[
  {
    "path": ".asf.yaml",
    "content": "notifications:\n    commits:      commits@mxnet.apache.org\n    issues:       issues@mxnet.apache.org\n    pullrequests: commits@mxnet.apache.org\n\ngithub:\n  features:\n    wiki: true\n    issues: true\n    projects: true\n\n  enabled_merge_buttons:\n    squash:  true\n    merge:   false\n    rebase:  true\n"
  },
  {
    "path": ".clang-format",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n---\nLanguage: Cpp\nBasedOnStyle: Google\nColumnLimit: 100\nAlignConsecutiveAssignments: true\nAlignConsecutiveDeclarations: false\nAlignConsecutiveMacros: true\nDerivePointerAlignment: false\nSortIncludes: true\nMaxEmptyLinesToKeep: 1\nPointerAlignment: Left\nAllowAllParametersOfDeclarationOnNextLine: false\nAllowShortBlocksOnASingleLine: false\nAllowShortCaseLabelsOnASingleLine: false\nAllowShortFunctionsOnASingleLine: Empty\nAllowShortIfStatementsOnASingleLine: false\nAllowShortLoopsOnASingleLine: false\nAlwaysBreakAfterReturnType: None\nAlwaysBreakBeforeMultilineStrings: true\nAlwaysBreakTemplateDeclarations: true\nBinPackArguments: false\nBinPackParameters: false\nSortIncludes: false\nBreakBeforeTernaryOperators: false\n---\nLanguage: JavaScript\nDisableFormat: true\n"
  },
  {
    "path": ".clang-tidy",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# The checks defined here will be run and will display by default as warnings.\nChecks: >\n    -*, cppcoreguidelines-* clang-analyzer-*, modernize-*,\n    performance-faster-string-find, performance-for-range-copy,\n    performance-implicit-conversion-in-loop, performance-inefficient-algorithm,\n    performance-inefficient-string-concatenation, performance-trivially-destructible,\n    performance-inefficient-vector-operation, performance-move-const-arg,\n    performance-move-constructor-init, performance-noexcept-move-constructor,\n    performance-no-automatic-move, performance-unnecessary-copy-initialization,\n    performance-type-promotion-in-math-fn\n\n# performance checks not enabled due to segmentation fault in clang-tidy v8+:\n# performance-unnecessary-value-param\n\n# In order to trigger an error, you must have a rule defined both in checks and in this section.\nWarningsAsErrors: >\n    cppcoreguidelines-no-malloc, modernize-deprecated-headers,\n    modernize-loop-convert, modernize-make-shared, modernize-pass-by-value, modernize-make-unique,\n    modernize-raw-string-literal, modernize-redundant-void-arg, modernize-replace-auto-ptr,\n    modernize-replace-random-shuffle, modernize-return-braced-init-list, modernize-shrink-to-fit,\n    modernize-unary-static-assert, modernize-use-bool-literals, modernize-use-default-member-init,\n    modernize-use-emplace, modernize-use-equals-default, modernize-use-equals-delete,\n    modernize-use-noexcept, modernize-use-nullptr, modernize-use-override,\n    modernize-use-transparent-functors, modernize-use-using,\n    performance-faster-string-find, performance-implicit-conversion-in-loop,\n    performance-inefficient-algorithm, performance-inefficient-string-concatenation,\n    performance-trivially-destructible, performance-inefficient-vector-operation,\n    performance-move-const-arg, performance-move-constructor-init,\n    performance-noexcept-move-constructor, performance-no-automatic-move,\n    performance-unnecessary-copy-initialization, performance-type-promotion-in-math-fn\n\n# modernize checks not enforced:\n# modernize-use-auto\n# modernize-avoid-bind\n\n# performance checks not enforced due to segmentation fault\n# performance-for-range-copy\n\n# Todo: define a better regex match that includes most project headers, but excludes third party\n# code.\nHeaderFilterRegex: '^src/.*'\n"
  },
  {
    "path": ".cmakelintrc",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# build and install are separated so changes to build don't invalidate\n# the whole docker cache for the image\n\n# --filter= options: https://pypi.org/project/cmakelint/\n# \"-\" disable option\n# \"+\" enable option\nfilter=-convention/filename,-linelength,-package/consistency,-readability/logic,-readability/mixedcase,-readability/wonkycase,-syntax,-whitespace/eol,+whitespace/extra,-whitespace/indent,-whitespace/mismatch,-whitespace/newline,-whitespace/tabs\n"
  },
  {
    "path": ".codecov.yml",
    "content": "# Codecov.io configuration file\n# See https://docs.codecov.io/docs/codecovyml-reference\ncodecov:\n  notify:\n    require_ci_to_pass: yes\n\ncoverage:\n  status:\n    project: off\n    patch: on\n  precision: 2\n  round: down\n  range: \"70...100\"\n\nparsers:\n  gcov:\n    branch_detection:\n      conditional: yes\n      loop: yes\n      method: no\n      macro: no\n\nignore:\n - \"tests/**/*\"\n\n# Disable comments for now to gather data in the background\ncomment: false\n#  layout: \"header, diff\"\n#  behavior: default\n#  require_changes: no\n"
  },
  {
    "path": ".git-blame-ignore-revs",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Clang-formatter initial commit - /src directory is formatted\ne359bcd65e453d4bc86d3d8e5b1dee3916a2e426\n\n# Clang-formatter initial commit - OneDNN files\n718a860f3aa8f24acca2aec867a3b31bc60a6e79\n"
  },
  {
    "path": ".gitattributes",
    "content": ".gitattributes export-ignore\nR-package/* export-ignore\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/bug_report.md",
    "content": "---\nname: Bug report\nabout: Create a report to help us improve\ntitle: ''\nlabels: 'Bug, needs triage'\nassignees: ''\n\n---\n## Description\n(A clear and concise description of what the bug is.)\n\n### Error Message\n(Paste the complete error message. Please also include stack trace by setting environment variable `DMLC_LOG_STACK_TRACE_DEPTH=100` before running your script.)\n\n## To Reproduce\n(If you developed your own code, please provide a short script that reproduces the error. For existing examples, please provide link.)\n\n### Steps to reproduce\n(Paste the commands you ran that produced the error.)\n\n1.\n2.\n\n## What have you tried to solve it?\n\n1.\n2.\n\n## Environment\n\n***We recommend using our script for collecting the diagnostic information with the following command***\n`curl --retry 10 -s https://raw.githubusercontent.com/apache/incubator-mxnet/master/tools/diagnose.py | python3`\n\n<details>\n<summary>Environment Information</summary>\n\n```\n# Paste the diagnose.py command output here\n```\n\n</details>\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/config.yml",
    "content": "blank_issues_enabled: false\ncontact_links:\n  - name: GitHub Discussions\n    url: https://github.com/apache/mxnet/discussions\n    about: Use GitHub Discussions to ask and answer questions, exchange ideas, and share learning.\n  - name: Discourse Forum\n    url: https://discuss.mxnet.io/\n    about: Discuss forum for usage questions.\n  - name: Stack Overflow\n    url: https://stackoverflow.com/questions/tagged/mxnet\n    about: Ask and answer usage questions on Stack Overflow\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/feature_request.md",
    "content": "---\nname: Feature request\nabout: Suggest an idea for this project\ntitle: ''\nlabels: 'Feature request'\nassignees: ''\n\n---\n\n## Description\n(A clear and concise description of what the feature is.)\n- If the proposal is about a new model, provide description of what the model is.\n- If the proposal is about an API, provide mock examples if possible.\n\n## References\n- list reference and related literature\n- list known implementations\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/flaky_test.md",
    "content": "---\nname: Flaky test\nabout: Report a flaky test\ntitle: ''\nlabels: 'Flaky'\nassignees: ''\n\n---\n## Description\n(The location and name of the flaky test.)\n\n## Occurrences\n(Links to the known occurrences.)\n\n## What have you tried to solve it?\n\n1.\n2.\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/rfc.md",
    "content": "---\nname: Request for comment (RFC)\nabout: RFC process requests for review on the design of a new feature or bug fix that involves more efforts. This thread is automatically mirrored to the dev@mxnet.apache.org mailing list.\ntitle: '[RFC] '\nlabels: 'RFC'\nassignees: ''\n\n---\n\n## Problem statement\n(A clear and concise description of what this contribution is trying to solve.)\n\n## Proposed solutions\n(Description of the approach this contribution takes to solve the problem.)\n\n## References\n- list reference and related literature\n- list known implementations\n"
  },
  {
    "path": ".github/PULL_REQUEST_TEMPLATE.md",
    "content": "## Description ##\n(Brief description on what this PR is about)\n\n## Checklist ##\n### Essentials ###\n- [ ] PR's title starts with a category (e.g. [BUGFIX], [MODEL], [TUTORIAL], [FEATURE], [DOC], etc)\n- [ ] Changes are complete (i.e. I finished coding on this PR)\n- [ ] All changes have test coverage\n- [ ] Code is well-documented\n\n### Changes ###\n- [ ] Feature1, tests, (and when applicable, API doc)\n- [ ] Feature2, tests, (and when applicable, API doc)\n\n## Comments ##\n- If this change is a backward incompatible change, why must this change be made.\n- Interesting edge cases to note here\n"
  },
  {
    "path": ".github/workflows/greetings.yml",
    "content": "name: Greetings\n\non: [pull_request, issues]\n\njobs:\n  greeting:\n    runs-on: ubuntu-latest\n    steps:\n    - uses: actions/first-interaction@v1\n      env:\n        GITHUB_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}\n        GITHUB_PR_RUN_ID: ${{ github.run_id }}\n        GITHUB_PR_BASE_REF: ${{ github.event.pull_request.base.ref }} \n      with:\n        repo-token: ${{ secrets.GITHUB_TOKEN }}\n        issue-message: |\n          Welcome to Apache MXNet (incubating)! We are on a mission to democratize AI, and we are glad that you are contributing to it by opening this issue.\n          Please make sure to include all the relevant context, and one of the @apache/mxnet-committers will be here shortly.\n          If you are interested in contributing to our project, let us know! Also, be sure to check out our guide on [contributing to MXNet](https://mxnet.apache.org/community/contribute) and our [development guides wiki](https://cwiki.apache.org/confluence/display/MXNET/Developments).\n        pr-message: |\n          Welcome to Apache MXNet (incubating)! We are on a mission to democratize AI, and we are glad that you are contributing to it by opening this pull request.\n          Please make sure that the changes are covered by tests. One of the @apache/mxnet-committers will be here shortly.\n          If you run into any issue with the CI and tests, we recommend that you first check out our guide on [developer guides wiki](https://cwiki.apache.org/confluence/display/MXNET/Developments).\n          Let our @apache/mxnet-committers know if you need any help!\n"
  },
  {
    "path": ".github/workflows/license_check.yml",
    "content": "name: license check\n\non: [push, pull_request]\n\ndefaults:\n  run:\n    shell: bash\n\njobs:\n  licensecheck:\n    runs-on: ubuntu-latest\n    strategy:\n      fail-fast: false\n    steps:\n      - name: Checkout repository\n        uses: actions/checkout@v2\n\n      - name: Update Submodules\n        run: |\n          git submodule update --init --recursive\n\n      - name: Check License Header\n        uses: apache/skywalking-eyes@main\n        env:\n          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n"
  },
  {
    "path": ".github/workflows/link_check.yml",
    "content": "name: link check\n\non: [push, pull_request]\n\ndefaults:\n  run:\n    shell: bash\n\njobs:\n  linkcheck:\n    runs-on: ubuntu-20.04\n    strategy:\n      fail-fast: false\n    steps:\n      - name: Checkout repository\n        uses: actions/checkout@v2\n\n      - name: Compilation cache\n        uses: actions/cache@v2\n        with:\n          path: ~/.ccache\n          # We include the commit sha in the cache key, as new cache entries are\n          # only created if there is no existing entry for the key yet.\n          key: ${{ runner.os }}-ccache-${{ github.sha }}\n          # Restore any ccache cache entry, if none for\n          # ${{ runner.os }}-ccache-${{ github.sha }} exists\n          restore-keys: |\n            ${{ runner.os }}-ccache\n\n      - name: Setup python\n        uses: actions/setup-python@v2\n        with:\n          python-version: '3.8'\n          architecture: x64\n\n      - name: Install Dependencies\n        run: |\n          sudo apt-get update\n          sudo apt-get install -y libopenblas-dev ninja-build ccache python3-sphinx \\\n              pandoc gcc-7 g++-7 libopencv-dev protobuf-compiler libprotobuf-dev\n          ccache -M 500M  # Limit the ccache size; Github's overall cache limit is 5GB\n          python -m pip install pandoc-attributes==0.1.7\n          python -m pip install -r docs/python_docs/requirements\n          python -m pip install docs/python_docs/themes/mx-theme\n        shell: bash\n\n      - name: Build project\n        env:\n          CC: gcc-7\n          CXX: g++-7\n        run: |\n          git submodule update --init --recursive\n          mkdir build; cd build\n          CXXFLAGS=\"-Wno-error=strict-overflow\" cmake \\\n              -DCMAKE_BUILD_TYPE=\"RelWithDebInfo\" \\\n              -DUSE_ONEDNN=OFF \\\n              -DUSE_CUDA=OFF \\\n              -G Ninja ..\n          ninja\n          cd ..\n        shell: bash\n\n      - name: Setup Python\n        run: |\n          python -m pip install --user -e python\n\n      - name: Link Check\n        env:\n          MAX_RETRY: 3\n        run: |\n          for run in {1..$MAX_RETRY}\n          do\n            cd docs/python_docs/python\n            make clean\n            timeout 10m make linkcheck EVAL=0\n            if [[ $? -eq 0 ]]\n            then\n              break\n            else\n              if [[ run -eq $MAX_RETRY ]]\n              then\n                exit 1\n              fi\n            fi\n          done\n"
  },
  {
    "path": ".github/workflows/os_x_mklbuild.yml",
    "content": "name: mkl continuous build\n\non: [push, pull_request]\n\njobs:\n  macosx-x86_64:\n    runs-on: macos-10.15\n    steps:\n      - name: Checkout repository\n        uses: actions/checkout@v2\n\n      - name: Compilation cache\n        uses: actions/cache@v2\n        with:\n          path: ~/.ccache\n          # We include the commit sha in the cache key, as new cache entries are\n          # only created if there is no existing entry for the key yet.\n          key: ${{ runner.os }}-ccache-${{ github.sha }}\n          # Restore any ccache cache entry, if none for\n          # ${{ runner.os }}-ccache-${{ github.sha }} exists\n          restore-keys: |\n            ${{ runner.os }}-ccache\n\n      - name: Setup python\n        uses: actions/setup-python@v2\n        with:\n          python-version: '3.8'\n          architecture: x64\n\n      - name: Install Dependencies\n        run: |\n          brew install nasm automake ninja libtool cmake pkgconfig protobuf hdf5 zlib ccache\n          ccache -M 500M  # Limit the ccache size; Github's overall cache limit is 5GB\n          python -m pip install -r ci/docker/install/requirements\n        shell: bash\n\n      - name: Build project\n        run: |\n          ./tools/staticbuild/build.sh cpu mkl\n\n      - name: Setup Python\n        run: |\n          python -m pip install --user -e python\n\n      - name: Test project\n        run: |\n          python -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'not test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'\n          MXNET_ENGINE_TYPE=NaiveEngine python -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'\n          python -m pytest --durations=50 --verbose tests/python/unittest/ -k 'not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'serial'\n          python -m pytest -n 4 --durations=50 --verbose tests/python/dnnl -k 'not (test_bf16_operator or test_amp or test_amp_subgraph)'\n"
  },
  {
    "path": ".github/workflows/os_x_staticbuild.yml",
    "content": "name: continuous build\n\non: [push, pull_request]\n\njobs:\n  macosx-x86_64:\n    runs-on: macos-latest\n    steps:\n      - name: Checkout repository\n        uses: actions/checkout@v2\n\n      - name: Compilation cache\n        uses: actions/cache@v2\n        with:\n          path: ~/.ccache\n          # We include the commit sha in the cache key, as new cache entries are\n          # only created if there is no existing entry for the key yet.\n          key: ${{ runner.os }}-ccache-${{ github.sha }}\n          # Restore any ccache cache entry, if none for\n          # ${{ runner.os }}-ccache-${{ github.sha }} exists\n          restore-keys: |\n            ${{ runner.os }}-ccache\n\n      - name: Setup python\n        uses: actions/setup-python@v2\n        with:\n          python-version: '3.8'\n          architecture: x64\n\n      - name: Install Dependencies\n        run: |\n          brew install nasm automake ninja libtool cmake pkgconfig protobuf hdf5 zlib ccache\n          ccache -M 500M  # Limit the ccache size; Github's overall cache limit is 5GB\n          python -m pip install -r ci/docker/install/requirements\n        shell: bash\n\n      - name: Build project\n        run: |\n          CMAKE_STATICBUILD=1 ./tools/staticbuild/build.sh cpu\n\n      - name: Setup Python\n        run: |\n          python -m pip install --user -e python\n\n      - name: Build with Cython\n        run: |\n          cd python\n          python setup.py build_ext --inplace --with-cython\n\n      - name: Test project\n        env:\n          MXNET_ENABLE_CYTHON: 1\n        run: |\n          python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'not test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'\n          MXNET_ENGINE_TYPE=NaiveEngine python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'\n          python3 -m pytest --durations=50 --verbose tests/python/unittest/ -k 'not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'serial'\n\n      - name: Test Array API\n        env:\n          MXNET_ENABLE_CYTHON: 1\n        run: |\n          cd ..\n          git clone https://github.com/data-apis/array-api-tests.git\n          cd array-api-tests\n          git checkout c1dba80a196a03f880d2e0a998a272fb3867b720\n          export ARRAY_API_TESTS_MODULE=mxnet.numpy pytest\n          export DMLC_LOG_STACK_TRACE_DEPTH=100\n          python3 -m pytest --reruns 3 --durations=50 --verbose array_api_tests/test_creation_functions.py\n          python3 -m pytest --reruns 3 --durations=50 --verbose array_api_tests/test_indexing.py\n          python3 -m pytest --reruns 3 --durations=50 --verbose array_api_tests/test_constants.py\n          python3 -m pytest --reruns 3 --durations=50 --verbose array_api_tests/test_elementwise_functions.py\n          python3 -m pytest --reruns 3 --durations=50 --verbose array_api_tests/test_broadcasting.py\n          python3 -m pytest --reruns 3 --durations=50 --verbose \\\n              array_api_tests/test_type_promotion.py::test_elementwise_function_two_arg_bool_type_promotion\n          python3 -m pytest --reruns 3 --durations=50 --verbose \\\n              array_api_tests/test_type_promotion.py::test_elementwise_function_two_arg_promoted_type_promotion\n          python3 -m pytest --reruns 3 --durations=50 --verbose \\\n              array_api_tests/test_type_promotion.py::test_elementwise_function_one_arg_bool\n          python3 -m pytest --reruns 3 --durations=50 --verbose \\\n              array_api_tests/test_type_promotion.py::test_elementwise_function_one_arg_type_promotion\n          python3 -m pytest --reruns 3 --durations=50 --verbose \\\n              array_api_tests/test_type_promotion.py::test_operator_one_arg_type_promotion\n          python3 -m pytest --reruns 3 --durations=50 --verbose \\\n              array_api_tests/test_type_promotion.py::test_operator_two_arg_bool_promotion\n          python3 -m pytest --reruns 3 --durations=50 --verbose \\\n              array_api_tests/test_type_promotion.py::test_operator_two_arg_promoted_promotion\n          python3 -m pytest --reruns 3 --durations=50 --verbose \\\n              array_api_tests/test_type_promotion.py::test_operator_inplace_two_arg_promoted_promotion\n"
  },
  {
    "path": ".gitignore",
    "content": "# Compiled Object files\n*.slo\n*.lo\n*.o\n*.obj\n\n# Precompiled Headers\n*.gch\n*.pch\n\n# Compiled Dynamic libraries\n*.so\n*.dylib\n*.dll\n\n# Fortran module files\n*.mod\n\n# Compiled Static libraries\n*.lai\n*.la\n*.a\n*.lib\n\n# Executables\n*.exe\n*.out\n*.app\n*~\n\n# doc\ndoc/html\ndoc/latex\ndoc/doc\ndocs/web-data\n.jekyll-cache\n*.lock\n\n#dmlc\nconfig.mk\nconfig.cmake\n\n*.pyc\n.Rhistory\n*log\nDebug\n*suo\ntracker\n\n# vim\n*.swp\n*.swo\n*.swn\n.vimrc\n.ycm_extra_conf.py\n.ycm_extra_conf.pyc\n\n# Emacs\n.#*\n.clang_complete\n.dir-locals.el\n__pycache__\n*.pkl\n*.params\n*.states\n*.json\n*.d\ncmake-build*\ndata\nmodel\nrecommonmark\n\n# R\n*.Rcheck\n*.rds\n*.Rproj\n.Rproj.user\nR-package/inst/*\n*.tar.gz\n*.tgz\nR-package/man/*.Rd\nR-package/R/mxnet_generated.R\n\n# data\n*.rec\n*.lst\n*.zip\n*ubyte\n*.bin\n*.txt\n!CMakeLists.txt\n\n# ipython notebook\n*_pb2.py\n*.ipynb_checkpoints*\ninput.txt*\n\n# Jetbrain\n.idea\n.gradle\n*.iml\n\n# ctags\ntags\n\n# cscope\ncscope.out\ncscope.files\n\n# Eclipse project config\n.project\n.cproject\n.classpath\n.settings\n.pydevproject\nCMakeFiles\ncmake_install.cmake\n\n# Visual Studio Code\n.vscode\n\n# Mac OS X\n.DS_Store\n\n# Windows\nwindows_package.7z\nwindows_package\n\n#Notebook Automated Test\n!tests/nightly/test_tutorial_config.txt\n!tests/nightly/TestNotebook\ntests/nightly/tmp_notebook\n\n# pip building tools\ntools/pip_package/build\ntools/pip_package/dist\ntools/pip_package/mxnet.egg-info\ntools/pip_package/mxnet\n\n# temporary path for building dependencies when building wheel\ndeps/\nstaticdeps/\ntmp/\nbuild/\nlib/\nbin/\nmodel/\n\n# VTune\n./r0*hs\n\n# generated function signature for IDE auto-complete\npython/mxnet/symbol/gen_*\npython/mxnet/ndarray/gen_*\npython/.eggs\n\n# tests if built insource\n*CTestTestfile.cmake\n*DartConfiguration.tcl\ntests/Makefile\ntests/mxnet_unit_tests\n\n# Code coverage related\n.coverage\n*.gcov\n*.gcno\ncoverage.xml\n\n# Local CMake build config\ncmake_options.yml\n\n# header file generated at compile time\ninclude/onednn/oneapi/dnnl/dnnl_version.h\ninclude/onednn/oneapi/dnnl/dnnl_config.h\n"
  },
  {
    "path": ".gitmodules",
    "content": "[submodule \"3rdparty/dmlc-core\"]\n\tpath = 3rdparty/dmlc-core\n\turl = https://github.com/dmlc/dmlc-core.git\n[submodule \"3rdparty/ps-lite\"]\n\tpath = 3rdparty/ps-lite\n\turl = https://github.com/dmlc/ps-lite\n[submodule \"3rdparty/dlpack\"]\n\tpath = 3rdparty/dlpack\n\turl = https://github.com/dmlc/dlpack\n[submodule \"3rdparty/googletest\"]\n\tpath = 3rdparty/googletest\n\turl = https://github.com/google/googletest.git\n[submodule \"3rdparty/tvm\"]\n\tpath = 3rdparty/tvm\n\turl = https://github.com/apache/incubator-tvm.git\n[submodule \"3rdparty/onnx-tensorrt\"]\n\tpath = 3rdparty/onnx-tensorrt\n\turl = https://github.com/onnx/onnx-tensorrt.git\n[submodule \"3rdparty/nvidia_cub\"]\n\tpath = 3rdparty/nvidia_cub\n\turl = https://github.com/NVlabs/cub.git\n[submodule \"3rdparty/libzip\"]\n\tpath = 3rdparty/libzip\n\turl = https://github.com/nih-at/libzip.git\n[submodule \"3rdparty/intgemm\"]\n\tpath = 3rdparty/intgemm\n\turl = https://github.com/kpu/intgemm\n[submodule \"3rdparty/onednn\"]\n\tpath = 3rdparty/onednn\n\turl = https://github.com/oneapi-src/oneDNN\n"
  },
  {
    "path": ".licenserc.yaml",
    "content": "header:\n  license:\n    spdx-id: Apache-2.0\n    copyright-owner: Apache Software Foundation\n\n  paths-ignore:\n    - 'licenses'\n    - 'LICENSE'\n    - 'NOTICE'\n    - '3rdparty'\n    - 'DISCLAIMER'\n    - 'KEYS'\n    - 'tools/dependencies/LICENSE.binary.dependencies'\n    - 'tools/lint/git-clang-format-13'\n    # files not distributed in source archive (listed in tools/source-exclude-artifacts.txt)\n    - 'docs'\n    - 'CODEOWNERS'\n    - '.gitignore'\n    - '.codecov.yml'\n    - '.gitattributes'\n    - '.github'\n    - '.gitmodules'\n    - '.licenserc.yaml'\n    - '.asf.yaml'\n    - 'CODEOWNERS'\n    - 'python/mxnet/_cy3/README.md'\n    - 'tools/dependencies/LICENSE.binary.dependencies'\n    # files not distributed in source archive (listed in tools/source-exclude-artifacts.txt)\n    - 'docs'\n    # files licensed under apache-2.0 license but do not include full license headers recognized by skywalking-eyes\n    - '**/*.ipynb'\n    - 'src/operator/deformable_convolution-inl.h'\n    - 'src/operator/deformable_convolution.cc'\n    - 'src/operator/deformable_convolution.cu'\n    - 'src/operator/contrib/deformable_psroi_pooling-inl.h'\n    - 'src/operator/contrib/deformable_psroi_pooling.cc'\n    - 'src/operator/contrib/deformable_psroi_pooling.cu'\n    - 'src/operator/contrib/multi_proposal-inl.h'\n    - 'src/operator/contrib/multi_proposal.cc'\n    - 'src/operator/contrib/multi_proposal.cu'\n    - 'src/operator/contrib/psroi_pooling.cc'\n    - 'src/operator/contrib/psroi_pooling.cu'\n    - 'src/operator/nn/dnnl/dnnl_base-inl.h'\n    # files licensed under boost license\n    - 'cmake/Modules/FindJeMalloc.cmake'\n    # files licensed under bsd 2-clause + caffe\n    - 'src/operator/nn/pool.cuh'\n    - 'src/operator/nn/pool.h'\n    - 'src/operator/nn/im2col.cuh'\n    - 'src/operator/nn/im2col.h'\n    - 'src/operator/contrib/nn/deformable_im2col.cuh'\n    - 'src/operator/contrib/nn/deformable_im2col.h'\n    - 'src/operator/contrib/nn/modulated_deformable_im2col.cuh'\n    - 'src/operator/contrib/nn/modulated_deformable_im2col.h'\n    # files licensed under bsd 3-clause\n    - 'cmake/upstream/FindBLAS.cmake'\n    - 'cmake/upstream/FindCUDAToolkit.cmake'\n    - 'cmake/upstream/select_compute_arch.cmake'\n    - 'python/mxnet/onnx/mx2onnx/_export_onnx.py'\n    - 'python/mxnet/onnx/mx2onnx/_op_translations/_op_translations_opset12.py'\n    - 'python/mxnet/onnx/mx2onnx/_op_translations/_op_translations_opset13.py'\n    - 'src/operator/contrib/erfinv-inl.h'\n    - 'src/operator/numpy/np_einsum_op-inl.h'\n    - 'src/operator/numpy/np_einsum_op.cc'\n    - 'src/operator/numpy/np_einsum_path_op-inl.h'\n    # files licensed under mit license\n    - 'src/operator/modulated_deformable_convolution-inl.h'\n    - 'src/operator/modulated_deformable_convolution.cc'\n    - 'src/operator/modulated_deformable_convolution.cu'\n    - 'src/operator/nn/layer_norm_cpu.h'\n    # symlinks\n    - 'include/dlpack' # symlink to 3rdparty/dlpack/include/dlpack\n    - 'include/dmlc' # symlink to 3rdparty/dmlc-core/include/dmlc\n    - 'include/mshadow' # symlink to 3rdparty/mshadow/mshadow\n    - 'include/onednn' # symlinks to 3rdparty/onednn\n    - 'include/nnvm' # symlinks to 3rdparty/tvm/nnvm/include/nnvm\n    # test/build data\n    - 'tests/python/dnnl/data/test_dnnl_test_dnnl_model_model1.json'\n\n\n  comment: on-failure\n"
  },
  {
    "path": ".mxnet_root",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n\n#   http://www.apache.org/licenses/LICENSE-2.0\n\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# This file marks the root directory of the Apache MXNet repository.\n"
  },
  {
    "path": "3rdparty/ctc_include/LICENSE",
    "content": "                                 Apache License\n                           Version 2.0, January 2004\n                        http://www.apache.org/licenses/\n\n   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n   1. Definitions.\n\n      \"License\" shall mean the terms and conditions for use, reproduction,\n      and distribution as defined by Sections 1 through 9 of this document.\n\n      \"Licensor\" shall mean the copyright owner or entity authorized by\n      the copyright owner that is granting the License.\n\n      \"Legal Entity\" shall mean the union of the acting entity and all\n      other entities that control, are controlled by, or are under common\n      control with that entity. For the purposes of this definition,\n      \"control\" means (i) the power, direct or indirect, to cause the\n      direction or management of such entity, whether by contract or\n      otherwise, or (ii) ownership of fifty percent (50%) or more of the\n      outstanding shares, or (iii) beneficial ownership of such entity.\n\n      \"You\" (or \"Your\") shall mean an individual or Legal Entity\n      exercising permissions granted by this License.\n\n      \"Source\" form shall mean the preferred form for making modifications,\n      including but not limited to software source code, documentation\n      source, and configuration files.\n\n      \"Object\" form shall mean any form resulting from mechanical\n      transformation or translation of a Source form, including but\n      not limited to compiled object code, generated documentation,\n      and conversions to other media types.\n\n      \"Work\" shall mean the work of authorship, whether in Source or\n      Object form, made available under the License, as indicated by a\n      copyright notice that is included in or attached to the work\n      (an example is provided in the Appendix below).\n\n      \"Derivative Works\" shall mean any work, whether in Source or Object\n      form, that is based on (or derived from) the Work and for which the\n      editorial revisions, annotations, elaborations, or other modifications\n      represent, as a whole, an original work of authorship. For the purposes\n      of this License, Derivative Works shall not include works that remain\n      separable from, or merely link (or bind by name) to the interfaces of,\n      the Work and Derivative Works thereof.\n\n      \"Contribution\" shall mean any work of authorship, including\n      the original version of the Work and any modifications or additions\n      to that Work or Derivative Works thereof, that is intentionally\n      submitted to Licensor for inclusion in the Work by the copyright owner\n      or by an individual or Legal Entity authorized to submit on behalf of\n      the copyright owner. For the purposes of this definition, \"submitted\"\n      means any form of electronic, verbal, or written communication sent\n      to the Licensor or its representatives, including but not limited to\n      communication on electronic mailing lists, source code control systems,\n      and issue tracking systems that are managed by, or on behalf of, the\n      Licensor for the purpose of discussing and improving the Work, but\n      excluding communication that is conspicuously marked or otherwise\n      designated in writing by the copyright owner as \"Not a Contribution.\"\n\n      \"Contributor\" shall mean Licensor and any individual or Legal Entity\n      on behalf of whom a Contribution has been received by Licensor and\n      subsequently incorporated within the Work.\n\n   2. Grant of Copyright License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      copyright license to reproduce, prepare Derivative Works of,\n      publicly display, publicly perform, sublicense, and distribute the\n      Work and such Derivative Works in Source or Object form.\n\n   3. Grant of Patent License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      (except as stated in this section) patent license to make, have made,\n      use, offer to sell, sell, import, and otherwise transfer the Work,\n      where such license applies only to those patent claims licensable\n      by such Contributor that are necessarily infringed by their\n      Contribution(s) alone or by combination of their Contribution(s)\n      with the Work to which such Contribution(s) was submitted. If You\n      institute patent litigation against any entity (including a\n      cross-claim or counterclaim in a lawsuit) alleging that the Work\n      or a Contribution incorporated within the Work constitutes direct\n      or contributory patent infringement, then any patent licenses\n      granted to You under this License for that Work shall terminate\n      as of the date such litigation is filed.\n\n   4. Redistribution. You may reproduce and distribute copies of the\n      Work or Derivative Works thereof in any medium, with or without\n      modifications, and in Source or Object form, provided that You\n      meet the following conditions:\n\n      (a) You must give any other recipients of the Work or\n          Derivative Works a copy of this License; and\n\n      (b) You must cause any modified files to carry prominent notices\n          stating that You changed the files; and\n\n      (c) You must retain, in the Source form of any Derivative Works\n          that You distribute, all copyright, patent, trademark, and\n          attribution notices from the Source form of the Work,\n          excluding those notices that do not pertain to any part of\n          the Derivative Works; and\n\n      (d) If the Work includes a \"NOTICE\" text file as part of its\n          distribution, then any Derivative Works that You distribute must\n          include a readable copy of the attribution notices contained\n          within such NOTICE file, excluding those notices that do not\n          pertain to any part of the Derivative Works, in at least one\n          of the following places: within a NOTICE text file distributed\n          as part of the Derivative Works; within the Source form or\n          documentation, if provided along with the Derivative Works; or,\n          within a display generated by the Derivative Works, if and\n          wherever such third-party notices normally appear. The contents\n          of the NOTICE file are for informational purposes only and\n          do not modify the License. You may add Your own attribution\n          notices within Derivative Works that You distribute, alongside\n          or as an addendum to the NOTICE text from the Work, provided\n          that such additional attribution notices cannot be construed\n          as modifying the License.\n\n      You may add Your own copyright statement to Your modifications and\n      may provide additional or different license terms and conditions\n      for use, reproduction, or distribution of Your modifications, or\n      for any such Derivative Works as a whole, provided Your use,\n      reproduction, and distribution of the Work otherwise complies with\n      the conditions stated in this License.\n\n   5. Submission of Contributions. Unless You explicitly state otherwise,\n      any Contribution intentionally submitted for inclusion in the Work\n      by You to the Licensor shall be under the terms and conditions of\n      this License, without any additional terms or conditions.\n      Notwithstanding the above, nothing herein shall supersede or modify\n      the terms of any separate license agreement you may have executed\n      with Licensor regarding such Contributions.\n\n   6. Trademarks. This License does not grant permission to use the trade\n      names, trademarks, service marks, or product names of the Licensor,\n      except as required for reasonable and customary use in describing the\n      origin of the Work and reproducing the content of the NOTICE file.\n\n   7. Disclaimer of Warranty. Unless required by applicable law or\n      agreed to in writing, Licensor provides the Work (and each\n      Contributor provides its Contributions) on an \"AS IS\" BASIS,\n      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n      implied, including, without limitation, any warranties or conditions\n      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n      PARTICULAR PURPOSE. You are solely responsible for determining the\n      appropriateness of using or redistributing the Work and assume any\n      risks associated with Your exercise of permissions under this License.\n\n   8. Limitation of Liability. In no event and under no legal theory,\n      whether in tort (including negligence), contract, or otherwise,\n      unless required by applicable law (such as deliberate and grossly\n      negligent acts) or agreed to in writing, shall any Contributor be\n      liable to You for damages, including any direct, indirect, special,\n      incidental, or consequential damages of any character arising as a\n      result of this License or out of the use or inability to use the\n      Work (including but not limited to damages for loss of goodwill,\n      work stoppage, computer failure or malfunction, or any and all\n      other commercial damages or losses), even if such Contributor\n      has been advised of the possibility of such damages.\n\n   9. Accepting Warranty or Additional Liability. While redistributing\n      the Work or Derivative Works thereof, You may choose to offer,\n      and charge a fee for, acceptance of support, warranty, indemnity,\n      or other liability obligations and/or rights consistent with this\n      License. However, in accepting such obligations, You may act only\n      on Your own behalf and on Your sole responsibility, not on behalf\n      of any other Contributor, and only if You agree to indemnify,\n      defend, and hold each Contributor harmless for any liability\n      incurred by, or claims asserted against, such Contributor by reason\n      of your accepting any such warranty or additional liability.\n\n   END OF TERMS AND CONDITIONS\n\n   APPENDIX: How to apply the Apache License to your work.\n\n      To apply the Apache License to your work, attach the following\n      boilerplate notice, with the fields enclosed by brackets \"[]\"\n      replaced with your own identifying information. (Don't include\n      the brackets!)  The text should be enclosed in the appropriate\n      comment syntax for the file format. We also recommend that a\n      file or class name and description of purpose be included on the\n      same \"printed page\" as the copyright notice for easier\n      identification within third-party archives.\n\n   Copyright [yyyy] [name of copyright owner]\n\n   Licensed under the Apache License, Version 2.0 (the \"License\");\n   you may not use this file except in compliance with the License.\n   You may obtain a copy of the License at\n\n       http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing, software\n   distributed under the License is distributed on an \"AS IS\" BASIS,\n   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n   See the License for the specific language governing permissions and\n   limitations under the License.\n\n   ----\n\n   Copyright 2015-2016, Baidu USA LLC."
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/LICENSE",
    "content": "/******************************************************************************\n* Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n* \n* Redistribution and use in source and binary forms, with or without\n* modification, are permitted provided that the following conditions are met:\n*     * Redistributions of source code must retain the above copyright\n*       notice, this list of conditions and the following disclaimer.\n*     * Redistributions in binary form must reproduce the above copyright\n*       notice, this list of conditions and the following disclaimer in the\n*       documentation and/or other materials provided with the distribution.\n*     * Neither the name of the NVIDIA CORPORATION nor the\n*       names of its contributors may be used to endorse or promote products\n*       derived from this software without specific prior written permission.\n* \n* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" \n* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE \n* ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n*\n******************************************************************************/\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/device/ctaloadbalance.cuh",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#pragma once\n\n#include \"ctasearch.cuh\"\n#include \"loadstore.cuh\"\n\nnamespace mgpu {\n\n////////////////////////////////////////////////////////////////////////////////\n// DeviceLoadBalancingSearch\n// Upper Bound search from A (needles) into B (haystack). The A values are\n// natural numbers from aBegin to aEnd. bFirst is the index of the B value at\n// bBegin in shared memory.\n\ntemplate<int VT, bool RangeCheck>\nMGPU_DEVICE void DeviceSerialLoadBalanceSearch(const int* b_shared, int aBegin,\n\tint aEnd, int bFirst, int bBegin, int bEnd, int* a_shared) {\n\n\tint bKey = b_shared[bBegin];\n\n\t#pragma unroll\n\tfor(int i = 0; i < VT; ++i) {\n\t\tbool p;\n\t\tif(RangeCheck)\n\t\t\tp = (aBegin < aEnd) && ((bBegin >= bEnd) || (aBegin < bKey));\n\t\telse\n\t\t\tp = aBegin < bKey;\n\n\t\tif(p)\n\t\t\t// Advance A (the needle).\n\t\t\ta_shared[aBegin++] = bFirst + bBegin;\n\t\telse\n\t\t\t// Advance B (the haystack).\n\t\t\tbKey = b_shared[++bBegin];\n\t}\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// CTALoadBalance\n// Computes upper_bound(counting_iterator<int>(first), b_global) - 1.\n\n// Unlike most other CTA* functions, CTALoadBalance loads from global memory.\n// This returns the loaded B elements at the beginning or end of shared memory\n// depending on the aFirst argument.\n\n// CTALoadBalance requires NT * VT + 2 slots of shared memory.\ntemplate<int NT, int VT, typename InputIt>\nMGPU_DEVICE int4 CTALoadBalance(int destCount, InputIt b_global,\n\tint sourceCount, int block, int tid, const int* mp_global,\n\tint* indices_shared, bool loadPrecedingB) {\n\n\tint4 range = ComputeMergeRange(destCount, sourceCount, block, 0, NT * VT,\n\t\tmp_global);\n\n\tint a0 = range.x;\n\tint a1 = range.y;\n\tint b0 = range.z;\n\tint b1 = range.w;\n\tif(!b0) loadPrecedingB = false;\n\n\t// Load one trailing term from B. If we're already at the end, fill the\n\t// end of the buffer with destCount.\n\tint aCount = a1 - a0;\n\tint bCount = b1 - b0;\n\tint extended = b1 < sourceCount;\n\tint loadCount = bCount + extended;\n\tint fillCount = NT * VT + 1 - loadCount - aCount;\n\n\tint* a_shared = indices_shared;\n\tint* b_shared = indices_shared + aCount + (int)loadPrecedingB;\n\n\t// Load the B values.\n//\tDeviceMemToMemLoop<NT>(bCount + extended + (int)loadPrecedingB,\n//\t\tb_global + b0 - (int)loadPrecedingB, tid,\n//\t\tb_shared - (int)loadPrecedingB);\n\n\tfor(int i = tid - (int)loadPrecedingB; i < bCount + extended; i += NT)\n\t\tb_shared[i] = b_global[b0 + i];\n\n\t// Fill the end of the array with destCount.\n\tfor(int i = tid + extended; i < fillCount; i += NT)\n\t\tb_shared[bCount + i] = destCount;\n\t__syncthreads();\n\n\t// Run a merge path to find the start of the serial merge for each thread.\n\tint diag = VT * tid;\n\tint mp = MergePath<MgpuBoundsUpper>(mgpu::counting_iterator<int>(a0),\n\t\taCount, b_shared, bCount, diag, mgpu::less<int>());\n\n\tint a0tid = a0 + mp;\n\tint b0tid = diag - mp;\n\n\t// Subtract 1 from b0 because we want to return upper_bound - 1.\n\tDeviceSerialLoadBalanceSearch<VT, false>(b_shared, a0tid, a1, b0 - 1,\n\t\tb0tid, bCount, a_shared - a0);\n\t__syncthreads();\n\n\tb0 -= (int)loadPrecedingB;\n\treturn make_int4(a0, a1, b0, b1);\n}\n\n\n} // namespace mgpu\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/device/ctamerge.cuh",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#pragma once\n\n#include \"ctasearch.cuh\"\n#include \"loadstore.cuh\"\n#include \"sortnetwork.cuh\"\n\nnamespace mgpu {\n\n////////////////////////////////////////////////////////////////////////////////\n// SerialMerge\n\ntemplate<int VT, bool RangeCheck, typename T, typename Comp>\nMGPU_DEVICE void SerialMerge(const T* keys_shared, int aBegin, int aEnd,\n\tint bBegin, int bEnd, T* results, int* indices, Comp comp) {\n\n\tT aKey = keys_shared[aBegin];\n\tT bKey = keys_shared[bBegin];\n\n\t#pragma unroll\n\tfor(int i = 0; i < VT; ++i) {\n\t\tbool p;\n\t\tif(RangeCheck)\n\t\t\tp = (bBegin >= bEnd) || ((aBegin < aEnd) && !comp(bKey, aKey));\n\t\telse\n\t\t\tp = !comp(bKey, aKey);\n\n\t\tresults[i] = p ? aKey : bKey;\n\t\tindices[i] = p ? aBegin : bBegin - !RangeCheck;\n\n\t\tif(p) aKey = keys_shared[++aBegin];\n\t\telse bKey = keys_shared[++bBegin];\n\t}\n\t__syncthreads();\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// FindMergeFrame and FindMergesortInterval help mergesort (both CTA and global\n// merge pass levels) locate lists within the single source array.\n\n// Returns (offset of a, offset of b, length of list).\nMGPU_HOST_DEVICE int3 FindMergesortFrame(int coop, int block, int nv) {\n\t// coop is the number of CTAs or threads cooperating to merge two lists into\n\t// one. We round block down to the first CTA's ID that is working on this\n\t// merge.\n\tint start = ~(coop - 1) & block;\n\tint size = nv * (coop>> 1);\n\treturn make_int3(nv * start, nv * start + size, size);\n}\n\n// Returns (a0, a1, b0, b1) into mergesort input lists between mp0 and mp1.\nMGPU_HOST_DEVICE int4 FindMergesortInterval(int3 frame, int coop, int block,\n\tint nv, int count, int mp0, int mp1) {\n\n\t// Locate diag from the start of the A sublist.\n\tint diag = nv * block - frame.x;\n\tint a0 = frame.x + mp0;\n\tint a1 = min(count, frame.x + mp1);\n\tint b0 = min(count, frame.y + diag - mp0);\n\tint b1 = min(count, frame.y + diag + nv - mp1);\n\n\t// The end partition of the last block for each merge operation is computed\n\t// and stored as the begin partition for the subsequent merge. i.e. it is\n\t// the same partition but in the wrong coordinate system, so its 0 when it\n\t// should be listSize. Correct that by checking if this is the last block\n\t// in this merge operation.\n\tif(coop - 1 == ((coop - 1) & block)) {\n\t\ta1 = min(count, frame.x + frame.z);\n\t\tb1 = min(count, frame.y + frame.z);\n\t}\n\treturn make_int4(a0, a1, b0, b1);\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// ComputeMergeRange\n\nMGPU_HOST_DEVICE int4 ComputeMergeRange(int aCount, int bCount, int block,\n\tint coop, int NV, const int* mp_global) {\n\n\t// Load the merge paths computed by the partitioning kernel.\n\tint mp0 = mp_global[block];\n\tint mp1 = mp_global[block + 1];\n\tint gid = NV * block;\n\n\t// Compute the ranges of the sources in global memory.\n\tint4 range;\n\tif(coop) {\n\t\tint3 frame = FindMergesortFrame(coop, block, NV);\n\t\trange = FindMergesortInterval(frame, coop, block, NV, aCount, mp0,\n\t\t\tmp1);\n\t} else {\n\t\trange.x = mp0;\t\t\t\t\t\t\t\t\t\t\t// a0\n\t\trange.y = mp1;\t\t\t\t\t\t\t\t\t\t\t// a1\n\t\trange.z = gid - range.x;\t\t\t\t\t\t\t\t// b0\n\t\trange.w = min(aCount + bCount, gid + NV) - range.y;\t\t// b1\n\t}\n\treturn range;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// CTA mergesort support\n\ntemplate<int NT, int VT, typename T, typename Comp>\nMGPU_DEVICE void CTABlocksortPass(T* keys_shared, int tid, int count,\n\tint coop, T* keys, int* indices, Comp comp) {\n\n\tint list = ~(coop - 1) & tid;\n\tint diag = min(count, VT * ((coop - 1) & tid));\n\tint start = VT * list;\n\tint a0 = min(count, start);\n\tint b0 = min(count, start + VT * (coop / 2));\n\tint b1 = min(count, start + VT * coop);\n\n\tint p = MergePath<MgpuBoundsLower>(keys_shared + a0, b0 - a0,\n\t\tkeys_shared + b0, b1 - b0, diag, comp);\n\n\tSerialMerge<VT, true>(keys_shared, a0 + p, b0, b0 + diag - p, b1, keys,\n\t\tindices, comp);\n}\n\ntemplate<int NT, int VT, bool HasValues, typename KeyType, typename ValType,\n\ttypename Comp>\nMGPU_DEVICE void CTABlocksortLoop(ValType threadValues[VT],\n\tKeyType* keys_shared, ValType* values_shared, int tid, int count,\n\tComp comp) {\n\n\t#pragma unroll\n\tfor(int coop = 2; coop <= NT; coop *= 2) {\n\t\tint indices[VT];\n\t\tKeyType keys[VT];\n\t\tCTABlocksortPass<NT, VT>(keys_shared, tid, count, coop, keys,\n\t\t\tindices, comp);\n\n\t\tif(HasValues) {\n\t\t\t// Exchange the values through shared memory.\n\t\t\tDeviceThreadToShared<VT>(threadValues, tid, values_shared);\n\t\t\tDeviceGather<NT, VT>(NT * VT, values_shared, indices, tid,\n\t\t\t\tthreadValues);\n\t\t}\n\n\t\t// Store results in shared memory in sorted order.\n\t\tDeviceThreadToShared<VT>(keys, tid, keys_shared);\n\t}\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// CTAMergesort\n// Caller provides the keys in shared memory. This functions sorts the first\n// count elements.\n\ntemplate<int NT, int VT, bool Stable, bool HasValues, typename KeyType,\n\ttypename ValType, typename Comp>\nMGPU_DEVICE void CTAMergesort(KeyType threadKeys[VT], ValType threadValues[VT],\n\tKeyType* keys_shared, ValType* values_shared, int count, int tid,\n\tComp comp) {\n\n\t// Stable sort the keys in the thread.\n\tif(VT * tid < count) {\n\t\tif(Stable)\n\t\t\tOddEvenTransposeSort<VT>(threadKeys, threadValues, comp);\n\t\telse\n\t\t\tOddEvenMergesort<VT>(threadKeys, threadValues, comp);\n\t}\n\n\t// Store the locally sorted keys into shared memory.\n\tDeviceThreadToShared<VT>(threadKeys, tid, keys_shared);\n\n\t// Recursively merge lists until the entire CTA is sorted.\n\tCTABlocksortLoop<NT, VT, HasValues>(threadValues, keys_shared,\n\t\tvalues_shared, tid, count, comp);\n}\n\ntemplate<int NT, int VT, bool Stable, typename KeyType, typename Comp>\nMGPU_DEVICE void CTAMergesortKeys(KeyType threadKeys[VT],\n\tKeyType* keys_shared, int count, int tid, Comp comp) {\n\n\tint valuesTemp[VT];\n\tCTAMergesort<NT, VT, Stable, false>(threadKeys, valuesTemp, keys_shared,\n\t\t(int*)keys_shared, count, tid, comp);\n}\n\ntemplate<int NT, int VT, bool Stable, typename KeyType, typename ValType,\n\ttypename Comp>\nMGPU_DEVICE void CTAMergesortPairs(KeyType threadKeys[VT],\n\tValType threadValues[VT], KeyType* keys_shared, ValType* values_shared,\n\tint count, int tid, Comp comp) {\n\n\tCTAMergesort<NT, VT, Stable, true>(threadKeys, threadValues, keys_shared,\n\t\tvalues_shared, count, tid, comp);\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// DeviceMergeKeysIndices\n\ntemplate<int NT, int VT, bool LoadExtended, typename It1, typename It2,\n\ttypename T, typename Comp>\nMGPU_DEVICE void DeviceMergeKeysIndices(It1 a_global, int aCount, It2 b_global,\n\tint bCount, int4 range, int tid, T* keys_shared, T* results, int* indices,\n\tComp comp) {\n\n\tint a0 = range.x;\n\tint a1 = range.y;\n\tint b0 = range.z;\n\tint b1 = range.w;\n\n\tif(LoadExtended) {\n\t\tbool extended = (a1 < aCount) && (b1 < bCount);\n\t\taCount = a1 - a0;\n\t\tbCount = b1 - b0;\n\t\tint aCount2 = aCount + (int)extended;\n\t\tint bCount2 = bCount + (int)extended;\n\n\t\t// Load one element past the end of each input to avoid having to use\n\t\t// range checking in the merge loop.\n\t\tDeviceLoad2ToShared<NT, VT, VT + 1>(a_global + a0, aCount2,\n\t\t\tb_global + b0, bCount2, tid, keys_shared);\n\n\t\t// Run a Merge Path search for each thread's starting point.\n\t\tint diag = VT * tid;\n\t\tint mp = MergePath<MgpuBoundsLower>(keys_shared, aCount,\n\t\t\tkeys_shared + aCount2, bCount, diag, comp);\n\n\t\t// Compute the ranges of the sources in shared memory.\n\t\tint a0tid = mp;\n\t\tint b0tid = aCount2 + diag - mp;\n\t\tif(extended) {\n\t\t\tSerialMerge<VT, false>(keys_shared, a0tid, 0, b0tid, 0, results,\n\t\t\t\tindices, comp);\n\t\t} else {\n\t\t\tint a1tid = aCount;\n\t\t\tint b1tid = aCount2 + bCount;\n\t\t\tSerialMerge<VT, true>(keys_shared, a0tid, a1tid, b0tid, b1tid,\n\t\t\t\tresults, indices, comp);\n\t\t}\n\t} else {\n\t\t// Use the input intervals from the ranges between the merge path\n\t\t// intersections.\n\t\taCount = a1 - a0;\n\t\tbCount = b1 - b0;\n\n\t\t// Load the data into shared memory.\n\t\tDeviceLoad2ToShared<NT, VT, VT>(a_global + a0, aCount, b_global + b0,\n\t\t\tbCount, tid, keys_shared);\n\n\t\t// Run a merge path to find the start of the serial merge for each\n\t\t// thread.\n\t\tint diag = VT * tid;\n\t\tint mp = MergePath<MgpuBoundsLower>(keys_shared, aCount,\n\t\t\tkeys_shared + aCount, bCount, diag, comp);\n\n\t\t// Compute the ranges of the sources in shared memory.\n\t\tint a0tid = mp;\n\t\tint a1tid = aCount;\n\t\tint b0tid = aCount + diag - mp;\n\t\tint b1tid = aCount + bCount;\n\n\t\t// Serial merge into register.\n\t\tSerialMerge<VT, true>(keys_shared, a0tid, a1tid, b0tid, b1tid, results,\n\t\t\tindices, comp);\n\t}\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// DeviceMerge\n// Merge pairs from global memory into global memory. Useful factorization to\n// enable calling from merge, mergesort, and locality sort.\n\ntemplate<int NT, int VT, bool HasValues, bool LoadExtended, typename KeysIt1,\n\ttypename KeysIt2, typename KeysIt3, typename ValsIt1, typename ValsIt2,\n\ttypename KeyType, typename ValsIt3, typename Comp>\nMGPU_DEVICE void DeviceMerge(KeysIt1 aKeys_global, ValsIt1 aVals_global,\n\tint aCount, KeysIt2 bKeys_global, ValsIt2 bVals_global, int bCount,\n\tint tid, int block, int4 range, KeyType* keys_shared, int* indices_shared,\n\tKeysIt3 keys_global, ValsIt3 vals_global, Comp comp) {\n\n\tKeyType results[VT];\n\tint indices[VT];\n\tDeviceMergeKeysIndices<NT, VT, LoadExtended>(aKeys_global, aCount,\n\t\tbKeys_global, bCount, range, tid, keys_shared, results, indices, comp);\n\n\t// Store merge results back to shared memory.\n\tDeviceThreadToShared<VT>(results, tid, keys_shared);\n\n\t// Store merged keys to global memory.\n\taCount = range.y - range.x;\n\tbCount = range.w - range.z;\n\tDeviceSharedToGlobal<NT, VT>(aCount + bCount, keys_shared, tid,\n\t\tkeys_global + NT * VT * block);\n\n\t// Copy the values.\n\tif(HasValues) {\n\t\tDeviceThreadToShared<VT>(indices, tid, indices_shared);\n\n\t\tDeviceTransferMergeValuesShared<NT, VT>(aCount + bCount,\n\t\t\taVals_global + range.x, bVals_global + range.z, aCount,\n\t\t\tindices_shared, tid, vals_global + NT * VT * block);\n\t}\n}\n\n} // namespace mgpu\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/device/ctascan.cuh",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#pragma once\n\n#include \"../mgpuenums.h\"\n#include \"deviceutil.cuh\"\n#include \"intrinsics.cuh\"\n\nnamespace mgpu {\n\n////////////////////////////////////////////////////////////////////////////////\n// CTAReduce\n\ntemplate<int NT, typename Op = mgpu::plus<int> >\nstruct CTAReduce {\n\ttypedef typename Op::first_argument_type T;\n\tenum { Size = NT, Capacity = NT };\n\tstruct Storage { T shared[Capacity]; };\n\n\tMGPU_DEVICE static T Reduce(int tid, T x, Storage& storage, Op op = Op()) {\n\t\tstorage.shared[tid] = x;\n\t\t__syncthreads();\n\n\t\t// Fold the data in half with each pass.\n\t\t#pragma unroll\n\t\tfor(int destCount = NT / 2; destCount >= 1; destCount /= 2) {\n\t\t\tif(tid < destCount) {\n\t\t\t\t// Read from the right half and store to the left half.\n\t\t\t\tx = op(x, storage.shared[destCount + tid]);\n\t\t\t\tstorage.shared[tid] = x;\n\t\t\t}\n\t\t\t__syncthreads();\n\t\t}\n\t\tT total = storage.shared[0];\n\t\t__syncthreads();\n\t\treturn total;\n\t}\n};\n\n#if __CUDA_ARCH__ >= 300\n\ntemplate<int NT>\nstruct CTAReduce<NT, mgpu::plus<int> > {\n\ttypedef mgpu::plus<int> Op;\n\ttypedef int T;\n\tenum { Size = NT, Capacity = WARP_SIZE };\n\tstruct Storage { int shared[Capacity]; };\n\n\tMGPU_DEVICE static int Reduce(int tid, int x, Storage& storage,\n\t\tOp op = Op()) {\n\n\t\tconst int NumSections = WARP_SIZE;\n\t\tconst int SecSize = NT / NumSections;\n\t\tint lane = (SecSize - 1) & tid;\n\t\tint sec = tid / SecSize;\n\n\t\t// In the first phase, threads cooperatively find the reduction within\n\t\t// their segment. The segments are SecSize threads (NT / WARP_SIZE)\n\t\t// wide.\n\t\t#pragma unroll\n\t\tfor(int offset = 1; offset < SecSize; offset *= 2)\n\t\t\tx = shfl_add(x, offset, SecSize);\n\n\t\t// The last thread in each segment stores the local reduction to shared\n\t\t// memory.\n\t\tif(SecSize - 1 == lane) storage.shared[sec] = x;\n\t\t__syncthreads();\n\n\t\t// Reduce the totals of each input segment. The spine is WARP_SIZE\n\t\t// threads wide.\n\t\tif(tid < NumSections) {\n\t\t\tx = storage.shared[tid];\n\t\t\t#pragma unroll\n\t\t\tfor(int offset = 1; offset < NumSections; offset *= 2)\n\t\t\t\tx = shfl_add(x, offset, NumSections);\n\t\t\tstorage.shared[tid] = x;\n\t\t}\n\t\t__syncthreads();\n\n\t\tint reduction = storage.shared[NumSections - 1];\n\t\t__syncthreads();\n\n\t\treturn reduction;\n\t}\n};\n\ntemplate<int NT>\nstruct CTAReduce<NT, mgpu::maximum<int> > {\n\ttypedef mgpu::maximum<int> Op;\n\tenum { Size = NT, Capacity = WARP_SIZE };\n\tstruct Storage { int shared[Capacity]; };\n\n\tMGPU_DEVICE static int Reduce(int tid, int x, Storage& storage,\n\t\tOp op = Op()) {\n\n\t\tconst int NumSections = WARP_SIZE;\n\t\tconst int SecSize = NT / NumSections;\n\t\tint lane = (SecSize - 1) & tid;\n\t\tint sec = tid / SecSize;\n\n\t\t#pragma unroll\n\t\tfor(int offset = 1; offset < SecSize; offset *= 2)\n\t\t\tx = shfl_max(x, offset, SecSize);\n\n\t\tif(SecSize - 1 == lane) storage.shared[sec] = x;\n\t\t__syncthreads();\n\n\t\tif(tid < NumSections) {\n\t\t\tx = storage.shared[tid];\n\t\t\t#pragma unroll\n\t\t\tfor(int offset = 1; offset < NumSections; offset *= 2)\n\t\t\t\tx = shfl_max(x, offset, NumSections);\n\t\t\tstorage.shared[tid] = x;\n\t\t}\n\t\t__syncthreads();\n\n\t\tint reduction = storage.shared[NumSections - 1];\n\t\t__syncthreads();\n\n\t\treturn reduction;\n\t}\n};\n\n#endif // __CUDA_ARCH__ >= 300\n\n////////////////////////////////////////////////////////////////////////////////\n// CTAScan\n\ntemplate<int NT, typename Op = mgpu::plus<int> >\nstruct CTAScan {\n\ttypedef typename Op::result_type T;\n\tenum { Size = NT, Capacity = 2 * NT + 1 };\n\tstruct Storage { T shared[Capacity]; };\n\n\tMGPU_DEVICE static T Scan(int tid, T x, Storage& storage, T* total,\n\t\tMgpuScanType type = MgpuScanTypeExc, T identity = (T)0, Op op = Op()) {\n\n\t\tstorage.shared[tid] = x;\n\t\tint first = 0;\n\t\t__syncthreads();\n\n\t\t#pragma unroll\n\t\tfor(int offset = 1; offset < NT; offset += offset) {\n\t\t\tif(tid >= offset)\n\t\t\t\tx = op(storage.shared[first + tid - offset], x);\n\t\t\tfirst = NT - first;\n\t\t\tstorage.shared[first + tid] = x;\n\t\t\t__syncthreads();\n\t\t}\n\t\t*total = storage.shared[first + NT - 1];\n\n\t\tif(MgpuScanTypeExc == type)\n\t\t\tx = tid ? storage.shared[first + tid - 1] : identity;\n\n\t\t__syncthreads();\n\t\treturn x;\n\t}\n\tMGPU_DEVICE static T Scan(int tid, T x, Storage& storage) {\n\t\tT total;\n\t\treturn Scan(tid, x, storage, &total, MgpuScanTypeExc, (T)0, Op());\n\t}\n};\n\n////////////////////////////////////////////////////////////////////////////////\n// Special partial specialization for CTAScan<NT, ScanOpAdd> on Kepler.\n// This uses the shfl intrinsic to reduce scan latency.\n\n#if __CUDA_ARCH__ >= 300\n\ntemplate<int NT>\nstruct CTAScan<NT, mgpu::plus<int> > {\n\ttypedef mgpu::plus<int> Op;\n\tenum { Size = NT, NumSegments = WARP_SIZE, SegSize = NT / NumSegments };\n\tenum { Capacity = NumSegments + 1 };\n\tstruct Storage { int shared[Capacity + 1]; };\n\n\tMGPU_DEVICE static int Scan(int tid, int x, Storage& storage, int* total,\n\t\tMgpuScanType type = MgpuScanTypeExc, int identity = 0, Op op = Op()) {\n\n\t\t// Define WARP_SIZE segments that are NT / WARP_SIZE large.\n\t\t// Each warp makes log(SegSize) shfl_add calls.\n\t\t// The spine makes log(WARP_SIZE) shfl_add calls.\n\t\tint lane = (SegSize - 1) & tid;\n\t\tint segment = tid / SegSize;\n\n\t\t// Scan each segment using shfl_add.\n\t\tint scan = x;\n\t\t#pragma unroll\n\t\tfor(int offset = 1; offset < SegSize; offset *= 2)\n\t\t\tscan = shfl_add(scan, offset, SegSize);\n\n\t\t// Store the reduction (last element) of each segment into storage.\n\t\tif(SegSize - 1 == lane) storage.shared[segment] = scan;\n\t\t__syncthreads();\n\n\t\t// Warp 0 does a full shfl warp scan on the partials. The total is\n\t\t// stored to shared[NumSegments]. (NumSegments = WARP_SIZE)\n\t\tif(tid < NumSegments) {\n\t\t\tint y = storage.shared[tid];\n\t\t\tint scan = y;\n\t\t\t#pragma unroll\n\t\t\tfor(int offset = 1; offset < NumSegments; offset *= 2)\n\t\t\t\tscan = shfl_add(scan, offset, NumSegments);\n\t\t\tstorage.shared[tid] = scan - y;\n\t\t\tif(NumSegments - 1 == tid) storage.shared[NumSegments] = scan;\n\t\t}\n\t\t__syncthreads();\n\n\t\t// Add the scanned partials back in and convert to exclusive scan.\n\t\tscan += storage.shared[segment];\n\t\tif(MgpuScanTypeExc == type) {\n\t\t\tscan -= x;\n\t\t\tif(identity && !tid) scan = identity;\n\t\t}\n\t\t*total = storage.shared[NumSegments];\n\t\t__syncthreads();\n\n\t\treturn scan;\n\t}\n\tMGPU_DEVICE static int Scan(int tid, int x, Storage& storage) {\n\t\tint total;\n\t\treturn Scan(tid, x, storage, &total, MgpuScanTypeExc, 0);\n\t}\n};\n\n#endif // __CUDA_ARCH__ >= 300\n\n////////////////////////////////////////////////////////////////////////////////\n// CTABinaryScan\n\ntemplate<int NT>\nMGPU_DEVICE int CTABinaryScan(int tid, bool x, int* shared, int* total) {\n\tconst int NumWarps = NT / WARP_SIZE;\n\tint warp = tid / WARP_SIZE;\n\tint lane = (WARP_SIZE - 1);\n\n\t// Store the bit totals for each warp.\n\tuint bits = __ballot(x);\n\tshared[warp] = popc(bits);\n\t__syncthreads();\n\n#if __CUDA_ARCH__ >= 300\n\tif(tid < NumWarps) {\n\t\tint x = shared[tid];\n\t\tint scan = x;\n\t\t#pragma unroll\n\t\tfor(int offset = 1; offset < NumWarps; offset *= 2)\n\t\t\tscan = shfl_add(scan, offset, NumWarps);\n\t\tshared[tid] = scan - x;\n\t}\n\t__syncthreads();\n\n#else\n\t// Thread 0 scans warp totals.\n\tif(!tid) {\n\t\tint scan = 0;\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < NumWarps; ++i) {\n\t\t\tint y = shared[i];\n\t\t\tshared[i] = scan;\n\t\t\tscan += y;\n\t\t}\n\t\tshared[NumWarps] = scan;\n\t}\n\t__syncthreads();\n\n#endif // __CUDA_ARCH__ >= 300\n\n\t// Add the warp scan back into the partials.\n\tint scan = shared[warp] + __popc(bfe(bits, 0, lane));\n\t*total = shared[NumWarps];\n\t__syncthreads();\n\treturn scan;\n}\n\n} // namespace mgpu\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/device/ctasearch.cuh",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#pragma once\n\n#include \"deviceutil.cuh\"\n#include \"../mgpudevice.cuh\"\n\nnamespace mgpu {\n\ntemplate<MgpuBounds Bounds, typename IntT, typename It, typename T,\n\ttypename Comp>\nMGPU_HOST_DEVICE void BinarySearchIt(It data, int& begin, int& end, T key,\n\tint shift, Comp comp) {\n\n\tIntT scale = (1<< shift) - 1;\n\tint mid = (int)((begin + scale * end)>> shift);\n\n\tT key2 = data[mid];\n\tbool pred = (MgpuBoundsUpper == Bounds) ?\n\t\t!comp(key, key2) :\n\t\tcomp(key2, key);\n\tif(pred) begin = mid + 1;\n\telse end = mid;\n}\n\ntemplate<MgpuBounds Bounds, typename IntT, typename T, typename It,\n\ttypename Comp>\nMGPU_HOST_DEVICE int BiasedBinarySearch(It data, int count, T key, int levels,\n\tComp comp) {\n\n\tint begin = 0;\n\tint end = count;\n\n\tif(levels >= 4 && begin < end)\n\t\tBinarySearchIt<Bounds, IntT>(data, begin, end, key, 9, comp);\n\tif(levels >= 3 && begin < end)\n\t\tBinarySearchIt<Bounds, IntT>(data, begin, end, key, 7, comp);\n\tif(levels >= 2 && begin < end)\n\t\tBinarySearchIt<Bounds, IntT>(data, begin, end, key, 5, comp);\n\tif(levels >= 1 && begin < end)\n\t\tBinarySearchIt<Bounds, IntT>(data, begin, end, key, 4, comp);\n\n\twhile(begin < end)\n\t\tBinarySearchIt<Bounds, int>(data, begin, end, key, 1, comp);\n\treturn begin;\n}\n\ntemplate<MgpuBounds Bounds, typename T, typename It, typename Comp>\nMGPU_HOST_DEVICE int BinarySearch(It data, int count, T key, Comp comp) {\n\tint begin = 0;\n\tint end = count;\n\twhile(begin < end)\n\t\tBinarySearchIt<Bounds, int>(data, begin, end, key, 1, comp);\n\treturn begin;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// MergePath search\n\ntemplate<MgpuBounds Bounds, typename It1, typename It2, typename Comp>\nMGPU_HOST_DEVICE int MergePath(It1 a, int aCount, It2 b, int bCount, int diag,\n\tComp comp) {\n\n\ttypedef typename std::iterator_traits<It1>::value_type T;\n\tint begin = max(0, diag - bCount);\n\tint end = min(diag, aCount);\n\n\twhile(begin < end) {\n\t\tint mid = (begin + end)>> 1;\n\t\tT aKey = a[mid];\n\t\tT bKey = b[diag - 1 - mid];\n\t\tbool pred = (MgpuBoundsUpper == Bounds) ?\n\t\t\tcomp(aKey, bKey) :\n\t\t\t!comp(bKey, aKey);\n\t\tif(pred) begin = mid + 1;\n\t\telse end = mid;\n\t}\n\treturn begin;\n}\n\n\n////////////////////////////////////////////////////////////////////////////////\n// SegmentedMergePath search\n\ntemplate<typename InputIt, typename Comp>\nMGPU_HOST_DEVICE int SegmentedMergePath(InputIt keys, int aOffset, int aCount,\n\tint bOffset, int bCount, int leftEnd, int rightStart, int diag, Comp comp) {\n\n\t// leftEnd and rightStart are defined from the origin, and diag is defined\n\t// from aOffset.\n\t// We only need to run a Merge Path search if the diagonal intersects the\n\t// segment that strides the left and right halves (i.e. is between leftEnd\n\t// and rightStart).\n\tif(aOffset + diag <= leftEnd) return diag;\n\tif(aOffset + diag >= rightStart) return aCount;\n\n\tbCount = min(bCount, rightStart - bOffset);\n\tint begin = max(max(leftEnd - aOffset, 0), diag - bCount);\n\tint end = min(diag, aCount);\n\n\twhile(begin < end) {\n\t\tint mid = (begin + end)>> 1;\n\t\tint ai = aOffset + mid;\n\t\tint bi = bOffset + diag - 1 - mid;\n\n\t\tbool pred = !comp(keys[bi], keys[ai]);\n\t\tif(pred) begin = mid + 1;\n\t\telse end = mid;\n\t}\n\treturn begin;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// BalancedPath search\n\ntemplate<bool Duplicates, typename IntT, typename InputIt1, typename InputIt2,\n\ttypename Comp>\nMGPU_HOST_DEVICE int2 BalancedPath(InputIt1 a, int aCount, InputIt2 b,\n\tint bCount, int diag, int levels, Comp comp) {\n\n\ttypedef typename std::iterator_traits<InputIt1>::value_type T;\n\n\tint p = MergePath<MgpuBoundsLower>(a, aCount, b, bCount, diag, comp);\n\tint aIndex = p;\n\tint bIndex = diag - p;\n\n\tbool star = false;\n\tif(bIndex < bCount) {\n\t\tif(Duplicates) {\n\t\t\tT x = b[bIndex];\n\n\t\t\t// Search for the beginning of the duplicate run in both A and B.\n\t\t\t// Because\n\t\t\tint aStart = BiasedBinarySearch<MgpuBoundsLower, IntT>(a, aIndex, x,\n\t\t\t\tlevels, comp);\n\t\t\tint bStart = BiasedBinarySearch<MgpuBoundsLower, IntT>(b, bIndex, x,\n\t\t\t\tlevels, comp);\n\n\t\t\t// The distance between the merge path and the lower_bound is the\n\t\t\t// 'run'. We add up the a- and b- runs and evenly distribute them to\n\t\t\t// get a stairstep path.\n\t\t\tint aRun = aIndex - aStart;\n\t\t\tint bRun = bIndex - bStart;\n\t\t\tint xCount = aRun + bRun;\n\n\t\t\t// Attempt to advance b and regress a.\n\t\t\tint bAdvance = max(xCount>> 1, bRun);\n\t\t\tint bEnd = min(bCount, bStart + bAdvance + 1);\n\t\t\tint bRunEnd = BinarySearch<MgpuBoundsUpper>(b + bIndex,\n\t\t\t\tbEnd - bIndex, x, comp) + bIndex;\n\t\t\tbRun = bRunEnd - bStart;\n\n\t\t\tbAdvance = min(bAdvance, bRun);\n\t\t\tint aAdvance = xCount - bAdvance;\n\n\t\t\tbool roundUp = (aAdvance == bAdvance + 1) && (bAdvance < bRun);\n\t\t\taIndex = aStart + aAdvance;\n\n\t\t\tif(roundUp) star = true;\n\t\t} else {\n\t\t\tif(aIndex && aCount) {\n\t\t\t\tT aKey = a[aIndex - 1];\n\t\t\t\tT bKey = b[bIndex];\n\n\t\t\t\t// If the last consumed element in A (aIndex - 1) is the same as\n\t\t\t\t// the next element in B (bIndex), we're sitting at a starred\n\t\t\t\t// partition.\n\t\t\t\tif(!comp(aKey, bKey)) star = true;\n\t\t\t}\n\t\t}\n\t}\n\treturn make_int2(aIndex, star);\n}\n\n} // namespace mgpu\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/device/ctasegreduce.cuh",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#pragma once\n\n#include \"ctasegscan.cuh\"\n#include \"ctasearch.cuh\"\n\nnamespace mgpu {\n\n////////////////////////////////////////////////////////////////////////////////\n// Segmented reduce utility functions.\n\n// Extract the upper-bound indices from the coded ranges. Decrement to include\n// the first addressed row/segment.\n\nstruct SegReduceRange {\n\tint begin;\n\tint end;\n\tint total;\n\tbool flushLast;\n};\n\nMGPU_DEVICE SegReduceRange DeviceShiftRange(int limit0, int limit1) {\n\tSegReduceRange range;\n\trange.begin = 0x7fffffff & limit0;\n\trange.end = 0x7fffffff & limit1;\n\trange.total = range.end - range.begin;\n\trange.flushLast = 0 == (0x80000000 & limit1);\n\trange.end += !range.flushLast;\n\treturn range;\n}\n\n// Reconstitute row/segment indices from a starting row index and packed end\n// flags. Used for pre-processed versions of interval reduce and interval Spmv.\ntemplate<int VT>\nMGPU_DEVICE void DeviceExpandFlagsToRows(int first, int endFlags,\n\tint rows[VT + 1]) {\n\n\trows[0] = first;\n\t#pragma unroll\n\tfor(int i = 0; i < VT; ++i) {\n\t\tif((1<< i) & endFlags) ++first;\n\t\trows[i + 1] = first;\n\t}\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// After loading CSR terms into shared memory, each thread binary searches\n// (upper-bound) to find its starting point. Each thread then walks forward,\n// emitting the csr0-relative row indices to register.\n\ntemplate<int NT, int VT>\nMGPU_DEVICE int DeviceExpandCsrRows(int tidOffset, int* csr_shared,\n\tint numRows, int end, int rows[VT + 1], int rowStarts[VT]) {\n\n\t// Each thread binary searches for its starting row.\n\tint row = BinarySearch<MgpuBoundsUpper>(csr_shared, numRows, tidOffset,\n\t\tmgpu::less<int>()) - 1;\n\n\t// Each thread starts at row and scans forward, emitting row IDs into\n\t// register. Store the CTA-local row index (starts at 0) to rows and the\n\t// start of the row (globally) to rowStarts.\n\tint curOffset = csr_shared[row];\n\tint nextOffset = (row + 1 < numRows) ? csr_shared[row + 1] : end;\n\n\trows[0] = row;\n\trowStarts[0] = curOffset;\n\tint endFlags = 0;\n\n\t#pragma unroll\n\tfor(int i = 1; i <= VT; ++i) {\n\t\t// Advance the row cursor when the iterator hits the next row offset.\n\t\tif(tidOffset + i == nextOffset) {\n\t\t\t// Set an end flag when the cursor advances to the next row.\n\t\t\tendFlags |= 1<< (i - 1);\n\n\t\t\t// Advance the cursor and load the next row offset.\n\t\t\t++row;\n\t\t\tcurOffset = nextOffset;\n\t\t\tnextOffset = (row + 1 < numRows) ? csr_shared[row + 1] : end;\n\t\t}\n\t\trows[i] = row;\n\t\tif(i < VT) rowStarts[i] = curOffset;\n\t}\n\t__syncthreads();\n\n\treturn endFlags;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// DeviceSegReducePrepare\n// Expand non-empty interval of CSR elements into row indices. Compute end-flags\n// by comparing adjacent row IDs.\n\n// DeviceSegReducePrepare may be called either by a pre-processing kernel or by\n// the kernel that actually evaluates the segmented reduction if no preprocesing\n// is desired.\nstruct SegReduceTerms {\n\tint endFlags;\n\tint tidDelta;\n};\n\ntemplate<int NT, int VT>\nMGPU_DEVICE SegReduceTerms DeviceSegReducePrepare(int* csr_shared, int numRows,\n\tint tid, int gid, bool flushLast, int rows[VT + 1], int rowStarts[VT]) {\n\n\t// Pass a sentinel (end) to point to the next segment start. If we flush,\n\t// this is the end of this tile. Otherwise it is INT_MAX\n\tint endFlags = DeviceExpandCsrRows<NT, VT>(gid + VT * tid, csr_shared,\n\t\tnumRows, flushLast ? (gid + NT * VT) : INT_MAX, rows, rowStarts);\n\n\t// Find the distance to to scan to compute carry-in for each thread. Use the\n\t// existance of an end flag anywhere in the thread to determine if carry-out\n\t// values from the left should propagate through to the right.\n\tint tidDelta = DeviceFindSegScanDelta<NT>(tid, rows[0] != rows[VT],\n\t\tcsr_shared);\n\n\tSegReduceTerms terms = { endFlags, tidDelta };\n\treturn terms;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// CTASegReduce\n// Core segmented reduction code. Supports fast-path and slow-path for intra-CTA\n// segmented reduction. Stores partials to global memory.\n// Callers feed CTASegReduce::ReduceToGlobal values in thread order.\ntemplate<int NT, int VT, bool HalfCapacity, typename T, typename Op>\nstruct CTASegReduce {\n\ttypedef CTASegScan<NT, Op> SegScan;\n\n\tenum {\n\t\tNV = NT * VT,\n\t\tCapacity = HalfCapacity ? (NV / 2) : NV\n\t};\n\n\tunion Storage {\n\t\ttypename SegScan::Storage segScanStorage;\n\t\tT values[Capacity];\n\t};\n\n\ttemplate<typename DestIt>\n\tMGPU_DEVICE static void ReduceToGlobal(const int rows[VT + 1], int total,\n\t\tint tidDelta, int startRow, int block, int tid, T data[VT],\n\t\tDestIt dest_global, T* carryOut_global, T identity, Op op,\n\t\tStorage& storage) {\n\n\t\t// Run a segmented scan within the thread.\n\t\tT x, localScan[VT];\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i) {\n\t\t\tx = i ? op(x, data[i]) : data[i];\n\t\t\tlocalScan[i] = x;\n\t\t\tif(rows[i] != rows[i + 1]) x = identity;\n\t\t}\n\n\t\t// Run a parallel segmented scan over the carry-out values to compute\n\t\t// carry-in.\n\t\tT carryOut;\n\t\tT carryIn = SegScan::SegScanDelta(tid, tidDelta, x,\n\t\t\tstorage.segScanStorage, &carryOut, identity, op);\n\n\t\t// Store the carry-out for the entire CTA to global memory.\n\t\tif(!tid) carryOut_global[block] = carryOut;\n\n\t\tdest_global += startRow;\n\t\tif(HalfCapacity && total > Capacity) {\n\t\t\t// Add carry-in to each thread-local scan value. Store directly\n\t\t\t// to global.\n\t\t\t#pragma unroll\n\t\t\tfor(int i = 0; i < VT; ++i) {\n\t\t\t\t// Add the carry-in to the local scan.\n\t\t\t\tT x2 = op(carryIn, localScan[i]);\n\n\t\t\t\t// Store on the end flag and clear the carry-in.\n\t\t\t\tif(rows[i] != rows[i + 1]) {\n\t\t\t\t\tcarryIn = identity;\n\t\t\t\t\tdest_global[rows[i]] = x2;\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\t// All partials fit in shared memory. Add carry-in to each thread-\n\t\t\t// local scan value.\n\t\t\t#pragma unroll\n\t\t\tfor(int i = 0; i < VT; ++i) {\n\t\t\t\t// Add the carry-in to the local scan.\n\t\t\t\tT x2 = op(carryIn, localScan[i]);\n\n\t\t\t\t// Store reduction when the segment changes and clear the\n\t\t\t\t// carry-in.\n\t\t\t\tif(rows[i] != rows[i + 1]) {\n\t\t\t\t\tstorage.values[rows[i]] = x2;\n\t\t\t\t\tcarryIn = identity;\n\t\t\t\t}\n\t\t\t}\n\t\t\t__syncthreads();\n\n\t\t\t// Cooperatively store reductions to global memory.\n\t\t\tfor(int index = tid; index < total; index += NT)\n\t\t\t\tdest_global[index] = storage.values[index];\n\t\t\t__syncthreads();\n\t\t}\n\t}\n};\n\n} // namespace mgpu\n\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/device/ctasegscan.cuh",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#pragma once\n\n#include \"ctascan.cuh\"\n\nnamespace mgpu {\n\n////////////////////////////////////////////////////////////////////////////////\n// DeviceFindSegScanDelta\n// Runs an inclusive max-index scan over binary inputs.\n\ntemplate<int NT>\nMGPU_DEVICE int DeviceFindSegScanDelta(int tid, bool flag, int* delta_shared) {\n\tconst int NumWarps = NT / 32;\n\n\tint warp = tid / 32;\n\tint lane = 31 & tid;\n\tuint warpMask = 0xffffffff>> (31 - lane);\t\t// inclusive search\n\tuint ctaMask = 0x7fffffff>> (31 - lane);\t\t// exclusive search\n\n\tuint warpBits = __ballot(flag);\n\tdelta_shared[warp] = warpBits;\n\t__syncthreads();\n\n\tif(tid < NumWarps) {\n\t\tuint ctaBits = __ballot(0 != delta_shared[tid]);\n\t\tint warpSegment = 31 - clz(ctaMask & ctaBits);\n\t\tint start = (-1 != warpSegment) ?\n\t\t\t(31 - clz(delta_shared[warpSegment]) + 32 * warpSegment) : 0;\n\t\tdelta_shared[NumWarps + tid] = start;\n\t}\n\t__syncthreads();\n\n\t// Find the closest flag to the left of this thread within the warp.\n\t// Include the flag for this thread.\n\tint start = 31 - clz(warpMask & warpBits);\n\tif(-1 != start) start += ~31 & tid;\n\telse start = delta_shared[NumWarps + warp];\n\t__syncthreads();\n\n\treturn tid - start;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// CTASegScan\n\ntemplate<int NT, typename _Op = mgpu::plus<int> >\nstruct CTASegScan {\n\ttypedef _Op Op;\n\ttypedef typename Op::result_type T;\n\tenum { NumWarps = NT / 32, Size = NT, Capacity = 2 * NT };\n\tunion Storage {\n\t\tint delta[NumWarps];\n\t\tT values[Capacity];\n\t};\n\n\t// Each thread passes the reduction of the LAST SEGMENT that it covers.\n\t// flag is set to true if there's at least one segment flag in the thread.\n\t// SegScan returns the reduction of values for the first segment in this\n\t// thread over the preceding threads.\n\t// Return the value init for the first thread.\n\n\t// When scanning single elements per thread, interpret the flag as a BEGIN\n\t// FLAG. If tid's flag is set, its value belongs to thread tid + 1, not\n\t// thread tid.\n\n\t// The function returns the reduction of the last segment in the CTA.\n\n\tMGPU_DEVICE static T SegScanDelta(int tid, int tidDelta, T x,\n\t\tStorage& storage, T* carryOut, T identity = (T)0, Op op = Op()) {\n\n\t\t// Run an inclusive scan\n\t\tint first = 0;\n\t\tstorage.values[first + tid] = x;\n\t\t__syncthreads();\n\n\t\t#pragma unroll\n\t\tfor(int offset = 1; offset < NT; offset += offset) {\n\t\t\tif(tidDelta >= offset)\n\t\t\t\tx = op(storage.values[first + tid - offset], x);\n\t\t\tfirst = NT - first;\n\t\t\tstorage.values[first + tid] = x;\n\t\t\t__syncthreads();\n\t\t}\n\n\t\t// Get the exclusive scan.\n\t\tx = tid ? storage.values[first + tid - 1] : identity;\n\t\t*carryOut = storage.values[first + NT - 1];\n\t\t__syncthreads();\n\t\treturn x;\n\t}\n\n\tMGPU_DEVICE static T SegScan(int tid, T x, bool flag, Storage& storage,\n\t\tT* carryOut, T identity = (T)0, Op op = Op()) {\n\n\t\t// Find the left-most thread that covers the first segment of this\n\t\t// thread.\n\t\tint tidDelta = DeviceFindSegScanDelta<NT>(tid, flag, storage.delta);\n\n\t\treturn SegScanDelta(tid, tidDelta, x, storage, carryOut, identity, op);\n\t}\n};\n\n} // namespace mgpu\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/device/ctasegsort.cuh",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#pragma once\n\n#include \"ctascan.cuh\"\n#include \"ctasearch.cuh\"\n#include \"loadstore.cuh\"\n#include \"sortnetwork.cuh\"\n\nnamespace mgpu {\n\ntemplate<int VT, typename T, typename Comp>\nMGPU_DEVICE void SegmentedSerialMerge(const T* keys_shared, int aBegin,\n\tint aEnd, int bBegin, int bEnd, T results[VT], int indices[VT],\n\tint leftEnd, int rightStart, Comp comp, bool sync = true) {\n\n\tbEnd = min(rightStart, bEnd);\n\tT aKey = keys_shared[aBegin];\n\tT bKey = keys_shared[bBegin];\n\n\t#pragma unroll\n\tfor(int i = 0; i < VT; ++i) {\n\t\tbool p;\n\n\t\t// If A has run out of inputs, emit B.\n\t\tif(aBegin >= aEnd)\n\t\t\tp = false;\n\t\telse if(bBegin >= bEnd || aBegin < leftEnd)\n\t\t\t// B has hit the end of the middle segment.\n\t\t\t// Emit A if A has inputs remaining in the middle segment.\n\t\t\tp = true;\n\t\telse\n\t\t\t// Emit the smaller element in the middle segment.\n\t\t\tp = !comp(bKey, aKey);\n\n\t\tresults[i] = p ? aKey : bKey;\n\t\tindices[i] = p ? aBegin : bBegin;\n\t\tif(p) aKey = keys_shared[++aBegin];\n\t\telse bKey = keys_shared[++bBegin];\n\t}\n\tif(sync) { __syncthreads(); }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// CTASegsortPass\n\ntemplate<int NT, int VT, typename T, typename Comp>\nMGPU_DEVICE void CTASegsortPass(T* keys_shared, int* ranges_shared, int tid,\n\tint pass, T results[VT], int indices[VT], int2& activeRange, Comp comp) {\n\n\t// Locate the intervals of the input lists.\n\tint3 frame = FindMergesortFrame(2<< pass, tid, VT);\n\tint a0 = frame.x;\n\tint b0 = frame.y;\n\tint listLen = frame.z;\n\tint list = tid>> pass;\n\tint listParity = 1 & list;\n\tint diag = VT * tid - frame.x;\n\n\t// Fetch the active range for the list this thread's list is merging with.\n\tint siblingRange = ranges_shared[1 ^ list];\n\tint siblingStart = 0x0000ffff & siblingRange;\n\tint siblingEnd = siblingRange>> 16;\n\n\t// Create a new active range for the merge.\n\tint leftEnd = listParity ? siblingEnd : activeRange.y;\n\tint rightStart = listParity ? activeRange.x : siblingStart;\n\tactiveRange.x = min(activeRange.x, siblingStart);\n\tactiveRange.y = max(activeRange.y, siblingEnd);\n\n\tint p = SegmentedMergePath(keys_shared, a0, listLen, b0, listLen, leftEnd,\n\t\trightStart, diag, comp);\n\n\tint a0tid = a0 + p;\n\tint b0tid = b0 + diag - p;\n\tSegmentedSerialMerge<VT>(keys_shared, a0tid, b0, b0tid, b0 + listLen,\n\t\tresults, indices, leftEnd, rightStart, comp);\n\n\t// Store the ranges to shared memory.\n\tif(0 == diag)\n\t\tranges_shared[list>> 1] =\n\t\t\t(int)bfi(activeRange.y, activeRange.x, 16, 16);\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// CTASegsortLoop\n\ntemplate<int NT, int VT, bool HasValues, typename KeyType, typename ValType,\n\ttypename Comp>\nMGPU_DEVICE int2 CTASegsortLoop(KeyType threadKeys[VT],\n\tValType threadValues[VT], KeyType* keys_shared, ValType* values_shared,\n\tint* ranges_shared, int tid, int2 activeRange, Comp comp) {\n\n\tconst int NumPasses = sLogPow2<NT>::value;\n\t#pragma unroll\n\tfor(int pass = 0; pass < NumPasses; ++pass) {\n\t\tint indices[VT];\n\t\tCTASegsortPass<NT, VT>(keys_shared, ranges_shared, tid, pass,\n\t\t\tthreadKeys, indices, activeRange, comp);\n\n\t\tif(HasValues) {\n\t\t\t// Exchange values through shared memory.\n\t\t\tDeviceThreadToShared<VT>(threadValues, tid, values_shared);\n\t\t\tDeviceGather<NT, VT>(NT * VT, values_shared, indices, tid,\n\t\t\t\tthreadValues);\n\t\t}\n\n\t\t// Store results in shared memory in sorted order.\n\t\tDeviceThreadToShared<VT>(threadKeys, tid, keys_shared);\n\t}\n\treturn activeRange;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// CTASegsort\n// Pass keys and values in register. On return, values are returned in register\n// and keys returned in shared memory.\n\ntemplate<int NT, int VT, bool Stable, bool HasValues, typename KeyType,\n\ttypename ValType, typename Comp>\nMGPU_DEVICE int2 CTASegsort(KeyType threadKeys[VT], ValType threadValues[VT],\n\tint tid, int headFlags, KeyType* keys_shared, ValType* values_shared,\n\tint* ranges_shared, Comp comp) {\n\n\tif(Stable)\n\t\t// Odd-even transpose sort.\n\t\tOddEvenTransposeSortFlags<VT>(threadKeys, threadValues, headFlags,\n\t\t\tcomp);\n\telse\n\t\t// Batcher's odd-even mergesort.\n\t\tOddEvenMergesortFlags<VT>(threadKeys, threadValues, headFlags, comp);\n\n\t// Record the first and last occurrence of head flags in this segment.\n\tint blockEnd = 31 - clz(headFlags);\n\tif(-1 != blockEnd) blockEnd += VT * tid;\n\n\tint blockStart = ffs(headFlags);\n\tblockStart = blockStart ? (VT * tid - 1 + blockStart) : (NT * VT);\n\n\tranges_shared[tid] = (int)bfi(blockEnd, blockStart, 16, 16);\n\n\t// Store back to shared mem. The values are in VT-length sorted lists.\n\t// These are merged recursively.\n\tDeviceThreadToShared<VT>(threadKeys, tid, keys_shared);\n\n\tint2 activeRange = CTASegsortLoop<NT, VT, HasValues>(threadKeys,\n\t\tthreadValues, keys_shared, values_shared, ranges_shared, tid,\n\t\tmake_int2(blockStart, blockEnd), comp);\n\treturn activeRange;\n}\n\n\ntemplate<int NT, int VT, bool Stable, typename KeyType, typename Comp>\nMGPU_DEVICE int2 CTASegsortKeys(KeyType threadKeys[VT], int tid, int headFlags,\n\tKeyType* keys_shared, int* ranges_shared, Comp comp) {\n\n\tint valuesTemp[VT];\n\treturn CTASegsort<NT, VT, Stable, false>(threadKeys, valuesTemp, tid,\n\t\theadFlags, keys_shared, (int*)keys_shared, ranges_shared, comp);\n}\n\ntemplate<int NT, int VT, bool Stable, typename KeyType, typename ValType,\n\ttypename Comp>\nMGPU_DEVICE int2 CTASegsortPairs(KeyType threadKeys[VT],\n\tValType threadValues[VT], int tid, int headFlags, KeyType* keys_shared,\n\tValType* values_shared, int* ranges_shared, Comp comp) {\n\n\treturn CTASegsort<NT, VT, Stable, true>(threadKeys, threadValues, tid,\n\t\theadFlags, keys_shared, values_shared, ranges_shared, comp);\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// DeviceSegBlocksort\n// Load keys and values from global memory, sort in shared memory, and store\n// back to global memory. Store the left-most and right-most encountered\n// headflag locations to ranges_global to prepare for the next pass.\n// This function is factored out of the blocksort kernel to allow easier\n// customization of that kernel - we have two implementations currently:\n// sort over indices and sort over bitfield.\n\ntemplate<int NT, int VT, bool Stable, bool HasValues, typename InputIt1,\n\ttypename InputIt2, typename KeyType, typename ValType, typename OutputIt1,\n\ttypename OutputIt2, typename Comp>\nMGPU_DEVICE void DeviceSegBlocksort(InputIt1 keys_global,\n\tInputIt2 values_global, int count2, KeyType* keys_shared,\n\tValType* values_shared, int* ranges_shared, int headFlags, int tid,\n\tint block, OutputIt1 keysDest_global, OutputIt2 valsDest_global,\n\tint* ranges_global, Comp comp) {\n\n\t// Load keys into register in thread order.\n\tint gid = NT * VT * block;\n\tKeyType threadKeys[VT];\n\tDeviceGlobalToShared<NT, VT>(count2, keys_global + gid, tid, keys_shared);\n\tDeviceSharedToThread<VT>(keys_shared, tid, threadKeys);\n\n\t// Load the values from global memory and into register in thread order.\n\tValType threadValues[VT];\n\tif(HasValues) {\n\t\tDeviceGlobalToShared<NT, VT>(count2, values_global + gid, tid,\n\t\t\tvalues_shared);\n\t\tDeviceSharedToThread<VT>(values_shared, tid, threadValues);\n\t}\n\n\t// Run the CTA segmented blocksort.\n\tint2 activeRange = CTASegsort<NT, VT, Stable, HasValues>(threadKeys,\n\t\tthreadValues, tid, headFlags, keys_shared, values_shared, ranges_shared,\n\t\tcomp);\n\n\t// Store the keys to global memory.\n\tDeviceSharedToGlobal<NT, VT>(count2, keys_shared, tid,\n\t\t keysDest_global + gid);\n\n\tif(HasValues) {\n\t\t// Store the values to global memory.xk b\n\t\tDeviceThreadToShared<VT>(threadValues, tid, values_shared);\n\t\tDeviceSharedToGlobal<NT, VT>(count2, values_shared, tid,\n\t\t\tvalsDest_global + gid, false);\n\t}\n\n\t// Store the 16-bit packed ranges. These are used by all merge kernels and\n\t// the first level of global segmented merge path partitioning.\n\tif(!tid)\n\t\tranges_global[block] = bfi(activeRange.y, activeRange.x, 16, 16);\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// DeviceIndicesToHeadFlags\n// Load indices from an array and cooperatively turn into a head flag bitfield\n// for each thread.\n\ntemplate<int NT, int VT>\nMGPU_DEVICE int DeviceIndicesToHeadFlags(const int* indices_global,\n\tconst int* partitions_global, int tid, int block, int count2,\n\tint* words_shared, byte* flags_shared) {\n\n\tconst int FlagWordsPerThread = MGPU_DIV_UP(VT, 4);\n\tint gid = NT * VT * block;\n\tint p0 = partitions_global[block];\n\tint p1 = partitions_global[block + 1];\n\n\tint headFlags = 0;\n\tif(p1 > p0 || count2 < NT * VT) {\n\n\t\t// Clear the flag bytes, then loop through the indices and poke in flag\n\t\t// values.\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < FlagWordsPerThread; ++i)\n\t\t\twords_shared[NT * i + tid] = 0;\n\t\t__syncthreads();\n\n\t\tfor(int index = p0 + tid; index < p1; index += NT) {\n\t\t\tint headFlag = indices_global[index];\n\t\t\tflags_shared[headFlag - gid] = 1;\n\t\t}\n\t\t__syncthreads();\n\n\t\t// Combine all the head flags for this thread.\n\t\tint first = VT * tid;\n\t\tint offset = first / 4;\n\t\tint prev = words_shared[offset];\n\t\tint mask = 0x3210 + 0x1111 * (3 & first);\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < FlagWordsPerThread; ++i) {\n\t\t\t// Gather the next four flags.\n\t\t\tint next = words_shared[offset + 1 + i];\n\t\t\tint x = prmt(prev, next, mask);\n\t\t\tprev = next;\n\n\t\t\t// Set the head flag bits.\n\t\t\tif(0x00000001 & x) headFlags |= 1<< (4 * i);\n\t\t\tif(0x00000100 & x) headFlags |= 1<< (4 * i + 1);\n\t\t\tif(0x00010000 & x) headFlags |= 1<< (4 * i + 2);\n\t\t\tif(0x01000000 & x) headFlags |= 1<< (4 * i + 3);\n\t\t}\n\t\t__syncthreads();\n\n\t\t// Set head flags for out-of-range keys.\n\t\tint outOfRange = min(VT, first + VT - count2);\n\t\tif(outOfRange > 0)\n\t\t\theadFlags = bfi(0xffffffff, headFlags, VT - outOfRange, outOfRange);\n\n\t\t// Clear head flags above VT.\n\t\theadFlags &= (1<< VT) - 1;\n\t}\n\treturn headFlags;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// SegSortSupport\n\nstruct SegSortSupport {\n\tint* ranges_global;\n\tint2* ranges2_global;\n\n\tint4* mergeList_global;\n\tint* copyList_global;\n\tint2* queueCounters_global;\n\tint2* nextCounters_global;\n\n\tbyte* copyStatus_global;\n};\n\n////////////////////////////////////////////////////////////////////////////////\n// DeviceSegSortMerge\n\ntemplate<int NT, int VT, bool HasValues, typename KeyType, typename ValueType,\n\ttypename Comp>\nMGPU_DEVICE void DeviceSegSortMerge(const KeyType* keys_global,\n\tconst ValueType* values_global, int2 segmentRange, int tid,\n\tint block, int4 range, int pass, KeyType* keys_shared,\n\tint* indices_shared, KeyType* keysDest_global, ValueType* valsDest_global,\n\tComp comp) {\n\n\tconst int NV = NT * VT;\n\tint gid = NV * block;\n\n\t// Load the local compressed segment indices.\n\tint a0 = range.x;\n\tint aCount = range.y - range.x;\n\tint b0 = range.z;\n\tint bCount = range.w - range.z;\n\n\tDeviceLoad2ToShared<NT, VT, VT>(keys_global + a0, aCount, keys_global + b0,\n\t\tbCount, tid, keys_shared);\n\n\t////////////////////////////////////////////////////////////////////////////\n\t// Run a merge path to find the starting point for each thread to merge.\n\t// If the entire warp fits into the already-sorted segments, we can skip\n\t// sorting it and leave its keys in shared memory. Doing this on the warp\n\t// level rather than thread level (also legal) gives slightly better\n\t// performance.\n\n\tint segStart = segmentRange.x;\n\tint segEnd = segmentRange.y;\n\tint listParity = 1 & (block>> pass);\n\n\tint warpOffset = VT * (~31 & tid);\n\tbool sortWarp = listParity ?\n\t\t// The spliced segment is to the left (segStart).\n\t\t(warpOffset < segStart) :\n\t\t// The spliced segment is to the right (segEnd).\n\t\t(warpOffset + 32 * VT > segEnd);\n\n\tKeyType threadKeys[VT];\n\tint indices[VT];\n\tif(sortWarp) {\n\t\tint diag = VT * tid;\n\t\tint mp = SegmentedMergePath(keys_shared, 0, aCount, aCount, bCount,\n\t\t\tlistParity ? 0 : segEnd, listParity ? segStart : NV, diag, comp);\n\t\tint a0tid = mp;\n\t\tint a1tid = aCount;\n\t\tint b0tid = aCount + diag - mp;\n\t\tint b1tid = aCount + bCount;\n\n\t\t// Serial merge into register. All threads in the CTA so we hoist the\n\t\t// check for list parity outside the function call to simplify the\n\t\t// logic. Unlike in the blocksort, this does not cause warp divergence.\n\t\tSegmentedSerialMerge<VT>(keys_shared, a0tid, a1tid, b0tid, b1tid,\n\t\t\tthreadKeys, indices, listParity ? 0 : segEnd,\n\t\t\tlistParity ? segStart : NV, comp, false);\n\t}\n\t__syncthreads();\n\n\t// Store sorted data in register back to shared memory. Then copy to global.\n\tif(sortWarp)\n\t\tDeviceThreadToShared<VT>(threadKeys, tid, keys_shared, false);\n\t__syncthreads();\n\n\tDeviceSharedToGlobal<NT, VT>(aCount + bCount, keys_shared, tid,\n\t\tkeysDest_global + gid);\n\n\t////////////////////////////////////////////////////////////////////////////\n\t// Use the merge indices to gather values from global memory. Store directly\n\t// to valsDest_global.\n\n\tif(HasValues) {\n\t\t// Transpose the gather indices to help coalesce loads.\n\t\tif(sortWarp)\n\t\t\tDeviceThreadToShared<VT>(indices, tid, indices_shared, false);\n\t\telse {\n\t\t\t#pragma unroll\n\t\t\tfor(int i = 0; i < VT; ++i)\n\t\t\t\tindices_shared[VT * tid + i] = VT * tid + i;\n\t\t}\n\t\t__syncthreads();\n\n\t\tDeviceTransferMergeValuesShared<NT, VT>(aCount + bCount,\n\t\t\tvalues_global + a0,  values_global + b0, aCount, indices_shared,\n\t\t\ttid, valsDest_global + NV * block);\n\t}\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// DeviceSegSortCopy\n\ntemplate<int NT, int VT, bool HasValues, typename KeyType, typename ValueType>\nMGPU_DEVICE void DeviceSegSortCopy(const KeyType* keys_global,\n\tconst ValueType* values_global, int tid, int block, int count,\n\tKeyType* keysDest_global, ValueType* valsDest_global) {\n\n\tint gid = NT * VT * block;\n\tint count2 = min(NT * VT, count - gid);\n\n\tDeviceGlobalToGlobal<NT, VT>(count2, keys_global + gid, tid,\n\t\tkeysDest_global + gid);\n\tif(HasValues)\n\t\tDeviceGlobalToGlobal<NT, VT>(count2, values_global + gid, tid,\n\t\t\tvalsDest_global + gid);\n}\n\n} // namespace mgpu\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/device/ctasortedsearch.cuh",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#pragma once\n\n#include \"../mgpudevice.cuh\"\n#include \"ctasearch.cuh\"\n\nnamespace mgpu {\n\n\n////////////////////////////////////////////////////////////////////////////////\n// DeviceSerialSearch\n\ntemplate<int VT, MgpuBounds Bounds, bool RangeCheck, bool IndexA, bool MatchA,\n\tbool IndexB, bool MatchB, typename T, typename Comp>\nMGPU_DEVICE int3 DeviceSerialSearch(const T* keys_shared, int aBegin,\n\tint aEnd, int bBegin, int bEnd, int aOffset, int bOffset, int* indices,\n\tComp comp) {\n\n\tconst int FlagA = IndexA ? 0x80000000 : 1;\n\tconst int FlagB = IndexB ? 0x80000000 : 1;\n\n\tT aKey = keys_shared[aBegin];\n\tT bKey = keys_shared[bBegin];\n\tT aPrev, bPrev;\n\tif(aBegin > 0) aPrev = keys_shared[aBegin - 1];\n\tif(bBegin > 0) bPrev = keys_shared[bBegin - 1];\n\tint decisions = 0;\n\tint matchCountA = 0;\n\tint matchCountB = 0;\n\n\t#pragma unroll\n\tfor(int i = 0; i < VT; ++i) {\n\t\tbool p;\n\t\tif(RangeCheck && aBegin >= aEnd) p = false;\n\t\telse if(RangeCheck && bBegin >= bEnd) p = true;\n\t\telse p = (MgpuBoundsUpper == Bounds) ?\n\t\t\tcomp(aKey, bKey) :\n\t\t\t!comp(bKey, aKey);\n\n\t\tif(p) {\n\t\t\t// aKey is smaller than bKey, so it is inserted before bKey.\n\t\t\t// Save bKey's index (bBegin + first) as the result of the search\n\t\t\t// and advance to the next needle in A.\n\t\t\tbool match = false;\n\t\t\tif(MatchA) {\n\t\t\t\t// Test if there is an element in B that matches aKey.\n\t\t\t\tif(MgpuBoundsUpper == Bounds) {\n\t\t\t\t\t// Upper Bound: We're inserting aKey after bKey. If there\n\t\t\t\t\t// is a match for aKey it must be bPrev. Check that bPrev\n\t\t\t\t\t// is in range and equal to aKey.\n\t\t\t\t\t// The predicate test result !comp(aKey, bPrev) was\n\t\t\t\t\t// established on the previous A-advancing iteration (it\n\t\t\t\t\t// failed the comp(aKey, bKey) test to get us to this\n\t\t\t\t\t// point). Check the other half of the equality condition\n\t\t\t\t\t// with a second comparison.\n\t\t\t\t\tbool inRange = !RangeCheck || (bBegin > aEnd);\n\t\t\t\t\tmatch = inRange && !comp(bPrev, aKey);\n\t\t\t\t} else {\n\t\t\t\t\t// Lower Bound: We're inserting aKey before bKey. If there\n\t\t\t\t\t// is a match for aKey, it must be bKey. Check that bKey\n\t\t\t\t\t// is in range and equal to aKey.\n\t\t\t\t\t// The predicate test !comp(bKey, aKey) has established one\n\t\t\t\t\t// half of the equality condition. We establish the other\n\t\t\t\t\t// half with a second comparison.\n\t\t\t\t\tbool inRange = !RangeCheck || (bBegin < bEnd);\n\t\t\t\t\tmatch = inRange && !comp(aKey, bKey);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tint index = 0;\n\t\t \tif(IndexA) index = bOffset + bBegin;\n\t\t\tif(match) index |= FlagA;\n\t\t\tif(IndexA || MatchA) indices[i] = index;\n\t\t\tmatchCountA += match;\n\n\t\t\t// Mark the decision bit to indicate that this iteration has\n\t\t\t// progressed A (the needles).\n\t\t\tdecisions |= 1<< i;\n\t\t\taPrev = aKey;\n\t\t\taKey = keys_shared[++aBegin];\n\t\t} else {\n\t\t\t// aKey is larger than bKey, so it is inserted after bKey (but we\n\t\t\t// don't know where yet). Advance the B index to the next element in\n\t\t\t// the haystack to continue the search for the current needle.\n\t\t\tbool match = false;\n\t\t\tif(MatchB) {\n\t\t\t\tif(MgpuBoundsUpper == Bounds) {\n\t\t\t\t\t// Upper Bound: aKey is not smaller than bKey. We advance to\n\t\t\t\t\t// the next haystack element in B. If there is a match in A\n\t\t\t\t\t// for bKey it must be aKey. By entering this branch we've\n\t\t\t\t\t// verified that !comp(aKey, bKey). Making the reciprocal\n\t\t\t\t\t// comparison !comp(bKey, aKey) establishes aKey == bKey.\n\t\t\t\t\tbool inRange = !RangeCheck ||\n\t\t\t\t\t\t((bBegin < bEnd) && (aBegin < aEnd));\n\t\t\t\t\tmatch = inRange && !comp(bKey, aKey);\n\t\t\t\t} else {\n\t\t\t\t\t// Lower Bound: bKey is smaller than aKey. We advance to the\n\t\t\t\t\t// next element in B. If there is a match for bKey, it must\n\t\t\t\t\t// be aPrev. The previous A-advancing iteration proved that\n\t\t\t\t\t// !comp(bKey, aPrev). We test !comp(aPrev, bKey) for the\n\t\t\t\t\t// other half of the equality condition.\n\t\t\t\t\tbool inRange = !RangeCheck ||\n\t\t\t\t\t\t((bBegin < bEnd) && (aBegin > 0));\n\t\t\t\t\tmatch = inRange && !comp(aPrev, bKey);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tint index = 0;\n\t\t\tif(IndexB) index = aOffset + aBegin;\n\t\t\tif(match) index |= FlagB;\n\t\t\tif(IndexB || MatchB) indices[i] = index;\n\t\t\tmatchCountB += match;\n\n\t\t\t// Keep the decision bit cleared to indicate that this iteration\n\t\t\t// has progressed B (the haystack).\n\t\t\tbPrev = bKey;\n\t\t\tbKey = keys_shared[++bBegin];\n\t\t}\n\t}\n\treturn make_int3(decisions, matchCountA, matchCountB);\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// CTASortedSearch\n// Take keys in shared memory and return indices and b-match flags in shared\n// memory.\n// NOTE: This function doesn't do any strided-to-thread order transposes so\n// using an even number of values per thread will incur no additional bank\n// conflicts.\n\ntemplate<int NT, int VT, MgpuBounds Bounds, bool IndexA, bool MatchA,\n\tbool IndexB, bool MatchB, typename T, typename Comp>\nMGPU_DEVICE int2 CTASortedSearch(T* keys_shared, int aStart, int aCount,\n\tint aEnd, int a0, int bStart, int bCount, int bEnd, int b0, bool extended,\n\tint tid, int* indices_shared, Comp comp) {\n\n\t// Run a merge path to find the start of the serial search for each thread.\n\tint diag = VT * tid;\n\tint mp = MergePath<Bounds>(keys_shared + aStart, aCount,\n\t\tkeys_shared + bStart, bCount, diag, comp);\n\tint a0tid = mp;\n\tint b0tid = diag - mp;\n\n\t// Serial search into register.\n\tint3 results;\n\tint indices[VT];\n\tif(extended)\n\t\tresults = DeviceSerialSearch<VT, Bounds, false, IndexA, MatchA, IndexB,\n\t\t\tMatchB>(keys_shared, a0tid + aStart, aEnd, b0tid + bStart, bEnd,\n\t\t\ta0 - aStart, b0 - bStart, indices, comp);\n\telse\n\t\tresults = DeviceSerialSearch<VT, Bounds, true, IndexA, MatchA, IndexB,\n\t\t\tMatchB>(keys_shared, a0tid + aStart, aEnd, b0tid + bStart, bEnd,\n\t\t\ta0 - aStart, b0 - bStart, indices, comp);\n\t__syncthreads();\n\n\t// Compact the indices into shared memory. Use the decision bits (set is A,\n\t// cleared is B) to select the destination.\n\tint decisions = results.x;\n\tb0tid += aCount;\n\t#pragma unroll\n\tfor(int i = 0; i < VT; ++i) {\n\t\tif((1<< i) & decisions) {\n\t\t\tif(IndexA || MatchA) indices_shared[a0tid++] = indices[i];\n\t\t} else {\n\t\t\tif(IndexB || MatchB) indices_shared[b0tid++] = indices[i];\n\t\t}\n\t}\n\t__syncthreads();\n\n\t// Return the match counts for A and B keys.\n\treturn make_int2(results.y, results.z);\n}\n\n} // namespace mgpu\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/device/devicetypes.cuh",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n * \n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n * \n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" \n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE \n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#pragma once\n\n#if __CUDA_ARCH__ == 100\n\t#error \"COMPUTE CAPABILITY 1.0 NOT SUPPORTED BY MPGU. TRY 2.0!\"\n#endif \n\n#include <climits>\n#include \"../util/static.h\"\n\n#ifdef _MSC_VER\n#define INLINESYMBOL __forceinline__\n#else\n#define INLINESYMBOL inline\n#endif\n\nnamespace mgpu {\n\n#define MGPU_HOST __host__ INLINESYMBOL\n#define MGPU_DEVICE __device__ INLINESYMBOL\n#define MGPU_HOST_DEVICE __host__ __device__ INLINESYMBOL\n\nconst int WARP_SIZE = 32;\nconst int LOG_WARP_SIZE = 5;\n\n////////////////////////////////////////////////////////////////////////////////\n// Device-side comparison operators\n\ntemplate<typename T>\nstruct less : public std::binary_function<T, T, bool> {\n\tMGPU_HOST_DEVICE bool operator()(T a, T b) { return a < b; }\n};\ntemplate<typename T>\nstruct less_equal : public std::binary_function<T, T, bool> {\n\tMGPU_HOST_DEVICE bool operator()(T a, T b) { return a <= b; }\n};\ntemplate<typename T>\nstruct greater : public std::binary_function<T, T, bool> {\n\tMGPU_HOST_DEVICE bool operator()(T a, T b) { return a > b; }\n};\ntemplate<typename T>\nstruct greater_equal : public std::binary_function<T, T, bool> {\n\tMGPU_HOST_DEVICE bool operator()(T a, T b) { return a >= b; }\n};\ntemplate<typename T>\nstruct equal_to : public std::binary_function<T, T, bool> {\n\tMGPU_HOST_DEVICE bool operator()(T a, T b) { return a == b; }\n};\ntemplate<typename T>\nstruct not_equal_to : public std::binary_function<T, T, bool> {\n\tMGPU_HOST_DEVICE bool operator()(T a, T b) { return a != b; }\n};\n\n////////////////////////////////////////////////////////////////////////////////\n// Device-side arithmetic operators\n\ntemplate<typename T>\nstruct plus : public std::binary_function<T, T, T> {\n\tMGPU_HOST_DEVICE T operator()(T a, T b) { return a + b; }\n};\n\ntemplate<typename T>\nstruct minus : public std::binary_function<T, T, T> {\n\tMGPU_HOST_DEVICE T operator()(T a, T b) { return a - b; }\n};\n\ntemplate<typename T>\nstruct multiplies : public std::binary_function<T, T, T> {\n\tMGPU_HOST_DEVICE T operator()(T a, T b) { return a * b; }\n};\n\ntemplate<typename T>\nstruct modulus : public std::binary_function<T, T, T> {\n\tMGPU_HOST_DEVICE T operator()(T a, T b) { return a % b; }\n};\n\ntemplate<typename T>\nstruct bit_or : public std::binary_function<T, T, T> {\n\tMGPU_HOST_DEVICE T operator()(T a, T b) { return a | b; }\n};\n\ntemplate<typename T>\nstruct bit_and : public std::binary_function<T, T, T> {\n\tMGPU_HOST_DEVICE T operator()(T a, T b) { return a & b; }\n};\n\ntemplate<typename T>\nstruct bit_xor : public std::binary_function<T, T, T> {\n\tMGPU_HOST_DEVICE T operator()(T a, T b) { return a ^ b; }\n};\n\ntemplate<typename T>\nstruct maximum : public std::binary_function<T, T, T> {\n\tMGPU_HOST_DEVICE T operator()(T a, T b) { return max(a, b); }\n};\n\ntemplate<typename T>\nstruct minimum : public std::binary_function<T, T, T> {\n\tMGPU_HOST_DEVICE T operator()(T a, T b) { return min(a, b); }\n};\n\n////////////////////////////////////////////////////////////////////////////////\n\ntemplate<typename T>\nMGPU_HOST_DEVICE void swap(T& a, T& b) {\n\tT c = a;\n\ta = b;\n\tb = c;\n}\n\ntemplate<typename T>\nstruct DevicePair {\n\tT x, y;\n};\n\ntemplate<typename T>\nMGPU_HOST_DEVICE DevicePair<T> MakeDevicePair(T x, T y) {\n\tDevicePair<T> p = { x, y };\n\treturn p;\n}\n\ntemplate<typename T> struct numeric_limits;\ntemplate<> struct numeric_limits<int> {\n\tMGPU_HOST_DEVICE static int min() { return INT_MIN; }\n\tMGPU_HOST_DEVICE static int max() { return INT_MAX; }\n\tMGPU_HOST_DEVICE static int lowest() { return INT_MIN; }\n\tMGPU_HOST_DEVICE static int AddIdent() { return 0; }\n\tMGPU_HOST_DEVICE static int MulIdent() { return 1; }\n};\ntemplate<> struct numeric_limits<long long> {\n\tMGPU_HOST_DEVICE static long long min() { return LLONG_MIN; }\n\tMGPU_HOST_DEVICE static long long max() { return LLONG_MAX; }\n\tMGPU_HOST_DEVICE static long long lowest() { return LLONG_MIN; }\n\tMGPU_HOST_DEVICE static long long AddIdent() { return 0; }\n\tMGPU_HOST_DEVICE static long long MulIdent() { return 1; }\n};\ntemplate<> struct numeric_limits<uint> {\n\tMGPU_HOST_DEVICE static uint min() { return 0; }\n\tMGPU_HOST_DEVICE static uint max() { return UINT_MAX; }\n\tMGPU_HOST_DEVICE static uint lowest() { return 0; }\n\tMGPU_HOST_DEVICE static uint AddIdent() { return 0; }\n\tMGPU_HOST_DEVICE static uint MulIdent() { return 1; }\n};\ntemplate<> struct numeric_limits<unsigned long long> {\n\tMGPU_HOST_DEVICE static unsigned long long min() { return 0; }\n\tMGPU_HOST_DEVICE static unsigned long long max() { return ULLONG_MAX; }\n\tMGPU_HOST_DEVICE static unsigned long long lowest() { return 0; }\n\tMGPU_HOST_DEVICE static unsigned long long AddIdent() { return 0; }\n\tMGPU_HOST_DEVICE static unsigned long long MulIdent() { return 1; }\n};\ntemplate<> struct numeric_limits<float> {\n\tMGPU_HOST_DEVICE static float min() { return FLT_MIN; }\n\tMGPU_HOST_DEVICE static float max() { return FLT_MAX; }\n\tMGPU_HOST_DEVICE static float lowest() { return -FLT_MAX; }\n\tMGPU_HOST_DEVICE static float AddIdent() { return 0; }\n\tMGPU_HOST_DEVICE static float MulIdent() { return 1; }\n};\ntemplate<> struct numeric_limits<double> {\n\tMGPU_HOST_DEVICE static double min() { return DBL_MIN; }\n\tMGPU_HOST_DEVICE static double max() { return DBL_MAX; }\n\tMGPU_HOST_DEVICE static double lowest() { return -DBL_MAX; }\n\tMGPU_HOST_DEVICE static double AddIdent() { return 0; }\n\tMGPU_HOST_DEVICE static double MulIdent() { return 1; }\n};\n\n\nMGPU_HOST_DEVICE int2 operator+(int2 a, int2 b) {\n\treturn make_int2(a.x + b.x, a.y + b.y); \n}\nMGPU_HOST_DEVICE int2& operator+=(int2& a, int2 b) {\n\ta = a + b;\n\treturn a;\n}\nMGPU_HOST_DEVICE int2 operator*(int2 a, int2 b) {\n\treturn make_int2(a.x * b.x, a.y * b.y);\n}\nMGPU_HOST_DEVICE int2& operator*=(int2& a, int2 b) {\n\ta = a * b;\n\treturn a;\n}\n\ntemplate<typename T>\nMGPU_HOST_DEVICE T max(T a, T b) {\n#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ < 100)\n\treturn std::max(a, b);\n#else\n\treturn (a < b) ? b : a;\n#endif\n}\ntemplate<typename T>\nMGPU_HOST_DEVICE T min(T a, T b) {\n#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ < 100)\n\treturn std::min(a, b);\n#else\n\treturn (b < a) ? b : a;\n#endif\n}\n\nMGPU_HOST_DEVICE int2 max(int2 a, int2 b) {\n\treturn make_int2(max(a.x, b.x), max(a.y, b.y));\n}\n\nMGPU_HOST_DEVICE int2 min(int2 a, int2 b) {\n\treturn make_int2(min(a.x, b.x), min(a.y, b.y));\n}\n\ntemplate<> struct numeric_limits<int2> {\n\tMGPU_HOST_DEVICE static int2 min() { return make_int2(INT_MIN, INT_MIN); }\n\tMGPU_HOST_DEVICE static int2 max() { return make_int2(INT_MAX, INT_MAX); }\n\tMGPU_HOST_DEVICE static int2 lowest() { \n\t\treturn make_int2(INT_MIN, INT_MIN); \n\t}\n\tMGPU_HOST_DEVICE static int2 AddIdent() { return make_int2(0, 0); }\n\tMGPU_HOST_DEVICE static int2 MulIdent() { return make_int2(1, 1); }\n};\n\ntemplate<typename T>\nclass constant_iterator : public std::iterator_traits<const T*> {\npublic:\n\tMGPU_HOST_DEVICE constant_iterator(T value) : _value(value) { }\n\n\tMGPU_HOST_DEVICE T operator[](ptrdiff_t i) const { \n\t\treturn _value;\n\t}\n\tMGPU_HOST_DEVICE T operator*() const {\n\t\treturn _value;\n\t}\n\tMGPU_HOST_DEVICE constant_iterator operator+(ptrdiff_t diff) const {\n\t\treturn constant_iterator(_value);\n\t}\n\tMGPU_HOST_DEVICE constant_iterator operator-(ptrdiff_t diff) const {\n\t\treturn constant_iterator(_value);\n\t}\n\tMGPU_HOST_DEVICE constant_iterator& operator+=(ptrdiff_t diff) {\n\t\treturn *this;\n\t}\n\tMGPU_HOST_DEVICE constant_iterator& operator-=(ptrdiff_t diff) {\n\t\treturn *this;\n\t}\nprivate:\n\tT _value;\n};\n\ntemplate<typename T>\nclass counting_iterator : public std::iterator_traits<const T*> {\npublic:\n\tMGPU_HOST_DEVICE counting_iterator(T value) : _value(value) { }\n\n\tMGPU_HOST_DEVICE T operator[](ptrdiff_t i) { \n\t\treturn _value + i;\n\t}\n\tMGPU_HOST_DEVICE T operator*() {\n\t\treturn _value;\n\t}\n\tMGPU_HOST_DEVICE counting_iterator operator+(ptrdiff_t diff) {\n\t\treturn counting_iterator(_value + diff);\n\t}\n\tMGPU_HOST_DEVICE counting_iterator operator-(ptrdiff_t diff) {\n\t\treturn counting_iterator(_value - diff);\n\t}\n\tMGPU_HOST_DEVICE counting_iterator& operator+=(ptrdiff_t diff) {\n\t\t_value += diff;\n\t\treturn *this;\n\t}\n\tMGPU_HOST_DEVICE counting_iterator& operator-=(ptrdiff_t diff) {\n\t\t_value -= diff;\n\t\treturn *this;\n\t}\nprivate:\n\tT _value;\n};\n\ntemplate<typename T>\nclass step_iterator : public std::iterator_traits<const T*> {\npublic:\n\tMGPU_HOST_DEVICE step_iterator(T base, T step) :\n\t\t_base(base), _step(step), _offset(0) { }\n\n\tMGPU_HOST_DEVICE T operator[](ptrdiff_t i) { \n\t\treturn _base + (_offset + i) * _step; \n\t}\n\tMGPU_HOST_DEVICE T operator*() { \n\t\treturn _base + _offset * _step; \n\t} \n\tMGPU_HOST_DEVICE step_iterator operator+(ptrdiff_t diff) {\n\t\tstep_iterator it = *this;\n\t\tit._offset += diff;\n\t\treturn it;\n\t}\n\tMGPU_HOST_DEVICE step_iterator operator-(ptrdiff_t diff) {\n\t\tstep_iterator it = *this;\n\t\tit._offset -= diff;\n\t\treturn it;\n\t}\n\tMGPU_HOST_DEVICE step_iterator& operator+=(ptrdiff_t diff) { \n\t\t_offset += diff;\n\t\treturn *this;\n\t}\n\tMGPU_HOST_DEVICE step_iterator& operator-=(ptrdiff_t diff) { \n\t\t_offset -= diff;\n\t\treturn *this;\n\t}\nprivate:\n\tptrdiff_t _offset;\n\tT _base, _step;\t\n};\n\n} // namespace mgpu\n\n\ntemplate<typename T>\nMGPU_HOST_DEVICE mgpu::counting_iterator<T> operator+(ptrdiff_t diff,\n\tmgpu::counting_iterator<T> it) {\n\treturn it + diff;\n}\ntemplate<typename T>\nMGPU_HOST_DEVICE mgpu::counting_iterator<T> operator-(ptrdiff_t diff,\n\tmgpu::counting_iterator<T> it) {\n\treturn it + (-diff);\n}\ntemplate<typename T>\nMGPU_HOST_DEVICE mgpu::step_iterator<T> operator+(ptrdiff_t diff, \n\tmgpu::step_iterator<T> it) {\n\treturn it + diff;\n}\ntemplate<typename T>\nMGPU_HOST_DEVICE mgpu::step_iterator<T> operator-(ptrdiff_t diff, \n\tmgpu::step_iterator<T> it) {\n\treturn it + (-diff);\n}\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/device/deviceutil.cuh",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#pragma once\n\n#include \"intrinsics.cuh\"\n\nnamespace mgpu {\n\n// Get the difference between two pointers in bytes.\nMGPU_HOST_DEVICE ptrdiff_t PtrDiff(const void* a, const void* b) {\n\treturn (const byte*)b - (const byte*)a;\n}\n\n// Offset a pointer by i bytes.\ntemplate<typename T>\nMGPU_HOST_DEVICE const T* PtrOffset(const T* p, ptrdiff_t i) {\n\treturn (const T*)((const byte*)p + i);\n}\ntemplate<typename T>\nMGPU_HOST_DEVICE T* PtrOffset(T* p, ptrdiff_t i) {\n\treturn (T*)((byte*)p + i);\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// Task range support\n// Evenly distributes variable-length arrays over a fixed number of CTAs.\n\nMGPU_HOST int2 DivideTaskRange(int numItems, int numWorkers) {\n\tdiv_t d = div(numItems, numWorkers);\n\treturn make_int2(d.quot, d.rem);\n}\n\nMGPU_HOST_DEVICE int2 ComputeTaskRange(int block, int2 task) {\n\tint2 range;\n\trange.x = task.x * block;\n\trange.x += min(block, task.y);\n\trange.y = range.x + task.x + (block < task.y);\n\treturn range;\n}\n\nMGPU_HOST_DEVICE int2 ComputeTaskRange(int block, int2 task, int blockSize,\n\tint count) {\n\tint2 range = ComputeTaskRange(block, task);\n\trange.x *= blockSize;\n\trange.y = min(count, range.y * blockSize);\n\treturn range;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// DeviceExtractHeadFlags\n// Input array flags is a bit array with 32 head flags per word.\n// ExtractThreadHeadFlags returns numBits flags starting at bit index.\n\nMGPU_HOST_DEVICE uint DeviceExtractHeadFlags(const uint* flags, int index,\n\tint numBits) {\n\n\tint index2 = index>> 5;\n\tint shift = 31 & index;\n\tuint headFlags = flags[index2]>> shift;\n\tint shifted = 32 - shift;\n\n\tif(shifted < numBits)\n\t\t// We also need to shift in the next set of bits.\n\t\theadFlags = bfi(flags[index2 + 1], headFlags, shifted, shift);\n\theadFlags &= (1<< numBits) - 1;\n\treturn headFlags;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// DevicePackHeadFlags\n// Pack VT bits per thread at 32 bits/thread. Will consume an integer number of\n// words, because CTA size is a multiple of 32. The first NT * VT / 32 threads\n// return packed words.\n\ntemplate<int NT, int VT>\nMGPU_DEVICE uint DevicePackHeadFlags(uint threadBits, int tid,\n\tuint* flags_shared) {\n\n\tconst int WordCount = NT * VT / 32;\n\n\t// Each thread stores its thread bits to flags_shared[tid].\n\tflags_shared[tid] = threadBits;\n\t__syncthreads();\n\n\tuint packed = 0;\n\tif(tid < WordCount) {\n\t\tconst int Items = MGPU_DIV_UP(32, VT);\n\t\tint index = 32 * tid;\n\t\tint first = index / VT;\n\t\tint bit = 0;\n\n\t\tint rem = index - VT * first;\n\t\tpacked = flags_shared[first]>> rem;\n\t\tbit = VT - rem;\n\t\t++first;\n\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < Items; ++i) {\n\t\t\tif(i < Items - 1 || bit < 32) {\n\t\t\t\tuint x = flags_shared[first + i];\n\t\t\t\tif(bit < 32) packed |= x<< bit;\n\t\t\t\tbit += VT;\n\t\t\t}\n\t\t}\n\t}\n\t__syncthreads();\n\n\treturn packed;\n}\n\n} // namespace mgpu\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/device/intrinsics.cuh",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#include \"devicetypes.cuh\"\n\n#pragma once\n\n#pragma GCC diagnostic push\n#pragma GCC diagnostic ignored \"-Wstrict-aliasing\"\n\nnamespace mgpu {\n\nMGPU_HOST_DEVICE uint2 ulonglong_as_uint2(uint64 x) {\n\treturn *reinterpret_cast<uint2*>(&x);\n}\nMGPU_HOST_DEVICE uint64 uint2_as_ulonglong(uint2 x) {\n\treturn *reinterpret_cast<uint64*>(&x);\n}\n\nMGPU_HOST_DEVICE int2 longlong_as_int2(int64 x) {\n\treturn *reinterpret_cast<int2*>(&x);\n}\nMGPU_HOST_DEVICE int64 int2_as_longlong(int2 x) {\n\treturn *reinterpret_cast<int64*>(&x);\n}\n\nMGPU_HOST_DEVICE int2 double_as_int2(double x) {\n\treturn *reinterpret_cast<int2*>(&x);\n}\nMGPU_HOST_DEVICE double int2_as_double(int2 x) {\n\treturn *reinterpret_cast<double*>(&x);\n}\n\nMGPU_HOST_DEVICE void SetDoubleX(double& d, int x) {\n\treinterpret_cast<int*>(&d)[0] = x;\n}\nMGPU_HOST_DEVICE int GetDoubleX(double d) {\n\treturn double_as_int2(d).x;\n}\nMGPU_HOST_DEVICE void SetDoubleY(double& d, int y) {\n\treinterpret_cast<int*>(&d)[1] = y;\n}\nMGPU_HOST_DEVICE int GetDoubleY(double d) {\n\treturn double_as_int2(d).y;\n}\n\n\n////////////////////////////////////////////////////////////////////////////////\n// PTX for bfe and bfi\n\n#if __CUDA_ARCH__ >= 200\n\nMGPU_DEVICE uint bfe_ptx(uint x, uint bit, uint numBits) {\n\tuint result;\n\tasm(\"bfe.u32 %0, %1, %2, %3;\" :\n\t\t\"=r\"(result) : \"r\"(x), \"r\"(bit), \"r\"(numBits));\n\treturn result;\n}\n\n\nMGPU_DEVICE uint bfi_ptx(uint x, uint y, uint bit, uint numBits) {\n\tuint result;\n\tasm(\"bfi.b32 %0, %1, %2, %3, %4;\" :\n\t\t\"=r\"(result) : \"r\"(x), \"r\"(y), \"r\"(bit), \"r\"(numBits));\n\treturn result;\n}\n\nMGPU_DEVICE uint prmt_ptx(uint a, uint b, uint index) {\n\tuint ret;\n\tasm(\"prmt.b32 %0, %1, %2, %3;\" : \"=r\"(ret) : \"r\"(a), \"r\"(b), \"r\"(index));\n\treturn ret;\n}\n\n#endif // __CUDA_ARCH__ >= 200\n\n#if CUDA_VERSION >= 9000\n////////////////////////////////////////////////////////////////////////////////\n// shfl_add\n\nMGPU_DEVICE int shfl_add(int x, int offset, int width = WARP_SIZE, unsigned int threadmask = 0xFFFFFFFF) {\n\tint result = 0;\n#if __CUDA_ARCH__ >= 300\n\tint mask = (WARP_SIZE - width)<< 8;\n\tasm(\n\t\t\"{.reg .s32 r0;\"\n\t\t\".reg .pred p;\"\n\t\t\"shfl.sync.up.b32 r0|p, %1, %2, %3, %4;\"\n\t\t\"@p add.s32 r0, r0, %5;\"\n\t\t\"mov.s32 %0, r0; }\"\n\t\t: \"=r\"(result) : \"r\"(x), \"r\"(offset), \"r\"(mask), \"r\"(threadmask), \"r\"(x));\n#endif\n\treturn result;\n}\n\nMGPU_DEVICE int shfl_max(int x, int offset, int width = WARP_SIZE, unsigned int threadmask = 0xFFFFFFFF) {\n\tint result = 0;\n#if __CUDA_ARCH__ >= 300\n\tint mask = (WARP_SIZE - width)<< 8;\n\tasm(\n\t\t\"{.reg .s32 r0;\"\n\t\t\".reg .pred p;\"\n\t\t\"shfl.sync.up.b32 r0|p, %1, %2, %3, %4;\"\n\t\t\"@p max.s32 r0, r0, %5;\"\n\t\t\"mov.s32 %0, r0; }\"\n\t\t: \"=r\"(result) : \"r\"(x), \"r\"(offset), \"r\"(mask), \"r\"(threadmask), \"r\"(x));\n#endif\n\treturn result;\n}\n#else\n////////////////////////////////////////////////////////////////////////////////\n// shfl_add\n\nMGPU_DEVICE int shfl_add(int x, int offset, int width = WARP_SIZE) {\n\tint result = 0;\n#if __CUDA_ARCH__ >= 300\n\tint mask = (WARP_SIZE - width)<< 8;\n\tasm(\n\t\t\"{.reg .s32 r0;\"\n\t\t\".reg .pred p;\"\n\t\t\"shfl.up.b32 r0|p, %1, %2, %3;\"\n\t\t\"@p add.s32 r0, r0, %4;\"\n\t\t\"mov.s32 %0, r0; }\"\n\t\t: \"=r\"(result) : \"r\"(x), \"r\"(offset), \"r\"(mask), \"r\"(x));\n#endif\n\treturn result;\n}\n\nMGPU_DEVICE int shfl_max(int x, int offset, int width = WARP_SIZE) {\n\tint result = 0;\n#if __CUDA_ARCH__ >= 300\n\tint mask = (WARP_SIZE - width)<< 8;\n\tasm(\n\t\t\"{.reg .s32 r0;\"\n\t\t\".reg .pred p;\"\n\t\t\"shfl.up.b32 r0|p, %1, %2, %3;\"\n\t\t\"@p max.s32 r0, r0, %4;\"\n\t\t\"mov.s32 %0, r0; }\"\n\t\t: \"=r\"(result) : \"r\"(x), \"r\"(offset), \"r\"(mask), \"r\"(x));\n#endif\n\treturn result;\n}\n#endif\n\n////////////////////////////////////////////////////////////////////////////////\n// brev, popc, clz, bfe, bfi, prmt\n\n// Reverse the bits in an integer.\nMGPU_HOST_DEVICE uint brev(uint x) {\n#if __CUDA_ARCH__ >= 200\n\tuint y = __brev(x);\n#else\n\tuint y = 0;\n\tfor(int i = 0; i < 32; ++i)\n\t\ty |= (1 & (x>> i))<< (31 - i);\n#endif\n\treturn y;\n}\n\n// Count number of bits in a register.\nMGPU_HOST_DEVICE int popc(uint x) {\n#if __CUDA_ARCH__ >= 200\n\treturn __popc(x);\n#else\n\tint c;\n\tfor(c = 0; x; ++c)\n\t\tx &= x - 1;\n\treturn c;\n#endif\n}\n\n// Count leading zeros - start from most significant bit.\nMGPU_HOST_DEVICE int clz(int x) {\n#if __CUDA_ARCH__ >= 200\n\treturn __clz(x);\n#else\n\tfor(int i = 31; i >= 0; --i)\n\t\tif((1<< i) & x) return 31 - i;\n\treturn 32;\n#endif\n}\n\n// Find first set - start from least significant bit. LSB is 1. ffs(0) is 0.\nMGPU_HOST_DEVICE int ffs(int x) {\n#if __CUDA_ARCH__ >= 200\n\treturn __ffs(x);\n#else\n\tfor(int i = 0; i < 32; ++i)\n\t\tif((1<< i) & x) return i + 1;\n\treturn 0;\n#endif\n}\n\nMGPU_HOST_DEVICE uint bfe(uint x, uint bit, uint numBits) {\n#if __CUDA_ARCH__ >= 200\n\treturn bfe_ptx(x, bit, numBits);\n#else\n\treturn ((1<< numBits) - 1) & (x>> bit);\n#endif\n}\n\nMGPU_HOST_DEVICE uint bfi(uint x, uint y, uint bit, uint numBits) {\n\tuint result;\n#if __CUDA_ARCH__ >= 200\n\tresult = bfi_ptx(x, y, bit, numBits);\n#else\n\tif(bit + numBits > 32) numBits = 32 - bit;\n\tuint mask = ((1<< numBits) - 1)<< bit;\n\tresult = y & ~mask;\n\tresult |= mask & (x<< bit);\n#endif\n\treturn result;\n}\n\nMGPU_HOST_DEVICE uint prmt(uint a, uint b, uint index) {\n\tuint result;\n#if __CUDA_ARCH__ >= 200\n\tresult = prmt_ptx(a, b, index);\n#else\n\tresult = 0;\n\tfor(int i = 0; i < 4; ++i) {\n\t\tuint sel = 0xf & (index>> (4 * i));\n\t\tuint x = ((7 & sel) > 3) ? b : a;\n\t\tx = 0xff & (x>> (8 * (3 & sel)));\n\t\tif(8 & sel) x = (128 & x) ? 0xff : 0;\n\t\tresult |= x<< (8 * i);\n\t}\n#endif\n\treturn result;\n}\n\n// Find log2(x) and optionally round up to the next integer logarithm.\nMGPU_HOST_DEVICE int FindLog2(int x, bool roundUp = false) {\n\tint a = 31 - clz(x);\n\tif(roundUp) a += !MGPU_IS_POW_2(x);\n\treturn a;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// vset4\n\n#if __CUDA_ARCH__ >= 300\n\n// Performs four byte-wise comparisons and returns 1 for each byte that\n// satisfies the conditional, and zero otherwise.\nMGPU_DEVICE uint vset4_lt_add_ptx(uint a, uint b, uint c) {\n\tuint result;\n\tasm(\"vset4.u32.u32.lt.add %0, %1, %2, %3;\" :\n\t\t\"=r\"(result) : \"r\"(a), \"r\"(b), \"r\"(c));\n\treturn result;\n}\nMGPU_DEVICE uint vset4_eq_ptx(uint a, uint b) {\n\tuint result;\n\tasm(\"vset4.u32.u32.eq %0, %1, %2, %3;\" :\n\t\t\"=r\"(result) : \"r\"(a), \"r\"(b), \"r\"(0));\n\treturn result;\n}\n#endif // __CUDA_ARCH__ >= 300\n\nMGPU_HOST_DEVICE uint vset4_lt_add(uint a, uint b, uint c) {\n\tuint result;\n#if __CUDA_ARCH__ >= 300\n\tresult = vset4_lt_add_ptx(a, b, c);\n#else\n\tresult = c;\n\tif((0x000000ff & a) < (0x000000ff & b)) result += 0x00000001;\n\tif((0x0000ff00 & a) < (0x0000ff00 & b)) result += 0x00000100;\n\tif((0x00ff0000 & a) < (0x00ff0000 & b)) result += 0x00010000;\n\tif((0xff000000 & a) < (0xff000000 & b)) result += 0x01000000;\n#endif\n\treturn result;\n}\n\nMGPU_HOST_DEVICE uint vset4_eq(uint a, uint b) {\n\tuint result;\n#if __CUDA_ARCH__ >= 300\n\tresult = vset4_eq_ptx(a, b);\n#else\n\tresult = 0;\n\tif((0x000000ff & a) == (0x000000ff & b)) result = 0x00000001;\n\tif((0x0000ff00 & a) == (0x0000ff00 & b)) result += 0x00000100;\n\tif((0x00ff0000 & a) == (0x00ff0000 & b)) result += 0x00010000;\n\tif((0xff000000 & a) == (0xff000000 & b)) result += 0x01000000;\n#endif\n\treturn result;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n//\n\nMGPU_HOST_DEVICE uint umulhi(uint x, uint y) {\n#if __CUDA_ARCH__ >= 100\n\treturn __umulhi(x, y);\n#else\n\tuint64 product = (uint64)x * y;\n\treturn (uint)(product>> 32);\n#endif\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// ldg() function defined for all devices and all types. Only compiles to __ldg\n// intrinsic for __CUDA_ARCH__ >= 320 && __CUDA_ARCH__ < 400 for types supported\n// by __ldg in sm_32_intrinsics.h\n\ntemplate<typename T>\nstruct IsLdgType {\n\tenum { value = false };\n};\n#define DEFINE_LDG_TYPE(T) \\\n\ttemplate<> struct IsLdgType<T> { enum { value = true }; };\n\ntemplate<typename T, bool UseLDG = IsLdgType<T>::value>\nstruct LdgShim {\n\tMGPU_DEVICE static T Ldg(const T* p) {\n\t\treturn *p;\n\t}\n};\n\n#if __CUDA_ARCH__ >= 320 && __CUDA_ARCH__ < 400\n\n\t// List of __ldg-compatible types from sm_32_intrinsics.h.\n\tDEFINE_LDG_TYPE(char)\n\tDEFINE_LDG_TYPE(short)\n\tDEFINE_LDG_TYPE(int)\n\tDEFINE_LDG_TYPE(long long)\n\tDEFINE_LDG_TYPE(char2)\n\tDEFINE_LDG_TYPE(char4)\n\tDEFINE_LDG_TYPE(short2)\n\tDEFINE_LDG_TYPE(short4)\n\tDEFINE_LDG_TYPE(int2)\n\tDEFINE_LDG_TYPE(int4)\n\tDEFINE_LDG_TYPE(longlong2)\n\n\tDEFINE_LDG_TYPE(unsigned char)\n\tDEFINE_LDG_TYPE(unsigned short)\n\tDEFINE_LDG_TYPE(unsigned int)\n\tDEFINE_LDG_TYPE(unsigned long long)\n\tDEFINE_LDG_TYPE(uchar2)\n\tDEFINE_LDG_TYPE(uchar4)\n\tDEFINE_LDG_TYPE(ushort2)\n\tDEFINE_LDG_TYPE(ushort4)\n\tDEFINE_LDG_TYPE(uint2)\n\tDEFINE_LDG_TYPE(uint4)\n\tDEFINE_LDG_TYPE(ulonglong2)\n\n\tDEFINE_LDG_TYPE(float)\n\tDEFINE_LDG_TYPE(double)\n\tDEFINE_LDG_TYPE(float2)\n\tDEFINE_LDG_TYPE(float4)\n\tDEFINE_LDG_TYPE(double2)\n\n\ttemplate<typename T> struct LdgShim<T, true> {\n\t\tMGPU_DEVICE static T Ldg(const T* p) {\n\t\t\treturn __ldg(p);\n\t\t}\n\t};\n#endif\n\ntemplate<typename T>\nMGPU_DEVICE T ldg(const T* p) {\n\treturn LdgShim<T>::Ldg(p);\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n// Fast division for 31-bit integers.\n// Uses the method in Hacker's Delight (2nd edition) page 228.\n// Evaluates for denom > 1 and x < 2^31.\nstruct FastDivide {\n\tuint denom;\n\tuint coef;\n\tuint shift;\n\n\tMGPU_HOST_DEVICE uint Divide(uint x) {\n\t\treturn umulhi(x, coef)>> shift;\n\t}\n\tMGPU_HOST_DEVICE uint Modulus(uint x) {\n\t\treturn x - Divide(x) * denom;\n\t}\n\n\texplicit FastDivide(uint denom_) {\n\t\tdenom = denom_;\n\t\tuint p = 31 + FindLog2(denom, true);\n\t\tcoef = (uint)(((1ull<< p) + denom - 1) / denom);\n\t\tshift = p - 32;\n\t}\n};\n\n#pragma GCC diagnostic pop\n\n} // namespace mgpu\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/device/loadstore.cuh",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#pragma once\n\n#include \"../mgpudevice.cuh\"\n#include \"deviceutil.cuh\"\n#include \"intrinsics.cuh\"\n\nnamespace mgpu {\n\n////////////////////////////////////////////////////////////////////////////////\n// Cooperative load functions.\n\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceSharedToReg(InputIt data, int tid, T* reg,\n\tbool sync) {\n\n\t#pragma unroll\n\tfor(int i = 0; i < VT; ++i)\n\t\treg[i] = data[NT * i + tid];\n\n\tif(sync) __syncthreads();\n}\n\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToRegPred(int count, InputIt data, int tid,\n\tT* reg, bool sync) {\n\n\t// TODO: Attempt to issue 4 loads at a time.\n\t#pragma unroll\n\tfor(int i = 0; i < VT; ++i) {\n\t\tint index = NT * i + tid;\n\t\tif(index < count) reg[i] = data[index];\n\t}\n\tif(sync) __syncthreads();\n}\n\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToReg(int count, InputIt data, int tid,\n\tT* reg, bool sync) {\n\n\tif(count >= NT * VT) {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i)\n\t\t\treg[i] = data[NT * i + tid];\n\t} else\n\t\tDeviceGlobalToRegPred<NT, VT>(count, data, tid, reg, false);\n\tif(sync) __syncthreads();\n}\ntemplate<int NT, int VT0, int VT1, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToReg2(int count, InputIt data, int tid,\n\tT* reg, bool sync) {\n\n\tDeviceGlobalToReg<NT, VT0>(count, data, tid, reg, false);\n\t#pragma unroll\n\tfor(int i = VT0; i < VT1; ++i) {\n\t\tint index = NT * i + tid;\n\t\tif(index < count) reg[i] = data[index];\n\t}\n\tif(sync) __syncthreads();\n}\n\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToRegDefault(int count, InputIt data, int tid,\n\tT* reg, T init, bool sync) {\n\n\tif(count >= NT * VT) {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i)\n\t\t\treg[i] = data[NT * i + tid];\n\t} else {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i) {\n\t\t\tint index = NT * i + tid;\n\t\t\treg[i] = init;\n\t\t\tif(index < count) reg[i] = data[index];\n\t\t}\n\t}\n\tif(sync) __syncthreads();\n}\ntemplate<int NT, int VT0, int VT1, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToRegDefault2(int count, InputIt data, int tid,\n\tT* reg, T init, bool sync) {\n\n\tDeviceGlobalToRegDefault<NT, VT0>(count, data, tid, reg, init, false);\n\t#pragma unroll\n\tfor(int i = VT0; i < VT1; ++i) {\n\t\tint index = NT * i + tid;\n\t\treg[i] = init;\n\t\tif(index < count) reg[i] = data[index];\n\t}\n\tif(sync) __syncthreads();\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToThread(int count, InputIt data, int tid,\n\tT* reg) {\n\n\tdata += VT * tid;\n\tif(count >= NT * VT) {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i)\n\t\t\treg[i] = ldg(data + i);\n\t} else {\n\t\tcount -= VT * tid;\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i)\n\t\t\tif(i < count) reg[i] = ldg(data + i);\n\t}\n}\n\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToThreadDefault(int count, InputIt data, int tid,\n\tT* reg, T init) {\n\n\tdata += VT * tid;\n\tif(count >= NT * VT) {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i)\n\t\t\treg[i] = ldg(data + i);\n\t} else {\n\t\tcount -= VT * tid;\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i)\n\t\t\treg[i] = (i < count) ? ldg(data + i) : init;\n\t}\n}\n\n\n////////////////////////////////////////////////////////////////////////////////\n// Cooperative store functions.\n\ntemplate<int NT, int VT, typename OutputIt, typename T>\nMGPU_DEVICE void DeviceRegToShared(const T* reg, int tid,\n\tOutputIt dest, bool sync) {\n\n\ttypedef typename std::iterator_traits<OutputIt>::value_type T2;\n\t#pragma unroll\n\tfor(int i = 0; i < VT; ++i)\n\t\tdest[NT * i + tid] = (T2)reg[i];\n\n\tif(sync) __syncthreads();\n}\n\ntemplate<int NT, int VT, typename OutputIt, typename T>\nMGPU_DEVICE void DeviceRegToGlobal(int count, const T* reg, int tid,\n\tOutputIt dest, bool sync) {\n\n\t#pragma unroll\n\tfor(int i = 0; i < VT; ++i) {\n\t\tint index = NT * i + tid;\n\t\tif(index < count)\n\t\t\tdest[index] = reg[i];\n\t}\n\tif(sync) __syncthreads();\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// DeviceMemToMemLoop\n// Transfer from shared memory to global, or global to shared, for transfers\n// that are smaller than NT * VT in the average case. The goal is to reduce\n// unnecessary comparison logic.\n\ntemplate<int NT, int VT, typename InputIt, typename OutputIt>\nMGPU_DEVICE void DeviceMemToMem4(int count, InputIt source, int tid,\n\tOutputIt dest, bool sync) {\n\n\ttypedef typename std::iterator_traits<InputIt>::value_type T;\n\n\tT x[VT];\n\tconst int Count = (VT < 4) ? VT : 4;\n\tif(count >= NT * VT) {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < Count; ++i)\n\t\t\tx[i] = source[NT * i + tid];\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < Count; ++i)\n\t\t\tdest[NT * i + tid] = x[i];\n\t} else {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < Count; ++i) {\n\t\t\tint index = NT * i + tid;\n\t\t\tif(index < count)\n\t\t\t\tx[i] = source[NT * i + tid];\n\t\t}\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < Count; ++i) {\n\t\t\tint index = NT * i + tid;\n\t\t\tif(index < count)\n\t\t\t\tdest[index] = x[i];\n\t\t}\n\t}\n\tif(sync) __syncthreads();\n}\ntemplate<int NT, typename InputIt, typename OutputIt>\nMGPU_DEVICE void DeviceMemToMemLoop(int count, InputIt source, int tid,\n\tOutputIt dest, bool sync) {\n\n\tfor(int i = 0; i < count; i += 4 * NT)\n\t\tDeviceMemToMem4<NT, 4>(count - i, source + i, tid, dest + i,\n\t\t\tfalse);\n\tif(sync) __syncthreads();\n}\n\n\n////////////////////////////////////////////////////////////////////////////////\n// Functions to copy between shared and global memory where the average case is\n// to transfer NT * VT elements.\n\ntemplate<int NT, int VT, typename T, typename OutputIt>\nMGPU_DEVICE void DeviceSharedToGlobal(int count, const T* source, int tid,\n\tOutputIt dest, bool sync) {\n\n\ttypedef typename std::iterator_traits<OutputIt>::value_type T2;\n\t#pragma unroll\n\tfor(int i = 0; i < VT; ++i) {\n\t\tint index = NT * i + tid;\n\t\tif(index < count) dest[index] = (T2)source[index];\n\t}\n\tif(sync) __syncthreads();\n}\n\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToShared(int count, InputIt source, int tid,\n\tT* dest, bool sync) {\n\n\tT reg[VT];\n\tDeviceGlobalToReg<NT, VT>(count, source, tid, reg, false);\n\tDeviceRegToShared<NT, VT>(reg, tid, dest, sync);\n}\n\ntemplate<int NT, int VT0, int VT1, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToShared2(int count, InputIt source, int tid,\n\tT* dest, bool sync) {\n\n\tT reg[VT1];\n\tDeviceGlobalToReg2<NT, VT0, VT1>(count, source, tid, reg, false);\n\tDeviceRegToShared<NT, VT1>(reg, tid, dest, sync);\n}\n\n\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToSharedDefault(int count, InputIt source, int tid,\n\tT* dest, T init, bool sync) {\n\n\tT reg[VT];\n\tDeviceGlobalToRegDefault<NT, VT>(count, source, tid, reg, init, false);\n\tDeviceRegToShared<NT, VT>(reg, tid, dest, sync);\n}\n\ntemplate<int NT, int VT0, int VT1, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToSharedDefault2(int count, InputIt data, int tid,\n\tT* dest, T init, bool sync) {\n\n\tT reg[VT1];\n\tDeviceGlobalToRegDefault2<NT, VT0, VT1>(count, data, tid, reg, init, false);\n\tDeviceRegToShared<NT, VT1>(reg, tid, dest, sync);\n}\n\n\n////////////////////////////////////////////////////////////////////////////////\n\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToSharedLoop(int count, InputIt source, int tid,\n\tT* dest, bool sync) {\n\n\tconst int Granularity = MGPU_MIN(VT, 3);\n\tDeviceGlobalToShared<NT, Granularity>(count, source, tid, dest, false);\n\n\tint offset = Granularity * NT;\n\tif(count > offset)\n\t\tDeviceGlobalToShared<NT, VT - Granularity>(count - offset,\n\t\t\tsource + offset, tid, dest + offset, false);\n\n\tif(sync) __syncthreads();\n\n\t/*\n\tsource += tid;\n\twhile(count > 0) {\n\t\tT reg[Granularity];\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < Granularity; ++i) {\n\t\t\tint index = NT * i + tid;\n\t\t\tif(index < count)\n\t\t\t\treg[i] = source[NT * i];\n\t\t}\n\t\tDeviceRegToShared<NT, Granularity>(reg, tid, dest, false);\n\t\tsource += Granularity * NT;\n\t\tdest += Granularity * NT;\n\t\tcount -= Granularity * NT;\n\t}\n\tif(sync) __syncthreads();*/\n}\n\ntemplate<int NT, int VT, typename InputIt, typename OutputIt>\nMGPU_DEVICE void DeviceGlobalToGlobal(int count, InputIt source, int tid,\n\tOutputIt dest, bool sync) {\n\n\ttypedef typename std::iterator_traits<OutputIt>::value_type T;\n\tT values[VT];\n\tDeviceGlobalToReg<NT, VT>(count, source, tid, values, false);\n\tDeviceRegToGlobal<NT, VT>(count, values, tid, dest, sync);\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// Transponse VT elements in NT threads (x) into thread-order registers (y)\n// using only NT * VT / 2 elements of shared memory.\n\n//This function definitely has a bug, don't use!!! fix TODO(erich)\ntemplate<int NT, int VT, typename T>\nMGPU_DEVICE void HalfSmemTranspose(const T* x, int tid, T* shared, T* y) {\n    printf(\"HalfSmemTranspose has a bug, use WAR SmemTranpose or find bug before using in production\");\n\t// Transpose the first half values (tid < NT / 2)\n\t#pragma unroll\n\tfor(int i = 0; i <= VT / 2; ++i)\n\t\tif(i < VT / 2 || tid < NT / 2)\n\t\t\tshared[NT * i + tid] = x[i];\n\t__syncthreads();\n\n\tif(tid < NT / 2) {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i)\n\t\t\ty[i] = shared[VT * tid + i];\n\t}\n\t__syncthreads();\n\n\t// Transpose the second half values (tid >= NT / 2)\n\t#pragma unroll\n\tfor(int i = VT / 2; i < VT; ++i)\n\t\tif(i > VT / 2 || tid >= NT / 2)\n\t\t\tshared[NT * i - NT * VT / 2 + tid] = x[i];\n\t__syncthreads();\n\n\tif(tid >= NT / 2) {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i)\n\t\t\ty[i] = shared[VT * tid + i - NT * VT / 2];\n\t}\n\t__syncthreads();\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// Gather/scatter functions\n\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGather(int count, InputIt data, int indices[VT],\n\tint tid, T* reg, bool sync) {\n\n\tif(count >= NT * VT) {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i)\n\t\t\treg[i] = data[indices[i]];\n\t} else {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i) {\n\t\t\tint index = NT * i + tid;\n\t\t\tif(index < count)\n\t\t\t\treg[i] = data[indices[i]];\n\t\t}\n\t}\n\tif(sync) __syncthreads();\n}\n\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGatherDefault(int count, InputIt data, int indices[VT],\n\tint tid, T* reg, T identity, bool sync) {\n\n\tif(count >= NT * VT) {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i)\n\t\t\treg[i] = data[indices[i]];\n\t} else {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i) {\n\t\t\tint index = NT * i + tid;\n\t\t\treg[i] = (index < count) ? data[indices[i]] : identity;\n\t\t}\n\t}\n\tif(sync) __syncthreads();\n}\n\ntemplate<int NT, int VT, typename T, typename OutputIt>\nMGPU_DEVICE void DeviceScatter(int count, const T* reg, int tid,\n\tint indices[VT], OutputIt data, bool sync) {\n\n\tif(count >= NT * VT) {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i)\n\t\t\tdata[indices[i]] = reg[i];\n\t} else {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i) {\n\t\t\tint index = NT * i + tid;\n\t\t\tif(index < count)\n\t\t\t\tdata[indices[i]] = reg[i];\n\t\t}\n\t}\n\tif(sync) __syncthreads();\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// Cooperative transpose functions (strided to thread order)\n\ntemplate<int VT, typename T>\nMGPU_DEVICE void DeviceThreadToShared(const T* threadReg, int tid, T* shared,\n\tbool sync) {\n\n\tif(1 & VT) {\n\t\t// Odd grain size. Store as type T.\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i)\n\t\t\tshared[VT * tid + i] = threadReg[i];\n\t} else {\n\t\t// Even grain size. Store as DevicePair<T>. This lets us exploit the\n\t\t// 8-byte shared memory mode on Kepler.\n\t\tDevicePair<T>* dest = (DevicePair<T>*)(shared + VT * tid);\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT / 2; ++i)\n\t\t\tdest[i] = MakeDevicePair(threadReg[2 * i], threadReg[2 * i + 1]);\n\t}\n\tif(sync) __syncthreads();\n}\n\ntemplate<int VT, typename T>\nMGPU_DEVICE void DeviceSharedToThread(const T* shared, int tid, T* threadReg,\n\tbool sync) {\n\n\tif(1 & VT) {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i)\n\t\t\tthreadReg[i] = shared[VT * tid + i];\n\t} else {\n\t\tconst DevicePair<T>* source = (const DevicePair<T>*)(shared + VT * tid);\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT / 2; ++i) {\n\t\t\tDevicePair<T> p = source[i];\n\t\t\tthreadReg[2 * i] = p.x;\n\t\t\tthreadReg[2 * i + 1] = p.y;\n\t\t}\n\t}\n\tif(sync) __syncthreads();\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// DeviceLoad2 - load from pointers of the same type. Optimize for a single LD\n// statement.\n\ntemplate<int NT, int VT0, int VT1, typename T>\nMGPU_DEVICE void DeviceLoad2ToReg(const T* a_global, int aCount,\n\tconst T* b_global, int bCount, int tid, T* reg, bool sync) {\n\n\tint b0 = b_global - a_global - aCount;\n\tint total = aCount + bCount;\n\tif(total >= NT * VT0) {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT0; ++i) {\n\t\t\tint index = NT * i + tid;\n\t\t\treg[i] = a_global[index + ((index >= aCount) ? b0 : 0)];\n\t\t}\n\t} else {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT0; ++i) {\n\t\t\tint index = NT * i + tid;\n\t\t\tif(index < total)\n\t\t\t\treg[i] = a_global[index + ((index >= aCount) ? b0 : 0)];\n\t\t}\n\t}\n\t#pragma unroll\n\tfor(int i = VT0; i < VT1; ++i) {\n\t\tint index = NT * i + tid;\n\t\tif(index < total)\n\t\t\treg[i] = a_global[index + ((index >= aCount) ? b0 : 0)];\n\t}\n}\n\ntemplate<int NT, int VT0, int VT1, typename T>\nMGPU_DEVICE void DeviceLoad2ToShared(const T* a_global, int aCount,\n\tconst T* b_global, int bCount, int tid, T* shared, bool sync) {\n\n\tT reg[VT1];\n\tDeviceLoad2ToReg<NT, VT0, VT1>(a_global, aCount, b_global, bCount, tid,\n\t\treg, false);\n\tDeviceRegToShared<NT, VT1>(reg, tid, shared, sync);\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// DeviceLoad2 - load from pointers of different types. Uses two LD statements.\n\ntemplate<int NT, int VT0, int VT1, typename InputIt1, typename InputIt2,\n\ttypename T>\nMGPU_DEVICE void DeviceLoad2ToReg(InputIt1 a_global, int aCount,\n\tInputIt2 b_global, int bCount, int tid, T* reg, bool sync)  {\n\n\tb_global -= aCount;\n\tint total = aCount + bCount;\n\tif(total >= NT * VT0) {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT0; ++i) {\n\t\t\tint index = NT * i + tid;\n\t\t\tif(index < aCount) reg[i] = a_global[index];\n\t\t\telse reg[i] = b_global[index];\n\t\t}\n\t} else {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT0; ++i) {\n\t\t\tint index = NT * i + tid;\n\t\t\tif(index < aCount) reg[i] = a_global[index];\n\t\t\telse if(index < total) reg[i] = b_global[index];\n\t\t}\n\t}\n\t#pragma unroll\n\tfor(int i = VT0; i < VT1; ++i) {\n\t\tint index = NT * i + tid;\n\t\tif(index < aCount) reg[i] = a_global[index];\n\t\telse if(index < total) reg[i] = b_global[index];\n\t}\n}\n\ntemplate<int NT, int VT0, int VT1, typename InputIt1, typename InputIt2,\n\ttypename T>\nMGPU_DEVICE void DeviceLoad2ToShared(InputIt1 a_global, int aCount,\n\tInputIt2 b_global, int bCount, int tid, T* shared, bool sync) {\n\n\tT reg[VT1];\n\tDeviceLoad2ToReg<NT, VT0, VT1>(a_global, aCount, b_global, bCount, tid,\n\t\treg, false);\n\tDeviceRegToShared<NT, VT1>(reg, tid, shared, sync);\n}\n\n\n////////////////////////////////////////////////////////////////////////////////\n// DeviceGatherGlobalToGlobal\n\ntemplate<int NT, int VT, typename InputIt, typename OutputIt>\nMGPU_DEVICE void DeviceGatherGlobalToGlobal(int count, InputIt data_global,\n\tconst int* indices_shared, int tid, OutputIt dest_global, bool sync) {\n\n\ttypedef typename std::iterator_traits<InputIt>::value_type ValType;\n\tValType values[VT];\n\n\t#pragma unroll\n\tfor(int i = 0; i < VT; ++i) {\n\t\tint index = NT * i + tid;\n\t\tif(index < count) {\n\t\t\tint gather = indices_shared[index];\n\t\t\tvalues[i] = data_global[gather];\n\t\t}\n\t}\n\tif(sync) __syncthreads();\n\tDeviceRegToGlobal<NT, VT>(count, values, tid, dest_global, false);\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// DeviceTransferMergeValues\n// Gather in a merge-like value from two input arrays and store to a single\n// output. Like DeviceGatherGlobalToGlobal, but for two arrays at once.\n\ntemplate<int NT, int VT, typename InputIt1, typename InputIt2,\n\ttypename T>\nMGPU_DEVICE void DeviceTransferMergeValuesReg(int count, InputIt1 a_global,\n\tInputIt2 b_global, int bStart, const int* indices, int tid,\n\tT* reg, bool sync) {\n\n\tb_global -= bStart;\n\tif(count >= NT * VT) {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i) {\n\t\t\treg[i] = (indices[i] < bStart) ? a_global[indices[i]] :\n\t\t\t\tb_global[indices[i]];\n\t\t}\n\t} else {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i) {\n\t\t\tint index = NT * i + tid;\n\t\t\tif(index < count)\n\t\t\t\treg[i] = (indices[i] < bStart) ? a_global[indices[i]] :\n\t\t\t\t\tb_global[indices[i]];\n\t\t}\n\t}\n\tif(sync) __syncthreads();\n}\n\ntemplate<int NT, int VT, typename InputIt1, typename InputIt2,\n\ttypename OutputIt>\nMGPU_DEVICE void DeviceTransferMergeValuesShared(int count, InputIt1 a_global,\n\tInputIt2 b_global, int bStart, const int* indices_shared, int tid,\n\tOutputIt dest_global, bool sync) {\n\n\tint indices[VT];\n\tDeviceSharedToReg<NT, VT>(indices_shared, tid, indices);\n\n\ttypedef typename std::iterator_traits<InputIt1>::value_type ValType;\n\tValType reg[VT];\n\tDeviceTransferMergeValuesReg<NT, VT>(count, a_global, b_global, bStart,\n\t\tindices, tid, reg, sync);\n\tDeviceRegToGlobal<NT, VT>(count, reg, tid, dest_global, sync);\n}\n\ntemplate<int NT, int VT, typename T>\nMGPU_DEVICE void DeviceTransferMergeValuesReg(int count, const T* a_global,\n\tconst T* b_global, int bStart, const int* indices, int tid, T* reg,\n\tbool sync) {\n\n\tint bOffset = (int)(b_global - a_global - bStart);\n\n\tif(count >= NT * VT) {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i) {\n\t\t\tint gather = indices[i];\n\t\t\tif(gather >= bStart) gather += bOffset;\n\t\t\treg[i] = a_global[gather];\n\t\t}\n\t} else {\n\t\t#pragma unroll\n\t\tfor(int i = 0; i < VT; ++i) {\n\t\t\tint index = NT * i + tid;\n\t\t\tint gather = indices[i];\n\t\t\tif(gather >= bStart) gather += bOffset;\n\t\t\tif(index < count)\n\t\t\t\treg[i] = a_global[gather];\n\t\t}\n\t}\n\tif(sync) __syncthreads();\n}\n\ntemplate<int NT, int VT, typename T, typename OutputIt>\nMGPU_DEVICE void DeviceTransferMergeValuesShared(int count, const T* a_global,\n\tconst T* b_global, int bStart, const int* indices_shared, int tid,\n\tOutputIt dest_global, bool sync) {\n\n\tint indices[VT];\n\tDeviceSharedToReg<NT, VT>(indices_shared, tid, indices);\n\n\tT reg[VT];\n\tDeviceTransferMergeValuesReg<NT, VT>(count, a_global, b_global, bStart,\n\t\tindices, tid, reg, sync);\n\tDeviceRegToGlobal<NT, VT>(count, reg, tid, dest_global, sync);\n}\n\n} // namespace mgpu\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/device/serialsets.cuh",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#pragma once\n\n#include \"deviceutil.cuh\"\n\nnamespace mgpu {\n\n////////////////////////////////////////////////////////////////////////////////\n// SerialSetIntersection\n// Emit A if A and B are in range and equal.\n\ntemplate<int VT, bool RangeCheck, typename T, typename Comp>\nMGPU_DEVICE int SerialSetIntersection(const T* data, int aBegin, int aEnd,\n\tint bBegin, int bEnd, int end, T* results, int* indices, Comp comp) {\n\n\tconst int MinIterations = VT / 2;\n\tint commit = 0;\n\n\t#pragma unroll\n\tfor(int i = 0; i < VT; ++i) {\n\t\tbool test = RangeCheck ?\n\t\t\t((aBegin + bBegin < end) && (aBegin < aEnd) && (bBegin < bEnd)) :\n\t\t\t(i < MinIterations || (aBegin + bBegin < end));\n\n\t\tif(test) {\n\t\t\tT aKey = data[aBegin];\n\t\t\tT bKey = data[bBegin];\n\n\t\t\tbool pA = comp(aKey, bKey);\n\t\t\tbool pB = comp(bKey, aKey);\n\n\t\t\t// The outputs must come from A by definition of set interection.\n\t\t\tresults[i] = aKey;\n\t\t\tindices[i] = aBegin;\n\n\t\t\tif(!pB) ++aBegin;\n\t\t\tif(!pA) ++bBegin;\n\t\t\tif(pA == pB) commit |= 1<< i;\n\t\t}\n\t}\n\treturn commit;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// SerialSetUnion\n// Emit A if A <= B. Emit B if B < A.\n\ntemplate<int VT, bool RangeCheck, typename T, typename Comp>\nMGPU_DEVICE int SerialSetUnion(const T* data, int aBegin, int aEnd,\n\tint bBegin, int bEnd, int end, T* results, int* indices, Comp comp) {\n\n\tconst int MinIterations = VT / 2;\n\tint commit = 0;\n\n\t#pragma unroll\n\tfor(int i = 0; i < VT; ++i) {\n\t\tbool test = RangeCheck ?\n\t\t\t(aBegin + bBegin < end) :\n\t\t\t(i < MinIterations || (aBegin + bBegin < end));\n\n\t\tif(test) {\n\t\t\tT aKey = data[aBegin];\n\t\t\tT bKey = data[bBegin];\n\n\t\t\tbool pA = false, pB = false;\n\t\t\tif(RangeCheck && aBegin >= aEnd)\n\t\t\t\tpB = true;\n\t\t\telse if(RangeCheck && bBegin >= bEnd)\n\t\t\t\tpA = true;\n\t\t\telse {\n\t\t\t\t// Both are in range.\n\t\t\t\tpA = comp(aKey, bKey);\n\t\t\t\tpB = comp(bKey, aKey);\n\t\t\t}\n\n\t\t\t// Output A in case of a tie, so check if b < a.\n\t\t\tresults[i] = pB ? bKey : aKey;\n\t\t\tindices[i] = pB ? bBegin : aBegin;\n\t\t\tif(!pB) ++aBegin;\n\t\t\tif(!pA) ++bBegin;\n\t\t\tcommit |= 1<< i;\n\t\t}\n\t}\n\treturn commit;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// SerialSetDifference\n// Emit A if A < B.\n\ntemplate<int VT, bool RangeCheck, typename T, typename Comp>\nMGPU_DEVICE int SerialSetDifference(const T* data, int aBegin, int aEnd,\n\tint bBegin, int bEnd, int end, T* results, int* indices, Comp comp) {\n\n\tconst int MinIterations = VT / 2;\n\tint commit = 0;\n\n\t#pragma unroll\n\tfor(int i = 0; i < VT; ++i) {\n\t\tbool test = RangeCheck ?\n\t\t\t(aBegin + bBegin < end) :\n\t\t\t(i < MinIterations || (aBegin + bBegin < end));\n\t\tif(test) {\n\t\t\tT aKey = data[aBegin];\n\t\t\tT bKey = data[bBegin];\n\n\t\t\tbool pA = false, pB = false;\n\t\t\tif(RangeCheck && aBegin >= aEnd)\n\t\t\t\tpB = true;\n\t\t\telse if(RangeCheck && bBegin >= bEnd)\n\t\t\t\tpA = true;\n\t\t\telse {\n\t\t\t\tpA = comp(aKey, bKey);\n\t\t\t\tpB = comp(bKey, aKey);\n\t\t\t}\n\n\t\t\t// The outputs must come from A by definition of set difference.\n\t\t\tresults[i] = aKey;\n\t\t\tindices[i] = aBegin;\n\t\t\tif(!pB) ++aBegin;\n\t\t\tif(!pA) ++bBegin;\n\t\t\tif(pA) commit |= 1<< i;\n\t\t}\n\t}\n\treturn commit;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// SerialSetSymDiff\n// Emit A if A < B and emit B if B < A.\n\ntemplate<int VT, bool RangeCheck, typename T, typename Comp>\nMGPU_DEVICE int SerialSetSymDiff(const T* data, int aBegin, int aEnd,\n\tint bBegin, int bEnd, int end, T* results, int* indices, Comp comp) {\n\n\tconst int MinIterations = VT / 2;\n\tint commit = 0;\n\n\t#pragma unroll\n\tfor(int i = 0; i < VT; ++i) {\n\t\tbool test = RangeCheck ?\n\t\t\t(aBegin + bBegin < end) :\n\t\t\t(i < MinIterations || (aBegin + bBegin < end));\n\t\tif(test) {\n\t\t\tT aKey = data[aBegin];\n\t\t\tT bKey = data[bBegin];\n\n\t\t\tbool pA = false, pB = false;\n\t\t\tif(RangeCheck && (bBegin >= bEnd))\n\t\t\t\tpA = true;\n\t\t\telse if(RangeCheck && (aBegin >= aEnd))\n\t\t\t\tpB = true;\n\t\t\telse {\n\t\t\t\tpA = comp(aKey, bKey);\n\t\t\t\tpB = comp(bKey, aKey);\n\t\t\t}\n\n\t\t\tresults[i] = pA ? aKey : bKey;\n\t\t\tindices[i] = pA ? aBegin : bBegin;\n\t\t\tif(!pA) ++bBegin;\n\t\t\tif(!pB) ++aBegin;\n\t\t\tif(pA != pB) commit |= 1<< i;\n\t\t}\n\t}\n\treturn commit;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// SerialSetOp\n// Uses the MgpuSetOp enum to statically select one of the four serial ops\n// above.\n\ntemplate<int VT, bool RangeCheck, MgpuSetOp Op, typename T, typename Comp>\nMGPU_DEVICE int SerialSetOp(const T* data, int aBegin, int aEnd,\n\tint bBegin, int bEnd, int star, T* results, int* indices, Comp comp) {\n\n\tint end = aBegin + bBegin + VT - star;\n\tif(RangeCheck) end = min(end, aEnd + bEnd);\n\tint commit;\n\tswitch(Op) {\n\t\tcase MgpuSetOpIntersection:\n\t\t\tcommit = SerialSetIntersection<VT, RangeCheck>(data, aBegin,\n\t\t\t\taEnd, bBegin, bEnd, end, results, indices, comp);\n\t\t\tbreak;\n\t\tcase MgpuSetOpUnion:\n\t\t\tcommit = SerialSetUnion<VT, RangeCheck>(data, aBegin, aEnd,\n\t\t\t\tbBegin, bEnd, end, results, indices, comp);\n\t\t\tbreak;\n\t\tcase MgpuSetOpDiff:\n\t\t\tcommit = SerialSetDifference<VT, RangeCheck>(data, aBegin, aEnd,\n\t\t\t\tbBegin, bEnd, end, results, indices, comp);\n\t\t\tbreak;\n\t\tcase MgpuSetOpSymDiff:\n\t\t\tcommit = SerialSetSymDiff<VT, RangeCheck>(data, aBegin, aEnd,\n\t\t\t\tbBegin, bEnd, end, results, indices, comp);\n\t\t\tbreak;\n\t}\n\t__syncthreads();\n\treturn commit;\n}\n\n} // namespace mgpu\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/device/sortnetwork.cuh",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#pragma once\n\n#include \"deviceutil.cuh\"\n\nnamespace mgpu {\n\n////////////////////////////////////////////////////////////////////////////////\n// Odd-even transposition sorting network. Sorts keys and values in-place in\n// register.\n// http://en.wikipedia.org/wiki/Odd%E2%80%93even_sort\n\n// CUDA Compiler does not currently unroll these loops correctly. Write using\n// template loop unrolling.\n/*\ntemplate<int VT, typename T, typename V, typename Comp>\nMGPU_DEVICE void OddEvenTransposeSort(T* keys, V* values, Comp comp) {\n\t#pragma unroll\n\tfor(int level = 0; level < VT; ++level) {\n\n\t\t#pragma unroll\n\t\tfor(int i = 1 & level; i < VT - 1; i += 2) {\n\t\t\tif(comp(keys[i + 1], keys[i])) {\n\t\t\t\tmgpu::swap(keys[i], keys[i + 1]);\n\t\t\t\tmgpu::swap(values[i], values[i + 1]);\n\t\t\t}\n\t\t}\n\t}\n}*/\n\ntemplate<int I, int VT>\nstruct OddEvenTransposeSortT {\n\t// Sort segments marked by head flags. If the head flag between i and i + 1\n\t// is set (so that (2<< i) & flags is true), the values belong to different\n\t// segments and are not swapped.\n\ttemplate<typename K, typename V, typename Comp>\n\tstatic MGPU_DEVICE void Sort(K* keys, V* values, int flags, Comp comp) {\n\t\t#pragma unroll\n\t\tfor(int i = 1 & I; i < VT - 1; i += 2)\n\t\t\tif((0 == ((2<< i) & flags)) && comp(keys[i + 1], keys[i])) {\n\t\t\t\tmgpu::swap(keys[i], keys[i + 1]);\n\t\t\t\tmgpu::swap(values[i], values[i + 1]);\n\t\t\t}\n\t\tOddEvenTransposeSortT<I + 1, VT>::Sort(keys, values, flags, comp);\n\t}\n};\ntemplate<int I> struct OddEvenTransposeSortT<I, I> {\n\ttemplate<typename K, typename V, typename Comp>\n\tstatic MGPU_DEVICE void Sort(K* keys, V* values, int flags, Comp comp) { }\n};\n\ntemplate<int VT, typename K, typename V, typename Comp>\nMGPU_DEVICE void OddEvenTransposeSort(K* keys, V* values, Comp comp) {\n\tOddEvenTransposeSortT<0, VT>::Sort(keys, values, 0, comp);\n}\ntemplate<int VT, typename K, typename V, typename Comp>\nMGPU_DEVICE void OddEvenTransposeSortFlags(K* keys, V* values, int flags,\n\tComp comp) {\n\tOddEvenTransposeSortT<0, VT>::Sort(keys, values, flags, comp);\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// Batcher Odd-Even Mergesort network\n// Unstable but executes much faster than the transposition sort.\n// http://en.wikipedia.org/wiki/Batcher_odd%E2%80%93even_mergesort\n\ntemplate<int Width, int Low, int Count>\nstruct OddEvenMergesortT {\n\ttemplate<typename K, typename V, typename Comp>\n\tMGPU_DEVICE static void CompareAndSwap(K* keys, V* values, int flags,\n\t\tint a, int b, Comp comp) {\n\t\tif(b < Count) {\n\t\t\t// Mask the bits between a and b. Any head flags in this interval\n\t\t\t// means the keys are in different segments and must not be swapped.\n\t\t\tconst int Mask = ((2<< b) - 1) ^ ((2<< a) - 1);\n\t\t\tif(!(Mask & flags) && comp(keys[b], keys[a])) {\n\t\t\t\tmgpu::swap(keys[b], keys[a]);\n\t\t\t\tmgpu::swap(values[b], values[a]);\n\t\t\t}\n\t\t}\n\t}\n\n\ttemplate<int R, int Low2, bool Recurse = 2 * R < Width>\n\tstruct OddEvenMerge {\n\t\ttemplate<typename K, typename V, typename Comp>\n\t\tMGPU_DEVICE static void Merge(K* keys, V* values, int flags,\n\t\t\tComp comp) {\n\t\t\t// Compare and swap\n\t\t\tconst int M = 2 * R;\n\t\t\tOddEvenMerge<M, Low2>::Merge(keys, values, flags, comp);\n\t\t\tOddEvenMerge<M, Low2 + R>::Merge(keys, values, flags, comp);\n\n\t\t\t#pragma unroll\n\t\t\tfor(int i = Low2 + R; i + R < Low2 + Width; i += M)\n\t\t\t\tCompareAndSwap(keys, values, flags, i, i + R, comp);\n\t\t}\n\t};\n\ttemplate<int R, int Low2>\n\tstruct OddEvenMerge<R, Low2, false> {\n\t\ttemplate<typename K, typename V, typename Comp>\n\t\tMGPU_DEVICE static void Merge(K* keys, V* values, int flags,\n\t\t\tComp comp) {\n\t\t\tCompareAndSwap(keys, values, flags, Low2, Low2 + R, comp);\n\t\t}\n\t};\n\n\ttemplate<typename K, typename V, typename Comp>\n\tMGPU_DEVICE static void Sort(K* keys, V* values, int flags,\n\t\tComp comp) {\n\n\t\tconst int M = Width / 2;\n\t\tOddEvenMergesortT<M, Low, Count>::Sort(keys, values, flags, comp);\n\t\tOddEvenMergesortT<M, Low + M, Count>::Sort(keys, values, flags, comp);\n\t\tOddEvenMerge<1, Low>::Merge(keys, values, flags, comp);\n\t}\n};\ntemplate<int Low, int Count> struct OddEvenMergesortT<1, Low, Count> {\n\ttemplate<typename K, typename V, typename Comp>\n\tMGPU_DEVICE static void Sort(K* keys, V* values, int flags,\n\t\tComp comp) { }\n};\n\ntemplate<int VT, typename K, typename V, typename Comp>\nMGPU_DEVICE void OddEvenMergesort(K* keys, V* values, Comp comp) {\n\tconst int Width = 1<< sLogPow2<VT, true>::value;\n\tOddEvenMergesortT<Width, 0, VT>::Sort(keys, values, 0, comp);\n}\ntemplate<int VT, typename K, typename V, typename Comp>\nMGPU_DEVICE void OddEvenMergesortFlags(K* keys, V* values, int flags,\n\tComp comp) {\n\tconst int Width = 1<< sLogPow2<VT, true>::value;\n\tOddEvenMergesortT<Width, 0, VT>::Sort(keys, values, flags, comp);\n}\n\n} // namespace mgpu\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/mgpudevice.cuh",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#pragma once\n\n#include \"mgpuenums.h\"\n#include \"device/deviceutil.cuh\"\n\nnamespace mgpu {\n\n////////////////////////////////////////////////////////////////////////////////\n// device/loadstore.cuh\n\n// For 0 <= i < VT:\n//\t\tindex = NT * i + tid;\n//\t\treg[i] = data[index];\n// Synchronize after load.\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceSharedToReg(InputIt data, int tid, T* reg,\n\tbool sync = true);\n\n// For 0 <= i < VT:\n//\t\tindex = NT * i + tid;\n//\t\tif(index < count) reg[i] = data[index];\n// No synchronize after load.\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToReg(int count, InputIt data, int tid,\n\tT* reg, bool sync = false);\n\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToRegDefault(int count, InputIt data, int tid,\n\tT* reg, T init, bool sync = false);\n\n// For 0 <= i < VT:\n//\t\tindex = NT * i + tid;\n//\t\tif(index < count) reg[i] = data[index];\n// No synchronize after load.\ntemplate<int NT, int VT0, int VT1, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToReg(int count, InputIt data, int tid,\n\tT* reg, bool sync = false);\n\n// For 0 <= i < VT:\n//\t\tindex = NT * i + tid;\n//\t\tif(index < count) reg[i] = data[index];\n// No synchronize after load.\ntemplate<int NT, int VT0, int VT1, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToRegDefault2(int count, InputIt data, int tid,\n\tT* reg, T init, bool sync = false);\n\n// For 0 <= i < VT:\n//\t\tindex = NT * i + tid;\n//\t\tif(index < count) reg[i] = data[index];\n// No synchronize after load.\n// No optimized code path for count < NV (smaller generated code).\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToRegLoop(int count, InputIt data, int tid,\n\tT* reg, bool sync = false);\n\n\n// For 0 <= i < VT:\n//\t\tindex = VT * tid + i.\n//\t\tif(index < count) reg[i] = data[index];\n// No synchronize after load.\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToThread(int count, InputIt data, int tid,\n\tT* reg);\n\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToThreadDefault(int count, InputIt data, int tid,\n\tT* reg, T init);\n\n// For 0 <= i < VT:\n//\t\tindex = NT * i + tid;\n//\t\tif(index < count) data[index] = reg[i];\n// Synchronize after load.\ntemplate<int NT, int VT, typename OutputIt, typename T>\nMGPU_DEVICE void DeviceRegToShared(const T* reg, int tid, OutputIt dest,\n\tbool sync = true);\n\n// For 0 <= i < VT:\n//\t\tindex = NT * i + tid;\n//\t\tif(index < count) data[index] = reg[i];\n// No synchronize after load.\ntemplate<int NT, int VT, typename OutputIt, typename T>\nMGPU_DEVICE void DeviceRegToGlobal(int count, const T* reg, int tid,\n\tOutputIt dest, bool sync = false);\n\n// For 0 <= index < count:\n//\t\tdest[index] = source[index];\n// This function is intended to replace DeviceGlobalToShared in cases where\n// count is much less than NT * VT.\ntemplate<int NT, typename InputIt, typename OutputIt>\nMGPU_DEVICE void DeviceMemToMemLoop(int count, InputIt source, int tid,\n\tOutputIt dest, bool sync = true);\n\n// For 0 <= index < count:\n//\t\tdest[index] = source[index];\n// Synchronize after store.\ntemplate<int NT, int VT, typename T, typename OutputIt>\nMGPU_DEVICE void DeviceSharedToGlobal(int count, const T* source, int tid,\n\tOutputIt dest, bool sync = true);\n\n// For 0 <= index < count:\n//\t\tdest[index] = source[index];\n// Synchronize after store.\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToShared(int count, InputIt source, int tid,\n\tT* dest, bool sync = true);\n\ntemplate<int NT, int VT0, int VT1, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToShared2(int count, InputIt source, int tid,\n\tT* dest, bool sync = true);\n\n// For 0 <= index < count:\n//\t\tdest[index] = source[index];\n// Synchronize after store.\n// No optimized code path for count < NV (smaller generated code).\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToSharedLoop(int count, InputIt source, int tid,\n\tT* dest, bool sync = true);\n\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToSharedDefault(int count, InputIt source, int tid,\n\tT* dest, T init, bool sync = true);\n\ntemplate<int NT, int VT0, int VT1, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGlobalToSharedDefault2(int count, InputIt source,\n\tint tid, T* dest, T init, bool sync = true);\n\n// For 0 <= index < count:\n//\t\tdest[index] = source[index];\n// No synchronize.\ntemplate<int NT, int VT, typename InputIt, typename OutputIt>\nMGPU_DEVICE void DeviceGlobalToGlobal(int count, InputIt source, int tid,\n\tOutputIt dest, bool sync = false);\n\n// Transponse VT elements in NT threads (x) into thread-order registers (y)\n// using only NT * VT / 2 elements of shared memory.\ntemplate<int NT, int VT, typename T>\nMGPU_DEVICE void HalfSmemTranspose(const T* x, int tid, T* shared, T* y);\n\n// For 0 <= i < VT:\n//\t\tindex = NT * i + tid;\n//\t\tif(index < count)\n//\t\t\tgather = indices[index];\n//\t\t\treg[i] = data[gather];\n// Synchronize after load.\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGather(int count, InputIt data, int indices[VT],\n\tint tid, T* reg, bool sync = true);\n\ntemplate<int NT, int VT, typename InputIt, typename T>\nMGPU_DEVICE void DeviceGatherDefault(int count, InputIt data, int indices[VT],\n\tint tid, T* reg, T identity, bool sync = true);\n\n// For 0 <= i < VT:\n//\t\tindex = NT * i + tid;\n//\t\tif(index < count)\n//\t\t\tscatter = indices[index];\n//\t\t\tdata[scatter] = reg[i];\n// Synchronize after store.\ntemplate<int NT, int VT, typename T, typename OutputIt>\nMGPU_DEVICE void DeviceScatter(int count, const T* reg, int tid,\n\tint indices[VT], OutputIt data, bool sync = true);\n\n// For 0 <= i < VT:\n//\t\tshared[VT * tid + i] = threadReg[i];\n// Synchronize after store.\n// Note this function moves data in THREAD ORDER.\n// (DeviceRegToShared moves data in STRIDED ORDER).\ntemplate<int VT, typename T>\nMGPU_DEVICE void DeviceThreadToShared(const T* threadReg, int tid, T* shared,\n\tbool sync = true);\n\n// For 0 <= i < VT:\n//\t\tthreadReg[i] = shared[VT * tid + i];\n// Synchronize after load.\n// Note this function moves data in THREAD ORDER.\n// (DeviceSharedToReg moves data in STRIDED ORDER).\ntemplate<int VT, typename T>\nMGPU_DEVICE void DeviceSharedToThread(const T* shared, int tid, T* threadReg,\n\tbool sync = true);\n\n// For 0 <= index < aCount:\n//\t\tshared[index] = a_global[index];\n// For 0 <= index < bCount:\n//\t\tshared[aCount + index] = b_global[index];\n// VT0 is the lower-bound for predication-free execution:\n//\t\tIf count >= NT * VT0, a predication-free branch is taken.\n// VT1 is the upper-bound for loads:\n//\t\tNT * VT1 must >= aCount + bCount.\n\ntemplate<int NT, int VT0, int VT1, typename T>\nMGPU_DEVICE void DeviceLoad2ToReg(const T* a_global, int aCount,\n\tconst T* b_global, int bCount, int tid, T* reg, bool sync = false);\n\ntemplate<int NT, int VT0, int VT1, typename T>\nMGPU_DEVICE void DeviceLoad2ToShared(const T* a_global, int aCount,\n\tconst T* b_global, int bCount, int tid, T* shared, bool sync = true);\n\ntemplate<int NT, int VT0, int VT1, typename InputIt1, typename InputIt2,\n\ttypename T>\nMGPU_DEVICE void DeviceLoad2ToReg(InputIt1 a_global, int aCount,\n\tInputIt2 b_global, int bCount, int tid, T* reg, bool sync = false);\n\ntemplate<int NT, int VT0, int VT1, typename InputIt1, typename InputIt2,\n\ttypename T>\nMGPU_DEVICE void DeviceLoad2ToShared(InputIt1 a_global, int aCount,\n\tInputIt2 b_global, int bCount, int tid, T* shared, bool sync = true);\n\n// For 0 <= i < VT\n//\t\tindex = NT * i + tid;\n//\t\tif(index < count)\n//\t\t\tgather = indices_shared[index];\n//\t\t\tdest_global[index] = data_global[gather];\n// Synchronize after load.\ntemplate<int NT, int VT, typename InputIt, typename OutputIt>\nMGPU_DEVICE void DeviceGatherGlobalToGlobal(int count, InputIt data_global,\n\tconst int* indices_shared, int tid, OutputIt dest_global,\n\tbool sync = true);\n\n// For 0 <= i < VT\n//\t\tindex = NT * i + tid\n//\t\tif(index < count)\n//\t\t\tgather = indices[index];\n//\t\t\tif(gather < aCount) data = a_global[gather];\n//\t\t\telse data = b_global[gather - aCount];\n//\t\t\tdest_global[index] = data;\n// Synchronize after load.\ntemplate<int NT, int VT, typename InputIt1, typename InputIt2,\n\ttypename T>\nMGPU_DEVICE void DeviceTransferMergeValuesReg(int count, InputIt1 a_global,\n\tInputIt2 b_global, int bStart, const int* indices, int tid,\n\tT* reg, bool sync = false);\n\ntemplate<int NT, int VT, typename InputIt1, typename InputIt2,\n\ttypename OutputIt>\nMGPU_DEVICE void DeviceTransferMergeValuesShared(int count, InputIt1 a_global,\n\tInputIt2 b_global, int bStart, const int* indices_shared, int tid,\n\tOutputIt dest_global, bool sync = true);\n\ntemplate<int NT, int VT, typename T>\nMGPU_DEVICE void DeviceTransferMergeValuesReg(int count, const T* a_global,\n\tconst T* b_global, int bStart, const int* indices, int tid,\n\tT* reg, bool sync = false);\n\ntemplate<int NT, int VT, typename T, typename OutputIt>\nMGPU_DEVICE void DeviceTransferMergeValuesShared(int count, const T* a_global,\n\tconst T* b_global, int bStart, const int* indices_shared, int tid,\n\tOutputIt dest_global, bool sync = true);\n\n\n\n} // namespace mgpu\n\n\n#include \"device/loadstore.cuh\"\n#include \"device/ctasegscan.cuh\"\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/mgpuenums.h",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n * \n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n * \n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" \n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE \n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#pragma once \n\nnamespace mgpu {\n\nenum MgpuBounds {\n\tMgpuBoundsLower,\n\tMgpuBoundsUpper\n};\n\nenum MgpuScanType {\n\tMgpuScanTypeExc,\n\tMgpuScanTypeInc\n};\n\nenum MgpuSearchType {\n\tMgpuSearchTypeNone,\n\tMgpuSearchTypeIndex,\n\tMgpuSearchTypeMatch,\n\tMgpuSearchTypeIndexMatch\n};\n\nenum MgpuJoinKind {\n\tMgpuJoinKindInner,\n\tMgpuJoinKindLeft,\n\tMgpuJoinKindRight,\n\tMgpuJoinKindOuter\n};\n\nenum MgpuSetOp {\n\tMgpuSetOpIntersection,\n\tMgpuSetOpUnion,\n\tMgpuSetOpDiff,\n\tMgpuSetOpSymDiff\n};\n\n} // namespace mgpu\n"
  },
  {
    "path": "3rdparty/ctc_include/contrib/moderngpu/include/util/static.h",
    "content": "/******************************************************************************\n * Copyright (c) 2013, NVIDIA CORPORATION.  All rights reserved.\n * \n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *     * Redistributions of source code must retain the above copyright\n *       notice, this list of conditions and the following disclaimer.\n *     * Redistributions in binary form must reproduce the above copyright\n *       notice, this list of conditions and the following disclaimer in the\n *       documentation and/or other materials provided with the distribution.\n *     * Neither the name of the NVIDIA CORPORATION nor the\n *       names of its contributors may be used to endorse or promote products\n *       derived from this software without specific prior written permission.\n * \n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" \n * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE \n * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY\n * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n ******************************************************************************/\n\n/******************************************************************************\n *\n * Code and text by Sean Baxter, NVIDIA Research\n * See http://nvlabs.github.io/moderngpu for repository and documentation.\n *\n ******************************************************************************/\n\n#pragma once\n\n#include <functional>\n#include <iterator>\n#include <cfloat>\n#include <typeinfo>\n#include <vector>\n#include <list>\n#include <map>\n#include <algorithm>\n#include <cassert>\n#include <memory>\n#include <cmath>\n#include <cstdio>\n#include <cstdlib>\n\n#ifndef MGPU_MIN\n#define MGPU_MIN(x, y) (((x) <= (y)) ? (x) : (y))\n#define MGPU_MAX(x, y) (((x) >= (y)) ? (x) : (y))\n#define MGPU_MAX0(x) (((x) >= 0) ? (x) : 0)\n#define MGPU_ABS(x) (((x) >= 0) ? (x) : (-x))\n\n#define MGPU_DIV_UP(x, y) (((x) + (y) - 1) / (y))\n#define MGPU_DIV_ROUND(x, y) (((x) + (y) / 2) / (y))\n#define MGPU_ROUND_UP(x, y) ((y) * MGPU_DIV_UP(x, y))\n#define MGPU_SHIFT_DIV_UP(x, y) (((x) + ((1<< (y)) - 1))>> y)\n#define MGPU_ROUND_UP_POW2(x, y) (((x) + (y) - 1) & ~((y) - 1))\n#define MGPU_ROUND_DOWN_POW2(x, y) ((x) & ~((y) - 1))\n#define MGPU_IS_POW_2(x) (0 == ((x) & ((x) - 1)))\n\n#endif // MGPU_MIN\n\nnamespace mgpu {\n\n\ntypedef unsigned char byte;\n\ntypedef unsigned int uint;\ntypedef signed short int16;\n\ntypedef unsigned short ushort;\ntypedef unsigned short uint16;\n\ntypedef long long int64;\ntypedef unsigned long long uint64;\n\n// IsPow2<X>::value is true if X is a power of 2.\ntemplate<int X> struct sIsPow2 {\n\tenum { value = 0 == (X & (X - 1)) };\n};\n\n// Finds the base-2 logarithm of X. value is -1 if X is not a power of 2.\ntemplate<int X, bool roundUp = true> struct sLogPow2 { \n\tenum { extra = sIsPow2<X>::value ? 0 : (roundUp ? 1 : 0) };\n\tenum { inner = sLogPow2<X / 2>::inner + 1 };\n\tenum { value = inner + extra };\n};\ntemplate<bool roundUp> struct sLogPow2<0, roundUp> {\n\tenum { inner = 0 };\n\tenum { value = 0 };\n};\ntemplate<bool roundUp> struct sLogPow2<1, roundUp> { \n\tenum { inner = 0 };\n\tenum { value = 0 };\n};\n\ntemplate<int X, int Y>\nstruct sDivUp {\n\tenum { value = (X + Y - 1) / Y };\n};\n\ntemplate<int count, int levels> struct sDiv2RoundUp {\n\tenum { value = sDiv2RoundUp<sDivUp<count, 2>::value, levels - 1>::value };\n};\ntemplate<int count> struct sDiv2RoundUp<count, 0> {\n\tenum { value = count };\n};\n\ntemplate<int X, int Y>\nstruct sDivSafe {\n\tenum { value = X / Y };\n};\ntemplate<int X>\nstruct sDivSafe<X, 0> {\n\tenum { value = 0 };\n};\n\ntemplate<int X, int Y>\nstruct sRoundUp {\n\tenum { rem = X % Y };\n\tenum { value = X + (rem ? (Y - rem) : 0) };\n};\n\ntemplate<int X, int Y>\nstruct sRoundDown {\n\tenum { rem = X % Y };\n\tenum { value = X - rem };\n};\n\n// IntegerDiv is a template for avoiding divisions by zero in template \n// evaluation. Templates always evaluate both b and c in an expression like\n// a ? b : c, and will error if either rhs contains an illegal expression,\n// even if the ternary is explictly designed to guard against that.\ntemplate<int X, int Y>\nstruct sIntegerDiv {\n\tenum { value = X / (Y ? Y : (X + 1)) };\n};\n\ntemplate<int X, int Y>\nstruct sMax {\n\tenum { value = (X >= Y) ? X : Y };\n};\ntemplate<int X, int Y>\nstruct sMin {\n\tenum { value = (X <= Y) ? X : Y };\n};\n\ntemplate<int X>\nstruct sAbs {\n\tenum { value = (X >= 0) ? X : -X };\n};\n\n\n// Finds the number of powers of 2 in the prime factorization of X.\ntemplate<int X, int LSB = 1 & X> struct sNumFactorsOf2 {\n\tenum { shifted = X >> 1 };\n\tenum { value = 1 + sNumFactorsOf2<shifted>::value };\n};\ntemplate<int X> struct sNumFactorsOf2<X, 1> {\n\tenum { value = 0 };\n};\n\n// Returns the divisor for a conflict-free transpose.\ntemplate<int X, int NumBanks = 32> struct sBankConflictDivisor {\n\tenum { value = \n\t\t(1 & X) ? 0 : \n\t\t(sIsPow2<X>::value ? NumBanks :\n\t\t(1<< sNumFactorsOf2<X>::value)) }; \n\tenum { log_value = sLogPow2<value>::value };\n};\n\ntemplate<int NT, int X, int NumBanks = 32> struct sConflictFreeStorage {\n\tenum { count = NT * X };\n\tenum { divisor = sBankConflictDivisor<X, NumBanks>::value };\n\tenum { padding = sDivSafe<count, divisor>::value };\n\tenum { value = count + padding };\n};\n\n} // namespace mgpu\n"
  },
  {
    "path": "3rdparty/ctc_include/detail/cpu_ctc.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#pragma once\n\n#include <tuple>\n#include <cmath>\n#include <limits>\n#include <algorithm>\n#include <numeric>\n\n#include <dmlc/omp.h>\n\n#include \"ctc_helper.h\"\n\nnamespace mxnet_warpctc {\n\ntemplate<typename ProbT>\nclass CpuCTC {\npublic:\n    // Noncopyable\n    CpuCTC(int alphabet_size, int minibatch, void* workspace,\n           int blank_label) :\n            alphabet_size_(alphabet_size), minibatch_(minibatch),\n            workspace_(workspace), blank_label_(blank_label) {\n\n    };\n\n    CpuCTC(const CpuCTC&) = delete;\n    CpuCTC& operator=(const CpuCTC&) = delete;\n\n    ctcStatus_t cost_and_grad(const ProbT* const activations,\n                              ProbT *grads,\n                              ProbT* costs,\n                              const int* const flat_labels,\n                              const int* const label_lengths,\n                              const int* const input_lengths);\n\n\n    ctcStatus_t score_forward(const ProbT* const activations,\n                              ProbT* costs,\n                              const int* const flat_labels,\n                              const int* const label_lengths,\n                              const int* const input_lengths);\n\nprivate:\n\n    class CpuCTC_metadata {\n\n    private:\n        int setup_labels(const int* const labels, int blank_label, int L, int S);\n\n    public:\n        CpuCTC_metadata(int L, int S, int T, int mb, int alphabet_size,\n                        void* workspace, size_t bytes_used, int blank_label,\n                        const int* const labels);\n\n        ProbT* alphas;\n        ProbT* betas;\n        int* labels_w_blanks;\n        int* e_inc;\n        int* s_inc;\n        ProbT* output;\n        int repeats;\n    };\n\n    int alphabet_size_; // Number of characters plus blank\n    int minibatch_;\n    void* workspace_;\n    int blank_label_;\n\n    void log_softmax(const ProbT* const activations, ProbT* log_probs,\n                     const int* const input_lengths);\n\n    std::tuple<ProbT, bool>\n            cost_and_grad_kernel(ProbT *grad, const ProbT* const log_probs,\n                                 const int* const labels, int T, int L,\n                                 int mb, size_t bytes_used);\n\n    ProbT compute_alphas(const ProbT* log_probs, int repeats, int S, int T,\n                         const int* const e_inc,\n                         const int* const s_inc,\n                         const int* const labels,\n                         ProbT* alphas);\n\n    ProbT compute_betas_and_grad(ProbT* grad, const ProbT* const log_probs,\n                                 ProbT log_partition, int repeats,\n                                 int S, int T, const int* const e_inc,\n                                 const int* const s_inc,\n                                 const int* const labels,\n                                 ProbT* alphas,\n                                 ProbT* betas,\n                                 ProbT* output);\n};\n\ntemplate<typename ProbT>\nCpuCTC<ProbT>::CpuCTC_metadata::CpuCTC_metadata(int L, int S, int T, int mb,\n                                                int alphabet_size,\n                                                void* workspace, size_t bytes_used,\n                                                int blank_label,\n                                                const int* const labels) {\n\n    alphas = reinterpret_cast<ProbT *>(static_cast<char *>(workspace) + bytes_used);\n    bytes_used += sizeof(ProbT) * S * T;\n    std::fill(alphas, alphas + S * T, ctc_helper::neg_inf<ProbT>());\n    betas = reinterpret_cast<ProbT *>(static_cast<char *>(workspace) + bytes_used);\n    bytes_used += sizeof(ProbT) * S;\n    std::fill(betas, betas + S, ctc_helper::neg_inf<ProbT>());\n    labels_w_blanks = reinterpret_cast<int *>(static_cast<char *>(workspace) + bytes_used);\n    bytes_used += sizeof(int) * S;\n    e_inc = reinterpret_cast<int *>(static_cast<char *>(workspace) + bytes_used);\n    bytes_used += sizeof(int) * S;\n    s_inc = reinterpret_cast<int *>(static_cast<char *>(workspace) + bytes_used);\n    bytes_used += sizeof(int) * S;\n    output = reinterpret_cast<ProbT *>(static_cast<char *>(workspace) + bytes_used);\n    bytes_used += sizeof(ProbT) * alphabet_size;\n\n    repeats = setup_labels(labels, blank_label, L, S);\n}\n\ntemplate<typename ProbT>\nint CpuCTC<ProbT>::CpuCTC_metadata::setup_labels(const int* const labels,\n                                                 int blank_label, int L, int S) {\n    int e_counter = 0;\n    int s_counter = 0;\n\n    s_inc[s_counter++] = 1;\n\n    int repeats = 0;\n\n    for (int i = 1; i < L; ++i) {\n        if (labels[i-1] == labels[i]) {\n            s_inc[s_counter++] = 1;\n            s_inc[s_counter++] = 1;\n            e_inc[e_counter++] = 1;\n            e_inc[e_counter++] = 1;\n            ++repeats;\n        }\n        else {\n            s_inc[s_counter++] = 2;\n            e_inc[e_counter++] = 2;\n        }\n    }\n    e_inc[e_counter++] = 1;\n\n    for (int i = 0; i < L; ++i) {\n        labels_w_blanks[2 * i] = blank_label;\n        labels_w_blanks[2 * i + 1] = labels[i];\n    }\n    labels_w_blanks[S - 1] = blank_label;\n\n    return repeats;\n}\n\ntemplate<typename ProbT>\nvoid\nCpuCTC<ProbT>::log_softmax(const ProbT* const activations, ProbT* log_probs,\n                           const int* const input_lengths) {\n#pragma omp parallel for\n    for (int mb = 0; mb < minibatch_; ++mb) {\n        for(int c = 0; c < input_lengths[mb]; ++c) {\n            int col_offset = (mb + minibatch_ * c) * alphabet_size_;\n            ProbT max_activation = -std::numeric_limits<ProbT>::infinity();\n            for(int r = 0; r < alphabet_size_; ++r)\n                max_activation = std::max(max_activation, activations[r + col_offset]);\n\n            ProbT denom = ProbT(0.);\n            for(int r = 0; r < alphabet_size_; ++r) {\n                denom += std::exp(activations[r + col_offset] - max_activation);\n            }\n\n            for(int r = 0; r < alphabet_size_; ++r) {\n                log_probs[r + col_offset] = activations[r + col_offset]\n                                            - max_activation - std::log(denom);\n            }\n        }\n    }\n}\n\ntemplate<typename ProbT>\nstd::tuple<ProbT, bool>\nCpuCTC<ProbT>::cost_and_grad_kernel(ProbT *grad, const ProbT* const log_probs,\n                                    const int* const labels,\n                                    int T, int L, int mb, size_t bytes_used) {\n\n    const int S = 2*L + 1; // Number of labels with blanks\n\n    CpuCTC_metadata ctcm(L, S, T, mb, alphabet_size_, workspace_, bytes_used, blank_label_, labels);\n\n    bool over_threshold = false;\n\n    if (L + ctcm.repeats > T) {\n        return std::make_tuple(ProbT(0), over_threshold); // TODO, not right to return 0\n    }\n\n    ProbT llForward = compute_alphas(log_probs, ctcm.repeats, S, T, ctcm.e_inc,\n                                     ctcm.s_inc, ctcm.labels_w_blanks,\n                                     ctcm.alphas);\n\n    ProbT llBackward = compute_betas_and_grad(grad, log_probs, llForward, ctcm.repeats,\n                                              S, T, ctcm.e_inc, ctcm.s_inc,\n                                              ctcm.labels_w_blanks,\n                                              ctcm.alphas,\n                                              ctcm.betas,\n                                              ctcm.output);\n\n    ProbT diff = std::abs(llForward - llBackward);\n    if (diff > ctc_helper::threshold) {\n        over_threshold = true;\n    }\n\n    return std::make_tuple(-llForward, over_threshold);\n}\n\n// Computes forward probabilities\ntemplate<typename ProbT>\nProbT CpuCTC<ProbT>::compute_alphas(const ProbT* log_probs, int repeats, int S, int T,\n                                    const int* const e_inc,\n                                    const int* const s_inc,\n                                    const int* const labels,\n                                    ProbT* alphas) {\n\n    int start =  (((S /2) + repeats - T) < 0) ? 0 : 1,\n            end = S > 1 ? 2 : 1;\n\n    for (int i = start; i < end; ++i) {\n        alphas[i] = log_probs[labels[i]];\n    }\n\n    for(int t = 1; t < T; ++t) {\n        int remain = (S / 2) + repeats - (T - t);\n        if(remain >= 0)\n            start += s_inc[remain];\n        if(t <= (S / 2) + repeats)\n            end += e_inc[t - 1];\n        int startloop = start;\n        int idx1 = t * S, idx2 = (t - 1) * S, idx3 = t * (alphabet_size_ * minibatch_);\n\n        if (start == 0) {\n            alphas[idx1] = alphas[idx2] + log_probs[blank_label_ + idx3];\n            startloop += 1;\n        }\n\n        for(int i = startloop; i < end; ++i) {\n            ProbT prev_sum = ctc_helper::log_plus<ProbT>()(alphas[i + idx2], alphas[(i-1) + idx2]);\n\n            // Skip two if not on blank and not on repeat.\n            if (labels[i] != blank_label_ && i != 1 && labels[i] != labels[i-2])\n                prev_sum = ctc_helper::log_plus<ProbT>()(prev_sum, alphas[(i-2) + idx2]);\n\n            alphas[i + idx1] = prev_sum + log_probs[labels[i] + idx3];\n        }\n    }\n\n    ProbT loglike = ctc_helper::neg_inf<ProbT>();\n    for(int i = start; i < end; ++i) {\n        loglike = ctc_helper::log_plus<ProbT>()(loglike, alphas[i + (T - 1) * S]);\n    }\n\n    return loglike;\n}\n\n// Starting from T, we sweep backward over the alpha array computing one column\n// of betas as we go.  At each position we can update product alpha * beta and then\n// sum into the gradient associated with each label.\n// NOTE computes gradient w.r.t UNNORMALIZED final layer activations.\n// Assumed passed in grads are already zeroed!\ntemplate<typename ProbT>\nProbT CpuCTC<ProbT>::compute_betas_and_grad(ProbT* grad, const ProbT* const log_probs,\n                                            ProbT log_partition, int repeats,\n                                            int S, int T, const int* const e_inc,\n                                            const int* const s_inc,\n                                            const int* const labels,\n                                            ProbT* alphas,\n                                            ProbT* betas,\n                                            ProbT* output) {\n    int start = S > 1 ? (S - 2) : 0,\n            end = (T > (S / 2) + repeats) ? S : S-1;\n\n    std::fill(output, output + alphabet_size_, ctc_helper::neg_inf<ProbT>());\n\n    //set the starting values in the beta column at the very right edge\n    for (int i = start; i < end; ++i) {\n        betas[i] = log_probs[labels[i] + (T - 1) * (alphabet_size_ * minibatch_)];\n\n        //compute alpha * beta in log space at this position in (S, T) space\n        alphas[i + (T - 1) * S] += betas[i];\n\n        //update the gradient associated with this label\n        //essentially performing a reduce-by-key in a sequential manner\n        output[labels[i]] =\n                ctc_helper::log_plus<ProbT>()(alphas[i + (T - 1) * S], output[labels[i]]);\n    }\n\n    //update the gradient wrt to each unique label\n    for (int i = 0; i < alphabet_size_; ++i) {\n        int idx3 = (T - 1) * alphabet_size_ * minibatch_ + i;\n\n        if (output[i] == 0.0 || output[i] == ctc_helper::neg_inf<ProbT>() ||\n            log_probs[idx3] == ctc_helper::neg_inf<ProbT>()) {\n            grad[idx3] = std::exp(log_probs[idx3]);\n        } else {\n            grad[idx3] = std::exp(log_probs[idx3])\n                         - std::exp(output[i] - log_probs[idx3] - log_partition);\n        }\n    }\n\n    //loop from the second to last column all the way to the left\n    for(int t = T - 2; t >= 0; --t) {\n        int remain = (S / 2) + repeats - (T - t);\n        if(remain >= -1)\n            start -= s_inc[remain + 1];\n        if(t < (S / 2) + repeats)\n            end -= e_inc[t];\n\n        int endloop = end == S ? end - 1 : end;\n        int idx1 = t * S, idx3 = t * (alphabet_size_ * minibatch_);\n\n        std::fill(output, output + alphabet_size_, ctc_helper::neg_inf<ProbT>());\n\n        for(int i = start; i < endloop; ++i) {\n            ProbT next_sum = ctc_helper::log_plus<ProbT>()(betas[i], betas[(i+1)]);\n            // Skip two if not on blank and not on repeat.\n            if (labels[i] != blank_label_ && i != (S-2) && labels[i] != labels[i+2]){\n                next_sum = ctc_helper::log_plus<ProbT>()(next_sum, betas[(i+2)]);\n            }\n            betas[i] = next_sum + log_probs[labels[i] + idx3];\n\n            //compute alpha * beta in log space\n            alphas[i + idx1] += betas[i];\n\n            //update the gradient associated with this label\n            output[labels[i]] =\n                    ctc_helper::log_plus<ProbT>()(alphas[i + idx1], output[labels[i]]);\n        }\n\n        if (end == S) {\n            betas[(S-1)] = betas[(S-1)] + log_probs[blank_label_ + idx3];\n            alphas[(S-1) + idx1] += betas[(S-1)];\n\n            output[labels[S-1]] =\n                    ctc_helper::log_plus<ProbT>()(alphas[S-1 + idx1], output[labels[S-1]]);\n        }\n\n        //go over the unique labels and compute the final grad\n        // wrt to each one at this time step\n        for (int i = 0; i < alphabet_size_; ++i) {\n\n            if (output[i] == 0.0 || output[i] == ctc_helper::neg_inf<ProbT>() ||\n                log_probs[idx3] == ctc_helper::neg_inf<ProbT>()) {\n                grad[idx3] = std::exp(log_probs[idx3]);\n            } else {\n                grad[idx3] = std::exp(log_probs[idx3])\n                             - std::exp(output[i] - log_probs[idx3] - log_partition);\n            }\n            ++idx3;\n        }\n    }\n\n    ProbT loglike = ctc_helper::neg_inf<ProbT>();\n    for(int i = start; i < end; ++i) {\n        loglike = ctc_helper::log_plus<ProbT>()(loglike, betas[i]);\n    }\n\n    return loglike;\n}\n\ntemplate<typename ProbT>\nctcStatus_t\nCpuCTC<ProbT>::cost_and_grad(const ProbT* const activations,\n                             ProbT *grads,\n                             ProbT *costs,\n                             const int* const flat_labels,\n                             const int* const label_lengths,\n                             const int* const input_lengths) {\n    if (activations == nullptr ||\n        grads == nullptr ||\n        costs == nullptr ||\n        flat_labels == nullptr ||\n        label_lengths == nullptr ||\n        input_lengths == nullptr\n        )\n        return CTC_STATUS_INVALID_VALUE;\n\n    ProbT* log_probs = static_cast<ProbT *>(workspace_);\n\n    int maxT = *std::max_element(input_lengths, input_lengths + minibatch_);\n\n    size_t bytes_used = sizeof(ProbT) * minibatch_ * alphabet_size_ * maxT;\n\n    //per minibatch memory\n    size_t per_minibatch_bytes = 0;\n\n    int maxL = *std::max_element(label_lengths, label_lengths + minibatch_);;\n    int maxS = 2 * maxL + 1;\n\n    //output\n    per_minibatch_bytes += sizeof(float) * alphabet_size_;\n\n    //alphas\n    per_minibatch_bytes += sizeof(float) * maxS * maxT;\n\n    //betas\n    per_minibatch_bytes += sizeof(float) * maxS;\n\n    //labels w/blanks, e_inc, s_inc\n    per_minibatch_bytes += 3 * sizeof(int) * maxS;\n\n    log_softmax(activations, log_probs, input_lengths);\n\n#pragma omp parallel for\n    for (int mb = 0; mb < minibatch_; ++mb) {\n        const int T = input_lengths[mb]; // Length of utterance (time)\n        const int L = label_lengths[mb]; // Number of labels in transcription\n\n        bool mb_status;\n\n        std::tie(costs[mb], mb_status) =\n                cost_and_grad_kernel(grads + mb * alphabet_size_,\n                                     log_probs + mb * alphabet_size_,\n                                     flat_labels + std::accumulate(label_lengths, label_lengths + mb, 0),\n                                     T, L, mb,\n                                     bytes_used + mb * per_minibatch_bytes);\n    }\n\n    return CTC_STATUS_SUCCESS;\n}\n\ntemplate<typename ProbT>\nctcStatus_t CpuCTC<ProbT>::score_forward(const ProbT* const activations,\n                                         ProbT* costs,\n                                         const int* const flat_labels,\n                                         const int* const label_lengths,\n                                         const int* const input_lengths) {\n    if (activations == nullptr ||\n        costs == nullptr ||\n        flat_labels == nullptr ||\n        label_lengths == nullptr ||\n        input_lengths == nullptr\n        )\n        return CTC_STATUS_INVALID_VALUE;\n\n    ProbT* log_probs = static_cast<ProbT *>(workspace_);\n\n    int maxT = *std::max_element(input_lengths, input_lengths + minibatch_);\n\n    size_t bytes_used = sizeof(ProbT) * minibatch_ * alphabet_size_ * maxT;\n\n    //per minibatch memory\n    size_t per_minibatch_bytes = 0;\n\n    int maxL = *std::max_element(label_lengths, label_lengths + minibatch_);\n    int maxS = 2 * maxL + 1;\n\n    //output\n    per_minibatch_bytes += sizeof(float) * alphabet_size_;\n\n    //alphas\n    per_minibatch_bytes += sizeof(float) * maxS * maxT;\n\n    //betas\n    per_minibatch_bytes += sizeof(float) * maxS;\n\n    //labels w/blanks, e_inc, s_inc\n    per_minibatch_bytes += 3 * sizeof(int) * maxS;\n\n    log_softmax(activations, log_probs, input_lengths);\n\n#pragma omp parallel for\n    for (int mb = 0; mb < minibatch_; ++mb) {\n        const int T = input_lengths[mb]; // Length of utterance (time)\n        const int L = label_lengths[mb]; // Number of labels in transcription\n        const int S = 2*L + 1; // Number of labels with blanks\n\n        CpuCTC_metadata ctcm(L, S, T, mb, alphabet_size_, workspace_,\n                             bytes_used + mb * per_minibatch_bytes, blank_label_,\n                             flat_labels + std::accumulate(label_lengths, label_lengths + mb, 0));\n\n\n        if (L + ctcm.repeats > T)\n            costs[mb] = ProbT(0);\n        else {\n            costs[mb] = -compute_alphas(log_probs + mb * alphabet_size_, ctcm.repeats, S, T,\n                                        ctcm.e_inc, ctcm.s_inc, ctcm.labels_w_blanks,\n                                        ctcm.alphas);\n        }\n\n    }\n\n    return CTC_STATUS_SUCCESS;\n}\n\n} // mxnet_warpctc\n"
  },
  {
    "path": "3rdparty/ctc_include/detail/ctc_helper.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#pragma once\n\n#include <limits>\n#include <algorithm>\n#include <cmath>\n\n#include \"hostdevice.h\"\n\ntypedef enum {\n    CTC_STATUS_SUCCESS = 0,\n    CTC_STATUS_MEMOPS_FAILED = 1,\n    CTC_STATUS_INVALID_VALUE = 2,\n    CTC_STATUS_EXECUTION_FAILED = 3,\n    CTC_STATUS_UNKNOWN_ERROR = 4\n} ctcStatus_t;\n\ntypedef enum {\n    CTC_CPU = 0,\n    CTC_GPU = 1\n} ctcComputeLocation;\n\nnamespace ctc_helper {\n\nstatic const float threshold = 1e-1;\n\ntemplate<typename T>\nHOSTDEVICE\nT neg_inf() { return -T(INFINITY); }\n\ninline int div_up(int x, int y) {\n    return (x + y - 1) / y;\n}\n\ntemplate <typename Arg, typename Res = Arg> struct maximum {\n    HOSTDEVICE\n    Res operator()(const Arg& x, const Arg& y) const {\n        return x < y ? y : x;\n    }\n};\n\ntemplate <typename Arg, typename Res = Arg> struct add {\n    HOSTDEVICE\n    Res operator()(const Arg& x, const Arg& y) const {\n        return x + y;\n    }\n};\n\ntemplate <typename Arg, typename Res = Arg> struct identity {\n    HOSTDEVICE Res operator()(const Arg& x) const {return Res(x);}\n};\n\ntemplate <typename Arg, typename Res = Arg> struct negate {\n    HOSTDEVICE Res operator()(const Arg& x) const {return Res(-x);}\n};\n\ntemplate <typename Arg, typename Res = Arg> struct exponential {\n    HOSTDEVICE Res operator()(const Arg& x) const {return std::exp(x);}\n};\n\ntemplate<typename Arg1, typename Arg2 = Arg1, typename Res=Arg1>\nstruct log_plus {\n    typedef Res result_type;\n    HOSTDEVICE\n    Res operator()(const Arg1& p1, const Arg2& p2) {\n        if (p1 == neg_inf<Arg1>())\n            return p2;\n        if (p2 == neg_inf<Arg2>())\n            return p1;\n        Res result = log1p(exp(-fabs(p1 - p2))) + maximum<Res>()(p1, p2);\n        return result;\n    }\n};\n\n}\n"
  },
  {
    "path": "3rdparty/ctc_include/detail/gpu_ctc.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#pragma once\n\n\n#include \"ctc_helper.h\"\n#include \"gpu_ctc_kernels.h\"\n\nnamespace mxnet_warpctc {\n\ntemplate <typename ProbT>\nclass GpuCTC {\n    public:\n        GpuCTC(int alphabet_size,\n               int minibatch,\n               void *workspace,\n               CUstream stream,\n               int blank_label) :\n            out_dim_(alphabet_size), minibatch_(minibatch),\n            gpu_workspace_(workspace), stream_(stream),\n            blank_label_(blank_label) {};\n\n        // Noncopyable\n        GpuCTC(const GpuCTC&) = delete;\n        GpuCTC& operator=(const GpuCTC&) = delete;\n\n        ctcStatus_t\n        cost_and_grad(const ProbT* const activations,\n                      ProbT* grads,\n                      ProbT* costs,\n                      const int* const flat_labels,\n                      const int* const label_lengths,\n                      const int* const input_lengths);\n\n        ctcStatus_t\n        score_forward(const ProbT* const activations,\n                      ProbT* costs,\n                      const int* const flat_labels,\n                      const int* const label_lengths,\n                      const int* const input_lengths);\n\n    private:\n\n        template<int NT, int VT>\n        ctcStatus_t launch_alpha_beta_kernels(const ProbT* const log_probs,\n                                              ProbT *grads,\n                                              bool compute_alpha,\n                                              bool compute_beta);\n\n        ctcStatus_t\n        launch_gpu_kernels(const ProbT* const log_probs,\n                           ProbT *grads,\n                           size_t config,\n                           bool launch_alpha,\n                           bool launch_beta);\n\n        ctcStatus_t\n        setup_gpu_metadata(const int* const flat_labels,\n                           const int* const label_lengths,\n                           const int* const input_lengths);\n\n        ctcStatus_t\n        create_metadata_and_choose_config(const int* const label_lengths,\n                                          const int* const flat_labels,\n                                          const int* const input_lengths,\n                                          size_t& best_config);\n\n        ctcStatus_t\n        compute_log_probs(const ProbT* const activations);\n\n        ctcStatus_t\n        compute_cost_and_score(const ProbT* const activations,\n                               ProbT* grads,\n                               ProbT* costs,\n                               const int* const flat_labels,\n                               const int* const label_lengths,\n                               const int* const input_lengths,\n                               bool compute_alpha,\n                               bool compute_betas_and_grad);\n\n\n        int out_dim_; // Number of characters plus blank\n        int minibatch_;\n\n        int S_;\n        int T_;\n\n        int activation_cols_; // Number of columns in activations\n\n        void *gpu_workspace_; // Buffer for all temporary GPU memory\n        CUstream stream_;\n        int blank_label_;\n\n        int *utt_length_; // T\n        int *label_sizes_; // L\n        int *repeats_; // repeats_\n        int *label_offsets_;\n        int *labels_without_blanks_;\n        int *labels_with_blanks_;\n        ProbT *alphas_;\n        ProbT *nll_forward_;\n        ProbT *nll_backward_;\n        ProbT *denoms_; // Temporary storage for denoms for softmax\n        ProbT *log_probs_; // Temporary storage for probabilities (log softmax output)\n};\n\ntemplate<typename ProbT>\nctcStatus_t\nGpuCTC<ProbT>::setup_gpu_metadata(const int* const flat_labels,\n                                  const int* const label_lengths,\n                                  const int* const input_lengths)\n{\n    size_t gpu_bytes_used = 0;\n\n    nll_forward_ =\n        reinterpret_cast<ProbT *>(static_cast<char*>(gpu_workspace_) +\n                                  gpu_bytes_used);\n    gpu_bytes_used += minibatch_ * sizeof(ProbT);\n\n\n    nll_backward_ =\n        reinterpret_cast<ProbT *>(static_cast<char*>(gpu_workspace_) +\n                                  gpu_bytes_used);\n    gpu_bytes_used += minibatch_ * sizeof(ProbT);\n\n\n    repeats_ =\n        reinterpret_cast<int *>(static_cast<char*>(gpu_workspace_) +\n                                gpu_bytes_used);\n    gpu_bytes_used += minibatch_ * sizeof(int);\n\n    label_offsets_ =\n        reinterpret_cast<int *>(static_cast<char*>(gpu_workspace_) +\n                                gpu_bytes_used);\n    gpu_bytes_used += minibatch_ * sizeof(int);\n\n\n    // This is the max of all S and T for all valid examples in the minibatch.\n    // A valid example is one for which L + repeats <= T\n    S_ = 0;\n    T_ = 0;\n\n    // This is the max of all timesteps, valid or not. Needed to compute offsets\n    int Tmax = 0;\n\n    // This is the max of all labels, valid or not. Needed to compute offsets\n    int Lmax = 0;\n    int total_label_length = 0;\n\n    constexpr int cpu_buffer_size = 64;\n    int repeats[cpu_buffer_size];\n    int label_offsets[cpu_buffer_size];\n\n    const int num_passes = ctc_helper::div_up(minibatch_, cpu_buffer_size);\n\n    cudaError_t cuda_status;\n\n    for (int pass = 0; pass < num_passes; ++pass) {\n\n        const int start_idx = pass * cpu_buffer_size;\n        const int end_idx = std::min(minibatch_, (pass+1) * cpu_buffer_size);\n\n        for (int j = start_idx; j < end_idx; ++j) {\n            const int L = label_lengths[j];\n            const int local_T = input_lengths[j];\n            const int *label_ptr = &(flat_labels[total_label_length]);\n\n            label_offsets[j % cpu_buffer_size] = total_label_length;\n            total_label_length += L;\n\n            int repeat_counter = 0;\n\n            for (int i = 1; i < L; ++i)\n                repeat_counter += (label_ptr[i] == label_ptr[i-1]);\n\n            repeats[j % cpu_buffer_size] = repeat_counter;\n            const bool valid_label = ((L + repeat_counter) <= local_T);\n\n            // Only update S and T if label is valid\n            S_ = (valid_label) ? std::max(S_, L) : S_;\n            T_ = (valid_label) ? std::max(T_, local_T) : T_;\n\n            Tmax = std::max(Tmax, local_T);\n            Lmax = std::max(Lmax, L);\n        }\n\n        cuda_status = cudaMemcpyAsync(&(repeats_[start_idx]), repeats,\n                                      (end_idx - start_idx) * sizeof(int),\n                                      cudaMemcpyHostToDevice, stream_);\n        if (cuda_status != cudaSuccess)\n            return CTC_STATUS_MEMOPS_FAILED;\n\n\n        cuda_status = cudaMemcpyAsync(&(label_offsets_[start_idx]), label_offsets,\n                                      (end_idx - start_idx) * sizeof(int),\n                                      cudaMemcpyHostToDevice, stream_);\n        if (cuda_status != cudaSuccess)\n            return CTC_STATUS_MEMOPS_FAILED;\n    }\n\n    S_ = 2 * S_ + 1;\n    const int Smax = 2 * Lmax + 1;\n\n    activation_cols_ = minibatch_ * Tmax;\n\n    // Allocate memory for T\n    utt_length_ =\n        reinterpret_cast<int *>(static_cast<char*>(gpu_workspace_) +\n                                gpu_bytes_used);\n    gpu_bytes_used += minibatch_  * sizeof(int);\n\n    cuda_status = cudaMemcpyAsync(utt_length_, input_lengths,\n                                  minibatch_ * sizeof(int),\n                                  cudaMemcpyHostToDevice, stream_);\n    if (cuda_status != cudaSuccess)\n        return CTC_STATUS_MEMOPS_FAILED;\n\n    label_sizes_ =\n        reinterpret_cast<int *>(static_cast<char*>(gpu_workspace_) +\n                                gpu_bytes_used);\n    gpu_bytes_used += minibatch_ * sizeof(int);\n    cuda_status = cudaMemcpyAsync(label_sizes_, label_lengths,\n                                  minibatch_ * sizeof(int),\n                                  cudaMemcpyHostToDevice, stream_);\n    if (cuda_status != cudaSuccess)\n        return CTC_STATUS_MEMOPS_FAILED;\n\n    labels_without_blanks_ =\n        reinterpret_cast<int *>(static_cast<char*>(gpu_workspace_) +\n                                gpu_bytes_used);\n    gpu_bytes_used += Lmax * minibatch_ * sizeof(int);\n    cuda_status = cudaMemcpyAsync(labels_without_blanks_, flat_labels,\n                                  total_label_length * sizeof(int),\n                                  cudaMemcpyHostToDevice, stream_);\n    if (cuda_status != cudaSuccess)\n        return CTC_STATUS_MEMOPS_FAILED;\n\n    labels_with_blanks_ =\n        reinterpret_cast<int *>(static_cast<char*>(gpu_workspace_) +\n                                gpu_bytes_used);\n    gpu_bytes_used += Smax * minibatch_ * sizeof(int);\n\n    alphas_ =\n        reinterpret_cast<ProbT *>(static_cast<char*>(gpu_workspace_) +\n                                  gpu_bytes_used);\n    gpu_bytes_used += (S_ * T_) * minibatch_ * sizeof(ProbT);\n\n\n    denoms_ =\n        reinterpret_cast<ProbT *>(static_cast<char*>(gpu_workspace_) +\n                                  gpu_bytes_used);\n    gpu_bytes_used += activation_cols_ * sizeof(ProbT);\n\n    log_probs_ =\n        reinterpret_cast<ProbT *>(static_cast<char*>(gpu_workspace_) +\n                                  gpu_bytes_used);\n    gpu_bytes_used += out_dim_ * activation_cols_ * sizeof(ProbT);\n\n    return CTC_STATUS_SUCCESS;\n}\n\ntemplate<typename ProbT>\ntemplate<int NT, int VT>\nctcStatus_t GpuCTC<ProbT>::launch_alpha_beta_kernels(const ProbT* const log_probs,\n                                                     ProbT* grads,\n                                                     bool compute_alpha,\n                                                     bool compute_beta ) {\n\n    // One thread block per utterance\n    const int grid_size = minibatch_;\n\n    // The data is laid out so that the next timestep is minibatch entries\n    // away\n    const int stride = minibatch_;\n\n    if (compute_alpha)\n        compute_alpha_kernel<ProbT, NT, VT><<<grid_size, NT, 0, stream_>>>\n            (log_probs, label_sizes_, utt_length_,\n             repeats_, labels_without_blanks_, label_offsets_,\n             labels_with_blanks_, alphas_, nll_forward_,\n             stride, out_dim_, S_, T_, blank_label_);\n\n\n    if (compute_beta) {\n        compute_betas_and_grad_kernel<ProbT, NT, VT><<<grid_size, NT, 0, stream_>>>\n            (log_probs, label_sizes_, utt_length_, repeats_,\n             labels_with_blanks_, alphas_, nll_forward_, nll_backward_,\n             grads, stride, out_dim_, S_, T_, blank_label_);\n\n        cudaStreamSynchronize(stream_);\n    }\n\n    cudaError_t err = cudaGetLastError();\n    if (err != cudaSuccess)\n        return CTC_STATUS_EXECUTION_FAILED;\n\n    return CTC_STATUS_SUCCESS;\n}\n\ntemplate<typename ProbT>\nctcStatus_t\nGpuCTC<ProbT>::create_metadata_and_choose_config(const int* const flat_labels,\n                                                 const int* const label_lengths,\n                                                 const int* const input_lengths,\n                                                 size_t& best_config) {\n\n    // Setup the metadata for GPU\n    ctcStatus_t status = setup_gpu_metadata(flat_labels, label_lengths, input_lengths);\n    if (status != CTC_STATUS_SUCCESS)\n        return status;\n\n    constexpr int num_configs = 12;\n\n    int config_NT[num_configs] =\n        {32, 64, 128, 64, 128, 32, 64, 128, 64, 128, 128, 128};\n    int config_VT[num_configs] =\n        { 1,  1,   1,  3,   2,  9,  6,   4,  9,   6,   9,  10};\n\n    best_config = 0;\n\n    for (int i = 0; i < num_configs; ++i) {\n        if ((config_NT[i]* config_VT[i]) >= S_)\n            break;\n        else\n            best_config++;\n    }\n\n    if (best_config >= num_configs)\n        return CTC_STATUS_UNKNOWN_ERROR;\n\n    return CTC_STATUS_SUCCESS;\n}\n\ntemplate<typename ProbT>\nctcStatus_t\nGpuCTC<ProbT>::launch_gpu_kernels(const ProbT* const log_probs,\n                                  ProbT* grads,\n                                  size_t config,\n                                  bool l_a,\n                                  bool l_b) {\n\n    switch(config) {\n        case 0:   {return launch_alpha_beta_kernels<32,   1>(log_probs, grads, l_a, l_b);}\n        case 1:   {return launch_alpha_beta_kernels<64,   1>(log_probs, grads, l_a, l_b);}\n        case 2:   {return launch_alpha_beta_kernels<128,  1>(log_probs, grads, l_a, l_b);}\n        case 3:   {return launch_alpha_beta_kernels<64,   3>(log_probs, grads, l_a, l_b);}\n        case 4:   {return launch_alpha_beta_kernels<128,  2>(log_probs, grads, l_a, l_b);}\n        case 5:   {return launch_alpha_beta_kernels<32,   9>(log_probs, grads, l_a, l_b);}\n        case 6:   {return launch_alpha_beta_kernels<64,   6>(log_probs, grads, l_a, l_b);}\n        case 7:   {return launch_alpha_beta_kernels<128,  4>(log_probs, grads, l_a, l_b);}\n        case 8:   {return launch_alpha_beta_kernels<64,   9>(log_probs, grads, l_a, l_b);}\n        case 9:   {return launch_alpha_beta_kernels<128,  6>(log_probs, grads, l_a, l_b);}\n        case 10:  {return launch_alpha_beta_kernels<128,  9>(log_probs, grads, l_a, l_b);}\n        case 11:  {return launch_alpha_beta_kernels<128, 10>(log_probs, grads, l_a, l_b);}\n    }\n\n    return CTC_STATUS_EXECUTION_FAILED;\n}\n\ntemplate<typename ProbT>\nctcStatus_t\nGpuCTC<ProbT>::compute_log_probs(const ProbT* const activations) {\n\n    cudaError_t cuda_status;\n    cuda_status =\n        cudaMemcpyAsync(log_probs_, activations,\n                        activation_cols_ * out_dim_ *sizeof(ProbT),\n                        cudaMemcpyDeviceToDevice, stream_);\n    if (cuda_status != cudaSuccess)\n        return CTC_STATUS_MEMOPS_FAILED;\n\n\n\n    // create mshadow handles to data\n    using namespace mshadow;\n    using namespace mshadow::expr;\n    Stream<mxnet::gpu> mxstream;\n    mxstream.stream_ = stream_;\n    Tensor<mxnet::gpu, 2, ProbT> log_probs_handle(log_probs_, mshadow::Shape2(activation_cols_, out_dim_), &mxstream);\n    Tensor<mxnet::gpu, 1, ProbT> denoms_handle(denoms_, mshadow::Shape1(activation_cols_), &mxstream);\n    denoms_handle = reduce_with_axis<red::maximum, false>(log_probs_handle, 1);\n\n\n\n    // Kernel launch to subtract maximum\n    const int NT = 128;\n    const int VT = 1;\n    const int NV = NT * VT;\n    const int num_elements = out_dim_ * activation_cols_;\n    const int grid_size = ctc_helper::div_up(num_elements, NV);\n\n    prepare_stable_LSM_kernel<ProbT, VT> <<< grid_size, NT, 0, stream_>>>\n       (ctc_helper::identity<ProbT>(), log_probs_,\n        denoms_, out_dim_, num_elements);\n\n    // compute denominators for softmax\n    denoms_handle = reduce_with_axis<red::sum, false>(F<mxnet::op::mshadow_op::exp>(log_probs_handle), 1);\n\n    // Kernel launch to calculate probabilities\n    compute_log_probs_kernel<ProbT, VT><<<grid_size, NT, 0, stream_>>>\n        (ctc_helper::identity<ProbT>(), log_probs_,\n         denoms_, out_dim_, num_elements);\n\n    cuda_status = cudaGetLastError();\n    if (cuda_status != cudaSuccess)\n        return CTC_STATUS_EXECUTION_FAILED;\n\n    return CTC_STATUS_SUCCESS;\n}\n\ntemplate<typename ProbT>\nctcStatus_t\nGpuCTC<ProbT>::compute_cost_and_score(const ProbT* const activations,\n                                      ProbT* grads,\n                                      ProbT* costs,\n                                      const int* const flat_labels,\n                                      const int* const label_lengths,\n                                      const int* const input_lengths,\n                                      bool compute_alpha,\n                                      bool compute_betas_and_grad) {\n\n    size_t best_config;\n    ctcStatus_t status = create_metadata_and_choose_config(flat_labels,\n                                                           label_lengths,\n                                                           input_lengths,\n                                                           best_config);\n    if (status != CTC_STATUS_SUCCESS)\n        return status;\n\n    status = compute_log_probs(activations);\n    if (status != CTC_STATUS_SUCCESS)\n        return status;\n\n    launch_gpu_kernels(log_probs_, grads, best_config,\n                       compute_alpha, compute_betas_and_grad);\n\n    cudaError_t cuda_status_mem, cuda_status_sync;\n    cuda_status_mem = cudaMemcpyAsync(costs, nll_forward_,\n                                      sizeof(ProbT) * minibatch_,\n                                      cudaMemcpyDeviceToHost, stream_);\n    cuda_status_sync = cudaStreamSynchronize(stream_);\n    if (cuda_status_mem != cudaSuccess || cuda_status_sync != cudaSuccess)\n        return CTC_STATUS_MEMOPS_FAILED;\n\n    return CTC_STATUS_SUCCESS;\n}\n\ntemplate<typename ProbT>\nctcStatus_t\nGpuCTC<ProbT>::cost_and_grad(const ProbT* const activations,\n                             ProbT* grads,\n                             ProbT* costs,\n                             const int* const flat_labels,\n                             const int* const label_lengths,\n                             const int* const input_lengths) {\n    if (activations == nullptr ||\n        grads == nullptr ||\n        costs == nullptr ||\n        flat_labels == nullptr ||\n        label_lengths == nullptr ||\n        input_lengths == nullptr\n        )\n        return CTC_STATUS_INVALID_VALUE;\n\n    return compute_cost_and_score(activations, grads, costs, flat_labels,\n                                  label_lengths, input_lengths, true, true);\n}\n\ntemplate<typename ProbT>\nctcStatus_t\nGpuCTC<ProbT>::score_forward(const ProbT* const activations,\n                             ProbT* costs,\n                             const int* const flat_labels,\n                             const int* const label_lengths,\n                             const int* const input_lengths) {\n    if (activations == nullptr ||\n        costs == nullptr ||\n        flat_labels == nullptr ||\n        label_lengths == nullptr ||\n        input_lengths == nullptr\n        )\n        return CTC_STATUS_INVALID_VALUE;\n\n    return compute_cost_and_score(activations, nullptr, costs, flat_labels,\n                                  label_lengths, input_lengths, true, false);\n}\n\n} // mxnet_warpctc\n"
  },
  {
    "path": "3rdparty/ctc_include/detail/gpu_ctc_kernels.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#pragma once\n\n#include \"../contrib/moderngpu/include/device/ctascan.cuh\"\n#include \"../contrib/moderngpu/include/device/ctamerge.cuh\"\n\n#include \"ctc_helper.h\"\n\nusing namespace mgpu;\n\ntemplate<int NT, int VT, typename T, typename KeyT, typename Op>\nstruct CTASegReduce {\n\n    enum {NV = NT * VT};\n\n    union Storage {\n        typename CTAScan<NT>::Storage scanStorage;\n        int indices[NV];\n    };\n\n    //adapted from global kernel KernelReduceByKeyPreprocess\n    __device__ static void preprocessKeys(KeyT *keys, int count,\n                                          int *numUniqueLabels, int seg_start[VT],\n                                          int seg_end[VT], int *scanout) {\n        __shared__ Storage shared;\n\n        const int tid = threadIdx.x;\n        // Compare adjacent keys within each thread and mark discontinuities\n        int endFlags = 0;\n        T key = keys[VT * tid];\n        #pragma unroll\n        for (int i = 0; i < VT; ++i) {\n            int index = VT * tid + 1 + i;\n            T next = keys[index];\n            if(index == count || (index < count && key != next)) {\n                endFlags |= 1 << i;\n            }\n            key = next;\n        }\n\n        __syncthreads();\n\n        //Count the number of encountered end flags\n        int scan = CTAScan<NT>::Scan(tid, popc(endFlags), shared.scanStorage, numUniqueLabels);\n\n        __syncthreads();\n\n        //output the unique keys\n        //use indices as scratch space\n        int outputPos = scan;\n        #pragma unroll\n        for (int i = 0; i < VT; ++i) {\n\n            if ( (endFlags >> i) & 1) {\n                shared.indices[outputPos] = keys[VT * tid + i];\n                scanout[outputPos] = VT * tid + i;\n                outputPos++;\n            }\n        }\n\n        __syncthreads();\n\n        // Create start and end\n        for (int idx = tid, j = 0; idx < (*numUniqueLabels); idx += blockDim.x, ++j) {\n            seg_start[j] = (idx == 0) ? 0 : (scanout[idx-1] + 1);\n            seg_end[j] = scanout[idx];\n        }\n\n        __syncthreads();\n\n        //copy from the scratch space back into the keys\n        #pragma unroll\n        for (int i = 0; i < VT; ++i) {\n            keys[i * NT + tid] = shared.indices[i * NT + tid];\n        }\n\n        __syncthreads();\n    }\n};\n\n// Computes forward probabilities. This fills in a T * S matrix.\n// The computation starts at t=1 (2nd row) and ends at t=T-1 (last row). Each row has\n// S elements where S = 2L + 1.\n//\n// We only need to read in probabilities corresponding to the labels, thus a sparse\n// set of values are read from the log probs matrix since the character set is much smaller\n// than the labels. This is much more true for Mandarin than English.\ntemplate<typename ProbT, int NT, int VT>\n__global__\nvoid compute_alpha_kernel (const ProbT* log_probs, const int *label_sizes,\n                           const int *utt_length, const int *repeats_in_labels,\n                           const int *labels_without_blanks, const int *label_offsets, \n                           int *labels_with_blanks, ProbT *alphas, \n                           ProbT* nll_forward, int stride, int out_dim,\n                           int S_memoffset, int T_memoffset, int blank_label) {\n\n    ctc_helper::log_plus<ProbT> log_plus_f;\n\n    const int tid = threadIdx.x;\n    const int L = label_sizes[blockIdx.x];\n    const int T = utt_length[blockIdx.x];\n    const int S = 2*L + 1;\n    const int prob_offset = out_dim * blockIdx.x;\n    const int repeats = repeats_in_labels[blockIdx.x];\n\n    const int NV = NT * VT;\n    __shared__ int label[NV];\n\n    if ((L + repeats) > T)\n        return;\n\n    // Generate labels with blanks from labels without blanks\n    {\n        const int label_start_offset = label_offsets[blockIdx.x];\n        for (int idx = tid; idx < L; idx += blockDim.x) {\n            const int offset = (blockIdx.x * S_memoffset) + 2 * idx;\n            labels_with_blanks[offset] = blank_label;\n            labels_with_blanks[offset+1] = labels_without_blanks[label_start_offset + idx];\n        }\n        if (tid == 0) {\n            labels_with_blanks[(blockIdx.x * S_memoffset) + 2 * L] = blank_label;\n        }\n    }\n    __syncthreads();\n\n    const int *labels = labels_with_blanks;\n    const int* label_global = &labels[blockIdx.x * S_memoffset];\n    ProbT* alpha = &alphas[blockIdx.x * (S_memoffset * T_memoffset)];\n\n    // Set the first row of alpha neg_inf - it is much more efficient to do it\n    // here than outside\n    #pragma unroll\n    for (int idx = tid; idx < min(S, NV); idx += blockDim.x) {\n        alpha[idx] = ctc_helper::neg_inf<ProbT>();\n    }\n\n    // Load labels into shared memory\n    #pragma unroll\n    for (int i = tid; i < S; i += NT) {\n        label[i] = label_global[i];\n    }\n\n    __syncthreads();\n\n    int start =  (L + repeats < T) ? 0 : 1;\n    int end = S > 1 ? 2 : 1;\n\n    // Initialize the first row corresponding to t=0;\n    for(int i = tid; i < (end-start); i += blockDim.x)\n        alpha[i + start] = log_probs[prob_offset + label[i + start]];\n\n    __syncthreads();\n\n    // Fill in the rest of matrix, one row at a time (outer loop).\n    for(int t = 1; t < T; ++t) {\n\n        // Start offsets into the current and previous row\n        const int start_cur_row = t * S;\n        const int start_prev_row = (t - 1) * S;\n\n        // The prob is a 2D column major array, with probabilites for each t strided\n        // by (out_dim * stride), where stride is the minibatch size\n        const int start_prob_col = t * (out_dim * stride);\n\n        // This is the first column and in this case there is nothing left of it\n        if (tid == 0) {\n            if (start == 0) {\n                alpha[start_cur_row] = alpha[start_prev_row] +\n                                       log_probs[prob_offset + start_prob_col + blank_label];\n            }\n            else if (start == 1) {\n                alpha[start_cur_row] = alpha[start_prev_row];\n            }\n        }\n\n        __syncthreads();\n\n        // Fill in the elements in each row. There is no loop dependence here since our\n        // input is the row above. We sum either two or three adjacent values from the\n        // row above depending on whether we have a blank or repeated characters. Finally\n        // we add the probability corresponding to this label at time t\n        #pragma unroll\n        for (int idx = (tid+1); idx < S; idx += blockDim.x) {\n\n            ProbT prev_sum = log_plus_f(alpha[idx + start_prev_row], alpha[(idx-1) + start_prev_row]);\n\n            // Skip two if not on blank and not on repeat.\n            if ((label[idx] != blank_label) &&\n                (idx != 1) && (label[idx] != label[idx-2]))\n                prev_sum = log_plus_f(prev_sum, alpha[(idx-2) + start_prev_row]);\n\n            alpha[idx + start_cur_row] =\n                prev_sum + log_probs[prob_offset + start_prob_col + label[idx]];\n        }\n\n        __syncthreads();\n    }\n\n    if (tid == 0) {\n        // Add and return the rightmost two/one element(s) in the last row.\n        ProbT loglike = ctc_helper::neg_inf<ProbT>();\n\n        // This is the total increment for s_inc and e_inc through the loop\n        const int val = 2 * (L-1) + 1 - (((L + repeats) == T) ? 1 : 0);\n\n        start = (val * (L!=0) + start);\n        end = (val * (L!=0) + end);\n\n        for(int i = start; i < end; ++i)\n            loglike = log_plus_f(loglike, alpha[i + (T - 1) * S]);\n\n        nll_forward[blockIdx.x] = -loglike;\n    }\n}\n\n// Computes backward probabilities. This also fills in a T * S matrix\n//\n// See comments above compute_alphas for more context.\ntemplate<typename ProbT, int NT, int VT>\n__global__\nvoid compute_betas_and_grad_kernel (const ProbT* log_probs, const int *label_sizes,\n                                    const int *utt_length, const int *repeats_in_labels,\n                                    const int *labels_with_blanks, ProbT *alphas,\n                                    const ProbT* nll_forward, ProbT *nll_backward,\n                                    ProbT *grads, int stride, int out_dim,\n                                    int S_memoffset, int T_memoffset, int blank_label) {\n\n    ctc_helper::log_plus<ProbT> log_plus_f;\n    typedef CTASegReduce<NT, VT, ProbT, int, ctc_helper::log_plus<ProbT>> SegReduce;\n\n    const int tid = threadIdx.x;\n    const int L = label_sizes[blockIdx.x];\n    const int T = utt_length[blockIdx.x];\n    const int S = 2*L + 1;\n    const int prob_offset = out_dim * blockIdx.x;\n    const int repeats = repeats_in_labels[blockIdx.x];\n    const ProbT log_partition = -nll_forward[blockIdx.x];\n\n    const int* labels = labels_with_blanks;\n    const int* label_global = &labels[blockIdx.x * S_memoffset];\n    ProbT* alpha = &alphas[blockIdx.x * (S_memoffset * T_memoffset)];\n\n    const int NV = NT * VT;\n\n    union TempStorage {\n        ProbT beta[NV];\n        int result[NV];\n    };\n\n    __shared__ TempStorage temp_buffer;\n\n    __shared__ int label[NV];\n\n    // Temporaries needed for segmented reduce\n    // TODO: see if we can combine the shared memory requirements\n    __shared__ int keys_shared[NV];\n    __shared__ int gather_indices[NV];\n    __shared__ ProbT output[NV];\n\n    ProbT beta_val[VT];\n\n    if ((L + repeats) > T)\n        return;\n\n    int start = S > 1 ? (S - 2) : 0;\n    int end = (L + repeats < T) ? S : S-1;\n\n    // Setup shared memory buffers\n    #pragma unroll\n    for (int idx = tid; idx < NV; idx += NT) {\n        label[idx] = (idx < S) ? label_global[idx] : INT_MAX;\n    }\n\n    __syncthreads();\n\n    // int flags;\n    int uniquelabels;\n    int seg_start[VT];\n    int seg_end[VT];\n\n    // Sort labels and record indices from which to gather from\n    {\n        int key[VT];\n        int gather_val[VT];\n\n        #pragma unroll\n        for (int i = 0; i < VT; ++i) {\n            const int idx = tid * VT + i;\n            gather_val[i] = idx;\n            key[i] = label[idx];\n        }\n\n        __syncthreads();\n\n        CTAMergesort<NT, VT, true, true, int, int, mgpu::less<int>>\n            (key, gather_val, keys_shared, gather_indices, S, tid, mgpu::less<int>());\n\n        __syncthreads();\n\n        for (int i = 0; i < VT; ++i) {\n            const int idx = tid * VT + i;\n            gather_indices[idx] = gather_val[i];\n        }\n\n        __syncthreads();\n\n        SegReduce::preprocessKeys(keys_shared, S, &uniquelabels, seg_start, seg_end,\n                                  temp_buffer.result);\n        __syncthreads();\n    }\n\n    // TODO: probably not necessary\n    __syncthreads();\n\n    // Load labels back\n    #pragma unroll\n    for (int idx = tid; idx < NV; idx += NT) {\n        temp_buffer.beta[idx] = ctc_helper::neg_inf<ProbT>();\n    }\n    __syncthreads();\n\n    // Initialize the two rightmost values in the last row (assuming L non-zero)\n    for(int i = tid; i < (end-start); i += blockDim.x)\n        temp_buffer.beta[i + start] =\n            log_probs[prob_offset + (T - 1) * (out_dim * stride) + label[i + start]];\n\n    __syncthreads();\n\n    // Load output data in registers through the transpose trick - should really be a function\n    #pragma unroll\n    for (int idx = tid; idx < S; idx += NT) {\n        output[idx] = alpha[idx + (T - 1) * S] + temp_buffer.beta[idx];\n    }\n\n    __syncthreads();\n\n    // Start at the second to last row and backward in time\n    for(int t = T - 1; t >= 0; --t) {\n\n        // Start offsets into the current and next row\n        const int start_cur_row = t * S;\n\n        // Starting offset of column that we read from the log probs array\n        const int start_prob_col = t * (out_dim * stride);\n\n        if (t < T-1) {\n\n            // Filling up one row at at time but going back in time from the last row\n            // to the first. As in the forward pass, there is no loop dependence and we\n            // do a variable length filter of maximum filter size of 3\n            #pragma unroll\n            for(int idx = tid, i = 0; idx < (S-1); idx += NT, i++) {\n                ProbT next_sum = log_plus_f(temp_buffer.beta[idx], temp_buffer.beta[idx+1]);\n\n                    // Skip two if not on blank and not on repeat.\n                if ((label[idx] != blank_label) &&\n                    (idx != (S-2)) && (label[idx] != label[idx+2]))\n                    next_sum = log_plus_f(next_sum, temp_buffer.beta[idx+2]);\n\n                beta_val[i] = next_sum + log_probs[prob_offset + start_prob_col + label[idx]];\n            }\n\n            __syncthreads();\n\n            // Initialize values for the rightmost column since there is nothing to the right\n            // Update input buffer for next iteration\n            if ((tid == 0) && (end == S))\n                temp_buffer.beta[(S-1)] = temp_buffer.beta[(S-1)] +\n                                          log_probs[prob_offset + start_prob_col + blank_label];\n\n            #pragma unroll\n            for(int idx = tid, i = 0; idx < (S-1); idx += NT, i++) {\n               temp_buffer.beta[idx] = beta_val[i];\n            }\n\n            __syncthreads();\n\n            // Beta Computation done - add to alpha and update the gradient. Reload\n            // the gradient back for segmented reduce later on\n            #pragma unroll\n            for(int idx = tid; idx < S; idx += NT) {\n               output[idx] = alpha[idx + start_cur_row] + temp_buffer.beta[idx];\n            }\n\n            __syncthreads();\n\n        }\n\n        __syncthreads();\n\n        // Compute segmented reduction of output by using label as key\n        {\n            // Somewhat faster key value reduce\n            ProbT accum[VT];\n\n            for (int idx = tid, j = 0; idx < uniquelabels; idx += blockDim.x, ++j) {\n\n                accum[j] = ctc_helper::neg_inf<ProbT>();\n                for (int i = seg_start[j]; i <= seg_end[j]; ++i) {\n                    accum[j] = log_plus_f(accum[j], output[gather_indices[i]]);\n                }\n            }\n            __syncthreads();\n\n            // Write accumulated value into output since that is not used\n            for (int idx = tid, j = 0; idx < uniquelabels; idx += blockDim.x, ++j) {\n                output[idx] = accum[j];\n            }\n            __syncthreads();\n\n            for (int idx = tid; idx < out_dim; idx += blockDim.x) {\n                const int grads_offset = prob_offset + start_prob_col + idx;\n                grads[grads_offset] = exp(log_probs[grads_offset]);\n            }\n\n            __syncthreads();\n\n            for (int idx = tid; idx < uniquelabels; idx += blockDim.x) {\n                const int grads_offset = prob_offset + start_prob_col + keys_shared[idx];\n\n                ProbT grad = output[idx];\n\n                if ((grad == 0.0) || (log_probs[grads_offset] == ctc_helper::neg_inf<ProbT>()) ||\n                    (grad == ctc_helper::neg_inf<ProbT>())) {\n                } else {\n                    grads[grads_offset] =\n                        exp(log_probs[grads_offset]) - exp(grad - log_probs[grads_offset] - log_partition);\n                }\n            }\n\n            __syncthreads();\n        }\n\n        // Output backward log likelihood\n        if ((t == 0) && (tid == 0)) {\n            ProbT loglike = ctc_helper::neg_inf<ProbT>();\n\n            const int val = 2 * (L-1) + 1 - (((L + repeats) == T) ? 1 : 0);\n\n            start = (-val * (L != 0) + start);\n            end = (-val * (L != 0) + end);\n\n            // Sum and return the leftmost one/two value(s) in first row\n            for(int i = start; i < end; ++i)\n                loglike = log_plus_f(loglike, temp_buffer.beta[i]);\n\n            nll_backward[blockIdx.x] = -loglike;\n        }\n\n        // For some reason this is important\n        __syncthreads();\n    }\n}\n\ntemplate <typename ProbT, int VT = 1, typename Op>\n__global__ void compute_log_probs_kernel(Op f, ProbT* log_probs,\n                                     const ProbT* const denom,\n                                     int alphabet_size,\n                                     int count) {\n\n    int idx = blockDim.x * blockIdx.x + threadIdx.x;\n    int stride = blockDim.x * gridDim.x;\n#pragma unroll\n    for(int i = 0; i < VT; i++) {\n        if (idx < count) {\n            const int column_idx = idx / alphabet_size;\n            log_probs[idx] = log_probs[idx] - log(denom[column_idx]);\n        }\n        idx += stride;\n    }\n}\n\ntemplate <typename ProbT, int VT = 1, typename Op>\n__global__ void prepare_stable_LSM_kernel(Op f, ProbT* log_probs,\n                                         const ProbT* const col_max,\n                                         int alphabet_size,\n                                         int count) {\n\n    int idx = blockDim.x * blockIdx.x + threadIdx.x;\n    int stride = blockDim.x * gridDim.x;\n#pragma unroll\n    for(int i = 0; i < VT; i++) {\n        if (idx < count) {\n            const int column_idx = idx / alphabet_size;\n            log_probs[idx] = f(log_probs[idx] - col_max[column_idx]);\n        }\n        idx += stride;\n    }\n}\n"
  },
  {
    "path": "3rdparty/ctc_include/detail/hostdevice.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n\n#pragma once\n\n#ifdef __CUDACC__\n    #define HOSTDEVICE __host__ __device__\n#else\n    #define HOSTDEVICE\n#endif\n"
  },
  {
    "path": "3rdparty/miniz/miniz.c",
    "content": "/**************************************************************************\n *\n * Copyright 2013-2014 RAD Game Tools and Valve Software\n * Copyright 2010-2014 Rich Geldreich and Tenacious Software LLC\n * All Rights Reserved.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n *\n **************************************************************************/\n\n#include  \"miniz.h\"\n\ntypedef unsigned char mz_validate_uint16[sizeof(mz_uint16) == 2 ? 1 : -1];\ntypedef unsigned char mz_validate_uint32[sizeof(mz_uint32) == 4 ? 1 : -1];\ntypedef unsigned char mz_validate_uint64[sizeof(mz_uint64) == 8 ? 1 : -1];\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n/* ------------------- zlib-style API's */\n\nmz_ulong mz_adler32(mz_ulong adler, const unsigned char *ptr, size_t buf_len)\n{\n    mz_uint32 i, s1 = (mz_uint32)(adler & 0xffff), s2 = (mz_uint32)(adler >> 16);\n    size_t block_len = buf_len % 5552;\n    if (!ptr)\n        return MZ_ADLER32_INIT;\n    while (buf_len)\n    {\n        for (i = 0; i + 7 < block_len; i += 8, ptr += 8)\n        {\n            s1 += ptr[0], s2 += s1;\n            s1 += ptr[1], s2 += s1;\n            s1 += ptr[2], s2 += s1;\n            s1 += ptr[3], s2 += s1;\n            s1 += ptr[4], s2 += s1;\n            s1 += ptr[5], s2 += s1;\n            s1 += ptr[6], s2 += s1;\n            s1 += ptr[7], s2 += s1;\n        }\n        for (; i < block_len; ++i)\n            s1 += *ptr++, s2 += s1;\n        s1 %= 65521U, s2 %= 65521U;\n        buf_len -= block_len;\n        block_len = 5552;\n    }\n    return (s2 << 16) + s1;\n}\n\n/* Karl Malbrain's compact CRC-32. See \"A compact CCITT crc16 and crc32 C implementation that balances processor cache usage against speed\": http://www.geocities.com/malbrain/ */\n#if 0\n    mz_ulong mz_crc32(mz_ulong crc, const mz_uint8 *ptr, size_t buf_len)\n    {\n        static const mz_uint32 s_crc32[16] = { 0, 0x1db71064, 0x3b6e20c8, 0x26d930ac, 0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c,\n                                               0xedb88320, 0xf00f9344, 0xd6d6a3e8, 0xcb61b38c, 0x9b64c2b0, 0x86d3d2d4, 0xa00ae278, 0xbdbdf21c };\n        mz_uint32 crcu32 = (mz_uint32)crc;\n        if (!ptr)\n            return MZ_CRC32_INIT;\n        crcu32 = ~crcu32;\n        while (buf_len--)\n        {\n            mz_uint8 b = *ptr++;\n            crcu32 = (crcu32 >> 4) ^ s_crc32[(crcu32 & 0xF) ^ (b & 0xF)];\n            crcu32 = (crcu32 >> 4) ^ s_crc32[(crcu32 & 0xF) ^ (b >> 4)];\n        }\n        return ~crcu32;\n    }\n#else\n/* Faster, but larger CPU cache footprint.\n */\nmz_ulong mz_crc32(mz_ulong crc, const mz_uint8 *ptr, size_t buf_len)\n{\n    static const mz_uint32 s_crc_table[256] =\n        {\n          0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, 0xE963A535,\n          0x9E6495A3, 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD,\n          0xE7B82D07, 0x90BF1D91, 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, 0x1ADAD47D,\n          0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC,\n          0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, 0x3B6E20C8, 0x4C69105E, 0xD56041E4,\n          0xA2677172, 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, 0x35B5A8FA, 0x42B2986C,\n          0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, 0x26D930AC,\n          0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F,\n          0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, 0x2F6F7C87, 0x58684C11, 0xC1611DAB,\n          0xB6662D3D, 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F,\n          0x9FBFE4A5, 0xE8B8D433, 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB,\n          0x086D3D2D, 0x91646C97, 0xE6635C01, 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E,\n          0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA,\n          0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, 0x4DB26158, 0x3AB551CE,\n          0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, 0x4369E96A,\n          0x346ED9FC, 0xAD678846, 0xDA60B8D0, 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9,\n          0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409,\n          0xCE61E49F, 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81,\n          0xB7BD5C3B, 0xC0BA6CAD, 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, 0xEAD54739,\n          0x9DD277AF, 0x04DB2615, 0x73DC1683, 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8,\n          0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, 0xF00F9344, 0x8708A3D2, 0x1E01F268,\n          0x6906C2FE, 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, 0xFED41B76, 0x89D32BE0,\n          0x10DA7A5A, 0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, 0xD6D6A3E8,\n          0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B,\n          0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF,\n          0x4669BE79, 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, 0xCC0C7795, 0xBB0B4703,\n          0x220216B9, 0x5505262F, 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7,\n          0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A,\n          0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE,\n          0x0CB61B38, 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, 0x86D3D2D4, 0xF1D4E242,\n          0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, 0x88085AE6,\n          0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45,\n          0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7, 0x4969474D,\n          0x3E6E77DB, 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5,\n          0x47B2CF7F, 0x30B5FFE9, 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605,\n          0xCDD70693, 0x54DE5729, 0x23D967BF, 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94,\n          0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D\n        };\n\n    mz_uint32 crc32 = (mz_uint32)crc ^ 0xFFFFFFFF;\n    const mz_uint8 *pByte_buf = (const mz_uint8 *)ptr;\n\n    while (buf_len >= 4)\n    {\n        crc32 = (crc32 >> 8) ^ s_crc_table[(crc32 ^ pByte_buf[0]) & 0xFF];\n        crc32 = (crc32 >> 8) ^ s_crc_table[(crc32 ^ pByte_buf[1]) & 0xFF];\n        crc32 = (crc32 >> 8) ^ s_crc_table[(crc32 ^ pByte_buf[2]) & 0xFF];\n        crc32 = (crc32 >> 8) ^ s_crc_table[(crc32 ^ pByte_buf[3]) & 0xFF];\n        pByte_buf += 4;\n        buf_len -= 4;\n    }\n\n    while (buf_len)\n    {\n        crc32 = (crc32 >> 8) ^ s_crc_table[(crc32 ^ pByte_buf[0]) & 0xFF];\n        ++pByte_buf;\n        --buf_len;\n    }\n\n    return ~crc32;\n}\n#endif\n\nvoid mz_free(void *p)\n{\n    MZ_FREE(p);\n}\n\nvoid *miniz_def_alloc_func(void *opaque, size_t items, size_t size)\n{\n    (void)opaque, (void)items, (void)size;\n    return MZ_MALLOC(items * size);\n}\nvoid miniz_def_free_func(void *opaque, void *address)\n{\n    (void)opaque, (void)address;\n    MZ_FREE(address);\n}\nvoid *miniz_def_realloc_func(void *opaque, void *address, size_t items, size_t size)\n{\n    (void)opaque, (void)address, (void)items, (void)size;\n    return MZ_REALLOC(address, items * size);\n}\n\nconst char *mz_version(void)\n{\n    return MZ_VERSION;\n}\n\n#ifndef MINIZ_NO_ZLIB_APIS\n\nint mz_deflateInit(mz_streamp pStream, int level)\n{\n    return mz_deflateInit2(pStream, level, MZ_DEFLATED, MZ_DEFAULT_WINDOW_BITS, 9, MZ_DEFAULT_STRATEGY);\n}\n\nint mz_deflateInit2(mz_streamp pStream, int level, int method, int window_bits, int mem_level, int strategy)\n{\n    tdefl_compressor *pComp;\n    mz_uint comp_flags = TDEFL_COMPUTE_ADLER32 | tdefl_create_comp_flags_from_zip_params(level, window_bits, strategy);\n\n    if (!pStream)\n        return MZ_STREAM_ERROR;\n    if ((method != MZ_DEFLATED) || ((mem_level < 1) || (mem_level > 9)) || ((window_bits != MZ_DEFAULT_WINDOW_BITS) && (-window_bits != MZ_DEFAULT_WINDOW_BITS)))\n        return MZ_PARAM_ERROR;\n\n    pStream->data_type = 0;\n    pStream->adler = MZ_ADLER32_INIT;\n    pStream->msg = NULL;\n    pStream->reserved = 0;\n    pStream->total_in = 0;\n    pStream->total_out = 0;\n    if (!pStream->zalloc)\n        pStream->zalloc = miniz_def_alloc_func;\n    if (!pStream->zfree)\n        pStream->zfree = miniz_def_free_func;\n\n    pComp = (tdefl_compressor *)pStream->zalloc(pStream->opaque, 1, sizeof(tdefl_compressor));\n    if (!pComp)\n        return MZ_MEM_ERROR;\n\n    pStream->state = (struct mz_internal_state *)pComp;\n\n    if (tdefl_init(pComp, NULL, NULL, comp_flags) != TDEFL_STATUS_OKAY)\n    {\n        mz_deflateEnd(pStream);\n        return MZ_PARAM_ERROR;\n    }\n\n    return MZ_OK;\n}\n\nint mz_deflateReset(mz_streamp pStream)\n{\n    if ((!pStream) || (!pStream->state) || (!pStream->zalloc) || (!pStream->zfree))\n        return MZ_STREAM_ERROR;\n    pStream->total_in = pStream->total_out = 0;\n    tdefl_init((tdefl_compressor *)pStream->state, NULL, NULL, ((tdefl_compressor *)pStream->state)->m_flags);\n    return MZ_OK;\n}\n\nint mz_deflate(mz_streamp pStream, int flush)\n{\n    size_t in_bytes, out_bytes;\n    mz_ulong orig_total_in, orig_total_out;\n    int mz_status = MZ_OK;\n\n    if ((!pStream) || (!pStream->state) || (flush < 0) || (flush > MZ_FINISH) || (!pStream->next_out))\n        return MZ_STREAM_ERROR;\n    if (!pStream->avail_out)\n        return MZ_BUF_ERROR;\n\n    if (flush == MZ_PARTIAL_FLUSH)\n        flush = MZ_SYNC_FLUSH;\n\n    if (((tdefl_compressor *)pStream->state)->m_prev_return_status == TDEFL_STATUS_DONE)\n        return (flush == MZ_FINISH) ? MZ_STREAM_END : MZ_BUF_ERROR;\n\n    orig_total_in = pStream->total_in;\n    orig_total_out = pStream->total_out;\n    for (;;)\n    {\n        tdefl_status defl_status;\n        in_bytes = pStream->avail_in;\n        out_bytes = pStream->avail_out;\n\n        defl_status = tdefl_compress((tdefl_compressor *)pStream->state, pStream->next_in, &in_bytes, pStream->next_out, &out_bytes, (tdefl_flush)flush);\n        pStream->next_in += (mz_uint)in_bytes;\n        pStream->avail_in -= (mz_uint)in_bytes;\n        pStream->total_in += (mz_uint)in_bytes;\n        pStream->adler = tdefl_get_adler32((tdefl_compressor *)pStream->state);\n\n        pStream->next_out += (mz_uint)out_bytes;\n        pStream->avail_out -= (mz_uint)out_bytes;\n        pStream->total_out += (mz_uint)out_bytes;\n\n        if (defl_status < 0)\n        {\n            mz_status = MZ_STREAM_ERROR;\n            break;\n        }\n        else if (defl_status == TDEFL_STATUS_DONE)\n        {\n            mz_status = MZ_STREAM_END;\n            break;\n        }\n        else if (!pStream->avail_out)\n            break;\n        else if ((!pStream->avail_in) && (flush != MZ_FINISH))\n        {\n            if ((flush) || (pStream->total_in != orig_total_in) || (pStream->total_out != orig_total_out))\n                break;\n            return MZ_BUF_ERROR; /* Can't make forward progress without some input.\n */\n        }\n    }\n    return mz_status;\n}\n\nint mz_deflateEnd(mz_streamp pStream)\n{\n    if (!pStream)\n        return MZ_STREAM_ERROR;\n    if (pStream->state)\n    {\n        pStream->zfree(pStream->opaque, pStream->state);\n        pStream->state = NULL;\n    }\n    return MZ_OK;\n}\n\nmz_ulong mz_deflateBound(mz_streamp pStream, mz_ulong source_len)\n{\n    (void)pStream;\n    /* This is really over conservative. (And lame, but it's actually pretty tricky to compute a true upper bound given the way tdefl's blocking works.) */\n    return MZ_MAX(128 + (source_len * 110) / 100, 128 + source_len + ((source_len / (31 * 1024)) + 1) * 5);\n}\n\nint mz_compress2(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len, int level)\n{\n    int status;\n    mz_stream stream;\n    memset(&stream, 0, sizeof(stream));\n\n    /* In case mz_ulong is 64-bits (argh I hate longs). */\n    if ((source_len | *pDest_len) > 0xFFFFFFFFU)\n        return MZ_PARAM_ERROR;\n\n    stream.next_in = pSource;\n    stream.avail_in = (mz_uint32)source_len;\n    stream.next_out = pDest;\n    stream.avail_out = (mz_uint32)*pDest_len;\n\n    status = mz_deflateInit(&stream, level);\n    if (status != MZ_OK)\n        return status;\n\n    status = mz_deflate(&stream, MZ_FINISH);\n    if (status != MZ_STREAM_END)\n    {\n        mz_deflateEnd(&stream);\n        return (status == MZ_OK) ? MZ_BUF_ERROR : status;\n    }\n\n    *pDest_len = stream.total_out;\n    return mz_deflateEnd(&stream);\n}\n\nint mz_compress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len)\n{\n    return mz_compress2(pDest, pDest_len, pSource, source_len, MZ_DEFAULT_COMPRESSION);\n}\n\nmz_ulong mz_compressBound(mz_ulong source_len)\n{\n    return mz_deflateBound(NULL, source_len);\n}\n\ntypedef struct\n{\n    tinfl_decompressor m_decomp;\n    mz_uint m_dict_ofs, m_dict_avail, m_first_call, m_has_flushed;\n    int m_window_bits;\n    mz_uint8 m_dict[TINFL_LZ_DICT_SIZE];\n    tinfl_status m_last_status;\n} inflate_state;\n\nint mz_inflateInit2(mz_streamp pStream, int window_bits)\n{\n    inflate_state *pDecomp;\n    if (!pStream)\n        return MZ_STREAM_ERROR;\n    if ((window_bits != MZ_DEFAULT_WINDOW_BITS) && (-window_bits != MZ_DEFAULT_WINDOW_BITS))\n        return MZ_PARAM_ERROR;\n\n    pStream->data_type = 0;\n    pStream->adler = 0;\n    pStream->msg = NULL;\n    pStream->total_in = 0;\n    pStream->total_out = 0;\n    pStream->reserved = 0;\n    if (!pStream->zalloc)\n        pStream->zalloc = miniz_def_alloc_func;\n    if (!pStream->zfree)\n        pStream->zfree = miniz_def_free_func;\n\n    pDecomp = (inflate_state *)pStream->zalloc(pStream->opaque, 1, sizeof(inflate_state));\n    if (!pDecomp)\n        return MZ_MEM_ERROR;\n\n    pStream->state = (struct mz_internal_state *)pDecomp;\n\n    tinfl_init(&pDecomp->m_decomp);\n    pDecomp->m_dict_ofs = 0;\n    pDecomp->m_dict_avail = 0;\n    pDecomp->m_last_status = TINFL_STATUS_NEEDS_MORE_INPUT;\n    pDecomp->m_first_call = 1;\n    pDecomp->m_has_flushed = 0;\n    pDecomp->m_window_bits = window_bits;\n\n    return MZ_OK;\n}\n\nint mz_inflateInit(mz_streamp pStream)\n{\n    return mz_inflateInit2(pStream, MZ_DEFAULT_WINDOW_BITS);\n}\n\nint mz_inflateReset(mz_streamp pStream)\n{\n    inflate_state *pDecomp;\n    if (!pStream)\n        return MZ_STREAM_ERROR;\n\n    pStream->data_type = 0;\n    pStream->adler = 0;\n    pStream->msg = NULL;\n    pStream->total_in = 0;\n    pStream->total_out = 0;\n    pStream->reserved = 0;\n\n    pDecomp = (inflate_state *)pStream->state;\n\n    tinfl_init(&pDecomp->m_decomp);\n    pDecomp->m_dict_ofs = 0;\n    pDecomp->m_dict_avail = 0;\n    pDecomp->m_last_status = TINFL_STATUS_NEEDS_MORE_INPUT;\n    pDecomp->m_first_call = 1;\n    pDecomp->m_has_flushed = 0;\n    /* pDecomp->m_window_bits = window_bits */;\n\n    return MZ_OK;\n}\n\nint mz_inflate(mz_streamp pStream, int flush)\n{\n    inflate_state *pState;\n    mz_uint n, first_call, decomp_flags = TINFL_FLAG_COMPUTE_ADLER32;\n    size_t in_bytes, out_bytes, orig_avail_in;\n    tinfl_status status;\n\n    if ((!pStream) || (!pStream->state))\n        return MZ_STREAM_ERROR;\n    if (flush == MZ_PARTIAL_FLUSH)\n        flush = MZ_SYNC_FLUSH;\n    if ((flush) && (flush != MZ_SYNC_FLUSH) && (flush != MZ_FINISH))\n        return MZ_STREAM_ERROR;\n\n    pState = (inflate_state *)pStream->state;\n    if (pState->m_window_bits > 0)\n        decomp_flags |= TINFL_FLAG_PARSE_ZLIB_HEADER;\n    orig_avail_in = pStream->avail_in;\n\n    first_call = pState->m_first_call;\n    pState->m_first_call = 0;\n    if (pState->m_last_status < 0)\n        return MZ_DATA_ERROR;\n\n    if (pState->m_has_flushed && (flush != MZ_FINISH))\n        return MZ_STREAM_ERROR;\n    pState->m_has_flushed |= (flush == MZ_FINISH);\n\n    if ((flush == MZ_FINISH) && (first_call))\n    {\n        /* MZ_FINISH on the first call implies that the input and output buffers are large enough to hold the entire compressed/decompressed file. */\n        decomp_flags |= TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF;\n        in_bytes = pStream->avail_in;\n        out_bytes = pStream->avail_out;\n        status = tinfl_decompress(&pState->m_decomp, pStream->next_in, &in_bytes, pStream->next_out, pStream->next_out, &out_bytes, decomp_flags);\n        pState->m_last_status = status;\n        pStream->next_in += (mz_uint)in_bytes;\n        pStream->avail_in -= (mz_uint)in_bytes;\n        pStream->total_in += (mz_uint)in_bytes;\n        pStream->adler = tinfl_get_adler32(&pState->m_decomp);\n        pStream->next_out += (mz_uint)out_bytes;\n        pStream->avail_out -= (mz_uint)out_bytes;\n        pStream->total_out += (mz_uint)out_bytes;\n\n        if (status < 0)\n            return MZ_DATA_ERROR;\n        else if (status != TINFL_STATUS_DONE)\n        {\n            pState->m_last_status = TINFL_STATUS_FAILED;\n            return MZ_BUF_ERROR;\n        }\n        return MZ_STREAM_END;\n    }\n    /* flush != MZ_FINISH then we must assume there's more input. */\n    if (flush != MZ_FINISH)\n        decomp_flags |= TINFL_FLAG_HAS_MORE_INPUT;\n\n    if (pState->m_dict_avail)\n    {\n        n = MZ_MIN(pState->m_dict_avail, pStream->avail_out);\n        memcpy(pStream->next_out, pState->m_dict + pState->m_dict_ofs, n);\n        pStream->next_out += n;\n        pStream->avail_out -= n;\n        pStream->total_out += n;\n        pState->m_dict_avail -= n;\n        pState->m_dict_ofs = (pState->m_dict_ofs + n) & (TINFL_LZ_DICT_SIZE - 1);\n        return ((pState->m_last_status == TINFL_STATUS_DONE) && (!pState->m_dict_avail)) ? MZ_STREAM_END : MZ_OK;\n    }\n\n    for (;;)\n    {\n        in_bytes = pStream->avail_in;\n        out_bytes = TINFL_LZ_DICT_SIZE - pState->m_dict_ofs;\n\n        status = tinfl_decompress(&pState->m_decomp, pStream->next_in, &in_bytes, pState->m_dict, pState->m_dict + pState->m_dict_ofs, &out_bytes, decomp_flags);\n        pState->m_last_status = status;\n\n        pStream->next_in += (mz_uint)in_bytes;\n        pStream->avail_in -= (mz_uint)in_bytes;\n        pStream->total_in += (mz_uint)in_bytes;\n        pStream->adler = tinfl_get_adler32(&pState->m_decomp);\n\n        pState->m_dict_avail = (mz_uint)out_bytes;\n\n        n = MZ_MIN(pState->m_dict_avail, pStream->avail_out);\n        memcpy(pStream->next_out, pState->m_dict + pState->m_dict_ofs, n);\n        pStream->next_out += n;\n        pStream->avail_out -= n;\n        pStream->total_out += n;\n        pState->m_dict_avail -= n;\n        pState->m_dict_ofs = (pState->m_dict_ofs + n) & (TINFL_LZ_DICT_SIZE - 1);\n\n        if (status < 0)\n            return MZ_DATA_ERROR; /* Stream is corrupted (there could be some uncompressed data left in the output dictionary - oh well). */\n        else if ((status == TINFL_STATUS_NEEDS_MORE_INPUT) && (!orig_avail_in))\n            return MZ_BUF_ERROR; /* Signal caller that we can't make forward progress without supplying more input or by setting flush to MZ_FINISH. */\n        else if (flush == MZ_FINISH)\n        {\n            /* The output buffer MUST be large to hold the remaining uncompressed data when flush==MZ_FINISH. */\n            if (status == TINFL_STATUS_DONE)\n                return pState->m_dict_avail ? MZ_BUF_ERROR : MZ_STREAM_END;\n            /* status here must be TINFL_STATUS_HAS_MORE_OUTPUT, which means there's at least 1 more byte on the way. If there's no more room left in the output buffer then something is wrong. */\n            else if (!pStream->avail_out)\n                return MZ_BUF_ERROR;\n        }\n        else if ((status == TINFL_STATUS_DONE) || (!pStream->avail_in) || (!pStream->avail_out) || (pState->m_dict_avail))\n            break;\n    }\n\n    return ((status == TINFL_STATUS_DONE) && (!pState->m_dict_avail)) ? MZ_STREAM_END : MZ_OK;\n}\n\nint mz_inflateEnd(mz_streamp pStream)\n{\n    if (!pStream)\n        return MZ_STREAM_ERROR;\n    if (pStream->state)\n    {\n        pStream->zfree(pStream->opaque, pStream->state);\n        pStream->state = NULL;\n    }\n    return MZ_OK;\n}\n\nint mz_uncompress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len)\n{\n    mz_stream stream;\n    int status;\n    memset(&stream, 0, sizeof(stream));\n\n    /* In case mz_ulong is 64-bits (argh I hate longs). */\n    if ((source_len | *pDest_len) > 0xFFFFFFFFU)\n        return MZ_PARAM_ERROR;\n\n    stream.next_in = pSource;\n    stream.avail_in = (mz_uint32)source_len;\n    stream.next_out = pDest;\n    stream.avail_out = (mz_uint32)*pDest_len;\n\n    status = mz_inflateInit(&stream);\n    if (status != MZ_OK)\n        return status;\n\n    status = mz_inflate(&stream, MZ_FINISH);\n    if (status != MZ_STREAM_END)\n    {\n        mz_inflateEnd(&stream);\n        return ((status == MZ_BUF_ERROR) && (!stream.avail_in)) ? MZ_DATA_ERROR : status;\n    }\n    *pDest_len = stream.total_out;\n\n    return mz_inflateEnd(&stream);\n}\n\nconst char *mz_error(int err)\n{\n    static struct\n    {\n        int m_err;\n        const char *m_pDesc;\n    } s_error_descs[] =\n        {\n          { MZ_OK, \"\" }, { MZ_STREAM_END, \"stream end\" }, { MZ_NEED_DICT, \"need dictionary\" }, { MZ_ERRNO, \"file error\" }, { MZ_STREAM_ERROR, \"stream error\" }, { MZ_DATA_ERROR, \"data error\" }, { MZ_MEM_ERROR, \"out of memory\" }, { MZ_BUF_ERROR, \"buf error\" }, { MZ_VERSION_ERROR, \"version error\" }, { MZ_PARAM_ERROR, \"parameter error\" }\n        };\n    mz_uint i;\n    for (i = 0; i < sizeof(s_error_descs) / sizeof(s_error_descs[0]); ++i)\n        if (s_error_descs[i].m_err == err)\n            return s_error_descs[i].m_pDesc;\n    return NULL;\n}\n\n#endif /*MINIZ_NO_ZLIB_APIS */\n\n#ifdef __cplusplus\n}\n#endif\n\n/*\n  This is free and unencumbered software released into the public domain.\n\n  Anyone is free to copy, modify, publish, use, compile, sell, or\n  distribute this software, either in source code form or as a compiled\n  binary, for any purpose, commercial or non-commercial, and by any\n  means.\n\n  In jurisdictions that recognize copyright laws, the author or authors\n  of this software dedicate any and all copyright interest in the\n  software to the public domain. We make this dedication for the benefit\n  of the public at large and to the detriment of our heirs and\n  successors. We intend this dedication to be an overt act of\n  relinquishment in perpetuity of all present and future rights to this\n  software under copyright law.\n\n  THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\n  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\n  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n  IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR\n  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,\n  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR\n  OTHER DEALINGS IN THE SOFTWARE.\n\n  For more information, please refer to <http://unlicense.org/>\n*/\n/**************************************************************************\n *\n * Copyright 2013-2014 RAD Game Tools and Valve Software\n * Copyright 2010-2014 Rich Geldreich and Tenacious Software LLC\n * All Rights Reserved.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n *\n **************************************************************************/\n\n\n\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n/* ------------------- Low-level Compression (independent from all decompression API's) */\n\n/* Purposely making these tables static for faster init and thread safety. */\nstatic const mz_uint16 s_tdefl_len_sym[256] =\n    {\n      257, 258, 259, 260, 261, 262, 263, 264, 265, 265, 266, 266, 267, 267, 268, 268, 269, 269, 269, 269, 270, 270, 270, 270, 271, 271, 271, 271, 272, 272, 272, 272,\n      273, 273, 273, 273, 273, 273, 273, 273, 274, 274, 274, 274, 274, 274, 274, 274, 275, 275, 275, 275, 275, 275, 275, 275, 276, 276, 276, 276, 276, 276, 276, 276,\n      277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278,\n      279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280,\n      281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281,\n      282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282,\n      283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283,\n      284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 285\n    };\n\nstatic const mz_uint8 s_tdefl_len_extra[256] =\n    {\n      0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n      4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,\n      5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n      5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0\n    };\n\nstatic const mz_uint8 s_tdefl_small_dist_sym[512] =\n    {\n      0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11,\n      11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 13,\n      13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,\n      14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,\n      14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,\n      15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,\n      16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,\n      16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,\n      16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17,\n      17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17,\n      17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17,\n      17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17\n    };\n\nstatic const mz_uint8 s_tdefl_small_dist_extra[512] =\n    {\n      0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5,\n      5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n      6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n      6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,\n      7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,\n      7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,\n      7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,\n      7, 7, 7, 7, 7, 7, 7, 7\n    };\n\nstatic const mz_uint8 s_tdefl_large_dist_sym[128] =\n    {\n      0, 0, 18, 19, 20, 20, 21, 21, 22, 22, 22, 22, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 25, 25, 25, 25, 25, 25, 25, 25, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26,\n      26, 26, 26, 26, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,\n      28, 28, 28, 28, 28, 28, 28, 28, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29\n    };\n\nstatic const mz_uint8 s_tdefl_large_dist_extra[128] =\n    {\n      0, 0, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,\n      12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,\n      13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13\n    };\n\n/* Radix sorts tdefl_sym_freq[] array by 16-bit key m_key. Returns ptr to sorted values. */\ntypedef struct\n{\n    mz_uint16 m_key, m_sym_index;\n} tdefl_sym_freq;\nstatic tdefl_sym_freq *tdefl_radix_sort_syms(mz_uint num_syms, tdefl_sym_freq *pSyms0, tdefl_sym_freq *pSyms1)\n{\n    mz_uint32 total_passes = 2, pass_shift, pass, i, hist[256 * 2];\n    tdefl_sym_freq *pCur_syms = pSyms0, *pNew_syms = pSyms1;\n    MZ_CLEAR_OBJ(hist);\n    for (i = 0; i < num_syms; i++)\n    {\n        mz_uint freq = pSyms0[i].m_key;\n        hist[freq & 0xFF]++;\n        hist[256 + ((freq >> 8) & 0xFF)]++;\n    }\n    while ((total_passes > 1) && (num_syms == hist[(total_passes - 1) * 256]))\n        total_passes--;\n    for (pass_shift = 0, pass = 0; pass < total_passes; pass++, pass_shift += 8)\n    {\n        const mz_uint32 *pHist = &hist[pass << 8];\n        mz_uint offsets[256], cur_ofs = 0;\n        for (i = 0; i < 256; i++)\n        {\n            offsets[i] = cur_ofs;\n            cur_ofs += pHist[i];\n        }\n        for (i = 0; i < num_syms; i++)\n            pNew_syms[offsets[(pCur_syms[i].m_key >> pass_shift) & 0xFF]++] = pCur_syms[i];\n        {\n            tdefl_sym_freq *t = pCur_syms;\n            pCur_syms = pNew_syms;\n            pNew_syms = t;\n        }\n    }\n    return pCur_syms;\n}\n\n/* tdefl_calculate_minimum_redundancy() originally written by: Alistair Moffat, alistair@cs.mu.oz.au, Jyrki Katajainen, jyrki@diku.dk, November 1996. */\nstatic void tdefl_calculate_minimum_redundancy(tdefl_sym_freq *A, int n)\n{\n    int root, leaf, next, avbl, used, dpth;\n    if (n == 0)\n        return;\n    else if (n == 1)\n    {\n        A[0].m_key = 1;\n        return;\n    }\n    A[0].m_key += A[1].m_key;\n    root = 0;\n    leaf = 2;\n    for (next = 1; next < n - 1; next++)\n    {\n        if (leaf >= n || A[root].m_key < A[leaf].m_key)\n        {\n            A[next].m_key = A[root].m_key;\n            A[root++].m_key = (mz_uint16)next;\n        }\n        else\n            A[next].m_key = A[leaf++].m_key;\n        if (leaf >= n || (root < next && A[root].m_key < A[leaf].m_key))\n        {\n            A[next].m_key = (mz_uint16)(A[next].m_key + A[root].m_key);\n            A[root++].m_key = (mz_uint16)next;\n        }\n        else\n            A[next].m_key = (mz_uint16)(A[next].m_key + A[leaf++].m_key);\n    }\n    A[n - 2].m_key = 0;\n    for (next = n - 3; next >= 0; next--)\n        A[next].m_key = A[A[next].m_key].m_key + 1;\n    avbl = 1;\n    used = dpth = 0;\n    root = n - 2;\n    next = n - 1;\n    while (avbl > 0)\n    {\n        while (root >= 0 && (int)A[root].m_key == dpth)\n        {\n            used++;\n            root--;\n        }\n        while (avbl > used)\n        {\n            A[next--].m_key = (mz_uint16)(dpth);\n            avbl--;\n        }\n        avbl = 2 * used;\n        dpth++;\n        used = 0;\n    }\n}\n\n/* Limits canonical Huffman code table's max code size. */\nenum\n{\n    TDEFL_MAX_SUPPORTED_HUFF_CODESIZE = 32\n};\nstatic void tdefl_huffman_enforce_max_code_size(int *pNum_codes, int code_list_len, int max_code_size)\n{\n    int i;\n    mz_uint32 total = 0;\n    if (code_list_len <= 1)\n        return;\n    for (i = max_code_size + 1; i <= TDEFL_MAX_SUPPORTED_HUFF_CODESIZE; i++)\n        pNum_codes[max_code_size] += pNum_codes[i];\n    for (i = max_code_size; i > 0; i--)\n        total += (((mz_uint32)pNum_codes[i]) << (max_code_size - i));\n    while (total != (1UL << max_code_size))\n    {\n        pNum_codes[max_code_size]--;\n        for (i = max_code_size - 1; i > 0; i--)\n            if (pNum_codes[i])\n            {\n                pNum_codes[i]--;\n                pNum_codes[i + 1] += 2;\n                break;\n            }\n        total--;\n    }\n}\n\nstatic void tdefl_optimize_huffman_table(tdefl_compressor *d, int table_num, int table_len, int code_size_limit, int static_table)\n{\n    int i, j, l, num_codes[1 + TDEFL_MAX_SUPPORTED_HUFF_CODESIZE];\n    mz_uint next_code[TDEFL_MAX_SUPPORTED_HUFF_CODESIZE + 1];\n    MZ_CLEAR_OBJ(num_codes);\n    if (static_table)\n    {\n        for (i = 0; i < table_len; i++)\n            num_codes[d->m_huff_code_sizes[table_num][i]]++;\n    }\n    else\n    {\n        tdefl_sym_freq syms0[TDEFL_MAX_HUFF_SYMBOLS], syms1[TDEFL_MAX_HUFF_SYMBOLS], *pSyms;\n        int num_used_syms = 0;\n        const mz_uint16 *pSym_count = &d->m_huff_count[table_num][0];\n        for (i = 0; i < table_len; i++)\n            if (pSym_count[i])\n            {\n                syms0[num_used_syms].m_key = (mz_uint16)pSym_count[i];\n                syms0[num_used_syms++].m_sym_index = (mz_uint16)i;\n            }\n\n        pSyms = tdefl_radix_sort_syms(num_used_syms, syms0, syms1);\n        tdefl_calculate_minimum_redundancy(pSyms, num_used_syms);\n\n        for (i = 0; i < num_used_syms; i++)\n            num_codes[pSyms[i].m_key]++;\n\n        tdefl_huffman_enforce_max_code_size(num_codes, num_used_syms, code_size_limit);\n\n        MZ_CLEAR_OBJ(d->m_huff_code_sizes[table_num]);\n        MZ_CLEAR_OBJ(d->m_huff_codes[table_num]);\n        for (i = 1, j = num_used_syms; i <= code_size_limit; i++)\n            for (l = num_codes[i]; l > 0; l--)\n                d->m_huff_code_sizes[table_num][pSyms[--j].m_sym_index] = (mz_uint8)(i);\n    }\n\n    next_code[1] = 0;\n    for (j = 0, i = 2; i <= code_size_limit; i++)\n        next_code[i] = j = ((j + num_codes[i - 1]) << 1);\n\n    for (i = 0; i < table_len; i++)\n    {\n        mz_uint rev_code = 0, code, code_size;\n        if ((code_size = d->m_huff_code_sizes[table_num][i]) == 0)\n            continue;\n        code = next_code[code_size]++;\n        for (l = code_size; l > 0; l--, code >>= 1)\n            rev_code = (rev_code << 1) | (code & 1);\n        d->m_huff_codes[table_num][i] = (mz_uint16)rev_code;\n    }\n}\n\n#define TDEFL_PUT_BITS(b, l)                                       \\\n    do                                                             \\\n    {                                                              \\\n        mz_uint bits = b;                                          \\\n        mz_uint len = l;                                           \\\n        MZ_ASSERT(bits <= ((1U << len) - 1U));                     \\\n        d->m_bit_buffer |= (bits << d->m_bits_in);                 \\\n        d->m_bits_in += len;                                       \\\n        while (d->m_bits_in >= 8)                                  \\\n        {                                                          \\\n            if (d->m_pOutput_buf < d->m_pOutput_buf_end)           \\\n                *d->m_pOutput_buf++ = (mz_uint8)(d->m_bit_buffer); \\\n            d->m_bit_buffer >>= 8;                                 \\\n            d->m_bits_in -= 8;                                     \\\n        }                                                          \\\n    }                                                              \\\n    MZ_MACRO_END\n\n#define TDEFL_RLE_PREV_CODE_SIZE()                                                                                       \\\n    {                                                                                                                    \\\n        if (rle_repeat_count)                                                                                            \\\n        {                                                                                                                \\\n            if (rle_repeat_count < 3)                                                                                    \\\n            {                                                                                                            \\\n                d->m_huff_count[2][prev_code_size] = (mz_uint16)(d->m_huff_count[2][prev_code_size] + rle_repeat_count); \\\n                while (rle_repeat_count--)                                                                               \\\n                    packed_code_sizes[num_packed_code_sizes++] = prev_code_size;                                         \\\n            }                                                                                                            \\\n            else                                                                                                         \\\n            {                                                                                                            \\\n                d->m_huff_count[2][16] = (mz_uint16)(d->m_huff_count[2][16] + 1);                                        \\\n                packed_code_sizes[num_packed_code_sizes++] = 16;                                                         \\\n                packed_code_sizes[num_packed_code_sizes++] = (mz_uint8)(rle_repeat_count - 3);                           \\\n            }                                                                                                            \\\n            rle_repeat_count = 0;                                                                                        \\\n        }                                                                                                                \\\n    }\n\n#define TDEFL_RLE_ZERO_CODE_SIZE()                                                         \\\n    {                                                                                      \\\n        if (rle_z_count)                                                                   \\\n        {                                                                                  \\\n            if (rle_z_count < 3)                                                           \\\n            {                                                                              \\\n                d->m_huff_count[2][0] = (mz_uint16)(d->m_huff_count[2][0] + rle_z_count);  \\\n                while (rle_z_count--)                                                      \\\n                    packed_code_sizes[num_packed_code_sizes++] = 0;                        \\\n            }                                                                              \\\n            else if (rle_z_count <= 10)                                                    \\\n            {                                                                              \\\n                d->m_huff_count[2][17] = (mz_uint16)(d->m_huff_count[2][17] + 1);          \\\n                packed_code_sizes[num_packed_code_sizes++] = 17;                           \\\n                packed_code_sizes[num_packed_code_sizes++] = (mz_uint8)(rle_z_count - 3);  \\\n            }                                                                              \\\n            else                                                                           \\\n            {                                                                              \\\n                d->m_huff_count[2][18] = (mz_uint16)(d->m_huff_count[2][18] + 1);          \\\n                packed_code_sizes[num_packed_code_sizes++] = 18;                           \\\n                packed_code_sizes[num_packed_code_sizes++] = (mz_uint8)(rle_z_count - 11); \\\n            }                                                                              \\\n            rle_z_count = 0;                                                               \\\n        }                                                                                  \\\n    }\n\nstatic mz_uint8 s_tdefl_packed_code_size_syms_swizzle[] = { 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };\n\nstatic void tdefl_start_dynamic_block(tdefl_compressor *d)\n{\n    int num_lit_codes, num_dist_codes, num_bit_lengths;\n    mz_uint i, total_code_sizes_to_pack, num_packed_code_sizes, rle_z_count, rle_repeat_count, packed_code_sizes_index;\n    mz_uint8 code_sizes_to_pack[TDEFL_MAX_HUFF_SYMBOLS_0 + TDEFL_MAX_HUFF_SYMBOLS_1], packed_code_sizes[TDEFL_MAX_HUFF_SYMBOLS_0 + TDEFL_MAX_HUFF_SYMBOLS_1], prev_code_size = 0xFF;\n\n    d->m_huff_count[0][256] = 1;\n\n    tdefl_optimize_huffman_table(d, 0, TDEFL_MAX_HUFF_SYMBOLS_0, 15, MZ_FALSE);\n    tdefl_optimize_huffman_table(d, 1, TDEFL_MAX_HUFF_SYMBOLS_1, 15, MZ_FALSE);\n\n    for (num_lit_codes = 286; num_lit_codes > 257; num_lit_codes--)\n        if (d->m_huff_code_sizes[0][num_lit_codes - 1])\n            break;\n    for (num_dist_codes = 30; num_dist_codes > 1; num_dist_codes--)\n        if (d->m_huff_code_sizes[1][num_dist_codes - 1])\n            break;\n\n    memcpy(code_sizes_to_pack, &d->m_huff_code_sizes[0][0], num_lit_codes);\n    memcpy(code_sizes_to_pack + num_lit_codes, &d->m_huff_code_sizes[1][0], num_dist_codes);\n    total_code_sizes_to_pack = num_lit_codes + num_dist_codes;\n    num_packed_code_sizes = 0;\n    rle_z_count = 0;\n    rle_repeat_count = 0;\n\n    memset(&d->m_huff_count[2][0], 0, sizeof(d->m_huff_count[2][0]) * TDEFL_MAX_HUFF_SYMBOLS_2);\n    for (i = 0; i < total_code_sizes_to_pack; i++)\n    {\n        mz_uint8 code_size = code_sizes_to_pack[i];\n        if (!code_size)\n        {\n            TDEFL_RLE_PREV_CODE_SIZE();\n            if (++rle_z_count == 138)\n            {\n                TDEFL_RLE_ZERO_CODE_SIZE();\n            }\n        }\n        else\n        {\n            TDEFL_RLE_ZERO_CODE_SIZE();\n            if (code_size != prev_code_size)\n            {\n                TDEFL_RLE_PREV_CODE_SIZE();\n                d->m_huff_count[2][code_size] = (mz_uint16)(d->m_huff_count[2][code_size] + 1);\n                packed_code_sizes[num_packed_code_sizes++] = code_size;\n            }\n            else if (++rle_repeat_count == 6)\n            {\n                TDEFL_RLE_PREV_CODE_SIZE();\n            }\n        }\n        prev_code_size = code_size;\n    }\n    if (rle_repeat_count)\n    {\n        TDEFL_RLE_PREV_CODE_SIZE();\n    }\n    else\n    {\n        TDEFL_RLE_ZERO_CODE_SIZE();\n    }\n\n    tdefl_optimize_huffman_table(d, 2, TDEFL_MAX_HUFF_SYMBOLS_2, 7, MZ_FALSE);\n\n    TDEFL_PUT_BITS(2, 2);\n\n    TDEFL_PUT_BITS(num_lit_codes - 257, 5);\n    TDEFL_PUT_BITS(num_dist_codes - 1, 5);\n\n    for (num_bit_lengths = 18; num_bit_lengths >= 0; num_bit_lengths--)\n        if (d->m_huff_code_sizes[2][s_tdefl_packed_code_size_syms_swizzle[num_bit_lengths]])\n            break;\n    num_bit_lengths = MZ_MAX(4, (num_bit_lengths + 1));\n    TDEFL_PUT_BITS(num_bit_lengths - 4, 4);\n    for (i = 0; (int)i < num_bit_lengths; i++)\n        TDEFL_PUT_BITS(d->m_huff_code_sizes[2][s_tdefl_packed_code_size_syms_swizzle[i]], 3);\n\n    for (packed_code_sizes_index = 0; packed_code_sizes_index < num_packed_code_sizes;)\n    {\n        mz_uint code = packed_code_sizes[packed_code_sizes_index++];\n        MZ_ASSERT(code < TDEFL_MAX_HUFF_SYMBOLS_2);\n        TDEFL_PUT_BITS(d->m_huff_codes[2][code], d->m_huff_code_sizes[2][code]);\n        if (code >= 16)\n            TDEFL_PUT_BITS(packed_code_sizes[packed_code_sizes_index++], \"\\02\\03\\07\"[code - 16]);\n    }\n}\n\nstatic void tdefl_start_static_block(tdefl_compressor *d)\n{\n    mz_uint i;\n    mz_uint8 *p = &d->m_huff_code_sizes[0][0];\n\n    for (i = 0; i <= 143; ++i)\n        *p++ = 8;\n    for (; i <= 255; ++i)\n        *p++ = 9;\n    for (; i <= 279; ++i)\n        *p++ = 7;\n    for (; i <= 287; ++i)\n        *p++ = 8;\n\n    memset(d->m_huff_code_sizes[1], 5, 32);\n\n    tdefl_optimize_huffman_table(d, 0, 288, 15, MZ_TRUE);\n    tdefl_optimize_huffman_table(d, 1, 32, 15, MZ_TRUE);\n\n    TDEFL_PUT_BITS(1, 2);\n}\n\nstatic const mz_uint mz_bitmasks[17] = { 0x0000, 0x0001, 0x0003, 0x0007, 0x000F, 0x001F, 0x003F, 0x007F, 0x00FF, 0x01FF, 0x03FF, 0x07FF, 0x0FFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF };\n\n#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN && MINIZ_HAS_64BIT_REGISTERS\nstatic mz_bool tdefl_compress_lz_codes(tdefl_compressor *d)\n{\n    mz_uint flags;\n    mz_uint8 *pLZ_codes;\n    mz_uint8 *pOutput_buf = d->m_pOutput_buf;\n    mz_uint8 *pLZ_code_buf_end = d->m_pLZ_code_buf;\n    mz_uint64 bit_buffer = d->m_bit_buffer;\n    mz_uint bits_in = d->m_bits_in;\n\n#define TDEFL_PUT_BITS_FAST(b, l)                    \\\n    {                                                \\\n        bit_buffer |= (((mz_uint64)(b)) << bits_in); \\\n        bits_in += (l);                              \\\n    }\n\n    flags = 1;\n    for (pLZ_codes = d->m_lz_code_buf; pLZ_codes < pLZ_code_buf_end; flags >>= 1)\n    {\n        if (flags == 1)\n            flags = *pLZ_codes++ | 0x100;\n\n        if (flags & 1)\n        {\n            mz_uint s0, s1, n0, n1, sym, num_extra_bits;\n            mz_uint match_len = pLZ_codes[0], match_dist = *(const mz_uint16 *)(pLZ_codes + 1);\n            pLZ_codes += 3;\n\n            MZ_ASSERT(d->m_huff_code_sizes[0][s_tdefl_len_sym[match_len]]);\n            TDEFL_PUT_BITS_FAST(d->m_huff_codes[0][s_tdefl_len_sym[match_len]], d->m_huff_code_sizes[0][s_tdefl_len_sym[match_len]]);\n            TDEFL_PUT_BITS_FAST(match_len & mz_bitmasks[s_tdefl_len_extra[match_len]], s_tdefl_len_extra[match_len]);\n\n            /* This sequence coaxes MSVC into using cmov's vs. jmp's. */\n            s0 = s_tdefl_small_dist_sym[match_dist & 511];\n            n0 = s_tdefl_small_dist_extra[match_dist & 511];\n            s1 = s_tdefl_large_dist_sym[match_dist >> 8];\n            n1 = s_tdefl_large_dist_extra[match_dist >> 8];\n            sym = (match_dist < 512) ? s0 : s1;\n            num_extra_bits = (match_dist < 512) ? n0 : n1;\n\n            MZ_ASSERT(d->m_huff_code_sizes[1][sym]);\n            TDEFL_PUT_BITS_FAST(d->m_huff_codes[1][sym], d->m_huff_code_sizes[1][sym]);\n            TDEFL_PUT_BITS_FAST(match_dist & mz_bitmasks[num_extra_bits], num_extra_bits);\n        }\n        else\n        {\n            mz_uint lit = *pLZ_codes++;\n            MZ_ASSERT(d->m_huff_code_sizes[0][lit]);\n            TDEFL_PUT_BITS_FAST(d->m_huff_codes[0][lit], d->m_huff_code_sizes[0][lit]);\n\n            if (((flags & 2) == 0) && (pLZ_codes < pLZ_code_buf_end))\n            {\n                flags >>= 1;\n                lit = *pLZ_codes++;\n                MZ_ASSERT(d->m_huff_code_sizes[0][lit]);\n                TDEFL_PUT_BITS_FAST(d->m_huff_codes[0][lit], d->m_huff_code_sizes[0][lit]);\n\n                if (((flags & 2) == 0) && (pLZ_codes < pLZ_code_buf_end))\n                {\n                    flags >>= 1;\n                    lit = *pLZ_codes++;\n                    MZ_ASSERT(d->m_huff_code_sizes[0][lit]);\n                    TDEFL_PUT_BITS_FAST(d->m_huff_codes[0][lit], d->m_huff_code_sizes[0][lit]);\n                }\n            }\n        }\n\n        if (pOutput_buf >= d->m_pOutput_buf_end)\n            return MZ_FALSE;\n\n        *(mz_uint64 *)pOutput_buf = bit_buffer;\n        pOutput_buf += (bits_in >> 3);\n        bit_buffer >>= (bits_in & ~7);\n        bits_in &= 7;\n    }\n\n#undef TDEFL_PUT_BITS_FAST\n\n    d->m_pOutput_buf = pOutput_buf;\n    d->m_bits_in = 0;\n    d->m_bit_buffer = 0;\n\n    while (bits_in)\n    {\n        mz_uint32 n = MZ_MIN(bits_in, 16);\n        TDEFL_PUT_BITS((mz_uint)bit_buffer & mz_bitmasks[n], n);\n        bit_buffer >>= n;\n        bits_in -= n;\n    }\n\n    TDEFL_PUT_BITS(d->m_huff_codes[0][256], d->m_huff_code_sizes[0][256]);\n\n    return (d->m_pOutput_buf < d->m_pOutput_buf_end);\n}\n#else\nstatic mz_bool tdefl_compress_lz_codes(tdefl_compressor *d)\n{\n    mz_uint flags;\n    mz_uint8 *pLZ_codes;\n\n    flags = 1;\n    for (pLZ_codes = d->m_lz_code_buf; pLZ_codes < d->m_pLZ_code_buf; flags >>= 1)\n    {\n        if (flags == 1)\n            flags = *pLZ_codes++ | 0x100;\n        if (flags & 1)\n        {\n            mz_uint sym, num_extra_bits;\n            mz_uint match_len = pLZ_codes[0], match_dist = (pLZ_codes[1] | (pLZ_codes[2] << 8));\n            pLZ_codes += 3;\n\n            MZ_ASSERT(d->m_huff_code_sizes[0][s_tdefl_len_sym[match_len]]);\n            TDEFL_PUT_BITS(d->m_huff_codes[0][s_tdefl_len_sym[match_len]], d->m_huff_code_sizes[0][s_tdefl_len_sym[match_len]]);\n            TDEFL_PUT_BITS(match_len & mz_bitmasks[s_tdefl_len_extra[match_len]], s_tdefl_len_extra[match_len]);\n\n            if (match_dist < 512)\n            {\n                sym = s_tdefl_small_dist_sym[match_dist];\n                num_extra_bits = s_tdefl_small_dist_extra[match_dist];\n            }\n            else\n            {\n                sym = s_tdefl_large_dist_sym[match_dist >> 8];\n                num_extra_bits = s_tdefl_large_dist_extra[match_dist >> 8];\n            }\n            MZ_ASSERT(d->m_huff_code_sizes[1][sym]);\n            TDEFL_PUT_BITS(d->m_huff_codes[1][sym], d->m_huff_code_sizes[1][sym]);\n            TDEFL_PUT_BITS(match_dist & mz_bitmasks[num_extra_bits], num_extra_bits);\n        }\n        else\n        {\n            mz_uint lit = *pLZ_codes++;\n            MZ_ASSERT(d->m_huff_code_sizes[0][lit]);\n            TDEFL_PUT_BITS(d->m_huff_codes[0][lit], d->m_huff_code_sizes[0][lit]);\n        }\n    }\n\n    TDEFL_PUT_BITS(d->m_huff_codes[0][256], d->m_huff_code_sizes[0][256]);\n\n    return (d->m_pOutput_buf < d->m_pOutput_buf_end);\n}\n#endif /* MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN && MINIZ_HAS_64BIT_REGISTERS */\n\nstatic mz_bool tdefl_compress_block(tdefl_compressor *d, mz_bool static_block)\n{\n    if (static_block)\n        tdefl_start_static_block(d);\n    else\n        tdefl_start_dynamic_block(d);\n    return tdefl_compress_lz_codes(d);\n}\n\nstatic int tdefl_flush_block(tdefl_compressor *d, int flush)\n{\n    mz_uint saved_bit_buf, saved_bits_in;\n    mz_uint8 *pSaved_output_buf;\n    mz_bool comp_block_succeeded = MZ_FALSE;\n    int n, use_raw_block = ((d->m_flags & TDEFL_FORCE_ALL_RAW_BLOCKS) != 0) && (d->m_lookahead_pos - d->m_lz_code_buf_dict_pos) <= d->m_dict_size;\n    mz_uint8 *pOutput_buf_start = ((d->m_pPut_buf_func == NULL) && ((*d->m_pOut_buf_size - d->m_out_buf_ofs) >= TDEFL_OUT_BUF_SIZE)) ? ((mz_uint8 *)d->m_pOut_buf + d->m_out_buf_ofs) : d->m_output_buf;\n\n    d->m_pOutput_buf = pOutput_buf_start;\n    d->m_pOutput_buf_end = d->m_pOutput_buf + TDEFL_OUT_BUF_SIZE - 16;\n\n    MZ_ASSERT(!d->m_output_flush_remaining);\n    d->m_output_flush_ofs = 0;\n    d->m_output_flush_remaining = 0;\n\n    *d->m_pLZ_flags = (mz_uint8)(*d->m_pLZ_flags >> d->m_num_flags_left);\n    d->m_pLZ_code_buf -= (d->m_num_flags_left == 8);\n\n    if ((d->m_flags & TDEFL_WRITE_ZLIB_HEADER) && (!d->m_block_index))\n    {\n        TDEFL_PUT_BITS(0x78, 8);\n        TDEFL_PUT_BITS(0x01, 8);\n    }\n\n    TDEFL_PUT_BITS(flush == TDEFL_FINISH, 1);\n\n    pSaved_output_buf = d->m_pOutput_buf;\n    saved_bit_buf = d->m_bit_buffer;\n    saved_bits_in = d->m_bits_in;\n\n    if (!use_raw_block)\n        comp_block_succeeded = tdefl_compress_block(d, (d->m_flags & TDEFL_FORCE_ALL_STATIC_BLOCKS) || (d->m_total_lz_bytes < 48));\n\n    /* If the block gets expanded, forget the current contents of the output buffer and send a raw block instead. */\n    if (((use_raw_block) || ((d->m_total_lz_bytes) && ((d->m_pOutput_buf - pSaved_output_buf + 1U) >= d->m_total_lz_bytes))) &&\n        ((d->m_lookahead_pos - d->m_lz_code_buf_dict_pos) <= d->m_dict_size))\n    {\n        mz_uint i;\n        d->m_pOutput_buf = pSaved_output_buf;\n        d->m_bit_buffer = saved_bit_buf, d->m_bits_in = saved_bits_in;\n        TDEFL_PUT_BITS(0, 2);\n        if (d->m_bits_in)\n        {\n            TDEFL_PUT_BITS(0, 8 - d->m_bits_in);\n        }\n        for (i = 2; i; --i, d->m_total_lz_bytes ^= 0xFFFF)\n        {\n            TDEFL_PUT_BITS(d->m_total_lz_bytes & 0xFFFF, 16);\n        }\n        for (i = 0; i < d->m_total_lz_bytes; ++i)\n        {\n            TDEFL_PUT_BITS(d->m_dict[(d->m_lz_code_buf_dict_pos + i) & TDEFL_LZ_DICT_SIZE_MASK], 8);\n        }\n    }\n    /* Check for the extremely unlikely (if not impossible) case of the compressed block not fitting into the output buffer when using dynamic codes. */\n    else if (!comp_block_succeeded)\n    {\n        d->m_pOutput_buf = pSaved_output_buf;\n        d->m_bit_buffer = saved_bit_buf, d->m_bits_in = saved_bits_in;\n        tdefl_compress_block(d, MZ_TRUE);\n    }\n\n    if (flush)\n    {\n        if (flush == TDEFL_FINISH)\n        {\n            if (d->m_bits_in)\n            {\n                TDEFL_PUT_BITS(0, 8 - d->m_bits_in);\n            }\n            if (d->m_flags & TDEFL_WRITE_ZLIB_HEADER)\n            {\n                mz_uint i, a = d->m_adler32;\n                for (i = 0; i < 4; i++)\n                {\n                    TDEFL_PUT_BITS((a >> 24) & 0xFF, 8);\n                    a <<= 8;\n                }\n            }\n        }\n        else\n        {\n            mz_uint i, z = 0;\n            TDEFL_PUT_BITS(0, 3);\n            if (d->m_bits_in)\n            {\n                TDEFL_PUT_BITS(0, 8 - d->m_bits_in);\n            }\n            for (i = 2; i; --i, z ^= 0xFFFF)\n            {\n                TDEFL_PUT_BITS(z & 0xFFFF, 16);\n            }\n        }\n    }\n\n    MZ_ASSERT(d->m_pOutput_buf < d->m_pOutput_buf_end);\n\n    memset(&d->m_huff_count[0][0], 0, sizeof(d->m_huff_count[0][0]) * TDEFL_MAX_HUFF_SYMBOLS_0);\n    memset(&d->m_huff_count[1][0], 0, sizeof(d->m_huff_count[1][0]) * TDEFL_MAX_HUFF_SYMBOLS_1);\n\n    d->m_pLZ_code_buf = d->m_lz_code_buf + 1;\n    d->m_pLZ_flags = d->m_lz_code_buf;\n    d->m_num_flags_left = 8;\n    d->m_lz_code_buf_dict_pos += d->m_total_lz_bytes;\n    d->m_total_lz_bytes = 0;\n    d->m_block_index++;\n\n    if ((n = (int)(d->m_pOutput_buf - pOutput_buf_start)) != 0)\n    {\n        if (d->m_pPut_buf_func)\n        {\n            *d->m_pIn_buf_size = d->m_pSrc - (const mz_uint8 *)d->m_pIn_buf;\n            if (!(*d->m_pPut_buf_func)(d->m_output_buf, n, d->m_pPut_buf_user))\n                return (d->m_prev_return_status = TDEFL_STATUS_PUT_BUF_FAILED);\n        }\n        else if (pOutput_buf_start == d->m_output_buf)\n        {\n            int bytes_to_copy = (int)MZ_MIN((size_t)n, (size_t)(*d->m_pOut_buf_size - d->m_out_buf_ofs));\n            memcpy((mz_uint8 *)d->m_pOut_buf + d->m_out_buf_ofs, d->m_output_buf, bytes_to_copy);\n            d->m_out_buf_ofs += bytes_to_copy;\n            if ((n -= bytes_to_copy) != 0)\n            {\n                d->m_output_flush_ofs = bytes_to_copy;\n                d->m_output_flush_remaining = n;\n            }\n        }\n        else\n        {\n            d->m_out_buf_ofs += n;\n        }\n    }\n\n    return d->m_output_flush_remaining;\n}\n\n#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES\n#ifdef MINIZ_UNALIGNED_USE_MEMCPY\nstatic mz_uint16 TDEFL_READ_UNALIGNED_WORD(const mz_uint8* p)\n{\n\tmz_uint16 ret;\n\tmemcpy(&ret, p, sizeof(mz_uint16));\n\treturn ret;\n}\nstatic mz_uint16 TDEFL_READ_UNALIGNED_WORD2(const mz_uint16* p)\n{\n\tmz_uint16 ret;\n\tmemcpy(&ret, p, sizeof(mz_uint16));\n\treturn ret;\n}\n#else\n#define TDEFL_READ_UNALIGNED_WORD(p) *(const mz_uint16 *)(p)\n#define TDEFL_READ_UNALIGNED_WORD2(p) *(const mz_uint16 *)(p)\n#endif\nstatic MZ_FORCEINLINE void tdefl_find_match(tdefl_compressor *d, mz_uint lookahead_pos, mz_uint max_dist, mz_uint max_match_len, mz_uint *pMatch_dist, mz_uint *pMatch_len)\n{\n    mz_uint dist, pos = lookahead_pos & TDEFL_LZ_DICT_SIZE_MASK, match_len = *pMatch_len, probe_pos = pos, next_probe_pos, probe_len;\n    mz_uint num_probes_left = d->m_max_probes[match_len >= 32];\n    const mz_uint16 *s = (const mz_uint16 *)(d->m_dict + pos), *p, *q;\n    mz_uint16 c01 = TDEFL_READ_UNALIGNED_WORD(&d->m_dict[pos + match_len - 1]), s01 = TDEFL_READ_UNALIGNED_WORD2(s);\n    MZ_ASSERT(max_match_len <= TDEFL_MAX_MATCH_LEN);\n    if (max_match_len <= match_len)\n        return;\n    for (;;)\n    {\n        for (;;)\n        {\n            if (--num_probes_left == 0)\n                return;\n#define TDEFL_PROBE                                                                             \\\n    next_probe_pos = d->m_next[probe_pos];                                                      \\\n    if ((!next_probe_pos) || ((dist = (mz_uint16)(lookahead_pos - next_probe_pos)) > max_dist)) \\\n        return;                                                                                 \\\n    probe_pos = next_probe_pos & TDEFL_LZ_DICT_SIZE_MASK;                                       \\\n    if (TDEFL_READ_UNALIGNED_WORD(&d->m_dict[probe_pos + match_len - 1]) == c01)                \\\n        break;\n            TDEFL_PROBE;\n            TDEFL_PROBE;\n            TDEFL_PROBE;\n        }\n        if (!dist)\n            break;\n        q = (const mz_uint16 *)(d->m_dict + probe_pos);\n        if (TDEFL_READ_UNALIGNED_WORD2(q) != s01)\n            continue;\n        p = s;\n        probe_len = 32;\n        do\n        {\n        } while ((TDEFL_READ_UNALIGNED_WORD2(++p) == TDEFL_READ_UNALIGNED_WORD2(++q)) && (TDEFL_READ_UNALIGNED_WORD2(++p) == TDEFL_READ_UNALIGNED_WORD2(++q)) &&\n                 (TDEFL_READ_UNALIGNED_WORD2(++p) == TDEFL_READ_UNALIGNED_WORD2(++q)) && (TDEFL_READ_UNALIGNED_WORD2(++p) == TDEFL_READ_UNALIGNED_WORD2(++q)) && (--probe_len > 0));\n        if (!probe_len)\n        {\n            *pMatch_dist = dist;\n            *pMatch_len = MZ_MIN(max_match_len, (mz_uint)TDEFL_MAX_MATCH_LEN);\n            break;\n        }\n        else if ((probe_len = ((mz_uint)(p - s) * 2) + (mz_uint)(*(const mz_uint8 *)p == *(const mz_uint8 *)q)) > match_len)\n        {\n            *pMatch_dist = dist;\n            if ((*pMatch_len = match_len = MZ_MIN(max_match_len, probe_len)) == max_match_len)\n                break;\n            c01 = TDEFL_READ_UNALIGNED_WORD(&d->m_dict[pos + match_len - 1]);\n        }\n    }\n}\n#else\nstatic MZ_FORCEINLINE void tdefl_find_match(tdefl_compressor *d, mz_uint lookahead_pos, mz_uint max_dist, mz_uint max_match_len, mz_uint *pMatch_dist, mz_uint *pMatch_len)\n{\n    mz_uint dist, pos = lookahead_pos & TDEFL_LZ_DICT_SIZE_MASK, match_len = *pMatch_len, probe_pos = pos, next_probe_pos, probe_len;\n    mz_uint num_probes_left = d->m_max_probes[match_len >= 32];\n    const mz_uint8 *s = d->m_dict + pos, *p, *q;\n    mz_uint8 c0 = d->m_dict[pos + match_len], c1 = d->m_dict[pos + match_len - 1];\n    MZ_ASSERT(max_match_len <= TDEFL_MAX_MATCH_LEN);\n    if (max_match_len <= match_len)\n        return;\n    for (;;)\n    {\n        for (;;)\n        {\n            if (--num_probes_left == 0)\n                return;\n#define TDEFL_PROBE                                                                               \\\n    next_probe_pos = d->m_next[probe_pos];                                                        \\\n    if ((!next_probe_pos) || ((dist = (mz_uint16)(lookahead_pos - next_probe_pos)) > max_dist))   \\\n        return;                                                                                   \\\n    probe_pos = next_probe_pos & TDEFL_LZ_DICT_SIZE_MASK;                                         \\\n    if ((d->m_dict[probe_pos + match_len] == c0) && (d->m_dict[probe_pos + match_len - 1] == c1)) \\\n        break;\n            TDEFL_PROBE;\n            TDEFL_PROBE;\n            TDEFL_PROBE;\n        }\n        if (!dist)\n            break;\n        p = s;\n        q = d->m_dict + probe_pos;\n        for (probe_len = 0; probe_len < max_match_len; probe_len++)\n            if (*p++ != *q++)\n                break;\n        if (probe_len > match_len)\n        {\n            *pMatch_dist = dist;\n            if ((*pMatch_len = match_len = probe_len) == max_match_len)\n                return;\n            c0 = d->m_dict[pos + match_len];\n            c1 = d->m_dict[pos + match_len - 1];\n        }\n    }\n}\n#endif /* #if MINIZ_USE_UNALIGNED_LOADS_AND_STORES */\n\n#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN\n#ifdef MINIZ_UNALIGNED_USE_MEMCPY\nstatic mz_uint32 TDEFL_READ_UNALIGNED_WORD32(const mz_uint8* p)\n{\n\tmz_uint32 ret;\n\tmemcpy(&ret, p, sizeof(mz_uint32));\n\treturn ret;\n}\n#else\n#define TDEFL_READ_UNALIGNED_WORD32(p) *(const mz_uint32 *)(p)\n#endif\nstatic mz_bool tdefl_compress_fast(tdefl_compressor *d)\n{\n    /* Faster, minimally featured LZRW1-style match+parse loop with better register utilization. Intended for applications where raw throughput is valued more highly than ratio. */\n    mz_uint lookahead_pos = d->m_lookahead_pos, lookahead_size = d->m_lookahead_size, dict_size = d->m_dict_size, total_lz_bytes = d->m_total_lz_bytes, num_flags_left = d->m_num_flags_left;\n    mz_uint8 *pLZ_code_buf = d->m_pLZ_code_buf, *pLZ_flags = d->m_pLZ_flags;\n    mz_uint cur_pos = lookahead_pos & TDEFL_LZ_DICT_SIZE_MASK;\n\n    while ((d->m_src_buf_left) || ((d->m_flush) && (lookahead_size)))\n    {\n        const mz_uint TDEFL_COMP_FAST_LOOKAHEAD_SIZE = 4096;\n        mz_uint dst_pos = (lookahead_pos + lookahead_size) & TDEFL_LZ_DICT_SIZE_MASK;\n        mz_uint num_bytes_to_process = (mz_uint)MZ_MIN(d->m_src_buf_left, TDEFL_COMP_FAST_LOOKAHEAD_SIZE - lookahead_size);\n        d->m_src_buf_left -= num_bytes_to_process;\n        lookahead_size += num_bytes_to_process;\n\n        while (num_bytes_to_process)\n        {\n            mz_uint32 n = MZ_MIN(TDEFL_LZ_DICT_SIZE - dst_pos, num_bytes_to_process);\n            memcpy(d->m_dict + dst_pos, d->m_pSrc, n);\n            if (dst_pos < (TDEFL_MAX_MATCH_LEN - 1))\n                memcpy(d->m_dict + TDEFL_LZ_DICT_SIZE + dst_pos, d->m_pSrc, MZ_MIN(n, (TDEFL_MAX_MATCH_LEN - 1) - dst_pos));\n            d->m_pSrc += n;\n            dst_pos = (dst_pos + n) & TDEFL_LZ_DICT_SIZE_MASK;\n            num_bytes_to_process -= n;\n        }\n\n        dict_size = MZ_MIN(TDEFL_LZ_DICT_SIZE - lookahead_size, dict_size);\n        if ((!d->m_flush) && (lookahead_size < TDEFL_COMP_FAST_LOOKAHEAD_SIZE))\n            break;\n\n        while (lookahead_size >= 4)\n        {\n            mz_uint cur_match_dist, cur_match_len = 1;\n            mz_uint8 *pCur_dict = d->m_dict + cur_pos;\n            mz_uint first_trigram = TDEFL_READ_UNALIGNED_WORD32(pCur_dict) & 0xFFFFFF;\n            mz_uint hash = (first_trigram ^ (first_trigram >> (24 - (TDEFL_LZ_HASH_BITS - 8)))) & TDEFL_LEVEL1_HASH_SIZE_MASK;\n            mz_uint probe_pos = d->m_hash[hash];\n            d->m_hash[hash] = (mz_uint16)lookahead_pos;\n\n            if (((cur_match_dist = (mz_uint16)(lookahead_pos - probe_pos)) <= dict_size) && ((TDEFL_READ_UNALIGNED_WORD32(d->m_dict + (probe_pos &= TDEFL_LZ_DICT_SIZE_MASK)) & 0xFFFFFF) == first_trigram))\n            {\n                const mz_uint16 *p = (const mz_uint16 *)pCur_dict;\n                const mz_uint16 *q = (const mz_uint16 *)(d->m_dict + probe_pos);\n                mz_uint32 probe_len = 32;\n                do\n                {\n                } while ((TDEFL_READ_UNALIGNED_WORD2(++p) == TDEFL_READ_UNALIGNED_WORD2(++q)) && (TDEFL_READ_UNALIGNED_WORD2(++p) == TDEFL_READ_UNALIGNED_WORD2(++q)) &&\n                         (TDEFL_READ_UNALIGNED_WORD2(++p) == TDEFL_READ_UNALIGNED_WORD2(++q)) && (TDEFL_READ_UNALIGNED_WORD2(++p) == TDEFL_READ_UNALIGNED_WORD2(++q)) && (--probe_len > 0));\n                cur_match_len = ((mz_uint)(p - (const mz_uint16 *)pCur_dict) * 2) + (mz_uint)(*(const mz_uint8 *)p == *(const mz_uint8 *)q);\n                if (!probe_len)\n                    cur_match_len = cur_match_dist ? TDEFL_MAX_MATCH_LEN : 0;\n\n                if ((cur_match_len < TDEFL_MIN_MATCH_LEN) || ((cur_match_len == TDEFL_MIN_MATCH_LEN) && (cur_match_dist >= 8U * 1024U)))\n                {\n                    cur_match_len = 1;\n                    *pLZ_code_buf++ = (mz_uint8)first_trigram;\n                    *pLZ_flags = (mz_uint8)(*pLZ_flags >> 1);\n                    d->m_huff_count[0][(mz_uint8)first_trigram]++;\n                }\n                else\n                {\n                    mz_uint32 s0, s1;\n                    cur_match_len = MZ_MIN(cur_match_len, lookahead_size);\n\n                    MZ_ASSERT((cur_match_len >= TDEFL_MIN_MATCH_LEN) && (cur_match_dist >= 1) && (cur_match_dist <= TDEFL_LZ_DICT_SIZE));\n\n                    cur_match_dist--;\n\n                    pLZ_code_buf[0] = (mz_uint8)(cur_match_len - TDEFL_MIN_MATCH_LEN);\n#ifdef MINIZ_UNALIGNED_USE_MEMCPY\n\t\t\t\t\tmemcpy(&pLZ_code_buf[1], &cur_match_dist, sizeof(cur_match_dist));\n#else\n                    *(mz_uint16 *)(&pLZ_code_buf[1]) = (mz_uint16)cur_match_dist;\n#endif\n                    pLZ_code_buf += 3;\n                    *pLZ_flags = (mz_uint8)((*pLZ_flags >> 1) | 0x80);\n\n                    s0 = s_tdefl_small_dist_sym[cur_match_dist & 511];\n                    s1 = s_tdefl_large_dist_sym[cur_match_dist >> 8];\n                    d->m_huff_count[1][(cur_match_dist < 512) ? s0 : s1]++;\n\n                    d->m_huff_count[0][s_tdefl_len_sym[cur_match_len - TDEFL_MIN_MATCH_LEN]]++;\n                }\n            }\n            else\n            {\n                *pLZ_code_buf++ = (mz_uint8)first_trigram;\n                *pLZ_flags = (mz_uint8)(*pLZ_flags >> 1);\n                d->m_huff_count[0][(mz_uint8)first_trigram]++;\n            }\n\n            if (--num_flags_left == 0)\n            {\n                num_flags_left = 8;\n                pLZ_flags = pLZ_code_buf++;\n            }\n\n            total_lz_bytes += cur_match_len;\n            lookahead_pos += cur_match_len;\n            dict_size = MZ_MIN(dict_size + cur_match_len, (mz_uint)TDEFL_LZ_DICT_SIZE);\n            cur_pos = (cur_pos + cur_match_len) & TDEFL_LZ_DICT_SIZE_MASK;\n            MZ_ASSERT(lookahead_size >= cur_match_len);\n            lookahead_size -= cur_match_len;\n\n            if (pLZ_code_buf > &d->m_lz_code_buf[TDEFL_LZ_CODE_BUF_SIZE - 8])\n            {\n                int n;\n                d->m_lookahead_pos = lookahead_pos;\n                d->m_lookahead_size = lookahead_size;\n                d->m_dict_size = dict_size;\n                d->m_total_lz_bytes = total_lz_bytes;\n                d->m_pLZ_code_buf = pLZ_code_buf;\n                d->m_pLZ_flags = pLZ_flags;\n                d->m_num_flags_left = num_flags_left;\n                if ((n = tdefl_flush_block(d, 0)) != 0)\n                    return (n < 0) ? MZ_FALSE : MZ_TRUE;\n                total_lz_bytes = d->m_total_lz_bytes;\n                pLZ_code_buf = d->m_pLZ_code_buf;\n                pLZ_flags = d->m_pLZ_flags;\n                num_flags_left = d->m_num_flags_left;\n            }\n        }\n\n        while (lookahead_size)\n        {\n            mz_uint8 lit = d->m_dict[cur_pos];\n\n            total_lz_bytes++;\n            *pLZ_code_buf++ = lit;\n            *pLZ_flags = (mz_uint8)(*pLZ_flags >> 1);\n            if (--num_flags_left == 0)\n            {\n                num_flags_left = 8;\n                pLZ_flags = pLZ_code_buf++;\n            }\n\n            d->m_huff_count[0][lit]++;\n\n            lookahead_pos++;\n            dict_size = MZ_MIN(dict_size + 1, (mz_uint)TDEFL_LZ_DICT_SIZE);\n            cur_pos = (cur_pos + 1) & TDEFL_LZ_DICT_SIZE_MASK;\n            lookahead_size--;\n\n            if (pLZ_code_buf > &d->m_lz_code_buf[TDEFL_LZ_CODE_BUF_SIZE - 8])\n            {\n                int n;\n                d->m_lookahead_pos = lookahead_pos;\n                d->m_lookahead_size = lookahead_size;\n                d->m_dict_size = dict_size;\n                d->m_total_lz_bytes = total_lz_bytes;\n                d->m_pLZ_code_buf = pLZ_code_buf;\n                d->m_pLZ_flags = pLZ_flags;\n                d->m_num_flags_left = num_flags_left;\n                if ((n = tdefl_flush_block(d, 0)) != 0)\n                    return (n < 0) ? MZ_FALSE : MZ_TRUE;\n                total_lz_bytes = d->m_total_lz_bytes;\n                pLZ_code_buf = d->m_pLZ_code_buf;\n                pLZ_flags = d->m_pLZ_flags;\n                num_flags_left = d->m_num_flags_left;\n            }\n        }\n    }\n\n    d->m_lookahead_pos = lookahead_pos;\n    d->m_lookahead_size = lookahead_size;\n    d->m_dict_size = dict_size;\n    d->m_total_lz_bytes = total_lz_bytes;\n    d->m_pLZ_code_buf = pLZ_code_buf;\n    d->m_pLZ_flags = pLZ_flags;\n    d->m_num_flags_left = num_flags_left;\n    return MZ_TRUE;\n}\n#endif /* MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN */\n\nstatic MZ_FORCEINLINE void tdefl_record_literal(tdefl_compressor *d, mz_uint8 lit)\n{\n    d->m_total_lz_bytes++;\n    *d->m_pLZ_code_buf++ = lit;\n    *d->m_pLZ_flags = (mz_uint8)(*d->m_pLZ_flags >> 1);\n    if (--d->m_num_flags_left == 0)\n    {\n        d->m_num_flags_left = 8;\n        d->m_pLZ_flags = d->m_pLZ_code_buf++;\n    }\n    d->m_huff_count[0][lit]++;\n}\n\nstatic MZ_FORCEINLINE void tdefl_record_match(tdefl_compressor *d, mz_uint match_len, mz_uint match_dist)\n{\n    mz_uint32 s0, s1;\n\n    MZ_ASSERT((match_len >= TDEFL_MIN_MATCH_LEN) && (match_dist >= 1) && (match_dist <= TDEFL_LZ_DICT_SIZE));\n\n    d->m_total_lz_bytes += match_len;\n\n    d->m_pLZ_code_buf[0] = (mz_uint8)(match_len - TDEFL_MIN_MATCH_LEN);\n\n    match_dist -= 1;\n    d->m_pLZ_code_buf[1] = (mz_uint8)(match_dist & 0xFF);\n    d->m_pLZ_code_buf[2] = (mz_uint8)(match_dist >> 8);\n    d->m_pLZ_code_buf += 3;\n\n    *d->m_pLZ_flags = (mz_uint8)((*d->m_pLZ_flags >> 1) | 0x80);\n    if (--d->m_num_flags_left == 0)\n    {\n        d->m_num_flags_left = 8;\n        d->m_pLZ_flags = d->m_pLZ_code_buf++;\n    }\n\n    s0 = s_tdefl_small_dist_sym[match_dist & 511];\n    s1 = s_tdefl_large_dist_sym[(match_dist >> 8) & 127];\n    d->m_huff_count[1][(match_dist < 512) ? s0 : s1]++;\n\n    if (match_len >= TDEFL_MIN_MATCH_LEN)\n        d->m_huff_count[0][s_tdefl_len_sym[match_len - TDEFL_MIN_MATCH_LEN]]++;\n}\n\nstatic mz_bool tdefl_compress_normal(tdefl_compressor *d)\n{\n    const mz_uint8 *pSrc = d->m_pSrc;\n    size_t src_buf_left = d->m_src_buf_left;\n    tdefl_flush flush = d->m_flush;\n\n    while ((src_buf_left) || ((flush) && (d->m_lookahead_size)))\n    {\n        mz_uint len_to_move, cur_match_dist, cur_match_len, cur_pos;\n        /* Update dictionary and hash chains. Keeps the lookahead size equal to TDEFL_MAX_MATCH_LEN. */\n        if ((d->m_lookahead_size + d->m_dict_size) >= (TDEFL_MIN_MATCH_LEN - 1))\n        {\n            mz_uint dst_pos = (d->m_lookahead_pos + d->m_lookahead_size) & TDEFL_LZ_DICT_SIZE_MASK, ins_pos = d->m_lookahead_pos + d->m_lookahead_size - 2;\n            mz_uint hash = (d->m_dict[ins_pos & TDEFL_LZ_DICT_SIZE_MASK] << TDEFL_LZ_HASH_SHIFT) ^ d->m_dict[(ins_pos + 1) & TDEFL_LZ_DICT_SIZE_MASK];\n            mz_uint num_bytes_to_process = (mz_uint)MZ_MIN(src_buf_left, TDEFL_MAX_MATCH_LEN - d->m_lookahead_size);\n            const mz_uint8 *pSrc_end = pSrc + num_bytes_to_process;\n            src_buf_left -= num_bytes_to_process;\n            d->m_lookahead_size += num_bytes_to_process;\n            while (pSrc != pSrc_end)\n            {\n                mz_uint8 c = *pSrc++;\n                d->m_dict[dst_pos] = c;\n                if (dst_pos < (TDEFL_MAX_MATCH_LEN - 1))\n                    d->m_dict[TDEFL_LZ_DICT_SIZE + dst_pos] = c;\n                hash = ((hash << TDEFL_LZ_HASH_SHIFT) ^ c) & (TDEFL_LZ_HASH_SIZE - 1);\n                d->m_next[ins_pos & TDEFL_LZ_DICT_SIZE_MASK] = d->m_hash[hash];\n                d->m_hash[hash] = (mz_uint16)(ins_pos);\n                dst_pos = (dst_pos + 1) & TDEFL_LZ_DICT_SIZE_MASK;\n                ins_pos++;\n            }\n        }\n        else\n        {\n            while ((src_buf_left) && (d->m_lookahead_size < TDEFL_MAX_MATCH_LEN))\n            {\n                mz_uint8 c = *pSrc++;\n                mz_uint dst_pos = (d->m_lookahead_pos + d->m_lookahead_size) & TDEFL_LZ_DICT_SIZE_MASK;\n                src_buf_left--;\n                d->m_dict[dst_pos] = c;\n                if (dst_pos < (TDEFL_MAX_MATCH_LEN - 1))\n                    d->m_dict[TDEFL_LZ_DICT_SIZE + dst_pos] = c;\n                if ((++d->m_lookahead_size + d->m_dict_size) >= TDEFL_MIN_MATCH_LEN)\n                {\n                    mz_uint ins_pos = d->m_lookahead_pos + (d->m_lookahead_size - 1) - 2;\n                    mz_uint hash = ((d->m_dict[ins_pos & TDEFL_LZ_DICT_SIZE_MASK] << (TDEFL_LZ_HASH_SHIFT * 2)) ^ (d->m_dict[(ins_pos + 1) & TDEFL_LZ_DICT_SIZE_MASK] << TDEFL_LZ_HASH_SHIFT) ^ c) & (TDEFL_LZ_HASH_SIZE - 1);\n                    d->m_next[ins_pos & TDEFL_LZ_DICT_SIZE_MASK] = d->m_hash[hash];\n                    d->m_hash[hash] = (mz_uint16)(ins_pos);\n                }\n            }\n        }\n        d->m_dict_size = MZ_MIN(TDEFL_LZ_DICT_SIZE - d->m_lookahead_size, d->m_dict_size);\n        if ((!flush) && (d->m_lookahead_size < TDEFL_MAX_MATCH_LEN))\n            break;\n\n        /* Simple lazy/greedy parsing state machine. */\n        len_to_move = 1;\n        cur_match_dist = 0;\n        cur_match_len = d->m_saved_match_len ? d->m_saved_match_len : (TDEFL_MIN_MATCH_LEN - 1);\n        cur_pos = d->m_lookahead_pos & TDEFL_LZ_DICT_SIZE_MASK;\n        if (d->m_flags & (TDEFL_RLE_MATCHES | TDEFL_FORCE_ALL_RAW_BLOCKS))\n        {\n            if ((d->m_dict_size) && (!(d->m_flags & TDEFL_FORCE_ALL_RAW_BLOCKS)))\n            {\n                mz_uint8 c = d->m_dict[(cur_pos - 1) & TDEFL_LZ_DICT_SIZE_MASK];\n                cur_match_len = 0;\n                while (cur_match_len < d->m_lookahead_size)\n                {\n                    if (d->m_dict[cur_pos + cur_match_len] != c)\n                        break;\n                    cur_match_len++;\n                }\n                if (cur_match_len < TDEFL_MIN_MATCH_LEN)\n                    cur_match_len = 0;\n                else\n                    cur_match_dist = 1;\n            }\n        }\n        else\n        {\n            tdefl_find_match(d, d->m_lookahead_pos, d->m_dict_size, d->m_lookahead_size, &cur_match_dist, &cur_match_len);\n        }\n        if (((cur_match_len == TDEFL_MIN_MATCH_LEN) && (cur_match_dist >= 8U * 1024U)) || (cur_pos == cur_match_dist) || ((d->m_flags & TDEFL_FILTER_MATCHES) && (cur_match_len <= 5)))\n        {\n            cur_match_dist = cur_match_len = 0;\n        }\n        if (d->m_saved_match_len)\n        {\n            if (cur_match_len > d->m_saved_match_len)\n            {\n                tdefl_record_literal(d, (mz_uint8)d->m_saved_lit);\n                if (cur_match_len >= 128)\n                {\n                    tdefl_record_match(d, cur_match_len, cur_match_dist);\n                    d->m_saved_match_len = 0;\n                    len_to_move = cur_match_len;\n                }\n                else\n                {\n                    d->m_saved_lit = d->m_dict[cur_pos];\n                    d->m_saved_match_dist = cur_match_dist;\n                    d->m_saved_match_len = cur_match_len;\n                }\n            }\n            else\n            {\n                tdefl_record_match(d, d->m_saved_match_len, d->m_saved_match_dist);\n                len_to_move = d->m_saved_match_len - 1;\n                d->m_saved_match_len = 0;\n            }\n        }\n        else if (!cur_match_dist)\n            tdefl_record_literal(d, d->m_dict[MZ_MIN(cur_pos, sizeof(d->m_dict) - 1)]);\n        else if ((d->m_greedy_parsing) || (d->m_flags & TDEFL_RLE_MATCHES) || (cur_match_len >= 128))\n        {\n            tdefl_record_match(d, cur_match_len, cur_match_dist);\n            len_to_move = cur_match_len;\n        }\n        else\n        {\n            d->m_saved_lit = d->m_dict[MZ_MIN(cur_pos, sizeof(d->m_dict) - 1)];\n            d->m_saved_match_dist = cur_match_dist;\n            d->m_saved_match_len = cur_match_len;\n        }\n        /* Move the lookahead forward by len_to_move bytes. */\n        d->m_lookahead_pos += len_to_move;\n        MZ_ASSERT(d->m_lookahead_size >= len_to_move);\n        d->m_lookahead_size -= len_to_move;\n        d->m_dict_size = MZ_MIN(d->m_dict_size + len_to_move, (mz_uint)TDEFL_LZ_DICT_SIZE);\n        /* Check if it's time to flush the current LZ codes to the internal output buffer. */\n        if ((d->m_pLZ_code_buf > &d->m_lz_code_buf[TDEFL_LZ_CODE_BUF_SIZE - 8]) ||\n            ((d->m_total_lz_bytes > 31 * 1024) && (((((mz_uint)(d->m_pLZ_code_buf - d->m_lz_code_buf) * 115) >> 7) >= d->m_total_lz_bytes) || (d->m_flags & TDEFL_FORCE_ALL_RAW_BLOCKS))))\n        {\n            int n;\n            d->m_pSrc = pSrc;\n            d->m_src_buf_left = src_buf_left;\n            if ((n = tdefl_flush_block(d, 0)) != 0)\n                return (n < 0) ? MZ_FALSE : MZ_TRUE;\n        }\n    }\n\n    d->m_pSrc = pSrc;\n    d->m_src_buf_left = src_buf_left;\n    return MZ_TRUE;\n}\n\nstatic tdefl_status tdefl_flush_output_buffer(tdefl_compressor *d)\n{\n    if (d->m_pIn_buf_size)\n    {\n        *d->m_pIn_buf_size = d->m_pSrc - (const mz_uint8 *)d->m_pIn_buf;\n    }\n\n    if (d->m_pOut_buf_size)\n    {\n        size_t n = MZ_MIN(*d->m_pOut_buf_size - d->m_out_buf_ofs, d->m_output_flush_remaining);\n        memcpy((mz_uint8 *)d->m_pOut_buf + d->m_out_buf_ofs, d->m_output_buf + d->m_output_flush_ofs, n);\n        d->m_output_flush_ofs += (mz_uint)n;\n        d->m_output_flush_remaining -= (mz_uint)n;\n        d->m_out_buf_ofs += n;\n\n        *d->m_pOut_buf_size = d->m_out_buf_ofs;\n    }\n\n    return (d->m_finished && !d->m_output_flush_remaining) ? TDEFL_STATUS_DONE : TDEFL_STATUS_OKAY;\n}\n\ntdefl_status tdefl_compress(tdefl_compressor *d, const void *pIn_buf, size_t *pIn_buf_size, void *pOut_buf, size_t *pOut_buf_size, tdefl_flush flush)\n{\n    if (!d)\n    {\n        if (pIn_buf_size)\n            *pIn_buf_size = 0;\n        if (pOut_buf_size)\n            *pOut_buf_size = 0;\n        return TDEFL_STATUS_BAD_PARAM;\n    }\n\n    d->m_pIn_buf = pIn_buf;\n    d->m_pIn_buf_size = pIn_buf_size;\n    d->m_pOut_buf = pOut_buf;\n    d->m_pOut_buf_size = pOut_buf_size;\n    d->m_pSrc = (const mz_uint8 *)(pIn_buf);\n    d->m_src_buf_left = pIn_buf_size ? *pIn_buf_size : 0;\n    d->m_out_buf_ofs = 0;\n    d->m_flush = flush;\n\n    if (((d->m_pPut_buf_func != NULL) == ((pOut_buf != NULL) || (pOut_buf_size != NULL))) || (d->m_prev_return_status != TDEFL_STATUS_OKAY) ||\n        (d->m_wants_to_finish && (flush != TDEFL_FINISH)) || (pIn_buf_size && *pIn_buf_size && !pIn_buf) || (pOut_buf_size && *pOut_buf_size && !pOut_buf))\n    {\n        if (pIn_buf_size)\n            *pIn_buf_size = 0;\n        if (pOut_buf_size)\n            *pOut_buf_size = 0;\n        return (d->m_prev_return_status = TDEFL_STATUS_BAD_PARAM);\n    }\n    d->m_wants_to_finish |= (flush == TDEFL_FINISH);\n\n    if ((d->m_output_flush_remaining) || (d->m_finished))\n        return (d->m_prev_return_status = tdefl_flush_output_buffer(d));\n\n#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN\n    if (((d->m_flags & TDEFL_MAX_PROBES_MASK) == 1) &&\n        ((d->m_flags & TDEFL_GREEDY_PARSING_FLAG) != 0) &&\n        ((d->m_flags & (TDEFL_FILTER_MATCHES | TDEFL_FORCE_ALL_RAW_BLOCKS | TDEFL_RLE_MATCHES)) == 0))\n    {\n        if (!tdefl_compress_fast(d))\n            return d->m_prev_return_status;\n    }\n    else\n#endif /* #if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN */\n    {\n        if (!tdefl_compress_normal(d))\n            return d->m_prev_return_status;\n    }\n\n    if ((d->m_flags & (TDEFL_WRITE_ZLIB_HEADER | TDEFL_COMPUTE_ADLER32)) && (pIn_buf))\n        d->m_adler32 = (mz_uint32)mz_adler32(d->m_adler32, (const mz_uint8 *)pIn_buf, d->m_pSrc - (const mz_uint8 *)pIn_buf);\n\n    if ((flush) && (!d->m_lookahead_size) && (!d->m_src_buf_left) && (!d->m_output_flush_remaining))\n    {\n        if (tdefl_flush_block(d, flush) < 0)\n            return d->m_prev_return_status;\n        d->m_finished = (flush == TDEFL_FINISH);\n        if (flush == TDEFL_FULL_FLUSH)\n        {\n            MZ_CLEAR_OBJ(d->m_hash);\n            MZ_CLEAR_OBJ(d->m_next);\n            d->m_dict_size = 0;\n        }\n    }\n\n    return (d->m_prev_return_status = tdefl_flush_output_buffer(d));\n}\n\ntdefl_status tdefl_compress_buffer(tdefl_compressor *d, const void *pIn_buf, size_t in_buf_size, tdefl_flush flush)\n{\n    MZ_ASSERT(d->m_pPut_buf_func);\n    return tdefl_compress(d, pIn_buf, &in_buf_size, NULL, NULL, flush);\n}\n\ntdefl_status tdefl_init(tdefl_compressor *d, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags)\n{\n    d->m_pPut_buf_func = pPut_buf_func;\n    d->m_pPut_buf_user = pPut_buf_user;\n    d->m_flags = (mz_uint)(flags);\n    d->m_max_probes[0] = 1 + ((flags & 0xFFF) + 2) / 3;\n    d->m_greedy_parsing = (flags & TDEFL_GREEDY_PARSING_FLAG) != 0;\n    d->m_max_probes[1] = 1 + (((flags & 0xFFF) >> 2) + 2) / 3;\n    if (!(flags & TDEFL_NONDETERMINISTIC_PARSING_FLAG))\n        MZ_CLEAR_OBJ(d->m_hash);\n    d->m_lookahead_pos = d->m_lookahead_size = d->m_dict_size = d->m_total_lz_bytes = d->m_lz_code_buf_dict_pos = d->m_bits_in = 0;\n    d->m_output_flush_ofs = d->m_output_flush_remaining = d->m_finished = d->m_block_index = d->m_bit_buffer = d->m_wants_to_finish = 0;\n    d->m_pLZ_code_buf = d->m_lz_code_buf + 1;\n    d->m_pLZ_flags = d->m_lz_code_buf;\n    d->m_num_flags_left = 8;\n    d->m_pOutput_buf = d->m_output_buf;\n    d->m_pOutput_buf_end = d->m_output_buf;\n    d->m_prev_return_status = TDEFL_STATUS_OKAY;\n    d->m_saved_match_dist = d->m_saved_match_len = d->m_saved_lit = 0;\n    d->m_adler32 = 1;\n    d->m_pIn_buf = NULL;\n    d->m_pOut_buf = NULL;\n    d->m_pIn_buf_size = NULL;\n    d->m_pOut_buf_size = NULL;\n    d->m_flush = TDEFL_NO_FLUSH;\n    d->m_pSrc = NULL;\n    d->m_src_buf_left = 0;\n    d->m_out_buf_ofs = 0;\n    if (!(flags & TDEFL_NONDETERMINISTIC_PARSING_FLAG))\n        MZ_CLEAR_OBJ(d->m_dict);\n    memset(&d->m_huff_count[0][0], 0, sizeof(d->m_huff_count[0][0]) * TDEFL_MAX_HUFF_SYMBOLS_0);\n    memset(&d->m_huff_count[1][0], 0, sizeof(d->m_huff_count[1][0]) * TDEFL_MAX_HUFF_SYMBOLS_1);\n    return TDEFL_STATUS_OKAY;\n}\n\ntdefl_status tdefl_get_prev_return_status(tdefl_compressor *d)\n{\n    return d->m_prev_return_status;\n}\n\nmz_uint32 tdefl_get_adler32(tdefl_compressor *d)\n{\n    return d->m_adler32;\n}\n\nmz_bool tdefl_compress_mem_to_output(const void *pBuf, size_t buf_len, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags)\n{\n    tdefl_compressor *pComp;\n    mz_bool succeeded;\n    if (((buf_len) && (!pBuf)) || (!pPut_buf_func))\n        return MZ_FALSE;\n    pComp = (tdefl_compressor *)MZ_MALLOC(sizeof(tdefl_compressor));\n    if (!pComp)\n        return MZ_FALSE;\n    succeeded = (tdefl_init(pComp, pPut_buf_func, pPut_buf_user, flags) == TDEFL_STATUS_OKAY);\n    succeeded = succeeded && (tdefl_compress_buffer(pComp, pBuf, buf_len, TDEFL_FINISH) == TDEFL_STATUS_DONE);\n    MZ_FREE(pComp);\n    return succeeded;\n}\n\ntypedef struct\n{\n    size_t m_size, m_capacity;\n    mz_uint8 *m_pBuf;\n    mz_bool m_expandable;\n} tdefl_output_buffer;\n\nstatic mz_bool tdefl_output_buffer_putter(const void *pBuf, int len, void *pUser)\n{\n    tdefl_output_buffer *p = (tdefl_output_buffer *)pUser;\n    size_t new_size = p->m_size + len;\n    if (new_size > p->m_capacity)\n    {\n        size_t new_capacity = p->m_capacity;\n        mz_uint8 *pNew_buf;\n        if (!p->m_expandable)\n            return MZ_FALSE;\n        do\n        {\n            new_capacity = MZ_MAX(128U, new_capacity << 1U);\n        } while (new_size > new_capacity);\n        pNew_buf = (mz_uint8 *)MZ_REALLOC(p->m_pBuf, new_capacity);\n        if (!pNew_buf)\n            return MZ_FALSE;\n        p->m_pBuf = pNew_buf;\n        p->m_capacity = new_capacity;\n    }\n    memcpy((mz_uint8 *)p->m_pBuf + p->m_size, pBuf, len);\n    p->m_size = new_size;\n    return MZ_TRUE;\n}\n\nvoid *tdefl_compress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, size_t *pOut_len, int flags)\n{\n    tdefl_output_buffer out_buf;\n    MZ_CLEAR_OBJ(out_buf);\n    if (!pOut_len)\n        return MZ_FALSE;\n    else\n        *pOut_len = 0;\n    out_buf.m_expandable = MZ_TRUE;\n    if (!tdefl_compress_mem_to_output(pSrc_buf, src_buf_len, tdefl_output_buffer_putter, &out_buf, flags))\n        return NULL;\n    *pOut_len = out_buf.m_size;\n    return out_buf.m_pBuf;\n}\n\nsize_t tdefl_compress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void *pSrc_buf, size_t src_buf_len, int flags)\n{\n    tdefl_output_buffer out_buf;\n    MZ_CLEAR_OBJ(out_buf);\n    if (!pOut_buf)\n        return 0;\n    out_buf.m_pBuf = (mz_uint8 *)pOut_buf;\n    out_buf.m_capacity = out_buf_len;\n    if (!tdefl_compress_mem_to_output(pSrc_buf, src_buf_len, tdefl_output_buffer_putter, &out_buf, flags))\n        return 0;\n    return out_buf.m_size;\n}\n\nstatic const mz_uint s_tdefl_num_probes[11] = { 0, 1, 6, 32, 16, 32, 128, 256, 512, 768, 1500 };\n\n/* level may actually range from [0,10] (10 is a \"hidden\" max level, where we want a bit more compression and it's fine if throughput to fall off a cliff on some files). */\nmz_uint tdefl_create_comp_flags_from_zip_params(int level, int window_bits, int strategy)\n{\n    mz_uint comp_flags = s_tdefl_num_probes[(level >= 0) ? MZ_MIN(10, level) : MZ_DEFAULT_LEVEL] | ((level <= 3) ? TDEFL_GREEDY_PARSING_FLAG : 0);\n    if (window_bits > 0)\n        comp_flags |= TDEFL_WRITE_ZLIB_HEADER;\n\n    if (!level)\n        comp_flags |= TDEFL_FORCE_ALL_RAW_BLOCKS;\n    else if (strategy == MZ_FILTERED)\n        comp_flags |= TDEFL_FILTER_MATCHES;\n    else if (strategy == MZ_HUFFMAN_ONLY)\n        comp_flags &= ~TDEFL_MAX_PROBES_MASK;\n    else if (strategy == MZ_FIXED)\n        comp_flags |= TDEFL_FORCE_ALL_STATIC_BLOCKS;\n    else if (strategy == MZ_RLE)\n        comp_flags |= TDEFL_RLE_MATCHES;\n\n    return comp_flags;\n}\n\n#ifdef _MSC_VER\n#pragma warning(push)\n#pragma warning(disable : 4204) /* nonstandard extension used : non-constant aggregate initializer (also supported by GNU C and C99, so no big deal) */\n#endif\n\n/* Simple PNG writer function by Alex Evans, 2011. Released into the public domain: https://gist.github.com/908299, more context at\n http://altdevblogaday.org/2011/04/06/a-smaller-jpg-encoder/.\n This is actually a modification of Alex's original code so PNG files generated by this function pass pngcheck. */\nvoid *tdefl_write_image_to_png_file_in_memory_ex(const void *pImage, int w, int h, int num_chans, size_t *pLen_out, mz_uint level, mz_bool flip)\n{\n    /* Using a local copy of this array here in case MINIZ_NO_ZLIB_APIS was defined. */\n    static const mz_uint s_tdefl_png_num_probes[11] = { 0, 1, 6, 32, 16, 32, 128, 256, 512, 768, 1500 };\n    tdefl_compressor *pComp = (tdefl_compressor *)MZ_MALLOC(sizeof(tdefl_compressor));\n    tdefl_output_buffer out_buf;\n    int i, bpl = w * num_chans, y, z;\n    mz_uint32 c;\n    *pLen_out = 0;\n    if (!pComp)\n        return NULL;\n    MZ_CLEAR_OBJ(out_buf);\n    out_buf.m_expandable = MZ_TRUE;\n    out_buf.m_capacity = 57 + MZ_MAX(64, (1 + bpl) * h);\n    if (NULL == (out_buf.m_pBuf = (mz_uint8 *)MZ_MALLOC(out_buf.m_capacity)))\n    {\n        MZ_FREE(pComp);\n        return NULL;\n    }\n    /* write dummy header */\n    for (z = 41; z; --z)\n        tdefl_output_buffer_putter(&z, 1, &out_buf);\n    /* compress image data */\n    tdefl_init(pComp, tdefl_output_buffer_putter, &out_buf, s_tdefl_png_num_probes[MZ_MIN(10, level)] | TDEFL_WRITE_ZLIB_HEADER);\n    for (y = 0; y < h; ++y)\n    {\n        tdefl_compress_buffer(pComp, &z, 1, TDEFL_NO_FLUSH);\n        tdefl_compress_buffer(pComp, (mz_uint8 *)pImage + (flip ? (h - 1 - y) : y) * bpl, bpl, TDEFL_NO_FLUSH);\n    }\n    if (tdefl_compress_buffer(pComp, NULL, 0, TDEFL_FINISH) != TDEFL_STATUS_DONE)\n    {\n        MZ_FREE(pComp);\n        MZ_FREE(out_buf.m_pBuf);\n        return NULL;\n    }\n    /* write real header */\n    *pLen_out = out_buf.m_size - 41;\n    {\n        static const mz_uint8 chans[] = { 0x00, 0x00, 0x04, 0x02, 0x06 };\n        mz_uint8 pnghdr[41] = { 0x89, 0x50, 0x4e, 0x47, 0x0d,\n                                0x0a, 0x1a, 0x0a, 0x00, 0x00,\n                                0x00, 0x0d, 0x49, 0x48, 0x44,\n                                0x52, 0x00, 0x00, 0x00, 0x00,\n                                0x00, 0x00, 0x00, 0x00, 0x08,\n                                0x00, 0x00, 0x00, 0x00, 0x00,\n                                0x00, 0x00, 0x00, 0x00, 0x00,\n                                0x00, 0x00, 0x49, 0x44, 0x41,\n                                0x54 };\n        pnghdr[18] = (mz_uint8)(w >> 8);\n        pnghdr[19] = (mz_uint8)w;\n        pnghdr[22] = (mz_uint8)(h >> 8);\n        pnghdr[23] = (mz_uint8)h;\n        pnghdr[25] = chans[num_chans];\n        pnghdr[33] = (mz_uint8)(*pLen_out >> 24);\n        pnghdr[34] = (mz_uint8)(*pLen_out >> 16);\n        pnghdr[35] = (mz_uint8)(*pLen_out >> 8);\n        pnghdr[36] = (mz_uint8)*pLen_out;\n        c = (mz_uint32)mz_crc32(MZ_CRC32_INIT, pnghdr + 12, 17);\n        for (i = 0; i < 4; ++i, c <<= 8)\n            ((mz_uint8 *)(pnghdr + 29))[i] = (mz_uint8)(c >> 24);\n        memcpy(out_buf.m_pBuf, pnghdr, 41);\n    }\n    /* write footer (IDAT CRC-32, followed by IEND chunk) */\n    if (!tdefl_output_buffer_putter(\"\\0\\0\\0\\0\\0\\0\\0\\0\\x49\\x45\\x4e\\x44\\xae\\x42\\x60\\x82\", 16, &out_buf))\n    {\n        *pLen_out = 0;\n        MZ_FREE(pComp);\n        MZ_FREE(out_buf.m_pBuf);\n        return NULL;\n    }\n    c = (mz_uint32)mz_crc32(MZ_CRC32_INIT, out_buf.m_pBuf + 41 - 4, *pLen_out + 4);\n    for (i = 0; i < 4; ++i, c <<= 8)\n        (out_buf.m_pBuf + out_buf.m_size - 16)[i] = (mz_uint8)(c >> 24);\n    /* compute final size of file, grab compressed data buffer and return */\n    *pLen_out += 57;\n    MZ_FREE(pComp);\n    return out_buf.m_pBuf;\n}\nvoid *tdefl_write_image_to_png_file_in_memory(const void *pImage, int w, int h, int num_chans, size_t *pLen_out)\n{\n    /* Level 6 corresponds to TDEFL_DEFAULT_MAX_PROBES or MZ_DEFAULT_LEVEL (but we can't depend on MZ_DEFAULT_LEVEL being available in case the zlib API's where #defined out) */\n    return tdefl_write_image_to_png_file_in_memory_ex(pImage, w, h, num_chans, pLen_out, 6, MZ_FALSE);\n}\n\n#ifndef MINIZ_NO_MALLOC\n/* Allocate the tdefl_compressor and tinfl_decompressor structures in C so that */\n/* non-C language bindings to tdefL_ and tinfl_ API don't need to worry about */\n/* structure size and allocation mechanism. */\ntdefl_compressor *tdefl_compressor_alloc()\n{\n    return (tdefl_compressor *)MZ_MALLOC(sizeof(tdefl_compressor));\n}\n\nvoid tdefl_compressor_free(tdefl_compressor *pComp)\n{\n    MZ_FREE(pComp);\n}\n#endif\n\n#ifdef _MSC_VER\n#pragma warning(pop)\n#endif\n\n#ifdef __cplusplus\n}\n#endif\n/**************************************************************************\n *\n * Copyright 2013-2014 RAD Game Tools and Valve Software\n * Copyright 2010-2014 Rich Geldreich and Tenacious Software LLC\n * All Rights Reserved.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n *\n **************************************************************************/\n\n\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n/* ------------------- Low-level Decompression (completely independent from all compression API's) */\n\n#define TINFL_MEMCPY(d, s, l) memcpy(d, s, l)\n#define TINFL_MEMSET(p, c, l) memset(p, c, l)\n\n#define TINFL_CR_BEGIN  \\\n    switch (r->m_state) \\\n    {                   \\\n        case 0:\n#define TINFL_CR_RETURN(state_index, result) \\\n    do                                       \\\n    {                                        \\\n        status = result;                     \\\n        r->m_state = state_index;            \\\n        goto common_exit;                    \\\n        case state_index:;                   \\\n    }                                        \\\n    MZ_MACRO_END\n#define TINFL_CR_RETURN_FOREVER(state_index, result) \\\n    do                                               \\\n    {                                                \\\n        for (;;)                                     \\\n        {                                            \\\n            TINFL_CR_RETURN(state_index, result);    \\\n        }                                            \\\n    }                                                \\\n    MZ_MACRO_END\n#define TINFL_CR_FINISH }\n\n#define TINFL_GET_BYTE(state_index, c)                                                                                                                           \\\n    do                                                                                                                                                           \\\n    {                                                                                                                                                            \\\n        while (pIn_buf_cur >= pIn_buf_end)                                                                                                                       \\\n        {                                                                                                                                                        \\\n            TINFL_CR_RETURN(state_index, (decomp_flags & TINFL_FLAG_HAS_MORE_INPUT) ? TINFL_STATUS_NEEDS_MORE_INPUT : TINFL_STATUS_FAILED_CANNOT_MAKE_PROGRESS); \\\n        }                                                                                                                                                        \\\n        c = *pIn_buf_cur++;                                                                                                                                      \\\n    }                                                                                                                                                            \\\n    MZ_MACRO_END\n\n#define TINFL_NEED_BITS(state_index, n)                \\\n    do                                                 \\\n    {                                                  \\\n        mz_uint c;                                     \\\n        TINFL_GET_BYTE(state_index, c);                \\\n        bit_buf |= (((tinfl_bit_buf_t)c) << num_bits); \\\n        num_bits += 8;                                 \\\n    } while (num_bits < (mz_uint)(n))\n#define TINFL_SKIP_BITS(state_index, n)      \\\n    do                                       \\\n    {                                        \\\n        if (num_bits < (mz_uint)(n))         \\\n        {                                    \\\n            TINFL_NEED_BITS(state_index, n); \\\n        }                                    \\\n        bit_buf >>= (n);                     \\\n        num_bits -= (n);                     \\\n    }                                        \\\n    MZ_MACRO_END\n#define TINFL_GET_BITS(state_index, b, n)    \\\n    do                                       \\\n    {                                        \\\n        if (num_bits < (mz_uint)(n))         \\\n        {                                    \\\n            TINFL_NEED_BITS(state_index, n); \\\n        }                                    \\\n        b = bit_buf & ((1 << (n)) - 1);      \\\n        bit_buf >>= (n);                     \\\n        num_bits -= (n);                     \\\n    }                                        \\\n    MZ_MACRO_END\n\n/* TINFL_HUFF_BITBUF_FILL() is only used rarely, when the number of bytes remaining in the input buffer falls below 2. */\n/* It reads just enough bytes from the input stream that are needed to decode the next Huffman code (and absolutely no more). It works by trying to fully decode a */\n/* Huffman code by using whatever bits are currently present in the bit buffer. If this fails, it reads another byte, and tries again until it succeeds or until the */\n/* bit buffer contains >=15 bits (deflate's max. Huffman code size). */\n#define TINFL_HUFF_BITBUF_FILL(state_index, pHuff)                             \\\n    do                                                                         \\\n    {                                                                          \\\n        temp = (pHuff)->m_look_up[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)];     \\\n        if (temp >= 0)                                                         \\\n        {                                                                      \\\n            code_len = temp >> 9;                                              \\\n            if ((code_len) && (num_bits >= code_len))                          \\\n                break;                                                         \\\n        }                                                                      \\\n        else if (num_bits > TINFL_FAST_LOOKUP_BITS)                            \\\n        {                                                                      \\\n            code_len = TINFL_FAST_LOOKUP_BITS;                                 \\\n            do                                                                 \\\n            {                                                                  \\\n                temp = (pHuff)->m_tree[~temp + ((bit_buf >> code_len++) & 1)]; \\\n            } while ((temp < 0) && (num_bits >= (code_len + 1)));              \\\n            if (temp >= 0)                                                     \\\n                break;                                                         \\\n        }                                                                      \\\n        TINFL_GET_BYTE(state_index, c);                                        \\\n        bit_buf |= (((tinfl_bit_buf_t)c) << num_bits);                         \\\n        num_bits += 8;                                                         \\\n    } while (num_bits < 15);\n\n/* TINFL_HUFF_DECODE() decodes the next Huffman coded symbol. It's more complex than you would initially expect because the zlib API expects the decompressor to never read */\n/* beyond the final byte of the deflate stream. (In other words, when this macro wants to read another byte from the input, it REALLY needs another byte in order to fully */\n/* decode the next Huffman code.) Handling this properly is particularly important on raw deflate (non-zlib) streams, which aren't followed by a byte aligned adler-32. */\n/* The slow path is only executed at the very end of the input buffer. */\n/* v1.16: The original macro handled the case at the very end of the passed-in input buffer, but we also need to handle the case where the user passes in 1+zillion bytes */\n/* following the deflate data and our non-conservative read-ahead path won't kick in here on this code. This is much trickier. */\n#define TINFL_HUFF_DECODE(state_index, sym, pHuff)                                                                                  \\\n    do                                                                                                                              \\\n    {                                                                                                                               \\\n        int temp;                                                                                                                   \\\n        mz_uint code_len, c;                                                                                                        \\\n        if (num_bits < 15)                                                                                                          \\\n        {                                                                                                                           \\\n            if ((pIn_buf_end - pIn_buf_cur) < 2)                                                                                    \\\n            {                                                                                                                       \\\n                TINFL_HUFF_BITBUF_FILL(state_index, pHuff);                                                                         \\\n            }                                                                                                                       \\\n            else                                                                                                                    \\\n            {                                                                                                                       \\\n                bit_buf |= (((tinfl_bit_buf_t)pIn_buf_cur[0]) << num_bits) | (((tinfl_bit_buf_t)pIn_buf_cur[1]) << (num_bits + 8)); \\\n                pIn_buf_cur += 2;                                                                                                   \\\n                num_bits += 16;                                                                                                     \\\n            }                                                                                                                       \\\n        }                                                                                                                           \\\n        if ((temp = (pHuff)->m_look_up[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0)                                               \\\n            code_len = temp >> 9, temp &= 511;                                                                                      \\\n        else                                                                                                                        \\\n        {                                                                                                                           \\\n            code_len = TINFL_FAST_LOOKUP_BITS;                                                                                      \\\n            do                                                                                                                      \\\n            {                                                                                                                       \\\n                temp = (pHuff)->m_tree[~temp + ((bit_buf >> code_len++) & 1)];                                                      \\\n            } while (temp < 0);                                                                                                     \\\n        }                                                                                                                           \\\n        sym = temp;                                                                                                                 \\\n        bit_buf >>= code_len;                                                                                                       \\\n        num_bits -= code_len;                                                                                                       \\\n    }                                                                                                                               \\\n    MZ_MACRO_END\n\ntinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_next, size_t *pIn_buf_size, mz_uint8 *pOut_buf_start, mz_uint8 *pOut_buf_next, size_t *pOut_buf_size, const mz_uint32 decomp_flags)\n{\n    static const int s_length_base[31] = { 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0 };\n    static const int s_length_extra[31] = { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 0, 0 };\n    static const int s_dist_base[32] = { 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577, 0, 0 };\n    static const int s_dist_extra[32] = { 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13 };\n    static const mz_uint8 s_length_dezigzag[19] = { 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };\n    static const int s_min_table_sizes[3] = { 257, 1, 4 };\n\n    tinfl_status status = TINFL_STATUS_FAILED;\n    mz_uint32 num_bits, dist, counter, num_extra;\n    tinfl_bit_buf_t bit_buf;\n    const mz_uint8 *pIn_buf_cur = pIn_buf_next, *const pIn_buf_end = pIn_buf_next + *pIn_buf_size;\n    mz_uint8 *pOut_buf_cur = pOut_buf_next, *const pOut_buf_end = pOut_buf_next + *pOut_buf_size;\n    size_t out_buf_size_mask = (decomp_flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF) ? (size_t)-1 : ((pOut_buf_next - pOut_buf_start) + *pOut_buf_size) - 1, dist_from_out_buf_start;\n\n    /* Ensure the output buffer's size is a power of 2, unless the output buffer is large enough to hold the entire output file (in which case it doesn't matter). */\n    if (((out_buf_size_mask + 1) & out_buf_size_mask) || (pOut_buf_next < pOut_buf_start))\n    {\n        *pIn_buf_size = *pOut_buf_size = 0;\n        return TINFL_STATUS_BAD_PARAM;\n    }\n\n    num_bits = r->m_num_bits;\n    bit_buf = r->m_bit_buf;\n    dist = r->m_dist;\n    counter = r->m_counter;\n    num_extra = r->m_num_extra;\n    dist_from_out_buf_start = r->m_dist_from_out_buf_start;\n    TINFL_CR_BEGIN\n\n    bit_buf = num_bits = dist = counter = num_extra = r->m_zhdr0 = r->m_zhdr1 = 0;\n    r->m_z_adler32 = r->m_check_adler32 = 1;\n    if (decomp_flags & TINFL_FLAG_PARSE_ZLIB_HEADER)\n    {\n        TINFL_GET_BYTE(1, r->m_zhdr0);\n        TINFL_GET_BYTE(2, r->m_zhdr1);\n        counter = (((r->m_zhdr0 * 256 + r->m_zhdr1) % 31 != 0) || (r->m_zhdr1 & 32) || ((r->m_zhdr0 & 15) != 8));\n        if (!(decomp_flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF))\n            counter |= (((1U << (8U + (r->m_zhdr0 >> 4))) > 32768U) || ((out_buf_size_mask + 1) < (size_t)(1U << (8U + (r->m_zhdr0 >> 4)))));\n        if (counter)\n        {\n            TINFL_CR_RETURN_FOREVER(36, TINFL_STATUS_FAILED);\n        }\n    }\n\n    do\n    {\n        TINFL_GET_BITS(3, r->m_final, 3);\n        r->m_type = r->m_final >> 1;\n        if (r->m_type == 0)\n        {\n            TINFL_SKIP_BITS(5, num_bits & 7);\n            for (counter = 0; counter < 4; ++counter)\n            {\n                if (num_bits)\n                    TINFL_GET_BITS(6, r->m_raw_header[counter], 8);\n                else\n                    TINFL_GET_BYTE(7, r->m_raw_header[counter]);\n            }\n            if ((counter = (r->m_raw_header[0] | (r->m_raw_header[1] << 8))) != (mz_uint)(0xFFFF ^ (r->m_raw_header[2] | (r->m_raw_header[3] << 8))))\n            {\n                TINFL_CR_RETURN_FOREVER(39, TINFL_STATUS_FAILED);\n            }\n            while ((counter) && (num_bits))\n            {\n                TINFL_GET_BITS(51, dist, 8);\n                while (pOut_buf_cur >= pOut_buf_end)\n                {\n                    TINFL_CR_RETURN(52, TINFL_STATUS_HAS_MORE_OUTPUT);\n                }\n                *pOut_buf_cur++ = (mz_uint8)dist;\n                counter--;\n            }\n            while (counter)\n            {\n                size_t n;\n                while (pOut_buf_cur >= pOut_buf_end)\n                {\n                    TINFL_CR_RETURN(9, TINFL_STATUS_HAS_MORE_OUTPUT);\n                }\n                while (pIn_buf_cur >= pIn_buf_end)\n                {\n                    TINFL_CR_RETURN(38, (decomp_flags & TINFL_FLAG_HAS_MORE_INPUT) ? TINFL_STATUS_NEEDS_MORE_INPUT : TINFL_STATUS_FAILED_CANNOT_MAKE_PROGRESS);\n                }\n                n = MZ_MIN(MZ_MIN((size_t)(pOut_buf_end - pOut_buf_cur), (size_t)(pIn_buf_end - pIn_buf_cur)), counter);\n                TINFL_MEMCPY(pOut_buf_cur, pIn_buf_cur, n);\n                pIn_buf_cur += n;\n                pOut_buf_cur += n;\n                counter -= (mz_uint)n;\n            }\n        }\n        else if (r->m_type == 3)\n        {\n            TINFL_CR_RETURN_FOREVER(10, TINFL_STATUS_FAILED);\n        }\n        else\n        {\n            if (r->m_type == 1)\n            {\n                mz_uint8 *p = r->m_tables[0].m_code_size;\n                mz_uint i;\n                r->m_table_sizes[0] = 288;\n                r->m_table_sizes[1] = 32;\n                TINFL_MEMSET(r->m_tables[1].m_code_size, 5, 32);\n                for (i = 0; i <= 143; ++i)\n                    *p++ = 8;\n                for (; i <= 255; ++i)\n                    *p++ = 9;\n                for (; i <= 279; ++i)\n                    *p++ = 7;\n                for (; i <= 287; ++i)\n                    *p++ = 8;\n            }\n            else\n            {\n                for (counter = 0; counter < 3; counter++)\n                {\n                    TINFL_GET_BITS(11, r->m_table_sizes[counter], \"\\05\\05\\04\"[counter]);\n                    r->m_table_sizes[counter] += s_min_table_sizes[counter];\n                }\n                MZ_CLEAR_OBJ(r->m_tables[2].m_code_size);\n                for (counter = 0; counter < r->m_table_sizes[2]; counter++)\n                {\n                    mz_uint s;\n                    TINFL_GET_BITS(14, s, 3);\n                    r->m_tables[2].m_code_size[s_length_dezigzag[counter]] = (mz_uint8)s;\n                }\n                r->m_table_sizes[2] = 19;\n            }\n            for (; (int)r->m_type >= 0; r->m_type--)\n            {\n                int tree_next, tree_cur;\n                tinfl_huff_table *pTable;\n                mz_uint i, j, used_syms, total, sym_index, next_code[17], total_syms[16];\n                pTable = &r->m_tables[r->m_type];\n                MZ_CLEAR_OBJ(total_syms);\n                MZ_CLEAR_OBJ(pTable->m_look_up);\n                MZ_CLEAR_OBJ(pTable->m_tree);\n                for (i = 0; i < r->m_table_sizes[r->m_type]; ++i)\n                    total_syms[pTable->m_code_size[i]]++;\n                used_syms = 0, total = 0;\n                next_code[0] = next_code[1] = 0;\n                for (i = 1; i <= 15; ++i)\n                {\n                    used_syms += total_syms[i];\n                    next_code[i + 1] = (total = ((total + total_syms[i]) << 1));\n                }\n                if ((65536 != total) && (used_syms > 1))\n                {\n                    TINFL_CR_RETURN_FOREVER(35, TINFL_STATUS_FAILED);\n                }\n                for (tree_next = -1, sym_index = 0; sym_index < r->m_table_sizes[r->m_type]; ++sym_index)\n                {\n                    mz_uint rev_code = 0, l, cur_code, code_size = pTable->m_code_size[sym_index];\n                    if (!code_size)\n                        continue;\n                    cur_code = next_code[code_size]++;\n                    for (l = code_size; l > 0; l--, cur_code >>= 1)\n                        rev_code = (rev_code << 1) | (cur_code & 1);\n                    if (code_size <= TINFL_FAST_LOOKUP_BITS)\n                    {\n                        mz_int16 k = (mz_int16)((code_size << 9) | sym_index);\n                        while (rev_code < TINFL_FAST_LOOKUP_SIZE)\n                        {\n                            pTable->m_look_up[rev_code] = k;\n                            rev_code += (1 << code_size);\n                        }\n                        continue;\n                    }\n                    if (0 == (tree_cur = pTable->m_look_up[rev_code & (TINFL_FAST_LOOKUP_SIZE - 1)]))\n                    {\n                        pTable->m_look_up[rev_code & (TINFL_FAST_LOOKUP_SIZE - 1)] = (mz_int16)tree_next;\n                        tree_cur = tree_next;\n                        tree_next -= 2;\n                    }\n                    rev_code >>= (TINFL_FAST_LOOKUP_BITS - 1);\n                    for (j = code_size; j > (TINFL_FAST_LOOKUP_BITS + 1); j--)\n                    {\n                        tree_cur -= ((rev_code >>= 1) & 1);\n                        if (!pTable->m_tree[-tree_cur - 1])\n                        {\n                            pTable->m_tree[-tree_cur - 1] = (mz_int16)tree_next;\n                            tree_cur = tree_next;\n                            tree_next -= 2;\n                        }\n                        else\n                            tree_cur = pTable->m_tree[-tree_cur - 1];\n                    }\n                    tree_cur -= ((rev_code >>= 1) & 1);\n                    pTable->m_tree[-tree_cur - 1] = (mz_int16)sym_index;\n                }\n                if (r->m_type == 2)\n                {\n                    for (counter = 0; counter < (r->m_table_sizes[0] + r->m_table_sizes[1]);)\n                    {\n                        mz_uint s;\n                        TINFL_HUFF_DECODE(16, dist, &r->m_tables[2]);\n                        if (dist < 16)\n                        {\n                            r->m_len_codes[counter++] = (mz_uint8)dist;\n                            continue;\n                        }\n                        if ((dist == 16) && (!counter))\n                        {\n                            TINFL_CR_RETURN_FOREVER(17, TINFL_STATUS_FAILED);\n                        }\n                        num_extra = \"\\02\\03\\07\"[dist - 16];\n                        TINFL_GET_BITS(18, s, num_extra);\n                        s += \"\\03\\03\\013\"[dist - 16];\n                        TINFL_MEMSET(r->m_len_codes + counter, (dist == 16) ? r->m_len_codes[counter - 1] : 0, s);\n                        counter += s;\n                    }\n                    if ((r->m_table_sizes[0] + r->m_table_sizes[1]) != counter)\n                    {\n                        TINFL_CR_RETURN_FOREVER(21, TINFL_STATUS_FAILED);\n                    }\n                    TINFL_MEMCPY(r->m_tables[0].m_code_size, r->m_len_codes, r->m_table_sizes[0]);\n                    TINFL_MEMCPY(r->m_tables[1].m_code_size, r->m_len_codes + r->m_table_sizes[0], r->m_table_sizes[1]);\n                }\n            }\n            for (;;)\n            {\n                mz_uint8 *pSrc;\n                for (;;)\n                {\n                    if (((pIn_buf_end - pIn_buf_cur) < 4) || ((pOut_buf_end - pOut_buf_cur) < 2))\n                    {\n                        TINFL_HUFF_DECODE(23, counter, &r->m_tables[0]);\n                        if (counter >= 256)\n                            break;\n                        while (pOut_buf_cur >= pOut_buf_end)\n                        {\n                            TINFL_CR_RETURN(24, TINFL_STATUS_HAS_MORE_OUTPUT);\n                        }\n                        *pOut_buf_cur++ = (mz_uint8)counter;\n                    }\n                    else\n                    {\n                        int sym2;\n                        mz_uint code_len;\n#if TINFL_USE_64BIT_BITBUF\n                        if (num_bits < 30)\n                        {\n                            bit_buf |= (((tinfl_bit_buf_t)MZ_READ_LE32(pIn_buf_cur)) << num_bits);\n                            pIn_buf_cur += 4;\n                            num_bits += 32;\n                        }\n#else\n                        if (num_bits < 15)\n                        {\n                            bit_buf |= (((tinfl_bit_buf_t)MZ_READ_LE16(pIn_buf_cur)) << num_bits);\n                            pIn_buf_cur += 2;\n                            num_bits += 16;\n                        }\n#endif\n                        if ((sym2 = r->m_tables[0].m_look_up[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0)\n                            code_len = sym2 >> 9;\n                        else\n                        {\n                            code_len = TINFL_FAST_LOOKUP_BITS;\n                            do\n                            {\n                                sym2 = r->m_tables[0].m_tree[~sym2 + ((bit_buf >> code_len++) & 1)];\n                            } while (sym2 < 0);\n                        }\n                        counter = sym2;\n                        bit_buf >>= code_len;\n                        num_bits -= code_len;\n                        if (counter & 256)\n                            break;\n\n#if !TINFL_USE_64BIT_BITBUF\n                        if (num_bits < 15)\n                        {\n                            bit_buf |= (((tinfl_bit_buf_t)MZ_READ_LE16(pIn_buf_cur)) << num_bits);\n                            pIn_buf_cur += 2;\n                            num_bits += 16;\n                        }\n#endif\n                        if ((sym2 = r->m_tables[0].m_look_up[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0)\n                            code_len = sym2 >> 9;\n                        else\n                        {\n                            code_len = TINFL_FAST_LOOKUP_BITS;\n                            do\n                            {\n                                sym2 = r->m_tables[0].m_tree[~sym2 + ((bit_buf >> code_len++) & 1)];\n                            } while (sym2 < 0);\n                        }\n                        bit_buf >>= code_len;\n                        num_bits -= code_len;\n\n                        pOut_buf_cur[0] = (mz_uint8)counter;\n                        if (sym2 & 256)\n                        {\n                            pOut_buf_cur++;\n                            counter = sym2;\n                            break;\n                        }\n                        pOut_buf_cur[1] = (mz_uint8)sym2;\n                        pOut_buf_cur += 2;\n                    }\n                }\n                if ((counter &= 511) == 256)\n                    break;\n\n                num_extra = s_length_extra[counter - 257];\n                counter = s_length_base[counter - 257];\n                if (num_extra)\n                {\n                    mz_uint extra_bits;\n                    TINFL_GET_BITS(25, extra_bits, num_extra);\n                    counter += extra_bits;\n                }\n\n                TINFL_HUFF_DECODE(26, dist, &r->m_tables[1]);\n                num_extra = s_dist_extra[dist];\n                dist = s_dist_base[dist];\n                if (num_extra)\n                {\n                    mz_uint extra_bits;\n                    TINFL_GET_BITS(27, extra_bits, num_extra);\n                    dist += extra_bits;\n                }\n\n                dist_from_out_buf_start = pOut_buf_cur - pOut_buf_start;\n                if ((dist > dist_from_out_buf_start) && (decomp_flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF))\n                {\n                    TINFL_CR_RETURN_FOREVER(37, TINFL_STATUS_FAILED);\n                }\n\n                pSrc = pOut_buf_start + ((dist_from_out_buf_start - dist) & out_buf_size_mask);\n\n                if ((MZ_MAX(pOut_buf_cur, pSrc) + counter) > pOut_buf_end)\n                {\n                    while (counter--)\n                    {\n                        while (pOut_buf_cur >= pOut_buf_end)\n                        {\n                            TINFL_CR_RETURN(53, TINFL_STATUS_HAS_MORE_OUTPUT);\n                        }\n                        *pOut_buf_cur++ = pOut_buf_start[(dist_from_out_buf_start++ - dist) & out_buf_size_mask];\n                    }\n                    continue;\n                }\n#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES\n                else if ((counter >= 9) && (counter <= dist))\n                {\n                    const mz_uint8 *pSrc_end = pSrc + (counter & ~7);\n                    do\n                    {\n#ifdef MINIZ_UNALIGNED_USE_MEMCPY\n\t\t\t\t\t\tmemcpy(pOut_buf_cur, pSrc, sizeof(mz_uint32)*2);\n#else\n                        ((mz_uint32 *)pOut_buf_cur)[0] = ((const mz_uint32 *)pSrc)[0];\n                        ((mz_uint32 *)pOut_buf_cur)[1] = ((const mz_uint32 *)pSrc)[1];\n#endif\n                        pOut_buf_cur += 8;\n                    } while ((pSrc += 8) < pSrc_end);\n                    if ((counter &= 7) < 3)\n                    {\n                        if (counter)\n                        {\n                            pOut_buf_cur[0] = pSrc[0];\n                            if (counter > 1)\n                                pOut_buf_cur[1] = pSrc[1];\n                            pOut_buf_cur += counter;\n                        }\n                        continue;\n                    }\n                }\n#endif\n                while(counter>2)\n                {\n                    pOut_buf_cur[0] = pSrc[0];\n                    pOut_buf_cur[1] = pSrc[1];\n                    pOut_buf_cur[2] = pSrc[2];\n                    pOut_buf_cur += 3;\n                    pSrc += 3;\n\t\t\t\t\tcounter -= 3;\n                }\n                if (counter > 0)\n                {\n                    pOut_buf_cur[0] = pSrc[0];\n                    if (counter > 1)\n                        pOut_buf_cur[1] = pSrc[1];\n                    pOut_buf_cur += counter;\n                }\n            }\n        }\n    } while (!(r->m_final & 1));\n\n    /* Ensure byte alignment and put back any bytes from the bitbuf if we've looked ahead too far on gzip, or other Deflate streams followed by arbitrary data. */\n    /* I'm being super conservative here. A number of simplifications can be made to the byte alignment part, and the Adler32 check shouldn't ever need to worry about reading from the bitbuf now. */\n    TINFL_SKIP_BITS(32, num_bits & 7);\n    while ((pIn_buf_cur > pIn_buf_next) && (num_bits >= 8))\n    {\n        --pIn_buf_cur;\n        num_bits -= 8;\n    }\n    bit_buf &= (tinfl_bit_buf_t)((((mz_uint64)1) << num_bits) - (mz_uint64)1);\n    MZ_ASSERT(!num_bits); /* if this assert fires then we've read beyond the end of non-deflate/zlib streams with following data (such as gzip streams). */\n\n    if (decomp_flags & TINFL_FLAG_PARSE_ZLIB_HEADER)\n    {\n        for (counter = 0; counter < 4; ++counter)\n        {\n            mz_uint s;\n            if (num_bits)\n                TINFL_GET_BITS(41, s, 8);\n            else\n                TINFL_GET_BYTE(42, s);\n            r->m_z_adler32 = (r->m_z_adler32 << 8) | s;\n        }\n    }\n    TINFL_CR_RETURN_FOREVER(34, TINFL_STATUS_DONE);\n\n    TINFL_CR_FINISH\n\ncommon_exit:\n    /* As long as we aren't telling the caller that we NEED more input to make forward progress: */\n    /* Put back any bytes from the bitbuf in case we've looked ahead too far on gzip, or other Deflate streams followed by arbitrary data. */\n    /* We need to be very careful here to NOT push back any bytes we definitely know we need to make forward progress, though, or we'll lock the caller up into an inf loop. */\n    if ((status != TINFL_STATUS_NEEDS_MORE_INPUT) && (status != TINFL_STATUS_FAILED_CANNOT_MAKE_PROGRESS))\n    {\n        while ((pIn_buf_cur > pIn_buf_next) && (num_bits >= 8))\n        {\n            --pIn_buf_cur;\n            num_bits -= 8;\n        }\n    }\n    r->m_num_bits = num_bits;\n    r->m_bit_buf = bit_buf & (tinfl_bit_buf_t)((((mz_uint64)1) << num_bits) - (mz_uint64)1);\n    r->m_dist = dist;\n    r->m_counter = counter;\n    r->m_num_extra = num_extra;\n    r->m_dist_from_out_buf_start = dist_from_out_buf_start;\n    *pIn_buf_size = pIn_buf_cur - pIn_buf_next;\n    *pOut_buf_size = pOut_buf_cur - pOut_buf_next;\n    if ((decomp_flags & (TINFL_FLAG_PARSE_ZLIB_HEADER | TINFL_FLAG_COMPUTE_ADLER32)) && (status >= 0))\n    {\n        const mz_uint8 *ptr = pOut_buf_next;\n        size_t buf_len = *pOut_buf_size;\n        mz_uint32 i, s1 = r->m_check_adler32 & 0xffff, s2 = r->m_check_adler32 >> 16;\n        size_t block_len = buf_len % 5552;\n        while (buf_len)\n        {\n            for (i = 0; i + 7 < block_len; i += 8, ptr += 8)\n            {\n                s1 += ptr[0], s2 += s1;\n                s1 += ptr[1], s2 += s1;\n                s1 += ptr[2], s2 += s1;\n                s1 += ptr[3], s2 += s1;\n                s1 += ptr[4], s2 += s1;\n                s1 += ptr[5], s2 += s1;\n                s1 += ptr[6], s2 += s1;\n                s1 += ptr[7], s2 += s1;\n            }\n            for (; i < block_len; ++i)\n                s1 += *ptr++, s2 += s1;\n            s1 %= 65521U, s2 %= 65521U;\n            buf_len -= block_len;\n            block_len = 5552;\n        }\n        r->m_check_adler32 = (s2 << 16) + s1;\n        if ((status == TINFL_STATUS_DONE) && (decomp_flags & TINFL_FLAG_PARSE_ZLIB_HEADER) && (r->m_check_adler32 != r->m_z_adler32))\n            status = TINFL_STATUS_ADLER32_MISMATCH;\n    }\n    return status;\n}\n\n/* Higher level helper functions. */\nvoid *tinfl_decompress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, size_t *pOut_len, int flags)\n{\n    tinfl_decompressor decomp;\n    void *pBuf = NULL, *pNew_buf;\n    size_t src_buf_ofs = 0, out_buf_capacity = 0;\n    *pOut_len = 0;\n    tinfl_init(&decomp);\n    for (;;)\n    {\n        size_t src_buf_size = src_buf_len - src_buf_ofs, dst_buf_size = out_buf_capacity - *pOut_len, new_out_buf_capacity;\n        tinfl_status status = tinfl_decompress(&decomp, (const mz_uint8 *)pSrc_buf + src_buf_ofs, &src_buf_size, (mz_uint8 *)pBuf, pBuf ? (mz_uint8 *)pBuf + *pOut_len : NULL, &dst_buf_size,\n                                               (flags & ~TINFL_FLAG_HAS_MORE_INPUT) | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF);\n        if ((status < 0) || (status == TINFL_STATUS_NEEDS_MORE_INPUT))\n        {\n            MZ_FREE(pBuf);\n            *pOut_len = 0;\n            return NULL;\n        }\n        src_buf_ofs += src_buf_size;\n        *pOut_len += dst_buf_size;\n        if (status == TINFL_STATUS_DONE)\n            break;\n        new_out_buf_capacity = out_buf_capacity * 2;\n        if (new_out_buf_capacity < 128)\n            new_out_buf_capacity = 128;\n        pNew_buf = MZ_REALLOC(pBuf, new_out_buf_capacity);\n        if (!pNew_buf)\n        {\n            MZ_FREE(pBuf);\n            *pOut_len = 0;\n            return NULL;\n        }\n        pBuf = pNew_buf;\n        out_buf_capacity = new_out_buf_capacity;\n    }\n    return pBuf;\n}\n\nsize_t tinfl_decompress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void *pSrc_buf, size_t src_buf_len, int flags)\n{\n    tinfl_decompressor decomp;\n    tinfl_status status;\n    tinfl_init(&decomp);\n    status = tinfl_decompress(&decomp, (const mz_uint8 *)pSrc_buf, &src_buf_len, (mz_uint8 *)pOut_buf, (mz_uint8 *)pOut_buf, &out_buf_len, (flags & ~TINFL_FLAG_HAS_MORE_INPUT) | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF);\n    return (status != TINFL_STATUS_DONE) ? TINFL_DECOMPRESS_MEM_TO_MEM_FAILED : out_buf_len;\n}\n\nint tinfl_decompress_mem_to_callback(const void *pIn_buf, size_t *pIn_buf_size, tinfl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags)\n{\n    int result = 0;\n    tinfl_decompressor decomp;\n    mz_uint8 *pDict = (mz_uint8 *)MZ_MALLOC(TINFL_LZ_DICT_SIZE);\n    size_t in_buf_ofs = 0, dict_ofs = 0;\n    if (!pDict)\n        return TINFL_STATUS_FAILED;\n    tinfl_init(&decomp);\n    for (;;)\n    {\n        size_t in_buf_size = *pIn_buf_size - in_buf_ofs, dst_buf_size = TINFL_LZ_DICT_SIZE - dict_ofs;\n        tinfl_status status = tinfl_decompress(&decomp, (const mz_uint8 *)pIn_buf + in_buf_ofs, &in_buf_size, pDict, pDict + dict_ofs, &dst_buf_size,\n                                               (flags & ~(TINFL_FLAG_HAS_MORE_INPUT | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF)));\n        in_buf_ofs += in_buf_size;\n        if ((dst_buf_size) && (!(*pPut_buf_func)(pDict + dict_ofs, (int)dst_buf_size, pPut_buf_user)))\n            break;\n        if (status != TINFL_STATUS_HAS_MORE_OUTPUT)\n        {\n            result = (status == TINFL_STATUS_DONE);\n            break;\n        }\n        dict_ofs = (dict_ofs + dst_buf_size) & (TINFL_LZ_DICT_SIZE - 1);\n    }\n    MZ_FREE(pDict);\n    *pIn_buf_size = in_buf_ofs;\n    return result;\n}\n\n#ifndef MINIZ_NO_MALLOC\ntinfl_decompressor *tinfl_decompressor_alloc()\n{\n    tinfl_decompressor *pDecomp = (tinfl_decompressor *)MZ_MALLOC(sizeof(tinfl_decompressor));\n    if (pDecomp)\n        tinfl_init(pDecomp);\n    return pDecomp;\n}\n\nvoid tinfl_decompressor_free(tinfl_decompressor *pDecomp)\n{\n    MZ_FREE(pDecomp);\n}\n#endif\n\n#ifdef __cplusplus\n}\n#endif\n/**************************************************************************\n *\n * Copyright 2013-2014 RAD Game Tools and Valve Software\n * Copyright 2010-2014 Rich Geldreich and Tenacious Software LLC\n * Copyright 2016 Martin Raiber\n * All Rights Reserved.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n *\n **************************************************************************/\n\n\n#ifndef MINIZ_NO_ARCHIVE_APIS\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n/* ------------------- .ZIP archive reading */\n\n#ifdef MINIZ_NO_STDIO\n#define MZ_FILE void *\n#else\n#include <sys/stat.h>\n\n#if defined(_MSC_VER) || defined(__MINGW64__)\nstatic FILE *mz_fopen(const char *pFilename, const char *pMode)\n{\n    FILE *pFile = NULL;\n    fopen_s(&pFile, pFilename, pMode);\n    return pFile;\n}\nstatic FILE *mz_freopen(const char *pPath, const char *pMode, FILE *pStream)\n{\n    FILE *pFile = NULL;\n    if (freopen_s(&pFile, pPath, pMode, pStream))\n        return NULL;\n    return pFile;\n}\n#ifndef MINIZ_NO_TIME\n#include <sys/utime.h>\n#endif\n#define MZ_FOPEN mz_fopen\n#define MZ_FCLOSE fclose\n#define MZ_FREAD fread\n#define MZ_FWRITE fwrite\n#define MZ_FTELL64 _ftelli64\n#define MZ_FSEEK64 _fseeki64\n#define MZ_FILE_STAT_STRUCT _stat64\n#define MZ_FILE_STAT _stat64\n#define MZ_FFLUSH fflush\n#define MZ_FREOPEN mz_freopen\n#define MZ_DELETE_FILE remove\n#elif defined(__MINGW32__)\n#ifndef MINIZ_NO_TIME\n#include <sys/utime.h>\n#endif\n#define MZ_FOPEN(f, m) fopen(f, m)\n#define MZ_FCLOSE fclose\n#define MZ_FREAD fread\n#define MZ_FWRITE fwrite\n#define MZ_FTELL64 ftello64\n#define MZ_FSEEK64 fseeko64\n#define MZ_FILE_STAT_STRUCT _stat\n#define MZ_FILE_STAT _stat\n#define MZ_FFLUSH fflush\n#define MZ_FREOPEN(f, m, s) freopen(f, m, s)\n#define MZ_DELETE_FILE remove\n#elif defined(__TINYC__)\n#ifndef MINIZ_NO_TIME\n#include <sys/utime.h>\n#endif\n#define MZ_FOPEN(f, m) fopen(f, m)\n#define MZ_FCLOSE fclose\n#define MZ_FREAD fread\n#define MZ_FWRITE fwrite\n#define MZ_FTELL64 ftell\n#define MZ_FSEEK64 fseek\n#define MZ_FILE_STAT_STRUCT stat\n#define MZ_FILE_STAT stat\n#define MZ_FFLUSH fflush\n#define MZ_FREOPEN(f, m, s) freopen(f, m, s)\n#define MZ_DELETE_FILE remove\n#elif defined(__GNUC__) && defined(_LARGEFILE64_SOURCE)\n#ifndef MINIZ_NO_TIME\n#include <utime.h>\n#endif\n#define MZ_FOPEN(f, m) fopen64(f, m)\n#define MZ_FCLOSE fclose\n#define MZ_FREAD fread\n#define MZ_FWRITE fwrite\n#define MZ_FTELL64 ftello64\n#define MZ_FSEEK64 fseeko64\n#define MZ_FILE_STAT_STRUCT stat64\n#define MZ_FILE_STAT stat64\n#define MZ_FFLUSH fflush\n#define MZ_FREOPEN(p, m, s) freopen64(p, m, s)\n#define MZ_DELETE_FILE remove\n#elif defined(__APPLE__)\n#ifndef MINIZ_NO_TIME\n#include <utime.h>\n#endif\n#define MZ_FOPEN(f, m) fopen(f, m)\n#define MZ_FCLOSE fclose\n#define MZ_FREAD fread\n#define MZ_FWRITE fwrite\n#define MZ_FTELL64 ftello\n#define MZ_FSEEK64 fseeko\n#define MZ_FILE_STAT_STRUCT stat\n#define MZ_FILE_STAT stat\n#define MZ_FFLUSH fflush\n#define MZ_FREOPEN(p, m, s) freopen(p, m, s)\n#define MZ_DELETE_FILE remove\n\n#else\n#pragma message(\"Using fopen, ftello, fseeko, stat() etc. path for file I/O - this path may not support large files.\")\n#ifndef MINIZ_NO_TIME\n#include <utime.h>\n#endif\n#define MZ_FOPEN(f, m) fopen(f, m)\n#define MZ_FCLOSE fclose\n#define MZ_FREAD fread\n#define MZ_FWRITE fwrite\n#ifdef __STRICT_ANSI__\n#define MZ_FTELL64 ftell\n#define MZ_FSEEK64 fseek\n#else\n#define MZ_FTELL64 ftello\n#define MZ_FSEEK64 fseeko\n#endif\n#define MZ_FILE_STAT_STRUCT stat\n#define MZ_FILE_STAT stat\n#define MZ_FFLUSH fflush\n#define MZ_FREOPEN(f, m, s) freopen(f, m, s)\n#define MZ_DELETE_FILE remove\n#endif /* #ifdef _MSC_VER */\n#endif /* #ifdef MINIZ_NO_STDIO */\n\n#define MZ_TOLOWER(c) ((((c) >= 'A') && ((c) <= 'Z')) ? ((c) - 'A' + 'a') : (c))\n\n/* Various ZIP archive enums. To completely avoid cross platform compiler alignment and platform endian issues, miniz.c doesn't use structs for any of this stuff. */\nenum\n{\n    /* ZIP archive identifiers and record sizes */\n    MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIG = 0x06054b50,\n    MZ_ZIP_CENTRAL_DIR_HEADER_SIG = 0x02014b50,\n    MZ_ZIP_LOCAL_DIR_HEADER_SIG = 0x04034b50,\n    MZ_ZIP_LOCAL_DIR_HEADER_SIZE = 30,\n    MZ_ZIP_CENTRAL_DIR_HEADER_SIZE = 46,\n    MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE = 22,\n\n    /* ZIP64 archive identifier and record sizes */\n    MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIG = 0x06064b50,\n    MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIG = 0x07064b50,\n    MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE = 56,\n    MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIZE = 20,\n    MZ_ZIP64_EXTENDED_INFORMATION_FIELD_HEADER_ID = 0x0001,\n    MZ_ZIP_DATA_DESCRIPTOR_ID = 0x08074b50,\n    MZ_ZIP_DATA_DESCRIPTER_SIZE64 = 24,\n    MZ_ZIP_DATA_DESCRIPTER_SIZE32 = 16,\n\n    /* Central directory header record offsets */\n    MZ_ZIP_CDH_SIG_OFS = 0,\n    MZ_ZIP_CDH_VERSION_MADE_BY_OFS = 4,\n    MZ_ZIP_CDH_VERSION_NEEDED_OFS = 6,\n    MZ_ZIP_CDH_BIT_FLAG_OFS = 8,\n    MZ_ZIP_CDH_METHOD_OFS = 10,\n    MZ_ZIP_CDH_FILE_TIME_OFS = 12,\n    MZ_ZIP_CDH_FILE_DATE_OFS = 14,\n    MZ_ZIP_CDH_CRC32_OFS = 16,\n    MZ_ZIP_CDH_COMPRESSED_SIZE_OFS = 20,\n    MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS = 24,\n    MZ_ZIP_CDH_FILENAME_LEN_OFS = 28,\n    MZ_ZIP_CDH_EXTRA_LEN_OFS = 30,\n    MZ_ZIP_CDH_COMMENT_LEN_OFS = 32,\n    MZ_ZIP_CDH_DISK_START_OFS = 34,\n    MZ_ZIP_CDH_INTERNAL_ATTR_OFS = 36,\n    MZ_ZIP_CDH_EXTERNAL_ATTR_OFS = 38,\n    MZ_ZIP_CDH_LOCAL_HEADER_OFS = 42,\n\n    /* Local directory header offsets */\n    MZ_ZIP_LDH_SIG_OFS = 0,\n    MZ_ZIP_LDH_VERSION_NEEDED_OFS = 4,\n    MZ_ZIP_LDH_BIT_FLAG_OFS = 6,\n    MZ_ZIP_LDH_METHOD_OFS = 8,\n    MZ_ZIP_LDH_FILE_TIME_OFS = 10,\n    MZ_ZIP_LDH_FILE_DATE_OFS = 12,\n    MZ_ZIP_LDH_CRC32_OFS = 14,\n    MZ_ZIP_LDH_COMPRESSED_SIZE_OFS = 18,\n    MZ_ZIP_LDH_DECOMPRESSED_SIZE_OFS = 22,\n    MZ_ZIP_LDH_FILENAME_LEN_OFS = 26,\n    MZ_ZIP_LDH_EXTRA_LEN_OFS = 28,\n    MZ_ZIP_LDH_BIT_FLAG_HAS_LOCATOR = 1 << 3,\n\n    /* End of central directory offsets */\n    MZ_ZIP_ECDH_SIG_OFS = 0,\n    MZ_ZIP_ECDH_NUM_THIS_DISK_OFS = 4,\n    MZ_ZIP_ECDH_NUM_DISK_CDIR_OFS = 6,\n    MZ_ZIP_ECDH_CDIR_NUM_ENTRIES_ON_DISK_OFS = 8,\n    MZ_ZIP_ECDH_CDIR_TOTAL_ENTRIES_OFS = 10,\n    MZ_ZIP_ECDH_CDIR_SIZE_OFS = 12,\n    MZ_ZIP_ECDH_CDIR_OFS_OFS = 16,\n    MZ_ZIP_ECDH_COMMENT_SIZE_OFS = 20,\n\n    /* ZIP64 End of central directory locator offsets */\n    MZ_ZIP64_ECDL_SIG_OFS = 0,                    /* 4 bytes */\n    MZ_ZIP64_ECDL_NUM_DISK_CDIR_OFS = 4,          /* 4 bytes */\n    MZ_ZIP64_ECDL_REL_OFS_TO_ZIP64_ECDR_OFS = 8,  /* 8 bytes */\n    MZ_ZIP64_ECDL_TOTAL_NUMBER_OF_DISKS_OFS = 16, /* 4 bytes */\n\n    /* ZIP64 End of central directory header offsets */\n    MZ_ZIP64_ECDH_SIG_OFS = 0,                       /* 4 bytes */\n    MZ_ZIP64_ECDH_SIZE_OF_RECORD_OFS = 4,            /* 8 bytes */\n    MZ_ZIP64_ECDH_VERSION_MADE_BY_OFS = 12,          /* 2 bytes */\n    MZ_ZIP64_ECDH_VERSION_NEEDED_OFS = 14,           /* 2 bytes */\n    MZ_ZIP64_ECDH_NUM_THIS_DISK_OFS = 16,            /* 4 bytes */\n    MZ_ZIP64_ECDH_NUM_DISK_CDIR_OFS = 20,            /* 4 bytes */\n    MZ_ZIP64_ECDH_CDIR_NUM_ENTRIES_ON_DISK_OFS = 24, /* 8 bytes */\n    MZ_ZIP64_ECDH_CDIR_TOTAL_ENTRIES_OFS = 32,       /* 8 bytes */\n    MZ_ZIP64_ECDH_CDIR_SIZE_OFS = 40,                /* 8 bytes */\n    MZ_ZIP64_ECDH_CDIR_OFS_OFS = 48,                 /* 8 bytes */\n    MZ_ZIP_VERSION_MADE_BY_DOS_FILESYSTEM_ID = 0,\n    MZ_ZIP_DOS_DIR_ATTRIBUTE_BITFLAG = 0x10,\n    MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_IS_ENCRYPTED = 1,\n    MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_COMPRESSED_PATCH_FLAG = 32,\n    MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_USES_STRONG_ENCRYPTION = 64,\n    MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_LOCAL_DIR_IS_MASKED = 8192,\n    MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_UTF8 = 1 << 11\n};\n\ntypedef struct\n{\n    void *m_p;\n    size_t m_size, m_capacity;\n    mz_uint m_element_size;\n} mz_zip_array;\n\nstruct mz_zip_internal_state_tag\n{\n    mz_zip_array m_central_dir;\n    mz_zip_array m_central_dir_offsets;\n    mz_zip_array m_sorted_central_dir_offsets;\n\n    /* The flags passed in when the archive is initially opened. */\n    uint32_t m_init_flags;\n\n    /* MZ_TRUE if the archive has a zip64 end of central directory headers, etc. */\n    mz_bool m_zip64;\n\n    /* MZ_TRUE if we found zip64 extended info in the central directory (m_zip64 will also be slammed to true too, even if we didn't find a zip64 end of central dir header, etc.) */\n    mz_bool m_zip64_has_extended_info_fields;\n\n    /* These fields are used by the file, FILE, memory, and memory/heap read/write helpers. */\n    MZ_FILE *m_pFile;\n    mz_uint64 m_file_archive_start_ofs;\n\n    void *m_pMem;\n    size_t m_mem_size;\n    size_t m_mem_capacity;\n};\n\n#define MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(array_ptr, element_size) (array_ptr)->m_element_size = element_size\n\n#if defined(DEBUG) || defined(_DEBUG) || defined(NDEBUG)\nstatic MZ_FORCEINLINE mz_uint mz_zip_array_range_check(const mz_zip_array *pArray, mz_uint index)\n{\n    MZ_ASSERT(index < pArray->m_size);\n    return index;\n}\n#define MZ_ZIP_ARRAY_ELEMENT(array_ptr, element_type, index) ((element_type *)((array_ptr)->m_p))[mz_zip_array_range_check(array_ptr, index)]\n#else\n#define MZ_ZIP_ARRAY_ELEMENT(array_ptr, element_type, index) ((element_type *)((array_ptr)->m_p))[index]\n#endif\n\nstatic MZ_FORCEINLINE void mz_zip_array_init(mz_zip_array *pArray, mz_uint32 element_size)\n{\n    memset(pArray, 0, sizeof(mz_zip_array));\n    pArray->m_element_size = element_size;\n}\n\nstatic MZ_FORCEINLINE void mz_zip_array_clear(mz_zip_archive *pZip, mz_zip_array *pArray)\n{\n    pZip->m_pFree(pZip->m_pAlloc_opaque, pArray->m_p);\n    memset(pArray, 0, sizeof(mz_zip_array));\n}\n\nstatic mz_bool mz_zip_array_ensure_capacity(mz_zip_archive *pZip, mz_zip_array *pArray, size_t min_new_capacity, mz_uint growing)\n{\n    void *pNew_p;\n    size_t new_capacity = min_new_capacity;\n    MZ_ASSERT(pArray->m_element_size);\n    if (pArray->m_capacity >= min_new_capacity)\n        return MZ_TRUE;\n    if (growing)\n    {\n        new_capacity = MZ_MAX(1, pArray->m_capacity);\n        while (new_capacity < min_new_capacity)\n            new_capacity *= 2;\n    }\n    if (NULL == (pNew_p = pZip->m_pRealloc(pZip->m_pAlloc_opaque, pArray->m_p, pArray->m_element_size, new_capacity)))\n        return MZ_FALSE;\n    pArray->m_p = pNew_p;\n    pArray->m_capacity = new_capacity;\n    return MZ_TRUE;\n}\n\nstatic MZ_FORCEINLINE mz_bool mz_zip_array_reserve(mz_zip_archive *pZip, mz_zip_array *pArray, size_t new_capacity, mz_uint growing)\n{\n    if (new_capacity > pArray->m_capacity)\n    {\n        if (!mz_zip_array_ensure_capacity(pZip, pArray, new_capacity, growing))\n            return MZ_FALSE;\n    }\n    return MZ_TRUE;\n}\n\nstatic MZ_FORCEINLINE mz_bool mz_zip_array_resize(mz_zip_archive *pZip, mz_zip_array *pArray, size_t new_size, mz_uint growing)\n{\n    if (new_size > pArray->m_capacity)\n    {\n        if (!mz_zip_array_ensure_capacity(pZip, pArray, new_size, growing))\n            return MZ_FALSE;\n    }\n    pArray->m_size = new_size;\n    return MZ_TRUE;\n}\n\nstatic MZ_FORCEINLINE mz_bool mz_zip_array_ensure_room(mz_zip_archive *pZip, mz_zip_array *pArray, size_t n)\n{\n    return mz_zip_array_reserve(pZip, pArray, pArray->m_size + n, MZ_TRUE);\n}\n\nstatic MZ_FORCEINLINE mz_bool mz_zip_array_push_back(mz_zip_archive *pZip, mz_zip_array *pArray, const void *pElements, size_t n)\n{\n    size_t orig_size = pArray->m_size;\n    if (!mz_zip_array_resize(pZip, pArray, orig_size + n, MZ_TRUE))\n        return MZ_FALSE;\n    if (n > 0)\n        memcpy((mz_uint8 *)pArray->m_p + orig_size * pArray->m_element_size, pElements, n * pArray->m_element_size);\n    return MZ_TRUE;\n}\n\n#ifndef MINIZ_NO_TIME\nstatic MZ_TIME_T mz_zip_dos_to_time_t(int dos_time, int dos_date)\n{\n    struct tm tm;\n    memset(&tm, 0, sizeof(tm));\n    tm.tm_isdst = -1;\n    tm.tm_year = ((dos_date >> 9) & 127) + 1980 - 1900;\n    tm.tm_mon = ((dos_date >> 5) & 15) - 1;\n    tm.tm_mday = dos_date & 31;\n    tm.tm_hour = (dos_time >> 11) & 31;\n    tm.tm_min = (dos_time >> 5) & 63;\n    tm.tm_sec = (dos_time << 1) & 62;\n    return mktime(&tm);\n}\n\n#ifndef MINIZ_NO_ARCHIVE_WRITING_APIS\nstatic void mz_zip_time_t_to_dos_time(MZ_TIME_T time, mz_uint16 *pDOS_time, mz_uint16 *pDOS_date)\n{\n#ifdef _MSC_VER\n    struct tm tm_struct;\n    struct tm *tm = &tm_struct;\n    errno_t err = localtime_s(tm, &time);\n    if (err)\n    {\n        *pDOS_date = 0;\n        *pDOS_time = 0;\n        return;\n    }\n#else\n    struct tm *tm = localtime(&time);\n#endif /* #ifdef _MSC_VER */\n\n    *pDOS_time = (mz_uint16)(((tm->tm_hour) << 11) + ((tm->tm_min) << 5) + ((tm->tm_sec) >> 1));\n    *pDOS_date = (mz_uint16)(((tm->tm_year + 1900 - 1980) << 9) + ((tm->tm_mon + 1) << 5) + tm->tm_mday);\n}\n#endif /* MINIZ_NO_ARCHIVE_WRITING_APIS */\n\n#ifndef MINIZ_NO_STDIO\n#ifndef MINIZ_NO_ARCHIVE_WRITING_APIS\nstatic mz_bool mz_zip_get_file_modified_time(const char *pFilename, MZ_TIME_T *pTime)\n{\n    struct MZ_FILE_STAT_STRUCT file_stat;\n\n    /* On Linux with x86 glibc, this call will fail on large files (I think >= 0x80000000 bytes) unless you compiled with _LARGEFILE64_SOURCE. Argh. */\n    if (MZ_FILE_STAT(pFilename, &file_stat) != 0)\n        return MZ_FALSE;\n\n    *pTime = file_stat.st_mtime;\n\n    return MZ_TRUE;\n}\n#endif /* #ifndef MINIZ_NO_ARCHIVE_WRITING_APIS*/\n\nstatic mz_bool mz_zip_set_file_times(const char *pFilename, MZ_TIME_T access_time, MZ_TIME_T modified_time)\n{\n    struct utimbuf t;\n\n    memset(&t, 0, sizeof(t));\n    t.actime = access_time;\n    t.modtime = modified_time;\n\n    return !utime(pFilename, &t);\n}\n#endif /* #ifndef MINIZ_NO_STDIO */\n#endif /* #ifndef MINIZ_NO_TIME */\n\nstatic MZ_FORCEINLINE mz_bool mz_zip_set_error(mz_zip_archive *pZip, mz_zip_error err_num)\n{\n    if (pZip)\n        pZip->m_last_error = err_num;\n    return MZ_FALSE;\n}\n\nstatic mz_bool mz_zip_reader_init_internal(mz_zip_archive *pZip, mz_uint flags)\n{\n    (void)flags;\n    if ((!pZip) || (pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_INVALID))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    if (!pZip->m_pAlloc)\n        pZip->m_pAlloc = miniz_def_alloc_func;\n    if (!pZip->m_pFree)\n        pZip->m_pFree = miniz_def_free_func;\n    if (!pZip->m_pRealloc)\n        pZip->m_pRealloc = miniz_def_realloc_func;\n\n    pZip->m_archive_size = 0;\n    pZip->m_central_directory_file_ofs = 0;\n    pZip->m_total_files = 0;\n    pZip->m_last_error = MZ_ZIP_NO_ERROR;\n\n    if (NULL == (pZip->m_pState = (mz_zip_internal_state *)pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, sizeof(mz_zip_internal_state))))\n        return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n\n    memset(pZip->m_pState, 0, sizeof(mz_zip_internal_state));\n    MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_central_dir, sizeof(mz_uint8));\n    MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_central_dir_offsets, sizeof(mz_uint32));\n    MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_sorted_central_dir_offsets, sizeof(mz_uint32));\n    pZip->m_pState->m_init_flags = flags;\n    pZip->m_pState->m_zip64 = MZ_FALSE;\n    pZip->m_pState->m_zip64_has_extended_info_fields = MZ_FALSE;\n\n    pZip->m_zip_mode = MZ_ZIP_MODE_READING;\n\n    return MZ_TRUE;\n}\n\nstatic MZ_FORCEINLINE mz_bool mz_zip_reader_filename_less(const mz_zip_array *pCentral_dir_array, const mz_zip_array *pCentral_dir_offsets, mz_uint l_index, mz_uint r_index)\n{\n    const mz_uint8 *pL = &MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_array, mz_uint8, MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_offsets, mz_uint32, l_index)), *pE;\n    const mz_uint8 *pR = &MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_array, mz_uint8, MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_offsets, mz_uint32, r_index));\n    mz_uint l_len = MZ_READ_LE16(pL + MZ_ZIP_CDH_FILENAME_LEN_OFS), r_len = MZ_READ_LE16(pR + MZ_ZIP_CDH_FILENAME_LEN_OFS);\n    mz_uint8 l = 0, r = 0;\n    pL += MZ_ZIP_CENTRAL_DIR_HEADER_SIZE;\n    pR += MZ_ZIP_CENTRAL_DIR_HEADER_SIZE;\n    pE = pL + MZ_MIN(l_len, r_len);\n    while (pL < pE)\n    {\n        if ((l = MZ_TOLOWER(*pL)) != (r = MZ_TOLOWER(*pR)))\n            break;\n        pL++;\n        pR++;\n    }\n    return (pL == pE) ? (l_len < r_len) : (l < r);\n}\n\n#define MZ_SWAP_UINT32(a, b) \\\n    do                       \\\n    {                        \\\n        mz_uint32 t = a;     \\\n        a = b;               \\\n        b = t;               \\\n    }                        \\\n    MZ_MACRO_END\n\n/* Heap sort of lowercased filenames, used to help accelerate plain central directory searches by mz_zip_reader_locate_file(). (Could also use qsort(), but it could allocate memory.) */\nstatic void mz_zip_reader_sort_central_dir_offsets_by_filename(mz_zip_archive *pZip)\n{\n    mz_zip_internal_state *pState = pZip->m_pState;\n    const mz_zip_array *pCentral_dir_offsets = &pState->m_central_dir_offsets;\n    const mz_zip_array *pCentral_dir = &pState->m_central_dir;\n    mz_uint32 *pIndices;\n    mz_uint32 start, end;\n    const mz_uint32 size = pZip->m_total_files;\n\n    if (size <= 1U)\n        return;\n\n    pIndices = &MZ_ZIP_ARRAY_ELEMENT(&pState->m_sorted_central_dir_offsets, mz_uint32, 0);\n\n    start = (size - 2U) >> 1U;\n    for (;;)\n    {\n        mz_uint64 child, root = start;\n        for (;;)\n        {\n            if ((child = (root << 1U) + 1U) >= size)\n                break;\n            child += (((child + 1U) < size) && (mz_zip_reader_filename_less(pCentral_dir, pCentral_dir_offsets, pIndices[child], pIndices[child + 1U])));\n            if (!mz_zip_reader_filename_less(pCentral_dir, pCentral_dir_offsets, pIndices[root], pIndices[child]))\n                break;\n            MZ_SWAP_UINT32(pIndices[root], pIndices[child]);\n            root = child;\n        }\n        if (!start)\n            break;\n        start--;\n    }\n\n    end = size - 1;\n    while (end > 0)\n    {\n        mz_uint64 child, root = 0;\n        MZ_SWAP_UINT32(pIndices[end], pIndices[0]);\n        for (;;)\n        {\n            if ((child = (root << 1U) + 1U) >= end)\n                break;\n            child += (((child + 1U) < end) && mz_zip_reader_filename_less(pCentral_dir, pCentral_dir_offsets, pIndices[child], pIndices[child + 1U]));\n            if (!mz_zip_reader_filename_less(pCentral_dir, pCentral_dir_offsets, pIndices[root], pIndices[child]))\n                break;\n            MZ_SWAP_UINT32(pIndices[root], pIndices[child]);\n            root = child;\n        }\n        end--;\n    }\n}\n\nstatic mz_bool mz_zip_reader_locate_header_sig(mz_zip_archive *pZip, mz_uint32 record_sig, mz_uint32 record_size, mz_int64 *pOfs)\n{\n    mz_int64 cur_file_ofs;\n    mz_uint32 buf_u32[4096 / sizeof(mz_uint32)];\n    mz_uint8 *pBuf = (mz_uint8 *)buf_u32;\n\n    /* Basic sanity checks - reject files which are too small */\n    if (pZip->m_archive_size < record_size)\n        return MZ_FALSE;\n\n    /* Find the record by scanning the file from the end towards the beginning. */\n    cur_file_ofs = MZ_MAX((mz_int64)pZip->m_archive_size - (mz_int64)sizeof(buf_u32), 0);\n    for (;;)\n    {\n        int i, n = (int)MZ_MIN(sizeof(buf_u32), pZip->m_archive_size - cur_file_ofs);\n\n        if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pBuf, n) != (mz_uint)n)\n            return MZ_FALSE;\n\n        for (i = n - 4; i >= 0; --i)\n        {\n            mz_uint s = MZ_READ_LE32(pBuf + i);\n            if (s == record_sig)\n            {\n                if ((pZip->m_archive_size - (cur_file_ofs + i)) >= record_size)\n                    break;\n            }\n        }\n\n        if (i >= 0)\n        {\n            cur_file_ofs += i;\n            break;\n        }\n\n        /* Give up if we've searched the entire file, or we've gone back \"too far\" (~64kb) */\n        if ((!cur_file_ofs) || ((pZip->m_archive_size - cur_file_ofs) >= (MZ_UINT16_MAX + record_size)))\n            return MZ_FALSE;\n\n        cur_file_ofs = MZ_MAX(cur_file_ofs - (sizeof(buf_u32) - 3), 0);\n    }\n\n    *pOfs = cur_file_ofs;\n    return MZ_TRUE;\n}\n\nstatic mz_bool mz_zip_reader_read_central_dir(mz_zip_archive *pZip, mz_uint flags)\n{\n    mz_uint cdir_size = 0, cdir_entries_on_this_disk = 0, num_this_disk = 0, cdir_disk_index = 0;\n    mz_uint64 cdir_ofs = 0;\n    mz_int64 cur_file_ofs = 0;\n    const mz_uint8 *p;\n\n    mz_uint32 buf_u32[4096 / sizeof(mz_uint32)];\n    mz_uint8 *pBuf = (mz_uint8 *)buf_u32;\n    mz_bool sort_central_dir = ((flags & MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY) == 0);\n    mz_uint32 zip64_end_of_central_dir_locator_u32[(MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIZE + sizeof(mz_uint32) - 1) / sizeof(mz_uint32)];\n    mz_uint8 *pZip64_locator = (mz_uint8 *)zip64_end_of_central_dir_locator_u32;\n\n    mz_uint32 zip64_end_of_central_dir_header_u32[(MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE + sizeof(mz_uint32) - 1) / sizeof(mz_uint32)];\n    mz_uint8 *pZip64_end_of_central_dir = (mz_uint8 *)zip64_end_of_central_dir_header_u32;\n\n    mz_uint64 zip64_end_of_central_dir_ofs = 0;\n\n    /* Basic sanity checks - reject files which are too small, and check the first 4 bytes of the file to make sure a local header is there. */\n    if (pZip->m_archive_size < MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE)\n        return mz_zip_set_error(pZip, MZ_ZIP_NOT_AN_ARCHIVE);\n\n    if (!mz_zip_reader_locate_header_sig(pZip, MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIG, MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE, &cur_file_ofs))\n        return mz_zip_set_error(pZip, MZ_ZIP_FAILED_FINDING_CENTRAL_DIR);\n\n    /* Read and verify the end of central directory record. */\n    if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pBuf, MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE) != MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE)\n        return mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n\n    if (MZ_READ_LE32(pBuf + MZ_ZIP_ECDH_SIG_OFS) != MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIG)\n        return mz_zip_set_error(pZip, MZ_ZIP_NOT_AN_ARCHIVE);\n\n    if (cur_file_ofs >= (MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIZE + MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE))\n    {\n        if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs - MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIZE, pZip64_locator, MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIZE) == MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIZE)\n        {\n            if (MZ_READ_LE32(pZip64_locator + MZ_ZIP64_ECDL_SIG_OFS) == MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIG)\n            {\n                zip64_end_of_central_dir_ofs = MZ_READ_LE64(pZip64_locator + MZ_ZIP64_ECDL_REL_OFS_TO_ZIP64_ECDR_OFS);\n                if (zip64_end_of_central_dir_ofs > (pZip->m_archive_size - MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE))\n                    return mz_zip_set_error(pZip, MZ_ZIP_NOT_AN_ARCHIVE);\n\n                if (pZip->m_pRead(pZip->m_pIO_opaque, zip64_end_of_central_dir_ofs, pZip64_end_of_central_dir, MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE) == MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE)\n                {\n                    if (MZ_READ_LE32(pZip64_end_of_central_dir + MZ_ZIP64_ECDH_SIG_OFS) == MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIG)\n                    {\n                        pZip->m_pState->m_zip64 = MZ_TRUE;\n                    }\n                }\n            }\n        }\n    }\n\n    pZip->m_total_files = MZ_READ_LE16(pBuf + MZ_ZIP_ECDH_CDIR_TOTAL_ENTRIES_OFS);\n    cdir_entries_on_this_disk = MZ_READ_LE16(pBuf + MZ_ZIP_ECDH_CDIR_NUM_ENTRIES_ON_DISK_OFS);\n    num_this_disk = MZ_READ_LE16(pBuf + MZ_ZIP_ECDH_NUM_THIS_DISK_OFS);\n    cdir_disk_index = MZ_READ_LE16(pBuf + MZ_ZIP_ECDH_NUM_DISK_CDIR_OFS);\n    cdir_size = MZ_READ_LE32(pBuf + MZ_ZIP_ECDH_CDIR_SIZE_OFS);\n    cdir_ofs = MZ_READ_LE32(pBuf + MZ_ZIP_ECDH_CDIR_OFS_OFS);\n\n    if (pZip->m_pState->m_zip64)\n    {\n        mz_uint32 zip64_total_num_of_disks = MZ_READ_LE32(pZip64_locator + MZ_ZIP64_ECDL_TOTAL_NUMBER_OF_DISKS_OFS);\n        mz_uint64 zip64_cdir_total_entries = MZ_READ_LE64(pZip64_end_of_central_dir + MZ_ZIP64_ECDH_CDIR_TOTAL_ENTRIES_OFS);\n        mz_uint64 zip64_cdir_total_entries_on_this_disk = MZ_READ_LE64(pZip64_end_of_central_dir + MZ_ZIP64_ECDH_CDIR_NUM_ENTRIES_ON_DISK_OFS);\n        mz_uint64 zip64_size_of_end_of_central_dir_record = MZ_READ_LE64(pZip64_end_of_central_dir + MZ_ZIP64_ECDH_SIZE_OF_RECORD_OFS);\n        mz_uint64 zip64_size_of_central_directory = MZ_READ_LE64(pZip64_end_of_central_dir + MZ_ZIP64_ECDH_CDIR_SIZE_OFS);\n\n        if (zip64_size_of_end_of_central_dir_record < (MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE - 12))\n            return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n        if (zip64_total_num_of_disks != 1U)\n            return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_MULTIDISK);\n\n        /* Check for miniz's practical limits */\n        if (zip64_cdir_total_entries > MZ_UINT32_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_TOO_MANY_FILES);\n\n        pZip->m_total_files = (mz_uint32)zip64_cdir_total_entries;\n\n        if (zip64_cdir_total_entries_on_this_disk > MZ_UINT32_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_TOO_MANY_FILES);\n\n        cdir_entries_on_this_disk = (mz_uint32)zip64_cdir_total_entries_on_this_disk;\n\n        /* Check for miniz's current practical limits (sorry, this should be enough for millions of files) */\n        if (zip64_size_of_central_directory > MZ_UINT32_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_CDIR_SIZE);\n\n        cdir_size = (mz_uint32)zip64_size_of_central_directory;\n\n        num_this_disk = MZ_READ_LE32(pZip64_end_of_central_dir + MZ_ZIP64_ECDH_NUM_THIS_DISK_OFS);\n\n        cdir_disk_index = MZ_READ_LE32(pZip64_end_of_central_dir + MZ_ZIP64_ECDH_NUM_DISK_CDIR_OFS);\n\n        cdir_ofs = MZ_READ_LE64(pZip64_end_of_central_dir + MZ_ZIP64_ECDH_CDIR_OFS_OFS);\n    }\n\n    if (pZip->m_total_files != cdir_entries_on_this_disk)\n        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_MULTIDISK);\n\n    if (((num_this_disk | cdir_disk_index) != 0) && ((num_this_disk != 1) || (cdir_disk_index != 1)))\n        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_MULTIDISK);\n\n    if (cdir_size < pZip->m_total_files * MZ_ZIP_CENTRAL_DIR_HEADER_SIZE)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n    if ((cdir_ofs + (mz_uint64)cdir_size) > pZip->m_archive_size)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n    pZip->m_central_directory_file_ofs = cdir_ofs;\n\n    if (pZip->m_total_files)\n    {\n        mz_uint i, n;\n        /* Read the entire central directory into a heap block, and allocate another heap block to hold the unsorted central dir file record offsets, and possibly another to hold the sorted indices. */\n        if ((!mz_zip_array_resize(pZip, &pZip->m_pState->m_central_dir, cdir_size, MZ_FALSE)) ||\n            (!mz_zip_array_resize(pZip, &pZip->m_pState->m_central_dir_offsets, pZip->m_total_files, MZ_FALSE)))\n            return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n\n        if (sort_central_dir)\n        {\n            if (!mz_zip_array_resize(pZip, &pZip->m_pState->m_sorted_central_dir_offsets, pZip->m_total_files, MZ_FALSE))\n                return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n        }\n\n        if (pZip->m_pRead(pZip->m_pIO_opaque, cdir_ofs, pZip->m_pState->m_central_dir.m_p, cdir_size) != cdir_size)\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n\n        /* Now create an index into the central directory file records, do some basic sanity checking on each record */\n        p = (const mz_uint8 *)pZip->m_pState->m_central_dir.m_p;\n        for (n = cdir_size, i = 0; i < pZip->m_total_files; ++i)\n        {\n            mz_uint total_header_size, disk_index, bit_flags, filename_size, ext_data_size;\n            mz_uint64 comp_size, decomp_size, local_header_ofs;\n\n            if ((n < MZ_ZIP_CENTRAL_DIR_HEADER_SIZE) || (MZ_READ_LE32(p) != MZ_ZIP_CENTRAL_DIR_HEADER_SIG))\n                return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n            MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir_offsets, mz_uint32, i) = (mz_uint32)(p - (const mz_uint8 *)pZip->m_pState->m_central_dir.m_p);\n\n            if (sort_central_dir)\n                MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_sorted_central_dir_offsets, mz_uint32, i) = i;\n\n            comp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_COMPRESSED_SIZE_OFS);\n            decomp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS);\n            local_header_ofs = MZ_READ_LE32(p + MZ_ZIP_CDH_LOCAL_HEADER_OFS);\n            filename_size = MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS);\n            ext_data_size = MZ_READ_LE16(p + MZ_ZIP_CDH_EXTRA_LEN_OFS);\n\n            if ((!pZip->m_pState->m_zip64_has_extended_info_fields) &&\n                (ext_data_size) &&\n                (MZ_MAX(MZ_MAX(comp_size, decomp_size), local_header_ofs) == MZ_UINT32_MAX))\n            {\n                /* Attempt to find zip64 extended information field in the entry's extra data */\n                mz_uint32 extra_size_remaining = ext_data_size;\n\n                if (extra_size_remaining)\n                {\n\t\t\t\t\tconst mz_uint8 *pExtra_data;\n\t\t\t\t\tvoid* buf = NULL;\n\n\t\t\t\t\tif (MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + filename_size + ext_data_size > n)\n\t\t\t\t\t{\n\t\t\t\t\t\tbuf = MZ_MALLOC(ext_data_size);\n\t\t\t\t\t\tif(buf==NULL)\n\t\t\t\t\t\t\treturn mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n\n\t\t\t\t\t\tif (pZip->m_pRead(pZip->m_pIO_opaque, cdir_ofs + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + filename_size, buf, ext_data_size) != ext_data_size)\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tMZ_FREE(buf);\n\t\t\t\t\t\t\treturn mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tpExtra_data = (mz_uint8*)buf;\n\t\t\t\t\t}\n\t\t\t\t\telse\n\t\t\t\t\t{\n\t\t\t\t\t\tpExtra_data = p + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + filename_size;\n\t\t\t\t\t}\n\n                    do\n                    {\n                        mz_uint32 field_id;\n                        mz_uint32 field_data_size;\n\n\t\t\t\t\t\tif (extra_size_remaining < (sizeof(mz_uint16) * 2))\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tMZ_FREE(buf);\n\t\t\t\t\t\t\treturn mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\t\t\t\t\t\t}\n\n                        field_id = MZ_READ_LE16(pExtra_data);\n                        field_data_size = MZ_READ_LE16(pExtra_data + sizeof(mz_uint16));\n\n\t\t\t\t\t\tif ((field_data_size + sizeof(mz_uint16) * 2) > extra_size_remaining)\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tMZ_FREE(buf);\n\t\t\t\t\t\t\treturn mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\t\t\t\t\t\t}\n\n                        if (field_id == MZ_ZIP64_EXTENDED_INFORMATION_FIELD_HEADER_ID)\n                        {\n                            /* Ok, the archive didn't have any zip64 headers but it uses a zip64 extended information field so mark it as zip64 anyway (this can occur with infozip's zip util when it reads compresses files from stdin). */\n                            pZip->m_pState->m_zip64 = MZ_TRUE;\n                            pZip->m_pState->m_zip64_has_extended_info_fields = MZ_TRUE;\n                            break;\n                        }\n\n                        pExtra_data += sizeof(mz_uint16) * 2 + field_data_size;\n                        extra_size_remaining = extra_size_remaining - sizeof(mz_uint16) * 2 - field_data_size;\n                    } while (extra_size_remaining);\n\n\t\t\t\t\tMZ_FREE(buf);\n                }\n            }\n\n            /* I've seen archives that aren't marked as zip64 that uses zip64 ext data, argh */\n            if ((comp_size != MZ_UINT32_MAX) && (decomp_size != MZ_UINT32_MAX))\n            {\n                if (((!MZ_READ_LE32(p + MZ_ZIP_CDH_METHOD_OFS)) && (decomp_size != comp_size)) || (decomp_size && !comp_size))\n                    return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n            }\n\n            disk_index = MZ_READ_LE16(p + MZ_ZIP_CDH_DISK_START_OFS);\n            if ((disk_index == MZ_UINT16_MAX) || ((disk_index != num_this_disk) && (disk_index != 1)))\n                return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_MULTIDISK);\n\n            if (comp_size != MZ_UINT32_MAX)\n            {\n                if (((mz_uint64)MZ_READ_LE32(p + MZ_ZIP_CDH_LOCAL_HEADER_OFS) + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + comp_size) > pZip->m_archive_size)\n                    return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n            }\n\n            bit_flags = MZ_READ_LE16(p + MZ_ZIP_CDH_BIT_FLAG_OFS);\n            if (bit_flags & MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_LOCAL_DIR_IS_MASKED)\n                return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_ENCRYPTION);\n\n            if ((total_header_size = MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS) + MZ_READ_LE16(p + MZ_ZIP_CDH_EXTRA_LEN_OFS) + MZ_READ_LE16(p + MZ_ZIP_CDH_COMMENT_LEN_OFS)) > n)\n                return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n            n -= total_header_size;\n            p += total_header_size;\n        }\n    }\n\n    if (sort_central_dir)\n        mz_zip_reader_sort_central_dir_offsets_by_filename(pZip);\n\n    return MZ_TRUE;\n}\n\nvoid mz_zip_zero_struct(mz_zip_archive *pZip)\n{\n    if (pZip)\n        MZ_CLEAR_OBJ(*pZip);\n}\n\nstatic mz_bool mz_zip_reader_end_internal(mz_zip_archive *pZip, mz_bool set_last_error)\n{\n    mz_bool status = MZ_TRUE;\n\n    if (!pZip)\n        return MZ_FALSE;\n\n    if ((!pZip->m_pState) || (!pZip->m_pAlloc) || (!pZip->m_pFree) || (pZip->m_zip_mode != MZ_ZIP_MODE_READING))\n    {\n        if (set_last_error)\n            pZip->m_last_error = MZ_ZIP_INVALID_PARAMETER;\n\n        return MZ_FALSE;\n    }\n\n    if (pZip->m_pState)\n    {\n        mz_zip_internal_state *pState = pZip->m_pState;\n        pZip->m_pState = NULL;\n\n        mz_zip_array_clear(pZip, &pState->m_central_dir);\n        mz_zip_array_clear(pZip, &pState->m_central_dir_offsets);\n        mz_zip_array_clear(pZip, &pState->m_sorted_central_dir_offsets);\n\n#ifndef MINIZ_NO_STDIO\n        if (pState->m_pFile)\n        {\n            if (pZip->m_zip_type == MZ_ZIP_TYPE_FILE)\n            {\n                if (MZ_FCLOSE(pState->m_pFile) == EOF)\n                {\n                    if (set_last_error)\n                        pZip->m_last_error = MZ_ZIP_FILE_CLOSE_FAILED;\n                    status = MZ_FALSE;\n                }\n            }\n            pState->m_pFile = NULL;\n        }\n#endif /* #ifndef MINIZ_NO_STDIO */\n\n        pZip->m_pFree(pZip->m_pAlloc_opaque, pState);\n    }\n    pZip->m_zip_mode = MZ_ZIP_MODE_INVALID;\n\n    return status;\n}\n\nmz_bool mz_zip_reader_end(mz_zip_archive *pZip)\n{\n    return mz_zip_reader_end_internal(pZip, MZ_TRUE);\n}\nmz_bool mz_zip_reader_init(mz_zip_archive *pZip, mz_uint64 size, mz_uint flags)\n{\n    if ((!pZip) || (!pZip->m_pRead))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    if (!mz_zip_reader_init_internal(pZip, flags))\n        return MZ_FALSE;\n\n    pZip->m_zip_type = MZ_ZIP_TYPE_USER;\n    pZip->m_archive_size = size;\n\n    if (!mz_zip_reader_read_central_dir(pZip, flags))\n    {\n        mz_zip_reader_end_internal(pZip, MZ_FALSE);\n        return MZ_FALSE;\n    }\n\n    return MZ_TRUE;\n}\n\nstatic size_t mz_zip_mem_read_func(void *pOpaque, mz_uint64 file_ofs, void *pBuf, size_t n)\n{\n    mz_zip_archive *pZip = (mz_zip_archive *)pOpaque;\n    size_t s = (file_ofs >= pZip->m_archive_size) ? 0 : (size_t)MZ_MIN(pZip->m_archive_size - file_ofs, n);\n    memcpy(pBuf, (const mz_uint8 *)pZip->m_pState->m_pMem + file_ofs, s);\n    return s;\n}\n\nmz_bool mz_zip_reader_init_mem(mz_zip_archive *pZip, const void *pMem, size_t size, mz_uint flags)\n{\n    if (!pMem)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    if (size < MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE)\n        return mz_zip_set_error(pZip, MZ_ZIP_NOT_AN_ARCHIVE);\n\n    if (!mz_zip_reader_init_internal(pZip, flags))\n        return MZ_FALSE;\n\n    pZip->m_zip_type = MZ_ZIP_TYPE_MEMORY;\n    pZip->m_archive_size = size;\n    pZip->m_pRead = mz_zip_mem_read_func;\n    pZip->m_pIO_opaque = pZip;\n    pZip->m_pNeeds_keepalive = NULL;\n\n#ifdef __cplusplus\n    pZip->m_pState->m_pMem = const_cast<void *>(pMem);\n#else\n    pZip->m_pState->m_pMem = (void *)pMem;\n#endif\n\n    pZip->m_pState->m_mem_size = size;\n\n    if (!mz_zip_reader_read_central_dir(pZip, flags))\n    {\n        mz_zip_reader_end_internal(pZip, MZ_FALSE);\n        return MZ_FALSE;\n    }\n\n    return MZ_TRUE;\n}\n\n#ifndef MINIZ_NO_STDIO\nstatic size_t mz_zip_file_read_func(void *pOpaque, mz_uint64 file_ofs, void *pBuf, size_t n)\n{\n    mz_zip_archive *pZip = (mz_zip_archive *)pOpaque;\n    mz_int64 cur_ofs = MZ_FTELL64(pZip->m_pState->m_pFile);\n\n    file_ofs += pZip->m_pState->m_file_archive_start_ofs;\n\n    if (((mz_int64)file_ofs < 0) || (((cur_ofs != (mz_int64)file_ofs)) && (MZ_FSEEK64(pZip->m_pState->m_pFile, (mz_int64)file_ofs, SEEK_SET))))\n        return 0;\n\n    return MZ_FREAD(pBuf, 1, n, pZip->m_pState->m_pFile);\n}\n\nmz_bool mz_zip_reader_init_file(mz_zip_archive *pZip, const char *pFilename, mz_uint32 flags)\n{\n    return mz_zip_reader_init_file_v2(pZip, pFilename, flags, 0, 0);\n}\n\nmz_bool mz_zip_reader_init_file_v2(mz_zip_archive *pZip, const char *pFilename, mz_uint flags, mz_uint64 file_start_ofs, mz_uint64 archive_size)\n{\n    mz_uint64 file_size;\n    MZ_FILE *pFile;\n\n    if ((!pZip) || (!pFilename) || ((archive_size) && (archive_size < MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE)))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    pFile = MZ_FOPEN(pFilename, \"rb\");\n    if (!pFile)\n        return mz_zip_set_error(pZip, MZ_ZIP_FILE_OPEN_FAILED);\n\n    file_size = archive_size;\n    if (!file_size)\n    {\n        if (MZ_FSEEK64(pFile, 0, SEEK_END))\n        {\n            MZ_FCLOSE(pFile);\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_SEEK_FAILED);\n        }\n\n        file_size = MZ_FTELL64(pFile);\n    }\n\n    /* TODO: Better sanity check archive_size and the # of actual remaining bytes */\n\n    if (file_size < MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE)\n    {\n\tMZ_FCLOSE(pFile);\n        return mz_zip_set_error(pZip, MZ_ZIP_NOT_AN_ARCHIVE);\n    }\n\n    if (!mz_zip_reader_init_internal(pZip, flags))\n    {\n        MZ_FCLOSE(pFile);\n        return MZ_FALSE;\n    }\n\n    pZip->m_zip_type = MZ_ZIP_TYPE_FILE;\n    pZip->m_pRead = mz_zip_file_read_func;\n    pZip->m_pIO_opaque = pZip;\n    pZip->m_pState->m_pFile = pFile;\n    pZip->m_archive_size = file_size;\n    pZip->m_pState->m_file_archive_start_ofs = file_start_ofs;\n\n    if (!mz_zip_reader_read_central_dir(pZip, flags))\n    {\n        mz_zip_reader_end_internal(pZip, MZ_FALSE);\n        return MZ_FALSE;\n    }\n\n    return MZ_TRUE;\n}\n\nmz_bool mz_zip_reader_init_cfile(mz_zip_archive *pZip, MZ_FILE *pFile, mz_uint64 archive_size, mz_uint flags)\n{\n    mz_uint64 cur_file_ofs;\n\n    if ((!pZip) || (!pFile))\n        return mz_zip_set_error(pZip, MZ_ZIP_FILE_OPEN_FAILED);\n\n    cur_file_ofs = MZ_FTELL64(pFile);\n\n    if (!archive_size)\n    {\n        if (MZ_FSEEK64(pFile, 0, SEEK_END))\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_SEEK_FAILED);\n\n        archive_size = MZ_FTELL64(pFile) - cur_file_ofs;\n\n        if (archive_size < MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE)\n            return mz_zip_set_error(pZip, MZ_ZIP_NOT_AN_ARCHIVE);\n    }\n\n    if (!mz_zip_reader_init_internal(pZip, flags))\n        return MZ_FALSE;\n\n    pZip->m_zip_type = MZ_ZIP_TYPE_CFILE;\n    pZip->m_pRead = mz_zip_file_read_func;\n\n    pZip->m_pIO_opaque = pZip;\n    pZip->m_pState->m_pFile = pFile;\n    pZip->m_archive_size = archive_size;\n    pZip->m_pState->m_file_archive_start_ofs = cur_file_ofs;\n\n    if (!mz_zip_reader_read_central_dir(pZip, flags))\n    {\n        mz_zip_reader_end_internal(pZip, MZ_FALSE);\n        return MZ_FALSE;\n    }\n\n    return MZ_TRUE;\n}\n\n#endif /* #ifndef MINIZ_NO_STDIO */\n\nstatic MZ_FORCEINLINE const mz_uint8 *mz_zip_get_cdh(mz_zip_archive *pZip, mz_uint file_index)\n{\n    if ((!pZip) || (!pZip->m_pState) || (file_index >= pZip->m_total_files))\n        return NULL;\n    return &MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir, mz_uint8, MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir_offsets, mz_uint32, file_index));\n}\n\nmz_bool mz_zip_reader_is_file_encrypted(mz_zip_archive *pZip, mz_uint file_index)\n{\n    mz_uint m_bit_flag;\n    const mz_uint8 *p = mz_zip_get_cdh(pZip, file_index);\n    if (!p)\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n        return MZ_FALSE;\n    }\n\n    m_bit_flag = MZ_READ_LE16(p + MZ_ZIP_CDH_BIT_FLAG_OFS);\n    return (m_bit_flag & (MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_IS_ENCRYPTED | MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_USES_STRONG_ENCRYPTION)) != 0;\n}\n\nmz_bool mz_zip_reader_is_file_supported(mz_zip_archive *pZip, mz_uint file_index)\n{\n    mz_uint bit_flag;\n    mz_uint method;\n\n    const mz_uint8 *p = mz_zip_get_cdh(pZip, file_index);\n    if (!p)\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n        return MZ_FALSE;\n    }\n\n    method = MZ_READ_LE16(p + MZ_ZIP_CDH_METHOD_OFS);\n    bit_flag = MZ_READ_LE16(p + MZ_ZIP_CDH_BIT_FLAG_OFS);\n\n    if ((method != 0) && (method != MZ_DEFLATED))\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_METHOD);\n        return MZ_FALSE;\n    }\n\n    if (bit_flag & (MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_IS_ENCRYPTED | MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_USES_STRONG_ENCRYPTION))\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_ENCRYPTION);\n        return MZ_FALSE;\n    }\n\n    if (bit_flag & MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_COMPRESSED_PATCH_FLAG)\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_FEATURE);\n        return MZ_FALSE;\n    }\n\n    return MZ_TRUE;\n}\n\nmz_bool mz_zip_reader_is_file_a_directory(mz_zip_archive *pZip, mz_uint file_index)\n{\n    mz_uint filename_len, attribute_mapping_id, external_attr;\n    const mz_uint8 *p = mz_zip_get_cdh(pZip, file_index);\n    if (!p)\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n        return MZ_FALSE;\n    }\n\n    filename_len = MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS);\n    if (filename_len)\n    {\n        if (*(p + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + filename_len - 1) == '/')\n            return MZ_TRUE;\n    }\n\n    /* Bugfix: This code was also checking if the internal attribute was non-zero, which wasn't correct. */\n    /* Most/all zip writers (hopefully) set DOS file/directory attributes in the low 16-bits, so check for the DOS directory flag and ignore the source OS ID in the created by field. */\n    /* FIXME: Remove this check? Is it necessary - we already check the filename. */\n    attribute_mapping_id = MZ_READ_LE16(p + MZ_ZIP_CDH_VERSION_MADE_BY_OFS) >> 8;\n    (void)attribute_mapping_id;\n\n    external_attr = MZ_READ_LE32(p + MZ_ZIP_CDH_EXTERNAL_ATTR_OFS);\n    if ((external_attr & MZ_ZIP_DOS_DIR_ATTRIBUTE_BITFLAG) != 0)\n    {\n        return MZ_TRUE;\n    }\n\n    return MZ_FALSE;\n}\n\nstatic mz_bool mz_zip_file_stat_internal(mz_zip_archive *pZip, mz_uint file_index, const mz_uint8 *pCentral_dir_header, mz_zip_archive_file_stat *pStat, mz_bool *pFound_zip64_extra_data)\n{\n    mz_uint n;\n    const mz_uint8 *p = pCentral_dir_header;\n\n    if (pFound_zip64_extra_data)\n        *pFound_zip64_extra_data = MZ_FALSE;\n\n    if ((!p) || (!pStat))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    /* Extract fields from the central directory record. */\n    pStat->m_file_index = file_index;\n    pStat->m_central_dir_ofs = MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir_offsets, mz_uint32, file_index);\n    pStat->m_version_made_by = MZ_READ_LE16(p + MZ_ZIP_CDH_VERSION_MADE_BY_OFS);\n    pStat->m_version_needed = MZ_READ_LE16(p + MZ_ZIP_CDH_VERSION_NEEDED_OFS);\n    pStat->m_bit_flag = MZ_READ_LE16(p + MZ_ZIP_CDH_BIT_FLAG_OFS);\n    pStat->m_method = MZ_READ_LE16(p + MZ_ZIP_CDH_METHOD_OFS);\n#ifndef MINIZ_NO_TIME\n    pStat->m_time = mz_zip_dos_to_time_t(MZ_READ_LE16(p + MZ_ZIP_CDH_FILE_TIME_OFS), MZ_READ_LE16(p + MZ_ZIP_CDH_FILE_DATE_OFS));\n#endif\n    pStat->m_crc32 = MZ_READ_LE32(p + MZ_ZIP_CDH_CRC32_OFS);\n    pStat->m_comp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_COMPRESSED_SIZE_OFS);\n    pStat->m_uncomp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS);\n    pStat->m_internal_attr = MZ_READ_LE16(p + MZ_ZIP_CDH_INTERNAL_ATTR_OFS);\n    pStat->m_external_attr = MZ_READ_LE32(p + MZ_ZIP_CDH_EXTERNAL_ATTR_OFS);\n    pStat->m_local_header_ofs = MZ_READ_LE32(p + MZ_ZIP_CDH_LOCAL_HEADER_OFS);\n\n    /* Copy as much of the filename and comment as possible. */\n    n = MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS);\n    n = MZ_MIN(n, MZ_ZIP_MAX_ARCHIVE_FILENAME_SIZE - 1);\n    memcpy(pStat->m_filename, p + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE, n);\n    pStat->m_filename[n] = '\\0';\n\n    n = MZ_READ_LE16(p + MZ_ZIP_CDH_COMMENT_LEN_OFS);\n    n = MZ_MIN(n, MZ_ZIP_MAX_ARCHIVE_FILE_COMMENT_SIZE - 1);\n    pStat->m_comment_size = n;\n    memcpy(pStat->m_comment, p + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS) + MZ_READ_LE16(p + MZ_ZIP_CDH_EXTRA_LEN_OFS), n);\n    pStat->m_comment[n] = '\\0';\n\n    /* Set some flags for convienance */\n    pStat->m_is_directory = mz_zip_reader_is_file_a_directory(pZip, file_index);\n    pStat->m_is_encrypted = mz_zip_reader_is_file_encrypted(pZip, file_index);\n    pStat->m_is_supported = mz_zip_reader_is_file_supported(pZip, file_index);\n\n    /* See if we need to read any zip64 extended information fields. */\n    /* Confusingly, these zip64 fields can be present even on non-zip64 archives (Debian zip on a huge files from stdin piped to stdout creates them). */\n    if (MZ_MAX(MZ_MAX(pStat->m_comp_size, pStat->m_uncomp_size), pStat->m_local_header_ofs) == MZ_UINT32_MAX)\n    {\n        /* Attempt to find zip64 extended information field in the entry's extra data */\n        mz_uint32 extra_size_remaining = MZ_READ_LE16(p + MZ_ZIP_CDH_EXTRA_LEN_OFS);\n\n        if (extra_size_remaining)\n        {\n            const mz_uint8 *pExtra_data = p + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS);\n\n            do\n            {\n                mz_uint32 field_id;\n                mz_uint32 field_data_size;\n\n                if (extra_size_remaining < (sizeof(mz_uint16) * 2))\n                    return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n                field_id = MZ_READ_LE16(pExtra_data);\n                field_data_size = MZ_READ_LE16(pExtra_data + sizeof(mz_uint16));\n\n                if ((field_data_size + sizeof(mz_uint16) * 2) > extra_size_remaining)\n                    return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n                if (field_id == MZ_ZIP64_EXTENDED_INFORMATION_FIELD_HEADER_ID)\n                {\n                    const mz_uint8 *pField_data = pExtra_data + sizeof(mz_uint16) * 2;\n                    mz_uint32 field_data_remaining = field_data_size;\n\n                    if (pFound_zip64_extra_data)\n                        *pFound_zip64_extra_data = MZ_TRUE;\n\n                    if (pStat->m_uncomp_size == MZ_UINT32_MAX)\n                    {\n                        if (field_data_remaining < sizeof(mz_uint64))\n                            return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n                        pStat->m_uncomp_size = MZ_READ_LE64(pField_data);\n                        pField_data += sizeof(mz_uint64);\n                        field_data_remaining -= sizeof(mz_uint64);\n                    }\n\n                    if (pStat->m_comp_size == MZ_UINT32_MAX)\n                    {\n                        if (field_data_remaining < sizeof(mz_uint64))\n                            return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n                        pStat->m_comp_size = MZ_READ_LE64(pField_data);\n                        pField_data += sizeof(mz_uint64);\n                        field_data_remaining -= sizeof(mz_uint64);\n                    }\n\n                    if (pStat->m_local_header_ofs == MZ_UINT32_MAX)\n                    {\n                        if (field_data_remaining < sizeof(mz_uint64))\n                            return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n                        pStat->m_local_header_ofs = MZ_READ_LE64(pField_data);\n                        pField_data += sizeof(mz_uint64);\n                        field_data_remaining -= sizeof(mz_uint64);\n                    }\n\n                    break;\n                }\n\n                pExtra_data += sizeof(mz_uint16) * 2 + field_data_size;\n                extra_size_remaining = extra_size_remaining - sizeof(mz_uint16) * 2 - field_data_size;\n            } while (extra_size_remaining);\n        }\n    }\n\n    return MZ_TRUE;\n}\n\nstatic MZ_FORCEINLINE mz_bool mz_zip_string_equal(const char *pA, const char *pB, mz_uint len, mz_uint flags)\n{\n    mz_uint i;\n    if (flags & MZ_ZIP_FLAG_CASE_SENSITIVE)\n        return 0 == memcmp(pA, pB, len);\n    for (i = 0; i < len; ++i)\n        if (MZ_TOLOWER(pA[i]) != MZ_TOLOWER(pB[i]))\n            return MZ_FALSE;\n    return MZ_TRUE;\n}\n\nstatic MZ_FORCEINLINE int mz_zip_filename_compare(const mz_zip_array *pCentral_dir_array, const mz_zip_array *pCentral_dir_offsets, mz_uint l_index, const char *pR, mz_uint r_len)\n{\n    const mz_uint8 *pL = &MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_array, mz_uint8, MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_offsets, mz_uint32, l_index)), *pE;\n    mz_uint l_len = MZ_READ_LE16(pL + MZ_ZIP_CDH_FILENAME_LEN_OFS);\n    mz_uint8 l = 0, r = 0;\n    pL += MZ_ZIP_CENTRAL_DIR_HEADER_SIZE;\n    pE = pL + MZ_MIN(l_len, r_len);\n    while (pL < pE)\n    {\n        if ((l = MZ_TOLOWER(*pL)) != (r = MZ_TOLOWER(*pR)))\n            break;\n        pL++;\n        pR++;\n    }\n    return (pL == pE) ? (int)(l_len - r_len) : (l - r);\n}\n\nstatic mz_bool mz_zip_locate_file_binary_search(mz_zip_archive *pZip, const char *pFilename, mz_uint32 *pIndex)\n{\n    mz_zip_internal_state *pState = pZip->m_pState;\n    const mz_zip_array *pCentral_dir_offsets = &pState->m_central_dir_offsets;\n    const mz_zip_array *pCentral_dir = &pState->m_central_dir;\n    mz_uint32 *pIndices = &MZ_ZIP_ARRAY_ELEMENT(&pState->m_sorted_central_dir_offsets, mz_uint32, 0);\n    const uint32_t size = pZip->m_total_files;\n    const mz_uint filename_len = (mz_uint)strlen(pFilename);\n\n    if (pIndex)\n        *pIndex = 0;\n\n    if (size)\n    {\n        /* yes I could use uint32_t's, but then we would have to add some special case checks in the loop, argh, and */\n        /* honestly the major expense here on 32-bit CPU's will still be the filename compare */\n        mz_int64 l = 0, h = (mz_int64)size - 1;\n\n        while (l <= h)\n        {\n            mz_int64 m = l + ((h - l) >> 1);\n            uint32_t file_index = pIndices[(uint32_t)m];\n\n            int comp = mz_zip_filename_compare(pCentral_dir, pCentral_dir_offsets, file_index, pFilename, filename_len);\n            if (!comp)\n            {\n                if (pIndex)\n                    *pIndex = file_index;\n                return MZ_TRUE;\n            }\n            else if (comp < 0)\n                l = m + 1;\n            else\n                h = m - 1;\n        }\n    }\n\n    return mz_zip_set_error(pZip, MZ_ZIP_FILE_NOT_FOUND);\n}\n\nint mz_zip_reader_locate_file(mz_zip_archive *pZip, const char *pName, const char *pComment, mz_uint flags)\n{\n    mz_uint32 index;\n    if (!mz_zip_reader_locate_file_v2(pZip, pName, pComment, flags, &index))\n        return -1;\n    else\n        return (int)index;\n}\n\nmz_bool mz_zip_reader_locate_file_v2(mz_zip_archive *pZip, const char *pName, const char *pComment, mz_uint flags, mz_uint32 *pIndex)\n{\n    mz_uint file_index;\n    size_t name_len, comment_len;\n\n    if (pIndex)\n        *pIndex = 0;\n\n    if ((!pZip) || (!pZip->m_pState) || (!pName))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    /* See if we can use a binary search */\n    if (((pZip->m_pState->m_init_flags & MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY) == 0) &&\n        (pZip->m_zip_mode == MZ_ZIP_MODE_READING) &&\n        ((flags & (MZ_ZIP_FLAG_IGNORE_PATH | MZ_ZIP_FLAG_CASE_SENSITIVE)) == 0) && (!pComment) && (pZip->m_pState->m_sorted_central_dir_offsets.m_size))\n    {\n        return mz_zip_locate_file_binary_search(pZip, pName, pIndex);\n    }\n\n    /* Locate the entry by scanning the entire central directory */\n    name_len = strlen(pName);\n    if (name_len > MZ_UINT16_MAX)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    comment_len = pComment ? strlen(pComment) : 0;\n    if (comment_len > MZ_UINT16_MAX)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    for (file_index = 0; file_index < pZip->m_total_files; file_index++)\n    {\n        const mz_uint8 *pHeader = &MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir, mz_uint8, MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir_offsets, mz_uint32, file_index));\n        mz_uint filename_len = MZ_READ_LE16(pHeader + MZ_ZIP_CDH_FILENAME_LEN_OFS);\n        const char *pFilename = (const char *)pHeader + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE;\n        if (filename_len < name_len)\n            continue;\n        if (comment_len)\n        {\n            mz_uint file_extra_len = MZ_READ_LE16(pHeader + MZ_ZIP_CDH_EXTRA_LEN_OFS), file_comment_len = MZ_READ_LE16(pHeader + MZ_ZIP_CDH_COMMENT_LEN_OFS);\n            const char *pFile_comment = pFilename + filename_len + file_extra_len;\n            if ((file_comment_len != comment_len) || (!mz_zip_string_equal(pComment, pFile_comment, file_comment_len, flags)))\n                continue;\n        }\n        if ((flags & MZ_ZIP_FLAG_IGNORE_PATH) && (filename_len))\n        {\n            int ofs = filename_len - 1;\n            do\n            {\n                if ((pFilename[ofs] == '/') || (pFilename[ofs] == '\\\\') || (pFilename[ofs] == ':'))\n                    break;\n            } while (--ofs >= 0);\n            ofs++;\n            pFilename += ofs;\n            filename_len -= ofs;\n        }\n        if ((filename_len == name_len) && (mz_zip_string_equal(pName, pFilename, filename_len, flags)))\n        {\n            if (pIndex)\n                *pIndex = file_index;\n            return MZ_TRUE;\n        }\n    }\n\n    return mz_zip_set_error(pZip, MZ_ZIP_FILE_NOT_FOUND);\n}\n\nmz_bool mz_zip_reader_extract_to_mem_no_alloc(mz_zip_archive *pZip, mz_uint file_index, void *pBuf, size_t buf_size, mz_uint flags, void *pUser_read_buf, size_t user_read_buf_size)\n{\n    int status = TINFL_STATUS_DONE;\n    mz_uint64 needed_size, cur_file_ofs, comp_remaining, out_buf_ofs = 0, read_buf_size, read_buf_ofs = 0, read_buf_avail;\n    mz_zip_archive_file_stat file_stat;\n    void *pRead_buf;\n    mz_uint32 local_header_u32[(MZ_ZIP_LOCAL_DIR_HEADER_SIZE + sizeof(mz_uint32) - 1) / sizeof(mz_uint32)];\n    mz_uint8 *pLocal_header = (mz_uint8 *)local_header_u32;\n    tinfl_decompressor inflator;\n\n    if ((!pZip) || (!pZip->m_pState) || ((buf_size) && (!pBuf)) || ((user_read_buf_size) && (!pUser_read_buf)) || (!pZip->m_pRead))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    if (!mz_zip_reader_file_stat(pZip, file_index, &file_stat))\n        return MZ_FALSE;\n\n    /* A directory or zero length file */\n    if ((file_stat.m_is_directory) || (!file_stat.m_comp_size))\n        return MZ_TRUE;\n\n    /* Encryption and patch files are not supported. */\n    if (file_stat.m_bit_flag & (MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_IS_ENCRYPTED | MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_USES_STRONG_ENCRYPTION | MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_COMPRESSED_PATCH_FLAG))\n        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_ENCRYPTION);\n\n    /* This function only supports decompressing stored and deflate. */\n    if ((!(flags & MZ_ZIP_FLAG_COMPRESSED_DATA)) && (file_stat.m_method != 0) && (file_stat.m_method != MZ_DEFLATED))\n        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_METHOD);\n\n    /* Ensure supplied output buffer is large enough. */\n    needed_size = (flags & MZ_ZIP_FLAG_COMPRESSED_DATA) ? file_stat.m_comp_size : file_stat.m_uncomp_size;\n    if (buf_size < needed_size)\n        return mz_zip_set_error(pZip, MZ_ZIP_BUF_TOO_SMALL);\n\n    /* Read and parse the local directory entry. */\n    cur_file_ofs = file_stat.m_local_header_ofs;\n    if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pLocal_header, MZ_ZIP_LOCAL_DIR_HEADER_SIZE) != MZ_ZIP_LOCAL_DIR_HEADER_SIZE)\n        return mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n\n    if (MZ_READ_LE32(pLocal_header) != MZ_ZIP_LOCAL_DIR_HEADER_SIG)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n    cur_file_ofs += MZ_ZIP_LOCAL_DIR_HEADER_SIZE + MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_FILENAME_LEN_OFS) + MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_EXTRA_LEN_OFS);\n    if ((cur_file_ofs + file_stat.m_comp_size) > pZip->m_archive_size)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n    if ((flags & MZ_ZIP_FLAG_COMPRESSED_DATA) || (!file_stat.m_method))\n    {\n        /* The file is stored or the caller has requested the compressed data. */\n        if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pBuf, (size_t)needed_size) != needed_size)\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n\n#ifndef MINIZ_DISABLE_ZIP_READER_CRC32_CHECKS\n        if ((flags & MZ_ZIP_FLAG_COMPRESSED_DATA) == 0)\n        {\n            if (mz_crc32(MZ_CRC32_INIT, (const mz_uint8 *)pBuf, (size_t)file_stat.m_uncomp_size) != file_stat.m_crc32)\n                return mz_zip_set_error(pZip, MZ_ZIP_CRC_CHECK_FAILED);\n        }\n#endif\n\n        return MZ_TRUE;\n    }\n\n    /* Decompress the file either directly from memory or from a file input buffer. */\n    tinfl_init(&inflator);\n\n    if (pZip->m_pState->m_pMem)\n    {\n        /* Read directly from the archive in memory. */\n        pRead_buf = (mz_uint8 *)pZip->m_pState->m_pMem + cur_file_ofs;\n        read_buf_size = read_buf_avail = file_stat.m_comp_size;\n        comp_remaining = 0;\n    }\n    else if (pUser_read_buf)\n    {\n        /* Use a user provided read buffer. */\n        if (!user_read_buf_size)\n            return MZ_FALSE;\n        pRead_buf = (mz_uint8 *)pUser_read_buf;\n        read_buf_size = user_read_buf_size;\n        read_buf_avail = 0;\n        comp_remaining = file_stat.m_comp_size;\n    }\n    else\n    {\n        /* Temporarily allocate a read buffer. */\n        read_buf_size = MZ_MIN(file_stat.m_comp_size, (mz_uint64)MZ_ZIP_MAX_IO_BUF_SIZE);\n        if (((sizeof(size_t) == sizeof(mz_uint32))) && (read_buf_size > 0x7FFFFFFF))\n            return mz_zip_set_error(pZip, MZ_ZIP_INTERNAL_ERROR);\n\n        if (NULL == (pRead_buf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, (size_t)read_buf_size)))\n            return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n\n        read_buf_avail = 0;\n        comp_remaining = file_stat.m_comp_size;\n    }\n\n    do\n    {\n        /* The size_t cast here should be OK because we've verified that the output buffer is >= file_stat.m_uncomp_size above */\n        size_t in_buf_size, out_buf_size = (size_t)(file_stat.m_uncomp_size - out_buf_ofs);\n        if ((!read_buf_avail) && (!pZip->m_pState->m_pMem))\n        {\n            read_buf_avail = MZ_MIN(read_buf_size, comp_remaining);\n            if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pRead_buf, (size_t)read_buf_avail) != read_buf_avail)\n            {\n                status = TINFL_STATUS_FAILED;\n                mz_zip_set_error(pZip, MZ_ZIP_DECOMPRESSION_FAILED);\n                break;\n            }\n            cur_file_ofs += read_buf_avail;\n            comp_remaining -= read_buf_avail;\n            read_buf_ofs = 0;\n        }\n        in_buf_size = (size_t)read_buf_avail;\n        status = tinfl_decompress(&inflator, (mz_uint8 *)pRead_buf + read_buf_ofs, &in_buf_size, (mz_uint8 *)pBuf, (mz_uint8 *)pBuf + out_buf_ofs, &out_buf_size, TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF | (comp_remaining ? TINFL_FLAG_HAS_MORE_INPUT : 0));\n        read_buf_avail -= in_buf_size;\n        read_buf_ofs += in_buf_size;\n        out_buf_ofs += out_buf_size;\n    } while (status == TINFL_STATUS_NEEDS_MORE_INPUT);\n\n    if (status == TINFL_STATUS_DONE)\n    {\n        /* Make sure the entire file was decompressed, and check its CRC. */\n        if (out_buf_ofs != file_stat.m_uncomp_size)\n        {\n            mz_zip_set_error(pZip, MZ_ZIP_UNEXPECTED_DECOMPRESSED_SIZE);\n            status = TINFL_STATUS_FAILED;\n        }\n#ifndef MINIZ_DISABLE_ZIP_READER_CRC32_CHECKS\n        else if (mz_crc32(MZ_CRC32_INIT, (const mz_uint8 *)pBuf, (size_t)file_stat.m_uncomp_size) != file_stat.m_crc32)\n        {\n            mz_zip_set_error(pZip, MZ_ZIP_CRC_CHECK_FAILED);\n            status = TINFL_STATUS_FAILED;\n        }\n#endif\n    }\n\n    if ((!pZip->m_pState->m_pMem) && (!pUser_read_buf))\n        pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf);\n\n    return status == TINFL_STATUS_DONE;\n}\n\nmz_bool mz_zip_reader_extract_file_to_mem_no_alloc(mz_zip_archive *pZip, const char *pFilename, void *pBuf, size_t buf_size, mz_uint flags, void *pUser_read_buf, size_t user_read_buf_size)\n{\n    mz_uint32 file_index;\n    if (!mz_zip_reader_locate_file_v2(pZip, pFilename, NULL, flags, &file_index))\n        return MZ_FALSE;\n    return mz_zip_reader_extract_to_mem_no_alloc(pZip, file_index, pBuf, buf_size, flags, pUser_read_buf, user_read_buf_size);\n}\n\nmz_bool mz_zip_reader_extract_to_mem(mz_zip_archive *pZip, mz_uint file_index, void *pBuf, size_t buf_size, mz_uint flags)\n{\n    return mz_zip_reader_extract_to_mem_no_alloc(pZip, file_index, pBuf, buf_size, flags, NULL, 0);\n}\n\nmz_bool mz_zip_reader_extract_file_to_mem(mz_zip_archive *pZip, const char *pFilename, void *pBuf, size_t buf_size, mz_uint flags)\n{\n    return mz_zip_reader_extract_file_to_mem_no_alloc(pZip, pFilename, pBuf, buf_size, flags, NULL, 0);\n}\n\nvoid *mz_zip_reader_extract_to_heap(mz_zip_archive *pZip, mz_uint file_index, size_t *pSize, mz_uint flags)\n{\n    mz_uint64 comp_size, uncomp_size, alloc_size;\n    const mz_uint8 *p = mz_zip_get_cdh(pZip, file_index);\n    void *pBuf;\n\n    if (pSize)\n        *pSize = 0;\n\n    if (!p)\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n        return NULL;\n    }\n\n    comp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_COMPRESSED_SIZE_OFS);\n    uncomp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS);\n\n    alloc_size = (flags & MZ_ZIP_FLAG_COMPRESSED_DATA) ? comp_size : uncomp_size;\n    if (((sizeof(size_t) == sizeof(mz_uint32))) && (alloc_size > 0x7FFFFFFF))\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_INTERNAL_ERROR);\n        return NULL;\n    }\n\n    if (NULL == (pBuf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, (size_t)alloc_size)))\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n        return NULL;\n    }\n\n    if (!mz_zip_reader_extract_to_mem(pZip, file_index, pBuf, (size_t)alloc_size, flags))\n    {\n        pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf);\n        return NULL;\n    }\n\n    if (pSize)\n        *pSize = (size_t)alloc_size;\n    return pBuf;\n}\n\nvoid *mz_zip_reader_extract_file_to_heap(mz_zip_archive *pZip, const char *pFilename, size_t *pSize, mz_uint flags)\n{\n    mz_uint32 file_index;\n    if (!mz_zip_reader_locate_file_v2(pZip, pFilename, NULL, flags, &file_index))\n    {\n        if (pSize)\n            *pSize = 0;\n        return MZ_FALSE;\n    }\n    return mz_zip_reader_extract_to_heap(pZip, file_index, pSize, flags);\n}\n\nmz_bool mz_zip_reader_extract_to_callback(mz_zip_archive *pZip, mz_uint file_index, mz_file_write_func pCallback, void *pOpaque, mz_uint flags)\n{\n    int status = TINFL_STATUS_DONE;\n    mz_uint file_crc32 = MZ_CRC32_INIT;\n    mz_uint64 read_buf_size, read_buf_ofs = 0, read_buf_avail, comp_remaining, out_buf_ofs = 0, cur_file_ofs;\n    mz_zip_archive_file_stat file_stat;\n    void *pRead_buf = NULL;\n    void *pWrite_buf = NULL;\n    mz_uint32 local_header_u32[(MZ_ZIP_LOCAL_DIR_HEADER_SIZE + sizeof(mz_uint32) - 1) / sizeof(mz_uint32)];\n    mz_uint8 *pLocal_header = (mz_uint8 *)local_header_u32;\n\n    if ((!pZip) || (!pZip->m_pState) || (!pCallback) || (!pZip->m_pRead))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    if (!mz_zip_reader_file_stat(pZip, file_index, &file_stat))\n        return MZ_FALSE;\n\n    /* A directory or zero length file */\n    if ((file_stat.m_is_directory) || (!file_stat.m_comp_size))\n        return MZ_TRUE;\n\n    /* Encryption and patch files are not supported. */\n    if (file_stat.m_bit_flag & (MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_IS_ENCRYPTED | MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_USES_STRONG_ENCRYPTION | MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_COMPRESSED_PATCH_FLAG))\n        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_ENCRYPTION);\n\n    /* This function only supports decompressing stored and deflate. */\n    if ((!(flags & MZ_ZIP_FLAG_COMPRESSED_DATA)) && (file_stat.m_method != 0) && (file_stat.m_method != MZ_DEFLATED))\n        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_METHOD);\n\n    /* Read and do some minimal validation of the local directory entry (this doesn't crack the zip64 stuff, which we already have from the central dir) */\n    cur_file_ofs = file_stat.m_local_header_ofs;\n    if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pLocal_header, MZ_ZIP_LOCAL_DIR_HEADER_SIZE) != MZ_ZIP_LOCAL_DIR_HEADER_SIZE)\n        return mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n\n    if (MZ_READ_LE32(pLocal_header) != MZ_ZIP_LOCAL_DIR_HEADER_SIG)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n    cur_file_ofs += MZ_ZIP_LOCAL_DIR_HEADER_SIZE + MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_FILENAME_LEN_OFS) + MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_EXTRA_LEN_OFS);\n    if ((cur_file_ofs + file_stat.m_comp_size) > pZip->m_archive_size)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n    /* Decompress the file either directly from memory or from a file input buffer. */\n    if (pZip->m_pState->m_pMem)\n    {\n        pRead_buf = (mz_uint8 *)pZip->m_pState->m_pMem + cur_file_ofs;\n        read_buf_size = read_buf_avail = file_stat.m_comp_size;\n        comp_remaining = 0;\n    }\n    else\n    {\n        read_buf_size = MZ_MIN(file_stat.m_comp_size, (mz_uint64)MZ_ZIP_MAX_IO_BUF_SIZE);\n        if (NULL == (pRead_buf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, (size_t)read_buf_size)))\n            return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n\n        read_buf_avail = 0;\n        comp_remaining = file_stat.m_comp_size;\n    }\n\n    if ((flags & MZ_ZIP_FLAG_COMPRESSED_DATA) || (!file_stat.m_method))\n    {\n        /* The file is stored or the caller has requested the compressed data. */\n        if (pZip->m_pState->m_pMem)\n        {\n            if (((sizeof(size_t) == sizeof(mz_uint32))) && (file_stat.m_comp_size > MZ_UINT32_MAX))\n                return mz_zip_set_error(pZip, MZ_ZIP_INTERNAL_ERROR);\n\n            if (pCallback(pOpaque, out_buf_ofs, pRead_buf, (size_t)file_stat.m_comp_size) != file_stat.m_comp_size)\n            {\n                mz_zip_set_error(pZip, MZ_ZIP_WRITE_CALLBACK_FAILED);\n                status = TINFL_STATUS_FAILED;\n            }\n            else if (!(flags & MZ_ZIP_FLAG_COMPRESSED_DATA))\n            {\n#ifndef MINIZ_DISABLE_ZIP_READER_CRC32_CHECKS\n                file_crc32 = (mz_uint32)mz_crc32(file_crc32, (const mz_uint8 *)pRead_buf, (size_t)file_stat.m_comp_size);\n#endif\n            }\n\n            cur_file_ofs += file_stat.m_comp_size;\n            out_buf_ofs += file_stat.m_comp_size;\n            comp_remaining = 0;\n        }\n        else\n        {\n            while (comp_remaining)\n            {\n                read_buf_avail = MZ_MIN(read_buf_size, comp_remaining);\n                if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pRead_buf, (size_t)read_buf_avail) != read_buf_avail)\n                {\n                    mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n                    status = TINFL_STATUS_FAILED;\n                    break;\n                }\n\n#ifndef MINIZ_DISABLE_ZIP_READER_CRC32_CHECKS\n                if (!(flags & MZ_ZIP_FLAG_COMPRESSED_DATA))\n                {\n                    file_crc32 = (mz_uint32)mz_crc32(file_crc32, (const mz_uint8 *)pRead_buf, (size_t)read_buf_avail);\n                }\n#endif\n\n                if (pCallback(pOpaque, out_buf_ofs, pRead_buf, (size_t)read_buf_avail) != read_buf_avail)\n                {\n                    mz_zip_set_error(pZip, MZ_ZIP_WRITE_CALLBACK_FAILED);\n                    status = TINFL_STATUS_FAILED;\n                    break;\n                }\n\n                cur_file_ofs += read_buf_avail;\n                out_buf_ofs += read_buf_avail;\n                comp_remaining -= read_buf_avail;\n            }\n        }\n    }\n    else\n    {\n        tinfl_decompressor inflator;\n        tinfl_init(&inflator);\n\n        if (NULL == (pWrite_buf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, TINFL_LZ_DICT_SIZE)))\n        {\n            mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n            status = TINFL_STATUS_FAILED;\n        }\n        else\n        {\n            do\n            {\n                mz_uint8 *pWrite_buf_cur = (mz_uint8 *)pWrite_buf + (out_buf_ofs & (TINFL_LZ_DICT_SIZE - 1));\n                size_t in_buf_size, out_buf_size = TINFL_LZ_DICT_SIZE - (out_buf_ofs & (TINFL_LZ_DICT_SIZE - 1));\n                if ((!read_buf_avail) && (!pZip->m_pState->m_pMem))\n                {\n                    read_buf_avail = MZ_MIN(read_buf_size, comp_remaining);\n                    if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pRead_buf, (size_t)read_buf_avail) != read_buf_avail)\n                    {\n                        mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n                        status = TINFL_STATUS_FAILED;\n                        break;\n                    }\n                    cur_file_ofs += read_buf_avail;\n                    comp_remaining -= read_buf_avail;\n                    read_buf_ofs = 0;\n                }\n\n                in_buf_size = (size_t)read_buf_avail;\n                status = tinfl_decompress(&inflator, (const mz_uint8 *)pRead_buf + read_buf_ofs, &in_buf_size, (mz_uint8 *)pWrite_buf, pWrite_buf_cur, &out_buf_size, comp_remaining ? TINFL_FLAG_HAS_MORE_INPUT : 0);\n                read_buf_avail -= in_buf_size;\n                read_buf_ofs += in_buf_size;\n\n                if (out_buf_size)\n                {\n                    if (pCallback(pOpaque, out_buf_ofs, pWrite_buf_cur, out_buf_size) != out_buf_size)\n                    {\n                        mz_zip_set_error(pZip, MZ_ZIP_WRITE_CALLBACK_FAILED);\n                        status = TINFL_STATUS_FAILED;\n                        break;\n                    }\n\n#ifndef MINIZ_DISABLE_ZIP_READER_CRC32_CHECKS\n                    file_crc32 = (mz_uint32)mz_crc32(file_crc32, pWrite_buf_cur, out_buf_size);\n#endif\n                    if ((out_buf_ofs += out_buf_size) > file_stat.m_uncomp_size)\n                    {\n                        mz_zip_set_error(pZip, MZ_ZIP_DECOMPRESSION_FAILED);\n                        status = TINFL_STATUS_FAILED;\n                        break;\n                    }\n                }\n            } while ((status == TINFL_STATUS_NEEDS_MORE_INPUT) || (status == TINFL_STATUS_HAS_MORE_OUTPUT));\n        }\n    }\n\n    if ((status == TINFL_STATUS_DONE) && (!(flags & MZ_ZIP_FLAG_COMPRESSED_DATA)))\n    {\n        /* Make sure the entire file was decompressed, and check its CRC. */\n        if (out_buf_ofs != file_stat.m_uncomp_size)\n        {\n            mz_zip_set_error(pZip, MZ_ZIP_UNEXPECTED_DECOMPRESSED_SIZE);\n            status = TINFL_STATUS_FAILED;\n        }\n#ifndef MINIZ_DISABLE_ZIP_READER_CRC32_CHECKS\n        else if (file_crc32 != file_stat.m_crc32)\n        {\n            mz_zip_set_error(pZip, MZ_ZIP_DECOMPRESSION_FAILED);\n            status = TINFL_STATUS_FAILED;\n        }\n#endif\n    }\n\n    if (!pZip->m_pState->m_pMem)\n        pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf);\n\n    if (pWrite_buf)\n        pZip->m_pFree(pZip->m_pAlloc_opaque, pWrite_buf);\n\n    return status == TINFL_STATUS_DONE;\n}\n\nmz_bool mz_zip_reader_extract_file_to_callback(mz_zip_archive *pZip, const char *pFilename, mz_file_write_func pCallback, void *pOpaque, mz_uint flags)\n{\n    mz_uint32 file_index;\n    if (!mz_zip_reader_locate_file_v2(pZip, pFilename, NULL, flags, &file_index))\n        return MZ_FALSE;\n\n    return mz_zip_reader_extract_to_callback(pZip, file_index, pCallback, pOpaque, flags);\n}\n\nmz_zip_reader_extract_iter_state* mz_zip_reader_extract_iter_new(mz_zip_archive *pZip, mz_uint file_index, mz_uint flags)\n{\n    mz_zip_reader_extract_iter_state *pState;\n    mz_uint32 local_header_u32[(MZ_ZIP_LOCAL_DIR_HEADER_SIZE + sizeof(mz_uint32) - 1) / sizeof(mz_uint32)];\n    mz_uint8 *pLocal_header = (mz_uint8 *)local_header_u32;\n\n    /* Argument sanity check */\n    if ((!pZip) || (!pZip->m_pState))\n        return NULL;\n\n    /* Allocate an iterator status structure */\n    pState = (mz_zip_reader_extract_iter_state*)pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, sizeof(mz_zip_reader_extract_iter_state));\n    if (!pState)\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n        return NULL;\n    }\n\n    /* Fetch file details */\n    if (!mz_zip_reader_file_stat(pZip, file_index, &pState->file_stat))\n    {\n        pZip->m_pFree(pZip->m_pAlloc_opaque, pState);\n        return NULL;\n    }\n\n    /* Encryption and patch files are not supported. */\n    if (pState->file_stat.m_bit_flag & (MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_IS_ENCRYPTED | MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_USES_STRONG_ENCRYPTION | MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_COMPRESSED_PATCH_FLAG))\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_ENCRYPTION);\n        pZip->m_pFree(pZip->m_pAlloc_opaque, pState);\n        return NULL;\n    }\n\n    /* This function only supports decompressing stored and deflate. */\n    if ((!(flags & MZ_ZIP_FLAG_COMPRESSED_DATA)) && (pState->file_stat.m_method != 0) && (pState->file_stat.m_method != MZ_DEFLATED))\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_METHOD);\n        pZip->m_pFree(pZip->m_pAlloc_opaque, pState);\n        return NULL;\n    }\n\n    /* Init state - save args */\n    pState->pZip = pZip;\n    pState->flags = flags;\n\n    /* Init state - reset variables to defaults */\n    pState->status = TINFL_STATUS_DONE;\n#ifndef MINIZ_DISABLE_ZIP_READER_CRC32_CHECKS\n    pState->file_crc32 = MZ_CRC32_INIT;\n#endif\n    pState->read_buf_ofs = 0;\n    pState->out_buf_ofs = 0;\n    pState->pRead_buf = NULL;\n    pState->pWrite_buf = NULL;\n    pState->out_blk_remain = 0;\n\n    /* Read and parse the local directory entry. */\n    pState->cur_file_ofs = pState->file_stat.m_local_header_ofs;\n    if (pZip->m_pRead(pZip->m_pIO_opaque, pState->cur_file_ofs, pLocal_header, MZ_ZIP_LOCAL_DIR_HEADER_SIZE) != MZ_ZIP_LOCAL_DIR_HEADER_SIZE)\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n        pZip->m_pFree(pZip->m_pAlloc_opaque, pState);\n        return NULL;\n    }\n\n    if (MZ_READ_LE32(pLocal_header) != MZ_ZIP_LOCAL_DIR_HEADER_SIG)\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n        pZip->m_pFree(pZip->m_pAlloc_opaque, pState);\n        return NULL;\n    }\n\n    pState->cur_file_ofs += MZ_ZIP_LOCAL_DIR_HEADER_SIZE + MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_FILENAME_LEN_OFS) + MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_EXTRA_LEN_OFS);\n    if ((pState->cur_file_ofs + pState->file_stat.m_comp_size) > pZip->m_archive_size)\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n        pZip->m_pFree(pZip->m_pAlloc_opaque, pState);\n        return NULL;\n    }\n\n    /* Decompress the file either directly from memory or from a file input buffer. */\n    if (pZip->m_pState->m_pMem)\n    {\n        pState->pRead_buf = (mz_uint8 *)pZip->m_pState->m_pMem + pState->cur_file_ofs;\n        pState->read_buf_size = pState->read_buf_avail = pState->file_stat.m_comp_size;\n        pState->comp_remaining = pState->file_stat.m_comp_size;\n    }\n    else\n    {\n        if (!((flags & MZ_ZIP_FLAG_COMPRESSED_DATA) || (!pState->file_stat.m_method)))\n        {\n            /* Decompression required, therefore intermediate read buffer required */\n            pState->read_buf_size = MZ_MIN(pState->file_stat.m_comp_size, MZ_ZIP_MAX_IO_BUF_SIZE);\n            if (NULL == (pState->pRead_buf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, (size_t)pState->read_buf_size)))\n            {\n                mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n                pZip->m_pFree(pZip->m_pAlloc_opaque, pState);\n                return NULL;\n            }\n        }\n        else\n        {\n            /* Decompression not required - we will be reading directly into user buffer, no temp buf required */\n            pState->read_buf_size = 0;\n        }\n        pState->read_buf_avail = 0;\n        pState->comp_remaining = pState->file_stat.m_comp_size;\n    }\n\n    if (!((flags & MZ_ZIP_FLAG_COMPRESSED_DATA) || (!pState->file_stat.m_method)))\n    {\n        /* Decompression required, init decompressor */\n        tinfl_init( &pState->inflator );\n\n        /* Allocate write buffer */\n        if (NULL == (pState->pWrite_buf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, TINFL_LZ_DICT_SIZE)))\n        {\n            mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n            if (pState->pRead_buf)\n                pZip->m_pFree(pZip->m_pAlloc_opaque, pState->pRead_buf);\n            pZip->m_pFree(pZip->m_pAlloc_opaque, pState);\n            return NULL;\n        }\n    }\n\n    return pState;\n}\n\nmz_zip_reader_extract_iter_state* mz_zip_reader_extract_file_iter_new(mz_zip_archive *pZip, const char *pFilename, mz_uint flags)\n{\n    mz_uint32 file_index;\n\n    /* Locate file index by name */\n    if (!mz_zip_reader_locate_file_v2(pZip, pFilename, NULL, flags, &file_index))\n        return NULL;\n\n    /* Construct iterator */\n    return mz_zip_reader_extract_iter_new(pZip, file_index, flags);\n}\n\nsize_t mz_zip_reader_extract_iter_read(mz_zip_reader_extract_iter_state* pState, void* pvBuf, size_t buf_size)\n{\n    size_t copied_to_caller = 0;\n\n    /* Argument sanity check */\n    if ((!pState) || (!pState->pZip) || (!pState->pZip->m_pState) || (!pvBuf))\n        return 0;\n\n    if ((pState->flags & MZ_ZIP_FLAG_COMPRESSED_DATA) || (!pState->file_stat.m_method))\n    {\n        /* The file is stored or the caller has requested the compressed data, calc amount to return. */\n        copied_to_caller = (size_t)MZ_MIN( buf_size, pState->comp_remaining );\n\n        /* Zip is in memory....or requires reading from a file? */\n        if (pState->pZip->m_pState->m_pMem)\n        {\n            /* Copy data to caller's buffer */\n            memcpy( pvBuf, pState->pRead_buf, copied_to_caller );\n            pState->pRead_buf = ((mz_uint8*)pState->pRead_buf) + copied_to_caller;\n        }\n        else\n        {\n            /* Read directly into caller's buffer */\n            if (pState->pZip->m_pRead(pState->pZip->m_pIO_opaque, pState->cur_file_ofs, pvBuf, copied_to_caller) != copied_to_caller)\n            {\n                /* Failed to read all that was asked for, flag failure and alert user */\n                mz_zip_set_error(pState->pZip, MZ_ZIP_FILE_READ_FAILED);\n                pState->status = TINFL_STATUS_FAILED;\n                copied_to_caller = 0;\n            }\n        }\n\n#ifndef MINIZ_DISABLE_ZIP_READER_CRC32_CHECKS\n        /* Compute CRC if not returning compressed data only */\n        if (!(pState->flags & MZ_ZIP_FLAG_COMPRESSED_DATA))\n            pState->file_crc32 = (mz_uint32)mz_crc32(pState->file_crc32, (const mz_uint8 *)pvBuf, copied_to_caller);\n#endif\n\n        /* Advance offsets, dec counters */\n        pState->cur_file_ofs += copied_to_caller;\n        pState->out_buf_ofs += copied_to_caller;\n        pState->comp_remaining -= copied_to_caller;\n    }\n    else\n    {\n        do\n        {\n            /* Calc ptr to write buffer - given current output pos and block size */\n            mz_uint8 *pWrite_buf_cur = (mz_uint8 *)pState->pWrite_buf + (pState->out_buf_ofs & (TINFL_LZ_DICT_SIZE - 1));\n\n            /* Calc max output size - given current output pos and block size */\n            size_t in_buf_size, out_buf_size = TINFL_LZ_DICT_SIZE - (pState->out_buf_ofs & (TINFL_LZ_DICT_SIZE - 1));\n\n            if (!pState->out_blk_remain)\n            {\n                /* Read more data from file if none available (and reading from file) */\n                if ((!pState->read_buf_avail) && (!pState->pZip->m_pState->m_pMem))\n                {\n                    /* Calc read size */\n                    pState->read_buf_avail = MZ_MIN(pState->read_buf_size, pState->comp_remaining);\n                    if (pState->pZip->m_pRead(pState->pZip->m_pIO_opaque, pState->cur_file_ofs, pState->pRead_buf, (size_t)pState->read_buf_avail) != pState->read_buf_avail)\n                    {\n                        mz_zip_set_error(pState->pZip, MZ_ZIP_FILE_READ_FAILED);\n                        pState->status = TINFL_STATUS_FAILED;\n                        break;\n                    }\n\n                    /* Advance offsets, dec counters */\n                    pState->cur_file_ofs += pState->read_buf_avail;\n                    pState->comp_remaining -= pState->read_buf_avail;\n                    pState->read_buf_ofs = 0;\n                }\n\n                /* Perform decompression */\n                in_buf_size = (size_t)pState->read_buf_avail;\n                pState->status = tinfl_decompress(&pState->inflator, (const mz_uint8 *)pState->pRead_buf + pState->read_buf_ofs, &in_buf_size, (mz_uint8 *)pState->pWrite_buf, pWrite_buf_cur, &out_buf_size, pState->comp_remaining ? TINFL_FLAG_HAS_MORE_INPUT : 0);\n                pState->read_buf_avail -= in_buf_size;\n                pState->read_buf_ofs += in_buf_size;\n\n                /* Update current output block size remaining */\n                pState->out_blk_remain = out_buf_size;\n            }\n\n            if (pState->out_blk_remain)\n            {\n                /* Calc amount to return. */\n                size_t to_copy = MZ_MIN( (buf_size - copied_to_caller), pState->out_blk_remain );\n\n                /* Copy data to caller's buffer */\n                memcpy( (uint8_t*)pvBuf + copied_to_caller, pWrite_buf_cur, to_copy );\n\n#ifndef MINIZ_DISABLE_ZIP_READER_CRC32_CHECKS\n                /* Perform CRC */\n                pState->file_crc32 = (mz_uint32)mz_crc32(pState->file_crc32, pWrite_buf_cur, to_copy);\n#endif\n\n                /* Decrement data consumed from block */\n                pState->out_blk_remain -= to_copy;\n\n                /* Inc output offset, while performing sanity check */\n                if ((pState->out_buf_ofs += to_copy) > pState->file_stat.m_uncomp_size)\n                {\n                    mz_zip_set_error(pState->pZip, MZ_ZIP_DECOMPRESSION_FAILED);\n                    pState->status = TINFL_STATUS_FAILED;\n                    break;\n                }\n\n                /* Increment counter of data copied to caller */\n                copied_to_caller += to_copy;\n            }\n        } while ( (copied_to_caller < buf_size) && ((pState->status == TINFL_STATUS_NEEDS_MORE_INPUT) || (pState->status == TINFL_STATUS_HAS_MORE_OUTPUT)) );\n    }\n\n    /* Return how many bytes were copied into user buffer */\n    return copied_to_caller;\n}\n\nmz_bool mz_zip_reader_extract_iter_free(mz_zip_reader_extract_iter_state* pState)\n{\n    int status;\n\n    /* Argument sanity check */\n    if ((!pState) || (!pState->pZip) || (!pState->pZip->m_pState))\n        return MZ_FALSE;\n\n    /* Was decompression completed and requested? */\n    if ((pState->status == TINFL_STATUS_DONE) && (!(pState->flags & MZ_ZIP_FLAG_COMPRESSED_DATA)))\n    {\n        /* Make sure the entire file was decompressed, and check its CRC. */\n        if (pState->out_buf_ofs != pState->file_stat.m_uncomp_size)\n        {\n            mz_zip_set_error(pState->pZip, MZ_ZIP_UNEXPECTED_DECOMPRESSED_SIZE);\n            pState->status = TINFL_STATUS_FAILED;\n        }\n#ifndef MINIZ_DISABLE_ZIP_READER_CRC32_CHECKS\n        else if (pState->file_crc32 != pState->file_stat.m_crc32)\n        {\n            mz_zip_set_error(pState->pZip, MZ_ZIP_DECOMPRESSION_FAILED);\n            pState->status = TINFL_STATUS_FAILED;\n        }\n#endif\n    }\n\n    /* Free buffers */\n    if (!pState->pZip->m_pState->m_pMem)\n        pState->pZip->m_pFree(pState->pZip->m_pAlloc_opaque, pState->pRead_buf);\n    if (pState->pWrite_buf)\n        pState->pZip->m_pFree(pState->pZip->m_pAlloc_opaque, pState->pWrite_buf);\n\n    /* Save status */\n    status = pState->status;\n\n    /* Free context */\n    pState->pZip->m_pFree(pState->pZip->m_pAlloc_opaque, pState);\n\n    return status == TINFL_STATUS_DONE;\n}\n\n#ifndef MINIZ_NO_STDIO\nstatic size_t mz_zip_file_write_callback(void *pOpaque, mz_uint64 ofs, const void *pBuf, size_t n)\n{\n    (void)ofs;\n\n    return MZ_FWRITE(pBuf, 1, n, (MZ_FILE *)pOpaque);\n}\n\nmz_bool mz_zip_reader_extract_to_file(mz_zip_archive *pZip, mz_uint file_index, const char *pDst_filename, mz_uint flags)\n{\n    mz_bool status;\n    mz_zip_archive_file_stat file_stat;\n    MZ_FILE *pFile;\n\n    if (!mz_zip_reader_file_stat(pZip, file_index, &file_stat))\n        return MZ_FALSE;\n\n    if ((file_stat.m_is_directory) || (!file_stat.m_is_supported))\n        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_FEATURE);\n\n    pFile = MZ_FOPEN(pDst_filename, \"wb\");\n    if (!pFile)\n        return mz_zip_set_error(pZip, MZ_ZIP_FILE_OPEN_FAILED);\n\n    status = mz_zip_reader_extract_to_callback(pZip, file_index, mz_zip_file_write_callback, pFile, flags);\n\n    if (MZ_FCLOSE(pFile) == EOF)\n    {\n        if (status)\n            mz_zip_set_error(pZip, MZ_ZIP_FILE_CLOSE_FAILED);\n\n        status = MZ_FALSE;\n    }\n\n#if !defined(MINIZ_NO_TIME) && !defined(MINIZ_NO_STDIO)\n    if (status)\n        mz_zip_set_file_times(pDst_filename, file_stat.m_time, file_stat.m_time);\n#endif\n\n    return status;\n}\n\nmz_bool mz_zip_reader_extract_file_to_file(mz_zip_archive *pZip, const char *pArchive_filename, const char *pDst_filename, mz_uint flags)\n{\n    mz_uint32 file_index;\n    if (!mz_zip_reader_locate_file_v2(pZip, pArchive_filename, NULL, flags, &file_index))\n        return MZ_FALSE;\n\n    return mz_zip_reader_extract_to_file(pZip, file_index, pDst_filename, flags);\n}\n\nmz_bool mz_zip_reader_extract_to_cfile(mz_zip_archive *pZip, mz_uint file_index, MZ_FILE *pFile, mz_uint flags)\n{\n    mz_zip_archive_file_stat file_stat;\n\n    if (!mz_zip_reader_file_stat(pZip, file_index, &file_stat))\n        return MZ_FALSE;\n\n    if ((file_stat.m_is_directory) || (!file_stat.m_is_supported))\n        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_FEATURE);\n\n    return mz_zip_reader_extract_to_callback(pZip, file_index, mz_zip_file_write_callback, pFile, flags);\n}\n\nmz_bool mz_zip_reader_extract_file_to_cfile(mz_zip_archive *pZip, const char *pArchive_filename, MZ_FILE *pFile, mz_uint flags)\n{\n    mz_uint32 file_index;\n    if (!mz_zip_reader_locate_file_v2(pZip, pArchive_filename, NULL, flags, &file_index))\n        return MZ_FALSE;\n\n    return mz_zip_reader_extract_to_cfile(pZip, file_index, pFile, flags);\n}\n#endif /* #ifndef MINIZ_NO_STDIO */\n\nstatic size_t mz_zip_compute_crc32_callback(void *pOpaque, mz_uint64 file_ofs, const void *pBuf, size_t n)\n{\n    mz_uint32 *p = (mz_uint32 *)pOpaque;\n    (void)file_ofs;\n    *p = (mz_uint32)mz_crc32(*p, (const mz_uint8 *)pBuf, n);\n    return n;\n}\n\nmz_bool mz_zip_validate_file(mz_zip_archive *pZip, mz_uint file_index, mz_uint flags)\n{\n    mz_zip_archive_file_stat file_stat;\n    mz_zip_internal_state *pState;\n    const mz_uint8 *pCentral_dir_header;\n    mz_bool found_zip64_ext_data_in_cdir = MZ_FALSE;\n    mz_bool found_zip64_ext_data_in_ldir = MZ_FALSE;\n    mz_uint32 local_header_u32[(MZ_ZIP_LOCAL_DIR_HEADER_SIZE + sizeof(mz_uint32) - 1) / sizeof(mz_uint32)];\n    mz_uint8 *pLocal_header = (mz_uint8 *)local_header_u32;\n    mz_uint64 local_header_ofs = 0;\n    mz_uint32 local_header_filename_len, local_header_extra_len, local_header_crc32;\n    mz_uint64 local_header_comp_size, local_header_uncomp_size;\n    mz_uint32 uncomp_crc32 = MZ_CRC32_INIT;\n    mz_bool has_data_descriptor;\n    mz_uint32 local_header_bit_flags;\n\n    mz_zip_array file_data_array;\n    mz_zip_array_init(&file_data_array, 1);\n\n    if ((!pZip) || (!pZip->m_pState) || (!pZip->m_pAlloc) || (!pZip->m_pFree) || (!pZip->m_pRead))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    if (file_index > pZip->m_total_files)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    pState = pZip->m_pState;\n\n    pCentral_dir_header = mz_zip_get_cdh(pZip, file_index);\n\n    if (!mz_zip_file_stat_internal(pZip, file_index, pCentral_dir_header, &file_stat, &found_zip64_ext_data_in_cdir))\n        return MZ_FALSE;\n\n    /* A directory or zero length file */\n    if ((file_stat.m_is_directory) || (!file_stat.m_uncomp_size))\n        return MZ_TRUE;\n\n    /* Encryption and patch files are not supported. */\n    if (file_stat.m_is_encrypted)\n        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_ENCRYPTION);\n\n    /* This function only supports stored and deflate. */\n    if ((file_stat.m_method != 0) && (file_stat.m_method != MZ_DEFLATED))\n        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_METHOD);\n\n    if (!file_stat.m_is_supported)\n        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_FEATURE);\n\n    /* Read and parse the local directory entry. */\n    local_header_ofs = file_stat.m_local_header_ofs;\n    if (pZip->m_pRead(pZip->m_pIO_opaque, local_header_ofs, pLocal_header, MZ_ZIP_LOCAL_DIR_HEADER_SIZE) != MZ_ZIP_LOCAL_DIR_HEADER_SIZE)\n        return mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n\n    if (MZ_READ_LE32(pLocal_header) != MZ_ZIP_LOCAL_DIR_HEADER_SIG)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n    local_header_filename_len = MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_FILENAME_LEN_OFS);\n    local_header_extra_len = MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_EXTRA_LEN_OFS);\n    local_header_comp_size = MZ_READ_LE32(pLocal_header + MZ_ZIP_LDH_COMPRESSED_SIZE_OFS);\n    local_header_uncomp_size = MZ_READ_LE32(pLocal_header + MZ_ZIP_LDH_DECOMPRESSED_SIZE_OFS);\n    local_header_crc32 = MZ_READ_LE32(pLocal_header + MZ_ZIP_LDH_CRC32_OFS);\n    local_header_bit_flags = MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_BIT_FLAG_OFS);\n    has_data_descriptor = (local_header_bit_flags & 8) != 0;\n\n    if (local_header_filename_len != strlen(file_stat.m_filename))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n    if ((local_header_ofs + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + local_header_filename_len + local_header_extra_len + file_stat.m_comp_size) > pZip->m_archive_size)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n    if (!mz_zip_array_resize(pZip, &file_data_array, MZ_MAX(local_header_filename_len, local_header_extra_len), MZ_FALSE))\n        return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n\n    if (local_header_filename_len)\n    {\n        if (pZip->m_pRead(pZip->m_pIO_opaque, local_header_ofs + MZ_ZIP_LOCAL_DIR_HEADER_SIZE, file_data_array.m_p, local_header_filename_len) != local_header_filename_len)\n        {\n            mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n            goto handle_failure;\n        }\n\n        /* I've seen 1 archive that had the same pathname, but used backslashes in the local dir and forward slashes in the central dir. Do we care about this? For now, this case will fail validation. */\n        if (memcmp(file_stat.m_filename, file_data_array.m_p, local_header_filename_len) != 0)\n        {\n            mz_zip_set_error(pZip, MZ_ZIP_VALIDATION_FAILED);\n            goto handle_failure;\n        }\n    }\n\n    if ((local_header_extra_len) && ((local_header_comp_size == MZ_UINT32_MAX) || (local_header_uncomp_size == MZ_UINT32_MAX)))\n    {\n        mz_uint32 extra_size_remaining = local_header_extra_len;\n        const mz_uint8 *pExtra_data = (const mz_uint8 *)file_data_array.m_p;\n\n        if (pZip->m_pRead(pZip->m_pIO_opaque, local_header_ofs + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + local_header_filename_len, file_data_array.m_p, local_header_extra_len) != local_header_extra_len)\n        {\n            mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n            goto handle_failure;\n        }\n\n        do\n        {\n            mz_uint32 field_id, field_data_size, field_total_size;\n\n            if (extra_size_remaining < (sizeof(mz_uint16) * 2))\n                return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n            field_id = MZ_READ_LE16(pExtra_data);\n            field_data_size = MZ_READ_LE16(pExtra_data + sizeof(mz_uint16));\n            field_total_size = field_data_size + sizeof(mz_uint16) * 2;\n\n            if (field_total_size > extra_size_remaining)\n                return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n            if (field_id == MZ_ZIP64_EXTENDED_INFORMATION_FIELD_HEADER_ID)\n            {\n                const mz_uint8 *pSrc_field_data = pExtra_data + sizeof(mz_uint32);\n\n                if (field_data_size < sizeof(mz_uint64) * 2)\n                {\n                    mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n                    goto handle_failure;\n                }\n\n                local_header_uncomp_size = MZ_READ_LE64(pSrc_field_data);\n                local_header_comp_size = MZ_READ_LE64(pSrc_field_data + sizeof(mz_uint64));\n\n                found_zip64_ext_data_in_ldir = MZ_TRUE;\n                break;\n            }\n\n            pExtra_data += field_total_size;\n            extra_size_remaining -= field_total_size;\n        } while (extra_size_remaining);\n    }\n\n    /* TODO: parse local header extra data when local_header_comp_size is 0xFFFFFFFF! (big_descriptor.zip) */\n    /* I've seen zips in the wild with the data descriptor bit set, but proper local header values and bogus data descriptors */\n    if ((has_data_descriptor) && (!local_header_comp_size) && (!local_header_crc32))\n    {\n        mz_uint8 descriptor_buf[32];\n        mz_bool has_id;\n        const mz_uint8 *pSrc;\n        mz_uint32 file_crc32;\n        mz_uint64 comp_size = 0, uncomp_size = 0;\n\n        mz_uint32 num_descriptor_uint32s = ((pState->m_zip64) || (found_zip64_ext_data_in_ldir)) ? 6 : 4;\n\n        if (pZip->m_pRead(pZip->m_pIO_opaque, local_header_ofs + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + local_header_filename_len + local_header_extra_len + file_stat.m_comp_size, descriptor_buf, sizeof(mz_uint32) * num_descriptor_uint32s) != (sizeof(mz_uint32) * num_descriptor_uint32s))\n        {\n            mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n            goto handle_failure;\n        }\n\n        has_id = (MZ_READ_LE32(descriptor_buf) == MZ_ZIP_DATA_DESCRIPTOR_ID);\n        pSrc = has_id ? (descriptor_buf + sizeof(mz_uint32)) : descriptor_buf;\n\n        file_crc32 = MZ_READ_LE32(pSrc);\n\n        if ((pState->m_zip64) || (found_zip64_ext_data_in_ldir))\n        {\n            comp_size = MZ_READ_LE64(pSrc + sizeof(mz_uint32));\n            uncomp_size = MZ_READ_LE64(pSrc + sizeof(mz_uint32) + sizeof(mz_uint64));\n        }\n        else\n        {\n            comp_size = MZ_READ_LE32(pSrc + sizeof(mz_uint32));\n            uncomp_size = MZ_READ_LE32(pSrc + sizeof(mz_uint32) + sizeof(mz_uint32));\n        }\n\n        if ((file_crc32 != file_stat.m_crc32) || (comp_size != file_stat.m_comp_size) || (uncomp_size != file_stat.m_uncomp_size))\n        {\n            mz_zip_set_error(pZip, MZ_ZIP_VALIDATION_FAILED);\n            goto handle_failure;\n        }\n    }\n    else\n    {\n        if ((local_header_crc32 != file_stat.m_crc32) || (local_header_comp_size != file_stat.m_comp_size) || (local_header_uncomp_size != file_stat.m_uncomp_size))\n        {\n            mz_zip_set_error(pZip, MZ_ZIP_VALIDATION_FAILED);\n            goto handle_failure;\n        }\n    }\n\n    mz_zip_array_clear(pZip, &file_data_array);\n\n    if ((flags & MZ_ZIP_FLAG_VALIDATE_HEADERS_ONLY) == 0)\n    {\n        if (!mz_zip_reader_extract_to_callback(pZip, file_index, mz_zip_compute_crc32_callback, &uncomp_crc32, 0))\n            return MZ_FALSE;\n\n        /* 1 more check to be sure, although the extract checks too. */\n        if (uncomp_crc32 != file_stat.m_crc32)\n        {\n            mz_zip_set_error(pZip, MZ_ZIP_VALIDATION_FAILED);\n            return MZ_FALSE;\n        }\n    }\n\n    return MZ_TRUE;\n\nhandle_failure:\n    mz_zip_array_clear(pZip, &file_data_array);\n    return MZ_FALSE;\n}\n\nmz_bool mz_zip_validate_archive(mz_zip_archive *pZip, mz_uint flags)\n{\n    mz_zip_internal_state *pState;\n    uint32_t i;\n\n    if ((!pZip) || (!pZip->m_pState) || (!pZip->m_pAlloc) || (!pZip->m_pFree) || (!pZip->m_pRead))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    pState = pZip->m_pState;\n\n    /* Basic sanity checks */\n    if (!pState->m_zip64)\n    {\n        if (pZip->m_total_files > MZ_UINT16_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE);\n\n        if (pZip->m_archive_size > MZ_UINT32_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE);\n    }\n    else\n    {\n        if (pZip->m_total_files >= MZ_UINT32_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE);\n\n        if (pState->m_central_dir.m_size >= MZ_UINT32_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE);\n    }\n\n    for (i = 0; i < pZip->m_total_files; i++)\n    {\n        if (MZ_ZIP_FLAG_VALIDATE_LOCATE_FILE_FLAG & flags)\n        {\n            mz_uint32 found_index;\n            mz_zip_archive_file_stat stat;\n\n            if (!mz_zip_reader_file_stat(pZip, i, &stat))\n                return MZ_FALSE;\n\n            if (!mz_zip_reader_locate_file_v2(pZip, stat.m_filename, NULL, 0, &found_index))\n                return MZ_FALSE;\n\n            /* This check can fail if there are duplicate filenames in the archive (which we don't check for when writing - that's up to the user) */\n            if (found_index != i)\n                return mz_zip_set_error(pZip, MZ_ZIP_VALIDATION_FAILED);\n        }\n\n        if (!mz_zip_validate_file(pZip, i, flags))\n            return MZ_FALSE;\n    }\n\n    return MZ_TRUE;\n}\n\nmz_bool mz_zip_validate_mem_archive(const void *pMem, size_t size, mz_uint flags, mz_zip_error *pErr)\n{\n    mz_bool success = MZ_TRUE;\n    mz_zip_archive zip;\n    mz_zip_error actual_err = MZ_ZIP_NO_ERROR;\n\n    if ((!pMem) || (!size))\n    {\n        if (pErr)\n            *pErr = MZ_ZIP_INVALID_PARAMETER;\n        return MZ_FALSE;\n    }\n\n    mz_zip_zero_struct(&zip);\n\n    if (!mz_zip_reader_init_mem(&zip, pMem, size, flags))\n    {\n        if (pErr)\n            *pErr = zip.m_last_error;\n        return MZ_FALSE;\n    }\n\n    if (!mz_zip_validate_archive(&zip, flags))\n    {\n        actual_err = zip.m_last_error;\n        success = MZ_FALSE;\n    }\n\n    if (!mz_zip_reader_end_internal(&zip, success))\n    {\n        if (!actual_err)\n            actual_err = zip.m_last_error;\n        success = MZ_FALSE;\n    }\n\n    if (pErr)\n        *pErr = actual_err;\n\n    return success;\n}\n\n#ifndef MINIZ_NO_STDIO\nmz_bool mz_zip_validate_file_archive(const char *pFilename, mz_uint flags, mz_zip_error *pErr)\n{\n    mz_bool success = MZ_TRUE;\n    mz_zip_archive zip;\n    mz_zip_error actual_err = MZ_ZIP_NO_ERROR;\n\n    if (!pFilename)\n    {\n        if (pErr)\n            *pErr = MZ_ZIP_INVALID_PARAMETER;\n        return MZ_FALSE;\n    }\n\n    mz_zip_zero_struct(&zip);\n\n    if (!mz_zip_reader_init_file_v2(&zip, pFilename, flags, 0, 0))\n    {\n        if (pErr)\n            *pErr = zip.m_last_error;\n        return MZ_FALSE;\n    }\n\n    if (!mz_zip_validate_archive(&zip, flags))\n    {\n        actual_err = zip.m_last_error;\n        success = MZ_FALSE;\n    }\n\n    if (!mz_zip_reader_end_internal(&zip, success))\n    {\n        if (!actual_err)\n            actual_err = zip.m_last_error;\n        success = MZ_FALSE;\n    }\n\n    if (pErr)\n        *pErr = actual_err;\n\n    return success;\n}\n#endif /* #ifndef MINIZ_NO_STDIO */\n\n/* ------------------- .ZIP archive writing */\n\n#ifndef MINIZ_NO_ARCHIVE_WRITING_APIS\n\nstatic MZ_FORCEINLINE void mz_write_le16(mz_uint8 *p, mz_uint16 v)\n{\n    p[0] = (mz_uint8)v;\n    p[1] = (mz_uint8)(v >> 8);\n}\nstatic MZ_FORCEINLINE void mz_write_le32(mz_uint8 *p, mz_uint32 v)\n{\n    p[0] = (mz_uint8)v;\n    p[1] = (mz_uint8)(v >> 8);\n    p[2] = (mz_uint8)(v >> 16);\n    p[3] = (mz_uint8)(v >> 24);\n}\nstatic MZ_FORCEINLINE void mz_write_le64(mz_uint8 *p, mz_uint64 v)\n{\n    mz_write_le32(p, (mz_uint32)v);\n    mz_write_le32(p + sizeof(mz_uint32), (mz_uint32)(v >> 32));\n}\n\n#define MZ_WRITE_LE16(p, v) mz_write_le16((mz_uint8 *)(p), (mz_uint16)(v))\n#define MZ_WRITE_LE32(p, v) mz_write_le32((mz_uint8 *)(p), (mz_uint32)(v))\n#define MZ_WRITE_LE64(p, v) mz_write_le64((mz_uint8 *)(p), (mz_uint64)(v))\n\nstatic size_t mz_zip_heap_write_func(void *pOpaque, mz_uint64 file_ofs, const void *pBuf, size_t n)\n{\n    mz_zip_archive *pZip = (mz_zip_archive *)pOpaque;\n    mz_zip_internal_state *pState = pZip->m_pState;\n    mz_uint64 new_size = MZ_MAX(file_ofs + n, pState->m_mem_size);\n\n    if (!n)\n        return 0;\n\n    /* An allocation this big is likely to just fail on 32-bit systems, so don't even go there. */\n    if ((sizeof(size_t) == sizeof(mz_uint32)) && (new_size > 0x7FFFFFFF))\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_FILE_TOO_LARGE);\n        return 0;\n    }\n\n    if (new_size > pState->m_mem_capacity)\n    {\n        void *pNew_block;\n        size_t new_capacity = MZ_MAX(64, pState->m_mem_capacity);\n\n        while (new_capacity < new_size)\n            new_capacity *= 2;\n\n        if (NULL == (pNew_block = pZip->m_pRealloc(pZip->m_pAlloc_opaque, pState->m_pMem, 1, new_capacity)))\n        {\n            mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n            return 0;\n        }\n\n        pState->m_pMem = pNew_block;\n        pState->m_mem_capacity = new_capacity;\n    }\n    memcpy((mz_uint8 *)pState->m_pMem + file_ofs, pBuf, n);\n    pState->m_mem_size = (size_t)new_size;\n    return n;\n}\n\nstatic mz_bool mz_zip_writer_end_internal(mz_zip_archive *pZip, mz_bool set_last_error)\n{\n    mz_zip_internal_state *pState;\n    mz_bool status = MZ_TRUE;\n\n    if ((!pZip) || (!pZip->m_pState) || (!pZip->m_pAlloc) || (!pZip->m_pFree) || ((pZip->m_zip_mode != MZ_ZIP_MODE_WRITING) && (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING_HAS_BEEN_FINALIZED)))\n    {\n        if (set_last_error)\n            mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n        return MZ_FALSE;\n    }\n\n    pState = pZip->m_pState;\n    pZip->m_pState = NULL;\n    mz_zip_array_clear(pZip, &pState->m_central_dir);\n    mz_zip_array_clear(pZip, &pState->m_central_dir_offsets);\n    mz_zip_array_clear(pZip, &pState->m_sorted_central_dir_offsets);\n\n#ifndef MINIZ_NO_STDIO\n    if (pState->m_pFile)\n    {\n        if (pZip->m_zip_type == MZ_ZIP_TYPE_FILE)\n        {\n            if (MZ_FCLOSE(pState->m_pFile) == EOF)\n            {\n                if (set_last_error)\n                    mz_zip_set_error(pZip, MZ_ZIP_FILE_CLOSE_FAILED);\n                status = MZ_FALSE;\n            }\n        }\n\n        pState->m_pFile = NULL;\n    }\n#endif /* #ifndef MINIZ_NO_STDIO */\n\n    if ((pZip->m_pWrite == mz_zip_heap_write_func) && (pState->m_pMem))\n    {\n        pZip->m_pFree(pZip->m_pAlloc_opaque, pState->m_pMem);\n        pState->m_pMem = NULL;\n    }\n\n    pZip->m_pFree(pZip->m_pAlloc_opaque, pState);\n    pZip->m_zip_mode = MZ_ZIP_MODE_INVALID;\n    return status;\n}\n\nmz_bool mz_zip_writer_init_v2(mz_zip_archive *pZip, mz_uint64 existing_size, mz_uint flags)\n{\n    mz_bool zip64 = (flags & MZ_ZIP_FLAG_WRITE_ZIP64) != 0;\n\n    if ((!pZip) || (pZip->m_pState) || (!pZip->m_pWrite) || (pZip->m_zip_mode != MZ_ZIP_MODE_INVALID))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    if (flags & MZ_ZIP_FLAG_WRITE_ALLOW_READING)\n    {\n        if (!pZip->m_pRead)\n            return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n    }\n\n    if (pZip->m_file_offset_alignment)\n    {\n        /* Ensure user specified file offset alignment is a power of 2. */\n        if (pZip->m_file_offset_alignment & (pZip->m_file_offset_alignment - 1))\n            return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n    }\n\n    if (!pZip->m_pAlloc)\n        pZip->m_pAlloc = miniz_def_alloc_func;\n    if (!pZip->m_pFree)\n        pZip->m_pFree = miniz_def_free_func;\n    if (!pZip->m_pRealloc)\n        pZip->m_pRealloc = miniz_def_realloc_func;\n\n    pZip->m_archive_size = existing_size;\n    pZip->m_central_directory_file_ofs = 0;\n    pZip->m_total_files = 0;\n\n    if (NULL == (pZip->m_pState = (mz_zip_internal_state *)pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, sizeof(mz_zip_internal_state))))\n        return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n\n    memset(pZip->m_pState, 0, sizeof(mz_zip_internal_state));\n\n    MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_central_dir, sizeof(mz_uint8));\n    MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_central_dir_offsets, sizeof(mz_uint32));\n    MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_sorted_central_dir_offsets, sizeof(mz_uint32));\n\n    pZip->m_pState->m_zip64 = zip64;\n    pZip->m_pState->m_zip64_has_extended_info_fields = zip64;\n\n    pZip->m_zip_type = MZ_ZIP_TYPE_USER;\n    pZip->m_zip_mode = MZ_ZIP_MODE_WRITING;\n\n    return MZ_TRUE;\n}\n\nmz_bool mz_zip_writer_init(mz_zip_archive *pZip, mz_uint64 existing_size)\n{\n    return mz_zip_writer_init_v2(pZip, existing_size, 0);\n}\n\nmz_bool mz_zip_writer_init_heap_v2(mz_zip_archive *pZip, size_t size_to_reserve_at_beginning, size_t initial_allocation_size, mz_uint flags)\n{\n    pZip->m_pWrite = mz_zip_heap_write_func;\n    pZip->m_pNeeds_keepalive = NULL;\n\n    if (flags & MZ_ZIP_FLAG_WRITE_ALLOW_READING)\n        pZip->m_pRead = mz_zip_mem_read_func;\n\n    pZip->m_pIO_opaque = pZip;\n\n    if (!mz_zip_writer_init_v2(pZip, size_to_reserve_at_beginning, flags))\n        return MZ_FALSE;\n\n    pZip->m_zip_type = MZ_ZIP_TYPE_HEAP;\n\n    if (0 != (initial_allocation_size = MZ_MAX(initial_allocation_size, size_to_reserve_at_beginning)))\n    {\n        if (NULL == (pZip->m_pState->m_pMem = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, initial_allocation_size)))\n        {\n            mz_zip_writer_end_internal(pZip, MZ_FALSE);\n            return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n        }\n        pZip->m_pState->m_mem_capacity = initial_allocation_size;\n    }\n\n    return MZ_TRUE;\n}\n\nmz_bool mz_zip_writer_init_heap(mz_zip_archive *pZip, size_t size_to_reserve_at_beginning, size_t initial_allocation_size)\n{\n    return mz_zip_writer_init_heap_v2(pZip, size_to_reserve_at_beginning, initial_allocation_size, 0);\n}\n\n#ifndef MINIZ_NO_STDIO\nstatic size_t mz_zip_file_write_func(void *pOpaque, mz_uint64 file_ofs, const void *pBuf, size_t n)\n{\n    mz_zip_archive *pZip = (mz_zip_archive *)pOpaque;\n    mz_int64 cur_ofs = MZ_FTELL64(pZip->m_pState->m_pFile);\n\n    file_ofs += pZip->m_pState->m_file_archive_start_ofs;\n\n    if (((mz_int64)file_ofs < 0) || (((cur_ofs != (mz_int64)file_ofs)) && (MZ_FSEEK64(pZip->m_pState->m_pFile, (mz_int64)file_ofs, SEEK_SET))))\n    {\n        mz_zip_set_error(pZip, MZ_ZIP_FILE_SEEK_FAILED);\n        return 0;\n    }\n\n    return MZ_FWRITE(pBuf, 1, n, pZip->m_pState->m_pFile);\n}\n\nmz_bool mz_zip_writer_init_file(mz_zip_archive *pZip, const char *pFilename, mz_uint64 size_to_reserve_at_beginning)\n{\n    return mz_zip_writer_init_file_v2(pZip, pFilename, size_to_reserve_at_beginning, 0);\n}\n\nmz_bool mz_zip_writer_init_file_v2(mz_zip_archive *pZip, const char *pFilename, mz_uint64 size_to_reserve_at_beginning, mz_uint flags)\n{\n    MZ_FILE *pFile;\n\n    pZip->m_pWrite = mz_zip_file_write_func;\n    pZip->m_pNeeds_keepalive = NULL;\n\n    if (flags & MZ_ZIP_FLAG_WRITE_ALLOW_READING)\n        pZip->m_pRead = mz_zip_file_read_func;\n\n    pZip->m_pIO_opaque = pZip;\n\n    if (!mz_zip_writer_init_v2(pZip, size_to_reserve_at_beginning, flags))\n        return MZ_FALSE;\n\n    if (NULL == (pFile = MZ_FOPEN(pFilename, (flags & MZ_ZIP_FLAG_WRITE_ALLOW_READING) ? \"w+b\" : \"wb\")))\n    {\n        mz_zip_writer_end(pZip);\n        return mz_zip_set_error(pZip, MZ_ZIP_FILE_OPEN_FAILED);\n    }\n\n    pZip->m_pState->m_pFile = pFile;\n    pZip->m_zip_type = MZ_ZIP_TYPE_FILE;\n\n    if (size_to_reserve_at_beginning)\n    {\n        mz_uint64 cur_ofs = 0;\n        char buf[4096];\n\n        MZ_CLEAR_OBJ(buf);\n\n        do\n        {\n            size_t n = (size_t)MZ_MIN(sizeof(buf), size_to_reserve_at_beginning);\n            if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_ofs, buf, n) != n)\n            {\n                mz_zip_writer_end(pZip);\n                return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n            }\n            cur_ofs += n;\n            size_to_reserve_at_beginning -= n;\n        } while (size_to_reserve_at_beginning);\n    }\n\n    return MZ_TRUE;\n}\n\nmz_bool mz_zip_writer_init_cfile(mz_zip_archive *pZip, MZ_FILE *pFile, mz_uint flags)\n{\n    pZip->m_pWrite = mz_zip_file_write_func;\n    pZip->m_pNeeds_keepalive = NULL;\n\n    if (flags & MZ_ZIP_FLAG_WRITE_ALLOW_READING)\n        pZip->m_pRead = mz_zip_file_read_func;\n\n    pZip->m_pIO_opaque = pZip;\n\n    if (!mz_zip_writer_init_v2(pZip, 0, flags))\n        return MZ_FALSE;\n\n    pZip->m_pState->m_pFile = pFile;\n    pZip->m_pState->m_file_archive_start_ofs = MZ_FTELL64(pZip->m_pState->m_pFile);\n    pZip->m_zip_type = MZ_ZIP_TYPE_CFILE;\n\n    return MZ_TRUE;\n}\n#endif /* #ifndef MINIZ_NO_STDIO */\n\nmz_bool mz_zip_writer_init_from_reader_v2(mz_zip_archive *pZip, const char *pFilename, mz_uint flags)\n{\n    mz_zip_internal_state *pState;\n\n    if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_READING))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    if (flags & MZ_ZIP_FLAG_WRITE_ZIP64)\n    {\n        /* We don't support converting a non-zip64 file to zip64 - this seems like more trouble than it's worth. (What about the existing 32-bit data descriptors that could follow the compressed data?) */\n        if (!pZip->m_pState->m_zip64)\n            return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n    }\n\n    /* No sense in trying to write to an archive that's already at the support max size */\n    if (pZip->m_pState->m_zip64)\n    {\n        if (pZip->m_total_files == MZ_UINT32_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_TOO_MANY_FILES);\n    }\n    else\n    {\n        if (pZip->m_total_files == MZ_UINT16_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_TOO_MANY_FILES);\n\n        if ((pZip->m_archive_size + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + MZ_ZIP_LOCAL_DIR_HEADER_SIZE) > MZ_UINT32_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_TOO_LARGE);\n    }\n\n    pState = pZip->m_pState;\n\n    if (pState->m_pFile)\n    {\n#ifdef MINIZ_NO_STDIO\n        (void)pFilename;\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n#else\n        if (pZip->m_pIO_opaque != pZip)\n            return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n        if (pZip->m_zip_type == MZ_ZIP_TYPE_FILE)\n        {\n            if (!pFilename)\n                return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n            /* Archive is being read from stdio and was originally opened only for reading. Try to reopen as writable. */\n            if (NULL == (pState->m_pFile = MZ_FREOPEN(pFilename, \"r+b\", pState->m_pFile)))\n            {\n                /* The mz_zip_archive is now in a bogus state because pState->m_pFile is NULL, so just close it. */\n                mz_zip_reader_end_internal(pZip, MZ_FALSE);\n                return mz_zip_set_error(pZip, MZ_ZIP_FILE_OPEN_FAILED);\n            }\n        }\n\n        pZip->m_pWrite = mz_zip_file_write_func;\n        pZip->m_pNeeds_keepalive = NULL;\n#endif /* #ifdef MINIZ_NO_STDIO */\n    }\n    else if (pState->m_pMem)\n    {\n        /* Archive lives in a memory block. Assume it's from the heap that we can resize using the realloc callback. */\n        if (pZip->m_pIO_opaque != pZip)\n            return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n        pState->m_mem_capacity = pState->m_mem_size;\n        pZip->m_pWrite = mz_zip_heap_write_func;\n        pZip->m_pNeeds_keepalive = NULL;\n    }\n    /* Archive is being read via a user provided read function - make sure the user has specified a write function too. */\n    else if (!pZip->m_pWrite)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    /* Start writing new files at the archive's current central directory location. */\n    /* TODO: We could add a flag that lets the user start writing immediately AFTER the existing central dir - this would be safer. */\n    pZip->m_archive_size = pZip->m_central_directory_file_ofs;\n    pZip->m_central_directory_file_ofs = 0;\n\n    /* Clear the sorted central dir offsets, they aren't useful or maintained now. */\n    /* Even though we're now in write mode, files can still be extracted and verified, but file locates will be slow. */\n    /* TODO: We could easily maintain the sorted central directory offsets. */\n    mz_zip_array_clear(pZip, &pZip->m_pState->m_sorted_central_dir_offsets);\n\n    pZip->m_zip_mode = MZ_ZIP_MODE_WRITING;\n\n    return MZ_TRUE;\n}\n\nmz_bool mz_zip_writer_init_from_reader(mz_zip_archive *pZip, const char *pFilename)\n{\n    return mz_zip_writer_init_from_reader_v2(pZip, pFilename, 0);\n}\n\n/* TODO: pArchive_name is a terrible name here! */\nmz_bool mz_zip_writer_add_mem(mz_zip_archive *pZip, const char *pArchive_name, const void *pBuf, size_t buf_size, mz_uint level_and_flags)\n{\n    return mz_zip_writer_add_mem_ex(pZip, pArchive_name, pBuf, buf_size, NULL, 0, level_and_flags, 0, 0);\n}\n\ntypedef struct\n{\n    mz_zip_archive *m_pZip;\n    mz_uint64 m_cur_archive_file_ofs;\n    mz_uint64 m_comp_size;\n} mz_zip_writer_add_state;\n\nstatic mz_bool mz_zip_writer_add_put_buf_callback(const void *pBuf, int len, void *pUser)\n{\n    mz_zip_writer_add_state *pState = (mz_zip_writer_add_state *)pUser;\n    if ((int)pState->m_pZip->m_pWrite(pState->m_pZip->m_pIO_opaque, pState->m_cur_archive_file_ofs, pBuf, len) != len)\n        return MZ_FALSE;\n\n    pState->m_cur_archive_file_ofs += len;\n    pState->m_comp_size += len;\n    return MZ_TRUE;\n}\n\n#define MZ_ZIP64_MAX_LOCAL_EXTRA_FIELD_SIZE (sizeof(mz_uint16) * 2 + sizeof(mz_uint64) * 2)\n#define MZ_ZIP64_MAX_CENTRAL_EXTRA_FIELD_SIZE (sizeof(mz_uint16) * 2 + sizeof(mz_uint64) * 3)\nstatic mz_uint32 mz_zip_writer_create_zip64_extra_data(mz_uint8 *pBuf, mz_uint64 *pUncomp_size, mz_uint64 *pComp_size, mz_uint64 *pLocal_header_ofs)\n{\n    mz_uint8 *pDst = pBuf;\n    mz_uint32 field_size = 0;\n\n    MZ_WRITE_LE16(pDst + 0, MZ_ZIP64_EXTENDED_INFORMATION_FIELD_HEADER_ID);\n    MZ_WRITE_LE16(pDst + 2, 0);\n    pDst += sizeof(mz_uint16) * 2;\n\n    if (pUncomp_size)\n    {\n        MZ_WRITE_LE64(pDst, *pUncomp_size);\n        pDst += sizeof(mz_uint64);\n        field_size += sizeof(mz_uint64);\n    }\n\n    if (pComp_size)\n    {\n        MZ_WRITE_LE64(pDst, *pComp_size);\n        pDst += sizeof(mz_uint64);\n        field_size += sizeof(mz_uint64);\n    }\n\n    if (pLocal_header_ofs)\n    {\n        MZ_WRITE_LE64(pDst, *pLocal_header_ofs);\n        pDst += sizeof(mz_uint64);\n        field_size += sizeof(mz_uint64);\n    }\n\n    MZ_WRITE_LE16(pBuf + 2, field_size);\n\n    return (mz_uint32)(pDst - pBuf);\n}\n\nstatic mz_bool mz_zip_writer_create_local_dir_header(mz_zip_archive *pZip, mz_uint8 *pDst, mz_uint16 filename_size, mz_uint16 extra_size, mz_uint64 uncomp_size, mz_uint64 comp_size, mz_uint32 uncomp_crc32, mz_uint16 method, mz_uint16 bit_flags, mz_uint16 dos_time, mz_uint16 dos_date)\n{\n    (void)pZip;\n    memset(pDst, 0, MZ_ZIP_LOCAL_DIR_HEADER_SIZE);\n    MZ_WRITE_LE32(pDst + MZ_ZIP_LDH_SIG_OFS, MZ_ZIP_LOCAL_DIR_HEADER_SIG);\n    MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_VERSION_NEEDED_OFS, method ? 20 : 0);\n    MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_BIT_FLAG_OFS, bit_flags);\n    MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_METHOD_OFS, method);\n    MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_FILE_TIME_OFS, dos_time);\n    MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_FILE_DATE_OFS, dos_date);\n    MZ_WRITE_LE32(pDst + MZ_ZIP_LDH_CRC32_OFS, uncomp_crc32);\n    MZ_WRITE_LE32(pDst + MZ_ZIP_LDH_COMPRESSED_SIZE_OFS, MZ_MIN(comp_size, MZ_UINT32_MAX));\n    MZ_WRITE_LE32(pDst + MZ_ZIP_LDH_DECOMPRESSED_SIZE_OFS, MZ_MIN(uncomp_size, MZ_UINT32_MAX));\n    MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_FILENAME_LEN_OFS, filename_size);\n    MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_EXTRA_LEN_OFS, extra_size);\n    return MZ_TRUE;\n}\n\nstatic mz_bool mz_zip_writer_create_central_dir_header(mz_zip_archive *pZip, mz_uint8 *pDst,\n                                                       mz_uint16 filename_size, mz_uint16 extra_size, mz_uint16 comment_size,\n                                                       mz_uint64 uncomp_size, mz_uint64 comp_size, mz_uint32 uncomp_crc32,\n                                                       mz_uint16 method, mz_uint16 bit_flags, mz_uint16 dos_time, mz_uint16 dos_date,\n                                                       mz_uint64 local_header_ofs, mz_uint32 ext_attributes)\n{\n    (void)pZip;\n    memset(pDst, 0, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE);\n    MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_SIG_OFS, MZ_ZIP_CENTRAL_DIR_HEADER_SIG);\n    MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_VERSION_NEEDED_OFS, method ? 20 : 0);\n    MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_BIT_FLAG_OFS, bit_flags);\n    MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_METHOD_OFS, method);\n    MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_FILE_TIME_OFS, dos_time);\n    MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_FILE_DATE_OFS, dos_date);\n    MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_CRC32_OFS, uncomp_crc32);\n    MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_COMPRESSED_SIZE_OFS, MZ_MIN(comp_size, MZ_UINT32_MAX));\n    MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS, MZ_MIN(uncomp_size, MZ_UINT32_MAX));\n    MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_FILENAME_LEN_OFS, filename_size);\n    MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_EXTRA_LEN_OFS, extra_size);\n    MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_COMMENT_LEN_OFS, comment_size);\n    MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_EXTERNAL_ATTR_OFS, ext_attributes);\n    MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_LOCAL_HEADER_OFS, MZ_MIN(local_header_ofs, MZ_UINT32_MAX));\n    return MZ_TRUE;\n}\n\nstatic mz_bool mz_zip_writer_add_to_central_dir(mz_zip_archive *pZip, const char *pFilename, mz_uint16 filename_size,\n                                                const void *pExtra, mz_uint16 extra_size, const void *pComment, mz_uint16 comment_size,\n                                                mz_uint64 uncomp_size, mz_uint64 comp_size, mz_uint32 uncomp_crc32,\n                                                mz_uint16 method, mz_uint16 bit_flags, mz_uint16 dos_time, mz_uint16 dos_date,\n                                                mz_uint64 local_header_ofs, mz_uint32 ext_attributes,\n                                                const char *user_extra_data, mz_uint user_extra_data_len)\n{\n    mz_zip_internal_state *pState = pZip->m_pState;\n    mz_uint32 central_dir_ofs = (mz_uint32)pState->m_central_dir.m_size;\n    size_t orig_central_dir_size = pState->m_central_dir.m_size;\n    mz_uint8 central_dir_header[MZ_ZIP_CENTRAL_DIR_HEADER_SIZE];\n\n    if (!pZip->m_pState->m_zip64)\n    {\n        if (local_header_ofs > 0xFFFFFFFF)\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_TOO_LARGE);\n    }\n\n    /* miniz doesn't support central dirs >= MZ_UINT32_MAX bytes yet */\n    if (((mz_uint64)pState->m_central_dir.m_size + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + filename_size + extra_size + user_extra_data_len + comment_size) >= MZ_UINT32_MAX)\n        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_CDIR_SIZE);\n\n    if (!mz_zip_writer_create_central_dir_header(pZip, central_dir_header, filename_size, (mz_uint16)(extra_size + user_extra_data_len), comment_size, uncomp_size, comp_size, uncomp_crc32, method, bit_flags, dos_time, dos_date, local_header_ofs, ext_attributes))\n        return mz_zip_set_error(pZip, MZ_ZIP_INTERNAL_ERROR);\n\n    if ((!mz_zip_array_push_back(pZip, &pState->m_central_dir, central_dir_header, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE)) ||\n        (!mz_zip_array_push_back(pZip, &pState->m_central_dir, pFilename, filename_size)) ||\n        (!mz_zip_array_push_back(pZip, &pState->m_central_dir, pExtra, extra_size)) ||\n        (!mz_zip_array_push_back(pZip, &pState->m_central_dir, user_extra_data, user_extra_data_len)) ||\n        (!mz_zip_array_push_back(pZip, &pState->m_central_dir, pComment, comment_size)) ||\n        (!mz_zip_array_push_back(pZip, &pState->m_central_dir_offsets, &central_dir_ofs, 1)))\n    {\n        /* Try to resize the central directory array back into its original state. */\n        mz_zip_array_resize(pZip, &pState->m_central_dir, orig_central_dir_size, MZ_FALSE);\n        return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n    }\n\n    return MZ_TRUE;\n}\n\nstatic mz_bool mz_zip_writer_validate_archive_name(const char *pArchive_name)\n{\n    /* Basic ZIP archive filename validity checks: Valid filenames cannot start with a forward slash, cannot contain a drive letter, and cannot use DOS-style backward slashes. */\n    if (*pArchive_name == '/')\n        return MZ_FALSE;\n\n    /* Making sure the name does not contain drive letters or DOS style backward slashes is the responsibility of the program using miniz*/\n\n    return MZ_TRUE;\n}\n\nstatic mz_uint mz_zip_writer_compute_padding_needed_for_file_alignment(mz_zip_archive *pZip)\n{\n    mz_uint32 n;\n    if (!pZip->m_file_offset_alignment)\n        return 0;\n    n = (mz_uint32)(pZip->m_archive_size & (pZip->m_file_offset_alignment - 1));\n    return (mz_uint)((pZip->m_file_offset_alignment - n) & (pZip->m_file_offset_alignment - 1));\n}\n\nstatic mz_bool mz_zip_writer_write_zeros(mz_zip_archive *pZip, mz_uint64 cur_file_ofs, mz_uint32 n)\n{\n    char buf[4096];\n    memset(buf, 0, MZ_MIN(sizeof(buf), n));\n    while (n)\n    {\n        mz_uint32 s = MZ_MIN(sizeof(buf), n);\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_file_ofs, buf, s) != s)\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n\n        cur_file_ofs += s;\n        n -= s;\n    }\n    return MZ_TRUE;\n}\n\nmz_bool mz_zip_writer_add_mem_ex(mz_zip_archive *pZip, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags,\n                                 mz_uint64 uncomp_size, mz_uint32 uncomp_crc32)\n{\n    return mz_zip_writer_add_mem_ex_v2(pZip, pArchive_name, pBuf, buf_size, pComment, comment_size, level_and_flags, uncomp_size, uncomp_crc32, NULL, NULL, 0, NULL, 0);\n}\n\nmz_bool mz_zip_writer_add_mem_ex_v2(mz_zip_archive *pZip, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size,\n                                    mz_uint level_and_flags, mz_uint64 uncomp_size, mz_uint32 uncomp_crc32, MZ_TIME_T *last_modified,\n                                    const char *user_extra_data, mz_uint user_extra_data_len, const char *user_extra_data_central, mz_uint user_extra_data_central_len)\n{\n    mz_uint16 method = 0, dos_time = 0, dos_date = 0;\n    mz_uint level, ext_attributes = 0, num_alignment_padding_bytes;\n    mz_uint64 local_dir_header_ofs = pZip->m_archive_size, cur_archive_file_ofs = pZip->m_archive_size, comp_size = 0;\n    size_t archive_name_size;\n    mz_uint8 local_dir_header[MZ_ZIP_LOCAL_DIR_HEADER_SIZE];\n    tdefl_compressor *pComp = NULL;\n    mz_bool store_data_uncompressed;\n    mz_zip_internal_state *pState;\n    mz_uint8 *pExtra_data = NULL;\n    mz_uint32 extra_size = 0;\n    mz_uint8 extra_data[MZ_ZIP64_MAX_CENTRAL_EXTRA_FIELD_SIZE];\n    mz_uint16 bit_flags = 0;\n\n    if ((int)level_and_flags < 0)\n        level_and_flags = MZ_DEFAULT_LEVEL;\n\n    if (uncomp_size || (buf_size && !(level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA)))\n        bit_flags |= MZ_ZIP_LDH_BIT_FLAG_HAS_LOCATOR;\n\n    if (!(level_and_flags & MZ_ZIP_FLAG_ASCII_FILENAME))\n        bit_flags |= MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_UTF8;\n\n    level = level_and_flags & 0xF;\n    store_data_uncompressed = ((!level) || (level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA));\n\n    if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING) || ((buf_size) && (!pBuf)) || (!pArchive_name) || ((comment_size) && (!pComment)) || (level > MZ_UBER_COMPRESSION))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    pState = pZip->m_pState;\n\n    if (pState->m_zip64)\n    {\n        if (pZip->m_total_files == MZ_UINT32_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_TOO_MANY_FILES);\n    }\n    else\n    {\n        if (pZip->m_total_files == MZ_UINT16_MAX)\n        {\n            pState->m_zip64 = MZ_TRUE;\n            /*return mz_zip_set_error(pZip, MZ_ZIP_TOO_MANY_FILES); */\n        }\n        if ((buf_size > 0xFFFFFFFF) || (uncomp_size > 0xFFFFFFFF))\n        {\n            pState->m_zip64 = MZ_TRUE;\n            /*return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE); */\n        }\n    }\n\n    if ((!(level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA)) && (uncomp_size))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    if (!mz_zip_writer_validate_archive_name(pArchive_name))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_FILENAME);\n\n#ifndef MINIZ_NO_TIME\n    if (last_modified != NULL)\n    {\n        mz_zip_time_t_to_dos_time(*last_modified, &dos_time, &dos_date);\n    }\n    else\n    {\n        MZ_TIME_T cur_time;\n        time(&cur_time);\n        mz_zip_time_t_to_dos_time(cur_time, &dos_time, &dos_date);\n    }\n#endif /* #ifndef MINIZ_NO_TIME */\n\n\tif (!(level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA))\n\t{\n\t\tuncomp_crc32 = (mz_uint32)mz_crc32(MZ_CRC32_INIT, (const mz_uint8 *)pBuf, buf_size);\n\t\tuncomp_size = buf_size;\n\t\tif (uncomp_size <= 3)\n\t\t{\n\t\t\tlevel = 0;\n\t\t\tstore_data_uncompressed = MZ_TRUE;\n\t\t}\n\t}\n\n    archive_name_size = strlen(pArchive_name);\n    if (archive_name_size > MZ_UINT16_MAX)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_FILENAME);\n\n    num_alignment_padding_bytes = mz_zip_writer_compute_padding_needed_for_file_alignment(pZip);\n\n    /* miniz doesn't support central dirs >= MZ_UINT32_MAX bytes yet */\n    if (((mz_uint64)pState->m_central_dir.m_size + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + archive_name_size + MZ_ZIP64_MAX_CENTRAL_EXTRA_FIELD_SIZE + comment_size) >= MZ_UINT32_MAX)\n        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_CDIR_SIZE);\n\n    if (!pState->m_zip64)\n    {\n        /* Bail early if the archive would obviously become too large */\n        if ((pZip->m_archive_size + num_alignment_padding_bytes + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + archive_name_size \n\t\t\t+ MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + archive_name_size + comment_size + user_extra_data_len + \n\t\t\tpState->m_central_dir.m_size + MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE + user_extra_data_central_len\n\t\t\t+ MZ_ZIP_DATA_DESCRIPTER_SIZE32) > 0xFFFFFFFF)\n        {\n            pState->m_zip64 = MZ_TRUE;\n            /*return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE); */\n        }\n    }\n\n    if ((archive_name_size) && (pArchive_name[archive_name_size - 1] == '/'))\n    {\n        /* Set DOS Subdirectory attribute bit. */\n        ext_attributes |= MZ_ZIP_DOS_DIR_ATTRIBUTE_BITFLAG;\n\n        /* Subdirectories cannot contain data. */\n        if ((buf_size) || (uncomp_size))\n            return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n    }\n\n    /* Try to do any allocations before writing to the archive, so if an allocation fails the file remains unmodified. (A good idea if we're doing an in-place modification.) */\n    if ((!mz_zip_array_ensure_room(pZip, &pState->m_central_dir, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + archive_name_size + comment_size + (pState->m_zip64 ? MZ_ZIP64_MAX_CENTRAL_EXTRA_FIELD_SIZE : 0))) || (!mz_zip_array_ensure_room(pZip, &pState->m_central_dir_offsets, 1)))\n        return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n\n    if ((!store_data_uncompressed) && (buf_size))\n    {\n        if (NULL == (pComp = (tdefl_compressor *)pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, sizeof(tdefl_compressor))))\n            return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n    }\n\n    if (!mz_zip_writer_write_zeros(pZip, cur_archive_file_ofs, num_alignment_padding_bytes))\n    {\n        pZip->m_pFree(pZip->m_pAlloc_opaque, pComp);\n        return MZ_FALSE;\n    }\n\n    local_dir_header_ofs += num_alignment_padding_bytes;\n    if (pZip->m_file_offset_alignment)\n    {\n        MZ_ASSERT((local_dir_header_ofs & (pZip->m_file_offset_alignment - 1)) == 0);\n    }\n    cur_archive_file_ofs += num_alignment_padding_bytes;\n\n    MZ_CLEAR_OBJ(local_dir_header);\n\n    if (!store_data_uncompressed || (level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA))\n    {\n        method = MZ_DEFLATED;\n    }\n\n    if (pState->m_zip64)\n    {\n        if (uncomp_size >= MZ_UINT32_MAX || local_dir_header_ofs >= MZ_UINT32_MAX)\n        {\n            pExtra_data = extra_data;\n            extra_size = mz_zip_writer_create_zip64_extra_data(extra_data, (uncomp_size >= MZ_UINT32_MAX) ? &uncomp_size : NULL,\n                                                               (uncomp_size >= MZ_UINT32_MAX) ? &comp_size : NULL, (local_dir_header_ofs >= MZ_UINT32_MAX) ? &local_dir_header_ofs : NULL);\n        }\n\n        if (!mz_zip_writer_create_local_dir_header(pZip, local_dir_header, (mz_uint16)archive_name_size, (mz_uint16)(extra_size + user_extra_data_len), 0, 0, 0, method, bit_flags, dos_time, dos_date))\n            return mz_zip_set_error(pZip, MZ_ZIP_INTERNAL_ERROR);\n\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, local_dir_header_ofs, local_dir_header, sizeof(local_dir_header)) != sizeof(local_dir_header))\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n\n        cur_archive_file_ofs += sizeof(local_dir_header);\n\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, pArchive_name, archive_name_size) != archive_name_size)\n        {\n            pZip->m_pFree(pZip->m_pAlloc_opaque, pComp);\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n        }\n        cur_archive_file_ofs += archive_name_size;\n\n        if (pExtra_data != NULL)\n        {\n            if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, extra_data, extra_size) != extra_size)\n                return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n\n            cur_archive_file_ofs += extra_size;\n        }\n    }\n    else\n    {\n        if ((comp_size > MZ_UINT32_MAX) || (cur_archive_file_ofs > MZ_UINT32_MAX))\n            return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE);\n        if (!mz_zip_writer_create_local_dir_header(pZip, local_dir_header, (mz_uint16)archive_name_size, (mz_uint16)user_extra_data_len, 0, 0, 0, method, bit_flags, dos_time, dos_date))\n            return mz_zip_set_error(pZip, MZ_ZIP_INTERNAL_ERROR);\n\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, local_dir_header_ofs, local_dir_header, sizeof(local_dir_header)) != sizeof(local_dir_header))\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n\n        cur_archive_file_ofs += sizeof(local_dir_header);\n\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, pArchive_name, archive_name_size) != archive_name_size)\n        {\n            pZip->m_pFree(pZip->m_pAlloc_opaque, pComp);\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n        }\n        cur_archive_file_ofs += archive_name_size;\n    }\n\n\tif (user_extra_data_len > 0)\n\t{\n\t\tif (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, user_extra_data, user_extra_data_len) != user_extra_data_len)\n\t\t\treturn mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n\n\t\tcur_archive_file_ofs += user_extra_data_len;\n\t}\n\n    if (store_data_uncompressed)\n    {\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, pBuf, buf_size) != buf_size)\n        {\n            pZip->m_pFree(pZip->m_pAlloc_opaque, pComp);\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n        }\n\n        cur_archive_file_ofs += buf_size;\n        comp_size = buf_size;\n    }\n    else if (buf_size)\n    {\n        mz_zip_writer_add_state state;\n\n        state.m_pZip = pZip;\n        state.m_cur_archive_file_ofs = cur_archive_file_ofs;\n        state.m_comp_size = 0;\n\n        if ((tdefl_init(pComp, mz_zip_writer_add_put_buf_callback, &state, tdefl_create_comp_flags_from_zip_params(level, -15, MZ_DEFAULT_STRATEGY)) != TDEFL_STATUS_OKAY) ||\n            (tdefl_compress_buffer(pComp, pBuf, buf_size, TDEFL_FINISH) != TDEFL_STATUS_DONE))\n        {\n            pZip->m_pFree(pZip->m_pAlloc_opaque, pComp);\n            return mz_zip_set_error(pZip, MZ_ZIP_COMPRESSION_FAILED);\n        }\n\n        comp_size = state.m_comp_size;\n        cur_archive_file_ofs = state.m_cur_archive_file_ofs;\n    }\n\n    pZip->m_pFree(pZip->m_pAlloc_opaque, pComp);\n    pComp = NULL;\n\n    if (uncomp_size)\n    {\n        mz_uint8 local_dir_footer[MZ_ZIP_DATA_DESCRIPTER_SIZE64];\n        mz_uint32 local_dir_footer_size = MZ_ZIP_DATA_DESCRIPTER_SIZE32;\n\n        MZ_ASSERT(bit_flags & MZ_ZIP_LDH_BIT_FLAG_HAS_LOCATOR);\n\n        MZ_WRITE_LE32(local_dir_footer + 0, MZ_ZIP_DATA_DESCRIPTOR_ID);\n        MZ_WRITE_LE32(local_dir_footer + 4, uncomp_crc32);\n        if (pExtra_data == NULL)\n        {\n            if (comp_size > MZ_UINT32_MAX)\n                return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE);\n\n            MZ_WRITE_LE32(local_dir_footer + 8, comp_size);\n            MZ_WRITE_LE32(local_dir_footer + 12, uncomp_size);\n        }\n        else\n        {\n            MZ_WRITE_LE64(local_dir_footer + 8, comp_size);\n            MZ_WRITE_LE64(local_dir_footer + 16, uncomp_size);\n            local_dir_footer_size = MZ_ZIP_DATA_DESCRIPTER_SIZE64;\n        }\n\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, local_dir_footer, local_dir_footer_size) != local_dir_footer_size)\n            return MZ_FALSE;\n\n        cur_archive_file_ofs += local_dir_footer_size;\n    }\n\n    if (pExtra_data != NULL)\n    {\n        extra_size = mz_zip_writer_create_zip64_extra_data(extra_data, (uncomp_size >= MZ_UINT32_MAX) ? &uncomp_size : NULL,\n                                                           (uncomp_size >= MZ_UINT32_MAX) ? &comp_size : NULL, (local_dir_header_ofs >= MZ_UINT32_MAX) ? &local_dir_header_ofs : NULL);\n    }\n\n    if (!mz_zip_writer_add_to_central_dir(pZip, pArchive_name, (mz_uint16)archive_name_size, pExtra_data, (mz_uint16)extra_size, pComment,\n                                          comment_size, uncomp_size, comp_size, uncomp_crc32, method, bit_flags, dos_time, dos_date, local_dir_header_ofs, ext_attributes,\n                                          user_extra_data_central, user_extra_data_central_len))\n        return MZ_FALSE;\n\n    pZip->m_total_files++;\n    pZip->m_archive_size = cur_archive_file_ofs;\n\n    return MZ_TRUE;\n}\n\nmz_bool mz_zip_writer_add_read_buf_callback(mz_zip_archive *pZip, const char *pArchive_name, mz_file_read_func read_callback, void* callback_opaque, mz_uint64 size_to_add, const MZ_TIME_T *pFile_time, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags,\n                                const char *user_extra_data, mz_uint user_extra_data_len, const char *user_extra_data_central, mz_uint user_extra_data_central_len)\n{\n    mz_uint16 gen_flags = MZ_ZIP_LDH_BIT_FLAG_HAS_LOCATOR;\n    mz_uint uncomp_crc32 = MZ_CRC32_INIT, level, num_alignment_padding_bytes;\n    mz_uint16 method = 0, dos_time = 0, dos_date = 0, ext_attributes = 0;\n    mz_uint64 local_dir_header_ofs, cur_archive_file_ofs = pZip->m_archive_size, uncomp_size = size_to_add, comp_size = 0;\n    size_t archive_name_size;\n    mz_uint8 local_dir_header[MZ_ZIP_LOCAL_DIR_HEADER_SIZE];\n    mz_uint8 *pExtra_data = NULL;\n    mz_uint32 extra_size = 0;\n    mz_uint8 extra_data[MZ_ZIP64_MAX_CENTRAL_EXTRA_FIELD_SIZE];\n    mz_zip_internal_state *pState;\n\tmz_uint64 file_ofs = 0;\n\n    if (!(level_and_flags & MZ_ZIP_FLAG_ASCII_FILENAME))\n        gen_flags |= MZ_ZIP_GENERAL_PURPOSE_BIT_FLAG_UTF8;\n\n    if ((int)level_and_flags < 0)\n        level_and_flags = MZ_DEFAULT_LEVEL;\n    level = level_and_flags & 0xF;\n\n    /* Sanity checks */\n    if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING) || (!pArchive_name) || ((comment_size) && (!pComment)) || (level > MZ_UBER_COMPRESSION))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    pState = pZip->m_pState;\n\n    if ((!pState->m_zip64) && (uncomp_size > MZ_UINT32_MAX))\n    {\n        /* Source file is too large for non-zip64 */\n        /*return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE); */\n        pState->m_zip64 = MZ_TRUE;\n    }\n\n    /* We could support this, but why? */\n    if (level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    if (!mz_zip_writer_validate_archive_name(pArchive_name))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_FILENAME);\n\n    if (pState->m_zip64)\n    {\n        if (pZip->m_total_files == MZ_UINT32_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_TOO_MANY_FILES);\n    }\n    else\n    {\n        if (pZip->m_total_files == MZ_UINT16_MAX)\n        {\n            pState->m_zip64 = MZ_TRUE;\n            /*return mz_zip_set_error(pZip, MZ_ZIP_TOO_MANY_FILES); */\n        }\n    }\n\n    archive_name_size = strlen(pArchive_name);\n    if (archive_name_size > MZ_UINT16_MAX)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_FILENAME);\n\n    num_alignment_padding_bytes = mz_zip_writer_compute_padding_needed_for_file_alignment(pZip);\n\n    /* miniz doesn't support central dirs >= MZ_UINT32_MAX bytes yet */\n    if (((mz_uint64)pState->m_central_dir.m_size + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + archive_name_size + MZ_ZIP64_MAX_CENTRAL_EXTRA_FIELD_SIZE + comment_size) >= MZ_UINT32_MAX)\n        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_CDIR_SIZE);\n\n    if (!pState->m_zip64)\n    {\n        /* Bail early if the archive would obviously become too large */\n        if ((pZip->m_archive_size + num_alignment_padding_bytes + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + archive_name_size + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE\n\t\t\t+ archive_name_size + comment_size + user_extra_data_len + pState->m_central_dir.m_size + MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE + 1024\n\t\t\t+ MZ_ZIP_DATA_DESCRIPTER_SIZE32 + user_extra_data_central_len) > 0xFFFFFFFF)\n        {\n            pState->m_zip64 = MZ_TRUE;\n            /*return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE); */\n        }\n    }\n\n#ifndef MINIZ_NO_TIME\n    if (pFile_time)\n    {\n        mz_zip_time_t_to_dos_time(*pFile_time, &dos_time, &dos_date);\n    }\n#endif\n\n    if (uncomp_size <= 3)\n        level = 0;\n\n    if (!mz_zip_writer_write_zeros(pZip, cur_archive_file_ofs, num_alignment_padding_bytes))\n    {\n        return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n    }\n\n    cur_archive_file_ofs += num_alignment_padding_bytes;\n    local_dir_header_ofs = cur_archive_file_ofs;\n\n    if (pZip->m_file_offset_alignment)\n    {\n        MZ_ASSERT((cur_archive_file_ofs & (pZip->m_file_offset_alignment - 1)) == 0);\n    }\n\n    if (uncomp_size && level)\n    {\n        method = MZ_DEFLATED;\n    }\n\n    MZ_CLEAR_OBJ(local_dir_header);\n    if (pState->m_zip64)\n    {\n        if (uncomp_size >= MZ_UINT32_MAX || local_dir_header_ofs >= MZ_UINT32_MAX)\n        {\n            pExtra_data = extra_data;\n            extra_size = mz_zip_writer_create_zip64_extra_data(extra_data, (uncomp_size >= MZ_UINT32_MAX) ? &uncomp_size : NULL,\n                                                               (uncomp_size >= MZ_UINT32_MAX) ? &comp_size : NULL, (local_dir_header_ofs >= MZ_UINT32_MAX) ? &local_dir_header_ofs : NULL);\n        }\n\n        if (!mz_zip_writer_create_local_dir_header(pZip, local_dir_header, (mz_uint16)archive_name_size, (mz_uint16)(extra_size + user_extra_data_len), 0, 0, 0, method, gen_flags, dos_time, dos_date))\n            return mz_zip_set_error(pZip, MZ_ZIP_INTERNAL_ERROR);\n\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, local_dir_header, sizeof(local_dir_header)) != sizeof(local_dir_header))\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n\n        cur_archive_file_ofs += sizeof(local_dir_header);\n\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, pArchive_name, archive_name_size) != archive_name_size)\n        {\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n        }\n\n        cur_archive_file_ofs += archive_name_size;\n\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, extra_data, extra_size) != extra_size)\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n\n        cur_archive_file_ofs += extra_size;\n    }\n    else\n    {\n        if ((comp_size > MZ_UINT32_MAX) || (cur_archive_file_ofs > MZ_UINT32_MAX))\n            return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE);\n        if (!mz_zip_writer_create_local_dir_header(pZip, local_dir_header, (mz_uint16)archive_name_size, (mz_uint16)user_extra_data_len, 0, 0, 0, method, gen_flags, dos_time, dos_date))\n            return mz_zip_set_error(pZip, MZ_ZIP_INTERNAL_ERROR);\n\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, local_dir_header, sizeof(local_dir_header)) != sizeof(local_dir_header))\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n\n        cur_archive_file_ofs += sizeof(local_dir_header);\n\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, pArchive_name, archive_name_size) != archive_name_size)\n        {\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n        }\n\n        cur_archive_file_ofs += archive_name_size;\n    }\n\n    if (user_extra_data_len > 0)\n    {\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, user_extra_data, user_extra_data_len) != user_extra_data_len)\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n\n        cur_archive_file_ofs += user_extra_data_len;\n    }\n\n    if (uncomp_size)\n    {\n        mz_uint64 uncomp_remaining = uncomp_size;\n        void *pRead_buf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, MZ_ZIP_MAX_IO_BUF_SIZE);\n        if (!pRead_buf)\n        {\n            return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n        }\n\n        if (!level)\n        {\n            while (uncomp_remaining)\n            {\n                mz_uint n = (mz_uint)MZ_MIN((mz_uint64)MZ_ZIP_MAX_IO_BUF_SIZE, uncomp_remaining);\n                if ((read_callback(callback_opaque, file_ofs, pRead_buf, n) != n) || (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, pRead_buf, n) != n))\n                {\n                    pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf);\n                    return mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n                }\n\t\t\t\tfile_ofs += n;\n                uncomp_crc32 = (mz_uint32)mz_crc32(uncomp_crc32, (const mz_uint8 *)pRead_buf, n);\n                uncomp_remaining -= n;\n                cur_archive_file_ofs += n;\n            }\n            comp_size = uncomp_size;\n        }\n        else\n        {\n            mz_bool result = MZ_FALSE;\n            mz_zip_writer_add_state state;\n            tdefl_compressor *pComp = (tdefl_compressor *)pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, sizeof(tdefl_compressor));\n            if (!pComp)\n            {\n                pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf);\n                return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n            }\n\n            state.m_pZip = pZip;\n            state.m_cur_archive_file_ofs = cur_archive_file_ofs;\n            state.m_comp_size = 0;\n\n            if (tdefl_init(pComp, mz_zip_writer_add_put_buf_callback, &state, tdefl_create_comp_flags_from_zip_params(level, -15, MZ_DEFAULT_STRATEGY)) != TDEFL_STATUS_OKAY)\n            {\n                pZip->m_pFree(pZip->m_pAlloc_opaque, pComp);\n                pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf);\n                return mz_zip_set_error(pZip, MZ_ZIP_INTERNAL_ERROR);\n            }\n\n            for (;;)\n            {\n                size_t in_buf_size = (mz_uint32)MZ_MIN(uncomp_remaining, (mz_uint64)MZ_ZIP_MAX_IO_BUF_SIZE);\n                tdefl_status status;\n                tdefl_flush flush = TDEFL_NO_FLUSH;\n\n                if (read_callback(callback_opaque, file_ofs, pRead_buf, in_buf_size)!= in_buf_size)\n                {\n                    mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n                    break;\n                }\n\n\t\t\t\tfile_ofs += in_buf_size;\n                uncomp_crc32 = (mz_uint32)mz_crc32(uncomp_crc32, (const mz_uint8 *)pRead_buf, in_buf_size);\n                uncomp_remaining -= in_buf_size;\n\n                if (pZip->m_pNeeds_keepalive != NULL && pZip->m_pNeeds_keepalive(pZip->m_pIO_opaque))\n                    flush = TDEFL_FULL_FLUSH;\n\n                status = tdefl_compress_buffer(pComp, pRead_buf, in_buf_size, uncomp_remaining ? flush : TDEFL_FINISH);\n                if (status == TDEFL_STATUS_DONE)\n                {\n                    result = MZ_TRUE;\n                    break;\n                }\n                else if (status != TDEFL_STATUS_OKAY)\n                {\n                    mz_zip_set_error(pZip, MZ_ZIP_COMPRESSION_FAILED);\n                    break;\n                }\n            }\n\n            pZip->m_pFree(pZip->m_pAlloc_opaque, pComp);\n\n            if (!result)\n            {\n                pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf);\n                return MZ_FALSE;\n            }\n\n            comp_size = state.m_comp_size;\n            cur_archive_file_ofs = state.m_cur_archive_file_ofs;\n        }\n\n        pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf);\n    }\n\n    {\n        mz_uint8 local_dir_footer[MZ_ZIP_DATA_DESCRIPTER_SIZE64];\n        mz_uint32 local_dir_footer_size = MZ_ZIP_DATA_DESCRIPTER_SIZE32;\n\n        MZ_WRITE_LE32(local_dir_footer + 0, MZ_ZIP_DATA_DESCRIPTOR_ID);\n        MZ_WRITE_LE32(local_dir_footer + 4, uncomp_crc32);\n        if (pExtra_data == NULL)\n        {\n            if (comp_size > MZ_UINT32_MAX)\n                return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE);\n\n            MZ_WRITE_LE32(local_dir_footer + 8, comp_size);\n            MZ_WRITE_LE32(local_dir_footer + 12, uncomp_size);\n        }\n        else\n        {\n            MZ_WRITE_LE64(local_dir_footer + 8, comp_size);\n            MZ_WRITE_LE64(local_dir_footer + 16, uncomp_size);\n            local_dir_footer_size = MZ_ZIP_DATA_DESCRIPTER_SIZE64;\n        }\n\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, local_dir_footer, local_dir_footer_size) != local_dir_footer_size)\n            return MZ_FALSE;\n\n        cur_archive_file_ofs += local_dir_footer_size;\n    }\n\n    if (pExtra_data != NULL)\n    {\n        extra_size = mz_zip_writer_create_zip64_extra_data(extra_data, (uncomp_size >= MZ_UINT32_MAX) ? &uncomp_size : NULL,\n                                                           (uncomp_size >= MZ_UINT32_MAX) ? &comp_size : NULL, (local_dir_header_ofs >= MZ_UINT32_MAX) ? &local_dir_header_ofs : NULL);\n    }\n\n    if (!mz_zip_writer_add_to_central_dir(pZip, pArchive_name, (mz_uint16)archive_name_size, pExtra_data, (mz_uint16)extra_size, pComment, comment_size,\n                                          uncomp_size, comp_size, uncomp_crc32, method, gen_flags, dos_time, dos_date, local_dir_header_ofs, ext_attributes,\n                                          user_extra_data_central, user_extra_data_central_len))\n        return MZ_FALSE;\n\n    pZip->m_total_files++;\n    pZip->m_archive_size = cur_archive_file_ofs;\n\n    return MZ_TRUE;\n}\n\n#ifndef MINIZ_NO_STDIO\n\nstatic size_t mz_file_read_func_stdio(void *pOpaque, mz_uint64 file_ofs, void *pBuf, size_t n)\n{\n\tMZ_FILE *pSrc_file = (MZ_FILE *)pOpaque;\n\tmz_int64 cur_ofs = MZ_FTELL64(pSrc_file);\n\n\tif (((mz_int64)file_ofs < 0) || (((cur_ofs != (mz_int64)file_ofs)) && (MZ_FSEEK64(pSrc_file, (mz_int64)file_ofs, SEEK_SET))))\n\t\treturn 0;\n\n\treturn MZ_FREAD(pBuf, 1, n, pSrc_file);\n}\n\nmz_bool mz_zip_writer_add_cfile(mz_zip_archive *pZip, const char *pArchive_name, MZ_FILE *pSrc_file, mz_uint64 size_to_add, const MZ_TIME_T *pFile_time, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags,\n\tconst char *user_extra_data, mz_uint user_extra_data_len, const char *user_extra_data_central, mz_uint user_extra_data_central_len)\n{\n\treturn mz_zip_writer_add_read_buf_callback(pZip, pArchive_name, mz_file_read_func_stdio, pSrc_file, size_to_add, pFile_time, pComment, comment_size, level_and_flags,\n\t\tuser_extra_data, user_extra_data_len, user_extra_data_central, user_extra_data_central_len);\n}\n\nmz_bool mz_zip_writer_add_file(mz_zip_archive *pZip, const char *pArchive_name, const char *pSrc_filename, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags)\n{\n    MZ_FILE *pSrc_file = NULL;\n    mz_uint64 uncomp_size = 0;\n    MZ_TIME_T file_modified_time;\n    MZ_TIME_T *pFile_time = NULL;\n    mz_bool status;\n\n    memset(&file_modified_time, 0, sizeof(file_modified_time));\n\n#if !defined(MINIZ_NO_TIME) && !defined(MINIZ_NO_STDIO)\n    pFile_time = &file_modified_time;\n    if (!mz_zip_get_file_modified_time(pSrc_filename, &file_modified_time))\n        return mz_zip_set_error(pZip, MZ_ZIP_FILE_STAT_FAILED);\n#endif\n\n    pSrc_file = MZ_FOPEN(pSrc_filename, \"rb\");\n    if (!pSrc_file)\n        return mz_zip_set_error(pZip, MZ_ZIP_FILE_OPEN_FAILED);\n\n    MZ_FSEEK64(pSrc_file, 0, SEEK_END);\n    uncomp_size = MZ_FTELL64(pSrc_file);\n    MZ_FSEEK64(pSrc_file, 0, SEEK_SET);\n\n    status = mz_zip_writer_add_cfile(pZip, pArchive_name, pSrc_file, uncomp_size, pFile_time, pComment, comment_size, level_and_flags, NULL, 0, NULL, 0);\n\n    MZ_FCLOSE(pSrc_file);\n\n    return status;\n}\n#endif /* #ifndef MINIZ_NO_STDIO */\n\nstatic mz_bool mz_zip_writer_update_zip64_extension_block(mz_zip_array *pNew_ext, mz_zip_archive *pZip, const mz_uint8 *pExt, uint32_t ext_len, mz_uint64 *pComp_size, mz_uint64 *pUncomp_size, mz_uint64 *pLocal_header_ofs, mz_uint32 *pDisk_start)\n{\n    /* + 64 should be enough for any new zip64 data */\n    if (!mz_zip_array_reserve(pZip, pNew_ext, ext_len + 64, MZ_FALSE))\n        return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n\n    mz_zip_array_resize(pZip, pNew_ext, 0, MZ_FALSE);\n\n    if ((pUncomp_size) || (pComp_size) || (pLocal_header_ofs) || (pDisk_start))\n    {\n        mz_uint8 new_ext_block[64];\n        mz_uint8 *pDst = new_ext_block;\n        mz_write_le16(pDst, MZ_ZIP64_EXTENDED_INFORMATION_FIELD_HEADER_ID);\n        mz_write_le16(pDst + sizeof(mz_uint16), 0);\n        pDst += sizeof(mz_uint16) * 2;\n\n        if (pUncomp_size)\n        {\n            mz_write_le64(pDst, *pUncomp_size);\n            pDst += sizeof(mz_uint64);\n        }\n\n        if (pComp_size)\n        {\n            mz_write_le64(pDst, *pComp_size);\n            pDst += sizeof(mz_uint64);\n        }\n\n        if (pLocal_header_ofs)\n        {\n            mz_write_le64(pDst, *pLocal_header_ofs);\n            pDst += sizeof(mz_uint64);\n        }\n\n        if (pDisk_start)\n        {\n            mz_write_le32(pDst, *pDisk_start);\n            pDst += sizeof(mz_uint32);\n        }\n\n        mz_write_le16(new_ext_block + sizeof(mz_uint16), (mz_uint16)((pDst - new_ext_block) - sizeof(mz_uint16) * 2));\n\n        if (!mz_zip_array_push_back(pZip, pNew_ext, new_ext_block, pDst - new_ext_block))\n            return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n    }\n\n    if ((pExt) && (ext_len))\n    {\n        mz_uint32 extra_size_remaining = ext_len;\n        const mz_uint8 *pExtra_data = pExt;\n\n        do\n        {\n            mz_uint32 field_id, field_data_size, field_total_size;\n\n            if (extra_size_remaining < (sizeof(mz_uint16) * 2))\n                return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n            field_id = MZ_READ_LE16(pExtra_data);\n            field_data_size = MZ_READ_LE16(pExtra_data + sizeof(mz_uint16));\n            field_total_size = field_data_size + sizeof(mz_uint16) * 2;\n\n            if (field_total_size > extra_size_remaining)\n                return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n            if (field_id != MZ_ZIP64_EXTENDED_INFORMATION_FIELD_HEADER_ID)\n            {\n                if (!mz_zip_array_push_back(pZip, pNew_ext, pExtra_data, field_total_size))\n                    return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n            }\n\n            pExtra_data += field_total_size;\n            extra_size_remaining -= field_total_size;\n        } while (extra_size_remaining);\n    }\n\n    return MZ_TRUE;\n}\n\n/* TODO: This func is now pretty freakin complex due to zip64, split it up? */\nmz_bool mz_zip_writer_add_from_zip_reader(mz_zip_archive *pZip, mz_zip_archive *pSource_zip, mz_uint src_file_index)\n{\n    mz_uint n, bit_flags, num_alignment_padding_bytes, src_central_dir_following_data_size;\n    mz_uint64 src_archive_bytes_remaining, local_dir_header_ofs;\n    mz_uint64 cur_src_file_ofs, cur_dst_file_ofs;\n    mz_uint32 local_header_u32[(MZ_ZIP_LOCAL_DIR_HEADER_SIZE + sizeof(mz_uint32) - 1) / sizeof(mz_uint32)];\n    mz_uint8 *pLocal_header = (mz_uint8 *)local_header_u32;\n    mz_uint8 new_central_header[MZ_ZIP_CENTRAL_DIR_HEADER_SIZE];\n    size_t orig_central_dir_size;\n    mz_zip_internal_state *pState;\n    void *pBuf;\n    const mz_uint8 *pSrc_central_header;\n    mz_zip_archive_file_stat src_file_stat;\n    mz_uint32 src_filename_len, src_comment_len, src_ext_len;\n    mz_uint32 local_header_filename_size, local_header_extra_len;\n    mz_uint64 local_header_comp_size, local_header_uncomp_size;\n    mz_bool found_zip64_ext_data_in_ldir = MZ_FALSE;\n\n    /* Sanity checks */\n    if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING) || (!pSource_zip->m_pRead))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    pState = pZip->m_pState;\n\n    /* Don't support copying files from zip64 archives to non-zip64, even though in some cases this is possible */\n    if ((pSource_zip->m_pState->m_zip64) && (!pZip->m_pState->m_zip64))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    /* Get pointer to the source central dir header and crack it */\n    if (NULL == (pSrc_central_header = mz_zip_get_cdh(pSource_zip, src_file_index)))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    if (MZ_READ_LE32(pSrc_central_header + MZ_ZIP_CDH_SIG_OFS) != MZ_ZIP_CENTRAL_DIR_HEADER_SIG)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n    src_filename_len = MZ_READ_LE16(pSrc_central_header + MZ_ZIP_CDH_FILENAME_LEN_OFS);\n    src_comment_len = MZ_READ_LE16(pSrc_central_header + MZ_ZIP_CDH_COMMENT_LEN_OFS);\n    src_ext_len = MZ_READ_LE16(pSrc_central_header + MZ_ZIP_CDH_EXTRA_LEN_OFS);\n    src_central_dir_following_data_size = src_filename_len + src_ext_len + src_comment_len;\n\n    /* TODO: We don't support central dir's >= MZ_UINT32_MAX bytes right now (+32 fudge factor in case we need to add more extra data) */\n    if ((pState->m_central_dir.m_size + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + src_central_dir_following_data_size + 32) >= MZ_UINT32_MAX)\n        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_CDIR_SIZE);\n\n    num_alignment_padding_bytes = mz_zip_writer_compute_padding_needed_for_file_alignment(pZip);\n\n    if (!pState->m_zip64)\n    {\n        if (pZip->m_total_files == MZ_UINT16_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_TOO_MANY_FILES);\n    }\n    else\n    {\n        /* TODO: Our zip64 support still has some 32-bit limits that may not be worth fixing. */\n        if (pZip->m_total_files == MZ_UINT32_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_TOO_MANY_FILES);\n    }\n\n    if (!mz_zip_file_stat_internal(pSource_zip, src_file_index, pSrc_central_header, &src_file_stat, NULL))\n        return MZ_FALSE;\n\n    cur_src_file_ofs = src_file_stat.m_local_header_ofs;\n    cur_dst_file_ofs = pZip->m_archive_size;\n\n    /* Read the source archive's local dir header */\n    if (pSource_zip->m_pRead(pSource_zip->m_pIO_opaque, cur_src_file_ofs, pLocal_header, MZ_ZIP_LOCAL_DIR_HEADER_SIZE) != MZ_ZIP_LOCAL_DIR_HEADER_SIZE)\n        return mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n\n    if (MZ_READ_LE32(pLocal_header) != MZ_ZIP_LOCAL_DIR_HEADER_SIG)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n\n    cur_src_file_ofs += MZ_ZIP_LOCAL_DIR_HEADER_SIZE;\n\n    /* Compute the total size we need to copy (filename+extra data+compressed data) */\n    local_header_filename_size = MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_FILENAME_LEN_OFS);\n    local_header_extra_len = MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_EXTRA_LEN_OFS);\n    local_header_comp_size = MZ_READ_LE32(pLocal_header + MZ_ZIP_LDH_COMPRESSED_SIZE_OFS);\n    local_header_uncomp_size = MZ_READ_LE32(pLocal_header + MZ_ZIP_LDH_DECOMPRESSED_SIZE_OFS);\n    src_archive_bytes_remaining = local_header_filename_size + local_header_extra_len + src_file_stat.m_comp_size;\n\n    /* Try to find a zip64 extended information field */\n    if ((local_header_extra_len) && ((local_header_comp_size == MZ_UINT32_MAX) || (local_header_uncomp_size == MZ_UINT32_MAX)))\n    {\n        mz_zip_array file_data_array;\n        const mz_uint8 *pExtra_data;\n        mz_uint32 extra_size_remaining = local_header_extra_len;\n\n        mz_zip_array_init(&file_data_array, 1);\n        if (!mz_zip_array_resize(pZip, &file_data_array, local_header_extra_len, MZ_FALSE))\n        {\n            return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n        }\n\n        if (pSource_zip->m_pRead(pSource_zip->m_pIO_opaque, src_file_stat.m_local_header_ofs + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + local_header_filename_size, file_data_array.m_p, local_header_extra_len) != local_header_extra_len)\n        {\n            mz_zip_array_clear(pZip, &file_data_array);\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n        }\n\n        pExtra_data = (const mz_uint8 *)file_data_array.m_p;\n\n        do\n        {\n            mz_uint32 field_id, field_data_size, field_total_size;\n\n            if (extra_size_remaining < (sizeof(mz_uint16) * 2))\n            {\n                mz_zip_array_clear(pZip, &file_data_array);\n                return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n            }\n\n            field_id = MZ_READ_LE16(pExtra_data);\n            field_data_size = MZ_READ_LE16(pExtra_data + sizeof(mz_uint16));\n            field_total_size = field_data_size + sizeof(mz_uint16) * 2;\n\n            if (field_total_size > extra_size_remaining)\n            {\n                mz_zip_array_clear(pZip, &file_data_array);\n                return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n            }\n\n            if (field_id == MZ_ZIP64_EXTENDED_INFORMATION_FIELD_HEADER_ID)\n            {\n                const mz_uint8 *pSrc_field_data = pExtra_data + sizeof(mz_uint32);\n\n                if (field_data_size < sizeof(mz_uint64) * 2)\n                {\n                    mz_zip_array_clear(pZip, &file_data_array);\n                    return mz_zip_set_error(pZip, MZ_ZIP_INVALID_HEADER_OR_CORRUPTED);\n                }\n\n                local_header_uncomp_size = MZ_READ_LE64(pSrc_field_data);\n                local_header_comp_size = MZ_READ_LE64(pSrc_field_data + sizeof(mz_uint64)); /* may be 0 if there's a descriptor */\n\n                found_zip64_ext_data_in_ldir = MZ_TRUE;\n                break;\n            }\n\n            pExtra_data += field_total_size;\n            extra_size_remaining -= field_total_size;\n        } while (extra_size_remaining);\n\n        mz_zip_array_clear(pZip, &file_data_array);\n    }\n\n    if (!pState->m_zip64)\n    {\n        /* Try to detect if the new archive will most likely wind up too big and bail early (+(sizeof(mz_uint32) * 4) is for the optional descriptor which could be present, +64 is a fudge factor). */\n        /* We also check when the archive is finalized so this doesn't need to be perfect. */\n        mz_uint64 approx_new_archive_size = cur_dst_file_ofs + num_alignment_padding_bytes + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + src_archive_bytes_remaining + (sizeof(mz_uint32) * 4) +\n                                            pState->m_central_dir.m_size + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + src_central_dir_following_data_size + MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE + 64;\n\n        if (approx_new_archive_size >= MZ_UINT32_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE);\n    }\n\n    /* Write dest archive padding */\n    if (!mz_zip_writer_write_zeros(pZip, cur_dst_file_ofs, num_alignment_padding_bytes))\n        return MZ_FALSE;\n\n    cur_dst_file_ofs += num_alignment_padding_bytes;\n\n    local_dir_header_ofs = cur_dst_file_ofs;\n    if (pZip->m_file_offset_alignment)\n    {\n        MZ_ASSERT((local_dir_header_ofs & (pZip->m_file_offset_alignment - 1)) == 0);\n    }\n\n    /* The original zip's local header+ext block doesn't change, even with zip64, so we can just copy it over to the dest zip */\n    if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_dst_file_ofs, pLocal_header, MZ_ZIP_LOCAL_DIR_HEADER_SIZE) != MZ_ZIP_LOCAL_DIR_HEADER_SIZE)\n        return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n\n    cur_dst_file_ofs += MZ_ZIP_LOCAL_DIR_HEADER_SIZE;\n\n    /* Copy over the source archive bytes to the dest archive, also ensure we have enough buf space to handle optional data descriptor */\n    if (NULL == (pBuf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, (size_t)MZ_MAX(32U, MZ_MIN((mz_uint64)MZ_ZIP_MAX_IO_BUF_SIZE, src_archive_bytes_remaining)))))\n        return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n\n    while (src_archive_bytes_remaining)\n    {\n        n = (mz_uint)MZ_MIN((mz_uint64)MZ_ZIP_MAX_IO_BUF_SIZE, src_archive_bytes_remaining);\n        if (pSource_zip->m_pRead(pSource_zip->m_pIO_opaque, cur_src_file_ofs, pBuf, n) != n)\n        {\n            pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf);\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n        }\n        cur_src_file_ofs += n;\n\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_dst_file_ofs, pBuf, n) != n)\n        {\n            pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf);\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n        }\n        cur_dst_file_ofs += n;\n\n        src_archive_bytes_remaining -= n;\n    }\n\n    /* Now deal with the optional data descriptor */\n    bit_flags = MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_BIT_FLAG_OFS);\n    if (bit_flags & 8)\n    {\n        /* Copy data descriptor */\n        if ((pSource_zip->m_pState->m_zip64) || (found_zip64_ext_data_in_ldir))\n        {\n            /* src is zip64, dest must be zip64 */\n\n            /* name\t\t\tuint32_t's */\n            /* id\t\t\t\t1 (optional in zip64?) */\n            /* crc\t\t\t1 */\n            /* comp_size\t2 */\n            /* uncomp_size 2 */\n            if (pSource_zip->m_pRead(pSource_zip->m_pIO_opaque, cur_src_file_ofs, pBuf, (sizeof(mz_uint32) * 6)) != (sizeof(mz_uint32) * 6))\n            {\n                pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf);\n                return mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n            }\n\n            n = sizeof(mz_uint32) * ((MZ_READ_LE32(pBuf) == MZ_ZIP_DATA_DESCRIPTOR_ID) ? 6 : 5);\n        }\n        else\n        {\n            /* src is NOT zip64 */\n            mz_bool has_id;\n\n            if (pSource_zip->m_pRead(pSource_zip->m_pIO_opaque, cur_src_file_ofs, pBuf, sizeof(mz_uint32) * 4) != sizeof(mz_uint32) * 4)\n            {\n                pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf);\n                return mz_zip_set_error(pZip, MZ_ZIP_FILE_READ_FAILED);\n            }\n\n            has_id = (MZ_READ_LE32(pBuf) == MZ_ZIP_DATA_DESCRIPTOR_ID);\n\n            if (pZip->m_pState->m_zip64)\n            {\n                /* dest is zip64, so upgrade the data descriptor */\n                const mz_uint32 *pSrc_descriptor = (const mz_uint32 *)((const mz_uint8 *)pBuf + (has_id ? sizeof(mz_uint32) : 0));\n                const mz_uint32 src_crc32 = pSrc_descriptor[0];\n                const mz_uint64 src_comp_size = pSrc_descriptor[1];\n                const mz_uint64 src_uncomp_size = pSrc_descriptor[2];\n\n                mz_write_le32((mz_uint8 *)pBuf, MZ_ZIP_DATA_DESCRIPTOR_ID);\n                mz_write_le32((mz_uint8 *)pBuf + sizeof(mz_uint32) * 1, src_crc32);\n                mz_write_le64((mz_uint8 *)pBuf + sizeof(mz_uint32) * 2, src_comp_size);\n                mz_write_le64((mz_uint8 *)pBuf + sizeof(mz_uint32) * 4, src_uncomp_size);\n\n                n = sizeof(mz_uint32) * 6;\n            }\n            else\n            {\n                /* dest is NOT zip64, just copy it as-is */\n                n = sizeof(mz_uint32) * (has_id ? 4 : 3);\n            }\n        }\n\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_dst_file_ofs, pBuf, n) != n)\n        {\n            pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf);\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n        }\n\n        cur_src_file_ofs += n;\n        cur_dst_file_ofs += n;\n    }\n    pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf);\n\n    /* Finally, add the new central dir header */\n    orig_central_dir_size = pState->m_central_dir.m_size;\n\n    memcpy(new_central_header, pSrc_central_header, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE);\n\n    if (pState->m_zip64)\n    {\n        /* This is the painful part: We need to write a new central dir header + ext block with updated zip64 fields, and ensure the old fields (if any) are not included. */\n        const mz_uint8 *pSrc_ext = pSrc_central_header + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + src_filename_len;\n        mz_zip_array new_ext_block;\n\n        mz_zip_array_init(&new_ext_block, sizeof(mz_uint8));\n\n        MZ_WRITE_LE32(new_central_header + MZ_ZIP_CDH_COMPRESSED_SIZE_OFS, MZ_UINT32_MAX);\n        MZ_WRITE_LE32(new_central_header + MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS, MZ_UINT32_MAX);\n        MZ_WRITE_LE32(new_central_header + MZ_ZIP_CDH_LOCAL_HEADER_OFS, MZ_UINT32_MAX);\n\n        if (!mz_zip_writer_update_zip64_extension_block(&new_ext_block, pZip, pSrc_ext, src_ext_len, &src_file_stat.m_comp_size, &src_file_stat.m_uncomp_size, &local_dir_header_ofs, NULL))\n        {\n            mz_zip_array_clear(pZip, &new_ext_block);\n            return MZ_FALSE;\n        }\n\n        MZ_WRITE_LE16(new_central_header + MZ_ZIP_CDH_EXTRA_LEN_OFS, new_ext_block.m_size);\n\n        if (!mz_zip_array_push_back(pZip, &pState->m_central_dir, new_central_header, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE))\n        {\n            mz_zip_array_clear(pZip, &new_ext_block);\n            return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n        }\n\n        if (!mz_zip_array_push_back(pZip, &pState->m_central_dir, pSrc_central_header + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE, src_filename_len))\n        {\n            mz_zip_array_clear(pZip, &new_ext_block);\n            mz_zip_array_resize(pZip, &pState->m_central_dir, orig_central_dir_size, MZ_FALSE);\n            return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n        }\n\n        if (!mz_zip_array_push_back(pZip, &pState->m_central_dir, new_ext_block.m_p, new_ext_block.m_size))\n        {\n            mz_zip_array_clear(pZip, &new_ext_block);\n            mz_zip_array_resize(pZip, &pState->m_central_dir, orig_central_dir_size, MZ_FALSE);\n            return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n        }\n\n        if (!mz_zip_array_push_back(pZip, &pState->m_central_dir, pSrc_central_header + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + src_filename_len + src_ext_len, src_comment_len))\n        {\n            mz_zip_array_clear(pZip, &new_ext_block);\n            mz_zip_array_resize(pZip, &pState->m_central_dir, orig_central_dir_size, MZ_FALSE);\n            return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n        }\n\n        mz_zip_array_clear(pZip, &new_ext_block);\n    }\n    else\n    {\n        /* sanity checks */\n        if (cur_dst_file_ofs > MZ_UINT32_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE);\n\n        if (local_dir_header_ofs >= MZ_UINT32_MAX)\n            return mz_zip_set_error(pZip, MZ_ZIP_ARCHIVE_TOO_LARGE);\n\n        MZ_WRITE_LE32(new_central_header + MZ_ZIP_CDH_LOCAL_HEADER_OFS, local_dir_header_ofs);\n\n        if (!mz_zip_array_push_back(pZip, &pState->m_central_dir, new_central_header, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE))\n            return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n\n        if (!mz_zip_array_push_back(pZip, &pState->m_central_dir, pSrc_central_header + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE, src_central_dir_following_data_size))\n        {\n            mz_zip_array_resize(pZip, &pState->m_central_dir, orig_central_dir_size, MZ_FALSE);\n            return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n        }\n    }\n\n    /* This shouldn't trigger unless we screwed up during the initial sanity checks */\n    if (pState->m_central_dir.m_size >= MZ_UINT32_MAX)\n    {\n        /* TODO: Support central dirs >= 32-bits in size */\n        mz_zip_array_resize(pZip, &pState->m_central_dir, orig_central_dir_size, MZ_FALSE);\n        return mz_zip_set_error(pZip, MZ_ZIP_UNSUPPORTED_CDIR_SIZE);\n    }\n\n    n = (mz_uint32)orig_central_dir_size;\n    if (!mz_zip_array_push_back(pZip, &pState->m_central_dir_offsets, &n, 1))\n    {\n        mz_zip_array_resize(pZip, &pState->m_central_dir, orig_central_dir_size, MZ_FALSE);\n        return mz_zip_set_error(pZip, MZ_ZIP_ALLOC_FAILED);\n    }\n\n    pZip->m_total_files++;\n    pZip->m_archive_size = cur_dst_file_ofs;\n\n    return MZ_TRUE;\n}\n\nmz_bool mz_zip_writer_finalize_archive(mz_zip_archive *pZip)\n{\n    mz_zip_internal_state *pState;\n    mz_uint64 central_dir_ofs, central_dir_size;\n    mz_uint8 hdr[256];\n\n    if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    pState = pZip->m_pState;\n\n    if (pState->m_zip64)\n    {\n        if ((pZip->m_total_files > MZ_UINT32_MAX) || (pState->m_central_dir.m_size >= MZ_UINT32_MAX))\n            return mz_zip_set_error(pZip, MZ_ZIP_TOO_MANY_FILES);\n    }\n    else\n    {\n        if ((pZip->m_total_files > MZ_UINT16_MAX) || ((pZip->m_archive_size + pState->m_central_dir.m_size + MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE) > MZ_UINT32_MAX))\n            return mz_zip_set_error(pZip, MZ_ZIP_TOO_MANY_FILES);\n    }\n\n    central_dir_ofs = 0;\n    central_dir_size = 0;\n    if (pZip->m_total_files)\n    {\n        /* Write central directory */\n        central_dir_ofs = pZip->m_archive_size;\n        central_dir_size = pState->m_central_dir.m_size;\n        pZip->m_central_directory_file_ofs = central_dir_ofs;\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, central_dir_ofs, pState->m_central_dir.m_p, (size_t)central_dir_size) != central_dir_size)\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n\n        pZip->m_archive_size += central_dir_size;\n    }\n\n    if (pState->m_zip64)\n    {\n        /* Write zip64 end of central directory header */\n        mz_uint64 rel_ofs_to_zip64_ecdr = pZip->m_archive_size;\n\n        MZ_CLEAR_OBJ(hdr);\n        MZ_WRITE_LE32(hdr + MZ_ZIP64_ECDH_SIG_OFS, MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIG);\n        MZ_WRITE_LE64(hdr + MZ_ZIP64_ECDH_SIZE_OF_RECORD_OFS, MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE - sizeof(mz_uint32) - sizeof(mz_uint64));\n        MZ_WRITE_LE16(hdr + MZ_ZIP64_ECDH_VERSION_MADE_BY_OFS, 0x031E); /* TODO: always Unix */\n        MZ_WRITE_LE16(hdr + MZ_ZIP64_ECDH_VERSION_NEEDED_OFS, 0x002D);\n        MZ_WRITE_LE64(hdr + MZ_ZIP64_ECDH_CDIR_NUM_ENTRIES_ON_DISK_OFS, pZip->m_total_files);\n        MZ_WRITE_LE64(hdr + MZ_ZIP64_ECDH_CDIR_TOTAL_ENTRIES_OFS, pZip->m_total_files);\n        MZ_WRITE_LE64(hdr + MZ_ZIP64_ECDH_CDIR_SIZE_OFS, central_dir_size);\n        MZ_WRITE_LE64(hdr + MZ_ZIP64_ECDH_CDIR_OFS_OFS, central_dir_ofs);\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, pZip->m_archive_size, hdr, MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE) != MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE)\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n\n        pZip->m_archive_size += MZ_ZIP64_END_OF_CENTRAL_DIR_HEADER_SIZE;\n\n        /* Write zip64 end of central directory locator */\n        MZ_CLEAR_OBJ(hdr);\n        MZ_WRITE_LE32(hdr + MZ_ZIP64_ECDL_SIG_OFS, MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIG);\n        MZ_WRITE_LE64(hdr + MZ_ZIP64_ECDL_REL_OFS_TO_ZIP64_ECDR_OFS, rel_ofs_to_zip64_ecdr);\n        MZ_WRITE_LE32(hdr + MZ_ZIP64_ECDL_TOTAL_NUMBER_OF_DISKS_OFS, 1);\n        if (pZip->m_pWrite(pZip->m_pIO_opaque, pZip->m_archive_size, hdr, MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIZE) != MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIZE)\n            return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n\n        pZip->m_archive_size += MZ_ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIZE;\n    }\n\n    /* Write end of central directory record */\n    MZ_CLEAR_OBJ(hdr);\n    MZ_WRITE_LE32(hdr + MZ_ZIP_ECDH_SIG_OFS, MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIG);\n    MZ_WRITE_LE16(hdr + MZ_ZIP_ECDH_CDIR_NUM_ENTRIES_ON_DISK_OFS, MZ_MIN(MZ_UINT16_MAX, pZip->m_total_files));\n    MZ_WRITE_LE16(hdr + MZ_ZIP_ECDH_CDIR_TOTAL_ENTRIES_OFS, MZ_MIN(MZ_UINT16_MAX, pZip->m_total_files));\n    MZ_WRITE_LE32(hdr + MZ_ZIP_ECDH_CDIR_SIZE_OFS, MZ_MIN(MZ_UINT32_MAX, central_dir_size));\n    MZ_WRITE_LE32(hdr + MZ_ZIP_ECDH_CDIR_OFS_OFS, MZ_MIN(MZ_UINT32_MAX, central_dir_ofs));\n\n    if (pZip->m_pWrite(pZip->m_pIO_opaque, pZip->m_archive_size, hdr, MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE) != MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE)\n        return mz_zip_set_error(pZip, MZ_ZIP_FILE_WRITE_FAILED);\n\n#ifndef MINIZ_NO_STDIO\n    if ((pState->m_pFile) && (MZ_FFLUSH(pState->m_pFile) == EOF))\n        return mz_zip_set_error(pZip, MZ_ZIP_FILE_CLOSE_FAILED);\n#endif /* #ifndef MINIZ_NO_STDIO */\n\n    pZip->m_archive_size += MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE;\n\n    pZip->m_zip_mode = MZ_ZIP_MODE_WRITING_HAS_BEEN_FINALIZED;\n    return MZ_TRUE;\n}\n\nmz_bool mz_zip_writer_finalize_heap_archive(mz_zip_archive *pZip, void **ppBuf, size_t *pSize)\n{\n    if ((!ppBuf) || (!pSize))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    *ppBuf = NULL;\n    *pSize = 0;\n\n    if ((!pZip) || (!pZip->m_pState))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    if (pZip->m_pWrite != mz_zip_heap_write_func)\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    if (!mz_zip_writer_finalize_archive(pZip))\n        return MZ_FALSE;\n\n    *ppBuf = pZip->m_pState->m_pMem;\n    *pSize = pZip->m_pState->m_mem_size;\n    pZip->m_pState->m_pMem = NULL;\n    pZip->m_pState->m_mem_size = pZip->m_pState->m_mem_capacity = 0;\n\n    return MZ_TRUE;\n}\n\nmz_bool mz_zip_writer_end(mz_zip_archive *pZip)\n{\n    return mz_zip_writer_end_internal(pZip, MZ_TRUE);\n}\n\n#ifndef MINIZ_NO_STDIO\nmz_bool mz_zip_add_mem_to_archive_file_in_place(const char *pZip_filename, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags)\n{\n    return mz_zip_add_mem_to_archive_file_in_place_v2(pZip_filename, pArchive_name, pBuf, buf_size, pComment, comment_size, level_and_flags, NULL);\n}\n\nmz_bool mz_zip_add_mem_to_archive_file_in_place_v2(const char *pZip_filename, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags, mz_zip_error *pErr)\n{\n    mz_bool status, created_new_archive = MZ_FALSE;\n    mz_zip_archive zip_archive;\n    struct MZ_FILE_STAT_STRUCT file_stat;\n    mz_zip_error actual_err = MZ_ZIP_NO_ERROR;\n\n    mz_zip_zero_struct(&zip_archive);\n    if ((int)level_and_flags < 0)\n        level_and_flags = MZ_DEFAULT_LEVEL;\n\n    if ((!pZip_filename) || (!pArchive_name) || ((buf_size) && (!pBuf)) || ((comment_size) && (!pComment)) || ((level_and_flags & 0xF) > MZ_UBER_COMPRESSION))\n    {\n        if (pErr)\n            *pErr = MZ_ZIP_INVALID_PARAMETER;\n        return MZ_FALSE;\n    }\n\n    if (!mz_zip_writer_validate_archive_name(pArchive_name))\n    {\n        if (pErr)\n            *pErr = MZ_ZIP_INVALID_FILENAME;\n        return MZ_FALSE;\n    }\n\n    /* Important: The regular non-64 bit version of stat() can fail here if the file is very large, which could cause the archive to be overwritten. */\n    /* So be sure to compile with _LARGEFILE64_SOURCE 1 */\n    if (MZ_FILE_STAT(pZip_filename, &file_stat) != 0)\n    {\n        /* Create a new archive. */\n        if (!mz_zip_writer_init_file_v2(&zip_archive, pZip_filename, 0, level_and_flags))\n        {\n            if (pErr)\n                *pErr = zip_archive.m_last_error;\n            return MZ_FALSE;\n        }\n\n        created_new_archive = MZ_TRUE;\n    }\n    else\n    {\n        /* Append to an existing archive. */\n        if (!mz_zip_reader_init_file_v2(&zip_archive, pZip_filename, level_and_flags | MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY, 0, 0))\n        {\n            if (pErr)\n                *pErr = zip_archive.m_last_error;\n            return MZ_FALSE;\n        }\n\n        if (!mz_zip_writer_init_from_reader_v2(&zip_archive, pZip_filename, level_and_flags))\n        {\n            if (pErr)\n                *pErr = zip_archive.m_last_error;\n\n            mz_zip_reader_end_internal(&zip_archive, MZ_FALSE);\n\n            return MZ_FALSE;\n        }\n    }\n\n    status = mz_zip_writer_add_mem_ex(&zip_archive, pArchive_name, pBuf, buf_size, pComment, comment_size, level_and_flags, 0, 0);\n    actual_err = zip_archive.m_last_error;\n\n    /* Always finalize, even if adding failed for some reason, so we have a valid central directory. (This may not always succeed, but we can try.) */\n    if (!mz_zip_writer_finalize_archive(&zip_archive))\n    {\n        if (!actual_err)\n            actual_err = zip_archive.m_last_error;\n\n        status = MZ_FALSE;\n    }\n\n    if (!mz_zip_writer_end_internal(&zip_archive, status))\n    {\n        if (!actual_err)\n            actual_err = zip_archive.m_last_error;\n\n        status = MZ_FALSE;\n    }\n\n    if ((!status) && (created_new_archive))\n    {\n        /* It's a new archive and something went wrong, so just delete it. */\n        int ignoredStatus = MZ_DELETE_FILE(pZip_filename);\n        (void)ignoredStatus;\n    }\n\n    if (pErr)\n        *pErr = actual_err;\n\n    return status;\n}\n\nvoid *mz_zip_extract_archive_file_to_heap_v2(const char *pZip_filename, const char *pArchive_name, const char *pComment, size_t *pSize, mz_uint flags, mz_zip_error *pErr)\n{\n    mz_uint32 file_index;\n    mz_zip_archive zip_archive;\n    void *p = NULL;\n\n    if (pSize)\n        *pSize = 0;\n\n    if ((!pZip_filename) || (!pArchive_name))\n    {\n        if (pErr)\n            *pErr = MZ_ZIP_INVALID_PARAMETER;\n\n        return NULL;\n    }\n\n    mz_zip_zero_struct(&zip_archive);\n    if (!mz_zip_reader_init_file_v2(&zip_archive, pZip_filename, flags | MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY, 0, 0))\n    {\n        if (pErr)\n            *pErr = zip_archive.m_last_error;\n\n        return NULL;\n    }\n\n    if (mz_zip_reader_locate_file_v2(&zip_archive, pArchive_name, pComment, flags, &file_index))\n    {\n        p = mz_zip_reader_extract_to_heap(&zip_archive, file_index, pSize, flags);\n    }\n\n    mz_zip_reader_end_internal(&zip_archive, p != NULL);\n\n    if (pErr)\n        *pErr = zip_archive.m_last_error;\n\n    return p;\n}\n\nvoid *mz_zip_extract_archive_file_to_heap(const char *pZip_filename, const char *pArchive_name, size_t *pSize, mz_uint flags)\n{\n    return mz_zip_extract_archive_file_to_heap_v2(pZip_filename, pArchive_name, NULL, pSize, flags, NULL);\n}\n\n#endif /* #ifndef MINIZ_NO_STDIO */\n\n#endif /* #ifndef MINIZ_NO_ARCHIVE_WRITING_APIS */\n\n/* ------------------- Misc utils */\n\nmz_zip_mode mz_zip_get_mode(mz_zip_archive *pZip)\n{\n    return pZip ? pZip->m_zip_mode : MZ_ZIP_MODE_INVALID;\n}\n\nmz_zip_type mz_zip_get_type(mz_zip_archive *pZip)\n{\n    return pZip ? pZip->m_zip_type : MZ_ZIP_TYPE_INVALID;\n}\n\nmz_zip_error mz_zip_set_last_error(mz_zip_archive *pZip, mz_zip_error err_num)\n{\n    mz_zip_error prev_err;\n\n    if (!pZip)\n        return MZ_ZIP_INVALID_PARAMETER;\n\n    prev_err = pZip->m_last_error;\n\n    pZip->m_last_error = err_num;\n    return prev_err;\n}\n\nmz_zip_error mz_zip_peek_last_error(mz_zip_archive *pZip)\n{\n    if (!pZip)\n        return MZ_ZIP_INVALID_PARAMETER;\n\n    return pZip->m_last_error;\n}\n\nmz_zip_error mz_zip_clear_last_error(mz_zip_archive *pZip)\n{\n    return mz_zip_set_last_error(pZip, MZ_ZIP_NO_ERROR);\n}\n\nmz_zip_error mz_zip_get_last_error(mz_zip_archive *pZip)\n{\n    mz_zip_error prev_err;\n\n    if (!pZip)\n        return MZ_ZIP_INVALID_PARAMETER;\n\n    prev_err = pZip->m_last_error;\n\n    pZip->m_last_error = MZ_ZIP_NO_ERROR;\n    return prev_err;\n}\n\nconst char *mz_zip_get_error_string(mz_zip_error mz_err)\n{\n    switch (mz_err)\n    {\n        case MZ_ZIP_NO_ERROR:\n            return \"no error\";\n        case MZ_ZIP_UNDEFINED_ERROR:\n            return \"undefined error\";\n        case MZ_ZIP_TOO_MANY_FILES:\n            return \"too many files\";\n        case MZ_ZIP_FILE_TOO_LARGE:\n            return \"file too large\";\n        case MZ_ZIP_UNSUPPORTED_METHOD:\n            return \"unsupported method\";\n        case MZ_ZIP_UNSUPPORTED_ENCRYPTION:\n            return \"unsupported encryption\";\n        case MZ_ZIP_UNSUPPORTED_FEATURE:\n            return \"unsupported feature\";\n        case MZ_ZIP_FAILED_FINDING_CENTRAL_DIR:\n            return \"failed finding central directory\";\n        case MZ_ZIP_NOT_AN_ARCHIVE:\n            return \"not a ZIP archive\";\n        case MZ_ZIP_INVALID_HEADER_OR_CORRUPTED:\n            return \"invalid header or archive is corrupted\";\n        case MZ_ZIP_UNSUPPORTED_MULTIDISK:\n            return \"unsupported multidisk archive\";\n        case MZ_ZIP_DECOMPRESSION_FAILED:\n            return \"decompression failed or archive is corrupted\";\n        case MZ_ZIP_COMPRESSION_FAILED:\n            return \"compression failed\";\n        case MZ_ZIP_UNEXPECTED_DECOMPRESSED_SIZE:\n            return \"unexpected decompressed size\";\n        case MZ_ZIP_CRC_CHECK_FAILED:\n            return \"CRC-32 check failed\";\n        case MZ_ZIP_UNSUPPORTED_CDIR_SIZE:\n            return \"unsupported central directory size\";\n        case MZ_ZIP_ALLOC_FAILED:\n            return \"allocation failed\";\n        case MZ_ZIP_FILE_OPEN_FAILED:\n            return \"file open failed\";\n        case MZ_ZIP_FILE_CREATE_FAILED:\n            return \"file create failed\";\n        case MZ_ZIP_FILE_WRITE_FAILED:\n            return \"file write failed\";\n        case MZ_ZIP_FILE_READ_FAILED:\n            return \"file read failed\";\n        case MZ_ZIP_FILE_CLOSE_FAILED:\n            return \"file close failed\";\n        case MZ_ZIP_FILE_SEEK_FAILED:\n            return \"file seek failed\";\n        case MZ_ZIP_FILE_STAT_FAILED:\n            return \"file stat failed\";\n        case MZ_ZIP_INVALID_PARAMETER:\n            return \"invalid parameter\";\n        case MZ_ZIP_INVALID_FILENAME:\n            return \"invalid filename\";\n        case MZ_ZIP_BUF_TOO_SMALL:\n            return \"buffer too small\";\n        case MZ_ZIP_INTERNAL_ERROR:\n            return \"internal error\";\n        case MZ_ZIP_FILE_NOT_FOUND:\n            return \"file not found\";\n        case MZ_ZIP_ARCHIVE_TOO_LARGE:\n            return \"archive is too large\";\n        case MZ_ZIP_VALIDATION_FAILED:\n            return \"validation failed\";\n        case MZ_ZIP_WRITE_CALLBACK_FAILED:\n            return \"write calledback failed\";\n        default:\n            break;\n    }\n\n    return \"unknown error\";\n}\n\n/* Note: Just because the archive is not zip64 doesn't necessarily mean it doesn't have Zip64 extended information extra field, argh. */\nmz_bool mz_zip_is_zip64(mz_zip_archive *pZip)\n{\n    if ((!pZip) || (!pZip->m_pState))\n        return MZ_FALSE;\n\n    return pZip->m_pState->m_zip64;\n}\n\nsize_t mz_zip_get_central_dir_size(mz_zip_archive *pZip)\n{\n    if ((!pZip) || (!pZip->m_pState))\n        return 0;\n\n    return pZip->m_pState->m_central_dir.m_size;\n}\n\nmz_uint mz_zip_reader_get_num_files(mz_zip_archive *pZip)\n{\n    return pZip ? pZip->m_total_files : 0;\n}\n\nmz_uint64 mz_zip_get_archive_size(mz_zip_archive *pZip)\n{\n    if (!pZip)\n        return 0;\n    return pZip->m_archive_size;\n}\n\nmz_uint64 mz_zip_get_archive_file_start_offset(mz_zip_archive *pZip)\n{\n    if ((!pZip) || (!pZip->m_pState))\n        return 0;\n    return pZip->m_pState->m_file_archive_start_ofs;\n}\n\nMZ_FILE *mz_zip_get_cfile(mz_zip_archive *pZip)\n{\n    if ((!pZip) || (!pZip->m_pState))\n        return 0;\n    return pZip->m_pState->m_pFile;\n}\n\nsize_t mz_zip_read_archive_data(mz_zip_archive *pZip, mz_uint64 file_ofs, void *pBuf, size_t n)\n{\n    if ((!pZip) || (!pZip->m_pState) || (!pBuf) || (!pZip->m_pRead))\n        return mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n\n    return pZip->m_pRead(pZip->m_pIO_opaque, file_ofs, pBuf, n);\n}\n\nmz_uint mz_zip_reader_get_filename(mz_zip_archive *pZip, mz_uint file_index, char *pFilename, mz_uint filename_buf_size)\n{\n    mz_uint n;\n    const mz_uint8 *p = mz_zip_get_cdh(pZip, file_index);\n    if (!p)\n    {\n        if (filename_buf_size)\n            pFilename[0] = '\\0';\n        mz_zip_set_error(pZip, MZ_ZIP_INVALID_PARAMETER);\n        return 0;\n    }\n    n = MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS);\n    if (filename_buf_size)\n    {\n        n = MZ_MIN(n, filename_buf_size - 1);\n        memcpy(pFilename, p + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE, n);\n        pFilename[n] = '\\0';\n    }\n    return n + 1;\n}\n\nmz_bool mz_zip_reader_file_stat(mz_zip_archive *pZip, mz_uint file_index, mz_zip_archive_file_stat *pStat)\n{\n    return mz_zip_file_stat_internal(pZip, file_index, mz_zip_get_cdh(pZip, file_index), pStat, NULL);\n}\n\nmz_bool mz_zip_end(mz_zip_archive *pZip)\n{\n    if (!pZip)\n        return MZ_FALSE;\n\n    if (pZip->m_zip_mode == MZ_ZIP_MODE_READING)\n        return mz_zip_reader_end(pZip);\n#ifndef MINIZ_NO_ARCHIVE_WRITING_APIS\n    else if ((pZip->m_zip_mode == MZ_ZIP_MODE_WRITING) || (pZip->m_zip_mode == MZ_ZIP_MODE_WRITING_HAS_BEEN_FINALIZED))\n        return mz_zip_writer_end(pZip);\n#endif\n\n    return MZ_FALSE;\n}\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif /*#ifndef MINIZ_NO_ARCHIVE_APIS*/\n"
  },
  {
    "path": "3rdparty/miniz/miniz.h",
    "content": "/**************************************************************************\n *\n * Copyright 2013-2014 RAD Game Tools and Valve Software\n * Copyright 2010-2014 Rich Geldreich and Tenacious Software LLC\n * All Rights Reserved.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n *\n **************************************************************************/\n\n\n/* miniz.c 2.1.0 - public domain deflate/inflate, zlib-subset, ZIP reading/writing/appending, PNG writing\n   See \"unlicense\" statement at the end of this file.\n   Rich Geldreich <richgel99@gmail.com>, last updated Oct. 13, 2013\n   Implements RFC 1950: http://www.ietf.org/rfc/rfc1950.txt and RFC 1951: http://www.ietf.org/rfc/rfc1951.txt\n\n   Most API's defined in miniz.c are optional. For example, to disable the archive related functions just define\n   MINIZ_NO_ARCHIVE_APIS, or to get rid of all stdio usage define MINIZ_NO_STDIO (see the list below for more macros).\n\n   * Low-level Deflate/Inflate implementation notes:\n\n     Compression: Use the \"tdefl\" API's. The compressor supports raw, static, and dynamic blocks, lazy or\n     greedy parsing, match length filtering, RLE-only, and Huffman-only streams. It performs and compresses\n     approximately as well as zlib.\n\n     Decompression: Use the \"tinfl\" API's. The entire decompressor is implemented as a single function\n     coroutine: see tinfl_decompress(). It supports decompression into a 32KB (or larger power of 2) wrapping buffer, or into a memory\n     block large enough to hold the entire file.\n\n     The low-level tdefl/tinfl API's do not make any use of dynamic memory allocation.\n\n   * zlib-style API notes:\n\n     miniz.c implements a fairly large subset of zlib. There's enough functionality present for it to be a drop-in\n     zlib replacement in many apps:\n        The z_stream struct, optional memory allocation callbacks\n        deflateInit/deflateInit2/deflate/deflateReset/deflateEnd/deflateBound\n        inflateInit/inflateInit2/inflate/inflateReset/inflateEnd\n        compress, compress2, compressBound, uncompress\n        CRC-32, Adler-32 - Using modern, minimal code size, CPU cache friendly routines.\n        Supports raw deflate streams or standard zlib streams with adler-32 checking.\n\n     Limitations:\n      The callback API's are not implemented yet. No support for gzip headers or zlib static dictionaries.\n      I've tried to closely emulate zlib's various flavors of stream flushing and return status codes, but\n      there are no guarantees that miniz.c pulls this off perfectly.\n\n   * PNG writing: See the tdefl_write_image_to_png_file_in_memory() function, originally written by\n     Alex Evans. Supports 1-4 bytes/pixel images.\n\n   * ZIP archive API notes:\n\n     The ZIP archive API's where designed with simplicity and efficiency in mind, with just enough abstraction to\n     get the job done with minimal fuss. There are simple API's to retrieve file information, read files from\n     existing archives, create new archives, append new files to existing archives, or clone archive data from\n     one archive to another. It supports archives located in memory or the heap, on disk (using stdio.h),\n     or you can specify custom file read/write callbacks.\n\n     - Archive reading: Just call this function to read a single file from a disk archive:\n\n      void *mz_zip_extract_archive_file_to_heap(const char *pZip_filename, const char *pArchive_name,\n        size_t *pSize, mz_uint zip_flags);\n\n     For more complex cases, use the \"mz_zip_reader\" functions. Upon opening an archive, the entire central\n     directory is located and read as-is into memory, and subsequent file access only occurs when reading individual files.\n\n     - Archives file scanning: The simple way is to use this function to scan a loaded archive for a specific file:\n\n     int mz_zip_reader_locate_file(mz_zip_archive *pZip, const char *pName, const char *pComment, mz_uint flags);\n\n     The locate operation can optionally check file comments too, which (as one example) can be used to identify\n     multiple versions of the same file in an archive. This function uses a simple linear search through the central\n     directory, so it's not very fast.\n\n     Alternately, you can iterate through all the files in an archive (using mz_zip_reader_get_num_files()) and\n     retrieve detailed info on each file by calling mz_zip_reader_file_stat().\n\n     - Archive creation: Use the \"mz_zip_writer\" functions. The ZIP writer immediately writes compressed file data\n     to disk and builds an exact image of the central directory in memory. The central directory image is written\n     all at once at the end of the archive file when the archive is finalized.\n\n     The archive writer can optionally align each file's local header and file data to any power of 2 alignment,\n     which can be useful when the archive will be read from optical media. Also, the writer supports placing\n     arbitrary data blobs at the very beginning of ZIP archives. Archives written using either feature are still\n     readable by any ZIP tool.\n\n     - Archive appending: The simple way to add a single file to an archive is to call this function:\n\n      mz_bool mz_zip_add_mem_to_archive_file_in_place(const char *pZip_filename, const char *pArchive_name,\n        const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags);\n\n     The archive will be created if it doesn't already exist, otherwise it'll be appended to.\n     Note the appending is done in-place and is not an atomic operation, so if something goes wrong\n     during the operation it's possible the archive could be left without a central directory (although the local\n     file headers and file data will be fine, so the archive will be recoverable).\n\n     For more complex archive modification scenarios:\n     1. The safest way is to use a mz_zip_reader to read the existing archive, cloning only those bits you want to\n     preserve into a new archive using using the mz_zip_writer_add_from_zip_reader() function (which compiles the\n     compressed file data as-is). When you're done, delete the old archive and rename the newly written archive, and\n     you're done. This is safe but requires a bunch of temporary disk space or heap memory.\n\n     2. Or, you can convert an mz_zip_reader in-place to an mz_zip_writer using mz_zip_writer_init_from_reader(),\n     append new files as needed, then finalize the archive which will write an updated central directory to the\n     original archive. (This is basically what mz_zip_add_mem_to_archive_file_in_place() does.) There's a\n     possibility that the archive's central directory could be lost with this method if anything goes wrong, though.\n\n     - ZIP archive support limitations:\n     No zip64 or spanning support. Extraction functions can only handle unencrypted, stored or deflated files.\n     Requires streams capable of seeking.\n\n   * This is a header file library, like stb_image.c. To get only a header file, either cut and paste the\n     below header, or create miniz.h, #define MINIZ_HEADER_FILE_ONLY, and then include miniz.c from it.\n\n   * Important: For best perf. be sure to customize the below macros for your target platform:\n     #define MINIZ_USE_UNALIGNED_LOADS_AND_STORES 1\n     #define MINIZ_LITTLE_ENDIAN 1\n     #define MINIZ_HAS_64BIT_REGISTERS 1\n\n   * On platforms using glibc, Be sure to \"#define _LARGEFILE64_SOURCE 1\" before including miniz.c to ensure miniz\n     uses the 64-bit variants: fopen64(), stat64(), etc. Otherwise you won't be able to process large files\n     (i.e. 32-bit stat() fails for me on files > 0x7FFFFFFF bytes).\n*/\n#pragma once\n\n\n\n\n\n/* Defines to completely disable specific portions of miniz.c: \n   If all macros here are defined the only functionality remaining will be CRC-32, adler-32, tinfl, and tdefl. */\n\n/* Define MINIZ_NO_STDIO to disable all usage and any functions which rely on stdio for file I/O. */\n/*#define MINIZ_NO_STDIO */\n\n/* If MINIZ_NO_TIME is specified then the ZIP archive functions will not be able to get the current time, or */\n/* get/set file times, and the C run-time funcs that get/set times won't be called. */\n/* The current downside is the times written to your archives will be from 1979. */\n/*#define MINIZ_NO_TIME */\n\n/* Define MINIZ_NO_ARCHIVE_APIS to disable all ZIP archive API's. */\n/*#define MINIZ_NO_ARCHIVE_APIS */\n\n/* Define MINIZ_NO_ARCHIVE_WRITING_APIS to disable all writing related ZIP archive API's. */\n/*#define MINIZ_NO_ARCHIVE_WRITING_APIS */\n\n/* Define MINIZ_NO_ZLIB_APIS to remove all ZLIB-style compression/decompression API's. */\n/*#define MINIZ_NO_ZLIB_APIS */\n\n/* Define MINIZ_NO_ZLIB_COMPATIBLE_NAME to disable zlib names, to prevent conflicts against stock zlib. */\n/*#define MINIZ_NO_ZLIB_COMPATIBLE_NAMES */\n\n/* Define MINIZ_NO_MALLOC to disable all calls to malloc, free, and realloc. \n   Note if MINIZ_NO_MALLOC is defined then the user must always provide custom user alloc/free/realloc\n   callbacks to the zlib and archive API's, and a few stand-alone helper API's which don't provide custom user\n   functions (such as tdefl_compress_mem_to_heap() and tinfl_decompress_mem_to_heap()) won't work. */\n/*#define MINIZ_NO_MALLOC */\n\n#if defined(__TINYC__) && (defined(__linux) || defined(__linux__))\n/* TODO: Work around \"error: include file 'sys\\utime.h' when compiling with tcc on Linux */\n#define MINIZ_NO_TIME\n#endif\n\n#include <stddef.h>\n\n#if !defined(MINIZ_NO_TIME) && !defined(MINIZ_NO_ARCHIVE_APIS)\n#include <time.h>\n#endif\n\n#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__i386) || defined(__i486__) || defined(__i486) || defined(i386) || defined(__ia64__) || defined(__x86_64__)\n/* MINIZ_X86_OR_X64_CPU is only used to help set the below macros. */\n#define MINIZ_X86_OR_X64_CPU 1\n#else\n#define MINIZ_X86_OR_X64_CPU 0\n#endif\n\n#if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) || MINIZ_X86_OR_X64_CPU\n/* Set MINIZ_LITTLE_ENDIAN to 1 if the processor is little endian. */\n#define MINIZ_LITTLE_ENDIAN 1\n#else\n#define MINIZ_LITTLE_ENDIAN 0\n#endif\n\n/* Set MINIZ_USE_UNALIGNED_LOADS_AND_STORES only if not set */\n#if !defined(MINIZ_USE_UNALIGNED_LOADS_AND_STORES)\n#if MINIZ_X86_OR_X64_CPU\n/* Set MINIZ_USE_UNALIGNED_LOADS_AND_STORES to 1 on CPU's that permit efficient integer loads and stores from unaligned addresses. */\n#define MINIZ_USE_UNALIGNED_LOADS_AND_STORES 1\n#define MINIZ_UNALIGNED_USE_MEMCPY\n#else\n#define MINIZ_USE_UNALIGNED_LOADS_AND_STORES 0\n#endif\n#endif\n\n#if defined(_M_X64) || defined(_WIN64) || defined(__MINGW64__) || defined(_LP64) || defined(__LP64__) || defined(__ia64__) || defined(__x86_64__)\n/* Set MINIZ_HAS_64BIT_REGISTERS to 1 if operations on 64-bit integers are reasonably fast (and don't involve compiler generated calls to helper functions). */\n#define MINIZ_HAS_64BIT_REGISTERS 1\n#else\n#define MINIZ_HAS_64BIT_REGISTERS 0\n#endif\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n/* ------------------- zlib-style API Definitions. */\n\n/* For more compatibility with zlib, miniz.c uses unsigned long for some parameters/struct members. Beware: mz_ulong can be either 32 or 64-bits! */\ntypedef unsigned long mz_ulong;\n\n/* mz_free() internally uses the MZ_FREE() macro (which by default calls free() unless you've modified the MZ_MALLOC macro) to release a block allocated from the heap. */\nvoid mz_free(void *p);\n\n#define MZ_ADLER32_INIT (1)\n/* mz_adler32() returns the initial adler-32 value to use when called with ptr==NULL. */\nmz_ulong mz_adler32(mz_ulong adler, const unsigned char *ptr, size_t buf_len);\n\n#define MZ_CRC32_INIT (0)\n/* mz_crc32() returns the initial CRC-32 value to use when called with ptr==NULL. */\nmz_ulong mz_crc32(mz_ulong crc, const unsigned char *ptr, size_t buf_len);\n\n/* Compression strategies. */\nenum\n{\n    MZ_DEFAULT_STRATEGY = 0,\n    MZ_FILTERED = 1,\n    MZ_HUFFMAN_ONLY = 2,\n    MZ_RLE = 3,\n    MZ_FIXED = 4\n};\n\n/* Method */\n#define MZ_DEFLATED 8\n\n/* Heap allocation callbacks.\nNote that mz_alloc_func parameter types purpsosely differ from zlib's: items/size is size_t, not unsigned long. */\ntypedef void *(*mz_alloc_func)(void *opaque, size_t items, size_t size);\ntypedef void (*mz_free_func)(void *opaque, void *address);\ntypedef void *(*mz_realloc_func)(void *opaque, void *address, size_t items, size_t size);\n\n/* Compression levels: 0-9 are the standard zlib-style levels, 10 is best possible compression (not zlib compatible, and may be very slow), MZ_DEFAULT_COMPRESSION=MZ_DEFAULT_LEVEL. */\nenum\n{\n    MZ_NO_COMPRESSION = 0,\n    MZ_BEST_SPEED = 1,\n    MZ_BEST_COMPRESSION = 9,\n    MZ_UBER_COMPRESSION = 10,\n    MZ_DEFAULT_LEVEL = 6,\n    MZ_DEFAULT_COMPRESSION = -1\n};\n\n#define MZ_VERSION \"10.1.0\"\n#define MZ_VERNUM 0xA100\n#define MZ_VER_MAJOR 10\n#define MZ_VER_MINOR 1\n#define MZ_VER_REVISION 0\n#define MZ_VER_SUBREVISION 0\n\n#ifndef MINIZ_NO_ZLIB_APIS\n\n/* Flush values. For typical usage you only need MZ_NO_FLUSH and MZ_FINISH. The other values are for advanced use (refer to the zlib docs). */\nenum\n{\n    MZ_NO_FLUSH = 0,\n    MZ_PARTIAL_FLUSH = 1,\n    MZ_SYNC_FLUSH = 2,\n    MZ_FULL_FLUSH = 3,\n    MZ_FINISH = 4,\n    MZ_BLOCK = 5\n};\n\n/* Return status codes. MZ_PARAM_ERROR is non-standard. */\nenum\n{\n    MZ_OK = 0,\n    MZ_STREAM_END = 1,\n    MZ_NEED_DICT = 2,\n    MZ_ERRNO = -1,\n    MZ_STREAM_ERROR = -2,\n    MZ_DATA_ERROR = -3,\n    MZ_MEM_ERROR = -4,\n    MZ_BUF_ERROR = -5,\n    MZ_VERSION_ERROR = -6,\n    MZ_PARAM_ERROR = -10000\n};\n\n/* Window bits */\n#define MZ_DEFAULT_WINDOW_BITS 15\n\nstruct mz_internal_state;\n\n/* Compression/decompression stream struct. */\ntypedef struct mz_stream_s\n{\n    const unsigned char *next_in; /* pointer to next byte to read */\n    unsigned int avail_in;        /* number of bytes available at next_in */\n    mz_ulong total_in;            /* total number of bytes consumed so far */\n\n    unsigned char *next_out; /* pointer to next byte to write */\n    unsigned int avail_out;  /* number of bytes that can be written to next_out */\n    mz_ulong total_out;      /* total number of bytes produced so far */\n\n    char *msg;                       /* error msg (unused) */\n    struct mz_internal_state *state; /* internal state, allocated by zalloc/zfree */\n\n    mz_alloc_func zalloc; /* optional heap allocation function (defaults to malloc) */\n    mz_free_func zfree;   /* optional heap free function (defaults to free) */\n    void *opaque;         /* heap alloc function user pointer */\n\n    int data_type;     /* data_type (unused) */\n    mz_ulong adler;    /* adler32 of the source or uncompressed data */\n    mz_ulong reserved; /* not used */\n} mz_stream;\n\ntypedef mz_stream *mz_streamp;\n\n/* Returns the version string of miniz.c. */\nconst char *mz_version(void);\n\n/* mz_deflateInit() initializes a compressor with default options: */\n/* Parameters: */\n/*  pStream must point to an initialized mz_stream struct. */\n/*  level must be between [MZ_NO_COMPRESSION, MZ_BEST_COMPRESSION]. */\n/*  level 1 enables a specially optimized compression function that's been optimized purely for performance, not ratio. */\n/*  (This special func. is currently only enabled when MINIZ_USE_UNALIGNED_LOADS_AND_STORES and MINIZ_LITTLE_ENDIAN are defined.) */\n/* Return values: */\n/*  MZ_OK on success. */\n/*  MZ_STREAM_ERROR if the stream is bogus. */\n/*  MZ_PARAM_ERROR if the input parameters are bogus. */\n/*  MZ_MEM_ERROR on out of memory. */\nint mz_deflateInit(mz_streamp pStream, int level);\n\n/* mz_deflateInit2() is like mz_deflate(), except with more control: */\n/* Additional parameters: */\n/*   method must be MZ_DEFLATED */\n/*   window_bits must be MZ_DEFAULT_WINDOW_BITS (to wrap the deflate stream with zlib header/adler-32 footer) or -MZ_DEFAULT_WINDOW_BITS (raw deflate/no header or footer) */\n/*   mem_level must be between [1, 9] (it's checked but ignored by miniz.c) */\nint mz_deflateInit2(mz_streamp pStream, int level, int method, int window_bits, int mem_level, int strategy);\n\n/* Quickly resets a compressor without having to reallocate anything. Same as calling mz_deflateEnd() followed by mz_deflateInit()/mz_deflateInit2(). */\nint mz_deflateReset(mz_streamp pStream);\n\n/* mz_deflate() compresses the input to output, consuming as much of the input and producing as much output as possible. */\n/* Parameters: */\n/*   pStream is the stream to read from and write to. You must initialize/update the next_in, avail_in, next_out, and avail_out members. */\n/*   flush may be MZ_NO_FLUSH, MZ_PARTIAL_FLUSH/MZ_SYNC_FLUSH, MZ_FULL_FLUSH, or MZ_FINISH. */\n/* Return values: */\n/*   MZ_OK on success (when flushing, or if more input is needed but not available, and/or there's more output to be written but the output buffer is full). */\n/*   MZ_STREAM_END if all input has been consumed and all output bytes have been written. Don't call mz_deflate() on the stream anymore. */\n/*   MZ_STREAM_ERROR if the stream is bogus. */\n/*   MZ_PARAM_ERROR if one of the parameters is invalid. */\n/*   MZ_BUF_ERROR if no forward progress is possible because the input and/or output buffers are empty. (Fill up the input buffer or free up some output space and try again.) */\nint mz_deflate(mz_streamp pStream, int flush);\n\n/* mz_deflateEnd() deinitializes a compressor: */\n/* Return values: */\n/*  MZ_OK on success. */\n/*  MZ_STREAM_ERROR if the stream is bogus. */\nint mz_deflateEnd(mz_streamp pStream);\n\n/* mz_deflateBound() returns a (very) conservative upper bound on the amount of data that could be generated by deflate(), assuming flush is set to only MZ_NO_FLUSH or MZ_FINISH. */\nmz_ulong mz_deflateBound(mz_streamp pStream, mz_ulong source_len);\n\n/* Single-call compression functions mz_compress() and mz_compress2(): */\n/* Returns MZ_OK on success, or one of the error codes from mz_deflate() on failure. */\nint mz_compress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len);\nint mz_compress2(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len, int level);\n\n/* mz_compressBound() returns a (very) conservative upper bound on the amount of data that could be generated by calling mz_compress(). */\nmz_ulong mz_compressBound(mz_ulong source_len);\n\n/* Initializes a decompressor. */\nint mz_inflateInit(mz_streamp pStream);\n\n/* mz_inflateInit2() is like mz_inflateInit() with an additional option that controls the window size and whether or not the stream has been wrapped with a zlib header/footer: */\n/* window_bits must be MZ_DEFAULT_WINDOW_BITS (to parse zlib header/footer) or -MZ_DEFAULT_WINDOW_BITS (raw deflate). */\nint mz_inflateInit2(mz_streamp pStream, int window_bits);\n\n/* Quickly resets a compressor without having to reallocate anything. Same as calling mz_inflateEnd() followed by mz_inflateInit()/mz_inflateInit2(). */\nint mz_inflateReset(mz_streamp pStream);\n\n/* Decompresses the input stream to the output, consuming only as much of the input as needed, and writing as much to the output as possible. */\n/* Parameters: */\n/*   pStream is the stream to read from and write to. You must initialize/update the next_in, avail_in, next_out, and avail_out members. */\n/*   flush may be MZ_NO_FLUSH, MZ_SYNC_FLUSH, or MZ_FINISH. */\n/*   On the first call, if flush is MZ_FINISH it's assumed the input and output buffers are both sized large enough to decompress the entire stream in a single call (this is slightly faster). */\n/*   MZ_FINISH implies that there are no more source bytes available beside what's already in the input buffer, and that the output buffer is large enough to hold the rest of the decompressed data. */\n/* Return values: */\n/*   MZ_OK on success. Either more input is needed but not available, and/or there's more output to be written but the output buffer is full. */\n/*   MZ_STREAM_END if all needed input has been consumed and all output bytes have been written. For zlib streams, the adler-32 of the decompressed data has also been verified. */\n/*   MZ_STREAM_ERROR if the stream is bogus. */\n/*   MZ_DATA_ERROR if the deflate stream is invalid. */\n/*   MZ_PARAM_ERROR if one of the parameters is invalid. */\n/*   MZ_BUF_ERROR if no forward progress is possible because the input buffer is empty but the inflater needs more input to continue, or if the output buffer is not large enough. Call mz_inflate() again */\n/*   with more input data, or with more room in the output buffer (except when using single call decompression, described above). */\nint mz_inflate(mz_streamp pStream, int flush);\n\n/* Deinitializes a decompressor. */\nint mz_inflateEnd(mz_streamp pStream);\n\n/* Single-call decompression. */\n/* Returns MZ_OK on success, or one of the error codes from mz_inflate() on failure. */\nint mz_uncompress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len);\n\n/* Returns a string description of the specified error code, or NULL if the error code is invalid. */\nconst char *mz_error(int err);\n\n/* Redefine zlib-compatible names to miniz equivalents, so miniz.c can be used as a drop-in replacement for the subset of zlib that miniz.c supports. */\n/* Define MINIZ_NO_ZLIB_COMPATIBLE_NAMES to disable zlib-compatibility if you use zlib in the same project. */\n#ifndef MINIZ_NO_ZLIB_COMPATIBLE_NAMES\ntypedef unsigned char Byte;\ntypedef unsigned int uInt;\ntypedef mz_ulong uLong;\ntypedef Byte Bytef;\ntypedef uInt uIntf;\ntypedef char charf;\ntypedef int intf;\ntypedef void *voidpf;\ntypedef uLong uLongf;\ntypedef void *voidp;\ntypedef void *const voidpc;\n#define Z_NULL 0\n#define Z_NO_FLUSH MZ_NO_FLUSH\n#define Z_PARTIAL_FLUSH MZ_PARTIAL_FLUSH\n#define Z_SYNC_FLUSH MZ_SYNC_FLUSH\n#define Z_FULL_FLUSH MZ_FULL_FLUSH\n#define Z_FINISH MZ_FINISH\n#define Z_BLOCK MZ_BLOCK\n#define Z_OK MZ_OK\n#define Z_STREAM_END MZ_STREAM_END\n#define Z_NEED_DICT MZ_NEED_DICT\n#define Z_ERRNO MZ_ERRNO\n#define Z_STREAM_ERROR MZ_STREAM_ERROR\n#define Z_DATA_ERROR MZ_DATA_ERROR\n#define Z_MEM_ERROR MZ_MEM_ERROR\n#define Z_BUF_ERROR MZ_BUF_ERROR\n#define Z_VERSION_ERROR MZ_VERSION_ERROR\n#define Z_PARAM_ERROR MZ_PARAM_ERROR\n#define Z_NO_COMPRESSION MZ_NO_COMPRESSION\n#define Z_BEST_SPEED MZ_BEST_SPEED\n#define Z_BEST_COMPRESSION MZ_BEST_COMPRESSION\n#define Z_DEFAULT_COMPRESSION MZ_DEFAULT_COMPRESSION\n#define Z_DEFAULT_STRATEGY MZ_DEFAULT_STRATEGY\n#define Z_FILTERED MZ_FILTERED\n#define Z_HUFFMAN_ONLY MZ_HUFFMAN_ONLY\n#define Z_RLE MZ_RLE\n#define Z_FIXED MZ_FIXED\n#define Z_DEFLATED MZ_DEFLATED\n#define Z_DEFAULT_WINDOW_BITS MZ_DEFAULT_WINDOW_BITS\n#define alloc_func mz_alloc_func\n#define free_func mz_free_func\n#define internal_state mz_internal_state\n#define z_stream mz_stream\n#define deflateInit mz_deflateInit\n#define deflateInit2 mz_deflateInit2\n#define deflateReset mz_deflateReset\n#define deflate mz_deflate\n#define deflateEnd mz_deflateEnd\n#define deflateBound mz_deflateBound\n#define compress mz_compress\n#define compress2 mz_compress2\n#define compressBound mz_compressBound\n#define inflateInit mz_inflateInit\n#define inflateInit2 mz_inflateInit2\n#define inflateReset mz_inflateReset\n#define inflate mz_inflate\n#define inflateEnd mz_inflateEnd\n#define uncompress mz_uncompress\n#define crc32 mz_crc32\n#define adler32 mz_adler32\n#define MAX_WBITS 15\n#define MAX_MEM_LEVEL 9\n#define zError mz_error\n#define ZLIB_VERSION MZ_VERSION\n#define ZLIB_VERNUM MZ_VERNUM\n#define ZLIB_VER_MAJOR MZ_VER_MAJOR\n#define ZLIB_VER_MINOR MZ_VER_MINOR\n#define ZLIB_VER_REVISION MZ_VER_REVISION\n#define ZLIB_VER_SUBREVISION MZ_VER_SUBREVISION\n#define zlibVersion mz_version\n#define zlib_version mz_version()\n#endif /* #ifndef MINIZ_NO_ZLIB_COMPATIBLE_NAMES */\n\n#endif /* MINIZ_NO_ZLIB_APIS */\n\n#ifdef __cplusplus\n}\n#endif\n#pragma once\n#include <assert.h>\n#include <stdint.h>\n#include <stdlib.h>\n#include <string.h>\n\n/* ------------------- Types and macros */\ntypedef unsigned char mz_uint8;\ntypedef signed short mz_int16;\ntypedef unsigned short mz_uint16;\ntypedef unsigned int mz_uint32;\ntypedef unsigned int mz_uint;\ntypedef int64_t mz_int64;\ntypedef uint64_t mz_uint64;\ntypedef int mz_bool;\n\n#define MZ_FALSE (0)\n#define MZ_TRUE (1)\n\n/* Works around MSVC's spammy \"warning C4127: conditional expression is constant\" message. */\n#ifdef _MSC_VER\n#define MZ_MACRO_END while (0, 0)\n#else\n#define MZ_MACRO_END while (0)\n#endif\n\n#ifdef MINIZ_NO_STDIO\n#define MZ_FILE void *\n#else\n#include <stdio.h>\n#define MZ_FILE FILE\n#endif /* #ifdef MINIZ_NO_STDIO */\n\n#ifdef MINIZ_NO_TIME\ntypedef struct mz_dummy_time_t_tag\n{\n    int m_dummy;\n} mz_dummy_time_t;\n#define MZ_TIME_T mz_dummy_time_t\n#else\n#define MZ_TIME_T time_t\n#endif\n\n#define MZ_ASSERT(x) assert(x)\n\n#ifdef MINIZ_NO_MALLOC\n#define MZ_MALLOC(x) NULL\n#define MZ_FREE(x) (void)x, ((void)0)\n#define MZ_REALLOC(p, x) NULL\n#else\n#define MZ_MALLOC(x) malloc(x)\n#define MZ_FREE(x) free(x)\n#define MZ_REALLOC(p, x) realloc(p, x)\n#endif\n\n#define MZ_MAX(a, b) (((a) > (b)) ? (a) : (b))\n#define MZ_MIN(a, b) (((a) < (b)) ? (a) : (b))\n#define MZ_CLEAR_OBJ(obj) memset(&(obj), 0, sizeof(obj))\n\n#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN\n#define MZ_READ_LE16(p) *((const mz_uint16 *)(p))\n#define MZ_READ_LE32(p) *((const mz_uint32 *)(p))\n#else\n#define MZ_READ_LE16(p) ((mz_uint32)(((const mz_uint8 *)(p))[0]) | ((mz_uint32)(((const mz_uint8 *)(p))[1]) << 8U))\n#define MZ_READ_LE32(p) ((mz_uint32)(((const mz_uint8 *)(p))[0]) | ((mz_uint32)(((const mz_uint8 *)(p))[1]) << 8U) | ((mz_uint32)(((const mz_uint8 *)(p))[2]) << 16U) | ((mz_uint32)(((const mz_uint8 *)(p))[3]) << 24U))\n#endif\n\n#define MZ_READ_LE64(p) (((mz_uint64)MZ_READ_LE32(p)) | (((mz_uint64)MZ_READ_LE32((const mz_uint8 *)(p) + sizeof(mz_uint32))) << 32U))\n\n#ifdef _MSC_VER\n#define MZ_FORCEINLINE __forceinline\n#elif defined(__GNUC__)\n#define MZ_FORCEINLINE __inline__ __attribute__((__always_inline__))\n#else\n#define MZ_FORCEINLINE inline\n#endif\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nextern void *miniz_def_alloc_func(void *opaque, size_t items, size_t size);\nextern void miniz_def_free_func(void *opaque, void *address);\nextern void *miniz_def_realloc_func(void *opaque, void *address, size_t items, size_t size);\n\n#define MZ_UINT16_MAX (0xFFFFU)\n#define MZ_UINT32_MAX (0xFFFFFFFFU)\n\n#ifdef __cplusplus\n}\n#endif\n#pragma once\n\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n/* ------------------- Low-level Compression API Definitions */\n\n/* Set TDEFL_LESS_MEMORY to 1 to use less memory (compression will be slightly slower, and raw/dynamic blocks will be output more frequently). */\n#define TDEFL_LESS_MEMORY 0\n\n/* tdefl_init() compression flags logically OR'd together (low 12 bits contain the max. number of probes per dictionary search): */\n/* TDEFL_DEFAULT_MAX_PROBES: The compressor defaults to 128 dictionary probes per dictionary search. 0=Huffman only, 1=Huffman+LZ (fastest/crap compression), 4095=Huffman+LZ (slowest/best compression). */\nenum\n{\n    TDEFL_HUFFMAN_ONLY = 0,\n    TDEFL_DEFAULT_MAX_PROBES = 128,\n    TDEFL_MAX_PROBES_MASK = 0xFFF\n};\n\n/* TDEFL_WRITE_ZLIB_HEADER: If set, the compressor outputs a zlib header before the deflate data, and the Adler-32 of the source data at the end. Otherwise, you'll get raw deflate data. */\n/* TDEFL_COMPUTE_ADLER32: Always compute the adler-32 of the input data (even when not writing zlib headers). */\n/* TDEFL_GREEDY_PARSING_FLAG: Set to use faster greedy parsing, instead of more efficient lazy parsing. */\n/* TDEFL_NONDETERMINISTIC_PARSING_FLAG: Enable to decrease the compressor's initialization time to the minimum, but the output may vary from run to run given the same input (depending on the contents of memory). */\n/* TDEFL_RLE_MATCHES: Only look for RLE matches (matches with a distance of 1) */\n/* TDEFL_FILTER_MATCHES: Discards matches <= 5 chars if enabled. */\n/* TDEFL_FORCE_ALL_STATIC_BLOCKS: Disable usage of optimized Huffman tables. */\n/* TDEFL_FORCE_ALL_RAW_BLOCKS: Only use raw (uncompressed) deflate blocks. */\n/* The low 12 bits are reserved to control the max # of hash probes per dictionary lookup (see TDEFL_MAX_PROBES_MASK). */\nenum\n{\n    TDEFL_WRITE_ZLIB_HEADER = 0x01000,\n    TDEFL_COMPUTE_ADLER32 = 0x02000,\n    TDEFL_GREEDY_PARSING_FLAG = 0x04000,\n    TDEFL_NONDETERMINISTIC_PARSING_FLAG = 0x08000,\n    TDEFL_RLE_MATCHES = 0x10000,\n    TDEFL_FILTER_MATCHES = 0x20000,\n    TDEFL_FORCE_ALL_STATIC_BLOCKS = 0x40000,\n    TDEFL_FORCE_ALL_RAW_BLOCKS = 0x80000\n};\n\n/* High level compression functions: */\n/* tdefl_compress_mem_to_heap() compresses a block in memory to a heap block allocated via malloc(). */\n/* On entry: */\n/*  pSrc_buf, src_buf_len: Pointer and size of source block to compress. */\n/*  flags: The max match finder probes (default is 128) logically OR'd against the above flags. Higher probes are slower but improve compression. */\n/* On return: */\n/*  Function returns a pointer to the compressed data, or NULL on failure. */\n/*  *pOut_len will be set to the compressed data's size, which could be larger than src_buf_len on uncompressible data. */\n/*  The caller must free() the returned block when it's no longer needed. */\nvoid *tdefl_compress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, size_t *pOut_len, int flags);\n\n/* tdefl_compress_mem_to_mem() compresses a block in memory to another block in memory. */\n/* Returns 0 on failure. */\nsize_t tdefl_compress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void *pSrc_buf, size_t src_buf_len, int flags);\n\n/* Compresses an image to a compressed PNG file in memory. */\n/* On entry: */\n/*  pImage, w, h, and num_chans describe the image to compress. num_chans may be 1, 2, 3, or 4. */\n/*  The image pitch in bytes per scanline will be w*num_chans. The leftmost pixel on the top scanline is stored first in memory. */\n/*  level may range from [0,10], use MZ_NO_COMPRESSION, MZ_BEST_SPEED, MZ_BEST_COMPRESSION, etc. or a decent default is MZ_DEFAULT_LEVEL */\n/*  If flip is true, the image will be flipped on the Y axis (useful for OpenGL apps). */\n/* On return: */\n/*  Function returns a pointer to the compressed data, or NULL on failure. */\n/*  *pLen_out will be set to the size of the PNG image file. */\n/*  The caller must mz_free() the returned heap block (which will typically be larger than *pLen_out) when it's no longer needed. */\nvoid *tdefl_write_image_to_png_file_in_memory_ex(const void *pImage, int w, int h, int num_chans, size_t *pLen_out, mz_uint level, mz_bool flip);\nvoid *tdefl_write_image_to_png_file_in_memory(const void *pImage, int w, int h, int num_chans, size_t *pLen_out);\n\n/* Output stream interface. The compressor uses this interface to write compressed data. It'll typically be called TDEFL_OUT_BUF_SIZE at a time. */\ntypedef mz_bool (*tdefl_put_buf_func_ptr)(const void *pBuf, int len, void *pUser);\n\n/* tdefl_compress_mem_to_output() compresses a block to an output stream. The above helpers use this function internally. */\nmz_bool tdefl_compress_mem_to_output(const void *pBuf, size_t buf_len, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags);\n\nenum\n{\n    TDEFL_MAX_HUFF_TABLES = 3,\n    TDEFL_MAX_HUFF_SYMBOLS_0 = 288,\n    TDEFL_MAX_HUFF_SYMBOLS_1 = 32,\n    TDEFL_MAX_HUFF_SYMBOLS_2 = 19,\n    TDEFL_LZ_DICT_SIZE = 32768,\n    TDEFL_LZ_DICT_SIZE_MASK = TDEFL_LZ_DICT_SIZE - 1,\n    TDEFL_MIN_MATCH_LEN = 3,\n    TDEFL_MAX_MATCH_LEN = 258\n};\n\n/* TDEFL_OUT_BUF_SIZE MUST be large enough to hold a single entire compressed output block (using static/fixed Huffman codes). */\n#if TDEFL_LESS_MEMORY\nenum\n{\n    TDEFL_LZ_CODE_BUF_SIZE = 24 * 1024,\n    TDEFL_OUT_BUF_SIZE = (TDEFL_LZ_CODE_BUF_SIZE * 13) / 10,\n    TDEFL_MAX_HUFF_SYMBOLS = 288,\n    TDEFL_LZ_HASH_BITS = 12,\n    TDEFL_LEVEL1_HASH_SIZE_MASK = 4095,\n    TDEFL_LZ_HASH_SHIFT = (TDEFL_LZ_HASH_BITS + 2) / 3,\n    TDEFL_LZ_HASH_SIZE = 1 << TDEFL_LZ_HASH_BITS\n};\n#else\nenum\n{\n    TDEFL_LZ_CODE_BUF_SIZE = 64 * 1024,\n    TDEFL_OUT_BUF_SIZE = (TDEFL_LZ_CODE_BUF_SIZE * 13) / 10,\n    TDEFL_MAX_HUFF_SYMBOLS = 288,\n    TDEFL_LZ_HASH_BITS = 15,\n    TDEFL_LEVEL1_HASH_SIZE_MASK = 4095,\n    TDEFL_LZ_HASH_SHIFT = (TDEFL_LZ_HASH_BITS + 2) / 3,\n    TDEFL_LZ_HASH_SIZE = 1 << TDEFL_LZ_HASH_BITS\n};\n#endif\n\n/* The low-level tdefl functions below may be used directly if the above helper functions aren't flexible enough. The low-level functions don't make any heap allocations, unlike the above helper functions. */\ntypedef enum {\n    TDEFL_STATUS_BAD_PARAM = -2,\n    TDEFL_STATUS_PUT_BUF_FAILED = -1,\n    TDEFL_STATUS_OKAY = 0,\n    TDEFL_STATUS_DONE = 1\n} tdefl_status;\n\n/* Must map to MZ_NO_FLUSH, MZ_SYNC_FLUSH, etc. enums */\ntypedef enum {\n    TDEFL_NO_FLUSH = 0,\n    TDEFL_SYNC_FLUSH = 2,\n    TDEFL_FULL_FLUSH = 3,\n    TDEFL_FINISH = 4\n} tdefl_flush;\n\n/* tdefl's compression state structure. */\ntypedef struct\n{\n    tdefl_put_buf_func_ptr m_pPut_buf_func;\n    void *m_pPut_buf_user;\n    mz_uint m_flags, m_max_probes[2];\n    int m_greedy_parsing;\n    mz_uint m_adler32, m_lookahead_pos, m_lookahead_size, m_dict_size;\n    mz_uint8 *m_pLZ_code_buf, *m_pLZ_flags, *m_pOutput_buf, *m_pOutput_buf_end;\n    mz_uint m_num_flags_left, m_total_lz_bytes, m_lz_code_buf_dict_pos, m_bits_in, m_bit_buffer;\n    mz_uint m_saved_match_dist, m_saved_match_len, m_saved_lit, m_output_flush_ofs, m_output_flush_remaining, m_finished, m_block_index, m_wants_to_finish;\n    tdefl_status m_prev_return_status;\n    const void *m_pIn_buf;\n    void *m_pOut_buf;\n    size_t *m_pIn_buf_size, *m_pOut_buf_size;\n    tdefl_flush m_flush;\n    const mz_uint8 *m_pSrc;\n    size_t m_src_buf_left, m_out_buf_ofs;\n    mz_uint8 m_dict[TDEFL_LZ_DICT_SIZE + TDEFL_MAX_MATCH_LEN - 1];\n    mz_uint16 m_huff_count[TDEFL_MAX_HUFF_TABLES][TDEFL_MAX_HUFF_SYMBOLS];\n    mz_uint16 m_huff_codes[TDEFL_MAX_HUFF_TABLES][TDEFL_MAX_HUFF_SYMBOLS];\n    mz_uint8 m_huff_code_sizes[TDEFL_MAX_HUFF_TABLES][TDEFL_MAX_HUFF_SYMBOLS];\n    mz_uint8 m_lz_code_buf[TDEFL_LZ_CODE_BUF_SIZE];\n    mz_uint16 m_next[TDEFL_LZ_DICT_SIZE];\n    mz_uint16 m_hash[TDEFL_LZ_HASH_SIZE];\n    mz_uint8 m_output_buf[TDEFL_OUT_BUF_SIZE];\n} tdefl_compressor;\n\n/* Initializes the compressor. */\n/* There is no corresponding deinit() function because the tdefl API's do not dynamically allocate memory. */\n/* pBut_buf_func: If NULL, output data will be supplied to the specified callback. In this case, the user should call the tdefl_compress_buffer() API for compression. */\n/* If pBut_buf_func is NULL the user should always call the tdefl_compress() API. */\n/* flags: See the above enums (TDEFL_HUFFMAN_ONLY, TDEFL_WRITE_ZLIB_HEADER, etc.) */\ntdefl_status tdefl_init(tdefl_compressor *d, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags);\n\n/* Compresses a block of data, consuming as much of the specified input buffer as possible, and writing as much compressed data to the specified output buffer as possible. */\ntdefl_status tdefl_compress(tdefl_compressor *d, const void *pIn_buf, size_t *pIn_buf_size, void *pOut_buf, size_t *pOut_buf_size, tdefl_flush flush);\n\n/* tdefl_compress_buffer() is only usable when the tdefl_init() is called with a non-NULL tdefl_put_buf_func_ptr. */\n/* tdefl_compress_buffer() always consumes the entire input buffer. */\ntdefl_status tdefl_compress_buffer(tdefl_compressor *d, const void *pIn_buf, size_t in_buf_size, tdefl_flush flush);\n\ntdefl_status tdefl_get_prev_return_status(tdefl_compressor *d);\nmz_uint32 tdefl_get_adler32(tdefl_compressor *d);\n\n/* Create tdefl_compress() flags given zlib-style compression parameters. */\n/* level may range from [0,10] (where 10 is absolute max compression, but may be much slower on some files) */\n/* window_bits may be -15 (raw deflate) or 15 (zlib) */\n/* strategy may be either MZ_DEFAULT_STRATEGY, MZ_FILTERED, MZ_HUFFMAN_ONLY, MZ_RLE, or MZ_FIXED */\nmz_uint tdefl_create_comp_flags_from_zip_params(int level, int window_bits, int strategy);\n\n#ifndef MINIZ_NO_MALLOC\n/* Allocate the tdefl_compressor structure in C so that */\n/* non-C language bindings to tdefl_ API don't need to worry about */\n/* structure size and allocation mechanism. */\ntdefl_compressor *tdefl_compressor_alloc(void);\nvoid tdefl_compressor_free(tdefl_compressor *pComp);\n#endif\n\n#ifdef __cplusplus\n}\n#endif\n#pragma once\n\n/* ------------------- Low-level Decompression API Definitions */\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n/* Decompression flags used by tinfl_decompress(). */\n/* TINFL_FLAG_PARSE_ZLIB_HEADER: If set, the input has a valid zlib header and ends with an adler32 checksum (it's a valid zlib stream). Otherwise, the input is a raw deflate stream. */\n/* TINFL_FLAG_HAS_MORE_INPUT: If set, there are more input bytes available beyond the end of the supplied input buffer. If clear, the input buffer contains all remaining input. */\n/* TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF: If set, the output buffer is large enough to hold the entire decompressed stream. If clear, the output buffer is at least the size of the dictionary (typically 32KB). */\n/* TINFL_FLAG_COMPUTE_ADLER32: Force adler-32 checksum computation of the decompressed bytes. */\nenum\n{\n    TINFL_FLAG_PARSE_ZLIB_HEADER = 1,\n    TINFL_FLAG_HAS_MORE_INPUT = 2,\n    TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF = 4,\n    TINFL_FLAG_COMPUTE_ADLER32 = 8\n};\n\n/* High level decompression functions: */\n/* tinfl_decompress_mem_to_heap() decompresses a block in memory to a heap block allocated via malloc(). */\n/* On entry: */\n/*  pSrc_buf, src_buf_len: Pointer and size of the Deflate or zlib source data to decompress. */\n/* On return: */\n/*  Function returns a pointer to the decompressed data, or NULL on failure. */\n/*  *pOut_len will be set to the decompressed data's size, which could be larger than src_buf_len on uncompressible data. */\n/*  The caller must call mz_free() on the returned block when it's no longer needed. */\nvoid *tinfl_decompress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, size_t *pOut_len, int flags);\n\n/* tinfl_decompress_mem_to_mem() decompresses a block in memory to another block in memory. */\n/* Returns TINFL_DECOMPRESS_MEM_TO_MEM_FAILED on failure, or the number of bytes written on success. */\n#define TINFL_DECOMPRESS_MEM_TO_MEM_FAILED ((size_t)(-1))\nsize_t tinfl_decompress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void *pSrc_buf, size_t src_buf_len, int flags);\n\n/* tinfl_decompress_mem_to_callback() decompresses a block in memory to an internal 32KB buffer, and a user provided callback function will be called to flush the buffer. */\n/* Returns 1 on success or 0 on failure. */\ntypedef int (*tinfl_put_buf_func_ptr)(const void *pBuf, int len, void *pUser);\nint tinfl_decompress_mem_to_callback(const void *pIn_buf, size_t *pIn_buf_size, tinfl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags);\n\nstruct tinfl_decompressor_tag;\ntypedef struct tinfl_decompressor_tag tinfl_decompressor;\n\n#ifndef MINIZ_NO_MALLOC\n/* Allocate the tinfl_decompressor structure in C so that */\n/* non-C language bindings to tinfl_ API don't need to worry about */\n/* structure size and allocation mechanism. */\ntinfl_decompressor *tinfl_decompressor_alloc(void);\nvoid tinfl_decompressor_free(tinfl_decompressor *pDecomp);\n#endif\n\n/* Max size of LZ dictionary. */\n#define TINFL_LZ_DICT_SIZE 32768\n\n/* Return status. */\ntypedef enum {\n    /* This flags indicates the inflator needs 1 or more input bytes to make forward progress, but the caller is indicating that no more are available. The compressed data */\n    /* is probably corrupted. If you call the inflator again with more bytes it'll try to continue processing the input but this is a BAD sign (either the data is corrupted or you called it incorrectly). */\n    /* If you call it again with no input you'll just get TINFL_STATUS_FAILED_CANNOT_MAKE_PROGRESS again. */\n    TINFL_STATUS_FAILED_CANNOT_MAKE_PROGRESS = -4,\n\n    /* This flag indicates that one or more of the input parameters was obviously bogus. (You can try calling it again, but if you get this error the calling code is wrong.) */\n    TINFL_STATUS_BAD_PARAM = -3,\n\n    /* This flags indicate the inflator is finished but the adler32 check of the uncompressed data didn't match. If you call it again it'll return TINFL_STATUS_DONE. */\n    TINFL_STATUS_ADLER32_MISMATCH = -2,\n\n    /* This flags indicate the inflator has somehow failed (bad code, corrupted input, etc.). If you call it again without resetting via tinfl_init() it it'll just keep on returning the same status failure code. */\n    TINFL_STATUS_FAILED = -1,\n\n    /* Any status code less than TINFL_STATUS_DONE must indicate a failure. */\n\n    /* This flag indicates the inflator has returned every byte of uncompressed data that it can, has consumed every byte that it needed, has successfully reached the end of the deflate stream, and */\n    /* if zlib headers and adler32 checking enabled that it has successfully checked the uncompressed data's adler32. If you call it again you'll just get TINFL_STATUS_DONE over and over again. */\n    TINFL_STATUS_DONE = 0,\n\n    /* This flag indicates the inflator MUST have more input data (even 1 byte) before it can make any more forward progress, or you need to clear the TINFL_FLAG_HAS_MORE_INPUT */\n    /* flag on the next call if you don't have any more source data. If the source data was somehow corrupted it's also possible (but unlikely) for the inflator to keep on demanding input to */\n    /* proceed, so be sure to properly set the TINFL_FLAG_HAS_MORE_INPUT flag. */\n    TINFL_STATUS_NEEDS_MORE_INPUT = 1,\n\n    /* This flag indicates the inflator definitely has 1 or more bytes of uncompressed data available, but it cannot write this data into the output buffer. */\n    /* Note if the source compressed data was corrupted it's possible for the inflator to return a lot of uncompressed data to the caller. I've been assuming you know how much uncompressed data to expect */\n    /* (either exact or worst case) and will stop calling the inflator and fail after receiving too much. In pure streaming scenarios where you have no idea how many bytes to expect this may not be possible */\n    /* so I may need to add some code to address this. */\n    TINFL_STATUS_HAS_MORE_OUTPUT = 2\n} tinfl_status;\n\n/* Initializes the decompressor to its initial state. */\n#define tinfl_init(r)     \\\n    do                    \\\n    {                     \\\n        (r)->m_state = 0; \\\n    }                     \\\n    MZ_MACRO_END\n#define tinfl_get_adler32(r) (r)->m_check_adler32\n\n/* Main low-level decompressor coroutine function. This is the only function actually needed for decompression. All the other functions are just high-level helpers for improved usability. */\n/* This is a universal API, i.e. it can be used as a building block to build any desired higher level decompression API. In the limit case, it can be called once per every byte input or output. */\ntinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_next, size_t *pIn_buf_size, mz_uint8 *pOut_buf_start, mz_uint8 *pOut_buf_next, size_t *pOut_buf_size, const mz_uint32 decomp_flags);\n\n/* Internal/private bits follow. */\nenum\n{\n    TINFL_MAX_HUFF_TABLES = 3,\n    TINFL_MAX_HUFF_SYMBOLS_0 = 288,\n    TINFL_MAX_HUFF_SYMBOLS_1 = 32,\n    TINFL_MAX_HUFF_SYMBOLS_2 = 19,\n    TINFL_FAST_LOOKUP_BITS = 10,\n    TINFL_FAST_LOOKUP_SIZE = 1 << TINFL_FAST_LOOKUP_BITS\n};\n\ntypedef struct\n{\n    mz_uint8 m_code_size[TINFL_MAX_HUFF_SYMBOLS_0];\n    mz_int16 m_look_up[TINFL_FAST_LOOKUP_SIZE], m_tree[TINFL_MAX_HUFF_SYMBOLS_0 * 2];\n} tinfl_huff_table;\n\n#if MINIZ_HAS_64BIT_REGISTERS\n#define TINFL_USE_64BIT_BITBUF 1\n#else\n#define TINFL_USE_64BIT_BITBUF 0\n#endif\n\n#if TINFL_USE_64BIT_BITBUF\ntypedef mz_uint64 tinfl_bit_buf_t;\n#define TINFL_BITBUF_SIZE (64)\n#else\ntypedef mz_uint32 tinfl_bit_buf_t;\n#define TINFL_BITBUF_SIZE (32)\n#endif\n\nstruct tinfl_decompressor_tag\n{\n    mz_uint32 m_state, m_num_bits, m_zhdr0, m_zhdr1, m_z_adler32, m_final, m_type, m_check_adler32, m_dist, m_counter, m_num_extra, m_table_sizes[TINFL_MAX_HUFF_TABLES];\n    tinfl_bit_buf_t m_bit_buf;\n    size_t m_dist_from_out_buf_start;\n    tinfl_huff_table m_tables[TINFL_MAX_HUFF_TABLES];\n    mz_uint8 m_raw_header[4], m_len_codes[TINFL_MAX_HUFF_SYMBOLS_0 + TINFL_MAX_HUFF_SYMBOLS_1 + 137];\n};\n\n#ifdef __cplusplus\n}\n#endif\n\n#pragma once\n\n\n/* ------------------- ZIP archive reading/writing */\n\n#ifndef MINIZ_NO_ARCHIVE_APIS\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nenum\n{\n    /* Note: These enums can be reduced as needed to save memory or stack space - they are pretty conservative. */\n    MZ_ZIP_MAX_IO_BUF_SIZE = 64 * 1024,\n    MZ_ZIP_MAX_ARCHIVE_FILENAME_SIZE = 512,\n    MZ_ZIP_MAX_ARCHIVE_FILE_COMMENT_SIZE = 512\n};\n\ntypedef struct\n{\n    /* Central directory file index. */\n    mz_uint32 m_file_index;\n\n    /* Byte offset of this entry in the archive's central directory. Note we currently only support up to UINT_MAX or less bytes in the central dir. */\n    mz_uint64 m_central_dir_ofs;\n\n    /* These fields are copied directly from the zip's central dir. */\n    mz_uint16 m_version_made_by;\n    mz_uint16 m_version_needed;\n    mz_uint16 m_bit_flag;\n    mz_uint16 m_method;\n\n#ifndef MINIZ_NO_TIME\n    MZ_TIME_T m_time;\n#endif\n\n    /* CRC-32 of uncompressed data. */\n    mz_uint32 m_crc32;\n\n    /* File's compressed size. */\n    mz_uint64 m_comp_size;\n\n    /* File's uncompressed size. Note, I've seen some old archives where directory entries had 512 bytes for their uncompressed sizes, but when you try to unpack them you actually get 0 bytes. */\n    mz_uint64 m_uncomp_size;\n\n    /* Zip internal and external file attributes. */\n    mz_uint16 m_internal_attr;\n    mz_uint32 m_external_attr;\n\n    /* Entry's local header file offset in bytes. */\n    mz_uint64 m_local_header_ofs;\n\n    /* Size of comment in bytes. */\n    mz_uint32 m_comment_size;\n\n    /* MZ_TRUE if the entry appears to be a directory. */\n    mz_bool m_is_directory;\n\n    /* MZ_TRUE if the entry uses encryption/strong encryption (which miniz_zip doesn't support) */\n    mz_bool m_is_encrypted;\n\n    /* MZ_TRUE if the file is not encrypted, a patch file, and if it uses a compression method we support. */\n    mz_bool m_is_supported;\n\n    /* Filename. If string ends in '/' it's a subdirectory entry. */\n    /* Guaranteed to be zero terminated, may be truncated to fit. */\n    char m_filename[MZ_ZIP_MAX_ARCHIVE_FILENAME_SIZE];\n\n    /* Comment field. */\n    /* Guaranteed to be zero terminated, may be truncated to fit. */\n    char m_comment[MZ_ZIP_MAX_ARCHIVE_FILE_COMMENT_SIZE];\n\n} mz_zip_archive_file_stat;\n\ntypedef size_t (*mz_file_read_func)(void *pOpaque, mz_uint64 file_ofs, void *pBuf, size_t n);\ntypedef size_t (*mz_file_write_func)(void *pOpaque, mz_uint64 file_ofs, const void *pBuf, size_t n);\ntypedef mz_bool (*mz_file_needs_keepalive)(void *pOpaque);\n\nstruct mz_zip_internal_state_tag;\ntypedef struct mz_zip_internal_state_tag mz_zip_internal_state;\n\ntypedef enum {\n    MZ_ZIP_MODE_INVALID = 0,\n    MZ_ZIP_MODE_READING = 1,\n    MZ_ZIP_MODE_WRITING = 2,\n    MZ_ZIP_MODE_WRITING_HAS_BEEN_FINALIZED = 3\n} mz_zip_mode;\n\ntypedef enum {\n    MZ_ZIP_FLAG_CASE_SENSITIVE = 0x0100,\n    MZ_ZIP_FLAG_IGNORE_PATH = 0x0200,\n    MZ_ZIP_FLAG_COMPRESSED_DATA = 0x0400,\n    MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY = 0x0800,\n    MZ_ZIP_FLAG_VALIDATE_LOCATE_FILE_FLAG = 0x1000, /* if enabled, mz_zip_reader_locate_file() will be called on each file as its validated to ensure the func finds the file in the central dir (intended for testing) */\n    MZ_ZIP_FLAG_VALIDATE_HEADERS_ONLY = 0x2000,     /* validate the local headers, but don't decompress the entire file and check the crc32 */\n    MZ_ZIP_FLAG_WRITE_ZIP64 = 0x4000,               /* always use the zip64 file format, instead of the original zip file format with automatic switch to zip64. Use as flags parameter with mz_zip_writer_init*_v2 */\n    MZ_ZIP_FLAG_WRITE_ALLOW_READING = 0x8000,\n    MZ_ZIP_FLAG_ASCII_FILENAME = 0x10000\n} mz_zip_flags;\n\ntypedef enum {\n    MZ_ZIP_TYPE_INVALID = 0,\n    MZ_ZIP_TYPE_USER,\n    MZ_ZIP_TYPE_MEMORY,\n    MZ_ZIP_TYPE_HEAP,\n    MZ_ZIP_TYPE_FILE,\n    MZ_ZIP_TYPE_CFILE,\n    MZ_ZIP_TOTAL_TYPES\n} mz_zip_type;\n\n/* miniz error codes. Be sure to update mz_zip_get_error_string() if you add or modify this enum. */\ntypedef enum {\n    MZ_ZIP_NO_ERROR = 0,\n    MZ_ZIP_UNDEFINED_ERROR,\n    MZ_ZIP_TOO_MANY_FILES,\n    MZ_ZIP_FILE_TOO_LARGE,\n    MZ_ZIP_UNSUPPORTED_METHOD,\n    MZ_ZIP_UNSUPPORTED_ENCRYPTION,\n    MZ_ZIP_UNSUPPORTED_FEATURE,\n    MZ_ZIP_FAILED_FINDING_CENTRAL_DIR,\n    MZ_ZIP_NOT_AN_ARCHIVE,\n    MZ_ZIP_INVALID_HEADER_OR_CORRUPTED,\n    MZ_ZIP_UNSUPPORTED_MULTIDISK,\n    MZ_ZIP_DECOMPRESSION_FAILED,\n    MZ_ZIP_COMPRESSION_FAILED,\n    MZ_ZIP_UNEXPECTED_DECOMPRESSED_SIZE,\n    MZ_ZIP_CRC_CHECK_FAILED,\n    MZ_ZIP_UNSUPPORTED_CDIR_SIZE,\n    MZ_ZIP_ALLOC_FAILED,\n    MZ_ZIP_FILE_OPEN_FAILED,\n    MZ_ZIP_FILE_CREATE_FAILED,\n    MZ_ZIP_FILE_WRITE_FAILED,\n    MZ_ZIP_FILE_READ_FAILED,\n    MZ_ZIP_FILE_CLOSE_FAILED,\n    MZ_ZIP_FILE_SEEK_FAILED,\n    MZ_ZIP_FILE_STAT_FAILED,\n    MZ_ZIP_INVALID_PARAMETER,\n    MZ_ZIP_INVALID_FILENAME,\n    MZ_ZIP_BUF_TOO_SMALL,\n    MZ_ZIP_INTERNAL_ERROR,\n    MZ_ZIP_FILE_NOT_FOUND,\n    MZ_ZIP_ARCHIVE_TOO_LARGE,\n    MZ_ZIP_VALIDATION_FAILED,\n    MZ_ZIP_WRITE_CALLBACK_FAILED,\n    MZ_ZIP_TOTAL_ERRORS\n} mz_zip_error;\n\ntypedef struct\n{\n    mz_uint64 m_archive_size;\n    mz_uint64 m_central_directory_file_ofs;\n\n    /* We only support up to UINT32_MAX files in zip64 mode. */\n    mz_uint32 m_total_files;\n    mz_zip_mode m_zip_mode;\n    mz_zip_type m_zip_type;\n    mz_zip_error m_last_error;\n\n    mz_uint64 m_file_offset_alignment;\n\n    mz_alloc_func m_pAlloc;\n    mz_free_func m_pFree;\n    mz_realloc_func m_pRealloc;\n    void *m_pAlloc_opaque;\n\n    mz_file_read_func m_pRead;\n    mz_file_write_func m_pWrite;\n    mz_file_needs_keepalive m_pNeeds_keepalive;\n    void *m_pIO_opaque;\n\n    mz_zip_internal_state *m_pState;\n\n} mz_zip_archive;\n\ntypedef struct\n{\n    mz_zip_archive *pZip;\n    mz_uint flags;\n\n    int status;\n#ifndef MINIZ_DISABLE_ZIP_READER_CRC32_CHECKS\n    mz_uint file_crc32;\n#endif\n    mz_uint64 read_buf_size, read_buf_ofs, read_buf_avail, comp_remaining, out_buf_ofs, cur_file_ofs;\n    mz_zip_archive_file_stat file_stat;\n    void *pRead_buf;\n    void *pWrite_buf;\n\n    size_t out_blk_remain;\n\n    tinfl_decompressor inflator;\n\n} mz_zip_reader_extract_iter_state;\n\n/* -------- ZIP reading */\n\n/* Inits a ZIP archive reader. */\n/* These functions read and validate the archive's central directory. */\nmz_bool mz_zip_reader_init(mz_zip_archive *pZip, mz_uint64 size, mz_uint flags);\n\nmz_bool mz_zip_reader_init_mem(mz_zip_archive *pZip, const void *pMem, size_t size, mz_uint flags);\n\n#ifndef MINIZ_NO_STDIO\n/* Read a archive from a disk file. */\n/* file_start_ofs is the file offset where the archive actually begins, or 0. */\n/* actual_archive_size is the true total size of the archive, which may be smaller than the file's actual size on disk. If zero the entire file is treated as the archive. */\nmz_bool mz_zip_reader_init_file(mz_zip_archive *pZip, const char *pFilename, mz_uint32 flags);\nmz_bool mz_zip_reader_init_file_v2(mz_zip_archive *pZip, const char *pFilename, mz_uint flags, mz_uint64 file_start_ofs, mz_uint64 archive_size);\n\n/* Read an archive from an already opened FILE, beginning at the current file position. */\n/* The archive is assumed to be archive_size bytes long. If archive_size is < 0, then the entire rest of the file is assumed to contain the archive. */\n/* The FILE will NOT be closed when mz_zip_reader_end() is called. */\nmz_bool mz_zip_reader_init_cfile(mz_zip_archive *pZip, MZ_FILE *pFile, mz_uint64 archive_size, mz_uint flags);\n#endif\n\n/* Ends archive reading, freeing all allocations, and closing the input archive file if mz_zip_reader_init_file() was used. */\nmz_bool mz_zip_reader_end(mz_zip_archive *pZip);\n\n/* -------- ZIP reading or writing */\n\n/* Clears a mz_zip_archive struct to all zeros. */\n/* Important: This must be done before passing the struct to any mz_zip functions. */\nvoid mz_zip_zero_struct(mz_zip_archive *pZip);\n\nmz_zip_mode mz_zip_get_mode(mz_zip_archive *pZip);\nmz_zip_type mz_zip_get_type(mz_zip_archive *pZip);\n\n/* Returns the total number of files in the archive. */\nmz_uint mz_zip_reader_get_num_files(mz_zip_archive *pZip);\n\nmz_uint64 mz_zip_get_archive_size(mz_zip_archive *pZip);\nmz_uint64 mz_zip_get_archive_file_start_offset(mz_zip_archive *pZip);\nMZ_FILE *mz_zip_get_cfile(mz_zip_archive *pZip);\n\n/* Reads n bytes of raw archive data, starting at file offset file_ofs, to pBuf. */\nsize_t mz_zip_read_archive_data(mz_zip_archive *pZip, mz_uint64 file_ofs, void *pBuf, size_t n);\n\n/* All mz_zip funcs set the m_last_error field in the mz_zip_archive struct. These functions retrieve/manipulate this field. */\n/* Note that the m_last_error functionality is not thread safe. */\nmz_zip_error mz_zip_set_last_error(mz_zip_archive *pZip, mz_zip_error err_num);\nmz_zip_error mz_zip_peek_last_error(mz_zip_archive *pZip);\nmz_zip_error mz_zip_clear_last_error(mz_zip_archive *pZip);\nmz_zip_error mz_zip_get_last_error(mz_zip_archive *pZip);\nconst char *mz_zip_get_error_string(mz_zip_error mz_err);\n\n/* MZ_TRUE if the archive file entry is a directory entry. */\nmz_bool mz_zip_reader_is_file_a_directory(mz_zip_archive *pZip, mz_uint file_index);\n\n/* MZ_TRUE if the file is encrypted/strong encrypted. */\nmz_bool mz_zip_reader_is_file_encrypted(mz_zip_archive *pZip, mz_uint file_index);\n\n/* MZ_TRUE if the compression method is supported, and the file is not encrypted, and the file is not a compressed patch file. */\nmz_bool mz_zip_reader_is_file_supported(mz_zip_archive *pZip, mz_uint file_index);\n\n/* Retrieves the filename of an archive file entry. */\n/* Returns the number of bytes written to pFilename, or if filename_buf_size is 0 this function returns the number of bytes needed to fully store the filename. */\nmz_uint mz_zip_reader_get_filename(mz_zip_archive *pZip, mz_uint file_index, char *pFilename, mz_uint filename_buf_size);\n\n/* Attempts to locates a file in the archive's central directory. */\n/* Valid flags: MZ_ZIP_FLAG_CASE_SENSITIVE, MZ_ZIP_FLAG_IGNORE_PATH */\n/* Returns -1 if the file cannot be found. */\nint mz_zip_reader_locate_file(mz_zip_archive *pZip, const char *pName, const char *pComment, mz_uint flags);\nint mz_zip_reader_locate_file_v2(mz_zip_archive *pZip, const char *pName, const char *pComment, mz_uint flags, mz_uint32 *file_index);\n\n/* Returns detailed information about an archive file entry. */\nmz_bool mz_zip_reader_file_stat(mz_zip_archive *pZip, mz_uint file_index, mz_zip_archive_file_stat *pStat);\n\n/* MZ_TRUE if the file is in zip64 format. */\n/* A file is considered zip64 if it contained a zip64 end of central directory marker, or if it contained any zip64 extended file information fields in the central directory. */\nmz_bool mz_zip_is_zip64(mz_zip_archive *pZip);\n\n/* Returns the total central directory size in bytes. */\n/* The current max supported size is <= MZ_UINT32_MAX. */\nsize_t mz_zip_get_central_dir_size(mz_zip_archive *pZip);\n\n/* Extracts a archive file to a memory buffer using no memory allocation. */\n/* There must be at least enough room on the stack to store the inflator's state (~34KB or so). */\nmz_bool mz_zip_reader_extract_to_mem_no_alloc(mz_zip_archive *pZip, mz_uint file_index, void *pBuf, size_t buf_size, mz_uint flags, void *pUser_read_buf, size_t user_read_buf_size);\nmz_bool mz_zip_reader_extract_file_to_mem_no_alloc(mz_zip_archive *pZip, const char *pFilename, void *pBuf, size_t buf_size, mz_uint flags, void *pUser_read_buf, size_t user_read_buf_size);\n\n/* Extracts a archive file to a memory buffer. */\nmz_bool mz_zip_reader_extract_to_mem(mz_zip_archive *pZip, mz_uint file_index, void *pBuf, size_t buf_size, mz_uint flags);\nmz_bool mz_zip_reader_extract_file_to_mem(mz_zip_archive *pZip, const char *pFilename, void *pBuf, size_t buf_size, mz_uint flags);\n\n/* Extracts a archive file to a dynamically allocated heap buffer. */\n/* The memory will be allocated via the mz_zip_archive's alloc/realloc functions. */\n/* Returns NULL and sets the last error on failure. */\nvoid *mz_zip_reader_extract_to_heap(mz_zip_archive *pZip, mz_uint file_index, size_t *pSize, mz_uint flags);\nvoid *mz_zip_reader_extract_file_to_heap(mz_zip_archive *pZip, const char *pFilename, size_t *pSize, mz_uint flags);\n\n/* Extracts a archive file using a callback function to output the file's data. */\nmz_bool mz_zip_reader_extract_to_callback(mz_zip_archive *pZip, mz_uint file_index, mz_file_write_func pCallback, void *pOpaque, mz_uint flags);\nmz_bool mz_zip_reader_extract_file_to_callback(mz_zip_archive *pZip, const char *pFilename, mz_file_write_func pCallback, void *pOpaque, mz_uint flags);\n\n/* Extract a file iteratively */\nmz_zip_reader_extract_iter_state* mz_zip_reader_extract_iter_new(mz_zip_archive *pZip, mz_uint file_index, mz_uint flags);\nmz_zip_reader_extract_iter_state* mz_zip_reader_extract_file_iter_new(mz_zip_archive *pZip, const char *pFilename, mz_uint flags);\nsize_t mz_zip_reader_extract_iter_read(mz_zip_reader_extract_iter_state* pState, void* pvBuf, size_t buf_size);\nmz_bool mz_zip_reader_extract_iter_free(mz_zip_reader_extract_iter_state* pState);\n\n#ifndef MINIZ_NO_STDIO\n/* Extracts a archive file to a disk file and sets its last accessed and modified times. */\n/* This function only extracts files, not archive directory records. */\nmz_bool mz_zip_reader_extract_to_file(mz_zip_archive *pZip, mz_uint file_index, const char *pDst_filename, mz_uint flags);\nmz_bool mz_zip_reader_extract_file_to_file(mz_zip_archive *pZip, const char *pArchive_filename, const char *pDst_filename, mz_uint flags);\n\n/* Extracts a archive file starting at the current position in the destination FILE stream. */\nmz_bool mz_zip_reader_extract_to_cfile(mz_zip_archive *pZip, mz_uint file_index, MZ_FILE *File, mz_uint flags);\nmz_bool mz_zip_reader_extract_file_to_cfile(mz_zip_archive *pZip, const char *pArchive_filename, MZ_FILE *pFile, mz_uint flags);\n#endif\n\n#if 0\n/* TODO */\n\ttypedef void *mz_zip_streaming_extract_state_ptr;\n\tmz_zip_streaming_extract_state_ptr mz_zip_streaming_extract_begin(mz_zip_archive *pZip, mz_uint file_index, mz_uint flags);\n\tuint64_t mz_zip_streaming_extract_get_size(mz_zip_archive *pZip, mz_zip_streaming_extract_state_ptr pState);\n\tuint64_t mz_zip_streaming_extract_get_cur_ofs(mz_zip_archive *pZip, mz_zip_streaming_extract_state_ptr pState);\n\tmz_bool mz_zip_streaming_extract_seek(mz_zip_archive *pZip, mz_zip_streaming_extract_state_ptr pState, uint64_t new_ofs);\n\tsize_t mz_zip_streaming_extract_read(mz_zip_archive *pZip, mz_zip_streaming_extract_state_ptr pState, void *pBuf, size_t buf_size);\n\tmz_bool mz_zip_streaming_extract_end(mz_zip_archive *pZip, mz_zip_streaming_extract_state_ptr pState);\n#endif\n\n/* This function compares the archive's local headers, the optional local zip64 extended information block, and the optional descriptor following the compressed data vs. the data in the central directory. */\n/* It also validates that each file can be successfully uncompressed unless the MZ_ZIP_FLAG_VALIDATE_HEADERS_ONLY is specified. */\nmz_bool mz_zip_validate_file(mz_zip_archive *pZip, mz_uint file_index, mz_uint flags);\n\n/* Validates an entire archive by calling mz_zip_validate_file() on each file. */\nmz_bool mz_zip_validate_archive(mz_zip_archive *pZip, mz_uint flags);\n\n/* Misc utils/helpers, valid for ZIP reading or writing */\nmz_bool mz_zip_validate_mem_archive(const void *pMem, size_t size, mz_uint flags, mz_zip_error *pErr);\nmz_bool mz_zip_validate_file_archive(const char *pFilename, mz_uint flags, mz_zip_error *pErr);\n\n/* Universal end function - calls either mz_zip_reader_end() or mz_zip_writer_end(). */\nmz_bool mz_zip_end(mz_zip_archive *pZip);\n\n/* -------- ZIP writing */\n\n#ifndef MINIZ_NO_ARCHIVE_WRITING_APIS\n\n/* Inits a ZIP archive writer. */\n/*Set pZip->m_pWrite (and pZip->m_pIO_opaque) before calling mz_zip_writer_init or mz_zip_writer_init_v2*/\n/*The output is streamable, i.e. file_ofs in mz_file_write_func always increases only by n*/\nmz_bool mz_zip_writer_init(mz_zip_archive *pZip, mz_uint64 existing_size);\nmz_bool mz_zip_writer_init_v2(mz_zip_archive *pZip, mz_uint64 existing_size, mz_uint flags);\n\nmz_bool mz_zip_writer_init_heap(mz_zip_archive *pZip, size_t size_to_reserve_at_beginning, size_t initial_allocation_size);\nmz_bool mz_zip_writer_init_heap_v2(mz_zip_archive *pZip, size_t size_to_reserve_at_beginning, size_t initial_allocation_size, mz_uint flags);\n\n#ifndef MINIZ_NO_STDIO\nmz_bool mz_zip_writer_init_file(mz_zip_archive *pZip, const char *pFilename, mz_uint64 size_to_reserve_at_beginning);\nmz_bool mz_zip_writer_init_file_v2(mz_zip_archive *pZip, const char *pFilename, mz_uint64 size_to_reserve_at_beginning, mz_uint flags);\nmz_bool mz_zip_writer_init_cfile(mz_zip_archive *pZip, MZ_FILE *pFile, mz_uint flags);\n#endif\n\n/* Converts a ZIP archive reader object into a writer object, to allow efficient in-place file appends to occur on an existing archive. */\n/* For archives opened using mz_zip_reader_init_file, pFilename must be the archive's filename so it can be reopened for writing. If the file can't be reopened, mz_zip_reader_end() will be called. */\n/* For archives opened using mz_zip_reader_init_mem, the memory block must be growable using the realloc callback (which defaults to realloc unless you've overridden it). */\n/* Finally, for archives opened using mz_zip_reader_init, the mz_zip_archive's user provided m_pWrite function cannot be NULL. */\n/* Note: In-place archive modification is not recommended unless you know what you're doing, because if execution stops or something goes wrong before */\n/* the archive is finalized the file's central directory will be hosed. */\nmz_bool mz_zip_writer_init_from_reader(mz_zip_archive *pZip, const char *pFilename);\nmz_bool mz_zip_writer_init_from_reader_v2(mz_zip_archive *pZip, const char *pFilename, mz_uint flags);\n\n/* Adds the contents of a memory buffer to an archive. These functions record the current local time into the archive. */\n/* To add a directory entry, call this method with an archive name ending in a forwardslash with an empty buffer. */\n/* level_and_flags - compression level (0-10, see MZ_BEST_SPEED, MZ_BEST_COMPRESSION, etc.) logically OR'd with zero or more mz_zip_flags, or just set to MZ_DEFAULT_COMPRESSION. */\nmz_bool mz_zip_writer_add_mem(mz_zip_archive *pZip, const char *pArchive_name, const void *pBuf, size_t buf_size, mz_uint level_and_flags);\n\n/* Like mz_zip_writer_add_mem(), except you can specify a file comment field, and optionally supply the function with already compressed data. */\n/* uncomp_size/uncomp_crc32 are only used if the MZ_ZIP_FLAG_COMPRESSED_DATA flag is specified. */\nmz_bool mz_zip_writer_add_mem_ex(mz_zip_archive *pZip, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags,\n                                 mz_uint64 uncomp_size, mz_uint32 uncomp_crc32);\n\nmz_bool mz_zip_writer_add_mem_ex_v2(mz_zip_archive *pZip, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags,\n                                    mz_uint64 uncomp_size, mz_uint32 uncomp_crc32, MZ_TIME_T *last_modified, const char *user_extra_data_local, mz_uint user_extra_data_local_len,\n                                    const char *user_extra_data_central, mz_uint user_extra_data_central_len);\n\n/* Adds the contents of a file to an archive. This function also records the disk file's modified time into the archive. */\n/* File data is supplied via a read callback function. User mz_zip_writer_add_(c)file to add a file directly.*/\nmz_bool mz_zip_writer_add_read_buf_callback(mz_zip_archive *pZip, const char *pArchive_name, mz_file_read_func read_callback, void* callback_opaque, mz_uint64 size_to_add,\n\tconst MZ_TIME_T *pFile_time, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags, const char *user_extra_data_local, mz_uint user_extra_data_local_len,\n\tconst char *user_extra_data_central, mz_uint user_extra_data_central_len);\n\n#ifndef MINIZ_NO_STDIO\n/* Adds the contents of a disk file to an archive. This function also records the disk file's modified time into the archive. */\n/* level_and_flags - compression level (0-10, see MZ_BEST_SPEED, MZ_BEST_COMPRESSION, etc.) logically OR'd with zero or more mz_zip_flags, or just set to MZ_DEFAULT_COMPRESSION. */\nmz_bool mz_zip_writer_add_file(mz_zip_archive *pZip, const char *pArchive_name, const char *pSrc_filename, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags);\n\n/* Like mz_zip_writer_add_file(), except the file data is read from the specified FILE stream. */\nmz_bool mz_zip_writer_add_cfile(mz_zip_archive *pZip, const char *pArchive_name, MZ_FILE *pSrc_file, mz_uint64 size_to_add,\n                                const MZ_TIME_T *pFile_time, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags, const char *user_extra_data_local, mz_uint user_extra_data_local_len,\n                                const char *user_extra_data_central, mz_uint user_extra_data_central_len);\n#endif\n\n/* Adds a file to an archive by fully cloning the data from another archive. */\n/* This function fully clones the source file's compressed data (no recompression), along with its full filename, extra data (it may add or modify the zip64 local header extra data field), and the optional descriptor following the compressed data. */\nmz_bool mz_zip_writer_add_from_zip_reader(mz_zip_archive *pZip, mz_zip_archive *pSource_zip, mz_uint src_file_index);\n\n/* Finalizes the archive by writing the central directory records followed by the end of central directory record. */\n/* After an archive is finalized, the only valid call on the mz_zip_archive struct is mz_zip_writer_end(). */\n/* An archive must be manually finalized by calling this function for it to be valid. */\nmz_bool mz_zip_writer_finalize_archive(mz_zip_archive *pZip);\n\n/* Finalizes a heap archive, returning a poiner to the heap block and its size. */\n/* The heap block will be allocated using the mz_zip_archive's alloc/realloc callbacks. */\nmz_bool mz_zip_writer_finalize_heap_archive(mz_zip_archive *pZip, void **ppBuf, size_t *pSize);\n\n/* Ends archive writing, freeing all allocations, and closing the output file if mz_zip_writer_init_file() was used. */\n/* Note for the archive to be valid, it *must* have been finalized before ending (this function will not do it for you). */\nmz_bool mz_zip_writer_end(mz_zip_archive *pZip);\n\n/* -------- Misc. high-level helper functions: */\n\n/* mz_zip_add_mem_to_archive_file_in_place() efficiently (but not atomically) appends a memory blob to a ZIP archive. */\n/* Note this is NOT a fully safe operation. If it crashes or dies in some way your archive can be left in a screwed up state (without a central directory). */\n/* level_and_flags - compression level (0-10, see MZ_BEST_SPEED, MZ_BEST_COMPRESSION, etc.) logically OR'd with zero or more mz_zip_flags, or just set to MZ_DEFAULT_COMPRESSION. */\n/* TODO: Perhaps add an option to leave the existing central dir in place in case the add dies? We could then truncate the file (so the old central dir would be at the end) if something goes wrong. */\nmz_bool mz_zip_add_mem_to_archive_file_in_place(const char *pZip_filename, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags);\nmz_bool mz_zip_add_mem_to_archive_file_in_place_v2(const char *pZip_filename, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags, mz_zip_error *pErr);\n\n/* Reads a single file from an archive into a heap block. */\n/* If pComment is not NULL, only the file with the specified comment will be extracted. */\n/* Returns NULL on failure. */\nvoid *mz_zip_extract_archive_file_to_heap(const char *pZip_filename, const char *pArchive_name, size_t *pSize, mz_uint flags);\nvoid *mz_zip_extract_archive_file_to_heap_v2(const char *pZip_filename, const char *pArchive_name, const char *pComment, size_t *pSize, mz_uint flags, mz_zip_error *pErr);\n\n#endif /* #ifndef MINIZ_NO_ARCHIVE_WRITING_APIS */\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif /* MINIZ_NO_ARCHIVE_APIS */\n"
  },
  {
    "path": "3rdparty/mshadow/.gitignore",
    "content": "# Compiled Object files\n*.slo\n*.lo\n*.o\n\n# Compiled Dynamic libraries\n*.so\n*.dylib\n\n# Compiled Static libraries\n*.lai\n*.la\n*.a\n*~\ndoc/html\ndoc/latex\nrabit\ndmlc-core\n*.db\n*.bak\nbuild\n"
  },
  {
    "path": "3rdparty/mshadow/.travis.yml",
    "content": "# disable sudo to use container based build\nsudo: false\n\n# Use Build Matrix to do lint and build seperately\nenv:\n  matrix:\n    - TASK=lint LINT_LANG=cpp\n    - TASK=doc\n    - TASK=build CXX=g++\n\n# dependent apt packages\naddons:\n  apt:\n    packages:\n      - doxygen\n      - wget\n      - unzip\n      - libblas-dev\n      - python3-pip\n\nbefore_install:\n  - git clone https://github.com/dmlc/dmlc-core\n  - export TRAVIS=dmlc-core/scripts/travis\n  - source ${TRAVIS}/travis_setup_env.sh\n\ninstall:\n  - pip3 install --upgrade pip --user\n  - pip3 install  --user  cpplint pylint\n  \nscript: scripts/travis_script.sh\n\nbefore_cache:\n  - ${TRAVIS}/travis_before_cache.sh\n\ncache:\n  directories:\n    - ${HOME}/.cache/usr\n\nnotifications:\n  email:\n    on_success: change\n    on_failure: always\n\n"
  },
  {
    "path": "3rdparty/mshadow/CHANGES.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nChange Log\n=====\n\nmshadow-1.0\n=====\n* Initial release\n\nmshadow-2.0: in progress\n=====\n* Support multiple data type\n* Great refactoring of code\n* Parameter server interface for MultiGPU and distributed learning\n"
  },
  {
    "path": "3rdparty/mshadow/CMakeLists.txt",
    "content": "cmake_minimum_required(VERSION 3.13)\nproject(mshadow C CXX)\n\ninclude(CMakeDependentOption)\noption(USE_CUDA \"Build with CUDA support\" ON)\noption(USE_CUDNN ON)\ncmake_dependent_option(USE_SSE \"Build with x86 SSE instruction support\" ON\n  \"CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL amd64\" OFF)\ncmake_dependent_option(USE_F16C \"Build with x86 F16C instruction support\" ON\n  \"CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL amd64\" OFF)   # autodetects support if ON\noption(USE_INT64_TENSOR_SIZE \"Use int64_t to represent the total number of elements in a tensor\" OFF)\noption(MSHADOW_IN_CXX11 ON)\n\nadd_library(mshadow INTERFACE)\nfile(GLOB_RECURSE MSHADOWSOURCE \"mshadow/*.h\")\ntarget_include_directories(mshadow INTERFACE \"${CMAKE_CURRENT_SOURCE_DIR}\")\ntarget_sources(mshadow INTERFACE ${MSHADOWSOURCE})\nif(UNIX)\n  target_compile_options(mshadow INTERFACE\n    \"$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:Clang>>:-Wno-braced-scalar-init>\"\n    \"$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:Clang>>:-Wno-pass-failed>\"\n    # TODO Replace Wno-unused-lambda-capture with [[maybe_unused]] annotation once requiring C++17\n    \"$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:Clang>>:-Wno-unused-lambda-capture>\"\n    # TODO Fixing the warning leads to compile error on 4.8; fix once 4.8 support is dropped\n    \"$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:Clang>>:-Wno-undefined-var-template>\"\n    \"$<$<COMPILE_LANGUAGE:CXX>:-Wno-unused-parameter>\"\n    \"$<$<COMPILE_LANGUAGE:CXX>:-Wno-unknown-pragmas>\"\n    \"$<$<COMPILE_LANGUAGE:CXX>:-Wno-unused-local-typedefs>\"\n    \"$<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>\")\nendif()\n\nif(USE_CUDA)\n  enable_language(CUDA)\n  file(GLOB_RECURSE MSHADOW_CUDASOURCE \"mshadow/*.cuh\")\n  target_sources(mshadow INTERFACE ${MSHADOW_CUDASOURCE})\n  target_compile_definitions(mshadow INTERFACE MSHADOW_USE_CUDA=1\n    MSHADOW_FORCE_STREAM)\nelse()\n  target_compile_definitions(mshadow INTERFACE MSHADOW_USE_CUDA=0)\nendif()\nif(USE_SSE)\n  # For cross compilation, we can't rely on the compiler checks, but mshadow\n  # will add platform specific includes not available in other arches\n  include(CheckCXXCompilerFlag)\n  check_cxx_compiler_flag(\"-msse3\" SUPPORT_MSSE3)\n  check_cxx_compiler_flag(\"-msse2\" SUPPORT_MSSE2)\n  if(SUPPORT_MSSE3)\n    target_compile_definitions(mshadow INTERFACE MSHADOW_USE_SSE)\n    target_compile_options(mshadow INTERFACE $<$<COMPILE_LANGUAGE:CXX>:-msse3>)\n  elseif(SUPPORT_MSSE2)\n    target_compile_definitions(mshadow INTERFACE MSHADOW_USE_SSE)\n    target_compile_options(mshadow INTERFACE $<$<COMPILE_LANGUAGE:CXX>:-msse2>)\n  else()\n    target_compile_definitions(mshadow INTERFACE MSHADOW_USE_SSE=0)\n  endif()\nelse()\n  target_compile_definitions(mshadow INTERFACE MSHADOW_USE_SSE=0)\nendif()\nif(USE_CUDNN)\n  target_compile_definitions(mshadow INTERFACE MSHADOW_USE_CUDNN)\nendif()\nif(USE_CUTENSOR)\n  target_compile_definitions(mshadow INTERFACE MSHADOW_USE_CUTENSOR)\nendif()\nif(MSHADOW_IN_CXX11)\n  target_compile_definitions(mshadow INTERFACE MSHADOW_IN_CXX11)\nendif()\nif(USE_F16C)\n  # Determine if hardware supports F16C instruction set\n  message(STATUS \"Determining F16C support\")\n  include(cmake/AutoDetectF16C.cmake)\n  if(SUPPORT_F16C)\n    target_compile_options(mshadow INTERFACE $<$<COMPILE_LANGUAGE:CXX>:-mf16c>)\n  else()\n    target_compile_definitions(mshadow INTERFACE MSHADOW_USE_F16C=0)\n  endif()\nelse()\n  target_compile_definitions(mshadow INTERFACE MSHADOW_USE_F16C=0)\nendif()\nif(USE_INT64_TENSOR_SIZE)\n  message(STATUS \"Using 64-bit integer for tensor size\")\n  target_compile_definitions(mshadow INTERFACE MSHADOW_INT64_TENSOR_SIZE=1)\nelse()\n  target_compile_definitions(mshadow INTERFACE MSHADOW_INT64_TENSOR_SIZE=0)\nendif()\n\nset(mshadow_LINT_DIRS mshadow mshadow-ps)\nfind_package(Python3)\nadd_custom_target(mshadow_lint COMMAND ${CMAKE_COMMAND} -DMSVC=${MSVC}\n  -DPYTHON_EXECUTABLE=${Python3_EXECUTABLE} -DLINT_DIRS=${mshadow_LINT_DIRS}\n  -DPROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR} -DPROJECT_NAME=mshadow\n  -P ${PROJECT_SOURCE_DIR}/../dmlc-core/cmake/lint.cmake)\n"
  },
  {
    "path": "3rdparty/mshadow/LICENSE",
    "content": "Licensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n    \n   http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n"
  },
  {
    "path": "3rdparty/mshadow/README.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nmshadow: Matrix Shadow\n======\n[![Build Status](https://travis-ci.org/dmlc/mshadow.svg?branch=master)](https://travis-ci.org/dmlc/mshadow)\n\nMShadow is a lightweight CPU/GPU Matrix/Tensor Template Library in C++/CUDA. The goal of mshadow is to support ***efficient***,\n***device invariant*** and ***simple*** tensor library for machine learning project that aims for maximum performance and control, while also emphasize simplicity.\n\nMShadow also provides interface that allows writing Multi-GPU and distributed deep learning programs in an easy and unified way.\n\n* [Contributors](https://github.com/tqchen/mshadow/graphs/contributors)\n* [Tutorial](guide)\n* [Documentation](doc)\n* [Parameter Server Interface for GPU Tensor](guide/mshadow-ps)\n\nFeatures\n--------\n* Efficient: all the expression you write will be lazily evaluated and compiled into optimized code\n  - No temporal memory allocation will happen for expression you write\n  - mshadow will generate specific kernel for every expression you write in compile time.\n* Device invariant: you can write one code and it will run on both CPU and GPU\n* Simple: mshadow allows you to write machine learning code using expressions.\n* Whitebox: put a float* into the Tensor struct and take the benefit of the package, no memory allocation is happened unless explicitly called\n* Lightweight library: light amount of code to support frequently used functions in machine learning\n* Extendable: user can write simple functions that plugs into mshadow and run on GPU/CPU, no experience in CUDA is required.\n* MultiGPU and Distributed ML: mshadow-ps interface allows user to write efficient MultiGPU and distributed programs in an unified way.\n\nVersion\n-------\n* This version mshadow-2.x, there are a lot of changes in the interface and it is not backward compatible with mshadow-1.0\n  - If you use older version of cxxnet, you will need to use the legacy mshadow code\n* For legacy code, refer to [Here](https://github.com/tqchen/mshadow/releases/tag/v1.1)\n* Change log in [CHANGES.md](CHANGES.md)\n\nProjects Using MShadow\n----------------------\n* [MXNet: Efficient and Flexible Distributed Deep Learning Framework](https://github.com/apache/mxnet)\n* [CXXNet: A lightweight  C++ based deep learnig framework](https://github.com/dmlc/cxxnet)\n"
  },
  {
    "path": "3rdparty/mshadow/cmake/AutoDetectF16C.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Determines whether hardware and compiler support F16C \n# instruction set\n#\n# The following are set after configuration is done:\n#  SUPPORT_F16C\n\nif(AUTO_DETECT_F16_CMAKE_INCLUDED)\n  return()\nendif()\nset(AUTO_DETECT_F16_CMAKE_INCLUDED True)\nset(SUPPORT_F16C False)\nif(MSVC)\n    message(\"F16C instruction set is not yet supported for MSVC\")\n    return()\nendif()\ninclude(CheckCXXCompilerFlag)\ncheck_cxx_compiler_flag(\"-mf16c\" COMPILER_SUPPORT_MF16C)\nif(CMAKE_SYSTEM_NAME STREQUAL \"Linux\")\n    execute_process(COMMAND cat /proc/cpuinfo\n            COMMAND grep flags\n            COMMAND grep f16c\n            OUTPUT_VARIABLE CPU_SUPPORT_F16C)\nelseif(CMAKE_SYSTEM_NAME STREQUAL \"Darwin\")\n    execute_process(COMMAND sysctl -a\n            COMMAND grep machdep.cpu.features\n            COMMAND grep F16C\n            OUTPUT_VARIABLE CPU_SUPPORT_F16C)\nendif()\nif(NOT CPU_SUPPORT_F16C)\n    message(\"CPU does not support F16C instructions\")\n    return()\nendif()\nif(CPU_SUPPORT_F16C AND COMPILER_SUPPORT_MF16C)\t\n    set(SUPPORT_F16C TRUE)\nendif()\n"
  },
  {
    "path": "3rdparty/mshadow/doc/Doxyfile",
    "content": "# Doxyfile 1.8.8\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# This file describes the settings to be used by the documentation system\n# doxygen (www.doxygen.org) for a project.\n#\n# All text after a double hash (##) is considered a comment and is placed in\n# front of the TAG it is preceding.\n#\n# All text after a single hash (#) is considered a comment and will be ignored.\n# The format is:\n# TAG = value [value, ...]\n# For lists, items can also be appended using:\n# TAG += value [value, ...]\n# Values that contain spaces should be placed between quotes (\\\" \\\").\n\n#---------------------------------------------------------------------------\n# Project related configuration options\n#---------------------------------------------------------------------------\n\n# This tag specifies the encoding used for all characters in the config file\n# that follow. The default is UTF-8 which is also the encoding used for all text\n# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv\n# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv\n# for the list of possible encodings.\n# The default value is: UTF-8.\n\nDOXYFILE_ENCODING      = UTF-8\n\n# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by\n# double-quotes, unless you are using Doxywizard) that should identify the\n# project for which the documentation is generated. This name is used in the\n# title of most generated pages and in a few other places.\n# The default value is: My Project.\n\nPROJECT_NAME           = \"mshadow\"\n\n# The PROJECT_NUMBER tag can be used to enter a project or revision number. This\n# could be handy for archiving the generated documentation or if some version\n# control system is used.\n\nPROJECT_NUMBER         =\n\n# Using the PROJECT_BRIEF tag one can provide an optional one line description\n# for a project that appears at the top of each page and should give viewer a\n# quick idea about the purpose of the project. Keep the description short.\n\nPROJECT_BRIEF          =\n\n# With the PROJECT_LOGO tag one can specify an logo or icon that is included in\n# the documentation. The maximum height of the logo should not exceed 55 pixels\n# and the maximum width should not exceed 200 pixels. Doxygen will copy the logo\n# to the output directory.\n\nPROJECT_LOGO           =\n\n# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path\n# into which the generated documentation will be written. If a relative path is\n# entered, it will be relative to the location where doxygen was started. If\n# left blank the current directory will be used.\n\nOUTPUT_DIRECTORY       = doc\n\n# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create 4096 sub-\n# directories (in 2 levels) under the output directory of each output format and\n# will distribute the generated files over these directories. Enabling this\n# option can be useful when feeding doxygen a huge amount of source files, where\n# putting all generated files in the same directory would otherwise causes\n# performance problems for the file system.\n# The default value is: NO.\n\nCREATE_SUBDIRS         = NO\n\n# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII\n# characters to appear in the names of generated files. If set to NO, non-ASCII\n# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode\n# U+3044.\n# The default value is: NO.\n\nALLOW_UNICODE_NAMES    = NO\n\n# The OUTPUT_LANGUAGE tag is used to specify the language in which all\n# documentation generated by doxygen is written. Doxygen will use this\n# information to generate all constant output in the proper language.\n# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese,\n# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States),\n# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian,\n# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages),\n# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian,\n# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian,\n# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish,\n# Ukrainian and Vietnamese.\n# The default value is: English.\n\nOUTPUT_LANGUAGE        = English\n\n# If the BRIEF_MEMBER_DESC tag is set to YES doxygen will include brief member\n# descriptions after the members that are listed in the file and class\n# documentation (similar to Javadoc). Set to NO to disable this.\n# The default value is: YES.\n\nBRIEF_MEMBER_DESC      = YES\n\n# If the REPEAT_BRIEF tag is set to YES doxygen will prepend the brief\n# description of a member or function before the detailed description\n#\n# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the\n# brief descriptions will be completely suppressed.\n# The default value is: YES.\n\nREPEAT_BRIEF           = YES\n\n# This tag implements a quasi-intelligent brief description abbreviator that is\n# used to form the text in various listings. Each string in this list, if found\n# as the leading text of the brief description, will be stripped from the text\n# and the result, after processing the whole list, is used as the annotated\n# text. Otherwise, the brief description is used as-is. If left blank, the\n# following values are used ($name is automatically replaced with the name of\n# the entity):The $name class, The $name widget, The $name file, is, provides,\n# specifies, contains, represents, a, an and the.\n\nABBREVIATE_BRIEF       =\n\n# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then\n# doxygen will generate a detailed section even if there is only a brief\n# description.\n# The default value is: NO.\n\nALWAYS_DETAILED_SEC    = NO\n\n# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all\n# inherited members of a class in the documentation of that class as if those\n# members were ordinary class members. Constructors, destructors and assignment\n# operators of the base classes will not be shown.\n# The default value is: NO.\n\nINLINE_INHERITED_MEMB  = NO\n\n# If the FULL_PATH_NAMES tag is set to YES doxygen will prepend the full path\n# before files name in the file list and in the header files. If set to NO the\n# shortest path that makes the file name unique will be used\n# The default value is: YES.\n\nFULL_PATH_NAMES        = YES\n\n# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path.\n# Stripping is only done if one of the specified strings matches the left-hand\n# part of the path. The tag can be used to show relative paths in the file list.\n# If left blank the directory from which doxygen is run is used as the path to\n# strip.\n#\n# Note that you can specify absolute paths here, but also relative paths, which\n# will be relative from the directory where doxygen is started.\n# This tag requires that the tag FULL_PATH_NAMES is set to YES.\n\nSTRIP_FROM_PATH        =\n\n# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the\n# path mentioned in the documentation of a class, which tells the reader which\n# header file to include in order to use a class. If left blank only the name of\n# the header file containing the class definition is used. Otherwise one should\n# specify the list of include paths that are normally passed to the compiler\n# using the -I flag.\n\nSTRIP_FROM_INC_PATH    =\n\n# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but\n# less readable) file names. This can be useful is your file systems doesn't\n# support long names like on DOS, Mac, or CD-ROM.\n# The default value is: NO.\n\nSHORT_NAMES            = NO\n\n# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the\n# first line (until the first dot) of a Javadoc-style comment as the brief\n# description. If set to NO, the Javadoc-style will behave just like regular Qt-\n# style comments (thus requiring an explicit @brief command for a brief\n# description.)\n# The default value is: NO.\n\nJAVADOC_AUTOBRIEF      = NO\n\n# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first\n# line (until the first dot) of a Qt-style comment as the brief description. If\n# set to NO, the Qt-style will behave just like regular Qt-style comments (thus\n# requiring an explicit \\brief command for a brief description.)\n# The default value is: NO.\n\nQT_AUTOBRIEF           = NO\n\n# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a\n# multi-line C++ special comment block (i.e. a block of //! or /// comments) as\n# a brief description. This used to be the default behavior. The new default is\n# to treat a multi-line C++ comment block as a detailed description. Set this\n# tag to YES if you prefer the old behavior instead.\n#\n# Note that setting this tag to YES also means that rational rose comments are\n# not recognized any more.\n# The default value is: NO.\n\nMULTILINE_CPP_IS_BRIEF = NO\n\n# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the\n# documentation from any documented member that it re-implements.\n# The default value is: YES.\n\nINHERIT_DOCS           = YES\n\n# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce a\n# new page for each member. If set to NO, the documentation of a member will be\n# part of the file/class/namespace that contains it.\n# The default value is: NO.\n\nSEPARATE_MEMBER_PAGES  = NO\n\n# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen\n# uses this value to replace tabs by spaces in code fragments.\n# Minimum value: 1, maximum value: 16, default value: 4.\n\nTAB_SIZE               = 8\n\n# This tag can be used to specify a number of aliases that act as commands in\n# the documentation. An alias has the form:\n# name=value\n# For example adding\n# \"sideeffect=@par Side Effects:\\n\"\n# will allow you to put the command \\sideeffect (or @sideeffect) in the\n# documentation, which will result in a user-defined paragraph with heading\n# \"Side Effects:\". You can put \\n's in the value part of an alias to insert\n# newlines.\n\nALIASES                =\n\n# This tag can be used to specify a number of word-keyword mappings (TCL only).\n# A mapping has the form \"name=value\". For example adding \"class=itcl::class\"\n# will allow you to use the command class in the itcl::class meaning.\n\nTCL_SUBST              =\n\n# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources\n# only. Doxygen will then generate output that is more tailored for C. For\n# instance, some of the names that are used will be different. The list of all\n# members will be omitted, etc.\n# The default value is: NO.\n\nOPTIMIZE_OUTPUT_FOR_C  = YES\n\n# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or\n# Python sources only. Doxygen will then generate output that is more tailored\n# for that language. For instance, namespaces will be presented as packages,\n# qualified scopes will look different, etc.\n# The default value is: NO.\n\nOPTIMIZE_OUTPUT_JAVA   = NO\n\n# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran\n# sources. Doxygen will then generate output that is tailored for Fortran.\n# The default value is: NO.\n\nOPTIMIZE_FOR_FORTRAN   = NO\n\n# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL\n# sources. Doxygen will then generate output that is tailored for VHDL.\n# The default value is: NO.\n\nOPTIMIZE_OUTPUT_VHDL   = NO\n\n# Doxygen selects the parser to use depending on the extension of the files it\n# parses. With this tag you can assign which parser to use for a given\n# extension. Doxygen has a built-in mapping, but you can override or extend it\n# using this tag. The format is ext=language, where ext is a file extension, and\n# language is one of the parsers supported by doxygen: IDL, Java, Javascript,\n# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran:\n# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran:\n# Fortran. In the later case the parser tries to guess whether the code is fixed\n# or free formatted code, this is the default for Fortran type files), VHDL. For\n# instance to make doxygen treat .inc files as Fortran files (default is PHP),\n# and .f files as C (default is Fortran), use: inc=Fortran f=C.\n#\n# Note For files without extension you can use no_extension as a placeholder.\n#\n# Note that for custom extensions you also need to set FILE_PATTERNS otherwise\n# the files are not read by doxygen.\n\nEXTENSION_MAPPING      =\n\n# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments\n# according to the Markdown format, which allows for more readable\n# documentation. See http://daringfireball.net/projects/markdown/ for details.\n# The output of markdown processing is further processed by doxygen, so you can\n# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in\n# case of backward compatibilities issues.\n# The default value is: YES.\n\nMARKDOWN_SUPPORT       = YES\n\n# When enabled doxygen tries to link words that correspond to documented\n# classes, or namespaces to their corresponding documentation. Such a link can\n# be prevented in individual cases by by putting a % sign in front of the word\n# or globally by setting AUTOLINK_SUPPORT to NO.\n# The default value is: YES.\n\nAUTOLINK_SUPPORT       = YES\n\n# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want\n# to include (a tag file for) the STL sources as input, then you should set this\n# tag to YES in order to let doxygen match functions declarations and\n# definitions whose arguments contain STL classes (e.g. func(std::string);\n# versus func(std::string) {}). This also make the inheritance and collaboration\n# diagrams that involve STL classes more complete and accurate.\n# The default value is: NO.\n\nBUILTIN_STL_SUPPORT    = NO\n\n# If you use Microsoft's C++/CLI language, you should set this option to YES to\n# enable parsing support.\n# The default value is: NO.\n\nCPP_CLI_SUPPORT        = NO\n\n# Set the SIP_SUPPORT tag to YES if your project consists of sip (see:\n# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen\n# will parse them like normal C++ but will assume all classes use public instead\n# of private inheritance when no explicit protection keyword is present.\n# The default value is: NO.\n\nSIP_SUPPORT            = NO\n\n# For Microsoft's IDL there are propget and propput attributes to indicate\n# getter and setter methods for a property. Setting this option to YES will make\n# doxygen to replace the get and set methods by a property in the documentation.\n# This will only work if the methods are indeed getting or setting a simple\n# type. If this is not the case, or you want to show the methods anyway, you\n# should set this option to NO.\n# The default value is: YES.\n\nIDL_PROPERTY_SUPPORT   = YES\n\n# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC\n# tag is set to YES, then doxygen will reuse the documentation of the first\n# member in the group (if any) for the other members of the group. By default\n# all members of a group must be documented explicitly.\n# The default value is: NO.\n\nDISTRIBUTE_GROUP_DOC   = NO\n\n# Set the SUBGROUPING tag to YES to allow class member groups of the same type\n# (for instance a group of public functions) to be put as a subgroup of that\n# type (e.g. under the Public Functions section). Set it to NO to prevent\n# subgrouping. Alternatively, this can be done per class using the\n# \\nosubgrouping command.\n# The default value is: YES.\n\nSUBGROUPING            = YES\n\n# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions\n# are shown inside the group in which they are included (e.g. using \\ingroup)\n# instead of on a separate page (for HTML and Man pages) or section (for LaTeX\n# and RTF).\n#\n# Note that this feature does not work in combination with\n# SEPARATE_MEMBER_PAGES.\n# The default value is: NO.\n\nINLINE_GROUPED_CLASSES = NO\n\n# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions\n# with only public data fields or simple typedef fields will be shown inline in\n# the documentation of the scope in which they are defined (i.e. file,\n# namespace, or group documentation), provided this scope is documented. If set\n# to NO, structs, classes, and unions are shown on a separate page (for HTML and\n# Man pages) or section (for LaTeX and RTF).\n# The default value is: NO.\n\nINLINE_SIMPLE_STRUCTS  = NO\n\n# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or\n# enum is documented as struct, union, or enum with the name of the typedef. So\n# typedef struct TypeS {} TypeT, will appear in the documentation as a struct\n# with name TypeT. When disabled the typedef will appear as a member of a file,\n# namespace, or class. And the struct will be named TypeS. This can typically be\n# useful for C code in case the coding convention dictates that all compound\n# types are typedef'ed and only the typedef is referenced, never the tag name.\n# The default value is: NO.\n\nTYPEDEF_HIDES_STRUCT   = NO\n\n# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This\n# cache is used to resolve symbols given their name and scope. Since this can be\n# an expensive process and often the same symbol appears multiple times in the\n# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small\n# doxygen will become slower. If the cache is too large, memory is wasted. The\n# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range\n# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536\n# symbols. At the end of a run doxygen will report the cache usage and suggest\n# the optimal cache size from a speed point of view.\n# Minimum value: 0, maximum value: 9, default value: 0.\n\nLOOKUP_CACHE_SIZE      = 0\n\n#---------------------------------------------------------------------------\n# Build related configuration options\n#---------------------------------------------------------------------------\n\n# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in\n# documentation are documented, even if no documentation was available. Private\n# class members and static file members will be hidden unless the\n# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES.\n# Note: This will also disable the warnings about undocumented members that are\n# normally produced when WARNINGS is set to YES.\n# The default value is: NO.\n\nEXTRACT_ALL            = NO\n\n# If the EXTRACT_PRIVATE tag is set to YES all private members of a class will\n# be included in the documentation.\n# The default value is: NO.\n\nEXTRACT_PRIVATE        = NO\n\n# If the EXTRACT_PACKAGE tag is set to YES all members with package or internal\n# scope will be included in the documentation.\n# The default value is: NO.\n\nEXTRACT_PACKAGE        = NO\n\n# If the EXTRACT_STATIC tag is set to YES all static members of a file will be\n# included in the documentation.\n# The default value is: NO.\n\nEXTRACT_STATIC         = NO\n\n# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) defined\n# locally in source files will be included in the documentation. If set to NO\n# only classes defined in header files are included. Does not have any effect\n# for Java sources.\n# The default value is: YES.\n\nEXTRACT_LOCAL_CLASSES  = YES\n\n# This flag is only useful for Objective-C code. When set to YES local methods,\n# which are defined in the implementation section but not in the interface are\n# included in the documentation. If set to NO only methods in the interface are\n# included.\n# The default value is: NO.\n\nEXTRACT_LOCAL_METHODS  = NO\n\n# If this flag is set to YES, the members of anonymous namespaces will be\n# extracted and appear in the documentation as a namespace called\n# 'anonymous_namespace{file}', where file will be replaced with the base name of\n# the file that contains the anonymous namespace. By default anonymous namespace\n# are hidden.\n# The default value is: NO.\n\nEXTRACT_ANON_NSPACES   = NO\n\n# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all\n# undocumented members inside documented classes or files. If set to NO these\n# members will be included in the various overviews, but no documentation\n# section is generated. This option has no effect if EXTRACT_ALL is enabled.\n# The default value is: NO.\n\nHIDE_UNDOC_MEMBERS     = NO\n\n# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all\n# undocumented classes that are normally visible in the class hierarchy. If set\n# to NO these classes will be included in the various overviews. This option has\n# no effect if EXTRACT_ALL is enabled.\n# The default value is: NO.\n\nHIDE_UNDOC_CLASSES     = YES\n\n# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend\n# (class|struct|union) declarations. If set to NO these declarations will be\n# included in the documentation.\n# The default value is: NO.\n\nHIDE_FRIEND_COMPOUNDS  = NO\n\n# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any\n# documentation blocks found inside the body of a function. If set to NO these\n# blocks will be appended to the function's detailed documentation block.\n# The default value is: NO.\n\nHIDE_IN_BODY_DOCS      = NO\n\n# The INTERNAL_DOCS tag determines if documentation that is typed after a\n# \\internal command is included. If the tag is set to NO then the documentation\n# will be excluded. Set it to YES to include the internal documentation.\n# The default value is: NO.\n\nINTERNAL_DOCS          = NO\n\n# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file\n# names in lower-case letters. If set to YES upper-case letters are also\n# allowed. This is useful if you have classes or files whose names only differ\n# in case and if your file system supports case sensitive file names. Windows\n# and Mac users are advised to set this option to NO.\n# The default value is: system dependent.\n\nCASE_SENSE_NAMES       = YES\n\n# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with\n# their full class and namespace scopes in the documentation. If set to YES the\n# scope will be hidden.\n# The default value is: NO.\n\nHIDE_SCOPE_NAMES       = NO\n\n# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of\n# the files that are included by a file in the documentation of that file.\n# The default value is: YES.\n\nSHOW_INCLUDE_FILES     = YES\n\n# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each\n# grouped member an include statement to the documentation, telling the reader\n# which file to include in order to use the member.\n# The default value is: NO.\n\nSHOW_GROUPED_MEMB_INC  = NO\n\n# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include\n# files with double quotes in the documentation rather than with sharp brackets.\n# The default value is: NO.\n\nFORCE_LOCAL_INCLUDES   = NO\n\n# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the\n# documentation for inline members.\n# The default value is: YES.\n\nINLINE_INFO            = YES\n\n# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the\n# (detailed) documentation of file and class members alphabetically by member\n# name. If set to NO the members will appear in declaration order.\n# The default value is: YES.\n\nSORT_MEMBER_DOCS       = YES\n\n# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief\n# descriptions of file, namespace and class members alphabetically by member\n# name. If set to NO the members will appear in declaration order. Note that\n# this will also influence the order of the classes in the class list.\n# The default value is: NO.\n\nSORT_BRIEF_DOCS        = NO\n\n# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the\n# (brief and detailed) documentation of class members so that constructors and\n# destructors are listed first. If set to NO the constructors will appear in the\n# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS.\n# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief\n# member documentation.\n# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting\n# detailed member documentation.\n# The default value is: NO.\n\nSORT_MEMBERS_CTORS_1ST = NO\n\n# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy\n# of group names into alphabetical order. If set to NO the group names will\n# appear in their defined order.\n# The default value is: NO.\n\nSORT_GROUP_NAMES       = NO\n\n# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by\n# fully-qualified names, including namespaces. If set to NO, the class list will\n# be sorted only by class name, not including the namespace part.\n# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.\n# Note: This option applies only to the class list, not to the alphabetical\n# list.\n# The default value is: NO.\n\nSORT_BY_SCOPE_NAME     = NO\n\n# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper\n# type resolution of all parameters of a function it will reject a match between\n# the prototype and the implementation of a member function even if there is\n# only one candidate or it is obvious which candidate to choose by doing a\n# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still\n# accept a match between prototype and implementation in such cases.\n# The default value is: NO.\n\nSTRICT_PROTO_MATCHING  = NO\n\n# The GENERATE_TODOLIST tag can be used to enable ( YES) or disable ( NO) the\n# todo list. This list is created by putting \\todo commands in the\n# documentation.\n# The default value is: YES.\n\nGENERATE_TODOLIST      = YES\n\n# The GENERATE_TESTLIST tag can be used to enable ( YES) or disable ( NO) the\n# test list. This list is created by putting \\test commands in the\n# documentation.\n# The default value is: YES.\n\nGENERATE_TESTLIST      = YES\n\n# The GENERATE_BUGLIST tag can be used to enable ( YES) or disable ( NO) the bug\n# list. This list is created by putting \\bug commands in the documentation.\n# The default value is: YES.\n\nGENERATE_BUGLIST       = YES\n\n# The GENERATE_DEPRECATEDLIST tag can be used to enable ( YES) or disable ( NO)\n# the deprecated list. This list is created by putting \\deprecated commands in\n# the documentation.\n# The default value is: YES.\n\nGENERATE_DEPRECATEDLIST= YES\n\n# The ENABLED_SECTIONS tag can be used to enable conditional documentation\n# sections, marked by \\if <section_label> ... \\endif and \\cond <section_label>\n# ... \\endcond blocks.\n\nENABLED_SECTIONS       =\n\n# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the\n# initial value of a variable or macro / define can have for it to appear in the\n# documentation. If the initializer consists of more lines than specified here\n# it will be hidden. Use a value of 0 to hide initializers completely. The\n# appearance of the value of individual variables and macros / defines can be\n# controlled using \\showinitializer or \\hideinitializer command in the\n# documentation regardless of this setting.\n# Minimum value: 0, maximum value: 10000, default value: 30.\n\nMAX_INITIALIZER_LINES  = 30\n\n# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at\n# the bottom of the documentation of classes and structs. If set to YES the list\n# will mention the files that were used to generate the documentation.\n# The default value is: YES.\n\nSHOW_USED_FILES        = YES\n\n# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This\n# will remove the Files entry from the Quick Index and from the Folder Tree View\n# (if specified).\n# The default value is: YES.\n\nSHOW_FILES             = YES\n\n# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces\n# page. This will remove the Namespaces entry from the Quick Index and from the\n# Folder Tree View (if specified).\n# The default value is: YES.\n\nSHOW_NAMESPACES        = YES\n\n# The FILE_VERSION_FILTER tag can be used to specify a program or script that\n# doxygen should invoke to get the current version for each file (typically from\n# the version control system). Doxygen will invoke the program by executing (via\n# popen()) the command command input-file, where command is the value of the\n# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided\n# by doxygen. Whatever the program writes to standard output is used as the file\n# version. For an example see the documentation.\n\nFILE_VERSION_FILTER    =\n\n# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed\n# by doxygen. The layout file controls the global structure of the generated\n# output files in an output format independent way. To create the layout file\n# that represents doxygen's defaults, run doxygen with the -l option. You can\n# optionally specify a file name after the option, if omitted DoxygenLayout.xml\n# will be used as the name of the layout file.\n#\n# Note that if you run doxygen from a directory containing a file called\n# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE\n# tag is left empty.\n\nLAYOUT_FILE            =\n\n# The CITE_BIB_FILES tag can be used to specify one or more bib files containing\n# the reference definitions. This must be a list of .bib files. The .bib\n# extension is automatically appended if omitted. This requires the bibtex tool\n# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info.\n# For LaTeX the style of the bibliography can be controlled using\n# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the\n# search path. See also \\cite for info how to create references.\n\nCITE_BIB_FILES         =\n\n#---------------------------------------------------------------------------\n# Configuration options related to warning and progress messages\n#---------------------------------------------------------------------------\n\n# The QUIET tag can be used to turn on/off the messages that are generated to\n# standard output by doxygen. If QUIET is set to YES this implies that the\n# messages are off.\n# The default value is: NO.\n\nQUIET                  = NO\n\n# The WARNINGS tag can be used to turn on/off the warning messages that are\n# generated to standard error ( stderr) by doxygen. If WARNINGS is set to YES\n# this implies that the warnings are on.\n#\n# Tip: Turn warnings on while writing the documentation.\n# The default value is: YES.\n\nWARNINGS               = YES\n\n# If the WARN_IF_UNDOCUMENTED tag is set to YES, then doxygen will generate\n# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag\n# will automatically be disabled.\n# The default value is: YES.\n\nWARN_IF_UNDOCUMENTED   = YES\n\n# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for\n# potential errors in the documentation, such as not documenting some parameters\n# in a documented function, or documenting parameters that don't exist or using\n# markup commands wrongly.\n# The default value is: YES.\n\nWARN_IF_DOC_ERROR      = YES\n\n# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that\n# are documented, but have no documentation for their parameters or return\n# value. If set to NO doxygen will only warn about wrong or incomplete parameter\n# documentation, but not about the absence of documentation.\n# The default value is: NO.\n\nWARN_NO_PARAMDOC       = YES\n\n# The WARN_FORMAT tag determines the format of the warning messages that doxygen\n# can produce. The string should contain the $file, $line, and $text tags, which\n# will be replaced by the file and line number from which the warning originated\n# and the warning text. Optionally the format may contain $version, which will\n# be replaced by the version of the file (if it could be obtained via\n# FILE_VERSION_FILTER)\n# The default value is: $file:$line: $text.\n\nWARN_FORMAT            = \"$file:$line: $text\"\n\n# The WARN_LOGFILE tag can be used to specify a file to which warning and error\n# messages should be written. If left blank the output is written to standard\n# error (stderr).\n\nWARN_LOGFILE           =\n\n#---------------------------------------------------------------------------\n# Configuration options related to the input files\n#---------------------------------------------------------------------------\n\n# The INPUT tag is used to specify the files and/or directories that contain\n# documented source files. You may enter file names like myfile.cpp or\n# directories like /usr/src/myproject. Separate the files or directories with\n# spaces.\n# Note: If this tag is empty the current directory is searched.\n\nINPUT                  = mshadow \\\n                         mshadow-ps\n\n# This tag can be used to specify the character encoding of the source files\n# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses\n# libiconv (or the iconv built into libc) for the transcoding. See the libiconv\n# documentation (see: http://www.gnu.org/software/libiconv) for the list of\n# possible encodings.\n# The default value is: UTF-8.\n\nINPUT_ENCODING         = UTF-8\n\n# If the value of the INPUT tag contains directories, you can use the\n# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and\n# *.h) to filter out the source-files in the directories. If left blank the\n# following patterns are tested:*.c, *.cc, *.cxx, *.cpp, *.c++, *.java, *.ii,\n# *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, *.hh, *.hxx, *.hpp,\n# *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, *.m, *.markdown,\n# *.md, *.mm, *.dox, *.py, *.f90, *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf,\n# *.qsf, *.as and *.js.\n\nFILE_PATTERNS          =\n\n# The RECURSIVE tag can be used to specify whether or not subdirectories should\n# be searched for input files as well.\n# The default value is: NO.\n\nRECURSIVE              = NO\n\n# The EXCLUDE tag can be used to specify files and/or directories that should be\n# excluded from the INPUT source files. This way you can easily exclude a\n# subdirectory from a directory tree whose root is specified with the INPUT tag.\n#\n# Note that relative paths are relative to the directory from which doxygen is\n# run.\n\nEXCLUDE                =\n\n# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or\n# directories that are symbolic links (a Unix file system feature) are excluded\n# from the input.\n# The default value is: NO.\n\nEXCLUDE_SYMLINKS       = NO\n\n# If the value of the INPUT tag contains directories, you can use the\n# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude\n# certain files from those directories.\n#\n# Note that the wildcards are matched against the file with absolute path, so to\n# exclude all test directories for example use the pattern */test/*\n\nEXCLUDE_PATTERNS       = *-inl.* \\\n                         utils.h \\\n                         thread_util.h \\\n                         thread.h \\\n                         kv_array.h\n\n# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names\n# (namespaces, classes, functions, etc.) that should be excluded from the\n# output. The symbol name can be a fully qualified name, a word, or if the\n# wildcard * is used, a substring. Examples: ANamespace, AClass,\n# AClass::ANamespace, ANamespace::*Test\n#\n# Note that the wildcards are matched against the file with absolute path, so to\n# exclude all test directories use the pattern */test/*\n\nEXCLUDE_SYMBOLS        = mshadow::expr::Plan* \\\n                         mshadow::expr::*Engine*\n\n# The EXAMPLE_PATH tag can be used to specify one or more files or directories\n# that contain example code fragments that are included (see the \\include\n# command).\n\nEXAMPLE_PATH           =\n\n# If the value of the EXAMPLE_PATH tag contains directories, you can use the\n# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and\n# *.h) to filter out the source-files in the directories. If left blank all\n# files are included.\n\nEXAMPLE_PATTERNS       =\n\n# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be\n# searched for input files to be used with the \\include or \\dontinclude commands\n# irrespective of the value of the RECURSIVE tag.\n# The default value is: NO.\n\nEXAMPLE_RECURSIVE      = NO\n\n# The IMAGE_PATH tag can be used to specify one or more files or directories\n# that contain images that are to be included in the documentation (see the\n# \\image command).\n\nIMAGE_PATH             =\n\n# The INPUT_FILTER tag can be used to specify a program that doxygen should\n# invoke to filter for each input file. Doxygen will invoke the filter program\n# by executing (via popen()) the command:\n#\n# <filter> <input-file>\n#\n# where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the\n# name of an input file. Doxygen will then use the output that the filter\n# program writes to standard output. If FILTER_PATTERNS is specified, this tag\n# will be ignored.\n#\n# Note that the filter must not add or remove lines; it is applied before the\n# code is scanned, but not when the output code is generated. If lines are added\n# or removed, the anchors will not be placed correctly.\n\nINPUT_FILTER           =\n\n# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern\n# basis. Doxygen will compare the file name with each pattern and apply the\n# filter if there is a match. The filters are a list of the form: pattern=filter\n# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how\n# filters are used. If the FILTER_PATTERNS tag is empty or if none of the\n# patterns match the file name, INPUT_FILTER is applied.\n\nFILTER_PATTERNS        =\n\n# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using\n# INPUT_FILTER ) will also be used to filter the input files that are used for\n# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES).\n# The default value is: NO.\n\nFILTER_SOURCE_FILES    = NO\n\n# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file\n# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and\n# it is also possible to disable source filtering for a specific pattern using\n# *.ext= (so without naming a filter).\n# This tag requires that the tag FILTER_SOURCE_FILES is set to YES.\n\nFILTER_SOURCE_PATTERNS =\n\n# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that\n# is part of the input, its contents will be placed on the main page\n# (index.html). This can be useful if you have a project on for instance GitHub\n# and want to reuse the introduction page also for the doxygen output.\n\nUSE_MDFILE_AS_MAINPAGE =\n\n#---------------------------------------------------------------------------\n# Configuration options related to source browsing\n#---------------------------------------------------------------------------\n\n# If the SOURCE_BROWSER tag is set to YES then a list of source files will be\n# generated. Documented entities will be cross-referenced with these sources.\n#\n# Note: To get rid of all source code in the generated output, make sure that\n# also VERBATIM_HEADERS is set to NO.\n# The default value is: NO.\n\nSOURCE_BROWSER         = NO\n\n# Setting the INLINE_SOURCES tag to YES will include the body of functions,\n# classes and enums directly into the documentation.\n# The default value is: NO.\n\nINLINE_SOURCES         = NO\n\n# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any\n# special comment blocks from generated source code fragments. Normal C, C++ and\n# Fortran comments will always remain visible.\n# The default value is: YES.\n\nSTRIP_CODE_COMMENTS    = YES\n\n# If the REFERENCED_BY_RELATION tag is set to YES then for each documented\n# function all documented functions referencing it will be listed.\n# The default value is: NO.\n\nREFERENCED_BY_RELATION = NO\n\n# If the REFERENCES_RELATION tag is set to YES then for each documented function\n# all documented entities called/used by that function will be listed.\n# The default value is: NO.\n\nREFERENCES_RELATION    = NO\n\n# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set\n# to YES, then the hyperlinks from functions in REFERENCES_RELATION and\n# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will\n# link to the documentation.\n# The default value is: YES.\n\nREFERENCES_LINK_SOURCE = YES\n\n# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the\n# source code will show a tooltip with additional information such as prototype,\n# brief description and links to the definition and documentation. Since this\n# will make the HTML file larger and loading of large files a bit slower, you\n# can opt to disable this feature.\n# The default value is: YES.\n# This tag requires that the tag SOURCE_BROWSER is set to YES.\n\nSOURCE_TOOLTIPS        = YES\n\n# If the USE_HTAGS tag is set to YES then the references to source code will\n# point to the HTML generated by the htags(1) tool instead of doxygen built-in\n# source browser. The htags tool is part of GNU's global source tagging system\n# (see http://www.gnu.org/software/global/global.html). You will need version\n# 4.8.6 or higher.\n#\n# To use it do the following:\n# - Install the latest version of global\n# - Enable SOURCE_BROWSER and USE_HTAGS in the config file\n# - Make sure the INPUT points to the root of the source tree\n# - Run doxygen as normal\n#\n# Doxygen will invoke htags (and that will in turn invoke gtags), so these\n# tools must be available from the command line (i.e. in the search path).\n#\n# The result: instead of the source browser generated by doxygen, the links to\n# source code will now point to the output of htags.\n# The default value is: NO.\n# This tag requires that the tag SOURCE_BROWSER is set to YES.\n\nUSE_HTAGS              = NO\n\n# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a\n# verbatim copy of the header file for each class for which an include is\n# specified. Set to NO to disable this.\n# See also: Section \\class.\n# The default value is: YES.\n\nVERBATIM_HEADERS       = YES\n\n# If the CLANG_ASSISTED_PARSING tag is set to YES, then doxygen will use the\n# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the\n# cost of reduced performance. This can be particularly helpful with template\n# rich C++ code for which doxygen's built-in parser lacks the necessary type\n# information.\n# Note: The availability of this option depends on whether or not doxygen was\n# compiled with the --with-libclang option.\n# The default value is: NO.\n\nCLANG_ASSISTED_PARSING = NO\n\n# If clang assisted parsing is enabled you can provide the compiler with command\n# line options that you would normally use when invoking the compiler. Note that\n# the include paths will already be set by doxygen for the files and directories\n# specified with INPUT and INCLUDE_PATH.\n# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES.\n\nCLANG_OPTIONS          =\n\n#---------------------------------------------------------------------------\n# Configuration options related to the alphabetical class index\n#---------------------------------------------------------------------------\n\n# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all\n# compounds will be generated. Enable this if the project contains a lot of\n# classes, structs, unions or interfaces.\n# The default value is: YES.\n\nALPHABETICAL_INDEX     = YES\n\n# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in\n# which the alphabetical index list will be split.\n# Minimum value: 1, maximum value: 20, default value: 5.\n# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.\n\nCOLS_IN_ALPHA_INDEX    = 5\n\n# In case all classes in a project start with a common prefix, all classes will\n# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag\n# can be used to specify a prefix (or a list of prefixes) that should be ignored\n# while generating the index headers.\n# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.\n\nIGNORE_PREFIX          =\n\n#---------------------------------------------------------------------------\n# Configuration options related to the HTML output\n#---------------------------------------------------------------------------\n\n# If the GENERATE_HTML tag is set to YES doxygen will generate HTML output\n# The default value is: YES.\n\nGENERATE_HTML          = YES\n\n# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a\n# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of\n# it.\n# The default directory is: html.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_OUTPUT            = html\n\n# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each\n# generated HTML page (for example: .htm, .php, .asp).\n# The default value is: .html.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_FILE_EXTENSION    = .html\n\n# The HTML_HEADER tag can be used to specify a user-defined HTML header file for\n# each generated HTML page. If the tag is left blank doxygen will generate a\n# standard header.\n#\n# To get valid HTML the header file that includes any scripts and style sheets\n# that doxygen needs, which is dependent on the configuration options used (e.g.\n# the setting GENERATE_TREEVIEW). It is highly recommended to start with a\n# default header using\n# doxygen -w html new_header.html new_footer.html new_stylesheet.css\n# YourConfigFile\n# and then modify the file new_header.html. See also section \"Doxygen usage\"\n# for information on how to generate the default header that doxygen normally\n# uses.\n# Note: The header is subject to change so you typically have to regenerate the\n# default header when upgrading to a newer version of doxygen. For a description\n# of the possible markers and block names see the documentation.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_HEADER            =\n\n# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each\n# generated HTML page. If the tag is left blank doxygen will generate a standard\n# footer. See HTML_HEADER for more information on how to generate a default\n# footer and what special commands can be used inside the footer. See also\n# section \"Doxygen usage\" for information on how to generate the default footer\n# that doxygen normally uses.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_FOOTER            =\n\n# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style\n# sheet that is used by each HTML page. It can be used to fine-tune the look of\n# the HTML output. If left blank doxygen will generate a default style sheet.\n# See also section \"Doxygen usage\" for information on how to generate the style\n# sheet that doxygen normally uses.\n# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as\n# it is more robust and this tag (HTML_STYLESHEET) will in the future become\n# obsolete.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_STYLESHEET        =\n\n# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined\n# cascading style sheets that are included after the standard style sheets\n# created by doxygen. Using this option one can overrule certain style aspects.\n# This is preferred over using HTML_STYLESHEET since it does not replace the\n# standard style sheet and is therefor more robust against future updates.\n# Doxygen will copy the style sheet files to the output directory.\n# Note: The order of the extra stylesheet files is of importance (e.g. the last\n# stylesheet in the list overrules the setting of the previous ones in the\n# list). For an example see the documentation.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_EXTRA_STYLESHEET  =\n\n# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or\n# other source files which should be copied to the HTML output directory. Note\n# that these files will be copied to the base HTML output directory. Use the\n# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these\n# files. In the HTML_STYLESHEET file, use the file name only. Also note that the\n# files will be copied as-is; there are no commands or markers available.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_EXTRA_FILES       =\n\n# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen\n# will adjust the colors in the stylesheet and background images according to\n# this color. Hue is specified as an angle on a colorwheel, see\n# http://en.wikipedia.org/wiki/Hue for more information. For instance the value\n# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300\n# purple, and 360 is red again.\n# Minimum value: 0, maximum value: 359, default value: 220.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_COLORSTYLE_HUE    = 220\n\n# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors\n# in the HTML output. For a value of 0 the output will use grayscales only. A\n# value of 255 will produce the most vivid colors.\n# Minimum value: 0, maximum value: 255, default value: 100.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_COLORSTYLE_SAT    = 100\n\n# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the\n# luminance component of the colors in the HTML output. Values below 100\n# gradually make the output lighter, whereas values above 100 make the output\n# darker. The value divided by 100 is the actual gamma applied, so 80 represents\n# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not\n# change the gamma.\n# Minimum value: 40, maximum value: 240, default value: 80.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_COLORSTYLE_GAMMA  = 80\n\n# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML\n# page will contain the date and time when the page was generated. Setting this\n# to NO can help when comparing the output of multiple runs.\n# The default value is: YES.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_TIMESTAMP         = YES\n\n# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML\n# documentation will contain sections that can be hidden and shown after the\n# page has loaded.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_DYNAMIC_SECTIONS  = NO\n\n# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries\n# shown in the various tree structured indices initially; the user can expand\n# and collapse entries dynamically later on. Doxygen will expand the tree to\n# such a level that at most the specified number of entries are visible (unless\n# a fully collapsed tree already exceeds this amount). So setting the number of\n# entries 1 will produce a full collapsed tree by default. 0 is a special value\n# representing an infinite number of entries and will result in a full expanded\n# tree by default.\n# Minimum value: 0, maximum value: 9999, default value: 100.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_INDEX_NUM_ENTRIES = 100\n\n# If the GENERATE_DOCSET tag is set to YES, additional index files will be\n# generated that can be used as input for Apple's Xcode 3 integrated development\n# environment (see: http://developer.apple.com/tools/xcode/), introduced with\n# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a\n# Makefile in the HTML output directory. Running make will produce the docset in\n# that directory and running make install will install the docset in\n# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at\n# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html\n# for more information.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nGENERATE_DOCSET        = NO\n\n# This tag determines the name of the docset feed. A documentation feed provides\n# an umbrella under which multiple documentation sets from a single provider\n# (such as a company or product suite) can be grouped.\n# The default value is: Doxygen generated docs.\n# This tag requires that the tag GENERATE_DOCSET is set to YES.\n\nDOCSET_FEEDNAME        = \"Doxygen generated docs\"\n\n# This tag specifies a string that should uniquely identify the documentation\n# set bundle. This should be a reverse domain-name style string, e.g.\n# com.mycompany.MyDocSet. Doxygen will append .docset to the name.\n# The default value is: org.doxygen.Project.\n# This tag requires that the tag GENERATE_DOCSET is set to YES.\n\nDOCSET_BUNDLE_ID       = org.doxygen.Project\n\n# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify\n# the documentation publisher. This should be a reverse domain-name style\n# string, e.g. com.mycompany.MyDocSet.documentation.\n# The default value is: org.doxygen.Publisher.\n# This tag requires that the tag GENERATE_DOCSET is set to YES.\n\nDOCSET_PUBLISHER_ID    = org.doxygen.Publisher\n\n# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher.\n# The default value is: Publisher.\n# This tag requires that the tag GENERATE_DOCSET is set to YES.\n\nDOCSET_PUBLISHER_NAME  = Publisher\n\n# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three\n# additional HTML index files: index.hhp, index.hhc, and index.hhk. The\n# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop\n# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on\n# Windows.\n#\n# The HTML Help Workshop contains a compiler that can convert all HTML output\n# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML\n# files are now used as the Windows 98 help format, and will replace the old\n# Windows help format (.hlp) on all Windows platforms in the future. Compressed\n# HTML files also contain an index, a table of contents, and you can search for\n# words in the documentation. The HTML workshop also contains a viewer for\n# compressed HTML files.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nGENERATE_HTMLHELP      = NO\n\n# The CHM_FILE tag can be used to specify the file name of the resulting .chm\n# file. You can add a path in front of the file if the result should not be\n# written to the html output directory.\n# This tag requires that the tag GENERATE_HTMLHELP is set to YES.\n\nCHM_FILE               =\n\n# The HHC_LOCATION tag can be used to specify the location (absolute path\n# including file name) of the HTML help compiler ( hhc.exe). If non-empty\n# doxygen will try to run the HTML help compiler on the generated index.hhp.\n# The file has to be specified with full path.\n# This tag requires that the tag GENERATE_HTMLHELP is set to YES.\n\nHHC_LOCATION           =\n\n# The GENERATE_CHI flag controls if a separate .chi index file is generated (\n# YES) or that it should be included in the master .chm file ( NO).\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTMLHELP is set to YES.\n\nGENERATE_CHI           = NO\n\n# The CHM_INDEX_ENCODING is used to encode HtmlHelp index ( hhk), content ( hhc)\n# and project file content.\n# This tag requires that the tag GENERATE_HTMLHELP is set to YES.\n\nCHM_INDEX_ENCODING     =\n\n# The BINARY_TOC flag controls whether a binary table of contents is generated (\n# YES) or a normal table of contents ( NO) in the .chm file. Furthermore it\n# enables the Previous and Next buttons.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTMLHELP is set to YES.\n\nBINARY_TOC             = NO\n\n# The TOC_EXPAND flag can be set to YES to add extra items for group members to\n# the table of contents of the HTML help documentation and to the tree view.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTMLHELP is set to YES.\n\nTOC_EXPAND             = NO\n\n# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and\n# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that\n# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help\n# (.qch) of the generated HTML documentation.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nGENERATE_QHP           = NO\n\n# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify\n# the file name of the resulting .qch file. The path specified is relative to\n# the HTML output folder.\n# This tag requires that the tag GENERATE_QHP is set to YES.\n\nQCH_FILE               =\n\n# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help\n# Project output. For more information please see Qt Help Project / Namespace\n# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace).\n# The default value is: org.doxygen.Project.\n# This tag requires that the tag GENERATE_QHP is set to YES.\n\nQHP_NAMESPACE          = org.doxygen.Project\n\n# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt\n# Help Project output. For more information please see Qt Help Project / Virtual\n# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual-\n# folders).\n# The default value is: doc.\n# This tag requires that the tag GENERATE_QHP is set to YES.\n\nQHP_VIRTUAL_FOLDER     = doc\n\n# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom\n# filter to add. For more information please see Qt Help Project / Custom\n# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-\n# filters).\n# This tag requires that the tag GENERATE_QHP is set to YES.\n\nQHP_CUST_FILTER_NAME   =\n\n# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the\n# custom filter to add. For more information please see Qt Help Project / Custom\n# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-\n# filters).\n# This tag requires that the tag GENERATE_QHP is set to YES.\n\nQHP_CUST_FILTER_ATTRS  =\n\n# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this\n# project's filter section matches. Qt Help Project / Filter Attributes (see:\n# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes).\n# This tag requires that the tag GENERATE_QHP is set to YES.\n\nQHP_SECT_FILTER_ATTRS  =\n\n# The QHG_LOCATION tag can be used to specify the location of Qt's\n# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the\n# generated .qhp file.\n# This tag requires that the tag GENERATE_QHP is set to YES.\n\nQHG_LOCATION           =\n\n# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be\n# generated, together with the HTML files, they form an Eclipse help plugin. To\n# install this plugin and make it available under the help contents menu in\n# Eclipse, the contents of the directory containing the HTML and XML files needs\n# to be copied into the plugins directory of eclipse. The name of the directory\n# within the plugins directory should be the same as the ECLIPSE_DOC_ID value.\n# After copying Eclipse needs to be restarted before the help appears.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nGENERATE_ECLIPSEHELP   = NO\n\n# A unique identifier for the Eclipse help plugin. When installing the plugin\n# the directory name containing the HTML and XML files should also have this\n# name. Each documentation set should have its own identifier.\n# The default value is: org.doxygen.Project.\n# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES.\n\nECLIPSE_DOC_ID         = org.doxygen.Project\n\n# If you want full control over the layout of the generated HTML pages it might\n# be necessary to disable the index and replace it with your own. The\n# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top\n# of each HTML page. A value of NO enables the index and the value YES disables\n# it. Since the tabs in the index contain the same information as the navigation\n# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nDISABLE_INDEX          = NO\n\n# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index\n# structure should be generated to display hierarchical information. If the tag\n# value is set to YES, a side panel will be generated containing a tree-like\n# index structure (just like the one that is generated for HTML Help). For this\n# to work a browser that supports JavaScript, DHTML, CSS and frames is required\n# (i.e. any modern browser). Windows users are probably better off using the\n# HTML help feature. Via custom stylesheets (see HTML_EXTRA_STYLESHEET) one can\n# further fine-tune the look of the index. As an example, the default style\n# sheet generated by doxygen has an example that shows how to put an image at\n# the root of the tree instead of the PROJECT_NAME. Since the tree basically has\n# the same information as the tab index, you could consider setting\n# DISABLE_INDEX to YES when enabling this option.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nGENERATE_TREEVIEW      = NO\n\n# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that\n# doxygen will group on one line in the generated HTML documentation.\n#\n# Note that a value of 0 will completely suppress the enum values from appearing\n# in the overview section.\n# Minimum value: 0, maximum value: 20, default value: 4.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nENUM_VALUES_PER_LINE   = 4\n\n# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used\n# to set the initial width (in pixels) of the frame in which the tree is shown.\n# Minimum value: 0, maximum value: 1500, default value: 250.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nTREEVIEW_WIDTH         = 250\n\n# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open links to\n# external symbols imported via tag files in a separate window.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nEXT_LINKS_IN_WINDOW    = NO\n\n# Use this tag to change the font size of LaTeX formulas included as images in\n# the HTML documentation. When you change the font size after a successful\n# doxygen run you need to manually remove any form_*.png images from the HTML\n# output directory to force them to be regenerated.\n# Minimum value: 8, maximum value: 50, default value: 10.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nFORMULA_FONTSIZE       = 10\n\n# Use the FORMULA_TRANPARENT tag to determine whether or not the images\n# generated for formulas are transparent PNGs. Transparent PNGs are not\n# supported properly for IE 6.0, but are supported on all modern browsers.\n#\n# Note that when changing this option you need to delete any form_*.png files in\n# the HTML output directory before the changes have effect.\n# The default value is: YES.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nFORMULA_TRANSPARENT    = YES\n\n# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see\n# http://www.mathjax.org) which uses client side Javascript for the rendering\n# instead of using prerendered bitmaps. Use this if you do not have LaTeX\n# installed or if you want to formulas look prettier in the HTML output. When\n# enabled you may also need to install MathJax separately and configure the path\n# to it using the MATHJAX_RELPATH option.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nUSE_MATHJAX            = NO\n\n# When MathJax is enabled you can set the default output format to be used for\n# the MathJax output. See the MathJax site (see:\n# http://docs.mathjax.org/en/latest/output.html) for more details.\n# Possible values are: HTML-CSS (which is slower, but has the best\n# compatibility), NativeMML (i.e. MathML) and SVG.\n# The default value is: HTML-CSS.\n# This tag requires that the tag USE_MATHJAX is set to YES.\n\nMATHJAX_FORMAT         = HTML-CSS\n\n# When MathJax is enabled you need to specify the location relative to the HTML\n# output directory using the MATHJAX_RELPATH option. The destination directory\n# should contain the MathJax.js script. For instance, if the mathjax directory\n# is located at the same level as the HTML output directory, then\n# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax\n# Content Delivery Network so you can quickly see the result without installing\n# MathJax. However, it is strongly recommended to install a local copy of\n# MathJax from http://www.mathjax.org before deployment.\n# The default value is: http://cdn.mathjax.org/mathjax/latest.\n# This tag requires that the tag USE_MATHJAX is set to YES.\n\nMATHJAX_RELPATH        = http://www.mathjax.org/mathjax\n\n# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax\n# extension names that should be enabled during MathJax rendering. For example\n# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols\n# This tag requires that the tag USE_MATHJAX is set to YES.\n\nMATHJAX_EXTENSIONS     =\n\n# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces\n# of code that will be used on startup of the MathJax code. See the MathJax site\n# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an\n# example see the documentation.\n# This tag requires that the tag USE_MATHJAX is set to YES.\n\nMATHJAX_CODEFILE       =\n\n# When the SEARCHENGINE tag is enabled doxygen will generate a search box for\n# the HTML output. The underlying search engine uses javascript and DHTML and\n# should work on any modern browser. Note that when using HTML help\n# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET)\n# there is already a search function so this one should typically be disabled.\n# For large projects the javascript based search engine can be slow, then\n# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to\n# search using the keyboard; to jump to the search box use <access key> + S\n# (what the <access key> is depends on the OS and browser, but it is typically\n# <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down\n# key> to jump into the search results window, the results can be navigated\n# using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel\n# the search. The filter options can be selected when the cursor is inside the\n# search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys>\n# to select a filter and <Enter> or <escape> to activate or cancel the filter\n# option.\n# The default value is: YES.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nSEARCHENGINE           = YES\n\n# When the SERVER_BASED_SEARCH tag is enabled the search engine will be\n# implemented using a web server instead of a web client using Javascript. There\n# are two flavors of web server based searching depending on the EXTERNAL_SEARCH\n# setting. When disabled, doxygen will generate a PHP script for searching and\n# an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing\n# and searching needs to be provided by external tools. See the section\n# \"External Indexing and Searching\" for details.\n# The default value is: NO.\n# This tag requires that the tag SEARCHENGINE is set to YES.\n\nSERVER_BASED_SEARCH    = NO\n\n# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP\n# script for searching. Instead the search results are written to an XML file\n# which needs to be processed by an external indexer. Doxygen will invoke an\n# external search engine pointed to by the SEARCHENGINE_URL option to obtain the\n# search results.\n#\n# Doxygen ships with an example indexer ( doxyindexer) and search engine\n# (doxysearch.cgi) which are based on the open source search engine library\n# Xapian (see: http://xapian.org/).\n#\n# See the section \"External Indexing and Searching\" for details.\n# The default value is: NO.\n# This tag requires that the tag SEARCHENGINE is set to YES.\n\nEXTERNAL_SEARCH        = NO\n\n# The SEARCHENGINE_URL should point to a search engine hosted by a web server\n# which will return the search results when EXTERNAL_SEARCH is enabled.\n#\n# Doxygen ships with an example indexer ( doxyindexer) and search engine\n# (doxysearch.cgi) which are based on the open source search engine library\n# Xapian (see: http://xapian.org/). See the section \"External Indexing and\n# Searching\" for details.\n# This tag requires that the tag SEARCHENGINE is set to YES.\n\nSEARCHENGINE_URL       =\n\n# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed\n# search data is written to a file for indexing by an external tool. With the\n# SEARCHDATA_FILE tag the name of this file can be specified.\n# The default file is: searchdata.xml.\n# This tag requires that the tag SEARCHENGINE is set to YES.\n\nSEARCHDATA_FILE        = searchdata.xml\n\n# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the\n# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is\n# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple\n# projects and redirect the results back to the right project.\n# This tag requires that the tag SEARCHENGINE is set to YES.\n\nEXTERNAL_SEARCH_ID     =\n\n# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen\n# projects other than the one defined by this configuration file, but that are\n# all added to the same external search index. Each project needs to have a\n# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of\n# to a relative location where the documentation can be found. The format is:\n# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ...\n# This tag requires that the tag SEARCHENGINE is set to YES.\n\nEXTRA_SEARCH_MAPPINGS  =\n\n#---------------------------------------------------------------------------\n# Configuration options related to the LaTeX output\n#---------------------------------------------------------------------------\n\n# If the GENERATE_LATEX tag is set to YES doxygen will generate LaTeX output.\n# The default value is: YES.\n\nGENERATE_LATEX         = YES\n\n# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a\n# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of\n# it.\n# The default directory is: latex.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_OUTPUT           = latex\n\n# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be\n# invoked.\n#\n# Note that when enabling USE_PDFLATEX this option is only used for generating\n# bitmaps for formulas in the HTML output, but not in the Makefile that is\n# written to the output directory.\n# The default file is: latex.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_CMD_NAME         = latex\n\n# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate\n# index for LaTeX.\n# The default file is: makeindex.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nMAKEINDEX_CMD_NAME     = makeindex\n\n# If the COMPACT_LATEX tag is set to YES doxygen generates more compact LaTeX\n# documents. This may be useful for small projects and may help to save some\n# trees in general.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nCOMPACT_LATEX          = NO\n\n# The PAPER_TYPE tag can be used to set the paper type that is used by the\n# printer.\n# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x\n# 14 inches) and executive (7.25 x 10.5 inches).\n# The default value is: a4.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nPAPER_TYPE             = a4\n\n# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names\n# that should be included in the LaTeX output. To get the times font for\n# instance you can specify\n# EXTRA_PACKAGES=times\n# If left blank no extra packages will be included.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nEXTRA_PACKAGES         =\n\n# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the\n# generated LaTeX document. The header should contain everything until the first\n# chapter. If it is left blank doxygen will generate a standard header. See\n# section \"Doxygen usage\" for information on how to let doxygen write the\n# default header to a separate file.\n#\n# Note: Only use a user-defined header if you know what you are doing! The\n# following commands have a special meaning inside the header: $title,\n# $datetime, $date, $doxygenversion, $projectname, $projectnumber,\n# $projectbrief, $projectlogo. Doxygen will replace $title with the empy string,\n# for the replacement values of the other commands the user is refered to\n# HTML_HEADER.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_HEADER           =\n\n# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the\n# generated LaTeX document. The footer should contain everything after the last\n# chapter. If it is left blank doxygen will generate a standard footer. See\n# LATEX_HEADER for more information on how to generate a default footer and what\n# special commands can be used inside the footer.\n#\n# Note: Only use a user-defined footer if you know what you are doing!\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_FOOTER           =\n\n# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or\n# other source files which should be copied to the LATEX_OUTPUT output\n# directory. Note that the files will be copied as-is; there are no commands or\n# markers available.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_EXTRA_FILES      =\n\n# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is\n# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will\n# contain links (just like the HTML output) instead of page references. This\n# makes the output suitable for online browsing using a PDF viewer.\n# The default value is: YES.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nPDF_HYPERLINKS         = YES\n\n# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate\n# the PDF file directly from the LaTeX files. Set this option to YES to get a\n# higher quality PDF documentation.\n# The default value is: YES.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nUSE_PDFLATEX           = YES\n\n# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode\n# command to the generated LaTeX files. This will instruct LaTeX to keep running\n# if errors occur, instead of asking the user for help. This option is also used\n# when generating formulas in HTML.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_BATCHMODE        = NO\n\n# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the\n# index chapters (such as File Index, Compound Index, etc.) in the output.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_HIDE_INDICES     = NO\n\n# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source\n# code with syntax highlighting in the LaTeX output.\n#\n# Note that which sources are shown also depends on other settings such as\n# SOURCE_BROWSER.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_SOURCE_CODE      = NO\n\n# The LATEX_BIB_STYLE tag can be used to specify the style to use for the\n# bibliography, e.g. plainnat, or ieeetr. See\n# http://en.wikipedia.org/wiki/BibTeX and \\cite for more info.\n# The default value is: plain.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_BIB_STYLE        = plain\n\n#---------------------------------------------------------------------------\n# Configuration options related to the RTF output\n#---------------------------------------------------------------------------\n\n# If the GENERATE_RTF tag is set to YES doxygen will generate RTF output. The\n# RTF output is optimized for Word 97 and may not look too pretty with other RTF\n# readers/editors.\n# The default value is: NO.\n\nGENERATE_RTF           = NO\n\n# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a\n# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of\n# it.\n# The default directory is: rtf.\n# This tag requires that the tag GENERATE_RTF is set to YES.\n\nRTF_OUTPUT             = rtf\n\n# If the COMPACT_RTF tag is set to YES doxygen generates more compact RTF\n# documents. This may be useful for small projects and may help to save some\n# trees in general.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_RTF is set to YES.\n\nCOMPACT_RTF            = NO\n\n# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will\n# contain hyperlink fields. The RTF file will contain links (just like the HTML\n# output) instead of page references. This makes the output suitable for online\n# browsing using Word or some other Word compatible readers that support those\n# fields.\n#\n# Note: WordPad (write) and others do not support links.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_RTF is set to YES.\n\nRTF_HYPERLINKS         = NO\n\n# Load stylesheet definitions from file. Syntax is similar to doxygen's config\n# file, i.e. a series of assignments. You only have to provide replacements,\n# missing definitions are set to their default value.\n#\n# See also section \"Doxygen usage\" for information on how to generate the\n# default style sheet that doxygen normally uses.\n# This tag requires that the tag GENERATE_RTF is set to YES.\n\nRTF_STYLESHEET_FILE    =\n\n# Set optional variables used in the generation of an RTF document. Syntax is\n# similar to doxygen's config file. A template extensions file can be generated\n# using doxygen -e rtf extensionFile.\n# This tag requires that the tag GENERATE_RTF is set to YES.\n\nRTF_EXTENSIONS_FILE    =\n\n#---------------------------------------------------------------------------\n# Configuration options related to the man page output\n#---------------------------------------------------------------------------\n\n# If the GENERATE_MAN tag is set to YES doxygen will generate man pages for\n# classes and files.\n# The default value is: NO.\n\nGENERATE_MAN           = NO\n\n# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a\n# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of\n# it. A directory man3 will be created inside the directory specified by\n# MAN_OUTPUT.\n# The default directory is: man.\n# This tag requires that the tag GENERATE_MAN is set to YES.\n\nMAN_OUTPUT             = man\n\n# The MAN_EXTENSION tag determines the extension that is added to the generated\n# man pages. In case the manual section does not start with a number, the number\n# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is\n# optional.\n# The default value is: .3.\n# This tag requires that the tag GENERATE_MAN is set to YES.\n\nMAN_EXTENSION          = .3\n\n# The MAN_SUBDIR tag determines the name of the directory created within\n# MAN_OUTPUT in which the man pages are placed. If defaults to man followed by\n# MAN_EXTENSION with the initial . removed.\n# This tag requires that the tag GENERATE_MAN is set to YES.\n\nMAN_SUBDIR             =\n\n# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it\n# will generate one additional man file for each entity documented in the real\n# man page(s). These additional files only source the real man page, but without\n# them the man command would be unable to find the correct page.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_MAN is set to YES.\n\nMAN_LINKS              = NO\n\n#---------------------------------------------------------------------------\n# Configuration options related to the XML output\n#---------------------------------------------------------------------------\n\n# If the GENERATE_XML tag is set to YES doxygen will generate an XML file that\n# captures the structure of the code including all documentation.\n# The default value is: NO.\n\nGENERATE_XML           = NO\n\n# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a\n# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of\n# it.\n# The default directory is: xml.\n# This tag requires that the tag GENERATE_XML is set to YES.\n\nXML_OUTPUT             = xml\n\n# If the XML_PROGRAMLISTING tag is set to YES doxygen will dump the program\n# listings (including syntax highlighting and cross-referencing information) to\n# the XML output. Note that enabling this will significantly increase the size\n# of the XML output.\n# The default value is: YES.\n# This tag requires that the tag GENERATE_XML is set to YES.\n\nXML_PROGRAMLISTING     = YES\n\n#---------------------------------------------------------------------------\n# Configuration options related to the DOCBOOK output\n#---------------------------------------------------------------------------\n\n# If the GENERATE_DOCBOOK tag is set to YES doxygen will generate Docbook files\n# that can be used to generate PDF.\n# The default value is: NO.\n\nGENERATE_DOCBOOK       = NO\n\n# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put.\n# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in\n# front of it.\n# The default directory is: docbook.\n# This tag requires that the tag GENERATE_DOCBOOK is set to YES.\n\nDOCBOOK_OUTPUT         = docbook\n\n# If the DOCBOOK_PROGRAMLISTING tag is set to YES doxygen will include the\n# program listings (including syntax highlighting and cross-referencing\n# information) to the DOCBOOK output. Note that enabling this will significantly\n# increase the size of the DOCBOOK output.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_DOCBOOK is set to YES.\n\nDOCBOOK_PROGRAMLISTING = NO\n\n#---------------------------------------------------------------------------\n# Configuration options for the AutoGen Definitions output\n#---------------------------------------------------------------------------\n\n# If the GENERATE_AUTOGEN_DEF tag is set to YES doxygen will generate an AutoGen\n# Definitions (see http://autogen.sf.net) file that captures the structure of\n# the code including all documentation. Note that this feature is still\n# experimental and incomplete at the moment.\n# The default value is: NO.\n\nGENERATE_AUTOGEN_DEF   = NO\n\n#---------------------------------------------------------------------------\n# Configuration options related to the Perl module output\n#---------------------------------------------------------------------------\n\n# If the GENERATE_PERLMOD tag is set to YES doxygen will generate a Perl module\n# file that captures the structure of the code including all documentation.\n#\n# Note that this feature is still experimental and incomplete at the moment.\n# The default value is: NO.\n\nGENERATE_PERLMOD       = NO\n\n# If the PERLMOD_LATEX tag is set to YES doxygen will generate the necessary\n# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI\n# output from the Perl module output.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_PERLMOD is set to YES.\n\nPERLMOD_LATEX          = NO\n\n# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be nicely\n# formatted so it can be parsed by a human reader. This is useful if you want to\n# understand what is going on. On the other hand, if this tag is set to NO the\n# size of the Perl module output will be much smaller and Perl will parse it\n# just the same.\n# The default value is: YES.\n# This tag requires that the tag GENERATE_PERLMOD is set to YES.\n\nPERLMOD_PRETTY         = YES\n\n# The names of the make variables in the generated doxyrules.make file are\n# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful\n# so different doxyrules.make files included by the same Makefile don't\n# overwrite each other's variables.\n# This tag requires that the tag GENERATE_PERLMOD is set to YES.\n\nPERLMOD_MAKEVAR_PREFIX =\n\n#---------------------------------------------------------------------------\n# Configuration options related to the preprocessor\n#---------------------------------------------------------------------------\n\n# If the ENABLE_PREPROCESSING tag is set to YES doxygen will evaluate all\n# C-preprocessor directives found in the sources and include files.\n# The default value is: YES.\n\nENABLE_PREPROCESSING   = NO\n\n# If the MACRO_EXPANSION tag is set to YES doxygen will expand all macro names\n# in the source code. If set to NO only conditional compilation will be\n# performed. Macro expansion can be done in a controlled way by setting\n# EXPAND_ONLY_PREDEF to YES.\n# The default value is: NO.\n# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.\n\nMACRO_EXPANSION        = NO\n\n# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then\n# the macro expansion is limited to the macros specified with the PREDEFINED and\n# EXPAND_AS_DEFINED tags.\n# The default value is: NO.\n# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.\n\nEXPAND_ONLY_PREDEF     = NO\n\n# If the SEARCH_INCLUDES tag is set to YES the includes files in the\n# INCLUDE_PATH will be searched if a #include is found.\n# The default value is: YES.\n# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.\n\nSEARCH_INCLUDES        = YES\n\n# The INCLUDE_PATH tag can be used to specify one or more directories that\n# contain include files that are not input files but should be processed by the\n# preprocessor.\n# This tag requires that the tag SEARCH_INCLUDES is set to YES.\n\nINCLUDE_PATH           =\n\n# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard\n# patterns (like *.h and *.hpp) to filter out the header-files in the\n# directories. If left blank, the patterns specified with FILE_PATTERNS will be\n# used.\n# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.\n\nINCLUDE_FILE_PATTERNS  =\n\n# The PREDEFINED tag can be used to specify one or more macro names that are\n# defined before the preprocessor is started (similar to the -D option of e.g.\n# gcc). The argument of the tag is a list of macros of the form: name or\n# name=definition (no spaces). If the definition and the \"=\" are omitted, \"=1\"\n# is assumed. To prevent a macro definition from being undefined via #undef or\n# recursively expanded use the := operator instead of the = operator.\n# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.\n\nPREDEFINED             =\n\n# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this\n# tag can be used to specify a list of macro names that should be expanded. The\n# macro definition that is found in the sources will be used. Use the PREDEFINED\n# tag if you want to use a different macro definition that overrules the\n# definition found in the source code.\n# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.\n\nEXPAND_AS_DEFINED      =\n\n# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will\n# remove all references to function-like macros that are alone on a line, have\n# an all uppercase name, and do not end with a semicolon. Such function macros\n# are typically used for boiler-plate code, and will confuse the parser if not\n# removed.\n# The default value is: YES.\n# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.\n\nSKIP_FUNCTION_MACROS   = YES\n\n#---------------------------------------------------------------------------\n# Configuration options related to external references\n#---------------------------------------------------------------------------\n\n# The TAGFILES tag can be used to specify one or more tag files. For each tag\n# file the location of the external documentation should be added. The format of\n# a tag file without this location is as follows:\n# TAGFILES = file1 file2 ...\n# Adding location for the tag files is done as follows:\n# TAGFILES = file1=loc1 \"file2 = loc2\" ...\n# where loc1 and loc2 can be relative or absolute paths or URLs. See the\n# section \"Linking to external documentation\" for more information about the use\n# of tag files.\n# Note: Each tag file must have a unique name (where the name does NOT include\n# the path). If a tag file is not located in the directory in which doxygen is\n# run, you must also specify the path to the tagfile here.\n\nTAGFILES               =\n\n# When a file name is specified after GENERATE_TAGFILE, doxygen will create a\n# tag file that is based on the input files it reads. See section \"Linking to\n# external documentation\" for more information about the usage of tag files.\n\nGENERATE_TAGFILE       =\n\n# If the ALLEXTERNALS tag is set to YES all external class will be listed in the\n# class index. If set to NO only the inherited external classes will be listed.\n# The default value is: NO.\n\nALLEXTERNALS           = NO\n\n# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed in\n# the modules index. If set to NO, only the current project's groups will be\n# listed.\n# The default value is: YES.\n\nEXTERNAL_GROUPS        = YES\n\n# If the EXTERNAL_PAGES tag is set to YES all external pages will be listed in\n# the related pages index. If set to NO, only the current project's pages will\n# be listed.\n# The default value is: YES.\n\nEXTERNAL_PAGES         = YES\n\n# The PERL_PATH should be the absolute path and name of the perl script\n# interpreter (i.e. the result of 'which perl').\n# The default file (with absolute path) is: /usr/bin/perl.\n\nPERL_PATH              = /usr/bin/perl\n\n#---------------------------------------------------------------------------\n# Configuration options related to the dot tool\n#---------------------------------------------------------------------------\n\n# If the CLASS_DIAGRAMS tag is set to YES doxygen will generate a class diagram\n# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to\n# NO turns the diagrams off. Note that this option also works with HAVE_DOT\n# disabled, but it is recommended to install and use dot, since it yields more\n# powerful graphs.\n# The default value is: YES.\n\nCLASS_DIAGRAMS         = YES\n\n# You can define message sequence charts within doxygen comments using the \\msc\n# command. Doxygen will then run the mscgen tool (see:\n# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the\n# documentation. The MSCGEN_PATH tag allows you to specify the directory where\n# the mscgen tool resides. If left empty the tool is assumed to be found in the\n# default search path.\n\nMSCGEN_PATH            =\n\n# You can include diagrams made with dia in doxygen documentation. Doxygen will\n# then run dia to produce the diagram and insert it in the documentation. The\n# DIA_PATH tag allows you to specify the directory where the dia binary resides.\n# If left empty dia is assumed to be found in the default search path.\n\nDIA_PATH               =\n\n# If set to YES, the inheritance and collaboration graphs will hide inheritance\n# and usage relations if the target is undocumented or is not a class.\n# The default value is: YES.\n\nHIDE_UNDOC_RELATIONS   = YES\n\n# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is\n# available from the path. This tool is part of Graphviz (see:\n# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent\n# Bell Labs. The other options in this section have no effect if this option is\n# set to NO\n# The default value is: YES.\n\nHAVE_DOT               = NO\n\n# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed\n# to run in parallel. When set to 0 doxygen will base this on the number of\n# processors available in the system. You can set it explicitly to a value\n# larger than 0 to get control over the balance between CPU load and processing\n# speed.\n# Minimum value: 0, maximum value: 32, default value: 0.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_NUM_THREADS        = 0\n\n# When you want a differently looking font in the dot files that doxygen\n# generates you can specify the font name using DOT_FONTNAME. You need to make\n# sure dot is able to find the font, which can be done by putting it in a\n# standard location or by setting the DOTFONTPATH environment variable or by\n# setting DOT_FONTPATH to the directory containing the font.\n# The default value is: Helvetica.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_FONTNAME           = Helvetica\n\n# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of\n# dot graphs.\n# Minimum value: 4, maximum value: 24, default value: 10.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_FONTSIZE           = 10\n\n# By default doxygen will tell dot to use the default font as specified with\n# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set\n# the path where dot can find it using this tag.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_FONTPATH           =\n\n# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for\n# each documented class showing the direct and indirect inheritance relations.\n# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nCLASS_GRAPH            = YES\n\n# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a\n# graph for each documented class showing the direct and indirect implementation\n# dependencies (inheritance, containment, and class references variables) of the\n# class with other documented classes.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nCOLLABORATION_GRAPH    = YES\n\n# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for\n# groups, showing the direct groups dependencies.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nGROUP_GRAPHS           = YES\n\n# If the UML_LOOK tag is set to YES doxygen will generate inheritance and\n# collaboration diagrams in a style similar to the OMG's Unified Modeling\n# Language.\n# The default value is: NO.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nUML_LOOK               = NO\n\n# If the UML_LOOK tag is enabled, the fields and methods are shown inside the\n# class node. If there are many fields or methods and many nodes the graph may\n# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the\n# number of items for each type to make the size more manageable. Set this to 0\n# for no limit. Note that the threshold may be exceeded by 50% before the limit\n# is enforced. So when you set the threshold to 10, up to 15 fields may appear,\n# but if the number exceeds 15, the total amount of fields shown is limited to\n# 10.\n# Minimum value: 0, maximum value: 100, default value: 10.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nUML_LIMIT_NUM_FIELDS   = 10\n\n# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and\n# collaboration graphs will show the relations between templates and their\n# instances.\n# The default value is: NO.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nTEMPLATE_RELATIONS     = NO\n\n# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to\n# YES then doxygen will generate a graph for each documented file showing the\n# direct and indirect include dependencies of the file with other documented\n# files.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nINCLUDE_GRAPH          = YES\n\n# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are\n# set to YES then doxygen will generate a graph for each documented file showing\n# the direct and indirect include dependencies of the file with other documented\n# files.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nINCLUDED_BY_GRAPH      = YES\n\n# If the CALL_GRAPH tag is set to YES then doxygen will generate a call\n# dependency graph for every global function or class method.\n#\n# Note that enabling this option will significantly increase the time of a run.\n# So in most cases it will be better to enable call graphs for selected\n# functions only using the \\callgraph command.\n# The default value is: NO.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nCALL_GRAPH             = NO\n\n# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller\n# dependency graph for every global function or class method.\n#\n# Note that enabling this option will significantly increase the time of a run.\n# So in most cases it will be better to enable caller graphs for selected\n# functions only using the \\callergraph command.\n# The default value is: NO.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nCALLER_GRAPH           = NO\n\n# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical\n# hierarchy of all classes instead of a textual one.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nGRAPHICAL_HIERARCHY    = YES\n\n# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the\n# dependencies a directory has on other directories in a graphical way. The\n# dependency relations are determined by the #include relations between the\n# files in the directories.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDIRECTORY_GRAPH        = YES\n\n# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images\n# generated by dot.\n# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order\n# to make the SVG files visible in IE 9+ (other browsers do not have this\n# requirement).\n# Possible values are: png, png:cairo, png:cairo:cairo, png:cairo:gd, png:gd,\n# png:gd:gd, jpg, jpg:cairo, jpg:cairo:gd, jpg:gd, jpg:gd:gd, gif, gif:cairo,\n# gif:cairo:gd, gif:gd, gif:gd:gd and svg.\n# The default value is: png.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_IMAGE_FORMAT       = png\n\n# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to\n# enable generation of interactive SVG images that allow zooming and panning.\n#\n# Note that this requires a modern browser other than Internet Explorer. Tested\n# and working are Firefox, Chrome, Safari, and Opera.\n# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make\n# the SVG files visible. Older versions of IE do not have SVG support.\n# The default value is: NO.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nINTERACTIVE_SVG        = NO\n\n# The DOT_PATH tag can be used to specify the path where the dot tool can be\n# found. If left blank, it is assumed the dot tool can be found in the path.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_PATH               =\n\n# The DOTFILE_DIRS tag can be used to specify one or more directories that\n# contain dot files that are included in the documentation (see the \\dotfile\n# command).\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOTFILE_DIRS           =\n\n# The MSCFILE_DIRS tag can be used to specify one or more directories that\n# contain msc files that are included in the documentation (see the \\mscfile\n# command).\n\nMSCFILE_DIRS           =\n\n# The DIAFILE_DIRS tag can be used to specify one or more directories that\n# contain dia files that are included in the documentation (see the \\diafile\n# command).\n\nDIAFILE_DIRS           =\n\n# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the\n# path where java can find the plantuml.jar file. If left blank, it is assumed\n# PlantUML is not used or called during a preprocessing step. Doxygen will\n# generate a warning when it encounters a \\startuml command in this case and\n# will not generate output for the diagram.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nPLANTUML_JAR_PATH      =\n\n# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes\n# that will be shown in the graph. If the number of nodes in a graph becomes\n# larger than this value, doxygen will truncate the graph, which is visualized\n# by representing a node as a red box. Note that doxygen if the number of direct\n# children of the root node in a graph is already larger than\n# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that\n# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.\n# Minimum value: 0, maximum value: 10000, default value: 50.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_GRAPH_MAX_NODES    = 50\n\n# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs\n# generated by dot. A depth value of 3 means that only nodes reachable from the\n# root by following a path via at most 3 edges will be shown. Nodes that lay\n# further from the root node will be omitted. Note that setting this option to 1\n# or 2 may greatly reduce the computation time needed for large code bases. Also\n# note that the size of a graph can be further restricted by\n# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.\n# Minimum value: 0, maximum value: 1000, default value: 0.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nMAX_DOT_GRAPH_DEPTH    = 0\n\n# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent\n# background. This is disabled by default, because dot on Windows does not seem\n# to support this out of the box.\n#\n# Warning: Depending on the platform used, enabling this option may lead to\n# badly anti-aliased labels on the edges of a graph (i.e. they become hard to\n# read).\n# The default value is: NO.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_TRANSPARENT        = NO\n\n# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output\n# files in one run (i.e. multiple -o and -T options on the command line). This\n# makes dot run faster, but since only newer versions of dot (>1.8.10) support\n# this, this feature is disabled by default.\n# The default value is: NO.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_MULTI_TARGETS      = YES\n\n# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page\n# explaining the meaning of the various boxes and arrows in the dot generated\n# graphs.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nGENERATE_LEGEND        = YES\n\n# If the DOT_CLEANUP tag is set to YES doxygen will remove the intermediate dot\n# files that are used to generate the various graphs.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_CLEANUP            = YES\n"
  },
  {
    "path": "3rdparty/mshadow/doc/README.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nMShadow Documentation\n=====\nThis is the documentation for mshadow: A Lightweight CPU/GPU Matrix/Tensor Template Library in C++/CUDA. \n\n### Links to Topics\n\n* [Tutorial](../guide)\n* [API Documentation](http://homes.cs.washington.edu/~tqchen/mshadow/doc)\n  - You can run ```./mkdoc.sh``` to make the document locally\n* [Tutorial about Expression Template](../guide/exp-template)\n* [Writing Multi-GPU and Distributed ML](../guide/mshadow-ps)\n* [Compile Configuration script](../make)\n* [Expression API](#expression-api)\n  - Expression api introduces the concept of expression in mshadow\n\nExpression API\n=====\nExpression is the key concept in mshadow, a common operation of mshadow is ```tensor = some code to construct expression```\n\nThere are three major types of expression:\n* Mapper expression: only contain element-wise operations of Mapper expressions  \n  - Mapper expression can used as composition component of other operations.\n  - Tensor, scalar are Mapper expressions\n  - Example: ``` weight =  - eta * (grad + lambda * weight)```  is a Mapper expression.\n  - Mapper expressions are translated using expression template code implemented by mshadow.\n  - ***Assign safety***: Element-wise mapping are assign safe, which means, we can write ```A = A * 2 + B```, making lvalue appear in expression, the results are still correct.\n* Chainer expression: may contain element-wise operation such as reduction and broadcast\n  - Example: ```dst = mirror(src)``` is a chainer expression\n  - ***Assign safety***: Most of the chainer extensions are not assignment safe, which means user should avoid putting target in source epression.\n* Complex expression: complex operations, need special translation rule to translate to specific implementations.\n   - Complex expression can not be used as composition component of other operations.\n   - Example: ``` dot(lhs.T(), rhs)```,  is complex expression, we can not write\n``` dst =  1.0 + dot(lhs.T(), rhs)```\n   - But limited syntax is supported depending on specification, for example, we do support ``` dst +=  2.0f * dot(lhs.T(), rhs)```\n   - Complex expressions are translated into specific implementations such as BLAS.\n\n### Element-wise Operations\nThe basic binary operators are overloaded to composite Mapper expressions, so we can write \n```c++\nweight = (-eta) * (grad + lambda * weight);\n```\nWe can also use customized binary operators, and unary operators:\n```c++\nstruct maximum {\n  MSHADOW_XINLINE static float Map(float a, float b) {\n    return a > b ? a : b;\n  }\n};\ntemplate<typename xpu>\nvoid ExampleMaximum(Tensor<xpu, 2> out,\n                    const Tensor<xpu, 2> &A,\n                    const Tensor<xpu, 2> &B) {\n  out= 10.0f * F<maximum>(A+1.0f, B); \n}\nstruct sigmoid {\n  MSHADOW_XINLINE static float Map(float a) {\n    return 1.0f/(1.0f+expf(-a));\n  }\n};\ntemplate<typename xpu>\nvoid ExampleSigmoid(Tensor<xpu, 2> out, const Tensor<xpu, 2> &in) {\n  // equivalent to out = sigmoid(in*2) + 1; \n  out = F<op::plus>(F<sigmoid>(in * 2.0f), ScalarExp(1.0f));\n}\n```\n### Matrix Multiplications\nMatrix multiplications are supported by following syntax, with things brackets [] are optional\n```\ndst <sv> [scale*] dot(lhs [.T()] , rhs [.T()]), <sv> can be =,+=,-=\n```\nExample:\n```c++\ntemplate<typename xpu>\nvoid Backprop(Tensor<xpu, 2> gradin,\n              const Tensor<xpu, 2> &gradout,\n              const Tensor<xpu, 2> &netweight) {\n  gradin = 2.0 * dot(gradout, netweight.T());\n}\n```\n\n### Introducing Expression Extensions\nNaming conventions:\n* ```Tensor<xpu, dim>``` to refer to any Tensor with device any device and dimension. \n* ```xpu```, ```dim```, are implicit template parameters. \n* ```Expr<xpu, dim>``` will be used to refer to any mapper expression with type ```Tensor<xpu,dim>```.\n\nList of functions:\n* [reshape](#reshape): reshapes a tensor to another shape, number of content must be same\n* [broadcast<?>](#broadcast): replicate a 1 dimension tensor in certain dimension\n* [repmat](#repmat), special case of broadcast<0>: repeat vector over rows to form a matrix\n* [sumall_except_dim<?>](#sumall_except_dim): sum over all the dimensions, except the dimension specified in template parameter\n* [sum_rows](#sum_rows): special case of sumall_except_dim<0>, sum of rows in the matrix\n* [unpack_patch2col](#unpack_patch2col): unpack local (overlap) patches of image to column of mat, can be used to implement convolution\n* [pack_col2patch](#pack_col2patch): reverse operation of unpack_patch2col, can be used to implement deconvolution\n* [pool](#pool): do pooling on image\n* [unpool](#unpool): get gradient of pooling result\n* [crop](#crop): crop the original image to a smaller size\n* [mirror](#mirror): get the mirrored result of input expression\n\n======\n##### reshape\n* ```reshape(Expr<xpu,dim> src, Shape<dimdst> oshape)```\n* reshapes a tensor to another shape, total number of elements must be same\n* parameters:\n  - src:  input data\n  - oshape: target shape\n* result expression type: ```Tensor<xpu, dimdst>``` with ```shape=oshape```, is Mapper expression\n```c++\nvoid ExampleReshape(void) {\n  Tensor<cpu, 2> dst = NewTensor<cpu>(Shape2(4, 5));\n  Tensor<cpu, 1> src = NewTensor<cpu>(Shape1(20), 1.0f); \n  dst = reshape(src, dst.shape_);\n  ...\n}\n```\n======\n\n##### broadcast\n* ```broadcast<dimcast>(Tensor<xpu,1> src, Shape<dimdst> oshape)```\n* replicate a 1 dimension tensor certain dimension, specified by template parameter dimcast\n* parameters:\n  - src: input 1 dimensional tensor\n  - oshape: shape of output\n* return expression type: ```Tensor<xpu, dimdst>```, ```shape = oshape```, is Chainer expression \n```c++\nvoid ExampleBroadcast(void) {\n  Tensor<cpu, 2> dst = NewTensor<cpu>(Shape2(2, 3));\n  Tensor<cpu, 1> src = NewTensor<cpu>(Shape1(2), 1.0f);\n  src[0] = 2.0f; src[1] = 1.0f;\n  dst = broadcast<0>(src, dst.shape_);\n  // dst[0][0] = 2, dst[0][1] = 2; dst[1][0]=1, dst[1][1] = 1\n  ...\n}\n```\n======\n##### repmat\n* ```repmat(Tensor<xpu, 1> src, int nrows) ```\n* special case of broadcast, repeat 1d tensor over rows\n* input parameters:\n  - src: input vector\n  - nrows: number of rows in target\n* return expression type:  ```Tensor<xpu, 2>```, with ```shape=(nrows, src.size(0))```,  is Chainer expression\n```c++\nvoid ExampleRepmat(void) {\n  Tensor<cpu,2> dst = NewTensor<cpu>(Shape2(3, 2));\n  Tensor<cpu,1> src = NewTensor<cpu>(Shape1(2), 1.0f);\n  src[0] = 2.0f; src[1] = 1.0f;\n  dst = repmat(src, 3);\n  // dst[0][0] = 2, dst[0][1] = 1; dst[1][0]=2, dst[1][1] = 1\n  ...\n}\n```\n======\n##### sumall_except_dim\n* ```sumall_except_dim<dimkeep>(Expr<xpu,dim> src) ```\n* sum over all dimensions, except dimkeep\n* input parameters:\n  - src: input mapper expression\n* return expression type:  ```Tensor<xpu, 1>```, with ```shape=(src.size(dimkeep))```,  is Complex expression\n* Syntax: ```dst [sv] [scale*] sumall_except_dim<dimkeep>(src) , <sv> can be =, +=, -=, *=, /=````\n```c++\nvoid ExampleSumAllExceptDim(void) {\n  Tensor<cpu,3> src = NewTensor<cpu>(Shape3(2, 3, 2), 1.0f);\n  Tensor<cpu,1> dst = NewTensor<cpu>(Shape1(3), 1.0f);\n  dst += sum_all_except<1>(src * 2.0f);\n  // dst[0] = 1.0 + 4.0 *2.0 = 9.0\n  ...\n}\n```\n======\n##### sum_rows\n* ```sum_rows(Expr<xpu, 2> src) ```\n* sum of rows in the matrix\n* input parameters:\n  - src: input mapper  expression\n* return expression type:  ```Tensor<xpu,1>```, with ```shape=(src.size(0))```,  is Complex expression\n* Syntax: ```dst [sv] [scale*] sum_rows(src) , <sv> can be =,+=,-=,*=,/=````\n```c++\nvoid ExampleSumRows(void) {\n  Tensor<cpu, 2> src = NewTensor<cpu>(Shape2(3, 2), 1.0f);\n  Tensor<cpu, 1> dst = NewTensor<cpu>(Shape1(2), 1.0f);\n  dst += sum_rows(src + 1.0f);\n  // dst[0] = 1.0 + 3.0 *(1.0+1.0) = 7.0\n  ...\n}\n```\n======\n##### unpack_patch2col\n* ```unpack_patch2col(Expr<xpu,3> img, int psize_y, int p_size_x, int pstride) ```\n* unpack local (overlap) patches of image to column of mat, can be used to implement convolution, after getting unpacked mat, we can use: ```output = dot(weight, mat)``` to get covolved results, the relations:\n  - weight; shape[0]: out_channel, shape[1]: ichannel * psize_y * psize_x\n  - output; shape[0]: out_channel, shape[1]: out_height * out_width * num_of_images\n  -  out_height = (in_height - psize_y) / pstride + 1, this means we pad inperfect patch with 0\n  - out_width  = (in_width - psize_x) / pstride + 1\n* input parameters:\n  - img: source image, can be expression; (in_channels, in_height, in_width)\n  - psize_y height of each patch\n  - psize_x width of each patch\n  - pstride: stride of each patch\n* return expression type:  ```Tensor<xpu, 2>```, with ```shape=(in_channel*psize_x*psize_y, out_height*out_width)```,  is Chainer expression\n```c++\nvoid ExampleCovolution(Tensor<cpu, 3> dst, Tensor<cpu, 3> src,\n                       Tensor<cpu, 2> weight, int ksize, int stride) {\n  int o_height = (src.size(1)- ksize) / stride + 1;\n  int o_width  = (src.size(2)- ksize) / stride + 1;\n  utils::Assert(weight.size(1) == src.size(0) * ksize * ksize);\n  TensorContainer<cpu, 2> tmp_col(Shape2(src.size(0) * ksize * ksize,\n                                         o_height * o_width)); \n  TensorContainer<cpu, 2> tmp_dst(Shape2(weight.size(0),\n                                         o_height * o_width)); \n  tmp_col = unpack_patch2col(src, ksize, ksize, stride);\n  tmp_dst = dot(weight, tmp_col);\n  dst = reshape(tmp_dst, dst.shape_);\n}\n```\n\n======\n##### pack_col2patch\n* ```pack_col2patch(Tensor<xpu, 2> mat, Shape<3> imshape, int psize_y, int psize_x, int pstride) ````\n* reverse operation of unpack_patch2col, can be used to implement deconvolution\n* input parameters:\n  - mat: source mat, same shape as output of unpack_patch2col\n  - imshape: shape of target image\n  - psize_y height of each patch\n  - psize_x width of each patch\n  - pstride: stride of each patch\n* return expression type:  ```Tensor<xpu, 3>```, with ```shape = imshape```,  is Chainer expression\n```c++\nvoid ExampleDecovolution(Tensor<cpu, 3> bottom, Tensor<cpu, 3> top,\n                         Tensor<cpu, 2> weight, int ksize, int stride) {\n  int o_height = (bottom.size(1)- ksize) / stride + 1;\n  int o_width  = (bottom.size(2)- ksize) / stride + 1;\n  utils::Assert(weight.size(1) == bottom.size(0) * ksize * ksize);\n  TensorContainer<cpu, 2> tmp_col(Shape2(bottom.size(0) * ksize * ksize,\n                                         o_height * o_width)); \n  TensorContainer<cpu, 2> tmp_dst(Shape2(weight.size(0), o_height*o_width)); \n  tmp_dst = reshape(top, tmp_dst.shape_);\n  tmp_col = dot(weight.T(), tmp_dst);\n  bottom = pack_col2patch(tmp_col, bottom.shape_, ksize, ksize, stride);\n}\n```\n\n======\n##### pool\n* ```pool<Reducer>(Expr<xpu, dim> img, [Shape<2> pshape,] int ksize_y, int ksize_x, int kstride)```\n* Pooling on image with specify kernel size and stride, can be used to implement max pooilng and other pooling layer\n* input parameters:\n  - Reducer: operation can be max or sum\n  - img: source image, can be expression; (in_channels, in_height, in_width)\n  - [optional] Shape<2> pshape, output shape\n  - ksize_y height of each patch\n  - ksize_x width of each patch\n  - kstride: stride of each patch\n* return expression:  ```Expr<xpu, dim>```, with ```shape = (in_channel, (out_height - ksize) / kstride + 1, (out_width - ksize) / kstride + 1)```, or expression in pshape\n  - Chainer expression\n```c++\nvoid ExampleMaxPooling(TensorContainer<cpu, 3> &data, int ksize, int stride) {\n  TensorContainer<cpu, 3> pooled(Shape3(data.size(0),\n                                        (data.size(2) - ksize) / kstride + 1), \n                                        (data.size(1) - ksize) / kstride + 1));\n  pooled = pool<red::maximum>(data, ksize, ksize, stride);\n}\n```\n\n======\n##### unpool\n* ```unpool<Reducer>(Tensor<xpu, 4> data_src, Tensor<xpu, 4> data_pooled, Tensor<xpu, 4> grad_pooled, int ksize_y,  int ksize_x, int kstride)```\n* Unpooling on image with specify kernel size and stride, can be used to implement backprop of max pooilng and other pooling layer\n* input parameters:\n  - Reducer: operation can be max or sum\n  - data_src: source image batch. \n  - data_pooled: pooled image batch. \n  - grad_pooled: gradient of upper layer\n  - ksize_y height of each patch\n  - ksize_x width of each patch\n  - kstride: stride of each patch\n* return:\n  Expression, same shape to data_src\n```c++\nvoid ExampleMaxUnpooling(Tensor<cpu, 4> &data_src, Tensor<cpu, 4> &data_pooled, \n                         Tensor<cpu, 4> &grad_pooled, int ksize, int kstride) {\n  TensorContainer<cpu, 4> grad(data_src.shape_);\n  grad = unpool<red::maximum>(data_src, data_pooled,\n                              grad_pooled, ksize, ksize, kstride);\n}\n```\n\n======\n##### crop\n* ```crop(Expr<xpu, dim> src, Shape<2> oshape, int start_height, int start_width)```\n* input parameters:\n - src: input expression \n - oshape: output shape after crop\n - start_height: start height for cropping\n - start_width: start width for cropping\n* Can also be ```crop(Expr<xpu, dim> src, Shape<2> oshape)``` where the crop will happen in center. \n* return\n - cropped expression\n```c++\nvoid ExampleCrop(TensorContainer<cpu, 3> img, int start_height, int start_width) {\n  TensorContainer<cpu> cropped(Shape3(img.size(0),\n                                      img.size(1) - start_height,\n                                      img.size(2) - start_width));\n  cropped = crop(img, start_height, start_width);\n}\n```\n\n======\n##### mirror\n* ```mirrow(Expr<xpu, dim> src)```\n* input:\n    - src, source expression to be mirrored\n* output:\n    - expression of mirrored result\n```c++\nvoid ExampleMirror(TensorContainer<cpu, 3> img) {\n  TensorContainer<cpu> mirrored(img.shape_);\n  mirrored = mirror(img);\n}\n```\n\n"
  },
  {
    "path": "3rdparty/mshadow/doc/mkdoc.sh",
    "content": "#!/bin/bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\ncd ..\ndoxygen doc/Doxyfile\ncd doc\n"
  },
  {
    "path": "3rdparty/mshadow/guide/.gitignore",
    "content": "defop\nbasic\nconfig.mk\n"
  },
  {
    "path": "3rdparty/mshadow/guide/Makefile",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# set LD_LIBRARY_PATH\nexport CC  = gcc\nexport CXX = g++\nexport NVCC =nvcc\ninclude config.mk\ninclude ../make/mshadow.mk\nexport CFLAGS = -Wall -O3 -std=c++17 -I../ $(MSHADOW_CFLAGS)\nexport LDFLAGS= -lm $(MSHADOW_LDFLAGS)\nexport NVCCFLAGS = -O3 --use_fast_math -ccbin $(CXX) $(MSHADOW_NVCCFLAGS)\n\n# specify tensor path\nBIN = basic defop\nOBJ =\nCUOBJ =\nCUBIN =\n.PHONY: clean all\n\nall: $(BIN) $(OBJ) $(CUBIN) $(CUOBJ)\n\nbasic: basic.cpp\ndefop: defop.cpp\nbasic_stream: basic_stream.cu\n\n$(BIN) :\n\t$(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c, $^)  $(LDFLAGS)\n\n$(OBJ) :\n\t$(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c, $^) )\n\n$(CUOBJ) :\n\t$(NVCC) -c -o $@ $(NVCCFLAGS) -Xcompiler \"$(CFLAGS)\" $(filter %.cu, $^)\n\n$(CUBIN) :\n\t$(NVCC) -o $@ $(NVCCFLAGS) -Xcompiler \"$(CFLAGS)\" -Xlinker \"$(LDFLAGS)\" $(filter %.cu %.cpp %.o, $^)\n\nclean:\n\t$(RM) $(OBJ) $(BIN) $(CUBIN) $(CUOBJ) *~\n"
  },
  {
    "path": "3rdparty/mshadow/guide/README.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nTutorial of mshadow\n=====\nThis is a beginner's tutorial for mshadow. If you like mshadow and have ideas to improve this tutorial, you are more than welcome to contribute :)\nPlease send a pull-request if you would like to share your experience.\n\nSee also other related materials about mshadow\n* [Expression Template Tutorial](exp-template)\n* [Writing Multi-GPU and Distributed ML](mshadow-ps)\n\n**List of Topics**\n* [Tensor Data Structure](#tensor-data-structure)\n* [Memory Allocation](#memory-allocation)\n* [Elementwise Operations](#elementwise-operations)\n* [One code for both CPU and GPU](#one-code-for-both-cpu-and-gpu)\n* [Matrix Multiplications](#matrix-multiplications)\n* [User Defined Operator](#user-defined-operator)\n\nTensor Data Structure\n====\nThe basic data structure of mshadow is Tensor. The following is a simplified equivalent version of\nthe declaration in [mashadow/tensor.h](../mshadow/tensor.h)\n```c++\ntypedef unsigned index_t;\ntemplate<int dimension>\nstruct Shape {\n  index_t shape_[dimension];\n};\ntemplate<typename Device, int dimension, typename DType = float>\nstruct Tensor {\n  DType *dptr_;\n  Shape<dimension> shape_;\n  Stream<Device> stream_;\n  index_t stride_;\n};\n// this is how shape object declaration look like\nShape<2> shape2;\n// this is how tensor object declaration look like\n// you can\nTensor<cpu, 2> ts2;\nTensor<gpu, 3, float> ts3;\n```\n``` Tensor<cpu,2>``` is a two dimensional tensor in host memory, while ```Tensor<gpu,3>``` is a three dimensional tensor in device memory.\n```Shape<k>``` gives the shape information of a k-dimensional tensor. The declarations use templates and\ncan be specialized to tensors on a specific device and of a specific dimension. This is what a two dimensional tensor would look like:\n```c++\nstruct Shape<2> {\n  index_t shape_[2];\n};\nstruct Tensor<cpu, 2, float> {\n  float *dptr_;\n  Shape<2> shape_;\n  index_t stride_;\n};\n```\n* ``` Tensor<cpu, 2>``` contains ```dptr_```, which points to the space that backs up the tensor.\n* ```Shape<2>``` is a structure that stores shape information, the convention is the same as numpy.\n* ```stride_``` gives the number of cell spaces allocated in the smallest dimension (if we use numpy convention, the dimension corresponds to shape_[-1]).\nThis is introduced when we introduce some padding cells in lowest dimension to make sure memory is aligned. ```stride_``` is automatically set during\nmemory allocation of a tensor in mshadow.\n\nTo understand the data structure, consider the following code:\n``` c++\nfloat data[9] = {0, 1, 2, 3, 4, 5, 6, 7, 8};\nTensor<cpu, 2> ts;\nts.dptr_ = data;\nts.shape_ = mshadow::Shape2(3, 2);\nts.stride_ = 3;\n// now: ts[0][0] == 0, ts[0][1] == 1 , ts[1][0] == 3, ts[1][1] == 4\nfor (index_t i = 0; i < ts.size(0); ++i) {\n  for (index_t j = 0; j < ts.size(1); ++j) {\n    printf(\"ts[%u][%u]=%f\\n\", i, j, ts[i][j]);\n  }\n}\n```\nThe result ts should be a 3 * 2 matrix, where data[2], data[5], data[8] are padding cells that are ignored. If you want a continuous memory, set ```stride_=shape_[1]```.\n\nNOTICE: We highly recommend use stream in ```gpu``` mode, there will be an error thrown out if no stream is set. Check [basic_stream.cu](basic_stream.cu) for more detail.\n\nMemory Allocation\n====\nAn important design choice in mshadow was making the data structure ```Tensor``` a **whitebox**,\nit works so long as we set the space pointer ```dptr_``` corresponding ```shape_``` and ```stride_```:\n* For ```Tensor<cpu, k>``` ```dptr_``` must point to space created by ```new float[]``` or to some existing space such as the float array in the last example.\n* For ```Tensor<gpu, k>``` ```dptr_``` must point to space on the device created by ```cudaMallocPitch```.\n\nmshadow also provides an explicit memory allocation routine, as shown in following code:\n``` c++\n// create a 5 x 3 tensor on the device, and allocate space\nTensor<gpu, 2> ts2(Shape2(5, 3));\nAllocSpace(&ts2);\n// allocate 5 x 3 x 2 tensor on the host, initialized by 0\nTensor<cpu, 3> ts3 = NewTensor<cpu>(Shape3(5,3,2), 0.0f);\n// free space\nFreeSpace(&ts2); FreeSpace(&ts3);\n```\nAll memory allocations in mshadow are **explicit**. There are **no** implicit memory allocations or de-allocations during any operations.\nThis means ```Tensor<cpu, k>``` variable is more like a reference handle(pointer), instead of a object. If we assign a tensor to another variable, the two share the same content space.\n\nThis also allows user to use mshadow in their existing project easily, simply give mshadow the pointer of the memory and you can get the benefit of all the mshadow expressions with zero cost:)\n\nWe also have STL style container object called ```TensorContainer```, they behave exactly the same as Tensors, but the memory will be automatically freed during destruction.\n\nElementwise Operations\n====\nAll the operators(+, -, *, /, += etc.) in mshadow are element-wise. Consider the following SGD update code:\n```c++\nvoid UpdateSGD(Tensor<cpu, 2> weight, Tensor<cpu, 2> grad, float eta, float lambda) {\n  weight -= eta * (grad + lambda * weight);\n}\n```\nDuring compilation, this code will be translated to the following form:\n```c++\nvoid UpdateSGD(Tensor<cpu,2> weight, Tensor<cpu,2> grad, float eta, float lambda) {\n  for (index_t y = 0; y < weight.size(0); ++y) {\n    for (index_t x = 0; x < weight.size(1); ++x) {\n      weight[y][x] -= eta * (grad[y][x] + lambda * weight[y][x]);\n    }\n  }\n}\n```\nAs we can see, *no memory allocation* happens in the translated code. For ```Tensor<gpu, k>```, the corresponding function will be translated into a CUDA kernel of the same spirit.\nUsing an [Expression Template](exp-template), the translation happens at compile time. We can write simple lines of code while getting the full performance of the translated code.\n\nOne code for both CPU and GPU\n====\nSince mshadow has an identical interface for ```Tensor<cpu, k>``` and ```Tensor<gpu, k>```, we can easily write code that works on both the CPU and GPU.\nFor example, the following code compiles for both GPU and CPU Tensors.\n```c++\ntemplate<typename xpu>\nvoid UpdateSGD(Tensor<xpu, 2> weight, const Tensor<xpu, 2> &grad,\n               float eta, float lambda) {\n  weight -= eta * (grad + lambda * weight);\n}\n```\nMatrix Multiplications\n====\nWe also have a shorthand for dot product that will be translated to call standard packages such as MKL and CuBLAS.\n```c++\ntemplate<typename xpu>\nvoid Backprop(Tensor<xpu, 2> gradin,\n              const Tensor<xpu, 2> &gradout,\n              const Tensor<xpu, 2> &netweight) {\n  gradin = dot(gradout, netweight.T());\n}\n```\nAgain, the code can compile for both GPU and CPU Tensors.\n\nUser Defined Operator\n====\nThere are common cases when we want to define our own function. For example, assume we do not have an element-wise sigmoid transformation in mshadow.\nWe simply use the following code to add ```sigmoid``` to mshadow\n```c++\nstruct sigmoid {\n  MSHADOW_XINLINE static float Map(float a) {\n    return 1.0f / (1.0f + expf(-a));\n  }\n};\ntemplate<typename xpu>\nvoid ExampleSigmoid(Tensor<xpu, 2> out, const Tensor<xpu, 2> &in) {\n  out = F<sigmoid>(in * 2.0f) + 1.0f;\n}\n```\nThe translated code for CPU is given by\n```c++\ntemplate<typename xpu>\nvoid ExampleSigmoid(Tensor<xpu, 2> out, const Tensor<xpu, 2> &in) {\n  for (index_t y = 0; y < out.size(0); ++y) {\n    for(index_t x = 0; x < out.size(1); ++x) {\n      out[y][x] = sigmoid::Map(in[y][x] * 2.0f) + 1.0f;\n    }\n  }\n}\n```\nAlso note that the defined operation can be **composited into expressions**, not only we can write ```out = F<sigmoid>(in)```,\nwe can also write ```out = F<sigmoid>+2.0``` or ```out = F<sigmoid>(F<sigmoid>(in))```.\n\nThere will also be a translated CUDA kernel version that runs on the GPU. Check out [defop.cpp](defop.cpp) for a complete example.\n\nComplete Example\n====\nThe following code is from [basic.cpp](basic.cpp). It illustrates basic usage of mshadow.\n\n```c++\n// header file to use mshadow\n#include \"mshadow/tensor.h\"\n// this namespace contains all data structures, functions\nusing namespace mshadow;\n// this namespace contains all operator overloads\nusing namespace mshadow::expr;\n\nint main(void) {\n  // intialize tensor engine before using tensor operation, needed for CuBLAS\n  InitTensorEngine<cpu>();\n  // assume we have a float space\n  float data[20];\n  // create a 2 x 5 x 2 tensor, from existing space\n  Tensor<cpu, 3> ts(data, Shape3(2,5,2));\n    // take first subscript of the tensor\n  Tensor<cpu, 2> mat = ts[0];\n  // Tensor object is only a handle, assignment means they have same data content\n  // we can specify content type of a Tensor, if not specified, it is float bydefault\n  Tensor<cpu, 2, float> mat2 = mat;\n\n  // shape of matrix, note size order is the same as numpy\n  printf(\"%u X %u matrix\\n\", mat.size(0), mat.size(1));\n\n  // initialize all element to zero\n  mat = 0.0f;\n  // assign some values\n  mat[0][1] = 1.0f; mat[1][0] = 2.0f;\n  // elementwise operations\n  mat += (mat + 10.0f) / 10.0f + 2.0f;\n\n  // print out matrix, note: mat2 and mat1 are handles(pointers)\n  for (index_t i = 0; i < mat.size(0); ++i) {\n    for (index_t j = 0; j < mat.size(1); ++j) {\n      printf(\"%.2f \", mat2[i][j]);\n    }\n    printf(\"\\n\");\n  }\n  // shutdown tensor enigne after usage\n  ShutdownTensorEngine<cpu>();\n  return 0;\n}\n```\n\n"
  },
  {
    "path": "3rdparty/mshadow/guide/basic.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n// header file to use mshadow\n#include \"mshadow/tensor.h\"\n// this namespace contains all data structures, functions\nusing namespace mshadow;\n// this namespace contains all operator overloads\nusing namespace mshadow::expr;\n\nint main(void) {\n  // intialize tensor engine before using tensor operation, needed for CuBLAS\n  InitTensorEngine<cpu>();\n  // assume we have a float space\n  float data[20];\n  // create a 2 x 5 x 2 tensor, from existing space\n  Tensor<cpu, 3> ts(data, Shape3(2,5,2));\n  // take first subscript of the tensor\n  Tensor<cpu, 2> mat = ts[0];\n  // Tensor object is only a handle, assignment means they have same data content\n  // we can specify content type of a Tensor, if not specified, it is float bydefault\n  Tensor<cpu, 2, float> mat2 = mat;\n  mat = Tensor<cpu, 1>(data, Shape1(10)).FlatTo2D();\n\n  // shaape of matrix, note size order is same as numpy\n  printf(\"%u X %u matrix\\n\", mat.size(0), mat.size(1));\n\n  // initialize all element to zero\n  mat = 0.0f;\n  // assign some values\n  mat[0][1] = 1.0f; mat[1][0] = 2.0f;\n  // elementwise operations\n  mat += (mat + 10.0f) / 10.0f + 2.0f;\n\n  // print out matrix, note: mat2 and mat1 are handles(pointers)\n  for (index_t i = 0; i < mat.size(0); ++i) {\n    for (index_t j = 0; j < mat.size(1); ++j) {\n      printf(\"%.2f \", mat2[i][j]);\n    }\n    printf(\"\\n\");\n  }\n\n  TensorContainer<cpu, 2> lhs(Shape2(2, 3)), rhs(Shape2(2, 3)), ret(Shape2(2,2));\n  lhs = 1.0;\n  rhs = 1.0;\n  ret = implicit_dot(lhs, rhs.T());\n  VectorDot(ret[0].Slice(0, 1), lhs[0], rhs[0]);\n  printf(\"vdot=%f\\n\", ret[0][0]);\n  int cnt = 0;\n  for (index_t i = 0; i < ret.size(0); ++i) {\n    for (index_t j = 0; j < ret.size(1); ++j) {\n      printf(\"%.2f \", ret[i][j]);\n    }\n    printf(\"\\n\");\n  }\n\n  printf(\"\\n\");\n\n  for (index_t i = 0; i < lhs.size(0); ++i) {\n    for (index_t j = 0; j < lhs.size(1); ++j) {\n      lhs[i][j] = cnt++;\n      printf(\"%.2f \", lhs[i][j]);\n    }\n    printf(\"\\n\");\n  }\n  printf(\"\\n\");\n  TensorContainer<cpu, 1> index(Shape1(2)), choosed(Shape1(2));\n  index[0] = 1; index[1] = 2;\n  choosed = mat_choose_row_element(lhs, index);\n  for (index_t i = 0; i < choosed.size(0); ++i) {\n    printf(\"%.2f \", choosed[i]);\n  }\n  printf(\"\\n\");\n\n  TensorContainer<cpu, 2> recover_lhs(Shape2(2, 3)), small_mat(Shape2(2, 3));\n  small_mat = -100.0f;\n  recover_lhs = mat_fill_row_element(small_mat, choosed, index);\n  for (index_t i = 0; i < recover_lhs.size(0); ++i) {\n    for (index_t j = 0; j < recover_lhs.size(1); ++j) {\n      printf(\"%.2f \", recover_lhs[i][j] - lhs[i][j]);\n    }\n  }\n  printf(\"\\n\");\n\n  rhs = one_hot_encode(index, 3);\n\n  for (index_t i = 0; i < lhs.size(0); ++i) {\n    for (index_t j = 0; j < lhs.size(1); ++j) {\n      printf(\"%.2f \", rhs[i][j]);\n    }\n    printf(\"\\n\");\n  }\n  printf(\"\\n\");\n  TensorContainer<cpu, 1> idx(Shape1(3));\n  idx[0] = 8;\n  idx[1] = 0;\n  idx[2] = 1;\n\n  TensorContainer<cpu, 2> weight(Shape2(10, 5));\n  TensorContainer<cpu, 2> embed(Shape2(3, 5));\n\n  for (index_t i = 0; i < weight.size(0); ++i) {\n    for (index_t j = 0; j < weight.size(1); ++j) {\n      weight[i][j] = i;\n    }\n  }\n  embed = take(idx, weight);\n  for (index_t i = 0; i < embed.size(0); ++i) {\n    for (index_t j = 0; j < embed.size(1); ++j) {\n      printf(\"%.2f \", embed[i][j]);\n    }\n    printf(\"\\n\");\n  }\n  printf(\"\\n\\n\");\n  weight = take_grad(idx, embed, 10);\n  for (index_t i = 0; i < weight.size(0); ++i) {\n    for (index_t j = 0; j < weight.size(1); ++j) {\n      printf(\"%.2f \", weight[i][j]);\n    }\n    printf(\"\\n\");\n  }\n\n  printf(\"upsampling\\n\");\n  TensorContainer<cpu, 2> small(Shape2(2, 2));\n  small[0][0] = 1.0f;\n  small[0][1] = 2.0f;\n  small[1][0] = 3.0f;\n  small[1][1] = 4.0f;\n  TensorContainer<cpu, 2> large(Shape2(6, 6));\n  large = upsampling_nearest(small, 3);\n  for (index_t i = 0; i < large.size(0); ++i) {\n    for (index_t j = 0; j < large.size(1); ++j) {\n      printf(\"%.2f \", large[i][j]);\n    }\n    printf(\"\\n\");\n  }\n  small = pool<red::sum>(large, small.shape_, 3, 3, 3, 3);\n  // shutdown tensor enigne after usage\n  for (index_t i = 0; i < small.size(0); ++i) {\n    for (index_t j = 0; j < small.size(1); ++j) {\n      printf(\"%.2f \", small[i][j]);\n    }\n    printf(\"\\n\");\n  }\n\n  printf(\"mask\\n\");\n  TensorContainer<cpu, 2> mask_data(Shape2(6, 8));\n  TensorContainer<cpu, 2> mask_out(Shape2(6, 8));\n  TensorContainer<cpu, 1> mask_src(Shape1(6));\n\n  mask_data = 1.0f;\n  for (int i = 0; i < 6; ++i) {\n    mask_src[i] = static_cast<float>(i);\n  }\n  mask_out = mask(mask_src, mask_data);\n  for (index_t i = 0; i < mask_out.size(0); ++i) {\n    for (index_t j = 0; j < mask_out.size(1); ++j) {\n      printf(\"%.2f \", mask_out[i][j]);\n    }\n    printf(\"\\n\");\n  }\n  ShutdownTensorEngine<cpu>();\n  return 0;\n}\n"
  },
  {
    "path": "3rdparty/mshadow/guide/basic_stream.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n// header file to use mshadow\n#include \"mshadow/tensor.h\"\n// this namespace contains all data structures, functions\nusing namespace mshadow;\n// this namespace contains all operator overloads\nusing namespace mshadow::expr;\n\nint main(void) {\n  // intialize tensor engine before using tensor operation, needed for CuBLAS\n  InitTensorEngine<gpu>();\n  // create a 2 x 5 tensor, from existing space\n  Stream<gpu> *sm1 = NewStream<gpu>();\n  Stream<gpu> *sm2 = NewStream<gpu>();\n  Tensor<gpu, 2, float> ts1 = NewTensor<gpu, float>(Shape2(2, 5), 0.0f, sm1);\n  Tensor<gpu, 2, float> ts2 = NewTensor<gpu, float>(Shape2(2, 5), 0.0f, sm2);\n  ts1 = 1; // Should use stream 0.\n  ts2 = 2; // Should use stream 1. Can run in parallel with stream 0.\n  Tensor<gpu, 2> res = NewTensor<gpu, float>(Shape2(2, 2), 0.0f);\n  res.stream_ = NewStream<gpu>();\n  res = dot(ts1, ts2.T()); //Should use stream 2.\n\n  Tensor<cpu, 2> cpu_res = NewTensor<cpu, float>(Shape2(2, 2), 0.0f);\n  Copy(cpu_res, res); // default stream, should be 0.\n  for (index_t i = 0; i < cpu_res.size(0); ++i){\n    for (index_t j = 0; j < cpu_res.size(1); ++j){\n      printf(\"%.2f \", cpu_res[i][j]);\n    }\n    printf(\"\\n\");\n  }\n  // shutdown tensor enigne after usage\n  DeleteStream(sm1);\n  DeleteStream(sm2);\n  ShutdownTensorEngine<gpu>();\n  return 0;\n}\n"
  },
  {
    "path": "3rdparty/mshadow/guide/defop.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#include <cmath>\n// header file to use mshadow\n#include \"mshadow/tensor.h\"\n// this namespace contains all data structures, functions\nusing namespace mshadow;\n// this namespace contains all operator overloads\nusing namespace mshadow::expr;\n\n// user defined unary operator addone\nstruct addone {\n  // map can be template function\n  template<typename DType>\n  MSHADOW_XINLINE static DType Map(DType a) {\n    return  a + static_cast<DType>(1);\n  }\n};\n// user defined binary operator max of two\nstruct maxoftwo {\n  // map can also be normal functions,\n  // however, this can only be applied to float tensor\n  MSHADOW_XINLINE static float Map(float a, float b) {\n    if(a > b) return a;\n    else return b;\n  }\n};\n\nint main(void) {\n  // intialize tensor engine before using tensor operation, needed for CuBLAS\n  InitTensorEngine<cpu>();\n  // take first subscript of the tensor\n  Stream<cpu> *stream_ = NewStream<cpu>(0);\n  Tensor<cpu,2, float> mat = NewTensor<cpu>(Shape2(2,3), 0.0f, stream_);\n  Tensor<cpu,2, float> mat2= NewTensor<cpu>(Shape2(2,3), 0.0f, stream_);\n\n  mat[0][0] = -2.0f;\n  mat = F<maxoftwo>(F<addone>(mat) + 0.5f, mat2);\n\n  for (index_t i = 0; i < mat.size(0); ++i) {\n    for (index_t j = 0; j < mat.size(1); ++j) {\n      printf(\"%.2f \", mat[i][j]);\n    }\n    printf(\"\\n\");\n  }\n  FreeSpace(&mat); FreeSpace(&mat2);\n  DeleteStream(stream_);\n  // shutdown tensor enigne after usage\n  ShutdownTensorEngine<cpu>();\n  return 0;\n}\n"
  },
  {
    "path": "3rdparty/mshadow/guide/exp-template/.gitignore",
    "content": "exp_*"
  },
  {
    "path": "3rdparty/mshadow/guide/exp-template/Makefile",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# set LD_LIBRARY_PATH\nexport CC  = gcc\nexport CXX = g++\nexport CFLAGS = -Wall -O3 \n# specify tensor path\nBIN = exp_lazy exp_template exp_template_op\n\n.PHONY: clean all\n\nall: $(BIN) \n\nexp_lazy: exp_lazy.cpp\nexp_template: exp_template.cpp\nexp_template_op: exp_template_op.cpp\n\n$(BIN) :\n\t$(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c, $^)\n\nclean:\n\trm -rf $(BIN) *~\n"
  },
  {
    "path": "3rdparty/mshadow/guide/exp-template/README.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nExpression Template Tutorial\n====\nThis page explains how mshadow works. The main trick behind mshadow is called [Expression Template](http://en.wikipedia.org/wiki/Expression_templates).\nWe will explain how it will affect the performance of compiled code. Expression template is the major trick behind the C++ matrix libraries such as Eigen, GSL, boost.uBLAS.\n\nHow to write efficient machine learning code\n====\nBefore we start, let us think of the question above. Assume we want to write down the update rule (for illustration purpose here, while typical update would be `weight += - eta * (grad + lambda * weight)` )\n```c++\nweight =  - eta * (grad + lambda * weight);\n```\nWhere weight and grad are vectors of length ```n```. When you choose C++ as your programming language,\nI guess the major concern is efficiency. There is one principle that is important and used in most C/C++ programs:\n* Pre-allocate necessary memory, **no temporal memory allocation** during running.\n\nAn example code is like\n```c++\nvoid UpdateWeight (const float *grad, float eta, float lambda,\n                   int n, float *weight) {\n  for (int i = 0; i < n; ++i) {\n    weight[i] =  - eta * (grad[i] + lambda * weight[i]);\n  }\n}\n``` \nThe function takes the pre-allocated space grad, and weight, and run the calculation. Writing these functions are simple,\nhowever, it can be annoying when we write them repeatedly. So the question is, can we write as follows, and get same performance as previous code?\n```c++\nvoid UpdateWeight (const Vec& grad, float eta, float lambda, Vec& weight) {\n  weight = -eta * (grad + lambda * weight);\n} \n```\nThe answer is yes, but not by the most obvious solution.\n\nA Naive Bad Solution\n====\nLet us first take a look at a most straight forward solution: operator overloading.\n```c++\n// Naive solution for vector operation overloading \nstruct Vec {\n  int len;\n  float* dptr;\n  Vec(int len) : len(len) { \n    dptr = new float[len];\n  }\n  Vec(const Vec& src) : len(src.len) {\n    dptr = new float[len];\n    memcpy(dptr, src.dptr, sizeof(float)*len ); \n  }\n  ~Vec(void) {\n    delete [] dptr;\n  }\n};\n\ninline Vec operator+(const Vec &lhs, const Vec &rhs) {\n  Vec res(lhs.len);\n  for (int i = 0; i < lhs.len; ++i) {\n    res.dptr[i] = lhs.dptr[i] + rhs.dptr[i];\n  } \n  return res;\n} \n```\nIf we add more operators overloading in the same style, we can get what we want, and write equations instead of loop.\nHowever, this kind of approach is inefficient, because temporal memory is allocated and de-allocated during each operation, while we could have done better.\n\nAn alternative, more effective way is only overload operator+=, operator-=, which can be implemented without temporal memory allocation. But this limits the equations we can write.\n\nWe will discuss why we still need expression template although C++11 provides move assignment operator and rvalue reference at the end of this tutorial. \n\nLazy Evaluation\n====\nLet us think why we need temporal memory allocation when doing operator+. This is because we *do not know* the target that will be assigned to in operator+,\notherwise we could have directly storing into target memory instead of temporal memory. \n\nWhat if we can know the target? The following code ([exp_lazy.cpp](exp_lazy.cpp)) achieves this. \n```c++\n// Example Lazy evaluation code\n// for simplicity, we use struct and make all members public\n#include <cstdio>\nstruct Vec;\n// expression structure holds the expression\nstruct BinaryAddExp {\n  const Vec &lhs;\n  const Vec &rhs;\n  BinaryAddExp(const Vec &lhs, const Vec &rhs)\n  : lhs(lhs), rhs(rhs) {}\n};\n// no constructor and destructor to allocate and de-allocate memory,\n//  allocation done by user\nstruct Vec {\n  int len;\n  float* dptr;\n  Vec(void) {}\n  Vec(float *dptr, int len)\n      : len(len), dptr(dptr) {}\n  // here is where evaluation happens\n  inline Vec &operator=(const BinaryAddExp &src) {\n    for (int i = 0; i < len; ++i) {\n      dptr[i] = src.lhs.dptr[i] + src.rhs.dptr[i];\n    }\n    return *this;\n  }\n};\n// no evaluation happens here\ninline BinaryAddExp operator+(const Vec &lhs, const Vec &rhs) {\n  return BinaryAddExp(lhs, rhs);\n}\n\nconst int n = 3;\nint main(void) {\n  float sa[n] = {1, 2, 3};\n  float sb[n] = {2, 3, 4};\n  float sc[n] = {3, 4, 5};\n  Vec A(sa, n), B(sb, n), C(sc, n);\n  // run expression\n  A = B + C;\n  for (int i = 0; i < n; ++i) {\n    printf(\"%d:%f==%f+%f\\n\", i, A.dptr[i], B.dptr[i], C.dptr[i]);\n  }\n  return 0;\n}\n```\nThe idea is that we do not actually do computation in operator+, but only return a expression structure (like abstract syntax tree),\nand when we overload operator=, we see the target, as well as all the operands, and we can run computation without introducing extra memory!\nSimilarly, we can define a DotExp and lazily evaluate at operator=, and redirect matrix(vector) multiplications to BLAS.\n\n\nMore Lengthy Expressions and Expression Template\n====\nBy using lazy evaluation, we are cool by avoiding temporal memory allocations. But the ability of the code is limited:\n* We can only write ```A=B+C```, but not more lengthy expressions.\n* When we add more expression, we need to write more operator= to evaluate each equations.\n\nHere is where the magic of template programming comes to rescue. The following code ([exp_template.cpp](exp_template.cpp)),\nwhich is a bit more lengthy, also allows you to write lengthy equations.\n```c++\n// Example code, expression template, and more length equations\n// for simplicity, we use struct and make all members public\n#include <cstdio>\n\n// this is expression, all expressions must inheritate it,\n//  and put their type in subtype\ntemplate<typename SubType>\nstruct Exp {\n  // returns const reference of the actual type of this expression\n  inline const SubType& self(void) const {\n    return *static_cast<const SubType*>(this);\n  }\n};\n\n// binary add expression\n// note how it is inheritates from Exp\n// and put its own type into the template argument\ntemplate<typename TLhs, typename TRhs>\nstruct BinaryAddExp: public Exp<BinaryAddExp<TLhs, TRhs> > {\n  const TLhs &lhs;\n  const TRhs &rhs;\n  BinaryAddExp(const TLhs& lhs, const TRhs& rhs)\n      : lhs(lhs), rhs(rhs) {}\n  // evaluation function, evaluate this expression at position i\n  inline float Eval(int i) const {\n    return lhs.Eval(i) + rhs.Eval(i);\n  }\n};\n// no constructor and destructor to allocate\n// and de-allocate memory, allocation done by user\nstruct Vec: public Exp<Vec> {\n  int len;\n  float* dptr;\n  Vec(void) {}\n  Vec(float *dptr, int len)\n      :len(len), dptr(dptr) {}\n  // here is where evaluation happens\n  template<typename EType>\n  inline Vec& operator= (const Exp<EType>& src_) {\n    const EType &src = src_.self();\n    for (int i = 0; i < len; ++i) {\n      dptr[i] = src.Eval(i);\n    }\n    return *this;\n  }\n  // evaluation function, evaluate this expression at position i\n  inline float Eval(int i) const {\n    return dptr[i];\n  }\n};\n// template add, works for any expressions\ntemplate<typename TLhs, typename TRhs>\ninline BinaryAddExp<TLhs, TRhs>\noperator+(const Exp<TLhs> &lhs, const Exp<TRhs> &rhs) {\n  return BinaryAddExp<TLhs, TRhs>(lhs.self(), rhs.self());\n}\n\nconst int n = 3;\nint main(void) {\n  float sa[n] = {1, 2, 3};\n  float sb[n] = {2, 3, 4};\n  float sc[n] = {3, 4, 5};\n  Vec A(sa, n), B(sb, n), C(sc, n);\n  // run expression, this expression is longer:)\n  A = B + C + C;\n  for (int i = 0; i < n; ++i) {\n    printf(\"%d:%f == %f + %f + %f\\n\", i,\n           A.dptr[i], B.dptr[i],\n           C.dptr[i], C.dptr[i]);\n  }\n  return 0;\n}\n```\nThe key idea of the code is the template ```Exp<SubType>``` takes type of its derived class as template argument, so it can convert itself to\nthe SubType via ```self()```.  BinaryAddExp now is a template class that can composite expressions together, like a template version of Composite pattern.\nThe evaluation is done through function Eval, which is done in a recursive way in BinaryAddExp.\n* Due to inlining, the function calls of ```src.Eval(i)``` in ```operator=``` will be compiled into ```B.dptr[i] + C.dptr[i] + C.dptr[i]``` in compile time.\n* We can write equations for element-wise operations with same efficiency as if we write a loop  \n\nMake it more flexible\n====\nAs we can find in the previous example, template programming is a powerful to make things flexible in compile time, our final example,\nwhich is closer to mshadow, allows user customized binary operators ([exp_template_op.cpp](exp_template_op.cpp)). \n```c++\n// Example code, expression template\n// with binary operator definition and extension\n// for simplicity, we use struct and make all members public\n#include <cstdio>\n\n// this is expression, all expressions must inheritate it,\n// and put their type in subtype\ntemplate<typename SubType>\nstruct Exp{\n  // returns const reference of the actual type of this expression\n  inline const SubType& self(void) const {\n    return *static_cast<const SubType*>(this);\n  }\n};\n\n// binary operators\nstruct mul{\n  inline static float Map(float a, float b) {\n    return a * b;\n  }\n};\n\n// binary add expression\n// note how it is inheritates from Exp\n// and put its own type into the template argument\ntemplate<typename OP, typename TLhs, typename TRhs>\nstruct BinaryMapExp: public Exp<BinaryMapExp<OP, TLhs, TRhs> >{\n  const TLhs& lhs;\n  const TRhs& rhs;\n  BinaryMapExp(const TLhs& lhs, const TRhs& rhs)\n      :lhs(lhs), rhs(rhs) {}\n  // evaluation function, evaluate this expression at position i\n  inline float Eval(int i) const {\n    return OP::Map(lhs.Eval(i), rhs.Eval(i));\n  }\n};\n// no constructor and destructor to allocate and de-allocate memory\n// allocation done by user\nstruct Vec: public Exp<Vec>{\n  int len;\n  float* dptr;\n  Vec(void) {}\n  Vec(float *dptr, int len)\n      : len(len), dptr(dptr) {}\n  // here is where evaluation happens\n  template<typename EType>\n  inline Vec& operator=(const Exp<EType>& src_) {\n    const EType &src = src_.self();\n    for (int i = 0; i < len; ++i) {\n      dptr[i] = src.Eval(i);\n    }\n    return *this;\n  }\n  // evaluation function, evaluate this expression at position i\n  inline float Eval(int i) const {\n    return dptr[i];\n  }\n};\n// template binary operation, works for any expressions\ntemplate<typename OP, typename TLhs, typename TRhs>\ninline BinaryMapExp<OP, TLhs, TRhs>\nF(const Exp<TLhs>& lhs, const Exp<TRhs>& rhs) {\n  return BinaryMapExp<OP, TLhs, TRhs>(lhs.self(), rhs.self());\n}\n\ntemplate<typename TLhs, typename TRhs>\ninline BinaryMapExp<mul, TLhs, TRhs>\noperator*(const Exp<TLhs>& lhs, const Exp<TRhs>& rhs) {\n  return F<mul>(lhs, rhs);\n}\n\n// user defined operation\nstruct maximum{\n  inline static float Map(float a, float b) {\n    return a > b ? a : b;\n  }\n};\n\nconst int n = 3;\nint main(void) {\n  float sa[n] = {1, 2, 3};\n  float sb[n] = {2, 3, 4};\n  float sc[n] = {3, 4, 5};\n  Vec A(sa, n), B(sb, n), C(sc, n);\n  // run expression, this expression is longer:)\n  A = B * F<maximum>(C, B);\n  for (int i = 0; i < n; ++i) {\n    printf(\"%d:%f == %f * max(%f, %f)\\n\",\n           i, A.dptr[i], B.dptr[i], C.dptr[i], B.dptr[i]);\n  }\n  return 0;\n}\n```\n\nSummary\n=====\nUp to this point, you should have understand basic ideas how it works:\n* Lazy evaluation, to allow us see all the operands and target\n* Template composition and recursive evaluation, to allows us evaluate arbitrary composite expressions for element-wise operations.\n* Due to template and inlining, writing expressions are as efficient as if we directly write a for loop to implement the update rule:)\n\nSo write expressions when you write machine learning codes, and focus your energy on the algorithm part that matters.\n\nThe Expression Template in MShadow\n=====\nExpression template in mshadow use the same key points as we introduced in the tutorial, with some minor differences:\n* We separate evaluation code from expression construction and composition code.  \n    - Instead of putting Eval in Exp class. A Plan class is created from expression, and used to evaluate the result. \n    - This allows us to put less variables in Plan, for example, we do not need array length when we evaluate a data.\n    - One important reason is CUDA kernel cannot take class with const references \n    - This design choice is debatable, but we find it is useful so far.\n* Lazy support for complex expressions such as matrix dot product\n    - Besides element-wise expressions, we also want to support sugars such as ```A = dot(B.T(), C)```,  again, lazy evaluation is used and no extra memory is allocated.\n* Type checking and array length checking.\n\nNotes\n====\n* Expression Template and C++11: in C++11, move constructor can be used to save repetitive allocation memory, which removes some need to expression template. However, the space still needs to be allocated at least once. \n   - This only removes the need of expression template then expression generate space, say dst = A+B+C, dst does not contain space allocated before assignment.\n   - If we want to keep the syntax that everything is pre-allocated, and expression executes without memory allocation (which is what we did in mshadow), we still need expression template.\n\n"
  },
  {
    "path": "3rdparty/mshadow/guide/mshadow-ps/.gitignore",
    "content": "log\n*cpu\n*gpu\ncore*\n"
  },
  {
    "path": "3rdparty/mshadow/guide/mshadow-ps/Makefile",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# set LD_LIBRARY_PATH\nexport CC  = gcc\nexport CXX = g++\nexport NVCC =nvcc\ninclude config.mk\ninclude ../../make/mshadow.mk\nexport CFLAGS = -Wall -O3 -std=c++17 -fopenmp -I../../ $(MSHADOW_CFLAGS)\nexport LDFLAGS= -lm $(MSHADOW_LDFLAGS)\nexport NVCCFLAGS = -O3 --use_fast_math -ccbin $(CXX) $(MSHADOW_NVCCFLAGS)\n\n# specify tensor path local_sum.cpu\nBIN = local_sum.cpu\nOBJ =\nCUOBJ =\nCUBIN = local_sum.gpu\n\nifeq ($(USE_DIST_PS),1)\nBIN = dist_async_sum.cpu\nLDFLAGS += -lunwind\nendif\n\n.PHONY: clean all\n\nall: $(BIN) #$(CUBIN)\n\nlocal_sum.cpu: local_sum.cpp\nlocal_sum.gpu: local_sum.cu\n\ndist_async_sum.cpu: dist_async_sum.cpp dist_async_sum-inl.h\ndist_sync_sum.cpu: dist_sync_sum.cpp\n\n$(BIN) :\n\t$(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c, $^)  $(LDFLAGS) $(PS_LIB)\n\n$(OBJ) :\n\t$(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c, $^) )\n\n$(CUOBJ) :\n\t$(NVCC) -c -o $@ $(NVCCFLAGS) -Xcompiler \"$(CFLAGS)\" $(filter %.cu, $^)\n\n$(CUBIN) :\n\t$(NVCC) -o $@ $(NVCCFLAGS) -Xcompiler \"$(CFLAGS)\" -Xlinker \"$(LDFLAGS)\" $(filter %.cu %.cpp %.o, $^)\n\nclean:\n\t$(RM) $(OBJ) $(BIN) $(CUBIN) $(CUOBJ) *~\n"
  },
  {
    "path": "3rdparty/mshadow/guide/mshadow-ps/README.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nmshadow-ps\n====\n### Parameter Server Interface for GPU Tensor\n\nmshadow-ps provides asynchronize parameter server interface for mshadow GPU/CPU Tensor.\nThis allows you to do ***multi-GPU*** and ***disrtibuted*** (deep) learning in\nan ***easy*** and ***unified*** way.\n\nmshadow-ps implemented a two-level parameter server. The architecture is shown\nin the following figure. Typically, a GPU card or a cpu core runs a worker node,\nthen a level-1 server node communicates with the worker nodes on the same\nmachine.  Inter-machine communication is then via the level-2 server nodes.\n\nThe rational is that both the bandwidth and latency between the worker nodes in\na single machine is usually 10x better than the inter-machine ones. By using the\ntwo level parameter server, we can use different consistency models at different\nlevel to better trade-off the algorithm efficiency and system performance. For\nexample, we can use a sequential consistency model, also known as\n[BSP](http://en.wikipedia.org/wiki/Bulk_synchronous_parallel), on level 1 for\nguaranteed algorithm convergence, but use a\n[eventual consistency model](http://en.wikipedia.org/wiki/Eventual_consistency)\non level 2 to hide the network latency. See our\n[OSDI'14 paper](https://www.usenix.org/conference/osdi14/technical-sessions/presentation/li_mu)\nfor more details.\n\n![Arch](2-levels.png?raw=true \"arch\")\n\n####List of Resources\n* [API Documentation](http://homes.cs.washington.edu/~tqchen/mshadow/doc/namespacemshadow_1_1mshadow_ps.html)\n* [Library Interface Header](../../mshadow-ps/mshadow_ps.h)\n* Tutorial in this page\n\nWorking with Level-1 Server\n====\nSuppose that we are now implementing a Multi-GPU learning program.\nOne way to do that is through data parallelism. We can launch many\nthreads, with each thread compute gradient on one GPU, and aggregate\nthe statistics together.\nHowever, the gradient synchronization step could be cost time, and in\nmany cases, we can do the computation in an smarter way, so that\nwe ***overlaps the computation with the synchronization***.\n\nmshadow-ps provides interface to do such synchronization in an easy way.\nThe following documents provides a way\n\n### Getting Sum from Multiple GPUs\nWe first get familiar with the interface of mshadow-mshadow_ps. Through the following\nprogram in [local_sum-inl.h](local_sum-inl.h). You can compile the program\nby setup the [config.mk](config.mk) according to your computers's enviroment, and type make.\n\nIn the following program, each thread first does some computation locally, then tries to get the sum\nof ```data``` through mshadow-ps interface.\nThere are four key functions in ```ISharedModel``` interface\n* [InitKey](../../mshadow-ps/mshadow_ps.h#L76) allocates a key to specific tensor shape\n* [Push](../../mshadow-ps/mshadow_ps.h#L100) pushes out the local data to the synchronization interface\n  - The data pushed by different devices will be aggregated together by key\n  - Push is an asynchronize call and returns immediately\n* [PullReq](../../mshadow-ps/mshadow_ps.h#L122) requests the result of synchronization to be copied back\n  - In the local default case, the synchronized result is the sum of pushed data\n  - mshadow-ps also support the weight update on server side, where the result of PullReq is the updated weight instead of sum of gradient\n  - PullReq is also asynchronize\n* [PullWait](../../mshadow-ps/mshadow_ps.h#L87) wait until the pull request of corresponding key finishes\n\n```c++\n// this function is runed by specific thread\ntemplate<typename xpu>\ninline void RunWorkerThread(int devid,\n                            mshadow::ps::ISharedModel<xpu, float> *ps) {\n  // initialize tensor engine\n  mshadow::InitTensorEngine<xpu>(devid);\n  mshadow::Stream<xpu> *stream  = mshadow::NewStream<xpu>();\n  // allocate tensor on xpu\n  mshadow::TensorContainer<xpu, 2> data(mshadow::Shape2(2, 3));\n  // set the computation stream to the new allocated stream\n  // this will make subsequent computation whose target is data\n  // to use the stream, stream is needed for async execution in GPU\n  data.set_stream(stream);\n  // assume these operations sets the content of dataient\n  data[0] = 1.0f;\n  data[1] = devid + data[0];\n  printf(\"dev%d: before sync, data:\\n\", devid);\n  // use print to show result, do not call\n  // print normally since Copy will block\n  Print(data);\n  printf(\"====================\\n\");\n  // intiaialize the key, register the shape on parameter server\n  ps->InitKey(data[0].shape_, 0, devid);\n  ps->InitKey(data[1].shape_, 1, devid);\n  // push data[0] out, for update, or aggregation\n  // 0 is the key of the data, devid is the current device id\n  ps->Push(data[0], 0, devid);\n  // pull request is used to request the data to be copied back\n  // once computation is done\n  ps->PullReq(data[0], 0, devid);\n  // computation can be done here..\n  // the pull request handler will be overlapped with\n  // similar as previous call\n  ps->Push(data[1], 1, devid);\n  ps->PullReq(data[1], 1, devid);\n  // more computation can be done here...\n  // the computation will be overlapped\n  // PullWait will block until these request finishes\n  ps->PullWait(0, devid);\n  ps->PullWait(1, devid);\n  printf(\"dev%d: after sync, data:\\n\", devid);\n  // use print to show result, do not call\n  // print normally since Copy will block\n  Print(data);\n  printf(\"====================\\n\");\n  mshadow::DeleteStream(stream);\n  mshadow::ShutdownTensorEngine<xpu>();\n}\n\ntemplate<typename xpu>\ninline int Run(int argc, char *argv[]) {\n  if (argc < 2) {\n    printf(\"Usage: device list\\n\"\\\n           \"\\tfor CPU the device list can be arbitrary\\n\"\\\n           \"\\tfor GPU the device list need to be actual device index\\n\");\n    return 0;\n  }\n  // list of device ids\n  std::vector<int> devs;\n  // initialization\n  for (int i = 1; i < argc; ++i) {\n    // record the device id\n    devs.push_back(atoi(argv[i]));\n  }\n  mshadow::ps::ISharedModel<xpu, float>\n      *ps = mshadow::ps::CreateSharedModel<xpu, float>(\"local\");\n  // intiaialize the ps\n  ps->Init(devs);\n  // use openmp to launch #devs threads\n  #pragma omp parallel num_threads(devs.size())\n  {\n    int tid = omp_get_thread_num();\n    RunWorkerThread<xpu>(devs[tid], ps);\n  }\n  delete ps;\n  return 0;\n}\n```\nIn the above example, we did not do weight update on server side, so the synchronization result is\nsimply the sum of data on each device. The key property of this interface is that the Push and PullReq are asynchronize.\n* We can call these two functions once the gradient is ready, and the mshadow-ps will do the data synchronization in the background.\n* When we need the result of synchronization, we simply call PullWait to wait the synchronization task to finish.\n* Such interface allows us to do additional computation between the Push/PullReq and PullWait\n\n### A MultiGPU Neural Net\nTo get a more concrete understanding of the interface. We give an example of multi-GPU two layer neuralnet\nin [../neuralnet/nnet_ps.cu](../neuralnet/nnet_ps.cu). The general idea is follows\n* Push and PullReq is called once we get the gradient of certain layer\n* PullWait is called before we do forward on that layer next time\n* This creates a ***time lag*** between the backprop and next forward to that layer\n  - mshadow-ps do synchronization concurrently with computations during the time lag\n  - The time lag is big for latter layers, which also usually need more time to synchronize\n\nThere are several note of the mshadow-ps on the neural net code\n* Callback function in PullReq\n  - A callback function can be pass to PullReq to be called when the request complete\n  - We place weight update in the callback to perform update when we get the gradient sum\n* Computing stream\n  - Due to GPU's programming model, we need to do computation on non-default stream\n  - Use set_stream in mshadow tensors to set stream to computation stream\n  - To report error when you did not use stream, you can compile with -DMSHADOW_FORCE_STREAM\n\nWe should note thate because the example runs on MNIST, which is an quite small dataset, you may not observe\nspeedup with multiple cards. However, you will find significant speedup when you run on other tasks.\nThe newest version of [cxxnet](https://github.com/antinucleon/cxxnet)\n\n### Moving Parameter Update to the Server\nIn all the examples so far, we use mshadow-ps to get the aggregated sum of gradients, and update\nweights locally on each GPU. For more advanced usage of mshadow-ps, we can move the weight update\nto the server. The communication pattern is as follows\n* Each thread still call Push to push out gradient\n* The server will apply the update rule to update the weight\n* Each thread call PullReq to pull back the weight from server\n\nSuch update pattern is suitable under distributed setting. To do so, user need to implement an\n[IModelUpdater](../../mshadow-ps/mshadow_ps.h#L202) interface. And define the following CreateModelUpdater function\nin the program\n```c++\nnamespace mshadow {\nnamespace ps {\ntemplate<>\nIModelUpdater<float> *CreateModelUpdater() {\n  return new MyModelUpdater();\n}\n}\n}\n```\nBefore calling ISharedModel.Init, user need to call ```ps->SetParam(\"update_on_server\", \"1\")``` to set the update\nmode on the server side. If user uses distributed shared model, user must define ModelUpdater.\n\nWorking with Level-2 Server\n====\n\nFirst build the parameter server (replace `ps_dir` to any convenient directory)\n\n```bash\ngit clone https://github.com/dmlc/parameter_server -b dev ps_dir\ncd ps_dir\n./script/install_third.sh\nmake -j8\n```\n\nNext change `config.mk` to\n```bash\nUSE_DIST_PS = 1\nPS_PATH = ps_dir\n```\n\nThen `make`.\n\nNext start 1 server node, 3 worker nodes with 2 devices in each worker node:\n```bash\n./local.sh 1 3 ./dist_async_sum.cpu 1 2\n```\n\nThe `dist_async_sum-inl.h` is similar to `local_sum-inl.h`. The main differences\nare 1) we create the server at a remote node, and set\n`update_on_server` to be true.\n```c++\nauto* ps = mshadow::ps::CreateSharedModel<xpu, float>(\"dist\");\nps->SetParam(\"update_on_server\", \"1\");\n```\n2) we explicitly create server node and worker node at `dist_async_sum.cpp`\n"
  },
  {
    "path": "3rdparty/mshadow/guide/mshadow-ps/dbstr.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#pragma once\n#include <mshadow/tensor.h>\n#include <sstream>\n\ntemplate<typename DType>\nstd::string dbstr(mshadow::Tensor<mshadow::cpu, 1, DType> ts) {\n  std::stringstream ss;\n  for (mshadow::index_t i = 0; i < ts.size(0); ++i)\n    ss << ts[i] << \" \";\n  ss << \"\\n\";\n  return ss.str();\n}\n\ntemplate<typename DType>\nstd::string dbstr(mshadow::Tensor<mshadow::cpu, 2, DType> ts) {\n  std::stringstream ss;\n  for (mshadow::index_t i = 0; i < ts.size(0); ++i) {\n    for (mshadow::index_t j = 0; j < ts.size(1); ++j) {\n      ss << ts[i][j] << \" \";\n    }\n    ss << \"\\n\";\n  }\n  ss << \"\\n\";\n  return ss.str();\n}\n\ntemplate<typename DType>\nstd::string dbstr(mshadow::Tensor<mshadow::cpu, 3, DType> ts) {\n  std::stringstream ss;\n  for (mshadow::index_t i = 0; i < ts.size(0); ++i) {\n    ss << dbstr(ts[i]) << \"\\n\";\n  }\n  ss << \"\\n\";\n  return ss.str();\n}\n"
  },
  {
    "path": "3rdparty/mshadow/guide/mshadow-ps/dist_async_sum-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/**\n * @brief  Simple test of KVLayer\n */\n#include \"ps.h\"\n#include \"parameter/kv_layer.h\"\n#include <cstdio>\n#include <iostream>\n#include <omp.h>\n#include <map>\n#include <mshadow/tensor.h>\n#include <mshadow-ps/mshadow_ps.h>\n#include \"dbstr.h\"\n#include \"glog/logging.h\"\n\nnamespace mshadow {\nnamespace ps {\n\n\ntemplate<typename DType>\nclass Updater : public IModelUpdater<DType> {\n protected:\n  void InitModel_(int key, Tensor<cpu, 1, DType> data) {\n    data = 0;\n    data_[key] = data;\n  }\n\n  void Update_(int key, Tensor<cpu, 1, DType> data) {\n    data_[key] += data;\n    // LOG(ERROR) << dbstr(data_[key]);\n  }\n  std::map<int, Tensor<cpu, 1, DType> > data_;\n};\n\ntemplate<typename DType>\nIModelUpdater<DType> *CreateModelUpdater(void) {\n  return new Updater<DType>();\n}\n\n}  // namespace ps\n}  // namespace mshadow\n\n// this function is runed by specific thread\ntemplate<typename xpu>\ninline void RunWorkerThread(int devid,\n                            mshadow::ps::ISharedModel<xpu, float> *ps) {\n  // initialize tensor engine\n  mshadow::InitTensorEngine<xpu>(devid);\n  mshadow::Stream<xpu> *stream  = mshadow::NewStream<xpu>();\n  // allocate tensor on xpu\n  mshadow::TensorContainer<xpu, 2> data(mshadow::Shape2(2, 3));\n  // set the computation stream to the new allocated stream\n  // this will make subsequent computation whose target is data\n  // to use the stream, stream is needed for async execution in GPU\n  data.set_stream(stream);\n  // intiaialize the key, register the shape on parameter server\n  ps->InitKey(data[0].shape_, 0, devid);\n  ps->InitKey(data[1].shape_, 1, devid);\n  // first step, pull the data back from server\n  ps->PullReq(data[0], 0, devid);\n  ps->PullReq(data[1], 1, devid);\n\n  // PullWait will block until these request finishes\n  ps->PullWait(0, devid);\n  ps->PullWait(1, devid);\n\n  data[1] = devid + data[0];\n\n  LOG(ERROR) << \"node \" << ::ps::MyNodeID() << \", dev \" << devid << \": before sync\\n\"\n             << dbstr(data);\n\n  // push data[0] out, for update, or aggregation\n  // 0 is the key of the data, devid is the current device id\n  ps->Push(data[0], 0, devid);\n  // pull request is used to request the data to be copied back\n  // once computation is done\n  ps->PullReq(data[0], 0, devid);\n  // computation can be done here..\n  // the pull request handler will be overlapped with\n  // similar as previous call\n  ps->PullWait(0, devid);\n\n  ps->Push(data[1], 1, devid);\n  ps->PullReq(data[1], 1, devid);\n  // more computation can be done here...\n  // the computation will be overlapped\n  // PullWait will block until these request finishes\n  ps->PullWait(1, devid);\n\n  LOG(ERROR) << \"node \" << ::ps::MyNodeID() << \", dev \" << devid\n             << \": after sync\\n\" << dbstr(data);\n\n  mshadow::DeleteStream(stream);\n  mshadow::ShutdownTensorEngine<xpu>();\n}\n\ntemplate<typename xpu>\ninline int Run(int argc, char *argv[]) {\n  if (argc < 2) {\n    printf(\"Usage: device list\\n\"\\\n           \"\\tfor CPU the device list can be arbitrary\\n\"\\\n           \"\\tfor GPU the device list need to be actual device index\\n\");\n    return 0;\n  }\n  // list of device ids\n  std::vector<int> devs;\n  // initialization\n  for (int i = 1; i < argc; ++i) {\n    // record the device id\n    devs.push_back(atoi(argv[i]));\n  }\n  mshadow::ps::ISharedModel<xpu, float>\n      *ps = mshadow::ps::CreateSharedModel<xpu, float>(\"dist\");\n  // intiaialize the ps\n  ps->SetParam(\"update_on_server\", \"1\");\n  ps->Init(devs);\n  // use openmp to launch #devs threads\n  #pragma omp parallel num_threads(devs.size())\n  {\n    int tid = omp_get_thread_num();\n    RunWorkerThread<xpu>(devs[tid], ps);\n  }\n  delete ps;\n  return 0;\n}\n"
  },
  {
    "path": "3rdparty/mshadow/guide/mshadow-ps/dist_async_sum.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#include \"./dist_async_sum-inl.h\"\n\nint CreateServerNode(int argc, char *argv[]) {\n  mshadow::ps::MShadowServerNode<float> server(argc, argv);\n  return 0;\n}\n\n\nint WorkerNodeMain(int argc, char *argv[]) {\n  return Run<mshadow::cpu>(argc, argv);\n}\n"
  },
  {
    "path": "3rdparty/mshadow/guide/mshadow-ps/local.sh",
    "content": "#!/bin/bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# set -x\n# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../third_party/lib\nif [ $# -lt 3 ]; then\n    echo \"usage: $0 num_servers num_workers bin [args..]\"\n    exit -1;\nfi\n\nnum_servers=$1\nshift\nnum_workers=$1\nshift\nbin=$1\nshift\narg=\"-num_servers ${num_servers} -num_workers ${num_workers} -log_dir log $@\"\n\n\n# killall -q $(basename ${bin})\n# killall -q ${bin}\n\n# start the scheduler\nSch=\"role:SCHEDULER,hostname:'127.0.0.1',port:8001,id:'H'\"\n${bin} -my_node ${Sch} -scheduler ${Sch} ${arg} &\n\n# start servers\nfor ((i=0; i<${num_servers}; ++i)); do\n    port=$((9600 + ${i}))\n    N=\"role:SERVER,hostname:'127.0.0.1',port:${port},id:'S${i}'\"\n    ${bin} -my_node ${N} -scheduler ${Sch} ${arg} &\ndone\n\n# start workers\nfor ((i=0; i<${num_workers}; ++i)); do\n    port=$((9500 + ${i}))\n    N=\"role:WORKER,hostname:'127.0.0.1',port:${port},id:'W${i}'\"\n    ${bin} -my_node ${N} -scheduler ${Sch} ${arg} &\ndone\n\nwait\n"
  },
  {
    "path": "3rdparty/mshadow/guide/mshadow-ps/local_sum-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n// This is an example demonstrating the usage of mshadow ps\n#include <cstdio>\n// use openmp to launch multiple threads\n#include <omp.h>\n#include <mshadow/tensor.h>\n#include <mshadow-ps/mshadow_ps.h>\n\n// simple util to print result\nvoid Print_(mshadow::Tensor<mshadow::cpu, 2, float> ts) {\n  for (mshadow::index_t i = 0; i < ts.size(0); ++i) {\n    for (mshadow::index_t j = 0; j < ts.size(1); ++j) {\n      printf(\"%g \", ts[i][j]);\n    }\n    printf(\"\\n\");\n  }\n}\ntemplate<typename xpu>\ninline void Print(mshadow::Tensor<xpu, 2, float> ts) {\n  mshadow::TensorContainer<mshadow::cpu, 2, float> tmp;\n  tmp.Resize(ts.shape_);\n  mshadow::Copy(tmp, ts);\n  Print_(tmp);\n}\n\n// this function is runed by specific thread\ntemplate<typename xpu>\ninline void RunWorkerThread(int devid,\n                            mshadow::ps::ISharedModel<xpu, float> *ps) {\n  // initialize tensor engine\n  mshadow::InitTensorEngine<xpu>(devid);\n  mshadow::Stream<xpu> *stream  = mshadow::NewStream<xpu>(devid);\n  // allocate tensor on xpu\n  mshadow::TensorContainer<xpu, 2> data(mshadow::Shape2(2, 3));\n  // set the computation stream to the new allocated stream\n  // this will make subsequent computation whose target is data\n  // to use the stream, stream is needed for async execution in GPU\n  data.set_stream(stream);\n  // assume these operations sets the content of dataient\n  data[0] = 1.0f;\n  data[1] = devid + data[0];\n  printf(\"dev%d: before sync, data:\\n\", devid);\n  // use print to show result, do not call\n  // print normally since Copy will block\n  Print(data);\n  printf(\"====================\\n\");\n  // intiaialize the key, register the shape on parameter server\n  ps->InitKey(data[0].shape_, 0, devid);\n  ps->InitKey(data[1].shape_, 1, devid);\n  // push data[0] out, for update, or aggregation\n  // 0 is the key of the data, devid is the current device id\n  ps->Push(data[0], 0, devid);\n  // pull request is used to request the data to be copied back\n  // once computation is done\n  ps->PullReq(data[0], 0, devid);\n  // computation can be done here..\n  // the pull request handler will be overlapped with\n  // similar as previous call\n  ps->Push(data[1], 1, devid);\n  ps->PullReq(data[1], 1, devid);\n  // more computation can be done here...\n  // the computation will be overlapped\n  // PullWait will block until these request finishes\n  ps->PullWait(0, devid);\n  ps->PullWait(1, devid);\n  printf(\"dev%d: after sync, data:\\n\", devid);\n  // use print to show result, do not call\n  // print normally since Copy will block\n  Print(data);\n  printf(\"====================\\n\");\n  mshadow::DeleteStream(stream);\n  mshadow::ShutdownTensorEngine<xpu>();\n}\n\nnamespace mshadow {\nnamespace ps {\n// model updater is used when update is happening on server side\n// if we only use parameter server for sum aggregation\n// this is not needed, but we must declare this function to return NULL\ntemplate<>\nIModelUpdater<float> *CreateModelUpdater(void) {\n  return NULL;\n}\n}\n}\n\ntemplate<typename xpu>\ninline int Run(int argc, char *argv[]) {\n  if (argc < 2) {\n    printf(\"Usage: device list\\n\"\\\n           \"\\tfor CPU the device list can be arbitrary\\n\"\\\n           \"\\tfor GPU the device list need to be actual device index\\n\");\n    return 0;\n  }\n#if MSHADOW_RABIT_PS\n  rabit::Init(argc, argv);\n#endif\n  // list of device ids\n  std::vector<int> devs;\n  // initialization\n  for (int i = 1; i < argc; ++i) {\n    // record the device id\n    devs.push_back(atoi(argv[i]));\n  }\n  mshadow::ps::ISharedModel<xpu, float>\n      *ps = mshadow::ps::CreateSharedModel<xpu, float>(\"local\");\n  // intiaialize the ps\n  ps->Init(devs);\n  // use openmp to launch #devs threads\n  #pragma omp parallel num_threads(devs.size())\n  {\n    int tid = omp_get_thread_num();\n    RunWorkerThread<xpu>(devs[tid], ps);\n  }\n  delete ps;\n#if MSHADOW_RABIT_PS\n  rabit::Finalize();\n#endif\n  return 0;\n}\n"
  },
  {
    "path": "3rdparty/mshadow/guide/mshadow-ps/local_sum.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#include \"./local_sum-inl.h\"\nint main(int argc, char *argv[]) {\n  return Run<mshadow::cpu>(argc, argv);\n}\n"
  },
  {
    "path": "3rdparty/mshadow/guide/mshadow-ps/local_sum.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#include \"./local_sum-inl.h\"\nint main(int argc, char *argv[]) {\n  return Run<mshadow::gpu>(argc, argv);\n}\n"
  },
  {
    "path": "3rdparty/mshadow/guide/neuralnet/Makefile",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# set LD_LIBRARY_PATH\nexport CC  = gcc\nexport CXX = g++\nexport NVCC =nvcc\ninclude config.mk\ninclude ../../make/mshadow.mk\nexport CFLAGS = -Wall -O3 -I../../ -fopenmp $(MSHADOW_CFLAGS)\nexport LDFLAGS= -lm $(MSHADOW_LDFLAGS)\nexport NVCCFLAGS = -O3 --use_fast_math -ccbin $(CXX) $(MSHADOW_NVCCFLAGS)\n\n# specify tensor path\nBIN =\nOBJ =\nCUOBJ =\nCUBIN = nnet convnet nnet_ps\n.PHONY: clean all\n\nall: $(BIN) $(OBJ) $(CUBIN) $(CUOBJ)\n\nnnet: nnet.cu\nnnet_ps: nnet_ps.cu\nconvnet: convnet.cu\n\n$(BIN) :\n\t$(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c, $^)  $(LDFLAGS)\n\n$(OBJ) :\n\t$(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c, $^) )\n\n$(CUOBJ) :\n\t$(NVCC) -c -o $@ $(NVCCFLAGS) -Xcompiler \"$(CFLAGS)\" $(filter %.cu, $^)\n\n$(CUBIN) :\n\t$(NVCC) -o $@ $(NVCCFLAGS) -Xcompiler \"$(CFLAGS)\" -Xlinker \"$(LDFLAGS)\" $(filter %.cu %.cpp %.o, $^)\n\nclean:\n\t$(RM) $(OBJ) $(BIN) $(CUBIN) $(CUOBJ) *~\n\n"
  },
  {
    "path": "3rdparty/mshadow/guide/neuralnet/README.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nExample Neural Net code with MShadow\n====\n\nTo compile the code, modify ```config.mk``` to the setting you like and type make\n* You will need to have CUDA and  a version of BLAS\n\nTo run the demo, download  MNIST dataset from: http://yann.lecun.com/exdb/mnist/\nunzip all the files into current folder\n\nand run by  ./nnet cpu or ./nnet gpu. ./convnet cpu or ./convnet gpu\n\nMultiGPU Version\n====\n* If you have two GPUs, you can run it by ```./nnet_ps gpu 0 1```.\n* You can run it using CPUs ```./nnet_ps cpu 0 1```.\n* This is an demonstration of mshadow-ps interface, see introduction in [../mshadow-ps](../mshadow-ps)\n"
  },
  {
    "path": "3rdparty/mshadow/guide/neuralnet/convnet.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n// this implements a simple convolution neural net: conv-maxpool-fullc\n#include <vector>\n// header file to use mshadow\n#include \"mshadow/tensor.h\"\n// helper function to load mnist dataset\n#include \"util.h\"\n// this namespace contains all data structures, functions\nusing namespace mshadow;\n// this namespace contains all operator overloads\nusing namespace mshadow::expr;\n\n// define operations\nstruct relu{\n  MSHADOW_XINLINE static real_t Map(real_t a) {\n    using namespace std;\n    return max(a, 0.0f);\n  }\n};\nstruct relu_grad {\n  MSHADOW_XINLINE static real_t Map(real_t a) {\n    return a > 0.0f ? 1.0f : 0.0f;\n  }\n};\n\n/*! \\brief interface for nnet, interfacd allows use to use GPU/CPU implementation in a unified way */\nclass INNet{\n public:\n  virtual void Forward(const Tensor<cpu, 4, real_t>& inbatch, Tensor<cpu, 2, real_t> &oubatch) = 0;\n  virtual void Backprop(const Tensor<cpu, 2, real_t>& gradout) = 0;\n  virtual void Update(void) = 0;\n  virtual ~INNet() {}\n};\n\n/*!\n * \\brief simple two layer conv-net conv-pool-flat-fullc\n *        this implementation is device invariant\n */\ntemplate<typename xpu>\nclass ConvNet : public INNet {\n public:\n  // initialize the network\n  ConvNet(int batch_size, int insize, int nchannel, int ksize, int kstride, int psize, int num_out)\n      :rnd(0), ksize(ksize), kstride(kstride), psize(psize) {\n    // setup stream\n    Stream<xpu> *stream = NewStream<xpu>();\n    ninput.set_stream(stream);\n    nhidden.set_stream(stream);\n    nhiddenbak.set_stream(stream);\n    npool.set_stream(stream);\n    npoolbak.set_stream(stream);\n    nflat.set_stream(stream);\n    nout.set_stream(stream);\n    hbias.set_stream(stream); g_hbias.set_stream(stream);\n    obias.set_stream(stream);  g_obias.set_stream(stream);\n    Ki2h.set_stream(stream);  g_Ki2h.set_stream(stream);\n    Wh2o.set_stream(stream);   g_Wh2o.set_stream(stream);\n    tmp_col.set_stream(stream);\n    tmp_dst.set_stream(stream);\n    // setup nodes\n    ninput.Resize(Shape4(batch_size, 1, insize, insize));\n    nhidden.Resize(Shape4(batch_size, nchannel, (insize - ksize)/kstride+1, (insize -ksize)/kstride+1));\n    nhiddenbak.Resize(nhidden.shape_);\n    npool.Resize(Shape4(batch_size, nchannel, (nhidden.size(2)+1-psize)/psize, (nhidden.size(3)+1-psize)/psize));\n    npoolbak.Resize(npool.shape_);\n    nflat.Resize(Shape2(batch_size, npool.size(1)*npool.size(2)*npool.size(3)));\n    nout.Resize(Shape2(batch_size, num_out));\n    // setup bias\n    hbias.Resize(Shape1(nchannel)); g_hbias.Resize(hbias.shape_);\n    obias.Resize(Shape1(num_out));  g_obias.Resize(obias.shape_);\n    hbias = 0.0f; obias = 0.0f;\n    // setup weights\n    Ki2h.Resize(Shape2(nchannel, ksize*ksize));  g_Ki2h.Resize(Ki2h.shape_);\n    Wh2o.Resize(Shape2(nflat.size(1), num_out));   g_Wh2o.Resize(Wh2o.shape_);\n    rnd.SampleGaussian(&Ki2h, 0, 0.01f);\n    rnd.SampleGaussian(&Wh2o, 0, 0.01f);\n\n    printf(\"conv=%d, pool=%d\\n\", nhidden.size(3), npool.size(3));\n  }\n  virtual ~ConvNet() {}\n  // forward propagation\n  virtual void Forward(const Tensor<cpu, 4, real_t>& inbatch, Tensor<cpu, 2, real_t> &oubatch) {\n    index_t batch_size = inbatch.size(0);\n    // copy data to input layer\n    Copy(ninput, inbatch, ninput.stream_);\n    // first layer, conv, use stride=2\n    ConvForward(ninput, Ki2h, nhidden, ksize, kstride, tmp_col, tmp_dst);\n    // add bias\n    nhidden += broadcast<1>(hbias, nhidden.shape_);\n    // activation, relu, backup activation in nhidden\n    nhidden = F<relu>(nhidden);\n    Copy(nhiddenbak, nhidden, nhiddenbak.stream_);\n    // max pooling\n    npool = pool<red::maximum>(nhiddenbak, npool[0][0].shape_, psize, psize, psize);\n    Copy(npoolbak, npool, npoolbak.stream_);\n    // flat\n    nflat = reshape(npool, nflat.shape_);\n    // second layer fullc\n    nout = dot(nflat, Wh2o);\n    nout += repmat(obias, batch_size);\n    // softmax calculation\n    Softmax(nout, nout);\n    // copy result out\n    Copy(oubatch, nout, nout.stream_);\n  }\n  // back propagation\n  virtual void Backprop(const Tensor<cpu, 2, real_t>& gradout) {\n    // copy gradient to output layer\n    Copy(nout, gradout, nout.stream_);\n    // calc grad of final layer\n    g_obias = sum_rows(nout);\n    g_Wh2o  = dot(nflat.T(), nout);\n    // backprop to previous layer\n    nflat = dot(nout, Wh2o.T());\n    npool = reshape(nflat, npool.shape_);\n    // backprop pooling layer\n    nhiddenbak = unpool<red::maximum>(nhiddenbak, npoolbak, npool, psize, psize, psize);\n    // calculate gradient of relu layer\n    nhidden = F<relu_grad>(nhidden) * nhiddenbak;\n    // calc grad of layer 1\n    g_hbias = sumall_except_dim<1>(nhidden);\n    ConvBackWard(nhidden, Ki2h, g_Ki2h, ninput, ksize, kstride, tmp_col, tmp_dst);\n  }\n  // update weight\n  virtual void Update(void) {\n    // run SGD\n    const float eta = 0.1;\n    const float wd = 0.00001;\n    // update weight\n    Ki2h -= eta * (wd * Ki2h + g_Ki2h);\n    Wh2o -= eta * (wd * Wh2o + g_Wh2o);\n    // no regularization for bias\n    hbias-= eta * g_hbias;\n    obias-= eta * g_obias;\n  }\n private:\n  // forward convolution, tmp_col and tmp_dst are helper structure\n  inline static void ConvForward(const Tensor<xpu, 4, real_t> &in,\n                                 const Tensor<xpu, 2, real_t> &kernel,\n                                 Tensor<xpu, 4, real_t> &out,\n                                 int ksize, int kstride,\n                                 TensorContainer<xpu, 2, real_t> &tmp_col,\n                                 TensorContainer<xpu, 2, real_t> &tmp_dst) {\n    index_t oheight  = (in.size(2) - ksize)/kstride + 1;\n    index_t owidth   = (in.size(3) - ksize)/kstride + 1;\n    index_t nbatch   = in.size(0);\n    index_t nchannel = out.size(1);\n    // we directly unpack all local patches and do a dot product\n    // this cost lots of memory, normally for large image, only unpack several image at a time\n    tmp_col.Resize(Shape2(in.size(1)*ksize*ksize, nbatch*oheight*owidth));\n    tmp_dst.Resize(Shape2(nchannel, nbatch*oheight*owidth));\n    // unpack local patches , stride=1\n\ttmp_col = unpack_patch2col(in, ksize, ksize, kstride, kstride, 1, 1);\n    tmp_dst = dot(kernel, tmp_col);\n    // reshape, then swap axis, we chain equations together\n    out = swapaxis<1,0>(reshape(tmp_dst, Shape4(nchannel, nbatch, oheight, owidth)));\n  }\n  // backward convolution, calculate gradient of kernel, and backprop back to in\n  inline static void ConvBackWard(const Tensor<xpu, 4, real_t> &out,\n                                  const Tensor<xpu, 2, real_t> &kernel,\n                                  Tensor<xpu, 2, real_t> &g_kernel,\n                                  Tensor<xpu, 4, real_t> &in,\n                                  int ksize, int kstride,\n                                  TensorContainer<xpu, 2, real_t> &tmp_col,\n                                  TensorContainer<xpu, 2, real_t> &tmp_dst) {\n    index_t oheight  = (in.size(2) - ksize)/kstride + 1;\n    index_t owidth   = (in.size(3) - ksize)/kstride + 1;\n    index_t nbatch   = in.size(0);\n    index_t nchannel = out.size(1);\n    // we directly unpack all local patches and do a dot product\n    // this cost lots of memory, normally for large image, only unpack several image at a time\n    tmp_col.Resize(Shape2(in.size(1) * ksize * ksize,\n                          nbatch * oheight * owidth));\n    tmp_dst.Resize(Shape2(nchannel, nbatch * oheight * owidth));\n    // unpack local patches\n    tmp_col = unpack_patch2col(in, ksize, ksize, kstride, kstride, 1, 1);\n    tmp_dst = reshape(swapaxis<1,0>(out), tmp_dst.shape_);\n    g_kernel = dot(tmp_dst, tmp_col.T());\n        // backpropgation: not necessary for first layer, but included anyway\n    tmp_col = dot(kernel.T(), tmp_dst);\n    in = pack_col2patch(tmp_col, in.shape_, ksize, ksize, kstride, kstride, 1, 1);\n  }\n private:\n  // random seed generator\n  Random<xpu, real_t> rnd;\n  // kernel size, pooling size\n  int ksize, kstride, psize;\n  // nodes in neural net\n  TensorContainer<xpu, 4, real_t> ninput, nhidden, nhiddenbak, npool, npoolbak;\n  TensorContainer<xpu, 2, real_t> nflat, nout;\n  // temp helper structure\n  TensorContainer<xpu, 2, real_t> tmp_col, tmp_dst;\n  // hidden bias, gradient\n  TensorContainer<xpu, 1, real_t> hbias, obias, g_hbias, g_obias;\n  // weight, gradient: Ki2h is actually convoltuion kernel, with shape=(num_channel,ksize*ksize)\n  TensorContainer<xpu, 2, real_t> Ki2h,  Wh2o, g_Ki2h, g_Wh2o;\n};\n\n// helper function to get the max inde\ninline int MaxIndex(Tensor<cpu, 1, real_t> pred) {\n  int maxidx = 0;\n  for (index_t i = 1; i < pred.size(0); ++i) {\n    if(pred[i] > pred[maxidx]) maxidx = (int)i;\n  }\n  return maxidx;\n}\n\nint main(int argc, char *argv[]) {\n  if(argc < 2) {\n    printf(\"Usage: cpu or gpu\\n\"); return 0;\n  }\n  srand(0);\n  // settings\n  int batch_size = 100;\n  int insize = 28;\n  int nchannel = 10;\n  int ksize = 5;\n  int kstride = 1;\n  int psize = 2;\n  int num_out = 10;\n\n  // choose which version to use\n  INNet *net;\n  if (!strcmp(argv[1], \"gpu\")) {\n    InitTensorEngine<gpu>();\n    net = new ConvNet<gpu>(batch_size, insize, nchannel, ksize, kstride, psize, num_out);\n  } else {\n    InitTensorEngine<cpu>();\n    net = new ConvNet<cpu>(batch_size, insize, nchannel, ksize, kstride, psize, num_out);\n  }\n\n  // temp output layer\n  TensorContainer<cpu, 2, real_t> pred;\n  pred.Resize(Shape2(batch_size, num_out));\n\n  // label\n  std::vector<int> ytrain, ytest;\n  // data\n  TensorContainer<cpu, 2, real_t> xtrain_, xtest_;\n  LoadMNIST(\"train-images-idx3-ubyte\", \"train-labels-idx1-ubyte\", ytrain, xtrain_, true);\n  LoadMNIST(\"t10k-images-idx3-ubyte\", \"t10k-labels-idx1-ubyte\", ytest, xtest_, false);\n\n  TensorContainer<cpu, 4, real_t> xtrain(Shape4(xtrain_.size(0), 1, insize, insize));\n  TensorContainer<cpu, 4, real_t> xtest(Shape4(xtest_.size(0),  1, insize, insize));\n  xtrain = reshape(xtrain_, xtrain.shape_);\n  xtest = reshape(xtest_, xtest.shape_);\n\n  int num_iter = 20;\n\n  for (int i = 0; i < num_iter; ++ i) {\n    // training\n    for (index_t j = 0; j + batch_size <= xtrain.size(0); j += batch_size) {\n      net->Forward(xtrain.Slice(j, j + batch_size), pred);\n      // set gradient into pred\n      for (int k = 0; k < batch_size; ++ k) {\n        pred[k][ ytrain[k+j] ] -= 1.0f;\n      }\n      // scale gradient by batchs zie\n      pred *= 1.0f / batch_size;\n      // run backprop\n      net->Backprop(pred);\n      // update net parameters\n      net->Update();\n    }\n    // evaluation\n    long nerr = 0;\n    for (index_t j = 0; j + batch_size <= xtest.size(0); j += batch_size) {\n      net->Forward(xtest.Slice(j, j + batch_size), pred);\n      for (int k = 0; k < batch_size; ++ k) {\n        nerr += MaxIndex(pred[k]) != ytest[j+k];\n      }\n    }\n    printf(\"round %d: test-err=%f\\n\", i, (float)nerr/xtest.size(0));\n  }\n  delete net;\n\n  if (!strcmp(argv[1], \"gpu\")) {\n    ShutdownTensorEngine<gpu>();\n  } else {\n    ShutdownTensorEngine<cpu>();\n  }\n  return 0;\n}\n"
  },
  {
    "path": "3rdparty/mshadow/guide/neuralnet/nnet.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n// this implements a simple two layer neural net\n#include <vector>\n#include <cmath>\n// header file to use mshadow\n#include \"mshadow/tensor.h\"\n// helper function to load mnist dataset\n#include \"util.h\"\n// this namespace contains all data structures, functions\nusing namespace mshadow;\n// this namespace contains all operator overloads\nusing namespace mshadow::expr;\n\n// define sigmoid operation\nstruct sigmoid{\n  MSHADOW_XINLINE static real_t Map(real_t a) {\n    return  1.0f/(1.0f+expf(-a));\n  }\n};\n\n/*! \\brief interface for nnet, interfacd allows use to use GPU/CPU implementation in a unified way */\nclass INNet{\n public:\n  virtual void Forward(const Tensor<cpu, 2, real_t>& inbatch, Tensor<cpu, 2, real_t> &oubatch) = 0;\n  virtual void Backprop(const Tensor<cpu, 2, real_t>& gradout) = 0;\n  virtual void Update(void) = 0;\n  virtual ~INNet() {}\n};\n\n/*!\n * \\brief simple two layer neural net\n *        this implementation is device invariant\n */\ntemplate<typename xpu>\nclass NNet : public INNet {\n public:\n  // initialize the network\n  NNet(int batch_size, int num_in, int num_hidden, int num_out) : rnd(0) {\n    // setup stream\n    Stream<xpu> *stream = NewStream<xpu>();\n    ninput.set_stream(stream);\n    nhidden.set_stream(stream);\n    nhiddenbak.set_stream(stream);\n    nout.set_stream(stream);\n    hbias.set_stream(stream);\n    g_hbias.set_stream(stream);\n    g_obias.set_stream(stream);\n    obias.set_stream(stream);\n    Wi2h.set_stream(stream);\n    Wh2o.set_stream(stream);\n    g_Wi2h.set_stream(stream);\n    g_Wh2o.set_stream(stream);\n    // setup nodes\n    ninput.Resize(Shape2(batch_size, num_in));\n    nhidden.Resize(Shape2(batch_size, num_hidden));\n    nhiddenbak.Resize(nhidden.shape_);\n    nout.Resize(Shape2(batch_size, num_out));\n    // setup bias\n    hbias.Resize(Shape1(num_hidden)); g_hbias.Resize(hbias.shape_);\n    obias.Resize(Shape1(num_out)); g_obias.Resize(obias.shape_);\n    hbias = 0.0f; obias = 0.0f;\n    // setup weights\n    Wi2h.Resize(Shape2(num_in, num_hidden));  g_Wi2h.Resize(Wi2h.shape_);\n    Wh2o.Resize(Shape2(num_hidden, num_out)); g_Wh2o.Resize(Wh2o.shape_);\n    rnd.SampleGaussian(&Wi2h, 0, 0.01f);\n    rnd.SampleGaussian(&Wh2o, 0, 0.01f);\n  }\n  virtual ~NNet() {}\n  // forward propagation\n  virtual void Forward(const Tensor<cpu, 2, real_t>& inbatch,\n                       Tensor<cpu, 2, real_t> &oubatch) {\n    // size is same conventsion as numpy\n    index_t batch_size = inbatch.size(0);\n    // copy data to input layer\n    Copy(ninput, inbatch, ninput.stream_);\n    // first layer, fullc\n    nhidden = dot(ninput, Wi2h);\n    nhidden+= repmat(hbias, batch_size);\n    // activation, sigmloid, backup activation in nhidden\n    nhidden = F<sigmoid>(nhidden);\n    Copy(nhiddenbak, nhidden, nhiddenbak.stream_);\n    // second layer fullc\n    nout = dot(nhiddenbak, Wh2o);\n    nout += repmat(obias, batch_size);\n    // softmax calculation\n    Softmax(nout, nout);\n    // copy result out\n    Copy(oubatch, nout, nout.stream_);\n  }\n  // back propagation\n  virtual void Backprop(const Tensor<cpu, 2, real_t>& gradout) {\n    // copy gradient to output layer\n    Copy(nout, gradout, nout.stream_);\n    // calc grad of layer 2\n    g_obias = sum_rows(nout);\n    g_Wh2o  = dot(nhiddenbak.T(), nout);\n    // backprop to layer 1\n    nhiddenbak = dot(nout, Wh2o.T());\n    // calculate gradient of sigmoid layer\n    nhidden = nhidden * (1.0f-nhidden) * nhiddenbak;\n    // calc grad of layer 1\n    g_hbias = sum_rows(nhidden);\n    g_Wi2h  = dot(ninput.T(), nhidden);\n  }\n  // update weight\n  virtual void Update(void) {\n    // run SGD\n    const float eta = 0.8;\n    const float wd = 0.00001;\n    // update weight\n    Wi2h -= eta * (wd * Wi2h + g_Wi2h);\n    Wh2o -= eta * (wd * Wh2o + g_Wh2o);\n    // no regularization for bias\n    hbias-= eta * g_hbias;\n    obias-= eta * g_obias;\n  }\n private:\n  // random seed generator\n  Random<xpu, real_t> rnd;\n  // nodes in neural net\n  TensorContainer<xpu, 2, real_t> ninput, nhidden, nhiddenbak, nout;\n  // hidden bias, gradient\n  TensorContainer<xpu, 1, real_t> hbias, obias, g_hbias, g_obias;\n  // weight gradient\n  TensorContainer<xpu, 2, real_t> Wi2h, Wh2o, g_Wi2h, g_Wh2o;\n};\n// helper function to get the max inde\ninline int MaxIndex(Tensor<cpu, 1, real_t> pred) {\n  int maxidx = 0;\n  for(index_t i = 1; i < pred.size(0); ++i) {\n    if(pred[i] > pred[maxidx]) maxidx = (int)i;\n  }\n  return maxidx;\n}\n\nint main(int argc, char *argv[]) {\n  if(argc < 2) {\n    printf(\"Usage: cpu or gpu\\n\"); return 0;\n  }\n  srand(0);\n\n  // settings\n  int batch_size = 100;\n  int num_in = 28 * 28;\n  int num_hidden = 100;\n  int num_out = 10;\n  // choose which version to use\n  INNet *net;\n  if (!strcmp(argv[1], \"gpu\")) {\n    InitTensorEngine<gpu>();\n    net = new NNet<gpu>(batch_size, num_in, num_hidden, num_out);\n  } else {\n    InitTensorEngine<cpu>();\n    net = new NNet<cpu>(batch_size, num_in, num_hidden, num_out);\n  }\n\n  // temp output layer\n  TensorContainer<cpu, 2, real_t> pred;\n  pred.Resize(Shape2(batch_size, num_out));\n\n  // label\n  std::vector<int> ytrain, ytest;\n  // data\n  TensorContainer<cpu,2> xtrain, xtest;\n  LoadMNIST(\"train-images-idx3-ubyte\", \"train-labels-idx1-ubyte\", ytrain, xtrain, true);\n  LoadMNIST(\"t10k-images-idx3-ubyte\", \"t10k-labels-idx1-ubyte\", ytest, xtest, false);\n\n  int num_iter = 20;\n\n  for (int i = 0; i < num_iter; ++ i) {\n    // training\n    for (index_t j = 0; j + batch_size <= xtrain.size(0); j += batch_size) {\n      net->Forward(xtrain.Slice(j, j + batch_size), pred);\n      // set gradient into pred\n      for (int k = 0; k < batch_size; ++ k) {\n        pred[k][ ytrain[k+j] ] -= 1.0f;\n      }\n      // scale gradient by batchs zie\n      pred *= 1.0f / batch_size;\n      // run backprop\n      net->Backprop(pred);\n      // update net parameters\n      net->Update();\n    }\n    // evaluation\n    long nerr = 0;\n    for (index_t j = 0; j + batch_size <= xtest.size(0); j += batch_size) {\n      net->Forward(xtest.Slice(j, j + batch_size), pred);\n      for (int k = 0; k < batch_size; ++ k) {\n        nerr += MaxIndex(pred[k]) != ytest[j+k];\n\n      }\n    }\n    printf(\"round %d: test-err=%f\\n\", i, (float)nerr/xtest.size(0));\n  }\n  delete net;\n  if (!strcmp(argv[1], \"gpu\")) {\n    ShutdownTensorEngine<gpu>();\n  } else {\n    ShutdownTensorEngine<cpu>();\n  }\n  return 0;\n}\n"
  },
  {
    "path": "3rdparty/mshadow/guide/neuralnet/nnet_ps.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n// this implements a simple two layer Multi-GPU neural net\n// this implementation uses mshadow-ps to get gradient aggregation\n// between cards\n// this code is modified from nnet.cu\n#include <vector>\n#include <cmath>\n#include <omp.h>\n// header file to use mshadow\n#include <mshadow/tensor.h>\n#include <mshadow-ps/mshadow_ps.h>\n// helper function to load mnist dataset\n#include \"./util.h\"\n// this namespace contains all data structures, functions\nusing namespace mshadow;\n// this namespace contains all operator overloads\nusing namespace mshadow::expr;\n\n// define sigmoid operation\nstruct sigmoid {\n  MSHADOW_XINLINE static real_t Map(real_t a) {\n    return 1.0f / (1.0f + expf(-a));\n  }\n};\n\n/*! \\brief interface for nnet, interfacd allows use to use GPU/CPU implementation in a unified way */\nclass INNet{\n public:\n  virtual void Forward(const Tensor<cpu, 2, real_t>& inbatch,\n                       Tensor<cpu, 2, real_t> &oubatch) = 0;\n  virtual void Backprop(const Tensor<cpu, 2, real_t>& gradout) = 0;\n  virtual ~INNet() {}\n};\n\n/*!\n * \\brief simple two layer neural net\n *        this implementation is device invariant\n */\ntemplate<typename xpu>\nclass NNet : public INNet {\n public:\n  // initialize the network\n  NNet(int batch_size, int num_in, int num_hidden, int num_out,\n       int devid, mshadow::ps::ISharedModel<xpu, real_t> *ps)\n      : rnd(0), devid(devid), ps(ps) {\n    mshadow::SetDevice<xpu>(devid);\n    stream = mshadow::NewStream<xpu>();\n    // set the computing streams\n    ninput.set_stream(stream);\n    nhidden.set_stream(stream);\n    nhiddenbak.set_stream(stream);\n    nout.set_stream(stream);\n    hbias.set_stream(stream);\n    obias.set_stream(stream);\n    g_hbias.set_stream(stream);\n    g_obias.set_stream(stream);\n    Wi2h.set_stream(stream);\n    Wh2o.set_stream(stream);\n    g_Wi2h.set_stream(stream);\n    g_Wh2o.set_stream(stream);\n    rnd.set_stream(stream);\n    // setup nodes\n    ninput.Resize(Shape2(batch_size, num_in));\n    nhidden.Resize(Shape2(batch_size, num_hidden));\n    nhiddenbak.Resize(nhidden.shape_);\n    nout.Resize(Shape2(batch_size, num_out));\n    // setup bias\n    hbias.Resize(Shape1(num_hidden)); g_hbias.Resize(hbias.shape_);\n    obias.Resize(Shape1(num_out)); g_obias.Resize(obias.shape_);\n    hbias = 0.0f; obias = 0.0f;\n    // setup weights\n    Wi2h.Resize(Shape2(num_in, num_hidden));  g_Wi2h.Resize(Wi2h.shape_);\n    Wh2o.Resize(Shape2(num_hidden, num_out)); g_Wh2o.Resize(Wh2o.shape_);\n    rnd.SampleGaussian(&Wi2h, 0, 0.01f);\n    rnd.SampleGaussian(&Wh2o, 0, 0.01f);\n    // initialize the key\n    ps->InitKey(Wi2h.shape_, 0, devid);\n    ps->InitKey(hbias.shape_, 1, devid);\n    ps->InitKey(Wh2o.shape_, 2, devid);\n    ps->InitKey(obias.shape_, 3, devid);\n  }\n  virtual ~NNet() {\n    mshadow::SetDevice<xpu>(devid);\n    mshadow::DeleteStream(stream);\n  }\n  // forward propagation\n  virtual void Forward(const Tensor<cpu, 2, real_t> &inbatch,\n                       Tensor<cpu, 2, real_t> &oubatch) {\n    // size is same conventsion as numpy\n    index_t batch_size = inbatch.size(0);\n    // copy data to input layer\n    Copy(ninput, inbatch, stream);\n    // wait the last pull requst on layer to complete\n    ps->PullWait(0, devid);\n    // first layer, fullc\n    nhidden = dot(ninput, Wi2h);\n    // wait the pull request on hbias to complete\n    ps->PullWait(1, devid);\n    nhidden+= repmat(hbias, batch_size);\n    // activation, sigmloid, backup activation in nhidden\n    nhidden = F<sigmoid>(nhidden);\n    Copy(nhiddenbak, nhidden, stream);\n    // second layer fullc\n    ps->PullWait(2, devid);\n    nout = dot(nhiddenbak, Wh2o);\n    ps->PullWait(3, devid);\n    nout += repmat(obias, batch_size);\n    // softmax calculation\n    Softmax(nout, nout);\n    // copy result out\n    Copy(oubatch, nout, stream);\n    // Copy with stream is non-blocking, use wait to wait until copy finishes\n    stream->Wait();\n  }\n  // back propagation\n  virtual void Backprop(const Tensor<cpu, 2, real_t> &gradout) {\n    // copy gradient to output layer\n    Copy(nout, gradout, stream);\n    // calc grad of layer 2\n    g_obias = sum_rows(nout);\n    // sync proc defines the synchronization step\n    this->SyncProc(obias, g_obias, 3);\n    // update second layer weights\n    g_Wh2o = dot(nhiddenbak.T(), nout);\n    // backprop to layer 1\n    nhiddenbak = dot(nout, Wh2o.T());\n    this->SyncProc(Wh2o, g_Wh2o, 2);\n    // calculate gradient of sigmoid layer\n    nhidden = nhidden * (1.0f-nhidden) * nhiddenbak;\n    // calc grad of layer 1\n    g_hbias = sum_rows(nhidden);\n    this->SyncProc(hbias, g_hbias, 1);\n    g_Wi2h = dot(ninput.T(), nhidden);\n    this->SyncProc(Wi2h, g_Wi2h, 0);\n  }\n  // synchronization function\n  template<int dim>\n  inline void SyncProc(mshadow::Tensor<xpu, dim> weight,\n                       mshadow::Tensor<xpu, dim> grad,\n                       int data_key) {\n    // wait till last computation finishes\n    stream->Wait();\n    ps->Push(grad, data_key, devid, -data_key);\n    ps->PullReq(grad, data_key, devid, -data_key,\n                UpdateEntry::ApplyUpdate,\n                new UpdateEntry(weight.FlatTo2D(), grad.FlatTo2D(), dim == 1));\n  }\n  // data structure defined to help using callback function\n  struct UpdateEntry {\n    mshadow::Tensor<xpu, 2> weight;\n    mshadow::Tensor<xpu, 2> grad;\n    bool is_bias;\n    // constructor\n    UpdateEntry(mshadow::Tensor<xpu, 2> weight,\n                mshadow::Tensor<xpu, 2> grad,\n                bool is_bias)\n        : weight(weight), grad(grad),\n          is_bias(is_bias) {}\n    inline void Update(mshadow::Stream<xpu> *stream) {\n      weight.set_stream(stream);\n      const float wd = 0.00001;\n      const float eta = 0.8;\n      if (!is_bias) {\n        weight -= eta * (wd * weight + grad);\n      } else {\n        weight -= eta * grad;\n      }\n    }\n    // callback function to apply update\n    inline static void ApplyUpdate(mshadow::Stream<xpu> *stream, void *arg) {\n      UpdateEntry *e = static_cast<UpdateEntry*>(arg);\n      e->Update(stream);\n      delete e;\n    }\n  };\n\n private:\n  // computing stream\n  mshadow::Stream<xpu> *stream;\n  // device id\n  int devid;\n  // parameter server interface\n  mshadow::ps::ISharedModel<xpu, real_t> *ps;\n  // random seed generator\n  Random<xpu, real_t> rnd;\n  // nodes in neural net\n  TensorContainer<xpu, 2, real_t> ninput, nhidden, nhiddenbak, nout;\n  // hidden bias, gradient\n  TensorContainer<xpu, 1, real_t> hbias, obias, g_hbias, g_obias;\n  // weight gradient\n  TensorContainer<xpu, 2, real_t> Wi2h, Wh2o, g_Wi2h, g_Wh2o;\n};\n\n// helper function to get the max inde\ninline int MaxIndex(Tensor<cpu, 1, real_t> pred) {\n  int maxidx = 0;\n  for(index_t i = 1; i < pred.size(0); ++i) {\n    if(pred[i] > pred[maxidx]) maxidx = (int)i;\n  }\n  return maxidx;\n}\n\nnamespace mshadow {\nnamespace ps {\n// model updater is used when update is happening on server side\n// if we only use parameter server for sum aggregation\n// this is not needed, but we must declare this function to return NULL\ntemplate<>\nIModelUpdater<float> *CreateModelUpdater(void) {\n  return NULL;\n}\n}\n}\n\ntemplate<typename xpu>\ninline int Run(int argc, char *argv[]) {\n  srand(0);\n  // settings\n  int batch_size = 100;\n  int num_in = 28 * 28;\n  int num_hidden = 100;\n  int num_out = 10;\n  int ndev = argc - 2;\n  if (batch_size % ndev != 0) {\n    fprintf(stderr, \"choose number of devices ndev such that 100 MOD ndev == 0\\n\");\n    return 0;\n  }\n  // choose which version to use\n  std::vector<int> devs;\n  for (int i = 2; i < argc; ++i) {\n    devs.push_back(atoi(argv[i]));\n  }\n  mshadow::ps::ISharedModel<xpu, real_t>\n      *ps = mshadow::ps::CreateSharedModel<xpu, real_t>(\"local\");\n  ps->Init(devs);\n\n  std::vector<INNet *> nets(ndev);\n  for (int i = 0; i < ndev; ++i) {\n    mshadow::InitTensorEngine<xpu>(devs[i]);\n    nets[i] = new NNet<xpu>(batch_size / ndev, num_in, num_hidden, num_out, devs[i], ps);\n  }\n\n  // label\n  std::vector<int> ytrain, ytest;\n  // data\n  TensorContainer<cpu,2> xtrain, xtest;\n  LoadMNIST(\"train-images-idx3-ubyte\", \"train-labels-idx1-ubyte\", ytrain, xtrain, true);\n  LoadMNIST(\"t10k-images-idx3-ubyte\", \"t10k-labels-idx1-ubyte\", ytest, xtest, false);\n  int num_iter = 20;\n\n  for (int i = 0; i < num_iter; ++ i) {\n    // mini-batch per device\n    int step = batch_size / ndev;\n    // running parallel threads\n    #pragma omp parallel num_threads(ndev)\n    {\n      // temp output layer\n      TensorContainer<cpu, 2, real_t> pred;\n      pred.Resize(Shape2(step, num_out));\n      int tid = omp_get_thread_num();\n      mshadow::SetDevice<xpu>(devs[tid]);\n      for (index_t j = 0; j + batch_size <= xtrain.size(0); j += batch_size) {\n        nets[tid]->Forward(xtrain.Slice(j + tid * step, j + (tid + 1) * step), pred);\n        // set gradient into pred\n        for (int k = 0; k < step; ++ k) {\n          pred[k][ytrain[j + tid * step + k]] -= 1.0f;\n        }\n        // scale gradient by batchs zie\n        pred *= 1.0f / batch_size;\n        // run backprop\n        nets[tid]->Backprop(pred);\n      }\n    }\n    // evaluation\n    long nerr = 0;\n    #pragma omp parallel num_threads(ndev) reduction(+:nerr)\n    {\n      // temp output layer\n      TensorContainer<cpu, 2, real_t> pred;\n      pred.Resize(Shape2(step, num_out));\n      int tid = omp_get_thread_num();\n      mshadow::SetDevice<xpu>(devs[tid]);\n      for (index_t j = 0; j + batch_size <= xtest.size(0); j += batch_size) {\n        nets[tid]->Forward(xtest.Slice(j + tid * step, j + (tid + 1) * step), pred);\n        for (int k = 0; k < step; ++ k) {\n          nerr += MaxIndex(pred[k]) != ytest[j + tid * step + k];\n        }\n      }\n    }\n    printf(\"round %d: test-err=%f\\n\", i, (float)nerr/xtest.size(0));\n  }\n\n  for(int i = 0; i < ndev; ++i) {\n    mshadow::SetDevice<xpu>(devs[i]);\n    delete nets[i];\n    ShutdownTensorEngine<xpu>();\n  }\n  return 0;\n}\nint main(int argc, char *argv[]) {\n  if (argc < 3) {\n    printf(\"Usage: <device> devicelist\\n\"\\\n           \"\\tExample1: ./nnet_ps cpu 1 2 3\\n\"\\\n           \"\\tExample2: ./nnet_ps gpu 0 1\\n\");\n    return 0;\n  }\n  if (!strcmp(argv[1], \"cpu\")) {\n    Run<mshadow::cpu>(argc, argv);\n  } else {\n    Run<mshadow::gpu>(argc, argv);\n  }\n  return 0;\n}\n"
  },
  {
    "path": "3rdparty/mshadow/guide/neuralnet/util.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#pragma once\n#include <assert.h>\n#include <cstdio>\n#include <cstdlib>\n#include \"mshadow/tensor.h\"\n\ntypedef float real_t;\n\nusing namespace mshadow;\n\nint pack(unsigned char zz[4]){\n    return (int)(zz[3]) \n        | (((int)(zz[2])) << 8)\n        | (((int)(zz[1])) << 16)\n        | (((int)(zz[0])) << 24);\n}\n\ntemplate<typename T>\ninline void shuffle(T *data, size_t sz){\n  if(sz == 0) return;\n  for(size_t i = sz - 1; i > 0; i--){\n    std::swap(data[i], data[rand() % (i+1)]);\n  } \n}\n// random shuffle the data inside, require PRNG \ntemplate<typename T>\ninline void shuffle(std::vector<T> &data){\n  shuffle(&data[0], data.size());\n}\n\n// simple function to load in mnist\ninline void LoadMNIST(const char *path_img, const char *path_label,\n                      std::vector<int> &ylabel,\n                      TensorContainer<cpu, 2, real_t> &xdata,\n                      bool do_shuffle){\n  // load in data\n  FILE *fi = fopen(path_img, \"rb\");\n  if (fi == NULL) {\n    printf(\"cannot open %s\\n\", path_img);\n    exit(-1);\n  }\n  unsigned char zz[4];\n  unsigned char *t_data, *l_data;\n  int num_image, width, height, nlabel;            \n  assert(fread(zz, 4 , 1, fi));\n  assert(fread(zz, 4 , 1, fi));    \n  num_image = pack(zz);\n  assert(fread(zz, 4 , 1, fi));                \n  width = pack(zz);\n  assert(fread(zz, 4 , 1, fi));                    \n  height = pack(zz);\n  \n  int step = width * height;\n  t_data = new unsigned char[num_image * step];    \n  assert(fread(t_data, step*num_image , 1 , fi));\n  fclose(fi);\n  \n  // load in label\n  fi = fopen(path_label, \"rb\");\n  assert(fread(zz, 4 , 1, fi));\n  assert(fread(zz, 4 , 1, fi));    \n  nlabel = pack(zz);\n  assert(num_image == nlabel);\n  l_data = new unsigned char[num_image];\n  assert(fread(l_data, num_image , 1 , fi));    \n  // try to do shuffle \n  std::vector<int> rindex;\n  for (int i = 0; i < num_image; ++ i) {\n    rindex.push_back(i);\n  }\n  if (do_shuffle) {\n    shuffle(rindex);\n  }\n  \n  // save out result\n  ylabel.resize(num_image);\n  xdata.Resize(Shape2(num_image, width * height));\n  for (int i = 0 ; i < num_image ; ++i) {\n    for(int j = 0; j < step; ++j) {\n      xdata[i][j] = (float)(t_data[rindex[i]*step + j]) / 256.0f;            \n    }        \n    ylabel[i] = l_data[rindex[i]];\n  }\n  delete[] t_data; delete [] l_data;\n  printf(\"finish loading %dx%d matrix from %s, shuffle=%d\\n\", num_image, step, path_img, (int)do_shuffle);\n}\n"
  },
  {
    "path": "3rdparty/mshadow/make/README.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nMakefile Configuration of MShadow\n=====\nMShadow is a template library, you only need to include mshadow to use it. So this folder is not used to build mshadow library file.\n\nHowever, mshadow is a flexible library that allows you to compile with different configurations. For example,\nyou can compile mshadow without CUDA, and specify your own choice of BLAS.\nThere are different compile flags that you might need to set in your own configuration.\nThis folder provides a Makefile script to help you do that.\n\nUsage\n=====\n* Set the configurations via variables in your Makefile, see example in [../guide/config.mk](../guide/config.mk)\n* include [mshadow.mk](mshadow.mk) in your Makefile\n* mshadow.mk will give you compiler variables that you can include when compiling\n  - Add MSHADOW_CFLAGS to the compile flags\n  - Add MSHADOW_LDFLAGS to the linker flags\n  - Add MSHADOW_NVCCFLAGS to the nvcc compile flags\n* For example Makefile, see [../guide/Makefile](../guide/Makefile)\n"
  },
  {
    "path": "3rdparty/mshadow/make/mshadow.mk",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n#---------------------------------------------------------------------------------------\n#  mshadow configuration script\n#\n#  include mshadow.mk after the variables are set\n#\n#  Add MSHADOW_CFLAGS to the compile flags\n#  Add MSHADOW_LDFLAGS to the linker flags\n#  Add MSHADOW_NVCCFLAGS to the nvcc compile flags\n#----------------------------------------------------------------------------------------\n\nMSHADOW_CFLAGS = -funroll-loops -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs\nMSHADOW_LDFLAGS = -lm\nMSHADOW_NVCCFLAGS =\n\n\n# atlas blas library has different name on CentOS\nOS := $(shell cat /etc/system-release 2>/dev/null)\nifeq ($(findstring CentOS,$(OS)), CentOS)\n  ATLAS_LDFLAGS := -lsatlas -L/usr/lib64/atlas\nelse\n  ATLAS_LDFLAGS := -lcblas\nendif\n\nifndef USE_SSE\n\tUSE_SSE=1\nendif\n\nifeq ($(USE_SSE), 1)\n\tMSHADOW_CFLAGS += -msse3\nelse\n\tMSHADOW_CFLAGS += -DMSHADOW_USE_SSE=0\nendif\n\n# whether to use F16C instruction set extension for fast fp16 compute on CPU\n# if cross compiling you may want to explicitly turn it off if target system does not support it\nifndef USE_F16C\n    ifneq ($(OS),Windows_NT)\n        detected_OS := $(shell uname -s)\n        ifeq ($(detected_OS),Darwin)\n            F16C_SUPP = $(shell sysctl -a | grep machdep.cpu.features | grep F16C)\n        endif\n        ifeq ($(detected_OS),Linux)\n            F16C_SUPP = $(shell cat /proc/cpuinfo | grep flags | grep f16c)\n        endif\n\tifneq ($(strip $(F16C_SUPP)),)\n                USE_F16C=1\n        else\n                USE_F16C=0\n        endif\n    endif\n    # if OS is Windows, check if your processor and compiler support F16C architecture.\n    # One way to check if processor supports it is to download the tool \n    # https://docs.microsoft.com/en-us/sysinternals/downloads/coreinfo.\n    # If coreinfo -c shows F16C and compiler supports it, \n    # then you can set USE_F16C=1 explicitly to leverage that capability\"\nendif\n\nifeq ($(USE_F16C), 1)\n        MSHADOW_CFLAGS += -mf16c\nelse\n        MSHADOW_CFLAGS += -DMSHADOW_USE_F16C=0\nendif\n\nifeq ($(USE_CUDA), 0)\n\tMSHADOW_CFLAGS += -DMSHADOW_USE_CUDA=0\nelse\n\tMSHADOW_LDFLAGS += -lcudart -lcublas -lcurand -lcusolver\nendif\nifneq ($(USE_CUDA_PATH), NONE)\n\tMSHADOW_CFLAGS += -I$(USE_CUDA_PATH)/include\n\tMSHADOW_LDFLAGS += -L$(USE_CUDA_PATH)/lib64 -L$(USE_CUDA_PATH)/lib\nendif\n\nifeq ($(USE_BLAS), mkl)\nifneq ($(USE_INTEL_PATH), NONE)\n\tUNAME_S := $(shell uname -s)\n\tifeq ($(UNAME_S),Darwin)\n\t\tMSHADOW_LDFLAGS += -L$(USE_INTEL_PATH)/mkl/lib\n\t\tMSHADOW_LDFLAGS += -L$(USE_INTEL_PATH)/lib\n\telse\n\t\tMSHADOW_LDFLAGS += -L$(USE_INTEL_PATH)/mkl/lib/intel64\n\t\tMSHADOW_LDFLAGS += -L$(USE_INTEL_PATH)/compiler/lib/intel64\n\t\tMSHADOW_LDFLAGS += -L$(USE_INTEL_PATH)/lib/intel64\n\tendif\n\tMSHADOW_CFLAGS += -I$(USE_INTEL_PATH)/mkl/include\nendif\nifneq ($(USE_STATIC_MKL), NONE)\nifeq ($(USE_INTEL_PATH), NONE)\n\tMKLROOT = /opt/intel/mkl\nelse\n\tMKLROOT = $(USE_INTEL_PATH)/mkl\nendif\n\tMSHADOW_LDFLAGS += -L${MKLROOT}/../compiler/lib/intel64 -Wl,--start-group ${MKLROOT}/lib/intel64/libmkl_intel_lp64.a ${MKLROOT}/lib/intel64/libmkl_core.a ${MKLROOT}/lib/intel64/libmkl_intel_thread.a -Wl,--end-group -liomp5 -ldl -lpthread -lm\nelse\nifneq ($(USE_MKLML), 1)\n  MSHADOW_LDFLAGS += -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -liomp5\nendif\nendif\nelse\nifneq ($(USE_BLAS), NONE)\n\tMSHADOW_CFLAGS += -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0\nendif\nendif\n\nifeq ($(USE_MKLML), 1)\n\tMSHADOW_CFLAGS += -I$(MKLROOT)/include\n\tifneq ($(shell uname),Darwin)\n\t\tMSHADOW_LDFLAGS += -Wl,--as-needed -lmklml_intel -lmklml_gnu\n\telse\n\t\tMSHADOW_LDFLAGS += -lmklml\n\tendif\n\tMSHADOW_LDFLAGS += -liomp5 -L$(MKLROOT)/lib/\nendif\n\nifeq ($(USE_BLAS), openblas)\n\tMSHADOW_LDFLAGS += -lopenblas\nelse ifeq ($(USE_BLAS), perfblas)\n\tMSHADOW_LDFLAGS += -lperfblas\nelse ifeq ($(USE_BLAS), atlas)\n\tMSHADOW_LDFLAGS += $(ATLAS_LDFLAGS)\nelse ifeq ($(USE_BLAS), blas)\n\tMSHADOW_LDFLAGS += -lblas\nelse ifeq ($(USE_BLAS), apple)\n\tMSHADOW_CFLAGS += -I/System/Library/Frameworks/Accelerate.framework/Versions/Current/Frameworks/vecLib.framework/Versions/Current/Headers/\n\tMSHADOW_LDFLAGS += -framework Accelerate\nendif\n\nifeq ($(PS_PATH), NONE)\n\tPS_PATH = ..\nendif\nifeq ($(PS_THIRD_PATH), NONE)\n\tPS_THIRD_PATH = $(PS_PATH)/third_party\nendif\n\nifndef RABIT_PATH\n\tRABIT_PATH = rabit\nendif\n\nifeq ($(RABIT_PATH), NONE)\n\tRABIT_PATH = rabit\nendif\n\nifeq ($(USE_RABIT_PS),1)\n\tMSHADOW_CFLAGS += -I$(RABIT_PATH)/include\n\tMSHADOW_LDFLAGS += -L$(RABIT_PATH)/lib -lrabit_base\n\tMSHADOW_CFLAGS += -DMSHADOW_RABIT_PS=1\nelse\n\tMSHADOW_CFLAGS += -DMSHADOW_RABIT_PS=0\nendif\n\nifeq ($(USE_DIST_PS),1)\nMSHADOW_CFLAGS += -DMSHADOW_DIST_PS=1 -std=c++17 \\\n\t-I$(PS_PATH)/src -I$(PS_THIRD_PATH)/include\nPS_LIB = $(addprefix $(PS_PATH)/build/, libps.a libps_main.a) \\\n\t$(addprefix $(PS_THIRD_PATH)/lib/, libgflags.a libzmq.a libprotobuf.a \\\n\tlibglog.a libz.a libsnappy.a)\n\t# -L$(PS_THIRD_PATH)/lib -lgflags -lzmq -lprotobuf -lglog -lz -lsnappy\nMSHADOW_NVCCFLAGS += --std=c++14\nelse\n\tMSHADOW_CFLAGS+= -DMSHADOW_DIST_PS=0\nendif\n\n# MSHADOW_USE_PASCAL=1 used to enable true-fp16 gemms.  Now, mshadow\n# only uses pseudo-fp16 gemms, so this flag will be removed after\n# dependent projects no longer reference it.\nMSHADOW_CFLAGS += -DMSHADOW_USE_PASCAL=0\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/README.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nCode Guide\n====\nThis readme contains notes about code in mshadow. MShadow generally follows Google's C++ Style.\n\nConvention\n====\n* Basically, all the files ends in ```-inl.h, -inl.cuh``` are implementations, and can be ignored if only using mshadow\n* The files ends in ```.h``` are heavily commented with [doxyen format](http://www.doxygen.org/), and can be used to generate the corresponding document.\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/base.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file base.h\n * \\brief definitions of base types, operators, macros functions\n *\n * \\author Bing Xu, Tianqi Chen\n */\n#ifndef MSHADOW_BASE_H_\n#define MSHADOW_BASE_H_\n#ifdef _MSC_VER\n#ifndef _CRT_SECURE_NO_WARNINGS\n#define _CRT_SECURE_NO_WARNINGS\n#endif\n#ifndef _CRT_SECURE_NO_DEPRECATE\n#define _CRT_SECURE_NO_DEPRECATE\n#endif\n#ifndef NOMINMAX\n#define NOMINMAX\n#endif\n#endif\n#include <algorithm>\n#include <cfloat>\n#include <climits>\n#include <cmath>\n#include <cstdio>\n#include <functional>\n#include <limits>\n#include <sstream>\n#include <string>\n\n#ifdef _MSC_VER\n//! \\cond Doxygen_Suppress\ntypedef signed char int8_t;\ntypedef __int16 int16_t;\ntypedef __int32 int32_t;\ntypedef __int64 int64_t;\ntypedef unsigned char uint8_t;\ntypedef unsigned __int16 uint16_t;\ntypedef unsigned __int32 uint32_t;\ntypedef unsigned __int64 uint64_t;\n//! \\endcond\n#else\n#include <inttypes.h>\n#endif\n// macro defintiions\n/*!\n * \\brief if this macro is define to be 1,\n * mshadow should compile without any of other libs\n */\n#ifndef MSHADOW_STAND_ALONE\n#define MSHADOW_STAND_ALONE 0\n#endif\n/*! \\brief whether do padding during allocation */\n#ifndef MSHADOW_ALLOC_PAD\n#define MSHADOW_ALLOC_PAD true\n#endif\n/*!\n * \\brief\n *  x dimension of data must be bigger pad_size * ratio to be alloced padded memory,\n *  otherwise use tide allocation\n *  for example, if pad_ratio=2, GPU memory alignement size is 32,\n *  then we will only allocate padded memory if x dimension > 64\n *  set it to 0 then we will always allocate padded memory\n */\n#ifndef MSHADOW_MIN_PAD_RATIO\n  #define MSHADOW_MIN_PAD_RATIO 2\n#endif\n\n#if MSHADOW_STAND_ALONE\n  #define MSHADOW_USE_CBLAS 0\n  #define MSHADOW_USE_MKL   0\n  #define MSHADOW_USE_CUDA  0\n#endif\n\n/*!\n * \\brief force user to use GPU stream during computation\n *  error will be shot when default stream NULL is used\n */\n#ifndef MSHADOW_FORCE_STREAM\n#define MSHADOW_FORCE_STREAM 1\n#endif\n\n/*! \\brief use CBLAS for CBLAS */\n#ifndef MSHADOW_USE_CBLAS\n  #define MSHADOW_USE_CBLAS 0\n#endif\n/*! \\brief use MKL for BLAS */\n#ifndef MSHADOW_USE_MKL\n  #define MSHADOW_USE_MKL   1\n#endif\n\n/*!\n * \\brief use CUDA support, must ensure that the cuda include path is correct,\n * or directly compile using nvcc\n */\n#ifndef MSHADOW_USE_CUDA\n  #define MSHADOW_USE_CUDA   1\n#endif\n\n/*!\n * \\brief use CUDNN support, must ensure that the cudnn include path is correct\n */\n#ifndef MSHADOW_USE_CUDNN\n  #define MSHADOW_USE_CUDNN 0\n#endif\n\n/*!\n * \\brief use CUTENSOR support, must ensure that the cutensor include path is correct\n */\n#ifndef MSHADOW_USE_CUTENSOR\n  #define MSHADOW_USE_CUTENSOR 0\n#endif\n\n/*!\n * \\brief use CUSOLVER support\n */\n#ifndef MSHADOW_USE_CUSOLVER\n  #define MSHADOW_USE_CUSOLVER MSHADOW_USE_CUDA\n#endif\n\n/*!\n * \\brief seems CUDAARCH is deprecated in future NVCC\n * set this to 1 if you want to use CUDA version smaller than 2.0\n */\n#ifndef MSHADOW_OLD_CUDA\n#define MSHADOW_OLD_CUDA 0\n#endif\n\n/*! \\brief whether use SSE */\n#ifndef MSHADOW_USE_SSE\n  #define MSHADOW_USE_SSE 1\n#endif\n\n/*! \\brief whether use F16C instruction set architecture extension */\n#ifndef MSHADOW_USE_F16C\n  #if defined(_MSC_VER) || defined(__CUDACC__)\n    #define MSHADOW_USE_F16C 0\n  #elif defined(__clang__) && \\\n        ((__clang_major__ < 8) || ((__clang_major__ == 8) && (__clang_minor__ < 1)))\n    #define MSHADOW_USE_F16C 0\n  #else\n    #define MSHADOW_USE_F16C 1\n  #endif\n#endif\n\n/*! \\brief whether use NVML to get dynamic info */\n#ifndef MSHADOW_USE_NVML\n  #define MSHADOW_USE_NVML 0\n#endif\n// SSE is conflict with cudacc\n#ifdef __CUDACC__\n  #undef MSHADOW_USE_SSE\n  #define MSHADOW_USE_SSE 0\n#endif\n\n#if MSHADOW_USE_CBLAS\nextern \"C\" {\n    #include <cblas.h>\n}\n#elif MSHADOW_USE_MKL\n  #if MSHADOW_INT64_TENSOR_SIZE == 1\n    // Define MKL_INT here to use exactly the same 64bits integer type definitions.\n    // If MKL_INT will not be defined here, the mkl header defines it as long long int.\n    #define MKL_INT int64_t\n    #define MKL_UINT uint64_t\n  #endif\n  #include <mkl_blas.h>\n  #include <mkl_cblas.h>\n  #include <mkl_vsl.h>\n  #include <mkl_vsl_functions.h>\n  #include <mkl_version.h>\n#endif\n\n#if MSHADOW_USE_CUDA\n  #include <cuda.h>\n  #include <cublas_v2.h>\n  #include <curand.h>\n#endif\n\n#if MSHADOW_USE_CUDNN == 1\n  #include <cudnn.h>\n#endif\n\n#if MSHADOW_USE_CUTENSOR == 1\n  #include <cutensor.h>\n#endif\n\n#if MSHADOW_USE_CUSOLVER == 1\n  #include <cusolverDn.h>\n#endif\n\n#if MSHADOW_USE_NVML\n  #include <nvml.h>\n#endif\n\n// --------------------------------\n// MSHADOW_XINLINE is used for inlining template code for both CUDA and CPU code\n#ifdef MSHADOW_XINLINE\n  #error \"MSHADOW_XINLINE must not be defined\"\n#endif\n#ifdef _MSC_VER\n#define MSHADOW_FORCE_INLINE __forceinline\n#pragma warning(disable : 4068)\n#else\n#define MSHADOW_FORCE_INLINE inline __attribute__((always_inline))\n#endif\n#ifdef __CUDACC__\n  #define MSHADOW_XINLINE MSHADOW_FORCE_INLINE __device__ __host__\n#else\n  #define MSHADOW_XINLINE MSHADOW_FORCE_INLINE\n#endif\n/*! \\brief cpu force inline */\n#define MSHADOW_CINLINE MSHADOW_FORCE_INLINE\n\n/*!\n * \\brief default data type for tensor string\n *  in code release, change it to default_real_t\n *  during development, change it to empty string so that missing\n *  template arguments can be detected\n */\n#ifndef MSHADOW_DEFAULT_DTYPE\n#define MSHADOW_DEFAULT_DTYPE = ::mshadow::default_real_t\n#endif\n\n/*!\n * \\brief DMLC marco for logging\n */\n#ifndef MSHADOW_USE_GLOG\n#define MSHADOW_USE_GLOG DMLC_USE_GLOG\n#endif  // MSHADOW_USE_GLOG\n\n#define MSHADOW_THROW_EXCEPTION noexcept(false)\n#define MSHADOW_NO_EXCEPTION  noexcept(true)\n\n#if defined(_MSC_VER)\n#define MSHADOW_ALIGNED(x) __declspec(align(x))\n#else\n#define MSHADOW_ALIGNED(x) __attribute__ ((aligned(x)))\n#endif\n\n/*!\n * \\brief Protected cuda call in mshadow\n * \\param func Expression to call.\n * It checks for CUDA errors after invocation of the expression.\n */\n#define MSHADOW_CUDA_CALL(func)                                    \\\n  {                                                                \\\n    cudaError_t e = (func);                                        \\\n    if (e == cudaErrorCudartUnloading) {                           \\\n      throw dmlc::Error(cudaGetErrorString(e));                    \\\n    }                                                              \\\n    CHECK_EQ(e, cudaSuccess)                                       \\\n        << \"CUDA: \" << cudaGetErrorString(e);                      \\\n  }\n\n/*!\n * \\brief Run function and catch error, log unknown error.\n * \\param func Expression to call.\n */\n#define MSHADOW_CATCH_ERROR(func)                                     \\\n  {                                                                   \\\n    try {                                                             \\\n      (func);                                                         \\\n    } catch (const dmlc::Error &e) {                                    \\\n      std::string what = e.what();                                      \\\n      if (what.find(\"driver shutting down\") == std::string::npos) {     \\\n        LOG(ERROR) << \"Ignore CUDA Error \" << what;                     \\\n      }                                                                 \\\n    }                                                                   \\\n  }\n\n#include \"./half.h\"\n#include \"./bfloat.h\"\n#define MSHADOW_HALF_BF_OPERATOR(RTYPE, OP)                                               \\\n  MSHADOW_XINLINE RTYPE operator OP(mshadow::half::half_t a, mshadow::bfloat::bf16_t b) { \\\n    return float(a) OP float(b); /* NOLINT(*) */                                          \\\n  }                                                                                       \\\n  MSHADOW_XINLINE RTYPE operator OP(mshadow::bfloat::bf16_t a, mshadow::half::half_t b) { \\\n    return float(a) OP float(b); /* NOLINT(*) */                                          \\\n  }\n\n/*! \\brief overloaded + operator between half_t and bf16_t */\nMSHADOW_HALF_BF_OPERATOR(float, +)\n/*! \\brief overloaded - operator between half_t and bf16_t */\nMSHADOW_HALF_BF_OPERATOR(float, -)\n/*! \\brief overloaded * operator between half_t and bf16_t */\nMSHADOW_HALF_BF_OPERATOR(float, *)\n/*! \\brief overloaded / operator between half_t and bf16_t */\nMSHADOW_HALF_BF_OPERATOR(float, /)\n/*! \\brief overloaded > operator between half_t and bf16_t */\nMSHADOW_HALF_BF_OPERATOR(bool, >)\n/*! \\brief overloaded < operator between half_t and bf16_t */\nMSHADOW_HALF_BF_OPERATOR(bool, <)\n/*! \\brief overloaded >= operator between half_t and bf16_t */\nMSHADOW_HALF_BF_OPERATOR(bool, >=)\n/*! \\brief overloaded <= operator between half_t and bf16_t */\nMSHADOW_HALF_BF_OPERATOR(bool, <=)\n\n#include \"dmlc/logging.h\"\n/*! \\brief namespace for mshadow */\nnamespace mshadow {\n/*! \\brief buffer size for each random number generator */\nconst unsigned kRandBufferSize = 1000000;\n/*! \\brief pi  */\nconst float kPi = 3.1415926f;\n/*! \\brief type that will be used for index */\n#if MSHADOW_INT64_TENSOR_SIZE == 1\n  typedef int64_t index_t;\n#else\n  typedef int32_t index_t;\n#endif\n\n#ifdef _WIN32\n  /*! \\brief openmp index for windows */\n  typedef int64_t openmp_index_t;\n#else\n  /*! \\brief openmp index for linux */\n  typedef index_t openmp_index_t;\n#endif\n\n\n#if (MSHADOW_USE_MKL && MXNET_USE_LAPACK) || MXNET_USE_ILP64_LAPACKE\n  // lapack_index_t could be replaced by index_t and removed when all blas library support large tensor\n  typedef index_t lapack_index_t;\n#else\n  typedef int lapack_index_t;\n#endif\n\n/*! \\brief float point type that will be used in default by mshadow */\ntypedef float default_real_t;\n\n/*! \\brief data type flag */\nenum TypeFlag {\n  kFloat32 = 0,\n  kFloat64 = 1,\n  kFloat16 = 2,\n  kUint8 = 3,\n  kInt32 = 4,\n  kInt8  = 5,\n  kInt64 = 6,\n  kBool = 7,\n  kInt16 = 8,\n  kUint16 = 9,\n  kUint32 = 10,\n  kUint64 = 11,\n  kBfloat16 = 12\n};\n\ntemplate<typename DType>\nstruct DataType;\n\ntemplate<>\nstruct DataType<float> {\n  static const int kFlag = kFloat32;\n  static const int kLanes = 1;\n#if MSHADOW_USE_CUDA\n#if (CUDA_VERSION >= 8000)\n  static const cudaDataType_t kCudaFlag = CUDA_R_32F;\n#endif\n#if MSHADOW_USE_CUDNN\n  static const cudnnDataType_t kCudnnFlag = CUDNN_DATA_FLOAT;\n  typedef float ScaleType;\n#endif\n#endif\n};\ntemplate<>\nstruct DataType<double> {\n  static const int kFlag = kFloat64;\n  static const int kLanes = 1;\n#if MSHADOW_USE_CUDA\n#if (CUDA_VERSION >= 8000)\n  static const cudaDataType_t kCudaFlag = CUDA_R_64F;\n#endif\n#if MSHADOW_USE_CUDNN\n  static const cudnnDataType_t kCudnnFlag = CUDNN_DATA_DOUBLE;\n  typedef double ScaleType;\n#endif\n#endif\n};\ntemplate<>\nstruct DataType<half::half_t> {\n  static const int kFlag = kFloat16;\n  static const int kLanes = 1;\n#if MSHADOW_USE_CUDA\n#if (CUDA_VERSION >= 8000)\n  static const cudaDataType_t kCudaFlag = CUDA_R_16F;\n#endif\n#if MSHADOW_USE_CUDNN\n  static const cudnnDataType_t kCudnnFlag = CUDNN_DATA_HALF;\n  typedef float ScaleType;\n#endif\n#endif\n};\ntemplate <>\nstruct DataType<bfloat::bf16_t> {\n  static const int kFlag = kBfloat16;\n  static const int kLanes = 1;\n};\ntemplate<>\nstruct DataType<uint8_t> {\n  static const int kFlag = kUint8;\n  static const int kLanes = 1;\n#if MSHADOW_USE_CUDA\n#if (CUDA_VERSION >= 8000)\n  static const cudaDataType_t kCudaFlag = CUDA_R_8U;\n#endif\n#if (MSHADOW_USE_CUDNN == 1 && CUDNN_MAJOR >= 6)\n  // no uint8 in cudnn for now\n  static const cudnnDataType_t kCudnnFlag = CUDNN_DATA_INT8;\n  typedef uint8_t ScaleType;\n#endif\n#endif\n};\ntemplate<>\nstruct DataType<int8_t> {\n  static const int kFlag = kInt8;\n  static const int kLanes = 1;\n#if MSHADOW_USE_CUDA\n#if (CUDA_VERSION >= 8000)\n  static const cudaDataType_t kCudaFlag = CUDA_R_8I;\n#endif\n#if (MSHADOW_USE_CUDNN == 1 && CUDNN_MAJOR >= 6)\n  static const cudnnDataType_t kCudnnFlag = CUDNN_DATA_INT8;\n  typedef int8_t ScaleType;\n#endif\n#endif\n};\ntemplate<>\nstruct DataType<int32_t> {\n  static const int kFlag = kInt32;\n  static const int kLanes = 1;\n#if MSHADOW_USE_CUDA\n#if (CUDA_VERSION >= 8000)\n  static const cudaDataType_t kCudaFlag = CUDA_R_32I;\n#endif\n#if (MSHADOW_USE_CUDNN == 1 && CUDNN_MAJOR >= 6)\n  static const cudnnDataType_t kCudnnFlag = CUDNN_DATA_INT32;\n  typedef int32_t ScaleType;\n#endif\n#endif\n};\ntemplate<>\nstruct DataType<int64_t> {\n  static const int kFlag = kInt64;\n  static const int kLanes = 1;\n};\ntemplate<>\nstruct DataType<bool> {\n  static const int kFlag = kBool;\n  static const int kLanes = 1;\n};\ntemplate<>\nstruct DataType<int16_t> {\n  static const int kFlag = kInt16;\n  static const int kLanes = 1;\n};\ntemplate<>\nstruct DataType<uint16_t> {\n  static const int kFlag = kUint16;\n  static const int kLanes = 1;\n};\ntemplate<>\nstruct DataType<uint32_t> {\n  static const int kFlag = kUint32;\n  static const int kLanes = 1;\n};\ntemplate<>\nstruct DataType<uint64_t> {\n  static const int kFlag = kUint64;\n  static const int kLanes = 1;\n};\n\n/*! \\brief type enum value for default real type */\nconst int default_type_flag = DataType<default_real_t>::kFlag;\n\n/*! \\brief TypeFlag value for type of indexes */\nconst int index_type_flag = DataType<lapack_index_t>::kFlag;\n\n/*! layout flag */\nenum LayoutFlag {\n  kUNKNOWN = -1,\n\n  kNCHW = 0,\n  kNHWC,\n  kCHWN,\n\n  kNCW = 1 << 3,\n  kNWC,\n  kCWN,\n\n  kNCDHW = 1 << 5,\n  kNDHWC,\n  kCDHWN\n};\n\ninline LayoutFlag layoutFlag(std::string layoutstr) {\n  switch (layoutstr.length()) {\n    case 4:\n      if (layoutstr == \"NHWC\")\n        return kNHWC;\n      if (layoutstr == \"NCHW\")\n        return kNCHW;\n      if (layoutstr == \"CHWN\")\n        return kCHWN;\n      return kUNKNOWN;\n    case 3:\n      if (layoutstr == \"NWC\")\n        return kNWC;\n      if (layoutstr == \"NCW\")\n        return kNCW;\n      if (layoutstr == \"CWN\")\n        return kCWN;\n      return kUNKNOWN;\n    case 5:\n      if (layoutstr == \"NDHWC\")\n        return kNDHWC;\n      if (layoutstr == \"NCDHW\")\n        return kNCDHW;\n      if (layoutstr == \"CDHWN\")\n        return kCDHWN;\n      return kUNKNOWN;\n    default:\n      return kUNKNOWN;\n  }\n}\n\ninline std::string toString(LayoutFlag layout) {\n  switch (layout) {\n    case kUNKNOWN:\n      return \"\";\n    case kNCHW:\n      return \"NCHW\";\n    case kNHWC:\n      return \"NHWC\";\n    case kCHWN:\n      return \"CHWN\";\n    case kNCW:\n      return \"NCW\";\n    case kNWC:\n      return \"NWC\";\n    case kCWN:\n      return \"CWN\";\n    case kNCDHW:\n      return \"NCDHW\";\n    case kNDHWC:\n      return \"NDHWC\";\n    case kCDHWN:\n      return \"CDHWN\";\n    default:\n      return \"\";\n  }\n}\n\ntemplate<int layout>\nstruct LayoutType;\n\ntemplate<>\nstruct LayoutType<kNCHW> {\n  static const index_t kNdim = 4;\n#if (MSHADOW_USE_CUDA && MSHADOW_USE_CUDNN == 1 && CUDNN_MAJOR >= 4)\n  static const cudnnTensorFormat_t kCudnnFlag = CUDNN_TENSOR_NCHW;\n#else\n  static const int kCudnnFlag = -1;\n#endif\n};\n\ntemplate<>\nstruct LayoutType<kNHWC> {\n  static const index_t kNdim = 4;\n#if (MSHADOW_USE_CUDA && MSHADOW_USE_CUDNN == 1 && CUDNN_MAJOR >= 4)\n  static const cudnnTensorFormat_t kCudnnFlag = CUDNN_TENSOR_NHWC;\n#else\n  static const int kCudnnFlag = -1;\n#endif\n};\n\n/*! \\brief default layout for 4d tensor */\nconst int default_layout = kNCHW;\n\ntemplate<>\nstruct LayoutType<kNCDHW> {\n  static const index_t kNdim = 5;\n#if (MSHADOW_USE_CUDA && MSHADOW_USE_CUDNN == 1 && CUDNN_MAJOR >= 4)\n  static const cudnnTensorFormat_t kCudnnFlag = CUDNN_TENSOR_NCHW;\n#else\n  static const int kCudnnFlag = -1;\n#endif\n};\n\ntemplate<>\nstruct LayoutType<kNDHWC> {\n  static const index_t kNdim = 5;\n#if (MSHADOW_USE_CUDA && MSHADOW_USE_CUDNN == 1 && CUDNN_MAJOR >= 4)\n  static const cudnnTensorFormat_t kCudnnFlag = CUDNN_TENSOR_NHWC;\n#else\n  static const int kCudnnFlag = -1;\n#endif\n};\n\n/*! \\brief default layout for 5d tensor */\nconst int default_layout_5d = kNCDHW;\n\n/*! \\brief namespace for operators */\nnamespace op {\n// binary operator\n/*! \\brief mul operator */\nstruct mul{\n  /*! \\brief map a, b to result using defined operation */\n  template<typename DType>\n  MSHADOW_XINLINE static DType Map(DType a, DType b) {\n    return a * b;\n  }\n};\n/*! \\brief plus operator */\nstruct plus {\n  /*! \\brief map a, b to result using defined operation */\n  template<typename DType>\n  MSHADOW_XINLINE static DType Map(DType a, DType b) {\n    return a + b;\n  }\n};\n/*! \\brief minus operator */\nstruct minus {\n  /*! \\brief map a, b to result using defined operation */\n  template<typename DType>\n  MSHADOW_XINLINE static DType Map(DType a, DType b) {\n    return a - b;\n  }\n};\n/*! \\brief divide operator */\nstruct div {\n  /*! \\brief map a, b to result using defined operation */\n  template<typename DType>\n  MSHADOW_XINLINE static DType Map(DType a, DType b) {\n    return a / b;\n  }\n};\n/*! \\brief get rhs */\nstruct right {\n  /*! \\brief map a, b to result using defined operation */\n  template<typename DType>\n  MSHADOW_XINLINE static DType Map(DType a, DType b) {\n    return b;\n  }\n};\n// unary operator/ function: example\n// these operators can be defined by user,\n// in the same style as binary and unary operator\n// to use, simply write F<op::identity>( src )\n/*! \\brief identity function that maps a real number to it self */\nstruct identity{\n  /*! \\brief map a to result using defined operation */\n  template<typename DType>\n  MSHADOW_XINLINE static DType Map(DType a) {\n    return a;\n  }\n};\n}  // namespace op\n/*! \\brief namespace for savers */\nnamespace sv {\n/*! \\brief save to saver: = */\nstruct saveto {\n  /*! \\brief save b to a using save method */\n  template<typename DType>\n  MSHADOW_XINLINE static void Save(DType &a, DType b) { // NOLINT(*)\n    a = b;\n  }\n  /*! \\brief helper constant to use BLAS, alpha */\n  inline static default_real_t AlphaBLAS(void) { return 1.0f; }\n  /*! \\brief helper constant to use BLAS, beta */\n  inline static default_real_t BetaBLAS(void) { return 0.0f; }\n  /*! \\brief corresponding binary operator type */\n  typedef op::right OPType;\n};\n/*! \\brief save to saver: += */\nstruct plusto {\n  /*! \\brief save b to a using save method */\n  template<typename DType>\n  MSHADOW_XINLINE static void Save(DType &a, DType b) { // NOLINT(*)\n    a += b;\n  }\n  /*! \\brief helper constant to use BLAS, alpha */\n  inline static default_real_t AlphaBLAS(void) { return 1.0f; }\n  /*! \\brief helper constant to use BLAS, beta */\n  inline static default_real_t BetaBLAS(void) { return 1.0f; }\n  /*! \\brief corresponding binary operator type */\n  typedef op::plus OPType;\n};\n/*! \\brief minus to saver: -= */\nstruct minusto {\n  /*! \\brief save b to a using save method */\n  template<typename DType>\n  MSHADOW_XINLINE static void Save(DType &a, DType b) { // NOLINT(*)\n    a -= b;\n  }\n  /*! \\brief helper constant to use BLAS, alpha */\n  inline static default_real_t AlphaBLAS(void) { return -1.0f; }\n  /*! \\brief helper constant to use BLAS, beta */\n  inline static default_real_t BetaBLAS(void) { return 1.0f; }\n  /*! \\brief corresponding binary operator type */\n  typedef op::minus OPType;\n};\n/*! \\brief multiply to saver: *= */\nstruct multo {\n  /*! \\brief save b to a using save method */\n  template<typename DType>\n  MSHADOW_XINLINE static void Save(DType &a, DType b) { // NOLINT(*)\n    a *= b;\n  }\n  /*! \\brief corresponding binary operator type */\n  typedef op::mul OPType;\n};\n/*! \\brief divide to saver: /= */\nstruct divto {\n  /*! \\brief save b to a using save method */\n  template<typename DType>\n  MSHADOW_XINLINE static void Save(DType& a, DType b) { // NOLINT(*)\n    a /= b;\n  }\n  /*! \\brief corresponding binary operator type */\n  typedef op::div OPType;\n};\n}  // namespace sv\n\n#ifndef __CUDA_ARCH__\nusing std::isnan;\nusing std::isinf;\n#endif\n\n/*! \\brief\n *  determines if the given floating point\n *  number is not a number */\nnamespace isnan_typed {\n  template<typename DType>\n  MSHADOW_XINLINE bool IsNan(volatile DType val) {\n    return false;\n  }\n  template<>\n  MSHADOW_XINLINE bool IsNan(volatile float val) {\n    return isnan(val);\n  }\n  template<>\n  MSHADOW_XINLINE bool IsNan(volatile double val) {\n    return isnan(val);\n  }\n  template<>\n  MSHADOW_XINLINE bool IsNan(volatile long double val) {\n    return isnan(val);\n  }\n  template<>\n  MSHADOW_XINLINE bool IsNan(volatile mshadow::half::half_t val) {\n    return (val.half_ & (~MSHADOW_HALF_SIGN_BIT)) > MSHADOW_HALF_EXPONENT_BITS;\n  }\n  template <>\n  MSHADOW_XINLINE bool IsNan(volatile mshadow::bfloat::bf16_t val) {\n    return (val.bf16_ & (~MSHADOW_BF16_SIGN_BIT)) > MSHADOW_BF16_EXPONENT_BITS;\n  }\n}  // namespace isnan_typed\n\n/*! \\brief\n *  determines if the given floating point\n *  number is a positive or negative infinity */\nnamespace isinf_typed {\n  template<typename DType>\n  MSHADOW_XINLINE bool IsInf(volatile DType val) {\n    return false;\n  }\n  template<>\n  MSHADOW_XINLINE bool IsInf(volatile float val) {\n    return isinf(val);\n  }\n  template<>\n  MSHADOW_XINLINE bool IsInf(volatile double val) {\n    return isinf(val);\n  }\n  template<>\n  MSHADOW_XINLINE bool IsInf(volatile long double val) {\n    return isinf(val);\n  }\n  template<>\n  MSHADOW_XINLINE bool IsInf(volatile mshadow::half::half_t val) {\n    return (val.half_ & (~MSHADOW_HALF_SIGN_BIT)) == MSHADOW_HALF_EXPONENT_BITS;\n  }\n  template <>\n  MSHADOW_XINLINE bool IsInf(volatile mshadow::bfloat::bf16_t val) {\n    return (val.bf16_ & (~MSHADOW_BF16_SIGN_BIT)) == MSHADOW_BF16_EXPONENT_BITS;\n  }\n}  // namespace isinf_typed\n\n/*! \\brief namespace for potential reducer operations */\nnamespace red {\nnamespace limits {\n/*!\n * \\brief minimum value of certain types\n * \\tparam DType data type\n */\ntemplate<typename DType>\nMSHADOW_XINLINE DType MinValue(void);\n/*! \\brief minimum value of float */\ntemplate<>\nMSHADOW_XINLINE float MinValue<float>(void) {\n  return -FLT_MAX;\n}\n/*! \\brief minimum value of double */\ntemplate<>\nMSHADOW_XINLINE double MinValue<double>(void) {\n  return -DBL_MAX;\n}\n/*! \\brief minimum value of half */\ntemplate<>\nMSHADOW_XINLINE half::half_t MinValue<half::half_t>(void) {\n  return MSHADOW_HALF_MIN;\n}\n/*! \\brief minimum value of bf16 */\ntemplate<>\nMSHADOW_XINLINE bfloat::bf16_t MinValue<bfloat::bf16_t>(void) {\n  return MSHADOW_BF16_MIN;\n}\n/*! \\brief minimum value of uint8_t */\ntemplate<>\nMSHADOW_XINLINE uint8_t MinValue<uint8_t>(void) {\n  return 0;\n}\n/*! \\brief minimum value of int8_t */\ntemplate<>\nMSHADOW_XINLINE int8_t MinValue<int8_t>(void) {\n  return SCHAR_MIN;\n}\n/*! \\brief minimum value of int32_t */\ntemplate<>\nMSHADOW_XINLINE int MinValue<int32_t>(void) {\n  return INT_MIN;\n}\n/*! \\brief minimum value of int64_t */\ntemplate<>\nMSHADOW_XINLINE int64_t MinValue<int64_t>(void) {\n  return LLONG_MIN;\n}\n/*! \\brief minimum value of bool */\ntemplate<>\nMSHADOW_XINLINE bool MinValue<bool>(void) {\n  return false;\n}\n/*! \\brief minimum value of unsigned int */\ntemplate<>\nMSHADOW_XINLINE unsigned int MinValue<unsigned int>(void) {\n  return 0;\n}\n\n/*!\n * \\brief negative infinity of certain types\n * \\tparam DType data type\n */\ntemplate<typename DType>\nMSHADOW_XINLINE DType NegInfValue(void) {\n  return MinValue<DType>();\n}\n/*! \\brief negative infinity value of float */\ntemplate<>\nMSHADOW_XINLINE float NegInfValue<float>(void) {\n  return -HUGE_VALF;\n}\n/*! \\brief negative infinity value of double */\ntemplate<>\nMSHADOW_XINLINE double NegInfValue<double>(void) {\n  return -HUGE_VAL;\n}\n/*! \\brief negative infinity value of float16 */\ntemplate<>\nMSHADOW_XINLINE half::half_t NegInfValue<half::half_t>(void) {\n  return half::half_t::Binary(\n      MSHADOW_HALF_SIGN_BIT | MSHADOW_HALF_EXPONENT_BITS);\n}\n/*! \\brief negative infinity value of bfloat16 */\ntemplate <>\nMSHADOW_XINLINE bfloat::bf16_t NegInfValue<bfloat::bf16_t>(void) {\n  return bfloat::bf16_t::Binary(MSHADOW_BF16_SIGN_BIT | MSHADOW_BF16_EXPONENT_BITS);\n}\n\n/*!\n * \\brief maximum value of certain types\n * \\tparam DType data type\n */\ntemplate<typename DType>\nMSHADOW_XINLINE DType MaxValue(void);\n/*! \\brief maximum value of float */\ntemplate<>\nMSHADOW_XINLINE float MaxValue<float>(void) {\n  return FLT_MAX;\n}\n/*! \\brief maximum value of double */\ntemplate<>\nMSHADOW_XINLINE double MaxValue<double>(void) {\n  return DBL_MAX;\n}\n/*! \\brief maximum value of half */\ntemplate<>\nMSHADOW_XINLINE half::half_t MaxValue<half::half_t>(void) {\n  return MSHADOW_HALF_MAX;\n}\n/*! \\brief maximum value of bf16 */\ntemplate<>\nMSHADOW_XINLINE bfloat::bf16_t MaxValue<bfloat::bf16_t>(void) {\n  return MSHADOW_BF16_MAX;\n}\n/*! \\brief maximum value of uint8_t */\ntemplate<>\nMSHADOW_XINLINE uint8_t MaxValue<uint8_t>(void) {\n  return UCHAR_MAX;\n}\n/*! \\brief maximum value of int8_t */\ntemplate<>\nMSHADOW_XINLINE int8_t MaxValue<int8_t>(void) {\n  return SCHAR_MAX;\n}\n/*! \\brief maximum value of int32_t */\ntemplate<>\nMSHADOW_XINLINE int MaxValue<int32_t>(void) {\n  return INT_MAX;\n}\n/*! \\brief maximum value of int64_t */\ntemplate<>\nMSHADOW_XINLINE int64_t MaxValue<int64_t>(void) {\n  return LLONG_MAX;\n}\n/*! \\brief maximum value of bool */\ntemplate<>\nMSHADOW_XINLINE bool MaxValue<bool>(void) {\n  return true;\n}\n/*! \\brief maximum value of uint32_t */\ntemplate<>\nMSHADOW_XINLINE uint32_t MaxValue<uint32_t>(void) {\n  return std::numeric_limits<uint32_t>::max();\n}\n\n/*!\n * \\brief positive infinity of certain types\n * \\tparam DType data type\n */\ntemplate<typename DType>\nMSHADOW_XINLINE DType PosInfValue(void) {\n  return MaxValue<DType>();\n}\n/*! \\brief positive infinity value of float */\ntemplate<>\nMSHADOW_XINLINE float PosInfValue<float>(void) {\n  return HUGE_VALF;\n}\n/*! \\brief positive infinity value of double */\ntemplate<>\nMSHADOW_XINLINE double PosInfValue<double>(void) {\n  return HUGE_VAL;\n}\n/*! \\brief positive infinity value of float16 */\ntemplate<>\nMSHADOW_XINLINE half::half_t PosInfValue<half::half_t>(void) {\n  return half::half_t::Binary(MSHADOW_HALF_EXPONENT_BITS);\n}\n/*! \\brief positive infinity value of bfloat16 */\ntemplate <>\nMSHADOW_XINLINE bfloat::bf16_t PosInfValue<bfloat::bf16_t>(void) {\n  return bfloat::bf16_t::Binary(MSHADOW_BF16_EXPONENT_BITS);\n}\n\n}  // namespace limits\n\n/*! \\brief sum reducer */\nstruct sum {\n  /*! \\brief do reduction into dst */\n  template<typename DType>\n  MSHADOW_XINLINE static void Reduce(volatile DType& dst,  volatile DType src) { // NOLINT(*)\n    dst += src;\n  }\n  /*! \\brief do stable reduction into dst */\n  template<typename DType>\n  MSHADOW_XINLINE static void Reduce(volatile DType& dst,  volatile DType src, volatile DType& residual) { // NOLINT(*)\n    DType y = src - residual;\n    DType t = dst + y;\n    if (isinf_typed::IsInf(t)) {\n      residual = 0;\n    } else {\n      residual = (t - dst) - y;\n    }\n    dst = t;\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  MSHADOW_XINLINE static void Merge(volatile DType& dst_val, volatile DType& src_val) { // NOLINT(*)\n    Reduce(dst_val, src_val);\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  MSHADOW_XINLINE static void Merge(volatile DType& dst_val, volatile DType& dst_residual, volatile DType& src_val, volatile DType& src_residual) { // NOLINT(*)\n    DType t1 = dst_val + src_val;\n    if (isinf_typed::IsInf(t1)) {\n      dst_val = t1;\n      dst_residual = 0;\n    } else {\n      DType e = t1 - dst_val;\n      DType t2 = ((src_val - e) + (dst_val - (t1 - e))) + dst_residual + src_residual;\n      dst_val = t1 + t2;\n      dst_residual = t2 - (dst_val - t1);\n    }\n  }\n  /*! \\brief finalize reduction */\n  template<typename DType>\n  MSHADOW_XINLINE static void Finalize(volatile DType& dst) {} // NOLINT(*)\n  /*! \\brief finalize reduction */\n  template<typename DType>\n  MSHADOW_XINLINE static void Finalize(volatile DType& dst, volatile DType& residual) {} // NOLINT(*)\n  /*!\n   *\\brief calculate gradient of redres with respect to redsrc,\n   * redres: reduced result, redsrc: one of reduction element\n   */\n  template<typename DType>\n  MSHADOW_XINLINE static DType PartialGrad(DType redres, DType redsrc) {\n    return 1;\n  }\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  MSHADOW_XINLINE static void SetInitValue(DType &initv) { // NOLINT(*)\n    initv = 0;\n  }\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  MSHADOW_XINLINE static void SetInitValue(DType &initv, DType &residual) { // NOLINT(*)\n    SetInitValue(initv);\n    residual = 0;\n  }\n};\n/*! \\brief maximum reducer */\nstruct maximum {\n  /*! \\brief do reduction into dst */\n  template<typename DType>\n  MSHADOW_XINLINE static void Reduce(volatile DType& dst,  volatile DType src) { // NOLINT(*)\n    if (!isnan_typed::IsNan(dst)) {\n      if (!(dst >= src)) dst = src;\n    }\n  }\n  /*! \\brief do reduction into dst */\n  template<typename DType>\n  MSHADOW_XINLINE static void Reduce(volatile DType& dst,  volatile DType src, volatile DType &none) { // NOLINT(*)\n    Reduce(dst, src);\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  MSHADOW_XINLINE static void Merge(volatile DType& dst_val, volatile DType& src_val) { // NOLINT(*)\n    Reduce(dst_val, src_val);\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  MSHADOW_XINLINE static void Merge(volatile DType& dst_val, volatile DType& dst_residual, volatile DType& src_val, volatile DType& src_residual) { // NOLINT(*)\n    Reduce(dst_val, src_val);\n  }\n  /*! \\brief finalize reduction */\n  template<typename DType>\n  MSHADOW_XINLINE static void Finalize(volatile DType& dst) {} // NOLINT(*)\n  /*! \\brief finalize reduction */\n  template<typename DType>\n  MSHADOW_XINLINE static void Finalize(volatile DType& dst, volatile DType& residual) {} // NOLINT(*)\n  /*!\n   * \\brief calculate gradient of redres with respect to redsrc,\n   * redres: reduced result, redsrc: one of reduction element\n   */\n  template<typename DType>\n  MSHADOW_XINLINE static DType PartialGrad(DType redres, DType redsrc) {\n    return redres == redsrc ? 1: 0;\n  }\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  MSHADOW_XINLINE static void SetInitValue(DType &initv) { // NOLINT(*)\n    initv = limits::NegInfValue<DType>();\n  }\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  MSHADOW_XINLINE static void SetInitValue(DType &initv, DType &none) { // NOLINT(*)\n    SetInitValue(initv);\n  }\n};\n/*! \\brief minimum reducer */\nstruct minimum {\n  /*! \\brief do reduction into dst */\n  template<typename DType>\n  MSHADOW_XINLINE static void Reduce(volatile DType& dst,  volatile DType src) { // NOLINT(*)\n    if (!isnan_typed::IsNan(dst)) {\n      if (!(dst <= src)) dst = src;\n    }\n  }\n  /*! \\brief do reduction into dst */\n  template<typename DType>\n  MSHADOW_XINLINE static void Reduce(volatile DType& dst,  volatile DType src, volatile DType &none) { // NOLINT(*)\n    Reduce(dst, src);\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  MSHADOW_XINLINE static void Merge(volatile DType& dst_val, volatile DType& src_val) { // NOLINT(*)\n    Reduce(dst_val, src_val);\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  MSHADOW_XINLINE static void Merge(volatile DType& dst_val, volatile DType& dst_residual, volatile DType& src_val, volatile DType& src_residual) { // NOLINT(*)\n    Reduce(dst_val, src_val);\n  }\n  /*! \\brief finalize reduction */\n  template<typename DType>\n  MSHADOW_XINLINE static void Finalize(volatile DType& dst) {} // NOLINT(*)\n  /*! \\brief finalize reduction */\n  template<typename DType>\n  MSHADOW_XINLINE static void Finalize(volatile DType& dst, volatile DType& residual) {} // NOLINT(*)\n  /*!\n   * \\brief calculate gradient of redres with respect to redsrc,\n   * redres: reduced result, redsrc: one of reduction element\n   */\n  template<typename DType>\n  MSHADOW_XINLINE static DType PartialGrad(DType redres, DType redsrc) {\n    return redres == redsrc ? 1: 0;\n  }\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  MSHADOW_XINLINE static void SetInitValue(DType &initv) { // NOLINT(*)\n    initv = limits::PosInfValue<DType>();\n  }\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  MSHADOW_XINLINE static void SetInitValue(DType &initv, DType &none) { // NOLINT(*)\n    SetInitValue(initv);\n  }\n};\n}  // namespace red\n\n#ifndef __NVCC__\n#define MSHADOW_TYPE_SWITCH(type, DType, ...)       \\\n  switch (type) {                                   \\\n  case mshadow::kFloat32:                           \\\n    {                                               \\\n      typedef float DType;                          \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kFloat64:                           \\\n    {                                               \\\n      typedef double DType;                         \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kFloat16:                           \\\n    {                                               \\\n      typedef mshadow::half::half_t DType;          \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kBfloat16:                          \\\n    {                                               \\\n      typedef mshadow::bfloat::bf16_t DType;        \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kUint8:                             \\\n    {                                               \\\n      typedef uint8_t DType;                        \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kInt8:                              \\\n    {                                               \\\n      typedef int8_t DType;                         \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kInt32:                             \\\n    {                                               \\\n      typedef int32_t DType;                        \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kInt64:                             \\\n    {                                               \\\n      typedef int64_t DType;                        \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kBool:                              \\\n    LOG(FATAL) << \"This operation does not \"        \\\n                  \"support bool type\";              \\\n    break;                                          \\\n  case mshadow::kInt16:                             \\\n    LOG(FATAL) << \"This operation does not \"        \\\n                  \"support int16 type\";             \\\n    break;                                          \\\n  case mshadow::kUint16:                            \\\n    LOG(FATAL) << \"This operation does not \"        \\\n                  \"support uint16 type\";            \\\n    break;                                          \\\n  case mshadow::kUint32:                            \\\n    LOG(FATAL) << \"This operation does not \"        \\\n                  \"support uint32 type\";            \\\n    break;                                          \\\n  case mshadow::kUint64:                            \\\n    LOG(FATAL) << \"This operation does not \"        \\\n                  \"support uint64 type\";            \\\n    break;                                          \\\n  default:                                          \\\n    LOG(FATAL) << \"Unknown type enum \" << type;     \\\n  }\n#else\n#define MSHADOW_TYPE_SWITCH(type, DType, ...)       \\\n  switch (type) {                                   \\\n  case mshadow::kFloat32:                           \\\n    {                                               \\\n      typedef float DType;                          \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kFloat64:                           \\\n    {                                               \\\n      typedef double DType;                         \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kFloat16:                           \\\n    {                                               \\\n      typedef mshadow::half::half_t DType;          \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kUint8:                             \\\n    {                                               \\\n      typedef uint8_t DType;                        \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kInt8:                              \\\n    {                                               \\\n      typedef int8_t DType;                         \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kInt32:                             \\\n    {                                               \\\n      typedef int32_t DType;                        \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kInt64:                             \\\n    {                                               \\\n      typedef int64_t DType;                        \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kBool:                              \\\n    LOG(FATAL) << \"This operation does not \"        \\\n                  \"support bool type\";              \\\n    break;                                          \\\n  case mshadow::kInt16:                             \\\n    LOG(FATAL) << \"This operation does not \"        \\\n                  \"support int16 type\";             \\\n    break;                                          \\\n  case mshadow::kUint16:                            \\\n    LOG(FATAL) << \"This operation does not \"        \\\n                  \"support uint16 type\";            \\\n    break;                                          \\\n  case mshadow::kUint32:                            \\\n    LOG(FATAL) << \"This operation does not \"        \\\n                  \"support uint32 type\";            \\\n    break;                                          \\\n  case mshadow::kUint64:                            \\\n    LOG(FATAL) << \"This operation does not \"        \\\n                  \"support uint64 type\";            \\\n    break;                                          \\\n  default:                                          \\\n    LOG(FATAL) << \"Unknown type enum \" << type;     \\\n  }\n#endif\n\n#define MSHADOW_SGL_DBL_TYPE_SWITCH(type, DType, ...)  \\\n  switch (type) {                                      \\\n  case mshadow::kFloat32:                              \\\n    {                                                  \\\n      typedef float DType;                             \\\n      {__VA_ARGS__}                                    \\\n    }                                                  \\\n    break;                                             \\\n  case mshadow::kFloat64:                              \\\n    {                                                  \\\n      typedef double DType;                            \\\n      {__VA_ARGS__}                                    \\\n    }                                                  \\\n    break;                                             \\\n  default:                                             \\\n    LOG(FATAL) << \"This operation only supports \"      \\\n                  \"32-bit and 64-bit floating point\";  \\\n  }\n\n#define MSHADOW_REAL_TYPE_SWITCH(type, DType, ...)  \\\n  switch (type) {                                   \\\n  case mshadow::kFloat32:                           \\\n    {                                               \\\n      typedef float DType;                          \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kFloat64:                           \\\n    {                                               \\\n      typedef double DType;                         \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kFloat16:                           \\\n    {                                               \\\n      typedef mshadow::half::half_t DType;          \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kUint8:                             \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types not uint8\"; \\\n    break;                                          \\\n  case mshadow::kInt8:                              \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types not int8\";  \\\n    break;                                          \\\n  case mshadow::kInt32:                             \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types, not int32\";\\\n    break;                                          \\\n  case mshadow::kInt64:                             \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types, not int64\";\\\n    break;                                          \\\n  case mshadow::kBool:                              \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types, not bool\"; \\\n    break;                                          \\\n  case mshadow::kInt16:                             \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types, not int16\";\\\n    break;                                          \\\n  case mshadow::kUint16:                            \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types not uint16\";\\\n    break;                                          \\\n  case mshadow::kUint32:                            \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types not uint32\";\\\n    break;                                          \\\n  case mshadow::kUint64:                            \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types not uint64\";\\\n    break;                                          \\\n  default:                                          \\\n    LOG(FATAL) << \"Unknown type enum \" << type;     \\\n  }\n\n#ifndef __NVCC__\n#define MSHADOW_REAL_TYPE_SWITCH_EX(type$, DType$, DLargeType$, ...)  \\\n  switch (type$) {                                  \\\n  case mshadow::kFloat32:                           \\\n    {                                               \\\n      typedef float DType$;                         \\\n      typedef float DLargeType$;                    \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kFloat64:                           \\\n    {                                               \\\n      typedef double DType$;                        \\\n      typedef double DLargeType$;                   \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kFloat16:                           \\\n    {                                               \\\n      typedef mshadow::half::half_t DType$;         \\\n      typedef float DLargeType$;                    \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kBfloat16:                          \\\n    {                                               \\\n      typedef mshadow::bfloat::bf16_t DType$;       \\\n      typedef float DLargeType$;                    \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kUint8:                             \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types not uint8\"; \\\n    break;                                          \\\n  case mshadow::kInt8:                              \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types not int8\";  \\\n    break;                                          \\\n  case mshadow::kInt32:                             \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types, not int32\";\\\n    break;                                          \\\n  case mshadow::kInt64:                             \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types, not int64\";\\\n    break;                                          \\\n  case mshadow::kBool:                              \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types, not bool\"; \\\n    break;                                          \\\n  case mshadow::kInt16:                             \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types, not int16\";\\\n    break;                                          \\\n  case mshadow::kUint16:                            \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types not uint16\";\\\n    break;                                          \\\n  case mshadow::kUint32:                            \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types not uint32\";\\\n    break;                                          \\\n  case mshadow::kUint64:                            \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types not uint64\";\\\n    break;                                          \\\n  default:                                          \\\n    LOG(FATAL) << \"Unknown type enum \" << type$;    \\\n  }\n#else\n#define MSHADOW_REAL_TYPE_SWITCH_EX(type$, DType$, DLargeType$, ...)  \\\n  switch (type$) {                                  \\\n  case mshadow::kFloat32:                           \\\n    {                                               \\\n      typedef float DType$;                         \\\n      typedef float DLargeType$;                    \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kFloat64:                           \\\n    {                                               \\\n      typedef double DType$;                        \\\n      typedef double DLargeType$;                   \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kFloat16:                           \\\n    {                                               \\\n      typedef mshadow::half::half_t DType$;         \\\n      typedef float DLargeType$;                    \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kUint8:                             \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types not uint8\"; \\\n    break;                                          \\\n  case mshadow::kInt8:                              \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types not int8\";  \\\n    break;                                          \\\n  case mshadow::kInt32:                             \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types, not int32\";\\\n    break;                                          \\\n  case mshadow::kInt64:                             \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types, not int64\";\\\n    break;                                          \\\n  case mshadow::kBool:                              \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types, not bool\"; \\\n    break;                                          \\\n  case mshadow::kInt16:                             \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types, not int16\";\\\n    break;                                          \\\n  case mshadow::kUint16:                            \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types not uint16\";\\\n    break;                                          \\\n  case mshadow::kUint32:                            \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types not uint32\";\\\n    break;                                          \\\n  case mshadow::kUint64:                            \\\n    LOG(FATAL) << \"This operation only support \"    \\\n                  \"floating point types not uint64\";\\\n    break;                                          \\\n  default:                                          \\\n    LOG(FATAL) << \"Unknown type enum \" << type$;    \\\n  }\n#endif\n#define MSHADOW_LAYOUT_SWITCH(layout, Layout, ...)  \\\n  switch (layout) {                                 \\\n  case mshadow::kNCHW:                              \\\n    {                                               \\\n      const int Layout = kNCHW;                     \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kNHWC:                              \\\n    {                                               \\\n      const int Layout = kNHWC;                     \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kNCDHW:                             \\\n    {                                               \\\n      const int Layout = kNCDHW;                    \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  case mshadow::kNDHWC:                             \\\n    {                                               \\\n      const int Layout = kNDHWC;                    \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  default:                                          \\\n    LOG(FATAL) << \"Unknown layout enum \" << layout; \\\n  }\n\n/*!\n * \\brief Only supports int64 index type for aux_data\n * in NDArray class for now.\n */\n#define MSHADOW_IDX_TYPE_SWITCH(type, DType, ...)   \\\n  switch (type) {                                   \\\n  case mshadow::kInt64:                             \\\n    {                                               \\\n      typedef int64_t DType;                        \\\n      {__VA_ARGS__}                                 \\\n    }                                               \\\n    break;                                          \\\n  default:                                          \\\n    LOG(FATAL) << \"Unknown type enum \" << type;     \\\n  }\n\n#define MSHADOW_TYPE_SWITCH_WITH_BOOL(type, DType, ...)       \\\n  switch (type) {                                             \\\n  case mshadow::kFloat32:                                     \\\n    {                                                         \\\n      typedef float DType;                                    \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kFloat64:                                     \\\n    {                                                         \\\n      typedef double DType;                                   \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kFloat16:                                     \\\n    {                                                         \\\n      typedef mshadow::half::half_t DType;                    \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kBfloat16:                                    \\\n    {                                                         \\\n      typedef mshadow::bfloat::bf16_t DType;                  \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kUint8:                                       \\\n    {                                                         \\\n      typedef uint8_t DType;                                  \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kInt8:                                        \\\n    {                                                         \\\n      typedef int8_t DType;                                   \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kInt32:                                       \\\n    {                                                         \\\n      typedef int32_t DType;                                  \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kInt64:                                       \\\n    {                                                         \\\n      typedef int64_t DType;                                  \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kBool:                                        \\\n    {                                                         \\\n      typedef bool DType;                                     \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kInt16:                                       \\\n    LOG(FATAL) << \"This operation does not \"                  \\\n                  \"support int16 type\";                       \\\n    break;                                                    \\\n  case mshadow::kUint16:                                      \\\n    LOG(FATAL) << \"This operation does not \"                  \\\n                  \"support uint16 type\";                      \\\n    break;                                                    \\\n  case mshadow::kUint32:                                      \\\n    LOG(FATAL) << \"This operation does not \"                  \\\n                  \"support uint32 type\";                      \\\n    break;                                                    \\\n  case mshadow::kUint64:                                      \\\n    LOG(FATAL) << \"This operation does not \"                  \\\n                  \"support uint64 type\";                      \\\n    break;                                                    \\\n  default:                                                    \\\n    LOG(FATAL) << \"Unknown type enum \" << type;               \\\n  }\n\n#define MSHADOW_TYPE_SWITCH_EXT(type, DType, ...)             \\\n  switch (type) {                                             \\\n  case mshadow::kFloat32:                                     \\\n    {                                                         \\\n      typedef float DType;                                    \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kFloat64:                                     \\\n    {                                                         \\\n      typedef double DType;                                   \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kFloat16:                                     \\\n    {                                                         \\\n      typedef mshadow::half::half_t DType;                    \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kBfloat16:                                    \\\n    {                                                         \\\n      typedef mshadow::bfloat::bf16_t DType;                  \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kUint8:                                       \\\n    {                                                         \\\n      typedef uint8_t DType;                                  \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kInt8:                                        \\\n    {                                                         \\\n      typedef int8_t DType;                                   \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kInt32:                                       \\\n    {                                                         \\\n      typedef int32_t DType;                                  \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kInt64:                                       \\\n    {                                                         \\\n      typedef int64_t DType;                                  \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kInt16:                                       \\\n    {                                                         \\\n      typedef int16_t DType;                                  \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kUint16:                                      \\\n    {                                                         \\\n      typedef uint16_t DType;                                 \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kUint32:                                      \\\n    {                                                         \\\n      typedef uint32_t DType;                                 \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kUint64:                                      \\\n    {                                                         \\\n      typedef uint64_t DType;                                 \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  default:                                                    \\\n    LOG(FATAL) << \"Unknown type enum \" << type;               \\\n  }\n\n#define MSHADOW_TYPE_SWITCH_EXT_WITH_BOOL(type, DType, ...)   \\\n  switch (type) {                                             \\\n  case mshadow::kFloat32:                                     \\\n    {                                                         \\\n      typedef float DType;                                    \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kFloat64:                                     \\\n    {                                                         \\\n      typedef double DType;                                   \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kFloat16:                                     \\\n    {                                                         \\\n      typedef mshadow::half::half_t DType;                    \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kBfloat16:                                    \\\n    {                                                         \\\n      typedef mshadow::bfloat::bf16_t DType;                  \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kUint8:                                       \\\n    {                                                         \\\n      typedef uint8_t DType;                                  \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kInt8:                                        \\\n    {                                                         \\\n      typedef int8_t DType;                                   \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kInt32:                                       \\\n    {                                                         \\\n      typedef int32_t DType;                                  \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kInt64:                                       \\\n    {                                                         \\\n      typedef int64_t DType;                                  \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kBool:                                        \\\n    {                                                         \\\n      typedef bool DType;                                     \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kInt16:                                       \\\n    {                                                         \\\n      typedef int16_t DType;                                  \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kUint16:                                      \\\n    {                                                         \\\n      typedef uint16_t DType;                                 \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kUint32:                                      \\\n    {                                                         \\\n      typedef uint32_t DType;                                 \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  case mshadow::kUint64:                                      \\\n    {                                                         \\\n      typedef uint64_t DType;                                 \\\n      {__VA_ARGS__}                                           \\\n    }                                                         \\\n    break;                                                    \\\n  default:                                                    \\\n    LOG(FATAL) << \"Unknown type enum \" << type;               \\\n  }\n\n/*! \\brief get data type size from type enum */\ninline size_t mshadow_sizeof(int type) {\n  int size = 0;\n  MSHADOW_TYPE_SWITCH_EXT_WITH_BOOL(type, DType, size = sizeof(DType););\n  return size;\n}\n\n/*/ \\brief get string with the type name from type enum */\ninline std::string dtype_string(const int dtype) {\n  switch (dtype) {\n    case mshadow::kFloat32:\n      return \"float\";\n    case mshadow::kFloat64:\n      return \"double\";\n    case mshadow::kFloat16:\n      return \"half\";\n    case mshadow::kUint8:\n      return \"unsigned char\";\n    case mshadow::kInt8:\n      return \"char\";\n    case mshadow::kInt32:\n      return \"int\";\n    case mshadow::kInt64:\n      return \"long long\";\n    case mshadow::kBool:\n      return \"bool\";\n    case mshadow::kInt16:\n      return \"short\";\n    case mshadow::kUint16:\n      return \"unsigned short\";\n    case mshadow::kUint32:\n      return \"unsigned int\";\n    case mshadow::kUint64:\n      return \"unsigned long long\";\n    default:\n      LOG(FATAL) << \"Unknown type enum \" << dtype;\n  }\n  return \"unknown\";\n}\n\n}  // namespace mshadow\n#endif  // MSHADOW_BASE_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/bfloat.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file bfloat.h\n * \\brief definition of bfloat type.\n *\n * \\author Zhennan Qin\n */\n#ifndef MSHADOW_BFLOAT_H_\n#define MSHADOW_BFLOAT_H_\n#include \"./base.h\"\n\n/*! \\brief namespace for mshadow */\nnamespace mshadow {\n/* \\brief name space for host/device portable bfloats */\nnamespace bfloat {\n\n#define MSHADOW_BF16_OPERATOR_TYPE(RTYPE, ITYPE, OP)                      \\\n  MSHADOW_XINLINE RTYPE operator OP (ITYPE a, bf16_t b) {                 \\\n    return RTYPE(a OP float(b));  /* NOLINT(*) */                         \\\n  }                                                                       \\\n  MSHADOW_XINLINE RTYPE operator OP (bf16_t a, ITYPE b) {                 \\\n    return RTYPE(float(a) OP b);  /* NOLINT(*) */                         \\\n  }\n\n#define MSHADOW_BF16_OPERATOR(RTYPE, OP)                                  \\\n  MSHADOW_XINLINE RTYPE operator OP (bf16_t a, bf16_t b) {                \\\n    return RTYPE(static_cast<float>(a) OP float(b));  /* NOLINT(*) */     \\\n  }                                                                       \\\n  MSHADOW_BF16_OPERATOR_TYPE(float, float, OP)                            \\\n  MSHADOW_BF16_OPERATOR_TYPE(double, double, OP)                          \\\n  MSHADOW_BF16_OPERATOR_TYPE(float, int8_t, OP)                           \\\n  MSHADOW_BF16_OPERATOR_TYPE(float, uint8_t, OP)                          \\\n  MSHADOW_BF16_OPERATOR_TYPE(float, int32_t, OP)                          \\\n  MSHADOW_BF16_OPERATOR_TYPE(float, uint32_t, OP)                         \\\n  MSHADOW_BF16_OPERATOR_TYPE(float, int64_t, OP)                          \\\n  MSHADOW_BF16_OPERATOR_TYPE(float, uint64_t, OP)\n\n#define MSHADOW_BF16_ASSIGNOP(AOP, OP)                                    \\\n  template<typename T>                                                    \\\n  MSHADOW_XINLINE bf16_t operator AOP (const T& a) {                      \\\n    return *this = bf16_t(float(*this) OP float(a));  /* NOLINT(*)*/      \\\n  }                                                                       \\\n  template<typename T>                                                    \\\n  MSHADOW_XINLINE bf16_t operator AOP (const volatile T& a) volatile {    \\\n    return *this = bf16_t(float(*this) OP float(a));  /* NOLINT(*)*/      \\\n  }\n\n#define MSHADOW_BF16_CONVERSIONOP(T)                                      \\\n  MSHADOW_XINLINE operator T() const {                                    \\\n    return T(BF16ToFloat(bf16_));  /* NOLINT(*)*/                            \\\n  }                                                                       \\\n  MSHADOW_XINLINE operator T() const volatile {                           \\\n    return T(BF16ToFloat(bf16_));  /* NOLINT(*)*/                            \\\n  }\n\nclass MSHADOW_ALIGNED(2) bf16_t {\n public:\n  uint16_t bf16_;\n\nstatic MSHADOW_XINLINE bf16_t Binary(uint16_t value) {\n  bf16_t res;\n  res.bf16_ = value;\n  return res;\n  }\n\n  MSHADOW_XINLINE bf16_t() {}\n\n  MSHADOW_XINLINE bf16_t(const float& value) { constructor(value); }\n  MSHADOW_XINLINE explicit bf16_t(const double& value) { constructor(value); }\n  MSHADOW_XINLINE explicit bf16_t(const int8_t& value) { constructor(value); }\n  MSHADOW_XINLINE explicit bf16_t(const uint8_t& value) { constructor(value); }\n  MSHADOW_XINLINE explicit bf16_t(const int32_t& value) { constructor(value); }\n  MSHADOW_XINLINE explicit bf16_t(const uint32_t& value) { constructor(value); }\n  MSHADOW_XINLINE explicit bf16_t(const int64_t& value) { constructor(value); }\n  MSHADOW_XINLINE explicit bf16_t(const uint64_t& value) { constructor(value); }\n\n  MSHADOW_BF16_CONVERSIONOP(float)\n\n  MSHADOW_BF16_ASSIGNOP(+=, +)\n  MSHADOW_BF16_ASSIGNOP(-=, -)\n  MSHADOW_BF16_ASSIGNOP(*=, *)\n  MSHADOW_BF16_ASSIGNOP(/=, /)\n\n  MSHADOW_XINLINE bf16_t operator+() {\n    return *this;\n  }\n\n  MSHADOW_XINLINE bf16_t operator-() {\n    return bf16_t(-float(*this));  // NOLINT(*)\n  }\n\n  MSHADOW_XINLINE bf16_t operator=(const bf16_t& a) {\n    bf16_ = a.bf16_;\n    return a;\n  }\n\n  template<typename T>\n  MSHADOW_XINLINE bf16_t operator=(const T& a) {\n    return *this = bf16_t(a);  /* NOLINT(*)*/\n  }\n\n  MSHADOW_XINLINE bf16_t operator=(const bf16_t& a) volatile {\n    bf16_ = a.bf16_;\n    return a;\n  }\n\n  template<typename T>\n  MSHADOW_XINLINE bf16_t operator=(const T& a) volatile {\n    return *this = bf16_t(a);  /* NOLINT(*)*/\n  }\n\n private:\n  union Bits {\n    float f;\n    int32_t si;\n    uint32_t ui;\n  };\n\n  MSHADOW_XINLINE uint16_t FloatToBF16(const float& value) const {\n    return reinterpret_cast<const uint16_t*>(&value)[1];\n  }\n\n  // Same as above routine, except for addition of volatile keyword\n  MSHADOW_XINLINE uint16_t FloatToBF16(const volatile float& value) const volatile {  // NOLINT (*)\n    return reinterpret_cast<const volatile uint16_t*>(&value)[1];\n  }\n\n  MSHADOW_XINLINE float BF16ToFloat(const uint16_t& value) const {\n    float ret = 0.f;\n    reinterpret_cast<uint16_t*>(&ret)[1] = value;\n    return ret;\n  }\n\n  MSHADOW_XINLINE float BF16ToFloat(const volatile uint16_t& value) const volatile {  // NOLINT(*)\n    float ret = 0.f;\n    reinterpret_cast<uint16_t*>(&ret)[1] = value;\n    return ret;\n  }\n\n  template<typename T>\n  MSHADOW_XINLINE void constructor(const T& value) {\n    bf16_ = FloatToBF16(float(value));  // NOLINT(*)\n  }\n};\n\n/*! \\brief overloaded + operator for bf16_t */\nMSHADOW_BF16_OPERATOR(bf16_t, +)\n/*! \\brief overloaded - operator for bf16_t */\nMSHADOW_BF16_OPERATOR(bf16_t, -)\n/*! \\brief overloaded * operator for bf16_t */\nMSHADOW_BF16_OPERATOR(bf16_t, *)\n/*! \\brief overloaded / operator for bf16_t */\nMSHADOW_BF16_OPERATOR(bf16_t, /)\n/*! \\brief overloaded > operator for bf16_t */\nMSHADOW_BF16_OPERATOR(bool, >)\n/*! \\brief overloaded < operator for bf16_t */\nMSHADOW_BF16_OPERATOR(bool, <)\n/*! \\brief overloaded >= operator for bf16_t */\nMSHADOW_BF16_OPERATOR(bool, >=)\n/*! \\brief overloaded <= operator for bf16_t */\nMSHADOW_BF16_OPERATOR(bool, <=)\n\n#define MSHADOW_BF16_MIN mshadow::bfloat::bf16_t::Binary(0xFF7F);\n#define MSHADOW_BF16_MAX mshadow::bfloat::bf16_t::Binary(0x7F7F);\n#define MSHADOW_BF16_SIGN_BIT      0x8000\n#define MSHADOW_BF16_EXPONENT_BITS 0x7f80\n}  // namespace bfloat\n}  // namespace mshadow\n#endif  // MSHADOW_BFLOAT_H_"
  },
  {
    "path": "3rdparty/mshadow/mshadow/cuda/reduce.cuh",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file reduce.cuh\n * \\brief helper functions to do reduction\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_CUDA_REDUCE_CUH_\n#define MSHADOW_CUDA_REDUCE_CUH_\n\nnamespace mshadow {\nnamespace cuda {\n/*\n * \\brief reduce over the dimension x\n * \\tparam Reducer reducer\n * \\tparam x_bits dimension = 1<<x_bits\n * \\tparam DType content data type\n */\ntemplate<typename Reducer, int x_bits, typename DType>\ninline __device__ void Reduce1D(volatile DType buf[1 << x_bits]);\n/*\n * \\brief reduce over the dimension x\n * \\tparam Reducer reducer\n * \\tparam xmax_bits maximum size of buffer\n * \\tparam DType content data type\n * \\param xsize size of x dimension, not sure if aligned\n */\ntemplate<typename Reducer, int xmax_bits, typename DType>\ninline __device__ void\nReduce1DNotAlign(volatile DType buf[1 << xmax_bits], int xsize);\n// ===============================================x===\n//  implementations afterwards,\n//  no need to read if only use the functions\n// --------------------------------------------------\n#ifdef  __DEVICE_EMULATION__\n#define __syncwarp() __syncthreads()\n#else\n#if CUDA_VERSION < 9000\n#define __syncwarp()\n#endif\n#endif\n\ntemplate<typename Reducer, int x_bits, typename DType>\ninline __device__ void ReduceX(volatile DType  buf[], int tid) {\n  if (x_bits >= 10) {\n    if (tid < 512) Reducer::Reduce(buf[tid] , buf[tid + 512]);\n    __syncthreads();\n  }\n  if (x_bits >= 9) {\n    if (tid < 256) Reducer::Reduce(buf[tid] , buf[tid + 256]);\n    __syncthreads();\n  }\n  if (x_bits >= 8) {\n    if (tid < 128) Reducer::Reduce(buf[tid] , buf[tid + 128]);\n    __syncthreads();\n  }\n  if (x_bits >= 7) {\n    if (tid < 64) Reducer::Reduce(buf[tid] , buf[tid + 64]);\n    __syncthreads();\n  }\n  if (x_bits >= 6) {\n    if (tid < 32) Reducer::Reduce(buf[tid] , buf[tid + 32]);\n    __syncthreads();\n  }\n  // in warp optimization\n  if (x_bits >= 5) {\n    if (tid < 16) Reducer::Reduce(buf[tid] , buf[tid + 16]);\n#if MSHADOW_OLD_CUDA\n    __syncthreads();\n#else\n    __syncwarp();\n#endif\n  }\n  if (x_bits >= 4) {\n    if (tid < 8) Reducer::Reduce(buf[tid] , buf[tid + 8]);\n    __syncwarp();\n  }\n  if (x_bits >= 3) {\n    if (tid < 4) Reducer::Reduce(buf[tid] , buf[tid + 4]);\n    __syncwarp();\n  }\n  if (x_bits >= 2) {\n    if (tid < 2) Reducer::Reduce(buf[tid] , buf[tid + 2]);\n    __syncwarp();\n  }\n  if (x_bits >= 1) {\n    if (tid < 1) Reducer::Reduce(buf[tid] , buf[tid + 1]);\n    __syncwarp();\n  }\n}\ntemplate<typename Reducer, int x_bits, typename DType>\ninline __device__ void Reduce1D(volatile DType buf[1 << x_bits]) {\n  ReduceX<Reducer, x_bits>(buf, threadIdx.x);\n}\n// reduce with a upper bound\n#define __RD_NON_ALIGN(els, x_bits)                                     \\\n  els                                                                   \\\n  if (xmax_bits >= x_bits && x_size >= (1 << x_bits)) {                 \\\n    if (tid < (1 << x_bits) && tid + (1 << x_bits) < x_size) {          \\\n      Reducer::Reduce(buf[tid] , buf[tid + (1 << x_bits)]);             \\\n    }                                                                   \\\n    __syncthreads();                                                    \\\n    ReduceX<Reducer, x_bits>(buf, tid);                                 \\\n  }                                                                     \\\n\ntemplate<typename Reducer, int xmax_bits, typename DType>\ninline __device__ void Reduce1DNotAlign(volatile DType buf[], int x_size) {\n  int tid = threadIdx.x;\n  __RD_NON_ALIGN(, 8)\n  __RD_NON_ALIGN(else, 7)\n  __RD_NON_ALIGN(else, 6)\n  __RD_NON_ALIGN(else, 5)\n  __RD_NON_ALIGN(else, 4)\n  __RD_NON_ALIGN(else, 3)\n  __RD_NON_ALIGN(else, 2)\n  __RD_NON_ALIGN(else, 1)\n}\n}  // namespace cuda\n}  // namespace mshadow\n#endif  // MSHADOW_CUDA_REDUCE_CUH_\n\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/cuda/tensor_gpu-inl.cuh",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file tensor_gpu-inl.cuh\n * \\brief implementation of GPU code using CUDA\n * \\author Bing Xu, Tianqi Chen\n */\n#ifndef MSHADOW_CUDA_TENSOR_GPU_INL_CUH_\n#define MSHADOW_CUDA_TENSOR_GPU_INL_CUH_\n#include <thrust/device_ptr.h>\n#include <thrust/sort.h>\n#if CUDA_VERSION >= 7000\n#include <thrust/system/cuda/execution_policy.h>\n#endif\n#include \"../tensor.h\"\n#include \"./reduce.cuh\"\n#define MSHADOW_CUDA_POST_KERNEL_CHECK(x) \\\n  /* Code block avoids redefinition of cudaError_t err */ \\\n  do { \\\n    cudaError err = cudaGetLastError(); \\\n    CHECK_EQ(err, cudaSuccess) << \"Name: \" << #x << \" ErrStr:\" << cudaGetErrorString(err); \\\n  } while (0)\nnamespace mshadow {\nnamespace cuda {\n/* load unit for memory access, if CUDAARCH not defined, this is advanced nvcc */\n#if MSHADOW_OLD_CUDA\nconst int kMemUnitBits = 4;\nconst int kMaxThreadsPerBlock = 512;\n#else\nconst int kMemUnitBits = 5;\nconst int kMaxThreadsPerBlock = 1024;\n#endif\n/*! \\brief number of units that can do synchronized update, half warp size */\nconst int kMemUnit = 1 << kMemUnitBits;\n/*! \\brief mask that could be helpful sometime */\nconst int kMemUnitMask = kMemUnit - 1;\n/*! \\brief suggested thread number(logscale) for mapping kernel */\nconst int kBaseThreadBits = 8;\n/*! \\brief suggested thread number for mapping kernel */\nconst int kBaseThreadNum  = 1 << kBaseThreadBits;\n/*! \\brief maximum value of grid */\nconst int kMaxGridNum = 65535;\n/*! \\brief maximum value of grid within each dimension */\nconst int kMaxGridDim = 65535;\n/*! \\brief suggested grid number for mapping kernel */\nconst int kBaseGridNum = 1024;\n/*! \\brief get align stride for given size in x dimension */\ninline index_t GetAlignStride(index_t xsize) {\n  if (xsize >= MSHADOW_MIN_PAD_RATIO * 32) {\n    return ((xsize  + kMemUnit - 1) >> kMemUnitBits) << kMemUnitBits;\n  } else {\n    // if originally space is not aligned, no necessary to to alligned thread allocation\n    return xsize;\n  }\n}\ninline void CheckLaunchParam(dim3 dimGrid, dim3 dimBlock, const char *estr = \"\") {\n  if (dimBlock.x * dimBlock.y * dimBlock.z > static_cast<unsigned>(kMaxThreadsPerBlock) ||\n      dimGrid.x > kMaxGridDim || dimGrid.y > kMaxGridDim) {\n    LOG(FATAL) << \"too large launch parameter: \"\n      << estr << \"[\"\n      << dimGrid.x << \",\"\n      << dimGrid.y << \"], [\"\n      << dimBlock.x << \",\"\n      << dimBlock.y << \",\"\n      << dimBlock.z << \"]\";\n  }\n}\ntemplate<typename Saver, typename DstPlan,\n         typename Plan, int block_dim_bits>\n__device__ void MapPlanProc(DstPlan dst, index_t xstride,\n                            Shape<2> dshape, const Plan plan, int block_idx) {\n  const index_t tid = (block_idx << block_dim_bits) + threadIdx.x;\n  const int y = tid / xstride;\n  const int x = tid % xstride;\n  if (y < dshape[0] && x < dshape[1]) {\n    Saver::Save(dst.REval(y, x), plan.Eval(y, x));\n  }\n}\ntemplate<typename Saver, int block_dim_bits,\n         typename DstPlan, typename Plan>\n__global__ void MapPlanKernel(DstPlan dst, index_t xstride,\n                              Shape<2> dshape, const Plan plan) {\n  MapPlanProc<Saver, DstPlan, Plan, block_dim_bits>\n      (dst, xstride, dshape, plan, blockIdx.x);\n}\ntemplate<typename Saver, int block_dim_bits, int grid_size,\n         typename DstPlan, typename Plan>\n__global__ void MapPlanLargeKernel(DstPlan dst, index_t xstride,\n                                   Shape<2> dshape, const Plan plan, int repeat) {\n  for (int i = 0; i < repeat; ++i) {\n  MapPlanProc<Saver, DstPlan, Plan, block_dim_bits>\n      (dst, xstride, dshape, plan, blockIdx.x + i * grid_size);\n  }\n}\n\ntemplate<typename Saver, typename DstExp, typename E, typename DType>\ninline void MapPlan(expr::Plan<DstExp, DType> dst,\n                    const expr::Plan<E, DType> &plan,\n                    Shape<2> dshape,\n                    cudaStream_t stream) {\n  const index_t xstride = GetAlignStride(dshape[1]);\n  const int num_block = (dshape[0] * xstride + kBaseThreadNum-1) / kBaseThreadNum;\n  dim3 dimBlock(kBaseThreadNum, 1, 1);\n\n  if (num_block < kMaxGridNum) {\n    dim3 dimGrid(num_block, 1, 1);\n    MapPlanKernel<Saver, kBaseThreadBits,\n                  expr::Plan<DstExp, DType>,\n                  expr::Plan<E, DType> >\n        <<<dimGrid, dimBlock, 0, stream>>>(dst, xstride, dshape, plan);\n    MSHADOW_CUDA_POST_KERNEL_CHECK(MapPlanKernel);\n  } else {\n    int repeat = (num_block + kBaseGridNum-1) / kBaseGridNum;\n    dim3 dimGrid(kBaseGridNum, 1 , 1);\n    MapPlanLargeKernel<Saver, kBaseThreadBits, kBaseGridNum,\n                       expr::Plan<DstExp, DType>,\n                       expr::Plan<E, DType> >\n        <<<dimGrid, dimBlock, 0, stream>>>(dst, xstride, dshape, plan, repeat);\n    MSHADOW_CUDA_POST_KERNEL_CHECK(MapPlanLargeKernel);\n  }\n}\n\ntemplate<typename Saver, typename Reducer, int warp_bits,\n         typename DType, typename DstPlan, typename Plan>\n__global__ void\n__launch_bounds__(kMemUnit*kMemUnit, 1)\nMapRedKeepLowestKernel(DstPlan dst, Plan plan,\n                       DType scale, Shape<2> eshape) {\n  const unsigned warp_size = 1 << warp_bits;\n  const unsigned x = (blockIdx.x << warp_bits) + threadIdx.x;\n  // to avoid bank conflict\n  __shared__ DType s_res[warp_size][warp_size + 1];\n  // note: reverse store [y][x], so that we can reduce over threadIdx.x, use warp optimization\n  if (threadIdx.y < eshape[0] && x < eshape[1]) {\n    s_res[threadIdx.x][threadIdx.y] = plan.Eval(threadIdx.y, x);\n  }\n  for (unsigned y = warp_size; y < eshape[0]; y += warp_size) {\n    if (threadIdx.y + y < eshape[0] && x < eshape[1]) {\n      Reducer::Reduce(s_res[threadIdx.x][threadIdx.y], plan.Eval(threadIdx.y + y, x));\n    }\n  }\n  __syncthreads();\n  if (eshape[0] >= warp_size) {\n    Reduce1D<Reducer, warp_bits>(s_res[threadIdx.y]);\n  } else {\n    Reduce1DNotAlign<Reducer, warp_bits>(s_res[threadIdx.y], eshape[0]);\n  }\n  __syncthreads();\n\n  if (threadIdx.y == 0 && x < eshape[1]) {\n    Saver::Save(dst.REval(0, x),  DType(s_res[threadIdx.x][0] * scale));\n  }\n}\n\ntemplate<typename Saver, typename Reducer,\n         typename DstExp, typename E, typename DType>\ninline void MapReduceKeepLowest(expr::Plan<DstExp, DType> dst,\n                                const expr::Plan<E, DType> &plan,\n                                DType scale, Shape<2> eshape,\n                                cudaStream_t stream) {\n  dim3 dimBlock(kMemUnit, kMemUnit);\n  dim3 dimGrid((eshape[1] + kMemUnit - 1) >> kMemUnitBits);\n  CheckLaunchParam(dimGrid, dimBlock, \"MapRedKeepLowestKernel\");\n  MapRedKeepLowestKernel<Saver, Reducer, kMemUnitBits, DType,\n                         expr::Plan<DstExp, DType>,\n                         expr::Plan<E, DType> >\n      <<<dimGrid, dimBlock, 0, stream>>>(dst, plan, scale, eshape);\n  MSHADOW_CUDA_POST_KERNEL_CHECK(MapRedKeepLowestKernel);\n}\n\ntemplate<typename Saver, typename Reducer, int block_dim_bits,\n         typename DType, typename DstPlan, typename Plan>\n__global__ void MapReduceKeepDim1Kernel(DstPlan dst, Plan plan, DType scale, Shape<4> pshape) {\n  const int block_size = 1 << block_dim_bits;\n  __shared__ DType s_rec[block_size];\n  const int c = blockIdx.x + blockIdx.y * gridDim.x;\n  const index_t tot = pshape[3] * pshape[2] * pshape[0];\n\n  if (c < pshape[1]) {\n    DType res; Reducer::SetInitValue(res);\n    for (index_t i_offset = 0; i_offset < tot; i_offset += block_size) {\n      index_t i = i_offset + threadIdx.x;\n      if (i< tot) {\n        const index_t x = i % pshape[3];\n        i /= pshape[3];\n        const index_t y = i % pshape[2];\n        const index_t n = i / pshape[2];\n        Reducer::Reduce(res, plan.Eval((n * pshape[1] + c) * pshape[2] + y, x));\n      }\n    }\n    s_rec[threadIdx.x] = res;\n    __syncthreads();\n    Reduce1D<Reducer, block_dim_bits>(s_rec);\n    if (threadIdx.x == 0) {\n      Saver::Save(dst.REval(0, c), DType(s_rec[0] * scale));\n    }\n  }\n}\n\ntemplate<typename Saver, typename Reducer, typename DstExp, typename E, typename DType>\ninline void MapReduceKeepDim1(expr::Plan<DstExp, DType> dst,\n                              const expr::Plan<E, DType> &plan,\n                              DType scale, Shape<4> pshape,\n                              cudaStream_t stream) {\n  dim3 dimBlock(kBaseThreadNum);\n  const int grid_dim_x = (pshape[1] > kMaxGridNum) ? kMaxGridNum : pshape[1];\n  const int grid_dim_y = (pshape[1] > kMaxGridNum) ? (pshape[1] + kMaxGridNum - 1) / kMaxGridNum\n                                                   : 1;\n  dim3 dimGrid(grid_dim_x, grid_dim_y);\n  CheckLaunchParam(dimGrid, dimBlock, \"MapReduceKeepDim1\");\n  MapReduceKeepDim1Kernel<Saver, Reducer, kBaseThreadBits, DType,\n                          expr::Plan<DstExp, DType>,\n                          expr::Plan<E, DType> >\n      <<<dimGrid, dimBlock, 0, stream>>>(dst, plan, scale, pshape);\n  MSHADOW_CUDA_POST_KERNEL_CHECK(MapReduceKeepDim1Kernel);\n}\n\ntemplate<int x_bits, typename DType>\n__global__ void GetBatchedViewKernel(DType **dst, DType *src, int num, int stride) {\n  const int x_size = 1 << x_bits;\n  const int start = threadIdx.x;\n  // Copy the addresses of src to dst every stride steps\n  for (int i = start; i < num; i += x_size) {\n    dst[i] = src + i * stride;\n  }\n}\n\ntemplate<typename DType>\ninline void GetBatchedView(DType **dst, DType *src, int num, int stride,\n                           Stream<gpu> *stream) {\n  cudaStream_t stream_ = Stream<gpu>::GetStream(stream);\n  dim3 dimBlock(kBaseThreadNum);\n  dim3 dimGrid(1);\n  CheckLaunchParam(dimGrid, dimBlock, \"GetBatchedView\");\n  GetBatchedViewKernel<kBaseThreadBits, DType>\n    <<<dimGrid, dimBlock, 0, stream_>>> (dst, src, num, stride);\n  MSHADOW_CUDA_POST_KERNEL_CHECK(GetBatchedViewKernel);\n}\n\ntemplate<int x_bits, typename DType, typename DstPlan, typename SrcPlan1, typename SrcPlan2>\n__global__ void SoftmaxGradKernel(DstPlan dst, SrcPlan1 src, SrcPlan2 label, index_t xmax) {\n  const unsigned x_size = 1 << x_bits;\n  const int y = blockIdx.x;\n  const int k = static_cast<int>(label.Eval(0, y));\n\n  // calculate normalizer, with writeback\n  for (unsigned x = 0; x < xmax; x += x_size) {\n    const unsigned xindex = x + threadIdx.x;\n    if (xindex < xmax) {\n      if (xindex == k) {\n        dst.REval(y, xindex) = src.Eval(y, xindex) - 1.0f;\n      } else {\n        dst.REval(y, xindex) = src.Eval(y, xindex);\n      }\n    }\n  }\n}\n\ntemplate<int x_bits, typename DType, typename DstPlan, typename SrcPlan1, typename SrcPlan2>\n__global__ void SmoothSoftmaxGradKernel(DstPlan dst, SrcPlan1 src, SrcPlan2 label, index_t xmax,\n                                        float alpha) {\n  const unsigned x_size = 1 << x_bits;\n  const int y = blockIdx.x;\n  const int k = static_cast<int>(label.Eval(0, y));\n  // xmax is the number of classes in our distribution\n  const float smooth_grad = (alpha / (xmax - 1));\n\n  // calculate normalizer, with writeback\n  for (unsigned x = 0; x < xmax; x += x_size) {\n    const unsigned xindex = x + threadIdx.x;\n    if (xindex < xmax) {\n      if (xindex == k) {\n        dst.REval(y, xindex) = src.Eval(y, xindex) - 1.0f + alpha;\n      } else {\n        dst.REval(y, xindex) = src.Eval(y, xindex) - smooth_grad;\n      }\n    }\n  }\n}\n\ntemplate<int x_bits, typename DType, typename DstPlan, typename SrcPlan1, typename SrcPlan2>\n__global__ void SoftmaxGradKernel(DstPlan dst, SrcPlan1 src, SrcPlan2 label, index_t xmax,\n                                  DType ignore_label) {\n  const unsigned x_size = 1 << x_bits;\n  const int y = blockIdx.x;\n  const int k = static_cast<int>(label.Eval(0, y));\n\n  // calculate normalizer, with writeback\n  for (unsigned x = 0; x < xmax; x += x_size) {\n    const unsigned xindex = x + threadIdx.x;\n    if (xindex < xmax) {\n      if (static_cast<int>(ignore_label) == k) {\n        dst.REval(y, xindex) = 0.0f;\n      } else {\n        if (xindex == k) {\n          dst.REval(y, xindex) = src.Eval(y, xindex) - 1.0f;\n        } else {\n          dst.REval(y, xindex) = src.Eval(y, xindex);\n        }\n      }\n    }\n  }\n}\n\ntemplate<int x_bits, typename DType, typename DstPlan, typename SrcPlan1, typename SrcPlan2>\n__global__ void SmoothSoftmaxGradKernel(DstPlan dst, SrcPlan1 src, SrcPlan2 label, index_t xmax,\n                                  DType ignore_label, float alpha) {\n  const unsigned x_size = 1 << x_bits;\n  const int y = blockIdx.x;\n  const int k = static_cast<int>(label.Eval(0, y));\n  // xmax is the number of classes in our distribution\n  const float smooth_grad = (alpha / (xmax - 1));\n\n  // calculate normalizer, with writeback\n  for (unsigned x = 0; x < xmax; x += x_size) {\n    const unsigned xindex = x + threadIdx.x;\n    if (xindex < xmax) {\n      if (static_cast<int>(ignore_label) == k) {\n        dst.REval(y, xindex) = 0.0f;\n      } else {\n        if (xindex == k) {\n          dst.REval(y, xindex) = src.Eval(y, xindex) - 1.0f + alpha;\n        } else {\n          dst.REval(y, xindex) = src.Eval(y, xindex) - smooth_grad;\n        }\n      }\n    }\n  }\n}\n\ntemplate<int x_bits, typename DType,  typename DstPlan, typename SrcPlan>\n__global__ void SoftmaxKernel(DstPlan dst, SrcPlan src, index_t xmax) {\n  const unsigned x_size = 1 << x_bits;\n  const int y = blockIdx.x;\n  __shared__ DType s_rec[x_size];\n  // step 1: get max\n  if (threadIdx.x < xmax) {\n    s_rec[threadIdx.x] = src.Eval(y, threadIdx.x);\n  }\n  for (unsigned x = x_size; x < xmax; x += x_size) {\n    if (x + threadIdx.x < xmax) {\n      DType a = src.Eval(y, x + threadIdx.x);\n      s_rec[threadIdx.x] = max(a, s_rec[threadIdx.x]);\n    }\n  }\n  __syncthreads();\n  if (threadIdx.x >= xmax) {\n    s_rec[threadIdx.x] = s_rec[0];\n  }\n  __syncthreads();\n  Reduce1D<red::maximum, x_bits>(s_rec);\n  __syncthreads();\n  DType smax = s_rec[0];\n  __syncthreads();\n  s_rec[threadIdx.x] = 0.0f;\n  __syncthreads();\n\n  // calculate normalizer, with writeback\n  for (unsigned x = 0; x < xmax; x += x_size) {\n    if (x + threadIdx.x < xmax) {\n      DType p = expf(src.Eval(y, x + threadIdx.x) - smax);\n      s_rec[threadIdx.x] += p;\n      // write back first, will fetch later\n      dst.REval(y, x + threadIdx.x) = p;\n    }\n  }\n  // calculate normalizer\n  __syncthreads();\n  Reduce1D<red::sum, x_bits>(s_rec);\n  __syncthreads();\n  DType ssum = s_rec[0];\n\n  for (unsigned x = 0; x < xmax; x += x_size) {\n    if (x + threadIdx.x < xmax) {\n      dst.REval(y, x + threadIdx.x) /= ssum;\n    }\n  }\n}\n\ntemplate<typename DType>\ninline void Softmax(const Tensor<gpu, 2, DType> &dst,\n                    const Tensor<gpu, 2, DType> &src) {\n  dim3 dimBlock(kBaseThreadNum);\n  dim3 dimGrid(dst.size(0));\n  CHECK_EQ(dst.shape_, src.shape_) << \"Softmax: shape mismatch\";\n  CheckLaunchParam(dimGrid, dimBlock, \"Softmax\");\n  cudaStream_t stream = Stream<gpu>::GetStream(dst.stream_);\n  SoftmaxKernel<kBaseThreadBits, DType>\n      <<<dimGrid, dimBlock, 0, stream>>>\n      (expr::MakePlan(dst),\n       expr::MakePlan(src),\n       dst.size(1));\n  MSHADOW_CUDA_POST_KERNEL_CHECK(SoftmaxKernel);\n}\n\ntemplate<typename DType>\ninline void SoftmaxGrad(const Tensor<gpu, 2, DType> &dst,\n                        const Tensor<gpu, 2, DType> &src,\n                        const Tensor<gpu, 1, DType> &label) {\n  dim3 dimBlock(kBaseThreadNum);\n  dim3 dimGrid(dst.size(0));\n  CHECK_EQ(dst.shape_, src.shape_) << \"SoftmaxGrad: shape mismatch\";\n  CHECK_EQ(dst.size(0), label.size(0)) << \"SoftmaxGrad: label shape mismatch\";\n  CheckLaunchParam(dimGrid, dimBlock, \"SoftmaxGrad\");\n  cudaStream_t stream = Stream<gpu>::GetStream(dst.stream_);\n  SoftmaxGradKernel<kBaseThreadBits, DType>\n      <<<dimGrid, dimBlock, 0, stream>>>\n      (expr::MakePlan(dst),\n       expr::MakePlan(src),\n       expr::MakePlan(label),\n       dst.size(1));\n  MSHADOW_CUDA_POST_KERNEL_CHECK(SoftmaxGradKernel);\n}\n\ntemplate<typename DType>\ninline void SmoothSoftmaxGrad(const Tensor<gpu, 2, DType> &dst,\n                              const Tensor<gpu, 2, DType> &src,\n                              const Tensor<gpu, 1, DType> &label,\n                              const float alpha) {\n  dim3 dimBlock(kBaseThreadNum);\n  dim3 dimGrid(dst.size(0));\n  CHECK_EQ(dst.shape_, src.shape_) << \"SoftmaxGrad: shape mismatch\";\n  CHECK_EQ(dst.size(0), label.size(0)) << \"SoftmaxGrad: label shape mismatch\";\n  CheckLaunchParam(dimGrid, dimBlock, \"SoftmaxGrad\");\n  cudaStream_t stream = Stream<gpu>::GetStream(dst.stream_);\n  SmoothSoftmaxGradKernel<kBaseThreadBits, DType>\n      <<<dimGrid, dimBlock, 0, stream>>>\n      (expr::MakePlan(dst),\n       expr::MakePlan(src),\n       expr::MakePlan(label),\n       dst.size(1),\n       alpha);\n  MSHADOW_CUDA_POST_KERNEL_CHECK(SoftmaxGradKernel);\n}\n\ntemplate<typename DType>\ninline void SoftmaxGrad(const Tensor<gpu, 2, DType> &dst,\n                        const Tensor<gpu, 2, DType> &src,\n                        const Tensor<gpu, 1, DType> &label,\n                        const DType &ignore_label) {\n  dim3 dimBlock(kBaseThreadNum);\n  dim3 dimGrid(dst.size(0));\n  CHECK_EQ(dst.shape_, src.shape_) << \"SoftmaxGrad: shape mismatch\";\n  CHECK_EQ(dst.size(0), label.size(0)) << \"SoftmaxGrad: label shape mismatch\";\n  CheckLaunchParam(dimGrid, dimBlock, \"SoftmaxGrad\");\n  cudaStream_t stream = Stream<gpu>::GetStream(dst.stream_);\n  SoftmaxGradKernel<kBaseThreadBits, DType>\n      <<<dimGrid, dimBlock, 0, stream>>>\n      (expr::MakePlan(dst),\n       expr::MakePlan(src),\n       expr::MakePlan(label),\n       dst.size(1),\n       ignore_label);\n  MSHADOW_CUDA_POST_KERNEL_CHECK(SoftmaxGradKernel);\n}\n\ntemplate<typename DType>\ninline void SmoothSoftmaxGrad(const Tensor<gpu, 2, DType> &dst,\n                              const Tensor<gpu, 2, DType> &src,\n                              const Tensor<gpu, 1, DType> &label,\n                              const DType &ignore_label,\n                              const float alpha) {\n  dim3 dimBlock(kBaseThreadNum);\n  dim3 dimGrid(dst.size(0));\n  CHECK_EQ(dst.shape_, src.shape_) << \"SoftmaxGrad: shape mismatch\";\n  CHECK_EQ(dst.size(0), label.size(0)) << \"SoftmaxGrad: label shape mismatch\";\n  CheckLaunchParam(dimGrid, dimBlock, \"SoftmaxGrad\");\n  cudaStream_t stream = Stream<gpu>::GetStream(dst.stream_);\n  SmoothSoftmaxGradKernel<kBaseThreadBits, DType>\n      <<<dimGrid, dimBlock, 0, stream>>>\n      (expr::MakePlan(dst),\n       expr::MakePlan(src),\n       expr::MakePlan(label),\n       dst.size(1),\n       ignore_label,\n       alpha);\n  MSHADOW_CUDA_POST_KERNEL_CHECK(SoftmaxGradKernel);\n}\n\ntemplate<int n_bits, typename DType>\n__global__ void Softmax3DGradKernel(Tensor<gpu, 3, DType> dst,\n                                    const Tensor<gpu, 3, DType> src,\n                                    const Tensor<gpu, 2, DType> label) {\n  const index_t xmax = dst.size(1);\n  const index_t nmax = dst.size(2);\n  const unsigned n_size = 1 << n_bits;\n  const int y = blockIdx.x;\n  const int n = threadIdx.x;\n\n  for (index_t n_index = n; n_index < nmax; n_index += n_size) {\n    const int k = static_cast<int>(label[y][n_index]);\n    for (index_t i = 0; i < xmax; ++i) {\n      if (i == k) {\n        dst[y][i][n_index] = src[y][i][n_index] - 1.0f;\n      } else {\n        dst[y][i][n_index] = src[y][i][n_index];\n      }\n    }\n  }\n}\n\ntemplate<int n_bits, typename DType>\n__global__ void Softmax3DGradKernel(Tensor<gpu, 3, DType> dst,\n                                    const Tensor<gpu, 3, DType> src,\n                                    const Tensor<gpu, 2, DType> label,\n                                    DType ignore_label) {\n  const index_t xmax = dst.size(1);\n  const index_t nmax = dst.size(2);\n  const unsigned n_size = 1 << n_bits;\n  const int y = blockIdx.x;\n  const int n = threadIdx.x;\n  for (index_t n_index = n; n_index < nmax; n_index += n_size) {\n    int k = static_cast<int>(label[y][n_index]);\n    if (k == static_cast<int>(ignore_label)) {\n      for (index_t i = 0; i < xmax; ++i) {\n        dst[y][i][n_index] = 0.0f;\n      }\n    } else {\n      for (index_t i = 0; i < xmax; ++i) {\n        if (i == k) {\n          dst[y][i][n_index] = src[y][i][n_index] - 1.0f;\n        } else {\n          dst[y][i][n_index] = src[y][i][n_index];\n        }\n      }\n    }\n  }\n}\n\ntemplate<int n_bits, typename DType>\n__global__ void Softmax3DKernel(Tensor<gpu, 3, DType> dst,\n                    const Tensor<gpu, 3, DType> src) {\n  const index_t xmax = dst.size(1);\n  const index_t nmax = dst.size(2);\n  const unsigned n_size = 1 << n_bits;\n  const int y = blockIdx.x;\n  const int n = threadIdx.x;\n\n  for (index_t n_index = n; n_index < nmax; n_index += n_size) {\n    DType smax = src[y][0][n_index];\n    for (index_t i = 1; i < xmax; ++i) {\n      smax = max(smax, src[y][i][n_index]);  // NOLINT(*)\n    }\n    DType ssum = 0.0f;\n    for (index_t i = 0; i < xmax; ++i) {\n      DType p = expf(src[y][i][n_index] - smax);\n      ssum += p;\n      dst[y][i][n_index] = p;\n    }\n    for (index_t i = 0; i < xmax; ++i) {\n      dst[y][i][n_index] /= ssum;\n    }\n  }\n}\n\ntemplate<typename DType>\ninline void Softmax(const Tensor<gpu, 3, DType> &dst,\n                    const Tensor<gpu, 3, DType> &src) {\n  dim3 dimBlock(kBaseThreadNum);\n  dim3 dimGrid(dst.size(0));\n  CHECK_EQ(dst.shape_, src.shape_) << \"Softmax: shape mismatch\";\n  CheckLaunchParam(dimGrid, dimBlock, \"Softmax\");\n  cudaStream_t stream = Stream<gpu>::GetStream(dst.stream_);\n  Softmax3DKernel<kBaseThreadBits, DType><<<dimGrid, dimBlock, 0, stream>>>(dst, src);\n  MSHADOW_CUDA_POST_KERNEL_CHECK(Softmax3DKernel);\n}\n\ntemplate<typename DType>\ninline void SoftmaxGrad(const Tensor<gpu, 3, DType> &dst,\n                        const Tensor<gpu, 3, DType> &src,\n                        const Tensor<gpu, 2, DType> &label) {\n  dim3 dimBlock(kBaseThreadNum);\n  dim3 dimGrid(dst.size(0));\n  CHECK_EQ(dst.shape_, src.shape_) << \"SoftmaxGrad: shape mismatch\";\n  CHECK_EQ(dst.size(0), label.size(0)) << \"SoftmaxGrad: label shape mismatch\";\n  CHECK_EQ(dst.size(2), label.size(1)) << \"SoftmaxGrad: label shape mismatch\";\n  CheckLaunchParam(dimGrid, dimBlock, \"SoftmaxGrad\");\n  cudaStream_t stream = Stream<gpu>::GetStream(dst.stream_);\n  Softmax3DGradKernel<kBaseThreadBits, DType><<<dimGrid, dimBlock, 0, stream>>>(dst, src, label);\n  MSHADOW_CUDA_POST_KERNEL_CHECK(Softmax3DGradKernel);\n}\n\ntemplate<typename DType>\ninline void SoftmaxGrad(const Tensor<gpu, 3, DType> &dst,\n                        const Tensor<gpu, 3, DType> &src,\n                        const Tensor<gpu, 2, DType> &label,\n                        const DType &ignore_label) {\n  dim3 dimBlock(kBaseThreadNum);\n  dim3 dimGrid(dst.size(0));\n  CHECK_EQ(dst.shape_, src.shape_) << \"SoftmaxGrad: shape mismatch\";\n  CHECK_EQ(dst.size(0), label.size(0)) << \"SoftmaxGrad: label shape mismatch\";\n  CHECK_EQ(dst.size(2), label.size(1)) << \"SoftmaxGrad: label shape mismatch\";\n  CheckLaunchParam(dimGrid, dimBlock, \"SoftmaxGrad\");\n  cudaStream_t stream = Stream<gpu>::GetStream(dst.stream_);\n  Softmax3DGradKernel<kBaseThreadBits, DType><<<dimGrid, dimBlock, 0, stream>>>(\n    dst, src, label, ignore_label);\n  MSHADOW_CUDA_POST_KERNEL_CHECK(Softmax3DGradKernel);\n}\n\ntemplate<bool clip, int x_bits, typename DType, typename DstPlan,\n         typename SrcPlan1, typename SrcPlan2>\n__global__ void AddTakeGradKernel(DstPlan dst,\n                                  SrcPlan1 index, SrcPlan2 src,\n                                  index_t ymax, index_t xmax, const int K) {\n  const unsigned x_size = 1 << x_bits;\n  const int xindex = blockIdx.x * x_size + threadIdx.x;\n  __shared__ int ptr;\n  for (unsigned y = 0; y < ymax; ++y) {\n    if (threadIdx.x == 0) {\n      ptr = index.Eval(0, y);\n      if (clip) {\n        if (ptr <= 0) ptr = 0;\n        else if (ptr >= K) ptr = K - 1;\n      } else {\n        ptr %= K;\n        if (ptr < 0) ptr += K;\n      }\n    }\n    __syncthreads();\n    if (xindex < xmax) {\n      dst.REval(ptr, xindex) += src.Eval(y, xindex);\n    }\n  }\n}\n\ntemplate<bool clip, int x_bits, typename DstPlan, typename ATypePlan,\n         typename SrcPlan1, typename SrcPlan2>\n__global__ void AddTakeGradKernel(DstPlan dst,\n                                  ATypePlan temp,\n                                  SrcPlan1 index, SrcPlan2 src,\n                                  index_t ymax, index_t xmax, const int K) {\n  const unsigned x_size = 1 << x_bits;\n  const int xindex = blockIdx.x * x_size + threadIdx.x;\n  __shared__ int ptr;\n  if (xindex < xmax) {\n    for (unsigned y = 0; y < K; ++y) {\n      temp.REval(y, xindex) = dst.Eval(y, xindex);\n    }\n  }\n  for (unsigned y = 0; y < ymax; ++y) {\n    if (threadIdx.x == 0) {\n      ptr = index.Eval(0, y);\n      if (clip) {\n        if (ptr <= 0) ptr = 0;\n        else if (ptr >= K) ptr = K - 1;\n      } else {\n        ptr %= K;\n        if (ptr < 0) ptr += K;\n      }\n    }\n    __syncthreads();\n    if (xindex < xmax) {\n      temp.REval(ptr, xindex) += src.Eval(y, xindex);\n    }\n  }\n  if (xindex < xmax) {\n    for (unsigned y = 0; y < K; ++y) {\n      dst.REval(y, xindex) = temp.Eval(y, xindex);\n    }\n  }\n}\n\ntemplate<int warp_bits, int SZ, typename DType, typename IdxType>\n__global__ void AddTakeGradLargeBatchKernel(DType* dst,\n                                            const IdxType *sorted, const IdxType *index,\n                                            const DType *src,\n                                            int ymax, int xmax) {\n  // Based on Torch's Version https://github.com/torch/cunn/blob/master/lib/THCUNN/LookupTable.cu\n  // Each warp is responsible for an input into the LookupTable.\n  // If the preceeding input has the same as this input, then the warp\n  // exits immediately. The warp also processes subsequent inputs with the\n  // same value.\n  //\n  // Input Warp\n  // 1     <warp 1>\n  // 1     <warp 1> (<warp 2> exits without doing any work)\n  // 5     <warp 3>\n  // 8     <warp 4>\n  // Also, all warp will loop for SZ times to increase the throughput.\n\n  const int warp_size = 1 << warp_bits;\n  int idx = blockIdx.x * blockDim.y + threadIdx.y;\n\n  if (idx < ymax\n    && (idx == 0 || sorted[idx] != sorted[idx - 1])) {\n    do {\n      const int start_feature = threadIdx.x + blockIdx.y * blockDim.x * SZ;\n      const int dst_row = static_cast<int>(sorted[idx]) * xmax;\n      const int src_row = static_cast<int>(index[idx]) * xmax;\n      float grad_out[SZ];\n      float grad_weight[SZ];\n      #pragma unroll\n      for (int ii = 0; ii < SZ; ii++) {\n        int feature_dim = start_feature + ii * warp_size;\n        if (feature_dim < xmax) {\n          grad_out[ii] = src[src_row + feature_dim];\n          grad_weight[ii] = dst[dst_row + feature_dim];\n        }\n      }\n\n      #pragma unroll\n      for (int ii = 0; ii < SZ; ii++) {\n        grad_weight[ii] += grad_out[ii];\n      }\n\n      #pragma unroll\n      for (int ii = 0; ii < SZ; ii++) {\n        int feature_dim = start_feature + ii * warp_size;\n        if (feature_dim < xmax) {\n          dst[dst_row + feature_dim] = grad_weight[ii];\n        }\n      }\n      idx++;\n    } while (idx < ymax && (sorted[idx] == sorted[idx - 1]));\n  }\n}\n\ntemplate<bool clip = true, typename IndexType, typename DType>\ninline void AddTakeGrad(Tensor<gpu, 2, DType> dst,\n                        const Tensor<gpu, 1, IndexType>& index,\n                        const Tensor<gpu, 2, DType> &src) {\n  CHECK_EQ(dst.CheckContiguous(), true);\n  CHECK_EQ(index.CheckContiguous(), true);\n  CHECK_EQ(src.CheckContiguous(), true);\n  const int kUnitBits = kMemUnitBits + 1;\n  dim3 dimBlock(1 << kUnitBits);\n  dim3 dimGrid((dst.size(1) + (1 << kUnitBits) - 1) >> kUnitBits);\n\n  CHECK_EQ(dst.size(1), src.size(1)) << \"AddTakeGrad: shape mismatch\";\n  CHECK_EQ(index.size(0), src.size(0)) << \"AddTakeGrad: shape mismatch\";\n  CheckLaunchParam(dimGrid, dimBlock, \"AddTakeGrad\");\n  cudaStream_t stream = Stream<gpu>::GetStream(dst.stream_);\n  const int K = dst.shape_[0];\n\n  if (clip) {\n    AddTakeGradKernel<true, kUnitBits, DType>\n      <<<dimGrid, dimBlock, 0, stream>>>\n      (expr::MakePlan(dst),\n       expr::MakePlan(index),\n       expr::MakePlan(src),\n       src.size(0),\n       src.size(1), K);\n  } else {\n    AddTakeGradKernel<false, kUnitBits, DType>\n      <<<dimGrid, dimBlock, 0, stream>>>\n      (expr::MakePlan(dst),\n       expr::MakePlan(index),\n       expr::MakePlan(src),\n       src.size(0),\n       src.size(1), K);\n  }\n  MSHADOW_CUDA_POST_KERNEL_CHECK(AddTakeGradKernel);\n}\n\ntemplate<bool clip = true, typename IndexType, typename DType, typename AType>\ninline void AddTakeGrad(Tensor<gpu, 2, DType> dst,\n                        Tensor<gpu, 2, AType> temp,\n                        const Tensor<gpu, 1, IndexType>& index,\n                        const Tensor<gpu, 2, DType> &src) {\n  CHECK_EQ(dst.CheckContiguous(), true);\n  CHECK_EQ(index.CheckContiguous(), true);\n  CHECK_EQ(src.CheckContiguous(), true);\n  const int kUnitBits = kMemUnitBits + 1;\n  dim3 dimBlock(1 << kUnitBits);\n  dim3 dimGrid((dst.size(1) + (1 << kUnitBits) - 1) >> kUnitBits);\n\n  CHECK_EQ(dst.size(1), src.size(1)) << \"AddTakeGrad: shape mismatch\";\n  CHECK_EQ(index.size(0), src.size(0)) << \"AddTakeGrad: shape mismatch\";\n  CheckLaunchParam(dimGrid, dimBlock, \"AddTakeGrad\");\n  cudaStream_t stream = Stream<gpu>::GetStream(dst.stream_);\n  const int K = dst.shape_[0];\n\n  if (clip) {\n    AddTakeGradKernel<true, kUnitBits>\n      <<<dimGrid, dimBlock, 0, stream>>>\n      (expr::MakePlan(dst),\n       expr::MakePlan(temp),\n       expr::MakePlan(index),\n       expr::MakePlan(src),\n       src.size(0),\n       src.size(1), K);\n  } else {\n    AddTakeGradKernel<false, kUnitBits>\n      <<<dimGrid, dimBlock, 0, stream>>>\n      (expr::MakePlan(dst),\n       expr::MakePlan(temp),\n       expr::MakePlan(index),\n       expr::MakePlan(src),\n       src.size(0),\n       src.size(1), K);\n  }\n  MSHADOW_CUDA_POST_KERNEL_CHECK(AddTakeGradKernel);\n}\n\ntemplate<typename IndexType, typename DType>\ninline void AddTakeGradLargeBatch(Tensor<gpu, 2, DType> dst,\n                                  const Tensor<gpu, 1, IndexType>& sorted,\n                                  const Tensor<gpu, 1, IndexType>& index,\n                                  const Tensor<gpu, 2, DType> &src) {\n  CHECK_EQ(dst.CheckContiguous(), true);\n  CHECK_EQ(sorted.CheckContiguous(), true);\n  CHECK_EQ(index.CheckContiguous(), true);\n  CHECK_EQ(src.CheckContiguous(), true);\n  const int kWarpBits = kMemUnitBits;\n  const int SZ = 4;\n  const int block_dim_x = 1 << kWarpBits;\n  const int block_dim_y = 4;\n  const int grid_dim_x = (src.size(0) + block_dim_y - 1) / block_dim_y;\n  const int grid_dim_y = (src.size(1) + block_dim_x * SZ - 1) / (block_dim_x * SZ);\n  dim3 dimBlock(block_dim_x, block_dim_y);\n  dim3 dimGrid(grid_dim_x, grid_dim_y);\n\n  CHECK_EQ(dst.size(1), src.size(1)) << \"AddTakeGradLargeBatch: shape mismatch\";\n  CHECK_EQ(index.size(0), src.size(0)) << \"AddTakeGradLargeBatch: shape mismatch\";\n  CheckLaunchParam(dimGrid, dimBlock, \"AddTakeGradLargeBatch\");\n  cudaStream_t stream = Stream<gpu>::GetStream(dst.stream_);\n\n  AddTakeGradLargeBatchKernel<kWarpBits, SZ, DType>\n      <<<dimGrid, dimBlock, 0, stream>>>\n      (dst.dptr_,\n       sorted.dptr_,\n       index.dptr_,\n       src.dptr_,\n       static_cast<int>(src.size(0)),\n       static_cast<int>(src.size(1)));\n  MSHADOW_CUDA_POST_KERNEL_CHECK(AddTakeGradLargeBatchKernel);\n}\n\ntemplate<int warp_bits, typename DType, typename DstPlan, typename IndexPlan, typename SrcPlan>\n__global__ void IndexFillKernel(DstPlan dst,\n                                const IndexPlan index,\n                                const SrcPlan src,\n                                const int ymax,\n                                const int xmax) {\n  int bid = blockIdx.y * blockDim.x + blockIdx.x;\n  int tid = bid * blockDim.x * blockDim.y + threadIdx.y * blockDim.x + threadIdx.x;\n  if (tid < ymax * xmax) {\n    int i = tid / xmax;\n    int j = tid % xmax;\n    int k = static_cast<int>(index.Eval(0, i));\n    dst.REval(k, j) = src.Eval(i, j);\n  }\n}\n\ntemplate<typename IndexType, typename DType>\ninline void IndexFill(Tensor<gpu, 2, DType> dst,\n                      const Tensor<gpu, 1, IndexType>& index,\n                      const Tensor<gpu, 2, DType> &src) {\n  CHECK_EQ(dst.CheckContiguous(), true);\n  CHECK_EQ(index.CheckContiguous(), true);\n  CHECK_EQ(src.CheckContiguous(), true);\n  CHECK_EQ(dst.size(1), src.size(1)) << \"IndexFill: shape mismatch\";\n  CHECK_EQ(index.size(0), src.size(0)) << \"IndexFill: shape mismatch\";\n  const int block_dim_x = 1 << kMemUnitBits;\n  const int block_dim_y = 1 << kMemUnitBits;\n  const int block_size = block_dim_x * block_dim_y;\n  int grid_dim_x = (src.size(0) * src.size(1) + block_size - 1) / block_size;\n  int grid_dim_y = 1;\n  while (grid_dim_x > kMaxGridDim) {\n    grid_dim_x = (grid_dim_x + 1) / 2;\n    grid_dim_y *= 2;\n  }\n  dim3 dimBlock(block_dim_x, block_dim_y);\n  dim3 dimGrid(grid_dim_x, grid_dim_y);\n  CheckLaunchParam(dimGrid, dimBlock, \"IndexFill\");\n  cudaStream_t stream = Stream<gpu>::GetStream(dst.stream_);\n\n  IndexFillKernel<kMemUnitBits, DType>\n      <<<dimGrid, dimBlock, 0, stream>>>\n      (expr::MakePlan(dst),\n       expr::MakePlan(index),\n       expr::MakePlan(src),\n       src.size(0),\n       src.size(1));\n  MSHADOW_CUDA_POST_KERNEL_CHECK(IndexFillKernel);\n}\n\ntemplate<typename KDType, typename VDType>\ninline void SortByKey(Tensor<gpu, 1, KDType> keys, Tensor<gpu, 1, VDType> values,\n                      bool is_ascend) {\n  CHECK_EQ(keys.CheckContiguous(), true);\n  CHECK_EQ(values.CheckContiguous(), true);\n#if CUDA_VERSION >= 7000\n  cudaStream_t stream = Stream<gpu>::GetStream(keys.stream_);\n  thrust::device_ptr<KDType> key_iter = thrust::device_pointer_cast(keys.dptr_);\n  thrust::device_ptr<VDType> value_iter = thrust::device_pointer_cast(values.dptr_);\n  if (is_ascend) {\n    thrust::stable_sort_by_key(\n      thrust::cuda::par.on(stream),\n      key_iter, key_iter + keys.size(0), value_iter, thrust::less<KDType>());  // NOLINT(*)\n  } else {\n    thrust::stable_sort_by_key(\n      thrust::cuda::par.on(stream),\n      key_iter, key_iter + keys.size(0), value_iter, thrust::greater<KDType>());  // NOLINT(*)\n  }\n  MSHADOW_CUDA_POST_KERNEL_CHECK(SortByKey);\n#else\n  LOG(FATAL) << \"SortByKey is only supported for CUDA version >=7.0!\";\n#endif\n}\n\ntemplate<typename DType>\ninline void SortByKey(Tensor<gpu, 1, mshadow::half::half_t> keys, Tensor<gpu, 1, DType> values,\n                      bool is_ascend) {\n  LOG(FATAL) << \"SortByKey for half_t is not implemented!\";\n}\n\ntemplate<typename DType>\ninline void SortByKey(Tensor<gpu, 1, DType> keys, Tensor<gpu, 1, mshadow::half::half_t> values,\n  bool is_ascend) {\n  LOG(FATAL) << \"SortByKey for half_t is not implemented!\";\n}\n\n// break ambiguous template deduction for <half_t, half_t>\ninline void SortByKey(Tensor<gpu, 1, mshadow::half::half_t> keys,\n  Tensor<gpu, 1, mshadow::half::half_t> values,\n  bool is_ascend) {\n  LOG(FATAL) << \"SortByKey for half_t is not implemented!\";\n}\n}  // namespace cuda\n}  // namespace mshadow\n#endif  // MSHADOW_CUDA_TENSOR_GPU_INL_CUH_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/dot_engine-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file dot_engine-inl.h\n * \\brief definitions of how Matrix Multiplications can be evaluated\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_DOT_ENGINE_INL_H_\n#define MSHADOW_DOT_ENGINE_INL_H_\n\n#include <vector>\n#include \"./base.h\"\n#include \"./extension/implicit_gemm.h\"\n\n#ifdef __CUDACC__\n#include \"./cuda/tensor_gpu-inl.cuh\"\n#endif  // #ifdef __CUDACC__\n\nnamespace mshadow {\n /*!\n* \\brief CPU/GPU: Get a batched view of the src array. dst[i] = src + i * stride\n* \\param dst 2D pointer\n* \\param src 1D pointer\n* \\param num number of batches\n* \\param stride size of each batch\n* \\param stream\n*/\ntemplate<typename Device, typename DType>\ninline void GetBatchedView(DType **dst, DType *src, int num, int stride,\n                           Stream<Device> *stream);\ntemplate<typename DType>\ninline void GetBatchedView(DType **dst, DType *src, int num, int stride,\n                           Stream<cpu> *stream) {\n  for (int i = 0; i < num; i++) {\n    dst[i] = src + i * stride;\n  }\n}\n#ifdef __CUDACC__\nnamespace cuda {};\ntemplate<typename DType>\ninline void GetBatchedView(DType **dst, DType *src, int num, int stride,\n                           Stream<gpu> *stream) {\n  cuda::GetBatchedView(dst, src, num, stride, stream);\n}\n#endif  // #ifdef __CUDACC__\n\nnamespace expr {\n//---------------------------------------------------------------------\n// Matrix Multiplications, depends on BLAS Engine\n//---------------------------------------------------------------------\ntemplate<typename SV, typename Device, int ddim, int ldim,\n         int rdim, bool ltrans, bool rtrans, typename DType>\nstruct DotEngine {\n  inline static void Eval(Tensor<Device, ddim, DType> *p_dst,\n                          const Tensor<Device, ldim, DType> &lhs,\n                          const Tensor<Device, rdim, DType> &rhs,\n                          DType scale);\n};\n// handles the dot, use CblasColMajor\ntemplate<typename Device, typename DType = default_real_t>\nstruct BLASEngine {\n  inline static bool GetT(bool t) {\n    return t ? true : false;\n  }\n  inline static void SetStream(Stream<Device> *stream) {\n  }\n  inline static void gemm(Stream<Device> *stream,\n                          bool transa, bool transb,\n                          int m, int n, int k, DType alpha,\n                          const DType *A, int lda, const DType *B, int ldb,\n                          DType beta, DType *C, int ldc) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void batched_gemm(Stream<Device> *stream,\n                                  bool transa, bool transb,\n                                  int m, int n, int k, DType alpha,\n                                  const DType *A, int lda, const DType *B, int ldb,\n                                  DType beta, DType *C, int ldc, int batch_count,\n                                  DType **workspace) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void gemv(Stream<Device> *stream,\n                          bool trans, int m, int n,\n                          DType alpha, const DType *A, int lda,\n                          const DType *X, int incX,\n                          DType beta, DType *Y, int incY) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void batched_gemv(Stream<Device> *stream,\n                                  bool trans, int m, int n,\n                                  DType alpha, const DType *A, int lda,\n                                  const DType *X, int incX,\n                                  DType beta, DType *Y, int incY, int batch_count) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void ger(Stream<Device> *stream,\n                         int m, int n, DType alpha,\n                         const DType *X, int incX,\n                         const DType *Y, int incY, DType *A, int lda) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void batched_ger(Stream<Device> *stream,\n                         int m, int n, DType alpha,\n                         const DType *X, int incX,\n                         const DType *Y, int incY, DType *A, int lda, int batch_count) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void dot(Stream<Device> *stream,\n                         int n,\n                         const DType* X, int incX,\n                         const DType* Y, int incY,\n                         DType* ret) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n};\n\n#if MSHADOW_STAND_ALONE\ntemplate<>\nstruct BLASEngine<cpu, float> {\n  inline static bool GetT(bool t) {\n    return t ? true : false;\n  }\n  inline static void SetStream(Stream<cpu> *stream) {\n  }\n  inline static void gemm(Stream<cpu> *stream,\n                          bool transa, bool transb,\n                          int m, int n, int k, float alpha,\n                          const float *A, int lda, const float *B, int ldb,\n                          float beta, float *C, int ldc) {\n    if (alpha == 1.0f && beta == 0.0f) {\n      bool transpose_left = transb;\n      bool transpose_right = transa;\n      Tensor<cpu, 2, float> lhs((float*)B, Shape2(transpose_left ? k : n, transpose_left ? n : k));  // NOLINT(*)\n      Tensor<cpu, 2, float> rhs((float*)A, Shape2(transpose_right ? m : k, transpose_right ? k : m));  // NOLINT(*)\n      Tensor<cpu, 2, float> dst(C, Shape2(m, n));\n      if (!transpose_left && !transpose_right) {\n        dst = expr::implicit_dot(lhs, rhs); return;\n      } else if (!transpose_left && transpose_right) {\n        dst = expr::implicit_dot(lhs, rhs.T()); return;\n      } else if (transpose_left && !transpose_right) {\n        dst = expr::implicit_dot(lhs.T(), rhs); return;\n      } else {\n        LOG(FATAL) << \"Not implmented!\";\n      }\n    } else {\n      LOG(FATAL) << \"Not implmented!\";\n    }\n  }\n  inline static void batched_gemm(Stream<cpu> *stream,\n                                  bool transa, bool transb,\n                                  int m, int n, int k, float alpha,\n                                  const float *A, int lda, const float *B, int ldb,\n                                  float beta, float *C, int ldc, int batch_count,\n                                  float **workspace) {\n    for (int i = 0; i < batch_count; ++i) {\n      gemm(stream, transa, transb, m, n, k, alpha,\n           A + i * m * k, lda, B + i * k * n, ldb,\n           beta, C + i * m * n, ldc);\n    }\n  }\n  inline static void gemv(Stream<cpu> *stream,\n                          bool trans, int m, int n,\n                          float alpha, const float *A, int lda,\n                          const float *X, int incX,\n                          float beta, float *Y, int incY) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void batched_gemv(Stream<cpu> *stream,\n                                  bool trans, int m, int n,\n                                  float alpha, const float *A, int lda,\n                                  const float *X, int incX,\n                                  float beta, float *Y, int incY, int batch_count) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void ger(Stream<cpu> *stream,\n                         int m, int n, float alpha,\n                         const float *X, int incX,\n                         const float *Y, int incY, float *A, int lda) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void batched_ger(Stream<cpu> *stream,\n                         int m, int n, float alpha,\n                         const float *X, int incX,\n                         const float *Y, int incY, float *A, int lda, int batch_count) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void dot(Stream<cpu> *stream,\n                         int n,\n                         const float* X, int incX,\n                         const float* Y, int incY,\n                         float* ret) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n};\n\ntemplate<>\nstruct BLASEngine<cpu, double> {\n  inline static bool GetT(bool t) {\n    return t ? true : false;\n  }\n  inline static void SetStream(Stream<cpu> *stream) {\n  }\n  inline static void gemm(Stream<cpu> *stream,\n                          bool transa, bool transb,\n                          int m, int n, int k, double alpha,\n                          const double *A, int lda, const double *B, int ldb,\n                          double beta, double *C, int ldc) {\n    if (alpha == 1.0f && beta == 0.0f) {\n      bool transpose_left = transb;\n      bool transpose_right = transa;\n      Tensor<cpu, 2, double> lhs((double*)B, Shape2(transpose_left ? k : n, transpose_left ? n : k));  // NOLINT(*)\n      Tensor<cpu, 2, double> rhs((double*)A, Shape2(transpose_right ? m : k, transpose_right ? k : m));  // NOLINT(*)\n      Tensor<cpu, 2, double> dst(C, Shape2(m, n));\n      if (!transpose_left && !transpose_right) {\n        dst = expr::implicit_dot(lhs, rhs); return;\n      } else if (!transpose_left && transpose_right) {\n        dst = expr::implicit_dot(lhs, rhs.T()); return;\n      } else if (transpose_left && !transpose_right) {\n        dst = expr::implicit_dot(lhs.T(), rhs); return;\n      } else {\n        LOG(FATAL) << \"Not implmented!\";\n      }\n    } else {\n      LOG(FATAL) << \"Not implmented!\";\n    }\n  }\n  inline static void batched_gemm(Stream<cpu> *stream,\n                                  bool transa, bool transb,\n                                  int m, int n, int k, double alpha,\n                                  const double *A, int lda, const double *B, int ldb,\n                                  double beta, double *C, int ldc, int batch_count,\n                                  double **workspace) {\n    for (int i = 0; i < batch_count; ++i) {\n      gemm(stream, transa, transb, m, n, k, alpha,\n           A + i * m * k, lda, B + i * k * n, ldb,\n           beta, C + i * m * n, ldc);\n    }\n  }\n  inline static void gemv(Stream<cpu> *stream,\n                          bool trans, int m, int n,\n                          double alpha, const double *A, int lda,\n                          const double *X, int incX,\n                          double beta, double *Y, int incY) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void batched_gemv(Stream<cpu> *stream,\n                                  bool trans, int m, int n,\n                                  double alpha, const double *A, int lda,\n                                  const double *X, int incX,\n                                  double beta, double *Y, int incY, int batch_count) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void ger(Stream<cpu> *stream,\n                         int m, int n, double alpha,\n                         const double *X, int incX,\n                         const double *Y, int incY, double *A, int lda) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void batched_ger(Stream<cpu> *stream,\n                         int m, int n, double alpha,\n                         const double *X, int incX,\n                         const double *Y, int incY, double *A, int lda, int batch_count) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void dot(Stream<cpu> *stream,\n                         int n,\n                         const double* X, int incX,\n                         const double* Y, int incY,\n                         double* ret) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n};\n\n#elif (MSHADOW_USE_MKL || MSHADOW_USE_CBLAS)  // NOLINT(*)\ntemplate<>\nstruct BLASEngine<cpu, float> {\n  inline static CBLAS_TRANSPOSE GetT(bool t) {\n    return t ? CblasTrans : CblasNoTrans;\n  }\n  inline static void SetStream(Stream<cpu> *stream) {\n  }\n  inline static void gemm(Stream<cpu> *stream,\n                          bool transa, bool transb,\n                          index_t m, index_t n, index_t k, float alpha,\n                          const float *A, index_t lda, const float *B, index_t ldb,\n                          float beta, float *C, index_t ldc) {\n    cblas_sgemm(CblasColMajor, GetT(transa), GetT(transb),\n                m, n, k, alpha, A, lda, B, ldb, beta, C, ldc);\n  }\n  inline static void batched_gemm(Stream<cpu> *stream,\n                                  bool transa, bool transb,\n                                  index_t m, index_t n, index_t k, float alpha,\n                                  const float *A, index_t lda, const float *B, index_t ldb,\n                                  float beta, float *C, index_t ldc, index_t batch_count,\n                                  float **workspace) {\n#if (MSHADOW_USE_MKL && INTEL_MKL_VERSION >= 20160000)\n  // since same m/n/k is used for all single gemms, so we put all gemms into one group\n  const int GROUP_SIZE = 1;\n  MKL_INT p_m[GROUP_SIZE] = {static_cast<MKL_INT>(m)};\n  MKL_INT p_n[GROUP_SIZE] = {static_cast<MKL_INT>(n)};\n  MKL_INT p_k[GROUP_SIZE] = {static_cast<MKL_INT>(k)};\n  MKL_INT p_lda[GROUP_SIZE] = {static_cast<MKL_INT>(lda)};\n  MKL_INT p_ldb[GROUP_SIZE] = {static_cast<MKL_INT>(ldb)};\n  MKL_INT p_ldc[GROUP_SIZE] = {static_cast<MKL_INT>(ldc)};\n\n  float p_alpha[GROUP_SIZE] = {alpha};\n  float p_beta[GROUP_SIZE] = {beta};\n\n  CBLAS_TRANSPOSE cblas_a_trans = GetT(transa);\n  CBLAS_TRANSPOSE cblas_b_trans = GetT(transb);\n\n  MKL_INT p_group_sizeb[GROUP_SIZE] = {static_cast<MKL_INT>(batch_count)};\n  CBLAS_TRANSPOSE p_transa[GROUP_SIZE] = {cblas_a_trans};\n  CBLAS_TRANSPOSE p_transb[GROUP_SIZE] = {cblas_b_trans};\n\n  std::vector<const float*> pp_A(batch_count, nullptr);\n  std::vector<const float*> pp_B(batch_count, nullptr);\n  std::vector<float*> pp_C(batch_count, nullptr);\n\n  auto m_k = m * k;\n  auto k_n = k * n;\n  auto m_n = m * n;\n\n  for (int i = 0; i < batch_count; i++) {\n    pp_A[i] = A + i * m_k;\n    pp_B[i] = B + i * k_n;\n    pp_C[i] = C + i * m_n;\n  }\n\n  cblas_sgemm_batch(CblasColMajor, p_transa, p_transb,\n                    p_m, p_n, p_k, p_alpha, pp_A.data(), p_lda, pp_B.data(),\n                    p_ldb, p_beta, pp_C.data(), p_ldc, GROUP_SIZE, p_group_sizeb);\n#else\n    for (int i = 0; i < batch_count; ++i) {\n      gemm(stream, transa, transb, m, n, k, alpha,\n           A + i * m * k, lda, B + i * k * n, ldb,\n           beta, C + i * m * n, ldc);\n    }\n#endif\n  }\n  inline static void gemv(Stream<cpu> *stream,\n                          bool trans, int m, int n,\n                          float alpha, const float *A, int lda,\n                          const float *X, int incX,\n                          float beta, float *Y, int incY) {\n    cblas_sgemv(CblasColMajor, GetT(trans), m, n, alpha,\n                A, lda, X, incX, beta, Y, incY);\n  }\n  inline static void batched_gemv(Stream<cpu> *stream,\n                                  bool trans, int m, int n,\n                                  float alpha, const float *A, int lda,\n                                  const float *X, int incX,\n                                  float beta, float *Y, int incY, int batch_count) {\n    for (int i = 0; i < batch_count; ++i) {\n      gemv(stream, trans, m, n, alpha, A + i * m * n, lda,\n           X + i * (trans ? m : n) * incX, incX,\n           beta, Y + i * (trans ? n : m) * incY, incY);\n    }\n  }\n  inline static void ger(Stream<cpu> *stream,\n                         int m, int n, float alpha,\n                         const float *X, int incX,\n                         const float *Y, int incY, float *A, int lda) {\n    cblas_sger(CblasColMajor, m, n, alpha, X, incX, Y, incY, A, lda);\n  }\n  inline static void batched_ger(Stream<cpu> *stream,\n                         int m, int n, float alpha,\n                         const float *X, int incX,\n                         const float *Y, int incY, float *A, int lda, int batch_count) {\n    for (int i = 0; i < batch_count; ++i) {\n      ger(stream, m, n, alpha, X + i * m * incX, incX, Y + i * n * incY, incY,\n          A + i * lda * n, lda);\n    }\n  }\n  inline static void dot(Stream<cpu> *stream,\n                         int n,\n                         const float* X, int incX,\n                         const float* Y, int incY,\n                         float* ret) {\n    *ret = cblas_sdot(n, X, incX, Y, incY);\n  }\n};\n\ntemplate<>\nstruct BLASEngine<cpu, double> {\n  inline static CBLAS_TRANSPOSE GetT(bool t) {\n    return t ? CblasTrans : CblasNoTrans;\n  }\n  inline static void SetStream(Stream<cpu> *stream) {\n  }\n  inline static void gemm(Stream<cpu> *stream,\n                          bool transa, bool transb,\n                          index_t m, index_t n, index_t k, double alpha,\n                          const double *A, index_t lda, const double *B, index_t ldb,\n                          double beta, double *C, index_t ldc) {\n    cblas_dgemm(CblasColMajor, GetT(transa), GetT(transb),\n                m, n, k, alpha, A, lda, B, ldb, beta, C, ldc);\n  }\n  inline static void batched_gemm(Stream<cpu> *stream,\n                                  bool transa, bool transb,\n                                  index_t m, index_t n, index_t k, double alpha,\n                                  const double *A, index_t lda, const double *B, index_t ldb,\n                                  double beta, double *C, index_t ldc, index_t batch_count,\n                                  double **workspace) {\n#if (MSHADOW_USE_MKL && INTEL_MKL_VERSION >= 20160000)\n  // since same m/n/k is used for all single gemms, so we put all gemms into one group\n  const int GROUP_SIZE = 1;\n  MKL_INT p_m[GROUP_SIZE] = {static_cast<MKL_INT>(m)};\n  MKL_INT p_n[GROUP_SIZE] = {static_cast<MKL_INT>(n)};\n  MKL_INT p_k[GROUP_SIZE] = {static_cast<MKL_INT>(k)};\n  MKL_INT p_lda[GROUP_SIZE] = {static_cast<MKL_INT>(lda)};\n  MKL_INT p_ldb[GROUP_SIZE] = {static_cast<MKL_INT>(ldb)};\n  MKL_INT p_ldc[GROUP_SIZE] = {static_cast<MKL_INT>(ldc)};\n\n  double p_alpha[GROUP_SIZE] = {alpha};\n  double p_beta[GROUP_SIZE] = {beta};\n\n  CBLAS_TRANSPOSE cblas_a_trans = GetT(transa);\n  CBLAS_TRANSPOSE cblas_b_trans = GetT(transb);\n\n  MKL_INT p_group_sizeb[GROUP_SIZE] = {static_cast<MKL_INT>(batch_count)};\n  CBLAS_TRANSPOSE p_transa[GROUP_SIZE] = {cblas_a_trans};\n  CBLAS_TRANSPOSE p_transb[GROUP_SIZE] = {cblas_b_trans};\n\n  std::vector<const double*> pp_A(batch_count, nullptr);\n  std::vector<const double*> pp_B(batch_count, nullptr);\n  std::vector<double*> pp_C(batch_count, nullptr);\n\n  auto m_k = m * k;\n  auto k_n = k * n;\n  auto m_n = m * n;\n\n  for (int i = 0; i < batch_count; i++) {\n    pp_A[i] = A + i * m_k;\n    pp_B[i] = B + i * k_n;\n    pp_C[i] = C + i * m_n;\n  }\n\n  cblas_dgemm_batch(CblasColMajor, p_transa, p_transb,\n                    p_m, p_n, p_k, p_alpha, pp_A.data(), p_lda, pp_B.data(),\n                    p_ldb, p_beta, pp_C.data(), p_ldc, GROUP_SIZE, p_group_sizeb);\n#else\n    for (int i = 0; i < batch_count; ++i) {\n      gemm(stream, transa, transb, m, n, k, alpha,\n           A + i * m * k, lda, B + i * k * n, ldb,\n           beta, C + i * m * n, ldc);\n    }\n#endif\n  }\n  inline static void gemv(Stream<cpu> *stream,\n                          bool trans, int m, int n, double alpha,\n                          const double *A, int lda,\n                          const double *X, int incX,\n                          double beta, double *Y, int incY) {\n    cblas_dgemv(CblasColMajor, GetT(trans), m, n, alpha,\n                A, lda, X, incX, beta, Y, incY);\n  }\n  inline static void batched_gemv(Stream<cpu> *stream,\n                                  bool trans, int m, int n,\n                                  double alpha, const double *A, int lda,\n                                  const double *X, int incX,\n                                  double beta, double *Y, int incY, int batch_count) {\n    for (int i = 0; i < batch_count; ++i) {\n      gemv(stream, trans, m, n, alpha, A + i * m * n, lda,\n           X + i * (trans ? m : n) * incX, incX,\n           beta, Y + i * (trans ? n : m) * incY, incY);\n    }\n  }\n  inline static void ger(Stream<cpu> *stream,\n                         int m, int n, double alpha,\n                         const double *X, int incX,\n                         const double *Y, int incY, double *A, int lda) {\n    cblas_dger(CblasColMajor, m, n, alpha, X, incX, Y, incY, A, lda);\n  }\n  inline static void batched_ger(Stream<cpu> *stream,\n                         int m, int n, double alpha,\n                         const double *X, int incX,\n                         const double *Y, int incY, double *A, int lda, int batch_count) {\n    for (int i = 0; i < batch_count; ++i) {\n      ger(stream, m, n, alpha, X + i * m * incX, incX, Y + i * n * incY, incY,\n          A + i * lda * n, lda);\n    }\n  }\n  inline static void dot(Stream<cpu> *stream,\n                         int n,\n                         const double* X, int incX,\n                         const double* Y, int incY,\n                         double* ret) {\n    *ret = cblas_ddot(n, X, incX, Y, incY);\n  }\n};\n#endif  // MSHADOW_USE_CBLAS || MSHADOW_USE_MKL || MSHADOW_STAND_ALONE\n// CuBLAS redirect code\n#if MSHADOW_USE_CUDA\n// All CuBLAS goes to here, use legacy API: not threadsafe\ntemplate<>\nstruct BLASEngine<gpu, half::half_t> {\n  inline static cublasOperation_t GetT(bool t) {\n    return t ? CUBLAS_OP_T : CUBLAS_OP_N;\n  }\n  inline static void SetStream(Stream<gpu> *stream) {\n    cublasStatus_t err = cublasSetStream(Stream<gpu>::GetBlasHandle(stream),\n                    Stream<gpu>::GetStream(stream));\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas set stream fail\";\n  }\n  inline static void gemm(Stream<gpu> *stream,\n                          bool transa, bool transb,\n                          int m, int n, int k, half::half_t alpha,\n                          const half::half_t *A, int lda,\n                          const half::half_t *B, int ldb, half::half_t beta,\n                          half::half_t *C, int ldc) {\n#if defined(CUDA_VERSION) && CUDA_VERSION >= 7050\n  // Always use pseudo-fp16: fp32 compute with fp16 I/O.\n  float alpha_f = float(alpha);  // NOLINT(*)\n  float beta_f = float(beta);  // NOLINT(*)\n  #if CUDA_VERSION >= 8000\n    cublasStatus_t err = cublasSgemmEx(Stream<gpu>::GetBlasHandle(stream),\n                                       GetT(transa), GetT(transb), m, n, k, &alpha_f,\n                                       A, CUDA_R_16F, lda, B, CUDA_R_16F,\n                                       ldb, &beta_f, C, CUDA_R_16F, ldc);\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas SgemmEx fail\";\n  #else\n    cublasStatus_t err = cublasSgemmEx(Stream<gpu>::GetBlasHandle(stream),\n                                       GetT(transa), GetT(transb), m, n, k, &alpha_f,\n                                       A, CUBLAS_DATA_HALF, lda, B, CUBLAS_DATA_HALF,\n                                       ldb, &beta_f, C, CUBLAS_DATA_HALF, ldc);\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas SgemmEx fail\";\n  #endif  // CUDA_VERSION >= 8000\n#else\n    LOG(FATAL) << \"Require CUDA version >= 7.5!\";\n#endif  // defined(CUDA_VERSION) && CUDA_VERSION >= 7050\n  }\n  inline static void batched_gemm(Stream<gpu> *stream,\n                                  bool transa, bool transb,\n                                  int m, int n, int k, half::half_t alpha,\n                                  const half::half_t *A, int lda, const half::half_t *B, int ldb,\n                                  half::half_t beta, half::half_t *C, int ldc, int batch_count,\n                                  half::half_t **workspace) {\n#if defined(__CUDACC__) && CUDA_VERSION >= 9000\n    int major = stream->prop.major;\n    int minor = stream->prop.minor;\n    // fp16 is not supported before ARCH 53\n    if ((major > 5) || (major == 5 && minor >= 3)) {\n      const __half* A_h = reinterpret_cast<const __half*>(A);\n      const __half* B_h = reinterpret_cast<const __half*>(B);\n      __half* alpha_h = reinterpret_cast<__half*>(&alpha);\n      __half* beta_h = reinterpret_cast<__half*>(&beta);\n      __half* C_h = reinterpret_cast<__half*>(C);\n      cublasStatus_t err = cublasHgemmStridedBatched(Stream<gpu>::GetBlasHandle(stream),\n        GetT(transa), GetT(transb), m, n, k, alpha_h,\n        A_h, lda, m * k,\n        B_h, ldb, k * n,\n        beta_h, C_h, ldc, m * n,\n        batch_count);\n      CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas: HgemmStridedBatched fail\";\n      return;\n    }\n#endif\n    for (int i = 0; i < batch_count; ++i) {\n      gemm(stream, transa, transb, m, n, k, alpha,\n           A + i * m * k, lda, B + i * k * n, ldb,\n           beta, C + i * m * n, ldc);\n    }\n  }\n  inline static void gemv(Stream<gpu> *stream,\n                          bool trans, int m, int n, half::half_t alpha,\n                          const half::half_t *A, int lda,\n                          const half::half_t *X, int incX, half::half_t beta,\n                          half::half_t *Y, int incY) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void batched_gemv(Stream<gpu> *stream,\n                                  bool trans, int m, int n,\n                                  half::half_t alpha, const half::half_t *A, int lda,\n                                  const half::half_t *X, int incX,\n                                  half::half_t beta, half::half_t *Y, int incY, int batch_count) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void ger(Stream<gpu> *stream,\n                         int m, int n, half::half_t alpha,\n                         const half::half_t *X, int incX,\n                         const half::half_t *Y, int incY, half::half_t *A, int lda) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void batched_ger(Stream<gpu> *stream,\n                         int m, int n, half::half_t alpha,\n                         const half::half_t *X, int incX, const half::half_t *Y, int incY,\n                         half::half_t *A, int lda, int batch_count) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n  inline static void dot(Stream<gpu> *stream,\n                         int n,\n                         const half::half_t* X, int incX,\n                         const half::half_t* Y, int incY,\n                         half::half_t *ret) {\n    LOG(FATAL) << \"Not implmented!\";\n  }\n};\n\ntemplate<>\nstruct BLASEngine<gpu, float> {\n  inline static cublasOperation_t GetT(bool t) {\n    return t ? CUBLAS_OP_T : CUBLAS_OP_N;\n  }\n  inline static void SetStream(Stream<gpu> *stream) {\n    cublasStatus_t err = cublasSetStream(Stream<gpu>::GetBlasHandle(stream),\n                    Stream<gpu>::GetStream(stream));\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas: set stream fail\";\n  }\n  inline static void gemm(Stream<gpu> *stream,\n                          bool transa, bool transb,\n                          int m, int n, int k, float alpha,\n                          const float *A, int lda,\n                          const float *B, int ldb, float beta,\n                          float *C, int ldc) {\n    cublasStatus_t err = cublasSgemm(Stream<gpu>::GetBlasHandle(stream),\n                GetT(transa), GetT(transb), m, n, k, &alpha,\n                A, lda, B, ldb, &beta, C, ldc);\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas: Sgemm fail\";\n  }\n  inline static void batched_gemm(Stream<gpu> *stream,\n                                  bool transa, bool transb,\n                                  int m, int n, int k, float alpha,\n                                  const float *A, int lda, const float *B, int ldb,\n                                  float beta, float *C, int ldc, int batch_count,\n                                  float **workspace) {\n#if defined(__CUDACC__) && CUDA_VERSION >= 4010 && CUDA_VERSION < 8000\n    // Cast DType* to DType** using workspace as a buffer\n    bool alloc_workspace = false;\n    if (workspace == NULL) {\n      // Allocate the workspace if it's NULL.\n      // TODO(sxjscience) Try to move the allocation inside Tensor, which is thread-safe.\n      cudaMalloc(reinterpret_cast<void**>(&workspace), 3 * batch_count * sizeof(float*));\n      alloc_workspace = true;\n    }\n    GetBatchedView(workspace, const_cast<float*>(A), batch_count, m * k, stream);\n    GetBatchedView(workspace + batch_count,\n                   const_cast<float*>(B), batch_count, k * n, stream);\n    GetBatchedView(workspace + 2 * batch_count, C, batch_count, m * n, stream);\n    cublasStatus_t err = cublasSgemmBatched(Stream<gpu>::GetBlasHandle(stream),\n                                            GetT(transa), GetT(transb), m, n, k, &alpha,\n                                            (const float**)workspace, lda,\n                                            (const float**)(workspace + batch_count), ldb,\n                                            &beta, workspace + 2 * batch_count, ldc, batch_count);\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas: SgemmBatched fail\";\n    if (alloc_workspace) {\n      cudaFree(workspace);\n    }\n#elif defined(__CUDACC__) && CUDA_VERSION >= 8000\n    cublasStatus_t err = cublasSgemmStridedBatched(Stream<gpu>::GetBlasHandle(stream),\n      GetT(transa), GetT(transb), m, n, k, &alpha,\n      A, lda, m * k,\n      B, ldb, k * n,\n      &beta, C, ldc, m * n,\n      batch_count);\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas: SgemmStridedBatched fail\";\n#else\n    for (int i = 0; i < batch_count; ++i) {\n      gemm(stream, transa, transb, m, n, k, alpha,\n           A + i * m * k, lda, B + i * k * n, ldb,\n           beta, C + i * m * n, ldc);\n    }\n#endif  // defined(__CUDACC__) && CUDA_VERSION >= 4010\n  }\n  inline static void gemv(Stream<gpu> *stream,\n                          bool trans, int m, int n, float alpha,\n                          const float *A, int lda,\n                          const float *X, int incX, float beta,\n                          float *Y, int incY) {\n    cublasStatus_t err = cublasSgemv(Stream<gpu>::GetBlasHandle(stream),\n                GetT(trans), m, n, &alpha, A, lda, X, incX, &beta, Y, incY);\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas: Sgemv fail\";\n  }\n  inline static void batched_gemv(Stream<gpu> *stream,\n                                  bool trans, int m, int n,\n                                  float alpha, const float *A, int lda,\n                                  const float *X, int incX,\n                                  float beta, float *Y, int incY, int batch_count) {\n    for (int i = 0; i < batch_count; ++i) {\n      gemv(stream, trans, m, n, alpha, A + i * m * n, lda,\n           X + i * (trans ? m : n) * incX, incX,\n           beta, Y + i * (trans ? n : m) * incY, incY);\n    }\n  }\n  inline static void ger(Stream<gpu> *stream,\n                         int m, int n, float alpha,\n                         const float *X, int incX,\n                         const float *Y, int incY, float *A, int lda) {\n    cublasStatus_t err = cublasSger(Stream<gpu>::GetBlasHandle(stream),\n                                    m, n, &alpha, X, incX, Y, incY, A, lda);\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas: Sger fail\";\n  }\n  inline static void batched_ger(Stream<gpu> *stream,\n                         int m, int n, float alpha,\n                         const float *X, int incX,\n                         const float *Y, int incY, float *A, int lda, int batch_count) {\n    for (int i = 0; i < batch_count; ++i) {\n      ger(stream, m, n, alpha, X + i * m * incX, incX, Y + i * n * incY, incY,\n          A + i * lda * n, lda);\n    }\n  }\n  inline static void dot(Stream<gpu> *stream,\n                         int n,\n                         const float* X, int incX,\n                         const float* Y, int incY,\n                         float *ret) {\n    cublasSetPointerMode(Stream<gpu>::GetBlasHandle(stream),\n                         CUBLAS_POINTER_MODE_DEVICE);\n    cublasStatus_t err = cublasSdot(Stream<gpu>::GetBlasHandle(stream),\n                                    n, X, incX, Y, incY, ret);\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas: Dot fail\";\n    cublasSetPointerMode(Stream<gpu>::GetBlasHandle(stream),\n                         CUBLAS_POINTER_MODE_HOST);\n  }\n};\n\ntemplate<>\nstruct BLASEngine<gpu, double> {\n  inline static cublasOperation_t GetT(bool t) {\n    return t ? CUBLAS_OP_T : CUBLAS_OP_N;\n  }\n  inline static void SetStream(Stream<gpu> *stream) {\n    cublasStatus_t err = cublasSetStream(Stream<gpu>::GetBlasHandle(stream),\n                    Stream<gpu>::GetStream(stream));\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas: set stream fail\";\n  }\n  inline static void gemm(Stream<gpu> *stream,\n                          bool transa, bool transb,\n                          int m, int n, int k, double alpha,\n                          const double *A, int lda,\n                          const double *B, int ldb,\n                          double beta, double *C, int ldc) {\n    cublasStatus_t err = cublasDgemm(Stream<gpu>::GetBlasHandle(stream),\n                GetT(transa), GetT(transb), m, n, k, &alpha,\n                A, lda, B, ldb, &beta, C, ldc);\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas: Dgemm fail\";\n  }\n  inline static void batched_gemm(Stream<gpu> *stream,\n                                  bool transa, bool transb,\n                                  int m, int n, int k, double alpha,\n                                  const double *A, int lda, const double *B, int ldb,\n                                  double beta, double *C, int ldc, int batch_count,\n                                  double **workspace) {\n#if defined(__CUDACC__) && CUDA_VERSION >= 4010 && CUDA_VERSION < 8000\n    // Cast DType* to DType** using workspace as a buffer\n    bool alloc_workspace = false;\n    if (workspace == NULL) {\n      // Allocate the workspace if it's NULL.\n      // TODO(sxjscience) Try to move the allocation inside Tensor, which is thread-safe.\n      cudaMalloc(reinterpret_cast<void**>(&workspace), 3 * batch_count * sizeof(double*));\n      alloc_workspace = true;\n    }\n    GetBatchedView(workspace, const_cast<double*>(A), batch_count, m * k, stream);\n    GetBatchedView(workspace + batch_count,\n                   const_cast<double*>(B), batch_count, k * n, stream);\n    GetBatchedView(workspace + 2 * batch_count, C, batch_count, m * n, stream);\n    cublasStatus_t err = cublasDgemmBatched(Stream<gpu>::GetBlasHandle(stream),\n                                            GetT(transa), GetT(transb), m, n, k, &alpha,\n                                            (const double**)workspace, lda,\n                                            (const double**)(workspace + batch_count), ldb,\n                                            &beta, workspace + 2 * batch_count, ldc, batch_count);\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas: DgemmBatched fail\";\n    if (alloc_workspace) {\n      cudaFree(workspace);\n    }\n#elif defined(__CUDACC__) && CUDA_VERSION >= 8000\n    cublasStatus_t err = cublasDgemmStridedBatched(Stream<gpu>::GetBlasHandle(stream),\n      GetT(transa), GetT(transb), m, n, k, &alpha,\n      A, lda, m * k,\n      B, ldb, k * n,\n      &beta, C, ldc, m * n,\n      batch_count);\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas: DgemmStridedBatched fail\";\n#else\n    for (int i = 0; i < batch_count; ++i) {\n      gemm(stream, transa, transb, m, n, k, alpha,\n           A + i * m * k, lda, B + i * k * n, ldb,\n           beta, C + i * m * n, ldc);\n    }\n#endif  // defined(__CUDACC__) && CUDA_VERSION >= 4010\n  }\n  inline static void gemv(Stream<gpu> *stream,\n                          bool trans, int m, int n, double alpha,\n                          const double *A, int lda,\n                          const double *X, int incX,\n                          double beta, double *Y, int incY) {\n    cublasStatus_t err = cublasDgemv(Stream<gpu>::GetBlasHandle(stream),\n                GetT(trans), m, n, &alpha, A, lda, X, incX, &beta, Y, incY);\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas: Dgemv fail\";\n  }\n  inline static void batched_gemv(Stream<gpu> *stream,\n                                  bool trans, int m, int n,\n                                  double alpha, const double *A, int lda,\n                                  const double *X, int incX,\n                                  double beta, double *Y, int incY, int batch_count) {\n    for (int i = 0; i < batch_count; ++i) {\n      gemv(stream, trans, m, n, alpha, A + i * m * n, lda,\n           X + i * (trans ? m : n) * incX, incX,\n           beta, Y + i * (trans ? n : m) * incY, incY);\n    }\n  }\n  inline static void ger(Stream<gpu> *stream,\n                         int m, int n, double alpha,\n                         const double *X, int incX,\n                         const double *Y, int incY, double *A, int lda) {\n    cublasStatus_t err = cublasDger(Stream<gpu>::GetBlasHandle(stream),\n                                    m, n, &alpha, X, incX, Y, incY, A, lda);\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas: Dger fail\";\n  }\n  inline static void batched_ger(Stream<gpu> *stream,\n                         int m, int n, double alpha,\n                         const double *X, int incX,\n                         const double *Y, int incY, double *A, int lda, int batch_count) {\n    for (int i = 0; i < batch_count; ++i) {\n      ger(stream, m, n, alpha, X + i * m * incX, incX, Y + i * n * incY, incY,\n          A + i * lda * n, lda);\n    }\n  }\n  inline static void dot(Stream<gpu> *stream,\n                         int n,\n                         const double* X, int incX,\n                         const double* Y, int incY,\n                         double *ret) {\n    cublasSetPointerMode(Stream<gpu>::GetBlasHandle(stream),\n                         CUBLAS_POINTER_MODE_DEVICE);\n    cublasStatus_t err = cublasDdot(Stream<gpu>::GetBlasHandle(stream),\n                                    n, X, incX, Y, incY, ret);\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Cublas: Dot fail\";\n    cublasSetPointerMode(Stream<gpu>::GetBlasHandle(stream),\n                         CUBLAS_POINTER_MODE_HOST);\n  }\n};\n#endif  // MSHADOW_USE_CUDA\n// helper function to decide which shape we are in\ninline Shape<2> GetShape(const Shape<2> &shape, bool transpose) {\n  return transpose ? Shape2(shape[1], shape[0]) : shape;\n}\n// dst = dot(lhs[.T], rhs[.T])\ntemplate<typename SV, typename xpu,\n         bool transpose_left, bool transpose_right, typename DType>\nstruct DotEngine<SV, xpu, 2, 2, 2, transpose_left, transpose_right, DType> {\n  inline static void Eval(Tensor<xpu, 2, DType> *p_dst,\n                          const Tensor<xpu, 2, DType> &lhs,\n                          const Tensor<xpu, 2, DType> &rhs,\n                          DType scale) {\n    Tensor<xpu, 2, DType> &dst = *p_dst;\n#if MSHADOW_STAND_ALONE\n    if (xpu::kDevMask == cpu::kDevMask && scale == 1.0f) {\n      if (!transpose_left && !transpose_right) {\n        dst = expr::implicit_dot(lhs, rhs); return;\n      } else if (!transpose_left && transpose_right) {\n        dst = expr::implicit_dot(lhs, rhs.T()); return;\n      } else if (transpose_left && !transpose_right) {\n        dst = expr::implicit_dot(lhs.T(), rhs); return;\n      }\n    }\n#endif\n    // set kernel stream\n    // if there is no stream, crush\n    BLASEngine<xpu, DType>::SetStream(dst.stream_);\n    Shape<2> sleft = GetShape(lhs.shape_, transpose_left);\n    Shape<2> sright = GetShape(rhs.shape_, transpose_right);\n    CHECK(dst.size(0) == sleft[0] && dst.size(1) == sright[1] && sleft[1] == sright[0])\n      << \"dot-gemm: matrix shape mismatch\";\n    // use column major argument to compatible with most BLAS\n    BLASEngine<xpu, DType>::gemm\n        (dst.stream_,\n         transpose_right , transpose_left,\n         transpose_right ? rhs.size(0) : rhs.size(1),\n         transpose_left  ? lhs.size(1) : lhs.size(0),\n         transpose_right ? rhs.size(1) : rhs.size(0),\n         DType(scale * SV::AlphaBLAS()),\n         rhs.dptr_, rhs.stride_,\n         lhs.dptr_, lhs.stride_,\n         DType(SV::BetaBLAS()),\n         dst.dptr_, dst.stride_);\n  }\n};\ntemplate<typename SV, typename xpu, bool transpose_right, typename DType>\nstruct DotEngine<SV, xpu, 1, 1, 2, false, transpose_right, DType> {\n  inline static void Eval(Tensor<xpu, 1, DType> *p_dst,\n                          const Tensor<xpu, 1, DType> &lhs,\n                          const Tensor<xpu, 2, DType> &rhs,\n                          DType scale) {\n    Tensor<xpu, 1, DType> &dst = *p_dst;\n    // set kernel stream\n    // if there is no stream, crush\n    BLASEngine<xpu, DType>::SetStream(dst.stream_);\n    Shape<2> sright = GetShape(rhs.shape_, transpose_right);\n    CHECK(dst.size(0) == sright[1] && lhs.size(0) == sright[0])\n      << \"dot-gemv: matrix shape mismatch\"\n      << \"dst: \" << dst.shape_ << \"\\n\"\n      << \"lhs: \" << lhs.shape_ << \"\\n\"\n      << \"rhs: \" << sright << \"\\n\";\n    BLASEngine<xpu, DType>::gemv\n        (dst.stream_,\n         transpose_right,\n         rhs.size(1), rhs.size(0), scale * SV::AlphaBLAS(),\n         rhs.dptr_, rhs.stride_,\n         lhs.dptr_, 1, SV::BetaBLAS(),\n         dst.dptr_, 1);\n  }\n};\ntemplate<typename SV, typename xpu, typename DType>\nstruct DotEngine<SV, xpu, 2, 1, 1, true, false, DType> {\n  inline static void Eval(Tensor<xpu, 2, DType> *p_dst,\n                          const Tensor<xpu, 1, DType> &lhs,\n                          const Tensor<xpu, 1, DType> &rhs,\n                          DType scale) {\n    Tensor<xpu, 2, DType> &dst = *p_dst;\n    // set kernel stream\n    // if there is no stream, crush\n    BLASEngine<xpu, DType>::SetStream(dst.stream_);\n    CHECK(dst.size(0) == lhs.size(0) && dst.size(1) == rhs.size(0))\n      << \"dot-ger: matrix shape mismatch\"\n      << \"dst: \" << dst.shape_ << \"\\n\"\n      << \"lhs: \" << lhs.shape_ << \"\\n\"\n      << \"rhs: \" << rhs.shape_;\n    if (SV::BetaBLAS() == 0.0f) {\n      BLASEngine<xpu, DType>::ger\n          (dst.stream_, rhs.size(0), lhs.size(0), scale * SV::AlphaBLAS(),\n           rhs.dptr_, 1, lhs.dptr_, 1, dst.dptr_, dst.stride_);\n    } else {\n      DotEngine<SV, xpu, 2, 2, 2, true, false,\n                DType>::Eval(p_dst, lhs.FlatTo2D(), rhs.FlatTo2D(), scale);\n    }\n  }\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_DOT_ENGINE_INL_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/expr_engine-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file expr_engine-inl.h\n * \\brief definitions of how expressions should be evaluated\n * \\author Tianqi Chen, Bing Xu\n */\n#ifndef MSHADOW_EXPR_ENGINE_INL_H_\n#define MSHADOW_EXPR_ENGINE_INL_H_\n#include <utility>\n#include <algorithm>\n#include \"dmlc/logging.h\"\n#include \"./expression.h\"\n#include \"./tensor.h\"\n\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief a general class that allows extension that makes tensors of some shape\n * \\tparam SubType type of subclass\n * \\tparam SrcExp source expression of the MakeTensorExp, the source of operation\n * \\tparam dim dimension of the expression\n * \\tparam DType the type of elements\n */\ntemplate<typename SubType, typename SrcExp, int dim, typename DType>\nstruct MakeTensorExp\n    : public Exp<MakeTensorExp<SubType, SrcExp, dim, DType>,\n                 DType, type::kChainer> {\n  /*! \\brief the shape of this expression */\n  Shape<dim> shape_;\n  /*! \\brief true self of subtype */\n  inline const SubType& real_self(void) const{\n    return *static_cast<const SubType*>(this);\n  }\n};\n//----------------------------------------------------------------------\n// This part of code gives plan that can be used to carry out execution\n//---------------------------------------------------------------------\n// Declarations of plans\ntemplate<typename ExpType, typename DType>\nclass Plan {\n public:\n  /*!\n   * \\brief evaluate the expression at index [y][x]\n   *  to be implemented by SubType, for RValue, the return type will be DType &\n   */\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const;\n};\n// tensor plan\ntemplate <typename Device, int dim, typename DType>\nclass Plan<Tensor<Device, dim, DType>, DType> {\n public:\n  explicit Plan(const Tensor<Device, dim, DType> &t)\n      : dptr_(t.dptr_), stride_(t.stride_) {}\n  // for RValue, the return type should be reference\n  MSHADOW_XINLINE DType &REval(index_t y, index_t x) {\n    return dptr_[y * stride_ + x];\n  }\n  // const evaluation\n  MSHADOW_XINLINE const DType &Eval(index_t y, index_t x) const {\n    return dptr_[y * stride_ + x];\n  }\n\n private:\n  DType  *dptr_;\n  index_t stride_;\n};\n// special evaluation case for 1d tensor, no stride\ntemplate <typename Device, typename DType>\nclass Plan<Tensor<Device, 1, DType>, DType> {\n public:\n  explicit Plan(const Tensor<Device, 1, DType> &t) : dptr_(t.dptr_) {}\n  MSHADOW_XINLINE DType &REval(index_t y, index_t x) {\n    return dptr_[x];\n  }\n  MSHADOW_XINLINE const DType &Eval(index_t y, index_t x) const {\n    return dptr_[x];\n  }\n\n private:\n  DType  *dptr_;\n};\n// scalar\ntemplate<typename DType>\nclass Plan<ScalarExp<DType>, DType> {\n public:\n  explicit Plan(DType scalar) : scalar_(scalar) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    return scalar_;\n  }\n\n private:\n  DType scalar_;\n};\n// unary expression\ntemplate<typename DstDType, typename SrcDType,\n         typename EType, int etype>\nclass Plan<TypecastExp<DstDType, SrcDType, EType, etype>, DstDType> {\n public:\n  explicit Plan(const Plan<EType, SrcDType> &src) : src_(src) {}\n  MSHADOW_XINLINE DstDType Eval(index_t y, index_t x) const {\n    return DstDType(src_.Eval(y, x));  // NOLINT(*)\n  }\n\n private:\n  Plan<EType, SrcDType> src_;\n};\n\n// ternary expression\ntemplate<typename OP, typename TA, typename TB, typename TC, int etype, typename DType>\nclass Plan<TernaryMapExp<OP, TA, TB, TC, DType, etype>, DType> {\n public:\n  explicit Plan(const Plan<TA, DType> &item1, const Plan<TB, DType> &item2,\n       const Plan<TC, DType> &item3)\n      : item1_(item1), item2_(item2), item3_(item3) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    return OP::Map(item1_.Eval(y, x), item2_.Eval(y, x), item3_.Eval(y, x));\n  }\n\n private:\n  Plan<TA, DType> item1_;\n  Plan<TB, DType> item2_;\n  Plan<TC, DType> item3_;\n};\n// binary expression\ntemplate<typename OP, typename TA, typename TB, int etype, typename DType>\nclass Plan<BinaryMapExp<OP, TA, TB, DType, etype>, DType> {\n public:\n  explicit Plan(const Plan<TA, DType> &lhs, const Plan<TB, DType> &rhs)\n      : lhs_(lhs), rhs_(rhs) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    return OP::Map(lhs_.Eval(y, x), rhs_.Eval(y, x));\n  }\n\n private:\n  Plan<TA, DType> lhs_;\n  Plan<TB, DType> rhs_;\n};\n// unary expression\ntemplate<typename OP, typename TA, int etype, typename DType>\nclass Plan<UnaryMapExp<OP, TA, DType, etype>, DType> {\n public:\n  explicit Plan(const Plan<TA, DType> &src) : src_(src) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    return OP::Map(src_.Eval(y, x));\n  }\n\n private:\n  Plan<TA, DType> src_;\n};\n// remaps map tensor expression to subtype's plan\ntemplate<typename SubType, typename SrcExp, int dim, typename DType>\nstruct Plan<MakeTensorExp<SubType, SrcExp, dim, DType>, DType> {\n public:\n  Plan(const Plan<SubType, DType> &src) : src_(src) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    return src_.Eval(y, x);\n  }\n\n private:\n  Plan<SubType, DType> src_;\n};\n// tranpsoe\ntemplate<typename EType, typename DType>\nclass Plan<TransposeExp<EType, DType>, DType> {\n public:\n  explicit Plan(const Plan<EType, DType> &src) : src_(src) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    return src_.Eval(x, y);\n  }\n\n private:\n  Plan<EType, DType> src_;\n};\n//----------------------------------------------------------------------\n// Mappings from expression to plans\n//---------------------------------------------------------------------\ntemplate<typename OP, typename TA, typename TB, typename DType, int etype>\ninline Plan<BinaryMapExp<OP, TA, TB, DType, etype>, DType>\nMakePlan(const BinaryMapExp<OP, TA, TB, DType, etype> &e);\n\ntemplate<typename OP, typename TA, typename TB, typename TC, typename DType, int etype>\ninline Plan<TernaryMapExp<OP, TA, TB, TC, DType, etype>, DType>\nMakePlan(const TernaryMapExp<OP, TA, TB, TC, DType, etype> &e);\n\ntemplate<typename DType>\ninline Plan<ScalarExp<DType>, DType> MakePlan(const ScalarExp<DType> &e) {\n  return Plan<ScalarExp<DType>, DType>(e.scalar_);\n}\n\ntemplate<typename DstDType, typename SrcDType, typename EType, int etype>\ninline Plan<TypecastExp<DstDType, SrcDType, EType, etype>, DstDType>\nMakePlan(const TypecastExp<DstDType, SrcDType, EType, etype> &e) {\n  return Plan<TypecastExp<DstDType, SrcDType, EType, etype>, DstDType>(MakePlan(e.exp));\n}\n\ntemplate<typename T, typename DType>\ninline Plan<T, DType> MakePlan(const RValueExp<T, DType> &e) {\n  return Plan<T, DType>(e.self());\n}\n\ntemplate<typename T, typename DType>\ninline Plan<TransposeExp<T, DType>, DType>\nMakePlan(const TransposeExp<T, DType> &e) {\n  return Plan<TransposeExp<T, DType>, DType>(MakePlan(e.exp));\n}\n\ntemplate<typename T, typename SrcExp, int dim, typename DType>\ninline Plan<T, DType>\nMakePlan(const MakeTensorExp<T, SrcExp, dim, DType> &e) {\n  return Plan<T, DType>(e.real_self());\n}\n\ntemplate<typename OP, typename TA, typename DType, int etype>\ninline Plan<UnaryMapExp<OP, TA, DType, etype>, DType>\nMakePlan(const UnaryMapExp<OP, TA, DType, etype> &e) {\n  return Plan<UnaryMapExp<OP, TA, DType, etype>, DType>(MakePlan(e.src_));\n}\n\ntemplate<typename OP, typename TA, typename TB, typename DType, int etype>\ninline Plan<BinaryMapExp<OP, TA, TB, DType, etype>, DType>\nMakePlan(const BinaryMapExp<OP, TA, TB, DType, etype> &e) {\n  return Plan<BinaryMapExp<OP, TA, TB, DType, etype>,\n              DType>(MakePlan(e.lhs_), MakePlan(e.rhs_));\n}\n\n// Ternary\ntemplate<typename OP, typename TA, typename TB, typename TC, typename DType, int etype>\ninline Plan<TernaryMapExp<OP, TA, TB, TC, DType, etype>, DType>\nMakePlan(const TernaryMapExp<OP, TA, TB, TC, DType, etype> &e) {\n  return Plan<TernaryMapExp<OP, TA, TB, TC, DType, etype>,\n              DType>(MakePlan(e.item1_), MakePlan(e.item2_), MakePlan(e.item3_));\n}\n//----------------------------------------------------------------\n// Static Type inference and Type Checking\n//----------------------------------------------------------------\n/*!\n * \\brief static type inference template,\n *        used to get the dimension of each expression,\n *        if ExpInfo<E>::kDim == -1, this means here are mismatch in expression\n *        if (ExpInfo<E>::kDevMask & cpu::kDevMask) != 0, this means this expression can be assigned to cpu\n * \\tparam E expression\n */\ntemplate<typename E>\nstruct ExpInfo {\n  static const int kDim = -1;\n  static const int kDevMask = 0;\n};\ntemplate<typename DType>\nstruct ExpInfo< ScalarExp<DType> > {\n  static const int kDim = 0;\n  static const int kDevMask = 0xffff;\n};\ntemplate<typename E, typename DType>\nstruct ExpInfo<TransposeExp<E, DType> > {\n  static const int kDim = ExpInfo<E>::kDim;\n  static const int kDevMask = ExpInfo<E>::kDevMask;\n};\ntemplate<typename DstDType, typename SrcDType, typename EType, int etype>\nstruct ExpInfo<TypecastExp<DstDType, SrcDType, EType, etype> > {\n  static const int kDim = ExpInfo<EType>::kDim;\n  static const int kDevMask = ExpInfo<EType>::kDevMask;\n};\ntemplate<typename Device, int dim, typename DType>\nstruct ExpInfo<Tensor<Device, dim, DType> > {\n  static const int kDim = dim;\n  static const int kDevMask = Device::kDevMask;\n};\ntemplate<typename T, typename SrcExp, int dim, typename DType>\nstruct ExpInfo<MakeTensorExp<T, SrcExp, dim, DType> > {\n  static const int kDimSrc = ExpInfo<SrcExp>::kDim;\n  static const int kDim = kDimSrc >= 0 ? dim : -1;\n  static const int kDevMask = ExpInfo<SrcExp>::kDevMask;\n};\ntemplate<typename OP, typename TA, typename DType, int etype>\nstruct ExpInfo<UnaryMapExp<OP, TA, DType, etype> > {\n  static const int kDim = ExpInfo<TA>::kDim;\n  static const int kDevMask = ExpInfo<TA>::kDevMask;\n};\ntemplate<typename OP, typename TA, typename TB, typename DType, int etype>\nstruct ExpInfo<BinaryMapExp<OP, TA, TB, DType, etype> > {\n  static const int kDimLhs = ExpInfo<TA>::kDim;\n  static const int kDimRhs = ExpInfo<TB>::kDim;\n  static const int kDim = (kDimLhs >= 0 && kDimRhs >= 0) ?\\\n      (kDimLhs == 0 ?\\\n       kDimRhs :\\\n       ((kDimRhs == 0 || kDimLhs == kDimRhs) ? kDimLhs : -1)) : -1;\n  static const int kDevMask = ExpInfo<TA>::kDevMask & ExpInfo<TB>::kDevMask;\n};\ntemplate<typename OP, typename TA, typename TB, typename TC, typename DType, int etype>\nstruct ExpInfo<TernaryMapExp<OP, TA, TB, TC, DType, etype> > {\n  static const int kDimItem1 = ExpInfo<TA>::kDim;\n  static const int kDimItem2 = ExpInfo<TB>::kDim;\n  static const int kDimItem3 = ExpInfo<TC>::kDim;\n  static const int kDim = kDimItem1;\n  static const int kDevMask = ExpInfo<TA>::kDevMask & ExpInfo<TB>::kDevMask & ExpInfo<TC>::kDevMask;\n};\n\n/*! \\brief template to do type check */\ntemplate<typename Device, int dim, typename DType, typename E>\nstruct TypeCheck {\n  /*! \\brief dimension of expression*/\n  static const int kExpDim = ExpInfo<E>::kDim;\n  /*! \\brief whether the expression device type matches */\n  static const bool kDevPass = (ExpInfo<E>::kDevMask & Device::kDevMask) != 0;\n  /*! \\brief whether the expression can be mapped to expression of dim */\n  static const bool kMapPass = (kExpDim == 0 || kExpDim == dim) && kDevPass;\n  /*! \\brief whether the expression can be reduced to expression of dim */\n  static const bool kRedPass = (kExpDim > dim) && kDevPass;\n};\n/*! \\brief used to help static type check*/\ntemplate<bool kPass>\nstruct TypeCheckPass;\n// Todo : add static assert using C++11\ntemplate<>\nstruct TypeCheckPass<false> {};\ntemplate<>\nstruct TypeCheckPass<true> {\n  inline static void Error_All_Tensor_in_Exp_Must_Have_Same_Type(void) {}\n  inline static void Error_TypeCheck_Not_Pass_For_Reduce_Exp(void) {}\n  inline static void Error_Expression_Does_Not_Meet_Dimension_Req(void) {}\n};\n\n//----------------------------------------------------------------\n// Runtime Stream Getting\n//----------------------------------------------------------------\ntemplate<typename Device, typename E>\nstruct StreamInfo {\n  inline static Stream<Device> *Get(const E &t);\n};\ntemplate<int dim, typename Device, typename DType>\nstruct StreamInfo<Device, Tensor<Device, dim, DType> > {\n  inline static Stream<Device> *Get(const Tensor<Device, dim, DType> &t) {\n    return t.stream_;\n  }\n};\n//----------------------------------------------------------------\n// Runtime Shape Checking\n//----------------------------------------------------------------\n/*!\n * \\brief runtime shape checking template\n *    get the shape of an expression, report error if shape mismatch\n * \\tparam dim the dimension of the shape\n * \\tparam E expression\n */\ntemplate<int dim, typename E>\nstruct ShapeCheck {\n  inline static Shape<dim> Check(const E &t);\n};\ntemplate<int dim, typename DType>\nstruct ShapeCheck<dim, ScalarExp<DType> > {\n  inline static Shape<dim> Check(const ScalarExp<DType> &exp) {\n    // use lowest dimension to mark scalar exp\n    Shape<dim> shape;\n    for (int i = 0; i < dim; ++i) {\n      shape[i] = 0;\n    }\n    return shape;\n  }\n};\ntemplate<int dim, typename DstDType, typename SrcDType, typename EType, int etype>\nstruct ShapeCheck<dim, TypecastExp<DstDType, SrcDType, EType, etype> > {\n  inline static Shape<dim>\n  Check(const TypecastExp<DstDType, SrcDType, EType, etype> &exp) {\n    return ShapeCheck<dim, EType>::Check(exp.exp);\n  }\n};\ntemplate<int dim, typename E, typename DType>\nstruct ShapeCheck<dim, TransposeExp<E, DType> > {\n  inline static Shape<dim> Check(const TransposeExp<E, DType> &e) {\n    // swap the lowest two dimensions\n    Shape<dim> s = ShapeCheck<dim, E>::Check(e.exp);\n    std::swap(s[0], s[1]);\n    return s;\n  }\n};\ntemplate<int dim, typename Device, typename DType>\nstruct ShapeCheck<dim, Tensor<Device, dim, DType> > {\n  inline static Shape<dim> Check(const Tensor<Device, dim, DType> &t) {\n    return t.shape_;\n  }\n};\ntemplate<int dim, typename SrcExp, typename T, typename DType>\nstruct ShapeCheck<dim, MakeTensorExp<T, SrcExp, dim, DType> > {\n  inline static Shape<dim>\n  Check(const MakeTensorExp<T, SrcExp, dim, DType> &t) {\n    return t.shape_;\n  }\n};\ntemplate<int dim, typename OP, typename TA, typename DType, int etype>\nstruct ShapeCheck<dim, UnaryMapExp<OP, TA, DType, etype> > {\n  inline static Shape<dim> Check(const UnaryMapExp<OP, TA, DType, etype> &t) {\n    Shape<dim> s = ShapeCheck<dim, TA>::Check(t.src_);\n    return s;\n  }\n};\n\ntemplate<int dim, typename OP, typename TA, typename TB,\n         typename DType, int etype>\nstruct ShapeCheck<dim, BinaryMapExp<OP, TA, TB, DType, etype> > {\n  inline static Shape<dim>\n  Check(const BinaryMapExp<OP, TA, TB, DType, etype> &t) {\n    Shape<dim> shape1 = ShapeCheck<dim, TA>::Check(t.lhs_);\n    Shape<dim> shape2 = ShapeCheck<dim, TB>::Check(t.rhs_);\n    if (shape1[0] == 0) return shape2;\n    if (shape2[0] == 0) return shape1;\n    CHECK_EQ(shape1, shape2) << \"BinaryMapExp: Shapes of operands are not the same, \" <<\n      \"Shape1=\" << shape1 << \", Shape2=\" << shape2;\n    return shape1;\n  }\n};\n\ntemplate<int dim, typename OP, typename TA, typename TB, typename TC,\n         typename DType, int etype>\nstruct ShapeCheck<dim, TernaryMapExp<OP, TA, TB, TC, DType, etype> > {\n  inline static Shape<dim>\n  Check(const TernaryMapExp<OP, TA, TB, TC, DType, etype> &t) {\n    Shape<dim> shape1 = ShapeCheck<dim, TA>::Check(t.item1_);\n    Shape<dim> shape2 = ShapeCheck<dim, TB>::Check(t.item2_);\n    Shape<dim> shape3 = ShapeCheck<dim, TC>::Check(t.item3_);\n    bool same = (shape1 == shape2) && (shape2 == shape3);\n    CHECK(same) << \"TernaryMapExp: Shapes of operands are not the same, \" <<\n      \"Shape1=\" << shape1 << \", Shape2=\" << shape2 << \", Shape3=\" << shape3;\n\n    return shape1;\n  }\n};\n}  // namespace expr\n\n}  // namespace mshadow\n// include definition of dot engine\n#include \"./dot_engine-inl.h\"\n\nnamespace mshadow {\nnamespace expr {\n/*! \\brief some engine that evaluate complex expression */\ntemplate<typename SV, typename RV, typename E, typename DType>\nstruct ExpComplexEngine {\n  inline static void Eval(RV *dst, const E &exp);\n};\n/*! \\brief the engine that dispatches simple operations*/\ntemplate<typename SV, typename RV, typename DType>\nstruct ExpEngine {\n  template<typename E>\n  inline static void Eval(RV *dst,\n                          const Exp<E, DType, type::kMapper> &exp) {\n    MapExp<SV>(dst, exp);\n  }\n  template<typename E>\n  inline static void Eval(RV *dst,\n                          const Exp<E, DType, type::kChainer> &exp) {\n    MapExp<SV>(dst, exp);\n  }\n  template<typename E>\n  inline static void Eval(RV *dst,\n                          const Exp<E, DType, type::kRValue> &exp) {\n    MapExp<SV>(dst, exp);\n  }\n  template<typename E>\n  inline static void Eval(RV *dst,\n                          const Exp<E, DType, type::kComplex> &exp) {\n    ExpComplexEngine<SV, RV, E, DType>::Eval(dst->ptrself(), exp.self());\n  }\n};\ntemplate<typename SV, typename Device, int dim, int ldim,\n         int rdim, bool ltrans, bool rtrans, typename DType>\nstruct ExpComplexEngine<SV,\n                        Tensor<Device, dim, DType>,\n                        DotExp<Tensor<Device, ldim, DType>,\n                               Tensor<Device, rdim, DType>,\n                               ltrans, rtrans, DType>,\n                        DType> {\n  inline static void Eval(Tensor<Device, dim, DType> *dst,\n                          const DotExp<Tensor<Device, ldim, DType>,\n                                       Tensor<Device, rdim, DType>,\n                                       ltrans, rtrans, DType> &exp) {\n    DotEngine<SV, Device, dim, ldim, rdim,\n              ltrans, rtrans, DType>::Eval(dst, exp.lhs_, exp.rhs_, exp.scale_);\n  }\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXPR_ENGINE_INL_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/expr_scalar-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file expr_scalar-inl.h\n * \\brief definitions of operators in expression with respect to scalar\n *  this file will be included several times, each time with MACRO MSHADOW_SCALAR_ to be different types\n *\n * DO NOT add pragma once or macro guard\n * \\author Tianqi Chen, Bing Xu\n */\n// macro guard is harmful, used to pass the cpplint\n#ifndef MSHADOW_EXPR_SCALAR_INL_H_\n#define MSHADOW_EXPR_SCALAR_INL_H_\n// undef the guard so it can be included multiple times\n#undef MSHADOW_EXPR_SCALAR_INL_H_\n\nnamespace mshadow {\nnamespace expr {\n// DotExp\n/*! \\brief dot operator def */\ntemplate<typename TA, typename TB, bool ltrans, bool rtrans>\ninline DotExp<TA, TB, ltrans, rtrans, MSHADOW_SCALAR_>\noperator*(const DotExp<TA, TB, ltrans, rtrans, MSHADOW_SCALAR_> &lhs,\n          MSHADOW_SCALAR_ rhs) {\n  return DotExp<TA, TB, ltrans, rtrans,\n                MSHADOW_SCALAR_>(lhs.lhs_, lhs.rhs_, lhs.scale_ * rhs);\n}\n/*! \\brief scale of dot operation */\ntemplate<typename TA, typename TB, bool ltrans, bool rtrans>\ninline DotExp<TA, TB, ltrans, rtrans, MSHADOW_SCALAR_>\noperator*(MSHADOW_SCALAR_ lhs,\n          const DotExp<TA, TB, ltrans, rtrans, MSHADOW_SCALAR_> &rhs) {\n  return DotExp<TA, TB, ltrans, rtrans,\n                MSHADOW_SCALAR_>(rhs.lhs_, rhs.rhs_, rhs.scale_ * lhs);\n}\n\n/*! \\brief operator overload */\ntemplate<typename E, typename DType, typename R, int d>\ninline ReduceTo1DExp<E, DType, R, d>\noperator*(const ReduceTo1DExp<E, DType, R, d> &e, MSHADOW_SCALAR_ scale) {\n  return ReduceTo1DExp<E, DType, R, d>(e.src_, e.scale_ * scale);\n}\n/*! \\brief operator overload */\ntemplate<typename E, typename DType, typename R, int d>\ninline ReduceTo1DExp<E, DType, R, d>\noperator*(MSHADOW_SCALAR_ scale, const ReduceTo1DExp<E, DType, R, d> &e) {\n  return ReduceTo1DExp<E, DType, R, d>(e.src_, e.scale_ * scale);\n}\n\n/*! \\brief operator overload for const */\ntemplate<typename OP, typename TA, int ta>\ninline BinaryMapExp<OP, TA, ScalarExp<MSHADOW_SCALAR_>,\n                    MSHADOW_SCALAR_, (ta|type::kMapper)>\nF(const Exp<TA, MSHADOW_SCALAR_, ta> &lhs, const ScalarExp<MSHADOW_SCALAR_> &rhs) {\n  return MakeExp<OP>(lhs, rhs);\n}\n/*! \\brief operator overload for const */\ntemplate<typename OP, typename TB, int tb>\ninline BinaryMapExp<OP, ScalarExp<MSHADOW_SCALAR_>, TB,\n                    MSHADOW_SCALAR_, (tb|type::kMapper)>\nF(const ScalarExp<MSHADOW_SCALAR_> &lhs, const Exp<TB, MSHADOW_SCALAR_, tb> &rhs) {\n  return MakeExp<OP>(lhs, rhs);\n}\n/*! \\brief operator overload for const */\ntemplate<typename OP>\ninline BinaryMapExp<OP, ScalarExp<MSHADOW_SCALAR_>, ScalarExp<MSHADOW_SCALAR_>,\n                    MSHADOW_SCALAR_, (1|type::kMapper)>\nF(const ScalarExp<MSHADOW_SCALAR_> &lhs, const ScalarExp<MSHADOW_SCALAR_> &rhs) {\n  return MakeExp<OP>(lhs, rhs);\n}\n// constant operators\n/*! \\brief operator overload */\ntemplate<typename TA, int ta>\ninline BinaryMapExp<op::plus, TA, ScalarExp<MSHADOW_SCALAR_>,\n                    MSHADOW_SCALAR_, (ta|type::kMapper)>\noperator+(const Exp<TA, MSHADOW_SCALAR_, ta> &lhs,\n          const ScalarExp<MSHADOW_SCALAR_> &rhs) {\n  return MakeExp<op::plus>(lhs, rhs);\n}\n/*! \\brief operator overload */\ntemplate<typename TA, int ta>\ninline BinaryMapExp<op::minus, TA, ScalarExp<MSHADOW_SCALAR_>,\n                    MSHADOW_SCALAR_, (ta|type::kMapper)>\noperator-(const Exp<TA, MSHADOW_SCALAR_, ta> &lhs,\n          const ScalarExp<MSHADOW_SCALAR_> &rhs) {\n  return MakeExp<op::minus>(lhs, rhs);\n}\n/*! \\brief operator overload */\ntemplate<typename TA, int ta>\ninline BinaryMapExp<op::mul, TA, ScalarExp<MSHADOW_SCALAR_>,\n                    MSHADOW_SCALAR_, (ta|type::kMapper)>\noperator*(const Exp<TA, MSHADOW_SCALAR_, ta> &lhs,\n          const ScalarExp<MSHADOW_SCALAR_> &rhs) {\n  return MakeExp<op::mul>(lhs, rhs);\n}\n/*! \\brief operator overload */\ntemplate<typename TA, int ta>\ninline BinaryMapExp<op::div, TA, ScalarExp<MSHADOW_SCALAR_>,\n                    MSHADOW_SCALAR_, (ta|type::kMapper)>\noperator/(const Exp<TA, MSHADOW_SCALAR_, ta> &lhs,\n          const ScalarExp<MSHADOW_SCALAR_> &rhs) {\n  return MakeExp<op::div>(lhs, rhs);\n}\n// constant operators 2\n/*! \\brief operator overload */\ntemplate<typename TB, int tb>\ninline BinaryMapExp<op::plus, ScalarExp<MSHADOW_SCALAR_>, TB,\n                    MSHADOW_SCALAR_, (tb|type::kMapper)>\noperator+(const ScalarExp<MSHADOW_SCALAR_> &lhs,\n          const Exp<TB, MSHADOW_SCALAR_, tb> &rhs) {\n  return MakeExp<op::plus>(lhs, rhs);\n}\n/*! \\brief operator overload */\ntemplate<typename TB, int tb>\ninline BinaryMapExp<op::minus, ScalarExp<MSHADOW_SCALAR_>, TB,\n                    MSHADOW_SCALAR_, (tb|type::kMapper)>\noperator-(const ScalarExp<MSHADOW_SCALAR_> &lhs,\n          const Exp<TB, MSHADOW_SCALAR_, tb> &rhs) {\n  return MakeExp<op::minus>(lhs, rhs);\n}\n/*! \\brief operator overload */\ntemplate<typename TB, int tb>\ninline BinaryMapExp<op::mul, ScalarExp<MSHADOW_SCALAR_>, TB,\n                    MSHADOW_SCALAR_, (tb|type::kMapper)>\noperator*(const ScalarExp<MSHADOW_SCALAR_> &lhs,\n          const Exp<TB, MSHADOW_SCALAR_, tb> &rhs) {\n  return MakeExp<op::mul>(lhs, rhs);\n}\n/*! \\brief operator overload */\ntemplate<typename TB, int tb>\ninline BinaryMapExp<op::div, ScalarExp<MSHADOW_SCALAR_>, TB,\n                    MSHADOW_SCALAR_, (tb|type::kMapper)>\noperator/(const ScalarExp<MSHADOW_SCALAR_> &lhs, const Exp<TB, MSHADOW_SCALAR_, tb> &rhs) {\n  return MakeExp<op::div>(lhs, rhs);\n}\n// constant operators 3\n/*! \\brief operator overload */\ninline BinaryMapExp<op::plus, ScalarExp<MSHADOW_SCALAR_>, ScalarExp<MSHADOW_SCALAR_>,\n                    MSHADOW_SCALAR_, (1|type::kMapper)>\noperator+(const ScalarExp<MSHADOW_SCALAR_> &lhs,\n          const ScalarExp<MSHADOW_SCALAR_> &rhs) {\n  return MakeExp<op::plus>(lhs, rhs);\n}\n/*! \\brief operator overload */\ninline BinaryMapExp<op::minus, ScalarExp<MSHADOW_SCALAR_>, ScalarExp<MSHADOW_SCALAR_>,\n                    MSHADOW_SCALAR_, (1|type::kMapper)>\noperator-(const ScalarExp<MSHADOW_SCALAR_> &lhs,\n          const ScalarExp<MSHADOW_SCALAR_> &rhs) {\n  return MakeExp<op::minus>(lhs, rhs);\n}\n/*! \\brief operator overload */\ninline BinaryMapExp<op::mul, ScalarExp<MSHADOW_SCALAR_>, ScalarExp<MSHADOW_SCALAR_>,\n                    MSHADOW_SCALAR_, (1|type::kMapper)>\noperator*(const ScalarExp<MSHADOW_SCALAR_> &lhs,\n          const ScalarExp<MSHADOW_SCALAR_> &rhs) {\n  return MakeExp<op::mul>(lhs, rhs);\n}\n/*! \\brief operator overload */\ninline BinaryMapExp<op::div, ScalarExp<MSHADOW_SCALAR_>, ScalarExp<MSHADOW_SCALAR_>,\n                    MSHADOW_SCALAR_, (1|type::kMapper)>\noperator/(const ScalarExp<MSHADOW_SCALAR_> &lhs, const ScalarExp<MSHADOW_SCALAR_> &rhs) {\n  return MakeExp<op::div>(lhs, rhs);\n}\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXPR_SCALAR_INL_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/expression.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file expression.h\n * \\brief definitions of abstract expressions and expressions template\n * \\author Tianqi Chen, Bing Xu\n */\n#ifndef MSHADOW_EXPRESSION_H_\n#define MSHADOW_EXPRESSION_H_\n#include \"./base.h\"\n\nnamespace mshadow {\n/*!\n * \\brief namespace for abstract expressions and expressions template,\n *        have no dependency on tensor.h,\n *        These data structure takes no charge in computations,\n *        they are only used to define operations and represent expression in a symbolic way\n */\nnamespace expr {\n/*! \\brief type of expressions */\nnamespace type {\n// type expression type are defined as bitmask\n// subtype relationshop kRValue < kMapper < kPull < kComplex\n/*!\n * \\brief this expression directly correspnds to a data class,\n *   can be used to assign data\n */\nconst int kRValue = 0;\n/*!\n * \\brief expression contains element-wise tensor operations,\n *   map a expression to same shape\n */\nconst int kMapper = 1;\n/*!\n * \\brief expression that can be chained with other expressiones\n *    Usually it have function Eval(i,j) defined, which pulls the result (i, j) from input\n *    expression and output the result at certain position.\n */\nconst int kChainer = 3;\n/*! \\brief othercase: e.g dot product */\nconst int kComplex = 7;\n}  // namespace type\n/*!\n * \\brief expression engine that actually interprets these expressions\n *   this is a function template that needed to be implemented for specific expressions\n * \\tparam Saver the save method\n * \\tparam RValue the type of RValue to be saved\n * \\sa namespace sv\n */\ntemplate<typename Saver, typename RValue, typename DType>\nstruct ExpEngine;\n/*! \\brief defines how expression exp can be evaluated and stored into dst */\n// template<typename EType>\n// inline static void Eval(RValue *dst, const EType &exp);\n/*!\n * \\brief base class for expression\n * \\tparam SubType inheritated class must put their type into this parameter\n * \\tparam DType the data type of each element in the expression\n * \\tparam exp_type expression type, see namespace type\n */\ntemplate<typename SubType, typename DType, int exp_type>\nstruct Exp {\n public:\n  /*! \\return  subtype instance of current class */\n  inline const SubType& self(void) const {\n    return *static_cast<const SubType*>(this);\n  }\n  /*! \\return reference of subtype instance of current class */\n  inline SubType* ptrself(void) {\n    return static_cast<SubType*>(this);\n  }\n};\n/*!\n * \\brief scalar expression\n * \\tparam DType the data type of the scalar\n */\ntemplate<typename DType>\nstruct ScalarExp: public Exp<ScalarExp<DType>, DType, type::kMapper> {\n  /*! \\brief scalar value */\n  DType scalar_;\n  /*! \\brief implicit constructor, MUST NOT BE explicit */\n  ScalarExp(DType scalar) : scalar_(scalar) {}  // NOLINT(*)\n};\n/*! \\brief create an scalar expression */\ntemplate<typename DType>\ninline ScalarExp<DType> scalar(DType s) {\n  return ScalarExp<DType>(s);\n}\n/*!\n * \\brief typecast expression, cast the type of elements\n * \\tparam DstDType the target type we want to cast into\n * \\tparam SrcDType the target type we want to cast from\n * \\tparam EType the type of the source expression\n * \\tparam etype the type of expression after cast\n */\ntemplate<typename DstDType, typename SrcDType, typename EType, int etype>\nstruct TypecastExp:\n      public Exp<TypecastExp<DstDType, SrcDType, EType, etype>,\n                 DstDType, etype> {\n  /*! \\brief expression to be typecasted */\n  const EType &exp;\n  /*! \\brief constructor */\n  explicit TypecastExp(const EType &e) : exp(e) {}\n};\n/*! \\brief create an scalar expression */\ntemplate<typename DstDType, typename SrcDType,\n         typename EType, int etype>\ninline TypecastExp<DstDType, SrcDType, EType, (etype|type::kMapper)>\ntcast(const Exp<EType, SrcDType, etype> &exp) {\n  return TypecastExp<DstDType, SrcDType, EType, (etype|type::kMapper)>(exp.self());\n}\n/*! \\brief represent a transpose expression of a container */\ntemplate<typename EType, typename DType>\nstruct TransposeExp: public Exp<TransposeExp<EType, DType>,\n                                DType, type::kChainer> {\n  /*! \\brief expression to be transposed */\n  const EType &exp;\n  /*! \\brief constructor */\n  explicit TransposeExp(const EType &e) : exp(e) {}\n  /*! \\brief transpose expression */\n  inline const EType &T(void) const {\n    return exp;\n  }\n};\n/*!\n * \\brief base class of all rvalues\n * \\tparam Container the actually class of data container, e.g. Tensor1D\n * \\tparam DataType the element data type of each element in the container\n */\ntemplate<typename Container, typename DType>\nclass RValueExp: public Exp<Container, DType, type::kRValue> {\n public:\n  /*!\n   *\\brief transpose of a matrix\n   *\\return transpose of current expression\n   */\n  inline const TransposeExp<Container, DType> T(void) const {\n    return TransposeExp<Container, DType>(this->self());\n  }\n  /*! \\brief operator overload */\n  inline Container &operator+=(DType s) {\n    ExpEngine<sv::plusto, Container, DType>::Eval(this->ptrself(), scalar<DType>(s));\n    return *(this->ptrself());\n  }\n  /*! \\brief operator overload */\n  inline Container &operator-=(DType s) {\n    ExpEngine<sv::minusto, Container, DType>::Eval(this->ptrself(), scalar<DType>(s));\n    return *(this->ptrself());\n  }\n  /*! \\brief operator overload */\n  inline Container &operator*=(DType s) {\n    ExpEngine<sv::multo, Container, DType>::Eval(this->ptrself(), scalar<DType>(s));\n    return *(this->ptrself());\n  }\n  /*! \\brief operator overload */\n  inline Container &operator/=(DType s) {\n    ExpEngine<sv::divto, Container, DType>::Eval(this->ptrself(), scalar<DType>(s));\n    return *(this->ptrself());\n  }\n  /*! \\brief operator overload */\n  inline Container &__assign(DType s) {\n    ExpEngine<sv::saveto, Container, DType>::Eval(this->ptrself(), scalar<DType>(s));\n    return *(this->ptrself());\n  }\n  /*! \\brief  we can not define container = container */\n  template<typename E, int etype>\n  inline Container &__assign(const Exp<E, DType, etype> &exp) {\n    ExpEngine<sv::saveto, Container, DType>::Eval(this->ptrself(), exp.self());\n    return *(this->ptrself());\n  }\n  /*! \\brief operator overload, assign */\n  inline Container &__assign(const Exp<Container, DType, type::kRValue> &exp);\n  /*! \\brief implementation of operator+= */\n  template<typename E, int etype>\n  inline Container &operator+=(const Exp<E, DType, etype> &exp) {\n    ExpEngine<sv::plusto, Container, DType>::Eval(this->ptrself(), exp.self());\n    return *(this->ptrself());\n  }\n  /*! \\brief implementation of operator-= */\n  template<typename E, int etype>\n  inline Container &operator-=(const Exp<E, DType, etype> &exp) {\n    ExpEngine<sv::minusto, Container, DType>::Eval(this->ptrself(), exp.self());\n    return *(this->ptrself());\n  }\n  /*! \\brief implementation of operator*= */\n  template<typename E, int etype>\n  inline Container &operator*=(const Exp<E, DType, etype> &exp) {\n    ExpEngine<sv::multo, Container, DType>::Eval(this->ptrself(), exp.self());\n    return *(this->ptrself());\n  }\n  /*! \\brief implementation of operator/= */\n  template<typename E, int etype>\n  inline Container &operator/=(const Exp<E, DType, etype> &exp) {\n    ExpEngine<sv::divto, Container, DType>::Eval(this->ptrself(), exp.self());\n    return *(this->ptrself());\n  }\n};\n/*!\n * \\brief matrix multiplication expression dot(lhs[.T], rhs[.T])\n * \\tparam TA type of lhs\n * \\tparam TB type of rhs\n * \\tparam ltrans whether lhs is transposed\n * \\tparam rtrans whether rhs is transposed\n * \\tparam DType the data type of the scalar\n */\ntemplate<typename TA, typename TB, bool ltrans, bool rtrans, typename DType>\nstruct DotExp: public Exp<DotExp<TA, TB, ltrans, rtrans, DType>,\n                          DType, type::kComplex> {\n  /*! \\brief left operand */\n  const TA &lhs_;\n  /*! \\brief right operand */\n  const TB &rhs_;\n  /*! \\brief scale over result */\n  DType scale_;\n  /*! \\brief constructor */\n  explicit DotExp(const TA &lhs, const TB &rhs, DType scale)\n      : lhs_(lhs), rhs_(rhs), scale_(scale) {}\n};\n// definition of dot expression\n/*! \\brief dot operator def */\ntemplate<typename TA, typename TB, typename DType>\ninline DotExp<TA, TB, false, false, DType>\ndot(const RValueExp<TA, DType> &lhs, const RValueExp<TB, DType> &rhs) {\n  return DotExp<TA, TB, false, false, DType>(lhs.self(), rhs.self(), DType(1.0f));\n}\n/*! \\brief dot operator def */\ntemplate<typename TA, typename TB, typename DType>\ninline DotExp<TA, TB, true, false, DType>\ndot(const TransposeExp<TA, DType> &lhs, const RValueExp<TB, DType> &rhs) {\n  return DotExp<TA, TB, true, false, DType>(lhs.exp, rhs.self(), DType(1.0f));\n}\n/*! \\brief dot operator def */\ntemplate<typename TA, typename TB, typename DType>\ninline DotExp<TA, TB, false, true, DType>\ndot(const RValueExp<TA, DType> &lhs, const TransposeExp<TB, DType> &rhs) {\n  return DotExp<TA, TB, false, true, DType>(lhs.self(), rhs.exp, DType(1.0f));\n}\n/*! \\brief dot operator def */\ntemplate<typename TA, typename TB, typename DType>\ninline DotExp<TA, TB, true, true, DType>\ndot(const TransposeExp<TA, DType> &lhs, const TransposeExp<TB, DType> &rhs) {\n  return DotExp<TA, TB, true, true, DType>(lhs.exp, rhs.exp, DType(1.0f));\n}\n/*! \\brief batch_dot operator def */\ntemplate<bool transpose_left, bool transpose_right, typename TA, typename TB, typename DType>\ninline DotExp<TA, TB, transpose_left, transpose_right, DType>\nbatch_dot(const RValueExp<TA, DType> &lhs, const RValueExp<TB, DType> &rhs) {\n  return DotExp<TA, TB, transpose_left, transpose_right, DType>(\n    lhs.self(), rhs.self(), DType(1.0f));\n}\n//---------------\n// TernaryMapExp\n// --------------\n/*!\n * \\brief ternary map expression\n * \\tparam OP operator\n * \\tparam TA type of item1\n * \\tparam TB type of item2\n * \\tparam etype expression type, sa namespace::type\n */\ntemplate<typename OP, typename TA, typename TB, typename TC, typename DType, int etype>\nstruct TernaryMapExp: public Exp<TernaryMapExp<OP, TA, TB, TC, DType, etype>,\n                                DType, etype> {\n  /*! \\brief first operand */\n  const TA &item1_;\n  /*! \\brief second operand */\n  const TB &item2_;\n  /*! \\brief third  operand */\n  const TC &item3_;\n  /*! \\brief constructor */\n  explicit TernaryMapExp(const TA &item1, const TB &item2, const TC &item3)\n      :item1_(item1), item2_(item2), item3_(item3) {}\n};\n\n/*! \\brief make expression */\ntemplate<typename OP, typename TA, typename TB, typename TC, typename DType, int ta, int tb, int tc>\ninline TernaryMapExp<OP, TA, TB, TC, DType, (ta|tb|tc|type::kMapper)>\nMakeExp(const Exp<TA, DType, ta> &item1, const Exp<TB, DType, tb> &item2,\n const Exp<TC, DType, tc> &item3) {\n  return TernaryMapExp<OP, TA, TB, TC, DType,\n                      (ta|tb|tc|type::kMapper)>(item1.self(), item2.self(), item3.self());\n}\n/*!\n * \\brief short hand for MakeExp, usage F<op>(item1,item2,item3). create a ternary operation expression\n * \\param item1 first operand\n * \\param item2 second operand\n * \\param item3 third operand\n * \\return the result expression\n * \\tparam ternary operator\n * \\tparam TA item1 expression\n * \\tparam ta item1 expression type\n * \\tparam TB item2 expression\n * \\tparam tb item2 expression type\n * \\tparam TC item3 expression\n * \\tparam tc item3 expression type\n * \\sa mshadow::op\n */\n\n// Ternary\ntemplate<typename OP, typename TA, typename TB, typename TC, typename DType, int ta, int tb, int tc>\ninline TernaryMapExp<OP, TA, TB, TC, DType, (ta|tb|tc|type::kMapper)>\nF(const Exp<TA, DType, ta> &item1, const Exp<TB, DType, tb> &item2,\n const Exp<TC, DType, tc> &item3) {\n  return MakeExp<OP>(item1, item2, item3);\n}\n//---------------\n// BinaryMapExp\n// --------------\n/*!\n * \\brief binary map expression lhs [op] rhs\n * \\tparam OP operator\n * \\tparam TA type of lhs\n * \\tparam TB type of rhs\n * \\tparam etype expression type, sa namespace::type\n */\ntemplate<typename OP, typename TA, typename TB, typename DType, int etype>\nstruct BinaryMapExp: public Exp<BinaryMapExp<OP, TA, TB, DType, etype>,\n                                DType, etype> {\n  /*! \\brief left operand */\n  const TA &lhs_;\n  /*! \\brief right operand */\n  const TB &rhs_;\n  /*! \\brief constructor */\n  explicit BinaryMapExp(const TA &lhs, const TB &rhs)\n      :lhs_(lhs), rhs_(rhs) {}\n};\n\n/*! \\brief make expression */\ntemplate<typename OP, typename TA, typename TB, typename DType, int ta, int tb>\ninline BinaryMapExp<OP, TA, TB, DType, (ta|tb|type::kMapper)>\nMakeExp(const Exp<TA, DType, ta> &lhs, const Exp<TB, DType, tb> &rhs) {\n  return BinaryMapExp<OP, TA, TB, DType,\n                      (ta|tb|type::kMapper)>(lhs.self(), rhs.self());\n}\n/*!\n * \\brief short hand for MakeExp, usage F<op>(lhs, rhs). create a binary operation expression\n * \\param lhs left operand\n * \\param rhs right operand\n * \\return the result expression\n * \\tparam binary operator\n * \\tparam TA lhs expression\n * \\tparam ta lhs expression type\n * \\tparam TB rhs expression\n * \\tparam tb rhs expression type\n * \\sa mshadow::op\n */\ntemplate<typename OP, typename TA, typename TB, typename DType, int ta, int tb>\ninline BinaryMapExp<OP, TA, TB, DType, (ta|tb|type::kMapper)>\nF(const Exp<TA, DType, ta> &lhs, const Exp<TB, DType, tb> &rhs) {\n  return MakeExp<OP>(lhs, rhs);\n}\n// operator rules\n/*! \\brief operator overload */\ntemplate<typename TA, typename TB, typename DType, int ta, int tb>\ninline BinaryMapExp<op::plus, TA, TB, DType, (ta|tb|type::kMapper)>\noperator+(const Exp<TA, DType, ta> &lhs, const Exp<TB, DType, tb> &rhs) {\n  return MakeExp<op::plus>(lhs, rhs);\n}\n/*! \\brief operator overload */\ntemplate<typename TA, typename TB, typename DType, int ta, int tb>\ninline BinaryMapExp<op::minus, TA, TB, DType, (ta|tb|type::kMapper)>\noperator-(const Exp<TA, DType, ta> &lhs, const Exp<TB, DType, tb> &rhs) {\n  return MakeExp<op::minus>(lhs, rhs);\n}\n/*! \\brief operator overload */\ntemplate<typename TA, typename TB, typename DType, int ta, int tb>\ninline BinaryMapExp<op::mul, TA, TB, DType, (ta|tb|type::kMapper)>\noperator*(const Exp<TA, DType, ta> &lhs, const Exp<TB, DType, tb> &rhs) {\n  return MakeExp<op::mul>(lhs, rhs);\n}\n/*! \\brief operator overload */\ntemplate<typename TA, typename TB, typename DType, int ta, int tb>\ninline BinaryMapExp<op::div, TA, TB, DType, (ta|tb|type::kMapper)>\noperator/(const Exp<TA, DType, ta> &lhs, const Exp<TB, DType, tb> &rhs) {\n  return MakeExp<op::div>(lhs, rhs);\n}\n//---------------\n// UnaryMapExp\n// --------------\n/*!\n * \\brief unary map expression op(src)\n * \\tparam OP operator\n * \\tparam TA type of src\n * \\tparam etype expression type, sa namespace::type\n */\ntemplate<typename OP, typename TA, typename DType, int etype>\nstruct UnaryMapExp: public Exp<UnaryMapExp<OP, TA, DType, etype>,\n                               DType, etype> {\n  /*! \\brief source expression */\n  const TA &src_;\n  /*! \\brief constructor */\n  explicit UnaryMapExp(const TA &src) : src_(src) {}\n};\n\n/*! \\brief make expression */\ntemplate<typename OP, typename TA, typename DType, int ta>\ninline UnaryMapExp<OP, TA, DType, (ta|type::kMapper)>\nMakeExp(const Exp<TA, DType, ta> &src) {\n  return UnaryMapExp<OP, TA, DType, (ta|type::kMapper)>(src.self());\n}\n/*!\n * \\brief short hand for MakeExp, usage F<op>(src), create a unary operation expression\n * \\param src source expression\n * \\return the result expression\n * \\tparam operator\n * \\tparam TA source expression\n * \\tparam ta source expression type\n * \\sa mshadow::op\n */\ntemplate<typename OP, typename TA, typename DType, int ta>\ninline UnaryMapExp<OP, TA, DType, (ta|type::kMapper)>\nF(const Exp<TA, DType, ta> &src) {\n  return MakeExp<OP>(src);\n}\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXPRESSION_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/broadcast.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file broadcast.h\n * \\brief support for broadcast and repmat\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_EXTENSION_BROADCAST_H_\n#define MSHADOW_EXTENSION_BROADCAST_H_\n#include \"../extension.h\"\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief broadcast Tensor1D into a higher dimension Tensor\n * input: Tensor<Device,1>: ishape[0]\n * output: Tensor<Device,dimdst> : oshape[dimcast] = ishape[0]\n * \\tparam SrcExp type of input expression\n * \\tparam DType the type of elements\n * \\tparam dimdst  target tensor dimension\n * \\tparam dimcast_m_dst  dimdst - dimcast\n */\ntemplate<typename SrcExp, typename DType, int dimdst, int dimdst_m_cast>\nstruct Broadcast1DExp:\n      public MakeTensorExp<Broadcast1DExp<SrcExp, DType, dimdst, dimdst_m_cast>,\n                           SrcExp, dimdst, DType> {\n  /*! \\brief source operand */\n  const SrcExp &src_;\n  /*! \\brief constructor */\n  Broadcast1DExp(const SrcExp &src, Shape<dimdst> shape)\n      : src_(src) {\n    this->shape_ = shape;\n  }\n};\n\n/*!\n * \\brief broadcast scalar into a higher dimension Tensor\n * input: Tensor<Device,1>: ishape = {1}\n * output: Tensor<Device, dimdst> : oshape[dimcast] = ishape[0]\n * \\tparam SrcExp type of input expression\n * \\tparam DType the type of elements\n * \\tparam dimdst  target tensor dimension\n */\ntemplate<typename SrcExp, typename DType, int dimdst>\nstruct BroadcastScalarExp:\n      public MakeTensorExp<BroadcastScalarExp<SrcExp, DType, dimdst>,\n                           SrcExp, dimdst, DType> {\n  /*! \\brief source operand */\n  const SrcExp &src_;\n  /*! \\brief constructor */\n  BroadcastScalarExp(const SrcExp &src, Shape<dimdst> shape)\n      : src_(src) {\n    this->shape_ = shape;\n  }\n};\n\n/*!\n * \\brief a expression that replicate a 1 dimension tensor in dimension dimcast\n * \\param src Tensor<Device,1>: shape[0]\n * \\param shape shape of output\n * \\return a expresion with type Tensor<Device,dimdst>\n * \\tparam dimcast target dimension where the 1D tensor will be broadcasted\n * \\tparam SrcExp type of input expression\n * \\tparam DType the type of elements\n * \\tparam dimdst dimension of destination tensor\n * \\tparam dimcast_lowest the dimension we want to cast the data into\n */\ntemplate<int dimcast, typename SrcExp, typename DType,\n         int etype, int dimdst>\ninline Broadcast1DExp<SrcExp, DType, dimdst, dimdst - dimcast>\nbroadcast(const expr::Exp<SrcExp, DType, etype> &src, Shape<dimdst> shape) {\n  TypeCheckPass<dimcast < dimdst && ExpInfo<SrcExp>::kDim == 1>\n                ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  typedef ShapeCheck<1, SrcExp> ShapeCheckDim1SrcExp;\n  CHECK_EQ(ShapeCheckDim1SrcExp::Check(src.self())[0], shape[dimcast])\n    << \"broadcast, shape mismatch\";\n  return Broadcast1DExp<SrcExp, DType, dimdst,\n                        dimdst - dimcast>(src.self(), shape);\n}\n\n/*!\n * \\brief a expression that replicate a scalar tensor to target dimension.\n * \\param src Tensor<Device,1>: shape[0] == 1\n * \\param shape shape of output\n * \\return a expresion with type Tensor<Device, dimdst>\n * \\tparam dimcast target dimension where the 1D tensor will be broadcasted\n * \\tparam SrcExp type of input expression\n * \\tparam DType the type of elements\n * \\tparam dimdst dimension of destination tensor\n */\ntemplate<typename SrcExp, typename DType, int etype, int dimdst>\ninline BroadcastScalarExp<SrcExp, DType, dimdst>\nbroadcast_scalar(const expr::Exp<SrcExp, DType, etype> &src, Shape<dimdst> shape) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim == 1>\n                ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  typedef ShapeCheck<1, SrcExp> ShapeCheckDim1SrcExp;\n  CHECK_EQ(ShapeCheckDim1SrcExp::Check(src.self())[0], 1U)\n      << \"broadcast_scalar, source need to be scalar expression\";\n  return BroadcastScalarExp<SrcExp, DType, dimdst>(src.self(), shape);\n}\n// short cut functions\n/*!\n * \\brief a expression that replicate a 1 dimension tensor for nrow times\n * \\param src Tensor<Device,1>: shape[0]\n * \\param nrow number of rows to replicate\n * \\return a expresion with type Tensor<Device,2> size(1), size(0) = nrow\n * \\tparam Device which device it lies\n */\ntemplate<typename SrcExp, typename DType, int etype>\ninline Broadcast1DExp<SrcExp, DType, 2, 1>\nrepmat(const expr::Exp<SrcExp, DType, etype> &src, index_t nrow) {\n  return broadcast<1>\n      (src, Shape2(nrow, ShapeCheck<1, SrcExp>::Check(src.self())[0]));\n}\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename SrcExp, typename DType, int dimdst, int dimdst_m_cast>\nstruct Plan<Broadcast1DExp<SrcExp, DType, dimdst, dimdst_m_cast>, DType> {\n public:\n  static const int dimcast = dimdst - dimdst_m_cast;\n  explicit Plan(const Broadcast1DExp<SrcExp, DType, dimdst, dimdst_m_cast> &e)\n      : src_(MakePlan(e.src_)),\n        ystride_(e.shape_.ProdShape(dimcast + 1, dimdst - 1)),\n        length_(e.shape_[dimcast]) {\n    TypeCheckPass<dimcast != dimdst - 1>\n        ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  }\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    return src_.Eval(0, (y / ystride_) % length_);\n  }\n\n private:\n  expr::Plan<SrcExp, DType> src_;\n  const index_t  ystride_, length_;\n};\n\n/*! \\brief execution plan of Broadcast1DExp */\ntemplate<typename SrcExp, typename DType, int dimdst>\nstruct Plan<Broadcast1DExp<SrcExp, DType, dimdst, 1>, DType>{\n public:\n  explicit Plan(const Broadcast1DExp<SrcExp, DType, dimdst, 1> &e)\n      : src_(MakePlan(e.src_)) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    return src_.Eval(0, x);\n  }\n\n private:\n  expr::Plan<SrcExp, DType> src_;\n};\n\n/*! \\brief execution plan of Broadcast1DExp */\ntemplate<typename SrcExp, typename DType, int dimdst>\nstruct Plan<BroadcastScalarExp<SrcExp, DType, dimdst>, DType>{\n public:\n  explicit Plan(const BroadcastScalarExp<SrcExp, DType, dimdst> &e)\n      : src_(MakePlan(e.src_)) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    return src_.Eval(0, 0);\n  }\n\n private:\n  expr::Plan<SrcExp, DType> src_;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_BROADCAST_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/broadcast_with_axis.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file broadcast_with_axis.h\n * \\brief\n * \\author Junyuan Xie, Xingjian Shi\n*/\n#ifndef MSHADOW_EXTENSION_BROADCAST_WITH_AXIS_H_\n#define MSHADOW_EXTENSION_BROADCAST_WITH_AXIS_H_\n\n#include <vector>\n#include \"../extension.h\"\n\nnamespace mshadow {\nnamespace expr {\n\n  /*!\n  * \\brief Broadcasting the tensor in the given axis. If keepdim is off, insert the broadcasting dim after axis. Otherwise broadcasting axis.\n  * \\tparam SrcExp source expression\n  * \\tparam DType  data type\n  * \\tparam dimsrc source dimension\n  * \\tparam dimdst destination dimension\n  */\ntemplate<typename SrcExp, typename DType, int dimsrc, int dimdst>\nstruct BroadcastWithAxisExp:\n    public MakeTensorExp<BroadcastWithAxisExp<SrcExp, DType, dimsrc, dimdst>,\n                         SrcExp, dimdst, DType> {\n  /*! \\brief data oprand */\n  const SrcExp &src_;\n  /*! \\brief size of the last dimension of dst */\n  index_t dst_last_;\n  /*! \\brief product of the dimensions after the broadcasting axis */\n  index_t trailing_;\n  /*! \\brief new dimension of the broadcasting axis*/\n  index_t size_;\n  /*! \\brief size of the last dimension of src*/\n  index_t last_;\n  /*! constructor */\n  BroadcastWithAxisExp(const SrcExp &src, const int axis, const index_t size)\n    : src_(src), size_(size) {\n    bool keepdim = (dimsrc == dimdst);\n    Shape<dimsrc> src_shape = ShapeCheck<dimsrc, SrcExp>::Check(src_);\n    this->trailing_ = 1;\n\n    if (!keepdim) {\n      CHECK(dimsrc > axis && axis >= -1) << \"broadcast axis (no keepdim) out of bound, \"  <<\n        \"axis must be between -1 and\" << dimsrc - 1 << \", given=\" << axis << \".\";\n      for (int i = 0; i <= axis; ++i) {\n        this->shape_[i] = src_shape[i];\n      }\n      this->shape_[axis + 1] = size_;\n      for (int i = axis + 1; i < dimsrc; ++i) {\n        this->trailing_ *= src_shape[i];\n        this->shape_[i + 1] = src_shape[i];\n      }\n    } else {\n      CHECK(dimdst > axis && axis >= 0) << \"broadcast axis (keepdim) out of bound, \" <<\n        \"axis must be between 0 and\" << dimdst - 1 << \", given=\" << axis << \".\";\n      CHECK_EQ(src_shape[axis], 1U) << \"Size of the dimension of the broadcasting axis must be 1\" <<\n        \" when keepdim is on, src_shape[\" << axis << \"]=\" << src_shape[axis] << \".\";\n      for (int i = 0; i <= axis - 1; ++i) {\n        this->shape_[i] = src_shape[i];\n      }\n      this->shape_[axis] = size_;\n      for (int i = axis + 1; i < dimdst; ++i) {\n        this->trailing_ *= src_shape[i];\n        this->shape_[i] = src_shape[i];\n      }\n    }\n\n    this->last_ = src_shape[dimsrc - 1];\n    this->dst_last_ = this->shape_[dimdst - 1];\n  }\n};  // struct BroadcastWithAxisExp\n\n/*!\n * \\brief Broadcasting the tensor after given axis.\n * \\tparam SrcExp source expression\n * \\tparam DType data type\n * \\tparam etype type of the expression\n */\ntemplate<typename SrcExp, typename DType, int etype>\ninline BroadcastWithAxisExp<SrcExp, DType, ExpInfo<SrcExp>::kDim,\n  ExpInfo<SrcExp>::kDim + 1>\nbroadcast_with_axis(const Exp<SrcExp, DType, etype> &src, const int axis, const index_t size) {\n  return BroadcastWithAxisExp<SrcExp, DType, ExpInfo<SrcExp>::kDim,\n    ExpInfo<SrcExp>::kDim + 1>(src.self(), axis, size);\n}\n\n/*!\n* \\brief Broadcasting the tensor in the given axis (keepdim turned on)\n* \\tparam SrcExp source expression\n* \\tparam DType data type\n* \\tparam etype type of the expression\n*/\ntemplate<typename SrcExp, typename DType, int etype>\ninline BroadcastWithAxisExp<SrcExp, DType, ExpInfo<SrcExp>::kDim,\n  ExpInfo<SrcExp>::kDim>\n  broadcast_keepdim(const Exp<SrcExp, DType, etype> &src, const int axis, const index_t size) {\n  return BroadcastWithAxisExp<SrcExp, DType, ExpInfo<SrcExp>::kDim,\n    ExpInfo<SrcExp>::kDim>(src.self(), axis, size);\n}\n\n/*!\n* \\brief Broadcasting the tensor in multiple axes. The dimension of the source tensor\n         in the given axes must be 1.\n* \\tparam SrcExp source expression\n* \\tparam DType  data type\n* \\tparam dimsrc source dimension\n* \\tparam axesnum number of broadcasting dimensions\n*/\ntemplate<typename SrcExp, typename DType, int dimsrc>\nstruct BroadcastWithMultiAxesExp :\n      public MakeTensorExp<BroadcastWithMultiAxesExp<SrcExp, DType, dimsrc>,\n  SrcExp, dimsrc, DType> {\n  /*! \\brief data oprand */\n  const SrcExp &src_;\n  /*! \\brief size of the last dimension of dst */\n  index_t dst_last_;\n  /*! \\brief number of broadcasting axes*/\n  index_t axesnum_;\n  /*! \\brief product of the dimensions after the broadcasting axses */\n  Shape<dimsrc> trailings_;\n  /*! \\brief new dimension of the broadcasting axes*/\n  Shape<dimsrc> sizes_;\n  /*! \\brief size of the last dimension of src*/\n  index_t last_;\n  /*! constructor */\n  template<typename TShape>\n  BroadcastWithMultiAxesExp(const SrcExp &src, const TShape& axes, const TShape& sizes)\n    : src_(src) {\n    Shape<dimsrc> src_shape = ShapeCheck<dimsrc, SrcExp>::Check(src_);\n    CHECK(axes.ndim() == sizes.ndim()) << \"ndim of axes and sizes must be equal.\";\n    this->axesnum_ = axes.ndim();\n    CHECK(this->axesnum_ <= dimsrc) << \"Number of broadcasting axes must be smaller than\"\n      \"the source ndim, number of axes=\" << this->axesnum_ << \" dimsrc=\" << dimsrc;\n    for (index_t i = 0; i < this->axesnum_; i++) {\n      CHECK(dimsrc > axes[i]) << \"broadcast axis (keepdim) out of bound, \" <<\n        \"all axes must be between 0 and\" << dimsrc - 1 << \", given axes[\" << i << \"] = \" << axes[i]\n        << \".\";\n      CHECK_EQ(src_shape[axes[i]], 1U) << \"Size of the dimension of the broadcasting axis must be 1\"\n        << \", src_shape[\" << axes[i] << \"]=\" << src_shape[axes[i]] << \".\";\n      if (i < this->axesnum_ - 1) {\n        CHECK(axes[i] < axes[i + 1]) << \"The given axes must be in increasing order.\";\n      }\n    }\n    for (index_t i = 0; i < dimsrc; i++) {\n      this->shape_[i] = src_shape[i];\n      this->sizes_[i] = 1;\n      this->trailings_[i] = 1;\n    }\n    for (index_t i = 0; i < this->axesnum_; i++) {\n      this->shape_[axes[i]] = sizes[i];\n      this->sizes_[i] = sizes[i];\n    }\n    for (index_t i = 0; i < this->axesnum_; i++) {\n      this->trailings_[i] = 1;\n      for (index_t j = axes[i] + 1; j < dimsrc; ++j) {\n        this->trailings_[i] *= this->shape_[j];\n      }\n    }\n    this->last_ = src_shape[dimsrc - 1];\n    this->dst_last_ = this->shape_[dimsrc - 1];\n  }\n};  // struct BroadcastWithMultiAxesExp\n\n/*!\n* \\brief Broadcasting the tensor in the given axis (keepdim turned on)\n* \\param src source\n* \\param axes broadcasting axes\n* \\param sizes sizes of the broadcasting axes\n* \\tparam SrcExp source expression\n* \\tparam DType data type\n* \\tparam etype type of the expression\n* \\tparam TShape the flexible shape type\n*/\ntemplate<typename SrcExp, typename DType, int etype, typename TShape>\ninline BroadcastWithMultiAxesExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>\nbroadcast_multi_axes(const Exp<SrcExp, DType, etype> &src,\nconst TShape &axes, const TShape &sizes) {\n  return BroadcastWithMultiAxesExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>(src.self(), axes, sizes);\n}\n\n/*!\n* \\brief Broadcasting the tensor to the target shape,\n         dimension of different sizes must be 1 in the original tensor.\n* \\param src source\n* \\param target_shape shape of the target broadcasting tensor\n* \\tparam SrcExp source expression\n* \\tparam DType data type\n* \\tparam etype type of the expression\n* \\tparam TShape the flexible shape type\n*/\ntemplate<typename SrcExp, typename DType, int etype, typename TShape>\ninline BroadcastWithMultiAxesExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>\nbroadcast_to(const Exp<SrcExp, DType, etype> &src, const TShape &target_shape) {\n  static const size_t dimsrc = ExpInfo<SrcExp>::kDim;\n  CHECK_EQ(target_shape.ndim(), dimsrc);\n  std::vector<index_t> axes_vec, sizes_vec;\n  Shape<dimsrc> src_shape = ShapeCheck<dimsrc, SrcExp>::Check(src.self());\n  for (size_t i = 0; i < dimsrc; ++i) {\n    if (src_shape[i] != target_shape[i]) {\n      CHECK_EQ(src_shape[i], 1U) << \"broadcasting axis must have size 1, received shape=\"\n        << src_shape << \" target_shape=\" << target_shape;\n      axes_vec.push_back(i);\n      sizes_vec.push_back(target_shape[i]);\n    }\n  }\n  TShape axes = TShape(axes_vec.begin(), axes_vec.end());\n  TShape sizes = TShape(sizes_vec.begin(), sizes_vec.end());\n  return BroadcastWithMultiAxesExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>(src.self(), axes, sizes);\n}\n\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename SrcExp, typename DType, int dimsrc, int dimdst>\nstruct Plan<BroadcastWithAxisExp<SrcExp, DType, dimsrc, dimdst>, DType> {\n public:\n  explicit Plan(const BroadcastWithAxisExp<SrcExp, DType, dimsrc, dimdst> &e)\n       : src_(MakePlan(e.src_)), dst_last_(e.dst_last_),\n         trailing_(e.trailing_), size_(e.size_), last_(e.last_) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    index_t x = (i * dst_last_ + j) / trailing_ / size_;\n    index_t y = (i * dst_last_ + j) % trailing_;\n    index_t z = x * trailing_ + y;\n    return src_.Eval(z / last_, z % last_);\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t dst_last_, trailing_, size_, last_;\n};\n\ntemplate<typename SrcExp, typename DType, int dimsrc>\nstruct Plan<BroadcastWithMultiAxesExp<SrcExp, DType, dimsrc>, DType> {\n public:\n  explicit Plan(const BroadcastWithMultiAxesExp<SrcExp, DType, dimsrc> &e)\n    : src_(MakePlan(e.src_)), dst_last_(e.dst_last_), last_(e.last_), axesnum_(e.axesnum_),\n    trailings_(e.trailings_), sizes_(e.sizes_) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    index_t indx = i * dst_last_ + j;\n    for (index_t p = 0; p < dimsrc; ++p) {\n      if (p >= axesnum_) {\n        break;\n      }\n      indx = (indx / trailings_[p] / sizes_[p]) * trailings_[p] + (indx % trailings_[p]);\n    }\n    return src_.Eval(indx / last_, indx % last_);\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t dst_last_, last_, axesnum_;\n  const Shape<dimsrc> trailings_, sizes_;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_BROADCAST_WITH_AXIS_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/channel_pool.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file channel_pool.h\n * \\brief support for chpool\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_EXTENSION_CHANNEL_POOL_H_\n#define MSHADOW_EXTENSION_CHANNEL_POOL_H_\n#include <algorithm>\n#include \"../extension.h\"\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief channel pooling expression, do reduction over (local nearby) channels,\n *        used to implement local response normalization\n * \\tparam Reducer reduction method during pooling\n * \\tparam SrcExp source expression to be pooled from\n * \\tparam DType the type of elements\n * \\tparam srcdim dimension of src\n */\ntemplate<typename Reducer, typename SrcExp, typename DType, int srcdim>\nstruct ChannelPoolingExp:\n      public MakeTensorExp<ChannelPoolingExp<Reducer, SrcExp, DType, srcdim>,\n                           SrcExp, srcdim, DType> {\n  /*! \\brief source operand */\n  const SrcExp &src_;\n  /*! \\brief neighbor size */\n  index_t nsize_;\n  /*! \\brief stride of pooling */\n  index_t stride_;\n  /*! \\brief pad of pooling of each side */\n  index_t pad_;\n  index_t src_channel_;\n  /*! \\brief constructor */\n  ChannelPoolingExp(const SrcExp &src, index_t nsize, index_t stride, index_t pad)\n      : src_(src), nsize_(nsize), stride_(stride), pad_(pad) {\n    this->shape_ = ShapeCheck<srcdim, SrcExp>::Check(src_);\n    this->src_channel_ = this->shape_[srcdim - 3];\n    CHECK_GE(this->shape_[srcdim - 3], nsize_)\n      << \"chpool: local size must be smaller than nchannels\";\n    this->shape_[srcdim - 3] = (this->src_channel_ - nsize + pad * 2 + 1) / stride;\n  }\n};\n/*!\n * \\brief  channel pooling, do reduction over (local nearby) channels,\n *         used to implement local response normalization\n * \\param src source data\n * \\param nsize neighbor size\n * \\return expression of pooled result\n * \\tparam Reducer reducer type\n * \\tparam SrcExp source expression\n * \\tparam DType the type of elements\n * \\tparam etype type of expression\n */\ntemplate<typename Reducer, typename SrcExp, typename DType, int etype>\ninline ChannelPoolingExp<Reducer, SrcExp, DType, ExpInfo<SrcExp>::kDim>\nchpool(const Exp<SrcExp, DType, etype> &src, index_t nsize) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim >= 3>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  CHECK_EQ(nsize % 2, 1U) << \"chpool: if no pad is specified, local size must be odd\";\n  return ChannelPoolingExp<Reducer, SrcExp,\n                           DType, ExpInfo<SrcExp>::kDim>(src.self(), nsize, 1, nsize / 2);\n}\n\ntemplate<typename Reducer, typename SrcExp, typename DType, int etype>\ninline ChannelPoolingExp<Reducer, SrcExp, DType, ExpInfo<SrcExp>::kDim>\nchpool(const Exp<SrcExp, DType, etype> &src, index_t nsize, index_t stride, index_t pad) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim >= 3>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return ChannelPoolingExp<Reducer, SrcExp,\n                           DType, ExpInfo<SrcExp>::kDim>(src.self(), nsize, stride, pad);\n}\n\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename Reducer, typename SrcExp, typename DType, int srcdim>\nstruct Plan<ChannelPoolingExp<Reducer, SrcExp, DType, srcdim>, DType> {\n public:\n  explicit Plan(const ChannelPoolingExp<Reducer, SrcExp, DType, srcdim> &e)\n      : src_(MakePlan(e.src_)), channel_(e.shape_[srcdim - 3]),\n        height_(e.shape_[srcdim - 2]), width_(e.shape_[srcdim - 1]),\n        hnsize_(e.nsize_), stride_(e.stride_), pad_(e.pad_),\n        src_channel_(e.src_channel_) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    using namespace std;\n    const index_t y = i % height_;\n    i /= height_;\n    const index_t c = i % channel_;\n    const index_t n = i / channel_;\n    const index_t x = j;\n    const index_t cstart = c * stride_ < pad_ ? 0  : c * stride_ - pad_;\n    const index_t cend   = min(c * stride_ - pad_ + hnsize_, channel_);\n    DType res; Reducer::SetInitValue(res);\n    for (index_t cc = cstart; cc < cend; ++cc) {\n      Reducer::Reduce(res, src_.Eval((n * src_channel_ + cc) * height_ + y, x));\n    }\n    return res;\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t channel_, height_, width_, hnsize_, stride_, pad_, src_channel_;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_CHANNEL_POOL_H_\n\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/channel_unpool.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file channel_pool.h\n * \\brief support for chpool\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_EXTENSION_CHANNEL_UNPOOL_H_\n#define MSHADOW_EXTENSION_CHANNEL_UNPOOL_H_\n#include <algorithm>\n#include \"../extension.h\"\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief channel pooling expression, do reduction over (local nearby) channels,\n *        used to implement local response normalization\n * \\tparam Reducer reduction method during pooling\n * \\tparam SrcExp source expression to be pooled from\n * \\tparam DType the type of elements\n * \\tparam srcdim dimension of src\n */\ntemplate<typename Reducer, typename SrcExp, typename DType, int srcdim>\nstruct ChannelUnpoolingExp:\n      public MakeTensorExp<ChannelUnpoolingExp<Reducer, SrcExp, DType, srcdim>,\n                           SrcExp, srcdim, DType> {\n  /*! \\brief source input, corresponds to src in pooling */\n  const SrcExp &data_src_;\n  /*! \\brief result of pooled data, corresponds to result of pooling */\n  const SrcExp &data_pooled_;\n  /*! \\brief gradient data of pooled part, to be propgate down */\n  const SrcExp &grad_pooled_;\n  /*! \\brief channel of pooled expression */\n  index_t pchannel_;\n  /*! \\brief kernel size in height */\n  index_t nsize_;\n  /*! \\brief kernel size in width */\n  index_t kstride_;\n  /*! \\brief pad */\n  index_t pad_;\n  /*! \\brief constructor */\n  ChannelUnpoolingExp(const SrcExp &data_src,\n               const SrcExp &data_pooled,\n               const SrcExp &grad_pooled,\n               index_t nsize, index_t kstride, index_t pad)\n      : data_src_(data_src), data_pooled_(data_pooled),\n        grad_pooled_(grad_pooled),\n        nsize_(nsize), kstride_(kstride), pad_(pad) {\n    Shape<srcdim> pshape = ShapeCheck<srcdim, SrcExp>::Check(grad_pooled);\n    typedef ShapeCheck<srcdim, SrcExp> ShapeCheckSrcDimSrcExp;\n    CHECK_EQ(pshape, ShapeCheckSrcDimSrcExp::Check(data_pooled))\n      << \"ChannelUnPoolingExp: data and grad shape mismatch\";\n    Shape<srcdim> sshape = ShapeCheck<srcdim, SrcExp>::Check(data_src);\n    for (int k = 0; k < srcdim; ++k) {\n      if (k == 1) {\n        continue;\n      }\n      CHECK_EQ(pshape[k], sshape[k])\n        << \"ChannelUnPoolingExp: pooled tensor and src tensor shape mismatch\"\n        << pshape[k]\n        << \" vs \"\n        << sshape[k];\n    }\n    pchannel_ = pshape[1];\n    this->shape_ = sshape;\n  }\n};\n/*!\n * \\brief  channel unpooling, do unroll over (local nearby) channels\n * \\param src source data\n * \\param nsize neighbor size\n * \\param stride stride of the pooling\n * \\param pad number of padding at each side\n * \\return expression of pooled result\n * \\tparam Reducer reducer type\n * \\tparam SrcExp source expression\n * \\tparam DType the type of elements\n * \\tparam etype type of expression\n */\ntemplate<typename Reducer, typename SrcExp, typename DType, int etype>\ninline ChannelUnpoolingExp<Reducer, SrcExp, DType, ExpInfo<SrcExp>::kDim>\nch_unpool(const Exp<SrcExp, DType, etype> &data_src,\n       const Exp<SrcExp, DType, etype> &data_pooled,\n       const Exp<SrcExp, DType, etype> &grad_pooled,\n      index_t nsize, index_t stride, index_t pad) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim >= 3>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return ChannelUnpoolingExp<Reducer, SrcExp, DType, ExpInfo<SrcExp>::kDim>\n        (data_src.self(), data_pooled.self(), grad_pooled.self(), nsize, stride, pad);\n}\n\ntemplate<typename Reducer, typename SrcExp, typename DType, int etype>\ninline ChannelUnpoolingExp<Reducer, SrcExp, DType, ExpInfo<SrcExp>::kDim>\nch_unpool(const Exp<SrcExp, DType, etype> &data_src,\n       const Exp<SrcExp, DType, etype> &data_pooled,\n       const Exp<SrcExp, DType, etype> &grad_pooled, index_t nsize) {\n  return ch_unpool(data_src, data_pooled, grad_pooled, nsize, 1, nsize / 2);\n}\n\n\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename Reducer, typename SrcExp, typename DType, int srcdim>\nstruct Plan<ChannelUnpoolingExp<Reducer, SrcExp, DType, srcdim>, DType> {\n public:\n  explicit Plan(const ChannelUnpoolingExp<Reducer, SrcExp, DType, srcdim> &e)\n      : data_src_(e.data_src_), data_pooled_(e.data_pooled_),\n        grad_pooled_(e.grad_pooled_), channel_(e.shape_[srcdim - 3]),\n        height_(e.shape_[srcdim - 2]), pchannel_(e.pchannel_),\n        hnsize_(e.nsize_), stride_(e.kstride_), pad_(e.pad_) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    using namespace std;\n    const DType vsrc = data_src_.Eval(i, j);\n    const index_t y = i % height_;\n    i /= height_;\n    const index_t c = i % channel_;\n    const index_t n = i / channel_;\n    const index_t x = j;\n    const index_t cstart = c < hnsize_ - pad_ ? 0\n                        : (c - (hnsize_ - pad_) + stride_) / stride_;\n    const index_t cend = min((c + pad_ + stride_) / stride_, channel_);\n    DType val = static_cast<DType>(0);\n    for (index_t cc = cstart; cc < cend; ++cc) {\n      val += Reducer::PartialGrad(vsrc,\n                                  data_pooled_.Eval((n * pchannel_ + cc) * height_ + y, x)) *\n                                  grad_pooled_.Eval((n * pchannel_ + cc) * height_ + y, x);\n    }\n    return val;\n  }\n\n private:\n  Plan<SrcExp, DType> data_src_, data_pooled_, grad_pooled_;\n  const index_t channel_, height_, pchannel_, hnsize_, stride_, pad_;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_CHANNEL_UNPOOL_H_\n\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/choose.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file choose.h\n * \\brief support for implicit array selection operation\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_EXTENSION_CHOOSE_H_\n#define MSHADOW_EXTENSION_CHOOSE_H_\n\n#include \"../extension.h\"\n\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief Make a choice of index in the lowest changing dimension.\n * \\tparam SrcExp type of lhs expression\n * \\tparam IndexExp type of index expression\n * \\tparam DType the type of elements\n */\ntemplate<typename SrcExp, typename IndexExp, typename DType>\nstruct MatChooseRowElementExp:\n      public Exp<MatChooseRowElementExp<SrcExp, IndexExp, DType>,\n                 DType, type::kChainer> {\n  /*! \\brief source operand */\n  const SrcExp &src_;\n  /*! \\brief index operand */\n  const IndexExp &index_;\n  /*! \\brief constructor */\n  MatChooseRowElementExp(const SrcExp &src, const IndexExp &index)\n      : src_(src), index_(index) {}\n};\n\ntemplate<typename SrcExp, typename IndexExp,\n         typename DType, typename IDType, int e1, int e2>\ninline MatChooseRowElementExp<SrcExp, IndexExp, DType>\nmat_choose_row_element(const Exp<SrcExp, DType, e1> &src,\n                       const Exp<IndexExp, IDType, e2> &index) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim == 2 && ExpInfo<IndexExp>::kDim == 1>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return MatChooseRowElementExp<SrcExp, IndexExp, DType>(src.self(), index.self());\n}\n\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename SrcExp, typename IndexExp, typename DType>\nstruct Plan<MatChooseRowElementExp<SrcExp, IndexExp, DType>, DType> {\n public:\n  explicit Plan(const MatChooseRowElementExp<SrcExp, IndexExp, DType> &e)\n      : src_(MakePlan(e.src_)),\n        index_(MakePlan(e.index_)) {\n  }\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    index_t idx = static_cast<index_t>(index_.Eval(0, x));\n    return src_.Eval(x, idx);\n  }\n\n private:\n  expr::Plan<SrcExp, DType> src_;\n  expr::Plan<IndexExp, DType> index_;\n};\n\ntemplate<typename SrcExp, typename IndexExp, typename DType>\ninline Plan<MatChooseRowElementExp<SrcExp, IndexExp, DType>, DType>\nMakePlan(const MatChooseRowElementExp<SrcExp, IndexExp, DType> &exp) {\n  return Plan<MatChooseRowElementExp<SrcExp, IndexExp, DType>, DType>(exp);\n}\n\ntemplate<int dim, typename SrcExp, typename IndexExp, typename DType>\nstruct ShapeCheck<dim, MatChooseRowElementExp<SrcExp, IndexExp, DType> > {\n  inline static Shape<dim>\n  Check(const MatChooseRowElementExp<SrcExp, IndexExp, DType> &t) {\n    CHECK(dim == 1)\n        << \"MatChooseRowElementExp only support 1 dimension output\";\n    Shape<2> shape1 = ShapeCheck<2, SrcExp>::Check(t.src_);\n    Shape<dim> shape2 = ShapeCheck<dim, IndexExp>::Check(t.index_);\n    CHECK_EQ(shape1[0], shape2[0])\n        << \"mat_choose_row_element index length and number of rows in matrix\";\n    return shape2;\n  }\n};\n\ntemplate<typename SrcExp, typename IndexExp, typename DType>\nstruct ExpInfo<MatChooseRowElementExp<SrcExp, IndexExp, DType> > {\n  static const int kDim = 1;\n  static const int kDevMask = ExpInfo<SrcExp>::kDevMask & ExpInfo<IndexExp>::kDevMask;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_CHOOSE_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/complex.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file complex.h\n * \\brief support for complex operations\n * \\author Xingjian Shi\n */\n#ifndef MSHADOW_EXTENSION_COMPLEX_H_\n#define MSHADOW_EXTENSION_COMPLEX_H_\n#include <algorithm>\n#include \"../extension.h\"\n\nnamespace mshadow {\nnamespace op {\nnamespace complex {\nenum BinaryCalculationType { kBinaryCC, kBinaryCR, kBinaryRC};\nenum UnitaryCalculationType { kUnitaryC2R, kUnitaryC2C, kUnitaryR2C };\nstruct mul {\n  /*! \\brief map a_real, a_imag, b_real, b_imag to result using defined operation */\n  template<typename DType>\n  MSHADOW_XINLINE static DType RealMap(DType a_real, DType a_imag,\n    DType b_real, DType b_imag) {\n    return a_real * b_real - a_imag * b_imag;\n  }\n  template<typename DType>\n  MSHADOW_XINLINE static DType ImagMap(DType a_real, DType a_imag,\n    DType b_real, DType b_imag) {\n    return a_real * b_imag + b_real * a_imag;\n  }\n};\n\nstruct div {\n  /*! \\brief map a_real, a_imag, b_real, b_imag to result using defined operation */\n  template<typename DType>\n  MSHADOW_XINLINE static DType RealMap(DType a_real, DType a_imag,\n    DType b_real, DType b_imag) {\n    return (a_real * b_real + a_imag * b_imag) / (b_real * b_real + b_imag * b_imag);\n  }\n  template<typename DType>\n  MSHADOW_XINLINE static DType ImagMap(DType a_real, DType a_imag,\n    DType b_real, DType b_imag) {\n    return (b_real * a_imag - a_real * b_imag) / (b_real * b_real + b_imag * b_imag);\n  }\n};\n\nstruct conjugate {\n  template<typename TA, typename DType>\n  MSHADOW_XINLINE static DType RealMap(const expr::Plan<TA, DType> &src_,\n    index_t real_i, index_t real_j, index_t imag_i, index_t imag_j) {\n    return src_.Eval(real_i, real_j);\n  }\n  template<typename TA, typename DType>\n  MSHADOW_XINLINE static DType ImagMap(const expr::Plan<TA, DType> &src_,\n    index_t real_i, index_t real_j, index_t imag_i, index_t imag_j) {\n    return -src_.Eval(imag_i, imag_j);\n  }\n};\n\nstruct exchange {\n  template<typename TA, typename DType>\n  MSHADOW_XINLINE static DType RealMap(const expr::Plan<TA, DType> &src_,\n    index_t real_i, index_t real_j, index_t imag_i, index_t imag_j) {\n    return src_.Eval(imag_i, imag_j);\n  }\n  template<typename TA, typename DType>\n  MSHADOW_XINLINE static DType ImagMap(const expr::Plan<TA, DType> &src_,\n    index_t real_i, index_t real_j, index_t imag_i, index_t imag_j) {\n    return src_.Eval(real_i, real_j);\n  }\n};\n\n// r2c operator\nstruct pad_imag {\n  template<typename TA, typename DType>\n  MSHADOW_XINLINE static DType RealMap(const expr::Plan<TA, DType> &src_,\n    index_t real_i, index_t real_j) {\n    return src_.Eval(real_i, real_j);\n  }\n  template<typename TA, typename DType>\n  MSHADOW_XINLINE static DType ImagMap(const expr::Plan<TA, DType> &src_,\n    index_t real_i, index_t real_j) {\n    return 0;\n  }\n};\n\n// c2r operator\nstruct toreal {\n  template<typename TA, typename DType>\n  MSHADOW_XINLINE static DType RealMap(const expr::Plan<TA, DType> &src_,\n    index_t real_i, index_t real_j, index_t imag_i, index_t imag_j) {\n    DType real_val = src_.Eval(real_i, real_j);\n    return real_val;\n  }\n};\n\nstruct abs_square {\n  template<typename TA, typename DType>\n  MSHADOW_XINLINE static DType RealMap(const expr::Plan<TA, DType> &src_,\n    index_t real_i, index_t real_j, index_t imag_i, index_t imag_j) {\n    DType real_val = src_.Eval(real_i, real_j);\n    DType image_val = src_.Eval(imag_i, imag_j);\n    return real_val * real_val + image_val * image_val;\n  }\n};\n\nstruct sum_real_imag {\n  template<typename TA, typename DType>\n  MSHADOW_XINLINE static DType RealMap(const expr::Plan<TA, DType> &src_,\n    index_t real_i, index_t real_j, index_t imag_i, index_t imag_j) {\n    DType real_val = src_.Eval(real_i, real_j);\n    DType image_val = src_.Eval(imag_i, imag_j);\n    return real_val + image_val;\n  }\n};\n}  // namespace complex\n}  // namespace op\n\nnamespace expr {\n//--------------------\n// ComplexBinaryMapExp\n//--------------------\n  /*!\n* \\brief binary map expression lhs [op] rhs where lhs and rhs are complex tensors\n* \\tparam OP operator\n* \\tparam calctype type of the calculation\n* \\tparam TA type of lhs\n* \\tparam TB type of rhs\n* \\tparam etype expression type, sa namespace::type\n*/\ntemplate<int calctype, typename OP, typename TA, typename TB, typename DType, int etype>\nstruct ComplexBinaryMapExp : public Exp<ComplexBinaryMapExp<calctype, OP, TA, TB, DType, etype>,\n  DType, etype> {\n  /*! \\brief left operand */\n  const TA &lhs_;\n  /*! \\brief right operand */\n  const TB &rhs_;\n  /*! \\brief constructor */\n  explicit ComplexBinaryMapExp(const TA &lhs, const TB &rhs)\n    :lhs_(lhs), rhs_(rhs) {}\n};\n\n//-------------------\n// ComplexConjExp\n//-------------------\n/*!\n* \\brief compute conj(src) where src is a complex tensor\n* \\tparam TA type of src\n* \\tparam etype expression type, sa namespace::type\n*/\ntemplate<int calctype, typename OP, typename TA, typename DType, int etype>\nstruct ComplexUnitaryExp : public Exp<ComplexUnitaryExp<calctype, OP, TA, DType, etype>,\n  DType, etype> {\n  /*! \\brief source expression */\n  const TA &src_;\n  /*! \\brief constructor */\n  explicit ComplexUnitaryExp(const TA &src) : src_(src) {}\n};\n\n\n\ntemplate<int calctype, typename OP, typename TA, typename TB, typename DType, int ta, int tb>\ninline ComplexBinaryMapExp<calctype, OP, TA, TB, DType, (ta | tb | type::kMapper)>\nComplexF(const Exp<TA, DType, ta> &lhs, const Exp<TB, DType, tb> &rhs) {\n  return ComplexBinaryMapExp<calctype, OP, TA, TB, DType,\n    (ta | tb | type::kMapper)>(lhs.self(), rhs.self());\n}\n\n/*!\n* \\brief conj Negation the imaginary part of A where A is a complex tensor\n* \\param src source tensor\n* \\tparam e1 type of source expression\n*/\ntemplate<int calctype, typename OP, typename SrcExp, typename DType, int e1>\ninline ComplexUnitaryExp<calctype, OP, SrcExp, DType, (e1 | type::kMapper)>\nComplexF(const Exp<SrcExp, DType, e1> &src) {\n  return ComplexUnitaryExp<calctype, OP, SrcExp, DType, (e1 | type::kMapper)>(src.self());\n}\n\n/*!\n* \\brief complex_mul_cc Complex multipilication two complex tensors, A * B\n*/\ntemplate<typename TA, typename TB, typename DType, int ta, int tb>\ninline ComplexBinaryMapExp<op::complex::kBinaryCC, op::complex::mul,\n  TA, TB, DType, (ta | tb | type::kMapper)>\ncomplex_mul_cc(const Exp<TA, DType, ta> &lhs, const Exp<TB, DType, tb> &rhs) {\n  return ComplexF<op::complex::kBinaryCC, op::complex::mul>(lhs, rhs);\n}\n\n/*!\n* \\brief complex_mul_cr Complex multipilication a complex tensor A and a real tensor B\n*/\ntemplate<typename TA, typename TB, typename DType, int ta, int tb>\ninline ComplexBinaryMapExp<op::complex::kBinaryCR, op::complex::mul,\n  TA, TB, DType, (ta | tb | type::kMapper)>\ncomplex_mul_cr(const Exp<TA, DType, ta> &lhs, const Exp<TB, DType, tb> &rhs) {\n  return ComplexF<op::complex::kBinaryCR, op::complex::mul>(lhs, rhs);\n}\n\n/*!\n* \\brief complex_mul_rc Complex multipilication of a real tensor B and a complex tensor A\n*/\ntemplate<typename TA, typename TB, typename DType, int ta, int tb>\ninline ComplexBinaryMapExp<op::complex::kBinaryRC, op::complex::mul,\n  TA, TB, DType, (ta | tb | type::kMapper)>\ncomplex_mul_rc(const Exp<TA, DType, ta> &lhs, const Exp<TB, DType, tb> &rhs) {\n  return ComplexF<op::complex::kBinaryRC, op::complex::mul>(lhs, rhs);\n}\n\n/*!\n* \\brief complex_mul_cc Complex multipilication two complex tensors, A * B\n*/\ntemplate<typename TA, typename TB, typename DType, int ta, int tb>\ninline ComplexBinaryMapExp<op::complex::kBinaryCC, op::complex::div,\n  TA, TB, DType, (ta | tb | type::kMapper)>\ncomplex_div_cc(const Exp<TA, DType, ta> &lhs, const Exp<TB, DType, tb> &rhs) {\n  return ComplexF<op::complex::kBinaryCC, op::complex::div>(lhs, rhs);\n}\n\n/*!\n* \\brief complex_mul_cr Complex multipilication a complex tensor A and a real tensor B\n*/\ntemplate<typename TA, typename TB, typename DType, int ta, int tb>\ninline ComplexBinaryMapExp<op::complex::kBinaryCR, op::complex::div,\n  TA, TB, DType, (ta | tb | type::kMapper)>\ncomplex_div_cr(const Exp<TA, DType, ta> &lhs, const Exp<TB, DType, tb> &rhs) {\n  return ComplexF<op::complex::kBinaryCR, op::complex::div>(lhs, rhs);\n}\n\n/*!\n* \\brief complex_mul_rc Complex multipilication of a real tensor A and a complex tensor B\n*/\ntemplate<typename TA, typename TB, typename DType, int ta, int tb>\ninline ComplexBinaryMapExp<op::complex::kBinaryRC, op::complex::div,\n  TA, TB, DType, (ta | tb | type::kMapper)>\ncomplex_div_rc(const Exp<TA, DType, ta> &lhs, const Exp<TB, DType, tb> &rhs) {\n  return ComplexF<op::complex::kBinaryRC, op::complex::div>(lhs, rhs);\n}\n\n/*!\n* \\brief conj Negation the imaginary part of A where A is a complex tensor\n* \\param src source tensor\n* \\tparam e1 type of source expression\n*/\ntemplate<typename SrcExp, typename DType, int e1>\ninline ComplexUnitaryExp<op::complex::kUnitaryC2C, op::complex::conjugate,\n  SrcExp, DType, (e1|type::kMapper)>\nconj(const Exp<SrcExp, DType, e1> &src) {\n  return ComplexF<op::complex::kUnitaryC2C, op::complex::conjugate>(src);\n}\n\n/*!\n* \\brief complex_exchange Exchange the real and imaginary part of A where A is a complex tensor\n* \\param src source tensor\n* \\tparam e1 type of source expression\n*/\ntemplate<typename SrcExp, typename DType, int e1>\ninline ComplexUnitaryExp<op::complex::kUnitaryC2C, op::complex::exchange,\n  SrcExp, DType, (e1|type::kMapper)>\ncomplex_exchange(const Exp<SrcExp, DType, e1> &src) {\n  return ComplexF<op::complex::kUnitaryC2C, op::complex::exchange>(src);\n}\n\n/*!\n* \\brief complex_pad_imag Transform real matrix into complex matrix\n* \\param src source tensor\n* \\tparam e1 type of source expression\n*/\ntemplate<typename SrcExp, typename DType, int e1>\ninline ComplexUnitaryExp<op::complex::kUnitaryR2C, op::complex::pad_imag,\n  SrcExp, DType, (e1|type::kMapper)>\ncomplex_pad_imag(const Exp<SrcExp, DType, e1> &src) {\n  return ComplexF<op::complex::kUnitaryR2C, op::complex::pad_imag>(src);\n}\n\n/*!\n* \\brief complex_toreal convert complex matrix to real matrix, keep only real part\n* \\param src source tensor\n* \\tparam e1 type of source expression\n*/\ntemplate<typename SrcExp, typename DType, int e1>\ninline ComplexUnitaryExp<op::complex::kUnitaryC2R, op::complex::toreal,\n  SrcExp, DType, (e1 | type::kMapper)>\ncomplex_toreal(const Exp<SrcExp, DType, e1> &src) {\n  return ComplexF<op::complex::kUnitaryC2R, op::complex::toreal>(src);\n}\n\n/*!\n* \\brief complex_abs_square calculate the square of the modulus of A where A is a complex tensor\n* \\param src source tensor\n* \\tparam e1 type of source expression\n*/\ntemplate<typename SrcExp, typename DType, int e1>\ninline ComplexUnitaryExp<op::complex::kUnitaryC2R, op::complex::abs_square,\n  SrcExp, DType, (e1 | type::kMapper)>\ncomplex_abs_square(const Exp<SrcExp, DType, e1> &src) {\n  return ComplexF<op::complex::kUnitaryC2R, op::complex::abs_square>(src);\n}\n\ntemplate<typename SrcExp, typename DType, int e1>\ninline ComplexUnitaryExp<op::complex::kUnitaryC2R, op::complex::sum_real_imag,\n  SrcExp, DType, (e1 | type::kMapper)>\ncomplex_sum_real_imag(const Exp<SrcExp, DType, e1> &src) {\n  return ComplexF<op::complex::kUnitaryC2R, op::complex::sum_real_imag>(src);\n}\n\ntemplate<int dim, int calctype, typename OP, typename TA, typename TB,\n  typename DType, int etype>\nstruct ShapeCheck<dim, ComplexBinaryMapExp<calctype, OP, TA, TB, DType, etype> > {\n  inline static Shape<dim>\n    Check(const ComplexBinaryMapExp<calctype, OP, TA, TB, DType, etype> &t) {\n    Shape<dim> shape1 = ShapeCheck<dim, TA>::Check(t.lhs_);\n    Shape<dim> shape2 = ShapeCheck<dim, TB>::Check(t.rhs_);\n    if (shape1[0] == 0) return shape2;\n    if (shape2[0] == 0) return shape1;\n    if (calctype == op::complex::kBinaryCC) {\n      CHECK_EQ(shape1, shape2) << \"ComplexBinaryMapExp (CC): Shapes of operands are not the same.\";\n      CHECK_EQ(shape1[dim - 1] % 2, 0) <<\n        \"ComplexBinaryMapExp (CC): Shape of the last dimension is not even. \"\n        \"We must have real part + imaginary part.\";\n      return shape1;\n    } else if (calctype == op::complex::kBinaryCR) {\n      for (int i = 0; i < dim - 1; ++i) {\n        CHECK_EQ(shape1.shape_[i], shape2.shape_[i]) <<\n          \"ComplexBinaryMapExp (CR): Shapes of operands are not the same.\";\n      }\n      CHECK_EQ(shape1[dim - 1], shape2[dim - 1] * 2) <<\n        \"ComplexBinaryMapExp (CR): Shapes of operands do not match.\";\n      return shape1;\n    } else if (calctype == op::complex::kBinaryRC) {\n      for (int i = 0; i < dim - 1; ++i) {\n        CHECK_EQ(shape1.shape_[i], shape2.shape_[i]) <<\n          \"ComplexBinaryMapExp (RC): Shapes of operands are not the same.\";\n      }\n      CHECK_EQ(shape2[dim - 1], shape1[dim - 1] * 2) <<\n        \"ComplexBinaryMapExp (RC): Shapes of operands do not match.\";\n      return shape2;\n    } else {\n      LOG(FATAL) << \"ComplexBinaryMapExp: Unexpected Calculation Type!\";\n      return shape1;\n    }\n  }\n};\n\ntemplate<int dim, int calctype, typename OP, typename TA, typename DType, int etype>\nstruct ShapeCheck<dim, ComplexUnitaryExp<calctype, OP, TA, DType, etype> > {\n  inline static Shape<dim> Check(const ComplexUnitaryExp<calctype, OP, TA, DType, etype> &t) {\n    Shape<dim> s = ShapeCheck<dim, TA>::Check(t.src_);\n    CHECK_EQ(s[dim - 1] % 2, 0) << \"ComplexUnitaryExp: Shape of the last dimension is not even. \"\n      \"We must have real + imaginary.\";\n    if (calctype == op::complex::kUnitaryC2C) {\n      return s;\n    } else if (calctype == op::complex::kUnitaryC2R) {\n      Shape<dim> s_ret = s;\n      s_ret[dim - 1] /= 2;\n      return s_ret;\n    } else if (calctype == op::complex::kUnitaryR2C) {\n      Shape<dim> s_ret = s;\n      s_ret[dim-1] *= 2;\n      return s_ret;\n    } else {\n      LOG(FATAL) << \"ComplexUnitaryExp: Unexpected Calculation Type!\";\n      return s;\n    }\n  }\n};\n\n\n\n// complex binary expression (cc)\ntemplate<typename OP, typename TA, typename TB, int etype, typename DType>\nclass Plan<ComplexBinaryMapExp<op::complex::kBinaryCC, OP, TA, TB, DType, etype>, DType> {\n public:\n  explicit Plan(const Plan<TA, DType> &lhs, const Plan<TB, DType> &rhs)\n    : lhs_(lhs), rhs_(rhs) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    const index_t base_x = static_cast<index_t>(x / 2) * 2;\n    if (x % 2 == 0) {\n      return OP::RealMap(lhs_.Eval(y, base_x), lhs_.Eval(y, base_x + 1),\n        rhs_.Eval(y, base_x), rhs_.Eval(y, base_x + 1));\n    } else {\n      return OP::ImagMap(lhs_.Eval(y, base_x), lhs_.Eval(y, base_x + 1),\n        rhs_.Eval(y, base_x), rhs_.Eval(y, base_x + 1));\n    }\n  }\n\n private:\n  Plan<TA, DType> lhs_;\n  Plan<TB, DType> rhs_;\n};\n\n// complex binary expression (cr)\ntemplate<typename OP, typename TA, typename TB, int etype, typename DType>\nclass Plan<ComplexBinaryMapExp<op::complex::kBinaryCR, OP, TA, TB, DType, etype>, DType> {\n public:\n  explicit Plan(const Plan<TA, DType> &lhs, const Plan<TB, DType> &rhs)\n    : lhs_(lhs), rhs_(rhs) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    const index_t base_x = static_cast<index_t>(x / 2) * 2;\n    if (x % 2 == 0) {\n      return OP::RealMap(lhs_.Eval(y, base_x), lhs_.Eval(y, base_x + 1),\n        rhs_.Eval(y, base_x / 2), static_cast<DType>(0));\n    } else {\n      return OP::ImagMap(lhs_.Eval(y, base_x), lhs_.Eval(y, base_x + 1),\n        rhs_.Eval(y, base_x / 2), static_cast<DType>(0));\n    }\n  }\n\n private:\n  Plan<TA, DType> lhs_;\n  Plan<TB, DType> rhs_;\n};\n\n\n// complex binary expression (rc)\ntemplate<typename OP, typename TA, typename TB, int etype, typename DType>\nclass Plan<ComplexBinaryMapExp<op::complex::kBinaryRC, OP, TA, TB, DType, etype>, DType> {\n public:\n  explicit Plan(const Plan<TA, DType> &lhs, const Plan<TB, DType> &rhs)\n    : lhs_(lhs), rhs_(rhs) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    const index_t base_x = static_cast<index_t>(x / 2) * 2;\n    if (x % 2 == 0) {\n      return OP::RealMap(lhs_.Eval(y, base_x / 2), static_cast<DType>(0),\n        rhs_.Eval(y, base_x), rhs_.Eval(y, base_x + 1));\n    } else {\n      return OP::ImagMap(lhs_.Eval(y, base_x / 2), static_cast<DType>(0),\n        rhs_.Eval(y, base_x), rhs_.Eval(y, base_x + 1));\n    }\n  }\n\n private:\n  Plan<TA, DType> lhs_;\n  Plan<TB, DType> rhs_;\n};\n\n\n// complex unitary expression (c2c)\ntemplate<typename OP, typename TA, int etype, typename DType>\nclass Plan<ComplexUnitaryExp<op::complex::kUnitaryC2C, OP, TA, DType, etype>, DType> {\n public:\n  explicit Plan(const Plan<TA, DType> &src) : src_(src) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    const index_t base_x = static_cast<index_t>(x / 2) * 2;\n    if (0 == x % 2) {\n      return OP::RealMap(src_, y, base_x, y, base_x + 1);\n    } else {\n      return OP::ImagMap(src_, y, base_x, y, base_x + 1);\n    }\n  }\n\n private:\n  Plan<TA, DType> src_;\n};\n\n// complex unitary expression (r2c)\ntemplate<typename OP, typename TA, int etype, typename DType>\nclass Plan<ComplexUnitaryExp<op::complex::kUnitaryR2C, OP, TA, DType, etype>, DType> {\n public:\n  explicit Plan(const Plan<TA, DType> &src) : src_(src) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    const index_t real_x = static_cast<index_t>(x / 2);\n    if (0 == x%2) {\n      // x,y should be coordinates in the complex matrix\n      // this defines how we will give value to the real part from the real matrix src_,\n      // thus the index has only 2 dimensions\n      return OP::RealMap(src_, y, real_x);\n    } else {\n      return OP::ImagMap(src_, y, real_x);\n    }\n  }\n\n private:\n  Plan<TA, DType> src_;\n};\n\n// complex unitary expression (c2r)\ntemplate<typename OP, typename TA, int etype, typename DType>\nclass Plan<ComplexUnitaryExp<op::complex::kUnitaryC2R, OP, TA, DType, etype>, DType> {\n public:\n  explicit Plan(const Plan<TA, DType> &src) : src_(src) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    return OP::RealMap(src_, y, x * 2, y, x * 2 + 1);\n  }\n\n private:\n  Plan<TA, DType> src_;\n};\n\n\n\ntemplate<int calctype, typename OP, typename TA, typename TB, typename DType, int etype>\ninline Plan<ComplexBinaryMapExp<calctype, OP, TA, TB, DType, etype>, DType>\nMakePlan(const ComplexBinaryMapExp<calctype, OP, TA, TB, DType, etype> &e) {\n  return Plan<ComplexBinaryMapExp<calctype, OP, TA, TB, DType, etype>,\n    DType>(MakePlan(e.lhs_), MakePlan(e.rhs_));\n}\n\ntemplate<int calctype, typename OP, typename TA, typename DType, int etype>\ninline Plan<ComplexUnitaryExp<calctype, OP, TA, DType, etype>, DType>\nMakePlan(const ComplexUnitaryExp<calctype, OP, TA, DType, etype> &e) {\n  return Plan<ComplexUnitaryExp<calctype, OP, TA, DType, etype>,\n    DType>(MakePlan(e.src_));\n}\n\n\n\ntemplate<int calctype, typename OP, typename TA, typename TB, typename DType, int etype>\nstruct ExpInfo<ComplexBinaryMapExp<calctype, OP, TA, TB, DType, etype> > {\n  static const int kDimLhs = ExpInfo<TA>::kDim;\n  static const int kDimRhs = ExpInfo<TB>::kDim;\n  static const int kDim = (kDimLhs >= 0 && kDimRhs >= 0) ? \\\n    (kDimLhs == 0 ? \\\n  kDimRhs : \\\n            ((kDimRhs == 0 || kDimLhs == kDimRhs) ? kDimLhs : -1)) : -1;\n  static const int kDevMask = ExpInfo<TA>::kDevMask & ExpInfo<TB>::kDevMask;\n};\n\ntemplate<int calctype, typename OP, typename TA, typename DType, int etype>\nstruct ExpInfo<ComplexUnitaryExp<calctype, OP, TA, DType, etype> > {\n  static const int kDim = ExpInfo<TA>::kDim;\n  static const int kDevMask = ExpInfo<TA>::kDevMask;\n};\n\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_COMPLEX_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/concat.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file concat.h\n * \\brief support for concatenation\n */\n#ifndef MSHADOW_EXTENSION_CONCAT_H_\n#define MSHADOW_EXTENSION_CONCAT_H_\n\n#include \"../extension.h\"\n\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief concat expression, concat two tensor's channel\n * \\tparam LhsExp left expression\n * \\tparam RhsExp right expression\n * \\tparam DType the type of elements\n * \\tparam srcdim dimension of src\n * \\tparam dimsrc_m_cat dimsrc - dimcat\n */\ntemplate<typename LhsExp, typename RhsExp,\n         typename Device, typename DType,\n         int srcdim, int dimsrc_m_cat>\nstruct ConcatExp : public TRValue<ConcatExp<LhsExp, RhsExp,\n                                            Device, DType,\n                                            srcdim, dimsrc_m_cat>,\n                                  Device, srcdim, DType> {\n  static const int dimcat = srcdim - dimsrc_m_cat;\n  const LhsExp &src1_;\n  const RhsExp &src2_;\n  index_t dcat_src1_;\n  index_t dcat_src2_;\n  Shape<4> shape_;\n  ConcatExp(const LhsExp &src1, const RhsExp &src2) : src1_(src1), src2_(src2) {\n    Shape<srcdim> sshape1 = ShapeCheck<srcdim, LhsExp>::Check(src1_);\n    Shape<srcdim> sshape2 = ShapeCheck<srcdim, RhsExp>::Check(src2_);\n    #pragma unroll\n    for (int i = 0; i < srcdim; ++i) {\n      if (i != dimcat) {\n        CHECK_EQ(sshape1[i], sshape2[i]) << \"ConcatExp: shape mismatch\";\n      }\n    }\n    this->shape_ = sshape1;\n    this->shape_[dimcat] = sshape1[dimcat] + sshape2[dimcat];\n    this->dcat_src1_ = sshape1[dimcat];\n    this->dcat_src2_ = sshape2[dimcat];\n  }\n  template<typename E, int etype>\n  inline void\n  operator=(const expr::Exp<E, DType, etype> &exp) {\n    this->__assign(exp);\n  }\n  inline void\n  operator=(const DType &exp) {\n    this->__assign(exp);\n  }\n};  // struct ConcatExp\n/*!\n * \\brief concat two 4D tensor\n * \\param src1 source tensor1\n * \\param src2 source tensor2\n * \\return concated 4D tensor\n * \\tparam cdim the dimension to concatnate on\n * \\tparam SrcExp source expression\n * \\tparam DType the type of elements\n * \\tparam etype type of expression\n */\ntemplate<int cdim, typename LhsExp, typename RhsExp,\n         typename Device, typename DType, int srcdim>\ninline ConcatExp<LhsExp, RhsExp, Device, DType, srcdim, srcdim - cdim>\nconcat(const TRValue<LhsExp, Device, srcdim, DType> &src1,\n       const TRValue<RhsExp, Device, srcdim, DType> &src2) {\n  TypeCheckPass<ExpInfo<LhsExp>::kDim == ExpInfo<RhsExp>::kDim>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  TypeCheckPass<cdim < srcdim && ExpInfo<LhsExp>::kDim == srcdim>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return ConcatExp<LhsExp, RhsExp, Device, DType, srcdim, srcdim - cdim>\n      (src1.self(), src2.self());\n}\n//------------------------\n//  engine plugin\n//------------------------\n// runtime shapecheck\ntemplate<typename LhsExp, typename RhsExp,\n         typename Device, typename DType,\n         int srcdim, int dimsrc_m_cat>\nstruct ShapeCheck<srcdim, ConcatExp<LhsExp, RhsExp, Device, DType, srcdim, dimsrc_m_cat> >{\n  inline static Shape<srcdim> Check(const ConcatExp<LhsExp, RhsExp,\n                                    Device, DType, srcdim, dimsrc_m_cat> &t) {\n    return t.shape_;\n  }\n};\ntemplate<typename LhsExp, typename RhsExp,\n         typename Device, typename DType,\n         int srcdim, int dimsrc_m_cat>\nstruct StreamInfo<Device, ConcatExp<LhsExp, RhsExp, Device, DType, srcdim, dimsrc_m_cat> >{\n  inline static Stream<Device> *\n  Get(const ConcatExp<LhsExp, RhsExp, Device, DType, srcdim, dimsrc_m_cat> &t) {\n    Stream<Device> *lhs = StreamInfo<Device, LhsExp>::Get(t.src1_);\n    Stream<Device> *rhs = StreamInfo<Device, RhsExp>::Get(t.src2_);\n    if (lhs != rhs) return NULL;\n    return lhs;\n  }\n};\n// static typecheck\ntemplate<typename LhsExp, typename RhsExp,\n         typename Device, typename DType,\n         int srcdim, int dimsrc_m_cat>\nstruct ExpInfo<ConcatExp<LhsExp, RhsExp, Device, DType, srcdim, dimsrc_m_cat> >{\n  static const int kDimLhs = ExpInfo<LhsExp>::kDim;\n  static const int kDimRhs = ExpInfo<RhsExp>::kDim;\n  // copy from binarymap\n  static const int kDim = (kDimLhs >= 0 && kDimRhs >= 0) ?\\\n      (kDimLhs == 0 ?\\\n       kDimRhs :\\\n       ((kDimRhs == 0 || kDimLhs == kDimRhs) ? kDimLhs : -1)) : -1;\n  static const int kDevMask = ExpInfo<LhsExp>::kDevMask & ExpInfo<RhsExp>::kDevMask;\n};\n//----------------------\n// Execution plan\n//---------------------\ntemplate<typename LhsExp, typename RhsExp,\n         typename Device, typename DType,\n         int srcdim, int dimsrc_m_cat>\nstruct Plan<ConcatExp<LhsExp, RhsExp, Device, DType, srcdim, dimsrc_m_cat>, DType> {\n public:\n  static const int dimcat = srcdim - dimsrc_m_cat;\n  explicit Plan(const ConcatExp<LhsExp, RhsExp, Device, DType, srcdim, dimsrc_m_cat> &e)\n      : src1_(MakePlan(e.src1_)), src2_(MakePlan(e.src2_)),\n        height_(e.shape_.ProdShape(dimcat + 1, srcdim - 1)),\n        ch_src1_(e.dcat_src1_), ch_src2_(e.dcat_src2_), ch_(e.shape_[dimcat]) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    const index_t y = i % height_;\n    i /= height_;\n    const index_t c = i % ch_;\n    const index_t b = i / ch_;\n    const index_t x = j;\n    if (c < ch_src1_) {\n      return src1_.Eval((b * ch_src1_ + c) * height_ + y, x);\n    } else {\n      return src2_.Eval((b * ch_src2_ + c - ch_src1_) * height_ + y, x);\n    }\n  }\n  MSHADOW_XINLINE DType &REval(index_t i, index_t j) {\n    const index_t y = i % height_;\n    i /= height_;\n    const index_t c = i % ch_;\n    const index_t b = i / ch_;\n    const index_t x = j;\n    if (c < ch_src1_) {\n      return src1_.REval((b * ch_src1_ + c) * height_ + y, x);\n    } else {\n      return src2_.REval((b * ch_src2_ + c - ch_src1_) * height_ + y, x);\n    }\n  }\n\n private:\n  Plan<LhsExp, DType> src1_;\n  Plan<RhsExp, DType> src2_;\n  const index_t height_, ch_src1_, ch_src2_, ch_;\n};  // struct Plan\n\n// specialize for concat in x\ntemplate<typename LhsExp, typename RhsExp,\n         typename Device, typename DType,\n         int srcdim>\nstruct Plan<ConcatExp<LhsExp, RhsExp, Device, DType, srcdim, 1>, DType> {\n public:\n  explicit Plan(const ConcatExp<LhsExp, RhsExp, Device, DType, srcdim, 1> &e)\n      : src1_(MakePlan(e.src1_)), src2_(MakePlan(e.src2_)),\n        width_src1_(e.dcat_src1_) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    if (x < width_src1_) {\n      return src1_.Eval(y, x);\n    } else {\n      return src2_.Eval(y, x - width_src1_);\n    }\n  }\n  MSHADOW_XINLINE DType &REval(index_t y, index_t x) {\n    if (x < width_src1_) {\n      return src1_.REval(y, x);\n    } else {\n      return src2_.REval(y, x - width_src1_);\n    }\n  }\n\n private:\n  Plan<LhsExp, DType> src1_;\n  Plan<RhsExp, DType> src2_;\n  const index_t width_src1_;\n};\n}  // namespace expr\n}   // namespace mshadow\n#endif  // MSHADOW_EXTENSION_CONCAT_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/crop.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file crop.h\n * \\brief support for crop\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_EXTENSION_CROP_H_\n#define MSHADOW_EXTENSION_CROP_H_\n#include \"../extension.h\"\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief crop expression, cut off the boundary region, reverse operation of padding\n * \\tparam SrcExp source expression to be pooled from\n * \\tparam DType the type of elements\n * \\tparam srcdim dimension of src\n */\ntemplate<typename SrcExp, typename DType, int srcdim>\nstruct CroppingExp:\n      public MakeTensorExp<CroppingExp<SrcExp, DType, srcdim>,\n                           SrcExp, srcdim, DType> {\n  /*! \\brief source operand */\n  const SrcExp &src_;\n  /*! \\brief pad height */\n  index_t pad_height_;\n  /*! \\brief pad height */\n  index_t pad_width_;\n  /*! \\brief src height */\n  index_t src_height_;\n  /*! \\brief constructor */\n  explicit CroppingExp(const SrcExp &src, Shape<2> cshape)\n      : src_(src) {\n    this->shape_ = ShapeCheck<srcdim, SrcExp>::Check(src_);\n    CHECK_GE(this->shape_[srcdim - 2], cshape[0]) << \"CroppingExp: height requirement not met\";\n    CHECK_GE(this->shape_[srcdim - 1], cshape[1]) << \"CroppingExp: width requirement not met\";\n    pad_height_ = (this->shape_[srcdim - 2] - cshape[0]) / 2;\n    pad_width_ = (this->shape_[srcdim - 1] - cshape[1]) / 2;\n    src_height_ = this->shape_[srcdim - 2];\n    this->shape_[srcdim - 2] = cshape[0];  // height\n    this->shape_[srcdim - 1] = cshape[1];  // width\n  }\n  /*! \\brief constructor */\n  explicit CroppingExp(const SrcExp &src, Shape<2> cshape,\n                       index_t start_height, index_t start_width)\n      : src_(src), pad_height_(start_height), pad_width_(start_width) {\n    this->shape_ = ShapeCheck<srcdim, SrcExp>::Check(src_);\n    CHECK_GE(this->shape_[srcdim - 2], cshape[0] + start_height)\n      << \"CroppingExp: height requirement not met\";\n    CHECK_GE(this->shape_[srcdim - 1], cshape[1] + start_width)\n      << \"CroppingExp: width requirement not met\";\n    src_height_ = this->shape_[srcdim - 2];\n    this->shape_[srcdim - 2] = cshape[0];  // height\n    this->shape_[srcdim - 1] = cshape[1];  // width\n  }\n};  // struct CroppingExp\n/*!\n * \\brief revserse operationg of padding, cut off boundaries,\n *   crop output from center of input\n * \\param src original image batches\n * \\param oshape output shape to be cropped\n * \\return expression corresponding to padded result\n * \\tparam SrcExp source expression\n * \\tparam DType the type of elements\n * \\tparam etype type of expression\n */\ntemplate<typename SrcExp, typename DType, int etype>\ninline CroppingExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>\ncrop(const Exp<SrcExp, DType, etype> &src, Shape<2> oshape) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim >= 2>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return CroppingExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>(src.self(), oshape);\n}\n/*!\n * \\brief same as crop, but can specify starting position to do cropping\n * \\param src original image batches\n * \\param oshape output shape to be cropped\n * \\param start_height start height position to do cropping\n * \\param start_width  start width position to do cropping\n * \\return expression corresponding to padded result\n * \\tparam SrcExp source expression\n * \\tparam DType the type of elements\n * \\tparam etype type of expression\n */\ntemplate<typename SrcExp, typename DType, int etype>\ninline CroppingExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>\ncrop(const Exp<SrcExp, DType, etype> &src, Shape<2> oshape,\n     index_t start_height, index_t start_width) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim >= 2>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return CroppingExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>\n      (src.self(), oshape, start_height, start_width);\n}\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename SrcExp, typename DType, int srcdim>\nstruct Plan<CroppingExp<SrcExp, DType, srcdim>, DType> {\n public:\n  explicit Plan(const CroppingExp<SrcExp, DType, srcdim> &e)\n      : src_(MakePlan(e.src_)),\n        pad_height_(e.pad_height_), pad_width_(e.pad_width_),\n        new_height_(e.shape_[srcdim - 2]), src_height_(e.src_height_) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    const index_t x = j;\n    const index_t y = i % new_height_;\n    const index_t c = i / new_height_;\n    const index_t h = y + pad_height_;\n    const index_t w = x + pad_width_;\n    return src_.Eval(c * src_height_ + h, w);\n  }\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t pad_height_, pad_width_;\n  const index_t new_height_;\n  const index_t src_height_;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_CROP_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/fill.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file fill.h\n * \\brief support for implicit array filling operation\n * \\author Xingjian Shi\n */\n#ifndef MSHADOW_EXTENSION_FILL_H_\n#define MSHADOW_EXTENSION_FILL_H_\n\n#include \"../extension.h\"\n\n\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief Set value of a specific element in each line of the data matrix.\n * \\tparam SrcExp type of src expression\n * \\tparam ValExp type of val expression\n * \\tparam IndexExp type of index expression\n * \\tparam DType the type of ret expression\n */\ntemplate<typename SrcExp, typename ValExp, typename IndexExp, typename DType>\nstruct MatFillRowElementExp:\n      public Exp<MatFillRowElementExp<SrcExp, ValExp, IndexExp, DType>,\n                 DType, type::kChainer> {\n  /*! \\brief src operand */\n  const SrcExp &src_;\n  const ValExp &val_;\n  /*! \\brief index operand */\n  const IndexExp &index_;\n  /*! \\brief constructor */\n  MatFillRowElementExp(const SrcExp &src, const ValExp &val, const IndexExp &index)\n      : src_(src), val_(val), index_(index) {}\n};\n\ntemplate<typename SrcExp, typename ValExp, typename IndexExp,\n        typename SDType, typename VDType, typename IDType, int e1, int e2, int e3>\ninline MatFillRowElementExp<SrcExp, ValExp, IndexExp, SDType>\nmat_fill_row_element(const Exp<SrcExp, SDType, e1> &src,\n                     const Exp<ValExp, VDType, e2> &val,\n                     const Exp<IndexExp, IDType, e3> &index) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim == 2 && ExpInfo<ValExp>::kDim == 1\n                && ExpInfo<IndexExp>::kDim == 1>::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return MatFillRowElementExp<SrcExp, ValExp, IndexExp, SDType>(src.self(),\n                                                                val.self(), index.self());\n}\n\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename SrcExp, typename ValExp, typename IndexExp, typename DType>\nstruct Plan<MatFillRowElementExp<SrcExp, ValExp, IndexExp, DType>, DType> {\n public:\n  explicit Plan(const MatFillRowElementExp<SrcExp, ValExp, IndexExp, DType> &e)\n      : src_(MakePlan(e.src_)),\n        val_(MakePlan(e.val_)),\n        index_(MakePlan(e.index_)) {\n  }\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    index_t idx = static_cast<index_t>(index_.Eval(0, y));\n    if (idx == x) {\n      return static_cast<DType>(val_.Eval(0, y));\n    } else {\n      return static_cast<DType>(src_.Eval(y, x));\n    }\n  }\n\n private:\n  expr::Plan<SrcExp, DType> src_;\n  expr::Plan<ValExp, DType> val_;\n  expr::Plan<IndexExp, DType> index_;\n};\n\ntemplate<typename SrcExp, typename ValExp, typename IndexExp, typename DType>\ninline Plan<MatFillRowElementExp<SrcExp, ValExp, IndexExp, DType>, DType>\nMakePlan(const MatFillRowElementExp<SrcExp, ValExp, IndexExp, DType> &exp) {\n  return Plan<MatFillRowElementExp<SrcExp, ValExp, IndexExp, DType>, DType>(exp);\n}\n\ntemplate<int dim, typename SrcExp, typename ValExp, typename IndexExp, typename DType>\nstruct ShapeCheck<dim, MatFillRowElementExp<SrcExp, ValExp, IndexExp, DType> > {\n  inline static Shape<dim>\n  Check(const MatFillRowElementExp<SrcExp, ValExp, IndexExp, DType> &t) {\n    CHECK(dim == 2)\n        << \"MatFillRowElementExp only support 2 dimension output\";\n    Shape<2> shape_src = ShapeCheck<2, SrcExp>::Check(t.src_);\n    Shape<1> shape_val = ShapeCheck<1, ValExp>::Check(t.val_);\n    Shape<1> shape_index = ShapeCheck<1, IndexExp>::Check(t.index_);\n    CHECK((shape_src[0] == shape_index[0]) && (shape_index[0] == shape_val[0]))\n        << \"mat_fill_row_element index length, val length and number of rows in matrix\";\n    return shape_src;\n  }\n};\n\ntemplate<typename SrcExp, typename ValExp, typename IndexExp, typename DType>\nstruct ExpInfo<MatFillRowElementExp<SrcExp, ValExp, IndexExp, DType> > {\n  static const int kDim = 2;\n  static const int kDevMask =\n          ExpInfo<SrcExp>::kDevMask & ExpInfo<ValExp>::kDevMask & ExpInfo<IndexExp>::kDevMask;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_FILL_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/flip.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file flip.h\n * \\brief support for flip a certain dimension.\n * \\author Junyuan Xie\n */\n#ifndef MSHADOW_EXTENSION_FLIP_H_\n#define MSHADOW_EXTENSION_FLIP_H_\n\n#include \"../extension.h\"\n\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief slice expression, slice a tensor's channel\n * \\tparam SrcExp left expression\n * \\tparam DType the type of elements\n * \\tparam srcdim dimension of src\n * \\tparam dimsrc_m_cat dimsrc - dimcat\n */\ntemplate<typename SrcExp, typename Device,\n         typename DType, int srcdim>\nstruct FlipExp : public TRValue<FlipExp<SrcExp,\n                                        Device, DType,\n                                        srcdim>,\n                                Device, srcdim, DType> {\n  const SrcExp &src_;\n  index_t trailing_;\n  index_t stride_;\n  index_t stride_j_;\n  Shape<srcdim> shape_;\n  FlipExp(const SrcExp &src, int dim)\n      : src_(src) {\n    shape_ = ShapeCheck<srcdim, SrcExp>::Check(src_);\n    stride_ = shape_[dim];\n    stride_j_ = shape_[srcdim-1];\n    trailing_ = 1;\n    for (int i = dim + 1; i < srcdim; ++i) {\n      trailing_ *= shape_[i];\n    }\n  }\n  template<typename E, int etype>\n  inline void\n  operator=(const expr::Exp<E, DType, etype> &exp) {\n    this->__assign(exp);\n  }\n  inline void\n  operator=(const DType &exp) {\n    this->__assign(exp);\n  }\n};  // struct Flip\n\n/*!\n * \\brief Flip a Tensor\n * \\param src source tensor\n * \\param begin The beginning slice.\n * \\param end The end slice.\n * \\return sliced tensor\n * \\tparam sdim the dimension to slice on\n * \\tparam SrcExp source expression\n * \\tparam DType the type of elements\n * \\tparam etype type of expression\n */\ntemplate<typename SrcExp, typename Device,\n         typename DType, int srcdim>\ninline FlipExp<SrcExp, Device, DType, srcdim>\nflip(const TRValue<SrcExp, Device, srcdim, DType> &src, int dim) {\n  return FlipExp<SrcExp, Device, DType, srcdim>(src.self(), dim);\n}\n//------------------------\n//  engine plugin\n//------------------------\n// runtime shapecheck\ntemplate<typename SrcExp, typename Device,\n         typename DType, int srcdim>\nstruct ShapeCheck<srcdim, FlipExp<SrcExp, Device, DType, srcdim> >{\n  inline static Shape<srcdim> Check(const FlipExp<SrcExp,\n                                    Device, DType, srcdim> &t) {\n    return t.shape_;\n  }\n};\ntemplate<typename SrcExp, typename Device,\n         typename DType, int srcdim>\nstruct StreamInfo<Device, FlipExp<SrcExp, Device, DType, srcdim> >{\n  inline static Stream<Device> *\n  Get(const FlipExp<SrcExp, Device, DType, srcdim> &t) {\n    return StreamInfo<Device, SrcExp>::Get(t.src_);\n  }\n};\n// static typecheck\ntemplate<typename SrcExp, typename Device,\n         typename DType, int srcdim>\nstruct ExpInfo<FlipExp<SrcExp, Device, DType, srcdim> >{\n  static const int kDim = ExpInfo<SrcExp>::kDim;\n  static const int kDevMask = ExpInfo<SrcExp>::kDevMask;\n};\n//----------------------\n// Execution plan\n//---------------------\ntemplate<typename SrcExp, typename Device,\n         typename DType, int srcdim>\nstruct Plan<FlipExp<SrcExp, Device, DType, srcdim>, DType> {\n public:\n  explicit Plan(const FlipExp<SrcExp, Device, DType, srcdim> &e)\n      : src_(MakePlan(e.src_)), stride_j_(e.stride_j_),\n        trailing_(e.trailing_), stride_(e.stride_) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    index_t idx = i*stride_j_+j;\n    const index_t low = idx%trailing_;\n    index_t high = idx/trailing_;\n    const index_t x = high%stride_;\n    high /= stride_;\n    idx = (high*stride_+stride_-1-x)*trailing_+low;\n    return src_.Eval(idx/stride_j_, idx%stride_j_);\n  }\n  MSHADOW_XINLINE DType &REval(index_t i, index_t j) const {\n    index_t idx = i*stride_j_+j;\n    const index_t low = idx%trailing_;\n    index_t high = idx/trailing_;\n    const index_t x = high%stride_;\n    high /= stride_;\n    idx = (high*stride_+stride_-1-x)*trailing_+low;\n    return src_.REval(idx/stride_j_, idx%stride_j_);\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t stride_j_, trailing_, stride_;\n};  // struct Plan\n}  // namespace expr\n}   // namespace mshadow\n#endif  // MSHADOW_EXTENSION_FLIP_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/implicit_gemm.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file implicit_gemm.h\n * \\brief support for implicit GEMM operation\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_EXTENSION_IMPLICIT_GEMM_H_\n#define MSHADOW_EXTENSION_IMPLICIT_GEMM_H_\n\n#include \"../extension.h\"\n#include \"../packet-inl.h\"\n\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief Matrix multiplication.\n * \\tparam LhsExp type of lhs expression\n * \\tparam LhsExp type of rhs expression\n * \\tparam DType the type of elements\n */\ntemplate<typename LhsExp, typename RhsExp, typename DType>\nstruct ImplicitGEMMExp:\n      public Exp<ImplicitGEMMExp<LhsExp, RhsExp, DType>,\n                 DType, type::kChainer> {\n  /*! \\brief lhs operand */\n  const LhsExp &lhs_;\n  /*! \\brief rhs operand */\n  const RhsExp &rhs_;\n  /*! \\brief internal production size*/\n  index_t prod_size_;\n  /*! \\brief the shape of this expression */\n  Shape<2> shape_;\n  /*! \\brief constructor */\n  ImplicitGEMMExp(const LhsExp &lhs, const RhsExp &rhs)\n      : lhs_(lhs), rhs_(rhs) {\n    Shape<2> slhs = ShapeCheck<2, LhsExp>::Check(lhs_);\n    Shape<2> srhs = ShapeCheck<2, RhsExp>::Check(rhs_);\n    this->shape_ = mshadow::Shape2(slhs[0], srhs[1]);\n    prod_size_ = slhs[1];\n  }\n};\n\n\ntemplate<typename LhsExp, typename RhsExp, typename DType, int e1, int e2>\ninline ImplicitGEMMExp<LhsExp, RhsExp, DType>\nimplicit_dot(const Exp<LhsExp, DType, e1> &lhs,\n             const Exp<RhsExp, DType, e2> &rhs) {\n  TypeCheckPass<ExpInfo<LhsExp>::kDim == 2 && ExpInfo<RhsExp>::kDim == 2>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return ImplicitGEMMExp<LhsExp, RhsExp, DType>(lhs.self(), rhs.self());\n}\n\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename LhsExp, typename RhsExp, typename DType>\nstruct Plan<ImplicitGEMMExp<LhsExp, RhsExp, DType>, DType> {\n public:\n  explicit Plan(const ImplicitGEMMExp<LhsExp, RhsExp, DType> &e)\n      : lhs_(MakePlan(e.lhs_)),\n        rhs_(MakePlan(e.rhs_)),\n        prod_size_(e.prod_size_),\n        prod_size_lower_align_(packet::LowerAlign<DType, MSHADOW_DEFAULT_PACKET>(e.prod_size_)) {\n  }\n\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    typedef packet::Packet<DType> Packet;\n    Packet sum = Packet::Fill(0);\n\n    const size_t packetSize = Packet::size;\n    DType lhs_temp[packetSize], rhs_temp[packetSize];\n\n    for (index_t i = 0; i < prod_size_lower_align_; i += packetSize) {\n      // unroll\n      for (index_t j = 0; j < packetSize; ++j) {\n        lhs_temp[j] = lhs_.Eval(y, i + j);\n      }\n      for (index_t j = 0; j < packetSize; ++j) {\n        rhs_temp[j] = rhs_.Eval(i + j, x);\n      }\n      sum = sum + Packet::LoadUnAligned(lhs_temp) * Packet::LoadUnAligned(rhs_temp);\n    }\n    DType ret_result = sum.Sum();\n\n    for (index_t i =  prod_size_lower_align_; i < prod_size_; ++i) {\n      ret_result += lhs_.Eval(y, i) * rhs_.Eval(i, x);\n    }\n    return ret_result;\n  }\n\n private:\n  expr::Plan<LhsExp, DType> lhs_;\n  expr::Plan<RhsExp, DType> rhs_;\n  const index_t prod_size_;\n  const index_t prod_size_lower_align_;\n};\n\ntemplate<typename LhsExp, typename RhsExp, typename DType>\ninline Plan<ImplicitGEMMExp<LhsExp, RhsExp, DType>, DType>\nMakePlan(const ImplicitGEMMExp<LhsExp, RhsExp, DType> &exp) {\n  return Plan<ImplicitGEMMExp<LhsExp, RhsExp, DType>, DType>(exp);\n}\n\n\ntemplate<int dim, typename LhsExp, typename RhsExp, typename DType>\nstruct ShapeCheck<dim, ImplicitGEMMExp<LhsExp, RhsExp, DType> > {\n  inline static Shape<dim>\n  Check(const ImplicitGEMMExp<LhsExp, RhsExp, DType> &t) {\n    CHECK(dim == 2)\n        << \"ImplicitGEMMExp only support 2 dimension\";\n    Shape<dim> shape1 = ShapeCheck<dim, LhsExp>::Check(t.lhs_);\n    Shape<dim> shape2 = ShapeCheck<dim, RhsExp>::Check(t.rhs_);\n    CHECK_EQ(shape1[1], shape2[0])\n      << \"implicit_dot The matrix shape do  not match\";\n    return t.shape_;\n  }\n};\n\ntemplate<typename LhsExp, typename RhsExp, typename DType>\nstruct ExpInfo<ImplicitGEMMExp<LhsExp, RhsExp, DType> > {\n  static const int kDim = 2;\n  static const int kDevMask = ExpInfo<LhsExp>::kDevMask & ExpInfo<RhsExp>::kDevMask;\n};\n\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_IMPLICIT_GEMM_H_\n\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/mask.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file mask.h\n * \\brief\n * \\author Bing Xu\n*/\n#ifndef MSHADOW_EXTENSION_MASK_H_\n#define MSHADOW_EXTENSION_MASK_H_\n\n#include \"../extension.h\"\n\nnamespace mshadow {\nnamespace expr {\n\n/*! \\brief Broadcast a mask and do element-wise multiplication\n *  \\tparam IndexExp type of index expression\n *  \\tparam SrcExp type of src expression\n *  \\tparam DType data type\n */\ntemplate<typename IndexExp, typename SrcExp, typename DType>\nstruct MaskExp: public Exp<MaskExp<IndexExp, SrcExp, DType>,\n                           DType, type::kChainer> {\n  /*! \\brief index oprand */\n  const IndexExp &index_;\n  /*! \\brief matrix oprand */\n  const SrcExp &src_;\n  /*! constructor */\n  MaskExp(const IndexExp &index, const SrcExp &src)\n    : index_(index), src_(src) {}\n};  // struct MaskExp\n\n\n\ntemplate<typename IndexExp,\n         typename SrcExp,\n         typename DType,\n         int e1, int e2>\ninline MaskExp<IndexExp, SrcExp, DType>\nmask(const Exp<IndexExp, DType, e1> &index,\n     const Exp<SrcExp, DType, e2> &src) {\n  return MaskExp<IndexExp, SrcExp, DType>(index.self(), src.self());\n}\n\n\n//----------------------\n// Execution plan\n//----------------------\n\ntemplate<typename IndexExp, typename SrcExp, typename DType>\nstruct Plan<MaskExp<IndexExp, SrcExp, DType>, DType> {\n public:\n  explicit Plan(const MaskExp<IndexExp, SrcExp, DType> &e)\n    : index_(MakePlan(e.index_)), src_(MakePlan(e.src_)) {\n  }\n\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    return static_cast<DType>(src_.Eval(y, x) * index_.Eval(0, y));\n  }\n\n private:\n  expr::Plan<IndexExp, DType> index_;\n  expr::Plan<SrcExp, DType> src_;\n};  // struct Plan\n\ntemplate<typename IndexExp, typename SrcExp, typename DType>\ninline Plan<MaskExp<IndexExp, SrcExp, DType>, DType>\nMakePlan(const MaskExp<IndexExp, SrcExp, DType> &exp) {\n  return Plan<MaskExp<IndexExp, SrcExp, DType>, DType>(exp);\n}\n\ntemplate<int dim, typename IndexExp, typename SrcExp, typename DType>\nstruct ShapeCheck<dim, MaskExp<IndexExp, SrcExp, DType> > {\n  inline static Shape<dim>\n  Check(const MaskExp<IndexExp, SrcExp, DType> &t) {\n    CHECK(dim == 2)\n      << \"MaskExp only support 2D output\";\n    Shape<1> dshape = ShapeCheck<1, IndexExp>::Check(t.index_);\n    Shape<2> wshape = ShapeCheck<2, SrcExp>::Check(t.src_);\n    CHECK_EQ(dshape[0], wshape[0]) << \"MaskExp require inputs in same first dimention\";\n    Shape<dim> ret;\n    ret[0] = wshape[0];\n    ret[1] = wshape[1];\n    return ret;\n  }\n};\n\n\ntemplate<typename IndexExp, typename SrcExp, typename DType>\nstruct ExpInfo<MaskExp<IndexExp, SrcExp, DType> > {\n  static const int kDim = 2;\n  static const int kDevMask = ExpInfo<IndexExp>::kDevMask;\n};\n\n}  // namespace expr\n}  // namespace mshadow\n\n#endif  // MSHADOW_EXTENSION_MASK_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/mirror.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file mirror.h\n * \\brief support for mirror\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_EXTENSION_MIRROR_H_\n#define MSHADOW_EXTENSION_MIRROR_H_\n#include \"../extension.h\"\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief mirror expression, mirror a image in width\n * \\tparam SrcExp source expression to be mirrored\n * \\tparam DType the type of elements\n * \\tparam srcdim dimension of src\n */\ntemplate<typename SrcExp, typename DType, int srcdim>\nstruct MirroringExp:\n      public MakeTensorExp<MirroringExp<SrcExp, DType, srcdim>,\n                           SrcExp, srcdim, DType> {\n  /*! \\brief source operand */\n  const SrcExp &src_;\n  /*! \\brief constructor */\n  explicit MirroringExp(const SrcExp &src) : src_(src) {\n    this->shape_ = ShapeCheck<srcdim, SrcExp>::Check(src_);\n  }\n};\n/*!\n * \\brief mirroring expression, mirror images in width\n * \\param src original image batches\n * \\return expression corresponding to mirrored result\n * \\tparam SrcExp source expression\n * \\tparam DType the type of elements\n * \\tparam etype type of expression\n */\ntemplate<typename SrcExp, typename DType, int etype>\ninline MirroringExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>\nmirror(const Exp<SrcExp, DType, etype> &src) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim >= 2>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return MirroringExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>(src.self());\n}\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename SrcExp, typename DType, int srcdim>\nstruct Plan<MirroringExp<SrcExp, DType, srcdim>, DType> {\n public:\n  explicit Plan(const MirroringExp<SrcExp, DType, srcdim> &e)\n      : src_(MakePlan(e.src_)), width_(e.shape_[srcdim - 1]) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    return src_.Eval(i, width_ - j - 1);\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t width_;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_MIRROR_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/one_hot.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file one_hot.h\n * \\brief Create one-hot indicator array based on the index.\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_EXTENSION_ONE_HOT_H_\n#define MSHADOW_EXTENSION_ONE_HOT_H_\n\n#include \"../extension.h\"\n\n\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief Create a one-hot indicator array.\n * \\tparam IndexExp type of index expression\n * \\tparam DType the type of elements\n */\ntemplate<typename IndexExp, typename DType>\nstruct OneHotEncodeExp:\n      public Exp<OneHotEncodeExp<IndexExp, DType>,\n                 DType, type::kChainer> {\n  /*! \\brief index operand */\n  const IndexExp &index_;\n  /*! \\brief number of choices we can have. */\n  index_t num_choices_;\n  /*! \\brief constructor */\n  OneHotEncodeExp(const IndexExp &index, index_t num_choices)\n      : index_(index), num_choices_(num_choices) {}\n};\n\ntemplate<typename IndexExp,\n         typename IDType, int e1>\ninline OneHotEncodeExp<IndexExp, default_real_t>\none_hot_encode(const Exp<IndexExp, IDType, e1> &index, index_t num_choices) {\n  TypeCheckPass<ExpInfo<IndexExp>::kDim == 1>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return OneHotEncodeExp<IndexExp, default_real_t>(index.self(), num_choices);\n}\n\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename IndexExp, typename DType>\nstruct Plan<OneHotEncodeExp<IndexExp, DType>, DType> {\n public:\n  explicit Plan(const OneHotEncodeExp<IndexExp, DType> &e)\n      : index_(MakePlan(e.index_)) {\n  }\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    index_t idx = static_cast<index_t>(index_.Eval(0, y));\n    return static_cast<DType>(x == idx);\n  }\n\n private:\n  expr::Plan<IndexExp, DType> index_;\n};\n\ntemplate<typename IndexExp, typename DType>\ninline Plan<OneHotEncodeExp<IndexExp, DType>, DType>\nMakePlan(const OneHotEncodeExp<IndexExp, DType> &exp) {\n  return Plan<OneHotEncodeExp<IndexExp, DType>, DType>(exp);\n}\n\ntemplate<int dim, typename IndexExp, typename DType>\nstruct ShapeCheck<dim, OneHotEncodeExp<IndexExp, DType> > {\n  inline static Shape<dim>\n  Check(const OneHotEncodeExp<IndexExp, DType> &t) {\n    CHECK(dim == 2)\n        << \"OneHotEncodeExp only support 2 dimension output\";\n    Shape<1> shape = ShapeCheck<1, IndexExp>::Check(t.index_);\n    Shape<dim> ret;\n    ret[0] = shape[0];\n    ret[1] = t.num_choices_;\n    return ret;\n  }\n};\n\ntemplate<typename IndexExp, typename DType>\nstruct ExpInfo<OneHotEncodeExp<IndexExp, DType> > {\n  static const int kDim = 2;\n  static const int kDevMask = ExpInfo<IndexExp>::kDevMask;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_ONE_HOT_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/pack_col2patch.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file pack_col2patch.h\n * \\brief support for pack\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_EXTENSION_PACK_COL2PATCH_H_\n#define MSHADOW_EXTENSION_PACK_COL2PATCH_H_\n#include <algorithm>\n#include \"../extension.h\"\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief reverse operation of UnpackPatchToCol,\n *    used to backprop gradient back\n *    this is a version supporting multiple images\n * \\tparam SrcExp source expression\n * \\tparam DType the type of elements\n * \\tparam dstdim destination dimension\n */\ntemplate<typename SrcExp, typename DType, int dstdim>\nstruct PackColToPatchXExp:\n      public MakeTensorExp<PackColToPatchXExp<SrcExp, DType, dstdim>,\n                           SrcExp, dstdim, DType> {\n  /*! \\brief source operand */\n  const SrcExp &src_;\n  /*! \\brief patch height */\n  index_t psize_y_;\n  /*! \\brief patch height */\n  index_t psize_x_;\n  /*! \\brief patch stride */\n  index_t pstride_y_;\n  index_t pstride_x_;\n  /*! \\brief patch dilate */\n  index_t pdilate_y_;\n  index_t pdilate_x_;\n  /*! \\brief constructor */\n  PackColToPatchXExp(const SrcExp &src, Shape<dstdim> imshape,\n                     index_t psize_y, index_t psize_x,\n                     index_t pstride_y, index_t pstride_x,\n                     index_t pdilate_y, index_t pdilate_x)\n      :src_(src), psize_y_(psize_y), psize_x_(psize_x),\n       pstride_y_(pstride_y), pstride_x_(pstride_x),\n       pdilate_y_(pdilate_y), pdilate_x_(pdilate_x){\n    this->shape_ = imshape;\n    const index_t o_height = (imshape[dstdim - 2] -\n        (pdilate_y * (psize_y - 1)+ 1))/pstride_y + 1;\n    const index_t o_width  = (imshape[dstdim - 1] -\n        (pdilate_x * (psize_x - 1) + 1)) / pstride_x + 1;\n    Shape<2> sshape = ShapeCheck<2, SrcExp>::Check(src_);\n    CHECK_EQ(sshape[1], o_height * o_width * imshape.ProdShape(0, dstdim - 3))\n      << \"PackColToPatchExp: src.size(1) mismatch\";\n    CHECK_EQ(sshape[0], psize_y * psize_x * imshape[dstdim - 3])\n      << \"PackColToPatchExp: src.size(0) mismatch\";\n  }\n};\n/*!\n * \\brief reverse operation of pack_col2patch, can be used to implement deconvolution\n * \\return packed img expression\n * \\param mat source matrix\n * \\param imshape shape of target img\n * \\param psize_y height of each patch\n * \\param psize_x height of each patch\n * \\param pstride stride of each patch\n * \\tparam SrcExp source expression\n * \\tparam DType the type of elements\n * \\tparam dstdim destination dimension\n * \\tparam etype type of expression\n */\ntemplate<typename SrcExp, typename DType, int dstdim, int etype>\ninline PackColToPatchXExp<SrcExp, DType, dstdim>\npack_col2patch(const expr::Exp<SrcExp, DType, etype> &src,\n               Shape<dstdim> imshape, index_t psize_y,\n               index_t psize_x, index_t pstride, index_t pdilate) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim == 2>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  CHECK(imshape[dstdim - 1] >= psize_x && imshape[dstdim - 2] >= psize_y)\n    << \"PackColToPatch:image shape smaller than patch size\";\n  return PackColToPatchXExp<SrcExp, DType, dstdim>(src.self(), imshape,\n                                                   psize_y, psize_x, pstride, pstride,\n                                                   pdilate, pdilate);\n}\n/*!\n *if you want to specify kstride_y and kstride_x\n */\ntemplate<typename SrcExp, typename DType, int dstdim, int etype>\ninline PackColToPatchXExp<SrcExp, DType, dstdim>\npack_col2patch(const expr::Exp<SrcExp, DType, etype> &src,\n               Shape<dstdim> imshape, index_t psize_y,\n               index_t psize_x, index_t pstride_y, index_t pstride_x,\n               index_t pdilate_y, index_t pdilate_x) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim == 2>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  CHECK(imshape[dstdim - 1] >= psize_x && imshape[dstdim - 2] >= psize_y)\n    << \"PackColToPatch:image shape smaller than patch size\";\n  return PackColToPatchXExp<SrcExp, DType, dstdim>(src.self(), imshape,\n                                                   psize_y, psize_x, pstride_y, pstride_x,\n                                                   pdilate_y, pdilate_x);\n}\n\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename SrcExp, typename DType, int dstdim>\nstruct Plan<PackColToPatchXExp<SrcExp, DType, dstdim>, DType> {\n public:\n  explicit Plan(const PackColToPatchXExp<SrcExp, DType, dstdim> &e)\n      :src_(MakePlan(e.src_)), psize_y_(e.psize_y_),\n       psize_x_(e.psize_x_), pstride_y_(e.pstride_y_), pstride_x_(e.pstride_x_),\n       i_channel_(e.shape_[dstdim - 3]), pdilate_y_(e.pdilate_y_), pdilate_x_(e.pdilate_x_),\n       i_height_(e.shape_[dstdim - 2]),\n       o_height_((e.shape_[dstdim - 2] - (pdilate_y_ * (psize_y_ - 1) + 1)) /\n               pstride_y_ + 1),\n       o_width_((e.shape_[dstdim - 1] - (pdilate_x_ * (psize_x_ - 1) + 1)) /\n               pstride_x_ + 1) {\n    // note: i/o convention are same as unpack\n  }\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    using namespace std;\n    const index_t y = i % i_height_;\n    const index_t idivh = i / i_height_;\n    const index_t c = idivh % i_channel_;\n    const index_t n = idivh / i_channel_;\n    const index_t x = j;\n\n    const index_t psize_y_dilate = (pdilate_y_ * (psize_y_ - 1) + 1);\n    const index_t psize_x_dilate = (pdilate_x_ * (psize_x_ - 1) + 1);\n\n    const index_t py_min =\n        y < psize_y_dilate ? y % pdilate_y_ : (y-psize_y_dilate + pstride_y_) / pstride_y_;\n    const index_t px_min =\n        x < psize_x_dilate ? x % pdilate_x_ : (x-psize_x_dilate + pstride_x_) / pstride_x_;\n    const index_t py_max = min((y + pstride_y_) / pstride_y_, o_height_);\n    const index_t px_max = min((x + pstride_x_) / pstride_x_, o_width_);\n    DType res = static_cast<DType>(0);\n    for (index_t py = py_min; py < py_max; py += pdilate_y_) {\n      for (index_t px = px_min; px < px_max; px += pdilate_x_) {\n        res += src_.Eval(((c * psize_y_ + (y - py*pstride_y_) / pdilate_y_) * psize_x_ +\n                         (x - px * pstride_x_) / pdilate_x_),\n                         (n * o_height_ + py) * o_width_ + px);\n      }\n    }\n    return res;\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t psize_y_, psize_x_, pstride_y_, pstride_x_, i_channel_;\n  const index_t pdilate_y_, pdilate_x_;\n  const index_t i_height_, o_height_, o_width_;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_PACK_COL2PATCH_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/pad.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file pad.h\n * \\brief support for pad\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_EXTENSION_PAD_H_\n#define MSHADOW_EXTENSION_PAD_H_\n#include \"../extension.h\"\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief padding expression, pad a image with zeros\n * \\tparam SrcExp source expression\n * \\tparam DType the type of elements\n * \\tparam srcdim dimension of src\n */\ntemplate<typename SrcExp, typename DType, int srcdim>\nstruct PaddingExp:\n      public MakeTensorExp<PaddingExp<SrcExp, DType, srcdim>,\n                           SrcExp, srcdim, DType> {\n  /*! \\brief source operand */\n  const SrcExp &src_;\n  /*! \\brief pad size in y */\n  index_t pad_y_;\n  /*! \\brief pad size in x */\n  index_t pad_x_;\n  /*! \\brief source tensor height */\n  index_t src_height_;\n  /*! \\brief source tensor width */\n  index_t src_width_;\n  /*! \\brief constructor */\n  PaddingExp(const SrcExp &src, index_t pad_y, index_t pad_x)\n      : src_(src), pad_y_(pad_y), pad_x_(pad_x) {\n    this->shape_ = ShapeCheck<srcdim, SrcExp>::Check(src_);\n    src_height_ = this->shape_[srcdim - 2];\n    src_width_  = this->shape_[srcdim - 1];\n    this->shape_[srcdim - 2] += pad_y * 2;  // height\n    this->shape_[srcdim - 1] += pad_x * 2;  // width\n  }\n};\n/*!\n * \\brief padding expression, pad a image with zeros on boundaries, padding affects shape[0], and shape[1]\n * \\param src original image batches\n * \\param pad padding size\n * \\return expression corresponding to padded result\n * \\tparam SrcExp source expression\n * \\tparam DType the content data type\n * \\tparam etype type of expression\n */\ntemplate<typename SrcExp, typename DType, int etype>\ninline PaddingExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>\npad(const Exp<SrcExp, DType, etype> &src, index_t pad) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim >= 2>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return PaddingExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>(src.self(), pad, pad);\n}\n/*!\n * \\brief padding expression, pad a image with zeros on boundaries, padding affects shape[0], and shape[1]\n * \\param src original image batches\n * \\param pad_y padding size in y\n * \\param pad_x padding size in x\n * \\return expression corresponding to padded result\n * \\tparam SrcExp source expression\n * \\tparam DType the content data type\n * \\tparam etype type of expression\n */\ntemplate<typename SrcExp, typename DType, int etype>\ninline PaddingExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>\npad(const Exp<SrcExp, DType, etype> &src, index_t pad_y, index_t pad_x) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim >= 2>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return PaddingExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>\n      (src.self(), pad_y, pad_x);\n}\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename SrcExp, typename DType, int srcdim>\nstruct Plan<PaddingExp<SrcExp, DType, srcdim>, DType> {\n public:\n  explicit Plan(const PaddingExp<SrcExp, DType, srcdim> &e)\n      : src_(MakePlan(e.src_)),\n        pad_y_(e.pad_y_), pad_x_(e.pad_x_),\n        new_height_(e.shape_[srcdim - 2]),\n        src_height_(e.src_height_), src_width_(e.src_width_) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    const index_t x = j;\n    const index_t y = i % new_height_;\n    const index_t c = i / new_height_;\n    if (y < pad_y_ || x < pad_x_) return static_cast<DType>(0);\n    const index_t h = y - pad_y_;\n    const index_t w = x - pad_x_;\n    if (h < src_height_ && w < src_width_) {\n      return src_.Eval(c * src_height_ + h, w);\n    } else {\n      return static_cast<DType>(0);\n    }\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t pad_y_;\n  const index_t pad_x_;\n  const index_t new_height_;\n  const index_t src_height_;\n  const index_t src_width_;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_PAD_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/range.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file range.h\n * \\brief support generating a range vector\n * \\author Xingjian Shi\n */\n#ifndef MSHADOW_EXTENSION_RANGE_H_\n#define MSHADOW_EXTENSION_RANGE_H_\n\n#include \"../extension.h\"\n\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief Generate a range vector similar to python: range(start, stop[, step][, repeat]).\n          If step is positive, the last element is the largest start + i * step less than stop\n          If step is negative, the last element is the smallest start + i * step greater than stop.\n          All elements are repeated for `repeat` times, e.g range(0, 4, 2, 3) --> 0, 0, 0, 2, 2, 2\n * \\tparam SrcExp type of lhs expression\n * \\tparam IndexExp type of index expression\n * \\tparam DType the type of elements\n */\ntemplate<typename DType>\nstruct RangeExp:\n      public Exp<RangeExp<DType>, DType, type::kMapper> {\n  const DType start_;\n  const DType stop_;\n  const DType step_;\n  const int repeat_;\n  /*! \\brief constructor */\n  RangeExp(DType start, DType stop, DType step, int repeat)\n      : start_(start), stop_(stop), step_(step), repeat_(repeat) {}\n};\n\ntemplate<typename DType>\ninline RangeExp<DType>\nrange(DType start, DType stop, DType step = 1, int repeat = 1) {\n  return RangeExp<DType>(start, stop, step, repeat);\n}\n\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename DType>\nstruct Plan<RangeExp<DType>, DType> {\n public:\n  explicit Plan(const RangeExp<DType> &e)\n      : start_(e.start_),\n        stop_(e.stop_),\n        step_(e.step_),\n        repeat_(e.repeat_) {\n  }\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    return start_ + static_cast<DType>((static_cast<int>(x) / repeat_)) * step_;\n  }\n\n private:\n  const DType start_;\n  const DType stop_;\n  const DType step_;\n  const int repeat_;\n};\n\ntemplate<typename DType>\ninline Plan<RangeExp<DType>, DType>\nMakePlan(const RangeExp<DType> &exp) {\n  return Plan<RangeExp<DType>, DType>(exp);\n}\n\n\ntemplate<typename DType>\ninline int RangeOutSize(DType start, DType stop, DType step, int repeat) {\n  return repeat * ((stop - start - 1) / step + 1);\n}\n\ntemplate<>\ninline int RangeOutSize<float>(float start, float stop, float step, int repeat) {\n  double d_start = static_cast<double>(start);\n  double d_stop = static_cast<double>(stop);\n  double d_step = static_cast<double>(step);\n  return repeat * static_cast<int>(ceil((d_stop - d_start) / d_step));\n}\n\ntemplate<>\ninline int RangeOutSize<double>(double start, double stop, double step, int repeat) {\n  return repeat * static_cast<int>(ceil((stop - start) / step));\n}\n\n\ntemplate<int dim, typename DType>\nstruct ShapeCheck<dim, RangeExp<DType> > {\n  inline static Shape<dim>\n  Check(const RangeExp<DType> &t) {\n    CHECK(dim == 1)\n        << \"RangeExp only support 1 dimension output, received \" << dim;\n    CHECK(t.step_ != 0)\n        << \"RangeExp does not support step=0, received \" << t.step_;\n    CHECK(t.repeat_ > 0)\n      << \"RangeExp only supports repeat > 0, received \" << t.repeat_;\n    if (t.step_ > 0) {\n      CHECK(t.start_ < t.stop_) << \"RangeExp does not support (start, stop, step) = \"\n                                << \"(\" << t.start_ << \",\" << t.stop_ << \",\" << t.step_ << \")\";\n    } else {\n      CHECK(t.start_ > t.stop_) << \"RangeExp does not support (start, stop, step)= \"\n                                << \"(\" << t.start_ << \",\" << t.stop_ << \",\" << t.step_ << \")\";\n    }\n    return Shape1(RangeOutSize<DType>(t.start_, t.stop_, t.step_, t.repeat_));\n  }\n};\n\ntemplate<typename DType>\nstruct ExpInfo<RangeExp<DType> > {\n  static const int kDim = 1;\n  static const int kDevMask = 0xffff;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_RANGE_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/reduce_with_axis.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file reduce_with_axis.h\n * \\brief\n * \\author Junyuan Xie\n*/\n#ifndef MSHADOW_EXTENSION_REDUCE_WITH_AXIS_H_\n#define MSHADOW_EXTENSION_REDUCE_WITH_AXIS_H_\n\n#include \"../extension.h\"\n\nnamespace mshadow {\nnamespace expr {\n\n/*! \\brief reduce out the dimension of src labeled by axis.\n *  \\tparam Reducer type of reducer\n *  \\tparam SrcExp type of source expression\n *  \\tparam DType data type\n */\ntemplate<typename Reducer, typename SrcExp, typename DType, int dimsrc, bool mask, int dimdst>\nstruct ReduceWithAxisExp:\n    public MakeTensorExp<ReduceWithAxisExp<Reducer, SrcExp, DType, dimsrc, mask, dimdst>,\n                         SrcExp, dimdst, DType> {\n  /*! \\brief source oprand */\n  const SrcExp &src_;\n  /*! \\brief size of last destination dimension */\n  index_t last_dst_dim_;\n  /*! \\brief size of trailing dimensions */\n  index_t trailing_;\n  /*! \\brief size of axis dimension */\n  index_t size_;\n  /*! \\brief size of last src dimension */\n  index_t last_;\n  /*! constructor */\n  explicit ReduceWithAxisExp(const SrcExp &src, int axis)\n    : src_(src) {\n    bool keepdim = (dimsrc == dimdst);\n    CHECK(dimsrc > axis) << \"reduce axis out of bound\";\n    Shape<dimsrc> src_shape = ShapeCheck<dimsrc, SrcExp>::Check(src_);\n    for (int i = 0; i < axis; ++i) {\n      this->shape_[i] = src_shape[i];\n    }\n    this->size_ = src_shape[axis];\n    this->trailing_ = 1;\n    if (!keepdim) {\n      for (int i = axis + 1; i < dimsrc; ++i) {\n        this->trailing_ *= src_shape[i];\n        this->shape_[i - 1] = src_shape[i];\n      }\n    } else {\n      this->shape_[axis] = 1;\n      for (index_t i = axis + 1; i < dimsrc; ++i) {\n        this->trailing_ *= src_shape[i];\n        this->shape_[i] = src_shape[i];\n      }\n    }\n\n    this->last_ = src_shape[dimsrc - 1];\n    this->last_dst_dim_ = this->shape_[dimdst - 1];\n  }\n};  // struct ReduceWithAxisExp\n\n/*!\n * \\brief reduce out the dimension of src labeled by axis.\n * \\param Reducer type of the reducing operation\n * \\param mask whether to output the unmask indices\n * \\tparam SrcExp source expression\n * \\tparam DType data type\n * \\tparam etype type of the expression\n */\ntemplate<typename Reducer, bool mask, typename SrcExp, typename DType, int etype>\ninline ReduceWithAxisExp<Reducer, SrcExp, DType, ExpInfo<SrcExp>::kDim, mask,\n  ExpInfo<SrcExp>::kDim - 1>\nreduce_with_axis(const Exp<SrcExp, DType, etype> &src, int axis) {\n  return ReduceWithAxisExp<Reducer, SrcExp, DType, ExpInfo<SrcExp>::kDim, mask,\n    ExpInfo<SrcExp>::kDim- 1>(src.self(), axis);\n}\n\n/*!\n* \\brief reduce out the dimension of src labeled by axis, keepdim turned on.\n* \\param Reducer type of the reducing operation\n* \\param mask whether to output the unmask indices\n* \\tparam SrcExp source expression\n* \\tparam DType data type\n* \\tparam etype type of the expression\n*/\ntemplate<typename Reducer, bool mask, typename SrcExp, typename DType, int etype>\ninline ReduceWithAxisExp<Reducer, SrcExp, DType, ExpInfo<SrcExp>::kDim, mask,\n  ExpInfo<SrcExp>::kDim>\n  reduce_keepdim(const Exp<SrcExp, DType, etype> &src, int axis) {\n  return ReduceWithAxisExp<Reducer, SrcExp, DType, ExpInfo<SrcExp>::kDim, mask,\n    ExpInfo<SrcExp>::kDim>(src.self(), axis);\n}\n\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename Reducer, typename SrcExp, typename DType, int dimsrc, bool mask, int dimdst>\nstruct Plan<ReduceWithAxisExp<Reducer, SrcExp, DType, dimsrc, mask, dimdst>, DType> {\n public:\n  explicit Plan(const ReduceWithAxisExp<Reducer, SrcExp, DType, dimsrc, mask, dimdst> &e)\n      : src_(MakePlan(e.src_)), last_dst_dim_(e.last_dst_dim_), trailing_(e.trailing_),\n        size_(e.size_), last_(e.last_) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    index_t x = (i*last_dst_dim_ + j)/trailing_;\n    index_t y = (i*last_dst_dim_ + j)%trailing_;\n\n    if (mask) {\n      index_t idx = 0;\n      DType res; Reducer::SetInitValue(res);\n      for (index_t k = 0; k < size_; ++k) {\n        index_t z = (x*size_+k)*trailing_+y;\n        DType tmp = res;\n        Reducer::Reduce(res, src_.Eval(z/last_, z%last_));\n        if (tmp != res && !isnan_typed::IsNan(tmp)) {\n          idx = k;\n        }\n      }\n      return static_cast<DType>(static_cast<int>(idx));\n    } else {\n      DType res; Reducer::SetInitValue(res);\n      for (index_t k = 0; k < size_; ++k) {\n        index_t z = (x*size_+k)*trailing_+y;\n        Reducer::Reduce(res, src_.Eval(z/last_, z%last_));\n      }\n      return res;\n    }\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t last_dst_dim_, trailing_, size_, last_;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_REDUCE_WITH_AXIS_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/reduceto1d.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file reduceto1d.h\n * \\brief support for sum_rows and sumall_except_dim\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_EXTENSION_REDUCETO1D_H_\n#define MSHADOW_EXTENSION_REDUCETO1D_H_\n#include \"../extension.h\"\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief reduction to 1 dimension tensor\n * input: Tensor<Device,k>: ishape\n * output: Tensor<Device,1> shape[0] = ishape[dimkeep];\n *\n * \\tparam SrcExp type of expression to be reduced\n * \\tparam DType the data type of the scalar\n * \\tparam Reducer which reducer to use\n * \\tparam m_dimkeep which dimension to be kept, encoded with dimsrc - dimkeep\n */\ntemplate<typename SrcExp, typename DType, typename Reducer, int m_dimkeep>\nstruct ReduceTo1DExp:\n      public Exp<ReduceTo1DExp<SrcExp, DType, Reducer, m_dimkeep>,\n                 DType, type::kComplex> {\n  /*! \\brief source operand */\n  const SrcExp &src_;\n  /*! \\brief source operand, scale of the  */\n  DType scale_;\n  /*! \\brief construct a repmat expression from src and nrow */\n  ReduceTo1DExp(const SrcExp& src, DType scale) : src_(src), scale_(scale) {}\n};\n/*!\n * \\brief a sum over all dimensions, except dimkeep\n * \\param exp input expression that must be a matrix Tensor<?,2>\n * \\return a expresion with type Tensor<Device,1>\n * \\tparam dimkeep the dimension that will be kept\n * \\tparam SrcExp expression\n * \\tparam etype type of expression\n */\ntemplate<int dimkeep,  typename SrcExp, typename DType, int etype>\ninline ReduceTo1DExp<SrcExp, DType, red::sum,\n                     ExpInfo<SrcExp>::kDim - dimkeep>\nsumall_except_dim(const Exp<SrcExp, DType, etype> &exp) {\n  return ReduceTo1DExp<SrcExp, DType, red::sum,\n                       ExpInfo<SrcExp>::kDim - dimkeep>(exp.self(), DType(1));\n}\n/*!\n * \\brief reduce over all dimensions, except dimkeep\n * \\param exp input expression that must be a matrix Tensor<?,2>\n * \\return a expresion with type Tensor<Device,1>\n * \\tparam dimkeep the dimension that will be kept\n * \\tparam SrcExp expression\n * \\tparam etype type of expression\n */\ntemplate<int dimkeep, typename Reducer, typename SrcExp, typename DType, int etype>\ninline ReduceTo1DExp<SrcExp, DType, Reducer,\n                     ExpInfo<SrcExp>::kDim - dimkeep>\nreduce_except_dim(const Exp<SrcExp, DType, etype> &exp) {\n  return ReduceTo1DExp<SrcExp, DType, Reducer,\n                       ExpInfo<SrcExp>::kDim - dimkeep>(exp.self(), DType(1));\n}\n/*!\n * \\brief a expression that sum over rows of a matrix\n * \\param exp input expression that must be a matrix Tensor<?, 2>\n * \\return a expresion with type Tensor<Device, 1>\n * \\tparam SrcExp expression\n * \\tparam etype type of expression\n */\ntemplate<typename SrcExp, typename DType, int etype>\ninline ReduceTo1DExp<SrcExp, DType, red::sum, 1>\nsum_rows(const Exp<SrcExp, DType, etype> &exp) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim ==2>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return sumall_except_dim<1>(exp);\n}\ntemplate<typename SV, typename Device, typename DType,\n         typename SrcExp, typename Reducer, int m_dimkeep>\nstruct ExpComplexEngine<SV,\n                        Tensor<Device, 1, DType>,\n                        ReduceTo1DExp<SrcExp, DType, Reducer, m_dimkeep>,\n                        DType> {\n  static const int dimkeep = ExpInfo<SrcExp>::kDim - m_dimkeep;\n  inline static void Eval(Tensor<Device, 1, DType> *dst,\n                          const ReduceTo1DExp<SrcExp, DType,\n                                              Reducer, m_dimkeep> &exp) {\n    TypeCheckPass<m_dimkeep != 1>\n        ::Error_Expression_Does_Not_Meet_Dimension_Req();\n    MapReduceKeepHighDim<SV, Reducer, dimkeep>(dst, exp.src_, exp.scale_);\n  }\n};\ntemplate<typename SV, typename Device, typename DType,\n         typename SrcExp, typename Reducer>\nstruct ExpComplexEngine<SV,\n                        Tensor<Device, 1, DType>,\n                        ReduceTo1DExp<SrcExp, DType, Reducer, 1>, DType> {\n  inline static void Eval(Tensor<Device, 1, DType> *dst,\n                          const ReduceTo1DExp<SrcExp, DType, Reducer, 1> &exp) {\n    MapReduceKeepLowest<SV, Reducer>(dst, exp.src_, exp.scale_);\n  }\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_REDUCETO1D_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/reshape.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file reshape.h\n * \\brief support for reshape\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_EXTENSION_RESHAPE_H_\n#define MSHADOW_EXTENSION_RESHAPE_H_\n#include \"../extension.h\"\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief reshape the content to another shape\n * input: Tensor<Device,dimsrc>: ishape\n * output: Tensor<Device,dimdst> ishape.Size() == oshape.Size()\n * \\tparam SrcExp source expression\n * \\tparam dimdst target dimension\n * \\tparam dimsrc source dimension\n */\ntemplate<typename SrcExp, typename DType, int dimdst, int dimsrc>\nstruct ReshapeExp:\n      public MakeTensorExp<ReshapeExp<SrcExp, DType, dimdst, dimsrc>,\n                           SrcExp, dimdst, DType> {\n  /*! \\brief source expression */\n  const SrcExp &src_;\n  /*! \\brief smallest dimension of input */\n  index_t ishapex_;\n  /*! \\brief constructor */\n  ReshapeExp(const SrcExp &src, Shape<dimdst> shape)\n      : src_(src) {\n    Shape<dimsrc> ishape = ShapeCheck<dimsrc, SrcExp>::Check(src_);\n    CHECK_EQ(ishape.Size(), shape.Size()) << \"reshape size must match\";\n    ishapex_ = ishape[dimsrc - 1];\n    this->shape_ = shape;\n  }\n};\n/*!\n * \\brief a expression that reshapes a tensor to another shape\n * \\param src Tensor<Device,dimsrc>:\n * \\param oshape target shape\n * \\return a expresion with type Tensor<Device,dimdst>\n * \\tparam SrcExp source expression\n * \\tparam etype source expression type\n * \\tparam dimdst target dimension\n */\ntemplate<typename SrcExp, typename DType, int etype, int dimdst>\ninline ReshapeExp<SrcExp, DType, dimdst, ExpInfo<SrcExp>::kDim>\nreshape(const Exp<SrcExp, DType, etype> &src, Shape<dimdst> oshape) {\n  return ReshapeExp<SrcExp, DType, dimdst, ExpInfo<SrcExp>::kDim>\n      (src.self(), oshape);\n}\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename SrcExp, typename DType, int dimdst, int dimsrc>\nstruct Plan<ReshapeExp<SrcExp, DType, dimdst, dimsrc>, DType> {\n public:\n  explicit Plan(const ReshapeExp<SrcExp, DType, dimdst, dimsrc> &e)\n      : src_(MakePlan(e.src_)),\n        oshapex_(e.shape_[dimdst - 1]), ishapex_(e.ishapex_) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    const index_t idx = y * oshapex_ + x;\n    return src_.Eval(idx / ishapex_, idx % ishapex_);\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t oshapex_, ishapex_;\n};\n// special work plan for 1 dimensional data\ntemplate<typename SrcExp, typename DType, int dimdst>\nstruct Plan<ReshapeExp<SrcExp, DType, dimdst, 1>, DType> {\n public:\n  explicit Plan(const ReshapeExp<SrcExp, DType, dimdst, 1> &e)\n      : src_(MakePlan(e.src_)), oshapex_(e.shape_[dimdst - 1]) {\n  }\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    return src_.Eval(0, y * oshapex_ + x);\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t oshapex_;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_RESHAPE_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/slice.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file slice.h\n * \\brief support for slice a certain dimension.\n */\n#ifndef MSHADOW_EXTENSION_SLICE_H_\n#define MSHADOW_EXTENSION_SLICE_H_\n\n#include \"../extension.h\"\n\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief slice expression, slice a tensor's channel\n * \\tparam SrcExp left expression\n * \\tparam DType the type of elements\n * \\tparam srcdim dimension of src\n * \\tparam dimsrc_m_cat dimsrc - dimcat\n */\ntemplate<typename SrcExp,\n         typename Device, typename DType,\n         int srcdim, int dimsrc_m_slice>\nstruct SliceExp : public TRValue<SliceExp<SrcExp,\n                                          Device, DType,\n                                          srcdim, dimsrc_m_slice>,\n                                 Device, srcdim, DType> {\n  static const int dimslice = srcdim - dimsrc_m_slice;\n  const SrcExp &src_;\n  index_t ch_begin_;\n  index_t ch_old_;\n  Shape<srcdim> shape_;\n  SliceExp(const SrcExp &src, index_t begin, index_t end)\n      : src_(src), ch_begin_(begin) {\n    shape_ = ShapeCheck<srcdim, SrcExp>::Check(src_);\n    ch_old_ = shape_[dimslice];\n    CHECK(begin <= shape_[dimslice] && end <= shape_[dimslice])\n        << \"The slice went out of range. \";\n    shape_[dimslice] = end - begin;\n  }\n  template<typename E, int etype>\n  inline void\n  operator=(const expr::Exp<E, DType, etype> &exp) {\n    this->__assign(exp);\n  }\n  inline void\n  operator=(const DType &exp) {\n    this->__assign(exp);\n  }\n};  // struct Slice\n\n/*!\n * \\brief Slice a Tensor\n * \\param src source tensor\n * \\param begin The beginning slice.\n * \\param end The end slice.\n * \\return sliced tensor\n * \\tparam sdim the dimension to slice on\n * \\tparam SrcExp source expression\n * \\tparam DType the type of elements\n * \\tparam etype type of expression\n */\ntemplate<int sdim, typename SrcExp,\n         typename Device, typename DType, int srcdim>\ninline SliceExp<SrcExp, Device, DType, srcdim, srcdim - sdim>\nslice(const TRValue<SrcExp, Device, srcdim, DType> &src, index_t begin, index_t end) {\n  TypeCheckPass<sdim < srcdim && ExpInfo<SrcExp>::kDim == srcdim>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return SliceExp<SrcExp, Device, DType, srcdim, srcdim - sdim>(src.self(), begin, end);\n}\n//------------------------\n//  engine plugin\n//------------------------\n// runtime shapecheck\ntemplate<typename SrcExp,\n         typename Device, typename DType,\n         int srcdim, int dimsrc_m_slice>\nstruct ShapeCheck<srcdim, SliceExp<SrcExp, Device, DType, srcdim, dimsrc_m_slice> >{\n  inline static Shape<srcdim> Check(const SliceExp<SrcExp,\n                                    Device, DType, srcdim, dimsrc_m_slice> &t) {\n    return t.shape_;\n  }\n};\ntemplate<typename SrcExp,\n         typename Device, typename DType,\n         int srcdim, int dimsrc_m_slice>\nstruct StreamInfo<Device, SliceExp<SrcExp, Device, DType, srcdim, dimsrc_m_slice> >{\n  inline static Stream<Device> *\n  Get(const SliceExp<SrcExp, Device, DType, srcdim, dimsrc_m_slice> &t) {\n    return StreamInfo<Device, SrcExp>::Get(t.src_);\n  }\n};\n// static typecheck\ntemplate<typename SrcExp,\n         typename Device, typename DType,\n         int srcdim, int dimsrc_m_slice>\nstruct ExpInfo<SliceExp<SrcExp, Device, DType, srcdim, dimsrc_m_slice> >{\n  static const int kDim = ExpInfo<SrcExp>::kDim;\n  static const int kDevMask = ExpInfo<SrcExp>::kDevMask;\n};\n//----------------------\n// Execution plan\n//---------------------\ntemplate<typename SrcExp,\n         typename Device, typename DType,\n         int srcdim, int dimsrc_m_slice>\nstruct Plan<SliceExp<SrcExp, Device, DType, srcdim, dimsrc_m_slice>, DType> {\n public:\n  static const int dimslice = srcdim - dimsrc_m_slice;\n  explicit Plan(const SliceExp<SrcExp, Device, DType, srcdim, dimsrc_m_slice> &e)\n      : src_(MakePlan(e.src_)),\n        height_(e.shape_.ProdShape(dimslice + 1, srcdim - 1)),\n        ch_begin_(e.ch_begin_), ch_old_(e.ch_old_), ch_(e.shape_[dimslice]) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    const index_t y = i % height_;\n    i /= height_;\n    const index_t c = i % ch_ + ch_begin_;\n    const index_t b = i / ch_;\n    const index_t x = j;\n    return src_.Eval((b * ch_old_ + c) * height_ + y, x);\n  }\n  MSHADOW_XINLINE DType &REval(index_t i, index_t j) {\n    const index_t y = i % height_;\n    i /= height_;\n    const index_t c = i % ch_ + ch_begin_;\n    const index_t b = i / ch_;\n    const index_t x = j;\n    return src_.REval((b * ch_old_ + c) * height_ + y, x);\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t height_, ch_begin_, ch_old_, ch_;\n};  // struct Plan\n\ntemplate<typename SrcExp,\n         typename Device, typename DType,\n         int srcdim>\nstruct Plan<SliceExp<SrcExp, Device, DType, srcdim, 1>, DType> {\n public:\n  explicit Plan(const SliceExp<SrcExp, Device, DType, srcdim, 1> &e)\n      : src_(MakePlan(e.src_)),\n        ch_begin_(e.ch_begin_) {}\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    return src_.Eval(y, x + ch_begin_);\n  }\n  MSHADOW_XINLINE DType &REval(index_t y, index_t x) {\n    return src_.REval(y, x + ch_begin_);\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t ch_begin_;\n};\n}  // namespace expr\n}   // namespace mshadow\n#endif  // MSHADOW_EXTENSION_SLICE_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/slice_ex.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file slice.h\n * \\brief support for slice a certain dimension.\n */\n#ifndef MSHADOW_EXTENSION_SLICE_EX_H_\n#define MSHADOW_EXTENSION_SLICE_EX_H_\n\n#include \"../extension.h\"\n\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief slice expression, slice a tensor's channel\n * \\tparam SrcExp left expression\n * \\tparam DType the type of elements\n * \\tparam srcdim dimension of src\n * \\tparam dimsrc_m_cat dimsrc - dimcat\n */\ntemplate<typename SrcExp, typename Device,\n         typename DType, int srcdim>\nstruct SliceExExp : public TRValue<SliceExExp<SrcExp,\n                                              Device, DType,\n                                              srcdim>,\n                                   Device, srcdim, DType> {\n  const SrcExp &src_;\n  Shape<srcdim> src_shape_;\n  Shape<srcdim> shape_;\n  const Shape<srcdim> begin_;\n  const Shape<srcdim> end_;\n  SliceExExp(const SrcExp &src, Shape<srcdim> begin, Shape<srcdim> end)\n      : src_(src), begin_(begin), end_(end) {\n    src_shape_ = ShapeCheck<srcdim, SrcExp>::Check(src_);\n    for (int i = 0; i < srcdim; ++i) {\n      shape_[i] = end_[i] - begin_[i];\n    }\n  }\n  template<typename E, int etype>\n  inline void\n  operator=(const expr::Exp<E, DType, etype> &exp) {\n    this->__assign(exp);\n  }\n  inline void\n  operator=(const DType &exp) {\n    this->__assign(exp);\n  }\n};  // struct SliceEx\n\n/*!\n * \\brief SliceEx a Tensor\n * \\param src source tensor\n * \\param begin The beginning slice.\n * \\param end The end slice.\n * \\return sliced tensor\n * \\tparam sdim the dimension to slice on\n * \\tparam SrcExp source expression\n * \\tparam DType the type of elements\n * \\tparam etype type of expression\n */\ntemplate<typename SrcExp, typename Device,\n         typename DType, int srcdim>\ninline SliceExExp<SrcExp, Device, DType, srcdim>\nslice(const TRValue<SrcExp, Device, srcdim, DType> &src, Shape<srcdim> begin, Shape<srcdim> end) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim == srcdim>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return SliceExExp<SrcExp, Device, DType, srcdim>(src.self(), begin, end);\n}\n//------------------------\n//  engine plugin\n//------------------------\n// runtime shapecheck\ntemplate<typename SrcExp, typename Device,\n         typename DType, int srcdim>\nstruct ShapeCheck<srcdim, SliceExExp<SrcExp, Device, DType, srcdim> >{\n  inline static Shape<srcdim> Check(const SliceExExp<SrcExp,\n                                    Device, DType, srcdim> &t) {\n    return t.shape_;\n  }\n};\n\ntemplate<typename SrcExp, typename Device,\n         typename DType, int srcdim>\nstruct StreamInfo<Device, SliceExExp<SrcExp, Device, DType, srcdim> >{\n  inline static Stream<Device> *\n  Get(const SliceExExp<SrcExp, Device, DType, srcdim> &t) {\n    return StreamInfo<Device, SrcExp>::Get(t.src_);\n  }\n};\n// static typecheck\ntemplate<typename SrcExp, typename Device,\n         typename DType, int srcdim>\nstruct ExpInfo<SliceExExp<SrcExp, Device, DType, srcdim> >{\n  static const int kDim = ExpInfo<SrcExp>::kDim;\n  static const int kDevMask = ExpInfo<SrcExp>::kDevMask;\n};\n//----------------------\n// Execution plan\n//---------------------\ntemplate<typename SrcExp, typename Device,\n         typename DType, int srcdim>\nstruct Plan<SliceExExp<SrcExp, Device, DType, srcdim>, DType> {\n public:\n  explicit Plan(const SliceExExp<SrcExp, Device, DType, srcdim> &e)\n      : src_(MakePlan(e.src_)), begin_(e.begin_),\n        src_shape_(e.src_shape_), shape_(e.shape_) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    index_t idx = 0;\n    index_t stride = 1;\n    #pragma unroll\n    for (int k = srcdim-2; k >= 0; --k) {\n      idx += stride * (i%shape_[k] + begin_[k]);\n      i /= shape_[k];\n      stride *= src_shape_[k];\n    }\n    return src_.Eval(idx, j + begin_[srcdim-1]);\n  }\n  MSHADOW_XINLINE DType &REval(index_t i, index_t j) {\n    index_t idx = 0;\n    index_t stride = 1;\n    #pragma unroll\n    for (int k = srcdim-2; k >= 0; --k) {\n      idx += stride * (i%shape_[k] + begin_[k]);\n      i /= shape_[k];\n      stride *= src_shape_[k];\n    }\n    return src_.REval(idx, j + begin_[srcdim-1]);\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const Shape<srcdim> begin_, src_shape_, shape_;\n};  // struct Plan\n}  // namespace expr\n}   // namespace mshadow\n#endif  // MSHADOW_EXTENSION_SLICE_EX_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/spatial_pool.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file spatial_pool.h\n * \\brief support for spatial pooling\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_EXTENSION_SPATIAL_POOL_H_\n#define MSHADOW_EXTENSION_SPATIAL_POOL_H_\n#include <algorithm>\n#include \"../extension.h\"\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief pooling expression, do reduction over local patches of a image\n * \\tparam Reducer reduction method during pooling\n * \\tparam SrcExp source expression to be pooled from\n * \\tparam DType the content data type\n * \\tparam srcdim dimension of src\n */\ntemplate<typename Reducer, typename SrcExp, typename DType, int srcdim>\nstruct PoolingExp:\n      public MakeTensorExp<PoolingExp<Reducer, SrcExp, DType, srcdim>,\n                           SrcExp, srcdim, DType> {\n  /*! \\brief source operand */\n  const SrcExp &src_;\n  /*! \\brief kernel size in height */\n  index_t ksize_y_;\n  /*! \\brief kernel size in width */\n  index_t ksize_x_;\n  /*! \\brief kernel stride in y directory */\n  index_t kstride_y_;\n  /*! \\brief kernel stride in x directory */\n  index_t kstride_x_;\n  /*! \\brief source height shape[1] */\n  index_t src_height_;\n  /*! \\brief source width shape[0] */\n  index_t src_width_;\n  /*! \\brief constructor */\n  PoolingExp(const SrcExp &src,\n             index_t ksize_y, index_t ksize_x, index_t kstride_y, index_t kstride_x)\n             : src_(src), ksize_y_(ksize_y), ksize_x_(ksize_x),\n               kstride_y_(kstride_y), kstride_x_(kstride_x) {\n    Shape<srcdim> sshape = ShapeCheck<srcdim, SrcExp>::Check(src_);\n    CHECK(sshape[srcdim - 1] >= ksize_x && sshape[srcdim - 2] >= ksize_y)\n      << \"PoolingExp: kernel must be smaller than image\";\n    this->src_height_ = sshape[srcdim - 2];\n    this->src_width_  = sshape[srcdim - 1];\n    this->shape_ = sshape;\n    this->shape_[srcdim - 2] = (src_height_ - ksize_y) / kstride_y + 1;\n    this->shape_[srcdim - 1] = (src_width_  - ksize_x) / kstride_x + 1;\n  }\n  /*! \\brief constructor, specify shape */\n  PoolingExp(const SrcExp &src, Shape<2> pshape,\n             index_t ksize_y, index_t ksize_x, index_t kstride_y, index_t kstride_x)\n             : src_(src), ksize_y_(ksize_y), ksize_x_(ksize_x),\n               kstride_y_(kstride_y), kstride_x_(kstride_x) {\n    Shape<srcdim> sshape = ShapeCheck<srcdim, SrcExp>::Check(src_);\n    CHECK(sshape[srcdim - 1] >= ksize_x && sshape[srcdim - 2] >= ksize_y)\n      << \"PoolingExp: kernel must be smaller than image\";\n    this->src_height_ = sshape[srcdim - 2];\n    this->src_width_  = sshape[srcdim - 1];\n    this->shape_ = sshape;\n    this->shape_[srcdim - 2] = pshape[0];\n    this->shape_[srcdim - 1] = pshape[1];\n  }\n};\n/*!\n * \\brief pooling subregion results together\n * \\param src source image, shape: (batch, channel, height, width)\n * \\param ksize_y kernel size in height\n * \\param ksize_x kernel size in width\n * \\param kstride_y stride in y directory\n * \\param kstride_x stride in x directory\n * \\return expression of pooled result\n * \\tparam Reducer reducer type\n * \\tparam SrcExp source expression\n * \\tparam DType the content data type\n * \\tparam etype type of expression\n */\ntemplate<typename Reducer, typename SrcExp, typename DType, int etype>\ninline PoolingExp<Reducer, SrcExp, DType, ExpInfo<SrcExp>::kDim>\npool(const Exp<SrcExp, DType, etype> &src,\n     index_t ksize_y, index_t ksize_x, index_t kstride_y, index_t kstride_x) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim >= 2>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return PoolingExp<Reducer, SrcExp, DType, ExpInfo<SrcExp>::kDim>\n      (src.self(), ksize_y, ksize_x, kstride_y, kstride_x);\n}\n/*!\n * \\brief same as pool, except the output shape is specified by pshape\n * \\param src source image\n * \\param pshape ouput shape\n * \\param ksize_y kernel size in y\n * \\param ksize_x kernel size in x\n * \\param kstride_y stride in y directory\n * \\param kstride_x stride in x directory\n * \\return expression of pooled result\n * \\tparam Reducer reducer type\n * \\tparam SrcExp source expression\n * \\tparam DType the content data type\n * \\tparam etype type of expression\n */\ntemplate<typename Reducer, typename SrcExp,\n         typename DType, int etype>\ninline PoolingExp<Reducer, SrcExp, DType, ExpInfo<SrcExp>::kDim>\npool(const Exp<SrcExp, DType, etype> &src, Shape<2> pshape,\n     index_t ksize_y, index_t ksize_x, index_t kstride_y, index_t kstride_x) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim >= 2>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return PoolingExp<Reducer, SrcExp, DType, ExpInfo<SrcExp>::kDim>\n     (src.self(), pshape, ksize_y, ksize_x, kstride_y, kstride_x);\n}\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename Reducer, typename SrcExp, typename DType, int srcdim>\nstruct Plan<PoolingExp< Reducer, SrcExp, DType, srcdim>, DType> {\n public:\n  explicit Plan(const PoolingExp<Reducer, SrcExp, DType, srcdim> &e)\n      : src_(MakePlan(e.src_)),\n      ksize_y_(e.ksize_y_), ksize_x_(e.ksize_x_),\n      kstride_y_(e.kstride_y_), kstride_x_(e.kstride_x_),\n        src_height_(e.src_height_), src_width_(e.src_width_),\n        new_height_(e.shape_[srcdim - 2]) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    using namespace std;\n    const index_t py = i % new_height_;\n    const index_t y_start = py * kstride_y_;\n    const index_t y_end = min(y_start + ksize_y_, src_height_);\n    const index_t px = j;\n    const index_t x_start = px * kstride_x_;\n    const index_t x_end = min(x_start + ksize_x_, src_width_);\n    const index_t c = i / new_height_;\n\n    DType res; Reducer::SetInitValue(res);\n    for (index_t y = y_start; y < y_end; ++y) {\n      for (index_t x = x_start; x < x_end; ++x) {\n        Reducer::Reduce(res, src_.Eval(c * src_height_ + y, x));\n      }\n    }\n    return res;\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t ksize_y_, ksize_x_, kstride_y_, kstride_x_;\n  const index_t src_height_, src_width_;\n  const index_t new_height_;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_SPATIAL_POOL_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/spatial_unpool.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file spatial_unpool.h\n * \\brief support for unpool\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_EXTENSION_SPATIAL_UNPOOL_H_\n#define MSHADOW_EXTENSION_SPATIAL_UNPOOL_H_\n#include <algorithm>\n#include \"../extension.h\"\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief unpooling expr reverse operation of pooling, used to pass gradient back\n * \\tparam Reducer reduction method during pooling\n * \\tparam SrcExp source expression to be pooled from\n * \\tparam DType the content data type\n * \\tparam srcdim dimension of src\n */\ntemplate<typename Reducer, typename SrcExp, typename DType, int srcdim>\nstruct UnPoolingExp:\n      public MakeTensorExp<UnPoolingExp<Reducer, SrcExp, DType, srcdim>,\n                           SrcExp, srcdim, DType> {\n  /*! \\brief source input, corresponds to src in pooling */\n  const SrcExp &data_src_;\n  /*! \\brief result of pooled data, corresponds to result of pooling */\n  const SrcExp &data_pooled_;\n  /*! \\brief gradient data of pooled part, to be propgate down */\n  const SrcExp &grad_pooled_;\n  /*! \\brief shape of pooled expression */\n  index_t pshape_y_;\n  /*! \\brief shape of pooled expression */\n  index_t pshape_x_;\n  /*! \\brief kernel size in height */\n  index_t ksize_y_;\n  /*! \\brief kernel size in width */\n  index_t ksize_x_;\n  /*! \\brief kernel stride in y directory */\n  index_t kstride_y_;\n  /*! \\brief kernel stride in x directory */\n  index_t kstride_x_;\n  /*! \\brief constructor */\n  UnPoolingExp(const SrcExp &data_src,\n               const SrcExp &data_pooled,\n               const SrcExp &grad_pooled,\n               index_t ksize_y, index_t ksize_x, index_t kstride_y, index_t kstride_x)\n      : data_src_(data_src), data_pooled_(data_pooled),\n        grad_pooled_(grad_pooled),\n    ksize_y_(ksize_y), ksize_x_(ksize_x),\n    kstride_y_(kstride_y), kstride_x_(kstride_x) {\n    Shape<srcdim> pshape = ShapeCheck<srcdim, SrcExp>::Check(grad_pooled);\n    typedef ShapeCheck<srcdim, SrcExp> ShapeCheckSrcDimSrcExp;\n    CHECK_EQ(pshape, ShapeCheckSrcDimSrcExp::Check(data_pooled))\n      << \"UnPoolingExp: pooled shape mismatch\";\n    Shape<srcdim> sshape = ShapeCheck<srcdim, SrcExp>::Check(data_src);\n    for (int k = 0;  k < srcdim - 2; ++k) {\n      CHECK_EQ(pshape[k], sshape[k]) << \"UnPoolingExp: pool and src shape mismatch\";\n    }\n    pshape_x_ = pshape[srcdim - 1];\n    pshape_y_ = pshape[srcdim - 2];\n    this->shape_ = sshape;\n  }\n};\n/*!\n * \\brief unpooling gradient for 4D, backprop gradient value back, revserse operation of pooling,\n *   same as unpooling, but allows unequal size of kernel\n * \\param data_src  source input, corresponds to src in pooling\n * \\param data_pooled result of pooled data, corresponds to result of pooling\n * \\param grad_pooled gradient data of pooled part, to be propgate down\n * \\param ksize_y kernel height\n * \\param ksize_x kernel width\n * \\param kstride_y stride in y directory\n * \\param kstride_x stride in x directory\n * \\return expression corresponding to unpooled 4D Tensor, storing backproped gradient\n * \\tparam Reducer reducer type\n * \\tparam SrcExp source expression\n * \\tparam DType the content data type\n * \\tparam etype type of expression\n */\ntemplate<typename Reducer, typename SrcExp, typename DType, int etype>\ninline UnPoolingExp<Reducer, SrcExp, DType, ExpInfo<SrcExp>::kDim>\nunpool(const Exp<SrcExp, DType, etype> &data_src,\n       const Exp<SrcExp, DType, etype> &data_pooled,\n       const Exp<SrcExp, DType, etype> &grad_pooled,\n       index_t ksize_y, index_t ksize_x, index_t kstride_y, index_t kstride_x) {\n  return UnPoolingExp<Reducer, SrcExp, DType, ExpInfo<SrcExp>::kDim>\n      (data_src.self(), data_pooled.self(), grad_pooled.self(),\n       ksize_y, ksize_x, kstride_y, kstride_x);\n}\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename Reducer, typename SrcExp, typename DType, int srcdim>\nstruct Plan<UnPoolingExp<Reducer, SrcExp, DType, srcdim>, DType> {\n public:\n  explicit Plan(const UnPoolingExp<Reducer, SrcExp, DType, srcdim> &e)\n      : data_src_(MakePlan(e.data_src_)), data_pooled_(MakePlan(e.data_pooled_)),\n        grad_pooled_(MakePlan(e.grad_pooled_)), sshape_y_(e.shape_[srcdim - 2]),\n        pshape_y_(e.pshape_y_),  pshape_x_(e.pshape_x_),\n        ksize_y_(e.ksize_y_), ksize_x_(e.ksize_x_),\n        kstride_y_(e.kstride_y_), kstride_x_(e.kstride_x_) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    using namespace std;\n    const index_t x = j;\n    const index_t y = i % sshape_y_;\n    const index_t c = i / sshape_y_;\n    const DType vsrc = data_src_.Eval(i, j);\n    const index_t py_min =\n        y < ksize_y_ ? 0 : (y - ksize_y_ + kstride_y_) / kstride_y_;\n    const index_t px_min =\n        x < ksize_x_ ? 0 : (x - ksize_x_ + kstride_x_) / kstride_x_;\n    const index_t py_max = min((y + kstride_y_) / kstride_y_, pshape_y_);\n    const index_t px_max = min((x + kstride_x_) / kstride_x_, pshape_x_);\n\n    DType val = static_cast<DType>(0);\n    for (index_t py = py_min; py < py_max; ++py) {\n      for (index_t px = px_min; px < px_max; ++px) {\n        val += Reducer::PartialGrad(vsrc,\n                                    data_pooled_.Eval(c * pshape_y_ + py, px)) *\n                                    grad_pooled_.Eval(c * pshape_y_ + py, px);\n      }\n    }\n\n    return val;\n  }\n\n private:\n  Plan<SrcExp, DType> data_src_, data_pooled_, grad_pooled_;\n  const index_t sshape_y_, pshape_y_, pshape_x_;\n  const index_t ksize_y_, ksize_x_;\n  const index_t kstride_y_, kstride_x_;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_SPATIAL_UNPOOL_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/spatial_upsampling_nearest.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file spatial_upsampling.h\n * \\brief\n * \\author Bing Xu\n*/\n#ifndef MSHADOW_EXTENSION_SPATIAL_UPSAMPLING_NEAREST_H_\n#define MSHADOW_EXTENSION_SPATIAL_UPSAMPLING_NEAREST_H_\n#include \"../extension.h\"\n\nnamespace mshadow {\nnamespace expr {\n\n/*! \\brief nearest neighboor upsampling\n *         out(x, y) = in(int(x / scale_x), int(y / scale_y))\n *  \\tparam SrcExp source expression\n *  \\tparam DType data type\n *  \\tparam srcdim source dimension\n */\ntemplate<typename SrcExp, typename DType, int srcdim>\nstruct UpSamplingNearestExp :\n  public MakeTensorExp<UpSamplingNearestExp<SrcExp, DType, srcdim>,\n                       SrcExp, srcdim, DType> {\n  /*! \\brief source oprand */\n  const SrcExp &src_;\n  /*! \\brief up sampling scale */\n  index_t scale_;\n  /*! \\brief constructor */\n  UpSamplingNearestExp(const SrcExp &src, index_t scale)\n    : src_(src), scale_(scale) {\n    this->shape_ = ShapeCheck<srcdim, SrcExp>::Check(src_);\n    this->shape_[srcdim - 2] *= scale_;\n    this->shape_[srcdim - 1] *= scale_;\n  }\n};\n\n\ntemplate<typename SrcExp, typename DType, int etype>\ninline UpSamplingNearestExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>\nupsampling_nearest(const Exp<SrcExp, DType, etype> &src, index_t scale) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim >= 2>\n    ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return UpSamplingNearestExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>(src.self(), scale);\n}\n\ntemplate<typename SrcExp, typename DType, int srcdim>\nstruct Plan<UpSamplingNearestExp<SrcExp, DType, srcdim>, DType> {\n public:\n  explicit Plan(const UpSamplingNearestExp<SrcExp, DType, srcdim> &e)\n    : src_(MakePlan(e.src_)),\n      scale_(e.scale_),\n      new_height_(e.shape_[srcdim - 2]),\n      src_height_(static_cast<index_t>(e.shape_[srcdim - 2] / e.scale_)) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    const index_t x = j;\n    const index_t y = i % new_height_;\n    const index_t c = i / new_height_;\n    const index_t h = static_cast<index_t>(y / scale_);\n    const index_t w = static_cast<index_t>(x / scale_);\n    return src_.Eval(c * src_height_ + h, w);\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t scale_;\n  const index_t new_height_;\n  const index_t src_height_;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_SPATIAL_UPSAMPLING_NEAREST_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/swapaxis.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file swapaxis.h\n * \\brief support for swapaxis\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_EXTENSION_SWAPAXIS_H_\n#define MSHADOW_EXTENSION_SWAPAXIS_H_\n#include <algorithm>\n#include <utility>\n#include \"../extension.h\"\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief swap two axis of a tensor\n * input: Tensor<Device,dim>: ishape\n * output: Tensor<Device,dimdst> oshape[a1],oshape[a2] = ishape[a2],oshape[a1]\n *\n * \\tparam SrcExp type of source expression\n * \\tparam DType the type of elements \n * \\tparam dimsrc source dimension, assert a1 > a2\n * \\tparam m_a1 one dimension to be swapped, encoded by dimsrc - a1 \n * \\tparam a2 second dimension to be swapped, encoded by a2\n */\ntemplate<typename SrcExp, typename DType, int dimsrc, int m_a1, int a2>\nstruct SwapAxisExp:\n      public MakeTensorExp<SwapAxisExp<SrcExp, DType, dimsrc, m_a1, a2>,\n                           SrcExp, dimsrc, DType> {\n  // decode the a1, a2\n  static const int a1 = dimsrc - m_a1;\n  /*! \\brief source expression */\n  const SrcExp &src_;\n  /*! \\brief constructor */\n  explicit SwapAxisExp(const SrcExp &src) : src_(src) {\n    this->shape_ = ShapeCheck<dimsrc, SrcExp>::Check(src);\n    std::swap(this->shape_[a1], this->shape_[a2]);\n  }\n};\n/*!\n * \\brief a expression that reshapes a tensor to another shape\n * \\param src Tensor<Device,dimsrc>:\n * \\return a expresion with type Tensor<Device,dimdst>\n * \\tparam a1 higher dimension to be swapped, assert a1 > a2\n * \\tparam a2 lower dimension to be swapped\n * \\tparam SrcExp source expression\n * \\tparam DType the type of elements \n * \\tparam etype source expression type\n */\ntemplate<int a1, int a2, typename SrcExp, typename DType, int etype>\ninline SwapAxisExp<SrcExp, DType, ExpInfo<SrcExp>::kDim,\n                   ExpInfo<SrcExp>::kDim - a1, a2>\nswapaxis(const Exp<SrcExp, DType, etype> &src) {\n  typedef ExpInfo<SrcExp> Info;\n  TypeCheckPass<Info::kDim >= a1 + 1 && Info::kDim >= a2 + 1 &&\n                a2 < a1>::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return SwapAxisExp<SrcExp, DType, ExpInfo<SrcExp>::kDim,\n                     ExpInfo<SrcExp>::kDim - a1, a2>(src.self());\n}\ntemplate<typename SrcExp, typename DType, int dimsrc, int m_a1, int a2>\nstruct Plan<SwapAxisExp<SrcExp, DType, dimsrc, m_a1, a2>, DType> {\n public:\n  // decode the a1\n  static const int a1 = dimsrc - m_a1;\n  explicit Plan(const SwapAxisExp<SrcExp, DType, dimsrc, m_a1, a2> &e)\n      : src_(MakePlan(e.src_)),\n        shapey_(e.shape_.ProdShape(a1 + 1, dimsrc - 1)),\n        shapez_(e.shape_[a1]),\n        shapec_(e.shape_.ProdShape(a2 + 1, a1)),\n        shapen_(e.shape_[a2]) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    const index_t y = i % shapey_;\n    i /= shapey_;\n    const index_t z = i % shapez_;\n    i /= shapez_;\n    const index_t c = i % shapec_;\n    i /= shapec_;\n    const index_t n = i % shapen_;\n    // swap z and n\n    return src_.Eval(((((i / shapen_) * shapez_ + z) * shapec_ +\n                          c) * shapen_ + n) * shapey_ + y, j);\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t shapey_, shapez_, shapec_, shapen_;\n};\ntemplate<typename SrcExp, typename DType, int dimsrc, int a2>\nstruct Plan<SwapAxisExp<SrcExp, DType, dimsrc, 1, a2>, DType> {\n public:\n  explicit Plan(const SwapAxisExp<SrcExp, DType, dimsrc, 1, a2> &e)\n      : src_(MakePlan(e.src_)),\n        shapex_(e.shape_[dimsrc - 1]),\n        shapey_(e.shape_.ProdShape(a2 + 1, dimsrc - 1)),\n        shapez_(e.shape_[a2]) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t x) const {\n    // swap x and z\n    const index_t y = i % shapey_;\n    i /= shapey_;\n    const index_t z = i % shapez_;\n    const index_t n = i / shapez_;\n    return src_.Eval((n * shapex_ + x) * shapey_ + y , z);\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t shapex_, shapey_, shapez_;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_SWAPAXIS_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/take.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file take.h\n * \\brief\n * \\author Bing Xu\n*/\n#ifndef MSHADOW_EXTENSION_TAKE_H_\n#define MSHADOW_EXTENSION_TAKE_H_\n\n#include \"../extension.h\"\n\nnamespace mshadow {\nnamespace expr {\n\n/*! \\brief Take a column from a matrix\n *  \\tparam IndexExp type of index expression\n *  \\tparam SrcExp type of src expression\n *  \\tparam DType data type\n */\ntemplate<typename IndexExp, typename SrcExp, typename DType>\nstruct TakeExp: public Exp<TakeExp<IndexExp, SrcExp, DType>,\n                           DType, type::kChainer> {\n  /*! \\brief index oprand */\n  const IndexExp &index_;\n  /*! \\brief embediing oprand */\n  const SrcExp &src_;\n  /*! constructor */\n  TakeExp(const IndexExp &index, const SrcExp &src)\n    : index_(index), src_(src) {}\n};  // struct TakeExp\n\n\n\ntemplate<typename IndexExp,\n         typename SrcExp,\n         typename DType,\n         int e1, int e2>\ninline TakeExp<IndexExp, SrcExp, DType>\ntake(const Exp<IndexExp, DType, e1> &index,\n     const Exp<SrcExp, DType, e2> &src) {\n  return TakeExp<IndexExp, SrcExp, DType>(index.self(), src.self());\n}\n\n\n//----------------------\n// Execution plan\n//----------------------\n\ntemplate<typename IndexExp, typename SrcExp, typename DType>\nstruct Plan<TakeExp<IndexExp, SrcExp, DType>, DType> {\n public:\n  explicit Plan(const TakeExp<IndexExp, SrcExp, DType> &e)\n    : index_(MakePlan(e.index_)), src_(MakePlan(e.src_)) {\n  }\n\n  // TODO(xx): discuss W shape: in * out or out * in\n  // Now I use in * out\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    index_t idx = static_cast<index_t>(index_.Eval(0, y));\n    return static_cast<DType>(src_.Eval(idx, x));\n  }\n\n private:\n  expr::Plan<IndexExp, DType> index_;\n  expr::Plan<SrcExp, DType> src_;\n};  // struct Plan\n\ntemplate<typename IndexExp, typename SrcExp, typename DType>\ninline Plan<TakeExp<IndexExp, SrcExp, DType>, DType>\nMakePlan(const TakeExp<IndexExp, SrcExp, DType> &exp) {\n  return Plan<TakeExp<IndexExp, SrcExp, DType>, DType>(exp);\n}\n\ntemplate<int dim, typename IndexExp, typename SrcExp, typename DType>\nstruct ShapeCheck<dim, TakeExp<IndexExp, SrcExp, DType> > {\n  inline static Shape<dim>\n  Check(const TakeExp<IndexExp, SrcExp, DType> &t) {\n    CHECK(dim == 2)\n      << \"TakeExp only support 2D output\";\n    Shape<1> dshape = ShapeCheck<1, IndexExp>::Check(t.index_);\n    Shape<2> wshape = ShapeCheck<2, SrcExp>::Check(t.src_);\n    Shape<dim> ret;\n    ret[0] = dshape[0];\n    ret[1] = wshape[1];\n    return ret;\n  }\n};\n\n\ntemplate<typename IndexExp, typename SrcExp, typename DType>\nstruct ExpInfo<TakeExp<IndexExp, SrcExp, DType> > {\n  static const int kDim = 2;\n  static const int kDevMask = ExpInfo<IndexExp>::kDevMask;\n};\n\n}  // namespace expr\n}  // namespace mshadow\n\n#endif  // MSHADOW_EXTENSION_TAKE_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/take_grad.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file take_grad.h\n * \\brief\n * \\author Bing Xu\n*/\n#ifndef MSHADOW_EXTENSION_TAKE_GRAD_H_\n#define MSHADOW_EXTENSION_TAKE_GRAD_H_\n\n#include \"../extension.h\"\n\nnamespace mshadow {\nnamespace expr {\n\n/*! \\brief Calculate embedding gradient\n *  \\tparam IndexExp type of index expression\n *  \\tparam SrcExp type of src expression\n *  \\tparam DType data type\n */\n\ntemplate<typename IndexExp, typename SrcExp, typename DType>\nstruct TakeGradExp : public Exp<TakeGradExp<IndexExp, SrcExp, DType>,\n                                DType, type::kChainer> {\n  /*! \\brief index oprand */\n  const IndexExp &index_;\n  /*! \\brief out gradient oprand */\n  const SrcExp &src_;\n  /*! \\brief batch size */\n  const index_t input_dim_;\n  /*! \\brief constructor */\n  TakeGradExp(const IndexExp &index, const SrcExp &src, const index_t input_dim)\n    : index_(index), src_(src), input_dim_(input_dim) {}\n};  // struct TakeGradExp\n\n\ntemplate<typename IndexExp,\n         typename SrcExp,\n         typename DType,\n         int e1, int e2>\ninline TakeGradExp<IndexExp, SrcExp, DType>\ntake_grad(const Exp<IndexExp, DType, e1> &index,\n          const Exp<SrcExp, DType, e2> &src,\n          const index_t input_dim) {\n  return TakeGradExp<IndexExp, SrcExp, DType>(index.self(),\n                                                       src.self(),\n                                                       input_dim);\n}\n\n//----------------------\n// Execution plan\n//----------------------\n\ntemplate<typename IndexExp, typename SrcExp, typename DType>\nstruct Plan<TakeGradExp<IndexExp, SrcExp, DType>, DType> {\n public:\n  explicit Plan(const TakeGradExp<IndexExp, SrcExp, DType> &e)\n    : index_(MakePlan(e.index_)),\n      src_(MakePlan(e.src_)),\n      batch_size_(ShapeCheck<1, IndexExp>::Check(e.index_)[0]) {\n  }\n\n  // now return shape: in * out\n  MSHADOW_XINLINE DType Eval(index_t y, index_t x) const {\n    DType ret = 0.f;\n    for (index_t i = 0; i < batch_size_; ++i) {\n      index_t idx = static_cast<index_t>(index_.Eval(0, i));\n      if (idx == y) {\n        ret += static_cast<DType>(src_.Eval(i, x));\n      }\n    }\n    return ret;\n  }\n\n private:\n  expr::Plan<IndexExp, DType> index_;\n  expr::Plan<SrcExp, DType> src_;\n  const index_t batch_size_;\n};  // struct Plan\n\n\ntemplate<typename IndexExp, typename SrcExp, typename DType>\ninline Plan<TakeGradExp<IndexExp, SrcExp, DType>, DType>\nMakePlan(const TakeGradExp<IndexExp, SrcExp, DType> &exp) {\n  return Plan<TakeGradExp<IndexExp, SrcExp, DType>, DType>(exp);\n}\n\ntemplate<int dim, typename IndexExp, typename SrcExp, typename DType>\nstruct ShapeCheck<dim, TakeGradExp<IndexExp, SrcExp, DType> > {\n  inline static Shape<dim>\n  Check(const TakeGradExp<IndexExp, SrcExp, DType> &t) {\n    CHECK(dim == 2)\n      << \"TakeGradExp only support 2D output\";\n    // Shape<1> dshape = ShapeCheck<1, IndexExp>::Check(t.index_);\n    Shape<2> gshape = ShapeCheck<2, SrcExp>::Check(t.src_);\n    Shape<dim> ret;\n    ret[0] = t.input_dim_;\n    ret[1] = gshape[1];\n    return ret;\n  }\n};  // struct ShapeCheck\n\ntemplate<typename IndexExp, typename SrcExp, typename DType>\nstruct ExpInfo<TakeGradExp<IndexExp, SrcExp, DType> > {\n  static const int kDim = 2;\n  static const int kDevMask = ExpInfo<IndexExp>::kDevMask;\n};\n\n}  // namespace expr\n}  // namespace mshadow\n\n#endif  // MSHADOW_EXTENSION_TAKE_GRAD_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/transpose.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file transpose.h\n * \\brief support for transpose\n * \\author Junyuan Xie\n */\n#ifndef MSHADOW_EXTENSION_TRANSPOSE_H_\n#define MSHADOW_EXTENSION_TRANSPOSE_H_\n#include <algorithm>\n#include \"../extension.h\"\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief transpose axes of a tensor\n * input: Tensor<Device,dim>: ishape\n * output: Tensor<Device,dimdst> oshape[a1],oshape[a2] = ishape[a2],oshape[a1]\n *\n * \\tparam SrcExp type of source expression\n * \\tparam DType the type of elements\n * \\tparam dimsrc source dimension, assert a1 > a2\n * \\tparam m_a1 one dimension to be swapped, encoded by dimsrc - a1\n * \\tparam a2 second dimension to be swapped, encoded by a2\n */\ntemplate<typename SrcExp, typename DType, int dimsrc>\nstruct TransposeExExp:\n      public MakeTensorExp<TransposeExExp<SrcExp, DType, dimsrc>,\n                           SrcExp, dimsrc, DType> {\n  /*! \\brief source expression */\n  const SrcExp &src_;\n  const Shape<dimsrc> axes_;\n  Shape<dimsrc> dst_in_src_stride_;  // Holds the corresponding stride of the dst axes in src\n  index_t src_stride_;\n  /*! \\brief constructor */\n  explicit TransposeExExp(const SrcExp &src, Shape<dimsrc> axes) : src_(src), axes_(axes) {\n    Shape<dimsrc> src_shape = ShapeCheck<dimsrc, SrcExp>::Check(src);\n    src_stride_ = src_shape[dimsrc - 1];\n    Shape<dimsrc> src_stride;\n    src_stride[dimsrc-1] = 1;\n    for (int i = dimsrc-2; i >= 0; --i) src_stride[i] = src_shape[i+1]*src_stride[i+1];\n    for (int i = 0; i < dimsrc; ++i) {\n      dst_in_src_stride_[i] = src_stride[axes[i]];\n      this->shape_[i] = src_shape[axes[i]];\n    }\n  }\n};\n/*!\n * \\brief a expression that reshapes a tensor to another shape\n * \\param src Tensor<Device,dimsrc>:\n * \\return a expresion with type Tensor<Device,dimdst>\n * \\tparam a1 higher dimension to be swapped, assert a1 > a2\n * \\tparam a2 lower dimension to be swapped\n * \\tparam SrcExp source expression\n * \\tparam DType the type of elements\n * \\tparam etype source expression type\n */\ntemplate<typename SrcExp, typename DType, int etype>\ninline TransposeExExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>\ntranspose(const Exp<SrcExp, DType, etype> &src, Shape<ExpInfo<SrcExp>::kDim> axes) {\n  return TransposeExExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>(src.self(), axes);\n}\n\ntemplate<typename SrcExp, typename DType, int dimsrc>\nstruct Plan<TransposeExExp<SrcExp, DType, dimsrc>, DType> {\n public:\n  explicit Plan(const TransposeExExp<SrcExp, DType, dimsrc> &e)\n      : src_(MakePlan(e.src_)),\n        src_stride_(e.src_stride_),\n        dst_in_src_stride_(e.dst_in_src_stride_),\n        dst_shape_(e.shape_) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    index_t idx = j * dst_in_src_stride_[dimsrc - 1];\n    #pragma unroll\n    for (int k = dimsrc-2; k >= 0; --k) {\n      idx += (i % dst_shape_[k]) * dst_in_src_stride_[k];\n      i /= dst_shape_[k];\n    }\n    return src_.Eval(idx/src_stride_, idx%src_stride_);\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t src_stride_;\n  const Shape<dimsrc> dst_in_src_stride_, dst_shape_;\n};\n\n/*!\n * \\brief transform contiguous indices of the source tensor to indices of the transposed tensor.\n * input: Tensor<Device, k>: ishape\n * output: Tensor<Device, k>: oshape = ishape\n *\n * \\tparam SrcExp type of source expression\n * \\tparam DType the type of elements\n * \\tparam dimsrc source dimension\n * \\tparam etype source type\n */\ntemplate<typename SrcExp, typename DType, int dimsrc, int etype>\nstruct TransposeIndicesExp:\n      public Exp<TransposeIndicesExp<SrcExp, DType, dimsrc, etype>, DType, etype> {\n  /*! \\brief source expression */\n  const SrcExp &src_indices_;  // Expression of the source indices\n  Shape<dimsrc> src_shape_;  // Holds the corresponding stride of the source axes in dst\n  const Shape<dimsrc> axes_;  // The transpose axes\n  Shape<dimsrc> src_in_dst_stride_;  // Holds the corresponding stride of the source axes in dst\n  /*! \\brief constructor */\n  explicit TransposeIndicesExp(const SrcExp &src_indices,\n                               Shape<dimsrc> src_shape,\n                               Shape<dimsrc> axes) : src_indices_(src_indices),\n                                                     src_shape_(src_shape), axes_(axes) {\n    Shape<dimsrc> dst_shape_;\n    Shape<dimsrc> dst_stride_;\n    bool axes_checking_flag[dimsrc] = { 0 };\n    for (int i = 0; i < dimsrc; ++i) {\n      CHECK_LT(static_cast<int>(axes[i]), dimsrc)\n        << \"Invalid axes input! All elements of axes must be between 0 and \" << dimsrc\n        << \", find axes=\" << axes;\n      dst_shape_[i] = src_shape[axes[i]];\n      axes_checking_flag[axes[i]] = true;\n    }\n    // check if the input axes is valid\n    for (int i = 0; i < dimsrc; ++i) {\n      CHECK_EQ(axes_checking_flag[i], true)\n        << \"Invalid axes input! All elements of axes must be between 0 and \" << dimsrc\n        << \", find axes=\" << axes;\n    }\n    dst_stride_[dimsrc - 1] = 1;\n    for (int i = dimsrc - 2; i >= 0; --i) dst_stride_[i] = dst_shape_[i+1] * dst_stride_[i+1];\n    for (int i = 0; i < dimsrc; ++i) {\n      src_in_dst_stride_[axes[i]] = dst_stride_[i];\n    }\n  }\n};\n\n/*!\n * \\brief a expression that reshapes a tensor to another shape\n * \\param src Tensor<Device,dimsrc>:\n * \\return a expresion with type Tensor<Device,dimdst>\n * \\tparam a1 higher dimension to be swapped, assert a1 > a2\n * \\tparam a2 lower dimension to be swapped\n * \\tparam SrcExp source expression\n * \\tparam DType the type of elements\n * \\tparam etype source expression type\n */\ntemplate<typename SrcExp, typename DType, int dimsrc, int etype>\ninline TransposeIndicesExp<SrcExp, DType, dimsrc, etype>\ntranspose_indices(const Exp<SrcExp, DType, etype> &src_indices,\n                  Shape<dimsrc> src_shape,\n                  Shape<dimsrc> axes) {\n  return TransposeIndicesExp<SrcExp, DType, dimsrc, etype>(src_indices.self(), src_shape, axes);\n}\n\ntemplate<typename SrcExp, typename DType, int dimsrc, int etype>\nstruct Plan<TransposeIndicesExp<SrcExp, DType, dimsrc, etype>, DType> {\n public:\n  explicit Plan(const TransposeIndicesExp<SrcExp, DType, dimsrc, etype> &e)\n      : src_indices_(MakePlan(e.src_indices_)),\n        src_in_dst_stride_(e.src_in_dst_stride_),\n        src_shape_(e.src_shape_) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    index_t src_idx = static_cast<index_t>(src_indices_.Eval(i, j));\n    index_t dst_idx = 0;\n    #pragma unroll\n    for (int k = dimsrc - 1; k >= 0; --k) {\n      dst_idx += (src_idx % src_shape_[k]) * src_in_dst_stride_[k];\n      src_idx /= src_shape_[k];\n    }\n    return static_cast<DType>(dst_idx);\n  }\n\n private:\n  Plan<SrcExp, DType> src_indices_;\n  const Shape<dimsrc> src_in_dst_stride_, src_shape_;\n};\n\n//----------------------\n// Execution plan\n//----------------------\n/*! \\brief make expression */\ntemplate<typename SrcExp, typename DType, int dimsrc, int etype>\ninline Plan<TransposeIndicesExp<SrcExp, DType, dimsrc, etype>, DType>\nMakePlan(const TransposeIndicesExp<SrcExp, DType, dimsrc, etype> &e) {\n  return Plan<TransposeIndicesExp<SrcExp, DType, dimsrc, etype>, DType>(e);\n}\n\ntemplate<int dim, typename SrcExp, typename DType, int dimsrc, int etype>\nstruct ShapeCheck<dim, TransposeIndicesExp<SrcExp, DType, dimsrc, etype> > {\n  inline static Shape<dim>\n  Check(const TransposeIndicesExp<SrcExp, DType, dimsrc, etype> &t) {\n    Shape<dim> s = ShapeCheck<dim, SrcExp>::Check(t.src_indices_);\n    return s;\n  }\n};\n\ntemplate<typename SrcExp, typename DType, int dimsrc, int etype>\nstruct ExpInfo<TransposeIndicesExp<SrcExp, DType, dimsrc, etype> > {\n  static const int kDim = ExpInfo<SrcExp>::kDim;\n  static const int kDevMask = ExpInfo<SrcExp>::kDevMask;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_TRANSPOSE_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension/unpack_patch2col.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file unpack_patch2col.h\n * \\brief support for unpack\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_EXTENSION_UNPACK_PATCH2COL_H_\n#define MSHADOW_EXTENSION_UNPACK_PATCH2COL_H_\n#include \"../extension.h\"\nnamespace mshadow {\nnamespace expr {\n/*!\n * \\brief unpack local (overlap) patches of image to column of mat,\n *  can be used to implement convolution, this expression allow unpack of a batch\n *  this is a version support unpacking multiple images\n *  after getting unpacked mat, we can use: output = dot(weight, mat) to get covolved results, the relations:\n * \\tparam SrcExp source expression\n * \\tparam dstdim destination dimension\n */\ntemplate<typename SrcExp, typename DType, int srcdim>\nstruct UnpackPatchToColXExp:\n      public MakeTensorExp<UnpackPatchToColXExp<SrcExp, DType, srcdim>,\n                           SrcExp, 2, DType>{\n  /*! \\brief source operand */\n  const SrcExp &img_;\n  /*! \\brief patch height */\n  index_t psize_y_;\n  /*! \\brief patch width */\n  index_t psize_x_;\n  /*! \\brief patch stride */\n  index_t pstride_y_;\n  index_t pstride_x_;\n  /*! \\brief patch dilate */\n  index_t pdilate_y_;\n  index_t pdilate_x_;\n  /*! \\brief number of input channel */\n  index_t i_channel_;\n  /*! \\brief height of img */\n  index_t i_height_;\n  /*! \\brief width of img */\n  index_t i_width_;\n  /*! \\brief constructor */\n  UnpackPatchToColXExp(const SrcExp &img,\n                       index_t psize_y,\n                       index_t psize_x,\n                       index_t pstride_y,\n                       index_t pstride_x,\n                       index_t pdilate_y,\n                       index_t pdilate_x)\n      : img_(img), psize_y_(psize_y), psize_x_(psize_x),\n      pstride_y_(pstride_y), pstride_x_(pstride_x),\n      pdilate_y_(pdilate_y), pdilate_x_(pdilate_x){\n    Shape<srcdim> imshape = ShapeCheck<srcdim, SrcExp>::Check(img_);\n    CHECK(imshape[srcdim - 1] >= psize_x && imshape[srcdim - 2] >= psize_y)\n      << \"UnpackPatchToCol:image shape smaller than patch size\";\n    this->i_channel_ = imshape[srcdim - 3];\n    this->i_height_  = imshape[srcdim - 2];\n    this->i_width_   = imshape[srcdim - 1];\n    // calculate number of batches\n    const index_t num = imshape.ProdShape(0, srcdim - 3);\n    const index_t o_height = (i_height_ -\n        (pdilate_y * (psize_y - 1) + 1)) / pstride_y + 1;\n    const index_t o_width  = (i_width_  -\n        (pdilate_x * (psize_x - 1) + 1)) / pstride_x + 1;\n    this->shape_[1] = o_height * o_width * num;\n    this->shape_[0] = psize_y * psize_x * i_channel_;\n  }\n};\n\n/*!\n * \\brief  unpack local (overlap) patches of image to column of mat, can be used to implement convolution\n *  after getting unpacked mat, we can use: output = dot(weight, mat) to get covolved results, the relations:\n *\n *  weight; shape[0]: out_channel, shape[1]: ichannel * psize_y * psize_x\n *  output; shape[0]: out_channel, shape[1]: out_height * out_width * num_of_images\n *  out_height = (in_height - psize_y) / pstride + 1, this means we pad inperfect patch with 0\n *  out_width  = (in_width - psize_x) / pstride + 1\n *\n * \\return mat target matrix; shape[0]: in_channel*psize_y*psize_x  shape[1]: out_height*out_width * num_of_images\n * \\param img source image; shape[-3]: in_channels, shape[-2]: in_height, shape[-1]: in_width, can be 3D or 4D tensor(multiple images)\n * \\param psize_y height of each patch\n * \\param psize_x width of each patch\n * \\param pstride stride of each patch\n * \\param pdilate dilate of each patch\n * \\tparam SrcExp source expression\n * \\tparam DType the type of elements\n * \\tparam etype type of expression\n */\ntemplate<typename SrcExp, typename DType, int etype>\ninline UnpackPatchToColXExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>\nunpack_patch2col(const Exp<SrcExp, DType, etype> &img,\n                 index_t psize_y, index_t psize_x, index_t pstride, index_t pdilate) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim >= 3>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return UnpackPatchToColXExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>\n      (img.self(), psize_y, psize_x, pstride, pstride, pdilate, pdilate);\n}\n\n/*!\n *if you want to specify stride_x and stride_y\n */\ntemplate<typename SrcExp, typename DType, int etype>\ninline UnpackPatchToColXExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>\nunpack_patch2col(const Exp<SrcExp, DType, etype> &img,\n                 index_t psize_y, index_t psize_x, index_t pstride_y_, index_t pstride_x_,\n                 index_t pdilate_y_, index_t pdilate_x_) {\n  TypeCheckPass<ExpInfo<SrcExp>::kDim >= 3>\n      ::Error_Expression_Does_Not_Meet_Dimension_Req();\n  return UnpackPatchToColXExp<SrcExp, DType, ExpInfo<SrcExp>::kDim>\n      (img.self(), psize_y, psize_x, pstride_y_, pstride_x_, pdilate_y_, pdilate_x_);\n}\n//----------------------\n// Execution plan\n//----------------------\ntemplate<typename SrcExp, typename DType, int srcdim>\nstruct Plan<UnpackPatchToColXExp<SrcExp, DType, srcdim>, DType> {\n public:\n  explicit Plan(const UnpackPatchToColXExp<SrcExp, DType, srcdim> &e)\n      :src_(MakePlan(e.img_)),\n       psize_y_(e.psize_y_), psize_x_(e.psize_x_),\n       pstride_y_(e.pstride_y_), pstride_x_(e.pstride_x_),\n       i_channel_(e.i_channel_), pdilate_y_(e.pdilate_y_), pdilate_x_(e.pdilate_x_),\n       i_height_(e.i_height_), i_width_(e.i_width_),\n       o_height_((i_height_ - (pdilate_y_ * (psize_y_ - 1) + 1)) / pstride_y_ + 1),\n       o_width_((i_width_ - (pdilate_x_ * (psize_x_ - 1) + 1)) / pstride_x_ + 1) {}\n  MSHADOW_XINLINE DType Eval(index_t i, index_t j) const {\n    const index_t x_offset = i % psize_x_ * pdilate_x_;\n    const index_t idivp    = i / psize_x_;\n    const index_t y_offset = idivp % psize_y_ * pdilate_y_;\n    const index_t c = idivp / psize_y_;\n    const index_t x = (j % o_width_) * pstride_x_ + x_offset;\n    const index_t jdivw = j / o_width_;\n    const index_t y = (jdivw % o_height_) * pstride_y_ + y_offset;\n    const index_t n = jdivw / o_height_;\n\n    if (x < i_width_ && y < i_height_) {\n      return src_.Eval((n * i_channel_  + c) * i_height_ + y, x);\n    } else {\n      return DType(0.0f);\n    }\n  }\n\n private:\n  Plan<SrcExp, DType> src_;\n  const index_t psize_y_, psize_x_, pstride_y_, pstride_x_, i_channel_;\n  const index_t pdilate_y_, pdilate_x_;\n  const index_t i_height_, i_width_, o_height_, o_width_;\n};\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_EXTENSION_UNPACK_PATCH2COL_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/extension.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file extension.h\n * \\brief some extension of expressions,\n *  used to support something beyond elementwise op\n * \\author Tianqi Chen, Bing Xu\n */\n#ifndef MSHADOW_EXTENSION_H_\n#define MSHADOW_EXTENSION_H_\n#include \"./expr_engine-inl.h\"\n#include \"./extension/broadcast.h\"\n#include \"./extension/unpack_patch2col.h\"\n#include \"./extension/pack_col2patch.h\"\n#include \"./extension/reshape.h\"\n#include \"./extension/swapaxis.h\"\n#include \"./extension/reduceto1d.h\"\n#include \"./extension/spatial_pool.h\"\n#include \"./extension/spatial_unpool.h\"\n#include \"./extension/channel_pool.h\"\n#include \"./extension/channel_unpool.h\"\n#include \"./extension/pad.h\"\n#include \"./extension/crop.h\"\n#include \"./extension/mirror.h\"\n#include \"./extension/concat.h\"\n#include \"./extension/implicit_gemm.h\"\n#include \"./extension/choose.h\"\n#include \"./extension/fill.h\"\n#include \"./extension/one_hot.h\"\n#include \"./extension/slice.h\"\n#include \"./extension/slice_ex.h\"\n#include \"./extension/take.h\"\n#include \"./extension/take_grad.h\"\n#include \"./extension/reduce_with_axis.h\"\n#include \"./extension/broadcast_with_axis.h\"\n#include \"./extension/spatial_upsampling_nearest.h\"\n#include \"./extension/transpose.h\"\n#include \"./extension/flip.h\"\n#include \"./extension/complex.h\"\n#include \"./extension/range.h\"\n#include \"./extension/mask.h\"\n#endif  // MSHADOW_EXTENSION_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/half.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file half.h\n * \\brief definition of half (float16) type.\n *\n * \\author Junyuan Xie\n */\n#ifndef MSHADOW_HALF_H_\n#define MSHADOW_HALF_H_\n#include \"./base.h\"\n\n#if MSHADOW_USE_F16C\n  #include <x86intrin.h>\n#endif  // MSHADOW_USE_F16C\n\n// This flag dictates rounding for the float2half() routine only (used generally on Windows),\n// not the f16c lib or cuda v7.5 (or later) behavior which is fixed at round-to-nearest-even.\n#ifndef MSHADOW_HALF_ROUND_TO_NEAREST\n#define MSHADOW_HALF_ROUND_TO_NEAREST 1\n#endif\n\n#if (MSHADOW_USE_CUDA && CUDA_VERSION >= 7050)\n  #define MSHADOW_CUDA_HALF 1\n  #include <cuda_fp16.h>\n  #if defined(__CUDA_ARCH__)\n    /*! \\brief __half2float_warp */\n    MSHADOW_XINLINE float __half2float_warp(const volatile __half& h) { /* NOLINT(*) */\n      __half val;\n#if CUDA_VERSION >= 9000\n      val = const_cast<__half&>(h);\n#else\n      val.x = h.x;\n#endif\n      return __half2float(val);\n    }\n  #endif\n#else\n  #define MSHADOW_CUDA_HALF 0\n#endif\n\n/*! \\brief namespace for mshadow */\nnamespace mshadow {\n/* \\brief name space for host/device portable half-precision floats */\nnamespace half {\n#define MSHADOW_HALF_OPERATOR(RTYPE, OP)                                  \\\n  MSHADOW_XINLINE RTYPE operator OP (half_t a, half_t b) {                \\\n    return RTYPE(float(a) OP float(b));  /* NOLINT(*) */                  \\\n  }                                                                       \\\n  template<typename T>                                                    \\\n  MSHADOW_XINLINE RTYPE operator OP (half_t a, T b) {                     \\\n    return RTYPE(float(a) OP float(b));  /* NOLINT(*) */                  \\\n  }                                                                       \\\n  template<typename T>                                                    \\\n  MSHADOW_XINLINE RTYPE operator OP (T a, half_t b) {                     \\\n    return RTYPE(float(a) OP float(b));  /* NOLINT(*) */                  \\\n  }\n\n#define MSHADOW_HALF_ASSIGNOP(AOP, OP)                                    \\\n  template<typename T>                                                    \\\n  MSHADOW_XINLINE half_t operator AOP (const T& a) {                      \\\n    return *this = half_t(float(*this) OP float(a));  /* NOLINT(*)*/      \\\n  }                                                                       \\\n  template<typename T>                                                    \\\n  MSHADOW_XINLINE half_t operator AOP (const volatile T& a) volatile {    \\\n    return *this = half_t(float(*this) OP float(a));  /* NOLINT(*)*/      \\\n  }\n\n#if (MSHADOW_CUDA_HALF && defined(__CUDA_ARCH__))\n#define MSHADOW_HALF_CONVERSIONOP(T)                                      \\\n  MSHADOW_XINLINE operator T() const {                                    \\\n    return T(__half2float(cuhalf_));  /* NOLINT(*)*/                      \\\n  }                                                                       \\\n  MSHADOW_XINLINE operator T() const volatile {                           \\\n    return T(__half2float_warp(cuhalf_));  /* NOLINT(*)*/                 \\\n  }\n#elif(MSHADOW_USE_F16C)\n#define MSHADOW_HALF_CONVERSIONOP(T)                                      \\\n  MSHADOW_XINLINE operator T() const {                                    \\\n    return T(_cvtsh_ss(half_));   /* NOLINT(*)*/                          \\\n  }                                                                       \\\n  MSHADOW_XINLINE operator T() const volatile {                           \\\n    return T(_cvtsh_ss(half_));   /* NOLINT(*)*/                          \\\n  }\n#else\n#define MSHADOW_HALF_CONVERSIONOP(T)                                      \\\n  MSHADOW_XINLINE operator T() const {                                    \\\n    return T(half2float(half_));  /* NOLINT(*)*/                          \\\n  }                                                                       \\\n  MSHADOW_XINLINE operator T() const volatile {                           \\\n    return T(half2float(half_));  /* NOLINT(*)*/                          \\\n  }\n#endif  // (MSHADOW_CUDA_HALF && defined(__CUDA_ARCH__))\n\nclass MSHADOW_ALIGNED(2) half_t {\n public:\n  union {\n    uint16_t half_;\n#if MSHADOW_CUDA_HALF\n    __half cuhalf_;\n#endif  // MSHADOW_CUDA_HALF\n  };\n\n  static MSHADOW_XINLINE half_t Binary(uint16_t value) {\n    half_t res;\n    res.half_ = value;\n    return res;\n  }\n\n  MSHADOW_XINLINE half_t() {}\n\n#if MSHADOW_CUDA_HALF\n  MSHADOW_XINLINE explicit half_t(const __half& value) {\n    cuhalf_ = value;\n  }\n#endif  // MSHADOW_CUDA_HALF\n\n  MSHADOW_XINLINE half_t(const float& value) { constructor(value); }\n  MSHADOW_XINLINE explicit half_t(const double& value) { constructor(value); }\n  MSHADOW_XINLINE explicit half_t(const int8_t& value) { constructor(value); }\n  MSHADOW_XINLINE explicit half_t(const uint8_t& value) { constructor(value); }\n  MSHADOW_XINLINE explicit half_t(const int32_t& value) { constructor(value); }\n  MSHADOW_XINLINE explicit half_t(const uint32_t& value) { constructor(value); }\n  MSHADOW_XINLINE explicit half_t(const int64_t& value) { constructor(value); }\n  MSHADOW_XINLINE explicit half_t(const uint64_t& value) { constructor(value); }\n\n  MSHADOW_HALF_CONVERSIONOP(float)\n\n  MSHADOW_HALF_ASSIGNOP(+=, +)\n  MSHADOW_HALF_ASSIGNOP(-=, -)\n  MSHADOW_HALF_ASSIGNOP(*=, *)\n  MSHADOW_HALF_ASSIGNOP(/=, /)\n\n  MSHADOW_XINLINE half_t operator+() {\n    return *this;\n  }\n\n  MSHADOW_XINLINE half_t operator-() {\n    return half_t(-float(*this));  // NOLINT(*)\n  }\n\n  MSHADOW_XINLINE half_t operator=(const half_t& a) {\n    half_ = a.half_;\n    return a;\n  }\n\n  template<typename T>\n  MSHADOW_XINLINE half_t operator=(const T& a) {\n    return *this = half_t(a);  /* NOLINT(*)*/\n  }\n\n  MSHADOW_XINLINE half_t operator=(const half_t& a) volatile {\n    half_ = a.half_;\n    return a;\n  }\n\n  template<typename T>\n  MSHADOW_XINLINE half_t operator=(const T& a) volatile {\n    return *this = half_t(a);  /* NOLINT(*)*/\n  }\n\n private:\n  union Bits {\n    float f;\n    int32_t si;\n    uint32_t ui;\n  };\n\n  static int const fp16FractionBits = 10;\n  static int const fp32FractionBits = 23;\n  static int32_t const fp32FractionMask = ~(~0u << fp32FractionBits);  // == 0x7fffff\n  static int32_t const fp32HiddenBit = 1 << fp32FractionBits;         // == 0x800000\n  static int const shift = fp32FractionBits - fp16FractionBits;       // == 13\n  static int const shiftSign = 16;\n  static int32_t const expAdjust = 127 - 15;    // exp32-127 = exp16-15, so exp16 = exp32 - (127-15)\n\n  static int32_t const infN = 0x7F800000;  // flt32 infinity\n  static int32_t const maxN = 0x477FFFFF;  // max flt32 that's a flt16 normal after >> by shift\n  static int32_t const minN = 0x38800000;  // min flt16 normal as a flt32\n  static int32_t const maxZ = 0x33000000;  // max fp32 number that's still rounded to zero in fp16\n  static int32_t const signN = 0x80000000;  // flt32 sign bit\n\n  static int32_t const infC = infN >> shift;\n  static int32_t const nanN = (infC + 1) << shift;  // minimum flt16 nan as a flt32\n  static int32_t const maxC = maxN >> shift;\n  static int32_t const minC = minN >> shift;\n  static int32_t const signC = signN >> shiftSign;  // flt16 sign bit\n\n  static int32_t const mulN = 0x52000000;  // (1 << 23) / minN\n  static int32_t const mulC = 0x33800000;  // minN / (1 << (23 - shift))\n\n  static int32_t const subC = 0x003FF;  // max flt32 subnormal down shifted\n  static int32_t const norC = 0x00400;  // min flt32 normal down shifted\n\n  static int32_t const maxD = infC - maxC - 1;\n  static int32_t const minD = minC - subC - 1;\n\n  MSHADOW_XINLINE uint16_t float2half(const float& value) const {\n    Bits v;\n    v.f = value;\n    uint32_t sign = v.si & signN;    // grab sign bit\n    v.si ^= sign;                    // clear sign bit from v\n    sign >>= shiftSign;              // logical shift sign to fp16 position\n\n    if (v.si <= maxZ) {\n      // Handle eventual zeros here to ensure vshift will not exceed 32 below.\n      v.ui = 0;\n    } else if (v.si < minN) {\n      // Handle denorms\n      uint32_t exp32 = v.ui >> fp32FractionBits;\n      int32_t exp16 = exp32 - expAdjust;\n      // If exp16 == 0 (just into the denorm range), then significant should be shifted right 1.\n      // Smaller (so negative) exp16 values should result in greater right shifts.\n      uint32_t vshift = 1 - exp16;\n      uint32_t significand = fp32HiddenBit | (v.ui & fp32FractionMask);\n      v.ui = significand >> vshift;\n      // The only time it's *not* OK to add 0x1000 (i.e. half the flt16 fraction lsb) is\n      // when the lsb of the flt16 fraction == 0 (so not rounding up to even) and the additional\n      // bits to the right of the lsb are 1000... (including flt32 significand bits\n      // that may be lost during the above vshift).  The first term below will always\n      // be true for vshift >=12 (since even the 'hidden bit' has been shifted to the\n      // right of the '1' bit in 0x1000). And when vshift <= 11, both terms combine to make\n      // the proper test of the flt32 significand bits, including those lost during the vshift.\n#if MSHADOW_HALF_ROUND_TO_NEAREST == 1\n      // Rounding may increase the exponent to 1, but that's OK.\n      v.ui += (v.ui & 0x3fff) != 0x1000 || (significand & 0x7ff) ? 0x1000 : 0;\n#endif\n    } else if (v.si <= maxN) {\n      // Handle norms\n#if MSHADOW_HALF_ROUND_TO_NEAREST == 1\n      // Rounding may increase the exponent, possibly creating an inf, but that's OK.\n      v.ui += (v.ui & 0x3fff) != 0x1000 ? 0x1000 : 0;\n#endif\n      v.ui -= expAdjust << fp32FractionBits;\n    } else if (v.si <= infN) {\n      v.si = infN;\n    } else if (v.si < nanN) {\n      v.si = nanN;\n    }\n\n    v.ui >>= shift;\n    return sign | (v.ui & 0x7fff);\n  }\n\n  // Same as above routine, except for addition of volatile keyword\n  MSHADOW_XINLINE uint16_t float2half(const volatile float& value) const volatile {  // NOLINT (*)\n    Bits v;\n    v.f = value;\n    uint32_t sign = v.si & signN;    // grab sign bit\n    v.si ^= sign;                    // clear sign bit from v\n    sign >>= shiftSign;              // logical shift sign to fp16 position\n\n    if (v.si <= maxZ) {\n      // Handle eventual zeros here to ensure vshift will not exceed 32 below.\n      v.ui = 0;\n    } else if (v.si < minN) {\n      // Handle denorms\n      uint32_t exp32 = v.ui >> fp32FractionBits;\n      int32_t exp16 = exp32 - expAdjust;\n      // If exp16 == 0 (just into the denorm range), then significant should be shifted right 1.\n      // Smaller (so negative) exp16 values should result in greater right shifts.\n      uint32_t vshift = 1 - exp16;\n      uint32_t significand = fp32HiddenBit | (v.ui & fp32FractionMask);\n      v.ui = significand >> vshift;\n#if MSHADOW_HALF_ROUND_TO_NEAREST == 1\n      // Rounding may increase the exponent to 1, but that's OK.\n      v.ui += (v.ui & 0x3fff) != 0x1000 || (significand & 0x7ff) ? 0x1000 : 0;\n#endif\n    } else if (v.si <= maxN) {\n      // Handle norms\n#if MSHADOW_HALF_ROUND_TO_NEAREST == 1\n      // Rounding may increase the exponent, possibly creating an inf, but that's OK.\n      v.ui += (v.ui & 0x3fff) != 0x1000 ? 0x1000 : 0;\n#endif\n      v.ui -= expAdjust << fp32FractionBits;\n    } else if (v.si <= infN) {\n      v.si = infN;\n    } else if (v.si < nanN) {\n      v.si = nanN;\n    }\n\n    v.ui >>= shift;\n    return sign | (v.ui & 0x7fff);\n  }\n\n  MSHADOW_XINLINE float half2float(const uint16_t& value) const {\n    Bits v;\n    v.ui = value;\n    int32_t sign = v.si & signC;\n    v.si ^= sign;\n    sign <<= shiftSign;\n    v.si ^= ((v.si + minD) ^ v.si) & -(v.si > subC);\n    v.si ^= ((v.si + maxD) ^ v.si) & -(v.si > maxC);\n    Bits s;\n    s.si = mulC;\n    s.f *= v.si;\n    int32_t mask = -(norC > v.si);\n    v.si <<= shift;\n    v.si ^= (s.si ^ v.si) & mask;\n    v.si |= sign;\n    return v.f;\n  }\n\n  MSHADOW_XINLINE float half2float(const volatile uint16_t& value) const volatile {  // NOLINT(*)\n    Bits v;\n    v.ui = value;\n    int32_t sign = v.si & signC;\n    v.si ^= sign;\n    sign <<= shiftSign;\n    v.si ^= ((v.si + minD) ^ v.si) & -(v.si > subC);\n    v.si ^= ((v.si + maxD) ^ v.si) & -(v.si > maxC);\n    Bits s;\n    s.si = mulC;\n    s.f *= v.si;\n    int32_t mask = -(norC > v.si);\n    v.si <<= shift;\n    v.si ^= (s.si ^ v.si) & mask;\n    v.si |= sign;\n    return v.f;\n  }\n\n  template<typename T>\n  MSHADOW_XINLINE void constructor(const T& value) {\n#if (MSHADOW_CUDA_HALF && defined(__CUDA_ARCH__))\n    cuhalf_ = __float2half(float(value));  // NOLINT(*)\n#elif(MSHADOW_USE_F16C)\n    half_ = _cvtss_sh(static_cast<float>(value), 0);\n#else /* !MSHADOW_CUDA_HALF && !MSHADOW_USE_F16C */\n    half_ = float2half(float(value));  // NOLINT(*)\n#endif /* !MSHADOW_CUDA_HALF && !MSHADOW_USE_F16C */\n  }\n};\n\n/*! \\brief overloaded + operator for half_t */\nMSHADOW_HALF_OPERATOR(half_t, +)\n/*! \\brief overloaded - operator for half_t */\nMSHADOW_HALF_OPERATOR(half_t, -)\n/*! \\brief overloaded * operator for half_t */\nMSHADOW_HALF_OPERATOR(half_t, *)\n/*! \\brief overloaded / operator for half_t */\nMSHADOW_HALF_OPERATOR(half_t, /)\n/*! \\brief overloaded > operator for half_t */\nMSHADOW_HALF_OPERATOR(bool, >)\n/*! \\brief overloaded < operator for half_t */\nMSHADOW_HALF_OPERATOR(bool, <)\n/*! \\brief overloaded >= operator for half_t */\nMSHADOW_HALF_OPERATOR(bool, >=)\n/*! \\brief overloaded <= operator for half_t */\nMSHADOW_HALF_OPERATOR(bool, <=)\n\n#define MSHADOW_HALF_MIN mshadow::half::half_t::Binary(0xFBFF);\n#define MSHADOW_HALF_MAX mshadow::half::half_t::Binary(0x7BFF);\n#define MSHADOW_HALF_SIGN_BIT 0x8000\n#define MSHADOW_HALF_EXPONENT_BITS 0x7c00\n}  // namespace half\n}  // namespace mshadow\n#endif  // MSHADOW_HALF_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/io.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file io.h\n * \\brief definitions of I/O functions for mshadow tensor\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_IO_H_\n#define MSHADOW_IO_H_\n#include \"./tensor.h\"\n\nnamespace mshadow {\nnamespace utils {\n/*!\n * \\brief interface of stream I/O, used to serialize data,\n *   mshadow does not restricted to only this interface in SaveBinary/LoadBinary\n *   mshadow accept all class that implements Read and Write\n */\nclass IStream {\n public:\n  /*!\n   * \\brief read data from stream\n   * \\param ptr pointer to memory buffer\n   * \\param size size of block\n   * \\return usually is the size of data readed\n   */\n  virtual size_t Read(void *ptr, size_t size) = 0;\n  /*!\n   * \\brief write data to stream\n   * \\param ptr pointer to memory buffer\n   * \\param size size of block\n   */\n  virtual void Write(const void *ptr, size_t size) = 0;\n  /*! \\brief virtual destructor */\n  virtual ~IStream(void) {}\n};\n}  // namespace utils\n/*!\n * \\brief CPU/GPU: save a tensor by binary format, for GPU version, a temp Tensor<cpu,dim> storage will be allocated\n * \\param fo output binary stream\n * \\param src source data file\n * \\tparam dim dimension of tensor\n * \\tparam DType type of element in tensor\n * \\tparam TStream type of stream, need to support Read, Write, one example is utils::IStream.\n */\ntemplate<int dim, typename DType, typename TStream>\ninline void SaveBinary(TStream &fo, const Tensor<cpu, dim, DType> &src);  // NOLINT(*)\n/*!\n * \\brief CPU/GPU: save a tensor by binary format, for GPU version, a temp Tensor<cpu,dim> storage will be allocated\n * \\param fo output binary stream\n * \\param src source data file\n * \\tparam dim dimension of tensor\n * \\tparam DType type of element in tensor\n * \\tparam TStream type of stream, need to support Read, Write, one example is utils::IStream.\n */\ntemplate<int dim, typename DType, typename TStream>\ninline void SaveBinary(TStream &fo, const Tensor<gpu, dim, DType> &src); // NOLINT(*)\n/*!\n * \\brief CPU/GPU: load a tensor by binary format, for GPU version, a temp Tensor<cpu,dim> storage will be allocated\n *       if pre_alloc is true , then space in dst is preallocated, and must have same shape of the tensor loaded\n *       if pre_alloc is false, then dst originally does not have space allocated, LoadBinary will allocate space for dst\n * \\param fi output binary stream\n * \\param dst destination file\n * \\param pre_alloc whether space is pre-allocated, if false, space allocation will happen\n * \\tparam dim dimension of tensor\n * \\tparam DType type of element in tensor\n * \\tparam TStream type of stream, need to support Read, Write, one example is utils::IStream.\n */\ntemplate<int dim, typename DType, typename TStream>\ninline void LoadBinary(TStream &fi,  // NOLINT(*)\n                       Tensor<cpu, dim, DType> *dst, bool pre_alloc);\n/*!\n * \\brief CPU/GPU: load a tensor by binary format, for GPU version, a temp Tensor<cpu,dim> storage will be allocated\n *       if pre_alloc is true , then space in dst is preallocated, and must have same shape of the tensor loaded\n *       if pre_alloc is false, then dst originally does not have space allocated, LoadBinary will allocate space for dst\n * \\param fi output binary stream\n * \\param dst destination file\n * \\param pre_alloc whether space is pre-allocated, if false, space allocation will happen\n * \\tparam dim dimension of tensor\n * \\tparam DType type of element in tensor\n * \\tparam TStream type of stream, need to support Read, Write, one example is utils::IStream.\n */\n\ntemplate<int dim, typename DType, typename TStream>\ninline void LoadBinary(TStream &fi, // NOLINT(*)\n                       Tensor<gpu, dim, DType> *dst, bool pre_alloc);\n\n// implementations\ntemplate<int dim, typename DType, typename TStream>\ninline void SaveBinary(TStream &fo, const Tensor<cpu, dim, DType> &src_) { // NOLINT(*)\n  fo.Write(&src_.shape_, sizeof(src_.shape_));\n  Tensor<cpu, 2, DType> src = src_.FlatTo2D();\n  for (index_t i = 0; i < src.size(0); ++i) {\n    fo.Write(src[i].dptr_, sizeof(DType) * src.size(1));\n  }\n}\ntemplate<int dim, typename DType, typename TStream>\ninline void SaveBinary(TStream &fo, const Tensor<gpu, dim, DType> &src) { // NOLINT(*)\n  // copy to CPU, then save\n  Tensor<cpu, dim, DType> tmp(src.shape_);\n  AllocSpace(&tmp);\n  Stream<gpu> stream;\n  Copy(tmp, src, &stream);\n  SaveBinary(fo, tmp);\n  FreeSpace(&tmp);\n}\ntemplate<int dim, typename DType, typename TStream>\ninline void LoadBinary(TStream &fi, // NOLINT(*)\n                       Tensor<cpu, dim, DType> *dst_, bool pre_alloc) {\n  Shape<dim> shape;\n  CHECK_NE(fi.Read(&shape, sizeof(shape)), 0) << \"mshadow::LoadBinary\";\n  if (pre_alloc) {\n    CHECK_EQ(shape, dst_->shape_) << \"LoadBinary, shape do not match pre-allocated shape\";\n  } else {\n    dst_->shape_ = shape; AllocSpace(dst_);\n  }\n  Tensor<cpu, 2, DType> dst = dst_->FlatTo2D();\n  if (dst.size(0) == 0) return;\n  for (index_t i = 0; i < dst.size(0); ++i) {\n    CHECK_NE(fi.Read(dst[i].dptr_, sizeof(DType) * dst.size(1)), 0) << \"mshadow::LoadBinary\";\n  }\n}\ntemplate<int dim, typename DType, typename TStream>\ninline void LoadBinary(TStream &fi, // NOLINT(*)\n                       Tensor<gpu, dim, DType> *dst, bool pre_alloc) {\n  Tensor<cpu, dim, DType> tmp;\n  LoadBinary(fi, &tmp, false);\n  if (pre_alloc) {\n    CHECK_EQ(tmp.shape, dst->shape_) << \"LoadBinary, shape do not match pre-allocated shape\";\n  } else {\n    dst->shape = tmp.shape; AllocSpace(dst);\n  }\n  Stream<gpu> stream;\n  Copy(*dst, tmp, &stream);\n  FreeSpace(&tmp);\n}\n}  // namespace mshadow\n#endif  // MSHADOW_IO_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/packet/plain-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file plain-inl.h\n * \\brief support of plain packet that use the plain datatype.\n */\n#ifndef MSHADOW_PACKET_PLAIN_INL_H_\n#define MSHADOW_PACKET_PLAIN_INL_H_\n\n#include \"../base.h\"\n#include \"../packet-inl.h\"\n\nnamespace mshadow {\nnamespace packet {\ntemplate<typename DType>\nstruct Packet<DType, kPlain> {\n public:\n  /*! \\brief number of float in vector */\n  static constexpr index_t size = 1;\n  /*! \\brief The internal data */\n  DType data_;\n  // enable default copy constructor\n  Packet(void) {}\n  // constructor from the intrinsic type\n  explicit Packet(DType data) : data_(data) {}\n  // create a fill with the target value s\n  MSHADOW_CINLINE static Packet<DType, kPlain> Fill(DType s) {\n    return Packet<DType, kPlain>(s);\n  }\n  // load from address\n  MSHADOW_CINLINE static Packet<DType, kPlain> Load(const DType* src) {\n    return Packet<DType, kPlain>(*src);\n  }\n  // load from address\n  MSHADOW_CINLINE static Packet<DType, kPlain> LoadUnAligned(const DType* src) {\n    return Packet<DType, kPlain>(*src);\n  }\n  // fill it with value s\n  MSHADOW_CINLINE Packet<DType, kPlain>& operator=(DType s) {\n    data_ = s;\n    return *this;\n  }\n  // store data into dst\n  MSHADOW_CINLINE void Store(DType* dst) const {\n    *dst = data_;\n  }\n  // get the sum of all contents\n  MSHADOW_CINLINE DType Sum() const {\n    return data_;\n  }\n};\n\ntemplate<typename DType>\nMSHADOW_CINLINE Packet<DType, kPlain> operator+(const Packet<DType, kPlain>& lhs,\n                                                const Packet<DType, kPlain>& rhs) {\n  return Packet<DType, kPlain>(lhs.data_ + rhs.data_);\n}\n\ntemplate<typename DType>\nMSHADOW_CINLINE Packet<DType, kPlain> operator-(const Packet<DType, kPlain>& lhs,\n                                                const Packet<DType, kPlain>& rhs) {\n  return Packet<DType, kPlain>(lhs.data_ - rhs.data_);\n}\ntemplate<typename DType>\nMSHADOW_CINLINE Packet<DType, kPlain> operator*(const Packet<DType, kPlain>& lhs,\n                                                    const Packet<DType, kPlain>& rhs) {\n  return Packet<DType, kPlain>(lhs.data_ * rhs.data_);\n}\n\ntemplate<typename DType>\nMSHADOW_CINLINE Packet<DType, kPlain> operator/(const Packet<DType, kPlain>& lhs,\n                                                    const Packet<DType, kPlain>& rhs) {\n  return Packet<DType, kPlain>(lhs.data_ / rhs.data_);\n}\n}  // namespace packet\n}  // namespace mshadow\n#endif  // MSHADOW_PACKET_PLAIN_INL_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/packet/sse-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file sse-inl.h\n * \\brief support of sse2 packet optimization of some operations\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_PACKET_SSE_INL_H_\n#define MSHADOW_PACKET_SSE_INL_H_\n\n#include <emmintrin.h>\n#include \"../base.h\"\n#include \"../packet-inl.h\"\n\nnamespace mshadow {\nnamespace packet {\ntemplate<>\nstruct Packet<float, kSSE2> {\n public:\n  /*! \\brief number of float in vector */\n  static constexpr index_t size = 4;\n  /*! \\brief The internal data */\n  __m128 data_;\n  // enable default copy constructor\n  Packet(void) {}\n  // constructor from the intrinsic type\n  explicit Packet(__m128 data) : data_(data) {}\n  // create a fill with the target value s\n  MSHADOW_CINLINE static Packet<float, kSSE2> Fill(float s) {\n    return Packet<float, kSSE2>(_mm_set1_ps(s));\n  }\n  // load from address\n  MSHADOW_CINLINE static Packet<float, kSSE2> Load(const float* src) {\n    return Packet<float, kSSE2>(_mm_load_ps(src));\n  }\n  // load from address\n  MSHADOW_CINLINE static Packet<float, kSSE2> LoadUnAligned(const float* src) {\n    return Packet<float, kSSE2>(_mm_loadu_ps(src));\n  }\n  // fill it with value s\n  MSHADOW_CINLINE Packet<float, kSSE2>& operator=(float s) {\n    data_ = _mm_set1_ps(s);\n    return *this;\n  }\n  // store data into dst\n  MSHADOW_CINLINE void Store(float* dst) const {\n    _mm_store_ps(dst, data_);\n  }\n  // get the sum of all contents\n  MSHADOW_CINLINE float Sum() const {\n    __m128 ans  = _mm_add_ps(data_, _mm_movehl_ps(data_, data_));\n    __m128 rst  = _mm_add_ss(ans, _mm_shuffle_ps(ans, ans, 1));\n#if defined(_MSC_VER) && (_MSC_VER <= 1500) && defined(_WIN64)\n    return rst.m128_f32[0];\n#else\n    float rr = _mm_cvtss_f32(rst);\n    return rr;\n#endif\n  }\n};\n\n\n/*! \\brief vector real type for float */\ntemplate<>\nstruct Packet<double, kSSE2> {\n  /*! \\brief number of float in vector */\n  static constexpr index_t size = 2;\n  // internal data\n  __m128d data_;\n  // constructor\n  Packet(void) {}\n  explicit Packet(__m128d data) : data_(data) {}\n  // create a fill with the target value s\n  MSHADOW_CINLINE static Packet<double, kSSE2> Fill(double s) {\n    return Packet<double, kSSE2>(_mm_set1_pd(s));\n  }\n  // load from address\n  MSHADOW_CINLINE static Packet<double, kSSE2> Load(const double* src) {\n    return Packet<double, kSSE2>(_mm_load_pd(src));\n  }\n  MSHADOW_CINLINE static Packet<double, kSSE2> LoadUnAligned(const double* src) {\n    return Packet<double, kSSE2>(_mm_loadu_pd(src));\n  }\n  // fill it with value s\n  MSHADOW_CINLINE Packet<double, kSSE2>& operator=(double s) {\n    data_ = _mm_set1_pd(s);\n    return *this;\n  }\n  // store data into dst\n  MSHADOW_CINLINE void Store(double* dst) const {\n    _mm_store_pd(dst, data_);\n  }\n  // get sum of all content\n  inline double Sum(void) const {\n    __m128d tmp =  _mm_add_sd(data_, _mm_unpackhi_pd(data_, data_));\n#if defined(_MSC_VER) && (_MSC_VER <= 1500) && defined(_WIN64)\n    return tmp.m128d_f64[0];\n#else\n    double ans = _mm_cvtsd_f64(tmp);\n    return ans;\n#endif\n  }\n};\n\nMSHADOW_CINLINE Packet<float, kSSE2> operator+(const Packet<float, kSSE2>& lhs,\n                                                    const Packet<float, kSSE2>& rhs) {\n  return Packet<float, kSSE2>(_mm_add_ps(lhs.data_, rhs.data_));\n}\n\nMSHADOW_CINLINE Packet<double, kSSE2> operator+(const Packet<double, kSSE2>& lhs,\n                                                     const Packet<double, kSSE2>& rhs) {\n  return Packet<double, kSSE2>(_mm_add_pd(lhs.data_, rhs.data_));\n}\n\nMSHADOW_CINLINE Packet<float, kSSE2> operator-(const Packet<float, kSSE2>& lhs,\n                                                    const Packet<float, kSSE2>& rhs) {\n  return Packet<float, kSSE2>(_mm_sub_ps(lhs.data_, rhs.data_));\n}\n\nMSHADOW_CINLINE Packet<double, kSSE2> operator-(const Packet<double, kSSE2>& lhs,\n                                                     const Packet<double, kSSE2>& rhs) {\n  return Packet<double, kSSE2>(_mm_sub_pd(lhs.data_, rhs.data_));\n}\n\nMSHADOW_CINLINE Packet<float, kSSE2> operator*(const Packet<float, kSSE2>& lhs,\n                                                    const Packet<float, kSSE2>& rhs) {\n  return Packet<float, kSSE2>(_mm_mul_ps(lhs.data_, rhs.data_));\n}\n\nMSHADOW_CINLINE Packet<double, kSSE2> operator*(const Packet<double, kSSE2>& lhs,\n                                                     const Packet<double, kSSE2>& rhs) {\n  return Packet<double, kSSE2>(_mm_mul_pd(lhs.data_, rhs.data_));\n}\n\n\nMSHADOW_CINLINE Packet<float, kSSE2> operator/(const Packet<float, kSSE2>& lhs,\n                                                    const Packet<float, kSSE2>& rhs) {\n  return Packet<float, kSSE2>(_mm_div_ps(lhs.data_, rhs.data_));\n}\n\nMSHADOW_CINLINE Packet<double, kSSE2> operator/(const Packet<double, kSSE2>& lhs,\n                                                     const Packet<double, kSSE2>& rhs) {\n  return Packet<double, kSSE2>(_mm_div_pd(lhs.data_, rhs.data_));\n}\n\n}  // namespace packet\n}  // namespace mshadow\n#endif  // MSHADOW_PACKET_SSE_INL_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/packet-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file packet-inl.h\n * \\brief Generic packet vectorization code\n */\n#ifndef MSHADOW_PACKET_INL_H_\n#define MSHADOW_PACKET_INL_H_\n\n#if defined(__APPLE__) || defined(__FreeBSD__)\n#include <stdlib.h>\n#else\n#include <malloc.h>\n#endif\n#include \"./base.h\"\n#include \"./tensor.h\"\n#include \"./expression.h\"\n\n\nnamespace mshadow {\n/*! \\brief namespace of packet math*/\nnamespace packet {\n\nenum PacketArch {\n  kPlain,\n  kSSE2,\n};\n\n#if MSHADOW_USE_SSE\n#define MSHADOW_DEFAULT_PACKET  ::mshadow::packet::kSSE2\n#else\n#define MSHADOW_DEFAULT_PACKET  ::mshadow::packet::kPlain\n#endif\n\n// whether packet operator is enabled.\n/*!\n * \\brief Generic packet type\n * \\tparam DType The data type of the packet.\n * \\tparam Arch the Arch of the packet.\n */\ntemplate<typename DType, PacketArch Arch = MSHADOW_DEFAULT_PACKET>\nstruct Packet;\n\ntemplate<PacketArch Arch>\nstruct AlignBytes {\n  static const index_t value = 4;\n};\n\n}  // namespace packet\n}  // namespace mshadow\n\nnamespace mshadow {\nnamespace packet {\n/*!\n * \\brief analog to cudaMallocPitch, allocate a aligned space with num_line * lspace cells\n * \\param out_pitch output parameter, the actuall space allocated for each line\n * \\param lspace number of cells required for each line\n * \\param num_line number of lines to be allocated\n */\ninline void* AlignedMallocPitch(size_t *out_pitch,\n                                size_t lspace,\n                                size_t num_line) {\n  const index_t bits = AlignBytes<MSHADOW_DEFAULT_PACKET>::value;\n  const index_t mask = (1 << bits) - 1;\n\n  size_t pitch = ((lspace + mask) >> bits) << bits;\n  *out_pitch = pitch;\n#ifdef _MSC_VER\n  void *res = _aligned_malloc(pitch * num_line, 1 << bits);\n#else\n  void *res;\n  int ret = posix_memalign(&res, 1 << bits, pitch * num_line);\n  CHECK_EQ(ret, 0) << \"AlignedMallocPitch failed\";\n#endif\n  if (res == NULL) {\n    LOG(FATAL) << \"AlignedMallocPitch failed\";\n  }\n#if __GNUC__ >= 6\n#pragma GCC diagnostic ignored \"-Wmaybe-uninitialized\"\n#endif\n  return res;\n#pragma GCC diagnostic pop\n}\n\n/*!\n * \\brief free aligned space\n * \\param ptr pointer to space to be freed\n */\ninline void AlignedFree(void *ptr) {\n#ifdef _MSC_VER\n  _aligned_free(ptr);\n#else\n  free(ptr);\n#endif\n}\n\n/*! \\brief check if a pointer is aligned */\ntemplate<PacketArch Arch>\ninline bool CheckAlign(size_t pitch) {\n  const index_t bits = AlignBytes<Arch>::value;\n  return !(pitch & ((1 << bits) - 1));\n}\n\n/*! \\brief check if a pointer is aligned */\ntemplate<PacketArch Arch>\ninline bool CheckAlign(void *ptr) {\n  return CheckAlign<Arch>(reinterpret_cast<size_t>(ptr));\n}\n\n/*!\n * \\brief get upper bound of aligned index of size\n * \\param size size of the array\n * \\param fsize size of float\n */\ntemplate<typename DType, PacketArch Arch>\ninline index_t UpperAlign(index_t size) {\n  const index_t bits = AlignBytes<MSHADOW_DEFAULT_PACKET>::value;\n  const index_t mask = (1 << bits) - 1;\n  const index_t fsize = sizeof(DType);\n  return (((size * fsize + mask) >> bits) << bits) / fsize;\n}\n\n/*!\n * \\brief get lower bound of aligned index of size\n * \\param size size of the array\n * \\param fsize size of float\n */\ntemplate<typename DType, PacketArch Arch>\ninline index_t LowerAlign(index_t size) {\n  const index_t bits = AlignBytes<MSHADOW_DEFAULT_PACKET>::value;\n  const index_t fsize = sizeof(DType);\n  return (((size * fsize) >> bits) << bits) / fsize;\n}\n\n/*!\n * \\brief generic Packet operator\n * \\tparam OP The operator\n * \\tparam DType The data type\n * \\tparam Arch The architecture.\n */\ntemplate<typename OP, typename DType, PacketArch Arch>\nstruct PacketOp {\n  static const bool kEnabled = false;\n};\n// specialization of operators\ntemplate<typename DType, PacketArch Arch>\nstruct PacketOp<op::plus, DType, Arch> {\n  static const bool kEnabled = true;\n  MSHADOW_CINLINE static Packet<DType, Arch> Map(const Packet<DType, Arch>& lhs,\n                                                   const Packet<DType, Arch>& rhs) {\n    return lhs + rhs;\n  }\n};\ntemplate<typename DType, PacketArch Arch>\nstruct PacketOp<op::minus, DType, Arch> {\n  static const bool kEnabled = true;\n  MSHADOW_CINLINE static Packet<DType, Arch> Map(const Packet<DType, Arch>& lhs,\n                                                  const Packet<DType, Arch>& rhs) {\n    return lhs - rhs;\n  }\n};\ntemplate<typename DType, PacketArch Arch>\nstruct PacketOp<op::mul, DType, Arch> {\n  static const bool kEnabled = true;\n  MSHADOW_CINLINE static Packet<DType, Arch> Map(const Packet<DType, Arch>& lhs,\n                                                  const Packet<DType, Arch>& rhs) {\n    return lhs * rhs;\n  }\n};\ntemplate<typename DType, PacketArch Arch>\nstruct PacketOp<op::div, DType, Arch> {\n  static const bool kEnabled = true;\n  MSHADOW_CINLINE static Packet<DType, Arch> Map(const Packet<DType, Arch>& lhs,\n                                                  const Packet<DType, Arch>& rhs) {\n    return lhs / rhs;\n  }\n};\n\ntemplate<typename DType, PacketArch Arch>\nstruct PacketOp<op::identity, DType, Arch> {\n  static const bool kEnabled = true;\n  MSHADOW_CINLINE static Packet<DType, Arch> Map(const Packet<DType, Arch>& src) {\n    return src;\n  }\n};\n\n\n// savers to do storage\ntemplate<typename SV, typename TFloat, PacketArch Arch>\nstruct Saver{\n  MSHADOW_CINLINE static void Save(TFloat *dst, const Packet<TFloat, Arch>& src) {\n    Packet<TFloat, Arch> lhs = Packet<TFloat, Arch>::Load(dst);\n    Packet<TFloat, Arch> ans = PacketOp<typename SV::OPType, TFloat, Arch>::Map(lhs, src);\n    ans.Store(dst);\n  }\n};\ntemplate<typename TFloat, PacketArch Arch>\nstruct Saver<sv::saveto, TFloat, Arch> {\n  MSHADOW_CINLINE static void Save(TFloat *dst, const Packet<TFloat, Arch>& src) {\n    src.Store(dst);\n  }\n};\n}  // namespace packet\n}  // namespace mshadow\n\n#include \"packet/plain-inl.h\"\n#if MSHADOW_USE_SSE && !defined(__CUDACC__)\n#include \"packet/sse-inl.h\"\n#endif\n\nnamespace mshadow {\nnamespace expr {\n\ntypedef packet::PacketArch PacketArch;\n\n// same as plan, but use packet\ntemplate<typename ExpType, typename DType, PacketArch Arch>\nclass PacketPlan {\n public:\n  /*!\n   * \\brief evaluate the expression at index [y][x],\n   * x will be aligned to Packet<DType, Arch>::Size()\n   */\n  MSHADOW_CINLINE packet::Packet<DType, Arch> EvalPacket(index_t y, index_t x) const;\n  MSHADOW_CINLINE DType Eval(index_t y, index_t x) const;\n};\n\ntemplate <typename Device, int dim, typename DType, PacketArch Arch>\nclass PacketPlan<Tensor<Device, dim, DType>, DType, Arch> {\n public:\n  explicit PacketPlan(const Tensor<Device, dim, DType> &t)\n      :dptr_(t.dptr_), stride_(t.stride_) {}\n  MSHADOW_CINLINE packet::Packet<DType, Arch> EvalPacket(index_t y, index_t x) const {\n    return packet::Packet<DType, Arch>::Load(&dptr_[y * stride_ + x]);\n  }\n  MSHADOW_CINLINE DType Eval(index_t y, index_t x) const {\n    return dptr_[y * stride_ + x];\n  }\n\n private:\n  const DType  *dptr_;\n  index_t stride_;\n};\n\ntemplate<typename DType, PacketArch Arch>\nclass PacketPlan<ScalarExp<DType>, DType, Arch> {\n public:\n  explicit PacketPlan(DType scalar) : scalar_(scalar) {}\n  MSHADOW_CINLINE packet::Packet<DType, Arch> EvalPacket(index_t y, index_t x) const {\n    return packet::Packet<DType, Arch>::Fill(scalar_);\n  }\n  MSHADOW_CINLINE DType Eval(index_t y, index_t x) const {\n    return scalar_;\n  }\n\n private:\n  DType scalar_;\n};\n\ntemplate<typename OP, typename TA, typename TB, int etype, typename DType, PacketArch Arch>\nclass PacketPlan<BinaryMapExp<OP, TA, TB, DType, etype>, DType, Arch> {\n public:\n  PacketPlan(const PacketPlan<TA, DType, Arch> &lhs, const PacketPlan<TB, DType, Arch> &rhs)\n      : lhs_(lhs), rhs_(rhs) {}\n  MSHADOW_CINLINE packet::Packet<DType, Arch> EvalPacket(index_t y, index_t x) const {\n    return packet::PacketOp<OP, DType, Arch>::Map(lhs_.EvalPacket(y, x), rhs_.EvalPacket(y, x));\n  }\n  MSHADOW_CINLINE DType Eval(index_t y, index_t x) const {\n    return OP::Map(lhs_.Eval(y, x), rhs_.Eval(y, x));\n  }\n\n private:\n  PacketPlan<TA, DType, Arch> lhs_;\n  PacketPlan<TB, DType, Arch> rhs_;\n};\n\ntemplate<typename OP, typename TA, int etype, typename DType, PacketArch Arch>\nclass PacketPlan<UnaryMapExp<OP, TA, DType, etype>, DType, Arch> {\n public:\n  PacketPlan(const PacketPlan<TA, DType, Arch> &src) : src_(src) {}\n  MSHADOW_CINLINE packet::Packet<DType> EvalPacket(index_t y, index_t x) const {\n    return packet::PacketOp<OP, DType, Arch>::Map(src_.EvalPacket(y, x));\n  }\n  MSHADOW_CINLINE DType Eval(index_t y, index_t x) const {\n    return OP::Map(src_.Eval(y, x));\n  }\n\n private:\n  PacketPlan<TA, DType, Arch> src_;\n};\n\ntemplate<PacketArch Arch, typename OP, typename TA, typename TB, typename DType, int etype>\ninline PacketPlan<BinaryMapExp<OP, TA, TB, DType, etype>, DType, Arch>\nMakePacketPlan(const BinaryMapExp<OP, TA, TB, DType, etype> &e);\n\ntemplate<PacketArch Arch, typename DType>\ninline PacketPlan<ScalarExp<DType>, DType, Arch> MakePacketPlan(const ScalarExp<DType> &e) {\n  return PacketPlan<ScalarExp<DType>, DType, Arch>(e.scalar_);\n}\ntemplate<PacketArch Arch, typename T, typename DType>\ninline PacketPlan<T, DType, Arch> MakePacketPlan(const RValueExp<T, DType> &e) {\n  return PacketPlan<T, DType, Arch>(e.self());\n}\ntemplate<PacketArch Arch, typename T, int dim, typename DType>\ninline PacketPlan<T, DType, Arch>\nMakePacketPlan(const MakeTensorExp<T, cpu, dim, DType> &e) {\n  return PacketPlan<T, DType, Arch>(e.real_self());\n}\ntemplate<PacketArch Arch, typename OP, typename TA, typename DType, int etype>\ninline PacketPlan<UnaryMapExp<OP, TA, DType, etype>, DType, Arch>\nMakePacketPlan(const UnaryMapExp<OP, TA, DType, etype> &e) {\n  return PacketPlan<UnaryMapExp<OP, TA, DType, etype>, DType, Arch>(MakePacketPlan<Arch>(e.src_));\n}\ntemplate<PacketArch Arch, typename OP, typename TA, typename TB, typename DType, int etype>\ninline PacketPlan<BinaryMapExp<OP, TA, TB, DType, etype>, DType, Arch>\nMakePacketPlan(const BinaryMapExp<OP, TA, TB, DType, etype> &e) {\n  return PacketPlan<BinaryMapExp<OP, TA, TB, DType, etype>,\n                    DType, Arch>(MakePacketPlan<Arch>(e.lhs_), MakePacketPlan<Arch>(e.rhs_));\n}\n\n/*!\n * \\brief static check packet enable\n *\n * \\tparam Device the type of Device\n * \\tparam dim dimension of the tensor\n * \\tparam E expression\n */\ntemplate<typename E, PacketArch Arch>\nstruct PacketCheck{\n  static const bool kPass = false;\n};\ntemplate<PacketArch Arch>\nstruct PacketCheck<float, Arch> {\n  static const bool kPass = true;\n};\ntemplate<PacketArch Arch>\nstruct PacketCheck<double, Arch> {\n  static const bool kPass = true;\n};\ntemplate<typename DType, PacketArch Arch>\nstruct PacketCheck<ScalarExp<DType>, Arch> {\n  static const bool kPass = PacketCheck<DType, Arch>::kPass;\n};\ntemplate<int dim, typename DType, PacketArch Arch>\nstruct PacketCheck<Tensor<cpu, dim, DType>, Arch> {\n  static const bool kPass = PacketCheck<DType, Arch>::kPass;\n};\ntemplate<typename OP, typename TA, typename DType, int etype, PacketArch Arch>\nstruct PacketCheck<UnaryMapExp<OP, TA, DType, etype>, Arch> {\n  static const bool kPass = PacketCheck<TA, Arch>::kPass &&\n      packet::PacketOp<OP, DType, Arch>::kEnabled;\n};\ntemplate<typename OP, typename TA, typename TB, typename DType, int etype, PacketArch Arch>\nstruct PacketCheck< BinaryMapExp<OP, TA, TB, DType, etype>, Arch> {\n  static const bool kPass = packet::PacketOp<OP, DType, Arch>::kEnabled &&\n      PacketCheck<TA, Arch>::kPass && PacketCheck<TB, Arch>::kPass;\n};\n//----------------------------------------------------\n// Check if data is aligned and allow packet operation\n//----------------------------------------------------\ntemplate<int dim, typename E, PacketArch Arch>\nstruct PacketAlignCheck {\n  inline static bool Check(const E &exp) {\n    return false;\n  }\n};\ntemplate<int dim, typename DType, PacketArch Arch>\nstruct PacketAlignCheck<dim, ScalarExp<DType>, Arch> {\n  inline static bool Check(const ScalarExp<DType> &exp) {\n    return true;\n  }\n};\ntemplate<int dim, typename DType, PacketArch Arch>\nstruct PacketAlignCheck<dim, Tensor<cpu, dim, DType>, Arch> {\n  inline static bool Check(const Tensor<cpu, dim, DType> &t) {\n    return packet::CheckAlign<Arch>(t.dptr_) &&\n        packet::CheckAlign<Arch>(t.stride_ * sizeof(DType));\n  }\n};\ntemplate<int dim, typename OP, typename TA, typename DType, int etype, PacketArch Arch>\nstruct PacketAlignCheck<dim, UnaryMapExp<OP, TA, DType, etype>, Arch> {\n  inline static bool Check(const UnaryMapExp<OP, TA, DType, etype> &t) {\n    return PacketAlignCheck<dim, TA, Arch>::Check(t.src_);\n  }\n};\ntemplate<int dim, typename OP, typename TA, typename TB,\n         typename DType, int etype, PacketArch Arch>\nstruct PacketAlignCheck<dim, BinaryMapExp<OP, TA, TB, DType, etype>, Arch> {\n  inline static bool Check(const BinaryMapExp<OP, TA, TB, DType, etype> &t) {\n    return PacketAlignCheck<dim, TA, Arch>::Check(t.lhs_) &&\n        PacketAlignCheck<dim, TB, Arch>::Check(t.rhs_);\n  }\n};\n\n/*!\n * \\brief use PacketPlan to compute result\n */\ntemplate<typename SV, typename E, int dim, typename DType, PacketArch Arch>\ninline void MapPacketPlan(Tensor<cpu, dim, DType> _dst,\n                          const expr::PacketPlan<E, DType, Arch>& plan) {\n  Tensor<cpu, 2, DType> dst = _dst.FlatTo2D();\n  const index_t xlen = packet::LowerAlign<DType, Arch>(dst.size(1));\n  const size_t packetSize = packet::Packet<DType, Arch>::size;\n#ifndef __CUDACC__\n  #pragma omp parallel for\n#endif\n  for (openmp_index_t y = 0; y < dst.size(0); ++y) {\n    for (index_t x = 0; x < xlen; x += packetSize) {\n      packet::Saver<SV, DType, Arch>::Save(&dst[y][x], plan.EvalPacket(y, x));\n    }\n    for (index_t x = xlen; x < dst.size(1); ++x) {\n      SV::Save(dst[y][x], plan.Eval(y, x));\n    }\n  }\n}\n}  // namespace expr\n}  // namespace mshadow\n#endif  // MSHADOW_PACKET_INL_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/random.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n *  \\file random.h\n *  \\brief Random inline functions for tensor.\n *  \\author Bing Xu, Tianqi Chen\n *   Based on curand|MKL|stdlib\n */\n#ifndef MSHADOW_RANDOM_H_\n#define MSHADOW_RANDOM_H_\n\n#include <cstdlib>\n#include <algorithm>\n#include <random>\n#include \"./base.h\"\n#include \"./tensor.h\"\n#include \"./tensor_container.h\"\n#include <random>\n\n\nnamespace mshadow {\n/*!\n * \\brief random number generator\n * \\tparam Device the device of random number generator\n * \\tparam DType the target data type of random number can be float for double\n */\ntemplate<typename Device, typename DType MSHADOW_DEFAULT_DTYPE>\nclass Random {};\n\n/*! \\brief CPU random number generator */\ntemplate<typename DType>\nclass Random<cpu, DType> {\n public:\n  /*!\n   * \\brief constructor of random engine\n   * \\param seed random number seed\n   */\n  explicit Random(int seed) {\n    this->Seed(seed);\n    buffer_.Resize(Shape1(kRandBufferSize));\n  }\n  ~Random(void) {\n  }\n  /*!\n   * \\brief seed random number generator using this seed\n   * \\param seed seed of prng\n   */\n  inline void Seed(int seed) {\n    rnd_engine_.seed(seed);\n    this->rseed_ = static_cast<unsigned>(seed);\n  }\n  /*!\n   * \\brief get random seed used in random generator\n   * \\return seed in unsigned\n   */\n  inline unsigned GetSeed() const {\n    return rseed_;\n  }\n  /*!\n   * \\brief set the stream of computation\n   * \\param stream computation stream\n   */\n  inline void set_stream(Stream<cpu> *stream) {\n  }\n\n  /*!\n   * \\brief get some random integer\n   * \\return integer as unsigned\n   */\n  inline unsigned GetRandInt() {\n    return rnd_engine_();\n  }\n\n  /*!\n   * \\brief get a set of random integers\n   */\n  inline void GetRandInt(const Tensor<cpu, 1, unsigned>& dst) {\n    std::generate_n(dst.dptr_, dst.size(0), [&](){ return rnd_engine_(); });\n  }\n\n  /*!\n   * \\brief generate data from a distribution\n   * \\param dst destination\n   * \\tparam dim dimension of tensor\n   * \\param sampler sampler of the distribution\n   */\n  template<int dim, class Sampler>\n  inline void SampleDistribution(Tensor<cpu, dim, DType> *dst, Sampler sampler) {\n    if (dst->CheckContiguous()) {\n      std::generate_n(dst->dptr_, dst->shape_.Size(), sampler);\n    } else {\n      Tensor<cpu, 2, DType> mat = dst->FlatTo2D();\n      for (index_t i = 0; i < mat.size(0); ++i) {\n        std::generate_n(mat[i].dptr_, mat.size(1), sampler);\n      }\n    }\n  }\n\n  /*!\n   * \\brief generate data from uniform [a,b)\n   * \\param dst destination\n   * \\param a lower bound of uniform\n   * \\param b upper bound of uniform\n   * \\tparam dim dimension of tensor\n   */\n  template<int dim, typename PType>\n  inline void SampleUniform(Tensor<cpu, dim, DType> *dst,\n                            PType a = 0.0f , PType b = 1.0f ) {\n    // Ensure that half_t is handled correctly.\n    typedef typename std::conditional<std::is_floating_point<DType>::value,\n                                      DType, double>::type FType;\n    typedef typename std::conditional<std::is_integral<DType>::value,\n                                      std::uniform_int_distribution<DType>,\n                                      std::uniform_real_distribution<FType>>::type GType;\n    GType dist_uniform(a, b);\n    SampleDistribution(dst, [&](){ return dist_uniform(rnd_engine_);});\n  }\n\n  /*!\n   * \\brief generate data from standard gaussian\n   * \\param dst destination\n   * \\param mu mean variable\n   * \\param sigma standard deviation\n   * \\tparam dim dimension of tensor\n   */\n  template<int dim, typename PType>\n  inline void SampleGaussian(Tensor<cpu, dim, DType> *dst,\n                             PType mu = 0.0f, PType sigma = 1.0f ) {\n    if (sigma <= 0) {\n      *dst = mu; return;\n    }\n    typedef typename std::conditional<std::is_floating_point<DType>::value,\n                                      DType, double>::type GType;\n    std::normal_distribution<GType> dist_normal(mu, sigma);\n    SampleDistribution(dst, [&](){ return dist_normal(rnd_engine_);});\n  }\n\n  /*!\n   * \\brief generate data from a gamma distribution\n   * \\param dst destination\n   * \\param alpha (shape) parameter\n   * \\param beta (scale) parameter\n   * \\tparam dim dimension of tensor\n   */\n  template<int dim, typename PType>\n  inline void SampleGamma(Tensor<cpu, dim, DType> *dst,\n                          PType alpha, PType beta) {\n    typedef typename std::conditional<std::is_floating_point<DType>::value,\n                                      DType, double>::type GType;\n    std::gamma_distribution<GType> dist_gamma(alpha, beta);\n    SampleDistribution(dst, [&](){ return dist_gamma(rnd_engine_);});\n  }\n\n  /*!\n   * \\brief generate data from an exponential distribution\n   * \\param dst destination\n   * \\param lambda parameter (rate) of the distribution\n   * \\tparam dim dimension of tensor\n   */\n  template<int dim, typename PType>\n  inline void SampleExponential(Tensor<cpu, dim, DType> *dst, PType lambda ) {\n    typedef typename std::conditional<std::is_floating_point<DType>::value,\n                                      DType, double>::type GType;\n    std::exponential_distribution<GType> dist_exp(lambda);\n    SampleDistribution(dst, [&](){ return dist_exp(rnd_engine_);});\n  }\n\n  /*!\n   * \\brief generate data from a poisson distribution\n   * \\param dst destination\n   * \\param lambda parameter (rate) of the distribution\n   * \\tparam dim dimension of tensor\n   */\n  template<int dim, typename PType>\n  inline void SamplePoisson(Tensor<cpu, dim, DType> *dst, PType lambda) {\n    typedef typename std::conditional<std::is_integral<DType>::value, DType, int>::type GType;\n    std::poisson_distribution<GType> dist_poisson(lambda);\n    SampleDistribution(dst, [&](){ return static_cast<DType>(dist_poisson(rnd_engine_));});\n  }\n\n  /*!\n   * \\brief generate data from a negative binomial distribution\n   * \\param dst destination\n   * \\param k limit on number of failures\n   * \\param p success probability\n   * \\tparam dim dimension of tensor\n   */\n  template<int dim, typename PType1, typename PType2>\n  inline void SampleNegativeBinomial(Tensor<cpu, dim, DType> *dst, PType1 k, PType2 p) {\n    typedef typename std::conditional<std::is_integral<DType>::value, DType, int>::type GType;\n    std::negative_binomial_distribution<GType> dist_negbinomial(k, p);\n    SampleDistribution(dst, [&](){ return static_cast<DType>(dist_negbinomial(rnd_engine_));});\n  }\n\n  /*!\n   * \\brief generate data from a generalized negative binomial distribution\n   * \\param dst destination\n   * \\param mu parameter (mean) of the distribution\n   * \\param alpha parameter (over dispersion) of the distribution\n   *   (for alpha=0 this gives a Poisson)\n   * \\tparam dim dimension of tensor\n   */\n  template<int dim, typename PType>\n  inline void SampleGeneralizedNegativeBinomial(Tensor<cpu, dim, DType> *dst,\n                                                PType mu, PType alpha) {\n    if (alpha == PType(0)) {\n      SamplePoisson(dst, mu);  // limit of Poisson\n    } else {\n      PType r(PType(1) / alpha);\n      PType beta = mu * alpha;\n      std::gamma_distribution<> dist_gamma(r, beta);\n      typedef typename std::conditional<std::is_integral<DType>::value, DType, int>::type GType;\n      SampleDistribution(dst,\n        [&](){ std::poisson_distribution<GType> dist_poisson(dist_gamma(rnd_engine_));\n               return static_cast<DType>(dist_poisson(rnd_engine_));});\n    }\n  }\n\n  /*!\n   * \\brief return a temporal expression storing standard gaussian random variables\n   *        the temporal tensor is only valid before next call of gaussian or uniform\n   *        can be used as part of expression\n   *  Caution: this means expression such as A = gaussian(s1) * gaussian(s2) will give invalid result,\n   *           since second call of gaussian(s2) makes gaussian(s1) invalid\n   *           A = gaussian(s1)*B+C; is correct; use one gaussian/uniform in each expression\n   * \\param shape shape of the tensor\n   * \\return a temporal expression storing standard gaussian random variables\n   * \\tparam dim dimension of tensor\n   */\n  template<int dim>\n  inline expr::ReshapeExp<Tensor<cpu, 1, DType>, DType, dim, 1>\n  gaussian(Shape<dim> shape) {\n    buffer_.Resize(Shape1(shape.Size()));\n    this->SampleGaussian(&buffer_, 0.0f, 1.0f);\n    return expr::reshape(buffer_, shape);\n  }\n  /*!\n   * \\brief return a temporal expression storing standard uniform [0,1)\n   *        the temporal tensor is only valid before next call of gaussian or uniform\n   *        can be used as part of expression\n   *  Caution: this means expression such as A = uniform(s1) * uniform(s2) will give invalid result,\n   *           since second call of gaussian(s2) makes gaussian(s1) invalid\n   *           A = gaussian(s1)*B+C; is correct; use one gaussian/uniform in each expression\n   * \\param shape shape of the tensor\n   * \\return a temporal expression storing standard uniform [0,1)\n   * \\tparam dim dimension of tensor\n   */\n  template<int dim>\n  inline expr::ReshapeExp<Tensor<cpu, 1, DType>, DType, dim, 1>\n  uniform(Shape<dim> shape) {\n    buffer_.Resize(Shape1(shape.Size()));\n    this->SampleUniform(&buffer_, 0.0f, 1.0f);\n    return expr::reshape(buffer_, shape);\n  }\n\n  std::mt19937 &GetRndEngine() {\n    return rnd_engine_;\n  }\n\n private:\n  /*! \\brief use c++11 random engine. */\n  std::mt19937 rnd_engine_;\n  /*! \\brief random number seed used in random engine */\n  unsigned rseed_;\n  /*! \\brief temporal space used to store random numbers */\n  TensorContainer<cpu, 1, DType> buffer_;\n};  // class Random<cpu, DType>\n\n// only allow GPU PRNG when cuda is enabled\n#if MSHADOW_USE_CUDA\n/*! \\brief GPU random number generator */\ntemplate<typename DType>\nclass Random<gpu, DType> {\n public:\n  /*!\n   * \\brief constructor of random engine\n   * \\param seed random number seed\n   */\n  explicit Random(int seed) : gen_(NULL) {\n    this->Seed(seed);\n    buffer_.Resize(Shape1(kRandBufferSize));\n  }\n  ~Random(void) MSHADOW_THROW_EXCEPTION {\n    DeleteGenerator();\n  }\n  /*!\n   * \\brief set the stream of computation\n   * \\param stream computation stream\n   */\n  inline void set_stream(Stream<gpu> *stream) {\n    curandStatus_t status;\n    status = curandSetStream(gen_, Stream<gpu>::GetStream(stream));\n\n    CHECK_EQ(status, CURAND_STATUS_SUCCESS) << \"set_stream CURAND failed\";\n  }\n  /*!\n   * \\brief seed random number generator using this seed\n   * \\param seed seed of prng\n   */\n  inline void Seed(int seed) {\n    // Create a new rng, either initially or if the RNG type can't reset its offset.\n    if (gen_ == NULL || (curandSetGeneratorOffset(gen_, 0ULL) != CURAND_STATUS_SUCCESS))\n      CreateGenerator();\n    // Now set the seed.\n    curandStatus_t status;\n    status = curandSetPseudoRandomGeneratorSeed(gen_, static_cast<uint64_t>(seed));\n    CHECK_EQ(status, CURAND_STATUS_SUCCESS) << \"Set CURAND seed failed.\";\n  }\n  /*!\n   * \\brief get a set of random integers\n   */\n  inline void GetRandInt(const Tensor<gpu, 1, unsigned>& dst) {\n    curandStatus_t status;\n    status = curandGenerate(gen_, dst.dptr_, dst.size(0));\n    CHECK_EQ(status, CURAND_STATUS_SUCCESS) << \"CURAND Gen rand ints failed.\"\n                                            << \" size = \" << dst.size(0);\n  }\n  /*!\n   * \\brief generate data from uniform [a,b)\n   * \\param dst destination\n   * \\param a lower bound of uniform\n   * \\param b upper bound of uniform\n   * \\tparam dim dimension of tensor\n   */\n  template<int dim>\n  inline void SampleUniform(Tensor<gpu, dim, DType> *dst,\n                            DType a = 0.0f, DType b = 1.0f);\n\n  /*!\n   * \\brief generate data from standard gaussian\n   * \\param dst destination\n   * \\param mu mean variable\n   * \\param sigma standard deviation\n   * \\tparam dim dimension of tensor\n   */\n  template<int dim>\n  inline void SampleGaussian(Tensor<gpu, dim, DType> *dst,\n                             DType mu = 0.0f, DType sigma = 1.0f);\n  /*!\n   * \\brief return a temporal expression storing standard gaussian random variables\n   *        the temporal tensor is only valid before next call of gaussian or uniform\n   *        can be used as part of expression\n   *  Caution: this means expression such as A = gaussian(s1) * gaussian(s2) will give invalid result,\n   *           since second call of gaussian(s2) makes gaussian(s1) invalid\n   *           A = gaussian(s1)*B+C; is correct; use one gaussian/uniform in each expression\n   * \\param shape shape of the tensor\n   * \\param mu mean\n   * \\param sigma variance\n   * \\return a temporal expression storing standard gaussian random variables\n   * \\tparam dim dimension of tensor\n   */\n  template<int dim>\n  inline expr::ReshapeExp<Tensor<gpu, 1, DType>, DType, dim, 1>\n  gaussian(Shape<dim> shape, DType mu = 0.0f, DType sigma = 1.0f);\n  /*!\n   * \\brief return a temporal expression storing standard uniform [0,1)\n   *        the temporal tensor is only valid before next call of gaussian or uniform\n   *        can be used as part of expression\n   *  Caution: this means expression such as A = gaussian(s1) * gaussian(s2) will give invalid result,\n   *           since second call of gaussian(s2) makes gaussian(s1) invalid\n   *           A = gaussian(s1)*B+C; is correct; use one gaussian/uniform in each expression\n   * \\param shape shape of the tensor\n   * \\return a temporal expression storing standard uniform [0,1)\n   * \\tparam dim dimension of tensor\n   */\n  template<int dim>\n  inline expr::ReshapeExp<Tensor<gpu, 1, DType>, DType, dim, 1>\n  uniform(Shape<dim> shape);\n\n private:\n  inline void GenGaussian(float *dptr, size_t size, float mu, float sigma) {\n    curandStatus_t status;\n    status = curandGenerateNormal(gen_, dptr, size, mu, sigma);\n    CHECK_EQ(status, CURAND_STATUS_SUCCESS) << \"CURAND Gen Normal float failed.\"\n                                            << \" size = \" << size\n                                            << \",mu = \" << mu\n                                            << \",sigma = \" << sigma;\n  }\n  inline void GenGaussian(double *dptr, size_t size, double mu, double sigma) {\n    curandStatus_t status;\n    status = curandGenerateNormalDouble(gen_, dptr, size, mu, sigma);\n    CHECK_EQ(status, CURAND_STATUS_SUCCESS) << \"CURAND Gen Normal double failed.\"\n                                            << \" size = \" << size\n                                            << \",mu = \" << mu\n                                            << \",sigma = \" << sigma;\n  }\n  inline void GenUniform(float *dptr, size_t size) {\n    curandStatus_t status;\n    status = curandGenerateUniform(gen_, dptr, size);\n    CHECK_EQ(status, CURAND_STATUS_SUCCESS) << \"CURAND Gen Uniform float failed.\"\n                                            << \" size = \" << size;\n  }\n  inline void GenUniform(double *dptr, size_t size) {\n    curandStatus_t status;\n    status = curandGenerateUniformDouble(gen_, dptr, size);\n    CHECK_EQ(status, CURAND_STATUS_SUCCESS) << \"CURAND Gen Uniform double failed.\"\n                                            << \" size = \" << size;\n  }\n  inline void CreateGenerator() {\n    if (gen_ != NULL)\n      DeleteGenerator();\n    curandStatus_t status;\n    status = curandCreateGenerator(&gen_, CURAND_RNG_PSEUDO_DEFAULT);\n    CHECK_EQ(status, CURAND_STATUS_SUCCESS) << \"Cannot create CURAND Generator\";\n  }\n  inline void DeleteGenerator() {\n    if (gen_ != NULL) {\n      curandStatus_t status;\n      status = curandDestroyGenerator(gen_);\n      CHECK_EQ(status, CURAND_STATUS_SUCCESS) << \"Destory CURAND Gen failed\";\n      gen_ = NULL;\n    }\n  }\n  /*! \\brief random number generator */\n  curandGenerator_t gen_;\n  /*! \\brief templ buffer */\n  TensorContainer<gpu, 1, DType> buffer_;\n};  // class Random<gpu, DType>\n#endif  // MSHADOW_USE_CUDA\n\n#ifdef __CUDACC__\n// implementations that depends on cuda kernels\ntemplate<typename DType>\ntemplate<int dim>\ninline void Random<gpu, DType>::SampleUniform(\n    Tensor<gpu, dim, DType> *dst, DType a, DType b) {\n  if (a == 0.0f && b == 1.0f) {\n    if (dst->CheckContiguous()) {\n      this->GenUniform(dst->dptr_, dst->shape_.Size());\n    } else {\n      *dst = this->uniform(dst->shape_);\n    }\n  } else {\n    *dst = this->uniform(dst->shape_) * (b - a) + a;\n  }\n}\ntemplate<typename DType>\ntemplate<int dim>\ninline void Random<gpu, DType>::SampleGaussian(\n    Tensor<gpu, dim, DType> *dst, DType mu, DType sigma) {\n  // We need to check whether the shape size is even since CuRand supports only normal distribution\n  // generation of even number of elements.\n  if (dst->CheckContiguous() && (dst->shape_.Size() % 2 == 0)) {\n    this->GenGaussian(dst->dptr_, dst->shape_.Size(), mu, sigma);\n  } else {\n    *dst = this->gaussian(dst->shape_, mu, sigma);\n  }\n}\n\ntemplate<typename DType>\ntemplate<int dim>\ninline expr::ReshapeExp<Tensor<gpu, 1, DType>, DType, dim, 1>\nRandom<gpu, DType>::gaussian(Shape<dim> shape, DType mu, DType sigma) {\n  size_t aligned_sz = ((shape.Size() + 1UL) >> 1) << 1;\n  // allocate alligned size\n  buffer_.Resize(Shape1(aligned_sz));\n  buffer_.Resize(Shape1(shape.Size()));\n  this->GenGaussian(buffer_.dptr_, aligned_sz, mu, sigma);\n  return expr::reshape(buffer_, shape);\n}\n\ntemplate<typename DType>\ntemplate<int dim>\ninline expr::ReshapeExp<Tensor<gpu, 1, DType>, DType, dim, 1>\nRandom<gpu, DType>::uniform(Shape<dim> shape) {\n  buffer_.Resize(Shape1(shape.Size()));\n  this->GenUniform(buffer_.dptr_, buffer_.size(0));\n  return expr::reshape(buffer_, shape);\n}\n#endif  // __CUDACC__\n}  // namespace mshadow\n#endif  // MSHADOW_RANDOM_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/stream_gpu-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file stream_gpu-inl.h\n * \\brief implementation of GPU code\n * \\author Bing Xu, Tianqi Chen\n */\n#ifndef MSHADOW_STREAM_GPU_INL_H_\n#define MSHADOW_STREAM_GPU_INL_H_\n#include <memory>\n#include \"./base.h\"\n#include \"./tensor.h\"\n#include \"dmlc/logging.h\"\n\nnamespace mshadow {\n#if MSHADOW_USE_CUDA == 1\n// Stream alocation\n// actual implementation of GPU stream in CUDA\ntemplate<>\nstruct Stream<gpu> {\n  /*! \\brief handle state */\n  enum HandleState {\n    NoHandle = 0,\n    OwnHandle = 1,\n  };\n  /*! \\brief cudaStream */\n  cudaStream_t stream_;\n  /*! \\brief cublas handle */\n  cublasHandle_t blas_handle_;\n  /*! \\brief cusolver handle */\n  #if MSHADOW_USE_CUSOLVER == 1\n  cusolverDnHandle_t solver_handle_;\n  #endif\n  /*! \\brief cudnn handle */\n  #if MSHADOW_USE_CUDNN == 1\n  cudnnHandle_t dnn_handle_;\n  #endif\n  /*! \\brief cutensor handle */\n  #if MSHADOW_USE_CUTENSOR== 1\n  cutensorHandle_t cutensor_handle_;\n  #endif\n  /*! \\brief cublas handle ownership */\n  HandleState blas_handle_ownership_;\n  /*! \\brief cusolver handle ownership */\n  HandleState solver_handle_ownership_;\n  /*! \\brief cudnn handle ownership */\n  HandleState dnn_handle_ownership_;\n  /*! \\brief cutensor handle ownership */\n  HandleState cutensor_handle_ownership_;\n  void* cutensor_cachelines_ = nullptr;\n  /*! \\brief cudaDeviceProp */\n  cudaDeviceProp prop;\n  /*! \\brief dev id */\n  int dev_id;\n\n  Stream(void)\n    : stream_(0)\n      , blas_handle_(0)\n#if MSHADOW_USE_CUDNN == 1\n      , dnn_handle_(0)\n#endif\n      //, cutensor_handle_()\n      , blas_handle_ownership_(NoHandle)\n      , solver_handle_ownership_(NoHandle)\n      , dnn_handle_ownership_(NoHandle)\n      , cutensor_handle_ownership_(NoHandle)\n      , cutensor_cachelines_(nullptr){}\n  /*!\n   * \\brief wait for all the computation associated\n   *  with this stream to complete\n   */\n  inline void Wait(void) {\n    MSHADOW_CUDA_CALL(cudaStreamSynchronize(stream_));\n  }\n  /*!\n   * \\brief query whether the the stream is idle\n   * \\return true if the stream is idle and all the job have been completed\n   */\n  inline bool CheckIdle(void) {\n    cudaError_t err = cudaStreamQuery(stream_);\n    if (err == cudaSuccess) return true;\n    if (err == cudaErrorNotReady) return false;\n    LOG(FATAL) << cudaGetErrorString(err);\n    return false;\n  }\n  /*!\n   * \\brief returns actual cudaStream_t given an input GPU stream pointer\n   * \\param stream pointer to GPU stream\n   */\n  inline static cudaStream_t GetStream(Stream<gpu> *stream) {\n    if (stream == NULL) {\n#if MSHADOW_FORCE_STREAM\n      LOG(FATAL) << \"Default GPU stream was used when MSHADOW_FORCE_STREAM was on\";\n#endif\n      return 0;\n    } else {\n      return stream->stream_;\n    }\n  }\n  /*!\n   * \\brief return actual cublasHandle\n   * \\param pointer to GPU stream\n   */\n  inline static cublasHandle_t GetBlasHandle(Stream<gpu> *stream) {\n    if (stream == NULL) {\n      return 0;\n    } else {\n      CHECK_NE(stream->blas_handle_ownership_, NoHandle)\n        << \"No handle exist in source stream\";\n      return stream->blas_handle_;\n    }\n  }\n  /*! \\brief Destory cublas handle if own it */\n  inline void DestroyBlasHandle() {\n    if (blas_handle_ownership_ == OwnHandle) {\n      cublasStatus_t err = cublasDestroy(blas_handle_);\n      blas_handle_ownership_ = NoHandle;\n      CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Destory cublas handle failed\";\n    }\n  }\n  /*! \\brief Destory original blas handle and create a new one */\n  inline void CreateBlasHandle() {\n    this->DestroyBlasHandle();\n    cublasStatus_t err = cublasCreate(&blas_handle_);\n    blas_handle_ownership_ = OwnHandle;\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Create cublas handle failed\";\n    err = cublasSetStream(blas_handle_, stream_);\n    CHECK_EQ(err, CUBLAS_STATUS_SUCCESS) << \"Setting cublas stream failed\";\n  }\n#if MSHADOW_USE_CUSOLVER == 1\n  inline static cusolverDnHandle_t GetSolverHandle(Stream<gpu> *stream) {\n    if (stream == NULL) {\n      return 0;\n    } else {\n      CHECK_NE(stream->solver_handle_ownership_, NoHandle) << \"No handle exist in source stream\";\n      return stream->solver_handle_;\n    }\n  }\n#endif\n  inline void DestroySolverHandle() {\n#if MSHADOW_USE_CUSOLVER == 1\n    if (solver_handle_ownership_ == OwnHandle) {\n      cusolverStatus_t err = cusolverDnDestroy(solver_handle_);\n      CHECK_EQ(err, CUSOLVER_STATUS_SUCCESS) << \"Destory cusolver handle failed\";\n    }\n#endif\n  }\n  inline void CreateSolverHandle() {\n#if MSHADOW_USE_CUSOLVER == 1\n    this->DestroySolverHandle();\n    cusolverStatus_t err = cusolverDnCreate(&solver_handle_);\n    CHECK_EQ(err, CUSOLVER_STATUS_SUCCESS) << \"Create cusolver handle failed\";\n    err = cusolverDnSetStream(solver_handle_, stream_);\n    CHECK_EQ(err, CUSOLVER_STATUS_SUCCESS) << \"Setting cusolver stream failed\";\n    this->solver_handle_ownership_ = OwnHandle;\n#endif\n  }\n// #if MSHADOW_USE_CUDNN && defined(__CUDACC__)\n#if MSHADOW_USE_CUDNN == 1\n  inline static cudnnHandle_t GetDnnHandle(Stream<gpu> *stream) {\n    if (stream == NULL) {\n      return 0;\n    } else {\n      CHECK_NE(stream->dnn_handle_ownership_, NoHandle) << \"No handle exist in source stream\";\n      return stream->dnn_handle_;\n    }\n  }\n#endif\n  inline void DestroyDnnHandle() {\n// #if MSHADOW_USE_CUDNN && defined(__CUDACC__)\n#if MSHADOW_USE_CUDNN == 1\n    if (dnn_handle_ownership_ == OwnHandle) {\n      cudnnStatus_t err = cudnnDestroy(dnn_handle_);\n      this->dnn_handle_ownership_ = NoHandle;\n      CHECK_EQ(err, CUDNN_STATUS_SUCCESS) << cudnnGetErrorString(err);\n    }\n#endif\n  }\n  inline void CreateDnnHandle() {\n// #if MSHADOW_USE_CUDNN == 1 && defined(__CUDACC__)\n#if MSHADOW_USE_CUDNN == 1\n    this->DestroyDnnHandle();\n    cudnnStatus_t err = cudnnCreate(&dnn_handle_);\n    CHECK_EQ(err, CUDNN_STATUS_SUCCESS) << cudnnGetErrorString(err);\n    // At this point, we have the resource which may need to be freed\n    this->dnn_handle_ownership_ = OwnHandle;\n    err = cudnnSetStream(dnn_handle_, stream_);\n    CHECK_EQ(err, CUDNN_STATUS_SUCCESS) << cudnnGetErrorString(err);\n#endif\n  }\n  inline void DestroyCuTensorHandle() {\n#if MSHADOW_USE_CUTENSOR == 1\n    if (cutensor_handle_ownership_ == OwnHandle) {\n      // not destroy method available\n      if (cutensor_cachelines_ != nullptr) {\n        cutensorStatus_t err;\n        const char* cacheFilename = getenv(\"MXNET_CUTENSOR_CACHEFILE\");\n        if (cacheFilename != nullptr) {\n          err = cutensorHandleWriteCacheToFile(&cutensor_handle_, cacheFilename);\n          CHECK_EQ(err, CUTENSOR_STATUS_SUCCESS) << cutensorGetErrorString(err);\n        }\n        err = cutensorHandleDetachPlanCachelines(&cutensor_handle_);\n        CHECK_EQ(err, CUTENSOR_STATUS_SUCCESS) << cutensorGetErrorString(err);\n        free(cutensor_cachelines_);\n        cutensor_cachelines_ = nullptr;\n      }\n      this->cutensor_handle_ownership_ = NoHandle;\n    }\n#endif\n  }\n  inline void CreateCuTensorHandle() {\n#if MSHADOW_USE_CUTENSOR == 1\n    this->DestroyCuTensorHandle();\n    cutensorStatus_t err = cutensorInit(&cutensor_handle_);\n    CHECK_EQ(err, CUTENSOR_STATUS_SUCCESS) << cutensorGetErrorString(err);\n    const char* cacheFilename = getenv(\"MXNET_CUTENSOR_CACHEFILE\");\n    if (cacheFilename != nullptr) {\n      constexpr int32_t numCachelines = 1024;\n      size_t sizeCache = numCachelines * sizeof(cutensorPlanCacheline_t);\n      cutensor_cachelines_ = malloc(sizeCache);\n      err = cutensorHandleAttachPlanCachelines(&cutensor_handle_, (cutensorPlanCacheline_t*) cutensor_cachelines_, numCachelines);\n      CHECK_EQ(err, CUTENSOR_STATUS_SUCCESS) << cutensorGetErrorString(err);\n\n      uint32_t numCachelinesRead = 0;\n      cutensorStatus_t status = cutensorHandleReadCacheFromFile(&cutensor_handle_, cacheFilename, &numCachelinesRead);\n      if (status == CUTENSOR_STATUS_IO_ERROR) {\n        printf(\"File (%s) doesn't seem to exist.\\n\", cacheFilename);\n      } else if (status == CUTENSOR_STATUS_INSUFFICIENT_WORKSPACE) {\n        printf(\"Cannot read cache: Please attach at least %d cachelines to the handle.\\n\", numCachelinesRead);\n      }\n    }\n    // At this point, we have the resource which may need to be freed\n    this->cutensor_handle_ownership_ = OwnHandle;\n#endif\n  }\n};\ntemplate<>\ninline void DeleteStream<gpu>(Stream<gpu> *stream) {\n  if (stream) {\n    stream->DestroyCuTensorHandle();\n    MSHADOW_CUDA_CALL(cudaStreamDestroy(stream->stream_));\n    stream->DestroyBlasHandle();\n    stream->DestroySolverHandle();\n    stream->DestroyDnnHandle();\n    delete stream;\n  }\n}\ntemplate<>\ninline Stream<gpu> *NewStream<gpu>(bool create_blas_handle,\n                                   bool create_dnn_handle,\n                                   int dev_id) {\n  // RAII on Cuda exception\n  struct StreamDeleter { void operator()(Stream<gpu> *ptr) const { DeleteStream<gpu>(ptr); } };\n  std::unique_ptr<Stream<gpu>, StreamDeleter> st(new Stream<gpu>());\n  MSHADOW_CUDA_CALL(cudaStreamCreate(&st->stream_));\n  if (create_blas_handle) {\n    st->CreateBlasHandle();\n    st->CreateSolverHandle();\n  }\n  if (create_dnn_handle) {\n    st->CreateDnnHandle();\n  }\n#if MSHADOW_USE_CUTENSOR == 1\n  st->CreateCuTensorHandle();\n#endif\n  st->dev_id = dev_id;\n  if (dev_id != -1) {\n    MSHADOW_CUDA_CALL(cudaGetDeviceProperties(&st->prop, dev_id));\n  }\n  return st.release();\n}\n#endif\n}  // namespace mshadow\n#endif  // MSHADOW_STREAM_GPU_INL_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/tensor.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file tensor.h\n * \\brief header file of tensor data structure and functions\n *  This lib requires explicit memory allocation and de-allocation\n *  all the data structure Tensor<cpu,1>, Tensor<gpu,1> are like handles(pointers),\n *  no memory allocation is happening during calculation\n *\n *  For STL style tensor, see tensor_container.h\n * \\author Bing Xu, Tianqi Chen\n */\n#ifndef MSHADOW_TENSOR_H_\n#define MSHADOW_TENSOR_H_\n#include <string>\n#include <iostream>\n#include \"./base.h\"\n#include \"./expression.h\"\n\nnamespace mshadow {\n/*! \\brief device name CPU */\nstruct cpu {\n  /*! \\brief whether this device is CPU or not */\n  static const bool kDevCPU = true;\n  /*! \\brief device flag number, identifies this device */\n  static const int kDevMask = 1 << 0;\n};\n/*! \\brief device name GPU */\nstruct gpu {\n  /*! \\brief whether this device is CPU or not */\n  static const bool kDevCPU = false;\n  /*! \\brief device flag number, identifies this device */\n  static const int kDevMask = 1 << 1;\n};\n\ntemplate <typename xpu>\nstruct LapackIndex {\n    using IndexT = lapack_index_t;\n};\n\ntemplate <>\nstruct LapackIndex <gpu> {\n    using IndexT = int;\n};\n\ntemplate<int ndim>\nstruct Shape;\n\n/*!\n * \\brief allow string printing of the shape\n * \\param os the output stream\n * \\param shape the shape\n * \\return the ostream\n */\ntemplate<int ndim>\ninline std::ostream &operator<<(std::ostream &os, const Shape<ndim> &shape); // NOLINT(*)\n\n/*!\n * \\brief shape of a tensor\n * \\tparam dimension dimension of tensor\n */\ntemplate<int dimension>\nstruct Shape {\n  /*! \\brief dimension of current shape */\n  static const int kDimension = dimension;\n  /*! \\brief dimension of current shape minus one */\n  static const int kSubdim = dimension - 1;\n  /*! \\brief storing the dimension information */\n  index_t shape_[kDimension];\n  /*! \\brief default constructor, do nothing */\n  MSHADOW_XINLINE Shape(void) {}\n  /*! \\brief constuctor */\n  MSHADOW_XINLINE Shape(const Shape<kDimension> &s) {\n    #pragma unroll\n    for (int i = 0; i < kDimension; ++i) {\n      this->shape_[i] = s[i];\n    }\n  }\n  /*!\n   * \\brief get corresponding index\n   * \\param idx dimension index\n   * \\return the corresponding dimension size\n   */\n  MSHADOW_XINLINE index_t &operator[](int idx) {\n    return shape_[idx];\n  }\n  /*!\n   * \\brief get corresponding index\n   * \\param idx dimension index\n   * \\return the corresponding dimension size\n   */\n  MSHADOW_XINLINE const index_t &operator[](int idx) const {\n#pragma GCC diagnostic push\n#pragma GCC diagnostic ignored \"-Warray-bounds\"\n    return shape_[idx];\n#pragma GCC diagnostic pop\n  }\n  /*!\n   * \\return whether two shape equals\n   * \\param s the shape to compare against\n   */\n  MSHADOW_XINLINE bool operator==(const Shape<kDimension> &s) const {\n    #pragma unroll\n    for (int i = 0; i < kDimension; ++i) {\n      if (s.shape_[i] != this->shape_[i]) return false;\n    }\n    return true;\n  }\n  /*!\n   * \\return whether two shape not equal\n   * \\param s the shape to compare against\n   */\n  MSHADOW_XINLINE bool operator!=(const Shape<kDimension> &s) const {\n    return !(*this == s);\n  }\n  /*!\n   * flatten the tensor, return a 1D shape\n   * \\return the flat 1d shape\n   */\n  MSHADOW_XINLINE Shape<1> FlatTo1D(void) const {\n    Shape<1> s;\n    s[0] = this->Size();\n    return s;\n  }\n  /*!\n   * flatten the higher dimension to second dimension, return a 2D shape\n   * \\return the flat 2d shape\n   */\n  MSHADOW_XINLINE Shape<2> FlatTo2D(void) const {\n    Shape<2> s;\n    s.shape_[1] = this->shape_[kDimension - 1];\n    index_t ymax = 1;\n    #pragma unroll\n    for (int i = 0; i < kDimension - 1; ++i) {\n      ymax *= this->shape_[i];\n    }\n    s.shape_[0] = ymax;\n    return s;\n  }\n  /*! \\return number of valid elements */\n  MSHADOW_XINLINE index_t Size(void) const {\n    index_t size = this->shape_[0];\n    #pragma unroll\n    for (int i = 1; i < kDimension; ++i) {\n      size *= this->shape_[i];\n    }\n    return size;\n  }\n  /*!\n   * \\return product shape in [dimstart,dimend)\n   * \\param dimstart start dimension\n   * \\param dimend end dimension\n   */\n  MSHADOW_XINLINE index_t ProdShape(int dimstart, int dimend) const {\n    index_t num = 1;\n    #pragma unroll\n    for (int i = dimstart; i < dimend; ++i) {\n      num *= this->shape_[i];\n    }\n    return num;\n  }\n  /*!\n   * \\brief get subshape that takes off largest dimension\nv   * \\return subshape\n   */\n  MSHADOW_XINLINE Shape<kSubdim> SubShape(void) const {\n    Shape<kSubdim> s;\n    // for cuda\n    #pragma unroll\n    for (int i = 0; i < kSubdim; ++i) {\n      s.shape_[i] = this->shape_[i + 1];\n    }\n    return s;\n  }\n  /*!\n   * \\brief slice the shape from start to end\n   * \\tparam dimstart start dimension\n   * \\tparam dimend end dimension\n   * \\return the sliced shape\n   */\n  template<int dimstart, int dimend>\n  MSHADOW_XINLINE Shape<dimend - dimstart> Slice(void) const {\n    Shape<dimend - dimstart> s;\n    #pragma unroll\n    for (int i = dimstart; i < dimend; ++i) {\n      s[i - dimstart] = this->shape_[i];\n    }\n    return s;\n  }\n  //! \\cond Doxygen_Suppress\n  template<int dim>\n  friend std::ostream &operator<<(std::ostream &os, const Shape<dim> &shape); // NOLINT(*)\n  //! \\endcond\n};  // Shape\n//------------------------------------------------\n// useful construction functions to generate shape\n//-------------------------------------------------\n/*!\n * \\brief construct a one dimension shape, stride will equal s0\n * \\param s0 size of dimension 0\n * \\return the shape construction\n */\nMSHADOW_XINLINE Shape<1> Shape1(index_t s0) {\n  Shape<1> s; s[0] = s0;\n  return s;\n}\n/*!\n * \\brief construct a two dimension shape, stride will equal s0\n * \\param s0 size of dimension 0\n * \\param s1 size of dimension 1\n * \\return the shape construction\n */\nMSHADOW_XINLINE Shape<2> Shape2(index_t s0, index_t s1) {\n  Shape<2> s; s[0] = s0; s[1] = s1;\n  return s;\n}\n/*!\n * \\brief construct a three dimension shape, stride will equal s0\n * \\param s0 size of dimension 0\n * \\param s1 size of dimension 1\n * \\param s2 size of dimension 2\n * \\return the shape construction\n */\nMSHADOW_XINLINE Shape<3> Shape3(index_t s0, index_t s1, index_t s2) {\n  Shape<3> s;\n  s[0] = s0; s[1] = s1; s[2] = s2;\n  return s;\n}\n/*!\n * \\brief construct a four dimension shape, stride will equal s0\n * \\param s0 size of dimension 0\n * \\param s1 size of dimension 1\n * \\param s2 size of dimension 2\n * \\param s3 size of dimension 3\n * \\return the shape construction\n */\nMSHADOW_XINLINE Shape<4> Shape4(index_t s0, index_t s1,\n                                index_t s2, index_t s3) {\n  Shape<4> s;\n  s[0] = s0; s[1] = s1; s[2] = s2; s[3] = s3;\n  return s;\n}\n/*!\n* \\brief construct a five dimension shape, stride will equal s0\n* \\param s0 size of dimension 0\n* \\param s1 size of dimension 1\n* \\param s2 size of dimension 2\n* \\param s3 size of dimension 3\n* \\param s4 size of dimension 4\n* \\return the shape construction\n*/\nMSHADOW_XINLINE Shape<5> Shape5(index_t s0, index_t s1, index_t s2,\n                                index_t s3, index_t s4) {\n  Shape<5> s;\n  s[0] = s0; s[1] = s1; s[2] = s2; s[3] = s3; s[4] = s4;\n  return s;\n}\n\n/*!\n* \\brief Convert shape in src_layout to shape in dst_layout\n* \\param src original shape\n* \\param src_layout layout of original shape\n* \\param dst_layout target layout\n* \\return shape in target layout\n*/\ninline Shape<3> ConvertLayout(const Shape<3>& src, int src_layout, int dst_layout) {\n  Shape<3> dst;\n  switch (src_layout) {\n  case kNCW:\n    dst = src;\n    break;\n  case kNWC:\n    dst[0] = src[0];\n    dst[1] = src[2];\n    dst[2] = src[1];\n    break;\n  default:\n    LOG(FATAL) << \"Invalid layout for 3d shape \" << src_layout;\n  }\n  switch (dst_layout) {\n  case kNCW:\n    return dst;\n  case kNWC:\n    {\n      index_t tmp = dst[1];\n      dst[1] = dst[2];\n      dst[2] = tmp;\n    }\n    break;\n  default:\n    LOG(FATAL) << \"Invalid layout for 3d shape \" << src_layout;\n  }\n  return dst;\n}\n\n/*!\n* \\brief Convert shape in src_layout to shape in dst_layout\n* \\param src original shape\n* \\param src_layout layout of original shape\n* \\param dst_layout target layout\n* \\return shape in target layout\n*/\ninline Shape<4> ConvertLayout(const Shape<4>& src, int src_layout, int dst_layout) {\n  Shape<4> dst;\n  switch (src_layout) {\n  case kNCHW:\n    dst = src;\n    break;\n  case kNHWC:\n    dst[0] = src[0];\n    dst[2] = src[1];\n    dst[3] = src[2];\n    dst[1] = src[3];\n    break;\n  default:\n    LOG(FATAL) << \"Invalid layout for 4d shape \" << src_layout;\n    dst = src;  // fixes compiler warning\n  }\n  Shape<4> dst2;\n  switch (dst_layout) {\n  case kNCHW:\n    return dst;\n  case kNHWC:\n    dst2[0] = dst[0];\n    dst2[1] = dst[2];\n    dst2[2] = dst[3];\n    dst2[3] = dst[1];\n    break;\n  default:\n    LOG(FATAL) << \"Invalid layout for 4d shape \" << src_layout;\n    dst2 = src;  // fixes compiler warning\n  }\n  return dst2;\n}\n\n/*!\n* \\brief Convert shape in src_layout to shape in dst_layout\n* \\param src original shape\n* \\param src_layout layout of original shape\n* \\param dst_layout target layout\n* \\return shape in target layout\n*/\ninline Shape<5> ConvertLayout(const Shape<5>& src, int src_layout, int dst_layout) {\n  Shape<5> dst;\n  switch (src_layout) {\n  case kNCDHW:\n    dst = src;\n    break;\n  case kNDHWC:\n    dst[0] = src[0];\n    dst[2] = src[1];\n    dst[3] = src[2];\n    dst[4] = src[3];\n    dst[1] = src[4];\n    break;\n  default:\n    LOG(FATAL) << \"Invalid layout for 5d shape \" << src_layout;\n  }\n  Shape<5> dst2;\n  switch (dst_layout) {\n  case kNCDHW:\n    return dst;\n  case kNDHWC:\n    dst2[0] = dst[0];\n    dst2[1] = dst[2];\n    dst2[2] = dst[3];\n    dst2[3] = dst[4];\n    dst2[4] = dst[1];\n    break;\n  default:\n    LOG(FATAL) << \"Invalid layout for 5d shape \" << src_layout;\n  }\n  return dst2;\n}\n\n/*!\n * \\brief returns axes of transpose operation\n *        that needs to be performed between src layout and dst\n * \\param src_layout input layout\n * \\param dst_layout output layout\n * \\return vector of required type describing axes of a transpose operation\n */\ntemplate <typename dim_t>\ninline std::vector<dim_t> getTranspAxes(const LayoutFlag src_layout, const LayoutFlag dst_layout) {\n  auto apply = [](const std::vector<dim_t>& v, const std::vector<dim_t>& op) {\n    CHECK_EQ(v.size(), op.size()) << \"Layout ndims does not match\";\n    std::vector<dim_t> ret(v.size());\n    for (size_t i = 0; i < v.size(); i++) {\n      ret[i] = v[op[i]];\n    }\n    return ret;\n  };\n  std::vector<dim_t> axes;\n  // transpose from `case` to ND?H?WC\n  switch (src_layout) {\n    case kUNKNOWN:\n      LOG(FATAL) << \"Unknown source layout\";\n      break;\n    case kNHWC:\n      axes = std::vector<dim_t>({0, 1, 2, 3});\n      break;\n    case kNCHW:\n      axes = std::vector<dim_t>({0, 2, 3, 1});\n      break;\n    case kCHWN:\n      axes = std::vector<dim_t>({3, 1, 2, 0});\n      break;\n    case kNWC:\n      axes = std::vector<dim_t>({0, 1, 2});\n      break;\n    case kNCW:\n      axes = std::vector<dim_t>({0, 2, 1});\n      break;\n    case kCWN:\n      axes = std::vector<dim_t>({2, 1, 0});\n      break;\n    case kNDHWC:\n      axes = std::vector<dim_t>({0, 1, 2, 3, 4});\n      break;\n    case kNCDHW:\n      axes = std::vector<dim_t>({0, 2, 3, 4, 1});\n      break;\n    case kCDHWN:\n      axes = std::vector<dim_t>({4, 1, 2, 3, 0});\n      break;\n    default:\n      LOG(FATAL) << \"Invalid source layout \" << src_layout;\n  }\n  // transpose from ND?H?WC to `case`\n  switch (dst_layout) {\n    case kUNKNOWN:\n      LOG(FATAL) << \"Unknown destination layout\";\n      break;\n    case kNHWC:\n      axes = apply(axes, {0, 1, 2, 3});\n      break;\n    case kNCHW:\n      axes = apply(axes, {0, 3, 1, 2});\n      break;\n    case kCHWN:\n      axes = apply(axes, {3, 1, 2, 0});\n      break;\n    case kNWC:\n      axes = apply(axes, {0, 1, 2});\n      break;\n    case kNCW:\n      axes = apply(axes, {0, 2, 1});\n      break;\n    case kCWN:\n      axes = apply(axes, {2, 1, 0});\n      break;\n    case kNDHWC:\n      axes = apply(axes, {0, 1, 2, 3, 4});\n      break;\n    case kNCDHW:\n      axes = apply(axes, {0, 4, 1, 2, 3});\n      break;\n    case kCDHWN:\n      axes = apply(axes, {4, 1, 2, 3, 0});\n      break;\n    default:\n      LOG(FATAL) << \"Invalid destination layout \" << src_layout;\n  }\n  return axes;\n}\n\n/*!\n * \\brief computaion stream structure, used for asynchronous computations\n */\ntemplate<typename Device>\nstruct Stream {\n  // this is only a dummy implementation for CPU\n  // for GPU, the actual implementation will be specialized in tensor_gpu-inl.h\n  /*!\n   * \\brief wait for all the computations associated\n   *  with this stream to complete\n   */\n  inline void Wait(void) {}\n  /*!\n   * \\brief query whether the the stream is idle\n   * \\return true if the stream is idle and all the jobs have been completed\n   */\n  inline bool CheckIdle(void) {\n    return true;\n  }\n  /*! \\brief create a blas handle */\n  inline void CreateBlasHandle() {}\n};\n/*!\n * \\brief Tensor RValue, this is the super type of all kinds of possible tensors\n * \\tparam Container the tensor type\n * \\tparam Device which device the tensor is on\n * \\tparam dimension dimension of the tensor\n * \\tparam DType the type of elements in the tensor\n */\ntemplate<typename Container, typename Device, int dimension, typename DType>\nstruct TRValue: public expr::RValueExp<Container, DType> {\n};\n// more compact template\n/*!\n * \\brief general tensor\n * \\tparam Device which device the tensor is on\n * \\tparam dimension dimension of the tensor\n * \\tparam DType the type of elements in the tensor\n */\ntemplate<typename Device, int dimension,\n         typename DType MSHADOW_DEFAULT_DTYPE>\nstruct Tensor: public TRValue<Tensor<Device, dimension, DType>,\n                              Device, dimension, DType> {\n public:\n  //--------------------------------\n  // struct memembers\n  //--------------------------------\n  /*! \\brief whether current type lies in cpu */\n  static const bool kDevCPU = Device::kDevCPU;\n  /*! \\brief dimension of subtype */\n  static const int  kSubdim = dimension - 1;\n  //--------------------------------\n  // struct memembers\n  //--------------------------------\n  /*! \\brief pointer to the data */\n  DType *dptr_ = nullptr;\n  /*! \\brief shape of the tensor */\n  Shape<dimension> shape_;\n  /*!\n   * \\brief storing the stride information in x dimension\n   *    this is used to deal with pitch allocation in gpu or sse(align x dimension to 64bit) for efficiency\n   */\n  index_t stride_;\n  /*!\n   * \\brief stream where the computation lies\n   * stream is a device dependency concept where each computation\n   */\n  Stream<Device> *stream_;\n  //--------------------------------\n  // functions\n  //--------------------------------\n  /*! \\brief default constructor */\n  MSHADOW_XINLINE Tensor(void) : stream_(NULL) {}\n  /*! \\brief constructor from shape  */\n  MSHADOW_XINLINE Tensor(const Shape<dimension> &shape)\n      : shape_(shape), stream_(NULL) {}\n  /*! \\brief constructor from data pointer and shape, without stride */\n  MSHADOW_XINLINE Tensor(DType *dptr, const Shape<dimension> &shape)\n      : dptr_(dptr), shape_(shape), stride_(shape[kSubdim]), stream_(NULL) {}\n  /*! \\brief constructor from data pointer and shape, without stride */\n  MSHADOW_XINLINE Tensor(DType *dptr, const Shape<dimension> &shape,\n                         Stream<Device> *stream)\n    : dptr_(dptr), shape_(shape), stride_(shape[kSubdim]), stream_(stream) {}\n  /*! \\brief constructor from data pointer and shape  */\n  MSHADOW_XINLINE Tensor(DType *dptr,\n                         const Shape<dimension> &shape,\n                         index_t stride, Stream<Device> *stream)\n      : dptr_(dptr), shape_(shape), stride_(stride), stream_(stream) {}\n  /*!\n   * \\brief set the stream to do computation of current tensor\n   * \\param stream the computation stream\n   */\n  inline void set_stream(Stream<Device> *stream) {\n    this->stream_ = stream;\n  }\n  /*!\n   * \\return memory cost of the tensor, including the aligned x dimension\n   * \\tparam startdim the starting dimension\n   */\n  template<int startdim>\n  MSHADOW_XINLINE index_t MemSize(void) const {\n    index_t memsz = this->stride_;\n    #pragma unroll\n    for (int i = startdim; i < kSubdim; ++i) {\n      memsz *= this->shape_[i];\n    }\n    return memsz;\n  }\n  /*!\n   * \\return whether the tensor's memory is continuous\n   * x dimension same as stride\n   */\n  MSHADOW_XINLINE bool CheckContiguous(void) const {\n    return this->shape_[dimension - 1] == stride_;\n  }\n  /*!\n   * \\return memory cost of the tensor, including the aligned x dimension\n   */\n  MSHADOW_XINLINE index_t MSize(void) const {\n    return this->MemSize<0>();\n  }\n  /*!\n   * \\brief return size of i-th dimension, start counting from highest dimension\n   * \\param idx the dimension count from the highest dimensin\n   * \\return the size\n   */\n  MSHADOW_XINLINE index_t size(int idx) const {\n    return shape_[idx];\n  }\n  /*!\n   * \\brief flatten the tensor to 1 dimension\n   * \\return tensor after flatten\n   */\n  MSHADOW_XINLINE Tensor<Device, 1, DType> FlatTo1D(void) const {\n    return Tensor<Device, 1, DType>(dptr_, shape_.FlatTo1D(), stride_, stream_);\n  }\n  /*!\n   * \\brief flatten the tensor to 2 dimension, collapse the higher dimensions together\n   * \\return tensor after flatten\n   */\n  MSHADOW_XINLINE Tensor<Device, 2, DType> FlatTo2D(void) const {\n    return Tensor<Device, 2, DType>(dptr_, shape_.FlatTo2D(), stride_, stream_);\n  }\n  /*!\n   * \\brief get a element of dimension - 1\n   * \\param idx index\n   * \\return the result tensor\n   */\n  MSHADOW_XINLINE Tensor<Device, kSubdim, DType> operator[](index_t idx) const {\n    return Tensor<Device, kSubdim, DType>(dptr_ + this->MemSize<1>() * idx,\n                                          shape_.SubShape(), stride_, stream_);\n  }\n  /*!\n   * \\brief slice the tensor in highest dimension [begin,end)\n   * \\param begin begin position of slice\n   * \\param end end position of slice\n   * \\return tensor after slice\n   */\n  MSHADOW_XINLINE Tensor<Device, dimension, DType>\n  Slice(index_t begin, index_t end) const {\n    Shape<dimension> s = this->shape_;\n    s[0] = end - begin;\n    return Tensor<Device, dimension, DType>(dptr_ + this->MemSize<1>() * begin,\n                                            s, stride_, stream_);\n  }\n  /*!\\brief implement the assignment of same type */\n  inline Tensor<Device, dimension, DType> &\n  operator=(const Tensor<Device, dimension, DType> &exp) {\n    dptr_ = exp.dptr_;\n    shape_ = exp.shape_;\n    stride_ = exp.stride_;\n    stream_ = exp.stream_;\n    return *this;\n  }\n  /*!\\brief functions to fit expression template */\n  template<typename E, int etype>\n  inline Tensor<Device, dimension, DType> &\n  operator=(const expr::Exp<E, DType, etype> &exp) {\n    return this->__assign(exp);\n  }\n  /*!\\brief functions to fit expression template */\n  inline Tensor<Device, dimension, DType> &operator=(const DType &exp) {\n    return this->__assign(exp);\n  }\n};\n/*\n *  respecialized class Tensor1D, thei is due to different implementation in operator[]\n */\ntemplate<typename Device, typename DType>\nstruct Tensor<Device, 1, DType>:\n      public TRValue<Tensor<Device, 1, DType>, Device, 1, DType> {\n public:\n  DType *dptr_;\n  Shape<1> shape_;\n  index_t stride_;\n  Stream<Device> *stream_;\n  // constructor\n  MSHADOW_XINLINE Tensor(void) : stream_(NULL) {}\n  MSHADOW_XINLINE Tensor(const Shape<1> &shape)\n      : shape_(shape), stream_(NULL) {}\n  MSHADOW_XINLINE Tensor(DType *dptr, Shape<1> shape)\n      : dptr_(dptr), shape_(shape), stride_(shape[0]), stream_(NULL) {}\n  MSHADOW_XINLINE Tensor(DType *dptr, Shape<1> shape, Stream<Device> *stream)\n      : dptr_(dptr), shape_(shape), stride_(shape[0]), stream_(stream) {}\n  MSHADOW_XINLINE Tensor(DType *dptr, Shape<1> shape,\n                         index_t stride, Stream<Device> *stream)\n      : dptr_(dptr), shape_(shape), stride_(stride), stream_(stream) {}\n  inline void set_stream(Stream<Device> *stream) {\n    this->stream_ = stream;\n  }\n  MSHADOW_XINLINE Tensor<Device, 1, DType> FlatTo1D(void) const {\n    return *this;\n  }\n  MSHADOW_XINLINE Tensor<Device, 2, DType> FlatTo2D(void) const {\n    return Tensor<Device, 2, DType>(dptr_, shape_.FlatTo2D(), stride_, stream_);\n  }\n  MSHADOW_XINLINE Tensor<Device, 1, DType> Slice(index_t begin, index_t end) const {\n    Shape<1> s;\n    s[0] = end  - begin;\n    return Tensor<Device, 1, DType>(dptr_ + begin, s, s[0], stream_);\n  }\n  MSHADOW_XINLINE bool CheckContiguous(void) const {\n    return true;\n  }\n  MSHADOW_XINLINE index_t MSize(void) const {\n    return shape_[0];\n  }\n  MSHADOW_XINLINE index_t size(index_t i) const {\n    return shape_[0];\n  }\n  MSHADOW_XINLINE DType &operator[](index_t idx) {\n    return dptr_[idx];\n  }\n  MSHADOW_XINLINE const DType &operator[](index_t idx) const {\n    return dptr_[idx];\n  }\n  /*!\\brief implement the assignment of same type */\n  inline Tensor<Device, 1, DType> &\n  operator=(const Tensor<Device, 1, DType> &exp) {\n    dptr_ = exp.dptr_;\n    shape_ = exp.shape_;\n    stride_ = exp.stride_;\n    stream_ = exp.stream_;\n    return *this;\n  }\n  template<typename E, int etype>\n  inline Tensor<Device, 1, DType> &\n  operator=(const expr::Exp<E, DType, etype> &exp) {\n    return this->__assign(exp);\n  }\n  inline Tensor<Device, 1, DType> &operator=(const DType &exp) {\n    return this->__assign(exp);\n  }\n};\n//------------------------\n// Function Declarations\n//-----------------------\n/*!\n * \\brief initialize tensor engine, used to call intialization functions of dependent libs\n *        this function should be called before all GPU tensor operations,\n *        for using tensors in CPU, this call is actually not needed\n * \\param device_id GPU device id to be choosed\n * \\tparam Device the device type\n */\ntemplate<typename Device>\ninline void InitTensorEngine(int device_id = 0);\n/*!\n * \\brief Shutdown tensor engine on current device\n *     this function should be called after all GPU tensor operations,\n *     for using tensors in CPU, this call is actually not needed\n * \\tparam Device the device type\n */\ntemplate<typename Device>\ninline void ShutdownTensorEngine(void);\n/*!\n * \\brief set the device of current thread to work on\n * \\param devid the device id\n * \\tparam Device the device type\n */\ntemplate<typename Device>\ninline void SetDevice(int devid);\n/*!\n * \\brief create a new stream from system\n * \\param create_blas_handle whether create blas & cusolver handle in stream\n * \\param create_dnn_handle whether create cudnn handle in stream\n * \\param dev_id device id\n * \\return a pointer to the created stream\n * \\tparam Device the device type\n */\ntemplate<typename Device>\ninline Stream<Device> *NewStream(bool create_blas_handle,\n                                 bool create_dnn_handle,\n                                 int dev_id = -1);\n/*! \\brief default behavior: create cublas handle\n *  \\param dev_id device id\n *  \\return a pointer to the created stream\n */\ntemplate<typename Device>\ninline Stream<Device> *NewStream(int dev_id) {\n  return NewStream<Device>(true, false, dev_id);\n}\n/*!\n * \\brief delete the computing stream\n * \\param stream the stream parameter to be deleted\n */\ntemplate<typename Device>\ninline void DeleteStream(Stream<Device> *stream);\n/*!\n * \\brief CPU/CPU: allocate space for CTensor, according to the shape in the obj\n *        this function is responsible to set the stride_ in each obj.shape\n * \\param obj the tensor object, with shape specified\n * \\param pad whether padding dimension 0, to make last dimension aligned,\n *            padding may help improve efficiency of matrix multiplications\n *            if true, will allocate space with stride_ that may not equals shape[0]\n *            if false, will allocate continuous space\n * \\tparam dim specify the dim of tensor\n * \\tparam DType type of element in tensor\n */\ntemplate<int dim, typename DType>\ninline void AllocSpace(Tensor<cpu, dim, DType> *obj,\n                       bool pad = MSHADOW_ALLOC_PAD);\n/*!\n * \\brief CPU/CPU: allocate space for CTensor, according to the shape in the obj\n *        this function is responsible to set the stride_ in each obj.shape\n * \\param obj the tensor object, with shape specified\n * \\param pad whether padding dimension 0, to make last dimension aligned,\n *            padding may help improve efficiency of matrix multiplications\n *            if true, will allocate space with stride_ that may not equals shape[0]\n *            if false, will allocate continuous space\n * \\tparam dim specify the dim of tensor\n * \\tparam DType type of element in tensor\n */\ntemplate<int dim, typename DType>\ninline void AllocSpace(Tensor<gpu, dim, DType> *obj,\n                       bool pad = MSHADOW_ALLOC_PAD);\n/*!\n * \\brief CPU/GPU: free the space of tensor, will set obj.dptr to NULL\n * \\param obj the tensor object\n * \\tparam dim specify the dim of tensor\n * \\tparam DType type of element in tensor\n */\ntemplate<int dim, typename DType>\ninline void FreeSpace(Tensor<cpu, dim, DType> *obj);\n/*!\n * \\brief CPU/GPU: free the space of tensor, will set obj.dptr to NULL\n * \\param obj the tensor object\n * \\tparam dim specify the dim of tensor\n * \\tparam DType type of element in tensor\n */\ntemplate<int dim, typename DType>\ninline void FreeSpace(Tensor<gpu, dim, DType> *obj);\n/*!\n * \\brief CPU/GPU: short cut to allocate and initialize a Tensor\n * \\param shape: shape of tensor\n * \\param initv: initialization value\n * \\param pad : padding option\n * \\param stream : stream of tensor\n * \\tparam Device device of tensor\n * \\tparam DType type of element in tensor\n * \\tparam dim dimention of tensor\n * \\return a new allocated tensor\n * \\sa AllocSpace\n */\ntemplate<typename Device, typename DType, int dim>\ninline Tensor<Device, dim, DType> NewTensor(const Shape<dim> &shape,\n                                            DType initv,\n                                            bool pad = MSHADOW_ALLOC_PAD,\n                                            Stream<Device> *stream = NULL);\n/*!\n * \\brief copy data from one tensor to another, with same shape\n * \\param dst target tensor\n * \\param src source tensor\n * \\param stream the stream, when specified, the copy can exhibit asynchronize behavior\n * \\tparam dim specify the dim of tensor\n * \\tparam DType type of element in tensor\n */\ntemplate<int dim, typename DType>\ninline void Copy(Tensor<cpu, dim, DType> dst,\n                 const Tensor<cpu, dim, DType> &src,\n                 Stream<cpu> *stream = NULL);\n/*!\n * \\brief copy data from one tensor to another, with same shape\n * \\param dst target tensor\n * \\param src source tensor\n * \\param stream the stream, when specified, the copy can exhibit asynchronize behavior\n * \\tparam dim specify the dim of tensor\n * \\tparam DType type of element in tensor\n */\ntemplate<int dim, typename DType>\ninline void Copy(Tensor<cpu, dim, DType> dst,\n                 const Tensor<gpu, dim, DType> &src,\n                 Stream<gpu> *stream = NULL);\n/*!\n * \\brief copy data from one tensor to another, with same shape\n * \\param dst target tensor\n * \\param src source tensor\n * \\param stream the stream, when specified, the copy can exhibit asynchronize behavior\n * \\tparam dim specify the dim of tensor\n * \\tparam DType type of element in tensor\n */\ntemplate<int dim, typename DType>\ninline void Copy(Tensor<gpu, dim, DType> dst,\n                 const Tensor<cpu, dim, DType> &src,\n                 Stream<gpu> *stream = NULL);\n/*!\n * \\brief copy data from one tensor to another, with same shape\n * \\param dst target tensor\n * \\param src source tensor\n * \\param stream the stream, when specified, the copy can exhibit asynchronize behavior\n * \\tparam dim specify the dim of tensor\n * \\tparam DType type of element in tensor\n */\ntemplate<int dim, typename DType>\ninline void Copy(Tensor<gpu, dim, DType> dst,\n                 const Tensor<gpu, dim, DType> &src,\n                 Stream<gpu> *stream = NULL);\n/*!\n * \\brief CPU/GPU: normalize softmax: dst[i][j] = exp(energy[i][j]) /(sum_j exp(energy[i][j]))\n * \\param dst destination\n * \\param energy input energy\n */\ntemplate<typename DType>\ninline void Softmax(Tensor<cpu, 2, DType> dst, const Tensor<cpu, 2, DType> &energy);\n/*!\n * \\brief CPU/GPU: normalize softmax: dst[i][j] = exp(energy[i][j]) /(sum_j exp(energy[i][j]))\n * \\param dst destination\n * \\param energy input energy\n */\ntemplate<typename DType>\ninline void Softmax(Tensor<gpu, 2, DType> dst, const Tensor<gpu, 2, DType> &energy);\n\n/*!\n * \\brief CPU/GPU: softmax gradient\n * \\param dst destination\n * \\param src source output\n * \\param label label info\n */\ntemplate<typename DType>\ninline void SoftmaxGrad(Tensor<cpu, 2, DType> dst,\n                        const Tensor<cpu, 2, DType> &src,\n                        const Tensor<cpu, 1, DType> &label);\n/*!\n * \\brief CPU/GPU: softmax gradient\n * \\param dst destination\n * \\param src source output\n * \\param label label info\n */\ntemplate<typename DType>\ninline void SoftmaxGrad(const Tensor<gpu, 2, DType> &dst,\n                        const Tensor<gpu, 2, DType> &src,\n                        const Tensor<gpu, 1, DType> &label);\n/*!\n * \\brief CPU/GPU: Gradient accumulate of embedding matrix.\n                   dst[index[i]] += src[i]\n                   Called when the featuredim of src is much larger than the batchsize\n * \\param dst destination\n * \\param index index to take\n * \\param src source output\n */\ntemplate<bool clip = true, typename IndexType, typename DType>\ninline void AddTakeGrad(Tensor<cpu, 2, DType> dst,\n                        const Tensor<cpu, 1, IndexType>& index,\n                        const Tensor<cpu, 2, DType> &src);\n/*!\n * \\brief CPU/GPU: Gradient accumulate of embedding matrix.\n                   dst[index[i]] += src[i]\n                   Called when the featuredim of src is much larger than the batchsize\n * \\param dst destination\n * \\param index index to take\n * \\param src source output\n */\ntemplate<bool clip = true, typename IndexType, typename DType, typename AType>\ninline void AddTakeGrad(Tensor<cpu, 2, DType> dst,\n                        Tensor<cpu, 2, AType> temp,\n                        const Tensor<cpu, 1, IndexType>& index,\n                        const Tensor<cpu, 2, DType> &src);\n/*!\n * \\brief CPU/GPU: Gradient accumulate of embedding matrix with safe accumulation.\n                   dst[index[i]] += src[i]\n * \\param dst destination\n * \\temp temporal storage for safe accumulation\n * \\param index index to take\n * \\param src source output\n */\ntemplate<bool clip = true, typename IndexType, typename DType>\ninline void AddTakeGrad(Tensor<gpu, 2, DType> dst,\n                        const Tensor<gpu, 1, IndexType>& index,\n                        const Tensor<gpu, 2, DType> &src);\n/*!\n * \\brief CPU/GPU: Gradient accumulate of embedding matrix.\n                   dst[sorted[i]] += src[index[i]]\n                   Called when the batchsize of src is larger than the featuredim\n * \\param dst destination\n * \\param sorted the sorted indices\n * \\param index original index of the sorted indices\n * \\param src source output\n */\ntemplate<bool clip = true, typename IndexType, typename DType, typename AType>\ninline void AddTakeGrad(Tensor<gpu, 2, DType> dst,\n                        Tensor<gpu, 2, AType> temp,\n                        const Tensor<gpu, 1, IndexType>& index,\n                        const Tensor<gpu, 2, DType> &src);\n/*!\n * \\brief CPU/GPU: Gradient accumulate of embedding matrix with safe accumulation.\n                   dst[index[i]] += src[i]\n * \\param dst destination\n * \\temp temporal storage for safe accumulation\n * \\param index index to take\n * \\param src source output\n */\ntemplate<typename IndexType, typename DType>\ninline void AddTakeGradLargeBatch(Tensor<cpu, 2, DType> dst,\n                                  const Tensor<cpu, 1, IndexType>& sorted,\n                                  const Tensor<cpu, 1, IndexType>& index,\n                                  const Tensor<cpu, 2, DType> &src);\n/*!\n * \\brief CPU/GPU: Gradient accumulate of embedding matrix.\n                   dst[sorted[i]] += src[index[i]]\n                   Called when the batchsize of src is larger than the featuredim\n * \\param dst destination\n * \\param sorted the sorted indices\n * \\param index original index of the sorted indices\n * \\param src source output\n */\ntemplate<typename IndexType, typename DType>\ninline void AddTakeGradLargeBatch(Tensor<gpu, 2, DType> dst,\n                                  const Tensor<gpu, 1, IndexType>& sorted,\n                                  const Tensor<gpu, 1, IndexType>& index,\n                                  const Tensor<gpu, 2, DType> &src);\n/*!\n * \\brief CPU/GPU: Fill the values of the destination matrix to specific rows in the source matrix.\n                   dst[index[i]] = src[i]\n                   Will use atomicAdd in the inner implementation and the result may not be deterministic.\n * \\param dst destination\n * \\param index the index to accumulate value\n * \\param src source output\n */\ntemplate<typename IndexType, typename DType>\ninline void IndexFill(Tensor<cpu, 2, DType> dst,\n                      const Tensor<cpu, 1, IndexType>& index,\n                      const Tensor<cpu, 2, DType> &src);\n/*!\n * \\brief CPU/GPU: Fill the values of the destination matrix to specific rows in the source matrix.\n                   dst[index[i]] = src[i]\n                   Will use atomicAdd in the inner implementation and the result may not be deterministic.\n * \\param dst destination\n * \\param index the index to accumulate value\n * \\param src source output\n */\ntemplate<typename IndexType, typename DType>\ninline void IndexFill(Tensor<gpu, 2, DType> dst,\n                      const Tensor<gpu, 1, IndexType>& index,\n                      const Tensor<gpu, 2, DType> &src);\n/*!\n * \\brief CPU/GPU: Sort key-value pairs stored in separate places. (Stable sort is performed!)\n * \\param keys the keys to sort\n * \\param values the values that sorts w.r.t the key\n * \\param is_ascend whether to sort key in ascending order\n */\ntemplate<typename KDType, typename VDType>\ninline void SortByKey(Tensor<cpu, 1, KDType> keys, Tensor<cpu, 1, VDType> values,\n                      bool is_ascend = true);\n/*!\n * \\brief CPU/GPU: Sort key-value pairs stored in separate places. (Stable sort is performed!)\n * \\param keys the keys to sort\n * \\param values the values that sorts w.r.t the key\n * \\param is_ascend whether to sort key in ascending order\n */\ntemplate<typename KDType, typename VDType>\ninline void SortByKey(Tensor<gpu, 1, KDType> keys, Tensor<gpu, 1, VDType> values,\n                      bool is_ascend = true);\n/*!\n * \\brief CPU/GPU: Sort the keys within each segment. (Stable sort is performed!)\n                   Segments is defined as an ascending ordered vector like [0, 0, 0, 1, 1, 2, 3, 3, 3,...]\n                   We sort separately the keys labeled by 0 and 1, 2, 3, etc.\n                   Currently only supports sorting in ascending order !!\n * \\param values the data to sort\n * \\param segments segment indicator\n */\ntemplate<typename Device, typename VDType, typename SDType>\ninline void VectorizedSort(Tensor<Device, 1, VDType> values, Tensor<Device, 1, SDType> segments);\n\n// function declarations to support expression, no need to understand them\n// these functions do not need to be directly used\n/*!\n * \\brief CPU/GPU: map a expression to a tensor, this function calls MapPlan\n * \\tparam Saver specify storage method\n * \\tparam R specifies the storage type of the tensor\n * \\tparam dim dim of the tensor, during usage, there is no need to specify this parameter\n * \\tparam DType the type of elements in the tensor\n * \\tparam E specifies the expression type, not need to specify this parameter during usage\n * \\tparam etype expression type\n * \\param dst destination\n * \\param exp expression\n * \\sa namespace mshadow:sv, mshadow::op, mshadow::expr\n */\ntemplate<typename Saver, typename R, int dim,\n         typename DType, typename E, int etype>\ninline void MapExp(TRValue<R, cpu, dim, DType> *dst,\n                   const expr::Exp<E, DType, etype> &exp);\n/*!\n * \\brief CPU/GPU: map a expression to a tensor, this function calls MapPlan\n * \\tparam Saver specify storage method\n * \\tparam R specifies the storage type of the tensor\n * \\tparam dim dim of the tensor, during usage, there is no need to specify this parameter\n * \\tparam DType the type of elements in the tensor\n * \\tparam E specifies the expression type, not need to specify this parameter during usage\n * \\tparam etype expression type\n * \\param dst destination\n * \\param exp expression\n * \\sa namespace mshadow:sv, mshadow::op, mshadow::expr\n */\ntemplate<typename Saver, typename R, int dim,\n         typename DType, typename E, int etype>\ninline void MapExp(TRValue<R, gpu, dim, DType> *dst,\n                   const expr::Exp<E, DType, etype> &exp);\n/*!\n * \\brief CPU/GPU: map a expression, do reduction to 1D Tensor in lowest dimension (dimension 0)\n * \\tparam Saver specify storage method\n * \\tparam Reducer specify a reducer method\n * \\tparam R specifies the storage type of the tensor\n * \\tparam DType the type of elements in the tensor\n * \\tparam E specifies the expression type, not need to specify this parameter during usage\n * \\tparam etype expression type\n * \\param dst destination\n * \\param exp expression\n * \\param scale scale the result before save\n * \\sa namespace mshadow:sv, mshadow::op, mshadow::red, mshadow::expr\n */\ntemplate<typename Saver, typename Reducer,\n         typename R, typename DType, typename E, int etype>\ninline void MapReduceKeepLowest(TRValue<R, cpu, 1, DType> *dst,\n                                const expr::Exp<E, DType, etype> &exp,\n                                DType scale = 1);\n/*!\n * \\brief CPU/GPU: map a expression, do reduction to 1D Tensor in lowest dimension (dimension 0)\n * \\tparam Saver specify storage method\n * \\tparam Reducer specify a reducer method\n * \\tparam R specifies the storage type of the tensor\n * \\tparam DType the type of elements in the tensor\n * \\tparam E specifies the expression type, not need to specify this parameter during usage\n * \\tparam etype expression type\n * \\param dst destination\n * \\param exp expression\n * \\param scale scale the result before save\n * \\sa namespace mshadow:sv, mshadow::op, mshadow::red, mshadow::expr\n */\ntemplate<typename Saver, typename Reducer, typename R,\n         typename DType, typename E, int etype>\ninline void MapReduceKeepLowest(TRValue<R, gpu, 1, DType> *dst,\n                                const expr::Exp<E, DType, etype> &exp,\n                                DType scale = 1);\n/*!\n * \\brief CPU/GPU: map a expression, do reduction to 1D Tensor in third dimension (dimension 2)\n * \\tparam Saver specify storage method\n * \\tparam Reducer specify a reducer method\n * \\tparam R specifies the storage type of the tensor\n * \\tparam DType the type of elements in the tensor\n * \\tparam dimkeep the target dimension to be kept, should be larger than 0, for 0, use MapReduceKeepLowest\n * \\tparam E specifies the expression type, not need to specify this parameter during usage\n * \\tparam etype expression type\n * \\param dst destination\n * \\param exp expression\n * \\param scale scale the result before save\n * \\sa namespace mshadow:sv, mshadow::op, mshadow::red, mshadow::expr\n */\ntemplate<typename Saver, typename Reducer, int dimkeep,\n         typename R, typename DType, typename E, int etype>\ninline void MapReduceKeepHighDim(TRValue<R, cpu, 1, DType> *dst,\n                                 const expr::Exp<E, DType, etype> &exp,\n                                 DType scale = 1);\n/*!\n * \\brief CPU/GPU: map a expression, do reduction to 1D Tensor in third dimension (dimension 2)\n * \\tparam Saver specify storage method\n * \\tparam Reducer specify a reducer method\n * \\tparam R specifies the storage type of the tensor\n * \\tparam DType the type of elements in the tensor\n * \\tparam dimkeep the target dimension to be kept, should be larger than 0, for 0, use MapReduceKeepLowest\n * \\tparam E specifies the expression type, not need to specify this parameter during usage\n * \\tparam etype expression type\n * \\param dst destination\n * \\param exp expression\n * \\param scale scale the result before save\n * \\sa namespace mshadow:sv, mshadow::op, mshadow::red, mshadow::expr\n */\ntemplate<typename Saver, typename Reducer, int dimkeep,\n         typename R, typename DType, typename E, int etype>\ninline void MapReduceKeepHighDim(TRValue<R, gpu, 1, DType> *dst,\n                                 const expr::Exp<E, DType, etype> &exp,\n                                 DType scale = 1);\n/*!\n * \\brief CPU/GPU: 1 dimension vector dot\n * \\param dst Length 1 vector, used to hold the result.\n * \\param lhs Left operand vector\n * \\param rhs Right operand vector\n */\ntemplate<typename Device, typename DType>\ninline void VectorDot(Tensor<Device, 1, DType> dst,\n                      const Tensor<Device, 1, DType> &lhs,\n                      const Tensor<Device, 1, DType> &rhs);\n/*!\n * \\brief CPU/GPU: dst = alpha * op(lhs) op(rhs) + beta * dst\n * \\param dst Length 3 tensor, used to hold the result\n * \\param lhs Left operand vector\n * \\param rhs Right operand vector\n * \\param alpha multiplier of op(lhs)op(rhs)\n * \\param beta multiplier of dst\n * \\param workspace Workspace for casting DType* to DType** (batched-view), must have size >= 3 * batch_size\n */\ntemplate<bool transpose_left, bool transpose_right, typename Device, typename DType>\ninline void BatchGEMM(Tensor<Device, 3, DType> dst,\n                      const Tensor<Device, 3, DType> &lhs,\n                      const Tensor<Device, 3, DType> &rhs,\n                      DType alpha,\n                      DType beta,\n                      Tensor<Device, 1, DType*> workspace);\n}  // namespace mshadow\n// include headers\n#include \"./stream_gpu-inl.h\"\n#include \"./extension.h\"\n#include \"./expr_engine-inl.h\"\n#include \"./tensor_cpu-inl.h\"\n#include \"./tensor_gpu-inl.h\"\n#include \"./io.h\"\n#include \"./tensor_container.h\"\n#include \"./random.h\"\n// add definition of scalar related operators\n#ifdef MSHADOW_SCALAR_\n  #error \"MSHADOW_SCALAR_ must not be defined\"\n#endif\n// enumerate all the scalar data type we aim to be good at\n#define MSHADOW_SCALAR_ float\n#include \"./expr_scalar-inl.h\"\n#undef MSHADOW_SCALAR_\n#define MSHADOW_SCALAR_ double\n#include \"./expr_scalar-inl.h\"\n#undef MSHADOW_SCALAR_\n#define MSHADOW_SCALAR_ int32_t\n#include \"./expr_scalar-inl.h\"\n#undef MSHADOW_SCALAR_\n#define MSHADOW_SCALAR_ int64_t\n#include \"./expr_scalar-inl.h\"\n#undef MSHADOW_SCALAR_\n#define MSHADOW_SCALAR_ mshadow::half::half_t\n#include \"./expr_scalar-inl.h\"\n#undef MSHADOW_SCALAR_\n#endif  // MSHADOW_TENSOR_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/tensor_container.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file tensor_container.h\n * \\brief tensor container that does memory allocation and resize like STL\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_TENSOR_CONTAINER_H_\n#define MSHADOW_TENSOR_CONTAINER_H_\n#include \"./tensor.h\"\n#include \"./io.h\"\n\nnamespace mshadow {\n/*!\n * \\brief tensor container that does memory allocation and resize like STL,\n *        use it to save the lines of FreeSpace in class.\n *        Do not abuse it, efficiency can come from pre-allocation and no re-allocation\n *\n * \\tparam Device which device the tensor is on\n * \\tparam dimension dimension of the tensor\n */\ntemplate<typename Device, int dimension, typename DType = default_real_t>\nclass TensorContainer: public Tensor<Device, dimension, DType> {\n public:\n  /*!\n   * \\brief constructor\n   * \\param pad whether use padding alignment in space allocation\n   */\n  explicit TensorContainer(bool pad = MSHADOW_ALLOC_PAD) {\n    this->pad_ = pad;\n    this->dptr_ = data_.dptr_ = NULL;\n    this->shape_[0] = 0;\n    this->stride_ = 0;\n    this->data_.stride_ = 0;\n    this->data_.shape_[0] = 0;\n  }\n  /*!\n   * \\brief constructor\n   * \\param shape intial shape\n   */\n  explicit TensorContainer(const Shape<dimension> &shape) {\n    this->pad_ = MSHADOW_ALLOC_PAD;\n    data_.dptr_ = NULL;\n    this->AllocByShape(shape);\n  }\n  /*!\n   * \\brief constructor\n   * \\param shape intial shape\n   * \\param initv intial value\n   */\n  explicit TensorContainer(const Shape<dimension> &shape, DType initv) {\n    this->pad_ = MSHADOW_ALLOC_PAD;\n    data_.dptr_ = NULL;\n    this->AllocByShape(shape);\n    (*this) = initv;\n  }\n  /*!\n   * \\brief copy constructor\n   * \\param src source value\n   */\n  TensorContainer\n  (const TensorContainer<Device, dimension, DType> &src)\n      : pad_(src.pad_) {\n    this->dptr_ = data_.dptr_ = NULL;\n    this->shape_[0] = 0;\n    this->stride_ = 0;\n    this->data_.stride_ = 0;\n    this->data_.shape_[0] = 0;\n    this->stream_ = src.stream_;\n    if (src.dptr_ != NULL) {\n      this->AllocByShape(src.shape_);\n      mshadow::Copy(*this, src, this->stream_);\n    }\n  }\n  ~TensorContainer(void) MSHADOW_THROW_EXCEPTION {\n    this->Release();\n  }\n  /*!\n   * \\brief resize the container to given shape, content is NOT preserved\n   * \\param shape target shape\n   */\n  inline void Resize(const Shape<dimension> &shape) {\n    Shape<2> s2 = shape.FlatTo2D();\n    if (s2.shape_[1] > data_.stride_ || s2.shape_[0] > data_.size(0)) {\n      this->AllocByShape(shape);\n    } else {\n      this->shape_ = shape;\n      if (this->pad_) {\n        this->stride_ = data_.stride_;\n      } else {\n        this->stride_ = s2.shape_[1];\n      }\n    }\n  }\n  /*!\n   * \\brief resize the container to given shape, and initialize, content is NOT preserved\n   * \\param shape target shape\n   * \\param initv initialization value\n   */\n  inline void Resize(const Shape<dimension> &shape, DType initv) {\n    this->Resize(shape);\n    (*this) = initv;\n  }\n  /*! \\brief set whether padding is allowed in tensor */\n  inline void set_pad(bool pad) {\n    this->pad_ = pad;\n  }\n  /*!\n   * \\brief save by binary format\n   * \\param fo output binary stream\n   * \\tparam TStream type of stream, need to support Read, Write, one example is utils::IStream.\n   */\n  template<typename TStream>\n  inline void SaveBinary(TStream &fo) const { // NOLINT(*)\n    mshadow::SaveBinary(fo, *this);\n  }\n  /*!\n   * \\brief load by binary format, a temp Tensor<cpu,dim> storage will be allocated\n   * \\param fi input binary stream\n   * \\tparam TStream type of stream, need to support Read, Write, one example is utils::IStream.\n   */\n  template<typename TStream>\n  inline void LoadBinary(TStream &fi) { // NOLINT(*)\n    Tensor<cpu, dimension, DType> tmp;\n    mshadow::LoadBinary(fi, &tmp, false);\n    this->Resize(tmp.shape_);\n    Stream<Device> stream;\n    Copy(*this, tmp, &stream);\n    mshadow::FreeSpace(&tmp);\n  }\n  /*!\n   * \\brief assign operator from TensorContainer\n   * \\param src source value\n   * \\return reference of self\n   */\n  inline TensorContainer &operator=\n  (const TensorContainer<Device, dimension, DType> &src) {\n    this->pad_ = src.pad_;\n    this->stream_ = src.stream_;\n    if (src.dptr_ != NULL) {\n      this->Resize(src.shape_);\n      mshadow::Copy(*this, src, this->stream_);\n    }\n    return *this;\n  }\n  /*!\\brief functions to fit expression template */\n  inline Tensor<Device, dimension, DType> &operator=(DType s) {\n    return this->__assign(s);\n  }\n  /*!\\brief functions to fit expression template */\n  template<typename E>\n  inline Tensor<Device, dimension, DType> &\n  operator=(const expr::Exp<E, DType, expr::type::kMapper> &exp) {\n    return this->__assign(exp);\n  }\n  /*!\\brief functions to fit expression template */\n  template<typename E>\n  inline Tensor<Device, dimension, DType> &\n  operator=(const expr::Exp<E, DType, expr::type::kChainer> &exp) {\n    return this->__assign(exp);\n  }\n  /*!\\brief functions to fit expression template */\n  template<typename E>\n  inline Tensor<Device, dimension, DType> &\n  operator=(const expr::Exp<E, DType, expr::type::kComplex> &exp) {\n    return this->__assign(exp);\n  }\n  /*!\n   * \\brief Release the llocated space,\n   *  The TensorContainer is still functionable,\n   *  but will restart allocating space when Resize is called.\n   */\n  inline void Release(void) {\n    if (data_.dptr_ != NULL) {\n      this->shape_[0] = 0;\n      this->stride_ = 0;\n      this->data_.stride_ = 0;\n      this->data_.shape_[0] = 0;\n      try {\n        mshadow::FreeSpace(&data_);\n      } catch (const dmlc::Error &e) {\n        this->dptr_ = data_.dptr_ = NULL;\n        throw e;\n      }\n      this->dptr_ = data_.dptr_ = NULL;\n    }\n  }\n\n private:\n  /*! \\brief whether we do padding in the space */\n  bool pad_;\n  /*! \\brief the shape of data_ is actually current data space */\n  Tensor<Device, 2, DType> data_;\n\n  inline void AllocByShape(const Shape<dimension>& shape) {\n    if (data_.dptr_ != NULL) this->Release();\n    data_.shape_ = shape.FlatTo2D();\n    mshadow::AllocSpace(&data_, pad_);\n    this->dptr_ = data_.dptr_;\n    this->shape_ = shape;\n    if (this->pad_) {\n      this->stride_ = data_.stride_;\n    } else {\n      this->stride_ = data_.size(1);\n    }\n  }\n};\n}  // namespace mshadow\n#endif  // MSHADOW_TENSOR_CONTAINER_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/tensor_cpu-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file tensor_cpu-inl.h\n * \\brief implementation of CPU host code\n * \\author Bing Xu, Tianqi Chen\n */\n#ifndef MSHADOW_TENSOR_CPU_INL_H_\n#define MSHADOW_TENSOR_CPU_INL_H_\n#include <cstring>\n#include <functional>\n#include <utility>\n#include <vector>\n#include \"./base.h\"\n#include \"./tensor.h\"\n#include \"./packet-inl.h\"\n#include \"./dot_engine-inl.h\"\n\nnamespace mshadow {\ntemplate<>\ninline void InitTensorEngine<cpu>(int dev_id) {\n}\ntemplate<>\ninline void ShutdownTensorEngine<cpu>(void) {\n}\n\ntemplate<>\ninline void SetDevice<cpu>(int devid) {\n}\ntemplate<>\ninline Stream<cpu> *NewStream<cpu>(bool create_blas_handle,\n                                   bool create_dnn_handle,\n                                   int dev_id) {\n  return new Stream<cpu>();\n}\ntemplate<>\ninline void DeleteStream<cpu>(Stream<cpu> *stream) {\n  delete stream;\n}\n\ntemplate<int ndim>\ninline std::ostream &operator<<(std::ostream &os, const Shape<ndim> &shape) { // NOLINT(*)\n  os << '(';\n  for (int i = 0; i < ndim; ++i) {\n    if (i != 0) os << ',';\n    os << shape[i];\n  }\n  // python style tuple\n  if (ndim == 1) os << ',';\n  os << ')';\n  return os;\n}\n\ntemplate<typename xpu>\ninline void *AllocHost_(size_t size);\ntemplate<typename xpu>\ninline void FreeHost_(void * dptr);\n\n#ifdef __CUDACC__\ntemplate<>\ninline void *AllocHost_<gpu>(size_t size) {\n  void *dptr;\n  MSHADOW_CUDA_CALL(cudaMallocHost(&dptr, size, cudaHostAllocPortable));\n  return dptr;\n}\ntemplate<>\ninline void FreeHost_<gpu>(void *dptr) {\n  MSHADOW_CUDA_CALL(cudaFreeHost(dptr));\n}\n#endif\n\ntemplate<>\ninline void *AllocHost_<cpu>(size_t size) {\n  size_t pitch;\n  return packet::AlignedMallocPitch(&pitch, size, 1);\n}\ntemplate<>\ninline void FreeHost_<cpu>(void *dptr) {\n  packet::AlignedFree(dptr);\n}\n\ntemplate<typename xpu, int dim, typename DType>\ninline void AllocHost(Tensor<cpu, dim, DType> *obj) {\n  obj->stride_ = obj->size(dim - 1);\n  CHECK_EQ(obj->CheckContiguous(), true) << \"AllocHost\";\n  void *dptr = AllocHost_<xpu>(obj->MSize() * sizeof(DType));\n  obj->dptr_ = reinterpret_cast<DType*>(dptr);\n}\ntemplate<typename xpu, int dim, typename DType>\ninline void FreeHost(Tensor<cpu, dim, DType> *obj) {\n  if (obj->dptr_ == NULL) {\n    LOG(FATAL) << \"FreeHost:: double free\";\n  }\n  FreeHost_<xpu>(obj->dptr_);\n  obj->dptr_ = NULL;\n}\n\ntemplate<int dim, typename DType>\ninline void AllocSpace(Tensor<cpu, dim, DType> *obj, bool pad) {\n  size_t pitch;\n  void *dptr;\n  if (pad) {\n    dptr = packet::AlignedMallocPitch\n        (&pitch, obj->size(dim - 1) * sizeof(DType), obj->shape_.FlatTo2D()[0]);\n    obj->stride_ = static_cast<index_t>(pitch / sizeof(DType));\n  } else {\n    obj->stride_ = obj->size(dim - 1);\n    dptr = packet::AlignedMallocPitch\n        (&pitch, obj->shape_.Size() * sizeof(DType), 1);\n  }\n  obj->dptr_ = reinterpret_cast<DType*>(dptr);\n}\ntemplate<typename Device, typename DType, int dim>\ninline Tensor<Device, dim, DType>\nNewTensor(const Shape<dim> &shape, DType initv, bool pad, Stream<Device> *stream_) {\n  Tensor<Device, dim, DType> obj(shape);\n  obj.stream_ = stream_;\n  AllocSpace(&obj, pad);\n  MapExp<sv::saveto>(&obj, expr::ScalarExp<DType>(initv));\n  return obj;\n}\ntemplate<int dim, typename DType>\ninline void FreeSpace(Tensor<cpu, dim, DType> *obj) {\n  packet::AlignedFree(obj->dptr_);\n  obj->dptr_ = NULL;\n}\ntemplate<int dim, typename DType>\ninline void Copy(Tensor<cpu, dim, DType> _dst,\n                 const Tensor<cpu, dim, DType> &_src,\n                 Stream<cpu> *stream) {\n#pragma GCC diagnostic push\n#if __GNUC__ >= 8\n#pragma GCC diagnostic ignored \"-Wclass-memaccess\"\n#endif\n  CHECK_EQ(_dst.shape_, _src.shape_)\n      << \"Copy:shape mismatch:\" << _dst.shape_ << \" vs \" << _src.shape_;\n  if (_dst.CheckContiguous() && _src.CheckContiguous()) {\n    memcpy(_dst.dptr_, _src.dptr_, sizeof(DType) * _dst.shape_.Size());\n  } else {\n    Tensor<cpu, 2, DType> dst = _dst.FlatTo2D();\n    Tensor<cpu, 2, DType> src = _src.FlatTo2D();\n    for (index_t y = 0; y < dst.size(0); ++y) {\n      memcpy(dst[y].dptr_, src[y].dptr_, sizeof(DType) * dst.size(1));\n    }\n  }\n#pragma GCC diagnostic pop\n}\n\ntemplate<typename Saver, typename R, int dim,\n         typename DType, typename E>\ninline void MapPlan(TRValue<R, cpu, dim, DType> *dst,\n                    const expr::Plan<E, DType> &plan) {\n  Shape<2> shape = expr::ShapeCheck<dim, R>::Check(dst->self()).FlatTo2D();\n  expr::Plan<R, DType> dplan = expr::MakePlan(dst->self());\n#ifndef __CUDACC__\n  #pragma omp parallel for\n#endif\n  // temp remove openmp, as default setting throttles CPU\n  for (openmp_index_t y = 0; y < shape[0]; ++y) {\n    for (index_t x = 0; x < shape[1]; ++x) {\n      // trust your compiler! -_- they will optimize it\n      Saver::template Save<DType>(dplan.REval(y, x), plan.Eval(y, x));\n    }\n  }\n}\n// code to handle SSE optimization\ntemplate<bool pass_check, typename Saver,\n         typename R, int dim,\n         typename DType, typename E, int etype>\nstruct MapExpCPUEngine {\n  inline static void Map(TRValue<R, cpu, dim, DType> *dst,\n                         const expr::Exp<E, DType, etype> &exp) {\n    MapPlan<Saver>(dst, MakePlan(exp.self()));\n  }\n};\n\ntemplate<typename SV, int dim, typename DType, typename E, int etype>\nstruct MapExpCPUEngine<true, SV, Tensor<cpu, dim, DType>,\n                       dim, DType, E, etype> {\n  inline static void Map(Tensor<cpu, dim, DType> *dst,\n                         const expr::Exp<E, DType, etype> &exp) {\n    if (expr::PacketAlignCheck<dim, E, MSHADOW_DEFAULT_PACKET>::Check(exp.self()) &&\n        expr::PacketAlignCheck<dim, Tensor<cpu, dim, DType>, MSHADOW_DEFAULT_PACKET>::Check(*dst)) {\n      expr::MapPacketPlan<SV>(dst->self(),\n                              expr::MakePacketPlan<MSHADOW_DEFAULT_PACKET>(exp.self()));\n    } else {\n      MapPlan<SV>(dst, MakePlan(exp.self()));\n    }\n  }\n};\n\n\ntemplate<typename Saver, typename R, int dim,\n         typename DType, typename E, int etype>\ninline void MapExp(TRValue<R, cpu, dim, DType> *dst,\n                   const expr::Exp<E, DType, etype> &exp) {\n  expr::TypeCheckPass<expr::TypeCheck<cpu, dim, DType, E>::kMapPass>\n      ::Error_All_Tensor_in_Exp_Must_Have_Same_Type();\n  Shape<dim> eshape = expr::ShapeCheck<dim, E>::Check(exp.self());\n  Shape<dim> dshape = expr::ShapeCheck<dim, R>::Check(dst->self());\n  CHECK(eshape[0] == 0 || eshape == dshape)\n      << \"Assignment: Shape of Tensors are not consistent with target, \"\n      << \"eshape: \" << eshape << \" dshape:\" << dshape;\n  MapExpCPUEngine<expr::PacketCheck<E, MSHADOW_DEFAULT_PACKET>::kPass,\n                  Saver, R, dim, DType, E, etype>\n  ::Map(dst->ptrself(), exp);\n}\n\ntemplate<typename Saver, typename Reducer,\n         typename R, typename DType, typename E, int etype>\ninline void MapReduceKeepLowest(TRValue<R, cpu, 1, DType> *dst,\n                                const expr::Exp<E, DType, etype> &exp,\n                                DType scale) {\n  expr::TypeCheckPass<expr::TypeCheck<cpu, 1, DType, E>::kRedPass>\n      ::Error_TypeCheck_Not_Pass_For_Reduce_Exp();\n  Shape<2> eshape = expr::ShapeCheck<expr::ExpInfo<E>::kDim, E>\n      ::Check(exp.self()).FlatTo2D();\n  Shape<1> dshape = expr::ShapeCheck<1, R>::Check(dst->self());\n  CHECK_EQ(eshape[1], dshape[0]) << \"MapReduceKeepLowest::reduction dimension do not match\";\n  CHECK_NE(eshape[0], 0U) << \"can not reduce over empty tensor\";\n  // execution\n  expr::Plan<R, DType> dplan = MakePlan(dst->self());\n  expr::Plan<E, DType> splan = MakePlan(exp.self());\n#ifndef __CUDACC__\n  #pragma omp parallel for\n#endif\n  for (openmp_index_t x = 0; x < eshape[1]; ++x) {\n    DType res = splan.Eval(0, x);\n    for (index_t y = 1; y < eshape[0]; ++y) {\n      Reducer::Reduce(res, splan.Eval(y, x));\n    }\n    Saver::template Save<DType>(dplan.REval(0, x), res * scale);\n  }\n}\n\ntemplate<typename Saver, typename Reducer, int dimkeep,\n         typename R, typename DType, typename E, int etype>\ninline void MapReduceKeepHighDim(TRValue<R, cpu, 1, DType> *dst,\n                                 const expr::Exp<E, DType, etype> &exp,\n                                 DType scale) {\n  expr::TypeCheckPass<expr::TypeCheck<cpu, dimkeep, DType, E>::kRedPass>\n      ::Error_TypeCheck_Not_Pass_For_Reduce_Exp();\n  typedef Shape<expr::ExpInfo<E>::kDim> EShape;\n  EShape eshape = expr::ShapeCheck<expr::ExpInfo<E>::kDim, E>\n      ::Check(exp.self());\n  Shape<1> dshape = expr::ShapeCheck<1, R>::Check(dst->self());\n  CHECK_EQ(eshape[dimkeep], dshape[0])\n    << \"MapReduceKeepHighDim::reduction dimension do not match\";\n  // use equvalent form\n  Shape<4> pshape = Shape4(eshape.ProdShape(0, dimkeep),\n                           eshape[dimkeep],\n                           eshape.ProdShape(dimkeep + 1, EShape::kSubdim),\n                           eshape[EShape::kSubdim]);\n  // execution\n  expr::Plan<R, DType> dplan = MakePlan(dst->self());\n  expr::Plan<E, DType> splan = MakePlan(exp.self());\n#ifndef __CUDACC__\n  #pragma omp parallel for\n#endif\n  for (openmp_index_t c = 0; c < pshape[1]; ++c) {\n    DType res; Reducer::SetInitValue(res);\n    for (index_t n = 0; n < pshape[0]; ++n) {\n      DType tres; Reducer::SetInitValue(tres);\n      for (index_t y = 0; y < pshape[2]; ++y) {\n        for (index_t x = 0; x < pshape[3]; ++x) {\n          Reducer::Reduce(tres,\n                          splan.Eval((n * pshape[1] + c) * pshape[2] + y, x));\n        }\n      }\n      Reducer::Reduce(res, tres);\n    }\n    Saver::template Save<DType>(dplan.REval(0, c), DType(res * scale));\n  }\n}\n\ntemplate<typename DType>\ninline void Softmax(Tensor<cpu, 1, DType> dst,\n                    const Tensor<cpu, 1, DType> &energy) {\n  DType mmax = energy[0];\n  for (index_t x = 1; x < dst.size(0); ++x) {\n    if (mmax < energy[x]) mmax = energy[x];\n  }\n  DType sum = DType(0.0f);\n  for (index_t x = 0; x < dst.size(0); ++x) {\n    dst[x] = std::exp(energy[x] - mmax);\n    sum += dst[x];\n  }\n  for (index_t x = 0; x < dst.size(0); ++x) {\n    dst[x] /= sum;\n  }\n}\n\ntemplate<typename DType>\ninline void SoftmaxGrad(Tensor<cpu, 2, DType> dst,\n                        const Tensor<cpu, 2, DType> &src,\n                        const Tensor<cpu, 1, DType> &label) {\n#pragma omp parallel for\n  for (openmp_index_t y = 0; y < dst.size(0); ++y) {\n    const index_t k = static_cast<int>(label[y]);\n    for (index_t x = 0; x < dst.size(1); ++x) {\n      if (x == k) {\n        dst[y][k] = src[y][k] - 1.0f;\n      } else {\n        dst[y][x] = src[y][x];\n      }\n    }\n  }\n}\n\ntemplate<typename DType>\ninline void SmoothSoftmaxGrad(Tensor<cpu, 2, DType> dst,\n                        const Tensor<cpu, 2, DType> &src,\n                        const Tensor<cpu, 1, DType> &label,\n                        const float alpha) {\n  const float smooth_grad = (alpha / (dst.size(1) - 1));\n#pragma omp parallel for\n  for (openmp_index_t y = 0; y < dst.size(0); ++y) {\n    const index_t k = static_cast<int>(label[y]);\n    for (index_t x = 0; x < dst.size(1); ++x) {\n      if (x == k) {\n        dst[y][k] = src[y][k] - 1.0f + alpha;\n      } else {\n        dst[y][x] = src[y][x] - smooth_grad;\n      }\n    }\n  }\n}\n\n\ntemplate<typename DType>\ninline void SoftmaxGrad(Tensor<cpu, 2, DType> dst,\n                        const Tensor<cpu, 2, DType> &src,\n                        const Tensor<cpu, 1, DType> &label,\n                        const DType &ignore_label) {\n#pragma omp parallel for\n  for (openmp_index_t y = 0; y < dst.size(0); ++y) {\n    const int k = static_cast<int>(label[y]);\n    for (int x = 0; x < static_cast<int>(dst.size(1)); ++x) {\n      if (static_cast<int>(ignore_label) == k) {\n        dst[y][x] = 0.0f;\n      } else {\n        if (x == k) {\n          dst[y][k] = src[y][k] - 1.0f;\n        } else {\n          dst[y][x] = src[y][x];\n        }\n      }\n    }\n  }\n}\n\ntemplate<typename DType>\ninline void SmoothSoftmaxGrad(Tensor<cpu, 2, DType> dst,\n                              const Tensor<cpu, 2, DType> &src,\n                              const Tensor<cpu, 1, DType> &label,\n                              const DType &ignore_label,\n                              const float alpha) {\n  const float smooth_grad = (alpha / (dst.size(1) - 1));\n#pragma omp parallel for\n  for (openmp_index_t y = 0; y < dst.size(0); ++y) {\n    const int k = static_cast<int>(label[y]);\n    for (int x = 0; x < static_cast<int>(dst.size(1)); ++x) {\n      if (static_cast<int>(ignore_label) == k) {\n        dst[y][x] = 0.0f;\n      } else {\n        if (x == k) {\n          dst[y][k] = src[y][k] - 1.0f + alpha;\n        } else {\n          dst[y][x] = src[y][x] - smooth_grad;\n        }\n      }\n    }\n  }\n}\n\ntemplate<typename DType>\ninline void SoftmaxGrad(Tensor<cpu, 3, DType> dst,\n                        const Tensor<cpu, 3, DType> &src,\n                        const Tensor<cpu, 2, DType> &label) {\n#pragma omp parallel for\n  for (openmp_index_t n = 0; n < dst.size(2); ++n) {\n    for (index_t y = 0; y < dst.size(0); ++y) {\n      const int k = static_cast<int>(label[y][n]);\n      for (int x = 0; x < static_cast<int>(dst.size(1)); ++x) {\n        if (x == k) {\n          dst[y][k][n] = src[y][k][n] - 1.0f;\n        } else {\n          dst[y][x][n] = src[y][x][n];\n        }\n      }\n    }\n  }\n}\n\ntemplate<typename DType>\ninline void SmoothSoftmaxGrad(Tensor<cpu, 3, DType> dst,\n                        const Tensor<cpu, 3, DType> &src,\n                        const Tensor<cpu, 2, DType> &label,\n                        const float alpha) {\n  const float smooth_grad = (alpha / (dst.size(1) - 1));\n#pragma omp parallel for\n  for (openmp_index_t n = 0; n < dst.size(2); ++n) {\n    for (index_t y = 0; y < dst.size(0); ++y) {\n      const int k = static_cast<int>(label[y][n]);\n      for (int x = 0; x < static_cast<int>(dst.size(1)); ++x) {\n        if (x == k) {\n          dst[y][k][n] = src[y][k][n] - 1.0f + alpha;\n        } else {\n          dst[y][x][n] = src[y][x][n] - smooth_grad;\n        }\n      }\n    }\n  }\n}\n\ntemplate<typename DType>\ninline void SoftmaxGrad(Tensor<cpu, 3, DType> dst,\n                        const Tensor<cpu, 3, DType> &src,\n                        const Tensor<cpu, 2, DType> &label,\n                        const DType &ignore_label) {\n#pragma omp parallel for\n  for (openmp_index_t n = 0; n < dst.size(2); ++n) {\n    for (index_t y = 0; y < dst.size(0); ++y) {\n      const int k = static_cast<int>(label[y][n]);\n      if (k == static_cast<int>(ignore_label)) {\n        for (int x = 0; x < static_cast<int>(dst.size(1)); ++x) {\n          dst[y][x][n] = DType(0.0f);\n        }\n      } else {\n        for (int x = 0; x < static_cast<int>(dst.size(1)); ++x) {\n          if (x == k) {\n            dst[y][k][n] = src[y][k][n] - 1.0f;\n          } else {\n            dst[y][x][n] = src[y][x][n];\n          }\n        }\n      }\n    }\n  }\n}\n\ntemplate<typename DType>\ninline void SmoothSoftmaxGrad(Tensor<cpu, 3, DType> dst,\n                        const Tensor<cpu, 3, DType> &src,\n                        const Tensor<cpu, 2, DType> &label,\n                        const DType &ignore_label,\n                        const float alpha) {\n  const float smooth_grad = (alpha / (dst.size(1) - 1));\n#pragma omp parallel for\n  for (openmp_index_t n = 0; n < dst.size(2); ++n) {\n    for (index_t y = 0; y < dst.size(0); ++y) {\n      const int k = static_cast<int>(label[y][n]);\n      if (k == static_cast<int>(ignore_label)) {\n        for (int x = 0; x < static_cast<int>(dst.size(1)); ++x) {\n          dst[y][x][n] = DType(0.0f);\n        }\n      } else {\n        for (int x = 0; x < static_cast<int>(dst.size(1)); ++x) {\n          if (x == k) {\n            dst[y][k][n] = src[y][k][n] - 1.0f + alpha;\n          } else {\n            dst[y][x][n] = src[y][x][n] - smooth_grad;\n          }\n        }\n      }\n    }\n  }\n}\n\ntemplate<typename DType>\ninline void Softmax(Tensor<cpu, 2, DType> dst,\n                    const Tensor<cpu, 2, DType> &energy) {\n  CHECK_EQ(dst.shape_, energy.shape_) << \"Softmax: shape mismatch\";\n#pragma omp parallel for\n  for (openmp_index_t y = 0; y < dst.size(0); ++y) {\n    Softmax(dst[y], energy[y]);\n  }\n}\n\ntemplate<typename DType>\ninline void Softmax(Tensor<cpu, 3, DType> dst,\n                    const Tensor<cpu, 3, DType> &energy) {\n  CHECK_EQ(dst.shape_, energy.shape_) << \"Softmax: shape mismatch\";\n#pragma omp parallel for\n  for (openmp_index_t y = 0; y < dst.size(0); ++y) {\n    for (index_t n = 0; n < dst.size(2); ++n) {\n      DType mmax = energy[y][0][n];\n      for (index_t x = 1; x < dst.size(1); ++x) {\n        if (mmax < energy[y][x][n]) mmax = energy[y][x][n];\n      }\n      DType sum = DType(0.0f);\n      for (index_t x = 0; x < dst.size(1); ++x) {\n        dst[y][x][n] = std::exp(energy[y][x][n] - mmax);\n        sum += dst[y][x][n];\n      }\n      for (index_t x = 0; x < dst.size(1); ++x) {\n        dst[y][x][n] /= sum;\n      }\n    }\n  }\n}\n\ntemplate<bool clip, typename IndexType, typename DType>\ninline void AddTakeGrad(Tensor<cpu, 2, DType> dst,\n                        const Tensor<cpu, 1, IndexType>& index,\n                        const Tensor<cpu, 2, DType> &src) {\n  const index_t K = dst.shape_[0];\n  const index_t C = dst.shape_[1];\n  for (index_t y = 0; y < index.size(0); ++y) {\n    index_t j = index[y];\n    if (clip) {\n      if (j <= 0) j = 0;\n      else if (j >= K) j = K - 1;\n    } else {\n      j %= K;\n      if (j < 0) j += K;\n    }\n    for (index_t i = 0; i < C; ++i) {\n      dst[j][i] += src[y][i];\n    }\n  }\n}\n\n// safe accumulation\ntemplate<bool clip, typename IndexType, typename DType, typename AType>\ninline void AddTakeGrad(Tensor<cpu, 2, DType> dst,\n                        Tensor<cpu, 2, AType> temp,\n                        const Tensor<cpu, 1, IndexType>& index,\n                        const Tensor<cpu, 2, DType> &src) {\n  const index_t K = dst.shape_[0];\n  const index_t C = dst.shape_[1];\n  for (index_t j = 0; j < K; ++j) {\n    for (index_t i = 0; i < C; ++i) {\n      temp[j][i] = dst[j][i];\n    }\n  }\n  for (index_t y = 0; y < index.size(0); ++y) {\n    index_t j = index[y];\n    if (clip) {\n      if (j <= 0) j = 0;\n      else if (j >= K) j = K - 1;\n    } else {\n      j %= K;\n      if (j < 0) j += K;\n    }\n    for (index_t i = 0; i < C; ++i) {\n      temp[j][i] += src[y][i];\n    }\n  }\n  for (index_t j = 0; j < K; ++j) {\n    for (index_t i = 0; i < C; ++i) {\n      dst[j][i] = temp[j][i];\n    }\n  }\n}\n\ntemplate<typename IndexType, typename DType>\ninline void AddTakeGradLargeBatch(Tensor<cpu, 2, DType> dst,\n                                  const Tensor<cpu, 1, IndexType>& sorted,\n                                  const Tensor<cpu, 1, IndexType>& index,\n                                  const Tensor<cpu, 2, DType> &src) {\n  for (index_t y = 0; y < sorted.size(0); ++y) {\n    dst[sorted[y]] += src[index[y]];\n  }\n}\n\ntemplate<typename IndexType, typename DType>\ninline void IndexFill(Tensor<cpu, 2, DType> dst,\n                      const Tensor<cpu, 1, IndexType>& index,\n                      const Tensor<cpu, 2, DType> &src) {\n  for (index_t y = 0; y < index.size(0); ++y) {\n    for (index_t j = 0; j < src.size(1); j++) {\n      dst[index[y]][j] = src[y][j];\n    }\n  }\n}\n\ntemplate<typename KDType, typename VDType>\ninline void SortByKey(Tensor<cpu, 1, KDType> keys, Tensor<cpu, 1, VDType> values,\n                      bool is_ascend) {\n  CHECK_EQ(keys.CheckContiguous(), true);\n  CHECK_EQ(values.CheckContiguous(), true);\n  CHECK_EQ(keys.size(0), values.size(0))\n    << \"The sizes of key/value are not equal! keys_size: \" << keys.size(0)\n    << \"values_size: \" << values.size(0);\n  std::vector<size_t> idx(keys.size(0));\n  std::vector<KDType> keys_vec(keys.size(0));\n  std::vector<VDType> values_vec(values.size(0));\n  for (int i = 0; i < keys.size(0); i++) {\n    idx[i] = i;\n    keys_vec[i] = keys[i];\n    values_vec[i] = values[i];\n  }\n  if (is_ascend) {\n    std::stable_sort(idx.begin(), idx.end(),\n                     [&keys_vec](size_t i1, size_t i2)\n                       {return keys_vec[i1] < keys_vec[i2]; });\n  } else {\n    std::stable_sort(idx.begin(), idx.end(),\n                     [&keys_vec](size_t i1, size_t i2)\n                       {return keys_vec[i1] > keys_vec[i2]; });\n  }\n  for (index_t i = 0; i < values.size(0); i++) {\n    keys[i] = keys_vec[idx[i]];\n    values[i] = values_vec[idx[i]];\n  }\n}\n\ntemplate<typename Device, typename VDType, typename SDType>\ninline void VectorizedSort(Tensor<Device, 1, VDType> values, Tensor<Device, 1, SDType> segments) {\n  // We can sort each segments using two stable sorts\n  SortByKey(values, segments, true);\n  SortByKey(segments, values, true);\n}\n\n// blas related\ntemplate<typename Device, typename DType>\ninline void VectorDot(Tensor<Device, 1, DType> dst,\n                      const Tensor<Device, 1, DType> &lhs,\n                      const Tensor<Device, 1, DType> &rhs) {\n  CHECK_EQ(lhs.size(0), rhs.size(0))\n      << \"VectorDot: Shape mismatch\";\n  CHECK_EQ(dst.size(0), 1U)\n      << \"VectorDot: expect dst to be scalar\";\n  expr::BLASEngine<Device, DType>::SetStream(lhs.stream_);\n  mshadow::expr::BLASEngine<Device, DType>::dot(\n      lhs.stream_, lhs.size(0), lhs.dptr_, 1, rhs.dptr_, 1, dst.dptr_);\n}\n\ntemplate<bool transpose_left, bool transpose_right, typename Device, typename DType>\ninline void BatchGEMM(Tensor<Device, 3, DType> dst,\n                      const Tensor<Device, 3, DType> &lhs,\n                      const Tensor<Device, 3, DType> &rhs,\n                      DType alpha,\n                      DType beta,\n                      Tensor<Device, 1, DType*> workspace) {\n  index_t batch_size = dst.shape_[0];\n  expr::BLASEngine<Device, DType>::SetStream(dst.stream_);\n  Shape<3> sleft = transpose_left ? Shape3(lhs.shape_[0], lhs.shape_[2], lhs.shape_[1])\n    : lhs.shape_;\n  Shape<3> sright = transpose_right ? Shape3(rhs.shape_[0], rhs.shape_[2], rhs.shape_[1])\n    : rhs.shape_;\n  CHECK_EQ(dst.CheckContiguous(), true);\n  CHECK_EQ(lhs.CheckContiguous(), true);\n  CHECK_EQ(rhs.CheckContiguous(), true);\n  CHECK(sleft[0] == batch_size && sright[0] == batch_size)\n    << \"BatchGEMM: batchsize must be equal.\"\n    << \"dst: \" << dst.shape_ << \"\\n\"\n    << \"lhs: \" << sleft << \"\\n\"\n    << \"rhs: \" << sright << \"\\n\";\n  CHECK(dst.size(1) == sleft[1] && dst.size(2) == sright[2] && sleft[2] == sright[1])\n    << \"BatchGEMM: matrix shape mismatch\"\n    << \"dst: \" << dst.shape_ << \"\\n\"\n    << \"lhs: \" << sleft << \"\\n\"\n    << \"rhs: \" << sright << \"\\n\";\n  CHECK(workspace.size(0) >= 3 * batch_size)\n    << \"Workspace Size must be bigger than \" << 3 * batch_size;\n  CHECK_EQ(workspace.CheckContiguous(), true);\n  // use column major argument to compatible with most BLAS\n  expr::BLASEngine<Device, DType>::batched_gemm\n    (dst.stream_,\n    transpose_right, transpose_left,\n    transpose_right ? rhs.size(1) : rhs.size(2),\n    transpose_left ? lhs.size(2) : lhs.size(1),\n    transpose_right ? rhs.size(2) : rhs.size(1),\n    alpha,\n    rhs.dptr_, rhs.stride_,\n    lhs.dptr_, lhs.stride_,\n    beta,\n    dst.dptr_, dst.stride_, batch_size,\n    workspace.dptr_);\n}\n}  // namespace mshadow\n#endif  // MSHADOW_TENSOR_CPU_INL_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow/tensor_gpu-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file tensor_gpu-inl.h\n * \\brief implementation of GPU host code\n * \\author Bing Xu, Tianqi Chen\n */\n#ifndef MSHADOW_TENSOR_GPU_INL_H_\n#define MSHADOW_TENSOR_GPU_INL_H_\n#include \"./base.h\"\n#include \"./tensor.h\"\n\nnamespace mshadow {\n#if MSHADOW_USE_CUDA\ntemplate<>\ninline void InitTensorEngine<gpu>(int dev_id) {\n  cudaDeviceProp prop;\n  int device_id = 0;\n  int device_count = 0;\n  cudaGetDeviceCount(&device_count);\n  CHECK_GT(device_count, 0) << \"Cannot find CUDA device. Please check CUDA-Configuration\";\n  if (dev_id < 0) {\n    device_id = 0;\n  } else {\n    device_id = dev_id;\n  }\n  CHECK_LT(device_id, device_count) << \"Incorrect Device ID\";\n  MSHADOW_CUDA_CALL(cudaSetDevice(device_id));\n  MSHADOW_CUDA_CALL(cudaGetDeviceProperties(&prop, device_id));\n}\ntemplate<>\ninline void ShutdownTensorEngine<gpu>(void) {\n}\ntemplate<>\ninline void SetDevice<gpu>(int devid) {\n  MSHADOW_CUDA_CALL(cudaSetDevice(devid));\n}\ntemplate<int dim, typename DType>\ninline void AllocSpace(Tensor<gpu, dim, DType> *obj, bool pad) {\n  size_t pitch;\n  // common choice for cuda mem align unit is 32\n  if (pad && obj->size(dim - 1) >= MSHADOW_MIN_PAD_RATIO * 32) {\n    MSHADOW_CUDA_CALL(cudaMallocPitch(reinterpret_cast<void**>(&(obj->dptr_)), &pitch,\n                                      obj->size(dim - 1) * sizeof(DType),\n                                      obj->shape_.FlatTo2D()[0]));\n    obj->stride_ = static_cast<index_t>(pitch / sizeof(DType));\n  } else {\n    obj->stride_ = obj->size(dim - 1);\n    MSHADOW_CUDA_CALL(cudaMallocPitch(reinterpret_cast<void**>(&(obj->dptr_)), &pitch,\n                                      obj->shape_.Size() * sizeof(DType), 1));\n  }\n}\ntemplate<int dim, typename DType>\ninline void FreeSpace(Tensor<gpu, dim, DType> *obj) {\n  MSHADOW_CUDA_CALL(cudaFree(obj->dptr_));\n  obj->dptr_ = NULL;\n}\ntemplate<typename A, typename B, int dim, typename DType>\ninline void Copy(Tensor<A, dim, DType> _dst,\n                 Tensor<B, dim, DType> _src,\n                 cudaMemcpyKind kind,\n                 Stream<gpu> *stream) {\n  CHECK_EQ(_dst.shape_, _src.shape_) << \"Copy:shape mismatch\";\n  Tensor<A, 2, DType> dst = _dst.FlatTo2D();\n  Tensor<B, 2, DType> src = _src.FlatTo2D();\n  MSHADOW_CUDA_CALL(cudaMemcpy2DAsync(dst.dptr_, dst.stride_ * sizeof(DType),\n                                      src.dptr_, src.stride_ * sizeof(DType),\n                                      dst.size(1) * sizeof(DType),\n                                      dst.size(0), kind,\n                                      Stream<gpu>::GetStream(stream)));\n  // use synchronize call behavior for zero stream\n  if (stream == NULL) {\n    MSHADOW_CUDA_CALL(cudaStreamSynchronize(0));\n  }\n}\ntemplate<int dim, typename DType>\ninline void Copy(Tensor<cpu, dim, DType> dst,\n                 const Tensor<gpu, dim, DType> &src,\n                 Stream<gpu> *stream) {\n  Copy(dst, src, cudaMemcpyDeviceToHost, stream);\n}\ntemplate<int dim, typename DType>\ninline void Copy(Tensor<gpu, dim, DType> dst,\n                 const Tensor<gpu, dim, DType> &src,\n                 Stream<gpu> *stream) {\n  Copy(dst, src, cudaMemcpyDeviceToDevice, stream);\n}\ntemplate<int dim, typename DType>\ninline void Copy(Tensor<gpu, dim, DType> dst,\n                 const Tensor<cpu, dim, DType> &src,\n                 Stream<gpu> *stream) {\n  Copy(dst, src, cudaMemcpyHostToDevice, stream);\n}\n#endif  // MSHADOW_USE_CUDA\n}  // namespace mshadow\n\n// the following part is included only if compiler is nvcc\n#ifdef __CUDACC__\n#include \"./cuda/tensor_gpu-inl.cuh\"\n\nnamespace mshadow {\ntemplate<typename Saver, typename R, int dim,\n         typename DType, typename E, int etype>\ninline void MapExp(TRValue<R, gpu, dim, DType> *dst,\n                   const expr::Exp<E, DType, etype> &exp) {\n  expr::TypeCheckPass<expr::TypeCheck<gpu, dim, DType, E>::kMapPass>\n      ::Error_All_Tensor_in_Exp_Must_Have_Same_Type();\n  Shape<dim> eshape = expr::ShapeCheck<dim, E>::Check(exp.self());\n  Shape<dim> dshape = expr::ShapeCheck<dim, R>::Check(dst->self());\n  CHECK(eshape[0] == 0 || eshape == dshape)\n    << \"Assignment: Shape of Tensors are not consistent with target, \"\n    << \"eshape: \" << eshape << \" dshape:\" << dshape;\n  cuda::MapPlan<Saver>(MakePlan(dst->self()),\n                       MakePlan(exp.self()),\n                       dshape.FlatTo2D(),\n                       Stream<gpu>::GetStream(expr::StreamInfo<gpu, R>::Get(dst->self())));\n}\n\ntemplate<typename Saver, typename Reducer,\n         typename R, typename DType, typename E, int etype>\ninline void MapReduceKeepLowest(TRValue<R, gpu, 1, DType> *dst,\n                                const expr::Exp<E, DType, etype> &exp,\n                                DType scale) {\n  expr::TypeCheckPass<expr::TypeCheck<gpu, 1, DType, E>::kRedPass>\n      ::Error_TypeCheck_Not_Pass_For_Reduce_Exp();\n  Shape<2> eshape = expr::ShapeCheck<expr::ExpInfo<E>::kDim, E>\n      ::Check(exp.self()).FlatTo2D();\n  Shape<1> dshape = expr::ShapeCheck<1, R>::Check(dst->self());\n  CHECK_EQ(eshape[1], dshape[0]) << \"MapReduceKeepLowest::reduction dimension do not match\";\n  CHECK_NE(eshape[0], 0U) << \"can not reduce over empty tensor\";\n  cuda::MapReduceKeepLowest<Saver, Reducer>\n      (MakePlan(dst->self()), MakePlan(exp.self()), scale, eshape,\n       Stream<gpu>::GetStream(expr::StreamInfo<gpu, R>::Get(dst->self())));\n}\n\ntemplate<typename Saver, typename Reducer, int dimkeep,\n         typename R, typename DType, typename E, int etype>\ninline void MapReduceKeepHighDim(TRValue<R, gpu, 1, DType> *dst,\n                                 const expr::Exp<E, DType, etype> &exp,\n                                 DType scale) {\n  expr::TypeCheckPass<expr::TypeCheck<gpu, dimkeep, DType, E>::kRedPass>\n      ::Error_TypeCheck_Not_Pass_For_Reduce_Exp();\n  typedef Shape<expr::ExpInfo<E>::kDim> EShape;\n  EShape eshape = expr::ShapeCheck<expr::ExpInfo<E>::kDim, E>\n      ::Check(exp.self());\n    Shape<1> dshape = expr::ShapeCheck<1, R>::Check(dst->self());\n  CHECK_EQ(eshape[dimkeep], dshape[0]) << \"MapReduceKeepHighDim::reduction dimension do not match\";\n  // use equvalent form\n  Shape<4> pshape = Shape4(eshape.ProdShape(0, dimkeep),\n                           eshape[dimkeep],\n                           eshape.ProdShape(dimkeep + 1, EShape::kSubdim),\n                           eshape[EShape::kSubdim]);\n  // call equavalent map red dim 2\n  cuda::MapReduceKeepDim1<Saver, Reducer>\n      (MakePlan(dst->self()), MakePlan(exp.self()), scale, pshape,\n       Stream<gpu>::GetStream(expr::StreamInfo<gpu, R>::Get(dst->self())));\n}\ntemplate<typename DType>\ninline void Softmax(Tensor<gpu, 2, DType> dst,\n                    const Tensor<gpu, 2, DType>& src) {\n  cuda::Softmax(dst, src);\n}\n\ntemplate<typename DType>\ninline void Softmax(Tensor<gpu, 3, DType> dst,\n                    const Tensor<gpu, 3, DType>& src) {\n  cuda::Softmax(dst, src);\n}\n\ntemplate<typename DType>\ninline void SoftmaxGrad(const Tensor<gpu, 2, DType> &dst,\n                        const Tensor<gpu, 2, DType> &src,\n                        const Tensor<gpu, 1, DType> &label) {\n  cuda::SoftmaxGrad(dst, src, label);\n}\n\ntemplate<typename DType>\ninline void SmoothSoftmaxGrad(const Tensor<gpu, 2, DType> &dst,\n                              const Tensor<gpu, 2, DType> &src,\n                              const Tensor<gpu, 1, DType> &label,\n                              const float alpha) {\n  cuda::SmoothSoftmaxGrad(dst, src, label, alpha);\n}\n\ntemplate<typename DType>\ninline void SoftmaxGrad(const Tensor<gpu, 2, DType> &dst,\n                        const Tensor<gpu, 2, DType> &src,\n                        const Tensor<gpu, 1, DType> &label,\n                        const DType &ignore_label) {\n  cuda::SoftmaxGrad(dst, src, label, ignore_label);\n}\n\ntemplate<typename DType>\ninline void SmoothSoftmaxGrad(const Tensor<gpu, 2, DType> &dst,\n                              const Tensor<gpu, 2, DType> &src,\n                              const Tensor<gpu, 1, DType> &label,\n                              const DType &ignore_label,\n                              const float alpha) {\n  cuda::SmoothSoftmaxGrad(dst, src, label, ignore_label, alpha);\n}\n\ntemplate<typename DType>\ninline void SoftmaxGrad(const Tensor<gpu, 3, DType> &dst,\n                        const Tensor<gpu, 3, DType> &src,\n                        const Tensor<gpu, 2, DType> &label) {\n  cuda::SoftmaxGrad(dst, src, label);\n}\n\ntemplate<typename DType>\ninline void SoftmaxGrad(const Tensor<gpu, 3, DType> &dst,\n                        const Tensor<gpu, 3, DType> &src,\n                        const Tensor<gpu, 2, DType> &label,\n                        const DType &ignore_label) {\n  cuda::SoftmaxGrad(dst, src, label, ignore_label);\n}\n\ntemplate<bool clip, typename IndexType, typename DType>\ninline void AddTakeGrad(Tensor<gpu, 2, DType> dst,\n                        const Tensor<gpu, 1, IndexType>& index,\n                        const Tensor<gpu, 2, DType> &src) {\n  cuda::AddTakeGrad<clip, IndexType, DType>(dst, index, src);\n}\n\ntemplate<bool clip, typename IndexType, typename DType, typename AType>\ninline void AddTakeGrad(Tensor<gpu, 2, DType> dst,\n                        Tensor<gpu, 2, AType> temp,\n                        const Tensor<gpu, 1, IndexType>& index,\n                        const Tensor<gpu, 2, DType> &src) {\n  cuda::AddTakeGrad<clip, IndexType, DType>(dst, temp, index, src);\n}\n\ntemplate<typename IndexType, typename DType>\ninline void AddTakeGradLargeBatch(Tensor<gpu, 2, DType> dst,\n                                  const Tensor<gpu, 1, IndexType>& sorted,\n                                  const Tensor<gpu, 1, IndexType>& index,\n                                  const Tensor<gpu, 2, DType> &src) {\n  cuda::AddTakeGradLargeBatch(dst, sorted, index, src);\n}\n\ntemplate<typename KDType, typename VDType>\ninline void SortByKey(Tensor<gpu, 1, KDType> keys, Tensor<gpu, 1, VDType> values,\n                      bool is_ascend) {\n  cuda::SortByKey(keys, values, is_ascend);\n}\n\ntemplate<typename IndexType, typename DType>\ninline void IndexFill(Tensor<gpu, 2, DType> dst,\n                      const Tensor<gpu, 1, IndexType>& index,\n                      const Tensor<gpu, 2, DType> &src) {\n  cuda::IndexFill(dst, index, src);\n}\n}  // namespace mshadow\n#endif  // __CUDACC__\n#endif  // MSHADOW_TENSOR_GPU_INL_H_\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow-ps/.gitignore",
    "content": "Makefile\ntest\ntest.cpp\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow-ps/README.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nmshadow-ps\n====\nThis folder contains mshadow-ps parameter server interface for mshadow GPU/CPU Tensor. See [guide on mshadow-ps](../guide/mshadow-ps) for introduction of the interface.\n\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow-ps/mshadow_ps.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file mshadow_ps.h\n * \\brief parameter server abstraction for mshadow tensor\n *  this is a plugin of mshadow that can be used to syncrhonize\n *  parameters across device and machines\n *\n * \\author Tianqi Chen, Mu Li\n */\n#ifndef MSHADOW_PS_H_  // NOLINT(*)\n#define MSHADOW_PS_H_  // NOLINT(*)\n#include <vector>\n// optionally support of lambda function in C++11, if available\n#if __cplusplus >= 201103L\n#include <functional>\n#endif  // C++11\n#include \"../mshadow/tensor.h\"\n\n/*! \\brief whether to adapt distributed PS from parameter-server */\n#ifndef MSHADOW_DIST_PS\n#define MSHADOW_DIST_PS 1\n#endif\n\n/*! \\brief whether to support BSP rabit API of PS*/\n#ifndef MSHADOW_RABIT_PS\n#define MSHADOW_RABIT_PS 1\n#endif\n\nnamespace mshadow {\n/*! \\brief namespace of mshadow-ps */\nnamespace ps {\n/*!\n * \\brief interface of parameter server\n * \\tparam xpu the device of the data lies\n * \\tparam DType the type of element in the tensor\n */\ntemplate<typename xpu,\n         typename DType MSHADOW_DEFAULT_DTYPE>\nclass ISharedModel {\n public:\n  /*!\n   * \\brief callback function that will be executed when pull request finishes\n   *        before calling the callback, the thread context is already switched\n   *        to the device of pullrequest\n   * \\param stream the stream of callback thread, it is recommended to operate using this stream\n   * \\param arg the argument of callback function\n   */\n  typedef void (CallbackFunction) (Stream<xpu> *stream, void *arg);\n  /*! \\brief virtual destructor */\n  virtual ~ISharedModel(void) {}\n  /*!\n   * \\brief Set param for the layer from string\n   * \\param name parameter name\n   * \\param val string for configuration\n   */\n  virtual void SetParam(const char *name, const char *val) {}\n  /*!\n   * \\brief initialize the paramerver server client\n   * \\param devices specifies the possible device id\n   *   to be input from Push and Pull,\n   */\n  virtual void Init(const std::vector<int> &devices) {}\n  /*!\n   * \\brief initialize the paramerver server client\n   * without specifying the devices, only device 0 is allowed\n   */\n  inline void Init(void) {\n    std::vector<int> dev;\n    dev.push_back(0);\n    this->Init(dev);\n  }\n  /*!\n   * \\brief initialize a key with certain shape\n   *  must be called before using Push/PullReq/PullWait\n   *  on the corresponding key\n   * \\param shape the shape content of the key\n   * \\param key the unique key to indicate the tensor\n   *        this is unique per device\n   * \\param devid the device id this tensor lies in\n   */\n  template<int dim>\n  inline void InitKey(Shape<dim> shape,\n                      int key, int devid) {\n    this->InitKey_(shape.FlatTo2D(), key, devid);\n  }\n  /*!\n   * \\brief wait until the pull event finishes\n   * if there was no pull request, wait will directly returns\n   * \\param key the unique key to indicate the tensor\n   *        this is unique per device\n   * \\param devid the device id this tensor lies in\n   */\n  virtual void PullWait(int key, int devid) = 0;\n  /*!\n   * \\brief check if the weight was correct on the current device\n   *\n   * \\param data the data\n   * \\param key the unique key to indicate the tensor\n   *        this is unique per device\n   * \\param devid the device id this tensor lies in\n   */\n  template<int dim>\n  inline void CheckWeight(Tensor<xpu, dim, DType> data,\n                          int key,\n                          int devid) {\n    this->CheckWeight_(data.FlatTo2D(), key, devid);\n  }\n  /*!\n   * \\brief push out a tensor to parameter server\n   *  this call is asynchronize and returns immediately\n   *\n   * \\param data the data\n   * \\param key the unique key to indicate the tensor\n   *        this is unique per device\n   * \\param devid the device id this tensor lies in\n   * \\param priority the priority of this operation,\n   *   the bigger the number is the higher the priority will be\n   */\n  template<int dim>\n  inline void Push(Tensor<xpu, dim, DType> data,\n                   int key,\n                   int devid,\n                   int priority = 0) {\n    this->Push_(data.FlatTo2D(), key, devid, priority);\n  }\n  /*!\n   * \\brief send a pull request, to pull parameter into data\n   *  this call is asynchronize and returns immediately\n   *  use PullWait to wait the event of copy finish\n   *\n   * \\param data the data\n   * \\param key the unique key to indicate the tensor,\n   *        this is unique per device\n   * \\param devid the device id this tensor lies in\n   * \\param priority the priority of this operation,\n   *   the bigger the number is the higher the priority will be\n   * \\param callback the callback function that will\n   *                 be invoked when the request finishes\n   * \\param callback_arg the argument to pass to callback\n   */\n  template<int dim>\n  inline void PullReq(Tensor<xpu, dim, DType> data,\n                      int key,\n                      int devid,\n                      int priority = 0,\n                      CallbackFunction callback = NULL,\n                      void *callback_arg = NULL) {\n    this->PullReq_(data.FlatTo2D(), key,\n                   devid, priority, callback, callback_arg);\n  }\n#if __cplusplus >= 201103L\n  /*!\n   * \\brief send a pull request, to pull parameter into data\n   *  this call is asynchronize and returns immediately\n   *  use PullWait to wait the event of copy finish\n   *  this is the c++11 version that allows lambda function as callback\n   * \\param data the data\n   * \\param key the unique key to indicate the tensor,\n   *        this is unique per device\n   * \\param devid the device id this tensor lies in\n   * \\param priority the priority of this operation,\n   *   the bigger the number is the higher the priority will be\n   * \\param callback the callback function\n   */\n  template<int dim>\n  inline void PullReq(Tensor<xpu, dim, DType> data,\n                      int key,\n                      int devid,\n                      int priority,\n                      std::function<void(Stream<xpu> *stream)> callback) {\n    // need to allocate space, because callback can happen latter..\n    auto calbk = new std::function<void(Stream<xpu> *stream)>();\n    *calbk = callback;\n    this->PullReq(data, key, devid, priority, InvokeLambda_, calbk);\n  }\n#endif  // C++11\n\n  /*!\n   * \\brief set weight of corresponding key in server\n   *   this is a debug function that was not necessarily\n   *   implemented by the server\n   * \\param data the data to set\n   * \\param key the unique key to indicate the tensor\n   *        this is unique per device\n   * \\param devid the device id this tensor lies in\n   */\n  virtual void SetWeight_(Tensor<xpu, 2, DType> data,\n                          int key,\n                          int devid) = 0;\n  /*!\n   * \\brief check if the weight matches the server side\n   *   this is a debug function that was not necessarily\n   *   implemented by the server\n   * \\param data the data to set\n   * \\param key the unique key to indicate the tensor\n   *        this is unique per device\n   * \\param devid the device id this tensor lies in\n   */\n  virtual void CheckWeight_(Tensor<xpu, 2, DType> data,\n                            int key,\n                            int devid) = 0;\n\n protected:\n  /*!\n   * \\brief initialize a key with certain shape\n   * \\param shape the shape content of the key\n   * \\param key the unique key to indicate the tensor\n   *        this is unique per device\n   * \\param devid the device id this tensor lies in\n   */\n  virtual void InitKey_(Shape<2> shape,\n                        int key, int devid) = 0;\n  /*!\n   * \\brief push out a tensor to parameter server\n   *  this call is asynchronize and returns immediately\n   *\n   * \\param data the data\n   * \\param key the unique key to indicate the tensor\n   *        this is unique per device\n   * \\param devid the device id this tensor lies in\n   * \\param priority the priority of this operation,\n   *   the bigger the number is the higher the priority will be\n   */\n  virtual void Push_(Tensor<xpu, 2, DType> data,\n                     int key,\n                     int devid,\n                     int priority = 0) = 0;\n  /*!\n   * \\brief send a pull request, to pull parameter into data\n   *  this call is asynchronize and returns immediately\n   *  use PullWait to wait the event of copy finish\n   *\n   * \\param data the data\n   * \\param key the unique key to indicate the tensor,\n   *        this is unique per device\n   * \\param devid the device id this tensor lies in\n   * \\param priority the priority of this operation,\n   *   the bigger the number is the higher the priority will be\n   * \\param callback the callback function that will\n   *                 be invoked when the request finishes\n   * \\param callback_arg the argument to pass to callback\n   */\n  virtual void PullReq_(Tensor<xpu, 2, DType> data,\n                        int key,\n                        int devid,\n                        int priority,\n                        CallbackFunction callback,\n                        void *callback_arg) = 0;\n\n private:\n// C++11 support for lambda prepare function\n#if __cplusplus >= 201103L\n  /*! \\brief hack function to convert lambda to callback function */\n  inline static void InvokeLambda_(Stream<xpu> *stream, void *fun) {\n    auto *fp = static_cast<std::function<void(Stream<xpu> *stream)>*>(fun);\n    (*fp)(stream);\n    delete fp;\n  }\n#endif  // C++11\n};\n/*! \\brief interface for customized mshadow server */\ntemplate<typename DType>\nclass IModelUpdater {\n public:\n  virtual ~IModelUpdater(void) {}\n  /*!\n   * \\brief set parameters from outside\n   * \\param name name of parameter\n   * \\param val value of parameter\n   */\n  virtual void SetParam(const char *name, const char *val) {}\n  /*!\n   * \\brief init the model updater\n   * \\param rank the rank of the node\n   * \\param argc number of arguments\n   * \\param argv arguments\n   */\n  virtual void InitUpdater(int rank, int argc, char *argv[]) {}\n  /*!\n   * \\brief initialize the model\n   * \\param key the key of data we point to\n   * \\param dptr the data pointer\n   * \\param size size of the parameter key\n   */\n  virtual void InitModel(int key, DType *dptr, size_t size) {\n    this->InitModel_(key, Tensor<cpu, 1, DType>(dptr, Shape1(size)));\n  }\n  /*!\n   * update the model\n   * \\param key the key of data we point to\n   * \\param dptr the data pointer\n   * \\param size size of the parameter key\n   */\n  virtual void Update(int key, DType *dptr, size_t size) {\n    this->Update_(key, Tensor<cpu, 1, DType>(dptr, Shape1(size)));\n  }\n\n protected:\n  /*!\n   * \\brief initialize the model, user can implement this one\n   *   to take advantage of tensor operations\n   * \\param key the key of data we point to\n   * \\param data the tensor data corresponding to the data we want to initialize\n   */\n  virtual void InitModel_(int key, Tensor<cpu, 1, DType> data) {\n    LOG(FATAL) << \"InitModel: not implemented\";\n  }\n  /*!\n   * \\brief update the model, user can implement this one\n   *    to take advantage of tensor operations\n   * \\param key the key of data we point to\n   * \\param data the tensor data corresponding to the data we want to initialize\n   */\n  virtual void Update_(int key, Tensor<cpu, 1, DType> data) {\n    LOG(FATAL) << \"InitModel: not implemented\";\n  }\n};\n/*!\n * \\brief create customized server\n * this is a server defined by user\n * \\return new server\n */\ntemplate<typename DType>\nIModelUpdater<DType> *CreateModelUpdater(void);\n}  // namespace ps\n}  // namespace mshadow\n\n#include \"./ps_local-inl.h\"\n#include \"./ps_dist-inl.h\"\n#include \"./ps_rabit-inl.h\"\nnamespace mshadow {\nnamespace ps {\n/*!\n * \\brief create a parameter server implementation\n * \\param type the type of paramerver server\n *     can either be \"local\" or \"dist\"\n * \\return the ISharedModel that can be used to synchronize weights\n */\ntemplate<typename xpu, typename DType>\ninline ISharedModel<xpu, DType> *CreateSharedModel(const char *type) {\n  if (!strcmp(\"local\", type)) {\n#if MSHADOW_RABIT_PS\n    // allreduce on one machine pays no cost\n    if (rabit::IsDistributed()) {\n      return new RabitModel<xpu, DType>();\n    }\n#endif\n    return new LocalModel<xpu, DType>();\n  }\n#if MSHADOW_DIST_PS\n  if (!strcmp(\"dist\", type)) return new DistModel<xpu, DType>();\n#endif\n  LOG(FATAL) << \"unknown server type \" << type;\n  return NULL;\n}\n}  // namespace ps\n}  // namespace mshadow\n#endif  // MSHADOW_PS_H_  NOLINT(*)\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow-ps/ps_dist-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file ps_dist-inl.h\n * \\brief distributed version of PS\n *\n * \\author Tianqi Chen, Mu Li\n */\n#ifndef MSHADOW_PS_DIST_INL_H_ // NOLINT(*)\n#define MSHADOW_PS_DIST_INL_H_ // NOLINT(*)\n\n#include <vector>\n#include \"./mshadow_ps.h\"\n#include \"./ps_local-inl.h\"\n\n#if MSHADOW_DIST_PS\n#include \"parameter/kv_layer.h\"\nnamespace mshadow {\nnamespace ps {\n\n/**\n * @brief bridge IModelUpdater to KVLayerUpdater\n */\ntemplate<typename DType>\nclass UpdaterWrapper {\n public:\n  explicit UpdaterWrapper(IModelUpdater<DType> * updater)\n      : updater_(updater) { }\n  ~UpdaterWrapper() { delete updater_; }\n\n  /// @brief initialize the data\n  void Init(int id, size_t size, DType* data) {\n    updater_->InitModel(id, data, size);\n  }\n\n  /// @brief update the model by using received data\n  void Update(int id, size_t size, const DType* recv_data, DType* data) {\n    updater_->Update(id, (DType*)recv_data, size);  // NOLINT(*)\n  }\n private:\n  IModelUpdater<DType> *updater_;\n};\n\n\ntemplate<typename xpu, typename DType>\nclass DistModel : public LocalModel<xpu, DType> {\n public:\n  // parent type\n  typedef LocalModel<xpu, DType> Parent;\n\n  // initialize the parameter server\n  virtual void Init(const std::vector<int> &devices) {\n    Parent::Init(devices);\n    if (this->custom_server != NULL) {\n      delete this->custom_server;\n      this->custom_server = NULL;\n    }\n  }\n  virtual ~DistModel(void) {\n  }\n\n protected:\n  // do nothing\n  virtual void InitCustomerServer(void) {\n  }\n  virtual void ServerInitKey(Tensor<cpu, 2> weight, int key) {\n    // this is called when key get initialized for the first time\n    // weight can be used to hold the model that pulled back\n    // use this to initialize the key on serverside\n    shared_model_.Pull(\n        ::ps::Parameter::Request(key), weight.dptr_, weight.MSize(),\n        [this, weight, key]() {\n          // call PullReady to notify LocalServer pulling is ready\n          this->PullReady(weight, key);\n        });\n  }\n  // override this function, to use parameter server\n  virtual void HandlePushFinish(Tensor<cpu, 3, DType> data,\n                                int key) {\n    // summation the data fron all devices\n    LocalModel<xpu, DType>::ReduceSum(data);\n\n    // push and pull\n    Tensor<cpu, 2> sendrecv = data[0];\n    CHECK_EQ(data[0].CheckContiguous(), true) << \"data must be contiguous\";\n\n    int ts = shared_model_.Push(\n        ::ps::Parameter::Request(key), sendrecv.dptr_, sendrecv.MSize(), false);\n\n    // let this pull request wait the push finish at the server node\n    shared_model_.Pull(\n        ::ps::Parameter::Request(key, -1, {ts}), sendrecv.dptr_, sendrecv.MSize(),\n        [this, sendrecv, key]() {\n          // call PullReady to notify LocalServer pulling is ready\n          this->PullReady(sendrecv, key);\n        });\n  }\n\n private:\n  ::ps::KVLayer<DType, UpdaterWrapper<DType> > shared_model_;\n};\n\n\ntemplate<typename DType>\nclass MShadowServerNode {\n public:\n  // conf: get from the flag -app_conf\n  MShadowServerNode(int argc, char *argv[]) {\n    IModelUpdater<DType> *updater = CreateModelUpdater<DType>();\n    updater->InitUpdater(::ps::MyRank(), argc, argv);\n\n    UpdaterWrapper<DType> *wrapper = new UpdaterWrapper<DType>(updater);\n    typedef ::ps::KVLayer<DType, UpdaterWrapper<DType> > PSServer;\n    PSServer *shared_model_ = new PSServer();\n    shared_model_->set_updater(wrapper);\n    ::ps::Postoffice::instance().manager().TransferCustomer(\n         CHECK_NOTNULL(shared_model_));\n  }\n  virtual ~MShadowServerNode() { }\n};\n\n// NOTE: do not add PS::CreateServer here add it in the program that uses\n// mshadow-ps\n}  // namespace ps\n}  // namespace mshadow\n#endif\n#endif  // MSHADOW_PS_DIST_INL_H_  NOLINT(*)\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow-ps/ps_local-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file ps_local-inl.h\n * \\brief local multi-threading implementation of PS abstraction\n *\n * \\author Tianqi Chen, Mu Li\n */\n#ifndef MSHADOW_PS_LOCAL_INL_H_  // NOLINT(*)\n#define MSHADOW_PS_LOCAL_INL_H_  // NOLINT(*)\n#include <map>\n#include <utility>\n#include <string>\n#include <vector>\n#if defined(_OPENMP)\n#include <omp.h>\n#ifdef _MSC_VER\ntypedef int ms_omp_uint;\n#else\ntypedef unsigned ms_omp_uint;\n#endif\n#endif\n\n#include \"./thread.h\"\n#include \"./thread_util.h\"\n\nnamespace mshadow {\nnamespace ps {\n// multi-threaded implementation of\ntemplate<typename xpu, typename DType>\nclass LocalModel : public ISharedModel<xpu, DType> {\n public:\n  // redefine callback function\n  typedef typename ISharedModel<xpu, DType>::CallbackFunction\n  CallbackFunction;\n  // constructor\n  LocalModel(void) {\n    init_end = 0;\n    perdev_pull_thread = 1;\n    perdev_push_thread = 1;\n    use_fifo_push_queue = 0;\n    bigarray_bound = 1000 * 1000;\n    nthread_reduction = 8;\n    use_pin_memory = 1;\n    test_on_server = 0;\n    update_on_server = 0;\n    destroy_signal = false;\n    custom_server = NULL;\n  }\n  // destructor\n  virtual ~LocalModel(void) {\n    this->Destroy();\n  }\n  inline void Destroy(void) {\n    if (init_end != 0) {\n      destroy_signal = true;\n      for (size_t i = 0; i < push_queues.size(); ++i) {\n        push_queues[i].Abort(1);\n      }\n      for (size_t i = 0; i < pull_queues.size(); ++i) {\n        pull_queues[i].Abort(1);\n      }\n      for (size_t i = 0; i < thread_push_handler.size(); ++i) {\n        thread_push_handler[i].Join();\n      }\n      for (size_t i = 0; i < thread_pull_handler.size(); ++i) {\n        thread_pull_handler[i].Join();\n      }\n      for (size_t i = 0; i < push_queues.size(); ++i) {\n        push_queues[i].Destroy();\n      }\n      push_map.Destroy();\n      push_lock.Destroy();\n      for (size_t i = 0; i < pull_queues.size(); ++i) {\n        pull_queues[i].Destroy();\n      }\n      pull_map.Destroy();\n      request_lock.Destroy();\n      wait_lock.Destroy();\n      wait_cond.Destroy();\n      init_end = 0;\n    }\n    if (custom_server != NULL) {\n      delete custom_server;\n      custom_server = NULL;\n    }\n  }\n  virtual void SetParam(const char *name, const char *val) {\n    int key;\n    if (sscanf(name, \"push_op[%d]\", &key) == 1) {\n      if (!strcmp(val, \"gather\")) {\n        request_lock.Lock();\n        push_operation[key] = kGather;\n        request_lock.Unlock();\n        return;\n      }\n      if (!strcmp(val, \"sum\")) {\n        push_operation[key] = kSum; return;\n      }\n      LOG(FATAL) << \"unknown push operation \" << val;\n    }\n    if (!strcmp(name, \"reduce_thread\")) {\n      nthread_reduction = atoi(val);\n    }\n    if (!strcmp(name, \"use_pin_memory\")) {\n      use_pin_memory = atoi(val);\n    }\n    if (!strcmp(name, \"bigarray_bound\")) {\n      bigarray_bound = static_cast<size_t>(atol(val));\n    }\n    if (!strcmp(name, \"pull_thread\")) {\n      if (!strcmp(val, \"ndev\")) {\n        perdev_pull_thread = 1;\n      } else if (!strcmp(val, \"one\")) {\n        perdev_pull_thread = 0;\n      } else {\n        LOG(FATAL) << \"invalid value for parameter pull_thread,\" << \" can only be ndev or one\";\n      }\n    }\n    if (!strcmp(name, \"push_thread\")) {\n      if (!strcmp(val, \"ndev\")) {\n        perdev_push_thread = 1;\n      } else if (!strcmp(val, \"one\")) {\n        perdev_push_thread = 0;\n      } else {\n        LOG(FATAL) << \"invalid value for parameter push_thread,\" << \" can only be ndev or one\";\n      }\n    }\n    if (!strcmp(name, \"update_on_server\")) {\n      update_on_server = atoi(val);\n    }\n    if (!strcmp(name, \"test_on_server\")) {\n      test_on_server = atoi(val);\n    }\n    // ignore message parameter\n    if (!strncmp(name, \"msg:\", 4)) return;\n    cfgvec.push_back(std::make_pair(std::string(name),\n                                    std::string(val)));\n  }\n  virtual void PullWait(int key, int devid) {\n    const int wid = GetWorkIndex(devid);\n    PullEntry *p = pull_map.Get(key);\n    if (p == NULL || p->wait.size() == 0) return;\n    PullEntry &e = *p;\n    // wake up waiters if any\n    CHECK_EQ(e.wait.size(), devices.size()) << \"PullWait: must initialize the wait\";\n    PullWaitRecord &w = e.wait[wid];\n    if (!w.finished) {\n      wait_lock.Lock();\n      w.nwait += 1;\n      while (!w.finished) {\n        wait_cond.Wait(&wait_lock);\n      }\n      w.nwait -= 1;\n      CHECK_GE(w.nwait, 0) << \"boundary check\";\n      wait_lock.Unlock();\n    }\n  }\n  virtual void Init(const std::vector<int> &devices) {\n    CHECK_EQ(init_end, 0) << \"LocalServer.Init can only call Init once\";\n    CHECK_NE(devices.size(), 0) << \"LocalServer.Init: must at least contain 1 devices\";\n    this->devices = devices;\n    destroy_signal = false;\n    // initialize device id to local index\n    dev2index.clear();\n    for (size_t i = 0; i < devices.size(); ++i) {\n      int devid = devices[i];\n      CHECK_GE(devid, 0) << \"device id must be bigger than 0\";\n      if (devid >= static_cast<int>(dev2index.size())) {\n        dev2index.resize(devid + 1, -1);\n      }\n      dev2index[devid] = static_cast<int>(i);\n    }\n    // allocate space\n    pull_stream.resize(devices.size());\n    push_stream.resize(devices.size());\n    // initialize all the thread related things\n    if (perdev_push_thread != 0) {\n      push_queues.resize(devices.size());\n    } else {\n      push_queues.resize(1);\n    }\n    for (size_t i = 0; i < push_queues.size(); ++i) {\n      push_queues[i].Init(use_fifo_push_queue != 0);\n    }\n    push_map.Init();\n    push_lock.Init();\n    pull_map.Init();\n    request_lock.Init();\n    wait_lock.Init();\n    wait_cond.Init();\n    if (perdev_pull_thread != 0) {\n      pull_queues.resize(devices.size());\n    } else {\n      pull_queues.resize(1);\n    }\n    for (size_t i = 0; i < pull_queues.size(); ++i) {\n      pull_queues[i].Init();\n    }\n    // initialize the thread\n    if (perdev_push_thread != 0) {\n      thread_push_handler.resize(devices.size());\n      for (size_t i = 0; i < devices.size(); ++i) {\n        std::pair<LocalModel*, size_t> *p\n            = new std::pair<LocalModel*, size_t>();\n        *p = std::make_pair(this, i);\n        thread_push_handler[i].Start(PushLocalThread, p);\n      }\n    } else {\n      thread_push_handler.resize(1);\n      thread_push_handler[0].Start(PushGlobalThread, this);\n    }\n    // initialize pull handler\n    if (perdev_pull_thread != 0) {\n      thread_pull_handler.resize(devices.size());\n      for (size_t i = 0; i < devices.size(); ++i) {\n        std::pair<LocalModel*, size_t> *p\n            = new std::pair<LocalModel*, size_t>();\n        *p = std::make_pair(this, i);\n        thread_pull_handler[i].Start(PullLocalThread, p);\n      }\n    } else {\n      thread_pull_handler.resize(1);\n      thread_pull_handler[0].Start(PullGlobalThread, this);\n    }\n    this->InitCustomerServer();\n    this->init_end = 1;\n  }\n\n  // set weight\n  virtual void SetWeight_(Tensor<xpu, 2, DType> data,\n                          int key,\n                          int devid) {\n    PushEntry &e = push_map.GetRef(key);\n    Stream<xpu> s;\n    push_lock.Lock();\n    mshadow::Copy(e.weight, data, &s);\n    push_lock.Unlock();\n  }\n  virtual void CheckWeight_(Tensor<xpu, 2, DType> data,\n                            int key,\n                            int devid) {\n    CHECK_NE(test_on_server, 0) << \"must be in pair debug mode\";\n    PushEntry &e = push_map.GetRef(key);\n    mshadow::TensorContainer<cpu, 2, DType> tmp(false);\n    tmp.Resize(data.shape_);\n    Stream<xpu> s;\n    push_lock.Lock();\n    // copy data\n    mshadow::Copy(tmp, data, &s);\n    index_t count = tmp.shape_.Size();\n    double diff = 0.0, ssum = 0.0, maxdiff = 0.0;\n    index_t mxidx = 0;\n    for (index_t i = 0; i < count; ++i) {\n      double d = std::abs(tmp.dptr_[i] - e.weight.dptr_[i]);\n      if (d > maxdiff) {\n        maxdiff = d; mxidx = i;\n      }\n      diff += d;\n      ssum += std::abs(tmp.dptr_[i]);\n    }\n    push_lock.Unlock();\n    // relative absolute error\n    double rerr = diff / ssum;\n    if (rerr > 1e-5 || diff != diff) {\n      fprintf(stderr, \"PSLocal:key=%d,dev=%d: err=%f, maxd[%u]=%f, diff=%f, ssum=%f\\n\",\n              key, devid, rerr, mxidx, maxdiff, diff, ssum);\n    } else {\n      fprintf(stderr, \"PSLocal:key=%d,dev=%d:check pass\\n\", key, devid);\n    }\n  }\n\n protected:\n  /*! \\brief operation performed locally in PS */\n  enum LocalOp {\n    /*! \\brief take sum of all devices over the same key */\n    kSum = 0,\n    /*!\n     * \\brief concatenate(gather),\n     *  the tensors in all devices with same key\n     */\n    kGather = 1\n  };\n  virtual void InitKey_(Shape<2> shape,\n                        int key, int devid) {\n    this->InitPullMap(key);\n    this->InitPushMap(key, shape);\n  }\n  virtual void Push_(Tensor<xpu, 2, DType> data,\n                     int key, int devid, int priority) {\n    PullEntry &e = pull_map.GetRef(key);\n    e.req[GetWorkIndex(devid)].ready = false;\n    if (perdev_push_thread != 0) {\n      int wid = GetWorkIndex(devid);\n      push_queues[wid].Push(PullTask(data, key, devid), priority);\n    } else {\n      push_queues[0].Push(PullTask(data, key, devid), priority);\n    }\n  }\n  virtual void PullReq_(Tensor<xpu, 2, DType> data,\n                        int key, int devid, int priority,\n                        CallbackFunction callback,\n                        void *callback_arg) {\n    PullEntry &e = pull_map.GetRef(key);\n    CHECK_EQ(e.req.size(), devices.size()) << \"PullReq: must initialize the key, req\";\n    CHECK_EQ(e.wait.size(), devices.size()) << \"PullReq: must initialize the key, wait\";\n    const int wid = GetWorkIndex(devid);\n    PullReqRecord &r = e.req[wid];\n    r.dest = data;\n    r.priority = priority;\n    r.callback = callback;\n    r.callback_arg = callback_arg;\n    // reset pull request finish mark\n    wait_lock.Lock();\n    e.wait[wid].finished = false;\n    wait_lock.Unlock();\n    // check ready event\n    request_lock.Lock();\n    CHECK_EQ(!r.pending, true) << \"key = \" << key\n      << \"cannot send duplicate pull request before it finishes\";\n    if (e.req[wid].ready) {\n      if (perdev_pull_thread != 0) {\n        pull_queues[wid].Push(std::make_pair(key, devid));\n      } else {\n        pull_queues[0].Push(std::make_pair(key, devid));\n      }\n    } else {\n      r.pending = true;\n    }\n    request_lock.Unlock();\n  }\n  /*!\n   * \\brief called to notify that the data is ready for pull\n   * \\param data the data that can be pulled back\n   * \\param the key of the data\n   */\n  virtual void PullReady(Tensor<cpu, 2> data, int key) {\n    PullEntry &e = pull_map.GetRef(key);\n    CHECK_EQ(e.req.size(), devices.size()) << \"PullReady: must initialize the key, req\";\n    request_lock.Lock();\n    e.src = data;\n    for (index_t i = 0; i < e.req.size(); ++i) {\n      e.req[i].ready = true;\n      if (e.req[i].pending) {\n        if (perdev_pull_thread != 0) {\n          pull_queues[i].Push(std::make_pair(key, devices[i]));\n        } else {\n          pull_queues[0].Push(std::make_pair(key, devices[i]));\n        }\n        e.req[i].pending = false;\n      }\n    }\n    request_lock.Unlock();\n  }\n  virtual void ServerInitKey(Tensor<cpu, 2> weight, int key) {\n    if (custom_server != NULL) {\n      // intialize server, and ready for pullback\n      custom_server->InitModel(key, weight.dptr_, weight.MSize());\n      if (update_on_server != 0) {\n        this->PullReady(weight, key);\n      }\n    }\n  }\n  /*!\n   * \\brief event handler for push finish\n   *  called when all the data with same key comes int\n   * \\param data the buffer holds the data in all devices\n   * \\param key the key of the data\n   */\n  virtual void HandlePushFinish(Tensor<cpu, 3, DType> data,\n                                int key) {\n    // LOG(ERROR) << dbstr(data);\n    LocalOp op = kSum;\n    typename std::map<int, LocalOp>::const_iterator\n        it = push_operation.find(key);\n    if (it != push_operation.end() && it->first == key) {\n      op = it->second;\n    }\n    // customized server\n    if (custom_server != NULL) {\n      this->ReduceSum(data);\n      custom_server->Update(key, data[0].dptr_, data[0].MSize());\n      if (update_on_server != 0) {\n        PushEntry &e = push_map.GetRef(key);\n        this->PullReady(e.weight, key);\n      } else {\n        CHECK_NE(test_on_server, 0) << \"test mode\";\n        this->PullReady(data[0], key);\n      }\n      return;\n    }\n    switch (op) {\n      case kSum: {\n        this->ReduceSum(data);\n        this->PullReady(data[0], key);\n        return;\n      }\n      case kGather: {\n        this->PullReady(data.FlatTo2D(), key);\n        return;\n      }\n      default: LOG(FATAL) << \"unknown LocalOp\";\n    }\n  }\n  /*!\n   * \\brief event handler for reduce finish\n   *  called when all the data with same key finishes the reduction\n   * \\param data the buffer holds the reduction result\n   * \\param key the key of the data\n   */\n  inline void HandleReduceFinish(Tensor<cpu, 2, DType> data,\n                                 int key) {\n    if (custom_server != NULL) {\n      custom_server->Update(key, data.dptr_, data.MSize());\n      if (update_on_server != 0) {\n        PushEntry &e = push_map.GetRef(key);\n        this->PullReady(e.weight, key);\n      } else {\n        CHECK_NE(test_on_server, 0) << \"test mode\";\n        this->PullReady(data, key);\n      }\n    } else {\n      this->PullReady(data, key);\n    }\n  }\n  virtual void InitCustomerServer(void) {\n    if (update_on_server != 0 || test_on_server != 0) {\n      custom_server = CreateModelUpdater<DType>();\n      for (size_t j = 0; j < cfgvec.size(); ++j) {\n        custom_server->SetParam(cfgvec[j].first.c_str(),\n                                cfgvec[j].second.c_str());\n      }\n      custom_server->InitUpdater(0, 0, NULL);\n    }\n  }\n\n protected:\n  // customized server\n  IModelUpdater<DType> *custom_server;\n  // whether use fifo push queue\n  int use_fifo_push_queue;\n\n  // perform sum reduction\n  inline void ReduceSum(Tensor<cpu, 3, DType> data) {\n    #if defined(_OPENMP)\n    if (data[0].MSize() >= bigarray_bound &&\n        nthread_reduction != 0) {\n      ms_omp_uint ntask = static_cast<ms_omp_uint>(data.size(1));\n      #pragma omp parallel for schedule(static) num_threads(nthread_reduction)\n      for (ms_omp_uint j = 0; j < ntask; ++j) {\n        for (index_t i = 1; i < data.size(0); ++i) {\n          data[0][j] += data[i][j];\n        }\n      }\n    } else  //NOLINT(*)\n      #endif\n    {\n      for (index_t i = 1; i < data.size(0); ++i) {\n        data[0] += data[i];\n      }\n    }\n  }\n\n private:\n  /*! \\brief task running */\n  struct PullTask {\n    /*! \\brief the task data source */\n    Tensor<xpu, 2, DType> data;\n    /*! \\brief the key to the tensor */\n    int key;\n    /*!\n     * \\brief the device id, (key,devid),\n     * uniquely identifies a mem location\n     */\n    int devid;\n    PullTask(void) {}\n    PullTask(Tensor<xpu, 2, DType> data, int key, int devid)\n        : data(data), key(key), devid(devid) {}\n  };\n  /*! \\brief data structure to hold temporal push result */\n  struct PushEntry {\n    // temporal space to hold input data\n    Tensor<cpu, 4, DType> data;\n    // temporal space to hold weight, if needed\n    Tensor<cpu, 2, DType> weight;\n    // indicator whether the certain devices is already copied in\n    std::vector<bool> copied;\n    // number of data copied in\n    int num_copied;\n    // version number of data used to hold incomming data in push\n    int copyin_version;\n    // use pinned memory\n    bool pin_memory;\n    // constructor\n    PushEntry(void)\n        : copyin_version(0) {\n      weight.dptr_ = NULL;\n    }\n    ~PushEntry(void) {\n      if (data.dptr_ != NULL) {\n        if (pin_memory) {\n          mshadow::FreeHost<xpu>(&data);\n          if (weight.dptr_ != NULL) {\n            mshadow::FreeHost<xpu>(&weight);\n          }\n        } else {\n          mshadow::FreeSpace(&data);\n          if (weight.dptr_ != NULL) {\n            mshadow::FreeSpace(&weight);\n          }\n        }\n      }\n    }\n    // constructor\n    inline void Init(int ndevice, Shape<2> shape,\n                     bool pin_memory, bool need_weight) {\n      this->pin_memory = pin_memory;\n      data.shape_ = Shape4(2, ndevice, shape[0], shape[1]);\n      weight.shape_ = shape;\n      if (pin_memory) {\n        mshadow::AllocHost<xpu>(&data);\n        if (need_weight) mshadow::AllocHost<xpu>(&weight);\n      } else {\n        mshadow::AllocSpace(&data, false);\n        if (need_weight) mshadow::AllocSpace(&weight);\n      }\n      CHECK_EQ(data.CheckContiguous(), true) << \"Data must be contiguous\";\n      CHECK(!need_weight || weight.CheckContiguous()) << \"Weight must be contiguous\";\n      num_copied = 0;\n      copied.resize(ndevice, false);\n    }\n  };\n  // a record to remember things related to pull request\n  struct PullReqRecord {\n    // whether this record contains a pending request\n    // whether pull is ready to go\n    bool ready;\n    // waiting for pull ready\n    bool pending;\n    // the destination to pull data into\n    Tensor<xpu, 2, DType> dest;\n    // the priority of the\n    int priority;\n    // callback function\n    CallbackFunction *callback;\n    // argument for callback\n    void *callback_arg;\n    PullReqRecord(void) : ready(false), pending(false) {\n    }\n  };\n  // a record to help handle pullwait\n  struct PullWaitRecord {\n    // number of thread that waits for the request to finish\n    int nwait;\n    // the request was finished\n    bool finished;\n    PullWaitRecord(void)\n        : nwait(0), finished(true) {\n      // set finished to true so pull without pull request returns\n    }\n  };\n  /*! \\brief data structure to hold pull request */\n  struct PullEntry {\n    // data to be pulled back\n    Tensor<cpu, 2, DType> src;\n    // pullrequest record\n    std::vector<PullReqRecord> req;\n    // whether there is thread waiting on this event\n    std::vector<PullWaitRecord> wait;\n    PullEntry(void) {\n    }\n  };\n  // signal to notify all the thread about class destruction\n  bool destroy_signal;\n  // vector of devices\n  std::vector<int> devices;\n  // device index to local index\n  std::vector<int> dev2index;\n  //----- data structure used to support push ----\n  // stream used by push thread each device for memcpy\n  std::vector<Stream<xpu>*> push_stream;\n  // the queue used for push task\n  std::vector<utils::ThreadPQueue<PullTask> > push_queues;\n  // thread to handle push task\n  std::vector<utils::Thread> thread_push_handler;\n  // lock to lock push field\n  utils::Mutex push_lock;\n  // the map of push buffer\n  utils::ThreadSafeMap<PushEntry> push_map;\n  // customized local reduction operation\n  std::map<int, LocalOp> push_operation;\n  //----- data structure used to support pull ----\n  // the queue used for pull task\n  std::vector<utils::ThreadPQueue<std::pair<int, int> > > pull_queues;\n  // stream used by pull thread each device for memcpy\n  std::vector<Stream<xpu>*> pull_stream;\n  // the map to store pull status\n  utils::ThreadSafeMap<PullEntry> pull_map;\n  // thread to handle pull task\n  std::vector<utils::Thread> thread_pull_handler;\n  // lock to lock request field\n  utils::Mutex request_lock;\n  // lock to lock wait field\n  utils::Mutex wait_lock;\n  // conditional variable to do waiting\n  utils::ConditionVariable wait_cond;\n  // ---------configurations of server-------\n  int init_end;\n  // whether perform update on serverside\n  int update_on_server;\n  // debug option\n  int test_on_server;\n  // use pinned memory\n  int use_pin_memory;\n  // number of reduction thread\n  int nthread_reduction;\n  // the threshold for big array\n  size_t bigarray_bound;\n  // whether use pull thread per device\n  int perdev_pull_thread;\n  // whether use push thread per device\n  int perdev_push_thread;\n  /*! \\brief history of configurations */\n  std::vector< std::pair<std::string, std::string> > cfgvec;\n  // push handler\n  inline void PushProc(utils::ThreadPQueue<PullTask> *queue) {\n    while (!destroy_signal) {\n      PullTask tsk;\n      if (queue->Pop(&tsk)) {\n        const int wid = GetWorkIndex(tsk.devid);\n        PushEntry &e = push_map.GetRef(tsk.key);\n        CHECK_EQ(e.data[0][0].shape_, tsk.data.shape_)\n          << \"Tensor with same key must share same shape \"\n          << e.data[0][0].shape_\n          << \" vs \"\n          << tsk.data.shape_;\n        CHECK_EQ(!e.copied[wid], true) << \"data inconsistency\";\n        // start copy\n        SetDevice<xpu>(tsk.devid);\n        Copy(e.data[e.copyin_version][wid], tsk.data, push_stream[wid]);\n        // wait till the copy finishes\n        push_stream[wid]->Wait();\n        // mark copied\n        e.copied[wid] = true;\n        push_lock.Lock();\n        e.num_copied += 1;\n        int cp_version = e.copyin_version;\n        bool push_finish = e.num_copied >= static_cast<int>(devices.size());\n        if (push_finish) {\n          // switch version\n          e.copyin_version = (e.copyin_version + 1) % e.data.size(0);\n          std::fill(e.copied.begin(), e.copied.end(), false);\n          e.num_copied = 0;\n        }\n        push_lock.Unlock();\n        if (push_finish) {\n          this->HandlePushFinish(e.data[cp_version], tsk.key);\n        }\n      } else {\n        CHECK_EQ(destroy_signal, true) << \"abort but not destroy\";\n      }\n    }\n  }\n  inline void PushHandlerGlobal(void) {\n    // allocate stream resources\n    for (size_t i = 0; i < devices.size(); ++i) {\n      SetDevice<xpu>(devices[i]);\n      push_stream[i] = NewStream<xpu>(devices[i]);\n    }\n    this->PushProc(&push_queues[0]);\n    // free resources\n    for (size_t i = 0; i < devices.size(); ++i) {\n      SetDevice<xpu>(devices[i]);\n      DeleteStream(push_stream[i]);\n    }\n  }\n  inline void PushHandlerLocal(size_t tid) {\n    CHECK_LT(tid, devices.size()) << \"threadid exceed boundary\";\n    CHECK_EQ(push_queues.size(), devices.size()) << \"must have one pull_queue per device\";\n    // allocate stream resources\n    SetDevice<xpu>(devices[tid]);\n    push_stream[tid] = NewStream<xpu>(devices[tid]);\n    this->PushProc(&push_queues[tid]);\n    SetDevice<xpu>(devices[tid]);\n    DeleteStream(push_stream[tid]);\n  }\n  /*!\\brief entry point of loader thread */\n  inline static MSHADOW_THREAD_PREFIX PushGlobalThread(void *pthread) {\n    static_cast<LocalModel*>(pthread)->PushHandlerGlobal();\n    utils::ThreadExit(NULL);\n    return NULL;\n  }\n  inline static MSHADOW_THREAD_PREFIX PushLocalThread(void *arg) {\n    std::pair<LocalModel*, size_t> *p\n        = static_cast<std::pair<LocalModel*, size_t>*>(arg);\n    p->first->PushHandlerLocal(p->second);\n    delete p;\n    return NULL;\n  }\n  // push handler procedure\n  inline void PullProc(utils::ThreadPQueue<std::pair<int, int> > *queue) {\n    while (!destroy_signal) {\n      std::pair<int, int> tsk;\n      if (queue->Pop(&tsk)) {\n        const int key = tsk.first;\n        const int devid = tsk.second;\n        const int wid = GetWorkIndex(devid);\n        PullEntry &e = pull_map.GetRef(key);\n        {\n          // handle request\n          CHECK_EQ(e.req.size(), devices.size()) << \"PullHandler: must initialize the key, req\";\n          PullReqRecord &r = e.req[wid];\n          SetDevice<xpu>(devid);\n          Copy(r.dest, e.src, pull_stream[wid]);\n          // callback, if any\n          if (r.callback != NULL) {\n            (*r.callback)(pull_stream[wid], r.callback_arg);\n          }\n          // wait till the operation finishes\n          pull_stream[wid]->Wait();\n        }\n        {\n          // wake up waiters if any\n          CHECK_EQ(e.wait.size(), devices.size()) << \"PullHandler, must initialize the key, req\";\n          PullWaitRecord &w = e.wait[wid];\n          wait_lock.Lock();\n          w.finished = true;\n          if (w.nwait != 0) {\n            wait_cond.Broadcast();\n          }\n          wait_lock.Unlock();\n        }\n      } else {\n        CHECK_EQ(destroy_signal, true) << \"abort but not destroy\";\n      }\n    }\n  }\n  // use one thread for all pull actions\n  inline void PullHandlerGlobal(void) {\n    // allocate stream resources\n    for (size_t i = 0; i < devices.size(); ++i) {\n      SetDevice<xpu>(devices[i]);\n      pull_stream[i] = NewStream<xpu>(devices[i]);\n    }\n    this->PullProc(&pull_queues[0]);\n    // free resources\n    for (size_t i = 0; i < devices.size(); ++i) {\n      SetDevice<xpu>(devices[i]);\n      DeleteStream(pull_stream[i]);\n    }\n  }\n  inline void PullHandlerLocal(size_t tid) {\n    CHECK_LT(tid, devices.size()) << \"threadid exceed boundary\";\n    CHECK_EQ(pull_queues.size(), devices.size()) << \"must have one pull_queue per device\";\n    // allocate stream resources\n    SetDevice<xpu>(devices[tid]);\n    pull_stream[tid] = NewStream<xpu>(devices[tid]);\n    this->PullProc(&pull_queues[tid]);\n    SetDevice<xpu>(devices[tid]);\n    DeleteStream(pull_stream[tid]);\n  }\n  /*!\\brief entry point of pull thread, one thread for all devices */\n  inline static MSHADOW_THREAD_PREFIX PullGlobalThread(void *arg) {\n    static_cast<LocalModel*>(arg)->PullHandlerGlobal();\n    return NULL;\n  }\n  inline static MSHADOW_THREAD_PREFIX PullLocalThread(void *arg) {\n    std::pair<LocalModel*, size_t> *p\n        = static_cast<std::pair<LocalModel*, size_t>*>(arg);\n    p->first->PullHandlerLocal(p->second);\n    delete p;\n    return NULL;\n  }\n  // get internal index of device\n  inline int GetWorkIndex(int devid) const {\n    CHECK(devid >= 0 &&\n          devid < static_cast<int>(dev2index.size()) &&\n          dev2index[devid] >= 0) << \"Push: invalid devid\";\n    return dev2index[devid];\n  }\n  // functions to handle pull\n  inline void InitPullMap(int key) {\n    pull_map.Init(key);\n    PullEntry &e = pull_map.GetRef(key);\n    request_lock.Lock();\n    // must recheck after lock\n    if (e.req.size() == 0) {\n      e.req.resize(devices.size(), PullReqRecord());\n    }\n    request_lock.Unlock();\n    // check wait map\n    wait_lock.Lock();\n    // must recheck after lock\n    if (e.wait.size() == 0) {\n      e.wait.resize(devices.size(), PullWaitRecord());\n    }\n    wait_lock.Unlock();\n  }\n  // functions to handle pull\n  inline void InitPushMap(int key, Shape<2> shape) {\n    push_map.Init(key);\n    PushEntry &e = push_map.GetRef(key);\n    push_lock.Lock();\n    if (e.copied.size() == 0) {\n      e.Init(devices.size(), shape,\n             use_pin_memory != 0,\n             update_on_server != 0 || test_on_server != 0);\n    }\n    this->ServerInitKey(e.weight, key);\n    push_lock.Unlock();\n  }\n};\n}  // namespace ps\n}  // namespace mshadow\n#endif // MSHADOW_PS_LOCAL_INL_H_  NOLINT(*)\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow-ps/ps_rabit-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file ps_rabit-inl.h\n * \\brief distributed version of PS using BSP\n *     synchronization in the backend\n * \\author Tianqi Chen, Mu Li\n */\n#ifndef MSHADOW_PS_RABIT_INL_H_ // NOLINT(*)\n#define MSHADOW_PS_RABIT_INL_H_ // NOLINT(*)\n#include <vector>\n#include \"./mshadow_ps.h\"\n#include \"./ps_local-inl.h\"\n\n#if MSHADOW_RABIT_PS\n#include <rabit.h>\nnamespace mshadow {\nnamespace ps {\n// multi-threaded implementation of\ntemplate<typename xpu, typename DType>\nclass RabitModel : public LocalModel<xpu, DType> {\n public:\n  // parent type\n  typedef LocalModel<xpu, DType> Parent;\n  // constructor\n  RabitModel() {\n    // enforce usage of fifo queue\n    this->use_fifo_push_queue = 1;\n    destroy_reduce_thread_ = false;\n    disable_allreduce_ = 0;\n    this->init_reducer_ = 0;\n  }\n  virtual ~RabitModel(void) {\n    Parent::Destroy();\n    if (init_reducer_ != 0) {\n      destroy_reduce_thread_ = true;\n      reduce_queue_.Abort(1);\n      thread_reduce_handler_.Join();\n      reduce_queue_.Destroy();\n    }\n  }\n  // initialize the parameter server\n  virtual void Init(const std::vector<int> &devices) {\n    this->use_fifo_push_queue = 1;\n    // use fifo\n    reduce_queue_.Init(true);\n    thread_reduce_handler_.Start(ReduceGlobalThread, this);\n    init_reducer_ = 1;\n    // initialize other things\n    Parent::Init(devices);\n  }\n  // set parameters\n  virtual void SetParam(const char *name, const char *val) {\n    if (!strcmp(name, \"msg:disable_allreduce\")) {\n      disable_allreduce_ = atoi(val);\n    }\n    Parent::SetParam(name, val);\n  }\n  // override this function, to use parameter server\n  virtual void HandlePushFinish(Tensor<cpu, 3, DType> data,\n                                int key) {\n    // summation the data fron all devices\n    LocalModel<xpu, DType>::ReduceSum(data);\n    CHECK_EQ(data[0].CheckContiguous(), true) << \"data must be contiguous\";\n    ReduceTask tsk;\n    tsk.data = data[0]; tsk.key = key;\n    reduce_queue_.Push(tsk, 0);\n  }\n\n private:\n  // reduce task\n  struct ReduceTask {\n    int key;\n    mshadow::Tensor<cpu, 2> data;\n  };\n  // destroy reduce\n  bool destroy_reduce_thread_;\n  // whether reducer is initialized\n  int init_reducer_;\n  // check disable_allreduce functionalities\n  int disable_allreduce_;\n  // reduce handler thread\n  utils::Thread thread_reduce_handler_;\n  // queue for allreduce task\n  utils::ThreadPQueue<ReduceTask> reduce_queue_;\n  // reduce handler\n  inline void ReduceHandler(void) {\n    while (!destroy_reduce_thread_) {\n      ReduceTask tsk;\n      if (reduce_queue_.Pop(&tsk)) {\n        CHECK_EQ(disable_allreduce_, 0) << \"Allreduce disabled error\";\n        int key = tsk.key;\n        rabit::Allreduce<rabit::op::Max>(&key, 1);\n        CHECK_EQ(key, tsk.key) << \"Allreduce not concensus\";\n        rabit::Allreduce<rabit::op::Sum>\n            (tsk.data.dptr_, tsk.data.MSize());\n        tsk.data *= 1.0f / rabit::GetWorldSize();\n        CHECK_EQ(disable_allreduce_, 0) << \"Allreduce disabled error\";\n        this->HandleReduceFinish(tsk.data, tsk.key);\n      } else {\n        CHECK_EQ(destroy_reduce_thread_, true) << \"abort but not destroy\";\n      }\n    }\n  }\n  /*!\\brief entry point of reduce thread */\n  inline static MSHADOW_THREAD_PREFIX ReduceGlobalThread(void *pthread) {\n    static_cast<RabitModel*>(pthread)->ReduceHandler();\n    return NULL;\n  }\n};\n}  // namespace ps\n}  // namespace mshadow\n#endif  // MSHADOW_RABIT_PS\n#endif  // MSHADOW_PS_RABIT_INL_H_ // NOLINT(*)\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow-ps/thread.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file thread.h\n * \\brief this header include the minimum necessary resource\n * for multi-threading that can be compiled in windows, linux, mac\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_PS_THREAD_H_ // NOLINT(*)\n#define MSHADOW_PS_THREAD_H_ // NOLINT(*)\n\n#ifdef _MSC_VER\n#include <windows.h>\n#include <process.h>\n#include \"../mshadow/logging.h\"\nnamespace mshadow {\nnamespace utils {\n/*! \\brief simple semaphore used for synchronization */\nclass Semaphore {\n public :\n  inline void Init(int init_val) {\n    sem = CreateSemaphore(NULL, init_val, 10, NULL);\n    CHECK_NE(sem, NULL) << \"create Semaphore error\";\n  }\n  inline void Destroy(void) {\n    CloseHandle(sem);\n  }\n  inline void Wait(void) {\n    CHECK_EQ(WaitForSingleObject(sem, INFINITE), WAIT_OBJECT_0)\n      << \"WaitForSingleObject error\";\n  }\n  inline void Post(void) {\n    CHECK_NE(ReleaseSemaphore(sem, 1, NULL), 0) << \"ReleaseSemaphore error\";\n  }\n\n private:\n  HANDLE sem;\n};\n\n/*! \\brief mutex under windows */\nclass Mutex {\n public:\n  inline void Init(void) {\n    CHECK_NE(InitializeCriticalSectionAndSpinCount(&mutex, 0x00000400), 0)\n      << \"Mutex::Init fail\";\n  }\n  inline void Lock(void) {\n    EnterCriticalSection(&mutex);\n  }\n  inline void Unlock(void) {\n    LeaveCriticalSection(&mutex);\n  }\n  inline void Destroy(void) {\n    DeleteCriticalSection(&mutex);\n  }\n\n private:\n  friend class ConditionVariable;\n  CRITICAL_SECTION mutex;\n};\n\n// conditional variable that uses pthread\nclass ConditionVariable {\n public:\n  // initialize conditional variable\n  inline void Init(void) {\n    InitializeConditionVariable(&cond);\n  }\n  // destroy the thread\n  inline void Destroy(void) {\n    // DeleteConditionVariable(&cond);\n  }\n  // wait on the conditional variable\n  inline void Wait(Mutex *mutex) {\n    CHECK_NE(SleepConditionVariableCS(&cond, &(mutex->mutex), INFINITE), 0)\n      << \"ConditionVariable:Wait fail\";\n  }\n  inline void Broadcast(void) {\n    WakeAllConditionVariable(&cond);\n  }\n  inline void Signal(void) {\n    WakeConditionVariable(&cond);\n  }\n\n private:\n  CONDITION_VARIABLE cond;\n};\n\n/*! \\brief simple thread that wraps windows thread */\nclass Thread {\n private:\n  HANDLE    thread_handle;\n  unsigned  thread_id;\n public:\n  inline void Start(unsigned int __stdcall entry(void*p), void *param) {\n    thread_handle = (HANDLE)_beginthreadex(NULL, 0, entry, param, 0, &thread_id);\n  }\n  inline int Join(void) {\n    WaitForSingleObject(thread_handle, INFINITE);\n    return 0;\n  }\n};\n/*! \\brief exit function called from thread */\ninline void ThreadExit(void *status) {\n  _endthreadex(0);\n}\n#define MSHADOW_THREAD_PREFIX unsigned int __stdcall\n}  // namespace utils\n}  // namespace mshadow\n#else\n// thread interface using g++\n#include <semaphore.h>\n#include <pthread.h>\n#include <errno.h>\nnamespace mshadow {\nnamespace utils {\n/*!\\brief semaphore class */\nclass Semaphore {\n  #ifdef __APPLE__\n\n private:\n  sem_t* semPtr;\n  char sema_name[20];\n\n private:\n  inline void GenRandomString(char *s, const int len) {\n    static const char alphanum[] = \"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\";\n    for (int i = 0; i < len; ++i) {\n      s[i] = alphanum[rand() % (sizeof(alphanum) - 1)];\n    }\n    s[len] = 0;\n  }\n\n public:\n  inline void Init(int init_val) {\n    sema_name[0] = '/';\n    sema_name[1] = 's';\n    sema_name[2] = 'e';\n    sema_name[3] = '/';\n    GenRandomString(&sema_name[4], 16);\n    if ((semPtr = sem_open(sema_name, O_CREAT, 0644, init_val)) == SEM_FAILED) {\n      perror(\"sem_open\");\n      exit(1);\n    }\n    CHECK_NE(semPtr, NULL) << \"create Semaphore error\";\n  }\n  inline void Destroy(void) {\n    if (sem_close(semPtr) == -1) {\n      perror(\"sem_close\");\n      exit(EXIT_FAILURE);\n    }\n    if (sem_unlink(sema_name) == -1) {\n      perror(\"sem_unlink\");\n      exit(EXIT_FAILURE);\n    }\n  }\n  inline void Wait(void) {\n    sem_wait(semPtr);\n  }\n  inline void Post(void) {\n    sem_post(semPtr);\n  }\n  #else\n\n private:\n  sem_t sem;\n\n public:\n  inline void Init(int init_val) {\n    if (sem_init(&sem, 0, init_val) != 0) {\n      LOG(FATAL) << \"Semaphore.Init: \" << strerror(errno);\n    }\n  }\n  inline void Destroy(void) {\n    if (sem_destroy(&sem) != 0) {\n      LOG(FATAL) << \"Semaphore.Destroy: \" << strerror(errno);\n    }\n  }\n  inline void Wait(void) {\n    if (sem_wait(&sem) != 0) {\n      LOG(FATAL) << \"Semaphore.Wait: \" << strerror(errno);\n    }\n  }\n  inline void Post(void) {\n    if (sem_post(&sem) != 0) {\n      LOG(FATAL) << \"Semaphore.Post: \" << strerror(errno);\n    }\n  }\n  #endif\n};\n\n// mutex that works with pthread\nclass Mutex {\n public:\n  inline void Init(void) {\n    pthread_mutex_init(&mutex, NULL);\n  }\n  inline void Lock(void) {\n    pthread_mutex_lock(&mutex);\n  }\n  inline void Unlock(void) {\n    pthread_mutex_unlock(&mutex);\n  }\n  inline void Destroy(void) {\n    pthread_mutex_destroy(&mutex);\n  }\n\n private:\n  friend class ConditionVariable;\n  pthread_mutex_t mutex;\n};\n\n// conditional variable that uses pthread\nclass ConditionVariable {\n public:\n  // initialize conditional variable\n  inline void Init(void) {\n    pthread_cond_init(&cond, NULL);\n  }\n  // destroy the thread\n  inline void Destroy(void) {\n    pthread_cond_destroy(&cond);\n  }\n  // wait on the conditional variable\n  inline void Wait(Mutex *mutex) {\n    pthread_cond_wait(&cond, &(mutex->mutex));\n  }\n  inline void Broadcast(void) {\n    pthread_cond_broadcast(&cond);\n  }\n  inline void Signal(void) {\n    pthread_cond_signal(&cond);\n  }\n\n private:\n  pthread_cond_t cond;\n};\n\n/*!\\brief simple thread class */\nclass Thread {\n private:\n  pthread_t thread;\n public :\n  inline void Start(void * entry(void*), void *param) { // NOLINT(*)\n    pthread_attr_t attr;\n    pthread_attr_init(&attr);\n    pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);\n    pthread_create(&thread, &attr, entry, param);\n  }\n  inline int Join(void) {\n    void *status;\n    return pthread_join(thread, &status);\n  }\n};\ninline void ThreadExit(void *status) {\n  pthread_exit(status);\n}\n}  // namespace utils\n}  // namespace mshadow\n#define MSHADOW_THREAD_PREFIX void *\n#endif  // Linux\n#endif  // MSHADOW_PS_THREAD_H_  NOLINT(*)\n"
  },
  {
    "path": "3rdparty/mshadow/mshadow-ps/thread_util.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file thread_util.h\n * \\brief data structures for multi-threading communication\n * \\author Tianqi Chen\n */\n#ifndef MSHADOW_PS_THREAD_UTIL_H_  // NOLINT(*)\n#define MSHADOW_PS_THREAD_UTIL_H_  // NOLINT(*)\n\n#include <utility>\n#include <queue>\n#include <map>\n#include \"./thread.h\"\nnamespace mshadow {\nnamespace utils {\n/*!\n * \\brief thread safe queue that can be used for customer consumer model\n * in the future, it will support priority scheduling\n * \\tparam DType the content of the queue\n */\ntemplate<typename DType>\nclass ThreadPQueue {\n public:\n  // constructor\n  ThreadPQueue() : use_fifo_(false) {\n  }\n  /*! \\brief intitialize the queue, must call this before use */\n  inline void Init(bool use_fifo = false) {\n    use_fifo_ = use_fifo;\n    lock_.Init();\n    counter_.Init(0);\n  }\n  /*! \\brief destroy the resources on the queue */\n  inline void Destroy(void) {\n    lock_.Destroy();\n    counter_.Destroy();\n  }\n  /*!\n   * \\brief Destroy the queue\n   *        wake up all the threads waits on pop\n   *  this is usually used in class destructor\n   * \\param max_nthread the maximum number of thread that\n   *  could be waiting on the queue\n   */\n  inline void Abort(int max_nthread = 1) {\n    for (int i = 0; i < max_nthread; ++i) {\n      counter_.Post();\n    }\n  }\n  /*!\n   * \\brief push an element to the queue\n   * \\param data the data to be puhed into queue\n   * \\param optionally priority level to hint which\n   *        element should be poped first\n   */\n  inline void Push(const DType &data, int priority = 0) {\n    lock_.Lock();\n    if (use_fifo_) {\n      fqueue_.push(data);\n    } else {\n      pqueue_.push(Entry(data, priority));\n    }\n    lock_.Unlock();\n    counter_.Post();\n  }\n  /*!\n   * \\brief pop an element from the queue\n   * this will block the thread if the queue is empty\n   * \\param data_out the address to put output of the queue\n   * \\return true if a correct element is returned\n   *  false if abort is called and no element was left in queue\n   */\n  inline bool Pop(DType *data_out) {\n    counter_.Wait();\n    lock_.Lock();\n    if (use_fifo_) {\n      if (fqueue_.size() == 0) {\n        lock_.Unlock(); return false;\n      }\n    } else {\n      if (pqueue_.size() == 0) {\n        lock_.Unlock(); return false;\n      }\n    }\n    if (use_fifo_) {\n      CHECK_NE(fqueue_.size(), 0) << \"Queue.Pop\";\n      *data_out = fqueue_.front();\n      fqueue_.pop();\n    } else {\n      CHECK_NE(pqueue_.size(), 0) << \"Queue.Pop\";\n      *data_out = pqueue_.top().data;\n      pqueue_.pop();\n    }\n    lock_.Unlock();\n    return true;\n  }\n\n private:\n  // entry in the queue\n  struct Entry {\n    DType data;\n    int priority;\n    Entry(const DType &data, int priority)\n        : data(data), priority(priority) {}\n    inline bool operator<(const Entry &b) const {\n      return priority < b.priority;\n    }\n  };\n  // whether use FIFO queue\n  bool use_fifo_;\n  // a priority queue\n  std::priority_queue<Entry> pqueue_;\n  // a FIFO queue\n  std::queue<DType> fqueue_;\n  // lock for accessing the queue\n  utils::Mutex lock_;\n  // counter to count number of push tasks\n  utils::Semaphore counter_;\n};\n\n// naive implementation of threadsafe map\ntemplate<typename TValue>\nclass ThreadSafeMap {\n public:\n  inline void Init(void) {\n    lock_.Init();\n  }\n  inline void Destroy(void) {\n    for (typename std::map<int, TValue*>::iterator\n             it = map_.begin(); it != map_.end(); ++it) {\n      delete it->second;\n    }\n    lock_.Destroy();\n  }\n  inline TValue *Get(int key) {\n    TValue *ret;\n    lock_.Lock();\n    typename std::map<int, TValue*>::const_iterator\n        it = map_.find(key);\n    if (it == map_.end() || it->first != key) {\n      ret = NULL;\n    } else {\n      ret = it->second;\n    }\n    lock_.Unlock();\n    return ret;\n  }\n  inline TValue &GetRef(int key) {\n    TValue *ret = this->Get(key);\n    CHECK_NE(ret, NULL) << \"key = \" << key << \" does not exist\";\n    return *ret;\n  }\n  inline void Init(int key) {\n    lock_.Lock();\n    if (map_.count(key) == 0) {\n      map_[key] = new TValue();\n    }\n    lock_.Unlock();\n  }\n\n private:\n  // lock for accessing the queue\n  utils::Mutex lock_;\n  std::map<int, TValue*> map_;\n};\n\n}  // namespace utils\n}  // namespace mshadow\n#endif  // MSHADOW_PS_THREAD_UTIL_H_\n"
  },
  {
    "path": "3rdparty/mshadow/scripts/travis_script.sh",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# main script of travis\nif [ ${TASK} == \"lint\" ]; then\n    python3 dmlc-core/scripts/lint.py mshadow all mshadow mshadow-ps || exit -1\nfi\n\nif [ ${TASK} == \"doc\" ]; then\n    doxygen doc/Doxyfile 2>log.txt\n    (cat log.txt| grep -v ENABLE_PREPROCESSING |grep -v \"unsupported tag\" |grep nothing) && exit -1\nfi\n\nif [ ${TASK} == \"build\" ]; then\n    cd guide\n    echo \"USE_BLAS=blas\" >> config.mk\n    make all || exit -1\n    cd mshadow-ps\n    echo \"USE_BLAS=blas\" >> config.mk\n    echo \"USE_RABIT_PS=0\" >> config.mk    \n    make local_sum.cpu || exit -1\nfi\n"
  },
  {
    "path": "3rdparty/mshadow/test/Makefile",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# set LD_LIBRARY_PATH\nexport CC  = gcc\nexport CXX = g++\nexport NVCC =nvcc\nexport CFLAGS = -Wall -O3 -g -msse3 -Wno-unknown-pragmas -funroll-loops -I../\nexport LDFLAGS= -g -lm -lcublas -lcudart -lcusolver\nexport NVCCFLAGS = -O3 --use_fast_math -ccbin $(CXX)\n\n# specify tensor path\nBIN = test_tblob\nOBJ =\nCUOBJ =\nCUBIN = test\n.PHONY: clean all\n\nall: $(CUBIN) $(BIN)\n\ntest: test.cu\n\ntest_tblob: test_tblob.cc\n\n$(BIN) :\n\t$(CXX) $(CFLAGS) -std=c++17 -o $@ $(filter %.cpp %.o %.c %.cc, $^)  $(LDFLAGS)\n\n$(OBJ) :\n\t$(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c, $^) )\n\n$(CUOBJ) :\n\t$(NVCC) -c -o $@ $(NVCCFLAGS) -Xcompiler \"$(CFLAGS)\" $(filter %.cu, $^)\n\n$(CUBIN) :\n\t$(NVCC) -o $@ $(NVCCFLAGS) -Xcompiler \"$(CFLAGS)\" -Xlinker \"$(LDFLAGS)\" $(filter %.cu %.cpp %.o, $^)\n\nclean:\n\t$(RM) $(OBJ) $(BIN) $(CUBIN) $(CUOBJ) *~\n"
  },
  {
    "path": "3rdparty/mshadow/test/pairtest.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#include \"mshadow/tensor.h\"\n#include \"old/tensor.h\"\n#include \"assert.h\"\n#include <cstring>\n\nusing mshadow::index_t;\ntemplate<typename T>\nvoid Print(T const & ist, int I, int J) {\n  for (int i = 0; i < I; ++i) {\n    for (int j = 0; j < J; ++j) {\n      printf(\"%.2f \", ist[i][j]);\n    }\n    printf(\"\\n\");\n  }\n}\n\nbool Check(mshadow::TensorContainer<mshadow::cpu, 2, float> &mct, \\\n           Xmshadow::TensorContainer<Xmshadow::cpu, 2> &xct) {\n  for (index_t i = 0; i < mct.size(0); ++i) {\n    for (index_t j = 0; j < mct.size(1); ++j) {\n      assert(mct[i][j] == xct[i][j]);\n    }\n  }\n  return true;\n}\n\ntemplate<typename xpua, typename xpub>\nvoid RunTask() {\n  const int X = 6;\n  const int K = 2;\n  const int O = (X - K) / 2 + 1;\n  mshadow::TensorContainer<mshadow::cpu, 4, float> srcm(mshadow::Shape4(1,1,X, X));\n  Xmshadow::TensorContainer<Xmshadow::cpu, 4> srcx(Xmshadow::Shape4(1,1,X, X));\n  for (int i = 0; i < X; ++i) {\n    for (int j = 0; j < X; ++j) {\n      srcm[0][0][i][j] = i * 0.1f + j * 0.1f;\n      srcx[0][0][i][j] = i * 0.1f + j * 0.1f;\n    }\n  }\n  printf(\"Source:\\n\");\n  Print(srcm[0][0], X, X);\n  printf(\"\\n\");\n  mshadow::TensorContainer<xpua, 4, float> mct(mshadow::Shape4(1,1,X, X));\n  Xmshadow::TensorContainer<xpub, 4> xct(Xmshadow::Shape4(1,1,X, X));\n  mshadow::Copy(mct, srcm);\n  Xmshadow::Copy(xct, srcx);\n\n  \n  mshadow::TensorContainer<xpua, 4, float> pool_ct(mshadow::Shape4(1,1, O, O));\n  Xmshadow::TensorContainer<xpub, 4> pool_xct(Xmshadow::Shape4(1,1,O,O));\n\n  pool_ct = mshadow::expr::pool<mshadow::red::maximum>(mct, K, K, K);\n  pool_xct = Xmshadow::expr::pool<Xmshadow::red::maximum>(xct, K, K);\n\n  printf(\"New pool:\\n\");\n  Print(pool_ct[0][0], O, O);\n  printf(\"\\nOld pool:\\n\");\n  Print(pool_xct[0][0], O, O);\n  printf(\"\\n\");\n  mshadow::TensorContainer<mshadow::cpu, 4, float> gpool_src(mshadow::Shape4(1,1, O, O));\n  Xmshadow::TensorContainer<Xmshadow::cpu, 4> gpool_xsrc(Xmshadow::Shape4(1,1,O,O));\n  for (int i = 0; i < O; ++i) {\n    for (int j = 0; j < O; ++j) {\n      gpool_src[0][0][i][j] = 0.1f;\n      gpool_xsrc[0][0][i][j] = 0.1f;\n    }\n  }\n  mshadow::TensorContainer<xpua, 4, float> gpool_ct(mshadow::Shape4(1,1, O, O));\n  Xmshadow::TensorContainer<xpub, 4> gpool_xct(Xmshadow::Shape4(1,1,O,O));\n  mshadow::Copy(gpool_ct, gpool_src);\n  Xmshadow::Copy(gpool_xct, gpool_xsrc);\n\n  mshadow::TensorContainer<xpua, 4, float> mout(mshadow::Shape4(1,1,X, X));\n  Xmshadow::TensorContainer<xpub, 4> xout(Xmshadow::Shape4(1,1,X, X));\n\n  mout = mshadow::expr::unpool<mshadow::red::maximum>(mct, pool_ct, gpool_ct, K, K, K);\n  xout = Xmshadow::expr::unpool<Xmshadow::red::maximum>(xct, pool_xct, gpool_xct, K, K);\n\n  mshadow::Copy(srcm, mout);\n  Xmshadow::Copy(srcx, xout);\n\n  mshadow::TensorContainer<mshadow::cpu, 2> l1(mshadow::Shape2(X,X));\n  Xmshadow::TensorContainer<Xmshadow::cpu, 2> l2(Xmshadow::Shape2(X, X));\n  l1 = mshadow::expr::reshape(srcm, l1.shape_);\n  l2 = Xmshadow::expr::reshape(srcx, l2.shape);\n  printf(\"New unpool\\n\");\n  Print(l1, l1.size(0), l1.size(1));\n  printf(\"\\nOld unpool\\n\");\n  Print(l2, X, X);\n  if (Check(l1, l2)) {\n    printf(\"Pass\\n\");\n  }\n}\n\nint main(int argc, char** argv) {\n  if (argc < 1) {\n    printf(\"Usage: dev\\n\");\n    exit(-1);\n  }\n  if (!strcmp(argv[1], \"cpu\")) {\n    RunTask<mshadow::cpu, Xmshadow::cpu>();\n  } else {\n    RunTask<mshadow::gpu, Xmshadow::gpu>();\n  }\n}\n"
  },
  {
    "path": "3rdparty/mshadow/test/pool.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#include \"mshadow/tensor.h\"\n#include \"old/tensor.h\"\n#include \"assert.h\"\n#include <cstring>\n\nusing mshadow::index_t;\ntemplate<typename T>\nvoid Print(T const & ist) {\n  for (int i = 0; i < ist.size(0); ++i) {\n    for (int j = 0; j < ist.size(1); ++j) {\n      printf(\"%.2f \", ist[i][j]);\n    }\n    printf(\"\\n\");\n  }\n}\n\nbool Check(mshadow::TensorContainer<mshadow::cpu, 2, float> &mct, \\\n           Xmshadow::TensorContainer<Xmshadow::cpu, 2> &xct) {\n  for (index_t i = 0; i < mct.size(0); ++i) {\n    for (index_t j = 0; j < mct.size(1); ++j) {\n      assert(mct[i][j] == xct[i][j]);\n    }\n  }\n  return true;\n}\n\ntemplate<typename xpua, typename xpub>\nvoid RunTask() {\n  const int X = 6;\n  const int K = 2;\n  mshadow::TensorContainer<mshadow::cpu, 2, float> srcm(mshadow::Shape2(X, X));\n  Xmshadow::TensorContainer<Xmshadow::cpu, 2> srcx(Xmshadow::Shape2(X, X));\n  \n  mshadow::TensorContainer<xpua, 2, float> mct(mshadow::Shape2(X, X));\n  Xmshadow::TensorContainer<xpub, 2> xct(Xmshadow::Shape2(X, X));\n  for (int i = 0; i < X; ++i) {\n    for (int j = 0; j < X; ++j) {\n      srcm[i][j] = i * 0.1f + j * 0.1f;\n      srcx[i][j] = i * 0.1f + j * 0.1f;\n    }\n  }\n  mshadow::Copy(mct, srcm);\n  Xmshadow::Copy(xct, srcx);\n  mshadow::TensorContainer<xpua, 2, float> pool_ct(mshadow::Shape2((X-K)/2+1, (X-K)/2+1));\n  Xmshadow::TensorContainer<xpub, 2> pool_xct(Xmshadow::Shape2((X-K)/2+1, (X-K)/2+1));\n\n  pool_ct = mshadow::expr::pool<mshadow::red::maximum>(mct, K, K, K);\n  pool_xct = Xmshadow::expr::pool<Xmshadow::red::maximum>(xct, K, K);\n\n  mshadow::TensorContainer<mshadow::cpu, 2, float> cpool_ct(mshadow::Shape2((X-K)/2+1, (X-K)/2+1));\n  Xmshadow::TensorContainer<Xmshadow::cpu, 2> cpool_xct(Xmshadow::Shape2((X-K)/2+1, (X-K)/2+1));\n  mshadow::Copy(cpool_ct, pool_ct);\n  Xmshadow::Copy(cpool_xct, pool_xct);\n  if (Check(cpool_ct, cpool_xct)) {\n    printf(\"Pass\\n\");\n  }\n}\n\nint main(int argc, char** argv) {\n  if (argc < 2) {\n    printf(\"Usage: dev\\n\");\n    exit(-1);\n  }\n  if (!strcmp(argv[1], \"cpu\")) {\n    RunTask<mshadow::cpu, Xmshadow::cpu>();\n  } else {\n    RunTask<mshadow::gpu, Xmshadow::gpu>();\n  }\n}\n"
  },
  {
    "path": "3rdparty/mshadow/test/reshape.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#include \"mshadow/tensor.h\"\n#include \"old/tensor.h\"\n#include \"assert.h\"\n#include <cstring>\n\nusing mshadow::index_t;\ntemplate<typename T>\nvoid Print(T const & ist) {\n  for (int i = 0; i < ist.size(0); ++i) {\n    for (int j = 0; j < ist.size(1); ++j) {\n      printf(\"%.2f \", ist[i][j]);\n    }\n    printf(\"\\n\");\n  }\n}\n\nbool Check(mshadow::TensorContainer<mshadow::cpu, 2, float> &mct, \\\n           Xmshadow::TensorContainer<Xmshadow::cpu, 2> &xct) {\n  for (index_t i = 0; i < mct.size(0); ++i) {\n    for (index_t j = 0; j < mct.size(1); ++j) {\n      assert(mct[i][j] == xct[i][j]);\n    }\n  }\n  return true;\n}\n\ntemplate<typename xpua, typename xpub>\nvoid RunTask() {\n  const int X = 6;\n  const int K = 2;\n  mshadow::TensorContainer<mshadow::cpu, 2, float> srcm(mshadow::Shape2(X, X));\n  Xmshadow::TensorContainer<Xmshadow::cpu, 2> srcx(Xmshadow::Shape2(X, X));\n  \n  mshadow::TensorContainer<xpua, 2, float> mct(mshadow::Shape2(X, X));\n  Xmshadow::TensorContainer<xpub, 2> xct(Xmshadow::Shape2(X, X));\n  for (int i = 0; i < X; ++i) {\n    for (int j = 0; j < X; ++j) {\n      srcm[i][j] = i * 0.1f + j * 0.1f;\n      srcx[i][j] = i * 0.1f + j * 0.1f;\n    }\n  }\n  mshadow::Copy(mct, srcm);\n  Xmshadow::Copy(xct, srcx);\n\n  mshadow::TensorContainer<xpua, 4, float> mct4d(mshadow::Shape4(1, 1, X / K, X * K));\n  Xmshadow::TensorContainer<xpub, 4> xct4d(Xmshadow::Shape4(X / K, X * K, 1, 1));\n  \n  mct4d = mshadow::expr::reshape(mct, mct4d.shape_);\n  xct4d = Xmshadow::expr::reshape(xct, xct4d.shape);\n  \n  mct = mshadow::expr::reshape(mct4d, mct.shape_);\n  xct = Xmshadow::expr::reshape(xct4d, xct.shape);\n  \n  mshadow::TensorContainer<mshadow::cpu, 2, float> m_ct(mshadow::Shape2(X, X));\n  Xmshadow::TensorContainer<Xmshadow::cpu, 2> x_ct(Xmshadow::Shape2(X, X));\n  \n  mshadow::Copy(m_ct, mct);\n  Xmshadow::Copy(x_ct, xct);\n  if (Check(m_ct, x_ct)) {\n    printf(\"Pass\\n\");\n  }\n}\n\nint main(int argc, char** argv) {\n  if (argc < 2) {\n    printf(\"Usage: dev\\n\");\n    exit(-1);\n  }\n  if (!strcmp(argv[1], \"cpu\")) {\n    RunTask<mshadow::cpu, Xmshadow::cpu>();\n  } else {\n    RunTask<mshadow::gpu, Xmshadow::gpu>();\n  }\n}\n"
  },
  {
    "path": "3rdparty/mshadow/test/test.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#include \"test.h\"\n\nusing namespace mshadow;\n\n\nint main() {\n  InitTensorEngine<cpu>();\n  InitTensorEngine<gpu>();\n  Tensor<cpu, 3, float> tc = NewTensor<cpu, float>(Shape3(3, 2, 4), 0.0f);\n  Tensor<gpu, 3, float> tg = NewTensor<gpu, float>(tc.shape_, 0.0f);\n  // init\n  for (index_t i = 0; i < tc.size(0); ++i) {\n    for (index_t j = 0; j < tc.size(1); ++j) {\n      for (index_t k = 0; k < tc.size(2); ++k) {\n        tc[i][j][k] = i * 0.1f + j * 0.2f + k * 0.1f;\n      }\n    }\n  }\n  Copy(tg, tc);\n  // print\n  printf(\"\\n#print batch 0 of cpu tensor:\\n\");\n  Print2DTensor(tc[0]);\n  printf(\"\\n\");\n  Print2DTensor(tc[1]);\n  printf(\"\\n\");\n  Print2DTensor(tc[2]);\n  // check\n  if (Check2DTensor(tg[1], tc[1])) {\n    printf(\"batch 1 of gpu & cpu tensor are same.\\n\");\n  }\n  // sum of row\n  Tensor<cpu, 1, float> tmp_tc = NewTensor<cpu, float>(Shape1(tc[0].size(1)), 0.0f);\n  Tensor<gpu, 1, float> tmp_tg = NewTensor<gpu, float>(Shape1(tg[0].size(1)), 0.0f);\n  printf(\"\\n#sum_rows of batch 0:\\n\");\n  tmp_tc = sum_rows(tc[0]);\n  tmp_tg = sum_rows(tg[0]);\n  Print1DTensor(tmp_tc);\n  if (Check1DTensor(tmp_tg, tmp_tc)) {\n    printf(\"cpu & gpu result consists\\n\");\n  }\n  FreeSpace(&tmp_tc);\n  FreeSpace(&tmp_tg);\n  // sumall_except_dim\n  printf(\"\\n#sumall_except_dim<0> of batch 0:\\n\");\n  Tensor<cpu, 1, float> red_tc = NewTensor<cpu, float>(Shape1(tc.size(0)), 0.0f);\n  Tensor<gpu, 1, float> red_tg = NewTensor<gpu, float>(Shape1(tg.size(0)), 0.0f);\n  red_tc = sumall_except_dim<0>(tc);\n  red_tg = sumall_except_dim<0>(tg);\n  Print1DTensor(red_tc);\n  if (Check1DTensor(red_tg, red_tc)) {\n    printf(\"cpu & gpu result consists\\n\");\n  }\n  FreeSpace(&red_tc);\n  FreeSpace(&red_tg);\n  // softmax\n  printf(\"\\n#Softmax\\n\");\n  Tensor<cpu, 2, float> sm_tc = NewTensor<cpu, float>(tc[0].shape_, 0.0f);\n  Tensor<gpu, 2, float> sm_tg = NewTensor<gpu, float>(tg[0].shape_, 0.0f);\n  Softmax(sm_tc, tc[0]);\n  Softmax(sm_tg, tg[0]);\n  if (Check2DTensor(sm_tg, sm_tc)) {\n    printf(\"cpu & gpu result consists\\n\");\n  }\n  // mirror\n  printf(\"\\n#mirror\\n\");\n  sm_tc = mirror(tc[0]);\n  sm_tg = mirror(tg[0]);\n  if (Check2DTensor(sm_tg, sm_tc)) {\n    printf(\"cpu & gpu result consists\\n\");\n  }\n  FreeSpace(&sm_tc);\n  FreeSpace(&sm_tg);\n  // reshape\n  \n  FreeSpace(&tc);\n  FreeSpace(&tg);\n  ShutdownTensorEngine<cpu>();\n  ShutdownTensorEngine<gpu>();\n}\n"
  },
  {
    "path": "3rdparty/mshadow/test/test.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef TEST_H\n#define TEST_H\n\n#include \"mshadow/tensor.h\"\n#include \"assert.h\"\n\n#define EPS 0.0001\nusing namespace mshadow;\nusing namespace mshadow::expr;\n\n\ntemplate<typename xpu>\nvoid Print2DTensor(Tensor<xpu, 2, float> const &ts);\n\ntemplate<typename xpu>\nvoid Print1DTensor(Tensor<xpu, 1, float> const &ts);\n\ntemplate<>\nvoid Print1DTensor(Tensor<cpu, 1, float> const &ts) {\n  for (index_t i = 0; i < ts.size(0); ++i) {\n    printf(\"%.2f \", ts[i]);\n  }\n  printf(\"\\n\");\n}\n\n\ntemplate<>\nvoid Print2DTensor(Tensor<cpu, 2, float> const &ts) {\n  for (index_t i = 0; i < ts.size(0); ++i) {\n    Print1DTensor(ts[i]);\n  }\n}\n\ntemplate<>\nvoid Print2DTensor(Tensor<gpu, 2, float> const &tg) {\n  Tensor<cpu, 2, float> tc = NewTensor<cpu, float>(tg.shape_, 0.0f);\n  Copy(tc, tg);\n  Print2DTensor(tc);\n  FreeSpace(&tc);\n}\n\n\n\nbool Check2DTensor(Tensor<gpu, 2, float> const &tg, Tensor<cpu, 2, float> const &tc) {\n  Tensor<cpu, 2, float> tcc = NewTensor<cpu, float>(tg.shape_, 0.0f);\n  Copy(tcc, tg);\n  for (index_t i = 0; i < tc.size(0); ++i) {\n    for (index_t j = 0; j < tc.size(1); ++j) {\n      assert(abs(tcc[i][j] - tc[i][j]) < EPS);\n    }\n  }\n  FreeSpace(&tcc);\n  return true;\n}\n\nbool Check1DTensor(Tensor<gpu, 1, float> const &tg, Tensor<cpu, 1, float> const &tc) {\n  Tensor<cpu, 1, float> tcc = NewTensor<cpu, float>(tc.shape_, 0.0f);\n  Copy(tcc, tg);\n  printf(\"gpu result:\\n\");\n  Print1DTensor(tcc);\n  for (index_t i = 0; i < tc.size(0); ++i) {\n    assert(abs(tcc[i] - tc[i]) < EPS);\n  }\n  FreeSpace(&tcc);\n  return true;\n}\n#endif\n"
  },
  {
    "path": "3rdparty/mshadow/test/unpack.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#include \"mshadow/tensor.h\"\n#include \"old/tensor.h\"\n#include \"assert.h\"\n#include <cstring>\n\nusing mshadow::index_t;\ntemplate<typename T>\nvoid Print(T const & ist) {\n  for (int i = 0; i < ist.size(0); ++i) {\n    for (int j = 0; j < ist.size(1); ++j) {\n      printf(\"%.2f \", ist[i][j]);\n    }\n    printf(\"\\n\");\n  }\n}\n\nbool Check(mshadow::TensorContainer<mshadow::cpu, 2, float> &mct, \\\n           Xmshadow::TensorContainer<Xmshadow::cpu, 2> &xct) {\n  for (index_t i = 0; i < mct.size(0); ++i) {\n    for (index_t j = 0; j < mct.size(1); ++j) {\n      assert(mct[i][j] == xct[i][j]);\n    }\n  }\n  return true;\n}\n\ntemplate<typename xpua, typename xpub>\nvoid RunTask() {\n  const int ksize = 3;\n  const int kstride = 2;\n  const int X = 6;\n  Xmshadow::TensorContainer<Xmshadow::cpu, 4> xsrc(Xmshadow::Shape4(1, 1, X, X));\n  mshadow::TensorContainer<mshadow::cpu, 4> src(mshadow::Shape4(1, 1, X, X));\n\n  for (int i = 0; i < X; ++i) {\n    for (int j = 0; j < X; ++j) {\n      xsrc[0][0][i][j] = i * 0.1f + j * 0.2f;\n      src[0][0][i][j] = i * 0.1f + j * 0.2f;\n    }\n  }\n  Xmshadow::TensorContainer<xpub, 4> xin(Xmshadow::Shape4(1, 1, X, X));\n  mshadow::TensorContainer<xpua, 4> in(mshadow::Shape4(1, 1, X, X));\n\n  mshadow::Copy(in, src);\n  Xmshadow::Copy(xin, xsrc);\n\n  Xmshadow::TensorContainer<xpub, 2> xtmp_col;\n  mshadow::TensorContainer<xpua, 2> tmp_col;\n  \n\n  index_t oheight  = (in.size(2) - ksize)/kstride + 1;\n  index_t owidth   = (in.size(3) - ksize)/kstride + 1;\n  index_t nbatch   = in.size(0);\n\n  \n  xtmp_col.Resize( Xmshadow::Shape2( xin.shape[2]*ksize*ksize, nbatch*oheight*owidth ) );\n  tmp_col.Resize(mshadow::Shape2(in.size(1)*ksize*ksize, nbatch*oheight*owidth));\n  xtmp_col = Xmshadow::expr::unpack_patch2col( xin, ksize, kstride );\n  tmp_col = mshadow::expr::unpack_patch2col(in, ksize, ksize, kstride);\n\n  Xmshadow::TensorContainer<Xmshadow::cpu, 2> xtc;\n  mshadow::TensorContainer<mshadow::cpu, 2> tc;\n\n  xtc.Resize( Xmshadow::Shape2( xin.shape[2]*ksize*ksize, nbatch*oheight*owidth ) );\n  tc.Resize(mshadow::Shape2(in.size(1)*ksize*ksize, nbatch*oheight*owidth));\n\n  mshadow::Copy(tc, tmp_col);\n  Xmshadow::Copy(xtc, xtmp_col);\n  if (Check(tc, xtc)) {\n    printf(\"Pass\\n\");\n  }\n  \n}\n\nint main(int argc, char** argv) {\n  if (argc < 2) {\n    printf(\"Usage: dev\\n\");\n    exit(-1);\n  }\n  if (!strcmp(argv[1], \"cpu\")) {\n    RunTask<mshadow::cpu, Xmshadow::cpu>();\n  } else {\n    RunTask<mshadow::gpu, Xmshadow::gpu>();\n  }\n}\n"
  },
  {
    "path": "CMakeLists.txt",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\ncmake_minimum_required(VERSION 3.13)\n\n# workaround to store CMAKE_CROSSCOMPILING because is getting reset by the project command\nif(CMAKE_CROSSCOMPILING)\n  set(__CMAKE_CROSSCOMPILING ${CMAKE_CROSSCOMPILING})\n  set(__CMAKE_CROSSCOMPILING_OVERRIDE ON)\nendif()\n\nproject(mxnet C CXX)\nset(CMAKE_CXX_STANDARD 17)\nset(CMAKE_CXX_STANDARD_REQUIRED ON)\nset(CMAKE_CXX_EXTENSIONS ON)  # GNU extensions used by src/operator/random/shuffle_op.cc\n\n# Sanity checks for some popular compilers. Make sure their version is\n# sufficient. Cmake also automatically checks if a compiler supports c++17. But\n# some compilers claim they support c++17 without actually implementing crucial\n# parts of the standard leading to hard to understand compilation errors.\nif(CMAKE_CXX_COMPILER_ID STREQUAL \"GNU\")\n  if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0)\n    message(FATAL_ERROR \"MXNet 2 requires a C++17 compatible compiler. Please update to GCC version 7 or newer.\")\n  endif()\nelseif(CMAKE_CXX_COMPILER_ID STREQUAL \"Clang\")\n  if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6.0)\n    message(FATAL_ERROR \"MXNet 2 requires a C++17 compatible compiler. Please update to Clang version 6 or newer.\")\n  endif()\nendif()\n\nif(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/config.cmake)\n  # Load config.cmake only if mxnet is not compiled as a dependency of another project\n  include(${CMAKE_CURRENT_SOURCE_DIR}/config.cmake)\nendif()\n\nif(__CMAKE_CROSSCOMPILING_OVERRIDE)\n  set(CMAKE_CROSSCOMPILING ${__CMAKE_CROSSCOMPILING})\nendif()\n\ninclude(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Utils.cmake)\n\ninclude(CMakeDependentOption)\n#Some things have order. This must be put in front alone\noption(MXNET_BUILD_SHARED_LIBS \"Build shared libraries instead of static libraries\" ON)\noption(USE_CUDA \"Build with CUDA support\"   ON)\nset(MXNET_CUDA_ARCH \"Auto\" CACHE STRING \"Target NVIDIA GPU achitecture.\nFormat: Auto | Common | All | LIST(ARCH_AND_PTX ...)\n- \\\"Auto\\\" detects local machine GPU compute arch at runtime.\n- \\\"Common\\\" and \\\"All\\\" cover common and entire subsets of architectures\n- ARCH_AND_PTX : NAME | NUM.NUM | NUM.NUM(NUM.NUM) | NUM.NUM+PTX\n- NAME: Fermi Kepler Maxwell Kepler+Tegra Kepler+Tesla Maxwell+Tegra Pascal Volta Turing\n- NUM: Any number. Only those pairs are currently accepted by NVCC though:\n       2.0 2.1 3.0 3.2 3.5 3.7 5.0 5.2 5.3 6.0 6.2 7.0 7.2 7.5\")\noption(USE_NCCL \"Use NVidia NCCL with CUDA\" OFF)\noption(USE_OPENCV \"Build with OpenCV support\" ON)\noption(USE_OPENMP \"Build with Openmp support\" ON)\noption(USE_FATBIN_COMPRESSION \"Compress nvcc fatbin output\" ON)\ncmake_dependent_option(USE_NVML \"Build with nvml support if found\" ON \"USE_CUDA\" OFF)\ncmake_dependent_option(USE_CUDNN \"Build with cudnn support\" ON \"USE_CUDA\" OFF) # one could set CUDNN_ROOT for search path\ncmake_dependent_option(USE_CUTENSOR \"Build with cuTENSOR support\" ON \"USE_CUDA\" OFF) # one could set CUTENSOR_ROOT for search path\ncmake_dependent_option(USE_NVTX \"Build with nvtx support if found\" ON \"USE_CUDA\" OFF)\ncmake_dependent_option(USE_SSE \"Build with x86 SSE instruction support\" ON\n  \"CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL amd64\" OFF)\noption(USE_F16C \"Build with x86 F16C instruction support\" ON) # autodetects support if ON\noption(USE_LAPACK \"Build with lapack support\" ON)\noption(USE_MKL_LAYERNORM \"Use layer normalization from MKL, which is currently slower than internal. No effect unless USE_BLAS=MKL (or mkl).\" OFF)\nif((NOT APPLE) AND (NOT MSVC) AND (CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL \"x86_64\") AND (NOT CMAKE_CROSSCOMPILING))\n  option(USE_ONEDNN \"Build with oneDNN support\" ON)\nelse()\n  option(USE_ONEDNN \"Build with oneDNN support\" OFF)\nendif()\ncmake_dependent_option(USE_INTGEMM \"Build with x86_64 intgemm library for low-precision multiplication\" ON \"CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64\" OFF)\nif(NOT MSVC)\n  option(USE_OPERATOR_TUNING  \"Enable auto-tuning of operators\" ON)\nelse()\n  option(USE_OPERATOR_TUNING  \"Enable auto-tuning of operators\" OFF)\nendif()\noption(USE_GPERFTOOLS \"Build with GPerfTools support\" OFF)\noption(USE_JEMALLOC \"Build with Jemalloc support\" OFF)\noption(USE_LIBJPEG_TURBO \"Use libjpeg-turbo\" OFF)\noption(USE_DIST_KVSTORE \"Build with DIST_KVSTORE support\" OFF)\noption(USE_PLUGINS_WARPCTC \"Use WARPCTC Plugins\" OFF)\noption(USE_CPP_PACKAGE \"Build C++ Package\" OFF)\noption(USE_MXNET_LIB_NAMING \"Use MXNet library naming conventions.\" ON)\noption(USE_GPROF \"Compile with gprof (profiling) flag\" OFF)\noption(USE_VTUNE \"Enable use of Intel Amplifier XE (VTune)\" OFF) # one could set VTUNE_ROOT for search path\noption(USE_TVM_OP \"Enable use of TVM operator build system.\" OFF)\noption(BUILD_CPP_EXAMPLES \"Build cpp examples\" ON)\noption(INSTALL_EXAMPLES \"Install the example source files.\" OFF)\noption(USE_SIGNAL_HANDLER \"Print stack traces on segfaults.\" ON)\noption(USE_TENSORRT \"Enable inference optimization with TensorRT.\" OFF)\noption(USE_ASAN \"Enable Clang/GCC ASAN sanitizers.\" OFF)\ncmake_dependent_option(ENABLE_TESTCOVERAGE \"Enable compilation with test coverage metric output\" OFF \"NOT MSVC\" OFF)\noption(BUILD_EXTENSION_PATH \"Path to extension to build\" \"\")\noption(BUILD_CYTHON_MODULES \"Build cython modules.\" OFF)\noption(COLORIZE_OUTPUT \"Colorize output during compilation\" ON)\noption(LOG_FATAL_THROW \"Log exceptions but do not abort\" ON)\ncmake_dependent_option(USE_SPLIT_ARCH_DLL \"Build a separate DLL for each Cuda arch (Windows only).\" ON \"MSVC\" OFF)\ncmake_dependent_option(USE_CCACHE \"Attempt using CCache to wrap the compilation\" ON \"UNIX\" OFF)\ncmake_dependent_option(MXNET_FORCE_SHARED_CRT \"Build with dynamic CRT on Windows (/MD)\" ON \"MXNET_BUILD_SHARED_LIBS\" OFF)\n\nmessage(STATUS \"CMAKE_CROSSCOMPILING ${CMAKE_CROSSCOMPILING}\")\nmessage(STATUS \"CMAKE_HOST_SYSTEM_PROCESSOR ${CMAKE_HOST_SYSTEM_PROCESSOR}\")\nmessage(STATUS \"CMAKE_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}\")\n\nmessage(STATUS \"CMAKE_SYSTEM_NAME ${CMAKE_SYSTEM_NAME}\")\n\nfind_package(Git QUIET)\nif(${GIT_FOUND})\n  execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD \n    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}\n    OUTPUT_VARIABLE GIT_BRANCH\n    RESULT_VARIABLE BRANCH_FAILED\n  )\n  execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse HEAD\n    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}\n    OUTPUT_VARIABLE GIT_COMMIT\n    RESULT_VARIABLE COMMIT_FAILED\n  )\n  if(NOT BRANCH_FAILED)\n    string(REGEX REPLACE \"\\n$\" \"\" GIT_BRANCH \"${GIT_BRANCH}\")\n    add_compile_definitions(MXNET_BRANCH=\"${GIT_BRANCH}\")\n  else()\n    add_compile_definitions(MXNET_BRANCH=\"Unavailable\")\n  endif()\n  if(NOT COMMIT_FAILED)\n    string(REGEX REPLACE \"\\n$\" \"\" GIT_COMMIT \"${GIT_COMMIT}\")\n    add_compile_definitions(MXNET_COMMIT_HASH=\"${GIT_COMMIT}\")\n  else()\n    add_compile_definitions(MXNET_COMMIT_HASH=\"Unavailable\")\n  endif()\nendif()\n\nif(USE_TVM_OP)\n  add_definitions(-DMXNET_USE_TVM_OP=1)\nendif()\n\nif(MXNET_FORCE_SHARED_CRT)\n  set(DMLC_FORCE_SHARED_CRT ON)\n  set(gtest_force_shared_crt ON)\nendif()\n\nmessage(STATUS \"CMake version '${CMAKE_VERSION}' using generator '${CMAKE_GENERATOR}'\")\nif(USE_CUDA)\n  cmake_minimum_required(VERSION 3.13.2)  # CUDA 10 (Turing) detection available starting 3.13.2\n  include(CheckLanguage)\n  check_language(CUDA)\n  if(NOT CMAKE_CUDA_COMPILER AND UNIX AND EXISTS \"/usr/local/cuda/bin/nvcc\")\n    set(ENV{CUDACXX} \"/usr/local/cuda/bin/nvcc\")\n    message(WARNING \"CMAKE_CUDA_COMPILER guessed: \" $ENV{CUDACXX} \"\\n\"\n      \"Please fix your cuda installation: https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#mandatory-post\")\n  endif()\n  enable_language(CUDA)\n  set(CMAKE_CUDA_STANDARD 14)\n  set(CMAKE_CUDA_STANDARD_REQUIRED ON)\nendif()\n\nif(UNIX)\n  set(CMAKE_POSITION_INDEPENDENT_CODE ON)\nendif()\n\nif(USE_CCACHE)\n  find_program(CCACHE_PROGRAM ccache)\n  if(CCACHE_PROGRAM)\n    set(CMAKE_C_COMPILER_LAUNCHER \"${CCACHE_PROGRAM}\")\n    set(CMAKE_CXX_COMPILER_LAUNCHER \"${CCACHE_PROGRAM}\")\n    set(CMAKE_CUDA_COMPILER_LAUNCHER \"${CCACHE_PROGRAM}\")\n  else()\n    message(STATUS \"Could not find CCache. Consider installing CCache to speed up compilation.\")\n  endif()\nendif()\n\nif(MSVC)\n  set(SYSTEM_ARCHITECTURE x86_64)\n  enable_language(ASM_MASM)\nelse()\n  execute_process(COMMAND uname -m COMMAND tr -d '\\n' OUTPUT_VARIABLE SYSTEM_ARCHITECTURE)\nendif()\n\nif(CMAKE_BUILD_TYPE STREQUAL \"Distribution\")\n  if(UNIX AND NOT APPLE)\n    set(CMAKE_BUILD_WITH_INSTALL_RPATH ON)\n    set(CMAKE_INSTALL_RPATH $\\{ORIGIN\\})\n    # Enforce DT_PATH instead of DT_RUNPATH\n    set(CMAKE_SHARED_LINKER_FLAGS \"-Wl,--disable-new-dtags\")\n    set(CMAKE_EXE_LINKER_FLAGS \"-Wl,--disable-new-dtags\")\n  endif()\n  set(Protobuf_USE_STATIC_LIBS ON)\nendif()\n\nset(CMAKE_MODULE_PATH \"${CMAKE_CURRENT_SOURCE_DIR}/cmake/upstream;${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}\")\n\nSET(EXTRA_OPERATORS \"\" CACHE PATH \"EXTRA OPERATORS PATH\")\n\nif(\"$ENV{VERBOSE}\" STREQUAL \"1\")\n  message(STATUS \" Verbose Makefile ACTIVATED\")\n  set(CMAKE_VERBOSE_MAKEFILE ON)\nendif()\n\n#Switch off modern thread local for dmlc-core, please see: https://github.com/dmlc/dmlc-core/issues/571#issuecomment-543467484\nadd_definitions(-DDMLC_MODERN_THREAD_LOCAL=0)\n# disable stack trace in exception by default.\nadd_definitions(-DDMLC_LOG_STACK_TRACE_SIZE=0)\n\nadd_definitions(-DDMLC_USE_CXX11)\nadd_definitions(-DDMLC_STRICT_CXX11)\nadd_definitions(-DDMLC_USE_CXX14)\nadd_definitions(-DMSHADOW_IN_CXX11)\n\nif(\"${CMAKE_CXX_COMPILER_ID}\" MATCHES \"Clang\")\n  if(${COLORIZE_OUTPUT})\n    string(APPEND CMAKE_CXX_FLAGS \" -fcolor-diagnostics\")\n  endif()\nendif()\nif(\"${CMAKE_CXX_COMPILER_ID}\" STREQUAL \"GNU\" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 4.9)\n  if(${COLORIZE_OUTPUT})\n    string(APPEND CMAKE_CXX_FLAGS \" -fdiagnostics-color=always\")\n  endif()\nendif()\n\nif(MSVC)\n  add_definitions(-D_SCL_SECURE_NO_WARNINGS)\n  add_definitions(-D_CRT_SECURE_NO_WARNINGS)\n  add_definitions(-DMXNET_EXPORTS)\n  add_definitions(-DNNVM_EXPORTS)\n  add_definitions(-DNOMINMAX)\n  set(CMAKE_C_FLAGS \"/MP\")\n  set(CMAKE_CXX_FLAGS \"${CMAKE_C_FLAGS} ${CMAKE_CXX_FLAGS} /bigobj\")\nelse()\n  include(CheckCXXCompilerFlag)\n  set(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} -Wall -Wno-sign-compare\")\n  if(CMAKE_BUILD_TYPE STREQUAL \"Debug\")\n    set(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} -O0 -g\")\n    set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -D_GLIBCXX_ASSERTIONS\")\n  elseif(CMAKE_BUILD_TYPE STREQUAL \"RelWithDebInfo\")\n    set(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} -O3 -g\")\n    set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -D_GLIBCXX_ASSERTIONS\")\n  else()\n    add_definitions(-DNDEBUG=1)\n    set(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} -O3\")\n  endif()\n  set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS}\")\nendif()\n\nif(NOT mxnet_LINKER_LIBS)\n  set(mxnet_LINKER_LIBS \"\")\nendif()\n\nif(USE_GPROF)\n  message(STATUS \"Using GPROF\")\n  set(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} -fno-omit-frame-pointer -g -pg\")\n  set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -g -pg\")\n  set(CMAKE_LINK_LIBRARY_FILE_FLAG \"${CMAKE_LINK_LIBRARY_FILE_FLAG} -g -pg\")\nendif()\n\nif(USE_VTUNE)\n  message(STATUS \"Using VTUNE\")\n  if(NOT VTUNE_ROOT)\n    set(VTUNE_ROOT /opt/intel/vtune_amplifier_xe_2017)\n  endif()\n  set(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} -fno-omit-frame-pointer -g -pg\")\n  set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -g -pg\")\n  set(CMAKE_LINK_LIBRARY_FILE_FLAG \"${CMAKE_LINK_LIBRARY_FILE_FLAG} -g -pg\")\n  add_definitions(-DMXNET_USE_VTUNE=1)\n  include_directories(${VTUNE_ROOT}/include)\n  list(APPEND mxnet_LINKER_LIBS ${VTUNE_ROOT}/lib64/libittnotify.a)\n  list(APPEND mxnet_LINKER_LIBS dl)\nendif()\n\nif(USE_TENSORRT)\n  message(STATUS \"Using TensorRT\")\n  set(ONNX_PATH 3rdparty/onnx-tensorrt/third_party/onnx/build/)\n  set(ONNX_TRT_PATH 3rdparty/onnx-tensorrt/build/)\n\n  include_directories(${ONNX_PATH})\n  include_directories(3rdparty/onnx-tensorrt/)\n  include_directories(3rdparty/)\n  include_directories(3rdparty/onnx-tensorrt/third_party/onnx/)\n  add_definitions(-DMXNET_USE_TENSORRT=1)\n  add_definitions(-DONNX_NAMESPACE=onnx)\n  add_definitions(-DONNX_ML=1)\n\n  find_package(Protobuf REQUIRED)\n\n  find_library(ONNX_LIBRARY NAMES libonnx.so REQUIRED\n          PATHS ${ONNX_PATH}\n          DOC \"Path to onnx library.\")\n  find_library(ONNX_PROTO_LIBRARY NAMES libonnx_proto.so REQUIRED\n          PATHS ${ONNX_PATH}\n          DOC \"Path to onnx_proto library.\")\n  find_library(ONNX_TRT_PARSER_LIBRARY NAMES libnvonnxparser.so REQUIRED\n          PATHS ${ONNX_TRT_PATH}\n          DOC \"Path to onnx_proto parser library.\")\n\n  list(APPEND mxnet_LINKER_LIBS libnvinfer.so ${ONNX_TRT_PARSER_LIBRARY}\n          ${ONNX_PROTO_LIBRARY} ${ONNX_LIBRARY} ${PROTOBUF_LIBRARY})\nendif()\n\ninclude(GNUInstallDirs)\nif(USE_MKL_LAYERNORM)\n  add_definitions(-DMXNET_USE_MKL_LAYERNORM=1)\nendif()\nif(USE_ONEDNN)\n  # CPU architecture (e.g., C5) can't run on another architecture (e.g., g3).\n  if(MSVC)\n    set(CMAKE_CXX_FLAGS_DEBUG \"${CMAKE_CXX_FLAGS_DEBUG} /EHsc\")\n    set(CMAKE_CXX_FLAGS_RELEASE \"${CMAKE_CXX_FLAGS_RELEASE} /EHsc /Gy\")\n    set(CMAKE_CXX_FLAGS_RELWITHDEBINFO \"${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /EHsc /Gy\")\n    set(CMAKE_CXX_FLAGS_MINSIZEREL \"${CMAKE_CXX_FLAGS_MINSIZEREL} /EHsc /Gy\")\n    if(NOT MXNET_FORCE_SHARED_CRT)\n      set(CMAKE_CXX_FLAGS_DEBUG \"${CMAKE_CXX_FLAGS_DEBUG} /MTd\")\n      set(CMAKE_CXX_FLAGS_RELEASE \"${CMAKE_CXX_FLAGS_RELEASE} /MT\")\n      set(CMAKE_CXX_FLAGS_RELWITHDEBINFO \"${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /MT\")\n      set(CMAKE_CXX_FLAGS_MINSIZEREL \"${CMAKE_CXX_FLAGS_MINSIZEREL} /MT\")\n    endif()\n  endif()\n\n  function(load_onednn)\n    set(ONEDNN_BUILD_TESTS OFF CACHE INTERNAL \"\" FORCE)\n    set(ONEDNN_BUILD_EXAMPLES OFF CACHE INTERNAL \"\" FORCE)\n    set(ONEDNN_ARCH_OPT_FLAGS \"\" CACHE INTERNAL \"\" FORCE)\n    set(ONEDNN_ENABLE_JIT_PROFILING OFF CACHE INTERNAL \"\" FORCE)\n    set(ONEDNN_LIBRARY_TYPE STATIC CACHE INTERNAL \"\" FORCE)\n    set(ONEDNN_ENABLE_CONCURRENT_EXEC ON CACHE INTERNAL \"\" FORCE)\n    set(ONEDNN_ENABLE_PRIMITIVE_CACHE ON CACHE INTERNAL \"\" FORCE)\n\n    if(NOT USE_OPENMP)\n      set(ONEDNN_CPU_RUNTIME SEQ CACHE INTERNAL \"\" FORCE)\n    endif()\n\n    set(CMAKE_INSTALL_INCLUDEDIR \"${CMAKE_INSTALL_INCLUDEDIR}/onednn\")\n    add_subdirectory(3rdparty/onednn)\n  endfunction()\n  load_onednn()\n  include_directories(3rdparty/onednn/include)\n  include_directories(${PROJECT_BINARY_DIR}/3rdparty/onednn/include)\n  add_definitions(-DMXNET_USE_ONEDNN=1)\n  list(APPEND mxnet_LINKER_LIBS dnnl)\n  set_target_properties(dnnl PROPERTIES CXX_CLANG_TIDY \"\")  # don't lint 3rdparty dependency\nendif()\n\nif(USE_CPP_PACKAGE)\n    add_definitions(-DMXNET_USE_CPP_PACKAGE=1)\nendif()\n\nif(USE_INTGEMM)\n  message(STATUS \"Using intgemm\")\n  add_subdirectory(3rdparty/intgemm EXCLUDE_FROM_ALL)\n  add_definitions(-DMXNET_USE_INTGEMM=1)\nendif()\n\n# Allow Cuda compiles outside of src tree to find things in 'src' and 'include'\ninclude_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)\ninclude_directories(${CMAKE_CURRENT_SOURCE_DIR}/src)\n\ncmake_dependent_option(USE_INT64_TENSOR_SIZE \"Use int64_t to represent the total number of elements in a tensor\" ON \"CMAKE_SIZEOF_VOID_P EQUAL 8\" OFF)\n\ninclude(cmake/ChooseBlas.cmake)\n\nif(USE_ASAN)\n  set(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} -fno-omit-frame-pointer -fsanitize=address\")\n  set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -fsanitize=address\")\n  set(CMAKE_LINKER_FLAGS \"${CMAKE_LINKER_FLAGS} -fsanitize=address\")\n  set(GTEST_LIBRARIES \"${GTEST_LIBRARIES} -fsanitize=address\")\n  list(APPEND mxnet_LINKER_LIBS asan)\nendif()\n\nlist(APPEND mxnet_LINKER_LIBS ${mshadow_LINKER_LIBS})\nmessage(\"After choosing blas, linking to ${mxnet_LINKER_LIBS}\")\n\nforeach(var ${C_CXX_INCLUDE_DIRECTORIES})\n    include_directories(${var})\nendforeach()\n\ninclude_directories(\"include\")\ninclude_directories(\"3rdparty/tvm/nnvm/include\")\ninclude_directories(\"3rdparty/tvm/include\")\ninclude_directories(\"3rdparty/dmlc-core/include\")\ninclude_directories(\"3rdparty/dlpack/include\")\n\nif(UNIX)\n  find_library(RTLIB rt)\n  if(RTLIB)\n    list(APPEND mxnet_LINKER_LIBS ${RTLIB})\n  endif()\nendif()\n\nset(ALT_MALLOC_FLAGS \"-fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free\")\n\n# ---[ gperftools\nif(USE_GPERFTOOLS)\n  find_package(Gperftools)\n  if(GPERFTOOLS_FOUND)\n    message(STATUS \"Using Gperftools malloc (tcmalloc)\")\n    include_directories(${GPERFTOOLS_INCLUDE_DIR})\n    set(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} ${ALT_MALLOC_FLAGS}\")\n    set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} ${ALT_MALLOC_FLAGS}\")\n    set(mxnet_LINKER_LIBS ${mxnet_LINKER_LIBS} ${GPERFTOOLS_LIBRARIES})\n    set(USE_JEMALLOC 0)\n  endif()\nendif()\n\n# ---[ jemalloc\nif(USE_JEMALLOC)\n  if(GPERFTOOLS_FOUND)\n    message(ERROR \" Only one of USE_JEMALLOC and USE_GPERFTOOLS can be defined at once\")\n  endif()\n  find_package(JeMalloc)\n  if(JEMALLOC_FOUND)\n    message(STATUS \"Using JEMalloc malloc\")\n    add_definitions(-DUSE_JEMALLOC)\n    set(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} ${ALT_MALLOC_FLAGS}\")\n    set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} ${ALT_MALLOC_FLAGS}\")\n    include_directories(${JEMALLOC_INCLUDE_DIRS})\n    set(mxnet_LINKER_LIBS ${mxnet_LINKER_LIBS} ${JEMALLOC_LIBRARIES})\n  endif()\nendif()\n\nif(USE_LIBJPEG_TURBO)\n  find_package(PkgConfig REQUIRED)\n  pkg_search_module(TURBOJPEG REQUIRED libturbojpeg)\n  include_directories(SYSTEM ${TURBOJPEG_INCLUDE_DIRS})\n  list(APPEND mxnet_LINKER_LIBS ${TURBOJPEG_LINK_LIBRARIES})\n  add_definitions(-DMXNET_USE_LIBJPEG_TURBO=1)\nelse()\n  add_definitions(-DMXNET_USE_LIBJPEG_TURBO=0)\nendif()\n\n# ---[ OpenCV\nif(USE_OPENCV)\n  find_package(OpenCV COMPONENTS core highgui imgproc imgcodecs)\n  if(NOT OpenCV_FOUND) # if not OpenCV 3.x, then imgcodecs are not found\n    message(STATUS \"OpenCV imgcodecs missing\")\n    find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc)\n  endif()\n  include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS})\n  list(APPEND mxnet_LINKER_LIBS ${OpenCV_LIBS})\n  message(STATUS \"OpenCV ${OpenCV_VERSION} found (${OpenCV_CONFIG_PATH})\")\n  message(STATUS \" OpenCV_LIBS=${OpenCV_LIBS}\")\n  add_definitions(-DMXNET_USE_OPENCV=1)\nelse(USE_OPENCV)\n  message(STATUS \"OpenCV Disabled\")\n  add_definitions(-DMXNET_USE_OPENCV=0)\nendif()\n\n# ---[ OpenMP\nif(USE_OPENMP)\n  find_package(OpenMP REQUIRED)\n  if(OPENMP_FOUND)\n    set(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}\")\n    set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}\")\n    # Enable pragma omp simd\n    # \"While the name of this switch is 'experimental', the switch itself, and\n    # the functionality it enables is fully supported and production-ready.\n    # The name reflects that it doesn’t enable any complete subset or\n    # version of an OpenMP standard.\"\n    # -- https://devblogs.microsoft.com/cppblog/simd-extension-to-c-openmp-in-visual-studio/\n    if(MSVC)\n      set(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} -openmp:experimental\")\n      set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -openmp:experimental\")\n    endif()\n    if(NOT BLAS STREQUAL \"MKL\")\n      # Linker flags for Intel OMP are already set in case MKL is used. Only set if not MKL\n      set(CMAKE_SHARED_LINKER_FLAGS \"${CMAKE_SHARED_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}\")\n      set(CMAKE_EXE_LINKER_FLAGS \"${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}\")\n    endif()\n    add_definitions(-DMXNET_USE_OPENMP=1)\n  endif()\nelseif(UNIX)\n  # Prefer compiler pthread flag. This is the recommended way, but not backwards\n  # compatible and thus not the cmake default.\n  set(THREADS_PREFER_PTHREAD_FLAG ON)\n  find_package(Threads REQUIRED)\n  list(APPEND mxnet_LINKER_LIBS Threads::Threads)\nendif()\n\n# ---[ LAPack\nif(USE_LAPACK)\n  message(\"USE_LAPACK is ON\")\n  add_definitions(-DMXNET_USE_LAPACK=1)\n  if(NOT USE_LAPACKE_INTERFACE)\n    # BLAS=open case is handled in ChooseBlas.cmake\n    if(NOT MSVC AND NOT CMAKE_BUILD_TYPE STREQUAL \"Distribution\"\n       AND NOT BLAS STREQUAL \"Open\" AND NOT BLAS STREQUAL \"open\")\n      list(APPEND mxnet_LINKER_LIBS lapack)\n    endif()\n  endif()\nendif()\n\n# ---[ jemalloc\nif(USE_JEMALLOC)\n  find_package(JeMalloc)\n  if(JEMALLOC_FOUND)\n    add_definitions(-DUSE_JEMALLOC)\n    include_directories(${JEMALLOC_INCLUDE_DIRS})\n    set(mxnet_LINKER_LIBS ${mxnet_LINKER_LIBS} ${JEMALLOC_LIBRARIES})\n  endif()\nendif()\n\ninclude(CTest)\nset(GTEST_ROOT \"${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/googletest/googletest\")\nset(GTEST_INCLUDE_DIR ${GTEST_ROOT}/include)\nset(GTEST_MAIN_LIBRARY gtest_main)\nset(GTEST_LIBRARY gtest)\n\nadd_subdirectory(${GTEST_ROOT})\nset_target_properties(gtest PROPERTIES CXX_CLANG_TIDY \"\")  # don't lint 3rdparty dependency\nset_target_properties(gtest_main PROPERTIES CXX_CLANG_TIDY \"\")  # don't lint 3rdparty dependency\nfind_package(GTest REQUIRED)\n\n# cudnn detection\nif(USE_CUDNN)\n  find_package(CUDNN)\n  if(CUDNN_FOUND)\n    add_definitions(-DUSE_CUDNN)\n    include_directories(SYSTEM ${CUDNN_INCLUDE})\n    list(APPEND mxnet_LINKER_LIBS ${CUDNN_LIBRARY})\n  else()\n    set(USE_CUDNN OFF)\n  endif()\nendif()\n\n# cutensor detection\nif(USE_CUTENSOR)\n  find_package(CUTENSOR)\n  if(CUTENSOR_FOUND)\n    add_definitions(-DUSE_CUTENSOR)\n    include_directories(SYSTEM ${CUTENSOR_INCLUDE})\n    list(APPEND mxnet_LINKER_LIBS ${CUTENSOR_LIBRARY})\n  else()\n    set(USE_CUTENSOR OFF)\n  endif()\nendif()\n\nif(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/dmlc-core/cmake)\n  add_subdirectory(\"3rdparty/dmlc-core\")\n  set_target_properties(dmlc PROPERTIES CXX_CLANG_TIDY \"\")  # don't lint 3rdparty dependency\nendif()\n\nFILE(GLOB_RECURSE SOURCE \"src/*.cc\" \"src/*.h\" \"include/*.h\")\nFILE(GLOB_RECURSE CUDA \"src/*.cu\" \"src/*.cuh\")\n\nif(MSVC)\n  FILE(GLOB_RECURSE TVM_BRIDGE_SOURCE \"src/*/tvm_bridge.cc\")\n  list(REMOVE_ITEM SOURCE ${TVM_BRIDGE_SOURCE})\nendif()\n\nif(NOT USE_INTGEMM)\n  FILE(GLOB_RECURSE INTGEMM_OPERATOR_SOURCE \"src/operator/contrib/intgemm/*.cc\" \"src/operator/contrib/intgemm/*.h\")\n  list(REMOVE_ITEM SOURCE ${INTGEMM_OPERATOR_SOURCE})\nendif()\n\n# add nnvm to source\nFILE(GLOB_RECURSE NNVMSOURCE\n  3rdparty/tvm/nnvm/src/c_api/*.cc\n  3rdparty/tvm/nnvm/src/core/*.cc\n  3rdparty/tvm/nnvm/src/pass/*.cc\n  3rdparty/tvm/nnvm/src/c_api/*.h\n  3rdparty/tvm/nnvm/src/core/*.h\n  3rdparty/tvm/nnvm/src/pass/*.h\n  3rdparty/tvm/nnvm/include/*.h)\nadd_library(nnvm OBJECT ${NNVMSOURCE})\nset_target_properties(nnvm PROPERTIES CXX_CLANG_TIDY \"\")  # don't lint 3rdparty dependency\nlist(APPEND SOURCE $<TARGET_OBJECTS:nnvm>)\n\nadd_library(miniz STATIC \"3rdparty/miniz/miniz.c\")\ntarget_include_directories(miniz PUBLIC \"3rdparty/miniz\")\nlist(APPEND mxnet_LINKER_LIBS miniz)\n\n# add source group\nFILE(GLOB_RECURSE GROUP_SOURCE \"src/*.cc\" \"3rdparty/tvm/nnvm/*.cc\" \"plugin/*.cc\")\nFILE(GLOB_RECURSE GROUP_Include \"src/*.h\" \"3rdparty/tvm/nnvm/*.h\" \"3rdparty/mshadow/mshadow/*.h\" \"plugin/*.h\")\nFILE(GLOB_RECURSE GROUP_CUDA \"src/*.cu\" \"src/*.cuh\" \"3rdparty/mshadow/mshadow/*.cuh\" \"plugin/*.cu\"\n  \"plugin/*.cuh\" \"3rdparty/nvidia_cub/cub/*.cuh\")\nassign_source_group(\"Source\" ${GROUP_SOURCE})\nassign_source_group(\"Include\" ${GROUP_Include})\nassign_source_group(\"CUDA\" ${GROUP_CUDA})\n\nif(USE_PLUGINS_WARPCTC)\n    set(WARPCTC_INCLUDE  \"\" CACHE PATH \"WARPCTC include\")\n    set(WARPCTC_LIB_DEBUG  \"\" CACHE FILEPATH \"WARPCTC lib\")\n    set(WARPCTC_LIB_RELEASE  \"\" CACHE FILEPATH \"WARPCTC lib\")\n    include_directories(SYSTEM ${WARPCTC_INCLUDE})\n    list(APPEND mxnet_LINKER_LIBS ${WARPCTC_LIB})\n    FILE(GLOB_RECURSE PLUGINS_SOURCE \"plugin/warpctc/*.cc\" \"plugin/warpctc/*.h\")\n    FILE(GLOB_RECURSE PLUGINS_CUSRC \"plugin/warpctc/*.cu\")\n    list(APPEND SOURCE ${PLUGINS_SOURCE})\n    list(APPEND CUDA ${PLUGINS_CUSRC})\nendif()\n\nif(USE_OPERATOR_TUNING AND USE_OPENMP)\n  add_definitions(-DMXNET_USE_OPERATOR_TUNING=1)\nendif()\n\nif(NOT (EXTRA_OPERATORS STREQUAL \"\"))\n    mxnet_source_group(\"Extra\"   GLOB_RECURSE \"${EXTRA_OPERATORS}/*.cc\")\n    mxnet_source_group(\"Extra\\\\Cuda\"   GLOB_RECURSE \"${EXTRA_OPERATORS}/*.cu\")\n    FILE(GLOB_RECURSE EXTRA_SRC \"${EXTRA_OPERATORS}/*.cc\")\n    FILE(GLOB_RECURSE EXTRA_CUSRC \"${EXTRA_OPERATORS}/*.cu\")\n    list(APPEND SOURCE ${EXTRA_SRC} ${EXTRA_CUSRC})\nendif()\n\nif(MSVC AND NOT MXNET_FORCE_SHARED_CRT)\n  foreach(flag_var\n        CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE\n        CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)\n    if(${flag_var} MATCHES \"/MD\")\n      string(REGEX REPLACE \"/MD\" \"/MT\" ${flag_var} \"${${flag_var}}\")\n    elseif(${flag_var} MATCHES \"/MDd\")\n      string(REGEX REPLACE \"/MDd\" \"/MTd\" ${flag_var} \"${${flag_var}}\")    \n    endif()\n  endforeach(flag_var)\nendif()\n\nif(USE_CUDA)\n  # CUDA_SELECT_NVCC_ARCH_FLAGS is not deprecated, though part of deprecated\n  # FindCUDA https://gitlab.kitware.com/cmake/cmake/issues/19199\n  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/upstream/select_compute_arch.cmake)\n  CUDA_SELECT_NVCC_ARCH_FLAGS(CUDA_ARCH_FLAGS ${MXNET_CUDA_ARCH})\n  message(\"-- CUDA: Using the following NVCC architecture flags ${CUDA_ARCH_FLAGS}\")\n  set(arch_code_list)\n  foreach(arch_str ${CUDA_ARCH_FLAGS})\n    if((arch_str MATCHES \".*sm_[0-9]+\"))\n      string( REGEX REPLACE  \".*sm_([0-9]+)\" \"\\\\1\" arch_code ${arch_str} )\n      list(APPEND arch_code_list ${arch_code})\n    endif()\n  endforeach()\n\n  string(REPLACE \";\" \" \" CUDA_ARCH_FLAGS_SPACES \"${CUDA_ARCH_FLAGS}\")\n\n  find_package(CUDAToolkit REQUIRED cublas cufft cusolver curand nvrtc\n    OPTIONAL_COMPONENTS nvToolsExt)\n\n  list(APPEND mxnet_LINKER_LIBS CUDA::cudart CUDA::cublas CUDA::cufft CUDA::cusolver CUDA::curand\n                                CUDA::nvrtc)\n  list(APPEND SOURCE ${CUDA})\n  add_definitions(-DMXNET_USE_CUDA=1)\n\n  if(UNIX)\n    if(USE_NVML)\n      find_package(NVML)\n      if(NVML_FOUND)\n        include_directories(${NVML_INCLUDE_DIRS})\n        list(APPEND mxnet_LINKER_LIBS ${NVML_LIBRARIES})\n        add_definitions(-DMXNET_USE_NVML=1)\n      else()\n        add_definitions(-DMXNET_USE_NVML=0)\n        message(WARNING \"Could not find NVML libraries\")\n      endif()\n    endif()\n  endif()\n  if(USE_NCCL)\n    find_package(NCCL)\n    if(NCCL_FOUND)\n      include_directories(${NCCL_INCLUDE_DIRS})\n      list(APPEND mxnet_LINKER_LIBS ${NCCL_LIBRARIES})\n      add_definitions(-DMXNET_USE_NCCL=1)\n    else()\n      add_definitions(-DMXNET_USE_NCCL=0)\n      message(WARNING \"Could not find NCCL libraries\")\n    endif()\n  endif()\n  if(UNIX)\n    if(USE_NVTX AND CUDA_nvToolsExt_LIBRARY)\n      list(APPEND mxnet_LINKER_LIBS CUDA::nvToolsExt)\n      add_definitions(-DMXNET_USE_NVTX=1)\n    else()\n      message(\"Building without NVTX support.\")\n    endif()\n  endif()\n\n  include_directories(${CUDAToolkit_INCLUDE_DIRS})\n  link_directories(${CUDAToolkit_LIBRARY_DIR})\nendif()\n\nif(CUDAToolkit_VERSION_MAJOR LESS \"11\")\n  include_directories(\"3rdparty/nvidia_cub\")\nendif()\n\nif(MSVC)\n  set(CMAKE_CXX_FLAGS_DEBUG \"${CMAKE_CXX_FLAGS_DEBUG} /EHsc\")\n  set(CMAKE_CXX_FLAGS_RELEASE \"${CMAKE_CXX_FLAGS_RELEASE} /EHsc /Gy\")\n  set(CMAKE_CXX_FLAGS_MINSIZEREL \"${CMAKE_CXX_FLAGS_MINSIZEREL} /EHsc /Gy\")\n  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO \"${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /EHsc /Gy\")\n  set(CMAKE_SHARED_LINKER_FLAGS_RELEASE \"${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /OPT:REF /OPT:ICF\")\n  set(CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL \"${CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL} /OPT:REF /OPT:ICF\")\n  set(CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO \"${CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO} /OPT:REF /OPT:ICF\")\n\nendif()\n\n# Add cmake targets\nadd_subdirectory(\"3rdparty/mshadow\")\n\nif(USE_CUDA AND USE_FATBIN_COMPRESSION)\n  string(APPEND CMAKE_CUDA_FLAGS \" --fatbin-options --compress-all\")\n  message(\"-- CUDA: Adding NVCC options: --fatbin-options --compress-all\")\nendif()\nif(UNIX)\n  string(APPEND CMAKE_CUDA_FLAGS \" ${CUDA_ARCH_FLAGS_SPACES}\")\n  # Create dummy file since we want an empty shared library before linking\n  set(DUMMY_SOURCE ${CMAKE_BINARY_DIR}/dummy.c)\n  file(WRITE ${DUMMY_SOURCE} \"\")\n  if(MXNET_BUILD_SHARED_LIBS)\n    add_library(mxnet SHARED ${SOURCE})\n  else()\n    add_library(mxnet STATIC ${SOURCE})\n  endif()\n  target_link_libraries(mxnet PUBLIC mshadow)\n  target_link_libraries(mxnet PUBLIC ${CMAKE_DL_LIBS})\n  if(CMAKE_BUILD_TYPE STREQUAL \"RelWithDebInfo\")\n    target_compile_options(mxnet PRIVATE \"$<$<COMPILE_LANGUAGE:CXX>:-Werror>\")\n    # Ignore erroneous compiler warnings:\n    # 1) variables used in '#pragma omp parallel' are considered unused\n    target_compile_options(mxnet PRIVATE \"$<$<COMPILE_LANGUAGE:CXX>:-Wno-error=unused-variable>\")\n    if(USE_CUDA)\n      # Note: \"=\" is required to avoid breaking ccache\n      string(APPEND CMAKE_CUDA_FLAGS \" -Werror=cross-execution-space-call\")\n    endif()\n  endif()\n  if(ENABLE_TESTCOVERAGE)\n    find_program(GCOV_PATH gcov REQUIRED)\n    target_compile_options(mxnet PUBLIC \"--coverage\")\n    target_link_libraries(mxnet PUBLIC gcov)\n  endif()\n  if(APPLE)\n    set_target_properties(mxnet PROPERTIES LINK_FLAGS \"-Wl,-exported_symbols_list,${PROJECT_SOURCE_DIR}/cmake/libmxnet.sym\")\n  else()\n    set_target_properties(mxnet PROPERTIES LINK_FLAGS \"-Wl,--exclude-libs,ALL\")\n  endif()\nelseif(MSVC)\n  if(USE_CUDA)\n    if(USE_SPLIT_ARCH_DLL)\n      add_executable(gen_warp tools/windowsbuild/gen_warp.cpp)\n      add_library(mxnet SHARED tools/windowsbuild/warp_dll.cpp ${CMAKE_BINARY_DIR}/warp_gen_cpp.cpp\n                  ${CMAKE_BINARY_DIR}/warp_gen.asm)\n      target_link_libraries(mxnet PRIVATE cudart Shlwapi)\n      list(GET arch_code_list 0 mxnet_first_arch)\n      foreach(arch ${arch_code_list})\n        add_library(mxnet_${arch} SHARED ${SOURCE})\n        target_link_libraries(mxnet_${arch} PUBLIC mshadow)\n        target_compile_definitions(mxnet_${arch} PRIVATE -DWIN32_LEAN_AND_MEAN)\n        target_compile_options(\n          mxnet_${arch}\n          PRIVATE\n          \"$<$<COMPILE_LANGUAGE:CUDA>:--gpu-architecture=compute_${arch}>\"\n        )\n        target_compile_options(\n          mxnet_${arch}\n          PRIVATE\n          \"$<$<COMPILE_LANGUAGE:CUDA>:--gpu-code=sm_${arch},compute_${arch}>\"\n        )\n        if(MXNET_FORCE_SHARED_CRT)\n          target_compile_options(\n            mxnet_${arch} \n            PRIVATE \"$<$<AND:$<CONFIG:DEBUG>,$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-MDd -Gy /bigobj>\")\n          target_compile_options(\n            mxnet_${arch}\n            PRIVATE \"$<$<AND:$<CONFIG:RELEASE>,$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-MD -Gy /bigobj>\")          \n          target_compile_options(\n            mxnet_${arch}\n            PRIVATE \"$<$<AND:$<CONFIG:RELWITHDEBINFO>,$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-MD -Gy /bigobj>\")\n          target_compile_options(\n            mxnet_${arch}\n            PRIVATE \"$<$<AND:$<CONFIG:MINSIZEREL>,$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-MD -Gy /bigobj>\")\n        else()\n          target_compile_options(\n            mxnet_${arch} \n            PRIVATE \"$<$<AND:$<CONFIG:DEBUG>,$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-MTd -Gy /bigobj>\")\n          target_compile_options(\n            mxnet_${arch}\n            PRIVATE \"$<$<AND:$<CONFIG:RELEASE>,$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-MT -Gy /bigobj>\")          \n          target_compile_options(\n            mxnet_${arch}\n            PRIVATE \"$<$<AND:$<CONFIG:RELWITHDEBINFO>,$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-MT -Gy /bigobj>\")\n          target_compile_options(\n            mxnet_${arch}\n            PRIVATE \"$<$<AND:$<CONFIG:MINSIZEREL>,$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-MT -Gy /bigobj>\")\n        endif()\n      endforeach()\n\n      add_custom_command(\n        OUTPUT ${CMAKE_BINARY_DIR}/warp_gen_cpp.cpp ${CMAKE_BINARY_DIR}/warp_gen.asm\n        COMMAND gen_warp $<TARGET_FILE:mxnet_${mxnet_first_arch}> WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/ DEPENDS $<TARGET_FILE:mxnet_${mxnet_first_arch}>)\n    else(USE_SPLIT_ARCH_DLL)\n      string(REPLACE \";\" \" \" NVCC_FLAGS_ARCH \"${NVCC_FLAGS_ARCH}\")\n      string(APPEND CMAKE_CUDA_FLAGS \" ${CUDA_ARCH_FLAGS_SPACES}\")\n      add_library(mxnet SHARED ${SOURCE})\n      target_compile_definitions(mxnet PRIVATE -DWIN32_LEAN_AND_MEAN)\n      target_link_libraries(mxnet PUBLIC mshadow)\n      if(MXNET_FORCE_SHARED_CRT)\n        target_compile_options(\n            mxnet \n            PRIVATE \"$<$<AND:$<CONFIG:DEBUG>,$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-MDd -Gy /bigobj>\")\n        target_compile_options(\n            mxnet\n            PRIVATE \"$<$<AND:$<CONFIG:RELEASE>,$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-MD -Gy /bigobj>\")\n        target_compile_options(\n            mxnet\n            PRIVATE \"$<$<AND:$<CONFIG:RELWITHDEBINFO>,$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-MD -Gy /bigobj>\")\n        target_compile_options(\n            mxnet\n            PRIVATE \"$<$<AND:$<CONFIG:MINSIZEREL>,$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-MD -Gy /bigobj>\")\n      else()\n        target_compile_options(\n          mxnet \n          PRIVATE \"$<$<AND:$<CONFIG:DEBUG>,$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-MTd -Gy /bigobj>\")\n        target_compile_options(\n          mxnet\n          PRIVATE \"$<$<AND:$<CONFIG:RELEASE>,$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-MT -Gy /bigobj>\")          \n        target_compile_options(\n          mxnet\n          PRIVATE \"$<$<AND:$<CONFIG:RELWITHDEBINFO>,$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-MT -Gy /bigobj>\")\n        target_compile_options(\n          mxnet\n          PRIVATE \"$<$<AND:$<CONFIG:MINSIZEREL>,$<COMPILE_LANGUAGE:CUDA>>:-Xcompiler=-MT -Gy /bigobj>\")\n      endif()\n    endif(USE_SPLIT_ARCH_DLL)\n  else()\n    add_library(mxnet SHARED ${SOURCE})\n    target_compile_definitions(mxnet PRIVATE -DWIN32_LEAN_AND_MEAN)\n    target_link_libraries(mxnet PUBLIC mshadow)\n  endif()\nendif()\ntarget_compile_definitions(mxnet PUBLIC DMLC_LOG_FATAL_THROW=$<BOOL:${LOG_FATAL_THROW}>)\n\n# extension libraries (custom operators, custom subgraphs) are built by default\nadd_library(customop_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/example/extensions/lib_custom_op/gemm_lib.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/lib_api.cc)\nadd_library(transposecsr_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/example/extensions/lib_custom_op/transposecsr_lib.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/lib_api.cc)\nadd_library(transposerowsp_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/example/extensions/lib_custom_op/transposerowsp_lib.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/lib_api.cc)\nadd_library(subgraph_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/example/extensions/lib_subgraph/subgraph_lib.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/lib_api.cc)\nadd_library(pass_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/example/extensions/lib_pass/pass_lib.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/lib_api.cc)\n\nif(IS_DIRECTORY ${BUILD_EXTENSION_PATH})\n  if(MSVC)\n    message(FATAL_ERROR \"Windows builds are not support for external ops\")\n  else()\n    add_subdirectory(${BUILD_EXTENSION_PATH} ${BUILD_EXTENSION_PATH}/build)\n  endif()\nendif()\n\ntarget_include_directories(customop_lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/mxnet)\ntarget_include_directories(transposecsr_lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/mxnet)\ntarget_include_directories(transposerowsp_lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/mxnet)\ntarget_include_directories(subgraph_lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/mxnet)\ntarget_include_directories(pass_lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/mxnet)\nif(USE_CUDA)\n  add_library(customop_gpu_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/example/extensions/lib_custom_op/relu_lib.cu ${CMAKE_CURRENT_SOURCE_DIR}/example/extensions/lib_custom_op/relu_lib.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/lib_api.cc)\n  target_include_directories(customop_gpu_lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/mxnet ${CMAKE_CURRENT_SOURCE_DIR}/example/extensions/lib_custom_op)\nendif()\nif(UNIX)\n  if(USE_CUDA)\n    target_compile_options(customop_gpu_lib PUBLIC -shared)\n  endif()\nelseif(MSVC)\n  target_compile_options(customop_lib PUBLIC /LD)\n  target_compile_options(transposecsr_lib PUBLIC /LD)\n  target_compile_options(transposerowsp_lib PUBLIC /LD)\n  target_compile_options(subgraph_lib PUBLIC /LD)\n  target_compile_options(pass_lib PUBLIC /LD)\n  set_target_properties(customop_lib PROPERTIES PREFIX \"lib\")\n  set_target_properties(transposecsr_lib PROPERTIES PREFIX \"lib\")\n  set_target_properties(transposerowsp_lib PROPERTIES PREFIX \"lib\")\n  set_target_properties(subgraph_lib PROPERTIES PREFIX \"lib\")\n  set_target_properties(pass_lib PROPERTIES PREFIX \"lib\")\n  if(USE_CUDA)\n    if(MXNET_FORCE_SHARED_CRT)\n      target_compile_options(customop_gpu_lib PRIVATE \"$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=-LD -MD>\")\n      target_compile_options(customop_gpu_lib PRIVATE \"$<$<COMPILE_LANGUAGE:CXX>:/LD>\")\n      target_compile_options(customop_gpu_lib PRIVATE \"$<$<COMPILE_LANGUAGE:CXX>:/MD>\")\n    else()\n      target_compile_options(customop_gpu_lib PRIVATE \"$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=-LD -MT>\")\n      target_compile_options(customop_gpu_lib PRIVATE \"$<$<COMPILE_LANGUAGE:CXX>:/LD>\")\n      target_compile_options(customop_gpu_lib PRIVATE \"$<$<COMPILE_LANGUAGE:CXX>:/MT>\")\n    endif()\n    set_target_properties(customop_gpu_lib PROPERTIES PREFIX \"lib\")\n  endif()\nendif()\n\nif(USE_DIST_KVSTORE)\n  add_subdirectory(\"3rdparty/ps-lite\")\n  add_definitions(-DMXNET_USE_DIST_KVSTORE)\n  list(APPEND mxnet_LINKER_LIBS pslite)\n  set_target_properties(pslite PROPERTIES CXX_CLANG_TIDY \"\")  # don't lint 3rdparty dependency\nendif()\n\nif(USE_ONEDNN)\n    add_custom_command(TARGET mxnet POST_BUILD\n      COMMAND ${CMAKE_COMMAND} -E copy\n      ${CMAKE_BINARY_DIR}/3rdparty/onednn/include/oneapi/dnnl/dnnl_config.h  ${CMAKE_SOURCE_DIR}/include/onednn/oneapi/dnnl/\n      COMMAND ${CMAKE_COMMAND} -E copy\n      ${CMAKE_BINARY_DIR}/3rdparty/onednn/include/oneapi/dnnl/dnnl_version.h  ${CMAKE_SOURCE_DIR}/include/onednn/oneapi/dnnl/)\nendif()\n\nif(USE_INTGEMM)\n  target_link_libraries(mxnet PRIVATE intgemm)\nendif()\n\nfunction(BuildTVMOP)\n  # scope the variables in BuildTVM.cmake to avoid conflict\n  include(cmake/BuildTVM.cmake)\n  add_subdirectory(\"3rdparty/tvm\")\n  set_target_properties(tvm PROPERTIES CXX_CLANG_TIDY \"\")  # don't lint 3rdparty dependency\n  set_target_properties(tvm_runtime PROPERTIES CXX_CLANG_TIDY \"\")  # don't lint 3rdparty dependency\nendfunction()\n\nif(USE_TVM_OP)\n  list(APPEND mxnet_LINKER_LIBS tvm_runtime)\n  BuildTVMOP()\n  find_package(Python3 REQUIRED)\n  set(TVM_OP_COMPILE_OPTIONS \"-o${CMAKE_CURRENT_BINARY_DIR}\" \"--config\" \"${CMAKE_CURRENT_BINARY_DIR}/tvmop.conf\" \"-L\" \"${CMAKE_CURRENT_BINARY_DIR}/3rdparty/tvm\")\n  if(UNIX AND NOT APPLE)\n    set(LD_LIBRARY_PATH \"LD_LIBRARY_PATH\")\n  elseif(APPLE)\n    set(LD_LIBRARY_PATH \"DYLD_LIBRARY_PATH\")\n  endif()\n  if(USE_CUDA)\n    set(TVM_OP_COMPILE_OPTIONS \"${TVM_OP_COMPILE_OPTIONS}\" \"--cuda-arch\" \"\\\"${CUDA_ARCH_FLAGS}\\\"\")\n  endif()\n\n  add_custom_command(TARGET mxnet POST_BUILD\n    COMMAND ${CMAKE_COMMAND} -E env\n    PYTHONPATH=\"${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/tvm/python:${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/tvm/topi/python:${CMAKE_CURRENT_SOURCE_DIR}/contrib\"\n    ${LD_LIBRARY_PATH}=${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_CURRENT_BINARY_DIR}/3rdparty/tvm:$ENV{${LD_LIBRARY_PATH}}\n    ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/contrib/tvmop/compile.py ${TVM_OP_COMPILE_OPTIONS}\n  )\nendif()\n\nif(USE_PLUGINS_WARPCTC)\n  list(APPEND mxnet_LINKER_LIBS ${WARPCTC_LIB})\nendif()\n\nif(MSVC)\n  if(USE_SPLIT_ARCH_DLL AND USE_CUDA)\n    foreach(arch ${arch_code_list})\n      target_link_libraries(mxnet_${arch} PUBLIC ${mxnet_LINKER_LIBS})\n      target_link_libraries(mxnet_${arch} PUBLIC dmlc)\n    endforeach()\n  endif()\nendif()\n\ntarget_link_libraries(mxnet PUBLIC ${mxnet_LINKER_LIBS})\ntarget_link_libraries(mxnet PUBLIC dmlc)\n\nif(USE_OPENCV AND OpenCV_VERSION_MAJOR GREATER 2)\n  add_executable(im2rec \"tools/im2rec.cc\")\n  target_link_libraries(im2rec\n    ${mxnet_LINKER_LIBS}\n    ${OpenCV_LIBS}\n    mxnet\n    dmlc\n    )\nelse()\n    message(WARNING \"OpenCV_VERSION_MAJOR: ${OpenCV_VERSION_MAJOR}, version 3 with imgcodecs \\\n    is required for im2rec, im2rec will not be available\")\nendif()\n\n\nif(MSVC AND USE_MXNET_LIB_NAMING)\n  set_target_properties(mxnet PROPERTIES OUTPUT_NAME \"libmxnet\")\nendif()\n\n\n# NOTE: Public headers will be installed into ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}, see\n#       https://cmake.org/cmake/help/v3.13/variable/CMAKE_INSTALL_PREFIX.html\n#       https://cmake.org/cmake/help/v3.13/module/GNUInstallDirs.html\ninstall(TARGETS mxnet\n  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}\n          COMPONENT   MXNET_Runtime\n  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}\n          COMPONENT            MXNET_Runtime\n          NAMELINK_COMPONENT   MXNET_Development\n  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}\n          COMPONENT   MXNET_Development\n)\ninstall(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/dlpack/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})\ninstall(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/dmlc-core/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})\ninstall(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/mshadow/mshadow/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/mshadow)\ninstall(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/mxnet/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/mxnet)\ninstall(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/tvm/nnvm/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})\nif(INSTALL_EXAMPLES)\n  install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/example  DESTINATION ${CMAKE_INSTALL_DATADIR}/${PROJECT_NAME})\nendif()\n\nif(USE_SIGNAL_HANDLER)\n    add_definitions(-DMXNET_USE_SIGNAL_HANDLER=1)\nendif()\n\n# AUTO_INSTALL_DIR -> Optional: specify post-build install direcory\nif(AUTO_INSTALL_DIR)\n  # ---[ Install Includes\n  add_custom_command(TARGET mxnet POST_BUILD\n    COMMAND ${CMAKE_COMMAND} -E copy_directory\n    ${CMAKE_CURRENT_SOURCE_DIR}/include ${AUTO_INSTALL_DIR}/include\n    )\n\n  # ---[ Install Examples\n  add_custom_command(TARGET mxnet POST_BUILD\n    COMMAND ${CMAKE_COMMAND} -E copy_directory\n    ${CMAKE_CURRENT_SOURCE_DIR}/example ${AUTO_INSTALL_DIR}/example\n    )\nendif()\n\nif(INSTALL_PYTHON_VERSIONS)\n  message(STATUS \"Installing for python versions: ${INSTALL_PYTHON_VERSIONS}\")\n  foreach(version ${INSTALL_PYTHON_VERSIONS})\n    set(outdir ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/python${version}/site-packages/mxnet)\n    add_custom_command(TARGET mxnet POST_BUILD\n      COMMAND mkdir -p ${outdir}\n      COMMAND cp -ru ${CMAKE_CURRENT_SOURCE_DIR}/python/mxnet/* ${outdir}\n      )\n  endforeach()\nendif()\n\nif(USE_CPP_PACKAGE)\n  add_subdirectory(cpp-package)\n  target_compile_definitions(mxnet PUBLIC MXNET_USE_CPP_PACKAGE=1)\nendif()\n\nif(NOT CMAKE_BUILD_TYPE STREQUAL \"Distribution\")\n  # Staticbuild applies linker version script to hide private symbols, breaking unit tests\n  add_subdirectory(tests)\nendif()\n\n# ---[ Linter target\nfind_package(Python3)\nset(LINT_DIRS \"include src plugin tests\")\nset(EXCLUDE_PATH \"src/operator/contrib/ctc_include\")\nadd_custom_target(mxnet_lint COMMAND ${CMAKE_COMMAND} -DMSVC=${MSVC} -DPYTHON_EXECUTABLE=${Python3_EXECUTABLE} -DLINT_DIRS=${LINT_DIRS} -DPROJECT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} -DPROJECT_NAME=mxnet -DEXCLUDE_PATH=${EXCLUDE_PATH} -P ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/dmlc-core/cmake/lint.cmake)\n\nif(BUILD_CYTHON_MODULES)\n  include(cmake/BuildCythonModules.cmake)\n  add_cython_modules(3) # Build cython module for python3 if python3 is found\n  if(NOT ${PYTHON3_FOUND})\n    message(FATAL_ERROR \"No python interpreter found to build cython modules\")\n  endif()\nendif()\n\n# https://github.com/apache/incubator-mxnet/issues/20145\nif(UNIX)\n    if(USE_NVML)\n      find_package(NVML)\n      if(NVML_FOUND)\n         target_compile_definitions(mxnet PRIVATE NVML_NO_UNVERSIONED_FUNC_DEFS)\n    endif()\n  endif()\nendif()\n"
  },
  {
    "path": "CODEOWNERS",
    "content": "# Watchers and contributors to Apache MXNet repo directories/packages/files\n# Please see documentation of use of CODEOWNERS file at\n# https://help.github.com/articles/about-codeowners/ and\n# https://github.com/blog/2392-introducing-code-owners\n#\n# Anybody can add themselves or a team as additional watcher or contributor\n# to get notified about changes in a specific package.\n# See https://help.github.com/articles/about-teams how to setup teams.\n\n\n# Global owners\n*\t\t\t@apache/mxnet-committers\n\n# Language bindings\n/R-package/                       @thirdwing\n/scala-package/                   @yzhliu @nswamy @pllarroy\n/perl-package/                    @sergeykolychev\n/python/                          @szha @pllarroy\n/python/mxnet/kvstore.py          @eric-haibin-lin\n/python/mxnet/optimizer/          @eric-haibin-lin\n/python/mxnet/gluon/trainer.py    @eric-haibin-lin\n/contrib/clojure-package/         @gigasquid\n/julia/                           @iblis17\n\n# C++ base\n/src/kvstore/     @rahul003 @eric-haibin-lin @apeforest\n/include/         @pllarroy @eric-haibin-lin\n/src/c_api/       @eric-haibin-lin @apeforest\n/src/engine/      @eric-haibin-lin @apeforest\n/src/executor/    @eric-haibin-lin @apeforest\n/src/imperative/  @eric-haibin-lin @apeforest\n/src/io/          @eric-haibin-lin @apeforest\n/src/ndarray/     @eric-haibin-lin @apeforest\n/src/nnvm/        @eric-haibin-lin @apeforest\n/src/operator/    @eric-haibin-lin @apeforest\n/src/profiler/    @eric-haibin-lin\n/src/storage/     @eric-haibin-lin\n/cpp-package/     @nswamy @pllarroy\n/src/             @pllarroy\n/plugin/          @pllarroy\n\n# Build system\nCMakeLists.txt    @szha @pllarroy @leezu\n/cmake/           @szha @pllarroy @leezu\n/make/            @szha\n\n# MXNet CI\ndev_menu.py         @pllarroy\n/ci/                @pllarroy @marcoabreu @aaronmarkham\n/ci/publish/        @szha\n/docker/            @marcoabreu\n/tests/ci_build/    @marcoabreu\nJenkinsfile         @marcoabreu\n\n# MXNet CD\n/cd/                @szha\n\n# Build logic\nMakefile          @szha\nprepare_mkl.sh    @szha\n\n# Docs\n/docs/            @szha @pllarroy @aaronmarkham\n\n# Submodules\n.gitmodules       @szha\n\n# Examples\n/example/         @szha @pllarroy @aaronmarkham\n\n# Tools\n/tools/           @szha @pllarroy\n\n# Github configuration\n/.github/         @szha @leezu\n"
  },
  {
    "path": "CODE_OF_CONDUCT.md",
    "content": "<!---\n  Licensed to the Apache Software Foundation (ASF) under one\n  or more contributor license agreements.  See the NOTICE file\n  distributed with this work for additional information\n  regarding copyright ownership.  The ASF licenses this file\n  to you under the Apache License, Version 2.0 (the\n  \"License\"); you may not use this file except in compliance\n  with the License.  You may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing,\n  software distributed under the License is distributed on an\n  \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  KIND, either express or implied.  See the License for the\n  specific language governing permissions and limitations\n  under the License.\n-->\n\n# Code of Conduct\n\n* [Code of Conduct for The Apache Software Foundation][1]\n\n## Conflict Resolution\n\nConflicts in an open source project can take many forms, from someone having a bad day and using harsh and hurtful language in the issue queue, to more serious instances such as sexist/racist statements or threats of violence, and everything in between.\n\nIf the behavior is threatening or harassing, or for other reasons requires immediate escalation, please see below.\n\nHowever, for the vast majority of issues, we aim to empower individuals to first resolve conflicts themselves, asking for help when needed, and only after that fails to escalate further. This approach gives people more control over the outcome of their dispute. \n\nIf you are experiencing or witnessing conflict, we ask you to use the following escalation strategy to address the conflict:\n\n1.  Address the perceived conflict directly with those involved, preferably in a real-time medium.\n2.  If this fails, get a third party (e.g. a mutual friend, and/or someone with background on the issue, but not involved in the conflict) to intercede.\n3.  If you are still unable to resolve the conflict, and you believe it rises to harassment or another code of conduct violation, report it.\n\n## Reporting Violations\n\nViolations of the Code of Conduct can be reported to [Apache MXNet Project PMC (PPMC)](mailto:private@mxnet.apache.org). The PPMC will determine whether the Code of Conduct was violated, and will issue an appropriate sanction, possibly including a written warning or expulsion from the project, project sponsored spaces, or project forums. We ask that you make a good-faith effort to resolve your conflict via the conflict resolution policy before submitting a report.\n\n\n[1]: https://www.apache.org/foundation/policies/conduct.html\n"
  },
  {
    "path": "CONTRIBUTORS.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\nContributors of Apache MXNet\n=========================================\nApache MXNet adopts the Apache way and governs by merit. We believe that it is important to create\nan inclusive community where everyone can use, contribute to, and influence the direction of\nthe project. We actively invite contributors who have earned the merit to be part of the\ndevelopment community. See [MXNet Community Guide](https://mxnet.apache.org/community/community).\n\n\nPPMC\n---\nThe Podling Project Management Committee (PPMC) is a group of active committers that moderate the\ndiscussion, manage the project release, and appoint new committers and PPMC members. Here's the list of\nPPMC members in alphabetical order by first name:\n\n* [Anirudh Subramanian](https://github.com/anirudh2290)\n* [Bing Xu](https://github.com/antinucleon)\n  - Bing is the initiator and major contributor of operators and ndarray modules of MXNet.\n* [Carin Meier](https://github.com/gigasquid)\n  - Carin created and is the current maintainer for the Clojure interface.\n* [Chiyuan Zhang](https://github.com/pluskid)\n  - Chiyuan is the creator of MXNet Julia package.\n* [Chris Olivier](https://github.com/cjolivier01)\n* [Dick Carter](https://github.com/DickJC123)\n* [Junyuan Xie](https://github.com/piiswrong)\n* [Haibin Lin](https://github.com/eric-haibin-lin)\n* [Henri Yandell](https://github.com/hen)\n* [Hongliang Liu](https://github.com/phunterlau)\n* [Indhu Bharathi](https://github.com/indhub)\n* [Jackie Wu](https://github.com/wkcn)\n* [Jian Zhang](https://github.com/jzhang-zju)\n* [Joe Spisak](https://github.com/jspisak)\n* [Jun Wu](https://github.com/reminisce)\n* [Leonard Lausen](https://github.com/leezu)\n* [Liang Depeng](https://github.com/Ldpe2G)\n* [Ly Nguyen](https://github.com/lxn2)\n* [Madan Jampani](https://github.com/madjam)\n* [Marco de Abreu](https://github.com/marcoabreu)\n  - Marco is the creator of the current MXNet CI.\n* [Mu Li](https://github.com/mli)\n  - Mu is the contributor of the distributed key-value store in MXNet.\n* [Nan Zhu](https://github.com/CodingCat)\n* [Naveen Swamy](https://github.com/nswamy)\n* [Przemek Tredak](https://github.com/ptrendx)\n* [Qiang Kou](https://github.com/thirdwing)\n  - KK is a R ninja, he makes mxnet available for R users.\n* [Qing Lan](https://github.com/lanking520)\n* [Sandeep Krishnamurthy](https://github.com/sandeep-krishnamurthy)\n* [Sergey Kolychev](https://github.com/sergeykolychev)\n  - Sergey is original author and current maintainer of Perl5 interface.\n* [Sheng Zha](https://github.com/szha)\n* [Shiwen Hu](https://github.com/yajiedesign)\n* [Tao Lv](https://github.com/TaoLv)\n  - Tao is a major contributor to the MXNet MKL-DNN backend and performance on CPU.\n* [Terry Chen](https://github.com/terrychenism)\n* [Thomas Delteil](https://github.com/ThomasDelteil)\n* [Tianqi Chen](https://github.com/tqchen)\n  - Tianqi is one of the initiator of the MXNet project.\n* [Tong He](https://github.com/hetong007)\n  - Tong is the major maintainer of MXNet R package, he designs the MXNet interface and wrote many of the tutorials on R.\n* [Tsuyoshi Ozawa](https://github.com/oza)\n* [Xingjian Shi](https://github.com/sxjscience)\n* [Yifeng Geng](https://github.com/gengyifeng)\n* [Yizhi Liu](https://github.com/yzhliu)\n  - Yizhi is the main creator on mxnet scala project to make deep learning available for JVM stacks.\n* [Yu Zhang](https://github.com/yzhang87)\n* [Yuan Tang](https://github.com/terrytangyuan)\n  - Yuan is one of major maintainers of MXNet Scala package.\n* [Yutian Li](https://github.com/hotpxl)\n  - Yutian is the ninja behind the dependency and storage engine of MXNet.\n* [Zhi Zhang](https://github.com/zhreshold)\n* [Zihao Zheng](https://github.com/zihaolucky)\n* [Ziheng Jiang](https://github.com/zihengjiang)\n* [Ziyue Huang](https://github.com/ZiyueHuang)\n\n\nCommitters\n----------\nCommitters are individuals who are granted write access to the project. A committer is usually\nresponsible for a certain area or several areas of the code where they oversee the code review\nprocess. The area of contribution can take all forms, including code contributions and code\nreviews, documents, education, and outreach. Committers are essential for a high quality and\nhealthy project. The PPMC actively seeks to appoint new committers from the list of contributors.\n\n* [Aaron Markham](https://github.com/aaronmarkham)\n* [Alex Zai](https://github.com/azai91)\n* [Anirudh Acharya](https://github.com/anirudhacharya)\n* [Anna Karbownik](https://github.com/akarbown)\n* [Aston Zhang](https://github.com/astonzhang)\n* [Bartłomiej Gawrych](https://github.com/bgawrych)\n* [Chaitanya Bapat](https://github.com/ChaiBapchya)\n* [Ciyong Chen](https://github.com/ciyongch)\n* [Da Zheng](https://github.com/zheng-da)\n* [Ding Kuo](https://github.com/chinakook)\n* [Hao Jin](https://github.com/haojin2)\n* [Haozheng Fan](https://github.com/hzfan)\n* [Iblis Lin](https://github.com/iblis17)\n* [Jeremie Desgagne-Bouchard](https://github.com/jeremiedb)\n* [Jiajun Wang](https://github.com/arcadiaphy)\n* [Junru Shao](https://github.com/junrushao1994)\n* [Kedar Bellare](https://github.com/kedarbellare)\n* [Kellen Sunderland](https://github.com/KellenSunderland)\n* [Kevin Qin](https://github.com/ZhennanQin)\n* [Lai Wei](https://github.com/roywei)\n* [Lin Yuan](https://github.com/apeforest)\n  - Lin supports MXNet distributed training using Horovod and is also a major contributor to higher order gradients.\n* [Manu Seth](https://github.com/mseth10/)\n* [Moises Hernandez](https://github.com/MoisesHer/)\n* [Nicolas Modrzyk](https://github.com/hellonico)\n* [Patric Zhao](https://github.com/pengzhao-intel)\n  - Patric is a parallel computing expert and a major contributor to the MXNet MKL-DNN backend.\n* [Przemysław Trędak](https://github.com/ptrendx)\n* [Rahul Huilgol](https://github.com/rahul003)\n* [Roshani Nagmote](https://github.com/roshrini)\n* [Sam Skalicky](https://github.com/samskalicky)\n* [Steffen Rochel](https://github.com/srochel)\n* [Xi Wang](https://github.com/xidulu)\n* [Yang Shi](https://github.com/ys2843)\n* [Yuxi Hu](https://github.com/yuxihu)\n* [Zach Kimberg](https://github.com/zachgk)\n  - Zach is one of the major maintainers of the MXNet Scala package.\n* [Zhaoqi Zhu](https://github.com/Zha0q1)\n\n\nList of Contributors\n--------------------\n* [Top-100 Contributors](https://github.com/apache/mxnet/graphs/contributors)\n  - To contributors: please add your name to the list when you submit a patch to the project:)\n* [Aditya Trivedi](https://github.com/iadi7ya)\n* [Feng Wang](https://github.com/happynear)\n  - Feng makes MXNet compatible with Windows Visual Studio.\n* [Jack Deng](https://github.com/jdeng)\n  - Jack created the amalgamation script and Go bind for MXNet.\n* [Li Dong](https://github.com/donglixp)\n* [Piji Li](https://github.com/lipiji)\n* [Boyuan Deng](https://github.com/bryandeng)\n* [Junran He](https://github.com/junranhe)\n  - Junran makes device kvstore allocation strategy smarter\n* [Shuzhe Wu](https://github.com/II-Matto)\n* [Xiaodong](https://github.com/XD-DENG)\n* [Nan Xiao](https://github.com/road2stat)\n* [Wei Wu](https://github.com/tornadomeet)\n* [Michaël Benesty](https://github.com/pommedeterresautee)\n  - Michaël contributes the R visualization module of MXNet\n* [Kublai Jing](https://github.com/Kublai-Jing)\n* [chenjx1005](https://github.com/chenjx1005)\n* [ry](https://github.com/ry)\n* [Ming Zhang](https://github.com/starimpact)\n* [sxjscience](https://github.com/sxjscience)\n* [Zheng Xu](https://github.com/XericZephyr)\n* [Valentin Churavy](https://github.com/vchuravy)\n* [Luke Metz](https://github.com/lukemetz)\n* [Guosheng Dong](https://github.com/dongguosheng)\n* [yang1fan2](https://github.com/yang1fan2)\n* [Wang Gu](https://github.com/wangg12)\n* [Zhenchuan Huang](https://github.com/chuan92)\n* [Hang Su](https://github.com/suhangpro)\n* [Rinu Boney](https://github.com/rinuboney)\n* [nowozin](https://github.com/nowozin)\n* [Mathis](https://github.com/sveitser)\n* [sennendoko](https://github.com/sennendoko)\n* [srand99](https://github.com/srand99)\n* [Taiyun](https://github.com/taiyun)\n* [Yanghao Li](https://github.com/lyttonhao)\n* [Ye Zhou](https://github.com/zhouye)\n* [Zhang Chen](https://github.com/zhangchen-qinyinghua)\n* [Xianliang Wang](https://github.com/wangxianliang)\n* [Xiao Liu](https://github.com/skylook)\n* [Lowik CHANUSSOT](https://github.com/Nzeuwik)\n* [Alexander Skidanov](https://github.com/SkidanovAlex)\n* [Ruixiang Zhang](https://github.com/sodabeta7)\n* [Lodewic van Twillert](https://github.com/Lodewic)\n* [Aditya Kumar](https://github.com/hiraditya)\n* [Dan Becker](https://github.com/dansbecker)\n* [Yun Yan](https://github.com/Puriney)\n* [Tao Wei](https://github.com/taoari)\n* [Max Kuhn](https://github.com/topepo)\n* [Yuqi Li](https://github.com/ziyeqinghan)\n* [Kiko Qiu](https://github.com/kikoqiu)\n* [Yang Bo](https://github.com/Atry)\n* [Jonas Amaro](https://github.com/jonasrla)\n* [Yan Li](https://github.com/Godricly)\n* [Yuance Li](https://github.com/liyuance)\n* [Andre Moeller](https://github.com/andremoeller)\n* [Miguel Gonzalez-Fierro](https://github.com/miguelgfierro)\n* [Mingjie Xing](https://github.com/EricFisher)\n* [Sebastian Bodenstein](https://github.com/sbodenstein)\n* [Taliesin Beynon](https://github.com/taliesinb)\n* [Chi Zhang](https://github.com/WellyZhang)\n* [Wei Wu](https://github.com/lazyparser)\n* [Shishi Duan](https://github.com/burness)\n* [Yu Du](https://github.com/Answeror)\n* [Xu Dong](https://github.com/dsqx71)\n* [Chihiro Komaki](https://github.com/ckomaki)\n* [Piyush Singh](https://github.com/Piyush3dB)\n* [Freddy Chua](https://github.com/freddycct)\n* [Jie Zhang](https://github.com/luoyetx)\n* [Robert Stone](https://github.com/tlby)\n* [Pedro Larroy](https://github.com/larroy)\n* [Dom Divakaruni](https://github.com/domdivakaruni)\n* [David Salinas](https://github.com/geoalgo)\n* [Asmus Hetzel](https://github.com/asmushetzel)\n* [Chetan Khatri](https://github.com/chetkhatri/)\n* [James Liu](https://github.com/jamesliu/)\n* [Nir Ben-Zvi](https://github.com/nirbenz/)\n* [Arik Poznanski](https://github.com/arikpoz/)\n* [Yuwen Xiong](https://github.com/Orpine/)\n* [Haozhi Qi](https://github.com/Oh233/)\n* [Yi Li](https://github.com/liyi14/)\n* [Guodong Zhang](https://github.com/gd-zhang/)\n* [Xizhou Zhu](https://github.com/einsiedler0408/)\n* [Jean Kossaifi](https://github.com/JeanKossaifi/)\n* [Kenta Kubo](https://github.com/kkk669/)\n* [Calum Leslie](https://github.com/calumleslie)\n* [Andre Tamm](https://github.com/andretamm)\n* [Julian Salazar](https://github.com/JulianSlzr)\n* [Meghna Baijal](https://github.com/mbaijal)\n* [Tao Hu](https://github.com/dongzhuoyao)\n* [Sorokin Evgeniy](https://github.com/TheTweak)\n* [dwSun](https://github.com/dwSun/)\n* [David Braude](https://github.com/dabraude/)\n* [Nick Robinson](https://github.com/nickrobinson)\n* [Zheqin Wang](https://github.com/rasefon)\n* [Thom Lane](https://github.com/thomelane)\n* [Sina Afrooze](https://github.com/safrooze)\n* [Sergey Sokolov](https://github.com/Ishitori)\n* [Jesse Brizzi](https://github.com/jessebrizzi)\n* [Hang Zhang](http://hangzh.com)\n* [Kou Ding](https://github.com/chinakook)\n* [Istvan Fehervari](https://github.com/ifeherva)\n* [Per Goncalves da Silva](https://github.com/perdasilva)\n* [Zhijingcheng Yu](https://github.com/jasonyu1996)\n* [Cheng-Che Lee](https://github.com/stu1130)\n* [LuckyPigeon](https://github.com/LuckyPigeon)\n* [Anton Chernov](https://github.com/lebeg)\n* [Denisa Roberts](https://github.com/D-Roberts)\n* [Rahul Padmanabhan](https://github.com/rahul3)\n* [Harsh Patel](https://github.com/harshp8l)\n* [Xiao Wang](https://github.com/BeyonderXX)\n* [Piyush Ghai](https://github.com/piyushghai)\n* [Dang Trung Kien](https://github.com/kiendang)\n* [Zach Boldyga](https://github.com/zboldyga)\n* [Gordon Reid](https://github.com/gordon1992)\n* [Mikhail Lobanov](https://github.com/lobanov-m)\n* [Ming Yang](http://ufoym.com)\n* [Satya Krishna Gorti](https://github.com/satyakrishnagorti)\n* [Neo Chien](https://github.com/cchung100m)\n* [Wujie Zhou](https://github.com/eureka7mt)\n* [Hao Li](https://github.com/lihaofd)\n* [Jin Huang](https://github.com/jinhuang415)\n* [Luobao Zou](https://github.com/luobao-intel)\n* [Pengxin Yuan](https://github.com/pengxin99)\n* [Rong Zhang](https://github.com/rongzha1/)\n* [Shu Zhang](https://github.com/Sherry-Zhang)\n* [Shufan Wu](https://github.com/juliusshufan)\n* [Wenting Jiang](https://github.com/wentingj)\n* [Xiaotao Chen](https://github.com/XiaotaoChen)\n* [Xinyu Chen](https://github.com/xinyu-intel)\n* [Zhennan Qin](https://github.com/ZhennanQin)\n* [Zhiyuan Huang](https://github.com/huangzhiyuan)\n* [Zak Jost](https://github.com/zjost)\n* [Nick Guletskii](https://github.com/nickguletskii)\n* [Shoubhik Bhattacharya](https://github.com/shoubhik)\n* [Rohit Srivastava](https://github.com/access2rohit)\n* [Caner Turkmen](https://github.com/canerturkmen)\n* [Disi A](https://github.com/adis300)\n* [Vandana Kannan](https://github.com/vandanavk)\n* [Guanxin Qiao](https://github.com/guanxinq)\n* [dithyrambe](https://github.com/dithyrambe)\n* [Piljae Chae](https://github.com/IHateMint)\n* [Jonathan Tan](https://github.com/jonatan1626)\n* [Oliver Kowalke](https://github.com/olk)\n* [Connor Goggins](https://github.com/connorgoggins)\n* [Wei Chu](https://github.com/waytrue17)\n* [Joe Evans](https://github.com/josephevans)\n* [Nikolay Ulmasov](https://github.com/r3stl355)\n* [Paweł Głomski](https://github.com/PawelGlomski-Intel)\n* [Andrzej Kotlowski](https://github.com/anko-intel)\n* [Yingxiao Du](https://github.com/Duconnor)\n* [Bartosz Kuncer](https://github.com/bartekkuncer)\n* [Maria Boerner](https://github.com/mariaboerner1987)\n* [Zhenghui Jin](https://github.com/barry-jin)\n* [Dominika Jedynak](https://github.com/DominikaJedynak)\n* [Adam Grabowski](https://github.com/agrabows)\n* [Kacper Pietkun](https://github.com/Kacper-Pietkun)\n* [Hanna Jarlaczyńska](https://github.com/hankaj)\n\nLabel Bot\n---------\n* [mxnet-label-bot](https://github.com/mxnet-label-bot)\n  - mxnet-label-bot provides users with the functionality to manage labels for Issues/Pull Requests on the repository\n  - To use me, comment:\n    - @mxnet-label-bot add [specify comma separated labels here]\n    - @mxnet-label-bot remove [specify comma separated labels here]\n    - @mxnet-label-bot update [specify comma separated labels here]\n      (i.e. @mxnet-label-bot update [Bug, Python])\n\n  - Available label names which are supported: [Labels](https://github.com/apache/mxnet/labels)\n  - For further details: [My Wiki Page](https://cwiki.apache.org/confluence/display/MXNET/Machine+Learning+Based+GitHub+Bot)\n"
  },
  {
    "path": "DNNL_README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\nFile is moved to [docs/python_docs/python/tutorials/performance/backend/dnnl/dnnl_readme.md](docs/python_docs/python/tutorials/performance/backend/dnnl/dnnl_readme.md).\n"
  },
  {
    "path": "LICENSE",
    "content": "\n                                 Apache License\n                           Version 2.0, January 2004\n                        http://www.apache.org/licenses/\n\n   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n   1. Definitions.\n\n      \"License\" shall mean the terms and conditions for use, reproduction,\n      and distribution as defined by Sections 1 through 9 of this document.\n\n      \"Licensor\" shall mean the copyright owner or entity authorized by\n      the copyright owner that is granting the License.\n\n      \"Legal Entity\" shall mean the union of the acting entity and all\n      other entities that control, are controlled by, or are under common\n      control with that entity. For the purposes of this definition,\n      \"control\" means (i) the power, direct or indirect, to cause the\n      direction or management of such entity, whether by contract or\n      otherwise, or (ii) ownership of fifty percent (50%) or more of the\n      outstanding shares, or (iii) beneficial ownership of such entity.\n\n      \"You\" (or \"Your\") shall mean an individual or Legal Entity\n      exercising permissions granted by this License.\n\n      \"Source\" form shall mean the preferred form for making modifications,\n      including but not limited to software source code, documentation\n      source, and configuration files.\n\n      \"Object\" form shall mean any form resulting from mechanical\n      transformation or translation of a Source form, including but\n      not limited to compiled object code, generated documentation,\n      and conversions to other media types.\n\n      \"Work\" shall mean the work of authorship, whether in Source or\n      Object form, made available under the License, as indicated by a\n      copyright notice that is included in or attached to the work\n      (an example is provided in the Appendix below).\n\n      \"Derivative Works\" shall mean any work, whether in Source or Object\n      form, that is based on (or derived from) the Work and for which the\n      editorial revisions, annotations, elaborations, or other modifications\n      represent, as a whole, an original work of authorship. For the purposes\n      of this License, Derivative Works shall not include works that remain\n      separable from, or merely link (or bind by name) to the interfaces of,\n      the Work and Derivative Works thereof.\n\n      \"Contribution\" shall mean any work of authorship, including\n      the original version of the Work and any modifications or additions\n      to that Work or Derivative Works thereof, that is intentionally\n      submitted to Licensor for inclusion in the Work by the copyright owner\n      or by an individual or Legal Entity authorized to submit on behalf of\n      the copyright owner. For the purposes of this definition, \"submitted\"\n      means any form of electronic, verbal, or written communication sent\n      to the Licensor or its representatives, including but not limited to\n      communication on electronic mailing lists, source code control systems,\n      and issue tracking systems that are managed by, or on behalf of, the\n      Licensor for the purpose of discussing and improving the Work, but\n      excluding communication that is conspicuously marked or otherwise\n      designated in writing by the copyright owner as \"Not a Contribution.\"\n\n      \"Contributor\" shall mean Licensor and any individual or Legal Entity\n      on behalf of whom a Contribution has been received by Licensor and\n      subsequently incorporated within the Work.\n\n   2. Grant of Copyright License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      copyright license to reproduce, prepare Derivative Works of,\n      publicly display, publicly perform, sublicense, and distribute the\n      Work and such Derivative Works in Source or Object form.\n\n   3. Grant of Patent License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      (except as stated in this section) patent license to make, have made,\n      use, offer to sell, sell, import, and otherwise transfer the Work,\n      where such license applies only to those patent claims licensable\n      by such Contributor that are necessarily infringed by their\n      Contribution(s) alone or by combination of their Contribution(s)\n      with the Work to which such Contribution(s) was submitted. If You\n      institute patent litigation against any entity (including a\n      cross-claim or counterclaim in a lawsuit) alleging that the Work\n      or a Contribution incorporated within the Work constitutes direct\n      or contributory patent infringement, then any patent licenses\n      granted to You under this License for that Work shall terminate\n      as of the date such litigation is filed.\n\n   4. Redistribution. You may reproduce and distribute copies of the\n      Work or Derivative Works thereof in any medium, with or without\n      modifications, and in Source or Object form, provided that You\n      meet the following conditions:\n\n      (a) You must give any other recipients of the Work or\n          Derivative Works a copy of this License; and\n\n      (b) You must cause any modified files to carry prominent notices\n          stating that You changed the files; and\n\n      (c) You must retain, in the Source form of any Derivative Works\n          that You distribute, all copyright, patent, trademark, and\n          attribution notices from the Source form of the Work,\n          excluding those notices that do not pertain to any part of\n          the Derivative Works; and\n\n      (d) If the Work includes a \"NOTICE\" text file as part of its\n          distribution, then any Derivative Works that You distribute must\n          include a readable copy of the attribution notices contained\n          within such NOTICE file, excluding those notices that do not\n          pertain to any part of the Derivative Works, in at least one\n          of the following places: within a NOTICE text file distributed\n          as part of the Derivative Works; within the Source form or\n          documentation, if provided along with the Derivative Works; or,\n          within a display generated by the Derivative Works, if and\n          wherever such third-party notices normally appear. The contents\n          of the NOTICE file are for informational purposes only and\n          do not modify the License. You may add Your own attribution\n          notices within Derivative Works that You distribute, alongside\n          or as an addendum to the NOTICE text from the Work, provided\n          that such additional attribution notices cannot be construed\n          as modifying the License.\n\n      You may add Your own copyright statement to Your modifications and\n      may provide additional or different license terms and conditions\n      for use, reproduction, or distribution of Your modifications, or\n      for any such Derivative Works as a whole, provided Your use,\n      reproduction, and distribution of the Work otherwise complies with\n      the conditions stated in this License.\n\n   5. Submission of Contributions. Unless You explicitly state otherwise,\n      any Contribution intentionally submitted for inclusion in the Work\n      by You to the Licensor shall be under the terms and conditions of\n      this License, without any additional terms or conditions.\n      Notwithstanding the above, nothing herein shall supersede or modify\n      the terms of any separate license agreement you may have executed\n      with Licensor regarding such Contributions.\n\n   6. Trademarks. This License does not grant permission to use the trade\n      names, trademarks, service marks, or product names of the Licensor,\n      except as required for reasonable and customary use in describing the\n      origin of the Work and reproducing the content of the NOTICE file.\n\n   7. Disclaimer of Warranty. Unless required by applicable law or\n      agreed to in writing, Licensor provides the Work (and each\n      Contributor provides its Contributions) on an \"AS IS\" BASIS,\n      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n      implied, including, without limitation, any warranties or conditions\n      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n      PARTICULAR PURPOSE. You are solely responsible for determining the\n      appropriateness of using or redistributing the Work and assume any\n      risks associated with Your exercise of permissions under this License.\n\n   8. Limitation of Liability. In no event and under no legal theory,\n      whether in tort (including negligence), contract, or otherwise,\n      unless required by applicable law (such as deliberate and grossly\n      negligent acts) or agreed to in writing, shall any Contributor be\n      liable to You for damages, including any direct, indirect, special,\n      incidental, or consequential damages of any character arising as a\n      result of this License or out of the use or inability to use the\n      Work (including but not limited to damages for loss of goodwill,\n      work stoppage, computer failure or malfunction, or any and all\n      other commercial damages or losses), even if such Contributor\n      has been advised of the possibility of such damages.\n\n   9. Accepting Warranty or Additional Liability. While redistributing\n      the Work or Derivative Works thereof, You may choose to offer,\n      and charge a fee for, acceptance of support, warranty, indemnity,\n      or other liability obligations and/or rights consistent with this\n      License. However, in accepting such obligations, You may act only\n      on Your own behalf and on Your sole responsibility, not on behalf\n      of any other Contributor, and only if You agree to indemnify,\n      defend, and hold each Contributor harmless for any liability\n      incurred by, or claims asserted against, such Contributor by reason\n      of your accepting any such warranty or additional liability.\n\n   END OF TERMS AND CONDITIONS\n\n   APPENDIX: How to apply the Apache License to your work.\n\n      To apply the Apache License to your work, attach the following\n      boilerplate notice, with the fields enclosed by brackets \"[]\"\n      replaced with your own identifying information. (Don't include\n      the brackets!)  The text should be enclosed in the appropriate\n      comment syntax for the file format. We also recommend that a\n      file or class name and description of purpose be included on the\n      same \"printed page\" as the copyright notice for easier\n      identification within third-party archives.\n\n   Copyright [yyyy] [name of copyright owner]\n\n   Licensed under the Apache License, Version 2.0 (the \"License\");\n   you may not use this file except in compliance with the License.\n   You may obtain a copy of the License at\n\n       http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing, software\n   distributed under the License is distributed on an \"AS IS\" BASIS,\n   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n   See the License for the specific language governing permissions and\n   limitations under the License.\n\n    ======================================================================================\n    Apache MXNET Subcomponents:\n\n    The Apache MXNET project contains subcomponents with separate\n    copyright notices and license terms. Your use of the source code for the\n    these subcomponents is subject to the terms and conditions of the following\n    licenses. See licenses/ for text of these licenses.\n\n    If a folder hierarchy is listed as subcomponent, separate listings of\n    further subcomponents (files or folder hierarchies) part of the hierarchy\n    take precedence.\n\n    =======================================================================================\n    Apache-2.0 license\n    =======================================================================================\n\n    3rdparty/ctc_include\n    3rdparty/dlpack\n    3rdparty/dmlc-core\n    3rdparty/googletest/googlemock/scripts/generator\n    3rdparty/mshadow\n    3rdparty/onednn\n    3rdparty/onednn/doc/assets/mathjax\n    3rdparty/onednn/tests/benchdnn\n    3rdparty/onnx-tensorrt/third_party/onnx/third_party/benchmark\n    3rdparty/onnx-tensorrt/third_party/onnx/tools/protoc-gen-mypy.py\n    3rdparty/ps-lite\n    3rdparty/tvm\n    3rdparty/tvm/3rdparty/dlpack\n    3rdparty/tvm/3rdparty/dmlc-core\n    3rdparty/tvm/3rdparty/bfloat16/bfloat16.cc\n    include/dlpack (header symlinks to 3rdparty/dlpack/include/dlpack)\n    include/dmlc (header symlinks to 3rdparty/dmlc-core/include/dmlc)\n    include/onednn (header symlinks to 3rdparty/onednn)\n    include/mshadow (header symlinks to 3rdparty/mshadow/mshadow)\n    include/nnvm (header symlinks to 3rdparty/tvm/nnvm/include/nnvm)\n    src/operator/deformable_convolution-inl.h\n    src/operator/deformable_convolution.cc\n    src/operator/deformable_convolution.cu\n    src/operator/contrib/deformable_psroi_pooling-inl.h\n    src/operator/contrib/deformable_psroi_pooling.cc\n    src/operator/contrib/deformable_psroi_pooling.cu\n    src/operator/contrib/multi_proposal-inl.h\n    src/operator/contrib/multi_proposal.cc\n    src/operator/contrib/multi_proposal.cu\n    src/operator/contrib/psroi_pooling.cc\n    src/operator/contrib/psroi_pooling.cu\n    src/operator/nn/dnnl/dnnl_base-inl.h\n    src/operator/special_functions-inl.h\n    docs/python_docs/themes/mx-theme/mxtheme/static/material-design-icons-3.0.1 (Copy of the License available at top of current file)\n    docs/python_docs/themes/mx-theme/mxtheme/static/font/Roboto (Copy of the License available at top of current file)\n\n    =======================================================================================\n    MIT license\n    =======================================================================================\n\n    3rdparty/intgemm\n    3rdparty/miniz/miniz.c\n    3rdparty/miniz/miniz.h\n    3rdparty/onnx-tensorrt\n    3rdparty/onnx-tensorrt/third_party/onnx\n    3rdparty/tvm/3rdparty/cma\n    3rdparty/tvm/3rdparty/compiler-rt/builtin_fp16.h\n    docs/static_site/src/assets/js/clipboard.js\n    docs/python_docs/themes/mx-theme\n    src/operator/contrib/modulated_deformable_convolution-inl.h\n    src/operator/contrib/modulated_deformable_convolution.cc\n    src/operator/contrib/modulated_deformable_convolution.cu\n    src/operator/contrib/nn/modulated_deformable_im2col.cuh\n    src/operator/contrib/nn/modulated_deformable_im2col.h\n    src/operator/nn/layer_norm_cpu.h\n\n    =======================================================================================\n    3-clause BSD license\n    =======================================================================================\n\n    3rdparty/onednn/cmake/FindACL.cmake (see licenses/LICENSE.onednn.txt)\n    3rdparty/onednn/cmake/FindBLAS.cmake (see licenses/LICENSE.onednn.txt)\n    3rdparty/onednn/cmake/FindOpenCL.cmake (see licenses/LICENSE.onednn.txt)\n    3rdparty/onednn/src/common/ittnotify (see licenses/LICENSE.onednn.txt)\n    3rdparty/onednn/src/cpu/x64/xbyak (see licenses/LICENSE.onednn.txt)\n    3rdparty/onednn/tests/gtests/gtest (see licenses/LICENSE.onednn.txt)\n    3rdparty/onnx-tensorrt/third_party/onnx/third_party/pybind11/tools/FindPythonLibsNew.cmake\n    3rdparty/ctc_include/contrib/moderngpu\n    3rdparty/nvidia_cub\n    3rdparty/nvidia_cub/test/mersenne.h\n    3rdparty/googletest/googlemock\n    3rdparty/googletest/googletest\n    cmake/upstream/FindCUDAToolkit.cmake\n    cmake/upstream/FindBLAS.cmake\n    cmake/upstream/select_compute_arch.cmake\n    python/mxnet/onnx/mx2onnx/_export_onnx.py\n    python/mxnet/onnx/mx2onnx/_op_translations/_op_translations_opset12.py\n    python/mxnet/onnx/mx2onnx/_op_translations/_op_translations_opset13.py\n    src/operator/contrib/erfinv-inl.h\n    src/operator/numpy/np_einsum_op-inl.h\n    src/operator/numpy/np_einsum_path_op-inl.h\n    src/operator/numpy/np_einsum_op.cc\n\n    =======================================================================================\n    2-clause BSD license\n    =======================================================================================\n\n    3rdparty/dmlc-core/include/dmlc/concurrentqueue.h\n    3rdparty/onnx-tensorrt/third_party/onnx/third_party/pybind11/tools/FindEigen3.cmake\n    3rdparty/tvm/3rdparty/picojson/picojson.h\n    include/dmlc/concurrentqueue.h (symlink to 3rdparty/dmlc-core/include/dmlc/concurrentqueue.h)\n\n    =======================================================================================\n    Apache-2.0 license + LLVM Exceptions\n    =======================================================================================\n\n    3rdparty/openmp\n\n    =======================================================================================\n    2-clause BSD license + Caffe Copyright Notice and Disclaimer\n    =======================================================================================\n\n    src/operator/nn/pool.h\n    src/operator/nn/pool.cuh\n    src/operator/nn/im2col.h\n    src/operator/nn/im2col.cuh\n    src/operator/contrib/nn/deformable_im2col.h\n    src/operator/contrib/nn/deformable_im2col.cuh\n    src/operator/contrib/nn/modulated_deformable_im2col.h\n    src/operator/contrib/nn/modulated_deformable_im2col.cuh\n\n    =======================================================================================\n    2-clause BSD license + zlib license\n    =======================================================================================\n\n    3rdparty/dmlc-core/include/dmlc/blockingconcurrentqueue.h\n    include/dmlc/blockingconcurrentqueue.h (symlink to 3rdparty/dmlc-core/include/dmlc/blockingconcurrentqueue.h)\n\n    =======================================================================================\n    Apache-2.0 license + MIT License\n    =======================================================================================\n\n    src/serialization/cnpy.h (Copy of the AL2 License available at the top of this file, MIT License available at licenses/MIT)\n    src/serialization/cnpy.cc (Copy of the AL2 License available at the top of this file, MIT License available at licenses/MIT)\n\n    =======================================================================================\n    Boost Software License, Version 1.0\n    =======================================================================================\n\n    3rdparty/intgemm/test/3rd_party/catch.hpp\n    cmake/Modules/FindJeMalloc.cmake\n\n    =======================================================================================\n    LLVM Release License\n    =======================================================================================\n\n    3rdparty/onnx-tensorrt/third_party/onnx/third_party/pybind11/tools/clang\n\n    =======================================================================================\n    Unlicense\n    =======================================================================================\n\n    3rdparty/tvm/3rdparty/rang\n\n    =======================================================================================\n    SIL Open Font License (OFL)\n    =======================================================================================\n\n    docs/python_docs/themes/mx-theme/mxtheme/static/webfonts/ (Copy of the License available at licenses/OFL1_1)\n    \n    =======================================================================================\n    Apache-2.0 WITH LLVM-exception\n    =======================================================================================\n\n    tools/lint/git-clang-format-13\n    \n"
  },
  {
    "path": "NEWS.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\nMXNet Change Log\n================\n- [MXNet Change Log](#mxnet-change-log)\n  - [1.8.0](#180)\n    - [Features](#features)\n      * [CUDA Graphs](#cuda-graphs)\n      * [CUDA 11 Support](#cuda-11-support)\n      * [TensorRT](#tensorrt)\n      * [OneDNN](#onednn)\n      * [IntGemm](#intgemm)\n      * [Subgraph API](#subgraph-api)\n      * [Extensions](#extensions)\n      * [ONNX](#onnx)\n      * [Large Tensor](#large-tensor)\n      * [Website Improvements](#website-improvements)\n      * [Documentation](#documentation)\n      * [License](#license)\n      * [CI Improvements](#ci-improvements)\n      * [Bug Fixes](#bug-fixes)\n  - [1.7.0](#170)\n    - [New features](#new-features)\n      - [MXNet Extensions: custom operators, partitioning, and graph passes](#mxnet-extensions-custom-operators-partitioning-and-graph-passes)\n      - [OpPerf utility enabled in the binary distribution](#opperf-utility-enabled-in-the-binary-distribution)\n      - [MKL-DNN](#mkl-dnn)\n        - [MKL-DNN as the default CPU backend in binary distribution](#mkl-dnn-as-the-default-cpu-backend-in-binary-distribution)\n        - [Branding change to DNNL](#branding-change-to-dnnl)\n        - [Support bfloat16 datatype](#support-bfloat16-datatype)\n      - [New operators](#new-operators)\n    - [Feature improvements](#feature-improvements)\n      - [Numpy compatible interface(experimental)](#numpy-compatible-interfaceexperimental)\n      - [Large tensor support](#large-tensor-support)\n      - [MKL-DNN enhancement](#mkl-dnn-enhancement)\n      - [TensorRT integration](#tensorrt-integration)\n      - [Quantization](#quantization)\n      - [Profiler](#profiler)\n      - [ONNX](#onnx)\n      - [New models](#new-models)\n      - [Operator improvements](#operator-improvements)\n      - [Bug fixes](#bug-fixes)\n    - [Front end API](#front-end-api)\n      - [Gluon](#gluon)\n      - [Symbol](#symbol)\n    - [Language Bindings](#language-bindings)\n      - [Python](#python)\n      - [C/C++](#cc)\n      - [R](#r)\n      - [Clojure](#clojure)\n      - [Julia](#julia)\n      - [Perl](#perl)\n      - [Scala](#scala)\n    - [Performance improvements](#performance-improvements)\n    - [Example and tutorials](#example-and-tutorials)\n    - [Website and documentation](#website-and-documentation)\n    - [CI/CD](#cicd)\n    - [License](#license)\n    - [Miscellaneous changes](#miscellaneous-changes)\n  - [1.6.0](#160)\n    - [Deprecation of Python 2](#deprecation-of-python-2)\n    - [New features](#new-features-1)\n      - [NumPy compatible interface and using TVM to generate operators](#numpy-compatible-interface-and-using-tvm-to-generate-operators)\n      - [Graph optimizations](#graph-optimizations)\n        - [Pointwise fusion for GPU](#pointwise-fusion-for-gpu)\n        - [Eliminate common subexpressions](#eliminate-common-subexpressions)\n        - [Default MKLDNN Subgraph fusion](#default-mkldnn-subgraph-fusion)\n      - [New operators](#new-operators-1)\n    - [Feature improvements](#feature-improvements-1)\n      - [Automatic Mixed Precision](#automatic-mixed-precision)\n      - [Gluon Fit API](#gluon-fit-api)\n      - [MKLDNN](#mkldnn)\n      - [Large tensor support](#large-tensor-support-1)\n      - [TensorRT integration](#tensorrt-integration-1)\n      - [Higher order gradient support](#higher-order-gradient-support)\n      - [Operator improvements](#operator-improvements-1)\n      - [Profiler](#profiler-1)\n      - [ONNX import/export](#onnx-importexport)\n      - [Runtime discovery of features](#runtime-discovery-of-features)\n      - [Bug fixes](#bug-fixes-1)\n    - [Front end API](#front-end-api-1)\n      - [Gluon](#gluon-1)\n      - [Symbol](#symbol-1)\n    - [Language Bindings](#language-bindings-1)\n      - [Python](#python-1)\n      - [C/C++](#cc-1)\n      - [Clojure](#clojure-1)\n      - [Julia](#julia-1)\n      - [Perl](#perl-1)\n      - [Scala](#scala-1)\n    - [Performance improvements](#performance-improvements-1)\n    - [Examples and tutorials](#examples-and-tutorials)\n    - [Website and documentation](#website-and-documentation-1)\n    - [CI/CD](#cicd-1)\n    - [Misc](#misc)\n  - [1.5.1](#151)\n    - [Bug-fixes](#bug-fixes-2)\n  - [1.5.0](#150)\n    - [New Features](#new-features-2)\n      - [Automatic Mixed Precision(experimental)](#automatic-mixed-precisionexperimental)\n      - [MKL-DNN Reduced precision inference and RNN API support](#mkl-dnn-reduced-precision-inference-and-rnn-api-support)\n      - [Dynamic Shape(experimental)](#dynamic-shapeexperimental)\n      - [Large Tensor Support](#large-tensor-support-2)\n      - [Dependency Update](#dependency-update)\n      - [Gluon Fit API(experimental)](#gluon-fit-apiexperimental)\n      - [New Operators](#new-operators-2)\n    - [Feature Improvements](#feature-improvements-2)\n      - [Operators](#operators)\n      - [MKLDNN](#mkldnn-1)\n      - [ONNX](#onnx-1)\n      - [TensorRT](#tensorrt)\n      - [FP16 Support](#fp16-support)\n      - [Deep Graph Library(DGL) support](#deep-graph-librarydgl-support)\n      - [Horovod Integration](#horovod-integration)\n      - [Dynamic Shape](#dynamic-shape)\n      - [Backend Engine](#backend-engine)\n      - [Large Tensor Support](#large-tensor-support-3)\n      - [Quantization](#quantization-1)\n      - [Profiler](#profiler-2)\n      - [CoreML](#coreml)\n    - [Front End API](#front-end-api-2)\n      - [Gluon](#gluon-2)\n      - [Python](#python-2)\n    - [Language Bindings](#language-bindings-2)\n      - [Scala](#scala-2)\n      - [Java](#java)\n      - [C++](#c)\n      - [Clojure](#clojure-2)\n      - [Julia](#julia-2)\n      - [Perl:](#perl-2)\n      - [R](#r-1)\n    - [Performance Improvements](#performance-improvements-2)\n    - [Example and Tutorials](#example-and-tutorials-1)\n    - [Website](#website)\n    - [Documentation](#documentation)\n    - [Build and Test](#build-and-test)\n    - [Bug-fixes](#bug-fixes-3)\n    - [License](#license-1)\n    - [Depreciations](#depreciations)\n    - [Known Issues](#known-issues)\n  - [1.4.1](#141)\n    - [Bug-fixes](#bug-fixes-4)\n  - [1.4.0](#140)\n    - [New Features](#new-features-3)\n      - [Java Inference API](#java-inference-api)\n      - [Julia API](#julia-api)\n      - [Control Flow Operators (experimental)](#control-flow-operators-experimental)\n      - [SVRG Optimization](#svrg-optimization)\n      - [Subgraph API (experimental)](#subgraph-api-experimental)\n      - [JVM Memory Management](#jvm-memory-management)\n      - [Topology-aware AllReduce (experimental)](#topology-aware-allreduce-experimental)\n      - [MKLDNN backend: Graph optimization and Quantization (experimental)](#mkldnn-backend-graph-optimization-and-quantization-experimental)\n        - [Graph Optimization](#graph-optimization)\n        - [Quantization](#quantization-2)\n    - [New Operators](#new-operators-3)\n    - [Feature improvements](#feature-improvements-3)\n      - [Operator](#operator)\n      - [Optimizer](#optimizer)\n      - [Sparse](#sparse)\n      - [ONNX](#onnx-2)\n      - [MKLDNN](#mkldnn-2)\n      - [Inference](#inference)\n      - [Other](#other)\n    - [Frontend API updates](#frontend-api-updates)\n      - [Gluon](#gluon-3)\n      - [Symbol](#symbol-2)\n    - [Language API updates](#language-api-updates)\n      - [Java](#java-1)\n      - [R](#r-2)\n      - [Scala](#scala-3)\n      - [Clojure](#clojure-3)\n      - [Perl](#perl-3)\n      - [Julia](#julia-3)\n    - [Performance benchmarks and improvements](#performance-benchmarks-and-improvements)\n    - [Bug fixes](#bug-fixes-5)\n    - [Licensing updates](#licensing-updates)\n    - [Improvements](#improvements)\n      - [Tutorial](#tutorial)\n      - [Example](#example)\n      - [Documentation](#documentation-1)\n      - [Website](#website-1)\n      - [MXNet Distributions](#mxnet-distributions)\n      - [Installation](#installation)\n      - [Build and CI](#build-and-ci)\n      - [3rd party](#3rd-party)\n        - [TVM:](#tvm)\n        - [CUDNN:](#cudnn)\n        - [Horovod:](#horovod)\n    - [Deprications](#deprications)\n    - [Other](#other-1)\n    - [How to build MXNet](#how-to-build-mxnet)\n    - [List of submodules used by Apache MXNet (Incubating) and when they were updated last](#list-of-submodules-used-by-apache-mxnet-incubating-and-when-they-were-updated-last)\n  - [1.3.1](#131)\n    - [Bug fixes](#bug-fixes-6)\n    - [Documentation fixes](#documentation-fixes)\n    - [Other Improvements](#other-improvements)\n    - [Submodule updates](#submodule-updates)\n    - [Known issues](#known-issues-1)\n  - [1.3.0](#130)\n    - [New Features - Gluon RNN layers are now HybridBlocks](#new-features---gluon-rnn-layers-are-now-hybridblocks)\n    - [MKL-DNN improvements](#mkl-dnn-improvements)\n    - [New Features - Gluon Model Zoo Pre-trained Models](#new-features---gluon-model-zoo-pre-trained-models)\n    - [New Features - Clojure package (experimental)](#new-features---clojure-package-experimental)\n    - [New Features - Synchronized Cross-GPU Batch Norm (experimental)](#new-features---synchronized-cross-gpu-batch-norm-experimental)\n    - [New Features - Sparse Tensor Support for Gluon (experimental)](#new-features---sparse-tensor-support-for-gluon-experimental)\n    - [New Features - Control flow operators (experimental)](#new-features---control-flow-operators-experimental)\n    - [New Features - Scala API Improvements (experimental)](#new-features---scala-api-improvements-experimental)\n    - [New Features - Rounding GPU Memory Pool for dynamic networks with variable-length inputs and outputs (experimental)](#new-features---rounding-gpu-memory-pool-for-dynamic-networks-with-variable-length-inputs-and-outputs-experimental)\n    - [New Features - Topology-aware AllReduce (experimental)](#new-features---topology-aware-allreduce-experimental)\n    - [New Features - Export MXNet models to ONNX format (experimental)](#new-features---export-mxnet-models-to-onnx-format-experimental)\n    - [New Features - TensorRT Runtime Integration (experimental)](#new-features---tensorrt-runtime-integration-experimental)\n    - [New Examples - Scala](#new-examples---scala)\n    - [Maintenance - Flaky Tests improvement effort](#maintenance---flaky-tests-improvement-effort)\n    - [Maintenance - MXNet Model Backwards Compatibility Checker](#maintenance---mxnet-model-backwards-compatibility-checker)\n    - [Maintenance - Integrated testing for \"the Straight Dope\"](#maintenance---integrated-testing-for-%22the-straight-dope%22)\n    - [Bug-fixes](#bug-fixes-7)\n    - [Performance Improvements](#performance-improvements-3)\n    - [API Changes](#api-changes)\n    - [Other features](#other-features)\n    - [Usability Improvements](#usability-improvements)\n  - [1.2.0](#120)\n    - [New Features - Added Scala Inference APIs](#new-features---added-scala-inference-apis)\n    - [New Features - Added a Module to Import ONNX models into MXNet](#new-features---added-a-module-to-import-onnx-models-into-mxnet)\n    - [New Features - Added Support for Model Quantization with Calibration](#new-features---added-support-for-model-quantization-with-calibration)\n    - [New Features - MKL-DNN Integration](#new-features---mkl-dnn-integration)\n    - [New Features - Added Exception Handling Support for Operators](#new-features---added-exception-handling-support-for-operators)\n    - [New Features - Enhanced FP16 support](#new-features---enhanced-fp16-support)\n    - [New Features - Added Profiling Enhancements](#new-features---added-profiling-enhancements)\n    - [Breaking Changes](#breaking-changes)\n    - [Bug Fixes](#bug-fixes-8)\n    - [Performance Improvements](#performance-improvements-4)\n    - [API Changes](#api-changes-1)\n    - [Sparse Support](#sparse-support)\n    - [Deprecations](#deprecations)\n    - [Other Features](#other-features-1)\n    - [Usability Improvements](#usability-improvements-1)\n    - [Known Issues](#known-issues-2)\n  - [1.1.0](#110)\n    - [Usability Improvements](#usability-improvements-2)\n    - [Bug-fixes](#bug-fixes-9)\n    - [New Features](#new-features-4)\n    - [API Changes](#api-changes-2)\n    - [Deprecations](#deprecations-1)\n    - [Performance Improvements](#performance-improvements-5)\n    - [Known Issues](#known-issues-3)\n  - [1.0.0](#100)\n    - [Performance](#performance)\n    - [New Features - Gradient Compression [Experimental]](#new-features---gradient-compression-experimental)\n    - [New Features - Support of NVIDIA Collective Communication Library (NCCL) [Experimental]](#new-features---support-of-nvidia-collective-communication-library-nccl-experimental)\n    - [New Features - Advanced Indexing [General Availability]](#new-features---advanced-indexing-general-availability)\n    - [New Features - Gluon [General Availability]](#new-features---gluon-general-availability)\n    - [New Features - ARM / Raspberry Pi support [Experimental]](#new-features---arm--raspberry-pi-support-experimental)\n    - [New Features - NVIDIA Jetson support [Experimental]](#new-features---nvidia-jetson-support-experimental)\n    - [New Features - Sparse Tensor Support [General Availability]](#new-features---sparse-tensor-support-general-availability)\n    - [Bug-fixes](#bug-fixes-10)\n    - [Doc Updates](#doc-updates)\n  - [0.12.1](#0121)\n    - [Bug-fixes](#bug-fixes-11)\n  - [0.12.0](#0120)\n    - [Performance](#performance-1)\n    - [New Features - Gluon](#new-features---gluon)\n    - [New Features - Autograd](#new-features---autograd)\n    - [New Features - Sparse Tensor Support](#new-features---sparse-tensor-support)\n    - [Other New Features](#other-new-features)\n    - [API Changes](#api-changes-3)\n    - [Bug-fixes](#bug-fixes-12)\n  - [0.11.0](#0110)\n    - [Major Features](#major-features)\n    - [API Changes](#api-changes-4)\n    - [Performance Improvements](#performance-improvements-6)\n    - [Bugfixes](#bugfixes)\n    - [Refactors](#refactors)\n  - [0.10.0](#0100)\n  - [0.9.3](#093)\n  - [v0.8](#v08)\n  - [v0.7](#v07)\n  - [v0.5 (initial release)](#v05-initial-release)\n\n## 1.8.0\n### Features\n#### CUDA Graphs\n - Enable CUDA Graphs for TRT (#19184)\n - CUDA graphs support (#19142)\n - Update cudnn version. (#19375)\n#### CUDA 11 Support\n - Update CUB and include it only for CUDA < 11 #18799' (#18975)\n - Add new CI pipeline for building and testing with cuda 11.0. (#19149)\n - Enable CUDA 11.0 on nightly development builds (#19314)\n#### TensorRT\n - TensorRT: add int8 with calibration (#19011)\n - Add TRT verbose mode (#19100)\n - Backporting TensorRT-Gluon Partition API (and TensorRT 7 support) (#18916)\n - Backport TRT test update #19296 (#19298)\n#### OneDNN\n - Upgrade to oneDNN v1.6.3 (#19153) (#19161)\n - Update oneDNN to official v1.6 release (#18867) (#18867)\n - Upgrade to oneDNN v1.6 (#18822)\n - bumped version to v1.6.5 (#19437)\n - Upgrade to oneDNN v1.7 (#19560)\n#### IntGemm\n - Backport of intgemm #17559 (#19099)\n - Change intgemm to a submodule instead of fetch. (#19406)\n#### Subgraph API\n - Backport Fix for duplicate subgraph inputs/outputs (#16131) (#19112)\n#### Extensions\n - Backport #19103 (#19117)\n - Backporting #19016 (#19069)\n - Backport: Change Partition API's options_map to std::unordered_map #18929 (#18964)\n - Backporting #18779 to v1.x (#18894)\n - Backport extension bug fixes to v1.8.x (#19469) (#19504)\n - fix for MX_ERROR_MSG namespace (#19756)\n#### ONNX\n - Update onnx support to work with onnx 1.7.0 with most CV models (#19017)\n#### Large Tensor\n - Fix linalg_potri and linalg_potrf operators for large tensor. (#18752)\n - Add forward, backward test for linalg.gemm2 (#18784)\n - Add large matrix tests for linalg ops: det, inverse, trsm, trmm (#18744)\n - Add Large Tensor Test for linalg_syrk (#18782)\n - Add Large Dim Checks for linalg Operators (#18816)\n - Add forward & backward linalg.gemm test for large size (#18825)\n - Adding error message when attempting to use Large tensor with linalg_syevd (#18807)\n#### Website Improvements\n - v1.8 website patch (#19212)\n - Automate website artifacts uploading (#19244)\n#### Documentation\n - Fix mxnet.test_utils.check_numeric_gradient documentation (#19060)\n - Update windows_setup.md (#18874)\n#### License\n - Stop packaging GPL libquadmath.so (#19055)\n - Remove mention of nightly in pypi (#18635) (#18884)\n - Mkldnn header fix v1x for nightly binaries (#18797)\n - Update LICENSE for all submodules. (#19440)\n - LICENSE update (#19443)\n - Update LICENSE (#19704) (#19707)\n#### CI Improvements\n - Upgrade unix gpu toolchain (#18186) (#18785)\n - Fix CI in v1.x branch (#18907)\n - Remove extra --build-arg causing docker command to fail. (#19412)\n - Fix CI builds failing due to invalid GPG keys. (#19377) (#19388)\n#### Bug Fixes\n - Backport #19656 - fix R builds (#19658)\n - remove cleanup on side threads (#19557)\n - Don't use namespace for pow() function, since it is built into cuda math library, and cast the second argument so it will find an acceptable form. (#19533)\n - Remove temporary fix for RNN (#19451)\n - backport #19393 to v1.8.x (#19398)\n - Fix SoftReLU fused operator numerical stability (#17849) (#19390)\n - Temporary fix for RNN with oneDNN seg faults/core dumps (#19308)\n - Fix MKLDNN BatchNorm with even number of channels (#19150) #19299 #19425 (#19428)\n - Relaxing type requirements for broadcast_like (#17977) (#19448)\n - Backporting: Fixed setting attributes in reviewSubgraph (#19278)\n - Include oneDNN gemm fix (#19251)\n - Fix for breaking change introduced in #17123 when batch_axis=0 (#19283)\n - Backport PR #19272 to v1.8.x (#19273)\n - Backport PRs in v1.7.x missing from v1.x to v1.8.x (#19262)\n - Delete executor before reallocating it memory (#19222)\n - Nightly Large Tensor test cherrypicks (#19194) (#19215)\n - Tweeking syntax to be closer to other tests (#19186) (#19206)\n - ElementWiseSum fix for oneDNN (#18777) (#19200)\n - Fix flaky intgemm test in v1.8.x too (#19204)\n - Revert \"Fix memory leaks in Gluon (#18328) (#18359)\" (#19181)\n - Improve environment variable handling in unittests (#18424) (#19173)\n - Backport Unittest tolerance handling improvements (#18694). Also test seeding (#18762). (#19148)\n - Fix the error of gradient of np.pad (#19044) (#19167)\n - Backport Add cmake flag USE_FATBIN_COMPRESSION, ON by default (#19123) (#19158)\n - SymbolBlock.imports ignore_extra & allow_missing (#19156)\n - Fix race condition in NaiveEngine::PushAsync (#19108) (#19122)\n - Empty list cannot be cleared issue fixed. (#14882)\n - Update base_module.py (#19096)\n - Fix block.export (#17970) (#19075)\n - Support for fp16 in SpM x DnsM on GPU (#18930) (#19074)\n - Backport of Fix LeakyRelu behaviour on empty input (#18934) (#19009)\n - Get rid of monkey patching in LossScaler overflow handling (#18959) (#18973)\n - Remove upper bound (#18857) (#18910)\n - Fix gelu to use erf based algorithm (#18827) (#18946)\n - Cherry-pick #18635 to v1.7.x (#18935) (#18945)\n - Backporting backward inference from 2.x #18348 and #18378 (#18895)\n - Backport Invoke mkldnn and cudnn BatchNorm when axis != 1 to v1.7.x (#18676) (#18890)\n - Bump version to 1.8.0 (#18899)\n - Fixing ONNX spatial export for batchnorm (#17711) (#18846)\n - Fix softmax, logsoftmax failed on empty ndarray (#18602) (#18708)\n - Add unit tests for potri and potrf backward and check output shape in unit tests. (#18803)\n - Add syrk test shape check (#18812)\n - Back port optimization to broadcast_axis to MXNet1.x (#18773)\n - Fix crash when accessing already destructed static variables (#18768) (#18778)\n - Cherrypick #18677 #18713 (#18742)\n\n## 1.7.0\n\n### New features\n#### MXNet Extensions: custom operators, partitioning, and graph passes\n\nAdds support for extending MXNet with custom operators, partitioning strategies, and graph passes. All implemented in a library easily compiled separately from the MXNet codebase, and dynamically loaded at runtime into any prebuilt installation of MXNet.\n\n - fix for number of inputs/outputs for backward custom ops (#17069)\n - Enhancements for custom subgraph op (#17194)\n - Disable flaky test_custom_op_fork (#17481)\n - fix custom op makefile (#17516)\n - Update CustomOp doc with changes for GPU support (#17486)\n - [WIP] MXNet Extensions enhancements (#17885) (#18128)\n - Dynamic subgraph property (#17034)\n - Dynamic subgraph property doc (#17585)\n - [1.7] Backport MXNet Extension PRs (#17623, #17569, #17762) #18063 (#18069)\n\n#### OpPerf utility enabled in the binary distribution\n - [OpPerf] Add Neural network loss ops (#17482)\n - [OpPerf] Fixes the issue when you pass NDArray to run_perf_test (#17508)\n - [OpPerf] Fix markdown for native profile and add profile param in function desc (#17494)\n - [OpPerf] Add Indexing ops (#16253)\n - [OpPerf] Implement remaining random sampling ops (#17502)\n - [OpPerf] Implement remaining GEMM ops (#17501)\n - [OpPerf] Implement all linalg ops (#17528)\n - [OpPerf] Fixed native output ordering, added warmup & runs command line args (#17571)\n - [OpPerf] Add norm, cast ops, remaining optimizer ops (#17542)\n - [OpPerf] Fixed Python profiler bug (#17642)\n\n#### MKL-DNN\n##### MKL-DNN as the default CPU backend in binary distribution\n##### Branding change to DNNL\n - Upgrade MKL-DNN dependency to v1.1 (#16823)\n\n##### Support bfloat16 datatype\n - Add bfloat16 floating-point format support based on AMP  (#17265)\n\n#### New operators\n - [New Op] Add deformable conv v2 (#16341)\n - Add MXNet Ops for fast multihead attention (#16408)\n - Support boolean elemwise/broadcast binary add, multiply and true_divide (#16728)\n - add gammaln, erf, erfinv (#16811)\n - add aligned roi introduced in Detectron2 (#16619)\n - Implement atleast_1d/2d/3d (#17099)\n - Interleaved MHA for CPU path (#17138)\n - Lamb optimizer update (#16715)\n - Quantized Embedding (#16691)\n - Add gelu fuse ops (#18082) (#18092)\n\n### Feature improvements\n#### Numpy compatible interface(experimental)\n - [NumPy] NumPy support for linalg.inv (#16730)\n - add numpy op nan_to_num (#16717)\n - [Numpy] Add sampling method for bernoulli (#16638)\n - Fix numpy-compatible mean output type for integer inputs (#16792)\n - [Numpy] Fix collect_params().zero_grad() in gluon numpy interface (#16716)\n - [Numpy][Operator] 'where' Implementation in MXNet (#16829)\n - [Numpy] Random.normal() with backward (#16330)\n - Add OP diag [numpy] (#16786)\n - Mixed precison binary op backward (use in) for numpy (#16791)\n - add numpy op diagflat [numpy] (#16813)\n - add op bitwise_or [numpy] (#16801)\n - [Numpy] Implementation npx.{sample}_n (#16876)\n - [Numpy] Add NumPy support for np.linalg.det and np.linalg.slogdet (#16800)\n - Op Unravel_index PR [Numpy] (#16862)\n - [Numpy] Fix imperative basic indexing in numpy (#16902)\n - [Numpy] Basic indexing in symbolic interface of DeepNumpy (#16621)\n - [Numpy] add op full_like, c++ impl, fix zeros_like, ones_like type inference (#16804)\n - [Numpy] Implement numpy operator 'average' (#16720)\n - [Bugfix] [Numpy] Add `kAddTo` and kNullOp to Transpose (#16979)\n - set rtol = 1e-2 and atol = 1e-4 when dtype == np.float32 in test_numpy_op.py:test_np_linalg_solve (#17025)\n - Op_Diagonal [Numpy] (#16989)\n - numpy bincount (#16965)\n - [numpy] add op bitwise_not (#16947)\n - [Numpy ]Modify np.random.shuffle to enable inplace by default (#17133)\n - [numpy] fix argsort typo (#17150)\n - [numpy] add op round (#17175)\n - [numpy]Add op delete (#17023)\n - [numpy] add op flipud, fliplr (#17192)\n - [CI] Re-enable testing with numpy 1.18 (#17200)\n - [Numpy] Add broadcast_to scalar case (#17233)\n - [Numpy] Random.gamma() implemented (#16152)\n - [Numpy] add row_stack (=vstack) (#17171)\n - [Numpy] Add infra for performing constraint check (#17272)\n - porting numpy-compatible hstack to master and add dstack for interoperability (#17030)\n - adding asnumpy() to output of gather(implicitly called) to fix gather test in large vector and tensor tests (#17290)\n - [numpy] add op random.exponential (#17280)\n - [NumPy] Add NumPy support for norm (#17014)\n - [numpy]add op random.lognormal  (#17415)\n - Add numpy random weibull operator (#17505)\n - [numpy] Add np.random.pareto and np.random.power (#17517)\n - [Numpy] Add sort op (#17393)\n - [numpy]implement exponential backward (#17401)\n - [Numpy] Where operator scalar version (#17249)\n - [numpy] add op matmul (#16990)\n - [numpy]add op random.logistic, random.gumbel (#17302)\n - [numpy][Do Not Review]add op insert (#16865)\n - [numpy] add op random.rayleigh (#17541)\n - [numpy] add fallback ops (#17609)\n - [numpy] add op pad (#17328)\n - [numpy] add op fabs, sometrue, round_ (#17619)\n - Add arange_like to npx (#16883)\n - try to move shape_array to npx (#16897)\n - support np.argsort (#16949)\n - np.broadcast_to extension (#17358)\n - support bitwise_and (#16861)\n - fix np.argmax/argmin output data type (#17476)\n - add op random.beta (#17390)\n - add op isnan isinf (#17535)\n - array_split pr (#17032)\n - Mixed data type binary ops (#16699)\n - randn implemented (#17141)\n - refactor and reduce float types for some functions, also add bitwise_xor (#16827)\n - any/all (#17087)\n - amax (#17176)\n - fix format (#17100)\n - add op empty_like, add nan_to_num to dispatch (#17169)\n - handle array_like fill_value for np.full; add unit test coverage (#17245)\n - add np.amin (#17538)\n - add npx.gather_nd (#17477)\n - add np.random.chisquare (#17524)\n - add polyval (#17416)\n - add isposinf isneginf isfinite (#17563)\n - Support broadcast assign for `npi_boolean_mask_assign_tensor` (#17131)\n - Implement Weibull backward (#17590)\n - support np.dsplit, fix some error msgs and corner cases for hsplit and vsplit, add interoperability tests for h/v/dsplit (#17478)\n - add np.product (#17489)\n - Implement np.random.pareto backward (#17607)\n - add np.ediff1d (#17624)\n - more support for boolean indexing and assign (#18352)\n - Fix einsum gradient (#18482)\n - [v1.7.x] Backport PRs of numpy features (#18653)\n - [v1.7.x] backport mixed type binary ops to v1.7.x (#18649)\n - revise activations (#18700)\n\n#### Large tensor support\n - [Large Tensor] Add support to Random Sample & Pdf ops (#17445)\n - [Large Tensor] Add LT support for NN optimizers and 1 activation function (#17444)\n - [Large Tensor] Fixed SoftmaxActivation op (#17634)\n - [Large Tensor] Fixed col2im op (#17622)\n - [Large Tensor] Fixed Spatial Transformer op (#17617)\n - [Large Tensor] Fix ravel_multi_index op (#17644)\n - Sparse int64 Large tensor support (#16898)\n - Re-Enabling Large Tensor Nightly on GPU (#16164)\n - enabling build stage gpu_int64 to enable large tensor nightly runs (#17546)\n - [Large Tensor] Fixed Embedding op (#17599)\n\n#### MKL-DNN enhancement\n - MKLDNN FC : Add error info when mkldnn fc bias dimension is wrong (#16692)\n - [MKLDNN] support mkldnn gelu (#16710)\n - [MKLDNN] Fix int8 convolution/fc bias overflow (#16734)\n - [MKLDNN] use dim_t instead of int in slice/transpose operators (#16737)\n - Mkldnn fullyConnect bwd bug fix (#16890)\n - Revert Mkldnn fullyConnect bwd bug fix (#16890) (#16907)\n - [MKLDNN] Use MKLDNNRun (#16772)\n - [MKLDNN] mkldnn RNN operator enhancement (#17075)\n - [MKLDNN] enable MaxPooling with full pooling convention (#16860)\n - update mkldnn to v1.1.2 (#17165)\n - improve mkldnn doc (#17198)\n - [MKLDNN] Fix _copyto  (#17173)\n - [MKLDNN] Support channel wise quantization for FullyConnected (#17187)\n - fixed seed for mkldnn test (#17386)\n - add mkldnn softmax backward  (#17170)\n - cmake: copy dnnl headers to include/mkldnn (#17647)\n - [mkldnn]Mkldnn bn opt backport from master to 1.7x (#18009)\n - [v1.x] Update 3rdparty/mkldnn remote URL and pin to v1.3 (#17972) (#18033)\n - [v1.x] backport #17900 [MKLDNN] support using any format in pooling backward (#18067)\n - Static link MKL-DNN library (#16731)\n - Add large tensor nightly tests for MKL-DNN operators (#16184)\n -  [MKL-DNN] Enable and Optimization for s8 eltwise_add (#16931)\n - [MKL-DNN] Enhance Quantization Method (#17161)\n - Static Build and CD for mxnet-cu102/mxnet-cu102mkl (#17074)\n - MKL-DNN RNN backward path enhancement (#17183)\n - cmake: check USE_OPENMP and pass proper MKL-DNN build flags (#17356)\n - update mkl to 2020.0 (#17355)\n - Enable MKL-DNN by default in pip packages (#16899)\n - Enable MKL-DNN FullyConnected backward (#17318)\n - Softmax primitive cache and in-place computation (#17152)\n - boolean_mask_assign with start_axis (#16886)\n - use identity_with_cast (#16913)\n - change error tolerance for bf16 bn (#18110)\n - [v1.x] Backport #17689 and #17884 to v1.x branch (#18064)\n - refactor codes and add an option to skip/check weight's version to reduce overhead (#17707) (#18039)\n - [v1.x] Backport #17702 and #17872 to v1.x branch (#18038)\n\n#### TensorRT integration\n - Update TensorRT tutorial to build-from-source. (#14860)\n - Minor fix, use RAII for TensorRT builder and network object (#17189)\n\n#### Quantization\n - Add silent option to quantization script (#17094)\n\n#### Profiler\n - Implemented final two binary ops, added default params for functionality (#17407)\n - Implement remaining nn_activation ops in opperf (#17475)\n - Implement all miscellaneous ops (#17511)\n - Implement remaining nn_basic ops in opperf (#17456)\n\n#### ONNX\n - Fix memory leak reported by ASAN in NNVM to ONNX conversion (#15516)\n - ONNX export: Gather (#15995)\n - ONNX export: Slice op - Handle None value for ends (#14942)\n\n#### New models\n - [Model] Implement Neural Collaborative Filtering with MXNet (#16689)\n - Further optimization for NCF model (#17148)\n - HMM Model (#17120)\n\n#### Operator improvements\n - Faster GPU NMS operator (#16542)\n - [MXNET-1421] Added (CuDNN)BatchNorm operator to the list of mirrored operators (#16022)\n - dynamic custom operator support (#15921)\n - Multi Precision Lamb Update operator (#16885)\n - Add im2col and col2im operator (#16502)\n - Quantized Elemwise Mul Operator (#17147)\n - Enhancements for MXTensor for custom operators (#17204)\n - Enabling large tensor support for binary broadcast operators (#16755)\n - Fix operators lying about their number of inputs (#17049)\n - [WIP] Fallback mechanism for mx.np operators (#16923)\n - Dynamic custom operator GPU support (#17270)\n - Fix flaky - test_operator_gpu.test_np_insert (#17620)\n - MXNet FFI for Operator Imperative Invocation (#17510)\n - [MXNET-978] Higher Order Gradient Support `logp1`, `expm1`, `square`. (#15416)\n - [MXNET-978] Higher Order Gradient Support `arcsin`, `arccos`. (#15515)\n - [MXNET-978] Higher Order Gradient Support `rsqrt`, `rcbrt`. (#15476)\n - gather_nd: check bound and wrap negative indices (#17208)\n - Remove dilation restriction for conv3d (#17491)\n - Fix storage type infer of softmax backward (#17576)\n - Fix and optimize handling of vectorized memory accesses (#17767) (#18113)\n - Cherry-pick of #17995 and #17937 to 1.x branch (#18041)\n - No tensor cores for fp32 interleaved attention, remove div by 8 restriction (#17994) (#18085)\n - GPU gemms true fp16 (#17466) (#18023)\n - Add support for boolean inputs to FusedOp (#16796)\n\n#### Bug fixes\n - [BUG FIX] Always preserve batch dimension in batches returned from dataloader (#16233)\n - Fix SliceChannel Type inference (#16748)\n - change _generate_op_module_signature get_module_file open with encoding=utf-8,it fix some encode error in Chinese windows system. (#16738)\n - Fix rtrue_divide grad (#16769)\n - fix inv test flakiness using random matrices generated by SVD (#16782)\n - [MXNET-1426] Fix the wrong result of sum, mean, argmin, argmax when inputs contain inf or nan (#16234)\n - Fix (#16781)\n - fix expand_dims fall back when input's ndim is 0 (#16837)\n - [fix] missing input log higher order. (#15331)\n - Fix IndentationError in setup.py (#16857)\n - Fix a few np issues (#16849)\n - Fix InferAttr/InferShapeAttr not calling inference for all nodes in a graph (#16836)\n - fix for enable model parallelism for non-fp32 data (#16683)\n - Fix NDArrayIter iteration bug when last_batch_handle='pad' (#16166)\n - Fix crashing on Windows in ObjectPool ~ctor (#16941)\n - Fix NDArrayIter cant pad when size is large (#17001)\n - fix axis=-1 bug (#17016)\n - Fix CUDNN detection for CMake build (#17019)\n - Fix omp assert issue (#17039)\n - mshadow: fix vector access (#17021)\n - [BUGFIX] Fix race condition in kvstore.pushpull (#17007)\n - [BUGFIX] Fix trainer param order (#17068)\n - [BugFix] fix filter channel calculation in ModulatedDeformableConvV2 (#17070)\n - Fix reshape interoperability test (#17155)\n - fix norm sparse fallback (#17149)\n - fix py27 quantization (#17153)\n - fix int8 add ut (#17166)\n - Fix and clean up Ubuntu build from source instructions (#17229)\n - fix lstm layer with projection save params (#17266)\n - Fix rendering of ubuntu_setup.md codeblocks (#17294)\n - Fix #17267, add expected and got datatype for concat error msgs (#17271)\n - [BUGFIX] fix model zoo parallel download (#17372)\n - fix use int8, uint8, int32, int64 (#17188)\n - [Fix] Add ctx to the original ndarray and revise the usage of context to ctx (#16819)\n - Fix ndarray indexing bug (#16895)\n - fix requantize flaky test (#16709)\n - Initial checkin (#16856)\n - Fix flakey test_ndarray.py:test_reduce (#17312)\n - fix flaky test: boolean index and fix bugs (#17222)\n - Fix IOT Devices section of Get Started page (#17326)\n - add logic for no batch size while getting data arrays from executors (#17772) (#18122)\n - Fix reverse shape inference in LayerNorm (#17683)\n - fix full and full_like when input is boolean (#17668)\n - Fix MBCC inference (#17660)\n - Additional fix for vector access. (#17230)\n - Cherrypick Fix nightly large_vector test caused by incorrect with_seed path (#18178) (#18220)\n - [1.7] Pass args fix3 (#18237)\n - fixing batch_norm and layer_norm for large tensors (#17805) (#18261)\n - [1.7.x] Backport of LSTM and GRU fix (#17898) and RNN op (#17632) (#18316)\n - [v1.7.x] backport #18500 - [Bug Fixed] Fix batch norm when grad_req is `add` (#18517)\n - Fix the monitor_callback invalid issue during calibration with variable input shapes (#18632) (#18703)\n\n### Front end API\n - Fix the problem in printing feature in c++ API examples : feature_extract (#15686)\n - updating MXNet version to 1.6.0 in base.h for C APIs (#16905)\n - [API] unified API for custom kvstores (#17010)\n - fix parameter names in the estimator api (#17051)\n - adding docs for 64bit C APIs of large tensor (#17309)\n - Add API docs to INT64 APIs (#16617)\n\n#### Gluon\n - [Quantization] Enhance gluon quantization API (#16695)\n - [Gluon] Improve estimator usability and fix logging logic (#16810)\n - Fix test_gluon.py:test_sync_batchnorm when number of GPUS > 4 (#16834)\n - [Gluon] Update contrib.Estimator LoggingHandler to support logging per batch interval (#16922)\n - Include eval_net the validation model in the gluon estimator api (#16957)\n - Fix Gluon Estimator nightly test (#17042)\n - [MXNET-1431] Multiple channel support in Gluon PReLU (#16262)\n - Fix gluon.Trainer regression if no kvstore is used with sparse gradients (#17199)\n - refactor gluon.utils.split_data() following np.array_split() (#17123)\n - Add RandomApply in gluon's transforms (#17242)\n - Partitioning Gluon HybridBlocks (#15969)\n - Random rotation (#16794)\n - bump up atol for gradient check (#16843)\n - Extend estimator.evaluate() to support event handlers (#16971)\n - [MXNET-1438] Adding SDML loss function (#17298)\n\n#### Symbol\n - Add unoptimized symbol to executor for sharing (#16798)\n - Enforces NDArray type in get_symbol (#16871)\n - Fix #17164 symbolblock with BatchNorm inside during cast to fp16 (#17212)\n - autograd video and image link fixes and removing symbol tutorials (#17227)\n - Fix CosineEmbeddingLoss in when symbol API is used (#17308)\n - Fix Horovod build error due to missing exported symbols (#17348)\n - Update symbol.py (#17408)\n - update symbol to json (#16948)\n\n### Language Bindings\n#### Python\n - Python 2 compatibility fix in base.py\n - adding stacktrace in Jenkinsfile_utils.groovy to inspect Python2 failure cause in CI (#17065)\n - Fix image display in python autograd tutorial (#17243)\n - Fix Python 3 compatibility in example/speech_recognition (#17354)\n - Stop testing Python 2 on CI (#15990)\n - Docs: Python tutorials doc fixes (#17435)\n - pin python dependencies (#17556)\n - Python 2 cleanup (#17583)\n\n#### C/C++\n - Simplify C++ flags (#17413)\n\n#### R\n - fix R docs (#16733)\n - [R package] Make R package compilation support opencv 4.0 (#16934)\n - Support R-package with cmake build and fix installation instructions (#17228)\n - Fix R-package/src/Makevars for OpenCV 4 (#17404)\n - Fix typo in Install the MXNet Package for R (#17340)\n\n#### Clojure\n\n#### Julia\n - [MXNET-1440] julia: porting `current_context` (#17142)\n - julia: porting `context.empty_cache` (#17172)\n - pin Markdown version to 3.1 in Julia doc build (#17549)\n\n#### Perl\n - [Perl] - ndarray operator overloading enhancements (#16779)\n - MXNET-1447 [Perl] Runtime features and large tensor support. (#17610)\n\n#### Scala\n - Fix scala publish & nvidia-docker cublas issue (#16968)\n - Fix publishing scala gpu with cpu instance (#16987)\n - swap wget to curl in Scala scripts (#17041)\n - [Scala/Java] Remove unnecessary data slicing (#17544)\n - quantile_scalar (#17572)\n - Fix get_started scala gpu (#17434)\n - Fix MBCC & scala publish pipeline (#17643)\n - Bump up additional scala 1.x branch to 1.7.0 (#17765)\n\n### Performance improvements\n - Build.py improvement (#16976)\n - Improvements to config.cmake (#17639)\n - [Done] BilinearResize2D optimized (#16292)\n - Speed fused_op compilation by caching ptx and jit-compiled functions (#16783)\n - Improve the speed of the pointwise fusion graph pass (#17114)\n - broadcast_axis optimization (#17091)\n - Optimize AddTakeGrad Tensor Sum (#17906) (#18045)\n\n### Example and tutorials\n - Add CustomOp tutorial doc (#17241)\n - Correct the grammar in 1-ndarray tutorial (#17513)\n\n### Website and documentation\n - Website edits (#17050)\n - [Website 2.0] Nightly Build for v1.x (#17956)\n - [docs] Fix runtime feature detection documentation (#16746)\n - Adding user guidelines for using MXNet built with Large Tensor Support (#16894)\n - fix typo and doc (#16921)\n - large tensor faq doc fix (#16953)\n - [DOC] Add a few tips for running horovod (#17235)\n - Update NOTICE to fix copyright years (#17330)\n - [DOC] Fix tutorial link, and better error msg (#17057)\n - doc fix for argmax & argmin (#17604)\n\n### CI/CD\n - support mixed-precision true_divide (#16711)\n - Try to fix CI (#16908)\n - mixed precision for power (#16859)\n - Fix desired precision for test_ndarray.py:test_reduce (#16992)\n - [reproducibility] multi_sum_sq review, AtomicAdd removal (#17002)\n - fix precision problem in linalg_solve, linalg_tensorinv, linalg_cholesky op test (#16981)\n - grouping large array tests based on type and updating nightly CI function (#17305)\n - [LICENSE] fix cpp predcit license (#17377)\n - [CI] Fix static build pipeline (#17474)\n - skipping tests that cannot fit in nightly CI machine corrected imports (#17450)\n - Update Windows CI scripts to use syntax compatible with Win 2019 server powershell. (#17526)\n - Fix Non-ASCII character in docstring (#17600)\n - [CI] Follow redirects when downloading apache-maven-3.3.9-bin.tar.gz (#17608)\n - [CI] Upgrade sphinx and autodocsumm (#17594)\n - Reduce load on CI due to excessive log flood (#17629)\n - Enable users to specify BLAS (#17648)\n - [CI] Add AMI id to instance info on builds (#17649)\n - [v1.7.x] Backport staggered CI builds (#17999 & #18119) (#18142)\n - [v1.7.x] Backport #17177 to 1.7.x (Fix incorrect calculation results when the C locale is set to a locale that uses commas as the decimal separator) (#18147)\n - Fix formatting and typos in CD README.md (#16703)\n - [CD] dynamic libmxet pipeline fix + small fixes (#16966)\n - [CD] enable s3 publish for nightly builds in cd (#17112)\n - [CD] fix CD pipeline (#17259)\n - [CD] update publish path (#17453)\n - fix CD and remove leftover from #15990 (#17551)\n - Fix nightly build (#16773)\n - Update pypi_publish.py to disable nighlty build upload to Pypi (#17082)\n - [v1.7.x] update jetson dockerfile to support CUDA 10.0 (#18339)\n - Remove manually created symbolic link to ninja-build (#18437) (#18456)\n - Increase staggered build timeout to 180 min (#18568) (#18585)\n\n### License\n - Don't relicense FindCUDAToolkit.cmake (#17334)\n - fix license and copyright issues (#17364)\n - Update ps-lite LICENSE (#17351)\n - remove unused file with license issue (#17371)\n - Update LICENSE for fonts (#17365)\n - license np_einsum file under bsd (#17367)\n - Update Apache License for mshadow (#18109) (#18134)\n - Julia: remove downloading of the non-ASF binary build (#18489) (#18502)\n - Add missing license header for md files (#18541)\n - [v1.7.x]License checker enhancement (#18478)\n\n### Miscellaneous changes\n - Link fixes4 (#16764)\n - Refactoring names for mxnet version of nnvm to avoid conflicting with the original tvm/nnvm. (#15303)\n - minor typo fix (#17008)\n - Add micro averaging strategy to pearsonr metric (#16878)\n - introduce  gradient update handler to the  base estimator (#16900)\n - fix latency calculation and print issue (#17217)\n - add inference benchmark script (#16978)\n - change the wording and log level to be more in line with the general use (#16626)\n - Updated logos. (#16719)\n - Pinning rvm version to satisfy Jekyll build (#18016)\n - Workaround gnu_tls handshake error on Ubuntu 14.04 Nvidia Docker (#18044)\n\n## 1.6.0\n\n### Deprecation of Python 2\n\nMXNet community [voted](https://lists.apache.org/thread.html/r3a2db0f22a1680cc56804191446fef2289595798ca19fd17de1ff03e%40%3Cdev.mxnet.apache.org%3E) to no longer support Python 2 in future releases of MXNet. Therefore, MXNet 1.6 release is going to be the last MXNet release to support Python 2.\n\n### New features\n\n#### NumPy compatible interface and using TVM to generate operators\n\nNumPy has long been established as the standard math library in Python, the most prevalent language for the deep learning community. With this library as the cornerstone, there are now the largest ecosystem and community for scientific computing. The popularity of NumPy comes from its flexibility and generality.\n\nIn #14253, the MXNet community reached consensus on moving towards a NumPy-compatible programing experience and committed to a major endeavor on providing NumPy compatible operators.\n\nThe primary goal of the projects below is to provide the equivalent usability and expressiveness of NumPy in MXNet to facilitate Deep Learning model development, which not only helps existing deep learning practitioners but also provides people in the existing NumPy community with a shortcut for getting started in Deep Learning. The efforts towards this goal would also help a secondary goal, which is to enable the existing NumPy ecosystem to utilize GPUs and accelerators to speed up large scale computation.\n\n - Infra to use tvm write op kernels (#15550)\n - fix boolean_mask for 0-size output (#15731)\n - fix tvm cmake (#15781)\n - Numpy-compatible Infra (#15581)\n - [MXNET-1206] Support NDArray indexing with None and Ellipsis (#13143)\n - numpy-compatible sum (#15810)\n - [Numpy] Numpy compatible slicing (#15798)\n - Numpy Tensordot and Dot Operator (#15820)\n - numpy linspace (#15852)\n - tvm infra for op attrs (#15854)\n - Port several np ops to master (#15867)\n - numpy-compatible split upstream (#15841)\n - Numpy-compatible concatenate upstream (#15894)\n - Numpy-compatible stack upstream (#15842)\n - [Numpy] Numpy behavior random.uniform() (#15858)\n - Tvm broadcast backward (#15938)\n - np elemwise unary ops upstream (#15831)\n - [Numpy] random.randint() implemented (#15956)\n - Refines NDArray indexing and adds numpy ndarray indexing [READY FOR REVIEW] (#15942)\n - Port ops from np branch (#16018)\n - numpy-compatible cumsum upstream (#15924)\n - NumPy-compatible infrastructure on Gluon (#16024)\n - [OP] Support range as advanced index for ndarrays (#16047)\n - Numpy compatible max min (#16046)\n - NumPy-compatible Mean, Std and Var (#16014)\n - Add fluent methods mean, std, var for ndarray (#16077)\n - numpy multinomial op (#15878)\n - add numpy operator remainder (#16080)\n - [Numpy] Random.choice implemented (#16089)\n - Fix sample.normal shape inference\n - Numpy add numpy op indices (#15837)\n - [Numpy] Numpy copysign (#15851)\n - numpy operator ravel, derive from reshape (#16016)\n - Add __array_function__\n - Improved error mesages\n - Fix np.choice\n - add exception check for numpy reshape (#16180)\n - [Numpy] Numpy behavior normal distribution (#16109)\n - fix multinomial bug on gpu (#16204)\n - [Numpy] Differentiable svd (#15795)\n - add epsilon to sum(pvalue) upperbound (#16211)\n - np compatible vstack (#15850)\n - Numpy add numpy op roll (#15902)\n - add numpy compatible trace (#16008)\n - add numpy op hanning, hamming, blackman (#15815)\n - [Numpy]flip (#15819)\n - numpy operator around (#16126)\n - numpy operator arctan2 (#15890)\n - numpy operator nonzero (#15838)\n - numpy operator hypot (#15901)\n - tvm numpy operator deg2rad && rad2deg (#16015)\n - numpy op unique\n - try to fix bug\n - fix memory bug and disable some test\n - fix according to review\n - Numpy operators: `lcm`, `tril`, `identity` and `take` (#16264)\n - [numpy] Cosmetic improvement on mxnet.numpy builtin op signature in documentation (#16305)\n - Disable Pylint false error in numpy_op_signature  (#16370)\n - boolean_mask_assign operator for future boolean indexing (#16361)\n - Implements ldexp. (#15845)\n - Numpy Operators: Inner, Outer, vdot (#15846)\n - Numpy det and slogdet operators (#15861)\n - Fix random op signature\n - fix choice signature\n - add raise test for shape\n - Add boolean ndarray (#15940)\n - global numpy shape flag (#16335)\n - numpy-compatible histogram (#16266)\n - [Numpy] Numpy compatible dstack (#15871)\n - numpy eye op (#16132)\n - Numpy compatible vsplit; minor changes to split (#15983)\n - add numpy op logspace (#15825)\n - add numpy op bitwise_xor, hsplit, moveaxis, rot90 (#16257)\n - Fix optimizer bug for np attribute (#16494)\n - Tests of NumPy interoperability (#16469)\n - improve unary and binary operator handling and refactor tests (#16423)\n - [DOC] Fix numpy op doc  (#16504)\n - [Numpy] More numpy dispatch tests (#16426)\n - [Numpy] einsum (#15911)\n - Add test pipeline for USE_TVM_OP=OFF on Unix (#16450)\n - Numpy dispatch test of ...... (#16422)\n - setup and concatenate, copy, expand_dims, expm1 (#16493)\n - add sum for boolean type in mainline (#16436)\n - [Numpy] SVD outputs tuple (#16530)\n - numpy op doc: max, min, prod (#16506)\n - add interface for rand\n - Fix numpy bugs (#16537)\n - pickler override for np ndarrays (#16561)\n - [numpy]op test in new pattern (#16556)\n - Enforce adding documentation for builtin numpy operators (#16575)\n - [Numpy] Support N_D(N>=3) batch_dot (#16586)\n - [Numpy] Loading numpy-incompatible NDArray in numpy-compatible mode (#16597)\n - Fix index overflow bug in einsum (#16589)\n - add npx reshape (#16640)\n - add type switch to weight tensor (#16543)\n - numpy doc enhancement (#16637)\n - Infra for tvm op runtime dispatch (#16100)\n - [NumPy][Operator] NumPy operator `may_share_memory` and `shares_memory` (#16533)\n - [Numpy] Numpy operator diff (#15906)\n - Miscellaneous fix for several numpy issues (#16664)\n - [Numpy] implement np.column_stack (#16594)\n - [numpy] add numpy operator : append (#16564)\n - Backport of #16711, #16737, #16408 to 1.6 branch (#16763)\n - Backport to 1.6 (#16773, #16781, #16783, #16716, #16699, #16728, #16769, #16792) (#16832)\n - [Backport][v1.6.x] Fix the wrong result of sum, mean, argmin, argmax when inputs contain inf or nan (#16884)\n - Backport of #16827, #16791 and #16888 to 1.6 branch (#16901)\n - port shape op to 1.6.x (#16912)\n - [Numpy] Fix imperative basic indexing in numpy (#16902) (#16919)\n - Backport #16895, #16922, #16878, #16979 and #16900 to 1.6 (#17029)\n\n\n#### Graph optimizations\n\n##### Pointwise fusion for GPU\n\nDL models, besides compute intensive operations like convolutions and fully connected layers, feature a lot of simple pointwise (aka elementwise) operations (like elementwise addition etc.). Performance of those operations is fully memory bandwidth bound and so limit speedups from newer GPU hardware, which typically has high compute/memory bandwidth ratio. When multiple of such operations are chained one after another, it results in a series of unnecessary stores and loads as well as potential increased memory usage to store the intermediate results. Pointwise fusion helps in alleviating those problems by just-in-time generation of fused operators, which do not store intermediate results in memory, resulting in performance and memory usage improvements.\n\n - Pointwise fusion for GPU (#15167)\n - Backport #16798, #16836 and #16838 to 1.6 (#16874)\n - Add support for boolean inputs to FusedOp (#16796) (#16892)\n - Workaround problem with fusion in CUDA 9 (#17028) (#17035)\n\n##### Eliminate common subexpressions\n\n - Eliminate common expressions (#15657)\n\n##### Default MKLDNN Subgraph fusion\n\n - [MKLDNN] Enable subgraph backend mkldnn by default. (#15518)\n\n#### New operators\n\n - [OP] Add a new arange_like operator to contrib (#15400)\n - PDF operators for each distribution for which we have a random sampler (plus also the PDF of the Dirichlet).  Supports probabilities and log-probabilities, as well as gradients. (#14617)\n - Group Normalization (#14959)\n - Add RROIAlign (#16017)\n - Add fast implementation of LARS (#16122)\n - Round and sign straight-through-estimators C operators. (#16373)\n - New ops for RCNN + old ops improvements for RCNN (#16215)\n - Comparison ops implemented using mshadow (#16414)\n - Add mask target generator operator for Mask-RCNN (#16268)\n - Move MRCNNMaskTarget op to contrib (#16486)\n - Mxnet allclose (#14443)\n - Aggregated adamw update (#16398)\n - Make mrcnn_mask_target arg mask_size a 2d tuple (#16567)\n - Dgl ops 2 (#16416)\n - Lamb optimizer update (#16715)\n - [OP] changing data type of 't' to int in lamb_update_phase1 (#16903)\n - Multi Precision Lamb Update operator (#16885)\n - Interleaved MHA for CPU path (#17138) (#17211)\n\n### Feature improvements\n\n#### Automatic Mixed Precision\n\n - [AMP] Move topk from FP16_FP32_FUNCS to FP32_FUNCS (#15342)\n - Conversion from FP32 model to Mixed Precision model (#15118)\n - Update fp16 docs: Block.cast is inplace (#15458)\n - FP16 Support for C Predict API (#15245)\n - Add AMP Conversion support for BucketingModule (#15528)\n\n#### Gluon Fit API\n\n - Fixing build for gluon estimator test, including libtvm in pack libs (#16148)\n - [Estimator] handle composite metrics in estimator (#16676)\n - [Estimator] refactor estimator to allow overriding evaluate/fit of a batch (#16678)\n - [Estimator] refactor estimator and clarify docs (#16694)\n - [Gluon] Improve estimator usability and fix logging logic (#16810) (#16846)\n - Backport Gluon estimator changes to 1.6 (#17048)\n - fix parameter names in the estimator api (#17051) (#17162)\n\n\n#### MKLDNN\n\n - Upgrade MKL-DNN submodule to v0.20 release (#15422)\n - Fix quantized concat when inputs are mixed int8 and uint8 (#15693)\n - [MKLDNN]Enhance Quantization APIs and Tutorial (#15448)\n - Add quantization support for GluonCV (#15754)\n - add int8 bn mkldnn implementation and test (#15664)\n - [Quantization]support exclude operators while quantization (#15910)\n - [MKLDNN]Support fullyconnected and element-wise ops fusion (#15950)\n - Disable test coverage for Clang MKLDNN (#15977)\n - update support MKLDNN BN conditions (#15870)\n - [MKLDNN] Fix out of bound access of req vector (#16000)\n - add uint8 bn mkldnn implementation (#16003)\n - Improve quantization flow (#15961)\n - [MKLDNN] fix uint8 batch norm memory misuse (#16034)\n - MKL-DNN RNN checks NDArray version (#16071)\n - Float64 fallback for mkldnn subgraph and rnn op (#15853)\n - Update MKL-DNN dependency (#16073)\n - Integrate MKL-DNN leakyrelu (#16075)\n - [MKLDNN] NDArray reorder in C API and deconv (#16265)\n - Fix mkldnn reshape (#16455)\n - [MKLDNN] Fix uint quantized fc when not fusing with requantize (#16523)\n - [MKLDNN]Fix reorder2default (#16602)\n - Upgrade MKL-DNN dependency to v1.0 (#16555)\n - Revert \"[MKLDNN]Fix reorder2default (#16602)\" (#16697)\n - [v1.6.x] Backport #16837 into v1.6.x (#16847)\n - Initial checkin (#16856) (#16872)\n\n#### Large tensor support\n\n - [MXNET-1413] Adding Large Tensor support for sort operators (#15170)\n - Large Index Support for Slice (#15593)\n - Add large tensor support binary arithmetic (#15785)\n - Large tensor support for random ops (#15783)\n - Add Large Tensor Support for Sequence, NN Ops  (#15807)\n - Add power, exponent, log ops large tensor support (#15794)\n - removing unnecessary int64 C apis that were added to support Large Tensors and Vectors (#15944)\n - creating ndarray directly using mxnet ndarray primitives to reduce memory footprint of tests for topk, sort and argsort (#15900)\n - Adding tests to verify support for Large Tensors in additional Ops along with new C_Apis supporting 64bit indexing (#15895)\n - Added tests to verify Large Vector Support for initial set of ops  (#15943)\n - Added more tests for Large Indices (#15960)\n - Add Large tensor vector test cases (#15941)\n - Test large vector mean operator and fix a few bugs (#16079)\n - Reducing memory footprint of one_hot for Large Array Testing (#16136)\n - removing MXNDArrayLoadFromBuffer64 and MXNDArrayLoad64 (#16203)\n - Fix large array tests (#16328)\n - added more tests to verify support for large vector (#16477)\n - added support for large tensors for Dropout operator and tests to verify support for more operators (#16409)\n - adding large tensor support for add_n and tests for more ops (#16476)\n - adding large tensor support for pad operator (#15126)\n - Added large tensor support and test for gather_nd (#16371)\n - Large Vector tests for DGL Ops Part 2 (#16497)\n - Showing proper error message when an attempt is made to create large tensor but MXNet is not built with it (#16570)\n\n#### TensorRT integration\n\n - enable TensorRT integration with cpp api (#15335)\n - Add unit tests for TensorRT integration and fix some bugs (#15399)\n\n#### Higher order gradient support\n\n - [MXNET-978] Higher order gradient for sigmoid (#15288)\n - [MXNET-978] Higher Order Gradient Support `reciprocal`, `abs`. (#15413)\n - [MXNET-978] Add higher order gradient support `tan`, `tanh` (#15253)\n - [MXNET-978] Higher Order Gradient Support `arctan`, `arctanh`, `radians`. (#15531)\n - [MXNET-978] Higher Order Gradient Support `sqrt`, `cbrt`. (#15474)\n - [MXNET-978] Higher Order Gradient Support `clip`, `dropout`. (#15746)\n - [MXNET-978] Higher Order Gradient Support `sinh`, `cosh`. (#15412)\n - [MXNET-978] n-th order gradient test support. (#15611)\n - [MXNET-978] Fully connected, higher order grad (#14779)\n - [MXNET-978] Higher Order Gradient Support `arcsinh`, `arccosh`. (#15530)\n\n#### Operator improvements\n\n - broadcast axis is alias to broadcast axes; doc fix (#15546)\n - Utility to help developers debug operators: Tensor Inspector (#15490)\n - Softmax with length (#15169)\n - in-place reshape ops (#14053)\n - Add missing default axis value to symbol.squeeze op (#15707)\n - Add matrix determinant operator in linalg (#15007)\n - Add fp16 support for topk (#15560)\n - [MXNET-1399] multiclass-mcc metric enhancements (#14874)\n - new raise mode for nd.take and fix backward for wrap mode (#15887)\n\n#### Profiler\n\n - Fixing duplication in operator profiling (#15240)\n - Custom Operator Profiling Enhancement (#15210)\n - [Opperf] Make module/namespace of the operator parameterized (#15226)\n - Opperf: Support Python<3.6 (#15487)\n - Add transpose_conv, sorting and searching operator benchmarks to Opperf (#15475)\n - Deprecate USE_PROFILER flag (#15595)\n - Update profiler.md (#15477)\n - [Opperf] Add array rearrange operators to opperf (#15606)\n - [OpPerf] PDF Random ops fix (#15661)\n - [Opperf] Add  optimizer update operator benchmarks to opperf (#15522)\n - fix broadcast op param (#15714)\n - [OpPerf] Profiler flag for Python, Cpp  (#15881)\n - [Opperf] Filter out deprecated ops (#15541)\n - [OpPerf] Handle positional arguments (#15761)\n - [OpPerf] Take care of 4d param  (#15736)\n - Add Median,p50,p99 to python profiler (#15953)\n - adding \"total\" (total time) to profiler aggregate stats sorting criteria (#16055)\n\n#### ONNX import/export\n\n - Correct ONNX documentation (#15914)\n - [MXNET-895] ONNX import/export: TopK (#13627)\n\n#### Runtime discovery of features\n\n - Making Features as a singleton for improved caching (#15835)\n\n#### Bug fixes\n\n - [bug] fix higher grad log  (#15120)\n - Showing proper error when csr array is not 2D in shape. (#15242)\n - add 'asnumpy' dtype option to check_symbolic_backward (#15186)\n - point fix the vector declaration in MultiBoxDetection (#15300)\n - Temporarily Commenting out Flaky Test (#15436)\n - Fix memory leak in NaiveEngine (#15405)\n - fix nightly CI failure (#15452)\n - Small typo fixes in batch_norm-inl.h (#15527)\n - Bypass cuda/cudnn checks if no driver. (#15551)\n - Julia path patch (#15561)\n - Fix AMP Tutorial failures (#15526)\n - Fix warnings in CLang: (#15270)\n - Fix dumps for Constant initializer (#15150)\n - fix normalize mean error bug (#15539)\n - [fix] print `self` in warning. (#15614)\n - [MXNET-1411] solve pylint error issue#14851 (#15113)\n - [Flaky test] Skip test_operator_gpu.test_convolution_independent_gradients (#15631)\n - Fix subgraph with custom_op (#15671)\n - Fix USE_BLAS == openblas check (#15691)\n - update previous flaky naive engine test (#15651)\n - make TransposeShape infer shape form both sides (#15713)\n - Skip Flaky Test (#15722)\n - Revert \"Dynamic Library Loading Support\" (#15755)\n - Fix flaky test test_global_metric (#15756)\n - Fix PR #15489 (Dynamic Library Loading Support) (#15760)\n - Refactor LibraryInitializer so it's thread safe. Fixes random sporadical concurrency crashes. (#15762)\n - Fix backward_clip num inputs and type of clip params (#15688)\n - fixing problem with existing Singleton Caching (#15868)\n - Allow operators with multiple outputs in get_atomic_symbol (#15740)\n - Fix ConcatType backward type inference (#15829)\n - Add disable attr to subgraph property (#15926)\n - Re-enable flaky test_prelu (#15777)\n - declare explicitly the tblob default assign operator and copy constructor (#15937)\n - Discard needless test cases in `test_convolution_independent_gradients` (#15939)\n - fix naive engine for multi-threaded inference (#15574)\n - Fix get_rows_per_block (#15979)\n - Fix a memory misalignment in topk operator (#15948)\n - Decouple dtype from shape for Random multinomial (#15980)\n - Fix dtype inference in arange_like operator (#15930)\n - Disable laop_6 (#15976)\n - Fix flaky clojure profile test (#16058)\n - fix test_pick test time  is too long (#16066)\n - [fix] Support nullop in `transpose` (#15865)\n - fix flaky test (#16074)\n - fix some test files test time is too long (#16067)\n - Fix gradient tensor mutate in `{adam/ftrl/rmprop/rmspropalex}_update`. (#15768)\n - Fix unary operator ceil/floor/trunc when data type is integer (#14251)\n - Fix failing tests (#16117)\n - Fixes NAG optimizer #15543 (#16053)\n - avoid test relu at the origin due to discontinuous gradient (#16133)\n - Fix remaining errors reported by D2L (#16157)\n - use 1E-4 in groupnorm test(#16169)\n - Sequence last fix (#16156)\n - fixing test for model compatibility checker (#16159)\n - assert_allclose -> rtol=1e-10 (#16198)\n - [MEMORY] retry GPU memory allocation if fragmented (#16194)\n - improve dataloader signals and messages (#16114)\n - Update ndarray.py (#16205)\n - fix flaky test (#16191)\n - Solve #14116, #15143 (#15144)\n - [MXNET-1422] Fix wrong results of min([inf, inf]) and max([-inf,-inf]) (#16226)\n - Fix inconsistent interpolation method values (#16212)\n - set fixed seed for profiler (#16155)\n - Fix MXNDArrayGetData (#16289)\n - fix atol for test_preloaded_multi_sgd (#16356)\n - Fix windows flakiness (#16415)\n - cuDNN non-persistant bidirectional RNN dgrad sync fix (#16391)\n - [BUGFIX] Minor type issues in Squeeze (#16448)\n - Fix Nightly Tests for Binaries (#16451)\n - Fix dtype bug (#16467)\n - Fix flakey pylint CI failures (#16462)\n - Load NDArray only to GPU if GPU is present (#16432)\n - Bug fix for the input of same axes of the swapaxes operator (#16513)\n - Fix learning rate scheduler being unexpectedly overwritten by optimizer's default value (#16487)\n - disable tests (#16536)\n - fix pylint in CI (#16540)\n - image crop gpu (#16464)\n - Build dmlc-core with old thread_local implementation (#16526)\n - fix doc for topk (#16571)\n - RNNOp to call cudaEventCreate lazily (#16584)\n - add encoding to the stub files for potential utf8 char in doc strings (#16580)\n - Surpress subgraph log in CI (#16607)\n - Fix dequantize memory corruption (#16606)\n - Fix for wrong reqs set after switching from training to inference (#16553)\n - Disables test_bulking_operator_gpu due to flakiness (#16611)\n - Imagenet inference to nightly fix (#16599)\n - Move some subgraph verbose to MXNET_SUBGRAPH_VERBOSE=2 (#16622)\n - RNNOp only call cuda/cudnn if GPU ctx is requested (#16632)\n - fix bad encode (#16641)\n - Disable float16 test (#16643)\n - Fix GetMKLDNNData for delay alloc (#16618)\n - Move ops which don't support FP16 dtype to FP32 list (#16668)\n - no such method => modified function args (#16610)\n - fix cuDNN RNN dtype_with_fallback_ bug (#16671)\n - Add check if scipy is imported in sparse.py (#16574)\n - Added launch bounds to the reduce kernels (#16397)\n - fix install dir (#16690)\n - fix binary dependencies in CD and nightly (#16693)\n - Fix SliceChannel Type inference (#16748) (#16797)\n - fix flakiness of test_np_mixed_precision_binary_funcs (#16873)\n - Fix test_gluon.py:test_sync_batchnorm when number of GPUS > 4 (#16835)\n - Omp fork numthreads fix 1.6 (#17000)\n - [BUGFIX] Fix race condition in kvstore.pushpull (#17007) (#17052)\n - Backport #17002, #17068 and #17114 to 1.6 branch (#17137)\n - Backport 3rdparty/openmp fixes (#17193)\n - fix norm sparse fallback (#17149)\n\n### Front end API\n\n - Expose get_all_registered_operators and get_operator_arguments in the… (#15364)\n - Add magic method `abs` to NDArray and Symbol. (#15680)\n - Dynamic Library Loading Support (#15489)\n - [MXNET-1294] Add KVSTORE PushPull API (#15559)\n\n#### Gluon\n\n - [Dataset] Add take, filter, sample API to dataset (#16078)\n - Add register_op_hook for gluon (#15839)\n - [Dataset] add shard API (#16175)\n - Add list_ctx to ParameterDict (#16185)\n - [Gluon] Support None argument in HybridBlock (#16280)\n - Aggregated zero grad (#16446)\n - try to fix block (#16465)\n - [Gluon] Don't serialize shared parameters twice (#16582)\n - Initializer.__eq__ (#16680)\n\n#### Symbol\n\n - Add symbol api for randn and fix shape issue for randn ndarray and symbol api (#15772)\n - Graph Partition API (#15886)\n\n### Language Bindings\n\n#### Python\n\nMXNet community [voted](https://lists.apache.org/thread.html/r3a2db0f22a1680cc56804191446fef2289595798ca19fd17de1ff03e%40%3Cdev.mxnet.apache.org%3E) to no longer support Python 2 in future releases of MXNet. Therefore, MXNet 1.6 release is going to be the last MXNet release to support Python 2.\n\n#### C/C++\n\n - [C++] Improve inference script to support benchmark on Imagenet (#15164)\n - C Api for simplebind, fix comment for trigoops, add atol to assert (#16585)\n\n#### Clojure\n\n - Extend Clojure BERT example (#15023)\n - [Clojure] Add fastText example (#15340)\n - make clojure api generator tests less brittle (#15579)\n\n#### Julia\n\n - add julia env settings (#15523)\n - julia: bump window prebult binary version to v1.5.0 (#15608)\n - julia: remove Travis CI related files (#15616)\n - julia: bump binding version to v1.6.0 (#15607)\n - julia: rename build env var `MXNET_HOME` to `MXNET_ROOT` (#15568)\n - Revert \"julia: rename build env var `MXNET_HOME` to `MXNET_ROOT` (#15568)\" (#16147)\n - julia: fix `mx.forward` kwargs checking (#16138)\n - julia: implement `context.num_gpus` (#16236)\n - julia: add `AbstractMXError` as parent type (#16235)\n - [MXNET-1430] julia: implement context.gpu_memory_info (#16324)\n - julia/docs: more DRY on page rendering (#16396)\n\n#### Perl\n\n - [Perl] - simplify aliasing strategy (#15395)\n - [Perl] - ndarray to native array conversion fix (#16635)\n\n#### Scala\n\n - Add Sparse NDArray support for Scala (#15378)\n - fix the bug on Scala Sparse (#15500)\n - fix heap-use-after-free in scala (#15503)\n - Bump Scala version to 1.6 (#15660)\n - Fix Scala Symbolic API some/Some typo (#15687)\n - Faster Scala NDArray to BufferedImage function (#16219)\n\n### Performance improvements\n\n - Proper bulking of ops not using FCompute (#15272)\n - improve layernorm CPU performance (#15313)\n - Efficient MXNet sampling in the multinomial distribution (#15311)\n - Revert default return type for indices in argsort() and topk() back to float32 (#15360)\n - Use omp threads for cpu data loader (#15379)\n - Accelerate ROIPooling layer (#14894)\n - Avoid memory copy for dropout inference (#15521)\n - Add omp parallel optimization for _contrib_BilinearReisze2D (#15584)\n - Softmax optimization for GPU (#15545)\n - Speed up group executor (#16069)\n - FullyConnected Bias performance improvement on GPU (#16039)\n - Embedding gradient performance optimization on GPU (#16355)\n - Faster Transpose 2D (#16104)\n - Pseudo 2D transpose kernel (#16229)\n - Faster general take (#16615)\n\n### Examples and tutorials\n\n - [TUTORIAL] Gluon performance tips and tricks (#15427)\n - Updating profiler tutorial to include new custom operator profiling (#15403)\n - [TUTORIAL] Gluon and Sparse NDArray (#15396)\n - [TUTORIAL] Revise Naming tutorial (#15365)\n - Revise Symbol tutorial (#15343)\n - Two fixes for info_gan.md example Code (#15323)\n - Rebase #13757 to master (#15189)\n - Tensor Inspector Tutorial (#15517)\n - logging (#15106)\n - update profiler tutorial (#15580)\n - [MXNET-1358] Fit api tutorial (#15353)\n - Tutorials nighly fix (#16179)\n - Update add_op_in_backend.md (#16403)\n - typo fix in r doc lstm tutorial (#16546)\n - [MKL-DNN] Add mxnet mkldnn cmake tutorial (#16688)\n\n### Website and documentation\n\n - [DOC] Clarify that global pooling is going to reset padding (#15269)\n - Update sparse_retain Documentation (#15394)\n - nano instructions (#15117)\n - remove comments from nano instructions (#15433)\n - REAME   MTCNN   Link URL Error in original website (#15020)\n - Update Horovod docs links in README (#15366)\n - fix doc for sort and argsort (#15317)\n - fix comment (#15481)\n - Improve docs for AMP (#15455)\n - [Doc] Add MKL install method apt/yum into tutorial (#15491)\n - Julia docs (#15454)\n - Docs: Fix misprints (#15505)\n - website build for julia: fix path to be static (#15554)\n - some minor typos/clarifications (#15538)\n - refine Nano setup directions (#15524)\n - [Doc] add squeeze to Array change shape (#15549)\n - fix typo (#15648)\n - Fix url (404 error) (#15683)\n - update julia install doc (#15609)\n - [DOC] refine autograd docs (#15109)\n - [DOC] Fix many arguments in the doc: reshape_like, arange_like, shape_array (#15752)\n - Add Gather_nd Scatter_nd to NDArray API category doc (#15689)\n - [Dependency Update] [Doc] move the general prerequisite software to the top (#15896)\n - typo in docs (#16094)\n - [WIP] New Website: New Docs [1/3] (#15884)\n - [DOC] Fix doc for nn.Embedding, nn.Dense and nd.Embedding (#15869)\n - [DOC] Consistent capitalization: mxnet -> MXNet, scala -> Scala (#16041)\n - New Website: Remove Old Content [2/3] (#15885)\n - New Website: New Pipeline [3/3] (#15883)\n - Update KL Divergence formula (#16170)\n - fix broken links (#16255)\n - redirect to the 404 page (#16287)\n - add google-analytics config (#16271)\n - Fixing links for website + Fixing search (#16284)\n - Minor fix in ToTensor documentation. (#16299)\n - adding redirects so that old website API links surfaced from searches (#16342)\n - Fix code block formatting in Why MXNet doc page (#16334)\n - Julia: add API docs back (#16363)\n - Change mailing list url in footer to point to instructions about how to subscribe instead (#16384)\n - Add instructions to report a security vulnerability (#16383)\n - [DOC] fix installation selector wrong history (#16381)\n - Beta build (#16411)\n - [WIP] Improving Python Docs API (#16392)\n - fix autodoc for spurrious toggles (#16452)\n - [Doc] Update the download page with 1.5.1 release (#16442)\n - Fixing broken links (#16500)\n - add binary and docs build command options (#16514)\n - add option to remove indexes (#16525)\n - Correct Google Analytics Tracker (#16490)\n - [Doc] Use mirror link in the download page (#16501)\n - checking broken link fixes work (#16538)\n - detect number of procs during sphinx build (#16512)\n - fixed broken links across multiple files (#16581)\n - fix missing docs due to git add issues (#16496)\n - second round of fixing broken links in multiple files (#16598)\n - Python Docstring Convetion (#16550)\n - [MXNET-1434] Fix a broken link for basic C++ tutorial (#16461)\n - Fix python doc build issue (#16630)\n - fixing broken links in multiple files - round 3 (#16634)\n\n### CI/CD\n\n - Fix build_ccache_wrappers: (#14631)\n - Remove mhard-float option. This is already deprecated by Google. (#15435)\n - CI: upgrade Julia version from 1.0.3 to 1.0.4 (#15502)\n - Add -R option to ci/build.py to avoid rebuilding containers (#15426)\n - [Dependency Update] Bump up the CI Nvidia docker to CUDA 10.1 (#14986)\n - fixed config.mk and Makefile bugs for installing mkl (#15424)\n - Add -DMXNET_USE_OPENMP to Makefiles so libinfo gets updated accordingly (#15498)\n - [Dependency Update] Dependency update doc (#15045)\n - Remove Scala package test on build (#15915)\n - Refactor for windows CI 'out of heap space' errors (#15922)\n - Fix Nightly Maven GPU (#15989)\n - Windows cmake flags cleanup (#16013)\n - Disable flaky test in test_amp_conversion (#16031)\n - Updates git_init Jenkins utility function to support checking out a particular commit id\n - Adds artifact repository scripts\n - Adds CD pipeline framework\n - Adds static libmxnet release pipeline\n - Updates CD pipeline\n - Adds documentation\n - Updates kvstore functions to use pushd and popd\n - Throws exceptions instead o magic numbers\n - Updates artifact repository cli to use --libtype instead of --static or --dynamic\n - Clarifies ci_utils and cd_utils origin remark\n - Adds clarifying note on why ubuntu 14.04 is being used for compilation\n - Removes MXNET_SHA\n - Removes set_release_job_name\n - Adds license headers\n - Updates artifact repository to expect licenses\n - Moves ci/cd to cd directory\n - Takes downstream job name from environment\n - Updates order of parameters\n - Updates job type parameter to dropdown\n - Adds libmxnet feature extraction code comments\n - Removes ccache setup from static build\n - Disable test coverage of C++ codebase on CI  (#15981)\n - Update readme and project.clj comment (#16084)\n - Enable tvm_op for ci (#15889)\n - Not to search for coverage files when none exist (#16107)\n - Fixes openblas installation for static build\n - Update python dependencies (#16105)\n - CD Fixes (#16127)\n - Adds dynamic libmxnet to CD pipeline (#16163)\n - Fix README Build Status (#16183)\n - subscribe to build and CD changes (#16192)\n - [CD] Add COMMIT_ID param to release job (#16202)\n - Fix lack of dylib support in Makefile when use lapack (#15813)\n - Removes git status update stop gap solution (#16285)\n - add mkl installation temp fix (#16304)\n - add 'Release' cmake flag (#16294)\n - S3 upload artifacts (#16336)\n - Fix nightly scala pipeline (#16362)\n - remove redundant branch name (#16372)\n - Skipping installing nightly test (#16418)\n - Adds PyPI CD Pipeline (#16190)\n - upgrade the pytest version (#16429)\n - Revert \"add mkl installation temp fix (#16304)\" (#16369)\n - increase docker cache timeout (#16430)\n - Adds pip requirements file to nightly gpu ci image (#16472)\n - [CD] Adds python docker pipeline (#16547)\n - Move imagenet inference to nightly (#16577)\n - Backport #16980 #17031 #17018 #17019 to 1.6 branch (#17213)\n\n### Misc\n\n - update committer info (#15289)\n - Typo fix in plan_memory relase -> release. (#15299)\n - indent changes (#15321)\n - Had a few PRs merged. Hope to become an official contributor and potentially a commiter. (#15451)\n - cuda/cuDNN lib version checking.  Force cuDNN v7 usage. (#15449)\n - Improve diagnose.py, adding build features info and binary library path. (#15499)\n - update ratcheck for apache-rat 0.13 release (#15417)\n - add myself to interested modules (#15590)\n - 1.5.0 news (#15137)\n - bump up version from 1.5.0 to 1.6.0 on master (#15072)\n - Remove myself from CODEOWNERS (#15617)\n - remove mshadow submodule\n - import mshadow source tree\n - cuDNN support cleanup (#15812)\n - Remove requests_failed_to_import handling\n - Update CODEOWNERS. (#15972)\n - Improve diagnose.py to display environment variables (#15715)\n - Update README.md (#16035)\n - [Dev] update ps-lite dependency (#15936)\n - Typedef cleanup (#15899)\n - add KEY for Tao Lv (#16081)\n - remove 'foo' and other print msg from test (#16088)\n - Revert accidental change to CMakelists (#16040)\n - Update env_var.md (#16145)\n - Update dmlc-core (#16149)\n - adding codeowners (#16165)\n - Factorize CUDA_KERNEL_LOOP used in CUDA kernels (#16197)\n - add code of conduct and conflict resolution (#16343)\n - simple typo error in NEWS.md (#16344)\n - update NEWS.md and README.md (#16385)\n - split issue templates (#16558)\n - Create SECURITY.md (#16573)\n\n## 1.5.1\nApache MXNet (incubating) 1.5.1 is a maintenance release incorporating important bug fixes and important performance improvements. All users of Apache MXNet (incubating) 1.5.0 are advised to upgrade. You can install Apache MXNet (incubating) 1.5.1 at the usual place. Please review these Release Notes to learn the bug fixes.\n\n### Bug-fixes\n* add deconv in TRT subgraph (#15666) (#16043)\n* Update TRT tutorial with new APIs (#16044)\n* Fix _copy_to on MKLDNN backend (#15637) (#15803)\n* Benchmark doc fix (#15769) (#16029)\n* remove Julia cat image for license issue (#15964) (#16026)\n* added check for empty params file and unknown param (not arg/aux) (#15917)\n* fix license issues (#15806) (#15860)\n* prevent TRT_Logger to be destroyed before TRT engine (#14898) (#15877)\n* [MXNET-1086] added sub and mul to ONNX->TensorRT conversion (#15344) (#15875)\n* handle fix_gamma in tensorrt subgraph conversion correctly (#15645) (#15874)\n* fix LinearRegressionOutput with empty label (#15620) (#15873)\n* [v1.5.x] [MKLDNN] Independent gradients requests check with respect to weights… (#15805)\n* fix dropout mask output (#15697) (#15804)\n* fix fp32 flatten issue (#15351) (#15802)\n* Clojure package remove source images (#15828)\n* changed constructor args (#15601) (#15827)\n* Add MKLDNN 4c layout to fix gluoncv se_resnext101_64x4d (#15692) (#15801)\n* Fix the bug of `MXEnginePushAsyncND` and `MXEnginePushSyncND` (#15751) (#15792)\n\n## 1.5.0\n\n### New Features\n\n#### Automatic Mixed Precision(experimental)\nTraining Deep Learning networks is a very computationally intensive task. Novel model architectures tend to have increasing numbers of layers and parameters, which slow down training. Fortunately, software optimizations and new generations of training hardware make it a feasible task.\nHowever, most of the hardware and software optimization opportunities exist in exploiting lower precision (e.g. FP16) to, for example, utilize Tensor Cores available on new Volta and Turing GPUs. While training in FP16 showed great success in image classification tasks, other more complicated neural networks typically stayed in FP32 due to difficulties in applying the FP16 training guidelines.\nThat is where AMP (Automatic Mixed Precision) comes into play. It automatically applies the guidelines of FP16 training, using FP16 precision where it provides the most benefit, while conservatively keeping in full FP32 precision operations unsafe to do in FP16. To learn more about AMP, check out this [tutorial](https://github.com/apache/mxnet/blob/master/docs/tutorials/amp/amp_tutorial.md).\n\n#### MKL-DNN Reduced precision inference and RNN API support\nTwo advanced features, fused computation and reduced-precision kernels, are introduced by MKL-DNN in the recent version. These features can significantly speed up the inference performance on CPU for a broad range of deep learning topologies. MXNet MKL-DNN backend provides optimized implementations for various operators covering a broad range of applications including image classification, object detection, and natural language processing. Refer to the [MKL-DNN operator documentation](https://github.com/apache/mxnet/blob/v1.5.x/docs/tutorials/mkldnn/operator_list.md) for more information.\n\n#### Dynamic Shape(experimental)\nMXNet now supports Dynamic Shape in both imperative and symbolic mode. MXNet used to require that operators statically infer the output shapes from the input shapes. However, there exist some operators that don't meet this requirement. Examples are:\n* while_loop: its output size depends on the number of iterations in the loop.\n* boolean indexing: its output size depends on the value of the input data.\n* many operators can be extended to take a shape symbol as input and the shape symbol can determine the output shape of these operators (with this extension, the symbol interface of MXNet can fully support shape).\nTo support dynamic shape and such operators, we have modified MXNet backend. Now MXNet supports operators with dynamic shape such as [`contrib.while_loop`](https://mxnet.apache.org/api/python/ndarray/contrib.html#mxnet.ndarray.contrib.while_loop), [`contrib.cond`](https://mxnet.apache.org/api/python/ndarray/contrib.html#mxnet.ndarray.contrib.cond), and [`mxnet.ndarray.contrib.boolean_mask`](https://mxnet.apache.org/api/python/ndarray/contrib.html#contrib)\nNote: Currently dynamic shape does not work with Gluon deferred initialization.\n\n#### Large Tensor Support\nCurrently, MXNet supports maximal tensor size of around 4 billon (2^32). This is due to uint32_t being used as the default data type for tensor size, as well as variable indexing.\nThis limitation has created many problems when larger tensors are used in the model.\nA naive solution to this problem is to replace all uint32_t in the MXNet backend source code to int64_t.\nThis solution is not viable, however, because many data structures use uint32_t as the data type for its members.\nUnnecessarily replacing these variables to int64_t will increase the memory consumption causing another limitation. Second, MXNet has many submodule dependencies.\nUpdating the variable types in the MXNet repository is not enough. We also need to make sure different libraries, such as MKLDNN, MShadow etc. supports the int64_t integer data type.\nThird, many front end APIs assume unsigned 32-bit integer interface. Only updating the interface in C/C++ will cause all the language bindings to fail.\nTherefore, we need a systematic approach to enhance MXNet to support large tensors.\nNow you can enable large tensor support by changing the following build flag to 1: `USE_INT64_TENSOR_SIZE = 1`. Note this is set to 0 by default.\nFor more details please refer to the [design document](https://cwiki.apache.org/confluence/display/MXNET/Large+Tensor+Support).\n\n#### Dependency Update\nMXNet has added support for CUDA 10, CUDA 10.1, cudnn7.5, NCCL 2.4.2, and numpy 1.16.0.\nThese updates are available through PyPI packages and build from source, refer to [installation guide](https://mxnet.apache.org/versions/master/install/index.html) for more details.\n\n#### Gluon Fit API(experimental)\nTraining a model in Gluon requires users to write the training loop. This is useful because of its imperative nature, however repeating the same code across multiple models can become tedious and repetitive with boilerplate code.\nThe training loop can also be overwhelming to some users new to deep learning. We have introduced an Estimator and Fit API to help facilitate training loop.\nNote: this feature is still experimental, for more details, refer to [design document](https://cwiki.apache.org/confluence/display/MXNET/Gluon+Fit+API+-+Tech+Design).\n\n#### New Operators\n* split_v2 (#13687)\n* Gradient multiplier (contrib) operator (#13632)\n* Image normalize operator - GPU support, 3D/4D inputs (#13802)\n* Image ToTensor operator - GPU support, 3D/4D inputs (#13837)\n* Add Gluon Transformer Crop (#14259)\n* GELU (#14449)\n* AdamW operator (Fixing Weight Decay Regularization in Adam) (#13728)\n* [MXNET-1382] Add the index_array operator (#14638)\n* add an operator for computing the likelihood of a Hawkes self-exciting process (#14683)\n* Add numpy linspace (#14927)\n\n\n### Feature Improvements\n\n#### Operators\n* make ROIAlign support position-sensitive pooling (#13088)\n* Add erfinv operator for calculating inverse error function (#13811)\n* Added optional parameters to BilinearResize2D to do relative scaling (#13985)\n* MXNET-1295 Adding integer index support to Sequence* family of operators. (#13880)\n* Export resize and support batch size (#14014)\n* CUDNN dropout (#13896)\n* Relaxing type requirements for slice_like op (#14097)\n* Relaxing type requirements for reshape_like op (#14325)\n* Parallelize CPU version and add GPU version of boolean_mask op (#14090)\n* Add NHWC layout support to Pooling (cpu, gpu cuda, gpu cuDNN) (#13749)\n* Multi-precision AdamW update op (#14171)\n* [op] add back support for scalar type rescale_grad argument for adamw_update/mp_adamw_update (#14221)\n* move choose_element_0index to operator (#14273)\n* Optimize NMS (#14290)\n* Optimize NMS part 2 (#14352)\n* add background class in box_nms (#14058)\n* Use cudnn for dropout by default (#14278)\n* In-place updates for Nadam, Adadelta, Adamax and SGLD (#13960)\n* Aggregate SGD (#13346)\n* Add proper exception message for negative shape in array creation routines (#14362)\n* Support multi-threading for Custom Operator (#14363)\n* moveaxis operator now accepts negative indices and sequence of ints as well. (#14321)\n* Support SyncBatchNorm5D (#14542)\n* Add nd.power and sym.pow (#14606)\n* Change RNN OP to stateful (#14476)\n* Add imresize and copyMakeBorder to mx.image (#13357)\n* add ctx for rand_ndarray and rand_sparse_ndarray (#14966)\n* Add cpu implementation for Deformable PSROIPooling (#14886)\n* Add warning for fp16 inputs with MXNET_SAFE_ACCUMULATION=0 (#15046)\n* Safe LayerNorm (#15002)\n* use MXNET_SAFE_ACCUMULATION for softmax accumulator (#15037)\n* LayerNorm acceleration on GPU  (#14935)\n* Add matrix inversion operator in linalg (#14963)\n* implementation for equivalence of tf.moments (#14842)\n* Use env var to enforce safe accumulation in ReduceAxesCompute (#14830)\n* [MXNet-1211] Factor and \"Like\" modes in BilinearResize2D operator (#13226)\n* added extraction/generation of diagonal and triangonal matrices to linalg (#14501)\n* [Mxnet-1397] Support symbolic api for requantize and dequantize (#14749)\n* [MXNET-978] Support higher order gradient for `log`. (#14992)\n* Add cpu implementation for Deformable Convolution (#14879)\n\n#### MKLDNN\n* Feature/mkldnn static (#13628)\n* Feature/mkldnn static 2 (#13503)\n* support mkl log when dtype is fp32 or fp64 (#13150)\n* Add reshape op supported by MKL-DNN (#12980)\n* Move the debug output message into MXNET_MKLDNN_DEBUG (#13662)\n* Integrate MKLDNN Conv1d and support 3d layout (#13530)\n* Making MKL-DNN default on MXNet master (#13681)\n* Add mkldnn OP for slice (#13730)\n* mkldnn s8 conv API change for master (#13903)\n* [MKLDNN] Enable signed int8 support for convolution. (#13697)\n* add mkldnn softmax_output (#13699)\n* MKLDNN based Quantized FullyConnected Operator and its fusion (#14128)\n* Fix entropy for uint8 (#14150)\n* Update MKL-DNN to v0.18 release (was: fix the Dense layer issue) (#13668)\n* [MKL-DNN] Enable s8 support for inner product and 3d input with flatten=false (#14466)\n* Optimize transpose operator with MKL-DNN (#14545)\n* [MKLDNN] Remove repeat parts in MKLDNN.md (#14995)\n* [MKLDNN] Enable more convolution + activation fusion (#14819)\n* Update MKL-DNN submodule to v0.19 (#14783)\n* Add mkldnn_version.h to pip package (#14899)\n* [MKLDNN] add quantized sum (#14614)\n* [MKLDNN]Refactor requantize to speed up execution (#14608)\n* [MKLDNN]Add quantized relu (#14604)\n* Add MKLDNN headers to pip package (#14339)\n* add symbolic link to mkldnn header files in include (#14300)\n* disable default MKLDNN for cross compilation (#13893)\n* Update MKLDNN_README.md (#13653)\n* [Quantization] Support zero-size tensor input for quantization flow (#15031)\n* Support 3D input for MKL-DNN softmax operator (#14818)\n* Add primitive cache for MKL-DNN sum(elemwise_add operator (#14914)\n* Fix reshape to add in-place back (#14903)\n* [int8] Add MobileNetV2_1.0 & ResNet18 Quantization (#14823)\n* [MKLDNN]Improve quantizeV2 and dequantize latency (#14641)\n* added mkldnn dependency for plugin compile target (#14274)\n* Support Quantized Fully Connected by INT8 GEMM (#12922)\n\n#### ONNX\n* ONNX export: Instance normalization, Shape (#12920)\n* ONNX export: Logical operators (#12852)\n* ONNX import/export: Size (#13112)\n* ONNX export: Add Flatten before Gemm (#13356)\n* ONNX import/export: Add missing tests, ONNX export: LogSoftMax (#13654)\n* ONNX import: Hardmax (#13717)\n* [MXNET-898] ONNX import/export: Sample_multinomial, ONNX export: GlobalLpPool, LpPool (#13500)\n* ONNX ops: norm exported and lpnormalization imported (#13806)\n* [MXNET-880] ONNX export: Random uniform, Random normal, MaxRoiPool (#13676)\n* ONNX export: Add Crop, Deconvolution and fix the default stride of Pooling to 1 (#12399)\n* onnx export ops (#13821)\n* ONNX export: broadcast_to, tile ops (#13981)\n* ONNX export: Support equal length splits (#14121)\n\n#### TensorRT\n* [MXNET-1252][1 of 2] Decouple NNVM to ONNX from NNVM to TenosrRT conversion (#13659)\n* [MXNET-703] Update to TensorRT 5, ONNX IR 3. Fix inference bugs. (#13310)\n* [MXNET-703] Minor refactor of TensorRT code (#13311)\n* reformat trt to use subgraph API, add fp16 support (#14040)\n\n#### FP16 Support\n* Update mshadow to support batch_dot with fp16. (#13716)\n* float32 → float16 cast consistency across implementations (#13857)\n* modifying SyncBN doc for FP16 use case (#14041)\n* support dot(vector, vector) for fp16 inputs on GPU (#14102)\n* softmax for fp16 with fp32 accumulator (#14098)\n* [MXNET-1327] Allow RNN Layers to be initialized to fp16 (#14219)\n* fp16 safe norm operator (#14616)\n* NAG Optimizer with multi-precision support (#14568)\n\n#### Deep Graph Library(DGL) support\n* Add graph_compact operator. (#13436)\n* Accelerate DGL csr neighbor sampling (#13588)\n\n#### Horovod Integration\n* Add extra header file to export for error checking (#13795)\n* whitelist symbols for using MXNet error handling externally (#13812)\n* Use CPUPinned context in ImageRecordIOParser2 (#13980)\n* Add pin_device_id option to Gluon DataLoader (#14136)\n\n#### Dynamic Shape\n* [MXNET-1315] Add checks for dynamic-shaped operators in CachedOp (#14018)\n* [MXNET-1325] Make InferShapeAttr a standalone pass (#14193)\n* [MXNET-1324] Add NaiveRunGraph to imperative utils (#14192)\n* [MXNET-1352] Allow dynamic shape in while_loop and if conditionals (#14393)\n\n#### Backend Engine\n* Add infer_type_partial (#14214)\n* Tidy up storage allocation and deallocation (#14480)\n* Add MXEnginePushAsync and MXEnginePushSync C APIs (#14615)\n* Enhance subgraph API (#14113)\n* Enhance PartitionGraph (#14277)\n* Allow clearing gpu cache (#14252)\n* Fix warning / static function in header. (#14900)\n* Simplify creation of NodeEntry instances and use emplace_back (#14095)\n* Add unpooled gpu memory type (#14716)\n* [MXNET-1398] Enable zero-copy from numpy to MXNet NDArray (#14733)\n* Use DEFAULT macro in C APIs (#14767)\n* Avoid unnecessary vector copies in imperative_utils.cc (#14665)\n* Support populating errors back to MXNet engine in callback (#13922)\n* Restore save/load ndarray to 1.4.1 (#15073)\n* Enable serializing/deserializing ndarrays in np_shape semantics (#15090)\n* [numpy] Support zero-dim and zero-size tensors in MXNet (#14661)\n* Rename np_compat to np_shape (#15063)\n* [MXNET-1330] Bring nnvm::Tuple to mxnet::Tuple (#14270)\n\n#### Large Tensor Support\n* Large array support for randint (#14242)\n* [MXNET-1185] Support large array in several operators (part 1) (#13418)\n* [MXNET-1401] adding more operators to test support for Large Tensor (#14944)\n* [MXNET-1410]Adding Large Tensor Support for tensor transpose (#15059)\n\n#### Quantization\n* Exclude concat layer for gpu quantization (#14060)\n* Enhance gpu quantization (#14094)\n* Register fake grad to subgraph and quantized operators (#14275)\n* Add int8 data loader (#14123)\n\n#### Profiler\n* [MXNET-857] Add initial NVTX profiler implementation (#12328)\n\n#### CoreML\n* Add more support for mxnet_to_coreml (#14222)\n\n\n### Front End API\n\n#### Gluon\n* Add pixelshuffle layers (#13571)\n* [MXNET-766] add dynamic_unroll RNN for HybridBlock (#11948)\n* add pos_weight for SigmoidBinaryCrossEntropyLoss (#13612)\n* Rewrite dataloader with process pool, improves responsiveness and reliability (#13447)\n* Complimentary gluon DataLoader improvements (#13606)\n* [Fit-API] Adress PR comments (#14885)\n* [Fit API] update estimator (#14849)\n* [MXNET-1396][Fit-API] Update default handler logic (#14765)\n* [Fit API] improve event handlers (#14685)\n* move to gluon contrib (#14635)\n* move estimator to contrib (#14633)\n* [MXNet-1340][Fit API]Update train stats (#14494)\n* [MXNet-1334][Fit API]base class for estimator and eventhandler (#14346)\n* [MXNET-1333] Estimator and Fit API (#14629)\n* Add support for fast variable-length LSTM (#14208)\n* Add the Gluon Implementation of Deformable Convolution (#14810)\n* hybridize rnn and add model graph (#13244)\n\n#### Python\n* Python BucketingModule bind() with grad_req = 'add' (#13984)\n* Refine runtime feature discovery python API and add documentation to ... (#14130)\n* Runtime feature detection (#13549)\n* Add dtype visualization to plot_network (#14066)\n* [MXNET-1359] Adds a multiclass-MCC metric derived from Pearson (#14461)\n* support long for mx.random.seed (#14314)\n* Optimization of metric evaluation (#13471)\n* [MXNET-1403] Disable numpy's writability of NDArray once it is zero-copied to MXNet (#14948)\n* Refactor ImageRecordIter (#14824)\n\n\n### Language Bindings\n\n#### Scala\n* [MXNET-1260] Float64 DType computation support in Scala/Java (#13678)\n* [MXNET-1000] get Ndarray real value and form it from a NDArray (#12690)\n* Now passing DType of Label downstream to Label's DataDesc object (#14038)\n* Scala interpreter instructions (#14169)\n* Add default parameters for Scala NDArray.arange (#13816)\n* [MXNET-1287] Up scala comp (#14667)\n* [MXNET-1385] Improved Scala Init and Macros warning messages (#14656)\n* Remove all usages of makefile for scala (#14013)\n* Update scala-package gitignore configuration. (#13962)\n* [MXNET-1177]Adding Scala Demo to be run as a part of Nightly CI (#13823)\n* [MXNET-1287] Miscellaneous Scala warning fixes (#14658)\n* Fix jar path and add missing ones for spark jobs (#14020)\n* [MXNET-1155] Add scala packageTest utility (#13046)\n* [MXNET-1195] Cleanup Scala README file (#13582)\n* Add scalaclean to make clean (#14322)\n* Add maven wraper to scala project. (#13702)\n* Add new Maven build for Scala package (#13819)\n* [MXNET-1287] Feat dep (#14668)\n* add Apache header on all XML (#14138)\n* update the version name (#14076)\n* change to compile time (#13835)\n* [MXNET-918] Random module (#13039)\n* Avoid secondary deployment of package to local (#14647)\n\n#### Java\n* [MXNET-1180] Java Image API (#13807)\n* [MXNET-1285] Draw bounding box with Scala/Java Image API (#14474)\n* Add BERT QA Scala/Java example (#14592)\n* [MXNET-1232] fix demo and add Eclipse support (#13979)\n* [MXNET-1331] Removal of non-MXNET classes from JAR (#14303)\n* Java install info update (#13912)\n* [MXNET-1226] add Docs update for MXNet Java (#14395)\n* [MXNET-1383] Java new use of ParamObject (#14645)\n* MXNET-1302 Exclude commons-codec and commons-io from assembled JAR (#14000)\n\n#### C++\n* print error message for mxnet::cpp::Operator::Invoke when failed (#14318)\n* build docs with CPP package (#13983)\n* Update inception_inference.cpp (#14674)\n* Optimize C++ API (#13496)\n\n#### Clojure\n* [Clojure] - Add Spec Validations to the Optimizer namespace (#13499)\n* [Clojure] Add Spec Validations for the Random namespace (#13523)\n* [Clojure] Correct the versions in the README so they correspond to the latest maven.org release ([#13507)\n* Port of scala infer package to clojure (#13595)\n* Clojure example for fixed label-width captcha recognition (#13769)\n* Update project.clj file to use the snapshots repo to be able to pull (#13935)\n* [Clojure] Add resource scope to clojure package (#13993)\n* [clojure-package] improve docstrings in image.clj (#14307)\n* [Clojure] Helper function for n-dim vector to ndarray (#14305)\n* [clojure]: add comp-metric based on CompositeEvalMetric (#14553)\n* [Clojure] enhance draw bounding box (#14567)\n* [Clojure] Add methods based on NDArrayAPI/SymbolAPI (#14195)\n* [Clojure] Clojure BERT QA example (#14691)\n* [clojure-package][wip] add ->nd-vec function in ndarray.clj (#14308)\n* [Clojure] Correct the versions in the README so they correspond to the latest maven.org release (#13507)\n* Update version to v1.5.0 including clojure package (#13566)\n* [clojure][generator] ndarray/symbol api random merged (#14800)\n* upgrade codox to work with lein 2.9.0 (#14133)\n* [clojure] fix: image test does not rely on s3 to run (#15122)\n\n#### Julia\n* Julia v0.7/1.0 support and drop v0.6 support (#12845)\n* Julia: split ndarray.jl into several snippets (#14001)\n* Julia: split symbolic-node.jl into several snippets (#14024)\n* Julia: rename mx.clip to clamp for NDArray (#14027)\n* Julia: add binding for runtime feature detection (#13992)\n\n#### Perl:\n* Two more gluon loss classes. (#14194)\n\n#### R\n* add NAG optimizer to r api (#14023)\n* R-Package Makefile (#14068)\n\n\n### Performance Improvements\n\n* Less cudaGet/SetDevice calls in Gluon execution (#13764)\n* Improve bulking in Gluon (#13890)\n* Increase perfomance of BulkAppend and BulkFlush (#14067)\n* Performance improvement in ToTensor GPU Kernel (#14099)\n* Performance improvement in Normalize GPU Kernel (#14139)\n* Bulked op segments to allow Variable nodes (#14200)\n* Performance improving for MKL-DNN Quantized FullyConnected (#14528)\n* speedup SequenceMask on GPU (#14445)\n* Dual stream cudnn Convolution backward() with MXNET_GPU_WORKER_NSTREAMS=2. (#14006)\n* Speedup `_contrib_index_copy` (#14359)\n* use mkl sparse matrix to improve performance (#14492)\n* Re-enable static cached_op optimization (#14931)\n* Speed up SequenceReverse (#14627)\n* Improve FC perf when no_bias=False (#15033)\n* Improve cached_op performance for static mode (#14785)\n\n\n### Example and Tutorials\n\n* [MXNET-949] Module API to Gluon API tutorial (#12542)\n* Support SSD f32/int8 evaluation on COCO dataset (#14646)\n* [MXNET-1209] Tutorial transpose reshape  (#13208)\n* [Clojure] Add Fine Tuning Sentence Pair Classification BERT Example (#14769)\n* example/ssd/evaluate/eval_metric.py (#14561)\n* Add examples of running MXNet with Horovod (#14286)\n* Added link to landing page for Java examples (#14481)\n* Update lip reading example (#13647)\n* [MXNET-1121] Example to demonstrate the inference workflow using RNN (#13680)\n* [MXNET-1301] Remove the unnecessary WaitAll statements from inception_inference example (#13972)\n* Modifying clojure CNN text classification example (#13865)\n* [MXNET-1210 ] Gluon Audio - Example (#13325)\n* add examples and fix the dependency problem (#13620)\n* add quantization example to readme (#14186)\n* Add an inference script providing both accuracy and benchmark result for original wide_n_deep example (#13895)\n* Update autoencoder example (#12933)\n*  #13813 examples with opencv4/origami (#13813)\n* [MXNET-1083] Add the example to demonstrate the inference workflow using C++ API (#13294)\n* Add tutorial on how to use build from source jar (#14197)\n* Gluon end to end tutorial (#13411)\n* Update MXNetTutorialTemplate.ipynb (#13568)\n* Simplifications and some fun stuff for the MNIST Gluon tutorial (#13094)\n* Clarify dependency on OpenCV in CNN Visualization tutorial. (#13495)\n* Update row_sparse tutorial (#13414)\n* add clojure tutorials to index (#14814)\n* Update lstm_crf.py (#14865)\n\n\n### Website\n\n* Version switching user experience improvements (#13921)\n* fix toctree Sphinx errors (#13489)\n* fix link (#15036)\n* fix website build (#14148)\n* Fixed mailing list addresses (#13766)\n* website publish updates (#14015)\n* use relative links; update links (#13741)\n* update social media section (#13705)\n* [MXNET] Updated http://data.dmlc.ml/ links to http://data.mxnet.io/ (#15065)\n\n### Documentation\n* [MXNET-1402] MXNet docs change for 1.4.1 release (#14949)\n* Add API documentation for upsampling operator with examples (#14919)\n* Make docblocks for Gluon BatchNorm and SyncBatchNorm consistent with the code (#14840)\n* [DOC] Update ubuntu install instructions from source (#14534)\n* [Clojure] Better api docstrings by replacing newlines (#14752)\n* Fix documentation for bilinear upsampling and add unit test (#14035)\n* Updated docs for R-package installation (#14269)\n* [docstring] improve docstring and indentation in `module.clj` (#14705)\n* The folder python-howto was removed in an earlier commit. The reference to that folder was not removed. Making a PR to remove the reference to this folder to keep documents consistent (#14573)\n* Updated documentation about nightly tests (#14493)\n* [Doc] Start the tutorials for MKL-DNN backend (#14202)\n* [DOC] fix sym.arange doc (#14237)\n* fix render issue in NDArray linalg docs (#14258)\n* [clojure-package] fix docstrings in `normal.clj` (#14295)\n* [DOC] Refine documentation of runtime feature detection (#14238)\n* [MXNET-1178] updating scala docs (#14070)\n* Fix website scala doc (#14065)\n*  Return value docs for nd.random.* and sym.random.* (#13994)\n* Fixing the doc for symbolic version of rand_zipfian (#13978)\n* fix doc of take operator (#13947)\n* beta doc fixes (#13860)\n* [MXNET-1255] update hybridize documentation (#13597)\n* Update Adam optimizer documentation (#13754)\n* local docs build feature (#13682)\n* gluon docfix (#13631)\n* Added javadocs and improved example instructions (#13711)\n* [MXNET-1164] Generate the document for cpp-package using Doxygen (#12977)\n* Fix warning in waitall doc (#13618)\n* Updated docs for randint operator (#13541)\n* Update java setup docs for 1.4.0 (#13536)\n* clarify ops faq regarding docs strings (#13492)\n* [MXNET-1158] JVM Memory Management Documentation (#13105)\n* Fixing a 404 in the ubuntu setup doc (#13542)\n* Fix READMEs for examples (#14179)\n* [Doc] Add MKL-DNN operator list (#14891)\n* Fixed some typos in AvgPooling Docs (#14324)\n* doc fix (#13465)\n* Change Straight Dope to Dive into Deep Learning (#14465)\n* [DEV] update code owner (#14862)\n* Add notes about debug with libstdc++ symbols (#13533)\n* Mention additional language bindings and add links (#14798)\n* add contributors from intel (#14455)\n* what's new - add 1.4.0 release (#14435)\n* added note about cuda9.2 requirement (#14140)\n* Remove unnecessary \"also\" in README.md (#14543)\n* Updated news.md with the latest mkldnn submodule version (#14298)\n* add new cloud providers to install page (#14039)\n* Update NOTICE (#14043)\n* Update README.md (#13973)\n* Update profiler doc (#13901)\n* Add CODEOWNERS for Julia package (#13872)\n* update code owner (#13737)\n* Update git clone location to apache github (#13706)\n* NEWS.md backport from v1.4.x to master (#13693)\n* Update CODEOWNERS, add Pedro Larroy. (#13579)\n* [MXNET-1225] Always use config.mk in make install instructions (#13364)\n* Docs & website sphinx errors squished 🌦  (#13488)\n* add Qing's Key to master (#14180)\n* add KEY for zachgk (#14965)\n* corrected a spellign (#14247)\n* 1.4 release (#14297)\n\n\n### Build and Test\n\n* Fix scala doc build break for v1.3.1 (#13820)\n* Adds additional CUDA build environments (#14909)\n* Pins version of scikit-learn for python2 due to drop in support (#14928)\n* upgrade the libpng to 1.6.35 (#14620)\n* Updates to cudnn package installation (#14923)\n* Improve order of execution of install scripts. (#14867)\n* Installs qemu pip requirements from qemu requirements file (#14355)\n* update raspberry pi install instructions (#14172)\n* update the scala installation tutorial on intellij (#14033)\n* Removes unneeded nvidia driver ppa installation (#13814)\n* script for installing gpu libraries and build tools (#13646)\n* Set install path for libmxnet.so dynamic lib on Mac OS (#13629)\n* compatibility with opencv4 (#14313)\n* Flaky test #14189 (#14190)\n* Enforce determinism for backwards compatibility checker (#14463)\n* Change CUB submodule to track Nvidia CUB project. (#13322)\n* Updates gpu tests to use CUDNN_VERSION supplied by the environment but default to 7.0.3 if not set (#14595)\n* upgrade the version to 2.0.2 (#14621)\n* [Dependency Update] Upgrade the libtiff to 4.0.10 (#14623)\n* [Dependency Update] Upgrade cuDNN & NCCL (#14884)\n* [Dependency Update] Upgrade openssl to 1.1.1b (#14837)\n* [Dependency Update] Upgrade CI to use latest cuDNN (#14950)\n* GPU RNN to use TempSpace resource for workspace. (#15056)\n* Add vim-nox to ci/docker/install/ubuntu_core.sh (#14632)\n* Fix dockerized GPU builds in dev_menu (#14603)\n* [MXNET-1093] Add python3 Docker images for each MXNet release (#12791)\n* increased docker shared memory (#14119)\n* Fix permissions of ci/docker/install/ubuntu_publish.sh (#13840)\n* Dockerfiles for Publish Testing (#13707)\n* Fix test randint (#14990)\n* Silence excessive mkldnn logging output on tests. (#14947)\n* Fix test memory with ResourceScope (#14666)\n* Sync Horovod distributed training examples with latest changes (#14748)\n* use mx.context.num_gpus instead of mx.test_utils.list_gpus in MF recommender example (#14926)\n* [MXNET-1400] adding tests cases to verify large tensor support for depth_to_space and space_to_depth (#14797)\n* rewrite test_custom_op_exc (#14878)\n* [Clojure] Remove unneeded test files (#14813)\n* Use correct stash name when running nightly tests (#14809)\n* julia/ndarray: fix flaky test cases for `clamp` (#14776)\n* Updates tolerances for test_layer_bidirectional (#14682)\n* Adds context parameter to check_rnn_layer_forward calls in test_lstmp (#14529)\n* reenable the test (#14483)\n* temporarily disable integ tests with a dependency on origami repo (#14448)\n* Bypass ThreadedEngine in test_operator_gpu.py:test_convolution_multiple_streams. (#14338)\n* Updated the MLP test to accept the number of epochs. Reduced the epochs in ci_test.sh to shorten the CI build time (#14149)\n* follow up on fix nightly test (#14134)\n* Julia: enable integration test (#14025)\n* fix test_depthwise_convoltuion for occasional CI failures (#14016)\n* fix test_stn (#14063)\n* Add a test for SGLD optimizer with comparisons for set noise seeds. (#13762)\n* Code modification for  testcases of various network models in directory example (#12498)\n* Remove MXNET_STORAGE_FALLBACK_LOG_VERBOSE from test_autograd.py (#13830)\n* [MXNET-1263] Unit Tests for Java Predictor and Object Detector APIs (#13794)\n* ONNX test code cleanup (#13553)\n*  #13385 [Clojure] - Turn examples into integration tests (#13554)\n* add cpp example inception to nightly test (#13534)\n* Fix flaky test test_random:test_randint_generator (#13498)\n* Adding test for softmaxoutput (#13116)\n* [MXNET-1235] Add a test for AdaMax optimizer (#13467)\n* [MXNET-545] Fix broken cython build (#10951)\n* Update mkldnn window build instructions in MKLDNN_README.md (#14952)\n* Added USE_SIGNAL_HANDLER to other Linux builds which didn't had it (#14122)\n* Static build for Python (#13916)\n* Julia: add windows-cpu build (#13937)\n* Static build instruction for MXNet in general (#13914)\n* Jenkins nightly maven with static build script and gpu (#13767)\n* Re-organize Scala maven build (#13626)\n* disable error checking when building old versions (#13725)\n* scripts for building libmxnet binary and wheel (#13648)\n* Improve dev_menu usability, local build and virtualenv (#13529)\n* Scripts for building dependency libraries of MXNet (#13282)\n* [MXNET-1224]: improve scala maven jni build and packing. (#13493)\n* fix compile error in debug mode (#13873)\n* add ccache to docs build (#13832)\n* Decreases test sensitivity (#15014)\n* bump up atol for test_bilinear_resize_op (#15011)\n* Add STL checks via -D_GLIBCXX_ASSERTIONS in debug mode (#14896)\n* clean up duplicate cudnn installation (#14996)\n* fix custom op fork test (#14753)\n* fix pi instructions (#14746)\n* Reenable TensorRT step (#14654)\n* Fixes for CI downloads (#14504)\n* Fixed tutorial warnings (#14472)\n* Fixes static build script for cub directory rename (#14578)\n* add a compiler flag to use int64 as tensor size (#14570)\n* Upgrade Pylint version to 2.3.1 (#14807)\n* Fixes installation nightly test by filtering out the git commands (#14144)\n* fix nightly test on tutorials (#14036)\n* Fix MXNet R package build (#13952)\n* re-enable test after issue fixed https://github.com/apache/mxnet/issues/10973 (#14032)\n* Add back R tests and fix typo around R and perl tests (#13940)\n* Fix document build (#13927)\n* Temporarily disables windows pipeline to unblock PRs (#14261)\n* Fix USE_ONEDNN check in Makefile (#13775)\n* Fix spelling in threaded_engine_test (#14709)\n* Fix cmake options parsing in dev_menu (#13458)\n* Add Local test stage and option to jump directly to menu item from commandline (#13809)\n* Add CPU test coverage and refine cmake builds (#13338)\n* ONNX test code cleanup - part 2 (#13738)\n* Rearrange tests written only for update_on_kvstore = True (#13514)\n* add batch norm test (#13625)\n* Adadelta optimizer test (#13443)\n* Skip flaky test https://github.com/apache/mxnet/issues/13446 (#13480)\n* Comment out test_unix_python3_tensorrt_gpu step (#14642)\n* Enable bulking test on windows (#14392)\n* rewrote the concat test to avoid flaky failures (#14049)\n* #13624 clojure nightly tests (#13624)\n* Temporarily disable website testing (#13887)\n* adding tolerance to flaky test (#13850)\n* Add publish test of PyPi cu100mkl (#14637)\n* CMake: Enable installation of cpp-package headers (#13339)\n* Use USE_SIGNAL_HANDLER by default set to ON in CMakeLists.txt (#14599)\n* Improve CMake handling of sse2 and sse3 (#14757)\n* Update base CUDA image for CI to v10.0 cuDNN 7.3.1 (#14513)\n* Updates build_lib.sh to copy the cub library license (#14347)\n* Add license check to dev_menu, docs build with docker (#14166)\n* Print reproduction command on CI failure (#14815)\n* change mxnet_option behavior (#14743)\n* [DEP] upgrade dmlc-core (#14510)\n* Use ubuntu_rat container for rat check (#14678)\n* Added repeats for github status updates (#14530)\n* add filter to warnings (#14532)\n* CI Changes for Codified Windows AMIs (#14336)\n* Refactors USE_NVRTC setting to ENABLE_CUDA_RTC in pip make config files (#14250)\n* pypi package description. manifest/setup.py update (#14255)\n* make rat-excludes compliant with apache release policy (#14142)\n* Add libhdf5-dev to ubuntu_core.sh (#14079)\n* Added logging to GitHub commit status publishing (#13615)\n* [CI] Prevent timeouts when rebuilding containers with docker. (#13818)\n* [MXNET-862] Basic maven jenkins pipeline (#13450)\n* Scope requests so it's not needed for dev_menu (#13771)\n* Add timeout/retry logic to docker cache download (#13573)\n* turn on Sphinx warnings as errors (#13544)\n* [MXNET-1251] Basic configuration to do static-linking (#13621)\n* Improve CCache handling (#13456)\n* build config for maven and pip (#13556)\n* Add Intel MKL blas to Jenkins (#13607)\n* Add workspace cleaning after job finished (#13490)\n* Add a retry to qemu_provision (#13551)\n* Deprecate Jenkinsfile (#13474)\n* [MXNET-1408] Adding test to verify Large Tensor Support for ravel and unravel (#15048)\n* move amp test and change op support to warning (#15085)\n* Fixes call to build ubuntu gpu in nightly tests (#14964)\n* rat check make target (#15127)\n* add epsilon for tolerance level (#15098)\n* Change mx.test_utils.list_gpus to mx.context.num_gpus where possible (#14946)\n* bump up cudnn to 7.5.1 & nccl 2.4.2 (#14988)\n* Disables TensorRT build step (#14958)\n* disable flaky integration test (#14151)\n* Disables large tensor size cpu test step (#14982)\n* Disable Flaky Test test_poisson_generator (#14540)\n* Disabled flaky test test_negative_binomial_generator (#13784)\n* Disabled flaky test test_gluon_data.test_recordimage_dataset_with_data_loader_multiworker (#13527)\n\n\n### Bug-fixes\n\n* Improve dev_menu virtualenv handling (#14788)\n* Fallback to dense version for grad(reshape), grad(expand_dims) (#13599)\n* Fix the bug of BidirectionalCell (#13575)\n* set _scale in Trainer using optimizer rescale_grad (#14593)\n* [MXNET-1379] update reshape operator (#14600)\n* Add repr for SymbolBlock (#14423)\n* Cudnn conv dgrad algo filtering (#14310)\n* Fix memory leak for size-zero ndarray (#14365)\n* Fixes the test_sgld (#14473)\n* Revert \"Fix memory leak for size-zero ndarray (#14365)\" (#14477)\n* fix custom operation in fork (#14451)\n* Fixes test_operator_gpu.test_multinomial_generator (#14475)\n* support leading dimension of -1 in ravel/unravel (#14356)\n* begin=end not a valid input (#14403)\n* Fix NaN value comparisons in relu, max and min ops (#14262)\n* fix engine crash in shutdown phase (#14382)\n* fix OOM error during resource allocation (#14444)\n* Fix relative difference scala (#14417)\n* Correct update count with Gluon trainer and update_on_kvstore=False (#14377)\n* Fix crashes on visualization (#14425)\n* Reorder module import orders for dist-kvstore (#13742)\n* Fixes for trainer with update_on_kvstore=False (#13721)\n* Fix errors in docstrings for subgraph op; use code directive (#13463)\n* Add resiliency to onnx export code (#13426)\n* update github location for sampled_block.py (#13508)\n* Revert \"Manually track num_max_thread (#12380)\" (#13501)\n* Revert \"Feature/mkldnn static 2 (#13503)\" (#13540)\n* [MXNET-1110] Add header files required by horovod (#13062)\n* [MXAPPS-1020] Clean up some Sphinx warnings. (#13539)\n* [MXNET-1249] Fix Object Detector Performance with GPU (#13522)\n* [MXNET-769] Use MXNET_HOME in a tempdir in windows to prevent access denied due t… (#13531)\n* Chi_square_check for discrete distribution fix (#13543)\n* Fix use-before-assignment in convert_dot (#13511)\n* fix the situation where idx didn't align with rec (#13550)\n* fix link for gluon model zoo (#13583)\n* Fix exception handling api doc (#13519)\n* [MXNET-1253] fix control_flow_op (#13555)\n* fix the Float not showing correctly problem (#13617)\n* fix quantize pass error when the quantization supported Op are excluded in the model (#13596)\n* Fix for import mxnet taking long time if multiple process launched (#13602)\n* Revert \"Feature/mkldnn static (#13628)\" (#13638)\n* updated reference to Apache MXNet (#13645)\n* Fix incorrect delete in MXExecutorReshape exception handling (#13376)\n* add build fix for Scala/Java build (#13655)\n* remove omp which can cause ssd accuracy variance (#13622)\n* Fix Jetson compilation (#13532)\n* Revert \"Fix Jetson compilation\" (#13665)\n* Fix Jetson compilation (#13666)\n* Revert \"Revert \"[MXNET-43] Fix Jetson compilation\" (#13665)\" (#13672)\n* fix unpicklable transform_first on windows (#13686)\n* Fix NDArray ToDLPack Bug (#13698)\n* Fix the quantization script to support Python2 (#13700)\n* Update basic_layers.py (#13732)\n* [MXNET-1231] Allow not using Some in the Scala operators (#13619)\n* [MXNET-244] Work around likely compiler bug on nested inlines and temporary acces… (#13535)\n* Use curl to download sample data instead of wget. (#13761)\n* fix bipartite match memory corruption (#13727)\n* remove attributes clear on TRT nodes for GetOptimizedSymbol (#13703)\n* fix redirection issues; set default version to master (#13796)\n* fix for params with no dims in onnx (#13413)\n* Remove semicolon in libmxnet.sym file (#13822)\n* remove useless code (#13777)\n* Fixing a symlink issue with R install (#13708)\n* fix minor indentation (#13827)\n* Fix Tree Reduction on new instance type p3dn.24xlarge (#13852)\n* [Clojure] package infer tweaks (#13864)\n* Fix cpp examples build on Mac. (#13826)\n* Fix launch bounds in spatial transformer (#13188)\n* Update example scripts classpath. (#13849)\n* fix ssd quantization script error (#13843)\n* Avoid adding SegfaultLogger if process already has sig handler. (#13842)\n* fix the fetching GPU problem (#13889)\n* Fix SN-GAN example doc (#13877)\n* update Spectral Normalization Code (#13868)\n* Fixed java benchmark failing error by fixing the classpath (#13891)\n* Fix the order of error term's operands (#13745)\n* fix bug in nag optimizer (#13683)\n* Fix BatchNorm converter for CoreML when fix_gamma=True (#13557)\n* Fix for test always returning true (#13911)\n* Add error checking for cpp examples. (#13828)\n* julia: fix `argmax` for NDArray (#13871)\n* test_ImageRecordIter_seed_augmentation flaky test fix (#12485)\n* Julia: fix filename quoting in docstring (#13894)\n* Flaky maven binary download (#13974)\n* [MXNET-1293] Adding Iterables instead of List to method signature for infer APIs in Java (#13977)\n* Sample python bilinear initializer at integral points in y-direction (#12983)\n* Fix inconsistent handling for FResourceRequestEx for imperative and symbolic executor (#14007)\n* [MXNET-1258] fix unittest for ROIAlign Operator (#13609)\n* Fix performance regression in normalize operator (#14055)\n* Remove inplace support for ToTensor operator (#14083)\n* Addresses comments in runtime feature discovery API (#13964)\n* The latest version of leiningen has a dependency problem with codox (#14132)\n* Fix quote on LBSGD docs (#13975)\n* Fixes spelling (#14168)\n* Fix broken amalgamation (#12792)\n* Fix nd.pick large array issue (#14082)\n* Fix req=null in SliceLikeBackward (#14209)\n* onnx broadcast ops fixes (#13604)\n* fix update params (#14218)\n* MXNet Java bug fixes and experience improvement (#14213)\n* reverting broadcasting fixes (#14299)\n* fix memory-related issues to enable ASAN tests (#14223)\n* FIX: flaky test exponential generator (#14287)\n* fix SoftmaxOutput resource bug (#14302)\n* Fix shape inference pass (#14153)\n* Limit workspace for cudnnGet results (#14326)\n* #14199: catch subprocess.CalledProcessError in get_gpus() (#14212)\n* Fixes #14181, validate model output shape for ObjectDetector. (#14215)\n* Optimizer MXKVStoreUpdater bug fix in serializeState method (#14337)\n* Add proper exception message for negative shape in array creation routines (#14362)\n* Fix NaN value comparisons in relu, max and min ops (#14262)\n* fix engine crash in shutdown phase (#14382)\n* Flaky test #14189 (#14190)\n* Correct update count with Gluon trainer and update_on_kvstore=False (#14377)\n* Fix relative difference scala (#14417)\n* fix OOM error during resource allocation (#14444)\n* Fix crashes on visualization (#14425)\n* begin=end not a valid input (#14403)\n* Fix memory leak for size-zero ndarray (#14365)\n* Fixes the test_sgld (#14473)\n* Revert \"Fix memory leak for size-zero ndarray (#14365)\" (#14477)\n* fix custom operation in fork (#14451)\n* Fixes test_operator_gpu.test_multinomial_generator (#14475)\n* Fix script retrieval (#14519)\n* Memory fixes. Resolves #10867, and resolves #14080 (#14372)\n* Chouffe/clojure fix tests (#14531)\n* [clojure][image] add draw-bounding-box interop (#14533)\n* fix tests (#14565)\n* Do not touch GPU 0 during ReleaseAll (#14550)\n* [MXNET-1357] Fix the cpp-examples to add exception handling (#14441)\n* fix build cpp examples option (#14562)\n* Fix flaky test poisson generator & test_negative_binomial_generator (#14571)\n* Fixing unintentional variable overloading (#14438)\n* fix quantize graph pass (#14605)\n* replace std::random_shuffle to std::shuffle (#14523)\n* Add exception handling support for waitall (#14397)\n* split_and_load can now handle num_ctx > num_data. Issue #13909 (#14607)\n* Fix aspect ratio sampling for RandomResizedCrop (#14585)\n* [MXNET-400] support string type for kvstore key in cpp-package (#10792)\n* Fix warning on macro expansion using defined. (#14598)\n* Fix scaladoc scalastyle violations in Infer package (#14671)\n* Fix profiler check (#14677)\n* Tweak the copy for the cudnn autotuning warning. (#14680)\n* Properly handling custom op exception by modify engine (#14693)\n* Disable USE_GPERFTOOLS (#14711)\n* Reference engine from chunk via weak pointer (#14591)\n* [C++] fix type inconsistent issue when loading quantized parameters (#15038)\n* Fix crash in random.shuffle operator (#15041)\n* [MXNET-1406] [BUG] Fix DLManagedTensor deleter (#15016)\n* Fixes lint issue in AMP (#15015)\n* Fixed issue where the estimator was printing beyond the dataset size ... (#14464)\n* Fixes cuDNN version for CUDA 9.0 build environment (#15001)\n* Fix the incorrect MKLDNN/MKL logic in cmake  (#14877)\n* Fixed and re-enables TensorRT steps (#14960)\n* Fix the return type of sparse.clip operator (#14856)\n* Fix sample_multinomial number of outputs bug (#14873)\n* [MXNET-13578] Fix cmake installation failed (#14692)\n* Fix iterator over symbol when multiple children have the same name (#14597)\n* Fixes for wine detection tutorial (#13886)\n* Scala/Java Predict API fix #14756 (#14804)\n* Fix GELU backward possible NaN (#14782)\n* fix shape index bug (#14518)\n* [BUGFIX] fix ELU function will appear nan when calculating the gradient (#14673)\n* Change size_t to int within for loop to fix windows build error (#14740)\n* [contrib][op] fix MultiBoxPrior confusing results if first ratio is not 1.0 (#13763)\n* Fix scalastyle (#14669)\n* fix Makefile (#14424)\n* [v1.4.x] Update MKL-DNN to fix the OSX build issue (#14141) (#14182)\n* add preprocessed data and pretrained model info; minor format/spelling fixes (#14170)\n* Fixes libjpeg-turbo dependency under Ubuntu 16.04 (#14127)\n* Fix website error pages (#13963)\n* fix Makefile for rpkg (#13590)\n* fix c complier to clang (#13778)\n* Fix #13521 (#13537)\n* [MXNET-1234] Fix shape inference problems in Activation backward (#13409)\n* Revert the change broadcast_to param shape (#14998)\n* Fix infer shape partial after unknown shape changed to -1 (#14869)\n* fix add_n bug: when input mem overlap with output mem, results is wrong (#14889)\n* [Bugfix] Fix layer norm for large input shape (#14870)\n* Fix Clojure BERT example's context argument (#14843)\n* fix min max on zero-sized ndarray (#14745)\n* fix acc_type_switch macro with extra tests (#14773)\n* fix bug in profiler tutorial when using cpu (#13695)\n* [MXNET-1291] solve pylint errors in examples with issue no.12205 (#13815)\n* data preparation file moved in example (#14781)\n* [MXNET-1291] solve pylint errors in examples with issue no.12205 (#13848)\n*  Prevent crashes for opencv exception and std::exception (#14433)\n* Set idx2name for Optimizer object (#14703)\n* Revert \"Bumped minor version from 1.4.0 to 1.5.0 on master, updated License file\" (#13558)\n* [BUGFIX] fix unknown parameter shapes when np_shape is turned on. (#15097)\n* Add gluonCV to fix AMP Tutorial (#15039)\n* fix the if condition for LayerNorm (#15094)\n* [MKLDNN]Fix mkldnn deconvolution forward with bias (#15088)\n* NER example: fix divisions by zero (#15068)\n* remove warning in tutorial: (#15135)\n* [MXNET-1291] solve pylint errors in examples with issue no.12205 (#13938)\n* Revert \"Improve cached_op performance for static mode (#14785)\" (#14868)\n* Fix mkldnn backend when using naive engine (#15089)\n* fix gluon rnn cell single step unroll (#15081)\n* Revert \"Improve FC perf when no_bias=False (#15033)\" (#15099)\n\n\n### License\n\n* Updates python setup.py for recent license changes (#14778)\n* [MXNET-1377] Add static-dependencies licenses (#14726)\n* add license (#13793)\n* License update  (#13565)\n* Bumped minor version from 1.4.0 to 1.5.0 on master, updated License file (#13478)\n* License Googletest and Appendix (#14687)\n* Add copyrights for third party licenses to license file (#13851)\n* Improve license_header tool by only traversing files under revision c… (#13803)\n* Update LICENSE File with subcomponents (#13808)\n\n### Depreciations\n\n* Julia: deprecate `mx.empty`, replace it with `UndefInitializer` (#13934)\n * Deprecate NDArrayCollector and instead use ResourceScope (#14780)\n\n### Known Issues\n* Amalgamation compile problems(#14808)\n* Dynamic Shape does not support reverse shape inference and deferred initialization. (#14983)\n* Disables flaky test_random_size_crop (#15019)\n* Disables flaky test_l2_normalization (#15006)\n* Disables flaky TestStochasticTiming_2D test (#14412)\n* Disables flaky test_operator.test_sgld test (#14410)\n* Disables test_bulking due to flakyness (#14971)\n* Disabled flaky test (#13758)\n* Disables flaky test_droupout (#15003)\n* Disables flaky test_operator_gpu.test_activation (#14969)\n\n\n## 1.4.1\n\nApache MXNet (incubating) 1.4.1 is a maintenance release incorporating important bug fixes and important performance improvements. All users of Apache MXNet (incubating) 1.4.0 are advised to upgrade. You can install Apache MXNet (incubating) 1.4.1 at the usual place. Please review these Release Notes to learn the bug fixes.\n\n### Bug-fixes\n* Java bug-fix cherry pick (#14834)\n* Use DEFAULT macro in C APIs (#14767) (#14789)\n* Set idx2name for Optimizer object (#14703) (#14772)\n* Add pin_device_id option to Gluon DataLoader (#14136) (#14771)\n* Tidy up storage allocation and deallocation (#14480) (#14768)\n* Add MXEnginePushAsync and MXEnginePushSync C APIs (#14615) (#14770)\n* Less cudaGet/SetDevice calls in Gluon execution (#13764)\n* Fix nightly build of 1.4.x (#14556)\n* Memory fixes. Resolves #10867, and resolves #14080 (#14372) (#14586)\n* Fixes for data links (#14526)\n* Backport of Windows CI Fixes (#14420)\n\n\n## 1.4.0\n\n- [New Features](#new-features-2)\n  * [Java Inference API](#java-inference-api)\n  * [Julia API](#julia-api)\n  * [Control Flow Operators (experimental)](#control-flow-operators-experimental)\n  * [SVRG Optimization](#svrg-optimization)\n  * [Subgraph API (experimental)](#subgraph-api-experimental)\n  * [JVM Memory Management](#jvm-memory-management)\n  * [Topology-aware AllReduce (experimental)](#topology-aware-allreduce-experimental)\n  * [MKLDNN backend: Graph optimization and Quantization (experimental)](#mkldnn-backend--graph-optimization-and-quantization-experimental)\n    + [Graph Optimization](#graph-optimization)\n    + [Quantization](#quantization)\n- [New Operators](#new-operators-3)\n- [Feature improvements](#feature-improvements-3)\n  * [Operator](#operator)\n  * [Optimizer](#optimizer)\n  * [Sparse](#sparse)\n  * [ONNX](#onnx)\n  * [MKLDNN](#mkldnn-2)\n  * [Inference](#inference)\n  * [Other](#other)\n- [Frontend API updates](#frontend-api-updates)\n  * [Gluon](#gluon-2)\n  * [Symbol](#symbol-1)\n- [Language API updates](#language-api-updates)\n  * [Java](#java)\n  * [R](#r)\n  * [Scala](#scala-2)\n  * [Clojure](#clojure-2)\n  * [Perl](#perl-2)\n  * [Julia](#julia-2)\n- [Performance benchmarks and improvements](#performance-benchmarks-and-improvements)\n- [Bug fixes](#bug-fixes-4)\n- [Licensing updates](#licensing-updates)\n- [Improvements](#improvements)\n  * [Tutorial](#tutorial)\n  * [Example](#example)\n  * [Documentation](#documentation)\n  * [Website](#website)\n  * [MXNet Distributions](#mxnet-distributions)\n  * [Installation](#installation)\n  * [Build and CI](#build-and-ci)\n  * [3rd party](#3rd-party)\n    + [TVM:](#tvm)\n    + [CUDNN:](#cudnn)\n    + [Horovod:](#horovod)\n- [Deprications](#deprications)\n- [Other](#other-1)\n- [How to build MXNet](#how-to-build-mxnet)\n- [List of submodules used by Apache MXNet (Incubating) and when they were updated last](#list-of-submodules-used-by-apache-mxnet--incubating--and-when-they-were-updated-last)\n### New Features\n#### Java Inference API\n\nModel inference is often managed in a production ecosystem using primarily Java/Scala tools and frameworks. This release seeks to alleviate the need for software engineers to write custom MXNet wrappers to fit their production environment.\n\nInference on a trained model has a couple of common use cases:\n\n  1. Real-time or Online Inference - tasks that require immediate feedback, such as fraud detection\n  2. Batch or Offline Inference - tasks that don't require immediate feedback, these are use cases where you have massive amounts of data and want to run inference or pre-compute inference results\nReal-time Inference is often performed and deployed on popular web frameworks such as Tomcat, Netty, Jetty, etc., all of which use Java.\nBatch Inference is often performed on big data platforms such as Spark using Scala or Java.\n\nWith this project, we had the following goals:\n* Build a new set of APIs that are Java friendly, compatible with Java 7+, are easy to use for inference.\n* Lower the barrier to entry of consuming MXNet for production use cases.\n\nMore details can be found at the [Java Inference API document](https://cwiki.apache.org/confluence/display/MXNET/MXNet+Java+Inference+API).\n\n#### Julia API\n\nMXNet.jl is the Julia package of Apache MXNet. MXNet.jl brings flexible and efficient GPU computing and state-of-art deep learning to Julia. Some highlights of features include:\n\n  * Efficient tensor/matrix computation across multiple devices, including multiple CPUs, GPUs and distributed server nodes.\n  * Flexible manipulation of symbolic to composite for construction of state-of-the-art deep learning models.\n\n#### Control Flow Operators (experimental)\n\nToday we observe more and more dynamic neural network models, especially in the fields of natural language processing and graph analysis. The dynamics in these models come from multiple sources, including:\n\n  * Models are expressed with control flow, such as conditions and loops;\n  * NDArrays in a model may have dynamic shapes, meaning the NDArrays of a model or some of the NDArrays have different shapes for different batches;\n  * Models may want to use more dynamic data structures, such as lists or dictionaries.\nIt's natural to express dynamic models in frameworks with an imperative programming interface (e.g., Gluon, Pytorch, TensorFlow Eager). In this kind of interface, developers can use Python control flows, or NDArrays with any shape at any moment, or use Python lists and dictionaries to store data as they want. The problem of this approach is that it highly dependent on the originating front-end programming language (mainly Python). A model implemented in one language can only run in the same language.\n\nA common use case is that machine learning scientists want to develop their models in Python, whereas engineers who deploy the models usually have to use a different \"production\" language (e.g., Java or C). Gluon tries to close the gap between the model development and production deployment. Machine learning scientists design and implement their models in Python with the imperative interface, and then Gluon converts the implementations from imperative to symbolic by invoking `hybridize()` for model exporting.\n\nThe goal of this project is to enhance Gluon to turn a dynamic neural network into a static computation graph. The dynamic control flows are expressed by control flow operators with Gluon hybridization, and these are exported for deployment.\n\nMore information can be found at [Optimize dynamic neural network models with control flow operators](https://cwiki.apache.org/confluence/display/MXNET/Optimize+dynamic+neural+network+models+with+control+flow+operators)\n\n#### SVRG Optimization\n\nSVRG stands for Stochastic Variance Reduced Gradient, which was first introduced in the paper [Accelerating Stochastic Gradient Descent using Predicative Variance Reduction in 2013](https://papers.nips.cc/paper/4937-accelerating-stochastic-gradient-descent-using-predictive-variance-reduction.pdf). It is an optimization technique that complements SGD.\n\nSGD is known for large scale optimization, but it suffers from slow convergence asymptotically due to the inherent variance. SGD approximates the full gradient using a small batch of samples which introduces variance. In order to converge faster, SGD often needs to start with a smaller learning rate.\n\nSVRG remedies the slow convergence problem by keeping a version of the estimated weights that is close to the optimal parameters and maintains the average of the full gradient over the full pass of data. The average of the full gradients of all data is calculated w.r.t to parameters of last mth epochs. It has provable guarantees for strongly convex smooth functions; a detailed proof can be found in section 3 of the [paper](https://papers.nips.cc/paper/4937-accelerating-stochastic-gradient-descent-using-predictive-variance-reduction.pdf). SVRG uses a different update rule than SGD: gradients w.r.t current parameters minus gradients w.r.t parameters from the last mth epoch, plus the average of gradients over all data.\n\nKey Characteristics of SVRG:\n\n  * Explicit variance reduction\n  * Ability to use relatively large learning rate compared to SGD, which leads to faster convergence.\nMore details can be found at [SVRG Optimization in MXNet Python Module](https://cwiki.apache.org/confluence/display/MXNET/Unified+integration+with+external+backend+libraries)\n\n#### Subgraph API (experimental)\n\nMXNet can integrate with many different kinds of backend libraries, including TVM, MKLDNN, TensorRT, Intel nGraph and more. In general, these backends support a limited number of operators, so running computation in a model usually involves an interaction between backend-supported operators and MXNet operators. These backend libraries share some common requirements:\n\nTVM , MKLDNN and nGraph use customized data formats. Interaction between these backends with MXNet requires data format conversion.\nTVM, MKLDNN, TensorRT and nGraph fuses operators.\nIntegration with these backends should happen in the granularity of subgraphs instead of in the granularity of operators. To fuse operators, it's obvious that we need to divide a graph into subgraphs so that the operators in a subgraph can be fused into a single operator. To handle customized data formats, we should partition a computation graph into subgraphs as well. Each subgraph contains only TVM, MKLDNN or nGraph operators. In this way, MXNet converts data formats only when entering such a subgraph, and the operators inside a subgraph handle format conversion themselves if necessary. This makes interaction of TVM and MKLDNN with MXNet much easier. Neither the MXNet executor nor the MXNet operators need to deal with customized data formats. Even though invoking these libraries from MXNet requires similar steps, the partitioning rule and the subgraph execution of these backends can be different. As such, we define the following interface for backends to customize graph partitioning and subgraph execution inside an operator. More details can be found at PR 12157 and [Subgraph API](https://cwiki.apache.org/confluence/display/MXNET/Unified+integration+with+external+backend+libraries).\n\n#### JVM Memory Management\n\nThe MXNet Scala and Java API uses native memory to manage NDArray, Symbol, Executor, DataIterators using MXNet's internal C APIs.  The C APIs provide appropriate interfaces to create, access and free these objects. MXNet Scala has corresponding Wrappers and APIs that have pointer references to the native memory. Before this project, JVM users (e.g. Scala, Clojure, or Java) of MXNet have to manage MXNet objects manually using the dispose pattern. There are a few usability problems with this approach:\n\n* Users have to track the MXNet objects manually and remember to call `dispose`. This is not Java idiomatic and not user friendly. Quoting a user: \"this feels like I am writing C++ code which I stopped ages ago\".\n* Leads to memory leaks if `dispose` is not called.\n* Many objects in MXNet-Scala are managed in native memory, needing to use `dispose` on them as well.\n* Bloated code with `dispose()` methods.\n* Hard to debug memory-leaks.\nGoals of the project are:\n* Provide MXNet JVM users automated memory management that can release native memory when there are no references to JVM objects.\n* Provide automated memory management for both GPU and CPU memory without performance degradation.  More details can be found here: [JVM Memory Management](https://cwiki.apache.org/confluence/display/MXNET/JVM+Memory+Management)\n\n#### Topology-aware AllReduce (experimental)\nFor distributed training, the `Reduce` communication patterns used by NCCL and MXNet are not optimal for small batch sizes. The `Topology-aware AllReduce` approach is based on the idea of using trees to perform the `Reduce` and `Broadcast` operations. We can use the idea of minimum spanning trees to do a binary tree `Reduce` communication pattern to improve distributed training following this paper by Wang, Li, Edo and Smola [1]. Our strategy is to use:\n\n  * a single tree (latency-optimal for small messages) to handle `Reduce` on small messages\n  * multiple trees (bandwidth-optimal for large messages) to handle `Reduce` on large messages\n\nMore details can be found here: [Topology-aware AllReduce](https://cwiki.apache.org/confluence/display/MXNET/Single+machine+All+Reduce+Topology-aware+Communication)\nNote: This is an experimental feature and has known problems - see [13341](https://github.com/apache/mxnet/issues/13341). Please help to contribute to improve the robustness of the feature.\n\n#### MKLDNN backend: Graph optimization and Quantization (experimental)\n\nTwo advanced features, graph optimization (operator fusion) and reduced-precision (INT8) computation, are introduced to MKLDNN backend in this release ([#12530](https://github.com/apache/mxnet/pull/12530), [#13297](https://github.com/apache/mxnet/pull/13297), [#13260](https://github.com/apache/mxnet/pull/13260)).\nThese features significantly boost the inference performance on CPU (up to 4X) for a broad range of deep learning topologies. Currently, this feature is only available for inference on platforms with [supported Intel CPUs](https://github.com/intel/mkl-dnn#system-requirements).\n\n##### Graph Optimization\nMKLDNN backend takes advantage of MXNet subgraph to implement the most of possible operator fusions for inference, such as Convolution + ReLU, Batch Normalization folding, etc. When using mxnet-mkl package, users can easily enable this feature by setting export MXNET_SUBGRAPH_BACKEND=MKLDNN.\n\n##### Quantization\nPerformance of reduced-precision (INT8) computation is also dramatically improved after the graph optimization feature is applied on CPU Platforms. Various models are supported and can benefit from reduced-precision computation, including symbolic models, Gluon models and even custom models. Users can run most of the pre-trained models with only a few lines of commands and a new quantization script imagenet_gen_qsym_mkldnn.py. The observed accuracy loss is less than 0.5% for popular CNN networks, like ResNet-50, Inception-BN, MobileNet, etc.\n\nPlease find detailed information and performance/accuracy numbers here: [MKLDNN README](https://mxnet.apache.org/api/python/docs/tutorials/performance/backend/mkldnn/mkldnn_readme.html), [quantization README](https://github.com/apache/mxnet/tree/master/example/quantization#1) and [design proposal](https://cwiki.apache.org/confluence/display/MXNET/MXNet+Graph+Optimization+and+Quantization+based+on+subgraph+and+MKL-DNN)\n\n### New Operators\n\n* Add trigonometric operators (#12424)\n* [MXNET-807] Support integer label type in ctc_loss operator (#12468)\n* [MXNET-876] make CachedOp a normal operator (#11641)\n* Add index_copy() operator (#12810)\n* Fix getnnz operator for CSR matrix (#12908) - issue #12872\n* [MXNET-1173] Debug operators - isfinite, isinf and isnan (#12967)\n* Add sample_like operators (#13034)\n* Add gauss err function operator (#13229)\n* [MXNET -1030] Enhanced Cosine Embedding Loss (#12750)\n* Add bytearray support back to imdecode (#12855, #12868) (#12912)\n* Add Psroipooling CPU implementation (#12738)\n\n### Feature improvements\n#### Operator\n* [MXNET-912] Refactoring ctc loss operator (#12637)\n* Refactor L2_normalization (#13059)\n* Customized and faster `TakeOpForward` operator on CPU (#12997)\n* Allow stop of arange operator to be inferred from dims. (#12064)\n* Make check_isfinite, check_scale optional in clip_global_norm (#12042) add FListInputNames attribute to softmax_cross_entropy (#12701) [MXNET-867] Pooling1D with same padding (#12594)\n* Add support for more req patterns for bilinear sampler backward (#12386) [MXNET-882] Support for N-d arrays added to diag op. (#12430)\n\n#### Optimizer\n* Add a special version of Adagrad optimizer with row-wise learning rate (#12365)\n* Add a Python SVRGModule for performing SVRG Optimization Logic (#12376)\n\n#### Sparse\n\n* Fall back when sparse arrays are passed to MKLDNN-enabled operators (#11664)\n* Add Sparse support for logic operators (#12860)\n* Add Sparse support for take(csr, axis=0)  (#12889)\n\n#### ONNX\n\n* ONNX export - Clip operator (#12457)\n* ONNX version update from 1.2.1 to 1.3 in CI (#12633)\n* Use modern ONNX API to load a model from file (#12777)\n* [MXNET-892] ONNX export/import: DepthToSpace, SpaceToDepth operators (#12731)\n* ONNX export: Fully connected operator w/o bias, ReduceSum, Square (#12646)\n* ONNX export/import: Selu (#12785)\n* ONNX export: Cleanup (#12878)\n* [MXNET-892] ONNX export/import: DepthToSpace, SpaceToDepth operators (#12731)\n* ONNX export: Scalar, Reshape - Set appropriate tensor type (#13067)\n* [MXNET-886] ONNX export: HardSigmoid, Less, Greater, Equal (#12812)\n\n#### MKLDNN\n\n* MKLDNN Forward FullyConnected  op cache (#11611)\n* [MXNET-753] Fallback when using non-MKLDNN supported operators (#12019)\n* MKLDNN Backward op cache (#11301)\n* Implement mkldnn convolution fusion and quantization. (#12530)\n* Improve mkldnn fallback. (#12663)\n* Update MKL-DNN dependency (#12953)\n* Update MKLML dependency (#13181)\n* [MXNET-33] Enhance mkldnn pooling to support full convention (#11047)\n\n#### Inference\n* [MXNET-910] Multithreading inference. (#12456)\n* Tweaked the copy in c_predict_api.h (#12600)\n\n#### Other\n* support for upper triangular matrices in linalg (#12904)\n* Introduce Random module / Refactor code generation (#13038)\n* [MXNET-779]Add DLPack Transformation API (#12047)\n* Draw label name next to corresponding bounding boxes when the mapping of id to names is specified (#9496)\n* Track epoch metric separately (#12182)\n* Set correct update on kvstore flag in dist_device_sync mode (#12786)\n\n### Frontend API updates\n\n#### Gluon\n\n* Update basic_layers.py (#13299)\n* Gluon LSTM Projection and Clipping Support (#13056)\n* Make Gluon download function to be atomic (#12572)\n* [MXNET -1004] Poisson NegativeLog Likelihood loss (#12697)\n* Add activation information for `mxnet.gluon.nn._Conv` (#12354)\n* Gluon DataLoader: avoid recursionlimit error (#12622)\n\n#### Symbol\n* Addressed dumplicate object reference issues (#13214)\n* Throw exception if MXSymbolInferShape fails (#12733)\n* Infer dtype in SymbolBlock import from input symbol (#12412)\n\n### Language API updates\n#### Java\n* [MXNET-1198] MXNet Java API (#13162)\n\n#### R\n* Refactor R Optimizers to fix memory leak - 11374\n* Add new Vignettes to the R package\n  * Char-level Language modeling - 12670\n  * Multidimensional Time series forecasting - 12664\n* Fix broken Examples and tutorials\n  * Tutorial on neural network introduction - 12117\n  * CGAN example - 12283\n  * Test classification with LSTMs - 12263\n\n#### Scala\n* Explain the details for Scala Experimental (#12348)\n* [MXNET-716] Adding Scala Inference Benchmarks (#12721)\n* [MXNET-716][MIRROR #12723] Scala Benchmark Extension pack (#12758)\n* NativeResource Management in Scala (#12647)\n* Ignore generated Scala files (#12928)\n* Use ResourceScope in Model/Trainer/FeedForward.scala (#12882)\n* [MXNET-1180] Scala Image API (#12995)\n* Update log4j version of Scala package (#13131)\n* Review require() usages to add meaningful messages (#12570)\n* Fix Scala readme (#13082)\n\n#### Clojure\n* Introduction to Clojure-MXNet video link (#12754)\n* Improve the Clojure Package README to Make it Easier to Get Started (#12881)\n* MXNET-873 - Bring Clojure Package Inline with New DataDesc and Layout in Scala Package (#12387)\n* Port of Scala Image API to Clojure (#13107)\n\n#### Perl\n* [MXNET-1026] [Perl] Sync with recent changes in Python's API (#12739)\n\n#### Julia\n* Import Julia binding (#10149), how to use is available at https://github.com/apache/mxnet/tree/master/julia\n\n### Performance benchmarks and improvements\n* Update mshadow for omp acceleration when nvcc is not present  (#12674)\n* [MXNET-860] Avoid implicit double conversions (#12361)\n* Add more models to benchmark_score (#12780)\n* Add resnet50-v1 to benchmark_score (#12595)\n\n### Bug fixes\n* Fix for #10920 -  increase tolerance for sparse dot (#12527)\n* [MXNET-1234] Fix shape inference problems in Activation backward (#13409)\n* Fix a bug in `where` op with 1-D input (#12325)\n* [MXNET-825] Fix CGAN R Example with MNIST dataset (#12283)\n* [MXNET-535] Fix bugs in LR Schedulers and add warmup (#11234)\n* Fix speech recognition example (#12291)\n* Fix bug in 'device' type kvstore (#12350)\n* fix search result 404s (#12414)\n* Fix help in imread (#12420)\n* Fix render issue on &lt; and &gt; (#12482)\n* [MXNET-853] Fix for smooth_l1 operator scalar default value (#12284)\n* Fix subscribe links, remove disabled icons (#12474)\n* Fix broken URLs (#12508)\n* Fix/public internal header (#12374)\n* Fix lazy record io when used with dataloader and multi_worker > 0 (#12554)\n* Fix error in try/finally block for blc (#12561)\n* Add cudnn_off parameter to SpatialTransformer Op and fix the inconsistency between CPU & GPU code (#12557)\n* [MXNET-798] Fix the dtype cast from non float32 in Gradient computation (#12290)\n* Fix CodeCovs proper commit detection (#12551)\n* Add TensorRT tutorial to index and fix ToC (#12587)\n* Fixed typo in c_predict_api.cc (#12601)\n* Fix typo in profiler.h (#12599)\n* Fixed NoSuchMethodError for Jenkins Job for MBCC (#12618)\n* [MXNET-922] Fix memleak in profiler (#12499)\n* [MXNET-969] Fix buffer overflow in RNNOp (#12603)\n*  Fixed param coercion of clojure executor/forward (#12627) (#12630)\n* Fix version dropdown behavior (#12632)\n* Fix reference to wrong function (#12644)\n* Fix the location of the tutorial of control flow operators (#12638)\n* Fix issue 12613 (#12614)\n* [MXNET-780] Fix exception handling bug (#12051)\n* Fix bug in prelu, issue 12061 (#12660)\n* [MXNET-833] [R] Char-level RNN tutorial fix (#12670)\n* Fix static / dynamic linking of gperftools and jemalloc (#12714)\n* Fix #12672, importing numpy scalars (zero-dimensional arrays) (#12678)\n* [MXNET-623] Fixing an integer overflow bug in large NDArray (#11742)\n* Fix benchmark on control flow operators (#12693)\n* Fix regression in MKLDNN caused by PR 12019 (#12740)\n* Fixed broken link for Baidu's WARP CTC (#12774)\n* Fix CNN visualization tutorial (#12719)\n* [MXNET-979] Add fix_beta support in BatchNorm (#12625)\n* R fix metric shape (#12776)\n* Revert [MXNET-979] Add fix_beta support in BatchNorm (#12625) (#12789)\n* Fix mismatch shapes (#12793)\n* Fixed symbols naming in RNNCell, LSTMCell, GRUCell (#12794)\n* Fixed __setattr__ method of _MXClassPropertyMetaClass (#12811)\n* Fixed regex for matching platform type in Scala Benchmark scripts (#12826)\n* Fix broken links (#12856)\n* Fix Flaky Topk (#12798)\n* [MXNET-1033] Fix a bug in MultiboxTarget GPU implementation (#12840)\n* [MXNET-1107] Fix CPUPinned unexpected behaviour (#12031)\n* Fix __all__ in optimizer/optimizer.py (#12886)\n* Fix Batch input issue with Scala Benchmark (#12848)\n* fix type inference in index_copy. (#12890)\n* Fix the paths issue for downloading script (#12913)\n* Fix indpt[0] for take(csr) (#12927)\n* Fix the bug of assigning large integer to NDArray (#12921)\n* Fix Sphinx errors for tutorials and install ToCs (#12945)\n* Fix variable name in tutorial code snippet (#13052)\n* Fix example for mxnet.nd.contrib.cond and fix typo in src/engine (#12954)\n* Fix a typo in operator guide (#13115)\n* Fix variational autoencoder example (#12880)\n* Fix problem with some OSX not handling the cast on imDecode (#13207)\n* [MXNET-953] Fix oob memory read (#12631)\n* Fix Sphinx error in ONNX file (#13251)\n* [Example] Fixing Gradcam implementation (#13196)\n* Fix train mnist for inception-bn and resnet (#13239)\n* Fix a bug in index_copy (#13218)\n* Fix Sphinx errors in box_nms (#13261)\n* Fix Sphinx errors (#13252)\n* Fix the cpp example compiler flag (#13293)\n* Made fixes to sparse.py and sparse.md (#13305)\n* [Example] Gradcam- Fixing a link (#13307)\n* Manually track num_max_thread (#12380)\n* [Issue #11912] throw mxnet exceptions when decoding invalid images. (#12999)\n* Undefined name: load_model() --> utils.load_model() (#12867)\n* Change the way NDArrayIter handle the last batch (#12545)\n* Add embedding to print_summary (#12796)\n* Allow foreach on input with 0 length (#12471)\n* [MXNET-360]auto convert str to bytes in img.imdecode when py3 (#10697)\n* Fix unpicklable transform_first on windows (#13686)\n\n### Licensing updates\n* Add license headers to R-package (#12559)\n* License header (#13178)\n* add url and license to clojure package project (#13304)\n\n### Improvements\n#### Tutorial\n* [MXNET-422] Distributed training tutorial (#10955)\n* Add a tutorial for control flow operators. (#12340)\n* Add tutorial Gotchas using NumPy (#12007)\n* Updated Symbol tutorial with Gluon (#12190)\n* Improve tutorial redirection (#12607)\n* Include missing import in TensorRT tutorial (#12609)\n* Update Operator Implementation Tutorial (#12230)\n* Add a tutorial for the subgraph API. (#12698)\n* Improve clojure tutorial (#12974)\n* Update scala intellij tutorial (#12827)\n* [Example] Gradcam consolidation in tutorial (#13255)\n* [MXNET-1203] Tutorial infogan  (#13144)\n* [MXNET-703] Add a TensorRT walkthrough (#12548)\n\n#### Example\n* Update C++ example so it is easier to run (#12397)\n* [MXNET-580] Add SN-GAN example (#12419)\n* [MXNET-637] Multidimensional LSTM example for MXNetR (#12664)\n* [MXNET-982] Provide example to illustrate usage of CSVIter in C++ API (#12636)\n* [MXNET-947] Expand scala imclassification example with resnet (#12639)\n* MKL-DNN Quantization Examples and README (#12808)\n* Extending the DCGAN example implemented by gluon API to provide a more straight-forward evaluation on the generated image (#12790)\n* [MXNET-1017] Updating the readme file for cpp-package and adding readme file for example directory. (#12773)\n* Update tree lstm example (#12960)\n* Update bilstm integer array sorting example (#12929)\n* Updated / Deleted some examples (#12968)\n* Update module example (#12961)\n* Update adversary attack generation example (#12918)\n* Update Gluon example folder (#12951)\n* Update dec example (#12950)\n* Updated capsnet example (#12934)\n* Updates to several examples (#13068)\n* Update multi-task learning example (#12964)\n* Remove obsolete memory cost example (#13235)\n* [Example] Update cpp example README (#13280)\n* [Example]update NER example readme on module prediction (#13184)\n* Update proposal_target.py (#12709)\n* Removing the re-size for validation data, which breaking the validation accuracy of CIFAR training (#12362)\n* Update the README with instruction to redirect the user to gluon-cv (#13186)\n\n#### Documentation\n* Update ONNX API docs references (#12317)\n* Documentation update related to sparse support (#12367)\n* Edit shape.array doc and some style improvements (#12162)\n* Fixed docs/website build checkout bug (#12413)\n* Add Python API docs for test_utils and visualization (#12455)\n* Fix the installation doc for MKL-DNN backend (#12534)\n* Added comment to docs regarding ToTensor transform (#12186)\n* Pinned dockcross to a tag with fixed ABI for RPi (#12588)\n* Refine the documentation of im2rec (#12606)\n* Update and modify Windows docs (#12620)\n* update docs to list cmake required for build from source page (#12592)\n* update the distributed_training document (#12626)\n* Add docstring in im2rec.py (#12621)\n* [Doc] Change the description for pip packages (#12584)\n* Change dependencies documentation opencv2-->opencv (#12654)\n* Add documents for two new environment variables for memory pool. (#12668)\n* Scala Docs - Replace old Symbol api usages (#12759)\n* add/update infer_range docs (#12879)\n* Fix typo in formula in docstring for GRU cell and layer and add clarification to description (gluon.rnn) (#12896)\n* Fix the operator API documentation (#12942)\n* fix broken docs (#12871)\n* fix mac r install and windows python build from source docs (#12919)\n* Document the newly added env variable (#13049)\n* Add documentation on GPU performance on Quantization example (#13145)\n* Fix Sphinx python docstring formatting error. (#13177)\n* [Doc] Fix repo paths in Ubuntu build doc (#13101)\n* Fix Sphinx document parsing error. (#13195)\n* Fix #13090, Add image.imread to python API doc. (#13176)\n* Fix Sphinx docstring formatting error. (#13004, #13005, #13006) (#13175)\n* Fix #12944, Fix Sphinx python docstring formatting error. (#13174)\n* Fix #13013, Fix Sphinx python docstring error. (#13173)\n* Fixed Sparse astype doc string formatting error (#13171)\n* Fixed Documentation issues (#13215)\n* update the doc (#13205)\n* Fix Sphinx doc errors (#13170)\n* Fix Sphinx python docstring error: initializer.InitDesc (#12939) (#13148)\n* Fix Sphinx python docstring error: text contrib module (#12949) (#13149)\n* Fix Sphinx python docstrings (#13160)\n* Add Java API docs generation (#13071)\n* Fix scaladoc build errors (#13189)\n* Add missing documentations for getnnz (#13128)\n* Addressed ONNX module documentation warnings and added notes for short-form representation (#13259)\n* Doc fixes (#13256)\n* Addressed doc issues (#13165)\n* stop gap fix to let website builds through; scaladoc fix pending (#13298)\n* Fix Sphinx python docstring formatting error. (#13194)\n* Visualization doc fix. Added notes for shortform (#13291)\n* [Example] Add docstring for test optimizer and test score (#13286)\n* Fix descriptions in scaladocs for macro ndarray/sybmol APIs (#13210)\n* Sphinx error reduction (#12323)\n* Sphinx errors in Gluon (#13275)\n* Update env_var.md (#12702)\n* Updated the Instructions for use of the label bot (#13192)\n* Added/changed file_name, brief description comments in some files (#13033)\n\n#### Website\n* adding apache conf promo to home page (#12347)\n* Consistent website theme and custom 404 (#12426)\n* update apachecon links to https (#12521)\n* [HOLD] 1.3.0 release website updates (#12509)\n* add mentions of the gluon toolkits and links to resources (#12667)\n* remove apachecon promo (#12695)\n* [MXNet-1002] Add GluonCV and NLP tookits, Keras, and developer wiki to navigation (#12704)\n\n#### MXNet Distributions\n* Make the output of ci/docker/install/ubuntu_mklml.sh less verbose (#12422)\n* Fix tvm dependency for docker (#12479)\n* [MXNET-703] Add TensorRT runtime Dockerfile (#12549)\n* [MXNET-951] Python dockerfiles built on pip binaries and build/release script (#12556)\n* Change numpy version to 1.15.2 in python and docker install requirements (#12711)\n* Add mkl-dnn to docker install method (#12643)\n* Fix docker cleanup race condition (#13092)\n* Bugfix in ci/docker_cache.py (#13249)\n* Update PyPI version number (#11773)\n* update download links to apache distros (#12617)\n\n#### Installation\n* Installation instructions consolidation (#12388)\n* Refine mxnet python installation (#12696)\n* R install instructions update for macOS (#12832)\n* remove legacy installation of Roxygen2 5.0 and add R-specific clean target (#12993) (#12998)\n* Force APT cache update before executing install (#13285)\n* Make the Ubuntu scripts executable after download. (#12180)\n* replacing windows setup with newer instructions (#12504)\n* Updated download links and verification instructions (#12651)\n* Remove pip overwrites (#12604)\n\n#### Build and CI\n* [MXNET-908] Enable minimal OSX Travis build (#12462)\n* Use jom for parallel Windows builds (#12533)\n* [MXNET-950] Enable parallel R dep builds in CI (#12552)\n* Speed up CI Windows builds (#12563)\n* [MXNET-908] Speed up travis builds to avoid timeouts (#12706)\n* Simplify mac MKLDNN build (#12724)\n* [MXNET-674] Speed up GPU builds in CI (#12782)\n* Improved git reset for CI builds (#12784)\n* Improve cpp-package example project build files. (#13093)\n* Add --no-cache option to build.py when building containers (#13182)\n* Addressed sphinx build issue (#13246)\n* Tighten up PyLint directives again (#12322)\n* [MXNET-859] Add a clang-tidy stage to CI (#12282)\n* A solution to prevent zombie containers locally and in CI (#12381)\n*  [MXNET-696][PYTHON][UNDEFINED NAME] import logging in ci/util.py (#12488)\n* [MXNET-703] Static linking for libprotobuf with TensorRT (#12475)\n* Remove regression checks for website links (#12507)\n* [MXNET-953] - Add ASAN sanitizer, Enable in CI (#12370)\n* Allow custom path and static linking for custom mallocs in make (#12645)\n* Correct PR branch detection in code coverage (#12615)\n* Update osx.mk - Added apple to USE_BLAS comment (#12819)\n* [MXNET-953] Correct ASAN cflags flag (#12659)\n* [MXNET-1025] Add Jetpack 3.3 support to Jetson (#12735)\n* Fail the broken link job when broken links are found (#12905)\n* Removed unused header (#13066)\n* Maven Surefire bug workaround (#13081)\n* Add Turing and Volta support to arch_name (#13168)\n* Moves f16c autodetection to its own cmake module (#12331)\n* la_op_inline.h to la_op-inl.h for consistency (#13045)\n* [MXNET-793] Virtualized ARMv7 with Qemu CI integration (#13203)\n* Remove unused variable `rotateM_` (#10803)\n* Separate refactoring from #12276 in a prior PR (#12296)\n* [MXNET-860] Remove std::moves that have no affect (#12730)\n* [MXNET-860] Use emplace where helpful (#12694)\n* Enable C++ coverage (#12642)\n* [MXNET-860] Update to modern nullptr usage (#12352)\n* [MXNET-860] Reduce redundant copies, check for regressions with clang-tidy (#12355)\n\n\n#### 3rd party\n##### TVM:\n* Updated tvm submodule head (#12764)\n* Updated tvm submodule head (#12448)\n##### CUDNN:\n* [MXNET-1179] Enforce deterministic algorithms in convolution layers (#12992)\n* CudnnFind() usage improvements (#12804)\n* Add option for automatic downcasting dtype for cudnn to allow using Tensorcore for fp32  (#12722)\n##### Horovod:\n* [MXNET-1111] Remove CPUPinned in ImageRecordIter (#12666)\n\n### Deprications\n* Add a deprecate message (#13042) contrib_CTCLoss is deprecated. Added a message in command\n### Other\n* Updating news, readme files and bumping master version to 1.3.1 (#12525)\n* Add new name to CONTRIBUTORS.md (#12763)\n* Update contribute.md (#12685)\n* Updated CONTRIBUTORS.md to include lebeg and gigasquid, moved mabreu to committers section (#12766)\n* Update CONTRIBUTORS.md (#12996)\n* Updated CONTRIBUTORS.md to include mxnet-label-bot  (#13048)\n\n### How to build MXNet\nPlease follow the instructions at https://mxnet.apache.org/install/index.html\n\n### List of submodules used by Apache MXNet (Incubating) and when they were updated last\nSubmodule@commit ID::Last updated by MXNet:: Last update in submodule\n\n* cub@05eb57f::Jul 31, 2017 :: Jul 31, 2017\n* dlpack@10892ac:: Oct 30, 2017 :: Aug 23, 2018\n* dmlc-core@0a0e8ad:: Aug 15, 2018 :: Nov 15, 2018\n* googletest@ec44c6c:: July 14, 2016 :: July 14, 2016\n* mkldnn@722901c:: Feb 13, 2019 :: Feb 12, 2019\n* mshadow@696803b:: Sep 28, 2018 :: Nov 7,  2018\n* onnx-tensorrt@3d8ee04:: Aug 22, 2018 :: Nov 10, 2018\n* openmp@37c7212: Nov 22, 2017 :: Nov 13, 2018\n* ps-lite@8a76389: April 25, 2018 :: Oct 9, 2018\n* tvm@0f053c8: Oct 10, 2018 :: Oct 8, 2018\n\n## 1.3.1\n\n### Bug fixes\n\n* [MXNET-953] Fix oob memory read (v1.3.x) / [#13118](https://github.com/apache/mxnet/pull/13118)\nSimple bugfix addressing an out-of-bounds memory read.\n\n\n* [MXNET-969] Fix buffer overflow in RNNOp (v1.3.x) / [#13119](https://github.com/apache/mxnet/pull/13119)\nThis fixes an buffer overflow detected by ASAN.\n\n\n* CudnnFind() usage improvements (v1.3.x) / [#13123](https://github.com/apache/mxnet/pull/13123)\n  This PR improves the MXNet's use of cudnnFind() to address a few issues:\n  1. With the gluon imperative style, cudnnFind() is called during forward(), and so might have its timings perturbed by other GPU activity (including potentially other cudnnFind() calls).\n  2. With some cuda drivers versions, care is needed to ensure that the large I/O and workspace cudaMallocs() performed by cudnnFind() are immediately released and available to MXNet.\n  3. cudnnFind() makes both conv I/O and workspace allocations that must be covered by the GPU global memory headroom defined by MXNET_GPU_MEM_POOL_RESERVE. Per issue #12662, large convolutions can result in out-of-memory errors, even when MXNet's storage allocator has free memory in its pool.\n\n  This PR addresses these issues, providing the following benefits:\n  1. Consistent algo choice for a given convolution type in a model, both for instances in the same GPU and in other GPUs in a multi-GPU training setting.\n  2. Consistent algo choice from run to run, based on eliminating sources of interference of the cudnnFind() timing process.\n  3. Consistent model global memory footprint, both because of the consistent algo choice (algo's can have markedly different workspace requirements) and changes to MXNet's use of cudaMalloc.\n  4. Increased training performance based on being able to consistently run with models that approach the GPU's full global memory footprint.\n  5. Adds a unittest for and solves issue #12662.\n\n* [MXNET-922] Fix memleak in profiler (v1.3.x) / [#13120](https://github.com/apache/mxnet/pull/13120)\n  Fix a memleak reported locally by ASAN during a normal inference test.\n\n* Fix lazy record io when used with dataloader and multi_worker > 0 (v1.3.x) / [#13124](https://github.com/apache/mxnet/pull/13124)\n  Fixes multi_worker data loader when record file is used. The MXRecordIO instance needs to require a new file handler after fork to be safely manipulated simultaneously.\n\n  This fix also safely voids the previous temporary fixes #12093 #11370.\n\n* fixed symbols naming in RNNCell, LSTMCell, GRUCell (v1.3.x) / [#13158](https://github.com/apache/mxnet/pull/13158)\n  This fixes #12783, by assigning all nodes in hybrid_forward a unique name. Some operations were in fact performed without attaching the appropriate (time) prefix to the name, which makes serialized graphs non-deserializable.\n\n* Fixed `__setattr__` method of `_MXClassPropertyMetaClass` (v1.3.x) / [#13157](https://github.com/apache/mxnet/pull/13157)\n  Fixed `__setattr__` method\n\n* allow foreach on input with 0 length (v1.3.x) / [#13151](https://github.com/apache/mxnet/pull/13151)\n  Fix #12470. With this change, outs shape can be inferred correctly.\n\n* Infer dtype in SymbolBlock import from input symbol (v1.3.x) / [#13117](https://github.com/apache/mxnet/pull/13117)\n  Fix for the issue - #11849\n  Currently, Gluon symbol block cannot import any symbol with type other than fp32. All the parameters are created as FP32 leading to failure in importing the params when it is of type fp16, fp64 etc,\n  In this PR, we infer the type of the symbol being imported and create the Symbol Block Parameters with that inferred type.\n  Added the tests\n\n### Documentation fixes\n\n* Document the newly added env variable (v1.3.x) / [#13156](https://github.com/apache/mxnet/pull/13156)\n  Document the env variable: MXNET_ENFORCE_DETERMINISM added in PR: [#12992](https://github.com/apache/mxnet/pull/12992)\n\n* fix broken links (v1.3.x) / [#13155](https://github.com/apache/mxnet/pull/13155)\n  This PR fixes broken links on the website.\n\n* fix broken Python IO API docs (v1.3.x) / [#13154](https://github.com/apache/mxnet/pull/13154)\n  Fixes [#12854: Data Iterators documentation is broken](https://github.com/apache/mxnet/issues/12854)\n\n  This PR manually specifies members of the IO module so that the docs will render as expected. This is workaround in the docs to deal with a bug introduced in the Python code/structure since v1.3.0. See the comments for more info.\n\n  This PR also fixes another issue that may or may not be related. Cross references to same-named entities like name, shape, or type are confusing Sphinx and it seems to just link to whatever it last dealt with that has the same name, and not the current module. To fix this you have to be very specific. Don't use type, use np.type if that's what you want. Otherwise you might end up with mxnet.kvstore.KVStore.type. This is a known Sphinx issue, so it might be something we have to deal with for the time being.\n\n  This is important for any future modules - that they recognize this issue and make efforts to map the params and other elements.\n\n* add/update infer_range docs (v1.3.x) / [#13153](https://github.com/apache/mxnet/pull/13153)\n  This PR adds or updates the docs for the infer_range feature.\n\n  Clarifies the param in the C op docs\n  Clarifies the param in the Scala symbol docs\n  Adds the param for the Scala ndarray docs\n  Adds the param for the Python symbol docs\n  Adds the param for the Python ndarray docs\n\n### Other Improvements\n\n* [MXNET-1179] Enforce deterministic algorithms in convolution layers (v1.3.x) / [#13152](https://github.com/apache/mxnet/pull/13152)\n  Some of the CUDNN convolution algorithms are non-deterministic (see issue #11341). This PR adds an env variable to enforce determinism in the convolution operators. If set to true, only deterministic CUDNN algorithms will be used. If no deterministic algorithm is available, MXNet will error out.\n\n\n### Submodule updates\n\n* update mshadow (v1.3.x) / [#13122](https://github.com/apache/mxnet/pull/13122)\n  Update mshadow for omp acceleration when nvcc is not present\n\n### Known issues\n\nThe test test_operator.test_dropout has issues and has been disabled on the branch:\n\n* Disable flaky test test_operator.test_dropout (v1.3.x) / [#13200](https://github.com/apache/mxnet/pull/13200)\n\n\n\nFor more information and examples, see [full release notes](https://cwiki.apache.org/confluence/x/eZGzBQ)\n\n\n## 1.3.0\n\n### New Features - Gluon RNN layers are now HybridBlocks\n- In this release, Gluon RNN layers such as `gluon.rnn.RNN`, `gluon.rnn.LSTM`, `gluon.rnn.GRU` becomes `HybridBlock`s as part of [gluon.rnn improvements project](https://github.com/apache/mxnet/projects/11) (#11482).\n- This is the result of newly available fused RNN operators added for CPU: LSTM([#10104](https://github.com/apache/mxnet/pull/10104)), vanilla RNN([#11399](https://github.com/apache/mxnet/pull/11399)), GRU([#10311](https://github.com/apache/mxnet/pull/10311))\n- Now many dynamic networks that are based on Gluon RNN layers can now be completely hybridized, exported, and used in the inference APIs in other language bindings such as R, Scala, etc.\n\n### MKL-DNN improvements\n- Introducing more functionality support for MKL-DNN as follows:\n  - Added support for more activation functions like, \"sigmoid\", \"tanh\", \"softrelu\". ([#10336](https://github.com/apache/mxnet/pull/10336))\n  - Added Debugging functionality: Result check ([#12069](https://github.com/apache/mxnet/pull/12069)) and Backend switch ([#12058](https://github.com/apache/mxnet/pull/12058)).\n\n### New Features - Gluon Model Zoo Pre-trained Models\n- Gluon Vision Model Zoo now provides MobileNetV2 pre-trained models (#10879) in addition to\n  AlexNet, DenseNet, Inception V3, MobileNetV1, ResNet V1 and V2, SqueezeNet 1.0 and 1.1, and VGG\n  pretrained models.\n- Updated pre-trained models provide state-of-the-art performance on all resnetv1, resnetv2, and vgg16, vgg19, vgg16_bn, vgg19_bn models (#11327 #11860 #11830).\n\n### New Features - Clojure package (experimental)\n- MXNet now supports the Clojure programming language. The MXNet Clojure package brings flexible and efficient GPU computing and state-of-art deep learning to Clojure. It enables you to write seamless tensor/matrix computation with multiple GPUs in Clojure. It also lets you construct and customize the state-of-art deep learning models in Clojure, and apply them to tasks, such as image classification and data science challenges.([#11205](https://github.com/apache/mxnet/pull/11205))\n- Checkout examples and API documentation [here](https://mxnet.apache.org/api/clojure/index.html).\n\n### New Features - Synchronized Cross-GPU Batch Norm (experimental)\n- Gluon now supports Synchronized Batch Normalization (#11502).\n- This enables stable training on large-scale networks with high memory consumption such as FCN for image segmentation.\n\n### New Features - Sparse Tensor Support for Gluon (experimental)\n- Sparse gradient support is added to `gluon.nn.Embedding`. Set `sparse_grad=True` to enable when constructing the Embedding block. ([#10924](https://github.com/apache/mxnet/pull/10924))\n- Gluon Parameter now supports \"row_sparse\" storage type, which reduces communication cost and memory consumption for multi-GPU training for large models. `gluon.contrib.nn.SparseEmbedding` is an example empowered by this. ([#11001](https://github.com/apache/mxnet/pull/11001), [#11429](https://github.com/apache/mxnet/pull/11429))\n- Gluon HybridBlock now supports hybridization with sparse operators ([#11306](https://github.com/apache/mxnet/pull/11306)).\n\n### New Features - Control flow operators (experimental)\n- This is the first step towards optimizing dynamic neural networks with variable computation graphs, by adding symbolic and imperative control flow operators. [Proposal](https://cwiki.apache.org/confluence/display/MXNET/Optimize+dynamic+neural+network+models+with+control+flow+operators).\n- New operators introduced: foreach([#11531](https://github.com/apache/mxnet/pull/11531)), while_loop([#11566](https://github.com/apache/mxnet/pull/11566)), cond([#11760](https://github.com/apache/mxnet/pull/11760)).\n\n### New Features - Scala API Improvements (experimental)\n- Improvements to MXNet Scala API usability([#10660](https://github.com/apache/mxnet/pull/10660), [#10787](https://github.com/apache/mxnet/pull/10787), [#10991](https://github.com/apache/mxnet/pull/10991))\n- Symbol.api and NDArray.api would bring new set of functions that have complete definition for all arguments.\n- Please see this [Type safe API design document](https://cwiki.apache.org/confluence/display/MXNET/Scala+Type-safe+API+Design+Doc) for more details.\n\n### New Features - Rounding GPU Memory Pool for dynamic networks with variable-length inputs and outputs (experimental)\n- MXNet now supports a new memory pool type for GPU memory (#11041).\n- Unlike the default memory pool requires exact size match to reuse released memory chunks, this new memory pool uses exponential-linear rounding so that similar sized memory chunks can all be reused, which is more suitable for all the workloads with dynamic-shape inputs and outputs. Set environment variable `MXNET_GPU_MEM_POOL_TYPE=Round` to enable.\n\n### New Features - Topology-aware AllReduce (experimental)\n- This features uses trees to perform the Reduce and Broadcast. It uses the idea of minimum spanning trees to do a binary tree Reduce communication pattern to improve it. This topology aware approach reduces the existing limitations for single machine communication shown by mehods like parameter server and NCCL ring reduction. It is an experimental feature ([#11591](https://github.com/apache/mxnet/pull/11591)).\n- Paper followed for implementation: [Optimal message scheduling for aggregation](https://www.sysml.cc/doc/178.pdf).\n- Set environment variable `MXNET_KVSTORE_USETREE=1` to enable.\n\n### New Features - Export MXNet models to ONNX format (experimental)\n- With this feature, now MXNet models can be exported to ONNX format([#11213](https://github.com/apache/mxnet/pull/11213)). Currently, MXNet supports ONNX v1.2.1. [API documentation](https://mxnet.apache.org/api/python/contrib/onnx.html).\n- Checkout this [tutorial](https://mxnet.apache.org/tutorials/onnx/export_mxnet_to_onnx.html) which shows how to use MXNet to ONNX exporter APIs. ONNX protobuf so that those models can be imported in other frameworks for inference.\n\n### New Features - TensorRT Runtime Integration (experimental)\n- [TensorRT](https://developer.nvidia.com/tensorrt) provides significant acceleration of model inference on NVIDIA GPUs compared to running the full graph in MxNet using unfused GPU operators. In addition to faster fp32 inference, TensorRT optimizes fp16 inference, and is capable of int8 inference (provided the quantization steps are performed). Besides increasing throughput, TensorRT significantly reduces inference latency, especially for small batches.\n- This feature in MXNet now introduces runtime integration of TensorRT into MXNet, in order to accelerate inference.([#11325](https://github.com/apache/mxnet/pull/11325))\n- Currently, its in contrib package.\n\n### New Examples - Scala\n- Refurnished Scala Examples with improved API, documentation and CI test coverage. ([#11753](https://github.com/apache/mxnet/pull/11753), [#11621](https://github.com/apache/mxnet/pull/11621) )\n- Now all Scala examples have:\n  - No bugs block in the middle\n  - Good Readme to start with\n  - with Type-safe API usage inside\n  - monitored in CI in each PR runs\n\n### Maintenance - Flaky Tests improvement effort\n- Fixed 130 flaky tests on CI. Tracked progress of the project [here](https://github.com/apache/mxnet/projects/9).\n- Add flakiness checker (#11572)\n\n### Maintenance - MXNet Model Backwards Compatibility Checker\n- This tool ([#11626](https://github.com/apache/mxnet/pull/11626)) helps in ensuring consistency and sanity while performing inference on the latest version of MXNet using models trained on older versions of MXNet.\n- This tool will help in detecting issues earlier in the development cycle which break backwards compatibility on MXNet and would contribute towards ensuring a healthy and stable release of MXNet.\n\n### Maintenance - Integrated testing for \"the Straight Dope\"\n- [\"Deep Learning - The Straight Dope\"](http://gluon.mxnet.io) is a deep learning book based on Apache MXNet Gluon that are contributed by many Gluon users.\n- Now the testing of this book is integrated in the nightly tests.\n\n### Bug-fixes\n- Fix gperftools/jemalloc and lapack warning bug. (#11110)\n- Fix mkldnn performance regression + improve test logging (#11262)\n- Fix row_sparse_param.save() (#11266)\n- Fix trainer init_kvstore (#11266)\n- Fix axis Bug in MKLDNN Softmax (#11335)\n- Fix 'AttributeError: '_thread._local' object has no attribute 'value'' on distributed processing applications (#11332)\n- Fix recordfile dataset with multi worker (#11370)\n- Manually check node existence in CachedOp (#11545)\n- Javadoc fix (#11239)\n- Fix bugs in MKLDNN operators to handle the kAddTo request (#11129)\n- Fix InferStorage for sparse fallback in FullyConnected (#11498)\n- Fix batchnorm problem with sparse matrices when fix_gamma=True (#11656)\n- Fix rnn layer save (#11776)\n- Fix BucketSentenceIter bug related to #11430 (#11580)\n- Fix for _backward_softsign activation (#11827)\n- Fix a bug in CachedOp. (#11675)\n- Fix quantization divide by zero errors (#11833)\n- Refactor R optimizers to fix memory leak (#11374)\n- Avoid use of troublesome cudnnFind() results when grad_req='add' (#11338)\n- Fix shared memory with gluon dataloader, add option pin_memory (#11908)\n- Fix quantized graph pass bug (#11937)\n- Fix MXPredReshape in the c_predict_api (#11493)\n- Fix the topk regression issue (#12197)\n- Fix image-classification example and add missing optimizers w/ momentum support (#11826)\n\n### Performance Improvements\n- Added static allocation and static shape for HybridBloc gluon (#11320)\n- Fix RecordIO augmentation speed (#11474)\n- Improve sparse pull performance for gluon trainer (#11429)\n- CTC operator performance improvement from HawkAaron/MXNet-CTC (#11834)\n- Improve performance of broadcast ops backward pass (#11252)\n- Improved numerical stability as a result of using stable L2 norm (#11573)\n- Accelerate the performance of topk for GPU and CPU side (#12085 #10997 ; This changes the behavior of topk when nan values occur in the input)\n- Support for dot(dns, csr) = dns and dot(dns, csr.T) = dns on CPU ([#11113](https://github.com/apache/mxnet/pull/11113))\n- Performance improvement for Batch Dot on CPU from mshadow ([mshadow PR#342](https://github.com/dmlc/mshadow/pull/342))\n\n### API Changes\n- Allow Scala users to specify data/label names for NDArrayIter (#11256)\n- Allow user to define unknown token symbol to rnn encode_sentences() (#10461)\n- Added count_include_pad argument for Avg Pooling (#11021)\n- Add standard ResNet data augmentation for ImageRecordIter (#11027)\n- Add seed_aug parameter for ImageRecordIter to fix random seed for default augmentation (#11247)\n- Add support for accepting MXNet NDArrays in ColorNormalizeAug (#11606)\n- Enhancement of take operator (#11326)\n- Add temperature parameter in Softmax operator (#11466)\n- Add support for 1D inputs in leaky relu (#11850)\n- Add verify_ssl option to gluon.utils.download (#11546)\n\n### Other features\n- Added ccache reporting to CI (#11322)\n- Restructure dockcross dockerfiles to fix caching (#11302)\n- Added tests for MKLDNN backward operators  (#11232)\n- Add elemwise_add/sub between rsp and rsp on GPU (#11179)\n- Add clip_global_norm(row_sparse_grad) (#11266)\n- Add subgraph storage type inference to CachedOp  (#11306)\n- Enable support for dense weight and sparse grad Adagrad updates (#11355)\n- Added Histogram Operator (#10931)\n- Added Matthew's Correlation Coefficient to metrics (#10524)\n- Added support for add_n(dense, csr, dense) = dense on CPU & GPU (#11330)\n- Added support for add_n(any combination longer than 4 with at least one dense storage) = dense on CPU & GPU (#11330)\n- L1 Normalization (#11229)\n- Add support for int64 data type in CSVIter (#11446)\n- Add test for new int64 type in CSVIter (#11499)\n- Add sample ratio for ROI Align (#11145)\n- Shape and Size Operator (#10889)\n- Add HybidSequentialRNNCell, which can be nested in HybridBlock (#11003)\n- Support for a bunch of unary functions for csr matrices (#11559)\n- Added NDArrayCollector to dispose intermediate allocated NDArrays automatically (#11751)\n- Added the diag() operator (#11643)\n- Added broadcast_like operator (#11820)\n- Allow Partial shape infer for Slice (#11406)\n- Added support to profile kvstore server during distributed training  (#11215)\n- Add function for GPU Memory Query to C API (#12083)\n- Generalized reshape_like operator to be more flexible (#11928)\n- Add support for selu activation function (#12059)\n- Add support for accepting NDArray as input to Module predict API (#12166)\n- Add DataDesc type for the Scala Package (#11844)\n\n### Usability Improvements\n- Added NDArray auto-collector for Scala (#11751, #12232)\n- Added docs for mx.initializer.Constant (#10637)\n- Added build from souce instructions on windows (#11276)\n- Added a tutorial explaining how to use the profiler (#11274)\n- Added two tutorials on Learning Rate Schedules (#11296)\n- Added a tutorial for mixed precision training with float16 (#10391)\n- Create CPP test for concat MKLDNN operator (#11371)\n- Update large word language model example (#11405)\n- MNIST Examples for Scala new API (#11250)\n- Updated installation info to have latest packages and more clarity (#11503)\n- GAN MNIST Examples for Scala new API (#11547)\n- Added Learning Rate Finder tutorial (#11304)\n- Fix Installation instructions for R bindings on Linux systems. (#11590)\n- Integration Test for Scala (#11596)\n- Documentation enhancement for optimizers (#11657)\n- Update rcnn example (#11373)\n- Gluon ModelZoo, Gluon examples for Perl APIs (#11642)\n- Fix R installation in CI (#11761,#11755, #11768, #11805, #11954, #11976)\n- CNN Examples for Scala new API (#11292)\n- Custom Operator Example for Scala (#11401)\n- Added detailed doc about global pool layers in Gluon (#11832)\n- Updated MultiTask example to use new infer api (#11605)\n- Added logistic regression tutorial (#11651)\n- Added Support for integer type in ImageIter (#11864)\n- Added depth_to_space and space_to_depth operators (#11587)\n- Increased operator support for ONNX to MXNet importer (#11856)\n- Add linux and macos MKLDNN Building Instruction (#11049)\n- Add download utility for Scala APIs (#11866)\n- Improving documentation and error messages for Async distributed training with Gluon (#11910)\n- Added NeuralStyle Example for Scala (#11621)\n\nFor more information and examples, see [full release notes](https://cwiki.apache.org/confluence/display/MXNET/Apache+MXNet+%28incubating%29+1.3.0+Release+Notes)\n\n## 1.2.0\n### New Features - Added Scala Inference APIs\n- Implemented new [Scala Inference APIs](https://cwiki.apache.org/confluence/display/MXNET/MXNetScalaInferenceAPI) which offer an easy-to-use, Scala Idiomatic and thread-safe high level APIs for performing predictions with deep learning models trained with MXNet (#9678). Implemented a new ImageClassifier class which provides APIs for classification tasks on a Java BufferedImage using a pre-trained model you provide (#10054). Implemented a new ObjectDetector class which provides APIs for object and boundary detections on a Java BufferedImage using a pre-trained model you provide (#10229).\n\n### New Features - Added a Module to Import ONNX models into MXNet\n- Implemented a new ONNX module in MXNet which offers an easy to use API to import ONNX models into MXNet's symbolic interface (#9963). Checkout the [example](https://github.com/apache/mxnet/blob/master/example/onnx/super_resolution.py) on how you could use this [API](https://cwiki.apache.org/confluence/display/MXNET/ONNX-MXNet+API+Design) to import ONNX models and perform inference on MXNet. Currently, the ONNX-MXNet Import module is still experimental. Please use it with caution.\n\n### New Features - Added Support for Model Quantization with Calibration\n- Implemented model quantization by adopting the [TensorFlow approach](https://www.tensorflow.org/performance/quantization) with calibration by borrowing the idea from Nvidia's [TensorRT](http://on-demand.gputechconf.com/gtc/2017/presentation/s7310-8-bit-inference-with-tensorrt.pdf). The focus of this work is on keeping quantized models (ConvNets for now) inference accuracy loss under control when compared to their corresponding FP32 models. Please see the [example](https://github.com/apache/mxnet/tree/master/example/quantization) on how to quantize a FP32 model with or without calibration (#9552). Currently, the Quantization support is still experimental. Please use it with caution.\n\n### New Features - MKL-DNN Integration\n- MXNet now integrates with Intel MKL-DNN to accelerate neural network operators: Convolution, Deconvolution, FullyConnected, Pooling, Batch Normalization, Activation, LRN, Softmax, as well as some common operators: sum and concat (#9677). This integration allows NDArray to contain data with MKL-DNN layouts and reduces data layout conversion to get the maximal performance from MKL-DNN. Currently, the MKL-DNN integration is still experimental. Please use it with caution.\n\n### New Features - Added Exception Handling Support for Operators\n- Implemented [Exception Handling Support for Operators](https://cwiki.apache.org/confluence/display/MXNET/Improved+exception+handling+in+MXNet) in MXNet. MXNet now transports backend C++ exceptions to the different language front-ends and prevents crashes when exceptions are thrown during operator execution (#9681).\n\n### New Features - Enhanced FP16 support\n- Added support for distributed mixed precision training with FP16. It supports storing of master copy of weights in float32 with the multi_precision mode of optimizers (#10183). Improved speed of float16 operations on x86 CPU by 8 times through F16C instruction set. Added support for more operators to work with FP16 inputs (#10125, #10078, #10169). Added a tutorial on using mixed precision with FP16 (#10391).\n\n### New Features - Added Profiling Enhancements\n- Enhanced built-in profiler to support native Intel:registered: VTune:tm: Amplifier objects such as Task, Frame, Event, Counter and Marker from both C++ and Python -- which is also visible in the Chrome tracing view(#8972). Added Runtime tracking of symbolic and imperative operators as well as memory and API calls. Added Tracking and dumping of aggregate profiling data. Profiler also no longer affects runtime performance when not in use.\n\n### Breaking Changes\n- Changed Namespace for MXNet scala from `ml.dmlc.mxnet` to `org.apache.mxnet` (#10284).\n- Changed API for the Pooling operator from `mxnet.symbol.Pooling(data=None, global_pool=_Null, cudnn_off=_Null, kernel=_Null, pool_type=_Null, pooling_convention=_Null, stride=_Null, pad=_Null, name=None, attr=None, out=None, **kwargs)` to  `mxnet.symbol.Pooling(data=None,  kernel=_Null, pool_type=_Null, global_pool=_Null, cudnn_off=_Null, pooling_convention=_Null, stride=_Null, pad=_Null, name=None, attr=None, out=None, **kwargs)`. This is a breaking change when kwargs are not provided since the new api expects the arguments starting from `global_pool` at the fourth position instead of the second position. (#10000).\n\n### Bug Fixes\n- Fixed tests - Flakiness/Bugs - (#9598, #9951, #10259, #10197, #10136, #10422). Please see: [Tests Improvement Project](https://github.com/apache/mxnet/projects/9)\n- Fixed `cudnn_conv` and `cudnn_deconv` deadlock (#10392).\n- Fixed a race condition in `io.LibSVMIter` when batch size is large (#10124).\n- Fixed a race condition in converting data layouts in MKL-DNN (#9862).\n- Fixed MKL-DNN sigmoid/softrelu issue (#10336).\n- Fixed incorrect indices generated by device row sparse pull (#9887).\n- Fixed cast storage support for same stypes (#10400).\n- Fixed uncaught exception for bucketing module when symbol name not specified (#10094).\n- Fixed regression output layers (#9848).\n- Fixed crash with `mx.nd.ones` (#10014).\n- Fixed `sample_multinomial` crash when `get_prob=True` (#10413).\n- Fixed buggy type inference in correlation (#10135).\n- Fixed race condition for `CPUSharedStorageManager->Free` and launched workers at iter init stage to avoid frequent relaunch (#10096).\n- Fixed DLTensor Conversion for int64 (#10083).\n- Fixed issues where hex symbols of the profiler were not being recognized by chrome tracing tool(#9932)\n- Fixed crash when profiler was not enabled (#10306)\n- Fixed ndarray assignment issues (#10022, #9981, #10468).\n- Fixed incorrect indices generated by device row sparse pull (#9887).\n- Fixed `print_summary` bug in visualization module (#9492).\n- Fixed shape mismatch in accuracy metrics (#10446).\n- Fixed random samplers from uniform and random distributions in R bindings (#10450).\n- Fixed a bug that was causing training metrics to be printed as NaN sometimes (#10437).\n- Fixed a crash with non positive reps for tile ops (#10417).\n\n### Performance Improvements\n- On average, after the MKL-DNN change, the inference speed of MXNet + MKLDNN outperforms MXNet + OpenBLAS by a factor of 32, outperforms MXNet + MKLML by 82% and outperforms MXNet + MKLML with the experimental flag by 8%. The experiments were run for the image classifcation example, for different networks and different batch sizes.\n- Improved sparse SGD, sparse AdaGrad and sparse Adam optimizer speed on GPU by 30x (#9561, #10312, #10293, #10062).\n- Improved `sparse.retain` performance on CPU by 2.5x (#9722)\n- Replaced `std::swap_ranges` with memcpy (#10351)\n- Implemented DepthwiseConv2dBackwardFilterKernel which is over 5x faster (#10098)\n- Implemented CPU LSTM Inference (#9977)\n- Added Layer Normalization in C++ (#10029)\n- Optimized Performance for rtc (#10018)\n- Improved CPU performance of  ROIpooling operator by using OpenMP (#9958)\n- Accelerated the calculation of F1 (#9833)\n\n### API Changes\n- `Block.save_params` now match parameters according to model structure instead of names to avoid prefix mismatching problems during saving and loading (#10511).\n- Added an optional argument `ctx` to `mx.random.seed`. Seeding with `ctx` option produces random number sequence independent of device id. (#10367).\n- Added copy flag for astype (#10347).\n- Added context parameter to Scala Infer API - ImageClassifier and ObjectDetector (#10252).\n- Added axes support for dropout in gluon (#10032).\n- Added default `ctx` to cpu for `gluon.Block.load_params` (#10160).\n- Added support for variable sequence length in gluon.RecurrentCell (#9934).\n- Added convenience fluent method for squeeze op (#9734).\n- Made `array.reshape` compatible with numpy (#9790).\n- Added axis support and gradient for L2norm (#9740).\n\n### Sparse Support\n- Added support for multi-GPU training with `row_sparse` weights using `device` KVStore (#9987).\n- Added `Module.prepare` API for multi-GPU and multi-machine training with row_sparse weight (#10285).\n- Added `deterministic` option for `contrib.SparseEmbedding` operator (#9846).\n- Added `sparse.broadcast_mul` and `sparse.broadcast_div` with CSRNDArray and 1-D dense NDArray on CPU (#10208).\n- Added sparse support for Custom Operator (#10374).\n- Added Sparse feature for Perl (#9988).\n- Added `force_deterministic` option for sparse embedding (#9882).\n- Added `sparse.where` with condition being csr ndarray (#9481).\n\n### Deprecations\n- Deprecated `profiler_set_state` (#10156).\n\n### Other Features\n- Added constant parameter for gluon (#9893).\n- Added `contrib.rand.zipfian` (#9747).\n- Added Gluon PreLU, ELU, SELU, Swish activation layers for Gluon (#9662)\n- Added Squeeze Op (#9700).\n- Added multi-proposal operator (CPU version) and fixed bug in multi-proposal operator (GPU version) (#9939).\n- Added in Large-Batch SGD with a warmup, and a LARS startegy (#8918).\n- Added Language Modelling datasets and Sampler (#9514).\n- Added instance norm and reflection padding to Gluon (#7938).\n- Added micro-averaging strategy for F1 metric (#9777).\n- Added Softsign Activation Function (#9851).\n- Added eye operator, for default storage type (#9770).\n- Added TVM bridge support to JIT NDArray Function by TVM (#9880).\n- Added float16 support for correlation operator and L2Normalization operator (#10125, #10078).\n- Added random shuffle implementation for NDArray (#10048).\n- Added load from buffer functions for CPP package (#10261).\n\n### Usability Improvements\n- Added embedding learning example for Gluon (#9165).\n- Added tutorial on how to use data augmenters (#10055).\n- Added tutorial for Data Augmentation with Masks (#10178).\n- Added LSTNet example (#9512).\n- Added MobileNetV2 example (#9614).\n- Added tutorial for Gluon Datasets and DataLoaders (#10251).\n- Added Language model with Google's billion words dataset (#10025).\n- Added example for custom operator using RTC (#9870).\n- Improved image classification examples (#9799, #9633).\n- Added reshape predictor function to c_predict_api (#9984).\n- Added guide for implementing sparse ops (#10081).\n- Added naming tutorial for gluon blocks and parameters (#10511).\n\n### Known Issues\n- MXNet crash when built with `USE_GPERFTOOLS = 1` (#8968).\n- [DevGuide.md](https://github.com/google/googletest/blob/ec44c6c1675c25b9827aacd08c02433cccde7780/googlemock/docs/DevGuide.md) in the 3rdparty submodule googletest licensed under CC-BY-2.5.\n- Incompatibility in the behavior of MXNet Convolution operator for certain unsupported use cases: Raises an exception when MKLDNN is enabled, fails silently when it is not.\n- MXNet convolution generates wrong results for 1-element strides (#10689).\n- [Tutorial on fine-tuning an ONNX model](https://github.com/apache/mxnet/blob/v1.2.0/docs/tutorials/onnx/fine_tuning_gluon.md) fails when using cpu context.\n- CMake build ignores the `USE_MKLDNN` flag and doesn't build with MKLDNN support even with `-DUSE_MKLDNN=1`. To workaround the issue please see: #10801.\n- Linking the dmlc-core library fails with CMake build when building with `USE_OPENMP=OFF`. To workaround the issue, please use the updated CMakeLists in dmlc-core unit tests directory: https://github.com/dmlc/dmlc-core/pull/396. You can also workaround the issue by using make instead of cmake when building with `USE_OPENMP=OFF`.\n\nFor more information and examples, see [full release notes](https://cwiki.apache.org/confluence/display/MXNET/%5BWIP%5D+Apache+MXNet+%28incubating%29+1.2.0+Release+Notes)\n\n## 1.1.0\n### Usability Improvements\n- Improved the usability of examples and tutorials\n### Bug-fixes\n- Fixed I/O multiprocessing for too many open file handles (#8904), race condition (#8995), deadlock (#9126).\n- Fixed image IO integration with OpenCV 3.3 (#8757).\n- Fixed Gluon block printing (#8956).\n- Fixed float16 argmax when there is negative input. (#9149)\n- Fixed random number generator to ensure sufficient randomness. (#9119, #9256, #9300)\n- Fixed custom op multi-GPU scaling (#9283)\n- Fixed gradient of gather_nd when duplicate entries exist in index. (#9200)\n- Fixed overriden contexts in Module `group2ctx` option when using multiple contexts (#8867)\n- Fixed `swap_axes` operator with \"add_to\" gradient req (#9541)\n### New Features\n- Added experimental API in `contrib.text` for building vocabulary, and loading pre-trained word embeddings, with built-in support for 307 GloVe and FastText pre-trained embeddings. (#8763)\n- Added experimental structural blocks in `gluon.contrib`: `Concurrent`, `HybridConcurrent`, `Identity`. (#9427)\n- Added `sparse.dot(dense, csr)` operator (#8938)\n- Added `Khatri-Rao` operator (#7781)\n- Added `FTML` and `Signum` optimizer (#9220, #9262)\n- Added `ENABLE_CUDA_RTC` build option (#9428)\n### API Changes\n- Added zero gradients to rounding operators including `rint`, `ceil`, `floor`, `trunc`, and `fix` (#9040)\n- Added `use_global_stats` in `nn.BatchNorm` (#9420)\n- Added `axis` argument to `SequenceLast`, `SequenceMask` and `SequenceReverse` operators (#9306)\n- Added `lazy_update` option for standard `SGD` & `Adam` optimizer with `row_sparse` gradients (#9468, #9189)\n- Added `select` option in `Block.collect_params` to support regex (#9348)\n- Added support for (one-to-one and sequence-to-one) inference on explicit unrolled RNN models in R (#9022)\n### Deprecations\n- The Scala API name space is still called `ml.dmlc`. The name space is likely be changed in a future release to `org.apache` and might brake existing applications and scripts (#9579, #9324)\n### Performance Improvements\n- Improved GPU inference speed by 20% when batch size is 1 (#9055)\n- Improved `SequenceLast` operator speed (#9306)\n- Added multithreading for the class of broadcast_reduce operators on CPU (#9444)\n- Improved batching for GEMM/TRSM operators with large matrices on GPU (#8846)\n### Known Issues\n- \"Predict with pre-trained models\" tutorial is broken\n- \"example/numpy-ops/ndarray_softmax.py\" is broken\n\nFor more information and examples, see [full release notes](https://cwiki.apache.org/confluence/display/MXNET/Apache+MXNet+%28incubating%29+1.1.0+Release+Notes)\n\n\n## 1.0.0\n### Performance\n  - Enhanced the performance of `sparse.dot` operator.\n  - MXNet now automatically set OpenMP to use all available CPU cores to maximize CPU utilization when `NUM_OMP_THREADS` is not set.\n  - Unary and binary operators now avoid using OpenMP on small arrays if using OpenMP actually hurts performance due to multithreading overhead.\n  - Significantly improved performance of `broadcast_add`, `broadcast_mul`, etc on CPU.\n  - Added bulk execution to imperative mode. You can control segment size with `mxnet.engine.bulk`. As a result, the speed of Gluon in hybrid mode is improved, especially on small networks and multiple GPUs.\n  - Improved speed for `ctypes` invocation from Python frontend.\n### New Features - Gradient Compression [Experimental]\n  - Speed up multi-GPU and distributed training by compressing communication of gradients. This is especially effective when training networks with large fully-connected layers. In Gluon this can be activated with `compression_params` in Trainer.\n### New Features - Support of NVIDIA Collective Communication Library (NCCL) [Experimental]\n  - Use `kvstore=’nccl’` for (in some cases) faster training on multiple GPUs.\n  - Significantly faster than kvstore=’device’ when batch size is small.\n  - It is recommended to set environment variable `NCCL_LAUNCH_MODE` to `PARALLEL` when using NCCL version 2.1 or newer.\n### New Features - Advanced Indexing [General Availability]\n  - NDArray now supports advanced indexing (both slice and assign) as specified by the numpy standard: https://docs.scipy.org/doc/numpy-1.13.0/reference/arrays.indexing.html#combining-advanced-and-basic-indexing with the following restrictions:\n    - if key is a list type, only a list of integers is supported, e.g. `key=[1, 2]` is supported, while not for `key=[[1, 2]]`.\n    - Ellipsis (...) and np.newaxis are not supported.\n    - `Boolean` array indexing is not supported.\n### New Features - Gluon [General Availability]\n  - Performance optimizations discussed above.\n  - Added support for loading data in parallel with multiple processes to `gluon.data.DataLoader`. The number of workers can be set with `num_worker`. Does not support windows yet.\n  - Added Block.cast to support networks with different data types, e.g. `float16`.\n  - Added Lambda block for wrapping a user defined function as a block.\n  - Generalized `gluon.data.ArrayDataset` to support arbitrary number of arrays.\n### New Features - ARM / Raspberry Pi support [Experimental]\n  - MXNet now compiles and runs on ARMv6, ARMv7, ARMv64 including Raspberry Pi devices. See https://github.com/apache/mxnet/tree/master/docker_multiarch for more information.\n### New Features - NVIDIA Jetson support [Experimental]\n  - MXNet now compiles and runs on NVIDIA Jetson TX2 boards with GPU acceleration.\n  - You can install the python MXNet package on a Jetson board by running - `$ pip install mxnet-jetson-tx2`.\n### New Features - Sparse Tensor Support [General Availability]\n  - Added more sparse operators: `contrib.SparseEmbedding`, `sparse.sum` and `sparse.mean`.\n  - Added `asscipy()` for easier conversion to scipy.\n  - Added `check_format()` for sparse ndarrays to check if the array format is valid.\n### Bug-fixes\n  - Fixed a[-1] indexing doesn't work on `NDArray`.\n  - Fixed `expand_dims` if axis < 0.\n  - Fixed a bug that causes topk to produce incorrect result on large arrays.\n  - Improved numerical precision of unary and binary operators for `float64` data.\n  - Fixed derivatives of log2 and log10. They used to be the same with log.\n  - Fixed a bug that causes MXNet to hang after fork. Note that you still cannot use GPU in child processes after fork due to limitations of CUDA.\n  - Fixed a bug that causes `CustomOp` to fail when using auxiliary states.\n  - Fixed a security bug that is causing MXNet to listen on all available interfaces when running training in distributed mode.\n### Doc Updates\n  - Added a security best practices document under FAQ section.\n  - Fixed License Headers including restoring copyright attributions.\n  - Documentation updates.\n  - Links for viewing source.\n\n For more information and examples, see [full release notes](https://cwiki.apache.org/confluence/display/MXNET/Apache+MXNet+%28incubating%29+1.0+Release+Notes)\n\n\n## 0.12.1\n### Bug-fixes\n  - Added GPU support for the `syevd` operator which ensures that there is GPU support for all linalg-operators.\n  - Bugfix for `syevd` on CPU such that it works for `float32`.\n  - Fixed API call when `OMP_NUM_THREADS` environment variable is set.\n  - Fixed `MakeNonlossGradNode` bug.\n  - Fixed bug related to passing `dtype` to `array()`.\n  - Fixed some minor bugs for sparse distributed training.\n  - Fixed a bug on `Slice` accessing uninitialized memory in `param.begin` in the file `matrix_op-inl.h`.\n  - Fixed `gluon.data.RecordFileDataset`.\n  - Fixed a bug that caused `autograd` to crash on some networks.\n\n\n## 0.12.0\n### Performance\n  - Added full support for NVIDIA Volta GPU Architecture and CUDA 9. Training CNNs is up to 3.5x faster than Pascal when using float16 precision.\n  - Enabled JIT compilation. Autograd and Gluon hybridize now use less memory and has faster speed. Performance is almost the same with old symbolic style code.\n  - Improved ImageRecordIO image loading performance and added indexed RecordIO support.\n  - Added better openmp thread management to improve CPU performance.\n### New Features - Gluon\n  - Added enhancements to the Gluon package, a high-level interface designed to be easy to use while keeping most of the flexibility of low level API. Gluon supports both imperative and symbolic programming, making it easy to train complex models imperatively with minimal impact on performance. Neural networks (and other machine learning models) can be defined and trained with `gluon.nn` and `gluon.rnn` packages.\n  - Added new loss functions - `SigmoidBinaryCrossEntropyLoss`, `CTCLoss`, `HuberLoss`, `HingeLoss`, `SquaredHingeLoss`, `LogisticLoss`, `TripletLoss`.\n  - `gluon.Trainer` now allows reading and setting learning rate with `trainer.learning_rate` property.\n  - Added API `HybridBlock.export` for exporting gluon models to MXNet format.\n  - Added `gluon.contrib` package.\n    - Convolutional recurrent network cells for RNN, LSTM and GRU.\n    - `VariationalDropoutCell`\n### New Features - Autograd\n  - Added enhancements to `autograd` package, which enables automatic differentiation of NDArray operations.\n  - `autograd.Function` allows defining both forward and backward computation for custom operators.\n  - Added `mx.autograd.grad` and experimental second order gradient support (most operators don't support second order gradient yet).\n  - Autograd now supports cross-device graphs. Use `x.copyto(mx.gpu(i))` and `x.copyto(mx.cpu())` to do computation on multiple devices.\n### New Features - Sparse Tensor Support\n  - Added support for sparse matrices.\n  - Added limited cpu support for two sparse formats in `Symbol` and `NDArray` - `CSRNDArray` and `RowSparseNDArray`.\n  - Added a sparse dot product operator and many element-wise sparse operators.\n  - Added a data iterator for sparse data input - `LibSVMIter`.\n  - Added three optimizers for sparse gradient updates: `Ftrl`, `SGD` and `Adam`.\n  - Added `push` and `row_sparse_pull` with `RowSparseNDArray` in distributed kvstore.\n### Other New Features\n  - Added limited support for fancy indexing, which allows you to very quickly access and modify complicated subsets of an array's values. `x[idx_arr0, idx_arr1, ..., idx_arrn]` is now supported. Features such as combining and slicing are planned for the next release. Checkout master to get a preview.\n  - Random number generators in `mx.nd.random.*` and `mx.sym.random.*` now support both CPU and GPU.\n  - `NDArray` and `Symbol` now supports \"fluent\" methods. You can now use `x.exp()` etc instead of `mx.nd.exp(x)` or `mx.sym.exp(x)`.\n  - Added `mx.rtc.CudaModule` for writing and running CUDA kernels from python.\n  - Added `multi_precision` option to optimizer for easier float16 training.\n  - Better support for IDE auto-completion. IDEs like PyCharm can now correctly parse mxnet operators.\n### API Changes\n  - Operators like `mx.sym.linalg_*` and `mx.sym.random_*` are now moved to `mx.sym.linalg.*` and `mx.sym.random.*`. The old names are still available but deprecated.\n  - `sample_*` and `random_*` are now merged as `random.*`, which supports both scalar and  `NDArray` distribution parameters.\n### Bug-fixes\n  - Fixed a bug that causes `argsort` operator to fail on large tensors.\n  - Fixed numerical stability issues when summing large tensors.\n  - Fixed a bug that causes arange operator to output wrong results for large ranges.\n  - Improved numerical precision for unary and binary operators on `float64` inputs.\n\nFor more information and examples, see [full release notes](https://cwiki.apache.org/confluence/display/MXNET/MXNet+0.12.0+Release+Notes)\n\n\n## 0.11.0\n### Major Features\n  - Apple Core ML model converter\n  - Support for Keras v1.2.2\n  - For more information see [full release notes](https://cwiki.apache.org/confluence/display/MXNET/v0.11.0+Release+Notes)\n### API Changes\n  - Added `CachedOp`. You can now cache the operators that’s called frequently with the same set of arguments to reduce overhead.\n  - Added sample_multinomial for sampling from multinomial distributions.\n  - Added `trunc` operator for rounding towards zero.\n  - Added linalg_gemm, linalg_potrf, ... operators for lapack support.\n  - Added verbose option to Initializer for printing out initialization details.\n  - Added DeformableConvolution to contrib from the Deformable Convolutional Networks paper.\n  - Added float64 support for dot and batch_dot operator.\n  - `allow_extra` is added to Module.set_params to ignore extra parameters.\n  - Added `mod` operator for modulo.\n  - Added `multi_precision` option to SGD optimizer to improve training with float16. Resnet50 now achieves the same accuracy when trained with float16 and gives 50% speedup on Titan XP.\n### Performance Improvements\n  - ImageRecordIter now stores data in pinned memory to improve GPU memcopy speed.\n### Bugfixes\n  - Cython interface is fixed. `make cython` and `python setup.py install --with-cython` should install the cython interface and reduce overhead in applications that use imperative/bucketing.\n  - Fixed various bugs in Faster-RCNN example: https://github.com/apache/mxnet/pull/6486\n  - Fixed various bugs in SSD example.\n  - Fixed `out` argument not working for `zeros`, `ones`, `full`, etc.\n  - `expand_dims` now supports backward shape inference.\n  - Fixed a bug in rnn. BucketingSentenceIter that causes incorrect layout handling on multi-GPU.\n  - Fixed context mismatch when loading optimizer states.\n  - Fixed a bug in ReLU activation when using MKL.\n  - Fixed a few race conditions that causes crashes on shutdown.\n### Refactors\n  - Refactored TShape/TBlob to use int64 dimensions and DLTensor as internal storage. Getting ready for migration to DLPack. As a result TBlob::dev_mask_ and TBlob::stride_ are removed.\n\n\n## 0.10.0\n- Overhauled documentation for commonly used Python APIs, Installation instructions, Tutorials, HowTos and MXNet Architecture.\n- Updated mxnet.io for improved readability.\n- Pad operator now support reflection padding.\n- Fixed a memory corruption error in threadedengine.\n- Added CTC loss layer to contrib package. See mx.contrib.sym.ctc_loss.\n- Added new sampling operators for several distributions (normal,uniform,gamma,exponential,negative binomial).\n- Added documentation for experimental RNN APIs.\n\n## 0.9.3\n- Move symbolic API to NNVM @tqchen\n  - Most front-end C API are backward  compatible\n  - Removed symbolic API in MXNet and relies on NNVM\n- New features:\n  - MXNet profiler for profiling operator-level executions\n  - mxnet.image package for fast image loading and processing\n- Change of JSON format\n  - param and attr field are merged to attr\n  - New code is backward-compatible can load old json format\n- OpProperty registration now is deprecated\n  - New operators are encouraged to register their property to NNVM op registry attribute\n- Known features removed limitations to be fixed\n  - Bulk segment execution not yet added.\n\n## v0.8\nThis is the last release before the NNVM refactor.\n- CaffeOp and CaffeIter for interfacing with Caffe by @HrWangChengdu @cjolivier01\n- WrapCTC plugin for sequence learning by @xlvector\n- Improved Multi-GPU performance by @mli\n- CuDNN RNN support by @sbodenstein\n- OpenCV plugin for parallel image IO by @piiswrong\n- More operators as simple op\n    - Simple OP @tqchen\n    - element wise op with axis and broadcast @mli @sxjscience\n- Cudnn auto tuning for faster convolution by @piiswrong\n- More applications\n    - Faster RCNN by @precedenceguo\n\n\n## v0.7\n-  0.6 is skipped because there are a lot of improvements since initial release\n- More math operators\n  - elementwise ops and binary ops\n- Attribute support in computation graph\n  - Now user can use attributes to give various hints about specific learning rate, allocation plans etc\n- MXNet is more memory efficient\n  - Support user defined memory optimization with attributes\n- Support mobile applications by @antinucleon\n- Refreshed update of new documents\n- Model parallel training of LSTM by @tqchen\n- Simple operator refactor by @tqchen\n  - add operator_util.h to enable quick registration of both ndarray and symbolic ops\n- Distributed training by @mli\n- Support Torch Module by @piiswrong\n  - MXNet now can use any of the modules from Torch.\n- Support custom native operator by @piiswrong\n- Support data types including fp16, fp32, fp64, int32, and uint8 by @piiswrong\n- Support monitor for easy printing and debugging by @piiswrong\n- Support new module API by @pluskid\n  - Module API is a middle level API that can be used in imperative manner like Torch-Module\n- Support bucketing API for variable length input by @pluskid\n- Support CuDNN v5 by @antinucleon\n- More applications\n  - Speech recognition by @yzhang87\n  - [Neural art](https://github.com/apache/mxnet/tree/v0.7.0/example/neural-style) by @antinucleon\n  - [Detection](https://github.com/apache/mxnet/tree/v0.7.0/example/rcnn), RCNN bt @precedenceguo\n  - [Segmentation](https://github.com/apache/mxnet/tree/v0.7.0/example/fcn-xs), FCN by @tornadomeet\n  - [Face identification](https://github.com/tornadomeet/mxnet-face) by @tornadomeet\n  - More on the example\n\n## v0.5 (initial release)\n- All basic modules ready\n"
  },
  {
    "path": "NOTICE",
    "content": "    Apache MXNET\n    Copyright 2017-2023 The Apache Software Foundation\n\n    This product includes software developed at\n    The Apache Software Foundation (http://www.apache.org/).\n\n\n    --------------------------------------------------------------------------------\n    This product includes code from Apache TVM, which includes the\n    following in its NOTICE file:\n\n        Apache TVM (incubating)\n        Copyright 2019-2020 The Apache Software Foundation\n\n        This product includes software developed at\n        The Apache Software Foundation (http://www.apache.org/).\n"
  },
  {
    "path": "README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n<div align=\"center\">\n  <a href=\"https://mxnet.apache.org/\"><img src=\"https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/image/mxnet_logo_2.png\"></a><br>\n</div>\n\n[![banner](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/image/banner.png)](https://mxnet.apache.org)\n\nApache MXNet for Deep Learning\n===========================================\n[![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/apache/mxnet)](https://github.com/apache/mxnet/releases) [![GitHub stars](https://img.shields.io/github/stars/apache/mxnet)](https://github.com/apache/mxnet/stargazers) [![GitHub forks](https://img.shields.io/github/forks/apache/mxnet)](https://github.com/apache/mxnet/network) [![GitHub contributors](https://img.shields.io/github/contributors-anon/apache/mxnet)](https://github.com/apache/mxnet/graphs/contributors) [![GitHub issues](https://img.shields.io/github/issues/apache/mxnet)](https://github.com/apache/mxnet/issues) [![good first issue](https://img.shields.io/github/issues/apache/mxnet/good%20first%20issue)](https://github.com/apache/mxnet/labels/good%20first%20issue) [![GitHub pull requests by-label](https://img.shields.io/github/issues-pr/apache/mxnet/pr-awaiting-review)](https://github.com/apache/mxnet/labels/pr-awaiting-review) [![GitHub license](https://img.shields.io/github/license/apache/mxnet)](https://github.com/apache/mxnet/blob/master/LICENSE) [![Twitter](https://img.shields.io/twitter/url?style=social&url=https%3A%2F%2Fgithub.com%2Fapache%2Fmxnet)](https://twitter.com/intent/tweet?text=Wow:%20https%3A%2F%2Fgithub.com%2Fapache%2Fmxnet%20@ApacheMXNet) [![Twitter Follow](https://img.shields.io/twitter/follow/ApacheMXNet?style=social)](https://twitter.com/ApacheMXNet)\n\nApache MXNet is a deep learning framework designed for both *efficiency* and *flexibility*.\nIt allows you to ***mix*** [symbolic and imperative programming](https://mxnet.apache.org/api/architecture/program_model)\nto ***maximize*** efficiency and productivity.\nAt its core, MXNet contains a dynamic dependency scheduler that automatically parallelizes both symbolic and imperative operations on the fly.\nA graph optimization layer on top of that makes symbolic execution fast and memory efficient.\nMXNet is portable and lightweight, scalable to many GPUs and machines.\n\nApache MXNet is more than a deep learning project. It is a [community](https://mxnet.apache.org/versions/master/community)\non a mission of democratizing AI. It is a collection of [blue prints and guidelines](https://mxnet.apache.org/api/architecture/overview)\nfor building deep learning systems, and interesting insights of DL systems for hackers.\n\nLicensed under an [Apache-2.0](https://github.com/apache/mxnet/blob/master/LICENSE) license.\n\n| Branch  | Build Status  |\n|:-------:|:-------------:|\n| [master](https://github.com/apache/mxnet/tree/master) | [![CentOS CPU Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/centos-cpu/job/master/badge/icon?subject=build%20centos%20cpu)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/centos-cpu/job/master/) [![CentOS GPU Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/centos-gpu/job/master/badge/icon?subject=build%20centos%20gpu)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/centos-gpu/job/master/) [![Clang Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/clang/job/master/badge/icon?subject=build%20clang)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/clang/job/master/) <br> [![Edge Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/edge/job/master/badge/icon?subject=build%20edge)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/edge/job/master/) [![Miscellaneous Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/miscellaneous/job/master/badge/icon?subject=build%20miscellaneous)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/miscellaneous/job/master/) [![Sanity Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/sanity/job/master/badge/icon?subject=build%20sanity)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/sanity/job/master/) <br> [![Unix CPU Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/unix-cpu/job/master/badge/icon?subject=build%20unix%20cpu)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/unix-cpu/job/master/) [![Unix GPU Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/unix-gpu/job/master/badge/icon?subject=build%20unix%20gpu)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/unix-gpu/job/master/) [![Website Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/website/job/master/badge/icon?subject=build%20website)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/website/job/master/) <br> [![Windows CPU Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/windows-cpu/job/master/badge/icon?subject=build%20windows%20cpu)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/windows-cpu/job/master/) [![Windows GPU Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/windows-gpu/job/master/badge/icon?subject=build%20windows%20gpu)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/windows-gpu/job/master/) [![Documentation Status](http://jenkins.mxnet-ci.com/job/restricted-website-build/badge/icon)](https://mxnet.apache.org/) |\n| [v1.x](https://github.com/apache/mxnet/tree/v1.x) | [![CentOS CPU Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/centos-cpu/job/v1.x/badge/icon?subject=build%20centos%20cpu)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/centos-cpu/job/v1.x/) [![CentOS GPU Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/centos-gpu/job/v1.x/badge/icon?subject=build%20centos%20gpu)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/centos-gpu/job/v1.x/) [![Clang Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/clang/job/v1.x/badge/icon?subject=build%20clang)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/clang/job/v1.x/) <br> [![Edge Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/edge/job/v1.x/badge/icon?subject=build%20edge)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/edge/job/v1.x/) [![Miscellaneous Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/miscellaneous/job/v1.x/badge/icon?subject=build%20miscellaneous)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/miscellaneous/job/v1.x/) [![Sanity Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/sanity/job/v1.x/badge/icon?subject=build%20sanity)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/sanity/job/v1.x/) <br> [![Unix CPU Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/unix-cpu/job/v1.x/badge/icon?subject=build%20unix%20cpu)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/unix-cpu/job/v1.x/) [![Unix GPU Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/unix-gpu/job/v1.x/badge/icon?subject=build%20unix%20gpu)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/unix-gpu/job/v1.x/) [![Website Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/website/job/v1.x/badge/icon?subject=build%20website)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/website/job/v1.x/) <br> [![Windows CPU Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/windows-cpu/job/v1.x/badge/icon?subject=build%20windows%20cpu)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/windows-cpu/job/v1.x/) [![Windows GPU Build Status](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/windows-gpu/job/v1.x/badge/icon?subject=build%20windows%20gpu)](http://jenkins.mxnet-ci.com/job/mxnet-validation/job/windows-gpu/job/v1.x/) [![Documentation Status](http://jenkins.mxnet-ci.com/job/restricted-website-build/badge/icon)](https://mxnet.apache.org/) |\n\nFeatures\n--------\n* NumPy-like programming interface, and is integrated with the new, easy-to-use Gluon 2.0 interface. NumPy users can easily adopt MXNet and start in deep learning.\n* Automatic hybridization provides imperative programming with the performance of traditional symbolic programming.\n* Lightweight, memory-efficient, and portable to smart devices through native cross-compilation support on ARM, and through ecosystem projects such as [TVM](https://tvm.ai), [TensorRT](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html), [OpenVINO](https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit.html).\n* Scales up to multi GPUs and distributed setting with auto parallelism through [ps-lite](https://github.com/dmlc/ps-lite), [Horovod](https://github.com/horovod/horovod), and [BytePS](https://github.com/bytedance/byteps).\n* Extensible backend that supports full customization, allowing integration with custom accelerator libraries and in-house hardware without the need to maintain a fork.\n* Support for [Python](https://mxnet.apache.org/api/python), [Java](https://mxnet.apache.org/api/java), [C++](https://mxnet.apache.org/api/cpp), [R](https://mxnet.apache.org/api/r), [Scala](https://mxnet.apache.org/api/scala), [Clojure](https://mxnet.apache.org/api/clojure), [Go](https://github.com/jdeng/gomxnet/), [Javascript](https://github.com/dmlc/mxnet.js/), [Perl](https://mxnet.apache.org/api/perl), and [Julia](https://mxnet.apache.org/api/julia).\n* Cloud-friendly and directly compatible with AWS and Azure.\n\nContents\n--------\n* [Installation](https://mxnet.apache.org/get_started)\n* [Tutorials](https://mxnet.apache.org/api/python/docs/tutorials/)\n* [Ecosystem](https://mxnet.apache.org/ecosystem)\n* [API Documentation](https://mxnet.apache.org/api)\n* [Examples](https://github.com/apache/mxnet-examples)\n* [Stay Connected](#stay-connected)\n* [Social Media](#social-media)\n\nWhat's New\n----------\n* [1.9.1 Release](https://github.com/apache/mxnet/releases/tag/1.9.1) - MXNet 1.9.1 Release.\n* [1.8.0 Release](https://github.com/apache/mxnet/releases/tag/1.8.0) - MXNet 1.8.0 Release.\n* [1.7.0 Release](https://github.com/apache/mxnet/releases/tag/1.7.0) - MXNet 1.7.0 Release.\n* [1.6.0 Release](https://github.com/apache/mxnet/releases/tag/1.6.0) - MXNet 1.6.0 Release.\n* [1.5.1 Release](https://github.com/apache/mxnet/releases/tag/1.5.1) - MXNet 1.5.1 Patch Release.\n* [1.5.0 Release](https://github.com/apache/mxnet/releases/tag/1.5.0) - MXNet 1.5.0 Release.\n* [1.4.1 Release](https://github.com/apache/mxnet/releases/tag/1.4.1) - MXNet 1.4.1 Patch Release.\n* [1.4.0 Release](https://github.com/apache/mxnet/releases/tag/1.4.0) - MXNet 1.4.0 Release.\n* [1.3.1 Release](https://github.com/apache/mxnet/releases/tag/1.3.1) - MXNet 1.3.1 Patch Release.\n* [1.3.0 Release](https://github.com/apache/mxnet/releases/tag/1.3.0) - MXNet 1.3.0 Release.\n* [1.2.0 Release](https://github.com/apache/mxnet/releases/tag/1.2.0) - MXNet 1.2.0 Release.\n* [1.1.0 Release](https://github.com/apache/mxnet/releases/tag/1.1.0) - MXNet 1.1.0 Release.\n* [1.0.0 Release](https://github.com/apache/mxnet/releases/tag/1.0.0) - MXNet 1.0.0 Release.\n* [0.12.1 Release](https://github.com/apache/mxnet/releases/tag/0.12.1) - MXNet 0.12.1 Patch Release.\n* [0.12.0 Release](https://github.com/apache/mxnet/releases/tag/0.12.0) - MXNet 0.12.0 Release.\n* [0.11.0 Release](https://github.com/apache/mxnet/releases/tag/0.11.0) - MXNet 0.11.0 Release.\n* [Apache Incubator](http://incubator.apache.org/projects/mxnet.html) - We are now an Apache Incubator project.\n* [0.10.0 Release](https://github.com/apache/mxnet/releases/tag/v0.10.0) - MXNet 0.10.0 Release.\n* [0.9.3 Release](./docs/architecture/release_note_0_9.md) - First 0.9 official release.\n* [0.9.1 Release (NNVM refactor)](./docs/architecture/release_note_0_9.md) - NNVM branch is merged into master now. An official release will be made soon.\n* [0.8.0 Release](https://github.com/apache/mxnet/releases/tag/v0.8.0)\n\n### Ecosystem News\n\n* [oneDNN for Faster CPU Performance](docs/python_docs/python/tutorials/performance/backend/dnnl/dnnl_readme.md)\n* [MXNet Memory Monger, Training Deeper Nets with Sublinear Memory Cost](https://github.com/dmlc/mxnet-memonger)\n* [Tutorial for NVidia GTC 2016](https://github.com/dmlc/mxnet-gtc-tutorial)\n* [MXNet.js: Javascript Package for Deep Learning in Browser (without server)](https://github.com/dmlc/mxnet.js/)\n* [Guide to Creating New Operators (Layers)](https://mxnet.apache.org/api/faq/new_op)\n* [Go binding for inference](https://github.com/songtianyi/go-mxnet-predictor)\n\nStay Connected\n--------------\n\n| Channel | Purpose |\n|---|---|\n| [Follow MXNet Development on Github](https://github.com/apache/mxnet/issues) | See what's going on in the MXNet project. |\n| [MXNet Confluence Wiki for Developers](https://cwiki.apache.org/confluence/display/MXNET/Apache+MXNet+Home) <i class=\"fas fa-external-link-alt\"> | MXNet developer wiki for information related to project development, maintained by contributors and developers. To request write access, send an email to [send request to the dev list](mailto:dev@mxnet.apache.org?subject=Requesting%20CWiki%20write%20access) <i class=\"far fa-envelope\"></i>. |\n| [dev@mxnet.apache.org mailing list](https://lists.apache.org/list.html?dev@mxnet.apache.org) | The \"dev list\". Discussions about the development of MXNet. To subscribe, send an email to [dev-subscribe@mxnet.apache.org](mailto:dev-subscribe@mxnet.apache.org) <i class=\"far fa-envelope\"></i>. |\n| [discuss.mxnet.io](https://discuss.mxnet.io) <i class=\"fas fa-external-link-alt\"></i> | Asking & answering MXNet usage questions. |\n| [Apache Slack #mxnet Channel](https://the-asf.slack.com/archives/C7FN4FCP9) <i class=\"fas fa-external-link-alt\"> | Connect with MXNet and other Apache developers. To join the MXNet slack channel [send request to the dev list](mailto:dev@mxnet.apache.org?subject=Requesting%20slack%20access) <i class=\"far fa-envelope\"></i>. |\n| [Follow MXNet on Social Media](#social-media) | Get updates about new features and events. |\n\n\n### Social Media\n\nKeep connected with the latest MXNet news and updates.\n\n<p>\n<a href=\"https://twitter.com/apachemxnet\"><img src=\"https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/social/twitter.svg?sanitize=true\" height=\"30px\"/> Apache MXNet on Twitter</a>\n</p>\n<p>\n<a href=\"https://medium.com/apache-mxnet\"><img src=\"https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/social/medium_black.svg?sanitize=true\" height=\"30px\"/> Contributor and user blogs about MXNet</a>\n</p>\n<p>\n<a href=\"https://reddit.com/r/mxnet\"><img src=\"https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/social/reddit_blue.svg?sanitize=true\" height=\"30px\" alt=\"reddit\"/> Discuss MXNet on r/mxnet</a>\n</p>\n<p>\n<a href=\"https://www.youtube.com/apachemxnet\"><img src=\"https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/social/youtube_red.svg?sanitize=true\" height=\"30px\"/> Apache MXNet YouTube channel</a>\n</p>\n<p>\n<a href=\"https://www.linkedin.com/company/apache-mxnet\"><img src=\"https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/social/linkedin.svg?sanitize=true\" height=\"30px\"/> Apache MXNet on LinkedIn</a>\n</p>\n\n\nHistory\n-------\nMXNet emerged from a collaboration by the authors of [cxxnet](https://github.com/dmlc/cxxnet), [minerva](https://github.com/dmlc/minerva), and [purine2](https://github.com/purine/purine2). The project reflects what we have learned from the past projects. MXNet combines aspects of each of these projects to achieve flexibility, speed, and memory efficiency.\n\nTianqi Chen, Mu Li, Yutian Li, Min Lin, Naiyan Wang, Minjie Wang, Tianjun Xiao,\nBing Xu, Chiyuan Zhang, and Zheng Zhang.\n[MXNet: A Flexible and Efficient Machine Learning Library for Heterogeneous Distributed Systems](https://github.com/dmlc/web-data/raw/master/mxnet/paper/mxnet-learningsys.pdf).\nIn Neural Information Processing Systems, Workshop on Machine Learning Systems, 2015\n"
  },
  {
    "path": "SECURITY.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# Security Policy\n\n## Reporting a Vulnerability\nThe Apache Software Foundation takes a very active stance in eliminating security problems and denial of service attacks against its products.\n\nWe strongly encourage folks to report such problems to our private security mailing list first, before disclosing them in a public forum.\n\nFor instructions how to report a security vulnerability, please consult our [security guide](https://mxnet.apache.org/api/faq/security).\n"
  },
  {
    "path": "benchmark/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n"
  },
  {
    "path": "benchmark/opperf/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# MXNet Operator Performance Benchmarks\n\nA Python utility for benchmarking and profiling individual MXNet operator execution.\n\nWith this utility, for each MXNet operator you can get the following details:\n\n**Timing**\n1. Forward execution time\n2. Backward execution time\n\n**Memory**\n1. Average and Max memory allocated\n\nNOTE: This is the `pool memory`. It does not reflect the exact memory requested by the operator.\n\n# Motivation\n\nBenchmarks are usually done end-to-end for a given Network Architecture. For example: ResNet-50 benchmarks on ImageNet data. This is good measurement of overall performance and health of a deep learning framework. However, it is important to note the following important factors:\n1. Users use a lot more operators that are not part of a standard network like ResNet. Example: Tensor manipulation operators like mean, max, topk, argmax, sort etc.   \n2. A standard Network Architecture like ResNet-50 is made up of many operators Ex: Convolution2D, Softmax, Dense and more. Consider the following scenarios:\n    1. We improved the performance of Convolution2D operator, but due to a bug, Softmax performance went down. Overall, we may observe end to end benchmarks are running fine, we may miss out the performance degradation of a single operator which can accumulate and become untraceable.\n    2. You need to see in a given network, which operator is taking maximum time and plan optimization work. With end to end benchmarks, it is hard to get more fine grained numbers at operator level.\n3. We need to know on different hardware infrastructure (Ex: CPU with oneDNN, GPU with NVIDIA CUDA and cuDNN) how different operators performs. With these details, we can plan the optimization work at operator level, which could exponentially boost up end to end performance.\n4. You want to have nightly performance tests across all operators in a deep learning framework to catch regressions early. \n5. We can integrate this framework with a CI/CD system to run per operator performance tests for PRs. Example: When a PR modifies the kernel of TransposeConv2D, we can run benchmarks of TransposeConv2D operator to verify performance.\n\nHence, in this utility, we will build the functionality to allow users and developers of deep learning frameworks to easily run benchmarks for individual operators.\n\n# How to use\n\n## Prerequisites\n\nProvided you have MXNet installed (any version >= 1.5.1), all you need to use opperf utility is to add path to your cloned MXNet repository to the PYTHONPATH.\n\nNote: \n1. Currently, opperf utility requires a cloned mxnet repo. It isn't supported on PyPi binary yet. [Work in Progress]\n2. To install MXNet, refer [Installing MXNet page](https://mxnet.apache.org/versions/master/install/index.html)\n\n```\nexport PYTHONPATH=$PYTHONPATH:/path/to/mxnet/\n```\n\n## Usecase 1 - Run benchmarks for all the operators\n\nBelow command runs all the MXNet operators (NDArray) benchmarks with default inputs and saves the final result as JSON in the given file.\n\n```\npython mxnet/benchmark/opperf/opperf.py --output-format json --output-file mxnet_operator_benchmark_results.json\n```\n\n**Other Supported Options:**\n\n1. **output-format** : `json` or `md` for markdown file output.\n\n2. **ctx** : `cpu` or `gpu`. By default, cpu on CPU machine, gpu(0) on GPU machine. You can override and set the global context for all operator benchmarks. Example: --ctx gpu(2).\n\n3. **dtype** : By default, `float32`. You can override and set the global dtype for all operator benchmarks. Example: --dtype float64.\n\n4. **profiler** : `native` or `python`. By default, 'native'. You can override and set the global profiler for all operator benchmarks. Example: --profiler 'python'.\nNative profiler uses MXNet C++ based built-in profiler. Python profiler uses Python package time. Generally, native profiler is used by developers and python profiler is used by users.\n\n5. **int64-tensor** : `on` or `off`. By default, 'off'. You can override and set the large tensor flag to ON. Example: --int64-tensor ON\n\n## Usecase 2 - Run benchmarks for all the operators in a specific category\n\nFor example, you want to run benchmarks for all NDArray Broadcast Binary Operators, Ex: broadcast_add, broadcast_mod, broadcast_pow etc., You just run the following python script.\n\n```\n#!/usr/bin/python\nfrom benchmark.opperf.nd_operations.binary_operators import run_mx_binary_broadcast_operators_benchmarks\n\n# Run all Binary Broadcast operations benchmarks with default input values\nprint(run_mx_binary_broadcast_operators_benchmarks())\n```\n\nOutput for the above benchmark run, on a CPU machine, would look something like below:\n\n```\n{'broadcast_mod': [{'avg_time_forward_broadcast_mod': 28.7063, 'avg_time_mem_alloc_cpu/0': 4194.3042,\n                    'avg_time_backward_broadcast_mod': 12.0954, 'inputs': {'lhs': (1024, 1024), 'rhs': (1024, 1024)}},\n                   {'avg_time_forward_broadcast_mod': 2.7332, 'avg_time_mem_alloc_cpu/0': 400.0,\n                    'avg_time_backward_broadcast_mod': 1.1288, 'inputs': {'lhs': (10000, 10), 'rhs': (10000, 10)}},\n                   {'avg_time_forward_broadcast_mod': 30.5322, 'avg_time_mem_alloc_cpu/0': 4000.0,\n                    'avg_time_backward_broadcast_mod': 225.0255, 'inputs': {'lhs': (10000, 1), 'rhs': (10000, 100)}}],\n 'broadcast_power': [{'avg_time_backward_broadcast_power': 49.5871, 'avg_time_forward_broadcast_power': 18.0954,\n                      'avg_time_mem_alloc_cpu/0': 4194.3042, 'inputs': {'lhs': (1024, 1024), 'rhs': (1024, 1024)}},\n                     {'avg_time_backward_broadcast_power': 4.6623, 'avg_time_forward_broadcast_power': 1.8283,\n                      'avg_time_mem_alloc_cpu/0': 400.0, 'inputs': {'lhs': (10000, 10), 'rhs': (10000, 10)}},\n                     {'avg_time_backward_broadcast_power': 279.922, 'avg_time_forward_broadcast_power': 24.4621,\n                      'avg_time_mem_alloc_cpu/0': 4000.0, 'inputs': {'lhs': (10000, 1), 'rhs': (10000, 100)}}],\n.....\n.....                      \n```\n\n## Usecase 3 - Run benchmarks for specific operator\nFor example, you want to run benchmarks for `nd.add` operator in MXNet, you just run the following python script.\n\n```\n#!/usr/bin/python\nimport mxnet as mx\nfrom mxnet import nd\n\nfrom benchmark.opperf.utils.benchmark_utils import run_performance_test\n\nadd_res = run_performance_test(nd.add, run_backward=True, dtype='float32', ctx=mx.cpu(),\n                               inputs=[{\"lhs\": (1024, 1024),\n                                        \"rhs\": (1024, 1024)}],\n                               warmup=10, runs=25)\nprint(add_res)\n```\n\nOutput for the above benchmark run, on a CPU machine, would look something like below:\n\n```\n{'add': [{'avg_time_mem_alloc_cpu/0': 102760.4453,\n          'avg_time_forward_broadcast_add': 4.0372,\n          'avg_time_backward_broadcast_add': 5.3841,\n          'inputs': {'lhs': (1024, 1024), 'rhs': (1024, 1024)}}]}\n\n```\n\n## Usecase 4 - Run benchmarks for group of operators with same input\nFor example, you want to run benchmarks for `nd.add`, `nd.sub` operator in MXNet, with the same set of inputs. You just run the following python script.\n\n```\n#!/usr/bin/python\nimport mxnet as mx\nfrom mxnet import nd\n\nfrom benchmark.opperf.utils.benchmark_utils import run_performance_test\n\nadd_res = run_performance_test([nd.add, nd.subtract], run_backward=True, dtype='float32', ctx=mx.cpu(),\n                               inputs=[{\"lhs\": (1024, 1024),\n                                        \"rhs\": (1024, 1024)}],\n                               warmup=10, runs=25)\nprint(add_res)\n```\n\nOutput for the above benchmark run, on a CPU machine, would look something like below:\n\n```\n{'add': [{'avg_time_mem_alloc_cpu/0': 102760.4453,\n          'avg_time_forward_broadcast_add': 4.0372,\n          'avg_time_backward_broadcast_add': 5.3841,\n          'inputs': {'lhs': (1024, 1024), 'rhs': (1024, 1024)}}],\n'subtract': [{'avg_time_forward_broadcast_sub': 5.5137, \n               'avg_time_mem_alloc_cpu/0': 207618.0469,\n               'avg_time_backward_broadcast_sub': 7.2976, \n               'inputs': {'lhs': (1024, 1024), 'rhs': (1024, 1024)}}\n             ]}\n\n```\n\n## Usecase 5 - Profile internal operators locally\nCurrently, opperf supports operators in `mx.nd.*` namespace.\nHowever, locally, one can profile internal operators in `mx.nd.internal.*` namespace.\n\n## Usecase 6 - Compare performance for chosen operator from both NDArray library and its Numpy/Numpy_extension counterpart\nFor example, you want to compare add operator from `mx.nd` and `mx.np`. You just run the following python script.\n\n```\n#!/usr/bin/python\nfrom benchmark.opperf.utils.benchmark_utils import run_benchmark_operator\n\nrun_benchmark_operator(name = \"add\", run_backward=True)\n```\n\nOutput for the above benchmark run, on a CPU machine, would look something like below:\n\n```\n<module 'mxnet.ndarray'>\n[{'add': [{'inputs': {'lhs': (128, 128), 'rhs': (128, 128)},\n           'max_storage_mem_alloc_cpu/0': 32.768,\n           'avg_time_forward_add': 0.0496,\n           'avg_time_backward_add': 0.0793}]}]\n<module 'mxnet.numpy'>\n[{'add': [{'inputs': {'x1': (128, 128), 'x2': (128, 128)},\n           'max_storage_mem_alloc_cpu/0': 32.768,\n           'avg_time_forward_add': 0.0484,\n           'avg_time_backward_add': 0.0898}]}]\n\n```\nThis function uses `run_performance_test` function mentioned in Usecase 3 and Usecase 4 and it is possible to change all parameters from it.\nAll arguments that are of type NDArray will be automatically provided with shape that is passed as `size`.\nIf any fuction requires more arguments or different shaped NDArrays, provide those arguments as `additional_inputs` as it is shown below:\n```\nrun_benchmark_operator(name = \"pick\", size = (128,128), additional_inputs = {\"index\": (128,1)})\n```\n\n\n#### Changes\nRemove the hasattr check for `op.__name__` to be in `mx.nd`\n\nThe resulting diff would look like :\n##### Old Code\n```\n-        if hasattr(mx.nd, op.__name__):\n-            benchmark_result = _run_nd_operator_performance_test(op, inputs, run_backward, warmup, runs, kwargs_list, profiler)\n-        else:\n-            raise ValueError(\"Unknown NDArray operator provided to benchmark. -  \", op.__name__)\n```\n##### New Code\n```\n+        #if hasattr(mx.nd, op.__name__):\n+        benchmark_result = _run_nd_operator_performance_test(op, inputs, run_backward, warmup, runs, kwargs_list, profiler)\n+        #else:\n+            #raise ValueError(\"Unknown NDArray operator provided to benchmark. -  \", op.__name__)\n```\n\n#### Result\nThis should allow profiling of any operator in MXNet provided user provides valid parameters [`inputs`, `run_backward`, etc] to the `run_performance_test` function.\n\n#### Example\nProvided the source code change is made in the `benchmark/opperf/utils/benchmark_utils.py`\n```\n>>> import mxnet as mx\n>>> from mxnet import nd\n>>> from benchmark.opperf.utils.benchmark_utils import run_performance_test\n>>> run_performance_test(mx.nd._internal._copyto,inputs=[{\"data\":mx.nd.array([1,2]),\"out\":mx.nd.empty(shape=mx.nd.array([1,2]).shape,ctx=mx.cpu())}])\nINFO:root:Begin Benchmark - _copyto\nINFO:root:Complete Benchmark - _copyto\n[{'_copyto': [{'inputs': {'data': '<NDArray 2 @cpu(0)>', 'out': '<NDArray 2 @cpu(0)>'}, 'max_storage_mem_alloc_cpu/0': 0.004}]}]\n```\n\n# How does it work under the hood?\n\nUnder the hood, executes NDArray operator using randomly generated data. Use MXNet profiler to get summary of the operator execution:\n1. Memory\n2. Computation time (forward, backward)\n\nSee the design proposal document for more details - https://cwiki.apache.org/confluence/display/MXNET/MXNet+Operator+Benchmarks \n\n**NOTE:**\n\nThis utility queries MXNet operator registry to fetch all operators registered with MXNet, generate inputs and run benchmarks.\nHowever, fully automated tests are enabled only for simpler operators such as - broadcast operators, element_wise operators etc... For the purpose of readability and giving more control to the users, complex operators such as convolution (2D, 3D), Pooling, Recurrent are not fully automated but expressed as default rules.\nSee `utils/op_registry_utils.py` for more details.\n\n## Use python timer\nOptionally, you could use the python time package as the profiler engine to caliberate runtime in each operator.\nTo use python timer for all operators, use the argument --profiler 'python':\n```\npython mxnet/benchmark/opperf/opperf.py --profiler='python'\n```\n\nTo use python timer for a specific operator, pass the argument profiler to the run_performance_test method:\n```\nadd_res = run_performance_test([nd.add, nd.subtract], run_backward=True, dtype='float32', ctx=mx.cpu(),\n                               inputs=[{\"lhs\": (1024, 1024),\n                                        \"rhs\": (1024, 1024)}],\n                               warmup=10, runs=25, profiler='python')\n```\nBy default, MXNet profiler is used as the profiler engine.\n\n\n# TODO\n\nAll contributions are welcome. Below is the list of desired features:\n\n1. ~~Cover all MXNet operators~~.\n2. Enhance MXNet profiler with additional APIs to programmatically fetch and process profiler data.\n3. Integration with CI/CD system to run operator benchmarks for PR builds, nightly builds.\n4. Dashboards and other modes of presentation of results for analyzing and planning tasks such as operator performance improvements.\n5. Randomized Tensor Shape generation for profiling to identify bottlenecks in the operators.\n"
  },
  {
    "path": "benchmark/opperf/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n"
  },
  {
    "path": "benchmark/opperf/custom_operations/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n"
  },
  {
    "path": "benchmark/opperf/custom_operations/custom_operations.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport mxnet as mx\n\n\"\"\"\nMXNet's Custom Operator Benchmark Tests.\n\nIt does a simple element wise addition to make sure computation\nis not too much and we can observe custom operator logistics overhead.\n\"\"\"\n\n\n# 1. Define Custom Operator - Element wise Addition Multiplication\nclass CustomAddOne(mx.operator.CustomOp):\n    def forward(self, is_train, req, in_data, out_data, aux):\n        self.assign(out_data[0], req[0], in_data[0] + 1)\n\n    def backward(self, req, out_grad, in_data, out_data, in_grad, aux):\n        self.assign(in_grad[0], req[0], out_grad[0])\n\n\n@mx.operator.register(\"CustomAddOne\")\nclass CustomAddOneProp(mx.operator.CustomOpProp):\n    def __init__(self):\n        super(CustomAddOneProp, self).__init__(need_top_grad=True)\n\n    def list_arguments(self):\n        return ['in']\n\n    def list_outputs(self):\n        return ['output']\n\n    def infer_shape(self, in_shape):\n        # inputs, outputs, aux\n        return [in_shape[0]], [in_shape[0]], []\n\n    def create_operator(self, ctx, shapes, dtypes):\n        return CustomAddOne()\n\n\n\"\"\"Helps to benchmark MXNet's Custom Op for Element wise addition on a (1000, 1) tensor.\n    Performs both forward and backward operation.\n\n    This test mainly uncovers core custom op overhead in MXNet.\n\n    Benchmark will be done on the following operation:\n    native_add -> native_add -> native_add -> CUSTOM_ADD -> native_add -> native_add -> native_add\n\n    By default run on 'float32' precision.\n\"\"\"\n\n# TODO\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# TODO - Operators not covered in this Benchmark Utility\n\n**NOTE:** This list is AUTOGENERATED when you run opperf.py utility\n\n0. preloaded_multi_sgd_update\n1. multi_mp_sgd_mom_update\n2. IdentityAttachKLSparseReg\n3. unravel_index\n4. mp_lamb_update_phase1\n5. mp_lamb_update_phase2\n6. scatter_nd\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/array_manipulation_operators.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport mxnet as mx\n\nfrom mxnet import nd\nfrom benchmark.opperf.utils.benchmark_utils import run_performance_test\nfrom benchmark.opperf.utils.common_utils import merge_map_list\nfrom benchmark.opperf.rules.default_params import MX_OP_MODULE\n\nfrom benchmark.opperf.utils.benchmark_utils import run_op_benchmarks\nfrom benchmark.opperf.utils.op_registry_utils import get_all_rearrange_operators, \\\n    get_all_shape_operators, get_all_expanding_operators, get_all_rounding_operators\n\n\"\"\"Performance benchmark tests for MXNet Array Manipulation Operators.\n\nArray Rearrange Operators\n1. transpose\n2. swapaxes (alias SwapAxis)\n3. flip (alias reverse)\n4. depth_to_space\n5. space_to_depth\n\nArray Shape Manipulation Operators\n1. split (alias SliceChannel)\n2. diag\n3. reshape\n4. reshape_like\n5. size_array\n6. shape_array\n\nArray Expanding Operators\n1. broadcast_axes (alias broadcast_axis)\n2. broadcast_to\n3. broadcast_like\n4. repeat\n5. tile\n6. pad\n7. expand_dims\n\n\nArray Rounding Operators\n1. round\n2. rint\n3. fix\n4. floor\n5. ceil\n6. trunc\n\nArray Join & Split Operators\n1. concat\n2. split\n3. stack\n\n\"\"\"\n\n\ndef run_rearrange_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context and precision (dtype) for all the\n    rearrange operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    # Fetch all array rearrange operators\n    mx_rearrange_ops = get_all_rearrange_operators()\n\n    # Run benchmarks\n    mx_rearrange_op_results = run_op_benchmarks(mx_rearrange_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return mx_rearrange_op_results\n\n\ndef run_shape_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context and precision (dtype) for all the\n    array shape operators  in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    # Fetch all array shape operators\n    mx_shape_ops = get_all_shape_operators()\n\n    # Run benchmarks\n    mx_shape_op_results = run_op_benchmarks(mx_shape_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return mx_shape_op_results\n\n\ndef run_expanding_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context and precision (dtype) for all the\n    array expanding operators  in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    # Fetch all array expanding operators\n    mx_expanding_ops = get_all_expanding_operators()\n\n    # Run benchmarks\n    mx_expanding_op_results = run_op_benchmarks(mx_expanding_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return mx_expanding_op_results\n\n\ndef run_rounding_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context and precision (dtype) for all the\n    array rounding operators  in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    # Fetch all array rounding operators\n    mx_rounding_ops = get_all_rounding_operators()\n\n    # Run benchmarks\n    mx_rounding_op_results = run_op_benchmarks(mx_rounding_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return mx_rounding_op_results\n\n\ndef run_join_split_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context and precision (dtype) for all the\n    join & split operators  in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    # backward not supported for all 3 ops - concat, stack, split\n    # concat\n    concat_benchmark_res = run_performance_test([getattr(MX_OP_MODULE, \"concat\")],\n                                                      run_backward=False,\n                                                      dtype=dtype,\n                                                      ctx=ctx,\n                                                      profiler=profiler,\n                                                      inputs=[{\"args0\":nd.random_normal(shape=(100,100)),\n                                                               \"args1\":nd.random_normal(shape=(100,100)),\n                                                               \"args2\":nd.random_normal(shape=(100,100))}\n                                                              ],\n                                                      warmup=warmup,\n                                                      runs=runs)\n\n    # split\n    split_benchmark_res = run_performance_test([getattr(MX_OP_MODULE, \"split\")],\n                                                      run_backward=False,\n                                                      dtype=dtype,\n                                                      ctx=ctx,\n                                                      profiler=profiler,\n                                                      inputs=[{\"data\": (1024, 1024), \"num_outputs\": 2},\n                                                              {\"data\": (10000, 1), \"num_outputs\": 1},\n                                                              {\"data\": (10000, 100), \"num_outputs\": 10}\n                                                              ],\n                                                      warmup=warmup,\n                                                      runs=runs)\n\n    # stack\n    stack_benchmark_res = run_performance_test([getattr(MX_OP_MODULE, \"stack\")],\n                                                      run_backward=False,\n                                                      dtype=dtype,\n                                                      ctx=ctx,\n                                                      profiler=profiler,\n                                                      inputs=[{\"args0\":nd.random_normal(shape=(100,100)),\n                                                               \"args1\":nd.random_normal(shape=(100,100)),\n                                                               \"args2\":nd.random_normal(shape=(100,100))}\n                                                              ],\n                                                      warmup=warmup,\n                                                      runs=runs)\n    mx_join_split_op_results = merge_map_list(concat_benchmark_res + split_benchmark_res + stack_benchmark_res)\n    return mx_join_split_op_results\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/array_rearrange.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport mxnet as mx\nfrom benchmark.opperf.utils.benchmark_utils import run_op_benchmarks\nfrom benchmark.opperf.utils.op_registry_utils import get_all_rearrange_operators\n\n\"\"\"Performance benchmark tests for MXNet NDArray Rearrange Operators.\n\n1. transpose\n2. swapaxes\n3. flip\n4. depth_to_space\n5. space_to_depth\n\"\"\"\n\n\ndef run_rearrange_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and input data size (int64_tensor) for all the\n    rearrange operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    # Fetch all array rerrange operators\n    mx_rearrange_ops = get_all_rearrange_operators()\n\n    # Run benchmarks\n    mx_rearrange_op_results = run_op_benchmarks(mx_rearrange_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return mx_rearrange_op_results\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/binary_operators.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Performance benchmark tests for MXNet NDArray Binary Operations - covers both broadcast and element_wise.\n1. Operators are automatically fetched from MXNet operator registry.\n2. Default Inputs are generated. See rules/default_params.py. You can override the default values.\n\nBelow 20 binary broadcast Operators are covered:\n\n['broadcast_add', 'broadcast_div', 'broadcast_equal', 'broadcast_greater', 'broadcast_greater_equal',\n'broadcast_hypot', 'broadcast_lesser', 'broadcast_lesser_equal', 'broadcast_logical_and',\n'broadcast_logical_or', 'broadcast_logical_xor', 'broadcast_maximum', 'broadcast_minimum',\n'broadcast_minus', 'broadcast_mod', 'broadcast_mul', 'broadcast_not_equal', 'broadcast_plus',\n'broadcast_power', 'broadcast_sub']\n\nBelow 4 binary element_wise Operators are covered:\n['elemwise_add', 'elemwise_mul', 'elemwise_sub', 'elemwise_div']\n\n\"\"\"\nimport mxnet as mx\n\nfrom benchmark.opperf.utils.benchmark_utils import run_op_benchmarks\nfrom benchmark.opperf.utils.op_registry_utils import get_all_broadcast_binary_operators, \\\n    get_all_elemen_wise_binary_operators, get_all_misc_binary_operators\n\n\ndef run_mx_binary_misc_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and input data size (int64_tensor) for all the miscellaneous\n    binary operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    # Fetch all Miscellaneous Binary Operators\n    mx_binary_misc_ops = get_all_misc_binary_operators()\n    # Run benchmarks\n    mx_binary_op_results = run_op_benchmarks(mx_binary_misc_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return mx_binary_op_results\n\n\ndef run_mx_binary_broadcast_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and input data size (int64_tensor) for all the binary\n    broadcast operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    # Fetch all Binary Broadcast Operators\n    mx_binary_broadcast_ops = get_all_broadcast_binary_operators()\n    # Run benchmarks\n    mx_binary_op_results = run_op_benchmarks(mx_binary_broadcast_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return mx_binary_op_results\n\n\ndef run_mx_binary_element_wise_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and input data size (int64_tensor) for all the binary\n    element_wise operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 10\n        Number of times to run for warmup\n    runs: int, default 50\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    # Fetch all Binary Element_wise Operators\n    mx_binary_element_wise_ops = get_all_elemen_wise_binary_operators()\n    # Run benchmarks\n    mx_binary_op_results = run_op_benchmarks(mx_binary_element_wise_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return mx_binary_op_results\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/gemm_operators.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport mxnet as mx\nfrom benchmark.opperf.utils.benchmark_utils import run_performance_test\nfrom benchmark.opperf.utils.common_utils import merge_map_list\nfrom benchmark.opperf.rules.default_params import MX_OP_MODULE\n\"\"\"Performance benchmark tests for MXNet NDArray GEMM Operators.\n\n1. dot\n2. batch_dot\n3. khatri_rao\n\nTODO\n3. As part of default tests, following needs to be added:\n    3.1 Sparse dot. (csr, default) -> row_sparse\n    3.2 Sparse dot. (csr, row_sparse) -> default\n    3.3 With Transpose of lhs\n    3.4 With Transpose of rhs\n4. 1D array: inner product of vectors\n\"\"\"\n\n\ndef run_gemm_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and input data size (int64_tensor) for all the GEMM\n    operators (dot, batch_dot, khatri_rao) in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    standard_inputs_dot = [{\"lhs\": (1024, 1024),\n                            \"rhs\": (1024, 1024)},\n                           {\"lhs\": (1000, 10),\n                            \"rhs\": (1000, 10),\n                            \"transpose_b\": True},\n                           {\"lhs\": (1000, 1),\n                            \"rhs\": (100, 1000),\n                            \"transpose_a\": True,\n                            \"transpose_b\": True}]\n    int64_tensor_inputs_dot = [{\"lhs\": (2**16, 2**16),\n                                \"rhs\": (2**16, 2**16)},\n                               {\"lhs\": (4, 2**30),\n                                \"rhs\": (4, 2**30),\n                                \"transpose_b\": True},\n                               {\"lhs\": (2**28, 16),\n                                \"rhs\": (16, 2**28),\n                                \"transpose_a\": True,\n                                \"transpose_b\": True}]\n    standard_inputs_batch_dot = [{\"lhs\": (32, 1024, 1024),\n                                  \"rhs\": (32, 1024, 1024)},\n                                 {\"lhs\": (32, 1000, 10),\n                                  \"rhs\": (32, 1000, 10),\n                                  \"transpose_b\": True},\n                                 {\"lhs\": (32, 1000, 1),\n                                  \"rhs\": (32, 100, 1000),\n                                  \"transpose_a\": True,\n                                  \"transpose_b\": True}]\n    int64_tensor_inputs_batch_dot = [{\"lhs\": (1, 2**16, 2**16),\n                                      \"rhs\": (1, 2**16, 2**16)},\n                                     {\"lhs\": (1, 4, 2**30),\n                                      \"rhs\": (1, 4, 2**30),\n                                      \"transpose_b\": True},\n                                     {\"lhs\": (1, 2**28, 16),\n                                      \"rhs\": (1, 16, 2**28),\n                                      \"transpose_a\": True,\n                                      \"transpose_b\": True}]\n    standard_inputs_khatri_rao = [{\"args\": [(32, 32), (32, 32)]},\n                                  {\"args\": [(64, 64), (64, 64)]}]\n    int64_tensor_inputs_khatri_rao = [{\"args\": [(2**32, 1), (2**32, 1)]}]\n\n    if int64_tensor == 'on':\n        inputs_dot = int64_tensor_inputs_dot\n        inputs_batch_dot = int64_tensor_inputs_batch_dot\n        inputs_khatri_rao = int64_tensor_inputs_khatri_rao\n    else:\n        inputs_dot = standard_inputs_dot\n        inputs_batch_dot = standard_inputs_batch_dot\n        inputs_khatri_rao = standard_inputs_khatri_rao\n\n    # Benchmark tests for dot and batch_dot operators\n    dot_benchmark_res = run_performance_test(\n        [getattr(MX_OP_MODULE, \"dot\")], run_backward=True,\n        dtype=dtype, ctx=ctx,\n        inputs=inputs_dot,\n        warmup=warmup, runs=runs, profiler=profiler)\n\n    batch_dot_benchmark_res = run_performance_test(\n        [getattr(MX_OP_MODULE, \"batch_dot\")], run_backward=True,\n        dtype=dtype, ctx=ctx,\n        inputs=inputs_batch_dot,\n        warmup=warmup, runs=runs, profiler=profiler)\n        # Operator khatri_rao is not yet implemented for GPU\n    khatri_rao_benchmark_res = []\n    if ctx != mx.gpu():\n        # Benchmark tests for khatri_rao operator\n        khatri_rao_benchmark_res = run_performance_test(\n            [getattr(MX_OP_MODULE, \"khatri_rao\")], run_backward=False,\n            dtype=dtype, ctx=ctx,\n            inputs=inputs_khatri_rao,\n            warmup=warmup, runs=runs, profiler=profiler)\n\n    # Prepare combined results for GEMM operators\n    mx_gemm_op_results = merge_map_list(dot_benchmark_res + batch_dot_benchmark_res + khatri_rao_benchmark_res)\n    return mx_gemm_op_results\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/indexing_routines.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport mxnet as mx\nfrom benchmark.opperf.utils.benchmark_utils import run_op_benchmarks\nfrom benchmark.opperf.utils.op_registry_utils import get_all_indexing_routines\n\n\"\"\"Performance benchmark tests for MXNet Indexing routines.\n\n1. slice\n2. slice_axis\n3. slice_like\n4. take\n5. pick\n6. where\n7. ravel_multi_index\n8. unravel_index [to do]\n9. gather_nd\n10. scatter_nd [to do]\n11. one_hot\n\"\"\"\n\n\ndef run_indexing_routines_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and data size (int64_tensor) for all the indexing routines\n    in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    # Fetch all indexing routines\n    mx_indexing_ops = get_all_indexing_routines()\n\n    # Run benchmarks\n    mx_indexing_op_results = run_op_benchmarks(mx_indexing_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return mx_indexing_op_results\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/linalg_operators.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Performance benchmark tests for MXNet NDArray Linear Algebra Operations.\n\nBelow 17 Linear Algebra Operators are covered:\n\n['linalg_potri', 'linalg_gemm2', 'linalg_extractdiag', 'linalg_trsm', 'linalg_gelqf', 'linalg_gemm', 'linalg_sumlogdiag',\n'linalg_potrf', 'linalg_makediag', 'linalg_syrk', 'linalg_maketrian', 'linalg_trmm', 'linalg_extracttrian',\n'linalg_slogdet', 'linalg_det', 'linalg_inverse', 'moments']\n\n\"\"\"\n\nimport mxnet as mx\n\nfrom benchmark.opperf.utils.benchmark_utils import run_op_benchmarks\nfrom benchmark.opperf.utils.op_registry_utils import get_all_linalg_operators\n\nfrom benchmark.opperf.utils.benchmark_utils import run_performance_test\nfrom benchmark.opperf.utils.common_utils import merge_map_list\nfrom benchmark.opperf.rules.default_params import MX_OP_MODULE\n\ndef run_linalg_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and data size (int64_tensor) for all the linear algebra\n    operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    # Individual tests for ops with specific requirements on input data\n    # linalg_potrf requires a positive definite matrix as input\n    linalg_potrf_benchmark = run_performance_test(getattr(MX_OP_MODULE, \"linalg_potrf\"),\n                                                  run_backward=False,\n                                                  dtype=dtype,\n                                                  ctx=ctx,\n                                                  profiler=profiler,\n                                                  inputs=[{\"A\": [[1, 0],\n                                                                 [0, 1]]},\n                                                          {\"A\": [[2, -1, 0],\n                                                                 [-1, 2, -1],\n                                                                 [0, -1, 2]]}],\n                                                  warmup=warmup,\n                                                  runs=runs)\n\n    # Fetch all Linear Algebra Operators\n    mx_linalg_ops = get_all_linalg_operators()\n    # Run benchmarks\n    mx_linalg_op_results = run_op_benchmarks(mx_linalg_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return merge_map_list(linalg_potrf_benchmark + [mx_linalg_op_results])\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/misc_operators.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Performance benchmark tests for MXNet NDArray Miscellaneous Operations.\n\nBelow 16 Miscellaneous Operators are covered:\n\n['reset_arrays', 'multi_all_finite', 'multi_sum_sq', 'add_n', 'UpSampling', 'Custom', 'squeeze',\n'all_finite', 'clip', 'multi_lars', 'SequenceReverse', 'SequenceLast', 'SequenceMask', 'cast_storage',\n'cumsum', 'fill_element_0index']\n\n\"\"\"\n\nimport mxnet as mx\n\nfrom benchmark.opperf.utils.benchmark_utils import run_op_benchmarks\nfrom benchmark.opperf.utils.op_registry_utils import get_remaining_miscellaneous_operators\n\nfrom benchmark.opperf.utils.benchmark_utils import run_performance_test\nfrom benchmark.opperf.utils.common_utils import merge_map_list\nfrom benchmark.opperf.rules.default_params import MX_OP_MODULE\n\nfrom benchmark.opperf.custom_operations.custom_operations import CustomAddOneProp\n\n\ndef run_mx_misc_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context and precision (dtype) for all the miscellaneous\n    operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n\n    standard_inputs_array_ops = [{\"args\": [(1024, 1024)],\n                                  \"num_arrays\": 1},\n                                 {\"args\": [(10000, 1)],\n                                  \"num_arrays\": 1},\n                                 {\"args\": [(10000, 10)],\n                                  \"num_arrays\": 1}]\n    int64_tensor_inputs_array_ops = [{\"args\": [(2**32, 1)],\n                                      \"num_arrays\":1}]\n    standard_inputs_add_n = [{\"args\": [(1024, 1024)]},\n                             {\"args\": [(10000, 1)]},\n                             {\"args\": [(10000, 10)]}]\n    int64_tensor_inputs_add_n = [{\"args\": [(2**16, 2**16)]}]\n    standard_inputs_upsampling = [{\"args\": (32, 3, 256, 256),\n                                   \"scale\": 2,\n                                   \"sample_type\": \"nearest\"},\n                                  {\"args\": (32, 3, 10000, 1),\n                                   \"scale\": 4,\n                                   \"sample_type\": \"nearest\"}]\n    int64_tensor_inputs_upsampling = [{\"args\": (2**32 + 1, 1, 1, 1),\n                                       \"scale\": 2,\n                                       \"sample_type\": \"nearest\"}]\n    standard_inputs_custom = [{\"args\": [(1024, 1024)],\n                               \"op_type\": \"CustomAddOne\"},\n                              {\"args\": [(10000, 1)],\n                               \"op_type\": \"CustomAddOne\"},\n                              {\"args\": [(10000, 10)],\n                               \"op_type\": \"CustomAddOne\"}]\n    int64_tensor_inputs_custom = [{\"args\": [(2**32 + 1, 1)],\n                                   \"op_type\": \"CustomAddOne\"}]\n\n    if int64_tensor == 'on':\n        inputs_array_ops = int64_tensor_inputs_array_ops\n        inputs_add_n = int64_tensor_inputs_add_n\n        inputs_upsampling = int64_tensor_inputs_upsampling\n        inputs_custom = int64_tensor_inputs_custom\n    else:\n        inputs_array_ops = standard_inputs_array_ops\n        inputs_add_n = standard_inputs_add_n\n        inputs_upsampling = standard_inputs_upsampling\n        inputs_custom = standard_inputs_custom\n\n    # Individual tests for ops with positional args\n    array_ops_benchmark = run_performance_test([getattr(MX_OP_MODULE, \"reset_arrays\"),\n                                                getattr(MX_OP_MODULE, \"multi_all_finite\"),\n                                                getattr(MX_OP_MODULE, \"multi_sum_sq\")],\n                                               run_backward=False,\n                                               dtype=dtype,\n                                               ctx=ctx,\n                                               profiler=profiler,\n                                               inputs=inputs_array_ops,\n                                               warmup=warmup,\n                                               runs=runs)\n    add_n_benchmark = run_performance_test([getattr(MX_OP_MODULE, \"add_n\")],\n                                           run_backward=True,\n                                           dtype=dtype,\n                                           ctx=ctx,\n                                           profiler=profiler,\n                                           inputs=inputs_add_n,\n                                           warmup=warmup,\n                                           runs=runs)\n    # There are currently issus with UpSampling with bilinear interpolation.\n    # track issue here: https://github.com/apache/mxnet/issues/9138\n    upsampling_benchmark = run_performance_test([getattr(MX_OP_MODULE, \"UpSampling\")],\n                                                run_backward=True,\n                                                dtype=dtype,\n                                                ctx=ctx,\n                                                profiler=profiler,\n                                                inputs=inputs_upsampling,\n                                                warmup=warmup,\n                                                runs=runs)\n    # Create and register CustomAddOne operator for use in Custom op testing\n    c = CustomAddOneProp()\n    c.create_operator(ctx, [(1024,1024)], [dtype])\n    custom_benchmark = run_performance_test([getattr(MX_OP_MODULE, \"Custom\")],\n                                            run_backward=True,\n                                            dtype=dtype,\n                                            ctx=ctx,\n                                            profiler=profiler,\n                                            inputs=inputs_custom,\n                                            warmup=warmup,\n                                            runs=runs)\n\n    # Fetch remaining Miscellaneous Operators\n    mx_misc_ops = get_remaining_miscellaneous_operators()\n    # Run benchmarks\n    mx_misc_op_results = run_op_benchmarks(mx_misc_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return merge_map_list(array_ops_benchmark + add_n_benchmark + upsampling_benchmark + custom_benchmark + [mx_misc_op_results])\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/nn_activation_operators.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport mxnet as mx\n\nfrom benchmark.opperf.utils.op_registry_utils import get_all_nn_activation_operators\nfrom benchmark.opperf.utils.benchmark_utils import run_op_benchmarks\n\n\"\"\"Performance benchmark tests for MXNet NDArray Activation Operators.\n\n1. LeakyReLU\n    1.1 elu\n    1.2 selu\n    1.3 leaky\n    1.4 gelu\n2. hard_sigmoid\n3. Softmax\n4. SoftmaxActivation\n5. softmax\n6. log_softmax\n7. softmin\n8. Activation\n    8.1 relu\n    8.2 sigmoid\n    8.3 log_sigmoid\n    8.4 mish\n    8.5 softrelu\n    8.6 softsign\n    8.7 tanh\n\n\"\"\"\n\n\ndef run_activation_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and input data size (int64_tensor) for all the activation\n    operators (relu, sigmoid, softmax) in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Module to use for tracking benchmark excecution time\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n\n    # Fetch all NN Activation Operators\n    mx_activation_ops = get_all_nn_activation_operators()\n\n    # Run benchmarks\n    mx_activation_op_results = run_op_benchmarks(mx_activation_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return mx_activation_op_results\n    "
  },
  {
    "path": "benchmark/opperf/nd_operations/nn_basic_operators.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport mxnet as mx\n\nfrom benchmark.opperf.utils.op_registry_utils import get_all_nn_basic_operators\nfrom benchmark.opperf.utils.benchmark_utils import run_op_benchmarks\n\nfrom benchmark.opperf.utils.benchmark_utils import run_performance_test\nfrom benchmark.opperf.utils.common_utils import merge_map_list\nfrom benchmark.opperf.rules.default_params import MX_OP_MODULE\n\n\"\"\"Performance benchmark tests for MXNet NDArray basic NN Operators.\n\n1. FullyConnected\n2. Dropout\n3. BatchNorm\n9. L2Normalization\n10. LayerNorm\n11. InstanceNorm\n12. Embedding\n13. Correlation\n14. SpatialTransformer\n15. im2col\n16. col2im\n17. GroupNorm\n18. RNN\n19. LRN\n\n\"\"\"\n\n\ndef run_nn_basic_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and data size (int64_tensor) for all the basic neural network\n    operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n\n    standard_data_list = [(1024, 4, 4)]\n    int64_tensor_data_list = [(2**28, 4, 4)]\n\n    if int64_tensor == 'on':\n        data_list = int64_tensor_data_list\n    else:\n        data_list = standard_data_list\n\n    for data in data_list:\n        rnn_relu_benchmark = run_performance_test([getattr(MX_OP_MODULE, \"RNN\")],\n                                                  run_backward=True,\n                                                  dtype=dtype,\n                                                  ctx=ctx,\n                                                  profiler=profiler,\n                                                  inputs=[{\"data\": data,\n                                                           \"parameters\": (7,),\n                                                           \"state\": (1, 4, 1),\n                                                           \"mode\": \"rnn_relu\",\n                                                           \"state_size\": 1,\n                                                           \"num_layers\": 1}],\n                                                  warmup=warmup,\n                                                  runs=runs)\n        rnn_tanh_benchmark = run_performance_test([getattr(MX_OP_MODULE, \"RNN\")],\n                                                  run_backward=True,\n                                                  dtype=dtype,\n                                                  ctx=ctx,\n                                                  profiler=profiler,\n                                                  inputs=[{\"data\": data,\n                                                           \"parameters\": (7,),\n                                                           \"state\": (1, 4, 1),\n                                                           \"mode\": \"rnn_tanh\",\n                                                           \"state_size\": 1,\n                                                           \"num_layers\": 1}],\n                                                  warmup=warmup,\n                                                  runs=runs)\n        rnn_lstm_benchmark = run_performance_test([getattr(MX_OP_MODULE, \"RNN\")],\n                                                  run_backward=True,\n                                                  dtype=dtype,\n                                                  ctx=ctx,\n                                                  profiler=profiler,\n                                                  inputs=[{\"data\": data,\n                                                           \"parameters\": (28,),\n                                                           \"state\": (1, 4, 1),\n                                                           \"state_cell\": (1, 4, 1),\n                                                           \"mode\": \"lstm\",\n                                                           \"state_size\": 1,\n                                                           \"num_layers\": 1}],\n                                                  warmup=warmup,\n                                                  runs=runs)\n        rnn_gru_benchmark = run_performance_test([getattr(MX_OP_MODULE, \"RNN\")],\n                                                 run_backward=True,\n                                                 dtype=dtype,\n                                                 ctx=ctx,\n                                                 profiler=profiler,\n                                                 inputs=[{\"data\": data,\n                                                          \"parameters\": (21,),\n                                                          \"state\": (1, 4, 1),\n                                                          \"mode\": \"gru\",\n                                                          \"state_size\": 1,\n                                                          \"num_layers\": 1}],\n                                                 warmup=warmup,\n                                                 runs=runs)\n    # Fetch all NN Basic Operators\n    mx_nn_basic_ops = get_all_nn_basic_operators()\n    \n    # Run benchmarks\n    mx_nn_basic_op_results = run_op_benchmarks(mx_nn_basic_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return merge_map_list(rnn_relu_benchmark + rnn_tanh_benchmark + rnn_lstm_benchmark + rnn_gru_benchmark + [mx_nn_basic_op_results])\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/nn_conv_operators.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport mxnet as mx\nfrom benchmark.opperf.utils.benchmark_utils import run_performance_test\nfrom benchmark.opperf.utils.common_utils import merge_map_list\nfrom benchmark.opperf.rules.default_params import MX_OP_MODULE\n\n\"\"\"Performance benchmark tests for MXNet NDArray Convolution and Pooling Operators.\n\nMXNet NDArray Pooling Operators\n\n1. MaxPool1D\n2. MaxPool2D\n3. SumPool1D\n4. SumPool2D\n4. AvgPool1D\n5. AvgPool2D\n6. GlobalMaxPool1D\n7. GlobalMaxPool2D\n8. GlobalAvgPool1D\n9. GlobalAvgPool2D\n10.GlobalSumPool1D\n11.GlobalSumPool2D\n12.ROIPooling\n\n(Under the hood uses mx.nd.pooling)\n\nMXNet NDArray NN Convolution Operators\n\n1. Conv1D\n2. Conv2D\n3. Conv1DTranspose (DeConvolution)\n4. Conv2DTranspose (DeConvolution)\n\n(Under the hood uses mx.nd.convolution, mx.nd.Deconvolution)\n\n\"\"\"\n\n\ndef run_pooling_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and input data size (int64_tensor) for all the pooling\n    operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    pool_types = ['avg', 'max', 'sum']\n    global_pool_types = [0, 1]\n\n    standard_data_list_pool1d = [(32, 3, 256), (32, 3, 64)]\n    int64_tensor_data_list_pool1d = [(1, 1, 2**32)]\n    standard_data_list_pool2d = [(32, 3, 256, 256), (32, 3, 64, 64)]\n    int64_tensor_data_list_pool2d = [(2**28, 1, 4, 4)]\n    standard_data_list_roipool = [(32, 3, 256, 256), (32, 3, 64, 64)]\n    int64_tensor_data_list_roipool = [(32, 3, 2**13, 2**13)]\n\n    if int64_tensor == 'on':\n        data_list_pool1d = int64_tensor_data_list_pool1d\n        data_list_pool2d = int64_tensor_data_list_pool2d\n        data_list_roipool = int64_tensor_data_list_roipool\n    else:\n        data_list_pool1d = standard_data_list_pool1d\n        data_list_pool2d = standard_data_list_pool2d\n        data_list_roipool = standard_data_list_roipool\n\n    # Run 1D and 2D Pooling performance runs\n    pool1d_benchmark_res = []\n    pool2d_benchmark_res = []\n    for pool_type in pool_types:\n        for global_pool in global_pool_types:\n            for pool1d_data in data_list_pool1d:\n                pool1d_benchmark_res += run_performance_test([getattr(MX_OP_MODULE, \"Pooling\")],\n                                                             run_backward=True,\n                                                             dtype=dtype,\n                                                             ctx=ctx,\n                                                             profiler=profiler,\n                                                             inputs=[{\"data\": pool1d_data,\n                                                                      \"kernel\": 3,\n                                                                      \"pool_type\": pool_type,\n                                                                      \"global_pool\": global_pool,\n                                                                      \"stride\": 1,\n                                                                      \"pad\": 1}\n                                                                    ],\n                                                             warmup=warmup,\n                                                             runs=runs)\n            for pool2d_data in data_list_pool2d:\n                pool2d_benchmark_res += run_performance_test([getattr(MX_OP_MODULE, \"Pooling\")],\n                                                             run_backward=True,\n                                                             dtype=dtype,\n                                                             ctx=ctx,\n                                                             profiler=profiler,\n                                                             inputs=[{\"data\": pool2d_data,\n                                                                      \"kernel\": (3, 3),\n                                                                      \"pool_type\": pool_type,\n                                                                      \"global_pool\": global_pool,\n                                                                      \"stride\": (1, 1),\n                                                                      \"pad\": (0, 0)}\n                                                                    ],\n                                                             warmup=warmup,\n                                                             runs=runs)\n            # Run ROI Pooling performance runs\n            roipool_benchmark_res = []\n            for roipool_data in data_list_roipool:\n                roipool_benchmark_res += run_performance_test([getattr(MX_OP_MODULE, \"ROIPooling\")],\n                                                              run_backward=True,\n                                                              dtype=dtype,\n                                                              ctx=ctx,\n                                                              profiler=profiler,\n                                                              inputs=[{\"data\": roipool_data,\n                                                                       \"rois\": (32, 5),\n                                                                       \"pooled_size\": (2, 2),\n                                                                       \"spatial_scale\": .5}\n                                                                     ],\n                                                              warmup=warmup,\n                                                              runs=runs)\n    # Prepare combined results\n    mx_pooling_op_results = merge_map_list(pool1d_benchmark_res + pool2d_benchmark_res + roipool_benchmark_res)\n    return mx_pooling_op_results\n\n\ndef run_convolution_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and input data size (int64_tensor) for all the convolution\n    operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n\n    standard_data_list_conv1d = [(32, 3, 256), (32, 3, 64)]\n    int64_tensor_data_list_conv1d = [(2**30, 1, 4)]\n    standard_weight_conv1d = (1, 3, 3)\n    int64_tensor_weight_conv1d = (1, 1, 1)\n    standard_kernel_conv1d = (3,)\n    int64_tensor_kernel_conv1d = (1,)\n    standard_data_list_conv2d = [(32, 3, 256, 256), (32, 3, 64, 64)]\n    int64_tensor_data_list_conv2d = [(2**28, 1, 4, 4)]\n    standard_weight_conv2d = (1, 3, 3, 3)\n    int64_tensor_weight_conv2d = (1, 1, 1, 1)\n    standard_kernel_conv2d = (3, 3)\n    int64_tensor_kernel_conv2d = (1, 1)\n\n    if int64_tensor == 'on':\n        data_list_conv1d = int64_tensor_data_list_conv1d\n        weight_conv1d = int64_tensor_weight_conv1d\n        kernel_conv1d = int64_tensor_kernel_conv1d\n        data_list_conv2d = int64_tensor_data_list_conv2d\n        weight_conv2d = int64_tensor_weight_conv2d\n        kernel_conv2d = int64_tensor_kernel_conv2d\n    else:\n        data_list_conv1d = standard_data_list_conv1d\n        weight_conv1d = standard_weight_conv1d\n        kernel_conv1d = standard_kernel_conv1d\n        data_list_conv2d = standard_data_list_conv2d\n        weight_conv2d = standard_weight_conv2d\n        kernel_conv2d = standard_kernel_conv2d\n\n    conv1d_benchmark_res = []\n    conv2d_benchmark_res = []\n    # Conv1D Benchmarks\n    for conv_data in data_list_conv1d:\n        conv1d_benchmark_res += run_performance_test([getattr(MX_OP_MODULE, \"Convolution\")],\n                                                     run_backward=True,\n                                                     dtype=dtype,\n                                                     ctx=ctx,\n                                                     profiler=profiler,\n                                                     inputs=[{\"data\": conv_data,\n                                                              \"weight\": weight_conv1d,\n                                                              \"bias\": (1,),\n                                                              \"kernel\": kernel_conv1d,\n                                                              \"stride\": (1,),\n                                                              \"dilate\": (1,),\n                                                              \"pad\": (0,),\n                                                              \"num_filter\": 1,\n                                                              \"layout\": 'NCW'}],\n                                                     warmup=warmup,\n                                                     runs=runs)\n    # Conv2D Benchmarks\n    for conv_data in data_list_conv2d:\n        conv2d_benchmark_res += run_performance_test([getattr(MX_OP_MODULE, \"Convolution\")],\n                                                     run_backward=True,\n                                                     dtype=dtype,\n                                                     ctx=ctx,\n                                                     profiler=profiler,\n                                                     inputs=[{\"data\": conv_data,\n                                                              \"weight\": weight_conv2d,\n                                                              \"bias\": (1,),\n                                                              \"kernel\": kernel_conv2d,\n                                                              \"stride\": (1, 1),\n                                                              \"dilate\": (1, 1),\n                                                              \"pad\": (0, 0),\n                                                              \"num_filter\": 1,\n                                                              \"layout\": 'NCHW'}],\n                                                     warmup=warmup,\n                                                     runs=runs)\n    # Prepare combined results\n    mx_conv_op_results = merge_map_list(conv1d_benchmark_res + conv2d_benchmark_res)\n    return mx_conv_op_results\n\n\ndef run_transpose_convolution_operators_benchmarks(ctx=mx.cpu(), profiler='native', int64_tensor='off', dtype='float32', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and input data size (int64_tensor) for all the transpose convolution\n    operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n\n    standard_data_list_conv1d_transpose = [(32, 3, 256), (32, 3, 64)]\n    int64_tensor_data_list_conv1d_transpose = [(2**30, 1, 4)]\n    standard_weight_conv1d_transpose = (3, 1, 3)\n    int64_tensor_weight_conv1d_transpose = (1, 1, 1)\n    standard_kernel_conv1d_transpose = (3,)\n    int64_tensor_kernel_conv1d_transpose = (1,)\n    standard_data_list_conv2d_transpose = [(32, 3, 256, 256), (32, 3, 64, 64)]\n    int64_tensor_data_list_conv2d_transpose = [(2**28, 1, 4, 4)]\n    standard_weight_conv2d_transpose = (3, 1, 3, 3)\n    int64_tensor_weight_conv2d_transpose = (1, 1, 1, 1)\n    standard_kernel_conv2d_transpose = (3, 3)\n    int64_tensor_kernel_conv2d_transpose = (1, 1)\n\n    if int64_tensor == 'on':\n        data_list_conv1d_transpose = int64_tensor_data_list_conv1d_transpose\n        weight_conv1d_transpose = int64_tensor_weight_conv1d_transpose\n        kernel_conv1d_transpose = int64_tensor_kernel_conv1d_transpose\n        data_list_conv2d_transpose = int64_tensor_data_list_conv2d_transpose\n        weight_conv2d_transpose = int64_tensor_weight_conv2d_transpose\n        kernel_conv2d_transpose = int64_tensor_kernel_conv2d_transpose\n    else:\n        data_list_conv1d_transpose = standard_data_list_conv1d_transpose\n        weight_conv1d_transpose = standard_weight_conv1d_transpose\n        kernel_conv1d_transpose = standard_kernel_conv1d_transpose\n        data_list_conv2d_transpose = standard_data_list_conv2d_transpose\n        weight_conv2d_transpose = standard_weight_conv2d_transpose\n        kernel_conv2d_transpose = standard_kernel_conv2d_transpose\n\n    # Conv1DTranspose Benchmarks\n    conv1d_transpose_benchmark_res = []\n    for conv_data in data_list_conv1d_transpose:\n        conv1d_transpose_benchmark_res += run_performance_test([getattr(MX_OP_MODULE, \"Deconvolution\")],\n                                                                   run_backward=True,\n                                                                   dtype=dtype,\n                                                                   ctx=ctx,\n                                                                   profiler=profiler,\n                                                                   inputs=[{\"data\": conv_data,\n                                                                            \"weight\": weight_conv1d_transpose,\n                                                                            \"bias\": (1,),\n                                                                            \"kernel\": kernel_conv1d_transpose,\n                                                                            \"stride\": (1,),\n                                                                            \"dilate\": (1,),\n                                                                            \"pad\": (0,),\n                                                                            \"num_filter\": 1,\n                                                                            \"no_bias\": False,\n                                                                            \"layout\": 'NCW'}],\n                                                                   warmup=warmup,\n                                                                   runs=runs)\n    # Conv2DTranspose Benchmarks\n    conv2d_transpose_benchmark_res = []\n    for conv_data in data_list_conv2d_transpose:\n        conv2d_transpose_benchmark_res += run_performance_test([getattr(MX_OP_MODULE, \"Deconvolution\")],\n                                                                   run_backward=True,\n                                                                   dtype=dtype,\n                                                                   ctx=ctx,\n                                                                   profiler=profiler,\n                                                                   inputs=[{\"data\": conv_data,\n                                                                            \"weight\": weight_conv2d_transpose,\n                                                                            \"bias\": (1,),\n                                                                            \"kernel\": kernel_conv2d_transpose,\n                                                                            \"stride\": (1, 1),\n                                                                            \"pad\": (0, 0),\n                                                                            \"num_filter\": 1,\n                                                                            \"no_bias\": False,\n                                                                            \"layout\": 'NCHW'}],\n                                                                   warmup=warmup,\n                                                                   runs=runs)\n    # Prepare combined results\n    mx_transpose_conv_op_results = merge_map_list(conv1d_transpose_benchmark_res + conv2d_transpose_benchmark_res)\n    return mx_transpose_conv_op_results\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/nn_loss_operators.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport mxnet as mx\nfrom benchmark.opperf.utils.benchmark_utils import run_op_benchmarks\nfrom benchmark.opperf.utils.op_registry_utils import get_all_loss_operators\n\n\"\"\"Performance benchmark tests for MXNet Neural Network Loss Operators\n\n1. smooth_l1\n2. CTCLoss\n3. MakeLoss\n4. softmax_cross_entropy\n\"\"\"\n\n\ndef run_loss_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and data size (int64_tensor) for all the\n    Neural Network loss operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    # Fetch all loss operators\n    mx_loss_ops = get_all_loss_operators()\n\n    # Run benchmarks\n    mx_loss_op_results = run_op_benchmarks(mx_loss_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return mx_loss_op_results\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/nn_optimizer_operators.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport mxnet as mx\nfrom mxnet import nd\n\nfrom benchmark.opperf.utils.benchmark_utils import run_performance_test\nfrom benchmark.opperf.utils.benchmark_utils import run_op_benchmarks\nfrom benchmark.opperf.utils.op_registry_utils import get_all_optimizer_operators\nfrom benchmark.opperf.utils.common_utils import merge_map_list\nfrom benchmark.opperf.rules.default_params import MX_OP_MODULE\n\n\"\"\"Performance benchmark tests for MXNet Neural Network Optimizer Update Operators.\n\n1. Stochastic Gradient Descent (SGD)\n    1.1 mp_sgd_update\n    1.2 sgd_mom_update\n    1.3 signsgd_update\n    1.4 mp_sgd_mom_update\n    1.5 sgd_update\n2. signum_update\n3. rmspropalex_update\n4. ftml_update\n5. rmsprop_update\n6. ftrl_update\n7. adam_update\n8. preloaded_multi_*\n    8.1 preloaded_multi_sgd_mom_update\n    8.2 preloaded_multi_sgd_update\n    8.3 preloaded_multi_mp_sgd_update\n    8.4 preloaded_multi_mp_sgd_mom_update\n9. lamb_*\n    9.1 lamb_update_phase1\n    9.2 lamb_update_phase2\n10. multi_*\n    10.1 multi_sgd_update\n    10.2 multi_sgd_mom_update\n    10.3 multi_mp_sgd_update\n    10.4 multi_mp_sgd_mom_update\n\"\"\"\n\n\ndef run_optimizer_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and input data size (int64_tensor) for all the neural network\n    optimizer update operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    standard_shape = (5, 5)\n    int64_tensor_shape = (2**16, 2**16)\n\n    if int64_tensor == 'on':\n        arg_shape = int64_tensor_shape\n    else:\n        arg_shape = standard_shape\n\n    # Run independent tests for ops that need specific input data\n    multi_mp_sgd_mom_res = run_performance_test([getattr(MX_OP_MODULE, \"multi_mp_sgd_mom_update\")],\n                                                inputs=[{\"args0\": nd.random_normal(shape=arg_shape),\n                                                \"args1\": nd.random_normal(shape=arg_shape), \"args2\": nd.random_normal(shape=arg_shape),\n                                                \"args3\": nd.random_normal(shape=arg_shape), \"lrs\": 0.1, \"wds\": 0.2,\n                                                \"out\": nd.random_normal(shape=arg_shape)}],run_backward=False)\n\n    multi_sgd_mom_res = run_performance_test([getattr(MX_OP_MODULE, \"multi_sgd_mom_update\")],\n                                             inputs=[{\"args0\": nd.random_normal(shape=arg_shape),\n                                             \"args1\": nd.random_normal(shape=arg_shape),\"args2\": nd.random_normal(shape=arg_shape),\n                                             \"lrs\": 0.1, \"wds\": 0.2, \"out\": nd.random_normal(shape=arg_shape)}], run_backward=False)\n\n    multi_sgd_res = run_performance_test([getattr(MX_OP_MODULE, \"multi_sgd_update\")],\n                                         inputs=[{\"args0\": nd.random_normal(shape=arg_shape),\n                                         \"args1\": nd.random_normal(shape=arg_shape), \"lrs\": 0.1, \"wds\": 0.2,\n                                         \"out\": nd.random_normal(shape=arg_shape)}], run_backward=False)\n\n    multi_mp_sgd_res = run_performance_test([getattr(MX_OP_MODULE, \"multi_mp_sgd_update\")],\n                                            inputs=[{\"args0\": nd.random_normal(shape=arg_shape),\n                                            \"args1\": nd.random_normal(shape=arg_shape),\"args2\": nd.random_normal(shape=arg_shape),\n                                            \"lrs\": 0.1, \"wds\": 0.2, \"out\": nd.random_normal(shape=arg_shape)}], run_backward=False)\n\n    preloaded_multi_mp_sgd_res = run_performance_test(\n                                 [getattr(MX_OP_MODULE, \"preloaded_multi_mp_sgd_update\")],\n                                 inputs=[{\"args0\": nd.random_normal(shape=arg_shape),\n                                          \"args1\": nd.random_normal(shape=arg_shape), \"args2\": nd.random_normal(shape=arg_shape),\n                                          \"args3\": nd.random_normal(shape=(1)), \"args4\": nd.random_normal(shape=(1)),\n                                          \"out\": nd.random_normal(shape=arg_shape)}], run_backward=False)\n\n    preloaded_multi_sgd_mom_res = run_performance_test(\n                                  [getattr(MX_OP_MODULE, \"preloaded_multi_sgd_mom_update\")],\n                                  inputs=[{\"args0\": nd.random_normal(shape=arg_shape),\n                                           \"args1\": nd.random_normal(shape=arg_shape), \"args2\": nd.random_normal(shape=arg_shape),\n                                           \"args3\": nd.random_normal(shape=(1)), \"args4\": nd.random_normal(shape=(1)),\n                                           \"out\": nd.random_normal(shape=arg_shape)}], run_backward=False)\n\n    preloaded_multi_sgd_res = run_performance_test(\n                              [getattr(MX_OP_MODULE, \"preloaded_multi_sgd_update\")],\n                              inputs=[{\"args0\": nd.random_normal(shape=arg_shape), \"args1\": nd.random_normal(shape=arg_shape),\n                                       \"args4\": nd.random_normal(shape=(1)), \"args5\": nd.random_normal(shape=(1)),\n                                       \"out\": nd.random_normal(shape=arg_shape)}], run_backward=False)\n\n    preloaded_multi_mp_sgd_mom_res = run_performance_test(\n                                     [getattr(MX_OP_MODULE, \"preloaded_multi_mp_sgd_mom_update\")],\n                                     inputs=[{\"args0\": nd.random_normal(shape=arg_shape), \"args1\": nd.random_normal(shape=arg_shape),\n                                              \"args2\": nd.random_normal(shape=arg_shape), \"args3\": nd.random_normal(shape=arg_shape),\n                                              \"args4\": nd.random_normal(shape=(1)), \"args5\": nd.random_normal(shape=(1)),\n                                              \"out\": nd.random_normal(shape=arg_shape)}], run_backward=False)\n\n    # Fetch remaining optimizer operators\n    mx_optimizer_ops = get_all_optimizer_operators()\n\n    # Run benchmarks\n    mx_optimizer_op_results = run_op_benchmarks(mx_optimizer_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return merge_map_list(multi_sgd_mom_res + multi_sgd_mom_res + multi_sgd_res + multi_mp_sgd_res + preloaded_multi_mp_sgd_res +\\\n                          preloaded_multi_sgd_mom_res + preloaded_multi_mp_sgd_res + preloaded_multi_mp_sgd_mom_res +\\\n                          multi_mp_sgd_mom_res + preloaded_multi_sgd_res + [mx_optimizer_op_results])\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/random_sampling_operators.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Performance benchmark tests for MXNet NDArray Random Sampling Operations.\n1. Operators are automatically fetched from MXNet operator registry.\n2. Default Inputs are generated. See rules/default_params.py. You can override the default values.\n\nBelow 18 random sampling Operators are covered:\n\n['random_exponential', 'random_gamma', 'random_generalized_negative_binomial', 'random_negative_binomial',\n'random_normal', 'random_poisson', 'random_randint', 'random_uniform', 'sample_exponential', 'sample_gamma',\n'sample_generalized_negative_binomial', 'sample_multinomial', 'sample_negative_binomial', 'sample_normal',\n'sample_poisson', 'sample_uniform', 'GridGenerator', 'BilinearSampler']\n\n\"\"\"\n\nimport mxnet as mx\n\nfrom benchmark.opperf.utils.benchmark_utils import run_op_benchmarks\nfrom benchmark.opperf.utils.op_registry_utils import get_all_random_sampling_operators\n\n\ndef run_mx_random_sampling_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and input data size (int64_tensor) for all the random sampling\n    operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    # Fetch all Random Sampling Operators\n    mx_random_sample_ops = get_all_random_sampling_operators()\n    # Run benchmarks\n    mx_random_sample_op_results = run_op_benchmarks(mx_random_sample_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return mx_random_sample_op_results\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/reduction_operators.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Performance benchmark tests for MXNet NDArray Reduction Operations.\n1. Operators are automatically fetched from MXNet operator registry.\n2. Default Inputs are generated. See rules/default_params.py. You can override the default values.\n\nBelow 10 reduction Operators are covered:\n\n['max', 'max_axis', 'mean', 'min', 'min_axis', 'nanprod', 'nansum', 'prod', 'sum', 'sum_axis']\n\n\"\"\"\n\nimport mxnet as mx\n\nfrom benchmark.opperf.utils.op_registry_utils import get_all_reduction_operators\nfrom benchmark.opperf.utils.benchmark_utils import run_op_benchmarks\n\n\ndef run_mx_reduction_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and input data size (int64_tensor) for all the reduction\n    operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    # Fetch all Reduction Operators\n    mx_reduction_broadcast_ops = get_all_reduction_operators()\n    # Run benchmarks\n    mx_reduction_op_results = run_op_benchmarks(mx_reduction_broadcast_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return mx_reduction_op_results\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/sorting_searching_operators.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport mxnet as mx\nfrom benchmark.opperf.utils.benchmark_utils import run_op_benchmarks\nfrom benchmark.opperf.utils.op_registry_utils import get_all_sorting_searching_operators\n\n\n\"\"\" Performance benchmark tests for MXNet NDArray Sorting and Searching Operations\n1. sort\n2. argsort\n3. topk\n4. argmax\n5. argmin\n\"\"\"\n\n\ndef run_sorting_searching_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and input data size (int64_tensor) for all the sorting and searching\n    operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n    # Fetch all Random Sampling Operators\n    mx_sort_search_ops = get_all_sorting_searching_operators()\n    # Run benchmarks\n    mx_sort_search_op_results = run_op_benchmarks(mx_sort_search_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return mx_sort_search_op_results\n"
  },
  {
    "path": "benchmark/opperf/nd_operations/unary_operators.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Performance benchmark tests for MXNet NDArray Unary Operations.\n1. Operators are automatically fetched from MXNet operator registry.\n2. Default Inputs are generated. See rules/default_params.py. You can override the default values.\n\nBelow 54 unary Operators are covered:\n\n['BlockGrad', 'Flatten', 'abs', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctanh',\n'argmax_channel', 'cbrt', 'ceil', 'cos', 'cosh', 'degrees', 'erf', 'erfinv', 'exp', 'expm1', 'fix', 'flatten',\n'floor', 'gamma', 'gammaln', 'identity', 'log', 'log10', 'log1p', 'log2', 'logical_not', 'make_loss', 'negative',\n'ones_like', 'radians', 'rcbrt', 'reciprocal', 'relu', 'rint', 'round', 'rsqrt', 'shuffle', 'sigmoid', 'sign',\n'sin', 'sinh', 'size_array', 'softsign', 'sqrt', 'square', 'stop_gradient', 'tan', 'tanh', 'trunc', 'zeros_like']\n\n\"\"\"\n\nimport mxnet as mx\n\nfrom benchmark.opperf.utils.op_registry_utils import get_all_unary_operators\nfrom benchmark.opperf.utils.benchmark_utils import run_op_benchmarks\n\nfrom benchmark.opperf.utils.benchmark_utils import run_performance_test\nfrom benchmark.opperf.utils.common_utils import merge_map_list\nfrom benchmark.opperf.rules.default_params import MX_OP_MODULE\n\ndef run_mx_unary_operators_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Runs benchmarks with the given context, precision (dtype), and input data size (int64_tensor) for all the unary\n    operators in MXNet.\n\n    Parameters\n    ----------\n    ctx: mx.ctx\n        Context to run benchmarks\n    dtype: str, default 'float32'\n        Precision to use for benchmarks\n    profiler: str, default 'native'\n        Type of Profiler to use (native/python)\n    int64_tensor: str, default 'off'\n        Input tensor size to use for tests (if on, dimensions >= 2**32)\n    warmup: int, default 25\n        Number of times to run for warmup\n    runs: int, default 100\n        Number of runs to capture benchmark results\n\n    Returns\n    -------\n    Dictionary of results. Key -> Name of the operator, Value -> Benchmark results.\n\n    \"\"\"\n\n    standard_inputs = [{\"args\": [(1024, 1024)],\n                        \"num_outputs\":1},\n                       {\"args\": [(10000, 1)],\n                        \"num_outputs\":1}]\n    int64_tensor_inputs = [{\"args\": [(2**32, 1)],\n                            \"num_outputs\":1}]\n\n    if int64_tensor == 'on':\n        inputs = int64_tensor_inputs\n    else:\n        inputs = standard_inputs\n\n    # Run amp_multicast as it needs data as positional argument\n    amp_multicast_benchmark = run_performance_test([getattr(MX_OP_MODULE, \"amp_multicast\")],\n                                                   run_backward=True,\n                                                   dtype=dtype,\n                                                   ctx=ctx,\n                                                   profiler=profiler,\n                                                   inputs=inputs,\n                                                   warmup=warmup,\n                                                   runs=runs)\n\n    # Fetch all Unary Operators\n    mx_unary_broadcast_ops = get_all_unary_operators()\n\n    # Run benchmarks\n    mx_unary_op_results = run_op_benchmarks(mx_unary_broadcast_ops, dtype, ctx, profiler, int64_tensor, warmup, runs)\n    return merge_map_list(amp_multicast_benchmark + [mx_unary_op_results])\n"
  },
  {
    "path": "benchmark/opperf/opperf.py",
    "content": "#!/usr/bin/env python3\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# -*- coding: utf-8 -*-\n\n\"\"\"Commandline utility to run operator benchmarks\"\"\"\n\nimport argparse\nimport logging\nimport os\nimport sys\n\nimport mxnet as mx\n\nfrom benchmark.opperf.nd_operations.unary_operators import run_mx_unary_operators_benchmarks\nfrom benchmark.opperf.nd_operations.binary_operators import run_mx_binary_broadcast_operators_benchmarks, \\\n    run_mx_binary_element_wise_operators_benchmarks, run_mx_binary_misc_operators_benchmarks\nfrom benchmark.opperf.nd_operations.gemm_operators import run_gemm_operators_benchmarks\nfrom benchmark.opperf.nd_operations.random_sampling_operators import run_mx_random_sampling_operators_benchmarks\nfrom benchmark.opperf.nd_operations.reduction_operators import run_mx_reduction_operators_benchmarks\nfrom benchmark.opperf.nd_operations.sorting_searching_operators import run_sorting_searching_operators_benchmarks\nfrom benchmark.opperf.nd_operations.nn_activation_operators import run_activation_operators_benchmarks\nfrom benchmark.opperf.nd_operations.nn_conv_operators import run_pooling_operators_benchmarks, \\\n    run_convolution_operators_benchmarks, run_transpose_convolution_operators_benchmarks\nfrom benchmark.opperf.nd_operations.nn_basic_operators import run_nn_basic_operators_benchmarks\nfrom benchmark.opperf.nd_operations.nn_optimizer_operators import run_optimizer_operators_benchmarks\nfrom benchmark.opperf.nd_operations.indexing_routines import run_indexing_routines_benchmarks\nfrom benchmark.opperf.nd_operations.nn_loss_operators import run_loss_operators_benchmarks\nfrom benchmark.opperf.nd_operations.linalg_operators import run_linalg_operators_benchmarks\nfrom benchmark.opperf.nd_operations.misc_operators import run_mx_misc_operators_benchmarks\nfrom benchmark.opperf.nd_operations.array_manipulation_operators import run_rearrange_operators_benchmarks, \\\n    run_shape_operators_benchmarks, run_expanding_operators_benchmarks, run_rounding_operators_benchmarks, \\\n    run_join_split_operators_benchmarks\n\nfrom benchmark.opperf.utils.common_utils import merge_map_list, save_to_file\nfrom benchmark.opperf.utils.op_registry_utils import get_operators_with_no_benchmark, \\\n    get_current_runtime_features\n\n\ndef run_all_mxnet_operator_benchmarks(ctx=mx.cpu(), dtype='float32', profiler='native', int64_tensor='off', warmup=25, runs=100):\n    \"\"\"Run all the MXNet operators (NDArray) benchmarks.\n\n    Returns\n    -------\n    Dictionary of benchmark results.\n    \"\"\"\n    mxnet_operator_benchmark_results = []\n\n    # *************************MXNET TENSOR OPERATOR BENCHMARKS*****************************\n\n    # Run all Unary operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_mx_unary_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all Binary Broadcast, element_wise, and miscellaneous operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_mx_binary_broadcast_operators_benchmarks(ctx=ctx,\n                                                                                         dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n    mxnet_operator_benchmark_results.append(run_mx_binary_element_wise_operators_benchmarks(ctx=ctx,\n                                                                                            dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    mxnet_operator_benchmark_results.append(run_mx_binary_misc_operators_benchmarks(ctx=ctx,\n                                                                                         dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all GEMM operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_gemm_operators_benchmarks(ctx=ctx,\n                                                                          dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all Random sampling operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_mx_random_sampling_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all Reduction operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_mx_reduction_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all Sorting and Searching operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_sorting_searching_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all Indexing routines benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_indexing_routines_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all Array Rearrange operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_rearrange_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all Array Shape Manipulation operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_shape_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all Array Expansion operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_expanding_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all Array Rounding operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_rounding_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all Array Join & Split operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_join_split_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # ************************ MXNET NN OPERATOR BENCHMARKS ****************************\n\n    # Run all basic NN operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_nn_basic_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all Activation operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_activation_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all Pooling operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_pooling_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all Convolution operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_convolution_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all Optimizer operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_optimizer_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all Transpose Convolution operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_transpose_convolution_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all NN loss operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_loss_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Run all Miscellaneous operations benchmarks with default input values\n    mxnet_operator_benchmark_results.append(run_mx_misc_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # Linear Algebra operators do not work with int64 tensor data. Issue tracked here: https://github.com/apache/incubator-mxnet/issues/17716\n    if int64_tensor == 'off':\n        # Run all Linear Algebra operations benchmarks with default input values\n        mxnet_operator_benchmark_results.append(run_linalg_operators_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs))\n\n    # ****************************** PREPARE FINAL RESULTS ********************************\n    final_benchmark_result_map = merge_map_list(mxnet_operator_benchmark_results)\n    return final_benchmark_result_map\n\n\ndef _parse_mxnet_context(ctx):\n    if not ctx:\n        raise ValueError(\"Context cannot be null or empty\")\n\n    if ctx.lower() in ['cpu', 'gpu']:\n        return mx.context.Context(ctx)\n    elif ctx.lower().startwith('gpu('):\n        device_id = int(ctx[4:-1])\n        return mx.gpu(device_id)\n\n\ndef main():\n    # 1. GET USER INPUTS\n    parser = argparse.ArgumentParser(description='Run all the MXNet operator benchmarks')\n\n    parser.add_argument('--ctx', type=str, default='cpu',\n                        help='Global context to run all benchmarks. By default, cpu on a '\n                             'CPU machine, gpu(0) on a GPU machine. '\n                             'Valid Inputs - cpu, gpu, gpu(0), gpu(1)...')\n    parser.add_argument('--dtype', type=str, default='float32', help='DType (Precision) to run benchmarks. By default, '\n                                                                     'float32. Valid Inputs - float32, float64, int32, '\n                                                                     'int64')\n    parser.add_argument('-f', '--output-format', type=str, default='json',\n                        choices=['json', 'md'],\n                        help='Benchmark result output format. By default, json. '\n                             'Valid Inputs - json, md')\n\n    parser.add_argument('-o', '--output-file', type=str, default='./mxnet_operator_benchmarks.json',\n                        help='Name and path for the '\n                             'output file.')\n\n    parser.add_argument('-p', '--profiler', type=str, default='native',\n                        help='Use built-in CPP profiler (native) or Python'\n                             'time module.'\n                             'Valid Inputs - native, python')\n\n    parser.add_argument('--int64-tensor', type=str, default='off',\n                        help='Run performance tests with large tensor input'\n                             'data (dimension >= 2**32) or standard input data.'\n                             'Valid Inputs - on, off')\n\n    parser.add_argument('-w', '--warmup', type=int, default=25,\n                        help='Number of times to run for warmup.'\n                             'Valid Inputs - positive integers')\n\n    parser.add_argument('-r', '--runs', type=int, default=100,\n                        help='Number of runs to capture benchmark results.'\n                             'Valid Inputs - positive integers') \n\n    args = parser.parse_args()\n    logging.info(f\"Running MXNet operator benchmarks with the following options: {args}\")\n    assert not os.path.isfile(args.output_file),\\\n        f\"Output file {args.output_file} already exists.\"\n\n    # 2. RUN BENCHMARKS\n    ctx = _parse_mxnet_context(args.ctx)\n    dtype = args.dtype\n    profiler = args.profiler\n    int64_tensor = args.int64_tensor\n    warmup = args.warmup\n    runs = args.runs\n    benchmark_results = run_all_mxnet_operator_benchmarks(ctx=ctx, dtype=dtype, profiler=profiler, int64_tensor=int64_tensor, warmup=warmup, runs=runs)\n\n    # Sort benchmark results alphabetically by op name\n    final_benchmark_results = dict()\n    for key in sorted(benchmark_results.keys()):\n        final_benchmark_results[key] = benchmark_results[key]\n\n    # 3. PREPARE OUTPUTS\n    run_time_features = get_current_runtime_features()\n    save_to_file(final_benchmark_results, args.output_file, args.output_format, run_time_features, profiler)\n\n    # 4. Generate list of MXNet operators not covered in benchmarks\n    ops_not_covered = get_operators_with_no_benchmark(final_benchmark_results.keys())\n    for idx, op in enumerate(ops_not_covered):\n        print(f\"{idx}. {op}\")\n\n    return 0\n\n\nif __name__ == '__main__':\n    sys.exit(main())\n"
  },
  {
    "path": "benchmark/opperf/results/mxnet_operator_benchmark_results_cpu.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# MXNet Operator Benchmarks\n\n## Settings\n\n1. MXNet - v1.4.1\n2. Instance - C5.8x\n\n| Operator | Avg Forward Time (ms) | Avg. Backward Time (ms) | Max Mem Usage (Storage) (Bytes) | Inputs |\n| :---: | :---: | :---: | :---:| :--- |\n| shuffle | 0.8901 | --- | 4194.3042 | {'data': (1024, 1024)} |\n| shuffle | 1.2146 | --- | 40.0 | {'data': (10000, 1)} |\n| shuffle | 1.8777 | --- | 4000.0 | {'data': (10000, 100)} |\n| broadcast_equal | 0.006 | --- | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| broadcast_hypot | 0.0108 | 0.0135 | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| ceil | 3.4305 | --- | 4194.3042 | {'data': (1024, 1024)} |\n| ceil | 0.0507 | --- | 40.0 | {'data': (10000, 1)} |\n| ceil | 3.317 | --- | 4000.0 | {'data': (10000, 100)} |\n| sum | 32.4206 | 25.5443 | 0.002 | {'data': (1024, 1024), 'axis': ()} |\n| sum | 0.3393 | 0.2507 | 0.004 | {'data': (10000, 1), 'axis': 0} |\n| sum | 31.0189 | 24.7422 | 0.002 | {'data': (10000, 100), 'axis': (0, 1)} |\n| broadcast_logical_xor | 0.0068 | --- | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| erf | 35.0669 | 16.5842 | 4194.3042 | {'data': (1024, 1024)} |\n| erf | 0.3982 | 0.1734 | 40.0 | {'data': (10000, 1)} |\n| erf | 29.4103 | 14.3537 | 4000.0 | {'data': (10000, 100)} |\n| tanh | 11.2211 | 6.1798 | 2097.1521 | {'data': (1024, 1024)} |\n| tanh | 0.1628 | 0.0622 | 40.0 | {'data': (10000, 1)} |\n| tanh | 10.7941 | 6.0085 | 4000.0 | {'data': (10000, 100)} |\n| arcsinh | 10.0168 | 8.5245 | 2097.1521 | {'data': (1024, 1024)} |\n| arcsinh | 0.1111 | 0.0905 | 40.0 | {'data': (10000, 1)} |\n| arcsinh | 9.4415 | 7.9082 | 2000.0 | {'data': (10000, 100)} |\n| fix | 15.541 | --- | 4194.3042 | {'data': (1024, 1024)} |\n| fix | 0.1615 | --- | 40.0 | {'data': (10000, 1)} |\n| fix | 14.591 | --- | 4000.0 | {'data': (10000, 100)} |\n| broadcast_maximum | 0.0097 | 0.0099 | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| sin | 14.4123 | 16.5642 | 2097.1521 | {'data': (1024, 1024)} |\n| sin | 0.1459 | 0.156 | 40.0 | {'data': (10000, 1)} |\n| sin | 13.821 | 15.4752 | 2000.0 | {'data': (10000, 100)} |\n| random_normal | 151.0089 | --- | 4194.3042 | {'shape': (1024, 1024)} |\n| random_normal | 1.456 | --- | 40.0 | {'shape': (10000, 1)} |\n| random_normal | 144.775 | --- | 2000.0 | {'shape': (10000, 100)} |\n| sqrt | 3.3861 | 5.1123 | 2097.1521 | {'data': (1024, 1024)} |\n| sqrt | 0.0393 | 0.0548 | 20.0 | {'data': (10000, 1)} |\n| sqrt | 3.3037 | 4.7883 | 2000.0 | {'data': (10000, 100)} |\n| BlockGrad | 0.3275 | --- | 4194.3042 | {'data': (1024, 1024)} |\n| BlockGrad | 0.0161 | --- | 40.0 | {'data': (10000, 1)} |\n| BlockGrad | 0.3118 | --- | 4000.0 | {'data': (10000, 100)} |\n| sample_exponential | 123.8534 | --- | 8388.6084 | {'lam': [1.0, 8.5], 'shape': (1024, 1024)} |\n| sample_exponential | 1.3394 | --- | 80.0 | {'lam': [1.0, 8.5], 'shape': (10000, 1)} |\n| sample_exponential | 118.4786 | --- | 8000.0 | {'lam': [1.0, 8.5], 'shape': (10000, 100)} |\n| sample_gamma | 529.0305 | --- | 8388.6084 | {'alpha': [0.0, 2.5], 'shape': (1024, 1024), 'beta': [1.0, 0.7]} |\n| sample_gamma | 5.7426 | --- | 80.0 | {'alpha': [0.0, 2.5], 'shape': (10000, 1), 'beta': [1.0, 0.7]} |\n| sample_gamma | 496.0531 | --- | 8000.0 | {'alpha': [0.0, 2.5], 'shape': (10000, 100), 'beta': [1.0, 0.7]} |\n| log2 | 12.3183 | 4.5842 | 2097.1521 | {'data': (1024, 1024)} |\n| log2 | 0.1269 | 0.0459 | 40.0 | {'data': (10000, 1)} |\n| log2 | 11.6719 | 4.2632 | 4000.0 | {'data': (10000, 100)} |\n| broadcast_greater_equal | 0.0092 | --- | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| FullyConnected | 18.4677 | 21.6917 | 8.192 | {'data': (32, 3, 256, 256), 'num_hidden': 64, 'weight': (64, 196608), 'bias': (64,), 'flatten': True} |\n| FullyConnected | 20.3379 | 38.8295 | 6291.4561 | {'data': (32, 3, 256, 256), 'num_hidden': 64, 'weight': (64, 256), 'bias': (64,), 'flatten': False} |\n| cos | 14.8699 | 16.8678 | 2097.1521 | {'data': (1024, 1024)} |\n| cos | 0.1511 | 0.1585 | 40.0 | {'data': (10000, 1)} |\n| cos | 14.0109 | 15.5246 | 2000.0 | {'data': (10000, 100)} |\n| broadcast_mul | 0.0075 | 0.0075 | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| arccos | 21.5631 | 12.8768 | 4194.3042 | {'data': (1024, 1024)} |\n| arccos | 0.1719 | 0.1084 | 40.0 | {'data': (10000, 1)} |\n| arccos | 15.3153 | 7.9161 | 2000.0 | {'data': (10000, 100)} |\n| stop_gradient | --- | --- | 4194.3042 | {'data': (1024, 1024)} |\n| stop_gradient | --- | --- | 40.0 | {'data': (10000, 1)} |\n| stop_gradient | --- | --- | 4000.0 | {'data': (10000, 100)} |\n| broadcast_sub | 0.0078 | 0.0059 | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| random_poisson | 112.7425 | --- | 4194.3042 | {'shape': (1024, 1024)} |\n| random_poisson | 1.0701 | --- | 40.0 | {'shape': (10000, 1)} |\n| random_poisson | 114.3405 | --- | 2000.0 | {'shape': (10000, 100)} |\n| rsqrt | 4.3564 | 7.0663 | 2097.1521 | {'data': (1024, 1024)} |\n| rsqrt | 0.075 | 0.0861 | 40.0 | {'data': (10000, 1)} |\n| rsqrt | 4.5076 | 6.6598 | 4000.0 | {'data': (10000, 100)} |\n| nansum | 34.2019 | 57.1624 | 0.002 | {'data': (1024, 1024), 'axis': ()} |\n| nansum | 0.3683 | 0.5326 | 0.002 | {'data': (10000, 1), 'axis': 0} |\n| nansum | 32.9698 | 55.4243 | 0.002 | {'data': (10000, 100), 'axis': (0, 1)} |\n| hard_sigmoid | 7.5926 | 6.5839 | 2097.1521 | {'data': (1024, 1024), 'alpha': 0.25, 'beta': 0.5} |\n| hard_sigmoid | 0.1086 | 0.0895 | 40.0 | {'data': (10000, 1), 'alpha': 0.25, 'beta': 0.5} |\n| hard_sigmoid | 8.1285 | 6.6014 | 4000.0 | {'data': (10000, 100), 'alpha': 0.25, 'beta': 0.5} |\n| softmax | 25.4074 | 9.4933 | 2097.1521 | {'data': (1024, 1024), 'axis': -1, 'temperature': 0.5} |\n| softmax | 0.4022 | 0.3145 | 40.0 | {'data': (10000, 1), 'axis': -1, 'temperature': 0.5} |\n| softmax | 25.604 | 9.4286 | 4000.0 | {'data': (10000, 100), 'axis': -1, 'temperature': 0.5} |\n| random_negative_binomial | 285.8721 | --- | 4194.3042 | {'k': 1, 'p': 1, 'shape': (1024, 1024)} |\n| random_negative_binomial | 2.839 | --- | 40.0 | {'k': 1, 'p': 1, 'shape': (10000, 1)} |\n| random_negative_binomial | 273.034 | --- | 2000.0 | {'k': 1, 'p': 1, 'shape': (10000, 100)} |\n| BatchNorm | 66.062 | 88.4693 | 25165.8359 | {'data': (32, 3, 256, 256), 'gamma': (3,), 'beta': (3,), 'moving_mean': (3,), 'moving_var': (3,)} |\n| BatchNorm | 101.3006 | 134.4362 | 38400.0117 | {'data': (32, 3, 10000, 10), 'gamma': (3,), 'beta': (3,), 'moving_mean': (3,), 'moving_var': (3,)} |\n| Pooling | 0.5533 | 0.6485 | 49.152 | {'data': (32, 3, 256), 'kernel': 3, 'pool_type': 'avg', 'global_pool': 0, 'stride': 1, 'pad': 1, 'layout': 'NCW'} |\n| radians | 3.3238 | 3.9704 | 4194.3042 | {'data': (1024, 1024)} |\n| radians | 0.0391 | 0.0436 | 40.0 | {'data': (10000, 1)} |\n| radians | 3.2462 | 3.775 | 4000.0 | {'data': (10000, 100)} |\n| arctanh | 13.3211 | 6.3172 | 2097.1521 | {'data': (1024, 1024)} |\n| arctanh | 0.1498 | 0.0683 | 40.0 | {'data': (10000, 1)} |\n| arctanh | 12.5376 | 6.0177 | 2000.0 | {'data': (10000, 100)} |\n| nanprod | 34.3464 | 57.9841 | 0.004 | {'data': (1024, 1024), 'axis': ()} |\n| nanprod | 0.3638 | 0.5336 | 0.004 | {'data': (10000, 1), 'axis': 0} |\n| nanprod | 32.83 | 55.2982 | 0.002 | {'data': (10000, 100), 'axis': (0, 1)} |\n| elemwise_add | 0.0065 | --- | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| cosh | 8.4872 | 10.6597 | 2097.1521 | {'data': (1024, 1024)} |\n| cosh | 0.1015 | 0.1201 | 40.0 | {'data': (10000, 1)} |\n| cosh | 8.3937 | 10.6244 | 4000.0 | {'data': (10000, 100)} |\n| tan | 15.4508 | 6.0752 | 2097.1521 | {'data': (1024, 1024)} |\n| tan | 0.1549 | 0.0591 | 40.0 | {'data': (10000, 1)} |\n| tan | 14.6992 | 5.802 | 2000.0 | {'data': (10000, 100)} |\n| broadcast_not_equal | 0.0054 | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| trunc | 3.493 | --- | 2097.1521 | {'data': (1024, 1024)} |\n| trunc | 0.0505 | --- | 40.0 | {'data': (10000, 1)} |\n| trunc | 3.1751 | --- | 2000.0 | {'data': (10000, 100)} |\n| min_axis | 36.7382 | --- | 0.004 | {'data': (1024, 1024), 'axis': ()} |\n| min_axis | 0.4225 | --- | 0.004 | {'data': (10000, 1), 'axis': 0} |\n| min_axis | 31.3261 | --- | 0.004 | {'data': (10000, 100), 'axis': (0, 1)} |\n| random_uniform | 44.7633 | --- | 4194.3042 | {'low': 0, 'high': 5, 'shape': (1024, 1024)} |\n| random_uniform | 0.4607 | --- | 40.0 | {'low': 0, 'high': 5, 'shape': (10000, 1)} |\n| random_uniform | 42.9135 | --- | 4000.0 | {'low': 0, 'high': 5, 'shape': (10000, 100)} |\n| abs | 4.3965 | 13.406 | 4194.3042 | {'data': (1024, 1024)} |\n| abs | 0.0696 | 0.1374 | 40.0 | {'data': (10000, 1)} |\n| abs | 4.3552 | 13.7197 | 4000.0 | {'data': (10000, 100)} |\n| broadcast_lesser_equal | 0.0054 | --- | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| random_randint | 65.414 | --- | 4194.3042 | {'low': 0, 'high': 5, 'shape': (1024, 1024)} |\n| random_randint | 0.6331 | --- | 40.0 | {'low': 0, 'high': 5, 'shape': (10000, 1)} |\n| random_randint | 61.32 | --- | 4000.0 | {'low': 0, 'high': 5, 'shape': (10000, 100)} |\n| log1p | 13.6758 | 5.2497 | 2097.1521 | {'data': (1024, 1024)} |\n| log1p | 0.1493 | 0.0562 | 40.0 | {'data': (10000, 1)} |\n| log1p | 12.9494 | 5.0609 | 2000.0 | {'data': (10000, 100)} |\n| log | 11.9666 | 5.1096 | 4194.3042 | {'data': (1024, 1024)} |\n| log | 0.1306 | 0.0588 | 40.0 | {'data': (10000, 1)} |\n| log | 11.8985 | 5.0319 | 2000.0 | {'data': (10000, 100)} |\n| round | 14.6427 | --- | 4194.3042 | {'data': (1024, 1024)} |\n| round | 0.1424 | --- | 20.0 | {'data': (10000, 1)} |\n| round | 13.58 | --- | 2000.0 | {'data': (10000, 100)} |\n| sample_negative_binomial | 1263.9417 | --- | 8388.6084 | {'k': [20, 49], 'shape': (1024, 1024), 'p': [0.4, 0.77]} |\n| sample_negative_binomial | 12.5213 | --- | 80.0 | {'k': [20, 49], 'shape': (10000, 1), 'p': [0.4, 0.77]} |\n| sample_negative_binomial | 1207.5739 | --- | 8000.0 | {'k': [20, 49], 'shape': (10000, 100), 'p': [0.4, 0.77]} |\n| max | 30.7008 | 55.863 | 0.002 | {'data': (1024, 1024), 'axis': ()} |\n| max | 0.3287 | 0.5147 | 0.004 | {'data': (10000, 1), 'axis': 0} |\n| max | 29.4913 | 53.255 | 0.002 | {'data': (10000, 100), 'axis': (0, 1)} |\n| mean | 31.9337 | 35.9235 | 0.002 | {'data': (1024, 1024), 'axis': ()} |\n| mean | 0.4088 | 0.3453 | 0.002 | {'data': (10000, 1), 'axis': 0} |\n| mean | 31.5658 | 34.609 | 0.004 | {'data': (10000, 100), 'axis': (0, 1)} |\n| sign | 10.1736 | 4.1682 | 4194.3042 | {'data': (1024, 1024)} |\n| sign | 0.1251 | 0.0588 | 40.0 | {'data': (10000, 1)} |\n| sign | 9.5196 | 3.9109 | 2000.0 | {'data': (10000, 100)} |\n| broadcast_power | 0.0117 | 0.0112 | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| argmax_channel | 10.9332 | --- | 4.096 | {'data': (1024, 1024)} |\n| argmax_channel | 0.2703 | --- | 40.0 | {'data': (10000, 1)} |\n| argmax_channel | 10.7759 | --- | 40.0 | {'data': (10000, 100)} |\n| flatten | --- | --- | 4194.3042 | {'data': (1024, 1024)} |\n| flatten | --- | --- | 40.0 | {'data': (10000, 1)} |\n| flatten | --- | --- | 4000.0 | {'data': (10000, 100)} |\n| ones_like | 2.127 | --- | 4194.3042 | {'data': (1024, 1024)} |\n| ones_like | 0.028 | --- | 40.0 | {'data': (10000, 1)} |\n| ones_like | 1.8846 | --- | 4000.0 | {'data': (10000, 100)} |\n| negative | 2.6672 | --- | 4194.3042 | {'data': (1024, 1024)} |\n| negative | 0.0321 | --- | 40.0 | {'data': (10000, 1)} |\n| negative | 2.4958 | --- | 4000.0 | {'data': (10000, 100)} |\n| elemwise_mul | 0.0054 | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| batch_dot | 766.5307 | 1365.6267 | 134217.7344 | {'lhs': (32, 1024, 1024), 'rhs': (32, 1024, 1024)} |\n| batch_dot | 37.618 | 46.1098 | 128000.0 | {'lhs': (32, 1000, 10), 'rhs': (32, 1000, 10), 'transpose_b': True} |\n| batch_dot | 1.3618 | 4.0882 | 6.4 | {'lhs': (32, 1000, 1), 'rhs': (32, 100, 1000), 'transpose_a': True, 'transpose_b': True} |\n| sum_axis | 33.2033 | --- | 0.004 | {'data': (1024, 1024), 'axis': ()} |\n| sum_axis | 0.3155 | --- | 0.004 | {'data': (10000, 1), 'axis': 0} |\n| sum_axis | 30.9792 | --- | 0.004 | {'data': (10000, 100), 'axis': (0, 1)} |\n| floor | 3.5835 | --- | 4194.3042 | {'data': (1024, 1024)} |\n| floor | 0.0499 | --- | 20.0 | {'data': (10000, 1)} |\n| floor | 3.3519 | --- | 4000.0 | {'data': (10000, 100)} |\n| logical_not | 3.0748 | --- | 4194.3042 | {'data': (1024, 1024)} |\n| logical_not | 0.0319 | --- | 40.0 | {'data': (10000, 1)} |\n| logical_not | 3.0173 | --- | 4000.0 | {'data': (10000, 100)} |\n| log10 | 12.3647 | 4.5036 | 2097.1521 | {'data': (1024, 1024)} |\n| log10 | 0.1647 | 0.0619 | 40.0 | {'data': (10000, 1)} |\n| log10 | 11.7758 | 4.231 | 2000.0 | {'data': (10000, 100)} |\n| rcbrt | 11.737 | 14.931 | 2097.1521 | {'data': (1024, 1024)} |\n| rcbrt | 0.1241 | 0.1421 | 40.0 | {'data': (10000, 1)} |\n| rcbrt | 11.2254 | 14.2139 | 2000.0 | {'data': (10000, 100)} |\n| broadcast_logical_or | 0.0093 | --- | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| sample_normal | 304.5372 | --- | 8388.6084 | {'mu': [2.0, 2.5], 'shape': (1024, 1024), 'sigma': [1.0, 3.7]} |\n| sample_normal | 2.8403 | --- | 80.0 | {'mu': [2.0, 2.5], 'shape': (10000, 1), 'sigma': [1.0, 3.7]} |\n| sample_normal | 284.6853 | --- | 8000.0 | {'mu': [2.0, 2.5], 'shape': (10000, 100), 'sigma': [1.0, 3.7]} |\n| broadcast_minimum | 0.0073 | 0.0073 | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| arctan | 10.4997 | 6.4532 | 2097.1521 | {'data': (1024, 1024)} |\n| arctan | 0.1269 | 0.0683 | 40.0 | {'data': (10000, 1)} |\n| arctan | 10.1779 | 6.1741 | 2000.0 | {'data': (10000, 100)} |\n| broadcast_mod | 0.0131 | 0.0127 | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| size_array | 0.0056 | --- | 0.008 | {'data': (1024, 1024)} |\n| size_array | 0.005 | --- | 0.008 | {'data': (10000, 1)} |\n| size_array | 0.0081 | --- | 0.004 | {'data': (10000, 100)} |\n| make_loss | 0.4874 | --- | 4194.3042 | {'data': (1024, 1024)} |\n| make_loss | 0.013 | --- | 40.0 | {'data': (10000, 1)} |\n| make_loss | 0.3483 | --- | 4000.0 | {'data': (10000, 100)} |\n| broadcast_greater | 0.0082 | --- | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| gammaln | 49.6217 | 105.7931 | 2097.1521 | {'data': (1024, 1024)} |\n| gammaln | 0.4789 | 0.9577 | 40.0 | {'data': (10000, 1)} |\n| gammaln | 48.474 | 102.211 | 4000.0 | {'data': (10000, 100)} |\n| broadcast_lesser | 0.0084 | --- | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| max_axis | 30.1487 | --- | 0.004 | {'data': (1024, 1024), 'axis': ()} |\n| max_axis | 0.3101 | --- | 0.004 | {'data': (10000, 1), 'axis': 0} |\n| max_axis | 29.4315 | --- | 0.004 | {'data': (10000, 100), 'axis': (0, 1)} |\n| degrees | 3.659 | 4.2964 | 2097.1521 | {'data': (1024, 1024)} |\n| degrees | 0.0595 | 0.0538 | 20.0 | {'data': (10000, 1)} |\n| degrees | 3.8676 | 4.1255 | 4000.0 | {'data': (10000, 100)} |\n| sinh | 8.9259 | 10.3014 | 2097.1521 | {'data': (1024, 1024)} |\n| sinh | 0.0989 | 0.1048 | 40.0 | {'data': (10000, 1)} |\n| sinh | 8.4579 | 9.7402 | 2000.0 | {'data': (10000, 100)} |\n| zeros_like | 2.4764 | --- | 4194.3042 | {'data': (1024, 1024)} |\n| zeros_like | 0.0056 | --- | 40.0 | {'data': (10000, 1)} |\n| zeros_like | 2.3254 | --- | 4000.0 | {'data': (10000, 100)} |\n| arccosh | 6.8035 | 7.7818 | 2097.1521 | {'data': (1024, 1024)} |\n| arccosh | 0.0764 | 0.0847 | 40.0 | {'data': (10000, 1)} |\n| arccosh | 6.444 | 7.5842 | 2000.0 | {'data': (10000, 100)} |\n| prod | 28.2885 | 55.9765 | 0.002 | {'data': (1024, 1024), 'axis': ()} |\n| prod | 0.2996 | 0.5213 | 0.004 | {'data': (10000, 1), 'axis': 0} |\n| prod | 26.9891 | 54.6354 | 0.004 | {'data': (10000, 100), 'axis': (0, 1)} |\n| random_gamma | 247.5786 | --- | 2097.1521 | {'shape': (1024, 1024)} |\n| random_gamma | 2.3986 | --- | 40.0 | {'shape': (10000, 1)} |\n| random_gamma | 237.5963 | --- | 2000.0 | {'shape': (10000, 100)} |\n| broadcast_minus | --- | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| Flatten | 0.3339 | --- | 4194.3042 | {'data': (1024, 1024)} |\n| Flatten | 0.0152 | --- | 40.0 | {'data': (10000, 1)} |\n| Flatten | 0.3546 | --- | 4000.0 | {'data': (10000, 100)} |\n| expm1 | 9.8241 | 11.7609 | 4194.3042 | {'data': (1024, 1024)} |\n| expm1 | 0.1844 | 0.1675 | 40.0 | {'data': (10000, 1)} |\n| expm1 | 9.0366 | 10.4387 | 4000.0 | {'data': (10000, 100)} |\n| elemwise_div | 0.0064 | --- | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| LeakyReLU | 10.3625 | 12.5441 | 4194.3042 | {'data': (1024, 1024), 'act_type': 'leaky', 'slope': 0.1} |\n| LeakyReLU | 0.1076 | 0.1277 | 40.0 | {'data': (10000, 1), 'act_type': 'leaky', 'slope': 0.1} |\n| LeakyReLU | 9.5913 | 11.7957 | 2000.0 | {'data': (10000, 100), 'act_type': 'leaky', 'slope': 0.1} |\n| LeakyReLU | 12.337 | 12.6383 | 2097.1521 | {'data': (1024, 1024), 'act_type': 'elu', 'slope': 0.1} |\n| LeakyReLU | 0.1305 | 0.1217 | 40.0 | {'data': (10000, 1), 'act_type': 'elu', 'slope': 0.1} |\n| LeakyReLU | 11.652 | 11.8465 | 4000.0 | {'data': (10000, 100), 'act_type': 'elu', 'slope': 0.1} |\n| LeakyReLU | 12.4973 | 11.4957 | 2097.1521 | {'data': (1024, 1024), 'act_type': 'selu'} |\n| LeakyReLU | 0.1295 | 0.1176 | 40.0 | {'data': (10000, 1), 'act_type': 'selu'} |\n| LeakyReLU | 12.2224 | 11.548 | 4000.0 | {'data': (10000, 100), 'act_type': 'selu'} |\n| LeakyReLU | 16.9543 | 306.6579 | 2097.1521 | {'data': (1024, 1024), 'act_type': 'prelu', 'gamma': (1, 1024)} |\n| LeakyReLU | 0.2859 | 1.9528 | 20.0 | {'data': (10000, 1), 'act_type': 'prelu', 'gamma': (1, 1)} |\n| LeakyReLU | 16.0125 | 231.8273 | 2000.0 | {'data': (10000, 100), 'act_type': 'prelu', 'gamma': (1, 100)} |\n| rint | 14.9397 | --- | 4194.3042 | {'data': (1024, 1024)} |\n| rint | 0.1535 | --- | 40.0 | {'data': (10000, 1)} |\n| rint | 14.5915 | --- | 4000.0 | {'data': (10000, 100)} |\n| identity | --- | --- | 4194.3042 | {'data': (1024, 1024)} |\n| identity | --- | --- | 40.0 | {'data': (10000, 1)} |\n| identity | --- | --- | 4000.0 | {'data': (10000, 100)} |\n| softsign | 3.9985 | 7.05 | 2097.1521 | {'data': (1024, 1024)} |\n| softsign | 0.0486 | 0.0737 | 40.0 | {'data': (10000, 1)} |\n| softsign | 3.7662 | 6.7975 | 2000.0 | {'data': (10000, 100)} |\n| broadcast_div | 0.0083 | 0.0075 | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| square | 4.2037 | 4.9639 | 2097.1521 | {'data': (1024, 1024)} |\n| square | 0.0467 | 0.0558 | 40.0 | {'data': (10000, 1)} |\n| square | 3.9986 | 4.6533 | 2000.0 | {'data': (10000, 100)} |\n| elemwise_sub | 0.0058 | --- | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| dot | 14.562 | 29.1605 | 4194.3042 | {'lhs': (1024, 1024), 'rhs': (1024, 1024)} |\n| dot | 0.745 | 1.5842 | 2000.0 | {'lhs': (1000, 10), 'rhs': (1000, 10), 'transpose_b': True} |\n| dot | 0.0579 | 0.1673 | 0.2 | {'lhs': (1000, 1), 'rhs': (100, 1000), 'transpose_a': True, 'transpose_b': True} |\n| broadcast_logical_and | 0.0071 | --- | 0.024 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| broadcast_add | 0.0081 | 0.0066 | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| random_exponential | 63.2732 | --- | 4194.3042 | {'shape': (1024, 1024)} |\n| random_exponential | 0.6453 | --- | 40.0 | {'shape': (10000, 1)} |\n| random_exponential | 59.2788 | --- | 2000.0 | {'shape': (10000, 100)} |\n| Dropout | 249.4661 | 23.5141 | 37748.7344 | {'data': (32, 3, 256, 256), 'p': 0.5, 'mode': 'always'} |\n| Dropout | 3.9634 | 0.3516 | 600.0 | {'data': (10000, 10), 'p': 0.5, 'mode': 'always'} |\n| exp | 8.9413 | --- | 4194.3042 | {'data': (1024, 1024)} |\n| exp | 0.0971 | --- | 40.0 | {'data': (10000, 1)} |\n| exp | 7.9211 | --- | 4000.0 | {'data': (10000, 100)} |\n| random_generalized_negative_binomial | 362.7789 | --- | 2097.1521 | {'shape': (1024, 1024)} |\n| random_generalized_negative_binomial | 3.4276 | --- | 40.0 | {'shape': (10000, 1)} |\n| random_generalized_negative_binomial | 344.3516 | --- | 4000.0 | {'shape': (10000, 100)} |\n| min | 30.8723 | 55.9413 | 0.002 | {'data': (1024, 1024), 'axis': ()} |\n| min | 0.3168 | 0.5206 | 0.002 | {'data': (10000, 1), 'axis': 0} |\n| min | 29.9547 | 53.8245 | 0.004 | {'data': (10000, 100), 'axis': (0, 1)} |\n| erfinv | 79.987 | 99.2274 | 2097.1521 | {'data': (1024, 1024)} |\n| erfinv | 0.7567 | 0.9105 | 40.0 | {'data': (10000, 1)} |\n| erfinv | 76.0479 | 95.5001 | 2000.0 | {'data': (10000, 100)} |\n| broadcast_plus | --- | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| arcsin | 16.3157 | 7.6156 | 2097.1521 | {'data': (1024, 1024)} |\n| arcsin | 0.1611 | 0.0758 | 40.0 | {'data': (10000, 1)} |\n| arcsin | 16.0225 | 7.5081 | 2000.0 | {'data': (10000, 100)} |\n| sample_generalized_negative_binomial | 629.1785 | --- | 8388.6084 | {'mu': [2.0, 2.5], 'shape': (1024, 1024), 'alpha': [0.0, 2.5]} |\n| sample_generalized_negative_binomial | 6.8681 | --- | 80.0 | {'mu': [2.0, 2.5], 'shape': (10000, 1), 'alpha': [0.0, 2.5]} |\n| sample_generalized_negative_binomial | 604.3484 | --- | 8000.0 | {'mu': [2.0, 2.5], 'shape': (10000, 100), 'alpha': [0.0, 2.5]} |\n| relu | 11.0979 | 8.3262 | 2097.1521 | {'data': (1024, 1024)} |\n| relu | 0.1163 | 0.0853 | 40.0 | {'data': (10000, 1)} |\n| relu | 10.6863 | 8.0702 | 4000.0 | {'data': (10000, 100)} |\n| cbrt | 11.3121 | 6.5254 | 2097.1521 | {'data': (1024, 1024)} |\n| cbrt | 0.1238 | 0.0687 | 40.0 | {'data': (10000, 1)} |\n| cbrt | 10.4631 | 6.0997 | 2000.0 | {'data': (10000, 100)} |\n| sample_uniform | 89.1332 | --- | 8388.6084 | {'low': [0.0, 2.5], 'shape': (1024, 1024), 'high': [1.0, 3.7]} |\n| sample_uniform | 0.8895 | --- | 80.0 | {'low': [0.0, 2.5], 'shape': (10000, 1), 'high': [1.0, 3.7]} |\n| sample_uniform | 84.4477 | --- | 8000.0 | {'low': [0.0, 2.5], 'shape': (10000, 100), 'high': [1.0, 3.7]} |\n| Convolution | 13.4072 | 17.0238 | 56610.418 | {'data': (32, 3, 256), 'weight': (64, 3, 3), 'bias': (64,), 'kernel': (3,), 'stride': (1,), 'dilate': (1,), 'pad': (0,), 'num_filter': 64, 'layout': 'NCW'} |\n| sample_poisson | 512.1068 | --- | 8388.6084 | {'lam': [1.0, 8.5], 'shape': (1024, 1024)} |\n| sample_poisson | 4.6203 | --- | 80.0 | {'lam': [1.0, 8.5], 'shape': (10000, 1)} |\n| sample_poisson | 474.1238 | --- | 8000.0 | {'lam': [1.0, 8.5], 'shape': (10000, 100)} |\n| log_softmax | 21.4413 | 15.7456 | 2097.1521 | {'data': (1024, 1024), 'axis': -1, 'temperature': 0.5} |\n| log_softmax | 0.4613 | 0.2958 | 20.0 | {'data': (10000, 1), 'axis': -1, 'temperature': 0.5} |\n| log_softmax | 21.9745 | 15.2407 | 4000.0 | {'data': (10000, 100), 'axis': -1, 'temperature': 0.5} |\n| gamma | 35.1027 | 124.2015 | 2097.1521 | {'data': (1024, 1024)} |\n| gamma | 0.3611 | 1.1177 | 20.0 | {'data': (10000, 1)} |\n| gamma | 33.636 | 117.6889 | 2000.0 | {'data': (10000, 100)} |\n| reciprocal | 3.4646 | 6.1106 | 2097.1521 | {'data': (1024, 1024)} |\n| reciprocal | 0.0413 | 0.0635 | 40.0 | {'data': (10000, 1)} |\n| reciprocal | 3.2553 | 5.8762 | 2000.0 | {'data': (10000, 100)} |\n| sigmoid | 9.8017 | 5.9639 | 2097.1521 | {'data': (1024, 1024)} |\n| sigmoid | 0.1095 | 0.0651 | 40.0 | {'data': (10000, 1)} |\n| sigmoid | 9.0443 | 5.7901 | 2000.0 | {'data': (10000, 100)} |"
  },
  {
    "path": "benchmark/opperf/results/mxnet_operator_benchmark_results_gpu.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# MXNet GPU Operator Benchmarks\n\n## Settings\n\n1. MXNet - v1.5.0 with CUDA 10.1\n2. Instance - P3.2x\n \n| Operator | Avg Forward Time (ms) | Avg. Backward Time (ms) | Max Mem Usage (Storage) (Bytes) | Inputs |\n| :---: | :---: | :---: | :---:| :--- |\n| rcbrt | 0.0384 | 0.0393 | 2097.1521 | {'data': (1024, 1024)} |\n| rcbrt | 0.0268 | 0.0252 | 20.0 | {'data': (10000, 1)} |\n| rcbrt | 0.0378 | 0.039 | 2000.0 | {'data': (10000, 100)} |\n| min | 0.0679 | 0.0955 | 0.002 | {'axis': (), 'data': (1024, 1024)} |\n| min | 0.0337 | 0.022 | 0.002 | {'axis': 0, 'data': (10000, 1)} |\n| min | 0.0669 | 0.0921 | 0.002 | {'axis': (0, 1), 'data': (10000, 100)} |\n| broadcast_maximum | 0.0284 | 0.0308 | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| Flatten | 0.0416 | --- | 2097.1521 | {'data': (1024, 1024)} |\n| Flatten | 0.03 | --- | 20.0 | {'data': (10000, 1)} |\n| Flatten | 0.0437 | --- | 2000.0 | {'data': (10000, 100)} |\n| stop_gradient | --- | --- | 2097.1521 | {'data': (1024, 1024)} |\n| stop_gradient | --- | --- | 20.0 | {'data': (10000, 1)} |\n| stop_gradient | --- | --- | 2000.0 | {'data': (10000, 100)} |\n| broadcast_minimum | 0.0284 | 0.0294 | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| random_generalized_negative_binomial | 63.299 | --- | 2097.1521 | {'shape': (1024, 1024)} |\n| random_generalized_negative_binomial | 0.6491 | --- | 20.0 | {'shape': (10000, 1)} |\n| random_generalized_negative_binomial | 60.3705 | --- | 2000.0 | {'shape': (10000, 100)} |\n| broadcast_sub | 0.0198 | 0.0208 | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| round | 0.0363 | --- | 2097.1521 | {'data': (1024, 1024)} |\n| round | 0.0248 | --- | 20.0 | {'data': (10000, 1)} |\n| round | 0.0358 | --- | 2000.0 | {'data': (10000, 100)} |\n| cosh | 0.0384 | 0.0389 | 2097.1521 | {'data': (1024, 1024)} |\n| cosh | 0.0271 | 0.0255 | 20.0 | {'data': (10000, 1)} |\n| cosh | 0.0378 | 0.0387 | 2000.0 | {'data': (10000, 100)} |\n| max | 0.068 | 0.095 | 0.002 | {'axis': (), 'data': (1024, 1024)} |\n| max | 0.0336 | 0.0219 | 0.002 | {'axis': 0, 'data': (10000, 1)} |\n| max | 0.0665 | 0.0918 | 0.002 | {'axis': (0, 1), 'data': (10000, 100)} |\n| tanh | 0.0391 | 0.038 | 2097.1521 | {'data': (1024, 1024)} |\n| tanh | 0.0271 | 0.025 | 20.0 | {'data': (10000, 1)} |\n| tanh | 0.0373 | 0.0376 | 2000.0 | {'data': (10000, 100)} |\n| relu | 0.0382 | 0.0383 | 2097.1521 | {'data': (1024, 1024)} |\n| relu | 0.0272 | 0.0251 | 20.0 | {'data': (10000, 1)} |\n| relu | 0.0383 | 0.038 | 2000.0 | {'data': (10000, 100)} |\n| negative | 0.0367 | --- | 2097.1521 | {'data': (1024, 1024)} |\n| negative | 0.025 | --- | 20.0 | {'data': (10000, 1)} |\n| negative | 0.0361 | --- | 2000.0 | {'data': (10000, 100)} |\n| random_randint | 9.0025 | --- | 2097.1521 | {'shape': (1024, 1024), 'high': 5, 'low': 0} |\n| random_randint | 0.0976 | --- | 20.0 | {'shape': (10000, 1), 'high': 5, 'low': 0} |\n| random_randint | 8.589 | --- | 2000.0 | {'shape': (10000, 100), 'high': 5, 'low': 0} |\n| trunc | 0.0367 | --- | 2097.1521 | {'data': (1024, 1024)} |\n| trunc | 0.0251 | --- | 20.0 | {'data': (10000, 1)} |\n| trunc | 0.0359 | --- | 2000.0 | {'data': (10000, 100)} |\n| log_softmax | 0.0433 | 0.0387 | 2097.1521 | {'axis': -1, 'data': (1024, 1024), 'temperature': 0.5} |\n| log_softmax | 0.055 | 0.0386 | 20.0 | {'axis': -1, 'data': (10000, 1), 'temperature': 0.5} |\n| log_softmax | 0.08 | 0.0553 | 2000.0 | {'axis': -1, 'data': (10000, 100), 'temperature': 0.5} |\n| broadcast_greater_equal | 0.0191 | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| arctan | 0.0386 | 0.04 | 2097.1521 | {'data': (1024, 1024)} |\n| arctan | 0.0266 | 0.0248 | 20.0 | {'data': (10000, 1)} |\n| arctan | 0.0378 | 0.0394 | 2000.0 | {'data': (10000, 100)} |\n| sqrt | 0.0385 | 0.041 | 2097.1521 | {'data': (1024, 1024)} |\n| sqrt | 0.0269 | 0.025 | 20.0 | {'data': (10000, 1)} |\n| sqrt | 0.0378 | 0.0397 | 2000.0 | {'data': (10000, 100)} |\n| dot | 0.215 | 0.4045 | 2097.1521 | {'lhs': (1024, 1024), 'rhs': (1024, 1024)} |\n| dot | 0.031 | 0.0633 | 2000.0 | {'lhs': (1000, 10), 'transpose_b': True, 'rhs': (1000, 10)} |\n| dot | 0.042 | 0.0388 | 0.2 | {'lhs': (1000, 1), 'transpose_b': True, 'transpose_a': True, 'rhs': (100, 1000)} |\n| floor | 0.0366 | --- | 2097.1521 | {'data': (1024, 1024)} |\n| floor | 0.0249 | --- | 20.0 | {'data': (10000, 1)} |\n| floor | 0.0366 | --- | 2000.0 | {'data': (10000, 100)} |\n| broadcast_logical_xor | 0.0264 | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| shuffle | 0.1016 | --- | 2097.1521 | {'data': (1024, 1024)} |\n| shuffle | 0.2845 | --- | 60.0 | {'data': (10000, 1)} |\n| shuffle | 0.2798 | --- | 2000.0 | {'data': (10000, 100)} |\n| nansum | 0.0741 | 0.0948 | 0.002 | {'axis': (), 'data': (1024, 1024)} |\n| nansum | 0.0368 | 0.0221 | 0.002 | {'axis': 0, 'data': (10000, 1)} |\n| nansum | 0.0731 | 0.0918 | 0.002 | {'axis': (0, 1), 'data': (10000, 100)} |\n| nanprod | 0.0717 | 0.0955 | 0.002 | {'axis': (), 'data': (1024, 1024)} |\n| nanprod | 0.0344 | 0.0223 | 0.002 | {'axis': 0, 'data': (10000, 1)} |\n| nanprod | 0.0711 | 0.0921 | 0.002 | {'axis': (0, 1), 'data': (10000, 100)} |\n| broadcast_div | 0.02 | 0.0227 | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| LeakyReLU | 0.0326 | 0.0333 | 2097.1521 | {'slope': 0.1, 'data': (1024, 1024), 'act_type': 'leaky'} |\n| LeakyReLU | 0.0347 | 0.0199 | 20.0 | {'slope': 0.1, 'data': (10000, 1), 'act_type': 'leaky'} |\n| LeakyReLU | 0.0316 | 0.0327 | 2000.0 | {'slope': 0.1, 'data': (10000, 100), 'act_type': 'leaky'} |\n| LeakyReLU | 0.0328 | 0.0334 | 2097.1521 | {'slope': 0.1, 'data': (1024, 1024), 'act_type': 'elu'} |\n| LeakyReLU | 0.0208 | 0.0194 | 20.0 | {'slope': 0.1, 'data': (10000, 1), 'act_type': 'elu'} |\n| LeakyReLU | 0.0315 | 0.0326 | 2000.0 | {'slope': 0.1, 'data': (10000, 100), 'act_type': 'elu'} |\n| LeakyReLU | 0.0323 | 0.033 | 2097.1521 | {'data': (1024, 1024), 'act_type': 'selu'} |\n| LeakyReLU | 0.0209 | 0.0194 | 20.0 | {'data': (10000, 1), 'act_type': 'selu'} |\n| LeakyReLU | 0.0315 | 0.0325 | 2000.0 | {'data': (10000, 100), 'act_type': 'selu'} |\n| LeakyReLU | 0.0349 | 0.0917 | 2097.1521 | {'gamma': (1, 1024), 'data': (1024, 1024), 'act_type': 'prelu'} |\n| LeakyReLU | 0.0244 | 0.0465 | 20.0 | {'gamma': (1, 1), 'data': (10000, 1), 'act_type': 'prelu'} |\n| LeakyReLU | 0.034 | 0.1003 | 2000.0 | {'gamma': (1, 100), 'data': (10000, 100), 'act_type': 'prelu'} |\n| FullyConnected | 0.3141 | 0.3486 | 4.096 | {'weight': (64, 196608), 'num_hidden': 64, 'data': (32, 3, 256, 256), 'bias': (64,), 'flatten': True} |\n| FullyConnected | 0.1381 | 0.5142 | 3145.728 | {'weight': (64, 256), 'num_hidden': 64, 'data': (32, 3, 256, 256), 'bias': (64,), 'flatten': False} |\n| square | 0.038 | 0.038 | 2097.1521 | {'data': (1024, 1024)} |\n| square | 0.0281 | 0.0262 | 20.0 | {'data': (10000, 1)} |\n| square | 0.0424 | 0.0422 | 2000.0 | {'data': (10000, 100)} |\n| elemwise_mul | 0.0186 | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| broadcast_not_equal | 0.0262 | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| broadcast_logical_or | 0.019 | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| logical_not | 0.0366 | --- | 2097.1521 | {'data': (1024, 1024)} |\n| logical_not | 0.0247 | --- | 20.0 | {'data': (10000, 1)} |\n| logical_not | 0.0357 | --- | 2000.0 | {'data': (10000, 100)} |\n| broadcast_add | 0.0284 | 0.0289 | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| broadcast_hypot | 0.0202 | 0.0233 | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| sign | 0.038 | 0.0333 | 2097.1521 | {'data': (1024, 1024)} |\n| sign | 0.0269 | 0.0252 | 20.0 | {'data': (10000, 1)} |\n| sign | 0.0379 | 0.0326 | 2000.0 | {'data': (10000, 100)} |\n| arccos | 0.0385 | 0.0424 | 2097.1521 | {'data': (1024, 1024)} |\n| arccos | 0.0268 | 0.0253 | 20.0 | {'data': (10000, 1)} |\n| arccos | 0.0374 | 0.0413 | 2000.0 | {'data': (10000, 100)} |\n| erf | 0.0383 | 0.0384 | 2097.1521 | {'data': (1024, 1024)} |\n| erf | 0.0266 | 0.0247 | 20.0 | {'data': (10000, 1)} |\n| erf | 0.0374 | 0.0384 | 2000.0 | {'data': (10000, 100)} |\n| degrees | 0.0375 | 0.0326 | 2097.1521 | {'data': (1024, 1024)} |\n| degrees | 0.0265 | 0.0247 | 20.0 | {'data': (10000, 1)} |\n| degrees | 0.0376 | 0.0317 | 2000.0 | {'data': (10000, 100)} |\n| fix | 0.0366 | --- | 2097.1521 | {'data': (1024, 1024)} |\n| fix | 0.0251 | --- | 20.0 | {'data': (10000, 1)} |\n| fix | 0.0362 | --- | 2000.0 | {'data': (10000, 100)} |\n| cos | 0.0386 | 0.0389 | 2097.1521 | {'data': (1024, 1024)} |\n| cos | 0.0276 | 0.0255 | 20.0 | {'data': (10000, 1)} |\n| cos | 0.0385 | 0.0388 | 2000.0 | {'data': (10000, 100)} |\n| broadcast_plus | --- | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| sample_exponential | 0.1359 | --- | 4194.3042 | {'lam': [1.0, 8.5], 'shape': (1024, 1024)} |\n| sample_exponential | 0.0734 | --- | 40.0 | {'lam': [1.0, 8.5], 'shape': (10000, 1)} |\n| sample_exponential | 0.1323 | --- | 4000.0 | {'lam': [1.0, 8.5], 'shape': (10000, 100)} |\n| make_loss | 0.0415 | --- | 2097.1521 | {'data': (1024, 1024)} |\n| make_loss | 0.0301 | --- | 20.0 | {'data': (10000, 1)} |\n| make_loss | 0.0432 | --- | 2000.0 | {'data': (10000, 100)} |\n| argmax_channel | 0.288 | --- | 2.048 | {'data': (1024, 1024)} |\n| argmax_channel | 0.0264 | --- | 20.0 | {'data': (10000, 1)} |\n| argmax_channel | 0.0439 | --- | 20.0 | {'data': (10000, 100)} |\n| BlockGrad | 0.042 | --- | 2097.1521 | {'data': (1024, 1024)} |\n| BlockGrad | 0.0295 | --- | 20.0 | {'data': (10000, 1)} |\n| BlockGrad | 0.0438 | --- | 2000.0 | {'data': (10000, 100)} |\n| BatchNorm | 1.1379 | 2.3158 | 12582.9238 | {'gamma': (3,), 'moving_var': (3,), 'data': (32, 3, 256, 256), 'beta': (3,), 'moving_mean': (3,)} |\n| BatchNorm | 1.7465 | 3.5398 | 19200.0117 | {'gamma': (3,), 'moving_var': (3,), 'data': (32, 3, 10000, 10), 'beta': (3,), 'moving_mean': (3,)} |\n| sample_negative_binomial | 1.5065 | --- | 4194.3042 | {'shape': (1024, 1024), 'p': [0.4, 0.77], 'k': [20, 49]} |\n| sample_negative_binomial | 0.8542 | --- | 40.0 | {'shape': (10000, 1), 'p': [0.4, 0.77], 'k': [20, 49]} |\n| sample_negative_binomial | 1.476 | --- | 4000.0 | {'shape': (10000, 100), 'p': [0.4, 0.77], 'k': [20, 49]} |\n| batch_dot | 4.7282 | 9.3181 | 67108.8672 | {'lhs': (32, 1024, 1024), 'rhs': (32, 1024, 1024)} |\n| batch_dot | 0.3021 | 1.229 | 64000.0 | {'lhs': (32, 1000, 10), 'transpose_b': True, 'rhs': (32, 1000, 10)} |\n| batch_dot | 0.0523 | 0.0636 | 6.4 | {'lhs': (32, 1000, 1), 'transpose_b': True, 'transpose_a': True, 'rhs': (32, 100, 1000)} |\n| rsqrt | 0.0386 | 0.0422 | 2097.1521 | {'data': (1024, 1024)} |\n| rsqrt | 0.0268 | 0.0252 | 20.0 | {'data': (10000, 1)} |\n| rsqrt | 0.0377 | 0.0408 | 2000.0 | {'data': (10000, 100)} |\n| broadcast_logical_and | 0.0191 | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| mean | 0.0818 | 0.043 | 0.002 | {'axis': (), 'data': (1024, 1024)} |\n| mean | 0.0384 | 0.026 | 0.002 | {'axis': 0, 'data': (10000, 1)} |\n| mean | 0.0801 | 0.0421 | 0.002 | {'axis': (0, 1), 'data': (10000, 100)} |\n| ceil | 0.0363 | --- | 2097.1521 | {'data': (1024, 1024)} |\n| ceil | 0.0249 | --- | 20.0 | {'data': (10000, 1)} |\n| ceil | 0.0357 | --- | 2000.0 | {'data': (10000, 100)} |\n| min_axis | 0.0569 | --- | 0.002 | {'axis': (), 'data': (1024, 1024)} |\n| min_axis | 0.0319 | --- | 0.002 | {'axis': 0, 'data': (10000, 1)} |\n| min_axis | 0.0563 | --- | 0.002 | {'axis': (0, 1), 'data': (10000, 100)} |\n| sigmoid | 0.0395 | 0.0392 | 2097.1521 | {'data': (1024, 1024)} |\n| sigmoid | 0.0412 | 0.0256 | 20.0 | {'data': (10000, 1)} |\n| sigmoid | 0.0378 | 0.0384 | 2000.0 | {'data': (10000, 100)} |\n| broadcast_power | 0.0209 | 0.023 | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| gamma | 0.0415 | 0.0761 | 2097.1521 | {'data': (1024, 1024)} |\n| gamma | 0.0279 | 0.0279 | 20.0 | {'data': (10000, 1)} |\n| gamma | 0.041 | 0.0736 | 2000.0 | {'data': (10000, 100)} |\n| radians | 0.0378 | 0.0327 | 2097.1521 | {'data': (1024, 1024)} |\n| radians | 0.0268 | 0.0248 | 20.0 | {'data': (10000, 1)} |\n| radians | 0.0373 | 0.032 | 2000.0 | {'data': (10000, 100)} |\n| prod | 0.0671 | 0.0949 | 0.002 | {'axis': (), 'data': (1024, 1024)} |\n| prod | 0.0334 | 0.0218 | 0.002 | {'axis': 0, 'data': (10000, 1)} |\n| prod | 0.0663 | 0.091 | 0.002 | {'axis': (0, 1), 'data': (10000, 100)} |\n| abs | 0.0382 | 0.039 | 2097.1521 | {'data': (1024, 1024)} |\n| abs | 0.0271 | 0.0254 | 20.0 | {'data': (10000, 1)} |\n| abs | 0.0369 | 0.0381 | 2000.0 | {'data': (10000, 100)} |\n| reciprocal | 0.0378 | 0.0399 | 2097.1521 | {'data': (1024, 1024)} |\n| reciprocal | 0.0267 | 0.0248 | 20.0 | {'data': (10000, 1)} |\n| reciprocal | 0.0374 | 0.0391 | 2000.0 | {'data': (10000, 100)} |\n| sample_generalized_negative_binomial | 1.0903 | --- | 4194.3042 | {'mu': [2.0, 2.5], 'alpha': [0.0, 2.5], 'shape': (1024, 1024)} |\n| sample_generalized_negative_binomial | 0.6051 | --- | 40.0 | {'mu': [2.0, 2.5], 'alpha': [0.0, 2.5], 'shape': (10000, 1)} |\n| sample_generalized_negative_binomial | 1.0655 | --- | 4000.0 | {'mu': [2.0, 2.5], 'alpha': [0.0, 2.5], 'shape': (10000, 100)} |\n| rint | 0.0369 | --- | 2097.1521 | {'data': (1024, 1024)} |\n| rint | 0.025 | --- | 20.0 | {'data': (10000, 1)} |\n| rint | 0.0357 | --- | 2000.0 | {'data': (10000, 100)} |\n| arcsin | 0.0383 | 0.0412 | 2097.1521 | {'data': (1024, 1024)} |\n| arcsin | 0.0269 | 0.0253 | 20.0 | {'data': (10000, 1)} |\n| arcsin | 0.038 | 0.0402 | 2000.0 | {'data': (10000, 100)} |\n| sample_poisson | 0.4035 | --- | 4194.3042 | {'lam': [1.0, 8.5], 'shape': (1024, 1024)} |\n| sample_poisson | 0.3288 | --- | 40.0 | {'lam': [1.0, 8.5], 'shape': (10000, 1)} |\n| sample_poisson | 0.4029 | --- | 4000.0 | {'lam': [1.0, 8.5], 'shape': (10000, 100)} |\n| Pooling | 0.0225 | 0.0262 | 49.152 | {'stride': 1, 'pool_type': 'avg', 'data': (32, 3, 256), 'layout': 'NCW', 'global_pool': 0, 'kernel': 3, 'pad': 1} |\n| sample_uniform | 0.1386 | --- | 4194.3042 | {'shape': (1024, 1024), 'high': [1.0, 3.7], 'low': [0.0, 2.5]} |\n| sample_uniform | 0.0776 | --- | 40.0 | {'shape': (10000, 1), 'high': [1.0, 3.7], 'low': [0.0, 2.5]} |\n| sample_uniform | 0.1358 | --- | 4000.0 | {'shape': (10000, 100), 'high': [1.0, 3.7], 'low': [0.0, 2.5]} |\n| ones_like | 0.0326 | --- | 2097.1521 | {'data': (1024, 1024)} |\n| ones_like | 0.0251 | --- | 40.0 | {'data': (10000, 1)} |\n| ones_like | 0.032 | --- | 2000.0 | {'data': (10000, 100)} |\n| identity | --- | --- | 2097.1521 | {'data': (1024, 1024)} |\n| identity | --- | --- | 40.0 | {'data': (10000, 1)} |\n| identity | --- | --- | 2000.0 | {'data': (10000, 100)} |\n| broadcast_lesser_equal | 0.0189 | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| broadcast_lesser | 0.0189 | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| zeros_like | 0.0326 | --- | 2097.1521 | {'data': (1024, 1024)} |\n| zeros_like | 0.0252 | --- | 40.0 | {'data': (10000, 1)} |\n| zeros_like | 0.0321 | --- | 2000.0 | {'data': (10000, 100)} |\n| random_uniform | 3.1027 | --- | 2097.1521 | {'shape': (1024, 1024), 'high': 5, 'low': 0} |\n| random_uniform | 0.0588 | --- | 20.0 | {'shape': (10000, 1), 'high': 5, 'low': 0} |\n| random_uniform | 2.9599 | --- | 2000.0 | {'shape': (10000, 100), 'high': 5, 'low': 0} |\n| broadcast_mod | 0.0196 | 0.0227 | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| cbrt | 0.0383 | 0.0405 | 2097.1521 | {'data': (1024, 1024)} |\n| cbrt | 0.0272 | 0.0251 | 20.0 | {'data': (10000, 1)} |\n| cbrt | 0.0377 | 0.0394 | 2000.0 | {'data': (10000, 100)} |\n| broadcast_minus | --- | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| random_negative_binomial | 52.4089 | --- | 2097.1521 | {'shape': (1024, 1024), 'p': 1, 'k': 1} |\n| random_negative_binomial | 0.5224 | --- | 20.0 | {'shape': (10000, 1), 'p': 1, 'k': 1} |\n| random_negative_binomial | 49.9987 | --- | 2000.0 | {'shape': (10000, 100), 'p': 1, 'k': 1} |\n| log2 | 0.0382 | 0.0405 | 2097.1521 | {'data': (1024, 1024)} |\n| log2 | 0.0269 | 0.0253 | 20.0 | {'data': (10000, 1)} |\n| log2 | 0.0376 | 0.0403 | 2000.0 | {'data': (10000, 100)} |\n| broadcast_greater | 0.0191 | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| max_axis | 0.0564 | --- | 0.002 | {'axis': (), 'data': (1024, 1024)} |\n| max_axis | 0.0315 | --- | 0.002 | {'axis': 0, 'data': (10000, 1)} |\n| max_axis | 0.0561 | --- | 0.002 | {'axis': (0, 1), 'data': (10000, 100)} |\n| sample_gamma | 0.3435 | --- | 4194.3042 | {'alpha': [0.0, 2.5], 'beta': [1.0, 0.7], 'shape': (1024, 1024)} |\n| sample_gamma | 0.2938 | --- | 40.0 | {'alpha': [0.0, 2.5], 'beta': [1.0, 0.7], 'shape': (10000, 1)} |\n| sample_gamma | 0.341 | --- | 4000.0 | {'alpha': [0.0, 2.5], 'beta': [1.0, 0.7], 'shape': (10000, 100)} |\n| sin | 0.0382 | 0.0386 | 2097.1521 | {'data': (1024, 1024)} |\n| sin | 0.0268 | 0.0244 | 20.0 | {'data': (10000, 1)} |\n| sin | 0.0374 | 0.0386 | 2000.0 | {'data': (10000, 100)} |\n| sum | 0.0797 | 0.0341 | 0.002 | {'axis': (), 'data': (1024, 1024)} |\n| sum | 0.0362 | 0.021 | 0.002 | {'axis': 0, 'data': (10000, 1)} |\n| sum | 0.0771 | 0.0335 | 0.002 | {'axis': (0, 1), 'data': (10000, 100)} |\n| erfinv | 0.1069 | 0.1112 | 2097.1521 | {'data': (1024, 1024)} |\n| erfinv | 0.029 | 0.0271 | 20.0 | {'data': (10000, 1)} |\n| erfinv | 0.1043 | 0.1055 | 2000.0 | {'data': (10000, 100)} |\n| random_gamma | 46.4529 | --- | 2097.1521 | {'shape': (1024, 1024)} |\n| random_gamma | 0.466 | --- | 20.0 | {'shape': (10000, 1)} |\n| random_gamma | 44.4208 | --- | 2000.0 | {'shape': (10000, 100)} |\n| broadcast_mul | 0.019 | 0.0222 | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| arccosh | 0.0405 | 0.0414 | 2097.1521 | {'data': (1024, 1024)} |\n| arccosh | 0.0269 | 0.0253 | 20.0 | {'data': (10000, 1)} |\n| arccosh | 0.0396 | 0.0402 | 2000.0 | {'data': (10000, 100)} |\n| log1p | 0.0383 | 0.0399 | 2097.1521 | {'data': (1024, 1024)} |\n| log1p | 0.0269 | 0.0248 | 20.0 | {'data': (10000, 1)} |\n| log1p | 0.0376 | 0.0394 | 2000.0 | {'data': (10000, 100)} |\n| size_array | 0.0229 | --- | 0.004 | {'data': (1024, 1024)} |\n| size_array | 0.0367 | --- | 0.004 | {'data': (10000, 1)} |\n| size_array | 0.022 | --- | 0.004 | {'data': (10000, 100)} |\n| arcsinh | 0.0396 | 0.0406 | 2097.1521 | {'data': (1024, 1024)} |\n| arcsinh | 0.0269 | 0.0249 | 20.0 | {'data': (10000, 1)} |\n| arcsinh | 0.0388 | 0.0396 | 2000.0 | {'data': (10000, 100)} |\n| elemwise_div | 0.0188 | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| elemwise_add | 0.019 | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| elemwise_sub | 0.0186 | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| arctanh | 0.0385 | 0.0397 | 2097.1521 | {'data': (1024, 1024)} |\n| arctanh | 0.0266 | 0.0245 | 20.0 | {'data': (10000, 1)} |\n| arctanh | 0.0376 | 0.0391 | 2000.0 | {'data': (10000, 100)} |\n| log | 0.0381 | 0.0399 | 2097.1521 | {'data': (1024, 1024)} |\n| log | 0.0268 | 0.025 | 20.0 | {'data': (10000, 1)} |\n| log | 0.0377 | 0.0394 | 2000.0 | {'data': (10000, 100)} |\n| gammaln | 0.0461 | 0.0664 | 2097.1521 | {'data': (1024, 1024)} |\n| gammaln | 0.0273 | 0.0266 | 20.0 | {'data': (10000, 1)} |\n| gammaln | 0.0455 | 0.0645 | 2000.0 | {'data': (10000, 100)} |\n| Dropout | 0.122 | 0.0957 | 25165.8242 | {'mode': 'always', 'data': (32, 3, 256, 256), 'p': 0.5} |\n| Dropout | 0.0739 | 0.0241 | 400.0 | {'mode': 'always', 'data': (10000, 10), 'p': 0.5} |\n| softmax | 0.0454 | 0.0438 | 2097.1521 | {'axis': -1, 'data': (1024, 1024), 'temperature': 0.5} |\n| softmax | 0.056 | 0.0377 | 20.0 | {'axis': -1, 'data': (10000, 1), 'temperature': 0.5} |\n| softmax | 0.0805 | 0.0609 | 2000.0 | {'axis': -1, 'data': (10000, 100), 'temperature': 0.5} |\n| expm1 | 0.0386 | 0.0387 | 2097.1521 | {'data': (1024, 1024)} |\n| expm1 | 0.0275 | 0.025 | 20.0 | {'data': (10000, 1)} |\n| expm1 | 0.0375 | 0.0381 | 2000.0 | {'data': (10000, 100)} |\n| log10 | 0.0382 | 0.0403 | 2097.1521 | {'data': (1024, 1024)} |\n| log10 | 0.0274 | 0.0251 | 20.0 | {'data': (10000, 1)} |\n| log10 | 0.0369 | 0.0386 | 2000.0 | {'data': (10000, 100)} |\n| random_poisson | 15.4201 | --- | 2097.1521 | {'shape': (1024, 1024)} |\n| random_poisson | 0.176 | --- | 20.0 | {'shape': (10000, 1)} |\n| random_poisson | 14.9428 | --- | 2000.0 | {'shape': (10000, 100)} |\n| sinh | 0.0381 | 0.0381 | 2097.1521 | {'data': (1024, 1024)} |\n| sinh | 0.0269 | 0.0246 | 20.0 | {'data': (10000, 1)} |\n| sinh | 0.0376 | 0.038 | 2000.0 | {'data': (10000, 100)} |\n| random_normal | 16.4453 | --- | 2097.1521 | {'shape': (1024, 1024)} |\n| random_normal | 0.1864 | --- | 20.0 | {'shape': (10000, 1)} |\n| random_normal | 15.553 | --- | 2000.0 | {'shape': (10000, 100)} |\n| hard_sigmoid | 0.031 | 0.0336 | 2097.1521 | {'alpha': 0.25, 'data': (1024, 1024), 'beta': 0.5} |\n| hard_sigmoid | 0.0197 | 0.0182 | 20.0 | {'alpha': 0.25, 'data': (10000, 1), 'beta': 0.5} |\n| hard_sigmoid | 0.0303 | 0.0322 | 2000.0 | {'alpha': 0.25, 'data': (10000, 100), 'beta': 0.5} |\n| flatten | --- | --- | 2097.1521 | {'data': (1024, 1024)} |\n| flatten | --- | --- | 20.0 | {'data': (10000, 1)} |\n| flatten | --- | --- | 2000.0 | {'data': (10000, 100)} |\n| random_exponential | 15.9188 | --- | 2097.1521 | {'shape': (1024, 1024)} |\n| random_exponential | 0.1663 | --- | 40.0 | {'shape': (10000, 1)} |\n| random_exponential | 15.1982 | --- | 2000.0 | {'shape': (10000, 100)} |\n| tan | 0.0395 | 0.0377 | 2097.1521 | {'data': (1024, 1024)} |\n| tan | 0.027 | 0.025 | 20.0 | {'data': (10000, 1)} |\n| tan | 0.0384 | 0.037 | 2000.0 | {'data': (10000, 100)} |\n| broadcast_equal | 0.019 | --- | 0.012 | {'lhs': [(1024, 1024), (10000, 10), (10000, 1)], 'rhs': [(1024, 1024), (10000, 10), (10000, 1)]} |\n| softsign | 0.0384 | 0.0402 | 2097.1521 | {'data': (1024, 1024)} |\n| softsign | 0.0269 | 0.025 | 20.0 | {'data': (10000, 1)} |\n| softsign | 0.0375 | 0.04 | 2000.0 | {'data': (10000, 100)} |\n| Convolution | 0.0578 | 0.1121 | 1040.384 | {'stride': (1,), 'data': (32, 3, 256), 'bias': (64,), 'layout': 'NCW', 'dilate': (1,), 'kernel': (3,), 'weight': (64, 3, 3), 'pad': (0,), 'num_filter': 64} |\n| sample_normal | 0.1549 | --- | 4194.3042 | {'mu': [2.0, 2.5], 'sigma': [1.0, 3.7], 'shape': (1024, 1024)} |\n| sample_normal | 0.1027 | --- | 40.0 | {'mu': [2.0, 2.5], 'sigma': [1.0, 3.7], 'shape': (10000, 1)} |\n| sample_normal | 0.1522 | --- | 4000.0 | {'mu': [2.0, 2.5], 'sigma': [1.0, 3.7], 'shape': (10000, 100)} |\n| exp | 0.0365 | --- | 2097.1521 | {'data': (1024, 1024)} |\n| exp | 0.0246 | --- | 20.0 | {'data': (10000, 1)} |\n| exp | 0.0356 | --- | 2000.0 | {'data': (10000, 100)} |\n| sum_axis | 0.0637 | --- | 0.004 | {'axis': (), 'data': (1024, 1024)} |\n| sum_axis | 0.0351 | --- | 0.002 | {'axis': 0, 'data': (10000, 1)} |\n| sum_axis | 0.0631 | --- | 0.002 | {'axis': (0, 1), 'data': (10000, 100)} |"
  },
  {
    "path": "benchmark/opperf/rules/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n"
  },
  {
    "path": "benchmark/opperf/rules/default_params.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\nimport sys\n\n# We will use all operators inside NDArray Module\n# If you want to run benchmark for all operators in different namespace,\n# for example mxnet.numpy.op, update here. All operators for benchmarks\n# will be picked up from this module\nMX_OP_MODULE = sys.modules[\"mxnet.ndarray.op\"]\n\n\n\"\"\"Default Input Tensor shapes to use for benchmarking\"\"\"\n\n# For operators like concat, ElementWiseSum, squeeze, stack\n# argument data is passed as variable arg (*args)\nDEFAULT_ARGS = [(1024, 1024)]\n\n# For Unary operators like abs, arccos, arcsin etc..\nDEFAULT_DATA = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_DTYPE = ['float32', 'int32', 'float32']  # required parameter for amp_cast, cast\nDEFAULT_DTYPE_INT = ['int32', 'int64', 'int32']  # randint works for int* types only\nDEFAULT_DTYPE_FLOAT = ['float16', 'float32', 'float64']  # random_exp works for float* types only\n\nDEFAULT_DATA_LARGE_TENSOR = [(2**16, 2**16)]\n\n# For Binary miscellaneous operators like choose_element0_index\n# argument data must be indexed via an NDArray.\n# NOTE: Data used is DEFAULT_DATA\nDEFAULT_INDEX = [(1, 1024), (1, 1), (1, 100)]\n\nDEFAULT_INDEX_LARGE_TENSOR = [(1, 2**16)]\n\n# For Binary broadcast operators like - broadcast_add/sub/mod/logical_and etc..\nDEFAULT_LHS = [(1024, 1024), (10000, 10), (10000, 1)]\nDEFAULT_RHS = [(1024, 1024), (10000, 10), (10000, 1)]\n\nDEFAULT_LHS_LARGE_TENSOR = [(2**16, 2**16), (2**28, 2**4), (2**32, 1)]\nDEFAULT_RHS_LARGE_TENSOR = [(2**16, 2**16), (2**28, 2**4), (2**32, 1)]\n\n# For operators like - random_uniform, random_normal etc..\nDEFAULT_SHAPE = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_SAMPLE = [(2,)]\nDEFAULT_LOW = [0]\nDEFAULT_HIGH = [5]\nDEFAULT_K = [1]\nDEFAULT_P = [1]\n\nDEFAULT_SHAPE_LARGE_TENSOR = [(2**16, 2**16)]#, (2**32, 1), (2**25, 2**7)]\nDEFAULT_SAMPLE_LARGE_TENSOR = [(2**32,)]\nDEFAULT_DATA_RPD_LARGE_TENSOR = [(2**32 + 1, 5)]\nDEFAULT_ALPHA_RPD_LARGE_TENSOR = [(2**32,)]\nDEFAULT_SAMPLE_RPE_LARGE_TENSOR = [(1, 2**32)]\nDEFAULT_LAM_RPE_LARGE_TENSOR = [(1,)]\nDEFAULT_SAMPLE_RPG_LARGE_TENSOR = [(1, 2**32 + 1)]\nDEFAULT_ALPHA_RPG_LARGE_TENSOR = [(1,)]\n\n# For operators like - sample_uniform, sample_normal etc..\n# NOTE: There are many overlapping operators in random_* and sample_*,\n# Ex: random_uniform, sample_uniform. Parameter names are same, but, for\n# random_* operators they are float/int and for sample_* operators they are NDArray.\n# Hence, below we append ND to mark the difference.\nDEFAULT_LOW_ND = [[0.0, 2.5]]\nDEFAULT_HIGH_ND = [[1.0, 3.7]]\nDEFAULT_MU_ND = [[2.0, 2.5]]\nDEFAULT_SIGMA = [[1.0, 3.7]]\nDEFAULT_ALPHA_ND = [[0.0, 2.5]]\nDEFAULT_BETA_ND = [[1.0, 0.7]]\nDEFAULT_LAM = [[1.0, 8.5]]\nDEFAULT_K_ND = [[20, 49]]\nDEFAULT_P_ND = [[0.4, 0.77]]\nDEFAULT_GRID = [(32, 2, 256, 256)]\nDEFAULT_DATA_BILINEAR = [(32, 2, 256, 256)]\nDEFAULT_TRANSFORM_TYPE = ['warp', 'affine']\nDEFAULT_DATA_GRIDGEN = [(32, 2, 256, 256), (256, 6)]\nDEFAULT_TARGET_SHAPE = [(256, 6)]\nDEFAULT_DATA_SM = [(32, 32), (64, 64)]\n\nDEFAULT_LOW_ND_LARGE_TENSOR = [[0.0] * 2**16 + [2.5] * 2**16]\nDEFAULT_HIGH_ND_LARGE_TENSOR = [[1.0] * 2**16 + [3.7] * 2**16]\nDEFAULT_MU_ND_LARGE_TENSOR = [[2.0] * 2**16 + [2.5] * 2**16]\nDEFAULT_SIGMA_LARGE_TENSOR = [[1.0] * 2**16 + [3.7] * 2**16]\nDEFAULT_ALPHA_ND_LARGE_TENSOR = [[0.0] * 2**16 + [2.5] * 2**16]\nDEFAULT_BETA_ND_LARGE_TENSOR = [[1.0] * 2**16 + [0.7] * 2**16]\nDEFAULT_LAM_ND_LARGE_TENSOR = [[1.0] * 2**16 + [8.5] * 2**16]\nDEFAULT_K_ND_LARGE_TENSOR = [[20] * 2**16 + [49] * 2**16]\nDEFAULT_P_ND_LARGE_TENSOR = [[0.4] * 2**16 + [0.77] * 2**16]\nDEFAULT_DATA_BILINEAR_LARGE_TENSOR = [(2**32, 1, 1, 1)]\nDEFAULT_GRID_LARGE_TENSOR = [(2**32, 2, 1, 1)]\nDEFAULT_DATA_GRIDGEN_LARGE_TENSOR = [(2**31, 2, 1, 1), (1, 6)]\nDEFAULT_TARGET_SHAPE_LARGE_TENSOR = [(1, 6)]\nDEFAULT_DATA_SM_LARGE_TENSOR = [(2**32,)]\nDEFAULT_SHAPE_SE_LARGE_TENSOR = [(1,)]\nDEFAULT_LAM_SE_LARGE_TENSOR = [(2**32 + 1,)]\nDEFAULT_SHAPE_SU_LARGE_TENSOR = [(2**32,)]\n\n# For sorting and searching operators\n# NOTE: Data used is DEFAULT_DATA\nDEFAULT_AXIS = [0]\n\n# For NN basic operators\n# General\nDEFAULT_DATA_NN_BASIC = [(32, 3, 256, 256), (32, 3, 10000, 10)]\nDEFAULT_NUM_HIDDEN = [64]\nDEFAULT_BIAS = [(64,)]\nDEFAULT_FLATTEN = [True, False]\nDEFAULT_GAMMA = [(3,)]\nDEFAULT_BETA = [(3,)]\nDEFAULT_MOVING_MEAN = [(3,)]\nDEFAULT_MOVING_VAR = [(3,)]\nDEFAULT_LABEL_REG = [(32, 3, 256, 256), (32, 3, 10000, 10)]\nDEFAULT_GRAD_SCALE = [.5]\nDEFAULT_NORMALIZATION = [\"batch\"]\nDEFAULT_MARGIN = [.5]\nDEFAULT_REG_COEFF = [.5]\nDEFAULT_INPUT_DIM = [3, 16]\nDEFAULT_OUTPUT_DIM = [4, 9]\nDEFAULT_SPARSE_GRAD = [False]\nDEFAULT_KERNEL_SIZE = [3]\nDEFAULT_MAX_DISPLACEMENT = [2]\nDEFAULT_STRIDE_1 = [2]\nDEFAULT_STRIDE_2 = [2]\nDEFAULT_ALPHA = [.001]\nDEFAULT_NSIZE = [3]\nDEFAULT_PARAMETERS = [(7,), (104,)]\nDEFAULT_STATE = [(1, 4, 1), (2, 10000, 4)]\nDEFAULT_STATE_SIZE = [1, 4]\nDEFAULT_NUM_LAYERS = [1, 2]\nDEFAULT_NUM_GROUPS = [1, 10]\nDEFAULT_TRANSFORM = [\"affine\"]\nDEFAULT_SAMPLER = [\"bilinear\"]\nDEFAULT_DILATE = [(1,), (1, 1)]\nDEFAULT_PAD = [(1,), (1, 1)]\nDEFAULT_OUTPUT_SIZE = [(64, 16, 1), (32, 8, 1)]\nDEFAULT_KERNEL = [(1, 1, 1), (1, 1, 1)]\nDEFAULT_STRIDE = [(2, 2, 2), (1, 1, 1)]\n\nDEFAULT_DATA_NN_BASIC_LARGE_TENSOR = [(2**32 + 1, 1)]\nDEFAULT_NUM_HIDDEN_LARGE_TENSOR = [(1,)]\nDEFAULT_BIAS_LARGE_TENSOR = [(1,)]\nDEFAULT_FLATTEN_LARGE_TENSOR = [False]\nDEFAULT_GAMMA_LARGE_TENSOR = [(1,)]\nDEFAULT_BETA_LARGE_TENSOR = [(1,)]\nDEFAULT_MOVING_MEAN_LARGE_TENSOR = [(2**32 + 1,)]\nDEFAULT_MOVING_VAR_LARGE_TENSOR = [(2**32 + 1,)]\nDEFAULT_INPUT_DIM_LARGE_TENSOR = [2**32]\nDEFAULT_OUTPUT_DIM_LARGE_TENSOR = [1]\nDEFAULT_KERNEL_SIZE_LARGE_TENSOR = [1]\nDEFAULT_MAX_DISPLACEMENT_LARGE_TENSOR = [1]\nDEFAULT_STRIDE_1_LARGE_TENSOR = [1]\nDEFAULT_STRIDE_2_LARGE_TENSOR = [1]\nDEFAULT_DILATE_LARGE_TENSOR = [[]]\nDEFAULT_PAD_LARGE_TENSOR = [[]]\nDEFAULT_OUTPUT_SIZE_LARGE_TENSOR = [(2, 2, 1)]\nDEFAULT_KERNEL_LARGE_TENSOR = [(1, 1, 1)]\nDEFAULT_STRIDE_LARGE_TENSOR = [[]]\nDEFAULT_PARAMETERS_LARGE_TENSOR = [(7,)]\nDEFAULT_STATE_LARGE_TENSOR = [(1, 4, 1)]\nDEFAULT_STATE_SIZE_LARGE_TENSOR = [1]\nDEFAULT_NUM_LAYERS_LARGE_TENSOR = [1]\n\n# BatchNorm\nDEFAULT_AXIS_BN = [1]\n\n# LayerNorm\nDEFAULT_GAMMA_LN = [(32,), (32,)]\nDEFAULT_BETA_LN = [(32,), (32,)]\n\n# L2Normalization\nDEFAULT_MODE_L2 = ['channel', 'instance', 'spatial']\n\nDEFAULT_DATA_SVM_LARGE_TENSOR = [(2**29, 2, 2, 2)]\n\nDEFAULT_DATA_SO_LARGE_TENSOR = [(2**29, 2, 2, 2)]\nDEFAULT_LABEL_SO_LARGE_TENSOR = [(2**29, 2, 2)]\n\n# FullyConnected\nDEFAULT_WEIGHT_FC = [(64, 3 * 256 * 256), (64, 10)]\n\nDEFAULT_DATA_FC_LARGE_TENSOR = [(2**32, 1)]\nDEFAULT_WEIGHT_FC_LARGE_TENSOR = [(1, 1)]\nDEFAULT_NUM_HIDDEN_FC_LARGE_TENSOR = [1]\n\n# Embedding\nDEFAULT_WEIGHT_EMBEDDING = [(3, 4), (16, 9)]\n\nDEFAULT_WEIGHT_EMBEDDING_LARGE_TENSOR = [(2**32, 1)]\n\n# GroupNorm\nDEFAULT_DATA_GN = [(32, 3, 256, 256), (32, 10, 10000, 10)]\nDEFAULT_BETA_GAMMA_GN = [(1,), (10,)]\n\nDEFAULT_DATA_GN_LARGE_TENSOR = [(2**27, 4, 4, 2)]\nDEFAULT_BETA_GAMMA_GN_LARGE_TENSOR = [(1,)]\n\n# Dropout\nDEFAULT_DATA_DROPOUT = [(32, 3, 256, 256), (10000, 10)]\nDEFAULT_MODE_DROPOUT = [\"always\"]\n\nDEFAULT_DATA_DROPOUT_LARGE_TENSOR = [(2**32 + 1,)]\nDEFAULT_P_DROPOUT_LARGE_TENSOR = [.5]\nDEFAULT_AXES_DROPOUT_LARGE_TENSOR = [[]]\n\n# SpatialTransformer\nDEFAULT_DATA_ST = [(32, 3, 256, 6), (256, 3, 10000, 6)]\nDEFAULT_LOC_TAR_ST = [(32, 6), (256, 6)]\n\nDEFAULT_DATA_ST_LARGE_TENSOR = [(2, 2**29, 1, 6)]\nDEFAULT_LOC_TAR_ST_LARGE_TENSOR = [(2, 6)]\n\n# im2col\nDEFAULT_KERNEL_I2C = [(3,), (3, 3)]\nDEFAULT_STRIDE_I2C = [(1,), (1, 1)]\n\nDEFAULT_DATA_I2C_LARGE_TENSOR = [(2**29, 2, 2, 6)]\nDEFAULT_KERNEL_I2C_LARGE_TENSOR = [(1,)]\nDEFAULT_STRIDE_I2C_LARGE_TENSOR = [[]]\n\n# col2im\nDEFAULT_DATA_C2I = [(32, 64, 256), (32, 64, 256)]\n\nDEFAULT_DATA_C2I_LARGE_TENSOR = [(1, 2**30, 4)]\n\n# LRN\nDEFAULT_BETA_LRN = [.2]\n\nDEFAULT_DATA_LRN_LARGE_TENSOR = [(2**27, 4, 4, 2)]\n\n# Correlation\nDEFAULT_DATA1_LARGE_TENSOR = [(2**23, 8, 8, 8)]\nDEFAULT_DATA2_LARGE_TENSOR = [(2**23, 8, 8, 8)]\n\n# For regression operators\nDEFAULT_DATA_REG_LARGE_TENSOR = [(2**29, 2, 2, 2)]\nDEFAULT_LABEL_REG_LARGE_TENSOR = [(2**29, 2, 2, 2)]\n\n# For normalization operators\nDEFAULT_DATA_NORM_LARGE_TENSOR = [(2**29, 2, 2, 2)]\nDEFAULT_GAMMA_NORM_LARGE_TENSOR = [(2,)]\nDEFAULT_BETA_NORM_LARGE_TENSOR = [(2,)]\nDEFAULT_AXIS_LARGE_TENSOR = [-1]\n\n# For optimizer operators\nDEFAULT_WEIGHT = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_GRAD = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_MOM = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_MEAN = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_VAR = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_N = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_D = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_V = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_Z = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_G = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_R1 = [(1, 1024), (1, 1), (1, 100)]\nDEFAULT_R2 = [(1, 1024), (1, 1), (1, 100)]\nDEFAULT_DELTA = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_LRS = [(0.1,0.1)]\nDEFAULT_LR = [0.1, 0.5, 0.9]\nDEFAULT_WD = [0.1, 0.5, 0.9]\nDEFAULT_RHO = [0.1, 0.5, 0.9]\nDEFAULT_MOMENTUM = [0.1, 0.5, 0.9]\nDEFAULT_EPSILON = [1e-05]\nDEFAULT_BETA_1 = [0.1, 0.5, 0.9]\nDEFAULT_BETA_2 = [0.1, 0.5, 0.9]\nDEFAULT_T = [1, 5]\nDEFAULT_RESCALE_GRAD = [0.4, 0.77]\nDEFAULT_CLIP_GRADIENT = [-1.0, 0.8]\nDEFAULT_CLIP_WEIGHTS = [-1.0, 0.8]\nDEFAULT_LAZY_UPDATE = [0, 1]\n\nDEFAULT_WEIGHT_LARGE_TENSOR = [(2**16, 2**16), (2**32, 1), (2**25, 2**7)]\nDEFAULT_GRAD_LARGE_TENSOR = [(2**16, 2**16), (2**32, 1), (2**25, 2**7)]\nDEFAULT_MOM_LARGE_TENSOR = [(2**16, 2**16), (2**32, 1), (2**25, 2**7)]\nDEFAULT_MEAN_LARGE_TENSOR = [(2**16, 2**16), (2**32, 1), (2**25, 2**7)]\nDEFAULT_VAR_LARGE_TENSOR = [(2**16, 2**16), (2**32, 1), (2**25, 2**7)]\nDEFAULT_N_LARGE_TENSOR = [(2**16, 2**16), (2**32, 1), (2**25, 2**7)]\nDEFAULT_D_LARGE_TENSOR = [(2**16, 2**16), (2**32, 1), (2**25, 2**7)]\nDEFAULT_V_LARGE_TENSOR = [(2**16, 2**16), (2**32, 1), (2**25, 2**7)]\nDEFAULT_Z_LARGE_TENSOR = [(2**16, 2**16), (2**32, 1), (2**25, 2**7)]\nDEFAULT_G_LARGE_TENSOR = [(2**16, 2**16), (2**32, 1), (2**25, 2**7)]\nDEFAULT_R1_LARGE_TENSOR = [(1,)]\nDEFAULT_R2_LARGE_TENSOR = [(1,)]\nDEFAULT_DELTA_LARGE_TENSOR = [(2**16, 2**16), (2**32, 1), (2**25, 2**7)]\n\n# For array manipulation operators\n# NOTE: Data needs to be a 4D tensor for  operators like space_to_depth, depth_to_space etc\n# Hence below we append 4d to mark the difference.\n# For depth_to_space, dimension 3 needs to be a multiple of 'block' and 1 should be a multiple of `block^2`\nDEFAULT_DATA_4d = [(1, 4, 2, 4), (10, 25, 10, 100)]\nDEFAULT_BLOCK_SIZE = [2, 5]\nDEFAULT_NUM_OUTPUTS = [1]\nDEFAULT_PAD_WIDTH_4d = [(0, 0, 0, 0, 1, 1, 1, 1)]\nDEFAULT_MODE_4d = [\"constant\"]\nDEFAULT_REPEATS = [2]\n\n# broadcast_axis needs input array with atleast 1 dim of size 1\n# since axis is 0 (default) size(dim0)=1\nDEFAULT_DATA_DIM1 = [(1, 1024), (1, 1), (1, 100)]\nDEFAULT_SIZE = [2]\n\nDEFAULT_DATA_4d_LARGE_TENSOR = [(1, 4, 2, 2**29), (1,2**4,2**4,2**24)]\nDEFAULT_BLOCK_SIZE_LARGE_TENSOR = [2, 4]\n\n# For miscellaneous operators\nDEFAULT_DATA_SQUEEZE = [(1, 1024, 1024), (32, 1, 256, 256)]\nDEFAULT_AXIS_SQUEEZE = [0, 1]\nDEFAULT_A_MIN = [0.1]\nDEFAULT_A_MAX = [0.9]\nDEFAULT_LRS = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_WSS = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_GSS = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_WDS = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_ETA = [.5]\nDEFAULT_STYPE = ['default', 'csr', 'row_sparse']\nDEFAULT_A = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_LHS_FEI = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_MHS = [(1024,), (10000,), (10000,)]\nDEFAULT_RHS_FEI = [(1024,), (10000,), (10000,)]\n\nDEFAULT_DATA_SQUEEZE_LARGE_TENSOR = [(2**32, 1)]\nDEFAULT_AXIS_SQUEEZE_LARGE_TENSOR = [1]\nDEFAULT_WSS_LARGE_TENSOR = [(2**32, 1)]\nDEFAULT_GSS_LARGE_TENSOR = [(2**32, 1)]\nDEFAULT_WDS_LARGE_TENSOR = [(2**32, 1)]\nDEFAULT_LHS_FEI_LARGE_TENSOR = [(2, 2**32 + 1)]\nDEFAULT_RHS_FEI_LARGE_TENSOR = [(2,)]\nDEFAULT_MHS_LARGE_TENSOR = [(2,)]\n\n# For swapaxis operator\nDEFAULT_DIM_1 = [0]\nDEFAULT_DIM_2 = [1]\n\n# For indexing routines\nDEFAULT_INDEX = [(1,1024), (1,1), (1,100)]\nDEFAULT_INDICES = [(1, 1)]\nDEFAULT_BEGIN = [0] # slice_axis expects int, slice can have tuple/int\nDEFAULT_END =[1] # same as above\nDEFAULT_SHAPE_LIKE = [(100, 100), (10, 1), (100, 10)]\nDEFAULT_X = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_Y = [(1024, 1024), (10000, 1), (10000, 100)]\nDEFAULT_COND = [(1024,), (10000,), (10000,)]\nDEFAULT_DEPTH = [0]\n\n# For ravel_multi_index op, ndim(shape) = 2; hence data NDArray's first dim = 2\n# First dimension of input of ravel operator should match shape parameter dimension\n# DEFAULT_SHAPE is reused for ravel_multi_index op\nRAVEL_DATA = [(2, 1024)]\n\nRAVEL_DATA_LARGE_TENSOR = [(2, 2**32)]\nDEFAULT_X_LARGE_TENSOR = [(2**32, 1)]\n\n# For loss operators\nDEFAULT_DATA_3d = [(1024, 100, 100)]\nDEFAULT_LABEL = [(100,100)]\nDEFAULT_DATA_SMCE = [(1024, 1024)]\nDEFAULT_LABEL_SMCE = [(1024,)]\n\nDEFAULT_LABEL_LARGE_TENSOR = [(1, 1)]\nDEFAULT_DATA_CTCLOSS = [(2**32, 1, 1)]\nDEFAULT_DATA_SMCE_LARGE_TENSOR = [(2**32 + 1, 1)]\nDEFAULT_LABEL_SMCE_LARGE_TENSOR = [(2**32 + 1,)]\n\n# For NN operators\nDEFAULT_ACT_TYPE_LR = ['leaky', 'elu', 'selu', 'gelu']\nDEFAULT_ACT_TYPE_ACTIVATION = ['relu', 'sigmoid', 'log_sigmoid', 'mish', 'softrelu', 'softsign', 'tanh']\nDEFAULT_LABEL_SOFTMAX = [(1024, 1024), (10000, 1), (10000, 100)]\n\nDEFAULT_LABEL_SOFTMAX_LARGE_TENSOR = [(2**32, 1)]\n\n# For linalg operators\nDEFAULT_A = [(1024, 1024)]\nDEFAULT_B = [(1024, 1024)]\nDEFAULT_C = [(1024, 1024)]\nDEFAULT_A_MT = [(1024, 1035)]\nDEFAULT_AXES = [[0, 1]]\n\nDEFAULT_A_LARGE_TENSOR = [(2**16, 2**16)]\nDEFAULT_B_LARGE_TENSOR = [(2**16, 2**16)]\nDEFAULT_C_LARGE_TENSOR = [(2**16, 2**16)]\nDEFAULT_A_MT_LARGE_TENSOR = [(2**32 + 1, 1)]\n\n# Default Inputs. MXNet Op Param Name to Default Input mapping\nDEFAULTS_INPUTS = {\"data\": DEFAULT_DATA,\n                   \"dtype\": DEFAULT_DTYPE,\n                   \"dtype_int\": DEFAULT_DTYPE_INT,\n                   \"dtype_float\": DEFAULT_DTYPE_FLOAT,\n                   \"sample\": DEFAULT_SAMPLE,\n                   \"lhs\": DEFAULT_LHS,\n                   \"rhs\": DEFAULT_RHS,\n                   \"shape\": DEFAULT_SHAPE,\n                   \"low\": DEFAULT_LOW,\n                   \"high\": DEFAULT_HIGH,\n                   \"low_nd\": DEFAULT_LOW_ND,\n                   \"high_nd\": DEFAULT_HIGH_ND,\n                   \"mu_nd\": DEFAULT_MU_ND,\n                   \"sigma\": DEFAULT_SIGMA,\n                   \"alpha_nd\": DEFAULT_ALPHA_ND,\n                   \"beta_nd\": DEFAULT_BETA_ND,\n                   \"lam_nd\": DEFAULT_LAM,\n                   \"k\": DEFAULT_K,\n                   \"p\": DEFAULT_P,\n                   \"k_nd\": DEFAULT_K_ND,\n                   \"p_nd\": DEFAULT_P_ND,\n                   \"axis\": DEFAULT_AXIS,\n                   \"weight\" : DEFAULT_WEIGHT,\n                   \"weight32\" : DEFAULT_WEIGHT,\n                   \"grad\" : DEFAULT_GRAD,\n                   \"mean\" : DEFAULT_MEAN,\n                   \"var\" : DEFAULT_VAR,\n                   \"mom\" : DEFAULT_MOM,\n                   \"r1\" : DEFAULT_R1,\n                   \"r2\" : DEFAULT_R2,\n                   \"n\" : DEFAULT_N,\n                   \"d\" : DEFAULT_D,\n                   \"v\" : DEFAULT_V,\n                   \"z\" : DEFAULT_Z,\n                   \"g\" : DEFAULT_G,\n                   \"delta\" : DEFAULT_DELTA,\n                   \"lr\" : DEFAULT_LR,\n                   \"lrs\" : DEFAULT_LRS,\n                   \"wd\" : DEFAULT_WD,\n                   \"rho\" : DEFAULT_RHO,\n                   \"momentum\" : DEFAULT_MOMENTUM,\n                   \"epsilon\" : DEFAULT_EPSILON,\n                   \"beta1\" : DEFAULT_BETA_1,\n                   \"beta2\" : DEFAULT_BETA_2,\n                   \"t\" : DEFAULT_T,\n                   \"rescale_grad\" : DEFAULT_RESCALE_GRAD,\n                   \"clip_grad\" : DEFAULT_CLIP_GRADIENT,\n                   \"lazy_update\" : DEFAULT_LAZY_UPDATE,\n                   \"data_4d\": DEFAULT_DATA_4d,\n                   \"dim1\": DEFAULT_DIM_1,\n                   \"dim2\": DEFAULT_DIM_2,\n                   \"block_size\": DEFAULT_BLOCK_SIZE,\n                   \"args\": DEFAULT_ARGS,\n                   \"a\": DEFAULT_DATA,\n                   \"index\": DEFAULT_INDEX,\n                   \"indices\": DEFAULT_INDICES,\n                   \"begin\": DEFAULT_BEGIN,\n                   \"end\": DEFAULT_END,\n                   \"shape_like\": DEFAULT_SHAPE_LIKE,\n                   \"x\": DEFAULT_X,\n                   \"y\": DEFAULT_Y,\n                   \"condition\": DEFAULT_COND,\n                   \"depth\": DEFAULT_DEPTH,\n                   \"ravel_data\": RAVEL_DATA,\n                   \"data_smce\": DEFAULT_DATA_SMCE,\n                   \"data_3d\": DEFAULT_DATA_3d,\n                   \"label_smce\": DEFAULT_LABEL_SMCE,\n                   \"label\": DEFAULT_LABEL,\n                   \"num_outputs\": DEFAULT_NUM_OUTPUTS,\n                   \"data_dim1\": DEFAULT_DATA_DIM1,\n                   \"size\": DEFAULT_SIZE,\n                   \"mode_4d\": DEFAULT_MODE_4d,\n                   \"pad_width_4d\": DEFAULT_PAD_WIDTH_4d,\n                   \"repeats\": DEFAULT_REPEATS,\n                   \"reps\": DEFAULT_REPEATS,\n                   \"grid\": DEFAULT_GRID,\n                   \"data_bilinearsampler\": DEFAULT_DATA_BILINEAR,\n                   \"transform_type\": DEFAULT_TRANSFORM_TYPE,\n                   \"data_gridgenerator\": DEFAULT_DATA_GRIDGEN,\n                   \"target_shape_gridgenerator\": DEFAULT_TARGET_SHAPE,\n                   \"data_sample_multinomial\": DEFAULT_DATA_SM,\n                   \"A\": DEFAULT_A,\n                   \"B\": DEFAULT_B,\n                   \"C\": DEFAULT_C,\n                   \"A_linalg_maketrian\": DEFAULT_A_MT,\n                   \"axes\": DEFAULT_AXES,\n                   \"act_type_leakyrelu\": DEFAULT_ACT_TYPE_LR,\n                   \"label_softmax\": DEFAULT_LABEL_SOFTMAX,\n                   \"act_type_activation\": DEFAULT_ACT_TYPE_ACTIVATION,\n                   \"data_squeeze\": DEFAULT_DATA_SQUEEZE,\n                   \"axis_squeeze\": DEFAULT_AXIS_SQUEEZE,\n                   \"a_min\": DEFAULT_A_MIN,\n                   \"a_max\": DEFAULT_A_MAX,\n                   \"lrs\": DEFAULT_LRS,\n                   \"weights_sum_sq\": DEFAULT_WSS,\n                   \"grads_sum_sq\": DEFAULT_GSS,\n                   \"wds\": DEFAULT_WDS,\n                   \"eta\": DEFAULT_ETA,\n                   \"eps\": DEFAULT_EPSILON,\n                   \"stype\": DEFAULT_STYPE,\n                   \"a\": DEFAULT_A,\n                   \"lhs_fill_element_0index\": DEFAULT_LHS_FEI,\n                   \"rhs_fill_element_0index\": DEFAULT_RHS_FEI,\n                   \"mhs\": DEFAULT_MHS,\n                   \"data_spatialtransformer\": DEFAULT_DATA_ST,\n                   \"loc_spatialtransformer\": DEFAULT_LOC_TAR_ST,\n                   \"target_shape\": DEFAULT_LOC_TAR_ST,\n                   \"transform_type_spatialtransformer\": DEFAULT_TRANSFORM,\n                   \"sampler_type\": DEFAULT_SAMPLER,\n                   \"data_col2im\": DEFAULT_DATA_C2I,\n                   \"output_size\": DEFAULT_OUTPUT_SIZE,\n                   \"kernel_col2im\": DEFAULT_KERNEL,\n                   \"stride_col2im\": DEFAULT_STRIDE,\n                   \"parameters\": DEFAULT_PARAMETERS,\n                   \"state\": DEFAULT_STATE,\n                   \"state_size\": DEFAULT_STATE_SIZE,\n                   \"num_layers\": DEFAULT_NUM_LAYERS,\n                   \"data_groupnorm\": DEFAULT_DATA_GN,\n                   \"gamma_groupnorm\": DEFAULT_BETA_GAMMA_GN,\n                   \"beta_groupnorm\": DEFAULT_BETA_GAMMA_GN,\n                   \"num_groups\": DEFAULT_NUM_GROUPS,\n                   \"data_dropout\": DEFAULT_DATA_DROPOUT,\n                   \"mode_dropout\": DEFAULT_MODE_DROPOUT,\n                   \"p_dropout\": DEFAULT_P,\n                   \"data_nn_basic\": DEFAULT_DATA_NN_BASIC,\n                   \"num_hidden\": DEFAULT_NUM_HIDDEN,\n                   \"data_fullyconnected\": DEFAULT_DATA_NN_BASIC,\n                   \"weight_fullyconnected\": DEFAULT_WEIGHT_FC,\n                   \"weight_embedding\": DEFAULT_WEIGHT_EMBEDDING,\n                   \"bias\": DEFAULT_BIAS,\n                   \"flatten\": DEFAULT_FLATTEN,\n                   \"data_batchnorm\": DEFAULT_DATA_NN_BASIC,\n                   \"gamma_batchnorm\": DEFAULT_GAMMA,\n                   \"beta_batchnorm\": DEFAULT_BETA,\n                   \"moving_mean_batchnorm\": DEFAULT_MOVING_MEAN,\n                   \"moving_var_batchnorm\": DEFAULT_MOVING_VAR,\n                   \"axis_batchnorm\": DEFAULT_AXIS_BN,\n                   \"grad_scale\": DEFAULT_GRAD_SCALE,\n                   \"normalization\": DEFAULT_NORMALIZATION,\n                   \"margin\": DEFAULT_MARGIN,\n                   \"regularization_coefficient\": DEFAULT_REG_COEFF,\n                   \"data_l2normalization\": DEFAULT_DATA_NN_BASIC,\n                   \"mode_l2normalization\": DEFAULT_MODE_L2,\n                   \"gamma_layernorm\": DEFAULT_GAMMA_LN,\n                   \"beta_layernorm\": DEFAULT_BETA_LN,\n                   \"data_instancenorm\": DEFAULT_DATA_NN_BASIC,\n                   \"gamma_instancenorm\": DEFAULT_GAMMA,\n                   \"beta_instancenorm\": DEFAULT_BETA,\n                   \"input_dim\": DEFAULT_INPUT_DIM,\n                   \"output_dim\": DEFAULT_OUTPUT_DIM,\n                   \"sparse_grad\": DEFAULT_SPARSE_GRAD,\n                   \"data1\": DEFAULT_DATA_NN_BASIC,\n                   \"data2\": DEFAULT_DATA_NN_BASIC,\n                   \"kernel_size\": DEFAULT_KERNEL_SIZE,\n                   \"max_displacement\": DEFAULT_MAX_DISPLACEMENT,\n                   \"stride1\": DEFAULT_STRIDE_1,\n                   \"stride2\": DEFAULT_STRIDE_2,\n                   \"data_im2col\": DEFAULT_DATA_NN_BASIC,\n                   \"kernel_im2col\": DEFAULT_KERNEL_I2C,\n                   \"stride_im2col\": DEFAULT_STRIDE_I2C,\n                   \"dilate_im2col\": DEFAULT_DILATE,\n                   \"pad_im2col\": DEFAULT_PAD,\n                   \"data_lrn\": DEFAULT_DATA_NN_BASIC,\n                   \"alpha_lrn\": DEFAULT_ALPHA,\n                   \"beta_lrn\": DEFAULT_BETA_LRN,\n                   \"nsize\": DEFAULT_NSIZE,\n                   \"data_layernorm\": DEFAULT_DATA_NN_BASIC,\n                   \"axis_layernorm\": DEFAULT_AXIS}\n\n# Default Inputs for Large Tensor. MXNet Op Param Name to Default Input mapping\nDEFAULTS_INPUTS_LARGE_TENSOR = {\"data\": DEFAULT_DATA_LARGE_TENSOR,\n                                \"dtype\": DEFAULT_DTYPE,\n                                \"dtype_int\": DEFAULT_DTYPE_INT,\n                                \"dtype_float\": DEFAULT_DTYPE_FLOAT,\n                                \"sample\": DEFAULT_SAMPLE_LARGE_TENSOR,\n                                \"lhs\": DEFAULT_LHS_LARGE_TENSOR,\n                                \"rhs\": DEFAULT_RHS_LARGE_TENSOR,\n                                \"shape\": DEFAULT_SHAPE_LARGE_TENSOR,\n                                \"low\": DEFAULT_LOW,\n                                \"high\": DEFAULT_HIGH,\n                                \"low_nd\": DEFAULT_LOW_ND_LARGE_TENSOR,\n                                \"high_nd\": DEFAULT_HIGH_ND_LARGE_TENSOR,\n                                \"mu_nd\": DEFAULT_MU_ND_LARGE_TENSOR,\n                                \"sigma\": DEFAULT_SIGMA_LARGE_TENSOR,\n                                \"alpha_nd\": DEFAULT_ALPHA_ND_LARGE_TENSOR,\n                                \"beta_nd\": DEFAULT_BETA_ND_LARGE_TENSOR,\n                                \"lam_nd\": DEFAULT_LAM_ND_LARGE_TENSOR,\n                                \"lam_random_pdf_exponential\": DEFAULT_LAM_RPE_LARGE_TENSOR,\n                                \"sample_random_pdf_exponential\": DEFAULT_SAMPLE_RPE_LARGE_TENSOR,\n                                \"k\": DEFAULT_K,\n                                \"p\": DEFAULT_P,\n                                \"k_nd\": DEFAULT_K_ND_LARGE_TENSOR,\n                                \"p_nd\": DEFAULT_P_ND_LARGE_TENSOR,\n                                \"axis\": DEFAULT_AXIS,\n                                \"weight\" : DEFAULT_WEIGHT_LARGE_TENSOR,\n                                \"weight32\" : DEFAULT_WEIGHT_LARGE_TENSOR,\n                                \"grad\" : DEFAULT_GRAD_LARGE_TENSOR,\n                                \"mean\" : DEFAULT_MEAN_LARGE_TENSOR,\n                                \"var\" : DEFAULT_VAR_LARGE_TENSOR,\n                                \"mom\" : DEFAULT_MOM_LARGE_TENSOR,\n                                \"r1\": DEFAULT_R1_LARGE_TENSOR,\n                                \"r2\": DEFAULT_R2_LARGE_TENSOR,\n                                \"n\" : DEFAULT_N_LARGE_TENSOR,\n                                \"d\" : DEFAULT_D_LARGE_TENSOR,\n                                \"v\" : DEFAULT_V_LARGE_TENSOR,\n                                \"z\" : DEFAULT_Z_LARGE_TENSOR,\n                                \"g\" : DEFAULT_G_LARGE_TENSOR,\n                                \"delta\" : DEFAULT_DELTA_LARGE_TENSOR,\n                                \"lr\" : DEFAULT_LR,\n                                \"lrs\" : DEFAULT_LRS,\n                                \"wd\": DEFAULT_WD,\n                                \"rho\" : DEFAULT_RHO,\n                                \"momentum\" : DEFAULT_MOMENTUM,\n                                \"epsilon\" : DEFAULT_EPSILON,\n                                \"beta1\" : DEFAULT_BETA_1,\n                                \"beta2\" : DEFAULT_BETA_2,\n                                \"t\" : DEFAULT_T,\n                                \"rescale_grad\" : DEFAULT_RESCALE_GRAD,\n                                \"clip_grad\" : DEFAULT_CLIP_GRADIENT,\n                                \"lazy_update\" : DEFAULT_LAZY_UPDATE,\n                                \"data_4d\": DEFAULT_DATA_4d_LARGE_TENSOR,\n                                \"dim1\": DEFAULT_DIM_1,\n                                \"dim2\": DEFAULT_DIM_2,\n                                \"block_size\": DEFAULT_BLOCK_SIZE_LARGE_TENSOR,\n                                \"args\": DEFAULT_ARGS,\n                                \"index\": DEFAULT_INDEX_LARGE_TENSOR,\n                                \"data_smce\": DEFAULT_DATA_SMCE_LARGE_TENSOR,\n                                \"label_smce\": DEFAULT_LABEL_SMCE_LARGE_TENSOR,\n                                \"grid\": DEFAULT_GRID_LARGE_TENSOR,\n                                \"data_bilinearsampler\": DEFAULT_DATA_BILINEAR_LARGE_TENSOR,\n                                \"transform_type\": DEFAULT_TRANSFORM_TYPE,\n                                \"data_gridgenerator\": DEFAULT_DATA_GRIDGEN_LARGE_TENSOR,\n                                \"target_shape_gridgenerator\": DEFAULT_TARGET_SHAPE_LARGE_TENSOR,\n                                \"data_sample_multinomial\": DEFAULT_DATA_SM_LARGE_TENSOR,\n                                \"data_random_pdf_dirichlet\": DEFAULT_DATA_RPD_LARGE_TENSOR,\n                                \"alpha_random_pdf_dirichlet\": DEFAULT_ALPHA_RPD_LARGE_TENSOR,\n                                \"sample_random_pdf_gamma\": DEFAULT_SAMPLE_RPG_LARGE_TENSOR,\n                                \"alpha_random_pdf_gamma\": DEFAULT_ALPHA_RPG_LARGE_TENSOR,\n                                \"beta_random_pdf_gamma\": DEFAULT_BETA_LARGE_TENSOR,\n                                \"sample_random_pdf_generalized_negative_binomial\": DEFAULT_SAMPLE_RPG_LARGE_TENSOR,\n                                \"mu_random_pdf_generalized_negative_binomial\": DEFAULT_ALPHA_RPG_LARGE_TENSOR,\n                                \"alpha_random_pdf_generalized_negative_binomial\": DEFAULT_ALPHA_RPG_LARGE_TENSOR,\n                                \"sample_random_pdf_negative_binomial\": DEFAULT_SAMPLE_RPG_LARGE_TENSOR,\n                                \"k_random_pdf_negative_binomial\": DEFAULT_ALPHA_RPG_LARGE_TENSOR,\n                                \"p_random_pdf_negative_binomial\": DEFAULT_ALPHA_RPG_LARGE_TENSOR,\n                                \"sample_random_pdf_normal\": DEFAULT_SAMPLE_RPG_LARGE_TENSOR,\n                                \"mu_random_pdf_normal\": DEFAULT_ALPHA_RPG_LARGE_TENSOR,\n                                \"sigma_random_pdf_normal\": DEFAULT_ALPHA_RPG_LARGE_TENSOR,\n                                \"sample_random_pdf_poisson\": DEFAULT_SAMPLE_RPG_LARGE_TENSOR,\n                                \"lam_random_pdf_poisson\": DEFAULT_ALPHA_RPG_LARGE_TENSOR,\n                                \"sample_random_pdf_uniform\": DEFAULT_SAMPLE_RPG_LARGE_TENSOR,\n                                \"low_random_pdf_uniform\": DEFAULT_ALPHA_RPG_LARGE_TENSOR,\n                                \"high_random_pdf_uniform\": DEFAULT_ALPHA_RPG_LARGE_TENSOR,\n                                \"shape_sample_exponential\": DEFAULT_SHAPE_SE_LARGE_TENSOR,\n                                \"lam_sample_exponential\": DEFAULT_LAM_SE_LARGE_TENSOR,\n                                \"mu_sample_normal\": DEFAULT_LAM_SE_LARGE_TENSOR,\n                                \"sigma_sample_normal\": DEFAULT_LAM_SE_LARGE_TENSOR,\n                                \"shape_sample_poisson\": DEFAULT_LAM_SE_LARGE_TENSOR,\n                                \"lam_sample_poisson\": DEFAULT_SHAPE_SE_LARGE_TENSOR,\n                                \"shape_sample_uniform\": DEFAULT_SHAPE_SU_LARGE_TENSOR,\n                                \"low_sample_uniform\": DEFAULT_LAM_SE_LARGE_TENSOR,\n                                \"high_sample_uniform\": DEFAULT_LAM_SE_LARGE_TENSOR,\n                                \"alpha_sample_gamma\": DEFAULT_SHAPE_SU_LARGE_TENSOR,\n                                \"beta_sample_gamma\": DEFAULT_SHAPE_SU_LARGE_TENSOR,\n                                \"mu_sample_generalized_negative_binomial\": DEFAULT_SHAPE_SU_LARGE_TENSOR,\n                                \"shape_sample_generalized_negative_binomial\": DEFAULT_SHAPE_SU_LARGE_TENSOR,\n                                \"alpha_sample_generalized_negative_binomial\": DEFAULT_SHAPE_SU_LARGE_TENSOR,\n                                \"shape_sample_negative_binomial\": DEFAULT_SHAPE_SU_LARGE_TENSOR,\n                                \"k_sample_negative_binomial\": DEFAULT_SHAPE_SU_LARGE_TENSOR,\n                                \"p_sample_negative_binomial\": DEFAULT_SHAPE_SU_LARGE_TENSOR,\n                                \"A\": DEFAULT_A_LARGE_TENSOR,\n                                \"B\": DEFAULT_B_LARGE_TENSOR,\n                                \"C\": DEFAULT_C_LARGE_TENSOR,\n                                \"A_linalg_maketrian\": DEFAULT_A_MT_LARGE_TENSOR,\n                                \"axes\": DEFAULT_AXES,\n                                \"act_type_leakyrelu\": DEFAULT_ACT_TYPE_LR,\n                                \"label_softmax\": DEFAULT_LABEL_SOFTMAX_LARGE_TENSOR,\n                                \"act_type_activation\": DEFAULT_ACT_TYPE_ACTIVATION,\n                                \"data_squeeze\": DEFAULT_DATA_SQUEEZE_LARGE_TENSOR,\n                                \"axis_squeeze\": DEFAULT_AXIS_SQUEEZE_LARGE_TENSOR,\n                                \"a_min\": DEFAULT_A_MIN,\n                                \"a_max\": DEFAULT_A_MAX,\n                                \"weights_sum_sq\": DEFAULT_WSS_LARGE_TENSOR,\n                                \"grads_sum_sq\": DEFAULT_GSS_LARGE_TENSOR,\n                                \"wds\": DEFAULT_WDS_LARGE_TENSOR,\n                                \"eta\": DEFAULT_ETA,\n                                \"eps\": DEFAULT_EPSILON,\n                                \"stype\": DEFAULT_STYPE,\n                                \"indices\": DEFAULT_INDICES,\n                                \"begin\": DEFAULT_BEGIN,\n                                \"end\": DEFAULT_END,\n                                \"shape_like\": DEFAULT_DATA_LARGE_TENSOR,\n                                \"depth\": DEFAULT_DEPTH,\n                                \"condition\": DEFAULT_X_LARGE_TENSOR,\n                                \"x\": DEFAULT_X_LARGE_TENSOR,\n                                \"y\": DEFAULT_X_LARGE_TENSOR,\n                                \"ravel_data\": RAVEL_DATA_LARGE_TENSOR,\n                                \"a\": DEFAULT_A_LARGE_TENSOR,\n                                \"lhs_fill_element_0index\": DEFAULT_LHS_FEI_LARGE_TENSOR,\n                                \"rhs_fill_element_0index\": DEFAULT_RHS_FEI_LARGE_TENSOR,\n                                \"mhs\": DEFAULT_MHS_LARGE_TENSOR,\n                                \"lrs_multi_lars\": DEFAULT_WSS_LARGE_TENSOR,\n                                \"data_softmax\": DEFAULT_LABEL_SOFTMAX_LARGE_TENSOR,\n                                \"data_spatialtransformer\": DEFAULT_DATA_ST_LARGE_TENSOR,\n                                \"loc_spatialtransformer\": DEFAULT_LOC_TAR_ST_LARGE_TENSOR,\n                                \"target_shape\": DEFAULT_LOC_TAR_ST_LARGE_TENSOR,\n                                \"transform_type_spatialtransformer\": DEFAULT_TRANSFORM,\n                                \"sampler_type\": DEFAULT_SAMPLER,\n                                \"data_col2im\": DEFAULT_DATA_C2I_LARGE_TENSOR,\n                                \"output_size\": DEFAULT_OUTPUT_SIZE_LARGE_TENSOR,\n                                \"kernel_col2im\": DEFAULT_KERNEL_LARGE_TENSOR,\n                                \"stride_col2im\": DEFAULT_STRIDE_LARGE_TENSOR,\n                                \"data_ctcloss\": DEFAULT_DATA_CTCLOSS,\n                                \"label_ctcloss\": DEFAULT_LABEL_LARGE_TENSOR,\n                                \"data_ctc_loss\": DEFAULT_DATA_CTCLOSS,\n                                \"label_ctc_loss\": DEFAULT_LABEL_LARGE_TENSOR,\n                                \"parameters\": DEFAULT_PARAMETERS_LARGE_TENSOR,\n                                \"state\": DEFAULT_STATE_LARGE_TENSOR,\n                                \"state_size\": DEFAULT_STATE_SIZE_LARGE_TENSOR,\n                                \"num_layers\": DEFAULT_NUM_LAYERS_LARGE_TENSOR,\n                                \"data_groupnorm\": DEFAULT_DATA_GN_LARGE_TENSOR,\n                                \"gamma_groupnorm\": DEFAULT_BETA_GAMMA_GN_LARGE_TENSOR,\n                                \"beta_groupnorm\": DEFAULT_BETA_GAMMA_GN_LARGE_TENSOR,\n                                \"data_dropout\": DEFAULT_DATA_DROPOUT_LARGE_TENSOR,\n                                \"mode_dropout\": DEFAULT_MODE_DROPOUT,\n                                \"p_dropout\": DEFAULT_P_DROPOUT_LARGE_TENSOR,\n                                \"axes_dropout\": DEFAULT_AXES_DROPOUT_LARGE_TENSOR,\n                                \"data_nn_basic\": DEFAULT_DATA_NN_BASIC_LARGE_TENSOR,\n                                \"num_hidden\": DEFAULT_NUM_HIDDEN_LARGE_TENSOR,\n                                \"data_fullyconnected\": DEFAULT_DATA_FC_LARGE_TENSOR,\n                                \"weight_fullyconnected\": DEFAULT_WEIGHT_FC_LARGE_TENSOR,\n                                \"num_hidden_fullyconnected\": DEFAULT_NUM_HIDDEN_FC_LARGE_TENSOR,\n                                \"weight_embedding\": DEFAULT_WEIGHT_EMBEDDING_LARGE_TENSOR,\n                                \"bias\": DEFAULT_BIAS_LARGE_TENSOR,\n                                \"flatten\": DEFAULT_FLATTEN_LARGE_TENSOR,\n                                \"data_batchnorm\": DEFAULT_DATA_NN_BASIC_LARGE_TENSOR,\n                                \"gamma_batchnorm\": DEFAULT_GAMMA_LARGE_TENSOR,\n                                \"beta_batchnorm\": DEFAULT_BETA_LARGE_TENSOR,\n                                \"moving_mean_batchnorm\": DEFAULT_MOVING_MEAN_LARGE_TENSOR,\n                                \"moving_var_batchnorm\": DEFAULT_MOVING_VAR_LARGE_TENSOR,\n                                \"axis_batchnorm\": DEFAULT_AXIS_BN,\n                                \"grad_scale\": DEFAULT_GRAD_SCALE,\n                                \"normalization\": DEFAULT_NORMALIZATION,\n                                \"margin\": DEFAULT_MARGIN,\n                                \"regularization_coefficient\": DEFAULT_REG_COEFF,\n                                \"data_l2normalization\": DEFAULT_DATA_NORM_LARGE_TENSOR,\n                                \"mode_l2normalization\": DEFAULT_MODE_L2,\n                                \"gamma_layernorm\": DEFAULT_GAMMA_NORM_LARGE_TENSOR,\n                                \"beta_layernorm\": DEFAULT_BETA_NORM_LARGE_TENSOR,\n                                \"data_instancenorm\": DEFAULT_DATA_NORM_LARGE_TENSOR,\n                                \"gamma_instancenorm\": DEFAULT_GAMMA_NORM_LARGE_TENSOR,\n                                \"beta_instancenorm\": DEFAULT_GAMMA_NORM_LARGE_TENSOR,\n                                \"input_dim\": DEFAULT_INPUT_DIM_LARGE_TENSOR,\n                                \"output_dim\": DEFAULT_OUTPUT_DIM_LARGE_TENSOR,\n                                \"sparse_grad\": DEFAULT_SPARSE_GRAD,\n                                \"data1\": DEFAULT_DATA1_LARGE_TENSOR,\n                                \"data2\": DEFAULT_DATA2_LARGE_TENSOR,\n                                \"kernel_size\": DEFAULT_KERNEL_SIZE_LARGE_TENSOR,\n                                \"max_displacement\": DEFAULT_MAX_DISPLACEMENT_LARGE_TENSOR,\n                                \"stride1\": DEFAULT_STRIDE_1_LARGE_TENSOR,\n                                \"stride2\": DEFAULT_STRIDE_2_LARGE_TENSOR,\n                                \"data_im2col\": DEFAULT_DATA_I2C_LARGE_TENSOR,\n                                \"kernel_im2col\": DEFAULT_KERNEL_I2C_LARGE_TENSOR,\n                                \"stride_im2col\": DEFAULT_STRIDE_I2C_LARGE_TENSOR,\n                                \"dilate_im2col\": DEFAULT_DILATE_LARGE_TENSOR,\n                                \"pad_im2col\": DEFAULT_PAD_LARGE_TENSOR,\n                                \"data_lrn\": DEFAULT_DATA_LRN_LARGE_TENSOR,\n                                \"alpha_lrn\": DEFAULT_ALPHA,\n                                \"beta_lrn\": DEFAULT_BETA_LRN,\n                                \"nsize\": DEFAULT_NSIZE,\n                                \"data_layernorm\": DEFAULT_DATA_NORM_LARGE_TENSOR,\n                                \"axis_layernorm\": DEFAULT_AXIS_LARGE_TENSOR}\n\n# These are names of MXNet operator parameters that is of type NDArray.\n# We maintain this list to automatically recognize these parameters are to be\n# given as NDArray and translate users inputs such as a shape tuple, Numpy Array or\n# a list to MXNet NDArray. This is just a convenience added so benchmark utility users\n# can just say shape of the tensor, and we automatically create Tensors.\nPARAMS_OF_TYPE_NDARRAY = [\"lhs\", \"rhs\", \"data\", \"base\", \"exp\", \"sample\",\n                          \"mu\", \"sigma\", \"lam\", \"alpha\", \"beta\", \"gamma\", \"k\", \"p\",\n                          \"low\", \"high\", \"weight\", \"bias\", \"moving_mean\", \"moving_var\",\n                          \"weight\", \"weight32\", \"grad\", \"mean\", \"var\", \"mom\", \"n\", \"d\",\n                          \"v\", \"z\", \"g\", \"delta\", \"args\", \"indices\", \"shape_like\", \"y\",\n                          \"x\", \"condition\", \"a\", \"index\", \"raveL_data\", \"label\", \"grid\",\n                          \"A\", \"B\", \"C\", \"r1\", \"r2\", \"rois\", \"lrs\", \"wds\", \"weights_sum_sq\",\n                          \"grads_sum_sq\", \"mhs\", \"data1\", \"data2\", \"loc\", \"parameters\", \"state\",\n                          \"state_cell\", \"tensor\", \"arrays\", \"mask\", \"running_mean\", \"running_var\"]\n\nPARAMS_OF_TYPE_NP_ARRAY = [\"x1\", \"x2\", \"prototype\", \"source_array\", \"object\", \"a\", \"b\", \"fill_value\", \"array\", \"x\", \"arr\",\n                           \"values\", \"ary\", \"seq\", \"arrays\", \"tup\", \"indices\", \"m\", \"ar\", \"q\", \"p\", \"condition\",\n                           \"arys\", \"v\", \"A\", \"xp\", \"fp\", \"data\", \"gamma\", \"beta\", \"running_mean\", \"moving_mean\", \"moving_var\",\n                           \"running_var\", \"weight\", \"index\", \"lhs\", \"rhs\", \"parameters\", \"state\", \"mask\", \"bias\"]\n\n"
  },
  {
    "path": "benchmark/opperf/utils/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n"
  },
  {
    "path": "benchmark/opperf/utils/benchmark_operators_pytest.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport mxnet as mx\nimport pytest\n\nfrom benchmark.opperf.utils.benchmark_utils import run_benchmark_operator\n\ntest_cases = {\n    \"reshape\"           : [((128,128,128), {\"newshape\": (128,256,-1)}),\n                           ((256,256,256), {\"newshape\": (256,512,-1)}),\n                           ((512,512,512), {\"newshape\": (512,1024,-1)}),],\n    \"swapaxes\"          : [((64,128,64), {\"axis1\": 1, \"axis2\": 2}),\n                           ((128,256,128), {\"axis1\": 1, \"axis2\": 2}),\n                           ((256,512,256), {\"axis1\": 1, \"axis2\": 2})],\n    \"activation\"        : [((128,128,128), {\"actType\": \"relu\"}),\n                           ((256,256,256), {\"actType\": \"relu\"}),\n                           ((512,512,512), {\"actType\": \"relu\"})],\n    \"batch_norm\"        : [((128,128,128), {}),\n                           ((256,256,256), {}),\n                           ((512,512,512), {})],\n    \"convolution\"       : [((16,16,16,16,16), {\"numFilter\": 8, \"kernel\": (3,3,3)}),\n                           ((32,32,16,16,16), {\"numFilter\": 16, \"kernel\": (5,5,5)}),\n                           ((32,32,32,32,32), {\"numFilter\": 16, \"kernel\": (7,7,7)})],\n    \"add\"               : [((128,128,128), {}),\n                           ((256,256,256), {}),\n                           ((512,512,512), {})],\n    \"masked_softmax\"    : [((128,128,128), {}),\n                           ((256,256,256), {}),\n                           ((512,512,512), {})],\n    \"slice\"             : [((128,128,128), {\"begin\": (32,32,32), \"end\": (-32,-32,-32)}),\n                           ((256,256,256), {\"begin\": (64,64,64), \"end\": (-64,-64,-64)}),\n                           ((512,512,512), {\"begin\": (96,96,96), \"end\": (-96,-96,-96)})],\n    \"fully_connected\"   : [((20,20,20,20), {\"numHidden\": 30}),\n                           ((60,60,60,60), {\"numHidden\": 60}),\n                           ((90,90,90,90), {\"numHidden\": 90}),],\n    \"batch_dot\"         : [((10,10,10), {\"matrix1\": (20,30), \"matrix2\": (30,40)}),\n                           ((20,20,20), {\"matrix1\": (40,50), \"matrix2\": (50,60)}),\n                           ((40,40,40), {\"matrix1\": (60,70), \"matrix2\": (70,80)})]\n}\n\ndef generate_test_cases():\n    tests = []\n    for op_name, cases in test_cases.items():\n        for case in cases:\n            tests.append((op_name, case[0], case[1]))\n    return tests\n\ndef generate_test_ids():\n    test_ids = []\n    for op_name, cases in test_cases.items():\n        for case in cases:\n            s = op_name + \"-shape_\"\n            for i in range(len(case[0])):\n                s += str(case[0][i])\n                if (i != len(case[0])-1):\n                    s += \"x\"\n            params = case[1].items()\n            if len(params) != 0:\n                s += \"-params\"\n                for key, value in params:\n                    s += \"_\" + str(key) + \"_\"\n                    if isinstance(value, tuple):\n                        for i in range(len(value)):\n                            s += str(value[i])\n                            if (i != len(value)-1):\n                                s += \"x\"\n                    else:\n                        s += str(value)\n            test_ids.append(s)\n    return test_ids\n\ngenerate_inputs = {\n    \"reshape\"               : lambda shape, metadata: {\"newshape\": metadata[\"newshape\"], \"shape\": metadata[\"newshape\"]},\n    \"swapaxes\"              : lambda shape, metadata: {\"axis1\": metadata[\"axis1\"], \"axis2\": metadata[\"axis2\"],\n                                                       \"dim1\": metadata[\"axis1\"], \"dim2\": metadata[\"axis2\"]},\n    \"activation\"            : lambda shape, metadata: {\"act_type\": metadata[\"actType\"]},\n    \"batch_norm\"            : lambda shape, metadata: {\"gamma\": (shape[1],), \"beta\": (shape[1],), \"running_mean\": (shape[1],), \"running_var\": (shape[1],),\n                                                       \"moving_mean\": (shape[1],), \"moving_var\": (shape[1],)},\n    \"convolution\"           : lambda shape, metadata: {\"weight\": (metadata[\"numFilter\"], shape[1]) + metadata[\"kernel\"], \"kernel\": metadata[\"kernel\"],\n                                                       \"bias\": (metadata[\"numFilter\"],), \"num_filter\": metadata[\"numFilter\"]},\n    \"masked_softmax\"        : lambda shape, metadata: {\"mask\": mx.np.array(round(mx.np.random.rand(*shape)), dtype=\"bool\")},\n    \"fully_connected\"       : lambda shape, metadata: {\"weight\": (metadata[\"numHidden\"], shape[-1]), \"bias\": (metadata[\"numHidden\"],), \n                                                       \"num_hidden\": metadata[\"numHidden\"], \"flatten\": False},\n    \"batch_dot\"             : lambda shape, metadata: {\"lhs\": shape + metadata[\"matrix1\"], \"a\": shape + metadata[\"matrix1\"],\n                                                       \"rhs\": shape + metadata[\"matrix2\"], \"b\": shape + metadata[\"matrix2\"]},\n    \"slice\"                 : lambda shape, metadata: {\"begin\": metadata[\"begin\"], \"end\": metadata[\"end\"]}\n}\n\n@pytest.mark.parametrize(argnames=(\"op_name, shape, params\"), argvalues=generate_test_cases(), ids=generate_test_ids())\ndef test(op_name, shape, params):\n    if op_name in generate_inputs.keys():\n        additional_inputs = generate_inputs[op_name](shape,params)\n    else:\n        additional_inputs = {}\n    run_benchmark_operator(name=op_name, size=shape, additional_inputs=additional_inputs, profiler=\"python\")\n"
  },
  {
    "path": "benchmark/opperf/utils/benchmark_utils.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport logging\nimport inspect\n\nimport mxnet as mx\nfrom mxnet import nd\nfrom mxnet import np\n\nfrom .ndarray_utils import get_mx_ndarray, nd_forward_and_profile, nd_forward_backward_and_profile\nfrom .common_utils import merge_map_list\nfrom .op_registry_utils import prepare_op_inputs\nfrom benchmark.opperf.rules.default_params import PARAMS_OF_TYPE_NDARRAY, PARAMS_OF_TYPE_NP_ARRAY\nfrom .profiler_utils import cpp_profile, python_profile\n\nno_backward = {'gather_nd', 'softmax_cross_entropy', 'linalg_gelqf', 'linalg_slogdet', 'moments', 'SequenceLast', 'Embedding'}\n\ndef _prepare_op_inputs(inputs, run_backward, dtype, ctx, module):\n    mx.random.seed(41)\n    kwargs_list = []\n    if module == 'mxnet.numpy_extension' or module == 'mxnet.numpy':\n        PARAMS_TYPE = PARAMS_OF_TYPE_NP_ARRAY\n        get_array_fn = get_mx_np_ndarray\n    else:\n        PARAMS_TYPE = PARAMS_OF_TYPE_NDARRAY\n        get_array_fn = get_mx_ndarray\n\n    for inp in inputs:\n        kwargs = {}\n        for key, value in inp.items():\n            if key in PARAMS_TYPE:\n                kwargs[key] = get_array_fn(ctx=ctx, in_tensor=value,\n                                           dtype=dtype,\n                                           initializer=nd.normal,\n                                           attach_grad=run_backward)\n            else:\n                kwargs[key] = value\n        kwargs_list.append(kwargs)\n    return kwargs_list\n\ndef get_mx_np_ndarray(ctx, in_tensor, dtype, initializer, attach_grad=True):\n    \"\"\"Helper function to prepare a MXNet Numpy NDArray tensor in given Context (ctx) of type (dtype).\n    You can get a new Tensor by providing only \"Shape\" or \"Numpy NDArray\" or another MXNet NDArray as\n    \"in_tensor\".\n\n    NOTE: This is a sync call and waits for the Tensor to be created.\n\n    Parameters\n    ----------\n    ctx: mx.ctx, default mx.cpu()\n        Context of the new MXNet NDArray Tensor.\n    in_tensor: Numpy NDArray or MXNet NDArray or Tuple of shape\n        Can be a tuple of shape or Numpy NDArray or MXNet NDArray.\n    dtype: str\n        Precision or Dtype of the expected Tensor. Ex: \"float32\", \"Int64\"\n    initializer:\n        Function reference to the initialize to use. Ex: mx.nd.random.normal, mx.nd.zeros\n    attach_grad: Boolean, default True\n        To attach a gradient for the Tensor. Default is True.\n\n    Returns\n    -------\n    MXNet NDArray Tensor.\n    \"\"\"\n    if isinstance(in_tensor, int) or isinstance(in_tensor, float):\n        return in_tensor\n\n    if isinstance(in_tensor, tuple):\n        nd_ndarray = get_mx_ndarray(ctx=ctx, in_tensor=in_tensor,\n                                             dtype=\"float32\",\n                                             initializer=initializer,\n                                             attach_grad=attach_grad)\n        tensor = nd_ndarray.as_np_ndarray().astype(dtype=dtype)\n    elif isinstance(in_tensor, list):\n        tensor = np.array(in_tensor, ctx=ctx)\n    elif isinstance(in_tensor, nd.NDArray):\n        tensor = in_tensor.as_np_ndarray()\n    elif isinstance(in_tensor, np.ndarray):\n        tensor = in_tensor.as_in_context(ctx)\n    else:\n        raise ValueError(\"Invalid input type for creating input tensor. Input can be tuple() of shape or Numpy Array or\"\n                         \" MXNet NDArray. Given - \", in_tensor)\n    if attach_grad:\n        tensor.attach_grad()\n\n    tensor.wait_to_read()\n    return tensor\n\ndef adjust_op_name(module, name):\n    np_to_nd_func = {\n        \"batch_norm\":           \"BatchNorm\",\n        \"fully_connected\":      \"FullyConnected\",\n        \"activation\":           \"Activation\",\n        \"convolution\":          \"Convolution\" }\n    nd_to_np_func = {\n        \"BatchNorm\":            \"batch_norm\",\n        \"FullyConnected\":       \"fully_connected\",\n        \"Activation\":           \"activation\",\n        \"Convolution\":          \"convolution\" }\n\n    if (module == mx.nd and (hasattr(mx.np, name) or hasattr(mx.npx, name)) and name in np_to_nd_func.keys()):\n        return np_to_nd_func[name]\n    elif ((module == mx.np or module == mx.npx) and hasattr(mx.nd, name) and name in nd_to_np_func.keys()):\n        return nd_to_np_func[name]\n    else:\n        return name\n\ndef parse_input_ndarray(input_dict):\n    \"\"\"Parse input for ndarray and extract array shape for better readability\n\n    Parameters\n    ----------\n    input_dict : dict\n         Dictionary of input\n\n    Input Dictionary\n\n    'inputs': {'weight':\n    [[ 2.2122064   0.7740038   1.0434405   1.1839255   1.8917114 ]\n     [-1.2347414  -1.771029   -0.45138445  0.57938355 -1.856082  ]\n     [-1.9768796  -0.20801921  0.2444218  -0.03716067 -0.48774993]\n     [-0.02261727  0.57461417  1.4661262   0.6862904   0.35496104]\n     [ 1.0731696   0.12017461 -0.9711102  -0.77569664 -0.7882176 ]]\n    <NDArray 5x5 @cpu(0)>, 'grad':\n    [[ 0.7417728  -1.4734439  -1.0730928  -1.0424827  -1.3278849 ]\n     [-1.4749662  -0.52414197  1.2662556   0.8950642  -0.6015945 ]\n     [ 1.2040559  -0.9712193  -0.58256227  0.3717077   0.9300072 ]\n     [-1.4225755  -0.5176199   2.0088325   0.2863085   0.5604595 ]\n     [ 0.96975976 -0.52853745 -1.88909     0.65479124 -0.45481315]]\n    <NDArray 5x5 @cpu(0)>, 'mean':\n    [[ 0.32510808 -1.3002341   0.3679345   1.4534262   0.24154152]\n     [ 0.47898006  0.96885103 -1.0218245  -0.06812762 -0.31868345]\n     [-0.17634277  0.35655284  0.74419165  0.7787424   0.6087823 ]\n     [ 1.0741756   0.06642842  0.8486986  -0.8003802  -0.16882208]\n     [ 0.93632793  0.357444    0.77932847 -1.0103073  -0.39157307]]\n    <NDArray 5x5 @cpu(0)>, 'var':\n    [[ 1.3166187  -0.43292624  0.71535987  0.9254156  -0.90495086]\n     [-0.074684    0.82254    -1.8785107   0.8858836   1.9118724 ]\n     [ 0.33342266  0.11883813 -1.9198899  -0.67558455  1.007749  ]\n     [-0.35391203  1.6323917  -0.33354783 -1.7378405   0.7737382 ]\n     [ 0.89126545  3.2904532  -1.1976235   1.8938874  -0.5669272 ]]\n    <NDArray 5x5 @cpu(0)>, 't': 1, 'wd': 0.1}\n\n    Output\n    {'inputs': {'weight': '<NDArray 5x5 @cpu(0)>', 'grad': '<NDArray 5x5 @cpu(0)>', 'mean': '<NDArray 5x5 @cpu(0)>', 'var': '<NDArray 5x5 @cpu(0)>', 't': 1, 'wd': 0.1}\n    \"\"\"\n    no_new_line_input_dict=dict()\n    for key,value in input_dict.items():\n        if isinstance(value,nd.NDArray):\n            # if value in input is NDArray then extract last line only\n            val = str(value).split('\\n')[-1]\n            no_new_line_input_dict[key]=val\n        else:\n            no_new_line_input_dict[key]=value\n    return no_new_line_input_dict\n\n\ndef _run_operator_performance_test(op, inputs, run_backward, warmup, runs, kwargs_list, profiler):\n    if profiler == 'native':\n        if run_backward:\n            benchmark_helper_func = cpp_profile(nd_forward_backward_and_profile)\n        else:\n            benchmark_helper_func = cpp_profile(nd_forward_and_profile)\n    elif profiler == 'python':\n        if run_backward:\n            benchmark_helper_func = python_profile(nd_forward_backward_and_profile)\n        else:\n            benchmark_helper_func = python_profile(nd_forward_and_profile)\n    else:\n        raise ValueError(\"Incorrect input for profiler. Valid input - 'python' or 'native'\")\n\n    # Warm up, ignore the profiler output\n    _, _ = benchmark_helper_func(op, warmup, **kwargs_list[0])\n\n    # Run Benchmarks\n    op_benchmark_result = {op.__name__: []}\n    logging.info(f\"Begin Benchmark - {op.__name__}\")\n\n    for idx, kwargs in enumerate(kwargs_list):\n        _, profiler_output = benchmark_helper_func(op, runs, **kwargs)\n\n        # Add inputs used for profiling this operator into result\n        # parse input if it contains ndarray, replace with shape info for better markdown readability\n        new_inp = parse_input_ndarray(inputs[idx])\n        profiler_output = merge_map_list([{\"inputs\": new_inp}] + [profiler_output])\n        op_benchmark_result[op.__name__].append(profiler_output)\n    logging.info(f\"Complete Benchmark - {op.__name__}\")\n    return op_benchmark_result\n\n\ndef run_performance_test(ops, inputs, run_backward=True,\n                         dtype='float32', ctx=mx.cpu(), profiler='native',\n                         warmup=10, runs=50):\n    \"\"\"Run operator benchmark for given operator or list of operators, ops, with the given inputs.\n\n    Returns benchmark results as a list of dictionary where each dictionary represents benchmarks result per operator.\n    key -> name of the operator and value -> map of results (forward time, backward time, time spent in memory\n    operations.\n\n    Parameters\n    ----------\n    ops: [Str]\n        One or list of operators to benchmark. Should be an NDArray, Numpy or Numpy_extension operator.\n    inputs: map\n        Inputs for operator. Key should be name of parameter for operator.\n        Example: inputs = {\"lhs\": (1024, 1024), \"rhs\": (1024, 1024)} for mx.nd.add or\n                 inputs = {\"x1\": (1024, 1024), \"x2\": (1024, 1024)} for mx.np.add\n    run_backward: Boolean, Default is True\n        Should we have backward operator benchmarks.\n    dtype: Str, default 'float32'\n        Precision to use for input tensors. Defaults to float32. Example: 'float32', 'int64'\n    ctx: mx.ctx, default mx.cpu()\n        Context to use for benchmarks. Default to mx.cpu()\n    profiler: Str, default 'native'\n        Type of profiler to run benchmarks. Default to 'native'\n        Option - ['python', 'native']\n    warmup: int, default 10\n        Number of warmup runs\n    runs: int, default 50\n        Number of runs for capturing benchmark results\n\n    Returns\n    -------\n    List of dictionary of benchmark results. key -> name of the operator, Value is benchmark results.\n\n    Note: when run_performance_test is called on the nd.Embedding operator with run_backward=True, an error will\n    be thrown. Track issue here: https://github.com/apache/mxnet/issues/11314\n    \"\"\"\n    if not isinstance(ops, list):\n        ops = [ops]\n\n    op_benchmark_result = []\n    for op in ops:\n        if hasattr(mx.nd, op.__name__) or hasattr(mx.np, op.__name__) or hasattr(mx.npx, op.__name__):\n            kwargs_list = _prepare_op_inputs(inputs, run_backward, dtype, ctx, op.__module__)\n            benchmark_result = _run_operator_performance_test(op, inputs, run_backward, warmup, runs, kwargs_list, profiler)\n        else:\n            raise ValueError(f\"Unknown {op.__module__} operator provided to benchmark. - {op.__name__}\")\n        op_benchmark_result.append(benchmark_result)\n    return op_benchmark_result\n\ndef run_benchmark_operator(name, size = (128,128), additional_inputs = {},\n                           dtype = 'float32', run_backward = False, ctx = mx.cpu(),\n                           warmup=10, runs=50, profiler=\"native\"):\n    arg_list = {mx.nd: PARAMS_OF_TYPE_NDARRAY, mx.np: PARAMS_OF_TYPE_NP_ARRAY, mx.npx: PARAMS_OF_TYPE_NP_ARRAY}\n    modules = [mx.nd, mx.np, mx.npx]\n    responses = []\n    for module in modules:\n        name = adjust_op_name(module, name)\n        if hasattr(module, name):\n            function = getattr(module, name)\n            args = inspect.signature(function).parameters.keys()\n            inputs = {}\n            for arg in args:\n                if arg in additional_inputs.keys():\n                    inputs.update({arg: additional_inputs[arg]})\n                elif arg in arg_list[module]:\n                    inputs.update({arg:size})\n            res = run_performance_test(function, run_backward=run_backward, dtype=dtype, ctx=ctx,\n                                       inputs=[inputs], warmup=warmup, runs=runs, profiler=profiler)\n            responses.append(res)\n        else:\n            responses.append(str(module.__name__) + \" does not have operator \" + name)\n    for i in range(len(modules)):\n        print(modules[i].__name__)\n        print(responses[i])\n\ndef run_op_benchmarks(ops, dtype, ctx, profiler, int64_tensor, warmup, runs):\n    # Running im2col either forwards or backwards on GPU results in errors\n    # track issue here: https://github.com/apache/mxnet/issues/17493\n    gpu_disabled_ops = ['im2col']\n\n    # For each operator, run benchmarks\n    mx_op_benchmark_results = []\n    for op, op_params in ops.items():\n        if ctx == mx.cpu() or op not in gpu_disabled_ops:\n            # Prepare inputs for the operator\n            inputs = prepare_op_inputs(op, op_params, int64_tensor)\n\n            # setting backward false for ops with known issue\n            if op in no_backward:\n                op_params[\"has_backward\"] = False\n\n            # Run benchmarks\n            cur_op_res = run_performance_test(op_params[\"nd_op_handle\"],\n                                              run_backward=op_params[\"has_backward\"],\n                                              dtype=dtype, ctx=ctx,\n                                              profiler=profiler,\n                                              inputs=inputs,\n                                              warmup=warmup, runs=runs)\n            mx_op_benchmark_results += cur_op_res\n\n    # Prepare combined results for all operators\n    mx_op_benchmark_results = merge_map_list(mx_op_benchmark_results)\n    return mx_op_benchmark_results\n"
  },
  {
    "path": "benchmark/opperf/utils/common_utils.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport os\nimport json\nfrom operator import itemgetter\n\nimport logging\nlogging.basicConfig(level=logging.INFO)\n\n\ndef merge_map_list(map_list):\n    \"\"\"Merge all the Map in map_list into one final Map.\n\n    Useful when you have a list of benchmark result maps and you want to\n    prepare one final map combining all results.\n\n    Parameters\n    ----------\n    map_list: List[maps]\n        List of maps to be merged.\n\n    Returns\n    -------\n    map where all individual maps in the into map_list are merged\n\n    \"\"\"\n    # Preserve order of underlying maps and keys when converting to a single map\n    final_map = dict()\n\n    for current_map in map_list:\n        for key in current_map:\n            final_map[key] =  current_map[key]\n\n    return final_map\n\n\ndef save_to_file(inp_dict, out_filepath, out_format='json', runtime_features=None, profiler='native'):\n    \"\"\"Saves the given input dictionary to the given output file.\n\n    By default, saves the input dictionary as JSON file. Other supported formats include:\n    1. md\n\n    Parameters\n    ----------\n    inp_dict: map\n        Input dictionary to be saved\n    out_filepath: str\n        Output file path\n    out_format: str, default 'json'\n        Format of the output file. Supported options - 'json', 'md'. Default - json.\n    runtime_features: map\n        Dictionary of runtime_features.\n\n    \"\"\"\n    if out_format == 'json':\n        # Save as JSON\n        with open(out_filepath, \"w\") as result_file:\n            json.dump(inp_dict, result_file, indent=4, sort_keys=False)\n    elif out_format == 'md':\n        # Save as md\n        with open(out_filepath, \"w\") as result_file:\n            result_file.write(_prepare_markdown(inp_dict, runtime_features, profiler))\n    else:\n        raise ValueError(f\"Invalid output file format provided - '{out_format}'. Supported - json, md\")\n\n\ndef get_json(inp_dict):\n    \"\"\"Converts a given dictionary to prettified JSON string.\n\n    Parameters\n    ----------\n    inp_dict: map\n        Input dictionary to be converted to JSON.\n\n    Returns\n    -------\n    Prettified JSON string\n\n    \"\"\"\n    return json.dumps(inp_dict, indent=4)\n\n\ndef _prepare_op_benchmark_result(op, op_bench_result, profiler):\n    operator_name = op\n    avg_forward_time = \"---\"\n    avg_backward_time = \"---\"\n    max_mem_usage = \"---\"\n    inputs = \"---\"\n    avg_time = \"---\"\n    p50_time = \"---\"\n    p90_time = \"---\"\n    p99_time = \"---\"\n\n    for key, value in op_bench_result.items():\n        if \"avg_time_forward\" in key:\n            avg_forward_time = value\n        elif \"avg_time_backward\" in key:\n            avg_backward_time = value\n        elif \"max_storage_mem_alloc_\" in key:\n            max_mem_usage = value\n        elif \"inputs\" in key:\n            inputs = value\n        elif \"avg_time\" in key:\n            avg_time = value\n        elif \"p50_time\" in key:\n            p50_time = value\n        elif \"p90_time\" in key:\n            p90_time = value\n        elif \"p99_time\" in key:\n            p99_time = value\n\n    result = \"\"\n    if profiler == \"native\":\n        result = f\"| {operator_name} | {inputs} | {max_mem_usage} | {avg_forward_time} | {avg_backward_time} |\"\n    elif profiler == \"python\":\n        result = f\"| {operator_name} | {avg_time} | {p50_time} | {p90_time} | {p99_time} | {inputs} |\"\n    return result\n\n\ndef _prepare_markdown(results, runtime_features=None, profiler='native'):\n    results_markdown = []\n    if runtime_features and 'runtime_features' in runtime_features:\n        results_markdown.append(\"# Runtime Features\")\n        idx = 0\n        for key, value in runtime_features['runtime_features'].items():\n            results_markdown.append(f'{idx}. {key} : {value}')\n\n    results_markdown.append(\"# Benchmark Results\")\n    if profiler == 'native':\n        results_markdown.append(\n            \"| Operator | Inputs | Max Mem Usage (Storage) (Bytes) | Avg Forward Time (ms)\"\n            \" | Avg. Backward Time (ms) |\")\n        results_markdown.append(\"| :---: | :---: | :---: | :---: | :---: |\")\n    elif profiler == 'python':\n        results_markdown.append(\n            \"| Operator | Avg Time (ms) | P50 Time (ms) | P90 Time (ms) | P99 Time (ms) | Inputs |\")\n        results_markdown.append(\"| :---: | :---: | :---: | :---: | :---: | :---: |\")\n\n    for op, op_bench_results in sorted(results.items(), key=itemgetter(0)):\n        for op_bench_result in op_bench_results:\n            results_markdown.append(_prepare_op_benchmark_result(op, op_bench_result, profiler))\n\n    return os.linesep.join(results_markdown)\n"
  },
  {
    "path": "benchmark/opperf/utils/ndarray_utils.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport numpy as np\nimport mxnet as mx\nimport mxnet.ndarray as nd\n\n\ndef nd_forward_backward_and_profile(op, runs, **kwargs):\n    \"\"\"Helper function to run a given NDArray operator (op) for 'runs' number of times with\n    given args and kwargs. Executes both forward and backward pass.\n\n    NOTE: This is a sync call and waits for all the operations execution to complete.\n\n    Parameters\n    ----------\n    op: Str\n        NDArray operator (Function reference) to execute. Example: mx.nd.add\n    runs: int\n        Number of times to execute the operation\n    kwargs:\n        Key value arguments for the NDArray operator (op) being executed.\n\n    Returns\n    -------\n    any results from NDArray operation execution\n\n    \"\"\"\n    for _ in range(runs):\n        with mx.autograd.record():\n            args = []\n            # need to create a new dictionary because can't update dict while iterating\n            kwargs_new = dict()\n            for key in kwargs:\n                # separate positional args from key-worded args\n                if key.startswith(\"args\"):\n                    args.append(kwargs[key])\n                else:\n                    kwargs_new[key]=kwargs[key]\n            # check for positional args\n            if len(args):\n                res = op(*args, **kwargs_new)\n            else:\n                res = op(**kwargs_new)\n        res.backward()\n        nd.waitall()\n    return res\n\n\ndef nd_forward_and_profile(op, runs, **kwargs):\n    \"\"\"Helper function to run a given NDArray operator (op) for 'runs' number of times with\n    given args and kwargs. Executes ONLY forward pass.\n\n    NOTE: This is a sync call and waits for all the operations execution to complete.\n\n    Parameters\n    ----------\n    op: Str\n        NDArray operator (Function reference) to execute. Example: mx.nd.add\n    runs: int\n        Number of time to execute the operation\n    kwargs:\n        Key value arguments for the NDArray operator (op) being executed.\n\n    Returns\n    -------\n    any results from NDArray operation execution\n    \"\"\"\n    for _ in range(runs):\n        args = []\n        # need to create a new dictionary because can't update dict while iterating\n        kwargs_new = dict()\n        for key in kwargs:\n            # separate positional args from key-worded args\n            if key.startswith(\"args\"):\n                args.append(kwargs[key])\n            else:\n                kwargs_new[key]=kwargs[key]\n        # check for positional args\n        if len(args):\n            res = op(*args, **kwargs_new)\n        else:\n            res = op(**kwargs_new)\n        nd.waitall()\n    return res\n\n\ndef get_mx_ndarray(ctx, in_tensor, dtype, initializer, attach_grad=True):\n    \"\"\"Helper function to prepare a MXNet NDArray tensor in given Context (ctx) of type (dtype) with given\n    initializer. You can get a new Tensor by providing only \"Shape\" or \"Numpy NDArray\" or another MXNet NDArray as\n    \"in_tensor\".\n\n    NOTE: This is a sync call and waits for the Tensor to be created.\n\n    Parameters\n    ----------\n    ctx: mx.ctx, default mx.cpu()\n        Context of the new MXNet NDArray Tensor.\n    in_tensor: Numpy NDArray or MXNet NDArray or Tuple of shape\n        Can be a tuple of shape or Numpy NDArray or MXNet NDArray.\n    dtype: str\n        Precision or Dtype of the expected Tensor. Ex: \"float32\", \"Int64\"\n    initializer:\n        Function reference to the initialize to use. Ex: mx.nd.random.normal, mx.nd.zeros\n    attach_grad: Boolean, default True\n        To attach a gradient for the Tensor. Default is True.\n\n    Returns\n    -------\n    MXNet NDArray Tensor.\n    \"\"\"\n    if isinstance(in_tensor, int) or isinstance(in_tensor, float):\n        return in_tensor\n\n    if isinstance(in_tensor, tuple):\n        tensor = initializer(ctx=ctx, shape=in_tensor, dtype=dtype)\n    elif isinstance(in_tensor, list):\n        tensor = nd.array(in_tensor, ctx=ctx, dtype=dtype)\n    elif isinstance(in_tensor, np.ndarray):\n        tensor = nd.array(in_tensor)\n    elif isinstance(in_tensor, mx.np.ndarray):\n        tensor = in_tensor.as_nd_ndarray()\n    elif isinstance(in_tensor, nd.NDArray):\n        tensor = in_tensor.as_in_context(ctx)\n    else:\n        raise ValueError(\"Invalid input type for creating input tensor. Input can be tuple() of shape or Numpy Array or\"\n                         \" MXNet NDArray. Given - \", in_tensor)\n\n    if attach_grad:\n        tensor.attach_grad()\n\n    tensor.wait_to_read()\n    return tensor\n"
  },
  {
    "path": "benchmark/opperf/utils/op_registry_utils.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Utilities to interact with MXNet operator registry.\"\"\"\nfrom operator import itemgetter\nfrom mxnet import runtime\nimport mxnet as mx\n\nfrom benchmark.opperf.rules.default_params import DEFAULTS_INPUTS, DEFAULTS_INPUTS_LARGE_TENSOR, MX_OP_MODULE\n\n\ndef _select_ops(operator_names, filters=(\"_contrib\", \"_\"), merge_op_forward_backward=True):\n    \"\"\"From a given list of operators, filter out all operator names starting with given filters and prepares\n    a dictionary of operator with attributes - 'has_backward' and 'nd_op_handle = mxnet.ndarray.op'\n\n    By default, merge forward and backward operators for a given op into one operator and sets the attribute\n    'has_backward' for the operator.\n\n    By default, filter out all Contrib operators that starts with '_contrib' and internal operators that\n    starts with '_'.\n\n    Note - All deprecated operators are filtered out as well.\n\n    Parameters\n    ----------\n    operator_names: List[str]\n        List of operator names.\n    filters: Tuple(str)\n        Tuple of filters to apply on operator names.\n    merge_op_forward_backward: Boolean, Default - True\n        Merge forward and backward operators for a given op in to one op.\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\"}}\n    \"\"\"\n    mx_operators = {}\n    operators_with_backward = []\n\n    # Filter out deprecated operators\n    filters += (\"normal\", \"uniform\", \"Flatten\", \"contrib_CTCLoss\", \"Pad\", \"Cast\",\n                \"Pooling_v1\", \"Concat\", \"Reshape\", \"Convolution_v1\", \"SliceChannel\", \"Crop\",\n                \"crop\", \"onehot_encode\", \"batch_take\")\n\n    if merge_op_forward_backward:\n        filters += (\"_backward\",)\n\n    for cur_op_name in operator_names:\n        if not cur_op_name.startswith(filters):\n            mx_operators[cur_op_name] = {\"has_backward\": False,\n                                         \"nd_op_handle\": getattr(MX_OP_MODULE, cur_op_name)}\n\n        if cur_op_name.startswith(\"_backward_\"):\n            operators_with_backward.append(cur_op_name)\n\n    if merge_op_forward_backward:\n        # Identify all operators that can run backward.\n        for op_with_backward in operators_with_backward:\n            op_name = op_with_backward.split(\"_backward_\")[1]\n            if op_name in mx_operators:\n                mx_operators[op_name][\"has_backward\"] = True\n\n    return mx_operators\n\n\ndef _set_op_arguments(mx_operators):\n    \"\"\"Fetch and set operator arguments - nargs, arg_names, arg_types\n    \"\"\"\n    for op_name in mx_operators:\n        operator_arguments = mx.operator.get_operator_arguments(op_name)\n        mx_operators[op_name][\"params\"] = {\"narg\": operator_arguments.narg,\n                                           \"arg_names\": operator_arguments.names,\n                                           \"arg_types\": operator_arguments.types}\n\n\ndef _get_all_mxnet_operators():\n    # Step 1 - Get all registered op names and filter it\n    operator_names = mx.operator.get_all_registered_operators()\n    mx_operators = _select_ops(operator_names)\n\n    # Step 2 - Get all parameters for the operators\n    _set_op_arguments(mx_operators)\n    return mx_operators\n\n\ndef prepare_op_inputs(arg_params, arg_values):\n    inputs = []\n\n    for arg_value in arg_values:\n        inp = {}\n        for arg_name in arg_params[\"params\"][\"arg_names\"]:\n            if arg_name in arg_value:\n                inp[arg_name] = arg_value[arg_name]\n        inputs.append(inp)\n    return inputs\n\n\ndef prepare_op_inputs(op, arg_params, int64_tensor):\n    inputs = []\n\n    # 4d tensor is needed by following ops\n    ops_4d = ['depth_to_space', 'space_to_depth', 'pad']\n\n    # 3d tensor is needed by following ops\n    ops_3d = {'CTCLoss', 'ctc_loss'}\n\n    # For ops with args that need to change shape/value for different ops\n    custom_data = {'Activation', 'LeakyReLU', 'Softmax', 'BilinearSampler', 'GridGenerator', 'sample_multinomial', 'linalg_maketrian',\n                   'SpatialTransformer', 'col2im', 'GroupNorm', 'Dropout', 'FullyConnected',\n                   'BatchNorm',\n                   'L2Normalization', 'LayerNorm', 'InstanceNorm',\n                   'Embedding', 'Correlation', 'im2col', 'LRN', 'squeeze', 'fill_element_0index'}\n\n    custom_data_int64 = {'random_pdf_dirichlet', 'random_pdf_exponential', 'random_pdf_gamma',\n                         'random_pdf_generalized_negative_binomial', 'random_pdf_negative_binomial',\n                         'random_pdf_normal', 'random_pdf_poisson', 'random_pdf_uniform', 'sample_exponential',\n                         'sample_normal', 'sample_poisson', 'sample_uniform', 'sample_gamma',\n                         'sample_generalized_negative_binomial', 'sample_negative_binomial', 'CTCLoss',\n                         'ctc_loss', 'multi_lars'}\n\n    int_only = {'random_randint'}\n    float_only = {'log_softmax', 'softmax', 'softmin'}\n\n    # following ops need atleast 1 dim of size 1\n    ops_dim1 = ['broadcast_axis', 'broadcast_like', 'broadcast_to', 'broadcast_axes']\n\n    if int64_tensor == 'on':\n        default_inputs = DEFAULTS_INPUTS_LARGE_TENSOR\n        custom_data |= custom_data_int64\n    else:\n        default_inputs = DEFAULTS_INPUTS\n\n    # Prepare op to default input mapping\n    arg_values = {}\n    for arg_name, arg_type in zip(arg_params[\"params\"][\"arg_names\"],\n                                  arg_params[\"params\"][\"arg_types\"]):\n        # Due to lack of an internal API for fetching permissible dtype\n        # added a logic for using float only dtype as input for ops that take only floats\n        # same for randint (which is the only op that takes only int as input)\n        # rest all operators take int as well as float\n        if \"NDArray\" in arg_type:\n            if op in int_only and arg_name == \"dtype\":\n                arg_values[arg_name] = DEFAULTS_INPUTS[\"dtype_int\"]\n            elif (op.startswith(('random','sample')) or op in float_only) and arg_name == \"dtype\":\n                arg_values[arg_name] = DEFAULTS_INPUTS[\"dtype_float\"]\n            elif op == \"ravel_multi_index\":\n                arg_values[arg_name] = DEFAULTS_INPUTS[\"ravel_data\"]\n            elif op in custom_data and arg_name + \"_\" + op.lower() in DEFAULTS_INPUTS:\n                arg_values[arg_name] = DEFAULTS_INPUTS[arg_name + \"_\" + op.lower()]\n            elif arg_name + \"_nd\" in DEFAULTS_INPUTS:\n                arg_values[arg_name] = DEFAULTS_INPUTS[arg_name + \"_nd\"]\n            elif op in ops_3d and arg_name + \"_3d\" in DEFAULTS_INPUTS:\n                arg_values[arg_name] = DEFAULTS_INPUTS[arg_name + \"_3d\"]\n            elif op == 'softmax_cross_entropy':\n                arg_values[arg_name] = DEFAULTS_INPUTS[arg_name + \"_smce\"]\n            elif op in ops_4d and arg_name + \"_4d\" in DEFAULTS_INPUTS:\n                arg_values[arg_name] = DEFAULTS_INPUTS[arg_name + \"_4d\"]\n            elif op in ops_dim1 and arg_name + \"_dim1\" in DEFAULTS_INPUTS:\n                arg_values[arg_name] = DEFAULTS_INPUTS[arg_name + \"_dim1\"]\n            # default case\n            elif arg_name in DEFAULTS_INPUTS:\n                arg_values[arg_name] = DEFAULTS_INPUTS[arg_name]\n        else:\n            # arg_type is not NDArray\n            if op in int_only and arg_name == \"dtype\":\n                arg_values[arg_name] = DEFAULTS_INPUTS[\"dtype_int\"]\n            elif (op.startswith(('random','sample')) or op in float_only) and arg_name == \"dtype\":\n                arg_values[arg_name] = DEFAULTS_INPUTS[\"dtype_float\"]\n            elif op in custom_data and arg_name + \"_\" + op.lower() in DEFAULTS_INPUTS:\n                arg_values[arg_name] = DEFAULTS_INPUTS[arg_name + \"_\" + op.lower()]\n            elif op in ops_4d and arg_name + \"_4d\" in DEFAULTS_INPUTS:\n                arg_values[arg_name] = DEFAULTS_INPUTS[arg_name + \"_4d\"]\n            elif op in ops_dim1 and arg_name + \"_dim1\" in DEFAULTS_INPUTS:\n                arg_values[arg_name] = DEFAULTS_INPUTS[arg_name + \"_dim1\"]\n            #default case\n            elif arg_name in DEFAULTS_INPUTS:\n                arg_values[arg_name] = DEFAULTS_INPUTS[arg_name]\n\n    # Number of different inputs we want to use to test\n    # the operator\n    num_input_combinations = max([len(value) for value in arg_values.values()])\n\n    # Prepare key/value args for param to input value\n    for idx in range(num_input_combinations):\n        inp = {}\n        for arg_name in arg_params[\"params\"][\"arg_names\"]:\n            if arg_name in arg_values:\n                if len(arg_values[arg_name]) == num_input_combinations:\n                    inp[arg_name] = arg_values[arg_name][idx]\n                else:\n                    # This is required when we want to use a param same across all\n                    # input combination. Example: keeping low and high same for random sampling\n                    # operator for all different types of Tensor shape.\n                    inp[arg_name] = arg_values[arg_name][0]\n\n        inputs.append(inp)\n    return inputs\n\n\ndef get_all_unary_operators():\n    \"\"\"Gets all Unary operators registered with MXNet.\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n    \"\"\"\n    # Cast operators (cast & amp_cast are unary)\n    cast_ops = {'cast', 'amp_cast'}\n\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for unary broadcast operators\n    unary_broadcast_mx_operators = {}\n    for op_name, op_params in mx_operators.items():\n        if (op_params[\"params\"][\"narg\"] == 1 and \\\n                \"data\" in op_params[\"params\"][\"arg_names\"]) or \\\n                op_name in cast_ops:\n            unary_broadcast_mx_operators[op_name] = mx_operators[op_name]\n    return unary_broadcast_mx_operators\n\n\ndef get_all_broadcast_binary_operators():\n    \"\"\"Gets all binary broadcast operators registered with MXNet.\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n    \"\"\"\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for binary broadcast operators\n    binary_broadcast_mx_operators = {}\n    for op_name, op_params in mx_operators.items():\n        if op_name.startswith(\"broadcast_\") and op_params[\"params\"][\"narg\"] == 2 and \\\n                \"lhs\" in op_params[\"params\"][\"arg_names\"] and \\\n                \"rhs\" in op_params[\"params\"][\"arg_names\"]:\n            binary_broadcast_mx_operators[op_name] = mx_operators[op_name]\n    return binary_broadcast_mx_operators\n\n\ndef get_all_misc_binary_operators():\n    \"\"\"Gets all miscellaneous binary operators registered with MXNet.\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n    \"\"\"\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for miscellaneous binary operators\n    binary_misc_mx_operators = {}\n    for op_name, _ in mx_operators.items():\n        if \"choose_element_0index\" == op_name:\n            binary_misc_mx_operators[op_name] = mx_operators[op_name]\n        elif \"reshape_like\" == op_name:\n            binary_misc_mx_operators[op_name] = mx_operators[op_name]\n    return binary_misc_mx_operators\n\n\ndef get_all_elemen_wise_binary_operators():\n    \"\"\"Gets all binary elemen_wise operators registered with MXNet.\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n    \"\"\"\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for binary elemen_wise operators\n    binary_elemen_wise_mx_operators = {}\n    for op_name, op_params in mx_operators.items():\n        if op_name.startswith(\"elemwise_\") and op_params[\"params\"][\"narg\"] == 2 and \\\n                \"lhs\" in op_params[\"params\"][\"arg_names\"] and \\\n                \"rhs\" in op_params[\"params\"][\"arg_names\"]:\n            binary_elemen_wise_mx_operators[op_name] = mx_operators[op_name]\n        elif \"ElementWiseSum\" == op_name:\n            binary_elemen_wise_mx_operators[op_name] = mx_operators[op_name]\n    return binary_elemen_wise_mx_operators\n\n\ndef get_all_random_sampling_operators():\n    \"\"\"Gets all Random Sampling operators registered with MXNet.\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n    \"\"\"\n    # Additional Random Sampling ops which do not start with \"random_\" or \"sample_\"\n    additional_random_sampling_ops = {'GridGenerator', 'BilinearSampler'}\n\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for Random Sampling operators\n    random_sampling_mx_operators = {}\n    for op_name, _ in mx_operators.items():\n        if op_name.startswith((\"random_\", \"sample_\")) or op_name in additional_random_sampling_ops:\n            random_sampling_mx_operators[op_name] = mx_operators[op_name]\n    return random_sampling_mx_operators\n\n\ndef get_all_linalg_operators():\n    \"\"\"Gets all Linear Algebra operators registered with MXNet.\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n    \"\"\"\n    other_linalg_ops = {'moments'}\n\n    # Already tested linalg_potrf independently\n    independently_tested = {'linalg_potrf'}\n\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for Linear Algebra operators\n    linalg_mx_operators = {}\n    for op_name, _ in mx_operators.items():\n        if (op_name.startswith(\"linalg_\") and op_name not in independently_tested) or op_name in other_linalg_ops:\n            linalg_mx_operators[op_name] = mx_operators[op_name]\n    return linalg_mx_operators\n\n\ndef get_all_reduction_operators():\n    \"\"\"Gets all Reduction operators registered with MXNet.\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n    \"\"\"\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for Reduction operators\n    reduction_mx_operators = {}\n    for op_name, op_params in mx_operators.items():\n        if (op_params[\"params\"][\"narg\"] == 4 and \\\n                set([\"data\", \"axis\", \"exclude\", \"keepdims\"]).issubset(set(op_params[\"params\"][\"arg_names\"])) \\\n                or op_name == 'norm'):\n            reduction_mx_operators[op_name] = mx_operators[op_name]\n    return reduction_mx_operators\n\ndef get_all_nn_basic_operators():\n    \"\"\"Gets all NN basic operators registered with MXNet.\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n    \"\"\"\n    nn_basic_ops = ['FullyConnected', 'Dropout', 'BatchNorm',\n                    'L2Normalization',\n                    'LayerNorm', 'InstanceNorm', 'Embedding', 'Correlation', 'SpatialTransformer', 'im2col',\n                    'col2im', 'GroupNorm', 'LRN']\n\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for NN Basic operators\n    nn_basic_mx_operators = {}\n    for op_name, _ in mx_operators.items():\n         if op_name in nn_basic_ops:\n             nn_basic_mx_operators[op_name] = mx_operators[op_name]\n    return nn_basic_mx_operators\n\ndef get_all_nn_activation_operators():\n    \"\"\"Gets all NN Activation operators registered with MXNet.\n\n     Returns\n     -------\n     {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n     \"\"\"\n    nn_activation_ops = {'Softmax', 'SoftmaxActivation', 'softmin', 'Activation', 'LeakyReLU', 'hard_sigmoid', 'softmax', 'log_softmax'}\n\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for NN Activation operators\n    nn_activation_mx_operators = {}\n    for op_name, _ in mx_operators.items():\n         if op_name in nn_activation_ops:\n             nn_activation_mx_operators[op_name] = mx_operators[op_name]\n    return nn_activation_mx_operators\n\n\ndef get_all_optimizer_operators():\n    \"\"\"Gets all Optimizer operators registered with MXNet.\n\n     Returns\n     -------\n     {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n     \"\"\"\n    optimizer_ops = {'mp_sgd_update', 'signum_update', 'rmspropalex_update', 'ftml_update', 'rmsprop_update',\n                     'sgd_mom_update', 'signsgd_update', 'mp_sgd_mom_update', 'ftrl_update', 'sgd_update',\n                     'adam_update', 'mp_nag_mom_update', 'nag_mom_update', 'lamb_update_phase1',\n                     'lamb_update_phase2'}\n\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for Optimizer operators\n    optimizer_mx_operators = {}\n    for op_name, _ in mx_operators.items():\n        if op_name in optimizer_ops:\n            optimizer_mx_operators[op_name] = mx_operators[op_name]\n    return optimizer_mx_operators\n\ndef get_all_sorting_searching_operators():\n    \"\"\"Gets all Sorting and Searching operators registered with MXNet.\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n    \"\"\"\n    sort_search_ops = {'sort', 'argsort', 'argmax', 'argmin', 'topk'}\n\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for Sort and search operators\n    sort_search_mx_operators = {}\n    for op_name, _ in mx_operators.items():\n        if op_name in sort_search_ops:\n            sort_search_mx_operators[op_name] = mx_operators[op_name]\n    return sort_search_mx_operators\n\n\ndef get_all_rearrange_operators():\n    \"\"\"Gets all array rearrange operators registered with MXNet.\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n    \"\"\"\n    rearrange_ops = ['transpose', 'swapaxes', 'flip', 'depth_to_space',\n                     'space_to_depth', 'SwapAxis', 'reverse']\n\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for Array Rearrange operators\n    rearrange_mx_operators = {}\n    for op_name, _ in mx_operators.items():\n        if op_name in rearrange_ops:\n            rearrange_mx_operators[op_name] = mx_operators[op_name]\n    return rearrange_mx_operators\n\n\ndef get_remaining_miscellaneous_operators():\n    \"\"\"Gets remaining Miscellaneous operators registered with MXNet not covered by individual tests.\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n    \"\"\"\n    misc_ops = {'squeeze', 'all_finite', 'clip', 'multi_lars', 'SequenceReverse', 'SequenceLast', 'SequenceMask', 'cast_storage', 'cumsum', 'fill_element_0index'}\n\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for Miscellaneous operators\n    misc_mx_operators = {}\n    for op_name, _ in mx_operators.items():\n        if op_name in misc_ops:\n            misc_mx_operators[op_name] = mx_operators[op_name]\n    return misc_mx_operators\n\ndef get_all_indexing_routines():\n    \"\"\"Gets all indexing routines registered with MXNet.\n\n    # @ChaiBapchya unravel_index errors out on certain inputs\n    # tracked here https://github.com/apache/mxnet/issues/16771\n    # @ChaiBapchya scatter_nd errors with core dump\n    # tracked here https://github.com/apache/mxnet/issues/17480\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n    \"\"\"\n    indexing_routines = {'slice', 'slice_axis', 'slice_like', 'take', 'one_hot',\n                         'where', 'ravel_multi_index', 'gather_nd', 'pick'}\n\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for Indexing routines\n    indexing_mx_routines = {}\n    for op_name, _ in mx_operators.items():\n        if op_name in indexing_routines:\n            indexing_mx_routines[op_name] = mx_operators[op_name]\n    return indexing_mx_routines\n\n\ndef get_all_loss_operators():\n    \"\"\"Gets all Neural Network loss operators registered with MXNet.\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n    \"\"\"\n    loss_ops = {'smooth_l1', 'CTCLoss', 'ctc_loss', 'MakeLoss', 'softmax_cross_entropy'}\n\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for NN Loss operators\n    loss_mx_operators = {}\n    for op_name, _ in mx_operators.items():\n        if op_name in loss_ops:\n            loss_mx_operators[op_name] = mx_operators[op_name]\n    return loss_mx_operators\n\n\ndef get_all_shape_operators():\n    \"\"\"Gets all array shape manipulation operators registered with MXNet.\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n    \"\"\"\n    shape_ops = ['split', 'SliceChannel', 'diag', 'reshape',\n                     'reshape_like', 'size_array', 'shape_array']\n\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for Array Shape Manipulation operators\n    shape_mx_operators = {}\n    for op_name, op_params in mx_operators.items():\n        if op_name in shape_ops:\n            shape_mx_operators[op_name] = mx_operators[op_name]\n    return shape_mx_operators\n\n\ndef get_all_expanding_operators():\n    \"\"\"Gets all array expanding operators registered with MXNet.\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n    \"\"\"\n    expanding_ops = ['broadcast_axes', 'broadcast_axis', 'broadcast_to', 'broadcast_like',\n                     'repeat', 'tile', 'pad', 'expand_dims']\n\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for Array Expanding operators\n    expanding_mx_operators = {}\n    for op_name, op_params in mx_operators.items():\n        if op_name in expanding_ops:\n            expanding_mx_operators[op_name] = mx_operators[op_name]\n    return expanding_mx_operators\n\n\ndef get_all_rounding_operators():\n    \"\"\"Gets all array rounding operators registered with MXNet.\n\n    Returns\n    -------\n    {\"operator_name\": {\"has_backward\", \"nd_op_handle\", \"params\"}}\n    \"\"\"\n    rounding_ops = ['round', 'rint', 'fix', 'floor',\n                     'ceil', 'trunc']\n\n    # Get all mxnet operators\n    mx_operators = _get_all_mxnet_operators()\n\n    # Filter for Array Rounding operators\n    rounding_mx_operators = {}\n    for op_name, op_params in mx_operators.items():\n        if op_name in rounding_ops:\n            rounding_mx_operators[op_name] = mx_operators[op_name]\n    return rounding_mx_operators\n\n\ndef get_operators_with_no_benchmark(operators_with_benchmark):\n    \"\"\"Gets all MXNet operators with not benchmark.\n\n    Retrieve all operators registered with MXNet and prepares a list of operators that are not part of given\n    operators with benchmark list.\n\n    Parameters\n    ----------\n    operators_with_benchmark: list[Str]\n        List of operator names that has benchmarks\n\n    Returns\n    -------\n    list[Str]\n        List of operator names that is registered with MXNet but has no benchmarks.\n    \"\"\"\n    all_mxnet_operators = _get_all_mxnet_operators().keys()\n    return list(set(all_mxnet_operators) - set(operators_with_benchmark))\n\n\ndef get_current_runtime_features():\n    \"\"\"Get all current runtime time flags/configuration for MXNet.\n\n    Returns\n    -------\n    Map of current runtime features such as compile flags used by MXNet.\n        Example: {'runtime_features': {'OPENCV' : '✔ OPENCV', 'CUDA': '✖ CUDA'}}\n    \"\"\"\n    features = runtime.Features()\n    runtime_features = {}\n    for feature, config in sorted(features.items(), key=itemgetter(0)):\n        runtime_features[feature] = config\n\n    return {'runtime_features': runtime_features}\n"
  },
  {
    "path": "benchmark/opperf/utils/profiler_utils.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport time\nimport functools\nimport numpy as np\n\nfrom .common_utils import merge_map_list\nfrom mxnet import profiler\n\n\"\"\"\nTODO: Below we are using logic of parsing the MXNet profiler output string to\nfetch the benchmark results. Note that this is a temporary solution till we add\na new utility API into MXNet profiler to get_summary(), reset(). All the below\nparsing logic should be removed once these read APIs are available in Profiler.\n\n\"\"\"\n\n\ndef _get_memory_profile(memory_profile_results):\n    memory_profile = {}\n    for line in memory_profile_results:\n        if line.startswith(\"Memory:\"):\n            device_id = line.split()[1]\n            avg_time_memory_alloc = float(line.split()[-1])\n            memory_profile[\"max_storage_mem_alloc_\" + device_id] = avg_time_memory_alloc\n\n    return memory_profile\n\n\ndef _get_operator_profile(operator_name, operator_profile_results):\n    operator_profile = {}\n\n    # alias map : dictionary of the form {\"alias\" : \"registered_name\"}\n    # allows to retrieve alias operator profile from the profiler results\n    alias_map = {\"broadcast_plus\": \"broadcast_add\", \"broadcast_minus\": \"broadcast_sub\", \"flatten\": \"Flatten\", \"max_axis\": \"max\", \"Custom\": \"CustomAddOne\",\n                 \"swapaxes\": \"SwapAxis\", \"flip\": \"reverse\", \"reshape\": \"Reshape\", \"crop\": \"slice\", \"sum_axis\": \"sum\", \"min_axis\": \"min\", \"ctc_loss\": \"CTCLoss\",\n                 \"fill_element_0index\": \"TernaryOp\", \"identity\": \"_copy\", \"ElementWiseSum\": \"add_n\", \"choose_element_0index\": \"pick\", \"stop_gradient\": \"BlockGrad\",\n                 \"broadcast_axes\": \"broadcast_axis\"}\n\n    op_name = None\n\n    if operator_name in alias_map:\n        op_name = alias_map[operator_name]\n    else:\n        op_name = operator_name\n\n    # Variables to store forward/backward performance results\n    forward_res, backward_res = None, None\n\n    for line in operator_profile_results:\n        if op_name in line or op_name[:3] + \" \" in line:\n            operation = line.split()[0]\n            operation_avg_time = float(line.split()[-1])\n            if \"_backward\" in operation:\n                backward_res = operation_avg_time\n            else:\n                forward_res = operation_avg_time\n\n    # Add forward and backward performance results to the dict in the correct order\n    if forward_res:\n        operator_profile[\"avg_time_forward_\" + operator_name] = forward_res\n\n    if backward_res:\n        operator_profile[\"avg_time_backward_\" + operator_name] = backward_res\n\n    return operator_profile\n\n\ndef parse_profiler_dump(operator_name, profiler_dump):\n    \"\"\"Parse the MXNet profiler dump output, fetch Memory profile results and\n    Operator compute profiler results.\n\n    Parameters\n    ----------\n    profiler_dump: string\n        MXNet profiler output from mx.profiler.dumps() API.\n\n    Returns\n    -------\n    map, Memory and Compute profiler results.\n\n    \"\"\"\n    if not profiler_dump:\n        raise AssertionError(\"Invalid MXNet profiler output provided to parse!\")\n\n    \"\"\"\n    MXNet profiler output from mx.profiler.dumps() API looks like below. This function parses\n    this string profiler output to fetch Memory and Compute metrics.\n\n    Profile Statistics.\n    Note that counter items are counter values and not time units.\n    Device Storage\n    =================\n    Name                          Total Count        Time (ms)    Min Time (ms)    Max Time (ms)    Avg Time (ms)\n    ----                          -----------        ---------    -------------    -------------    -------------\n    Memory: cpu/0                         100     2097152.0000     1681915.8750     2097152.0000      207618.0469\n\n    MXNET_C_API\n    =================\n    Name                          Total Count        Time (ms)    Min Time (ms)    Max Time (ms)    Avg Time (ms)\n    ----                          -----------        ---------    -------------    -------------    -------------\n    MXNDArrayFree                          49           1.1220           0.0170           0.0360           0.0229\n    MXAutogradBackwardEx                   50          11.5460           0.1980           0.3360           0.2309\n    MXNet C API Calls                     399           1.9990           1.6010           1.9990           0.1990\n    MXImperativeInvoke                     50           4.4810           0.0700           0.1330           0.0896\n    MXNDArrayWaitAll                       50         769.0570          14.0200          24.5030          15.3811\n    MXAutogradSetIsTraining               100           0.0190           0.0000           0.0010           0.0002\n    MXAutogradSetIsRecording              100           0.0400           0.0000           0.0010           0.0004\n    MXNet C API Concurrency               798           0.0000           0.0000           0.0010           0.0005\n\n    operator\n    =================\n    Name                          Total Count        Time (ms)    Min Time (ms)    Max Time (ms)    Avg Time (ms)\n    ----                          -----------        ---------    -------------    -------------    -------------\n    DeleteVariable                        196           1.4490           0.0040           0.0250           0.0074\n    _backward_broadcast_add               100         521.2320           4.8070           8.5970           5.2123\n    SetValueOp                            100         645.8060           5.8820          10.0380           6.4581\n    broadcast_add                         100         394.8910           3.5230           5.8790           3.9489\n    \"\"\"\n\n    # String Patterns to look out for when parsing\n    memory_profile_result_start = \"Device Storage\"  # Helps identify start of Memory profile\n    c_api_profile_result_start = \"MXNET_C_API\"  # Helps identify end of Memory profile\n\n    if operator_name == \"Custom\":\n        operator_profile_result_start = \"Custom Operator\"  # Helps identify start of Custom Operator profile\n    else:\n        operator_profile_result_start = \"operator\"  # Helps identify start of Operator profile\n\n    memory_profile_results = []\n    operator_profile_results = []\n\n    # Parse lines corresponding to Memory and Computation profiling\n    read_memory_profile = False\n    read_operator_profile = False\n    for line in profiler_dump.splitlines():\n        if line.startswith(memory_profile_result_start):\n            read_memory_profile = True\n        elif line.startswith(operator_profile_result_start):\n            read_operator_profile = True\n        elif line.startswith(c_api_profile_result_start):\n            read_memory_profile = False\n\n        if read_memory_profile:\n            memory_profile_results.append(line)\n        elif read_operator_profile:\n            operator_profile_results.append(line)\n\n    # Prepare results\n    memory_profile = _get_memory_profile(memory_profile_results)\n    operator_profile = _get_operator_profile(operator_name, operator_profile_results)\n\n    return merge_map_list([memory_profile, operator_profile])\n\n\ndef cpp_profile(func):\n    \"\"\"Decorator for profiling MXNet operation.\n    Uses MXNet profiler to collect metrics on memory usage and execution time\n    of the operation.\n\n    Parameters\n    ----------\n    func:\n        Operation to be executed and timed.\n\n    Returns\n    -------\n    res, profiler output. res being result returned after operator execution.\n    profiler output is a dictionary with summary of operation execution.\n    Example output : { \"add\": [{\"avg_time_mem_alloc_cpu/0\": 207618.0469,\n                                \"avg_time_forward_broadcast_add\": 4.204,\n                                \"avg_time_backward_broadcast_add\": 5.6288,\n                                \"inputs\": {\n                                            \"lhs\": [1024, 1024],\n                                            \"rhs\": [1024,1024]\n                                          }]\n                     }\n    \"\"\"\n\n    @functools.wraps(func)\n    def cpp_profile_it(*args, **kwargs):\n        # Profile the operation\n        profiler.set_config(profile_all=True, aggregate_stats=True)\n        profiler.set_state('run')\n        res = func(*args, **kwargs)\n        profiler.set_state('stop')\n\n        # Prepare the results\n        profiler_dump = profiler.dumps(reset=True)\n\n        # args[0] is assumed to be operator name, if not found check for block name.\n        # NOTE: This parameter should be removed when we get away from parsing\n        # profiler output and start using new profiler APIs - get_summary(), reset()\n        if len(args) > 0:\n            operator_name = args[0].__name__\n        elif 'block' in kwargs:\n            operator_name = kwargs['block']._op_name\n        else:\n            raise ValueError(\"Unable to identify operator name to extract profiler output!\")\n\n        # Get the MXNet profile output\n        profiler_output = parse_profiler_dump(operator_name, profiler_dump)\n        return res, profiler_output\n\n    return cpp_profile_it\n\n\ndef python_profile(func):\n    \"\"\"Decorator for profiling MXNet operation.\n    Uses Python's time module to collect execution time information\n    of the operation.\n\n    Parameters\n    ----------\n    func:\n        Operation to be executed and timed.\n\n    Returns\n    -------\n    res, timing output. res being result returned after operator execution.\n    profiler output is a dictionary with summary of operation execution.\n    Example output : { \"add\": [{\"avg_time_add\": 0.4053089120425284,\n                                'p50_time_add': 16.761042876169086,\n                                'p90_time_add': 18.081666342914108,\n                                'p99_time_add': 19.060144051909447,\n                                \"inputs\": {\n                                    \"lhs\": [1024, 1024],\n                                    \"rhs\": [1024,1024]\n                                }]\n                     }\n    \"\"\"\n\n    @functools.wraps(func)\n    def python_profile_it(*args, **kwargs):\n        runs = args[1]\n        modified_args = (args[0], 1)\n        times = []\n\n        for _ in range(runs):\n            start_time = time.perf_counter()    # 1\n            res = func(*modified_args, **kwargs)\n            end_time = time.perf_counter()      # 2\n            run_time = (end_time - start_time)*1000    # 3\n            times.append(run_time)\n\n        # NOTE : same as cpp_profile_it\n        if len(args) > 0:\n            operator_name = args[0].__name__\n        elif 'block' in kwargs:\n            operator_name = kwargs['block']._op_name\n        else:\n            raise ValueError(\"Unable to identify operator name to extract profiler output!\")\n\n        avg_run_time = np.mean(times)\n        p50_run_time = np.percentile(times, 50)\n        p90_run_time = np.percentile(times, 90)\n        p99_run_time = np.percentile(times, 99)\n\n        profiler_output = {'avg_time_'+str(operator_name): avg_run_time,\n                           'p50_time_'+str(operator_name): p50_run_time,\n                           'p90_time_'+str(operator_name): p90_run_time,\n                           'p99_time_'+str(operator_name): p99_run_time,\n                           }\n        return res, profiler_output\n    return python_profile_it\n"
  },
  {
    "path": "benchmark/python/control_flow/rnn.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nfrom __future__ import print_function\nfrom six.moves import range\n\nimport argparse\nimport subprocess\nfrom itertools import product\nfrom time import time\n\nimport mxnet as mx\nimport numpy as onp\nfrom mxnet import gluon, np, npx\n\n\n_parser = argparse.ArgumentParser(description='Benchmark foreach and while_loop on RNN tasks.')\n_parser.add_argument('--benchmark', choices=[\"foreach\", \"while_loop\"], required=True)\n_parser.add_argument('--warmup_rounds', type=int, default=20)\n_parser.add_argument('--test_rounds', type=int, default=100)\n_parser.add_argument('--gpu', type=bool, default=False)\nargs = _parser.parse_args()\n\n\nclass ForeachRNN(gluon.HybridBlock):\n    def __init__(self, cell, length):\n        super(ForeachRNN, self).__init__()\n        self.length = length\n        self.cell = cell\n\n    def forward(self, inputs, states):\n        out, states = npx.foreach(self.cell, inputs, states)\n        return out\n\n\nclass WhileRNN(gluon.HybridBlock):\n    def __init__(self, cell, length):\n        super(WhileRNN, self).__init__()\n        self.length = length\n        self.cell = cell\n\n    def forward(self, inputs, states):\n        def _func(*states):\n            i = states[0]\n            s = states[1: ]\n            data = np.squeeze(np.take(inputs, i), axis=0)\n            out, new_s = self.cell(data, s)\n            new_s = [i + 1] + new_s\n            return out, new_s\n        out, states = npx.while_loop(\n            cond=lambda i, *_: i < self.length,\n            func=_func,\n            loop_vars=states,\n            max_iterations=self.length,\n        )\n        return out\n\n\ndef _zeros(shape, ctx):\n    return mx.np.zeros(shape=shape, ctx=ctx)\n\n\ndef _array(shape, ctx):\n    return mx.np.random.normal(loc=0.0, scale=1.0, size=shape, ctx=ctx)\n\n\ndef _get_gpus():\n    return range(mx.util.get_gpu_count())\n\ndef run_benchmark(cell_type, ctx, seq_len, batch_size, hidden_dim):\n    obj = {\"foreach\": ForeachRNN, \"while_loop\": WhileRNN}[args.benchmark]\n    inputs = _array((seq_len, batch_size, hidden_dim), ctx)\n    states = [_array((batch_size, hidden_dim), ctx) for _ in cell_type(0).state_info()]\n    if args.benchmark == \"while_loop\":\n        states.insert(0, _zeros((1, ), ctx))\n\n    for is_train, is_hyb_cell, is_hyb_layer in product([True, False], [False, True], [False, True]):\n        cell = cell_type(hidden_dim)\n        cell.infer_shape(0, inputs, False)\n        if is_hyb_cell:\n            cell.hybridize(static_alloc=True)\n        layer = obj(cell, seq_len)\n        layer.initialize(ctx=ctx)\n        if is_hyb_layer:\n            layer.hybridize(static_alloc=True)\n        print(\n            f\"is_train = {repr(is_train)}, hybridize_cell = {repr(is_hyb_cell)}, hybridize_layer = {repr(is_hyb_layer)}\")\n        times = []\n        for _ in range(args.warmup_rounds + args.test_rounds):\n            tick = time()\n            if not is_train:\n                res = layer(inputs, states)\n            else:\n                with mx.autograd.record():\n                    res = layer(inputs, states)\n            if is_train:\n                res.backward()\n            mx.npx.waitall()\n            tock = time()\n            times.append((tock - tick) * 1000.0)\n        times = times[args.warmup_rounds: ]\n        print(f\"Time used: mean = {onp.mean(times):.3f} ms, std = {onp.std(times):.3f} ms\")\n\n\ndef main():\n    # testing configurations\n    cell_types = [gluon.rnn.RNNCell,\n                  gluon.rnn.GRUCell,\n                  gluon.rnn.LSTMCell]\n    ctxs = [mx.cpu(0)]\n    if args.gpu:\n        ctxs = ctxs + [mx.gpu(i) for i in _get_gpus()]\n    seq_lens = [100]\n    batch_sizes = [1, 32]\n    hidden_dims = [512]\n    print(\"--------------------------------------\")\n    print(\"Benchmarking\", args.benchmark)\n    for cell_type, ctx, seq_len, batch_size, hidden_dim in product(  \\\n        cell_types, ctxs, seq_lens, batch_sizes, hidden_dims):\n        print(\"--------------------------------------\")\n        print(f\"cell: {cell_type.__name__}  ctx: {str(ctx)}  length: {seq_len}  batch size: {batch_size} dim: {hidden_dim}\")\n        run_benchmark(cell_type, ctx, seq_len, batch_size, hidden_dim)\n\n\nif __name__ == \"__main__\":\n    main()\n"
  },
  {
    "path": "benchmark/python/dnnl/fc_add.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport time\nimport gc\nimport sys\nimport mxnet as mx\nfrom mxnet.gluon import nn\nfrom mxnet.contrib import quantization\n\n#shape, num_hidden:\nsizes = [\n    ((  1, 224),   512),\n    ((  1, 224),  4096),\n    (( 16, 1024), 1024),\n    (( 32, 4096), 1024),\n    (( 32, 4096), 4096),\n    ((512,  512), 4096)]\n\nrounds = 1000\nwarmup = 10\n\ntest_header = \"--no_test_header\" not in sys.argv\ntable_header = \"--no_table_header\" not in sys.argv\ntable_left_colums = \"--no_size_column\" not in sys.argv\ndump_graph = \"--dump_graph\" in sys.argv\n\ndef dump_graph_fn(net, postfix):\n    if dump_graph:\n        net.export(\"/tmp/fc_add_\" + postfix)\n\ndef operator_string(elemwise_add):\n    return 'elemwise_add' if elemwise_add else 'npi_add'\n\ndef print_header(header):\n    print(\"\\n\")\n    print(header if test_header else \"\", \"\\n\")\n    if table_header:\n        if table_left_colums:\n            print(\"|    Shape    | Hidden | Mean [ms] |\" )\n            print(\"|------------:|-------:|----------:|\" )\n        else:\n            print(\" Mean [ms] |\" )\n            print(\"----------:|\" )\n\ndef print_value(shape, hidden, mean):\n    if table_left_colums:\n        print(f\"| ({shape[0]:4},{shape[1]:4}) | {hidden:6} | {mean:9.3f} |\")\n    else:\n        print(f\" {mean:9.3f} |\")\n\n\ndef measure(net, data0, data1, data2, shape, nhid):\n    mx.nd.waitall()\n    gc.collect()\n    gc.disable()\n    for i in range(rounds + warmup):\n        if i == warmup:\n            start_time = time.time()\n        o = net(data0, data1, data2)\n        o.wait_to_read()\n    end_time = time.time()\n    run_time = (end_time - start_time)\n    print_value(shape, nhid, 1000 * run_time / rounds)\n    gc.enable()\n\n\nclass FCWithSum(nn.HybridBlock):\n    def __init__(self, num_in, num_hidden, elemwise_add, **kwargs):\n        super(FCWithSum, self).__init__(**kwargs)\n        self.fc0 = nn.Dense(units=num_hidden, in_units=num_in)\n        self.fc1 = nn.Dense(units=num_hidden)\n        self.elemwise_add = elemwise_add\n\n    def forward(self, data0, data1, data2):\n        _fc0 = self.fc0(data0)\n        _fc1 = self.fc1(data1)\n        if  self.elemwise_add:\n            _sum0 = mx.nd.elemwise_add(data2.as_nd_ndarray(), _fc0.as_nd_ndarray()).as_np_ndarray()\n            _sum1 = mx.nd.elemwise_add(_fc1.as_nd_ndarray(), _sum0.as_nd_ndarray()).as_np_ndarray()\n        else:\n            _sum0 = data2 + _fc0\n            _sum1 = _fc1 + _sum0\n        return _sum1\n\ndef benchmark_float(elemwise_add, broadcast=False):\n    header = operator_string(elemwise_add) + ', float' + (' , broadcast' if broadcast else \"\")\n    print_header(header)\n    for shape, nhid in sizes:\n        net = FCWithSum(shape[1], nhid, elemwise_add)\n        net.initialize()\n        net.hybridize(static_alloc=True, static_shape=True)\n        data0 = mx.np.random.uniform(size=shape, low=-1.0, high=1.0)\n        data1 = mx.np.random.uniform(size=shape, low=-1.0, high=1.0)\n        shape2 = (shape[0], nhid)\n        if broadcast and not elemwise_add:\n            # broadcast is allowed only for npi_add version\n            shape2 = (1, 1)\n        data2 = mx.np.random.uniform(size=shape2, low=-1.0, high=1.0)\n        net.optimize_for(data0, data1, data2, backend='ONEDNN')\n        measure(net, data0, data1, data2, shape, nhid)\n    dump_graph_fn(net, operator_string(elemwise_add) + '_float')\n\nclass CalibIter(mx.io.DataIter):\n    def __init__(self, batch, data_shape, batch_size):\n        super(CalibIter, self).__init__(batch_size)\n        self.label_shape = (batch_size,)\n        self.data_shape = data_shape\n        if isinstance(data_shape, tuple):\n            self.provide_data = [('data', data_shape)]\n        else:\n            self.provide_data = data_shape\n        self.provide_label = []\n        self.batch = batch\n    def __iter__(self):\n        yield self.batch\n\ndef benchmark_int8(quantize_mode, quantize_granularity, elemwise_add, broadcast = False):\n    header = operator_string(elemwise_add) + ', mode = ' + quantize_mode + \\\n             ', granularity = ' + quantize_granularity + (' , broadcast' if broadcast else \"\")\n    print_header(header)\n    for shape, nhid in sizes:\n        net = FCWithSum(shape[1], nhid, elemwise_add)\n        net.initialize()\n        net.hybridize(static_alloc=True, static_shape=True)\n        data0 = mx.np.random.uniform(size=shape, low=-1.0, high=1.0)\n        data1 = mx.np.random.uniform(size=shape, low=-1.0, high=1.0)\n        shape2 = (shape[0], nhid)\n        if broadcast and not elemwise_add:\n            # broadcast is allowed only for npi_add\n            shape2 = (shape[0], 1)\n        data2 = mx.np.random.uniform(size=shape2, low=-1.0, high=1.0)\n        data = mx.gluon.data.ArrayDataset(data0, data1, data2)\n        calib_data = mx.gluon.data.DataLoader(data, batch_size=1)\n        net = quantization.quantize_net(net,\n                                        device=mx.cpu(),\n                                        exclude_layers=None,\n                                        exclude_operators=None,\n                                        calib_mode='naive',\n                                        calib_data=calib_data,\n                                        num_calib_batches=1,\n                                        quantize_mode=quantize_mode,\n                                        quantize_granularity=quantize_granularity\n                                        )\n        net.hybridize(static_alloc=True, static_shape=True)\n        measure(net, data0, data1, data2, shape, nhid)\n    dump_graph_fn(net, operator_string(elemwise_add) + \\\n                    '_' + str(quantize_mode) + '_' + str(quantize_granularity))\n\nfor elemwise_add in [True, False]:\n    benchmark_float(elemwise_add)\n\nfor quantize_mode in ['smart', 'full']:\n    for quantize_granularity in ['tensor-wise', 'channel-wise']:\n        for elemwise_add in [True, False]:\n            benchmark_int8(quantize_mode, quantize_granularity, elemwise_add)\n\n# Benchmark FC + npi_add with broadcasted input\nbenchmark_float(False, True)\n\n# Benchmark quantized FC + npi_add with broadcasted input\nfor quantize_mode in ['smart', 'full']:\n    for quantize_granularity in ['tensor-wise', 'channel-wise']:\n        benchmark_int8(quantize_mode, quantize_granularity, False, True)\n"
  },
  {
    "path": "benchmark/python/dnnl/run.sh",
    "content": "#!/bin/bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Script for running python benchmark with properly setting OMP prarameters for it\n\ncheck_parametrs() {\n \tif [ \"$#\" -eq 0 ] ; then\n\t\techo \"Please give python script to run as parameter.\"\n\t\techo \"Optionally you can give number of threads to use and python scripts parameters:\"\n\t\techo \"    `basename \"$0\"`  [num_threads] python_script [python script parameters]\"\n\t\texit\n\tfi\n}\n\ncheck_parametrs $@\n\nNUM_SOCKET=`lscpu | grep 'Socket(s)' | awk '{print $NF}'`\nCORES_PER_SOCKET=`lscpu | grep 'Core(s) per socket' | awk '{print $NF}'`\nNUM_CORES=$((CORES_PER_SOCKET * NUM_SOCKET))\n\ninteger_reg='^[0-9]+$'\nif [[ $1 =~ $integer_reg ]] ; then\n\tif (($1 > $NUM_CORES)); then\n\t\techo >&2\n\t\techo \"WARNING: given number of threads = $1\" \\\n\t\t\t\" is greater than number of physical cores = $NUM_CORES.\" >&2\n\t\techo >&2\n\tfi\n\tNUM_CORES=$1\n\tshift\n\tcheck_parametrs $@\nfi\n\nCORES={0}:${NUM_CORES}:1\n\nINSTRUCTION=\"OMP_NUM_THREADS=${NUM_CORES} OMP_PROC_BIND=TRUE OMP_PLACES=${CORES} python3 -u $@\"\necho $INSTRUCTION >&2\neval $INSTRUCTION\n"
  },
  {
    "path": "benchmark/python/dnnl/run_per_thread.sh",
    "content": "#!/bin/bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Script for running python benchmark against number of used OMP threads\n\n\nhelp_and_exit() {\n\techo \"Usage:\"\n\techo \"    `basename \"$0\"`  [start_num_threads step_num_threads end_num_threads] python_script [python script parameters]\"\n\techo \"Number of threads range parameters and python script are optional.\"\n\texit\n}\n\nif [ \"$#\" -eq 0 ] ; then\n\thelp_and_exit\nfi\n\nNUM_SOCKET=`lscpu | grep 'Socket(s)' | awk '{print $NF}'`\nCORES_PER_SOCKET=`lscpu | grep 'Core(s) per socket' | awk '{print $NF}'`\nNUM_CORES=$((CORES_PER_SOCKET * NUM_SOCKET))\n\nNT_START=1\nNT_STEP=1\nNT_END=$NUM_CORES\n\ninteger_reg='^[0-9]+$'\nsigned_integer_reg='^[+-]*[0-9]+$'\nif [[ $1 =~ $integer_reg ]] ; then\n\tif [[ $2 =~ $signed_integer_reg ]] && [[ $3 =~ $integer_reg ]]; then\n\t\tNT_START=$1\n\t\tNT_STEP=$2\n\t\tNT_END=$3\n\t\tshift 3\n\t\tif [ \"$#\" -eq 0 ] ; then\n\t\t\thelp_and_exit\n\t\tfi\n\telse\n\t\techo \"Provide 3 numbers for threads range: start, step and the end.\"\n\t\thelp_and_exit\n\tfi\nfi\n\nNT_SEQUENCE=`seq $NT_START $NT_STEP $NT_END`\nif [ -z \"$NT_SEQUENCE\" ]; then\n\techo \"Given threads range produce empy sequence.\"\n\thelp_and_exit\nelse\n\techo \"Start python script $1 for following number of threads:\"  >&2\n\techo $NT_SEQUENCE  >&2\nfi\n\nRUN_SCRIPT=`dirname \"$0\"`/run.sh\nfor NT in $NT_SEQUENCE;\ndo\n\tTMP_FILE=/tmp/_result_${NT}.txt\n\techo  1>${TMP_FILE}\n\tif [[ $NT -eq $NT_START ]]; then\n\t\techo \"NUM_THREADS = $NT\" 1>>${TMP_FILE}\n\t\t$RUN_SCRIPT $NT $@ 1>>${TMP_FILE}\n\telse\n\t\techo \" $NT\" 1>>${TMP_FILE}\n\t\t$RUN_SCRIPT $NT $@ --no_size_column --no_test_header 1>>${TMP_FILE}\n\tfi\n\tTMP_FILES+=\" ${TMP_FILE}\"\ndone\npaste -d \"\" ${TMP_FILES}\n"
  },
  {
    "path": "benchmark/python/einsum/benchmark_einsum.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport time\nimport mxnet as mx\nfrom mxnet import np, npx\n\ndef measure_cost(repeat, func_name, *args, **kwargs):\n    \"\"\"Measure time cost of running a function\n    \"\"\"\n    mx.nd.waitall()\n    start = time.time()\n    for _ in range(repeat):\n        func_name(*args, **kwargs)\n    mx.nd.waitall()\n    end = time.time()\n    diff = end - start\n    return diff / repeat\n\n\ndef test_np_einsum():\n    print(\"Path optimization test:\")\n    # Basic einsum\n    a = np.ones(64).reshape(2,4,8)\n    args = ['ijk,ilm,njm,nlk,abc->', a, a, a, a, a]\n    cost = measure_cost(500, np.einsum, *args)\n    print(f\"Basic einsum: {cost * 1000} ms\")\n\n    # Sub-optimal einsum\n    # cost = measure_cost(500, np.einsum, *args, optimize='optimal')\n    # print(\"Optimal einsum: {} ms\".format(cost * 1000))\n\n    # Greedy einsum\n    cost = measure_cost(500, np.einsum, *args, optimize=True)\n    print(f\"Greedy einsum: {cost * 1000} ms\")\n\n    print(\"RNN Use Case:\")\n    a = np.random.uniform(0, 1, size=(64, 128, 512))\n    b = np.random.uniform(0, 1, size=(128, 512, 2, 2))\n    args = ['bij, ijkl->bkl', a, b]\n    cost = measure_cost(2, np.einsum, *args, optimize=True)\n    print(f'Greedy einsum: {cost * 1000} ms')\n    cost = measure_cost(2, np.einsum, *args)\n    print(f'Basic einsum: {cost * 1000} ms')\n\n    print('Inner Product:')\n    a = np.ones(6000000)\n    b = np.ones(6000000)\n    args = [a, b]\n    cost = measure_cost(50, np.tensordot, *args, axes=([0],[0]))\n    print(f'Tensordot: {cost * 1000} ms')\n    args = ['i, i', a, b]\n    cost = measure_cost(50, np.einsum, *args, optimize=True)\n    print(f'Greedy einsum: {cost * 1000} ms')\n    cost = measure_cost(50, np.einsum, *args)\n    print(f'Basic einsum: {cost * 1000} ms')\n\n    print('Matrix Product:')\n    a = np.ones(600000).reshape(200, 3000)\n    b = np.ones(600000).reshape(3000, 200)\n    args = [a, b]\n    cost = measure_cost(50, np.tensordot, *args, axes=([1],[0]))\n    print(f'Tensordot: {cost * 1000} ms')\n    args = ['ij, jk', a, b]\n    cost = measure_cost(50, np.einsum, *args, optimize=True)\n    print(f'Greedy einsum: {cost * 1000} ms')\n    cost = measure_cost(50, np.einsum, *args)\n    print(f'Basic einsum: {cost * 1000} ms')\n\n\nif __name__ == \"__main__\":\n    npx.set_np(dtype=False)\n    test_np_einsum()\n"
  },
  {
    "path": "benchmark/python/ffi/benchmark_ffi.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\nimport timeit\nimport itertools\nimport argparse\nimport os\n\nclass OpArgMngr(object):\n    \"\"\"Operator argument manager for storing operator workloads.\"\"\"\n    args = {}\n\n    @staticmethod\n    def add_workload(funcname, *args, **kwargs):\n        if \"_specifier\" not in kwargs:\n            _specifier = funcname\n        else:\n            _specifier = kwargs[\"_specififer\"]\n            del kwargs[\"_specififer\"]\n        if _specifier in OpArgMngr.args:\n            raise ValueError(f\"duplicate {_specifier}\")\n        OpArgMngr.args[_specifier] = {'args': args, 'kwargs': kwargs, 'funcname': funcname}\n\n\ndef generate_workloads():\n    array_pool = {}\n    shapes = []\n    for ndim in range(4):\n        shapes.extend(list(itertools.product(range(4), repeat=ndim)))\n    for shape in shapes:\n        name = 'x'.join(str(i) for i in shape)\n        if name in array_pool:\n            raise ValueError(f\"duplicate array {name}\")\n        array_pool[name] = dnp.ones(shape)\n    return array_pool\n\n\ndef prepare_workloads():\n    pool = generate_workloads()\n    OpArgMngr.add_workload(\"zeros\", (2, 2))\n    OpArgMngr.add_workload(\"full\", (2, 2), 10)\n    OpArgMngr.add_workload(\"identity\", 3)\n    OpArgMngr.add_workload(\"ones\", (2, 2))\n    OpArgMngr.add_workload(\"einsum\", \"ii\", pool['2x2'], optimize=False)\n    OpArgMngr.add_workload(\"unique\", pool['1'], return_index=True, return_inverse=True, return_counts=True, axis=-1)\n    OpArgMngr.add_workload(\"dstack\", (pool['2x1'], pool['2x1'], pool['2x1'], pool['2x1']))\n    OpArgMngr.add_workload(\"polyval\", dnp.arange(10), pool['2x2'])\n    OpArgMngr.add_workload(\"ediff1d\", pool['2x2'], pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"nan_to_num\", pool['2x2'])\n    OpArgMngr.add_workload(\"tri\", 2, 3, 4)\n    OpArgMngr.add_workload(\"tensordot\", pool['2x2'], pool['2x2'], ((1, 0), (0, 1)))\n    OpArgMngr.add_workload(\"cumsum\", pool['3x2'], axis=0, out=pool['3x2'])\n    OpArgMngr.add_workload(\"random.shuffle\", pool['3'])\n    OpArgMngr.add_workload(\"equal\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"not_equal\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"less\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"greater_equal\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"less_equal\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"maximum\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"minimum\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"sum\", pool['2x2'], axis=0, keepdims=True, out=pool['1x2'])\n    OpArgMngr.add_workload(\"std\", pool['2x2'], axis=0, ddof=0, keepdims=True, out=pool['1x2'])\n    OpArgMngr.add_workload(\"var\", pool['2x2'], axis=0, ddof=1, keepdims=True, out=pool['1x2'])\n    OpArgMngr.add_workload(\"average\", pool['2x2'], weights=pool['2'], axis=1, returned=True)\n    OpArgMngr.add_workload(\"histogram\", pool['2x2'], bins=10, range=(0.0, 10.0))\n    OpArgMngr.add_workload(\"add\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"cross\", pool['2'], pool['2'])\n    OpArgMngr.add_workload(\"linalg.eig\", pool['3x3'])\n    OpArgMngr.add_workload(\"linalg.eigh\", pool['3x3'])\n    OpArgMngr.add_workload(\"linalg.det\", pool['3x3'])\n    OpArgMngr.add_workload(\"linalg.slogdet\", pool['3x3'])\n    OpArgMngr.add_workload(\"linalg.matrix_rank\", pool['3x3'], pool['1'], hermitian=False)\n    OpArgMngr.add_workload(\"linalg.svd\", pool['3x3'])\n    OpArgMngr.add_workload(\"linalg.cholesky\", pool['1x1'])\n    OpArgMngr.add_workload(\"linalg.qr\", pool['3x3'])\n    OpArgMngr.add_workload(\"linalg.lstsq\", pool['2x1'], pool['2'], rcond=None)\n    OpArgMngr.add_workload(\"linalg.eigvals\", pool['1x1'])\n    OpArgMngr.add_workload(\"linalg.eigvalsh\", pool['1x1'], UPLO='L')\n    OpArgMngr.add_workload(\"linalg.inv\", pool['1x1'])\n    OpArgMngr.add_workload(\"linalg.pinv\", pool['2x3x3'], pool['1'], hermitian=False)\n    OpArgMngr.add_workload(\"linalg.solve\", pool['1x1'], pool['1'])\n    OpArgMngr.add_workload(\"linalg.tensorinv\", pool['1x1'], ind=2)\n    OpArgMngr.add_workload(\"linalg.norm\", pool['3x3'])\n    OpArgMngr.add_workload(\"linalg.tensorsolve\", pool['1x1x1'], pool['1x1x1'], (2, 0, 1))\n    OpArgMngr.add_workload(\"tile\", pool['2x2'], 1)\n    OpArgMngr.add_workload(\"trace\", pool['2x2'])\n    OpArgMngr.add_workload(\"transpose\", pool['2x2'])\n    OpArgMngr.add_workload(\"split\", pool['3x3'], (0, 1, 2), axis=1)\n    OpArgMngr.add_workload(\"vstack\", (pool['3x3'], pool['3x3'], pool['3x3']))\n    OpArgMngr.add_workload(\"argmax\", pool['3x2'], axis=-1)\n    OpArgMngr.add_workload(\"argmin\", pool['3x2'], axis=-1)\n    OpArgMngr.add_workload(\"atleast_1d\", pool['2'], pool['2x2'])\n    OpArgMngr.add_workload(\"atleast_2d\", pool['2'], pool['2x2'])\n    OpArgMngr.add_workload(\"atleast_3d\", pool['2'], pool['2x2'])\n    OpArgMngr.add_workload(\"argsort\", pool['3x2'], axis=-1)\n    OpArgMngr.add_workload(\"sort\", pool['3x2'], axis=-1)\n    OpArgMngr.add_workload(\"indices\", dimensions=(1, 2, 3))\n    OpArgMngr.add_workload(\"subtract\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"multiply\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"mod\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"remainder\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"divide\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"true_divide\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"power\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"lcm\", pool['2x2'].astype('int32'), pool['2x2'].astype('int32'))\n    OpArgMngr.add_workload(\"diff\", pool['2x2'], n=1, axis=-1)\n    OpArgMngr.add_workload(\"inner\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"random.multinomial\", n=2, pvals=[1/6.]*6, size=(2,2))\n    OpArgMngr.add_workload(\"random.rand\", 3, 2)\n    OpArgMngr.add_workload(\"random.randn\", 2, 2)\n    OpArgMngr.add_workload(\"nonzero\", pool['2x2'])\n    OpArgMngr.add_workload(\"tril\", pool['2x2'], k=0)\n    OpArgMngr.add_workload(\"random.choice\", pool['2'], size=(2, 2))\n    OpArgMngr.add_workload(\"take\", pool['2'], dnp.array([1,0], dtype='int64'))\n    OpArgMngr.add_workload(\"clip\", pool['2x2'], 0, 1)\n    OpArgMngr.add_workload(\"expand_dims\", pool['2x2'], axis=0)\n    OpArgMngr.add_workload(\"broadcast_to\", pool['2x2'], (2, 2, 2))\n    OpArgMngr.add_workload(\"full_like\", pool['2x2'], 2)\n    OpArgMngr.add_workload(\"zeros_like\", pool['2x2'])\n    OpArgMngr.add_workload(\"ones_like\", pool['2x2'])\n    OpArgMngr.add_workload(\"bitwise_and\", pool['2x2'].astype(int), pool['2x2'].astype(int))\n    OpArgMngr.add_workload(\"bitwise_xor\", pool['2x2'].astype(int), pool['2x2'].astype(int))\n    OpArgMngr.add_workload(\"bitwise_or\", pool['2x2'].astype(int), pool['2x2'].astype(int))\n    OpArgMngr.add_workload(\"copysign\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"arctan2\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"hypot\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"ldexp\", pool['2x2'].astype(int), pool['2x2'].astype(int))\n    OpArgMngr.add_workload(\"logical_and\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"logical_or\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"logical_xor\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"random.uniform\", low=0, high=1, size=1)\n    OpArgMngr.add_workload(\"random.exponential\", scale=2, size=(2,2))\n    OpArgMngr.add_workload(\"random.rayleigh\", scale=2, size=(2,2))\n    OpArgMngr.add_workload(\"random.weibull\", a=2, size=(2,2))\n    OpArgMngr.add_workload(\"random.pareto\", a=2, size=(2,2))\n    OpArgMngr.add_workload(\"random.power\", a=2, size=(2,2))\n    OpArgMngr.add_workload(\"random.logistic\", loc=2, scale=2, size=(2,2))\n    OpArgMngr.add_workload(\"random.gumbel\", loc=2, scale=2, size=(2,2))\n    OpArgMngr.add_workload(\"where\", pool['2x3'], pool['2x3'], pool['2x1'])\n    OpArgMngr.add_workload(\"may_share_memory\", pool['2x3'][:0], pool['2x3'][:1])\n    OpArgMngr.add_workload('squeeze', pool['2x2'], axis=None)\n    OpArgMngr.add_workload(\"pad\", pool['2x2'], pad_width=((1,2),(1,2)), mode=\"constant\")\n    OpArgMngr.add_workload(\"prod\", pool['2x2'], axis=1, dtype=\"float64\", keepdims=False)\n    OpArgMngr.add_workload(\"around\", pool['2x2'], decimals=0)\n    OpArgMngr.add_workload(\"round\", pool['2x2'], decimals=1)\n    OpArgMngr.add_workload(\"repeat\", pool['2x2'], repeats=1, axis=None)\n    OpArgMngr.add_workload(\"diagflat\", pool['2x2'], k=1)\n    OpArgMngr.add_workload(\"diag\", pool['2x2'], k=1)\n    OpArgMngr.add_workload(\"diagonal\", pool['2x2x2'], offset=-1, axis1=0, axis2=1)\n    OpArgMngr.add_workload(\"diag_indices_from\", pool['2x2'])\n    OpArgMngr.add_workload(\"bincount\", dnp.arange(3, dtype=int), pool['3'], minlength=4)\n    OpArgMngr.add_workload(\"percentile\", pool['2x2x2'], 80, axis=0, out=pool['2x2'],\\\n                           interpolation='midpoint')\n    OpArgMngr.add_workload(\"quantile\", pool['2x2x2'], 0.8, axis=0, out=pool['2x2'],\\\n                           interpolation='midpoint')\n    OpArgMngr.add_workload(\"all\", pool['2x2x2'], axis=(0, 1),\\\n                           out=dnp.array([False, False], dtype=bool), keepdims=False)\n    OpArgMngr.add_workload(\"any\", pool['2x2x2'], axis=(0, 1),\\\n                           out=dnp.array([False, False], dtype=bool), keepdims=False)\n    OpArgMngr.add_workload(\"roll\", pool[\"2x2\"], 1, axis=0)\n    OpArgMngr.add_workload(\"rot90\", pool[\"2x2\"], 2)\n    OpArgMngr.add_workload(\"column_stack\", (pool['3x3'], pool['3x3'], pool['3x3']))\n    OpArgMngr.add_workload(\"hstack\", (pool['3x3'], pool['3x3'], pool['3x3']))\n    OpArgMngr.add_workload(\"triu\", pool['3x3'])\n    OpArgMngr.add_workload(\"array_split\", pool['2x2'], 2, axis=1)\n    OpArgMngr.add_workload(\"vsplit\", pool['2x2'], 2)\n    OpArgMngr.add_workload(\"hsplit\", pool['2x2'], 2)\n    OpArgMngr.add_workload(\"dsplit\", pool['2x2x2'], 2)\n    OpArgMngr.add_workload(\"arange\", 10)\n    OpArgMngr.add_workload(\"concatenate\", (pool['1x2'], pool['1x2'], pool['1x2']), axis=0)\n    OpArgMngr.add_workload(\"append\", pool['2x2'], pool['1x2'], axis=0)\n    OpArgMngr.add_workload(\"insert\", pool['3x2'], 1, pool['1x1'], axis=0)\n    OpArgMngr.add_workload(\"delete\", pool['3x2'], 1, axis=0)\n    OpArgMngr.add_workload(\"blackman\", 12)\n    OpArgMngr.add_workload(\"eye\", 5)\n    OpArgMngr.add_workload(\"hamming\", 12)\n    OpArgMngr.add_workload(\"hanning\", 12)\n    OpArgMngr.add_workload(\"linspace\", 0, 10, 8, endpoint=False)\n    OpArgMngr.add_workload(\"logspace\", 2.0, 3.0, num=4, base=2.0, dtype=onp.float32)\n    OpArgMngr.add_workload(\"matmul\", pool['2x2'], pool['2x2'])\n    OpArgMngr.add_workload(\"mean\", pool['2x2'], axis=0, keepdims=True)\n    OpArgMngr.add_workload(\"random.gamma\", 1, size=(2, 3))\n    OpArgMngr.add_workload(\"random.normal\", 1, size=(2, 3))\n    OpArgMngr.add_workload(\"max\", pool[\"2x2\"], axis=0, out=pool['2'], keepdims=False)\n    OpArgMngr.add_workload(\"min\", pool[\"2x2\"], axis=0, out=pool['2'], keepdims=False)\n    OpArgMngr.add_workload(\"amax\", pool[\"2x2\"], axis=1, out=pool['2'], keepdims=False)\n    OpArgMngr.add_workload(\"amin\", pool[\"2x2\"], axis=1, out=pool['2'], keepdims=False)\n\n    unary_ops = ['negative', 'reciprocal', 'abs', 'sign', 'rint', 'ceil', 'floor',\n                 'bitwise_not', 'trunc', 'fix', 'square', 'sqrt', 'cbrt', 'exp',\n                 'log', 'log10', 'log2', 'log1p', 'expm1', 'logical_not', 'isnan',\n                 'isinf', 'isposinf', 'isneginf', 'isfinite', 'sin', 'cos', 'tan',\n                 'arcsin', 'arccos', 'arctan', 'degrees', 'radians', 'sinh', 'cosh',\n                 'tanh', 'arcsinh', 'arccosh', 'arctanh']  # 'rad2deg', 'deg2rad' cannot run without tvm\n    for unary_op in unary_ops:\n        if unary_op == \"bitwise_not\":\n            OpArgMngr.add_workload(unary_op, dnp.ones((2, 2), dtype=int))\n        else:\n            OpArgMngr.add_workload(unary_op, pool['2x2'])\n\n\ndef benchmark_helper(f, *args, **kwargs):\n    number = 10000\n    return timeit.timeit(lambda: f(*args, **kwargs), number=number) / number\n\n\ndef get_op(module, funcname):\n    funcname = funcname.split(\".\")\n    for fname in funcname:\n        module = getattr(module, fname)\n    return module\n\n\ndef run_benchmark(packages):\n    results = {}\n    for (k, v) in OpArgMngr.args.items():\n        result = {}\n        for (name, package) in packages.items():\n            print(f'{name}.{k} running...')\n            op = get_op(package[\"module\"], v[\"funcname\"])\n            args = [package[\"data\"](arg) for arg in v[\"args\"]]\n            kwargs = {k: package[\"data\"](v) for (k, v) in v[\"kwargs\"].items()}\n            benchmark = benchmark_helper(op, *args, **kwargs)\n            result[name] = benchmark\n        results[k] = result\n    return results\n\n\ndef show_results(results):\n    print(f'{\"name\":>24}{\"package\":>24}{\"time(us)\":>24}')\n    for (specifier, d) in results.items():\n        for (k, v) in d.items():\n            print(f\"{specifier:>24}{k:>24}{v * 10 ** 6:>24}\")\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument('ffi_type')\n    parsed = parser.parse_args()\n    if parsed.ffi_type == \"cython\":\n        os.environ['MXNET_ENABLE_CYTHON'] = '1'\n        os.environ['MXNET_ENFORCE_CYTHON'] = '1'\n    elif parsed.ffi_type == \"ctypes\":\n        os.environ['MXNET_ENABLE_CYTHON'] = '0'\n    else:\n        raise ValueError(\"unknown ffi_type {}\",format(parsed.ffi_type))\n    os.environ[\"MXNET_ENGINE_TYPE\"] = \"NaiveEngine\"\n    import mxnet as mx\n    import numpy as onp\n    from mxnet import np as dnp\n\n    mx.npx.set_np(dtype=False)\n    packages = {\n        \"onp\": {\n            \"module\": onp,\n            \"data\": lambda arr: arr.asnumpy() if isinstance(arr, dnp.ndarray) else arr\n        },\n        \"dnp\": {\n            \"module\": dnp,\n            \"data\": lambda arr: arr\n        }\n    }\n    prepare_workloads()\n    results = run_benchmark(packages)\n    show_results(results)\n"
  },
  {
    "path": "benchmark/python/metric/benchmark_metric.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nfrom __future__ import print_function\n\nimport itertools\nimport mxnet as mx\nimport sys\nimport time\n\n\nclass MetricDataGen(object):\n    \"\"\" Base class for generating random data for metric benchmarking \"\"\"\n    def __init__(self, n, c, pred_ctx, label_ctx):\n        self.n = n\n        self.c = c\n        self.pred_ctx = pred_ctx\n        self.label_ctx = label_ctx\n\n    def data(self):\n        mx.random.seed(0)\n        pred = mx.nd.random_uniform(0.0, 1.0, (self.n, self.c), ctx=self.pred_ctx)\n        label = mx.nd.random_uniform(0.0, self.c - 1, (self.n,), ctx=self.label_ctx).round()\n        return label, pred\n\n    @property\n    def batch_size(self):\n        return self.n\n\n    @property\n    def output_dim(self):\n        return self.c\n\n\nclass F1MetricDataGen(MetricDataGen):\n    \"\"\" Class for generating random data for F1 metric benchmarking \"\"\"\n    def __init__(self, n, c, pred_ctx, label_ctx):\n        super(F1MetricDataGen, self).__init__(n, 2, pred_ctx, label_ctx)\n\n\nclass PearsonMetricDataGen(MetricDataGen):\n    \"\"\" Class for generating random data for Pearson Correlation metric benchmarking \"\"\"\n    def __init__(self, n, c, pred_ctx, label_ctx):\n        super(PearsonMetricDataGen, self).__init__(n, c, pred_ctx, label_ctx)\n\n    def data(self):\n        mx.random.seed(0)\n        pred = mx.nd.random_uniform(0.0, 1.0, (self.n, self.c), ctx=self.pred_ctx)\n        label = mx.nd.random_uniform(0.0, 1.0, (self.n, self.c), ctx=self.label_ctx)\n        return label, pred\n\n\ndef run_metric(name, data_gen_cls, i, n, c, pred_ctx, label_ctx, **kwargs):\n    \"\"\" Helper function for running one metric benchmark \"\"\"\n    metric = mx.gluon.metric.create(name, **kwargs)\n    data_gen = data_gen_cls(n, c, pred_ctx, label_ctx)\n    try:\n        label, pred = data_gen.data()\n        mx.nd.waitall()\n        before = time.time()\n        metric.update([label] * i, [pred] * i)\n        mx.nd.waitall()\n        elapsed = time.time() - before\n        elapsed_str = f\"{elapsed:<.5}\"\n    except mx.MXNetError:\n        elapsed_str = \"FAILED\"\n    print(f\"{name:<15}{pred_ctx:<10}{label_ctx:<12}{i * n:<12}{data_gen.batch_size:<15}{data_gen.output_dim:<15}{elapsed_str:<}\", file=sys.stderr)\n\n\ndef test_metric_performance():\n    \"\"\" unittest entry for metric performance benchmarking \"\"\"\n    # Each dictionary entry is (metric_name:(kwargs, DataGenClass))\n    metrics = [\n        ('acc', ({}, MetricDataGen)),\n        ('top_k_acc', ({'top_k': 5}, MetricDataGen)),\n        ('F1', ({}, F1MetricDataGen)),\n        ('Perplexity', ({'ignore_label': -1}, MetricDataGen)),\n        ('MAE', ({}, MetricDataGen)),\n        ('MSE', ({}, MetricDataGen)),\n        ('RMSE', ({}, MetricDataGen)),\n        ('ce', ({}, MetricDataGen)),\n        ('nll_loss', ({}, MetricDataGen)),\n        ('pearsonr', ({}, PearsonMetricDataGen)),\n    ]\n\n    data_size = 1024 * 128\n\n    batch_sizes = [16, 64, 256, 1024]\n    output_dims = [128, 1024, 8192]\n    ctxs = [mx.cpu(), mx.gpu()]\n\n    print(\"\\nmx.gluon.metric benchmarks\", file=sys.stderr)\n    print(\n        f\"{'Metric':15}{'Data-Ctx':10}{'Label-Ctx':12}{'Data Size':12}{'Batch Size':15}{'Output Dim':15}{'Elapsed Time'}\",\n        file=sys.stderr)\n    print(f\"{'':-^90}\", file=sys.stderr)\n    for k, v in metrics:\n        for c in output_dims:\n            for n in batch_sizes:\n                for pred_ctx, label_ctx in itertools.product(ctxs, ctxs):\n                    run_metric(k, v[1], (data_size * 128), (n * c), n, c, pred_ctx, label_ctx, **v[0])\n                print(f\"{'':-^90}\", file=sys.stderr)\n"
  },
  {
    "path": "benchmark/python/quantization/benchmark_op.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport time\nimport mxnet as mx\nfrom mxnet.test_utils import check_speed\n\n\ndef quantize_int8_helper(data):\n    min_data = mx.nd.min(data)\n    max_data = mx.nd.max(data)\n    return mx.nd.contrib.quantize(data, min_data, max_data, out_type='int8')\n\n\ndef benchmark_convolution(data_shape, kernel, num_filter, pad, stride, no_bias=True, layout='NCHW', repeats=20):\n    ctx_gpu = mx.gpu(0)\n    data = mx.sym.Variable(name=\"data\", shape=data_shape, dtype='float32')\n    # conv cudnn\n    conv_cudnn = mx.sym.Convolution(data=data, kernel=kernel, num_filter=num_filter, pad=pad, stride=stride,\n                                    no_bias=no_bias, layout=layout, cudnn_off=False, name=\"conv_cudnn\")\n    arg_shapes, _, _ = conv_cudnn.infer_shape(data=data_shape)\n    input_data = mx.nd.random.normal(0, 0.2, shape=data_shape, ctx=ctx_gpu)\n    conv_weight_name = conv_cudnn.list_arguments()[1]\n    args = {data.name: input_data, conv_weight_name: mx.random.normal(0, 1, shape=arg_shapes[1], ctx=ctx_gpu)}\n    conv_cudnn_time = check_speed(sym=conv_cudnn, location=args, ctx=ctx_gpu, N=repeats,\n                                  grad_req='null', typ='forward') * 1000\n\n    # quantized_conv2d\n    qdata = mx.sym.Variable(name='qdata', shape=data_shape, dtype='int8')\n    weight = mx.sym.Variable(name='weight', shape=arg_shapes[1], dtype='int8')\n    min_data = mx.sym.Variable(name='min_data', shape=(1,), dtype='float32')\n    max_data = mx.sym.Variable(name='max_data', shape=(1,), dtype='float32')\n    min_weight = mx.sym.Variable(name='min_weight', shape=(1,), dtype='float32')\n    max_weight = mx.sym.Variable(name='max_weight', shape=(1,), dtype='float32')\n    quantized_conv2d = mx.sym.contrib.quantized_conv(data=qdata, weight=weight, min_data=min_data, max_data=max_data,\n                                                     min_weight=min_weight, max_weight=max_weight,\n                                                     kernel=kernel, num_filter=num_filter, pad=pad, stride=stride,\n                                                     no_bias=no_bias, layout=layout, cudnn_off=False,\n                                                     name='quantized_conv2d')\n    qargs = {qdata.name: quantize_int8_helper(input_data)[0],\n             min_data.name: quantize_int8_helper(input_data)[1],\n             max_data.name: quantize_int8_helper(input_data)[2],\n             weight.name: quantize_int8_helper(args[conv_weight_name])[0],\n             min_weight.name: quantize_int8_helper(args[conv_weight_name])[1],\n             max_weight.name: quantize_int8_helper(args[conv_weight_name])[2]}\n    qconv_time = check_speed(sym=quantized_conv2d, location=qargs, ctx=ctx_gpu, N=repeats,\n                             grad_req='null', typ='forward') * 1000\n\n    print('==================================================================================================')\n    print(f'data={data_shape}, kernel={kernel}, num_filter={num_filter}, pad={pad}, stride={stride}, no_bias={no_bias}, layout={layout}, repeats={repeats}')\n    print(f'{conv_cudnn.name}-FP32 , ctx={ctx_gpu}, time={conv_cudnn_time:.2f} ms')\n    print(f'{quantized_conv2d.name}, ctx={ctx_gpu}, time={qconv_time:.2f} ms')\n    print(f'quantization speedup:               {conv_cudnn_time / qconv_time:.1f}X')\n    print('\\n')\n\n\nif __name__ == '__main__':\n    for batch_size in [32, 64, 128]:\n        benchmark_convolution(data_shape=(batch_size, 64, 56, 56), kernel=(1, 1), num_filter=256,\n                              pad=(0, 0), stride=(1, 1), layout='NCHW', repeats=20)\n\n        benchmark_convolution(data_shape=(batch_size, 256, 56, 56), kernel=(1, 1), num_filter=64,\n                              pad=(0, 0), stride=(1, 1), layout='NCHW', repeats=20)\n\n        benchmark_convolution(data_shape=(batch_size, 256, 56, 56), kernel=(1, 1), num_filter=128,\n                              pad=(0, 0), stride=(2, 2), layout='NCHW', repeats=20)\n\n        benchmark_convolution(data_shape=(batch_size, 128, 28, 28), kernel=(3, 3), num_filter=128,\n                              pad=(1, 1), stride=(1, 1), layout='NCHW', repeats=20)\n\n        benchmark_convolution(data_shape=(batch_size, 1024, 14, 14), kernel=(1, 1), num_filter=256,\n                              pad=(0, 0), stride=(1, 1), layout='NCHW', repeats=20)\n\n        benchmark_convolution(data_shape=(batch_size, 2048, 7, 7), kernel=(1, 1), num_filter=512,\n                              pad=(0, 0), stride=(1, 1), layout='NCHW', repeats=20)\n"
  },
  {
    "path": "benchmark/python/sparse/cast_storage.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport ctypes\n\nfrom mxnet.test_utils import *\nimport os\nimport time\nimport argparse\n\nfrom mxnet.base import check_call, _LIB\n\nparser = argparse.ArgumentParser(description=\"Benchmark cast storage operators\",\n                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)\nparser.add_argument('--num-omp-threads', type=int, default=1, help='number of omp threads to set in MXNet')\nargs = parser.parse_args()\n\ndef measure_cost(repeat, f, *args, **kwargs):\n    start = time.time()\n    results = []\n    for i in range(repeat):\n        (f(*args, **kwargs)).wait_to_read()\n    end = time.time()\n    diff = end - start\n    return diff / repeat\n\n\ndef run_cast_storage_synthetic():\n    def dense_to_sparse(m, n, density, ctx, repeat, stype):\n        set_default_device(ctx)\n        data_shape = (m, n)\n        dns_data = rand_ndarray(data_shape, stype, density).tostype('default')\n        dns_data.wait_to_read()\n\n        # do one warm up run, verify correctness\n        assert same(mx.nd.cast_storage(dns_data, stype).asnumpy(), dns_data.asnumpy())\n\n        # start benchmarking\n        cost = measure_cost(repeat, mx.nd.cast_storage, dns_data, stype)\n        results = f'{density*100:10.1f} {str(ctx):>10} {m:8d} {n:8d} {cost * 1000:10.2f}'\n        print(results)\n\n    check_call(_LIB.MXSetNumOMPThreads(ctypes.c_int(args.num_omp_threads)))\n\n    # params\n    # m           number of rows\n    # n           number of columns\n    # density     density of the matrix\n    # num_repeat  number of benchmark runs to average over\n    # contexts    mx.cpu(), mx.gpu()\n    #             note: benchmark different contexts separately; to benchmark cpu, compile without CUDA\n    # benchmarks  dns_to_csr, dns_to_rsp\n    m = [  512,    512]\n    n = [50000, 100000]\n    density = [1.00, 0.80, 0.60, 0.40, 0.20, 0.10, 0.05, 0.02, 0.01]\n    num_repeat = 10\n    contexts = [mx.gpu()]\n    benchmarks = [\"dns_to_csr\", \"dns_to_rsp\"]\n\n    # run benchmark\n    for b in benchmarks:\n        stype = ''\n        print(\"==================================================\")\n        if b is \"dns_to_csr\":\n            stype = 'csr'\n            print(\" cast_storage benchmark: dense to csr, size m x n \")\n        elif b is \"dns_to_rsp\":\n            stype = 'row_sparse'\n            print(\" cast_storage benchmark: dense to rsp, size m x n \")\n        else:\n            print(f\"invalid benchmark: {b}\")\n            continue\n        print(\"==================================================\")\n        headline = f\"{'density(%)':>10} {'context':>10} {'m':>8} {'n':>8} {'time(ms)':>10}\"\n        print(headline)\n        for i in range(len(n)):\n            for ctx in contexts:\n                for den in density:\n                    dense_to_sparse(m[i], n[i], den, ctx, num_repeat, stype)\n            print(\"\")\n        print(\"\")\n\n\nif __name__ == \"__main__\":\n    run_cast_storage_synthetic()\n"
  },
  {
    "path": "benchmark/python/sparse/dot.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport ctypes\n\nimport os\nimport time\nimport argparse\nimport subprocess\nimport scipy.sparse as sp\n\nimport mxnet as mx\nimport numpy as np\nimport numpy.random as rnd\nfrom mxnet.test_utils import rand_ndarray, set_default_device, assert_almost_equal, get_bz2_data\nfrom mxnet.base import check_call, _LIB\nfrom util import estimate_density\n\nPARSER = argparse.ArgumentParser(description=\"Benchmark sparse operators\",\n                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)\nPARSER.add_argument('--num-omp-threads', type=int,\n                    default=1, help='number of omp threads to set in MXNet')\nPARSER.add_argument('--gpu', action='store_true',\n                    help=\"to be run on gpu\")\n# TODO: Use logging later\nPARSER.add_argument('--verbose', action='store_true',\n                    help=\"Verbose output\")\nARGS = PARSER.parse_args()\n\n# some data information\nKDDA = {\n    'data_mini': 'kdda.t.mini',\n    'data_name': 'kdda.t',\n    'data_origin_name': 'kdda.t.bz2',\n    'url': \"https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/kdda.t.bz2\",\n    'feature_dim': 20216830,\n    'm': [1, 8, 32],\n    'batch_size': [64],\n    'default_index': {'batch_size': 0,\n                      'output_dim': 2},\n    'num_batches': 10\n}\n\nAVAZU = {\n    'data_mini': 'avazu-app.t.mini',\n    'data_name': 'avazu-app.t',\n    'data_origin_name': 'avazu-app.t.bz2',\n    'url': \"https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/avazu-app.t.bz2\",\n    'feature_dim': 1000000,\n    'm': [1, 1000, 2000],\n    'batch_size': [128, 256],\n    'default_index': {'batch_size': 0,\n                      'output_dim': 1},\n    'num_batches': 10\n}\n\nCRITEO = {\n    'data_mini': 'criteo.t.mini',\n    'data_name': 'criteo.t',\n    'data_origin_name': 'criteo.t.bz2',\n    'url': \"https://s3-us-west-2.amazonaws.com/sparse-dataset/criteo.t.bz2\",\n    'feature_dim': 8388621,\n    'm': [1, 8, 16, 32, 64],\n    'batch_size': [64, 128],\n    'default_index': {'batch_size': 1,\n                      'output_dim': 3},\n    'num_batches': 10\n}\n\nSYNTHETIC1 = {\n    'feature_dim': [1000000],\n    'm': [256, 1000],\n    'density': [0.001, 0.005, 0.01, 0.02, 0.05,\n                0.1, 0.2, 0.5, 0.65],\n    'batch_size': [64, 128],\n    'default_index': {'batch_size': 1,\n                      'density': 2,\n                      'output_dim': 1,\n                      'feature_dim': 0},\n    'num_repeat': 10\n}\n\nSYNTHETIC2 = {\n    'feature_dim': [8000000, 16000000],\n    'm': [1, 32],\n    'density': [0.001, 0.005, 0.01, 0.02, 0.05,\n                0.1, 0.2, 0.5, 0.65],\n    'batch_size': [64, 128],\n    'default_index': {'batch_size': 1,\n                      'density': 2,\n                      'output_dim': 1,\n                      'feature_dim': 0},\n    'num_repeat': 10\n}\n\ndef measure_cost(repeat, scipy_trans_lhs, scipy_dns_lhs, func_name, *args, **kwargs):\n    \"\"\"Measure time cost of running a function\n    \"\"\"\n    mx.nd.waitall()\n    args_list = []\n    for arg in args:\n        args_list.append(arg)\n    start = time.time()\n    if scipy_trans_lhs:\n        args_list[0] = np.transpose(args_list[0]) if scipy_dns_lhs else sp.spmatrix.transpose(args_list[0])\n    for _ in range(repeat):\n        func_name(*args_list, **kwargs)\n    mx.nd.waitall()\n    end = time.time()\n    diff = end - start\n    return diff / repeat\n\n\ndef _get_iter(path, data_shape, batch_size):\n    data_train = mx.io.LibSVMIter(data_libsvm=path,\n                                  data_shape=data_shape,\n                                  batch_size=batch_size)\n    data_iter = iter(data_train)\n    return data_iter\n\n\ndef _line_count(path):\n    return int(subprocess.check_output('wc -l {}'.format(path), shell=True).split()[0])\n\n\ndef _compare_sparse_dense(data_dir, file_name, mini_file_name, feature_dim,\n                          output_dim, density, batch_size, num_batches=3, num_repeat=5, transpose=False,\n                          rsp=False):\n\n    def create_mini_path(mini_path, path, num_batches):\n        \"\"\"Samples batches of size: batch_size, total number: num_batches\n        from the dataset files for running benchmarks\"\"\"\n        if not os.path.exists(mini_path):\n            last = _line_count(path) - num_batches * batch_size\n            last = last if last >= 1 else 1\n            start = int(rnd.uniform(1, last))\n            os.system(\"sed -n '{},{}p' {} > {}\".format(\n                start, start + num_batches * batch_size, repr(path), repr(mini_path)))\n            assert os.path.exists(mini_path)\n\n    def run_benchmark(mini_path):\n        \"\"\"Run benchmarks\n        \"\"\"\n        data_shape = (feature_dim, )\n        train_iter = _get_iter(mini_path, data_shape, batch_size)\n        weight_row_dim = batch_size if transpose else feature_dim\n        weight_shape = (weight_row_dim, output_dim)\n        if not rsp:\n            weight = mx.nd.random.uniform(low=0, high=1, shape=weight_shape)\n        else:\n            weight = rand_ndarray(weight_shape, \"row_sparse\", density=0.05, distribution=\"uniform\")\n        total_cost = {}\n        average_cost = {}\n        count = 0\n        total_cost[\"sparse\"] = 0.\n        total_cost[\"dense\"] = 0.\n        for _ in train_iter:\n            csr_data = train_iter.getdata()\n            dns_data = csr_data.tostype('default')\n            cost_sparse = measure_cost(num_repeat, False, False, mx.nd.sparse.dot, csr_data, weight, transpose_a=transpose)\n            cost_dense = measure_cost(num_repeat, False, False, mx.nd.dot, dns_data, weight, transpose_a=transpose)\n            total_cost[\"sparse\"] += cost_sparse\n            total_cost[\"dense\"] += cost_dense\n            count = count + 1\n        average_cost[\"sparse\"] = total_cost[\"sparse\"] / count\n        average_cost[\"dense\"] = total_cost[\"dense\"] / count\n        return (average_cost[\"sparse\"], average_cost[\"dense\"])\n\n    def print_result(average_cost_sparse, average_cost_dense):\n        \"\"\"Print result of comparison between sparse and dense\n        \"\"\"\n        ratio = average_cost_dense / average_cost_sparse\n        fmt = '{:15.4f} {:10d} {:10d} {:10d} {:20.2f} {:15.2f} {:15.2f} {:10} {:10}'\n        print(fmt.format(density * 100, batch_size, output_dim, feature_dim,\n                         ratio, average_cost_dense*1000, average_cost_sparse*1000,\n                         transpose, rsp))\n\n    mini_path = os.path.join(data_dir, mini_file_name)\n    path = os.path.join(data_dir, file_name)\n    create_mini_path(mini_path, path, num_batches)\n    average_cost_sparse, average_cost_dense = run_benchmark(mini_path)\n    print_result(average_cost_sparse, average_cost_dense)\n\n\ndef test_dot_real(data_dict):\n    \"\"\"Dot operator testing with real datasets\"\"\"\n    data_dir = os.path.join(os.getcwd(), 'data')\n\n    path = os.path.join(data_dir, data_dict['data_name'])\n    if not os.path.exists(path):\n        get_bz2_data(\n            data_dir,\n            data_dict['data_name'],\n            data_dict['url'],\n            data_dict['data_origin_name']\n        )\n        assert os.path.exists(path)\n\n    k = data_dict['feature_dim']\n    m = data_dict['m']\n    batch_size_list = data_dict['batch_size']\n\n    default_output_index = data_dict['default_index']['output_dim']\n    default_batch_size_index = data_dict['default_index']['batch_size']\n    density = estimate_density(path, data_dict['feature_dim'])\n    num_batches = data_dict['num_batches']\n\n    assert default_batch_size_index < len(batch_size_list)\n    assert default_output_index < len(m)\n    if ARGS.verbose:\n        print(f\"Running Benchmarking on {repr(data_dict['data_mini'])} data\")\n    print('{:>15} {:>10} {:>10} {:>10} {:>20} {:>15} {:>15} {:>10} {:>10}'.format('density(%)',\n                                                                                  'n',\n                                                                                  'm',\n                                                                                  'k',\n                                                                                  't_dense/t_sparse',\n                                                                                  't_dense(ms)',\n                                                                                  't_sparse(ms)',\n                                                                                  'is_transpose',\n                                                                                  'rhs_rsp'))\n\n    for output_dim in m:\n        _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'],\n                              k, output_dim, density,\n                              batch_size_list[default_batch_size_index], num_batches)\n        _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'],\n                              k, output_dim, density,\n                              batch_size_list[default_batch_size_index], num_batches,\n                              transpose=True)\n        _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'],\n                              k, output_dim, density,\n                              batch_size_list[default_batch_size_index], num_batches, rsp=True)\n\n    for batch_size in batch_size_list:\n        _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'],\n                              k, m[default_output_index], density, batch_size, num_batches)\n        _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'],\n                              k, m[default_output_index], density, batch_size, num_batches,\n                              transpose=True)\n        _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'],\n                              k, output_dim, density,\n                              batch_size_list[default_batch_size_index], num_batches, rsp=True)\n\n\ndef test_dot_synthetic(data_dict):\n    \"\"\"benchmark sparse mxnet dot and scipy dot operator with matrices of given density.\n    `t_sparse` is the runtime of the invoked sparse dot operator in ms, while `t_dense` is the\n    runtime of dot(dns, dns), with the same matrices except that they are in default storage type.\n    \"\"\"\n    # Benchmark MXNet and Scipys dot operator\n    def bench_dot(lhs_shape, rhs_shape, lhs_stype, rhs_stype,\n                  lhs_den, rhs_den, trans_lhs, ctx, num_repeat=10, fw=\"mxnet\", distribution=\"uniform\"):\n        set_default_device(ctx)\n        assert fw == \"mxnet\" or fw == \"scipy\"\n        # Set funcs\n        dot_func_sparse = mx.nd.sparse.dot if fw == \"mxnet\" else sp.spmatrix.dot\n        dot_func_dense = mx.nd.dot if fw == \"mxnet\" else np.dot\n        # Create matrix instances\n        lhs_nd = rand_ndarray(lhs_shape, lhs_stype, density=lhs_den, distribution=distribution)\n        # only uniform distribution supported for rhs\n        if rhs_stype == 'csr':\n            rhs_nd = rand_ndarray(rhs_shape, rhs_stype, density=rhs_den, distribution=distribution)\n        else:\n            rhs_nd = rand_ndarray(rhs_shape, rhs_stype, density=rhs_den, distribution=\"uniform\")\n        lhs_dns = None\n        rhs_dns = None\n        dense_cost = None\n        sparse_cost = None\n\n        if fw == \"mxnet\":\n            lhs_dns = lhs_nd if lhs_stype == 'default' else lhs_nd.tostype('default')\n            rhs_dns = rhs_nd if rhs_stype == 'default' else rhs_nd.tostype('default')\n            # One warm up run, verify correctness\n            out = dot_func_sparse(lhs_nd, rhs_dns, trans_lhs)\n            out_expected = dot_func_dense(lhs_dns, rhs_dns, trans_lhs)\n            assert_almost_equal(out.asnumpy(), out_expected.asnumpy(), rtol=1e-1, atol=1e-1)\n            sparse_cost = measure_cost(num_repeat, False, False, dot_func_sparse, lhs_nd, rhs_nd, trans_lhs)\n            dense_cost = measure_cost(num_repeat, False, False, dot_func_dense, lhs_dns, rhs_dns, trans_lhs)\n        else:\n            lhs_dns = lhs_nd.asnumpy()\n            rhs_dns = rhs_nd.asnumpy()\n            lhs_nd = sp.csr_matrix(lhs_nd.asnumpy())\n            rhs_nd = rhs_nd.asnumpy()\n            # One warm up run, verify correctness\n            lhs_nd_copy = sp.spmatrix.transpose(lhs_nd) if trans_lhs else lhs_nd\n            out = dot_func_sparse(lhs_nd_copy, rhs_dns)\n            sparse_cost = measure_cost(num_repeat, trans_lhs, False, dot_func_sparse, lhs_nd, rhs_nd)\n            dense_cost = measure_cost(num_repeat, trans_lhs, True, dot_func_dense, lhs_dns, rhs_dns)\n\n        speedup = dense_cost / sparse_cost\n        # Print results\n        m = lhs_shape[0]\n        k = lhs_shape[1]\n        n = rhs_shape[1]\n        result_pattern = '{:15.1f} {:15.1f} {:>10} {:8d} {:8d} {:8d} {:13.2f} {:13.2f} {:8.2f}'\n        results = result_pattern.format(lhs_den*100,\n                                        rhs_den*100,\n                                        str(ctx),\n                                        m,\n                                        k,\n                                        n,\n                                        sparse_cost*1000,\n                                        dense_cost*1000,\n                                        speedup)\n        print(results)\n\n    def print_benchmark_info(lhs, rhs, lhs_trans, fw):\n        trans_str = \"^T\" if lhs_trans else \"\"\n        print(\"========================================================\")\n        print(f\"  {fw} sparse dot benchmark: dot({lhs}, {rhs}) = {rhs}  \")\n        print(\n            f\"  (matrix multiplication: (m x k){trans_str} * (k x n) = m x n)  \")\n        print(\"========================================================\")\n        headline_pattern = '{:>15} {:>15} {:>10} {:>8} {:>8} {:>8} {:>13} {:>13} {:>8}'\n        headline = headline_pattern.format('lhs_density(%)',\n                                           'rhs_density(%)',\n                                           'context',\n                                           'm', 'k', 'n',\n                                           't_sparse(ms)',\n                                           't_dense(ms)',\n                                           'speedup')\n        print(headline)\n\n    def run_benchmark(ctx=None, lhs=\"csr\", lhs_trans=False, rhs=\"dns\", fw=\"mxnet\", rhs_density=1,\n                      distribution=\"uniform\"):\n\n        if rhs_density > 1 or rhs_density < 0:\n            raise ValueError(\"rhs_density has to be between 0 and 1\")\n\n        print_benchmark_info(lhs, rhs, lhs_trans, fw)\n\n        if rhs == \"csr\":\n            lhs_stype = \"default\"\n            rhs_stype = \"csr\"\n            assert (lhs_stype == 'default'), \"Only dot(default, csr) supported\"\n            # Arrange dimensions according to use case. For below csr will have num_rows << num_cols\n            feature_dim_list = data_dict['batch_size']\n            batch_size_list = data_dict['m']\n            output_dim_list = data_dict['feature_dim']\n            density_list = data_dict['density']\n            default_output_index = data_dict['default_index']['feature_dim']\n            default_density_index = data_dict['default_index']['density']\n            default_feature_index = data_dict['default_index']['batch_size']\n            default_batch_size_index = data_dict['default_index']['output_dim']\n            num_repeat = data_dict['num_repeat']\n\n        else:\n            lhs_stype = \"csr\"\n            rhs_stype = \"row_sparse\" if rhs == \"rsp\" else \"default\"\n\n            feature_dim_list = data_dict['feature_dim']\n            output_dim_list = data_dict['m']\n            batch_size_list = data_dict['batch_size']\n            density_list = data_dict['density']\n\n            default_output_index = data_dict['default_index']['output_dim']\n            default_batch_size_index = data_dict['default_index']['batch_size']\n            default_feature_index = data_dict['default_index']['feature_dim']\n            default_density_index = data_dict['default_index']['density']\n            num_repeat = data_dict['num_repeat']\n\n        for output_dim in output_dim_list:\n            if lhs_trans:\n                output_row_dim = batch_size_list[default_batch_size_index]\n            else:\n                output_row_dim = feature_dim_list[default_feature_index]\n            bench_dot((batch_size_list[default_batch_size_index],\n                       feature_dim_list[default_feature_index]),\n                      (output_row_dim, output_dim),\n                      lhs_stype, rhs_stype,\n                      density_list[default_density_index], rhs_density,\n                      lhs_trans, ctx, num_repeat=num_repeat,\n                      fw=fw, distribution=distribution)\n\n        for feature_dim in feature_dim_list:\n            if lhs_trans:\n                output_row_dim = batch_size_list[default_batch_size_index]\n            else:\n                output_row_dim = feature_dim\n            bench_dot((batch_size_list[default_batch_size_index], feature_dim),\n                      (output_row_dim, output_dim_list[default_output_index]),\n                      lhs_stype, rhs_stype, density_list[default_density_index], rhs_density,\n                      lhs_trans, ctx, num_repeat=num_repeat, fw=fw, distribution=distribution)\n\n        for batch_size in batch_size_list:\n            if lhs_trans:\n                output_row_dim = batch_size\n            else:\n                output_row_dim = feature_dim_list[default_feature_index]\n            bench_dot((batch_size, feature_dim_list[default_feature_index]),\n                      (output_row_dim,\n                       output_dim_list[default_output_index]),\n                      lhs_stype, rhs_stype, density_list[default_density_index],\n                      rhs_density, lhs_trans, ctx, num_repeat=num_repeat,\n                      fw=fw, distribution=distribution)\n\n        for density in density_list:\n            if lhs_trans:\n                output_row_dim = batch_size_list[default_batch_size_index]\n            else:\n                output_row_dim = feature_dim_list[default_feature_index]\n            bench_dot((batch_size_list[default_batch_size_index],\n                       feature_dim_list[default_feature_index]),\n                      (output_row_dim,\n                       output_dim_list[default_output_index]),\n                      lhs_stype, rhs_stype, density, density, lhs_trans, ctx,\n                      num_repeat=num_repeat, fw=fw, distribution=distribution)\n\n    check_call(_LIB.MXSetNumOMPThreads(ctypes.c_int(ARGS.num_omp_threads)))\n    context = mx.gpu() if ARGS.gpu else mx.cpu()\n    # TODO(anirudh): make the data dicts to config which can be passed at runtime\n    distributions = [\"uniform\", \"powerlaw\"]\n    for distribution in distributions:\n        run_benchmark(context, lhs=\"csr\",\n                      rhs=\"default\", lhs_trans=False,\n                      fw=\"mxnet\", rhs_density=1,\n                      distribution=distribution)\n        run_benchmark(context, lhs=\"csr\",\n                      rhs=\"default\", lhs_trans=True,\n                      fw=\"mxnet\", rhs_density=1,\n                      distribution=distribution)\n        run_benchmark(context, lhs=\"csr\",\n                      rhs=\"rsp\", lhs_trans=False,\n                      fw=\"mxnet\", rhs_density=0.05,\n                      distribution=distribution)\n        run_benchmark(context, lhs=\"default\",\n                      rhs=\"csr\", lhs_trans=False,\n                      fw=\"mxnet\", rhs_density=0.001,\n                      distribution=distribution)\n        if not ARGS.gpu:\n            run_benchmark(context, lhs=\"csr\",\n                          rhs=\"default\", lhs_trans=False,\n                          fw=\"scipy\", rhs_density=1,\n                          distribution=distribution)\n            run_benchmark(context, lhs=\"csr\",\n                          rhs=\"default\", lhs_trans=True,\n                          fw=\"scipy\", rhs_density=1,\n                          distribution=distribution)\n\n\nif __name__ == \"__main__\":\n    begin_time = time.time()\n    test_dot_real(KDDA)\n    test_dot_real(AVAZU)\n    test_dot_real(CRITEO)\n    test_dot_synthetic(SYNTHETIC1)\n    test_dot_synthetic(SYNTHETIC2)\n    total_time = time.time() - begin_time\n    print(f\"total time is {total_time}\")\n"
  },
  {
    "path": "benchmark/python/sparse/memory_benchmark.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Should be run with valgrind to get memory consumption\n   for sparse format storage and dot operators. This script can be\n   used for memory benchmarking on CPU only\"\"\"\nimport ctypes\nimport sys\nimport argparse\nimport mxnet as mx\nfrom mxnet.test_utils import rand_ndarray\nfrom mxnet.base import check_call, _LIB\n\n\ndef parse_args():\n    \"\"\" Function to parse arguments\n    \"\"\"\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--lhs-row-dim\",\n                        required=True,\n                        help=\"Provide batch_size\")\n    parser.add_argument(\"--lhs-col-dim\",\n                        required=True,\n                        help=\"Provide feature_dim\")\n    parser.add_argument(\"--rhs-col-dim\",\n                        required=True,\n                        help=\"Provide output_dim\")\n    parser.add_argument(\"--density\",\n                        required=True,\n                        help=\"Density for lhs\")\n    parser.add_argument(\"--num-omp-threads\", type=int,\n                        default=1, help=\"number of omp threads to set in MXNet\")\n    parser.add_argument(\"--lhs-stype\", default=\"csr\",\n                        choices=[\"csr\", \"default\", \"row_sparse\"],\n                        help=\"stype for lhs\",\n                        required=True)\n    parser.add_argument(\"--rhs-stype\", default=\"default\",\n                        choices=[\"default\", \"row_sparse\"],\n                        help=\"rhs stype\",\n                        required=True)\n    parser.add_argument(\"--only-storage\",\n                        action=\"store_true\",\n                        help=\"only storage\")\n    parser.add_argument(\"--rhs-density\",\n                        help=\"rhs_density\")\n    return parser.parse_args()\n\n\ndef main():\n    args = parse_args()\n    lhs_row_dim = int(args.lhs_row_dim)\n    lhs_col_dim = int(args.lhs_col_dim)\n    rhs_col_dim = int(args.rhs_col_dim)\n    density = float(args.density)\n    lhs_stype = args.lhs_stype\n    rhs_stype = args.rhs_stype\n    if args.rhs_density:\n        rhs_density = float(args.rhs_density)\n    else:\n        rhs_density = density\n    dot_func = mx.nd.sparse.dot if lhs_stype == \"csr\" else mx.nd.dot\n    check_call(_LIB.MXSetNumOMPThreads(ctypes.c_int(args.num_omp_threads)))\n    bench_dot(lhs_row_dim, lhs_col_dim, rhs_col_dim, density,\n              rhs_density, dot_func, False, lhs_stype, rhs_stype, args.only_storage)\n\ndef bench_dot(lhs_row_dim, lhs_col_dim, rhs_col_dim, density,\n              rhs_density, dot_func, trans_lhs, lhs_stype,\n              rhs_stype, only_storage, distribution=\"uniform\"):\n    \"\"\" Benchmarking both storage and dot\n    \"\"\"\n    lhs_nd = rand_ndarray((lhs_row_dim, lhs_col_dim), lhs_stype, density, distribution=distribution)\n    if not only_storage:\n        rhs_nd = rand_ndarray((lhs_col_dim, rhs_col_dim), rhs_stype,\n                              density=rhs_density, distribution=distribution)\n        out = dot_func(lhs_nd, rhs_nd, trans_lhs)\n    mx.nd.waitall()\n\n\nif __name__ == '__main__':\n    sys.exit(main())\n"
  },
  {
    "path": "benchmark/python/sparse/sparse_op.py",
    "content": "from __future__ import print_function\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport ctypes\n\nfrom mxnet.test_utils import *\nimport scipy.sparse as sp\nimport os\nimport time\nimport argparse\n\nfrom mxnet.base import check_call, _LIB\nfrom mxnet.test_utils import get_bz2_data\nfrom util import estimate_density\n\nparser = argparse.ArgumentParser(description=\"Benchmark sparse operators\",\n                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)\nparser.add_argument('--num-omp-threads', type=int, default=1, help='number of omp threads to set in MXNet')\nargs = parser.parse_args()\n\n# some data information\nkdda = {\n    'data_mini': 'kdda.t.mini',\n    'data_name': 'kdda.t',\n    'data_origin_name': 'kdda.t.bz2',\n    'url': \"https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/kdda.t.bz2\",\n    'feature_dim': 20216830,\n    'm': 200,\n    'batch_size': [64]\n}\n\navazu = {\n    'data_mini': 'avazu-app.t.mini',\n    'data_name': 'avazu-app.t',\n    'data_origin_name': 'avazu-app.t.bz2',\n    'url': \"https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/avazu-app.t.bz2\",\n    'feature_dim': 1000000,\n    'm': 500,\n    'batch_size': [64, 128]\n}\n\n\ndef measure_cost(repeat, f, *args, **kwargs):\n    # start bench\n    start = time.time()\n    results = []\n    for i in range(repeat):\n        results.append(f(*args, **kwargs))\n    for result in results:\n        result.wait_to_read()\n    end = time.time()\n    diff = end - start\n    return diff / repeat\n\n\ndef test_dot_real(data_dict):\n    def get_iter(path, data_shape, batch_size):\n        data_train = mx.io.LibSVMIter(data_libsvm=path,\n                                      data_shape=data_shape,\n                                      batch_size=batch_size)\n        data_iter = iter(data_train)\n        return data_iter\n\n    data_dir = os.path.join(os.getcwd(), 'data')\n\n    path = os.path.join(data_dir, data_dict['data_name'])\n    if not os.path.exists(path):\n        get_bz2_data(\n            data_dir,\n            data_dict['data_name'],\n            data_dict['url'],\n            data_dict['data_origin_name']\n        )\n        assert os.path.exists(path)\n\n    k = data_dict['feature_dim']\n    m = data_dict['m']\n    density = estimate_density(path, data_dict['feature_dim'])\n\n    mini_path = os.path.join(data_dir, data_dict['data_mini'])\n    if not os.path.exists(mini_path):\n        os.system(f\"head -n 2000 {repr(path)} > {repr(mini_path)}\")\n        assert os.path.exists(mini_path)\n\n    print(f\"Running Benchmarking on {repr(data_dict['data_mini'])} data\")\n    for batch_size in data_dict['batch_size']:  # iterator through different batch size of choice\n        print(f\"batch_size is {batch_size}\")\n        # model\n        data_shape = (k, )\n        train_iter = get_iter(mini_path, data_shape, batch_size)\n        weight = mx.nd.random.uniform(low=0, high=1, shape=(k, m))\n\n        csr_data = []\n        dns_data = []\n        num_batch = 0\n        for batch in train_iter:\n            data = train_iter.getdata()\n            csr_data.append(data)\n            dns_data.append(data.tostype('default'))\n            num_batch += 1\n        bag_of_data = [csr_data, dns_data]\n        num_repeat = 5\n        costs = []\n        for d in bag_of_data:\n            weight.wait_to_read()\n            cost = 0.\n            count = 0\n            for d_batch in d:\n                d_batch.wait_to_read()\n                cost += measure_cost(num_repeat, mx.nd.dot, d_batch, weight)\n                count += 1\n            costs.append(cost/count)\n        t_sparse = costs[0]\n        t_dense = costs[1]\n        ratio = t_dense / t_sparse\n        print('density(%)\\tn\\tm\\tk\\tt_dense/t_sparse\\tt_dense\\tt_sparse')\n        fmt = \"{:0.4f}\\t\\t{}\\t{}\\t{}\\t{:0.2f}\\t\\t\\t{:0.4f}\\t{:0.6f}\"\n        print(fmt.format(density * 100, batch_size, m, k, ratio, t_dense, t_sparse))\n\n\ndef test_dot_synthetic():\n    \"\"\"benchmark mx.nd.dot(sparse_ndarray, dense_ndarray) with given density.\n    `t_sparse` is the time cost of dot(csr, dns), while `t_dense` is the time cost\n    of dot(dns, dns), with the same matrix except that it is in default storage type.\n    \"\"\"\n    def measure_cost_forward_baseline(repeat, dot, lhs, rhs):\n        start = time.time()\n        for i in range(repeat):\n            dot(lhs, rhs)\n        end = time.time()\n        diff = end - start\n        return diff / repeat\n\n    def measure_cost_backward_baseline(repeat, dot, transpose, lhs, rhs):\n        start = time.time()\n        for i in range(repeat):\n            dot(transpose(lhs), rhs)\n        end = time.time()\n        diff = end - start\n        return diff / repeat\n\n    def bench_dot_forward(m, k, n, density, ctx, repeat):\n        set_default_device(ctx)\n        dns = mx.nd.random.uniform(shape=(k, n)).copyto(ctx)\n        data_shape = (m, k)\n        csr_data = rand_ndarray(data_shape, 'csr', density)\n        dns_data = csr_data.tostype('default')\n        rhs_dns_np = dns.asnumpy()\n        lhs_csr_sp = sp.csr_matrix(dns_data.asnumpy())  # csr in scipy\n        lhs_dns_np = lhs_csr_sp.tostype('default')\n\n        data = [dns_data, csr_data]\n        costs = []\n        for d in data:\n            dns.wait_to_read()\n            d.wait_to_read()\n            cost = measure_cost(repeat, mx.nd.dot, d, dns)\n            costs.append(cost)\n        ratio = costs[0] / costs[1]\n\n        costs_baseline = []\n        cost = measure_cost_forward_baseline(repeat, np.dot, lhs_dns_np, rhs_dns_np)\n        costs_baseline.append(cost)\n        cost = measure_cost_forward_baseline(repeat, sp.spmatrix.dot, lhs_csr_sp, rhs_dns_np)\n        costs_baseline.append(cost)\n        ratio_baseline = costs_baseline[0] / costs_baseline[1]\n        fmt = \"{:0.1f}\\t\\t{}\\t{}\\t{}\\t{}\\t{:0.2f}\\t\\t\\t{:0.2f}\\t{:0.5f}\\t\\t{:0.2f}\\t\\t\\t\\t{:0.6f}\\t{:0.5f}\"\n        print(fmt.format(density * 100, str(ctx), n, m, k, ratio, costs[0], costs[1],\n                     ratio_baseline, costs_baseline[0], costs_baseline[1]))\n\n    def bench_dot_backward(m, k, n, density, ctx, repeat):\n        set_default_device(ctx)\n        dns = mx.nd.random.uniform(shape=(m, n)).copyto(ctx)\n        data_shape = (m, k)\n        csr_data = rand_ndarray(data_shape, 'csr', density)\n        dns_data = csr_data.tostype('default')\n        rhs_dns_np = dns.asnumpy()\n        lhs_csr_sp = sp.csr_matrix(dns_data.asnumpy())\n        lhs_dns_np = lhs_csr_sp.tostype('default')\n\n        data = [dns_data, csr_data]\n        costs = []\n        for d in data:\n            dns.wait_to_read()\n            d.wait_to_read()\n            cost = measure_cost(repeat, mx.nd.dot, d, dns, transpose_a=True)\n            costs.append(cost)\n        ratio = costs[0] / costs[1]\n\n        costs_baseline = []\n        cost = measure_cost_backward_baseline(repeat, np.dot, np.transpose, lhs_dns_np, rhs_dns_np)\n        costs_baseline.append(cost)\n        cost = measure_cost_backward_baseline(repeat, sp.spmatrix.dot, sp.spmatrix.transpose, lhs_csr_sp, rhs_dns_np)\n        costs_baseline.append(cost)\n        ratio_baseline = costs_baseline[0] / costs_baseline[1]\n        fmt = \"{:0.1f}\\t\\t{}\\t{}\\t{}\\t{}\\t{:0.2f}\\t\\t\\t{:0.2f}\\t{:0.5f}\\t\\t{:0.2f}\\t\\t\\t\\t{:0.6f}\\t{:0.5f}\"\n        print(fmt.format(density * 100, str(ctx), n, m, k, ratio, costs[0], costs[1],\n                     ratio_baseline, costs_baseline[0], costs_baseline[1]))\n\n    print(\"A = sparse NDArray of shape(m, k)\")\n    print(\"B = dense NDArray of shape(k, n)\")\n    print(\"dot_forward\\tdot(csr, dns)\")\n    print('density(%)\\tcontext\\tn\\tm\\tk\\tt_dense/t_sparse\\tt_dense\\tt_sparse'\n          '\\tt_scipy_dense/t_scipy_sparse\\tt_scipy_dense\\tt_scipy_sparse')\n\n    check_call(_LIB.MXSetNumOMPThreads(ctypes.c_int(args.num_omp_threads)))\n    # TODO(haibin) make these runtime options\n    m = 512\n    k = [50000, 100000]\n    n = [64, 128]\n    density = [1.00, 0.90, 0.70, 0.50, 0.30, 0.20, 0.10, 0.07, 0.05, 0.02, 0.01, 0.005, 0.001]\n    num_repeat = 10\n    # contexts = [mx.cpu(), mx.gpu(0)]\n    contexts = [mx.cpu()]\n    for i in range(2):\n        for ctx in contexts:\n            for den in density:\n                bench_dot_forward(m, k[i], n[i], den, ctx, num_repeat)\n\n    print(\"dot_backward\\tdot(csr.T, dns)\")\n    print('density(%)\\tcontext\\tn\\tm\\tk\\tt_dense/t_sparse\\tt_dense\\tt_sparse'\n          '\\tt_scipy_dense/t_scipy_sparse\\tt_scipy_dense\\tt_scipy_sparse')\n    for i in range(2):\n        for ctx in contexts:\n            for den in density:\n                bench_dot_backward(m, k[i], n[i], den, ctx, num_repeat)\n\n\nif __name__ == \"__main__\":\n    test_dot_real(avazu)\n    test_dot_real(kdda)\n    test_dot_synthetic()\n"
  },
  {
    "path": "benchmark/python/sparse/updater.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport time\nimport mxnet as mx\nfrom mxnet.ndarray.sparse import adam_update\nimport numpy as np\nimport argparse\n\nmx.random.seed(0)\nnp.random.seed(0)\n\nparser = argparse.ArgumentParser(description='Benchmark adam updater')\nparser.add_argument('--dim-in', type=int, default=240000, help='weight.shape[0]')\nparser.add_argument('--dim-out', type=int, default=512, help='weight.shape[1]')\nparser.add_argument('--nnr', type=int, default=5000, help='grad.indices.shape[0]')\nparser.add_argument('--repeat', type=int, default=1000, help='num repeat')\nparser.add_argument('--dense-grad', action='store_true',\n                    help='if set to true, both gradient and weight are dense.')\nparser.add_argument('--dense-state', action='store_true',\n                    help='if set to true, states are dense, indicating standard update')\nparser.add_argument('--cpu', action='store_true')\n\n\nargs = parser.parse_args()\ndim_in = args.dim_in\ndim_out = args.dim_out\nnnr = args.nnr\nctx = mx.cpu() if args.cpu else mx.gpu()\n\nones = mx.nd.ones((dim_in, dim_out), ctx=ctx)\n\nif not args.dense_grad:\n    weight = ones.tostype('row_sparse')\n    indices = np.arange(dim_in)\n    np.random.shuffle(indices)\n    indices = np.unique(indices[:nnr])\n    indices = mx.nd.array(indices, ctx=ctx)\n    grad = mx.nd.sparse.retain(weight, indices)\nelse:\n    weight = ones.copy()\n    grad = ones.copy()\n\nif args.dense_state:\n    mean = ones.copy()\nelse:\n    mean = ones.tostype('row_sparse')\n\nvar = mean.copy()\n\n# warmup \nfor i in range(10):\n    adam_update(weight, grad, mean, var, out=weight, lr=1, wd=0, beta1=0.9,\n                beta2=0.99, rescale_grad=0.5, epsilon=1e-8)\nweight.wait_to_read()\n\n# measure speed\na = time.time()\nfor i in range(args.repeat):\n    adam_update(weight, grad, mean, var, out=weight, lr=1, wd=0, beta1=0.9,\n                beta2=0.99, rescale_grad=0.5, epsilon=1e-8)\nweight.wait_to_read()\nb = time.time()\nprint(b - a)\n"
  },
  {
    "path": "benchmark/python/sparse/util.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport os\nimport random\n\ndef estimate_density(DATA_PATH, feature_size):\n    \"\"\"sample 10 times of a size of 1000 for estimating the density of the sparse dataset\"\"\"\n    if not os.path.exists(DATA_PATH):\n        raise Exception(\"Data is not there!\")\n    density = []\n    P = 0.01\n    for _ in range(10):\n        num_non_zero = 0\n        num_sample = 0\n        with open(DATA_PATH) as f:\n            for line in f:\n                if (random.random() < P):\n                    num_non_zero += len(line.split(\" \")) - 1\n                    num_sample += 1\n        density.append(num_non_zero * 1.0 / (feature_size * num_sample))\n    return sum(density) / len(density)\n\n"
  },
  {
    "path": "benchmark/python/tvmop/benchmark_tvmop.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport time\nimport mxnet as mx\nimport numpy as _np\nfrom mxnet import np, npx\n\ndef measure_cost(repeat, func_name, *args, **kwargs):\n    \"\"\"Measure time cost of running a function\n    \"\"\"\n    mx.nd.waitall()\n    start = time.time()\n    for _ in range(repeat):\n        func_name(*args, **kwargs)\n    mx.nd.waitall()\n    end = time.time()\n    diff = end - start\n    return diff / repeat\n\n\ndef test_tvm_dot():\n    # benchmark\n    for i in list(range(1000, 1100, 4)):\n        m = i\n        k = i\n        n = i\n        print(\"{} * {} X {} * {}\".format(m, k, k, n))\n        a = mx.nd.random.uniform(shape=(m, k), dtype='float32')\n        b = mx.nd.random.uniform(shape=(k, n), dtype='float32')\n        cost = measure_cost(2, mx.nd.contrib.tvm_dot, a, b)\n        print(\"dispatch cost: {} ms\".format(cost * 1000))\n        a = mx.nd.random.uniform(shape=(m, k), dtype='float32')\n        b = mx.nd.random.uniform(shape=(k, n), dtype='float32')\n        cost = measure_cost(2, mx.nd.contrib.tvm_dot_fallback, a, b)\n        print(\"fallback cost: {} ms\".format(cost * 1000))\n        a = mx.nd.random.uniform(shape=(m, k), dtype='float32')\n        b = mx.nd.random.uniform(shape=(k, n), dtype='float32')\n        cost = measure_cost(2, mx.nd.dot, a, b)\n        print(\"dot cost: {} ms\".format(cost * 1000))\n\nif __name__ == \"__main__\":\n    test_tvm_dot()\n"
  },
  {
    "path": "cd/Jenkinsfile_cd_pipeline",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// MXNet Continuous Delivery Pipeline\n// Orchestrates the release of the different artifact by calling downstream release jobs.\n\npipeline {\n  agent {\n    label 'restricted-utility'\n  }\n  options {\n    // Because each pass of the CD pipeline\n    // updates Jenkins' state of the release job\n    // to avoid crazy issues, we don't allow concurrent builds.\n    disableConcurrentBuilds()\n  }\n\n  parameters {\n    // Release parameters\n    string(defaultValue: \"cpu,native,cu101,cu102,cu110,cu112\", description: \"Comma separated list of variants\", name: \"MXNET_VARIANTS\")\n    booleanParam(defaultValue: false, description: 'Whether this is a release build or not', name: \"RELEASE_BUILD\")\n  }\n\n  stages {\n    stage(\"Init\") {\n      steps {\n        script {\n          cd_utils = load('cd/Jenkinsfile_utils.groovy')\n          // Update release job state in Jenkins\n          cd_utils.update_release_job_state(params.CD_RELEASE_JOB_NAME)\n        }\n      }\n    }\n\n    stage(\"MXNet Release\") {\n      steps {\n        script {\n          stage(\"Build libmxnet\") {\n            cd_utils.trigger_release_job(params.CD_RELEASE_JOB_NAME, \"Build libmxnet\", \"mxnet_lib\", params.MXNET_VARIANTS)\n          }\n          stage(\"Releases\") {\n            cd_utils.error_checked_parallel([\n              \"PyPI Release\": {\n                echo \"Building PyPI Release\"\n                cd_utils.trigger_release_job(params.CD_RELEASE_JOB_NAME, \"Release PyPI Packages\", \"python/pypi\", params.MXNET_VARIANTS)\n              },\n              \"Python Docker Release\": {\n                echo \"Building Python Docker Release\"\n                cd_utils.trigger_release_job(params.CD_RELEASE_JOB_NAME, \"Release Python Docker Images\", \"python/docker\", params.MXNET_VARIANTS)\n              }\n            ])\n          }\n        }\n      }\n    }\n  }\n}\n"
  },
  {
    "path": "cd/Jenkinsfile_release_job",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// This job executes a \"generic\" (CD) release job.\n// The pipeline to be executed is defined by the \"RELEASE_JOB_TYPE\" pipeline parameter.\n// This is the path (relative to the CD directory) to a directory containing the following file groovy script\n// `Jenkins_pipeline.groovy`.\n// See `cd` directory README for more details.\n\n// timeout of each step in minutes\nmax_time = 180\n\npipeline {\n  agent {\n    label 'restricted-utility'\n  }\n\n  parameters {\n    // Release parameters\n    string(defaultValue: \"Generic release job\", description: \"Optional Job name\", name: \"RELEASE_JOB_NAME\")\n    string(defaultValue: \"master\", description: \"Git Commit to Build\", name: \"COMMIT_ID\")\n\n    // Using string instead of choice parameter to keep the changes to the parameters minimal to avoid\n    // any disruption caused by different COMMIT_ID values chaning the job parameter configuration on\n    // Jenkins.\n    string(defaultValue: \"mxnet_lib\", description: \"Pipeline to build\", name: \"RELEASE_JOB_TYPE\")\n    string(defaultValue: \"cpu,native,cu101,cu102,cu110,cu112\", description: \"Comma separated list of variants\", name: \"MXNET_VARIANTS\")\n    booleanParam(defaultValue: false, description: 'Whether this is a release build or not', name: \"RELEASE_BUILD\")\n    string(defaultValue: \"nightly\", description: \"String used for naming docker images\", name: \"VERSION\")\n  }\n\n  stages {\n    stage(\"Init\") {\n      steps {\n        script {\n          cd_utils = load('cd/Jenkinsfile_utils.groovy')\n          ci_utils = load('ci/Jenkinsfile_utils.groovy')\n          ci_utils.assign_node_labels(\n            utility: 'restricted-utility',\n            linux_cpu: 'restricted-mxnetlinux-cpu',\n            linux_gpu: 'restricted-mxnetlinux-gpu',\n            linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3',\n            windows_cpu: 'restricted-mxnetwindows-cpu',\n            windows_gpu: 'restricted-mxnetwindows-gpu'\n          )\n\n          // Skip Jenkins state update jobs\n          if (env.RELEASE_JOB_TYPE == cd_utils.STATE_UPDATE) {\n            echo \"\"\"\\\n            |Job Type: ${env.RELEASE_JOB_TYPE}\n            |Commit Id: ${env.GIT_COMMIT}\"\"\".stripMargin()\n          } else {\n            echo \"\"\"\\\n            |Job Name: ${env.RELEASE_JOB_NAME}\n            |Job Type: ${env.RELEASE_JOB_TYPE}\n            |Release Build: ${params.RELEASE_BUILD}\n            |Commit Id: ${env.GIT_COMMIT}\n            |Branch: ${env.GIT_BRANCH}\n            |Version: ${VERSION}\n            |Variants: ${params.MXNET_VARIANTS}\"\"\".stripMargin()\n          }\n        }\n      }\n    }\n    stage(\"Release Job\") {\n      steps {\n        script {\n          // Skip builds for state update job\n          if (env.RELEASE_JOB_TYPE == cd_utils.STATE_UPDATE) {\n            currentBuild.result = \"SUCCESS\"\n            return\n          }\n\n          // Add new job types here\n          def valid_job_types = [\n            \"mxnet_lib\",\n            \"python/pypi\",\n            \"python/docker\"\n          ]\n\n          // Convert mxnet variants to a list\n          def mxnet_variants = params.MXNET_VARIANTS.trim().split(',').inject([]) { list, item ->\n            list << item.trim()\n          }.findAll { item -> ! (item == null || item.isEmpty()) }\n\n          // Exit successfully if there are no variants to build\n          if (mxnet_variants.size() == 0) {\n            error \"No variants to build...\"\n          }\n\n          // Only execute from allowed release job types\n          if (! (valid_job_types.contains(params.RELEASE_JOB_TYPE))) {\n            error \"Unknown release job type '${params.RELEASE_JOB_TYPE}'\"\n          }\n\n          // Load script for the supplied job type\n          def custom_steps = load(\"cd/${params.RELEASE_JOB_TYPE}/Jenkins_pipeline.groovy\")\n\n          // Extract the pipelines for the variants\n          def pipelines = [:]\n\n          for (variant in mxnet_variants) {\n            pipelines << [\"${variant}\": custom_steps.get_pipeline(variant)]\n          }\n\n          // Execute them in parallel\n          // The build result will be set to:\n          //  - SUCCESS if all pipelines succeed\n          //  - UNSTABLE if some (but not all) pipelines fail\n          //  - FAILURE if all pipelines fail\n          cd_utils.error_checked_parallel(pipelines)\n        }\n      }\n    }\n  }\n}\n"
  },
  {
    "path": "cd/Jenkinsfile_utils.groovy",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n\n// Triggers a downstream jenkins job responsible for building, testing\n// and publishing all the variants for a particular 'job_type'.\n// The 'job_type' should be the name of the directory that contains the\n// 'Jenkins_pipeline.groovy' file and has the pipeline definition for the\n// artifact (docker image, binary, pypi or maven package, etc.) that should\n// be published.\n\nSTATE_UPDATE=\"State Update\"\n\ndef trigger_release_job(cd_release_job, job_name, job_type, mxnet_variants) {\n  def run = build(\n    job: cd_release_job,\n    parameters: [\n      string(name: \"RELEASE_JOB_NAME\", value: \"${job_name}\"),\n      string(name: \"RELEASE_JOB_TYPE\", value: \"${job_type}\"),\n      string(name: \"MXNET_VARIANTS\", value: \"${mxnet_variants}\"),\n      booleanParam(name: \"RELEASE_BUILD\", value: \"${env.RELEASE_BUILD}\"),\n      string(name: \"VERSION\", value: \"${env.VERSION}\"),\n      string(name: \"COMMIT_ID\", value: \"${env.GIT_COMMIT}\")\n    ],\n    // If propagate is true, any result other than successful will\n    // mark this call as failure (inc. unstable).\n    // https://jenkins.io/doc/pipeline/steps/pipeline-build-step\n    propagate: false)\n\n  def result = run.getResult()\n\n  // In case the underlying release job is unstable,\n  // e.g. one or more (but not all) the variants failed, or;\n  // it is aborted (e.g. one of steps timed out),\n  // continue with the pipeline and try to post as many releases as possible\n  // but mark it as unstable\n  if (result == \"UNSTABLE\" || result == \"ABORTED\") {\n    currentBuild.result = \"UNSTABLE\"\n  }\n\n  // Throw an exception on failure, because this would mean the whole\n  // pipeline failed (i.e. for every variant)\n  if (result == \"FAILURE\") {\n    error \"Downstream job: ${job_name} failed\"\n  }\n}\n\n\n// This triggers a downstream release job with no\n// variants and not job type. This will update\n// the configuration of the release job in jenkins\n// to the configuration of release job as defined in the\n// Jenkinsfile _release_job for env.GIT_COMMIT revision\ndef update_release_job_state(cd_release_job) {\n  build(\n    job: cd_release_job,\n    parameters: [\n      string(name: \"RELEASE_JOB_TYPE\", value: STATE_UPDATE),\n\n      // Should be set to the current git commit\n      string(name: \"COMMIT_ID\", value: \"${env.GIT_COMMIT}\")\n    ])\n}\n\n// Wraps variant pipeline with error catching and\n// job status setting code\n// If there's an error in one of the pipelines, set status to UNSTABLE\n// If all pipelines fail, set to FAILURE\n// This is to be used in conjunction with the error_checked_parallel\ndef wrap_variant_pipeline_fn(variant_pipeline, total_num_pipelines) {\n  // do not add def - seems to affect the scope\n  count = 0\n  return {\n    try {\n      variant_pipeline()\n    } catch (ex) {\n      count++\n      currentBuild.result = \"UNSTABLE\"\n\n      if (count == total_num_pipelines) {\n        currentBuild.result = \"FAILURE\"\n        throw ex\n      }\n    }\n  }\n}\n\n// Takes a map of key -> closure values to be executed in parallel.\n// The outcome of the execution of each parallel step will affect\n// the result (SUCCESS, FAILURE, ABORTED, UNSTABLE) of the overall job.\n// If all steps fail or are aborted, the job will be set to failed.\n// If some steps fail or are aborted, the job will be set to unstable.\ndef error_checked_parallel(variant_pipelines) {\n  pipelines = variant_pipelines.inject([:]) { mp, key, value ->\n    mp << [\"${key}\": wrap_variant_pipeline_fn(value, variant_pipelines.size())]\n  }\n  parallel pipelines\n}\n\n// pushes artifact to repository\ndef push_artifact(libmxnet_path, variant, libtype, license_paths = '', dependency_paths = '') {\n  if(license_paths == null) license_paths = ''\n  if(dependency_paths == null) dependency_paths = ''\n\n  sh \"python3 ./cd/utils/artifact_repository.py --push --verbose --libtype ${libtype} --variant ${variant} --libmxnet ${libmxnet_path} --licenses ${license_paths} --dependencies ${dependency_paths} --os ubuntu18.04\"\n}\n\n// pull artifact from repository\ndef pull_artifact(variant, libtype, destination = '') {\n  sh \"python3 ./cd/utils/artifact_repository.py --pull --verbose --libtype ${libtype} --variant ${variant} --destination ${destination} --os ubuntu18.04\"\n}\n\n// pulls artifact from repository and places files in the appropriate directories\ndef restore_artifact(variant, libtype) {\n\n  pull_artifact(variant, libtype, 'mxnet_artifact')\n\n  // move libraries to lib directory\n  dir('lib') {\n    sh \"mv ../mxnet_artifact/libmxnet.so .\"\n    if (fileExists('../mxnet_artifact/dependencies')) {\n      sh \"\"\"find \"../mxnet_artifact/dependencies\" -type f -name \"*.so*\" -exec mv {} . \\\\;\"\"\"\n      sh \"ls .\"\n    }\n  }\n\n  dir('cd_misc') {\n    if (fileExists('../mxnet_artifact/dependencies')) {\n      // All library files (*.so*) should have be moved\n      // to the lib directory. If anything is left, it will be\n      // other supporting files (header files, etc.)\n      sh \"\"\"find \"../mxnet_artifact/dependencies\" -type f -exec mv {} . \\\\;\"\"\"\n      sh \"ls .\"\n    }\n  }\n\n  dir('licenses') {\n    if (fileExists('../mxnet_artifact/licenses')) {\n      sh \"\"\"find \"../mxnet_artifact/licenses\" -type f -exec mv {} . \\\\;\"\"\"\n      sh \"ls .\"\n    }\n  }\n\n  dir('mxnet_artifact') {\n    deleteDir()\n  }\n}\n\n\n// Restores the statically linked libmxnet for the given variant\ndef restore_static_libmxnet(variant) {\n  restore_artifact(variant, 'static')\n}\n\n\n// Restores the dynamically linked libmxnet for the given variant\ndef restore_dynamic_libmxnet(variant) {\n  restore_artifact(variant, 'dynamic')\n}\n\n// A generic pipeline that can be used by *most* CD jobs\n// It can be used when implementing the pipeline steps in the Jenkins_steps.groovy\n// script for a particular delivery channel. However, it should also implement the\n// build, test, and push steps.\n// NOTE: Be mindful of the expected time that a step should take. If it will take a long time,\n// and it can be done in a CPU node, do it in a CPU node. We should avoid using GPU instances unless\n// we *have* to.\n// However, if it is only packaging libmxnet and that doesn't take long. Then, the pipeline can\n// just run on a single node. As is done bellow.\n// For examples of multi-node CD pipelines, see the the binary_release/static and binary_release/dynamic\n// pipeline.\ndef generic_pipeline(mxnet_variant, custom_steps, node_type = \"restricted-mxnetlinux-cpu\") {\n  return {\n    node(node_type) {\n      stage(\"${mxnet_variant}\") {\n\n        stage('Build') {\n          custom_steps.build(mxnet_variant)\n        }\n\n        stage('Test') {\n          custom_steps.test(mxnet_variant)\n        }\n\n        stage('Push') {\n          custom_steps.push(mxnet_variant)\n        }\n      }\n    }\n  }\n}\n\nreturn this\n"
  },
  {
    "path": "cd/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# MXNet Continuous Delivery\n\n## Introduction\n\nMXNet aims to support a variety of frontends, e.g. Python, Java, Perl, R, etc. as well as environments (Windows, Linux, Mac, with or without GPU, with or without oneDNN support, etc.). This package contains a small continuous delivery (CD) framework used to automate the delivery nightly and release builds across our delivery channels.\n\n<!-- TODO: Add links to the actual jobs, once this is live on PROD -->\n\nThe CD process is driven by the [CD pipeline job](Jenkinsfile_cd_pipeline), which orchestrates the order in which the artifacts are delivered. For instance, first publish the libmxnet library before publishing the pip package. It does this by triggering the [release job](Jenkinsfile_release_job) with a specific set of parameters for each delivery channel. The release job executes the specific release pipeline for a delivery channel across all MXNet *variants*.\n\nA variant is a specific environment or features for which MXNet is compiled. For instance CPU, GPU with CUDA v10.1, CUDA v10.2 with oneDNN support, etc.\n\nCurrently, below variants are supported. All of these variants except native have oneDNN backend enabled.\n\n* *cpu*: CPU\n* *native*: CPU without oneDNN\n* *cu101*: CUDA 10.1\n* *cu102*: CUDA 10.2\n* *cu110*: CUDA 11.0\n* *cu112*: CUDA 11.2\n\n*For more on variants, see [here](https://github.com/apache/mxnet/issues/8671)*\n\n## Framework Components\n\n### CD Pipeline Job\n\nThe [CD pipeline job](Jenkinsfile_cd_pipeline) take two parameters:\n\n * **RELEASE_BUILD**: Flags the run as a *release build*. The underlying jobs can then use this environment variable to disambiguate between nightly and release builds. Defaults to *false*.\n * **MXNET_VARIANTS**: A comma separated list of variants to build. Defaults to *all* variants.\n\nThis job defines and executes the CD pipeline. For example, first publish the MXNet library, then, in parallel, execute the python and maven releases. Every step of the pipeline executes a trigger for a [release job](Jenkinsfile_release_job).\n\n### Release Job\n\nThe [release job](Jenkinsfile_release_job) takes five parameters:\n\n * **RELEASE_BUILD**: Flags the run as a *release build*. The underlying jobs can then use this environment variable to disambiguate between nightly and release builds. Defaults to *false*.\n * **MXNET_VARIANTS**: A comma separated list of variants to build. Defaults to *all* variants.\n * **RELEASE\\_JOB\\_NAME**: A name for this release job (Optional). Defaults to \"Generic release job\". It is used for debug output purposes.\n * **RELEASE\\_JOB\\_TYPE**: Defines the release pipeline you want to execute.\n * **COMMIT_ID**: The commit id to build\n\nThe release job executes, in parallel, the release pipeline for each of the variants (**MXNET_VARIANTS**) for the job type (**RELEASE\\_JOB\\_TYPE**). The job type the path to a directory (relative to the `cd` directory) that includes a `Jenkins_pipeline.groovy` file ([e.g.](mxnet_lib/Jenkins_pipeline.groovy)).\n\nNOTE: The **COMMIT_ID** is a little tricky and we must be very careful with it. It is necessary to ensure that the same commit is built through out the pipeline, but at the same time, it has the potential to change the current state of the release job configuration - specifically the parameter configuration. Any changes to this configuration will require a \"dry-run\" of the release job to ensure Jenkins has the current (master) version. This is acceptable as there will be few changes to the parameter configuration for the job, if any at all. But, it's something to keep in mind.\n\nTo avoid potential issues as much as possible, the CD pipeline executes this \"dry run\" and ensures that Jenkins' state of the release job matches what is defined for the release job in the specified COMMIT_ID. This is done by setting the **RELEASE_JOB_TYPE** to *Status Update*.\n\nIt should be noted that the 'Pipeline' section of the configuration should use the *$COMMIT_ID* parameter as the specifier and 'lightweight checkout' unchecked. For example:\n\n![job setup example](img/job_setup.png)\n\n### Release Pipelines: Jenkins_pipeline.groovy\n\nThis file defines the release pipeline for a particular release channel. It defines a function `get_pipeline(mxnet_variant)`, which returns a closure with the pipeline to be executed. For instance:\n\n```\ndef get_pipeline(mxnet_variant) {\n  return {\n    stage(\"${mxnet_variant}\") {\n      stage(\"Build\") {\n        timeout(time: max_time, unit: 'MINUTES') {\n          build(mxnet_variant)\n        }\n      }\n      stage(\"Test\") {\n        timeout(time: max_time, unit: 'MINUTES') {\n          test(mxnet_variant)\n        }\n      }\n      stage(\"Publish\") {\n        timeout(time: max_time, unit: 'MINUTES') {\n          publish(mxnet_variant)\n        }\n      }\n    }\n  }\n}\n\ndef build(mxnet_variant) {\n  node(UBUNTU_CPU) {\n    ...\n  }\n}\n...\n```\n\n## Binary Releases\n\nThe \"first mile\" of the CD process is posting the mxnet binaries to the [artifact repository](utils/artifact_repository.md). Once this step is complete, the pipelines for the different release channels (PyPI, Maven, etc.) can begin from the compiled binary, and focus solely on packaging it, testing the package, and posting it to the particular distribution channel.\n\n<!-- TODO: Once all the artifact repository Jenkins utility functions are in, list them here -->\n\n## Adding New Release Pipelines\n\n1. Create a directory under `cd` which represents your release channel, e.g. `python/pypi`.\n2. Add a `Jenkins_pipeline.groovy` there with a `get_pipeline(mxnet_variant)` function that describes your pipeline.\n3. Add a call to your pipeline to the [CD pipeline job](Jenkinsfile_cd_pipeline).\n\n#### General Guidelines:\n\n##### Timeout\n\nWe shouldn't set global timeouts for the pipelines. Rather, the `step` being executed should be rapped with a `timeout` function (as in the pipeline example above). The `max_time` is a global variable set at the [release job](Jenkinsfile_release_job) level.\n\n##### Node of execution\n\nEnsure that either your steps, or the whole pipeline are wrapped in a `node` call. The jobs execute in an `utility` node. If you don't wrap your pipeline, or its individual steps, in a `node` call, this will lead to problems.\n\nExamples of the two approaches:\n\n<!-- TODO: Add links to examples once the all pipelines are in -->\n\n**Whole pipeline**\n\nThe release pipeline is executed on a single node, depending on the variant building released.\nThis approach is fine, as long as the stages that don't need specialized hardware (e.g. compilation, packaging, publishing), are short lived.\n\n```\ndef get_pipeline(mxnet_variant) {\n  def node_type = mxnet_variant.startsWith('cu') ? NODE_LINUX_GPU : NODE_LINUX_CPU\n\n  return {\n    node (node_type) {\n      stage(\"${mxnet_variant}\") {\n        stage(\"Build\") {\n          ...\n        }\n        stage(\"Test\") {\n          ...\n        }\n        ...\n      }\n    }\n  }\n}\n```\n\nExamples:\n\n * [PyPI Release](python/pypi/Jenkins_pipeline.groovy): In this pipeline, the majority of time is overwhelmingly spent on testing. Therefore, it should be ok to execute the whole pipeline on a GPU node (i.e. packaging, testing, and publishing).\n\n**Per step**\n\nUse this approach in cases where you have long running stages that don't depend on specialized/expensive hardware.\n\n```\ndef get_pipeline(mxnet_variant) {\n  return {\n    stage(\"${mxnet_variant}\") {\n      stage(\"Build\") {\n        ...\n      }\n      ...\n    }\n  }\n}\n\ndef build(mxnet_variant) {\n  node(UBUNTU_CPU) {\n    ...\n  }\n}\n\ndef test(mxnet_variant) {\n  def node_type = mxnet_variant.startsWith('cu') ? NODE_LINUX_GPU : NODE_LINUX_CPU\n  node(node_type) {\n    ...\n  }\n}\n```\n\nExamples:\n\nThe [libmxnet](mxnet_lib/Jenkins_pipeline.groovy) pipeline has long running compilation and testing stages that **do not** require specialized/expensive hardware (e.g. GPUs). Therefore, as much as possible, it is important to run each stage in on its own node, and design the pipeline to spend the least amount of time possible on expensive hardware. E.g. for GPU builds, only run GPU tests on GPU instances, all other stages can be executed on CPU nodes.\n"
  },
  {
    "path": "cd/mxnet_lib/Jenkins_pipeline.groovy",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// To avoid confusion, please note:\n// ci_utils is loaded by the originating Jenkins job, e.g. jenkins/Jenkinsfile_release_job\n\n// NOTE: the following variables are referenced in the mxnet_lib_pipeline jenkins file imported bellow\n\n// libmxnet location\nlibmxnet = 'lib/libmxnet.so'\n\n// licenses\nlicenses = 'licenses/*'\n\n// libmxnet dependencies\nmx_native_deps = 'lib/libgfortran.so.*, lib/libopenblas.so.0'\nmx_deps = 'lib/libgfortran.so.*, lib/libopenblas.so.0, include/onednn/oneapi/dnnl/dnnl_version.h, include/onednn/oneapi/dnnl/dnnl_config.h'\n\n// library type\n// either static or dynamic - depending on how it links to its dependencies\nlibtype = 'static'\n\nlibmxnet_pipeline = load('cd/mxnet_lib/mxnet_lib_pipeline.groovy')\n\n// Builds the static binary for the specified mxnet variant\ndef build(mxnet_variant) {\n  node(NODE_LINUX_CPU) {\n    ws(\"workspace/mxnet_${libtype}/${mxnet_variant}/${env.BUILD_NUMBER}\") {\n      def image = libmxnet_pipeline.get_environment(mxnet_variant)\n      ci_utils.init_git()\n      ci_utils.docker_run(image, \"build_static_libmxnet ${mxnet_variant}\", false)\n      ci_utils.pack_lib(\"mxnet_${mxnet_variant}\", libmxnet_pipeline.get_stash(mxnet_variant))\n    }\n  }\n}\n\ndef get_pipeline(mxnet_variant) {\n  return libmxnet_pipeline.get_pipeline(mxnet_variant, this.&build)\n}\n\nreturn this\n"
  },
  {
    "path": "cd/mxnet_lib/mxnet_lib_pipeline.groovy",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// To avoid confusion, please note:\n// ci_utils and cd_utils are loaded by the originating Jenkins job, e.g. jenkins/Jenkinsfile_release_job\n\ndef get_pipeline(mxnet_variant, build_fn) {\n  return {\n    stage(\"${mxnet_variant}\") {\n      stage('Build') {\n        timeout(time: max_time, unit: 'MINUTES') {\n          build_fn(mxnet_variant)\n        }\n      }\n\n      stage('Test') {\n        def tests = [:]\n        tests[\"${mxnet_variant}: Python 3\"] = {\n          stage(\"${mxnet_variant}: Python 3\") {\n            timeout(time: max_time, unit: 'MINUTES') {\n              unittest_py3(mxnet_variant)\n            }\n          }\n        }\n\n        parallel tests\n      }\n\n      stage('Push') {\n        timeout(time: max_time, unit: 'MINUTES') {\n          push(mxnet_variant)\n        }\n      }\n    }\n  }\n}\n\n// Returns a string of comma separated resources to be stashed b/w stages\n// E.g. the libmxnet library and any other dependencies\ndef get_stash(mxnet_variant) {\n  def deps = mxnet_variant.endsWith('native') ? mx_native_deps : mx_deps\n  return \"${libmxnet}, ${licenses}, ${deps}\"\n}\n\n// Returns the (Docker) environment for the given variant\n// The environment corresponds to the docker files in the 'docker' directory\ndef get_environment(mxnet_variant) {\n  if (mxnet_variant.startsWith(\"cu\")) {\n    return \"centos7_gpu_${mxnet_variant}\"\n  }\n  return \"centos7_cpu\"\n}\n\n// Returns the variant appropriate jenkins node test in which\n// to run a step\ndef get_jenkins_node_label(mxnet_variant) {\n  if (mxnet_variant.startsWith('cu')) {\n    return NODE_LINUX_GPU\n  }\n  return NODE_LINUX_CPU\n}\n\n// Runs unit tests using python 3\ndef unittest_py3(mxnet_variant) {\n  def node_label = get_jenkins_node_label(mxnet_variant)\n\n  node(node_label) {\n    ws(\"workspace/mxnet_${libtype}/${mxnet_variant}/${env.BUILD_NUMBER}\") {\n      def image = get_environment(mxnet_variant)\n      def use_nvidia_docker = mxnet_variant.startsWith('cu')\n      ci_utils.unpack_and_init(\"mxnet_${mxnet_variant}\", get_stash(mxnet_variant), false)\n      ci_utils.docker_run(image, \"cd_unittest_ubuntu ${mxnet_variant}\", use_nvidia_docker)\n    }\n  }\n}\n\n// Pushes artifact to artifact repository\ndef push(mxnet_variant) {\n  node(NODE_LINUX_CPU) {\n    ws(\"workspace/mxnet_${libtype}/${mxnet_variant}/${env.BUILD_NUMBER}\") {\n      def deps = (mxnet_variant.endsWith('native')? mx_native_deps : mx_deps).replaceAll(',', '')\n      ci_utils.unpack_and_init(\"mxnet_${mxnet_variant}\", get_stash(mxnet_variant), false)\n      cd_utils.push_artifact(libmxnet, mxnet_variant, libtype, licenses, deps)\n    }\n  }\n}\n\nreturn this\n"
  },
  {
    "path": "cd/python/docker/Dockerfile",
    "content": "# -*- mode: dockerfile -*-\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# Python MXNet Dockerfile\n\n# NOTE: Assumes wheel_build directory is the context root when building\n\nARG BASE_IMAGE\nFROM ${BASE_IMAGE}\n\nRUN apt-get update && \\\n    apt-get install -y software-properties-common && \\\n    add-apt-repository -y ppa:deadsnakes/ppa && \\\n    apt-get update && \\\n    apt-get install -y python3.8-dev python3.8-distutils virtualenv wget && \\\n    ln -sf /usr/bin/python3.8 /usr/local/bin/python3 && \\\n    wget -nv https://bootstrap.pypa.io/get-pip.py && \\\n    python3 get-pip.py\n\nRUN apt-get install -y libgomp1 libquadmath0\n\nARG MXNET_COMMIT_ID\nENV MXNET_COMMIT_ID=${MXNET_COMMIT_ID}\n\nRUN mkdir -p /mxnet\nCOPY dist/*.whl /mxnet/.\n\nWORKDIR /mxnet\nRUN WHEEL_FILE=$(ls -t /mxnet | head -n 1) && pip install ${WHEEL_FILE} && rm -f ${WHEEL_FILE}\n\n"
  },
  {
    "path": "cd/python/docker/Dockerfile.test",
    "content": "# -*- mode: dockerfile -*-\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# Python MXNet Dockerfile\n\n# NOTE: Assumes 'ci' directory is root of the context when building\n\nARG BASE_IMAGE\nFROM ${BASE_IMAGE}\n\nARG USER_ID=1001\nARG GROUP_ID=1001\n\nCOPY ./docker/install/ubuntu_adduser.sh /work/ubuntu_adduser.sh\nCOPY ./docker/install/requirements /work/requirements\n\nRUN mkdir -p /work\nRUN /work/ubuntu_adduser.sh\nRUN pip install -r /work/requirements\n\nWORKDIR /work/mxnet\n"
  },
  {
    "path": "cd/python/docker/Jenkins_pipeline.groovy",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// NOTE: \n// ci_utils and cd_utils are loaded by the originating Jenkins job, e.g. jenkins/Jenkinsfile_release_job\n\ndef get_pipeline(mxnet_variant) {\n  def node_type = mxnet_variant.startsWith('cu') ? NODE_LINUX_GPU : NODE_LINUX_CPU\n  return cd_utils.generic_pipeline(mxnet_variant, this, node_type)\n}\n\n// Returns the (Docker) environment for the given variant\n// The environment corresponds to the docker files in the 'docker' directory\ndef get_environment(mxnet_variant) {\n  if (mxnet_variant.startsWith(\"cu\")) {\n    return \"centos7_gpu_${mxnet_variant}\"\n  }\n  return \"centos7_cpu\"\n}\n\n\ndef build(mxnet_variant) {\n  ws(\"workspace/python_docker/${mxnet_variant}/${env.BUILD_NUMBER}\") {\n    ci_utils.init_git()\n    cd_utils.restore_static_libmxnet(mxnet_variant)\n\n    // package wheel file\n    def nvidia_docker = mxnet_variant.startsWith('cu')\n    def environment = get_environment(mxnet_variant)\n    ci_utils.docker_run(environment, \"cd_package_pypi ${mxnet_variant}\", nvidia_docker)\n\n    // build python docker images\n    sh \"./cd/python/docker/python_images.sh build ${mxnet_variant}\"\n  }\n}\n\ndef test(mxnet_variant) {\n  ws(\"workspace/python_docker/${mxnet_variant}/${env.BUILD_NUMBER}\") {\n    // test python docker images\n    sh \"./cd/python/docker/python_images.sh test ${mxnet_variant}\"\n  }\n}\n\ndef push(mxnet_variant) {\n  ws(\"workspace/python_docker/${mxnet_variant}/${env.BUILD_NUMBER}\") {\n    // push python docker images\n    sh \"./cd/python/docker/python_images.sh push ${mxnet_variant}\"\n  }\n}\n\nreturn this\n"
  },
  {
    "path": "cd/python/docker/python_images.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Executes mxnet python images pipeline functions: build, test, publish\n# Assumes script is run from the root of the mxnet repository\n# Assumes script is being run within MXNet CD infrastructure\n\nset -xe\n\nusage=\"Usage: python_images.sh <build|test|push> MXNET-VARIANT\"\n\ncommand=${1:?$usage}\nmxnet_variant=${2:?$usage}\n\ncd_utils='cd/utils'\nci_utils='ci/'\n\ndocker_tags=($(./${cd_utils}/docker_tag.sh ${mxnet_variant}))\nmain_tag=\"${docker_tags[0]}_py3\"\nbase_image=$(./${cd_utils}/mxnet_base_image.sh ${mxnet_variant})\nrepository=\"python\"\nimage_name=\"${repository}:${main_tag}\"\n\nresources_path='cd/python/docker'\n\nif [ ! -z \"${RELEASE_PUBLIC_ECR_REPOSITORY}\" ]; then\n    image_name=\"${RELEASE_PUBLIC_ECR_REPOSITORY}/${image_name}\"\nfi\n\nbuild() {\n    # NOTE: Ensure the correct context root is passed in when building - Dockerfile expects ./wheel_build\n    docker build -t \"${image_name}\" --build-arg BASE_IMAGE=\"${base_image}\" --build-arg MXNET_COMMIT_ID=${GIT_COMMIT} -f ${resources_path}/Dockerfile ./wheel_build\n}\n\ntest() {\n    local runtime_param=\"\"\n    if [[ ${mxnet_variant} == cu* ]]; then\n        runtime_param=\"--runtime=nvidia\"\n    fi\n    local test_image_name=\"${image_name}_test\"\n\n    # Ensure the correct context root is passed in when building - Dockerfile.test expects ci directory\n    docker build -t \"${test_image_name}\" --build-arg USER_ID=`id -u` --build-arg GROUP_ID=`id -g` --build-arg BASE_IMAGE=\"${image_name}\" -f ${resources_path}/Dockerfile.test ./ci\n}\n\npush() {\n    if [ -z \"${RELEASE_PUBLIC_ECR_REPOSITORY}\" ]; then\n        echo \"Cannot publish image without RELEASE_PUBLIC_ECR_REPOSITORY environment variable being set.\"\n        exit 1\n    fi\n\n    # Retrieve an authentication token and authenticate Docker client to registry\n    aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/w6z5f7h2\n\n    # Push image\n    docker push \"${image_name}\"\n}\n\ncase ${command} in\n    \"build\")\n        build\n        ;;\n\n    \"test\")\n        test\n        ;;\n\n    \"push\")\n        push\n        ;;\n\n    *)\n        echo $usage\n        exit 1\nesac\n"
  },
  {
    "path": "cd/python/docker/test_python_image.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# To be run _within_ a runtime image\n# Tests the Runtime docker image\n# Assumes the mxnet source directory is mounted on /mxnet and cwd is /mxnet\n\nset -ex\n\n# Variant parameter should be passed in\nmxnet_variant=${1:?\"Missing mxnet variant\"}\n\nif [ -z \"${MXNET_COMMIT_ID}\" ]; then\n    echo \"MXNET_COMMIT_ID environment variable is empty. Please rebuild the image with MXNET_COMMIT_ID build-arg specified.\"\n    exit 1\nfi\n\n# Execute tests\nif [[ $mxnet_variant != native ]]; then\n    python3 tests/python/dnnl/test_dnnl.py\nfi\n\n# TODO: Add more tests (18549)\n\n"
  },
  {
    "path": "cd/python/pypi/Jenkins_pipeline.groovy",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// NOTE:\n// ci_utils and cd_utils are loaded by the originating Jenkins job, e.g. jenkins/Jenkinsfile_release_job\n\n// Only post the following variants to PyPI.\n// This is a temporary solution until we are confident with the packages generated by CI\n// This should be removed in the not too distant future.\n// We only skip the publish step so we can still QA the other variants.\npypi_releases = []\n\ndef get_pipeline(mxnet_variant) {\n  def node_type = mxnet_variant.startsWith('cu') ? NODE_LINUX_GPU : NODE_LINUX_CPU\n  return cd_utils.generic_pipeline(mxnet_variant, this, node_type)\n}\n\ndef get_environment(mxnet_variant) {\n  if (mxnet_variant.startsWith('cu')) {\n    return \"centos7_gpu_${mxnet_variant}\"\n  }\n  return \"centos7_cpu\"\n}\n\ndef build(mxnet_variant) {\n  ws(\"workspace/python_pypi/${mxnet_variant}/${env.BUILD_NUMBER}\") {\n    ci_utils.init_git()\n    cd_utils.restore_static_libmxnet(mxnet_variant)\n\n    // create wheel file\n    def environment = get_environment(mxnet_variant)\n    def nvidia_docker = mxnet_variant.startsWith('cu')\n    ci_utils.docker_run(environment, \"cd_package_pypi ${mxnet_variant}\", nvidia_docker, '500m', \"RELEASE_BUILD='${env.RELEASE_BUILD}'\")\n  }\n}\n\ndef test(mxnet_variant) {\n  ws(\"workspace/python_pypi/${mxnet_variant}/${env.BUILD_NUMBER}\") {\n    // test wheel file\n    def environment = get_environment(mxnet_variant)\n    def nvidia_docker = mxnet_variant.startsWith('cu')\n    ci_utils.docker_run(environment, \"cd_integration_test_pypi ${nvidia_docker}\", nvidia_docker)\n  }\n}\n\ndef push(mxnet_variant) {\n  ws(\"workspace/python_pypi/${mxnet_variant}/${env.BUILD_NUMBER}\") {\n    // publish package to pypi\n    if (mxnet_variant in pypi_releases) {\n      sh \"./ci/docker/runtime_functions.sh cd_pypi_publish\"\n    } else {\n      echo \"Temporarily skipping publishing PyPI package for '${mxnet_variant}'.\"\n    }\n    sh \"./ci/docker/runtime_functions.sh cd_s3_publish\"\n  }\n}\n\nreturn this\n"
  },
  {
    "path": "cd/python/pypi/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# PyPI CD Pipeline\n\nThe Jenkins pipelines for continuous delivery of the PyPI MXNet packages.\nThe pipelines for each variant are run, and fail, independently. Each depends\non a successful build of the statically linked libmxet library.\n\nThe pipeline relies on the scripts and resources located in [tools/pip](https://github.com/apache/mxnet/tree/master/tools/pip)\nto build the PyPI packages.\n\n## Credentials\n\nThe pipeline depends on the following environment variables in order to successfully\nretrieve the credentials for the PyPI account:\n\n* CD_PYPI_SECRET_NAME\n* DOCKERHUB_SECRET_ENDPOINT_URL\n* DOCKERHUB_SECRET_ENDPOINT_REGION\n\nThe credentials are stored in the Secrets Manager of the AWS account hosting Jenkins.\nThe [pypi_publish.py](pypi_publish.sh) script is in charge of retrieving the credentials.\n\n## Mock publishing\n\nBecause of space limitations on PyPI, we don't want to push test packages from Jenkins Dev\neverytime the pipeline is run. Therefore, the [pypi_publish.sh](pypi_publish.sh) \nscript will fake publishing packages if the `username` is *skipPublish*.\n"
  },
  {
    "path": "cd/python/pypi/pypi_package.sh",
    "content": "#!/usr/bin/env bash\n#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nset -ex\n\n# variant = cpu, native, cu101, cu102, etc.\nexport mxnet_variant=${1:?\"Please specify the mxnet variant\"}\n\n# Due to this PR: https://github.com/apache/mxnet/pull/14899\n# The setup.py expects that dnnl_version.h be present in\n# mxnet-build/3rdparty/onednn/build/install/include\n# The artifact repository stores this file in the dependencies\n# and CD unpacks it to a directory called cd_misc\nif [ -f \"cd_misc/dnnl_version.h\" ]; then\n  mkdir -p 3rdparty/onednn/include/oneapi/dnnl\n  cp cd_misc/dnnl_version.h 3rdparty/onednn/include/oneapi/dnnl/.\n  cp cd_misc/dnnl_config.h 3rdparty/onednn/include/oneapi/dnnl/.\nfi\n\n# Create wheel workspace\nrm -rf wheel_build\nmkdir wheel_build\ncd wheel_build\n\n# Setup workspace\n# setup.py expects mxnet-build to be the\n# mxnet directory\nln -s ../. mxnet-build\n\n# Copy the setup.py and other package resources\ncp -R ../tools/pip/* .\n\n# Remove comment lines from pip doc files\npushd doc\nfor file in $(ls); do\n  sed -i '/<!--/d' ${file}\ndone\npopd\n\necho \"Building python package with environment:\"\nprintenv\necho \"-----------------------------------------\"\npip3 install --user pypandoc\n\n# Build wheel file - placed in wheel_build/dist\npython3 setup.py bdist_wheel\n"
  },
  {
    "path": "cd/python/pypi/pypi_publish.py",
    "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport json\nimport logging\nimport os\nimport subprocess\nimport sys\n\nimport boto3\nfrom botocore.exceptions import ClientError\n\n\ndef post_wheel(path):\n    \"\"\"\n    Posts mxnet wheel file to PyPI\n    \"\"\"\n    logging.info('Posting {} to PyPI'.format(path))\n    pypi_credentials = get_secret()\n\n    cmd = 'python3 -m twine upload {}'.format(path)\n    version = os.path.basename(path).split('-')[1]\n\n    # The PyPI credentials for DEV has username set to 'skipPublish'\n    # This way we do not attempt to publish the PyPI package\n    # Just print a helpful message\n    if pypi_credentials['username'] == 'skipPublish':\n        print('In DEV account, skipping publish')\n        print('Would have run: {}'.format(cmd))\n        return 0\n    elif any(test_version_mark in version for test_version_mark in ['a', 'b', 'dev']):\n        print('Skipping publishing nightly builds to Pypi.')\n        print('See https://github.com/pypa/pypi-support/issues/50 for details')\n        return 0\n    else:\n        env = os.environ.copy()\n        env['TWINE_USERNAME'] = pypi_credentials['username']\n        env['TWINE_PASSWORD'] = pypi_credentials['password']\n        p = subprocess.run(cmd.split(' '), stdout=subprocess.PIPE, env=env)\n        logging.info(p.stdout)\n        return p.returncode\n\ndef get_secret():\n    secret_name = os.environ['CD_PYPI_SECRET_NAME']\n    endpoint_url = os.environ['DOCKERHUB_SECRET_ENDPOINT_URL']\n    region_name = os.environ['DOCKERHUB_SECRET_ENDPOINT_REGION']\n\n    session = boto3.Session()\n    client = session.client(\n        service_name='secretsmanager',\n        region_name=region_name,\n        endpoint_url=endpoint_url\n    )\n\n    try:\n        get_secret_value_response = client.get_secret_value(SecretId=secret_name)\n    except ClientError as e:\n        if e.response['Error']['Code'] == 'DecryptionFailureException':\n            raise e\n        elif e.response['Error']['Code'] == 'InternalServiceErrorException':\n            raise e\n        elif e.response['Error']['Code'] == 'InvalidParameterException':\n            raise e\n        elif e.response['Error']['Code'] == 'InvalidRequestException':\n            raise e\n        elif e.response['Error']['Code'] == 'ResourceNotFoundException':\n            raise e\n    else:\n        return json.loads(get_secret_value_response['SecretString'])\n\n\nif __name__ == '__main__':\n    sys.exit(post_wheel(sys.argv[1]))\n"
  },
  {
    "path": "cd/utils/artifact_repository.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# Artifact Repository - Pushing and Pulling libmxnet\n\nThe artifact repository is an S3 bucket accessible only to restricted Jenkins nodes. It is used to store compiled MXNet artifacts that can be used by downstream CD pipelines to package the compiled libraries for different delivery channels (e.g. DockerHub, PyPI, Maven, etc.). The S3 object keys for the files being posted will be prefixed with the following distinguishing characteristics of the binary: branch, commit id, operating system, variant and dependency linking strategy (static or dynamic). For instance, s3://bucket/73b29fa90d3eac0b1fae403b7583fdd1529942dc/ubuntu16.04/cu102mkl/static/libmxnet.so\n\nAn MXNet artifact is defined as the following set of files:\n\n* The compiled libmxnet.so\n* License files for dependencies that required their licenses to be shipped with the binary\n* Dependencies that should be shipped together with the binary. For instance, for packaging the python wheel files, some dependencies that cannot be statically linked to the library need to also be included, see here (https://github.com/apache/mxnet/blob/master/tools/pip/setup.py#L142).\n\nThe artifact_repository.py script automates the upload and download of the specified files with the appropriate S3 object keys by taking explicitly set, or automatically derived, values for the different characteristics of the artifact.\n\n### Determining Artifact Characteristics\n\nAn mxnet compiled library, or artifact for our purposes, is identified by the following distinguishing characteristics, which when not explicitly stated, will be (as much as possible) ascertained from the environment by the artifact_repository.py script: commit id, variant, operating system, and library type.\n\n**Commit Id**\n\nManually configured through the --git-sha argument. \n\nIf not set, derived by:\n\n1. Using the values of the MXNET_SHA environment variable, which are set during the bootstrap process for *CD* Jenkins pipelines; otherwise\n2. Using the values of the GIT_COMMIT environment variable, which are set automatically by Jenkins in the *CI* pipelines; otherwise\n3. Using the output of git rev-parse HEAD for the commit id; otherwise\n4. Fail with error\n\n**Operating System**\n\nManually configured through the --os argument.\n\nIf not set, derived through the value of sys.platform (https://docs.python.org/3/library/sys.html#sys.platform). That is:\n\n* if, linux*, extract the ID and VERSION_ID from /etc/*release, and return a concatenated string of these values, eg. ubuntu16.04, centos7, etc.\n* otherwise, return the value given by sys.platform, eg. win32, darwin, etc.\n\n**Variant**\n\nManually configured through the --variant argument. The current variants are: cpu, native, cu101, cu102, cu110, cu112.\n\nAs long as the tool is being run from the MXNet code base, the runtime feature detection tool (https://github.com/larroy/mxnet/blob/dd432b7f241c9da2c96bcb877c2dc84e6a1f74d4/docs/api/python/libinfo/libinfo.md) can be used to detect whether the library has been compiled with oneDNN (library has oneDNN feature enabled) and/or CUDA support (compiled with CUDA feature enabled).\n\nIf it has been compiled with CUDA support, the output of /usr/local/cuda/bin/nvcc --version can be mined for the exact CUDA version (eg. 8.0, 9.0, etc.).\n\nBy knowing which features are enabled on the binary, and if necessary, which CUDA version is installed on the machine, the value for the variant argument can be calculated. Eg. if CUDA features are enabled, and nvcc reports cuda version 10.2, then the variant would be cu102. If neither oneDNN nor CUDA features are enabled, the variant would be native. \n\n**Dependency Linking**\n\nThe library dependencies can be either statically or dynamically linked. This property will need to be manually set by user through either the `--static` or `--dynamic` arguments. There is no foolproof and programmatic way (that I could find) that can easily discern whether the library dependencies are statically or dynamically linked.\n\n### Uploading an Artifact\n\nThe user must specify the path to the libmxnet.so, any license files, and any dependencies. The latter two are optional.\n \nExample:\n\n`./artifact_repository.py --push --static --libmxnet /path/to/libmxnet.so --licenses path/to/license1.txt /path/to/other_licenses/*.txt --dependencies /path/to/dependencies/*.so`\n\n`./artifact_repository.py --push --dynamic --libmxnet /path/to/libmxnet.so`\n\nNOTE: There is nothing stopping the user from uploading licenses and dependencies for dynamically linked libraries.\n\n### Downloading An Artifact\n\nThe user must specify the directory to which the artifact should be downloaded. The user will also need to specify the variant, since different variants can work with the host operating system.\n\nExample:\n\n`./artifact_repository.py --pull --static --variant=cu102 ./dist`\n\nThis would result in the following directory structure:\n\n```\ndist\n  |-----> libmxnet.so\n  |-----> libmxnet.meta\n  |-----> licenses\n             |-----> MKL_LICENSE.txt\n             |-----> CUP_LICENSE.txt\n             |-----> ...\n  |-----> dependencies\n             |-----> libxxx.so\n             |-----> libyyy.so\n             |-----> ...\n```\n\nThe libmxnet.meta file will include the characteristics of the artifact (ie. library type, variant, git commit id, etc.) in a “property” file format.\n\n"
  },
  {
    "path": "cd/utils/artifact_repository.py",
    "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"\nTool for uploading artifacts to the artifact repository\n\"\"\"\n\n__author__ = 'Per Goncalves da Silva'\n__version__ = '0.1'\n\nimport argparse\nimport ctypes\nimport glob\nimport logging\nimport os\nimport re\nimport sys\nfrom itertools import chain\nfrom subprocess import CalledProcessError, check_output\nfrom typing import Dict, List, Optional\n\nimport boto3\nimport botocore.exceptions\nimport yaml\n\ns3 = boto3.client('s3')\nlogger = logging.getLogger(__name__)\n\n\ndef config_logging():\n    \"\"\"\n    Configures default logging settings\n    \"\"\"\n    logging.root.setLevel(logging.WARNING)\n    logger.setLevel(logging.INFO)\n    logging.basicConfig(format='%(levelname)s: %(message)s')\n\n\ndef s3_upload(bucket: str, s3_key_prefix: str, paths: List[str]):\n    \"\"\"\n    Uploads a list of files to an S3 bucket with a particular S3 key prefix\n    :param bucket: The name of the S3 bucket\n    :param s3_key_prefix: The key prefix to apply to each of the files\n    :param paths: A list of paths to files\n    \"\"\"\n    for path in paths:\n        s3_key = \"{}/{}\".format(s3_key_prefix, os.path.basename(path))\n        logger.info('Uploading {}'.format(path))\n        logger.debug(\"Uploading {} to s3://{}/{}\".format(path, bucket, s3_key))\n        with open(path, 'rb') as data:\n            s3.upload_fileobj(Fileobj=data, Key=s3_key, Bucket=bucket)\n\n\ndef write_libmxnet_meta(args: argparse.Namespace, destination: str):\n    \"\"\"\n    Writes a file called libmxnet.meta in the 'destination' folder that contains\n    the libmxnet library information (commit id, type, etc.).\n    :param args: A Namespace object containing the library\n    :param destination: The folder in which to place the libmxnet.meta\n    \"\"\"\n    with open(os.path.join(destination, 'libmxnet.meta'), 'w') as fp:\n        fp.write(yaml.dump({\n            \"variant\": args.variant,\n            \"os\": args.os,\n            \"commit_id\": args.git_sha,\n            \"dependency_linking\": args.libtype,\n        }))\n\n\ndef try_s3_download(bucket, s3_key_prefix, destination) -> bool:\n    \"\"\"\n    Downloads a list of files to an S3 bucket with a particular S3 key prefix to 'destination'\n    :param bucket: The name of the S3 bucket\n    :param s3_key_prefix: The key prefix to apply to each of the files\n    :param destination the path to which to download the files\n    :return False if not artifacts were found, True otherwise\n    \"\"\"\n    response = s3.list_objects_v2(Bucket=bucket, Prefix=s3_key_prefix)\n    if not response:\n        raise RuntimeError('Error listing S3 objects')\n\n    if response.get('KeyCount') is None:\n        logger.debug('Invalid S3 list objects response format')\n        logger.debug(response)\n        raise RuntimeError('Invalid response format.')\n\n    key_count = response.get('KeyCount')\n    if key_count == 0:\n        logger.debug('No artifacts found')\n        return False\n\n    if not response.get('Contents'):\n        logger.debug('Invalid S3 list objects response format')\n        logger.debug(response)\n        raise RuntimeError('Invalid response format.')\n\n    for obj in response.get('Contents'):\n        key = obj['Key']\n\n        # extract file path with any subdirectories and remove the leading file separator\n        output_path = os.path.join(destination, key[len(s3_key_prefix):].lstrip(os.sep))\n        os.makedirs(os.path.dirname(output_path), exist_ok=True)\n        logger.info('Downloading {}'.format(output_path))\n        logger.debug(\"Downloading s3://{}/{} to {}\".format(bucket, key, output_path))\n        with open(output_path, 'wb') as fp:\n            s3.download_fileobj(Fileobj=fp, Key=key, Bucket=bucket)\n\n    return True\n\n\ndef get_commit_id_from_cmd() -> Optional[str]:\n    \"\"\"\n    Returns the output of 'git rev-parse HEAD'\n    :return: A commit id, or None if the command fails\n    \"\"\"\n    try:\n        logger.debug('Executing \"git rev-parse HEAD\"')\n        commit_id = check_output(\"git rev-parse HEAD\".split(\" \")).decode('UTF-8').strip()\n        logger.debug('Found commit id: {}'.format(commit_id))\n        return commit_id\n    except CalledProcessError as e:\n        logger.debug('Error getting commit id:')\n        logger.debug(format(e))\n        return None\n\n\ndef probe_commit_id() -> str:\n    \"\"\"\n    Probes the system in an attempt to ascertain the mxnet commit id\n    :return: The commit id, or None if not found\n    \"\"\"\n    logger.debug('Probing for commit id')\n    commit_id = os.environ.get('MXNET_SHA')\n    if not commit_id:\n        logger.debug('MXNET_SHA environment variable not set. Trying GIT_COMMIT')\n        commit_id = os.environ.get('GIT_COMMIT')\n    if not commit_id:\n        logger.debug('GIT_COMMIT environment variable not set. Trying git command')\n        commit_id = get_commit_id_from_cmd()\n    if not commit_id:\n        logger.debug('Could not determine git commit id')\n    else:\n        logger.debug('Commit id is: {}'.format(commit_id))\n    return commit_id\n\n\ndef get_linux_os_release_properties() -> Optional[Dict[str, str]]:\n    \"\"\"\n    Makes a dictionary out of /etc/os-release\n    :return: A dictionary of os release properties\n    \"\"\"\n    logger.debug('Extracting operating system properties from /etc/os-release')\n    if not os.path.isfile('/etc/os-release'):\n        logger.debug('Error: /etc/os-release not found')\n        return None\n\n    try:\n        with open('/etc/os-release', 'r') as fp:\n            # removes empty spaces and quotation marks from line\n            property_tuple_list = [line.strip().replace('\"', '').split('=') for line in fp if line.strip()]\n            return {key: value for (key, value) in property_tuple_list}\n    except Exception as e:\n        logger.debug('Error parsing /etc/os-release')\n        logger.debug(e)\n        return None\n\n\ndef get_linux_distribution_and_version() -> Optional[str]:\n    \"\"\"\n    Returns the linux distribution and version by taking\n    the values of ID and VERSION_ID from /etc/os-release and\n    concatenating them. Eg. centos7, ubuntu16.04, etc.\n    :return: The linux distribution and version string, or None if not found.\n    \"\"\"\n    logger.debug('Getting linux distribution and version')\n    os_properties = get_linux_os_release_properties()\n    if os_properties:\n        logger.debug('os properties: {}'.format(os_properties))\n        distribution = os_properties['ID']\n        version = os_properties['VERSION_ID']\n        return \"{}{}\".format(distribution, version)\n\n    logger.debug('Error getting linux distribution and version. Could not determine os properties.')\n    return None\n\n\ndef probe_operating_system() -> str:\n    \"\"\"\n    Probes the system to determine the operating system.\n    :return: The name of the operating system, e.g. win32, darwin, ubuntu16.04, centos7, etc.\n    \"\"\"\n    logger.debug('Determining operating system')\n    operating_system = sys.platform\n    logger.debug('Found platform: {}'.format(operating_system))\n    if operating_system.startswith('linux'):\n        operating_system = get_linux_distribution_and_version()\n\n    logger.debug('Operating system is {}'.format(operating_system))\n    return operating_system\n\n\ndef get_libmxnet_features(libmxnet_path: str) -> Optional[Dict[str, bool]]:\n    \"\"\"\n    Returns a string -> boolean dictionary mapping feature name\n    to whether it is enabled or not\n    :param libmxnet_path: path to the libmxnet library\n    :return: dictionary of features to whether they are enabled\n    \"\"\"\n    logger.debug('Getting feature dictionary from {}'.format(libmxnet_path))\n\n    class Feature(ctypes.Structure):\n        _fields_ = [(\"_name\", ctypes.c_char_p), (\"enabled\", ctypes.c_bool)]\n\n        @property\n        def name(self):\n            return self._name.decode()\n\n    # we are not using the mxnet python bindings here because we cannot assume\n    # they are present and in the python path, or that they would point to the\n    # specified libmxnet.so. Therefore, we load the libmxnet.so library independently\n    # to extract its features\n    try:\n        libmxnet = ctypes.CDLL(libmxnet_path, ctypes.RTLD_LOCAL)\n    except Exception as e:\n        logger.error('Error loading {}. '\n                     'Please check check path to libmxnet library is correct.'.format(libmxnet_path))\n        logger.error(e)\n        return None\n\n    libmxnet.MXGetLastError.restype = ctypes.c_char_p\n    feature_array = ctypes.POINTER(Feature)()\n    feature_array_size = ctypes.c_size_t()\n    if libmxnet.MXLibInfoFeatures(ctypes.byref(feature_array), ctypes.byref(feature_array_size)) != 0:\n        logger.error('Could not determine features from {}. '\n                     'Please specify the variant manually using the \"--variant\" argument.'.format(libmxnet_path))\n        return None\n    features = {feature_array[i].name: feature_array[i].enabled for i in range(feature_array_size.value)}\n    logger.debug('Found features: {}'.format(features))\n    return features\n\n\ndef get_cuda_version() -> Optional[str]:\n    \"\"\"\n    Returns the major and minor cuda version without the '.'\n    eg. 10.0 => 100, 9.2 => 92, etc.\n    :return: CUDA version\n    \"\"\"\n    logger.debug('Determining cuda version')\n\n    try:\n        logger.debug('Executing \"nvcc -V\"')\n        nvcc_version = check_output(\"nvcc -V\".split(\" \")).decode('UTF-8').strip()\n    except CalledProcessError as e:\n        logger.error('Error getting nvcc version')\n        logger.error(e)\n        return None\n\n    logger.debug('Extracting cuda version from {}'.format(nvcc_version))\n    # eg. \"Cuda compilation tools, release 10.0, V10.0.130\"\n    match = re.search(r' ([0-9]+.[0-9]+)', nvcc_version)\n    if match:\n        cuda_version = match.group(1).replace('.', '')\n        logger.debug('Found cuda version: {}'.format(cuda_version))\n        return cuda_version\n\n    logger.debug('Could not determine cuda version from \"{}\"'.format(nvcc_version))\n    return None\n\n\ndef probe_cpu_variant(mxnet_features: Dict[str, bool]) -> str:\n    \"\"\"\n    Returns the mxnet cpu targeted variant depending on which mxnet features are enabled\n    :param mxnet_features: An mxnet feature dictionary of feature to boolean (True = enabled)\n    :return: Either cpu, or mkl as the variant\n    \"\"\"\n    logger.debug('Determining cpu variant')\n    if not mxnet_features['ONEDNN']:\n        logger.debug('variant is: native')\n        return 'native'\n\n    logger.debug('variant is: cpu')\n    return 'cpu'\n\n\ndef probe_gpu_variant(mxnet_features: Dict[str, bool]) -> Optional[str]:\n    \"\"\"\n    Returns the mxnet gpu variant depending on which mxnet features are enabled\n    :param mxnet_features: An mxnet feature dictionary of feature to boolean (True = enabled)\n    :return: The mxnet gpu variant, eg. cu102, cu102mkl, etc.\n    :raises RuntimeError is the CUDA feature is not enabled in the library\n    \"\"\"\n    if not mxnet_features['CUDA']:\n        raise RuntimeError('Cannot determine gpu variant. CUDA feature is disabled.')\n\n    cuda_version = get_cuda_version()\n    if cuda_version:\n        variant = 'cu{}'.format(cuda_version)\n        if not mxnet_features['ONEDNN']:\n            RuntimeError('Error determining mxnet variant: oneDNN should be enabled for cuda variants')\n        logger.debug('variant is: {}'.format(variant))\n        return variant\n\n    raise RuntimeError('Error determining mxnet variant: Could not retrieve cuda version')\n\n\ndef probe_mxnet_variant(limxnet_path: str) -> Optional[str]:\n    \"\"\"\n    Probes the libmxnet library and environment to determine\n    the mxnet variant, eg. cpu, cu102, etc.\n    :return:\n    \"\"\"\n    logger.debug('Probing for mxnet variant')\n    features = get_libmxnet_features(limxnet_path)\n    if not features:\n        logger.debug('Error: could not determine variant. Features could not be extracted from libmxnet')\n        return None\n\n    if features['CUDA']:\n        return probe_gpu_variant(features)\n    return probe_cpu_variant(features)\n\n\ndef probe_artifact_repository_bucket() -> Optional[str]:\n    \"\"\"\n    Probes environment variables in search of artifact repository bucket\n    :return: string containing the artifact repository bucket name\n    \"\"\"\n    logger.debug('Probing for artifact repository bucket name')\n    bucket = os.environ.get('ARTIFACT_REPOSITORY_BUCKET')\n    if not bucket:\n        logger.debug('ARTIFACT_REPOSITORY_BUCKET environment variable not found')\n    return bucket\n\n\ndef probe(args: argparse.Namespace) -> argparse.Namespace:\n    \"\"\"\n    Probes the system to set any arguments that weren't manually set.\n    Modifies the input Namespace object with the probed parameters.\n    :param args: The namespace object given by argparse.parse()\n    \"\"\"\n    logger.debug('Trying to auto-determine arguments from environment')\n    if not args.git_sha:\n        commit_id = probe_commit_id()\n        if not commit_id:\n            logger.error('Could not determine commit id. '\n                         'Please set it manually with --git-sha, or ensure you are in a cloned '\n                         'mxnet repository directory')\n            sys.exit(1)\n        args.git_sha = commit_id\n\n    if not args.variant:\n        variant = probe_mxnet_variant(args.libmxnet)\n        if not variant:\n            logger.error('Could not determine mxnet variant. Please set it manually with --variant')\n            sys.exit(1)\n        args.variant = variant\n\n    if not args.os:\n        operating_system = probe_operating_system()\n        if not operating_system:\n            logger.error('Could not determine operating system. Please set it manually with --os')\n            sys.exit(1)\n        args.os = operating_system\n\n    if not args.bucket:\n        artifact_repo_bucket = probe_artifact_repository_bucket()\n        if not artifact_repo_bucket:\n            logger.error('Could not determine artifact repository bucket. Please set it manually with --bucket')\n            sys.exit(1)\n        args.bucket = artifact_repo_bucket\n\n    return args\n\n\ndef get_s3_key_prefix(args: argparse.Namespace, subdir: str = '') -> str:\n    \"\"\"\n    Returns the S3 key prefix given the arguments namespace\n    :param args: The arguments passed in by the user or derived by the script\n    :param subdir: An optional subdirectory in which to store the files. Post-pended to the end of the prefix.\n    :return: A string containing the S3 key prefix to be used to uploading and downloading files to the artifact repository\n    \"\"\"\n    prefix = \"{git_sha}/{libtype}/{os}/{variant}/\".format(**vars(args))\n    if subdir:\n        return \"{}{}/\".format(prefix, subdir)\n    return prefix\n\n\ndef push_artifact(args: argparse.Namespace):\n    \"\"\"\n    Pushes the artifact to the artifact repository\n    :param args: The arguments passed in to this script by the user\n    :return 0 for success, non-zero for failure\n    \"\"\"\n\n    args = probe(args)\n\n    logger.info('Pushing artifact with: ')\n    logger.info('COMMIT ID   : {}'.format(args.git_sha))\n    logger.info('OS          : {}'.format(args.os))\n    logger.info('VARIANT     : {}'.format(args.variant))\n    logger.info(\"LIBMXNET    : {}\".format(args.libmxnet))\n    logger.info(\"LICENSES    : {}\".format(args.licenses))\n    logger.info(\"DEPENDENCIES: {}\".format(args.dependencies))\n    logger.info(\"\")\n\n    if not args.licenses:\n        raise RuntimeError('No licenses defined. Please submit the licenses to be shipped with the binary.')\n\n    # Upload mxnet\n    try:\n        logger.info('Uploading libmxnet library...')\n        s3_upload(args.bucket, get_s3_key_prefix(args), [args.libmxnet])\n        logger.info(\"\")\n\n        # Upload licenses\n        logger.info('Uploading licenses...')\n        s3_upload(args.bucket, get_s3_key_prefix(args, subdir='licenses'), args.licenses)\n        logger.info(\"\")\n\n        # Upload dependencies, if necessary\n        if args.dependencies:\n            logger.info('Uploading dependencies...')\n            s3_upload(args.bucket, get_s3_key_prefix(args, subdir='dependencies'), args.dependencies)\n            logger.info(\"\")\n    except botocore.exceptions.BotoCoreError as e:\n        logger.error('Error uploading artifact')\n        logger.error(e)\n        raise e\n\n    logger.info('Successfully pushed artifact')\n\n\ndef pull_artifact(args: argparse.Namespace):\n    \"\"\"\n    Pulls the artifact from the artifact repository\n    :param args: The arguments passed in to this script by the user\n    :return 0 for success, 1 for unexpected failure, 2 for no artifact found failure\n    \"\"\"\n    if not args.variant:\n        logger.warning('''variant not set. Using 'cpu' by default.''')\n        args.variant = 'cpu'\n\n    args = probe(args)\n\n    logger.info('Pulling artifact with: ')\n    logger.info('COMMIT ID   : {}'.format(args.git_sha))\n    logger.info('OS          : {}'.format(args.os))\n    logger.info('VARIANT     : {}'.format(args.variant))\n    logger.info('To directory: {}'.format(args.destination))\n\n    try:\n        if not try_s3_download(args.bucket, get_s3_key_prefix(args), args.destination):\n            raise RuntimeError('No artifacts found for this configuration.')\n        write_libmxnet_meta(args=args, destination=args.destination)\n    except botocore.exceptions.BotoCoreError as e:\n        logger.error('Error downloading artifact')\n        logger.error(e)\n        raise e\n\n    logger.info('Successfully pulled artifact')\n\n\ndef is_file(path: str) -> str:\n    \"\"\"\n    Returns true or false if path points to an existing file\n    :param path: A path to a file\n    :return: True if file exists and is a file, False otherwise\n    :raises FileNotFoundError if file does not exist\n    \"\"\"\n    if not os.path.exists(path):\n        raise FileNotFoundError('''File '{}' not found'''.format(path))\n    return os.path.isfile(path)\n\n\ndef sanitize_path_array(paths: List[str]) -> List[str]:\n    \"\"\"\n    Expands supplied paths and removes empty or non-file entries.\n    :param paths: A list of paths\n    :return: A sanitized list of paths\n    :raises FileNotFoundError if a file does not exist\n    \"\"\"\n    expanded_paths = list(chain.from_iterable(glob.glob(path.strip()) for path in paths if path.strip() != ''))\n    return [path.strip() for path in expanded_paths if path.strip() != '' and is_file(path)]\n\n\ndef main() -> int:\n    config_logging()\n\n    logger.info(\"MXNet-CD Artifact Repository Tool\")\n\n    parser = argparse.ArgumentParser(description=\"Utility for uploading and downloading MXNet artifacts\")\n\n    parser.add_argument(\"--push\",\n                        help=\"Upload artifact to repository\",\n                        required=False,\n                        action='store_true')\n\n    parser.add_argument(\"--pull\",\n                        help=\"Download artifact from repository\",\n                        required=False,\n                        action='store_true')\n\n    parser.add_argument(\"--libmxnet\",\n                        help=\"Path to libmxnet library\",\n                        required=False,\n                        type=str)\n\n    parser.add_argument(\"--licenses\",\n                        help=\"Paths to license files\",\n                        required=False,\n                        nargs=argparse.ZERO_OR_MORE,\n                        default=[])\n\n    parser.add_argument(\"--dependencies\",\n                        help=\"Paths to dependencies\",\n                        required=False,\n                        nargs=argparse.ZERO_OR_MORE,\n                        default=[])\n\n    parser.add_argument(\"--os\",\n                        help=\"Target operating system\",\n                        type=str)\n\n    parser.add_argument(\"--git-sha\",\n                        help=\"MXNet repository commit id\",\n                        required=False,\n                        type=str)\n\n    parser.add_argument(\"--variant\",\n                        help=\"MXNet binary variant. Eg. cpu, native, cu102, etc.\",\n                        required=False,\n                        type=str)\n\n    parser.add_argument(\"--libtype\",\n                        help=\"libmxnet dependency linking type\",\n                        choices=['static', 'dynamic'],\n                        default='dynamic',\n                        required=False)\n\n    parser.add_argument('--bucket',\n                        help=\"S3 bucket to store files\",\n                        type=str,\n                        required=False)\n\n    parser.add_argument('--destination',\n                        help=\"Destination for downloaded library and supporting files\",\n                        type=str,\n                        default=os.getcwd(),\n                        required=False)\n\n    parser.add_argument('--verbose',\n                        help='Verbose',\n                        action='store_true',\n                        default=False)\n\n    parser.add_argument('--debug',\n                        help='Debug mode',\n                        action='store_true',\n                        default=False)\n\n    args = parser.parse_args()\n\n    if args.verbose:\n        logger.setLevel(logging.DEBUG)\n\n    if args.debug:\n        # Set debug level on root logger (ie. for all other loggers)\n        logging.getLogger().setLevel(logging.DEBUG)\n\n    if not args.push and not args.pull:\n        logger.info('''Mode not specified. Using 'push' by default.''')\n        args.push = True\n\n    # libmxnet argument is required for push mode\n    if args.push and not args.libmxnet:\n        logger.error('Path to libmxnet library must be specified when in push mode. '\n                     'Please specify it with --libmxnet.')\n        return 1\n\n    # sanitize license and dependency arrays\n    # Remove empty or directory entries\n    args.licenses = sanitize_path_array(args.licenses)\n    args.dependencies = sanitize_path_array(args.dependencies)\n\n    # expand destination path\n    args.destination = os.path.abspath(args.destination)\n\n    try:\n        if args.push:\n            push_artifact(args)\n\n        elif args.pull:\n            pull_artifact(args)\n    except RuntimeError as err:\n        logger.error(err)\n        return 1\n\n    return 0\n\n\nif __name__ == '__main__':\n    sys.exit(main())\n"
  },
  {
    "path": "cd/utils/docker_tag.sh",
    "content": "#!/usr/bin/env bash\n# -*- coding: utf-8 -*-\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\nmxnet_variant=${1:?\"Please specify the mxnet variant as the first parameter\"}\nis_release=${RELEASE_BUILD:-false}\nversion=${VERSION:-nightly}\n\nif [[ ${version} == \"null\" ]]; then\n    version=\"nightly\"\nfi\n\n# The docker tags will be in the form <version>_<hardware>\n# Eg. nightly_cpu, 2.0.0_cpu, nightly_gpu_cu110, etc.\n\nif [[ ${mxnet_variant} == \"cpu\" ]]; then\n    tag_suffix=\"cpu\"\nelif [[ ${mxnet_variant} == \"native\" ]]; then\n    tag_suffix=\"native\"\nelif [[ ${mxnet_variant} == cu* ]]; then\n    tag_suffix=\"gpu_${mxnet_variant}\"\n\nelse\n    echo \"Error: Unrecognized mxnet variant: '${mxnet_variant}'.\"\n    exit 1\nfi\n\necho \"${version}_${tag_suffix}\"\n"
  },
  {
    "path": "cd/utils/mxnet_base_image.sh",
    "content": "#!/usr/bin/env bash\n# -*- coding: utf-8 -*-\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nmxnet_variant=${1:?\"Please specify the mxnet variant as the first parameter\"}\n\ncase ${mxnet_variant} in\n    cu101*)\n    echo \"nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04\"\n    ;;\n    cu102*)\n    echo \"nvidia/cuda:10.2-cudnn7-runtime-ubuntu18.04\"\n    ;;\n    cu110*)\n    echo \"nvidia/cuda:11.0-cudnn8-runtime-ubuntu18.04\"\n    ;;\n    cu112*)\n    echo \"nvidia/cuda:11.2.0-cudnn8-runtime-ubuntu18.04\"\n    ;;\n    cpu)\n    echo \"ubuntu:18.04\"\n    ;;\n    native)\n    echo \"ubuntu:18.04\"\n    ;;\n    *)\n    echo \"Error: Unrecognized mxnet-variant: '${mxnet_variant}'\"\n    exit 1\n    ;;\nesac\n"
  },
  {
    "path": "cd/utils/test_artifact_repository.py",
    "content": "# -*- coding: utf-8 -*-\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport unittest\nfrom unittest.mock import MagicMock, mock_open, patch\n\nfrom artifact_repository import *\n\n\nclass TestArtifactRepositoryTool(unittest.TestCase):\n\n    @staticmethod\n    def create_argparse_namespace(libmxnet_path: Optional[str] = 'path_to_libmxnet',\n                                  git_sha: Optional[str] = 'abc123',\n                                  variant: Optional[str] = 'cpu',\n                                  operating_system: Optional[str] = 'linux',\n                                  libtype: Optional[str] = 'static',\n                                  bucket: Optional[str] = 's3bucket',\n                                  licenses: Optional[List[str]] = [],\n                                  dependencies: Optional[List[str]] = []) -> argparse.Namespace:\n        \"\"\"\n        Returns a namespace object containing the script's arguments with sample or specified values\n        \"\"\"\n        ns = argparse.Namespace()\n        ns.libmxnet = libmxnet_path\n        ns.git_sha = git_sha\n        ns.variant = variant\n        ns.os = operating_system\n        ns.libtype = libtype\n        ns.bucket = bucket\n        ns.licenses = licenses\n        ns.dependencies = dependencies\n\n        return ns\n\n    @patch('artifact_repository.check_output')\n    def test_get_commit_id_from_cmd_returns_none_on_fail(self, mock):\n        \"\"\"\n        Tests get_commit_id_from_cmd returns None if the command fails\n        \"\"\"\n        not_sucessful = 255\n        mock.side_effect = CalledProcessError(cmd=\"some command\", returncode=not_sucessful)\n        self.assertIsNone(get_commit_id_from_cmd())\n\n    def test_probe_commit_id_mxnet_sha(self):\n        \"\"\"\n        Tests the value of MXNET_SHA env var is returned\n        \"\"\"\n        with patch.dict('os.environ', {'MXNET_SHA': 'abcd1234'}):\n            self.assertEqual(probe_commit_id(), 'abcd1234')\n\n    def test_probe_commit_id_git_commit(self):\n        \"\"\"\n        Tests the value of GIT_COMMIT env. var is returned\n        if MXNET_SHA env var is not present\n        \"\"\"\n        with patch.dict('os.environ', {'GIT_COMMIT': 'abcd1234'}):\n            self.assertEqual(probe_commit_id(), 'abcd1234')\n\n        with patch.dict('os.environ', {'MXNET_SHA': 'efgh5678', 'GIT_COMMIT': 'abcd1234'}):\n            self.assertEqual(probe_commit_id(), 'efgh5678')\n\n    @patch('artifact_repository.get_commit_id_from_cmd')\n    def test_probe_commit_id_git_cmd(self, mock):\n        \"\"\"\n        Tests the git commit id from the git command is returned\n        if neither MXNET_SHA nor GIT_COMMIT env vars are set\n        \"\"\"\n        mock.return_value = 'abcd1234'\n        self.assertEqual(probe_commit_id(), 'abcd1234')\n\n    def test_get_linux_os_release_properties(self):\n        properties = \"\"\"\n        KEY=value\n        KEY2=value2\n        KEY3=value3\n        \"\"\"\n        mock = mock_open(read_data=properties)\n        with(patch('artifact_repository.os.path')) as path_mock:\n            path_mock.is_file.return_value = True\n            with patch('artifact_repository.open', mock, create=True):\n                properties = get_linux_os_release_properties()\n        self.assertEqual(properties['KEY3'], 'value3')\n\n    def test_get_linux_os_release_properties_with_quotes(self):\n        \"\"\"\n        Tests quote marks are removed from values\n        \"\"\"\n        properties = \"\"\"\n        KEY=\"value\"\n        \"\"\"\n        mock = mock_open(read_data=properties)\n        with(patch('artifact_repository.os.path')) as path_mock:\n            path_mock.is_file.return_value = True\n            with patch('artifact_repository.open', mock, create=True):\n                properties = get_linux_os_release_properties()\n        self.assertEqual(properties['KEY'], 'value')\n\n    @patch('artifact_repository.sys')\n    def test_probe_operating_system_windows(self, mock):\n        mock.platform = 'win32'\n        self.assertEqual(probe_operating_system(), 'win32')\n\n    @patch('artifact_repository.sys')\n    def test_probe_operating_system_darwin(self, mock):\n        mock.platform = 'darwin'\n        self.assertEqual(probe_operating_system(), 'darwin')\n\n    @patch('artifact_repository.sys')\n    @patch('artifact_repository.get_linux_os_release_properties')\n    def test_probe_operating_system_linux(self, mock_props, mock_sys):\n        mock_props.return_value = {'ID': 'ubuntu', 'VERSION_ID': '16.04'}\n\n        mock_sys.platform = 'linux'\n        self.assertEqual(probe_operating_system(), 'ubuntu16.04')\n\n        # sys.platform can return linux or linux2\n        mock_sys.platform = 'linux2'\n        self.assertEqual(probe_operating_system(), 'ubuntu16.04')\n\n    @patch('artifact_repository.check_output')\n    def test_get_cuda_version(self, mock):\n        \"\"\"\n        Tests correct cuda version with the right format is returned\n        :return:\n        \"\"\"\n        mock.return_value = b'Cuda compilation tools, release 10.2, V10.2.130'\n        cuda_version = get_cuda_version()\n        self.assertEqual(cuda_version, '102')\n\n        mock.return_value = b'Cuda compilation tools, release 11.0, V11.0.148'\n        cuda_version = get_cuda_version()\n        self.assertEqual(cuda_version, '110')\n\n    @patch('artifact_repository.check_output')\n    def test_get_cuda_version_not_found(self, mock):\n        \"\"\"\n        Tests None is returned there's an error retrieving the cuda version\n        :return:\n        \"\"\"\n        not_sucessful = 255\n        mock.side_effect = CalledProcessError(cmd=\"nvidia version command\", returncode=not_sucessful)\n        self.assertIsNone(get_cuda_version())\n\n    @patch('artifact_repository.get_libmxnet_features')\n    def test_probe_variant_native(self, mock_features):\n        \"\"\"\n        Tests 'native' is returned if oneDNN and CUDA features are OFF\n        \"\"\"\n        mock_features.return_value = {'ONEDNN': False, 'CUDA': False}\n        self.assertEqual(probe_mxnet_variant('libmxnet.so'), 'native')\n\n    @patch('artifact_repository.get_libmxnet_features')\n    def test_probe_variant_cpu(self, mock_features):\n        \"\"\"\n        Tests 'cpu' is returned if oneDNN is ON and CUDA is OFF\n        \"\"\"\n        mock_features.return_value = {'ONEDNN': True, 'CUDA': False}\n        self.assertEqual(probe_mxnet_variant('libmxnet.so'), 'cpu')\n\n    @patch('artifact_repository.get_libmxnet_features')\n    @patch('artifact_repository.get_cuda_version')\n    def test_probe_variant_cuda(self, mock_cuda_version, mock_features):\n        \"\"\"\n        Tests 'cu102' is returned if oneDNN is OFF and CUDA is ON and CUDA version is 10.2\n        \"\"\"\n        mock_features.return_value = {'ONEDNN': True, 'CUDA': True}\n        mock_cuda_version.return_value = '102'\n        self.assertEqual(probe_mxnet_variant('libmxnet.so'), 'cu102')\n\n    @patch('artifact_repository.get_libmxnet_features')\n    def test_probe_variant_cuda_returns_none_on_no_features(self, mock_features):\n        \"\"\"\n        Tests None is returned if the mxnet features could not be extracted from the libmxnet.so file\n        \"\"\"\n        mock_features.return_value = None\n        self.assertIsNone(probe_mxnet_variant('libmxnet.so'))\n\n    @patch('artifact_repository.get_libmxnet_features')\n    @patch('artifact_repository.get_cuda_version')\n    def test_probe_variant_cuda_mkl(self, mock_cuda_version, mock_features):\n        \"\"\"\n        Tests exception is raised if CUDA feature is ON but cuda version could not be determined\n        \"\"\"\n        mock_features.return_value = {'ONEDNN': True, 'CUDA': True}\n        mock_cuda_version.return_value = None\n        with self.assertRaises(RuntimeError):\n            probe_mxnet_variant('libmxnet.so')\n\n    def test_probe_artifact_repository_bucket(self):\n        \"\"\"\n        Tests artiact repository bucket is retrieved from environment variable ARTIFACT_REPOSITORY_BUCKET\n        \"\"\"\n        with patch.dict('os.environ', {'ARTIFACT_REPOSITORY_BUCKET': 'some bucket'}):\n            self.assertEqual(probe_artifact_repository_bucket(), 'some bucket')\n\n    @patch('artifact_repository.probe_commit_id')\n    def test_probe_no_commit_id(self, mock):\n        \"\"\"\n        Tests commit id gets probed if not set by user\n        \"\"\"\n        fake_args = TestArtifactRepositoryTool.create_argparse_namespace(git_sha=None)\n        mock.return_value = 'deadbeef'\n        probe(fake_args)\n        mock.assert_called_once()\n        self.assertEqual(fake_args.git_sha, 'deadbeef')\n\n    @patch('artifact_repository.probe_commit_id')\n    def test_probe_no_commit_id_failed(self, mock):\n        \"\"\"\n        Tests script will exit if commid id probe fails\n        \"\"\"\n        fake_args = TestArtifactRepositoryTool.create_argparse_namespace(git_sha=None)\n        mock.return_value = None\n        with self.assertRaises(SystemExit):\n            probe(fake_args)\n\n\n    @patch('artifact_repository.probe_operating_system')\n    def test_probe_no_operating_system(self, mock):\n        \"\"\"\n        Tests operating system gets probed if not set by user\n        \"\"\"\n        fake_args = TestArtifactRepositoryTool.create_argparse_namespace(operating_system=None)\n        mock.return_value = 'be/os'\n        probe(fake_args)\n        mock.assert_called_once()\n        self.assertEqual(fake_args.os, 'be/os')\n\n    @patch('artifact_repository.probe_operating_system')\n    def test_probe_no_operating_system_failed(self, mock):\n        \"\"\"\n        Tests script will exit if operating system probe fails\n        \"\"\"\n        fake_args = TestArtifactRepositoryTool.create_argparse_namespace(operating_system=None)\n        mock.return_value = None\n        with self.assertRaises(SystemExit):\n            probe(fake_args)\n\n    @patch('artifact_repository.probe_mxnet_variant')\n    def test_probe_no_variant(self, mock):\n        \"\"\"\n        Tests mxnet variant gets probed if not set by user\n        \"\"\"\n        fake_args = TestArtifactRepositoryTool.create_argparse_namespace(variant=None)\n        mock.return_value = 'cpu90mkl'\n        probe(fake_args)\n        mock.assert_called_once()\n        self.assertEqual(fake_args.variant, 'cpu90mkl')\n\n    @patch('artifact_repository.probe_mxnet_variant')\n    def test_probe_no_mxnet_variant_failed(self, mock):\n        \"\"\"\n        Tests script will exit if mxnet variant probe fails\n        \"\"\"\n        fake_args = TestArtifactRepositoryTool.create_argparse_namespace(variant=None)\n        mock.return_value = None\n        with self.assertRaises(SystemExit):\n            probe(fake_args)\n\n    @patch('artifact_repository.probe_artifact_repository_bucket')\n    def test_probe_no_bucket(self, mock):\n        \"\"\"\n        Tests artifact repository bucket gets probed if not set by user\n        \"\"\"\n        fake_args = TestArtifactRepositoryTool.create_argparse_namespace(bucket=None)\n        mock.return_value = 'bucket'\n        probe(fake_args)\n        mock.assert_called_once()\n        self.assertEqual(fake_args.bucket, 'bucket')\n\n    @patch('artifact_repository.probe_artifact_repository_bucket')\n    def test_probe_no_bucket_failed(self, mock):\n        \"\"\"\n        Tests script will exit if bucket probe fails\n        \"\"\"\n        fake_args = TestArtifactRepositoryTool.create_argparse_namespace(bucket=None)\n        mock.return_value = None\n        with self.assertRaises(SystemExit):\n            probe(fake_args)\n\n    def test_get_s3_key_prefix(self):\n        \"\"\"\n        Tests S3 key prefix is properly formated\n        \"\"\"\n        fake_args = TestArtifactRepositoryTool.create_argparse_namespace(git_sha=\"abc123\",\n                                                                         operating_system='linux',\n                                                                         variant='cpu',\n                                                                         libtype='static')\n\n        self.assertEqual(get_s3_key_prefix(fake_args), 'abc123/static/linux/cpu/')\n\n    def test_get_s3_key_prefix_with_subdir(self):\n        \"\"\"\n        Tests S3 key prefix with sub-directory is properly formated\n        \"\"\"\n        fake_args = TestArtifactRepositoryTool.create_argparse_namespace(git_sha=\"abc123\",\n                                                                         operating_system='linux',\n                                                                         variant='cpu',\n                                                                         libtype='static')\n\n        self.assertEqual(get_s3_key_prefix(fake_args, subdir='subdir'), 'abc123/static/linux/cpu/subdir/')\n\n    @patch('artifact_repository.s3')\n    def test_try_s3_download_fails_on_bad_response(self, mock_s3):\n        \"\"\"\n        Tests RuntimeError is thrown if the response is malformed\n        \"\"\"\n        key_prefix = 'some/key/prefix'\n        mock_s3.list_objects_v2.return_value = {\n            'something': 'not quite right'\n        }\n\n        with self.assertRaises(RuntimeError):\n            try_s3_download(bucket='bucket', s3_key_prefix=key_prefix, destination='')\n\n    @patch('artifact_repository.s3')\n    def test_try_s3_download_returns_false_on_no_keys(self, mock_s3):\n        \"\"\"\n        Tests False is returned when there are no keys for the prefix. Ie. no artifact to download\n        \"\"\"\n        key_prefix = 'some/key/prefix'\n        mock_s3.list_objects_v2.return_value = {\n            'KeyCount': 0\n        }\n        self.assertFalse(try_s3_download(bucket='bucket', s3_key_prefix=key_prefix, destination=''))\n\n    @patch('artifact_repository.os.makedirs', autospec=True)\n    @patch('artifact_repository.s3')\n    def test_try_s3_download_with_destination(self, mock_s3, mock_makedirs):\n        \"\"\"\n        Tests files are downloaded to the right destinations when destination parameter is not empty\n        \"\"\"\n        key_prefix = 'some/key/prefix'\n        s3_keys = [\n            {'Key': '{}/file.txt'.format(key_prefix)},\n            {'Key': '{}/subdir/other.txt'.format(key_prefix)},\n            {'Key': '{}/another/sub/dir/f.txt'.format(key_prefix)}\n        ]\n\n        mock_s3.list_objects_v2.return_value = {\n            'Contents': s3_keys,\n            'KeyCount': 3\n        }\n\n        mock_s3.download_fileobj = MagicMock()\n        mock_fopen = mock_open()\n\n        with patch('artifact_repository.open', mock_fopen, create=True):\n            dest = os.path.join('dest', 'ination')\n            self.assertTrue(try_s3_download(bucket='bucket', s3_key_prefix=key_prefix, destination=dest))\n\n            # Assert directories are created\n            mock_makedirs.assert_has_calls([\n                unittest.mock.call(dest, exist_ok=True),\n                unittest.mock.call(os.path.join(dest, 'subdir'), exist_ok=True),\n                unittest.mock.call(os.path.join(dest, 'another', 'sub', 'dir'), exist_ok=True)\n            ])\n\n            # Assert files are downloaded\n            mock_s3.download_fileobj.assert_has_calls([\n                unittest.mock.call(Bucket='bucket', Fileobj=unittest.mock.ANY, Key=s3_keys[0]['Key']),\n                unittest.mock.call(Bucket='bucket', Fileobj=unittest.mock.ANY, Key=s3_keys[1]['Key']),\n                unittest.mock.call(Bucket='bucket', Fileobj=unittest.mock.ANY, Key=s3_keys[2]['Key']),\n            ])\n\n    @patch('artifact_repository.os.makedirs', autospec=True)\n    @patch('artifact_repository.s3')\n    def test_try_s3_download(self, mock_s3, mock_makedirs):\n        \"\"\"\n        Tests files are downloaded to the right destinations when destination parameter is empty\n        \"\"\"\n        key_prefix = 'some/key/prefix'\n        s3_keys = [\n            {'Key': '{}/file.txt'.format(key_prefix)},\n            {'Key': '{}/subdir/other.txt'.format(key_prefix)},\n            {'Key': '{}/another/sub/dir/f.txt'.format(key_prefix)}\n        ]\n\n        mock_s3.list_objects_v2.return_value = {\n            'Contents': s3_keys,\n            'KeyCount': 3\n        }\n\n        mock_s3.download_fileobj = MagicMock()\n        mock_fopen = mock_open()\n\n        with patch('artifact_repository.open', mock_fopen, create=True):\n            dest = ''\n            self.assertTrue(try_s3_download(bucket='bucket', s3_key_prefix=key_prefix, destination=dest))\n\n            # Assert directories are created\n            mock_makedirs.assert_has_calls([\n                unittest.mock.call(dest, exist_ok=True),\n                unittest.mock.call(os.path.join(dest, 'subdir'), exist_ok=True),\n                unittest.mock.call(os.path.join(dest, 'another', 'sub', 'dir'), exist_ok=True)\n            ])\n\n            # Assert files are downloaded\n            mock_s3.download_fileobj.assert_has_calls([\n                unittest.mock.call(Bucket='bucket', Fileobj=unittest.mock.ANY, Key=s3_keys[0]['Key']),\n                unittest.mock.call(Bucket='bucket', Fileobj=unittest.mock.ANY, Key=s3_keys[1]['Key']),\n                unittest.mock.call(Bucket='bucket', Fileobj=unittest.mock.ANY, Key=s3_keys[2]['Key']),\n            ])\n\n    @patch('artifact_repository.s3')\n    def test_s3_upload(self, mock_s3):\n        \"\"\"\n        Tests files are uploaded using the supplied s3_key_prefix\n        \"\"\"\n        key_prefix = 'some/key/prefix'\n        paths = [\n            os.path.join('mainfile.txt'),\n            os.path.join('some/dir/file.txt'),\n            os.path.join('some/other/dir/another.txt'),\n        ]\n\n        mock_s3.upload_fileobj = MagicMock()\n        mock_fopen = mock_open(read_data=b'some data')\n\n        with patch('artifact_repository.open', mock_fopen, create=True):\n            s3_upload(bucket='bucket', s3_key_prefix=key_prefix, paths=paths)\n            mock_s3.upload_fileobj.assert_has_calls([\n                unittest.mock.call(Fileobj=unittest.mock.ANY, Key='some/key/prefix/mainfile.txt', Bucket='bucket'),\n                unittest.mock.call(Fileobj=unittest.mock.ANY, Key='some/key/prefix/file.txt', Bucket='bucket'),\n                unittest.mock.call(Fileobj=unittest.mock.ANY, Key='some/key/prefix/another.txt', Bucket='bucket'),\n            ])\n\n    @patch('artifact_repository.os.path.isfile')\n    @patch('artifact_repository.os.path.exists')\n    def test_is_file_is_file(self, mock_exists, mock_isfile):\n        \"\"\"\n        Tests is file returns True when path exists and is a file\n        \"\"\"\n        mock_exists.return_value = True\n        mock_isfile.return_value = True\n        self.assertTrue(is_file('some/path'))\n\n    @patch('artifact_repository.os.path.isfile')\n    @patch('artifact_repository.os.path.exists')\n    def test_is_file_not_file(self, mock_exists, mock_isfile):\n        \"\"\"\n        Tests is file returns False when path exists and is _not_ a file\n        \"\"\"\n        mock_exists.return_value = True\n        mock_isfile.return_value = False\n        self.assertFalse(is_file('some/path'))\n\n    @patch('artifact_repository.os.path.exists')\n    def test_is_file_not_found(self, mock_exists):\n        \"\"\"\n        Tests FileNotFound error thrown if file not found\n        \"\"\"\n        mock_exists.return_value = False\n        with self.assertRaises(FileNotFoundError) as ctx:\n            is_file('some/path')\n        self.assertEqual(str(ctx.exception), 'File \\'{}\\' not found'.format('some/path'))\n\n    def test_sanitize_path_array_empty_paths(self):\n        \"\"\"\n        Tests empty paths are removed\n        \"\"\"\n        self.assertListEqual(sanitize_path_array([' ', '\\t', '     \\n']), [])\n\n    @patch('artifact_repository.is_file')\n    @patch('artifact_repository.glob', autospec=True)\n    def test_sanitize_path_array_directories(self, mock_glob, mock_isfile):\n        \"\"\"\n        Tests directory paths are removed\n        \"\"\"\n        mock_isfile.side_effect = [False, True, False]\n        mock_glob.glob = lambda x: [x]\n        self.assertListEqual(sanitize_path_array(['dir1', 'file', 'dir2']), ['file'])\n\n    def test_write_libmxnet_meta(self):\n        \"\"\"\n        Tests libmxnet.meta is properly written out\n        \"\"\"\n        mock_fopen = mock_open()\n        with patch('artifact_repository.open', mock_fopen, create=True):\n            fake_args = TestArtifactRepositoryTool.create_argparse_namespace(git_sha='abcd1234',\n                                                                             variant='gpu',\n                                                                             operating_system='lunix',\n                                                                             libtype='stynamic')\n            write_libmxnet_meta(args=fake_args, destination='dest')\n            mock_fopen.assert_called_once_with(os.path.join('dest', 'libmxnet.meta'), 'w')\n            mock_fopen().write.called_with(\n                'commit_id: abcd1234\\ndependency_linking: stynamic\\nos: lunix\\nvariant: gpu\\n')\n\n    def test_push_artifact_throws_no_license_error(self):\n        \"\"\"\n        Tests push artifact throwns error if no licenses are defined\n        \"\"\"\n        args = TestArtifactRepositoryTool.create_argparse_namespace(licenses=[])\n        with self.assertRaises(RuntimeError) as ctx:\n            push_artifact(args)\n        self.assertEqual(str(ctx.exception),\n                          \"No licenses defined. Please submit the licenses to be shipped with the binary.\")\n\n        args = args = TestArtifactRepositoryTool.create_argparse_namespace(licenses=None)\n        with self.assertRaises(RuntimeError) as ctx:\n            push_artifact(args)\n        self.assertEqual(str(ctx.exception),\n                          \"No licenses defined. Please submit the licenses to be shipped with the binary.\")\n"
  },
  {
    "path": "ci/Jenkinsfile_docker_cache",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n\n// Jenkins pipeline to generate the centralized docker cache\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\ntotal_timeout = 300\n\nnode('restricted-utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n}\nutils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')\n\nutils.main_wrapper(\ncore_logic: {\n  stage(\"Docker cache build & publish\") {\n    node(NODE_LINUX_CPU) {\n      ws('workspace/docker_cache') {\n        timeout(time: total_timeout, unit: 'MINUTES') {\n          utils.init_git()\n          sh \"cd ci && python3 ./docker_login.py --secret-name ${env.DOCKERHUB_SECRET_NAME} && docker-compose -f docker/docker-compose.yml pull && docker-compose -f docker/docker-compose.yml build --parallel && COMPOSE_HTTP_TIMEOUT=600 docker-compose -f docker/docker-compose.yml push && docker logout\"\n        }\n      }\n    }\n  }\n}\n,\nfailure_handler:\n{\n  if (currentBuild.result == \"FAILURE\") {\n    emailext body: 'Generating the Docker Cache has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[DOCKER CACHE FAILED] Run ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n\n"
  },
  {
    "path": "ci/Jenkinsfile_utils.groovy",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n\n// initialize source codes\ndef init_git() {\n  deleteDir()\n  retry(5) {\n    try {\n      // Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of\n      // retries as this will increase the amount of requests and worsen the throttling\n      timeout(time: 15, unit: 'MINUTES') {\n        checkout scm\n        sh 'git clean -xdff'\n        sh 'git reset --hard'\n        sh 'git submodule update --init --recursive'\n        sh 'git submodule foreach --recursive git clean -ffxd'\n        sh 'git submodule foreach --recursive git reset --hard'\n      }\n    } catch (exc) {\n      deleteDir()\n      error \"Failed to fetch source codes with ${exc}\"\n      sleep 2\n    }\n  }\n}\n\ndef init_git_win() {\n  deleteDir()\n  retry(5) {\n    try {\n      // Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of\n      // retries as this will increase the amount of requests and worsen the throttling\n      timeout(time: 15, unit: 'MINUTES') {\n        checkout scm\n        bat 'git clean -xdff'\n        bat 'git reset --hard'\n        bat 'git submodule update --init --recursive'\n        bat 'git submodule foreach --recursive git clean -ffxd'\n        bat 'git submodule foreach --recursive git reset --hard'\n      }\n    } catch (exc) {\n      deleteDir()\n      error \"Failed to fetch source codes with ${exc}\"\n      sleep 2\n    }\n  }\n}\n\n// pack libraries for later use\ndef pack_lib(name, libs, include_gcov_data = false) {\n  sh returnStatus: true, script: \"\"\"\nset +e\necho \"Packing ${libs} into ${name}\"\nfor i in \\$(echo ${libs} | sed -e 's/,/ /g'); do md5sum \\$i; ls -lh \\$i; done\nreturn 0\n\"\"\"\n  stash includes: libs, name: name\n\n  if (include_gcov_data) {\n    // Store GCNO files that are required for GCOV to operate during runtime\n    sh \"find . -name '*.gcno'\"\n    stash name: \"${name}_gcov_data\", includes: \"**/*.gcno\"\n  }\n}\n\n// unpack libraries saved before\ndef unpack_and_init(name, libs, include_gcov_data = false) {\n  init_git()\n  unstash name\n  sh returnStatus: true, script: \"\"\"\nset +e\necho \"Unpacked ${libs} from ${name}\"\nfor i in \\$(echo ${libs} | sed -e 's/,/ /g'); do md5sum \\$i; done\nreturn 0\n\"\"\"\n  if (include_gcov_data) {\n    // Restore GCNO files that are required for GCOV to operate during runtime\n    unstash \"${name}_gcov_data\"\n  }\n}\n\ndef get_jenkins_master_url() {\n    return env.BUILD_URL.split('/')[2].split(':')[0]\n}\n\ndef get_git_commit_hash() {\n  lastCommitMessage = sh (script: \"git log -1 --pretty=%B\", returnStdout: true)\n  lastCommitMessage = lastCommitMessage.trim()\n  if (lastCommitMessage.startsWith(\"Merge commit '\") && lastCommitMessage.endsWith(\"' into HEAD\")) {\n      // Merge commit applied by Jenkins, skip that commit\n      git_commit_hash = sh (script: \"git rev-parse @~\", returnStdout: true)\n  } else {\n      git_commit_hash = sh (script: \"git rev-parse @\", returnStdout: true)\n  }\n  return git_commit_hash\n}\n\ndef publish_test_coverage() {\n    run \"aws s3 cp s3://mxnet-ci-codecov/codecov ./ && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN}\"\n}\n\ndef collect_test_results_unix(original_file_name, new_file_name) {\n    if (fileExists(original_file_name)) {\n        // Rename file to make it distinguishable. Unfortunately, it's not possible to get STAGE_NAME in a parallel stage\n        // Thus, we have to pick a name manually and rename the files so that they can be stored separately.\n        sh 'cp ' + original_file_name + ' ' + new_file_name\n        archiveArtifacts artifacts: new_file_name\n        try {\n          s3Upload(file:new_file_name, bucket:env.MXNET_CI_UNITTEST_ARTIFACT_BUCKET, path:env.JOB_NAME+\"/\"+env.BUILD_NUMBER+\"/\"+new_file_name)\n        } catch (Exception e) {\n          echo \"S3 Upload failed ${e}\"\n          throw new Exception(\"S3 upload failed\", e)\n        }\n    }\n}\n\ndef collect_test_results_windows(original_file_name, new_file_name) {\n    // Rename file to make it distinguishable. Unfortunately, it's not possible to get STAGE_NAME in a parallel stage\n    // Thus, we have to pick a name manually and rename the files so that they can be stored separately.\n    if (fileExists(original_file_name)) {\n        bat 'xcopy ' + original_file_name + ' ' + new_file_name + '*'\n        archiveArtifacts artifacts: new_file_name\n        try {\n          s3Upload(file:new_file_name, bucket:env.MXNET_CI_UNITTEST_ARTIFACT_BUCKET, path:env.JOB_NAME+\"/\"+env.BUILD_NUMBER+\"/\"+new_file_name)\n        } catch (Exception e) {\n          echo \"S3 Upload failed ${e}\"\n          throw new Exception(\"S3 upload failed\", e)\n        }\n    }\n}\n\n\ndef docker_run(platform, function_name, use_nvidia = false, shared_mem = '500m', env_vars = [],\n               build_args = \"\") {\n  def command = \"ci/build.py %ENV_VARS% %BUILD_ARGS% --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%\"\n  if (env_vars instanceof String || env_vars instanceof GString) {\n    env_vars = [env_vars]\n  }\n  env_vars << \"BRANCH=${env.BRANCH_NAME}\"\n  def env_vars_str = \"-e \" + env_vars.join(' ')\n  command = command.replaceAll('%ENV_VARS%', env_vars_str)\n  command = command.replaceAll('%BUILD_ARGS%', build_args.length() > 0 ? \"${build_args}\" : '')\n  command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '')\n  command = command.replaceAll('%PLATFORM%', platform)\n  command = command.replaceAll('%FUNCTION_NAME%', function_name)\n  command = command.replaceAll('%SHARED_MEM%', shared_mem)\n\n  sh command\n}\n\n// Allow publishing to GitHub with a custom context (the status shown under a PR)\n// Credit to https://plugins.jenkins.io/github\ndef get_repo_url() {\n  checkout scm\n  return sh(returnStdout: true, script: \"git config --get remote.origin.url\").trim()\n}\n\ndef update_github_commit_status(state, message) {\n  node(NODE_UTILITY) {\n    // NOTE: https://issues.jenkins-ci.org/browse/JENKINS-39482\n    //The GitHubCommitStatusSetter requires that the Git Server is defined under\n    //*Manage Jenkins > Configure System > GitHub > GitHub Servers*.\n    //Otherwise the GitHubCommitStatusSetter is not able to resolve the repository name\n    //properly and you would see an empty list of repos:\n    //[Set GitHub commit status (universal)] PENDING on repos [] (sha:xxxxxxx) with context:test/mycontext\n    //See https://cwiki.apache.org/confluence/display/MXNET/Troubleshooting#Troubleshooting-GitHubcommit/PRstatusdoesnotgetpublished\n\n    echo \"Publishing commit status...\"\n\n    repoUrl = get_repo_url()\n    echo \"repoUrl=${repoUrl}\"\n\n    commitSha = get_git_commit_hash()\n    echo \"commitSha=${commitSha}\"\n\n    context = get_github_context()\n    echo \"context=${context}\"\n\n    echo \"Publishing commit status...\"\n    step([\n      $class: 'GitHubCommitStatusSetter',\n      reposSource: [$class: \"ManuallyEnteredRepositorySource\", url: repoUrl],\n      contextSource: [$class: \"ManuallyEnteredCommitContextSource\", context: context],\n      commitShaSource: [$class: \"ManuallyEnteredShaSource\", sha: commitSha],\n      statusBackrefSource: [$class: \"ManuallyEnteredBackrefSource\", backref: \"${env.RUN_DISPLAY_URL}\"],\n      errorHandlers: [[$class: 'ShallowAnyErrorHandler']],\n      statusResultSource: [\n        $class: 'ConditionalStatusResultSource',\n        results: [[$class: \"AnyBuildResult\", message: message, state: state]]\n      ]\n    ])\n\n    echo \"Publishing commit status done.\"\n\n  }\n}\n\ndef get_github_context() {\n  // Since we use multi-branch pipelines, Jenkins appends the branch name to the job name\n  if (env.BRANCH_NAME) {\n    short_job_name = JOB_NAME.substring(0, JOB_NAME.lastIndexOf('/'))\n  } else {\n    short_job_name = JOB_NAME\n  }\n\n  return \"ci/jenkins/${short_job_name}\"\n}\n\ndef parallel_stage(stage_name, steps) {\n    // Allow to pass an array of steps that will be executed in parallel in a stage\n    new_map = [:]\n\n    for (def step in steps) {\n        new_map = new_map << step\n    }\n\n    stage(stage_name) {\n      parallel new_map\n    }\n}\n\ndef assign_node_labels(args) {\n  // This function allows to assign instance labels to the generalized placeholders.\n  // This serves two purposes:\n  // 1. Allow generalized placeholders (e.g. NODE_WINDOWS_CPU) in the job definition\n  //    in order to abstract away the underlying node label. This allows to schedule a job\n  //    onto a different node for testing or security reasons. This could be, for example,\n  //    when you want to test a new set of slaves on separate labels or when a job should\n  //    only be run on restricted slaves\n  // 2. Restrict the allowed job types within a Jenkinsfile. For example, a UNIX-CPU-only\n  //    Jenkinsfile should not allowed access to Windows or GPU instances. This prevents\n  //    users from just copy&pasting something into an existing Jenkinsfile without\n  //    knowing about the limitations.\n  NODE_LINUX_CPU = args.linux_cpu\n  NODE_LINUX_GPU = args.linux_gpu\n  NODE_LINUX_GPU_G4 = args.linux_gpu_g4\n  NODE_LINUX_GPU_G5 = args.linux_gpu_g5\n  NODE_LINUX_GPU_P3 = args.linux_gpu_p3\n  NODE_WINDOWS_CPU = args.windows_cpu\n  NODE_WINDOWS_GPU = args.windows_gpu\n  NODE_UTILITY = args.utility\n}\n\ndef main_wrapper(args) {\n  // Main Jenkinsfile pipeline wrapper handler that allows to wrap core logic into a format\n  // that supports proper failure handling\n  // args:\n  // - core_logic: Jenkins pipeline containing core execution logic\n  // - failure_handler: Failure handler\n\n  // assign any caught errors here\n  err = null\n  try {\n    update_github_commit_status('PENDING', 'Job has been enqueued')\n    args['core_logic']()\n\n    // set build status to success at the end\n    currentBuild.result = \"SUCCESS\"\n    update_github_commit_status('SUCCESS', 'Job succeeded')\n  } catch (caughtError) {\n    node(NODE_UTILITY) {\n      echo \"caught ${caughtError}\"\n      err = caughtError\n      currentBuild.result = \"FAILURE\"\n      update_github_commit_status('FAILURE', 'Job failed')\n    }\n  } finally {\n    node(NODE_UTILITY) {\n      // Call failure handler\n      args['failure_handler']()\n\n      // Clean workspace to reduce space requirements\n      cleanWs()\n\n      // Remember to rethrow so the build is marked as failing\n      if (err) {\n        throw err\n      }\n    }\n  }\n}\n\nreturn this\n"
  },
  {
    "path": "ci/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# Containerized build & test utilities\n\nThis folder contains scripts and dockerfiles used to build and test MXNet using\nDocker containers\n\nYou need `docker` and `docker-compose`. Install them on Ubuntu via `sudo apt-get\ninstall docker.io docker-compose python3-docker`. To run docker without\nadministrative priviledges as your local user, further run `sudo usermod -a -G\ndocker $USER`.\n\n## build.py\n\nThe main utility to build is build.py which will run docker and mount the mxnet\nfolder as a volume to do in-place builds.\n\nThe build.py script does two functions, build the docker image, and it can be\nalso used to run commands inside this image with the propper mounts and\nparaphernalia required to build mxnet inside docker from the sources on the\nparent folder.\n\nA set of helper shell functions are in `docker/runtime_functions.sh`.\n`build.py` without arguments or `build.py --help` will display usage\ninformation about the tool.\n\nTo build for armv7 for example:\n\n```\n./build.py -p armv7\n```\n\nTo work inside a container with a shell you can do:\n\n```\n./build.py -p ubuntu_cpu -i\n```\n\nWhen building, the artifacts are located in the build/ directory in the project root. In case\n`build.py -a` is invoked, the artifacts are located in build.<platform>/\n\n## Testing with ARM / Edge devices with QEMU\n\nWe build on [QEMU](https://www.qemu.org/) and Linux [Kernel Support for\nmiscellaneous Binary\nFormats](https://www.kernel.org/doc/html/v5.6/admin-guide/binfmt-misc.html) for\ntesting MXNet on edge devices. Test can be invoked with the same syntax as for\nnon-virtualized platforms:\n\n```\n./build.py -p armv7\n./build.py -p test.armv7 /work/runtime_functions.sh unittest_ubuntu_python3_arm\n```\n\nFor the test step to succeed, you must run Linux kernel 4.8 or later and have qemu installed.\n\nOn Debian and Ubuntu systems, run the following command to install the dependencies:\n```\nsudo apt install binfmt-support qemu-user-static\n\n# Use qemu-binfmt-conf.sh to register all binary types with the kernel\nwget https://raw.githubusercontent.com/qemu/qemu/stable-4.1/scripts/qemu-binfmt-conf.sh\nchmod +x qemu-binfmt-conf.sh\nsudo ./qemu-binfmt-conf.sh --persistent yes --qemu-suffix \"-static\" --qemu-path \"/usr/bin\" --systemd ALL\n```\n\nIf you run into segmentation faults at the beginning of the emulated tests, you\nprobably have a ancient version of Qemu on your system (or found a bug in\nupstream Qemu). In that situation, you can rely on the\n`multiarch/qemu-user-static` Docker project to register a set of up-to-date Qemu\nbinaries from their Docker image with your kernel:\n\n```\ndocker run --rm --privileged multiarch/qemu-user-static --reset -p yes\n```\n\n# Development\n\n## Add a platform\n\nTo add a platform, you should add the appropriate dockerfile in\ndocker/Dockerfile.build.<platform> and add a shell function named\nbuild_<platform> to the file docker/runtime_functions.sh with build\ninstructions for that platform.\n\n## ccache\nFor all builds a directory from the host system is mapped where ccache will store cached\ncompiled object files (defaults to /tmp/ci_ccache). This will speed up rebuilds\nsignificantly. You can set this directory explicitly by setting CCACHE_DIR environment\nvariable. All ccache instances are currently set to be 10 Gigabytes max in size.\n\n## Docker container cleanup (Zombie containers)\n\nDocker has a client-server architecture, so when the program that is executing the docker client\ndies or receieves a signal, the container keeps running as it's started by the docker daemon.\nWe implement signal handlers that catch sigterm and sigint and cleanup containers before exit. In\nJenkins there's not enough time between sigterm and sigkill so we guarantee that containers are not\nleft running by propagating environment variables used by the Jenkins process tree killer to\nidentify which process to kill when the job is stopped. This has the effect of stopping the\ncontainer given that the process inside the container is terminated.\n\nHow to test this is working propperly: On the console you can hit ^C while a container is running\n(not just building) and see that the container is stopped by running `docker ps` on another\nterminal. In Jenkins this has been tested by stopping the job which has containers running and\nverifying that the container stops shortly afterwards by running docker ps.\n"
  },
  {
    "path": "ci/__init__.py",
    "content": "# -*- coding: utf-8 -*-\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n"
  },
  {
    "path": "ci/build.py",
    "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Multi arch dockerized build tool.\"\"\"\n\n__author__ = 'Marco de Abreu, Kellen Sunderland, Anton Chernov, Pedro Larroy, Leonard Lausen'\n__version__ = '0.4'\n\nimport argparse\nimport pprint\nimport os\nimport signal\nimport subprocess\nfrom itertools import chain\nfrom subprocess import check_call\nfrom typing import *\n\nimport yaml\n\nfrom util import *\n\n\ndef get_platforms() -> List[str]:\n    \"\"\"Get a list of architectures declared in docker-compose.yml\"\"\"\n    with open(\"docker/docker-compose.yml\", \"r\") as f:\n        compose_config = yaml.load(f.read(), yaml.SafeLoader)\n    return list(compose_config[\"services\"].keys())\n\ndef get_docker_tag(platform: str, registry: str) -> str:\n    \"\"\":return: docker tag to be used for the container\"\"\"\n    with open(\"docker/docker-compose.yml\", \"r\") as f:\n        compose_config = yaml.load(f.read(), yaml.SafeLoader)\n        return compose_config[\"services\"][platform][\"image\"].replace('${DOCKER_CACHE_REGISTRY}', registry)\n\ndef build_docker(platform: str, registry: str, num_retries: int, no_cache: bool,\n                 cache_intermediate: bool = False) -> str:\n    \"\"\"\n    Build a container for the given platform\n    :param platform: Platform\n    :param registry: Dockerhub registry name\n    :param num_retries: Number of retries to build the docker image\n    :param no_cache: pass no-cache to docker to rebuild the images\n    :return: Id of the top level image\n    \"\"\"\n    logging.info('Building docker container \\'%s\\' based on ci/docker/docker-compose.yml', platform)\n    # We add a user with the same group as the executing non-root user so files created in the\n    # container match permissions of the local user. Same for the group.\n    cmd = ['docker-compose', '-f', 'docker/docker-compose.yml', 'build',\n           \"--build-arg\", \"USER_ID={}\".format(os.getuid()),\n           \"--build-arg\", \"GROUP_ID={}\".format(os.getgid())]\n    if cache_intermediate:\n        cmd.append('--no-rm')\n    cmd.append(platform)\n\n    env = os.environ.copy()\n    env[\"DOCKER_CACHE_REGISTRY\"] = registry\n\n    @retry(subprocess.CalledProcessError, tries=num_retries)\n    def run_cmd(env=None):\n        logging.info(\"Running command: '%s'\", ' '.join(cmd))\n        check_call(cmd, env=env)\n\n    run_cmd(env=env)\n\n\ndef buildir() -> str:\n    return os.path.join(get_mxnet_root(), \"build\")\n\n\ndef default_ccache_dir() -> str:\n    \"\"\":return: ccache directory for the current platform\"\"\"\n    # Share ccache across containers\n    if 'CCACHE_DIR' in os.environ:\n        ccache_dir = os.path.realpath(os.environ['CCACHE_DIR'])\n        try:\n            os.makedirs(ccache_dir, exist_ok=True)\n            return ccache_dir\n        except PermissionError:\n            logging.info('Unable to make dirs at %s, falling back to local temp dir', ccache_dir)\n    # In osx tmpdir is not mountable by default\n    import platform\n    if platform.system() == 'Darwin':\n        ccache_dir = \"/tmp/_mxnet_ccache\"\n        os.makedirs(ccache_dir, exist_ok=True)\n        return ccache_dir\n    return os.path.join(os.path.expanduser(\"~\"), \".ccache\")\n\n\ndef container_run(platform: str,\n                  nvidia_runtime: bool,\n                  docker_registry: str,\n                  shared_memory_size: str,\n                  local_ccache_dir: str,\n                  command: List[str],\n                  environment: Dict[str, str],\n                  dry_run: bool = False) -> int:\n    \"\"\"Run command in a container\"\"\"\n    # set default environment variables\n    environment.update({\n        'CCACHE_MAXSIZE': '500G',\n        'CCACHE_TEMPDIR': '/tmp/ccache',  # temp dir should be local and not shared\n        'CCACHE_DIR': '/work/ccache',  # this path is inside the container as /work/ccache is mounted\n        'CCACHE_LOGFILE': '/tmp/ccache.log',  # a container-scoped log, useful for ccache verification.\n    })\n    environment.update({k: os.environ[k] for k in ['CCACHE_MAXSIZE'] if k in os.environ})\n    if 'RELEASE_BUILD' not in environment:\n        environment['RELEASE_BUILD'] = 'false'\n\n    tag = get_docker_tag(platform=platform, registry=docker_registry)\n    mx_root = get_mxnet_root()\n    local_build_folder = buildir()\n    # We need to create it first, otherwise it will be created by the docker daemon with root only permissions\n    os.makedirs(local_build_folder, exist_ok=True)\n    os.makedirs(local_ccache_dir, exist_ok=True)\n    logging.info(\"Using ccache directory: %s\", local_ccache_dir)\n\n    # Log enviroment\n    logging.info(\"environment ---> {0}\".format(environment))\n\n    # Build docker command\n    docker_arg_list = [\n        \"--cap-add\", \"SYS_PTRACE\", # Required by ASAN\n        '--rm',\n        '--shm-size={}'.format(shared_memory_size),\n        # mount mxnet root\n        '-v', \"{}:/work/mxnet\".format(mx_root),\n        # mount mxnet/build for storing build\n        '-v', \"{}:/work/build\".format(local_build_folder),\n        '-v', \"{}:/work/ccache\".format(local_ccache_dir),\n        '-u', '{}:{}'.format(os.getuid(), os.getgid()),\n        '-e', 'CCACHE_MAXSIZE={}'.format(environment['CCACHE_MAXSIZE']),\n        # temp dir should be local and not shared\n        '-e', 'CCACHE_TEMPDIR={}'.format(environment['CCACHE_TEMPDIR']),\n        # this path is inside the container as /work/ccache is mounted\n        '-e', 'CCACHE_DIR={}'.format(environment['CCACHE_DIR']),\n        # a container-scoped log, useful for ccache verification.\n        '-e', 'CCACHE_LOGFILE={}'.format(environment['CCACHE_LOGFILE']),\n        # whether this is a release build or not\n        '-e', 'RELEASE_BUILD={}'.format(environment['RELEASE_BUILD']),\n    ]\n    docker_arg_list += [tag]\n    docker_arg_list.extend(command)\n\n    def docker_run_cmd(cmd):\n        logging.info(\"Running %s in container %s\", command, tag)\n        logging.info(\"Executing command:\\n%s\\n\", ' \\\\\\n\\t'.join(cmd))\n        subprocess.run(cmd, stdout=sys.stdout, stderr=sys.stderr, check=True)\n\n    if not dry_run:\n        if not nvidia_runtime:\n            docker_run_cmd(['docker', 'run'] + docker_arg_list)\n        else:\n            try:\n                docker_run_cmd(['docker', 'run', '--gpus', 'all'] + docker_arg_list)\n            except subprocess.CalledProcessError as e:\n                if e.returncode == 125:\n                    docker_run_cmd(['docker', 'run', '--runtime', 'nvidia'] + docker_arg_list)\n                else:\n                    raise\n\n    return 0\n\n\ndef list_platforms() -> str:\n    return \"\\nSupported platforms:\\n{}\".format('\\n'.join(get_platforms()))\n\n\ndef load_docker_cache(platform, tag, docker_registry) -> None:\n    \"\"\"Imports tagged container from the given docker registry\"\"\"\n    if docker_registry:\n        env = os.environ.copy()\n        env[\"DOCKER_CACHE_REGISTRY\"] = docker_registry\n        cmd = ['docker-compose', '-f', 'docker/docker-compose.yml', 'pull', platform]\n        logging.info(\"Running command: 'DOCKER_CACHE_REGISTRY=%s %s'\", docker_registry, ' '.join(cmd))\n        check_call(cmd, env=env)\n    else:\n        logging.info('Distributed docker cache disabled')\n\n\ndef log_environment():\n    instance_info = ec2_instance_info()\n    if instance_info:\n        logging.info(\"EC2: %s\", instance_info)\n    pp = pprint.PrettyPrinter(indent=4)\n    logging.debug(\"Build environment: %s\", pp.pformat(dict(os.environ)))\n\n\ndef main() -> int:\n    config_logging()\n\n    logging.info(\"MXNet container based build tool.\")\n    log_environment()\n    chdir_to_script_directory()\n\n    parser = argparse.ArgumentParser(description=\"\"\"Utility for building and testing MXNet on docker\n    containers\"\"\", epilog=\"\")\n    parser.add_argument(\"-p\", \"--platform\", type=str, help= \\\n                        \"Platform. See ci/docker/docker-compose.yml for list of supported \" \\\n                        \"platforms (services).\")\n\n    parser.add_argument(\"-b\", \"--build-only\",\n                        help=\"Only build the container, don't build the project\",\n                        action='store_true')\n\n    parser.add_argument(\"-R\", \"--run-only\",\n                        help=\"Only run the container, don't rebuild the container\",\n                        action='store_true')\n\n    parser.add_argument(\"-n\", \"--nvidiadocker\",\n                        help=\"Use nvidia docker\",\n                        action='store_true')\n\n    parser.add_argument(\"--shm-size\",\n                        help=\"Size of the shared memory /dev/shm allocated in the container (e.g '1g')\",\n                        default='500m',\n                        dest=\"shared_memory_size\")\n\n    parser.add_argument(\"-l\", \"--list\",\n                        help=\"List platforms\",\n                        action='store_true')\n\n    parser.add_argument(\"--print-docker-run\",\n                        help=\"print docker run command for manual inspection\",\n                        action='store_true')\n\n    parser.add_argument(\"-d\", \"--docker-registry\",\n                        help=\"Dockerhub registry name to retrieve cache from.\",\n                        default='mxnetci',\n                        type=str)\n\n    parser.add_argument(\"-r\", \"--docker-build-retries\",\n                        help=\"Number of times to retry building the docker image. Default is 1\",\n                        default=1,\n                        type=int)\n\n    parser.add_argument(\"--no-pull\", action=\"store_true\",\n                        help=\"Don't pull from dockerhub registry to initialize cache.\")\n\n    parser.add_argument(\"--no-cache\", action=\"store_true\",\n                        help=\"passes --no-cache to docker build\")\n\n    parser.add_argument(\"--cache-intermediate\", action=\"store_true\",\n                        help=\"passes --rm=false to docker build\")\n\n    parser.add_argument(\"-e\", \"--environment\", nargs=\"*\", default=[],\n                        help=\"Environment variables for the docker container. \"\n                        \"Specify with a list containing either names or name=value\")\n\n    parser.add_argument(\"command\",\n                        help=\"command to run in the container\",\n                        nargs='*', action='append', type=str)\n\n    parser.add_argument(\"--ccache-dir\",\n                        default=default_ccache_dir(),\n                        help=\"ccache directory\",\n                        type=str)\n\n    args = parser.parse_args()\n\n    command = list(chain.from_iterable(args.command))\n    environment = dict([(e.split('=')[:2] if '=' in e else (e, os.environ[e]))\n                        for e in args.environment])\n\n    if args.list:\n        print(list_platforms())\n    elif args.platform:\n        platform = args.platform\n        tag = get_docker_tag(platform=platform, registry=args.docker_registry)\n        if args.docker_registry and not args.no_pull:\n            load_docker_cache(platform=platform, tag=tag, docker_registry=args.docker_registry)\n        if not args.run_only:\n            build_docker(platform=platform, registry=args.docker_registry, num_retries=args.docker_build_retries,\n                         no_cache=args.no_cache, cache_intermediate=args.cache_intermediate)\n        else:\n            logging.info(\"Skipping docker build step.\")\n\n        if args.build_only:\n            logging.warning(\"Container was just built. Exiting due to build-only.\")\n            return 0\n\n        # noinspection PyUnusedLocal\n        ret = 0\n        if command:\n            ret = container_run(\n                platform=platform, nvidia_runtime=args.nvidiadocker,\n                shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,\n                local_ccache_dir=args.ccache_dir, environment=environment)\n        elif args.print_docker_run:\n            command = []\n            ret = container_run(\n                platform=platform, nvidia_runtime=args.nvidiadocker,\n                shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,\n                local_ccache_dir=args.ccache_dir, dry_run=True, environment=environment)\n        else:\n            # With no commands, execute a build function for the target platform\n            command = [\"/work/mxnet/ci/docker/runtime_functions.sh\", \"build_{}\".format(platform)]\n            logging.info(\"No command specified, trying default build: %s\", ' '.join(command))\n            ret = container_run(\n                platform=platform, nvidia_runtime=args.nvidiadocker,\n                shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,\n                local_ccache_dir=args.ccache_dir, environment=environment)\n\n        if ret != 0:\n            logging.critical(\"Execution of %s failed with status: %d\", command, ret)\n            return ret\n\n    else:\n        parser.print_help()\n        list_platforms()\n        print(\"\"\"\nExamples:\n\n./build.py -p armv7\n\n    Will build a docker container with cross compilation tools and build MXNet for armv7 by\n    running: ci/docker/runtime_functions.sh build_armv7 inside the container.\n\n./build.py -p armv7 ls\n\n    Will execute the given command inside the armv7 container\n\n./build.py -p armv7 --print-docker-run\n\n    Will print a docker run command to get inside the container in a shell\n\n./build.py -a\n\n    Builds for all platforms and leaves artifacts in build_<platform>\n\n    \"\"\")\n\n    return 0\n\n\nif __name__ == '__main__':\n    sys.exit(main())\n"
  },
  {
    "path": "ci/build_windows.py",
    "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"User friendly / multi platform builder script\"\"\"\n\nimport argparse\nimport datetime\nimport glob\nimport logging\nimport os\nimport platform\nimport shutil\nimport sys\nimport tempfile\nimport time\nimport zipfile\nimport requests\nfrom distutils.dir_util import copy_tree\nfrom enum import Enum\nfrom subprocess import check_call, call\n\nfrom util import *\n\nKNOWN_VCVARS = {\n    # https://gitlab.kitware.com/cmake/cmake/issues/18920\n    'VS 2015': r'C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat',\n    'VS 2017': r'C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsx86_amd64.bat',\n    'VS 2019': r'C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Community\\VC\\Auxiliary\\Build\\vcvars64.bat',\n}\n\n\nclass BuildFlavour(Enum):\n    WIN_CPU = 'WIN_CPU'\n    WIN_CPU_ONEDNN = 'WIN_CPU_ONEDNN'\n    WIN_CPU_ONEDNN_MKL = 'WIN_CPU_ONEDNN_MKL'\n    WIN_CPU_MKL = 'WIN_CPU_MKL'\n    WIN_GPU = 'WIN_GPU'\n    WIN_GPU_ONEDNN = 'WIN_GPU_ONEDNN'\n\n\nCMAKE_FLAGS = {\n    'WIN_CPU': (\n        '-DCMAKE_C_COMPILER=cl '\n        '-DCMAKE_CXX_COMPILER=cl '\n        '-DUSE_CUDA=OFF '\n        '-DUSE_CUDNN=OFF '\n        '-DUSE_OPENCV=ON '\n        '-DUSE_OPENMP=ON '\n        '-DUSE_BLAS=open '\n        '-DUSE_LAPACK=ON '\n        '-DUSE_DIST_KVSTORE=OFF '\n        '-DBUILD_CPP_EXAMPLES=ON '\n        '-DCMAKE_BUILD_TYPE=Release')\n\n    , 'WIN_CPU_ONEDNN': (\n        '-DCMAKE_C_COMPILER=cl '\n        '-DCMAKE_CXX_COMPILER=cl '\n        '-DUSE_CUDA=OFF '\n        '-DUSE_CUDNN=OFF '\n        '-DUSE_OPENCV=ON '\n        '-DUSE_OPENMP=ON '\n        '-DUSE_BLAS=open '\n        '-DUSE_LAPACK=ON '\n        '-DUSE_DIST_KVSTORE=OFF '\n        '-DUSE_ONEDNN=ON '\n        '-DCMAKE_BUILD_TYPE=Release')\n\n    , 'WIN_CPU_ONEDNN_MKL': (\n        '-DCMAKE_C_COMPILER=cl '\n        '-DCMAKE_CXX_COMPILER=cl '\n        '-DUSE_CUDA=OFF '\n        '-DUSE_CUDNN=OFF '\n        '-DUSE_OPENCV=ON '\n        '-DUSE_OPENMP=ON '\n        '-DUSE_BLAS=mkl '\n        '-DUSE_LAPACK=ON '\n        '-DUSE_DIST_KVSTORE=OFF '\n        '-DUSE_ONEDNN=ON '\n        '-DCMAKE_BUILD_TYPE=Release')\n\n    , 'WIN_CPU_MKL': (\n        '-DCMAKE_C_COMPILER=cl '\n        '-DCMAKE_CXX_COMPILER=cl '\n        '-DUSE_CUDA=OFF '\n        '-DUSE_CUDNN=OFF '\n        '-DUSE_OPENCV=ON '\n        '-DUSE_OPENMP=ON '\n        '-DUSE_BLAS=mkl '\n        '-DUSE_LAPACK=ON '\n        '-DUSE_DIST_KVSTORE=OFF '\n        '-DUSE_ONEDNN=OFF '\n        '-DCMAKE_BUILD_TYPE=Release')\n\n    , 'WIN_GPU': (\n        '-DCMAKE_C_COMPILER=cl '\n        '-DCMAKE_CXX_COMPILER=cl '\n        '-DUSE_CUDA=ON '\n        '-DUSE_CUDNN=ON '\n        '-DUSE_OPENCV=ON  '\n        '-DUSE_OPENMP=ON '\n        '-DUSE_BLAS=open '\n        '-DUSE_LAPACK=ON '\n        '-DUSE_DIST_KVSTORE=OFF '\n        '-DMXNET_CUDA_ARCH=\"5.2 7.5\" '\n        '-DCMAKE_BUILD_TYPE=Release')\n\n    , 'WIN_GPU_ONEDNN': (\n        '-DCMAKE_C_COMPILER=cl '\n        '-DCMAKE_CXX_COMPILER=cl '\n        '-DUSE_CUDA=ON '\n        '-DUSE_CUDNN=ON '\n        '-DUSE_OPENCV=ON '\n        '-DUSE_OPENMP=ON '\n        '-DUSE_BLAS=open '\n        '-DUSE_LAPACK=ON '\n        '-DUSE_DIST_KVSTORE=OFF '\n        '-DMXNET_CUDA_ARCH=\"5.2 7.5\" '\n        '-DUSE_ONEDNN=ON '\n        '-DCMAKE_BUILD_TYPE=Release')\n\n}\n\n\ndef windows_build(args):\n    logging.info(\"Using vcvars environment:\\n{}\".format(args.vcvars))\n    if args.vcvars_ver:\n        logging.info(\"Using vcvars version:\\n{}\".format(args.vcvars_ver))\n\n    path = args.output\n\n    mxnet_root = get_mxnet_root()\n    logging.info(\"Found MXNet root: {}\".format(mxnet_root))\n\n    # cuda thrust / CUB + VS 2019 is flaky: try multiple times if fail\n    MAXIMUM_TRY = 1\n    build_try = 0\n\n    while build_try < MAXIMUM_TRY:\n        if os.path.exists(path):\n            shutil.rmtree(path)\n        os.makedirs(path, exist_ok=True)\n\n        with remember_cwd():\n            os.chdir(path)\n            env = os.environ.copy()\n            if 'GPU' in args.flavour:\n                env[\"CXXFLAGS\"] = '/FS /MD /O2 /Ob2'\n            if not args.vcvars_ver:\n                cmd = \"\\\"{}\\\" && cmake -GNinja {} {}\".format(args.vcvars,\n                                                             CMAKE_FLAGS[args.flavour],\n                                                             mxnet_root)\n            else:\n                cmd = \"\\\"{}\\\" -vcvars_ver={} && cmake -GNinja {} {}\".format(args.vcvars,\n                                                                            args.vcvars_ver,\n                                                                            CMAKE_FLAGS[args.flavour],\n                                                                            mxnet_root)\n            logging.info(\"Generating project with CMake:\\n{}\".format(cmd))\n            check_call(cmd, shell=True, env=env)\n\n            if not args.vcvars_ver:\n                cmd = \"\\\"{}\\\" && ninja\".format(args.vcvars)\n            else:\n                cmd = \"\\\"{}\\\" -vcvars_ver={} && ninja\".format(args.vcvars, args.vcvars_ver)\n            logging.info(\"Building:\\n{}\".format(cmd))\n\n            t0 = int(time.time())\n            ret = call(cmd, shell=True)\n\n\n            if ret != 0:\n                build_try += 1\n                logging.info(\"{} build(s) have failed\".format(build_try))\n            else:\n                logging.info(\"Build flavour: {} complete in directory: \\\"{}\\\"\".format(args.flavour, os.path.abspath(path)))\n                logging.info(\"Build took {}\".format(datetime.timedelta(seconds=int(time.time() - t0))))\n                break\n\n    if ret == 0:\n        windows_package(args)\n    else:\n        logging.info(\"Build failed\")\n        sys.exit(1)\n\n\ndef windows_package(args):\n    pkgfile = 'windows_package.7z'\n    pkgdir = os.path.abspath('windows_package')\n    logging.info(\"Packaging libraries and headers in package: %s\", pkgfile)\n    j = os.path.join\n    pkgdir_lib = os.path.abspath(j(pkgdir, 'lib'))\n    with remember_cwd():\n        os.chdir(args.output)\n        logging.info(\"Looking for static libraries and dlls in: \\\"%s\", os.getcwd())\n        libs = list(glob.iglob('**/*.lib', recursive=True))\n        dlls = list(glob.iglob('**/*.dll', recursive=True))\n        os.makedirs(pkgdir_lib, exist_ok=True)\n        for lib in libs:\n            logging.info(\"packing lib: %s\", lib)\n            shutil.copy(lib, pkgdir_lib)\n        for dll in dlls:\n            logging.info(\"packing dll: %s\", dll)\n            shutil.copy(dll, pkgdir_lib)\n        os.chdir(get_mxnet_root())\n        logging.info('packing python bindings')\n        copy_tree('python', j(pkgdir, 'python'))\n        logging.info('packing headers')\n        copy_tree('include', j(pkgdir, 'include'))\n        logging.info(\"Compressing package: %s\", pkgfile)\n        check_call(['7z', 'a', pkgfile, pkgdir])\n\n\ndef nix_build(args):\n    path = args.output\n    os.makedirs(path, exist_ok=True)\n    with remember_cwd():\n        os.chdir(path)\n        logging.info(\"Generating project with CMake\")\n        check_call(\"cmake \\\n            -DUSE_CUDA=OFF \\\n            -DUSE_BLAS=open \\\n            -DUSE_OPENCV=OFF \\\n            -DUSE_OPENMP=OFF \\\n            -DCMAKE_BUILD_TYPE=Debug \\\n            -GNinja ..\", shell=True)\n        check_call(\"ninja\", shell=True)\n\n\ndef main():\n    logging.getLogger().setLevel(logging.INFO)\n    logging.basicConfig(format='%(asctime)-15s %(message)s')\n    logging.info(\"MXNet Windows build helper\")\n    instance_info = ec2_instance_info()\n    if instance_info:\n        logging.info(\"EC2: %s\", instance_info)\n\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"-o\", \"--output\",\n        help=\"output directory\",\n        default='build',\n        type=str)\n\n    parser.add_argument(\"--vcvars\",\n        help=\"vcvars batch file location, typically inside vs studio install dir\",\n        default=KNOWN_VCVARS['VS 2019'],\n        type=str)\n\n    parser.add_argument(\"--vcvars_ver\",\n        help=\"Optionally specifies the Visual Studio compiler toolset to use.\\\n            By default, the environment is set to use the current Visual Studio compiler toolset.\",\n        default=None,\n        type=str)\n\n    parser.add_argument(\"--arch\",\n        help=\"architecture\",\n        default='x64',\n        type=str)\n\n    parser.add_argument(\"-f\", \"--flavour\",\n        help=\"build flavour\",\n        default='WIN_CPU',\n        choices=[x.name for x in BuildFlavour],\n        type=str)\n\n    args = parser.parse_args()\n    logging.info(\"Build flavour: %s\", args.flavour)\n\n    system = platform.system()\n    if system == 'Windows':\n        logging.info(\"Detected Windows platform\")\n        if 'OpenBLAS_HOME' not in os.environ:\n            os.environ[\"OpenBLAS_HOME\"] = \"C:\\\\Program Files\\\\OpenBLAS-v0.2.19\"\n        if 'OpenCV_DIR' not in os.environ:\n            os.environ[\"OpenCV_DIR\"] = \"C:\\\\Program Files\\\\OpenCV-v3.4.1\\\\build\"\n        if 'CUDA_PATH' not in os.environ:\n            os.environ[\"CUDA_PATH\"] = \"C:\\\\Program Files\\\\NVIDIA GPU Computing Toolkit\\\\CUDA\\\\v10.2\"\n        if 'MKLROOT' not in os.environ:\n            os.environ[\"MKLROOT\"] = \"C:\\\\Program Files (x86)\\\\IntelSWTools\\\\compilers_and_libraries\\\\windows\\\\mkl\"\n        windows_build(args)\n\n    elif system == 'Linux' or system == 'Darwin':\n        nix_build(args)\n\n    else:\n        logging.error(\"Don't know how to build for {} yet\".format(platform.system()))\n\n    return 0\n\n\nif __name__ == '__main__':\n    sys.exit(main())\n"
  },
  {
    "path": "ci/dev_menu.py",
    "content": "#!/usr/bin/env python3\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# -*- coding: utf-8 -*-\n\"\"\"Tool to ease working with the build system and reproducing test results\"\"\"\n\nimport argparse\nimport os\nimport sys\nfrom subprocess import check_call\nimport shlex\nfrom ci.util import retry, remember_cwd\nfrom typing import List\nfrom collections import OrderedDict\nimport logging\nimport yaml\nimport shutil\n\n\nDEFAULT_PYENV=os.environ.get('DEFAULT_PYENV','py3_venv')\nDEFAULT_PYTHON=os.environ.get('DEFAULT_PYTHON','python3')\nDEFAULT_CMAKE_OPTIONS=os.environ.get('DEFAULT_CMAKE_OPTIONS','cmake_options.yml')\n\n\nclass Confirm(object):\n    def __init__(self, cmds):\n        self.cmds = cmds\n\n    def __call__(self):\n        resp = input(\"This will run the following command(s) '{}' are you sure? yes / no: \".format(self.cmds))\n        while True:\n            if resp.lower() == 'yes':\n                handle_commands(self.cmds)\n                return\n            elif resp.lower() == 'no':\n                return\n            else:\n                resp = input(\"Please answer yes or no: \")\n\n\nclass CMake(object):\n    def __init__(self, cmake_options_yaml=DEFAULT_CMAKE_OPTIONS, cmake_options_yaml_default='cmake/cmake_options.yml'):\n        if os.path.exists(cmake_options_yaml):\n            self.cmake_options_yaml = cmake_options_yaml\n        else:\n            self.cmake_options_yaml = cmake_options_yaml_default\n        logging.info('Using {} for CMake configuration'.format(self.cmake_options_yaml))\n        self.cmake_options = None\n        self.read_config()\n\n    def cmake_command(self) -> str:\n        \"\"\"\n        :return: Cmake command to run given the options\n        \"\"\"\n        cmd_lst = ['cmake', '-C', 'config.cmake']\n        cmd_lst.extend(self._cmdlineflags())\n        return cmd_lst\n\n    def __call__(self, build_dir='build', generator='Ninja', build_cmd='ninja'):\n        logging.info(\"CMake / {} build in directory {}\".format(\n            generator, os.path.abspath(build_dir)))\n        cmd_lst = self.cmake_command()\n        os.makedirs(build_dir, exist_ok=True)\n        with remember_cwd():\n            os.chdir(build_dir)\n            cmd_lst.extend(['-G{}'.format(generator), '..'])\n            logging.info('Executing: {}'.format('\\t\\n'.join(cmd_lst)))\n            check_call(cmd_lst)\n            logging.info('Now building')\n            check_call(shlex.split(build_cmd))\n\n\ndef create_virtualenv(venv_exe, pyexe, venv) -> None:\n    logging.info(\"Creating virtualenv in %s with python %s\", venv, pyexe)\n    if not (venv_exe and pyexe and venv):\n        logging.warn(\"Skipping creation of virtualenv\")\n        return\n    check_call([venv_exe, '-p', pyexe, venv])\n\n\ndef create_virtualenv_default():\n    create_virtualenv('virtualenv', DEFAULT_PYTHON, DEFAULT_PYENV)\n    logging.info(\"You can use the virtualenv by executing 'source %s/bin/activate'\", DEFAULT_PYENV)\n\n\ndef provision_virtualenv(venv_path=DEFAULT_PYENV):\n    pip = os.path.join(venv_path, 'bin', 'pip')\n    if os.path.exists(pip):\n        # Install MXNet python bindigs\n        check_call([pip, 'install', '--upgrade', '--force-reinstall', '-e', 'python'])\n        # Install test dependencies\n        check_call([pip, 'install', '--upgrade', '--force-reinstall', '-r',\n                    os.path.join('ci', 'docker', 'install', 'requirements')])\n    else:\n        logging.warn(\"Can't find pip: '%s' not found\", pip)\n\n\nCOMMANDS = OrderedDict([\n    ('[Local] BUILD CMake/Ninja (using cmake_options.yaml (cp cmake/cmake_options.yml .) and edit) ({} virtualenv in \"{}\")'.format(DEFAULT_PYTHON, DEFAULT_PYENV),\n    [\n        CMake(),\n        create_virtualenv_default,\n        provision_virtualenv,\n    ]),\n    ('[Local] Python Unit tests',\n        \"pytest -v tests/python/unittest/\"\n    ),\n    ('[Docker] Build the MXNet binary - outputs to \"lib/\"',\n        \"ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu\"),\n    ('[Docker] Build the Jekyll website - outputs to \"docs/static_site/build/html/\"',\n        \"ci/build.py --platform ubuntu_cpu_jekyll /work/runtime_functions.sh build_jekyll_docs\"),\n    ('[Docker] Build the Python API docs - outputs to \"docs/python_docs/python/build/_build/html/\"',\n        \"ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_python_docs\"),\n    ('[Docker] sanity_check. Check for linting and code formatting and licenses.',\n    [\n        \"ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh sanity_check\",\n    ]),\n    ('[Docker] Python3 CPU unittests',\n    [\n        \"ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_openblas\",\n        \"ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_python3_cpu\",\n    ]),\n    ('[Docker] Python3 GPU unittests',\n    [\n        \"ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh build_ubuntu_gpu\",\n        \"ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_python3_gpu\",\n    ]),\n    ('[Docker] Python3 GPU+oneDNN unittests',\n    [\n        \"ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh build_ubuntu_gpu_onednn\",\n        \"ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_python3_gpu\",\n    ]),\n    ('[Docker] Python3 CPU oneDNN unittests',\n    [\n        \"ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_onednn\",\n        \"ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_python3_cpu\",\n    ]),\n    ('[Docker] Python3 ARMv7 unittests (QEMU)',\n    [\n        \"ci/build.py -p armv7\",\n        \"ci/build.py -p test.armv7 /work/runtime_functions.sh unittest_ubuntu_python3_arm\"\n    ]),\n    ('Clean (RESET HARD) repository (Warning! erases local changes / DATA LOSS)',\n       Confirm(\"ci/docker/runtime_functions.sh clean_repo\"))\n])\n\ndef clip(x, mini, maxi):\n    return min(max(x,mini), maxi)\n\n@retry((ValueError, RuntimeError), 3, delay_s = 0)\ndef show_menu(items: List[str], header=None) -> int:\n    print('\\n-- MXNet dev menu --\\n')\n    def hr():\n        print(''.join(['-']*30))\n    if header:\n        print(header)\n    hr()\n    for i,x in enumerate(items,1):\n        print('{}. {}'.format(i,x))\n    hr()\n    choice = int(input('Choose option> ')) - 1\n    if choice < 0 or choice >= len(items):\n        raise RuntimeError('Choice must be between {} and {}'.format(1, len(items)))\n    return choice\n\ndef handle_commands(cmds) -> None:\n    def handle_command(cmd):\n        logging.info(\"Executing command: %s\",cmd)\n        check_call(shlex.split(cmd))\n\n    if type(cmds) is list:\n        for cmd in cmds:\n            handle_commands(cmd)\n    elif type(cmds) is str:\n        handle_command(cmds)\n    elif callable(cmds):\n        cmds()\n    else:\n        raise RuntimeError(\"handle_commands(cmds): argument should be str or List[str] but is %s\", type(cmds))\n\ndef use_menu_ui(args) -> None:\n    command_list = list(COMMANDS.keys())\n    if hasattr(args, 'choice') and args.choice and args.choice[0].isdigit():\n        choice = int(args.choice[0]) - 1\n    else:\n        choice = show_menu(command_list, 'Available actions')\n    handle_commands(COMMANDS[command_list[choice]])\n\ndef build(args) -> None:\n    \"\"\"Build using CMake\"\"\"\n    venv_exe = shutil.which('virtualenv')\n    pyexe = shutil.which(args.pyexe)\n    if not venv_exe:\n        logging.warn(\"virtualenv wasn't found in path, it's recommended to install virtualenv to manage python environments\")\n    if not pyexe:\n        logging.warn(\"Python executable %s not found in path\", args.pyexe)\n    if args.cmake_options:\n        cmake = CMake(args.cmake_options)\n    else:\n        cmake = CMake()\n    cmake()\n    create_virtualenv_default()\n    provision_virtualenv()\n\ndef main():\n    logging.getLogger().setLevel(logging.INFO)\n    parser = argparse.ArgumentParser(description=\"\"\"Utility for compiling and testing MXNet easily\"\"\")\n    parser.set_defaults(command='use_menu_ui')\n\n    subparsers = parser.add_subparsers(help='sub-command help')\n    build_parser = subparsers.add_parser('build', help='build with the specified flags from file')\n    build_parser.add_argument('cmake_options', nargs='?',\n        help='File containing CMake options in YAML')\n    build_parser.add_argument('-v', '--venv',\n        type=str,\n        default=DEFAULT_PYENV,\n        help='virtualenv dir')\n    build_parser.add_argument('-p', '--pyexe',\n        type=str,\n        default=DEFAULT_PYTHON,\n        help='python executable')\n    build_parser.set_defaults(command='build')\n\n    menu_parser = subparsers.add_parser('menu', help='jump to menu option #')\n    menu_parser.set_defaults(command='use_menu_ui')\n    menu_parser.add_argument('choice', nargs=1)\n\n    args = parser.parse_args()\n    globals()[args.command](args)\n    return 0\n\nif __name__ == '__main__':\n    sys.exit(main())\n"
  },
  {
    "path": "ci/docker/Dockerfile.build.android",
    "content": "# -*- mode: dockerfile -*-\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# Dockerfile to build MXNet for Android\n\n####################################################################################################\n# Shared base for all Android targets\n####################################################################################################\nFROM ubuntu:20.04 AS base\n\nWORKDIR /usr/local\n\nRUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \\\n    build-essential \\\n    ninja-build \\\n    cmake \\\n    ccache \\\n    git \\\n    curl \\\n    unzip \\\n && rm -rf /var/lib/apt/lists/*\n\nRUN curl -o android-ndk-r19c-linux-x86_64.zip -L https://dl.google.com/android/repository/android-ndk-r19c-linux-x86_64.zip && \\\n    unzip android-ndk-r19c-linux-x86_64.zip && \\\n    rm android-ndk-r19c-linux-x86_64.zip\nENV CMAKE_TOOLCHAIN_FILE=/usr/local/android-ndk-r19c/build/cmake/android.toolchain.cmake\n\nARG USER_ID=0\nARG GROUP_ID=0\nCOPY install/ubuntu_adduser.sh /work/\nRUN /work/ubuntu_adduser.sh\n\nCOPY runtime_functions.sh /work/\n\n\n####################################################################################################\n# Specialize base image for ARMv7\n####################################################################################################\nFROM base as armv7\nENV ARCH=armv7l \\\n    HOSTCC=gcc \\\n    HOSTCXX=g++ \\\n    TARGET=ARMV7\n\nRUN git clone --recursive -b v0.3.12 https://github.com/xianyi/OpenBLAS.git && \\\n    cd /usr/local/OpenBLAS && \\\n    export TOOLCHAIN=/usr/local/android-ndk-r19c/toolchains/llvm/prebuilt/linux-x86_64 && \\\n    make NOFORTRAN=1 ARM_SOFTFP_ABI=1 NO_SHARED=1 \\\n        LDFLAGS=\"-L/usr/local/android-ndk-r19c/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/lib/gcc/arm-linux-androideabi/4.9.x -lm\" \\\n        CC=$TOOLCHAIN/bin/armv7a-linux-androideabi16-clang AR=$TOOLCHAIN/bin/arm-linux-androideabi-ar && \\\n    make PREFIX=/usr/local/android-ndk-r19c/toolchains/llvm/prebuilt/linux-x86_64/sysroot/ NO_SHARED=1 install && \\\n    cd /usr/local && \\\n    rm -rf OpenBLAS\n\nWORKDIR /work/build\n\n\n####################################################################################################\n# Specialize base image for ARMv8\n####################################################################################################\nFROM base as armv8\nENV ARCH=aarch64 \\\n    HOSTCC=gcc \\\n    HOSTCXX=g++ \\\n    TARGET=ARMV8\n\nRUN git clone --recursive -b v0.3.12 https://github.com/xianyi/OpenBLAS.git && \\\n    cd /usr/local/OpenBLAS && \\\n    export TOOLCHAIN=/usr/local/android-ndk-r19c/toolchains/llvm/prebuilt/linux-x86_64 && \\\n    make NOFORTRAN=1 NO_SHARED=1 \\\n        LDFLAGS=\"-L/usr/local/android-ndk-r21/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/lib/gcc/aarch64-linux-android/4.9.x -lm\" \\\n        CC=$TOOLCHAIN/bin/aarch64-linux-android21-clang AR=$TOOLCHAIN/bin/aarch64-linux-android-ar && \\\n    make PREFIX=/usr/local/android-ndk-r19c/toolchains/llvm/prebuilt/linux-x86_64/sysroot/ NO_SHARED=1 install && \\\n    cd /usr/local && \\\n    rm -rf OpenBLAS\n\nWORKDIR /work/build\n"
  },
  {
    "path": "ci/docker/Dockerfile.build.arm",
    "content": "# -*- mode: dockerfile -*-\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# Dockerfile to build MXNet for ARM\n\n####################################################################################################\n# Shared base for all ARM targets\n####################################################################################################\nFROM ubuntu:20.04 AS base\n\nWORKDIR /usr/local\n\nRUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \\\n    build-essential \\\n    ninja-build \\\n    cmake \\\n    ccache \\\n    git \\\n    curl \\\n    zip \\\n    python3 \\\n    python3-pip \\\n && rm -rf /var/lib/apt/lists/*\n\nARG USER_ID=0\nARG GROUP_ID=0\nCOPY install/ubuntu_adduser.sh /work/\nRUN /work/ubuntu_adduser.sh\n\nCOPY runtime_functions.sh /work/\n\n\n####################################################################################################\n# Specialize base image for ARMv6\n####################################################################################################\nFROM base as armv6\n\nENV ARCH=armv6l \\\n    HOSTCC=gcc \\\n    HOSTCXX=g++ \\\n    TARGET=ARMV6\n\n# We use a toolchain from toolchains.bootlin.com instead of Debian / Ubunut\n# crossbuild-essential-armel toolchain, as the latter targets ARM architecture\n# versions 4T, 5T, and 6, whereas we only wish to target ARMV6 and like to use\n# ARMV6 specific features. https://wiki.debian.org/ArmEabiPort\nRUN curl -o armv6-eabihf--glibc--stable-2020.02-2.tar.bz2 -L https://toolchains.bootlin.com/downloads/releases/toolchains/armv6-eabihf/tarballs/armv6-eabihf--glibc--stable-2020.02-2.tar.bz2 && \\\n    tar xf armv6-eabihf--glibc--stable-2020.02-2.tar.bz2 && \\\n    rm armv6-eabihf--glibc--stable-2020.02-2.tar.bz2\nENV CMAKE_TOOLCHAIN_FILE=/usr/local/armv6-eabihf--glibc--stable-2020.02-2/share/buildroot/toolchainfile.cmake\n\nRUN git clone --recursive -b v0.3.12 https://github.com/xianyi/OpenBLAS.git && \\\n    cd /usr/local/OpenBLAS && \\\n    make NOFORTRAN=1 NO_SHARED=1 CC=/usr/local/armv6-eabihf--glibc--stable-2020.02-2/bin/arm-linux-gcc && \\\n    make PREFIX=/usr/local/armv6-eabihf--glibc--stable-2020.02-2/arm-buildroot-linux-gnueabihf/sysroot NO_SHARED=1 install && \\\n    cd /usr/local && \\\n    rm -rf OpenBLAS\n\nWORKDIR /work/mxnet\n\n\n####################################################################################################\n# Specialize base image for ARMv7\n####################################################################################################\nFROM base as armv7\n\nENV ARCH=armv7l \\\n    HOSTCC=gcc \\\n    HOSTCXX=g++ \\\n    TARGET=ARMV7\n\nRUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \\\n    crossbuild-essential-armhf \\\n && rm -rf /var/lib/apt/lists/*\n\nCOPY toolchains/arm-linux-gnueabihf-toolchain.cmake /usr/local\nENV CMAKE_TOOLCHAIN_FILE=/usr/local/arm-linux-gnueabihf-toolchain.cmake\n\nRUN git clone --recursive -b v0.3.12 https://github.com/xianyi/OpenBLAS.git && \\\n    cd /usr/local/OpenBLAS && \\\n    make NOFORTRAN=1 NO_SHARED=1 CC=arm-linux-gnueabihf-gcc && \\\n    make PREFIX=/usr/local/arm-linux-gnueabihf NO_SHARED=1 install && \\\n    cd /usr/local && \\\n    rm -rf OpenBLAS\n\nRUN git clone --recursive -b v1.2.11 https://github.com/madler/zlib.git && \\\n    cd /usr/local/zlib && \\\n    CHOST=arm \\\n    CC=arm-linux-gnueabihf-gcc \\\n    AR=arm-linux-gnueabihf-ar \\\n    RANLIB=arm-linux-gnueabihf-ranlib \\\n    ./configure --static --prefix=/usr/local/arm-linux-gnueabihf && \\\n    make -j$(nproc) && \\\n    make install && \\\n    cd /usr/local && \\\n    rm -rf zlib\n\nWORKDIR /work/mxnet\n\n\n####################################################################################################\n# Specialize base image for ARMv8\n####################################################################################################\nFROM base as armv8\n\nENV ARCH=aarch64 \\\n    HOSTCC=gcc \\\n    HOSTCXX=g++ \\\n    TARGET=ARMV8\n\nRUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \\\n    crossbuild-essential-arm64 \\\n && rm -rf /var/lib/apt/lists/*\n\nCOPY toolchains/aarch64-linux-gnu-toolchain.cmake /usr\nENV CMAKE_TOOLCHAIN_FILE=/usr/aarch64-linux-gnu-toolchain.cmake\n\nRUN git clone --recursive -b v0.3.12 https://github.com/xianyi/OpenBLAS.git && \\\n    cd /usr/local/OpenBLAS && \\\n    make NOFORTRAN=1 NO_SHARED=1 CC=aarch64-linux-gnu-gcc && \\\n    make PREFIX=/usr/aarch64-linux-gnu NO_SHARED=1 install && \\\n    cd /usr/local && \\\n    rm -rf OpenBLAS\n\nRUN git clone --recursive -b v1.2.11 https://github.com/madler/zlib.git && \\\n    cd /usr/local/zlib && \\\n    CHOST=arm \\\n    CC=aarch64-linux-gnu-gcc \\\n    AR=aarch64-linux-gnu-ar \\\n    RANLIB=aarch64-linux-gnu-ranlib \\\n    ./configure --static --prefix=/usr/aarch64-linux-gnu && \\\n    make -j$(nproc) && \\\n    make install && \\\n    cd /usr/local && \\\n    rm -rf zlib\n\nWORKDIR /work/mxnet\n"
  },
  {
    "path": "ci/docker/Dockerfile.build.centos7",
    "content": "# -*- mode: dockerfile -*-\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# Dockerfile for CentOS 7 based builds.\n# Via the CentOS 7 Dockerfiles, we ensure MXNet continues to run fine on older systems.\n#\n# See docker-compose.yml for supported BASE_IMAGE ARGs and targets.\n\n####################################################################################################\n# The Dockerfile uses a dynamic BASE_IMAGE (for example centos:7,\n# nvidia/cuda:10.2-cudnn7-devel-centos7 etc).\n# On top of BASE_IMAGE we install all dependencies shared by all MXNet build\n# environments into a \"base\" target. At the end of this file, we specialize\n# \"base\" for specific usecases. The target built by docker can be selected via\n# \"--target\" option or docker-compose.yml\n####################################################################################################\nARG BASE_IMAGE\nFROM $BASE_IMAGE AS base\n\nWORKDIR /work/deps\n\nRUN yum -y check-update || true && \\\n    yum -y install epel-release centos-release-scl && \\\n    yum install -y \\\n        # Utilities\n        wget \\\n        unzip \\\n        patchelf \\\n        pandoc \\\n        # Development tools\n        git \\\n        make \\\n        ninja-build \\\n        automake \\\n        autoconf \\\n        libtool \\\n        protobuf-compiler \\\n        protobuf-devel \\\n        # CentOS Software Collections https://www.softwarecollections.org\n        devtoolset-7 \\\n        devtoolset-8 \\\n        rh-python38 \\\n        rh-maven35 \\\n        # Libraries\n        # Provide clbas headerfiles\n        atlas-devel \\\n        opencv-devel \\\n        openssl-devel \\\n        zeromq-devel \\\n        # Build-dependencies for ccache 3.7.9\n        gperf \\\n        libb2-devel \\\n        libzstd-devel \\\n        # Required by openblas build\n        gcc-gfortran && \\\n    yum clean all\n\n# Build OpenBLAS from source\nRUN mkdir ~/openblas && \\\n    cd ~/openblas && \\\n    OPENBLAS_VERSION=0.3.10 && \\\n    wget \\\n        https://github.com/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.zip \\\n        -O openblas.zip && \\\n    unzip -q openblas.zip -d . && \\\n    cd OpenBLAS-${OPENBLAS_VERSION} && \\\n    CXX=\"g++ -fPIC\" CC=\"gcc -fPIC\" make -j DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 && \\\n    make PREFIX=/usr/local install\n\n# Make Python 3.8 and Maven 3.3 Software Collections available by default during\n# the following build steps in this Dockerfile\nSHELL [ \"/usr/bin/scl\", \"enable\", \"devtoolset-7\", \"rh-python38\", \"rh-maven35\" ]\n\n# Install minimum required cmake version\nRUN cd /usr/local/src && \\\n    wget -nv --no-check-certificate https://cmake.org/files/v3.13/cmake-3.13.5-Linux-x86_64.sh && \\\n    sh cmake-3.13.5-Linux-x86_64.sh --prefix=/usr/local --skip-license && \\\n    rm cmake-3.13.5-Linux-x86_64.sh\n\n# ccache 3.7.9 has fixes for caching nvcc outputs\nRUN cd /usr/local/src && \\\n    git clone --recursive https://github.com/ccache/ccache.git && \\\n    cd ccache && \\\n    git checkout v3.7.9 && \\\n    ./autogen.sh && \\\n    ./configure --disable-man && \\\n    make -j$(nproc) && \\\n    make install && \\\n    cd /usr/local/src && \\\n    rm -rf ccache\n\n# Fix the en_DK.UTF-8 locale to test locale invariance\nRUN localedef -i en_DK -f UTF-8 en_DK.UTF-8\n\n# Python dependencies\nRUN python3 -m pip install --upgrade pip\nCOPY install/requirements /work/\nRUN python3 -m pip install -r /work/requirements\n\nARG USER_ID=0\nCOPY install/docker_filepermissions.sh /work/\nRUN /work/docker_filepermissions.sh\n\nENV PYTHONPATH=./python/\n# Verify that MXNet works correctly when the C locale is set to a locale that uses a comma as the\n# decimal separator. Please see #16134 for an example of a bug caused by incorrect handling of\n# number serialization and deserialization.\nENV LC_NUMERIC=en_DK.UTF-8\nWORKDIR /work/mxnet\n\nCOPY runtime_functions.sh /work/\n"
  },
  {
    "path": "ci/docker/Dockerfile.build.jetson",
    "content": "# -*- mode: dockerfile -*-\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# Dockerfile to build libmxnet.so, and a python wheel for the Jetson TX1/TX2\n# This script assumes /work/mxnet exists and contains the mxnet code you wish to compile and\n# that /work/build exists and is the target for your output.\n\nFROM nvidia/cuda:10.2-cudnn7-devel-ubuntu18.04\n\nENV ARCH=aarch64 \\\n    HOSTCC=gcc \\\n    TARGET=ARMV8\n\nWORKDIR /usr/local\n\nRUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \\\n    build-essential \\\n    ninja-build \\\n    git \\\n    wget \\\n    zip \\\n    unzip \\\n    python3 \\\n    python3-pip \\\n    awscli \\\n    crossbuild-essential-arm64 \\\n && rm -rf /var/lib/apt/lists/*\n\n# cmake on Ubuntu 18.04 is too old\nRUN python3 -m pip install cmake\n\n# ccache on Ubuntu 18.04 is too old to support Cuda correctly\nCOPY install/deb_ubuntu_ccache.sh /work/\nRUN /work/deb_ubuntu_ccache.sh\n\nCOPY toolchains/aarch64-linux-gnu-toolchain.cmake /usr\nENV CMAKE_TOOLCHAIN_FILE=/usr/aarch64-linux-gnu-toolchain.cmake\n\nRUN git clone --recursive -b v0.3.12 https://github.com/xianyi/OpenBLAS.git && \\\n    cd /usr/local/OpenBLAS && \\\n    make NOFORTRAN=1 CC=aarch64-linux-gnu-gcc && \\\n    make PREFIX=/usr/aarch64-linux-gnu install && \\\n    cd /usr/local && \\\n    rm -rf OpenBLAS\n\n# Install aarch64 cross depedencies based on Jetpack 4.4\n# Dependencies require cuda-toolkit-10.2 which isn't installed in nvidia docker container\n# It contains cuda-compat instead. However deb files currently depend on cuda-toolkit alone.\n# Hence force dpkg configure\nRUN wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-cross-aarch64-10-2_10.2.89-1_all.deb && \\\n    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-cross-aarch64_10.2.89-1_all.deb && \\\n    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-cudart-cross-aarch64-10-2_10.2.89-1_all.deb && \\\n    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-cufft-cross-aarch64-10-2_10.2.89-1_all.deb && \\\n    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-cupti-cross-aarch64-10-2_10.2.89-1_all.deb && \\\n    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-curand-cross-aarch64-10-2_10.2.89-1_all.deb && \\\n    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-cusolver-cross-aarch64-10-2_10.2.89-1_all.deb && \\\n    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-cusparse-cross-aarch64-10-2_10.2.89-1_all.deb && \\\n    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-driver-cross-aarch64-10-2_10.2.89-1_all.deb && \\\n    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-misc-headers-cross-aarch64-10-2_10.2.89-1_all.deb && \\\n    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-npp-cross-aarch64-10-2_10.2.89-1_all.deb && \\\n    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-nsight-compute-addon-l4t-10-2_10.2.89-1_all.deb && \\\n    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-nvgraph-cross-aarch64-10-2_10.2.89-1_all.deb && \\\n    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-nvml-cross-aarch64-10-2_10.2.89-1_all.deb && \\\n    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-nvrtc-cross-aarch64-10-2_10.2.89-1_all.deb && \\\n    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cublas/libcublas-cross-aarch64_10.2.2.89-1_all.deb && \\\n    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/n/nsight-compute/nsight-compute-addon-l4t-2019.5.0_2019.5.0.14-1_all.deb && \\\n    dpkg -i --force-all  *.deb && \\\n    rm *.deb && \\\n    apt-get update && \\\n    apt-get install -y -f && \\\n    apt-get install -y cuda-cross-aarch64 cuda-cross-aarch64-10-2 && \\\n    rm -rf /var/lib/apt/lists/*\n\n# nvidia jetpack 4.4 installs libcublas.so at /usr/lib/aarch64-linux-gnu\n# while previously it used to store it at /usr/local/cuda/targets/aarch64-linux/lib/stubs\nRUN ln -s /usr/lib/aarch64-linux-gnu/libcublas.so /usr/local/cuda/targets/aarch64-linux/lib/stubs/libcublas.so\n\nARG USER_ID=0\nARG GROUP_ID=0\nCOPY install/ubuntu_adduser.sh /work/\nRUN /work/ubuntu_adduser.sh\n\nCOPY runtime_functions.sh /work/\nWORKDIR /work/mxnet\n"
  },
  {
    "path": "ci/docker/Dockerfile.build.ubuntu",
    "content": "# -*- mode: dockerfile -*-\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# Dockerfile for Ubuntu based builds.\n#\n# See docker-compose.yml for supported BASE_IMAGE ARGs and targets.\n\n####################################################################################################\n# The Dockerfile uses a dynamic BASE_IMAGE (for example ubuntu:20.04\n# nvidia/cuda:11.1-cudnn8-devel-ubuntu20.04 etc).\n# On top of BASE_IMAGE we install all dependencies shared by all MXNet build\n# environments into a \"base\" target. At the end of this file, we can specialize\n# \"base\" for specific usecases. The target built by docker can be selected via\n# \"--target\" option or docker-compose.yml\n####################################################################################################\nARG BASE_IMAGE\nFROM $BASE_IMAGE AS base\n\nWORKDIR /work/deps\n\nSHELL [\"/bin/bash\", \"-c\"]\nRUN export DEBIAN_FRONTEND=noninteractive && \\\n    export OS_RELEASE=\"$(cat /etc/os-release)\" && \\\n    apt-get clean && \\\n    apt-get update && \\\n    apt-get install -y wget software-properties-common && \\\n    wget -qO - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB -O - | apt-key add -; \\\n    add-apt-repository \"deb https://apt.repos.intel.com/oneapi all main\"; \\\n    INTEL_MKL=\"-2022.0.2\"; \\\n    apt-get update && \\\n    apt-get install -y \\\n        ## Utilities\n        curl \\\n        zip \\ \n        unzip \\\n        pandoc \\\n        ## Development tools\n        build-essential \\\n        ninja-build \\\n        git \\\n        protobuf-compiler \\\n        libprotobuf-dev \\\n        default-jdk \\\n        clang-6.0 \\\n        clang-format \\\n        python-yaml \\\n        clang-10 \\\n        clang-tidy-10 \\\n        g++ \\\n        g++-7 \\\n        g++-8 \\\n        intel-oneapi-mkl${INTEL_MKL} \\\n        intel-oneapi-mkl-devel${INTEL_MKL} \\\n        libomp-dev \\\n        ## Dependencies\n        libgomp1 \\\n        libturbojpeg0-dev \\\n        libcurl4-openssl-dev \\\n        libatlas-base-dev \\\n        libzmq3-dev \\\n        libopencv-dev \\\n        libxml2-dev \\\n        # BytePS\n        numactl \\\n        libnuma-dev \\\n        ## Frontend languages\n        # Python\n        python3 \\\n        python3-pip \\\n        ## Documentation\n        doxygen \\\n        pandoc \\\n        ## Build-dependencies for ccache 3.7.9\n        autoconf \\\n        gperf \\\n        libb2-dev \\\n        libzstd-dev \\\n        gfortran && \\\n    rm -rf /var/lib/apt/lists/* && \\\n    add-apt-repository -r \"deb https://apt.repos.intel.com/oneapi all main\"\n\n# Build OpenBLAS from source\nRUN export LIBRARY_PATH=$LIBRARY_PATH:/usr/lib/gcc/x86_64-linux-gnu/7/ && \\\n    mkdir ~/openblas && \\\n    cd ~/openblas && \\\n    OPENBLAS_VERSION=0.3.10 && \\\n    wget \\\n        https://github.com/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.zip \\\n        -O openblas.zip && \\\n    unzip -q openblas.zip -d . && \\\n    cd OpenBLAS-${OPENBLAS_VERSION} && \\\n    CXX=\"clang++-6.0 -fPIC\" CC=\"clang-6.0 -fPIC\" make -j DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 \\\n        USE_OPENMP=0 INTERFACE64=1 BINARY=64 && \\\n    make PREFIX=/usr/local/openblas-clang install && \\\n    cd .. && \\\n    rm -rf OpenBLAS-${OPENBLAS_VERSION} && \\\n    unzip -q openblas.zip -d . && \\\n    cd OpenBLAS-${OPENBLAS_VERSION} && \\\n    CXX=\"g++ -fPIC\" CC=\"gcc -fPIC\" make -j DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 \\\n        USE_OPENMP=1 INTERFACE64=1 BINARY=64 && \\\n    make PREFIX=/usr/local install\n\n# ccache 3.7.9 has fixes for caching nvcc outputs\nRUN cd /usr/local/src && \\\n    git clone --recursive https://github.com/ccache/ccache.git && \\\n    cd ccache && \\\n    git checkout v3.7.9 && \\\n    ./autogen.sh && \\\n    ./configure --disable-man && \\\n    make -j$(nproc) && \\\n    make install && \\\n    cd /usr/local/src && \\\n    rm -rf ccache\n\n# RAT License Checker tool\nRUN cd /usr/local/src && \\\n    wget https://archive.apache.org/dist/creadur/apache-rat-0.13/apache-rat-0.13-bin.tar.gz && \\\n    tar xf apache-rat-0.13-bin.tar.gz\n\n# Python & cmake\nCOPY install/requirements /work/\nRUN python3 -m pip install --upgrade pip && \\\n    python3 -m pip install cmake==3.16.6 && \\\n    python3 -m pip install -r /work/requirements\n\nARG USER_ID=0\nCOPY install/docker_filepermissions.sh /work/\nRUN /work/docker_filepermissions.sh\n\nENV PYTHONPATH=./python/\nWORKDIR /work/mxnet\n\nCOPY runtime_functions.sh /work/\n\n####################################################################################################\n# Specialize base image to install more gpu specific dependencies.\n# The target built by docker can be selected via \"--target\" option or docker-compose.yml\n####################################################################################################\nFROM base as gpu\n\n# Install TensorRT\n# Use bash as it has better support for string comparisons in if clauses\nSHELL [\"/bin/bash\", \"-c\"]\n# We need to redeclare ARG due to\n# https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact\nARG BASE_IMAGE\nRUN apt-get update && \\\n        apt-get install -y --allow-change-held-packages libcudnn8 libcudnn8-dev && \\\n        rm -rf /var/lib/apt/lists/*\n\nARG TRT_VERSION\nRUN if [ ! -z \"${TRT_VERSION}\" ]; then \\\n        apt-get update && \\\n        TRT_MAJOR_VERSION=$(echo $TRT_VERSION | cut -d. -f 1) && \\\n        apt-get install -y libnvinfer${TRT_MAJOR_VERSION}=${TRT_VERSION} \\\n                           libnvinfer-dev=${TRT_VERSION} \\\n                           libnvinfer-plugin${TRT_MAJOR_VERSION}=${TRT_VERSION} \\\n                           libnvinfer-plugin-dev=${TRT_VERSION}; \\\n        rm -rf /var/lib/apt/lists/*; \\\n    fi\n\n\n"
  },
  {
    "path": "ci/docker/Dockerfile.build.ubuntu_cpu_jekyll",
    "content": "# -*- mode: dockerfile -*-\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU\n\nFROM ruby:2.6.5-buster\n\nWORKDIR /work/deps\n\nENV BUNDLE_HOME=/work/deps/bundle\nENV BUNDLE_APP_CONFIG=/work/deps/bundle\nENV BUNDLE_BIN=/work/deps/bundle/bin\nENV GEM_BIN=/work/deps/gem/bin\nENV GEM_HOME=/work/deps/gem\n\nRUN echo \"gem: --no-ri --no-rdoc\" > ~/.gemrc && \\\n    yes | gem update --system && \\\n    yes | gem install --force bundler && \\\n    gem install jekyll\n\nENV PATH=$BUNDLE_BIN:$GEM_BIN:$PATH\n\nCOPY runtime_functions.sh /work/\n\nARG USER_ID=0\nARG GROUP_ID=0\nCOPY install/ubuntu_adduser.sh /work/\nRUN /work/ubuntu_adduser.sh\n\nWORKDIR /work/mxnet\n"
  },
  {
    "path": "ci/docker/Dockerfile.publish.test.centos7",
    "content": "# -*- mode: dockerfile -*-\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# Dockerfile for CentOS 7 based publish artifacts tests.\n#\n# See docker-compose.yml for supported BASE_IMAGE ARGs and targets.\n\n####################################################################################################\n# The Dockerfile uses a dynamic BASE_IMAGE (for example centos:7,\n# nvidia/cuda:10.2-cudnn7-devel-centos7 etc).\n# On top of BASE_IMAGE we install all dependencies required for the tests of\n# binary artifacts.\n####################################################################################################\nARG BASE_IMAGE\nFROM $BASE_IMAGE\n\nWORKDIR /work/deps\n\n# Install runtime dependencies for publish tests\n# - make is used to run tests ci/publish/scala/test.sh\n# - unzip is used to run org.apache.mxnetexamples.neuralstyle.NeuralStyleSuite\n# - gcc to provide libgomp.so.1 (may want to drop this in the future and ship\n#   inside jar)\n# - rh-maven35 to run ci/publish/scala/test.sh\nRUN yum -y check-update || true && \\\n    yum -y install epel-release centos-release-scl && \\\n    yum -y install \\\n        make \\\n        gcc \\\n        unzip \\\n        rh-maven35 && \\\n    yum clean all\n\nARG USER_ID=0\nCOPY install/docker_filepermissions.sh /work/\nRUN /work/docker_filepermissions.sh\n\nENV PYTHONPATH=./python/\nWORKDIR /work/mxnet\n\nCOPY runtime_functions.sh /work/\n"
  },
  {
    "path": "ci/docker/Dockerfile.test.arm",
    "content": "# -*- mode: dockerfile -*-\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# Dockerfile to test MXNet on Ubuntu 20.04 ARM\n\nARG BASE_IMAGE\nFROM $BASE_IMAGE\n\nWORKDIR /usr/local\n\nRUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \\\n    python3 \\\n    python3-pip \\\n    python3-numpy \\\n    python3-scipy \\\n    python3-requests \\\n && rm -rf /var/lib/apt/lists/*\n\n\n# Python dependencies\nRUN pip3 install --no-cache-dir --upgrade pip && \\\n    pip3 install --no-cache-dir \\\n    pytest==6.1.2 \\\n    pytest-env==0.6.2 \\\n    pytest-cov==2.10.1 \\\n    pytest-xdist==2.1.0 \\\n    pytest-timeout==1.4.2 \\\n    mock==2.0.0\n\nARG USER_ID=0\nARG GROUP_ID=0\nCOPY install/ubuntu_adduser.sh /work/\nRUN /work/ubuntu_adduser.sh\n\nCOPY runtime_functions.sh /work/\nWORKDIR /work/mxnet\n"
  },
  {
    "path": "ci/docker/docker-compose.yml",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# We use the cache_from feature introduced in file form version 3.4 (released 2017-11-01)\nversion: \"3.4\"\n\n# For simplicity, only the centos7_cpu is commented. But the comments apply to\n# all other services as well.\nservices:\n  ###################################################################################################\n  # Dockerfile.build.centos7 based images used for building on CentOS7. On\n  # CentOS7, we respectively test the oldest supported toolchain and dependency\n  # versions\n  ###################################################################################################\n  centos7_cpu:\n    # The resulting image will be named build.centos7_cpu:latest and will be\n    # pushed to the dockerhub user specified in the environment variable\n    # ${DOCKER_CACHE_REGISTRY} (typicall \"mxnetci\") under this name\n    image: ${DOCKER_CACHE_REGISTRY}/build.centos7_cpu:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.build.centos7\n      # Use \"base\" target declared in Dockerfile.build.centos7 as \"build.centos7_cpu:latest\"  \n      target: base\n      args:\n        # BASE_IMAGE is used to dynamically specify the FROM image in Dockerfile.build.centos7\n        BASE_IMAGE: centos:7\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/build.centos7_cpu:latest\n  centos7_gpu_cu101:\n    image: ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu101:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.build.centos7\n      target: base\n      args:\n        BASE_IMAGE: nvidia/cuda:10.1-cudnn8-devel-centos7\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu101:latest\n  centos7_gpu_cu102:\n    image: ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu102:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.build.centos7\n      target: base\n      args:\n        BASE_IMAGE: nvidia/cuda:10.2-cudnn8-devel-centos7\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu102:latest\n  centos7_gpu_cu110:\n    image: ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu110:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.build.centos7\n      target: base\n      args:\n        BASE_IMAGE: nvidia/cuda:11.0.3-cudnn8-devel-centos7\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu110:latest\n  centos7_gpu_cu112:\n    image: ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu112:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.build.centos7\n      target: base\n      args:\n        BASE_IMAGE: nvidia/cuda:11.2.0-cudnn8-devel-centos7\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu112:latest\n  ###################################################################################################\n  # Dockerfile.build.ubuntu based images. On Ubuntu we test more recent\n  # toolchain and dependency versions compared to CentOS7. We attempt to update\n  # the Ubuntu base image every 6 months, following the Ubuntu release cycle,\n  # and testing the dependencies in their version provided by the respective\n  # Ubuntu release.\n  ###################################################################################################\n  ubuntu_cpu:\n    image: ${DOCKER_CACHE_REGISTRY}/build.ubuntu_cpu:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.build.ubuntu\n      target: base\n      args:\n        BASE_IMAGE: ubuntu:20.04\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/build.ubuntu_cpu:latest\n  ubuntu_tensorrt_cu114:\n    image: ${DOCKER_CACHE_REGISTRY}/build.ubuntu_tensorrt_cu114:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.build.ubuntu\n      target: gpu\n      args:\n        BASE_IMAGE: nvidia/cuda:11.4.0-cudnn8-devel-ubuntu20.04\n        TRT_VERSION: 8.2.4-1+cuda11.4\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/build.ubuntu_tensorrt_cu114:latest\n  ubuntu_gpu_cu111:\n    image: ${DOCKER_CACHE_REGISTRY}/build.ubuntu_gpu_cu111:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.build.ubuntu\n      target: gpu\n      args:\n        BASE_IMAGE: nvidia/cuda:11.1.1-cudnn8-devel-ubuntu20.04\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/build.ubuntu_gpu_cu111:latest\n  ###################################################################################################\n  # Dockerfile.build.android based images used for testing cross-compilation for plain ARM\n  ###################################################################################################\n  armv6:\n    image: ${DOCKER_CACHE_REGISTRY}/build.armv6:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.build.arm\n      target: armv6\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/build.armv6:latest\n  armv7:\n    image: ${DOCKER_CACHE_REGISTRY}/build.armv7:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.build.arm\n      target: armv7\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/build.armv7:latest\n  armv8:\n    image: ${DOCKER_CACHE_REGISTRY}/build.armv8:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.build.arm\n      target: armv8\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/build.armv8:latest\n  ###################################################################################################\n  # Dockerfile.test.arm based images for testing ARM artefacts via QEMU\n  ###################################################################################################\n  test.armv7:\n    image: ${DOCKER_CACHE_REGISTRY}/test.armv7:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.test.arm\n      args:\n        BASE_IMAGE: arm32v7/ubuntu:20.04\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/test.armv7:latest\n  test.armv8:\n    image: ${DOCKER_CACHE_REGISTRY}/test.armv8:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.test.arm\n      args:\n        BASE_IMAGE: arm64v8/ubuntu:20.04\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/test.armv8:latest\n  ###################################################################################################\n  # Dockerfile.build.android based images used for testing cross-compilation for Android\n  ###################################################################################################\n  android_armv7:\n    image: ${DOCKER_CACHE_REGISTRY}/build.android_armv7:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.build.android\n      target: armv7\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/build.android_armv7:latest\n  android_armv8:\n    image: ${DOCKER_CACHE_REGISTRY}/build.android_armv8:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.build.android\n      target: armv8\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/build.android_armv8:latest\n  ###################################################################################################\n  # Dockerfile.publish.test based images used for testing binary artifacts on minimal systems.\n  ###################################################################################################\n  publish.test.centos7_cpu:\n    image: ${DOCKER_CACHE_REGISTRY}/publish.test.centos7_cpu:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.publish.test.centos7\n      args:\n        BASE_IMAGE: centos:7\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/publish.test.centos7_cpu:latest\n  publish.test.centos7_gpu:\n    image: ${DOCKER_CACHE_REGISTRY}/publish.test.centos7_gpu:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.publish.test.centos7\n      args:\n        BASE_IMAGE: nvidia/cuda:9.2-cudnn7-devel-centos7\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/publish.test.centos7_gpu:latest\n  ###################################################################################################\n  # Miscellaneous containers\n  ###################################################################################################\n  jetson:\n    image: ${DOCKER_CACHE_REGISTRY}/build.jetson:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.build.jetson\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/build.jetson:latest\n  ubuntu_cpu_jekyll:\n    image: ${DOCKER_CACHE_REGISTRY}/build.ubuntu_cpu_jekyll:latest\n    build:\n      context: .\n      dockerfile: Dockerfile.build.ubuntu_cpu_jekyll\n      cache_from:\n        - ${DOCKER_CACHE_REGISTRY}/build.ubuntu_cpu_jekyll:latest\n"
  },
  {
    "path": "ci/docker/install/deb_ubuntu_ccache.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Script to build ccache for debian and ubuntu based images.\n\nset -ex\n\npushd .\n\napt update\napt install -y \\\n    autoconf \\\n    gperf \\\n    xsltproc\n\nmkdir -p /work/deps\ncd /work/deps\n\ngit clone --recursive https://github.com/ccache/ccache.git\ncd ccache\ngit checkout v3.7.9\n\n./autogen.sh\n./configure --disable-man\nmake -j$(nproc)\nmake install\n\nrm -rf /work/deps/ccache\n\npopd\n"
  },
  {
    "path": "ci/docker/install/docker_filepermissions.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Add user in order to make sure the assumed user the container is running under\n# actually exists inside the container to avoid problems like missing home dir\n\nset -ex\n\n# Add user in order to make sure the assumed user the container is running under\n# actually exists inside the container to avoid problems like missing home dir\nif [[ \"$USER_ID\" -gt 0 ]]; then\n    # -no-log-init required due to https://github.com/moby/moby/issues/5419\n    useradd -m --no-log-init --uid $USER_ID --system jenkins_slave\n    # By default, docker creates all WORK_DIRs with root owner\n    mkdir /work/mxnet\n    mkdir /work/build\n    chown -R jenkins_slave /work/\nfi\n"
  },
  {
    "path": "ci/docker/install/requirements",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# build and install are separated so changes to build don't invalidate\n# the whole docker cache for the image\n\n# Required dependencies\nnumpy>=1.17,<1.20.0\nrequests>=2.20.0,<3\ngraphviz<0.9.0,>=0.8.1\ncontextvars;python_version<\"3.7\"\n\n# Optional dependencies\nonnx==1.8.0\nonnxruntime==1.7.0\nprotobuf==3.14.0\nscipy==1.4.1\ntabulate==0.7.5\nCython==0.29.24\n\n# Development dependencies\ncpplint==1.3.0\npylint<3,>=2.8.3 # pylint and astroid need to be aligned\nastroid<2.9,>=2.8.0  # pylint and astroid need to be aligned\npytest==6.1.2\npytest-env==0.6.2\npytest-cov==2.10.1\npytest-xdist==2.1.0\npytest-timeout==1.4.2\npytest-rerunfailures==10.2\nflaky==3.7.0\nsetuptools==49.6.0  # https://github.com/pypa/setuptools/issues/2352\nwheel\npackaging\n\n# TVM dependencies\ndecorator==4.4.0\n\n# Used in examples\nboto3==1.26.48\nh5py==2.10.0\n\n# Array API Standardization requirements\nhypothesis==6.14.0\n\n# Static code checker for CMake files\ncmakelint==1.4.1\n\n# Prospector - Python Static Analysis\nprospector==1.5.1\n\n# pyflakes - passive checker of Python programs\npyflakes<2.4.0,>=2.2.0\n"
  },
  {
    "path": "ci/docker/install/ubuntu_adduser.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Add user in order to make sure the assumed user the container is running under\n# actually exists inside the container to avoid problems like missing home dir\n\n\nset -ex\n\n# $USER_ID is coming from build.py:build_docker passed as --build-arg\nif [[ \"$USER_ID\" -gt 0 ]]\nthen\n    # -no-log-init required due to https://github.com/moby/moby/issues/5419\n    if [[ -n \"$GROUP_ID\" ]] && [[ \"$GROUP_ID\" -gt 0 ]]\n    then\n        groupadd --gid $GROUP_ID --system jenkins_slave\n        useradd -m --no-log-init --uid $USER_ID --gid $GROUP_ID --system jenkins_slave\n    else\n        useradd -m --no-log-init --uid $USER_ID --system jenkins_slave\n    fi\n    usermod -aG sudo jenkins_slave\n\n    # By default, docker creates all WORK_DIRs with root owner\n    mkdir /work/mxnet\n    mkdir /work/build\n    chown -R jenkins_slave /work/\n\n    # Later on, we have to override the links because underlying build systems ignore our compiler settings. Thus,\n    # we have to give the process the proper permission to these files. This is hacky, but unfortunately \n    # there's no better way to do this without patching all our submodules.\n    chown -R jenkins_slave /usr/local/bin\nfi\n"
  },
  {
    "path": "ci/docker/runtime_functions.sh",
    "content": "#!/bin/bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# build and install are separated so changes to build don't invalidate\n# the whole docker cache for the image\n\nset -ex\n\n# compute capabilities for CI instances supported by CUDA 10.x (i.e. p3, g4)\nCI_CMAKE_CUDA10_ARCH=\"5.2 7.5\"\n\n# compute capabilities for CI instances supported by CUDA >= 11.1 (i.e. p3, g4, g5)\nCI_CMAKE_CUDA_ARCH=\"5.2 7.5 8.6\"\n\n# On newer nvidia cuda containers, these environment variables\n#  are prefixed with NV_, so provide compatibility\nif [ ! -z \"$NV_CUDNN_VERSION\" ]; then\n    if [ -z \"$CUDNN_VERSION\" ]; then\n        export CUDNN_VERSION=$NV_CUDNN_VERSION\n    fi\nfi\n\nclean_repo() {\n    set -ex\n    git clean -xfd\n    git submodule foreach --recursive git clean -xfd\n    git reset --hard\n    git submodule foreach --recursive git reset --hard\n    git submodule update --init --recursive\n}\n\nscala_prepare() {\n    # Clean up maven logs\n    export MAVEN_OPTS=\"-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn\"\n}\n\ncheck_cython() {\n    set -ex\n    local is_cython_used=$(python3 <<EOF\nimport sys\nimport mxnet as mx\ncython_ndarraybase = 'mxnet._cy3.ndarray'\nprint(mx.nd._internal.NDArrayBase.__module__ == cython_ndarraybase)\nEOF\n)\n\n    if [ \"${is_cython_used}\" != \"True\" ]; then\n        echo \"ERROR: cython is not used.\"\n        return 1\n    else\n        echo \"NOTE: cython is used.\"\n        return 0\n    fi\n}\n\nbuild_wheel() {\n\n    set -ex\n    pushd .\n\n    PYTHON_DIR=${1:-/work/mxnet/python}\n    BUILD_DIR=${2:-/work/build}\n\n    # build\n\n    export MXNET_LIBRARY_PATH=${BUILD_DIR}/libmxnet.so\n\n    cd ${PYTHON_DIR}\n    python3 setup.py bdist_wheel\n\n    # repackage\n\n    # Fix pathing issues in the wheel.  We need to move libmxnet.so from the data folder to the\n    # mxnet folder, then repackage the wheel.\n    WHEEL=`readlink -f dist/*.whl`\n    TMPDIR=`mktemp -d`\n    unzip -d ${TMPDIR} ${WHEEL}\n    rm ${WHEEL}\n    cd ${TMPDIR}\n    mv *.data/data/mxnet/libmxnet.so mxnet\n    zip -r ${WHEEL} .\n    cp ${WHEEL} ${BUILD_DIR}\n    rm -rf ${TMPDIR}\n\n    popd\n}\n\ngather_licenses() {\n    mkdir -p licenses\n\n    cp tools/dependencies/LICENSE.binary.dependencies licenses/\n    cp NOTICE licenses/\n    cp LICENSE licenses/\n}\n\n# Compiles the dynamic mxnet library\n# Parameters:\n# $1 -> mxnet_variant: the mxnet variant to build, e.g. cpu, native, cu101, cu102, etc.\nbuild_dynamic_libmxnet() {\n    set -ex\n\n    local mxnet_variant=${1:?\"This function requires a mxnet variant as the first argument\"}\n\n    # relevant licenses will be placed in the licenses directory\n    gather_licenses\n\n    cd /work/build\n    source /opt/rh/devtoolset-8/enable\n    # Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.\n    export CXXFLAGS=\"-fabi-version=11 -fabi-compat-version=7\"\n    if [[ ${mxnet_variant} = \"cpu\" ]]; then\n        cmake -DUSE_BLAS=Open \\\n            -DUSE_ONEDNN=ON \\\n            -DUSE_CUDA=OFF \\\n            -G Ninja /work/mxnet\n    elif [[ ${mxnet_variant} = \"native\" ]]; then\n        cmake -DUSE_BLAS=Open \\\n            -DUSE_ONEDNN=OFF \\\n            -DUSE_CUDA=OFF \\\n            -G Ninja /work/mxnet\n    elif [[ ${mxnet_variant} =~ cu[0-9]+$ ]]; then\n        cmake -DUSE_BLAS=Open \\\n            -DUSE_ONEDNN=ON \\\n            -DUSE_DIST_KVSTORE=ON \\\n            -DUSE_CUDA=ON \\\n            -G Ninja /work/mxnet\n    else\n        echo \"Error: Unrecognized mxnet variant '${mxnet_variant}'\"\n        exit 1\n    fi\n    ninja\n}\n\nbuild_jetson() {\n    set -ex\n    cd /work/build\n    cmake \\\n        -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \\\n        -DUSE_CUDA=ON \\\n        -DMXNET_CUDA_ARCH=\"5.2\" \\\n        -DUSE_OPENCV=OFF \\\n        -DUSE_OPENMP=ON \\\n        -DUSE_LAPACK=OFF \\\n        -DUSE_BLAS=Open \\\n        -DCMAKE_BUILD_TYPE=Release \\\n        -G Ninja /work/mxnet\n    ninja\n    build_wheel\n}\n\n#\n# ARM builds\n#\n\nbuild_armv6() {\n    set -ex\n    cd /work/build\n\n    # We do not need OpenMP, since most armv6 systems have only 1 core\n\n    cmake \\\n        -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \\\n        -DUSE_CUDA=OFF \\\n        -DUSE_OPENCV=OFF \\\n        -DUSE_OPENMP=OFF \\\n        -DCMAKE_BUILD_TYPE=Release \\\n        -DUSE_LAPACK=OFF \\\n        -DUSE_BLAS=Open \\\n        -DBUILD_CPP_EXAMPLES=OFF \\\n        -G Ninja /work/mxnet\n\n    ninja\n    build_wheel\n}\n\nbuild_armv7() {\n    set -ex\n    cd /work/build\n\n    cmake \\\n        -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \\\n        -DUSE_CUDA=OFF \\\n        -DUSE_OPENCV=OFF \\\n        -DUSE_OPENMP=ON \\\n        -DCMAKE_BUILD_TYPE=Release \\\n        -DUSE_LAPACK=OFF \\\n        -DUSE_BLAS=Open \\\n        -DBUILD_CPP_EXAMPLES=OFF \\\n        -G Ninja /work/mxnet\n\n    ninja\n    build_wheel\n}\n\nbuild_armv8() {\n    cd /work/build\n    cmake \\\n        -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \\\n        -DUSE_CUDA=OFF \\\n        -DUSE_OPENCV=OFF \\\n        -DUSE_OPENMP=ON \\\n        -DUSE_LAPACK=OFF \\\n        -DUSE_BLAS=Open \\\n        -DCMAKE_BUILD_TYPE=Release \\\n        -G Ninja /work/mxnet\n    ninja\n    build_wheel\n}\n\n\n#\n# ANDROID builds\n#\n\nbuild_android_armv7() {\n    set -ex\n    cd /work/build\n    # ANDROID_ABI and ANDROID_STL are options of the CMAKE_TOOLCHAIN_FILE\n    # provided by Android NDK\n    cmake \\\n        -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \\\n        -DANDROID_ABI=\"armeabi-v7a\" \\\n        -DANDROID_STL=\"c++_shared\" \\\n        -DUSE_CUDA=OFF \\\n        -DUSE_LAPACK=OFF \\\n        -DUSE_BLAS=Open \\\n        -DUSE_OPENCV=OFF \\\n        -DUSE_OPENMP=OFF \\\n        -G Ninja /work/mxnet\n    ninja\n}\n\nbuild_android_armv8() {\n    set -ex\n    cd /work/build\n    # ANDROID_ABI and ANDROID_STL are options of the CMAKE_TOOLCHAIN_FILE\n    # provided by Android NDK\n    cmake \\\n        -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \\\n        -DANDROID_ABI=\"arm64-v8a\" \\\n        -DANDROID_STL=\"c++_shared\" \\\n        -DUSE_CUDA=OFF \\\n        -DUSE_LAPACK=OFF \\\n        -DUSE_BLAS=Open \\\n        -DUSE_OPENCV=OFF \\\n        -DUSE_OPENMP=OFF \\\n        -DUSE_SIGNAL_HANDLER=ON \\\n        -G Ninja /work/mxnet\n    ninja\n}\n\nbuild_centos7_cpu() {\n    set -ex\n    cd /work/build\n    source /opt/rh/devtoolset-7/enable\n    # Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.\n    export CXXFLAGS=\"-fabi-version=11 -fabi-compat-version=7\"\n    cmake \\\n        -DCMAKE_BUILD_TYPE=\"RelWithDebInfo\" \\\n        -DUSE_ONEDNN=OFF \\\n        -DUSE_DIST_KVSTORE=ON \\\n        -DUSE_CUDA=OFF \\\n        -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \\\n        -DUSE_INT64_TENSOR_SIZE=OFF \\\n        -DUSE_BLAS=Open \\\n        -G Ninja /work/mxnet\n    ninja\n}\n\nbuild_centos7_onednn() {\n    set -ex\n    cd /work/build\n    source /opt/rh/devtoolset-7/enable\n    # Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.\n    export CXXFLAGS=\"-fabi-version=11 -fabi-compat-version=7\"\n    cmake -DUSE_BLAS=Open \\\n        -DUSE_ONEDNN=ON \\\n        -DUSE_CUDA=OFF \\\n        -DUSE_INT64_TENSOR_SIZE=OFF \\\n        -G Ninja /work/mxnet\n    ninja\n}\n\nbuild_centos7_gpu() {\n    set -ex\n    cd /work/build\n    source /opt/rh/devtoolset-7/enable\n    # Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.\n    export CXXFLAGS=\"-fabi-version=11 -fabi-compat-version=7\"\n    cmake \\\n        -DCMAKE_BUILD_TYPE=\"RelWithDebInfo\" \\\n        -DUSE_BLAS=Open \\\n        -DUSE_ONEDNN=ON \\\n        -DUSE_CUDA=ON \\\n        -DMXNET_CUDA_ARCH=\"$CI_CMAKE_CUDA10_ARCH\" \\\n        -DUSE_DIST_KVSTORE=ON \\\n        -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \\\n        -DUSE_INT64_TENSOR_SIZE=OFF \\\n        -G Ninja /work/mxnet\n    ninja\n}\n\nbuild_ubuntu_cpu() {\n    build_ubuntu_cpu_openblas\n}\n\nbuild_ubuntu_cpu_openblas() {\n    set -ex\n    cd /work/build\n    CXXFLAGS=\"-Wno-error=strict-overflow\" CC=gcc-7 CXX=g++-7 cmake \\\n        -DCMAKE_BUILD_TYPE=\"RelWithDebInfo\" \\\n        -DENABLE_TESTCOVERAGE=ON \\\n        -DUSE_TVM_OP=ON \\\n        -DUSE_BLAS=Open \\\n        -DUSE_ONEDNN=OFF \\\n        -DUSE_CUDA=OFF \\\n        -DUSE_DIST_KVSTORE=ON \\\n        -DBUILD_CYTHON_MODULES=ON \\\n        -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \\\n        -G Ninja /work/mxnet\n    ninja -j$(($(nproc)/2))\n}\n\nbuild_ubuntu_cpu_mkl() {\n    set -ex\n    cd /work/build\n    CC=gcc-7 CXX=g++-7 cmake \\\n        -DCMAKE_BUILD_TYPE=\"RelWithDebInfo\" \\\n        -DENABLE_TESTCOVERAGE=OFF \\\n        -DUSE_ONEDNN=OFF \\\n        -DUSE_CUDA=OFF \\\n        -DUSE_TVM_OP=ON \\\n        -DUSE_MKL_LAYERNORM=ON \\\n        -DUSE_BLAS=MKL \\\n        -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \\\n        -GNinja /work/mxnet\n    ninja\n}\n\nbuild_ubuntu_cpu_cmake_debug() {\n    set -ex\n    cd /work/build\n    CC=gcc-7 CXX=g++-7 cmake \\\n        -DCMAKE_BUILD_TYPE=Debug \\\n        -DENABLE_TESTCOVERAGE=ON \\\n        -DUSE_CUDA=OFF \\\n        -DUSE_TVM_OP=ON \\\n        -DUSE_BLAS=Open \\\n        -DUSE_OPENMP=OFF \\\n        -DUSE_OPENCV=ON \\\n        -DUSE_SIGNAL_HANDLER=ON \\\n        -G Ninja \\\n        /work/mxnet\n    ninja\n}\n\nbuild_ubuntu_cpu_cmake_no_tvm_op() {\n    set -ex\n    cd /work/build\n    CC=gcc-7 CXX=g++-7 cmake \\\n        -DUSE_CUDA=OFF \\\n        -DUSE_TVM_OP=OFF \\\n        -DUSE_BLAS=Open \\\n        -DUSE_OPENMP=OFF \\\n        -DUSE_OPENCV=ON \\\n        -DUSE_SIGNAL_HANDLER=ON \\\n        -DCMAKE_BUILD_TYPE=Release \\\n        -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \\\n        -G Ninja \\\n        /work/mxnet\n\n    ninja\n}\n\nbuild_ubuntu_cpu_cmake_asan() {\n    set -ex\n\n    cd /work/build\n    cmake \\\n        -DUSE_CUDA=OFF \\\n        -DUSE_BLAS=Open \\\n        -DUSE_ONEDNN=OFF \\\n        -DUSE_OPENMP=OFF \\\n        -DUSE_OPENCV=OFF \\\n        -DCMAKE_BUILD_TYPE=Debug \\\n        -DUSE_GPERFTOOLS=OFF \\\n        -DUSE_JEMALLOC=OFF \\\n        -DUSE_ASAN=ON \\\n        /work/mxnet\n    make -j $(nproc) mxnet\n}\n\nbuild_ubuntu_cpu_gcc8_werror() {\n    set -ex\n    cd /work/build\n    CC=gcc-8 CXX=g++-8 cmake \\\n        -DUSE_BLAS=Open \\\n        -DUSE_CUDA=OFF \\\n        -DCMAKE_BUILD_TYPE=\"RelWithDebInfo\" \\\n        -GNinja /work/mxnet\n    ninja\n}\n\nbuild_ubuntu_cpu_clang10_werror() {\n    set -ex\n    cd /work/build\n    CXX=clang++-10 CC=clang-10 cmake \\\n       -DUSE_BLAS=Open \\\n       -DUSE_CUDA=OFF \\\n       -DCMAKE_BUILD_TYPE=\"RelWithDebInfo\" \\\n       -GNinja /work/mxnet\n    ninja\n}\n\nbuild_ubuntu_gpu_clang10_werror() {\n    set -ex\n    cd /work/build\n    # Disable cpp package as OpWrapperGenerator.py dlopens libmxnet.so,\n    # requiring presence of cuda driver libraries that are missing on CI host\n    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda-10.1/targets/x86_64-linux/lib/stubs\n    # Workaround https://github.com/thrust/thrust/issues/1072\n    # Can be deleted on Cuda 11\n    export CXXFLAGS=\"-I/usr/local/thrust\"\n\n    CXX=clang++-10 CC=clang-10 cmake \\\n       -DUSE_BLAS=Open \\\n       -DUSE_CUDA=ON \\\n       -DUSE_NVML=OFF \\\n       -DMXNET_CUDA_ARCH=\"$CI_CMAKE_CUDA_ARCH\" \\\n       -DCMAKE_BUILD_TYPE=\"RelWithDebInfo\" \\\n       -GNinja /work/mxnet\n    ninja\n}\n\nbuild_ubuntu_cpu_clang6() {\n    set -ex\n    cd /work/build\n    export OpenBLAS_HOME=/usr/local/openblas-clang/\n    CXX=clang++-6.0 CC=clang-6.0 cmake \\\n        -DUSE_BLAS=Open \\\n        -DUSE_ONEDNN=OFF \\\n        -DUSE_CUDA=OFF \\\n        -DUSE_OPENMP=OFF \\\n        -DUSE_DIST_KVSTORE=ON \\\n        -G Ninja /work/mxnet\n    ninja\n}\n\nbuild_ubuntu_cpu_clang100() {\n    set -ex\n    cd /work/build\n    export OpenBLAS_HOME=/usr/local/openblas-clang/\n    CXX=clang++-10 CC=clang-10 cmake \\\n       -DUSE_BLAS=Open \\\n       -DUSE_ONEDNN=OFF \\\n       -DUSE_CUDA=OFF \\\n       -DUSE_OPENMP=ON \\\n       -DUSE_DIST_KVSTORE=ON \\\n       -G Ninja /work/mxnet\n    ninja\n}\n\nbuild_ubuntu_cpu_clang_tidy() {\n    set -ex\n    cd /work/build\n    export OpenBLAS_HOME=/usr/local/openblas-clang/\n    # TODO(leezu) USE_OPENMP=OFF 3rdparty/dmlc-core/CMakeLists.txt:79 broken?\n    CXX=clang++-10 CC=clang-10 cmake \\\n       -DUSE_BLAS=Open \\\n       -DUSE_ONEDNN=OFF \\\n       -DUSE_CUDA=OFF \\\n       -DUSE_OPENMP=OFF \\\n       -DCMAKE_BUILD_TYPE=Debug \\\n       -DUSE_DIST_KVSTORE=ON \\\n       -DCMAKE_CXX_CLANG_TIDY=clang-tidy-10 \\\n       -G Ninja /work/mxnet\n    ninja\n}\n\nbuild_ubuntu_cpu_clang6_onednn() {\n    set -ex\n    cd /work/build\n    export OpenBLAS_HOME=/usr/local/openblas-clang/\n    CXX=clang++-6.0 CC=clang-6.0 cmake \\\n       -DUSE_BLAS=Open \\\n       -DUSE_ONEDNN=ON \\\n       -DUSE_CUDA=OFF \\\n       -DUSE_OPENMP=OFF \\\n       -G Ninja /work/mxnet\n    ninja\n}\n\nbuild_ubuntu_cpu_clang100_onednn() {\n    set -ex\n    cd /work/build\n    export OpenBLAS_HOME=/usr/local/openblas-clang/\n    CXX=clang++-10 CC=clang-10 cmake \\\n       -DUSE_BLAS=Open \\\n       -DUSE_ONEDNN=ON \\\n       -DUSE_CUDA=OFF \\\n       -G Ninja /work/mxnet\n    ninja\n}\n\nbuild_ubuntu_cpu_onednn() {\n    set -ex\n    cd /work/build\n    CC=gcc-7 CXX=g++-7 cmake \\\n        -DCMAKE_BUILD_TYPE=\"RelWithDebInfo\" \\\n        -DENABLE_TESTCOVERAGE=ON \\\n        -DUSE_TVM_OP=ON \\\n        -DUSE_BLAS=Open \\\n        -DUSE_ONEDNN=ON \\\n        -DUSE_CUDA=OFF \\\n        -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \\\n        -G Ninja /work/mxnet\n    ninja\n}\n\nbuild_ubuntu_cpu_onednn_mkl() {\n    set -ex\n    cd /work/build\n    CC=gcc-7 CXX=g++-7 cmake \\\n        -DCMAKE_BUILD_TYPE=\"RelWithDebInfo\" \\\n        -DENABLE_TESTCOVERAGE=OFF \\\n        -DUSE_ONEDNN=ON \\\n        -DUSE_CUDA=OFF \\\n        -DUSE_TVM_OP=ON \\\n        -DUSE_BLAS=MKL \\\n        -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \\\n        -GNinja /work/mxnet\n    ninja\n}\n\nbuild_ubuntu_gpu_tensorrt() {\n\n    set -ex\n\n    export CC=gcc-7\n    export CXX=g++-7\n    export ONNX_NAMESPACE=onnx\n    export PYBIN=$(which python3)\n    PYVERFULL=$($PYBIN -V | awk '{print $2}')\n    export PYVER=${PYVERFULL%.*}\n\n    # Build ONNX\n    pushd .\n    echo \"Installing ONNX.\"\n    cd 3rdparty/onnx-tensorrt/third_party/onnx\n    rm -rf build\n    mkdir -p build\n    cd build\n    cmake -DPYTHON_EXECUTABLE=$PYBIN -DCMAKE_CXX_FLAGS=-I/usr/include/python${PYVER} -DBUILD_SHARED_LIBS=ON ..\n    make -j$(nproc)\n    export LIBRARY_PATH=`pwd`:`pwd`/onnx/:$LIBRARY_PATH\n    export CPLUS_INCLUDE_PATH=`pwd`:$CPLUS_INCLUDE_PATH\n    export CXXFLAGS=-I`pwd`\n\n    popd\n\n    # Build ONNX-TensorRT\n    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib\n    export CPLUS_INCLUDE_PATH=${CPLUS_INCLUDE_PATH}:/usr/local/cuda/targets/x86_64-linux/include/\n    pushd .\n    cd 3rdparty/onnx-tensorrt/\n    mkdir -p build\n    cd build\n    cmake -DPYTHON_EXECUTABLE=$PYBIN -DONNX_NAMESPACE=$ONNX_NAMESPACE ..\n    make -j$(nproc)\n    export LIBRARY_PATH=`pwd`:$LIBRARY_PATH\n    popd\n\n    mkdir -p /work/mxnet/lib/\n    cp 3rdparty/onnx-tensorrt/third_party/onnx/build/*.so /work/mxnet/lib/\n    cp -L 3rdparty/onnx-tensorrt/build/libnvonnxparser.so /work/mxnet/lib/\n\n    cd /work/build\n    cmake -DUSE_CUDA=1                            \\\n          -DUSE_CUDNN=1                           \\\n          -DUSE_OPENCV=1                          \\\n          -DUSE_TENSORRT=1                        \\\n          -DUSE_INT64_TENSOR_SIZE=1               \\\n          -DUSE_OPENMP=0                          \\\n          -DUSE_BLAS=Open                         \\\n          -DUSE_ONEDNN=0                          \\\n          -DUSE_NVML=OFF                          \\\n          -DMXNET_CUDA_ARCH=\"$CI_CMAKE_CUDA_ARCH\" \\\n          -G Ninja                                \\\n          /work/mxnet\n\n    ninja\n}\n\nbuild_ubuntu_gpu_onednn() {\n    set -ex\n    cd /work/build\n    CC=gcc-7 CXX=g++-7 cmake \\\n        -DCMAKE_BUILD_TYPE=\"RelWithDebInfo\" \\\n        -DUSE_BLAS=Open \\\n        -DUSE_CUDA=ON \\\n        -DUSE_NVML=OFF \\\n        -DMXNET_CUDA_ARCH=\"$CI_CMAKE_CUDA_ARCH\" \\\n        -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \\\n        -G Ninja /work/mxnet\n    ninja\n}\n\nbuild_ubuntu_gpu_onednn_nocudnn() {\n    set -ex\n    cd /work/build\n    CC=gcc-7 CXX=g++-7 cmake \\\n        -DCMAKE_BUILD_TYPE=\"RelWithDebInfo\" \\\n        -DUSE_BLAS=Open \\\n        -DUSE_CUDA=ON \\\n        -DUSE_NVML=OFF \\\n        -DMXNET_CUDA_ARCH=\"$CI_CMAKE_CUDA_ARCH\" \\\n        -DUSE_CUDNN=OFF \\\n        -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \\\n        -G Ninja /work/mxnet\n    ninja\n}\n\nbuild_ubuntu_gpu() {\n    set -ex\n    cd /work/build\n    # Work around to link libcuda to libmxnet\n    # should be removed after https://github.com/apache/incubator-mxnet/issues/17858 is resolved. \n    ln -s -f /usr/local/cuda/targets/x86_64-linux/lib/stubs/libcuda.so libcuda.so.1\n    export LIBRARY_PATH=${LIBRARY_PATH}:/work/build\n    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/work/build\n    CC=gcc-7 CXX=g++-7 cmake \\\n        -DCMAKE_BUILD_TYPE=\"RelWithDebInfo\" \\\n        -DUSE_CUDA=ON \\\n        -DUSE_NVML=OFF \\\n        -DMXNET_CUDA_ARCH=\"$CI_CMAKE_CUDA_ARCH\" \\\n        -DUSE_CUDNN=ON \\\n        -DUSE_CPP_PACKAGE=ON \\\n        -DUSE_BLAS=Open \\\n        -DUSE_ONEDNN=OFF \\\n        -DUSE_DIST_KVSTORE=ON \\\n        -DBUILD_CYTHON_MODULES=ON \\\n        -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \\\n        -G Ninja /work/mxnet\n    ninja -j$(($(nproc)/2))\n}\n\nbuild_ubuntu_gpu_debug() {\n    set -ex\n    cd /work/build\n    CC=gcc-7 CXX=g++-7 cmake \\\n        -DCMAKE_BUILD_TYPE=Debug \\\n        -DUSE_CUDA=ON \\\n        -DUSE_NVML=OFF \\\n        -DMXNET_CUDA_ARCH=\"$CI_CMAKE_CUDA_ARCH\" \\\n        -DUSE_CUDNN=ON \\\n        -DUSE_BLAS=Open \\\n        -DUSE_ONEDNN=OFF \\\n        -DUSE_DIST_KVSTORE=ON \\\n        -DBUILD_CYTHON_MODULES=ON \\\n        -G Ninja /work/mxnet\n    ninja\n}\n\nbuild_ubuntu_cpu_large_tensor() {\n    set -ex\n    cd /work/build\n    CC=gcc-7 CXX=g++-7 cmake \\\n        -DUSE_SIGNAL_HANDLER=ON                 \\\n        -DUSE_CUDA=OFF                          \\\n        -DUSE_CUDNN=OFF                         \\\n        -DUSE_BLAS=Open                         \\\n        -DUSE_ONEDNN=ON                         \\\n        -G Ninja                                \\\n        /work/mxnet\n\n    ninja\n}\n\nbuild_ubuntu_gpu_large_tensor() {\n    set -ex\n    cd /work/build\n    CC=gcc-7 CXX=g++-7 cmake \\\n        -DUSE_SIGNAL_HANDLER=ON                 \\\n        -DUSE_CUDA=ON                           \\\n        -DUSE_CUDNN=ON                          \\\n        -DUSE_NVML=OFF                          \\\n        -DUSE_BLAS=Open                         \\\n        -DUSE_ONEDNN=ON                         \\\n        -DUSE_DIST_KVSTORE=ON                   \\\n        -DCMAKE_BUILD_TYPE=Release              \\\n        -DMXNET_CUDA_ARCH=\"$CI_CMAKE_CUDA_ARCH\" \\\n        -G Ninja                                \\\n        /work/mxnet\n\n    ninja\n}\n\n# Testing\n\nsanity_check() {\n    set -ex\n    sanity_clang\n    sanity_license\n    sanity_cmakelint\n    sanity_tutorial\n    sanity_python_prospector\n    sanity_cpp\n}\n\nsanity_cmakelint() {\n    set -exu\n    \n    git ls-files -z -- bootstrap '*.cmake' '*.cmake.in' '*CMakeLists.txt' | grep -E -z -v '^(3rdparty)|cmake/Modules/|cmake/upstream/' | xargs -0 cmakelint --config=.cmakelintrc --quiet\n}\n\nsanity_tutorial() {\n    set -ex\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/tutorials/test_sanity_tutorials.py\n}\n\nsanity_license() {\n    set -ex\n    tools/license_header.py check\n}\n\nsanity_cpp() {\n    set -ex\n    3rdparty/dmlc-core/scripts/lint.py mxnet cpp include src plugin cpp-package tests --exclude_path src/operator/contrib/ctc_include include/onednn\n}\n\nsanity_python_prospector() {\n    set -e\n    set +x\n\n    # Run Prospector\n    python3 -m prospector --profile prospector.yaml | tee prospector-output.txt\n    error_cnt=$(awk '/Messages Found:/{print $NF}' prospector-output.txt)\n    if [ $error_cnt -ne 0 ]; then\n        echo 'Please fix the above Prospector warnings.'\n        rm -rf prospector-output.txt\n        exit 1\n    fi\n    rm -rf prospector-output.txt\n}\n\nsanity_clang() {\n    set -e\n    set +x\n    # .github/workgflows/greetings.yml passes BASE_SHA, GITHUB_RUN_ID, GITHUB_BASE_REF for pull requests.\n    BASE_SHA=\"${GITHUB_PR_BASE_SHA}\"\n    GITHUB_RUN_ID=\"${GITHUB_PR_RUN_ID}\"\n    GITHUB_BASE_REF=\"${GITHUB_PR_BASE_REF}\"\n\n    if [ \"${BASE_SHA}\" == \"\" ]; then\n        BASE_SHA=`git show-ref --hash refs/remotes/origin/master`\n        if [ \"${GITHUB_RUN_ID}\" == \"\" ] || [ \"${GITHUB_BASE_REF}\" == \"\" ]; then\n             GITHUB_RUN_ID=`(git log --pretty=format:'%h' -n 1)`\n             GITHUB_BASE_REF=\"master\"\n        fi\n    fi\n\n    git remote add \"${GITHUB_RUN_ID}\" https://github.com/apache/incubator-mxnet.git\n    git fetch \"${GITHUB_RUN_ID}\" \"$GITHUB_BASE_REF\"\n    \n    tools/lint/clang_format_ci.sh \"${BASE_SHA}\"\n    GIT_DIFFERENCE=$(git diff)\n    if [[ -z $GIT_DIFFERENCE ]]; then\n        git remote remove \"${GITHUB_RUN_ID}\" # temporary remote is removed\n        return\n    fi\n\n    echo \"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\"\n    echo \"| Clang-format failures found! Run: \"\n    echo \"|    tools/lint/clang_format_ci.sh ${BASE_SHA} \"\n    echo \"| to fix this error. \"\n    echo \"| For more info, see: https://mxnet.apache.org/versions/master/community/clang_format_guide\"\n    echo \"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\"\n\n    echo \"$GIT_DIFFERENCE\"\n    git remote remove \"${GITHUB_RUN_ID}\" # temporary remote is removed\n    exit 1\n}\n\n# Tests libmxnet\n# Parameters:\n# $1 -> mxnet_variant: The variant of the libmxnet.so library\ncd_unittest_ubuntu() {\n    set -ex\n    source /opt/rh/rh-python38/enable\n    export PYTHONPATH=./python/\n    export MXNET_ONEDNN_DEBUG=0  # Ignored if not present\n    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0\n    export MXNET_SUBGRAPH_VERBOSE=0\n    export MXNET_ENABLE_CYTHON=0\n    export CD_JOB=1 # signal this is a CD run so any unecessary tests can be skipped\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n\n    local mxnet_variant=${1:?\"This function requires a mxnet variant as the first argument\"}\n\n    OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -n 4 --durations=50 --verbose tests/python/unittest\n    pytest -m 'serial' --durations=50 --verbose tests/python/unittest\n\n    # https://github.com/apache/mxnet/issues/11801\n    # if [[ ${mxnet_variant} = \"cpu\" ]] || [[ ${mxnet_variant} = \"mkl\" ]]; then\n        # integrationtest_ubuntu_cpu_dist_kvstore\n    # fi\n\n    if [[ ${mxnet_variant} = cu* ]]; then\n        MXNET_GPU_MEM_POOL_TYPE=Unpooled \\\n        MXNET_ENGINE_TYPE=NaiveEngine \\\n            OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --verbose tests/python/gpu\n        MXNET_GPU_MEM_POOL_TYPE=Unpooled \\\n            OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' -n 4 --durations=50 --verbose tests/python/gpu\n        pytest -m 'serial' --durations=50 --verbose tests/python/gpu\n        pytest --durations=50 --verbose tests/python/gpu/test_amp_init.py\n\n        # TODO(szha): fix and reenable the hanging issue. tracked in #18098\n        # integrationtest_ubuntu_gpu_dist_kvstore\n        # TODO(eric-haibin-lin): fix and reenable\n        # integrationtest_ubuntu_gpu_byteps\n    fi\n\n    if [[ ${mxnet_variant} = *mkl ]]; then\n        OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 --durations=50 --verbose tests/python/dnnl\n    fi\n}\n\nunittest_ubuntu_python3_cpu_onnx() {\n    set -ex\n    export PYTHONPATH=./python/\n    export MXNET_SUBGRAPH_VERBOSE=0\n    export DMLC_LOG_STACK_TRACE_DEPTH=10\n\n    pytest --cov-report xml:onnx_unittest.xml --verbose tests/python/onnx/test_operators.py\n    pytest --cov-report xml:onnx_unittest.xml --cov-append --verbose tests/python/onnx/test_models.py\n}\n\nunittest_ubuntu_python3_cpu() {\n    set -ex\n    export PYTHONPATH=./python/\n    export MXNET_ONEDNN_DEBUG=0  # Ignored if not present\n    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0\n    export MXNET_SUBGRAPH_VERBOSE=0\n    export MXNET_ENABLE_CYTHON=0\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest\n    MXNET_ENGINE_TYPE=NaiveEngine \\\n        OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest\n    pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest\n}\n\nunittest_ubuntu_python3_cpu_onednn() {\n    set -ex\n    export PYTHONPATH=./python/\n    export MXNET_ONEDNN_DEBUG=0  # Ignored if not present\n    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0\n    export MXNET_SUBGRAPH_VERBOSE=0\n    export MXNET_ENABLE_CYTHON=0\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest\n    MXNET_ENGINE_TYPE=NaiveEngine \\\n                     OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest\n    pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest\n    pytest --durations=50 --cov-report xml:tests_mkl.xml --verbose tests/python/dnnl\n}\n\nunittest_array_api_standardization() {\n    set -ex\n    python3 -m pip install -e /work/mxnet/python --user\n    cd ..\n    git clone https://github.com/data-apis/array-api-tests.git\n    pushd /work/array-api-tests\n    git checkout c1dba80a196a03f880d2e0a998a272fb3867b720\n    export ARRAY_API_TESTS_MODULE=mxnet.numpy pytest\n    export MXNET_ENABLE_CYTHON=1\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    python3 -m pytest --reruns 3 --durations=50 --cov-report xml:tests_api.xml --verbose array_api_tests/test_creation_functions.py\n    python3 -m pytest --reruns 3 --durations=50 --cov-report xml:tests_api.xml --verbose array_api_tests/test_indexing.py\n    python3 -m pytest --reruns 3 --durations=50 --cov-report xml:tests_api.xml --verbose array_api_tests/test_elementwise_functions.py\n    python3 -m pytest --reruns 3 --durations=50 --cov-report xml:tests_api.xml --verbose array_api_tests/test_constants.py\n    python3 -m pytest --reruns 3 --durations=50 --cov-report xml:tests_api.xml --verbose array_api_tests/test_broadcasting.py\n    python3 -m pytest --reruns 3 --durations=50 --cov-report xml:tests_api.xml --verbose \\\n        array_api_tests/test_type_promotion.py::test_elementwise_function_two_arg_bool_type_promotion\n    python3 -m pytest --reruns 3 --durations=50 --cov-report xml:tests_api.xml --verbose \\\n        array_api_tests/test_type_promotion.py::test_elementwise_function_two_arg_promoted_type_promotion\n    python3 -m pytest --reruns 3 --durations=50 --cov-report xml:tests_api.xml --verbose \\\n        array_api_tests/test_type_promotion.py::test_elementwise_function_one_arg_bool\n    python3 -m pytest --reruns 3 --durations=50 --cov-report xml:tests_api.xml --verbose \\\n        array_api_tests/test_type_promotion.py::test_elementwise_function_one_arg_type_promotion\n    python3 -m pytest --reruns 3 --durations=50 --cov-report xml:tests_api.xml --verbose \\\n        array_api_tests/test_type_promotion.py::test_operator_one_arg_type_promotion\n    python3 -m pytest --reruns 3 --durations=50 --cov-report xml:tests_api.xml --verbose \\\n        array_api_tests/test_type_promotion.py::test_operator_two_arg_bool_promotion\n    python3 -m pytest --reruns 3 --durations=50 --cov-report xml:tests_api.xml --verbose \\\n        array_api_tests/test_type_promotion.py::test_operator_two_arg_promoted_promotion\n    python3 -m pytest --reruns 3 --durations=50 --cov-report xml:tests_api.xml --verbose \\\n        array_api_tests/test_type_promotion.py::test_operator_inplace_two_arg_promoted_promotion\n    popd\n}\n\nunittest_ubuntu_python3_gpu() {\n    set -ex\n    export PYTHONPATH=./python/\n    export MXNET_ONEDNN_DEBUG=0 # Ignored if not present\n    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0\n    export MXNET_SUBGRAPH_VERBOSE=0\n    export CUDNN_VERSION=${CUDNN_VERSION:-7.0.3}\n    export MXNET_ENABLE_CYTHON=0\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    MXNET_GPU_MEM_POOL_TYPE=Unpooled \\\n        OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu\n    MXNET_GPU_MEM_POOL_TYPE=Unpooled \\\n    MXNET_ENGINE_TYPE=NaiveEngine \\\n        OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu\n    pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu\n    pytest --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu/test_amp_init.py\n}\n\nunittest_ubuntu_python3_gpu_cython() {\n    set -ex\n    export PYTHONPATH=./python/\n    export MXNET_ONEDNN_DEBUG=1 # Ignored if not present\n    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0\n    export MXNET_SUBGRAPH_VERBOSE=0\n    export CUDNN_VERSION=${CUDNN_VERSION:-7.0.3}\n    export MXNET_ENABLE_CYTHON=1\n    export MXNET_ENFORCE_CYTHON=1\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    check_cython\n    MXNET_GPU_MEM_POOL_TYPE=Unpooled \\\n        OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu\n    MXNET_GPU_MEM_POOL_TYPE=Unpooled \\\n    MXNET_ENGINE_TYPE=NaiveEngine \\\n        OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu\n    pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu\n    pytest --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu/test_amp_init.py\n}\n\nunittest_ubuntu_python3_gpu_nocudnn() {\n    set -ex\n    export PYTHONPATH=./python/\n    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0\n    export MXNET_SUBGRAPH_VERBOSE=0\n    export CUDNN_OFF_TEST_ONLY=true\n    export MXNET_ENABLE_CYTHON=0\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    MXNET_GPU_MEM_POOL_TYPE=Unpooled \\\n        OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu\n    MXNET_GPU_MEM_POOL_TYPE=Unpooled \\\n    MXNET_ENGINE_TYPE=NaiveEngine \\\n        OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu\n    pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu\n    pytest --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu/test_amp_init.py\n}\n\nunittest_cpp() {\n    set -ex\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    build/tests/mxnet_unit_tests\n}\n\nunittest_centos7_cpu() {\n    set -ex\n    source /opt/rh/rh-python38/enable\n    cd /work/mxnet\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest\n    MXNET_ENGINE_TYPE=NaiveEngine \\\n        OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest\n    python -m pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest\n    OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -n 4 --durations=50 --cov-report xml:tests_train.xml --verbose tests/python/train\n}\n\nunittest_centos7_gpu() {\n    set -ex\n    source /opt/rh/rh-python38/enable\n    cd /work/mxnet\n    export CUDNN_VERSION=${CUDNN_VERSION:-7.0.3}\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    MXNET_GPU_MEM_POOL_TYPE=Unpooled \\\n        OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu\n    MXNET_GPU_MEM_POOL_TYPE=Unpooled \\\n    MXNET_ENGINE_TYPE=NaiveEngine \\\n        OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu\n    pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu\n    pytest --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu/test_amp_init.py\n}\n\nintegrationtest_ubuntu_cpp_package_gpu() {\n    set -ex\n    export DMLC_LOG_STACK_TRACE_DEPTH=10\n    cpp-package/tests/ci_test.sh\n}\n\ntest_python3_data_interchange_gpu() {\n    set -ex\n    python3 -m pip install torch==1.10.0+cu113 torchvision==0.11.1+cu113 torchaudio==0.10.0+cu113 \\\n        -f https://download.pytorch.org/whl/cu113/torch_stable.html\n    MXNET_ENGINE_TYPE=ThreadedEngineAsync \\\n        python3 -m pytest --durations=50 tests/python/array-api/test_data_interchange.py\n}\n\nintegrationtest_ubuntu_cpu_onnx() {\n\tset -ex\n\texport PYTHONPATH=./python/\n\texport MXNET_SUBGRAPH_VERBOSE=0\n\texport DMLC_LOG_STACK_TRACE_DEPTH=100\n\tpython3 tests/python/unittest/onnx/backend_test.py\n\t#OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/mxnet_export_test.py\n\t#OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_models.py\n\t#OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_node.py\n\t#OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_onnxruntime.py\n}\n\nintegrationtest_ubuntu_cpu_dist_kvstore() {\n    set -ex\n    pushd .\n    export PYTHONPATH=./python/\n    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0\n    export MXNET_SUBGRAPH_VERBOSE=0\n    export MXNET_USE_OPERATOR_TUNING=0\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    cd tests/nightly/\n    python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --type=gluon_step_cpu\n    python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --type=gluon_sparse_step_cpu\n    python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --type=invalid_cpu\n    python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --type=gluon_type_cpu\n    python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py\n    python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --no-multiprecision\n    python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --type=compressed_cpu_1bit\n    python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --type=compressed_cpu_1bit --no-multiprecision\n    python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --type=compressed_cpu_2bit\n    python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --type=compressed_cpu_2bit --no-multiprecision\n    python3 ../../tools/launch.py -n 3 --launcher local python3 test_server_profiling.py\n    popd\n}\n\nintegrationtest_ubuntu_gpu_dist_kvstore() {\n    set -ex\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    pushd .\n    cd /work/mxnet/python\n    pip3 install -e .\n    pip3 install --no-cache-dir horovod\n    cd /work/mxnet/tests/nightly\n    ./test_distributed_training-gpu.sh\n    popd\n}\n\nintegrationtest_ubuntu_gpu_byteps() {\n    set -ex\n    pushd .\n    export PYTHONPATH=$PWD/python/\n    export BYTEPS_WITHOUT_PYTORCH=1\n    export BYTEPS_WITHOUT_TENSORFLOW=1\n    pip3 install byteps==0.2.3 --user\n    git clone -b v0.2.3 https://github.com/bytedance/byteps ~/byteps\n    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0\n    export MXNET_SUBGRAPH_VERBOSE=0\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    cd tests/nightly/\n\n    export NVIDIA_VISIBLE_DEVICES=0\n    export DMLC_WORKER_ID=0 # your worker id\n    export DMLC_NUM_WORKER=1 # one worker\n    export DMLC_ROLE=worker\n\n    # the following value does not matter for non-distributed jobs\n    export DMLC_NUM_SERVER=1\n    export DMLC_PS_ROOT_URI=0.0.0.127\n    export DMLC_PS_ROOT_PORT=1234\n\n    python3 ~/byteps/launcher/launch.py python3 dist_device_sync_kvstore_byteps.py\n\n    popd\n}\n\n\ntest_ubuntu_cpu_python3() {\n    set -ex\n    pushd .\n    export MXNET_LIBRARY_PATH=/work/build/libmxnet.so\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    VENV=mxnet_py3_venv\n    virtualenv -p `which python3` $VENV\n    source $VENV/bin/activate\n\n    cd /work/mxnet/python\n    pip3 install -e .\n    cd /work/mxnet\n    OMP_NUM_THREADS=$(expr $(nproc) / 4) python3 -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --verbose tests/python/unittest\n    MXNET_ENGINE_TYPE=NaiveEngine \\\n        OMP_NUM_THREADS=$(expr $(nproc) / 4) python3 -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --verbose tests/python/unittest\n    python3 -m pytest -m 'serial' --durations=50 --verbose tests/python/unittest\n\n    popd\n}\n\n# QEMU based ARM tests\nunittest_ubuntu_python3_arm() {\n    set -ex\n    export PYTHONPATH=./python/\n    export MXNET_ONEDNN_DEBUG=0  # Ignored if not present\n    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0\n    export MXNET_SUBGRAPH_VERBOSE=0\n    export MXNET_ENABLE_CYTHON=0\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    python3 -m pytest -n 2 --verbose tests/python/unittest/test_engine.py\n}\n\n# Functions that run the nightly Tests:\n\n#Runs Apache RAT Check on MXNet Source for License Headers\ntest_rat_check() {\n    set -e\n    set -o pipefail\n    pushd .\n\n    cd /usr/local/src/apache-rat-0.13\n\n    # Use shell number 5 to duplicate the log output. It get sprinted and stored in $OUTPUT at the same time https://stackoverflow.com/a/12451419\n    exec 5>&1\n    OUTPUT=$(java -jar apache-rat-0.13.jar -E /work/mxnet/rat-excludes -d /work/mxnet|tee >(cat - >&5))\n    ERROR_MESSAGE=\"Printing headers for text files without a valid license header\"\n\n\n    echo \"-------Process The Output-------\"\n\n    if [[ $OUTPUT =~ $ERROR_MESSAGE ]]; then\n        echo \"ERROR: RAT Check detected files with unknown licenses. Please fix and run test again!\";\n        exit 1\n    else\n        echo \"SUCCESS: There are no files with an Unknown License.\";\n    fi\n    popd\n}\n\n#Single Node KVStore Test\nnightly_test_KVStore_singleNode() {\n    set -ex\n    export PYTHONPATH=./python/\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    tests/nightly/test_kvstore.py\n}\n\n#Test Large Tensor Size\nnightly_test_large_tensor() {\n    set -ex\n    export PYTHONPATH=./python/\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    pytest -s --exitfirst --verbose --timeout=7200 tests/nightly/test_np_large_array.py\n}\n\n#Tests Model backwards compatibility on MXNet\nnightly_model_backwards_compat_test() {\n    set -ex\n    export PYTHONPATH=/work/mxnet/python/\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    ./tests/nightly/model_backwards_compatibility_check/model_backward_compat_checker.sh\n}\n\n#Backfills S3 bucket with models trained on earlier versions of mxnet\nnightly_model_backwards_compat_train() {\n    set -ex\n    export PYTHONPATH=./python/\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    ./tests/nightly/model_backwards_compatibility_check/train_mxnet_legacy_models.sh\n}\n\nnightly_tutorial_test_ubuntu_python3_gpu() {\n    set -ex\n    cd /work/mxnet/docs\n    export BUILD_VER=tutorial\n    export MXNET_DOCS_BUILD_MXNET=0\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    make html\n    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0\n    export MXNET_SUBGRAPH_VERBOSE=0\n    export PYTHONPATH=/work/mxnet/python/\n    export MXNET_TUTORIAL_TEST_KERNEL=python3\n    cd /work/mxnet/tests/tutorials\n    pytest --durations=50 --cov-report xml:tests_tutorials.xml --capture=no test_tutorials.py\n}\n\nnightly_estimator() {\n    set -ex\n    export DMLC_LOG_STACK_TRACE_DEPTH=100\n    cd /work/mxnet/tests/nightly/estimator\n    export PYTHONPATH=/work/mxnet/python/\n    pytest test_estimator_cnn.py\n    pytest test_sentiment_rnn.py\n}\n\n# For testing PRs\ndeploy_docs() {\n    set -ex\n    pushd .\n\n    export CC=\"ccache gcc\"\n    export CXX=\"ccache g++\"\n\n    build_python_docs\n\n    popd\n}\n\n\nbuild_docs_setup() {\n    build_folder=\"docs/_build\"\n    mxnetlib_folder=\"/work/mxnet/lib\"\n\n    mkdir -p $build_folder\n    mkdir -p $mxnetlib_folder\n}\n\nbuild_jekyll_docs() {\n    set -ex\n\n    pushd .\n    build_docs_setup\n    pushd docs/static_site\n    make clean\n    make html\n    popd\n\n    GZIP=-9 tar zcvf jekyll-artifacts.tgz -C docs/static_site/build html\n    mv jekyll-artifacts.tgz docs/_build/\n    popd\n}\n\n\nbuild_python_docs() {\n    set -ex\n    pushd .\n\n    build_docs_setup\n\n    pushd docs/python_docs\n    python3 -m pip install -r requirements\n    python3 -m pip install themes/mx-theme\n    python3 -m pip install -e /work/mxnet/python --user\n\n    export PATH=/home/jenkins_slave/.local/bin:$PATH\n\n    pushd python\n    cp tutorials/getting-started/crash-course/prepare_dataset.py .\n    make clean\n    make html EVAL=1\n\n    GZIP=-9 tar zcvf python-artifacts.tgz -C build/_build/html .\n    popd\n\n    mv python/python-artifacts.tgz /work/mxnet/docs/_build/\n    popd\n\n    popd\n}\n\n\nbuild_c_docs() {\n    set -ex\n    pushd .\n\n    build_docs_setup\n    doc_path=\"docs/cpp_docs\"\n    pushd $doc_path\n\n    make clean\n    make html\n\n    doc_artifact=\"c-artifacts.tgz\"\n    GZIP=-9 tar zcvf $doc_artifact -C build/html/html .\n    popd\n\n    mv $doc_path/$doc_artifact docs/_build/\n\n    popd\n}\n\n\nbuild_docs() {\n    pushd docs/_build\n    tar -xzf jekyll-artifacts.tgz\n    python_doc_folder='html/api/python/docs'\n    api_folder='html/api'\n\n    # Python has it's own landing page/site so we don't put it in /docs/api\n    mkdir -p $python_doc_folder && tar -xzf python-artifacts.tgz --directory $python_doc_folder\n    mkdir -p $api_folder/cpp/docs/api && tar -xzf c-artifacts.tgz --directory $api_folder/cpp/docs/api\n\n    # check if .asf.yaml file exists\n    if [ ! -f \"html/.asf.yaml\" ]; then\n        echo \"html/.asf.yaml file does not exist. Exiting 1\"\n        exit 1\n    fi\n    # check if .htaccess file exists\n    if [ ! -f \"html/.htaccess\" ]; then\n        echo \"html/.htaccess file does not exist. Exiting 1\"\n        exit 1\n    fi\n    # get the version\n    version=$(grep \"RewriteRule\" html/.htaccess | grep -E \"versions\\/[0-9]\" | sed -nre 's/^[^0-9]*(([0-9]+\\.)*[0-9]+).*/\\1/p')\n    # count how many versions are found\n    lines=$(echo \"$version\" | wc -l)\n    # check if multiple versions are found\n    if [ \"$lines\" != \"1\" ]; then\n        echo \"multiple versions detected: $lines. Exiting 1\"\n        exit 1\n    fi\n    # check if no version is found\n    if [ \"$version\" == \"\" ]; then\n        echo \"no version found. Exiting 1\"\n        exit 1\n    fi\n    # print the one and only default mxnet version\n    echo \"detected version is $version\"\n    # check if the artifacts for this version exist\n    if [ -d \"html/versions/$version/api\" ]; then\n        echo \"html/versions/$version/api directory exists\"\n    else\n        echo \"html/versions/$version/api directory does not exist! Exiting 1\"\n        exit 1\n    fi\n\n    # copy the full site for this version to versions folder\n    mkdir -p html/versions/master\n    for f in 404.html api assets community ecosystem features trusted_by feed.xml get_started index.html; do\n        cp -r html/$f html/versions/master/\n    done\n\n    # clean up temp files\n    find html -type f -name '.DS_Store' -delete\n\n    # archive artifact\n    GZIP=-9 tar -zcvf full_website.tgz -C html .\n    popd\n}\n\nbuild_docs_beta() {\n    pushd docs/_build\n    tar -xzf jekyll-artifacts.tgz\n    python_doc_folder=\"html/versions/$BRANCH/api/python/docs\"\n    cpp_doc_folder=\"html/versions/$BRANCH/api/cpp/docs\"\n    mkdir -p $python_doc_folder && tar -xzf python-artifacts.tgz --directory $python_doc_folder\n    mkdir -p $cpp_doc_folder && tar -xzf c-artifacts.tgz --directory $cpp_doc_folder\n    GZIP=-9 tar -zcvf beta_website.tgz -C html .\n    popd\n}\n\npush_docs() {\n    folder_name=$1\n    set -ex\n    export PATH=~/.local/bin:$PATH\n    pushd docs/_build\n    tar -xzf full_website.tgz --strip-components 1\n    # check if folder_name already exists in versions\n    pushd versions\n    if [ -d \"$folder_name\" ]; then\n        echo \"Folder $folder_name already exists in versions. Please double check the FOLDER_NAME variable in Jenkens pipeline\"\n        exit 1\n    fi\n    mv master $folder_name\n    popd\n    zip -r9 versions.zip versions/.\n    # Upload versions folder\n    aws s3 cp versions.zip s3://mxnet-website-static-artifacts --acl public-read\n    # Backup versions folder with the latest version name\n    backup_file=\"versions_backup_upto_$folder_name.zip\"\n    aws s3 cp s3://mxnet-website-static-artifacts/versions.zip s3://mxnet-website-static-artifacts/$backup_file --acl public-read\n    popd\n}\n\ncreate_repo() {\n   repo_folder=$1\n   mxnet_url=$2\n   git clone $mxnet_url $repo_folder --recursive\n   echo \"Adding MXNet upstream repo...\"\n   cd $repo_folder\n   git remote add upstream https://github.com/apache/mxnet\n   cd ..\n}\n\n\nrefresh_branches() {\n   repo_folder=$1\n   cd $repo_folder\n   git fetch\n   git fetch upstream\n   cd ..\n}\n\ncheckout() {\n   repo_folder=$1\n   cd $repo_folder\n   # Overriding configs later will cause a conflict here, so stashing...\n   git stash\n   # Fails to checkout if not available locally, so try upstream\n   git checkout \"$repo_folder\" || git branch $repo_folder \"upstream/$repo_folder\" && git checkout \"$repo_folder\" || exit 1\n   if [ $tag == 'master' ]; then\n      git pull\n      # master gets warnings as errors for Sphinx builds\n      OPTS=\"-W\"\n      else\n      OPTS=\n   fi\n   git submodule update --init --recursive\n   cd ..\n}\n\nbuild_static_libmxnet() {\n    set -ex\n    pushd .\n    source /opt/rh/devtoolset-8/enable\n    source /opt/rh/rh-python38/enable\n    # Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.\n    export CXXFLAGS=\"-fabi-version=11 -fabi-compat-version=7\"\n    local mxnet_variant=${1:?\"This function requires a python command as the first argument\"}\n    source tools/staticbuild/build.sh ${mxnet_variant}\n    popd\n}\n\n# Tests CD PyPI packaging in CI\nci_package_pypi() {\n    set -ex\n    # copies oneDNN header files to 3rdparty/onednn/include/oneapi/dnnl/ as in CD\n    mkdir -p 3rdparty/onednn/include/oneapi/dnnl\n    cp include/onednn/oneapi/dnnl/dnnl_version.h 3rdparty/onednn/include/oneapi/dnnl/.\n    cp include/onednn/oneapi/dnnl/dnnl_config.h 3rdparty/onednn/include/oneapi/dnnl/.\n    local mxnet_variant=${1:?\"This function requires a python command as the first argument\"}\n    cd_package_pypi ${mxnet_variant}\n    cd_integration_test_pypi\n}\n\n# Packages libmxnet into wheel file\ncd_package_pypi() {\n    set -ex\n    pushd .\n    source /opt/rh/devtoolset-8/enable\n    source /opt/rh/rh-python38/enable\n    # Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.\n    export CXXFLAGS=\"-fabi-version=11 -fabi-compat-version=7\"\n    local mxnet_variant=${1:?\"This function requires a python command as the first argument\"}\n    ./cd/python/pypi/pypi_package.sh ${mxnet_variant}\n    popd\n}\n\n# Sanity checks wheel file\ncd_integration_test_pypi() {\n    set -ex\n    source /opt/rh/rh-python38/enable\n\n    # install mxnet wheel package\n    pip3 install --user ./wheel_build/dist/*.whl\n\n    # execute tests\n    # TODO: Add tests (18549)\n}\n\n# Publishes wheel to PyPI\ncd_pypi_publish() {\n    set -ex\n    pip3 install --user twine\n    python3 ./cd/python/pypi/pypi_publish.py `readlink -f wheel_build/dist/*.whl`\n}\n\ncd_s3_publish() {\n    set -ex\n    filepath=$(readlink -f wheel_build/dist/*.whl)\n    filename=$(basename $filepath)\n    variant=$(echo $filename | cut -d'-' -f1 | cut -d'_' -f2 -s)\n    if [ -z \"${variant}\" ]; then\n        variant=\"cpu\"\n    fi\n    export PATH=/usr/local/bin:$PATH\n    aws s3 cp ${filepath} s3://apache-mxnet/dist/python/${variant}/${filename} --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=43f628fab72838a4f0b929d7f1993b14411f4b0294b011261bc6bd3e950a6822\n}\n\nbuild_static_python_cpu() {\n    set -ex\n    pushd .\n    export mxnet_variant=cpu\n    source /opt/rh/devtoolset-8/enable\n    source /opt/rh/rh-python38/enable\n    # Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.\n    export CXXFLAGS=\"-fabi-version=11 -fabi-compat-version=7\"\n    ./ci/publish/python/build.sh\n    popd\n}\n\nbuild_static_python_cu102() {\n    set -ex\n    pushd .\n    export mxnet_variant=cu102\n    source /opt/rh/devtoolset-8/enable\n    source /opt/rh/rh-python38/enable\n    # Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.\n    export CXXFLAGS=\"-fabi-version=11 -fabi-compat-version=7\"\n    ./ci/publish/python/build.sh\n    popd\n}\n\n# artifact repository unit tests\ntest_artifact_repository() {\n    set -ex\n    pushd .\n    cd cd/utils/\n    OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 test_artifact_repository.py\n    popd\n}\n\n##############################################################\n# MAIN\n#\n# Run function passed as argument\nset +x\nif [ $# -gt 0 ]\nthen\n    $@\nelse\n    cat<<EOF\n\n$0: Execute a function by passing it as an argument to the script:\n\nPossible commands:\n\nEOF\n    declare -F | cut -d' ' -f3\n    echo\nfi\n"
  },
  {
    "path": "ci/docker/toolchains/aarch64-linux-gnu-toolchain.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset(CMAKE_SYSTEM_NAME Linux)\nset(CMAKE_SYSTEM_PROCESSOR \"aarch64\")\nset(CMAKE_C_COMPILER aarch64-linux-gnu-gcc)\nset(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++)\nset(CMAKE_CUDA_COMPILER nvcc)\nset(CMAKE_CUDA_HOST_COMPILER aarch64-linux-gnu-gcc)\nset(CMAKE_FIND_ROOT_PATH \"/usr/aarch64-linux-gnu\")\n\nset(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)\nset(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)\nset(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)\nset(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)\n"
  },
  {
    "path": "ci/docker/toolchains/arm-linux-gnueabihf-toolchain.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset(CMAKE_SYSTEM_NAME Linux)\nset(CMAKE_SYSTEM_PROCESSOR \"armv7l\")\nset(CMAKE_C_COMPILER arm-linux-gnueabihf-gcc)\nset(CMAKE_CXX_COMPILER arm-linux-gnueabihf-g++)\nset(CMAKE_FIND_ROOT_PATH \"/usr/arm-linux-gnueabihf\" \"/usr/local/arm-linux-gnueabihf\")\n\nset(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)\nset(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)\nset(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)\nset(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)\n"
  },
  {
    "path": "ci/docker_login.py",
    "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport argparse\nimport json\nimport logging\nimport os\nimport subprocess\nimport sys\n\nfrom util import retry, config_logging\n\nDOCKERHUB_LOGIN_NUM_RETRIES = 5\nDOCKERHUB_RETRY_SECONDS = 5\n\n\ndef _get_dockerhub_credentials(secret_name: str, secret_endpoint_url: str, secret_endpoint_region_name: str):\n    import boto3\n    import botocore\n\n    session = boto3.Session()\n    client = session.client(\n        service_name='secretsmanager',\n        region_name=secret_endpoint_region_name,\n        endpoint_url=secret_endpoint_url\n    )\n    try:\n        get_secret_value_response = client.get_secret_value(\n            SecretId=secret_name\n        )\n    except botocore.exceptions.ClientError as client_error:\n        if client_error.response['Error']['Code'] == 'ResourceNotFoundException':\n            logging.exception(\"The requested secret %s was not found\", secret_name)\n        elif client_error.response['Error']['Code'] == 'InvalidRequestException':\n            logging.exception(\"The request was invalid due to:\")\n        elif client_error.response['Error']['Code'] == 'InvalidParameterException':\n            logging.exception(\"The request had invalid params:\")\n        raise\n    else:\n        secret = get_secret_value_response['SecretString']\n        secret_dict = json.loads(secret)\n        return secret_dict\n\n\n@retry(target_exception=subprocess.CalledProcessError, tries=DOCKERHUB_LOGIN_NUM_RETRIES,\n       delay_s=DOCKERHUB_RETRY_SECONDS)\ndef login_dockerhub(secret_name: str, secret_endpoint_url: str, secret_endpoint_region_name: str):\n    \"\"\"\n    Login to the Docker Hub account\n    :return: None\n    \"\"\"\n    dockerhub_credentials = _get_dockerhub_credentials(secret_name, secret_endpoint_url, secret_endpoint_region_name)\n\n    logging.info('Logging in to DockerHub')\n    # We use password-stdin instead of --password to avoid leaking passwords in case of an error.\n    # This method will produce the following output:\n    # > WARNING! Your password will be stored unencrypted in /home/jenkins_slave/.docker/config.json.\n    # > Configure a credential helper to remove this warning. See\n    # > https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n    # Since we consider the restricted slaves a secure environment, that's fine. Also, using this will require\n    # third party applications which would need a review first as well.\n    p = subprocess.run(['docker', 'login', '--username', dockerhub_credentials['username'], '--password-stdin'],\n                       stdout=subprocess.PIPE, input=str.encode(dockerhub_credentials['password']))\n    logging.info(p.stdout)\n    if p.returncode == 0:\n        logging.info('Successfully logged in to DockerHub')\n        return\n\n    raise RuntimeError(\"Failed to login to DockerHub\")\n\n\ndef logout_dockerhub():\n    \"\"\"\n    Log out of DockerHub to delete local credentials\n    :return: None\n    \"\"\"\n    logging.info('Logging out of DockerHub')\n    subprocess.call(['docker', 'logout'])\n    logging.info('Successfully logged out of DockerHub')\n\n\ndef main(command_line_arguments):\n    config_logging()\n\n    parser = argparse.ArgumentParser(\n        description=\"Safe docker login utility to avoid leaking passwords\",\n        epilog=\"\"\n    )\n    parser.add_argument(\"--secret-name\",\n                        help=\"Secret name\",\n                        type=str,\n                        required=True)\n\n    parser.add_argument(\"--secret-endpoint-url\",\n                        help=\"Endpoint Url\",\n                        type=str,\n                        default=os.environ.get(\"DOCKERHUB_SECRET_ENDPOINT_URL\", None))\n\n    parser.add_argument(\"--secret-endpoint-region\",\n                        help=\"AWS Region\",\n                        type=str,\n                        default=os.environ.get(\"DOCKERHUB_SECRET_ENDPOINT_REGION\", None))\n\n    args = parser.parse_args(args=command_line_arguments)\n\n    if args.secret_endpoint_url is None:\n        raise RuntimeError(\"Could not determine secret-endpoint-url, please specify with --secret-endpoint-url\")\n\n    if args.secret_endpoint_region is None:\n        raise RuntimeError(\"Could not determine secret-endpoint-region, please specify with --secret-endpoint-region\")\n\n    try:\n        login_dockerhub(args.secret_name, args.secret_endpoint_url, args.secret_endpoint_region)\n    except Exception as err:\n        logging.exception(err)\n        exit(1)\n\n\nif __name__ == '__main__':\n    main(sys.argv[1:])\n"
  },
  {
    "path": "ci/jenkins/Jenkins_steps.groovy",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// This file contains the steps that will be used in the\n// Jenkins pipelines\n\nutils = load('ci/Jenkinsfile_utils.groovy')\n\n// mxnet libraries\nmx_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, example/extensions/lib_external_ops/build/libexternal_lib.so'\nmx_lib_cython = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, example/extensions/lib_external_ops/build/libexternal_lib.so, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so'\n\n// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.\nmx_cmake_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/tests/mxnet_unit_tests'\nmx_cmake_lib_no_tvm_op = 'build/libmxnet.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, example/extensions/lib_external_ops/build/libexternal_lib.so, build/tests/mxnet_unit_tests'\nmx_cmake_lib_cython = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/tests/mxnet_unit_tests, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so'\n// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.\nmx_cmake_lib_debug = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/tests/mxnet_unit_tests'\nmx_onednn_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, example/extensions/lib_external_ops/build/libexternal_lib.so'\nmx_tensorrt_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'\nmx_lib_cpp_examples = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, example/extensions/lib_external_ops/build/libexternal_lib.so, build/cpp-package/example/*, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so'\nmx_lib_cpp_examples_no_tvm_op = 'build/libmxnet.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so'\nmx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/cpp-package/example/*'\nmx_cd_lib = 'lib/libmxnet.so, licenses/*, lib/libgfortran.so.*, lib/libopenblas.so.0, include/onednn/oneapi/dnnl/dnnl_version.h, include/onednn/oneapi/dnnl/dnnl_config.h'\n\n\n// Python unittest for CPU\n// Python 3\ndef python3_ut(docker_container_name) {\n  timeout(time: max_time, unit: 'MINUTES') {\n    utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu', false)\n  }\n}\n\ndef python3_ut_onnx(docker_container_name) {\n  timeout(time: max_time, unit: 'MINUTES') {\n    utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu_onnx', false)\n  }\n}\n\ndef python3_ut_onednn(docker_container_name) {\n  timeout(time: max_time, unit: 'MINUTES') {\n    utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu_onednn', false)\n  }\n}\n\ndef python3_ut_array_api(docker_container_name) {\n  timeout(time: max_time, unit: 'MINUTES') {\n    utils.docker_run(docker_container_name, 'unittest_array_api_standardization', false)\n  }\n}\n\n// GPU test has two parts. 1) run unittest on GPU, 2) compare the results on\n// both CPU and GPU\n// Python 3\ndef python3_gpu_ut(docker_container_name) {\n  timeout(time: max_time, unit: 'MINUTES') {\n    utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu', true)\n  }\n}\n\n// Python 3 NOCUDNN\ndef python3_gpu_ut_nocudnn(docker_container_name) {\n  timeout(time: max_time, unit: 'MINUTES') {\n    utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu_nocudnn', true)\n  }\n}\n\ndef python3_gpu_ut_cython(docker_container_name) {\n  timeout(time: max_time, unit: 'MINUTES') {\n    utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu_cython', true)\n  }\n}\n\n//------------------------------------------------------------------------------------------\n\ndef compile_unix_cpu_openblas(lib_name) {\n    return ['CPU: Openblas': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-cpu-openblas') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false)\n            utils.pack_lib(lib_name, mx_lib_cpp_examples, true)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_openblas_debug_cpu(lib_name) {\n    return ['CPU: Openblas, cmake, debug': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-cpu-openblas') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_cmake_debug', false)\n            utils.pack_lib(lib_name, mx_cmake_lib_debug, true)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_openblas_cpu_no_tvm_op(lib_name) {\n    return ['CPU: Openblas, cmake, TVM_OP OFF': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-cpu-openblas-no-tvm-op') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_cmake_no_tvm_op', false)\n            utils.pack_lib(lib_name, mx_cmake_lib_no_tvm_op)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_int64_cpu(lib_name) {\n    return ['CPU: USE_INT64_TENSOR_SIZE': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-cpu-int64') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run(lib_name, 'build_ubuntu_cpu_large_tensor', false)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_int64_gpu(lib_name) {\n    return ['GPU: USE_INT64_TENSOR_SIZE': {\n      node(NODE_LINUX_GPU_G4) {\n        ws('workspace/build-gpu-int64') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_gpu_cu111', 'build_ubuntu_gpu_large_tensor', false)\n            utils.pack_lib(lib_name, mx_cmake_lib)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_mkl_cpu(lib_name) {\n    return ['CPU: MKL': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-cpu-mkl') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkl', false)\n            utils.pack_lib(lib_name, mx_lib, false)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_onednn_cpu(lib_name) {\n    return ['CPU: oneDNN': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-onednn-cpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_onednn', false)\n            utils.pack_lib(lib_name, mx_onednn_lib, true)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_onednn_mkl_cpu(lib_name) {\n    return ['CPU: oneDNN-MKL': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-onednn-cpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_onednn_mkl', false)\n            utils.pack_lib(lib_name, mx_onednn_lib, false)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_onednn_gpu(lib_name) {\n    return ['GPU: oneDNN': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-onednn-gpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_gpu_cu111', 'build_ubuntu_gpu_onednn', false)\n            utils.pack_lib(lib_name, mx_onednn_lib)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_onednn_nocudnn_gpu(lib_name) {\n    return ['GPU: oneDNN-CUDNNOFF': {\n       node(NODE_LINUX_CPU) {\n         ws('workspace/build-onednn-gpu-nocudnn') {\n           timeout(time: max_time, unit: 'MINUTES') {\n             utils.init_git()\n             utils.docker_run('ubuntu_gpu_cu111', 'build_ubuntu_gpu_onednn_nocudnn', false)\n             utils.pack_lib(lib_name, mx_onednn_lib)\n           }\n         }\n       }\n    }]\n}\n\ndef compile_unix_full_gpu(lib_name) {\n    return ['GPU: CUDA+cuDNN': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-gpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_gpu_cu111', 'build_ubuntu_gpu', false)\n            utils.pack_lib(lib_name, mx_lib_cpp_examples)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_full_gpu_debug(lib_name) {\n    return ['GPU: CUDA+cuDNN, debug': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-gpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_gpu_cu111', 'build_ubuntu_gpu_debug', false)\n            utils.pack_lib(lib_name, mx_lib_cpp_examples)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_tensorrt_gpu(lib_name) {\n    return ['TensorRT': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-tensorrt') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_tensorrt_cu114', 'build_ubuntu_gpu_tensorrt', false)\n            utils.pack_lib(lib_name, mx_tensorrt_lib)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_centos7_cpu(lib_name) {\n    return ['CPU: CentOS 7': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-centos7-cpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('centos7_cpu', 'build_centos7_cpu', false)\n            utils.pack_lib(lib_name, mx_lib)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_centos7_cpu_onednn() {\n    return ['CPU: CentOS 7 oneDNN': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-centos7-onednn') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('centos7_cpu', 'build_centos7_onednn', false)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_centos7_gpu(lib_name) {\n    return ['GPU: CentOS 7': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-centos7-gpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('centos7_gpu_cu102', 'build_centos7_gpu', false)\n            utils.pack_lib(lib_name, mx_lib)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_clang_6_cpu() {\n    return ['CPU: Clang 6': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-cpu-clang39') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang6', false)\n          }\n        }\n      }\n    }]\n}\n\n// TODO(leezu) delete once DUSE_DIST_KVSTORE=ON builds in -WError build\ndef compile_unix_clang_10_cpu() {\n    return ['CPU: Clang 10': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-cpu-clang100') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang100', false)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_clang_tidy_cpu() {\n    return ['CPU: Clang Tidy': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-cpu-clang60_tidy') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang_tidy', false)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_clang_6_onednn_cpu() {\n    return ['CPU: Clang 6 oneDNN': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-cpu-onednn-clang6') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang6_onednn', false)\n          }\n        }\n      }\n    }]\n}\n\n// TODO(leezu) delete once DUSE_DIST_KVSTORE=ON builds in -WError build\ndef compile_unix_clang_10_onednn_cpu() {\n    return ['CPU: Clang 10 oneDNN': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-cpu-onednn-clang100') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang100_onednn', false)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_armv8_jetson_gpu() {\n    return ['NVidia Jetson / ARMv8':{\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-jetson-armv8') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('jetson', 'build_jetson', false)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_armv6_cpu(lib_name) {\n    return ['ARMv6':{\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-ARMv6') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('armv6', 'build_armv6', false)\n            utils.pack_lib(lib_name, mx_lib)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_armv7_cpu(lib_name) {\n    return ['ARMv7':{\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-ARMv7') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('armv7', 'build_armv7', false)\n            utils.pack_lib(lib_name, mx_lib)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_armv8_cpu(lib_name) {\n    return ['ARMv8':{\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-ARMv8') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('armv8', 'build_armv8', false)\n            utils.pack_lib(lib_name, mx_lib)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_armv8_android_cpu() {\n    return ['Android / ARMv8':{\n      node(NODE_LINUX_CPU) {\n        ws('workspace/android64') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('android_armv8', 'build_android_armv8', false)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_armv7_android_cpu() {\n    return ['Android / ARMv7':{\n      node(NODE_LINUX_CPU) {\n        ws('workspace/androidv7') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('android_armv7', 'build_android_armv7', false)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_asan_cpu(lib_name) {\n    return ['CPU: ASAN': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-cpu-asan') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_cmake_asan', false)\n            utils.pack_lib(lib_name, mx_lib_cpp_examples_cpu)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_gcc8_werror(lib_name) {\n    return ['CPU: GCC8 -WError': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-cpu-gcc8') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_gcc8_werror', false)\n            utils.pack_lib(lib_name, mx_lib)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_clang10_werror(lib_name) {\n    return ['CPU: Clang10 -WError': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-cpu-clang10') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang10_werror', false)\n            utils.pack_lib(lib_name, mx_lib)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_unix_clang10_cuda_werror(lib_name) {\n    return ['GPU: Clang10 -WError': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-cpu-clang10') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_gpu_cu111', 'build_ubuntu_gpu_clang10_werror', false)\n            utils.pack_lib(lib_name, mx_lib)\n          }\n        }\n      }\n    }]\n}\n\ndef compile_windows_cpu(lib_name) {\n    return ['Build CPU windows':{\n      node(NODE_WINDOWS_CPU) {\n        ws('workspace/build-cpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git_win()\n            powershell 'py -3 ci/build_windows.py -f WIN_CPU --vcvars_ver 14.28'\n            stash includes: 'windows_package.7z', name: lib_name\n          }\n        }\n      }\n    }]\n}\n\ndef compile_windows_cpu_onednn(lib_name) {\n    return ['Build CPU oneDNN windows':{\n      node(NODE_WINDOWS_CPU) {\n        ws('workspace/build-cpu-onednn') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git_win()\n            powershell 'py -3 ci/build_windows.py -f WIN_CPU_ONEDNN --vcvars_ver 14.28'\n            stash includes: 'windows_package.7z', name: lib_name\n          }\n        }\n      }\n    }]\n}\n\ndef compile_windows_cpu_onednn_mkl(lib_name) {\n    return ['Build CPU oneDNN MKL windows':{\n      node(NODE_WINDOWS_CPU) {\n        ws('workspace/build-cpu-onednn-mkl') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git_win()\n            powershell 'py -3 ci/build_windows.py -f WIN_CPU_ONEDNN_MKL --vcvars_ver 14.28'\n            stash includes: 'windows_package.7z', name: lib_name\n          }\n        }\n      }\n    }]\n}\n\ndef compile_windows_cpu_mkl(lib_name) {\n    return ['Build CPU MKL windows':{\n      node(NODE_WINDOWS_CPU) {\n        ws('workspace/build-cpu-mkl') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git_win()\n            powershell 'py -3 ci/build_windows.py -f WIN_CPU_MKL --vcvars_ver 14.28'\n            stash includes: 'windows_package.7z', name: lib_name\n          }\n        }\n      }\n    }]\n}\n\ndef compile_windows_gpu(lib_name) {\n    return ['Build GPU windows':{\n      node(NODE_WINDOWS_CPU) {\n        ws('workspace/build-gpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n              utils.init_git_win()\n              powershell 'py -3 ci/build_windows.py -f WIN_GPU --vcvars_ver 14.28'\n              stash includes: 'windows_package.7z', name: lib_name\n          }\n        }\n      }\n    }]\n}\n\ndef compile_windows_gpu_onednn(lib_name) {\n    return ['Build GPU oneDNN windows':{\n      node(NODE_WINDOWS_CPU) {\n        ws('workspace/build-gpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git_win()\n            powershell 'py -3 ci/build_windows.py -f WIN_GPU_ONEDNN --vcvars_ver 14.28'\n            stash includes: 'windows_package.7z', name: lib_name\n          }\n        }\n      }\n    }]\n}\n\ndef compile_static_python_cpu() {\n  return ['Static build CPU CentOS7 Python' : {\n    node(NODE_LINUX_CPU) {\n        ws('workspace/ut-publish-python-cpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('centos7_cpu', 'build_static_python_cpu', false)\n          }\n        }\n    }\n  }]\n}\n\ndef compile_static_cd_cpu(lib_name) {\n  return ['CPU: CD Static Build' : {\n    node(NODE_LINUX_CPU) {\n        ws('workspace/build-cd-static/cpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('centos7_cpu', 'build_static_libmxnet cpu', false)\n            utils.pack_lib(lib_name, mx_cd_lib)\n          }\n        }\n    }\n  }]\n}\n\ndef compile_static_python_gpu() {\n  return ['Static build GPU CentOS7 Python' : {\n    node(NODE_LINUX_GPU) {\n        ws('workspace/ut-publish-python-gpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('centos7_gpu_cu102', 'build_static_python_cu102')\n          }\n        }\n    }\n  }]\n}\n\ndef compile_static_cd_gpu(lib_name) {\n  return ['GPU: CD Static Build' : {\n    node(NODE_LINUX_CPU) {\n        ws('workspace/build-cd-static/gpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('centos7_gpu_cu102', 'build_static_libmxnet cu102', false)\n            utils.pack_lib(lib_name, mx_cd_lib)\n          }\n        }\n    }\n  }]\n}\n\ndef test_unix_python3_cpu(lib_name) {\n    return ['Python3: CPU': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/ut-python3-cpu') {\n          try {\n            utils.unpack_and_init(lib_name, mx_lib, true)\n            python3_ut('ubuntu_cpu')\n            utils.publish_test_coverage()\n          } finally {\n            utils.collect_test_results_unix('tests_unittest.xml', 'tests_python3_cpu_unittest.xml')\n            utils.collect_test_results_unix('tests_quantization.xml', 'tests_python3_cpu_quantization.xml')\n          }\n        }\n      }\n    }]\n}\n\ndef test_unix_python3_array_api(lib_name) {\n    return ['Python3: Array-API': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/ut-python3-cpu') {\n          utils.unpack_and_init(lib_name, mx_lib, false)\n          python3_ut_array_api('ubuntu_cpu')\n          utils.publish_test_coverage()\n        }\n      }\n    }]\n}\n\ndef test_unix_python3_mkl_cpu(lib_name) {\n    return ['Python3: MKL-CPU': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/ut-python3-cpu') {\n          try {\n            utils.unpack_and_init(lib_name, mx_lib)\n            python3_ut('ubuntu_cpu')\n            utils.publish_test_coverage()\n          } finally {\n            utils.collect_test_results_unix('tests_unittest.xml', 'tests_python3_cpu_unittest.xml')\n            utils.collect_test_results_unix('tests_quantization.xml', 'tests_python3_cpu_quantization.xml')\n          }\n        }\n      }\n    }]\n}\n\ndef test_unix_python3_gpu(lib_name) {\n    return ['Python3: GPU': {\n      node(NODE_LINUX_GPU_G4) {\n        ws('workspace/ut-python3-gpu') {\n          try {\n            utils.unpack_and_init(lib_name, mx_lib_cython)\n            python3_gpu_ut_cython('ubuntu_gpu_cu111')\n            utils.publish_test_coverage()\n          } finally {\n            utils.collect_test_results_unix('tests_gpu.xml', 'tests_python3_gpu.xml')\n          }\n        }\n      }\n    }]\n}\n\ndef test_unix_python3_ampere_gpu(lib_name) {\n    return ['Python3: Ampere-GPU': {\n      node(NODE_LINUX_GPU_G5) {\n        ws('workspace/ut-python3-gpu') {\n          try {\n            utils.unpack_and_init(lib_name, mx_lib_cython)\n            python3_gpu_ut_cython('ubuntu_gpu_cu111')\n            utils.publish_test_coverage()\n          } finally {\n            utils.collect_test_results_unix('tests_gpu.xml', 'tests_python3_ampere_gpu.xml')\n          }\n        }\n      }\n    }]\n}\n\ndef test_unix_python3_debug_cpu() {\n    return ['Python3: CPU debug': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/ut-python3-cpu-debug') {\n          try {\n            utils.unpack_and_init('cpu_debug', mx_cmake_lib_debug, true)\n            python3_ut('ubuntu_cpu')\n          } finally {\n            utils.collect_test_results_unix('tests_unittest.xml', 'tests_python3_cpu_debug_unittest.xml')\n            utils.collect_test_results_unix('tests_quantization.xml', 'tests_python3_cpu_debug_quantization.xml')\n          }\n        }\n      }\n    }]\n}\n\ndef test_unix_python3_cpu_no_tvm_op(lib_name) {\n    return ['Python3: CPU TVM_OP OFF': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/ut-python3-cpu-no-tvm-op') {\n          try {\n            utils.unpack_and_init(lib_name, mx_cmake_lib_no_tvm_op)\n            python3_ut('ubuntu_cpu')\n          } finally {\n            utils.collect_test_results_unix('tests_unittest.xml', 'tests_python3_cpu_no_tvm_op_unittest.xml')\n            utils.collect_test_results_unix('tests_quantization.xml', 'tests_python3_cpu_no_tvm_op_quantization.xml')\n          }\n        }\n      }\n    }]\n}\n\ndef test_unix_python3_onnx_cpu(lib_name) {\n    return ['Python3: ONNX-CPU': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/ut-python3-onnx-cpu') {\n          try {\n            utils.unpack_and_init(lib_name, mx_lib, true)\n            python3_ut_onnx('ubuntu_cpu')\n            utils.publish_test_coverage()\n          } finally {\n            utils.collect_test_results_unix('onnx_unittest.xml', 'tests_python3_onnx_cpu_unittest.xml')\n          }\n        }\n      }\n    }]\n}\n\ndef test_unix_python3_onednn_cpu(lib_name) {\n    return ['Python3: oneDNN-CPU': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/ut-python3-onednn-cpu') {\n          try {\n            utils.unpack_and_init(lib_name, mx_onednn_lib, true)\n            python3_ut_onednn('ubuntu_cpu')\n            utils.publish_test_coverage()\n          } finally {\n            utils.collect_test_results_unix('tests_unittest.xml', 'tests_python3_onednn_cpu_unittest.xml')\n            utils.collect_test_results_unix('tests_mkl.xml', 'tests_python3_onednn_cpu_mkl.xml')\n          }\n        }\n      }\n    }]\n}\n\ndef test_unix_python3_onednn_mkl_cpu(lib_name) {\n    return ['Python3: oneDNN-MKL-CPU': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/ut-python3-onednn-mkl-cpu') {\n          try {\n            utils.unpack_and_init(lib_name, mx_lib)\n            python3_ut_onednn('ubuntu_cpu')\n            utils.publish_test_coverage()\n          } finally {\n            utils.collect_test_results_unix('tests_unittest.xml', 'tests_python3_onednn_cpu_unittest.xml')\n            utils.collect_test_results_unix('tests_mkl.xml', 'tests_python3_onednn_cpu_mkl.xml')\n          }\n        }\n      }\n    }]\n}\n\ndef test_unix_python3_onednn_gpu(lib_name) {\n    return ['Python3: oneDNN-GPU': {\n      node(NODE_LINUX_GPU_G4) {\n        ws('workspace/ut-python3-onednn-gpu') {\n          try {\n            utils.unpack_and_init(lib_name, mx_onednn_lib)\n            python3_gpu_ut('ubuntu_gpu_cu111')\n            utils.publish_test_coverage()\n          } finally {\n            utils.collect_test_results_unix('tests_gpu.xml', 'tests_python3_onednn_gpu.xml')\n          }\n        }\n      }\n    }]\n}\n\ndef test_unix_python3_onednn_nocudnn_gpu(lib_name) {\n    return ['Python3: oneDNN-GPU-NOCUDNN': {\n      node(NODE_LINUX_GPU_G4) {\n        ws('workspace/ut-python3-onednn-gpu-nocudnn') {\n          try {\n            utils.unpack_and_init(lib_name, mx_onednn_lib)\n            python3_gpu_ut_nocudnn('ubuntu_gpu_cu111')\n            utils.publish_test_coverage()\n          } finally {\n            utils.collect_test_results_unix('tests_gpu.xml', 'tests_python3_onednn_gpu_nocudnn.xml')\n          }\n        }\n      }\n    }]\n}\n\ndef test_unix_onnx_cpu(lib_name) {\n    return ['Onnx: CPU Makefile': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/it-onnx-cpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.unpack_and_init(lib_name, mx_lib)\n            utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx', false)\n            utils.publish_test_coverage()\n          }\n        }\n      }\n    }]\n}\n\ndef test_unix_distributed_kvstore_cpu(lib_name) {\n    return ['dist-kvstore tests CPU': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/it-dist-kvstore') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.unpack_and_init(lib_name, mx_lib, true)\n            utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_dist_kvstore', false)\n            utils.publish_test_coverage()\n          }\n        }\n      }\n    }]\n}\n\ndef test_unix_byteps_gpu(lib_name) {\n    return ['byteps tests GPU': {\n      node(NODE_LINUX_GPU_G4) {\n        ws('workspace/it-byteps') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.unpack_and_init(lib_name, mx_lib)\n            utils.docker_run('ubuntu_gpu_cu111', 'integrationtest_ubuntu_gpu_byteps', true, '32768m')\n            utils.publish_test_coverage()\n          }\n        }\n      }\n    }]\n}\n\ndef test_unix_distributed_kvstore_gpu(lib_name) {\n    return ['dist-kvstore tests GPU': {\n      node(NODE_LINUX_GPU_G4) {\n        ws('workspace/it-dist-kvstore') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.unpack_and_init(lib_name, mx_lib)\n            utils.docker_run('ubuntu_gpu_cu111', 'integrationtest_ubuntu_gpu_dist_kvstore', true)\n            utils.publish_test_coverage()\n          }\n        }\n      }\n    }]\n}\n\ndef test_unix_cpp_package_gpu(lib_name) {\n    return ['cpp-package GPU Makefile': {\n      node(NODE_LINUX_GPU_G4) {\n        ws('workspace/it-cpp-package-gpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.unpack_and_init(lib_name, mx_lib_cpp_examples)\n            utils.docker_run('ubuntu_gpu_cu111', 'integrationtest_ubuntu_cpp_package_gpu', true)\n            utils.publish_test_coverage()\n          }\n        }\n      }\n    }]\n}\n\ndef test_unix_python3_data_interchange_gpu(lib_name) {\n    return ['Data Interchange': {\n      node(NODE_LINUX_GPU_G4) {\n        ws('workspace/it-data-interchange') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.unpack_and_init(lib_name, mx_lib)\n            utils.docker_run('ubuntu_gpu_cu111', 'test_python3_data_interchange_gpu', true)\n            utils.publish_test_coverage()\n          }\n        }\n      }\n    }]\n}\n\ndef test_centos7_python3_cpu(lib_name) {\n    return ['Python3: CentOS 7 CPU': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/build-centos7-cpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            try {\n              utils.unpack_and_init(lib_name, mx_lib)\n              utils.docker_run('centos7_cpu', 'unittest_centos7_cpu', false)\n              utils.publish_test_coverage()\n            } finally {\n              utils.collect_test_results_unix('tests_unittest.xml', 'tests_python3_centos7_cpu_unittest.xml')\n              utils.collect_test_results_unix('tests_train.xml', 'tests_python3_centos7_cpu_train.xml')\n            }\n          }\n        }\n      }\n    }]\n}\n\ndef test_centos7_python3_cd_cpu(lib_name) {\n    return ['Python3: CentOS 7 CPU CD': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/test-cd-static/cpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.unpack_and_init(lib_name, mx_cd_lib)\n            utils.docker_run('centos7_cpu', 'cd_unittest_ubuntu cpu', false)\n          }\n        }\n      }\n    }]\n}\n\ndef test_centos7_pypi_package_cd_cpu(lib_name) {\n    return ['PyPI package: CentOS 7 CPU CD': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/test-cd-pypi/cpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.unpack_and_init(lib_name, mx_cd_lib)\n            utils.docker_run('centos7_cpu', 'ci_package_pypi cpu', false)\n          }\n        }\n      }\n    }]\n}\n\ndef test_centos7_python3_gpu(lib_name) {\n    return ['Python3: CentOS 7 GPU': {\n      node(NODE_LINUX_GPU) {\n        ws('workspace/build-centos7-gpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            try {\n              utils.unpack_and_init(lib_name, mx_lib)\n              utils.docker_run('centos7_gpu_cu102', 'unittest_centos7_gpu', true)\n              utils.publish_test_coverage()\n            } finally {\n              utils.collect_test_results_unix('tests_gpu.xml', 'tests_python3_centos7_gpu.xml')\n            }\n          }\n        }\n      }\n    }]\n}\n\ndef test_centos7_python3_cd_gpu(lib_name) {\n    return ['Python3: CentOS 7 GPU CD': {\n      node(NODE_LINUX_GPU) {\n        ws('workspace/test-cd-static/gpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.unpack_and_init(lib_name, mx_cd_lib)\n            utils.docker_run('centos7_gpu_cu102', 'cd_unittest_ubuntu cu102', true)\n          }\n        }\n      }\n    }]\n}\n\ndef test_centos7_pypi_package_cd_gpu(lib_name) {\n    return ['PyPI package: CentOS 7 GPU CD': {\n      node(NODE_LINUX_GPU) {\n        ws('workspace/test-cd-pypi/gpu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.unpack_and_init(lib_name, mx_cd_lib)\n            utils.docker_run('centos7_gpu_cu102', 'ci_package_pypi cu102', true)\n          }\n        }\n      }\n    }]\n}\n\ndef test_windows_python3_gpu(lib_name) {\n    return ['Python 3: GPU Win':{\n      node(NODE_WINDOWS_GPU) {\n        timeout(time: max_time, unit: 'MINUTES') {\n          ws('workspace/ut-python-gpu') {\n            try {\n              utils.init_git_win()\n              unstash lib_name\n              powershell 'ci/windows/test_py3_gpu.ps1'\n            } finally {\n              utils.collect_test_results_windows('tests_forward.xml', 'tests_gpu_forward_windows_python3_gpu.xml')\n              utils.collect_test_results_windows('tests_operator.xml', 'tests_gpu_operator_windows_python3_gpu.xml')\n            }\n          }\n        }\n      }\n    }]\n}\n\ndef test_windows_python3_gpu_onednn(lib_name) {\n    return ['Python 3: oneDNN-GPU Win':{\n      node(NODE_WINDOWS_GPU) {\n        timeout(time: max_time, unit: 'MINUTES') {\n          ws('workspace/ut-python-gpu') {\n            try {\n              utils.init_git_win()\n              unstash lib_name\n              powershell 'ci/windows/test_py3_gpu.ps1'\n            } finally {\n              utils.collect_test_results_windows('tests_forward.xml', 'tests_gpu_forward_windows_python3_gpu_onednn.xml')\n              utils.collect_test_results_windows('tests_operator.xml', 'tests_gpu_operator_windows_python3_gpu_onednn.xml')\n            }\n          }\n        }\n      }\n    }]\n}\n\ndef test_windows_python3_cpu(lib_name) {\n    return ['Python 3: CPU Win': {\n      node(NODE_WINDOWS_CPU) {\n        timeout(time: max_time, unit: 'MINUTES') {\n          ws('workspace/ut-python-cpu') {\n            try {\n              utils.init_git_win()\n              unstash lib_name\n              powershell 'ci/windows/test_py3_cpu.ps1'\n            } finally {\n              utils.collect_test_results_windows('tests_unittest.xml', 'tests_unittest_windows_python3_cpu.xml')\n            }\n          }\n        }\n      }\n    }]\n}\n\ndef test_qemu_armv7_cpu(lib_name) {\n    return ['ARMv7 QEMU': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/ut-armv7-qemu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.unpack_and_init(lib_name, mx_lib)\n            utils.docker_run('test.armv7', 'unittest_ubuntu_python3_arm', false)\n          }\n        }\n      }\n    }]\n}\n\ndef test_qemu_armv8_cpu(lib_name) {\n    return ['ARMv8 QEMU': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/ut-armv8-qemu') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.unpack_and_init(lib_name, mx_lib)\n            utils.docker_run('test.armv8', 'unittest_ubuntu_python3_arm', false)\n          }\n        }\n      }\n    }]\n}\n\ndef should_pack_website() {\n  if (env.BRANCH_NAME) {\n    if (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"new_\")) {\n      return true\n    }\n  } else {\n    return true\n  }\n  return false\n}\n\n// Each of the docs_{lang} functions will build the docs...\n// Stashing is only needed for master for website publishing or for testing \"new_\"\n\n// Call this function from Jenkins to generate just the Python API microsite artifacts.\ndef docs_python(lib_name) {\n    return ['Python Docs': {\n      node(NODE_LINUX_GPU) {\n        ws('workspace/docs') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.unpack_and_init(lib_name, mx_lib_cython)\n            utils.docker_run('ubuntu_gpu_cu111', 'build_python_docs', true)\n            if (should_pack_website()) {\n              utils.pack_lib('python-artifacts', 'docs/_build/python-artifacts.tgz', false)\n            }\n          }\n        }\n      }\n    }]\n}\n\n// Call this function from Jenkins to generate just the C and C++ API microsite artifacts.\ndef docs_c(lib_name) {\n    return ['C Docs': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/docs') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.unpack_and_init(lib_name, mx_lib, false)\n            utils.docker_run('ubuntu_cpu', 'build_c_docs', false)\n            if (should_pack_website()) {\n              utils.pack_lib('c-artifacts', 'docs/_build/c-artifacts.tgz', false)\n            }\n          }\n        }\n      }\n    }]\n}\n\n\n// Call this function from Jenkins to generate just the main website artifacts.\ndef docs_jekyll() {\n    return ['Main Jekyll Website': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/docs') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n            utils.docker_run('ubuntu_cpu_jekyll', 'build_jekyll_docs', false)\n            if (should_pack_website()) {\n              utils.pack_lib('jekyll-artifacts', 'docs/_build/jekyll-artifacts.tgz', false)\n            }\n          }\n        }\n      }\n    }]\n}\n\n\n// This is for publishing the full website\n// Assumes you have run all of the docs generation functions\n// Called from Jenkins_website_full and Jenkins_website_full_pr\ndef docs_prepare() {\n    return ['Prepare for publication of the full website': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/docs') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n\n            unstash 'jekyll-artifacts'\n            unstash 'c-artifacts'\n            unstash 'python-artifacts'\n\n            utils.docker_run('ubuntu_cpu_jekyll', 'build_docs', false)\n\n            // only stash if we're going to unstash later\n            // utils.pack_lib('full_website', 'docs/_build/full_website.tgz', false)\n\n            // archive so the publish pipeline can access the artifact\n            archiveArtifacts 'docs/_build/full_website.tgz'\n          }\n        }\n      }\n    }]\n}\n\n// This is for updateing the new version of website artifact\n// Assumes you have run all of the docs generation functions\n// Called from Jenkins_website_version_artifacts\ndef docs_full_website() {\n    return ['Build artifacts full_website.tgz': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/docs') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n\n            unstash 'jekyll-artifacts'\n            unstash 'c-artifacts'\n            unstash 'python-artifacts'\n\n            utils.docker_run('ubuntu_cpu_jekyll', 'build_docs', false)\n\n            utils.pack_lib('full_website', 'docs/_build/full_website.tgz', false)\n\n            // archive so the publish pipeline can access the artifact\n            archiveArtifacts 'docs/_build/full_website.tgz'\n          }\n        }\n      }\n    }]\n}\n\ndef docs_prepare_beta() {\n    return ['Prepare for publication to the staging website': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/docs') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            utils.init_git()\n\n            unstash 'jekyll-artifacts'\n            unstash 'c-artifacts'\n            unstash 'python-artifacts'\n\n            utils.docker_run('ubuntu_cpu_jekyll', 'build_docs_beta', false)\n\n            // archive so the publish pipeline can access the artifact\n            archiveArtifacts 'docs/_build/beta_website.tgz'\n          }\n        }\n      }\n    }]\n}\n\n\ndef docs_archive() {\n    return ['Archive the full website': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/docs') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            archiveArtifacts 'docs/_build/full_website.tgz'\n          }\n        }\n      }\n    }]\n}\n\n\n// This is for the full website\ndef docs_publish() {\n    return ['Publish the full website': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/docs') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            // If used stashed files, you can retrieve them here\n            //unstash 'full_website'\n            //sh 'tar -xzf docs/_build/full_website.tgz --directory .'\n            try {\n              build 'restricted-website-publish-master'\n            }\n            catch (Exception e) {\n               println(e.getMessage())\n            }\n          }\n        }\n      }\n    }]\n}\n\n\n// This is for the beta website\ndef docs_publish_beta() {\n    return ['Publish the beta website to staging': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/docs') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            try {\n              build 'restricted-website-publish-master-beta'\n            }\n            catch (Exception e) {\n               println(e.getMessage())\n            }\n          }\n        }\n      }\n    }]\n}\n\n// This is for uploading website artifacts to S3 bucket\n// Assumes you have run docs_full_website function\ndef docs_upload_s3() {\n    return ['Upload artifacts to s3 bucket': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/docs') {\n          timeout(time: max_time, unit: 'MINUTES') {\n            if(env.FOLDER_NAME) {\n              utils.unpack_and_init('full_website', 'docs/_build/full_website.tgz')\n\n              utils.docker_run('ubuntu_cpu', \"push_docs ${env.FOLDER_NAME}\", false)\n\n              archiveArtifacts 'docs/_build/versions.zip'\n            } else {\n              sh 'echo Can not find website version for release. Please specify env var FOLDER_NAME in Jenkins pipeline'\n              sh 'exit 1'\n            }\n          }\n        }\n      }\n    }]\n}\n\ndef sanity_lint() {\n    return ['Lint': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/sanity-lint') {\n          utils.init_git()\n          utils.docker_run('ubuntu_cpu', 'sanity_check', false)\n        }\n      }\n    }]\n}\n\ndef sanity_rat_license() {\n    return ['RAT License': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/sanity-rat') {\n          utils.init_git()\n          utils.docker_run('ubuntu_cpu', 'test_rat_check', false)\n        }\n      }\n    }]\n}\n\ndef test_artifact_repository() {\n    return ['Test Artifact Repository Client': {\n      node(NODE_LINUX_CPU) {\n        ws('workspace/artifact-repo-client') {\n          utils.init_git()\n          utils.docker_run('ubuntu_cpu', 'test_artifact_repository', false)\n        }\n      }\n    }]\n}\n\ndef misc_test_docker_cache_build() {\n  return ['Test Docker cache build': {\n    node(NODE_LINUX_CPU) {\n      ws('workspace/docker_cache') {\n        utils.init_git()\n        sh \"cd ci && docker-compose -f docker/docker-compose.yml pull && docker-compose -f docker/docker-compose.yml build --parallel\"\n      }\n    }\n  }]\n}\n\nreturn this\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_centos_cpu",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 240\n\nnode('utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\nutils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Build', [\n    custom_steps.compile_centos7_cpu('centos7_cpu'),\n    custom_steps.compile_centos7_cpu_onednn(),\n    custom_steps.compile_static_python_cpu(),\n    custom_steps.compile_static_cd_cpu('centos7_cpu_cd')\n  ])\n\n  utils.parallel_stage('Tests', [\n    custom_steps.test_centos7_python3_cpu('centos7_cpu'),\n    custom_steps.test_centos7_python3_cd_cpu('centos7_cpu_cd'),\n    custom_steps.test_centos7_pypi_package_cd_cpu('centos7_cpu_cd')\n  ])\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_centos_gpu",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 240\n\nnode('utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\nutils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', linux_gpu_g4: 'mxnetlinux-gpu-g4')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Build', [\n    custom_steps.compile_centos7_gpu('centos7_gpu'),\n    custom_steps.compile_static_python_gpu(),\n    custom_steps.compile_static_cd_gpu('centos7_gpu_cd')\n  ])\n\n  utils.parallel_stage('Tests', [\n    custom_steps.test_centos7_python3_gpu('centos7_gpu'),\n    custom_steps.test_centos7_python3_cd_gpu('centos7_gpu_cd'),\n    custom_steps.test_centos7_pypi_package_cd_gpu('centos7_gpu_cd')\n  ])\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_clang",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 240\n\nnode('utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\nutils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Build', [\n    custom_steps.compile_unix_clang_6_cpu(),\n    custom_steps.compile_unix_clang_10_cpu(),\n    custom_steps.compile_unix_clang_tidy_cpu(),\n    custom_steps.compile_unix_clang_6_onednn_cpu(),\n    custom_steps.compile_unix_clang_10_onednn_cpu()\n  ]) \n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_edge",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 240\n\nnode('utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\nutils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Build', [\n    custom_steps.compile_armv8_jetson_gpu(),\n    custom_steps.compile_armv7_cpu('armv7'),\n    custom_steps.compile_armv6_cpu('armv6'),\n    custom_steps.compile_armv8_cpu('armv8'),\n    custom_steps.compile_armv8_android_cpu(),\n    custom_steps.compile_armv7_android_cpu()\n  ])\n\n  utils.parallel_stage('Tests', [\n    custom_steps.test_qemu_armv7_cpu('armv7'),\n    custom_steps.test_qemu_armv8_cpu('armv8')\n  ])\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_full",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\ndef max_time = 30\n\ndef buildJobs = [\n    'centos-cpu',\n    'centos-gpu',\n    'clang',\n    'edge',\n    'miscellaneous',\n    'unix-cpu',\n    'unix-gpu',\n    'website',\n    'windows-cpu',\n    'windows-gpu'\n]\n\n\nstage(\"full-build\") {\n    timeout(time: max_time, unit: 'MINUTES') {\n        // get the base path by removing build and branch portions\n        def jobPath = JOB_NAME.split('/')\n        def pipelineName = jobPath[0..jobPath.size()-3].join('/')\n        def sanityDone = false\n        while (!sanityDone) {\n            try {\n                println(\"Attempting to run sanity build...\")\n                build job: pipelineName + \"/sanity/\" + BRANCH_NAME, wait: true\n                sanityDone = true\n            } catch (hudson.AbortException e) {\n                println(\"Job doesn't yet exist, waiting for Jenkins to find job..\")\n                sleep(5)\n            }\n        }\n        buildJobs.each { subJob ->\n            build job: pipelineName + \"/\" + subJob + \"/\" + BRANCH_NAME, wait: false\n        }\n    }\n}\n\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_miscellaneous",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 240\n\n\nnode('utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\nutils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Build', [\n    custom_steps.compile_unix_asan_cpu('cpu_asan'),\n    custom_steps.compile_unix_gcc8_werror('cpu_gcc8'),\n    custom_steps.compile_unix_clang10_werror('cpu_clang10'),\n    custom_steps.compile_unix_clang10_cuda_werror('gpu_clang10')\n  ])\n\n  utils.parallel_stage('Tests', [\n    custom_steps.misc_test_docker_cache_build()\n  ])\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_sanity",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 240\n\nnode('utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\nutils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Sanity Check', [\n    custom_steps.sanity_lint(),\n    custom_steps.sanity_rat_license()\n  ])\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_tools",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// A place to add tests scripts for supporting tools\n\n// timeout in minutes\nmax_time = 240\n\nnode('utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\nutils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Tooling Tests', [\n    custom_steps.test_artifact_repository(),\n  ])\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_unix_cpu",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 240\n\nnode('utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\nutils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Build', [\n    custom_steps.compile_unix_cpu_openblas('cpu'),\n    custom_steps.compile_unix_openblas_debug_cpu('cpu_debug'),\n    custom_steps.compile_unix_mkl_cpu('cpu_mkl'),\n    custom_steps.compile_unix_onednn_cpu('onednn_cpu'),\n    custom_steps.compile_unix_onednn_mkl_cpu('onednn_mkl_cpu'),\n    custom_steps.compile_unix_int64_cpu('ubuntu_cpu'),\n    custom_steps.compile_unix_openblas_cpu_no_tvm_op('cpu_openblas_no_tvm_op'),\n  ])\n\n  utils.parallel_stage('Tests', [\n    custom_steps.test_unix_python3_cpu('cpu'),\n    custom_steps.test_unix_python3_onnx_cpu('cpu'),\n    // TVMOP has issue with NAN, see https://github.com/apache/incubator-mxnet/issues/20729\n    custom_steps.test_unix_python3_array_api('cpu_openblas_no_tvm_op'),\n    custom_steps.test_unix_python3_mkl_cpu('cpu_mkl'),\n    custom_steps.test_unix_python3_onednn_cpu('onednn_cpu'),\n    custom_steps.test_unix_python3_onednn_mkl_cpu('onednn_mkl_cpu'),\n    /* disable onnx tests for now, until onnx work is forwarded-ported to master\n    custom_steps.test_unix_onnx_cpu('cpu'),\n    */\n    /*  Disabled due to master build failure:\n     *  http://jenkins.mxnet-ci.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/\n     *  https://github.com/apache/incubator-mxnet/issues/11801\n    custom_steps.test_unix_distributed_kvstore_cpu('cpu')\n    */\n    custom_steps.test_unix_python3_cpu_no_tvm_op('cpu_openblas_no_tvm_op'),\n  ])\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_unix_gpu",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 240\n\nnode('utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\nutils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', linux_gpu_g4: 'mxnetlinux-gpu-g4', linux_gpu_g5: 'mxnetlinux-gpu-g5')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Build', [\n    custom_steps.compile_unix_onednn_gpu('onednn_gpu'),\n    custom_steps.compile_unix_onednn_nocudnn_gpu('onednn_gpu_nocudnn'),\n    custom_steps.compile_unix_full_gpu('gpu'),\n    custom_steps.compile_unix_full_gpu_debug('gpu_debug'),\n    custom_steps.compile_unix_tensorrt_gpu('tensorrt'),\n    custom_steps.compile_unix_int64_gpu('gpu_int64'),\n  ])\n\n  utils.parallel_stage('Tests', [\n    custom_steps.test_unix_python3_gpu('gpu'),\n    custom_steps.test_unix_python3_ampere_gpu('gpu'),\n    custom_steps.test_unix_python3_onednn_gpu('onednn_gpu'),\n    custom_steps.test_unix_python3_onednn_nocudnn_gpu('onednn_gpu_nocudnn'),\n    custom_steps.test_unix_cpp_package_gpu('gpu'),\n    custom_steps.test_unix_python3_data_interchange_gpu('gpu'),\n    // TODO(szha): fix and reenable the hanging issue. tracked in #18098\n    // custom_steps.test_unix_distributed_kvstore_gpu('gpu'),\n    // TODO(spanev): reenable when byteps is updated with the new dep engine API\n    // custom_steps.test_unix_byteps_gpu('gpu'),\n  ]) \n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_website_beta",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n// This pipeline will publish to https://mxnet-beta.staged.apache.org/\n\n// timeout in minutes\nmax_time = 240\n\nnode('restricted-utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\n\nutils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu-g4', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Build', [\n    custom_steps.compile_unix_cpu_openblas('libmxnet'),\n    custom_steps.compile_unix_full_gpu('libmxnet_gpu')\n  ])\n\n  utils.parallel_stage('Build Docs', [\n    // Only building a subset of the docs for previewing on staging\n    custom_steps.docs_jekyll(),\n    custom_steps.docs_c('libmxnet'),\n    custom_steps.docs_python('libmxnet_gpu')\n  ])\n\n  utils.parallel_stage('Prepare', [\n    custom_steps.docs_prepare_beta()\n  ])\n\n  utils.parallel_stage('Publish', [\n    custom_steps.docs_publish_beta()\n  ])\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_website_full",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 240\n\nnode('restricted-utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\n\nutils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu-g4', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Build', [\n    custom_steps.compile_unix_cpu_openblas('libmxnet'),\n    custom_steps.compile_unix_full_gpu('libmxnet_gpu')\n  ])\n\n  utils.parallel_stage('Build Docs', [\n    custom_steps.docs_jekyll(),\n    custom_steps.docs_c('libmxnet'),\n    custom_steps.docs_python('libmxnet_gpu'),\n  ])\n\n  utils.parallel_stage('Prepare', [\n    custom_steps.docs_prepare()\n  ])\n\n  utils.parallel_stage('Publish', [\n    custom_steps.docs_publish()\n  ])\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_website_full_pr",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 240\n\nnode('utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\nutils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu-g4')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Build', [\n    custom_steps.compile_unix_cpu_openblas('libmxnet'),\n    custom_steps.compile_unix_full_gpu('libmxnet_gpu')\n  ])\n\n  utils.parallel_stage('Build Docs', [\n    // Optimization would be to flag these not to stash if not previewing them\n    custom_steps.docs_jekyll(),\n    custom_steps.docs_c('libmxnet'),\n    custom_steps.docs_python('libmxnet_gpu'),\n  ])\n\n  // TODO: add a website preview function\n\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_website_jekyll_docs",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 20\n\nnode('utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\nutils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Jekyll Website Docs', [\n    custom_steps.docs_jekyll()\n  ])\n\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_website_mxnet_build",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 240\n\nnode('utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\nutils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Build', [\n    custom_steps.compile_unix_cpu_openblas('libmxnet')\n  ])\n\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_website_nightly",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 240\n\nnode('restricted-utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\n\nutils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu-g4')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Build', [\n    custom_steps.compile_unix_cpu_openblas('libmxnet'),\n    custom_steps.compile_unix_full_gpu('libmxnet_gpu')\n  ])\n\n  utils.parallel_stage('Build Docs', [\n    custom_steps.docs_jekyll(),\n    custom_steps.docs_c('libmxnet'),\n    custom_steps.docs_python('libmxnet_gpu'),\n  ])\n\n  utils.parallel_stage('Prepare', [\n    custom_steps.docs_prepare()\n  ])\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_website_python_docs",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 60\n\nnode('utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\nutils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu-g4')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Build', [\n    custom_steps.compile_unix_full_gpu('libmxnet_gpu')\n  ])\n\n  utils.parallel_stage('Python Docs', [\n    custom_steps.docs_python('libmxnet_gpu')\n  ])\n\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_website_version_artifacts",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 240\n\nnode('restricted-utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\n\nutils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu-g4', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Build', [\n    custom_steps.compile_unix_cpu_openblas('libmxnet'),\n    custom_steps.compile_unix_full_gpu('libmxnet_gpu')\n  ])\n\n  utils.parallel_stage('Build Docs', [\n    custom_steps.docs_jekyll(),\n    custom_steps.docs_c('libmxnet'),\n    custom_steps.docs_python('libmxnet_gpu'),\n  ])\n\n  utils.parallel_stage('Build Full Website', [\n    custom_steps.docs_full_website()\n  ])\n\n  utils.parallel_stage('Upload Docs', [\n    custom_steps.docs_upload_s3()\n  ])\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_windows_cpu",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 240\n\nnode('utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\nutils.assign_node_labels(utility: 'utility', windows_cpu: 'mxnetwindows-cpu')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Build', [\n    custom_steps.compile_windows_cpu('windows_package_cpu'),\n    custom_steps.compile_windows_cpu_onednn('windows_package_cpu_onednn'),\n    custom_steps.compile_windows_cpu_onednn_mkl('windows_package_cpu_onednn_mkl'),\n    custom_steps.compile_windows_cpu_mkl('windows_package_cpu_mkl')\n  ])\n\n  utils.parallel_stage('Tests', [\n    custom_steps.test_windows_python3_cpu('windows_package_cpu'),\n  ])\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/jenkins/Jenkinsfile_windows_gpu",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n//\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 240\n\nnode('utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n  custom_steps = load('ci/jenkins/Jenkins_steps.groovy')\n}\nutils.assign_node_labels(utility: 'utility', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu')\n\nutils.main_wrapper(\ncore_logic: {\n  utils.parallel_stage('Build', [\n    custom_steps.compile_windows_gpu('windows_package_gpu'),\n    custom_steps.compile_windows_gpu_onednn('windows_package_gpu_onednn')\n  ])\n\n  utils.parallel_stage('Tests', [\n    custom_steps.test_windows_python3_gpu('windows_package_gpu'),\n    custom_steps.test_windows_python3_gpu_onednn('windows_package_gpu_onednn')\n  ])\n}\n,\nfailure_handler: {\n  // Only send email if master or release branches failed\n  if (currentBuild.result == \"FAILURE\" && (env.BRANCH_NAME == \"master\" || env.BRANCH_NAME.startsWith(\"v\"))) {\n    emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/logging.conf",
    "content": "# -*- coding: utf-8 -*-\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n[loggers]\nkeys=root\n\n[handlers]\nkeys=consoleHandler\n\n[formatters]\nkeys=simpleFormatter\n\n[logger_root]\nlevel=DEBUG\nhandlers=consoleHandler\n\n[handler_consoleHandler]\nclass=StreamHandler\nlevel=DEBUG\nformatter=simpleFormatter\nargs=(sys.stdout,)\n\n[formatter_simpleFormatter]\nformat=%(asctime)s - %(name)s - %(levelname)s - %(message)s\ndatefmt="
  },
  {
    "path": "ci/other/ci_deploy_doc.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n#\n# Execute command outside a docker container\n#\n# Usage: ci_deploy_doc.sh <PR_ID> <BUILD_ID>\n#\n# PR_ID: the PR number\n#\n# BUILD_ID: the current build ID for the specified PR\n#\nset -ex\n\naws s3 sync --delete . s3://mxnet-ci-doc/$1/$2 \\\n    && echo \"Doc is hosted at https://mxnet-ci-doc.s3-accelerate.dualstack.amazonaws.com/$1/$2/index.html\"\n"
  },
  {
    "path": "ci/publish/Jenkinsfile",
    "content": "// -*- mode: groovy -*-\n\n// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements.  See the NOTICE file\n// distributed with this work for additional information\n// regarding copyright ownership.  The ASF licenses this file\n// to you under the Apache License, Version 2.0 (the\n// \"License\"); you may not use this file except in compliance\n// with the License.  You may obtain a copy of the License at\n//\n//   http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing,\n// software distributed under the License is distributed on an\n// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n// KIND, either express or implied.  See the License for the\n// specific language governing permissions and limitations\n// under the License.\n\n// Jenkins pipeline\n// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/\n\n// timeout in minutes\nmax_time = 120\n\nnode('restricted-utility') {\n  // Loading the utilities requires a node context unfortunately\n  checkout scm\n  utils = load('ci/Jenkinsfile_utils.groovy')\n}\nutils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')\n\n// CPU and GPU. OSX nodes are not currently supported by Jenkins\ndef nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU_P3]\ndef scalaOSMap = ['cpu': 'linux-x86_64-cpu', 'gpu': 'linux-x86_64-gpu']\ndef scalaVariantMap = ['cpu': 'cpu', 'gpu': 'cu92']\n\ndef wrapStep(nodeToRun, workspaceName, step) {\n  return {\n    node(nodeToRun) {\n      ws(\"workspace/${workspaceName}\") {\n        timeout(time: max_time, unit: 'MINUTES') {\n          step()\n        }\n      }\n    }\n  }\n}\n\nutils.main_wrapper(\ncore_logic: {\n  stage('Build Packages') {\n    parallel toBuild\n  }\n  stage('Test Packages') {\n    parallel toTest\n  }\n  stage('Deploy Packages') {\n    parallel toDeploy\n  }\n}\n,\nfailure_handler: {\n  if (currentBuild.result == \"FAILURE\") {\n    emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'\n  }\n}\n)\n"
  },
  {
    "path": "ci/publish/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# MXNet Publish Settings\n\nThis folder contains the configuration for restricted nodes on Jenkins for the publishing MXNet artifacts. It also contains a folder called `scala` that contains everything required for publishing to Maven. In this `README`, we provide a brief walkthrough of the Jenkins configuration as well as the usage of the Scala deployment files. Python publishing is TBD.\n\n## Jenkins\nCurrently, Jenkins contains three build stages, namely `Build Packages`, `Test Packages` and `Deploy Packages`. During the `build package` stages, all dependencies are built and a Scala package are created. In the second stage, the package created from the previous stage moves to this stage to specifically run the tests. In the final stage, the packages that pass the tests are deployed by the instances.\n\nThe job is scheduled to be triggered every 24 hours on a [restricted instance](http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/restricted-publish-artifacts).\n\nCurrently, we are supporting tests in the following systems:\n\n- Ubuntu 16.04\n- Ubuntu 18.04\n- Cent OS 7\n\nAll packages are currently built in `Cent OS 7` with Developer Toolset 7.\nDeveloper Toolset 7 provides `GCC 7` with C++17 support on `Cent OS 7`, enabling\nus to build binaries that support all major Linux distributions released after\n2014 (cf. Python Enhancement Proposals 599). All Dockerfile used for publishing\nare available in `ci/docker/` with prefix `Dockerfile.publish`.\n\nApart from that, the script used to create the environment and publish are available under `ci/docker/install`:\n\n- `ubuntu_base.sh` installs minimum dependencies required to run the published packages\n\n## Scala publishing\nCurrently Scala publish on Linux is fully supported on Jenkins. The `scala/` folder contains all files needed for publishing. Here is a brief introduction of the files:\n\n- `build.sh` Main executable files to build the backend as well as scala package\n- `buildkey.py` Main file used to extract password from the system and configure the maven\n- `deploy.sh` Script to deploy the package\n- `fullDeploy.sh` Used by CI to make full publish\n- `test.sh` Make Scala test on CI\n\n## Python publishing\nPython build support is TBD.\n"
  },
  {
    "path": "ci/publish/python/build.sh",
    "content": "#!/usr/bin/env bash\n#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nset -ex\n\nsource tools/staticbuild/build.sh $mxnet_variant\n\nset -ex\n\n# Compile tests for discovery later\nsource tools/staticbuild/build_wheel.sh\n"
  },
  {
    "path": "ci/publish/scala/build.sh",
    "content": "#!/usr/bin/env bash\n#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nset -ex\n\n# Setup Environment Variables\n# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu\n# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu\n\nsource tools/staticbuild/build.sh $mxnet_variant\n\nset -ex\n\n# Compile tests for discovery later\ncd scala-package\nmvn -B deploy -DskipTests=true\n"
  },
  {
    "path": "ci/publish/scala/buildkey.py",
    "content": "#!/usr/bin/env python3\n#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nimport os\nimport json\nimport logging\nimport subprocess\n\nHOME = os.environ['HOME']\nKEY_PATH = os.path.join(HOME, \".m2\")\n\n\n'''\nThis file would do the following items:\n    Import keys from AWS Credential services\n    Create settings.xml in .m2 with pass phrase\n    Create security-settings.xml in .m2 with master password\n    Import keys.asc the encrypted keys in gpg\n'''\n\n\ndef getCredentials():\n    import boto3\n    import botocore\n    endpoint_url = os.environ['MAVEN_PUBLISH_SECRET_ENDPOINT_URL']\n    secret_creds_name = os.environ['MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS']\n    secret_key_name = os.environ['MAVEN_PUBLISH_SECRET_NAME_GPG']\n    region_name = os.environ['DOCKERHUB_SECRET_ENDPOINT_REGION']\n\n    session = boto3.Session()\n    client = session.client(\n        service_name='secretsmanager',\n        region_name=region_name,\n        endpoint_url=endpoint_url\n    )\n    try:\n        get_secret_value_response = client.get_secret_value(\n            SecretId=secret_creds_name\n        )\n        get_secret_key_response = client.get_secret_value(\n            SecretId=secret_key_name\n        )\n    except botocore.exceptions.ClientError as client_error:\n        if client_error.response['Error']['Code'] == 'ResourceNotFoundException':\n            name = (secret_key_name if get_secret_value_response\n                    else secret_creds_name)\n            logging.exception(\"The requested secret %s was not found\", name)\n        elif client_error.response['Error']['Code'] == 'InvalidRequestException':\n            logging.exception(\"The request was invalid due to:\")\n        elif client_error.response['Error']['Code'] == 'InvalidParameterException':\n            logging.exception(\"The request had invalid params:\")\n        raise\n    else:\n        secret = get_secret_value_response['SecretString']\n        secret_dict = json.loads(secret)\n        secret_key = get_secret_key_response['SecretString']\n        return secret_dict, secret_key\n\n\ndef importASC(key, gpgPassphrase):\n    filename = os.path.join(KEY_PATH, \"key.asc\")\n    with open(filename, 'w') as f:\n        f.write(key)\n    subprocess.check_output(['gpg2', '--batch', '--yes',\n                    '--passphrase-fd', '0',\n                    \"--import\", \"{}\".format(filename)],\n                   input=str.encode(gpgPassphrase))\n\n\ndef encryptMasterPSW(password):\n    filename = os.path.join(KEY_PATH, \"encryptMasterPassword.exp\")\n    with open(filename, 'w') as f:\n        f.write('''\n        spawn mvn --encrypt-master-password\n        expect -exact \"Master password: \"\n        send -- \"{}\\r\"\n        expect eof\n        '''.format(password))\n    result = subprocess.check_output(['expect', filename])\n    return str(result).split('\\r\\n')[-1][2:-3]\n\n\ndef encryptPSW(password):\n    filename = os.path.join(KEY_PATH, \"encryptPassword.exp\")\n    with open(filename, 'w') as f:\n        f.write('''\n        spawn mvn --encrypt-password\n        expect -exact \"Password: \"\n        send -- \"{}\\r\"\n        expect eof\n        '''.format(password))\n    result = subprocess.check_output(['expect', filename])\n    return str(result).split('\\r\\n')[-1][2:-3]\n\n\ndef masterPSW(password):\n    with open(os.path.join(KEY_PATH, \"settings-security.xml\"), \"w\") as f:\n        f.write(\"<settingsSecurity>\\n <master>{}</master>\\n</settingsSecurity>\"\n                .format(password))\n\n\ndef serverPSW(username, password, gpgPassphrase):\n    with open(os.path.join(KEY_PATH, \"settings.xml\"), \"w\") as f:\n        settingsString = '''<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<settings xmlns=\"http://maven.apache.org/SETTINGS/1.0.0\"\nxmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\nxsi:schemaLocation=\"http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd\">\n<pluginGroups></pluginGroups>\n<proxies></proxies>\n<servers>\n<server>\n        <id>apache.snapshots.https</id>\n        <username>{}</username>\n        <password>{}</password>\n</server>\n<!-- To stage a release of some part of Maven -->\n<server>\n        <id>apache.releases.https</id>\n        <username>{}</username>\n        <password>{}</password>\n</server>\n</servers>\n<mirrors></mirrors>\n<profiles>\n<profile>\n        <id>gpg</id>\n        <properties>\n        <gpg.executable>gpg2</gpg.executable>\n        <gpg.passphrase>{}</gpg.passphrase>\n        <gpg.skip>true</gpg.skip>\n        </properties>\n</profile>\n</profiles>\n<activeProfiles>\n        <activeProfile>gpg</activeProfile>\n</activeProfiles>\n</settings> '''.format(username, password, username, password, gpgPassphrase)\n        f.write(settingsString)\n\n\nif __name__ == \"__main__\":\n    if not os.path.exists(KEY_PATH):\n        os.makedirs(KEY_PATH)\n    credentials, gpgKey = getCredentials()\n    masterPass = encryptMasterPSW(credentials['masterpass'])\n    masterPSW(masterPass)\n    passwordEncrypted = encryptPSW(credentials['password'])\n    serverPSW(credentials['user'], passwordEncrypted,\n             credentials['gpgPassphrase'])\n    importASC(gpgKey, credentials['gpgPassphrase'])\n"
  },
  {
    "path": "ci/publish/scala/deploy.sh",
    "content": "#!/usr/bin/env bash\n#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nset -ex\n\n# On Jenkins, run python script to configure keys\nif [[ $BUILD_ID ]]; then\n    python3 ci/publish/scala/buildkey.py\nfi\n\n# Updating cache\nmkdir -p ~/.gnupg\necho \"default-cache-ttl 14400\" > ~/.gnupg/gpg-agent.conf\necho \"max-cache-ttl 14400\" >> ~/.gnupg/gpg-agent.conf\necho \"allow-loopback-pinentry\" >> ~/.gnupg/gpg-agent.conf\necho \"pinentry-mode loopback\" >> ~/.gnupg/gpg-agent.conf\nexport GPG_TTY=$(tty)\n\ncd scala-package\n\nmvn -B deploy -Pnightly\n\n# On Jenkins, clear all password .xml files, exp files, and gpg key files\nif [[ $BUILD_ID ]]; then\n    rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp\nfi\n"
  },
  {
    "path": "ci/publish/scala/fullDeploy.sh",
    "content": "#!/usr/bin/env bash\n#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nset -ex\n\n./ci/publish/scala/build.sh\n./ci/publish/scala/test.sh\n./ci/publish/scala/deploy.sh\n"
  },
  {
    "path": "ci/publish/scala/test.sh",
    "content": "#!/usr/bin/env bash\n#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nset -ex\n\nif [ -z \"$JAVA_HOME\" ]; then\n    source /etc/profile\nfi\n\n# Test\ncd scala-package/packageTest\n\nif [[ $mxnet_variant == cu* ]]; then\n    export SCALA_TEST_ON_GPU=1\n    make testlocal USE_CUDA=1 CI=1\nelse\n    make testlocal CI=1\nfi\n"
  },
  {
    "path": "ci/publish/website/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# Website Deployment\n\nRefer to the [MXNet Developer Wiki](https://cwiki.apache.org/confluence/display/MXNET/Building+the+New+Website).\n"
  },
  {
    "path": "ci/publish/website/beta-deploy.sh",
    "content": "#!/bin/bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# A yaml file is written to trigger a staging deployment.\n# This file must be placed in the root of the site repo.\n# profile: sets custom the url using the pattern 'mxnet-PROFILE'\n# Example using 'beta': https://mxnet-beta.staged.apache.org/\n# Documentation: https://www.staged.apache.org/\n\nset -ex\n\nif [ ! -f ./.asf.yaml ]; then\n  echo -e \"\\nGenerating .asf.yaml file\"\n  cat > ./.asf.yaml <<EOL\nstaging:\n  profile: beta\nEOL\nfi\n"
  },
  {
    "path": "ci/publish/website/deploy.sh",
    "content": "#!/bin/bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# build and install are separated so changes to build don't invalidate\n# the whole docker cache for the image\n\n# This script requires that APACHE_PASSWORD and APACHE_USERNAME are set\n# environment variables. Also, artifacts must be previously uploaded to S3\n# in the MXNet public bucket (mxnet-public.s3.us-east-2.amazonaws.com).\n\nset -ex\n\n\n# Managed by Jenkins; set these env vars if running locally\n# export APACHE_USERNAME=\n# export APACHE_PASSWORD=\n\n# Configuration for artifacts\nversion=$2\napi_list=(\"python\")\njekyll_fork=ThomasDelteil\n\n\nsetup_mxnet_site_repo() {\n   fork=$1\n   if [ ! -d \"mxnet-site\" ]; then\n     git clone https://$APACHE_USERNAME:$APACHE_PASSWORD@github.com/aaronmarkham/mxnet-site.git\n   fi\n\n   cd mxnet-site\n   git checkout asf-site\n   rm -rf *\n   git rm -r *\n   cd ..\n}\n\n\nsetup_mxnet_site_repo()\n\n\nsetup_jekyll_repo() {\n   fork=$1\n   if [ ! -d \"mxnet.io-v2\" ]; then\n     git clone https://github.com/$fork/mxnet.io-v2.git\n   fi\n}\n\n\nsetup_jekyll_repo() $jekyll_fork\n\n# Copy in the main jekyll website artifacts\nweb_artifacts=mxnet.io-v2/release\nweb_dir=mxnet-site\ncp -a $web_artifacts/* $web_dir\n\n\nfetch_artifacts() {\n    api=$1\n    artifacts=https://mxnet-public.s3.us-east-2.amazonaws.com/docs/$version/$api-artifacts.tgz\n    dir=mxnet-site/api/\n    wget -q $artifacts\n    mkdir -p $dir\n    tar xf $api-artifacts.tgz -C $dir\n}\n\n# Download and untar each of the API artifacts\nfor i in \"${api_list[@]}\"\ndo\n    fetch_artifacts $i\ndone\n\n# Commit the updates\ncd mxnet-site\npwd\ngit branch\ngit add .\ngit commit -m \"Nightly build\"\ngit push origin asf-site\n# bump the site to force replication\ndate > date.txt\ngit add date.txt\ngit commit -m \"Bump the publish timestamp.\"\ngit push origin asf-site\n"
  },
  {
    "path": "ci/publish/website/publish_artifacts.sh",
    "content": "#!/bin/bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# build and install are separated so changes to build don't invalidate\n# the whole docker cache for the image\n\n# This script requires that APACHE_PASSWORD and APACHE_USERNAME are set\n# environment variables. Also, artifacts must be previously uploaded to S3\n# in the MXNet public bucket (mxnet-public.s3.us-east-2.amazonaws.com).\n\nset -ex\n\napi_list=(\"cpp\" \"clojure\" \"java\" \"julia\" \"python\" \"r\" \"scala\")\nversion=v1.5.0\nfor i in \"${api_list[@]}\"\ndo\n    tar cvf $i-artifacts.tgz $i && aws s3 cp $i-artifacts.tgz s3://mxnet-public/docs/$version/$i-artifacts.tgz\ndone\n"
  },
  {
    "path": "ci/test_docker_login.py",
    "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"\nDocker login tests\n\"\"\"\nimport os\nimport subprocess\nimport unittest\nfrom unittest.mock import create_autospec, patch, call, MagicMock\n\nimport boto3\nfrom boto3 import client\nfrom botocore.stub import Stubber\n\nfrom docker_login import login_dockerhub, logout_dockerhub, main, DOCKERHUB_RETRY_SECONDS, DOCKERHUB_LOGIN_NUM_RETRIES\n\n\nSECRET_NAME = \"secret_name\"\nSECRET_ENDPOINT_URL = \"https://endpoint.url\"\nSECRET_ENDPOINT_REGION = \"us-east-2\"\n\n\ndef mock_boto(num_calls: int = 1):\n    mock_client = client(\"secretsmanager\", region_name=\"us-east-1\")\n    mock_session = create_autospec(boto3.Session)\n    mock_session.client.return_value = mock_client\n\n    # Stub get_secret_value response\n    stub = Stubber(mock_client)\n    for i in range(num_calls):\n        stub.add_response(\n            method=\"get_secret_value\",\n            expected_params={\n                \"SecretId\": \"secret_name\"  # Matches os.environ['SECRET_NAME']\n            }, service_response={\n                \"SecretString\": \"\"\"{\"username\": \"myuser\", \"password\": \"mypass\"}\"\"\"\n            })\n    return mock_session, stub\n\n\nclass TestDockerLogin(unittest.TestCase):\n\n    @patch(\"subprocess.run\", name=\"mock_subprocess_run\")\n    def test_docker_login_success(self, mock_run):\n        \"\"\"\n        Tests successful docker login returns True and calls docker appropriately\n        \"\"\"\n        mock_session, stub = mock_boto()\n        stub.activate()\n        with patch(\"boto3.Session\", return_value=mock_session):\n            mock_process = MagicMock(auto_spec=subprocess.Popen, name=\"mock_process\")\n\n            # Simulate successful login\n            mock_process.returncode = 0\n            mock_run.return_value = mock_process\n\n            login_dockerhub(SECRET_NAME, SECRET_ENDPOINT_URL, SECRET_ENDPOINT_REGION)\n\n            # Check boto client is properly created\n            print(mock_session.client.call_args_list)\n            assert mock_session.client.call_args_list == [\n                call(service_name=\"secretsmanager\", region_name=\"us-east-2\", endpoint_url=\"https://endpoint.url\")\n            ]\n\n            # Check that login call passes in the password in the correct way\n            assert mock_run.call_args_list == [\n                call(\n                    [\"docker\", \"login\", \"--username\", \"myuser\", \"--password-stdin\"],\n                    stdout=subprocess.PIPE,\n                    input=str.encode(\"mypass\")\n                )\n            ]\n        stub.deactivate()\n\n    @patch(\"subprocess.run\", name=\"mock_subprocess_run\")\n    @patch(\"time.sleep\")\n    def test_docker_login_retry(self, mock_sleep, mock_run):\n        \"\"\"\n        Tests retry mechanism\n        \"\"\"\n        num_tries = 3\n        mock_session, stub = mock_boto(num_calls=num_tries)\n        stub.activate()\n        with patch(\"boto3.Session\", return_value=mock_session):\n            mock_process = MagicMock(auto_spec=subprocess.Popen, name=\"mock_process\")\n\n            # Simulate successful login\n            mock_process.returncode = 0\n\n            # Simulate (num_tries - 1) errors + 1 success\n            mock_run.side_effect = \\\n                [subprocess.CalledProcessError(1, \"cmd\", \"some error\")] * (num_tries - 1) + [mock_process]\n\n            login_dockerhub(SECRET_NAME, SECRET_ENDPOINT_URL, SECRET_ENDPOINT_REGION)\n\n            # Check boto client is properly created\n            print(mock_session.client.call_args_list)\n            assert mock_session.client.call_args_list == [\n                call(service_name=\"secretsmanager\", region_name=\"us-east-2\", endpoint_url=\"https://endpoint.url\")\n            ] * num_tries\n\n            # Check that login call passes in the password in the correct way\n            cmd = [\"docker\", \"login\", \"--username\", \"myuser\", \"--password-stdin\"]\n            assert mock_run.call_args_list == [\n                call(cmd, stdout=subprocess.PIPE, input=str.encode(\"mypass\"))\n            ] * num_tries\n\n            # Assert sleep was called appropriately\n            assert mock_sleep.call_args_list == [\n                call(2 ** retry_num * DOCKERHUB_RETRY_SECONDS) for retry_num in range(0, num_tries - 1)\n            ]\n        stub.deactivate()\n\n    @patch(\"subprocess.run\", name=\"mock_subprocess_run\")\n    @patch(\"time.sleep\")\n    def test_docker_login_retry_exhausted(self, mock_sleep, mock_run):\n        \"\"\"\n        Tests retry mechanism\n        \"\"\"\n        num_tries = DOCKERHUB_LOGIN_NUM_RETRIES\n        mock_session, stub = mock_boto(num_calls=num_tries)\n        stub.activate()\n        with patch(\"boto3.Session\", return_value=mock_session):\n            # Simulate num_tries errors\n            mock_run.side_effect = [subprocess.CalledProcessError(1, \"cmd\", \"some error\")] * num_tries\n\n            with self.assertRaises(subprocess.CalledProcessError):\n                login_dockerhub(SECRET_NAME, SECRET_ENDPOINT_URL, SECRET_ENDPOINT_REGION)\n\n            # Check boto client is properly created\n            assert mock_session.client.call_args_list == [\n                call(service_name=\"secretsmanager\", region_name=\"us-east-2\", endpoint_url=\"https://endpoint.url\")\n            ] * num_tries\n\n            # Check that login call passes in the password in the correct way\n            cmd = [\"docker\", \"login\", \"--username\", \"myuser\", \"--password-stdin\"]\n            assert mock_run.call_args_list == [\n                call(cmd, stdout=subprocess.PIPE, input=str.encode(\"mypass\"))\n            ] * num_tries\n\n            # Assert sleep was called appropriately\n            assert mock_sleep.call_args_list == [\n                call(2 ** retry_num * DOCKERHUB_RETRY_SECONDS) for retry_num in range(0, num_tries-1)\n            ]\n        stub.deactivate()\n\n    @patch(\"subprocess.run\", name=\"mock_subprocess_run\")\n    def test_docker_login_failed(self, mock_run):\n        \"\"\"\n        Tests failed docker login return false\n        \"\"\"\n        mock_session, stub = mock_boto()\n        stub.activate()\n        with patch(\"boto3.Session\", return_value=mock_session):\n\n            mock_process = MagicMock(auto_spec=subprocess.Popen, name=\"mock_process\")\n\n            # Simulate failed login\n            mock_process.returncode = 1\n            mock_run.return_value = mock_process\n\n            with self.assertRaises(RuntimeError):\n                login_dockerhub(SECRET_NAME, SECRET_ENDPOINT_URL, SECRET_ENDPOINT_REGION)\n        stub.deactivate()\n\n    @patch(\"subprocess.call\", name=\"mock_subprocess_call\")\n    def test_logout(self, mock_call):\n        \"\"\"\n        Tests logout calls docker command appropriately\n        \"\"\"\n        logout_dockerhub()\n        assert mock_call.call_args_list == [\n            call([\"docker\", \"logout\"])\n        ]\n\n    @patch(\"docker_login.login_dockerhub\")\n    def test_main_exit(self, mock_login):\n        \"\"\"\n        Tests main exits with error on failed docker login\n        \"\"\"\n        mock_login.side_effect = RuntimeError(\"Didn't work\")\n        with self.assertRaises(SystemExit):\n            main([\"--secret-name\", \"name\", \"--secret-endpoint-url\", \"url\", \"--secret-endpoint-region\", \"r\"])\n\n    @patch(\"docker_login.login_dockerhub\")\n    def test_main_default_argument_values(self, mock_login):\n        \"\"\"\n        Tests default arguments\n        \"\"\"\n\n        # Good env\n        env = {\n            \"DOCKERHUB_SECRET_ENDPOINT_URL\": \"url\",\n            \"DOCKERHUB_SECRET_ENDPOINT_REGION\": \"region\"\n        }\n        with patch.dict(os.environ, env):\n            main([\"--secret-name\", \"name\"])\n            assert mock_login.call_args_list == [\n                call(\"name\", \"url\", \"region\")\n            ]\n\n        # Bad envs - none or not all required vars defined\n        tests = [\n            {},\n            {\"DOCKERHUB_SECRET_ENDPOINT_URL\": \"url\"},\n            {\"DOCKERHUB_SECRET_ENDPOINT_REGION\": \"region\"}\n        ]\n        for bad_env in tests:\n            with patch.dict(os.environ, bad_env):\n                with self.assertRaises(RuntimeError):\n                    main([\"--secret-name\", \"name\"])\n\n"
  },
  {
    "path": "ci/util.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport contextlib\nimport logging\nimport logging.config\nimport os\nimport subprocess\nimport sys\n\nimport requests\n\n\ndef get_mxnet_root() -> str:\n    curpath = os.path.abspath(os.path.dirname(__file__))\n\n    def is_mxnet_root(path: str) -> bool:\n        return os.path.exists(os.path.join(path, \".mxnet_root\"))\n\n    while not is_mxnet_root(curpath):\n        parent = os.path.abspath(os.path.join(curpath, os.pardir))\n        if parent == curpath:\n            raise RuntimeError(\"Got to the root and couldn't find a parent folder with .mxnet_root\")\n        curpath = parent\n    return curpath\n\n\n@contextlib.contextmanager\ndef remember_cwd():\n    '''\n    Restore current directory when exiting context\n    '''\n    curdir = os.getcwd()\n    try: yield\n    finally: os.chdir(curdir)\n\n\ndef retry(target_exception, tries=4, delay_s=1, backoff=2):\n    \"\"\"Retry calling the decorated function using an exponential backoff.\n\n    http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/\n    original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry\n\n    :param target_exception: the exception to check. may be a tuple of\n        exceptions to check\n    :type target_exception: Exception or tuple\n    :param tries: number of times to try (not retry) before giving up\n    :type tries: int\n    :param delay_s: initial delay between retries in seconds\n    :type delay_s: int\n    :param backoff: backoff multiplier e.g. value of 2 will double the delay\n        each retry\n    :type backoff: int\n    \"\"\"\n    import time\n    from functools import wraps\n\n    def decorated_retry(f):\n        @wraps(f)\n        def f_retry(*args, **kwargs):\n            mtries, mdelay = tries, delay_s\n            while mtries > 1:\n                try:\n                    return f(*args, **kwargs)\n                except target_exception as e:\n                    logging.warning(\"Exception: %s, Retrying in %d seconds...\", str(e), mdelay)\n                    time.sleep(mdelay)\n                    mtries -= 1\n                    mdelay *= backoff\n            return f(*args, **kwargs)\n\n        return f_retry  # true decorator\n\n    return decorated_retry\n\n\n# noinspection SyntaxError\ndef under_ci() -> bool:\n    \"\"\":return: True if we run in Jenkins.\"\"\"\n    return 'JOB_NAME' in os.environ\n\n\ndef ec2_instance_info() -> str:\n    import requests\n    urls = [\n            \"http://instance-data/latest/meta-data/instance-type\",\n            \"http://instance-data/latest/meta-data/instance-id\",\n            \"http://instance-data/latest/meta-data/public-hostname\",\n            \"http://instance-data/latest/meta-data/ami-id\"\n    ]\n    if under_ci():\n        result = []\n        try:\n            for url in urls:\n                r = requests.get(url)\n                if r.status_code == 200:\n                    result.append(r.content.decode())\n            return ' '.join(result)\n        except ConnectionError:\n            pass\n        return '?'\n    else:\n        return ''\n\n\ndef chdir_to_script_directory():\n    # We need to be in the same directory than the script so the commands in the dockerfiles work as\n    # expected. But the script can be invoked from a different path\n    base = os.path.split(os.path.realpath(__file__))[0]\n    os.chdir(base)\n\n\ndef script_name() -> str:\n    \"\"\":returns: script name with leading paths removed\"\"\"\n    return os.path.split(sys.argv[0])[1]\n\n\ndef config_logging():\n    conf_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), \"logging.conf\")\n    logging.config.fileConfig(os.getenv('LOGGING_CONF', conf_path))\n\n    # Force botocore and requests are set to WARNING to avoid leaking any credentials\n    # or sensitive information\n    logging.getLogger(\"botocore\").setLevel(logging.WARNING)\n    logging.getLogger(\"requests\").setLevel(logging.WARNING)\n\n\n# Takes url and downloads it to the dest_path directory on Windows.\ndef download_file(url, dest_path):\n    file_name = url.split('/')[-1]\n    full_path = \"{}\\\\{}\".format(dest_path, file_name)\n    logging.info(\"Downloading: {}\".format(full_path))\n    r = requests.get(url, stream=True)\n    if r.status_code == 404:\n        return r.status_code\n    elif r.status_code != 200:\n        logging.error(\"{} returned status code {}\".format(url, r.status_code))\n    with open(full_path, 'wb') as f:\n        for chunk in r.iter_content(chunk_size=1024):\n            if chunk: # filter out keep-alive new chunks\n                f.write(chunk)\n    return full_path\n\n\n# Takes arguments and runs command on host.  Shell is disabled by default.\ndef run_command(args, shell=False):\n    try:\n        logging.info(\"Issuing command: {}\".format(args))\n        res = subprocess.check_output(args, shell=shell, timeout=1800).decode(\"utf-8\").replace(\"\\r\\n\", \"\")\n        logging.info(\"Output: {}\".format(res))\n    except subprocess.CalledProcessError as e:\n        raise RuntimeError(\"command '{}' return with error (code {}): {}\".format(e.cmd, e.returncode, e.output))\n    return res\n"
  },
  {
    "path": "ci/windows/test_py3_cpu.ps1",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n7z x -y windows_package.7z\n\n$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\\lib\\libmxnet.dll\n$env:PYTHONPATH=join-path $pwd.Path windows_package\\python\n$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0\n$env:MXNET_SUBGRAPH_VERBOSE=0\n$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home')\n\nC:\\Python38\\Scripts\\pip install -r ci\\docker\\install\\requirements\nC:\\Python38\\python.exe -m pytest -v -m 'not serial' -n 4 --durations=50 --cov-report xml:tests_unittest.xml tests\\python\\unittest\nif ($LastExitCode -ne 0) { Throw (\"Error running parallel unittest, python exited with status code \" + ('{0:X}' -f $LastExitCode)) }\nC:\\Python38\\python.exe -m pytest -v -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append tests\\python\\unittest\nif ($LastExitCode -ne 0) { Throw (\"Error running serial unittest, python exited with status code \" + ('{0:X}' -f $LastExitCode)) }\nC:\\Python38\\python.exe -m pytest -v -m 'not serial' -n 4 --durations=50 --cov-report xml:tests_train.xml tests\\python\\train\nif ($LastExitCode -ne 0) { Throw (\"Error running parallel train tests, python exited with status code \" + ('{0:X}' -f $LastExitCode)) }\nC:\\Python38\\python.exe -m pytest -v -m 'serial' --durations=50 --cov-report xml:tests_train.xml --cov-append tests\\python\\train\nif ($LastExitCode -ne 0) { Throw (\"Error running serial train tests, python exited with status code \" + ('{0:X}' -f $LastExitCode)) }\n# Adding this extra test since it's not possible to set env var on the fly in Windows.\nC:\\Python38\\python.exe -m pytest -v --durations=50 --cov-report xml:tests_unittest.xml --cov-append tests\\python\\unittest\\test_operator.py::test_norm\nif ($LastExitCode -ne 0) { Throw (\"Error running unittest, python exited with status code \" + ('{0:X}' -f $LastExitCode)) }\n\n# Need to explicitly set the environment variable for MXNET_MEMORY_OPT.\n$env:MXNET_MEMORY_OPT=1\nC:\\Python38\\python.exe -m pytest -v --durations=50 --cov-report xml:tests_unittest.xml --cov-append tests\\python\\unittest\\test_memory_opt.py\nif ($LastExitCode -ne 0) { Throw (\"Error running unittest, python exited with status code \" + ('{0:X}' -f $LastExitCode)) }\n"
  },
  {
    "path": "ci/windows/test_py3_gpu.ps1",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n7z x -y windows_package.7z\n\n$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\\lib\\libmxnet.dll\n$env:PYTHONPATH=join-path $pwd.Path windows_package\\python\n$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0\n$env:MXNET_SUBGRAPH_VERBOSE=0\n$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home')\n$env:MXNET_GPU_MEM_POOL_TYPE=\"Unpooled\"\n\nC:\\Python38\\Scripts\\pip install -r ci\\docker\\install\\requirements\nC:\\Python38\\python.exe -m pytest -v -m 'not serial' -n 4 --durations=50 --cov-report xml:tests_unittest.xml tests\\python\\unittest\nif ($LastExitCode -ne 0) { Throw (\"Error running parallel unittest, python exited with status code \" + ('{0:X}' -f $LastExitCode)) }\nC:\\Python38\\python.exe -m pytest -v -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append tests\\python\\unittest\nif ($LastExitCode -ne 0) { Throw (\"Error running serial unittest, python exited with status code \" + ('{0:X}' -f $LastExitCode)) }\n\nC:\\Python38\\python.exe -m pytest -v -m 'not serial' -n 4 --durations=50 --cov-report xml:tests_operator.xml tests\\python\\gpu\\test_operator_gpu.py\nif ($LastExitCode -ne 0) { Throw (\"Error running parallel tests, python exited with status code \" + ('{0:X}' -f $LastExitCode)) }\nC:\\Python38\\python.exe -m pytest -v -m 'serial' --durations=50 --cov-report xml:tests_operator.xml --cov-append tests\\python\\gpu\\test_operator_gpu.py\nif ($LastExitCode -ne 0) { Throw (\"Error running serial tests, python exited with status code \" + ('{0:X}' -f $LastExitCode)) }\n\nC:\\Python38\\python.exe -m pytest -v -m 'not serial' -n 4 --durations=50 --cov-report xml:tests_train.xml tests\\python\\train\nif ($LastExitCode -ne 0) { Throw (\"Error running parallel tests, python exited with status code \" + ('{0:X}' -f $LastExitCode)) }\nC:\\Python38\\python.exe -m pytest -v -m 'serial' --durations=50 --cov-report xml:tests_train.xml --cov-append tests\\python\\train\nif ($LastExitCode -ne 0) { Throw (\"Error running serial tests, python exited with status code \" + ('{0:X}' -f $LastExitCode)) }\n\n# Adding this extra test since it's not possible to set env var on the fly in Windows.\nC:\\Python38\\python.exe -m pytest -v --durations=50 --cov-report xml:tests_operator.xml --cov-append tests\\python\\gpu\\test_operator_gpu.py::test_norm\nif ($LastExitCode -ne 0) { Throw (\"Error running tests, python exited with status code \" + ('{0:X}' -f $LastExitCode)) }\nC:\\Python38\\python.exe -m pytest -v --durations=50 --cov-report xml:tests_tvm_op.xml tests\\python\\gpu\\test_tvm_op_gpu.py\nif ($LastExitCode -ne 0) { Throw (\"Error running TVM op tests, python exited with status code \" + ('{0:X}' -f $LastExitCode)) }\n\n# Need to explicitly set the environment variable for MXNET_MEMORY_OPT.\n$env:MXNET_MEMORY_OPT=1\nC:\\Python38\\python.exe -m pytest -v --durations=50 --cov-report xml:tests_unittest.xml --cov-append tests\\python\\unittest\\test_memory_opt.py\nif ($LastExitCode -ne 0) { Throw (\"Error running memory optimization tests, python exited with status code \" + ('{0:X}' -f $LastExitCode)) }\n"
  },
  {
    "path": "cmake/BuildCythonModules.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nfunction(add_cython_modules python_version)\n  find_package(Python3)\n  find_program(CYTHON_EXECUTABLE NAMES cython cython.bat cython3)\n  if(CYTHON_EXECUTABLE AND Python3_EXECUTABLE)\n    add_custom_command(COMMAND ${CMAKE_COMMAND} POST_BUILD\n                        -E env MXNET_LIBRARY_PATH=${CMAKE_BINARY_DIR}/libmxnet.so\n                        ${Python3_EXECUTABLE} setup.py build_ext --inplace --with-cython\n                        TARGET mxnet\n                        WORKING_DIRECTORY \"${PROJECT_SOURCE_DIR}/python\")\n    message(\"-- Cython modules will be built\")\n  else()\n    message(FATAL_ERROR \"-- Cython not found\")\n  endif()\nendfunction()\n"
  },
  {
    "path": "cmake/BuildTVM.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Whether enable ROCM runtime\n#\n# Possible values:\n# - ON: enable ROCM with cmake's auto search\n# - OFF: disable ROCM\n# - /path/to/rocm: use specific path to rocm\nset(USE_ROCM OFF)\n\n# Whether enable SDAccel runtime\nset(USE_SDACCEL OFF)\n\n# Whether enable Intel FPGA SDK for OpenCL (AOCL) runtime\nset(USE_AOCL OFF)\n\n# Whether enable OpenCL runtime\nset(USE_OPENCL OFF)\n\n# Whether enable Metal runtime\nset(USE_METAL OFF)\n\n# Whether enable Vulkan runtime\n#\n# Possible values:\n# - ON: enable Vulkan with cmake's auto search\n# - OFF: disable vulkan\n# - /path/to/vulkan-sdk: use specific path to vulkan-sdk\nset(USE_VULKAN OFF)\n\n# Whether enable OpenGL runtime\nset(USE_OPENGL OFF)\n\n# Whether to enable SGX runtime\n#\n# Possible values for USE_SGX:\n# - /path/to/sgxsdk: path to Intel SGX SDK\n# - OFF: disable SGX\n#\n# SGX_MODE := HW|SIM\nset(USE_SGX OFF)\nset(SGX_MODE \"SIM\")\nset(RUST_SGX_SDK \"/path/to/rust-sgx-sdk\")\n\n# Whether enable RPC runtime\nset(USE_RPC ON)\n\n# Whether embed stackvm into the runtime\nset(USE_STACKVM_RUNTIME OFF)\n\n# Whether enable tiny embedded graph runtime.\nset(USE_GRAPH_RUNTIME ON)\n\n# Whether enable additional graph debug functions\nset(USE_GRAPH_RUNTIME_DEBUG OFF)\n\n# Whether build with LLVM support\n# Requires LLVM version >= 4.0\n#\n# Possible values:\n# - ON: enable llvm with cmake's find search\n# - OFF: disable llvm\n# - /path/to/llvm-config: enable specific LLVM when multiple llvm-dev is available.\nset(USE_LLVM ON)\n\n#---------------------------------------------\n# Contrib libraries\n#---------------------------------------------\n# Whether use BLAS, choices: openblas, mkl, atlas, apple\nset(USE_BLAS none)\n\n# /path/to/mkl: mkl root path when use mkl blas library\n# set(USE_MKL /opt/intel/mkl) for UNIX\n# set(USE_MKL ../IntelSWTools/compilers_and_libraries_2018/windows/mkl) for WIN32\nset(USE_MKL OFF)\n\n# Whether use contrib.random in runtime\nset(USE_RANDOM OFF)\n\n# Whether use NNPack\nset(USE_NNPACK OFF)\n\n# First-class Cuda in modern CMake provides us with CMAKE_CUDA_COMPILER But TVM\n# uses the deprecated findCUDA functionality which requires\n# CUDA_TOOLKIT_ROOT_DIR We follow the FindCUDAToolkit.cmake logic to compute\n# CUDA_TOOLKIT_ROOT_DIR for TVM https://gitlab.kitware.com/cmake/cmake/merge_requests/4093/\nif(USE_CUDA)\n  get_filename_component(cuda_dir \"${CMAKE_CUDA_COMPILER}\" DIRECTORY)\n  set(CUDA_BIN_DIR \"${cuda_dir}\" CACHE PATH \"\" FORCE)\n  unset(cuda_dir)\n  get_filename_component(CUDA_TOOLKIT_ROOT_DIR ${CUDA_BIN_DIR} DIRECTORY ABSOLUTE)\n\n  message(\"CMAKE_CUDA_COMPILER: ${CMAKE_CUDA_COMPILER}\")\n  message(\"Inferred CUDA_TOOLKIT_ROOT_DIR for TVM as: ${CUDA_TOOLKIT_ROOT_DIR}\")\n  set(USE_CUDA ${CUDA_TOOLKIT_ROOT_DIR})\nendif()\n\n# Whether use cuBLAS\nset(USE_CUBLAS OFF)\n\n# Whether use MIOpen\nset(USE_MIOPEN OFF)\n\n# Whether use MPS\nset(USE_MPS OFF)\n\n# Whether use rocBlas\nset(USE_ROCBLAS OFF)\n\n# Whether use contrib sort\nset(USE_SORT OFF)\n\n# Build ANTLR parser for Relay text format\nset(USE_ANTLR OFF)\n\n# Build TSIM for VTA\nset(USE_VTA_TSIM OFF)\n\n# Whether use Relay debug mode\nset(USE_RELAY_DEBUG OFF)\n\n# Disable USE_MKLDNN for TVM\nset(USE_MKLDNN OFF)\n\n# Sanity checks\nif(NOT DEFINED USE_OPENMP)\n  message(FATAL_ERROR \"TVM expects USE_OPENMP is set. But USE_OPENMP was neither ON nor OFF.\")\nendif()\n"
  },
  {
    "path": "cmake/ChooseBlas.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset(BLAS \"Open\" CACHE STRING \"Selected BLAS library\")\nset_property(CACHE BLAS PROPERTY STRINGS \"Atlas;Open;MKL\")\n# ---[ Root folders\nset(INTEL_HOME_ROOT \"$ENV{HOME}/intel\" CACHE PATH \"Folder contains user-installed intel libs\")\nset(INTEL_OPT_ROOT \"/opt/intel\" CACHE PATH \"Folder contains root-installed intel libs\")\n\nif(DEFINED USE_BLAS)\n  set(BLAS \"${USE_BLAS}\")\nendif()\nif(USE_BLAS MATCHES \"MKL\" OR USE_BLAS MATCHES \"mkl\" OR NOT DEFINED USE_BLAS)\n  find_path(MKL_INCLUDE_DIR mkl_version.h\n    PATHS $ENV{MKLROOT} ${INTEL_HOME_ROOT}/mkl ${INTEL_OPT_ROOT}/mkl ${INTEL_OPT_ROOT}/oneapi/mkl/latest\n    PATH_SUFFIXES mkl latest include)\n  if(NOT MKL_INCLUDE_DIR STREQUAL \"MKL_INCLUDE_DIR-NOTFOUND\")\n    set(BLAS \"MKL\")\n  endif()\nendif()\n\nif(BLAS STREQUAL \"Atlas\" OR BLAS STREQUAL \"atlas\")\n  find_package(Atlas REQUIRED)\n  include_directories(SYSTEM ${Atlas_INCLUDE_DIR})\n  list(APPEND mshadow_LINKER_LIBS ${Atlas_LIBRARIES})\n  add_definitions(-DMSHADOW_USE_CBLAS=1)\n  add_definitions(-DMSHADOW_USE_MKL=0)\n  add_definitions(-DMXNET_USE_BLAS_ATLAS=1)\nelseif(BLAS STREQUAL \"Open\" OR BLAS STREQUAL \"open\")\n  find_package(OpenBLAS REQUIRED)\n  include_directories(SYSTEM ${OpenBLAS_INCLUDE_DIR})\n  list(APPEND mshadow_LINKER_LIBS ${OpenBLAS_LIB})\n  add_definitions(-DMSHADOW_USE_CBLAS=1)\n  add_definitions(-DMSHADOW_USE_MKL=0)\n  add_definitions(-DMXNET_USE_BLAS_OPEN=1)\n  if(NOT MSVC)\n    # check if we need to link to omp\n    execute_process(COMMAND ${CMAKE_NM} -g ${OpenBLAS_LIB}\n                    COMMAND grep omp_get_num_threads\n                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}\n                    OUTPUT_VARIABLE OPENBLAS_USES_OMP_OUT\n                    RESULT_VARIABLE OPENBLAS_USES_OMP_RET)\n    if(NOT OPENBLAS_USES_OMP_OUT STREQUAL \"\" AND NOT OPENBLAS_USES_OMP_RET AND NOT USE_OPENMP)\n      message(\"Openblas uses OMP, automatically linking to it\")\n      find_package(OpenMP REQUIRED)\n      message(\"OpenMP_CXX_LIBRARIES is ${OpenMP_CXX_LIBRARIES}\")\n      list(APPEND mshadow_LINKER_LIBS \"${OpenMP_CXX_LIBRARIES}\")\n    endif()\n    # check if we need to link to gfortran\n    execute_process(COMMAND ${CMAKE_NM} -g ${OpenBLAS_LIB}\n                    COMMAND grep gfortran\n                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}\n                    OUTPUT_VARIABLE OPENBLAS_USES_GFORTRAN_OUT\n                    RESULT_VARIABLE OPENBLAS_USES_GFORTRAN_RET)\n    if(NOT OPENBLAS_USES_GFORTRAN_OUT STREQUAL \"\" AND NOT OPENBLAS_USES_GFORTRAN_RET)\n      message(\"Openblas uses GFortran, automatically linking to it\")\n      file(WRITE \"${CMAKE_CURRENT_BINARY_DIR}/temp/CMakeLists.txt\"\n      \"cmake_minimum_required(VERSION ${CMAKE_VERSION})\nproject(CheckFortran Fortran)\nset(CMAKE_Fortran_COMPILER gfortran)\nfile(WRITE \\\"${CMAKE_CURRENT_BINARY_DIR}/temp/FortranDir.cmake\\\"\n\\\"\nset(FORTRAN_DIR \\\\\\\"\\$\\{CMAKE_Fortran_IMPLICIT_LINK_DIRECTORIES\\}\\\\\\\")\n\\\")\n\")\n      execute_process(\n        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/temp/\n        COMMAND ${CMAKE_COMMAND} .\n      )\n      set(FORTRAN_DIR \"\")\n      include(${CMAKE_CURRENT_BINARY_DIR}/temp/FortranDir.cmake)\n      find_library(FORTRAN_LIB NAMES gfortran HINTS ${FORTRAN_DIR})\n      message(\"FORTRAN_DIR is ${FORTRAN_DIR}\")\n      message(\"FORTRAN_LIB is ${FORTRAN_LIB}\")\n      list(APPEND mshadow_LINKER_LIBS ${FORTRAN_LIB})\n      file(REMOVE_RECURSE \"${CMAKE_CURRENT_BINARY_DIR}/temp/\")\n    endif()\n    # check the lapack flavor of openblas\n    include(CheckSymbolExists)\n    check_symbol_exists(OPENBLAS_USE64BITINT \"${OpenBLAS_INCLUDE_DIR}/openblas_config.h\" OPENBLAS_ILP64)\n    if(OPENBLAS_ILP64)\n      message(\"Using ILP64 OpenBLAS\")\n      if(NOT USE_INT64_TENSOR_SIZE)\n        message(FATAL_ERROR \"Must set USE_INT64_TENSOR_SIZE=1 when using ILP64 OpenBLAS\")\n      endif()\n    else()\n      message(\"Using LP64 OpenBLAS\")\n    endif()\n    if(USE_LAPACK)\n      if(EXISTS \"${OpenBLAS_INCLUDE_DIR}/lapacke.h\")\n        message(\"Detected lapacke.h, automatically using the LAPACKE interface\")\n        add_definitions(-DMXNET_USE_LAPACKE_INTERFACE=1)\n        set(USE_LAPACKE_INTERFACE 1)\n        if(OPENBLAS_ILP64)\n          message(\"Detected ILP64 LAPACKE\")\n         add_definitions(-DMXNET_USE_ILP64_LAPACKE=1)\n        endif()\n      else()\n        execute_process(COMMAND ${CMAKE_NM} -g ${OpenBLAS_LIB}\n                        COMMAND grep sgetri_\n                        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}\n                        OUTPUT_VARIABLE OPENBLAS_CONTAINS_C_LAPACK_OUT\n                        RESULT_VARIABLE OPENBLAS_CONTAINS_C_LAPACK_RET)\n        if(OPENBLAS_CONTAINS_C_LAPACK_OUT STREQUAL \"\"\n           AND NOT OPENBLAS_CONTAINS_C_LAPACK_RET)\n          list(APPEND mshadow_LINKER_LIBS lapack)\n        endif()\n      endif()\n    endif()\n  endif()\nelseif(BLAS STREQUAL \"MKL\" OR BLAS STREQUAL \"mkl\")\n  # ---[ MKL Options\n  file(STRINGS ${MKL_INCLUDE_DIR}/mkl_version.h MKL_VERSION_DEF REGEX \"INTEL_MKL_VERSION\")\n  string(REGEX MATCH \"([0-9]+)\" MKL_VERSION ${MKL_VERSION_DEF})\n  if(UNIX)\n    # Single dynamic library interface leads to conflicts between intel omp and llvm omp\n    # https://github.com/apache/incubator-mxnet/issues/17641\n    # Fixed in oneMKL 2021.3: [MKLD-11109] MKL is opening libgomp.so instead of\n    # libgomp.so.1 while SDL=1 & MKL_THREADING_LAYER=GNU\n    cmake_dependent_option(MKL_USE_SINGLE_DYNAMIC_LIBRARY \"Use single dynamic library interface\" ON\n      \"NOT BLA_STATIC;MKL_VERSION GREATER_EQUAL 20210003\" OFF)\n  else()\n    option(MKL_USE_SINGLE_DYNAMIC_LIBRARY \"Use single dynamic library interface\" ON)\n  endif()\n  cmake_dependent_option(BLA_STATIC \"Use static libraries\" ON \"NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY\" OFF)\n  option(MKL_MULTI_THREADED  \"Use multi-threading\" ON)\n\n  if(BLA_VENDOR)\n      message(FATAL_ERROR \"Do not set BLA_VENDOR manually. MKL version (BLA_VENDOR) is selected based on MKL_USE_SINGLE_DYNAMIC_LIBRARY, \"\n                          \"MKL_MULTI_THREADED and USE_INT64_TENSOR_SIZE flags. If you want to select specific MKL library version \"\n                          \"please set the above-mentioned flags instead.\")\n  endif()\n\n  if(MKL_USE_SINGLE_DYNAMIC_LIBRARY)\n    set(BLA_VENDOR Intel10_64_dyn)\n    add_definitions(-DMKL_USE_SINGLE_DYNAMIC_LIBRARY=1)\n  else()\n    if(CMAKE_SIZEOF_VOID_P EQUAL 4)\n      set(BLA_VENDOR Intel10_32)\n    else()\n      if(MKL_MULTI_THREADED)\n        if(USE_INT64_TENSOR_SIZE)\n          set(BLA_VENDOR Intel10_64ilp)\n        else()\n          set(BLA_VENDOR Intel10_64lp)\n        endif()\n      else()\n        if(USE_INT64_TENSOR_SIZE)\n          set(BLA_VENDOR Intel10_64ilp_seq)\n        else()\n          set(BLA_VENDOR Intel10_64lp_seq)\n        endif()\n      endif()\n    endif()\n  endif()\n  # In case of oneAPI 2021.3 if MKL_INCLUDE_DIR points to the subdirectory 'include',\n  # use the parent directory 'latest' instead\n  file(TO_CMAKE_PATH \"${MKL_INCLUDE_DIR}\" BLAS_mkl_MKLROOT)\n  get_filename_component(BLAS_mkl_MKLROOT_LAST_DIR \"${BLAS_mkl_MKLROOT}\" NAME)\n  if(BLAS_mkl_MKLROOT_LAST_DIR STREQUAL \"include\")\n      get_filename_component(BLAS_mkl_MKLROOT \"${BLAS_mkl_MKLROOT}\" DIRECTORY)\n  endif()\n  find_package(BLAS)\n  include_directories(SYSTEM ${MKL_INCLUDE_DIR})\n  list(APPEND mshadow_LINKER_LIBS ${BLAS_LIBRARIES})\n  if(USE_INT64_TENSOR_SIZE)\n    add_definitions(-DUSE_INT64_TENSOR_SIZE=1)\n  endif()\n  add_definitions(-DMSHADOW_USE_CBLAS=0)\n  add_definitions(-DMSHADOW_USE_MKL=1)\n  add_definitions(-DMXNET_USE_BLAS_MKL=1)\n  message(\"-- Found MKL (version: ${MKL_VERSION})\")\nelseif(BLAS STREQUAL \"apple\")\n  find_package(Accelerate REQUIRED)\n  include_directories(SYSTEM ${Accelerate_INCLUDE_DIR})\n  list(APPEND mshadow_LINKER_LIBS ${Accelerate_LIBRARIES})\n  add_definitions(-DMSHADOW_USE_MKL=0)\n  add_definitions(-DMSHADOW_USE_CBLAS=1)\n  add_definitions(-DMXNET_USE_BLAS_APPLE=1)\nendif()\n"
  },
  {
    "path": "cmake/Modules/FindAccelerate.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Find the Apple Accelerate framework\n#\n# The following are set after configuration is done:\n#  Accelerate_FOUND\n#  Accelerate_INCLUDE_DIRS\n#  Accelerate_LIBRARIES\n\nfile(TO_CMAKE_PATH \"$ENV{Accelerate_HOME}\" Accelerate_HOME)\nset(Accelerate_INCLUDE_SEARCH_PATHS\n  /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/Current\n  /System/Library/Frameworks/Accelerate.framework/Versions/Current/Frameworks/vecLib.framework/Versions/Current\n  ${Accelerate_HOME}\n)\n\nfind_path(Accelerate_CBLAS_INCLUDE_DIR NAMES cblas.h PATHS ${Accelerate_INCLUDE_SEARCH_PATHS} PATH_SUFFIXES Headers)\n\nset(LOOKED_FOR\n    Accelerate_CBLAS_INCLUDE_DIR\n)\n\ninclude(FindPackageHandleStandardArgs)\nfind_package_handle_standard_args(Accelerate DEFAULT_MSG ${LOOKED_FOR})\n\nif(Accelerate_FOUND)\n  set(Accelerate_INCLUDE_DIR ${Accelerate_CBLAS_INCLUDE_DIR})\n  set(Accelerate_LIBRARIES \"-framework Accelerate\")\n  mark_as_advanced(${LOOKED_FOR})\n\n  message(STATUS \"Found Accelerate (include: ${Accelerate_CBLAS_INCLUDE_DIR}, library: ${Accelerate_BLAS_LIBRARY})\")\nendif(Accelerate_FOUND)\n\n"
  },
  {
    "path": "cmake/Modules/FindAtlas.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Find the Atlas (and Lapack) libraries\n#\n# The following variables are optionally searched for defaults\n#  Atlas_ROOT_DIR:            Base directory where all Atlas components are found\n#  Atlas_NEED_LAPACK:         Whether need lapack libraries\n#\n# The following are set after configuration is done:\n#  Atlas_FOUND\n#  Atlas_INCLUDE_DIRS\n#  Atlas_LIBRARIES\n#  Atlas_LIBRARYRARY_DIRS\n\nset(Atlas_INCLUDE_SEARCH_PATHS\n  /usr/include/atlas\n  /usr/include/atlas-base\n  $ENV{Atlas_ROOT_DIR}\n  $ENV{Atlas_ROOT_DIR}/include\n  $ENV{Atlas_ROOT_DIR}/include/atlas\n)\n\nset(Atlas_LIB_SEARCH_PATHS\n  /usr/lib/atlas\n  /usr/lib/atlas-base\n  $ENV{Atlas_ROOT_DIR}\n  $ENV{Atlas_ROOT_DIR}/lib\n)\n\nfind_path(Atlas_CBLAS_INCLUDE_DIR   NAMES cblas.h   PATHS ${Atlas_INCLUDE_SEARCH_PATHS})\nfind_library(Atlas_CBLAS_LIBRARY NAMES  ptcblas_r ptcblas cblas_r cblas PATHS ${Atlas_LIB_SEARCH_PATHS})\nfind_library(Atlas_BLAS_LIBRARY NAMES   atlas_r   atlas                 PATHS ${Atlas_LIB_SEARCH_PATHS})\n\nset(LOOKED_FOR\n  Atlas_CBLAS_INCLUDE_DIR\n\n  Atlas_CBLAS_LIBRARY\n  Atlas_BLAS_LIBRARY\n)\n\nif(Atlas_NEED_LAPACK)\n  find_path(Atlas_CLAPACK_INCLUDE_DIR NAMES clapack.h PATHS ${Atlas_INCLUDE_SEARCH_PATHS})\n  find_library(Atlas_LAPACK_LIBRARY NAMES alapack_r alapack lapack_atlas  PATHS ${Atlas_LIB_SEARCH_PATHS})\n  set(LOOKED_FOR ${LOOKED_FOR} Atlas_CLAPACK_INCLUDE_DIR Atlas_LAPACK_LIBRARY)\nendif(Atlas_NEED_LAPACK)\n\ninclude(FindPackageHandleStandardArgs)\nfind_package_handle_standard_args(Atlas DEFAULT_MSG ${LOOKED_FOR})\n\nif(ATLAS_FOUND)\n  set(Atlas_INCLUDE_DIR ${Atlas_CBLAS_INCLUDE_DIR} ${Atlas_CLAPACK_INCLUDE_DIR})\n  set(Atlas_LIBRARIES ${Atlas_LAPACK_LIBRARY} ${Atlas_CBLAS_LIBRARY} ${Atlas_BLAS_LIBRARY})\n  mark_as_advanced(${LOOKED_FOR})\n\n  message(STATUS \"Found Atlas (include: ${Atlas_CBLAS_INCLUDE_DIR}, library: ${Atlas_BLAS_LIBRARY})\")\nendif(ATLAS_FOUND)\n"
  },
  {
    "path": "cmake/Modules/FindCUDNN.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\ninclude(FindPackageHandleStandardArgs)\n\nset(CUDNN_ROOT \"/usr/local/cuda/include\" CACHE PATH \"cuDNN root folder\")\n\nfind_path(CUDNN_INCLUDE cudnn.h\n  PATHS ${CUDNN_ROOT} $ENV{CUDNN_ROOT}\n  DOC \"Path to cuDNN include directory.\" )\n\nfind_library(CUDNN_LIBRARY NAMES libcudnn.so cudnn.lib # libcudnn_static.a\n  PATHS ${CUDNN_ROOT} $ENV{CUDNN_ROOT} ${CUDNN_INCLUDE}\n  PATH_SUFFIXES lib lib/x64  cuda/lib cuda/lib64 lib/x64\n  DOC \"Path to cuDNN library.\")\n\nfind_package_handle_standard_args(CUDNN DEFAULT_MSG CUDNN_LIBRARY CUDNN_INCLUDE)\n\nmark_as_advanced(CUDNN_ROOT CUDNN_INCLUDE CUDNN_LIBRARY)\n"
  },
  {
    "path": "cmake/Modules/FindCUTENSOR.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\ninclude(FindPackageHandleStandardArgs)\n\nset(CUTENSOR_ROOT \"/usr/local/cuda\" CACHE PATH \"cuTensor root folder\")\n\nfind_path(CUTENSOR_INCLUDE cutensor.h\n        PATHS ${CUTENSOR_ROOT} $ENV{CUTENSOR_ROOT}\n        DOC \"Path to cuTensor include directory.\" )\n\nfind_library(CUTENSOR_LIBRARY NAMES libcutensor.so # libcutensor_static.a\n        PATHS ${CUTENSOR_ROOT} $ENV{CUTENSOR_ROOT} ${CUTENSOR_INCLUDE}\n        PATH_SUFFIXES lib lib/x64  cuda/lib cuda/lib64 lib/x64\n        DOC \"Path to cuTensor library.\")\n\nfind_package_handle_standard_args(CUTENSOR DEFAULT_MSG CUTENSOR_LIBRARY CUTENSOR_INCLUDE)\n\nmark_as_advanced(CUTENSOR_ROOT CUTENSOR_INCLUDE CUTENSOR_LIBRARY)\n"
  },
  {
    "path": "cmake/Modules/FindGperftools.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Tries to find Gperftools.\n#\n# Usage of this module as follows:\n#\n#     find_package(Gperftools)\n#\n# Variables used by this module, they can change the default behaviour and need\n# to be set before calling find_package:\n#\n#  Gperftools_ROOT_DIR  Set this variable to the root installation of\n#                       Gperftools if the module has problems finding\n#                       the proper installation path.\n#\n# Variables defined by this module:\n#\n#  GPERFTOOLS_FOUND              System has Gperftools libs/headers\n#  GPERFTOOLS_LIBRARIES          The Gperftools libraries (tcmalloc & profiler)\n#  GPERFTOOLS_INCLUDE_DIR        The location of Gperftools headers\n\nfind_library(GPERFTOOLS_TCMALLOC\n  NAMES tcmalloc\n  HINTS ${Gperftools_ROOT_DIR}/lib)\n\nfind_library(GPERFTOOLS_PROFILER\n  NAMES profiler\n  HINTS ${Gperftools_ROOT_DIR}/lib)\n\nfind_library(GPERFTOOLS_TCMALLOC_AND_PROFILER\n  NAMES tcmalloc_and_profiler\n  HINTS ${Gperftools_ROOT_DIR}/lib)\n\nfind_path(GPERFTOOLS_INCLUDE_DIR\n  NAMES gperftools/heap-profiler.h\n  HINTS ${Gperftools_ROOT_DIR}/include)\n\nset(GPERFTOOLS_LIBRARIES ${GPERFTOOLS_TCMALLOC_AND_PROFILER})\n\ninclude(FindPackageHandleStandardArgs)\nfind_package_handle_standard_args(\n  Gperftools\n  DEFAULT_MSG\n  GPERFTOOLS_LIBRARIES\n  GPERFTOOLS_INCLUDE_DIR)\n\nmark_as_advanced(\n  Gperftools_ROOT_DIR\n  GPERFTOOLS_TCMALLOC\n  GPERFTOOLS_PROFILER\n  GPERFTOOLS_TCMALLOC_AND_PROFILER\n  GPERFTOOLS_LIBRARIES\n  GPERFTOOLS_INCLUDE_DIR)\n\n"
  },
  {
    "path": "cmake/Modules/FindJeMalloc.cmake",
    "content": "# Copyright (c)      2014 Thomas Heller\n# Copyright (c) 2007-2012 Hartmut Kaiser\n# Copyright (c) 2010-2011 Matt Anderson\n# Copyright (c) 2011      Bryce Lelbach\n#\n#----\n# Distributed under the Boost Software License, Version 1.0.\n# Boost Software License - Version 1.0 - August 17th, 2003\n#\n# Permission is hereby granted, free of charge, to any person or organization\n# obtaining a copy of the software and accompanying documentation covered by\n# this license (the \"Software\") to use, reproduce, display, distribute,\n# execute, and transmit the Software, and to prepare derivative works of the\n# Software, and to permit third-parties to whom the Software is furnished to\n# do so, all subject to the following:\n#\n# The copyright notices in the Software and this entire statement, including\n# the above license grant, this restriction and the following disclaimer,\n# must be included in all copies of the Software, in whole or in part, and\n# all derivative works of the Software, unless such copies or derivative\n# works are solely in the form of machine-executable object code generated by\n# a source language processor.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n# FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT\n# SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE\n# FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,\n# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n# DEALINGS IN THE SOFTWARE.\n\nfind_package(PkgConfig)\npkg_check_modules(PC_JEMALLOC QUIET jemalloc)\n\nfind_path(JEMALLOC_INCLUDE_DIR jemalloc/jemalloc.h\n  HINTS\n    ${JEMALLOC_ROOT} ENV JEMALLOC_ROOT\n    ${PC_JEMALLOC_MINIMAL_INCLUDEDIR}\n    ${PC_JEMALLOC_MINIMAL_INCLUDE_DIRS}\n    ${PC_JEMALLOC_INCLUDEDIR}\n    ${PC_JEMALLOC_INCLUDE_DIRS}\n  PATH_SUFFIXES include)\n\nfind_library(JEMALLOC_LIBRARY NAMES jemalloc libjemalloc\n  HINTS\n    ${JEMALLOC_ROOT} ENV JEMALLOC_ROOT\n    ${PC_JEMALLOC_MINIMAL_LIBDIR}\n    ${PC_JEMALLOC_MINIMAL_LIBRARY_DIRS}\n    ${PC_JEMALLOC_LIBDIR}\n    ${PC_JEMALLOC_LIBRARY_DIRS}\n  PATH_SUFFIXES lib lib64)\n\nset(JEMALLOC_LIBRARIES ${JEMALLOC_LIBRARY})\nset(JEMALLOC_INCLUDE_DIRS ${JEMALLOC_INCLUDE_DIR})\n\nfind_package_handle_standard_args(Jemalloc DEFAULT_MSG\n  JEMALLOC_LIBRARY JEMALLOC_INCLUDE_DIR)\n\nget_property(_type CACHE JEMALLOC_ROOT PROPERTY TYPE)\nif(_type)\n  set_property(CACHE JEMALLOC_ROOT PROPERTY ADVANCED 1)\n  if(\"x${_type}\" STREQUAL \"xUNINITIALIZED\")\n    set_property(CACHE JEMALLOC_ROOT PROPERTY TYPE PATH)\n  endif()\nendif()\n\nmark_as_advanced(JEMALLOC_ROOT JEMALLOC_LIBRARY JEMALLOC_INCLUDE_DIR)\n"
  },
  {
    "path": "cmake/Modules/FindNCCL.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Find the nccl libraries\n#\n# The following variables are optionally searched for defaults\n#  NCCL_ROOT_DIR: Base directory where all NCCL components are found\n#  NCCL_INCLUDE_DIR: Directory where NCCL header is found\n#  NCCL_LIB_DIR: Directory where NCCL library is found\n#\n# The following are set after configuration is done:\n#  NCCL_FOUND\n#  NCCL_INCLUDE_DIRS\n#  NCCL_LIBRARIES\n#\n# The path hints include CUDA_TOOLKIT_ROOT_DIR seeing as some folks\n# install NCCL in the same location as the CUDA toolkit.\n# See https://github.com/caffe2/caffe2/issues/1601\n\nif ($ENV{NCCL_ROOT_DIR})\n  message(WARNING \"NCCL_ROOT_DIR is deprecated. Please set NCCL_ROOT instead.\")\nendif()\n\nfind_path(NCCL_INCLUDE_DIRS\n  NAMES nccl.h\n  HINTS\n  ${NCCL_INCLUDE_DIR}\n  ${NCCL_ROOT_DIR}\n  ${NCCL_ROOT_DIR}/include\n  ${CUDA_TOOLKIT_ROOT_DIR}/include\n  $ENV{NCCL_DIR}/include\n  )\n\nif(CMAKE_BUILD_TYPE STREQUAL \"Distribution\" AND UNIX)\n  set(NCCL_LIB_NAME \"nccl_static\")\nelse()\n  set(NCCL_LIB_NAME \"nccl\")\nendif()\n\nfind_library(NCCL_LIBRARIES\n  NAMES ${NCCL_LIB_NAME}\n  HINTS\n  ${NCCL_LIB_DIR}\n  ${NCCL_ROOT_DIR}\n  ${NCCL_ROOT_DIR}/lib\n  ${NCCL_ROOT_DIR}/lib/x86_64-linux-gnu\n  ${NCCL_ROOT_DIR}/lib64\n  ${CUDA_TOOLKIT_ROOT_DIR}/lib64\n  $ENV{NCCL_DIR}/lib\n  )\n\n# if not found in any of the above paths, finally, check in the /usr/local/cuda for UNIX systems\nif (UNIX)\n  set (search_paths \"/usr/local/cuda\")\n\n  find_path(NCCL_INCLUDE_DIRS\n    NAMES nccl.h\n    PATHS ${search_paths}\n    PATH_SUFFIXES include\n  )\n\n  find_library(NCCL_LIBRARIES\n    NAMES ${NCCL_LIB_NAME}\n    PATHS ${search_paths}\n    PATH_SUFFIXES lib\n  )\nendif()\n\ninclude(FindPackageHandleStandardArgs)\nfind_package_handle_standard_args(NCCL DEFAULT_MSG NCCL_INCLUDE_DIRS NCCL_LIBRARIES)\n\nif(NCCL_FOUND)\n  message(STATUS \"Found NCCL (include: ${NCCL_INCLUDE_DIRS}, library: ${NCCL_LIBRARIES})\")\n  mark_as_advanced(NCCL_ROOT_DIR NCCL_INCLUDE_DIRS NCCL_LIBRARIES)\nendif()\n\n"
  },
  {
    "path": "cmake/Modules/FindNVML.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Find the nvml libraries\n#\n# The following variables are optionally searched for defaults\n#  NVML_ROOT_DIR: Base directory where all NVML components are found\n#  NVML_INCLUDE_DIR: Directory where NVML header is found\n#  NVML_LIB_DIR: Directory where NVML library is found\n#\n# The following are set after configuration is done:\n#  NVML_FOUND\n#  NVML_INCLUDE_DIRS\n#  NVML_LIBRARIES\n#\n# The path hints include CUDA_TOOLKIT_ROOT_DIR seeing as some folks\n# install NVML in the same location as the CUDA toolkit.\n# See https://github.com/caffe2/caffe2/issues/1601\n\nif ($ENV{NVML_ROOT_DIR})\n  message(WARNING \"NVML_ROOT_DIR is deprecated. Please set NVML_ROOT instead.\")\nendif()\n\nfind_path(NVML_INCLUDE_DIRS\n  NAMES nvml.h\n  HINTS\n  ${NVML_INCLUDE_DIR}\n  ${NVML_ROOT_DIR}\n  ${NVML_ROOT_DIR}/include\n  ${CUDA_TOOLKIT_ROOT_DIR}/include\n  $ENV{NVML_DIR}/include\n  )\n\nfind_library(NVML_LIBRARIES\n  NAMES nvidia-ml\n  HINTS\n  ${NVML_LIB_DIR}\n  ${NVML_ROOT_DIR}\n  ${NVML_ROOT_DIR}/lib\n  ${NVML_ROOT_DIR}/lib/x86_64-linux-gnu\n  ${NVML_ROOT_DIR}/lib64\n  ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs\n  $ENV{NVML_DIR}/lib\n  )\n\n# if not found in any of the above paths, finally, check in the /usr/local/cuda for UNIX systems\nif (UNIX)\n  set (search_paths \"/usr/local/cuda\")\n\n  find_path(NVML_INCLUDE_DIRS\n    NAMES nvml.h\n    PATHS ${search_paths}\n    PATH_SUFFIXES include\n  )\n\n  find_library(NVML_LIBRARIES\n    NAMES nvidia-ml\n    PATHS ${search_paths}\n    PATH_SUFFIXES lib64/stubs\n  )\nendif()\n\ninclude(FindPackageHandleStandardArgs)\nfind_package_handle_standard_args(NVML DEFAULT_MSG NVML_INCLUDE_DIRS NVML_LIBRARIES)\n\nif(NVML_FOUND)\n  message(STATUS \"Found NVML (include: ${NVML_INCLUDE_DIRS}, library: ${NVML_LIBRARIES})\")\n  mark_as_advanced(NVML_ROOT_DIR NVML_INCLUDE_DIRS NVML_LIBRARIES)\nendif()\n\n"
  },
  {
    "path": "cmake/Modules/FindNVTX.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nfind_path(NVTX_INCLUDE_DIRS\n  NAMES nvToolsExt.h\n  PATHS $ENV{NVTOOLSEXT_PATH} ${NVTX_ROOT_DIR}  ${CUDA_TOOLKIT_ROOT_DIR}\n  PATH_SUFFIXES include\n  )\n\nfind_library(NVTX_LIBRARIES\n  NAMES nvToolsExt64_1.lib nvToolsExt32_1.lib nvToolsExt\n  PATHS $ENV{NVTOOLSEXT_PATH} ${NVTX_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR}\n  PATH_SUFFIXES lib lib64 lib/Win32 lib/x64\n  )\n\ninclude(FindPackageHandleStandardArgs)\nfind_package_handle_standard_args(NVTX DEFAULT_MSG NVTX_INCLUDE_DIRS NVTX_LIBRARIES)\n\nif(NVTX_FOUND)\n  message(STATUS \"Found NVTX (include: ${NVTX_INCLUDE_DIRS}, library: ${NVTX_LIBRARIES})\")\n  mark_as_advanced(NVTX_ROOT_DIR NVTX_INCLUDE_DIRS NVTX_LIBRARIES)\nendif()\n"
  },
  {
    "path": "cmake/Modules/FindOpenBLAS.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nfile(TO_CMAKE_PATH \"$ENV{OpenBLAS_HOME}\" OpenBLAS_HOME)\nfile(TO_CMAKE_PATH \"$ENV{OpenBLAS}\" OpenBLAS_DIR)\n\nSET(Open_BLAS_INCLUDE_SEARCH_PATHS\n  ${OpenBLAS_HOME}\n  ${OpenBLAS_HOME}/include\n  /usr/include\n  /usr/include/openblas\n  /usr/include/openblas-base\n  /usr/local/include\n  /usr/local/include/openblas\n  /usr/local/include/openblas-base\n  /opt/OpenBLAS/include\n  /usr/local/opt/openblas/include\n  ${PROJECT_SOURCE_DIR}/3rdparty/OpenBLAS/include\n  ${PROJECT_SOURCE_DIR}/thirdparty/OpenBLAS/include\n)\n\nSET(Open_BLAS_LIB_SEARCH_PATHS\n        ${OpenBLAS_HOME}\n        ${OpenBLAS_HOME}/lib\n        /lib/\n        /lib/openblas-base\n        /lib64/\n        /usr/lib\n        /usr/lib/openblas-base\n        /usr/lib64\n        /usr/local/lib\n        /usr/local/lib64\n        /opt/OpenBLAS/lib\n        /usr/local/opt/openblas/lib\n        ${PROJECT_SOURCE_DIR}/3rdparty/OpenBLAS/lib\n        ${PROJECT_SOURCE_DIR}/thirdparty/OpenBLAS/lib\n\t${OpenBLAS_DIR}\n\t${OpenBLAS_DIR}/lib\n )\n\nFIND_PATH(OpenBLAS_INCLUDE_DIR NAMES cblas.h HINTS ${Open_BLAS_INCLUDE_SEARCH_PATHS})\nFIND_LIBRARY(OpenBLAS_LIB NAMES libopenblas.a HINTS ${Open_BLAS_LIB_SEARCH_PATHS})\nIF(NOT OpenBLAS_LIB)\n\tFIND_FILE(OpenBLAS_LIB NAMES libopenblas.dll.a PATHS ${Open_BLAS_LIB_SEARCH_PATHS})\nENDIF()\n\nSET(OpenBLAS_FOUND ON)\n\n#    Check include files\nIF(NOT OpenBLAS_INCLUDE_DIR)\n    SET(OpenBLAS_FOUND OFF)\n    MESSAGE(STATUS \"Could not find OpenBLAS include. Turning OpenBLAS_FOUND off\")\nENDIF()\n\n#    Check libraries\nIF(NOT OpenBLAS_LIB)\n    SET(OpenBLAS_FOUND OFF)\n    MESSAGE(STATUS \"Could not find OpenBLAS lib. Turning OpenBLAS_FOUND off\")\nENDIF()\n\nIF (OpenBLAS_FOUND)\n  IF (NOT OpenBLAS_FIND_QUIETLY)\n    MESSAGE(STATUS \"Found OpenBLAS libraries: ${OpenBLAS_LIB}\")\n    MESSAGE(STATUS \"Found OpenBLAS include: ${OpenBLAS_INCLUDE_DIR}\")\n  ENDIF (NOT OpenBLAS_FIND_QUIETLY)\nELSE (OpenBLAS_FOUND)\n  IF (OpenBLAS_FIND_REQUIRED)\n    MESSAGE(FATAL_ERROR \"Could not find OpenBLAS\")\n  ENDIF (OpenBLAS_FIND_REQUIRED)\nENDIF (OpenBLAS_FOUND)\n\nMARK_AS_ADVANCED(\n    OpenBLAS_INCLUDE_DIR\n    OpenBLAS_LIB\n    OpenBLAS\n)\n\n"
  },
  {
    "path": "cmake/Utils.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# For cmake_parse_arguments\ninclude(CMakeParseArguments)\n\n################################################################################################\n# Command alias for debugging messages\n# Usage:\n#   dmsg(<message>)\nfunction(dmsg)\n  message(STATUS ${ARGN})\nendfunction()\n\n################################################################################################\n# Removes duplicates from list(s)\n# Usage:\n#   mxnet_list_unique(<list_variable> [<list_variable>] [...])\nmacro(mxnet_list_unique)\n  foreach(__lst ${ARGN})\n    if(${__lst})\n      list(REMOVE_DUPLICATES ${__lst})\n    endif()\n  endforeach()\nendmacro()\n\n################################################################################################\n# Clears variables from list\n# Usage:\n#   mxnet_clear_vars(<variables_list>)\nmacro(mxnet_clear_vars)\n  foreach(_var ${ARGN})\n    unset(${_var})\n  endforeach()\nendmacro()\n\n################################################################################################\n# Removes duplicates from string\n# Usage:\n#   mxnet_string_unique(<string_variable>)\nfunction(mxnet_string_unique __string)\n  if(${__string})\n    set(__list ${${__string}})\n    separate_arguments(__list)\n    list(REMOVE_DUPLICATES __list)\n    foreach(__e ${__list})\n      set(__str \"${__str} ${__e}\")\n    endforeach()\n    set(${__string} ${__str} PARENT_SCOPE)\n  endif()\nendfunction()\n\n################################################################################################\n# Prints list element per line\n# Usage:\n#   mxnet_print_list(<list>)\nfunction(mxnet_print_list)\n  foreach(e ${ARGN})\n    message(STATUS ${e})\n  endforeach()\nendfunction()\n\n################################################################################################\n# Function merging lists of compiler flags to single string.\n# Usage:\n#   mxnet_merge_flag_lists(out_variable <list1> [<list2>] [<list3>] ...)\nfunction(mxnet_merge_flag_lists out_var)\n  set(__result \"\")\n  foreach(__list ${ARGN})\n    foreach(__flag ${${__list}})\n      string(STRIP ${__flag} __flag)\n      set(__result \"${__result} ${__flag}\")\n    endforeach()\n  endforeach()\n  string(STRIP ${__result} __result)\n  set(${out_var} ${__result} PARENT_SCOPE)\nendfunction()\n\n################################################################################################\n# Converts all paths in list to absolute\n# Usage:\n#   mxnet_convert_absolute_paths(<list_variable>)\nfunction(mxnet_convert_absolute_paths variable)\n  set(__dlist \"\")\n  foreach(__s ${${variable}})\n    get_filename_component(__abspath ${__s} ABSOLUTE)\n    list(APPEND __list ${__abspath})\n  endforeach()\n  set(${variable} ${__list} PARENT_SCOPE)\nendfunction()\n\n################################################################################################\n# Reads set of version defines from the header file\n# Usage:\n#   mxnet_parse_header(<file> <define1> <define2> <define3> ..)\nmacro(mxnet_parse_header FILENAME FILE_VAR)\n  set(vars_regex \"\")\n  set(__parnet_scope OFF)\n  set(__add_cache OFF)\n  foreach(name ${ARGN})\n    if(\"${name}\" STREQUAL \"PARENT_SCOPE\")\n      set(__parnet_scope ON)\n    elseif(\"${name}\" STREQUAL \"CACHE\")\n      set(__add_cache ON)\n    elseif(vars_regex)\n      set(vars_regex \"${vars_regex}|${name}\")\n    else()\n      set(vars_regex \"${name}\")\n    endif()\n  endforeach()\n  if(EXISTS \"${FILENAME}\")\n    file(STRINGS \"${FILENAME}\" ${FILE_VAR} REGEX \"#define[ \\t]+(${vars_regex})[ \\t]+[0-9]+\" )\n  else()\n    unset(${FILE_VAR})\n  endif()\n  foreach(name ${ARGN})\n    if(NOT \"${name}\" STREQUAL \"PARENT_SCOPE\" AND NOT \"${name}\" STREQUAL \"CACHE\")\n      if(${FILE_VAR})\n        if(${FILE_VAR} MATCHES \".+[ \\t]${name}[ \\t]+([0-9]+).*\")\n          string(REGEX REPLACE \".+[ \\t]${name}[ \\t]+([0-9]+).*\" \"\\\\1\" ${name} \"${${FILE_VAR}}\")\n        else()\n          set(${name} \"\")\n        endif()\n        if(__add_cache)\n          set(${name} ${${name}} CACHE INTERNAL \"${name} parsed from ${FILENAME}\" FORCE)\n        elseif(__parnet_scope)\n          set(${name} \"${${name}}\" PARENT_SCOPE)\n        endif()\n      else()\n        unset(${name} CACHE)\n      endif()\n    endif()\n  endforeach()\nendmacro()\n\n################################################################################################\n# Reads single version define from the header file and parses it\n# Usage:\n#   mxnet_parse_header_single_define(<library_name> <file> <define_name>)\nfunction(mxnet_parse_header_single_define LIBNAME HDR_PATH VARNAME)\n  set(${LIBNAME}_H \"\")\n  if(EXISTS \"${HDR_PATH}\")\n    file(STRINGS \"${HDR_PATH}\" ${LIBNAME}_H REGEX \"^#define[ \\t]+${VARNAME}[ \\t]+\\\"[^\\\"]*\\\".*$\" LIMIT_COUNT 1)\n  endif()\n\n  if(${LIBNAME}_H)\n    string(REGEX REPLACE \"^.*[ \\t]${VARNAME}[ \\t]+\\\"([0-9]+).*$\" \"\\\\1\" ${LIBNAME}_VERSION_MAJOR \"${${LIBNAME}_H}\")\n    string(REGEX REPLACE \"^.*[ \\t]${VARNAME}[ \\t]+\\\"[0-9]+\\\\.([0-9]+).*$\" \"\\\\1\" ${LIBNAME}_VERSION_MINOR  \"${${LIBNAME}_H}\")\n    string(REGEX REPLACE \"^.*[ \\t]${VARNAME}[ \\t]+\\\"[0-9]+\\\\.[0-9]+\\\\.([0-9]+).*$\" \"\\\\1\" ${LIBNAME}_VERSION_PATCH \"${${LIBNAME}_H}\")\n    set(${LIBNAME}_VERSION_MAJOR ${${LIBNAME}_VERSION_MAJOR} ${ARGN} PARENT_SCOPE)\n    set(${LIBNAME}_VERSION_MINOR ${${LIBNAME}_VERSION_MINOR} ${ARGN} PARENT_SCOPE)\n    set(${LIBNAME}_VERSION_PATCH ${${LIBNAME}_VERSION_PATCH} ${ARGN} PARENT_SCOPE)\n    set(${LIBNAME}_VERSION_STRING \"${${LIBNAME}_VERSION_MAJOR}.${${LIBNAME}_VERSION_MINOR}.${${LIBNAME}_VERSION_PATCH}\" PARENT_SCOPE)\n\n    # append a TWEAK version if it exists:\n    set(${LIBNAME}_VERSION_TWEAK \"\")\n    if(\"${${LIBNAME}_H}\" MATCHES \"^.*[ \\t]${VARNAME}[ \\t]+\\\"[0-9]+\\\\.[0-9]+\\\\.[0-9]+\\\\.([0-9]+).*$\")\n      set(${LIBNAME}_VERSION_TWEAK \"${CMAKE_MATCH_1}\" ${ARGN} PARENT_SCOPE)\n    endif()\n    if(${LIBNAME}_VERSION_TWEAK)\n      set(${LIBNAME}_VERSION_STRING \"${${LIBNAME}_VERSION_STRING}.${${LIBNAME}_VERSION_TWEAK}\" ${ARGN} PARENT_SCOPE)\n    else()\n      set(${LIBNAME}_VERSION_STRING \"${${LIBNAME}_VERSION_STRING}\" ${ARGN} PARENT_SCOPE)\n    endif()\n  endif()\nendfunction()\n\n################################################################################################\n# Utility macro for comparing two lists. Used for CMake debugging purposes\n# Usage:\n#   mxnet_compare_lists(<list_variable> <list2_variable> [description])\nfunction(mxnet_compare_lists list1 list2 desc)\n  set(__list1 ${${list1}})\n  set(__list2 ${${list2}})\n  list(SORT __list1)\n  list(SORT __list2)\n  list(LENGTH __list1 __len1)\n  list(LENGTH __list2 __len2)\n\n  if(NOT ${__len1} EQUAL ${__len2})\n    message(FATAL_ERROR \"Lists are not equal. ${__len1} != ${__len2}. ${desc}\")\n  endif()\n\n  foreach(__i RANGE 1 ${__len1})\n    math(EXPR __index \"${__i}- 1\")\n    list(GET __list1 ${__index} __item1)\n    list(GET __list2 ${__index} __item2)\n    if(NOT ${__item1} STREQUAL ${__item2})\n      message(FATAL_ERROR \"Lists are not equal. Differ at element ${__index}. ${desc}\")\n    endif()\n  endforeach()\nendfunction()\n\n################################################################################################\n# Command for disabling warnings for different platforms (see below for gcc and VisualStudio)\n# Usage:\n#   mxnet_warnings_disable(<CMAKE_[C|CXX]_FLAGS[_CONFIGURATION]> -Wshadow /wd4996 ..,)\nmacro(mxnet_warnings_disable)\n  set(_flag_vars \"\")\n  set(_msvc_warnings \"\")\n  set(_gxx_warnings \"\")\n\n  foreach(arg ${ARGN})\n    if(arg MATCHES \"^CMAKE_\")\n      list(APPEND _flag_vars ${arg})\n    elseif(arg MATCHES \"^/wd\")\n      list(APPEND _msvc_warnings ${arg})\n    elseif(arg MATCHES \"^-W\")\n      list(APPEND _gxx_warnings ${arg})\n    endif()\n  endforeach()\n\n  if(NOT _flag_vars)\n    set(_flag_vars CMAKE_C_FLAGS CMAKE_CXX_FLAGS)\n  endif()\n\n  if(MSVC AND _msvc_warnings)\n    foreach(var ${_flag_vars})\n      foreach(warning ${_msvc_warnings})\n        set(${var} \"${${var}} ${warning}\")\n      endforeach()\n    endforeach()\n  elseif((CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANGXX) AND _gxx_warnings)\n    foreach(var ${_flag_vars})\n      foreach(warning ${_gxx_warnings})\n        if(NOT warning MATCHES \"^-Wno-\")\n          string(REPLACE \"${warning}\" \"\" ${var} \"${${var}}\")\n          string(REPLACE \"-W\" \"-Wno-\" warning \"${warning}\")\n        endif()\n        set(${var} \"${${var}} ${warning}\")\n      endforeach()\n    endforeach()\n  endif()\n  mxnet_clear_vars(_flag_vars _msvc_warnings _gxx_warnings)\nendmacro()\n\n################################################################################################\n# Helper function get current definitions\n# Usage:\n#   mxnet_get_current_definitions(<definitions_variable>)\nfunction(mxnet_get_current_definitions definitions_var)\n  get_property(current_definitions DIRECTORY PROPERTY COMPILE_DEFINITIONS)\n  set(result \"\")\n\n  foreach(d ${current_definitions})\n    list(APPEND result -D${d})\n  endforeach()\n\n  mxnet_list_unique(result)\n  set(${definitions_var} ${result} PARENT_SCOPE)\nendfunction()\n\n################################################################################################\n# Helper function get current includes/definitions\n# Usage:\n#   mxnet_get_current_cflags(<cflagslist_variable>)\nfunction(mxnet_get_current_cflags cflags_var)\n  get_property(current_includes DIRECTORY PROPERTY INCLUDE_DIRECTORIES)\n  mxnet_convert_absolute_paths(current_includes)\n  mxnet_get_current_definitions(cflags)\n\n  foreach(i ${current_includes})\n    list(APPEND cflags \"-I${i}\")\n  endforeach()\n\n  mxnet_list_unique(cflags)\n  set(${cflags_var} ${cflags} PARENT_SCOPE)\nendfunction()\n\n################################################################################################\n# Helper function to parse current linker libs into link directories, libflags and osx frameworks\n# Usage:\n#   mxnet_parse_linker_libs(<mxnet_LINKER_LIBS_var> <directories_var> <libflags_var> <frameworks_var>)\nfunction(mxnet_parse_linker_libs mxnet_LINKER_LIBS_variable folders_var flags_var frameworks_var)\n\n  set(__unspec \"\")\n  set(__debug \"\")\n  set(__optimized \"\")\n  set(__framework \"\")\n  set(__varname \"__unspec\")\n\n  # split libs into debug, optimized, unspecified and frameworks\n  foreach(list_elem ${${mxnet_LINKER_LIBS_variable}})\n    if(list_elem STREQUAL \"debug\")\n      set(__varname \"__debug\")\n    elseif(list_elem STREQUAL \"optimized\")\n      set(__varname \"__optimized\")\n    elseif(list_elem MATCHES \"^-framework[ \\t]+([^ \\t].*)\")\n      list(APPEND __framework -framework ${CMAKE_MATCH_1})\n    else()\n      list(APPEND ${__varname} ${list_elem})\n      set(__varname \"__unspec\")\n    endif()\n  endforeach()\n\n  # attach debug or optimized libs to unspecified according to current configuration\n  if(CMAKE_BUILD_TYPE MATCHES \"Debug\")\n    set(__libs ${__unspec} ${__debug})\n  else()\n    set(__libs ${__unspec} ${__optimized})\n  endif()\n\n  set(libflags \"\")\n  set(folders \"\")\n\n  # convert linker libraries list to link flags\n  foreach(lib ${__libs})\n    if(TARGET ${lib})\n      list(APPEND folders $<TARGET_LINKER_FILE_DIR:${lib}>)\n      list(APPEND libflags -l${lib})\n    elseif(lib MATCHES \"^-l.*\")\n      list(APPEND libflags ${lib})\n    elseif(IS_ABSOLUTE ${lib})\n      get_filename_component(name_we ${lib} NAME_WE)\n      get_filename_component(folder  ${lib} PATH)\n\n      string(REGEX MATCH \"^lib(.*)\" __match ${name_we})\n      list(APPEND libflags -l${CMAKE_MATCH_1})\n      list(APPEND folders    ${folder})\n    else()\n      message(FATAL_ERROR \"Logic error. Need to update cmake script\")\n    endif()\n  endforeach()\n\n  mxnet_list_unique(libflags folders)\n\n  set(${folders_var} ${folders} PARENT_SCOPE)\n  set(${flags_var} ${libflags} PARENT_SCOPE)\n  set(${frameworks_var} ${__framework} PARENT_SCOPE)\nendfunction()\n\n################################################################################################\n# Helper function to detect Darwin version, i.e. 10.8, 10.9, 10.10, ....\n# Usage:\n#   mxnet_detect_darwin_version(<version_variable>)\nfunction(mxnet_detect_darwin_version output_var)\n  if(APPLE)\n    execute_process(COMMAND /usr/bin/sw_vers -productVersion\n                    RESULT_VARIABLE __sw_vers OUTPUT_VARIABLE __sw_vers_out\n                    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)\n\n    set(${output_var} ${__sw_vers_out} PARENT_SCOPE)\n  else()\n    set(${output_var} \"\" PARENT_SCOPE)\n  endif()\nendfunction()\n\n################################################################################################\n# Convenient command to setup source group for IDEs that support this feature (VS, XCode)\n# Usage:\n#   caffe_source_group(<group> GLOB[_RECURSE] <globbing_expression>)\nfunction(mxnet_source_group group)\n  message(WARNING \"mxnet_source_group function is obsolete, it not do anything now.\")\nendfunction()\n\n\nfunction(assign_source_group group)\n    foreach(_source IN ITEMS ${ARGN})\n        if(IS_ABSOLUTE \"${_source}\")\n            file(RELATIVE_PATH _source_rel \"${CMAKE_CURRENT_SOURCE_DIR}\" \"${_source}\")\n        else()\n            set(_source_rel \"${_source}\")\n        endif()\n        get_filename_component(_source_path \"${_source_rel}\" PATH)\n        string(REPLACE \"/\" \"\\\\\" _source_path_msvc \"${_source_path}\")\n        source_group(\"${group}\\\\${_source_path_msvc}\" FILES \"${_source}\")\n    endforeach()\nendfunction(assign_source_group)\n"
  },
  {
    "path": "cmake/libmxnet.sym",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n*MX*\n*NN*\n*mx*\n*nn*\n*NDArray*\n*Engine*Get*\n*Storage*Get*\n*on_enter_api*\n*on_exit_api*\n"
  },
  {
    "path": "cmake/upstream/FindBLAS.cmake",
    "content": "# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying\n# file Copyright.txt or https://cmake.org/licensing for details.\n\n#[=======================================================================[.rst:\nFindBLAS\n--------\n\nFind Basic Linear Algebra Subprograms (BLAS) library\n\nThis module finds an installed Fortran library that implements the\nBLAS linear-algebra interface (see http://www.netlib.org/blas/).\n\nThe approach follows that taken for the ``autoconf`` macro file,\n``acx_blas.m4`` (distributed at\nhttp://ac-archive.sourceforge.net/ac-archive/acx_blas.html).\n\nInput Variables\n^^^^^^^^^^^^^^^\n\nThe following variables may be set to influence this module's behavior:\n\n``BLA_STATIC``\n  if ``ON`` use static linkage\n\n``BLA_VENDOR``\n  If set, checks only the specified vendor, if not set checks all the\n  possibilities.  List of vendors valid in this module:\n\n  * ``Goto``\n  * ``FlexiBLAS``\n  * ``OpenBLAS``\n  * ``FLAME``\n  * ``ATLAS PhiPACK``\n  * ``CXML``\n  * ``DXML``\n  * ``SunPerf``\n  * ``SCSL``\n  * ``SGIMATH``\n  * ``IBMESSL``\n  * ``Intel10_32`` (intel mkl v10 32 bit, threaded code)\n  * ``Intel10_64lp`` (intel mkl v10+ 64 bit, threaded code, lp64 model)\n  * ``Intel10_64lp_seq`` (intel mkl v10+ 64 bit, sequential code, lp64 model)\n  * ``Intel10_64ilp`` (intel mkl v10+ 64 bit, threaded code, ilp64 model)\n  * ``Intel10_64ilp_seq`` (intel mkl v10+ 64 bit, sequential code, ilp64 model)\n  * ``Intel10_64_dyn`` (intel mkl v10+ 64 bit, single dynamic library)\n  * ``Intel`` (obsolete versions of mkl 32 and 64 bit)\n  * ``ACML``\n  * ``ACML_MP``\n  * ``ACML_GPU``\n  * ``Apple``\n  * ``NAS``\n  * ``Arm``\n  * ``Arm_mp``\n  * ``Arm_ilp64``\n  * ``Arm_ilp64_mp``\n  * ``EML``\n  * ``EML_mt``\n  * ``Generic``\n\n  .. versionadded:: 3.6\n    ``OpenBLAS`` support.\n\n  .. versionadded:: 3.11\n    ``FLAME`` support.\n\n  .. versionadded:: 3.13\n    Added ILP64 MKL variants (``Intel10_64ilp``, ``Intel10_64ilp_seq``).\n\n  .. versionadded:: 3.17\n    Added single dynamic library MKL variant (``Intel10_64_dyn``).\n\n  .. versionadded:: 3.18\n    Arm Performance Libraries support (``Arm``, ``Arm_mp``, ``Arm_ilp64``,\n    ``Arm_ilp64_mp``).\n\n  .. versionadded:: 3.19\n    ``FlexiBLAS`` support.\n\n  .. versionadded:: 3.20\n    Elbrus Math Library support (``EML``, ``EML_mt``).\n\n``BLA_F95``\n  if ``ON`` tries to find the BLAS95 interfaces\n\n``BLA_PREFER_PKGCONFIG``\n  .. versionadded:: 3.11\n\n  if set ``pkg-config`` will be used to search for a BLAS library first\n  and if one is found that is preferred\n\nImported targets\n^^^^^^^^^^^^^^^^\n\n.. versionadded:: 3.18\n\nThis module defines the following :prop_tgt:`IMPORTED` target:\n\n``BLAS::BLAS``\n  The libraries to use for BLAS, if found.\n\n\nResult Variables\n^^^^^^^^^^^^^^^^\n\nThis module defines the following variables:\n\n``BLAS_FOUND``\n  library implementing the BLAS interface is found\n``BLAS_LINKER_FLAGS``\n  uncached list of required linker flags (excluding ``-l`` and ``-L``).\n``BLAS_LIBRARIES``\n  uncached list of libraries (using full path name) to link against\n  to use BLAS (may be empty if compiler implicitly links BLAS)\n``BLAS95_LIBRARIES``\n  uncached list of libraries (using full path name) to link against\n  to use BLAS95 interface\n``BLAS95_FOUND``\n  library implementing the BLAS95 interface is found\n\n.. note::\n\n  C, CXX or Fortran must be enabled to detect a BLAS library.\n  C or CXX must be enabled to use Intel Math Kernel Library (MKL).\n\n  For example, to use Intel MKL libraries and/or Intel compiler:\n\n  .. code-block:: cmake\n\n    set(BLA_VENDOR Intel10_64lp)\n    find_package(BLAS)\n\nHints\n^^^^^\n\n``MKLROOT``\n  .. versionadded:: 3.15\n\n  Set this environment variable to a directory that contains an MKL\n  installation, or add the directory to the dynamic library loader environment\n  variable for your platform (``LIB``, ``DYLD_LIBRARY_PATH`` or\n  ``LD_LIBRARY_PATH``).\n\n#]=======================================================================]\n\n# Check the language being used\nif(NOT (CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED OR CMAKE_Fortran_COMPILER_LOADED))\n  if(BLAS_FIND_REQUIRED)\n    message(FATAL_ERROR \"FindBLAS requires Fortran, C, or C++ to be enabled.\")\n  else()\n    message(STATUS \"Looking for BLAS... - NOT found (Unsupported languages)\")\n    return()\n  endif()\nendif()\n\nfunction(_add_blas_target)\n  if(NOT TARGET BLAS::BLAS)\n    add_library(BLAS::BLAS INTERFACE IMPORTED)\n    if(BLAS_LIBRARIES)\n      set_target_properties(BLAS::BLAS PROPERTIES\n        INTERFACE_LINK_LIBRARIES \"${BLAS_LIBRARIES}\"\n      )\n    endif()\n  endif()\nendfunction()\n\nif(CMAKE_Fortran_COMPILER_LOADED)\n  include(${CMAKE_CURRENT_LIST_DIR}/CheckFortranFunctionExists.cmake)\nelse()\n# MXNET NOTE: This differs from CMake source by ${CMAKE_CURRENT_LIST_DIR}\n# replaced with ${CMAKE_ROOT}/Modules\n  include(CheckFunctionExists)\nendif()\n# MXNET NOTE: The second 2 lines differs from CMake source by ${CMAKE_CURRENT_LIST_DIR}\n# replaced with ${CMAKE_ROOT}/Modules\ninclude(CMakePushCheckState)\ninclude(FindPackageHandleStandardArgs)\ncmake_push_check_state()\nset(CMAKE_REQUIRED_QUIET ${BLAS_FIND_QUIETLY})\n\nif(BLA_PREFER_PKGCONFIG)\n  find_package(PkgConfig)\n  pkg_check_modules(PKGC_BLAS blas)\n  if(PKGC_BLAS_FOUND)\n    set(BLAS_FOUND ${PKGC_BLAS_FOUND})\n    set(BLAS_LIBRARIES \"${PKGC_BLAS_LINK_LIBRARIES}\")\n    _add_blas_target()\n    return()\n  endif()\nendif()\n\nset(_blas_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})\nif(BLA_STATIC)\n  if(WIN32)\n    set(CMAKE_FIND_LIBRARY_SUFFIXES .lib ${CMAKE_FIND_LIBRARY_SUFFIXES})\n  else()\n    set(CMAKE_FIND_LIBRARY_SUFFIXES .a ${CMAKE_FIND_LIBRARY_SUFFIXES})\n  endif()\nelse()\n  if(CMAKE_SYSTEM_NAME STREQUAL \"Linux\")\n    # for ubuntu's libblas3gf and liblapack3gf packages\n    set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES} .so.3gf)\n  endif()\nendif()\n\n# TODO: move this stuff to a separate module\n\nmacro(CHECK_BLAS_LIBRARIES LIBRARIES _prefix _name _flags _list _threadlibs _addlibdir _subdirs)\n  # This macro checks for the existence of the combination of fortran libraries\n  # given by _list.  If the combination is found, this macro checks (using the\n  # Check_Fortran_Function_Exists macro) whether can link against that library\n  # combination using the name of a routine given by _name using the linker\n  # flags given by _flags.  If the combination of libraries is found and passes\n  # the link test, LIBRARIES is set to the list of complete library paths that\n  # have been found.  Otherwise, LIBRARIES is set to FALSE.\n\n  # N.B. _prefix is the prefix applied to the names of all cached variables that\n  # are generated internally and marked advanced by this macro.\n  # _addlibdir is a list of additional search paths. _subdirs is a list of path\n  # suffixes to be used by find_library().\n\n  set(_libraries_work TRUE)\n  set(${LIBRARIES})\n  set(_combined_name)\n\n  set(_extaddlibdir \"${_addlibdir}\")\n  if(WIN32)\n    list(APPEND _extaddlibdir ENV LIB)\n  elseif(APPLE)\n    list(APPEND _extaddlibdir ENV DYLD_LIBRARY_PATH)\n  else()\n    list(APPEND _extaddlibdir ENV LD_LIBRARY_PATH)\n  endif()\n  list(APPEND _extaddlibdir \"${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}\")\n\n  foreach(_library ${_list})\n    if(_library MATCHES \"^-Wl,--(start|end)-group$\")\n      # Respect linker flags like --start/end-group (required by MKL)\n      set(${LIBRARIES} ${${LIBRARIES}} \"${_library}\")\n    else()\n      set(_combined_name ${_combined_name}_${_library})\n      if(NOT \"${_threadlibs}\" STREQUAL \"\")\n        set(_combined_name ${_combined_name}_threadlibs)\n      endif()\n      if(_libraries_work)\n        find_library(${_prefix}_${_library}_LIBRARY\n          NAMES ${_library}\n          NAMES_PER_DIR\n          PATHS ${_extaddlibdir}\n          PATH_SUFFIXES ${_subdirs}\n        )\n        #message(\"DEBUG: find_library(${_library}) got ${${_prefix}_${_library}_LIBRARY}\")\n        mark_as_advanced(${_prefix}_${_library}_LIBRARY)\n        set(${LIBRARIES} ${${LIBRARIES}} ${${_prefix}_${_library}_LIBRARY})\n        set(_libraries_work ${${_prefix}_${_library}_LIBRARY})\n      endif()\n    endif()\n  endforeach()\n\n  if(_libraries_work)\n    # Test this combination of libraries.\n    set(CMAKE_REQUIRED_LIBRARIES ${_flags} ${${LIBRARIES}} ${_threadlibs})\n    #message(\"DEBUG: CMAKE_REQUIRED_LIBRARIES = ${CMAKE_REQUIRED_LIBRARIES}\")\n    if(CMAKE_Fortran_COMPILER_LOADED)\n      check_fortran_function_exists(\"${_name}\" ${_prefix}${_combined_name}_WORKS)\n    else()\n      check_function_exists(\"${_name}_\" ${_prefix}${_combined_name}_WORKS)\n    endif()\n    set(CMAKE_REQUIRED_LIBRARIES)\n    set(_libraries_work ${${_prefix}${_combined_name}_WORKS})\n  endif()\n\n  if(_libraries_work)\n    if(\"${_list}\" STREQUAL \"\")\n      set(${LIBRARIES} \"${LIBRARIES}-PLACEHOLDER-FOR-EMPTY-LIBRARIES\")\n    else()\n      set(${LIBRARIES} ${${LIBRARIES}} ${_threadlibs})\n    endif()\n  else()\n    set(${LIBRARIES} FALSE)\n  endif()\n  #message(\"DEBUG: ${LIBRARIES} = ${${LIBRARIES}}\")\nendmacro()\n\nset(BLAS_LINKER_FLAGS)\nset(BLAS_LIBRARIES)\nset(BLAS95_LIBRARIES)\nif(NOT $ENV{BLA_VENDOR} STREQUAL \"\")\n  set(BLA_VENDOR $ENV{BLA_VENDOR})\nelse()\n  if(NOT BLA_VENDOR)\n    set(BLA_VENDOR \"All\")\n  endif()\nendif()\n\n# Implicitly linked BLAS libraries?\nif(BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\"\n      \"\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\nendif()\n\n# BLAS in the Intel MKL 10+ library?\nif(BLA_VENDOR MATCHES \"Intel\" OR BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    if(CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED)\n      # System-specific settings\n      if(WIN32)\n        if(BLA_STATIC)\n          set(BLAS_mkl_DLL_SUFFIX \"\")\n        else()\n          set(BLAS_mkl_DLL_SUFFIX \"_dll\")\n        endif()\n      else()\n        # MXNET NOTE: The second 2 lines differs from CMake source by ${CMAKE_CURRENT_LIST_DIR}\n        # replaced with ${CMAKE_ROOT}/Modules\n        # https://gitlab.kitware.com/cmake/cmake/-/issues/20548\n        if(BLA_STATIC AND NOT APPLE)\n          set(BLAS_mkl_START_GROUP \"-Wl,--start-group\")\n          set(BLAS_mkl_END_GROUP \"-Wl,--end-group\")\n        else()\n          set(BLAS_mkl_START_GROUP \"\")\n          set(BLAS_mkl_END_GROUP \"\")\n        endif()\n        # Switch to GNU Fortran support layer if needed (but not on Apple, where MKL does not provide it)\n        if(CMAKE_Fortran_COMPILER_LOADED AND CMAKE_Fortran_COMPILER_ID STREQUAL \"GNU\" AND NOT APPLE)\n            set(BLAS_mkl_INTFACE \"gf\")\n            set(BLAS_mkl_THREADING \"gnu\")\n            set(BLAS_mkl_OMP \"gomp\")\n        else()\n            set(BLAS_mkl_INTFACE \"intel\")\n            set(BLAS_mkl_THREADING \"intel\")\n            set(BLAS_mkl_OMP \"iomp5\")\n        endif()\n        set(BLAS_mkl_LM \"-lm\")\n        set(BLAS_mkl_LDL \"-ldl\")\n      endif()\n\n      if(BLAS_FIND_QUIETLY OR NOT BLAS_FIND_REQUIRED)\n        find_package(Threads)\n      else()\n        find_package(Threads REQUIRED)\n      endif()\n\n      if(BLA_VENDOR MATCHES \"_64ilp\")\n        set(BLAS_mkl_ILP_MODE \"ilp64\")\n      else()\n        set(BLAS_mkl_ILP_MODE \"lp64\")\n      endif()\n\n      set(BLAS_SEARCH_LIBS \"\")\n\n      if(BLA_F95)\n        set(BLAS_mkl_SEARCH_SYMBOL \"sgemm_f95\")\n        set(_LIBRARIES BLAS95_LIBRARIES)\n        if(WIN32)\n          # Find the main file (32-bit or 64-bit)\n          set(BLAS_SEARCH_LIBS_WIN_MAIN \"\")\n          if(BLA_VENDOR STREQUAL \"Intel10_32\" OR BLA_VENDOR STREQUAL \"All\")\n            list(APPEND BLAS_SEARCH_LIBS_WIN_MAIN\n              \"mkl_blas95${BLAS_mkl_DLL_SUFFIX} mkl_intel_c${BLAS_mkl_DLL_SUFFIX}\")\n          endif()\n\n          if(BLA_VENDOR MATCHES \"^Intel10_64i?lp\" OR BLA_VENDOR STREQUAL \"All\")\n            list(APPEND BLAS_SEARCH_LIBS_WIN_MAIN\n              \"mkl_blas95_${BLAS_mkl_ILP_MODE}${BLAS_mkl_DLL_SUFFIX} mkl_intel_${BLAS_mkl_ILP_MODE}${BLAS_mkl_DLL_SUFFIX}\")\n          endif()\n\n          # Add threading/sequential libs\n          set(BLAS_SEARCH_LIBS_WIN_THREAD \"\")\n          if(BLA_VENDOR MATCHES \"^Intel10_64i?lp$\" OR BLA_VENDOR STREQUAL \"All\")\n            # old version\n            list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD\n              \"libguide40 mkl_intel_thread${BLAS_mkl_DLL_SUFFIX}\")\n            # mkl >= 10.3\n            list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD\n              \"libiomp5md mkl_intel_thread${BLAS_mkl_DLL_SUFFIX}\")\n          endif()\n          if(BLA_VENDOR MATCHES \"^Intel10_64i?lp_seq$\" OR BLA_VENDOR STREQUAL \"All\")\n            list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD\n              \"mkl_sequential${BLAS_mkl_DLL_SUFFIX}\")\n          endif()\n\n          # Cartesian product of the above\n          foreach(MAIN ${BLAS_SEARCH_LIBS_WIN_MAIN})\n            foreach(THREAD ${BLAS_SEARCH_LIBS_WIN_THREAD})\n              list(APPEND BLAS_SEARCH_LIBS\n                \"${MAIN} ${THREAD} mkl_core${BLAS_mkl_DLL_SUFFIX}\")\n            endforeach()\n          endforeach()\n        else()\n          if(BLA_VENDOR STREQUAL \"Intel10_32\" OR BLA_VENDOR STREQUAL \"All\")\n            # old version\n            list(APPEND BLAS_SEARCH_LIBS\n              \"mkl_blas95 mkl_${BLAS_mkl_INTFACE} mkl_${BLAS_mkl_THREADING}_thread mkl_core guide\")\n\n            # mkl >= 10.3\n            list(APPEND BLAS_SEARCH_LIBS\n              \"${BLAS_mkl_START_GROUP} mkl_blas95 mkl_${BLAS_mkl_INTFACE} mkl_${BLAS_mkl_THREADING}_thread mkl_core ${BLAS_mkl_END_GROUP} ${BLAS_mkl_OMP}\")\n          endif()\n          if(BLA_VENDOR MATCHES \"^Intel10_64i?lp$\" OR BLA_VENDOR STREQUAL \"All\")\n            # old version\n            list(APPEND BLAS_SEARCH_LIBS\n              \"mkl_blas95 mkl_${BLAS_mkl_INTFACE}_${BLAS_mkl_ILP_MODE} mkl_${BLAS_mkl_THREADING}_thread mkl_core guide\")\n\n            # mkl >= 10.3\n            list(APPEND BLAS_SEARCH_LIBS\n              \"${BLAS_mkl_START_GROUP} mkl_blas95_${BLAS_mkl_ILP_MODE} mkl_${BLAS_mkl_INTFACE}_${BLAS_mkl_ILP_MODE} mkl_${BLAS_mkl_THREADING}_thread mkl_core ${BLAS_mkl_END_GROUP} ${BLAS_mkl_OMP}\")\n          endif()\n          if(BLA_VENDOR MATCHES \"^Intel10_64i?lp_seq$\" OR BLA_VENDOR STREQUAL \"All\")\n            list(APPEND BLAS_SEARCH_LIBS\n              \"${BLAS_mkl_START_GROUP} mkl_blas95_${BLAS_mkl_ILP_MODE} mkl_${BLAS_mkl_INTFACE}_${BLAS_mkl_ILP_MODE} mkl_sequential mkl_core ${BLAS_mkl_END_GROUP}\")\n          endif()\n        endif()\n      else()\n        set(BLAS_mkl_SEARCH_SYMBOL sgemm)\n        set(_LIBRARIES BLAS_LIBRARIES)\n        if(WIN32)\n          # Find the main file (32-bit or 64-bit)\n          set(BLAS_SEARCH_LIBS_WIN_MAIN \"\")\n          if(BLA_VENDOR STREQUAL \"Intel10_32\" OR BLA_VENDOR STREQUAL \"All\")\n            list(APPEND BLAS_SEARCH_LIBS_WIN_MAIN\n              \"mkl_intel_c${BLAS_mkl_DLL_SUFFIX}\")\n          endif()\n          if(BLA_VENDOR MATCHES \"^Intel10_64i?lp\" OR BLA_VENDOR STREQUAL \"All\")\n            list(APPEND BLAS_SEARCH_LIBS_WIN_MAIN\n              \"mkl_intel_${BLAS_mkl_ILP_MODE}${BLAS_mkl_DLL_SUFFIX}\")\n          endif()\n\n          # Add threading/sequential libs\n          set(BLAS_SEARCH_LIBS_WIN_THREAD \"\")\n          if(BLA_VENDOR STREQUAL \"Intel10_32\" OR BLA_VENDOR STREQUAL \"All\")\n            list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD\n              \"libiomp5md mkl_intel_thread${BLAS_mkl_DLL_SUFFIX}\")\n          endif()\n          if(BLA_VENDOR MATCHES \"^Intel10_64i?lp$\" OR BLA_VENDOR STREQUAL \"All\")\n            # old version\n            list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD\n              \"libguide40 mkl_intel_thread${BLAS_mkl_DLL_SUFFIX}\")\n            # mkl >= 10.3\n            list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD\n              \"libiomp5md mkl_intel_thread${BLAS_mkl_DLL_SUFFIX}\")\n          endif()\n          if(BLA_VENDOR MATCHES \"^Intel10_64i?lp_seq$\" OR BLA_VENDOR STREQUAL \"All\")\n            list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD\n              \"mkl_sequential${BLAS_mkl_DLL_SUFFIX}\")\n          endif()\n\n          # Cartesian product of the above\n          foreach(MAIN ${BLAS_SEARCH_LIBS_WIN_MAIN})\n            foreach(THREAD ${BLAS_SEARCH_LIBS_WIN_THREAD})\n              list(APPEND BLAS_SEARCH_LIBS\n                \"${MAIN} ${THREAD} mkl_core${BLAS_mkl_DLL_SUFFIX}\")\n            endforeach()\n          endforeach()\n        else()\n          if(BLA_VENDOR STREQUAL \"Intel10_32\" OR BLA_VENDOR STREQUAL \"All\")\n            # old version\n            list(APPEND BLAS_SEARCH_LIBS\n              \"mkl_${BLAS_mkl_INTFACE} mkl_${BLAS_mkl_THREADING}_thread mkl_core guide\")\n\n            # mkl >= 10.3\n            list(APPEND BLAS_SEARCH_LIBS\n              \"${BLAS_mkl_START_GROUP} mkl_${BLAS_mkl_INTFACE} mkl_${BLAS_mkl_THREADING}_thread mkl_core ${BLAS_mkl_END_GROUP} ${BLAS_mkl_OMP}\")\n          endif()\n          if(BLA_VENDOR MATCHES \"^Intel10_64i?lp$\" OR BLA_VENDOR STREQUAL \"All\")\n            # old version\n            list(APPEND BLAS_SEARCH_LIBS\n              \"mkl_${BLAS_mkl_INTFACE}_${BLAS_mkl_ILP_MODE} mkl_${BLAS_mkl_THREADING}_thread mkl_core guide\")\n\n            # mkl >= 10.3\n            list(APPEND BLAS_SEARCH_LIBS\n              \"${BLAS_mkl_START_GROUP} mkl_${BLAS_mkl_INTFACE}_${BLAS_mkl_ILP_MODE} mkl_${BLAS_mkl_THREADING}_thread mkl_core ${BLAS_mkl_END_GROUP} ${BLAS_mkl_OMP}\")\n          endif()\n          if(BLA_VENDOR MATCHES \"^Intel10_64i?lp_seq$\" OR BLA_VENDOR STREQUAL \"All\")\n            list(APPEND BLAS_SEARCH_LIBS\n              \"${BLAS_mkl_START_GROUP} mkl_${BLAS_mkl_INTFACE}_${BLAS_mkl_ILP_MODE} mkl_sequential mkl_core ${BLAS_mkl_END_GROUP}\")\n          endif()\n\n          #older versions of intel mkl libs\n          if(BLA_VENDOR STREQUAL \"Intel\" OR BLA_VENDOR STREQUAL \"All\")\n            list(APPEND BLAS_SEARCH_LIBS\n              \"mkl\")\n            list(APPEND BLAS_SEARCH_LIBS\n              \"mkl_ia32\")\n            list(APPEND BLAS_SEARCH_LIBS\n              \"mkl_em64t\")\n          endif()\n        endif()\n      endif()\n\n      if(BLA_VENDOR MATCHES \"^Intel10_64_dyn$\" OR BLA_VENDOR STREQUAL \"All\")\n        # mkl >= 10.3 with single dynamic library\n        list(APPEND BLAS_SEARCH_LIBS\n          \"mkl_rt\")\n      endif()\n\n      # MKL uses a multitude of partially platform-specific subdirectories:\n      if(BLA_VENDOR STREQUAL \"Intel10_32\")\n        set(BLAS_mkl_ARCH_NAME \"ia32\")\n      else()\n        set(BLAS_mkl_ARCH_NAME \"intel64\")\n      endif()\n      if(WIN32)\n        set(BLAS_mkl_OS_NAME \"win\")\n      elseif(APPLE)\n        set(BLAS_mkl_OS_NAME \"mac\")\n      else()\n        set(BLAS_mkl_OS_NAME \"lin\")\n      endif()\n      if(DEFINED ENV{MKLROOT})\n        file(TO_CMAKE_PATH \"$ENV{MKLROOT}\" BLAS_mkl_MKLROOT)\n        # If MKLROOT points to the subdirectory 'mkl', use the parent directory instead\n        # so we can better detect other relevant libraries in 'compiler' or 'tbb':\n        get_filename_component(BLAS_mkl_MKLROOT_LAST_DIR \"${BLAS_mkl_MKLROOT}\" NAME)\n        if(BLAS_mkl_MKLROOT_LAST_DIR STREQUAL \"mkl\")\n            get_filename_component(BLAS_mkl_MKLROOT \"${BLAS_mkl_MKLROOT}\" DIRECTORY)\n        endif()\n      endif()\n      set(BLAS_mkl_LIB_PATH_SUFFIXES\n          \"compiler/lib\" \"compiler/lib/${BLAS_mkl_ARCH_NAME}_${BLAS_mkl_OS_NAME}\"\n          \"compiler/lib/${BLAS_mkl_ARCH_NAME}\"\n          \"mkl/lib\" \"mkl/lib/${BLAS_mkl_ARCH_NAME}_${BLAS_mkl_OS_NAME}\"\n          \"mkl/lib/${BLAS_mkl_ARCH_NAME}\"\n          \"lib\" \"lib/${BLAS_mkl_ARCH_NAME}_${BLAS_mkl_OS_NAME}\"\n          \"lib/${BLAS_mkl_ARCH_NAME}\"\n          )\n\n      foreach(IT ${BLAS_SEARCH_LIBS})\n        string(REPLACE \" \" \";\" SEARCH_LIBS ${IT})\n        if(NOT ${_LIBRARIES})\n          check_blas_libraries(\n            ${_LIBRARIES}\n            BLAS\n            ${BLAS_mkl_SEARCH_SYMBOL}\n            \"\"\n            \"${SEARCH_LIBS}\"\n            \"${CMAKE_THREAD_LIBS_INIT};${BLAS_mkl_LM};${BLAS_mkl_LDL}\"\n            \"${BLAS_mkl_MKLROOT}\"\n            \"${BLAS_mkl_LIB_PATH_SUFFIXES}\"\n            )\n        endif()\n      endforeach()\n\n      unset(BLAS_mkl_ILP_MODE)\n      unset(BLAS_mkl_INTFACE)\n      unset(BLAS_mkl_THREADING)\n      unset(BLAS_mkl_OMP)\n      unset(BLAS_mkl_DLL_SUFFIX)\n      unset(BLAS_mkl_LM)\n      unset(BLAS_mkl_LDL)\n      unset(BLAS_mkl_MKLROOT)\n      unset(BLAS_mkl_MKLROOT_LAST_DIR)\n      unset(BLAS_mkl_ARCH_NAME)\n      unset(BLAS_mkl_OS_NAME)\n      unset(BLAS_mkl_LIB_PATH_SUFFIXES)\n    endif()\n  endif()\nendif()\n\nif(BLA_F95)\n  find_package_handle_standard_args(BLAS REQUIRED_VARS BLAS95_LIBRARIES)\n  set(BLAS95_FOUND ${BLAS_FOUND})\n  if(BLAS_FOUND)\n    set(BLAS_LIBRARIES \"${BLAS95_LIBRARIES}\")\n  endif()\nendif()\n\n# gotoblas? (http://www.tacc.utexas.edu/tacc-projects/gotoblas2)\nif(BLA_VENDOR STREQUAL \"Goto\" OR BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\"\n      \"goto2\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\nendif()\n\n# FlexiBLAS? (http://www.mpi-magdeburg.mpg.de/mpcsc/software/FlexiBLAS/)\nif(BLA_VENDOR STREQUAL \"FlexiBLAS\" OR BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\"\n      \"flexiblas\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\nendif()\n\n# OpenBLAS? (http://www.openblas.net)\nif(BLA_VENDOR STREQUAL \"OpenBLAS\" OR BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\"\n      \"openblas\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\n  if(NOT BLAS_LIBRARIES AND (CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED))\n    if(BLAS_FIND_QUIETLY OR NOT BLAS_FIND_REQUIRED)\n      find_package(Threads)\n    else()\n      find_package(Threads REQUIRED)\n    endif()\n    set(_threadlibs \"${CMAKE_THREAD_LIBS_INIT}\")\n    if(BLA_STATIC)\n      find_package(OpenMP COMPONENTS C)\n      list(PREPEND _threadlibs \"${OpenMP_C_LIBRARIES}\")\n    endif()\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\"\n      \"openblas\"\n      \"${_threadlibs}\"\n      \"\"\n      \"\"\n      )\n    unset(_threadlibs)\n  endif()\nendif()\n\n# ArmPL blas library? (https://developer.arm.com/tools-and-software/server-and-hpc/compile/arm-compiler-for-linux/arm-performance-libraries)\nif(BLA_VENDOR MATCHES \"Arm\" OR BLA_VENDOR STREQUAL \"All\")\n\n   # Check for 64bit Integer support\n   if(BLA_VENDOR MATCHES \"_ilp64\")\n     set(BLAS_armpl_LIB \"armpl_ilp64\")\n   else()\n     set(BLAS_armpl_LIB \"armpl_lp64\")\n   endif()\n\n   # Check for OpenMP support, VIA BLA_VENDOR of Arm_mp or Arm_ipl64_mp\n   if(BLA_VENDOR MATCHES \"_mp\")\n     set(BLAS_armpl_LIB \"${BLAS_armpl_LIB}_mp\")\n   endif()\n\n   if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\"\n      \"${BLAS_armpl_LIB}\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\n\nendif()\n\n# FLAME's blis library? (https://github.com/flame/blis)\nif(BLA_VENDOR STREQUAL \"FLAME\" OR BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\"\n      \"blis\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\nendif()\n\n# BLAS in the ATLAS library? (http://math-atlas.sourceforge.net/)\nif(BLA_VENDOR STREQUAL \"ATLAS\" OR BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      dgemm\n      \"\"\n      \"blas;f77blas;atlas\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\nendif()\n\n# BLAS in PhiPACK libraries? (requires generic BLAS lib, too)\nif(BLA_VENDOR STREQUAL \"PhiPACK\" OR BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\"\n      \"sgemm;dgemm;blas\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\nendif()\n\n# BLAS in Alpha CXML library?\nif(BLA_VENDOR STREQUAL \"CXML\" OR BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\"\n      \"cxml\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\nendif()\n\n# BLAS in Alpha DXML library? (now called CXML, see above)\nif(BLA_VENDOR STREQUAL \"DXML\" OR BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\"\n      \"dxml\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\nendif()\n\n# BLAS in Sun Performance library?\nif(BLA_VENDOR STREQUAL \"SunPerf\" OR BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"-xlic_lib=sunperf\"\n      \"sunperf;sunmath\"\n      \"\"\n      \"\"\n      \"\"\n      )\n    if(BLAS_LIBRARIES)\n      set(BLAS_LINKER_FLAGS \"-xlic_lib=sunperf\")\n    endif()\n  endif()\nendif()\n\n# BLAS in SCSL library?  (SGI/Cray Scientific Library)\nif(BLA_VENDOR STREQUAL \"SCSL\" OR BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\"\n      \"scsl\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\nendif()\n\n# BLAS in SGIMATH library?\nif(BLA_VENDOR STREQUAL \"SGIMATH\" OR BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\"\n      \"complib.sgimath\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\nendif()\n\n# BLAS in IBM ESSL library? (requires generic BLAS lib, too)\nif(BLA_VENDOR STREQUAL \"IBMESSL\" OR BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\"\n      \"essl;blas\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\nendif()\n\n# BLAS in acml library?\nif(BLA_VENDOR MATCHES \"ACML\" OR BLA_VENDOR STREQUAL \"All\")\n  if(((BLA_VENDOR STREQUAL \"ACML\") AND (NOT BLAS_ACML_LIB_DIRS)) OR\n    ((BLA_VENDOR STREQUAL \"ACML_MP\") AND (NOT BLAS_ACML_MP_LIB_DIRS)) OR\n    ((BLA_VENDOR STREQUAL \"ACML_GPU\") AND (NOT BLAS_ACML_GPU_LIB_DIRS))\n    )\n  # try to find acml in \"standard\" paths\n  if(WIN32)\n    file(GLOB _ACML_ROOT \"C:/AMD/acml*/ACML-EULA.txt\")\n  else()\n    file(GLOB _ACML_ROOT \"/opt/acml*/ACML-EULA.txt\")\n  endif()\n  if(WIN32)\n    file(GLOB _ACML_GPU_ROOT \"C:/AMD/acml*/GPGPUexamples\")\n  else()\n    file(GLOB _ACML_GPU_ROOT \"/opt/acml*/GPGPUexamples\")\n  endif()\n  list(GET _ACML_ROOT 0 _ACML_ROOT)\n  list(GET _ACML_GPU_ROOT 0 _ACML_GPU_ROOT)\n  if(_ACML_ROOT)\n    get_filename_component(_ACML_ROOT ${_ACML_ROOT} PATH)\n    if(SIZEOF_INTEGER EQUAL 8)\n      set(_ACML_PATH_SUFFIX \"_int64\")\n    else()\n      set(_ACML_PATH_SUFFIX \"\")\n    endif()\n    if(CMAKE_Fortran_COMPILER_ID STREQUAL \"Intel\")\n      set(_ACML_COMPILER32 \"ifort32\")\n      set(_ACML_COMPILER64 \"ifort64\")\n    elseif(CMAKE_Fortran_COMPILER_ID STREQUAL \"SunPro\")\n      set(_ACML_COMPILER32 \"sun32\")\n      set(_ACML_COMPILER64 \"sun64\")\n    elseif(CMAKE_Fortran_COMPILER_ID STREQUAL \"PGI\")\n      set(_ACML_COMPILER32 \"pgi32\")\n      if(WIN32)\n        set(_ACML_COMPILER64 \"win64\")\n      else()\n        set(_ACML_COMPILER64 \"pgi64\")\n      endif()\n    elseif(CMAKE_Fortran_COMPILER_ID STREQUAL \"Open64\")\n      # 32 bit builds not supported on Open64 but for code simplicity\n      # We'll just use the same directory twice\n      set(_ACML_COMPILER32 \"open64_64\")\n      set(_ACML_COMPILER64 \"open64_64\")\n    elseif(CMAKE_Fortran_COMPILER_ID STREQUAL \"NAG\")\n      set(_ACML_COMPILER32 \"nag32\")\n      set(_ACML_COMPILER64 \"nag64\")\n    else()\n      set(_ACML_COMPILER32 \"gfortran32\")\n      set(_ACML_COMPILER64 \"gfortran64\")\n    endif()\n\n    if(BLA_VENDOR STREQUAL \"ACML_MP\")\n      set(_ACML_MP_LIB_DIRS\n        \"${_ACML_ROOT}/${_ACML_COMPILER32}_mp${_ACML_PATH_SUFFIX}/lib\"\n        \"${_ACML_ROOT}/${_ACML_COMPILER64}_mp${_ACML_PATH_SUFFIX}/lib\")\n    else()\n      set(_ACML_LIB_DIRS\n        \"${_ACML_ROOT}/${_ACML_COMPILER32}${_ACML_PATH_SUFFIX}/lib\"\n        \"${_ACML_ROOT}/${_ACML_COMPILER64}${_ACML_PATH_SUFFIX}/lib\")\n    endif()\n  endif()\nelseif(BLAS_${BLA_VENDOR}_LIB_DIRS)\n  set(_${BLA_VENDOR}_LIB_DIRS ${BLAS_${BLA_VENDOR}_LIB_DIRS})\nendif()\n\nif(BLA_VENDOR STREQUAL \"ACML_MP\")\n  foreach(BLAS_ACML_MP_LIB_DIRS ${_ACML_MP_LIB_DIRS})\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\" \"acml_mp;acml_mv\" \"\" ${BLAS_ACML_MP_LIB_DIRS} \"\"\n      )\n    if(BLAS_LIBRARIES)\n      break()\n    endif()\n  endforeach()\nelseif(BLA_VENDOR STREQUAL \"ACML_GPU\")\n  foreach(BLAS_ACML_GPU_LIB_DIRS ${_ACML_GPU_LIB_DIRS})\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\" \"acml;acml_mv;CALBLAS\" \"\" ${BLAS_ACML_GPU_LIB_DIRS} \"\"\n      )\n    if(BLAS_LIBRARIES)\n      break()\n    endif()\n  endforeach()\nelse()\n  foreach(BLAS_ACML_LIB_DIRS ${_ACML_LIB_DIRS})\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\" \"acml;acml_mv\" \"\" ${BLAS_ACML_LIB_DIRS} \"\"\n      )\n    if(BLAS_LIBRARIES)\n      break()\n    endif()\n  endforeach()\nendif()\n\n# Either acml or acml_mp should be in LD_LIBRARY_PATH but not both\nif(NOT BLAS_LIBRARIES)\n  check_blas_libraries(\n    BLAS_LIBRARIES\n    BLAS\n    sgemm\n    \"\"\n    \"acml;acml_mv\"\n    \"\"\n    \"\"\n    \"\"\n    )\nendif()\nif(NOT BLAS_LIBRARIES)\n  check_blas_libraries(\n    BLAS_LIBRARIES\n    BLAS\n    sgemm\n    \"\"\n    \"acml_mp;acml_mv\"\n    \"\"\n    \"\"\n    \"\"\n    )\nendif()\nif(NOT BLAS_LIBRARIES)\n  check_blas_libraries(\n    BLAS_LIBRARIES\n    BLAS\n    sgemm\n    \"\"\n    \"acml;acml_mv;CALBLAS\"\n    \"\"\n    \"\"\n    \"\"\n    )\nendif()\nendif() # ACML\n\n# Apple BLAS library?\nif(BLA_VENDOR STREQUAL \"Apple\" OR BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      dgemm\n      \"\"\n      \"Accelerate\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\nendif()\n\n# Apple NAS (vecLib) library?\nif(BLA_VENDOR STREQUAL \"NAS\" OR BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      dgemm\n      \"\"\n      \"vecLib\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\nendif()\n\n# Elbrus Math Library?\nif(BLA_VENDOR MATCHES \"EML\" OR BLA_VENDOR STREQUAL \"All\")\n\n   set(BLAS_EML_LIB \"eml\")\n\n   # Check for OpenMP support, VIA BLA_VENDOR of eml_mt\n   if(BLA_VENDOR MATCHES \"_mt\")\n     set(BLAS_EML_LIB \"${BLAS_EML_LIB}_mt\")\n   endif()\n\n   if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\"\n      \"${BLAS_EML_LIB}\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\n\nendif()\n\n# Generic BLAS library?\nif(BLA_VENDOR STREQUAL \"Generic\" OR BLA_VENDOR STREQUAL \"All\")\n  if(NOT BLAS_LIBRARIES)\n    check_blas_libraries(\n      BLAS_LIBRARIES\n      BLAS\n      sgemm\n      \"\"\n      \"blas\"\n      \"\"\n      \"\"\n      \"\"\n      )\n  endif()\nendif()\n\nif(NOT BLA_F95)\n  find_package_handle_standard_args(BLAS REQUIRED_VARS BLAS_LIBRARIES)\nendif()\n\n\n# On compilers that implicitly link BLAS (such as ftn, cc, and CC on Cray HPC machines)\n# we used a placeholder for empty BLAS_LIBRARIES to get through our logic above.\nif(BLAS_LIBRARIES STREQUAL \"BLAS_LIBRARIES-PLACEHOLDER-FOR-EMPTY-LIBRARIES\")\n  set(BLAS_LIBRARIES \"\")\nendif()\n\n_add_blas_target()\ncmake_pop_check_state()\nset(CMAKE_FIND_LIBRARY_SUFFIXES ${_blas_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})\n"
  },
  {
    "path": "cmake/upstream/FindCUDAToolkit.cmake",
    "content": "# Copyright 2000-2019 Kitware, Inc. and Contributors\n# All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n#\n# * Redistributions of source code must retain the above copyright\n#   notice, this list of conditions and the following disclaimer.\n#\n# * Redistributions in binary form must reproduce the above copyright\n#   notice, this list of conditions and the following disclaimer in the\n#   documentation and/or other materials provided with the distribution.\n#\n# * Neither the name of Kitware, Inc. nor the names of Contributors\n#   may be used to endorse or promote products derived from this\n#   software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n# \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n#[=======================================================================[.rst:\nFindCUDAToolkit\n---------------\n\nThis script locates the NVIDIA CUDA toolkit and the associated libraries, but\ndoes not require the ``CUDA`` language be enabled for a given project. This\nmodule does not search for the NVIDIA CUDA Samples.\n\nSearch Behavior\n^^^^^^^^^^^^^^^\n\nFinding the CUDA Toolkit requires finding the ``nvcc`` executable, which is\nsearched for in the following order:\n\n1. If the ``CUDA`` language has been enabled we will use the directory\n   containing the compiler as the first search location for ``nvcc``.\n\n2. If the ``CUDAToolkit_ROOT`` cmake configuration variable (e.g.,\n   ``-DCUDAToolkit_ROOT=/some/path``) *or* environment variable is defined, it\n   will be searched.  If both an environment variable **and** a\n   configuration variable are specified, the *configuration* variable takes\n   precedence.\n\n   The directory specified here must be such that the executable ``nvcc`` can be\n   found underneath the directory specified by ``CUDAToolkit_ROOT``.  If\n   ``CUDAToolkit_ROOT`` is specified, but no ``nvcc`` is found underneath, this\n   package is marked as **not** found.  No subsequent search attempts are\n   performed.\n\n3. If the CUDA_PATH environment variable is defined, it will be searched.\n\n4. The user's path is searched for ``nvcc`` using :command:`find_program`.  If\n   this is found, no subsequent search attempts are performed.  Users are\n   responsible for ensuring that the first ``nvcc`` to show up in the path is\n   the desired path in the event that multiple CUDA Toolkits are installed.\n\n5. On Unix systems, if the symbolic link ``/usr/local/cuda`` exists, this is\n   used.  No subsequent search attempts are performed.  No default symbolic link\n   location exists for the Windows platform.\n\n6. The platform specific default install locations are searched.  If exactly one\n   candidate is found, this is used.  The default CUDA Toolkit install locations\n   searched are:\n\n   +-------------+-------------------------------------------------------------+\n   | Platform    | Search Pattern                                              |\n   +=============+=============================================================+\n   | macOS       | ``/Developer/NVIDIA/CUDA-X.Y``                              |\n   +-------------+-------------------------------------------------------------+\n   | Other Unix  | ``/usr/local/cuda-X.Y``                                     |\n   +-------------+-------------------------------------------------------------+\n   | Windows     | ``C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\vX.Y`` |\n   +-------------+-------------------------------------------------------------+\n\n   Where ``X.Y`` would be a specific version of the CUDA Toolkit, such as\n   ``/usr/local/cuda-9.0`` or\n   ``C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.0``\n\n   .. note::\n\n       When multiple CUDA Toolkits are installed in the default location of a\n       system (e.g., both ``/usr/local/cuda-9.0`` and ``/usr/local/cuda-10.0``\n       exist but the ``/usr/local/cuda`` symbolic link does **not** exist), this\n       package is marked as **not** found.\n\n       There are too many factors involved in making an automatic decision in\n       the presence of multiple CUDA Toolkits being installed.  In this\n       situation, users are encouraged to either (1) set ``CUDAToolkit_ROOT`` or\n       (2) ensure that the correct ``nvcc`` executable shows up in ``$PATH`` for\n       :command:`find_program` to find.\n\nOptions\n^^^^^^^\n\n``VERSION``\n    If specified, describes the version of the CUDA Toolkit to search for.\n\n``REQUIRED``\n    If specified, configuration will error if a suitable CUDA Toolkit is not\n    found.\n\n``QUIET``\n    If specified, the search for a suitable CUDA Toolkit will not produce any\n    messages.\n\n``EXACT``\n    If specified, the CUDA Toolkit is considered found only if the exact\n    ``VERSION`` specified is recovered.\n\nImported targets\n^^^^^^^^^^^^^^^^\n\nAn :ref:`imported target <Imported targets>` named ``CUDA::toolkit`` is provided.\n\nThis module defines :prop_tgt:`IMPORTED` targets for each\nof the following libraries that are part of the CUDAToolkit:\n\n- :ref:`CUDA Runtime Library<cuda_toolkit_rt_lib>`\n- :ref:`CUDA Driver Library<cuda_toolkit_driver_lib>`\n- :ref:`cuBLAS<cuda_toolkit_cuBLAS>`\n- :ref:`cuFFT<cuda_toolkit_cuFFT>`\n- :ref:`cuRAND<cuda_toolkit_cuRAND>`\n- :ref:`cuSOLVER<cuda_toolkit_cuSOLVER>`\n- :ref:`cuSPARSE<cuda_toolkit_cuSPARSE>`\n- :ref:`cuPTI<cuda_toolkit_cupti>`\n- :ref:`NPP<cuda_toolkit_NPP>`\n- :ref:`nvBLAS<cuda_toolkit_nvBLAS>`\n- :ref:`nvGRAPH<cuda_toolkit_nvGRAPH>`\n- :ref:`nvJPEG<cuda_toolkit_nvJPEG>`\n- :ref:`nvidia-ML<cuda_toolkit_nvML>`\n- :ref:`nvRTC<cuda_toolkit_nvRTC>`\n- :ref:`nvToolsExt<cuda_toolkit_nvToolsExt>`\n- :ref:`OpenCL<cuda_toolkit_opencl>`\n- :ref:`cuLIBOS<cuda_toolkit_cuLIBOS>`\n\n.. _`cuda_toolkit_rt_lib`:\n\nCUDA Runtime Library\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThe CUDA Runtime library (cudart) are what most applications will typically\nneed to link against to make any calls such as `cudaMalloc`, and `cudaFree`.\n\nTargets Created:\n\n- ``CUDA::cudart``\n- ``CUDA::cudart_static``\n\n.. _`cuda_toolkit_driver_lib`:\n\nCUDA Driver Library\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThe CUDA Driver library (cuda) are used by applications that use calls\nsuch as `cuMemAlloc`, and `cuMemFree`. This is generally used by advanced\n\n\nTargets Created:\n\n- ``CUDA::cuda_driver``\n- ``CUDA::cuda_driver``\n\n.. _`cuda_toolkit_cuBLAS`:\n\ncuBLAS\n\"\"\"\"\"\"\n\nThe `cuBLAS <https://docs.nvidia.com/cuda/cublas/index.html>`_ library.\n\nTargets Created:\n\n- ``CUDA::cublas``\n- ``CUDA::cublas_static``\n\n.. _`cuda_toolkit_cuFFT`:\n\ncuFFT\n\"\"\"\"\"\n\nThe `cuFFT <https://docs.nvidia.com/cuda/cufft/index.html>`_ library.\n\nTargets Created:\n\n- ``CUDA::cufft``\n- ``CUDA::cufftw``\n- ``CUDA::cufft_static``\n- ``CUDA::cufftw_static``\n\ncuRAND\n\"\"\"\"\"\"\n\nThe `cuRAND <https://docs.nvidia.com/cuda/curand/index.html>`_ library.\n\nTargets Created:\n\n- ``CUDA::curand``\n- ``CUDA::curand_static``\n\n.. _`cuda_toolkit_cuSOLVER`:\n\ncuSOLVER\n\"\"\"\"\"\"\"\"\n\nThe `cuSOLVER <https://docs.nvidia.com/cuda/cusolver/index.html>`_ library.\n\nTargets Created:\n\n- ``CUDA::cusolver``\n- ``CUDA::cusolver_static``\n\n.. _`cuda_toolkit_cuSPARSE`:\n\ncuSPARSE\n\"\"\"\"\"\"\"\"\n\nThe `cuSPARSE <https://docs.nvidia.com/cuda/cusparse/index.html>`_ library.\n\nTargets Created:\n\n- ``CUDA::cusparse``\n- ``CUDA::cusparse_static``\n\n.. _`cuda_toolkit_cupti`:\n\ncupti\n\"\"\"\"\"\n\nThe `NVIDIA CUDA Profiling Tools Interface <https://developer.nvidia.com/CUPTI>`_.\n\nTargets Created:\n\n- ``CUDA::cupti``\n- ``CUDA::cupti_static``\n\n.. _`cuda_toolkit_NPP`:\n\nNPP\n\"\"\"\n\nThe `NPP <https://docs.nvidia.com/cuda/npp/index.html>`_ libraries.\n\nTargets Created:\n\n- `nppc`:\n\n  - ``CUDA::nppc``\n  - ``CUDA::nppc_static``\n\n- `nppial`: Arithmetic and logical operation functions in `nppi_arithmetic_and_logical_operations.h`\n\n  - ``CUDA::nppial``\n  - ``CUDA::nppial_static``\n\n- `nppicc`: Color conversion and sampling functions in `nppi_color_conversion.h`\n\n  - ``CUDA::nppicc``\n  - ``CUDA::nppicc_static``\n\n- `nppicom`: JPEG compression and decompression functions in `nppi_compression_functions.h`\n\n  - ``CUDA::nppicom``\n  - ``CUDA::nppicom_static``\n\n- `nppidei`: Data exchange and initialization functions in `nppi_data_exchange_and_initialization.h`\n\n  - ``CUDA::nppidei``\n  - ``CUDA::nppidei_static``\n\n- `nppif`: Filtering and computer vision functions in `nppi_filter_functions.h`\n\n  - ``CUDA::nppif``\n  - ``CUDA::nppif_static``\n\n- `nppig`: Geometry transformation functions found in `nppi_geometry_transforms.h`\n\n  - ``CUDA::nppig``\n  - ``CUDA::nppig_static``\n\n- `nppim`: Morphological operation functions found in `nppi_morphological_operations.h`\n\n  - ``CUDA::nppim``\n  - ``CUDA::nppim_static``\n\n- `nppist`: Statistics and linear transform in `nppi_statistics_functions.h` and `nppi_linear_transforms.h`\n\n  - ``CUDA::nppist``\n  - ``CUDA::nppist_static``\n\n- `nppisu`: Memory support functions in `nppi_support_functions.h`\n\n  - ``CUDA::nppisu``\n  - ``CUDA::nppisu_static``\n\n- `nppitc`: Threshold and compare operation functions in `nppi_threshold_and_compare_operations.h`\n\n  - ``CUDA::nppitc``\n  - ``CUDA::nppitc_static``\n\n- `npps`:\n\n  - ``CUDA::npps``\n  - ``CUDA::npps_static``\n\n.. _`cuda_toolkit_nvBLAS`:\n\nnvBLAS\n\"\"\"\"\"\"\n\nThe `nvBLAS <https://docs.nvidia.com/cuda/nvblas/index.html>`_ libraries.\nThis is a shared library only.\n\nTargets Created:\n\n- ``CUDA::nvblas``\n\n.. _`cuda_toolkit_nvGRAPH`:\n\nnvGRAPH\n\"\"\"\"\"\"\"\n\nThe `nvGRAPH <https://docs.nvidia.com/cuda/nvgraph/index.html>`_ library.\n\nTargets Created:\n\n- ``CUDA::nvgraph``\n- ``CUDA::nvgraph_static``\n\n\n.. _`cuda_toolkit_nvJPEG`:\n\nnvJPEG\n\"\"\"\"\"\"\n\nThe `nvJPEG <https://docs.nvidia.com/cuda/nvjpeg/index.html>`_ library.\nIntroduced in CUDA 10.\n\nTargets Created:\n\n- ``CUDA::nvjpeg``\n- ``CUDA::nvjpeg_static``\n\n.. _`cuda_toolkit_nvRTC`:\n\nnvRTC\n\"\"\"\"\"\n\nThe `nvRTC <https://docs.nvidia.com/cuda/nvrtc/index.html>`_ (Runtime Compilation) library.\nThis is a shared library only.\n\nTargets Created:\n\n- ``CUDA::nvrtc``\n\n.. _`cuda_toolkit_nvml`:\n\nnvidia-ML\n\"\"\"\"\"\"\"\"\"\n\nThe `NVIDIA Management Library <https://developer.nvidia.com/nvidia-management-library-nvml>`_.\nThis is a shared library only.\n\nTargets Created:\n\n- ``CUDA::nvml``\n\n.. _`cuda_toolkit_nvToolsExt`:\n\nnvToolsExt\n\"\"\"\"\"\"\"\"\"\"\n\nThe `NVIDIA Tools Extension <https://docs.nvidia.com/gameworks/content/gameworkslibrary/nvtx/nvidia_tools_extension_library_nvtx.htm>`_.\nThis is a shared library only.\n\nTargets Created:\n\n- ``CUDA::nvToolsExt``\n\n.. _`cuda_toolkit_opencl`:\n\nOpenCL\n\"\"\"\"\"\"\n\nThe `NVIDIA OpenCL Library <https://developer.nvidia.com/opencl>`_.\nThis is a shared library only.\n\nTargets Created:\n\n- ``CUDA::OpenCL``\n\n.. _`cuda_toolkit_cuLIBOS`:\n\ncuLIBOS\n\"\"\"\"\"\"\"\n\nThe cuLIBOS library is a backend thread abstraction layer library which is\nstatic only.  The ``CUDA::cublas_static``, ``CUDA::cusparse_static``,\n``CUDA::cufft_static``, ``CUDA::curand_static``, and (when implemented) NPP\nlibraries all automatically have this dependency linked.\n\nTarget Created:\n\n- ``CUDA::culibos``\n\n**Note**: direct usage of this target by consumers should not be necessary.\n\n.. _`cuda_toolkit_cuRAND`:\n\n\n\nResult variables\n^^^^^^^^^^^^^^^^\n\n``CUDAToolkit_FOUND``\n    A boolean specifying whether or not the CUDA Toolkit was found.\n\n``CUDAToolkit_VERSION``\n    The exact version of the CUDA Toolkit found (as reported by\n    ``nvcc --version``).\n\n``CUDAToolkit_VERSION_MAJOR``\n    The major version of the CUDA Toolkit.\n\n``CUDAToolkit_VERSION_MAJOR``\n    The minor version of the CUDA Toolkit.\n\n``CUDAToolkit_VERSION_PATCH``\n    The patch version of the CUDA Toolkit.\n\n``CUDAToolkit_BIN_DIR``\n    The path to the CUDA Toolkit library directory that contains the CUDA\n    executable ``nvcc``.\n\n``CUDAToolkit_INCLUDE_DIRS``\n    The path to the CUDA Toolkit ``include`` folder containing the header files\n    required to compile a project linking against CUDA.\n\n``CUDAToolkit_LIBRARY_DIR``\n    The path to the CUDA Toolkit library directory that contains the CUDA\n    Runtime library ``cudart``.\n\n``CUDAToolkit_TARGET_DIR``\n    The path to the CUDA Toolkit directory including the target architecture\n    when cross-compiling. When not cross-compiling this will be equivalant to\n    ``CUDAToolkit_ROOT_DIR``.\n\n``CUDAToolkit_NVCC_EXECUTABLE``\n    The path to the NVIDIA CUDA compiler ``nvcc``.  Note that this path may\n    **not** be the same as\n    :variable:`CMAKE_CUDA_COMPILER <CMAKE_<LANG>_COMPILER>`.  ``nvcc`` must be\n    found to determine the CUDA Toolkit version as well as determining other\n    features of the Toolkit.  This variable is set for the convenience of\n    modules that depend on this one.\n\n\n#]=======================================================================]\n\n# NOTE: much of this was simply extracted from FindCUDA.cmake.\n\n#   James Bigler, NVIDIA Corp (nvidia.com - jbigler)\n#   Abe Stephens, SCI Institute -- http://www.sci.utah.edu/~abe/FindCuda.html\n#\n#   Copyright (c) 2008 - 2009 NVIDIA Corporation.  All rights reserved.\n#\n#   Copyright (c) 2007-2009\n#   Scientific Computing and Imaging Institute, University of Utah\n#\n#   This code is licensed under the MIT License.  See the FindCUDA.cmake script\n#   for the text of the license.\n\n# The MIT License\n#\n# License for the specific language governing rights and limitations under\n# Permission is hereby granted, free of charge, to any person obtaining a\n# copy of this software and associated documentation files (the \"Software\"),\n# to deal in the Software without restriction, including without limitation\n# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n# and/or sell copies of the Software, and to permit persons to whom the\n# Software is furnished to do so, subject to the following conditions:\n#\n# The above copyright notice and this permission notice shall be included\n# in all copies or substantial portions of the Software.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS\n# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n# DEALINGS IN THE SOFTWARE.\n#\n###############################################################################\n\nif(CMAKE_CUDA_COMPILER_LOADED AND NOT CUDAToolkit_BIN_DIR)\n  get_filename_component(cuda_dir \"${CMAKE_CUDA_COMPILER}\" DIRECTORY)\n  # use the already detected cuda compiler\n  set(CUDAToolkit_BIN_DIR \"${cuda_dir}\" CACHE PATH \"\")\n  mark_as_advanced(CUDAToolkit_BIN_DIR)\n  unset(cuda_dir)\nendif()\n\n# Try language- or user-provided path first.\nif(CUDAToolkit_BIN_DIR)\n  find_program(CUDAToolkit_NVCC_EXECUTABLE\n    NAMES nvcc nvcc.exe\n    PATHS ${CUDAToolkit_BIN_DIR}\n    NO_DEFAULT_PATH\n    )\nendif()\n\n# Search using CUDAToolkit_ROOT\nfind_program(CUDAToolkit_NVCC_EXECUTABLE\n  NAMES nvcc nvcc.exe\n  PATHS ENV CUDA_PATH\n  PATH_SUFFIXES bin\n)\n\n# If the user specified CUDAToolkit_ROOT but nvcc could not be found, this is an error.\nif (NOT CUDAToolkit_NVCC_EXECUTABLE AND (DEFINED CUDAToolkit_ROOT OR DEFINED ENV{CUDAToolkit_ROOT}))\n  # Declare error messages now, print later depending on find_package args.\n  set(fail_base \"Could not find nvcc executable in path specified by\")\n  set(cuda_root_fail \"${fail_base} CUDAToolkit_ROOT=${CUDAToolkit_ROOT}\")\n  set(env_cuda_root_fail \"${fail_base} environment variable CUDAToolkit_ROOT=$ENV{CUDAToolkit_ROOT}\")\n\n  if (CUDAToolkit_FIND_REQUIRED)\n    if (DEFINED CUDAToolkit_ROOT)\n      message(FATAL_ERROR ${cuda_root_fail})\n    elseif (DEFINED ENV{CUDAToolkit_ROOT})\n      message(FATAL_ERROR ${env_cuda_root_fail})\n    endif()\n  else()\n    if (NOT CUDAToolkit_FIND_QUIETLY)\n      if (DEFINED CUDAToolkit_ROOT)\n        message(STATUS ${cuda_root_fail})\n      elseif (DEFINED ENV{CUDAToolkit_ROOT})\n        message(STATUS ${env_cuda_root_fail})\n      endif()\n    endif()\n    set(CUDAToolkit_FOUND FALSE)\n    unset(fail_base)\n    unset(cuda_root_fail)\n    unset(env_cuda_root_fail)\n    return()\n  endif()\nendif()\n\n# CUDAToolkit_ROOT cmake / env variable not specified, try platform defaults.\n#\n# - Linux: /usr/local/cuda-X.Y\n# - macOS: /Developer/NVIDIA/CUDA-X.Y\n# - Windows: C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\vX.Y\n#\n# We will also search the default symlink location /usr/local/cuda first since\n# if CUDAToolkit_ROOT is not specified, it is assumed that the symlinked\n# directory is the desired location.\nif (NOT CUDAToolkit_NVCC_EXECUTABLE)\n  if (UNIX)\n    if (NOT APPLE)\n      set(platform_base \"/usr/local/cuda-\")\n    else()\n      set(platform_base \"/Developer/NVIDIA/CUDA-\")\n    endif()\n  else()\n    set(platform_base \"C:\\\\Program Files\\\\NVIDIA GPU Computing Toolkit\\\\CUDA\\\\v\")\n  endif()\n\n  # Build out a descending list of possible cuda installations, e.g.\n  file(GLOB possible_paths \"${platform_base}*\")\n  # Iterate the glob results and create a descending list.\n  set(possible_versions)\n  foreach (p ${possible_paths})\n    # Extract version number from end of string\n    string(REGEX MATCH \"[0-9][0-9]?\\\\.[0-9]$\" p_version ${p})\n    if (IS_DIRECTORY ${p} AND p_version)\n      list(APPEND possible_versions ${p_version})\n    endif()\n  endforeach()\n\n  # Cannot use list(SORT) because that is alphabetical, we need numerical.\n  # NOTE: this is not an efficient sorting strategy.  But even if a user had\n  # every possible version of CUDA installed, this wouldn't create any\n  # significant overhead.\n  set(versions)\n  foreach (v ${possible_versions})\n    list(LENGTH versions num_versions)\n    # First version, nothing to compare with so just append.\n    if (num_versions EQUAL 0)\n      list(APPEND versions ${v})\n    else()\n      # Loop through list.  Insert at an index when comparison is\n      # VERSION_GREATER since we want a descending list.  Duplicates will not\n      # happen since this came from a glob list of directories.\n      set(i 0)\n      set(early_terminate FALSE)\n      while (i LESS num_versions)\n        list(GET versions ${i} curr)\n        if (v VERSION_GREATER curr)\n          list(INSERT versions ${i} ${v})\n          set(early_terminate TRUE)\n          break()\n        endif()\n        math(EXPR i \"${i} + 1\")\n      endwhile()\n      # If it did not get inserted, place it at the end.\n      if (NOT early_terminate)\n        list(APPEND versions ${v})\n      endif()\n    endif()\n  endforeach()\n\n  # With a descending list of versions, populate possible paths to search.\n  set(search_paths)\n  foreach (v ${versions})\n    list(APPEND search_paths \"${platform_base}${v}\")\n  endforeach()\n\n  # Force the global default /usr/local/cuda to the front on Unix.\n  if (UNIX)\n    list(INSERT search_paths 0 \"/usr/local/cuda\")\n  endif()\n\n  # Now search for nvcc again using the platform default search paths.\n  find_program(CUDAToolkit_NVCC_EXECUTABLE\n    NAMES nvcc nvcc.exe\n    PATHS ${search_paths}\n    PATH_SUFFIXES bin\n  )\n\n  # We are done with these variables now, cleanup for caller.\n  unset(platform_base)\n  unset(possible_paths)\n  unset(possible_versions)\n  unset(versions)\n  unset(i)\n  unset(early_terminate)\n  unset(search_paths)\n\n  if (NOT CUDAToolkit_NVCC_EXECUTABLE)\n    if (CUDAToolkit_FIND_REQUIRED)\n      message(FATAL_ERROR \"Could not find nvcc, please set CUDAToolkit_ROOT.\")\n    elseif(NOT CUDAToolkit_FIND_QUIETLY)\n      message(STATUS \"Could not find nvcc, please set CUDAToolkit_ROOT.\")\n    endif()\n\n    set(CUDAToolkit_FOUND FALSE)\n    return()\n  endif()\nendif()\n\nif(NOT CUDAToolkit_BIN_DIR AND CUDAToolkit_NVCC_EXECUTABLE)\n  get_filename_component(cuda_dir \"${CUDAToolkit_NVCC_EXECUTABLE}\" DIRECTORY)\n  set(CUDAToolkit_BIN_DIR \"${cuda_dir}\" CACHE PATH \"\" FORCE)\n  mark_as_advanced(CUDAToolkit_BIN_DIR)\n  unset(cuda_dir)\nendif()\n\nif(CUDAToolkit_NVCC_EXECUTABLE AND\n   CUDAToolkit_NVCC_EXECUTABLE STREQUAL CMAKE_CUDA_COMPILER)\n  # Need to set these based off the already computed CMAKE_CUDA_COMPILER_VERSION value\n  # This if statement will always match, but is used to provide variables for MATCH 1,2,3...\n  if(CMAKE_CUDA_COMPILER_VERSION MATCHES [=[([0-9]+)\\.([0-9]+)\\.([0-9]+)]=])\n    set(CUDAToolkit_VERSION_MAJOR \"${CMAKE_MATCH_1}\")\n    set(CUDAToolkit_VERSION_MINOR \"${CMAKE_MATCH_2}\")\n    set(CUDAToolkit_VERSION_PATCH \"${CMAKE_MATCH_3}\")\n    set(CUDAToolkit_VERSION \"${CMAKE_CUDA_COMPILER_VERSION}\")\n  endif()\nelse()\n  # Compute the version by invoking nvcc\n  execute_process (COMMAND ${CUDAToolkit_NVCC_EXECUTABLE} \"--version\" OUTPUT_VARIABLE NVCC_OUT)\n  if(NVCC_OUT MATCHES [=[ V([0-9]+)\\.([0-9]+)\\.([0-9]+)]=])\n    set(CUDAToolkit_VERSION_MAJOR \"${CMAKE_MATCH_1}\")\n    set(CUDAToolkit_VERSION_MINOR \"${CMAKE_MATCH_2}\")\n    set(CUDAToolkit_VERSION_PATCH \"${CMAKE_MATCH_3}\")\n    set(CUDAToolkit_VERSION  \"${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}\")\n  endif()\n  unset(NVCC_OUT)\nendif()\n\n\nget_filename_component(CUDAToolkit_ROOT_DIR ${CUDAToolkit_BIN_DIR} DIRECTORY ABSOLUTE)\n\n# Handle cross compilation\nif(CMAKE_CROSSCOMPILING)\n  if(CMAKE_SYSTEM_PROCESSOR STREQUAL \"armv7-a\")\n    # Support for NVPACK\n    set (CUDAToolkit_TARGET_NAME \"armv7-linux-androideabi\")\n  elseif(CMAKE_SYSTEM_PROCESSOR MATCHES \"arm\")\n    # Support for arm cross compilation\n    set(CUDAToolkit_TARGET_NAME \"armv7-linux-gnueabihf\")\n  elseif(CMAKE_SYSTEM_PROCESSOR MATCHES \"aarch64\")\n    # Support for aarch64 cross compilation\n    if (ANDROID_ARCH_NAME STREQUAL \"arm64\")\n      set(CUDAToolkit_TARGET_NAME \"aarch64-linux-androideabi\")\n    else()\n      set(CUDAToolkit_TARGET_NAME \"aarch64-linux\")\n    endif (ANDROID_ARCH_NAME STREQUAL \"arm64\")\n  elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL \"x86_64\")\n      set(CUDAToolkit_TARGET_NAME \"x86_64-linux\")\n  endif()\n\n  if (EXISTS \"${CUDAToolkit_ROOT_DIR}/targets/${CUDAToolkit_TARGET_NAME}\")\n    set(CUDAToolkit_TARGET_DIR \"${CUDAToolkit_ROOT_DIR}/targets/${CUDAToolkit_TARGET_NAME}\")\n    # add known CUDA target root path to the set of directories we search for programs, libraries and headers\n    list(PREPEND CMAKE_FIND_ROOT_PATH \"${CUDAToolkit_TARGET_DIR}\")\n\n    # Mark that we need to pop the root search path changes after we have\n    # found all cuda libraries so that searches for our cross-compilation\n    # libraries work when another cuda sdk is in CMAKE_PREFIX_PATH or\n    # PATh\n    set(_CUDAToolkit_Pop_ROOT_PATH True)\n  endif()\nelse()\n  # Not cross compiling\n  set(CUDAToolkit_TARGET_DIR \"${CUDAToolkit_ROOT_DIR}\")\n  # Now that we have the real ROOT_DIR, find components inside it.\n  list(APPEND CMAKE_PREFIX_PATH ${CUDAToolkit_ROOT_DIR})\n\n  # Mark that we need to pop the prefix path changes after we have\n  # found the cudart library.\n  set(_CUDAToolkit_Pop_Prefix True)\nendif()\n\n\n# Find the include/ directory\nfind_path(CUDAToolkit_INCLUDE_DIR\n  NAMES cuda_runtime.h\n)\n\n# And find the CUDA Runtime Library libcudart\nfind_library(CUDA_CUDART\n  NAMES cudart\n  PATH_SUFFIXES lib64 lib/x64\n)\nif (NOT CUDA_CUDART)\n  find_library(CUDA_CUDART\n    NAMES cudart\n    PATH_SUFFIXES lib64/stubs lib/x64/stubs\n  )\nendif()\n\nif (NOT CUDA_CUDART AND NOT CUDAToolkit_FIND_QUIETLY)\n  message(STATUS \"Unable to find cudart library.\")\nendif()\n\nunset(CUDAToolkit_ROOT_DIR)\nif(_CUDAToolkit_Pop_Prefix)\n  list(REMOVE_AT CMAKE_PREFIX_PATH -1)\n  unset(_CUDAToolkit_Pop_Prefix)\nendif()\n\n#-----------------------------------------------------------------------------\n# Perform version comparison and validate all required variables are set.\n# MXNET NOTE: This differs from CMake source by ${CMAKE_CURRENT_LIST_DIR}\n# replaced with ${CMAKE_ROOT}/Modules\ninclude(${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake)\nfind_package_handle_standard_args(CUDAToolkit\n  REQUIRED_VARS\n    CUDAToolkit_INCLUDE_DIR\n    CUDA_CUDART\n    CUDAToolkit_NVCC_EXECUTABLE\n  VERSION_VAR\n    CUDAToolkit_VERSION\n)\nmark_as_advanced(CUDA_CUDART\n                 CUDAToolkit_INCLUDE_DIR\n                 CUDAToolkit_NVCC_EXECUTABLE\n                 )\n\n#-----------------------------------------------------------------------------\n# Construct result variables\nif(CUDAToolkit_FOUND)\n set(CUDAToolkit_INCLUDE_DIRS ${CUDAToolkit_INCLUDE_DIR})\n get_filename_component(CUDAToolkit_LIBRARY_DIR ${CUDA_CUDART} DIRECTORY ABSOLUTE)\nendif()\n\n#-----------------------------------------------------------------------------\n# Construct import targets\nif(CUDAToolkit_FOUND)\n\n  function(_CUDAToolkit_find_and_add_import_lib lib_name)\n    cmake_parse_arguments(arg \"\" \"\" \"ALT;DEPS;EXTRA_PATH_SUFFIXES\" ${ARGN})\n\n    set(search_names ${lib_name} ${arg_ALT})\n\n    find_library(CUDA_${lib_name}_LIBRARY\n      NAMES ${search_names}\n      HINTS ${CUDAToolkit_LIBRARY_DIR}\n            ENV CUDA_PATH\n      PATH_SUFFIXES nvidia/current lib64 lib/x64 lib\n                    ${arg_EXTRA_PATH_SUFFIXES}\n    )\n    # Don't try any stub directories intil we have exhausted all other\n    # search locations.\n    if(NOT CUDA_${lib_name}_LIBRARY)\n      find_library(CUDA_${lib_name}_LIBRARY\n        NAMES ${search_names}\n        HINTS ${CUDAToolkit_LIBRARY_DIR}\n              ENV CUDA_PATH\n        PATH_SUFFIXES lib64/stubs lib/x64/stubs lib/stubs stubs\n      )\n    endif()\n\n    mark_as_advanced(CUDA_${lib_name}_LIBRARY)\n\n    if (NOT TARGET CUDA::${lib_name} AND CUDA_${lib_name}_LIBRARY)\n      add_library(CUDA::${lib_name} IMPORTED INTERFACE)\n      target_include_directories(CUDA::${lib_name} SYSTEM INTERFACE \"${CUDAToolkit_INCLUDE_DIRS}\")\n      target_link_libraries(CUDA::${lib_name} INTERFACE \"${CUDA_${lib_name}_LIBRARY}\")\n      foreach(dep ${arg_DEPS})\n        if(TARGET CUDA::${dep})\n          target_link_libraries(CUDA::${lib_name} INTERFACE CUDA::${dep})\n        endif()\n      endforeach()\n    endif()\n  endfunction()\n\n  if(NOT TARGET CUDA::toolkit)\n    add_library(CUDA::toolkit IMPORTED INTERFACE)\n    target_include_directories(CUDA::toolkit SYSTEM INTERFACE \"${CUDAToolkit_INCLUDE_DIRS}\")\n    target_link_directories(CUDA::toolkit INTERFACE \"${CUDAToolkit_LIBRARY_DIR}\")\n  endif()\n\n  _CUDAToolkit_find_and_add_import_lib(cuda_driver ALT cuda)\n\n  _CUDAToolkit_find_and_add_import_lib(cudart)\n  _CUDAToolkit_find_and_add_import_lib(cudart_static)\n\n  # setup dependencies that are required for cudart_static when building\n  # on linux. These are generally only required when using the CUDA toolkit\n  # when CUDA language is disabled\n  if(NOT TARGET CUDA::cudart_static_deps\n     AND TARGET CUDA::cudart_static)\n\n    add_library(CUDA::cudart_static_deps IMPORTED INTERFACE)\n    target_link_libraries(CUDA::cudart_static INTERFACE CUDA::cudart_static_deps)\n\n    if(UNIX AND (CMAKE_C_COMPILER OR CMAKE_CXX_COMPILER))\n      find_package(Threads REQUIRED)\n      target_link_libraries(CUDA::cudart_static_deps INTERFACE Threads::Threads ${CMAKE_DL_LIBS})\n    endif()\n\n    if(UNIX AND NOT APPLE)\n      # On Linux, you must link against librt when using the static cuda runtime.\n      find_library(CUDAToolkit_rt_LIBRARY rt)\n      mark_as_advanced(CUDAToolkit_rt_LIBRARY)\n      if(NOT CUDAToolkit_rt_LIBRARY)\n        message(WARNING \"Could not find librt library, needed by CUDA::cudart_static\")\n      else()\n        target_link_libraries(CUDA::cudart_static_deps INTERFACE ${CUDAToolkit_rt_LIBRARY})\n      endif()\n    endif()\n  endif()\n\n  _CUDAToolkit_find_and_add_import_lib(culibos) # it's a static library\n  foreach (cuda_lib cublas cufft curand cusparse nppc nvjpeg)\n    _CUDAToolkit_find_and_add_import_lib(${cuda_lib})\n    _CUDAToolkit_find_and_add_import_lib(${cuda_lib}_static DEPS culibos)\n  endforeach()\n\n  # cuFFTW depends on cuFFT\n  _CUDAToolkit_find_and_add_import_lib(cufftw DEPS cufft)\n  _CUDAToolkit_find_and_add_import_lib(cufftw DEPS cufft_static)\n\n  # cuSOLVER depends on cuBLAS, and cuSPARSE\n  _CUDAToolkit_find_and_add_import_lib(cusolver DEPS cublas cusparse)\n  _CUDAToolkit_find_and_add_import_lib(cusolver_static DEPS cublas_static cusparse_static culibos)\n\n  # nvGRAPH depends on cuRAND, and cuSOLVER.\n  _CUDAToolkit_find_and_add_import_lib(nvgraph DEPS curand cusolver)\n  _CUDAToolkit_find_and_add_import_lib(nvgraph_static DEPS curand_static cusolver_static)\n\n  # Process the majority of the NPP libraries.\n  foreach (cuda_lib nppial nppicc nppidei nppif nppig nppim nppist nppitc npps nppicom nppisu)\n    _CUDAToolkit_find_and_add_import_lib(${cuda_lib} DEPS nppc)\n    _CUDAToolkit_find_and_add_import_lib(${cuda_lib}_static DEPS nppc_static)\n  endforeach()\n\n  _CUDAToolkit_find_and_add_import_lib(cupti\n                                       EXTRA_PATH_SUFFIXES ../extras/CUPTI/lib64/\n                                                           ../extras/CUPTI/lib/)\n  _CUDAToolkit_find_and_add_import_lib(cupti_static\n                                       EXTRA_PATH_SUFFIXES ../extras/CUPTI/lib64/\n                                                           ../extras/CUPTI/lib/)\n\n  _CUDAToolkit_find_and_add_import_lib(nvrtc DEPS cuda_driver)\n\n  _CUDAToolkit_find_and_add_import_lib(nvml ALT nvidia-ml nvml)\n\n  if(WIN32)\n    # nvtools can be installed outside the CUDA toolkit directory\n    # so prefer the NVTOOLSEXT_PATH windows only environment variable\n    # In addition on windows the most common name is nvToolsExt64_1\n    find_library(CUDA_nvToolsExt_LIBRARY\n      NAMES nvToolsExt64_1 nvToolsExt64 nvToolsExt\n      PATHS ENV NVTOOLSEXT_PATH\n            ENV CUDA_PATH\n      PATH_SUFFIXES lib/x64 lib\n    )\n  endif()\n  _CUDAToolkit_find_and_add_import_lib(nvToolsExt ALT nvToolsExt64)\n\n  _CUDAToolkit_find_and_add_import_lib(OpenCL)\nendif()\n\nif(_CUDAToolkit_Pop_ROOT_PATH)\n  list(REMOVE_AT CMAKE_FIND_ROOT_PATH 0)\n  unset(_CUDAToolkit_Pop_ROOT_PATH)\nendif()\n"
  },
  {
    "path": "cmake/upstream/select_compute_arch.cmake",
    "content": "# Copyright 2000-2019 Kitware, Inc. and Contributors\n# All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n#\n# * Redistributions of source code must retain the above copyright\n#   notice, this list of conditions and the following disclaimer.\n#\n# * Redistributions in binary form must reproduce the above copyright\n#   notice, this list of conditions and the following disclaimer in the\n#   documentation and/or other materials provided with the distribution.\n#\n# * Neither the name of Kitware, Inc. nor the names of Contributors\n#   may be used to endorse or promote products derived from this\n#   software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n# \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# Synopsis:\n#   CUDA_SELECT_NVCC_ARCH_FLAGS(out_variable [target_CUDA_architectures])\n#   -- Selects GPU arch flags for nvcc based on target_CUDA_architectures\n#      target_CUDA_architectures : Auto | Common | All | LIST(ARCH_AND_PTX ...)\n#       - \"Auto\" detects local machine GPU compute arch at runtime.\n#       - \"Common\" and \"All\" cover common and entire subsets of architectures\n#      ARCH_AND_PTX : NAME | NUM.NUM | NUM.NUM(NUM.NUM) | NUM.NUM+PTX\n#      NAME: Fermi Kepler Maxwell Kepler+Tegra Kepler+Tesla Maxwell+Tegra Pascal Volta Turing Ampere\n#      NUM: Any number. Only those pairs are currently accepted by NVCC though:\n#            2.0 2.1 3.0 3.2 3.5 3.7 5.0 5.2 5.3 6.0 6.2 7.0 7.2 7.5 8.0 8.6\n#      Returns LIST of flags to be added to CUDA_NVCC_FLAGS in ${out_variable}\n#      Additionally, sets ${out_variable}_readable to the resulting numeric list\n#      Example:\n#       CUDA_SELECT_NVCC_ARCH_FLAGS(ARCH_FLAGS 3.0 3.5+PTX 5.2(5.0) Maxwell)\n#        LIST(APPEND CUDA_NVCC_FLAGS ${ARCH_FLAGS})\n#\n#      More info on CUDA architectures: https://en.wikipedia.org/wiki/CUDA\n#\n\nif(CMAKE_CUDA_COMPILER_LOADED) # CUDA as a language\n  if(CMAKE_CUDA_COMPILER_ID STREQUAL \"NVIDIA\"\n      AND CMAKE_CUDA_COMPILER_VERSION MATCHES \"^([0-9]+\\\\.[0-9]+)\")\n    set(CUDA_VERSION \"${CMAKE_MATCH_1}\")\n  endif()\nendif()\n\n# See: https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#gpu-feature-list\n# Additions, deprecations, and removals can be found in the release notes:\n# https://developer.nvidia.com/cuda-toolkit-archive\n\n# The initial status here is for CUDA 7.0\nset(CUDA_KNOWN_GPU_ARCHITECTURES  \"Fermi\" \"Kepler\" \"Maxwell\" \"Kepler+Tegra\" \"Kepler+Tesla\" \"Maxwell+Tegra\")\nset(CUDA_COMMON_GPU_ARCHITECTURES \"2.0\" \"2.1\" \"3.0\" \"3.5\" \"5.0\" \"5.3\")\nset(CUDA_LIMIT_GPU_ARCHITECTURE \"6.0\")\nset(CUDA_ALL_GPU_ARCHITECTURES \"2.0\" \"2.1\" \"3.0\" \"3.2\" \"3.5\" \"3.7\" \"5.0\" \"5.2\" \"5.3\")\nset(_CUDA_MAX_COMMON_ARCHITECTURE \"5.2+PTX\")\n\n\nif(CUDA_VERSION VERSION_GREATER_EQUAL \"8.0\")\n  list(APPEND CUDA_KNOWN_GPU_ARCHITECTURES \"Pascal\")\n  list(APPEND CUDA_COMMON_GPU_ARCHITECTURES \"6.0\" \"6.1\")\n  list(APPEND CUDA_ALL_GPU_ARCHITECTURES \"6.0\" \"6.1\" \"6.2\")\n\n  set(_CUDA_MAX_COMMON_ARCHITECTURE \"6.2+PTX\")\n  set(CUDA_LIMIT_GPU_ARCHITECTURE \"7.0\")\n\n  list(REMOVE_ITEM CUDA_COMMON_GPU_ARCHITECTURES \"2.0\" \"2.1\")\nendif ()\n\nif(CUDA_VERSION VERSION_GREATER_EQUAL \"9.0\")\n  list(APPEND CUDA_KNOWN_GPU_ARCHITECTURES \"Volta\")\n  list(APPEND CUDA_COMMON_GPU_ARCHITECTURES \"7.0\")\n  list(APPEND CUDA_ALL_GPU_ARCHITECTURES \"7.0\" \"7.2\")\n\n  set(_CUDA_MAX_COMMON_ARCHITECTURE \"7.2+PTX\")\n  set(CUDA_LIMIT_GPU_ARCHITECTURE \"8.0\")\n\n  list(REMOVE_ITEM CUDA_KNOWN_GPU_ARCHITECTURES \"Fermi\")\n  list(REMOVE_ITEM CUDA_ALL_GPU_ARCHITECTURES \"2.0\" \"2.1\")\nendif()\n\nif(CUDA_VERSION VERSION_GREATER_EQUAL \"10.0\")\n  list(APPEND CUDA_KNOWN_GPU_ARCHITECTURES \"Turing\")\n  list(APPEND CUDA_COMMON_GPU_ARCHITECTURES \"7.5\")\n  list(APPEND CUDA_ALL_GPU_ARCHITECTURES \"7.5\")\n\n  set(_CUDA_MAX_COMMON_ARCHITECTURE \"7.5+PTX\")\n  set(CUDA_LIMIT_GPU_ARCHITECTURE \"8.0\")\n\n  list(REMOVE_ITEM CUDA_COMMON_GPU_ARCHITECTURES \"3.0\")\nendif()\n\n# https://docs.nvidia.com/cuda/archive/11.0/cuda-toolkit-release-notes/index.html#cuda-general-new-features\n# https://docs.nvidia.com/cuda/archive/11.0/cuda-toolkit-release-notes/index.html#deprecated-features\nif(CUDA_VERSION VERSION_GREATER_EQUAL \"11.0\")\n  list(APPEND CUDA_KNOWN_GPU_ARCHITECTURES \"Ampere\")\n  list(APPEND CUDA_COMMON_GPU_ARCHITECTURES \"8.0\")\n  list(APPEND CUDA_ALL_GPU_ARCHITECTURES \"8.0\")\n\n  set(_CUDA_MAX_COMMON_ARCHITECTURE \"8.0+PTX\")\n  set(CUDA_LIMIT_GPU_ARCHITECTURE \"8.6\")\n\n  list(REMOVE_ITEM CUDA_COMMON_GPU_ARCHITECTURES \"3.5\" \"5.0\")\n  list(REMOVE_ITEM CUDA_ALL_GPU_ARCHITECTURES \"3.0\" \"3.2\")\nendif()\n\nif(CUDA_VERSION VERSION_GREATER_EQUAL \"11.1\")\n  list(APPEND CUDA_COMMON_GPU_ARCHITECTURES \"8.6\")\n  list(APPEND CUDA_ALL_GPU_ARCHITECTURES \"8.6\")\n\n  set(_CUDA_MAX_COMMON_ARCHITECTURE \"8.6+PTX\")\n  set(CUDA_LIMIT_GPU_ARCHITECTURE \"9.0\")\nendif()\n\nlist(APPEND CUDA_COMMON_GPU_ARCHITECTURES \"${_CUDA_MAX_COMMON_ARCHITECTURE}\")\n\n# Check with: cmake -DCUDA_VERSION=7.0 -P select_compute_arch.cmake\nif(DEFINED CMAKE_SCRIPT_MODE_FILE)\n  include(CMakePrintHelpers)\n  cmake_print_variables(CUDA_KNOWN_GPU_ARCHITECTURES)\n  cmake_print_variables(CUDA_COMMON_GPU_ARCHITECTURES)\n  cmake_print_variables(CUDA_LIMIT_GPU_ARCHITECTURE)\n  cmake_print_variables(CUDA_ALL_GPU_ARCHITECTURES)\nendif()\n\n\n################################################################################################\n# A function for automatic detection of GPUs installed  (if autodetection is enabled)\n# Usage:\n#   CUDA_DETECT_INSTALLED_GPUS(OUT_VARIABLE)\n#\nfunction(CUDA_DETECT_INSTALLED_GPUS OUT_VARIABLE)\n  if(NOT CUDA_GPU_DETECT_OUTPUT)\n    if(CMAKE_CUDA_COMPILER_LOADED) # CUDA as a language\n      set(file \"${PROJECT_BINARY_DIR}/detect_cuda_compute_capabilities.cu\")\n    else()\n      set(file \"${PROJECT_BINARY_DIR}/detect_cuda_compute_capabilities.cpp\")\n    endif()\n\n    file(WRITE ${file} \"\"\n      \"#include <cuda_runtime.h>\\n\"\n      \"#include <cstdio>\\n\"\n      \"int main()\\n\"\n      \"{\\n\"\n      \"  int count = 0;\\n\"\n      \"  if (cudaSuccess != cudaGetDeviceCount(&count)) return -1;\\n\"\n      \"  if (count == 0) return -1;\\n\"\n      \"  for (int device = 0; device < count; ++device)\\n\"\n      \"  {\\n\"\n      \"    cudaDeviceProp prop;\\n\"\n      \"    if (cudaSuccess == cudaGetDeviceProperties(&prop, device))\\n\"\n      \"      std::printf(\\\"%d.%d \\\", prop.major, prop.minor);\\n\"\n      \"  }\\n\"\n      \"  return 0;\\n\"\n      \"}\\n\")\n\n    if(CMAKE_CUDA_COMPILER_LOADED) # CUDA as a language\n      try_run(run_result compile_result ${PROJECT_BINARY_DIR} ${file}\n              RUN_OUTPUT_VARIABLE compute_capabilities)\n    else()\n      try_run(run_result compile_result ${PROJECT_BINARY_DIR} ${file}\n              CMAKE_FLAGS \"-DINCLUDE_DIRECTORIES=${CUDA_INCLUDE_DIRS}\"\n              LINK_LIBRARIES ${CUDA_LIBRARIES}\n              RUN_OUTPUT_VARIABLE compute_capabilities)\n    endif()\n\n    # Filter unrelated content out of the output.\n    string(REGEX MATCHALL \"[0-9]+\\\\.[0-9]+\" compute_capabilities \"${compute_capabilities}\")\n\n    if(run_result EQUAL 0)\n      string(REPLACE \"2.1\" \"2.1(2.0)\" compute_capabilities \"${compute_capabilities}\")\n      set(CUDA_GPU_DETECT_OUTPUT ${compute_capabilities}\n        CACHE INTERNAL \"Returned GPU architectures from detect_gpus tool\" FORCE)\n    endif()\n  endif()\n\n  if(NOT CUDA_GPU_DETECT_OUTPUT)\n    message(STATUS \"Automatic GPU detection failed. Building for common architectures.\")\n    set(${OUT_VARIABLE} ${CUDA_COMMON_GPU_ARCHITECTURES} PARENT_SCOPE)\n  else()\n    # Filter based on CUDA version supported archs\n    set(CUDA_GPU_DETECT_OUTPUT_FILTERED \"\")\n    separate_arguments(CUDA_GPU_DETECT_OUTPUT)\n    foreach(ITEM IN ITEMS ${CUDA_GPU_DETECT_OUTPUT})\n        if(CUDA_LIMIT_GPU_ARCHITECTURE AND ITEM VERSION_GREATER_EQUAL CUDA_LIMIT_GPU_ARCHITECTURE)\n        list(GET CUDA_COMMON_GPU_ARCHITECTURES -1 NEWITEM)\n        string(APPEND CUDA_GPU_DETECT_OUTPUT_FILTERED \" ${NEWITEM}\")\n      else()\n        string(APPEND CUDA_GPU_DETECT_OUTPUT_FILTERED \" ${ITEM}\")\n      endif()\n    endforeach()\n\n    set(${OUT_VARIABLE} ${CUDA_GPU_DETECT_OUTPUT_FILTERED} PARENT_SCOPE)\n  endif()\nendfunction()\n\n\n################################################################################################\n# Function for selecting GPU arch flags for nvcc based on CUDA architectures from parameter list\n# Usage:\n#   SELECT_NVCC_ARCH_FLAGS(out_variable [list of CUDA compute archs])\nfunction(CUDA_SELECT_NVCC_ARCH_FLAGS out_variable)\n  set(CUDA_ARCH_LIST \"${ARGN}\")\n\n  if(\"X${CUDA_ARCH_LIST}\" STREQUAL \"X\" )\n    set(CUDA_ARCH_LIST \"Auto\")\n  endif()\n\n  set(cuda_arch_bin)\n  set(cuda_arch_ptx)\n\n  if(\"${CUDA_ARCH_LIST}\" STREQUAL \"All\")\n    set(CUDA_ARCH_LIST ${CUDA_KNOWN_GPU_ARCHITECTURES})\n  elseif(\"${CUDA_ARCH_LIST}\" STREQUAL \"Common\")\n    set(CUDA_ARCH_LIST ${CUDA_COMMON_GPU_ARCHITECTURES})\n  elseif(\"${CUDA_ARCH_LIST}\" STREQUAL \"Auto\")\n    CUDA_DETECT_INSTALLED_GPUS(CUDA_ARCH_LIST)\n    message(STATUS \"Autodetected CUDA architecture(s): ${CUDA_ARCH_LIST}\")\n  endif()\n\n  # Now process the list and look for names\n  string(REGEX REPLACE \"[ \\t]+\" \";\" CUDA_ARCH_LIST \"${CUDA_ARCH_LIST}\")\n  list(REMOVE_DUPLICATES CUDA_ARCH_LIST)\n  foreach(arch_name ${CUDA_ARCH_LIST})\n    set(arch_bin)\n    set(arch_ptx)\n    set(add_ptx FALSE)\n    # Check to see if we are compiling PTX\n    if(arch_name MATCHES \"(.*)\\\\+PTX$\")\n      set(add_ptx TRUE)\n      set(arch_name ${CMAKE_MATCH_1})\n    endif()\n    if(arch_name MATCHES \"^([0-9]\\\\.[0-9](\\\\([0-9]\\\\.[0-9]\\\\))?)$\")\n      set(arch_bin ${CMAKE_MATCH_1})\n      set(arch_ptx ${arch_bin})\n    else()\n      # Look for it in our list of known architectures\n      if(${arch_name} STREQUAL \"Fermi\")\n        set(arch_bin 2.0 \"2.1(2.0)\")\n      elseif(${arch_name} STREQUAL \"Kepler+Tegra\")\n        set(arch_bin 3.2)\n      elseif(${arch_name} STREQUAL \"Kepler+Tesla\")\n        set(arch_bin 3.7)\n      elseif(${arch_name} STREQUAL \"Kepler\")\n        set(arch_bin 3.0 3.5)\n        set(arch_ptx 3.5)\n      elseif(${arch_name} STREQUAL \"Maxwell+Tegra\")\n        set(arch_bin 5.3)\n      elseif(${arch_name} STREQUAL \"Maxwell\")\n        set(arch_bin 5.0 5.2)\n        set(arch_ptx 5.2)\n      elseif(${arch_name} STREQUAL \"Pascal\")\n        set(arch_bin 6.0 6.1)\n        set(arch_ptx 6.1)\n      elseif(${arch_name} STREQUAL \"Volta\")\n        set(arch_bin 7.0 7.0)\n        set(arch_ptx 7.0)\n      elseif(${arch_name} STREQUAL \"Turing\")\n        set(arch_bin 7.5)\n        set(arch_ptx 7.5)\n      elseif(${arch_name} STREQUAL \"Ampere\")\n        set(arch_bin 8.0)\n        set(arch_ptx 8.0)\n      else()\n        message(SEND_ERROR \"Unknown CUDA Architecture Name ${arch_name} in CUDA_SELECT_NVCC_ARCH_FLAGS\")\n      endif()\n    endif()\n    if(NOT arch_bin)\n      message(SEND_ERROR \"arch_bin wasn't set for some reason\")\n    endif()\n    list(APPEND cuda_arch_bin ${arch_bin})\n    if(add_ptx)\n      if (NOT arch_ptx)\n        set(arch_ptx ${arch_bin})\n      endif()\n      list(APPEND cuda_arch_ptx ${arch_ptx})\n    endif()\n  endforeach()\n\n  # remove dots and convert to lists\n  string(REGEX REPLACE \"\\\\.\" \"\" cuda_arch_bin \"${cuda_arch_bin}\")\n  string(REGEX REPLACE \"\\\\.\" \"\" cuda_arch_ptx \"${cuda_arch_ptx}\")\n  string(REGEX MATCHALL \"[0-9()]+\" cuda_arch_bin \"${cuda_arch_bin}\")\n  string(REGEX MATCHALL \"[0-9]+\"   cuda_arch_ptx \"${cuda_arch_ptx}\")\n\n  if(cuda_arch_bin)\n    list(REMOVE_DUPLICATES cuda_arch_bin)\n  endif()\n  if(cuda_arch_ptx)\n    list(REMOVE_DUPLICATES cuda_arch_ptx)\n  endif()\n\n  set(nvcc_flags \"\")\n  set(nvcc_archs_readable \"\")\n\n  # Tell NVCC to add binaries for the specified GPUs\n  foreach(arch ${cuda_arch_bin})\n    if(arch MATCHES \"([0-9]+)\\\\(([0-9]+)\\\\)\")\n      # User explicitly specified ARCH for the concrete CODE\n      list(APPEND nvcc_flags -gencode arch=compute_${CMAKE_MATCH_2},code=sm_${CMAKE_MATCH_1})\n      list(APPEND nvcc_archs_readable sm_${CMAKE_MATCH_1})\n    else()\n      # User didn't explicitly specify ARCH for the concrete CODE, we assume ARCH=CODE\n      list(APPEND nvcc_flags -gencode arch=compute_${arch},code=sm_${arch})\n      list(APPEND nvcc_archs_readable sm_${arch})\n    endif()\n  endforeach()\n\n  # Tell NVCC to add PTX intermediate code for the specified architectures\n  foreach(arch ${cuda_arch_ptx})\n    list(APPEND nvcc_flags -gencode arch=compute_${arch},code=compute_${arch})\n    list(APPEND nvcc_archs_readable compute_${arch})\n  endforeach()\n\n  string(REPLACE \";\" \" \" nvcc_archs_readable \"${nvcc_archs_readable}\")\n  set(${out_variable}          ${nvcc_flags}          PARENT_SCOPE)\n  set(${out_variable}_readable ${nvcc_archs_readable} PARENT_SCOPE)\nendfunction()\n"
  },
  {
    "path": "config/darwin.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n#-------------------------------------------------------------------------------\n#  Template configuration for compiling MXNet\n#\n#  If you want to change the configuration, please use the following steps.\n#  Assume you are on the root directory of mxnet. First copy this file so that\n#  any local changes will be ignored by git\n#\n#  $ cp config/darwin.cmake config.cmake\n#\n#  Next modify the according entries, and then compile by\n#\n#  $ mkdir build; cd build\n#  $ cmake  ..\n#  $ cmake --build .\n#\n# Specify `cmake --build . --parallel N` to set the number of parallel compilation jobs.\n# Default is derived from CPUs available.\n#\n#-------------------------------------------------------------------------------\n\n#---------------------------------------------\n# Common libraries\n#---------------------------------------------\nset(USE_BLAS \"apple\" CACHE STRING \"BLAS Vendor\")\n\nset(USE_OPENCV ON CACHE BOOL \"Build with OpenCV support\")\nset(OPENCV_ROOT \"\" CACHE BOOL \"OpenCV install path. Supports autodetection.\")\n\nset(USE_OPENMP OFF CACHE BOOL \"Build with Openmp support\")\n\nset(USE_ONEDNN ON CACHE BOOL \"Build with oneDNN support\")\n\nset(USE_LAPACK ON CACHE BOOL \"Build with lapack support\")\n\nset(USE_TVM_OP OFF CACHE BOOL \"Enable use of TVM operator build system.\")\n\n#---------------------\n# Compilers\n#--------------------\n# Compilers are usually autodetected. Uncomment and modify the next 3 lines to\n# choose manually:\n\n# set(CMAKE_C_COMPILER \"\" CACHE BOOL \"C compiler\")\n# set(CMAKE_CXX_COMPILER \"\" CACHE BOOL \"C++ compiler\")\n# set(CMAKE_CUDA_COMPILER \"\" CACHE BOOL \"Cuda compiler (nvcc)\")\n\n\n#---------------------------------------------\n# CPU instruction sets: The support is autodetected if turned ON\n#---------------------------------------------\nset(USE_SSE ON CACHE BOOL \"Build with x86 SSE instruction support\")\nset(USE_F16C ON CACHE BOOL \"Build with x86 F16C instruction support\")\n\n\n#----------------------------\n# distributed computing\n#----------------------------\nset(USE_DIST_KVSTORE OFF CACHE BOOL \"Build with DIST_KVSTORE support\")\n\n\n#----------------------------\n# performance settings\n#----------------------------\nset(USE_OPERATOR_TUNING ON CACHE BOOL  \"Enable auto-tuning of operators\")\nset(USE_GPERFTOOLS OFF CACHE BOOL \"Build with GPerfTools support\")\nset(USE_JEMALLOC OFF CACHE BOOL \"Build with Jemalloc support\")\n\n\n#----------------------------\n# additional operators\n#----------------------------\n# path to folders containing projects specific operators that you don't want to\n# put in src/operators\nSET(EXTRA_OPERATORS \"\" CACHE PATH \"EXTRA OPERATORS PATH\")\n\n\n#---------------------------------------------\n# GPU support\n#---------------------------------------------\nset(USE_CUDA OFF CACHE BOOL \"Build with CUDA support\")\nset(USE_CUDNN OFF CACHE BOOL \"Build with cudnn support, if found\")\nset(USE_CUTENSOR OFF CACHE BOOL \"Build with cutensor support, if found\")\n\n# Target NVIDIA GPU achitecture.\n# Valid options are \"Auto\" for autodetection, \"All\" for all available\n# architectures or a list of architectures by compute capability number, such as\n# \"7.0\" or \"7.0;7.5\" as well as name, such as \"Volta\" or \"Volta;Turing\".\n# The value specified here is passed to cmake's CUDA_SELECT_NVCC_ARCH_FLAGS to\n# obtain the compilation flags for nvcc.\n#\n# When compiling on a machine without GPU, autodetection will fail and you\n# should instead specify the target architecture manually to avoid excessive\n# compilation times.\nset(MXNET_CUDA_ARCH \"Auto\" CACHE STRING \"Target NVIDIA GPU achitecture\")\n\n\n#----------------------------\n# other features\n#----------------------------\n# Create C++ interface package\nset(USE_CPP_PACKAGE OFF CACHE BOOL \"Build C++ Package\")\n\n# Use int64_t type to represent the total number of elements in a tensor\n# This will cause performance degradation reported in issue #14496\n# Set to 1 for large tensor with tensor size greater than INT32_MAX i.e. 2147483647\n# Note: the size of each dimension is still bounded by INT32_MAX\nset(USE_INT64_TENSOR_SIZE ON CACHE BOOL \"Use int64_t to represent the total number of elements in a tensor\")\n\n# Other GPU features\nset(USE_NCCL \"Use NVidia NCCL with CUDA\" OFF)\nset(NCCL_ROOT \"\" CACHE BOOL \"NCCL install path. Supports autodetection.\")\nset(USE_NVTX ON CACHE BOOL \"Build with NVTX support\")\n"
  },
  {
    "path": "config/distribution/darwin_cpu.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset(CMAKE_BUILD_TYPE \"Distribution\" CACHE STRING \"Build type\")\nset(CFLAGS \"-mno-avx\" CACHE STRING \"CFLAGS\")\nset(CXXFLAGS \"-mno-avx\" CACHE STRING \"CXXFLAGS\")\n\nset(USE_BLAS \"apple\" CACHE STRING \"BLAS Vendor\")\n\nset(USE_CUDA OFF CACHE BOOL \"Build with CUDA support\")\nset(USE_OPENCV ON CACHE BOOL \"Build with OpenCV support\")\nset(USE_OPENMP OFF CACHE BOOL \"Build with Openmp support\")\nset(USE_ONEDNN ON CACHE BOOL \"Build with oneDNN support\")\nset(USE_LAPACK ON CACHE BOOL \"Build with lapack support\")\nset(USE_TVM_OP OFF CACHE BOOL \"Enable use of TVM operator build system.\")\nset(USE_SSE ON CACHE BOOL \"Build with x86 SSE instruction support\")\nset(USE_F16C OFF CACHE BOOL \"Build with x86 F16C instruction support\")\nset(USE_LIBJPEG_TURBO ON CACHE BOOL \"Build with libjpeg-turbo\")\nset(USE_DIST_KVSTORE ON CACHE BOOL \"Build with DIST_KVSTORE support\")\n"
  },
  {
    "path": "config/distribution/darwin_cpu_mkl.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset(CMAKE_BUILD_TYPE \"Distribution\" CACHE STRING \"Build type\")\nset(CFLAGS \"-mno-avx\" CACHE STRING \"CFLAGS\")\nset(CXXFLAGS \"-mno-avx\" CACHE STRING \"CXXFLAGS\")\n\nset(USE_BLAS \"mkl\" CACHE STRING \"BLAS Vendor\")\nset(BLA_STATIC ON CACHE BOOL \"Use static libraries\")\n\nset(USE_CUDA OFF CACHE BOOL \"Build with CUDA support\")\nset(USE_OPENCV ON CACHE BOOL \"Build with OpenCV support\")\nset(USE_OPENMP OFF CACHE BOOL \"Build with Openmp support\")\nset(USE_ONEDNN ON CACHE BOOL \"Build with oneDNN support\")\nset(USE_LAPACK ON CACHE BOOL \"Build with lapack support\")\nset(USE_TVM_OP OFF CACHE BOOL \"Enable use of TVM operator build system.\")\nset(USE_SSE ON CACHE BOOL \"Build with x86 SSE instruction support\")\nset(USE_F16C OFF CACHE BOOL \"Build with x86 F16C instruction support\")\nset(USE_LIBJPEG_TURBO ON CACHE BOOL \"Build with libjpeg-turbo\")\nset(USE_DIST_KVSTORE ON CACHE BOOL \"Build with DIST_KVSTORE support\")\n"
  },
  {
    "path": "config/distribution/darwin_native.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset(CMAKE_BUILD_TYPE \"Distribution\" CACHE STRING \"Build type\")\nset(CFLAGS \"-mno-avx\" CACHE STRING \"CFLAGS\")\nset(CXXFLAGS \"-mno-avx\" CACHE STRING \"CXXFLAGS\")\n\nset(USE_BLAS \"apple\" CACHE STRING \"BLAS Vendor\")\n\nset(USE_CUDA OFF CACHE BOOL \"Build with CUDA support\")\nset(USE_OPENCV ON CACHE BOOL \"Build with OpenCV support\")\nset(USE_OPENMP OFF CACHE BOOL \"Build with Openmp support\")\nset(USE_ONEDNN OFF CACHE BOOL \"Build with oneDNN support\")\nset(USE_LAPACK ON CACHE BOOL \"Build with lapack support\")\nset(USE_TVM_OP OFF CACHE BOOL \"Enable use of TVM operator build system.\")\nset(USE_SSE ON CACHE BOOL \"Build with x86 SSE instruction support\")\nset(USE_F16C OFF CACHE BOOL \"Build with x86 F16C instruction support\")\nset(USE_LIBJPEG_TURBO ON CACHE BOOL \"Build with libjpeg-turbo\")\nset(USE_DIST_KVSTORE ON CACHE BOOL \"Build with DIST_KVSTORE support\")\n"
  },
  {
    "path": "config/distribution/linux_cpu.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset(CMAKE_BUILD_TYPE \"Distribution\" CACHE STRING \"Build type\")\nset(CFLAGS \"-mno-avx\" CACHE STRING \"CFLAGS\")\nset(CXXFLAGS \"-mno-avx\" CACHE STRING \"CXXFLAGS\")\n\nset(USE_BLAS \"open\" CACHE STRING \"BLAS Vendor\")\nset(USE_CUDA OFF CACHE BOOL \"Build with CUDA support\")\nset(USE_OPENCV ON CACHE BOOL \"Build with OpenCV support\")\nset(USE_OPENMP ON CACHE BOOL \"Build with Openmp support\")\nset(USE_ONEDNN ON CACHE BOOL \"Build with oneDNN support\")\nset(USE_LAPACK ON CACHE BOOL \"Build with lapack support\")\nset(USE_TVM_OP OFF CACHE BOOL \"Enable use of TVM operator build system.\")\nset(USE_SSE ON CACHE BOOL \"Build with x86 SSE instruction support\")\nset(USE_F16C OFF CACHE BOOL \"Build with x86 F16C instruction support\")\nset(USE_LIBJPEG_TURBO ON CACHE BOOL \"Build with libjpeg-turbo\")\nset(USE_DIST_KVSTORE ON CACHE BOOL \"Build with DIST_KVSTORE support\")\n"
  },
  {
    "path": "config/distribution/linux_cpu_mkl.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset(CMAKE_BUILD_TYPE \"Distribution\" CACHE STRING \"Build type\")\nset(CFLAGS \"-mno-avx\" CACHE STRING \"CFLAGS\")\nset(CXXFLAGS \"-mno-avx\" CACHE STRING \"CXXFLAGS\")\n\nset(USE_BLAS \"mkl\" CACHE STRING \"BLAS Vendor\")\nset(BLA_STATIC ON CACHE BOOL \"Use static libraries\")\n\nset(USE_CUDA OFF CACHE BOOL \"Build with CUDA support\")\nset(USE_OPENCV ON CACHE BOOL \"Build with OpenCV support\")\nset(USE_OPENMP ON CACHE BOOL \"Build with Openmp support\")\nset(USE_ONEDNN ON CACHE BOOL \"Build with oneDNN support\")\nset(USE_LAPACK ON CACHE BOOL \"Build with lapack support\")\nset(USE_TVM_OP OFF CACHE BOOL \"Enable use of TVM operator build system.\")\nset(USE_SSE ON CACHE BOOL \"Build with x86 SSE instruction support\")\nset(USE_F16C OFF CACHE BOOL \"Build with x86 F16C instruction support\")\nset(USE_LIBJPEG_TURBO ON CACHE BOOL \"Build with libjpeg-turbo\")\nset(USE_DIST_KVSTORE ON CACHE BOOL \"Build with DIST_KVSTORE support\")\n"
  },
  {
    "path": "config/distribution/linux_cu100.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset(CMAKE_BUILD_TYPE \"Distribution\" CACHE STRING \"Build type\")\nset(CFLAGS \"-mno-avx\" CACHE STRING \"CFLAGS\")\nset(CXXFLAGS \"-mno-avx\" CACHE STRING \"CXXFLAGS\")\n\nset(USE_BLAS \"open\" CACHE STRING \"BLAS Vendor\")\nset(USE_CUDA ON CACHE BOOL \"Build with CUDA support\")\nset(USE_CUDNN ON CACHE BOOL \"Build with CUDNN support\")\nset(USE_NCCL ON CACHE BOOL \"Build with NCCL support\")\nset(USE_OPENCV ON CACHE BOOL \"Build with OpenCV support\")\nset(USE_OPENMP ON CACHE BOOL \"Build with Openmp support\")\nset(USE_ONEDNN ON CACHE BOOL \"Build with oneDNN support\")\nset(USE_LAPACK ON CACHE BOOL \"Build with lapack support\")\nset(USE_TVM_OP OFF CACHE BOOL \"Enable use of TVM operator build system.\")\nset(USE_SSE ON CACHE BOOL \"Build with x86 SSE instruction support\")\nset(USE_F16C OFF CACHE BOOL \"Build with x86 F16C instruction support\")\nset(USE_LIBJPEG_TURBO ON CACHE BOOL \"Build with libjpeg-turbo\")\nset(USE_DIST_KVSTORE ON CACHE BOOL \"Build with DIST_KVSTORE support\")\nset(CUDACXX \"/usr/local/cuda-10.0/bin/nvcc\" CACHE STRING \"Cuda compiler\")\nset(MXNET_CUDA_ARCH \"3.0;5.0;6.0;7.0\" CACHE STRING \"Cuda architectures\")\n"
  },
  {
    "path": "config/distribution/linux_cu101.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset(CMAKE_BUILD_TYPE \"Distribution\" CACHE STRING \"Build type\")\nset(CFLAGS \"-mno-avx\" CACHE STRING \"CFLAGS\")\nset(CXXFLAGS \"-mno-avx\" CACHE STRING \"CXXFLAGS\")\nset(CFLAGS \"-mno-avx\" CACHE STRING \"CFLAGS\")\nset(CXXFLAGS \"-mno-avx\" CACHE STRING \"CXXFLAGS\")\n\nset(USE_BLAS \"open\" CACHE STRING \"BLAS Vendor\")\nset(USE_CUDA ON CACHE BOOL \"Build with CUDA support\")\nset(USE_CUDNN ON CACHE BOOL \"Build with CUDNN support\")\nset(USE_NCCL ON CACHE BOOL \"Build with NCCL support\")\nset(USE_OPENCV ON CACHE BOOL \"Build with OpenCV support\")\nset(USE_OPENMP ON CACHE BOOL \"Build with Openmp support\")\nset(USE_ONEDNN ON CACHE BOOL \"Build with oneDNN support\")\nset(USE_LAPACK ON CACHE BOOL \"Build with lapack support\")\nset(USE_TVM_OP OFF CACHE BOOL \"Enable use of TVM operator build system.\")\nset(USE_SSE ON CACHE BOOL \"Build with x86 SSE instruction support\")\nset(USE_F16C OFF CACHE BOOL \"Build with x86 F16C instruction support\")\nset(USE_LIBJPEG_TURBO ON CACHE BOOL \"Build with libjpeg-turbo\")\nset(USE_DIST_KVSTORE ON CACHE BOOL \"Build with DIST_KVSTORE support\")\nset(CUDACXX \"/usr/local/cuda-10.1/bin/nvcc\" CACHE STRING \"Cuda compiler\")\nset(MXNET_CUDA_ARCH \"3.0;5.0;6.0;7.0\" CACHE STRING \"Cuda architectures\")\n"
  },
  {
    "path": "config/distribution/linux_cu102.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset(CMAKE_BUILD_TYPE \"Distribution\" CACHE STRING \"Build type\")\nset(CFLAGS \"-mno-avx\" CACHE STRING \"CFLAGS\")\nset(CXXFLAGS \"-mno-avx\" CACHE STRING \"CXXFLAGS\")\n\nset(USE_BLAS \"open\" CACHE STRING \"BLAS Vendor\")\nset(USE_CUDA ON CACHE BOOL \"Build with CUDA support\")\nset(USE_CUDNN ON CACHE BOOL \"Build with CUDNN support\")\nset(USE_NCCL ON CACHE BOOL \"Build with NCCL support\")\nset(USE_OPENCV ON CACHE BOOL \"Build with OpenCV support\")\nset(USE_OPENMP ON CACHE BOOL \"Build with Openmp support\")\nset(USE_ONEDNN ON CACHE BOOL \"Build with oneDNN support\")\nset(USE_LAPACK ON CACHE BOOL \"Build with lapack support\")\nset(USE_TVM_OP OFF CACHE BOOL \"Enable use of TVM operator build system.\")\nset(USE_SSE ON CACHE BOOL \"Build with x86 SSE instruction support\")\nset(USE_F16C OFF CACHE BOOL \"Build with x86 F16C instruction support\")\nset(USE_LIBJPEG_TURBO ON CACHE BOOL \"Build with libjpeg-turbo\")\nset(USE_DIST_KVSTORE ON CACHE BOOL \"Build with DIST_KVSTORE support\")\nset(CUDACXX \"/usr/local/cuda-10.2/bin/nvcc\" CACHE STRING \"Cuda compiler\")\nset(MXNET_CUDA_ARCH \"3.0;5.0;6.0;7.0\" CACHE STRING \"Cuda architectures\")\n"
  },
  {
    "path": "config/distribution/linux_cu110.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset(CMAKE_BUILD_TYPE \"Distribution\" CACHE STRING \"Build type\")\nset(CFLAGS \"-mno-avx\" CACHE STRING \"CFLAGS\")\nset(CXXFLAGS \"-mno-avx\" CACHE STRING \"CXXFLAGS\")\n\nset(USE_BLAS \"open\" CACHE STRING \"BLAS Vendor\")\nset(USE_CUDA ON CACHE BOOL \"Build with CUDA support\")\nset(USE_CUDNN ON CACHE BOOL \"Build with CUDNN support\")\nset(USE_NCCL ON CACHE BOOL \"Build with NCCL support\")\nset(USE_OPENCV ON CACHE BOOL \"Build with OpenCV support\")\nset(USE_OPENMP ON CACHE BOOL \"Build with Openmp support\")\nset(USE_ONEDNN ON CACHE BOOL \"Build with oneDNN support\")\nset(USE_LAPACK ON CACHE BOOL \"Build with lapack support\")\nset(USE_TVM_OP OFF CACHE BOOL \"Enable use of TVM operator build system.\")\nset(USE_SSE ON CACHE BOOL \"Build with x86 SSE instruction support\")\nset(USE_F16C OFF CACHE BOOL \"Build with x86 F16C instruction support\")\nset(USE_LIBJPEG_TURBO ON CACHE BOOL \"Build with libjpeg-turbo\")\nset(USE_DIST_KVSTORE ON CACHE BOOL \"Build with DIST_KVSTORE support\")\nset(CUDACXX \"/usr/local/cuda-11.0/bin/nvcc\" CACHE STRING \"Cuda compiler\")\nset(MXNET_CUDA_ARCH \"5.0;6.0;7.0;8.0\" CACHE STRING \"Cuda architectures\")\n"
  },
  {
    "path": "config/distribution/linux_cu112.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset(CMAKE_BUILD_TYPE \"Distribution\" CACHE STRING \"Build type\")\nset(CFLAGS \"-mno-avx\" CACHE STRING \"CFLAGS\")\nset(CXXFLAGS \"-mno-avx\" CACHE STRING \"CXXFLAGS\")\n\nset(USE_BLAS \"open\" CACHE STRING \"BLAS Vendor\")\nset(USE_CUDA ON CACHE BOOL \"Build with CUDA support\")\nset(USE_CUDNN ON CACHE BOOL \"Build with CUDNN support\")\nset(USE_NCCL ON CACHE BOOL \"Build with NCCL support\")\nset(USE_OPENCV ON CACHE BOOL \"Build with OpenCV support\")\nset(USE_OPENMP ON CACHE BOOL \"Build with Openmp support\")\nset(USE_ONEDNN ON CACHE BOOL \"Build with oneDNN support\")\nset(USE_LAPACK ON CACHE BOOL \"Build with lapack support\")\nset(USE_TVM_OP OFF CACHE BOOL \"Enable use of TVM operator build system.\")\nset(USE_SSE ON CACHE BOOL \"Build with x86 SSE instruction support\")\nset(USE_F16C OFF CACHE BOOL \"Build with x86 F16C instruction support\")\nset(USE_LIBJPEG_TURBO ON CACHE BOOL \"Build with libjpeg-turbo\")\nset(USE_DIST_KVSTORE ON CACHE BOOL \"Build with DIST_KVSTORE support\")\nset(CUDACXX \"/usr/local/cuda-11.2/bin/nvcc\" CACHE STRING \"Cuda compiler\")\nset(MXNET_CUDA_ARCH \"5.0;6.0;7.0;8.0;8.6\" CACHE STRING \"Cuda architectures\")\n"
  },
  {
    "path": "config/distribution/linux_cu92.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset(CMAKE_BUILD_TYPE \"Distribution\" CACHE STRING \"Build type\")\nset(CFLAGS \"-mno-avx\" CACHE STRING \"CFLAGS\")\nset(CXXFLAGS \"-mno-avx\" CACHE STRING \"CXXFLAGS\")\n\nset(USE_BLAS \"open\" CACHE STRING \"BLAS Vendor\")\nset(USE_CUDA ON CACHE BOOL \"Build with CUDA support\")\nset(USE_CUDNN ON CACHE BOOL \"Build with CUDNN support\")\nset(USE_NCCL ON CACHE BOOL \"Build with NCCL support\")\nset(USE_OPENCV ON CACHE BOOL \"Build with OpenCV support\")\nset(USE_OPENMP ON CACHE BOOL \"Build with Openmp support\")\nset(USE_ONEDNN ON CACHE BOOL \"Build with oneDNN support\")\nset(USE_LAPACK ON CACHE BOOL \"Build with lapack support\")\nset(USE_TVM_OP OFF CACHE BOOL \"Enable use of TVM operator build system.\")\nset(USE_SSE ON CACHE BOOL \"Build with x86 SSE instruction support\")\nset(USE_F16C OFF CACHE BOOL \"Build with x86 F16C instruction support\")\nset(USE_LIBJPEG_TURBO ON CACHE BOOL \"Build with libjpeg-turbo\")\nset(CUDACXX \"/usr/local/cuda-9.2/bin/nvcc\" CACHE STRING \"Cuda compiler\")\nset(MXNET_CUDA_ARCH \"3.0;5.0;6.0;7.0\" CACHE STRING \"Cuda architectures\")\n"
  },
  {
    "path": "config/distribution/linux_native.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset(CMAKE_BUILD_TYPE \"Distribution\" CACHE STRING \"Build type\")\nset(CFLAGS \"-mno-avx\" CACHE STRING \"CFLAGS\")\nset(CXXFLAGS \"-mno-avx\" CACHE STRING \"CXXFLAGS\")\n\nset(USE_BLAS \"open\" CACHE STRING \"BLAS Vendor\")\nset(USE_CUDA OFF CACHE BOOL \"Build with CUDA support\")\nset(USE_OPENCV ON CACHE BOOL \"Build with OpenCV support\")\nset(USE_OPENMP ON CACHE BOOL \"Build with Openmp support\")\nset(USE_ONEDNN OFF CACHE BOOL \"Build with oneDNN support\")\nset(USE_LAPACK ON CACHE BOOL \"Build with lapack support\")\nset(USE_TVM_OP OFF CACHE BOOL \"Enable use of TVM operator build system.\")\nset(USE_SSE ON CACHE BOOL \"Build with x86 SSE instruction support\")\nset(USE_F16C OFF CACHE BOOL \"Build with x86 F16C instruction support\")\nset(USE_LIBJPEG_TURBO ON CACHE BOOL \"Build with libjpeg-turbo\")\n"
  },
  {
    "path": "config/linux.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n#-------------------------------------------------------------------------------\n#  Template configuration for compiling MXNet\n#\n#  If you want to change the configuration, please use the following steps.\n#  Assume you are on the root directory of mxnet. First copy this file so that\n#  any local changes will be ignored by git\n#\n#  $ cp config/linux.cmake config.cmake\n#\n#  Next modify the according entries, and then compile by\n#\n#  $ mkdir build; cd build\n#  $ cmake ..\n#  $ cmake --build .\n#\n# Specify `cmake --build . --parallel N` to set the number of parallel compilation jobs.\n# Default is derived from CPUs available.\n#\n#-------------------------------------------------------------------------------\n\n#---------------------------------------------\n# GPU support\n#---------------------------------------------\nset(USE_CUDA OFF CACHE BOOL \"Build with CUDA support\")\nset(USE_CUDNN OFF CACHE BOOL \"Build with cudnn support, if found\")\nset(USE_CUTENSOR OFF CACHE BOOL \"Build with cutensor support, if found\")\n\n# Target NVIDIA GPU achitecture.\n# Valid options are \"Auto\" for autodetection, \"All\" for all available\n# architectures or a list of architectures by compute capability number, such as\n# \"7.0\" or \"7.0;7.5\" as well as name, such as \"Volta\" or \"Volta;Turing\".\n# The value specified here is passed to cmake's CUDA_SELECT_NVCC_ARCH_FLAGS to\n# obtain the compilation flags for nvcc.\n#\n# When compiling on a machine without GPU, autodetection will fail and you\n# should instead specify the target architecture manually to avoid excessive\n# compilation times.\nset(MXNET_CUDA_ARCH \"Auto\" CACHE STRING \"Target NVIDIA GPU achitecture\")\n\n#---------------------------------------------\n# Common libraries\n#---------------------------------------------\nset(USE_OPENCV ON CACHE BOOL \"Build with OpenCV support\")\nset(OPENCV_ROOT \"\" CACHE BOOL \"OpenCV install path. Supports autodetection.\")\n\nset(USE_OPENMP ON CACHE BOOL \"Build with Openmp support\")\n\nset(USE_ONEDNN ON CACHE BOOL \"Build with oneDNN support\")\n\nset(USE_LAPACK ON CACHE BOOL \"Build with lapack support\")\n\nset(USE_TVM_OP OFF CACHE BOOL \"Enable use of TVM operator build system.\")\n\n#---------------------\n# Compilers\n#--------------------\n# Compilers are usually autodetected. Uncomment and modify the next 3 lines to\n# choose manually:\n\n# set(CMAKE_C_COMPILER \"\" CACHE BOOL \"C compiler\")\n# set(CMAKE_CXX_COMPILER \"\" CACHE BOOL \"C++ compiler\")\n# set(CMAKE_CUDA_COMPILER \"\" CACHE BOOL \"Cuda compiler (nvcc)\")\n\n\n#---------------------------------------------\n# CPU instruction sets: The support is autodetected if turned ON\n#---------------------------------------------\nset(USE_SSE ON CACHE BOOL \"Build with x86 SSE instruction support\")\nset(USE_F16C ON CACHE BOOL \"Build with x86 F16C instruction support\")\n\n\n#----------------------------\n# distributed computing\n#----------------------------\nset(USE_DIST_KVSTORE OFF CACHE BOOL \"Build with DIST_KVSTORE support\")\n\n\n#----------------------------\n# performance settings\n#----------------------------\nset(USE_OPERATOR_TUNING ON CACHE BOOL  \"Enable auto-tuning of operators\")\nset(USE_GPERFTOOLS OFF CACHE BOOL \"Build with GPerfTools support\")\nset(USE_JEMALLOC OFF CACHE BOOL \"Build with Jemalloc support\")\n\n\n#----------------------------\n# additional operators\n#----------------------------\n# path to folders containing projects specific operators that you don't want to\n# put in src/operators\nSET(EXTRA_OPERATORS \"\" CACHE PATH \"EXTRA OPERATORS PATH\")\n\n\n#----------------------------\n# other features\n#----------------------------\n# Create C++ interface package\nset(USE_CPP_PACKAGE OFF CACHE BOOL \"Build C++ Package\")\n\n# Use int64_t type to represent index and the number of elements in a tensor\n# This will cause performance degradation reported in issue #14496\n# Set to 1 for large tensor with tensor size greater than INT32_MAX i.e. 2147483647\nset(USE_INT64_TENSOR_SIZE ON CACHE BOOL \"Use int64_t to represent the number of elements in a tensor\")\n\n# Other GPU features\nset(USE_NCCL \"Use NVidia NCCL with CUDA\" OFF)\nset(NCCL_ROOT \"\" CACHE BOOL \"NCCL install path. Supports autodetection.\")\nset(USE_NVTX ON CACHE BOOL \"Build with NVTX support\")\n"
  },
  {
    "path": "config/linux_gpu.cmake",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n#-------------------------------------------------------------------------------\n#  Template configuration for compiling MXNet\n#\n#  If you want to change the configuration, please use the following steps.\n#  Assume you are on the root directory of mxnet. First copy this file so that\n#  any local changes will be ignored by git\n#\n#  $ cp config/linux_gpu.cmake config.cmake\n#\n#  Next modify the entries in the config.cmake like MXNET_CUDA_ARCH to set the specific\n#  GPU architecture, and then compile by\n#\n#  $ mkdir build; cd build\n#  $ cmake ..\n#  $ cmake --build .\n#\n# Specify `cmake --build . --parallel N` to set the number of parallel compilation jobs.\n# Default is derived from CPUs available.\n#\n#-------------------------------------------------------------------------------\n\n#---------------------------------------------\n# GPU support\n#---------------------------------------------\nset(USE_CUDA ON CACHE BOOL \"Build with CUDA support\")\nset(USE_CUDNN ON CACHE BOOL \"Build with cudnn support, if found\")\nset(USE_CUTENSOR ON CACHE BOOL \"Build with cutensor support, if found\")\n\n# Target NVIDIA GPU achitecture.\n# Valid options are:\n#   - \"Auto\" for autodetection, will try and discover which GPU architecture to use by\n#            looking at the available GPUs on the machine that you're building on\n#   - \"All\" for all available GPU architectures supported by the version of CUDA installed\n#   - \"specific GPU architectures\" by giving the compute capability number such as\n#            \"7.0\" or \"7.0;7.5\" (ie. sm_70 or sm_75) or you can specify the name like:\n#            \"Volta\" or \"Volta;Turing\", be sure not to use quotes (ie. just set to 7.0)\n# The value specified here is passed to cmake's CUDA_SELECT_NVCC_ARCH_FLAGS to\n# obtain the compilation flags for nvcc.\n#\n# When compiling on a machine without GPU, autodetection will fail and you\n# should instead specify the target architecture manually.\nset(MXNET_CUDA_ARCH \"Auto\" CACHE STRING \"Target NVIDIA GPU achitecture\")\n\n#---------------------------------------------\n# Common libraries\n#---------------------------------------------\nset(USE_OPENCV ON CACHE BOOL \"Build with OpenCV support\")\nset(OPENCV_ROOT \"\" CACHE BOOL \"OpenCV install path. Supports autodetection.\")\n\nset(USE_OPENMP ON CACHE BOOL \"Build with Openmp support\")\n\nset(USE_ONEDNN ON CACHE BOOL \"Build with oneDNN support\")\n\nset(USE_LAPACK ON CACHE BOOL \"Build with lapack support\")\n\nset(USE_TVM_OP OFF CACHE BOOL \"Enable use of TVM operator build system.\")\n\n#---------------------\n# Compilers\n#--------------------\n# Compilers are usually autodetected. Uncomment and modify the next 3 lines to\n# choose manually:\n\n# set(CMAKE_C_COMPILER \"\" CACHE BOOL \"C compiler\")\n# set(CMAKE_CXX_COMPILER \"\" CACHE BOOL \"C++ compiler\")\n# set(CMAKE_CUDA_COMPILER \"\" CACHE BOOL \"Cuda compiler (nvcc)\")\n\n\n#---------------------------------------------\n# CPU instruction sets: The support is autodetected if turned ON\n#---------------------------------------------\nset(USE_SSE ON CACHE BOOL \"Build with x86 SSE instruction support\")\nset(USE_F16C ON CACHE BOOL \"Build with x86 F16C instruction support\")\n\n\n#----------------------------\n# distributed computing\n#----------------------------\nset(USE_DIST_KVSTORE OFF CACHE BOOL \"Build with DIST_KVSTORE support\")\n\n\n#----------------------------\n# performance settings\n#----------------------------\nset(USE_OPERATOR_TUNING ON CACHE BOOL  \"Enable auto-tuning of operators\")\nset(USE_GPERFTOOLS OFF CACHE BOOL \"Build with GPerfTools support\")\nset(USE_JEMALLOC OFF CACHE BOOL \"Build with Jemalloc support\")\n\n\n#----------------------------\n# additional operators\n#----------------------------\n# path to folders containing projects specific operators that you don't want to\n# put in src/operators\nSET(EXTRA_OPERATORS \"\" CACHE PATH \"EXTRA OPERATORS PATH\")\n\n\n#----------------------------\n# other features\n#----------------------------\n# Create C++ interface package\nset(USE_CPP_PACKAGE OFF CACHE BOOL \"Build C++ Package\")\n\n# Use int64_t type to represent the total number of elements in a tensor\n# This will cause performance degradation reported in issue #14496\n# Set to 1 for large tensor with tensor size greater than INT32_MAX i.e. 2147483647\n# Note: the size of each dimension is still bounded by INT32_MAX\nset(USE_INT64_TENSOR_SIZE ON CACHE BOOL \"Use int64_t to represent the total number of elements in a tensor\")\n\n# Other GPU features\nset(USE_NCCL \"Use NVidia NCCL with CUDA\" OFF)\nset(NCCL_ROOT \"\" CACHE BOOL \"NCCL install path. Supports autodetection.\")\nset(USE_NVML OFF CACHE BOOL \"Build with NVML support\")\nset(USE_NVTX ON CACHE BOOL \"Build with NVTX support\")\n"
  },
  {
    "path": "conftest.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"conftest.py contains configuration for pytest.\n\nConfiguration file for tests in tests/ and scripts/ folders.\n\nNote that fixtures of higher-scoped fixtures (such as ``session``) are\ninstantiated before lower-scoped fixtures (such as ``function``).\n\n\"\"\"\n\nimport logging\nimport os\nimport random\n\nimport pytest\n\n\ndef pytest_configure(config):\n    # Load the user's locale settings to verify that MXNet works correctly when the C locale is set\n    # to anything other than the default value. Please see #16134 for an example of a bug caused by\n    # incorrect handling of C locales.\n    import locale\n    locale.setlocale(locale.LC_ALL, \"\")\n\n\ndef pytest_sessionfinish(session, exitstatus):\n    if exitstatus == 5:  # Don't fail if no tests were run\n        session.exitstatus = 0\n\n\n@pytest.hookimpl(tryfirst=True, hookwrapper=True)\ndef pytest_runtest_makereport(item, call):\n    \"\"\"Make test outcome available to fixture.\n\n    https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures\n    \"\"\"\n    # execute all other hooks to obtain the report object\n    outcome = yield\n    rep = outcome.get_result()\n\n    # set a report attribute for each phase of a call, which can\n    # be \"setup\", \"call\", \"teardown\"\n    setattr(item, \"rep_\" + rep.when, rep)\n\n\n@pytest.fixture(scope='module', autouse=True)\ndef module_scope_waitall(request):\n    \"\"\"A module scope fixture to issue waitall() operations between test modules.\"\"\"\n    yield\n\n    try:\n        import mxnet as mx\n        mx.npx.waitall()\n    except:\n        # Use print() as module level fixture logging.warning messages never\n        # shown to users. https://github.com/pytest-dev/pytest/issues/7819\n        print('Unable to import numpy/mxnet. Skip mx.npx.waitall().')\n\n\n@pytest.fixture(scope='module', autouse=True)\ndef module_scope_seed(request):\n    \"\"\"Module scope fixture to help reproduce test segfaults\n\n    Sets and outputs rng seeds.\n\n    The segfault-debug procedure on a module called test_module.py is:\n\n    1. run \"pytest --verbose test_module.py\".  A seg-faulting output might be:\n\n       [INFO] np, mx and python random seeds = 4018804151\n       test_module.test1 ... ok\n       test_module.test2 ... Illegal instruction (core dumped)\n\n    2. Copy the module-starting seed into the next command, then run:\n\n       MXNET_MODULE_SEED=4018804151 pytest --log-level=DEBUG --verbose test_module.py\n\n       Output might be:\n\n       [WARNING] **** module-level seed is set: all tests running deterministically ****\n       [INFO] np, mx and python random seeds = 4018804151\n       test_module.test1 ... [DEBUG] np and mx random seeds = 3935862516\n       ok\n       test_module.test2 ... [DEBUG] np and mx random seeds = 1435005594\n       Illegal instruction (core dumped)\n\n    3. Copy the segfaulting-test seed into the command:\n       MXNET_TEST_SEED=1435005594 pytest --log-level=DEBUG --verbose test_module.py:test2\n       Output might be:\n\n       [INFO] np, mx and python random seeds = 2481884723\n       test_module.test2 ... [DEBUG] np and mx random seeds = 1435005594\n       Illegal instruction (core dumped)\n\n    3. Finally reproduce the segfault directly under gdb (might need additional os packages)\n       by editing the bottom of test_module.py to be\n\n       if __name__ == '__main__':\n           logging.getLogger().setLevel(logging.DEBUG)\n           test2()\n\n       MXNET_TEST_SEED=1435005594 gdb -ex r --args python test_module.py\n\n    4. When finished debugging the segfault, remember to unset any exported MXNET_ seed\n       variables in the environment to return to non-deterministic testing (a good thing).\n    \"\"\"\n    module_seed_str = os.getenv('MXNET_MODULE_SEED')\n    if module_seed_str is None:\n        seed = random.randint(0, 2**31-1)\n    else:\n        seed = int(module_seed_str)\n        # Use print() as module level fixture logging.warning messages never\n        # shown to users. https://github.com/pytest-dev/pytest/issues/7819\n        print('*** module-level seed is set: all tests running deterministically ***')\n    print('Setting module np/mx/python random seeds, '\n          f'use MXNET_MODULE_SEED={seed} to reproduce.')\n    old_state = random.getstate()\n    random.seed(seed)\n    try:\n        import numpy as np\n        import mxnet as mx\n        np.random.seed(seed)\n        mx.random.seed(seed)\n    except:\n        # Use print() as module level fixture logging.warning messages never\n        # shown to users. https://github.com/pytest-dev/pytest/issues/7819\n        print('Unable to import numpy/mxnet. Skip setting module-level seed.')\n\n    # The MXNET_TEST_SEED environment variable will override MXNET_MODULE_SEED for tests with\n    #  the 'with_seed()' decoration.  Inform the user of this once here at the module level.\n    if os.getenv('MXNET_TEST_SEED') is not None:\n        # Use print() as module level fixture logging.warning messages never\n        # shown to users. https://github.com/pytest-dev/pytest/issues/7819\n        print('*** test-level seed set: all \"@with_seed()\" tests run deterministically ***')\n\n    yield  # run all tests in the module\n\n    random.setstate(old_state)\n\n\n@pytest.fixture(scope='function', autouse=True)\ndef function_scope_seed(request):\n    \"\"\"A function scope fixture that manages rng seeds.\n\n    This fixture automatically initializes the python, numpy and mxnet random\n    number generators randomly on every test run.\n\n    def test_ok_with_random_data():\n        ...\n\n    To fix the seed used for a test case mark the test function with the\n    desired seed:\n\n    @pytest.mark.seed(1)\n    def test_not_ok_with_random_data():\n        '''This testcase actually works.'''\n        assert 17 == random.randint(0, 100)\n\n    When a test fails, the fixture outputs the seed used. The user can then set\n    the environment variable MXNET_TEST_SEED to the value reported, then rerun\n    the test with:\n\n        pytest --verbose -s <test_module_name.py> -k <failing_test>\n\n    To run a test repeatedly, install pytest-repeat and add the --count argument:\n\n        pip install pytest-repeat\n        pytest --verbose -s <test_module_name.py> -k <failing_test> --count 1000\n\n    \"\"\"\n\n    seed = request.node.get_closest_marker('seed')\n    env_seed_str = os.getenv('MXNET_TEST_SEED')\n\n    if seed is not None:\n        seed = seed.args[0]\n        assert isinstance(seed, int)\n    elif env_seed_str is not None:\n        seed = int(env_seed_str)\n    else:\n        seed = random.randint(0, 2**31-1)\n    old_state = random.getstate()\n    random.seed(seed)\n    try:\n        import numpy as np\n        import mxnet as mx\n        np.random.seed(seed)\n        mx.random.seed(seed)\n    except:\n        logging.warning('Unable to import numpy/mxnet. Skip setting function-level seed.')\n\n    seed_message = f'Setting np/mx/python random seeds to {seed}. Use MXNET_TEST_SEED={seed} to reproduce.'\n\n    # Always log seed on DEBUG log level. This makes sure we can find out the\n    # value of the seed even if the test case causes a segfault and subsequent\n    # teardown code is not run.\n    logging.debug(seed_message)\n\n    yield  # run the test\n\n    if request.node.rep_setup.failed:\n        logging.error(\"Setting up a test failed: {}\", request.node.nodeid)\n    elif request.node.rep_call.outcome == 'failed':\n        # Either request.node.rep_setup.failed or request.node.rep_setup.passed should be True\n        assert request.node.rep_setup.passed\n        # On failure also log seed on WARNING log level\n        error_message = f'Error seen with seeded test, use MXNET_TEST_SEED={seed} to reproduce'\n        logging.warning(error_message)\n\n    random.setstate(old_state)\n"
  },
  {
    "path": "contrib/tvmop/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\nfrom .opdef import defop\nfrom .utils import AllTypes, RealTypes\nfrom .utils import assign_by_req, reduce_axes\n\nfrom . import basic\nfrom . import core\n"
  },
  {
    "path": "contrib/tvmop/basic/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\nfrom . import ufunc\n"
  },
  {
    "path": "contrib/tvmop/basic/ufunc.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\nimport tvm\nfrom .. import defop, AllTypes, RealTypes\nfrom .. import assign_by_req, reduce_axes\n\ndef compute_add(dtype, ndim):\n    A = tvm.te.placeholder([tvm.te.size_var() for _ in range(ndim)], name='A', dtype=dtype)\n    B = tvm.te.placeholder([tvm.te.size_var() for _ in range(ndim)], name='B', dtype=dtype)\n    C = tvm.te.compute([tvm.te.size_var() for _ in range(ndim)],\n                    lambda *index: A[index] + B[index], name='C')\n    s = tvm.te.create_schedule(C.op)\n    return s, A, B, C\n\n\n@defop(name=\"vadd\", target=\"cpu\", auto_broadcast=True,\n       dtype=AllTypes, ndim=[5])\ndef vadd(dtype, ndim):\n    s, A, B, C = compute_add(dtype, ndim)\n    axes = [axis for axis in C.op.axis]\n    fused = s[C].fuse(*axes)\n    s[C].parallel(fused)\n\n    return s, [A, B, C]\n\n\n@defop(name=\"cuda_vadd\", target=\"cuda\", auto_broadcast=True,\n       dtype=[\"float32\", \"float64\"], ndim=[5])\ndef vadd_gpu(dtype, ndim):\n    s, A, B, C = compute_add(dtype, ndim)\n    s = tvm.te.create_schedule(C.op)\n    axes = [axis for axis in C.op.axis]\n    fused = s[C].fuse(*axes)\n    bx, tx = s[C].split(fused, factor=64)\n    s[C].bind(bx, tvm.te.thread_axis(\"blockIdx.x\"))\n    s[C].bind(tx, tvm.te.thread_axis(\"threadIdx.x\"))\n    return s, [A, B, C]\n\n\ndef compute_backward_vadd(dtype, ndim, reduce1st, req):\n    # The backward of broadcast op is basically a reduction on broadcast axes.\n    # We label the reduce axes as 1 and other axes as 0, and they form a bit string.\n    # Each bit string correponds to a kernel, so the number of kernels is as many as `2^n`\n    # To reduce it, the bit string is compressed by combining consecutive 0s or 1s.\n    # In this way, the number of bit string (the number of kernels) is reduced to `2 * n`\n    # They compressed bit string is stored in `axes`. And `reduce1st` represents the first bit\n    # of the compressed bit string. Credit to @junrushao1994 and @yzhliu.\n    axes = ([reduce1st, 1 - reduce1st] * ndim)[:ndim]\n    X = tvm.te.placeholder([tvm.te.size_var() for _ in range(ndim)], name='X', dtype=dtype)\n    reducer = tvm.te.comm_reducer(lambda x, y: x + y,\n        lambda t: tvm.tir.const(0, dtype=t), name=\"sum\")\n    ret = reduce_axes(X, axes, reducer)\n    in_grad_a, in_grad = assign_by_req(ret, req)\n    s = tvm.te.create_schedule(in_grad.op)\n    return s, X, in_grad_a, in_grad, [ret, in_grad]\n\n\n@defop(name=\"backward_vadd\", target=\"cpu\", dtype=AllTypes, \n       ndim=[5], reduce1st=[0, 1],\n       req=[\"kWriteTo\", \"kAddTo\"], attrs=[\"reduce1st\", \"req\"])\ndef backward_vadd(dtype, ndim, reduce1st, req):\n    s, X, in_grad_a, in_grad, c_list = compute_backward_vadd(dtype, ndim, reduce1st, req)\n    for t in c_list:\n        axes = [axis for axis in t.op.axis]\n        fused = s[t].fuse(*axes)\n        s[t].parallel(fused)\n    return s, [X, in_grad_a, in_grad]\n\n\n@defop(name=\"cuda_backward_vadd\", target=\"gpu\", dtype=[\"float32\", \"float64\"],\n       ndim=[5], reduce1st=[0, 1],\n       req=[\"kWriteTo\", \"kAddTo\"], attrs=[\"reduce1st\", \"req\"])\ndef backward_vadd_gpu(dtype, ndim, reduce1st, req):\n    s, X, in_grad_a, in_grad, c_list = compute_backward_vadd(dtype, ndim, reduce1st, req)\n    num_thread = 64\n    for t in c_list:\n        block_x = tvm.te.thread_axis(\"blockIdx.x\")\n        thread_x = tvm.te.thread_axis(\"threadIdx.x\")\n        axes = [axis for axis in t.op.axis]\n        fused = s[t].fuse(*axes)\n        bx, tx = s[t].split(fused, factor=num_thread)\n        s[t].bind(bx, block_x)\n        s[t].bind(tx, thread_x)\n    return s, [X, in_grad_a, in_grad]\n\n\ndef compute_degandrad(dtype, ndim, n):\n    A = tvm.te.placeholder([tvm.te.size_var() for _ in range(ndim)], name='A', dtype=dtype)\n    import math\n    if n == 0:\n        B = tvm.te.compute([tvm.te.size_var() for _ in range(ndim)],\n                        lambda *index: A[index] * tvm.tir.const(math.pi, dtype) / tvm.tir.const(180, dtype), name='B')\n    else:\n        B = tvm.te.compute([tvm.te.size_var() for _ in range(ndim)],\n                        lambda *index: A[index] / tvm.tir.const(math.pi, dtype) * tvm.tir.const(180, dtype), name='B')\n    s = tvm.te.create_schedule(B.op)\n    return s, A, B\n\n\n@defop(name=\"deg2rad\", target=\"cpu\", auto_broadcast=False,\n       dtype=[\"float32\", \"float64\"], ndim=list(range(0, 6)))\ndef deg2rad(dtype, ndim):\n    s, A, B = compute_degandrad(dtype, ndim, 0)\n    axes = [axis for axis in B.op.axis]\n    fused = s[B].fuse(*axes)\n    s[B].parallel(fused)\n    return s, [A, B]\n\n\n@defop(name=\"rad2deg\", target=\"cpu\", auto_broadcast=False,\n       dtype=[\"float32\", \"float64\"], ndim=list(range(0, 6)))\ndef rad2deg(dtype, ndim):\n    s, A, B = compute_degandrad(dtype, ndim, 1)\n    axes = [axis for axis in B.op.axis]\n    fused = s[B].fuse(*axes)\n    s[B].parallel(fused)\n    return s, [A, B]\n\n\n@defop(name=\"cuda_deg2rad\", target=\"cuda\", auto_broadcast=False,\n       dtype=[\"float32\", \"float64\"], ndim=list(range(0, 6)))\ndef deg2rad_gpu(dtype, ndim):\n    s, A, B = compute_degandrad(dtype, ndim, 0)\n    s = tvm.te.create_schedule(B.op)\n    axes = [axis for axis in B.op.axis]\n    fused = s[B].fuse(*axes)\n    bx, tx = s[B].split(fused, factor=64)\n    s[B].bind(bx, tvm.te.thread_axis(\"blockIdx.x\"))\n    s[B].bind(tx, tvm.te.thread_axis(\"threadIdx.x\"))\n    return s, [A, B]\n\n\n@defop(name=\"cuda_rad2deg\", target=\"cuda\", auto_broadcast=False,\n       dtype=[\"float32\", \"float64\"], ndim=list(range(0, 6)))\ndef rad2deg_gpu(dtype, ndim):\n    s, A, B = compute_degandrad(dtype, ndim, 1)\n    s = tvm.te.create_schedule(B.op)\n    axes = [axis for axis in B.op.axis]\n    fused = s[B].fuse(*axes)\n    bx, tx = s[B].split(fused, factor=64)\n    s[B].bind(bx, tvm.te.thread_axis(\"blockIdx.x\"))\n    s[B].bind(tx, tvm.te.thread_axis(\"threadIdx.x\"))\n    return s, [A, B]\n\n\ndef compute_backward_degandrad(dtype, ndim, req, n):\n    ishape = [tvm.te.size_var() for _ in range(ndim)]\n    in_grad_tmp = tvm.te.placeholder(ishape, name='in_grad_tmp', dtype=dtype)\n    in_grad = tvm.te.placeholder(ishape, name='in_grad', dtype=dtype)\n    out_grad = tvm.te.placeholder(ishape, name='out_grad', dtype=dtype)\n    import math\n    if n == 0:\n        ret = tvm.te.compute(ishape, lambda *index: out_grad[index] * tvm.tir.const(math.pi, dtype) / tvm.tir.const(180, dtype))\n    else:\n        ret = tvm.te.compute(ishape, lambda *index: out_grad[index] / tvm.tir.const(math.pi, dtype) * tvm.tir.const(180, dtype))\n    if (req == \"kAddTo\"):\n        in_grad = tvm.te.compute(ishape, lambda *index: in_grad_tmp[index] + ret[index])\n    else:\n        in_grad = tvm.te.compute(ishape, lambda *index: ret[index])\n    s = tvm.te.create_schedule(in_grad.op)\n    return s, out_grad, in_grad_tmp, in_grad, [ret, in_grad]\n\n\n@defop(name=\"backward_deg2rad\", target=\"cpu\", auto_broadcast=False,\n       dtype=[\"float32\", \"float64\"], ndim=list(range(0, 6)), req=[\"kWriteTo\", \"kAddTo\"],\n       attrs=[\"req\"])\ndef backward_deg2rad(dtype, ndim, req):\n    s, out_grad, in_grad_tmp, in_grad, c_list = compute_backward_degandrad(dtype, ndim, req, 0)\n    for t in c_list:\n        axes = [axis for axis in t.op.axis]\n        fused = s[t].fuse(*axes)\n        s[t].parallel(fused)\n    return s, [out_grad, in_grad, in_grad_tmp]\n\n\n@defop(name=\"backward_rad2deg\", target=\"cpu\", auto_broadcast=False,\n       dtype=[\"float32\", \"float64\"], ndim=list(range(0, 6)), req=[\"kWriteTo\", \"kAddTo\"],\n       attrs=[\"req\"])\ndef backward_rad2deg(dtype, ndim, req):\n    s, out_grad, in_grad_tmp, in_grad, c_list = compute_backward_degandrad(dtype, ndim, req, 1)\n    for t in c_list:\n        axes = [axis for axis in t.op.axis]\n        fused = s[t].fuse(*axes)\n        s[t].parallel(fused)\n    return s, [out_grad, in_grad, in_grad_tmp]\n\n\n@defop(name=\"cuda_backward_deg2rad\", target=\"gpu\", auto_broadcast=False,\n       dtype=[\"float32\", \"float64\"], ndim=list(range(0, 6)), req=[\"kWriteTo\", \"kAddTo\"],\n       attrs=[\"req\"])\ndef cuda_backward_deg2rad(dtype, ndim, req):\n    s, out_grad, in_grad_tmp, in_grad, c_list = compute_backward_degandrad(dtype, ndim, req, 0)\n    num_thread = 64\n    for t in c_list:\n        block_x = tvm.te.thread_axis(\"blockIdx.x\")\n        thread_x = tvm.te.thread_axis(\"threadIdx.x\")\n        axes = [axis for axis in t.op.axis]\n        fused = s[t].fuse(*axes)\n        bx, tx = s[t].split(fused, factor=num_thread)\n        s[t].bind(bx, block_x)\n        s[t].bind(tx, thread_x)\n    return s, [out_grad, in_grad, in_grad_tmp]\n\n\n@defop(name=\"cuda_backward_rad2deg\", target=\"gpu\", auto_broadcast=False,\n       dtype=[\"float32\", \"float64\"], ndim=list(range(0, 6)), req=[\"kWriteTo\", \"kAddTo\"],\n       attrs=[\"req\"])\ndef cuda_backward_rad2deg(dtype, ndim, req):\n    s, out_grad, in_grad_tmp, in_grad, c_list = compute_backward_degandrad(dtype, ndim, req, 1)\n    num_thread = 64\n    for t in c_list:\n        block_x = tvm.te.thread_axis(\"blockIdx.x\")\n        thread_x = tvm.te.thread_axis(\"threadIdx.x\")\n        axes = [axis for axis in t.op.axis]\n        fused = s[t].fuse(*axes)\n        bx, tx = s[t].split(fused, factor=num_thread)\n        s[t].bind(bx, block_x)\n        s[t].bind(tx, thread_x)\n    return s, [out_grad, in_grad, in_grad_tmp]\n"
  },
  {
    "path": "contrib/tvmop/compile.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"TVM Operator compile entry point\"\"\"\nimport tvm\nfrom tvm import autotvm\n\nimport os\nimport argparse\nimport re\nimport json\nimport logging\nimport sys\nimport subprocess\nfrom tvmop.opdef import __OP_DEF__\nfrom tvmop.space import ConfigSpaces, ConfigSpace\nfrom tvm.autotvm.measure.measure_methods import set_cuda_target_arch\n\nlogging.basicConfig(level=logging.INFO)\n\n\ndef create_shared(output,\n                  objects,\n                  options=None,\n                  cc=\"g++\"):\n    \"\"\"Create shared library.\n    Parameters\n    ----------\n    output : str\n        The target shared library.\n    objects : List[str]\n        List of object files.\n    options : List[str]\n        The list of additional options string.\n    cc : Optional[str]\n        The compiler command.\n    \"\"\"\n    if sys.platform == \"darwin\" or sys.platform.startswith(\"linux\"):\n        _linux_compile(output, objects, options, cc)\n    # TODO(yzhliu): elif sys.platform == \"win32\":\n    else:\n        raise ValueError(\"Unsupported platform\")\n\n\ndef _linux_compile(output, objects, options, compile_cmd=\"g++\"):\n    cmd = [compile_cmd]\n    if output.endswith(\".so\") or output.endswith(\".dylib\"):\n        cmd += [\"-shared\", \"-fPIC\"]\n        if sys.platform == \"darwin\":\n            cmd += [\"-undefined\", \"dynamic_lookup\"]\n    elif output.endswith(\".obj\"):\n        cmd += [\"-c\"]\n    cmd += [\"-o\", output]\n    if isinstance(objects, str):\n        cmd += [objects]\n    else:\n        cmd += objects\n    if options:\n        cmd += options\n    proc = subprocess.Popen(\n        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)\n    (out, _) = proc.communicate()\n    if proc.returncode != 0:\n        msg = \"Compilation error:\\n\"\n        msg += str(out)\n        raise RuntimeError(msg)\n\n\ndef get_target(device):\n    if device == \"cpu\":\n        return \"llvm\"\n    elif device == \"cuda\" or device == \"gpu\":\n        return \"cuda\"\n    assert False, \"Unknown device \" + device\n\n\ndef get_cuda_arch(arch):\n    if arch is None:\n        return None\n\n    if not isinstance(arch, str):\n        raise TypeError('Expecting parameter arch as a str, while got a {}'.format(str(type(arch))))\n\n    if len(arch) == 0:\n        return None\n\n    # an example of arch string,\n    # -gencode arch=compute_30,code=sm_30 -gencode arch=compute_35,code=sm_35\n    # -gencode;arch=compute_75,code=[sm_75,compute_75] --fatbin-options -compress-all\n    archs = []\n    flags = arch.replace(\"-gencode;\", \"-gencode \").split()\n    for flag in flags:\n        if flag.startswith('-gencode') or flag.startswith('arch='):\n            archs.append(flag)\n\n    return archs\n\n\nif __name__ == \"__main__\":\n    import sys\n    sys.path.append(os.path.dirname(sys.path[0]))\n    parser = argparse.ArgumentParser(description=\"Generate tvm operators\")\n    parser.add_argument(\"-o\", action=\"store\", required=True, dest=\"target_path\",\n                        help=\"Target path which stores compiled library\")\n    parser.add_argument(\"-L\", action=\"store\", default=None, dest=\"ld_path\",\n                        help=\"library link path\")\n    parser.add_argument('--cuda-arch', type=str, default=None, dest='cuda_arch',\n                        help='The cuda arch for compiling kernels for')\n    parser.add_argument(\"--config\", action=\"store\", required=True, dest=\"config_path\",\n                        help=\"Path which stores the config file\")\n    arguments = parser.parse_args()\n\n    mod_llvm = tvm.IRModule({})\n    mod_cuda = tvm.IRModule({})\n    has_cuda = False\n\n    # TODO: attach instruction features to the library, e.g., avx-512, etc.\n    for operator_def in __OP_DEF__:\n        for sch, args, name in operator_def.invoke_all():\n            name = operator_def.get_op_name(name, args)\n            if tvm.runtime.module.enabled(get_target(operator_def.target)):\n                func_lower = tvm.lower(sch, args,\n                                       name=name,\n                                       binds=operator_def.get_binds(args))\n                if operator_def.target == \"cpu\":\n                    mod = mod_llvm.update(func_lower)\n                else:\n                    has_cuda = True\n                    mod_cuda.update(func_lower)\n\n    lowered_funcs = {get_target(\"cpu\"): mod_llvm}\n    if has_cuda > 0:\n        lowered_funcs[get_target(\"cuda\")] = mod_cuda\n        cuda_arch = get_cuda_arch(arguments.cuda_arch)\n        if cuda_arch is None:\n            logging.info('No cuda arch specified. TVM will try to detect it from the build platform.')\n        else:\n            logging.info('Cuda arch {} set for compiling TVM operator kernels.'.format(cuda_arch))\n            set_cuda_target_arch(cuda_arch)\n    func_binary = tvm.build(lowered_funcs, name=\"tvmop\")\n    # we create libtvmop.o first, which gives us chance to link tvm_runtime together with the libtvmop\n    # to allow mxnet find external helper functions in libtvm_runtime\n    func_binary.save(arguments.target_path + \"/libtvmop.o\")\n    if len(func_binary.imported_modules):\n        func_binary.imported_modules[0].save(arguments.target_path + \"/libtvmop.cubin\")\n    ld_path = arguments.target_path if arguments.ld_path is None else arguments.ld_path\n    create_shared(arguments.target_path + \"/libtvmop.so\",\n                  arguments.target_path + \"/libtvmop.o\",\n                  options=[\"-L\", ld_path, \"-ltvm_runtime\"])\n\n    config_spaces = ConfigSpaces()\n    for operator_def in __OP_DEF__:\n        for config_space, name in operator_def.get_config_spaces():\n            config_spaces[name] = ConfigSpace.from_tvm(config_space)\n    with open(arguments.config_path, \"w\") as f:\n        json.dump(config_spaces.to_json_dict(), f)\n"
  },
  {
    "path": "contrib/tvmop/core/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nfrom . import umath, fromnumeric, multiarray\n"
  },
  {
    "path": "contrib/tvmop/core/fromnumeric.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\nimport tvm\nfrom .. import defop\nfrom ..utils import reduce_axes, assign_by_req\n\n\ndef _compute_sum(itype, otype, ndim, reduce1st_dim, req):\n    axes = ([reduce1st_dim, 1 - reduce1st_dim] * ndim)[:ndim]\n    a = tvm.te.placeholder([tvm.te.size_var() for _ in range(ndim)], name='a', dtype=itype)\n    reduce_output = reduce_axes(a, axes, tvm.tir.sum, otype)\n    output_placeholder, final_output = assign_by_req(reduce_output, req)\n    s = tvm.te.create_schedule(final_output.op)\n    return s, a, output_placeholder, final_output, [reduce_output, final_output]\n\n\n@defop(name='sum_cpu', target='cpu', itype=['bool'],\n       otype=['float32', 'float64', 'int32', 'int64'],\n       ndim=[5], req=['kWriteTo', 'kAddTo'], reduce1st_dim=[0, 1],\n       attrs=[\"reduce1st_dim\", \"req\"])\ndef _sum_cpu(itype, otype, ndim, reduce1st_dim, req):\n    s, a, output_placeholder, final_output, tensor_list = _compute_sum(\n        itype, otype, ndim, reduce1st_dim, req)\n    for t in tensor_list:\n        axes = [axis for axis in t.op.axis]\n        fused = s[t].fuse(*axes)\n        s[t].parallel(fused)\n    return s, [a, output_placeholder, final_output]\n\n\n@defop(name='sum_gpu', target='gpu', itype=['bool'],\n       otype=['float32', 'float64', 'int32', 'int64'],\n       ndim=[5], req=['kWriteTo', 'kAddTo'], reduce1st_dim=[0, 1],\n       attrs=[\"reduce1st_dim\", \"req\"])\ndef _sum_gpu(itype, otype, ndim, reduce1st_dim, req):\n    s, a, output_placeholder, final_output, tensor_list = _compute_sum(\n        itype, otype, ndim, reduce1st_dim, req)\n    num_threads = 64\n    for t in tensor_list:\n        block_x = tvm.te.thread_axis(\"blockIdx.x\")\n        thread_x = tvm.te.thread_axis(\"threadIdx.x\")\n        axes = [axis for axis in t.op.axis]\n        fused = s[t].fuse(*axes)\n        bx, tx = s[t].split(fused, factor=num_threads)\n        s[t].bind(bx, block_x)\n        s[t].bind(tx, thread_x)\n    return s, [a, output_placeholder, final_output]\n"
  },
  {
    "path": "contrib/tvmop/core/multiarray.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\nimport tvm\nfrom tvm import autotvm\nfrom .. import defop, AllTypes\nfrom .. import assign_by_req, reduce_axes\n\ndef compute_dot(A, B):\n    M = A.shape[0]\n    K = A.shape[1]\n    N = B.shape[1]\n    k = tvm.te.reduce_axis((0, K), 'k')\n    C = tvm.te.compute((M, N),\n                    lambda x, y: tvm.tir.sum(A[x, k] * B[k, y], axis=k),\n                    name='C')\n    return C\n\n\n@defop(name=\"dot\", target=\"cpu\", dtype=AllTypes)\ndef dot(dtype, fallback):\n    cfg = autotvm.get_config()\n    cfg.define_knob(\"bn\", [64] if fallback else [64, 32])\n    cfg.define_knob(\"factor\", [4] if fallback else [4])\n    M = tvm.te.size_var(\"M\")\n    K = tvm.te.size_var(\"K\")\n    N = tvm.te.size_var(\"N\")\n    A = tvm.te.placeholder((M, K), name='A', dtype=dtype)\n    B = tvm.te.placeholder((K, N), name='B', dtype=dtype)\n    C = compute_dot(A, B)\n    s = tvm.te.create_schedule(C.op)\n    # Blocking by loop tiling\n    xo, yo, xi, yi = s[C].tile(C.op.axis[0], C.op.axis[1], cfg[\"bn\"].val, cfg[\"bn\"].val)\n    k, = s[C].op.reduce_axis\n    ko, ki = s[C].split(k, factor=cfg[\"factor\"].val)\n    # Hoist reduction domain outside the blocking loop\n    s[C].reorder(xo, yo, ko, ki, xi, yi)\n    return s, [A, B, C]\n"
  },
  {
    "path": "contrib/tvmop/core/umath.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport tvm\nfrom .. import defop, AllTypes\n\n_bin_logic_op_map = {\n    'equal': lambda a, b, *idx: a[idx] == b[idx],\n    'not_equal': lambda a, b, *idx: a[idx] != b[idx],\n    'greater': lambda a, b, *idx: a[idx] > b[idx],\n    'less': lambda a, b, *idx: a[idx] < b[idx],\n    'greater_equal': lambda a, b, *idx: a[idx] >= b[idx],\n    'less_equal': lambda a, b, *idx: a[idx] <= b[idx],\n    'logical_and': lambda a, b, *idx: tvm.tir.all(a[idx] != 0, b[idx] != 0),\n    'logical_or': lambda a, b, *idx: tvm.tir.any(a[idx] != 0, b[idx] != 0),\n    'logical_xor': lambda a, b, *idx: tvm.tir.all(tvm.tir.any(a[idx] != 0, b[idx] != 0), tvm.tir.any(a[idx] == 0, b[idx] == 0)),\n}\n\n\ndef _compute_binary_logic(op, dtype, ndim):\n    a = tvm.te.placeholder([tvm.te.size_var() for _ in range(ndim)], dtype=dtype, name='a')\n    b = tvm.te.placeholder([tvm.te.size_var() for _ in range(ndim)], dtype=dtype, name='b')\n    c = tvm.te.compute([tvm.te.size_var() for _ in range(ndim)],\n                    lambda *idx: _bin_logic_op_map[op](a, b, *idx), name='c')\n    s = tvm.te.create_schedule(c.op)\n    return s, a, b, c\n\n\n_bin_logic_cpu_attrs = {\n    'compute_func': _compute_binary_logic,\n    'target': 'cpu',\n    'auto_broadcast': True,\n    'itype': AllTypes + ['bool'],\n    'ndim': list(range(6))\n}\n\n_bin_logic_gpu_attrs = {\n    'compute_func': _compute_binary_logic,\n    'target': 'gpu',\n    'auto_broadcast': True,\n    'itype': AllTypes + ['bool'],\n    'ndim': list(range(6))\n}\n\n\ndef _binary_logic_cpu(compute_func, op, itype, ndim):\n    s, a, b, c = compute_func(op, itype, ndim)\n    axes = [axis for axis in c.op.axis]\n    fused = s[c].fuse(*axes)\n    s[c].parallel(fused)\n    return s, [a, b, c]\n\n\ndef _binary_logic_gpu(compute_func, op, itype, ndim):\n    s, a, b, c = compute_func(op, itype, ndim)\n    axes = [axis for axis in c.op.axis]\n    fused = s[c].fuse(*axes)\n    bx, tx = s[c].split(fused, factor=64)\n    s[c].bind(bx, tvm.te.thread_axis('blockIdx.x'))\n    s[c].bind(tx, tvm.te.thread_axis('threadIdx.x'))\n    return s, [a, b, c]\n\n\n# register binary element-wise logic ops with broadcasting supported\nfor op_name in _bin_logic_op_map.keys():\n    defop(name='{}_cpu'.format(op_name), op=op_name, **_bin_logic_cpu_attrs)(_binary_logic_cpu)\n    defop(name='{}_gpu'.format(op_name), op=op_name, **_bin_logic_gpu_attrs)(_binary_logic_gpu)\n\n\n# Note that `b.dtype` is hard-coded as 'float64'.\n# We should always promote `a`'s elements to `b.dtype`.\n_bin_scalar_logic_op_map = {\n    'equal_scalar': lambda a, b, *idx: a[idx].astype(b.dtype) == b,\n    'not_equal_scalar': lambda a, b, *idx: a[idx].astype(b.dtype) != b,\n    'greater_scalar': lambda a, b, *idx: a[idx].astype(b.dtype) > b,\n    'less_scalar': lambda a, b, *idx: a[idx].astype(b.dtype) < b,\n    'greater_equal_scalar': lambda a, b, *idx: a[idx].astype(b.dtype) >= b,\n    'less_equal_scalar': lambda a, b, *idx: a[idx].astype(b.dtype) <= b,\n    'logical_and_scalar': lambda a, b, *idx: tvm.tir.all(a[idx].astype(b.dtype) != 0 , b != 0),\n    'logical_or_scalar': lambda a, b, *idx: tvm.tir.any(a[idx].astype(b.dtype) != 0, b != 0),\n    'logical_xor_scalar': lambda a, b, *idx: tvm.tir.all(tvm.tir.any(a[idx].astype(b.dtype) != 0, b != 0), tvm.tir.any(a[idx].astype(b.dtype) == 0, b == 0)),\n}\n\n\ndef _compute_binary_scalar_logic(op, dtype, ndim):\n    a = tvm.te.placeholder([tvm.te.size_var() for _ in range(ndim)], name='a', dtype=dtype)\n    b = tvm.te.var('b', dtype='float64')\n    c = tvm.te.compute([tvm.te.size_var() for _ in range(ndim)],\n                    lambda *idx: _bin_scalar_logic_op_map[op](a, b, *idx), name='c')\n    s = tvm.te.create_schedule(c.op)\n    return s, a, b, c\n\n\n_bin_scalar_logic_cpu_attrs = {\n    'compute_func': _compute_binary_scalar_logic,\n    'target': 'cpu',\n    'itype': AllTypes + ['bool'],\n    'ndim': list(range(6))\n}\n\n_bin_scalar_logic_gpu_attrs = {\n    'compute_func': _compute_binary_scalar_logic,\n    'target': 'gpu',\n    'itype': AllTypes + ['bool'],\n    'ndim': list(range(6))\n}\n\n\n# register binary element-wise scalar logic ops\nfor op_name in _bin_scalar_logic_op_map.keys():\n    defop(name='{}_cpu'.format(op_name), op=op_name,\n          **_bin_scalar_logic_cpu_attrs)(_binary_logic_cpu)\n    defop(name='{}_gpu'.format(op_name), op=op_name,\n          **_bin_scalar_logic_gpu_attrs)(_binary_logic_gpu)\n"
  },
  {
    "path": "contrib/tvmop/opdef.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\nimport tvm\nimport inspect\nfrom tvm import autotvm\nfrom itertools import product\n\n__OP_DEF__ = []\n\nclass OpDef:\n    \"\"\"Specify the properties of an operator and\n    construct the value combination of the arguments\n    e.g., ldtype=[\"float32\", \"int32\"], rdtype=[\"float16\", \"int16\"],\n    then the argument combination is\n    [\n        {\"ldtype\": \"float32\", \"rdtype\": \"float16\"},\n        {\"ldtype\": \"float32\", \"rdtype\": \"int16\"},\n        {\"ldtype\": \"int32\", \"rdtype\": \"float16\"},\n        {\"ldtype\": \"int32\", \"rdtype\": \"int16\"},\n    ]\n\n    Parameters\n    ----------\n    func : function\n         The function to define the operator (in tvm compute and schedule).\n         It will get the argument combination extracted by this class.\n    name : str\n         function name.\n    target : str\n         {\"cpu\", \"gpu\", \"cuda\"}\n    auto_broadcast : bool\n         auto_broadcast=True allows one to implement broadcast computation\n         without considering whether dimension size equals to one.\n         TVM maps buffer[i][j][k] -> buffer[i][0][k] if dimension i's shape equals 1.\n    \"\"\"\n    def __init__(self, func, name, target, auto_broadcast, **kwargs):\n        # construct the value combination of the arguments\n        # e.g., ldtype=[\"float32\", \"int32\"], rdtype=[\"float16\", \"int16\"]\n        # arg_combination = [\n        #   {\"ldtype\": \"float32\", \"rdtype\": \"float16\"},\n        #   {\"ldtype\": \"float32\", \"rdtype\": \"int16\"},\n        #   {\"ldtype\": \"int32\", \"rdtype\": \"float16\"},\n        #   {\"ldtype\": \"int32\", \"rdtype\": \"int16\"},\n        # ]\n        self.attrs = kwargs.pop('attrs', [])\n        self.attrs_valid = kwargs.pop('attrs_valid', lambda **kwargs: True)\n        args = [k for k in kwargs]\n        values = [kwargs[k] if isinstance(kwargs[k], (list, tuple)) else [kwargs[k]]\n                  for k in args]\n        cart_product = product(*values)\n        self.arg_combination = [{k: v for k, v in zip(args, comb_values)}\n                                for comb_values in cart_product]\n        self.func = func\n        self.name = name\n        self.target = target\n        self.auto_broadcast = auto_broadcast\n        self.dispatchable = 'fallback' in inspect.signature(self.func).parameters\n\n    def __call__(self, *args, **kwargs):\n        return self.func(*args, **kwargs)\n\n    def invoke_all(self):\n        for each_kwargs in self.arg_combination:\n            if self.attrs_valid(**each_kwargs):\n                name = self.name \\\n                    + ''.join([\"{}_{}\".format(key, each_kwargs[key]) for key in self.attrs])\n                if self.dispatchable is False:\n                    sch, args = self.func(**each_kwargs)\n                    yield sch, args, name\n                else:\n                    # register dispatch schedules\n                    config_space = autotvm.ConfigSpace()\n                    with autotvm.task.ApplyConfig(config_space):\n                            sch, args = self.func(fallback=False, **each_kwargs)\n                    for i in range(len(config_space)):\n                        config_entity = config_space.get(i)\n                        with autotvm.task.ApplyConfig(config_entity):\n                            sch, args = self.func(fallback=False, **each_kwargs)\n                        subname = name + \"index_\" + str(i)\n                        yield sch, args, subname\n                    # register fallback schedule\n                    config_space = autotvm.ConfigSpace()\n                    with autotvm.task.ApplyConfig(config_space):\n                            sch, args = self.func(fallback=True, **each_kwargs)\n                    subname = name + \"fallback\"\n                    yield sch, args, subname\n\n    def get_op_name(self, name, args):\n        return name + ''.join([f\"{arg.dtype}_{len(arg.shape)}\" for arg in args if hasattr(arg, 'shape')])\n\n    def get_config_spaces(self):\n        for each_kwargs in self.arg_combination:\n            if self.attrs_valid(**each_kwargs) and self.dispatchable is True:\n                name = self.name \\\n                    + ''.join([\"{}_{}\".format(key, each_kwargs[key]) for key in self.attrs])\n                config_space = autotvm.ConfigSpace()\n                with autotvm.task.ApplyConfig(config_space):\n                    self.func(fallback=False, **each_kwargs)\n                yield config_space, name\n\n    def get_binds(self, args):\n        if self.auto_broadcast:\n            return {arg: tvm.tir.decl_buffer(arg.shape, arg.dtype, buffer_type=\"auto_broadcast\")\n                    for arg in args}\n        return None\n\n\ndef defop(name, target=None, auto_broadcast=False, **kwargs):\n    \"\"\"Decorator to define a tvm operator.\n    Parameters\n    ----------\n    name : str\n        function name\n    target : str\n        {\"cpu\", \"gpu\", \"cuda\"}\n    auto_broadcast : bool\n        auto_broadcast=True allows one to implement broadcast computation\n        without considering whether dimension size equals to one.\n        TVM maps buffer[i][j][k] -> buffer[i][0][k] if dimension i's shape equals 1.\n    Returns\n    -------\n    fdef : function\n        A wrapped operator definition function, which returns (schedule, [tensors])\n    \"\"\"\n    assert name is not None and len(name) > 0\n    target = \"cpu\" if target is None else target\n\n    def _defop(func):\n        opdef = OpDef(func, name, target, auto_broadcast, **kwargs)\n        __OP_DEF__.append(opdef)\n        return opdef\n    return _defop\n\n"
  },
  {
    "path": "contrib/tvmop/space.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"ConfigSpace API.\"\"\"\nfrom collections import OrderedDict\nimport numpy as _np\n\nclass OtherOptionSpace(object):\n    \"\"\"The parameter space for general option\"\"\"\n    def __init__(self, entities):\n        self.entities = [OtherOptionEntity(e) for e in entities]\n\n    @classmethod\n    def from_tvm(cls, x):\n        return cls([e.val for e in x.entities])\n\n    def __len__(self):\n        return len(self.entities)\n\n    def __repr__(self):\n        return f\"OtherOption({self.entities}) len={len(self)}\"\n\n\nclass OtherOptionEntity(object):\n    \"\"\"The parameter entity for general option, with a detailed value\"\"\"\n    def __init__(self, val):\n        self.val = val\n\n    @classmethod\n    def from_tvm(cls, x):\n        \"\"\"Build a OtherOptionEntity from autotvm.OtherOptionEntity\n\n        Parameters\n        ----------\n        cls: class\n            Calling class\n        x: autotvm.OtherOptionEntity\n            The source object\n\n        Returns\n        -------\n        ret: OtherOptionEntity\n            The corresponding OtherOptionEntity object\n        \"\"\"\n        return cls(x.val)\n\n    def __repr__(self):\n        return str(self.val)\n\n\nclass ConfigSpace(object):\n    \"\"\"The configuration space of a schedule.\"\"\"\n    def __init__(self, space_map, _entity_map):\n        self.space_map = space_map\n        self._entity_map = _entity_map\n        self._length = None\n\n    @classmethod\n    def from_tvm(cls, x):\n        \"\"\"Build a ConfigSpace from autotvm.ConfigSpace\n\n        Parameters\n        ----------\n        cls: class\n            Calling class\n        x: autotvm.ConfigSpace\n            The source object\n\n        Returns\n        -------\n        ret: ConfigSpace\n            The corresponding ConfigSpace object\n        \"\"\"\n        space_map = OrderedDict([(k, OtherOptionSpace.from_tvm(v)) for k, v in x.space_map.items()])\n        _entity_map = OrderedDict([(k, OtherOptionEntity.from_tvm(v)) for k, v in x._entity_map.items()])\n        return cls(space_map, _entity_map)\n\n    def __len__(self):\n\n        if self._length is None:\n            self._length = int(_np.prod([len(x) for x in self.space_map.values()]))\n        return self._length\n\n    def __repr__(self):\n        res = f\"ConfigSpace (len={len(self)}, space_map=\\n\"\n        for i, (name, space) in enumerate(self.space_map.items()):\n            res += f\"  {i:2} {name}: {space}\\n\"\n        return res + \")\"\n\n    def to_json_dict(self):\n        \"\"\"convert to a json serializable dictionary\n\n        Return\n        ------\n        ret: dict\n            a json serializable dictionary\n        \"\"\"\n        ret = {}\n        entity_map = []\n        for k, v in self._entity_map.items():\n            if isinstance(v, OtherOptionEntity):\n                entity_map.append((k, 'ot', v.val))\n            else:\n                raise RuntimeError(\"Invalid entity instance: \" + v)\n        ret['e'] = entity_map\n        space_map = []\n        for k, v in self.space_map.items():\n            entities = [e.val for e in v.entities]\n            space_map.append((k, 'ot', entities))\n        ret['s'] = space_map\n        return ret\n\n    @classmethod\n    def from_json_dict(cls, json_dict):\n        \"\"\"Build a ConfigSpace from json serializable dictionary\n\n        Parameters\n        ----------\n        cls: class\n            The calling class\n        json_dict: dict\n            Json serializable dictionary.\n\n        Returns\n        -------\n        ret: ConfigSpace\n            The corresponding ConfigSpace object\n        \"\"\"\n        entity_map = OrderedDict()\n        for item in json_dict[\"e\"]:\n            key, knob_type, knob_args = item\n            if knob_type == 'ot':\n                entity = OtherOptionEntity(knob_args)\n            else:\n                raise RuntimeError(\"Invalid config knob type: \" + knob_type)\n            entity_map[str(key)] = entity\n        space_map = OrderedDict()\n        for item in json_dict[\"s\"]:\n            key, knob_type, knob_args = item\n            if knob_type == 'ot':\n                space = OtherOptionSpace(knob_args)\n            else:\n                raise RuntimeError(\"Invalid config knob type: \" + knob_type)\n            space_map[str(key)] = space\n        return cls(space_map, entity_map)\n\n\nclass ConfigSpaces(object):\n    \"\"\"The configuration spaces of all ops.\"\"\"\n    def __init__(self):\n        self.spaces = {}\n\n    def __setitem__(self, name, space):\n        self.spaces[name] = space\n\n    def __len__(self):\n        return len(self.spaces)\n\n    def __repr__(self):\n        res = f\"ConfigSpaces (len={len(self)}, config_space=\\n\"\n        for i, (key, val) in enumerate(self.spaces.items()):\n            res += f\"  {i:2} {key}:\\n {val}\\n\"\n        return res + \")\"\n\n    def to_json_dict(self):\n        \"\"\"convert to a json serializable dictionary\n\n        Return\n        ------\n        ret: dict\n            a json serializable dictionary\n        \"\"\"\n        ret = []\n        for k, v in self.spaces.items():\n            ret.append((k, v.to_json_dict()))\n        return ret\n\n    @classmethod\n    def from_json_dict(cls, json_dict):\n        \"\"\"Build a ConfigSpaces from json serializable dictionary\n\n        Parameters\n        ----------\n        cls: class\n            The calling class\n        json_dict: dict\n            Json serializable dictionary.\n\n        Returns\n        -------\n        ret: ConfigSpaces\n            The corresponding ConfigSpaces object\n        \"\"\"\n        ret = cls()\n        for key, val in json_dict:\n            ret.spaces[key] = ConfigSpace.from_json_dict(val)\n        return ret\n"
  },
  {
    "path": "contrib/tvmop/utils.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\nimport tvm\n\nAllTypes = [\"float32\", \"float64\", \"float16\", \"uint8\", \"uint16\",\n            \"uint32\", \"uint64\", \"int8\", \"int16\", \"int32\", \"int64\"]\nRealTypes = [\"float32\", \"float64\", \"float16\"]\n\n\ndef assign_by_req(a, req, otype=None):\n    b = tvm.te.placeholder(a.shape, name='assign_by_req_b', dtype=a.dtype)\n    if req == \"kAddTo\":\n        c = tvm.te.compute(a.shape, lambda *idx: a[idx].astype(otype) + b[idx]\n                                              if otype else a[idx] + b[idx])\n    else:\n        c = tvm.te.compute(a.shape, lambda *idx: a[idx].astype(otype) if otype else a[idx])\n    return b, c\n\n\ndef reduce_axes(X, axes, reducer, atype=None):\n    def get_index(idx, ridx):\n        j = 0\n        k = 0\n        ret = []\n        for val in axes:\n            ret.append(idx[j] if val == 0 else ridx[k])\n            j += (val == 0)\n            k += (val != 0)\n        return tuple(ret)\n    \n    ishape = X.shape\n    odim = (len(ishape) + 1 - axes[0]) // 2\n    oshape = [tvm.te.size_var() for _ in range(odim)]\n    ridx = [tvm.te.reduce_axis((0, ishape[i])) for (i, val) in enumerate(axes) if val == 1]\n    ret = tvm.te.compute(oshape, lambda *idx: reducer(X[get_index(idx, ridx)].astype(atype)\n                                                   if atype else X[get_index(idx, ridx)],\n                                                   axis=ridx), name='ret')\n    return ret\n"
  },
  {
    "path": "cpp-package/CMakeLists.txt",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\ncmake_minimum_required(VERSION 3.13)\nproject(mxnet_cpp C CXX)\n\nadd_library(mxnet_cpp INTERFACE)\n\nset(CPP_PACKAGE_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/include/)\ntarget_include_directories(mxnet_cpp INTERFACE \"${CPP_PACKAGE_INCLUDE_DIR}\")\nfile(GLOB_RECURSE CPP_PACKAGE_HEADERS\n  \"${CPP_PACKAGE_INCLUDE_DIR}/*.h\"\n  \"${CPP_PACKAGE_INCLUDE_DIR}/*.hpp\")\nset(CPP_PACKAGE_OP_H_HEADER ${CMAKE_CURRENT_LIST_DIR}/include/mxnet-cpp/op.h)\ntarget_sources(mxnet_cpp INTERFACE ${CPP_PACKAGE_HEADERS} ${CPP_PACKAGE_OP_H_HEADER})\ntarget_link_libraries(mxnet_cpp INTERFACE mxnet ${mxnet_LINKER_LIBS})\n\nadd_custom_target(\n  cpp_package_op_h ALL\n  BYPRODUCTS ${CPP_PACKAGE_OP_H_HEADER}\n  MAIN_DEPENDENCY mxnet\n  DEPENDS mxnet ${CMAKE_CURRENT_SOURCE_DIR}/scripts/OpWrapperGenerator.py\n  COMMAND echo \"Running: OpWrapperGenerator.py\"\n  COMMAND python3 OpWrapperGenerator.py $<TARGET_FILE:mxnet>\n  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/scripts\n)\nadd_dependencies(mxnet_cpp cpp_package_op_h)\n\nif(MSVC)\n  target_compile_options(mxnet_cpp INTERFACE \"/utf-8\")\nendif(MSVC)\n\nif(BUILD_CPP_EXAMPLES)\n  add_subdirectory(example)\n  add_subdirectory(example/inference)\nendif()\n\ninstall(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})\n"
  },
  {
    "path": "cpp-package/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# MXNet - C++ API\n\nThe MXNet C++ Package provides C++ API bindings to the users of MXNet.  Currently, these bindings are not available as standalone package.\nThe users of these bindings are required to build this package as mentioned below.\n\n## Building C++ Package\n\nThe cpp-package directory contains the implementation of C++ API. Users are required to build this directory or package before using it. \n**The cpp-package is built while building the MXNet shared library, *libmxnet.so*, with *USE\\_CPP\\_PACKAGE* option turned on. Please follow the steps to build the C++ package**\n\n### Steps to build the C++ package:\n1.  Building the MXNet C++ package requires building MXNet from source.\n2.  Clone the MXNet GitHub repository **recursively** to ensure the code in submodules is available for building MXNet.\n\t```\n\tgit clone --recursive https://github.com/apache/mxnet mxnet\n\t```\n\n3.  Install the [recommended dependencies](https://mxnet.apache.org/versions/master/get_started/build_from_source.html#installing-mxnet's-recommended-dependencies) and [optional dependencies](https://mxnet.apache.org/versions/master/get_started/build_from_source.html#overview-of-optional-dependencies-and-optional-features) for building MXNet from source.\n4.  There is a configuration file for cmake, [config/*.cmake](<https://github.com/apache/mxnet/tree/master/config>) that contains all the compilation options. You can edit this file and set the appropriate options prior to running the **cmake** command.\n5.  Please refer to  [cmake configuration files](https://github.com/apache/mxnet/blob/970a2cfbe77d09ee610fdd70afca1a93247cf4fb/config/linux_gpu.cmake#L18-L37) for more details on how to configure and compile MXNet.\n6.  For enabling the build of C++ Package, set the **-DUSE\\_CPP\\_PACKAGE = 1** in cmake options.\n\n### Cross-Compilation steps:\n1.  Build the C++ package for the **host** platform to generate op.h file.\n2.  Remove the following line in [CMakeLists.txt](<https://github.com/apache/mxnet/blob/master/cpp-package/CMakeLists.txt#L15>).\n    ```\n\tCOMMAND python OpWrapperGenerator.py $<TARGET_FILE:mxnet>\n\t``` \n3.  Re-configure cmake for cross-compilation to build the **target** C++ package.\n\n## Usage\n\nIn order to consume the C++ API please follow the steps below.\n\n1. Ensure that the MXNet shared library is built from source with the **USE\\_CPP\\_PACKAGE = 1**.\n2. Include the [MxNetCpp.h](<https://github.com/apache/mxnet/blob/master/cpp-package/include/mxnet-cpp/MxNetCpp.h>) in the program that is going to consume MXNet C++ API.\n\t```c++\n\t#include <mxnet-cpp/MxNetCpp.h>\n\t```\n3. While building the program, ensure that the correct paths to the directories containing header files and MXNet shared library.\n4. The program links the MXNet shared library dynamically. Hence the library needs to be accessible to the program during runtime. This can be achieved by including the path to the shared library in the environment variable  **LD\\_LIBRARY\\_PATH** for Linux, Mac. and Ubuntu OS and **PATH** for Windows OS.\n\n\n## Tutorial\n\nA basic tutorial can be found at <https://mxnet.apache.org/api/cpp/docs/tutorials/basics>.\n\n## Examples\n\nThe example directory contains examples for you to get started. Please build the MXNet C++ Package before building the examples.\n"
  },
  {
    "path": "cpp-package/example/CMakeLists.txt",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Explicitly set GENERATED property https://gitlab.kitware.com/cmake/cmake/issues/18399\nset_property(SOURCE ${CMAKE_CURRENT_LIST_DIR}/../include/mxnet-cpp/op.h PROPERTY GENERATED 1)\n\nadd_executable(lenet lenet.cpp)\ntarget_link_libraries(lenet mxnet_cpp)\n\nadd_executable(lenet_with_mxdataiter lenet_with_mxdataiter.cpp)\ntarget_link_libraries(lenet_with_mxdataiter mxnet_cpp)\n\nadd_executable(alexnet alexnet.cpp)\ntarget_link_libraries(alexnet mxnet_cpp)\n\nadd_executable(charRNN charRNN.cpp)\ntarget_link_libraries(charRNN mxnet_cpp)\n\nadd_executable(googlenet googlenet.cpp)\ntarget_link_libraries(googlenet mxnet_cpp)\n\nadd_executable(inception_bn inception_bn.cpp)\ntarget_link_libraries(inception_bn mxnet_cpp)\n\nadd_executable(mlp mlp.cpp)\ntarget_link_libraries(mlp mxnet_cpp)\n\nadd_executable(mlp_cpu mlp_cpu.cpp)\ntarget_link_libraries(mlp_cpu mxnet_cpp)\n\nadd_executable(mlp_gpu mlp_gpu.cpp)\ntarget_link_libraries(mlp_gpu mxnet_cpp)\n\nadd_executable(resnet resnet.cpp)\ntarget_link_libraries(resnet mxnet_cpp)\n\nadd_executable(test_optimizer test_optimizer.cpp)\ntarget_link_libraries(test_optimizer mxnet_cpp)\n\nadd_executable(test_ndarray_copy test_ndarray_copy.cpp)\ntarget_link_libraries(test_ndarray_copy mxnet_cpp)\n\nadd_executable(test_score test_score.cpp)\ntarget_link_libraries(test_score mxnet_cpp)\n\nadd_executable(mlp_csv mlp_csv.cpp)\ntarget_link_libraries(mlp_csv mxnet_cpp)\n\nadd_executable(test_kvstore test_kvstore.cpp)\ntarget_link_libraries(test_kvstore mxnet_cpp)\n\nadd_executable(test_regress_label test_regress_label.cpp)\ntarget_link_libraries(test_regress_label mxnet_cpp)\n\nadd_executable(sentiment_analysis_rnn ./inference/sentiment_analysis_rnn.cpp)\ntarget_link_libraries(sentiment_analysis_rnn mxnet_cpp)\n\nadd_executable(multi_threaded_inference ./inference/multi_threaded_inference/multi_threaded_inference.cc)\ntarget_link_libraries(multi_threaded_inference mxnet_cpp)\n\nif(MSVC)\n  add_custom_target(cpp_package_deploy_library ALL\n    DEPENDS mxnet\n    COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:mxnet> $<TARGET_FILE_DIR:mlp>)\nendif()\n"
  },
  {
    "path": "cpp-package/example/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# MXNet C++ Package Examples\n\n## Building C++ examples\n\nThe examples in this folder demonstrate the **training** workflow. The **inference workflow** related examples can be found in [inference](<https://github.com/apache/mxnet/blob/master/cpp-package/example/inference>) folder.\nPlease build the MXNet C++ Package as explained in the [README](<https://github.com/apache/mxnet/tree/master/cpp-package#building-c-package>) File.\nThe examples in this folder are built while building the MXNet library and cpp-package from source. You can get the executable files by just copying them from ```mxnet/build/cpp-package/example```\n\nThe examples that are built to be run on GPU may not work on the non-GPU machines.\n\n## Examples demonstrating training workflow\n\nThis directory contains following examples. In order to run the examples, ensure that the path to the MXNet shared library is added to the OS specific environment variable viz. **LD\\_LIBRARY\\_PATH** for Linux, Mac and Ubuntu OS and **PATH** for Windows OS. For example `export LD_LIBRARY_PATH=/usr/local/cuda/lib64:/home/ubuntu/mxnet/build` on ubuntu using gpu.\n\n### [alexnet.cpp](<https://github.com/apache/mxnet/blob/master/cpp-package/example/alexnet.cpp>)\n\nThe example implements the C++ version of AlexNet. The networks trains on MNIST data. The number of epochs can be specified as a command line argument. For example to train with 10 epochs use the following:\n\n```\nbuild/alexnet 10\n```\n\n### [googlenet.cpp](<https://github.com/apache/mxnet/blob/master/cpp-package/example/googlenet.cpp>)\n\nThe code implements a GoogLeNet/Inception network using the C++ API. The example uses MNIST data to train the network. By default, the example trains the model for 100 epochs. The number of epochs can also be specified in the command line. For example, to train the model for 10 epochs use the following:\n\n```\nbuild/googlenet 10\n```\n\n### [mlp.cpp](<https://github.com/apache/mxnet/blob/master/cpp-package/example/mlp.cpp>)\n\nThe code implements a multilayer perceptron from scratch. The example creates its own dummy data to train the model. The example does not require command line parameters. It trains the model for 20,000 epochs.\nTo run the example use the following command:\n\n```\nbuild/mlp\n```\n\n### [mlp_cpu.cpp](<https://github.com/apache/mxnet/blob/master/cpp-package/example/mlp_cpu.cpp>)\n\nThe code implements a multilayer perceptron to train the MNIST data. The code demonstrates the use of \"SimpleBind\"  C++ API and MNISTIter. The example is designed to work on CPU. The example does not require command line parameters.\nTo run the example use the following command:\n\n```\nbuild/mlp_cpu\n```\n\n### [mlp_gpu.cpp](<https://github.com/apache/mxnet/blob/master/cpp-package/example/mlp_gpu.cpp>)\n\nThe code implements a multilayer perceptron to train the MNIST data. The code demonstrates the use of the \"SimpleBind\"  C++ API and MNISTIter. The example is designed to work on GPU. The example does not require command line arguments. To run the example execute following command:\n\n```\nbuild/mlp_gpu\n```\n\n### [mlp_csv.cpp](<https://github.com/apache/mxnet/blob/master/cpp-package/example/mlp_csv.cpp>)\n\nThe code implements a multilayer perceptron to train the MNIST data. The code demonstrates the use of the \"SimpleBind\"  C++ API and CSVIter. The CSVIter can iterate data that is in CSV format. The example can be run on CPU or GPU. The example usage is as follows:\n\n```\nbuild/mlp_csv --train data/mnist_data/mnist_train.csv --test data/mnist_data/mnist_test.csv --epochs 10 --batch_size 100 --hidden_units \"128 64 64\" --gpu\n```\n* To get the `mnist_training_set.csv` and `mnist_test_set.csv` please run the following command:\n```python\n# in mxnet/cpp-package/example directory\npython mnist_to_csv.py ./data/mnist_data/train-images-idx3-ubyte ./data/mnist_data/train-labels-idx1-ubyte ./data/mnist_data/mnist_train.csv 60000\npython mnist_to_csv.py ./data/mnist_data/t10k-images-idx3-ubyte ./data/mnist_data/t10k-labels-idx1-ubyte ./data/mnist_data/mnist_test.csv 10000\n```\n\n### [resnet.cpp](<https://github.com/apache/mxnet/blob/master/cpp-package/example/resnet.cpp>)\n\nThe code implements a resnet model using the C++ API. The model is used to train MNIST data. The number of epochs for training the model can be specified on the command line. By default, model is trained for 100 epochs. For example, to train with 10 epochs use the following command:\n\n```\nbuild/resnet 10\n```\n\n### [lenet.cpp](<https://github.com/apache/mxnet/blob/master/cpp-package/example/lenet.cpp>)\n\nThe code implements a lenet model using the C++ API. It uses MNIST training data in CSV format to train the network. The example does not use built-in CSVIter to read the data from CSV file. The number of epochs can be specified on the command line. By default, the mode is trained for 100,000 epochs. For example, to train with 10 epochs use the following command:\n\n```\nbuild/lenet 10\n```\n### [lenet\\_with\\_mxdataiter.cpp](<https://github.com/apache/mxnet/blob/master/cpp-package/example/mlp_cpu.cpp>)\n\nThe code implements a lenet model using the C++ API. It uses MNIST training data to train the network. The example uses built-in MNISTIter to read the data. The number of epochs can be specified on the command line. By default, the mode is trained for 100 epochs. For example, to train with 10 epochs use the following command:\n\n```\nbuild/lenet_with_mxdataiter 10\n```\n\nIn addition, there is `run_lenet_with_mxdataiter.sh` that downloads the mnist data and run `lenet_with_mxdataiter` example.\n\n### [inception_bn.cpp](<https://github.com/apache/mxnet/blob/master/cpp-package/example/inception_bn.cpp>)\n\nThe code implements an Inception network using the C++ API with batch normalization. The example uses MNIST data to train the network. The model trains for 100 epochs. The example can be run by executing the following command:\n\n```\nbuild/inception_bn\n```\n"
  },
  {
    "path": "cpp-package/example/alexnet.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n */\n#include <iostream>\n#include <map>\n#include <string>\n#include <fstream>\n#include <cstdlib>\n#include \"utils.h\"\n#include \"mxnet-cpp/MxNetCpp.h\"\n\nusing namespace mxnet::cpp;\n\nSymbol AlexnetSymbol(int num_classes) {\n  auto input_data = Symbol::Variable(\"data\");\n  auto target_label = Symbol::Variable(\"label\");\n  /*stage 1*/\n  auto conv1 = Operator(\"Convolution\")\n                   .SetParam(\"kernel\", Shape(11, 11))\n                   .SetParam(\"num_filter\", 96)\n                   .SetParam(\"stride\", Shape(4, 4))\n                   .SetParam(\"dilate\", Shape(1, 1))\n                   .SetParam(\"pad\", Shape(0, 0))\n                   .SetParam(\"num_group\", 1)\n                   .SetParam(\"workspace\", 512)\n                   .SetParam(\"no_bias\", false)\n                   .SetInput(\"data\", input_data)\n                   .CreateSymbol(\"conv1\");\n  auto relu1 = Operator(\"Activation\")\n                   .SetParam(\"act_type\", \"relu\") /*relu,sigmoid,softrelu,tanh */\n                   .SetInput(\"data\", conv1)\n                   .CreateSymbol(\"relu1\");\n  auto pool1 = Operator(\"Pooling\")\n                   .SetParam(\"kernel\", Shape(3, 3))\n                   .SetParam(\"pool_type\", \"max\") /*avg,max,sum */\n                   .SetParam(\"global_pool\", false)\n                   .SetParam(\"stride\", Shape(2, 2))\n                   .SetParam(\"pad\", Shape(0, 0))\n                   .SetInput(\"data\", relu1)\n                   .CreateSymbol(\"pool1\");\n  auto lrn1 = Operator(\"LRN\")\n                  .SetParam(\"nsize\", 5)\n                  .SetParam(\"alpha\", 0.0001)\n                  .SetParam(\"beta\", 0.75)\n                  .SetParam(\"knorm\", 1)\n                  .SetInput(\"data\", pool1)\n                  .CreateSymbol(\"lrn1\");\n  /*stage 2*/\n  auto conv2 = Operator(\"Convolution\")\n                   .SetParam(\"kernel\", Shape(5, 5))\n                   .SetParam(\"num_filter\", 256)\n                   .SetParam(\"stride\", Shape(1, 1))\n                   .SetParam(\"dilate\", Shape(1, 1))\n                   .SetParam(\"pad\", Shape(2, 2))\n                   .SetParam(\"num_group\", 1)\n                   .SetParam(\"workspace\", 512)\n                   .SetParam(\"no_bias\", false)\n                   .SetInput(\"data\", lrn1)\n                   .CreateSymbol(\"conv2\");\n  auto relu2 = Operator(\"Activation\")\n                   .SetParam(\"act_type\", \"relu\") /*relu,sigmoid,softrelu,tanh */\n                   .SetInput(\"data\", conv2)\n                   .CreateSymbol(\"relu2\");\n  auto pool2 = Operator(\"Pooling\")\n                   .SetParam(\"kernel\", Shape(3, 3))\n                   .SetParam(\"pool_type\", \"max\") /*avg,max,sum */\n                   .SetParam(\"global_pool\", false)\n                   .SetParam(\"stride\", Shape(2, 2))\n                   .SetParam(\"pad\", Shape(0, 0))\n                   .SetInput(\"data\", relu2)\n                   .CreateSymbol(\"pool2\");\n  auto lrn2 = Operator(\"LRN\")\n                  .SetParam(\"nsize\", 5)\n                  .SetParam(\"alpha\", 0.0001)\n                  .SetParam(\"beta\", 0.75)\n                  .SetParam(\"knorm\", 1)\n                  .SetInput(\"data\", pool2)\n                  .CreateSymbol(\"lrn2\");\n  /*stage 3*/\n  auto conv3 = Operator(\"Convolution\")\n                   .SetParam(\"kernel\", Shape(3, 3))\n                   .SetParam(\"num_filter\", 384)\n                   .SetParam(\"stride\", Shape(1, 1))\n                   .SetParam(\"dilate\", Shape(1, 1))\n                   .SetParam(\"pad\", Shape(1, 1))\n                   .SetParam(\"num_group\", 1)\n                   .SetParam(\"workspace\", 512)\n                   .SetParam(\"no_bias\", false)\n                   .SetInput(\"data\", lrn2)\n                   .CreateSymbol(\"conv3\");\n  auto relu3 = Operator(\"Activation\")\n                   .SetParam(\"act_type\", \"relu\") /*relu,sigmoid,softrelu,tanh */\n                   .SetInput(\"data\", conv3)\n                   .CreateSymbol(\"relu3\");\n  auto conv4 = Operator(\"Convolution\")\n                   .SetParam(\"kernel\", Shape(3, 3))\n                   .SetParam(\"num_filter\", 384)\n                   .SetParam(\"stride\", Shape(1, 1))\n                   .SetParam(\"dilate\", Shape(1, 1))\n                   .SetParam(\"pad\", Shape(1, 1))\n                   .SetParam(\"num_group\", 1)\n                   .SetParam(\"workspace\", 512)\n                   .SetParam(\"no_bias\", false)\n                   .SetInput(\"data\", relu3)\n                   .CreateSymbol(\"conv4\");\n  auto relu4 = Operator(\"Activation\")\n                   .SetParam(\"act_type\", \"relu\") /*relu,sigmoid,softrelu,tanh */\n                   .SetInput(\"data\", conv4)\n                   .CreateSymbol(\"relu4\");\n  auto conv5 = Operator(\"Convolution\")\n                   .SetParam(\"kernel\", Shape(3, 3))\n                   .SetParam(\"num_filter\", 256)\n                   .SetParam(\"stride\", Shape(1, 1))\n                   .SetParam(\"dilate\", Shape(1, 1))\n                   .SetParam(\"pad\", Shape(1, 1))\n                   .SetParam(\"num_group\", 1)\n                   .SetParam(\"workspace\", 512)\n                   .SetParam(\"no_bias\", false)\n                   .SetInput(\"data\", relu4)\n                   .CreateSymbol(\"conv5\");\n  auto relu5 = Operator(\"Activation\")\n                   .SetParam(\"act_type\", \"relu\")\n                   .SetInput(\"data\", conv5)\n                   .CreateSymbol(\"relu5\");\n  auto pool3 = Operator(\"Pooling\")\n                   .SetParam(\"kernel\", Shape(3, 3))\n                   .SetParam(\"pool_type\", \"max\")\n                   .SetParam(\"global_pool\", false)\n                   .SetParam(\"stride\", Shape(2, 2))\n                   .SetParam(\"pad\", Shape(0, 0))\n                   .SetInput(\"data\", relu5)\n                   .CreateSymbol(\"pool3\");\n  /*stage4*/\n  auto flatten =\n      Operator(\"Flatten\").SetInput(\"data\", pool3).CreateSymbol(\"flatten\");\n  auto fc1 = Operator(\"FullyConnected\")\n                 .SetParam(\"num_hidden\", 4096)\n                 .SetParam(\"no_bias\", false)\n                 .SetInput(\"data\", flatten)\n                 .CreateSymbol(\"fc1\");\n  auto relu6 = Operator(\"Activation\")\n                   .SetParam(\"act_type\", \"relu\")\n                   .SetInput(\"data\", fc1)\n                   .CreateSymbol(\"relu6\");\n  auto dropout1 = Operator(\"Dropout\")\n                      .SetParam(\"p\", 0.5)\n                      .SetInput(\"data\", relu6)\n                      .CreateSymbol(\"dropout1\");\n  /*stage5*/\n  auto fc2 = Operator(\"FullyConnected\")\n                 .SetParam(\"num_hidden\", 4096)\n                 .SetParam(\"no_bias\", false)\n                 .SetInput(\"data\", dropout1)\n                 .CreateSymbol(\"fc2\");\n  auto relu7 = Operator(\"Activation\")\n                   .SetParam(\"act_type\", \"relu\")\n                   .SetInput(\"data\", fc2)\n                   .CreateSymbol(\"relu7\");\n  auto dropout2 = Operator(\"Dropout\")\n                      .SetParam(\"p\", 0.5)\n                      .SetInput(\"data\", relu7)\n                      .CreateSymbol(\"dropout2\");\n  /*stage6*/\n  auto fc3 = Operator(\"FullyConnected\")\n                 .SetParam(\"num_hidden\", num_classes)\n                 .SetParam(\"no_bias\", false)\n                 .SetInput(\"data\", dropout2)\n                 .CreateSymbol(\"fc3\");\n  auto softmax = Operator(\"SoftmaxOutput\")\n                     .SetParam(\"grad_scale\", 1)\n                     .SetParam(\"ignore_label\", -1)\n                     .SetParam(\"multi_output\", false)\n                     .SetParam(\"use_ignore\", false)\n                     .SetParam(\"normalization\", \"null\") /*batch,null,valid */\n                     .SetInput(\"data\", fc3)\n                     .SetInput(\"label\", target_label)\n                     .CreateSymbol(\"softmax\");\n  return softmax;\n}\n\nNDArray ResizeInput(NDArray data, const Shape new_shape) {\n  NDArray pic = data.Reshape(Shape(0, 1, 28, 28));\n  NDArray pic_1channel;\n  Operator(\"_contrib_BilinearResize2D\")\n    .SetParam(\"height\", new_shape[2])\n    .SetParam(\"width\", new_shape[3])\n    (pic).Invoke(pic_1channel);\n  NDArray output;\n  Operator(\"tile\")\n    .SetParam(\"reps\", Shape(1, 3, 1, 1))\n    (pic_1channel).Invoke(output);\n  return output;\n}\n\nint main(int argc, char const *argv[]) {\n  /*basic config*/\n  int max_epo = argc > 1 ? strtol(argv[1], nullptr, 10) : 100;\n  float learning_rate = 1e-4;\n  float weight_decay = 1e-4;\n\n  /*context*/\n  auto ctx = Context::cpu();\n  int num_gpu;\n  MXGetGPUCount(&num_gpu);\n  int batch_size = 32;\n#if MXNET_USE_CUDA\n  if (num_gpu > 0) {\n    ctx = Context::gpu();\n    batch_size = 256;\n  }\n#endif\n\n  TRY\n  /*net symbol*/\n  auto Net = AlexnetSymbol(10);\n\n  /*args_map and aux_map is used for parameters' saving*/\n  std::map<std::string, NDArray> args_map;\n  std::map<std::string, NDArray> aux_map;\n\n  /*we should tell mxnet the shape of data and label*/\n  const Shape data_shape = Shape(batch_size, 3, 256, 256),\n              label_shape = Shape(batch_size);\n  args_map[\"data\"] = NDArray(data_shape, ctx);\n  args_map[\"label\"] = NDArray(label_shape, ctx);\n\n  /*with data and label, executor can be generated automatically*/\n  auto *exec = Net.SimpleBind(ctx, args_map);\n  auto arg_names = Net.ListArguments();\n  aux_map = exec->aux_dict();\n  args_map = exec->arg_dict();\n\n  /*if fine tune from some pre-trained model, we should load the parameters*/\n  // NDArray::Load(\"./model/alex_params_3\", nullptr, &args_map);\n  /*else, we should use initializer Xavier to init the params*/\n  auto initializer = Uniform(0.07);\n  for (auto &arg : args_map) {\n    /*be careful here, the arg's name must has some specific ends or starts for\n     * initializer to call*/\n    initializer(arg.first, &arg.second);\n  }\n\n  /*these binary files should be generated using im2rc tools, which can be found\n   * in mxnet/bin*/\n  std::vector<std::string> data_files = { \"./data/mnist_data/train-images-idx3-ubyte\",\n                                \"./data/mnist_data/train-labels-idx1-ubyte\",\n                                \"./data/mnist_data/t10k-images-idx3-ubyte\",\n                                \"./data/mnist_data/t10k-labels-idx1-ubyte\"\n                              };\n\n  auto train_iter =  MXDataIter(\"MNISTIter\");\n  if (!setDataIter(&train_iter, \"Train\", data_files, batch_size)) {\n    return 1;\n  }\n\n  auto val_iter = MXDataIter(\"MNISTIter\");\n  if (!setDataIter(&val_iter, \"Label\", data_files, batch_size)) {\n    return 1;\n  }\n\n  Optimizer* opt = OptimizerRegistry::Find(\"sgd\");\n  opt->SetParam(\"momentum\", 0.9)\n     ->SetParam(\"rescale_grad\", 1.0 / batch_size)\n     ->SetParam(\"clip_gradient\", 10)\n     ->SetParam(\"lr\", learning_rate)\n     ->SetParam(\"wd\", weight_decay);\n\n  Accuracy acu_train, acu_val;\n  LogLoss logloss_train, logloss_val;\n  for (int epoch = 0; epoch < max_epo; ++epoch) {\n    LG << \"Train Epoch: \" << epoch;\n    /*reset the metric every epoch*/\n    acu_train.Reset();\n    /*reset the data iter every epoch*/\n    train_iter.Reset();\n    int iter = 0;\n    while (train_iter.Next()) {\n      auto batch = train_iter.GetDataBatch();\n      /*use copyto to feed new data and label to the executor*/\n      ResizeInput(batch.data, data_shape).CopyTo(&args_map[\"data\"]);\n      batch.label.CopyTo(&args_map[\"label\"]);\n      exec->Forward(true);\n      exec->Backward();\n      for (size_t i = 0; i < arg_names.size(); ++i) {\n        if (arg_names[i] == \"data\" || arg_names[i] == \"label\") continue;\n        opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]);\n      }\n\n      NDArray::WaitAll();\n      acu_train.Update(batch.label, exec->outputs[0]);\n      logloss_train.Reset();\n      logloss_train.Update(batch.label, exec->outputs[0]);\n      ++iter;\n      LG << \"EPOCH: \" << epoch << \" ITER: \" << iter\n         << \" Train Accuracy: \" << acu_train.Get()\n         << \" Train Loss: \" << logloss_train.Get();\n    }\n    LG << \"EPOCH: \" << epoch << \" Train Accuracy: \" << acu_train.Get();\n\n    LG << \"Val Epoch: \" << epoch;\n    acu_val.Reset();\n    val_iter.Reset();\n    logloss_val.Reset();\n    iter = 0;\n    while (val_iter.Next()) {\n      auto batch = val_iter.GetDataBatch();\n      ResizeInput(batch.data, data_shape).CopyTo(&args_map[\"data\"]);\n      batch.label.CopyTo(&args_map[\"label\"]);\n      exec->Forward(false);\n      NDArray::WaitAll();\n      acu_val.Update(batch.label, exec->outputs[0]);\n      logloss_val.Update(batch.label, exec->outputs[0]);\n      LG << \"EPOCH: \" << epoch << \" ITER: \" << iter << \" Val Accuracy: \" << acu_val.Get();\n      ++iter;\n    }\n    LG << \"EPOCH: \" << epoch << \" Val Accuracy: \" << acu_val.Get();\n    LG << \"EPOCH: \" << epoch << \" Val LogLoss: \" << logloss_val.Get();\n\n    /*save the parameters*/\n    std::stringstream ss;\n    ss << epoch;\n    std::string epoch_str;\n    ss >> epoch_str;\n    std::string save_path_param = \"alex_param_\" + epoch_str;\n    auto save_args = args_map;\n    /*we do not want to save the data and label*/\n    save_args.erase(save_args.find(\"data\"));\n    save_args.erase(save_args.find(\"label\"));\n    /*the alexnet does not get any aux array, so we do not need to save\n     * aux_map*/\n    LG << \"EPOCH: \" << epoch << \" Saving to...\" << save_path_param;\n    NDArray::Save(save_path_param, save_args);\n  }\n  /*don't foget to release the executor*/\n  delete exec;\n  delete opt;\n  MXNotifyShutdown();\n  CATCH\n  return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/charRNN.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * Hua Zhang mz24cn@hotmail.com\n * The code implements C++ version charRNN for mxnet\\example\\rnn\\char-rnn.ipynb with MXNet.cpp API.\n * The generated params file is compatiable with python version.\n * train() and predict() has been verified with original data samples.\n * 2017/1/23:\n * Add faster version charRNN based on built-in cuDNN RNN operator, 10 times faster.\n * Add time major computation graph, although no substantial performance difference.\n * Support continuing training from last params file.\n * Rename params file epoch number starts from zero.\n */\n\n#if _MSC_VER\n#pragma warning(disable: 4996)  // VS2015 complains on 'std::copy' ...\n#endif\n#include <cstring>\n#include <iostream>\n#include <fstream>\n#include <unordered_map>\n#include <vector>\n#include <string>\n#include <tuple>\n#include <algorithm>\n#include <functional>\n#include <thread>\n#include <chrono>\n#include \"mxnet-cpp/MxNetCpp.h\"\n#include \"utils.h\"\n\nusing namespace mxnet::cpp;\n\nstruct LSTMState {\n  Symbol C;\n  Symbol h;\n};\n\nstruct LSTMParam {\n  Symbol i2h_weight;\n  Symbol i2h_bias;\n  Symbol h2h_weight;\n  Symbol h2h_bias;\n};\n\nbool TIME_MAJOR = true;\n\n// LSTM Cell symbol\nLSTMState LSTM(int num_hidden, const Symbol& indata, const LSTMState& prev_state,\n    const LSTMParam& param, int seqidx, int layeridx, mx_float dropout = 0) {\n  auto input = dropout > 0? Dropout(indata, dropout) : indata;\n  auto prefix = std::string(\"t\") + std::to_string(seqidx) + \"_l\" + std::to_string(layeridx);\n  auto i2h = FullyConnected(prefix + \"_i2h\", input, param.i2h_weight, param.i2h_bias,\n      num_hidden * 4);\n  auto h2h = FullyConnected(prefix + \"_h2h\", prev_state.h, param.h2h_weight, param.h2h_bias,\n      num_hidden * 4);\n  auto gates = i2h + h2h;\n  auto slice_gates = SliceChannel(prefix + \"_slice\", gates, 4);\n  auto in_gate = Activation(slice_gates[0], ActivationActType::kSigmoid);\n  auto in_transform = Activation(slice_gates[1], ActivationActType::kTanh);\n  auto forget_gate = Activation(slice_gates[2], ActivationActType::kSigmoid);\n  auto out_gate = Activation(slice_gates[3], ActivationActType::kSigmoid);\n\n  LSTMState state;\n  state.C = (forget_gate * prev_state.C) + (in_gate * in_transform);\n  state.h = out_gate * Activation(state.C, ActivationActType::kTanh);\n  return state;\n}\n\nSymbol LSTMUnroll(int num_lstm_layer, int sequence_length, int input_dim,\n        int num_hidden, int num_embed, mx_float dropout = 0) {\n  auto isTrain = sequence_length > 1;\n  auto data = Symbol::Variable(\"data\");\n  if (TIME_MAJOR && isTrain)\n    data = transpose(data);\n  auto embed_weight = Symbol::Variable(\"embed_weight\");\n  auto embed = Embedding(\"embed\", data, embed_weight, input_dim, num_embed);\n  auto wordvec = isTrain? SliceChannel(embed, sequence_length, TIME_MAJOR? 0 : 1, true) : embed;\n\n  std::vector<LSTMState> last_states;\n  std::vector<LSTMParam> param_cells;\n  for (int l = 0; l < num_lstm_layer; l++) {\n    std::string layer = \"l\" + std::to_string(l);\n    LSTMParam param;\n    param.i2h_weight = Symbol::Variable(layer + \"_i2h_weight\");\n    param.i2h_bias = Symbol::Variable(layer + \"_i2h_bias\");\n    param.h2h_weight = Symbol::Variable(layer + \"_h2h_weight\");\n    param.h2h_bias = Symbol::Variable(layer + \"_h2h_bias\");\n    param_cells.push_back(param);\n    LSTMState state;\n    state.C = Symbol::Variable(layer + \"_init_c\");\n    state.h = Symbol::Variable(layer + \"_init_h\");\n    last_states.push_back(state);\n  }\n\n  std::vector<Symbol> hidden_all;\n  for (int i = 0; i < sequence_length; i++) {\n    auto hidden = wordvec[i];\n    for (int layer = 0; layer < num_lstm_layer; layer++) {\n      double dp_ratio = layer == 0? 0 : dropout;\n      auto next_state = LSTM(num_hidden, hidden, last_states[layer], param_cells[layer],\n          i, layer, dp_ratio);\n      hidden = next_state.h;\n      last_states[layer] = next_state;\n    }\n    if (dropout > 0)\n      hidden = Dropout(hidden, dropout);\n    hidden_all.push_back(hidden);\n  }\n\n  auto hidden_concat =\n      isTrain ? Concat(hidden_all, hidden_all.size(), dmlc::optional<int>(0)) : hidden_all[0];\n  auto cls_weight = Symbol::Variable(\"cls_weight\");\n  auto cls_bias = Symbol::Variable(\"cls_bias\");\n  auto pred = FullyConnected(\"pred\", hidden_concat, cls_weight, cls_bias, input_dim);\n\n  auto label = Symbol::Variable(\"softmax_label\");\n  label = transpose(label);\n  label = Reshape(label, Shape(), false, Shape(0), false);  // -1: infer from graph\n  auto sm = SoftmaxOutput(\"softmax\", pred, label);\n  if (isTrain)\n    return sm;\n\n  std::vector<Symbol> outputs = { sm };\n  for (auto& state : last_states) {\n    outputs.push_back(state.C);\n    outputs.push_back(state.h);\n  }\n  return Symbol::Group(outputs);\n}\n\n// Currently mxnet GPU version RNN operator is implemented via *fast* NVIDIA cuDNN.\nSymbol LSTMWithBuiltInRNNOp(int num_lstm_layer, int sequence_length, int input_dim,\n int num_hidden, int num_embed, mx_float dropout = 0) {\n  auto isTrain = sequence_length > 1;\n  auto data = Symbol::Variable(\"data\");\n  if (TIME_MAJOR && isTrain)\n    data = transpose(data);\n\n  auto embed_weight = Symbol::Variable(\"embed_weight\");\n  auto embed = Embedding(\"embed\", data, embed_weight, input_dim, num_embed);\n  auto label = Symbol::Variable(\"softmax_label\");\n  label = transpose(label);\n  label = Reshape(label, Shape(), false,\n                  Shape(0), false);  // FullyConnected requires one dimension\n  if (!TIME_MAJOR && isTrain)\n    embed = SwapAxis(embed, 0, 1);  // Change to time-major as cuDNN requires\n\n  // We need not do the SwapAxis op as python version does. Direct and better performance in C++!\n  auto rnn_h_init = Symbol::Variable(\"LSTM_init_h\");\n  auto rnn_c_init = Symbol::Variable(\"LSTM_init_c\");\n  auto rnn_params = Symbol::Variable(\"LSTM_parameters\");  // See explanations near RNNXavier class\n  auto variable_sequence_length = Symbol::Variable(\"sequence_length\");\n  auto rnn = RNN(embed, rnn_params, rnn_h_init, rnn_c_init, variable_sequence_length, num_hidden,\n                 num_lstm_layer, RNNMode::kLstm, false, dropout, !isTrain);\n  auto hidden = Reshape(rnn[0], Shape(), false, Shape(0, num_hidden), false);\n\n  auto cls_weight = Symbol::Variable(\"cls_weight\");\n  auto cls_bias = Symbol::Variable(\"cls_bias\");\n  auto pred = FullyConnected(\"pred\", hidden, cls_weight, cls_bias, input_dim);\n  /*In rnn-time-major/rnn_cell_demo.py, the author claimed time-major version speeds up\n   * 1.5~2 times versus batch version. I doubts on the conclusion. In my test, the performance\n   * of both codes are almost same. In fact, there are no substantially differences between\n   * two codes. They are both based on time major cuDNN, the computation graph only differs\n   * slightly on the choices of where to put Reshape/SwapAxis/transpose operation. Here I don't\n   * use Reshape on pred and keep label shape on SoftmaxOutput like time major version code,\n   * but Reshape on label for simplification. It doesn't make influence on performacne. */\n\n  auto sm = SoftmaxOutput(\"softmax\", pred, label);\n  if (isTrain)\n    return sm;\n  else\n    return Symbol::Group({ sm, rnn[1/*RNNOpOutputs::kStateOut=1*/],\n    rnn[2/*RNNOpOutputs::kStateCellOut=2*/] });\n}\n\nclass Shuffler {\n  std::vector<int> sequence;\n public:\n  explicit Shuffler(int size) : sequence(size) {\n    int* p = sequence.data();\n    for (int i = 0; i < size; i++)\n      *p++ = i;\n  }\n  void shuffle(std::function<void(int, int)> lambda = nullptr) {\n    std::random_device rd;\n    std::mt19937 g(rd());\n    std::shuffle(sequence.begin(), sequence.end(), g);\n    int n = 0;\n    if (lambda != nullptr)\n      for (int i : sequence)\n        lambda(n++, i);\n  }\n  const int* data() {\n    return sequence.data();\n  }\n};\n\nclass BucketSentenceIter : public DataIter {\n  Shuffler* random;\n  int batch, current, end;\n  unsigned int sequence_length;\n  Context device;\n  std::vector<std::vector<mx_float>> sequences;\n  std::vector<wchar_t> index2chars;\n  std::unordered_map<wchar_t, mx_float> charIndices;\n\n public:\n  BucketSentenceIter(std::string filename, int minibatch, Context context) : batch(minibatch),\n  current(-1), device(context) {\n    auto content = readContent(filename);\n    buildCharIndex(content);\n    sequences = convertTextToSequences(content, '\\n');\n\n    int N = sequences.size() / batch * batch;  // total used samples\n    sequences.resize(N);\n    sort(sequences.begin(), sequences.end(), [](const std::vector<mx_float>& a,\n        const std::vector<mx_float>& b) { return a.size() < b.size(); });\n\n    sequence_length = sequences.back().size();\n    random = new Shuffler(N);\n    // We still can get random results if call Reset() firstly\n//    std::vector<vector<mx_float>>* target = &sequences;\n//    random->shuffle([target](int n, int i) { (*target)[n].swap((*target)[i]); });\n    end = N / batch;\n  }\n  virtual ~BucketSentenceIter() {\n    delete random;\n  }\n\n  unsigned int maxSequenceLength() {\n    return sequence_length;\n  }\n\n  size_t characterSize() {\n    return charIndices.size();\n  }\n\n  virtual bool Next(void) {\n    return ++current < end;\n  }\n  virtual NDArray GetData(void) {\n    const int* indices = random->data();\n    mx_float *data = new mx_float[sequence_length * batch], *pdata = data;\n\n    for (int i = current * batch, end = i + batch; i < end; i++) {\n      memcpy(pdata, sequences[indices[i]].data(), sequences[indices[i]].size() * sizeof(mx_float));\n      if (sequences[indices[i]].size() < sequence_length)\n        memset(pdata + sequences[indices[i]].size(), 0,\n            (sequence_length - sequences[indices[i]].size()) * sizeof(mx_float));\n      pdata += sequence_length;\n    }\n    NDArray array(Shape(batch, sequence_length), device, false);\n    array.SyncCopyFromCPU(data, batch * sequence_length);\n    return array;\n  }\n  virtual NDArray GetLabel(void) {\n    const int* indices = random->data();\n    mx_float *label = new mx_float[sequence_length * batch], *plabel = label;\n\n    for (int i = current * batch, end = i + batch; i < end; i++) {\n      memcpy(plabel, sequences[indices[i]].data() + 1,\n          (sequences[indices[i]].size() - 1) * sizeof(mx_float));\n      memset(plabel + sequences[indices[i]].size() - 1, 0,\n          (sequence_length - sequences[indices[i]].size() + 1) * sizeof(mx_float));\n      plabel += sequence_length;\n    }\n    NDArray array(Shape(batch, sequence_length), device, false);\n    array.SyncCopyFromCPU(label, batch * sequence_length);\n    return array;\n  }\n  virtual int GetPadNum(void) {\n    return sequence_length - sequences[random->data()[current * batch]].size();\n  }\n  virtual std::vector<int> GetIndex(void) {\n    const int* indices = random->data();\n    std::vector<int> list(indices + current * batch, indices + current * batch + batch);\n    return list;\n  }\n  virtual void BeforeFirst(void) {\n    current = -1;\n    random->shuffle(nullptr);\n  }\n\n  std::wstring readContent(const std::string file) {\n    std::wifstream ifs(file, std::ios::binary);\n    if (ifs) {\n      std::wostringstream os;\n      os << ifs.rdbuf();\n      return os.str();\n    }\n    return L\"\";\n  }\n\n  void buildCharIndex(const std::wstring& content) {\n  // This version buildCharIndex() Compatiable with python version char_rnn dictionary\n    int n = 1;\n    charIndices['\\0'] = 0;  // padding character\n    index2chars.push_back(0);  // padding character index\n    for (auto c : content)\n      if (charIndices.find(c) == charIndices.end()) {\n        charIndices[c] = n++;\n        index2chars.push_back(c);\n      }\n  }\n//  void buildCharIndex(wstring& content) {\n//    for (auto c : content)\n//      charIndices[c]++; // char-frequency map; then char-index map\n//    std::vector<tuple<wchar_t, mx_float>> characters;\n//    for (auto& iter : charIndices)\n//      characters.push_back(make_tuple(iter.first, iter.second));\n//    sort(characters.begin(), characters.end(), [](const tuple<wchar_t, mx_float>& a,\n//      const tuple<wchar_t, mx_float>& b) { return get<1>(a) > get<1>(b); });\n//    mx_float index = 1; //0 is left for zero-padding\n//    index2chars.clear();\n//    index2chars.push_back(0); //zero-padding\n//    for (auto& t : characters) {\n//      charIndices[get<0>(t)] = index++;\n//      index2chars.push_back(get<0>(t));\n//    }s\n//  }\n\n  inline wchar_t character(int i) {\n    return index2chars[i];\n  }\n\n  inline mx_float index(wchar_t c) {\n    return charIndices[c];\n  }\n\n  void saveCharIndices(const std::string file) {\n    std::wofstream ofs(file, std::ios::binary);\n    if (ofs) {\n      ofs.write(index2chars.data() + 1, index2chars.size() - 1);\n      ofs.close();\n    }\n  }\n\n  static std::tuple<std::unordered_map<wchar_t, mx_float>, std::vector<wchar_t>> loadCharIndices(\n      const std::string file) {\n    std::wifstream ifs(file, std::ios::binary);\n    std::unordered_map<wchar_t, mx_float> map;\n    std::vector<wchar_t> chars;\n    if (ifs) {\n      std::wostringstream os;\n      os << ifs.rdbuf();\n      int n = 1;\n      map[L'\\0'] = 0;\n      chars.push_back(L'\\0');\n      for (auto c : os.str()) {\n        map[c] = (mx_float) n++;\n        chars.push_back(c);\n      }\n    }\n    // Note: Can't use {} because this would hit the explicit constructor\n    return std::tuple<std::unordered_map<wchar_t, mx_float>, std::vector<wchar_t>>(map, chars);\n  }\n\n  std::vector<std::vector<mx_float>>\n  convertTextToSequences(const std::wstring& content, wchar_t spliter) {\n    std::vector<std::vector<mx_float>> sequences;\n    sequences.push_back(std::vector<mx_float>());\n    for (auto c : content)\n      if (c == spliter && !sequences.back().empty())\n        sequences.push_back(std::vector<mx_float>());\n      else\n        sequences.back().push_back(charIndices[c]);\n    return sequences;\n  }\n};\n\nvoid OutputPerplexity(NDArray* labels, NDArray* output) {\n  std::vector<mx_float> charIndices, a;\n  labels->SyncCopyToCPU(&charIndices, 0L);  // 0L indicates all\n  output->SyncCopyToCPU(&a, 0L)/*4128*84*/;\n  mx_float loss = 0;\n  int batchSize = labels->GetShape()[0]/*32*/, sequenceLength = labels->GetShape()[1]/*129*/,\n      nSamples = output->GetShape()[0]/*4128*/, vocabSize = output->GetShape()[1]/*84*/;\n  for (int n = 0; n < nSamples; n++) {\n    int row = n % batchSize, column = n / batchSize, labelOffset = column +\n        row * sequenceLength;  // Search based on column storage: labels.T\n    mx_float safe_value = std::max(1e-10f, a[vocabSize * n +\n                                    static_cast<int>(charIndices[labelOffset])]);\n    loss += -log(safe_value);  // Calculate negative log-likelihood\n  }\n  loss = exp(loss / nSamples);\n  std::cout << \"Train-Perplexity=\" << loss << std::endl;\n}\n\nvoid SaveCheckpoint(const std::string filepath, Symbol net, Executor* exe) {\n  std::map<std::string, NDArray> params;\n  for (auto iter : exe->arg_dict())\n    if (iter.first.find(\"_init_\") == std::string::npos\n        && iter.first.rfind(\"data\") != iter.first.length() - 4\n        && iter.first.rfind(\"label\") != iter.first.length() - 5)\n      params.insert({\"arg:\" + iter.first, iter.second});\n  for (auto iter : exe->aux_dict())\n      params.insert({\"aux:\" + iter.first, iter.second});\n  NDArray::Save(filepath, params);\n}\n\nvoid LoadCheckpoint(const std::string filepath, Executor* exe) {\n  std::map<std::string, NDArray> params = NDArray::LoadToMap(filepath);\n  for (auto iter : params) {\n    std::string type = iter.first.substr(0, 4);\n    std::string name = iter.first.substr(4);\n    NDArray target;\n    if (type == \"arg:\")\n      target = exe->arg_dict()[name];\n    else if (type == \"aux:\")\n      target = exe->aux_dict()[name];\n    else\n      continue;\n    iter.second.CopyTo(&target);\n  }\n}\n\nint input_dim = 0;/*84*/\nint sequence_length_max = 0;/*129*/\nint num_embed = 256;\nint num_lstm_layer = 3;\nint num_hidden = 512;\nmx_float dropout = 0.2;\nvoid train(const std::string file, int batch_size, int max_epoch, int start_epoch) {\n  Context device(DeviceType::kGPU, 0);\n  BucketSentenceIter dataIter(file, batch_size, device);\n  std::string prefix = file.substr(0, file.rfind(\".\"));\n  dataIter.saveCharIndices(prefix + \".dictionary\");\n\n  input_dim = static_cast<int>(dataIter.characterSize());\n  sequence_length_max = dataIter.maxSequenceLength();\n\n  auto RNN = LSTMUnroll(num_lstm_layer, sequence_length_max, input_dim, num_hidden,\n      num_embed, dropout);\n  std::map<std::string, NDArray> args_map;\n  args_map[\"data\"] = NDArray(Shape(batch_size, sequence_length_max), device, false);\n  args_map[\"softmax_label\"] = NDArray(Shape(batch_size, sequence_length_max), device, false);\n  for (int i = 0; i < num_lstm_layer; i++) {\n    std::string key = \"l\" + std::to_string(i) + \"_init_\";\n    args_map[key + \"c\"] = NDArray(Shape(batch_size, num_hidden), device, false);\n    args_map[key + \"h\"] = NDArray(Shape(batch_size, num_hidden), device, false);\n  }\n  std::vector<mx_float> zeros(batch_size * num_hidden, 0);\n  // RNN.SimpleBind(device, args_map, {}, {{\"data\", kNullOp}});\n  Executor* exe = RNN.SimpleBind(device, args_map);\n\n  if (start_epoch == -1) {\n    auto initializer = Uniform(0.07);\n    for (auto &arg : exe->arg_dict()) {\n      initializer(arg.first, &arg.second);\n    }\n  } else {\n    LoadCheckpoint(prefix + \"-\" + std::to_string(start_epoch) + \".params\", exe);\n  }\n  start_epoch++;\n\n  mx_float learning_rate = 0.0002;\n  mx_float weight_decay = 0.000002;\n  Optimizer* opt = OptimizerRegistry::Find(\"sgd\");\n  opt->SetParam(\"lr\", learning_rate)\n     ->SetParam(\"wd\", weight_decay);\n//  opt->SetParam(\"momentum\", 0.9)->SetParam(\"rescale_grad\", 1.0 / batch_size)\n//  ->SetParam(\"clip_gradient\", 10);\n\n  for (int epoch = start_epoch; epoch < max_epoch; ++epoch) {\n    dataIter.Reset();\n    auto tic =  std::chrono::system_clock::now();\n    while (dataIter.Next()) {\n      auto data_batch = dataIter.GetDataBatch();\n      data_batch.data.CopyTo(&exe->arg_dict()[\"data\"]);\n      data_batch.label.CopyTo(&exe->arg_dict()[\"softmax_label\"]);\n      for (int l = 0; l < num_lstm_layer; l++) {\n        std::string key = \"l\" + std::to_string(l) + \"_init_\";\n        exe->arg_dict()[key + \"c\"].SyncCopyFromCPU(zeros);\n        exe->arg_dict()[key + \"h\"].SyncCopyFromCPU(zeros);\n      }\n      NDArray::WaitAll();\n\n      exe->Forward(true);\n      exe->Backward();\n      for (size_t i = 0; i < exe->arg_arrays.size(); ++i) {\n        opt->Update(i, exe->arg_arrays[i], exe->grad_arrays[i]);\n      }\n\n      NDArray::WaitAll();\n    }\n    auto toc =  std::chrono::system_clock::now();\n    std::cout << \"Epoch[\" << epoch << \"] Time Cost:\" <<\n         std::chrono::duration_cast< std::chrono::seconds>(toc - tic).count() << \" seconds \";\n    OutputPerplexity(&exe->arg_dict()[\"softmax_label\"], &exe->outputs[0]);\n    std::string filepath = prefix + \"-\" + std::to_string(epoch) + \".params\";\n    SaveCheckpoint(filepath, RNN, exe);\n  }\n\n  delete exe;\n  delete opt;\n}\n\n/*The original example, rnn_cell_demo.py, uses default Xavier as initalizer, which relies on\n * variable name, cannot initialize LSTM_parameters. Thus it was renamed to LSTM_bias,\n * which can be initialized as zero. But it cannot converge after 100 epochs in this corpus\n * example. Using RNNXavier, after 15 oscillating epochs,  it rapidly converges like old\n * LSTMUnroll version. */\nclass RNNXavier : public Xavier {\n public:\n  RNNXavier(RandType rand_type = gaussian, FactorType factor_type = avg,\n    float magnitude = 3) : Xavier(rand_type, factor_type, magnitude) {\n  }\n  virtual ~RNNXavier() {}\n protected:\n  virtual void InitDefault(NDArray* arr) {\n    Xavier::InitWeight(arr);\n  }\n};\n\nvoid trainWithBuiltInRNNOp(const std::string file, int batch_size, int max_epoch, int start_epoch) {\n  Context device(DeviceType::kGPU, 0);\n  BucketSentenceIter dataIter(file, batch_size, device);\n  std::string prefix = file.substr(0, file.rfind(\".\"));\n  dataIter.saveCharIndices(prefix + \".dictionary\");\n\n  input_dim = static_cast<int>(dataIter.characterSize());\n  sequence_length_max = dataIter.maxSequenceLength();\n\n  auto RNN = LSTMWithBuiltInRNNOp(num_lstm_layer, sequence_length_max, input_dim, num_hidden,\n      num_embed, dropout);\n  std::map<std::string, NDArray> args_map;\n  args_map[\"data\"] = NDArray(Shape(batch_size, sequence_length_max), device, false);\n  // Avoiding SwapAxis, batch_size is of second dimension.\n  args_map[\"LSTM_init_c\"] = NDArray(Shape(num_lstm_layer, batch_size, num_hidden), device, false);\n  args_map[\"LSTM_init_h\"] = NDArray(Shape(num_lstm_layer, batch_size, num_hidden), device, false);\n  args_map[\"softmax_label\"] = NDArray(Shape(batch_size, sequence_length_max), device, false);\n  std::vector<mx_float> zeros(batch_size * num_lstm_layer * num_hidden, 0);\n  Executor* exe = RNN.SimpleBind(device, args_map);\n\n  if (start_epoch == -1) {\n    RNNXavier xavier = RNNXavier(Xavier::gaussian, Xavier::in, 2.34);\n    for (auto &arg : exe->arg_dict())\n      xavier(arg.first, &arg.second);\n  } else {\n    LoadCheckpoint(prefix + \"-\" + std::to_string(start_epoch) + \".params\", exe);\n  }\n  start_epoch++;\n\n  Optimizer* opt = OptimizerRegistry::Find(\"ccsgd\");\n//  opt->SetParam(\"momentum\", 0.9)->SetParam(\"rescale_grad\", 1.0 / batch_size)\n//  ->SetParam(\"clip_gradient\", 10);\n\n  for (int epoch = start_epoch; epoch < max_epoch; ++epoch) {\n    dataIter.Reset();\n    auto tic =  std::chrono::system_clock::now();\n    while (dataIter.Next()) {\n      auto data_batch = dataIter.GetDataBatch();\n      data_batch.data.CopyTo(&exe->arg_dict()[\"data\"]);\n      data_batch.label.CopyTo(&exe->arg_dict()[\"softmax_label\"]);\n      exe->arg_dict()[\"LSTM_init_c\"].SyncCopyFromCPU(zeros);\n      exe->arg_dict()[\"LSTM_init_h\"].SyncCopyFromCPU(zeros);\n      NDArray::WaitAll();\n\n      exe->Forward(true);\n      exe->Backward();\n      for (size_t i = 0; i < exe->arg_arrays.size(); ++i) {\n        opt->Update(i, exe->arg_arrays[i], exe->grad_arrays[i]);\n      }\n      NDArray::WaitAll();\n    }\n    auto toc =  std::chrono::system_clock::now();\n    std::cout << \"Epoch[\" << epoch << \"] Time Cost:\" <<\n         std::chrono::duration_cast< std::chrono::seconds>(toc - tic).count() << \" seconds \";\n    OutputPerplexity(&exe->arg_dict()[\"softmax_label\"], &exe->outputs[0]);\n    std::string filepath = prefix + \"-\" + std::to_string(epoch) + \".params\";\n    SaveCheckpoint(filepath, RNN, exe);\n  }\n\n  delete exe;\n  delete opt;\n}\n\nvoid predict(std::wstring* ptext, int sequence_length, const std::string param_file,\n    const std::string dictionary_file) {\n  Context device(DeviceType::kGPU, 0);\n  auto results = BucketSentenceIter::loadCharIndices(dictionary_file);\n  auto dictionary = std::get<0>(results);\n  auto charIndices = std::get<1>(results);\n  input_dim = static_cast<int>(charIndices.size());\n  auto RNN = LSTMUnroll(num_lstm_layer, 1, input_dim, num_hidden, num_embed, 0);\n\n  std::map<std::string, NDArray> args_map;\n  args_map[\"data\"] = NDArray(Shape(1, 1), device, false);\n  args_map[\"softmax_label\"] = NDArray(Shape(1, 1), device, false);\n  std::vector<mx_float> zeros(1 * num_hidden, 0);\n  for (int l = 0; l < num_lstm_layer; l++) {\n    std::string key = \"l\" + std::to_string(l) + \"_init_\";\n    args_map[key + \"c\"] = NDArray(Shape(1, num_hidden), device, false);\n    args_map[key + \"h\"] = NDArray(Shape(1, num_hidden), device, false);\n    args_map[key + \"c\"].SyncCopyFromCPU(zeros);\n    args_map[key + \"h\"].SyncCopyFromCPU(zeros);\n  }\n  Executor* exe = RNN.SimpleBind(device, args_map);\n  LoadCheckpoint(param_file, exe);\n\n  mx_float index;\n  wchar_t next = 0;\n  std::vector<mx_float> softmax;\n  softmax.resize(input_dim);\n  for (auto c : *ptext) {\n    exe->arg_dict()[\"data\"].SyncCopyFromCPU(&dictionary[c], 1);\n    exe->Forward(false);\n\n    exe->outputs[0].SyncCopyToCPU(softmax.data(), input_dim);\n    for (int l = 0; l < num_lstm_layer; l++) {\n      std::string key = \"l\" + std::to_string(l) + \"_init_\";\n      exe->outputs[l * 2 + 1].CopyTo(&args_map[key + \"c\"]);\n      exe->outputs[l * 2 + 2].CopyTo(&args_map[key + \"h\"]);\n    }\n\n    size_t n = max_element(softmax.begin(), softmax.end()) - softmax.begin();\n    index = (mx_float) n;\n    next = charIndices[n];\n  }\n  ptext->push_back(next);\n\n  for (int i = 0; i < sequence_length; i++) {\n    exe->arg_dict()[\"data\"].SyncCopyFromCPU(&index, 1);\n    exe->Forward(false);\n\n    exe->outputs[0].SyncCopyToCPU(softmax.data(), input_dim);\n    for (int l = 0; l < num_lstm_layer; l++) {\n      std::string key = \"l\" + std::to_string(l) + \"_init_\";\n      exe->outputs[l * 2 + 1].CopyTo(&args_map[key + \"c\"]);\n      exe->outputs[l * 2 + 2].CopyTo(&args_map[key + \"h\"]);\n    }\n\n    size_t n = max_element(softmax.begin(), softmax.end()) - softmax.begin();\n    index = (mx_float) n;\n    next = charIndices[n];\n    ptext->push_back(next);\n  }\n\n  delete exe;\n}\n\nvoid predictWithBuiltInRNNOp(std::wstring* ptext, int sequence_length, const std::string param_file,\n  const std::string dictionary_file) {\n  Context device(DeviceType::kGPU, 0);\n  auto results = BucketSentenceIter::loadCharIndices(dictionary_file);\n  auto dictionary = std::get<0>(results);\n  auto charIndices = std::get<1>(results);\n  input_dim = static_cast<int>(charIndices.size());\n  auto RNN = LSTMWithBuiltInRNNOp(num_lstm_layer, 1, input_dim, num_hidden, num_embed, 0);\n\n  std::map<std::string, NDArray> args_map;\n  args_map[\"data\"] = NDArray(Shape(1, 1), device, false);\n  args_map[\"softmax_label\"] = NDArray(Shape(1, 1), device, false);\n  std::vector<mx_float> zeros(1 * num_lstm_layer * num_hidden, 0);\n  // Avoiding SwapAxis, batch_size=1 is of second dimension.\n  args_map[\"LSTM_init_c\"] = NDArray(Shape(num_lstm_layer, 1, num_hidden), device, false);\n  args_map[\"LSTM_init_h\"] = NDArray(Shape(num_lstm_layer, 1, num_hidden), device, false);\n  args_map[\"LSTM_init_c\"].SyncCopyFromCPU(zeros);\n  args_map[\"LSTM_init_h\"].SyncCopyFromCPU(zeros);\n  Executor* exe = RNN.SimpleBind(device, args_map);\n  LoadCheckpoint(param_file, exe);\n\n  mx_float index;\n  wchar_t next = 0;\n  std::vector<mx_float> softmax;\n  softmax.resize(input_dim);\n  for (auto c : *ptext) {\n    exe->arg_dict()[\"data\"].SyncCopyFromCPU(&dictionary[c], 1);\n    exe->Forward(false);\n\n    exe->outputs[0].SyncCopyToCPU(softmax.data(), input_dim);\n    exe->outputs[1].CopyTo(&args_map[\"LSTM_init_h\"]);\n    exe->outputs[2].CopyTo(&args_map[\"LSTM_init_c\"]);\n\n    size_t n = max_element(softmax.begin(), softmax.end()) - softmax.begin();\n    index = (mx_float) n;\n    next = charIndices[n];\n  }\n  ptext->push_back(next);\n\n  for (int i = 0; i < sequence_length; i++) {\n    exe->arg_dict()[\"data\"].SyncCopyFromCPU(&index, 1);\n    exe->Forward(false);\n\n    exe->outputs[0].SyncCopyToCPU(softmax.data(), input_dim);\n    exe->outputs[1].CopyTo(&args_map[\"LSTM_init_h\"]);\n    exe->outputs[2].CopyTo(&args_map[\"LSTM_init_c\"]);\n\n    size_t n = max_element(softmax.begin(), softmax.end()) - softmax.begin();\n    index = (mx_float) n;\n    next = charIndices[n];\n    ptext->push_back(next);\n  }\n\n  delete exe;\n}\n\nint main(int argc, char** argv) {\n  if (argc < 5) {\n    std::cout << \"Usage for training: charRNN train[BuiltIn][TimeMajor] {corpus file}\"\n            \" {batch size} {max epoch} [{starting epoch}]\" << std::endl;\n    std::cout <<\"Usage for prediction: charRNN predict[BuiltIn][TimeMajor] {params file}\"\n            \" {dictionary file} {beginning of text}\" << std::endl;\n    std::cout <<\"Note: The {params file} of train/trainBuiltIn/trainTimeMajor/trainBuiltInTimeMajor\"\n            \" are not compatible with each other.\" << std::endl;\n    return 0;\n  }\n\n  std::string task = argv[1];\n  bool builtIn = task.find(\"BuiltIn\") != std::string::npos;\n  TIME_MAJOR = task.find(\"TimeMajor\") != std::string::npos;\n  std::cout << \"use BuiltIn cuDNN RNN: \" << builtIn << std::endl\n         << \"use data as TimeMajor: \" << TIME_MAJOR << std::endl;\n  TRY\n  if (task.find(\"train\") == 0) {\n    std::cout << \"train batch size:      \" << argv[3] << std::endl\n           << \"train max epoch:       \" << argv[4] << std::endl;\n    int start_epoch = argc > 5? atoi(argv[5]) : -1;\n    // this function will generate dictionary file and params file.\n    if (builtIn)\n      trainWithBuiltInRNNOp(argv[2], atoi(argv[3]), atoi(argv[4]), start_epoch);\n    else\n      train(argv[2], atoi(argv[3]), atoi(argv[4]), start_epoch);  // ditto\n  } else if (task.find(\"predict\") == 0) {\n    std::wstring text;  // = L\"If there is anyone out there who still doubts \";\n    // Considering of extending to Chinese samples in future, use wchar_t instead of char\n    for (char c : std::string(argv[4]))\n      text.push_back((wchar_t) c);\n    /*Python version predicts text default to random selecltions. Here I didn't write the random\n    code, always choose the 'best' character. So the text length reduced to 600. Longer size often\n    leads to repeated sentances, since training sequence length is only 129 for obama corpus.*/\n    if (builtIn)\n      predictWithBuiltInRNNOp(&text, 600, argv[2], argv[3]);\n    else\n      predict(&text, 600, argv[2], argv[3]);\n    std::wcout << text << std::endl;\n  }\n\n  MXNotifyShutdown();\n  CATCH\n  return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/feature_extract/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\nThis example shows how to extract features with a pretrained model.\n\nExecute `run.sh` to:\n- Download a pretrained model\n- Download sample pictures (`dog.jpg` and `cat.jpg`)\n- Compile the files\n- Execute the featurization on `dog.jpg` and `cat.jpg`\n\n\nNote:\n1. The filename of network parameters may vary, line 67 in `feature_extract.cpp` should be updated accordingly.\n2. You need to build MXNet from source to get access to the `lib/libmxnet.so` or point `LD_LIBRARY_PATH` to where it is installed in your system\n"
  },
  {
    "path": "cpp-package/example/feature_extract/feature_extract.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n */\n#include <iostream>\n#include <fstream>\n#include <map>\n#include <string>\n#include <vector>\n#include \"mxnet-cpp/MxNetCpp.h\"\nusing namespace std;\nusing namespace mxnet::cpp;\n\n/*\n * This example shows how to extract features with a pretrained model.\n * Get the model here:\n *   https://github.com/dmlc/mxnet-model-gallery\n * */\n\n/*The global context, change them if necessary*/\nContext global_ctx(kGPU, 0);\n// Context global_ctx(kCPU,0);\n\nclass FeatureExtractor {\n private:\n  /*the mean image, get from the pretrained model*/\n  NDArray mean_img;\n  /*the following two maps store all the paramters need by the model*/\n  map<string, NDArray> args_map;\n  map<string, NDArray> aux_map;\n  Symbol net;\n  Executor *executor;\n  /*Get the feature layer we want to extract*/\n  void GetFeatureSymbol() {\n    /*\n     * use the following to check all the layers' names:\n     * */\n    /*\n    net=Symbol::Load(\"./model/Inception_BN-symbol.json\").GetInternals();\n    for(const auto & layer_name:net.ListOutputs()){\n      LG<<layer_name;\n    }\n    */\n    net = Symbol::Load(\"./model/Inception-BN-symbol.json\")\n              .GetInternals()[\"global_pool_output\"];\n  }\n  /*Fill the trained paramters into the model, a.k.a. net, executor*/\n  void LoadParameters() {\n    map<string, NDArray> paramters;\n    NDArray::Load(\"./model/Inception-BN-0126.params\", 0, &paramters);\n    for (const auto &k : paramters) {\n      if (k.first.substr(0, 4) == \"aux:\") {\n        auto name = k.first.substr(4, k.first.size() - 4);\n        aux_map[name] = k.second.Copy(global_ctx);\n      }\n      if (k.first.substr(0, 4) == \"arg:\") {\n        auto name = k.first.substr(4, k.first.size() - 4);\n        args_map[name] = k.second.Copy(global_ctx);\n      }\n    }\n    /*WaitAll is need when we copy data between GPU and the main memory*/\n    NDArray::WaitAll();\n  }\n  void GetMeanImg() {\n    mean_img = NDArray(Shape(1, 3, 224, 224), global_ctx, false);\n    mean_img.SyncCopyFromCPU(\n        NDArray::LoadToMap(\"./model/mean_224.nd\")[\"mean_img\"].GetData(),\n        1 * 3 * 224 * 224);\n    NDArray::WaitAll();\n  }\n\n public:\n  FeatureExtractor() {\n    /*prepare the model, fill the pretrained parameters, get the mean image*/\n    GetFeatureSymbol();\n    LoadParameters();\n    GetMeanImg();\n  }\n\n  void Extract(NDArray data) {\n    /*Normalize the pictures*/\n    data.Slice(0, 1) -= mean_img;\n    data.Slice(1, 2) -= mean_img;\n    args_map[\"data\"] = data;\n    /*bind the executor*/\n    executor = net.SimpleBind(global_ctx, args_map, map<string, NDArray>(),\n                              map<string, OpReqType>(), aux_map);\n    executor->Forward(false);\n    /*print out the features*/\n    auto array = executor->outputs[0].Copy(Context(kCPU, 0));\n    NDArray::WaitAll();\n    array = array.Reshape({2, 1024});\n    for (int i = 0; i < 1024; ++i) {\n      cout << array.At(0, i) << \",\";\n    }\n    cout << endl;\n  }\n};\n\nNDArray Data2NDArray() {\n  NDArray ret(Shape(2, 3, 224, 224), global_ctx, false);\n  ifstream inf(\"./img.dat\", ios::binary);\n  vector<float> data(2 * 3 * 224 * 224);\n  inf.read(reinterpret_cast<char *>(data.data()), 2 * 3 * 224 * 224 * sizeof(float));\n  inf.close();\n  ret.SyncCopyFromCPU(data.data(), 2 * 3 * 224 * 224);\n  NDArray::WaitAll();\n  return ret;\n}\n\nint main() {\n  /*\n   * get the data from a binary file ./img.data\n   * this file is generated by ./prepare_data_with_opencv\n   * it stores 2 pictures in NDArray format\n   *\n   */\n  auto data = Data2NDArray();\n  FeatureExtractor fe;\n  fe.Extract(data);\n  return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/feature_extract/prepare_data_with_opencv.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n */\n#include <fstream>\n#include <iostream>\n#include <string>\n#include <vector>\n#include <opencv2/opencv.hpp>\n\nusing namespace std;\n\n/*read images and store them the NDArray format that MXNet.cpp can handle*/\nvoid Mat2Array() {\n  string file_name_list[] = {\"./dog.jpg\", \"./cat.jpg\"};\n\n  std::vector<float> array;\n  for (auto &t : file_name_list) {\n    cv::Mat mat = cv::imread(t);\n    /*resize pictures to (224, 224) according to the pretrained model*/\n    cv::resize(mat, mat, cv::Size(224, 224));\n    for (int c = 0; c < 3; ++c) {\n      for (int i = 0; i < 224; ++i) {\n        for (int j = 0; j < 224; ++j) {\n          array.push_back(static_cast<float>(mat.data[(i * 224 + j) * 3 + c]));\n        }\n      }\n    }\n  }\n  ofstream outf(\"./img.dat\", ios::binary);\n  outf.write(reinterpret_cast<char *>(array.data()), array.size() * sizeof(float));\n  outf.close();\n}\n\nint main(int argc, char *argv[]) {\n  Mat2Array();\n  return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/feature_extract/run.sh",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Downloading the data and model\nmkdir -p model\nwget -nc -O model/Inception-BN-symbol.json \\\n    http://data.mxnet.io/mxnet/models/imagenet/inception-bn/Inception-BN-symbol.json\nwget -nc -O model/synset.txt \\\n    http://data.mxnet.io/mxnet/models/imagenet/synset.txt\nwget -nc -O model/Inception-BN-0126.params \\\n    http://data.mxnet.io/mxnet/models/imagenet/inception-bn/Inception-BN-0126.params?raw=true \nwget -nc -O cat.jpg https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/python/predict_image/cat.jpg?raw=true\nwget -nc -O dog.jpg https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/python/predict_image/dog.jpg?raw=true\nwget -nc -O model/mean_224.nd https://github.com/dmlc/web-data/raw/master/mxnet/example/feature_extract/mean_224.nd\ntar -xvzf inception-bn.tar.gz -C model --skip-old-files\n\n# Building\nmake\n\n# Preparing the data\n./prepare_data_with_opencv\n\n# Running the featurization\nLD_LIBRARY_PATH=../../../lib ./feature_extract\n"
  },
  {
    "path": "cpp-package/example/get_data.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset -e\n\nmkdir -p data/mnist_data\ncd data/mnist_data\n\ndownload () {\n    local URL=$1\n    local GZ_FILE_NAME=\"${URL##*/}\"\n\n    local FILE_NAME=\"${GZ_FILE_NAME%.*}\"\n    if [[ -f \"${FILE_NAME}\" ]]; then\n        echo \"File ${FILE_NAME} already downloaded.\"\n        return 0\n    fi\n\n    echo \"Downloading ${URL} ...\"\n    local CURL_OPTIONS=\"--connect-timeout 60 \\\n              --max-time 300 \\\n              --retry-delay 30 \\\n              --retry 5 \\\n              --location \\\n              --silent\"\n    curl ${CURL_OPTIONS} ${URL} -o ${GZ_FILE_NAME}\n\n    if [[ ! -f \"${GZ_FILE_NAME}\" ]]; then\n        echo \"File ${URL} couldn't be downloaded!\"\n        exit 1\n    fi\n\n    gzip -d ${GZ_FILE_NAME}\n    (($? != 0)) && exit 1 || return 0\n}\n\n# MNIST dataset from: http://yann.lecun.com/exdb/mnist/\nFILES=(\n    \"https://web.archive.org/web/20160828233817/http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\"\n    \"https://web.archive.org/web/20160828233817/http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\"\n    \"https://web.archive.org/web/20160828233817/http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\"\n    \"https://web.archive.org/web/20160828233817/http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\"\n    \"http://data.mxnet.io/data/mnist_train.csv.gz\")\n\nfor FILE in ${FILES[@]}; do\n    download ${FILE}\ndone\n"
  },
  {
    "path": "cpp-package/example/googlenet.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n */\n#include <string>\n#include <vector>\n#include <map>\n#include <fstream>\n#include \"utils.h\"\n#include \"mxnet-cpp/MxNetCpp.h\"\n\nusing namespace mxnet::cpp;\n\nSymbol ConvFactory(Symbol data, int num_filter,\n                   Shape kernel,\n                   Shape stride = Shape(1, 1),\n                   Shape pad = Shape(0, 0),\n                   const std::string & name = \"\",\n                   const std::string & suffix = \"\") {\n  Symbol conv_w(\"conv_\" + name + suffix + \"_w\"), conv_b(\"conv_\" + name + suffix + \"_b\");\n\n  Symbol conv = Convolution(\"conv_\" + name + suffix, data,\n                            conv_w, conv_b, kernel,\n                            num_filter, stride, Shape(1, 1), pad);\n  return Activation(\"relu_\" + name + suffix, conv, \"relu\");\n}\n\nSymbol InceptionFactory(Symbol data, int num_1x1, int num_3x3red,\n                        int num_3x3, int num_d5x5red, int num_d5x5,\n                        PoolingPoolType pool, int proj, const std::string & name) {\n  Symbol c1x1 = ConvFactory(data, num_1x1, Shape(1, 1),\n                            Shape(1, 1), Shape(0, 0), name + \"_1x1\");\n\n  Symbol c3x3r = ConvFactory(data, num_3x3red, Shape(1, 1),\n                             Shape(1, 1), Shape(0, 0), name + \"_3x3\", \"_reduce\");\n\n  Symbol c3x3 = ConvFactory(c3x3r, num_3x3, Shape(3, 3),\n                            Shape(1, 1), Shape(1, 1), name + \"_3x3\");\n\n  Symbol cd5x5r = ConvFactory(data, num_d5x5red, Shape(1, 1),\n                              Shape(1, 1), Shape(0, 0), name + \"_5x5\", \"_reduce\");\n\n  Symbol cd5x5 = ConvFactory(cd5x5r, num_d5x5, Shape(5, 5),\n                             Shape(1, 1), Shape(2, 2), name + \"_5x5\");\n\n  Symbol pooling = Pooling(name + \"_pool\", data, Shape(3, 3), pool,\n                           false, false, PoolingPoolingConvention::kValid,\n                           Shape(1, 1), Shape(1, 1));\n\n  Symbol cproj = ConvFactory(pooling, proj, Shape(1, 1),\n                             Shape(1, 1), Shape(0, 0), name + \"_proj\");\n\n  std::vector<Symbol> lst;\n  lst.push_back(c1x1);\n  lst.push_back(c3x3);\n  lst.push_back(cd5x5);\n  lst.push_back(cproj);\n  return Concat(\"ch_concat_\" + name + \"_chconcat\", lst, lst.size());\n}\n\nSymbol GoogleNetSymbol(int num_classes) {\n  // data and label\n  Symbol data = Symbol::Variable(\"data\");\n  Symbol data_label = Symbol::Variable(\"data_label\");\n\n  Symbol conv1 = ConvFactory(data, 64, Shape(7, 7), Shape(2, 2), Shape(3, 3), \"conv1\");\n  Symbol pool1 = Pooling(\"pool1\", conv1, Shape(3, 3), PoolingPoolType::kMax,\n                         false, false, PoolingPoolingConvention::kValid, Shape(2, 2));\n  Symbol conv2 = ConvFactory(pool1, 64, Shape(1, 1), Shape(1, 1),\n                             Shape(0, 0), \"conv2\");\n  Symbol conv3 = ConvFactory(conv2, 192, Shape(3, 3), Shape(1, 1), Shape(1, 1), \"conv3\");\n  Symbol pool3 = Pooling(\"pool3\", conv3, Shape(3, 3), PoolingPoolType::kMax,\n                         false, false, PoolingPoolingConvention::kValid, Shape(2, 2));\n\n  Symbol in3a = InceptionFactory(pool3, 64, 96, 128, 16, 32, PoolingPoolType::kMax, 32, \"in3a\");\n  Symbol in3b = InceptionFactory(in3a, 128, 128, 192, 32, 96, PoolingPoolType::kMax, 64, \"in3b\");\n  Symbol pool4 = Pooling(\"pool4\", in3b, Shape(3, 3), PoolingPoolType::kMax,\n                         false, false, PoolingPoolingConvention::kValid, Shape(2, 2));\n  Symbol in4a = InceptionFactory(pool4, 192, 96, 208, 16, 48, PoolingPoolType::kMax, 64, \"in4a\");\n  Symbol in4b = InceptionFactory(in4a, 160, 112, 224, 24, 64, PoolingPoolType::kMax, 64, \"in4b\");\n  Symbol in4c = InceptionFactory(in4b, 128, 128, 256, 24, 64, PoolingPoolType::kMax, 64, \"in4c\");\n  Symbol in4d = InceptionFactory(in4c, 112, 144, 288, 32, 64, PoolingPoolType::kMax, 64, \"in4d\");\n  Symbol in4e = InceptionFactory(in4d, 256, 160, 320, 32, 128, PoolingPoolType::kMax, 128, \"in4e\");\n  Symbol pool5 = Pooling(\"pool5\", in4e, Shape(3, 3), PoolingPoolType::kMax,\n                         false, false, PoolingPoolingConvention::kValid, Shape(2, 2));\n  Symbol in5a = InceptionFactory(pool5, 256, 160, 320, 32, 128, PoolingPoolType::kMax, 128, \"in5a\");\n  Symbol in5b = InceptionFactory(in5a, 384, 192, 384, 48, 128, PoolingPoolType::kMax, 128, \"in5b\");\n  Symbol pool6 = Pooling(\"pool6\", in5b, Shape(7, 7), PoolingPoolType::kAvg,\n                         false, false, PoolingPoolingConvention::kValid, Shape(1, 1));\n\n  Symbol flatten = Flatten(\"flatten\", pool6);\n\n  Symbol fc1_w(\"fc1_w\"), fc1_b(\"fc1_b\");\n  Symbol fc1 = FullyConnected(\"fc1\", flatten, fc1_w, fc1_b, num_classes);\n\n  return SoftmaxOutput(\"softmax\", fc1, data_label);\n}\n\nint main(int argc, char const *argv[]) {\n  int batch_size = 50;\n  int max_epoch = argc > 1 ? strtol(argv[1], nullptr, 10) : 100;\n  float learning_rate = 1e-4;\n  float weight_decay = 1e-4;\n\n  auto ctx = Context::gpu();\n#if !MXNET_USE_CUDA\n  ctx = Context::cpu();;\n#endif\n\n  TRY\n  auto googlenet = GoogleNetSymbol(10);\n  std::map<std::string, NDArray> args_map;\n  std::map<std::string, NDArray> aux_map;\n\n  args_map[\"data\"] = NDArray(Shape(batch_size, 3, 256, 256), ctx);\n  args_map[\"data_label\"] = NDArray(Shape(batch_size), ctx);\n  googlenet.InferArgsMap(ctx, &args_map, args_map);\n\n  std::vector<std::string> data_files = { \"./data/mnist_data/train-images-idx3-ubyte\",\n                                          \"./data/mnist_data/train-labels-idx1-ubyte\",\n                                          \"./data/mnist_data/t10k-images-idx3-ubyte\",\n                                          \"./data/mnist_data/t10k-labels-idx1-ubyte\"\n                                        };\n\n  auto train_iter =  MXDataIter(\"MNISTIter\");\n  if (!setDataIter(&train_iter, \"Train\", data_files, batch_size)) {\n    return 1;\n  }\n\n  auto val_iter = MXDataIter(\"MNISTIter\");\n  if (!setDataIter(&val_iter, \"Label\", data_files, batch_size)) {\n    return 1;\n  }\n\n  Optimizer* opt = OptimizerRegistry::Find(\"sgd\");\n  opt->SetParam(\"momentum\", 0.9)\n     ->SetParam(\"rescale_grad\", 1.0 / batch_size)\n     ->SetParam(\"clip_gradient\", 10)\n     ->SetParam(\"lr\", learning_rate)\n     ->SetParam(\"wd\", weight_decay);\n\n\n  auto *exec = googlenet.SimpleBind(ctx, args_map);\n  auto arg_names = googlenet.ListArguments();\n\n  for (int iter = 0; iter < max_epoch; ++iter) {\n    LG << \"Epoch: \" << iter;\n    train_iter.Reset();\n    while (train_iter.Next()) {\n      auto data_batch = train_iter.GetDataBatch();\n      data_batch.data.CopyTo(&args_map[\"data\"]);\n      data_batch.label.CopyTo(&args_map[\"data_label\"]);\n      NDArray::WaitAll();\n      exec->Forward(true);\n      exec->Backward();\n      for (size_t i = 0; i < arg_names.size(); ++i) {\n        if (arg_names[i] == \"data\" || arg_names[i] == \"data_label\") continue;\n        opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]);\n      }\n    }\n\n    Accuracy acu;\n    val_iter.Reset();\n    while (val_iter.Next()) {\n      auto data_batch = val_iter.GetDataBatch();\n      data_batch.data.CopyTo(&args_map[\"data\"]);\n      data_batch.label.CopyTo(&args_map[\"data_label\"]);\n      NDArray::WaitAll();\n      exec->Forward(false);\n      NDArray::WaitAll();\n      acu.Update(data_batch.label, exec->outputs[0]);\n    }\n    LG << \"Accuracy: \" << acu.Get();\n  }\n\n  delete exec;\n  delete opt;\n  MXNotifyShutdown();\n  CATCH\n  return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/inception_bn.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n */\n#include <map>\n#include <string>\n#include <fstream>\n#include <vector>\n#include \"utils.h\"\n#include \"mxnet-cpp/MxNetCpp.h\"\n\nusing namespace mxnet::cpp;\n\nSymbol ConvFactoryBN(Symbol data, int num_filter,\n                     Shape kernel, Shape stride, Shape pad,\n                     const std::string & name,\n                     const std::string & suffix = \"\") {\n  Symbol conv_w(\"conv_\" + name + suffix + \"_w\"), conv_b(\"conv_\" + name + suffix + \"_b\");\n\n  Symbol conv = Convolution(\"conv_\" + name + suffix, data,\n                            conv_w, conv_b, kernel,\n                            num_filter, stride, Shape(1, 1), pad);\n  std::string name_suffix = name + suffix;\n  Symbol gamma(name_suffix + \"_gamma\");\n  Symbol beta(name_suffix + \"_beta\");\n  Symbol mmean(name_suffix + \"_mmean\");\n  Symbol mvar(name_suffix + \"_mvar\");\n  Symbol bn = BatchNorm(\"bn_\" + name + suffix, conv, gamma, beta, mmean, mvar);\n  return Activation(\"relu_\" + name + suffix, bn, \"relu\");\n}\n\nSymbol InceptionFactoryA(Symbol data, int num_1x1, int num_3x3red,\n                         int num_3x3, int num_d3x3red, int num_d3x3,\n                         PoolingPoolType pool, int proj,\n                         const std::string & name) {\n  Symbol c1x1 = ConvFactoryBN(data, num_1x1, Shape(1, 1), Shape(1, 1),\n                              Shape(0, 0), name + \"1x1\");\n  Symbol c3x3r = ConvFactoryBN(data, num_3x3red, Shape(1, 1), Shape(1, 1),\n                               Shape(0, 0), name + \"_3x3r\");\n  Symbol c3x3 = ConvFactoryBN(c3x3r, num_3x3, Shape(3, 3), Shape(1, 1),\n                              Shape(1, 1), name + \"_3x3\");\n  Symbol cd3x3r = ConvFactoryBN(data, num_d3x3red, Shape(1, 1), Shape(1, 1),\n                                Shape(0, 0), name + \"_double_3x3\", \"_reduce\");\n  Symbol cd3x3 = ConvFactoryBN(cd3x3r, num_d3x3, Shape(3, 3), Shape(1, 1),\n                               Shape(1, 1), name + \"_double_3x3_0\");\n  cd3x3 = ConvFactoryBN(data = cd3x3, num_d3x3, Shape(3, 3), Shape(1, 1),\n                        Shape(1, 1), name + \"_double_3x3_1\");\n  Symbol pooling = Pooling(name + \"_pool\", data,\n                           Shape(3, 3), pool, false, false,\n                           PoolingPoolingConvention::kValid,\n                           Shape(1, 1), Shape(1, 1));\n  Symbol cproj = ConvFactoryBN(pooling, proj, Shape(1, 1), Shape(1, 1),\n                               Shape(0, 0), name + \"_proj\");\n  std::vector<Symbol> lst;\n  lst.push_back(c1x1);\n  lst.push_back(c3x3);\n  lst.push_back(cd3x3);\n  lst.push_back(cproj);\n  return Concat(\"ch_concat_\" + name + \"_chconcat\", lst, lst.size());\n}\n\nSymbol InceptionFactoryB(Symbol data, int num_3x3red, int num_3x3,\n                         int num_d3x3red, int num_d3x3, const std::string & name) {\n  Symbol c3x3r = ConvFactoryBN(data, num_3x3red, Shape(1, 1),\n                               Shape(1, 1), Shape(0, 0),\n                               name + \"_3x3\", \"_reduce\");\n  Symbol c3x3 = ConvFactoryBN(c3x3r, num_3x3, Shape(3, 3), Shape(2, 2),\n                              Shape(1, 1), name + \"_3x3\");\n  Symbol cd3x3r = ConvFactoryBN(data, num_d3x3red, Shape(1, 1), Shape(1, 1),\n                                Shape(0, 0), name + \"_double_3x3\", \"_reduce\");\n  Symbol cd3x3 = ConvFactoryBN(cd3x3r, num_d3x3, Shape(3, 3), Shape(1, 1),\n                               Shape(1, 1), name + \"_double_3x3_0\");\n  cd3x3 = ConvFactoryBN(cd3x3, num_d3x3, Shape(3, 3), Shape(2, 2),\n                        Shape(1, 1), name + \"_double_3x3_1\");\n  Symbol pooling = Pooling(\"max_pool_\" + name + \"_pool\", data,\n                           Shape(3, 3), PoolingPoolType::kMax,\n                           false, false, PoolingPoolingConvention::kValid,\n                           Shape(2, 2), Shape(1, 1));\n  std::vector<Symbol> lst;\n  lst.push_back(c3x3);\n  lst.push_back(cd3x3);\n  lst.push_back(pooling);\n  return Concat(\"ch_concat_\" + name + \"_chconcat\", lst, lst.size());\n}\n\nSymbol InceptionSymbol(int num_classes) {\n  // data and label\n  Symbol data = Symbol::Variable(\"data\");\n  Symbol data_label = Symbol::Variable(\"data_label\");\n\n  // stage 1\n  Symbol conv1 = ConvFactoryBN(data, 64, Shape(7, 7), Shape(2, 2), Shape(3, 3), \"conv1\");\n  Symbol pool1 = Pooling(\"pool1\", conv1, Shape(3, 3), PoolingPoolType::kMax,\n      false, false, PoolingPoolingConvention::kValid, Shape(2, 2));\n\n  // stage 2\n  Symbol conv2red = ConvFactoryBN(pool1, 64, Shape(1, 1), Shape(1, 1),  Shape(0, 0), \"conv2red\");\n  Symbol conv2 = ConvFactoryBN(conv2red, 192, Shape(3, 3), Shape(1, 1), Shape(1, 1), \"conv2\");\n  Symbol pool2 = Pooling(\"pool2\", conv2, Shape(3, 3), PoolingPoolType::kMax,\n      false, false, PoolingPoolingConvention::kValid, Shape(2, 2));\n\n  // stage 3\n  Symbol in3a = InceptionFactoryA(pool2, 64, 64, 64, 64, 96, PoolingPoolType::kAvg, 32, \"3a\");\n  Symbol in3b = InceptionFactoryA(in3a, 64, 64, 96, 64, 96, PoolingPoolType::kAvg, 64, \"3b\");\n  Symbol in3c = InceptionFactoryB(in3b, 128, 160, 64, 96, \"3c\");\n\n  // stage 4\n  Symbol in4a = InceptionFactoryA(in3c, 224, 64, 96, 96, 128, PoolingPoolType::kAvg, 128, \"4a\");\n  Symbol in4b = InceptionFactoryA(in4a, 192, 96, 128, 96, 128,  PoolingPoolType::kAvg, 128, \"4b\");\n  Symbol in4c = InceptionFactoryA(in4b, 160, 128, 160, 128, 160, PoolingPoolType::kAvg, 128, \"4c\");\n  Symbol in4d = InceptionFactoryA(in4c, 96, 128, 192, 160, 192,  PoolingPoolType::kAvg, 128, \"4d\");\n  Symbol in4e = InceptionFactoryB(in4d, 128, 192, 192, 256, \"4e\");\n\n  // stage 5\n  Symbol in5a = InceptionFactoryA(in4e, 352, 192, 320, 160, 224, PoolingPoolType::kAvg, 128, \"5a\");\n  Symbol in5b = InceptionFactoryA(in5a, 352, 192, 320, 192, 224, PoolingPoolType::kMax, 128, \"5b\");\n\n  // average pooling\n  Symbol avg = Pooling(\"global_pool\", in5b, Shape(7, 7), PoolingPoolType::kAvg);\n\n  // classifier\n  Symbol flatten = Flatten(\"flatten\", avg);\n  Symbol conv1_w(\"conv1_w\"), conv1_b(\"conv1_b\");\n  Symbol fc1 = FullyConnected(\"fc1\", flatten, conv1_w, conv1_b, num_classes);\n  return SoftmaxOutput(\"softmax\", fc1, data_label);\n}\n\nNDArray ResizeInput(NDArray data, const Shape new_shape) {\n  NDArray pic = data.Reshape(Shape(0, 1, 28, 28));\n  NDArray pic_1channel;\n  Operator(\"_contrib_BilinearResize2D\")\n    .SetParam(\"height\", new_shape[2])\n    .SetParam(\"width\", new_shape[3])\n    (pic).Invoke(pic_1channel);\n  NDArray output;\n  Operator(\"tile\")\n    .SetParam(\"reps\", Shape(1, 3, 1, 1))\n    (pic_1channel).Invoke(output);\n  return output;\n}\n\nint main(int argc, char const *argv[]) {\n  int batch_size = 40;\n  int max_epoch = argc > 1 ? strtol(argv[1], nullptr, 10) : 100;\n  float learning_rate = 1e-2;\n  float weight_decay = 1e-4;\n\n  /*context*/\n  auto ctx = Context::cpu();\n  int num_gpu;\n  MXGetGPUCount(&num_gpu);\n#if MXNET_USE_CUDA\n  if (num_gpu > 0) {\n    ctx = Context::gpu();\n  }\n#endif\n\n  TRY\n  auto inception_bn_net = InceptionSymbol(10);\n  std::map<std::string, NDArray> args_map;\n  std::map<std::string, NDArray> aux_map;\n\n  const Shape data_shape = Shape(batch_size, 3, 224, 224),\n              label_shape = Shape(batch_size);\n  args_map[\"data\"] = NDArray(data_shape, ctx);\n  args_map[\"data_label\"] = NDArray(label_shape, ctx);\n  inception_bn_net.InferArgsMap(ctx, &args_map, args_map);\n\n  std::vector<std::string> data_files = { \"./data/mnist_data/train-images-idx3-ubyte\",\n                                          \"./data/mnist_data/train-labels-idx1-ubyte\",\n                                          \"./data/mnist_data/t10k-images-idx3-ubyte\",\n                                          \"./data/mnist_data/t10k-labels-idx1-ubyte\"\n                                        };\n\n  auto train_iter =  MXDataIter(\"MNISTIter\");\n  if (!setDataIter(&train_iter, \"Train\", data_files, batch_size)) {\n    return 1;\n  }\n\n  auto val_iter = MXDataIter(\"MNISTIter\");\n  if (!setDataIter(&val_iter, \"Label\", data_files, batch_size)) {\n    return 1;\n  }\n\n  // initialize parameters\n  auto initializer = Uniform(0.07);\n  for (auto& arg : args_map) {\n    initializer(arg.first, &arg.second);\n  }\n\n  Optimizer* opt = OptimizerRegistry::Find(\"sgd\");\n  opt->SetParam(\"momentum\", 0.9)\n     ->SetParam(\"rescale_grad\", 1.0 / batch_size)\n     ->SetParam(\"clip_gradient\", 10)\n     ->SetParam(\"lr\", learning_rate)\n     ->SetParam(\"wd\", weight_decay);\n\n  auto *exec = inception_bn_net.SimpleBind(ctx, args_map);\n  auto arg_names = inception_bn_net.ListArguments();\n\n  // Create metrics\n  Accuracy train_acc, val_acc;\n  for (int iter = 0; iter < max_epoch; ++iter) {\n    LG << \"Epoch: \" << iter;\n    train_iter.Reset();\n    train_acc.Reset();\n    while (train_iter.Next()) {\n      auto data_batch = train_iter.GetDataBatch();\n      ResizeInput(data_batch.data, data_shape).CopyTo(&args_map[\"data\"]);\n      data_batch.label.CopyTo(&args_map[\"data_label\"]);\n      NDArray::WaitAll();\n\n      exec->Forward(true);\n      exec->Backward();\n      // Update parameters\n      for (size_t i = 0; i < arg_names.size(); ++i) {\n        if (arg_names[i] == \"data\" || arg_names[i] == \"data_label\") continue;\n        opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]);\n      }\n\n      NDArray::WaitAll();\n      train_acc.Update(data_batch.label, exec->outputs[0]);\n    }\n\n    val_iter.Reset();\n    val_acc.Reset();\n    while (val_iter.Next()) {\n      auto data_batch = val_iter.GetDataBatch();\n      ResizeInput(data_batch.data, data_shape).CopyTo(&args_map[\"data\"]);\n      data_batch.label.CopyTo(&args_map[\"data_label\"]);\n      NDArray::WaitAll();\n      exec->Forward(false);\n      NDArray::WaitAll();\n      val_acc.Update(data_batch.label, exec->outputs[0]);\n    }\n    LG << \"Train Accuracy: \" << train_acc.Get();\n    LG << \"Validation Accuracy: \" << val_acc.Get();\n  }\n  delete exec;\n  delete opt;\n  MXNotifyShutdown();\n  CATCH\n  return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/inference/CMakeLists.txt",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Explicitly set GENERATED property https://gitlab.kitware.com/cmake/cmake/issues/18399\nset_property(SOURCE ${CMAKE_CURRENT_LIST_DIR}/../../include/mxnet-cpp/op.h PROPERTY GENERATED 1)\n\nadd_executable(imagenet_inference \"imagenet_inference.cpp\")\ntarget_link_libraries(imagenet_inference mxnet_cpp)\n"
  },
  {
    "path": "cpp-package/example/inference/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# MXNet C++ Package Inference Workflow Examples\n\n## Building C++ Inference examples\n\nThe examples in this folder demonstrate the **inference** workflow. Please build the MXNet C++ Package as explained in the [README](<https://github.com/apache/mxnet/tree/master/cpp-package#building-c-package>) File. You can get the executable files by just copying them from ```mxnet/build/cpp-package/example```\n\n## Examples demonstrating inference workflow\n\nThis directory contains following examples. In order to run the examples, ensure that the path to the MXNet shared library is added to the OS specific environment variable viz. **LD\\_LIBRARY\\_PATH** for Linux, Mac and Ubuntu OS and **PATH** for Windows OS.\n\n## [imagenet_inference.cpp](<https://github.com/apache/mxnet/blob/master/cpp-package/example/inference/imagenet_inference.cpp>)\n\nThis example demonstrates image classification workflow with pre-trained models using MXNet C++ API. Now this script also supports inference with quantized CNN models generated by oneDNN (see this [quantization flow](https://github.com/apache/mxnet/blob/master/example/quantization/README.md)). By using C++ API, the latency of most models will be reduced to some extent compared with current Python implementation.\n\nMost of CNN models have been tested on Linux systems. And 50000 images are used to collect accuracy numbers. Please refer to this [README](https://github.com/apache/mxnet/blob/master/example/quantization/README.md) for  more details about accuracy.\n\nThe following performance numbers are collected via using C++ inference API on AWS EC2 C5.12xlarge. The environment variables are set like below:\n\n```\nexport KMP_AFFINITY=granularity=fine,noduplicates,compact,1,0\nexport OMP_NUM_THREADS=$(vCPUs/2)\nexport MXNET_ENGINE_TYPE=NaiveEngine\n```\nAlso users are recommended to use ```numactl``` or ```taskset``` to bind a running process to the specified cores.\n\n| Model | Dataset |BS=1 (imgs/sec) |BS=64 (imgs/sec) |\n|:---|:---|:---:|:---:|\n| |  |FP32 / INT8 | FP32 / INT8 |\n| ResNet18-V1  | [Validation Dataset](http://data.mxnet.io/data/val_256_q90.rec)  |369.00 / 778.82|799.7 / 2598.04|\n| ResNet50-V1  | [Validation Dataset](http://data.mxnet.io/data/val_256_q90.rec)  |160.72 / 405.84|349.73 / 1297.65 |\n| ResNet101-V1 | [Validation Dataset](http://data.mxnet.io/data/val_256_q90.rec)  | 89.56 / 197.55| 193.25 / 740.47|\n|Squeezenet 1.0|[Validation Dataset](http://data.mxnet.io/data/val_256_q90.rec)\t  | 294.46 /  899.28| 857.70 / 3065.13|\n|MobileNet 1.0|[Validation Dataset](http://data.mxnet.io/data/val_256_q90.rec)    |554.94 / 676.59|1279.44 / 3393.43|\n|MobileNetV2 1.0|[Validation Dataset](http://data.mxnet.io/data/val_256_q90.rec)  |303.40 / 776.40|994.25 / 4227.77|\n|Inception V3|[Validation Dataset](http://data.mxnet.io/data/val_256_q90.rec)     |108.20 /  219.20 | 232.22 / 870.09 |\n|ResNet152-V2|[Validation Dataset](http://data.mxnet.io/data/val_256_q90.rec)     |52.28 / 64.62|107.03 / 134.04 |\n|Inception-BN|[Validation Dataset](http://data.mxnet.io/data/val_256_q90.rec)     | 211.86 / 306.37| 632.79 / 2115.28|\n\nThe command line to launch inference by this script can accept are as shown below:\n```\n./imagenet_inference --help\nUsage:\nimagenet_inference  --symbol_file <model symbol file in json format>\n                    --params_file <model params file>\n\t\t    --dataset <dataset used to benchmark>\n\t\t    --data_nthreads <number of threads for data decoding, default: 60>\n\t\t    --input_shape <shape of input image e.g \"3 224 224\">]\n\t\t    --rgb_mean <mean value to be subtracted on R/G/B channel e.g \"0 0 0\">\n\t\t    --rgb_std <standard deviation on R/G/B channel. e.g \"1 1 1\">\n\t\t    --batch_size <number of images per batch>\n\t\t    --num_skipped_batches <skip the number of batches for inference>\n\t\t    --num_inference_batches <number of batches used for inference>\n\t\t    --data_layer_type <default: \"float32\", choices: [\"float32\", \"int8\", \"uint8\"]>\n\t\t    --gpu <whether to run inference on GPU, default: false>\n\t\t    --enableTRT  <whether to run inference with TensorRT, default: false>\"\n\t\t    --benchmark <whether to use dummy data to run inference, default: false>\n```\n\nFollow the below steps to do inference with more models.\n\n- Download the pre-trained FP32 models into ```./model``` directory.\n- Refer this [README](https://github.com/apache/mxnet/blob/master/example/quantization/README.md) to generate the corresponding quantized models and also put them into ```./model``` directory.\n- Prepare [validation dataset](http://data.mxnet.io/data/val_256_q90.rec) and put it into ```./data``` directory.\n\nThe below command lines show how to run inference with FP32/INT8 resnet50_v1 model. Because the C++ inference script provides the almost same command line as this [Python script](https://github.com/apache/mxnet/blob/master/example/quantization/imagenet_inference.py) and then users can easily go from Python to C++.\n```\n\n# FP32 inference\n./imagenet_inference --symbol_file \"./model/resnet50_v1-symbol.json\" --params_file \"./model/resnet50_v1-0000.params\" --dataset \"./data/val_256_q90.rec\" --rgb_mean \"123.68 116.779 103.939\" --rgb_std \"58.393 57.12 57.375\" --batch_size 64 --num_skipped_batches 50 --num_inference_batches 500\n\n# INT8 inference\n./imagenet_inference --symbol_file \"./model/resnet50_v1-quantized-5batches-naive-symbol.json\" --params_file \"./model/resnet50_v1-quantized-0000.params\" --dataset \"./data/val_256_q90.rec\" --rgb_mean \"123.68 116.779 103.939\" --rgb_std \"58.393 57.12 57.375\" --batch_size 64 --num_skipped_batches 50 --num_inference_batches 500\n\n# FP32 dummy data\n./imagenet_inference --symbol_file \"./model/resnet50_v1-symbol.json\" --batch_size 64 --num_inference_batches 500 --benchmark\n\n# INT8 dummy data\n./imagenet_inference --symbol_file \"./model/resnet50_v1-quantized-5batches-naive-symbol.json\" --batch_size 64 --num_inference_batches 500 --benchmark\n\n```\nFor a quick inference test, users can directly run [unit_test_imagenet_inference.sh](<https://github.com/apache/mxnet/blob/master/cpp-package/example/inference/unit_test_imagenet_inference.sh>) by using the below command. This script will automatically download the pre-trained **Inception-Bn** and **resnet50_v1_int8** model and **validation dataset** which are required for inference.\n\n```\n./unit_test_imagenet_inference.sh\n```\nAnd you may get the similiar outputs like below:\n```\n>>> INFO: FP32 real data\nimagenet_inference.cpp:282: Loading the model from ./model/Inception-BN-symbol.json\nimagenet_inference.cpp:295: Loading the model parameters from ./model/Inception-BN-0126.params\nimagenet_inference.cpp:443: INFO:Dataset for inference: ./data/val_256_q90.rec\nimagenet_inference.cpp:444: INFO:label_name = softmax_label\nimagenet_inference.cpp:445: INFO:rgb_mean: (123.68, 116.779, 103.939)\nimagenet_inference.cpp:447: INFO:rgb_std: (1, 1, 1)\nimagenet_inference.cpp:449: INFO:Image shape: (3, 224, 224)\nimagenet_inference.cpp:451: INFO:Finished inference with: 500 images\nimagenet_inference.cpp:453: INFO:Batch size = 1 for inference\nimagenet_inference.cpp:454: INFO:Accuracy: 0.744\nimagenet_inference.cpp:455: INFO:Throughput: xxxx images per second\n\n>>> INFO: FP32 dummy data\nimagenet_inference.cpp:282: Loading the model from ./model/Inception-BN-symbol.json\nimagenet_inference.cpp:372: Running the forward pass on model to evaluate the performance..\nimagenet_inference.cpp:387: benchmark completed!\nimagenet_inference.cpp:388: batch size: 1 num batch: 500 throughput: xxxx imgs/s latency:xxxx ms\n\n>>> INFO: INT8 dummy data\nimagenet_inference.cpp:282: Loading the model from ./model/resnet50_v1_int8-symbol.json\nimagenet_inference.cpp:372: Running the forward pass on model to evaluate the performance..\nimagenet_inference.cpp:387: benchmark completed!\nimagenet_inference.cpp:388: batch size: 1 num batch: 500 throughput: xxxx imgs/s latency:xxxx ms\n```\nFor running this example with TensorRT, you can quickly try the following example to run a benchmark test for testing Inception BN:\n```\n./imagenet_inference --symbol_file \"./model/Inception-BN-symbol.json\" --params_file \"./model/Inception-BN-0126.params\" --batch_size 16 --num_inference_batches 500 --benchmark --enableTRT\n```\nSample output will looks like this (the example is running on a AWS P3.2xl machine):\n```\nimagenet_inference.cpp:302: Loading the model from ./model/Inception-BN-symbol.json\nbuild_subgraph.cc:686: start to execute partition graph.\nimagenet_inference.cpp:317: Loading the model parameters from ./model/Inception-BN-0126.params\nimagenet_inference.cpp:424: Running the forward pass on model to evaluate the performance..\nimagenet_inference.cpp:439:  benchmark completed!\nimagenet_inference.cpp:440:  batch size: 16 num batch: 500 throughput: 6284.78 imgs/s latency:0.159115 ms\n```\n\n## [sentiment_analysis_rnn.cpp](<https://github.com/apache/mxnet/blob/master/cpp-package/example/inference/sentiment_analysis_rnn.cpp>)\nThis example demonstrates how you can load a pre-trained RNN model and use it to predict the sentiment expressed in the given movie review with the MXNet C++ API. The example is capable of processing variable legnth inputs. It performs the following tasks\n- Loads the pre-trained RNN model.\n- Loads the dictionary file containing the word to index mapping.\n- Splits the review in multiple lines separated by \".\"\n- The example predicts the sentiment score for individual lines and outputs the average score.\n\nThe example is capable of processing variable length input by implementing following technique:\n- The example creates executors for pre-determined input lenghts such as 5, 10, 15, 20, 25, etc called **buckets**.\n- Each bucket is identified by **bucket-key** representing the length on input required by corresponding executor.\n- For each line in the review, the example finds the number of words in the line and tries to find a closest bucket or executor.\n- If the bucket key does not match the number of words in the line, the example pads or trims the input line to match the required length.\n\nThe example uses a pre-trained RNN model trained with a IMDB dataset. The RNN model was built by exercising the [GluonNLP Sentiment Analysis Tutorial](<http://gluon-nlp.mxnet.io/examples/sentiment_analysis/sentiment_analysis.html#>). The tutorial uses 'standard_lstm_lm_200' available in Gluon Model Zoo and fine tunes it for the IMDB dataset\nThe model consists of :\n- Embedding Layer\n- 2 LSTM Layers with hidden dimension size of 200\n- Average pooling layer\n- Sigmoid output layer\nThe model was trained for 10 epochs to achieve 85% test accuracy.\nThe visual representation of the model is [here](<http://gluon-nlp.mxnet.io/examples/sentiment_analysis/sentiment_analysis.html#Sentiment-analysis-model-with-pre-trained-language-model-encoder>).\n\nThe model files can be found here.\n- [sentiment_analysis-symbol.json](< https://s3.amazonaws.com/mxnet-cpp/RNN_model/sentiment_analysis-symbol.json>)\n- [sentiment_analysis-0010.params](< https://s3.amazonaws.com/mxnet-cpp/RNN_model/sentiment_analysis-0010.params>)\n- [sentiment_token_to_idx.txt](<https://s3.amazonaws.com/mxnet-cpp/RNN_model/sentiment_token_to_idx.txt>) Each line of the dictionary file contains a word and a unique index for that word, separated by a space, with a total of 32787 words generated from the training dataset.\nThe example downloads the above files while running.\n\nThe example's command line parameters are as shown below:\n\n```\n./sentiment_analysis_rnn --help\nUsage:\nsentiment_analysis_rnn\n--input Input movie review. The review can be single line or multiline.e.g. \"This movie is the best.\" OR  \"This movie is the best. The direction is awesome.\"\n[--gpu]  Specify this option if workflow needs to be run in gpu context\nIf the review is multiline, the example predicts sentiment score for each line and the final score is the average of scores obtained for each line.\n\n```\n\nThe following command line shows running the example with the movie review containing only one line.\n\n```\n./sentiment_analysis_rnn --input \"This movie has the great story\"\n```\n\nThe above command will output the sentiment score as follows:\n```\nsentiment_analysis_rnn.cpp:346: Input Line : [This movie has the great story] Score : 0.999898\nsentiment_analysis_rnn.cpp:449: The sentiment score between 0 and 1, (1 being positive)=0.999898\n```\n\nThe following command line shows invoking the example with the multi-line review.\n\n```\n./sentiment_analysis_rnn --input \"This movie is the best. The direction is awesome.\"\n```\nThe above command will output the sentiment score for each line in the review and average score as follows:\n```\nInput Line : [This movie is the best] Score : 0.964498\nInput Line : [ The direction is awesome] Score : 0.968855\nThe sentiment score between 0 and 1, (1 being positive)=0.966677\n```\n\nAlternatively, you can run the [unit_test_sentiment_analysis_rnn.sh](<https://github.com/apache/mxnet/blob/master/cpp-package/example/inference/unit_test_sentiment_analysis_rnn.sh>) script.\n"
  },
  {
    "path": "cpp-package/example/inference/imagenet_inference.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*\n * This example demonstrates image classification workflow with pre-trained models using MXNet C++ API.\n * The example performs following tasks.\n * 1. Load the pre-trained model.\n * 2. Load the parameters of pre-trained model.\n * 3. Load the inference dataset and create a new ImageRecordIter.\n * 4. Run the forward pass and obtain throughput & accuracy.\n */\n#ifndef _WIN32\n#include <sys/time.h>\n#endif\n#include <fstream>\n#include <iostream>\n#include <map>\n#include <chrono>\n#include <string>\n#include <vector>\n#include <random>\n#include <type_traits>\n#include <opencv2/opencv.hpp>\n#include \"mxnet/c_api.h\"\n#include \"mxnet/tuple.h\"\n#include \"mxnet-cpp/MxNetCpp.h\"\n#include \"mxnet-cpp/initializer.h\"\n\nusing namespace mxnet::cpp;\n\ndouble ms_now() {\n  double ret;\n#ifdef _WIN32\n  auto timePoint = std::chrono::high_resolution_clock::now().time_since_epoch();\n  ret = std::chrono::duration<double, std::milli>(timePoint).count();\n#else\n  struct timeval time;\n  gettimeofday(&time, nullptr);\n  ret = 1e+3 * time.tv_sec + 1e-3 * time.tv_usec;\n#endif\n  return ret;\n}\n\n\n// define the data type for NDArray, aliged with the definition in mshadow/base.h\nenum TypeFlag {\n  kFloat32 = 0,\n  kFloat64 = 1,\n  kFloat16 = 2,\n  kUint8 = 3,\n  kInt32 = 4,\n  kInt8  = 5,\n  kInt64 = 6,\n};\n\n/*\n * class Predictor\n *\n * This class encapsulates the functionality to load the model, prepare dataset and run the forward pass.\n */\n\nclass Predictor {\n public:\n    Predictor() {}\n    Predictor(const std::string& model_json_file,\n              const std::string& model_params_file,\n              const Shape& input_shape,\n              bool use_gpu,\n              bool enable_tensorrt,\n              const std::string& dataset,\n              const int data_nthreads,\n              const std::string& data_layer_type,\n              const std::vector<float>& rgb_mean,\n              const std::vector<float>& rgb_std,\n              int shuffle_chunk_seed,\n              int seed, bool benchmark);\n    void BenchmarkScore(int num_inference_batches);\n    void Score(int num_skipped_batches, int num_inference_batches);\n    ~Predictor();\n\n private:\n    bool CreateImageRecordIter();\n    bool AdvanceDataIter(int skipped_batches);\n    void LoadModel(const std::string& model_json_file);\n    void LoadParameters(const std::string& model_parameters_file);\n    void SplitParamMap(const std::map<std::string, NDArray> &paramMap,\n        std::map<std::string, NDArray> *argParamInTargetContext,\n        std::map<std::string, NDArray> *auxParamInTargetContext,\n        Context targetContext);\n    void ConvertParamMapToTargetContext(const std::map<std::string, NDArray> &paramMap,\n        std::map<std::string, NDArray> *paramMapInTargetContext,\n        Context targetContext);\n    void InitParameters();\n\n    inline bool FileExists(const std::string &name) {\n      std::ifstream fhandle(name.c_str());\n      return fhandle.good();\n    }\n    int GetDataLayerType();\n\n    std::map<std::string, NDArray> args_map_;\n    std::map<std::string, NDArray> aux_map_;\n    Symbol net_;\n    Executor *executor_;\n    Shape input_shape_;\n    Context global_ctx_ = Context::cpu();\n\n    MXDataIter *val_iter_;\n    bool use_gpu_;\n    bool enable_tensorrt_;\n    std::string dataset_;\n    int data_nthreads_;\n    std::string data_layer_type_;\n    std::vector<float> rgb_mean_;\n    std::vector<float> rgb_std_;\n    int shuffle_chunk_seed_;\n    int seed_;\n    bool benchmark_;\n};\n\n\n/*\n * The constructor takes following parameters as input:\n * 1. model_json_file:  The model in json formatted file.\n * 2. model_params_file: File containing model parameters\n * 3. input_shape: Shape of input data to the model. Since this class will be running one inference at a time,\n *                 the input shape is required to be in format Shape(1, number_of_channels, height, width)\n *                 The input image will be resized to (height x width) size before running the inference.\n * 4. use_gpu: determine if run inference on GPU\n * 5. enable_tensorrt: determine if enable TensorRT\n * 6. dataset: data file (.rec) to be used for inference\n * 7. data_nthreads: number of threads for data loading\n * 8. data_layer_type: data type for data layer\n * 9. rgb_mean: mean value to be subtracted on R/G/B channel\n * 10. rgb_std: standard deviation on R/G/B channel\n * 11. shuffle_chunk_seed: shuffling chunk seed\n * 12. seed: shuffling seed\n * 13. benchmark: use dummy data for inference\n *\n * The constructor will:\n *  1. Create ImageRecordIter based on the given dataset file.\n *  2. Load the model and parameter files.\n *  3. Infer and construct NDArrays according to the input argument and create an executor.\n */\nPredictor::Predictor(const std::string& model_json_file,\n                     const std::string& model_params_file,\n                     const Shape& input_shape,\n                     bool use_gpu,\n                     bool enable_tensorrt,\n                     const std::string& dataset,\n                     const int data_nthreads,\n                     const std::string& data_layer_type,\n                     const std::vector<float>& rgb_mean,\n                     const std::vector<float>& rgb_std,\n                     int shuffle_chunk_seed,\n                     int seed, bool benchmark)\n    : input_shape_(input_shape),\n      use_gpu_(use_gpu),\n      enable_tensorrt_(enable_tensorrt),\n      dataset_(dataset),\n      data_nthreads_(data_nthreads),\n      data_layer_type_(data_layer_type),\n      rgb_mean_(rgb_mean),\n      rgb_std_(rgb_std),\n      shuffle_chunk_seed_(shuffle_chunk_seed),\n      seed_(seed),\n      benchmark_(benchmark) {\n  if (use_gpu) {\n    global_ctx_ = Context::gpu();\n  }\n\n  // initilize data iterator\n  if (!benchmark_ && !CreateImageRecordIter()) {\n    LG << \"Error: failed to create ImageRecordIter\";\n    throw std::runtime_error(\"ImageRecordIter cannot be created\");\n  }\n\n  // Load the model\n  LoadModel(model_json_file);\n  // Initilize the parameters\n  // benchmark=true && model_params_file.empty(), randomly initialize parameters\n  // else, load parameters\n  if (benchmark_ && model_params_file.empty()) {\n    InitParameters();\n  } else {\n    LoadParameters(model_params_file);\n  }\n\n  int dtype = GetDataLayerType();\n  if (dtype == -1) {\n    throw std::runtime_error(\"Unsupported data layer type...\");\n  }\n  args_map_[\"data\"] = NDArray(input_shape_, global_ctx_, false, dtype);\n  Shape label_shape(input_shape_[0]);\n  args_map_[\"softmax_label\"] = NDArray(label_shape, global_ctx_, false);\n  std::vector<NDArray> arg_arrays;\n  std::vector<NDArray> grad_arrays;\n  std::vector<OpReqType> grad_reqs;\n  std::vector<NDArray> aux_arrays;\n\n  // infer and create ndarrays according to the given input ndarrays.\n  net_.InferExecutorArrays(global_ctx_, &arg_arrays, &grad_arrays, &grad_reqs,\n                           &aux_arrays, args_map_, std::map<std::string, NDArray>(),\n                           std::map<std::string, OpReqType>(), aux_map_);\n  for (auto& i : grad_reqs) i = OpReqType::kNullOp;\n\n  // Create an executor after binding the model to input parameters.\n  executor_ = new Executor(net_, global_ctx_, arg_arrays, grad_arrays, grad_reqs, aux_arrays);\n}\n\n/*\n * The following function is used to get the data layer type for input data\n */\nint Predictor::GetDataLayerType() {\n  int ret_type = -1;\n  if (data_layer_type_ == \"float32\") {\n    ret_type = kFloat32;\n  } else if (data_layer_type_ == \"int8\") {\n    ret_type = kInt8;\n  } else if (data_layer_type_ == \"uint8\") {\n    ret_type = kUint8;\n  } else {\n    LG << \"Unsupported data layer type \" << data_layer_type_ << \"...\"\n       << \"Please use one of {float32, int8, uint8}\";\n  }\n  return ret_type;\n}\n\n/*\n * create a new ImageRecordIter according to the given parameters\n */\nbool Predictor::CreateImageRecordIter() {\n  val_iter_ = new MXDataIter(\"ImageRecordIter\");\n  if (!FileExists(dataset_)) {\n    LG << \"Error: \" << dataset_ << \" must be provided\";\n    return false;\n  }\n\n  std::vector<index_t> shape_vec;\n  for (index_t i = 1; i < input_shape_.ndim(); i++)\n    shape_vec.push_back(input_shape_[i]);\n  mxnet::TShape data_shape(shape_vec.begin(), shape_vec.end());\n\n  // set image record parser parameters\n  val_iter_->SetParam(\"path_imgrec\", dataset_);\n  val_iter_->SetParam(\"label_width\", 1);\n  val_iter_->SetParam(\"data_shape\", data_shape);\n  val_iter_->SetParam(\"preprocess_threads\", data_nthreads_);\n  val_iter_->SetParam(\"shuffle_chunk_seed\", shuffle_chunk_seed_);\n\n  // set Batch parameters\n  val_iter_->SetParam(\"batch_size\", input_shape_[0]);\n\n  // image record parameters\n  val_iter_->SetParam(\"shuffle\", true);\n  val_iter_->SetParam(\"seed\", seed_);\n\n  // set normalize parameters\n  val_iter_->SetParam(\"mean_r\", rgb_mean_[0]);\n  val_iter_->SetParam(\"mean_g\", rgb_mean_[1]);\n  val_iter_->SetParam(\"mean_b\", rgb_mean_[2]);\n  val_iter_->SetParam(\"std_r\", rgb_std_[0]);\n  val_iter_->SetParam(\"std_g\", rgb_std_[1]);\n  val_iter_->SetParam(\"std_b\", rgb_std_[2]);\n\n  // set prefetcher parameters\n  if (use_gpu_) {\n    val_iter_->SetParam(\"ctx\", \"gpu\");\n  } else {\n    val_iter_->SetParam(\"ctx\", \"cpu\");\n  }\n  val_iter_->SetParam(\"dtype\", data_layer_type_);\n\n  val_iter_->CreateDataIter();\n  return true;\n}\n\n/*\n * The following function loads the model from json file.\n */\nvoid Predictor::LoadModel(const std::string& model_json_file) {\n  if (!FileExists(model_json_file)) {\n    LG << \"Model file \" << model_json_file << \" does not exist\";\n    throw std::runtime_error(\"Model file does not exist\");\n  }\n  LG << \"Loading the model from \" << model_json_file << std::endl;\n  net_ = Symbol::Load(model_json_file);\n  if (enable_tensorrt_) {\n    net_ = net_.GetBackendSymbol(\"TensorRT\");\n  }\n}\n\n/*\n * The following function loads the model parameters.\n */\nvoid Predictor::LoadParameters(const std::string& model_parameters_file) {\n  if (!FileExists(model_parameters_file)) {\n    LG << \"Parameter file \" << model_parameters_file << \" does not exist\";\n    throw std::runtime_error(\"Model parameters does not exist\");\n  }\n  LG << \"Loading the model parameters from \" << model_parameters_file << std::endl;\n  std::map<std::string, NDArray> parameters;\n  NDArray::Load(model_parameters_file, 0, &parameters);\n  if (enable_tensorrt_) {\n    std::map<std::string, NDArray> intermediate_args_map;\n    std::map<std::string, NDArray> intermediate_aux_map;\n    SplitParamMap(parameters, &intermediate_args_map, &intermediate_aux_map, Context::cpu());\n    contrib::InitTensorRTParams(net_, &intermediate_args_map, &intermediate_aux_map);\n    ConvertParamMapToTargetContext(intermediate_args_map, &args_map_, global_ctx_);\n    ConvertParamMapToTargetContext(intermediate_aux_map, &aux_map_, global_ctx_);\n  } else {\n    SplitParamMap(parameters, &args_map_, &aux_map_, global_ctx_);\n  }\n  /*WaitAll is need when we copy data between GPU and the main memory*/\n  NDArray::WaitAll();\n}\n\n/*\n * The following function split loaded param map into arg parm\n *   and aux param with target context\n */\nvoid Predictor::SplitParamMap(const std::map<std::string, NDArray> &paramMap,\n    std::map<std::string, NDArray> *argParamInTargetContext,\n    std::map<std::string, NDArray> *auxParamInTargetContext,\n    Context targetContext) {\n  for (const auto& pair : paramMap) {\n    std::string type = pair.first.substr(0, 4);\n    std::string name = pair.first.substr(4);\n    if (type == \"arg:\") {\n      (*argParamInTargetContext)[name] = pair.second.Copy(targetContext);\n    } else if (type == \"aux:\") {\n      (*auxParamInTargetContext)[name] = pair.second.Copy(targetContext);\n    }\n  }\n}\n\n/*\n * The following function copy the param map into the target context\n */\nvoid Predictor::ConvertParamMapToTargetContext(const std::map<std::string, NDArray> &paramMap,\n    std::map<std::string, NDArray> *paramMapInTargetContext,\n    Context targetContext) {\n  for (const auto& pair : paramMap) {\n    (*paramMapInTargetContext)[pair.first] = pair.second.Copy(targetContext);\n  }\n}\n\n/*\n * The following function randomly initializes the parameters when benchmark_ is true.\n */\nvoid Predictor::InitParameters() {\n  std::vector<mx_uint> data_shape;\n  for (index_t i = 0; i < input_shape_.ndim(); i++) {\n    data_shape.push_back(input_shape_[i]);\n  }\n\n  std::map<std::string, std::vector<mx_uint> > arg_shapes;\n  std::vector<std::vector<mx_uint> > aux_shapes, in_shapes, out_shapes;\n  arg_shapes[\"data\"] = data_shape;\n  net_.InferShape(arg_shapes, &in_shapes, &aux_shapes, &out_shapes);\n\n  // initializer to call\n  Xavier xavier(Xavier::uniform, Xavier::avg, 2.0f);\n\n  auto arg_name_list = net_.ListArguments();\n  for (index_t i = 0; i < in_shapes.size(); i++) {\n    const auto &shape = in_shapes[i];\n    const auto &arg_name = arg_name_list[i];\n    int paramType = kFloat32;\n    if (Initializer::StringEndWith(arg_name, \"weight_quantize\") ||\n        Initializer::StringEndWith(arg_name, \"bias_quantize\")) {\n      paramType = kInt8;\n    }\n    NDArray tmp_arr(shape, global_ctx_, false, paramType);\n    xavier(arg_name, &tmp_arr);\n    args_map_[arg_name] = tmp_arr.Copy(global_ctx_);\n  }\n\n  auto aux_name_list = net_.ListAuxiliaryStates();\n  for (index_t i = 0; i < aux_shapes.size(); i++) {\n    const auto &shape = aux_shapes[i];\n    const auto &aux_name = aux_name_list[i];\n    NDArray tmp_arr(shape, global_ctx_, false);\n    xavier(aux_name, &tmp_arr);\n    aux_map_[aux_name] = tmp_arr.Copy(global_ctx_);\n  }\n  /*WaitAll is need when we copy data between GPU and the main memory*/\n  NDArray::WaitAll();\n}\n\n/*\n * The following function runs the forward pass on the model\n * and use dummy data for benchmark.\n */\nvoid Predictor::BenchmarkScore(int num_inference_batches) {\n  // Create dummy data\n  std::vector<float> dummy_data(input_shape_.Size());\n  std::default_random_engine generator;\n  std::uniform_real_distribution<float> val(0.0f, 1.0f);\n  for (size_t i = 0; i < static_cast<size_t>(input_shape_.Size()); ++i) {\n    dummy_data[i] = static_cast<float>(val(generator));\n  }\n  executor_->arg_dict()[\"data\"].SyncCopyFromCPU(\n        dummy_data.data(),\n        input_shape_.Size());\n  NDArray::WaitAll();\n\n  LG << \"Running the forward pass on model to evaluate the performance..\";\n\n  // warm up.\n  for (int i = 0; i < 5; i++) {\n    executor_->Forward(false);\n    NDArray::WaitAll();\n  }\n\n  // Run the forward pass.\n  double ms = ms_now();\n  for (int i = 0; i < num_inference_batches; i++) {\n    executor_->Forward(false);\n    NDArray::WaitAll();\n  }\n  ms = ms_now() - ms;\n  LG << \" benchmark completed!\";\n  LG << \" batch size: \" << input_shape_[0] << \" num batch: \" << num_inference_batches\n     << \" throughput: \" << 1000.0 * input_shape_[0] * num_inference_batches / ms\n     << \" imgs/s latency:\" << ms / input_shape_[0] / num_inference_batches << \" ms\";\n}\n\n/*\n * \\param skipped_batches skip the first number of batches\n *\n */\nbool Predictor::AdvanceDataIter(int skipped_batches) {\n  assert(skipped_batches >= 0);\n  if (skipped_batches == 0) return true;\n  int skipped_count = 0;\n  while (val_iter_->Next()) {\n    if (++skipped_count >= skipped_batches) break;\n  }\n  if (skipped_count != skipped_batches) return false;\n  return true;\n}\n\n/*\n * The following function runs the forward pass on the model\n * and use real data for testing accuracy and performance.\n */\nvoid Predictor::Score(int num_skipped_batches, int num_inference_batches) {\n  // Create metrics\n  Accuracy val_acc;\n\n  val_iter_->Reset();\n  val_acc.Reset();\n  int nBatch = 0;\n\n  if (!AdvanceDataIter(num_skipped_batches)) {\n    LG << \"skipped batches should less than total batches!\";\n    return;\n  }\n\n  double ms = ms_now();\n  while (val_iter_->Next()) {\n    auto data_batch = val_iter_->GetDataBatch();\n    data_batch.data.CopyTo(&args_map_[\"data\"]);\n    data_batch.label.CopyTo(&args_map_[\"softmax_label\"]);\n    NDArray::WaitAll();\n\n    // running on forward pass\n    executor_->Forward(false);\n    NDArray::WaitAll();\n    val_acc.Update(data_batch.label, executor_->outputs[0]);\n\n    if (++nBatch >= num_inference_batches) {\n      break;\n    }\n  }\n  ms = ms_now() - ms;\n  auto args_name = net_.ListArguments();\n  LG << \"INFO:\" << \"Dataset for inference: \" << dataset_;\n  LG << \"INFO:\" << \"label_name = \" << args_name[args_name.size()-1];\n  LG << \"INFO:\" << \"rgb_mean: \" << \"(\" << rgb_mean_[0] << \", \" << rgb_mean_[1]\n     << \", \" << rgb_mean_[2] << \")\";\n  LG << \"INFO:\" << \"rgb_std: \" << \"(\" << rgb_std_[0] << \", \" << rgb_std_[1]\n     << \", \" << rgb_std_[2] << \")\";\n  LG << \"INFO:\" << \"Image shape: \" << \"(\" << input_shape_[1] << \", \"\n     << input_shape_[2] << \", \" << input_shape_[3] << \")\";\n  LG << \"INFO:\" << \"Finished inference with: \" << nBatch * input_shape_[0]\n     << \" images \";\n  LG << \"INFO:\" << \"Batch size = \" << input_shape_[0] << \" for inference\";\n  LG << \"INFO:\" << \"Accuracy: \" << val_acc.Get();\n  LG << \"INFO:\" << \"Throughput: \" << (1000.0 * nBatch * input_shape_[0] / ms)\n     << \" images per second\";\n}\n\nPredictor::~Predictor() {\n  if (executor_) {\n    delete executor_;\n  }\n  if (!benchmark_ && val_iter_) {\n    delete val_iter_;\n  }\n  MXNotifyShutdown();\n}\n\n/*\n * Convert the input string of number into the vector.\n */\ntemplate<typename T>\nstd::vector<T> createVectorFromString(const std::string& input_string) {\n  std::vector<T> dst_vec;\n  char *p_next;\n  T elem;\n  bool bFloat = std::is_same<T, float>::value;\n  if (!bFloat) {\n    elem = strtol(input_string.c_str(), &p_next, 10);\n  } else {\n    elem = strtof(input_string.c_str(), &p_next);\n  }\n\n  dst_vec.push_back(elem);\n  while (*p_next) {\n    if (!bFloat) {\n      elem = strtol(p_next, &p_next, 10);\n    } else {\n      elem = strtof(p_next, &p_next);\n    }\n    dst_vec.push_back(elem);\n  }\n  return dst_vec;\n}\n\nvoid printUsage() {\n    std::cout << \"Usage:\" << std::endl;\n    std::cout << \"imagenet_inference --symbol_file <model symbol file in json format>\" << std::endl\n              << \"--params_file <model params file> \" << std::endl\n              << \"--dataset <dataset used to run inference> \" << std::endl\n              << \"--data_nthreads <default: 60> \" << std::endl\n              << \"--input_shape <shape of input image e.g \\\"3 224 224\\\">] \" << std::endl\n              << \"--rgb_mean <mean value to be subtracted on RGB channel e.g \\\"0 0 0\\\">\"\n              << std::endl\n              << \"--rgb_std <standard deviation on R/G/B channel. e.g \\\"1 1 1\\\"> \" << std::endl\n              << \"--batch_size <number of images per batch> \" << std::endl\n              << \"--num_skipped_batches <skip the number of batches for inference> \" << std::endl\n              << \"--num_inference_batches <number of batches used for inference> \" << std::endl\n              << \"--data_layer_type <default: \\\"float32\\\" \"\n              << \"choices: [\\\"float32\\\",\\\"int8\\\",\\\"uint8\\\"]>\" << std::endl\n              << \"--gpu  <whether to run inference on GPU, default: false>\" << std::endl\n              << \"--enableTRT  <whether to run inference with TensorRT, \"\n              << \"default: false>\" << std::endl\n              << \"--benchmark <whether to use dummy data to run inference, default: false>\"\n              << std::endl;\n}\n\nint main(int argc, char** argv) {\n  std::string model_file_json;\n  std::string model_file_params;\n  std::string dataset(\"\");\n  std::string input_rgb_mean(\"0 0 0\");\n  std::string input_rgb_std(\"1 1 1\");\n  bool use_gpu = false;\n  bool enable_tensorrt = false;\n  bool benchmark = false;\n  int batch_size = 64;\n  int num_skipped_batches = 0;\n  int num_inference_batches = 100;\n  std::string data_layer_type(\"float32\");\n  std::string input_shape(\"3 224 224\");\n  int seed = 48564309;\n  int shuffle_chunk_seed = 3982304;\n  int data_nthreads = 60;\n\n  int index = 1;\n  while (index < argc) {\n    if (strcmp(\"--symbol_file\", argv[index]) == 0) {\n      index++;\n      model_file_json = (index < argc ? argv[index]:\"\");\n    } else if (strcmp(\"--params_file\", argv[index]) == 0) {\n      index++;\n      model_file_params = (index < argc ? argv[index]:\"\");\n    } else if (strcmp(\"--dataset\", argv[index]) == 0) {\n      index++;\n      dataset = (index < argc ? argv[index]:dataset);\n    } else if (strcmp(\"--data_nthreads\", argv[index]) == 0) {\n      index++;\n      data_nthreads = strtol(argv[index], nullptr, 10);\n    } else if (strcmp(\"--input_shape\", argv[index]) == 0) {\n      index++;\n      input_shape = (index < argc ? argv[index]:input_shape);\n    } else if (strcmp(\"--rgb_mean\", argv[index]) == 0) {\n      index++;\n      input_rgb_mean = (index < argc ? argv[index]:input_rgb_mean);\n    } else if (strcmp(\"--rgb_std\", argv[index]) == 0) {\n      index++;\n      input_rgb_std = (index < argc ? argv[index]:input_rgb_std);\n    } else if (strcmp(\"--batch_size\", argv[index]) == 0) {\n      index++;\n      batch_size = strtol(argv[index], nullptr, 10);\n    }  else if (strcmp(\"--num_skipped_batches\", argv[index]) == 0) {\n      index++;\n      num_skipped_batches = strtol(argv[index], nullptr, 10);\n    }  else if (strcmp(\"--num_inference_batches\", argv[index]) == 0) {\n      index++;\n      num_inference_batches = strtol(argv[index], nullptr, 10);\n    } else if (strcmp(\"--data_layer_type\", argv[index]) == 0) {\n      index++;\n      data_layer_type = (index < argc ? argv[index]:data_layer_type);\n    } else if (strcmp(\"--gpu\", argv[index]) == 0) {\n      use_gpu = true;\n    } else if (strcmp(\"--enableTRT\", argv[index]) == 0) {\n      use_gpu = true;\n      enable_tensorrt = true;\n    } else if (strcmp(\"--benchmark\", argv[index]) == 0) {\n      benchmark = true;\n    } else if (strcmp(\"--help\", argv[index]) == 0) {\n      printUsage();\n      return 0;\n    }\n    index++;\n  }\n\n  if (model_file_json.empty()\n      || (!benchmark && model_file_params.empty())\n      || (enable_tensorrt && model_file_params.empty())) {\n    LG << \"ERROR: Model details such as symbol, param files are not specified\";\n    printUsage();\n    return 1;\n  }\n  std::vector<index_t> input_dimensions = createVectorFromString<index_t>(input_shape);\n  input_dimensions.insert(input_dimensions.begin(), batch_size);\n  Shape input_data_shape(input_dimensions);\n\n  std::vector<float> rgb_mean = createVectorFromString<float>(input_rgb_mean);\n  std::vector<float> rgb_std = createVectorFromString<float>(input_rgb_std);\n\n  // Initialize the predictor object\n  Predictor predict(model_file_json, model_file_params, input_data_shape, use_gpu, enable_tensorrt,\n                    dataset, data_nthreads, data_layer_type, rgb_mean, rgb_std, shuffle_chunk_seed,\n                    seed, benchmark);\n\n  if (benchmark) {\n    predict.BenchmarkScore(num_inference_batches);\n  } else {\n    predict.Score(num_skipped_batches, num_inference_batches);\n  }\n  return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/inference/multi_threaded_inference/get_model.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport logging\nimport argparse\nimport requests\nimport errno\nimport os\n\nmodels = [\"imagenet1k-inception-bn\", \"imagenet1k-resnet-50\",\n          \"imagenet1k-resnet-152\", \"imagenet1k-resnet-18\"]\n\ndef download(url, fname=None, dirname=None, overwrite=False, retries=5):\n    \"\"\"Download an given URL\n\n    Parameters\n    ----------\n\n    url : str\n        URL to download\n    fname : str, optional\n        filename of the downloaded file. If None, then will guess a filename\n        from url.\n    dirname : str, optional\n        output directory name. If None, then guess from fname or use the current\n        directory\n    overwrite : bool, optional\n        Default is false, which means skipping download if the local file\n        exists. If true, then download the url to overwrite the local file if\n        exists.\n    retries : integer, default 5\n        The number of times to attempt the download in case of failure or non 200 return codes\n\n    Returns\n    -------\n    str\n        The filename of the downloaded file\n    \"\"\"\n\n    assert retries >= 0, \"Number of retries should be at least 0\"\n\n    if fname is None:\n        fname = url.split('/')[-1]\n\n    if dirname is None:\n        dirname = os.path.dirname(fname)\n    else:\n        fname = os.path.join(dirname, fname)\n    if dirname != \"\":\n        if not os.path.exists(dirname):\n            try:\n                logging.info('create directory %s', dirname)\n                os.makedirs(dirname)\n            except OSError as exc:\n                if exc.errno != errno.EEXIST:\n                    raise OSError('failed to create ' + dirname)\n\n    if not overwrite and os.path.exists(fname):\n        logging.info(\"%s exists, skipping download\", fname)\n        return fname\n\n    while retries+1 > 0:\n        # Disable pyling too broad Exception\n        # pylint: disable=W0703\n        try:\n            r = requests.get(url, stream=True)\n            assert r.status_code == 200, f\"failed to open {url}\"\n            with open(fname, 'wb') as f:\n                for chunk in r.iter_content(chunk_size=1024):\n                    if chunk: # filter out keep-alive new chunks\n                        f.write(chunk)\n                break\n        except Exception as e:\n            retries -= 1\n            if retries <= 0:\n                raise e\n\n            print(\"download failed, retrying, {} attempt{} left\"\n                  .format(retries, 's' if retries > 1 else ''))\n    logging.info(\"downloaded %s into %s successfully\", url, fname)\n    return fname\n\ndef download_model(model_name, dst_dir='./', meta_info=None):\n    \"\"\"Download a model from data.mxnet.io\n\n    Parameters\n    ----------\n    model_name : str\n        Model name to download\n    dst_dir : str\n        Destination Directory to download the model\n    meta_info : dict of dict\n        Mapping from model_name to dict of the following structure:\n        {'symbol': url, 'params': url}\n\n    Returns\n    -------\n    Two element tuple containing model_name and epoch for the params saved\n    \"\"\"\n    _base_model_url = 'http://data.mxnet.io/models/'\n    _default_model_info = {\n        'imagenet1k-inception-bn': {'symbol':_base_model_url+'imagenet/inception-bn/Inception-BN-symbol.json',\n                                    'params':_base_model_url+'imagenet/inception-bn/Inception-BN-0126.params'},\n        'imagenet1k-resnet-18': {'symbol':_base_model_url+'imagenet/resnet/18-layers/resnet-18-symbol.json',\n                                 'params':_base_model_url+'imagenet/resnet/18-layers/resnet-18-0000.params'},\n        'imagenet1k-resnet-34': {'symbol':_base_model_url+'imagenet/resnet/34-layers/resnet-34-symbol.json',\n                                 'params':_base_model_url+'imagenet/resnet/34-layers/resnet-34-0000.params'},\n        'imagenet1k-resnet-50': {'symbol':_base_model_url+'imagenet/resnet/50-layers/resnet-50-symbol.json',\n                                 'params':_base_model_url+'imagenet/resnet/50-layers/resnet-50-0000.params'},\n        'imagenet1k-resnet-101': {'symbol':_base_model_url+'imagenet/resnet/101-layers/resnet-101-symbol.json',\n                                  'params':_base_model_url+'imagenet/resnet/101-layers/resnet-101-0000.params'},\n        'imagenet1k-resnet-152': {'symbol':_base_model_url+'imagenet/resnet/152-layers/resnet-152-symbol.json',\n                                  'params':_base_model_url+'imagenet/resnet/152-layers/resnet-152-0000.params'},\n        'imagenet1k-resnext-50': {'symbol':_base_model_url+'imagenet/resnext/50-layers/resnext-50-symbol.json',\n                                  'params':_base_model_url+'imagenet/resnext/50-layers/resnext-50-0000.params'},\n        'imagenet1k-resnext-101': {'symbol':_base_model_url+'imagenet/resnext/101-layers/resnext-101-symbol.json',\n                                   'params':_base_model_url+'imagenet/resnext/101-layers/resnext-101-0000.params'},\n        'imagenet1k-resnext-101-64x4d':\n            {'symbol':_base_model_url+'imagenet/resnext/101-layers/resnext-101-64x4d-symbol.json',\n             'params':_base_model_url+'imagenet/resnext/101-layers/resnext-101-64x4d-0000.params'},\n        'imagenet11k-resnet-152':\n            {'symbol':_base_model_url+'imagenet-11k/resnet-152/resnet-152-symbol.json',\n             'params':_base_model_url+'imagenet-11k/resnet-152/resnet-152-0000.params'},\n        'imagenet11k-place365ch-resnet-152':\n            {'symbol':_base_model_url+'imagenet-11k-place365-ch/resnet-152-symbol.json',\n             'params':_base_model_url+'imagenet-11k-place365-ch/resnet-152-0000.params'},\n        'imagenet11k-place365ch-resnet-50':\n            {'symbol':_base_model_url+'imagenet-11k-place365-ch/resnet-50-symbol.json',\n             'params':_base_model_url+'imagenet-11k-place365-ch/resnet-50-0000.params'},\n    }\n\n\n    if meta_info is None:\n        meta_info = _default_model_info\n    meta_info = dict(meta_info)\n    if model_name not in meta_info:\n        return (None, 0)\n    if not os.path.isdir(dst_dir):\n        os.mkdir(dst_dir)\n    meta = dict(meta_info[model_name])\n    assert 'symbol' in meta, \"missing symbol url\"\n    model_name = os.path.join(dst_dir, model_name)\n    download(meta['symbol'], model_name+'-symbol.json')\n    assert 'params' in meta, \"mssing parameter file url\"\n    download(meta['params'], model_name+'-0000.params')\n    download(_base_model_url + 'imagenet/synset.txt')\n    return (model_name, 0)\n\ndef main():\n    logging.basicConfig()\n    logger = logging.getLogger(\"logger\")\n    logger.setLevel(logging.INFO)\n    parser = argparse.ArgumentParser(description='Download model hybridize and save as symbolic model for multithreaded inference')\n    parser.add_argument(\"--model\", type=str, choices=models, required=True)\n    args = parser.parse_args()\n\n    download_model(args.model)\n\nif __name__ == \"__main__\":\n    main()\n"
  },
  {
    "path": "cpp-package/example/inference/multi_threaded_inference/multi_threaded_inference.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file multi_threaded_inference.cc\n * \\brief Multi Threaded inference example with CachedOp\n */\n\n#include <mxnet/ndarray.h>\n\n#include <cstdio>\n#include <cstdlib>\n#include <iostream>\n#include <fstream>\n#include <thread>\n#include <iomanip>\n#include <chrono>\n#include <random>\n#include \"mxnet-cpp/MxNetCpp.h\"\n#include <opencv2/opencv.hpp>\n\nconst float DEFAULT_MEAN = 117.0;\n\n// Code to load image, PrintOutput results, helper functions for the same obtained from:\n// https://github.com/apache/mxnet/blob/master/example/image-classification/predict-cpp/\n\nstatic std::string trim(const std::string& input) {\n  auto not_space = [](int ch) { return !std::isspace(ch); };\n  auto output    = input;\n  output.erase(output.begin(), std::find_if(output.begin(), output.end(), not_space));\n  output.erase(std::find_if(output.rbegin(), output.rend(), not_space).base(), output.end());\n  return output;\n}\n\nstd::vector<std::string> LoadSynset(const std::string& synset_file) {\n  std::ifstream fi(synset_file.c_str());\n\n  if (!fi.is_open()) {\n    std::cerr << \"Error opening synset file \" << synset_file << std::endl;\n    assert(false);\n  }\n\n  std::vector<std::string> output;\n\n  std::string synset, lemma;\n  while (fi >> synset) {\n    getline(fi, lemma);\n    output.push_back(lemma);\n  }\n\n  fi.close();\n\n  return output;\n}\n\nvoid PrintOutputResult(const float* data, size_t size, const std::vector<std::string>& synset) {\n  if (size != synset.size()) {\n    std::cerr << \"Result data and synset size do not match!\" << std::endl;\n  }\n\n  float best_accuracy  = 0.0;\n  std::size_t best_idx = 0;\n\n  for (std::size_t i = 0; i < size; ++i) {\n    if (data[i] > best_accuracy) {\n      best_accuracy = data[i];\n      best_idx      = i;\n    }\n  }\n\n  std::cout << \"Best Result: \" << trim(synset[best_idx]) << \" (id=\" << best_idx << \", \"\n            << \"accuracy=\" << std::setprecision(8) << best_accuracy << \")\" << std::endl;\n}\n\n// Read Image data into a float array\nvoid GetImageFile(const std::string& image_file,\n                  float* image_data,\n                  int channels,\n                  cv::Size resize_size) {\n  // Read all kinds of file into a BGR color 3 channels image\n  cv::Mat im_ori = cv::imread(image_file, cv::IMREAD_COLOR);\n\n  if (im_ori.empty()) {\n    std::cerr << \"Can't open the image. Plase check \" << image_file << \". \\n\";\n    assert(false);\n  }\n\n  cv::Mat im;\n  resize(im_ori, im, resize_size);\n\n  int size = im.rows * im.cols * channels;\n\n  float* ptr_image_r = image_data;\n  float* ptr_image_g = image_data + size / 3;\n  float* ptr_image_b = image_data + size / 3 * 2;\n\n  float mean_b, mean_g, mean_r;\n  mean_b = mean_g = mean_r = DEFAULT_MEAN;\n\n  for (int i = 0; i < im.rows; ++i) {\n    auto data = im.ptr<uchar>(i);\n    for (int j = 0; j < im.cols; j++) {\n      if (channels > 1) {\n        *ptr_image_b++ = static_cast<float>(*data++) - mean_b;\n        *ptr_image_g++ = static_cast<float>(*data++) - mean_g;\n      }\n    }\n    *ptr_image_r++ = static_cast<float>(*data++) - mean_r;\n  }\n}\n\nvoid prepare_input_data(const mxnet::cpp::Shape& shape,\n                        const mxnet::cpp::Context& ctx,\n                        int num_threads,\n                        std::vector<mxnet::cpp::NDArray>* data_arr,\n                        bool random_uniform = false) {\n  for (size_t i = 0; i < num_threads; ++i) {\n    data_arr->emplace_back(shape, ctx, false, 0);\n    int begin = i * 100;\n    int end   = begin + 100;\n    if (random_uniform) {\n      mxnet::cpp::Operator(\"_random_uniform\")(begin, end).Invoke((*data_arr)[i]);\n    }\n    mxnet::cpp::NDArray::WaitAll();\n  }\n}\n\n// Run inference on a model\nvoid run_inference(const std::string& model_name,\n                   const std::vector<mxnet::cpp::NDArray>& input_arrs,\n                   std::vector<mxnet::NDArray*>* output_mx_arr,\n                   int num_inf_per_thread = 1,\n                   bool random_sleep      = false,\n                   int num_threads        = 1,\n                   bool static_alloc      = false,\n                   bool static_shape      = false,\n                   bool is_gpu            = false) {\n  LOG(INFO) << \"Running inference for \" + model_name +\n                   \" num_threads: \" + std::to_string(num_threads) +\n                   \" num_inf_per_thread: \" + std::to_string(num_inf_per_thread) +\n                   \" random_sleep: \" + std::to_string(random_sleep) +\n                   \" static_alloc: \" + std::to_string(static_alloc) +\n                   \" static_shape: \" + std::to_string(static_shape);\n  std::string json_file        = model_name + \"-symbol.json\";\n  std::string param_file       = model_name + \"-0000.params\";\n  auto out                     = mxnet::cpp::Symbol::Load(json_file);\n  std::string static_alloc_str = static_alloc ? \"true\" : \"false\";\n  std::string static_shape_str = static_shape ? \"true\" : \"false\";\n\n  // Prepare context\n#if MXNET_USE_CUDA == 1\n  mxnet::Context backend_ctx;\n  mxnet::cpp::Context ctx = mxnet::cpp::Context::cpu(0);\n  if (is_gpu) {\n    backend_ctx = mxnet::Context::GPU(0);\n    ctx         = mxnet::cpp::Context::gpu(0);\n  } else {\n    backend_ctx = mxnet::Context::CPU(0);\n    ctx         = mxnet::cpp::Context::cpu(0);\n  }\n#else\n  mxnet::Context backend_ctx = mxnet::Context::CPU(0);\n  mxnet::cpp::Context ctx    = mxnet::cpp::Context::cpu(0);\n#endif\n\n  // Prepare input data and parameters\n  std::vector<mxnet::cpp::NDArray> data_arr(num_threads);\n  std::vector<mxnet::cpp::NDArray> softmax_arr;\n  std::vector<mxnet::cpp::NDArray> params;\n  mxnet::cpp::Shape data_shape    = mxnet::cpp::Shape(1, 3, 224, 224);\n  mxnet::cpp::Shape softmax_shape = mxnet::cpp::Shape(1);\n  int num_inputs                  = out.ListInputs().size();\n\n  for (size_t i = 0; i < data_arr.size(); ++i) {\n    data_arr[i] = input_arrs[i].Copy(ctx);\n  }\n  prepare_input_data(softmax_shape, ctx, num_threads, &softmax_arr);\n  std::map<std::string, mxnet::cpp::NDArray> parameters;\n  mxnet::cpp::NDArray::Load(param_file, 0, &parameters);\n\n  for (const std::string& name : out.ListInputs()) {\n    if (name == \"arg:data\") {\n      continue;\n    }\n    if (parameters.find(\"arg:\" + name) != parameters.end()) {\n      params.push_back(parameters[\"arg:\" + name].Copy(ctx));\n    } else if (parameters.find(\"aux:\" + name) != parameters.end()) {\n      params.push_back(parameters[\"aux:\" + name].Copy(ctx));\n    }\n  }\n\n  CachedOpHandle hdl = CachedOpHandle();\n\n  std::vector<std::string> flag_keys{\n      \"data_indices\", \"param_indices\", \"static_alloc\", \"static_shape\"};\n  std::string param_indices = \"[\";\n  for (size_t i = 1; i < num_inputs; ++i) {\n    param_indices += std::to_string(i);\n    param_indices += std::string(\", \");\n  }\n  param_indices += \"]\";\n  std::vector<std::string> flag_vals{\"[0]\", param_indices, static_alloc_str, static_shape_str};\n  std::vector<const char*> flag_key_cstrs, flag_val_cstrs;\n  flag_key_cstrs.reserve(flag_keys.size());\n  for (size_t i = 0; i < flag_keys.size(); ++i) {\n    flag_key_cstrs.emplace_back(flag_keys[i].c_str());\n  }\n  for (size_t i = 0; i < flag_vals.size(); ++i) {\n    flag_val_cstrs.emplace_back(flag_vals[i].c_str());\n  }\n\n  int ret1 = MXCreateCachedOp(\n      out.GetHandle(), flag_keys.size(), flag_key_cstrs.data(), flag_val_cstrs.data(), &hdl, true);\n  if (ret1 < 0) {\n    LOG(FATAL) << MXGetLastError();\n  }\n\n  // Prepare data structures and lambda to run in different threads\n  std::vector<NDArrayHandle*> cached_op_handles(num_threads);\n\n  std::vector<std::vector<NDArrayHandle>> arr_handles(num_threads);\n  for (size_t i = 0; i < num_threads; ++i) {\n    arr_handles[i].reserve(num_inputs);\n    arr_handles[i].emplace_back(data_arr[i].GetHandle());\n    for (size_t j = 1; j < num_inputs - 1; ++j) {\n      arr_handles[i].emplace_back(params[j - 1].GetHandle());\n    }\n    arr_handles[i].emplace_back(softmax_arr[i].GetHandle());\n  }\n\n  auto func = [&](int num) {\n    unsigned next = num;\n    if (random_sleep) {\n      static thread_local std::mt19937 generator;\n      std::uniform_int_distribution<int> distribution(0, 5);\n      int sleep_time = distribution(generator);\n      std::this_thread::sleep_for(std::chrono::seconds(sleep_time));\n    }\n    int num_output = 0;\n    const int* stypes;\n    int ret = MXInvokeCachedOp(hdl,\n                               arr_handles[num].size(),\n                               arr_handles[num].data(),\n                               ctx.GetDeviceType(),\n                               0,\n                               &num_output,\n                               &(cached_op_handles[num]),\n                               &stypes);\n    if (ret < 0) {\n      LOG(FATAL) << MXGetLastError();\n    }\n    (*output_mx_arr)[num] = static_cast<mxnet::NDArray*>(*cached_op_handles[num]);\n  };\n\n  // Spawn multiple threads, join and wait for threads to complete\n  std::vector<std::thread> worker_threads(num_threads);\n  int count = 0;\n  for (auto&& i : worker_threads) {\n    i = std::thread(func, count);\n    count++;\n  }\n\n  for (auto&& i : worker_threads) {\n    i.join();\n  }\n\n  mxnet::cpp::NDArray::WaitAll();\n\n  std::string synset_file = \"synset.txt\";\n  auto synset             = LoadSynset(synset_file);\n  std::vector<mxnet::NDArray> tmp(num_threads);\n  for (size_t i = 0; i < num_threads; i++) {\n    tmp[i] = (*output_mx_arr)[i]->Copy(mxnet::Context::CPU(0));\n    tmp[i].WaitToRead();\n    (*output_mx_arr)[i] = &tmp[i];\n  }\n  for (size_t i = 0; i < num_threads; ++i) {\n    PrintOutputResult(static_cast<float*>((*output_mx_arr)[i]->data().dptr_),\n                      (*output_mx_arr)[i]->shape().Size(),\n                      synset);\n  }\n  int ret2 = MXFreeCachedOp(hdl);\n  if (ret2 < 0) {\n    LOG(FATAL) << MXGetLastError();\n  }\n  mxnet::cpp::NDArray::WaitAll();\n}\n\nint main(int argc, char* argv[]) {\n  if (argc < 4) {\n    std::cout << \"Please provide a model name, is_gpu, test_image\" << std::endl\n              << \"Usage: ./multi_threaded_inference [model_name] [is_gpu] [file_names]\" << std::endl\n              << \"Example: ./.multi_threaded_inference imagenet1k-inception-bn 0 apple.jpg\"\n              << std::endl\n              << \"NOTE: Thread number ordering will be based on the ordering of file inputs\"\n              << std::endl\n              << \"NOTE: Epoch is assumed to be 0\" << std::endl;\n    return EXIT_FAILURE;\n  }\n  std::string model_name = std::string(argv[1]);\n  bool is_gpu            = std::atoi(argv[2]);\n  CHECK(argc >= 4) << \"Number of files provided should be atleast 1\";\n  int num_threads = argc - 3;\n  std::vector<std::string> test_files;\n  for (size_t i = 0; i < argc - 3; ++i) {\n    test_files.emplace_back(argv[3 + i]);\n  }\n  int epoch         = 0;\n  bool static_alloc = true;\n  bool static_shape = true;\n\n  // Image size and channels\n  size_t width    = 224;\n  size_t height   = 224;\n  size_t channels = 3;\n\n  size_t image_size = width * height * channels;\n\n  // Read Image Data\n  // load into an input arr\n  std::vector<std::vector<float>> files(num_threads);\n  std::vector<mxnet::cpp::NDArray> input_arrs;\n  mxnet::cpp::Shape input_shape = mxnet::cpp::Shape(1, 3, 224, 224);\n  for (size_t i = 0; i < files.size(); i++) {\n    files[i].resize(image_size);\n    GetImageFile(test_files[i], files[i].data(), channels, cv::Size(width, height));\n    input_arrs.emplace_back(\n        mxnet::cpp::NDArray(files[i].data(), input_shape, mxnet::cpp::Context::cpu(0)));\n  }\n\n  // load symbol\n  std::string static_alloc_str = static_alloc ? \"true\" : \"false\";\n  std::string static_shape_str = static_shape ? \"true\" : \"false\";\n  std::vector<mxnet::NDArray*> output_mx_arr(num_threads);\n  run_inference(model_name,\n                input_arrs,\n                &output_mx_arr,\n                1,\n                false,\n                num_threads,\n                static_alloc,\n                static_shape,\n                is_gpu);\n  mxnet::cpp::NDArray::WaitAll();\n\n  return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/inference/multi_threaded_inference/unit_test_multi_threaded_inference.sh",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# http://mxnet.apache.org/versions/master/api/cpp/docs/tutorials/multi_threaded_inference.html\n\n# Install test data.\nwget https://github.com/tensorflow/tensorflow/raw/master/tensorflow/examples/label_image/data/grace_hopper.jpg\nwget http://optipng.sourceforge.net/pngtech/img/lena.png\n\n# Get Model.\npython3 get_model.py --model imagenet1k-inception-bn\n\n# Run test\n./multi_threaded_inference imagenet1k-inception-bn 1 grace_hopper.jpg lena.png\n"
  },
  {
    "path": "cpp-package/example/inference/sentiment_analysis_rnn.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*\n * This example demonstrates sentiment prediction workflow with pre-trained RNN model using MXNet C++ API.\n * The example performs following tasks.\n * 1. Load the pre-trained RNN model,\n * 2. Load the dictionary file that contains word to index mapping.\n * 3. Create executors for pre-determined input lengths.\n * 4. Convert each line in the input to the vector of indices.\n * 5. Predictor finds the right executor for each line.\n * 4. Run the forward pass for each line and predicts the sentiment scores.\n * The example uses a pre-trained RNN model that is trained with the IMDB dataset.\n */\n\n#include <sys/stat.h>\n#include <iostream>\n#include <fstream>\n#include <cstdlib>\n#include <map>\n#include <string>\n#include <algorithm>\n#include <vector>\n#include <sstream>\n#include \"mxnet-cpp/MxNetCpp.h\"\n\nusing namespace mxnet::cpp;\n\nstatic const int DEFAULT_BUCKET_KEYS[] = {30, 25, 20, 15, 10, 5};\nstatic const char DEFAULT_S3_URL[] = \"https://s3.amazonaws.com/mxnet-cpp/RNN_model/\";\n\n\n/*\n * class Predictor\n *\n * This class encapsulates the functionality to load the model, process input image and run the forward pass.\n */\n\nclass Predictor {\n public:\n    Predictor() {}\n    Predictor(const std::string& model_json,\n              const std::string& model_params,\n              const std::string& input_dictionary,\n              const std::vector<int>& bucket_keys,\n              bool use_gpu = false);\n    float PredictSentiment(const std::string &input_review);\n    ~Predictor();\n\n private:\n    void LoadModel(const std::string& model_json_file);\n    void LoadParameters(const std::string& model_parameters_file);\n    void LoadDictionary(const std::string &input_dictionary);\n    inline bool FileExists(const std::string& name) {\n        struct stat buffer;\n        return (stat(name.c_str(), &buffer) == 0);\n    }\n    float PredictSentimentForOneLine(const std::string &input_line);\n    int ConvertToIndexVector(const std::string& input,\n                      std::vector<float> *input_vector);\n    int GetIndexForOutputSymbolName(const std::string& output_symbol_name);\n    float GetIndexForWord(const std::string& word);\n    int GetClosestBucketKey(int num_words);\n\n    std::map<std::string, NDArray> args_map;\n    std::map<std::string, NDArray> aux_map;\n    std::map<std::string, int>  wordToIndex;\n    Symbol net;\n    std::map<int, Executor*> executor_buckets;\n    Context global_ctx = Context::cpu();\n    int highest_bucket_key;\n};\n\n\n/*\n * The constructor takes the following parameters as input:\n * 1. model_json:  The RNN model in json formatted file.\n * 2. model_params: File containing model parameters\n * 3. input_dictionary: File containing the word and associated index.\n * 4. bucket_keys: A vector of bucket keys for creating executors.\n *\n * The constructor:\n *  1. Loads the model and parameter files.\n *  2. Loads the dictionary file to create index to word and word to index maps.\n *  3. For each bucket key in the input vector of bucket keys, it creates an executor.\n *     The executors share the memory. The bucket key determines the length of input data\n *     required for that executor.\n *  4. Creates a map of bucket key to corresponding executor.\n *  5. The model is loaded only once. The executors share the memory for the parameters.\n */\nPredictor::Predictor(const std::string& model_json,\n                     const std::string& model_params,\n                     const std::string& input_dictionary,\n                     const std::vector<int>& bucket_keys,\n                     bool use_gpu) {\n  if (use_gpu) {\n    global_ctx = Context::gpu();\n  }\n\n  /*\n   * Load the dictionary file that contains the word and its index.\n   * The function creates word to index and index to word map. The maps are used to create index\n   * vector for the input sentence.\n   */\n  LoadDictionary(input_dictionary);\n\n  // Load the model\n  LoadModel(model_json);\n\n  // Load the model parameters.\n  LoadParameters(model_params);\n\n  /*\n   * Create the executors for each bucket key. The bucket key represents the shape of input data.\n   * The executors will share the memory by using following technique:\n   * 1. Infer the executor arrays and bind the first executor with the first bucket key.\n   * 2. Then for creating the next bucket key, adjust the shape of input argument to match that key.\n   * 3. Create the executor for the next bucket key by passing the inferred executor arrays and\n   *    pointer to the executor created for the first key.\n   */\n  std::vector<NDArray> arg_arrays;\n  std::vector<NDArray> grad_arrays;\n  std::vector<OpReqType> grad_reqs;\n  std::vector<NDArray> aux_arrays;\n\n  /*\n   * Create master executor with highest bucket key for optimizing the shared memory between the\n   * executors for the remaining bucket keys.\n   */\n  highest_bucket_key = *(std::max_element(bucket_keys.begin(), bucket_keys.end()));\n  args_map[\"data0\"] = NDArray(Shape(highest_bucket_key, 1), global_ctx, false);\n  args_map[\"data1\"] = NDArray(Shape(1), global_ctx, false);\n\n  net.InferExecutorArrays(global_ctx, &arg_arrays, &grad_arrays, &grad_reqs,\n                          &aux_arrays, args_map, std::map<std::string, NDArray>(),\n                              std::map<std::string, OpReqType>(), aux_map);\n  Executor *master_executor = net.Bind(global_ctx, arg_arrays, grad_arrays, grad_reqs, aux_arrays,\n                                 std::map<std::string, Context>(), nullptr);\n  executor_buckets[highest_bucket_key] = master_executor;\n\n  for (int bucket : bucket_keys) {\n    if (executor_buckets.find(bucket) == executor_buckets.end()) {\n      arg_arrays[0]  = NDArray(Shape(bucket, 1), global_ctx, false);\n      Executor *executor = net.Bind(global_ctx, arg_arrays, grad_arrays, grad_reqs, aux_arrays,\n                                    std::map<std::string, Context>(), master_executor);\n      executor_buckets[bucket] = executor;\n    }\n  }\n}\n\n\n/*\n * The following function loads the model from json file.\n */\nvoid Predictor::LoadModel(const std::string& model_json_file) {\n  if (!FileExists(model_json_file)) {\n    LG << \"Model file \" << model_json_file << \" does not exist\";\n    throw std::runtime_error(\"Model file does not exist\");\n  }\n  LG << \"Loading the model from \" << model_json_file << std::endl;\n  net = Symbol::Load(model_json_file);\n}\n\n\n/*\n * The following function loads the model parameters.\n */\nvoid Predictor::LoadParameters(const std::string& model_parameters_file) {\n  if (!FileExists(model_parameters_file)) {\n    LG << \"Parameter file \" << model_parameters_file << \" does not exist\";\n    throw std::runtime_error(\"Model parameters does not exist\");\n  }\n  LG << \"Loading the model parameters from \" << model_parameters_file << std::endl;\n  std::map<std::string, NDArray> parameters;\n  NDArray::Load(model_parameters_file, 0, &parameters);\n  for (const auto &k : parameters) {\n    if (k.first.substr(0, 4) == \"aux:\") {\n      auto name = k.first.substr(4, k.first.size() - 4);\n      aux_map[name] = k.second.Copy(global_ctx);\n    }\n    if (k.first.substr(0, 4) == \"arg:\") {\n      auto name = k.first.substr(4, k.first.size() - 4);\n      args_map[name] = k.second.Copy(global_ctx);\n    }\n  }\n  /*WaitAll is need when we copy data between GPU and the main memory*/\n  NDArray::WaitAll();\n}\n\n\n/*\n * The following function loads the dictionary file.\n * The function constructs the word to index and index to word maps.\n * These maps will be used to represent words in the input sentence to their indices.\n * Ensure to use the same dictionary file that was used for training the network.\n */\nvoid Predictor::LoadDictionary(const std::string& input_dictionary) {\n  if (!FileExists(input_dictionary)) {\n    LG << \"Dictionary file \" << input_dictionary << \" does not exist\";\n    throw std::runtime_error(\"Dictionary file does not exist\");\n  }\n  LG << \"Loading the dictionary file.\";\n  std::ifstream fi(input_dictionary.c_str());\n  if (!fi.is_open()) {\n    std::cerr << \"Error opening dictionary file \" << input_dictionary << std::endl;\n    assert(false);\n  }\n\n  std::string line;\n  std::string word;\n  int index;\n  while (std::getline(fi, line)) {\n    std::istringstream stringline(line);\n    stringline >> word >> index;\n    wordToIndex[word] = index;\n  }\n  fi.close();\n}\n\n\n/*\n * The function returns the index associated with the word in the dictionary.\n * If the word is not present, the index representing \"<unk>\" is returned.\n * If the \"<unk>\" is not present then 0 is returned.\n */\nfloat Predictor::GetIndexForWord(const std::string& word) {\n  if (wordToIndex.find(word) == wordToIndex.end()) {\n    if (wordToIndex.find(\"<unk>\") == wordToIndex.end())\n      return 0;\n    else\n      return static_cast<float>(wordToIndex[\"<unk>\"]);\n  }\n  return static_cast<float>(wordToIndex[word]);\n}\n\n/*\n * The function populates the input vector with indices from the dictionary that\n * correspond to the words in the input string.\n * The function returns the number of words in the input line.\n */\nint Predictor::ConvertToIndexVector(const std::string& input, std::vector<float> *input_vector) {\n  std::istringstream input_string(input);\n  input_vector->clear();\n  const char delimiter = ' ';\n  std::string token;\n  size_t words = 0;\n  while (std::getline(input_string, token, delimiter) && (words <= input_vector->size())) {\n    input_vector->push_back(GetIndexForWord(token));\n    words++;\n  }\n  return words;\n}\n\n\n/*\n * The function returns the index at which the given symbol name will appear\n * in the output vector of NDArrays obtained after running the forward pass on the executor.\n */\nint Predictor::GetIndexForOutputSymbolName(const std::string& output_symbol_name) {\n  int index = 0;\n  for (const std::string op : net.ListOutputs()) {\n    if (op == output_symbol_name) {\n      return index;\n    } else {\n      index++;\n    }\n  }\n  throw std::runtime_error(\"The output symbol name can not be found\");\n}\n\n\n/*\n * The function finds the closest bucket for the given num_words in the input line.\n * If the exact bucket key exists, function returns that bucket key.\n * If the matching bucket key does not exist, function looks for the next bucket key\n * that is greater than given num_words.\n * If the next larger bucket does not exist, function returns the largest bucket key.\n */\nint Predictor::GetClosestBucketKey(int num_words) {\n  int closest_bucket_key = highest_bucket_key;\n\n  if (executor_buckets.lower_bound(num_words) != executor_buckets.end()) {\n    closest_bucket_key = executor_buckets.lower_bound(num_words)->first;\n  }\n  return closest_bucket_key;\n}\n\n\n/*\n * The following function runs the forward pass on the model for the given line.\n *\n */\nfloat Predictor::PredictSentimentForOneLine(const std::string& input_line) {\n  /*\n   * Initialize a vector of length equal to 'num_words' with index corresponding to <eos>.\n   * Convert the input string to a vector of indices that represent\n   * the words in the input string.\n   */\n  std::vector<float> index_vector(GetIndexForWord(\"<eos>\"));\n  int num_words = ConvertToIndexVector(input_line, &index_vector);\n  int bucket_key = GetClosestBucketKey(num_words);\n\n  /*\n   * The index_vector has size equal to num_words. The vector needs to be padded if\n   * the bucket_key is greater than num_words. The vector needs to be trimmed if\n   * the bucket_key is smaller than num_words.\n   */\n  index_vector.resize(bucket_key, GetIndexForWord(\"<eos>\"));\n\n  Executor* executor = executor_buckets[bucket_key];\n  executor->arg_dict()[\"data0\"].SyncCopyFromCPU(index_vector.data(), index_vector.size());\n  executor->arg_dict()[\"data1\"] = num_words;\n\n  // Run the forward pass.\n  executor->Forward(false);\n\n  /*\n   * The output is available in executor->outputs. It is a vector of\n   * NDArray. We need to find the index in that vector that\n   * corresponds to the output symbol \"sentimentnet0_hybridsequential0_dense0_fwd_output\".\n   */\n  const std::string output_symbol_name = \"sentimentnet0_hybridsequential0_dense0_fwd_output\";\n  int output_index = GetIndexForOutputSymbolName(output_symbol_name);\n  std::vector<NDArray> outputs = executor->outputs;\n  auto arrayout = executor->outputs[output_index].Copy(global_ctx);\n  /*\n   * We will run sigmoid operator to find out the sentiment score between\n   * 0 and 1 where 1 represents positive.\n   */\n  NDArray ret;\n  Operator(\"sigmoid\")(arrayout).Invoke(ret);\n  ret.WaitToRead();\n\n  return ret.At(0, 0);\n}\n\n\n/*\n * The function predicts the sentiment score for the input review.\n * The function splits the input review in lines (separated by '.').\n * It finds sentiment score for each line and computes the average.\n */\nfloat Predictor::PredictSentiment(const std::string& input_review) {\n  std::istringstream input_string(input_review);\n  int num_lines = 0;\n  float sentiment_score = 0.0f;\n\n  // Split the iput review in separate lines separated by '.'\n  const char delimiter = '.';\n  std::string line;\n  while (std::getline(input_string, line, delimiter)) {\n    // Predict the sentiment score for each line.\n    float score = PredictSentimentForOneLine(line);\n    LG << \"Input Line : [\" << line << \"] Score : \" << score;\n    sentiment_score += score;\n    num_lines++;\n  }\n\n  // Find the average sentiment score.\n  sentiment_score = sentiment_score / num_lines;\n  return sentiment_score;\n}\n\n\n/*\n * The destructor frees the executor and notifies MXNetEngine to shutdown.\n */\nPredictor::~Predictor() {\n  for (auto bucket : this->executor_buckets) {\n    Executor* executor = bucket.second;\n    delete executor;\n  }\n  MXNotifyShutdown();\n}\n\n\n/*\n * The function prints the usage information.\n */\nvoid printUsage() {\n    std::cout << \"Usage:\" << std::endl;\n    std::cout << \"sentiment_analysis_rnn \" << std::endl\n              << \"--input Input movie review. The review can be single line or multiline.\"\n              << \"e.g. \\\"This movie is the best.\\\" OR  \"\n              << \"\\\"This movie is the best. The direction is awesome.\\\" \" << std::endl\n              << \"[--gpu]  Specify this option if workflow needs to be run in gpu context \"\n              << std::endl\n              << \"If the review is multiline, the example predicts sentiment score for each line \"\n              << \"and the final score is the average of scores obtained for each line.\"\n              << std::endl;\n}\n\n\n/*\n * The function downloads the model files from s3 bucket.\n */\nvoid DownloadFiles(const std::vector<std::string> model_files) {\n  std::string wget_command(\"wget -nc \");\n  std::string s3_url(DEFAULT_S3_URL);\n  for (auto &file : model_files) {\n    std::ostringstream oss;\n    oss << wget_command << s3_url << file << \" -O \" << file;\n    int status = system(oss.str().c_str());\n    LG << \"Downloading \" << file << \" with status \" << status;\n  }\n  return;\n}\n\n\nint main(int argc, char** argv) {\n  std::string model_file_json = \"./sentiment_analysis-symbol.json\";\n  std::string model_file_params =\"./sentiment_analysis-0010.params\";\n  std::string input_dictionary = \"./sentiment_token_to_idx.txt\";\n  std::string input_review = \"This movie is the best\";\n  bool use_gpu = false;\n\n  int index = 1;\n  while (index < argc) {\n    if (strcmp(\"--input\", argv[index]) == 0) {\n      index++;\n      input_review = (index < argc ? argv[index]:input_review);\n    } else if (strcmp(\"--gpu\", argv[index]) == 0) {\n      use_gpu = true;\n    } else if (strcmp(\"--help\", argv[index]) == 0) {\n      printUsage();\n      return 0;\n    }\n    index++;\n  }\n\n\n  /*\n   * Download the trained RNN model file, param file and dictionary file.\n   * The dictionary file contains word to index mapping.\n   * Each line of the dictionary file contains a word and the unique index for that word separated\n   * by a space. For example:\n   * snippets 11172\n   * This dictionary file is created when the RNN model was trained with a particular dataset.\n   * Hence the dictionary file is specific to the dataset with which model was trained.\n   */\n  std::vector<std::string> files;\n  files.push_back(model_file_json);\n  files.push_back(model_file_params);\n  files.push_back(input_dictionary);\n\n  DownloadFiles(files);\n\n  std::vector<int> buckets(DEFAULT_BUCKET_KEYS,\n                           DEFAULT_BUCKET_KEYS + sizeof(DEFAULT_BUCKET_KEYS) / sizeof(int));\n\n  try {\n    // Initialize the predictor object\n    Predictor predict(model_file_json, model_file_params, input_dictionary, buckets, use_gpu);\n\n    // Run the forward pass to predict the sentiment score for the given review.\n    float sentiment_score = predict.PredictSentiment(input_review);\n    LG << \"The sentiment score between 0 and 1, (1 being positive)=\" << sentiment_score;\n  } catch (std::runtime_error &error) {\n    LG << MXGetLastError();\n    LG << \"Execution failed with ERROR: \" << error.what();\n    return 1;\n  } catch (...) {\n    /*\n     * If underlying MXNet code has thrown an exception the error message is\n     * accessible through MXGetLastError() function.\n     */\n    LG << \"Execution failed with following MXNet error\";\n    LG << MXGetLastError();\n    return 1;\n  }\n  return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/inference/unit_test_imagenet_inference.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset -ex\n# create ./model directory if not existed\nif [ ! -d model ]; then\n    mkdir -p model\nfi\n# create ./data directory if not existed\nif [ ! -d data ]; then\n    mkdir -p data\nfi\n# Downloading the data and model if not existed\nmodel_file=./model/Inception-BN-symbol.json\nparams_file=./model/Inception-BN-0126.params\nif [ ! -f ${model_file} ] || [ ! -f ${params_file} ]; then\n    wget -nc http://data.mxnet.io/models/imagenet/inception-bn.tar.gz\n    tar -xvzf inception-bn.tar.gz -C model\nfi\ncd model\nwget -nc https://raw.githubusercontent.com/dmlc/gluon-cv/master/gluoncv/model_zoo/quantized/resnet50_v1_int8-symbol.json\ncd ../data\nwget -nc http://data.mxnet.io/data/val_256_q90.rec\ncd ..\n\n# Running inference on imagenet.\nif [ \"$(uname)\" == \"Darwin\" ]; then\n    echo \">>> INFO: FP32 real data\"\n    DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}:../../../build ./imagenet_inference --symbol_file \"./model/Inception-BN-symbol.json\" --params_file \"./model/Inception-BN-0126.params\" --dataset \"./data/val_256_q90.rec\" --rgb_mean \"123.68 116.779 103.939\" --batch_size 1 --num_skipped_batches 50 --num_inference_batches 500\n\n    echo \">>> INFO: FP32 dummy data\"\n    DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}:../../../build ./imagenet_inference --symbol_file \"./model/Inception-BN-symbol.json\" --batch_size 1 --num_inference_batches 500 --benchmark\nelse\n    echo \">>> INFO: FP32 real data\"\n    LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:../../../build ./imagenet_inference --symbol_file \"./model/Inception-BN-symbol.json\" --params_file \"./model/Inception-BN-0126.params\" --dataset \"./data/val_256_q90.rec\" --rgb_mean \"123.68 116.779 103.939\" --batch_size 1 --num_skipped_batches 50 --num_inference_batches 500\n\n    echo \">>> INFO: FP32 dummy data\"\n    LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:../../../build ./imagenet_inference --symbol_file \"./model/Inception-BN-symbol.json\" --batch_size 1 --num_inference_batches 500 --benchmark\n\n    lib_name=$(ls -a ../../../build | grep -oE 'onednn' | tail -1)\n    if [[ -n ${lib_name} ]] && [[ 'onednn' =~ ${lib_name} ]]; then\n        echo \">>> INFO: INT8 dummy data\"\n        LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:../../../build ./imagenet_inference --symbol_file \"./model/resnet50_v1_int8-symbol.json\" --batch_size 1 --num_inference_batches 500 --benchmark\n    else\n        echo \"Skipped INT8 test because onednn was not found which is required for running inference with quantized models.\"\n    fi\nfi\n"
  },
  {
    "path": "cpp-package/example/inference/unit_test_sentiment_analysis_rnn.sh",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nfunction compare_range() {\n    perl -e \"{if($1>$2 && $1<=$3){print 1} else {print 0}}\"\n}\n\nset -e # exit on the first error\nexport EXE_NAME=\"sentiment_analysis_rnn\"\n\n# Running the example with a movie review.\nif [ \"$(uname)\" == \"Darwin\" ]; then\n    DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}:../../../build ./${EXE_NAME}  --input \"This movie is the best.\" 2&> ${EXE_NAME}.log\nelse\n    LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:../../../build ./${EXE_NAME}  --input \"This movie is the best.\" 2&> ${EXE_NAME}.log\nfi\nresult=`grep \"The sentiment score between 0 and 1.*\\=\" ${EXE_NAME}.log | cut -d '=' -f2`\nlower_bound=0.8\nupper_bound=0.99\nif [ $(compare_range $result $lower_bound $upper_bound) == 1 ];\nthen\n    echo \"PASS: ${EXE_NAME} correctly predicted the sentiment with score = $result\"\n    exit 0\nelse\n    echo \"FAIL: ${EXE_NAME} FAILED to predict the sentiment with score = $result\"\n    exit 1\nfi"
  },
  {
    "path": "cpp-package/example/lenet.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n */\n#include <fstream>\n#include <map>\n#include <string>\n#include <vector>\n#include <cstdlib>\n#include \"mxnet-cpp/MxNetCpp.h\"\n#include \"utils.h\"\n\nusing namespace mxnet::cpp;\n\nclass Lenet {\n public:\n  Lenet()\n      : ctx_cpu(Context(DeviceType::kCPU, 0)),\n#if !MXNET_USE_CUDA\n        ctx_dev(Context(DeviceType::kCPU, 0))\n#else\n        ctx_dev(Context(DeviceType::kGPU, 0))\n#endif\n        {}\n\n  void Run(int max_epoch) {\n    /*\n     * LeCun, Yann, Leon Bottou, Yoshua Bengio, and Patrick Haffner.\n     * \"Gradient-based learning applied to document recognition.\"\n     * Proceedings of the IEEE (1998)\n     * */\n\n    /*define the symbolic net*/\n    Symbol data = Symbol::Variable(\"data\");\n    Symbol data_label = Symbol::Variable(\"data_label\");\n    Symbol conv1_w(\"conv1_w\"), conv1_b(\"conv1_b\");\n    Symbol conv2_w(\"conv2_w\"), conv2_b(\"conv2_b\");\n    Symbol conv3_w(\"conv3_w\"), conv3_b(\"conv3_b\");\n    Symbol fc1_w(\"fc1_w\"), fc1_b(\"fc1_b\");\n    Symbol fc2_w(\"fc2_w\"), fc2_b(\"fc2_b\");\n\n    Symbol conv1 =\n        Convolution(\"conv1\", data, conv1_w, conv1_b, Shape(5, 5), 20);\n    Symbol tanh1 = Activation(\"tanh1\", conv1, ActivationActType::kTanh);\n    Symbol pool1 = Pooling(\"pool1\", tanh1, Shape(2, 2), PoolingPoolType::kMax,\n      false, false, PoolingPoolingConvention::kValid, Shape(2, 2));\n\n    Symbol conv2 = Convolution(\"conv2\", pool1, conv2_w, conv2_b,\n      Shape(5, 5), 50);\n    Symbol tanh2 = Activation(\"tanh2\", conv2, ActivationActType::kTanh);\n    Symbol pool2 = Pooling(\"pool2\", tanh2, Shape(2, 2), PoolingPoolType::kMax,\n      false, false, PoolingPoolingConvention::kValid, Shape(2, 2));\n\n    Symbol conv3 = Convolution(\"conv3\", pool2, conv3_w, conv3_b,\n      Shape(2, 2), 500);\n    Symbol tanh3 = Activation(\"tanh3\", conv3, ActivationActType::kTanh);\n    Symbol pool3 = Pooling(\"pool3\", tanh3, Shape(2, 2), PoolingPoolType::kMax,\n      false, false, PoolingPoolingConvention::kValid, Shape(1, 1));\n\n    Symbol flatten = Flatten(\"flatten\", pool3);\n    Symbol fc1 = FullyConnected(\"fc1\", flatten, fc1_w, fc1_b, 500);\n    Symbol tanh4 = Activation(\"tanh4\", fc1, ActivationActType::kTanh);\n    Symbol fc2 = FullyConnected(\"fc2\", tanh4, fc2_w, fc2_b, 10);\n\n    Symbol lenet = SoftmaxOutput(\"softmax\", fc2, data_label);\n\n    for (auto s : lenet.ListArguments()) {\n      LG << s;\n    }\n\n    /*setup basic configs*/\n    int val_fold = 1;\n    int W = 28;\n    int H = 28;\n    int batch_size = 42;\n    float learning_rate = 1e-4;\n    float weight_decay = 1e-4;\n\n    /*prepare the data*/\n    std::vector<float> data_vec, label_vec;\n    size_t data_count = GetData(&data_vec, &label_vec);\n    const float *dptr = data_vec.data();\n    const float *lptr = label_vec.data();\n    NDArray data_array = NDArray(Shape(data_count, 1, W, H), ctx_cpu,\n                                 false);  // store in main memory, and copy to\n    // device memory while training\n    NDArray label_array =\n      NDArray(Shape(data_count), ctx_cpu,\n                false);  // it's also ok if just store them all in device memory\n    data_array.SyncCopyFromCPU(dptr, data_count * W * H);\n    label_array.SyncCopyFromCPU(lptr, data_count);\n    data_array.WaitToRead();\n    label_array.WaitToRead();\n\n    size_t train_num = data_count * (1 - val_fold / 10.0);\n    train_data = data_array.Slice(0, train_num);\n    train_label = label_array.Slice(0, train_num);\n    val_data = data_array.Slice(train_num, data_count);\n    val_label = label_array.Slice(train_num, data_count);\n\n    LG << \"here read fin\";\n\n    /*init some of the args*/\n    // map<string, NDArray> args_map;\n    args_map[\"data\"] = data_array.Slice(0, batch_size).Copy(ctx_dev);\n    args_map[\"data_label\"] = label_array.Slice(0, batch_size).Copy(ctx_dev);\n    NDArray::WaitAll();\n\n    LG << \"here slice fin\";\n    /*\n     * we can also feed in some of the args other than the input all by\n     * ourselves,\n     * fc2-w , fc1-b for example:\n     * */\n    // args_map[\"fc2_w\"] =\n    // NDArray(mshadow::Shape2(500, 4 * 4 * 50), ctx_dev, false);\n    // NDArray::SampleGaussian(0, 1, &args_map[\"fc2_w\"]);\n    // args_map[\"fc1_b\"] = NDArray(mshadow::Shape1(10), ctx_dev, false);\n    // args_map[\"fc1_b\"] = 0;\n\n    lenet.InferArgsMap(ctx_dev, &args_map, args_map);\n    Optimizer* opt = OptimizerRegistry::Find(\"ccsgd\");\n    opt->SetParam(\"momentum\", 0.9)\n       ->SetParam(\"rescale_grad\", 1.0)\n       ->SetParam(\"clip_gradient\", 10)\n       ->SetParam(\"lr\", learning_rate)\n       ->SetParam(\"wd\", weight_decay);\n\n    Executor *exe = lenet.SimpleBind(ctx_dev, args_map);\n    auto arg_names = lenet.ListArguments();\n\n    for (int ITER = 0; ITER < max_epoch; ++ITER) {\n      size_t start_index = 0;\n      while (start_index < train_num) {\n        if (start_index + batch_size > train_num) {\n          start_index = train_num - batch_size;\n        }\n        args_map[\"data\"] =\n            train_data.Slice(start_index, start_index + batch_size)\n                .Copy(ctx_dev);\n        args_map[\"data_label\"] =\n            train_label.Slice(start_index, start_index + batch_size)\n                .Copy(ctx_dev);\n        start_index += batch_size;\n        NDArray::WaitAll();\n\n        exe->Forward(true);\n        exe->Backward();\n        // Update parameters\n        for (size_t i = 0; i < arg_names.size(); ++i) {\n          if (arg_names[i] == \"data\" || arg_names[i] == \"data_label\") continue;\n          opt->Update(i, exe->arg_arrays[i], exe->grad_arrays[i]);\n        }\n      }\n\n      LG << \"Iter \" << ITER\n         << \", accuracy: \" << ValAccuracy(batch_size * 10, lenet);\n    }\n    delete exe;\n    delete opt;\n  }\n\n private:\n  Context ctx_cpu;\n  Context ctx_dev;\n  std::map<std::string, NDArray> args_map;\n  NDArray train_data;\n  NDArray train_label;\n  NDArray val_data;\n  NDArray val_label;\n\n  size_t GetData(std::vector<float> *data, std::vector<float> *label) {\n    const char *train_data_path = \"./data/mnist_data/mnist_train.csv\";\n    std::ifstream inf(train_data_path);\n    std::string line;\n    inf >> line;  // ignore the header\n    size_t _N = 0;\n    while (inf >> line) {\n      for (auto &c : line) c = (c == ',') ? ' ' : c;\n      std::stringstream ss;\n      ss << line;\n      float _data;\n      ss >> _data;\n      label->push_back(_data);\n      while (ss >> _data) data->push_back(_data / 256.0);\n      _N++;\n    }\n    inf.close();\n    return _N;\n  }\n\n  float ValAccuracy(int batch_size, Symbol lenet) {\n    size_t val_num = val_data.GetShape()[0];\n\n    size_t correct_count = 0;\n    size_t all_count = 0;\n\n    size_t start_index = 0;\n    while (start_index < val_num) {\n      if (start_index + batch_size > val_num) {\n        start_index = val_num - batch_size;\n      }\n      args_map[\"data\"] =\n          val_data.Slice(start_index, start_index + batch_size).Copy(ctx_dev);\n      args_map[\"data_label\"] =\n          val_label.Slice(start_index, start_index + batch_size).Copy(ctx_dev);\n      start_index += batch_size;\n      NDArray::WaitAll();\n\n      Executor *exe = lenet.SimpleBind(ctx_dev, args_map);\n      exe->Forward(false);\n\n      const auto &out = exe->outputs;\n      NDArray out_cpu = out[0].Copy(ctx_cpu);\n      NDArray label_cpu =\n          val_label.Slice(start_index - batch_size, start_index).Copy(ctx_cpu);\n\n      NDArray::WaitAll();\n\n      const mx_float *dptr_out = out_cpu.GetData();\n      const mx_float *dptr_label = label_cpu.GetData();\n      for (int i = 0; i < batch_size; ++i) {\n        float label = dptr_label[i];\n        int cat_num = out_cpu.GetShape()[1];\n        float p_label = 0, max_p = dptr_out[i * cat_num];\n        for (int j = 0; j < cat_num; ++j) {\n          float p = dptr_out[i * cat_num + j];\n          if (max_p < p) {\n            p_label = j;\n            max_p = p;\n          }\n        }\n        if (label == p_label) correct_count++;\n      }\n      all_count += batch_size;\n\n      delete exe;\n    }\n    return correct_count * 1.0 / all_count;\n  }\n};\n\nint main(int argc, char const *argv[]) {\n  TRY\n  Lenet lenet;\n  lenet.Run(argc > 1 ? strtol(argv[1], nullptr, 10) : 100000);\n  MXNotifyShutdown();\n  CATCH\n  return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/lenet_with_mxdataiter.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n */\n#include <map>\n#include <string>\n#include <vector>\n#include <fstream>\n#include <chrono>\n#include <cstdlib>\n#include \"utils.h\"\n#include \"mxnet-cpp/MxNetCpp.h\"\n\nusing namespace mxnet::cpp;\n\nSymbol LenetSymbol() {\n  /*\n   * LeCun, Yann, Leon Bottou, Yoshua Bengio, and Patrick Haffner.\n   * \"Gradient-based learning applied to document recognition.\"\n   * Proceedings of the IEEE (1998)\n   * */\n\n  /*define the symbolic net*/\n  Symbol data = Symbol::Variable(\"data\");\n  Symbol data_label = Symbol::Variable(\"data_label\");\n  Symbol conv1_w(\"conv1_w\"), conv1_b(\"conv1_b\");\n  Symbol conv2_w(\"conv2_w\"), conv2_b(\"conv2_b\");\n  Symbol conv3_w(\"conv3_w\"), conv3_b(\"conv3_b\");\n  Symbol fc1_w(\"fc1_w\"), fc1_b(\"fc1_b\");\n  Symbol fc2_w(\"fc2_w\"), fc2_b(\"fc2_b\");\n\n  Symbol conv1 = Convolution(\"conv1\", data, conv1_w, conv1_b, Shape(5, 5), 20);\n  Symbol tanh1 = Activation(\"tanh1\", conv1, ActivationActType::kTanh);\n  Symbol pool1 = Pooling(\"pool1\", tanh1, Shape(2, 2), PoolingPoolType::kMax,\n      false, false, PoolingPoolingConvention::kValid, Shape(2, 2));\n\n  Symbol conv2 = Convolution(\"conv2\", pool1, conv2_w, conv2_b, Shape(5, 5), 50);\n  Symbol tanh2 = Activation(\"tanh2\", conv2, ActivationActType::kTanh);\n  Symbol pool2 = Pooling(\"pool2\", tanh2, Shape(2, 2), PoolingPoolType::kMax,\n      false, false, PoolingPoolingConvention::kValid, Shape(2, 2));\n\n  Symbol flatten = Flatten(\"flatten\", pool2);\n  Symbol fc1 = FullyConnected(\"fc1\", flatten, fc1_w, fc1_b, 500);\n  Symbol tanh3 = Activation(\"tanh3\", fc1, ActivationActType::kTanh);\n  Symbol fc2 = FullyConnected(\"fc2\", tanh3, fc2_w, fc2_b, 10);\n\n  Symbol lenet = SoftmaxOutput(\"softmax\", fc2, data_label);\n\n  return lenet;\n}\n\nNDArray ResizeInput(NDArray data, const Shape new_shape) {\n  NDArray pic = data.Reshape(Shape(0, 1, 28, 28));\n  NDArray output;\n  Operator(\"_contrib_BilinearResize2D\")\n    .SetParam(\"height\", new_shape[2])\n    .SetParam(\"width\", new_shape[3])\n    (pic).Invoke(output);\n  return output;\n}\n\nint main(int argc, char const *argv[]) {\n  /*setup basic configs*/\n  int W = 28;\n  int H = 28;\n  int batch_size = 128;\n  int max_epoch = argc > 1 ? strtol(argv[1], nullptr, 10) : 100;\n  float learning_rate = 1e-4;\n  float weight_decay = 1e-4;\n\n  auto dev_ctx = Context::cpu();\n  int num_gpu;\n  MXGetGPUCount(&num_gpu);\n#if MXNET_USE_CUDA\n  if (num_gpu > 0) {\n    dev_ctx = Context::gpu();\n  }\n#endif\n\n  TRY\n  auto lenet = LenetSymbol();\n  std::map<std::string, NDArray> args_map;\n\n  const Shape data_shape = Shape(batch_size, 1, H, W),\n              label_shape = Shape(batch_size);\n  args_map[\"data\"] = NDArray(data_shape, dev_ctx);\n  args_map[\"data_label\"] = NDArray(label_shape, dev_ctx);\n  lenet.InferArgsMap(dev_ctx, &args_map, args_map);\n\n  args_map[\"fc1_w\"] = NDArray(Shape(500, 4 * 4 * 50), dev_ctx);\n  NDArray::SampleGaussian(0, 1, &args_map[\"fc1_w\"]);\n  args_map[\"fc2_b\"] = NDArray(Shape(10), dev_ctx);\n  args_map[\"fc2_b\"] = 0;\n\n  std::vector<std::string> data_files = { \"./data/mnist_data/train-images-idx3-ubyte\",\n                                          \"./data/mnist_data/train-labels-idx1-ubyte\",\n                                          \"./data/mnist_data/t10k-images-idx3-ubyte\",\n                                          \"./data/mnist_data/t10k-labels-idx1-ubyte\"\n                                        };\n\n  auto train_iter =  MXDataIter(\"MNISTIter\");\n  if (!setDataIter(&train_iter, \"Train\", data_files, batch_size)) {\n    return 1;\n  }\n\n  auto val_iter = MXDataIter(\"MNISTIter\");\n  if (!setDataIter(&val_iter, \"Label\", data_files, batch_size)) {\n    return 1;\n  }\n\n  Optimizer* opt = OptimizerRegistry::Find(\"sgd\");\n  opt->SetParam(\"momentum\", 0.9)\n     ->SetParam(\"rescale_grad\", 1.0)\n     ->SetParam(\"clip_gradient\", 10)\n     ->SetParam(\"lr\", learning_rate)\n     ->SetParam(\"wd\", weight_decay);\n\n\n  auto *exec = lenet.SimpleBind(dev_ctx, args_map);\n  auto arg_names = lenet.ListArguments();\n\n  // Create metrics\n  Accuracy train_acc, val_acc;\n\n  for (int iter = 0; iter < max_epoch; ++iter) {\n      int samples = 0;\n      train_iter.Reset();\n      train_acc.Reset();\n\n      auto tic = std::chrono::system_clock::now();\n\n     while (train_iter.Next()) {\n      samples += batch_size;\n      auto data_batch = train_iter.GetDataBatch();\n\n      ResizeInput(data_batch.data, data_shape).CopyTo(&args_map[\"data\"]);\n      data_batch.label.CopyTo(&args_map[\"data_label\"]);\n      NDArray::WaitAll();\n\n      // Compute gradients\n      exec->Forward(true);\n      exec->Backward();\n\n      // Update parameters\n      for (size_t i = 0; i < arg_names.size(); ++i) {\n        if (arg_names[i] == \"data\" || arg_names[i] == \"data_label\") continue;\n        opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]);\n      }\n\n      // Update metric\n      train_acc.Update(data_batch.label, exec->outputs[0]);\n    }\n\n     // one epoch of training is finished\n     auto toc = std::chrono::system_clock::now();\n     float duration = std::chrono::duration_cast<std::chrono::milliseconds>\n                      (toc - tic).count() / 1000.0;\n     LG << \"Epoch[\" << iter << \"] \" << samples / duration \\\n         << \" samples/sec \" << \"Train-Accuracy=\" << train_acc.Get();;\n\n      val_iter.Reset();\n      val_acc.Reset();\n\n    Accuracy acu;\n    val_iter.Reset();\n    while (val_iter.Next()) {\n      auto data_batch = val_iter.GetDataBatch();\n      ResizeInput(data_batch.data, data_shape).CopyTo(&args_map[\"data\"]);\n      data_batch.label.CopyTo(&args_map[\"data_label\"]);\n      NDArray::WaitAll();\n\n      // Only forward pass is enough as no gradient is needed when evaluating\n      exec->Forward(false);\n      NDArray::WaitAll();\n      acu.Update(data_batch.label, exec->outputs[0]);\n      val_acc.Update(data_batch.label, exec->outputs[0]);\n    }\n    LG << \"Epoch[\" << iter << \"] Val-Accuracy=\" << val_acc.Get();\n  }\n\n  delete exec;\n  delete opt;\n  MXNotifyShutdown();\n  CATCH\n  return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/mlp.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n */\n\n#include <iostream>\n#include <vector>\n#include <string>\n#include \"mxnet-cpp/MxNetCpp.h\"\n#include \"utils.h\"\n\nusing namespace mxnet::cpp;\n\n/*\n * In this example,\n * we make by hand some data in 10 classes with some pattern\n * and try to use MLP to recognize the pattern.\n */\n\nvoid OutputAccuracy(mx_float* pred, mx_float* target) {\n  int right = 0;\n  for (int i = 0; i < 128; ++i) {\n    float mx_p = pred[i * 10 + 0];\n    float p_y = 0;\n    for (int j = 0; j < 10; ++j) {\n      if (pred[i * 10 + j] > mx_p) {\n        mx_p = pred[i * 10 + j];\n        p_y = j;\n      }\n    }\n    if (p_y == target[i]) right++;\n  }\n  std::cout << \"Accuracy: \" << right / 128.0 << std::endl;\n}\n\nvoid MLP(int max_epoch) {\n  auto sym_x = Symbol::Variable(\"X\");\n  auto sym_label = Symbol::Variable(\"label\");\n\n  const int nLayers = 2;\n  std::vector<int> layerSizes({512, 10});\n  std::vector<Symbol> weights(nLayers);\n  std::vector<Symbol> biases(nLayers);\n  std::vector<Symbol> outputs(nLayers);\n\n  Symbol null_sym;\n  for (int i = 0; i < nLayers; i++) {\n    std::string istr = std::to_string(i);\n    weights[i] = Symbol::Variable(std::string(\"w\") + istr);\n    biases[i] = Symbol::Variable(std::string(\"b\") + istr);\n    Symbol fc = FullyConnected(std::string(\"fc\") + istr,\n      i == 0? sym_x : outputs[i-1],\n      weights[i], biases[i], layerSizes[i]);\n    outputs[i] = LeakyReLU(std::string(\"act\") + istr, fc, null_sym, LeakyReLUActType::kLeaky);\n  }\n  auto sym_out = SoftmaxOutput(\"softmax\", outputs[nLayers - 1], sym_label);\n\n  Context ctx_dev(DeviceType::kCPU, 0);\n\n  NDArray array_x(Shape(128, 28), ctx_dev, false);\n  NDArray array_y(Shape(128), ctx_dev, false);\n\n  mx_float* aptr_x = new mx_float[128 * 28];\n  mx_float* aptr_y = new mx_float[128];\n\n  // we make the data by hand, in 10 classes, with some pattern\n  for (int i = 0; i < 128; i++) {\n    for (int j = 0; j < 28; j++) {\n      aptr_x[i * 28 + j] = i % 10 * 1.0f;\n    }\n    aptr_y[i] = i % 10;\n  }\n  array_x.SyncCopyFromCPU(aptr_x, 128 * 28);\n  array_x.WaitToRead();\n  array_y.SyncCopyFromCPU(aptr_y, 128);\n  array_y.WaitToRead();\n\n  // init the parameters\n  NDArray array_w_1(Shape(512, 28), ctx_dev, false);\n  NDArray array_b_1(Shape(512), ctx_dev, false);\n  NDArray array_w_2(Shape(10, 512), ctx_dev, false);\n  NDArray array_b_2(Shape(10), ctx_dev, false);\n\n  // the parameters should be initialized in some kind of distribution,\n  // so it learns fast\n  // but here just give a const value by hand\n  array_w_1 = 0.5f;\n  array_b_1 = 0.0f;\n  array_w_2 = 0.5f;\n  array_b_2 = 0.0f;\n\n  // the grads\n  NDArray array_w_1_g(Shape(512, 28), ctx_dev, false);\n  NDArray array_b_1_g(Shape(512), ctx_dev, false);\n  NDArray array_w_2_g(Shape(10, 512), ctx_dev, false);\n  NDArray array_b_2_g(Shape(10), ctx_dev, false);\n\n  // Bind the symolic network with the ndarray\n  // all the input args\n  std::vector<NDArray> in_args;\n  in_args.push_back(array_x);\n  in_args.push_back(array_w_1);\n  in_args.push_back(array_b_1);\n  in_args.push_back(array_w_2);\n  in_args.push_back(array_b_2);\n  in_args.push_back(array_y);\n  // all the grads\n  std::vector<NDArray> arg_grad_store;\n  arg_grad_store.push_back(NDArray());  // we don't need the grad of the input\n  arg_grad_store.push_back(array_w_1_g);\n  arg_grad_store.push_back(array_b_1_g);\n  arg_grad_store.push_back(array_w_2_g);\n  arg_grad_store.push_back(array_b_2_g);\n  arg_grad_store.push_back(\n      NDArray());  // neither do we need the grad of the loss\n  // how to handle the grad\n  std::vector<OpReqType> grad_req_type;\n  grad_req_type.push_back(kNullOp);\n  grad_req_type.push_back(kWriteTo);\n  grad_req_type.push_back(kWriteTo);\n  grad_req_type.push_back(kWriteTo);\n  grad_req_type.push_back(kWriteTo);\n  grad_req_type.push_back(kNullOp);\n  std::vector<NDArray> aux_states;\n\n  std::cout << \"make the Executor\" << std::endl;\n  Executor* exe = new Executor(sym_out, ctx_dev, in_args, arg_grad_store,\n                               grad_req_type, aux_states);\n\n  std::cout << \"Training\" << std::endl;\n  mx_float learning_rate = 0.0001;\n  for (int epoch_num = 0; epoch_num < max_epoch; ++epoch_num) {\n    exe->Forward(true);\n    // print accuracy every 100 epoch\n    if (epoch_num % 100 == 0) {\n      std::cout << \"epoch \" << epoch_num << std::endl;\n      std::vector<NDArray>& out = exe->outputs;\n      float* cptr = new float[128 * 10];\n      out[0].SyncCopyToCPU(cptr, 128 * 10);\n      NDArray::WaitAll();\n      OutputAccuracy(cptr, aptr_y);\n      delete[] cptr;\n    }\n\n    // update the parameters\n    exe->Backward();\n    for (int i = 1; i < 5; ++i) {\n      in_args[i] -= arg_grad_store[i] * learning_rate;\n    }\n    NDArray::WaitAll();\n  }\n\n  delete exe;\n  delete[] aptr_x;\n  delete[] aptr_y;\n}\n\nint main(int argc, char** argv) {\n  int max_epoch = argc > 1 ? strtol(argv[1], nullptr, 10) : 15000;\n  TRY\n  MLP(max_epoch);\n  MXNotifyShutdown();\n  CATCH\n  return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/mlp_cpu.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * Xin Li yakumolx@gmail.com\n */\n#include <chrono>\n#include \"utils.h\"\n#include \"mxnet-cpp/MxNetCpp.h\"\n\nusing namespace mxnet::cpp;\n\nSymbol mlp(const std::vector<int> &layers) {\n  auto x = Symbol::Variable(\"X\");\n  auto label = Symbol::Variable(\"label\");\n\n  std::vector<Symbol> weights(layers.size());\n  std::vector<Symbol> biases(layers.size());\n  std::vector<Symbol> outputs(layers.size());\n\n  for (size_t i = 0; i < layers.size(); ++i) {\n    weights[i] = Symbol::Variable(\"w\" + std::to_string(i));\n    biases[i] = Symbol::Variable(\"b\" + std::to_string(i));\n    Symbol fc = FullyConnected(\n      i == 0? x : outputs[i-1],  // data\n      weights[i],\n      biases[i],\n      layers[i]);\n    outputs[i] = i == layers.size()-1 ? fc : Activation(fc, ActivationActType::kRelu);\n  }\n\n  return SoftmaxOutput(outputs.back(), label);\n}\n\nint main(int argc, char** argv) {\n  const int image_size = 28;\n  const std::vector<int> layers{128, 64, 10};\n  const int batch_size = 100;\n  const int max_epoch = 10;\n  const float learning_rate = 0.1;\n  const float weight_decay = 1e-2;\n\n  std::vector<std::string> data_files = { \"./data/mnist_data/train-images-idx3-ubyte\",\n                                          \"./data/mnist_data/train-labels-idx1-ubyte\",\n                                          \"./data/mnist_data/t10k-images-idx3-ubyte\",\n                                          \"./data/mnist_data/t10k-labels-idx1-ubyte\"\n                                        };\n\n  auto train_iter =  MXDataIter(\"MNISTIter\");\n  if (!setDataIter(&train_iter, \"Train\", data_files, batch_size)) {\n    return 1;\n  }\n\n  auto val_iter = MXDataIter(\"MNISTIter\");\n  if (!setDataIter(&val_iter, \"Label\", data_files, batch_size)) {\n    return 1;\n  }\n\n  TRY\n  auto net = mlp(layers);\n\n  Context ctx = Context::cpu();  // Use CPU for training\n\n  std::map<std::string, NDArray> args;\n  args[\"X\"] = NDArray(Shape(batch_size, image_size*image_size), ctx);\n  args[\"label\"] = NDArray(Shape(batch_size), ctx);\n  // Let MXNet infer shapes other parameters such as weights\n  net.InferArgsMap(ctx, &args, args);\n\n  // Initialize all parameters with uniform distribution U(-0.01, 0.01)\n  auto initializer = Uniform(0.01);\n  for (auto& arg : args) {\n    // arg.first is parameter name, and arg.second is the value\n    initializer(arg.first, &arg.second);\n  }\n\n  // Create sgd optimizer\n  Optimizer* opt = OptimizerRegistry::Find(\"sgd\");\n  opt->SetParam(\"rescale_grad\", 1.0/batch_size)\n     ->SetParam(\"lr\", learning_rate)\n     ->SetParam(\"wd\", weight_decay);\n\n  // Create executor by binding parameters to the model\n  auto *exec = net.SimpleBind(ctx, args);\n  auto arg_names = net.ListArguments();\n\n  // Start training\n  for (int iter = 0; iter < max_epoch; ++iter) {\n    int samples = 0;\n    train_iter.Reset();\n\n    auto tic = std::chrono::system_clock::now();\n    while (train_iter.Next()) {\n      samples += batch_size;\n      auto data_batch = train_iter.GetDataBatch();\n      // Set data and label\n      data_batch.data.CopyTo(&args[\"X\"]);\n      data_batch.label.CopyTo(&args[\"label\"]);\n\n      // Compute gradients\n      exec->Forward(true);\n      exec->Backward();\n      // Update parameters\n      for (size_t i = 0; i < arg_names.size(); ++i) {\n        if (arg_names[i] == \"X\" || arg_names[i] == \"label\") continue;\n        opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]);\n      }\n    }\n    auto toc = std::chrono::system_clock::now();\n\n    Accuracy acc;\n    val_iter.Reset();\n    while (val_iter.Next()) {\n      auto data_batch = val_iter.GetDataBatch();\n      data_batch.data.CopyTo(&args[\"X\"]);\n      data_batch.label.CopyTo(&args[\"label\"]);\n      // Forward pass is enough as no gradient is needed when evaluating\n      exec->Forward(false);\n      acc.Update(data_batch.label, exec->outputs[0]);\n    }\n    float duration = std::chrono::duration_cast<std::chrono::milliseconds>\n                     (toc - tic).count() / 1000.0;\n    LG << \"Epoch: \" << iter << \" \" << samples/duration << \" samples/sec Accuracy: \" << acc.Get();\n  }\n\n  delete exec;\n  delete opt;\n  MXNotifyShutdown();\n  CATCH\n  return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/mlp_csv.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*\n * Example: mlp_csv\n * Description:\n * The following example demonstrates how to use CSVIter. This example creates\n * mlp (multi-layer perceptron) model and trains the MNIST data which is in\n * CSV format.\n */\n#include <chrono>\n#include <string>\n#include \"utils.h\"\n#include \"mxnet-cpp/MxNetCpp.h\"\n\nusing namespace mxnet::cpp;\n\n/*\n * Implementing the mlp symbol with given hidden units configuration.\n */\nSymbol mlp(const std::vector<int> &hidden_units) {\n    auto data = Symbol::Variable(\"data\");\n    auto label = Symbol::Variable(\"label\");\n\n    std::vector<Symbol> weights(hidden_units.size());\n    std::vector<Symbol> biases(hidden_units.size());\n    std::vector<Symbol> outputs(hidden_units.size());\n\n    for (size_t i = 0; i < hidden_units.size(); ++i) {\n        weights[i] = Symbol::Variable(\"w\" + std::to_string(i));\n        biases[i] = Symbol::Variable(\"b\" + std::to_string(i));\n        Symbol fc = FullyConnected(\n                                   i == 0? data : outputs[i-1],  // data\n                                   weights[i],\n                                   biases[i],\n                                   hidden_units[i]);\n        outputs[i] = i == hidden_units.size()-1 ? fc : Activation(fc, ActivationActType::kRelu);\n    }\n    return SoftmaxOutput(outputs.back(), label);\n}\n\n/*\n * Convert the input string of number of hidden units into the vector of integers.\n */\nstd::vector<int> getLayers(const std::string& hidden_units_string) {\n    std::vector<int> hidden_units;\n    char *pNext;\n    int num_unit = strtol(hidden_units_string.c_str(), &pNext, 10);\n    hidden_units.push_back(num_unit);\n    while (*pNext) {\n        num_unit = strtol(pNext, &pNext, 10);\n        hidden_units.push_back(num_unit);\n    }\n    return hidden_units;\n}\n\nvoid printUsage() {\n    std::cout << \"Usage:\" << std::endl;\n    std::cout << \"mlp_csv --train mnist_training_set.csv --test mnist_test_set.csv --epochs 10 \"\n    << \"--batch_size 100 --hidden_units \\\"128 64 64\\\" --gpu\" << std::endl;\n    std::cout << \"The example uses mnist data in CSV format. The MNIST data in CSV format assumes \"\n    << \"the column 0 to be label and the rest 784 column to be data.\" << std::endl;\n    std::cout << \"By default, the example uses 'cpu' context. If '--gpu' is specified, \"\n    << \"program uses 'gpu' context.\" <<std::endl;\n}\n\nint main(int argc, char** argv) {\n    const int image_size = 28;\n    const int num_mnist_features = image_size * image_size;\n    int batch_size = 100;\n    int max_epoch = 10;\n    const float learning_rate = 0.1;\n    const float weight_decay = 1e-2;\n    bool isGpu = false;\n\n    std::string training_set;\n    std::string test_set;\n    std::string hidden_units_string;\n    int index = 1;\n    while (index < argc) {\n        if (strcmp(\"--train\", argv[index]) == 0) {\n            index++;\n            training_set = argv[index];\n        } else if (strcmp(\"--test\", argv[index]) == 0) {\n            index++;\n            test_set = argv[index];\n        } else if (strcmp(\"--epochs\", argv[index]) == 0) {\n            index++;\n            max_epoch = strtol(argv[index], nullptr, 10);\n        } else if (strcmp(\"--batch_size\", argv[index]) == 0) {\n            index++;\n            batch_size = strtol(argv[index], nullptr, 10);\n        } else if (strcmp(\"--hidden_units\", argv[index]) == 0) {\n            index++;\n            hidden_units_string = argv[index];\n        } else if (strcmp(\"--gpu\", argv[index]) == 0) {\n            isGpu = true;\n            index++;\n        } else if (strcmp(\"--help\", argv[index]) == 0) {\n            printUsage();\n            return 0;\n        }\n        index++;\n    }\n\n    if (training_set.empty() || test_set.empty() || hidden_units_string.empty()) {\n        std::cout << \"ERROR: The mandatory arguments such as path to training and test data or \"\n        << \"number of hidden units for mlp are not specified.\" << std::endl << std::endl;\n        printUsage();\n        return 1;\n    }\n\n    std::vector<int> hidden_units = getLayers(hidden_units_string);\n\n    if (hidden_units.empty()) {\n        std::cout << \"ERROR: Number of hidden units are not provided in correct format.\"\n        << \"The numbers need to be separated by ' '.\" << std::endl << std::endl;\n        printUsage();\n        return 1;\n    }\n\n    /*\n     * The MNIST data in CSV format has 785 columns.\n     * The first column is \"Label\" and rest of the columns contain data.\n     * The mnist_train.csv has 60000 records and mnist_test.csv has\n     * 10000 records.\n     */\n    auto train_iter = MXDataIter(\"CSVIter\")\n    .SetParam(\"data_csv\", training_set)\n    .SetParam(\"data_shape\", Shape(num_mnist_features + 1, 1))\n    .SetParam(\"batch_size\", batch_size)\n    .SetParam(\"flat\", 1)\n    .SetParam(\"shuffle\", 0)\n    .CreateDataIter();\n\n    auto val_iter = MXDataIter(\"CSVIter\")\n    .SetParam(\"data_csv\", test_set)\n    .SetParam(\"data_shape\", Shape(num_mnist_features + 1, 1))\n    .SetParam(\"batch_size\", batch_size)\n    .SetParam(\"flat\", 1)\n    .SetParam(\"shuffle\", 0)\n    .CreateDataIter();\n\n    TRY\n    auto net = mlp(hidden_units);\n\n    Context ctx = Context::cpu();\n    if (isGpu) {\n        ctx = Context::gpu();\n    }\n\n    std::map<std::string, NDArray> args;\n    args[\"data\"] = NDArray(Shape(batch_size, num_mnist_features), ctx);\n    args[\"label\"] = NDArray(Shape(batch_size), ctx);\n    // Let MXNet infer shapes other parameters such as weights\n    net.InferArgsMap(ctx, &args, args);\n\n    // Initialize all parameters with uniform distribution U(-0.01, 0.01)\n    auto initializer = Uniform(0.01);\n    for (auto& arg : args) {\n        // arg.first is parameter name, and arg.second is the value\n        initializer(arg.first, &arg.second);\n    }\n\n    // Create sgd optimiz er\n    Optimizer* opt = OptimizerRegistry::Find(\"sgd\");\n    opt->SetParam(\"rescale_grad\", 1.0/batch_size)\n    ->SetParam(\"lr\", learning_rate)\n    ->SetParam(\"wd\", weight_decay);\n\n    // Create executor by binding parameters to the model\n    auto *exec = net.SimpleBind(ctx, args);\n    auto arg_names = net.ListArguments();\n\n    // Start training\n    for (int iter = 0; iter < max_epoch; ++iter) {\n        int samples = 0;\n        train_iter.Reset();\n\n        auto tic = std::chrono::system_clock::now();\n        while (train_iter.Next()) {\n            samples += batch_size;\n            auto data_batch = train_iter.GetDataBatch();\n\n            /*\n             * The shape of data_batch.data is (batch_size, (num_mnist_features + 1))\n             * Need to reshape this data so that label column can be extracted from this data.\n             */\n            NDArray reshapedData = data_batch.data.Reshape(Shape((num_mnist_features + 1),\n                                                                 batch_size));\n\n            /*\n             * Extract the label data by slicing the first column of the data and\n             * copy it to \"label\" arg.\n             */\n            reshapedData.Slice(0, 1).Reshape(Shape(batch_size)).CopyTo(&args[\"label\"]);\n\n            /*\n             * Extract the feature data by slicing the columns 1 to 785 of the data and\n             * copy it to \"data\" arg.\n             */\n            reshapedData.Slice(1, (num_mnist_features + 1)).Reshape(Shape(batch_size,\n                                                                         num_mnist_features))\n                                                           .CopyTo(&args[\"data\"]);\n\n            exec->Forward(true);\n\n            // Compute gradients\n            exec->Backward();\n            // Update parameters\n            for (size_t i = 0; i < arg_names.size(); ++i) {\n                if (arg_names[i] == \"data\" || arg_names[i] == \"label\") continue;\n                opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]);\n            }\n        }\n        auto toc = std::chrono::system_clock::now();\n\n        Accuracy acc;\n        val_iter.Reset();\n        while (val_iter.Next()) {\n            auto data_batch = val_iter.GetDataBatch();\n\n            /*\n             * The shape of data_batch.data is (batch_size, (num_mnist_features + 1))\n             * Need to reshape this data so that label column can be extracted from this data.\n             */\n            NDArray reshapedData = data_batch.data.Reshape(Shape((num_mnist_features + 1),\n                                                                 batch_size));\n\n            /*\n             * Extract the label data by slicing the first column of the data and\n             * copy it to \"label\" arg.\n             */\n            NDArray labelData = reshapedData.Slice(0, 1).Reshape(Shape(batch_size));\n            labelData.CopyTo(&args[\"label\"]);\n\n            /*\n             * Extract the feature data by slicing the columns 1 to 785 of the data and\n             * copy it to \"data\" arg.\n             */\n            reshapedData.Slice(1, (num_mnist_features + 1)).Reshape(Shape(batch_size,\n                                                                         num_mnist_features))\n                                                                   .CopyTo(&args[\"data\"]);\n\n            // Forward pass is enough as no gradient is needed when evaluating\n            exec->Forward(false);\n            acc.Update(labelData, exec->outputs[0]);\n        }\n        float duration = std::chrono::duration_cast<std::chrono::milliseconds>\n        (toc - tic).count() / 1000.0;\n        LG << \"Epoch[\" << iter << \"]  \" << samples/duration << \" samples/sec Accuracy: \"\n        << acc.Get();\n    }\n\n    delete exec;\n    delete opt;\n    MXNotifyShutdown();\n    CATCH\n    return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/mlp_gpu.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * Xin Li yakumolx@gmail.com\n */\n#include <chrono>\n#include \"utils.h\"\n#include \"mxnet-cpp/MxNetCpp.h\"\n\nusing namespace mxnet::cpp;\n\nSymbol mlp(const std::vector<int> &layers) {\n  auto x = Symbol::Variable(\"X\");\n  auto label = Symbol::Variable(\"label\");\n\n  std::vector<Symbol> weights(layers.size());\n  std::vector<Symbol> biases(layers.size());\n  std::vector<Symbol> outputs(layers.size());\n\n  for (size_t i = 0; i < layers.size(); ++i) {\n    weights[i] = Symbol::Variable(\"w\" + std::to_string(i));\n    biases[i] = Symbol::Variable(\"b\" + std::to_string(i));\n    Symbol fc = FullyConnected(\n      i == 0? x : outputs[i-1],  // data\n      weights[i],\n      biases[i],\n      layers[i]);\n    outputs[i] = i == layers.size()-1 ? fc : Activation(fc, ActivationActType::kRelu);\n  }\n\n  return SoftmaxOutput(outputs.back(), label);\n}\n\nint main(int argc, char** argv) {\n  const int image_size = 28;\n  const std::vector<int> layers{128, 64, 10};\n  const int batch_size = 100;\n  const int max_epoch = 10;\n  const float learning_rate = 0.1;\n  const float weight_decay = 1e-2;\n\n  std::vector<std::string> data_files = { \"./data/mnist_data/train-images-idx3-ubyte\",\n                                          \"./data/mnist_data/train-labels-idx1-ubyte\",\n                                          \"./data/mnist_data/t10k-images-idx3-ubyte\",\n                                          \"./data/mnist_data/t10k-labels-idx1-ubyte\"\n                                        };\n\n  auto train_iter =  MXDataIter(\"MNISTIter\");\n  if (!setDataIter(&train_iter, \"Train\", data_files, batch_size)) {\n    return 1;\n  }\n\n  auto val_iter = MXDataIter(\"MNISTIter\");\n  if (!setDataIter(&val_iter, \"Label\", data_files, batch_size)) {\n    return 1;\n  }\n\n  TRY\n  auto net = mlp(layers);\n\n  Context ctx = Context::gpu();  // Use GPU for training\n\n  std::map<std::string, NDArray> args;\n  args[\"X\"] = NDArray(Shape(batch_size, image_size*image_size), ctx);\n  args[\"label\"] = NDArray(Shape(batch_size), ctx);\n  // Let MXNet infer shapes of other parameters such as weights\n  net.InferArgsMap(ctx, &args, args);\n\n  // Initialize all parameters with uniform distribution U(-0.01, 0.01)\n  auto initializer = Uniform(0.01);\n  for (auto& arg : args) {\n    // arg.first is parameter name, and arg.second is the value\n    initializer(arg.first, &arg.second);\n  }\n\n  // Create sgd optimizer\n  Optimizer* opt = OptimizerRegistry::Find(\"sgd\");\n  opt->SetParam(\"rescale_grad\", 1.0/batch_size)\n     ->SetParam(\"lr\", learning_rate)\n     ->SetParam(\"wd\", weight_decay);\n  std::unique_ptr<LRScheduler> lr_sch(new FactorScheduler(5000, 0.1));\n  opt->SetLRScheduler(std::move(lr_sch));\n\n  // Create executor by binding parameters to the model\n  auto *exec = net.SimpleBind(ctx, args);\n  auto arg_names = net.ListArguments();\n\n  // Create metrics\n  Accuracy train_acc, val_acc;\n\n  // Start training\n  for (int iter = 0; iter < max_epoch; ++iter) {\n    int samples = 0;\n    train_iter.Reset();\n    train_acc.Reset();\n\n    auto tic = std::chrono::system_clock::now();\n    while (train_iter.Next()) {\n      samples += batch_size;\n      auto data_batch = train_iter.GetDataBatch();\n      // Data provided by DataIter are stored in memory, should be copied to GPU first.\n      data_batch.data.CopyTo(&args[\"X\"]);\n      data_batch.label.CopyTo(&args[\"label\"]);\n      // CopyTo is imperative, need to wait for it to complete.\n      NDArray::WaitAll();\n\n      // Compute gradients\n      exec->Forward(true);\n      exec->Backward();\n\n      // Update parameters\n      for (size_t i = 0; i < arg_names.size(); ++i) {\n        if (arg_names[i] == \"X\" || arg_names[i] == \"label\") continue;\n        opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]);\n      }\n      // Update metric\n      train_acc.Update(data_batch.label, exec->outputs[0]);\n    }\n    // one epoch of training is finished\n    auto toc = std::chrono::system_clock::now();\n    float duration = std::chrono::duration_cast<std::chrono::milliseconds>\n                     (toc - tic).count() / 1000.0;\n    LG << \"Epoch[\" << iter << \"] \" << samples/duration \\\n       << \" samples/sec \" << \"Train-Accuracy=\" << train_acc.Get();;\n\n    val_iter.Reset();\n    val_acc.Reset();\n    while (val_iter.Next()) {\n      auto data_batch = val_iter.GetDataBatch();\n      data_batch.data.CopyTo(&args[\"X\"]);\n      data_batch.label.CopyTo(&args[\"label\"]);\n      NDArray::WaitAll();\n\n      // Only forward pass is enough as no gradient is needed when evaluating\n      exec->Forward(false);\n      val_acc.Update(data_batch.label, exec->outputs[0]);\n    }\n    LG << \"Epoch[\" << iter << \"] Val-Accuracy=\" << val_acc.Get();\n  }\n\n  delete exec;\n  delete opt;\n  MXNotifyShutdown();\n  CATCH\n  return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/mnist_to_csv.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Following file converts the mnist data to CSV format.\n# Usage:\n# mnist_to_csv.py train-images-idx3-ubyte train-labels-idx1-ubyte mnist_train.csv 60000\n# mnist_to_csv.py t10k-images-idx3-ubyte t10k-labels-idx1-ubyte mnist_test.csv 10000\n#\n\nimport argparse\n\ndef convert_to_csv(args):\n    imageFile = open(args.imageFile, \"rb\")\n    labelFile = open(args.labelFile, \"rb\")\n    outputFile = open(args.outputFile, \"w\")\n\n    imageFile.read(16)\n    labelFile.read(8)\n    images = []\n\n    for i in range(args.num_records):\n        image = [ord(labelFile.read(1))]\n        for j in range(28 * 28):\n            image.append(ord(imageFile.read(1)))\n        images.append(image)\n\n    for image in images:\n        outputFile.write(\",\".join(str(pix) for pix in image) + \"\\n\")\n\n    imageFile.close()\n    outputFile.close()\n    labelFile.close()\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"imageFile\", type=str, help=\"image file in mnist format e.g. train-images-idx3-ubyte\")\n    parser.add_argument(\"labelFile\", type=str, help=\"label file in mnist format e.g train-labels-idx1-ubyte\")\n    parser.add_argument(\"outputFile\", type=str, help=\"Output file in CSV format e.g mnist_train_trial.csv\")\n    parser.add_argument(\"num_records\", type=int, help=\"Number of images in the input files.e.g 60000\")\n    args = parser.parse_args()\n\n    try:\n        convert_to_csv(args)\n    except Exception as e:\n        print(\"Error : Exception {}\".format(str(e)))\n"
  },
  {
    "path": "cpp-package/example/resnet.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n */\n#include <map>\n#include <string>\n#include <fstream>\n#include <vector>\n#include <cstdlib>\n#include \"utils.h\"\n#include \"mxnet-cpp/MxNetCpp.h\"\n\nusing namespace mxnet::cpp;\n\nSymbol ConvolutionNoBias(const std::string& symbol_name,\n                         Symbol data,\n                         Symbol weight,\n                         Shape kernel,\n                         int num_filter,\n                         Shape stride = Shape(1, 1),\n                         Shape dilate = Shape(1, 1),\n                         Shape pad = Shape(0, 0),\n                         int num_group = 1,\n                         int64_t workspace = 512) {\n  return Operator(\"Convolution\")\n      .SetParam(\"kernel\", kernel)\n      .SetParam(\"num_filter\", num_filter)\n      .SetParam(\"stride\", stride)\n      .SetParam(\"dilate\", dilate)\n      .SetParam(\"pad\", pad)\n      .SetParam(\"num_group\", num_group)\n      .SetParam(\"workspace\", workspace)\n      .SetParam(\"no_bias\", true)\n      .SetInput(\"data\", data)\n      .SetInput(\"weight\", weight)\n      .CreateSymbol(symbol_name);\n}\n\nSymbol getConv(const std::string & name, Symbol data,\n               int  num_filter,\n               Shape kernel, Shape stride, Shape pad,\n               bool with_relu,\n               mx_float bn_momentum) {\n  Symbol conv_w(name + \"_w\");\n  Symbol conv = ConvolutionNoBias(name, data, conv_w,\n                                  kernel, num_filter, stride, Shape(1, 1),\n                                  pad, 1, 512);\n\n  Symbol gamma(name + \"_gamma\");\n  Symbol beta(name + \"_beta\");\n  Symbol mmean(name + \"_mmean\");\n  Symbol mvar(name + \"_mvar\");\n\n  Symbol bn = BatchNorm(name + \"_bn\", conv, gamma,\n                        beta, mmean, mvar, 2e-5, bn_momentum, false);\n\n  if (with_relu) {\n    return Activation(name + \"_relu\", bn, \"relu\");\n  } else {\n    return bn;\n  }\n}\n\nSymbol makeBlock(const std::string & name, Symbol data, int num_filter,\n                 bool dim_match, mx_float bn_momentum) {\n  Shape stride;\n  if (dim_match) {\n    stride = Shape(1, 1);\n  } else {\n    stride = Shape(2, 2);\n  }\n\n  Symbol conv1 = getConv(name + \"_conv1\", data, num_filter,\n                         Shape(3, 3), stride, Shape(1, 1),\n                         true, bn_momentum);\n\n  Symbol conv2 = getConv(name + \"_conv2\", conv1, num_filter,\n                         Shape(3, 3), Shape(1, 1), Shape(1, 1),\n                         false, bn_momentum);\n\n  Symbol shortcut;\n\n  if (dim_match) {\n    shortcut = data;\n  } else {\n    Symbol shortcut_w(name + \"_proj_w\");\n    shortcut = ConvolutionNoBias(name + \"_proj\", data, shortcut_w,\n                                 Shape(2, 2), num_filter,\n                                 Shape(2, 2), Shape(1, 1), Shape(0, 0),\n                                 1, 512);\n  }\n\n  Symbol fused = shortcut + conv2;\n  return Activation(name + \"_relu\", fused, \"relu\");\n}\n\nSymbol getBody(Symbol data, int num_level, int num_block, int num_filter, mx_float bn_momentum) {\n  for (int level = 0; level < num_level; level++) {\n    for (int block = 0; block < num_block; block++) {\n      data = makeBlock(\"level\" + std::to_string(level + 1) + \"_block\" + std::to_string(block + 1),\n                       data, num_filter * (std::pow(2, level)),\n                       (level == 0 || block > 0), bn_momentum);\n    }\n  }\n  return data;\n}\n\nSymbol ResNetSymbol(int num_class, int num_level = 3, int num_block = 9,\n                    int num_filter = 16, mx_float bn_momentum = 0.9,\n                    mxnet::cpp::Shape pool_kernel = mxnet::cpp::Shape(8, 8)) {\n  // data and label\n  Symbol data = Symbol::Variable(\"data\");\n  Symbol data_label = Symbol::Variable(\"data_label\");\n\n  Symbol gamma(\"gamma\");\n  Symbol beta(\"beta\");\n  Symbol mmean(\"mmean\");\n  Symbol mvar(\"mvar\");\n\n  Symbol zscore = BatchNorm(\"zscore\", data, gamma,\n                            beta, mmean, mvar, 0.001, bn_momentum);\n\n  Symbol conv = getConv(\"conv0\", zscore, num_filter,\n                        Shape(3, 3), Shape(1, 1), Shape(1, 1),\n                        true, bn_momentum);\n\n  Symbol body = getBody(conv, num_level, num_block, num_filter, bn_momentum);\n\n  Symbol pool = Pooling(\"pool\", body, pool_kernel, PoolingPoolType::kAvg);\n\n  Symbol flat = Flatten(\"flatten\", pool);\n\n  Symbol fc_w(\"fc_w\"), fc_b(\"fc_b\");\n  Symbol fc = FullyConnected(\"fc\", flat, fc_w, fc_b, num_class);\n\n  return SoftmaxOutput(\"softmax\", fc, data_label);\n}\n\nNDArray ResizeInput(NDArray data, const Shape new_shape) {\n  NDArray pic = data.Reshape(Shape(0, 1, 28, 28));\n  NDArray pic_1channel;\n  Operator(\"_contrib_BilinearResize2D\")\n    .SetParam(\"height\", new_shape[2])\n    .SetParam(\"width\", new_shape[3])\n    (pic).Invoke(pic_1channel);\n  NDArray output;\n  Operator(\"tile\")\n    .SetParam(\"reps\", Shape(1, 3, 1, 1))\n    (pic_1channel).Invoke(output);\n  return output;\n}\n\nint main(int argc, char const *argv[]) {\n  int max_epoch = argc > 1 ? strtol(argv[1], nullptr, 10) : 100;\n  float learning_rate = 1e-4;\n  float weight_decay = 1e-4;\n\n  TRY\n  auto resnet = ResNetSymbol(10);\n  std::map<std::string, NDArray> args_map;\n  std::map<std::string, NDArray> aux_map;\n\n  /*context*/\n  auto ctx = Context::cpu();\n  int num_gpu;\n  MXGetGPUCount(&num_gpu);\n  int batch_size = 8;\n#if MXNET_USE_CUDA\n  if (num_gpu > 0) {\n    ctx = Context::gpu();\n    batch_size = 32;\n  }\n#endif\n\n  const Shape data_shape = Shape(batch_size, 3, 224, 224),\n              label_shape = Shape(batch_size);\n  args_map[\"data\"] = NDArray(data_shape, ctx);\n  args_map[\"data_label\"] = NDArray(label_shape, ctx);\n  resnet.InferArgsMap(ctx, &args_map, args_map);\n\n  std::vector<std::string> data_files = { \"./data/mnist_data/train-images-idx3-ubyte\",\n                                          \"./data/mnist_data/train-labels-idx1-ubyte\",\n                                          \"./data/mnist_data/t10k-images-idx3-ubyte\",\n                                          \"./data/mnist_data/t10k-labels-idx1-ubyte\"\n                                        };\n\n  auto train_iter =  MXDataIter(\"MNISTIter\");\n  if (!setDataIter(&train_iter, \"Train\", data_files, batch_size)) {\n    return 1;\n  }\n\n  auto val_iter = MXDataIter(\"MNISTIter\");\n  if (!setDataIter(&val_iter, \"Label\", data_files, batch_size)) {\n    return 1;\n  }\n\n  // initialize parameters\n  auto initializer = Uniform(0.07);\n  for (auto& arg : args_map) {\n    initializer(arg.first, &arg.second);\n  }\n\n  Optimizer* opt = OptimizerRegistry::Find(\"sgd\");\n  opt->SetParam(\"lr\", learning_rate)\n     ->SetParam(\"wd\", weight_decay)\n     ->SetParam(\"momentum\", 0.9)\n     ->SetParam(\"rescale_grad\", 1.0 / batch_size)\n     ->SetParam(\"clip_gradient\", 10);\n\n  auto *exec = resnet.SimpleBind(ctx, args_map);\n  auto arg_names = resnet.ListArguments();\n\n  // Create metrics\n  Accuracy train_acc, val_acc;\n  LogLoss logloss_train, logloss_val;\n  for (int epoch = 0; epoch < max_epoch; ++epoch) {\n    LG << \"Epoch: \" << epoch;\n    train_iter.Reset();\n    train_acc.Reset();\n    int iter = 0;\n    while (train_iter.Next()) {\n      auto data_batch = train_iter.GetDataBatch();\n      ResizeInput(data_batch.data, data_shape).CopyTo(&args_map[\"data\"]);\n      data_batch.label.CopyTo(&args_map[\"data_label\"]);\n      NDArray::WaitAll();\n\n      exec->Forward(true);\n      exec->Backward();\n\n      for (size_t i = 0; i < arg_names.size(); ++i) {\n        if (arg_names[i] == \"data\" || arg_names[i] == \"data_label\") continue;\n        opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]);\n      }\n      NDArray::WaitAll();\n      train_acc.Update(data_batch.label, exec->outputs[0]);\n      logloss_train.Reset();\n      logloss_train.Update(data_batch.label, exec->outputs[0]);\n      ++iter;\n      LG << \"EPOCH: \" << epoch << \" ITER: \" << iter\n         << \" Train Accuracy: \" << train_acc.Get()\n         << \" Train Loss: \" << logloss_train.Get();\n    }\n    LG << \"EPOCH: \" << epoch << \" Train Accuracy: \" << train_acc.Get();\n\n    val_iter.Reset();\n    val_acc.Reset();\n    iter = 0;\n    while (val_iter.Next()) {\n      auto data_batch = val_iter.GetDataBatch();\n      ResizeInput(data_batch.data, data_shape).CopyTo(&args_map[\"data\"]);\n      data_batch.label.CopyTo(&args_map[\"data_label\"]);\n      NDArray::WaitAll();\n      exec->Forward(false);\n      NDArray::WaitAll();\n      val_acc.Update(data_batch.label, exec->outputs[0]);\n      LG << \"EPOCH: \" << epoch << \" ITER: \" << iter << \" Val Accuracy: \" << val_acc.Get();\n      ++iter;\n    }\n    LG << \"Validation Accuracy: \" << val_acc.Get();\n  }\n  delete exec;\n  delete opt;\n  MXNotifyShutdown();\n  CATCH\n  return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/run_lenet_with_mxdataiter.sh",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nif [ ! -f \"./mnist.zip\" ]; then\n  wget https://repo.mxnet.io/gluon/dataset/mnist/\n  unzip -u mnist.zip\nfi\nmake lenet_with_mxdataiter\nLD_LIBRARY_PATH=../lib/linux ./lenet_with_mxdataiter\n"
  },
  {
    "path": "cpp-package/example/test_kvstore.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n#include \"mxnet/c_api.h\"  // MXGetGPUCount()\n#include \"mxnet-cpp/MxNetCpp.h\"\n\nusing namespace mxnet::cpp;\n\nstatic bool test_single_key(const Context &context, const std::string &context_str) {\n  std::string key = \"singlekeytest-\" + context_str;\n\n  NDArray result(Shape(4), context);\n  NDArray result_cpu;\n\n  // initialize data\n  NDArray data_cpu({0.f, 233.f, -0.12f, 9.f}, Shape(4), Context::cpu());\n  NDArray data = data_cpu.Copy(context);\n  NDArray::WaitAll();\n\n  KVStore::Init(key, data);\n  NDArray::WaitAll();\n\n  // retrieve result\n  KVStore::Pull(key, &result);\n  NDArray::WaitAll();\n\n  result_cpu = result.Copy(Context::cpu());\n  NDArray::WaitAll();\n\n  // compare\n  for (size_t j=0; j < result_cpu.Size(); j++) {\n    if (result_cpu.GetData()[j] != data_cpu.GetData()[j]) {\n      LG << \"Error: wrong initialized data in singlekeytest-\" << context_str\n          << \", expect \" << data_cpu.GetData()[j]\n          << \" got \" << result_cpu.GetData()[j];\n      return false;\n    }\n  }\n\n  // push gradient\n  NDArray grad_cpu({0.1f, -2.f, -4.4f, 0.f}, Shape(4), Context::cpu());\n  NDArray grad = grad_cpu.Copy(context);\n  NDArray::WaitAll();\n\n  KVStore::Push(key, grad);\n  NDArray::WaitAll();\n\n  // retrieve result\n  KVStore::Pull(key, &result);\n  NDArray::WaitAll();\n\n  result_cpu = result.Copy(Context::cpu());\n  NDArray::WaitAll();\n\n  // compare\n  for (size_t j=0; j < result_cpu.Size(); j++) {\n    if (result_cpu.GetData()[j] != grad_cpu.GetData()[j]) {\n      LG << \"Error: wrong gradient data in singlekeytest-\" << context_str\n          << \", expect \" << grad_cpu.GetData()[j]\n          << \" got \" << result_cpu.GetData()[j];\n      return false;\n    }\n  }\n\n  return true;\n}\n\nstatic bool test_multiple_key(const Context &context, const std::string &context_str) {\n  std::vector<std::string> keys(2);\n  keys[0] = \"multikeytest-0-\" + context_str;\n  keys[1] = \"multikeytest-1-\" + context_str;\n\n  std::vector<NDArray> results(2);\n  results[0] = NDArray(Shape(4), context);\n  results[1] = NDArray(Shape(4), context);\n  std::vector<NDArray> results_cpu(2);\n\n  // initialize data\n  std::vector<NDArray> data_cpu(2);\n  data_cpu[0] = NDArray({0.f, 2.f, -3.12f, 4.f}, Shape(4), Context::cpu());\n  data_cpu[1] = NDArray({0.8f, -2.f, 6.6f, 77.f}, Shape(4), Context::cpu());\n  std::vector<NDArray> data(2);\n  data[0] = data_cpu[0].Copy(context);\n  data[1] = data_cpu[1].Copy(context);\n  NDArray::WaitAll();\n\n  KVStore::Init(keys, data);\n  NDArray::WaitAll();\n\n  // retrieve result\n  KVStore::Pull(keys, &results);\n  NDArray::WaitAll();\n\n  results_cpu[0] = results[0].Copy(Context::cpu());\n  results_cpu[1] = results[1].Copy(Context::cpu());\n  NDArray::WaitAll();\n\n  // compare\n  for (size_t i=0; i < results_cpu.size(); i++) {\n    for (size_t j=0; j < results_cpu[i].Size(); j++) {\n      if (results_cpu[i].GetData()[j] != data_cpu[i].GetData()[j]) {\n        LG << \"Error: wrong initialized data in multikeytest-\" << context_str\n            << \", expect \" << data_cpu[i].GetData()[j]\n            << \" got \" << results_cpu[i].GetData()[j];\n        return false;\n      }\n    }\n  }\n\n  // push gradient, reduce for the second\n  std::vector<std::string> push_keys(3);\n  push_keys[0] = \"multikeytest-0-\" + context_str;\n  push_keys[1] = \"multikeytest-1-\" + context_str;\n  push_keys[2] = \"multikeytest-1-\" + context_str;\n\n  std::vector<NDArray> grads_cpu(3);\n  grads_cpu[0] = NDArray({0.2f, -0.3f, -1.1f, 0.0f}, Shape(4), Context::cpu());\n  grads_cpu[1] = NDArray({2.f, 4.f, -4.f, -5.f}, Shape(4), Context::cpu());\n  grads_cpu[2] = NDArray({-3.f, -0.2f, 12.f, -9.f}, Shape(4), Context::cpu());\n  std::vector<NDArray> grads(3);\n  grads[0] = grads_cpu[0].Copy(context);\n  grads[1] = grads_cpu[1].Copy(context);\n  grads[2] = grads_cpu[2].Copy(context);\n  NDArray::WaitAll();\n\n  KVStore::Push(push_keys, grads);\n  NDArray::WaitAll();\n\n  // retrieve result\n  KVStore::Pull(keys, &results);\n  NDArray::WaitAll();\n\n  results_cpu[0] = results[0].Copy(Context::cpu());\n  results_cpu[1] = results[1].Copy(Context::cpu());\n  NDArray::WaitAll();\n\n  // compare the first\n  for (size_t j=0; j < results_cpu[0].Size(); j++) {\n    if (results_cpu[0].GetData()[j] != grads_cpu[0].GetData()[j]) {\n      LG << \"Error: wrong gradient data in multikeytest-\" << context_str\n          << \", expect \" << grads_cpu[0].GetData()[j]\n          << \" got \" << results_cpu[0].GetData()[j];\n      return false;\n    }\n  }\n\n  // compare the second\n  for (size_t j=0; j < results_cpu[1].Size(); j++) {\n    if (results_cpu[1].GetData()[j] != (grads_cpu[1].GetData()[j] + grads_cpu[2].GetData()[j])) {\n      LG << \"Error: wrong reduced gradient data in multikeytest-\" << context_str\n          << \", expect \" << (grads_cpu[1].GetData()[j] + grads_cpu[2].GetData()[j])\n          << \" got \" << results_cpu[1].GetData()[j];\n      return false;\n    }\n  }\n\n  return true;\n}\n\nint main(int argc, char** argv) {\n  KVStore::SetType(\"local\");\n\n  bool success1 = test_single_key(Context::cpu(), \"cpu\");\n  bool success2 = test_multiple_key(Context::cpu(), \"cpu\");\n\n  bool success3 = true;\n  bool success4 = true;\n\n  int gpu_count = 0;\n  if (MXGetGPUCount(&gpu_count) != 0) {\n    LG << \"Error: MXGetGPUCount\";\n\n    MXNotifyShutdown();\n    return 1;\n  }\n\n  if (gpu_count > 0) {\n    success3 = test_single_key(Context::gpu(), \"gpu\");\n    success4 = test_multiple_key(Context::gpu(), \"gpu\");\n  }\n\n  int ret = (success1 && success2 && success3 && success4) ? 0 : 1;\n\n  MXNotifyShutdown();\n  return ret;\n}\n"
  },
  {
    "path": "cpp-package/example/test_ndarray_copy.cpp",
    "content": "/*\r\n * Licensed to the Apache Software Foundation (ASF) under one\r\n * or more contributor license agreements.  See the NOTICE file\r\n * distributed with this work for additional information\r\n * regarding copyright ownership.  The ASF licenses this file\r\n * to you under the Apache License, Version 2.0 (the\r\n * \"License\"); you may not use this file except in compliance\r\n * with the License.  You may obtain a copy of the License at\r\n *\r\n *   http://www.apache.org/licenses/LICENSE-2.0\r\n *\r\n * Unless required by applicable law or agreed to in writing,\r\n * software distributed under the License is distributed on an\r\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\r\n * KIND, either express or implied.  See the License for the\r\n * specific language governing permissions and limitations\r\n * under the License.\r\n * \r\n */\r\n#include <vector>\r\n#include \"mxnet/c_api.h\"\r\n#include \"dmlc/logging.h\"\r\n#include \"mxnet-cpp/MxNetCpp.h\"\r\nusing namespace mxnet::cpp;\r\n\r\nenum TypeFlag {\r\n  kFloat32 = 0,\r\n  kFloat64 = 1,\r\n  kFloat16 = 2,\r\n  kUint8 = 3,\r\n  kInt32 = 4,\r\n  kInt8  = 5,\r\n  kInt64 = 6,\r\n};\r\n\r\n/*\r\n * The file is used for testing if there exist type inconsistency\r\n * when using Copy API to create a new NDArray.\r\n * By running: build/test_ndarray.\r\n */\r\nint main(int argc, char** argv) {\r\n    std::vector<mx_uint> shape1{128, 2, 32};\r\n    Shape shape2(32, 8, 64);\r\n\r\n    int gpu_count = 0;\r\n    if (MXGetGPUCount(&gpu_count) != 0) {\r\n      LOG(ERROR) << \"MXGetGPUCount failed\";\r\n      return -1;\r\n    }\r\n\r\n    Context context = (gpu_count > 0) ? Context::gpu() : Context::cpu();\r\n\r\n    NDArray src1(shape1, context, true, kFloat16);\r\n    NDArray src2(shape2, context, false, kInt8);\r\n    NDArray dst1, dst2;\r\n    dst1 = src1.Copy(context);\r\n    dst2 = src2.Copy(context);\r\n    NDArray::WaitAll();\r\n    CHECK_EQ(src1.GetDType(), dst1.GetDType());\r\n    CHECK_EQ(src2.GetDType(), dst2.GetDType());\r\n    return 0;\r\n}\r\n"
  },
  {
    "path": "cpp-package/example/test_optimizer.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n * \n * The file is used for testing if the optimizer could be created more than 1.\n * By running: build/test_optimizer\n * It return 0(means no error) if it succeed otherwise 1(error).\n */\n#include \"mxnet-cpp/MxNetCpp.h\"\n\nusing namespace mxnet::cpp;\n\nint main(int argc, char** argv) {\n  // Confirm >1 optimizers can be created w/o error\n  Optimizer* opt = OptimizerRegistry::Find(\"sgd\");\n  opt = OptimizerRegistry::Find(\"adam\");\n  int ret = (opt == 0) ? 1 : 0;\n\n  delete opt;\n  MXNotifyShutdown();\n  return ret;\n}\n"
  },
  {
    "path": "cpp-package/example/test_regress_label.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n * \n * This file is used for testing LinearRegressionOutput can\n *   still bind if label is not provided\n */\n\n#include <iostream>\n#include <vector>\n#include <string>\n#include \"dmlc/logging.h\"\n#include \"mxnet-cpp/MxNetCpp.h\"\n\nusing namespace mxnet::cpp;\n\nint main() {\n    LOG(INFO) << \"Running LinearRegressionOutput symbol testing, \"\n                 \"executor should be able to bind without label.\";\n    Symbol data = Symbol::Variable(\"data\");\n    Symbol label = Symbol::Variable(\"regress_label\");\n    Symbol symbol = LinearRegressionOutput(data, label);\n    std::map<std::string, mxnet::cpp::OpReqType> opReqMap;\n    for (const auto& iter : symbol.ListArguments()) {\n        opReqMap[iter] = mxnet::cpp::OpReqType::kNullOp;\n    }\n    std::map<std::string, mxnet::cpp::NDArray> argMap({\n        {\"data\", NDArray(Shape{1, 3}, Context::cpu(), true)}\n    });\n\n    try {\n        symbol.SimpleBind(Context::cpu(),\n                argMap,\n                std::map<std::string, mxnet::cpp::NDArray>(),\n                opReqMap,\n                std::map<std::string, mxnet::cpp::NDArray>());\n    } catch (const std::exception& e) {\n        LOG(ERROR) << \"Error binding the symbol: \" << MXGetLastError() << \" \" << e.what();\n        throw;\n    }\n    return 0;\n}\n"
  },
  {
    "path": "cpp-package/example/test_score.cpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * Xin Li yakumolx@gmail.com\n * The file is used for testing if the score(accurary) we get\n * is better than the threshold we set using mlp model.\n * By running: build/test_score 0.75\n * 0.75 here means the threshold score\n * It return 0 if we can achieve higher score than threshold, otherwise 1\n */\n#include <chrono>\n#include \"utils.h\"\n#include \"mxnet-cpp/MxNetCpp.h\"\n\nusing namespace mxnet::cpp;\n\nSymbol mlp(const std::vector<int> &layers) {\n  auto x = Symbol::Variable(\"X\");\n  auto label = Symbol::Variable(\"label\");\n\n  std::vector<Symbol> weights(layers.size());\n  std::vector<Symbol> biases(layers.size());\n  std::vector<Symbol> outputs(layers.size());\n\n  for (size_t i = 0; i < layers.size(); ++i) {\n    weights[i] = Symbol::Variable(\"w\" + std::to_string(i));\n    biases[i] = Symbol::Variable(\"b\" + std::to_string(i));\n    Symbol fc = FullyConnected(\n      i == 0? x : outputs[i-1],  // data\n      weights[i],\n      biases[i],\n      layers[i]);\n    outputs[i] = i == layers.size()-1? fc : Activation(fc, ActivationActType::kRelu);\n  }\n\n  return SoftmaxOutput(outputs.back(), label);\n}\n\nint main(int argc, char** argv) {\n  const float MIN_SCORE = std::stof(argv[1]);\n\n  const int image_size = 28;\n  const std::vector<int> layers{128, 64, 10};\n  const int batch_size = 100;\n  const int max_epoch = 10;\n  const float learning_rate = 0.1;\n  const float weight_decay = 1e-2;\n  float score = 0;\n\n  std::vector<std::string> data_files = { \"./data/mnist_data/train-images-idx3-ubyte\",\n                                          \"./data/mnist_data/train-labels-idx1-ubyte\",\n                                          \"./data/mnist_data/t10k-images-idx3-ubyte\",\n                                          \"./data/mnist_data/t10k-labels-idx1-ubyte\"\n                                        };\n\n  auto train_iter =  MXDataIter(\"MNISTIter\");\n  if (!setDataIter(&train_iter, \"Train\", data_files, batch_size)) {\n    return 1;\n  }\n\n  auto val_iter = MXDataIter(\"MNISTIter\");\n  if (!setDataIter(&val_iter, \"Label\", data_files, batch_size)) {\n    return 1;\n  }\n\n  TRY\n  auto net = mlp(layers);\n\n  Context ctx = Context::gpu();  // Use GPU for training\n#if !MXNET_USE_CUDA\n  ctx = Context::cpu();\n#endif\n\n  std::map<std::string, NDArray> args;\n  args[\"X\"] = NDArray(Shape(batch_size, image_size*image_size), ctx);\n  args[\"label\"] = NDArray(Shape(batch_size), ctx);\n  // Let MXNet infer shapes of other parameters such as weights\n  net.InferArgsMap(ctx, &args, args);\n\n  // Initialize all parameters with uniform distribution U(-0.01, 0.01)\n  auto initializer = Uniform(0.01);\n  for (auto& arg : args) {\n    // arg.first is parameter name, and arg.second is the value\n    initializer(arg.first, &arg.second);\n  }\n\n  // Create sgd optimizer\n  Optimizer* opt = OptimizerRegistry::Find(\"sgd\");\n  opt->SetParam(\"rescale_grad\", 1.0/batch_size)\n     ->SetParam(\"lr\", learning_rate)\n     ->SetParam(\"wd\", weight_decay);\n  std::unique_ptr<LRScheduler> lr_sch(new FactorScheduler(5000, 0.1));\n  opt->SetLRScheduler(std::move(lr_sch));\n\n  // Create executor by binding parameters to the model\n  auto *exec = net.SimpleBind(ctx, args);\n  auto arg_names = net.ListArguments();\n\n  // Start training\n  for (int iter = 0; iter < max_epoch; ++iter) {\n    int samples = 0;\n    train_iter.Reset();\n\n    auto tic = std::chrono::system_clock::now();\n    while (train_iter.Next()) {\n      samples += batch_size;\n      auto data_batch = train_iter.GetDataBatch();\n      // Data provided by DataIter are stored in memory, should be copied to GPU first.\n      data_batch.data.CopyTo(&args[\"X\"]);\n      data_batch.label.CopyTo(&args[\"label\"]);\n      // CopyTo is imperative, need to wait for it to complete.\n      NDArray::WaitAll();\n\n      // Compute gradients\n      exec->Forward(true);\n      exec->Backward();\n      // Update parameters\n      for (size_t i = 0; i < arg_names.size(); ++i) {\n        if (arg_names[i] == \"X\" || arg_names[i] == \"label\") continue;\n        opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]);\n      }\n    }\n    auto toc = std::chrono::system_clock::now();\n\n    Accuracy acc;\n    val_iter.Reset();\n    while (val_iter.Next()) {\n      auto data_batch = val_iter.GetDataBatch();\n      data_batch.data.CopyTo(&args[\"X\"]);\n      data_batch.label.CopyTo(&args[\"label\"]);\n      NDArray::WaitAll();\n      // Only forward pass is enough as no gradient is needed when evaluating\n      exec->Forward(false);\n      acc.Update(data_batch.label, exec->outputs[0]);\n    }\n    float duration = std::chrono::duration_cast<std::chrono::milliseconds>\n                     (toc - tic).count() / 1000.0;\n    LG << \"Epoch: \" << iter << \" \" << samples/duration << \" samples/sec Accuracy: \" << acc.Get();\n    score = acc.Get();\n  }\n\n  delete exec;\n  delete opt;\n  MXNotifyShutdown();\n  CATCH\n  return score >= MIN_SCORE ? 0 : 1;\n}\n"
  },
  {
    "path": "cpp-package/example/unittests/unit_test_mlp_csv.sh",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# This file is a unit test for mlp_csv.cpp example in 'example' directory.\n# The file\n#    1. Downloads the MNIST data,\n#    2. Converts it into CSV format.\n#    3. Runs the mlp_csv example and ensures that the accuracy is more than expected.\n#\n\n#!/bin/bash\n\nset -e # exit on the first error\nexport EXE_NAME=mlp_csv\n\ncd $(dirname $(readlink -f $0))/../\nexport LD_LIBRARY_PATH=$(readlink -f ../../build):$LD_LIBRARY_PATH\n\nif [ ! -f ../../build/cpp-package/example/${EXE_NAME} ];\nthen\necho \"FAIL: ${EXE_NAME} does not exist\"\nexit\nfi\n\ncp ../../build/cpp-package/example/${EXE_NAME} .\n\n./get_data.sh\npython3 mnist_to_csv.py ./data/mnist_data/train-images-idx3-ubyte ./data/mnist_data/train-labels-idx1-ubyte ./data/mnist_data/mnist_train.csv 60000\npython3 mnist_to_csv.py ./data/mnist_data/t10k-images-idx3-ubyte ./data/mnist_data/t10k-labels-idx1-ubyte ./data/mnist_data/mnist_test.csv 10000\n\n./${EXE_NAME} --train ./data/mnist_data/mnist_train.csv --test ./data/mnist_data/mnist_test.csv --epochs 10 --batch_size 100 --hidden_units \"128 64 10\" 2&> ${EXE_NAME}.log\n\nif [ ! -f ${EXE_NAME}.log ];\nthen\necho \"FAIL: Log file ${EXE_NAME}.log does not exist.\"\nexit\nfi\n\n# Obtain the accuracy achieved by mlp model after training with MNIST data in CSV format.\nexport Acc_obtained=`grep -oP '.*\\K(?<=Accuracy: ).*$' ${EXE_NAME}.log | tail -1 | tr -d '\\n'`\nexport Acc_expected=0.98\n\n# If the obtained accuracy does not meet the expected accuracy, report the test as FAIL.\nif [ $(echo \"$Acc_obtained $Acc_expected\" | awk '{printf($1 >= $2) ? 1 : 0}') -eq 1 ] ;\nthen\necho \"PASS: ${EXE_NAME} obtained $Acc_obtained accuracy.\"\nelse\necho \"FAIL: Accuracy = $Acc_obtained is less than expected accuracy $Acc_expected.\"\nfi\n"
  },
  {
    "path": "cpp-package/example/utils.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef CPP_PACKAGE_EXAMPLE_UTILS_H_\n#define CPP_PACKAGE_EXAMPLE_UTILS_H_\n\n#include <string>\n#include <fstream>\n#include <vector>\n#include \"mxnet-cpp/MxNetCpp.h\"\n\nusing namespace mxnet::cpp;\n\n#define TRY try {\n#define CATCH                                 \\\n  }                                           \\\n  catch (dmlc::Error & err) {                 \\\n    LG << \"Status: FAIL\";                     \\\n    LG << \"With Error: \" << MXGetLastError(); \\\n    return 1;                                 \\\n  }\n\nbool isFileExists(const std::string& filename) {\n  std::ifstream fhandle(filename.c_str());\n  return fhandle.good();\n}\n\nbool check_datafiles(const std::vector<std::string>& data_files) {\n  for (size_t index = 0; index < data_files.size(); index++) {\n    if (!(isFileExists(data_files[index]))) {\n      LG << \"Error: File does not exist: \" << data_files[index];\n      return false;\n    }\n  }\n  return true;\n}\n\nbool setDataIter(MXDataIter* iter,\n                 const std::string& useType,\n                 const std::vector<std::string>& data_files,\n                 int batch_size) {\n  if (!check_datafiles(data_files)) {\n    return false;\n  }\n\n  iter->SetParam(\"batch_size\", batch_size);\n  iter->SetParam(\"shuffle\", 1);\n  iter->SetParam(\"flat\", 1);\n\n  if (useType == \"Train\") {\n    iter->SetParam(\"image\", data_files[0]);\n    iter->SetParam(\"label\", data_files[1]);\n  } else if (useType == \"Label\") {\n    iter->SetParam(\"image\", data_files[2]);\n    iter->SetParam(\"label\", data_files[3]);\n  }\n\n  iter->CreateDataIter();\n  return true;\n}\n\n#endif  // CPP_PACKAGE_EXAMPLE_UTILS_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/.gitignore",
    "content": "# Rebuildable file(s)\nop.h\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/CPPLINT.cfg",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nfilter=-runtime/references\nexclude_files=op.h\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/MxNetCpp.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file MxNetCpp.h\n * \\brief meta include file for mxnet.cpp\n * \\author Chuntao Hong, Zhang Chen\n */\n\n#ifndef MXNET_CPP_MXNETCPP_H_\n#define MXNET_CPP_MXNETCPP_H_\n\n#include \"mxnet-cpp/executor.hpp\"\n#include \"mxnet-cpp/symbol.hpp\"\n#include \"mxnet-cpp/ndarray.hpp\"\n#include \"mxnet-cpp/operator.hpp\"\n#include \"mxnet-cpp/optimizer.hpp\"\n#include \"mxnet-cpp/kvstore.hpp\"\n#include \"mxnet-cpp/op.h\"\n#include \"mxnet-cpp/op_suppl.h\"\n#include \"mxnet-cpp/io.hpp\"\n#include \"mxnet-cpp/metric.h\"\n#include \"mxnet-cpp/initializer.h\"\n#include \"mxnet-cpp/contrib.h\"\n\n#endif  // MXNET_CPP_MXNETCPP_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/base.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file base.h\n * \\brief base definitions for mxnetcpp\n * \\author Chuntao Hong, Zhang Chen\n */\n\n#ifndef MXNET_CPP_BASE_H_\n#define MXNET_CPP_BASE_H_\n\n#include <cstdlib>\n#include \"mxnet/c_api.h\"\n#include \"nnvm/c_api.h\"\n\nnamespace mxnet {\nnamespace cpp {\n\ntypedef unsigned index_t;\n\nenum OpReqType {\n  /*! \\brief no operation, do not write anything */\n  kNullOp,\n  /*! \\brief write gradient to provided space */\n  kWriteTo,\n  /*!\n   * \\brief perform an inplace write,\n   * Target shares memory with one of input arguments.\n   * This option only happen when\n   */\n  kWriteInplace,\n  /*! \\brief add to the provided space */\n  kAddTo\n};\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_BASE_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/contrib.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file contrib.h\n * \\brief utility function to enable some contrib features\n * \\author Haohuan Wang\n */\n#ifndef MXNET_CPP_CONTRIB_H_\n#define MXNET_CPP_CONTRIB_H_\n\n#include <iostream>\n#include <string>\n#include <map>\n#include <vector>\n#include \"mxnet-cpp/symbol.h\"\n\nnamespace mxnet {\nnamespace cpp {\nnamespace details {\n\n/*!\n * split a string with the given delimiter\n * @param str string to be parsed\n * @param delimiter delimiter\n * @return delimited list of string\n */\ninline std::vector<std::string> split(const std::string& str, const std::string& delimiter) {\n  std::vector<std::string> splitted;\n  size_t last = 0;\n  size_t next = 0;\n  while ((next = str.find(delimiter, last)) != std::string::npos) {\n    splitted.push_back(str.substr(last, next - last));\n    last = next + 1;\n  }\n  splitted.push_back(str.substr(last));\n  return splitted;\n}\n\n}  // namespace details\n\nnamespace contrib {\n\n// needs to be same with\n//   https://github.com/apache/mxnet/blob/1c874cfc807cee755c38f6486e8e0f4d94416cd8/src/operator/subgraph/tensorrt/tensorrt-inl.h#L190\nstatic const std::string TENSORRT_SUBGRAPH_PARAM_IDENTIFIER = \"subgraph_params_names\";  // NOLINT\n// needs to be same with\n//   https://github.com/apache/mxnet/blob/master/src/operator/subgraph/tensorrt/tensorrt.cc#L244\nstatic const std::string TENSORRT_SUBGRAPH_PARAM_PREFIX = \"subgraph_param_\";  // NOLINT\n/*!\n * this is a mimic to\n * https://github.com/apache/mxnet/blob/master/python/mxnet/contrib/tensorrt.py#L37\n * @param symbol symbol that already called subgraph api\n * @param argParams original arg params, params needed by tensorrt will be removed after calling\n * this function\n * @param auxParams original aux params, params needed by tensorrt will be removed after calling\n * this function\n */\ninline void InitTensorRTParams(const mxnet::cpp::Symbol& symbol,\n                               std::map<std::string, mxnet::cpp::NDArray>* argParams,\n                               std::map<std::string, mxnet::cpp::NDArray>* auxParams) {\n  mxnet::cpp::Symbol internals = symbol.GetInternals();\n  mx_uint numSymbol            = internals.GetNumOutputs();\n  for (mx_uint i = 0; i < numSymbol; ++i) {\n    std::map<std::string, std::string> attrs = internals[i].ListAttributes();\n    if (attrs.find(TENSORRT_SUBGRAPH_PARAM_IDENTIFIER) != attrs.end()) {\n      std::string new_params_names;\n      std::map<std::string, mxnet::cpp::NDArray> tensorrtParams;\n      std::vector<std::string> keys =\n          details::split(attrs[TENSORRT_SUBGRAPH_PARAM_IDENTIFIER], \";\");\n      for (const auto& key : keys) {\n        if (argParams->find(key) != argParams->end()) {\n          new_params_names += key + \";\";\n          tensorrtParams[TENSORRT_SUBGRAPH_PARAM_PREFIX + key] = (*argParams)[key];\n          argParams->erase(key);\n        } else if (auxParams->find(key) != auxParams->end()) {\n          new_params_names += key + \";\";\n          tensorrtParams[TENSORRT_SUBGRAPH_PARAM_PREFIX + key] = (*auxParams)[key];\n          auxParams->erase(key);\n        }\n      }\n      std::map<std::string, std::string> new_attrs = {};\n      for (const auto& kv : tensorrtParams) {\n        // passing the ndarray address into TRT node attributes to get the weight\n        uint64_t address    = reinterpret_cast<uint64_t>(kv.second.GetHandle());\n        new_attrs[kv.first] = std::to_string(address);\n      }\n      if (!new_attrs.empty()) {\n        internals[i].SetAttributes(new_attrs);\n        internals[i].SetAttribute(TENSORRT_SUBGRAPH_PARAM_IDENTIFIER,\n                                  new_params_names.substr(0, new_params_names.length() - 1));\n      }\n    }\n  }\n}\n\n}  // namespace contrib\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_CONTRIB_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/executor.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file executor.h\n * \\brief executor definition\n * \\author Chuntao Hong, Zhang Chen\n */\n\n#ifndef MXNET_CPP_EXECUTOR_H_\n#define MXNET_CPP_EXECUTOR_H_\n\n#include <vector>\n#include <map>\n#include <set>\n#include <string>\n#include <algorithm>\n#include \"mxnet-cpp/base.h\"\n#include \"mxnet-cpp/symbol.h\"\n\nnamespace mxnet {\nnamespace cpp {\n\nclass Optimizer;\n\n/*!\n * \\brief Executor interface\n */\nclass Executor {\n public:\n  Executor(const Symbol& symbol,\n           Context context,\n           const std::vector<NDArray>& arg_arrays,\n           const std::vector<NDArray>& grad_arrays,\n           const std::vector<OpReqType>& grad_reqs,\n           const std::vector<NDArray>& aux_arrays,\n           const std::map<std::string, Context>& group_to_ctx = std::map<std::string, Context>(),\n           Executor* shared_exec                              = nullptr);\n  explicit Executor(const CachedOpHandle& h) {\n    handle_ = h;\n  }\n  /*!\n   * \\brief Perform a Forward operation of Operator\n   *  After this operation, user can get the result by using function head.\n   */\n  void Forward(bool is_train) {\n    std::vector<NDArrayHandle> arg_handles;\n    for (const auto& array : combined_arrays) {\n      arg_handles.push_back(array.GetHandle());\n    }\n    int prev_is_record  = 0;\n    int prev_train_mode = 0;\n    CHECK_EQ(MXAutogradSetIsRecording(1, &prev_is_record), 0);\n    if (is_train == true) {\n      CHECK_EQ(MXAutogradSetIsTraining(1, &prev_train_mode), 0);\n    }\n    std::vector<NDArrayHandle> output_handles;\n    std::transform(\n        outputs.begin(), outputs.end(), std::back_inserter(output_handles), [](NDArray& a) {\n          return a.GetHandle();\n        });\n    int out_size             = 0;\n    NDArrayHandle* out_array = nullptr;\n    CHECK_EQ(MXInvokeCachedOp(handle_,\n                              arg_handles.size(),\n                              arg_handles.data(),\n                              device_type,\n                              device_id,\n                              &out_size,\n                              &out_array,\n                              nullptr),\n             0);\n    outputs.clear();\n    outputs.reserve(out_size);\n    for (mx_uint i = 0; i < out_size; ++i) {\n      outputs.push_back(NDArray(out_array[i]));\n    }\n    int cur_train_mode = prev_train_mode;\n    int cur_is_record  = prev_is_record;\n    if (is_train == true) {\n      CHECK_EQ(MXAutogradSetIsTraining(cur_train_mode, &prev_train_mode), 0);\n    }\n    CHECK_EQ(MXAutogradSetIsRecording(cur_is_record, &prev_is_record), 0);\n  }\n  /*!\n   * \\brief Perform a Backward operation of the Operator.\n   *  This must be called after Forward.\n   *  After this operation, NDArrays specified by grad_in_args_store will be\n   *updated accordingly.\n   *  User is allowed to pass in an empty Array if the head node is\n   *  loss function and head gradeitn is not needed.\n   *\n   * \\param head_grads the gradient of head nodes to be backproped.\n   */\n  void Backward(const std::vector<NDArray>& head_grads = std::vector<NDArray>()) {\n    if (require_grad == true) {\n      if (outputs.size() == 0) {\n        Forward(false);\n      }\n      std::vector<NDArrayHandle> out_handles;\n      for (const auto& array : outputs) {\n        out_handles.push_back(array.GetHandle());\n      }\n      std::vector<NDArrayHandle> head_grads_;\n      for (auto d : head_grads) {\n        head_grads_.push_back(d.GetHandle());\n      }\n      if (head_grads_.size() > 0) {\n        CHECK_EQ(MXAutogradBackwardEx(out_handles.size(),\n                                      out_handles.data(),\n                                      head_grads_.data(),\n                                      0,\n                                      nullptr,\n                                      0,\n                                      0,\n                                      1,\n                                      nullptr,\n                                      nullptr),\n                 0);\n      } else {\n        CHECK_EQ(MXAutogradBackwardEx(out_handles.size(),\n                                      out_handles.data(),\n                                      nullptr,\n                                      0,\n                                      nullptr,\n                                      0,\n                                      0,\n                                      1,\n                                      nullptr,\n                                      nullptr),\n                 0);\n      }\n      grad_arrays.clear();\n      grad_arrays.reserve(arg_arrays.size());\n      for (const auto& array : arg_arrays) {\n        NDArrayHandle grad;\n        CHECK_EQ(MXNDArrayGetGrad(array.GetHandle(), &grad), 0);\n        grad_arrays.push_back(NDArray(grad));\n      }\n    }\n  }\n  // TODO(zhangchen-qinyinghua)\n  // To implement reshape function\n  void Reshape();\n  /*!\n   * \\brief destructor, free the handle\n   */\n  ~Executor() {\n    MXFreeCachedOp(handle_);\n  }\n  std::vector<NDArray> arg_arrays;\n  std::vector<NDArray> grad_arrays;\n  std::vector<NDArray> aux_arrays;\n  std::vector<NDArray> combined_arrays;\n  int device_type;\n  int device_id;\n  bool require_grad;\n  /*!\n   * \\brief arrays store the outputs of forward\n   */\n  std::vector<NDArray> outputs;\n  std::map<std::string, NDArray> arg_dict() {\n    return GetDict(symbol_.ListArguments(), arg_arrays);\n  }\n  std::map<std::string, NDArray> grad_dict() {\n    return GetDict(symbol_.ListArguments(), grad_arrays);\n  }\n  std::map<std::string, NDArray> aux_dict() {\n    return GetDict(symbol_.ListAuxiliaryStates(), aux_arrays);\n  }\n\n private:\n  Executor(const Executor& e);\n  Executor& operator=(const Executor& e);\n  CachedOpHandle handle_;\n  Symbol symbol_;\n  std::map<std::string, NDArray> GetDict(const std::vector<std::string>& names,\n                                         const std::vector<NDArray>& arrays) {\n    std::map<std::string, NDArray> ret;\n    std::set<std::string> name_set;\n    for (const auto& s : names) {\n      CHECK(name_set.find(s) == name_set.end()) << \"Duplicate names detected, \" << s;\n      name_set.insert(s);\n    }\n    CHECK_EQ(name_set.size(), arrays.size()) << \"names size not equal to arrays size\";\n    for (size_t i = 0; i < names.size(); ++i) {\n      ret[names[i]] = arrays[i];\n    }\n    return ret;\n  }\n};\n}  // namespace cpp\n}  // namespace mxnet\n#endif  // MXNET_CPP_EXECUTOR_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/executor.hpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file executor.hpp\n * \\brief implementation of the executor\n * \\author Zhang Chen, Chuntao Hong\n */\n\n#ifndef MXNET_CPP_EXECUTOR_HPP_\n#define MXNET_CPP_EXECUTOR_HPP_\n\n#include <vector>\n#include <map>\n#include <string>\n#include \"mxnet-cpp/executor.h\"\n#include \"mxnet-cpp/optimizer.h\"\n\n\nnamespace mxnet {\nnamespace cpp {\ninline Executor::Executor(const Symbol &symbol, Context context,\n                          const std::vector<NDArray> &arg_arrays,\n                          const std::vector<NDArray> &grad_arrays,\n                          const std::vector<OpReqType> &grad_reqs,\n                          const std::vector<NDArray> &aux_arrays,\n                          const std::map<std::string, Context> &group_to_ctx,\n                          Executor *shared_exec) {\n  this->arg_arrays = arg_arrays;\n  this->grad_arrays = grad_arrays;\n  this->aux_arrays = aux_arrays;\n  this->symbol_ = symbol;\n  this->device_type = context.GetDeviceType();\n  this->device_id = context.GetDeviceId();\n\n  std::vector<NDArrayHandle> arg_handles;\n  std::vector<NDArrayHandle> grad_handles;\n\n  CHECK_EQ(arg_arrays.size(), grad_arrays.size())\n      << \"Number of input arg_arrays is different from the number of input grad_arrays\";\n  for (int i = 0; i < arg_arrays.size(); i++) {\n    if (grad_arrays[i].GetShape().size() != 0) {\n      grad_handles.push_back(grad_arrays[i].GetHandle());\n      arg_handles.push_back(arg_arrays[i].GetHandle());\n    }\n  }\n\n  this->require_grad = false;\n  std::vector<mx_uint> grad_reqs_uint;\n  for (auto s : grad_reqs) {\n    if (s != OpReqType::kNullOp) {\n      this->require_grad = true;\n    }\n    grad_reqs_uint.push_back(s);\n  }\n  CHECK_EQ(MXAutogradMarkVariables(arg_handles.size(), arg_handles.data(),\n                                   grad_reqs_uint.data(), grad_handles.data()), 0);\n\n  std::map<std::string, NDArray> arg_map = arg_dict();\n  std::map<std::string, NDArray> aux_map = aux_dict();\n  const auto input_name_list = symbol_.ListInputs();\n  std::vector<NDArray> combined_arrays;\n  for (size_t i = 0; i < input_name_list.size(); ++i) {\n    const auto &input_name = input_name_list[i];\n    auto iter_arg = arg_map.find(input_name);\n    if (iter_arg != arg_map.end()) {\n      combined_arrays.push_back(iter_arg->second);\n    } else {\n      auto iter_aux = aux_map.find(input_name);\n      CHECK(iter_aux != aux_map.end())\n          << \"Can not find name in args array and aux array\";\n      combined_arrays.push_back(iter_aux->second);\n    }\n  }\n  this->combined_arrays = combined_arrays;\n\n  CHECK_EQ(MXCreateCachedOp(symbol.GetHandle(), 0, nullptr, nullptr, &handle_, false), 0);\n}\n\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_EXECUTOR_HPP_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/initializer.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file initializer.h\n * \\brief random initializer\n * \\author Zhang Chen\n */\n\n#ifndef MXNET_CPP_INITIALIZER_H_\n#define MXNET_CPP_INITIALIZER_H_\n\n#include <cmath>\n#include <string>\n#include <vector>\n#include <random>\n#include \"mxnet-cpp/ndarray.h\"\n\nnamespace mxnet {\nnamespace cpp {\n\nclass Initializer {\n public:\n  static bool StringStartWith(const std::string& name, const std::string& check_str) {\n    return (name.size() >= check_str.size() && name.substr(0, check_str.size()) == check_str);\n  }\n  static bool StringEndWith(const std::string& name, const std::string& check_str) {\n    return (name.size() >= check_str.size() &&\n            name.substr(name.size() - check_str.size(), check_str.size()) == check_str);\n  }\n  virtual void operator()(const std::string& name, NDArray* arr) {\n    if (StringStartWith(name, \"upsampling\")) {\n      InitBilinear(arr);\n    } else if (StringEndWith(name, \"bias\")) {\n      InitBias(arr);\n    } else if (StringEndWith(name, \"gamma\")) {\n      InitGamma(arr);\n    } else if (StringEndWith(name, \"beta\")) {\n      InitBeta(arr);\n    } else if (StringEndWith(name, \"weight\")) {\n      InitWeight(arr);\n    } else if (StringEndWith(name, \"moving_mean\")) {\n      InitZero(arr);\n    } else if (StringEndWith(name, \"moving_var\")) {\n      InitOne(arr);\n    } else if (StringEndWith(name, \"moving_inv_var\")) {\n      InitZero(arr);\n    } else if (StringEndWith(name, \"moving_avg\")) {\n      InitZero(arr);\n    } else if (StringEndWith(name, \"min\")) {\n      InitZero(arr);\n    } else if (StringEndWith(name, \"max\")) {\n      InitOne(arr);\n    } else if (StringEndWith(name, \"weight_quantize\")) {\n      InitQuantizedWeight(arr);\n    } else if (StringEndWith(name, \"bias_quantize\")) {\n      InitQuantizedBias(arr);\n    } else {\n      InitDefault(arr);\n    }\n  }\n\n protected:\n  virtual void InitBilinear(NDArray* arr) {\n    Shape shape(arr->GetShape());\n    std::vector<float> weight(shape.Size(), 0);\n    int f   = std::ceil(shape[3] / 2.0);\n    float c = (2 * f - 1 - f % 2) / (2. * f);\n    for (size_t i = 0; i < shape.Size(); ++i) {\n      int x     = i % shape[3];\n      int y     = (i / shape[3]) % shape[2];\n      weight[i] = (1 - std::abs(x / f - c)) * (1 - std::abs(y / f - c));\n    }\n    (*arr).SyncCopyFromCPU(weight);\n  }\n  virtual void InitZero(NDArray* arr) {\n    (*arr) = 0.0f;\n  }\n  virtual void InitOne(NDArray* arr) {\n    (*arr) = 1.0f;\n  }\n  virtual void InitBias(NDArray* arr) {\n    (*arr) = 0.0f;\n  }\n  virtual void InitGamma(NDArray* arr) {\n    (*arr) = 1.0f;\n  }\n  virtual void InitBeta(NDArray* arr) {\n    (*arr) = 0.0f;\n  }\n  virtual void InitWeight(NDArray* arr) {}\n  virtual void InitQuantizedWeight(NDArray* arr) {\n    std::default_random_engine generator;\n    std::uniform_int_distribution<int32_t> _val(-127, 127);\n    (*arr) = _val(generator);\n  }\n  virtual void InitQuantizedBias(NDArray* arr) {\n    (*arr) = 0;\n  }\n  virtual void InitDefault(NDArray* arr) {}\n};\n\nclass Constant : public Initializer {\n public:\n  explicit Constant(float value) : value(value) {}\n  void operator()(const std::string& name, NDArray* arr) override {\n    (*arr) = value;\n  }\n\n protected:\n  float value;\n};\n\nclass Zero : public Constant {\n public:\n  Zero() : Constant(0.0f) {}\n};\n\nclass One : public Constant {\n public:\n  One() : Constant(1.0f) {}\n};\n\nclass Uniform : public Initializer {\n public:\n  explicit Uniform(float scale) : Uniform(-scale, scale) {}\n  Uniform(float begin, float end) : begin(begin), end(end) {}\n  void operator()(const std::string& name, NDArray* arr) override {\n    if (StringEndWith(name, \"weight_quantize\")) {\n      InitQuantizedWeight(arr);\n      return;\n    }\n    if (StringEndWith(name, \"bias_quantize\")) {\n      InitQuantizedBias(arr);\n      return;\n    }\n    NDArray::SampleUniform(begin, end, arr);\n  }\n\n protected:\n  float begin, end;\n};\n\nclass Normal : public Initializer {\n public:\n  Normal(float mu, float sigma) : mu(mu), sigma(sigma) {}\n  void operator()(const std::string& name, NDArray* arr) override {\n    if (StringEndWith(name, \"weight_quantize\")) {\n      InitQuantizedWeight(arr);\n      return;\n    }\n    if (StringEndWith(name, \"bias_quantize\")) {\n      InitQuantizedBias(arr);\n      return;\n    }\n    NDArray::SampleGaussian(mu, sigma, arr);\n  }\n\n protected:\n  float mu, sigma;\n};\n\nclass Bilinear : public Initializer {\n public:\n  Bilinear() {}\n  void operator()(const std::string& name, NDArray* arr) override {\n    if (StringEndWith(name, \"weight_quantize\")) {\n      InitQuantizedWeight(arr);\n      return;\n    }\n    if (StringEndWith(name, \"bias_quantize\")) {\n      InitQuantizedBias(arr);\n      return;\n    }\n    InitBilinear(arr);\n  }\n};\n\nclass Xavier : public Initializer {\n public:\n  enum RandType { gaussian, uniform } rand_type;\n  enum FactorType { avg, in, out } factor_type;\n  float magnitude;\n  Xavier(RandType rand_type     = gaussian,  // NOLINT\n         FactorType factor_type = avg,       // NOLINT\n         float magnitude        = 3)                // NOLINT\n      : rand_type(rand_type), factor_type(factor_type), magnitude(magnitude) {}\n\n  void operator()(const std::string& name, NDArray* arr) override {\n    if (StringEndWith(name, \"weight_quantize\")) {\n      InitQuantizedWeight(arr);\n      return;\n    }\n    if (StringEndWith(name, \"bias_quantize\")) {\n      InitQuantizedBias(arr);\n      return;\n    }\n\n    Shape shape(arr->GetShape());\n    float hw_scale = 1.0f;\n    if (shape.ndim() > 2) {\n      for (size_t i = 2; i < shape.ndim(); ++i) {\n        hw_scale *= shape[i];\n      }\n    }\n    float fan_in = shape[1] * hw_scale, fan_out = shape[0] * hw_scale;\n    float factor = 1.0f;\n    switch (factor_type) {\n      case avg:\n        factor = (fan_in + fan_out) / 2.0;\n        break;\n      case in:\n        factor = fan_in;\n        break;\n      case out:\n        factor = fan_out;\n    }\n    float scale = std::sqrt(magnitude / factor);\n    switch (rand_type) {\n      case uniform:\n        NDArray::SampleUniform(-scale, scale, arr);\n        break;\n      case gaussian:\n        NDArray::SampleGaussian(0, scale, arr);\n        break;\n    }\n  }\n};\n\nclass MSRAPrelu : public Xavier {\n public:\n  explicit MSRAPrelu(FactorType factor_type = avg, float slope = 0.25f)\n      : Xavier(gaussian, factor_type, 2. / (1 + slope * slope)) {}\n};\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_INITIALIZER_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/io.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file operator.h\n * \\brief definition of io, such as DataIter\n * \\author Zhang Chen\n */\n#ifndef MXNET_CPP_IO_H_\n#define MXNET_CPP_IO_H_\n\n#include <map>\n#include <string>\n#include <vector>\n#include <sstream>\n#include \"mxnet-cpp/base.h\"\n#include \"mxnet-cpp/ndarray.h\"\n#include \"dmlc/logging.h\"\n\nnamespace mxnet {\nnamespace cpp {\n/*!\n * \\brief Default object for holding a mini-batch of data and related\n * information.\n */\nclass DataBatch {\n public:\n  NDArray data;\n  NDArray label;\n  int pad_num;\n  std::vector<int> index;\n};\nclass DataIter {\n public:\n  virtual void BeforeFirst(void)          = 0;\n  virtual bool Next(void)                 = 0;\n  virtual NDArray GetData(void)           = 0;\n  virtual NDArray GetLabel(void)          = 0;\n  virtual int GetPadNum(void)             = 0;\n  virtual std::vector<int> GetIndex(void) = 0;\n\n  DataBatch GetDataBatch() {\n    return DataBatch{GetData(), GetLabel(), GetPadNum(), GetIndex()};\n  }\n  void Reset() {\n    BeforeFirst();\n  }\n\n  virtual ~DataIter() = default;\n};\n\nclass MXDataIterMap {\n public:\n  inline MXDataIterMap() {\n    mx_uint num_data_iter_creators      = 0;\n    DataIterCreator* data_iter_creators = nullptr;\n    int r = MXListDataIters(&num_data_iter_creators, &data_iter_creators);\n    CHECK_EQ(r, 0);\n    for (mx_uint i = 0; i < num_data_iter_creators; i++) {\n      const char* name;\n      const char* description;\n      mx_uint num_args;\n      const char** arg_names;\n      const char** arg_type_infos;\n      const char** arg_descriptions;\n      r = MXDataIterGetIterInfo(data_iter_creators[i],\n                                &name,\n                                &description,\n                                &num_args,\n                                &arg_names,\n                                &arg_type_infos,\n                                &arg_descriptions);\n      CHECK_EQ(r, 0);\n      mxdataiter_creators_[name] = data_iter_creators[i];\n    }\n  }\n  inline DataIterCreator GetMXDataIterCreator(const std::string& name) {\n    return mxdataiter_creators_[name];\n  }\n\n private:\n  std::map<std::string, DataIterCreator> mxdataiter_creators_;\n};\n\nstruct MXDataIterBlob {\n public:\n  MXDataIterBlob() : handle_(nullptr) {}\n  explicit MXDataIterBlob(DataIterHandle handle) : handle_(handle) {}\n  ~MXDataIterBlob() {\n    MXDataIterFree(handle_);\n  }\n  DataIterHandle handle_;\n\n private:\n  MXDataIterBlob& operator=(const MXDataIterBlob&);\n};\n\nclass MXDataIter : public DataIter {\n public:\n  explicit MXDataIter(const std::string& mxdataiter_type);\n  MXDataIter(const MXDataIter& other) {\n    creator_  = other.creator_;\n    params_   = other.params_;\n    blob_ptr_ = other.blob_ptr_;\n  }\n  void BeforeFirst();\n  bool Next();\n  NDArray GetData();\n  NDArray GetLabel();\n  int GetPadNum();\n  std::vector<int> GetIndex();\n  MXDataIter CreateDataIter();\n  /*!\n   * \\brief set config parameters\n   * \\param name name of the config parameter\n   * \\param value value of the config parameter\n   * \\return reference of self\n   */\n  template <typename T>\n  MXDataIter& SetParam(const std::string& name, const T& value) {\n    std::string value_str;\n    std::stringstream ss;\n    ss << value;\n    ss >> value_str;\n\n    params_[name] = value_str;\n    return *this;\n  }\n\n private:\n  DataIterCreator creator_;\n  std::map<std::string, std::string> params_;\n  std::shared_ptr<MXDataIterBlob> blob_ptr_;\n  static MXDataIterMap*& mxdataiter_map();\n};\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_IO_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/io.hpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n* \\file operator.hpp\n* \\brief implementation of data iter\n* \\author Zhang Chen\n*/\n#ifndef MXNET_CPP_IO_HPP_\n#define MXNET_CPP_IO_HPP_\n\n#include <string>\n#include <vector>\n#include \"mxnet-cpp/io.h\"\n\nnamespace mxnet {\nnamespace cpp {\n\ninline MXDataIterMap*& MXDataIter::mxdataiter_map() {\n    static MXDataIterMap* mxdataiter_map_ = new MXDataIterMap;\n    return mxdataiter_map_;\n}\n\ninline MXDataIter::MXDataIter(const std::string &mxdataiter_type) {\n  creator_ = mxdataiter_map()->GetMXDataIterCreator(mxdataiter_type);\n  blob_ptr_ = std::make_shared<MXDataIterBlob>(nullptr);\n}\n\ninline void MXDataIter::BeforeFirst() {\n  int r = MXDataIterBeforeFirst(blob_ptr_->handle_);\n  CHECK_EQ(r, 0);\n}\n\ninline bool MXDataIter::Next() {\n  int out;\n  int r = MXDataIterNext(blob_ptr_->handle_, &out);\n  CHECK_EQ(r, 0);\n  return out;\n}\n\ninline NDArray MXDataIter::GetData() {\n  NDArrayHandle handle;\n  int r = MXDataIterGetData(blob_ptr_->handle_, &handle);\n  CHECK_EQ(r, 0);\n  return NDArray(handle);\n}\n\ninline NDArray MXDataIter::GetLabel() {\n  NDArrayHandle handle;\n  int r = MXDataIterGetLabel(blob_ptr_->handle_, &handle);\n  CHECK_EQ(r, 0);\n  return NDArray(handle);\n}\n\ninline int MXDataIter::GetPadNum() {\n  int out;\n  int r = MXDataIterGetPadNum(blob_ptr_->handle_, &out);\n  CHECK_EQ(r, 0);\n  return out;\n}\ninline std::vector<int> MXDataIter::GetIndex() {\n  uint64_t *out_index, out_size;\n  int r = MXDataIterGetIndex(blob_ptr_->handle_, &out_index, &out_size);\n  CHECK_EQ(r, 0);\n  std::vector<int> ret;\n  for (uint64_t i = 0; i < out_size; ++i) {\n    ret.push_back(out_index[i]);\n  }\n  return ret;\n}\n\ninline MXDataIter MXDataIter::CreateDataIter() {\n  std::vector<const char *> param_keys;\n  std::vector<const char *> param_values;\n\n  for (auto &data : params_) {\n    param_keys.push_back(data.first.c_str());\n    param_values.push_back(data.second.c_str());\n  }\n\n  MXDataIterCreateIter(creator_, param_keys.size(), param_keys.data(),\n                       param_values.data(), &blob_ptr_->handle_);\n  return *this;\n}\n\n// MXDataIter MNIst\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_IO_HPP_\n\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/kvstore.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file kvstore.h\n * \\brief definition of kvstore\n * \\author Chuntao Hong\n */\n\n#ifndef MXNET_CPP_KVSTORE_H_\n#define MXNET_CPP_KVSTORE_H_\n\n#include <string>\n#include <vector>\n#include \"mxnet-cpp/ndarray.h\"\n\nnamespace mxnet {\nnamespace cpp {\n\nclass KVStore {\n public:\n  static void SetType(const std::string& type);\n  static void RunServer();\n  static void Init(int key, const NDArray& val);\n  static void Init(const std::string& key, const NDArray& val);\n  static void Init(const std::vector<int>& keys, const std::vector<NDArray>& vals);\n  static void Init(const std::vector<std::string>& keys, const std::vector<NDArray>& vals);\n  static void Push(int key, const NDArray& val, int priority = 0);\n  static void Push(const std::string& key, const NDArray& val, int priority = 0);\n  static void Push(const std::vector<int>& keys,\n                   const std::vector<NDArray>& vals,\n                   int priority = 0);\n  static void Push(const std::vector<std::string>& keys,\n                   const std::vector<NDArray>& vals,\n                   int priority = 0);\n  static void Pull(int key, NDArray* out, int priority = 0);\n  static void Pull(const std::string& key, NDArray* out, int priority = 0);\n  static void Pull(const std::vector<int>& keys, std::vector<NDArray>* outs, int priority = 0);\n  static void Pull(const std::vector<std::string>& keys,\n                   std::vector<NDArray>* outs,\n                   int priority = 0);\n  // TODO(lx): put lr in optimizer or not?\n  static void SetOptimizer(std::unique_ptr<Optimizer> optimizer, bool local = false);\n  static std::string GetType();\n  static int GetRank();\n  static int GetNumWorkers();\n  static void Barrier();\n  static std::string GetRole();\n\n private:\n  KVStore();\n  static KVStoreHandle& get_handle();\n  static std::unique_ptr<Optimizer>& get_optimizer();\n  static KVStore*& get_kvstore();\n  static void Controller(int head, const char* body, void* controller_handle);\n  static void Updater(int key, NDArrayHandle recv, NDArrayHandle local, void* handle_);\n};\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_KVSTORE_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/kvstore.hpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file kvstore.hpp\n * \\brief implementation of kvstore\n * \\author Xin Li\n */\n\n#include <algorithm>\n#include <map>\n#include <numeric>\n#include <string>\n#include <vector>\n\n#include \"mxnet-cpp/kvstore.h\"\n#include \"mxnet-cpp/optimizer.h\"\n\n#ifndef MXNET_CPP_KVSTORE_HPP_\n#define MXNET_CPP_KVSTORE_HPP_\n\nnamespace mxnet {\nnamespace cpp {\n\ninline void KVStore::Controller(int head, const char* body, void* controller_handle) {\n  if (head == 0) {\n    std::map<std::string, std::string> params;\n    std::istringstream sin(body);\n    std::string line;\n    while (getline(sin, line)) {\n      size_t n = line.find('=');\n      params.emplace(line.substr(0, n), line.substr(n+1));\n    }\n    std::unique_ptr<Optimizer> opt(OptimizerRegistry::Find(params.at(\"opt_type\")));\n    params.erase(\"opt_type\");\n    for (const auto& pair : params) {\n      opt->SetParam(pair.first, pair.second);\n    }\n    get_kvstore()->SetOptimizer(std::move(opt), true);\n  }\n}\n\ninline KVStoreHandle& KVStore::get_handle() {\n  static KVStoreHandle handle_ = nullptr;\n  return handle_;\n}\n\ninline std::unique_ptr<Optimizer>& KVStore::get_optimizer() {\n  static std::unique_ptr<Optimizer> optimizer_;\n  return optimizer_;\n}\n\ninline KVStore*& KVStore::get_kvstore() {\n  static KVStore* kvstore_ = new KVStore;\n  return kvstore_;\n}\n\ninline KVStore::KVStore() {}\n\ninline void KVStore::SetType(const std::string& type) {\n  CHECK_EQ(MXKVStoreCreate(type.c_str(), &(get_kvstore()->get_handle())), 0);\n}\n\ninline void KVStore::RunServer() {\n  CHECK_NE(GetRole(), \"worker\");\n  CHECK_EQ(MXKVStoreRunServer(get_kvstore()->get_handle(), &Controller, 0), 0);\n}\n\ninline void KVStore::Init(int key, const NDArray& val) {\n  NDArrayHandle val_handle = val.GetHandle();\n  CHECK_EQ(MXKVStoreInit(get_kvstore()->get_handle(), 1, &key, &val_handle), 0);\n}\n\ninline void KVStore::Init(const std::string& key, const NDArray& val) {\n  const char* key_handle = key.c_str();\n  NDArrayHandle val_handle = val.GetHandle();\n  CHECK_EQ(MXKVStoreInitEx(get_kvstore()->get_handle(), 1, &key_handle, &val_handle), 0);\n}\n\ninline void KVStore::Init(const std::vector<int>& keys, const std::vector<NDArray>& vals) {\n  CHECK_EQ(keys.size(), vals.size());\n  std::vector<NDArrayHandle> val_handles(vals.size());\n  std::transform(vals.cbegin(), vals.cend(), val_handles.begin(),\n      [](const NDArray& val) {\n        return val.GetHandle();\n      });\n\n  CHECK_EQ(MXKVStoreInit(get_kvstore()->get_handle(), keys.size(), keys.data(),\n      val_handles.data()), 0);\n}\n\ninline void KVStore::Init(const std::vector<std::string>& keys, const std::vector<NDArray>& vals) {\n  CHECK_EQ(keys.size(), vals.size());\n  std::vector<const char*> key_handles(keys.size());\n  std::transform(keys.cbegin(), keys.cend(), key_handles.begin(),\n      [](const std::string& key) {\n        return key.c_str();\n      });\n  std::vector<NDArrayHandle> val_handles(vals.size());\n  std::transform(vals.cbegin(), vals.cend(), val_handles.begin(),\n      [](const NDArray& val) {\n        return val.GetHandle();\n      });\n\n  CHECK_EQ(MXKVStoreInitEx(get_kvstore()->get_handle(), key_handles.size(), key_handles.data(),\n      val_handles.data()), 0);\n}\n\ninline void KVStore::Push(int key, const NDArray& val, int priority) {\n  NDArrayHandle val_handle = val.GetHandle();\n  CHECK_EQ(MXKVStorePush(get_kvstore()->get_handle(), 1, &key, &val_handle, priority), 0);\n}\n\ninline void KVStore::Push(const std::string& key, const NDArray& val, int priority) {\n  const char* key_handle = key.c_str();\n  NDArrayHandle val_handle = val.GetHandle();\n  CHECK_EQ(MXKVStorePushEx(get_kvstore()->get_handle(), 1, &key_handle, &val_handle, priority), 0);\n}\n\ninline void KVStore::Push(const std::vector<int>& keys,\n                          const std::vector<NDArray>& vals, int priority) {\n  CHECK_EQ(keys.size(), vals.size());\n  std::vector<NDArrayHandle> val_handles(vals.size());\n  std::transform(vals.cbegin(), vals.cend(), val_handles.begin(),\n      [](const NDArray& val) {\n        return val.GetHandle();\n      });\n\n  CHECK_EQ(MXKVStorePush(get_kvstore()->get_handle(), keys.size(), keys.data(),\n      val_handles.data(), priority), 0);\n}\n\ninline void KVStore::Push(const std::vector<std::string>& keys,\n                          const std::vector<NDArray>& vals, int priority) {\n  CHECK_EQ(keys.size(), vals.size());\n  std::vector<const char*> key_handles(keys.size());\n  std::transform(keys.cbegin(), keys.cend(), key_handles.begin(),\n      [](const std::string& key) {\n        return key.c_str();\n      });\n  std::vector<NDArrayHandle> val_handles(vals.size());\n  std::transform(vals.cbegin(), vals.cend(), val_handles.begin(),\n      [](const NDArray& val) {\n        return val.GetHandle();\n      });\n\n  CHECK_EQ(MXKVStorePushEx(get_kvstore()->get_handle(), key_handles.size(), key_handles.data(),\n      val_handles.data(), priority), 0);\n}\n\ninline void KVStore::Pull(int key, NDArray* out, int priority) {\n  NDArrayHandle out_handle = out->GetHandle();\n  CHECK_EQ(MXKVStorePull(get_kvstore()->get_handle(), 1, &key, &out_handle, priority), 0);\n}\n\ninline void KVStore::Pull(const std::string& key, NDArray* out, int priority) {\n  const char* key_handle = key.c_str();\n  NDArrayHandle out_handle = out->GetHandle();\n  CHECK_EQ(MXKVStorePullEx(get_kvstore()->get_handle(), 1, &key_handle, &out_handle, priority), 0);\n}\n\ninline void KVStore::Pull(const std::vector<int>& keys,\n                          std::vector<NDArray>* outs, int priority) {\n  CHECK_EQ(keys.size(), outs->size());\n\n  std::vector<NDArrayHandle> out_handles(keys.size());\n  std::transform(outs->cbegin(), outs->cend(), out_handles.begin(),\n      [](const NDArray& val) {\n        return val.GetHandle();\n      });\n\n  CHECK_EQ(MXKVStorePull(get_kvstore()->get_handle(), keys.size(), keys.data(),\n      out_handles.data(), priority), 0);\n}\n\ninline void KVStore::Pull(const std::vector<std::string>& keys,\n                          std::vector<NDArray>* outs, int priority) {\n  CHECK_EQ(keys.size(), outs->size());\n\n  std::vector<const char*> key_handles(keys.size());\n  std::transform(keys.cbegin(), keys.cend(), key_handles.begin(),\n      [](const std::string& key) {\n        return key.c_str();\n      });\n  std::vector<NDArrayHandle> out_handles(keys.size());\n  std::transform(outs->cbegin(), outs->cend(), out_handles.begin(),\n      [](const NDArray& val) {\n        return val.GetHandle();\n      });\n\n  CHECK_EQ(MXKVStorePullEx(get_kvstore()->get_handle(), key_handles.size(), key_handles.data(),\n      out_handles.data(), priority), 0);\n}\n\ninline void KVStore::Updater(int key, NDArrayHandle recv, NDArrayHandle local,\n                             void* handle_) {\n  Optimizer *opt = static_cast<Optimizer*>(handle_);\n  opt->Update(key, NDArray(local), NDArray(recv));\n}\n\ninline void KVStore::SetOptimizer(std::unique_ptr<Optimizer> optimizer, bool local) {\n  if (local) {\n    get_kvstore()->get_optimizer() = std::move(optimizer);\n    CHECK_EQ(MXKVStoreSetUpdater(get_kvstore()->get_handle(),\n                                 &Updater, get_kvstore()->get_optimizer().get()), 0);\n  } else {\n    CHECK_EQ(MXKVStoreSendCommmandToServers(get_kvstore()->get_handle(), 0,\n                                            (*optimizer).Serialize().c_str()), 0);\n  }\n}\n\ninline std::string KVStore::GetType() {\n  const char *type;\n  CHECK_EQ(MXKVStoreGetType(get_kvstore()->get_handle(), &type), 0);\n  return type;\n}\n\ninline int KVStore::GetRank() {\n  int rank;\n  CHECK_EQ(MXKVStoreGetRank(get_kvstore()->get_handle(), &rank), 0);\n  return rank;\n}\n\ninline int KVStore::GetNumWorkers() {\n  int num_workers;\n  CHECK_EQ(MXKVStoreGetGroupSize(get_kvstore()->get_handle(), &num_workers), 0);\n  return num_workers;\n}\n\ninline void KVStore::Barrier() {\n  CHECK_EQ(MXKVStoreBarrier(get_kvstore()->get_handle()), 0);\n}\n\ninline std::string KVStore::GetRole() {\n  int ret;\n  CHECK_EQ(MXKVStoreIsSchedulerNode(&ret), 0);\n  if (ret) {\n    return \"scheduler\";\n  }\n  CHECK_EQ(MXKVStoreIsServerNode(&ret), 0);\n  if (ret) {\n    return \"server\";\n  }\n  CHECK_EQ(MXKVStoreIsWorkerNode(&ret), 0);\n  CHECK(ret);\n  return \"worker\";\n}\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_KVSTORE_HPP_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/lr_scheduler.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file lr_scheduler.h\n * \\brief Scheduling learning rate\n */\n\n#ifndef MXNET_CPP_LR_SCHEDULER_H_\n#define MXNET_CPP_LR_SCHEDULER_H_\n\n#include \"dmlc/logging.h\"\n\nnamespace mxnet {\nnamespace cpp {\n\n/*!\n * \\brief lr scheduler interface\n */\nclass LRScheduler {\n public:\n  /*!\n   * \\brief constructor\n   * \\param base_lr the initial learning rate.\n   */\n  explicit LRScheduler(float base_lr = 0.01) : base_lr_(base_lr) {}\n  /*!\n   * \\brief set base lr\n   * \\param lr learning rate from optimizer\n   */\n  void SetLR(const float lr) {\n    base_lr_ = lr;\n  }\n  /*!\n   * \\brief get a new learning rate\n   */\n  virtual float GetLR(unsigned num_update) = 0;\n  /*!\n   * \\brief destructor\n   */\n  virtual ~LRScheduler() {}\n\n protected:\n  float base_lr_;\n};\n\nclass FactorScheduler : public LRScheduler {\n public:\n  explicit FactorScheduler(int step, float factor = 1, float stop_factor_lr = 1e-8)\n      : LRScheduler() {\n    step_           = step;\n    factor_         = factor;\n    stop_factor_lr_ = stop_factor_lr;\n  }\n\n  float GetLR(unsigned num_update) override {\n    while (num_update > unsigned(count_ + step_)) {\n      count_ += step_;\n      base_lr_ *= factor_;\n      if (base_lr_ < stop_factor_lr_) {\n        base_lr_ = stop_factor_lr_;\n        LG << \"Update[\" << num_update << \"]: now learning rate arrived at \" << base_lr_\n           << \", will not change in the future\";\n      } else {\n        LG << \"Update[\" << num_update << \"]: Change learning rate to \" << base_lr_;\n      }\n    }\n    return base_lr_;\n  }\n\n private:\n  int count_ = 0;\n  int step_;\n  float factor_;\n  float stop_factor_lr_;\n};\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_LR_SCHEDULER_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/metric.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file base.h\n * \\brief metrics defined\n * \\author Zhang Chen\n */\n\n#ifndef MXNET_CPP_METRIC_H_\n#define MXNET_CPP_METRIC_H_\n\n#include <cmath>\n#include <string>\n#include <vector>\n#include <algorithm>\n#include \"mxnet-cpp/ndarray.h\"\n#include \"dmlc/logging.h\"\n\nnamespace mxnet {\nnamespace cpp {\n\nclass EvalMetric {\n public:\n  explicit EvalMetric(const std::string& name, int num = 0) : name(name), num(num) {}\n  virtual void Update(NDArray labels, NDArray preds) = 0;\n  void Reset() {\n    num_inst   = 0;\n    sum_metric = 0.0f;\n  }\n  float Get() {\n    return sum_metric / num_inst;\n  }\n  void GetNameValue();\n\n protected:\n  std::string name;\n  int num;\n  float sum_metric = 0.0f;\n  int num_inst     = 0;\n\n  static void CheckLabelShapes(NDArray labels, NDArray preds, bool strict = false) {\n    if (strict) {\n      CHECK_EQ(Shape(labels.GetShape()), Shape(preds.GetShape()));\n    } else {\n      CHECK_EQ(labels.Size(), preds.Size());\n    }\n  }\n};\n\nclass Accuracy : public EvalMetric {\n public:\n  Accuracy() : EvalMetric(\"accuracy\") {}\n\n  void Update(NDArray labels, NDArray preds) override {\n    CHECK_EQ(labels.GetShape().size(), 1);\n    mx_uint len = labels.GetShape()[0];\n    std::vector<mx_float> pred_data(len);\n    std::vector<mx_float> label_data(len);\n    preds.ArgmaxChannel().SyncCopyToCPU(&pred_data, len);\n    labels.SyncCopyToCPU(&label_data, len);\n    for (mx_uint i = 0; i < len; ++i) {\n      sum_metric += (pred_data[i] == label_data[i]) ? 1 : 0;\n      num_inst += 1;\n    }\n  }\n};\n\nclass LogLoss : public EvalMetric {\n public:\n  LogLoss() : EvalMetric(\"logloss\") {}\n\n  void Update(NDArray labels, NDArray preds) override {\n    static const float epsilon = 1e-15;\n    mx_uint len                = labels.GetShape()[0];\n    mx_uint m                  = preds.GetShape()[1];\n    std::vector<mx_float> pred_data(len * m);\n    std::vector<mx_float> label_data(len);\n    preds.SyncCopyToCPU(&pred_data, pred_data.size());\n    labels.SyncCopyToCPU(&label_data, len);\n    for (mx_uint i = 0; i < len; ++i) {\n      sum_metric += -std::log(std::max(pred_data[i * m + label_data[i]], epsilon));\n      num_inst += 1;\n    }\n  }\n};\n\nclass MAE : public EvalMetric {\n public:\n  MAE() : EvalMetric(\"mae\") {}\n\n  void Update(NDArray labels, NDArray preds) override {\n    CheckLabelShapes(labels, preds);\n\n    std::vector<mx_float> pred_data;\n    preds.SyncCopyToCPU(&pred_data);\n    std::vector<mx_float> label_data;\n    labels.SyncCopyToCPU(&label_data);\n\n    size_t len   = preds.Size();\n    mx_float sum = 0;\n    for (size_t i = 0; i < len; ++i) {\n      sum += std::abs(pred_data[i] - label_data[i]);\n    }\n    sum_metric += sum / len;\n    ++num_inst;\n  }\n};\n\nclass MSE : public EvalMetric {\n public:\n  MSE() : EvalMetric(\"mse\") {}\n\n  void Update(NDArray labels, NDArray preds) override {\n    CheckLabelShapes(labels, preds);\n\n    std::vector<mx_float> pred_data;\n    preds.SyncCopyToCPU(&pred_data);\n    std::vector<mx_float> label_data;\n    labels.SyncCopyToCPU(&label_data);\n\n    size_t len   = preds.Size();\n    mx_float sum = 0;\n    for (size_t i = 0; i < len; ++i) {\n      mx_float diff = pred_data[i] - label_data[i];\n      sum += diff * diff;\n    }\n    sum_metric += sum / len;\n    ++num_inst;\n  }\n};\n\nclass RMSE : public EvalMetric {\n public:\n  RMSE() : EvalMetric(\"rmse\") {}\n\n  void Update(NDArray labels, NDArray preds) override {\n    CheckLabelShapes(labels, preds);\n\n    std::vector<mx_float> pred_data;\n    preds.SyncCopyToCPU(&pred_data);\n    std::vector<mx_float> label_data;\n    labels.SyncCopyToCPU(&label_data);\n\n    size_t len   = preds.Size();\n    mx_float sum = 0;\n    for (size_t i = 0; i < len; ++i) {\n      mx_float diff = pred_data[i] - label_data[i];\n      sum += diff * diff;\n    }\n    sum_metric += std::sqrt(sum / len);\n    ++num_inst;\n  }\n};\n\nclass PSNR : public EvalMetric {\n public:\n  PSNR() : EvalMetric(\"psnr\") {}\n\n  void Update(NDArray labels, NDArray preds) override {\n    CheckLabelShapes(labels, preds);\n\n    std::vector<mx_float> pred_data;\n    preds.SyncCopyToCPU(&pred_data);\n    std::vector<mx_float> label_data;\n    labels.SyncCopyToCPU(&label_data);\n\n    size_t len   = preds.Size();\n    mx_float sum = 0;\n    for (size_t i = 0; i < len; ++i) {\n      mx_float diff = pred_data[i] - label_data[i];\n      sum += diff * diff;\n    }\n    mx_float mse = sum / len;\n    if (mse > 0) {\n      sum_metric += 10 * std::log(255.0f / mse) / log10_;\n    } else {\n      sum_metric += 99.0f;\n    }\n    ++num_inst;\n  }\n\n private:\n  mx_float log10_ = std::log(10.0f);\n};\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_METRIC_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/model.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file model.h\n * \\brief MXNET.cpp model module\n * \\author Zhang Chen\n */\n\n#ifndef MXNET_CPP_MODEL_H_\n#define MXNET_CPP_MODEL_H_\n\n#include <string>\n#include <vector>\n#include \"mxnet-cpp/base.h\"\n#include \"mxnet-cpp/symbol.h\"\n#include \"mxnet-cpp/ndarray.h\"\n\nnamespace mxnet {\nnamespace cpp {\n\nstruct FeedForwardConfig {\n  Symbol symbol;\n  std::vector<Context> ctx = {Context::cpu()};\n  int num_epoch            = 0;\n  int epoch_size           = 0;\n  std::string optimizer    = \"sgd\";\n  // TODO(zhangchen-qinyinghua) More implement\n  // initializer=Uniform(0.01),\n  // numpy_batch_size=128,\n  // arg_params=None, aux_params=None,\n  // allow_extra_params=False,\n  // begin_epoch=0,\n  // **kwargs):\n  FeedForwardConfig(const FeedForwardConfig& other) {}\n  FeedForwardConfig() {}\n};\nclass FeedForward {\n public:\n  explicit FeedForward(const FeedForwardConfig& conf) : conf_(conf) {}\n  void Predict();\n  void Score();\n  void Fit();\n  void Save();\n  void Load();\n  static FeedForward Create();\n\n private:\n  void InitParams();\n  void InitPredictor();\n  void InitIter();\n  void InitEvalIter();\n  FeedForwardConfig conf_;\n};\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_MODEL_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/ndarray.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file ndarray.h\n * \\brief definition of ndarray\n * \\author Chuntao Hong, Zhang Chen\n */\n\n#ifndef MXNET_CPP_NDARRAY_H_\n#define MXNET_CPP_NDARRAY_H_\n\n#include <map>\n#include <memory>\n#include <string>\n#include <vector>\n#include <iostream>\n#include \"mxnet-cpp/base.h\"\n#include \"mxnet-cpp/shape.h\"\n\nnamespace mxnet {\nnamespace cpp {\n\nenum DeviceType { kCPU = 1, kGPU = 2, kCPUPinned = 3 };\n\n/*!\n * \\brief Context interface\n */\nclass Context {\n public:\n  /*!\n   * \\brief Context constructor\n   * \\param type type of the device\n   * \\param id id of the device\n   */\n  Context(const DeviceType& type, int id) : type_(type), id_(id) {}\n  /*!\n   * \\return the type of the device\n   */\n  DeviceType GetDeviceType() const {\n    return type_;\n  }\n  /*!\n   * \\return the id of the device\n   */\n  int GetDeviceId() const {\n    return id_;\n  }\n\n  /*!\n   * \\brief Return a GPU context\n   * \\param device_id id of the device\n   * \\return the corresponding GPU context\n   */\n  static Context gpu(int device_id = 0) {\n    return Context(DeviceType::kGPU, device_id);\n  }\n\n  /*!\n   * \\brief Return a CPU context\n   * \\param device_id id of the device. this is not needed by CPU\n   * \\return the corresponding CPU context\n   */\n  static Context cpu(int device_id = 0) {\n    return Context(DeviceType::kCPU, device_id);\n  }\n\n private:\n  DeviceType type_;\n  int id_;\n};\n\n/*!\n * \\brief struct to store NDArrayHandle\n */\nstruct NDBlob {\n public:\n  /*!\n   * \\brief default constructor\n   */\n  NDBlob() : handle_(nullptr) {}\n  /*!\n   * \\brief construct with a NDArrayHandle\n   * \\param handle NDArrayHandle to store\n   */\n  explicit NDBlob(NDArrayHandle handle) : handle_(handle) {}\n  /*!\n   * \\brief destructor, free the NDArrayHandle\n   */\n  ~NDBlob() {\n    MXNDArrayFree(handle_);\n  }\n  /*!\n   * \\brief the NDArrayHandle\n   */\n  NDArrayHandle handle_;\n\n private:\n  NDBlob(const NDBlob&);\n  NDBlob& operator=(const NDBlob&);\n};\n\n/*!\n * \\brief NDArray interface\n */\nclass NDArray {\n public:\n  /*!\n   * \\brief construct with a none handle\n   */\n  NDArray();\n  /*!\n   * \\brief construct with a NDArrayHandle\n   */\n  explicit NDArray(const NDArrayHandle& handle);\n  /*!\n   * \\brief construct a new dynamic NDArray\n   * \\param shape the shape of array\n   * \\param context context of NDArray\n   * \\param delay_alloc whether delay the allocation\n   * \\param dtype data type of NDArray\n   */\n  NDArray(const std::vector<mx_uint>& shape,\n          const Context& context,\n          bool delay_alloc = true,\n          int dtype        = 0);\n  /*!\n   * \\brief construct a new dynamic NDArray\n   * \\param shape the shape of array\n   * \\param constext context of NDArray\n   * \\param delay_alloc whether delay the allocation\n   * \\param dtype data type of NDArray\n   */\n  NDArray(const Shape& shape, const Context& context, bool delay_alloc = true, int dtype = 0);\n  NDArray(const mx_float* data, size_t size);\n  /*!\n   * \\brief construct a new dynamic NDArray\n   * \\param data the data to create NDArray from\n   * \\param shape the shape of array\n   * \\param constext context of NDArray\n   */\n  NDArray(const mx_float* data, const Shape& shape, const Context& context);\n  /*!\n   * \\brief construct a new dynamic NDArray\n   * \\param data the data to create NDArray from\n   * \\param shape the shape of array\n   * \\param constext context of NDArray\n   */\n  NDArray(const std::vector<mx_float>& data, const Shape& shape, const Context& context);\n  explicit NDArray(const std::vector<mx_float>& data);\n  NDArray operator+(mx_float scalar);\n  NDArray operator-(mx_float scalar);\n  NDArray operator*(mx_float scalar);\n  NDArray operator/(mx_float scalar);\n  NDArray operator%(mx_float scalar);\n  NDArray operator+(const NDArray&);\n  NDArray operator-(const NDArray&);\n  NDArray operator*(const NDArray&);\n  NDArray operator/(const NDArray&);\n  NDArray operator%(const NDArray&);\n  /*!\n   * \\brief set all the elements in ndarray to be scalar\n   * \\param scalar the scalar to set\n   * \\return reference of self\n   */\n  NDArray& operator=(mx_float scalar);\n  /*!\n   * \\brief elementwise add to current space\n   *  this mutate the current NDArray\n   * \\param scalar the data to add\n   * \\return reference of self\n   */\n  NDArray& operator+=(mx_float scalar);\n  /*!\n   * \\brief elementwise subtract from current ndarray\n   * this mutate the current NDArray\n   * \\param scalar the data to subtract\n   * \\return reference of self\n   */\n  NDArray& operator-=(mx_float scalar);\n  /*!\n   * \\brief elementwise multiplication to current ndarray\n   *  this mutate the current NDArray\n   * \\param scalar the data to subtract\n   * \\return reference of self\n   */\n  NDArray& operator*=(mx_float scalar);\n  /*!\n   * \\brief elementwise division from current ndarray\n   *  this mutate the current NDArray\n   * \\param scalar the data to subtract\n   * \\return reference of self\n   */\n  NDArray& operator/=(mx_float scalar);\n  /*!\n   * \\brief elementwise modulo from current ndarray\n   *  this mutate the current NDArray\n   * \\param scalar the data to subtract\n   * \\return reference of self\n   */\n  NDArray& operator%=(mx_float scalar);\n  /*!\n   * \\brief elementwise add to current space\n   *  this mutate the current NDArray\n   * \\param src the data to add\n   * \\return reference of self\n   */\n  NDArray& operator+=(const NDArray& src);\n  /*!\n   * \\brief elementwise subtract from current ndarray\n   * this mutate the current NDArray\n   * \\param src the data to subtract\n   * \\return reference of self\n   */\n  NDArray& operator-=(const NDArray& src);\n  /*!\n   * \\brief elementwise multiplication to current ndarray\n   *  this mutate the current NDArray\n   * \\param src the data to subtract\n   * \\return reference of self\n   */\n  NDArray& operator*=(const NDArray& src);\n  /*!\n   * \\brief elementwise division from current ndarray\n   *  this mutate the current NDArray\n   * \\param src the data to subtract\n   * \\return reference of self\n   */\n  NDArray& operator/=(const NDArray& src);\n  /*!\n   * \\brief elementwise modulo from current ndarray\n   *  this mutate the current NDArray\n   * \\param src the data to subtract\n   * \\return reference of self\n   */\n  NDArray& operator%=(const NDArray& src);\n  NDArray ArgmaxChannel();\n  /*!\n   * \\brief Do a synchronize copy from a contiguous CPU memory region.\n   *\n   *  This function will call WaitToWrite before the copy is performed.\n   *  This is useful to copy data from existing memory region that are\n   *  not wrapped by NDArray(thus dependency not being tracked).\n   *\n   * \\param data the data source to copy from.\n   * \\param size the memory size we want to copy from.\n   */\n  void SyncCopyFromCPU(const mx_float* data, size_t size);\n  /*!\n   * \\brief Do a synchronize copy from a contiguous CPU memory region.\n   *\n   *  This function will call WaitToWrite before the copy is performed.\n   *  This is useful to copy data from existing memory region that are\n   *  not wrapped by NDArray(thus dependency not being tracked).\n   *\n   * \\param data the data source to copy from, int the form of mx_float vector\n   */\n  void SyncCopyFromCPU(const std::vector<mx_float>& data);\n  /*!\n   * \\brief Do a synchronize copy to a contiguous CPU memory region.\n   *\n   *  This function will call WaitToRead before the copy is performed.\n   *  This is useful to copy data from existing memory region that are\n   *  not wrapped by NDArray(thus dependency not being tracked).\n   *\n   * \\param data the data source to copyinto.\n   * \\param size the memory size we want to copy into. Defualt value is Size()\n   */\n  void SyncCopyToCPU(mx_float* data, size_t size = 0);\n  /*!\n   * \\brief Do a synchronize copy to a contiguous CPU memory region.\n   *\n   *  This function will call WaitToRead before the copy is performed.\n   *  This is useful to copy data from existing memory region that are\n   *  not wrapped by NDArray(thus dependency not being tracked).\n   *\n   * \\param data the data source to copyinto.\n   * \\param size the memory size we want to copy into. Defualt value is Size()\n   */\n  void SyncCopyToCPU(std::vector<mx_float>* data, size_t size = 0);\n  /*!\n   * \\brief copy the content of current array to a target array.\n   * \\param other the target NDArray\n   * \\return the target NDarray\n   */\n  NDArray CopyTo(NDArray* other) const;\n  /*!\n   * \\brief return a new copy to this NDArray\n   * \\param Context the new context of this NDArray\n   * \\return the new copy\n   */\n  NDArray Copy(const Context&) const;\n  /*!\n   * \\brief return offset of the element at (h, w)\n   * \\param h height position\n   * \\param w width position\n   * \\return offset of two dimensions array\n   */\n  size_t Offset(size_t h = 0, size_t w = 0) const;\n  /*!\n   * \\brief return offset of three dimensions array\n   * \\param c channel position\n   * \\param h height position\n   * \\param w width position\n   * \\return offset of three dimensions array\n   */\n  size_t Offset(size_t c, size_t h, size_t w) const;\n  /*!\n   * \\brief return value of the element at (index)\n   * \\param index  position\n   * \\return value of one dimensions array\n   */\n  mx_float At(size_t index) const;\n  /*!\n   * \\brief return value of the element at (h, w)\n   * \\param h height position\n   * \\param w width position\n   * \\return value of two dimensions array\n   */\n  mx_float At(size_t h, size_t w) const;\n  /*!\n   * \\brief return value of three dimensions array\n   * \\param c channel position\n   * \\param h height position\n   * \\param w width position\n   * \\return value of three dimensions array\n   */\n  mx_float At(size_t c, size_t h, size_t w) const;\n  /*!\n   * \\brief Slice a NDArray\n   * \\param begin begin index in first dim\n   * \\param end end index in first dim\n   * \\return sliced NDArray\n   */\n  NDArray Slice(mx_uint begin, mx_uint end) const;\n  /*!\n   * \\brief Return a reshaped NDArray that shares memory with current one\n   * \\param new_shape the new shape\n   * \\return reshaped NDarray\n   */\n  NDArray Reshape(const Shape& new_shape) const;\n  /*!\n   * \\brief Block until all the pending write operations with respect\n   *    to current NDArray are finished, and read can be performed.\n   */\n  void WaitToRead() const;\n  /*!\n   * \\brief Block until all the pending read/write operations with respect\n   *    to current NDArray are finished, and write can be performed.\n   */\n  void WaitToWrite();\n  /*!\n   * \\brief Block until all the pending read/write operations with respect\n   *    to current NDArray are finished, and read/write can be performed.\n   */\n  static void WaitAll();\n  /*!\n   * \\brief Sample gaussian distribution for each elements of out.\n   * \\param mu mean of gaussian distribution.\n   * \\param sigma standard deviation of gaussian distribution.\n   * \\param out output NDArray.\n   */\n  static void SampleGaussian(mx_float mu, mx_float sigma, NDArray* out);\n  /*!\n   * \\brief Sample uniform distribution for each elements of out.\n   * \\param begin lower bound of distribution.\n   * \\param end upper bound of distribution.\n   * \\param out output NDArray.\n   */\n  static void SampleUniform(mx_float begin, mx_float end, NDArray* out);\n  /*!\n   * \\brief Load NDArrays from binary file.\n   * \\param file_name name of the binary file.\n   * \\param array_list a list of NDArrays returned, do not fill the list if\n   * nullptr is given.\n   * \\param array_map a map from names to NDArrays returned, do not fill the map\n   * if nullptr is given or no names is stored in binary file.\n   */\n  static void Load(const std::string& file_name,\n                   std::vector<NDArray>* array_list          = nullptr,\n                   std::map<std::string, NDArray>* array_map = nullptr);\n  /*!\n   * \\brief Load map of NDArrays from binary file.\n   * \\param file_name name of the binary file.\n   * \\return a list of NDArrays.\n   */\n  static std::map<std::string, NDArray> LoadToMap(const std::string& file_name);\n  /*!\n   * \\brief Load list of NDArrays from binary file.\n   * \\param file_name name of the binary file.\n   * \\return a map from names to NDArrays.\n   */\n  static std::vector<NDArray> LoadToList(const std::string& file_name);\n  /*!\n   * \\brief Load NDArrays from buffer.\n   * \\param buffer Pointer to buffer. (ie contents of param file)\n   * \\param size Size of buffer\n   * \\param array_list a list of NDArrays returned, do not fill the list if\n   * nullptr is given.\n   * \\param array_map a map from names to NDArrays returned, do not fill the map\n   * if nullptr is given or no names is stored in binary file.\n   */\n  static void LoadFromBuffer(const void* buffer,\n                             size_t size,\n                             std::vector<NDArray>* array_list          = nullptr,\n                             std::map<std::string, NDArray>* array_map = nullptr);\n  /*!\n   * \\brief Load map of NDArrays from buffer.\n   * \\param buffer Pointer to buffer. (ie contents of param file)\n   * \\param size Size of buffer\n   * \\return a list of NDArrays.\n   */\n  static std::map<std::string, NDArray> LoadFromBufferToMap(const void* buffer, size_t size);\n  /*!\n   * \\brief Load list of NDArrays from buffer.\n   * \\param buffer Pointer to buffer. (ie contents of param file)\n   * \\param size Size of buffer\n   * \\return a map from names to NDArrays.\n   */\n  static std::vector<NDArray> LoadFromBufferToList(const void* buffer, size_t size);\n  /*!\n   * \\brief save a map of string->NDArray to binary file.\n   * \\param file_name name of the binary file.\n   * \\param array_map a map from names to NDArrays.\n   */\n  static void Save(const std::string& file_name, const std::map<std::string, NDArray>& array_map);\n  /*!\n   * \\brief save a list of NDArrays to binary file.\n   * \\param file_name name of the binary file.\n   * \\param array_list a list of NDArrays.\n   */\n  static void Save(const std::string& file_name, const std::vector<NDArray>& array_list);\n  /*!\n   * \\return the size of current NDArray, a.k.a. the production of all shape dims\n   */\n  size_t Size() const;\n  /*!\n   * \\return the shape of current NDArray, in the form of mx_uint vector\n   */\n  std::vector<mx_uint> GetShape() const;\n  /*!\n   * \\return the data type of current NDArray\n   */\n  int GetDType() const;\n  /*!\n   * \\brief Get the pointer to data (IMPORTANT: The ndarray should not be in GPU)\n   * \\return the data pointer to the current NDArray\n   */\n  const mx_float* GetData() const;\n\n  /*!\n   * \\return the context of NDArray\n   */\n  Context GetContext() const;\n\n  /*!\n   * \\return the NDArrayHandle of the current NDArray\n   */\n  NDArrayHandle GetHandle() const {\n    return blob_ptr_->handle_;\n  }\n\n private:\n  std::shared_ptr<NDBlob> blob_ptr_;\n};\n\nstd::ostream& operator<<(std::ostream& out, const NDArray& ndarray);\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_NDARRAY_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/ndarray.hpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file ndarray.hpp\n * \\brief implementation of the ndarray\n * \\author Zhang Chen, Chuntao Hong\n */\n\n#ifndef MXNET_CPP_NDARRAY_HPP_\n#define MXNET_CPP_NDARRAY_HPP_\n\n#include <algorithm>\n#include <map>\n#include <string>\n#include <vector>\n#include <iterator>\n#include \"dmlc/logging.h\"\n#include \"mxnet-cpp/ndarray.h\"\n#include \"mxnet-cpp/operator.h\"\n\nnamespace mxnet {\nnamespace cpp {\n\ninline NDArray::NDArray() {\n  NDArrayHandle handle;\n  CHECK_EQ(MXNDArrayCreateNone(&handle), 0);\n  blob_ptr_ = std::make_shared<NDBlob>(handle);\n}\ninline NDArray::NDArray(const NDArrayHandle &handle) {\n  blob_ptr_ = std::make_shared<NDBlob>(handle);\n}\ninline NDArray::NDArray(const std::vector<mx_uint> &shape, const Context &context,\n                        bool delay_alloc, int dtype) {\n  NDArrayHandle handle;\n  CHECK_EQ(MXNDArrayCreateEx(shape.data(), shape.size(), context.GetDeviceType(),\n                             context.GetDeviceId(), delay_alloc, dtype, &handle),\n           0);\n  blob_ptr_ = std::make_shared<NDBlob>(handle);\n}\ninline NDArray::NDArray(const Shape &shape, const Context &context,\n                        bool delay_alloc, int dtype) {\n  NDArrayHandle handle;\n  CHECK_EQ(MXNDArrayCreateEx(shape.data(), shape.ndim(), context.GetDeviceType(),\n                             context.GetDeviceId(), delay_alloc, dtype, &handle),\n           0);\n  blob_ptr_ = std::make_shared<NDBlob>(handle);\n}\ninline NDArray::NDArray(const mx_float *data, size_t size) {\n  NDArrayHandle handle;\n  CHECK_EQ(MXNDArrayCreateNone(&handle), 0);\n  MXNDArraySyncCopyFromCPU(handle, data, size);\n  blob_ptr_ = std::make_shared<NDBlob>(handle);\n}\ninline NDArray::NDArray(const mx_float *data, const Shape &shape,\n                        const Context &context) {\n  NDArrayHandle handle;\n  CHECK_EQ(MXNDArrayCreate(shape.data(), shape.ndim(), context.GetDeviceType(),\n                           context.GetDeviceId(), false, 0, &handle),\n           0);\n  CHECK_EQ(MXNDArraySyncCopyFromCPU(handle, data, shape.Size()), 0);\n  blob_ptr_ = std::make_shared<NDBlob>(handle);\n}\ninline NDArray::NDArray(const std::vector<mx_float> &data, const Shape &shape,\n                        const Context &context) {\n  NDArrayHandle handle;\n  CHECK_EQ(MXNDArrayCreate(shape.data(), shape.ndim(), context.GetDeviceType(),\n                           context.GetDeviceId(), false, 0, &handle),\n           0);\n  MXNDArraySyncCopyFromCPU(handle, data.data(), shape.Size());\n  blob_ptr_ = std::make_shared<NDBlob>(handle);\n}\ninline NDArray::NDArray(const std::vector<mx_float> &data) {\n  NDArrayHandle handle;\n  CHECK_EQ(MXNDArrayCreateNone(&handle), 0);\n  MXNDArraySyncCopyFromCPU(handle, data.data(), data.size());\n  blob_ptr_ = std::make_shared<NDBlob>(handle);\n}\n\ninline NDArray NDArray::operator+(mx_float scalar) {\n  NDArray ret;\n  Operator(\"_plus_scalar\")(*this, scalar).Invoke(ret);\n  return ret;\n}\ninline NDArray NDArray::operator-(mx_float scalar) {\n  NDArray ret;\n  Operator(\"_minus_scalar\")(*this, scalar).Invoke(ret);\n  return ret;\n}\ninline NDArray NDArray::operator*(mx_float scalar) {\n  NDArray ret;\n  Operator(\"_mul_scalar\")(*this, scalar).Invoke(ret);\n  return ret;\n}\ninline NDArray NDArray::operator/(mx_float scalar) {\n  NDArray ret;\n  Operator(\"_div_scalar\")(*this, scalar).Invoke(ret);\n  return ret;\n}\ninline NDArray NDArray::operator%(mx_float scalar) {\n  NDArray ret;\n  Operator(\"_mod_scalar\")(*this, scalar).Invoke(ret);\n  return ret;\n}\ninline NDArray NDArray::operator+(const NDArray &rhs) {\n  NDArray ret;\n  Operator(\"_plus\")(*this, rhs).Invoke(ret);\n  return ret;\n}\ninline NDArray NDArray::operator-(const NDArray &rhs) {\n  NDArray ret;\n  Operator(\"_minus\")(*this, rhs).Invoke(ret);\n  return ret;\n}\ninline NDArray NDArray::operator*(const NDArray &rhs) {\n  NDArray ret;\n  Operator(\"_mul\")(*this, rhs).Invoke(ret);\n  return ret;\n}\ninline NDArray NDArray::operator/(const NDArray &rhs) {\n  NDArray ret;\n  Operator(\"_div\")(*this, rhs).Invoke(ret);\n  return ret;\n}\ninline NDArray NDArray::operator%(const NDArray &rhs) {\n  NDArray ret;\n  Operator(\"_mod\")(*this, rhs).Invoke(ret);\n  return ret;\n}\ninline NDArray &NDArray::operator=(mx_float scalar) {\n  Operator(\"_set_value\")(scalar).Invoke(*this);\n  return *this;\n}\ninline NDArray &NDArray::operator+=(mx_float scalar) {\n  Operator(\"_plus_scalar\")(*this, scalar).Invoke(*this);\n  return *this;\n}\ninline NDArray &NDArray::operator-=(mx_float scalar) {\n  Operator(\"_minus_scalar\")(*this, scalar).Invoke(*this);\n  return *this;\n}\ninline NDArray &NDArray::operator*=(mx_float scalar) {\n  Operator(\"_mul_scalar\")(*this, scalar).Invoke(*this);\n  return *this;\n}\ninline NDArray &NDArray::operator/=(mx_float scalar) {\n  Operator(\"_div_scalar\")(*this, scalar).Invoke(*this);\n  return *this;\n}\ninline NDArray &NDArray::operator%=(mx_float scalar) {\n  Operator(\"_mod_scalar\")(*this, scalar).Invoke(*this);\n  return *this;\n}\ninline NDArray &NDArray::operator+=(const NDArray &rhs) {\n  Operator(\"_plus\")(*this, rhs).Invoke(*this);\n  return *this;\n}\ninline NDArray &NDArray::operator-=(const NDArray &rhs) {\n  Operator(\"_minus\")(*this, rhs).Invoke(*this);\n  return *this;\n}\ninline NDArray &NDArray::operator*=(const NDArray &rhs) {\n  Operator(\"_mul\")(*this, rhs).Invoke(*this);\n  return *this;\n}\ninline NDArray &NDArray::operator/=(const NDArray &rhs) {\n  Operator(\"_div\")(*this, rhs).Invoke(*this);\n  return *this;\n}\ninline NDArray &NDArray::operator%=(const NDArray &rhs) {\n  Operator(\"_mod\")(*this, rhs).Invoke(*this);\n  return *this;\n}\n\ninline NDArray NDArray::ArgmaxChannel() {\n  NDArray ret;\n  Operator(\"argmax_channel\")(*this).Invoke(ret);\n  return ret;\n}\n\ninline void NDArray::SyncCopyFromCPU(const mx_float *data, size_t size) {\n  MXNDArraySyncCopyFromCPU(blob_ptr_->handle_, data, size);\n}\ninline void NDArray::SyncCopyFromCPU(const std::vector<mx_float> &data) {\n  MXNDArraySyncCopyFromCPU(blob_ptr_->handle_, data.data(), data.size());\n}\ninline void NDArray::SyncCopyToCPU(mx_float *data, size_t size) {\n  MXNDArraySyncCopyToCPU(blob_ptr_->handle_, data, size > 0 ? size : Size());\n}\ninline void NDArray::SyncCopyToCPU(std::vector<mx_float> *data, size_t size) {\n  size = size > 0 ? size : Size();\n  data->resize(size);\n  MXNDArraySyncCopyToCPU(blob_ptr_->handle_, data->data(), size);\n}\ninline NDArray NDArray::Copy(const Context &ctx) const {\n  NDArray ret(GetShape(), ctx, true, this->GetDType());\n  Operator(\"_copyto\")(*this).Invoke(ret);\n  return ret;\n}\ninline NDArray NDArray::CopyTo(NDArray * other) const {\n  Operator(\"_copyto\")(*this).Invoke(*other);\n  return *other;\n}\ninline NDArray NDArray::Slice(mx_uint begin, mx_uint end) const {\n  NDArrayHandle handle;\n  CHECK_EQ(MXNDArraySlice(GetHandle(), begin, end, &handle), 0);\n  return NDArray(handle);\n}\ninline NDArray NDArray::Reshape(const Shape &new_shape) const {\n  NDArrayHandle handle;\n  std::vector<int> dims(new_shape.ndim());\n  for (index_t i = 0; i < new_shape.ndim(); ++i) {\n    dims[i] = new_shape[i];\n  }\n  new_shape.data();\n  CHECK_EQ(\n      MXNDArrayReshape(GetHandle(), new_shape.ndim(), dims.data(), &handle), 0);\n  return NDArray(handle);\n}\ninline void NDArray::WaitToRead() const {\n  CHECK_EQ(MXNDArrayWaitToRead(blob_ptr_->handle_), 0) << MXGetLastError();\n}\ninline void NDArray::WaitToWrite() {\n  CHECK_EQ(MXNDArrayWaitToWrite(blob_ptr_->handle_), 0) << MXGetLastError();\n}\ninline void NDArray::WaitAll() { CHECK_EQ(MXNDArrayWaitAll(), 0) << MXGetLastError(); }\ninline void NDArray::SampleGaussian(mx_float mu, mx_float sigma, NDArray *out) {\n  Operator(\"_random_normal\")(mu, sigma).Invoke(*out);\n}\ninline void NDArray::SampleUniform(mx_float begin, mx_float end, NDArray *out) {\n  Operator(\"_random_uniform\")(begin, end).Invoke(*out);\n}\ninline void NDArray::Load(const std::string &file_name,\n                          std::vector<NDArray> *array_list,\n                          std::map<std::string, NDArray> *array_map) {\n  mx_uint out_size, out_name_size;\n  NDArrayHandle *out_arr;\n  const char **out_names;\n  CHECK_EQ(MXNDArrayLoad(file_name.c_str(), &out_size, &out_arr, &out_name_size,\n                         &out_names),\n           0);\n  if (array_list != nullptr) {\n    array_list->reserve(out_size);\n    for (mx_uint i = 0; i < out_size; ++i) {\n      array_list->push_back(NDArray(out_arr[i]));\n    }\n  }\n  if (array_map != nullptr && out_name_size > 0) {\n    CHECK_EQ(out_name_size, out_size);\n    for (mx_uint i = 0; i < out_size; ++i) {\n      (*array_map)[out_names[i]] = NDArray(out_arr[i]);\n    }\n  }\n}\ninline std::map<std::string, NDArray> NDArray::LoadToMap(\n    const std::string &file_name) {\n  std::map<std::string, NDArray> array_map;\n  mx_uint out_size, out_name_size;\n  NDArrayHandle *out_arr;\n  const char **out_names;\n  CHECK_EQ(MXNDArrayLoad(file_name.c_str(), &out_size, &out_arr, &out_name_size,\n                         &out_names),\n           0);\n  if (out_name_size > 0) {\n    CHECK_EQ(out_name_size, out_size);\n    for (mx_uint i = 0; i < out_size; ++i) {\n      array_map[out_names[i]] = NDArray(out_arr[i]);\n    }\n  }\n  return array_map;\n}\ninline std::vector<NDArray> NDArray::LoadToList(const std::string &file_name) {\n  std::vector<NDArray> array_list;\n  mx_uint out_size, out_name_size;\n  NDArrayHandle *out_arr;\n  const char **out_names;\n  CHECK_EQ(MXNDArrayLoad(file_name.c_str(), &out_size, &out_arr, &out_name_size,\n                         &out_names),\n           0);\n  array_list.reserve(out_size);\n  for (mx_uint i = 0; i < out_size; ++i) {\n    array_list.push_back(NDArray(out_arr[i]));\n  }\n  return array_list;\n}\ninline void NDArray::LoadFromBuffer(const void *buffer, size_t size,\n                          std::vector<NDArray> *array_list,\n                          std::map<std::string, NDArray> *array_map) {\n  mx_uint out_size, out_name_size;\n  NDArrayHandle *out_arr;\n  const char **out_names;\n  CHECK_EQ(MXNDArrayLoadFromBuffer(buffer, size, &out_size, &out_arr, &out_name_size,\n                         &out_names),\n           0);\n  if (array_list != nullptr) {\n    array_list->reserve(out_size);\n    for (mx_uint i = 0; i < out_size; ++i) {\n      array_list->push_back(NDArray(out_arr[i]));\n    }\n  }\n  if (array_map != nullptr && out_name_size > 0) {\n    CHECK_EQ(out_name_size, out_size);\n    for (mx_uint i = 0; i < out_size; ++i) {\n      (*array_map)[out_names[i]] = NDArray(out_arr[i]);\n    }\n  }\n}\ninline std::map<std::string, NDArray> NDArray::LoadFromBufferToMap(\n    const void *buffer, size_t size) {\n  std::map<std::string, NDArray> array_map;\n  mx_uint out_size, out_name_size;\n  NDArrayHandle *out_arr;\n  const char **out_names;\n  CHECK_EQ(MXNDArrayLoadFromBuffer(buffer, size, &out_size, &out_arr, &out_name_size,\n                         &out_names),\n           0);\n  if (out_name_size > 0) {\n    CHECK_EQ(out_name_size, out_size);\n    for (mx_uint i = 0; i < out_size; ++i) {\n      array_map[out_names[i]] = NDArray(out_arr[i]);\n    }\n  }\n  return array_map;\n}\ninline std::vector<NDArray> NDArray::LoadFromBufferToList(const void *buffer, size_t size) {\n  std::vector<NDArray> array_list;\n  mx_uint out_size, out_name_size;\n  NDArrayHandle *out_arr;\n  const char **out_names;\n  CHECK_EQ(MXNDArrayLoadFromBuffer(buffer, size, &out_size, &out_arr, &out_name_size,\n                         &out_names),\n           0);\n  array_list.reserve(out_size);\n  for (mx_uint i = 0; i < out_size; ++i) {\n    array_list.push_back(NDArray(out_arr[i]));\n  }\n  return array_list;\n}\ninline void NDArray::Save(const std::string &file_name,\n                          const std::map<std::string, NDArray> &array_map) {\n  std::vector<NDArrayHandle> args;\n  std::vector<const char *> keys;\n  for (const auto &t : array_map) {\n    args.push_back(t.second.GetHandle());\n    keys.push_back(t.first.c_str());\n  }\n  CHECK_EQ(\n      MXNDArraySave(file_name.c_str(), args.size(), args.data(), keys.data()),\n      0);\n}\ninline void NDArray::Save(const std::string &file_name,\n                          const std::vector<NDArray> &array_list) {\n  std::vector<NDArrayHandle> args;\n  for (const auto &t : array_list) {\n    args.push_back(t.GetHandle());\n  }\n  CHECK_EQ(MXNDArraySave(file_name.c_str(), args.size(), args.data(), nullptr),\n           0);\n}\n\ninline size_t NDArray::Offset(size_t h, size_t w) const {\n  auto const shape = GetShape();\n  CHECK_EQ(shape.size(), 2) << \"The NDArray needs to be 2 dimensional.\";\n\n  return (h * shape[1]) + w;\n}\n\ninline size_t NDArray::Offset(size_t c, size_t h, size_t w) const {\n  auto const shape = GetShape();\n  CHECK_EQ(shape.size(), 3) << \"The NDArray needs to be 3 dimensional.\";\n  return h * shape[0] * shape[2] + w * shape[0] + c;\n}\n\ninline mx_float NDArray::At(size_t h, size_t w) const {\n  return GetData()[Offset(h, w)];\n}\n\ninline mx_float NDArray::At(size_t c, size_t h, size_t w) const {\n  return GetData()[Offset(c, h, w)];\n}\n\ninline mx_float NDArray::At(size_t index) const {\n  auto shape = GetShape();\n  CHECK_EQ(shape.size(), 1) << \"The NDArray needs to be 1 dimensional.\";\n  CHECK_LT(index, shape[0]) << \"Specified index is out of range.\";\n  return GetData()[index];\n}\n\ninline size_t NDArray::Size() const {\n  size_t ret = 1;\n  for (auto &i : GetShape()) ret *= i;\n  return ret;\n}\n\ninline std::vector<mx_uint> NDArray::GetShape() const {\n  const int *out_pdata;\n  int out_dim;\n  MXNDArrayGetShape(blob_ptr_->handle_, &out_dim, &out_pdata);\n  std::vector<mx_uint> ret;\n  for (int i = 0; i < out_dim; ++i) {\n    ret.push_back(out_pdata[i]);\n  }\n  return ret;\n}\n\ninline int NDArray::GetDType() const {\n  int ret;\n  MXNDArrayGetDType(blob_ptr_->handle_, &ret);\n  return ret;\n}\n\ninline const mx_float *NDArray::GetData() const {\n  void *ret;\n  MXNDArrayGetData(blob_ptr_->handle_, &ret);\n  if (GetDType() != 0) {\n    return nullptr;\n  }\n  return static_cast<mx_float*>(ret);\n}\n\ninline Context NDArray::GetContext() const {\n  int out_dev_type;\n  int out_dev_id;\n  MXNDArrayGetContext(blob_ptr_->handle_, &out_dev_type, &out_dev_id);\n  return Context((DeviceType)out_dev_type, out_dev_id);\n}\n\ninline std::ostream & operator<<(std::ostream &out, const NDArray &ndarray) {\n  // TODO(lx75249): Consider DType / beautify like numpy\n  auto shape = ndarray.GetShape();\n  NDArray cpu_array(ndarray.GetShape(), Context::cpu());\n  if (ndarray.GetContext().GetDeviceType() != DeviceType::kGPU) {\n    cpu_array = ndarray;\n  } else {\n    ndarray.WaitToRead();\n    ndarray.CopyTo(&cpu_array);\n  }\n\n  out << '[';\n  cpu_array.WaitToRead();\n  std::copy(cpu_array.GetData(), cpu_array.GetData() + ndarray.Size(),\n      std::ostream_iterator<float>(out, \", \"));\n  out << ']';\n  return out;\n}\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_NDARRAY_HPP_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/op_map.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file op_map.h\n * \\brief definition of OpMap\n * \\author Chuntao Hong\n */\n\n#ifndef MXNET_CPP_OP_MAP_H_\n#define MXNET_CPP_OP_MAP_H_\n\n#include <map>\n#include <string>\n#include \"mxnet-cpp/base.h\"\n#include \"dmlc/logging.h\"\n\nnamespace mxnet {\nnamespace cpp {\n\n/*!\n * \\brief OpMap instance holds a map of all the symbol creators so we can\n *  get symbol creators by name.\n *  This is used internally by Symbol and Operator.\n */\nclass OpMap {\n public:\n  /*!\n   * \\brief Create an Mxnet instance\n   */\n  inline OpMap() {\n    mx_uint num_symbol_creators          = 0;\n    AtomicSymbolCreator* symbol_creators = nullptr;\n    int r = MXSymbolListAtomicSymbolCreators(&num_symbol_creators, &symbol_creators);\n    CHECK_EQ(r, 0);\n    for (mx_uint i = 0; i < num_symbol_creators; i++) {\n      const char* name;\n      const char* description;\n      mx_uint num_args;\n      const char** arg_names;\n      const char** arg_type_infos;\n      const char** arg_descriptions;\n      const char* key_var_num_args;\n      r = MXSymbolGetAtomicSymbolInfo(symbol_creators[i],\n                                      &name,\n                                      &description,\n                                      &num_args,\n                                      &arg_names,\n                                      &arg_type_infos,\n                                      &arg_descriptions,\n                                      &key_var_num_args);\n      CHECK_EQ(r, 0);\n      symbol_creators_[name] = symbol_creators[i];\n    }\n\n    nn_uint num_ops;\n    const char** op_names;\n    r = NNListAllOpNames(&num_ops, &op_names);\n    CHECK_EQ(r, 0);\n    for (nn_uint i = 0; i < num_ops; i++) {\n      OpHandle handle;\n      r = NNGetOpHandle(op_names[i], &handle);\n      CHECK_EQ(r, 0);\n      op_handles_[op_names[i]] = handle;\n    }\n  }\n\n  /*!\n   * \\brief Get a symbol creator with its name.\n   *\n   * \\param name name of the symbol creator\n   * \\return handle to the symbol creator\n   */\n  inline AtomicSymbolCreator GetSymbolCreator(const std::string& name) {\n    if (symbol_creators_.count(name) == 0)\n      return GetOpHandle(name);\n    return symbol_creators_[name];\n  }\n\n  /*!\n   * \\brief Get an op handle with its name.\n   *\n   * \\param name name of the op\n   * \\return handle to the op\n   */\n  inline OpHandle GetOpHandle(const std::string& name) {\n    return op_handles_[name];\n  }\n\n private:\n  std::map<std::string, AtomicSymbolCreator> symbol_creators_;\n  std::map<std::string, OpHandle> op_handles_;\n};\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_OP_MAP_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/op_suppl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file op_suppl.h\n * \\brief A supplement and amendment of the operators from op.h\n * \\author Zhang Chen, zhubuntu, Xin Li\n */\n\n#ifndef MXNET_CPP_OP_SUPPL_H_\n#define MXNET_CPP_OP_SUPPL_H_\n\n#include <cassert>\n#include <string>\n#include <vector>\n#include \"mxnet-cpp/base.h\"\n#include \"mxnet-cpp/shape.h\"\n#include \"mxnet-cpp/operator.h\"\n#include \"mxnet-cpp/MxNetCpp.h\"\n\nnamespace mxnet {\nnamespace cpp {\n\ninline Symbol _Plus(Symbol lhs, Symbol rhs) {\n  return Operator(\"_Plus\")(lhs, rhs).CreateSymbol();\n}\ninline Symbol _Mul(Symbol lhs, Symbol rhs) {\n  return Operator(\"_Mul\")(lhs, rhs).CreateSymbol();\n}\ninline Symbol _Minus(Symbol lhs, Symbol rhs) {\n  return Operator(\"_Minus\")(lhs, rhs).CreateSymbol();\n}\ninline Symbol _Div(Symbol lhs, Symbol rhs) {\n  return Operator(\"_Div\")(lhs, rhs).CreateSymbol();\n}\ninline Symbol _Mod(Symbol lhs, Symbol rhs) {\n  return Operator(\"_Mod\")(lhs, rhs).CreateSymbol();\n}\ninline Symbol _Power(Symbol lhs, Symbol rhs) {\n  return Operator(\"_Power\")(lhs, rhs).CreateSymbol();\n}\ninline Symbol _Maximum(Symbol lhs, Symbol rhs) {\n  return Operator(\"_Maximum\")(lhs, rhs).CreateSymbol();\n}\ninline Symbol _Minimum(Symbol lhs, Symbol rhs) {\n  return Operator(\"_Minimum\")(lhs, rhs).CreateSymbol();\n}\ninline Symbol _PlusScalar(Symbol lhs, mx_float scalar) {\n  return Operator(\"_PlusScalar\")(lhs).SetParam(\"scalar\", scalar).CreateSymbol();\n}\ninline Symbol _MinusScalar(Symbol lhs, mx_float scalar) {\n  return Operator(\"_MinusScalar\")(lhs).SetParam(\"scalar\", scalar).CreateSymbol();\n}\ninline Symbol _RMinusScalar(mx_float scalar, Symbol rhs) {\n  return Operator(\"_RMinusScalar\")(rhs).SetParam(\"scalar\", scalar).CreateSymbol();\n}\ninline Symbol _MulScalar(Symbol lhs, mx_float scalar) {\n  return Operator(\"_MulScalar\")(lhs).SetParam(\"scalar\", scalar).CreateSymbol();\n}\ninline Symbol _DivScalar(Symbol lhs, mx_float scalar) {\n  return Operator(\"_DivScalar\")(lhs).SetParam(\"scalar\", scalar).CreateSymbol();\n}\ninline Symbol _RDivScalar(mx_float scalar, Symbol rhs) {\n  return Operator(\"_RDivScalar\")(rhs).SetParam(\"scalar\", scalar).CreateSymbol();\n}\ninline Symbol _ModScalar(Symbol lhs, mx_float scalar) {\n  return Operator(\"_ModScalar\")(lhs).SetParam(\"scalar\", scalar).CreateSymbol();\n}\ninline Symbol _RModScalar(mx_float scalar, Symbol rhs) {\n  return Operator(\"_RModScalar\")(rhs).SetParam(\"scalar\", scalar).CreateSymbol();\n}\ninline Symbol _PowerScalar(Symbol lhs, mx_float scalar) {\n  return Operator(\"_PowerScalar\")(lhs).SetParam(\"scalar\", scalar).CreateSymbol();\n}\ninline Symbol _RPowerScalar(mx_float scalar, Symbol rhs) {\n  return Operator(\"_RPowerScalar\")(rhs).SetParam(\"scalar\", scalar).CreateSymbol();\n}\ninline Symbol _MaximumScalar(Symbol lhs, mx_float scalar) {\n  return Operator(\"_MaximumScalar\")(lhs).SetParam(\"scalar\", scalar).CreateSymbol();\n}\ninline Symbol _MinimumScalar(Symbol lhs, mx_float scalar) {\n  return Operator(\"_MinimumScalar\")(lhs).SetParam(\"scalar\", scalar).CreateSymbol();\n}\n// TODO(zhangcheng-qinyinghua)\n//  make crop function run in op.h\n//  This function is due to [zhubuntu](https://github.com/zhubuntu)\ninline Symbol Crop(const std::string& symbol_name,\n                   int num_args,\n                   Symbol data,\n                   Symbol crop_like,\n                   Shape offset     = Shape(0, 0),\n                   Shape h_w        = Shape(0, 0),\n                   bool center_crop = false) {\n  return Operator(\"Crop\")\n      .SetParam(\"num_args\", num_args)\n      .SetParam(\"offset\", offset)\n      .SetParam(\"h_w\", h_w)\n      .SetParam(\"center_crop\", center_crop)\n      .SetInput(\"arg0\", data)\n      .SetInput(\"arg1\", crop_like)\n      .CreateSymbol(symbol_name);\n}\n\n/*!\n * \\brief Apply activation function to input.\n *        Softmax Activation is only available with CUDNN on GPUand will be\n *        computed at each location across channel if input is 4D.\n * \\param symbol_name name of the resulting symbol.\n * \\param data Input data to activation function.\n * \\param act_type Activation function to be applied.\n * \\return new symbol\n */\ninline Symbol Activation(const std::string& symbol_name, Symbol data, const std::string& act_type) {\n  assert(act_type == \"relu\" || act_type == \"sigmoid\" || act_type == \"softrelu\" ||\n         act_type == \"tanh\");\n  return Operator(\"Activation\")\n      .SetParam(\"act_type\", act_type.c_str())\n      .SetInput(\"data\", data)\n      .CreateSymbol(symbol_name);\n}\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_OP_SUPPL_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/op_util.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file op_util.h\n * \\brief operator helper functions\n * \\author Chris Olivier\n */\n\n#ifndef MXNET_CPP_OP_UTIL_H_\n#define MXNET_CPP_OP_UTIL_H_\n\n#include <string>\n\n#if defined(MXNET_USE_CAFFE) && MXNET_USE_CAFFE != 0\n#include <caffe/proto/caffe.pb.h>\n#include <google/protobuf/text_format.h>\n#endif\n\nnamespace mxnet {\nnamespace cpp {\n\n#if defined(MXNET_USE_CAFFE) && MXNET_USE_CAFFE != 0\n\ninline ::caffe::LayerParameter textToCaffeLayerParameter(const std::string& text) {\n  caffe::NetParameter np;\n  const bool success = google::protobuf::TextFormat::ParseFromString(text, &np);\n  CHECK_EQ(success, true) << \"Invalid protpbuf layer string: \" << text;\n  return ::caffe::LayerParameter(np.layer(0));\n}\n\ntemplate <typename StreamType>\ninline StreamType& operator<<(StreamType& os, const ::caffe::LayerParameter& op) {\n  std::string s;\n  caffe::NetParameter np;\n  // Avoid wasting time making a copy -- just push in out default object's pointer\n  np.mutable_layer()->AddAllocated(const_cast<::caffe::LayerParameter*>(&op));\n  google::protobuf::TextFormat::PrintToString(np, &s);\n  np.mutable_layer()->ReleaseLast();\n  os << s;\n  return os;\n}\n#endif\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_OP_UTIL_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/operator.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file operator.h\n * \\brief definition of operator\n * \\author Chuntao Hong, Zhang Chen\n */\n\n#ifndef MXNET_CPP_OPERATOR_H_\n#define MXNET_CPP_OPERATOR_H_\n\n#include <map>\n#include <string>\n#include <vector>\n#include \"mxnet-cpp/base.h\"\n#include \"mxnet-cpp/op_map.h\"\n#include \"mxnet-cpp/symbol.h\"\n\nnamespace mxnet {\nnamespace cpp {\nclass Mxnet;\n/*!\n * \\brief Operator interface\n */\nclass Operator {\n public:\n  /*!\n   * \\brief Operator constructor\n   * \\param operator_name type of the operator\n   */\n  explicit Operator(const std::string& operator_name);\n  Operator& operator=(const Operator& rhs);\n  /*!\n   * \\brief set config parameters\n   * \\param name name of the config parameter\n   * \\param value value of the config parameter\n   * \\return reference of self\n   */\n  template <typename T>\n  Operator& SetParam(const std::string& name, const T& value) {\n    std::string value_str;\n    std::stringstream ss;\n    ss << value;\n    ss >> value_str;\n\n    params_[name] = value_str;\n    return *this;\n  }\n  /*!\n   * \\brief set config parameters from positional inputs\n   * \\param pos the position of parameter\n   * \\param value value of the config parameter\n   * \\return reference of self\n   */\n  template <typename T>\n  Operator& SetParam(int pos, const T& value) {\n    std::string value_str;\n    std::stringstream ss;\n    ss << value;\n    ss >> value_str;\n\n    params_[arg_names_[pos]] = value_str;\n    return *this;\n  }\n  /*!\n   * \\brief add an input symbol\n   * \\param name name of the input symbol\n   * \\param symbol the input symbol\n   * \\return reference of self\n   */\n  Operator& SetInput(const std::string& name, const Symbol& symbol);\n  /*!\n   * \\brief add an input symbol\n   * \\param symbol the input symbol\n   */\n  template <int N = 0>\n  void PushInput(const Symbol& symbol) {\n    input_symbols_.push_back(symbol.GetHandle());\n  }\n  /*!\n   * \\brief add input symbols\n   * \\return reference of self\n   */\n  Operator& operator()() {\n    return *this;\n  }\n  /*!\n   * \\brief add input symbols\n   * \\param symbol the input symbol\n   * \\return reference of self\n   */\n  Operator& operator()(const Symbol& symbol) {\n    input_symbols_.push_back(symbol.GetHandle());\n    return *this;\n  }\n  /*!\n   * \\brief add a list of input symbols\n   * \\param symbols the vector of the input symbols\n   * \\return reference of self\n   */\n  Operator& operator()(const std::vector<Symbol>& symbols) {\n    for (auto& s : symbols) {\n      input_symbols_.push_back(s.GetHandle());\n    }\n    return *this;\n  }\n  /*!\n   * \\brief create a Symbol from the current operator\n   * \\param name the name of the operator\n   * \\return the operator Symbol\n   */\n  Symbol CreateSymbol(const std::string& name = \"\");\n\n  /*!\n   * \\brief add an input ndarray\n   * \\param name name of the input ndarray\n   * \\param ndarray the input ndarray\n   * \\return reference of self\n   */\n  Operator& SetInput(const std::string& name, const NDArray& ndarray);\n  /*!\n   * \\brief add an input ndarray\n   * \\param ndarray the input ndarray\n   */\n  template <int N = 0>\n  Operator& PushInput(const NDArray& ndarray) {\n    input_ndarrays_.push_back(ndarray.GetHandle());\n    return *this;\n  }\n  /*!\n   * \\brief add positional inputs\n   */\n  template <class T, class... Args, int N = 0>\n  Operator& PushInput(const T& t, Args... args) {\n    SetParam(N, t);\n    PushInput<Args..., N + 1>(args...);\n    return *this;\n  }\n  /*!\n   * \\brief add the last positional input\n   */\n  template <class T, int N = 0>\n  Operator& PushInput(const T& t) {\n    SetParam(N, t);\n    return *this;\n  }\n  /*!\n   * \\brief add input ndarrays\n   * \\param ndarray the input ndarray\n   * \\return reference of self\n   */\n  Operator& operator()(const NDArray& ndarray) {\n    input_ndarrays_.push_back(ndarray.GetHandle());\n    return *this;\n  }\n  /*!\n   * \\brief add a list of input ndarrays\n   * \\param ndarrays the vector of the input ndarrays\n   * \\return reference of self\n   */\n  Operator& operator()(const std::vector<NDArray>& ndarrays) {\n    for (auto& s : ndarrays) {\n      input_ndarrays_.push_back(s.GetHandle());\n    }\n    return *this;\n  }\n  /*!\n   * \\brief add input ndarrays\n   * \\return reference of self\n   */\n  template <typename... Args>\n  Operator& operator()(Args... args) {\n    PushInput(args...);\n    return *this;\n  }\n  std::vector<NDArray> Invoke();\n  void Invoke(NDArray& output);\n  void Invoke(std::vector<NDArray>& outputs);\n\n private:\n  std::map<std::string, std::string> params_desc_;\n  bool variable_params_ = false;\n  std::map<std::string, std::string> params_;\n  std::vector<SymbolHandle> input_symbols_;\n  std::vector<NDArrayHandle> input_ndarrays_;\n  std::vector<std::string> input_keys_;\n  std::vector<std::string> arg_names_;\n  AtomicSymbolCreator handle_;\n  static OpMap*& op_map();\n};\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_OPERATOR_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/operator.hpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n* \\file operator.hpp\n* \\brief implementation of operator\n* \\author Chuntao Hong, Zhang Chen\n*/\n\n#ifndef MXNET_CPP_OPERATOR_HPP_\n#define MXNET_CPP_OPERATOR_HPP_\n\n#include <algorithm>\n#include <string>\n#include <vector>\n#include <iterator>\n#include \"mxnet-cpp/base.h\"\n#include \"mxnet-cpp/op_map.h\"\n#include \"mxnet-cpp/operator.h\"\n\nnamespace mxnet {\nnamespace cpp {\n\n/*\n * Pushing NDArray or Symbol as inputs here to avoid partial specialization\n * like PushInput<NDArray, Args..., N>, which is not allowed in C++\n */\ntemplate <>\ninline Operator& Operator::SetParam<NDArray>(int pos, const NDArray &value) {\n  input_ndarrays_.push_back(value.GetHandle());\n  return *this;\n}\ntemplate <>\ninline Operator& Operator::SetParam<Symbol>(int pos, const Symbol &value) {\n  input_symbols_.push_back(value.GetHandle());\n  return *this;\n}\n\ninline OpMap*& Operator::op_map() {\n  static OpMap *op_map_ = new OpMap();\n  return op_map_;\n}\n\ninline Operator::Operator(const std::string &operator_name) {\n  handle_ = op_map()->GetSymbolCreator(operator_name);\n  const char *name;\n  const char *description;\n  mx_uint num_args;\n  const char **arg_names;\n  const char **arg_type_infos;\n  const char **arg_descriptions;\n  const char *key_var_num_args;\n  MXSymbolGetAtomicSymbolInfo(handle_,\n      &name,\n      &description,\n      &num_args,\n      &arg_names,\n      &arg_type_infos,\n      &arg_descriptions,\n      &key_var_num_args);\n  for (mx_uint i = 0; i < num_args; ++i) {\n    arg_names_.push_back(arg_names[i]);\n  }\n}\n\ninline Symbol Operator::CreateSymbol(const std::string &name) {\n  if (input_keys_.size() > 0) {\n    CHECK_EQ(input_keys_.size(), input_symbols_.size());\n  }\n  const char *pname = name == \"\" ? nullptr : name.c_str();\n\n  SymbolHandle symbol_handle;\n  std::vector<const char *> input_keys;\n  std::vector<const char *> param_keys;\n  std::vector<const char *> param_values;\n\n  for (auto &data : params_) {\n    param_keys.push_back(data.first.c_str());\n    param_values.push_back(data.second.c_str());\n  }\n  for (auto &data : this->input_keys_) {\n    input_keys.push_back(data.c_str());\n  }\n  const char **input_keys_p =\n      (input_keys.size() > 0) ? input_keys.data() : nullptr;\n\n  MXSymbolCreateAtomicSymbol(handle_, param_keys.size(), param_keys.data(),\n                             param_values.data(), &symbol_handle);\n  MXSymbolCompose(symbol_handle, pname, input_symbols_.size(), input_keys_p,\n                  input_symbols_.data());\n  return Symbol(symbol_handle);\n}\n\ninline void Operator::Invoke(std::vector<NDArray> &outputs) {\n  if (input_keys_.size() > 0) {\n    CHECK_EQ(input_keys_.size(), input_ndarrays_.size());\n  }\n\n  std::vector<const char *> input_keys;\n  std::vector<const char *> param_keys;\n  std::vector<const char *> param_values;\n\n  for (auto &data : params_) {\n    param_keys.push_back(data.first.c_str());\n    param_values.push_back(data.second.c_str());\n  }\n\n  int num_inputs = input_ndarrays_.size();\n  int num_outputs = outputs.size();\n  std::vector<NDArrayHandle> output_handles;\n  std::transform(outputs.begin(), outputs.end(),\n      std::back_inserter(output_handles), [](NDArray& a) {\n        return a.GetHandle();\n      });\n\n  NDArrayHandle *outputs_receiver = nullptr;\n  if (num_outputs > 0) {\n    outputs_receiver = output_handles.data();\n  }\n\n  if (MXImperativeInvoke(handle_, num_inputs, input_ndarrays_.data(),\n                         &num_outputs, &outputs_receiver,\n                         param_keys.size(), param_keys.data(),\n                         param_values.data(), nullptr))\n      LOG(FATAL) << MXGetLastError();\n\n  if (outputs.size() > 0)\n    return;\n\n  std::transform(outputs_receiver, outputs_receiver+num_outputs,\n      std::back_inserter(outputs), [](const NDArrayHandle& handle) {\n        return NDArray(handle);\n      });\n}\n\ninline std::vector<NDArray> Operator::Invoke() {\n  std::vector<NDArray> outputs;\n  Invoke(outputs);\n  return outputs;\n}\n\ninline void Operator::Invoke(NDArray &output) {\n  std::vector<NDArray> outputs{output};\n  Invoke(outputs);\n}\n\ninline Operator &Operator::SetInput(const std::string &name, const Symbol &symbol) {\n    if (symbol.GetHandle()) {\n      input_keys_.push_back(name.c_str());\n      input_symbols_.push_back(symbol.GetHandle());\n    }\n    return *this;\n}\n\ninline Operator &Operator::SetInput(const std::string &name, const NDArray &ndarray) {\n  input_keys_.push_back(name.c_str());\n  input_ndarrays_.push_back(ndarray.GetHandle());\n  return *this;\n}\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_OPERATOR_HPP_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/optimizer.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file optimizer.h\n * \\brief definition of optimizer\n * \\author Chuntao Hong, Zhang Chen\n */\n\n#ifndef MXNET_CPP_OPTIMIZER_H_\n#define MXNET_CPP_OPTIMIZER_H_\n\n#include <dmlc/strtonum.h>\n#include <map>\n#include <vector>\n#include <string>\n#include <memory>\n#include <functional>\n#include \"mxnet-cpp/base.h\"\n#include \"dmlc/logging.h\"\n#include \"mxnet-cpp/ndarray.h\"\n#include \"mxnet-cpp/op_map.h\"\n#include \"mxnet-cpp/lr_scheduler.h\"\n\nnamespace mxnet {\nnamespace cpp {\n\n/*!\n * \\brief Optimizer interface\n */\nclass Optimizer {\n public:\n  /*!\n   * \\brief constructor\n   * \\param beign_num_update The initial number of updates\n   */\n  explicit Optimizer(unsigned begin_num_update);\n  /*!\n   * \\brief get optimizer type\n   * \\return string of optimizer type\n   */\n  virtual std::string GetType() const = 0;\n  /*!\n   * \\brief destructor\n   */\n  virtual ~Optimizer();\n  /*!\n   * \\brief set config parameters\n   * \\param name name of the config parameter\n   * \\param value value of the config parameter\n   * \\return reference of self\n   */\n  template <typename T>\n  Optimizer* SetParam(const std::string& name, const T& value) {\n    std::string value_str;\n    std::stringstream ss;\n    ss << value;\n    ss >> value_str;\n\n    params_[name] = value_str;\n    return this;\n  }\n  /*!\n   * \\bried set the lr scheduler\n   * \\param lrScheduler lr scheduler used for this optimizer\n   * \\return reference if self\n   */\n  Optimizer* SetLRScheduler(std::unique_ptr<LRScheduler> lrScheduler) {\n    CHECK(lrScheduler);\n    lrScheduler_ = std::move(lrScheduler);\n    lrScheduler_->SetLR(dmlc::stof(params_[\"lr\"]));\n    return this;\n  }\n  /*!\n   *  \\brief Update a weight with gradient.\n   *  \\param index the unique index for the weight.\n   *  \\param weight the weight to update.\n   *  \\param grad gradient for the weight.\n   */\n  virtual void Update(int index, NDArray weight, NDArray grad) = 0;\n  // TODO(zhangcheng-qinyinghua)\n  // implement Update a list of arrays, maybe in the form of map\n  // void Update(int index, std::vector<NDArray> weights, std::vector<NDArray>\n  // grad, mx_float lr);\n\n  /*!\n   *  \\brief Serialize the optimizer parameters to a string.\n   *  \\return serialization\n   */\n  std::string Serialize() const;\n\n protected:\n  std::map<std::string, std::string> params_;\n  static OpMap*& op_map();\n  const std::vector<const char*> GetParamKeys_() const;\n  const std::vector<const char*> GetParamValues_() const;\n  std::map<int, unsigned> count_;\n  unsigned begin_num_update_, num_update_;\n  unsigned UpdateCount_(int index);\n  float GetLR_(int index);\n  float GetWD_(int index);\n  virtual void CreateState_(int index, NDArray weight);\n  std::unique_ptr<LRScheduler> lrScheduler_ = nullptr;\n};\n\ntypedef std::function<Optimizer*()> OptimizerCreator;\n\nclass OptimizerRegistry {\n public:\n  static Optimizer* Find(const std::string& name);\n  static int __REGISTER__(const std::string& name, OptimizerCreator creator);\n\n private:\n  static std::map<std::string, OptimizerCreator>& cmap();\n  OptimizerRegistry()  = delete;\n  ~OptimizerRegistry() = delete;\n};\n#define MXNETCPP_REGISTER_OPTIMIZER(Name, OptimizerType) \\\n  OptimizerRegistry::__REGISTER__(#Name, []() { return new OptimizerType(); })\n\nclass SGDOptimizer : public Optimizer {\n public:\n  explicit SGDOptimizer(unsigned begin_num_update = 0);\n  std::string GetType() const override;\n  void Update(int index, NDArray weight, NDArray grad) override;\n\n private:\n  virtual ~SGDOptimizer();\n  void CreateState_(int index, NDArray weight) override;\n  std::map<int, NDArray*> states_;\n  AtomicSymbolCreator update_handle_;\n  AtomicSymbolCreator mom_update_handle_;\n};\n\nclass SignumOptimizer : public Optimizer {\n public:\n  explicit SignumOptimizer(unsigned begin_num_update = 0);\n  std::string GetType() const override;\n  void Update(int index, NDArray weight, NDArray grad) override;\n\n private:\n  virtual ~SignumOptimizer();\n  void CreateState_(int index, NDArray weight) override;\n  std::map<int, NDArray*> states_;\n  AtomicSymbolCreator update_handle_;\n  AtomicSymbolCreator mom_update_handle_;\n};\n\nclass RMSPropOptimizer : public Optimizer {\n public:\n  explicit RMSPropOptimizer(unsigned begin_num_update = 0);\n  std::string GetType() const override;\n  void Update(int index, NDArray weight, NDArray grad) override;\n\n private:\n  virtual ~RMSPropOptimizer();\n  void CreateState_(int index, NDArray weight) override;\n  std::map<int, NDArray*> n_, g_, delta_;\n  AtomicSymbolCreator update_handle_;\n  AtomicSymbolCreator alex_update_handle_;\n};\n\nclass AdamOptimizer : public Optimizer {\n public:\n  explicit AdamOptimizer(unsigned begin_num_update = 0);\n  std::string GetType() const override;\n  void Update(int index, NDArray weight, NDArray grad) override;\n\n private:\n  virtual ~AdamOptimizer();\n  void CreateState_(int index, NDArray weight) override;\n  std::map<int, NDArray*> mean_;\n  std::map<int, NDArray*> var_;\n  AtomicSymbolCreator update_handle_;\n};\n\nclass AdaGradOptimizer : public Optimizer {\n public:\n  explicit AdaGradOptimizer(unsigned begin_num_update = 0);\n  std::string GetType() const override;\n  void Update(int index, NDArray weight, NDArray grad) override;\n\n private:\n  virtual ~AdaGradOptimizer();\n  void CreateState_(int index, NDArray weight) override;\n  std::map<int, NDArray*> history_;\n};\n\nclass AdaDeltaOptimizer : public Optimizer {\n public:\n  explicit AdaDeltaOptimizer(unsigned begin_num_update = 0);\n  std::string GetType() const override;\n  void Update(int index, NDArray weight, NDArray grad) override;\n\n private:\n  virtual ~AdaDeltaOptimizer();\n  void CreateState_(int index, NDArray weight) override;\n  std::map<int, NDArray*> acc_g_, acc_delta_;\n};\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_OPTIMIZER_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/optimizer.hpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n* \\file optimizer.hpp\n* \\brief implementation of optimizer\n* \\author Chuntao Hong, Zhang Chen\n*/\n\n#ifndef MXNET_CPP_OPTIMIZER_HPP_\n#define MXNET_CPP_OPTIMIZER_HPP_\n\n#include <dmlc/strtonum.h>\n#include <algorithm>\n#include <utility>\n#include <numeric>\n#include <map>\n#include <cmath>\n#include <string>\n#include <vector>\n#include \"mxnet-cpp/optimizer.h\"\n#include \"mxnet-cpp/op.h\"\n#include \"mxnet-cpp/op_map.h\"\n\nnamespace {\n\n// TODO(lx75249): Add imperative operators to op.h under ndarray namespace\ninline void _clip(mxnet::cpp::NDArray &data, float limit) {\n  data = mxnet::cpp::Operator(\"clip\")\n    .SetParam(\"a_min\", -limit)\n    .SetParam(\"a_max\", limit)\n    .SetInput(\"data\", data)\n    .Invoke()[0];\n}\ninline mxnet::cpp::NDArray _sqrt(mxnet::cpp::NDArray data) {\n  return mxnet::cpp::Operator(\"sqrt\")\n    .SetInput(\"data\", data)\n    .Invoke()[0];\n}\n\n}  // namespace\n\nnamespace mxnet {\nnamespace cpp {\ninline Optimizer::Optimizer(unsigned begin_num_update)\n  : begin_num_update_(begin_num_update),\n    num_update_(begin_num_update_) {\n  params_[\"lr\"] = \"0.01f\";\n  params_[\"wd\"] = \"0.f\";\n}\n\ninline std::map<std::string, OptimizerCreator>& OptimizerRegistry::cmap() {\n  static std::map<std::string, OptimizerCreator> cmap_;\n  return cmap_;\n}\n\ninline OpMap*& Optimizer::op_map() {\n  static OpMap *op_map_ = new OpMap();\n  return op_map_;\n}\n\ninline Optimizer::~Optimizer() {}\n\ninline void Optimizer::CreateState_(int index, NDArray weight) {\n}\n\ninline std::string Optimizer::Serialize() const {\n  using ValueType = std::map<std::string, std::string>::value_type;\n  auto params = params_;\n  params.emplace(\"opt_type\", GetType());\n  return std::accumulate(params.cbegin(), params.cend(), std::string(\"\"),\n    [](const std::string& sum, const ValueType& i) {\n      return sum + '\\n' + i.first + '=' + i.second;\n    }).substr(1);\n}\n\ninline const std::vector<const char*> Optimizer::GetParamKeys_() const {\n  std::vector<const char*> keys;\n  for (auto& iter : params_)\n    keys.push_back(iter.first.c_str());\n  return keys;\n}\n\ninline const std::vector<const char*> Optimizer::GetParamValues_() const {\n  std::vector<const char*> values;\n  for (auto& iter : params_)\n    values.push_back(iter.second.c_str());\n  return values;\n}\n\ninline unsigned Optimizer::UpdateCount_(int index) {\n  if (count_.count(index) == 0) {\n    count_.emplace(index, begin_num_update_);\n  }\n  unsigned new_count = ++count_[index];\n  num_update_ = std::max(num_update_, new_count);\n  return new_count;\n}\n\ninline float Optimizer::GetLR_(int index) {\n  if (nullptr != lrScheduler_) {\n    return lrScheduler_->GetLR(num_update_);\n  }\n  return dmlc::stof(params_[\"lr\"]);\n}\n\ninline float Optimizer::GetWD_(int index) {\n  float wd = dmlc::stof(params_[\"wd\"]);\n  return wd;\n}\n\ninline Optimizer* OptimizerRegistry::Find(const std::string& name) {\n  if (cmap().empty()) {\n    // Optimizers should only be registered once\n    MXNETCPP_REGISTER_OPTIMIZER(sgd, SGDOptimizer);\n    MXNETCPP_REGISTER_OPTIMIZER(ccsgd, SGDOptimizer);  // For backward compatibility\n    MXNETCPP_REGISTER_OPTIMIZER(rmsprop, RMSPropOptimizer);\n    MXNETCPP_REGISTER_OPTIMIZER(adam, AdamOptimizer);\n    MXNETCPP_REGISTER_OPTIMIZER(adagrad, AdaGradOptimizer);\n    MXNETCPP_REGISTER_OPTIMIZER(adadelta, AdaDeltaOptimizer);\n    MXNETCPP_REGISTER_OPTIMIZER(signum, SignumOptimizer);\n  }\n  auto it = cmap().find(name);\n  if (it == cmap().end())\n    return nullptr;\n  return it->second();\n}\n\ninline int OptimizerRegistry::__REGISTER__(const std::string& name, OptimizerCreator creator) {\n  CHECK_EQ(cmap().count(name), 0) << name << \" already registered\";\n  cmap().emplace(name, std::move(creator));\n  return 0;\n}\n\ninline SGDOptimizer::SGDOptimizer(unsigned begin_num_update)\n  : Optimizer(begin_num_update) {\n  update_handle_ = op_map()->GetSymbolCreator(\"sgd_update\");\n  mom_update_handle_ = op_map()->GetSymbolCreator(\"sgd_mom_update\");\n}\n\ninline std::string SGDOptimizer::GetType() const {\n  return \"sgd\";\n}\n\ninline SGDOptimizer::~SGDOptimizer() {\n  for (auto &it : states_) {\n    delete it.second;\n  }\n}\n\ninline void SGDOptimizer::Update(int index, NDArray weight, NDArray grad) {\n  if (states_.count(index) == 0) {\n    CreateState_(index, weight);\n  }\n\n  params_[\"lr\"] = std::to_string(GetLR_(index));\n  params_[\"wd\"] = std::to_string(GetWD_(index));\n  UpdateCount_(index);\n  auto keys = GetParamKeys_();\n  auto values = GetParamValues_();\n  CHECK_EQ(keys.size(), values.size());\n\n  NDArrayHandle inputs[3];\n  inputs[0] = weight.GetHandle();\n  inputs[1] = grad.GetHandle();\n\n  int num_outputs = 1;\n  NDArrayHandle output = weight.GetHandle();\n  NDArrayHandle *outputs = &output;\n\n  if (states_[index] == nullptr) {\n    MXImperativeInvoke(update_handle_, 2, inputs,\n        &num_outputs, &outputs,\n        keys.size(), keys.data(), values.data(), nullptr);\n  } else {\n    inputs[2] = states_[index]->GetHandle();\n    MXImperativeInvoke(mom_update_handle_, 3, inputs,\n        &num_outputs, &outputs,\n        keys.size(), keys.data(), values.data(), nullptr);\n  }\n}\n\ninline void SGDOptimizer::CreateState_(int index, NDArray weight) {\n  if (params_.count(\"momentum\") == 0) {\n    states_[index] = nullptr;\n  } else {\n    states_[index] = new NDArray(weight.GetShape(), weight.GetContext());\n    *states_[index] = 0;\n  }\n}\n\n// inplementing Signum optimizer\n\ninline SignumOptimizer::SignumOptimizer(unsigned begin_num_update)\n  : Optimizer(begin_num_update) {\n  update_handle_ = op_map()->GetSymbolCreator(\"signsgd_update\");\n  mom_update_handle_ = op_map()->GetSymbolCreator(\"signum_update\");\n}\n\ninline std::string SignumOptimizer::GetType() const {\n  return \"signum\";\n}\n\ninline SignumOptimizer::~SignumOptimizer() {\n  for (auto &it : states_) {\n    delete it.second;\n  }\n}\n\ninline void SignumOptimizer::Update(int index, NDArray weight, NDArray grad) {\n  if (states_.count(index) == 0) {\n    CreateState_(index, weight);\n  }\n\n  params_[\"lr\"] = std::to_string(GetLR_(index));\n  params_[\"wd\"] = std::to_string(GetWD_(index));\n  UpdateCount_(index);\n  auto keys = GetParamKeys_();\n  auto values = GetParamValues_();\n  CHECK_EQ(keys.size(), values.size());\n\n  NDArrayHandle inputs[3];\n  inputs[0] = weight.GetHandle();\n  inputs[1] = grad.GetHandle();\n\n  int num_outputs = 1;\n  NDArrayHandle output = weight.GetHandle();\n  NDArrayHandle *outputs = &output;\n\n  if (states_[index] == nullptr) {\n    MXImperativeInvoke(update_handle_, 2, inputs,\n        &num_outputs, &outputs,\n        keys.size(), keys.data(), values.data(), nullptr);\n  } else {\n    inputs[2] = states_[index]->GetHandle();\n    MXImperativeInvoke(mom_update_handle_, 3, inputs,\n        &num_outputs, &outputs,\n        keys.size(), keys.data(), values.data(), nullptr);\n  }\n}\n\ninline void SignumOptimizer::CreateState_(int index, NDArray weight) {\n  if (params_.count(\"momentum\") == 0) {\n    states_[index] = nullptr;\n  } else {\n    states_[index] = new NDArray(weight.GetShape(), weight.GetContext());\n    *states_[index] = 0;\n  }\n}\n\n// finish implementing Signum\n\n\n\ninline RMSPropOptimizer::RMSPropOptimizer(unsigned begin_num_update)\n  : Optimizer(begin_num_update) {\n  update_handle_ = op_map()->GetSymbolCreator(\"rmsprop_update\");\n  alex_update_handle_ = op_map()->GetSymbolCreator(\"rmspropalex_update\");\n  SetParam(\"gamma1\", 0.9f);\n  SetParam(\"gamma2\", 0.9f);\n  SetParam(\"epsilon\", 1e-8);\n}\n\ninline std::string RMSPropOptimizer::GetType() const {\n  return \"rmsprop\";\n}\n\ninline RMSPropOptimizer::~RMSPropOptimizer() {\n  for (auto &it : n_) {\n    delete it.second;\n  }\n  for (auto &it : g_) {\n    delete it.second;\n  }\n  for (auto &it : delta_) {\n    delete it.second;\n  }\n}\n\ninline void RMSPropOptimizer::Update(int index, NDArray weight, NDArray grad) {\n  if (n_.count(index) == 0) {\n    CreateState_(index, weight);\n  }\n\n  params_[\"lr\"] = std::to_string(GetLR_(index));\n  params_[\"wd\"] = std::to_string(GetWD_(index));\n  UpdateCount_(index);\n  auto keys = GetParamKeys_();\n  auto values = GetParamValues_();\n  CHECK_EQ(keys.size(), values.size());\n\n  NDArrayHandle inputs[5];\n  inputs[0] = weight.GetHandle();\n  inputs[1] = grad.GetHandle();\n  inputs[2] = n_[index]->GetHandle();\n  inputs[3] = g_[index]->GetHandle();\n  inputs[4] = delta_[index]->GetHandle();\n\n  int num_outputs = 1;\n  NDArrayHandle output = weight.GetHandle();\n  NDArrayHandle *outputs = &output;\n\n  MXImperativeInvoke(alex_update_handle_, 5, inputs,\n      &num_outputs, &outputs,\n      keys.size(), keys.data(), values.data(), nullptr);\n}\n\ninline void RMSPropOptimizer::CreateState_(int index, NDArray weight) {\n  n_[index] = new NDArray(weight.GetShape(), weight.GetContext());\n  *n_[index] = 0;\n  g_[index] = new NDArray(weight.GetShape(), weight.GetContext());\n  *g_[index] = 0;\n  delta_[index] = new NDArray(weight.GetShape(), weight.GetContext());\n  *delta_[index] = 0;\n}\n\ninline AdamOptimizer::AdamOptimizer(unsigned begin_num_update)\n  : Optimizer(begin_num_update) {\n  update_handle_ = op_map()->GetSymbolCreator(\"adam_update\");\n  SetParam(\"beta1\", 0.9f);\n  SetParam(\"beta2\", 0.999f);\n  SetParam(\"epsilon\", 1e-8);\n}\n\ninline std::string AdamOptimizer::GetType() const {\n  return \"adam\";\n}\n\ninline AdamOptimizer::~AdamOptimizer() {\n  for (auto &it : mean_) {\n    delete it.second;\n  }\n  for (auto &it : var_) {\n    delete it.second;\n  }\n}\n\ninline void AdamOptimizer::Update(int index, NDArray weight, NDArray grad) {\n  if (mean_.count(index) == 0) {\n    CreateState_(index, weight);\n  }\n\n  params_[\"lr\"] = std::to_string(GetLR_(index));\n  params_[\"wd\"] = std::to_string(GetWD_(index));\n  UpdateCount_(index);\n  auto keys = GetParamKeys_();\n  auto values = GetParamValues_();\n  CHECK_EQ(keys.size(), values.size());\n\n  float lr = dmlc::stof(params_[\"lr\"]);\n  float b1 = dmlc::stof(params_[\"beta1\"]);\n  float b2 = dmlc::stof(params_[\"beta2\"]);\n  float t = count_[index];\n  float coef1 = 1.0f - std::pow(b1, t);\n  float coef2 = 1.0f - std::pow(b2, t);\n  lr *= std::sqrt(coef2) / coef1;\n\n  NDArrayHandle inputs[4];\n  inputs[0] = weight.GetHandle();\n  inputs[1] = grad.GetHandle();\n\n  int num_outputs = 1;\n  NDArrayHandle output = weight.GetHandle();\n  NDArrayHandle *outputs = &output;\n\n  inputs[2] = mean_[index]->GetHandle();\n  inputs[3] = var_[index]->GetHandle();\n\n  MXImperativeInvoke(update_handle_, 4, inputs,\n    &num_outputs, &outputs,\n    keys.size(), keys.data(), values.data(), nullptr);\n}\n\ninline void AdamOptimizer::CreateState_(int index, NDArray weight) {\n  mean_[index] = new NDArray(weight.GetShape(), weight.GetContext());\n  *mean_[index] = 0;\n  var_[index] = new NDArray(weight.GetShape(), weight.GetContext());\n  *var_[index] = 0;\n}\n\ninline AdaGradOptimizer::AdaGradOptimizer(unsigned begin_num_update)\n  : Optimizer(begin_num_update) {\n  SetParam(\"eps\", 1e-7);\n}\n\ninline std::string AdaGradOptimizer::GetType() const {\n  return \"adagrad\";\n}\n\ninline void AdaGradOptimizer::Update(int index, NDArray weight, NDArray grad) {\n  if (history_.count(index) == 0) {\n    CreateState_(index, weight);\n  }\n\n  float eps = dmlc::stof(params_[\"eps\"]);\n  float lr = GetLR_(index);\n  float wd = GetWD_(index);\n  UpdateCount_(index);\n  if (params_.count(\"rescale_grad\") > 0) {\n    grad *= dmlc::stof(params_[\"rescale_grad\"]);\n  }\n  if (params_.count(\"clip_gradient\") > 0) {\n    _clip(grad, dmlc::stof(params_[\"clip_gradient\"]));\n  }\n  auto& history = *history_[index];\n  history += grad * grad;\n  weight -= (grad / _sqrt(history + eps) + weight * wd) * lr;\n}\n\ninline AdaGradOptimizer::~AdaGradOptimizer() {\n  for (auto& it : history_) {\n    delete it.second;\n  }\n}\n\ninline void AdaGradOptimizer::CreateState_(int index, NDArray weight) {\n  history_[index] = new NDArray(weight.GetShape(), weight.GetContext());\n  *history_[index] = 0;\n}\n\ninline AdaDeltaOptimizer::AdaDeltaOptimizer(unsigned begin_num_update)\n  : Optimizer(begin_num_update) {\n  SetParam(\"rho\", 0.90f);\n  SetParam(\"epsilon\", 1e-5);\n}\n\ninline std::string AdaDeltaOptimizer::GetType() const {\n  return \"adadelta\";\n}\n\ninline void AdaDeltaOptimizer::Update(int index, NDArray weight, NDArray grad) {\n  if (acc_g_.count(index) == 0) {\n    CreateState_(index, weight);\n  }\n\n  float rho = dmlc::stof(params_[\"rho\"]);\n  float epsilon = dmlc::stof(params_[\"epsilon\"]);\n  float wd = GetWD_(index);\n  UpdateCount_(index);\n\n  if (params_.count(\"rescale_grad\") > 0) {\n    grad *= dmlc::stof(params_[\"rescale_grad\"]);\n  }\n  if (params_.count(\"clip_gradient\") > 0) {\n    _clip(grad, dmlc::stof(params_[\"clip_gradient\"]));\n  }\n\n  auto& acc_g = *acc_g_[index];\n  auto& acc_delta = *acc_delta_[index];\n  acc_g *= rho;\n  acc_g += grad * grad * (1.0f - rho);\n\n  auto delta = _sqrt(acc_delta + epsilon) / _sqrt(acc_g + epsilon) * grad;\n  acc_delta *= rho;\n  acc_delta += delta * delta * (1.0f - rho);\n  weight *= 1.0f - wd;\n  weight -= delta;\n}\n\ninline AdaDeltaOptimizer::~AdaDeltaOptimizer() {\n  for (auto& it : acc_g_) {\n    delete it.second;\n  }\n  for (auto& it : acc_delta_) {\n    delete it.second;\n  }\n}\n\ninline void AdaDeltaOptimizer::CreateState_(int index, NDArray weight) {\n  acc_g_[index] = new NDArray(weight.GetShape(), weight.GetContext());\n  *acc_g_[index] = 0;\n  acc_delta_[index] = new NDArray(weight.GetShape(), weight.GetContext());\n  *acc_delta_[index] = 0;\n}\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_OPTIMIZER_HPP_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/shape.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file shape.h\n * \\brief definition of shape\n * \\author Chuntao Hong, Zhang Chen\n */\n\n#ifndef MXNET_CPP_SHAPE_H_\n#define MXNET_CPP_SHAPE_H_\n\n#include <istream>\n#include <ostream>\n#include <algorithm>\n#include <vector>\n#include \"mxnet-cpp/base.h\"\n\nnamespace mxnet {\nnamespace cpp {\n\n/*!\n * \\brief dynamic shape class that can hold shape\n *   of arbirary dimension\n */\nstruct Shape {\n public:\n  /*! \\brief constructor */\n  Shape() : ndim_(0), num_heap_allocated_(0), data_heap_(nullptr) {}\n  /*!\n   * \\brief constructor from a vector of index_t\n   * \\param v the vector\n   */\n  explicit Shape(const std::vector<index_t>& v) : ndim_(v.size()) {\n    if (ndim_ <= kStackCache) {\n      data_heap_          = nullptr;\n      num_heap_allocated_ = 0;\n      std::copy(v.begin(), v.end(), data_stack_);\n    } else {\n      data_heap_          = new index_t[ndim_];\n      num_heap_allocated_ = ndim_;\n      std::copy(v.begin(), v.end(), data_heap_);\n    }\n  }\n  /*!\n   * \\brief constructor one dimmension shape\n   * \\param s1 size of the first dimmension\n   */\n  explicit Shape(index_t s1) : ndim_(1) {\n    if (ndim_ <= kStackCache) {\n      data_heap_          = nullptr;\n      num_heap_allocated_ = 0;\n      data_stack_[0]      = s1;\n    } else {\n      data_heap_          = new index_t[ndim_];\n      num_heap_allocated_ = ndim_;\n      data_heap_[0]       = s1;\n    }\n  }\n  /*!\n   * \\brief constructor two dimmension shape\n   * \\param s1 size of the first dimmension\n   * \\param s2 size of the second dimmension\n   */\n  Shape(index_t s1, index_t s2) : ndim_(2) {\n    if (ndim_ <= kStackCache) {\n      data_heap_          = nullptr;\n      num_heap_allocated_ = 0;\n      data_stack_[0]      = s1;\n      data_stack_[1]      = s2;\n    } else {\n      data_heap_          = new index_t[ndim_];\n      num_heap_allocated_ = ndim_;\n      data_heap_[0]       = s1;\n      data_heap_[1]       = s2;\n    }\n  }\n  /*!\n   * \\brief constructor three dimmension shape\n   * \\param s1 size of the first dimmension\n   * \\param s2 size of the second dimmension\n   * \\param s3 size of the third dimmension\n   */\n  Shape(index_t s1, index_t s2, index_t s3) : ndim_(3) {\n    if (ndim_ <= kStackCache) {\n      data_heap_          = nullptr;\n      num_heap_allocated_ = 0;\n      data_stack_[0]      = s1;\n      data_stack_[1]      = s2;\n      data_stack_[2]      = s3;\n    } else {\n      data_heap_          = new index_t[ndim_];\n      num_heap_allocated_ = ndim_;\n      data_heap_[0]       = s1;\n      data_heap_[1]       = s2;\n      data_heap_[2]       = s3;\n    }\n  }\n  /*!\n   * \\brief constructor four dimmension shape\n   * \\param s1 size of the first dimmension\n   * \\param s2 size of the second dimmension\n   * \\param s3 size of the third dimmension\n   * \\param s4 size of the fourth dimmension\n   */\n  Shape(index_t s1, index_t s2, index_t s3, index_t s4) : ndim_(4) {\n    if (ndim_ <= kStackCache) {\n      data_heap_          = nullptr;\n      num_heap_allocated_ = 0;\n      data_stack_[0]      = s1;\n      data_stack_[1]      = s2;\n      data_stack_[2]      = s3;\n      data_stack_[3]      = s4;\n    } else {\n      data_heap_          = new index_t[ndim_];\n      num_heap_allocated_ = ndim_;\n      data_heap_[0]       = s1;\n      data_heap_[1]       = s2;\n      data_heap_[2]       = s3;\n      data_heap_[3]       = s4;\n    }\n  }\n  /*!\n   * \\brief constructor five dimmension shape\n   * \\param s1 size of the first dimmension\n   * \\param s2 size of the second dimmension\n   * \\param s3 size of the third dimmension\n   * \\param s4 size of the fourth dimmension\n   * \\param s5 size of the fifth dimmension\n   */\n  Shape(index_t s1, index_t s2, index_t s3, index_t s4, index_t s5) : ndim_(5) {\n    if (ndim_ <= kStackCache) {\n      data_heap_          = nullptr;\n      num_heap_allocated_ = 0;\n      data_stack_[0]      = s1;\n      data_stack_[1]      = s2;\n      data_stack_[2]      = s3;\n      data_stack_[3]      = s4;\n      data_stack_[4]      = s5;\n    } else {\n      data_heap_          = new index_t[ndim_];\n      num_heap_allocated_ = ndim_;\n      data_heap_[0]       = s1;\n      data_heap_[1]       = s2;\n      data_heap_[2]       = s3;\n      data_heap_[3]       = s4;\n      data_heap_[4]       = s5;\n    }\n  }\n  /*!\n   * \\brief constructor from Shape\n   * \\param s the source shape\n   */\n  Shape(const Shape& s) : ndim_(s.ndim_) {\n    if (ndim_ <= kStackCache) {\n      data_heap_          = nullptr;\n      num_heap_allocated_ = 0;\n      std::copy(s.data_stack_, s.data_stack_ + ndim_, data_stack_);\n    } else {\n      data_heap_          = new index_t[ndim_];\n      num_heap_allocated_ = ndim_;\n      std::copy(s.data_heap_, s.data_heap_ + ndim_, data_heap_);\n    }\n  }\n#if MSHADOW_IN_CXX11\n  /*!\n   * \\brief move constructor from Shape\n   * \\param s the source shape\n   */\n  Shape(Shape&& s)\n      : ndim_(s.ndim_), num_heap_allocated_(s.num_heap_allocated_), data_heap_(s.data_heap_) {\n    if (ndim_ <= kStackCache) {\n      std::copy(s.data_stack_, s.data_stack_ + ndim_, data_stack_);\n    }\n    // remove data heap space from s\n    s.data_heap_ = nullptr;\n  }\n#endif\n  /*! \\brief destructor */\n  ~Shape() {\n    // data_heap_ can be nullptr\n    delete[] data_heap_;\n  }\n  /*!\n   * \\brief copy shape from content betwen two iterators\n   * \\param begin the beginning of iterator\n   * \\param end the end of the iterator\n   * \\tparam RandomAccessIterator iterator type\n   */\n  template <typename RandomAccessIterator>\n  inline void CopyFrom(RandomAccessIterator begin, RandomAccessIterator end) {\n    this->SetDim(end - begin);\n    std::copy(begin, end, data());\n  }\n  /*!\n   * \\brief assignment from shape\n   * \\param shape source shape\n   * \\return reference of self\n   */\n  inline Shape& operator=(const Shape& shape) {\n    this->SetDim(shape.ndim_);\n    const index_t* src = shape.data();\n    std::copy(src, src + ndim_, data());\n    return *this;\n  }\n  /*!\n   * \\brief assignment from vector\n   * \\param shape source shape\n   * \\return reference of self\n   */\n  inline Shape& operator=(const std::vector<index_t>& shape) {\n    this->CopyFrom(shape.begin(), shape.end());\n    return *this;\n  }\n  /*! \\return the data content of the shape */\n  inline const index_t* data() const {\n    return ndim_ <= kStackCache ? data_stack_ : data_heap_;\n  }\n  /*! \\return the data content of the shape */\n  inline index_t* data() {\n    return ndim_ <= kStackCache ? data_stack_ : data_heap_;\n  }\n  /*! \\brief return number of dimension of the tensor inside */\n  inline index_t ndim(void) const {\n    return ndim_;\n  }\n  /*!\n   * \\brief get corresponding index\n   * \\param i dimension index\n   * \\return the corresponding dimension size\n   */\n  inline index_t& operator[](index_t i) {\n    return data()[i];\n  }\n  /*!\n   * \\brief get corresponding index\n   * \\param i dimension index\n   * \\return the corresponding dimension size\n   */\n  inline const index_t& operator[](index_t i) const {\n    return data()[i];\n  }\n  /*! \\brief total number of elements in the tensor */\n  inline size_t Size(void) const {\n    size_t size      = 1;\n    const index_t* d = this->data();\n    for (index_t i = 0; i < ndim_; ++i) {\n      size *= d[i];\n    }\n    return size;\n  }\n  /*!\n   * \\return whether two shape equals\n   * \\param s the shape to compare against\n   */\n  inline bool operator==(const Shape& s) const {\n    if (ndim_ != s.ndim_)\n      return false;\n    if (ndim_ <= kStackCache) {\n      for (index_t i = 0; i < ndim_; ++i) {\n        if (data_stack_[i] != s.data_stack_[i])\n          return false;\n      }\n    } else {\n      for (index_t i = 0; i < ndim_; ++i) {\n        if (data_heap_[i] != s.data_heap_[i])\n          return false;\n      }\n    }\n    return true;\n  }\n  /*!\n   * \\return whether two shape not equals\n   * \\param s the shape to compare against\n   */\n  inline bool operator!=(const Shape& s) const {\n    return !(*this == s);\n  }\n\n  friend std::ostream& operator<<(std::ostream& os, const Shape& shape);\n  friend std::istream& operator>>(std::istream& is, Shape& shape);\n\n private:\n  // the shape will be stored in data_stack_\n  // when dimension is smaller than kStackCache\n  // when it is bigger, it will be stored in data_heap_;\n  /*! \\brief size of in stack space */\n  static const index_t kStackCache = 5;\n  /*! \\brief number of dimnsion of the shape */\n  index_t ndim_;\n  /*! \\brief number of cells allocated in data_heap_ */\n  index_t num_heap_allocated_;\n  /*! \\brief in stack space used to store shape when it is small */\n  index_t data_stack_[kStackCache];\n  /*! \\brief space to store shape when dimension is big*/\n  index_t* data_heap_;\n  /*!\n   * \\brief internal function to set the dimension\n   * \\param dim the dimension of the shape\n   */\n  inline void SetDim(index_t dim) {\n    if (dim > kStackCache && dim > num_heap_allocated_) {\n      // data_heap_ can be nullptr\n      delete[] data_heap_;\n      data_heap_          = new index_t[dim];\n      num_heap_allocated_ = dim;\n    }\n    ndim_ = dim;\n  }\n};\n\n/*!\n * \\brief allow string printing of the shape\n * \\param os the output stream\n * \\param shape the shape\n * \\return the ostream\n */\ninline std::ostream& operator<<(std::ostream& os, const Shape& shape) {\n  os << '(';\n  for (index_t i = 0; i < shape.ndim(); ++i) {\n    if (i != 0)\n      os << ',';\n    os << static_cast<int>(shape[i]);  // Supports negative Shape 'special codes' for inferring\n  }\n  // python style tuple\n  if (shape.ndim() == 1)\n    os << ',';\n  os << ')';\n  return os;\n}\n\n/*!\n * \\brief read shape from the istream\n * \\param is the input stream\n * \\param shape the shape\n * \\return the istream\n */\ninline std::istream& operator>>(std::istream& is, Shape& shape) {\n  // get (\n  while (true) {\n    char ch = is.get();\n    if (ch == '(')\n      break;\n    if (!isspace(ch)) {\n      is.setstate(std::ios::failbit);\n      return is;\n    }\n  }\n  index_t idx;\n  std::vector<index_t> tmp;\n  while (is >> idx) {\n    tmp.push_back(idx);\n    char ch;\n    do {\n      ch = is.get();\n    } while (isspace(ch));\n    if (ch == ',') {\n      while (true) {\n        ch = is.peek();\n        if (isspace(ch)) {\n          is.get();\n          continue;\n        }\n        if (ch == ')') {\n          is.get();\n          break;\n        }\n        break;\n      }\n      if (ch == ')')\n        break;\n    } else if (ch == ')') {\n      break;\n    } else {\n      is.setstate(std::ios::failbit);\n      return is;\n    }\n  }\n  shape.CopyFrom(tmp.begin(), tmp.end());\n  return is;\n}\n\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_SHAPE_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/symbol.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file symbol.h\n * \\brief definition of symbol\n * \\author Chuntao Hong, Zhang Chen\n */\n\n#ifndef MXNET_CPP_SYMBOL_H_\n#define MXNET_CPP_SYMBOL_H_\n\n#include <map>\n#include <string>\n#include <vector>\n#include \"mxnet-cpp/base.h\"\n#include \"mxnet-cpp/ndarray.h\"\n#include \"mxnet-cpp/op_map.h\"\n\nnamespace mxnet {\nnamespace cpp {\n\nclass Executor;\n\n/*!\n * \\brief struct to store SymbolHandle\n */\nstruct SymBlob {\n public:\n  /*!\n   * \\brief default constructor\n   */\n  SymBlob() : handle_(nullptr) {}\n  /*!\n   * \\brief construct with SymbolHandle to store\n   */\n  explicit SymBlob(SymbolHandle handle) : handle_(handle) {}\n  /*!\n   * \\brief destructor, free the SymbolHandle\n   */\n  ~SymBlob() {\n    MXSymbolFree(handle_);\n  }\n  /*!\n   * \\brief the SymbolHandle to store\n   */\n  SymbolHandle handle_;\n\n private:\n  SymBlob(const SymBlob&);\n  SymBlob& operator=(const SymBlob&);\n};\n\n/*!\n * \\brief Symbol interface\n */\nclass Symbol {\n public:\n  Symbol() {}\n  /*!\n   * \\brief construct a Symbol with SymbolHandle\n   * \\param handle the given SymbolHandle\n   */\n  explicit Symbol(SymbolHandle handle);\n  /*!\n   * \\brief construct a variable Symbol\n   * \\param name the name of the variable\n   */\n  explicit Symbol(const char* name);\n  /*!\n   * \\brief construct a variable Symbol\n   * \\param name the name of the variable\n   */\n  explicit Symbol(const std::string& name);\n  Symbol operator+(const Symbol& rhs) const;\n  Symbol operator-(const Symbol& rhs) const;\n  Symbol operator*(const Symbol& rhs) const;\n  Symbol operator/(const Symbol& rhs) const;\n  Symbol operator%(const Symbol& rhs) const;\n\n  Symbol operator+(mx_float scalar) const;\n  Symbol operator-(mx_float scalar) const;\n  Symbol operator*(mx_float scalar) const;\n  Symbol operator/(mx_float scalar) const;\n  Symbol operator%(mx_float scalar) const;\n  Symbol Copy() const;\n  /*!\n   * \\brief construct a variable Symbol\n   * \\param name the name of the variable\n   */\n  static Symbol Variable(const std::string& name = \"\");\n  Symbol operator[](int index);\n  Symbol operator[](const std::string& index);\n  /*!\n   * \\brief Create a symbol that groups symbols together\n   * \\param symbols List of symbols to be groupe\n   */\n  static Symbol Group(const std::vector<Symbol>& symbols);\n  /*!\n   * \\brief load Symbol from a JSON file\n   * \\param file_name the name of the file\n   */\n  static Symbol Load(const std::string& file_name);\n  /*!\n   * \\brief load Symbol from a JSON string\n   * \\param json_str the JSON string\n   */\n  static Symbol LoadJSON(const std::string& json_str);\n  /*!\n   * \\brief save Symbol to a file\n   * \\param file_name the name of the file\n   */\n  void Save(const std::string& file_name) const;\n  /*!\n   * \\brief save Symbol into a JSON string\n   */\n  std::string ToJSON() const;\n  /*!\n   * \\brief save Symbol into a JSON string\n   * \\retutrn the symbol whose outputs are all the internals.\n   */\n  Symbol GetInternals() const;\n  /*!\n   * \\return the SymbolHandle\n   */\n  SymbolHandle GetHandle() const {\n    return (blob_ptr_) ? blob_ptr_->handle_ : nullptr;\n  }\n  /*!\n   * \\brief construct an operator Symbol, with given input Symbol and config\n   * \\param name the name of the Symbol\n   * \\param input_keys the vector of keys of the input\n   * \\param input_values the vector of the intput Symbols\n   * \\param config_keys the vector of keys of the config\n   * \\param config_values the vecotr of values of the config\n   */\n  Symbol(const std::string& operator_name,\n         const std::string& name,\n         std::vector<const char*> input_keys,\n         std::vector<SymbolHandle> input_values,\n         std::vector<const char*> config_keys,\n         std::vector<const char*> config_values);\n  /*!\n   * \\brief infer the shapes by providing shapes of known argument shapes.\n   * \\param arg_shapes map of argument name to shape of arguments with known\n   * shapes.\n   * \\param in_shapes used to store infered shapes of input arguments.\n   * \\param out_shapes used to store infered shapes of outputs.\n   * \\param aux_shapes use to store the infered shapes of auxiliary states\n   */\n  void InferShape(const std::map<std::string, std::vector<mx_uint> >& arg_shapes,\n                  std::vector<std::vector<mx_uint> >* in_shape,\n                  std::vector<std::vector<mx_uint> >* aux_shape,\n                  std::vector<std::vector<mx_uint> >* out_shape) const;\n  /*!\n   * \\brief List the arguments names.\n   *\n   * The position of the returned list also corresponds to calling position in\n   *operator()\n   * \\return the arguments list of this symbol, they can be either named or\n   *unnamed (empty string).\n   */\n  std::vector<std::string> ListArguments() const;\n  /*! \\return lists all argument names and aux states of the symbol */\n  std::vector<std::string> ListInputs() const;\n  /*! \\return get the descriptions of outputs for this symbol */\n  std::vector<std::string> ListOutputs() const;\n  /*! \\return get the descriptions of auxiliary data for this symbol */\n  std::vector<std::string> ListAuxiliaryStates() const;\n  /*! \\return get all attributes for this symbol */\n  std::map<std::string, std::string> ListAttributes() const;\n  /*!\n   * \\brief set key-value attribute to the symbol\n   * @param key string represent the key for the attribute\n   * @param value string represent the value for the attribute\n   */\n  void SetAttribute(const std::string& key, const std::string& value);\n  /*!\n   * \\brief set a series of key-value attribute to the symbol\n   * @param attrs string:string map represent the key value attributes\n   */\n  void SetAttributes(const std::map<std::string, std::string>& attrs);\n  /*! \\return get number of outputs for this symbol */\n  mx_uint GetNumOutputs() const;\n  /*! \\return get the new symbol through subgraph API for this symbol */\n  mxnet::cpp::Symbol GetBackendSymbol(const std::string& backendName) const;\n  /*! \\return get the name of the symbol */\n  std::string GetName() const;\n  /*!\n   * \\brief infer and construct all the arrays to bind to executor by providing\n   * some known arrays.\n   * \\param context the context of all the infered arrays\n   * \\param arg_arrays infered input arguments arrays.\n   * \\param arad_arrays infered arrays to store the gradient output of the input\n   * arguments.\n   * \\param aux_arrays infered arrays that is used as internal state in op.\n   * \\param args_map map of some given arguments arrays.\n   * \\param args_grad_store map of some gradient given store arrays.\n   * \\param args_req_type map of some given type of gradient saving. Can only be\n   * in {kNullOp, kAddTo, kWriteTo}.\n   * \\param aux_map NDArray that stores the internal state in op\n   */\n  void InferExecutorArrays(\n      const Context& context,\n      std::vector<NDArray>* arg_arrays,\n      std::vector<NDArray>* grad_arrays,\n      std::vector<OpReqType>* grad_reqs,\n      std::vector<NDArray>* aux_arrays,\n      const std::map<std::string, NDArray>& args_map,\n      const std::map<std::string, NDArray>& arg_grad_store  = std::map<std::string, NDArray>(),\n      const std::map<std::string, OpReqType>& grad_req_type = std::map<std::string, OpReqType>(),\n      const std::map<std::string, NDArray>& aux_map = std::map<std::string, NDArray>()) const;\n  /*!\n   * \\brief infer and construct all the input arguments arrays to bind to\n   * executor by providing some known arguments arrays.\n   * \\param context the context of all the infered arrays.\n   * \\param args_map map of all the infered input arguments arrays.\n   * \\param known_args map of some given arguments arrays.\n   */\n  void InferArgsMap(const Context& context,\n                    std::map<std::string, NDArray>* args_map,\n                    const std::map<std::string, NDArray>& known_args) const;\n  /*!\n   * \\brief Create an executor by bind symbol with context and arguments.\n   *  If user do not want to compute the gradients of i-th argument,\n   *grad_req_type[i] can be kNullOp.\n   *  The input arrays in the given maps should have the same name with the input\n   *symbol.\n   *  Only need some of the necessary arrays, and the other arrays can be infered\n   *automatically.\n   *\n   * \\param context the context of binding.\n   * \\param args_map the NDArray that stores the input arguments to the symbol.\n   * \\param arg_grad_store NDArray that is used to store the gradient output of\n   *the input arguments.\n   * \\param grad_req_type requirment type of gradient saving. Can only be in\n   *{kNullOp, kAddTo, kWriteTo}.\n   * \\param aux_map NDArray that stores the internal state in op\n   * \\return a new executor, which need to be free manually.\n   */\n  Executor* SimpleBind(\n      const Context& context,\n      const std::map<std::string, NDArray>& args_map,\n      const std::map<std::string, NDArray>& arg_grad_store  = std::map<std::string, NDArray>(),\n      const std::map<std::string, OpReqType>& grad_req_type = std::map<std::string, OpReqType>(),\n      const std::map<std::string, NDArray>& aux_map         = std::map<std::string, NDArray>());\n  /*!\n   * \\brief Create an executor by bind symbol with context and arguments.\n   *  If user do not want to compute the gradients of i-th argument,\n   *grad_req_type[i] can be kNullOp.\n   *\n   * \\param context the context of binding.\n   * \\param arg_arrays the NDArray that stores the input arguments to the symbol.\n   * \\param grad_arrays NDArray that is used to store the gradient output of the\n   *input arguments.\n   * \\param grad_reqs requirment type of gradient saving. Can only be in\n   *{kNullOp, kAddTo, kWriteTo}.\n   * \\param aux_arrays NDArray that is used as internal state in op\n   * \\param group_to_ctx dict of string to mx.Context\n   * \\param shared_exec Executor to share memory with. This is intended for\n   *runtime reshaping, variable length sequencesn etc.  The returned executor\n   *shares state with shared_exec, and should not be used in parallel with it.\n   * \\return a new executor, which need to be free manually.\n   */\n  Executor* Bind(\n      const Context& context,\n      const std::vector<NDArray>& arg_arrays,\n      const std::vector<NDArray>& grad_arrays,\n      const std::vector<OpReqType>& grad_reqs,\n      const std::vector<NDArray>& aux_arrays,\n      const std::map<std::string, Context>& group_to_ctx = std::map<std::string, Context>(),\n      Executor* shared_exec                              = nullptr);\n\n private:\n  std::shared_ptr<SymBlob> blob_ptr_;\n  static OpMap*& op_map();\n};\nSymbol operator+(mx_float lhs, const Symbol& rhs);\nSymbol operator-(mx_float lhs, const Symbol& rhs);\nSymbol operator*(mx_float lhs, const Symbol& rhs);\nSymbol operator/(mx_float lhs, const Symbol& rhs);\nSymbol operator%(mx_float lhs, const Symbol& rhs);\n}  // namespace cpp\n}  // namespace mxnet\n#endif  // MXNET_CPP_SYMBOL_H_\n"
  },
  {
    "path": "cpp-package/include/mxnet-cpp/symbol.hpp",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file symbol.hpp\n * \\brief implementation of the symbol\n * \\author Zhang Chen, Chuntao Hong\n */\n\n#ifndef MXNET_CPP_SYMBOL_HPP_\n#define MXNET_CPP_SYMBOL_HPP_\n\n#include <map>\n#include <memory>\n#include <string>\n#include <vector>\n\n#include \"dmlc/logging.h\"\n#include \"mxnet-cpp/symbol.h\"\n\n#include \"mxnet-cpp/op_suppl.h\"\n\nnamespace mxnet {\nnamespace cpp {\ninline OpMap*& Symbol::op_map() {\n  static OpMap* op_map_ = new OpMap();\n  return op_map_;\n}\ninline Symbol::Symbol(SymbolHandle handle) {\n  blob_ptr_ = std::make_shared<SymBlob>(handle);\n}\ninline Symbol::Symbol(const char *name) {\n  SymbolHandle handle;\n  CHECK_EQ(MXSymbolCreateVariable(name, &(handle)), 0);\n  blob_ptr_ = std::make_shared<SymBlob>(handle);\n}\ninline Symbol::Symbol(const std::string &name) : Symbol(name.c_str()) {}\ninline Symbol Symbol::Variable(const std::string &name) { return Symbol(name); }\ninline Symbol Symbol::operator+(const Symbol &rhs) const { return _Plus(*this, rhs); }\ninline Symbol Symbol::operator-(const Symbol &rhs) const { return _Minus(*this, rhs); }\ninline Symbol Symbol::operator*(const Symbol &rhs) const { return _Mul(*this, rhs); }\ninline Symbol Symbol::operator/(const Symbol &rhs) const { return _Div(*this, rhs); }\ninline Symbol Symbol::operator%(const Symbol &rhs) const { return _Mod(*this, rhs); }\ninline Symbol Symbol::operator+(mx_float scalar) const {\n  return _PlusScalar(*this, scalar);\n}\ninline Symbol Symbol::operator-(mx_float scalar) const {\n  return _MinusScalar(*this, scalar);\n}\ninline Symbol Symbol::operator*(mx_float scalar) const {\n  return _MulScalar(*this, scalar);\n}\ninline Symbol Symbol::operator/(mx_float scalar) const {\n  return _DivScalar(*this, scalar);\n}\ninline Symbol Symbol::operator%(mx_float scalar) const {\n  return _ModScalar(*this, scalar);\n}\ninline Symbol Symbol::operator[](int index) {\n  SymbolHandle out;\n  MXSymbolGetOutput(GetHandle(), index, &out);\n  return Symbol(out);\n}\ninline Symbol Symbol::operator[](const std::string &index) {\n  auto outputs = ListOutputs();\n  for (mx_uint i = 0; i < outputs.size(); ++i) {\n    if (outputs[i] == index) {\n      return (*this)[i];\n    }\n  }\n  LOG(FATAL) << \"Cannot find output that matches name \" << index;\n  return (*this)[0];\n}\ninline Symbol Symbol::Group(const std::vector<Symbol> &symbols) {\n  SymbolHandle out;\n  std::vector<SymbolHandle> handle_list;\n  for (const auto &t : symbols) {\n    handle_list.push_back(t.GetHandle());\n  }\n  MXSymbolCreateGroup(handle_list.size(), handle_list.data(), &out);\n  return Symbol(out);\n}\ninline Symbol Symbol::Load(const std::string &file_name) {\n  op_map();\n  SymbolHandle handle;\n  CHECK_EQ(MXSymbolCreateFromFile(file_name.c_str(), &(handle)), 0);\n  return Symbol(handle);\n}\ninline Symbol Symbol::LoadJSON(const std::string &json_str) {\n  op_map();\n  SymbolHandle handle;\n  CHECK_EQ(MXSymbolCreateFromJSON(json_str.c_str(), &(handle)), 0);\n  return Symbol(handle);\n}\ninline void Symbol::Save(const std::string &file_name) const {\n  CHECK_EQ(MXSymbolSaveToFile(GetHandle(), file_name.c_str()), 0);\n}\ninline std::string Symbol::ToJSON() const {\n  const char *out_json;\n  CHECK_EQ(MXSymbolSaveToJSON(GetHandle(), &out_json), 0);\n  return std::string(out_json);\n}\ninline Symbol Symbol::GetInternals() const {\n  SymbolHandle handle;\n  CHECK_EQ(MXSymbolGetInternals(GetHandle(), &handle), 0);\n  return Symbol(handle);\n}\ninline Symbol::Symbol(const std::string &operator_name, const std::string &name,\n               std::vector<const char *> input_keys,\n               std::vector<SymbolHandle> input_values,\n               std::vector<const char *> config_keys,\n               std::vector<const char *> config_values) {\n  SymbolHandle handle;\n  AtomicSymbolCreator creator = op_map()->GetSymbolCreator(operator_name);\n  MXSymbolCreateAtomicSymbol(creator, config_keys.size(), config_keys.data(),\n                             config_values.data(), &handle);\n  MXSymbolCompose(handle, operator_name.c_str(), input_keys.size(),\n                  input_keys.data(), input_values.data());\n  blob_ptr_ = std::make_shared<SymBlob>(handle);\n}\n\ninline Symbol Symbol::Copy() const {\n  SymbolHandle handle;\n  CHECK_EQ(MXSymbolCopy(GetHandle(), &handle), 0);\n  return Symbol(handle);\n}\n\ninline std::vector<std::string> Symbol::ListArguments() const {\n  std::vector<std::string> ret;\n  mx_uint size;\n  const char **sarr;\n  MXSymbolListArguments(GetHandle(), &size, &sarr);\n  for (mx_uint i = 0; i < size; ++i) {\n    ret.push_back(std::string(sarr[i]));\n  }\n  return ret;\n}\n\ninline std::vector<std::string> Symbol::ListInputs() const {\n  std::vector<std::string> ret;\n  mx_uint size;\n  const char **sarr;\n  NNSymbolListInputNames(GetHandle(), 0, &size, &sarr);\n  for (mx_uint i = 0; i < size; ++i) {\n    ret.push_back(std::string(sarr[i]));\n  }\n  return ret;\n}\n\ninline std::vector<std::string> Symbol::ListOutputs() const {\n  std::vector<std::string> ret;\n  mx_uint size;\n  const char **sarr;\n  MXSymbolListOutputs(GetHandle(), &size, &sarr);\n  for (mx_uint i = 0; i < size; ++i) {\n    ret.push_back(std::string(sarr[i]));\n  }\n  return ret;\n}\ninline std::vector<std::string> Symbol::ListAuxiliaryStates() const {\n  std::vector<std::string> ret;\n  mx_uint size;\n  const char **sarr;\n  MXSymbolListAuxiliaryStates(GetHandle(), &size, &sarr);\n  for (mx_uint i = 0; i < size; ++i) {\n    ret.push_back(std::string(sarr[i]));\n  }\n  return ret;\n}\n\ninline std::map<std::string, std::string> Symbol::ListAttributes() const {\n    mx_uint size;\n    const char** pairs;\n    CHECK_EQ(MXSymbolListAttrShallow(GetHandle(), &size, &pairs), 0);\n    std::map<std::string, std::string> attributes;\n    for (mx_uint i = 0; i < size; ++i) {\n      // pairs is 2 * size with key, value pairs according to\n      //   https://github.com/apache/mxnet/blob/master/include/mxnet/c_api.h#L1428\n      attributes[pairs[2 * i]] = pairs[2 * i + 1];\n    }\n    return attributes;\n}\n\ninline void Symbol::SetAttribute(const std::string &key, const std::string &value) {\n    CHECK_EQ(MXSymbolSetAttr(GetHandle(), key.c_str(), value.c_str()), 0);\n}\n\ninline void Symbol::SetAttributes(const std::map<std::string, std::string> &attrs) {\n    for (const auto& kv : attrs) {\n        SetAttribute(kv.first, kv.second);\n    }\n}\n\ninline mx_uint Symbol::GetNumOutputs() const {\n    mx_uint numOutputs;\n    CHECK_EQ(MXSymbolGetNumOutputs(GetHandle(), &numOutputs), 0);\n    return numOutputs;\n}\n\ninline mxnet::cpp::Symbol Symbol::GetBackendSymbol(const std::string &backendName) const {\n    SymbolHandle symbolHandle;\n    CHECK_EQ(MXGenBackendSubgraph(GetHandle(), backendName.c_str(), &symbolHandle), 0);\n    return mxnet::cpp::Symbol(symbolHandle);\n}\n\ninline std::string Symbol::GetName() const {\n  int success;\n  const char* out_name;\n  CHECK_EQ(MXSymbolGetName(GetHandle(), &out_name, &success), 0);\n  CHECK_EQ(success, 1);\n  return std::string(out_name);\n}\n\ninline void Symbol::InferShape(\n    const std::map<std::string, std::vector<mx_uint> > &arg_shapes,\n    std::vector<std::vector<mx_uint> > *in_shape,\n    std::vector<std::vector<mx_uint> > *aux_shape,\n    std::vector<std::vector<mx_uint> > *out_shape) const {\n\n  std::vector<const char *> keys;\n  std::vector<mx_uint> arg_ind_ptr;\n  std::vector<int> arg_shape_data;\n\n  for (const auto &arg : arg_shapes) {\n    keys.push_back(arg.first.c_str());\n    arg_ind_ptr.push_back(arg_shape_data.size());\n    for (auto i : arg.second) {\n      arg_shape_data.push_back(i);\n    }\n  }\n  arg_ind_ptr.push_back(arg_shape_data.size());\n\n  mx_uint in_shape_size;\n  const int *in_shape_ndim;\n  const int **in_shape_data;\n  mx_uint out_shape_size;\n  const int *out_shape_ndim;\n  const int **out_shape_data;\n  mx_uint aux_shape_size;\n  const int *aux_shape_ndim;\n  const int **aux_shape_data;\n  int complete;\n\n  CHECK_EQ(MXSymbolInferShape(GetHandle(), keys.size(), keys.data(),\n                              arg_ind_ptr.data(), arg_shape_data.data(),\n                              &in_shape_size, &in_shape_ndim, &in_shape_data,\n                              &out_shape_size, &out_shape_ndim, &out_shape_data,\n                              &aux_shape_size, &aux_shape_ndim, &aux_shape_data,\n                              &complete),\n           0);\n\n  if (complete) {\n    for (mx_uint i = 0; i < in_shape_size; ++i) {\n      in_shape->push_back(std::vector<mx_uint>());\n      for (int j = 0; j < in_shape_ndim[i]; ++j) {\n        (*in_shape)[i].push_back(in_shape_data[i][j]);\n      }\n    }\n    for (mx_uint i = 0; i < aux_shape_size; ++i) {\n      aux_shape->push_back(std::vector<mx_uint>());\n      for (int j = 0; j < aux_shape_ndim[i]; ++j) {\n        (*aux_shape)[i].push_back(aux_shape_data[i][j]);\n      }\n    }\n    for (mx_uint i = 0; i < out_shape_size; ++i) {\n      out_shape->push_back(std::vector<mx_uint>());\n      for (int j = 0; j < out_shape_ndim[i]; ++j) {\n        (*out_shape)[i].push_back(out_shape_data[i][j]);\n      }\n    }\n  }\n}\n\ninline void Symbol::InferExecutorArrays(\n    const Context &context, std::vector<NDArray> *arg_arrays,\n    std::vector<NDArray> *grad_arrays, std::vector<OpReqType> *grad_reqs,\n    std::vector<NDArray> *aux_arrays,\n    const std::map<std::string, NDArray> &args_map,\n    const std::map<std::string, NDArray> &arg_grad_store,\n    const std::map<std::string, OpReqType> &grad_req_type,\n    const std::map<std::string, NDArray> &aux_map) const {\n\n  const auto arg_name_list = ListArguments();\n  std::vector<std::vector<mx_uint> > in_shapes, aux_shapes, out_shapes;\n  std::map<std::string, std::vector<mx_uint> > arg_shapes;\n\n  for (const auto &arg_name : arg_name_list) {\n    auto iter = args_map.find(arg_name);\n    if (iter != args_map.end()) {\n      arg_shapes[arg_name] = iter->second.GetShape();\n    }\n  }\n\n  InferShape(arg_shapes, &in_shapes, &aux_shapes, &out_shapes);\n\n  for (size_t i = 0; i < in_shapes.size(); ++i) {\n    const auto &shape = in_shapes[i];\n    const auto &arg_name = arg_name_list[i];\n    auto iter_arg = args_map.find(arg_name);\n    if (iter_arg != args_map.end()) {\n      arg_arrays->push_back(iter_arg->second);\n    } else {\n      arg_arrays->push_back(NDArray(shape, context, false));\n      NDArray::SampleGaussian(0, 1, &arg_arrays->back());\n    }\n    auto iter_grad = arg_grad_store.find(arg_name);\n    if (iter_grad != arg_grad_store.end()) {\n      grad_arrays->push_back(iter_grad->second);\n    } else {\n      grad_arrays->push_back(NDArray(shape, context, false));\n    }\n    auto iter_req = grad_req_type.find(arg_name);\n    if (iter_req != grad_req_type.end()) {\n      grad_reqs->push_back(iter_req->second);\n    } else if (arg_name.rfind(\"data\") != std::string::npos\n            || arg_name.rfind(\"label\") != std::string::npos) {\n      grad_reqs->push_back(OpReqType::kNullOp);\n    } else {\n      grad_reqs->push_back(OpReqType::kWriteTo);\n    }\n  }\n\n  const auto aux_name_list = ListAuxiliaryStates();\n  for (size_t i = 0; i < aux_shapes.size(); ++i) {\n    const auto &shape = aux_shapes[i];\n    const auto &aux_name = aux_name_list[i];\n    auto iter_aux = aux_map.find(aux_name);\n    if (iter_aux != aux_map.end()) {\n      aux_arrays->push_back(iter_aux->second);\n    } else {\n      aux_arrays->push_back(NDArray(shape, context, false));\n      NDArray::SampleGaussian(0, 1, &aux_arrays->back());\n    }\n  }\n}\ninline void Symbol::InferArgsMap(\n    const Context &context, std::map<std::string, NDArray> *args_map,\n    const std::map<std::string, NDArray> &known_args) const {\n\n  const auto arg_name_list = ListArguments();\n  std::vector<std::vector<mx_uint> > in_shapes, aux_shapes, out_shapes;\n  std::map<std::string, std::vector<mx_uint> > arg_shapes;\n\n  for (const auto &arg_name : arg_name_list) {\n    auto iter = known_args.find(arg_name);\n    if (iter != known_args.end()) {\n      arg_shapes[arg_name] = iter->second.GetShape();\n    }\n  }\n\n  InferShape(arg_shapes, &in_shapes, &aux_shapes, &out_shapes);\n\n  for (size_t i = 0; i < in_shapes.size(); ++i) {\n    const auto &shape = in_shapes[i];\n    const auto &arg_name = arg_name_list[i];\n    auto iter_arg = known_args.find(arg_name);\n    if (iter_arg != known_args.end()) {\n      (*args_map)[arg_name] = iter_arg->second;\n    } else {\n      (*args_map)[arg_name] = NDArray(shape, context, false);\n      NDArray::SampleGaussian(0, 1, &(*args_map)[arg_name]);\n    }\n  }\n}\n\ninline Executor *Symbol::SimpleBind(\n    const Context &context, const std::map<std::string, NDArray> &args_map,\n    const std::map<std::string, NDArray> &arg_grad_store,\n    const std::map<std::string, OpReqType> &grad_req_type,\n    const std::map<std::string, NDArray> &aux_map) {\n  std::vector<NDArray> arg_arrays;\n  std::vector<NDArray> grad_arrays;\n  std::vector<OpReqType> grad_reqs;\n  std::vector<NDArray> aux_arrays;\n\n  InferExecutorArrays(context, &arg_arrays, &grad_arrays, &grad_reqs,\n                      &aux_arrays, args_map, arg_grad_store, grad_req_type,\n                      aux_map);\n\n  return new Executor(*this, context, arg_arrays, grad_arrays, grad_reqs,\n                      aux_arrays);\n}\n\ninline Executor *Symbol::Bind(const Context &context,\n                       const std::vector<NDArray> &arg_arrays,\n                       const std::vector<NDArray> &grad_arrays,\n                       const std::vector<OpReqType> &grad_reqs,\n                       const std::vector<NDArray> &aux_arrays,\n                       const std::map<std::string, Context> &group_to_ctx,\n                       Executor *shared_exec) {\n  return new Executor(*this, context, arg_arrays, grad_arrays, grad_reqs,\n                      aux_arrays, group_to_ctx, shared_exec);\n}\ninline Symbol operator+(mx_float lhs, const Symbol &rhs) { return rhs + lhs; }\ninline Symbol operator-(mx_float lhs, const Symbol &rhs) {\n  return mxnet::cpp::_RMinusScalar(lhs, rhs);\n}\ninline Symbol operator*(mx_float lhs, const Symbol &rhs) { return rhs * lhs; }\ninline Symbol operator/(mx_float lhs, const Symbol &rhs) {\n  return mxnet::cpp::_RDivScalar(lhs, rhs);\n}\ninline Symbol operator%(mx_float lhs, const Symbol &rhs) {\n  return mxnet::cpp::_RModScalar(lhs, rhs);\n}\n}  // namespace cpp\n}  // namespace mxnet\n\n#endif  // MXNET_CPP_SYMBOL_HPP_\n"
  },
  {
    "path": "cpp-package/scripts/OpWrapperGenerator.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# -*- coding: utf-8 -*-\n# This is a python script that generates operator wrappers such as FullyConnected,\n# based on current libmxnet.dll. This script is written so that we don't need to\n# write new operator wrappers when new ones are added to the library.\n\nfrom ctypes import *\nfrom ctypes.util import find_library\nimport os\nimport logging\nimport platform\nimport re\nimport sys\nimport tempfile\nimport filecmp\nimport shutil\nimport codecs\n\ndef gen_enum_value(value):\n    return 'k' + value[0].upper() + value[1:]\n\nclass EnumType:\n    name = ''\n    enumValues = []\n    def __init__(self, typeName = 'ElementWiseOpType', \\\n                 typeString = \"{'avg', 'max', 'sum'}\"):\n        self.name = typeName\n        if (typeString[0] == '{'):  # is a enum type\n            isEnum = True\n            # parse enum\n            self.enumValues = typeString[typeString.find('{') + 1:typeString.find('}')].split(',')\n            for i in range(0, len(self.enumValues)):\n                self.enumValues[i] = self.enumValues[i].strip().strip(\"'\")\n        else:\n            logging.warn(f\"trying to parse none-enum type as enum: {typeString}\")\n    def GetDefinitionString(self, indent = 0):\n        indentStr = ' ' * indent\n        ret = indentStr + 'enum class {} {{\\n'.format(self.name)\n        for i in range(0, len(self.enumValues)):\n            ret = ret + indentStr + f'  {gen_enum_value(self.enumValues[i])} = {i}'\n            if (i != len(self.enumValues) -1):\n                ret = ret + \",\"\n            ret = ret + \"\\n\"\n        ret = ret + \"};\\n\"\n        return ret\n    def GetDefaultValueString(self, value = ''):\n        return self.name + \"::\" + gen_enum_value(value)\n    def GetEnumStringArray(self, indent = 0):\n        indentStr = ' ' * indent\n        ret = indentStr + 'static const char *{}Values[] = {{\\n'.format(self.name)\n        for i in range(0, len(self.enumValues)):\n            ret = ret + indentStr + f'  \"{self.enumValues[i]}\"'\n            if (i != len(self.enumValues) -1):\n                ret = ret + \",\"\n            ret = ret + \"\\n\"\n        ret = ret + indentStr + \"};\\n\"\n        return ret\n    def GetConvertEnumVariableToString(self, variable=''):\n        return f\"{self.name}Values[int({variable})]\"\n\n\nclass Arg:\n    typeDict = {'boolean':'bool',\\\n        'boolean or None':'dmlc::optional<bool>',\\\n        'Shape(tuple)':'Shape',\\\n        'Symbol':'Symbol',\\\n        'NDArray':'Symbol',\\\n        'NDArray-or-Symbol':'Symbol',\\\n        'Symbol[]':'const std::vector<Symbol>&',\\\n        'Symbol or Symbol[]':'const std::vector<Symbol>&',\\\n        'NDArray[]':'const std::vector<Symbol>&',\\\n        'caffe-layer-parameter':'::caffe::LayerParameter',\\\n        'NDArray-or-Symbol[]':'const std::vector<Symbol>&',\\\n        'float':'mx_float',\\\n        'real_t':'mx_float',\\\n        'int':'int',\\\n        'int (non-negative)': 'uint32_t',\\\n        'long (non-negative)': 'uint64_t',\\\n        'int or None':'dmlc::optional<int>',\\\n        'float or None':'dmlc::optional<float>',\\\n        'long':'int64_t',\\\n        'double':'double',\\\n        'double or None':'dmlc::optional<double>',\\\n        'Shape or None':'dmlc::optional<Shape>',\\\n        'string':'const std::string&',\\\n        'tuple of <float>':'nnvm::Tuple<mx_float>',\\\n        'tuple of <>':'mxnet::cpp::Shape',\\\n        '':'index_t'}\n    name = ''\n    type = ''\n    description = ''\n    isEnum = False\n    enum = None\n    hasDefault = False\n    defaultString = ''\n    def __init__(self, opName = '', argName = '', typeString = '', descString = ''):\n        self.name = argName\n        self.description = descString\n        if (typeString[0] == '{'):  # is enum type\n            self.isEnum = True\n            self.enum = EnumType(self.ConstructEnumTypeName(opName, argName), typeString)\n            self.type = self.enum.name\n        else:\n            try:\n                self.type = self.typeDict[typeString.split(',')[0]]\n            except:\n                print(f'argument \"{argName}\" of operator \"{opName}\" has unknown type \"{typeString}\"')\n                pass\n        if typeString.find('default=') != -1:\n            self.hasDefault = True\n            self.defaultString = typeString.split('default=')[1].strip().strip(\"'\")\n            if typeString.startswith('string'):\n                self.defaultString = self.MakeCString(self.defaultString)\n            elif self.isEnum:\n                self.defaultString = self.enum.GetDefaultValueString(self.defaultString)\n            elif self.defaultString == 'None':\n                self.defaultString = self.type + '()'\n            elif self.type == \"bool\":\n                if self.defaultString == \"1\" or self.defaultString == \"True\":\n                    self.defaultString = \"true\"\n                else:\n                    self.defaultString = \"false\"\n            elif self.defaultString[0] == '(':\n                self.defaultString = 'Shape' + self.defaultString\n            elif self.defaultString[0] == '[':\n                self.defaultString = 'Shape(' + self.defaultString[1:-1] + \")\"\n            elif self.type == 'dmlc::optional<int>':\n                self.defaultString = self.type + '(' + self.defaultString + ')'\n            elif self.type == 'dmlc::optional<bool>':\n                self.defaultString = self.type + '(' + self.defaultString + ')'\n            elif typeString.startswith('caffe-layer-parameter'):\n                self.defaultString = 'textToCaffeLayerParameter(' + self.MakeCString(self.defaultString) + ')'\n                hasCaffe = True\n\n    def MakeCString(self, str):\n        str = str.replace('\\n', \"\\\\n\")\n        str = str.replace('\\t', \"\\\\t\")\n        return '\\\"' + str + '\\\"'\n\n    def ConstructEnumTypeName(self, opName = '', argName = ''):\n        a = opName[0].upper()\n        # format ArgName so instead of act_type it returns ActType\n        argNameWords = argName.split('_')\n        argName = ''\n        for an in argNameWords:\n            argName = argName + an[0].upper() + an[1:]\n        typeName = a + opName[1:] + argName\n        return typeName\n\nclass Op:\n    name = ''\n    description = ''\n    args = []\n\n    def __init__(self, name = '', description = '', args = []):\n        self.name = name\n        self.description = description\n        # add a 'name' argument\n        nameArg = Arg(self.name, \\\n                      'symbol_name', \\\n                      'string', \\\n                      'name of the resulting symbol')\n        args.insert(0, nameArg)\n        # reorder arguments, put those with default value to the end\n        orderedArgs = []\n        for arg in args:\n            if not arg.hasDefault:\n                orderedArgs.append(arg)\n        for arg in args:\n            if arg.hasDefault:\n                orderedArgs.append(arg)\n        self.args = orderedArgs\n\n    def WrapDescription(self, desc = ''):\n        ret = []\n        sentences = desc.split('.')\n        lines = desc.split('\\n')\n        for line in lines:\n          line = line.strip()\n          if len(line) <= 80:\n            ret.append(line.strip())\n          else:\n            while len(line) > 80:\n              pos = line.rfind(' ', 0, 80)+1\n              if pos <= 0:\n                pos = line.find(' ')\n              if pos < 0:\n                pos = len(line)\n              ret.append(line[:pos].strip())\n              line = line[pos:]\n        return ret\n\n    def GenDescription(self, desc = '', \\\n                        firstLineHead = ' * \\\\brief ', \\\n                        otherLineHead = ' *        '):\n        ret = ''\n        descs = self.WrapDescription(desc)\n        ret = ret + firstLineHead\n        if len(descs) == 0:\n          return ret.rstrip()\n        ret = (ret + descs[0]).rstrip() + '\\n'\n        for i in range(1, len(descs)):\n            ret = ret + (otherLineHead + descs[i]).rstrip() + '\\n'\n        return ret\n\n    def GetOpDefinitionString(self, use_name, indent=0):\n        ret = ''\n        indentStr = ' ' * indent\n        # define enums if any\n        for arg in self.args:\n            if arg.isEnum and use_name:\n                # comments\n                ret = ret + self.GenDescription(arg.description, \\\n                                        '/*! \\\\brief ', \\\n                                        ' *        ')\n                ret = ret + \" */\\n\"\n                # definition\n                ret = ret + arg.enum.GetDefinitionString(indent) + '\\n'\n        # create function comments\n        ret = ret + self.GenDescription(self.description, \\\n                                        '/*!\\n * \\\\brief ', \\\n                                        ' *        ')\n        for arg in self.args:\n            if arg.name != 'symbol_name' or use_name:\n                ret = ret + self.GenDescription(arg.name + ' ' + arg.description, \\\n                                        ' * \\\\param ', \\\n                                        ' *        ')\n        ret = ret + \" * \\\\return new symbol\\n\"\n        ret = ret + \" */\\n\"\n        # create function header\n        declFirstLine = indentStr + f'inline Symbol {self.name}('\n        ret = ret + declFirstLine\n        argIndentStr = ' ' * len(declFirstLine)\n        arg_start = 0 if use_name else 1\n        if len(self.args) > arg_start:\n            ret = ret + self.GetArgString(self.args[arg_start])\n        for i in range(arg_start+1, len(self.args)):\n            ret = ret + ',\\n'\n            ret = ret + argIndentStr + self.GetArgString(self.args[i])\n        ret = ret + ') {\\n'\n        # create function body\n        # if there is enum, generate static enum<->string mapping\n        for arg in self.args:\n            if arg.isEnum:\n                ret = ret + arg.enum.GetEnumStringArray(indent + 2)\n        # now generate code\n        ret = ret + indentStr + f'  return Operator(\\\"{self.name}\\\")\\n'\n        for arg in self.args:   # set params\n            if arg.type == 'Symbol' or \\\n                arg.type == 'const std::string&' or \\\n                arg.type == 'const std::vector<Symbol>&':\n                continue\n            v = arg.name\n            if arg.isEnum:\n                v = arg.enum.GetConvertEnumVariableToString(v)\n            ret = ret + indentStr + ' ' * 11 + \\\n                f'.SetParam(\\\"{arg.name}\\\", {v})\\n'\n        #ret = ret[:-1]  # get rid of the last \\n\n        symbols = ''\n        inputAlreadySet = False\n        for arg in self.args:   # set inputs\n            if arg.type != 'Symbol':\n                continue\n            inputAlreadySet = True\n            #if symbols != '':\n            #    symbols = symbols + ', '\n            #symbols = symbols + arg.name\n            ret = ret + indentStr + ' ' * 11 + \\\n                f'.SetInput(\\\"{arg.name}\\\", {arg.name})\\n'\n        for arg in self.args:   # set input arrays vector<Symbol>\n            if arg.type != 'const std::vector<Symbol>&':\n                continue\n            if (inputAlreadySet):\n                logging.error(f\"op {self.name} has both Symbol[] and Symbol inputs!\")\n            inputAlreadySet = True\n            symbols = arg.name\n            ret = ret + f'({symbols})\\n'\n        ret = ret + indentStr + ' ' * 11\n        if use_name:\n            ret = ret + '.CreateSymbol(symbol_name);\\n'\n        else:\n            ret = ret + '.CreateSymbol();\\n'\n        ret = ret + indentStr + '}\\n'\n        return ret\n\n    def GetArgString(self, arg):\n        ret = f'{arg.type} {arg.name}'\n        if arg.hasDefault:\n            ret = ret + ' = ' + arg.defaultString\n        return ret\n\n\ndef ParseAllOps():\n    \"\"\"\n    MXNET_DLL int MXSymbolListAtomicSymbolCreators(mx_uint *out_size,\n                                                   AtomicSymbolCreator **out_array);\n\n    MXNET_DLL int MXSymbolGetAtomicSymbolInfo(AtomicSymbolCreator creator,\n                                              const char **name,\n                                              const char **description,\n                                              mx_uint *num_args,\n                                              const char ***arg_names,\n                                              const char ***arg_type_infos,\n                                              const char ***arg_descriptions,\n                                              const char **key_var_num_args);\n    \"\"\"\n    cdll.libmxnet = cdll.LoadLibrary(sys.argv[1])\n    ListOP = cdll.libmxnet.MXSymbolListAtomicSymbolCreators\n    GetOpInfo = cdll.libmxnet.MXSymbolGetAtomicSymbolInfo\n    ListOP.argtypes=[POINTER(c_int), POINTER(POINTER(c_void_p))]\n    GetOpInfo.argtypes=[c_void_p, \\\n        POINTER(c_char_p), \\\n        POINTER(c_char_p), \\\n        POINTER(c_int), \\\n        POINTER(POINTER(c_char_p)), \\\n        POINTER(POINTER(c_char_p)), \\\n        POINTER(POINTER(c_char_p)), \\\n        POINTER(c_char_p), \\\n        POINTER(c_char_p)\n        ]\n\n    nOps = c_int()\n    opHandlers = POINTER(c_void_p)()\n    r = ListOP(byref(nOps), byref(opHandlers))\n    ret = ''\n    ret2 = ''\n    for i in range(0, nOps.value):\n        handler = opHandlers[i]\n        name = c_char_p()\n        description = c_char_p()\n        nArgs = c_int()\n        argNames = POINTER(c_char_p)()\n        argTypes = POINTER(c_char_p)()\n        argDescs = POINTER(c_char_p)()\n        varArgName = c_char_p()\n        return_type = c_char_p()\n\n        GetOpInfo(handler, byref(name), byref(description), \\\n            byref(nArgs), byref(argNames), byref(argTypes), \\\n            byref(argDescs), byref(varArgName), byref(return_type))\n\n        if name.value.decode('utf-8').startswith('_'):     # get rid of functions like __init__\n            continue\n\n        args = []\n\n        for i in range(0, nArgs.value):\n            arg = Arg(name.value.decode('utf-8'),\n                      argNames[i].decode('utf-8'),\n                      argTypes[i].decode('utf-8'),\n                      argDescs[i].decode('utf-8'))\n            args.append(arg)\n\n        op = Op(name.value.decode('utf-8'), description.value.decode('utf-8'), args)\n\n        ret = ret + op.GetOpDefinitionString(True) + \"\\n\"\n        ret2 = ret2 + op.GetOpDefinitionString(False) + \"\\n\"\n    return ret + ret2\n\nif __name__ == \"__main__\":\n    #et = EnumType(typeName = 'MyET')\n    #print(et.GetDefinitionString())\n    #print(et.GetEnumStringArray())\n    #arg = Arg()\n    #print(arg.ConstructEnumTypeName('SoftmaxActivation', 'act_type'))\n    #arg = Arg(opName = 'FullConnected', argName='act_type', \\\n    #    typeString=\"{'elu', 'leaky', 'prelu', 'rrelu'},optional, default='leaky'\", \\\n    #    descString='Activation function to be applied.')\n    #print(arg.isEnum)\n    #print(arg.defaultString)\n    #arg = Arg(\"fc\", \"alpha\", \"float, optional, default=0.0001\", \"alpha\")\n    #decl = \"%s %s\" % (arg.type, arg.name)\n    #if arg.hasDefault:\n    #    decl = decl + \"=\" + arg.defaultString\n    #print(decl)\n\n    temp_file_name = \"\"\n    output_file = '../include/mxnet-cpp/op.h'\n    try:\n        # generate file header\n        patternStr = (\"/*!\\n\"\n                      \"* \\\\file op.h\\n\"\n                      \"* \\\\brief definition of all the operators\\n\"\n                      \"* \\\\author Chuntao Hong, Xin Li\\n\"\n                      \"*/\\n\"\n                      \"\\n\"\n                      \"#ifndef MXNET_CPP_OP_H_\\n\"\n                      \"#define MXNET_CPP_OP_H_\\n\"\n                      \"\\n\"\n                      \"#include <string>\\n\"\n                      \"#include <vector>\\n\"\n                      \"#include \\\"mxnet-cpp/base.h\\\"\\n\"\n                      \"#include \\\"mxnet-cpp/shape.h\\\"\\n\"\n                      \"#include \\\"mxnet-cpp/op_util.h\\\"\\n\"\n                      \"#include \\\"mxnet-cpp/operator.h\\\"\\n\"\n                      \"#include \\\"dmlc/optional.h\\\"\\n\"\n                      \"#include \\\"nnvm/tuple.h\\\"\\n\"\n                      \"\\n\"\n                      \"namespace mxnet {{\\n\"\n                      \"namespace cpp {{\\n\"\n                      \"\\n\"\n                      \"{}\"\n                      \"}} //namespace cpp\\n\"\n                      \"}} //namespace mxnet\\n\"\n                      \"#endif  // MXNET_CPP_OP_H_\\n\")\n\n        # Generate a temporary file name\n        tf = tempfile.NamedTemporaryFile()\n        temp_file_name = tf.name\n        tf.close()\n        with codecs.open(temp_file_name, 'w', 'utf-8') as f:\n            f.write(patternStr.format(ParseAllOps()))\n    except Exception as e:\n      if (os.path.exists(output_file)):\n        os.remove(output_file)\n      if len(temp_file_name) > 0:\n        os.remove(temp_file_name)\n      raise(e)\n    if os.path.exists(output_file):\n      if not filecmp.cmp(temp_file_name, output_file):\n          os.remove(output_file)\n    if not os.path.exists(output_file):\n      shutil.move(temp_file_name, output_file)\n"
  },
  {
    "path": "cpp-package/scripts/lint.py",
    "content": "#!/usr/bin/env python\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=protected-access, unused-variable, locally-disabled, redefined-variable-type\n\"\"\"Lint helper to generate lint summary of source.\nCopyright by Contributors\n\"\"\"\nfrom __future__ import print_function\nimport codecs\nimport sys\nimport re\nimport os\nimport cpplint\nfrom cpplint import _cpplint_state\nfrom pylint import epylint\n\nCXX_SUFFIX = set(['cc', 'c', 'cpp', 'h', 'cu', 'hpp'])\nPYTHON_SUFFIX = set(['py'])\n\nclass LintHelper(object):\n    \"\"\"Class to help runing the lint and records summary\"\"\"\n\n    @staticmethod\n    def _print_summary_map(strm, result_map, ftype):\n        \"\"\"Print summary of certain result map.\"\"\"\n        if len(result_map) == 0:\n            return 0\n        npass = len([x for k, x in result_map.iteritems() if len(x) == 0])\n        strm.write(f'====={npass}/{len(result_map)} {ftype} files passed check=====\\n')\n        for fname, emap in result_map.iteritems():\n            if len(emap) == 0:\n                continue\n            strm.write(f'{fname}: {sum(emap.values())} Errors of {len(emap)} Categories map={str(emap)}\\n')\n        return len(result_map) - npass\n\n    def __init__(self):\n        self.project_name = None\n        self.cpp_header_map = {}\n        self.cpp_src_map = {}\n        self.python_map = {}\n        pylint_disable = ['superfluous-parens',\n                          'too-many-instance-attributes',\n                          'too-few-public-methods']\n        # setup pylint\n        self.pylint_opts = ['--extension-pkg-whitelist=numpy',\n                            '--disable=' + ','.join(pylint_disable)]\n\n        self.pylint_cats = set(['error', 'warning', 'convention', 'refactor'])\n        # setup cpp lint\n        cpplint_args = ['.', '--extensions=' + (','.join(CXX_SUFFIX))]\n        _ = cpplint.ParseArguments(cpplint_args)\n        cpplint._SetFilters(','.join(['-build/c++11',\n                                      '-build/namespaces',\n                                      '-build/include',\n                                      '-build/header_guard',\n                                      '+build/include_what_you_use',\n                                      '+build/include_order']))\n        cpplint._SetCountingStyle('toplevel')\n        cpplint._line_length = 100\n\n    def process_cpp(self, path, suffix):\n        \"\"\"Process a cpp file.\"\"\"\n        _cpplint_state.ResetErrorCounts()\n        cpplint.ProcessFile(str(path), _cpplint_state.verbose_level)\n        _cpplint_state.PrintErrorCounts()\n        errors = _cpplint_state.errors_by_category.copy()\n\n        if suffix == 'h':\n            self.cpp_header_map[str(path)] = errors\n        else:\n            self.cpp_src_map[str(path)] = errors\n\n    def process_python(self, path):\n        \"\"\"Process a python file.\"\"\"\n        (pylint_stdout, pylint_stderr) = epylint.py_run(\n            ' '.join([str(path)] + self.pylint_opts), return_std=True)\n        emap = {}\n        print(pylint_stderr.read())\n        for line in pylint_stdout:\n            sys.stderr.write(line)\n            key = line.split(':')[-1].split('(')[0].strip()\n            if key not in self.pylint_cats:\n                continue\n            if key not in emap:\n                emap[key] = 1\n            else:\n                emap[key] += 1\n        sys.stderr.write('\\n')\n        self.python_map[str(path)] = emap\n\n    def print_summary(self, strm):\n        \"\"\"Print summary of lint.\"\"\"\n        nerr = 0\n        nerr += LintHelper._print_summary_map(strm, self.cpp_header_map, 'cpp-header')\n        nerr += LintHelper._print_summary_map(strm, self.cpp_src_map, 'cpp-soruce')\n        nerr += LintHelper._print_summary_map(strm, self.python_map, 'python')\n        if nerr == 0:\n            strm.write('All passed!\\n')\n        else:\n            strm.write(f'{nerr} files failed lint\\n')\n        return nerr\n\n# singleton helper for lint check\n_HELPER = LintHelper()\n\ndef get_header_guard_dmlc(filename):\n    \"\"\"Get Header Guard Convention for DMLC Projects.\n    For headers in include, directly use the path\n    For headers in src, use project name plus path\n    Examples: with project-name = dmlc\n        include/dmlc/timer.h -> DMLC_TIMTER_H_\n        src/io/libsvm_parser.h -> DMLC_IO_LIBSVM_PARSER_H_\n    \"\"\"\n    fileinfo = cpplint.FileInfo(filename)\n    file_path_from_root = fileinfo.RepositoryName()\n    inc_list = ['include', 'api', 'wrapper']\n\n    if file_path_from_root.find('src/') != -1 and _HELPER.project_name is not None:\n        idx = file_path_from_root.find('src/')\n        file_path_from_root = _HELPER.project_name +  file_path_from_root[idx + 3:]\n    else:\n        for spath in inc_list:\n            prefix = spath + os.sep\n            if file_path_from_root.startswith(prefix):\n                file_path_from_root = re.sub('^' + prefix, '', file_path_from_root)\n                break\n    return re.sub(r'[-./\\s]', '_', file_path_from_root).upper() + '_'\n\ncpplint.GetHeaderGuardCPPVariable = get_header_guard_dmlc\n\ndef process(fname, allow_type):\n    \"\"\"Process a file.\"\"\"\n    fname = str(fname)\n    # HACK: ignore op.h which is automatically generated\n    if fname.endswith('op.h'):\n      return\n    arr = fname.rsplit('.', 1)\n    if fname.find('#') != -1 or arr[-1] not in allow_type:\n        return\n    if arr[-1] in CXX_SUFFIX:\n        _HELPER.process_cpp(fname, arr[-1])\n    if arr[-1] in PYTHON_SUFFIX:\n        _HELPER.process_python(fname)\n\ndef main():\n    \"\"\"Main entry function.\"\"\"\n    if len(sys.argv) < 3:\n        print('Usage: <project-name> <filetype> <list-of-path to traverse>')\n        print('\\tfiletype can be python/cpp/all')\n        exit(-1)\n    _HELPER.project_name = sys.argv[1]\n    file_type = sys.argv[2]\n    allow_type = []\n    if file_type == 'python' or file_type == 'all':\n        allow_type += [x for x in PYTHON_SUFFIX]\n    if file_type == 'cpp' or file_type == 'all':\n        allow_type += [x for x in CXX_SUFFIX]\n    allow_type = set(allow_type)\n    if os.name != 'nt':\n        sys.stderr = codecs.StreamReaderWriter(sys.stderr,\n                                               codecs.getreader('utf8'),\n                                               codecs.getwriter('utf8'),\n                                               'replace')\n    for path in sys.argv[3:]:\n        if os.path.isfile(path):\n            process(path, allow_type)\n        else:\n            for root, dirs, files in os.walk(path):\n                for name in files:\n                    process(os.path.join(root, name), allow_type)\n\n    nerr = _HELPER.print_summary(sys.stderr)\n    sys.exit(nerr > 0)\n\nif __name__ == '__main__':\n    main()\n"
  },
  {
    "path": "cpp-package/tests/ci_test.sh",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nset -e # exit on the first error\ncd $(dirname $(readlink -f $0))/../example\necho $PWD\nexport LD_LIBRARY_PATH=$(readlink -f /work/build):$LD_LIBRARY_PATH\necho $LD_LIBRARY_PATH\nls -l /work/build/\n\n./get_data.sh\n\ncp /work/build/cpp-package/example/lenet .\n./lenet 1\n\ncp /work/build/cpp-package/example/alexnet .\n./alexnet 1\n\ncp /work/build/cpp-package/example/lenet_with_mxdataiter .\n./lenet_with_mxdataiter 1\n\ncp /work/build/cpp-package/example/resnet .\n./resnet 1\n\ncp /work/build/cpp-package/example/inception_bn .\n./inception_bn 1\n\ncp /work/build/cpp-package/example/mlp .\n./mlp 150\n\ncp /work/build/cpp-package/example/mlp_cpu .\n./mlp_cpu\n\ncp /work/build/cpp-package/example/mlp_gpu .\n./mlp_gpu\n\ncp /work/build/cpp-package/example/test_optimizer .\n./test_optimizer\n\ncp /work/build/cpp-package/example/test_kvstore .\n./test_kvstore\n\ncp /work/build/cpp-package/example/test_score .\n./test_score 0.93\n\ncp /work/build/cpp-package/example/test_ndarray_copy .\n./test_ndarray_copy\n\n# skippping temporarily, tracked by https://github.com/apache/mxnet/issues/20011\ncp /work/build/cpp-package/example/test_regress_label .\n./test_regress_label\n\nsh unittests/unit_test_mlp_csv.sh\n\ncd inference\n\ncp /work/build/cpp-package/example/sentiment_analysis_rnn .\n./unit_test_sentiment_analysis_rnn.sh\n\ncd multi_threaded_inference\n\ncp ../../../../build/cpp-package/example/multi_threaded_inference .\n./unit_test_multi_threaded_inference.sh\n\ncd ../..\n"
  },
  {
    "path": "doap.rdf",
    "content": "<?xml version=\"1.0\"?>\n<!---\n    Licensed to the Apache Software Foundation (ASF) under one or more\n    contributor license agreements.  See the NOTICE file distributed with\n    this work for additional information regarding copyright ownership.\n    The ASF licenses this file to You under the Apache License, Version 2.0\n    (the \"License\"); you may not use this file except in compliance with\n    the License.  You may obtain a copy of the License at\n\n         https://www.apache.org/licenses/LICENSE-2.0\n\n    Unless required by applicable law or agreed to in writing, software\n    distributed under the License is distributed on an \"AS IS\" BASIS,\n    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n    See the License for the specific language governing permissions and\n    limitations under the License.\n-->\n<?xml-stylesheet type=\"text/xsl\"?>\n<rdf:RDF xml:lang=\"en\"\n         xmlns=\"http://usefulinc.com/ns/doap#\"\n         xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\"\n         xmlns:asfext=\"http://projects.apache.org/ns/asfext#\"\n         xmlns:foaf=\"http://xmlns.com/foaf/0.1/\">\n  <Project rdf:about=\"https://mxnet.apache.org\">\n    <created>2020-09-15</created>\n    <license rdf:resource=\"https://spdx.org/licenses/Apache-2.0\" />\n    <name>Apache MXNet</name>\n    <homepage rdf:resource=\"https://mxnet.apache.org\" />\n    <asfext:pmc rdf:resource=\"https://incubator.apache.org\" />\n    <shortdesc>Apache MXNet is a deep learning framework designed for both efficiency and flexibility.</shortdesc>\n    <description>Apache MXNet is a deep learning framework designed for both efficiency and flexibility. It's lightweight, Portable, Flexible Distributed/Mobile Deep Learning with dynamic, mutation-aware data-flow dependency scheduler; for Python, R, Julia, Scala, Go, Javascript and more</description>\n    <bug-database rdf:resource=\"https://github.com/apache/mxnet/labels/Bug\" />\n    <mailing-list rdf:resource=\"https://lists.apache.org/list.html?dev@mxnet.apache.org\" />\n    <download-page rdf:resource=\"https://mxnet.apache.org/get_started/download\" />\n    <programming-language>C++</programming-language>\n    <category rdf:resource=\"https://projects.apache.org/category/big-data\" />\n    <release>\n      <Version>\n        <name>Apache MXNet (Incubating) 1.7.0</name>\n        <created>2020-09-13</created>\n        <revision>1.7.0</revision>\n      </Version>\n    </release>\n    <repository>\n      <GitRepository>\n        <location rdf:resource=\"https://github.com/apache/mxnet\"/>\n        <browse rdf:resource=\"https://github.com/apache/mxnet\"/>\n      </GitRepository>\n    </repository>\n    <maintainer>\n      <foaf:Person>\n        <foaf:name>ApacheMXNet Contributors</foaf:name>\n          <foaf:mbox rdf:resource=\"mailto:dev@mxnet.apache.org\"/>\n      </foaf:Person>\n    </maintainer>\n  </Project>\n</rdf:RDF>\n\n"
  },
  {
    "path": "docker/.gitignore",
    "content": "Dockerfile.*\n!Dockerfile.in.*\n"
  },
  {
    "path": "docker/Dockerfiles/Dockerfile.in.julia",
    "content": "# -*- mode: dockerfile -*-\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n# part of the dockerfile to install the julia binding\n\nCOPY install/julia.sh install/\nRUN install/julia.sh\nENV MXNET_HOME /mxnet\nRUN julia -e 'Pkg.add(\"MXNet\")'\n"
  },
  {
    "path": "docker/Dockerfiles/Dockerfile.in.lib.cpu",
    "content": "# -*- mode: dockerfile -*-\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n# dockerfile to build libmxnet.so on CPU\nFROM ubuntu:14.04\n\nCOPY install/cpp.sh install/\nRUN install/cpp.sh\n\nRUN git clone --recursive https://github.com/apache/mxnet && cd mxnet && \\\n    make -j$(nproc) && \\\n    rm -r build\n"
  },
  {
    "path": "docker/Dockerfiles/Dockerfile.in.lib.gpu",
    "content": "# -*- mode: dockerfile -*-\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n# dockerfile to build libmxnet.so on GPU\nFROM nvidia/cuda:8.0-cudnn5-devel\n\nCOPY install/cpp.sh install/\nRUN install/cpp.sh\n\nENV BUILD_OPTS \"USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1\"\nRUN git clone --recursive https://github.com/apache/mxnet && cd mxnet && \\\n    make -j$(nproc) $BUILD_OPTS\n"
  },
  {
    "path": "docker/Dockerfiles/Dockerfile.in.perl",
    "content": "# -*- mode: dockerfile -*-\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n# part of the dockerfile to install the perl binding\n\nCOPY install/perl.sh install/\nRUN install/perl.sh && \\\n    cd /mxnet/perl-package/AI-MXNetCAPI/ && perl Makefile.PL && make install && \\\n    cd /mxnet/perl-package/AI-NNVMCAPI/ && perl Makefile.PL && make install && \\\n    cd /mxnet/perl-package/AI-MXNet/ && perl Makefile.PL && make install\n"
  },
  {
    "path": "docker/Dockerfiles/Dockerfile.in.python",
    "content": "# -*- mode: dockerfile -*-\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n# part of the dockerfile to install the python binding\n\nCOPY install/python.sh install/\nRUN install/python.sh\nENV PYTHONPATH=/mxnet/python\n"
  },
  {
    "path": "docker/Dockerfiles/Dockerfile.in.r-lang",
    "content": "# -*- mode: dockerfile -*-\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n# part of the dockerfile to install the r binding\n\nCOPY install/r.sh install/\nADD https://raw.githubusercontent.com/dmlc/mxnet/master/R-package/DESCRIPTION  install/\nRUN install/r.sh\nRUN cd mxnet && make rpkg && R CMD INSTALL mxnet_current_r.tar.gz\n"
  },
  {
    "path": "docker/Dockerfiles/Dockerfile.in.scala",
    "content": "# -*- mode: dockerfile -*-\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n# part of the dockerfile to install the scala binding\n\nCOPY install/scala.sh install/\nRUN install/scala.sh\n\nRUN cd mxnet/scala-package && mvn package\n"
  },
  {
    "path": "docker/README.md",
    "content": "<!---\n  Licensed to the Apache Software Foundation (ASF) under one\n  or more contributor license agreements.  See the NOTICE file\n  distributed with this work for additional information\n  regarding copyright ownership.  The ASF licenses this file\n  to you under the Apache License, Version 2.0 (the\n  \"License\"); you may not use this file except in compliance\n  with the License.  You may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing,\n  software distributed under the License is distributed on an\n  \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  KIND, either express or implied.  See the License for the\n  specific language governing permissions and limitations\n  under the License.\n-->\n\n# Docker images for MXNET\n\n## How to use\n\nFirst make sure [docker](https://docs.docker.com/engine/installation/) is\ninstalled. The docker plugin\n[nvidia-docker](https://github.com/NVIDIA/nvidia-docker) is required to run on\nNvidia GPUs.\n\nPre-built docker containers are available at https://hub.docker.com/r/mxnet/\n\nFor example, the following command launches a container with the Python package\ninstalled. It will pull the docker images from docker hub if it does not exist\nlocally.\n\n```bash\ndocker run -ti --rm mxnet/python\n```\n\nThen you can run MXNet in python, e.g.:\n\n```bash\n# python -c 'import mxnet as mx; a = mx.nd.ones((2,3)); print((a*2).asnumpy())'\n[[ 2.  2.  2.]\n [ 2.  2.  2.]]\n```\n\nIf the host machine has at least one GPU installed and `nvidia-docker` is installed, namely\n`nvidia-docker run --rm nvidia/cuda nvidia-smi` runs successfully, then you can\nrun a container with GPU supports\n\n```bash\nnvidia-docker run -ti --rm mxnet/python:gpu\n```\n\nNow you can run the above example in `GPU 0`:\n\n```bash\n# python -c 'import mxnet as mx; a = mx.nd.ones((2,3), mx.gpu(0)); print((a*2).asnumpy())'\n[[ 2.  2.  2.]\n [ 2.  2.  2.]]\n```\n\n## Hosted containers\n\nAll images are based on Ubuntu 14.04. The `gpu` tag is built with CUDA 8.0 and\ncuDNN 5.\n\n### Python\n\nHosted at https://hub.docker.com/r/mxnet/python/\n\nPython versions: 2.7.12 and 3.5.2.\n\nAvailable tags:\n\n- mxnet/python\n- mxnet/python:gpu\n\n### R\n\nHosted at https://hub.docker.com/r/mxnet/r-lang/\n\nR version: 3.3.3\n\nAvailable tags:\n\n- mxnet/r-lang\n- mxnet/r-lang:gpu\n\n\n### Julia\n\nHosted at https://hub.docker.com/r/mxnet/julia/\n\nJulia version: 0.5.1\n\nAvailable tags:\n\n- mxnet/julia\n- mxnet/julia:gpu\n\n#### Scala\n\nHosted at https://hub.docker.com/r/mxnet/scala/\n\nScala version: 2.11.8\n\nAvailable tags:\n\n- mxnet/scala\n\n### Perl\n\nHosted at https://hub.docker.com/r/mxnet/perl/\n\nPerl version: 5.18.2\n\nAvailable tags:\n\n- mxnet/perl\n- mxnet/perl:gpu\n\n\n## How to build\n\nThe following command build the default Python package\n\n```bash\n./tool.sh build python cpu\n```\n\nRun `./tool.sh` for more details. Use\n\n\nTips: The following commands stop all docker containers and delete all docker images.\n\n```bash\ndocker stop $(docker ps -a -q)\ndocker rm $(docker ps -a -q)\n```\n\n```bash\ndocker rmi $(docker images -a -q)\n```\n"
  },
  {
    "path": "docker/docker-python/README.md",
    "content": "<!---\n  Licensed to the Apache Software Foundation (ASF) under one\n  or more contributor license agreements.  See the NOTICE file\n  distributed with this work for additional information\n  regarding copyright ownership.  The ASF licenses this file\n  to you under the Apache License, Version 2.0 (the\n  \"License\"); you may not use this file except in compliance\n  with the License.  You may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing,\n  software distributed under the License is distributed on an\n  \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  KIND, either express or implied.  See the License for the\n  specific language governing permissions and limitations\n  under the License.\n-->\n\n# Release Python Docker Images for MXNet\n\nThe `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release.  \nIt uses the appropriate pip binaries to build different docker images. Both python2 (default) and python3 images are available as -\n* {version}_cpu\n* {version}_cpu_mkl\n* {version}_gpu_cu92\n* {version}_gpu_cu92_mkl\n* {version}_cpu_py3\n* {version}_cpu_mkl_py3\n* {version}_gpu_cu92_py3\n* {version}_gpu_cu92_mkl_py3\n\nAnd the following tags will be available without the version string in the image name (for Benchmarking and other use cases):\n* latest (same as {version}_cpu)\n* gpu (same as {version}_gpu_cu90)\n* latest_cpu_mkl_py2 (same as {version}_cpu_mkl)\n* latest_cpu_mkl_py3 (same as {version}_cpu_mkl_py3)\n* latest_gpu_mkl_py2 (same as {version}_gpu_cu90_mkl)\n* latest_gpu_mkl_py3 (same as {version}_gpu_cu90_mkl_py3)\n\nRefer: https://pypi.org/project/mxnet/\n\n### Using the Build Script\n`./build_python_dockerfile.sh <mxnet_version> <pip_tag> <path_to_cloned_mxnet_repo>`\n\nFor example: \n`./build_python_dockerfile.sh 1.3.0 1.3.0.post0 ~/build-docker/mxnet`\n\n### Tests run\n* [test_mxnet.py](https://github.com/apache/mxnet/blob/master/docker/docker-python/test_mxnet.py): This script is used to make sure that the docker image builds the expected mxnet version. That is, the version picked by pip is the same as as the version passed as a parameter.\n\n### Dockerhub Credentials\nDockerhub credentials will be required to push images at the end of this script.\nCredentials can be provided in the following ways:\n* **Interactive Login:** Run the script as is and it will ask you for credentials interactively.\n* **Be Already Logged in:** Login to the mxnet dockerhub account before you run the build script and the script will complete build, test and push.\n* **Set Environment Variables:** Set the following environment variables which the script will pick up to login to dockerhub at runtime -\n    * $MXNET_DOCKERHUB_PASSWORD\n    * $MXNET_DOCKERHUB_USERNAME\n    \n\n### Using the Docker Images\n* The MXNet Python Docker images can be found here: https://hub.docker.com/r/mxnet/python/\n\n* Docker Pull Command: `docker pull mxnet/python:<image_tag>`\n* Get started: `docker run -it mxnet/python:<image_tag> bash`\n"
  },
  {
    "path": "docker/docker-python/build_python_dockerfile.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Check Params\nprogramname=$0\n\nfunction usage {\n    echo \"usage: $programname [version] [pip_tag] [path]\"\n    echo \"  [version]  Mxnet Version to build\"\n    echo \"  [pip_tag]  Pip Tag to use\"\n    echo \"  [path]     Path to MXNet repository (to run tests)\"\n    echo \" \"\n    exit 1\n}\n\nif [ $# -le 2 ] || [ $# -ge 4 ]\nthen\n    usage\n    exit 1\nfi\n\n# Two params provided\necho \"Building Docker Images for Apache MXNet (Incubating) v$1\"\nmxnet_version=\"${1}\"\npip_tag=\"${2}\"\ntest_dir=\"${3}\"\n\n# Remove the logs directory if it already exists else it may fail due to old logs.\nLOGDIR=~/temp/docker_logs\nif [ -d \"${LOGDIR}\" ]; then\n  rm -rf ${LOGDIR}\nfi\n\n# Create ~/temp if it does not exist\nmkdir -p ~/temp\nmkdir ${LOGDIR}\n\n\n# Functions\ndocker_test_image_cpu(){\n    image_tag=\"${1}\"\n    python_version=\"${2}\"\n    echo \"Running tests on mxnet/python:${image_tag}\"\n    docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c \"${python_version} /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}\"\n}\n\ndocker_test_image_gpu(){\n    image_tag=\"${1}\"\n    python_version=\"${2}\"\n    echo \"Running tests on mxnet/python:${1}\"\n    nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c \"${python_version} /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}\"\n}\n\n# if both $MXNET_DOCKERHUB_PASSWORD and $MXNET_DOCKERHUB_USERNAME environment variables are set, docker will automatically login\n# if env variables are not set, login will be interactive.\ndocker_account_login(){\n    if [[ -z $MXNET_DOCKERHUB_PASSWORD ]] || [[ -z $MXNET_DOCKERHUB_USERNAME ]]; then\n        docker login\n    else\n        echo $MXNET_DOCKERHUB_PASSWORD | docker login -u $MXNET_DOCKERHUB_USERNAME --password-stdin\n    fi\n}\n\ndocker_account_logout(){\n    docker logout\n}\n\ndocker_push_image(){\n    image_tag=\"${1}\"\n    docker push mxnet/python:${image_tag}\n}\n\ndocker_generate_image_cpu(){\n    image_tag=\"${1}\"\n    dockerfile=\"${2}\"\n    python_version=\"${3}\"\n    echo \"Building docker image mxnet/python:${image_tag}\"\n    docker build --build-arg version=${pip_tag} -t mxnet/python:${image_tag} -f ${dockerfile} .\n    docker_test_image_cpu ${image_tag} ${python_version}\n}\n\ndocker_tag_image_cpu(){\n    original_tag=\"${1}\"\n    image_tag=\"${2}\"\n    python_version=\"${3}\"\n    docker tag mxnet/python:${original_tag} mxnet/python:${image_tag}\n    docker_test_image_cpu ${image_tag} ${python_version}\n}\n\ndocker_generate_image_gpu(){\n    image_tag=\"${1}\"\n    dockerfile=\"${2}\"\n    python_version=\"${3}\"\n    echo \"Building docker image mxnet/python:${1}\"\n    docker build --build-arg version=${pip_tag} -t mxnet/python:${image_tag} -f ${dockerfile} .\n    docker_test_image_gpu ${image_tag} ${python_version}\n}\n\ndocker_tag_image_gpu(){\n    original_tag=\"${1}\"\n    image_tag=\"${2}\"\n    python_version=\"${3}\"\n    docker tag mxnet/python:${original_tag} mxnet/python:${image_tag}\n    docker_test_image_gpu ${image_tag} ${python_version}\n}\n\ncheck_errors(){\n    egrep -i \"not found|error|returned a non-zero code|fail\" ${LOGDIR}/docker*\n    if [ $? -eq 0 ]; then\n        echo \"ERROR: One of the build/test commands failed. Refer to the filename above to see which image tag caused it.\"\n        exit 1\n    else\n        echo \"Success: No errors found\"\n    fi\n}\n\n# Build and Test dockerfiles - CPU\ndocker_generate_image_cpu \"${mxnet_version}_cpu\" \"Dockerfile.mxnet.python.cpu\" \"python\"  > ${LOGDIR}/docker_cpu.out 2>&1 &\ndocker_generate_image_cpu \"${mxnet_version}_cpu_mkl\" \"Dockerfile.mxnet.python.cpu.mkl\" \"python\" > ${LOGDIR}/docker_cpu_mkl.out 2>&1 &\n\n\n#Build and Test dockerfiles - GPU\ndocker_generate_image_gpu \"${mxnet_version}_gpu_cu92\" \"Dockerfile.mxnet.python.gpu.cu92\" \"python\" > ${LOGDIR}/docker_gpu_cu92.out 2>&1 &\ndocker_generate_image_gpu \"${mxnet_version}_gpu_cu92_mkl\" \"Dockerfile.mxnet.python.gpu.cu92.mkl\" \"python\" > ${LOGDIR}/docker_gpu_cu92_mkl.out 2>&1\n\necho \"Waiting for MXNet Python2 Docker Images to Build\"\nwait\n\n# Build and Test Python3 dockerfiles - CPU\ndocker_generate_image_cpu \"${mxnet_version}_cpu_py3\" \"Dockerfile.mxnet.python3.cpu\" \"python3\" > ${LOGDIR}/docker_cpu_py3.out 2>&1 &\ndocker_generate_image_cpu \"${mxnet_version}_cpu_mkl_py3\" \"Dockerfile.mxnet.python3.cpu.mkl\" \"python3\" > ${LOGDIR}/docker_cpu_mkl_py3.out 2>&1 &\n\n#Build and Test Python3 dockerfiles - GPU\ndocker_generate_image_gpu \"${mxnet_version}_gpu_cu92_py3\" \"Dockerfile.mxnet.python3.gpu.cu92\" \"python3\" > ${LOGDIR}/docker_gpu_cu92_py3.out 2>&1 &\ndocker_generate_image_gpu \"${mxnet_version}_gpu_cu92_mkl_py3\" \"Dockerfile.mxnet.python3.gpu.cu92.mkl\" \"python3\" > ${LOGDIR}/docker_gpu_cu92_mkl_py3.out 2>&1\n\necho \"Waiting for MXNet Python3 Docker Images to Build\"\nwait\n\necho \"Re-Tag 6 images with version-free names (for Benchmarking) - only after previous builds complete. \"\ndocker_tag_image_cpu \"${mxnet_version}_cpu\" \"latest\" \"python\" > ${LOGDIR}/docker_latest.out 2>&1 &\ndocker_tag_image_cpu \"${mxnet_version}_cpu_mkl\" \"latest_cpu_mkl_py2\" \"python\" > ${LOGDIR}/docker_latest_cpu_mkl_py2.out 2>&1 &\ndocker_tag_image_cpu \"${mxnet_version}_cpu_mkl_py3\" \"latest_cpu_mkl_py3\" \"python3\" > ${LOGDIR}/docker_latest_cpu_mkl_py3.out 2>&1 &\nwait\n\n# Parse all the docker logfiles to make sure there is no error. Fail script if error is found.\ncheck_errors\n\n# Push dockerfiles\necho \"All images were successfully built. Now login to dockerhub and push images\"\ndocker_account_login\n\n# Python2\ndocker_push_image \"${mxnet_version}_cpu\"\ndocker_push_image \"${mxnet_version}_cpu_mkl\"\ndocker_push_image \"${mxnet_version}_gpu_cu92\"\ndocker_push_image \"${mxnet_version}_gpu_cu92_mkl\"\n\n# Python3\ndocker_push_image \"${mxnet_version}_cpu_py3\"\ndocker_push_image \"${mxnet_version}_cpu_mkl_py3\"\ndocker_push_image \"${mxnet_version}_gpu_cu92_py3\"\ndocker_push_image \"${mxnet_version}_gpu_cu92_mkl_py3\"\n\ndocker_push_image \"latest\"\ndocker_push_image \"gpu\"\ndocker_push_image \"latest_cpu_mkl_py2\"\ndocker_push_image \"latest_cpu_mkl_py3\"\ndocker_push_image \"latest_gpu_mkl_py2\"\ndocker_push_image \"latest_gpu_mkl_py3\"\n\n\ndocker_account_logout\n\necho \"Successfully Built, Tested and Pushed all Images to Dockerhub. Link: https://hub.docker.com/r/mxnet/python/tags/\"\n\n#Delete the log directory since everything succeeded:\nrm -rf ${LOGDIR}\n"
  },
  {
    "path": "docker/docker-python/test_mxnet.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# This checks that the version of mxnet imported matches the parameter passed to the build script.\nimport mxnet as mx\nimport sys\n\npip_version = mx.__version__\nexpected_version = sys.argv[1]\n\nif pip_version != expected_version:\n    raise ValueError(\"ERROR: Incorrect pip version. Please check the parameter passed or pip binary used.\")\n"
  },
  {
    "path": "docker/install/cpp.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# libraries for building mxnet c++ core on ubuntu\n\napt-get update && apt-get install -y \\\n    build-essential git libatlas-base-dev libopencv-dev python-opencv \\\n    libcurl4-openssl-dev libgtest-dev cmake wget unzip\n\ncd /usr/src/gtest && cmake CMakeLists.txt && make && cp *.a /usr/lib\n"
  },
  {
    "path": "docker/install/julia.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# install libraries for mxnet's julia package on ubuntu\n\n# the julia version shipped with ubuntu (version 0.4) is too low. so download a\n# new version\n# apt-get install -y julia\n\nwget -q https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5.1-linux-x86_64.tar.gz\ntar -zxf julia-0.5.1-linux-x86_64.tar.gz\nrm julia-0.5.1-linux-x86_64.tar.gz\nln -s $(pwd)/julia-6445c82d00/bin/julia /usr/bin/julia\n"
  },
  {
    "path": "docker/install/perl.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# install libraries for mxnet's perl package on ubuntu\napt-get update && apt-get install -y libmouse-perl pdl cpanminus swig libgraphviz-perl\ncpanm -q Function::Parameters Hash::Ordered PDL::CCS\n"
  },
  {
    "path": "docker/install/python.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# install libraries for mxnet's python package on ubuntu\n\napt-get update && apt-get install -y python3-dev\n\n# the version of the pip shipped with ubuntu may be too lower, install a recent version here\ncd /tmp && wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py\n\npip3 install pylint numpy requests Pillow pytest==6.1.2 pytest-env==0.6.2 pytest-cov==2.10.1 pytest-xdist==2.1.0\n"
  },
  {
    "path": "docker/install/r.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# install libraries for mxnet's r package on ubuntu\n\necho \"deb http://cran.rstudio.com/bin/linux/ubuntu trusty/\" >> /etc/apt/sources.list\ngpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9\ngpg -a --export E084DAB9 | apt-key add -\n\napt-get update\napt-get install -y r-base r-base-dev libxml2-dev libxt-dev libssl-dev\n\ncd \"$(dirname \"${BASH_SOURCE[0]}\")\"\n\nRscript -e \"install.packages('devtools', repo = 'https://cran.rstudio.com')\"\nRscript -e \"library(devtools); library(methods); options(repos=c(CRAN='https://cran.rstudio.com')); install_deps(dependencies = TRUE)\"\n"
  },
  {
    "path": "docker/install/scala.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# install libraries for mxnet's scala package on ubuntu\n\n\napt-get install -y software-properties-common\nadd-apt-repository -y ppa:webupd8team/java\napt-get update\necho \"oracle-java8-installer shared/accepted-oracle-license-v1-1 select true\" | debconf-set-selections\napt-get install -y oracle-java8-installer\napt-get install -y oracle-java8-set-default\n\napt-get install -y maven\n\nwget http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb\ndpkg -i scala-2.11.8.deb\nrm scala-2.11.8.deb\n"
  },
  {
    "path": "docker/run.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Build and push all docker containers\n\nDEVICES=('cpu' 'gpu')\nLANGUAGES=('python' 'julia' 'r-lang' 'scala' 'perl')\nfor DEV in \"${DEVICES[@]}\"; do\n    for LANG in \"${LANGUAGES[@]}\"; do\n        ./tool.sh build ${LANG} ${DEV}\n        ./tool.sh push ${LANG} ${DEV}\n    done\ndone\n"
  },
  {
    "path": "docker/tool.sh",
    "content": "#!/usr/bin/env bash\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n#\n# Script to build, test and push a docker container\n#\nSCRIPT_DIR=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\" && pwd)\"\n\nfunction show_usage() {\n    echo \"\"\n    echo \"Usage: $(basename $0) COMMAND LANGUAGE DEVICE\"\n    echo \"\"\n    echo \"   COMMAND: build or commit.\"\n    echo \"            commit needs logined in docker hub\"\n    echo \"   LANGUAGE: the language binding to buld, e.g. python, r-lang, julia, scala or perl\"\n    echo \"   DEVICE: targed device, e.g. cpu, or gpu\"\n    echo \"\"\n}\n\nif (( $# < 3 )); then\n    show_usage\n    exit -1\nfi\n\nCOMMAND=$( echo \"$1\" | tr '[:upper:]' '[:lower:]' )\nshift 1\nLANGUAGE=$( echo \"$1\" | tr '[:upper:]' '[:lower:]' )\nshift 1\nDEVICE=$( echo \"$1\" | tr '[:upper:]' '[:lower:]' )\nshift 1\n\nDOCKERFILE_LIB=\"${SCRIPT_DIR}/Dockerfiles/Dockerfile.in.lib.${DEVICE}\"\nif [ ! -e ${DOCKERFILE_LIB} ]; then\n    echo \"Error DEVICE=${DEVICE}, failed to find ${DOCKERFILE_LIB}\"\n    show_usage\n    exit 1\nfi\n\nDOCKERFILE_LANG=\"${SCRIPT_DIR}/Dockerfiles/Dockerfile.in.${LANGUAGE}\"\nif [ ! -e ${DOCKERFILE_LANG} ]; then\n    echo \"Error LANGUAGE=${LANGUAGE}, failed to find ${DOCKERFILE_LANG}\"\n    show_usage\n    exit 1\nfi\n\nif [[ \"${DEVICE}\" == *\"gpu\"* ]] && [[ \"{COMMAND}\" == \"test\" ]]; then\n    DOCKER_BINARY=\"nvidia-docker\"\nelse\n    DOCKER_BINARY=\"docker\"\nfi\n\nDOCKER_TAG=\"mxnet/${LANGUAGE}\"\nif [ \"${DEVICE}\" != 'cpu' ]; then\n    DOCKER_TAG=\"${DOCKER_TAG}:${DEVICE}\"\nfi\nDOCKERFILE=\"Dockerfile.${LANGUAGE}.${DEVICE}\"\n\n# print arguments\necho \"DOCKER_BINARY: ${DOCKER_BINARY}\"\necho \"DOCKERFILE: ${DOCKERFILE}\"\necho \"DOCKER_TAG: ${DOCKER_TAG}\"\n\nif [[ \"${COMMAND}\" == \"build\" ]]; then\n    rm -rf ${DOCKERFILE}\n    cp ${DOCKERFILE_LIB} ${DOCKERFILE}\n    cat ${DOCKERFILE_LANG} >>${DOCKERFILE}\n    # To remove the following error caused by opencv\n    #    libdc1394 error: Failed to initialize libdc1394\"\n    CMD=\"sh -c 'ln -s /dev/null /dev/raw1394';\"\n    # setup scala classpath\n    if [[ \"${LANGUAGE}\" == \"scala\" ]]; then\n        CMD+=\"CLASSPATH=\\${CLASSPATH}:\\`ls /mxnet/scala-package/assembly/linux-x86_64-*/target/*.jar | paste -sd \\\":\\\"\\` \"\n    fi\n    echo \"CMD ${CMD} bash\" >>${DOCKERFILE}\n    ${DOCKER_BINARY} build -t ${DOCKER_TAG} -f ${DOCKERFILE} .\nelif [[ \"${COMMAND}\" == \"push\" ]]; then\n    ${DOCKER_BINARY} push ${DOCKER_TAG}\nelse\n    echo \"Unknow COMMAND=${COMMAND}\"\n    show_usage\n    exit 1\nfi\n"
  },
  {
    "path": "docs/.dockerignore",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n\n#   http://www.apache.org/licenses/LICENSE-2.0\n\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nDockerfile\n_build\n\n"
  },
  {
    "path": "docs/.gitignore",
    "content": "_build/*\n*.pyc\ndoxygen\n"
  },
  {
    "path": "docs/README.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Building and Updating MXNet Documentation\n\nThe website is hosted at https://mxnet.apache.org/.\nhttps://mxnet.io redirects to this site and advised to use links with https://mxnet.apache.org/ instead of https://mxnet.io/.\n\n## Website & Documentation Contributions\n\nDetailed information on website development, continuous integration, and proposals for future projects can be found on the [MXNet Wiki](https://cwiki.apache.org/confluence/display/MXNET/Website).\n\nThe website is built using Jekyll. You may run your own version of the static website by following the instructions on the wiki.\n\nEach language documentation is built in a modular way, so that if you are a contributor to Julia, for example, you only need Julia-related tools to build it. Each language API has a section on installation and building along with how to build the docs locally.\n\nYou can also use the project's CI tools to emulate any changes with Docker. You can use these tools to install dependencies and run the parts of the build you want to test.\n\nRefer to the [MXNet Developer Wiki](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=125309983) for instructions on building the docs locally.\n\nIf you plan to contribute changes to the documentation or website, please submit a pull request. Contributions are welcome!\n\n## Python Docs\n\nMXNet's Python documentation is built with [Sphinx](https://www.sphinx-doc.org) and a variety of plugins including [pandoc](https://pandoc.org/), and [recommonmark](https://github.com/rtfd/recommonmark).\n\nMore information on the dependencies can be found in the [CI folder's installation scripts](https://github.com/apache/mxnet/tree/master/ci/docker/install/ubuntu_docs.sh).\n\nYou can run just the Python docs by following the instructions in the Python API guide.\n\n## Other API Docs\n\nThe docs are hosted on the website in each language API's section. You can find installation and build instructions there.\n\n## How to Build the MXNet Website for Development and QA\n\n`conda` or `miniconda` is recommended.\n* [Conda](https://www.anaconda.com/distribution/#download-section) (install to PATH)\n\nIf you only need to make changes to tutorials or other pages that are not generated from one of the API source code folders, then you can use a basic Python pip or conda installation. But if you want edit the API source and have the reference API docs update, you also need to build MXNet from source. Refer to the build from source instructions for this requirement.\n\n\n### Ubuntu Setup\n\nAs this is maintained for CI, Ubuntu is recommended. Refer to [ubuntu_doc.sh](https://github.com/apache/mxnet/tree/master/ci/docker/install/ubuntu_docs.sh) for the latest install script.\n\n### Caveat for Rendering Outputs\n\nNote that without a GPU you will not be able to generate the docs with the outputs in the tutorials.\n\n### GPU setup\nTo run the full build, including tests of all tutorials,\n**you will need at least two GPUs**.\nDistributed training is a key feature of MXNet,\nso multiple GPUs are required for running through every tutorial.\n* [CUDA 9.2](https://developer.nvidia.com/cuda-downloads)\n\n### CPU-only setup\nIn the `environment.yml` file:\n* Change `mxnet-cu92` to `mxnet`.\n\n### macOS setup\nIn the `environment.yml` file:\n* Change `mxnet-cu92` to `mxnet`. (There is no CUDA package for mac anyway.)\n\n### Windows Setup\nIf you have a GPU and have installed CUDA 9.2 you can leave the MXNet dependency alone.\nOtherwise, in the `environment.yml` file:\n* Change `mxnet-cu92` to `mxnet`.\n\nInstall recommended software:\n* [git bash](https://gitforwindows.org/)\n* Be sure to install `Conda` in `PATH`\n* Install `make` from a `git bash` terminal with Admin rights\n    - [Install chocolatey](https://chocolatey.org/install)\n    - Use `choco to install make`\n* Restart terminals after installations to make sure PATH is set.\n    - The `choco`, `make`, and `conda` commands should work in `git bash`.\n\n### Conda environment setup\nRun the following commands from the project root (`new-docs`) to setup the environment.\n\n```bash\nconda env create -f environment.yml\nsource activate mxnet-docs\n```\n\n## Build the docs\n\n* Change directories to `new-docs/python`.\n\nTo build without GPUs and without testing the notebooks (faster):\n\n```bash\nmake EVAL=0\n```\n\nTo build with testing the notebooks (requires GPU):\n\n```bash\nmake\n```\n\nThe build docs will be available at `build/_build/html`.\n\nEach build may take a few minutes even without evaluation. To accelerate it, we can use one of the following ways:\n\n1. open `build/conf.py`, add the folders you want to skip into `exclude_patterns`, such as `exclude_patterns = ['templates', 'api', 'develop', 'blog']`.\n2. move the files into a different folder, such as `mv api /tmp/`, and then `make clean`.\n\n## Check results\n\nTo run a server to see the website:\n\n1. Start a http server: `cd build/_build/html; python -m http.server`\n2. For viewing a remote machine, ssh to your machine with port forwarding: `ssh -L8000:localhost:8000 your_machine`\n3. Open http://localhost:8000 in your local machine\n\n## Run tutorials\n\nIn addition to view the built html pages, you can run the Jupyter notebook from a remote machine.\n1. Install `notedown` plugin: `pip install https://github.com/mli/notedown/tarball/master` in remote server\n2. Start Jupyter notebook `jupyter notebook --NotebookApp.contents_manager_class='notedown.NotedownContentsManager'` in remote server\n3. ssh to your machine with port forwarding: `ssh -L8888:localhost:8888 your_machine`\n4. Open http://localhost:8888 in your local machine and run the md files directly\n\nOptionally, one can run the following to launch the notedown plugin automatically when starting jupyter notebook.\n1. Generate the jupyter configure file `~/.jupyter/jupyter_notebook_config.py` if it\nis not existing by run `jupyter notebook --generate-config`\n2. Add `c.NotebookApp.contents_manager_class = 'notedown.NotedownContentsManager'` to `~/.jupyter/jupyter_notebook_config.py`\n3. Simply run `jupyter notebook`\n\n## Troubleshooting\nDependencies and the setup steps for this website are changing often. Here are some troubleshooting tips.\n\n* You might need to update the environment for the latest modules.\n```bash\nconda env update -f environment.yml\n```\n\nThe `-W` Sphinx option enforces \"warnings as errors\". This will help you debug your builds and get them through CI.\n**CI will not let a PR through if it breaks the website.** Refer to the [MXNet Developer wiki's documentation guide](https://cwiki.apache.org/confluence/display/MXNET/Documentation+Guide) for troubleshooting tips.\n\n\n## Production Website Deployment Process\n\n[Apache Jenkins MXNet website building job](https://builds.apache.org/job/mxnet-build-site/) is used to build MXNet website.\n\nThe Jenkins docs build job will fetch MXNet repository, build MXNet website and push all static files to [host repository](https://github.com/apache/mxnet-site.git).\n\nThe host repo is hooked with [Apache gitbox](https://gitbox.apache.org/repos/asf?p=mxnet-site.git;a=summary) to host website.\n\n### Processes for Running the Docs Build Jobs\n\nThis information is maintained on the [MXNet Wiki](https://cwiki.apache.org/confluence/display/MXNET/Website).\n\n\n## Other Docs Build Processes\n\n* Perl API docs are maintained separately at [metacpan](https://metacpan.org/release/AI-MXNet).\n\n\n## Troubleshooting\n\n- If C++ code has been changed, remove the previous results to trigger the rebuild for all pages. To do this, run `make clean_docs`.\n- If C++ code fails to build, run `make clean`.\n- If CSS or javascript are changed, clear the cache in the browser with a *forced refresh*.\n- If search doesn't work, run `make clean` and then `make docs`.\n"
  },
  {
    "path": "docs/cpp_docs/Doxyfile",
    "content": "# Doxyfile 1.8.8\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# This file describes the settings to be used by the documentation system\n# doxygen (www.doxygen.org) for a project.\n#\n# All text after a double hash (##) is considered a comment and is placed in\n# front of the TAG it is preceding.\n#\n# All text after a single hash (#) is considered a comment and will be ignored.\n# The format is:\n# TAG = value [value, ...]\n# For lists, items can also be appended using:\n# TAG += value [value, ...]\n# Values that contain spaces should be placed between quotes (\\\" \\\").\n\n#---------------------------------------------------------------------------\n# Project related configuration options\n#---------------------------------------------------------------------------\n\n# This tag specifies the encoding used for all characters in the config file\n# that follow. The default is UTF-8 which is also the encoding used for all text\n# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv\n# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv\n# for the list of possible encodings.\n# The default value is: UTF-8.\n\nDOXYFILE_ENCODING      = UTF-8\n\n# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by\n# double-quotes, unless you are using Doxywizard) that should identify the\n# project for which the documentation is generated. This name is used in the\n# title of most generated pages and in a few other places.\n# The default value is: My Project.\n\nPROJECT_NAME           = \"mxnet\"\n\n# The PROJECT_NUMBER tag can be used to enter a project or revision number. This\n# could be handy for archiving the generated documentation or if some version\n# control system is used.\n\nPROJECT_NUMBER         =\n\n# Using the PROJECT_BRIEF tag one can provide an optional one line description\n# for a project that appears at the top of each page and should give viewer a\n# quick idea about the purpose of the project. Keep the description short.\n\nPROJECT_BRIEF          =\n\n# With the PROJECT_LOGO tag one can specify an logo or icon that is included in\n# the documentation. The maximum height of the logo should not exceed 55 pixels\n# and the maximum width should not exceed 200 pixels. Doxygen will copy the logo\n# to the output directory.\n\nPROJECT_LOGO           =\n\n# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path\n# into which the generated documentation will be written. If a relative path is\n# entered, it will be relative to the location where doxygen was started. If\n# left blank the current directory will be used.\n\nOUTPUT_DIRECTORY       = build/html\n\n# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create 4096 sub-\n# directories (in 2 levels) under the output directory of each output format and\n# will distribute the generated files over these directories. Enabling this\n# option can be useful when feeding doxygen a huge amount of source files, where\n# putting all generated files in the same directory would otherwise causes\n# performance problems for the file system.\n# The default value is: NO.\n\nCREATE_SUBDIRS         = NO\n\n# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII\n# characters to appear in the names of generated files. If set to NO, non-ASCII\n# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode\n# U+3044.\n# The default value is: NO.\n\n#ALLOW_UNICODE_NAMES    = NO\n\n# The OUTPUT_LANGUAGE tag is used to specify the language in which all\n# documentation generated by doxygen is written. Doxygen will use this\n# information to generate all constant output in the proper language.\n# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese,\n# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States),\n# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian,\n# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages),\n# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian,\n# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian,\n# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish,\n# Ukrainian and Vietnamese.\n# The default value is: English.\n\nOUTPUT_LANGUAGE        = English\n\n# If the BRIEF_MEMBER_DESC tag is set to YES doxygen will include brief member\n# descriptions after the members that are listed in the file and class\n# documentation (similar to Javadoc). Set to NO to disable this.\n# The default value is: YES.\n\nBRIEF_MEMBER_DESC      = YES\n\n# If the REPEAT_BRIEF tag is set to YES doxygen will prepend the brief\n# description of a member or function before the detailed description\n#\n# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the\n# brief descriptions will be completely suppressed.\n# The default value is: YES.\n\nREPEAT_BRIEF           = YES\n\n# This tag implements a quasi-intelligent brief description abbreviator that is\n# used to form the text in various listings. Each string in this list, if found\n# as the leading text of the brief description, will be stripped from the text\n# and the result, after processing the whole list, is used as the annotated\n# text. Otherwise, the brief description is used as-is. If left blank, the\n# following values are used ($name is automatically replaced with the name of\n# the entity):The $name class, The $name widget, The $name file, is, provides,\n# specifies, contains, represents, a, an and the.\n\nABBREVIATE_BRIEF       =\n\n# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then\n# doxygen will generate a detailed section even if there is only a brief\n# description.\n# The default value is: NO.\n\nALWAYS_DETAILED_SEC    = NO\n\n# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all\n# inherited members of a class in the documentation of that class as if those\n# members were ordinary class members. Constructors, destructors and assignment\n# operators of the base classes will not be shown.\n# The default value is: NO.\n\nINLINE_INHERITED_MEMB  = NO\n\n# If the FULL_PATH_NAMES tag is set to YES doxygen will prepend the full path\n# before files name in the file list and in the header files. If set to NO the\n# shortest path that makes the file name unique will be used\n# The default value is: YES.\n\nFULL_PATH_NAMES        = YES\n\n# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path.\n# Stripping is only done if one of the specified strings matches the left-hand\n# part of the path. The tag can be used to show relative paths in the file list.\n# If left blank the directory from which doxygen is run is used as the path to\n# strip.\n#\n# Note that you can specify absolute paths here, but also relative paths, which\n# will be relative from the directory where doxygen is started.\n# This tag requires that the tag FULL_PATH_NAMES is set to YES.\n\nSTRIP_FROM_PATH        =\n\n# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the\n# path mentioned in the documentation of a class, which tells the reader which\n# header file to include in order to use a class. If left blank only the name of\n# the header file containing the class definition is used. Otherwise one should\n# specify the list of include paths that are normally passed to the compiler\n# using the -I flag.\n\nSTRIP_FROM_INC_PATH    =\n\n# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but\n# less readable) file names. This can be useful is your file systems doesn't\n# support long names like on DOS, Mac, or CD-ROM.\n# The default value is: NO.\n\nSHORT_NAMES            = NO\n\n# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the\n# first line (until the first dot) of a Javadoc-style comment as the brief\n# description. If set to NO, the Javadoc-style will behave just like regular Qt-\n# style comments (thus requiring an explicit @brief command for a brief\n# description.)\n# The default value is: NO.\n\nJAVADOC_AUTOBRIEF      = NO\n\n# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first\n# line (until the first dot) of a Qt-style comment as the brief description. If\n# set to NO, the Qt-style will behave just like regular Qt-style comments (thus\n# requiring an explicit \\brief command for a brief description.)\n# The default value is: NO.\n\nQT_AUTOBRIEF           = NO\n\n# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a\n# multi-line C++ special comment block (i.e. a block of //! or /// comments) as\n# a brief description. This used to be the default behavior. The new default is\n# to treat a multi-line C++ comment block as a detailed description. Set this\n# tag to YES if you prefer the old behavior instead.\n#\n# Note that setting this tag to YES also means that rational rose comments are\n# not recognized any more.\n# The default value is: NO.\n\nMULTILINE_CPP_IS_BRIEF = NO\n\n# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the\n# documentation from any documented member that it re-implements.\n# The default value is: YES.\n\nINHERIT_DOCS           = YES\n\n# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce a\n# new page for each member. If set to NO, the documentation of a member will be\n# part of the file/class/namespace that contains it.\n# The default value is: NO.\n\nSEPARATE_MEMBER_PAGES  = NO\n\n# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen\n# uses this value to replace tabs by spaces in code fragments.\n# Minimum value: 1, maximum value: 16, default value: 4.\n\nTAB_SIZE               = 8\n\n# This tag can be used to specify a number of aliases that act as commands in\n# the documentation. An alias has the form:\n# name=value\n# For example adding\n# \"sideeffect=@par Side Effects:\\n\"\n# will allow you to put the command \\sideeffect (or @sideeffect) in the\n# documentation, which will result in a user-defined paragraph with heading\n# \"Side Effects:\". You can put \\n's in the value part of an alias to insert\n# newlines.\n\nALIASES                =\n\n# This tag can be used to specify a number of word-keyword mappings (TCL only).\n# A mapping has the form \"name=value\". For example adding \"class=itcl::class\"\n# will allow you to use the command class in the itcl::class meaning.\n\nTCL_SUBST              =\n\n# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources\n# only. Doxygen will then generate output that is more tailored for C. For\n# instance, some of the names that are used will be different. The list of all\n# members will be omitted, etc.\n# The default value is: NO.\n\nOPTIMIZE_OUTPUT_FOR_C  = NO\n\n# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or\n# Python sources only. Doxygen will then generate output that is more tailored\n# for that language. For instance, namespaces will be presented as packages,\n# qualified scopes will look different, etc.\n# The default value is: NO.\n\nOPTIMIZE_OUTPUT_JAVA   = NO\n\n# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran\n# sources. Doxygen will then generate output that is tailored for Fortran.\n# The default value is: NO.\n\nOPTIMIZE_FOR_FORTRAN   = NO\n\n# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL\n# sources. Doxygen will then generate output that is tailored for VHDL.\n# The default value is: NO.\n\nOPTIMIZE_OUTPUT_VHDL   = NO\n\n# Doxygen selects the parser to use depending on the extension of the files it\n# parses. With this tag you can assign which parser to use for a given\n# extension. Doxygen has a built-in mapping, but you can override or extend it\n# using this tag. The format is ext=language, where ext is a file extension, and\n# language is one of the parsers supported by doxygen: IDL, Java, Javascript,\n# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran:\n# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran:\n# Fortran. In the later case the parser tries to guess whether the code is fixed\n# or free formatted code, this is the default for Fortran type files), VHDL. For\n# instance to make doxygen treat .inc files as Fortran files (default is PHP),\n# and .f files as C (default is Fortran), use: inc=Fortran f=C.\n#\n# Note For files without extension you can use no_extension as a placeholder.\n#\n# Note that for custom extensions you also need to set FILE_PATTERNS otherwise\n# the files are not read by doxygen.\n\nEXTENSION_MAPPING      =\n\n# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments\n# according to the Markdown format, which allows for more readable\n# documentation. See http://daringfireball.net/projects/markdown/ for details.\n# The output of markdown processing is further processed by doxygen, so you can\n# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in\n# case of backward compatibilities issues.\n# The default value is: YES.\n\n#MARKDOWN_SUPPORT       = YES\n\n# When enabled doxygen tries to link words that correspond to documented\n# classes, or namespaces to their corresponding documentation. Such a link can\n# be prevented in individual cases by by putting a % sign in front of the word\n# or globally by setting AUTOLINK_SUPPORT to NO.\n# The default value is: YES.\n\n#AUTOLINK_SUPPORT       = YES\n\n# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want\n# to include (a tag file for) the STL sources as input, then you should set this\n# tag to YES in order to let doxygen match functions declarations and\n# definitions whose arguments contain STL classes (e.g. func(std::string);\n# versus func(std::string) {}). This also make the inheritance and collaboration\n# diagrams that involve STL classes more complete and accurate.\n# The default value is: NO.\n\nBUILTIN_STL_SUPPORT    = NO\n\n# If you use Microsoft's C++/CLI language, you should set this option to YES to\n# enable parsing support.\n# The default value is: NO.\n\nCPP_CLI_SUPPORT        = NO\n\n# Set the SIP_SUPPORT tag to YES if your project consists of sip (see:\n# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen\n# will parse them like normal C++ but will assume all classes use public instead\n# of private inheritance when no explicit protection keyword is present.\n# The default value is: NO.\n\nSIP_SUPPORT            = NO\n\n# For Microsoft's IDL there are propget and propput attributes to indicate\n# getter and setter methods for a property. Setting this option to YES will make\n# doxygen to replace the get and set methods by a property in the documentation.\n# This will only work if the methods are indeed getting or setting a simple\n# type. If this is not the case, or you want to show the methods anyway, you\n# should set this option to NO.\n# The default value is: YES.\n\nIDL_PROPERTY_SUPPORT   = YES\n\n# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC\n# tag is set to YES, then doxygen will reuse the documentation of the first\n# member in the group (if any) for the other members of the group. By default\n# all members of a group must be documented explicitly.\n# The default value is: NO.\n\nDISTRIBUTE_GROUP_DOC   = NO\n\n# Set the SUBGROUPING tag to YES to allow class member groups of the same type\n# (for instance a group of public functions) to be put as a subgroup of that\n# type (e.g. under the Public Functions section). Set it to NO to prevent\n# subgrouping. Alternatively, this can be done per class using the\n# \\nosubgrouping command.\n# The default value is: YES.\n\nSUBGROUPING            = YES\n\n# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions\n# are shown inside the group in which they are included (e.g. using \\ingroup)\n# instead of on a separate page (for HTML and Man pages) or section (for LaTeX\n# and RTF).\n#\n# Note that this feature does not work in combination with\n# SEPARATE_MEMBER_PAGES.\n# The default value is: NO.\n\nINLINE_GROUPED_CLASSES = NO\n\n# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions\n# with only public data fields or simple typedef fields will be shown inline in\n# the documentation of the scope in which they are defined (i.e. file,\n# namespace, or group documentation), provided this scope is documented. If set\n# to NO, structs, classes, and unions are shown on a separate page (for HTML and\n# Man pages) or section (for LaTeX and RTF).\n# The default value is: NO.\n\nINLINE_SIMPLE_STRUCTS  = NO\n\n# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or\n# enum is documented as struct, union, or enum with the name of the typedef. So\n# typedef struct TypeS {} TypeT, will appear in the documentation as a struct\n# with name TypeT. When disabled the typedef will appear as a member of a file,\n# namespace, or class. And the struct will be named TypeS. This can typically be\n# useful for C code in case the coding convention dictates that all compound\n# types are typedef'ed and only the typedef is referenced, never the tag name.\n# The default value is: NO.\n\nTYPEDEF_HIDES_STRUCT   = NO\n\n# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This\n# cache is used to resolve symbols given their name and scope. Since this can be\n# an expensive process and often the same symbol appears multiple times in the\n# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small\n# doxygen will become slower. If the cache is too large, memory is wasted. The\n# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range\n# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536\n# symbols. At the end of a run doxygen will report the cache usage and suggest\n# the optimal cache size from a speed point of view.\n# Minimum value: 0, maximum value: 9, default value: 0.\n\nLOOKUP_CACHE_SIZE      = 0\n\n#---------------------------------------------------------------------------\n# Build related configuration options\n#---------------------------------------------------------------------------\n\n# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in\n# documentation are documented, even if no documentation was available. Private\n# class members and static file members will be hidden unless the\n# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES.\n# Note: This will also disable the warnings about undocumented members that are\n# normally produced when WARNINGS is set to YES.\n# The default value is: NO.\n\nEXTRACT_ALL            = YES\n\n# If the EXTRACT_PRIVATE tag is set to YES all private members of a class will\n# be included in the documentation.\n# The default value is: NO.\n\nEXTRACT_PRIVATE        = NO\n\n# If the EXTRACT_PACKAGE tag is set to YES all members with package or internal\n# scope will be included in the documentation.\n# The default value is: NO.\n\n#EXTRACT_PACKAGE        = NO\n\n# If the EXTRACT_STATIC tag is set to YES all static members of a file will be\n# included in the documentation.\n# The default value is: NO.\n\nEXTRACT_STATIC         = NO\n\n# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) defined\n# locally in source files will be included in the documentation. If set to NO\n# only classes defined in header files are included. Does not have any effect\n# for Java sources.\n# The default value is: YES.\n\nEXTRACT_LOCAL_CLASSES  = YES\n\n# This flag is only useful for Objective-C code. When set to YES local methods,\n# which are defined in the implementation section but not in the interface are\n# included in the documentation. If set to NO only methods in the interface are\n# included.\n# The default value is: NO.\n\nEXTRACT_LOCAL_METHODS  = NO\n\n# If this flag is set to YES, the members of anonymous namespaces will be\n# extracted and appear in the documentation as a namespace called\n# 'anonymous_namespace{file}', where file will be replaced with the base name of\n# the file that contains the anonymous namespace. By default anonymous namespace\n# are hidden.\n# The default value is: NO.\n\nEXTRACT_ANON_NSPACES   = NO\n\n# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all\n# undocumented members inside documented classes or files. If set to NO these\n# members will be included in the various overviews, but no documentation\n# section is generated. This option has no effect if EXTRACT_ALL is enabled.\n# The default value is: NO.\n\nHIDE_UNDOC_MEMBERS     = NO\n\n# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all\n# undocumented classes that are normally visible in the class hierarchy. If set\n# to NO these classes will be included in the various overviews. This option has\n# no effect if EXTRACT_ALL is enabled.\n# The default value is: NO.\n\nHIDE_UNDOC_CLASSES     = NO\n\n# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend\n# (class|struct|union) declarations. If set to NO these declarations will be\n# included in the documentation.\n# The default value is: NO.\n\nHIDE_FRIEND_COMPOUNDS  = NO\n\n# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any\n# documentation blocks found inside the body of a function. If set to NO these\n# blocks will be appended to the function's detailed documentation block.\n# The default value is: NO.\n\nHIDE_IN_BODY_DOCS      = NO\n\n# The INTERNAL_DOCS tag determines if documentation that is typed after a\n# \\internal command is included. If the tag is set to NO then the documentation\n# will be excluded. Set it to YES to include the internal documentation.\n# The default value is: NO.\n\nINTERNAL_DOCS          = NO\n\n# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file\n# names in lower-case letters. If set to YES upper-case letters are also\n# allowed. This is useful if you have classes or files whose names only differ\n# in case and if your file system supports case sensitive file names. Windows\n# and Mac users are advised to set this option to NO.\n# The default value is: system dependent.\n\nCASE_SENSE_NAMES       = YES\n\n# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with\n# their full class and namespace scopes in the documentation. If set to YES the\n# scope will be hidden.\n# The default value is: NO.\n\nHIDE_SCOPE_NAMES       = NO\n\n# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of\n# the files that are included by a file in the documentation of that file.\n# The default value is: YES.\n\nSHOW_INCLUDE_FILES     = YES\n\n# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each\n# grouped member an include statement to the documentation, telling the reader\n# which file to include in order to use the member.\n# The default value is: NO.\n\n#SHOW_GROUPED_MEMB_INC  = NO\n\n# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include\n# files with double quotes in the documentation rather than with sharp brackets.\n# The default value is: NO.\n\nFORCE_LOCAL_INCLUDES   = NO\n\n# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the\n# documentation for inline members.\n# The default value is: YES.\n\nINLINE_INFO            = YES\n\n# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the\n# (detailed) documentation of file and class members alphabetically by member\n# name. If set to NO the members will appear in declaration order.\n# The default value is: YES.\n\nSORT_MEMBER_DOCS       = YES\n\n# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief\n# descriptions of file, namespace and class members alphabetically by member\n# name. If set to NO the members will appear in declaration order. Note that\n# this will also influence the order of the classes in the class list.\n# The default value is: NO.\n\nSORT_BRIEF_DOCS        = NO\n\n# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the\n# (brief and detailed) documentation of class members so that constructors and\n# destructors are listed first. If set to NO the constructors will appear in the\n# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS.\n# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief\n# member documentation.\n# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting\n# detailed member documentation.\n# The default value is: NO.\n\nSORT_MEMBERS_CTORS_1ST = NO\n\n# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy\n# of group names into alphabetical order. If set to NO the group names will\n# appear in their defined order.\n# The default value is: NO.\n\nSORT_GROUP_NAMES       = NO\n\n# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by\n# fully-qualified names, including namespaces. If set to NO, the class list will\n# be sorted only by class name, not including the namespace part.\n# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.\n# Note: This option applies only to the class list, not to the alphabetical\n# list.\n# The default value is: NO.\n\nSORT_BY_SCOPE_NAME     = NO\n\n# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper\n# type resolution of all parameters of a function it will reject a match between\n# the prototype and the implementation of a member function even if there is\n# only one candidate or it is obvious which candidate to choose by doing a\n# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still\n# accept a match between prototype and implementation in such cases.\n# The default value is: NO.\n\nSTRICT_PROTO_MATCHING  = NO\n\n# The GENERATE_TODOLIST tag can be used to enable ( YES) or disable ( NO) the\n# todo list. This list is created by putting \\todo commands in the\n# documentation.\n# The default value is: YES.\n\nGENERATE_TODOLIST      = YES\n\n# The GENERATE_TESTLIST tag can be used to enable ( YES) or disable ( NO) the\n# test list. This list is created by putting \\test commands in the\n# documentation.\n# The default value is: YES.\n\nGENERATE_TESTLIST      = YES\n\n# The GENERATE_BUGLIST tag can be used to enable ( YES) or disable ( NO) the bug\n# list. This list is created by putting \\bug commands in the documentation.\n# The default value is: YES.\n\nGENERATE_BUGLIST       = YES\n\n# The GENERATE_DEPRECATEDLIST tag can be used to enable ( YES) or disable ( NO)\n# the deprecated list. This list is created by putting \\deprecated commands in\n# the documentation.\n# The default value is: YES.\n\nGENERATE_DEPRECATEDLIST= YES\n\n# The ENABLED_SECTIONS tag can be used to enable conditional documentation\n# sections, marked by \\if <section_label> ... \\endif and \\cond <section_label>\n# ... \\endcond blocks.\n\nENABLED_SECTIONS       =\n\n# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the\n# initial value of a variable or macro / define can have for it to appear in the\n# documentation. If the initializer consists of more lines than specified here\n# it will be hidden. Use a value of 0 to hide initializers completely. The\n# appearance of the value of individual variables and macros / defines can be\n# controlled using \\showinitializer or \\hideinitializer command in the\n# documentation regardless of this setting.\n# Minimum value: 0, maximum value: 10000, default value: 30.\n\nMAX_INITIALIZER_LINES  = 30\n\n# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at\n# the bottom of the documentation of classes and structs. If set to YES the list\n# will mention the files that were used to generate the documentation.\n# The default value is: YES.\n\nSHOW_USED_FILES        = YES\n\n# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This\n# will remove the Files entry from the Quick Index and from the Folder Tree View\n# (if specified).\n# The default value is: YES.\n\nSHOW_FILES             = YES\n\n# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces\n# page. This will remove the Namespaces entry from the Quick Index and from the\n# Folder Tree View (if specified).\n# The default value is: YES.\n\nSHOW_NAMESPACES        = YES\n\n# The FILE_VERSION_FILTER tag can be used to specify a program or script that\n# doxygen should invoke to get the current version for each file (typically from\n# the version control system). Doxygen will invoke the program by executing (via\n# popen()) the command command input-file, where command is the value of the\n# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided\n# by doxygen. Whatever the program writes to standard output is used as the file\n# version. For an example see the documentation.\n\nFILE_VERSION_FILTER    =\n\n# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed\n# by doxygen. The layout file controls the global structure of the generated\n# output files in an output format independent way. To create the layout file\n# that represents doxygen's defaults, run doxygen with the -l option. You can\n# optionally specify a file name after the option, if omitted DoxygenLayout.xml\n# will be used as the name of the layout file.\n#\n# Note that if you run doxygen from a directory containing a file called\n# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE\n# tag is left empty.\n\nLAYOUT_FILE            =\n\n# The CITE_BIB_FILES tag can be used to specify one or more bib files containing\n# the reference definitions. This must be a list of .bib files. The .bib\n# extension is automatically appended if omitted. This requires the bibtex tool\n# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info.\n# For LaTeX the style of the bibliography can be controlled using\n# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the\n# search path. See also \\cite for info how to create references.\n\nCITE_BIB_FILES         =\n\n#---------------------------------------------------------------------------\n# Configuration options related to warning and progress messages\n#---------------------------------------------------------------------------\n\n# The QUIET tag can be used to turn on/off the messages that are generated to\n# standard output by doxygen. If QUIET is set to YES this implies that the\n# messages are off.\n# The default value is: NO.\n\nQUIET                  = NO\n\n# The WARNINGS tag can be used to turn on/off the warning messages that are\n# generated to standard error ( stderr) by doxygen. If WARNINGS is set to YES\n# this implies that the warnings are on.\n#\n# Tip: Turn warnings on while writing the documentation.\n# The default value is: YES.\n\nWARNINGS               = YES\n\n# If the WARN_IF_UNDOCUMENTED tag is set to YES, then doxygen will generate\n# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag\n# will automatically be disabled.\n# The default value is: YES.\n\nWARN_IF_UNDOCUMENTED   = YES\n\n# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for\n# potential errors in the documentation, such as not documenting some parameters\n# in a documented function, or documenting parameters that don't exist or using\n# markup commands wrongly.\n# The default value is: YES.\n\nWARN_IF_DOC_ERROR      = YES\n\n# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that\n# are documented, but have no documentation for their parameters or return\n# value. If set to NO doxygen will only warn about wrong or incomplete parameter\n# documentation, but not about the absence of documentation.\n# The default value is: NO.\n\nWARN_NO_PARAMDOC       = YES\n\n# The WARN_FORMAT tag determines the format of the warning messages that doxygen\n# can produce. The string should contain the $file, $line, and $text tags, which\n# will be replaced by the file and line number from which the warning originated\n# and the warning text. Optionally the format may contain $version, which will\n# be replaced by the version of the file (if it could be obtained via\n# FILE_VERSION_FILTER)\n# The default value is: $file:$line: $text.\n\nWARN_FORMAT            = \"$file:$line: $text\"\n\n# The WARN_LOGFILE tag can be used to specify a file to which warning and error\n# messages should be written. If left blank the output is written to standard\n# error (stderr).\n\nWARN_LOGFILE           =\n\n#---------------------------------------------------------------------------\n# Configuration options related to the input files\n#---------------------------------------------------------------------------\n\n# The INPUT tag is used to specify the files and/or directories that contain\n# documented source files. You may enter file names like myfile.cpp or\n# directories like /usr/src/myproject. Separate the files or directories with\n# spaces.\n# Note: If this tag is empty the current directory is searched.\n\nINPUT                  = ../../include ../../src/common ../../cpp-package/include/mxnet-cpp\n\n# This tag can be used to specify the character encoding of the source files\n# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses\n# libiconv (or the iconv built into libc) for the transcoding. See the libiconv\n# documentation (see: http://www.gnu.org/software/libiconv) for the list of\n# possible encodings.\n# The default value is: UTF-8.\n\nINPUT_ENCODING         = UTF-8\n\n# If the value of the INPUT tag contains directories, you can use the\n# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and\n# *.h) to filter out the source-files in the directories. If left blank the\n# following patterns are tested:*.c, *.cc, *.cxx, *.cpp, *.c++, *.java, *.ii,\n# *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, *.hh, *.hxx, *.hpp,\n# *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, *.m, *.markdown,\n# *.md, *.mm, *.dox, *.py, *.f90, *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf,\n# *.qsf, *.as and *.js.\n\nFILE_PATTERNS          = *.h\n\n# The RECURSIVE tag can be used to specify whether or not subdirectories should\n# be searched for input files as well.\n# The default value is: NO.\n\nRECURSIVE              = YES\n\n# The EXCLUDE tag can be used to specify files and/or directories that should be\n# excluded from the INPUT source files. This way you can easily exclude a\n# subdirectory from a directory tree whose root is specified with the INPUT tag.\n#\n# Note that relative paths are relative to the directory from which doxygen is\n# run.\n\nEXCLUDE                = 3rdparty\n\n# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or\n# directories that are symbolic links (a Unix file system feature) are excluded\n# from the input.\n# The default value is: NO.\n\nEXCLUDE_SYMLINKS       = NO\n\n# If the value of the INPUT tag contains directories, you can use the\n# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude\n# certain files from those directories.\n#\n# Note that the wildcards are matched against the file with absolute path, so to\n# exclude all test directories for example use the pattern */test/*\n\nEXCLUDE_PATTERNS       = */test/* \\\n                         logging.h\n\n# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names\n# (namespaces, classes, functions, etc.) that should be excluded from the\n# output. The symbol name can be a fully qualified name, a word, or if the\n# wildcard * is used, a substring. Examples: ANamespace, AClass,\n# AClass::ANamespace, ANamespace::*Test\n#\n# Note that the wildcards are matched against the file with absolute path, so to\n# exclude all test directories use the pattern */test/*\n\nEXCLUDE_SYMBOLS        =\n\n# The EXAMPLE_PATH tag can be used to specify one or more files or directories\n# that contain example code fragments that are included (see the \\include\n# command).\n\nEXAMPLE_PATH           =\n\n# If the value of the EXAMPLE_PATH tag contains directories, you can use the\n# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and\n# *.h) to filter out the source-files in the directories. If left blank all\n# files are included.\n\nEXAMPLE_PATTERNS       =\n\n# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be\n# searched for input files to be used with the \\include or \\dontinclude commands\n# irrespective of the value of the RECURSIVE tag.\n# The default value is: NO.\n\nEXAMPLE_RECURSIVE      = NO\n\n# The IMAGE_PATH tag can be used to specify one or more files or directories\n# that contain images that are to be included in the documentation (see the\n# \\image command).\n\nIMAGE_PATH             =\n\n# The INPUT_FILTER tag can be used to specify a program that doxygen should\n# invoke to filter for each input file. Doxygen will invoke the filter program\n# by executing (via popen()) the command:\n#\n# <filter> <input-file>\n#\n# where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the\n# name of an input file. Doxygen will then use the output that the filter\n# program writes to standard output. If FILTER_PATTERNS is specified, this tag\n# will be ignored.\n#\n# Note that the filter must not add or remove lines; it is applied before the\n# code is scanned, but not when the output code is generated. If lines are added\n# or removed, the anchors will not be placed correctly.\n\nINPUT_FILTER           =\n\n# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern\n# basis. Doxygen will compare the file name with each pattern and apply the\n# filter if there is a match. The filters are a list of the form: pattern=filter\n# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how\n# filters are used. If the FILTER_PATTERNS tag is empty or if none of the\n# patterns match the file name, INPUT_FILTER is applied.\n\nFILTER_PATTERNS        =\n\n# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using\n# INPUT_FILTER ) will also be used to filter the input files that are used for\n# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES).\n# The default value is: NO.\n\nFILTER_SOURCE_FILES    = NO\n\n# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file\n# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and\n# it is also possible to disable source filtering for a specific pattern using\n# *.ext= (so without naming a filter).\n# This tag requires that the tag FILTER_SOURCE_FILES is set to YES.\n\nFILTER_SOURCE_PATTERNS =\n\n# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that\n# is part of the input, its contents will be placed on the main page\n# (index.html). This can be useful if you have a project on for instance GitHub\n# and want to reuse the introduction page also for the doxygen output.\n\n#USE_MDFILE_AS_MAINPAGE =\n\n#---------------------------------------------------------------------------\n# Configuration options related to source browsing\n#---------------------------------------------------------------------------\n\n# If the SOURCE_BROWSER tag is set to YES then a list of source files will be\n# generated. Documented entities will be cross-referenced with these sources.\n#\n# Note: To get rid of all source code in the generated output, make sure that\n# also VERBATIM_HEADERS is set to NO.\n# The default value is: NO.\n\nSOURCE_BROWSER         = NO\n\n# Setting the INLINE_SOURCES tag to YES will include the body of functions,\n# classes and enums directly into the documentation.\n# The default value is: NO.\n\nINLINE_SOURCES         = NO\n\n# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any\n# special comment blocks from generated source code fragments. Normal C, C++ and\n# Fortran comments will always remain visible.\n# The default value is: YES.\n\nSTRIP_CODE_COMMENTS    = YES\n\n# If the REFERENCED_BY_RELATION tag is set to YES then for each documented\n# function all documented functions referencing it will be listed.\n# The default value is: NO.\n\nREFERENCED_BY_RELATION = NO\n\n# If the REFERENCES_RELATION tag is set to YES then for each documented function\n# all documented entities called/used by that function will be listed.\n# The default value is: NO.\n\nREFERENCES_RELATION    = NO\n\n# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set\n# to YES, then the hyperlinks from functions in REFERENCES_RELATION and\n# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will\n# link to the documentation.\n# The default value is: YES.\n\nREFERENCES_LINK_SOURCE = YES\n\n# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the\n# source code will show a tooltip with additional information such as prototype,\n# brief description and links to the definition and documentation. Since this\n# will make the HTML file larger and loading of large files a bit slower, you\n# can opt to disable this feature.\n# The default value is: YES.\n# This tag requires that the tag SOURCE_BROWSER is set to YES.\n\n#SOURCE_TOOLTIPS        = YES\n\n# If the USE_HTAGS tag is set to YES then the references to source code will\n# point to the HTML generated by the htags(1) tool instead of doxygen built-in\n# source browser. The htags tool is part of GNU's global source tagging system\n# (see http://www.gnu.org/software/global/global.html). You will need version\n# 4.8.6 or higher.\n#\n# To use it do the following:\n# - Install the latest version of global\n# - Enable SOURCE_BROWSER and USE_HTAGS in the config file\n# - Make sure the INPUT points to the root of the source tree\n# - Run doxygen as normal\n#\n# Doxygen will invoke htags (and that will in turn invoke gtags), so these\n# tools must be available from the command line (i.e. in the search path).\n#\n# The result: instead of the source browser generated by doxygen, the links to\n# source code will now point to the output of htags.\n# The default value is: NO.\n# This tag requires that the tag SOURCE_BROWSER is set to YES.\n\nUSE_HTAGS              = NO\n\n# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a\n# verbatim copy of the header file for each class for which an include is\n# specified. Set to NO to disable this.\n# See also: Section \\class.\n# The default value is: YES.\n\nVERBATIM_HEADERS       = YES\n\n# If the CLANG_ASSISTED_PARSING tag is set to YES, then doxygen will use the\n# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the\n# cost of reduced performance. This can be particularly helpful with template\n# rich C++ code for which doxygen's built-in parser lacks the necessary type\n# information.\n# Note: The availability of this option depends on whether or not doxygen was\n# compiled with the --with-libclang option.\n# The default value is: NO.\n\n#CLANG_ASSISTED_PARSING = NO\n\n# If clang assisted parsing is enabled you can provide the compiler with command\n# line options that you would normally use when invoking the compiler. Note that\n# the include paths will already be set by doxygen for the files and directories\n# specified with INPUT and INCLUDE_PATH.\n# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES.\n\n#CLANG_OPTIONS          =\n\n#---------------------------------------------------------------------------\n# Configuration options related to the alphabetical class index\n#---------------------------------------------------------------------------\n\n# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all\n# compounds will be generated. Enable this if the project contains a lot of\n# classes, structs, unions or interfaces.\n# The default value is: YES.\n\nALPHABETICAL_INDEX     = YES\n\n# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in\n# which the alphabetical index list will be split.\n# Minimum value: 1, maximum value: 20, default value: 5.\n# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.\n\nCOLS_IN_ALPHA_INDEX    = 5\n\n# In case all classes in a project start with a common prefix, all classes will\n# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag\n# can be used to specify a prefix (or a list of prefixes) that should be ignored\n# while generating the index headers.\n# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.\n\nIGNORE_PREFIX          =\n\n#---------------------------------------------------------------------------\n# Configuration options related to the HTML output\n#---------------------------------------------------------------------------\n\n# If the GENERATE_HTML tag is set to YES doxygen will generate HTML output\n# The default value is: YES.\n\nGENERATE_HTML          = YES\n\n# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a\n# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of\n# it.\n# The default directory is: html.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_OUTPUT            = html\n\n# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each\n# generated HTML page (for example: .htm, .php, .asp).\n# The default value is: .html.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_FILE_EXTENSION    = .html\n\n# The HTML_HEADER tag can be used to specify a user-defined HTML header file for\n# each generated HTML page. If the tag is left blank doxygen will generate a\n# standard header.\n#\n# To get valid HTML the header file that includes any scripts and style sheets\n# that doxygen needs, which is dependent on the configuration options used (e.g.\n# the setting GENERATE_TREEVIEW). It is highly recommended to start with a\n# default header using\n# doxygen -w html new_header.html new_footer.html new_stylesheet.css\n# YourConfigFile\n# and then modify the file new_header.html. See also section \"Doxygen usage\"\n# for information on how to generate the default header that doxygen normally\n# uses.\n# Note: The header is subject to change so you typically have to regenerate the\n# default header when upgrading to a newer version of doxygen. For a description\n# of the possible markers and block names see the documentation.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_HEADER            =\n\n# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each\n# generated HTML page. If the tag is left blank doxygen will generate a standard\n# footer. See HTML_HEADER for more information on how to generate a default\n# footer and what special commands can be used inside the footer. See also\n# section \"Doxygen usage\" for information on how to generate the default footer\n# that doxygen normally uses.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_FOOTER            =\n\n# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style\n# sheet that is used by each HTML page. It can be used to fine-tune the look of\n# the HTML output. If left blank doxygen will generate a default style sheet.\n# See also section \"Doxygen usage\" for information on how to generate the style\n# sheet that doxygen normally uses.\n# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as\n# it is more robust and this tag (HTML_STYLESHEET) will in the future become\n# obsolete.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_STYLESHEET        =\n\n# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined\n# cascading style sheets that are included after the standard style sheets\n# created by doxygen. Using this option one can overrule certain style aspects.\n# This is preferred over using HTML_STYLESHEET since it does not replace the\n# standard style sheet and is therefor more robust against future updates.\n# Doxygen will copy the style sheet files to the output directory.\n# Note: The order of the extra stylesheet files is of importance (e.g. the last\n# stylesheet in the list overrules the setting of the previous ones in the\n# list). For an example see the documentation.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\n#HTML_EXTRA_STYLESHEET  =\n\n# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or\n# other source files which should be copied to the HTML output directory. Note\n# that these files will be copied to the base HTML output directory. Use the\n# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these\n# files. In the HTML_STYLESHEET file, use the file name only. Also note that the\n# files will be copied as-is; there are no commands or markers available.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_EXTRA_FILES       =\n\n# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen\n# will adjust the colors in the stylesheet and background images according to\n# this color. Hue is specified as an angle on a colorwheel, see\n# http://en.wikipedia.org/wiki/Hue for more information. For instance the value\n# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300\n# purple, and 360 is red again.\n# Minimum value: 0, maximum value: 359, default value: 220.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_COLORSTYLE_HUE    = 220\n\n# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors\n# in the HTML output. For a value of 0 the output will use grayscales only. A\n# value of 255 will produce the most vivid colors.\n# Minimum value: 0, maximum value: 255, default value: 100.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_COLORSTYLE_SAT    = 100\n\n# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the\n# luminance component of the colors in the HTML output. Values below 100\n# gradually make the output lighter, whereas values above 100 make the output\n# darker. The value divided by 100 is the actual gamma applied, so 80 represents\n# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not\n# change the gamma.\n# Minimum value: 40, maximum value: 240, default value: 80.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_COLORSTYLE_GAMMA  = 80\n\n# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML\n# page will contain the date and time when the page was generated. Setting this\n# to NO can help when comparing the output of multiple runs.\n# The default value is: YES.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_TIMESTAMP         = YES\n\n# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML\n# documentation will contain sections that can be hidden and shown after the\n# page has loaded.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nHTML_DYNAMIC_SECTIONS  = NO\n\n# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries\n# shown in the various tree structured indices initially; the user can expand\n# and collapse entries dynamically later on. Doxygen will expand the tree to\n# such a level that at most the specified number of entries are visible (unless\n# a fully collapsed tree already exceeds this amount). So setting the number of\n# entries 1 will produce a full collapsed tree by default. 0 is a special value\n# representing an infinite number of entries and will result in a full expanded\n# tree by default.\n# Minimum value: 0, maximum value: 9999, default value: 100.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\n#HTML_INDEX_NUM_ENTRIES = 100\n\n# If the GENERATE_DOCSET tag is set to YES, additional index files will be\n# generated that can be used as input for Apple's Xcode 3 integrated development\n# environment (see: http://developer.apple.com/tools/xcode/), introduced with\n# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a\n# Makefile in the HTML output directory. Running make will produce the docset in\n# that directory and running make install will install the docset in\n# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at\n# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html\n# for more information.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nGENERATE_DOCSET        = NO\n\n# This tag determines the name of the docset feed. A documentation feed provides\n# an umbrella under which multiple documentation sets from a single provider\n# (such as a company or product suite) can be grouped.\n# The default value is: Doxygen generated docs.\n# This tag requires that the tag GENERATE_DOCSET is set to YES.\n\nDOCSET_FEEDNAME        = \"Doxygen generated docs\"\n\n# This tag specifies a string that should uniquely identify the documentation\n# set bundle. This should be a reverse domain-name style string, e.g.\n# com.mycompany.MyDocSet. Doxygen will append .docset to the name.\n# The default value is: org.doxygen.Project.\n# This tag requires that the tag GENERATE_DOCSET is set to YES.\n\nDOCSET_BUNDLE_ID       = org.doxygen.Project\n\n# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify\n# the documentation publisher. This should be a reverse domain-name style\n# string, e.g. com.mycompany.MyDocSet.documentation.\n# The default value is: org.doxygen.Publisher.\n# This tag requires that the tag GENERATE_DOCSET is set to YES.\n\nDOCSET_PUBLISHER_ID    = org.doxygen.Publisher\n\n# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher.\n# The default value is: Publisher.\n# This tag requires that the tag GENERATE_DOCSET is set to YES.\n\nDOCSET_PUBLISHER_NAME  = Publisher\n\n# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three\n# additional HTML index files: index.hhp, index.hhc, and index.hhk. The\n# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop\n# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on\n# Windows.\n#\n# The HTML Help Workshop contains a compiler that can convert all HTML output\n# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML\n# files are now used as the Windows 98 help format, and will replace the old\n# Windows help format (.hlp) on all Windows platforms in the future. Compressed\n# HTML files also contain an index, a table of contents, and you can search for\n# words in the documentation. The HTML workshop also contains a viewer for\n# compressed HTML files.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nGENERATE_HTMLHELP      = NO\n\n# The CHM_FILE tag can be used to specify the file name of the resulting .chm\n# file. You can add a path in front of the file if the result should not be\n# written to the html output directory.\n# This tag requires that the tag GENERATE_HTMLHELP is set to YES.\n\nCHM_FILE               =\n\n# The HHC_LOCATION tag can be used to specify the location (absolute path\n# including file name) of the HTML help compiler ( hhc.exe). If non-empty\n# doxygen will try to run the HTML help compiler on the generated index.hhp.\n# The file has to be specified with full path.\n# This tag requires that the tag GENERATE_HTMLHELP is set to YES.\n\nHHC_LOCATION           =\n\n# The GENERATE_CHI flag controls if a separate .chi index file is generated (\n# YES) or that it should be included in the master .chm file ( NO).\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTMLHELP is set to YES.\n\nGENERATE_CHI           = NO\n\n# The CHM_INDEX_ENCODING is used to encode HtmlHelp index ( hhk), content ( hhc)\n# and project file content.\n# This tag requires that the tag GENERATE_HTMLHELP is set to YES.\n\nCHM_INDEX_ENCODING     =\n\n# The BINARY_TOC flag controls whether a binary table of contents is generated (\n# YES) or a normal table of contents ( NO) in the .chm file. Furthermore it\n# enables the Previous and Next buttons.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTMLHELP is set to YES.\n\nBINARY_TOC             = NO\n\n# The TOC_EXPAND flag can be set to YES to add extra items for group members to\n# the table of contents of the HTML help documentation and to the tree view.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTMLHELP is set to YES.\n\nTOC_EXPAND             = NO\n\n# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and\n# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that\n# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help\n# (.qch) of the generated HTML documentation.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nGENERATE_QHP           = NO\n\n# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify\n# the file name of the resulting .qch file. The path specified is relative to\n# the HTML output folder.\n# This tag requires that the tag GENERATE_QHP is set to YES.\n\nQCH_FILE               =\n\n# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help\n# Project output. For more information please see Qt Help Project / Namespace\n# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace).\n# The default value is: org.doxygen.Project.\n# This tag requires that the tag GENERATE_QHP is set to YES.\n\nQHP_NAMESPACE          = org.doxygen.Project\n\n# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt\n# Help Project output. For more information please see Qt Help Project / Virtual\n# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual-\n# folders).\n# The default value is: doc.\n# This tag requires that the tag GENERATE_QHP is set to YES.\n\nQHP_VIRTUAL_FOLDER     = doc\n\n# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom\n# filter to add. For more information please see Qt Help Project / Custom\n# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-\n# filters).\n# This tag requires that the tag GENERATE_QHP is set to YES.\n\nQHP_CUST_FILTER_NAME   =\n\n# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the\n# custom filter to add. For more information please see Qt Help Project / Custom\n# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-\n# filters).\n# This tag requires that the tag GENERATE_QHP is set to YES.\n\nQHP_CUST_FILTER_ATTRS  =\n\n# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this\n# project's filter section matches. Qt Help Project / Filter Attributes (see:\n# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes).\n# This tag requires that the tag GENERATE_QHP is set to YES.\n\nQHP_SECT_FILTER_ATTRS  =\n\n# The QHG_LOCATION tag can be used to specify the location of Qt's\n# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the\n# generated .qhp file.\n# This tag requires that the tag GENERATE_QHP is set to YES.\n\nQHG_LOCATION           =\n\n# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be\n# generated, together with the HTML files, they form an Eclipse help plugin. To\n# install this plugin and make it available under the help contents menu in\n# Eclipse, the contents of the directory containing the HTML and XML files needs\n# to be copied into the plugins directory of eclipse. The name of the directory\n# within the plugins directory should be the same as the ECLIPSE_DOC_ID value.\n# After copying Eclipse needs to be restarted before the help appears.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nGENERATE_ECLIPSEHELP   = NO\n\n# A unique identifier for the Eclipse help plugin. When installing the plugin\n# the directory name containing the HTML and XML files should also have this\n# name. Each documentation set should have its own identifier.\n# The default value is: org.doxygen.Project.\n# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES.\n\nECLIPSE_DOC_ID         = org.doxygen.Project\n\n# If you want full control over the layout of the generated HTML pages it might\n# be necessary to disable the index and replace it with your own. The\n# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top\n# of each HTML page. A value of NO enables the index and the value YES disables\n# it. Since the tabs in the index contain the same information as the navigation\n# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nDISABLE_INDEX          = NO\n\n# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index\n# structure should be generated to display hierarchical information. If the tag\n# value is set to YES, a side panel will be generated containing a tree-like\n# index structure (just like the one that is generated for HTML Help). For this\n# to work a browser that supports JavaScript, DHTML, CSS and frames is required\n# (i.e. any modern browser). Windows users are probably better off using the\n# HTML help feature. Via custom stylesheets (see HTML_EXTRA_STYLESHEET) one can\n# further fine-tune the look of the index. As an example, the default style\n# sheet generated by doxygen has an example that shows how to put an image at\n# the root of the tree instead of the PROJECT_NAME. Since the tree basically has\n# the same information as the tab index, you could consider setting\n# DISABLE_INDEX to YES when enabling this option.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nGENERATE_TREEVIEW      = NO\n\n# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that\n# doxygen will group on one line in the generated HTML documentation.\n#\n# Note that a value of 0 will completely suppress the enum values from appearing\n# in the overview section.\n# Minimum value: 0, maximum value: 20, default value: 4.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nENUM_VALUES_PER_LINE   = 4\n\n# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used\n# to set the initial width (in pixels) of the frame in which the tree is shown.\n# Minimum value: 0, maximum value: 1500, default value: 250.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nTREEVIEW_WIDTH         = 250\n\n# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open links to\n# external symbols imported via tag files in a separate window.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nEXT_LINKS_IN_WINDOW    = NO\n\n# Use this tag to change the font size of LaTeX formulas included as images in\n# the HTML documentation. When you change the font size after a successful\n# doxygen run you need to manually remove any form_*.png images from the HTML\n# output directory to force them to be regenerated.\n# Minimum value: 8, maximum value: 50, default value: 10.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nFORMULA_FONTSIZE       = 10\n\n# Use the FORMULA_TRANPARENT tag to determine whether or not the images\n# generated for formulas are transparent PNGs. Transparent PNGs are not\n# supported properly for IE 6.0, but are supported on all modern browsers.\n#\n# Note that when changing this option you need to delete any form_*.png files in\n# the HTML output directory before the changes have effect.\n# The default value is: YES.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nFORMULA_TRANSPARENT    = YES\n\n# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see\n# http://www.mathjax.org) which uses client side Javascript for the rendering\n# instead of using prerendered bitmaps. Use this if you do not have LaTeX\n# installed or if you want to formulas look prettier in the HTML output. When\n# enabled you may also need to install MathJax separately and configure the path\n# to it using the MATHJAX_RELPATH option.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nUSE_MATHJAX            = NO\n\n# When MathJax is enabled you can set the default output format to be used for\n# the MathJax output. See the MathJax site (see:\n# http://docs.mathjax.org/en/latest/output.html) for more details.\n# Possible values are: HTML-CSS (which is slower, but has the best\n# compatibility), NativeMML (i.e. MathML) and SVG.\n# The default value is: HTML-CSS.\n# This tag requires that the tag USE_MATHJAX is set to YES.\n\n#MATHJAX_FORMAT         = HTML-CSS\n\n# When MathJax is enabled you need to specify the location relative to the HTML\n# output directory using the MATHJAX_RELPATH option. The destination directory\n# should contain the MathJax.js script. For instance, if the mathjax directory\n# is located at the same level as the HTML output directory, then\n# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax\n# Content Delivery Network so you can quickly see the result without installing\n# MathJax. However, it is strongly recommended to install a local copy of\n# MathJax from http://www.mathjax.org before deployment.\n# The default value is: http://cdn.mathjax.org/mathjax/latest.\n# This tag requires that the tag USE_MATHJAX is set to YES.\n\nMATHJAX_RELPATH        = http://www.mathjax.org/mathjax\n\n# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax\n# extension names that should be enabled during MathJax rendering. For example\n# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols\n# This tag requires that the tag USE_MATHJAX is set to YES.\n\nMATHJAX_EXTENSIONS     =\n\n# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces\n# of code that will be used on startup of the MathJax code. See the MathJax site\n# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an\n# example see the documentation.\n# This tag requires that the tag USE_MATHJAX is set to YES.\n\n#MATHJAX_CODEFILE       =\n\n# When the SEARCHENGINE tag is enabled doxygen will generate a search box for\n# the HTML output. The underlying search engine uses javascript and DHTML and\n# should work on any modern browser. Note that when using HTML help\n# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET)\n# there is already a search function so this one should typically be disabled.\n# For large projects the javascript based search engine can be slow, then\n# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to\n# search using the keyboard; to jump to the search box use <access key> + S\n# (what the <access key> is depends on the OS and browser, but it is typically\n# <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down\n# key> to jump into the search results window, the results can be navigated\n# using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel\n# the search. The filter options can be selected when the cursor is inside the\n# search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys>\n# to select a filter and <Enter> or <escape> to activate or cancel the filter\n# option.\n# The default value is: YES.\n# This tag requires that the tag GENERATE_HTML is set to YES.\n\nSEARCHENGINE           = YES\n\n# When the SERVER_BASED_SEARCH tag is enabled the search engine will be\n# implemented using a web server instead of a web client using Javascript. There\n# are two flavors of web server based searching depending on the EXTERNAL_SEARCH\n# setting. When disabled, doxygen will generate a PHP script for searching and\n# an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing\n# and searching needs to be provided by external tools. See the section\n# \"External Indexing and Searching\" for details.\n# The default value is: NO.\n# This tag requires that the tag SEARCHENGINE is set to YES.\n\nSERVER_BASED_SEARCH    = NO\n\n# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP\n# script for searching. Instead the search results are written to an XML file\n# which needs to be processed by an external indexer. Doxygen will invoke an\n# external search engine pointed to by the SEARCHENGINE_URL option to obtain the\n# search results.\n#\n# Doxygen ships with an example indexer ( doxyindexer) and search engine\n# (doxysearch.cgi) which are based on the open source search engine library\n# Xapian (see: http://xapian.org/).\n#\n# See the section \"External Indexing and Searching\" for details.\n# The default value is: NO.\n# This tag requires that the tag SEARCHENGINE is set to YES.\n\n#EXTERNAL_SEARCH        = NO\n\n# The SEARCHENGINE_URL should point to a search engine hosted by a web server\n# which will return the search results when EXTERNAL_SEARCH is enabled.\n#\n# Doxygen ships with an example indexer ( doxyindexer) and search engine\n# (doxysearch.cgi) which are based on the open source search engine library\n# Xapian (see: http://xapian.org/). See the section \"External Indexing and\n# Searching\" for details.\n# This tag requires that the tag SEARCHENGINE is set to YES.\n\n#SEARCHENGINE_URL       =\n\n# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed\n# search data is written to a file for indexing by an external tool. With the\n# SEARCHDATA_FILE tag the name of this file can be specified.\n# The default file is: searchdata.xml.\n# This tag requires that the tag SEARCHENGINE is set to YES.\n\n#SEARCHDATA_FILE        = searchdata.xml\n\n# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the\n# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is\n# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple\n# projects and redirect the results back to the right project.\n# This tag requires that the tag SEARCHENGINE is set to YES.\n\n#EXTERNAL_SEARCH_ID     =\n\n# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen\n# projects other than the one defined by this configuration file, but that are\n# all added to the same external search index. Each project needs to have a\n# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of\n# to a relative location where the documentation can be found. The format is:\n# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ...\n# This tag requires that the tag SEARCHENGINE is set to YES.\n\n#EXTRA_SEARCH_MAPPINGS  =\n\n#---------------------------------------------------------------------------\n# Configuration options related to the LaTeX output\n#---------------------------------------------------------------------------\n\n# If the GENERATE_LATEX tag is set to YES doxygen will generate LaTeX output.\n# The default value is: YES.\n\nGENERATE_LATEX         = YES\n\n# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a\n# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of\n# it.\n# The default directory is: latex.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_OUTPUT           = latex\n\n# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be\n# invoked.\n#\n# Note that when enabling USE_PDFLATEX this option is only used for generating\n# bitmaps for formulas in the HTML output, but not in the Makefile that is\n# written to the output directory.\n# The default file is: latex.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_CMD_NAME         = latex\n\n# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate\n# index for LaTeX.\n# The default file is: makeindex.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nMAKEINDEX_CMD_NAME     = makeindex\n\n# If the COMPACT_LATEX tag is set to YES doxygen generates more compact LaTeX\n# documents. This may be useful for small projects and may help to save some\n# trees in general.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nCOMPACT_LATEX          = NO\n\n# The PAPER_TYPE tag can be used to set the paper type that is used by the\n# printer.\n# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x\n# 14 inches) and executive (7.25 x 10.5 inches).\n# The default value is: a4.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nPAPER_TYPE             = a4\n\n# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names\n# that should be included in the LaTeX output. To get the times font for\n# instance you can specify\n# EXTRA_PACKAGES=times\n# If left blank no extra packages will be included.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nEXTRA_PACKAGES         =\n\n# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the\n# generated LaTeX document. The header should contain everything until the first\n# chapter. If it is left blank doxygen will generate a standard header. See\n# section \"Doxygen usage\" for information on how to let doxygen write the\n# default header to a separate file.\n#\n# Note: Only use a user-defined header if you know what you are doing! The\n# following commands have a special meaning inside the header: $title,\n# $datetime, $date, $doxygenversion, $projectname, $projectnumber,\n# $projectbrief, $projectlogo. Doxygen will replace $title with the empy string,\n# for the replacement values of the other commands the user is refered to\n# HTML_HEADER.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_HEADER           =\n\n# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the\n# generated LaTeX document. The footer should contain everything after the last\n# chapter. If it is left blank doxygen will generate a standard footer. See\n# LATEX_HEADER for more information on how to generate a default footer and what\n# special commands can be used inside the footer.\n#\n# Note: Only use a user-defined footer if you know what you are doing!\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_FOOTER           =\n\n# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or\n# other source files which should be copied to the LATEX_OUTPUT output\n# directory. Note that the files will be copied as-is; there are no commands or\n# markers available.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\n#LATEX_EXTRA_FILES      =\n\n# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is\n# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will\n# contain links (just like the HTML output) instead of page references. This\n# makes the output suitable for online browsing using a PDF viewer.\n# The default value is: YES.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nPDF_HYPERLINKS         = YES\n\n# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate\n# the PDF file directly from the LaTeX files. Set this option to YES to get a\n# higher quality PDF documentation.\n# The default value is: YES.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nUSE_PDFLATEX           = YES\n\n# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode\n# command to the generated LaTeX files. This will instruct LaTeX to keep running\n# if errors occur, instead of asking the user for help. This option is also used\n# when generating formulas in HTML.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_BATCHMODE        = NO\n\n# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the\n# index chapters (such as File Index, Compound Index, etc.) in the output.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_HIDE_INDICES     = NO\n\n# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source\n# code with syntax highlighting in the LaTeX output.\n#\n# Note that which sources are shown also depends on other settings such as\n# SOURCE_BROWSER.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_SOURCE_CODE      = NO\n\n# The LATEX_BIB_STYLE tag can be used to specify the style to use for the\n# bibliography, e.g. plainnat, or ieeetr. See\n# http://en.wikipedia.org/wiki/BibTeX and \\cite for more info.\n# The default value is: plain.\n# This tag requires that the tag GENERATE_LATEX is set to YES.\n\nLATEX_BIB_STYLE        = plain\n\n#---------------------------------------------------------------------------\n# Configuration options related to the RTF output\n#---------------------------------------------------------------------------\n\n# If the GENERATE_RTF tag is set to YES doxygen will generate RTF output. The\n# RTF output is optimized for Word 97 and may not look too pretty with other RTF\n# readers/editors.\n# The default value is: NO.\n\nGENERATE_RTF           = NO\n\n# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a\n# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of\n# it.\n# The default directory is: rtf.\n# This tag requires that the tag GENERATE_RTF is set to YES.\n\nRTF_OUTPUT             = rtf\n\n# If the COMPACT_RTF tag is set to YES doxygen generates more compact RTF\n# documents. This may be useful for small projects and may help to save some\n# trees in general.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_RTF is set to YES.\n\nCOMPACT_RTF            = NO\n\n# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will\n# contain hyperlink fields. The RTF file will contain links (just like the HTML\n# output) instead of page references. This makes the output suitable for online\n# browsing using Word or some other Word compatible readers that support those\n# fields.\n#\n# Note: WordPad (write) and others do not support links.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_RTF is set to YES.\n\nRTF_HYPERLINKS         = NO\n\n# Load stylesheet definitions from file. Syntax is similar to doxygen's config\n# file, i.e. a series of assignments. You only have to provide replacements,\n# missing definitions are set to their default value.\n#\n# See also section \"Doxygen usage\" for information on how to generate the\n# default style sheet that doxygen normally uses.\n# This tag requires that the tag GENERATE_RTF is set to YES.\n\nRTF_STYLESHEET_FILE    =\n\n# Set optional variables used in the generation of an RTF document. Syntax is\n# similar to doxygen's config file. A template extensions file can be generated\n# using doxygen -e rtf extensionFile.\n# This tag requires that the tag GENERATE_RTF is set to YES.\n\nRTF_EXTENSIONS_FILE    =\n\n#---------------------------------------------------------------------------\n# Configuration options related to the man page output\n#---------------------------------------------------------------------------\n\n# If the GENERATE_MAN tag is set to YES doxygen will generate man pages for\n# classes and files.\n# The default value is: NO.\n\nGENERATE_MAN           = NO\n\n# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a\n# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of\n# it. A directory man3 will be created inside the directory specified by\n# MAN_OUTPUT.\n# The default directory is: man.\n# This tag requires that the tag GENERATE_MAN is set to YES.\n\nMAN_OUTPUT             = man\n\n# The MAN_EXTENSION tag determines the extension that is added to the generated\n# man pages. In case the manual section does not start with a number, the number\n# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is\n# optional.\n# The default value is: .3.\n# This tag requires that the tag GENERATE_MAN is set to YES.\n\nMAN_EXTENSION          = .3\n\n# The MAN_SUBDIR tag determines the name of the directory created within\n# MAN_OUTPUT in which the man pages are placed. If defaults to man followed by\n# MAN_EXTENSION with the initial . removed.\n# This tag requires that the tag GENERATE_MAN is set to YES.\n\n#MAN_SUBDIR             =\n\n# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it\n# will generate one additional man file for each entity documented in the real\n# man page(s). These additional files only source the real man page, but without\n# them the man command would be unable to find the correct page.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_MAN is set to YES.\n\nMAN_LINKS              = NO\n\n#---------------------------------------------------------------------------\n# Configuration options related to the XML output\n#---------------------------------------------------------------------------\n\n# If the GENERATE_XML tag is set to YES doxygen will generate an XML file that\n# captures the structure of the code including all documentation.\n# The default value is: NO.\n\nGENERATE_XML           = YES\n\n# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a\n# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of\n# it.\n# The default directory is: xml.\n# This tag requires that the tag GENERATE_XML is set to YES.\n\nXML_OUTPUT             = xml\n\n# If the XML_PROGRAMLISTING tag is set to YES doxygen will dump the program\n# listings (including syntax highlighting and cross-referencing information) to\n# the XML output. Note that enabling this will significantly increase the size\n# of the XML output.\n# The default value is: YES.\n# This tag requires that the tag GENERATE_XML is set to YES.\n\nXML_PROGRAMLISTING     = YES\n\n#---------------------------------------------------------------------------\n# Configuration options related to the DOCBOOK output\n#---------------------------------------------------------------------------\n\n# If the GENERATE_DOCBOOK tag is set to YES doxygen will generate Docbook files\n# that can be used to generate PDF.\n# The default value is: NO.\n\n#GENERATE_DOCBOOK       = NO\n\n# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put.\n# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in\n# front of it.\n# The default directory is: docbook.\n# This tag requires that the tag GENERATE_DOCBOOK is set to YES.\n\n#DOCBOOK_OUTPUT         = docbook\n\n# If the DOCBOOK_PROGRAMLISTING tag is set to YES doxygen will include the\n# program listings (including syntax highlighting and cross-referencing\n# information) to the DOCBOOK output. Note that enabling this will significantly\n# increase the size of the DOCBOOK output.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_DOCBOOK is set to YES.\n\n#DOCBOOK_PROGRAMLISTING = NO\n\n#---------------------------------------------------------------------------\n# Configuration options for the AutoGen Definitions output\n#---------------------------------------------------------------------------\n\n# If the GENERATE_AUTOGEN_DEF tag is set to YES doxygen will generate an AutoGen\n# Definitions (see http://autogen.sf.net) file that captures the structure of\n# the code including all documentation. Note that this feature is still\n# experimental and incomplete at the moment.\n# The default value is: NO.\n\nGENERATE_AUTOGEN_DEF   = NO\n\n#---------------------------------------------------------------------------\n# Configuration options related to the Perl module output\n#---------------------------------------------------------------------------\n\n# If the GENERATE_PERLMOD tag is set to YES doxygen will generate a Perl module\n# file that captures the structure of the code including all documentation.\n#\n# Note that this feature is still experimental and incomplete at the moment.\n# The default value is: NO.\n\nGENERATE_PERLMOD       = NO\n\n# If the PERLMOD_LATEX tag is set to YES doxygen will generate the necessary\n# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI\n# output from the Perl module output.\n# The default value is: NO.\n# This tag requires that the tag GENERATE_PERLMOD is set to YES.\n\nPERLMOD_LATEX          = NO\n\n# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be nicely\n# formatted so it can be parsed by a human reader. This is useful if you want to\n# understand what is going on. On the other hand, if this tag is set to NO the\n# size of the Perl module output will be much smaller and Perl will parse it\n# just the same.\n# The default value is: YES.\n# This tag requires that the tag GENERATE_PERLMOD is set to YES.\n\nPERLMOD_PRETTY         = YES\n\n# The names of the make variables in the generated doxyrules.make file are\n# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful\n# so different doxyrules.make files included by the same Makefile don't\n# overwrite each other's variables.\n# This tag requires that the tag GENERATE_PERLMOD is set to YES.\n\nPERLMOD_MAKEVAR_PREFIX =\n\n#---------------------------------------------------------------------------\n# Configuration options related to the preprocessor\n#---------------------------------------------------------------------------\n\n# If the ENABLE_PREPROCESSING tag is set to YES doxygen will evaluate all\n# C-preprocessor directives found in the sources and include files.\n# The default value is: YES.\n\nENABLE_PREPROCESSING   = YES\n\n# If the MACRO_EXPANSION tag is set to YES doxygen will expand all macro names\n# in the source code. If set to NO only conditional compilation will be\n# performed. Macro expansion can be done in a controlled way by setting\n# EXPAND_ONLY_PREDEF to YES.\n# The default value is: NO.\n# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.\n\nMACRO_EXPANSION        = NO\n\n# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then\n# the macro expansion is limited to the macros specified with the PREDEFINED and\n# EXPAND_AS_DEFINED tags.\n# The default value is: NO.\n# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.\n\nEXPAND_ONLY_PREDEF     = NO\n\n# If the SEARCH_INCLUDES tag is set to YES the includes files in the\n# INCLUDE_PATH will be searched if a #include is found.\n# The default value is: YES.\n# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.\n\nSEARCH_INCLUDES        = YES\n\n# The INCLUDE_PATH tag can be used to specify one or more directories that\n# contain include files that are not input files but should be processed by the\n# preprocessor.\n# This tag requires that the tag SEARCH_INCLUDES is set to YES.\n\nINCLUDE_PATH           =\n\n# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard\n# patterns (like *.h and *.hpp) to filter out the header-files in the\n# directories. If left blank, the patterns specified with FILE_PATTERNS will be\n# used.\n# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.\n\nINCLUDE_FILE_PATTERNS  =\n\n# The PREDEFINED tag can be used to specify one or more macro names that are\n# defined before the preprocessor is started (similar to the -D option of e.g.\n# gcc). The argument of the tag is a list of macros of the form: name or\n# name=definition (no spaces). If the definition and the \"=\" are omitted, \"=1\"\n# is assumed. To prevent a macro definition from being undefined via #undef or\n# recursively expanded use the := operator instead of the = operator.\n# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.\n\nPREDEFINED             = MXNET_USE_CUDA DMLC_USE_CXX11\n\n# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this\n# tag can be used to specify a list of macro names that should be expanded. The\n# macro definition that is found in the sources will be used. Use the PREDEFINED\n# tag if you want to use a different macro definition that overrules the\n# definition found in the source code.\n# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.\n\nEXPAND_AS_DEFINED      =\n\n# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will\n# remove all references to function-like macros that are alone on a line, have\n# an all uppercase name, and do not end with a semicolon. Such function macros\n# are typically used for boiler-plate code, and will confuse the parser if not\n# removed.\n# The default value is: YES.\n# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.\n\nSKIP_FUNCTION_MACROS   = YES\n\n#---------------------------------------------------------------------------\n# Configuration options related to external references\n#---------------------------------------------------------------------------\n\n# The TAGFILES tag can be used to specify one or more tag files. For each tag\n# file the location of the external documentation should be added. The format of\n# a tag file without this location is as follows:\n# TAGFILES = file1 file2 ...\n# Adding location for the tag files is done as follows:\n# TAGFILES = file1=loc1 \"file2 = loc2\" ...\n# where loc1 and loc2 can be relative or absolute paths or URLs. See the\n# section \"Linking to external documentation\" for more information about the use\n# of tag files.\n# Note: Each tag file must have a unique name (where the name does NOT include\n# the path). If a tag file is not located in the directory in which doxygen is\n# run, you must also specify the path to the tagfile here.\n\nTAGFILES               =\n\n# When a file name is specified after GENERATE_TAGFILE, doxygen will create a\n# tag file that is based on the input files it reads. See section \"Linking to\n# external documentation\" for more information about the usage of tag files.\n\nGENERATE_TAGFILE       =\n\n# If the ALLEXTERNALS tag is set to YES all external class will be listed in the\n# class index. If set to NO only the inherited external classes will be listed.\n# The default value is: NO.\n\nALLEXTERNALS           = NO\n\n# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed in\n# the modules index. If set to NO, only the current project's groups will be\n# listed.\n# The default value is: YES.\n\nEXTERNAL_GROUPS        = YES\n\n# If the EXTERNAL_PAGES tag is set to YES all external pages will be listed in\n# the related pages index. If set to NO, only the current project's pages will\n# be listed.\n# The default value is: YES.\n\n#EXTERNAL_PAGES         = YES\n\n# The PERL_PATH should be the absolute path and name of the perl script\n# interpreter (i.e. the result of 'which perl').\n# The default file (with absolute path) is: /usr/bin/perl.\n\nPERL_PATH              = /usr/bin/perl\n\n#---------------------------------------------------------------------------\n# Configuration options related to the dot tool\n#---------------------------------------------------------------------------\n\n# If the CLASS_DIAGRAMS tag is set to YES doxygen will generate a class diagram\n# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to\n# NO turns the diagrams off. Note that this option also works with HAVE_DOT\n# disabled, but it is recommended to install and use dot, since it yields more\n# powerful graphs.\n# The default value is: YES.\n\nCLASS_DIAGRAMS         = YES\n\n# You can define message sequence charts within doxygen comments using the \\msc\n# command. Doxygen will then run the mscgen tool (see:\n# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the\n# documentation. The MSCGEN_PATH tag allows you to specify the directory where\n# the mscgen tool resides. If left empty the tool is assumed to be found in the\n# default search path.\n\nMSCGEN_PATH            =\n\n# You can include diagrams made with dia in doxygen documentation. Doxygen will\n# then run dia to produce the diagram and insert it in the documentation. The\n# DIA_PATH tag allows you to specify the directory where the dia binary resides.\n# If left empty dia is assumed to be found in the default search path.\n\n#DIA_PATH               =\n\n# If set to YES, the inheritance and collaboration graphs will hide inheritance\n# and usage relations if the target is undocumented or is not a class.\n# The default value is: YES.\n\nHIDE_UNDOC_RELATIONS   = YES\n\n# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is\n# available from the path. This tool is part of Graphviz (see:\n# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent\n# Bell Labs. The other options in this section have no effect if this option is\n# set to NO\n# The default value is: YES.\n\nHAVE_DOT               = YES\n\n# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed\n# to run in parallel. When set to 0 doxygen will base this on the number of\n# processors available in the system. You can set it explicitly to a value\n# larger than 0 to get control over the balance between CPU load and processing\n# speed.\n# Minimum value: 0, maximum value: 32, default value: 0.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_NUM_THREADS        = 0\n\n# When you want a differently looking font in the dot files that doxygen\n# generates you can specify the font name using DOT_FONTNAME. You need to make\n# sure dot is able to find the font, which can be done by putting it in a\n# standard location or by setting the DOTFONTPATH environment variable or by\n# setting DOT_FONTPATH to the directory containing the font.\n# The default value is: Helvetica.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_FONTNAME           = Helvetica\n\n# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of\n# dot graphs.\n# Minimum value: 4, maximum value: 24, default value: 10.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_FONTSIZE           = 10\n\n# By default doxygen will tell dot to use the default font as specified with\n# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set\n# the path where dot can find it using this tag.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_FONTPATH           =\n\n# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for\n# each documented class showing the direct and indirect inheritance relations.\n# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nCLASS_GRAPH            = YES\n\n# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a\n# graph for each documented class showing the direct and indirect implementation\n# dependencies (inheritance, containment, and class references variables) of the\n# class with other documented classes.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nCOLLABORATION_GRAPH    = YES\n\n# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for\n# groups, showing the direct groups dependencies.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nGROUP_GRAPHS           = YES\n\n# If the UML_LOOK tag is set to YES doxygen will generate inheritance and\n# collaboration diagrams in a style similar to the OMG's Unified Modeling\n# Language.\n# The default value is: NO.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nUML_LOOK               = YES\n\n# If the UML_LOOK tag is enabled, the fields and methods are shown inside the\n# class node. If there are many fields or methods and many nodes the graph may\n# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the\n# number of items for each type to make the size more manageable. Set this to 0\n# for no limit. Note that the threshold may be exceeded by 50% before the limit\n# is enforced. So when you set the threshold to 10, up to 15 fields may appear,\n# but if the number exceeds 15, the total amount of fields shown is limited to\n# 10.\n# Minimum value: 0, maximum value: 100, default value: 10.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\n#UML_LIMIT_NUM_FIELDS   = 10\n\n# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and\n# collaboration graphs will show the relations between templates and their\n# instances.\n# The default value is: NO.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nTEMPLATE_RELATIONS     = NO\n\n# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to\n# YES then doxygen will generate a graph for each documented file showing the\n# direct and indirect include dependencies of the file with other documented\n# files.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nINCLUDE_GRAPH          = YES\n\n# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are\n# set to YES then doxygen will generate a graph for each documented file showing\n# the direct and indirect include dependencies of the file with other documented\n# files.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nINCLUDED_BY_GRAPH      = YES\n\n# If the CALL_GRAPH tag is set to YES then doxygen will generate a call\n# dependency graph for every global function or class method.\n#\n# Note that enabling this option will significantly increase the time of a run.\n# So in most cases it will be better to enable call graphs for selected\n# functions only using the \\callgraph command.\n# The default value is: NO.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nCALL_GRAPH             = NO\n\n# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller\n# dependency graph for every global function or class method.\n#\n# Note that enabling this option will significantly increase the time of a run.\n# So in most cases it will be better to enable caller graphs for selected\n# functions only using the \\callergraph command.\n# The default value is: NO.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nCALLER_GRAPH           = NO\n\n# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical\n# hierarchy of all classes instead of a textual one.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nGRAPHICAL_HIERARCHY    = YES\n\n# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the\n# dependencies a directory has on other directories in a graphical way. The\n# dependency relations are determined by the #include relations between the\n# files in the directories.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDIRECTORY_GRAPH        = YES\n\n# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images\n# generated by dot.\n# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order\n# to make the SVG files visible in IE 9+ (other browsers do not have this\n# requirement).\n# Possible values are: png, png:cairo, png:cairo:cairo, png:cairo:gd, png:gd,\n# png:gd:gd, jpg, jpg:cairo, jpg:cairo:gd, jpg:gd, jpg:gd:gd, gif, gif:cairo,\n# gif:cairo:gd, gif:gd, gif:gd:gd and svg.\n# The default value is: png.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_IMAGE_FORMAT       = png\n\n# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to\n# enable generation of interactive SVG images that allow zooming and panning.\n#\n# Note that this requires a modern browser other than Internet Explorer. Tested\n# and working are Firefox, Chrome, Safari, and Opera.\n# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make\n# the SVG files visible. Older versions of IE do not have SVG support.\n# The default value is: NO.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nINTERACTIVE_SVG        = NO\n\n# The DOT_PATH tag can be used to specify the path where the dot tool can be\n# found. If left blank, it is assumed the dot tool can be found in the path.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_PATH               =\n\n# The DOTFILE_DIRS tag can be used to specify one or more directories that\n# contain dot files that are included in the documentation (see the \\dotfile\n# command).\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOTFILE_DIRS           =\n\n# The MSCFILE_DIRS tag can be used to specify one or more directories that\n# contain msc files that are included in the documentation (see the \\mscfile\n# command).\n\nMSCFILE_DIRS           =\n\n# The DIAFILE_DIRS tag can be used to specify one or more directories that\n# contain dia files that are included in the documentation (see the \\diafile\n# command).\n\n#DIAFILE_DIRS           =\n\n# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the\n# path where java can find the plantuml.jar file. If left blank, it is assumed\n# PlantUML is not used or called during a preprocessing step. Doxygen will\n# generate a warning when it encounters a \\startuml command in this case and\n# will not generate output for the diagram.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\n#PLANTUML_JAR_PATH      =\n\n# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes\n# that will be shown in the graph. If the number of nodes in a graph becomes\n# larger than this value, doxygen will truncate the graph, which is visualized\n# by representing a node as a red box. Note that doxygen if the number of direct\n# children of the root node in a graph is already larger than\n# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that\n# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.\n# Minimum value: 0, maximum value: 10000, default value: 50.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_GRAPH_MAX_NODES    = 50\n\n# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs\n# generated by dot. A depth value of 3 means that only nodes reachable from the\n# root by following a path via at most 3 edges will be shown. Nodes that lay\n# further from the root node will be omitted. Note that setting this option to 1\n# or 2 may greatly reduce the computation time needed for large code bases. Also\n# note that the size of a graph can be further restricted by\n# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.\n# Minimum value: 0, maximum value: 1000, default value: 0.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nMAX_DOT_GRAPH_DEPTH    = 0\n\n# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent\n# background. This is disabled by default, because dot on Windows does not seem\n# to support this out of the box.\n#\n# Warning: Depending on the platform used, enabling this option may lead to\n# badly anti-aliased labels on the edges of a graph (i.e. they become hard to\n# read).\n# The default value is: NO.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_TRANSPARENT        = NO\n\n# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output\n# files in one run (i.e. multiple -o and -T options on the command line). This\n# makes dot run faster, but since only newer versions of dot (>1.8.10) support\n# this, this feature is disabled by default.\n# The default value is: NO.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_MULTI_TARGETS      = YES\n\n# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page\n# explaining the meaning of the various boxes and arrows in the dot generated\n# graphs.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nGENERATE_LEGEND        = YES\n\n# If the DOT_CLEANUP tag is set to YES doxygen will remove the intermediate dot\n# files that are used to generate the various graphs.\n# The default value is: YES.\n# This tag requires that the tag HAVE_DOT is set to YES.\n\nDOT_CLEANUP            = YES"
  },
  {
    "path": "docs/cpp_docs/Makefile",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nall: html\n\nhtml:\n\tmkdir -p build/html\n\tdoxygen Doxyfile\n\n\nclean:\n\trm -rf build\n"
  },
  {
    "path": "docs/python_docs/README.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# README\n\nPreview at https://mxnet-beta.staged.apache.org/\n\n## [Building the Docs and Website](https://cwiki.apache.org/confluence/display/MXNET/Building+the+New+Website)\n\n## [Technical details for the building the Python microsite](python/README.md)\n\n# Python binding docs\n\nThe following guide will help you build a local version of the Python API website,\nso that you may work on and test any contributions.\n\nIt is recommended that you read the MXNet developer wiki's info on [building the website & docs](https://cwiki.apache.org/confluence/display/MXNET/Building+the+New+Website) as that includes info on how to test and build the site using Docker. The following information should only be used if you can't use Docker or if you're trying to run the site locally.\n\n## Setup\n\nThe default configuration requires a GPU and CUDA 9.2 and expects Ubuntu.\nHowever, you may setup the website on macOS or Windows with or without a GPU.\n\n### Prerequisites\n\nTo run the full build, including tests of all tutorials, **you will need at\nleast two GPUs**. Distributed training is a key feature of MXNet, so multiple\nGPUs are required for running through some of the tutorials.\n\nYou need to install MXNet, for example, by following the build from source\nguide. Further, you need to install the Python requirements listed in the\n`requirements` file: \n\n```bash\npython3 -m pip install -r requirements\n```\n\n## Build the docs\n\n* Change directories to `python-docs/python`.\n\nTo build without GPUs and without testing the notebooks (faster):\n\n```bash\nmake EVAL=0\n```\n\nTo build with testing the notebooks (requires GPU):\n\n```bash\nmake\n```\n\nThe build docs will be available at `build/_build/html`.\n\nEach build may take a few minutes even without evaluation. To accelerate it, we can use one of the following ways:\n\n1. open `build/conf.py`, add the folders you want to skip into `exclude_patterns`, such as `exclude_patterns = ['templates', 'api', 'develop', 'blog']`.\n2. move the files into a different folder, such as `mv api /tmp/`, and then `make clean`.\n\n## Check results\n\nTo run a server to see the website:\n\n1. Start a http server: `cd build/_build/html; python -m http.server`\n2. For viewing a remote machine, ssh to your machine with port forwarding: `ssh -L8000:localhost:8000 your_machine`\n3. Open http://localhost:8000 in your local machine\n\n## Run tutorials\n\nIn addition to view the built html pages, you can run the Jupyter notebook from a remote machine.\n1. Install `notedown` plugin: `pip install https://github.com/mli/notedown/tarball/master` in remote server\n2. Start Jupyter notebook `jupyter notebook --NotebookApp.contents_manager_class='notedown.NotedownContentsManager'` in remote server\n3. ssh to your machine with port forwarding: `ssh -L8888:localhost:8888 your_machine`\n4. Open http://localhost:8888 in your local machine and run the md files directly\n\nOptionally, one can run the following to launch the notedown plugin automatically when starting jupyter notebook.\n1. Generate the jupyter configure file `~/.jupyter/jupyter_notebook_config.py` if it\nis not existing by run `jupyter notebook --generate-config`\n2. Add `c.NotebookApp.contents_manager_class = 'notedown.NotedownContentsManager'` to `~/.jupyter/jupyter_notebook_config.py`\n3. Simply run `jupyter notebook`\n"
  },
  {
    "path": "docs/python_docs/_static/autodoc.js",
    "content": "/*!\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/* Customizations to the Sphinx auto module plugin output */\nfunction auto_index() {\n    var targets = $(\"dl.class>dt,dl.function>dt\");\n    var li_node = $(\"li.current>span>a.current.reference.internal\").parent().parent();\n    var html = \"<ul id='autodoc'>\";\n    if (li_node.length > 0) {\n        if (targets.length > 0) {\n            for (var i = 0; i < targets.length; ++i) {\n                var id = $(targets[i]).attr('id');\n                if (id) {\n                    var paths = id.split('.')\n                    if (paths.length >= 2) {\n                        var id_simple = paths.pop();\n                        id_simple = paths.pop() + \".\" + id_simple;\n                    } else {\n                        var id_simple = id;\n                    }\n                    html += \"<li><span class='link-wrapper'><a class='reference internal' href='#\";\n                    html += id;\n                    html += \"'>\" + id_simple + \"</a></span</li>\";\n                }\n            }\n            html += \"</ul>\";\n            li_node.append(html);\n            li_node.prepend(\"<a><span id='autodoc_toggle' onclick='$(\\\"#autodoc\\\").toggle()'>[toggle]</span></a>\")\n        }\n    } else {\n        setTimeout(auto_index, 500);\n    }\n\n}\n$(document).ready(auto_index);"
  },
  {
    "path": "docs/python_docs/_static/feedback.css",
    "content": ".feedback-container {\n  text-align: center;\n}\n\n.feedback-answer-container {\n  display: inline-block;\n}\n\n.feedback-question {\n  display: inline-block;\n  padding: 0.5em 1em 0.5em 1em;\n}\n\n.feedback-answer {\n  display: inline-block;\n  padding: 0.5em 1em 0.5em 1em;\n  color: #048ccc;\n  cursor: pointer;\n}\n\n.feedback-answer:hover {\n  color: #ffffff;\n  background-color: #048ccc;\n}\n\n.feedback-thank-you {\n  display: none;\n  padding: 0.5em 1em 0.5em 1em;\n}\n\n.feedback-hr-top {\n  margin-top: 50px;\n}\n\n.feedback-hr-bottom {\n  margin-bottom: 30px;\n}\n"
  },
  {
    "path": "docs/python_docs/_static/matomo_analytics.js",
    "content": "/*!\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\nvar _paq = window._paq = window._paq || [];\n/* tracker methods like \"setCustomDimension\" should be called before \"trackPageView\" */\n/* We explicitly disable cookie tracking to avoid privacy issues */\n_paq.push(['disableCookies']);\n_paq.push(['trackPageView']);\n_paq.push(['enableLinkTracking']);\n(function() {\n  var u=\"https://analytics.apache.org/\";\n  _paq.push(['setTrackerUrl', u+'matomo.php']);\n  _paq.push(['setSiteId', '23']);\n  var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];\n  g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);\n})();\n"
  },
  {
    "path": "docs/python_docs/_static/mxnet.css",
    "content": "/* THOMAS EDITS */\n\n.logo {\n    width: 120px !important;\n    padding: 21px !important;\n    margin: auto !important;    \n}\n\n@media only screen and (min-width: 1025px) {\n    .mdl-navigation.breadcrumb {\n        margin-left: 100px !important;\n    }\n    .mdl-layout__drawer {\n        width: 300px !important;\n    }\n}\n\n.mdl-layout {\n    visibility: hidden;\n}\n\n.mdl-layout__header--waterfall.is-casting-shadow {\n    box-shadow: none !important;\n}\n\n.mdl-layout__header.is-compact {\n    max-height: inherit !important;\n}\n\n.mx-card {\n    box-shadow: none;\n    transition: box-shadow 0.3s linear;\n}\n\n.mx-card.head-card {\n    width: 100% !important;\n}\n\n.mx-card-title {\n    font-weight: 300 !important\n}\n\n.mx-card:hover .mx-card-title {\n    color: orangered !important;\n}\n\n.mdl-card:hover {\n    box-shadow: inset 0 -2px 0 0 #048ccc !important;\n}\n\n.mdl-layout__header-row {\n    height: 80px !important;\n}\n\n.mdl-shadow--2dp {\n    box-shadow: none !important;\n}\n\n\ndiv.mdl-layout__header-row.header-links {\n    display: none !important;\n    visibility: none !important;\n    background-color: #fafafa !important;\n}\n\n.header_links {\n    display: none !important;\n    visibility: none !important;\n    background-color: #fafafa !important;\n}\n\n\n.mdl-layout__header {\n    background-color: rgb(4,140,204);\n}\n.mdl-layout-title {\n    background-color: rgb(4,140,204);\n}\n\n\n.mdl-layout__drawer {\n    box-shadow: none !important;\n    border: none !important;\n}\n\n.pagenation {\n    visibility: hidden !important;\n}\n\nfooter.mdl-mini-footer {\n    width: 100%;\n    padding-left: 150px;\n    background-color: #424242 !important;\n}\n\n    \n/* END OF THOMAS EDITS */\n\n\nbody {\n    font-family: 'Roboto', sans-serif;\n}\n\np {\n    font-size: 16px;\n    /*     font-weight: 400; */\n    line-height: 1.5em;\n    margin: 16px 0;\n}\n\n.sidebar {\n    float: right;\n    display: block;\n    width: 30%;\n    padding: 0 20px;\n    margin: 0 20px;\n    background-color: #eee;\n    border-radius: 8px;\n\n}\n\n@media (max-width: 500px) {\n    .sidebar {\n        float: none;\n        width: 100%;\n        padding: 0 10px;\n        margin: 0 10px;\n        width: 80%;\n    }\n}\n\n.sidebar .sidebar-title {\n    text-align: center;\n    display: block;\n    margin-bottom: 0px;\n    display: none;\n}\n\n.align-center {\n    text-align: center;\n    display: block;\n    /* float: right; */\n    margin: auto;\n}\n\n/* API section */\n\n.mx-api .section .hidden-section {\n    display: none;\n}\n\n.mx-api h3.mdl-color-text--primary {\n    /* display: none; */\n\n    /* border-top-style: solid; */\n    /* border-color: #ccc; */\n    /* border-top-width: 1px; */\n    padding: 1em 0 0 0;\n    margin: 2em 0 0 0;\n    height: 0;\n}\n\n/* .section .viewcode-link { */\n/*     padding-left: 2em; */\n/*     font-size: 80%; */\n/* } */\n\n.section .class dt {\n    padding-bottom: 1em;\n}\n\n.install {\n    max-width: 800px;\n}\n.install .title {\n    display: inline-block;\n    min-width: 100px;\n    text-transform: uppercase;\n    font-size: 90%;\n    color: #555;\n}\n\n.install .option {\n    margin: 5px;\n}\n\n@media (max-width: 650px) {\n    .install .option, .install .title {\n        width: 90%;\n    }\n}\n\n.install .title {\n    margin-top: 1em;\n}\n\n/* autodoc */\n#autodoc_toggle {\n    float: right;\n    margin: 4px;\n    cursor: pointer;\n}\n\n.scrollUp {\n    transform: translateY(-80px);\n}\n"
  },
  {
    "path": "docs/python_docs/python/.gitignore",
    "content": "_build/\nbuild/**/*.rst\nbuild/**/*.ipynb\nbuild/**/*.md\n__pycache__\n/build.sh\n_autogen\n*.ndarray\n*.pickle\n/365px-Golden_Retriever_medium-to-light-coat.jpg\n/net.params\n/synset.txt\n*.pt\n**/raw/*-ubyte\ndogcat\n*.tar.gz\n*.jpg\n*checkpoint.md\n*.ipynb_checkpoints*\n*.json\n"
  },
  {
    "path": "docs/python_docs/python/Makefile",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nall: html\n\n# markdown files that will not be evaluated, simply copy to build/\nPURE_MARKDOWN =  ./README.md\n# markdown files that will be evaluated and then saved as ipynb files\nIPYNB_MARKDOWN = $(shell find . -not -path \"./build/*\" -not -path \"*.ipynb_checkpoints*\" -name \"*.md\" | sort -h)\n# RST files will be simply coped to build/\nRST = $(shell find . -not -path \"./build/*\" -not -path \"*.ipynb_checkpoints*\" -name \"*.rst\")\n\nOBJ = $(patsubst %.rst, build/%.rst, $(RST)) \\\n\t$(patsubst %.md, build/%.ipynb, \\\n\t\t$(filter-out $(PURE_MARKDOWN), $(IPYNB_MARKDOWN)))\n\nbuild/%.ipynb: %.md\n\t@mkdir -p $(@D)\n\tpython3 scripts/md2ipynb.py $< $@\n\n\nbuild/%.rst: %.rst\n\t@mkdir -p $(@D)\n\t# python3 scripts/process_rst.py $< $@\n\tcp $< $@\n\nbuild/%: %\n\t@mkdir -p $(@D)\n\t@cp -r $< $@\n\nlinkcheck: $(OBJ)\n\tmkdir -p build;\n\tcp Makefile_sphinx build/Makefile;\n\tcp -n -r ../_static build/ || true;\n\tsphinx-autogen build/api/*.rst build/api/**/*.rst   -t build/_templates/;\n\tmake -C build linkcheck;\n\nhtml: $(OBJ)\n\tmkdir -p build;\n\tcp Makefile_sphinx build/Makefile;\n\tcp -n -r ../_static build/ || true;\n\tsphinx-autogen build/api/*.rst build/api/**/*.rst   -t build/_templates/;\n\t# make -C build linkcheck doctest html\n\tmake -C build html;\n\tsed -i.bak 's/33\\,150\\,243/23\\,141\\,201/g'  build/_build/html/_static/material-design-lite-1.3.0/material.blue-deep_orange.min.css;\n\tmake update_github_link\n\nupdate_github_link:\n\tfor f in $(shell find build/_build/html/tutorials -type f -name '*.html'); do \\\n\t\techo \"Updating github edit link for $$f.\"; \\\n\t\tsed -i.bak 's/\\(href=\"https:\\/\\/github.com\\/apache\\/mxnet\\/edit\\/master\\/docs\\/python_docs\\/python\\/tutorials\\/[^\"]*\\).ipynb\"/\\1.md\"/g' $$f; \\\n\tdone;\n\nclean:\n\trm -rf build\n"
  },
  {
    "path": "docs/python_docs/python/Makefile_sphinx",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Makefile for Sphinx documentation\n#\n\n# Begin number of processors detection\nNPROCS := 1\nOS := $(shell uname)\nexport NPROCS\n\nifeq ($(NUMJOBS),)\n\nifeq ($(OS),Linux)\n  NPROCS := $(shell grep -c ^processor /proc/cpuinfo)\nelse ifeq ($(OS),Darwin)\n  NPROCS := $(shell sysctl -n hw.physicalcpu)\nendif # $(OS)\n\nelse\n  NPROCS := $(NUMJOBS)\nendif # $(NUMJOBS)\n# End number of processors detection\n\n# You can set these variables from the command line.\nSPHINXOPTS    = -j$(NPROCS) -c ../scripts --keep-going -W\nSPHINXBUILD   = sphinx-build\nPAPER         =\nBUILDDIR      = _build\n\n# User-friendly check for sphinx-build\nifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)\n$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)\nendif\n\n# Internal variables.\nPAPEROPT_a4     = -D latex_paper_size=a4\nPAPEROPT_letter = -D latex_paper_size=letter\nALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .\n# the i18n builder cannot share the environment and doctrees with the others\nI18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .\n\n.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext\n\nall: html\n\nhelp:\n\t@echo \"Please use \\`make <target>' where <target> is one of\"\n\t@echo \"  html       to make standalone HTML files\"\n\t@echo \"  dirhtml    to make HTML files named index.html in directories\"\n\t@echo \"  singlehtml to make a single large HTML file\"\n\t@echo \"  pickle     to make pickle files\"\n\t@echo \"  json       to make JSON files\"\n\t@echo \"  htmlhelp   to make HTML files and a HTML help project\"\n\t@echo \"  qthelp     to make HTML files and a qthelp project\"\n\t@echo \"  applehelp  to make an Apple Help Book\"\n\t@echo \"  devhelp    to make HTML files and a Devhelp project\"\n\t@echo \"  epub       to make an epub\"\n\t@echo \"  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter\"\n\t@echo \"  latexpdf   to make LaTeX files and run them through pdflatex\"\n\t@echo \"  latexpdfja to make LaTeX files and run them through platex/dvipdfmx\"\n\t@echo \"  text       to make text files\"\n\t@echo \"  man        to make manual pages\"\n\t@echo \"  texinfo    to make Texinfo files\"\n\t@echo \"  info       to make Texinfo files and run them through makeinfo\"\n\t@echo \"  gettext    to make PO message catalogs\"\n\t@echo \"  changes    to make an overview of all changed/added/deprecated items\"\n\t@echo \"  xml        to make Docutils-native XML files\"\n\t@echo \"  pseudoxml  to make pseudoxml-XML files for display purposes\"\n\t@echo \"  linkcheck  to check all external links for integrity\"\n\t@echo \"  doctest    to run all doctests embedded in the documentation (if enabled)\"\n\t@echo \"  coverage   to run coverage check of the documentation (if enabled)\"\n\nclean:\n\trm -rf $(BUILDDIR)/*\n\nlivehtml:\n\tsphinx-autobuild --ignore \"web-data/*\" -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html\n\nhtml:\n\texport BUILD_VER=$(BUILD_VER)\n\t@echo \"Env var set for BUILD_VER: $(BUILD_VER)\"\n\t$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/html.\"\n\ndirhtml:\n\t$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/dirhtml.\"\n\nsinglehtml:\n\t$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml\n\t@echo\n\t@echo \"Build finished. The HTML page is in $(BUILDDIR)/singlehtml.\"\n\npickle:\n\t$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle\n\t@echo\n\t@echo \"Build finished; now you can process the pickle files.\"\n\njson:\n\t$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json\n\t@echo\n\t@echo \"Build finished; now you can process the JSON files.\"\n\nhtmlhelp:\n\t$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp\n\t@echo\n\t@echo \"Build finished; now you can run HTML Help Workshop with the\" \\\n\t      \".hhp project file in $(BUILDDIR)/htmlhelp.\"\n\nqthelp:\n\t$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp\n\t@echo\n\t@echo \"Build finished; now you can run \"qcollectiongenerator\" with the\" \\\n\t      \".qhcp project file in $(BUILDDIR)/qthelp, like this:\"\n\t@echo \"# qcollectiongenerator $(BUILDDIR)/qthelp/zongyanwei.qhcp\"\n\t@echo \"To view the help file:\"\n\t@echo \"# assistant -collectionFile $(BUILDDIR)/qthelp/zongyanwei.qhc\"\n\napplehelp:\n\t$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp\n\t@echo\n\t@echo \"Build finished. The help book is in $(BUILDDIR)/applehelp.\"\n\t@echo \"N.B. You won't be able to view it unless you put it in\" \\\n\t      \"~/Library/Documentation/Help or install it in your application\" \\\n\t      \"bundle.\"\n\ndevhelp:\n\t$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp\n\t@echo\n\t@echo \"Build finished.\"\n\t@echo \"To view the help file:\"\n\t@echo \"# mkdir -p $$HOME/.local/share/devhelp/zongyanwei\"\n\t@echo \"# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/zongyanwei\"\n\t@echo \"# devhelp\"\n\nepub:\n\t$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub\n\t@echo\n\t@echo \"Build finished. The epub file is in $(BUILDDIR)/epub.\"\n\nlatex:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo\n\t@echo \"Build finished; the LaTeX files are in $(BUILDDIR)/latex.\"\n\t@echo \"Run \\`make' in that directory to run these through (pdf)latex\" \\\n\t      \"(use \\`make latexpdf' here to do that automatically).\"\n\nlatexpdf:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo \"Running LaTeX files through pdflatex...\"\n\t$(MAKE) -C $(BUILDDIR)/latex all-pdf\n\t@echo \"pdflatex finished; the PDF files are in $(BUILDDIR)/latex.\"\n\nlatexpdfja:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo \"Running LaTeX files through platex and dvipdfmx...\"\n\t$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja\n\t@echo \"pdflatex finished; the PDF files are in $(BUILDDIR)/latex.\"\n\ntext:\n\t$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text\n\t@echo\n\t@echo \"Build finished. The text files are in $(BUILDDIR)/text.\"\n\nman:\n\t$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man\n\t@echo\n\t@echo \"Build finished. The manual pages are in $(BUILDDIR)/man.\"\n\ntexinfo:\n\t$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo\n\t@echo\n\t@echo \"Build finished. The Texinfo files are in $(BUILDDIR)/texinfo.\"\n\t@echo \"Run \\`make' in that directory to run these through makeinfo\" \\\n\t      \"(use \\`make info' here to do that automatically).\"\n\ninfo:\n\t$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo\n\t@echo \"Running Texinfo files through makeinfo...\"\n\tmake -C $(BUILDDIR)/texinfo info\n\t@echo \"makeinfo finished; the Info files are in $(BUILDDIR)/texinfo.\"\n\ngettext:\n\t$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale\n\t@echo\n\t@echo \"Build finished. The message catalogs are in $(BUILDDIR)/locale.\"\n\nchanges:\n\t$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes\n\t@echo\n\t@echo \"The overview file is in $(BUILDDIR)/changes.\"\n\nlinkcheck:\n\t$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck\n\t@echo\n\t@echo \"Link check complete; look for any errors in the above output \" \\\n\t      \"or in $(BUILDDIR)/linkcheck/output.txt.\"\n\ndoctest:\n\t$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest\n\t@echo \"Testing of doctests in the sources finished, look at the \" \\\n\t      \"results in $(BUILDDIR)/doctest/output.txt.\"\n\ncoverage:\n\t$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage\n\t@echo \"Testing of coverage in the sources finished, look at the \" \\\n\t      \"results in $(BUILDDIR)/coverage/python.txt.\"\n\nxml:\n\t$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml\n\t@echo\n\t@echo \"Build finished. The XML files are in $(BUILDDIR)/xml.\"\n\npseudoxml:\n\t$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml\n\t@echo\n\t@echo \"Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml.\"\n"
  },
  {
    "path": "docs/python_docs/python/api/autograd/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.autograd\n==============\n\n.. automodule:: mxnet.autograd\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/contrib/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.contrib\n=============\n\n.. automodule:: mxnet.contrib\n\nContributed modules\n-------------------\n\n.. container:: cards\n\n   .. card::\n      :title: contrib.io\n      :link: io/index.html\n\n      Data iterators for common data formats and utility functions.\n\n   .. card::\n      :title: contrib.ndarray\n      :link: ndarray/index.html\n\n      Operations and functions on NDArrays.\n\n   .. card::\n      :title: contrib.onnx\n      :link: onnx/index.html\n\n      ONNX support.\n\n   .. card::\n      :title: contrib.symbol\n      :link: symbol/index.html\n\n      Symbolic API for MXNet.\n\n   .. card::\n      :title: contrib.tensorboard\n      :link: tensorboard/index.html\n\n      Tensorboard integration.\n\n   .. card::\n      :title: contrib.tensorrt\n      :link: tensorrt/index.html\n\n      Tensorrt integration.\n\n   .. card::\n      :title: contrib.text\n      :link: text/index.html\n\n      Functions for manipulating text data.\n\n   .. card::\n      :title: contrib.quantization\n      :link: quantization/index.html\n\n      Functions for precision reduction.\n\n.. toctree::\n   :hidden:\n   :maxdepth: 2\n   :glob:\n\n   */index\n"
  },
  {
    "path": "docs/python_docs/python/api/contrib/io/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ncontrib.io\n==========\n\n.. automodule:: mxnet.contrib.io\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/contrib/ndarray/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ncontrib.ndarray\n================\n\n.. automodule:: mxnet.contrib.ndarray\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/contrib/onnx/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ncontrib.onnx\n============\n\n.. automodule:: mxnet.contrib.onnx\n    :members:\n    :imported-members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/contrib/quantization/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ncontrib.quantization\n====================\n\n.. automodule:: mxnet.contrib.quantization\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/contrib/symbol/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ncontrib.symbol\n==============\n\n.. automodule:: mxnet.contrib.symbol\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/contrib/tensorboard/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ncontrib.tensorboard\n===================\n\n.. automodule:: mxnet.contrib.tensorboard\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/contrib/tensorrt/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ncontrib.tensorrt\n================\n\n.. automodule:: mxnet.contrib.tensorrt\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/contrib/text/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ncontrib.text\n============\n\n.. automodule:: mxnet.contrib.text\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/device/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.device\n=============\n\n.. automodule:: mxnet.device\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/engine/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.engine\n============\n\n.. automodule:: mxnet.engine\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/executor/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.executor\n===============\n\n.. automodule:: mxnet.executor\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/gluon/block.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ngluon.Block\n===========\n\n\n.. autoclass:: mxnet.gluon.Block\n    :members:\n    :inherited-members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/gluon/constant.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ngluon.Constant\n==============\n\n\n.. autoclass:: mxnet.gluon.Constant\n    :members:\n    :inherited-members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/gluon/contrib/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ngluon.contrib\n=============\n\nThis document lists the contrib APIs in Gluon:\n\n.. currentmodule:: mxnet.gluon.contrib\n\n.. autosummary::\n    :nosignatures:\n\n    mxnet.gluon.contrib\n\n\nThe `Gluon Contrib` API, defined in the `gluon.contrib` package, provides\nmany useful experimental APIs for new features.\nThis is a place for the community to try out the new features,\nso that feature contributors can receive feedback.\n\n\n.. warning:: This package contains experimental APIs and may change in the near future.\n\n\nIn the rest of this document, we list routines provided by the `gluon.contrib` package.\n\nVision Data\n-----------\n\n.. autosummary::\n    :nosignatures:\n\n    data.vision.create_image_augment\n    data.vision.ImageDataLoader\n    data.vision.ImageBboxDataLoader\n    data.vision.ImageBboxRandomFlipLeftRight\n    data.vision.ImageBboxCrop\n    data.vision.ImageBboxRandomCropWithConstraints\n    data.vision.ImageBboxResize\n\n\nEstimator\n---------\n\n.. currentmodule:: mxnet.gluon.contrib.estimator\n\n.. autosummary::\n    :nosignatures:\n\n    Estimator\n\n\nEvent Handler\n-------------\n\n.. currentmodule:: mxnet.gluon.contrib.estimator\n\n.. autosummary::\n    :nosignatures:\n\n    StoppingHandler\n    MetricHandler\n    ValidationHandler\n    LoggingHandler\n    CheckpointHandler\n    EarlyStoppingHandler\n\n\nAPI Reference\n-------------\n\n.. automodule:: mxnet.gluon.contrib\n    :members:\n\n.. automodule:: mxnet.gluon.contrib.estimator\n    :members:\n    :imported-members:\n"
  },
  {
    "path": "docs/python_docs/python/api/gluon/hybrid_block.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ngluon.HybridBlock\n=================\n\n\n.. autoclass:: mxnet.gluon.HybridBlock\n    :members:\n    :inherited-members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/gluon/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.gluon\n===========\n\nThe Gluon library in Apache MXNet provides a clear, concise, and simple API for deep learning.\nIt makes it easy to prototype, build, and train deep learning models without sacrificing training speed.\n\nExample\n-------\n\nThe following example shows how you might create a simple neural network with three layers:\none input layer, one hidden layer, and one output layer.\n\n.. code-block:: python\n\n   net = gluon.nn.Sequential()\n   # When instantiated, Sequential stores a chain of neural network layers.\n   # Once presented with data, Sequential executes each layer in turn, using\n   # the output of one layer as the input for the next\n   net.add(gluon.nn.Dense(256, activation=\"relu\")) # 1st layer (256 nodes)\n   net.add(gluon.nn.Dense(256, activation=\"relu\")) # 2nd hidden layer\n   net.add(gluon.nn.Dense(num_outputs))\n\n\n.. automodule:: mxnet.gluon\n\n\nTutorials\n---------\n\n.. container:: cards\n\n   .. card::\n      :title: Gluon Guide\n      :link: ../../tutorials/packages/gluon/index.html\n\n      The Gluon guide. Start here!\n\n   .. card::\n      :title: Gluon-CV Toolkit\n      :link: https://gluon-cv.mxnet.io/\n\n      A Gluon add-on module for computer vision.\n\n   .. card::\n      :title: Gluon-NLP Toolkit\n      :link: https://gluon-nlp.mxnet.io/\n\n      A Gluon add-on module for natural language processing.\n\n\nAPIs and Packages\n-----------------\n\nCore Modules\n~~~~~~~~~~~~\n\n.. container:: cards\n\n   .. card::\n      :title: gluon.nn\n      :link: nn/index.html\n\n      Neural network components.\n\n   .. card::\n      :title: gluon.rnn\n      :link: rnn/index.html\n\n      Recurrent neural network components.\n\nTraining\n~~~~~~~~\n\n.. container:: cards\n\n   .. card::\n      :title: gluon.loss\n      :link: loss/index.html\n\n      Loss functions for training neural networks.\n\n   .. card::\n      :title: gluon.metric\n      :link: metric/index.html\n\n      Metrics to evaluate the performance of a learned model.\n\n   .. card::\n      :title: gluon.Parameter\n      :link: parameter.html\n\n      Parameter getting and setting functions.\n\n   .. card::\n      :title: gluon.Trainer\n      :link: trainer.html\n\n      Functions for applying an optimizer on a set of parameters.\n\nData\n~~~~\n\n.. container:: cards\n\n   .. card::\n      :title: gluon.data\n      :link: data/index.html\n\n      Dataset utilities.\n\n   .. card::\n      :title: gluon.data.vision\n      :link: data/vision/index.html\n\n      Image dataset utilities.\n\n   .. card::\n      :title: gluon.model_zoo.vision\n      :link: model_zoo/index.html\n\n      A module for loading pre-trained neural network models.\n\n\nUtilities\n~~~~~~~~~\n\n.. container:: cards\n\n   .. card::\n      :title: gluon.utils\n      :link: utils/index.html\n\n      A variety of utilities for training.\n\n.. toctree::\n   :hidden:\n   :maxdepth: 2\n   :glob:\n\n   block\n   hybrid_block\n   symbol_block\n   constant\n   parameter\n   trainer\n   */index\n"
  },
  {
    "path": "docs/python_docs/python/api/gluon/loss/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ngluon.loss\n==========\n\nGluon provides pre-defined loss functions in the :py:mod:`mxnet.gluon.loss`\nmodule.\n\n.. automodule:: mxnet.gluon.loss\n    :members:\n    :imported-members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/gluon/metric/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ngluon.metric\n============\n\n.. automodule:: mxnet.gluon.metric\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/gluon/model_zoo/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ngluon.model_zoo.vision\n======================\n\n.. automodule:: mxnet.gluon.model_zoo.vision\n   :noindex:\n\n.. autosummary::\n\n    get_model\n\nResNet\n------\n\n.. autosummary::\n\n    resnet18_v1\n    resnet34_v1\n    resnet50_v1\n    resnet101_v1\n    resnet152_v1\n    resnet18_v2\n    resnet34_v2\n    resnet50_v2\n    resnet101_v2\n    resnet152_v2\n\n.. autosummary::\n\n    ResNetV1\n    ResNetV2\n    BasicBlockV1\n    BasicBlockV2\n    BottleneckV1\n    BottleneckV2\n    get_resnet\n\nVGG\n---\n\n.. autosummary::\n\n    vgg11\n    vgg13\n    vgg16\n    vgg19\n    vgg11_bn\n    vgg13_bn\n    vgg16_bn\n    vgg19_bn\n\n.. autosummary::\n\n    VGG\n    get_vgg\n\nAlexnet\n--------\n\n.. autosummary::\n\n    alexnet\n\n.. autosummary::\n\n    AlexNet\n\n\nDenseNet\n--------\n\n\n.. autosummary::\n\n    densenet121\n    densenet161\n    densenet169\n    densenet201\n\n\n\n.. autosummary::\n\n    DenseNet\n\n\nSqueezeNet\n------------\n\n\n.. autosummary::\n\n    squeezenet1_0\n    squeezenet1_1\n\n\n\n.. autosummary::\n\n    SqueezeNet\n\n\nInception\n---------\n\n\n.. autosummary::\n\n    inception_v3\n\n\n.. autosummary::\n\n    Inception3\n\n\nMobileNet\n---------\n\n\n.. autosummary::\n\n    mobilenet1_0\n    mobilenet0_75\n    mobilenet0_5\n    mobilenet0_25\n    mobilenet_v2_1_0\n    mobilenet_v2_0_75\n    mobilenet_v2_0_5\n    mobilenet_v2_0_25\n\n\n.. autosummary::\n\n    MobileNet\n    MobileNetV2\n\nAPI Reference\n-------------\n\n.. automodule:: mxnet.gluon.model_zoo.vision\n    :members:\n    :imported-members:\n    :autosummary:\n\n"
  },
  {
    "path": "docs/python_docs/python/api/gluon/nn/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ngluon.nn\n========\n\nGluon provides a large number of build-in neural network layers in the following\ntwo modules:\n\n..\n   Don't add toctree to these two modules, otherwise it will generate two pages in\n   the global TOC\n\n.. autosummary::\n    :nosignatures:\n\n    mxnet.gluon.nn\n\n\nWe group all layers in these two modules according to their categories.\n\n.. currentmodule:: mxnet.gluon\n\n\n\nSequential Containers\n---------------------\n\n.. autosummary::\n    :nosignatures:\n\n    nn.Sequential\n    nn.HybridSequential\n\n\nConcatenation Containers\n------------------------\n\n.. autosummary::\n    :nosignatures:\n\n    nn.Concatenate\n    nn.HybridConcatenate\n\n\nBasic Layers\n------------\n\n.. autosummary::\n    :nosignatures:\n\n    nn.Dense\n    nn.Activation\n    nn.Dropout\n    nn.Flatten\n    nn.Lambda\n    nn.HybridLambda\n    nn.Identity\n\nConvolutional Layers\n--------------------\n\n.. autosummary::\n    :nosignatures:\n\n    nn.Conv1D\n    nn.Conv2D\n    nn.Conv3D\n    nn.Conv1DTranspose\n    nn.Conv2DTranspose\n    nn.Conv3DTranspose\n    nn.DeformableConvolution\n    nn.ModulatedDeformableConvolution\n\nPixel Shuffle Layers\n--------------------\n\n.. autosummary::\n    :nosignatures:\n\n    nn.PixelShuffle1D\n    nn.PixelShuffle2D\n    nn.PixelShuffle3D\n\nPooling Layers\n--------------\n\n.. autosummary::\n   :nosignatures:\n\n    nn.MaxPool1D\n    nn.MaxPool2D\n    nn.MaxPool3D\n    nn.AvgPool1D\n    nn.AvgPool2D\n    nn.AvgPool3D\n    nn.GlobalMaxPool1D\n    nn.GlobalMaxPool2D\n    nn.GlobalMaxPool3D\n    nn.GlobalAvgPool1D\n    nn.GlobalAvgPool2D\n    nn.GlobalAvgPool3D\n    nn.ReflectionPad2D\n\nNormalization Layers\n--------------------\n\n.. autosummary::\n    :nosignatures:\n\n    nn.BatchNorm\n    nn.InstanceNorm\n    nn.LayerNorm\n    nn.SyncBatchNorm\n\nEmbedding Layers\n----------------\n\n.. autosummary::\n    :nosignatures:\n\n    nn.Embedding\n\n\nAdvanced Activation Layers\n--------------------------\n\n.. autosummary::\n    :nosignatures:\n\n    nn.LeakyReLU\n    nn.PReLU\n    nn.ELU\n    nn.SELU\n    nn.Swish\n    nn.SiLU\n    nn.GELU\n\nAPI Reference\n-------------\n.. automodule:: mxnet.gluon.nn\n    :members:\n    :imported-members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/gluon/parameter.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ngluon.Parameter\n===============\n\n\n.. autoclass:: mxnet.gluon.Parameter\n    :members:\n    :inherited-members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/gluon/rnn/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ngluon.rnn\n=========\n\nBuild-in recurrent neural network layers are provided in the following two modules:\n\n\n.. autosummary::\n    :nosignatures:\n\n    mxnet.gluon.rnn\n\n.. currentmodule:: mxnet.gluon\n\nRecurrent Cells\n---------------\n\n.. autosummary::\n    :nosignatures:\n\n    rnn.LSTMCell\n    rnn.GRUCell\n    rnn.RecurrentCell\n    rnn.LSTMPCell\n    rnn.SequentialRNNCell\n    rnn.BidirectionalCell\n    rnn.DropoutCell\n    rnn.VariationalDropoutCell\n    rnn.ZoneoutCell\n    rnn.ResidualCell\n\nConvolutional Recurrent Cells\n-----------------------------\n\n.. autosummary::\n    :nosignatures:\n\n    rnn.Conv1DLSTMCell\n    rnn.Conv2DLSTMCell\n    rnn.Conv3DLSTMCell\n    rnn.Conv1DGRUCell\n    rnn.Conv2DGRUCell\n    rnn.Conv3DGRUCell\n    rnn.Conv1DRNNCell\n    rnn.Conv2DRNNCell\n    rnn.Conv3DRNNCell\n\nRecurrent Layers\n----------------\n\n.. autosummary::\n    :nosignatures:\n\n    rnn.RNN\n    rnn.LSTM\n    rnn.GRU\n\nAPI Reference\n-------------\n.. automodule:: mxnet.gluon.rnn\n    :members:\n    :imported-members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/gluon/symbol_block.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ngluon.SymbolBlock\n=================\n\n.. autoclass:: mxnet.gluon.SymbolBlock\n    :members:\n    :inherited-members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/gluon/trainer.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ngluon.Trainer\n=============\n\n.. autoclass:: mxnet.gluon.Trainer\n    :members:\n    :inherited-members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/gluon/utils/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\ngluon.utils\n===========\n\n.. automodule:: mxnet.gluon.utils\n    :members:\n    :autosummary:\n\n"
  },
  {
    "path": "docs/python_docs/python/api/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nPython API\n==========\n\nOverview\n--------\n\nThis API section details functions, modules, and objects included in Apache MXNet,\ndescribing what they are and what they do. The APIs are grouped into the\nfollowing categories:\n\n\nImperative API\n--------------\n.. container:: cards\n\n   .. card::\n      :title: mxnet.np\n      :link: np/index.html\n\n      MXNet NP module, an array library that provides NumPy-compatible API.\n\n   .. card::\n      :title: mxnet.npx\n      :link: npx/index.html\n\n      MXNet NPX module, neural network extension to the NumPy-compatible API.\n\n   .. card::\n      :title: mxnet.gluon\n      :link: gluon/index.html\n\n      Imperative APIs to load data, construct and train neural networks.\n\n\n\nGluon related modules\n---------------------\n\n.. container:: cards\n\n   .. card::\n      :title: mxnet.autograd\n      :link: autograd/index.html\n\n      Functions for Automatic differentiation.\n\n   .. card::\n      :title: mxnet.optimizer\n      :link: optimizer/index.html\n\n      Functions for applying an optimizer on weights.\n\n   .. card::\n      :title: mxnet.initializer\n      :link: initializer/index.html\n\n      Default behaviors to initialize parameters.\n\n   .. card::\n      :title: mxnet.lr_scheduler\n      :link: lr_scheduler/index.html\n\n      Scheduling the learning rate.\n\n   .. card::\n      :title: mxnet.kvstore\n      :link: kvstore/index.html\n\n      Key value store interface of MXNet for parameter synchronization.\n\n   .. card::\n      :title: mxnet.device\n      :link: mxnet/device/index.html\n\n      CPU and GPU device information.\n\n   .. card::\n      :title: mxnet.profiler\n      :link: mxnet/profiler/index.html\n\n      Profiler setting methods.\n\n   .. card::\n      :title: mxnet.random\n      :link: mxnet/random/index.html\n\n      Imperative random distribution generator functions.\n\n\nAdvanced modules\n----------------\n\n.. container:: cards\n\n   .. card::\n      :title: mxnet.runtime\n      :link: runtime/index.html\n\n      API for querying MXNet enabled features.\n\n   .. card::\n      :title: mxnet.device\n      :link: device/index.html\n\n      MXNet array device for specifying in-memory storage device.\n\n   .. card::\n      :title: mxnet.profiler\n      :link: profiler/index.html\n\n      MXNet memory and performance profiler.\n\n   .. card::\n      :title: mxnet.executor\n      :link: executor/index.html\n\n      Managing symbolic graph execution.\n\n   .. card::\n      :title: mxnet.kvstore_server\n      :link: kvstore_server/index.html\n\n      Server node for the key value store.\n\n   .. card::\n      :title: mxnet.engine\n      :link: engine/index.html\n\n      Engine properties management.\n\n   .. card::\n      :title: mxnet.rtc\n      :link: rtc/index.html\n\n      Tools for compiling and running CUDA code from the python frontend.\n\n   .. card::\n      :title: mxnet.test_utils\n      :link: test_utils/index.html\n\n      Tools for using and testing MXNet.\n\n   .. card::\n      :title: mxnet.util\n      :link: util/index.html\n\n      General utility functions\n\nLegacy\n------\n\n.. container:: cards\n\n   .. card::\n      :title: mxnet.ndarray\n      :link: legacy/ndarray/index.html\n\n      Imperative APIs to manipulate multi-dimensional arrays.\n\n   .. card::\n      :title: mxnet.symbol\n      :link: legacy/symbol/index.html\n\n      Symbolic APIs for multi-dimensional arrays and neural network layers\n\n   .. card::\n      :title: mxnet.callback\n      :link: legacy/callback/index.html\n\n      Functions to track various statuses during an epoch.\n\n   .. card::\n      :title: mxnet.image\n      :link: legacy/image/index.html\n\n      Image iterators and image augmentation functions.\n\n   .. card::\n      :title: mxnet.io\n      :link: legacy/io/index.html\n\n      Data iterators for common data formats and utility functions.\n\n   .. card::\n      :title: mxnet.recordio\n      :link: legacy/recordio/index.html\n\n      Read and write for the RecordIO data format.\n\n   .. card::\n      :title: mxnet.visualization\n      :link: legacy/visualization/index.html\n\n      Functions for Symbol visualization.\n\n\n.. toctree::\n   :maxdepth: 1\n   :hidden:\n   :glob:\n\n   np/index\n   npx/index\n   gluon/index\n   autograd/index\n   initializer/index\n   optimizer/index\n   lr_scheduler/index\n   kvstore/index\n   contrib/index\n   legacy/index\n   */index*\n"
  },
  {
    "path": "docs/python_docs/python/api/initializer/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.initializer\n=================\n\n.. automodule:: mxnet.initializer\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/kvstore/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nKVStore: Communication for Distributed Training\n===============================================\n.. currentmodule:: mxnet.kvstore\n\n\nHorovod\n=======\n\n.. autosummary::\n   :toctree: generated/\n\n   Horovod\n\nBytePS\n======\n\n.. autosummary::\n   :toctree: generated/\n\n   BytePS\n\n\nKVStore Interface\n=================\n\n.. autosummary::\n   :toctree: generated/\n\n   KVStore\n   KVStoreBase\n   KVStoreServer\n"
  },
  {
    "path": "docs/python_docs/python/api/kvstore_server/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.kvstore_server\n====================\n\n.. automodule:: mxnet.kvstore_server\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/callback/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.callback\n==============\n\n.. automodule:: mxnet.callback\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/image/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.image\n===========\n\n.. note:: This API is best used in conjunction with ``mxnet.io`` data iterators.\n    For augmentation and transforms in gluon with Datasets and DataLoaders see ``mxnet.gluon.data``\n\n\n.. automodule:: mxnet.image\n    :members:\n    :imported-members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nLegacy\n======\n\nThis document hosts API for legacy modules that are being deprecated in MXNet 2.x.\n\n.. container:: cards\n\n   .. card::\n      :title: NDArray API\n      :link: ndarray/index.html\n\n      Imperative APIs to manipulate multi-dimensional arrays.\n\n   .. card::\n      :title: mxnet.symbol\n      :link: symbol/index.html\n\n      Symbolic APIs for multi-dimensional arrays and neural network layers\n\n   .. card::\n      :title: mxnet.callback\n      :link: callback/index.html\n\n      Functions to track various statuses during an epoch.\n\n   .. card::\n      :title: mxnet.io\n      :link: io/index.html\n\n      Data iterators for common data formats and utility functions.\n\n   .. card::\n      :title: mxnet.recordio\n      :link: recordio/index.html\n\n      Read and write for the RecordIO data format.\n\n   .. card::\n      :title: mxnet.image\n      :link: image/index.html\n\n      Image iterators and image augmentation functions.\n\n   .. card::\n      :title: mxnet.visualization\n      :link: visualization/index.html\n\n      Functions for Symbol visualization.\n\n\n.. toctree::\n   :hidden:\n   :glob:\n\n   */index*\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/io/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.io\n========\n\n.. automodule:: mxnet.io\n    :members:\n    :imported-members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/ndarray/contrib/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nndarray.contrib\n===============\n\n.. automodule:: mxnet.ndarray.contrib\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/ndarray/image/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nndarray.image\n=============\n\n.. automodule:: mxnet.ndarray.image\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/ndarray/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.ndarray\n=============\n\nThe NDArray library in Apache MXNet defines the core data structure for all mathematical computations. NDArray supports fast execution on a wide range of hardware configurations and automatically parallelizes multiple operations across the available hardware.\n\nExample\n-------\n\nThe following example shows how you can create an NDArray from a regular Python list using the 'array' function. \n\n.. code-block:: python\n\n\timport mxnet as mx\n\t# create a 1-dimensional array with a python list\n\ta = mx.nd.array([1,2,3])\n\t# create a 2-dimensional array with a nested python list\n\tb = mx.nd.array([[1,2,3], [2,3,4]])\n\t{'a.shape':a.shape, 'b.shape':b.shape}\n\n\n.. note:: ``mxnet.ndarray`` is similar to ``numpy.ndarray`` in some aspects. But the differences are not negligible. For instance:\n\n   - ``mxnet.ndarray.NDArray.T`` does real data transpose to return new a copied\n     array, instead of returning a view of the input array.\n   - ``mxnet.ndarray.dot`` performs dot product between the last axis of the\n     first input array and the first axis of the second input, while `numpy.dot`\n     uses the second last axis of the input array.\n\n   In addition, ``mxnet.ndarray.NDArray`` supports GPU computation and various neural\n   network layers.\n\n.. note:: ``ndarray`` provides almost the same routines as ``symbol``. Most\n  routines between these two packages share the source code. But ``ndarray``\n  differs from ``symbol`` in few aspects:\n\n  - ``ndarray`` adopts imperative programming, namely sentences are executed\n    step-by-step so that the results can be obtained immediately whereas\n    ``symbol`` adopts declarative programming.\n\n  - Most binary operators in ``ndarray`` such as ``+`` and ``>`` have\n    broadcasting enabled by default.\n\nTutorials\n---------\n\n.. container:: cards\n\n\n   .. card::\n      :title: NDArray Guide\n      :link: ../../tutorials/packages/ndarray/\n\n      The NDArray guide. Start here!\n\n\n\nNDArray API of MXNet\n--------------------\n\n.. container:: cards\n\n   .. card::\n      :title: NDArray\n      :link: ndarray.html\n\n      Imperative tensor operations using the NDArray API.\n\n\nSparse NDArray API of MXNet\n---------------------------\n\n.. container:: cards\n\n   .. card::\n      :title: Sparse routines\n      :link: sparse/index.html\n\n      Representing and manipulating sparse arrays.\n\n\n\n.. toctree::\n   :hidden:\n   :maxdepth: 2\n   :glob:\n\n   ndarray\n   */index\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/ndarray/linalg/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nndarray.linalg\n==============\n\n.. automodule:: mxnet.ndarray.linalg\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/ndarray/ndarray.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nndarray\n=======\n\n.. currentmodule:: mxnet.ndarray\n\n.. automodule:: mxnet.ndarray\n    :members:\n    :imported-members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/ndarray/op/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nndarray.op\n==========\n\n.. automodule:: mxnet.ndarray.op\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/ndarray/random/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nndarray.random\n==============\n\n.. automodule:: mxnet.ndarray.random\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/ndarray/register/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nndarray.register\n================\n\n.. automodule:: mxnet.ndarray.register\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/ndarray/sparse/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nndarray.sparse\n==============\n\n.. automodule:: mxnet.ndarray.sparse\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/ndarray/utils/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nndarray.utils\n=============\n\n.. automodule:: mxnet.ndarray.utils\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/recordio/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.recordio\n==============\n\n.. automodule:: mxnet.recordio\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/symbol/contrib/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nsymbol.contrib\n===============\n\n.. automodule:: mxnet.symbol.contrib\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/legacy/symbol/image/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nsymbol.image\n=============\n\n.. automodule:: mxnet.symbol.image\n    :members:\n    :autosummary:"
  },
  {
    "path": "docs/python_docs/python/api/legacy/symbol/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.symbol\n============\n\nThe Symbol API in Apache MXNet is an interface for symbolic programming. It features the use of computational graphs, reduced memory usage, and pre-use function optimization.\n\n\nExample\n-------\n\nThe following example shows how you might build a simple expression with the Symbol API.\n\n.. code-block:: python\n\n\timport mxnet as mx\n\t# Two placeholers are created with mx.sym.variable\n\ta = mx.sym.Variable('a')\n\tb = mx.sym.Variable('b')\n\t# The symbol is constructed using the '+' operator\n\tc = a + b\n\t(a, b, c)\n\n\nSymbol Package\n--------------\n\n.. container:: cards\n\n   .. card::\n      :title: Symbol\n      :link: symbol.html\n\n      Symbolic programming using the Symbol API.\n\n.. toctree::\n   :hidden:\n   :maxdepth: 2\n   :glob:\n\n   symbol\n   */index"
  },
  {
    "path": "docs/python_docs/python/api/legacy/symbol/linalg/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nsymbol.linalg\n=============\n\n.. automodule:: mxnet.symbol.linalg\n    :members:\n    :autosummary:"
  },
  {
    "path": "docs/python_docs/python/api/legacy/symbol/op/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nsymbol.op\n=========\n\n.. automodule:: mxnet.symbol.op\n    :members:\n    :autosummary:"
  },
  {
    "path": "docs/python_docs/python/api/legacy/symbol/random/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nsymbol.random\n=============\n\n.. automodule:: mxnet.symbol.random\n    :members:\n    :autosummary:"
  },
  {
    "path": "docs/python_docs/python/api/legacy/symbol/register/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nsymbol.register\n===============\n\n.. automodule:: mxnet.symbol.register\n    :members:\n    :autosummary:"
  },
  {
    "path": "docs/python_docs/python/api/legacy/symbol/sparse/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nsymbol.sparse\n=============\n\n.. automodule:: mxnet.symbol.sparse\n    :members:\n    :autosummary:"
  },
  {
    "path": "docs/python_docs/python/api/legacy/symbol/symbol.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nsymbol\n======\n\n.. currentmodule:: mxnet.symbol\n\n.. automodule:: mxnet.symbol\n    :members:\n    :imported-members:\n    :autosummary:"
  },
  {
    "path": "docs/python_docs/python/api/legacy/visualization/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.visualization\n===================\n\n.. automodule:: mxnet.visualization\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/lr_scheduler/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.lr_scheduler\n==================\n\n.. automodule:: mxnet.lr_scheduler\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/np/arrays.indexing.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\n.. _arrays.indexing:\n\nIndexing\n========\n\n.. sectionauthor:: adapted from \"Guide to NumPy\" by Travis E. Oliphant\n\n.. currentmodule:: mxnet.np\n\n.. index:: indexing, slicing\n\n:class:`ndarrays <ndarray>` can be indexed using the standard Python\n``x[obj]`` syntax, where *x* is the array and *obj* the selection.\nThere are three kinds of indexing available: basic\nslicing, advanced indexing, and boolean mask indexing. Which one occurs depends on *obj*.\n\n.. note::\n\n   In Python, ``x[(exp1, exp2, ..., expN)]`` is equivalent to\n   ``x[exp1, exp2, ..., expN]``; the latter is just syntactic sugar\n   for the former.\n\n\nBasic Slicing and Indexing\n--------------------------\n\nBasic slicing extends Python's basic concept of slicing to N\ndimensions. Basic slicing occurs when *obj* is a :class:`slice` object\n(constructed by ``start:stop:step`` notation inside of brackets), an\ninteger, or a tuple of slice objects and integers. :const:`Ellipsis`\nand :const:`newaxis` objects can be interspersed with these as\nwell.\n\nThe simplest case of indexing with *N* integers returns an array\nscalar representing the corresponding item.  As in\nPython, all indices are zero-based: for the *i*-th index :math:`n_i`,\nthe valid range is :math:`0 \\le n_i < d_i` where :math:`d_i` is the\n*i*-th element of the shape of the array.  Negative indices are\ninterpreted as counting from the end of the array (*i.e.*, if\n:math:`n_i < 0`, it means :math:`n_i + d_i`).\n\nAll arrays generated by basic slicing are always views\nof the original array if the fetched elements are contiguous in memory.\n\nThe standard rules of sequence slicing apply to basic slicing on a\nper-dimension basis (including using a step index). Some useful\nconcepts to remember include:\n\n- The basic slice syntax is ``i:j:k`` where *i* is the starting index,\n  *j* is the stopping index, and *k* is the step (:math:`k\\neq0`).\n  This selects the *m* elements (in the corresponding dimension) with\n  index values *i*, *i + k*, ..., *i + (m - 1) k* where\n  :math:`m = q + (r\\neq0)` and *q* and *r* are the quotient and remainder\n  obtained by dividing *j - i* by *k*: *j - i = q k + r*, so that\n  *i + (m - 1) k < j*.\n\n  .. admonition:: Example\n\n     >>> x = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])\n     >>> x[1:7:2]\n     array([1, 3, 5])\n\n- Negative *i* and *j* are interpreted as *n + i* and *n + j* where\n  *n* is the number of elements in the corresponding dimension.\n  Negative *k* makes stepping go towards smaller indices.\n\n  .. admonition:: Example\n\n      >>> x[-2:10]\n      array([8, 9])\n      >>> x[-3:3:-1]\n      array([7, 6, 5, 4])\n\n- Assume *n* is the number of elements in the dimension being\n  sliced. Then, if *i* is not given it defaults to 0 for *k > 0* and\n  *n - 1* for *k < 0* . If *j* is not given it defaults to *n* for *k > 0*\n  and *-n-1* for *k < 0* . If *k* is not given it defaults to 1. Note that\n  ``::`` is the same as ``:`` and means select all indices along this\n  axis.\n\n  .. admonition:: Example\n\n      >>> x[5:]\n      array([5, 6, 7, 8, 9])\n\n- If the number of objects in the selection tuple is less than\n  *N* , then ``:`` is assumed for any subsequent dimensions.\n\n  .. admonition:: Example\n\n      >>> x = np.array([[[1],[2],[3]], [[4],[5],[6]]])\n      >>> x.shape\n      (2, 3, 1)\n      >>> x[1:2]\n      array([[[4],\n              [5],\n              [6]]])\n\n- :const:`Ellipsis` expands to the number of ``:`` objects needed for the\n  selection tuple to index all dimensions. In most cases, this means that\n  length of the expanded selection tuple is ``x.ndim``. There may only be a\n  single ellipsis present.\n\n  .. admonition:: Example\n\n      >>> x[...,0]\n      array([[1, 2, 3],\n             [4, 5, 6]])\n\n- Each :const:`newaxis` object in the selection tuple serves to expand\n  the dimensions of the resulting selection by one unit-length\n  dimension.  The added dimension is the position of the :const:`newaxis`\n  object in the selection tuple.\n\n  .. admonition:: Example\n\n      >>> x[:,np.newaxis,:,:].shape\n      (2, 1, 3, 1)\n\n- An integer, *i*, returns the same values as ``i:i+1``\n  **except** the dimensionality of the returned object is reduced by\n  1. In particular, a selection tuple with the *p*-th\n  element an integer (and all other entries ``:``) returns the\n  corresponding sub-array with dimension *N - 1*. If *N = 1*\n  then the returned object is an scalar `ndarray` whose `ndim=0`.\n\n- If the selection tuple has all entries ``:`` except the\n  *p*-th entry which is a slice object ``i:j:k``,\n  then the returned array has dimension *N* formed by\n  concatenating the sub-arrays returned by integer indexing of\n  elements *i*, *i+k*, ..., *i + (m - 1) k < j*,\n\n- Basic slicing with more than one non-``:`` entry in the slicing\n  tuple, acts like repeated application of slicing using a single\n  non-``:`` entry, where the non-``:`` entries are successively taken\n  (with all other non-``:`` entries replaced by ``:``). Thus,\n  ``x[ind1,...,ind2,:]`` acts like ``x[ind1][...,ind2,:]`` under basic\n  slicing.\n\n  .. warning:: The above is **not** true for advanced indexing.\n\n- You may use slicing to set values in the array, but (unlike lists) you\n  can never grow the array. The size of the value to be set in\n  ``x[obj] = value`` must be (broadcastable) to the same shape as\n  ``x[obj]``.\n\n.. note::\n\n    Remember that a slicing tuple can always be constructed as *obj*\n    and used in the ``x[obj]`` notation. Slice objects can be used in\n    the construction in place of the ``[start:stop:step]``\n    notation. For example, ``x[1:10:5,::-1]`` can also be implemented\n    as ``obj = (slice(1,10,5), slice(None,None,-1)); x[obj]`` . This\n    can be useful for constructing generic code that works on arrays\n    of arbitrary dimension.\n\n.. data:: newaxis\n   :noindex:\n\n   The :const:`newaxis` object can be used in all slicing operations to\n   create an axis of length one. :const:`newaxis` is an alias for\n   'None', and 'None' can be used in place of this with the same result.\n\n\nAdvanced Indexing\n-----------------\n\nAdvanced indexing is triggered when the selection object, *obj*, is a\nnon-tuple sequence object, an :class:`ndarray` (of data type integer or bool),\nor a tuple with at least one sequence object or ndarray (of data type\ninteger or bool). There are two types of advanced indexing: integer\nand Boolean.\n\nAdvanced indexing always returns a *copy* of the data (contrast with\nsome cases in basic slicing that returns a view).\n\n.. warning::\n\n   The definition of advanced indexing means that ``x[(1,2,3),]`` is\n   fundamentally different than ``x[(1,2,3)]``. The latter is\n   equivalent to ``x[1,2,3]`` which will trigger basic selection while\n   the former will trigger advanced indexing. Be sure to understand\n   why this occurs.\n\n   Also recognize that ``x[[1,2,3]]`` will trigger advanced indexing,\n   whereas due to the deprecated Numeric compatibility mentioned above,\n   ``x[[1,2,slice(None)]]`` will trigger basic slicing in the official NumPy\n   which is not currently supported in MXNet `numpy` module.\n\nInteger array indexing\n^^^^^^^^^^^^^^^^^^^^^^\n\nInteger array indexing allows selection of arbitrary items in the array\nbased on their *N*-dimensional index. Each integer array represents a number\nof indexes into that dimension.\n\nPurely integer array indexing\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nWhen the index consists of as many integer arrays as the array being indexed\nhas dimensions, the indexing is straight forward, but different from slicing.\n\nAdvanced indexes always are broadcasting and\niterated as *one*::\n\n     result[i_1, ..., i_M] == x[ind_1[i_1, ..., i_M], ind_2[i_1, ..., i_M],\n                                ..., ind_N[i_1, ..., i_M]]\n\nNote that the result shape is identical to the (broadcast) indexing array\nshapes ``ind_1, ..., ind_N``.\n\n.. admonition:: Example\n\n    From each row, a specific element should be selected. The row index is just\n    ``[0, 1, 2]`` and the column index specifies the element to choose for the\n    corresponding row, here ``[0, 1, 0]``. Using both together the task\n    can be solved using advanced indexing:\n\n    >>> x = np.array([[1, 2], [3, 4], [5, 6]])\n    >>> x[[0, 1, 2], [0, 1, 0]]\n    array([1, 4, 5])\n\nCombining advanced and basic indexing\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nWhen there is at least one slice (``:``), ellipsis (``...``) or :const:`newaxis`\nin the index (or the array has more dimensions than there are advanced indexes),\nthen the behaviour can be more complicated. It is like concatenating the\nindexing result for each advanced index element\n\nIn the simplest case, there is only a *single* advanced index. A single\nadvanced index can for example replace a slice and the result array will be\nthe same, however, it is a copy and may have a different memory layout.\nA slice is preferable when it is possible.\n\n.. admonition:: Example\n\n    >>> x[1:2, 1:3]\n    array([[4, 5]])\n    >>> x[1:2, [1, 2]]\n    array([[4, 5]])\n\nThe easiest way to understand the situation may be to think in\nterms of the result shape. There are two parts to the indexing operation,\nthe subspace defined by the basic indexing (excluding integers) and the\nsubspace from the advanced indexing part. Two cases of index combination\nneed to be distinguished:\n\n* The advanced indexes are separated by a slice, :const:`Ellipsis` or :const:`newaxis`.\n  For example ``x[arr1, :, arr2]``.\n* The advanced indexes are all next to each other.\n  For example ``x[..., arr1, arr2, :]`` but *not* ``x[arr1, :, 1]``\n  since ``1`` is an advanced index in this regard.\n\nIn the first case, the dimensions resulting from the advanced indexing\noperation come first in the result array, and the subspace dimensions after\nthat.\nIn the second case, the dimensions from the advanced indexing operations\nare inserted into the result array at the same spot as they were in the\ninitial array (the latter logic is what makes simple advanced indexing\nbehave just like slicing).\n\n.. admonition:: Example\n\n Suppose ``x.shape`` is (10,20,30) and ``ind`` is a (2,3,4)-shaped\n indexing :class:`intp` array, then ``result = x[...,ind,:]`` has\n shape (10,2,3,4,30) because the (20,)-shaped subspace has been\n replaced with a (2,3,4)-shaped broadcasted indexing subspace. If\n we let *i, j, k* loop over the (2,3,4)-shaped subspace then\n ``result[...,i,j,k,:] = x[...,ind[i,j,k],:]``. This example\n produces the same result as :meth:`x.take(ind, axis=-2) <ndarray.take>`.\n\n.. admonition:: Example\n\n  Let ``x.shape`` be (10,20,30,40,50) and suppose ``ind_1``\n  and ``ind_2`` can be broadcast to the shape (2,3,4).  Then\n  ``x[:,ind_1,ind_2]`` has shape (10,2,3,4,40,50) because the\n  (20,30)-shaped subspace from X has been replaced with the\n  (2,3,4) subspace from the indices.  However,\n  ``x[:,ind_1,:,ind_2]`` has shape (2,3,4,10,30,50) because there\n  is no unambiguous place to drop in the indexing subspace, thus\n  it is tacked-on to the beginning. It is always possible to use\n  :meth:`.transpose() <ndarray.transpose>` to move the subspace\n  anywhere desired. Note that this example cannot be replicated\n  using :func:`take`.\n\n\nBoolean array indexing\n^^^^^^^^^^^^^^^^^^^^^^\n\nThis advanced indexing occurs when obj is an array object of Boolean\ntype, such as may be returned from comparison operators. A single\nboolean index array is practically identical to ``x[obj.nonzero()]`` where,\nas described above, :meth:`obj.nonzero() <ndarray.nonzero>` returns a\ntuple (of length :attr:`obj.ndim <ndarray.ndim>`) of integer index\narrays showing the :const:`True` elements of *obj*. However, it is\nfaster when ``obj.shape == x.shape``.\n\nIf ``obj.ndim == x.ndim``, ``x[obj]`` returns a 1-dimensional array\nfilled with the elements of *x* corresponding to the :const:`True`\nvalues of *obj*.  The search order will be row-major,\nC-style. If *obj* has :const:`True` values at entries that are outside\nof the bounds of *x*, then an index error will be raised. If *obj* is\nsmaller than *x* it is identical to filling it with :const:`False`.\n\n.. note::\n\n    Boolean indexing currently only supports a single boolean ndarray as a index.\n    An composite index including a boolean array is not supported for now.\n\nIf there is only one Boolean array and no integer indexing array present,\nthis is straight forward. Care must only be taken to make sure that the\nboolean index has *exactly* as many dimensions as it is supposed to work\nwith.\n\n.. admonition:: Example\n\n    From an array, select all rows which sum up to less or equal two:\n\n    >>> x = np.array([[0, 1], [1, 1], [2, 2]], dtype=np.int32)\n    >>> rowsum = x.sum(-1)\n    >>> x[rowsum <= 2]\n    array([[0, 1],\n           [1, 1]], dtype=int32)\n\n    But if ``rowsum`` would have two dimensions as well:\n\n    >>> rowsum = x.sum(-1, keepdims=True)\n    >>> rowsum.shape\n    (3, 1)\n    >>> x[rowsum <= 2]  # fail\n    IndexError: boolean index did not match indexed array along dimension 1\n\nDetailed notes\n--------------\n\nThese are some detailed notes, which are not of importance for day to day\nindexing (in no particular order):\n\n* For advanced assignments, there is in general no guarantee for the\n  iteration order. This means that if an element is set more than once,\n  it is not possible to predict the final result.\n* An empty (tuple) index is a full scalar index into a zero dimensional array.\n  ``x[()]`` returns a *scalar* `ndarray` if ``x`` has zero dimensions.\n  On the other hand ``x[...]`` always returns a view.\n* If a zero dimensional array is present in the index *and* it is *not considered as* a full\n  integer index as in NumPy. Advanced indexing is not triggered.\n* the ``nonzero`` equivalence for Boolean arrays does not hold for zero\n  dimensional boolean arrays.\n* When the result of an advanced indexing operation has no elements but an\n  individual index is out of bounds, currently no ``IndexError`` is\n  raised as in NumPy.\n\n.. index::\n   single: indexing\n   single: ndarray\n"
  },
  {
    "path": "docs/python_docs/python/api/np/arrays.ndarray.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\n.. _arrays.ndarray:\n\n******************************************\nThe N-dimensional array (:class:`ndarray`)\n******************************************\n\n.. currentmodule:: mxnet.np\n\nAn :class:`ndarray` is a (usually fixed-size) multidimensional\ncontainer of items of the same type and size. The number of dimensions\nand items in an array is defined by its :attr:`shape <ndarray.shape>`,\nwhich is a :class:`tuple` of *N* non-negative integers that specify the\nsizes of each dimension. The type of items in the array is specified by\na separate data-type object (dtype), one of which\nis associated with each ndarray.\n\nAs with other container objects in Python, the contents of an\n:class:`ndarray` can be accessed and modified by :ref:`indexing or\nslicing <arrays.indexing>` the array (using, for example, *N* integers),\nand via the methods and attributes of the :class:`ndarray`.\n\n.. index:: view, base\n\nDifferent :class:`ndarrays <ndarray>` can share the same data, so that\nchanges made in one :class:`ndarray` may be visible in another. That\nis, an ndarray can be a *\"view\"* to another ndarray, and the data it\nis referring to is taken care of by the *\"base\"* ndarray.\n\n\n.. admonition:: Example\n\n   A 2-dimensional array of size 2 x 3, composed of 4-byte integer\n   elements:\n\n   >>> x = np.array([[1, 2, 3], [4, 5, 6]], np.int32)\n   >>> type(x)\n   <class 'mxnet.numpy.ndarray'>\n   >>> x.shape\n   (2, 3)\n   >>> x.dtype\n   dtype('int32')\n\n   The array can be indexed using Python container-like syntax:\n\n   >>> # The element of x in the *second* row, *third* column, namely, 6.\n   >>> x[1, 2]\n   array(6, dtype=int32)  # this is different than the official NumPy which returns a np.int32 object\n\n   For example :ref:`slicing <arrays.indexing>` can produce views of\n   the array if the elements to be sliced is continguous in memory:\n\n   >>> y = x[1,:]\n   >>> y\n   array([9, 5, 6], dtype=int32)  # this also changes the corresponding element in x\n   >>> x\n   array([[1, 2, 3],\n           [9, 5, 6]], dtype=int32)\n\n\nConstructing arrays\n===================\n\nNew arrays can be constructed using the routines detailed in\n:ref:`routines.array-creation`, and also by using the low-level\n:class:`ndarray` constructor:\n\n.. autosummary::\n\n   ndarray\n\n\nIndexing arrays\n===============\n\nArrays can be indexed using an extended Python slicing syntax,\n``array[selection]``.\n\n.. seealso:: :ref:`Array Indexing <arrays.indexing>`.\n\n.. _memory-layout:\n\nInternal memory layout of an ndarray\n====================================\n\nAn instance of class :class:`ndarray` consists of a contiguous\none-dimensional segment of computer memory (owned by the array, or by\nsome other object), combined with an indexing scheme that maps *N*\nintegers into the location of an item in the block.  The ranges in\nwhich the indices can vary is specified by the :obj:`shape\n<ndarray.shape>` of the array. How many bytes each item takes and how\nthe bytes are interpreted is defined by the data-type object\nassociated with the array.\n\n.. index:: C-order, Fortran-order, row-major, column-major, stride,\n  offset\n\n.. note::\n\n    `mxnet.numpy.ndarray` currently only supports storing elements in\n    C-order/row-major and contiguous memory space. The following content\n    on explaining a variety of memory layouts of an ndarray\n    are copied from the official NumPy documentation as a comprehensive reference.\n\nA segment of memory is inherently 1-dimensional, and there are many\ndifferent schemes for arranging the items of an *N*-dimensional array\nin a 1-dimensional block. NumPy is flexible, and :class:`ndarray`\nobjects can accommodate any *strided indexing scheme*. In a strided\nscheme, the N-dimensional index :math:`(n_0, n_1, ..., n_{N-1})`\ncorresponds to the offset (in bytes):\n\n.. math:: n_{\\mathrm{offset}} = \\sum_{k=0}^{N-1} s_k n_k\n\nfrom the beginning of the memory block associated with the\narray. Here, :math:`s_k` are integers which specify the :obj:`strides\n<ndarray.strides>` of the array. The column-major order (used,\nfor example, in the Fortran language and in *Matlab*) and\nrow-major order (used in C) schemes are just specific kinds of\nstrided scheme, and correspond to memory that can be *addressed* by the strides:\n\n.. math::\n\n   s_k^{\\mathrm{column}} = \\mathrm{itemsize} \\prod_{j=0}^{k-1} d_j ,\n   \\quad  s_k^{\\mathrm{row}} = \\mathrm{itemsize} \\prod_{j=k+1}^{N-1} d_j .\n\n.. index:: single-segment, contiguous, non-contiguous\n\nwhere :math:`d_j` `= self.shape[j]`.\n\nBoth the C and Fortran orders are contiguous, *i.e.,*\nsingle-segment, memory layouts, in which every part of the\nmemory block can be accessed by some combination of the indices.\n\nWhile a C-style and Fortran-style contiguous array, which has the corresponding\nflags set, can be addressed with the above strides, the actual strides may be\ndifferent. This can happen in two cases:\n\n    1. If ``self.shape[k] == 1`` then for any legal index ``index[k] == 0``.\n       This means that in the formula for the offset :math:`n_k = 0` and thus\n       :math:`s_k n_k = 0` and the value of :math:`s_k` `= self.strides[k]` is\n       arbitrary.\n    2. If an array has no elements (``self.size == 0``) there is no legal\n       index and the strides are never used. Any array with no elements may be\n       considered C-style and Fortran-style contiguous.\n\nPoint 1. means that ``self`` and ``self.squeeze()`` always have the same\ncontiguity and ``aligned`` flags value. This also means\nthat even a high dimensional array could be C-style and Fortran-style\ncontiguous at the same time.\n\n.. index:: aligned\n\nAn array is considered aligned if the memory offsets for all elements and the\nbase offset itself is a multiple of `self.itemsize`. Understanding\n`memory-alignment` leads to better performance on most hardware.\n\n.. note::\n\n    Points (1) and (2) are not yet applied by default. Beginning with\n    NumPy 1.8.0, they are applied consistently only if the environment\n    variable ``NPY_RELAXED_STRIDES_CHECKING=1`` was defined when NumPy\n    was built. Eventually this will become the default.\n\n    You can check whether this option was enabled when your NumPy was\n    built by looking at the value of ``np.ones((10,1),\n    order='C').flags.f_contiguous``. If this is ``True``, then your\n    NumPy has relaxed strides checking enabled.\n\n.. warning::\n\n    It does *not* generally hold that ``self.strides[-1] == self.itemsize``\n    for C-style contiguous arrays or ``self.strides[0] == self.itemsize`` for\n    Fortran-style contiguous arrays is true.\n\nData in new :class:`ndarrays <ndarray>` is in the row-major\n(C) order, unless otherwise specified, but, for example, :ref:`basic\narray slicing <arrays.indexing>` often produces views\nin a different scheme.\n\n.. seealso: :ref:`Indexing <arrays.ndarray.indexing>`_\n\n.. note::\n\n   Several algorithms in NumPy work on arbitrarily strided arrays.\n   However, some algorithms require single-segment arrays. When an\n   irregularly strided array is passed in to such algorithms, a copy\n   is automatically made.\n\n.. _arrays.ndarray.attributes:\n\nArray attributes\n================\n\nArray attributes reflect information that is intrinsic to the array\nitself. Generally, accessing an array through its attributes allows\nyou to get and sometimes set intrinsic properties of the array without\ncreating a new array. The exposed attributes are the core parts of an\narray and only some of them can be reset meaningfully without creating\na new array. Information on each attribute is given below.\n\nMemory layout\n-------------\n\nThe following attributes contain information about the memory layout\nof the array:\n\n.. autosummary::\n\n   ndarray.shape\n   ndarray.ndim\n   ndarray.size\n\nData type\n---------\n\nThe data type object associated with the array can be found in the\n:attr:`dtype <ndarray.dtype>` attribute:\n\n.. autosummary::\n\n   ndarray.dtype\n\n.. _array.ndarray.methods:\n\nArray methods\n=============\n\nAn :class:`ndarray` object has many methods which operate on or with\nthe array in some fashion, typically returning an array result. These\nmethods are briefly explained below. (Each method's docstring has a\nmore complete description.)\n\nFor the following methods there are also corresponding functions in\n:mod:`numpy`: :func:`all`, :func:`any`, :func:`argmax`,\n:func:`argmin`, :func:`argpartition`, :func:`argsort`, :func:`choose`,\n:func:`clip`, :func:`compress`, :func:`copy`, :func:`cumprod`,\n:func:`cumsum`, :func:`diagonal`, :func:`imag`, :func:`max <amax>`,\n:func:`mean`, :func:`min <amin>`, :func:`nonzero`, :func:`partition`,\n:func:`prod`, :func:`ptp`, :func:`put`, :func:`ravel`, :func:`real`,\n:func:`repeat`, :func:`reshape`, :func:`round <around>`,\n:func:`searchsorted`, :func:`sort`, :func:`squeeze`, :func:`std`,\n:func:`sum`, :func:`swapaxes`, :func:`take`, :func:`trace`,\n:func:`transpose`, :func:`var`.\n\nArray conversion\n----------------\n\n.. autosummary::\n\n   ndarray.item\n   ndarray.copy\n   ndarray.tolist\n   ndarray.astype\n\n\nShape manipulation\n------------------\n\nFor reshape, resize, and transpose, the single tuple argument may be\nreplaced with ``n`` integers which will be interpreted as an n-tuple.\n\n.. autosummary::\n\n   ndarray.reshape\n   ndarray.transpose\n   ndarray.swapaxes\n   ndarray.flatten\n   ndarray.squeeze\n\nItem selection and manipulation\n-------------------------------\n\nFor array methods that take an *axis* keyword, it defaults to\n:const:`None`. If axis is *None*, then the array is treated as a 1-D\narray. Any other value for *axis* represents the dimension along which\nthe operation should proceed.\n\n.. autosummary::\n\n   ndarray.nonzero\n   ndarray.take\n   ndarray.repeat\n   ndarray.argsort\n   ndarray.sort\n\nCalculation\n-----------\n\n.. index:: axis\n\nMany of these methods take an argument named *axis*. In such cases,\n\n- If *axis* is *None* (the default), the array is treated as a 1-D\n  array and the operation is performed over the entire array. This\n  behavior is also the default if self is a 0-dimensional array or\n  array scalar. (An array scalar is an instance of the types/classes\n  float32, float64, etc., whereas a 0-dimensional array is an ndarray\n  instance containing precisely one array scalar.)\n\n- If *axis* is an integer, then the operation is done over the given\n  axis (for each 1-D subarray that can be created along the given axis).\n\n.. admonition:: Example of the *axis* argument\n\n   A 3-dimensional array of size 3 x 3 x 3, summed over each of its\n   three axes\n\n   >>> x\n   array([[[ 0,  1,  2],\n           [ 3,  4,  5],\n           [ 6,  7,  8]],\n          [[ 9, 10, 11],\n           [12, 13, 14],\n           [15, 16, 17]],\n          [[18, 19, 20],\n           [21, 22, 23],\n           [24, 25, 26]]])\n   >>> x.sum(axis=0)\n   array([[27, 30, 33],\n          [36, 39, 42],\n          [45, 48, 51]])\n   >>> # for sum, axis is the first keyword, so we may omit it,\n   >>> # specifying only its value\n   >>> x.sum(0), x.sum(1), x.sum(2)\n   (array([[27, 30, 33],\n           [36, 39, 42],\n           [45, 48, 51]]),\n    array([[ 9, 12, 15],\n           [36, 39, 42],\n           [63, 66, 69]]),\n    array([[ 3, 12, 21],\n           [30, 39, 48],\n           [57, 66, 75]]))\n\nThe parameter *dtype* specifies the data type over which a reduction\noperation (like summing) should take place. The default reduce data\ntype is the same as the data type of *self*. To avoid overflow, it can\nbe useful to perform the reduction using a larger data type.\n\nFor several methods, an optional *out* argument can also be provided\nand the result will be placed into the output array given. The *out*\nargument must be an :class:`ndarray` and have the same number of\nelements. It can have a different data type in which case casting will\nbe performed.\n\n.. autosummary::\n\n   ndarray.max\n   ndarray.argmax\n   ndarray.min\n   ndarray.argmin\n   ndarray.clip\n   ndarray.sum\n   ndarray.mean\n   ndarray.prod\n   ndarray.cumsum\n   ndarray.var\n   ndarray.std\n   ndarray.round\n   ndarray.all\n   ndarray.any\n\nArithmetic, matrix multiplication, and comparison operations\n============================================================\n\n.. index:: comparison, arithmetic, matrix, operation, operator\n\nArithmetic and comparison operations on :class:`ndarrays <ndarray>`\nare defined as element-wise operations, and generally yield\n:class:`ndarray` objects as results.\n\nEach of the arithmetic operations (``+``, ``-``, ``*``, ``/``, ``//``,\n``%``, ``divmod()``, ``**`` or ``pow()``, ``<<``, ``>>``, ``&``,\n``^``, ``|``, ``~``) and the comparisons (``==``, ``<``, ``>``,\n``<=``, ``>=``, ``!=``) is equivalent to the corresponding\nuniversal function (or ufunc for short) in NumPy.\n\nComparison operators:\n\n.. autosummary::\n\n   ndarray.__lt__\n   ndarray.__le__\n   ndarray.__gt__\n   ndarray.__ge__\n   ndarray.__eq__\n   ndarray.__ne__\n\nTruth value of an array (:func:`bool()`):\n\n.. autosummary::\n\n   ndarray.__bool__\n\n.. note::\n\n   Truth-value testing of an array invokes\n   :meth:`ndarray.__bool__`, which raises an error if the number of\n   elements in the array is larger than 1, because the truth value\n   of such arrays is ambiguous.\n\n\nUnary operations:\n\n.. autosummary::\n\n   ndarray.__neg__\n   ndarray.__abs__\n   ndarray.__invert__\n\nArithmetic:\n\n.. autosummary::\n\n   ndarray.__add__\n   ndarray.__sub__\n   ndarray.__mul__\n   ndarray.__truediv__\n   ndarray.__mod__\n   ndarray.__pow__\n   ndarray.__and__\n   ndarray.__or__\n   ndarray.__xor__\n\n.. note::\n\n   - Any third argument to :func:`pow()` is silently ignored,\n     as the underlying :func:`ufunc <power>` takes only two arguments.\n\n   - The three division operators are all defined; :obj:`div` is active\n     by default, :obj:`truediv` is active when\n     :obj:`__future__` division is in effect.\n\n   - Because :class:`ndarray` is a built-in type (written in C), the\n     ``__r{op}__`` special methods are not directly defined.\n\n   - The functions called to implement many arithmetic special methods\n     for arrays can be modified using :class:`__array_ufunc__ <numpy.class.__array_ufunc__>`.\n\nArithmetic, in-place:\n\n.. autosummary::\n\n   ndarray.__iadd__\n   ndarray.__isub__\n   ndarray.__imul__\n   ndarray.__itruediv__\n   ndarray.__imod__\n   ndarray.__iand__\n   ndarray.__ior__\n   ndarray.__ixor__\n\n\n.. warning::\n\n   In place operations will perform the calculation using the\n   precision decided by the data type of the two operands, but will\n   silently downcast the result (if necessary) so it can fit back into\n   the array.  Therefore, for mixed precision calculations,\n   ``A {op}= B`` can be different than ``A = A {op} B``. For example, suppose\n   ``a = ones((3,3))``. Then, ``a += 3j`` is different than ``a = a + 3j``:\n   while they both perform the same computation, ``a += 3``\n   casts the result to fit back in ``a``, whereas ``a = a + 3j``\n   re-binds the name ``a`` to the result.\n\n\nMatrix Multiplication:\n\n\n.. autosummary::\n\n   ndarray.__matmul__\n\n\nSpecial methods\n===============\n\nFor standard library functions:\n\n.. autosummary::\n\n   ndarray.__reduce__\n   ndarray.__setstate__\n\nBasic customization:\n\n.. autosummary::\n\n   ndarray.__new__\n\nContainer customization: (see :ref:`Indexing <arrays.indexing>`)\n\n.. autosummary::\n\n   ndarray.__len__\n   ndarray.__getitem__\n   ndarray.__setitem__\n\nConversion; the operations :func:`index()`, :func:`int()` and :func:`float()`.\nThey work only on arrays that have one element in them\nand return the appropriate scalar.\n\n.. autosummary::\n\n   ndarray.__index__\n   ndarray.__int__\n   ndarray.__float__\n\nString representations:\n\n.. autosummary::\n\n   ndarray.__str__\n   ndarray.__repr__\n"
  },
  {
    "path": "docs/python_docs/python/api/np/arrays.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\n.. _arrays:\n\n*************\nArray objects\n*************\n\n.. currentmodule:: mxnet.np\n\n``np`` provides an N-dimensional array type, the :ref:`ndarray\n<arrays.ndarray>`, which describes a collection of \"items\" of the same\ntype. The items can be :ref:`indexed <arrays.indexing>` using for\nexample N integers.\n\nAll ndarrays are homogenous: every item takes up the same size\nblock of memory, and all blocks are interpreted in exactly the same\nway. How each item in the array is to be interpreted is specified by a\nseparate data-type object, one of which is associated\nwith every array. In addition to basic types (integers, floats,\n*etc.*), the data type objects can also represent data structures.\n\nAn item extracted from an array, *e.g.*, by indexing, is represented\nby a Python object whose type is one of the array scalar types\nbuilt in NumPy. The array scalars allow easy manipulation\nof also more complicated arrangements of data.\n\n.. note::\n\n   A major difference to ``numpy.ndarray`` is that ``mxnet.np.ndarray``'s scalar\n   is a 0-dim ndarray instead of a scalar object (``numpy.generic``).\n\n.. toctree::\n   :maxdepth: 2\n\n   arrays.ndarray\n   arrays.indexing\n"
  },
  {
    "path": "docs/python_docs/python/api/np/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\n.. _reference:\n\nmxnet.np\n========\n\n\n.. module:: mxnet.np\n\nThis section contains the `mxnet.np` API reference documentation. The topics here explain the functions, modules, and objects\nincluded in `mxnet.np`. Use the links here to learn more.\n\n\n.. toctree::\n   :maxdepth: 2\n\n   arrays\n   routines\n\n\n**Acknowledgements**\n\nLarge parts of this manual originate from NumPy documents.\n"
  },
  {
    "path": "docs/python_docs/python/api/np/random/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\n.. _numpyrandom:\n\n.. currentmodule:: mxnet.np.random\n\nnp.random\n=========\n\n\nSimple random data\n------------------\n.. autosummary::\n   :toctree: generated/\n\n   choice\n\nPermutations\n------------\n.. autosummary::\n   :toctree: generated/\n\n   shuffle\n\nDistributions\n-------------\n.. autosummary::\n   :toctree: generated/\n\n   normal\n   uniform\n   rand\n   randint\n   beta\n   chisquare\n   exponential\n   f\n   gamma\n   gumbel\n   laplace\n   logistic\n   lognormal\n   multinomial\n   multivariate_normal\n   pareto\n   power\n   rayleigh\n   weibull\n"
  },
  {
    "path": "docs/python_docs/python/api/np/routines.array-creation.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\n.. _routines.array-creation:\n\nArray creation routines\n=======================\n\n.. currentmodule:: mxnet.np\n\nOnes and zeros\n--------------\n.. autosummary::\n   :toctree: generated/\n\n   eye\n   empty\n   full\n   identity\n   ones\n   ones_like\n   zeros\n   zeros_like\n\n.. code::\n\n   full_like\n   empty_like\n\nFrom existing data\n------------------\n.. autosummary::\n   :toctree: generated/\n\n   array\n   copy\n\n.. code::\n\n   frombuffer\n   fromfunction\n   fromiter\n   fromstring\n   loadtxt\n\n.. _routines.array-creation.rec:\n\nCreating record arrays (:mod:`np.rec`)\n-----------------------------------------\n\n.. note:: :mod:`np.rec` is the preferred alias for\n   :mod:`np.core.records`.\n\n.. autosummary::\n   :toctree: generated/\n\n.. code::\n\n   core.records.array\n   core.records.fromarrays\n   core.records.fromrecords\n   core.records.fromstring\n   core.records.fromfile\n\n.. _routines.array-creation.char:\n\nCreating character arrays (:mod:`np.char`)\n---------------------------------------------\n\n.. note:: :mod:`np.char` is the preferred alias for\n   :mod:`np.core.defchararray`.\n\n.. autosummary::\n   :toctree: generated/\n\n.. code::\n\n   core.defchararray.array\n   core.defchararray.asarray\n\nNumerical ranges\n----------------\n.. autosummary::\n   :toctree: generated/\n\n   arange\n   linspace\n   logspace\n   meshgrid\n\n.. code::\n\n   geomspace\n   mgrid\n   ogrid\n\nBuilding matrices\n-----------------\n.. autosummary::\n   :toctree: generated/\n\n   tril\n\n.. code::\n\n   diag\n   diagflat\n   tri\n   triu\n   vander\n"
  },
  {
    "path": "docs/python_docs/python/api/np/routines.array-manipulation.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nArray manipulation routines\n***************************\n\n.. currentmodule:: mxnet.np\n\nChanging array shape\n====================\n.. autosummary::\n   :toctree: generated/\n\n\n   reshape\n   ravel\n   ndarray.flatten\n\nTranspose-like operations\n=========================\n.. autosummary::\n   :toctree: generated/\n\n   swapaxes\n   ndarray.T\n   transpose\n   moveaxis\n   rollaxis\n\nChanging number of dimensions\n=============================\n.. autosummary::\n   :toctree: generated/\n\n   expand_dims\n   squeeze\n   broadcast_to\n   broadcast_arrays\n   atleast_1d\n   atleast_2d\n   atleast_3d\n\nJoining arrays\n==============\n.. autosummary::\n   :toctree: generated/\n\n   concatenate\n   stack\n   dstack\n   vstack\n   column_stack\n   hstack\n\nSplitting arrays\n================\n.. autosummary::\n   :toctree: generated/\n\n   split\n   hsplit\n   vsplit\n   array_split\n   dsplit\n\nTiling arrays\n=============\n.. autosummary::\n   :toctree: generated/\n\n   tile\n   repeat\n\nAdding and removing elements\n============================\n.. autosummary::\n   :toctree: generated/\n\n   unique\n   delete\n   insert\n   append\n   resize\n   trim_zeros\n\nRearranging elements\n====================\n.. autosummary::\n   :toctree: generated/\n\n   reshape\n   flip\n   roll\n   rot90\n   fliplr\n   flipud\n"
  },
  {
    "path": "docs/python_docs/python/api/np/routines.io.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nInput and output\n****************\n\n.. currentmodule:: mxnet.np\n\nThe format of these binary file types is documented in\n:py:mod:`numpy.lib.format`\n\nText files\n----------\n.. autosummary::\n   :toctree: generated/\n\n   genfromtxt\n   ndarray.tolist\n\nText formatting options\n-----------------------\n.. autosummary::\n   :toctree: generated/\n\n   set_printoptions\n"
  },
  {
    "path": "docs/python_docs/python/api/np/routines.linalg.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\n.. _routines.linalg:\n\n.. module:: mxnet.np.linalg\n\nLinear algebra (:mod:`numpy.linalg`)\n************************************\n\nThe NumPy linear algebra functions rely on BLAS and LAPACK to provide efficient\nlow level implementations of standard linear algebra algorithms. Those\nlibraries may be provided by NumPy itself using C versions of a subset of their\nreference implementations but, when possible, highly optimized libraries that\ntake advantage of specialized processor functionality are preferred. Examples\nof such libraries are OpenBLAS_, MKL (TM), and ATLAS. Because those libraries\nare multithreaded and processor dependent, environmental variables and external\npackages such as threadpoolctl_ may be needed to control the number of threads\nor specify the processor architecture.\n\n.. _OpenBLAS: https://www.openblas.net/\n.. _threadpoolctl: https://github.com/joblib/threadpoolctl\n\n.. currentmodule:: mxnet.np\n\nMatrix and vector products\n--------------------------\n.. autosummary::\n   :toctree: generated/\n\n   dot\n   vdot\n   inner\n   outer\n   tensordot\n   einsum\n   linalg.multi_dot\n   matmul\n   linalg.matrix_power\n   kron\n\nDecompositions\n--------------\n.. autosummary::\n   :toctree: generated/\n\n   linalg.svd\n   linalg.cholesky\n   linalg.qr\n\nMatrix eigenvalues\n------------------\n.. autosummary::\n   :toctree: generated/\n\n   linalg.eig\n   linalg.eigh\n   linalg.eigvals\n   linalg.eigvalsh\n\nNorms and other numbers\n-----------------------\n.. autosummary::\n   :toctree: generated/\n\n   linalg.norm\n   trace\n   linalg.cond\n   linalg.det\n   linalg.matrix_rank\n   linalg.slogdet\n\nSolving equations and inverting matrices\n----------------------------------------\n.. autosummary::\n   :toctree: generated/\n\n   linalg.solve\n   linalg.tensorsolve\n   linalg.lstsq\n   linalg.inv\n   linalg.pinv\n   linalg.tensorinv\n"
  },
  {
    "path": "docs/python_docs/python/api/np/routines.math.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nMathematical functions\n**********************\n\n.. currentmodule:: mxnet.np\n\n.. note::\n\n   Currently, most of the math functions only support inputs and outputs of the same dtype.\n   This limitation usually results in imprecise outputs for ndarrays with integral dtype\n   while floating-point values are expected in the output.\n   Appropriate handling of ndarrays integral dtypes is in active development.\n\n\nTrigonometric functions\n-----------------------\n.. autosummary::\n   :toctree: generated/\n\n   sin\n   cos\n   tan\n   arcsin\n   arccos\n   arctan\n   degrees\n   radians\n   hypot\n   arctan2\n   deg2rad\n   rad2deg\n   unwrap\n\n\nHyperbolic functions\n--------------------\n.. autosummary::\n   :toctree: generated/\n\n   sinh\n   cosh\n   tanh\n   arcsinh\n   arccosh\n   arctanh\n\n\nRounding\n--------\n.. autosummary::\n   :toctree: generated/\n\n   rint\n   fix\n   floor\n   ceil\n   trunc\n   around\n   round_\n\n\nSums, products, differences\n---------------------------\n.. autosummary::\n   :toctree: generated/\n\n   sum\n   prod\n   cumsum\n   nanprod\n   nansum\n   cumprod\n   nancumprod\n   nancumsum\n   diff\n   ediff1d\n   cross\n   trapz\n\n\nExponents and logarithms\n------------------------\n.. autosummary::\n   :toctree: generated/\n\n   exp\n   expm1\n   log\n   log10\n   log2\n   log1p\n   logaddexp\n\n\nOther special functions\n-----------------------\n.. autosummary::\n   :toctree: generated/\n\n   i0\n\n\nFloating point routines\n-----------------------\n.. autosummary::\n   :toctree: generated/\n\n   ldexp\n   signbit\n   copysign\n   frexp\n   spacing\n\n\nRational routines\n-----------------\n.. autosummary::\n   :toctree: generated/\n\n   lcm\n   gcd\n\n\nArithmetic operations\n---------------------\n.. autosummary::\n   :toctree: generated/\n\n   add\n   reciprocal\n   negative\n   divide\n   power\n   subtract\n   mod\n   multiply\n   true_divide\n   remainder\n   positive\n   float_power\n   fmod\n   modf\n   divmod\n   floor_divide\n\n\nMiscellaneous\n-------------\n.. autosummary::\n   :toctree: generated/\n\n   clip\n   sqrt\n   cbrt\n   square\n   absolute\n   sign\n   maximum\n   minimum\n   fabs\n   heaviside\n   fmax\n   fmin\n   nan_to_num\n   interp\n\n"
  },
  {
    "path": "docs/python_docs/python/api/np/routines.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nRoutines\n========\n\nIn this chapter routine docstrings are presented, grouped by functionality.\nMany docstrings contain example code, which demonstrates basic usage\nof the routine. The examples assume that the `np` module is imported with::\n\n  >>> from mxnet import np, npx\n  >>> npx.set_np()\n\nA convenient way to execute examples is the ``%doctest_mode`` mode of\nIPython, which allows for pasting of multi-line examples and preserves\nindentation.\n\n.. toctree::\n   :maxdepth: 2\n\n   routines.array-creation\n   routines.array-manipulation\n   routines.io\n   routines.linalg\n   routines.math\n   random/index\n   routines.sort\n   routines.statistics\n"
  },
  {
    "path": "docs/python_docs/python/api/np/routines.sort.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nSorting, searching, and counting\n================================\n\n.. currentmodule:: mxnet.np\n\nSorting\n-------\n.. autosummary::\n   :toctree: generated/\n\n   ndarray.sort\n   sort\n   lexsort\n   argsort\n   msort\n   partition\n   argpartition\n\nSearching\n---------\n.. autosummary::\n   :toctree: generated/\n\n   argmax\n   argmin\n   nanargmax\n   nanargmin\n   argwhere\n   nonzero\n   flatnonzero\n   where\n   searchsorted\n   extract\n\nCounting\n--------\n.. autosummary::\n   :toctree: generated/\n\n   count_nonzero\n"
  },
  {
    "path": "docs/python_docs/python/api/np/routines.statistics.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nStatistics\n==========\n\n.. currentmodule:: mxnet.np\n\n\nOrder statistics\n----------------\n\n.. autosummary::\n   :toctree: generated/\n\n   min\n   max\n   amin\n   amax\n   nanmin\n   nanmax\n   ptp\n   percentile\n   nanpercentile\n   quantile\n   nanquantile\n\nAverages and variances\n----------------------\n\n.. autosummary::\n   :toctree: generated/\n\n   mean\n   std\n   var\n   median\n   average\n   nanmedian\n   nanstd\n   nanvar\n\nCorrelating\n-----------\n\n.. autosummary::\n   :toctree: generated/\n\n   corrcoef\n   correlate\n   cov\n\nHistograms\n----------\n\n.. autosummary::\n   :toctree: generated/\n\n   histogram\n   histogram2d\n   histogramdd\n   bincount\n   histogram_bin_edges\n   digitize\n"
  },
  {
    "path": "docs/python_docs/python/api/npx/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nNPX: NumPy Neural Network Extension\n===================================\n\n.. currentmodule:: mxnet.npx\n\nCompatibility\n-------------\n\n.. autosummary::\n   :toctree: generated/\n\n   set_np\n   reset_np\n\n.. code::\n\n   is_np_array\n   use_np_array\n   is_np_shape\n   use_np_shape\n   np_array\n   np_shape\n\n\nDevices\n---------\n\n\n.. autosummary::\n   :toctree: generated/\n\n   cpu\n   cpu_pinned\n   gpu\n   gpu_memory_info\n   current_device\n   num_gpus\n\nNerual networks\n-----------------------\n\n.. autosummary::\n   :toctree: generated/\n\n   activation\n   batch_norm\n   convolution\n   dropout\n   embedding\n   fully_connected\n   layer_norm\n   pooling\n   rnn\n   leaky_relu\n   multibox_detection\n   multibox_prior\n   multibox_target\n   roi_pooling\n\n\nMore operators\n------------------\n\n.. autosummary::\n   :toctree: generated/\n\n   sigmoid\n   relu\n   smooth_l1\n   softmax\n   log_softmax\n   topk\n   waitall\n   load\n   save\n   one_hot\n   pick\n   reshape_like\n   batch_flatten\n   batch_dot\n   gamma\n   sequence_mask\n\n.. code::\n\n   seed\n"
  },
  {
    "path": "docs/python_docs/python/api/optimizer/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.optimizer\n===============\n\n.. automodule:: mxnet.optimizer\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/profiler/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.profiler\n===============\n\n.. automodule:: mxnet.profiler\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/rtc/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.rtc\n=========\n\n.. automodule:: mxnet.rtc\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/runtime/index.rst",
    "content": "..\n  Licensed to the Apache Software Foundation (ASF) under one\n  or more contributor license agreements.  See the NOTICE file\n  distributed with this work for additional information\n  regarding copyright ownership.  The ASF licenses this file\n  to you under the Apache License, Version 2.0 (the\n  \"License\"); you may not use this file except in compliance\n  with the License.  You may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing,\n  software distributed under the License is distributed on an\n  \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  KIND, either express or implied.  See the License for the\n  specific language governing permissions and limitations\n  under the License.\n\nmxnet.runtime\n=============\n\n.. currentmodule:: mxnet.runtime\n\n.. autosummary::\n   :toctree: generated/\n\n   Feature\n   Features\n   feature_list\n"
  },
  {
    "path": "docs/python_docs/python/api/test_utils/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.test_utils\n================\n\n.. automodule:: mxnet.test_utils\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/api/util/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nmxnet.util\n==========\n\n.. automodule:: mxnet.util\n    :members:\n    :autosummary:\n"
  },
  {
    "path": "docs/python_docs/python/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nPython Documentation\n====================\n\n\n.. toctree::\n   :maxdepth: 2\n   :hidden:\n\n   tutorials/index\n   api/index\n\n\nPython Tutorials\n----------------\n\n.. container:: cards\n\n   .. card::\n      :title: Tutorials\n      :link: tutorials/index.html\n      :is_head: true\n\n      Guides and Tutorials for using MXNet.\n\nPython API References\n---------------------\n\n.. container:: cards\n\n   .. card::\n      :title: API\n      :link: api/index.html\n      :is_head: true\n\n      API reference to all MXNet classes and methods.\n"
  },
  {
    "path": "docs/python_docs/python/scripts/conf.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# -*- coding: utf-8 -*-\nimport sys, os, re, subprocess\nfrom recommonmark.parser import CommonMarkParser\nfrom recommonmark.transform import AutoStructify\n\n# -- mock out modules\nMOCK_MODULES = ['scipy', 'scipy.sparse', 'sklearn']\n\n# -- General configuration -----------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\nneeds_sphinx = '1.5.6'\n\n# General information about the project.\nproject = 'Apache MXNet'\nauthor = f'{project} developers'\ncopyright = f'2015-2020, {author}'\ngithub_doc_root = 'https://github.com/apache/mxnet/tree/master/docs/'\ndoc_root = 'https://mxnet.apache.org/'\n\n# add markdown parser\nsource_parsers = {\n    '.md': CommonMarkParser,\n}\n# Version information.\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones\nextensions = [\n    'sphinx.ext.autodoc',\n    'sphinx.ext.doctest',\n    'sphinx.ext.intersphinx',\n    'sphinx.ext.todo',\n    'sphinx.ext.mathjax',\n    'sphinx.ext.ifconfig',\n    'sphinx.ext.viewcode',\n    # 'sphinxcontrib.fulltoc',\n    'nbsphinx',\n    'IPython.sphinxext.ipython_console_highlighting',\n    'sphinx.ext.autosummary',\n    'sphinx.ext.napoleon',\n    'breathe',\n#    'mxdoc'\n    'autodocsumm',\n]\n\ndoctest_global_setup = '''\nimport mxnet as mx\nfrom mxnet import np, npx\n'''\n\nautosummary_generate = True\nnumpydoc_show_class_members = False\n\n# Disable SSL verification in link check.\ntls_verify = False\n\nautodoc_member_order = 'alphabetical'\n\nautodoc_default_flags = ['members', 'show-inheritance']\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = ['_templates']\n\n# The suffix of source filenames.\nsource_suffix = ['.rst', '.ipynb', '.md', '.Rmd']\n\n# The encoding of source files.\n#source_encoding = 'utf-8-sig'\n\n# The master toctree document.\nmaster_doc = 'index'\n\n# The version info for the project you're documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\n# The short X.Y version.\n\n# Version and release are passed from CMake.\n#version = None\n\n# The full version, including alpha/beta/rc tags.\n#release = version\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n#language = None\n\n# There are two options for replacing |today|: either, you set today to some\n# non-false value, then it is used:\n#today = ''\n# Else, today_fmt is used as the format for a strftime call.\n#today_fmt = '%B %d, %Y'\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\nexclude_patterns = ['templates',\n                    # 'api',\n                    'guide/modules/others', 'guide/guide', 'blog']\n\n# The reST default role (used for this markup: `text`) to use for all documents.\n#default_role = None\n\n# If true, '()' will be appended to :func: etc. cross-reference text.\n#add_function_parentheses = True\n\n# If true, the current module name will be prepended to all description\n# unit titles (such as .. function::).\nadd_module_names = False\n\n# If true, sectionauthor and moduleauthor directives will be shown in the\n# output. They are ignored by default.\n#show_authors = False\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = 'sphinx'\n\n\n# A list of ignored prefixes for module index sorting.\n#modindex_common_prefix = []\n\nsuppress_warnings = [\n   'image.nonlocal_uri',\n]\n\n# -- Options for HTML output ---------------------------------------------------\n\n# Add any paths that contain custom themes here, relative to this directory.\nhtml_theme_path = ['../../themes/mx-theme']\n\n# The theme to use for HTML and HTML Help pages.  See the documentation for\n# a list of builtin themes.\nhtml_theme = 'mxtheme'\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further.  For a list of options available for each theme, see the\n# documentation.\nhtml_theme_options = {\n    'primary_color': 'blue',\n    'accent_color': 'deep_orange',\n    'show_footer': True,\n    'relative_url': os.environ.get('SPHINX_RELATIVE_URL', '/')\n}\n\n\n# The name for this set of Sphinx documents.  If None, it defaults to\n# \"<project> v<release> documentation\".\n#html_title = None\n\n# A shorter title for the navigation bar.  Default is the same as html_title.\n#html_short_title = None\n\n# The name of an image file (relative to this directory) to place at the top\n# of the sidebar.\nhtml_logo = '../../_static/mxnet_logo.png'\n\n# The name of an image file (within the static path) to use as favicon of the\n# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32\n# pixels large.\nhtml_favicon = '../../_static/mxnet-icon.png'\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\nhtml_static_path = ['../../_static']\n\nhtml_css_files = [\n    'mxnet.css',\n]\n\nhtml_js_files = [\n    'autodoc.js'\n]\n\n# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,\n# using the given strftime format.\n#html_last_updated_fmt = '%b %d, %Y'\n\n# If true, SmartyPants will be used to convert quotes and dashes to\n# typographically correct entities.\n#html_use_smartypants = True\n\n# Custom sidebar templates, maps document names to template names.\nhtml_sidebars = {\n  '**': 'relations.html'\n}\n\n# Additional templates that should be rendered to pages, maps page names to\n# template names.\n#html_additional_pages = {}\n\n# If false, no module index is generated.\n#html_domain_indices = True\n\n# If false, no index is generated.\n#html_use_index = True\n\n# If true, the index is split into individual pages for each letter.\n#html_split_index = False\n\n# If true, links to the reST sources are added to the pages.\n#html_show_sourcelink = True\n\n# If true, \"Created using Sphinx\" is shown in the HTML footer. Default is True.\nhtml_show_sphinx = False\n\n# If true, \"(C) Copyright ...\" is shown in the HTML footer. Default is True.\nhtml_show_copyright = False\n\n# If true, an OpenSearch description file will be output, and all pages will\n# contain a <link> tag referring to it.  The value of this option must be the\n# base URL from which the finished HTML is served.\n#html_use_opensearch = ''\n\n# This is the file name suffix for HTML files (e.g. \".xhtml\").\n#html_file_suffix = None\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = 'formatdoc'\n\nnbsphinx_execute = 'never'\n\n# let the source file format to be xxx.ipynb instead of xxx.ipynb.txt\nhtml_sourcelink_suffix = ''\n\nhtml_context = {\n    'display_github': True,\n    'github_user': 'apache',\n    'github_repo': 'mxnet',\n    'github_version': 'master',\n    'conf_py_path': '/docs/python_docs/python/',\n    'last_updated': False,\n    'commit': True\n}\n\ndef setup(app):\n    app.add_transform(AutoStructify)\n    app.add_config_value('recommonmark_config', {\n    }, True)\n    app.add_javascript('matomo_analytics.js')\n    import mxtheme\n    app.add_directive('card', mxtheme.CardDirective)\n"
  },
  {
    "path": "docs/python_docs/python/scripts/md2ipynb.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport sys\nimport os\nimport time\nimport notedown\nimport nbformat\n\ndef md2ipynb():\n    assert len(sys.argv) == 3, 'usage: input.md output.rst'\n    (src_fn, input_fn, output_fn) = sys.argv\n\n    # timeout for each notebook, in sec\n    timeout = 60 * 60\n    # if enable evaluation\n    do_eval = int(os.environ.get('EVAL', True))\n    \n    # Skip these notebooks as some APIs will no longer be used\n    skip_list = [\"pytorch.md\", \"mnist.md\", \"custom-loss.md\", \"fit_api_tutorial.md\", \\\n        \"01-ndarray-intro.md\", \"02-ndarray-operations.md\", \"03-ndarray-contexts.md\", \\\n        \"gotchas_numpy_in_mxnet.md\", \"csr.md\", \"row_sparse.md\", \"fine_tuning_gluon.md\", \\\n        \"inference_on_onnx_model.md\", \"amp.md\", \"profiler.md\"]\n\n    require_gpu = []\n    # the files will be ignored for execution\n    ignore_execution = skip_list + require_gpu\n\n    reader = notedown.MarkdownReader(match='strict')\n    with open(input_fn, 'r', encoding=\"utf8\") as f:\n        notebook = reader.read(f)\n    if do_eval:\n        if not any([i in input_fn for i in ignore_execution]):\n            tic = time.time()\n            notedown.run(notebook, timeout)\n            print(f'{src_fn}: Evaluated {input_fn} in {time.time()-tic} sec')\n    # need to add language info to for syntax highlight\n    notebook['metadata'].update({'language_info':{'name':'python'}})\n    with open(output_fn, 'w', encoding='utf-8') as f:\n        f.write(nbformat.writes(notebook))\n    print(f'{src_fn}: Write results into {output_fn}')\n\nif __name__ == '__main__':\n    md2ipynb()\n"
  },
  {
    "path": "docs/python_docs/python/scripts/process_rst.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport sys\nimport os\nimport re\n\ndef has_token(token, lines):\n    for line in lines:\n        if token in line:\n            return True\n    return False\n\ndef get_next_title_mark(lines):\n    available_marks = ['=', '-', '~', '^']\n    for mark in available_marks:\n        if has_token(mark*3, lines):\n            continue\n        else:\n            return mark\n    return None\n\ndef add_hidden_title(inputs):\n    \"\"\"\n    convert\n\n       .. autoclass:: Class\n\n    into\n\n       :hidden:`Class`\n       ~~~~~~~~~~~~~~~\n\n       .. autoclass:: Class\n    \"\"\"\n    lines = inputs.split('\\n')\n    if not has_token('doxygenfunction:', lines):\n        return inputs, None\n\n    outputs = \"\"\".. raw:: html\n\n   <div class=\"mx-api\">\n\n.. role:: hidden\n    :class: hidden-section\n\n\"\"\"\n    num = 0\n    FUNC = re.compile('\\.\\. doxygenfunction\\:\\:[ ]+([\\w\\.]+)')\n    mark = get_next_title_mark(lines)\n    assert mark is not None\n    for line in lines:\n        m = FUNC.match(line)\n        if m is not None:\n            name = ':hidden:`' + m.groups()[0] + '`'\n            outputs += '\\n' + name + '\\n' + mark * len(name) + '\\n\\n'\n            num += 1\n        outputs += line + '\\n'\n    outputs += '.. raw:: html\\n\\n    </div>\\n'\n    return outputs, num\n\n\nif __name__ == '__main__':\n    assert len(sys.argv) == 3, 'usage: input.rst output.rst'\n    (src_fn, input_fn, output_fn) = sys.argv\n    with open(input_fn, 'r') as f:\n        inputs = f.read()\n    outputs, num = add_hidden_title(inputs)\n    if num is not None:\n        print(f'{src_fn}: add {num} hidden sections for {input_fn}')\n    with open(output_fn, 'w') as f:\n        f.write(outputs)\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/deploy/export/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nExport\n======\n\nThe following tutorials will help you learn export MXNet models.\nModels are by default exported as a couple of `params` and `json` files,\nbut you also have the option to export most models to the ONNX format.\n\n.. container:: cards\n\n   .. card::\n      :title: Export with GluonCV\n      :link: https://gluon-cv.mxnet.io/build/examples_deployment/export_network.html\n\n      How to export models trained with MXNet GluonCV.\n\n   .. card::\n      :title: Export ONNX Models\n      :link: onnx.html\n\n      Export your MXNet model to the Open Neural Exchange Format\n\n.. toctree::\n   :hidden:\n   :maxdepth: 1\n   :glob:\n\n   *\n   Export Gluon CV Models <https://gluon-cv.mxnet.io/build/examples_deployment/export_network.html>\n   Save / Load Parameters <https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/blocks/save_load_params.html>\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/deploy/export/onnx.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Exporting to ONNX format\n\n[Open Neural Network Exchange (ONNX)](https://github.com/onnx/onnx) provides an open source format for AI models. It defines an extensible computation graph model, as well as definitions of built-in operators and standard data types.\n\nIn this tutorial, we will show how you can save MXNet models to the ONNX format.\n\nMXNet-ONNX operators coverage and features are updated regularly. Visit the [ONNX operator coverage](https://cwiki.apache.org/confluence/display/MXNET/ONNX+Operator+Coverage) page for the latest information.\n\nIn this tutorial, we will learn how to use MXNet to ONNX exporter on pre-trained models.\n\n## Prerequisites\n\nTo run the tutorial you will need to have installed the following python modules:\n- [MXNet >= 2.0.0](https://mxnet.apache.org/get_started)\n- [onnx]( https://github.com/onnx/onnx#user-content-installation) v1.7 & v1.8 (follow the install guide)\n\n*Note:* MXNet-ONNX importer and exporter follows version 12 & 13 of ONNX operator set which comes with ONNX v1.7 & v1.8.\n\n\n```{.python .input}\nimport mxnet as mx\nfrom mxnet import initializer as init, np, onnx as mxnet_onnx\nfrom mxnet.gluon import nn\nimport logging\nlogging.basicConfig(level=logging.INFO)\n```\n\n## Create a model from the MXNet Gluon\n\nLet's build a concise model with [MXNet gluon](../../../api/gluon/index.rst) package. The model is multilayer perceptrons with two fully-connected layers. The first one is our hidden layer, which contains 256 hidden units and applies ReLU activation function. The second is our output layer. \n\n```{.python .input}\nnet = nn.HybridSequential()\nnet.add(nn.Dense(256, activation='relu'), nn.Dense(10))\n```\n\nThen we initialize the model and export it into symbol file and parameter file. \n\n```{.python .input}\nnet.initialize(init.Normal(sigma=0.01))\nnet.hybridize()\ninput = np.ones(shape=(50,), dtype=np.float32)\noutput = net(input)\nnet.export(\"mlp\")\n```\n\nNow, we have exported the model symbol, params file on the disk.\n\n## MXNet to ONNX exporter API\n\nLet us describe the MXNet's `export_model` API.\n\n```{.python .input}\nhelp(mxnet_onnx.export_model)\n```\n\nOutput:\n\n```text\nHelp on function export_model in module mxnet.contrib.onnx.mx2onnx.export_model:\n\nexport_model(sym, params, input_shape, input_type=<type 'numpy.float32'>, onnx_file_path=u'model.onnx', verbose=False)\n    Exports the MXNet model file, passed as a parameter, into ONNX model.\n    Accepts both symbol,parameter objects as well as json and params filepaths as input.\n    Operator support and coverage - https://cwiki.apache.org/confluence/display/MXNET/MXNet-ONNX+Integration\n\n    Parameters\n    ----------\n    sym : str or symbol object\n        Path to the json file or Symbol object\n    params : str or symbol object\n        Path to the params file or params dictionary. (Including both arg_params and aux_params)\n    input_shape : List of tuple\n        Input shape of the model e.g [(1,3,224,224)]\n    input_type : data type\n        Input data type e.g. np.float32\n    onnx_file_path : str\n        Path where to save the generated onnx file\n    verbose : Boolean\n        If true will print logs of the model conversion\n\n    Returns\n    -------\n    onnx_file_path : str\n        Onnx file path\n```\n\n`export_model` API can accept the MXNet model in one of the following two ways.\n\n1. MXNet sym, params objects:\n    * This is useful if we are training a model. At the end of training, we just need to invoke the `export_model` function and provide sym and params objects as inputs with other attributes to save the model in ONNX format.\n2. MXNet's exported json and params files:\n    * This is useful if we have pre-trained models and we want to convert them to ONNX format.\n\nSince we have downloaded pre-trained model files, we will use the `export_model` API by passing the path for symbol and params files.\n\n## How to use MXNet to ONNX exporter API\n\nWe will use the downloaded pre-trained model files (sym, params) and define input variables.\n\n```{.python .input}\n# The input symbol and params files\nsym = './mlp-symbol.json'\nparams = './mlp-0000.params'\n\n# Standard Imagenet input - 3 channels, 224*224\ninput_shape = (50,)\n\n# Path of the output file\nonnx_file = './mxnet_exported_mlp.onnx'\n```\n\nWe have defined the input parameters required for the `export_model` API. Now, we are ready to covert the MXNet model into ONNX format.\n\n```{.python .input}\n# Invoke export model API. It returns path of the converted onnx model\nconverted_model_path = mxnet_onnx.export_model(sym, params, [input_shape], [np.float32], onnx_file)\n```\n\nThis API returns path of the converted model which you can later use to import the model into other frameworks.\n\n## Check validity of ONNX model\n\nNow we can check validity of the converted ONNX model by using ONNX checker tool. The tool will validate the model by checking if the content contains valid protobuf:\n\n```{.python .input}\nfrom onnx import checker\nimport onnx\n\n# Load onnx model\nmodel_proto = onnx.load_model(converted_model_path)\n\n# Check if converted ONNX protobuf is valid\nchecker.check_graph(model_proto.graph)\n```\n\nIf the converted protobuf format doesn't qualify to ONNX proto specifications, the checker will throw errors, but in this case it successfully passes.\n\nThis method confirms exported model protobuf is valid. Now, the model is ready to be imported in other frameworks for inference!\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/deploy/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nDeployment\n==========\n\nThe following tutorials will help you learn how to deploy MXNet on various\nplatforms and in different language environments.\n\nExport\n------\nThe following tutorials will help you learn export MXNet models.\n\n.. container:: cards\n\n   .. card::\n      :title: Export with GluonCV\n      :link: https://gluon-cv.mxnet.io/build/examples_deployment/export_network.html\n\n      How to export models trained with MXNet GluonCV.\n\n   .. card::\n      :title: Export ONNX Models\n      :link: export/onnx.html\n\n      Export your MXNet model to the Open Neural Exchange Format\n\nInference\n---------\nThe following tutorials will help you learn how to deploy MXNet models for inference applications.\n\n.. container:: cards\n\n   .. card::\n      :title: GluonCV Models in a C++ Inference Application\n      :link: https://gluon-cv.mxnet.io/build/examples_deployment/cpp_inference.html\n\n      An example application that works with an exported MXNet GluonCV YOLO model.\n\n   .. card::\n      :title: Inference with Quantized Models\n      :link: https://gluon-cv.mxnet.io/build/examples_deployment/int8_inference.html\n\n      How to use quantized GluonCV models for inference on Intel Xeon Processors to gain higher performance.\n\n   .. card::\n      :title: C++\n      :link: inference/cpp.html\n\n      How to use MXNet models in a C++ environment.\n\n   .. card::\n      :title: Image Classification on Jetson\n      :link: inference/image_classification_jetson.html\n\n      Example of running a pretrained image classification model on a Jetson module.\n\n   .. card::\n      :title: Object Detection on Jetson\n      :link: https://gluon-cv.mxnet.io/build/examples_detection/demo_jetson.html\n\n      Example of running a pretrained object detection model on a Jetson module.\n\nCloud\n-----\nThe following tutorials will show you how to use MXNet on AWS.\n\n.. container:: cards\n\n   .. card::\n      :title: MXNet on EC2\n      :link: run-on-aws/use_ec2.html\n\n      How to deploy MXNet on an Amazon EC2 instance.\n\n   .. card::\n      :title: MXNet on SageMaker\n      :link: run-on-aws/use_sagemaker.html\n\n      How to run MXNet using Amazon SageMaker.\n\n   .. card::\n      :title: MXNet on the cloud\n      :link: run-on-aws/cloud.html\n\n      How to run MXNet on the cloud\n\n   .. card::\n      :title: Training with Data from S3\n      :link: /api/faq/s3_integration\n\n      How to train with data from Amazon S3 buckets.\n\nSecurity\n--------\n\n.. container:: cards\n\n   .. card::\n      :title: Securing MXNet\n      :link: https://mxnet.apache.org/api/faq/security\n\n      Best practices and deployment considerations.\n\n\n.. toctree::\n   :hidden:\n   :maxdepth: 1\n\n   export/index\n   inference/index\n   run-on-aws/index\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/deploy/inference/cpp.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nDeploy into C++\n===============\n\nContributions welcome!\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/deploy/inference/image_classification_jetson.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Image Classication using pretrained ResNet-50 model on Jetson module\n\nThis tutorial shows how to install MXNet v1.6 with Jetson support and use it to deploy a pre-trained MXNet model for image classification on a Jetson module.\n\n## What's in this tutorial?\n\nThis tutorial shows how to:\n\n1. Install MXNet v1.6 along with its dependencies on a Jetson module (This tutorial has been tested on Jetson Xavier AGX and Jetson Nano modules)\n\n2. Deploy a pre-trained MXNet model for image classifcation on the module\n\n## Who's this tutorial for?\n\nThis tutorial would benefit developers working on Jetson modules implementing deep learning applications. It assumes that readers have a Jetson module setup with Jetpack installed, are familiar with the Jetson working environment and are somewhat familiar with deep learning using MXNet.\n\n## Prerequisites\n\nTo complete this tutorial, you need:\n\n* A [Jetson module](https://developer.nvidia.com/embedded/develop/hardware) setup with [Jetpack 4.4](https://docs.nvidia.com/jetson/jetpack/release-notes/) installed using NVIDIA [SDK Manager](https://developer.nvidia.com/nvidia-sdk-manager)\n\n* An SSH connection to the module OR display and keyboard setup to directly open shell on the module\n\n* [Swapfile](https://help.ubuntu.com/community/SwapFaq) installed, especially on Jetson Nano for additional memory (increase memory if the inference script terminates with a `Killed` message)\n\n## Installing MXNet v1.6 with Jetson support\n\nTo install MXNet with Jetson support, you can follow the [installation guide](https://mxnet.apache.org/get_started/jetson_setup) on MXNet official website.\n\nAlternatively, you can also directly install MXNet v1.6 wheel with Jetson support, hosted on a public s3 bucket. Here are the steps to install this wheel:\n\n*WARNING: this MXNet wheel is provided for your convenience but it contains packages that are not provided nor endorsed by the Apache Software Foundation.\nAs such, they might contain software components with more restrictive licenses than the Apache License and you'll need to decide whether they are appropriate for your usage. Like all Apache Releases, the\nofficial Apache MXNet (incubating) releases consist of source code only and are found at https://mxnet.apache.org/get_started/download .*\n\nWe start by installing MXNet dependencies\n```bash\nsudo apt-get update\nsudo apt-get install -y git build-essential libopenblas-dev libopencv-dev python3-pip\nsudo pip3 install -U pip\n```\n\nThen we download and install MXNet v1.6 wheel with Jetson support\n```bash\nwget https://mxnet-public.s3.us-east-2.amazonaws.com/install/jetson/1.6.0/mxnet_cu102-1.6.0-py2.py3-none-linux_aarch64.whl\nsudo pip3 install mxnet_cu102-1.6.0-py2.py3-none-linux_aarch64.whl\n```\n\nAnd we are done. You can test the installation now by importing mxnet from python3\n```bash\n>>> python3 -c 'import mxnet'\n```\n\n## Running a pre-trained ResNet-50 model on Jetson\n\nWe are now ready to run a pre-trained model and run inference on a Jetson module. In this tutorial we are using ResNet-50 model trained on Imagenet dataset. We run the following classification script with either cpu/gpu device using python3.\n\n```{.python .input}\nfrom mxnet import gluon\nimport mxnet as mx\n\n# set device\ngpus = mx.test_utils.list_gpus()\ndevice =  mx.gpu() if gpus else mx.cpu()\n\n# load pre-trained model\nnet = gluon.model_zoo.vision.resnet50_v1(pretrained=True, device=device)\nnet.hybridize(static_alloc=True, static_shape=True)\n\n# load labels\nlbl_path = gluon.utils.download('http://data.mxnet.io/models/imagenet/synset.txt')\nwith open(lbl_path, 'r') as f:\n    labels = [l.rstrip() for l in f]\n\n# download and format image as (batch, RGB, width, height)\nimg_path = gluon.utils.download('https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/python/predict_image/cat.jpg?raw=true')\nimg = mx.image.imread(img_path)\nimg = mx.image.imresize(img, 224, 224) # resize\nimg = mx.image.color_normalize(img.astype(dtype='float32')/255,\n                               mean=mx.np.array([0.485, 0.456, 0.406]),\n                               std=mx.np.array([0.229, 0.224, 0.225])) # normalize\nimg = img.transpose((2, 0, 1)) # channel first\nimg = mx.np.expand_dims(img, axis=0) # batchify\nimg = img.to_device(device)\n\nprob = mx.npx.softmax(net(img)) # predict and normalize output\nidx = mx.npx.topk(prob, k=5)[0] # get top 5 result\nfor i in idx:\n    i = int(i.item())\n    print('With prob = %.5f, it contains %s' % (prob[0,i].item(), labels[i]))\n```\n\nAfter running the above script, you should get the following output showing the five classes that the image most relates to with probability:\n```bash\nWith prob = 0.41940, it contains n02119789 kit fox, Vulpes macrotis\nWith prob = 0.28096, it contains n02119022 red fox, Vulpes vulpes\nWith prob = 0.06857, it contains n02124075 Egyptian cat\nWith prob = 0.03046, it contains n02120505 grey fox, gray fox, Urocyon cinereoargenteus\nWith prob = 0.02770, it contains n02441942 weasel\n```\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/deploy/inference/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nInference\n=========\n\nThe following tutorials will help you learn how to deploy MXNet models for inference applications.\n\n.. container:: cards\n\n   .. card::\n      :title: GluonCV Models in a C++ Inference Application\n      :link: https://gluon-cv.mxnet.io/build/examples_deployment/cpp_inference.html\n\n      An example application that works with an exported MXNet GluonCV YOLO model.\n\n   .. card::\n      :title: Inference with Quantized Models\n      :link: https://gluon-cv.mxnet.io/build/examples_deployment/int8_inference.html\n\n      How to use quantized GluonCV models for inference on Intel Xeon Processors to gain higher performance.\n\n   .. card::\n      :title: C++\n      :link: cpp.html\n\n      How to use MXNet models in a C++ environment.\n\n   .. card::\n      :title: Image Classification on Jetson\n      :link: image_classification_jetson.html\n\n      Example of running a pretrained image classification model on a Jetson module.\n\n   .. card::\n      :title: Object Detection on Jetson\n      :link: https://gluon-cv.mxnet.io/build/examples_detection/demo_jetson.html\n\n      Example of running a pretrained object detection model on a Jetson module.\n\n\n.. toctree::\n   :hidden:\n   :maxdepth: 1\n   :glob:\n\n   *\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/deploy/run-on-aws/cloud.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nMXNet on the Cloud\n==================\n\nDeep learning can require extremely powerful hardware, often for\nunpredictable durations of time. Moreover, *MXNet* can benefit from both multiple GPUs and multiple machines. Accordingly, cloud computing, as offered by AWS and others, is especially well suited to training deep learning models. Using AWS, we can rapidly fire up multiple machines with multiple GPUs each at will and maintain the resources for precisely the amount of time needed.\n\nHere are some ways you can use MXNet on AWS:\n\n1. Use [Amazon SageMaker](https://aws.amazon.com/sagemaker/developer-resources/)\n1. Use the [AWS Deep Learning AMI with Conda](https://docs.aws.amazon.com/dlami/latest/devguide/overview-conda.html)\n1. Use an [AWS Deep Learning Container](https://docs.aws.amazon.com/dlami/latest/devguide/deep-learning-containers.html)\n1. Install MXNet on a [AWS Deep Learning Base AMI](https://docs.aws.amazon.com/dlami/latest/devguide/overview-base.html)\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/deploy/run-on-aws/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nRun on AWS\n==========\n\nThe following tutorials will help you learn how to deploy MXNet on various AWS platforms.\n\n.. container:: cards\n\n   .. card::\n      :title: MXNet on EC2\n      :link: use_ec2.html\n\n      How to deploy MXNet on an Amazon EC2 instance.\n\n   .. card::\n      :title: MXNet on SageMaker\n      :link: use_sagemaker.html\n\n      How to run MXNet using Amazon SageMaker.\n\n   .. card::\n      :title: MXNet on the Cloud\n      :link: cloud.html\n\n      How to run MXNet in the cloud.\n\n   .. card::\n      :title: Training with Data from S3\n      :link: /api/faq/s3_integration\n\n      How to train with data from Amazon S3 buckets.\n\n.. toctree::\n   :hidden:\n   :maxdepth: 1\n\n   use_ec2\n   use_sagemaker\n   cloud\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/deploy/run-on-aws/use_ec2.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nRun on an EC2 Instance\n======================\n\nThis chapter shows, how to allocate a CPU/GPU instance in AWS and how to\nsetup the Deep Learning environment.\n\nWe first need `an AWS account <https://aws.amazon.com/>`_, and\nthen go the EC2 console after login in.\n\nThen click \"launch instance\" to select the operation system and instance\ntype.\n\nAWS offers\n`Deep Learning AMIs <https://docs.aws.amazon.com/dlami/latest/devguide/options.html>`_\nthat come with the latest versions of Deep Learning frameworks. The Deep\nLearning AMIs provide all necessary packages and drivers and allow you\nto directly start implementing and training your models. Deep Learning\nAMIs use use binaries that are optimized to run on AWS instances to\naccelerate model training and inference. In this tutorial we use Deep\nLearning AMI (Ubuntu) Version 19.0:\n\nWe choose \"p2.xlarge\", which contains a single Nvidia K80 GPU. Note that\nthere is a large number of instance, refer to\n`ec2instances.info <http://www.ec2instances.info/>`_ for detailed\nconfigurations and fees.\n\nNote that we need to check the instance limits to guarantee that we can\nrequest the resource. If running out of limits, we can request more\ncapacity by clicking the right link, which often takes about a single\nworkday to process.\n\nOn the next step we increased the disk from 8 GB to 40 GB so we have\nenough space store a reasonable size dataset. For large-scale datasets,\nwe can \"add new volume\". Also you selected a very powerful GPU instance\nsuch as \"p3.8xlarge\", make sure you selected \"Provisioned IOPS\" in the\nvolume type for better I/O performance.\n\nThen we launched with other options as the default values. The last step\nbefore launching is choosing the ssh key, you may need to generate and\nstore a key if you don't have one before.\n\nAfter clicked \"launch instances\", we can check the status by clicking\nthe instance ID link.\n\nOnce the status is green, we can right-click and select \"connect\" to get\nthe access instruction.\n\nWith the given address, we can log into our instance:\n\nThe login screen will show a long list of available conda environments\nfor the different Deep Learning frameworks, CUDA driver and Python\nversions. With ``conda activate`` you can easily switch into the\ndifferent environments. In the following example we switch to the MXNet\nPython 3.6 environment:\n\nNow you are ready to start developing and training MXNet models. Once\nyou start training, you can check the GPU status with ``nividia-smi``.\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/deploy/run-on-aws/use_sagemaker.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nRun on Amazon SageMaker\n-----------------------\n\nThis chapter will give a high level overview about running MXNet on Amazon SageMaker,\nin-depth tutorials can be found on the `Sagemaker\nwebsite <https://docs.aws.amazon.com/sagemaker/latest/dg/whatis.html>`__.\n\nSageMaker offers Jupyter notebooks and supports MXNet out-of-the box.\nYou can run your notebooks on CPU instances and as such profit from the\nfree tier. However, more powerful CPU instances or GPU instances are\ncharged by time. Within this notebook you can `fetch, explore and\nprepare training\ndata <https://docs.aws.amazon.com/sagemaker/latest/dg/how-it-works-notebooks-instances.html>`__.\n\nWith your own data on the notebook instance, you can easily launch training via the SageMaker\nSDK. So there is no need to manually configure and log into EC2\ninstances. You can either bring your own model or use SageMaker's\n`built-in\nalgorithms <https://docs.aws.amazon.com/sagemaker/latest/dg/algos.html>`__\nthat are tailored to specific use cases such as computer vision, NLP\netc. SageMaker encapsulates the process of training into the class\n``Estimator`` and we can now start the training on the local notebook\ninstance:\n\n::\n\n    from sagemaker.mxnet import MXNet as MXNetEstimator\n    estimator = MXNetEstimator(entry_point='train.py',\n                               role=sagemaker.get_execution_role(),\n                               train_instance_count=1,\n                               train_instance_type='local',\n                               hyperparameters={'batch_size': 1024,\n                                                'epochs': 30})\n    estimator.fit(inputs)\n\nIf you require a more powerful platform for training, then you only need\nto change the ``train_instance_type``. Once you call fit, SageMaker will\nautomatically create the required EC2 instances, train your model within\na Docker container and then immediately shutdown these instances.\n``Fit()`` requires an entry point (here ``train.py``) that describes the\nmodel and training loop. This script needs to provides certain\nfunctions, that will be automatically called by SageMaker once you train\nand deploy the model. More information about the entry point script can\nbe found\n`here <https://docs.aws.amazon.com/sagemaker/latest/dg/mxnet-training-inference-code-template.html>`__.\nWhen the model is ready for deployment you can use `SageMaker's hosting\nservices <https://docs.aws.amazon.com/sagemaker/latest/dg/how-it-works-hosting.html>`__\nthat create an HTTPS endpoint where model inference is provided.\n\n::\n\n    predictor = estimator.deploy(initial_instance_count=1,\n                                 instance_type='ml.m4.xlarge')\n\nThe following links show more advanced uses cases in SageMaker: -\n`Distributed training on multiple\nmachines <https://medium.com/apache-mxnet/94-accuracy-on-cifar-10-in-10-minutes-with-amazon-sagemaker-754e441d01d7>`__\n- `Hyperparameter Tuning\nJobs <https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-ex.html>`__\n- `Optimize a model with SageMaker\nNeo <https://docs.aws.amazon.com/sagemaker/latest/dg/neo.html>`__ -\n`Build Groundtruth\nDatasets <https://docs.aws.amazon.com/sagemaker/latest/dg/sms-getting-started.html>`__\n- `Getting started with\nSageMaker <https://medium.com/apache-mxnet/getting-started-with-sagemaker-ebe1277484c9>`__\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/extend/customop.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Custom Numpy Operators\n\nIn this tutorial, we will learn how to build custom operators with numpy in python. We will go through two examples:\n- Custom operator without any `Parameter`s\n- Custom operator with `Parameter`s\n\nCustom operator in python is easy to develop and good for prototyping, but may hurt performance. If you find it to be a bottleneck, please consider moving to a C++ based implementation in the backend.\n\n\n\n```{.python .input}\nimport numpy as np\nimport mxnet as mx\nfrom mxnet import gluon, autograd\nimport os\nmx.npx.reset_np()\n```\n\n## Parameter-less operators\n\nThis operator implements the standard sigmoid activation function. This is only for illustration purposes, in real life you would use the built-in operator `mx.npx.relu`.\n\n### Forward & backward implementation\n\nFirst we implement the forward and backward computation by sub-classing `mx.operator.CustomOp`:\n\n\n```{.python .input}\nclass Sigmoid(mx.operator.CustomOp):\n    def forward(self, is_train, req, in_data, out_data, aux):\n        \"\"\"Implements forward computation.\n\n        is_train : bool, whether forwarding for training or testing.\n        req : list of {'null', 'write', 'inplace', 'add'}, how to assign to out_data. 'null' means skip assignment, etc.\n        in_data : list of NDArray, input data.\n        out_data : list of NDArray, pre-allocated output buffers.\n        aux : list of NDArray, mutable auxiliary states. Usually not used.\n        \"\"\"\n        x = in_data[0].asnumpy()\n        y = 1.0 / (1.0 + np.exp(-x))\n        self.assign(out_data[0], req[0], mx.nd.array(y))\n\n    def backward(self, req, out_grad, in_data, out_data, in_grad, aux):\n        \"\"\"Implements backward computation\n\n        req : list of {'null', 'write', 'inplace', 'add'}, how to assign to in_grad\n        out_grad : list of NDArray, gradient w.r.t. output data.\n        in_grad : list of NDArray, gradient w.r.t. input data. This is the output buffer.\n        \"\"\"\n        y = out_data[0].asnumpy()\n        dy = out_grad[0].asnumpy()\n        dx = dy*(1.0 - y)*y\n        self.assign(in_grad[0], req[0], mx.nd.array(dx))\n```\n\n### Register custom operator\n\nThen we need to register the custom op and describe it's properties like input and output shapes so that mxnet can recognize it. This is done by sub-classing `mx.operator.CustomOpProp`:\n\n\n```{.python .input}\n@mx.operator.register(\"sigmoid\")  # register with name \"sigmoid\"\nclass SigmoidProp(mx.operator.CustomOpProp):\n    def __init__(self):\n        super(SigmoidProp, self).__init__(True)\n\n    def list_arguments(self):\n        #  this can be omitted if you only have 1 input.\n        return ['data']\n\n    def list_outputs(self):\n        #  this can be omitted if you only have 1 output.\n        return ['output']\n\n    def infer_shape(self, in_shapes):\n        \"\"\"Calculate output shapes from input shapes. This can be\n        omited if all your inputs and outputs have the same shape.\n\n        in_shapes : list of shape. Shape is described by a tuple of int.\n        \"\"\"\n        data_shape = in_shapes[0]\n        output_shape = data_shape\n        # return 3 lists representing inputs shapes, outputs shapes, and aux data shapes.\n        return (data_shape,), (output_shape,), ()\n\n    def create_operator(self, device, in_shapes, in_dtypes):\n        #  create and return the CustomOp class.\n        return Sigmoid()\n```\n\n### Example Usage\n\nWe can now use this operator by calling `mx.nd.Custom`:\n\n\n```{.python .input}\nx = mx.nd.array([0, 1, 2, 3])\n# attach gradient buffer to x for autograd\nx.attach_grad()\n# forward in a record() section to save computation graph for backward\n# see autograd tutorial to learn more.\nwith autograd.record():\n    y = mx.nd.Custom(x, op_type='sigmoid')\nprint(y)\n```\n\n```{.python .input}\n# call backward computation\ny.backward()\n# gradient is now saved to the grad buffer we attached previously\nprint(x.grad)\n```\n\n## Parametrized Operator\n\nIn the second use case we implement an operator with learnable weights. We implement the dense (or fully connected) layer that has one input, one output, and two learnable parameters: weight and bias.\n\nThe dense operator performs a dot product between data and weight, then add bias to it.\n\n### Forward & backward implementation\n\n\n```{.python .input}\nclass Dense(mx.operator.CustomOp):\n    def __init__(self, bias):\n        self._bias = bias\n\n    def forward(self, is_train, req, in_data, out_data, aux):\n        x = in_data[0].asnumpy()\n        weight = in_data[1].asnumpy()\n        y = x.dot(weight.T) + self._bias\n        self.assign(out_data[0], req[0], mx.nd.array(y))\n\n    def backward(self, req, out_grad, in_data, out_data, in_grad, aux):\n        x = in_data[0].asnumpy()\n        dy = out_grad[0].asnumpy()\n        dx = dy.T.dot(x)\n        self.assign(in_grad[0], req[0], mx.nd.array(dx))\n```\n\n### Registration\n\n\n```{.python .input}\n@mx.operator.register(\"dense\")  # register with name \"sigmoid\"\nclass DenseProp(mx.operator.CustomOpProp):\n    def __init__(self, bias):\n        super(DenseProp, self).__init__(True)\n        # we use constant bias here to illustrate how to pass arguments\n        # to operators. All arguments are in string format so you need\n        # to convert them back to the type you want.\n        self._bias = float(bias)\n\n    def list_arguments(self):\n        return ['data', 'weight']\n\n    def list_outputs(self):\n        #  this can be omitted if you only have 1 output.\n        return ['output']\n\n    def infer_shape(self, in_shapes):\n        data_shape = in_shapes[0]\n        weight_shape = in_shapes[1]\n        output_shape = (data_shape[0], weight_shape[0])\n        # return 3 lists representing inputs shapes, outputs shapes, and aux data shapes.\n        return (data_shape, weight_shape), (output_shape,), ()\n\n    def create_operator(self, device, in_shapes, in_dtypes):\n        #  create and return the CustomOp class.\n        return Dense(self._bias)\n```\n\n### Use CustomOp together with Block\n\nParameterized CustomOp are usually used together with Blocks, which holds the parameter.\n\n\n```{.python .input}\nclass DenseBlock(mx.gluon.Block):\n    def __init__(self, in_channels, channels, bias, **kwargs):\n        super(DenseBlock, self).__init__(**kwargs)\n        self._bias = bias\n        self.weight = gluon.Parameter('weight', shape=(channels, in_channels))\n\n    def forward(self, x):\n        device = x.device\n        return mx.nd.Custom(x, self.weight.data(device), bias=self._bias, op_type='dense')\n```\n\n### Example usage\n\n\n```{.python .input}\ndense = DenseBlock(3, 5, 0.1)\ndense.initialize()\nx = mx.nd.uniform(shape=(4, 3))\ny = dense(x)\nprint(y)\n```\n\n## Using custom operators with fork\nIn Linux systems, the default method in multiprocessing to create process is by using fork. If there are unfinished async custom operations when forking, the program will be blocked because of python GIL. Always use sync calls like `wait_to_read` or `waitall` before calling fork.\n\n```{.python}\nx = mx.nd.array([0, 1, 2, 3])\ny = mx.nd.Custom(x, op_type='sigmoid')\n# unfinished async sigmoid operation will cause blocking\nos.fork()\n```\n\nCorrectly handling this will make mxnet depend upon libpython, so the workaround now is to ensure that all custom operations are executed before forking process.\n\n```{.python}\nx = mx.nd.array([0, 1, 2, 3])\ny = mx.nd.Custom(x, op_type='sigmoid')\n# force execution by reading y\nprint(y.asnumpy())\nos.fork()\n```\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/extend/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nExtend\n======\n\nThe following tutorials will help you learn how to customize MXNet.\n\n.. container:: cards\n\n   .. card::\n      :title: Custom Layers for Gluon\n      :link: ../packages/gluon/blocks/custom-layer.html\n\n      How to add new layer functionality to MXNet's imperative interface.\n\n   .. card::\n      :title: Custom Loss\n      :link: ../packages/gluon/loss/custom-loss.html\n\n      A guide to implementing custom losses.\n\n   .. card::\n      :title: Custom Operators Using Numpy\n      :link: customop.html\n\n      How to use Numpy to create custom MXNet operators.\n\n   .. card::\n      :title: New Operator Creation\n      :link: /api/faq/new_op\n\n      How to create new MXNet operators using CustomOp (Python) or NNVM (C++).\n\n   .. card::\n      :title: A Beginner’s Guide to Implementing Operators in MXNet Backend\n      :link: /api/faq/add_op_in_backend\n\n      How to create new MXNet operators in MXNet's backend using C++.\n      An example custom quadratic function op.\n\n   .. card::\n      :title: Using runtime compilation (RTC) to write CUDA kernels in MXNet\n      :link: /api/faq/using_rtc\n\n      How to write CUDA kernels in MXNet using runtime compilation.\n\n\n.. toctree::\n   :hidden:\n   :glob:\n\n   *\n   New Operator Creation <https://mxnet.apache.org/api/faq/new_op>\n   New Operator in MXNet Backend <https://mxnet.apache.org/api/faq/add_op_in_backend>\n   Using RTC for CUDA kernels <https://mxnet.apache.org/api/faq/using_rtc>\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/getting-started/crash-course/0-introduction.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Introduction\n\n\n## About MXNet\n\nApache MXNet is an open-source deep learning framework that provides a comprehensive and flexible API to create deep learning models. Some of the key features of MXNet are:\n\n1.  **Fast and Scalable:** Easily supports multiple GPU's and distributed multi-host jobs. \n2.  **Multiple Programming language support:**  Python, Scala,  R, Java, C++, Julia, Matlab, JavaScript and Go interfaces. \n3.  **Supported:** Backed by Apache Software Foundation and supported by Amazon Web Services (AWS), Microsoft Azure and highly active open-source community.\n4.  **Portable:** Supports an efficient deployment on a wide range of hardware configurations and platforms i.e.  low end devices, internet of things devices, serverless computing and containers.\n5.  **Flexible:** Supports both imperative and symbolic programming.\n\n\n### Basic building blocks\n\n#### Tensors A.K.A Arrays\n\nTensors give us a generic way of describing $n$-dimensional **arrays** with an arbitrary number of axes. Vectors, for example, are first-order tensors, and matrices are second-order tensors. Tensors with more than two orders(axes) do not have special mathematical names. The [NP](../../../api/np/index.rst) package in MXNet provides a NumPy-compatible tensor implementation, `np.ndarray` with additional features. First, MXNet’s `np.ndarray` supports fast execution on a wide range of hardware configurations, including CPU, GPU, and multi-GPU machines where as NumPy only supports CPU computation. Second, MXNet’s `np.ndarray` executes code lazily, allowing it to automatically parallelize multiple operations across the available hardware.\n\nYou will get familiar to arrays in the [next section](./1-nparray.ipynb) of this crash course.\n\n### Computing paradigms\n\n#### Block\n\nNeural network designs like [ResNet-152](https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/He_Deep_Residual_Learning_CVPR_2016_paper.pdf) have a fair degree of regularity. They consist of _blocks_ of repeated (or at least similarly designed) layers; these blocks then form the basis of more complex network designs. A block can be a single layer, a component consisting of multiple layers, or the entire complex neural network itself! One benefit of working with the block abstraction is that you can combine blocks into larger artifacts; often recursively. By defining code to generate blocks of arbitrary complexity on demand, you can write surprisingly compact code and still implement complex neural networks.\n\n\nFrom a programming standpoint, a block is represented by a class and [Block](../../../api/gluon/block.rst#mxnet.gluon.Block)  is the base class for all neural networks layers in MXNet. Any subclass of it must define a forward propagation function that transforms its input into output and must store any necessary parameters if required.\n\nYou will see more about blocks in [Array](./1-nparray.ipynb) and [Create neural network](./2-create-nn.ipynb) sections.\n\n#### HybridBlock\n\nImperative and symbolic  programming represents two styles or paradigms of deep learning programming interface and historically most deep learning frameworks choose either imperative or symbolic programming. For example, both Theano and TensorFlow (inspired by the latter) make use of symbolic programming, while Chainer and its predecessor PyTorch utilize imperative programming. \n\nThe differences between imperative (interpreted) and symbolic programming are as follows:\n\n* __Imperative programming__ is easier. When imperative programming is used in Python, the majority of the code is straightforward and easy to write. It is also easier to debug imperative programming code. This is because it is easier to obtain and print all relevant intermediate variable values, or use Pythonʼs built-in debugging tools.\n    \n* __Symbolic programming__ is more efficient and easier to port. It makes it easier to optimize the code during compilation, while also having the ability to port the program into a format independent of Python. This allows the program to be run in a non-Python environment, thus avoiding any potential performance issues related to the Python interpreter.\n\nYou can learn more about the difference between symbolic vs. imperative programming from this [deep learning programming paradigm](https://mxnet.apache.org/versions/1.6/api/architecture/program_model) article\n\nWhen designing MXNet, developers considered whether it was possible to harness the benefits of both imperative and symbolic programming. The developers believed that users should be able to develop and debug using pure imperative programming, while having the ability to convert most programs into symbolic programming to be run when product-level computing performance and deployment are required. \n\nIn hybrid programming, you can build models using either the [HybridBlock](../../../api/gluon/hybrid_block.rst#mxnet.gluon.HybridBlock) or the [HybridSequential](../../../api/gluon/nn/index.rst#mxnet.gluon.nn.HybridSequential) and [HybridConcatenate](../../../api/gluon/nn/index.rst#mxnet.gluon.nn.HybridConcatenate) classes. By default, they are executed in the same way [Block](../../../api/gluon/block.rst#mxnet.gluon.Block) or [Sequential](../../../api/gluon/nn/index.rst#mxnet.gluon.nn.Sequential) and [Concurrent](../../../api/gluon/nn/index.rst#mxnet.gluon.nn.Concatenate) classes are executed in imperative programming. When the  `hybridize`  function is called, Gluon will convert the program’s execution into the style used in symbolic programming. This allows one to optimize the compute-intensive components without sacrifices in the way a model is implemented. In fact, most models can make use of hybrid programming’s execution style.\n\nYou will learn more about hybrid blocks and use them in the upcoming sections of the course.\n\n### Gluon\n\nGluon is an imperative high-level front end API in MXNet for deep learning that’s flexible and easy-to-use which comes with a lot of great features, and it can provide you everything you need: from experimentation to deploying the model without sacrificing training speed. This is because, as discussed above, you have access to both imperative and symbolic APIs through the introduction of hybrid programming. Gluon provides State of the Art models for many of the standard tasks such as Classification, Object Detection, Segmentation, etc. In one of the next sections of the tutorial, you will walk through an example of how to build a model using gluon, train it on a dataset, and make predictions with it.\n\n## Next steps\n\nDive deeper on [array representations](./1-nparray.ipynb) in MXNet.\n\n## References\n1.  [Dive into Deep Learning](http://d2l.ai/) \n"
  },
  {
    "path": "docs/python_docs/python/tutorials/getting-started/crash-course/1-nparray.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Step 1: Manipulate data with NP on MXNet\n\nThis getting started exercise introduces the MXNet `np` package for ndarrays.\nThese ndarrays extend the functionality of the common NumPy ndarrays, by adding\nsupport for gpu's and by adding auto-differentiation with autograd. Now, many\nNumPy methods are available within MXNet; therefore, we will only briefly cover\nsome of what is available.\n\n## Import packages and create an array\nTo get started, run the following commands to import the `np` package together\nwith the NumPy extensions package `npx`. Together, `np` with `npx` make up the\nNP on MXNet front end.\n\n```{.python .input}\nimport mxnet as mx\nfrom mxnet import np, npx\nnpx.set_np()  # Activate NumPy-like mode.\n```\n\nIn this step, create a 2D array (also called a matrix). The following code\nexample creates a matrix with values from two sets of numbers: 1, 2, 3 and 4, 5,\n6. This might also be referred to as a tuple of a tuple of integers.\n\n```{.python .input}\nnp.array(((1, 2, 3), (5, 6, 7)))\n```\n\nYou can also create a very simple matrix with the same shape (2 rows by 3\ncolumns), but fill it with 1's.\n\n```{.python .input}\nx = np.full((2, 3), 1) \nx\n```\n\nAlternatively, you could use the following array creation routine.\n\n```{.python .input}\nx = np.ones((2, 3)) \nx\n```\n\nYou can create arrays whose values are sampled randomly. For example, sampling\nvalues uniformly between -1 and 1. The following code example creates the same\nshape, but with random sampling.\n\n```{.python .input}\ny = np.random.uniform(-1, 1, (2, 3))\ny\n```\n\nAs with NumPy, the dimensions of each ndarray are shown by accessing the\n`.shape` attribute. As the following code example shows, you can also query for\n`size`, which is equal to the product of the components of the shape. In\naddition, `.dtype` tells the data type of the stored values. As you notice when\nwe generate random uniform values we generate `float32` not `float64` as normal\nNumPy arrays.\n\n```{.python .input}\n(x.shape, x.size, x.dtype)\n```\n\nYou could also specifiy the datatype when you create your ndarray.\n\n```{.python .input}\nx = np.full((2, 3), 1, dtype=\"int8\") \nx.dtype\n```\n\nVersus the default of `float32`.\n\n```{.python .input}\nx = np.full((2, 3), 1) \nx.dtype\n```\n\nWhen we multiply, by default we use the datatype with the most precision.\n\n```{.python .input}\nx = x.astype(\"int8\") + x.astype(int) + x.astype(\"float32\")\nx.dtype\n```\n\n## Performing operations on an array\n\nA ndarray supports a large number of standard mathematical operations. Here are\nsome examples. You can perform element-wise multiplication by using the\nfollowing code example.\n\n```{.python .input}\nx * y\n```\n\nYou can perform exponentiation by using the following code example.\n\n```{.python .input}\nnp.exp(y)\n```\n\nYou can also find a matrix’s transpose to compute a proper matrix-matrix product\nby using the following code example.\n\n```{.python .input}\nnp.dot(x, y.T)\n```\n\nAlternatively, you could use the matrix multiplication function.\n\n```{.python .input}\nnp.matmul(x, y.T)\n```\n\nYou can leverage built in operators, like summation.\n\n```{.python .input}\nx.sum()\n```\n\nYou can also gather a mean value.\n\n```{.python .input}\nx.mean()\n```\n\nYou can perform flatten and reshape just like you normally would in NumPy!\n\n```{.python .input}\nx.flatten()\n```\n\n```{.python .input}\nx.reshape(6, 1)\n```\n\n## Indexing an array\n\nThe ndarrays support slicing in many ways you might want to access your data.\nThe following code example shows how to read a particular element, which returns\na 1D array with shape `(1,)`.\n\n```{.python .input}\ny[1, 2]\n```\n\nThis example shows how to read the second and third columns from `y`.\n\n```{.python .input}\ny[:, 1:3]\n```\n\nThis example shows how to write to a specific element.\n\n```{.python .input}\ny[:, 1:3] = 2\ny\n```\n\nYou can perform multi-dimensional slicing, which is shown in the following code\nexample.\n\n```{.python .input}\ny[1:2, 0:2] = 4\ny\n```\n\n## Converting between MXNet ndarrays and NumPy arrays\n\nYou can convert MXNet ndarrays to and from NumPy ndarrays, as shown in the\nfollowing example. The converted arrays do not share memory.\n\n```{.python .input}\na = x.asnumpy()\n(type(a), a)\n```\n\n```{.python .input}\na = np.array(a)\n(type(a), a)\n```\n\nAdditionally, you can move them to different GPU devices. You will dive more\ninto this later, but here is an example for now.\n\n```{.python .input}\na.copyto(mx.gpu(0))\n```\n\n## Next Steps\n\nNdarrays also have some additional features which make Deep Learning possible\nand efficient. Namely, differentiation, and being able to leverage GPU's.\nAnother important feature of ndarrays that we will discuss later is \nautograd. But first, we will abstract an additional level and talk about building\nNeural Network Layers [Step 2: Create a neural network](./2-create-nn.ipynb)\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/getting-started/crash-course/2-create-nn.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Step 2: Create a neural network\n\nIn this step, you learn how to use NP on Apache MXNet to create neural networks\nin Gluon. In addition to the `np` package that you learned about in the previous\nstep [Step 1: Manipulate data with NP on MXNet](./1-nparray.ipynb), you also need to\nimport the neural network modules from `gluon`. Gluon includes built-in neural\nnetwork layers in the following two modules:\n\n1. `mxnet.gluon.nn`: NN module that maintained by the mxnet team\n2. `mxnet.gluon.contrib.nn`: Experiemental module that is contributed by the\ncommunity\n\nUse the following commands to import the packages required for this step.\n\n```{.python .input}\nfrom mxnet import np, npx\nfrom mxnet.gluon import nn\nnpx.set_np()  # Change MXNet to the numpy-like mode.\n```\n\n## Create your neural network's first layer\n\nIn this section, you will create a simple neural network with Gluon. One of the\nsimplest network you can create is a single **Dense** layer or **densely-\nconnected** layer. A dense layer consists of nodes in the input that are\nconnected to every node in the next layer. Use the following code example to\nstart with a dense layer with five output units.\n\n```{.python .input}\nlayer = nn.Dense(5)\nlayer\n# output: Dense(-1 -> 5, linear)\n```\n\nIn the example above, the output is `Dense(-1 -> 5, linear)`. The **-1** in the\noutput denotes that the size of the input layer is not specified during\ninitialization.\n\nYou can also call the **Dense** layer with an `in_units` parameter if you know\nthe shape of your input unit.\n\n```{.python .input}\nlayer = nn.Dense(5,in_units=3)\nlayer\n```\n\nIn addition to the `in_units` param, you can also add an activation function to\nthe layer using the `activation` param. The Dense layer implements the operation\n\n$$output = \\sigma(W \\cdot X + b)$$\n\nCall the Dense layer with an `activation` parameter to use an activation\nfunction.\n\n```{.python .input}\nlayer = nn.Dense(5, in_units=3,activation='relu')\n```\n\nVoila! Congratulations on creating a simple neural network. But for most of your\nuse cases, you will need to create a neural network with more than one dense\nlayer or with multiple types of other layers. In addition to the `Dense` layer,\nyou can find more layers at [mxnet nn layers](../../../api/gluon/nn/index.rst#module-mxnet.gluon.nn)\n\nSo now that you have created a neural network, you are probably wondering how to\npass data into your network?\n\nFirst, you need to initialize the network weights, if you use the default\ninitialization method which draws random values uniformly in the range $[-0.7,\n0.7]$. You can see this in the following example.\n\n**Note**: Initialization is discussed at a little deeper detail in the next\nnotebook\n\n```{.python .input}\nlayer.initialize()\n```\n\nNow that you have initialized your network, you can give it data. Passing data\nthrough a network is also called a forward pass. You can do a forward pass with\nrandom data, shown in the following example. First, you create a `(10,3)` shape\nrandom input `x` and feed the data into the layer to compute the output.\n\n```{.python .input}\nx = np.random.uniform(-1,1,(10,3))\nlayer(x)\n```\n\nThe layer produces a `(10,5)` shape output from your `(10,3)` input.\n\n**When you don't specify the `in_unit` parameter, the system  automatically\ninfers it during the first time you feed in data during the first forward step\nafter you create and initialize the weights.**\n\n\n```{.python .input}\nlayer.params\n```\n\nThe `weights` and `bias` can be accessed using the `.data()` method.\n\n```{.python .input}\nlayer.weight.data()\n```\n\n## Chain layers into a neural network using nn.Sequential\n\nSequential provides a special way of rapidly building networks when when the\nnetwork architecture follows a common design pattern: the layers look like a\nstack of pancakes. Many networks follow this pattern: a bunch of layers, one\nstacked on top of another, where the output of each layer is fed directly to the\ninput to the next layer. To use sequential, simply provide a list of layers\n(pass in the layers by calling `net.add(<Layer goes here!>`). To do this you can\nuse your previous example of Dense layers and create a 3-layer multi layer\nperceptron. You can create a sequential block using `nn.Sequential()` method and\nadd layers using `add()` method.\n\n```{.python .input}\nnet = nn.Sequential()\n\nnet.add(nn.Dense(5,in_units=3,activation='relu'),\n        nn.Dense(25, activation='relu'), nn.Dense(2))\nnet\n```\n\nThe layers are ordered exactly the way you defined your neural network with\nindex starting from 0. You can access the layers by indexing the network using\n`[]`.\n\n```{.python .input}\nnet[1]\n```\n\n## Create a custom neural network architecture flexibly\n\n`nn.Sequential()` allows you to create your multi-layer neural network with\nexisting layers from `gluon.nn`. It also includes a pre-defined `forward()`\nfunction that sequentially executes added layers. But what if the built-in\nlayers are not sufficient for your needs. If you want to create networks like\nResNet which has complex but repeatable components, how do you create such a\nnetwork?\n\nIn gluon, every neural network layer is defined by using a base class\n`nn.Block()`. A Block has one main job - define a forward method that takes some\ninput x and generates an output. A Block can just do something simple like apply\nan activation function. It can combine multiple layers together in a single\nblock or also combine a bunch of other Blocks together in creative ways to\ncreate complex networks like Resnet. In this case, you will construct three\nDense layers. The `forward()` method can then invoke the layers in turn to\ngenerate its output.\n\nCreate a subclass of `nn.Block` and implement two methods by using the following\ncode.\n\n- `__init__` create the layers\n- `forward` define the forward function.\n\n```{.python .input}\nclass Net(nn.Block):\n    def __init__(self):\n        super().__init__()\n    def forward(self, x):\n        return x\n```\n\n```{.python .input}\nclass MLP(nn.Block):\n    def __init__(self):\n        super().__init__()\n        self.dense1 = nn.Dense(5,activation='relu')\n        self.dense2 = nn.Dense(25,activation='relu')\n        self.dense3 = nn.Dense(2)\n\n    def forward(self, x):\n        layer1 = self.dense1(x)\n        layer2 = self.dense2(layer1)\n        layer3 = self.dense3(layer2)\n        return layer3\n\nnet = MLP()\nnet\n```\n\n```{.python .input}\nnet.dense1.params\n```\nEach layer includes parameters that are stored in a `Parameter` class. You can\naccess them using the `params()` method.\n\n## Creating custom layers using Parameters (Blocks API)\n\nMXNet includes a `Parameter` method to hold your parameters in each layer. You\ncan create custom layers using the `Parameter` class to include computation that\nmay otherwise be not included in the built-in layers. For example, for a dense\nlayer, the weights and biases will be created using the `Parameter` method. But\nif you want to add additional computation to the dense layer, you can create it\nusing parameter method.\n\nInstantiate a parameter, e.g weights with a size `(5,0)` using the `shape`\nargument.\n\n```{.python .input}\nfrom mxnet.gluon import Parameter\n\nweight = Parameter(\"custom_parameter_weight\",shape=(5,-1))\nbias = Parameter(\"custom_parameter_bias\",shape=(5,-1))\n\nweight,bias\n```\n\nThe `Parameter` method includes a `grad_req` argument that specifies how you\nwant to capture gradients for this Parameter. Under the hood, that lets gluon\nknow that it has to call `.attach_grad()` on the underlying array. By default,\nthe gradient is updated everytime the gradient is written to the grad\n`grad_req='write'`.\n\nNow that you know how parameters work, you are ready to create your very own\nfully-connected custom layer.\n\nTo create the custom layers using parameters, you can use the same skeleton with\n`nn.Block` base class. You will create a custom dense layer that takes parameter\nx and returns computed `w*x + b` without any activation function\n\n```{.python .input}\nclass custom_layer(nn.Block):\n   def __init__(self, out_units, in_units=0):\n       super().__init__()\n       self.weight = Parameter(\"weight\", shape=(in_units,out_units), allow_deferred_init=True)\n       self.bias = Parameter(\"bias\", shape=(out_units,), allow_deferred_init=True)\n   def forward(self, x):\n       return np.dot(x, self.weight.data()) + self.bias.data()\n```\n\nParameter can be instantiated before the corresponding data is instantiated. For\nexample, when you instantiate a Block but the shapes of each parameter still\nneed to be inferred, the Parameter will wait for the shape to be inferred before\nallocating memory.\n\n```{.python .input}\ndense = custom_layer(3,in_units=5)\ndense.initialize()\ndense(np.random.uniform(size=(4, 5)))\n```\n\nSimilarly, you can use the following code to implement a famous network called\n[LeNet](http://yann.lecun.com/exdb/lenet/) through `nn.Block` using the built-in\n`Dense` layer and using `custom_layer` as the last layer\n\n```{.python .input}\nclass LeNet(nn.Block):\n   def __init__(self):\n       super().__init__()\n       self.conv1 = nn.Conv2D(channels=6, kernel_size=3, activation='relu')\n       self.pool1 = nn.MaxPool2D(pool_size=2, strides=2)\n       self.conv2 = nn.Conv2D(channels=16, kernel_size=3, activation='relu')\n       self.pool2 = nn.MaxPool2D(pool_size=2, strides=2)\n       self.dense1 = nn.Dense(120, activation=\"relu\")\n       self.dense2 = nn.Dense(84, activation=\"relu\")\n       self.dense3 = nn.Dense(10)\n   def forward(self, x):\n       x = self.conv1(x)\n       x = self.pool1(x)\n       x = self.conv2(x)\n       x = self.pool2(x)\n       x = self.dense1(x)\n       x = self.dense2(x)\n       x = self.dense3(x)\n       return x\n\nlenet = LeNet()\n```\n\n```{.python .input}\nclass LeNet_custom(nn.Block):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2D(channels=6, kernel_size=3, activation='relu')\n        self.pool1 = nn.MaxPool2D(pool_size=2, strides=2)\n        self.conv2 = nn.Conv2D(channels=16, kernel_size=3, activation='relu')\n        self.pool2 = nn.MaxPool2D(pool_size=2, strides=2)\n        self.dense1 = nn.Dense(120, activation=\"relu\")\n        self.dense2 = nn.Dense(84, activation=\"relu\")\n        self.dense3 = custom_layer(10,84)\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.pool1(x)\n        x = self.conv2(x)\n        x = self.pool2(x)\n        x = self.dense1(x)\n        x = self.dense2(x)\n        x = self.dense3(x)\n        return x\n\nlenet_custom = LeNet_custom()\n```\n\n```{.python .input}\nimage_data = np.random.uniform(-1,1, (1,1,28,28))\n\nlenet.initialize()\nlenet_custom.initialize()\n\nprint(\"Lenet:\")\nprint(lenet(image_data))\n\nprint(\"Custom Lenet:\")\nprint(lenet_custom(image_data))\n```\n\n\nYou can use `.data` method to access the weights and bias of a particular layer.\nFor example, the following  accesses the first layer's weight and sixth layer's bias.\n\n```{.python .input}\nlenet.conv1.weight.data().shape, lenet.dense1.bias.data().shape\n```\n\n## Using predefined (pretrained) architectures\n\nTill now, you have seen how to create your own neural network architectures. But\nwhat if you want to replicate or baseline your dataset using some of the common\nmodels in computer visions or natural language processing (NLP). Gluon includes\ncommon architectures that you can directly use. The Gluon Model Zoo provides a\ncollection of off-the-shelf models e.g. RESNET, BERT etc. These architectures\nare found at:\n\n- [Gluon CV model zoo](https://cv.gluon.ai/model_zoo/index.html)\n\n- [Gluon NLP model zoo](https://nlp.gluon.ai/model_zoo/index.html)\n\n```{.python .input}\nfrom mxnet.gluon import model_zoo\n\nnet = model_zoo.vision.resnet50_v2(pretrained=True)\nnet.hybridize()\n\ndummy_input = np.ones(shape=(1,3,224,224))\noutput = net(dummy_input)\noutput.shape\n```\n\n## Deciding the paradigm for your network\n\nIn MXNet, Gluon API (Imperative programming paradigm) provides a user friendly\nway for quick prototyping, easy debugging and natural control flow for people\nfamiliar with python programming.\n\nHowever, at the backend, MXNET can also convert the network using Symbolic or\nDeclarative programming into static graphs with low level optimizations on\noperators. However, static graphs are less flexible because any logic must be\nencoded into the graph as special operators like scan, while_loop and cond. It’s\nalso hard to debug.\n\nSo how can you make use of symbolic programming while getting the flexibility of\nimperative programming to quickly prototype and debug?\n\nEnter **HybridBlock**\n\nHybridBlocks can run in a fully imperatively way where you define their\ncomputation with real functions acting on real inputs. But they’re also capable\nof running symbolically, acting on placeholders. Gluon hides most of this under\nthe hood so you will only need to know how it works when you want to write your\nown layers.\n\n```{.python .input}\nnet_hybrid_seq = nn.HybridSequential()\n\nnet_hybrid_seq.add(nn.Dense(5,in_units=3,activation='relu'),\n nn.Dense(25, activation='relu'), nn.Dense(2) )\nnet_hybrid_seq\n```\n\nTo compile and optimize `HybridSequential`, you can call its `hybridize` method.\n\n```{.python .input}\nnet_hybrid_seq.hybridize()\n```\n\n\n## Creating custom layers using Parameters (HybridBlocks API)\n\nWhen you instantiated your custom layer, you specified the input dimension\n`in_units` that initializes the weights with the shape specified by `in_units`\nand `out_units`. If you leave the shape of `in_unit` as unknown, you defer the\nshape to the first forward pass. For the custom layer, you define the\n`infer_shape()` method and let the shape be inferred at runtime.\n\n```{.python .input}\nclass CustomLayer(nn.HybridBlock):\n    def __init__(self, out_units, in_units=-1):\n        super().__init__()\n        self.weight = Parameter(\"weight\", shape=(in_units, out_units), allow_deferred_init=True)\n        self.bias = Parameter(\"bias\", shape=(out_units,), allow_deferred_init=True)\n\n    def forward(self, x):\n        print(self.weight.shape, self.bias.shape)\n        return np.dot(x, self.weight.data()) + self.bias.data()\n\n    def infer_shape(self, x):\n        print(self.weight.shape,x.shape)\n        self.weight.shape = (x.shape[-1],self.weight.shape[1])\n        dense = CustomLayer(3)\n\ndense.initialize()\ndense(np.random.uniform(size=(4, 5)))\n```\n\n### Performance\n\nTo get a sense of the speedup from hybridizing, you can compare the performance\nbefore and after hybridizing by measuring the time it takes to make 1000 forward\npasses through the network.\n\n```{.python .input}\nfrom time import time\n\ndef benchmark(net, x):\n    y = net(x)\n    start = time()\n    for i in range(1,1000):\n        y = net(x)\n    return time() - start\n\nx_bench = np.random.normal(size=(1,512))\n\nnet_hybrid_seq = nn.HybridSequential()\n\nnet_hybrid_seq.add(nn.Dense(256,activation='relu'),\n                   nn.Dense(128, activation='relu'),\n                   nn.Dense(2))\nnet_hybrid_seq.initialize()\n\nprint('Before hybridizing: %.4f sec'%(benchmark(net_hybrid_seq, x_bench)))\nnet_hybrid_seq.hybridize()\nprint('After hybridizing: %.4f sec'%(benchmark(net_hybrid_seq, x_bench)))\n```\n\nPeeling back another layer, you also have a `HybridBlock` which is the hybrid\nversion of the `Block` API.\n\nSimilar to the `Blocks` API, you define a `forward` function for `HybridBlock`\nthat takes an input `x`. MXNet takes care of hybridizing the model at the\nbackend so you don't have to make changes to your code to convert it to a\nsymbolic paradigm.\n\n```{.python .input}\nfrom mxnet.gluon import HybridBlock\n\nclass MLP_Hybrid(HybridBlock):\n    def __init__(self):\n        super().__init__()\n        self.dense1 = nn.Dense(256,activation='relu')\n        self.dense2 = nn.Dense(128,activation='relu')\n        self.dense3 = nn.Dense(2)\n    def forward(self, x):\n        layer1 = self.dense1(x)\n        layer2 = self.dense2(layer1)\n        layer3 = self.dense3(layer2)\n        return layer3\n\nnet_hybrid = MLP_Hybrid()\nnet_hybrid.initialize()\n\nprint('Before hybridizing: %.4f sec'%(benchmark(net_hybrid, x_bench)))\nnet_hybrid.hybridize()\nprint('After hybridizing: %.4f sec'%(benchmark(net_hybrid, x_bench)))\n```\n\nGiven a HybridBlock whose forward computation consists of going through other\nHybridBlocks, you can compile that section of the network by calling the\nHybridBlocks `.hybridize()` method.\n\nAll of MXNet’s predefined layers are HybridBlocks. This means that any network\nconsisting entirely of predefined MXNet layers can be compiled and run at much\nfaster speeds by calling `.hybridize()`.\n\n## Saving and Loading your models\n\nThe Blocks API also includes saving your models during and after training so\nthat you can host the model for inference or avoid training the model again from\nscratch. Another reason would be to train your model using one language (like\nPython that has a lot of tools for training) and run inference using a different\nlanguage.\n\nThere are two ways to save your model in MXNet.\n1. Save/load the model weights/parameters only\n2. Save/load the model weights/parameters and the architectures\n\n\n### 1. Save/load the model weights/parameters only\n\nYou can use `save_parameters` and `load_parameters` method to save and load the\nmodel weights. Take your simplest model `layer` and save your parameters first.\nThe model parameters are the params that you save **after** you train your\nmodel.\n\n```{.python .input}\nfile_name = 'layer.params'\nlayer.save_parameters(file_name)\n```\n\nAnd now load this model again. To load the parameters into a model, you will\nfirst have to build the model. To do this, you will need to create a simple\nfunction to build it.\n\n```{.python .input}\ndef build_model():\n    layer = nn.Dense(5, in_units=3,activation='relu')\n    return layer\n\nlayer_new = build_model()\n```\n\n```{.python .input}\nlayer_new.load_parameters('layer.params')\n```\n\n**Note**: The `save_parameters` and `load_parameters` method is used for models\nthat use a `Block` method instead of  `HybridBlock` method to build the model.\nThese models may have complex architectures where the model architectures may\nchange during execution. E.g. if you have a model that uses an if-else\nconditional statement to choose between two different architectures.\n\n### 2. Save/load the model weights/parameters and the architectures\n\nFor models that use the **HybridBlock**, the model architecture stays static and\ndo no change during execution. Therefore both model parameters **AND**\narchitecture can be saved and loaded using `export`, `imports` methods.\n\nNow look at your `MLP_Hybrid` model and export the model using the `export`\nfunction. The export function will export the model architecture into a `.json`\nfile and model parameters into a `.params` file.\n\n```{.python .input}\nnet_hybrid.export('MLP_hybrid')\n```\n\n```{.python .input}\nnet_hybrid.export('MLP_hybrid')\n```\n\nSimilarly, to load this model back, you can use `gluon.nn.SymbolBlock`. To\ndemonstrate that, load the network serialized above.\n\n```{.python .input}\nimport warnings\nwith warnings.catch_warnings():\n    warnings.simplefilter(\"ignore\")\n    net_loaded = nn.SymbolBlock.imports(\"MLP_hybrid-symbol.json\",\n                                        ['data'], \"MLP_hybrid-0000.params\",\n                                        device=None)\n```\n\n```{.python .input}\nnet_loaded(x_bench)\n```\n\n## Visualizing your models\n\nIn MXNet, the `Block.Summary()` method allows you to view the block’s shape\narguments and view the block’s parameters. When you combine multiple blocks into\na model, the `summary()` applied on the model allows you to view each block’s\nsummary, the total parameters, and the order of the blocks within the model. To\ndo this the `Block.summary()` method requires one forward pass of the data,\nthrough your network, in order to create the graph necessary for capturing the\ncorresponding shapes and parameters. Additionally, this method should be called\nbefore the hybridize method, since the hybridize method converts the graph into\na symbolic one, potentially changing the operations for optimal computation.\n\nLook at the following examples\n\n- layer: our single layer network\n- Lenet: a non-hybridized LeNet network\n- net_Hybrid: our MLP Hybrid network\n\n```{.python .input}\nlayer.summary(x)\n```\n\n```{.python .input}\nlenet.summary(image_data)\n```\n\nYou are able to print the summaries of the two networks `layer` and `lenet`\neasily since you didn't hybridize the two networks. However, the last network\n`net_Hybrid` was hybridized above and throws an `AssertionError` if you try\n`net_Hybrid.summary(x_bench)`. To print the summary for `net_Hybrid`, call\nanother instance of the same network and instantiate it for our summary and then\nhybridize it\n\n```{.python .input}\nnet_hybrid_summary = MLP_Hybrid()\n\nnet_hybrid_summary.initialize()\n\nnet_hybrid_summary.summary(x_bench)\n\nnet_hybrid_summary.hybridize()\n```\n\n## Next steps:\n\nNow that you have created a neural network, learn how to automatically compute\nthe gradients in [Step 3: Automatic differentiation with autograd](./3-autograd.ipynb).\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/getting-started/crash-course/3-autograd.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Step 3: Automatic differentiation with autograd\n\nIn this step, you learn how to use the MXNet `autograd` package to perform\ngradient calculations.\n\n## Basic use\n\nTo get started, import the `autograd` package with the following code.\n\n```{.python .input}\nfrom mxnet import np, npx\nfrom mxnet import autograd\nnpx.set_np()\n```\n\nAs an example, you could differentiate a function $f(x) = 2 x^2$ with respect to\nparameter $x$. For Autograd, you can start by assigning an initial value of $x$,\nas follows:\n\n```{.python .input}\nx = np.array([[1, 2], [3, 4]])\nx\n```\n\nAfter you compute the gradient of $f(x)$ with respect to $x$, you need a place\nto store it. In MXNet, you can tell a ndarray that you plan to store a gradient\nby invoking its `attach_grad` method, as shown in the following example.\n\n```{.python .input}\nx.attach_grad()\n```\n\nNext, define the function $y=f(x)$. To let MXNet store $y$, so that you can\ncompute gradients later, use the following code to put the definition inside an\n`autograd.record()` scope.\n\n```{.python .input}\nwith autograd.record():\n    y = 2 * x * x\n```\n\nYou can invoke back propagation (backprop) by calling `y.backward()`. When $y$\nhas more than one entry, `y.backward()` is equivalent to `y.sum().backward()`.\n\n```{.python .input}\ny.backward()\n```\n\nNext, verify whether this is the expected output. Note that $y=2x^2$ and\n$\\frac{dy}{dx} = 4x$, which should be `[[4, 8],[12, 16]]`. Check the\nautomatically computed results.\n\n```{.python .input}\nx.grad\n```\n\nNow you get to dive into `y.backward()` by first discussing a bit on gradients. As\nalluded to earlier `y.backward()` is equivalent to `y.sum().backward()`.\n\n```{.python .input}\nwith autograd.record():\n    y = np.sum(2 * x * x)\ny.backward()\nx.grad\n```\n\nAdditionally, you can only run backward once. Unless you use the flag\n`retain_graph` to be `True`.\n\n```{.python .input}\nwith autograd.record():\n    y = np.sum(2 * x * x)\ny.backward(retain_graph=True)\nprint(x.grad)\nprint(\"Since you have retained your previous graph you can run backward again\")\ny.backward()\nprint(x.grad)\n\ntry:\n    y.backward()\nexcept:\n    print(\"However, you can't do backward twice unless you retain the graph.\")\n```\n\n## Custom MXNet ndarray operations\n\nIn order to understand the `backward()` method it is beneficial to first\nunderstand how you can create custom operations. MXNet operators are classes\nwith a forward and backward method. Where the number of args in `backward()`\nmust equal the number of items returned in the `forward()` method. Additionally,\nthe number of arguments in the `forward()` method must match the number of\noutput arguments from `backward()`. You can modify the gradients in backward to\nreturn custom gradients. For instance, below you can return a different gradient then\nthe actual derivative.\n\n```{.python .input}\nclass MyFirstCustomOperation(autograd.Function):\n    def __init__(self):\n        super().__init__()\n\n    def forward(self,x,y):\n        return 2 * x, 2 * x * y, 2 * y\n\n    def backward(self, dx, dxy, dy):\n        \"\"\"\n        The input number of arguments must match the number of outputs from forward.\n        Furthermore, the number of output arguments must match the number of inputs from forward.\n        \"\"\"\n        return x, y\n```\n\nNow you can use the first custom operation you have built.\n\n```{.python .input}\nx = np.random.uniform(-1, 1, (2, 3)) \ny = np.random.uniform(-1, 1, (2, 3))\nx.attach_grad()\ny.attach_grad()\nwith autograd.record():\n    z = MyFirstCustomOperation()\n    z1, z2, z3 = z(x, y)\n    out = z1 + z2 + z3 \nout.backward()\nprint(np.array_equiv(x.asnumpy(), x.asnumpy()))\nprint(np.array_equiv(y.asnumpy(), y.asnumpy()))\n```\n\nAlternatively, you may want to have a function which is different depending on\nif you are training or not.\n\n```{.python .input}\ndef my_first_function(x):\n    if autograd.is_training(): # Return something else when training\n        return(4 * x)\n    else:\n        return(x)\n```\n\n```{.python .input}\ny = my_first_function(x)\nprint(np.array_equiv(y.asnumpy(), x.asnumpy()))\nwith autograd.record(train_mode=False):\n    y = my_first_function(x)\ny.backward()\nprint(x.grad)\nwith autograd.record(train_mode=True): # train_mode = True by default\n    y = my_first_function(x)\ny.backward()\nprint(x.grad)\n```\n\nYou could create functions with `autograd.record()`.\n\n```{.python .input}\ndef my_second_function(x):\n    with autograd.record():\n        return(2 * x)\n```\n\n```{.python .input}\ny = my_second_function(x)\ny.backward()\nprint(x.grad)\n```\n\nYou can also combine multiple functions.\n\n```{.python .input}\ny = my_second_function(x)\nwith autograd.record():\n    z = my_second_function(y) + 2\nz.backward()\nprint(x.grad)\n```\n\nAdditionally, MXNet records the execution trace and computes the gradient\naccordingly. The following function `f` doubles the inputs until its `norm`\nreaches 1000. Then it selects one element depending on the sum of its elements.\n\n```{.python .input}\ndef f(a):\n    b = a * 2\n    while np.abs(b).sum() < 1000:\n        b = b * 2\n    if b.sum() >= 0:\n        c = b[0]\n    else:\n        c = b[1]\n    return c\n```\n\nIn this example, you record the trace and feed in a random value.\n\n```{.python .input}\na = np.random.uniform(size=2)\na.attach_grad()\nwith autograd.record():\n    c = f(a)\nc.backward()\n```\n\nYou can see that `b` is a linear function of `a`, and `c` is chosen from `b`.\nThe gradient with respect to `a` be will be either `[c/a[0], 0]` or `[0,\nc/a[1]]`, depending on which element from `b` is picked. You see the results of\nthis example with this code:\n\n```{.python .input}\na.grad == c / a\n```\n\nAs you can notice there are 3 values along the dimension 0, so taking a `mean`\nalong this axis is the same as summing that axis and multiplying by `1/3`.\n\n## Advanced MXNet ndarray operations with Autograd\n\nYou can control gradients for different ndarray operations. For instance,\nperhaps you want to check that the gradients are propagating properly?\nthe `attach_grad()` method automatically detaches itself from the gradient.\nTherefore, the input up until y will no longer look like it has `x`. To\nillustrate this notice that `x.grad` and `y.grad` is not the same in the second\nexample.\n\n```{.python .input}\nwith autograd.record():\n    y = 3 * x\n    y.attach_grad()\n    z = 4 * y + 2 * x\nz.backward()\nprint(x.grad)\nprint(y.grad)\n```\n\nIs not the same as:\n\n```{.python .input}\nwith autograd.record():\n    y = 3 * x\n    z = 4 * y + 2 * x\nz.backward()\nprint(x.grad)\nprint(y.grad)\n```\n\n## Next steps\n\nLearn how to initialize weights, choose loss function, metrics and optimizers for training your neural network [Step 4: Necessary components\nto train the neural network](./4-components.ipynb).\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/getting-started/crash-course/4-components.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Step 4: Necessary components that are not in the network\n\nData and models are not the only components that\nyou need to train a deep learning model. In this notebook, you will\nlearn about the common components involved in training deep learning models. \nHere is a list of components necessary for training models in MXNet.\n\n1. Initialization\n2. Loss functions\n    1. Built-in\n    2. Custom\n3. Optimizers\n4. Metrics\n\n```{.python .input}\nfrom mxnet import np, npx,gluon\nimport mxnet as mx\nfrom mxnet.gluon import nn\nnpx.set_np()\n\ndevice = mx.cpu()\n```\n\n## Initialization\n\nIn a previous notebook, you used `net.initialize()` to initialize the network\nbefore a forward pass. Now, you will learn about initialization in a little more\ndetail.\n\nFirst, define and initialize the `sequential` network from earlier.\nAfter you initialize it, print the parameters using `collect_params()` method.\n\n```{.python .input}\nnet = nn.Sequential()\n\nnet.add(nn.Dense(5, in_units=3, activation=\"relu\"),\n        nn.Dense(25, activation=\"relu\"),\n        nn.Dense(2)\n       )\n\nnet\n```\n\n```{.python .input}\nnet.initialize()\nparams = net.collect_params()\n\nfor key, value in params.items():\n    print(key, value)\n\n\n```\n\nNext, you will print shape and params after the first forward pass.\n\n```{.python .input}\nx = np.random.uniform(-1, 1, (10, 3))\nnet(x)  # Forward computation\n\nparams = net.collect_params()\nfor key, value in params.items():\n    print(key, value)\n\n\n```\n\n#### Built-in Initialization\n\nMXNet makes it easy to initialize by providing many common initializers. A subset that you will be using in the following sections include:\n\n- Constant\n- Normal\n\nFor more information, see\n[Initializers](../../../api/initializer/index.rst)\n\nWhen you use `net.intialize()`, MXNet, by default, initializes the weight matrices uniformly\nby drawing random values with a uniform-distribution between −0.07 and 0.07 and\nupdates the bias parameters by setting them all to 0.\n\nTo initialize your network using different built-in types, you have to use the\n`init` keyword argument in the `initialize()` method. Here is an example using\n`constant` and `normal` initialization.\n\n```{.python .input}\nfrom mxnet import init\n\n# Constant init initializes the weights to be a constant value for all the params\nnet.initialize(init=init.Constant(3), device=device)\nprint(net[0].weight.data()[0])\n```\n\nIf you use Normal to initialize your weights then you will use a normal\ndistribution with a mean of zero and standard deviation of sigma. If you have\nalready initialized the weight but want to reinitialize the weight, set the\n`force_reinit` flag to `True`.\n\n```{.python .input}\nnet.initialize(init=init.Normal(sigma=0.2), force_reinit=True, device=device)\nprint(net[0].weight.data()[0])\n```\n\n## Components used in a training loop\n\nTill now you have seen how to create an algorithm and how to initialize it using mxnet\nAPIs; additionally you have learned the basics of using mxnet. When you start training the\nML algorithm, how do you actually teach the algorithm to learn or train?\n\nThere are three main components for training an algorithm.\n\n1. Loss function: calculates how far the model is from the true distribution\n2. Autograd: the mxnet auto differentiation tool that calculates the gradients to\noptimize the parameters\n3. Optimizer: updates the parameters based on an optimization algorithm\n\nYou have already learned about autograd in the previous notebook. In this\nnotebook, you will learn more about loss functions and optimizers.\n\n## Loss function\n\nLoss functions are used to train neural networks and help the algorithm learn\nfrom the data. The loss function computes the difference between the\noutput from the neural network and ground truth. This output is used to\nupdate the neural network weights during training. Next, you will look at a\nsimple example.\n\nSuppose you have a neural network `net` and the data is stored in a variable\n`data`. The data consists of 5 total records (rows) and two features (columns)\nand the output from the neural network after the first epoch is given by the\nvariable `nn_output`.\n\n```{.python .input}\nnet = gluon.nn.Dense(1)\nnet.initialize()\n\nnn_input = np.array([[1.2, 0.56],\n                     [3.0, 0.72],\n                     [0.89, 0.9],\n                     [0.89, 2.3],\n                     [0.99, 0.52]])\n\nnn_output = net(nn_input)\nnn_output\n```\n\nThe ground truth value of the data is stored in `groundtruth_label` is\n\n```{.python .input}\ngroundtruth_label = np.array([[0.0083],\n                             [0.00382],\n                             [0.02061],\n                             [0.00495],\n                             [0.00639]]).reshape(5, 1)\n```\n\nFor this problem, you will use the L2 Loss. L2Loss, also called Mean Squared Error, is a\nregression loss function that computes the squared distances between the target\nvalues and the output of the neural network. It is defined as:\n\n$$L = \\frac{1}{2N}\\sum_i{|label_i − pred_i|)^2}$$\n\nThe L2 loss function creates larger gradients for loss values which are farther apart due to the\nsquare operator and it also smooths the loss function space. \n\n```{.python .input}\ndef L2Loss(output_values, true_values):\n    return np.mean((output_values - true_values) ** 2, axis=1) / 2\n\nL2Loss(nn_output, groundtruth_label)\n```\n\nNow, you can do the same thing using the mxnet API\n\n```{.python .input}\nfrom mxnet.gluon import nn, loss as gloss\nloss = gloss.L2Loss()\n\nloss(nn_output, groundtruth_label)\n```\n\nA network can improve by iteratively updating its weights to minimise the loss.\nSome tasks use a combination of multiple loss functions, but often you will just\nuse one. MXNet Gluon provides a number of the most commonly used loss functions.\nThe choice of your loss function will depend on your network and task. Some\ncommon tasks and loss function pairs include:\n\n- regression: L1Loss, L2Loss\n\n- classification: SigmoidBinaryCrossEntropyLoss, SoftmaxCrossEntropyLoss\n\n- embeddings: HingeLoss\n\n#### Customizing your Loss functions\n\nYou can also create custom loss functions using **Loss Blocks**.\n\nYou can inherit the base `Loss` class and write your own `forward` method. The\nbackward propagation will be automatically computed by autograd. However, that\nonly holds true if you can build your loss from existing mxnet operators.\n\n```{.python .input}\nfrom mxnet.gluon.loss import Loss\n\nclass custom_L1_loss(Loss):\n    def __init__(self, weight=None, batch_axis=0, **kwargs):\n        super(custom_L1_loss, self).__init__(weight, batch_axis, **kwargs)\n\n    def forward(self, pred, label):\n        l = np.abs(label - pred)\n        l = l.reshape(len(l),)\n        return l\n    \nL1 = custom_L1_loss()\nL1(nn_output, groundtruth_label)\n```\n\n```{.python .input}\nl1=gloss.L1Loss()\nl1(nn_output, groundtruth_label)\n```\n\n## Optimizer\n\nThe loss function determines how much to change the parameters based on how far the\nmodel is from the groundtruth. Optimizer determines how the model\nweights or parameters are updated based on the loss function. In Gluon, this\noptimization step is performed by the `gluon.Trainer`.\n\nHere is a basic example of how to call the `gluon.Trainer` method.\n\n```{.python .input}\nfrom mxnet import optimizer\n```\n\n```{.python .input}\ntrainer = gluon.Trainer(net.collect_params(),\n                        optimizer=\"Adam\",\n                        optimizer_params={\n                            \"learning_rate\":0.1,\n                            \"wd\":0.001\n                        })\n```\n\nWhen creating a **Gluon Trainer**, you must provide the trainer object with\n1. A collection of parameters that need to be learnt. The collection of\nparameters will be the weights and biases of your network that you are training.\n2. An Optimization algorithm (optimizer) that you want to use for training. This\nalgorithm will be used to update the parameters every training iteration when\n`trainer.step` is called. For more information, see\n[optimizers](../../../api/optimizer/index.rst)\n\n```{.python .input}\ncurr_weight = net.weight.data()\nprint(curr_weight)\n```\n\n```{.python .input}\nbatch_size = len(nn_input)\ntrainer.step(batch_size, ignore_stale_grad=True)\nprint(net.weight.data())\n```\n\n```{.python .input}\nprint(curr_weight - net.weight.grad() * 1 / 5)\n```\n\n## Metrics\n\nMXNet includes a `metrics` API that you can use to evaluate how your model is\nperforming. This is typically used during training to monitor performance on the\nvalidation set. MXNet includes many commonly used metrics, a few are listed below:\n\n- [Accuracy](../../../api/gluon/metric/index.rst#mxnet.gluon.metric.Accuracy)\n- [CrossEntropy](../../../api/gluon/metric/index.rst#mxnet.gluon.metric.CrossEntropy)\n- [Mean squared error](../../../api/gluon/metric/index.rst#mxnet.gluon.metric.MSE)\n- [Root mean squared error (RMSE)](../../../api/gluon/metric/index.rst#mxnet.gluon.metric.RMSE)\n\nNow, you will define two arrays for a dummy binary classification example.\n\n```{.python .input}\n# Vector of likelihoods for all the classes\npred = np.array([[0.1, 0.9], [0.05, 0.95], [0.83, 0.17], [0.63, 0.37]])\n\nlabels = np.array([1, 1, 0, 1])\n```\n\nBefore you can calculate the accuracy of your model, the metric (accuracy)\nshould be instantiated before the training loop\n\n```{.python .input}\nfrom mxnet.gluon.metric import Accuracy\n\nacc = Accuracy()\n```\n\nTo run and calculate the updated accuracy for each batch or epoch, you can call\nthe `update()` method. This method uses labels and predictions which can be\neither class indexes or a vector of likelihoods for all of the classes.\n\n```{.python .input}\nacc.update(labels=labels, preds=pred)\n```\n\n#### Creating custom metrics\n\nIn addition to built-in metrics, if you want to create a custom metric, you can\nuse the following skeleton code. This code inherits from the `EvalMetric` base\nclass.\n\n```{.python .input}\ndef MyCustomMetric(EvalMetric):\n    def __init__(self):\n        super().init()\n\n    def update(self, labels, preds):\n        pass\n\n```\n\nHere is an example using the Precision metric. First, define the two values\n`labels` and `preds`.\n\n```{.python .input}\nlabels = np.array([0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1])\npreds = np.array([0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0])\n```\n\nNext, define the custom metric class `precision` and instantiate it\n\n```{.python .input}\nfrom mxnet.gluon.metric import EvalMetric\n\nclass precision(EvalMetric):\n    def __init__(self):\n        super().__init__(name=\"Precision\")\n        \n    def update(self,labels, preds):\n        tp_labels = (labels == 1)\n        true_positives = sum(preds[tp_labels] == 1)\n        fp_labels = (labels == 0)\n        false_positives = sum(preds[fp_labels] == 1)\n        return true_positives / (true_positives + false_positives)\n        \np = precision()\n```\n\nAnd finally, call the `update` method to return the results of `precision` for your data\n\n```{.python .input}\np.update(np.array(labels), np.array(preds))\n```\n\n## Next steps\n\nNow that you have learned all the components required to train a neural network,\nyou will see how to load your data using the Gluon API in [Step 5: Gluon\nDatasets and DataLoader](./5-datasets.ipynb)\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/getting-started/crash-course/5-datasets.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Step 5: `Dataset`s and `DataLoader`\n\nOne of the most critical steps for model training and inference is loading the data: without data you can't do Machine Learning! In this tutorial you will use the Gluon API to define a Dataset and use a DataLoader to iterate through the dataset in mini-batches.\n\n\n```{.python .input}\nimport mxnet as mx\nimport os\nimport time\nimport tarfile\n```\n\n## Introduction to `Dataset`s\n\nDataset objects are used to represent collections of data, and include methods to load and parse the data (that is often stored on disk). Gluon has a number of different `Dataset` classes for working with image data straight out-of-the-box, but you'll use the ArrayDataset to introduce the idea of a `Dataset`.\n\nYou will first start by generating random data `X` (with 3 variables) and corresponding random labels `y` to simulate a typical supervised learning task. You will generate 10 samples and pass them all to the `ArrayDataset`.\n\n\n\n```{.python .input}\nmx.np.random.seed(42) # Fix the seed for reproducibility\nX = mx.np.random.uniform(size=(10, 3))\ny = mx.np.random.uniform(size=(10, 1))\ndataset = mx.gluon.data.dataset.ArrayDataset(X, y)\n```\n\nA key feature of a `Dataset` is the __*ability to retrieve a single sample given an index*__. Our random data and labels were generated in memory, so this `ArrayDataset` doesn't have to load anything from disk, but the interface is the same for all `Dataset`'s.\n\n\n\n```{.python .input}\nsample_idx = 4\nsample = dataset[sample_idx]\n\nassert len(sample) == 2\nassert sample[0].shape == (3, )\nassert sample[1].shape == (1, )\nprint(sample)\n```\n\n\nYou get a tuple of a data sample and its corresponding label, which makes sense because you passed the data `X` and the labels `y` in that order when you instantiated the `ArrayDataset`. You don't usually retrieve individual samples from `Dataset` objects though (unless you're quality checking the output samples). Instead you use a `DataLoader`.\n\n## Introduction to `DataLoader`\n\nA DataLoader is used to create mini-batches of samples from a Dataset, and provides a convenient iterator interface for looping these batches. It's typically much more efficient to pass a mini-batch of data through a neural network than a single sample at a time, because the computation can be performed in parallel. A required parameter of `DataLoader` is the size of the mini-batches you want to create, called `batch_size`.\n\nAnother benefit of using `DataLoader` is the ability to easily load data in parallel using multiprocessing. You can set the `num_workers` parameter to the number of CPUs available on your machine for maximum performance, or limit it to a lower number to spare resources.\n\n```{.python .input}\nfrom multiprocessing import cpu_count\nCPU_COUNT = cpu_count()\n\ndata_loader = mx.gluon.data.DataLoader(dataset, batch_size=5, num_workers=CPU_COUNT)\n\nfor X_batch, y_batch in data_loader:\n    print(\"X_batch has shape {}, and y_batch has shape {}\".format(X_batch.shape, y_batch.shape))\n```\n\nYou can see 2 mini-batches of data (and labels), each with 5 samples, which makes sense given that you started with a dataset of 10 samples. When comparing the shape of the batches to the samples returned by the `Dataset`,you've gained an extra dimension at the start which is sometimes called the batch axis.\n\nOur `data_loader` loop will stop when every sample of `dataset` has been returned as part of a batch. Sometimes the dataset length isn't divisible by the mini-batch size, leaving a final batch with a smaller number of samples. `DataLoader`'s default behavior is to return this smaller mini-batch, but this can be changed by setting the `last_batch` parameter to `discard` (which ignores the last batch) or `rollover` (which starts the next epoch with the remaining samples).\n\n## Machine learning with `Dataset`s and `DataLoader`s\n\nYou will often use a few different `Dataset` objects in your Machine Learning project. It's essential to separate your training dataset from testing dataset, and it's also good practice to have validation dataset (a.k.a. development dataset) that can be used for optimising hyperparameters.\n\nUsing Gluon `Dataset` objects, you define the data to be included in each of these separate datasets. It's simple to create your own custom `Dataset` classes for other types of data. You can even use included `Dataset` objects for common datasets if you want to experiment quickly; they download and parse the data for you! In this example you use the [Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) dataset from Zalando Research.\n\nMany of the image `Dataset`'s accept a function (via the optional `transform` parameter) which is applied to each sample returned by the `Dataset`. It's useful for performing data augmentation, but can also be used for more simple data type conversion and pixel value scaling as seen below.\n\n```{.python .input}\ndef transform(data, label):\n    data = data.astype('float32')/255\n    return data, label\n\ntrain_dataset = mx.gluon.data.vision.datasets.FashionMNIST(train=True).transform(transform)\nvalid_dataset = mx.gluon.data.vision.datasets.FashionMNIST(train=False).transform(transform)\n```\n\n\n```{.python .input}\nfrom matplotlib.pylab import imshow\n\nsample_idx = 234\nsample = train_dataset[sample_idx]\ndata = sample[0]\nlabel = sample[1]\nlabel_desc = {0:'T-shirt/top', 1:'Trouser', 2:'Pullover', 3:'Dress', 4:'Coat', 5:'Sandal', 6:'Shirt', 7:'Sneaker', 8:'Bag', 9:'Ankle boot'}\n\nprint(\"Data type: {}\".format(data.dtype))\nprint(\"Label: {}\".format(label))\nprint(\"Label description: {}\".format(label_desc[label.item()]))\nimshow(data[:,:,0].asnumpy(), cmap='gray')\n```\n\n![datasets fashion mnist bag](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/gluon/datasets/fashion_mnist_bag.png)\n\nWhen training machine learning models it is important to shuffle the training samples every time you pass through the dataset (i.e. each epoch). Sometimes the order of your samples will have a spurious relationship with the target variable, and shuffling the samples helps remove this. With DataLoader it's as simple as adding `shuffle=True`. You don't need to shuffle the validation and testing data though.\n\n```{.python .input}\nbatch_size = 32\ntrain_data_loader = mx.gluon.data.DataLoader(train_dataset, batch_size, shuffle=True, num_workers=CPU_COUNT)\nvalid_data_loader = mx.gluon.data.DataLoader(valid_dataset, batch_size, num_workers=CPU_COUNT)\n```\n\nWith both `DataLoader`s defined, you can now train a model to classify each image and evaluate the validation loss at each epoch. See the next tutorial for how this is done.\n\n# Using own data with included `Dataset`s\n\nGluon has a number of different Dataset classes for working with your own image data straight out-of-the-box. You can get started quickly using the mxnet.gluon.data.vision.datasets.ImageFolderDataset which loads images directly from a user-defined folder, and infers the label (i.e. class) from the folders.\n\nHere you will run through an example for image classification, but a similar process applies for other vision tasks. If you already have your own collection of images to work with you should partition your data into training and test sets, and place all objects of the same class into seperate folders. Similar to:\n\n```\n./images/train/car/abc.jpg\n./images/train/car/efg.jpg\n./images/train/bus/hij.jpg\n./images/train/bus/klm.jpg\n./images/test/car/xyz.jpg\n./images/test/bus/uvw.jpg\n```\n\nYou can download the Caltech 101 dataset if you don't already have images to work with for this example, but please note the download is 126MB.\n\n\n```{.python .input}\ndata_folder = \"data\"\ndataset_name = \"101_ObjectCategories\"\narchive_file = \"{}.tar.gz\".format(dataset_name)\narchive_path = os.path.join(data_folder, archive_file)\ndata_url = \"https://s3.us-east-2.amazonaws.com/mxnet-public/\"\n\nif not os.path.isfile(archive_path):\n    mx.test_utils.download(\"{}{}\".format(data_url, archive_file), dirname = data_folder)\n    print('Extracting {} in {}...'.format(archive_file, data_folder))\n    tar = tarfile.open(archive_path, \"r:gz\")\n    tar.extractall(data_folder)\n    tar.close()\n    print('Data extracted.')\n```\n\nAfter downloading and extracting the data archive, you have two folders: `data/101_ObjectCategories` and `data/101_ObjectCategories_test`. You can then load the data into separate training and testing  ImageFolderDatasets.\n\ntraining_path = os.path.join(data_folder, dataset_name)\ntesting_path = os.path.join(data_folder, \"{}_test\".format(dataset_name))\n\nYou instantiate the ImageFolderDatasets by providing the path to the data, and the folder structure will be traversed to determine which image classes are available and which images correspond to each class. You must take care to ensure the same classes are both the training and testing datasets, otherwise the label encodings can get muddled.\n\nOptionally, you can pass a `transform` parameter to these `Dataset`'s as you've seen before.\n\n```{.python .input}\ntraining_path='./data/101_ObjectCategories'\ntesting_path='./data/101_ObjectCategories_test'\ntrain_dataset = mx.gluon.data.vision.datasets.ImageFolderDataset(training_path)\ntest_dataset = mx.gluon.data.vision.datasets.ImageFolderDataset(testing_path)\n```\n\nSamples from these datasets are tuples of data and label. Images are loaded from disk, decoded and optionally transformed when the `__getitem__(i)` method is called (equivalent to `train_dataset[i]`).\n\nAs with the Fashion MNIST dataset the labels will be integer encoded. You can use the `synsets` property of the ImageFolderDatasets to retrieve the original descriptions (e.g. `train_dataset.synsets[i]`).\n\n\n```{.python .input}\nsample_idx = 539\nsample = train_dataset[sample_idx]\ndata = sample[0]\nlabel = sample[1]\n\nprint(\"Data type: {}\".format(data.dtype))\nprint(\"Label: {}\".format(label))\nprint(\"Label description: {}\".format(train_dataset.synsets[label]))\nassert label == 1\n\nimshow(data.asnumpy(), cmap='gray')\n```\n\n\n![datasets caltech101 face](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/gluon/datasets/caltech101_face.png)<!--notebook-skip-line-->\n\n# Using your own data with custom `Dataset`s\n\nSometimes you have data that doesn't quite fit the format expected by the included Datasets. You might be able to preprocess your data to fit the expected format, but it is easy to create your own dataset to do this.\n\nAll you need to do is create a class that implements a `__getitem__` method, that returns a sample (i.e. a tuple of mx.np.ndarrays).\n\n# New in MXNet 2.0: faster C++ backend dataloaders\n\nAs part of an effort to speed up the current data loading pipeline using gluon dataset and dataloader, a new dataloader was created that uses only a C++ backend and avoids potentially slow calls to Python functions.\n\nSee [original issue](https://github.com/apache/incubator-mxnet/issues/17269), [pull request](https://github.com/apache/incubator-mxnet/pull/17464) and [implementation](https://github.com/apache/incubator-mxnet/pull/17841).\n\nThe current data loading pipeline is the major bottleneck for many training tasks. The flow can be summarized as:\n\n- `Dataset.__getitem__`\n- `Transform.__call__()/forward()`\n- `Batchify`\n- (optional communicate through shared_mem)\n- `split_and_load(devices)`\n- training on GPUs\n\nPerformance concerns include slow python dataset/transform functions, multithreading issues due to global interpreter lock, Python multiprocessing issues due to speed, and batchify issues due to poor memory management.\n\nThis new dataloader provides: \n- common C++ batchify functions that are split and context aware\n- a C++ MultithreadingDataLoader which inherit the same arguments as gluon.data.DataLoader but use MXNet internal multithreading rather than python multiprocessing.\n- fallback to python multiprocessing whenever the dataset is not fully supported by backend (e.g., there are custom python datasets) in the case that:\n    - the transform is not fully hybridizable\n    - batchify is not fully supported by backend\n\nUsers can continue to with the traditional gluon.data.Dataloader and the C++ backend will be applied automatically. The 'try_nopython' default is 'Auto', which detects whether the C++ backend is available given the dataset and transforms. \n\nHere you will show a performance increase on a t3.2xl instance for the CIFAR10 dataset with the C++ backend.\n\n## Using the C++ backend:\n\n```{.python .input}\ncpp_dl = mx.gluon.data.DataLoader(\n    mx.gluon.data.vision.CIFAR10(train=True, transform=None), batch_size=32, num_workers=2,try_nopython=True)\n```\n\n\n```{.python .input}\nstart = time.time()\nfor _ in range(3):\n    print(len(cpp_dl))\n    for _ in cpp_dl:\n        pass\nprint('Elapsed time for backend dataloader:', time.time() - start)\n```\n\n\n## Using the Python backend:\n\n```{.python .input}\ndl = mx.gluon.data.DataLoader(\n    mx.gluon.data.vision.CIFAR10(train=True, transform=None), batch_size=32, num_workers=2,try_nopython=False)\n```\n\n\n```{.python .input}\nstart = time.time()\nfor _ in range(3):\n    print(len(dl))\n    for _ in dl:\n        pass\nprint('Elapsed time for python dataloader:', time.time() - start)\n```\n\n## Next Steps\n\nNow that you have some experience with MXNet's datasets and dataloaders, it's time to use them for [Step 6: Training a Neural Network](./6-train-nn.ipynb).\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/getting-started/crash-course/6-train-nn.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Step 6: Train a Neural Network\n\nNow that you have seen all the necessary components for creating a neural network, you are\nnow ready to put all the pieces together and train a model end to end.\n\n## 1. Data preparation\n\nThe typical process for creating and training a model starts with loading and\npreparing the datasets. For this Network you will use a [dataset of leaf\nimages](https://data.mendeley.com/datasets/hb74ynkjcn/1) that consists of healthy\nand diseased examples of leafs from twelve different plant species. To get this\ndataset you have to download and extract it with the following commands.\n\n```{.python .input}\n# Import all the necessary libraries to train\nimport time\nimport os\nimport zipfile\n\nimport mxnet as mx\nfrom mxnet import np, npx, gluon, init, autograd\nfrom mxnet.gluon import nn\nfrom mxnet.gluon.data.vision import transforms\n\nimport matplotlib.pyplot as plt\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nfrom prepare_dataset import process_dataset #utility code to rearrange the data\n\nmx.np.random.seed(42)\n```\n\n```{.python .input}\n# Download dataset\nurl = 'https://prod-dcd-datasets-cache-zipfiles.s3.eu-west-1.amazonaws.com/hb74ynkjcn-1.zip'\nzip_file_path = mx.gluon.utils.download(url)\n\nos.makedirs('plants', exist_ok=True)\n\nwith zipfile.ZipFile(zip_file_path, 'r') as zf:\n    zf.extractall('plants')\n\nos.remove(zip_file_path)\n```\n\n#### Data inspection\n\nIf you take a look at the dataset you find the following structure for the directories:\n\n```\nplants\n|-- Alstonia Scholaris (P2)\n|-- Arjun (P1)\n|-- Bael (P4)\n    |-- diseased\n        |-- 0016_0001.JPG\n        |-- .\n        |-- .\n        |-- .\n        |-- 0016_0118.JPG\n|-- .\n|-- .\n|-- .\n|-- Mango (P0)\n    |-- diseased\n    |-- healthy\n```\n\nEach plant species has its own directory, for each of those directories you might\nfind subdirectories with examples of diseased leaves, healthy\nleaves, or both. With this dataset you can formulate different classification\nproblems; for example, you can create a multi-class classifier that determines\nthe species of a plant based on the leaves; you can instead create a binary\nclassifier that tells you whether the plant is healthy or diseased. Additionally, you can create\na multi-class, multi-label classifier that tells you both: what species a\nplant is and whether the plant is diseased or healthy. In this example you will stick to\nthe simplest classification question, which is whether a plant is healthy or not.\n\nTo do this, you need to manipulate the dataset in two ways. First, you need to\ncombine all images with labels consisting of healthy and diseased, regardless of the species, and then you\nneed to split the data into train, validation, and test sets. We prepared a\nsmall utility script that does this to get the dataset ready for you.\nOnce you run this utility code on the data, the structure will be\nalready organized in folders containing the right images in each of the classes,\nyou can use the `ImageFolderDataset` class to import the images from the file to MXNet.\n\n```{.python .input}\n# Call the utility function to rearrange the images\nprocess_dataset('plants')\n```\n\nThe dataset is located in the `datasets` folder and the new structure\nlooks like this:\n\n```\ndatasets\n|-- test\n    |-- diseased\n    |-- healthy\n|-- train\n|-- validation\n    |-- diseased\n    |-- healthy\n        |-- image1.JPG\n        |-- image2.JPG\n        |-- .\n        |-- .\n        |-- .\n        |-- imagen.JPG\n```\n\nNow, you need to create three different Dataset objects from the `train`,\n`validation`, and `test` folders, and the `ImageFolderDataset` class takes\ncare of inferring the classes from the directory names. If you don't remember\nhow the `ImageFolderDataset` works, take a look at [Step 5](5-datasets.md)\nof this course for a deeper description.\n\n```{.python .input}\n# Use ImageFolderDataset to create a Dataset object from directory structure\ntrain_dataset = gluon.data.vision.ImageFolderDataset('./datasets/train')\nval_dataset = gluon.data.vision.ImageFolderDataset('./datasets/validation')\ntest_dataset = gluon.data.vision.ImageFolderDataset('./datasets/test')\n```\n\nThe result from this operation is a different Dataset object for each folder.\nThese objects hold a collection of images and labels and as such they can be\nindexed, to get the $i$-th element from the dataset. The $i$-th element is a\ntuple with two objects, the first object of the tuple is the image in array\nform and the second is the corresponding label for that image.\n\n```{.python .input}\nsample_idx = 888 # choose a random sample\nsample = train_dataset[sample_idx]\ndata = sample[0]\nlabel = sample[1]\n\nplt.imshow(data.asnumpy())\nprint(f\"Data type: {data.dtype}\")\nprint(f\"Label: {label}\")\nprint(f\"Label description: {train_dataset.synsets[label]}\")\nprint(f\"Image shape: {data.shape}\")\n```\n\nAs you can see from the plot, the image size is very large 4000 x 6000 pixels.\nUsually, you downsize images before passing them to a neural network to reduce the training time.\nIt is also customary to make slight modifications to the images to improve generalization. That is why you add\ntransformations to the data in a process called Data Augmentation.\n\nYou can augment data in MXNet using `transforms`. For a complete list of all\nthe available transformations in MXNet check out\n[available transforms](../../../api/gluon/data/vision/transforms/index.rst).\nIt is very common to use more than one transform per image, and it is also\ncommon to process transforms sequentially. To this end, you can use the `transforms.Compose` class.\nThis class is very useful to create a transformation pipeline for your images.\n\nYou have to compose two different transformation pipelines, one for training\nand the other one for validating and testing. This is because each pipeline\nserves different pursposes. You need to downsize, convert to tensor and normalize\nimages across all the different datsets; however, you typically do not want to randomly flip\nor add color jitter to the validation or test images since you could reduce performance.\n\n```{.python .input}\n# Import transforms as compose a series of transformations to the images\nfrom mxnet.gluon.data.vision import transforms\n\njitter_param = 0.05\n\n# mean and std for normalizing image value in range (0,1)\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\n\ntraining_transformer = transforms.Compose([\n    transforms.Resize(size=224, keep_ratio=True),\n    transforms.CenterCrop(128),\n    transforms.RandomFlipLeftRight(),\n    transforms.RandomColorJitter(contrast=jitter_param),\n    transforms.ToTensor(),\n    transforms.Normalize(mean, std)\n])\n\nvalidation_transformer = transforms.Compose([\n    transforms.Resize(size=224, keep_ratio=True),\n    transforms.CenterCrop(128),\n    transforms.ToTensor(),\n    transforms.Normalize(mean, std)\n])\n```\n\nWith your augmentations ready, you can create the `DataLoaders` to use them. To\ndo this the `gluon.data.DataLoader` class comes in handy. You have to pass the dataset with\nthe applied transformations (notice the `.transform_first()` method on the datasets)\nto `gluon.data.DataLoader`. Additionally, you need to decide the batch size,\nwhich is how many images you will be passing to the network,\nand whether you want to shuffle the dataset.\n\n```{.python .input}\n# Create data loaders\nbatch_size = 4\ntrain_loader = gluon.data.DataLoader(train_dataset.transform_first(training_transformer),\n                                     batch_size=batch_size,\n                                     shuffle=True,\n                                     try_nopython=True)\nvalidation_loader = gluon.data.DataLoader(val_dataset.transform_first(validation_transformer),\n                                          batch_size=batch_size,\n                                          try_nopython=True)\ntest_loader = gluon.data.DataLoader(test_dataset.transform_first(validation_transformer),\n                                    batch_size=batch_size,\n                                    try_nopython=True)\n```\n\nNow, you can inspect the transformations that you made to the images. A prepared\nutility function has been provided for this.\n\n```{.python .input}\n# Function to plot batch\ndef show_batch(batch, columns=4, fig_size=(9, 5), pad=1):\n    labels = batch[1].asnumpy()\n    batch = batch[0] / 2 + 0.5     # unnormalize\n    batch = np.clip(batch.asnumpy(), 0, 1) # clip values\n    size = batch.shape[0]\n    rows = int(size / columns)\n    fig, axes = plt.subplots(rows, columns, figsize=fig_size)\n    for ax, img, label in zip(axes.flatten(), batch, labels):\n        ax.imshow(np.transpose(img, (1, 2, 0)))\n        ax.set(title=f\"Label: {label}\")\n    fig.tight_layout(h_pad=pad, w_pad=pad)\n    plt.show()\n```\n\n```{.python .input}\nfor batch in train_loader:\n    a = batch\n    break\n```\n\n```{.python .input}\nshow_batch(a)\n```\n\nYou can see that the original images changed to have different sizes and variations\nin color and lighting. These changes followed the specified transformations you stated\nin the pipeline. You are now ready to go to the next step: **Create the\narchitecture**.\n\n## 2. Create Neural Network\n\nConvolutional neural networks are a great tool to capture the spatial\nrelationship of pixel values within images, for this reason they have become the\ngold standard for computer vision. In this example you will create a small convolutional neural\nnetwork using what you learned from [Step 2](2-create-nn.md) of this crash course series.\nFirst, you can set up two functions that will generate the two types of blocks\nyou intend to use, the convolution block and the dense block. Then you can create an\nentire network based on these two blocks using a custom class.\n\n```{.python .input}\n# The convolutional block has a convolution layer, a max pool layer and a batch normalization layer\ndef conv_block(filters, kernel_size=2, stride=2, batch_norm=True):\n    conv_block = nn.HybridSequential()\n    conv_block.add(nn.Conv2D(channels=filters, kernel_size=kernel_size, activation='relu'),\n              nn.MaxPool2D(pool_size=4, strides=stride))\n    if batch_norm:\n        conv_block.add(nn.BatchNorm())\n    return conv_block\n\n# The dense block consists of a dense layer and a dropout layer\ndef dense_block(neurons, activation='relu', dropout=0.2):\n    dense_block = nn.HybridSequential()\n    dense_block.add(nn.Dense(neurons, activation=activation))\n    if dropout:\n        dense_block.add(nn.Dropout(dropout))\n    return dense_block\n```\n\n```{.python .input}\n# Create neural network blueprint using the blocks\nclass LeafNetwork(nn.HybridBlock):\n    def __init__(self):\n        super(LeafNetwork, self).__init__()\n        self.conv1 = conv_block(32)\n        self.conv2 = conv_block(64)\n        self.conv3 = conv_block(128)\n        self.flatten = nn.Flatten()\n        self.dense1 = dense_block(100)\n        self.dense2 = dense_block(10)\n        self.dense3 = nn.Dense(2)\n\n    def forward(self, batch):\n        batch = self.conv1(batch)\n        batch = self.conv2(batch)\n        batch = self.conv3(batch)\n        batch = self.flatten(batch)\n        batch = self.dense1(batch)\n        batch = self.dense2(batch)\n        batch = self.dense3(batch)\n\n        return batch\n```\n\nYou have concluded the architecting part of the network, so now you can actually\nbuild a model from that architecture for training. As you have seen\npreviously on [Step 4](4-components.md) of this\ncrash course series, to use the network you need to initialize the parameters and\nhybridize the model.\n\n```{.python .input}\n# Create the model based on the blueprint provided and initialize the parameters\ndevice = mx.gpu()\n\ninitializer = mx.initializer.Xavier()\n\nmodel = LeafNetwork()\nmodel.initialize(initializer, device=device)\nmodel.summary(mx.np.random.uniform(size=(4, 3, 128, 128), device=device))\nmodel.hybridize()\n```\n\n## 3. Choose Optimizer and Loss function\n\nWith the network created you can move on to choosing an optimizer and a loss\nfunction. The network you created uses these components to make an informed decision on how\nto tune the parameters to fit the final objective better. You can use the `gluon.Trainer` class to\nhelp with optimizing these parameters. The `gluon.Trainer` class needs two things to work\nproperly: the parameters needing to be tuned and the optimizer with its\ncorresponding hyperparameters. The trainer uses the error reported by the loss\nfunction to optimize these parameters.\n\nFor this particular dataset you will use Stochastic Gradient Descent as the\noptimizer and Cross Entropy as the loss function.\n\n```{.python .input}\n# SGD optimizer\noptimizer = 'sgd'\n\n# Set parameters\noptimizer_params = {'learning_rate': 0.001}\n\n# Define the trainer for the model\ntrainer = gluon.Trainer(model.collect_params(), optimizer, optimizer_params)\n\n# Define the loss function\nloss_fn = gluon.loss.SoftmaxCrossEntropyLoss()\n```\n\nFinally, you have to set up the training loop, and you need to create a function to evaluate the performance of the network on the validation dataset.\n\n```{.python .input}\n# Function to return the accuracy for the validation and test set\ndef test(val_data):\n    acc = gluon.metric.Accuracy()\n    for batch in val_data:\n        data = batch[0]\n        labels = batch[1]\n        outputs = model(data.to_device(device))\n        acc.update([labels], [outputs])\n\n    _, accuracy = acc.get()\n    return accuracy\n```\n\n## 4. Training Loop\n\nNow that you have everything set up, you can start training your network. This might\ntake some time to train depending on the hardware, number of layers, batch size and\nimages you use. For this particular case, you will only train for 2 epochs.\n\n```{.python .input}\n# Start the training loop\nepochs = 2\naccuracy = gluon.metric.Accuracy()\nlog_interval = 5\n\nfor epoch in range(epochs):\n    tic = time.time()\n    btic = time.time()\n    accuracy.reset()\n\n    for idx, batch in enumerate(train_loader):\n        data = batch[0]\n        label = batch[1]\n        with mx.autograd.record():\n            outputs = model(data.to_device(device))\n            loss = loss_fn(outputs, label.to_device(device))\n        mx.autograd.backward(loss)\n        trainer.step(batch_size)\n        accuracy.update([label], [outputs])\n        if log_interval and (idx + 1) % log_interval == 0:\n            _, acc = accuracy.get()\n\n            print(f\"\"\"Epoch[{epoch + 1}] Batch[{idx + 1}] Speed: {batch_size / (time.time() - btic)} samples/sec \\\n                  batch loss = {loss.mean().item()} | accuracy = {acc}\"\"\")\n            btic = time.time()\n\n    _, acc = accuracy.get()\n\n    acc_val = test(validation_loader)\n    print(f\"[Epoch {epoch + 1}] training: accuracy={acc}\")\n    print(f\"[Epoch {epoch + 1}] time cost: {time.time() - tic}\")\n    print(f\"[Epoch {epoch + 1}] validation: validation accuracy={acc_val}\")\n```\n\n## 5. Test on the test set\n\nNow that your network is trained and has reached a decent accuracy, you can\nevaluate the performance on the test set. For that, you can use the `test_loader` data\nloader and the test function you created previously.\n\n```{.python .input}\ntest(test_loader)\n```\n\nYou have a trained network that can confidently discriminate between plants that\nare healthy and the ones that are diseased. You can now start your garden and\nset cameras to automatically detect plants in distress! Or change your classification\nproblem to create a model that classify the species of the plants! Either way you\nmight be able to impress your botanist friends.\n\n## 6. Save the parameters\n\nIf you want to preserve the trained weights of the network you can save the\nparameters in a file. Later, when you want to use the network to make predictions\nyou can load the parameters back!\n\n```{.python .input}\n# Save parameters in the\nmodel.save_parameters('leaf_models.params')\n```\n\nThis is the end of this tutorial, to see how you can speed up the training by\nusing GPU hardware continue to the [next tutorial](./7-use-gpus.ipynb)\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/getting-started/crash-course/7-use-gpus.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Step 7: Load and Run a NN using GPU\n\nIn this step, you will learn how to use graphics processing units (GPUs) with MXNet. If you use GPUs to train and deploy neural networks, you may be able to train or perform inference quicker than with central processing units (CPUs).\n\n## Prerequisites\n\nBefore you start the steps, make sure you have at least one Nvidia GPU on your machine and make sure that you have CUDA properly installed. GPUs from AMD and Intel are not supported. Additionally, you will need to install the GPU-enabled version of MXNet. You can find information about how to install the GPU version of MXNet for your system [here](https://mxnet.apache.org/versions/1.4.1/install/ubuntu_setup.html).\n\nYou can use the following command to view the number GPUs that are available to MXNet.\n\n```{.python .input}\nfrom mxnet import np, npx, gluon, autograd\nfrom mxnet.gluon import nn\nimport time\nnpx.set_np()\n\nnpx.num_gpus() #This command provides the number of GPUs MXNet can access\n```\n\n## Allocate data to a GPU\n\nMXNet's ndarray is very similar to NumPy's. One major difference is that MXNet's ndarray has a `device` attribute specifying which device an array is on. By default, arrays are stored on `npx.cpu()`. To change it to the first GPU, you can use the following code, `npx.gpu()` or `npx.gpu(0)` to indicate the first GPU.\n\n```{.python .input}\ngpu = npx.gpu() if npx.num_gpus() > 0 else npx.cpu()\nx = np.ones((3,4), device=gpu)\nx\n```\n\nIf you're using a CPU, MXNet allocates data on the main memory and tries to use as many CPU cores as possible.  If there are multiple GPUs, MXNet will tell you which GPUs the ndarray is allocated on.\n\nAssuming there is at least two GPUs. You can create another ndarray and assign it to a different GPU. If you only have one GPU, then you will get an error trying to run this code. In the example code here, you will copy `x` to the second GPU, `npx.gpu(1)`:\n\n```{.python .input}\ngpu_1 = npx.gpu(1) if npx.num_gpus() > 1 else npx.cpu()\nx.copyto(gpu_1)\n```\n\nMXNet requries that users explicitly move data between devices. But several operators such as `print`, and `asnumpy`, will implicitly move data to main memory.\n\n## Choosing GPU Ids\nIf you have multiple GPUs on your machine, MXNet can access each of them through 0-indexing with `npx`. As you saw before, the first GPU was accessed using `npx.gpu(0)`, and the second using `npx.gpu(1)`. This extends to however many GPUs your machine has. So if your machine has eight GPUs, the last GPU is accessed using `npx.gpu(7)`. This allows you to select which GPUs to use for operations and training. You might find it particularly useful when you want to leverage multiple GPUs while training neural networks.\n\n## Run an operation on a GPU\n\nTo perform an operation on a particular GPU, you only need to guarantee that the input of an operation is already on that GPU. The output is allocated on the same GPU as well. Almost all operators in the `np` and `npx` module support running on a GPU.\n\n```{.python .input}\ny = np.random.uniform(size=(3,4), device=gpu)\nx + y\n```\n\nRemember that if the inputs are not on the same GPU, you will get an error.\n\n## Run a neural network on a GPU\n\nTo run a neural network on a GPU, you only need to copy and move the input data and parameters to the GPU. To demonstrate this you can reuse the previously defined LeafNetwork in [Training Neural Networks](6-train-nn.md). The following code example shows this.\n\n```{.python .input}\n# The convolutional block has a convolution layer, a max pool layer and a batch normalization layer\ndef conv_block(filters, kernel_size=2, stride=2, batch_norm=True):\n    conv_block = nn.HybridSequential()\n    conv_block.add(nn.Conv2D(channels=filters, kernel_size=kernel_size, activation='relu'),\n              nn.MaxPool2D(pool_size=4, strides=stride))\n    if batch_norm:\n        conv_block.add(nn.BatchNorm())\n    return conv_block\n\n# The dense block consists of a dense layer and a dropout layer\ndef dense_block(neurons, activation='relu', dropout=0.2):\n    dense_block = nn.HybridSequential()\n    dense_block.add(nn.Dense(neurons, activation=activation))\n    if dropout:\n        dense_block.add(nn.Dropout(dropout))\n    return dense_block\n\n# Create neural network blueprint using the blocks\nclass LeafNetwork(nn.HybridBlock):\n    def __init__(self):\n        super(LeafNetwork, self).__init__()\n        self.conv1 = conv_block(32)\n        self.conv2 = conv_block(64)\n        self.conv3 = conv_block(128)\n        self.flatten = nn.Flatten()\n        self.dense1 = dense_block(100)\n        self.dense2 = dense_block(10)\n        self.dense3 = nn.Dense(2)\n\n    def forward(self, batch):\n        batch = self.conv1(batch)\n        batch = self.conv2(batch)\n        batch = self.conv3(batch)\n        batch = self.flatten(batch)\n        batch = self.dense1(batch)\n        batch = self.dense2(batch)\n        batch = self.dense3(batch)\n\n        return batch\n```\n\nLoad the saved parameters onto GPU 0 directly as shown below; additionally, you could use `net.collect_params().reset_device(gpu)` to change the device.\n\n```{.python .input}\nnet = LeafNetwork()\nnet.load_parameters('leaf_models.params', device=gpu)\n```\n\nUse the following command to create input data on GPU 0. The forward function will then run on GPU 0.\n\n```{.python .input}\nx = np.random.uniform(size=(1, 3, 128, 128), device=gpu)\nnet(x)\n```\n\n## Training with multiple GPUs\n\nFinally, you will see how you can use multiple GPUs to jointly train a neural network through data parallelism. To elaborate on what data parallelism is, assume there are *n* GPUs, then you can split each data batch into *n* parts, and use a GPU on each of these parts to run the forward and backward passes on the seperate chunks of the data.\n\nFirst copy the data definitions with the following commands, and the transform functions from the tutorial [Training Neural Networks](6-train-nn.md).\n\n```{.python .input}\n# Import transforms as compose a series of transformations to the images\nfrom mxnet.gluon.data.vision import transforms\n\njitter_param = 0.05\n\n# mean and std for normalizing image value in range (0,1)\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\n\ntraining_transformer = transforms.Compose([\n    transforms.Resize(size=224, keep_ratio=True),\n    transforms.CenterCrop(128),\n    transforms.RandomFlipLeftRight(),\n    transforms.RandomColorJitter(contrast=jitter_param),\n    transforms.ToTensor(),\n    transforms.Normalize(mean, std)\n])\n\nvalidation_transformer = transforms.Compose([\n    transforms.Resize(size=224, keep_ratio=True),\n    transforms.CenterCrop(128),\n    transforms.ToTensor(),\n    transforms.Normalize(mean, std)\n])\n\n# Use ImageFolderDataset to create a Dataset object from directory structure\ntrain_dataset = gluon.data.vision.ImageFolderDataset('./datasets/train')\nval_dataset = gluon.data.vision.ImageFolderDataset('./datasets/validation')\ntest_dataset = gluon.data.vision.ImageFolderDataset('./datasets/test')\n\n# Create data loaders\nbatch_size = 4\ntrain_loader = gluon.data.DataLoader(train_dataset.transform_first(training_transformer),batch_size=batch_size, shuffle=True, try_nopython=True)\nvalidation_loader = gluon.data.DataLoader(val_dataset.transform_first(validation_transformer), batch_size=batch_size, try_nopython=True)\ntest_loader = gluon.data.DataLoader(test_dataset.transform_first(validation_transformer), batch_size=batch_size, try_nopython=True)\n```\n\n### Define a helper function\nThis is the same test function defined previously in the **Step 6**.\n\n```{.python .input}\n# Function to return the accuracy for the validation and test set\ndef test(val_data, devices):\n    acc = gluon.metric.Accuracy()\n    for batch in val_data:\n        data, label = batch[0], batch[1]\n        data_list = gluon.utils.split_and_load(data, devices)\n        label_list = gluon.utils.split_and_load(label, devices)\n        outputs = [net(X) for X in data_list]\n        acc.update(label_list, outputs)\n\n    _, accuracy = acc.get()\n    return accuracy\n```\n\nThe training loop is quite similar to that shown earlier. The major differences are highlighted in the following code.\n\n```{.python .input}\n# Diff 1: Use two GPUs for training.\navailable_gpus = [npx.gpu(i) for i in range(npx.num_gpus())]\nnum_gpus = 2\ndevices = available_gpus[:num_gpus]\nprint('Using {} GPUs'.format(len(devices)))\n\n# Diff 2: reinitialize the parameters and place them on multiple GPUs\nnet.initialize(force_reinit=True, device=devices)\n\n# Loss and trainer are the same as before\nloss_fn = gluon.loss.SoftmaxCrossEntropyLoss()\noptimizer = 'sgd'\noptimizer_params = {'learning_rate': 0.001}\ntrainer = gluon.Trainer(net.collect_params(), optimizer, optimizer_params)\n\nepochs = 2\naccuracy = gluon.metric.Accuracy()\nlog_interval = 5\n\nfor epoch in range(epochs):\n    train_loss = 0.\n    tic = time.time()\n    btic = time.time()\n    accuracy.reset()\n    for idx, batch in enumerate(train_loader):\n        data, label = batch[0], batch[1]\n\n        # Diff 3: split batch and load into corresponding devices\n        data_list = gluon.utils.split_and_load(data, devices)\n        label_list = gluon.utils.split_and_load(label, devices)\n\n        # Diff 4: run forward and backward on each devices.\n        # MXNet will automatically run them in parallel\n        with autograd.record():\n            outputs = [net(X)\n                      for X in data_list]\n            losses = [loss_fn(output, label)\n                      for output, label in zip(outputs, label_list)]\n        for l in losses:\n            l.backward()\n        trainer.step(batch_size)\n\n        # Diff 5: sum losses over all devices. Here, the float\n        # function will copy data into CPU.\n        train_loss += sum([float(l.sum()) for l in losses])\n        accuracy.update(label_list, outputs)\n        if log_interval and (idx + 1) % log_interval == 0:\n            _, acc = accuracy.get()\n\n            print(f\"\"\"Epoch[{epoch + 1}] Batch[{idx + 1}] Speed: {batch_size / (time.time() - btic)} samples/sec \\\n                  batch loss = {train_loss} | accuracy = {acc}\"\"\")\n            btic = time.time()\n\n    _, acc = accuracy.get()\n\n    acc_val = test(validation_loader, devices)\n    print(f\"[Epoch {epoch + 1}] training: accuracy={acc}\")\n    print(f\"[Epoch {epoch + 1}] time cost: {time.time() - tic}\")\n    print(f\"[Epoch {epoch + 1}] validation: validation accuracy={acc_val}\")\n```\n\n## Next steps\n\nNow that you have completed training and predicting with a neural network on GPUs, you reached the conclusion of the crash course. Congratulations.\nIf you are keen on studying more, checkout [D2L.ai](https://d2l.ai),\n[GluonCV](https://cv.gluon.ai/tutorials/index.html), [GluonNLP](https://nlp.gluon.ai),\n[GluonTS](https://ts.gluon.ai/), [AutoGluon](https://auto.gluon.ai).\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/getting-started/crash-course/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nCrash Course\n=============\n\nThis crash course will give you a quick overview of MXNet. You will review core concepts like NDArray (manipulating multiple dimensional arrays) and Gluon (create and train neural networks on CPU and GPU). The intended audience for this crash course is people already familiar with deep learning theory or other deep learning frameworks. For a deep dive into MXNet and deep learning architectures, please refer to [Dive Into Deep learning](http://d2l.ai/) textbook or [Introduction to Deep Learning Course](https://courses.d2l.ai/berkeley-stat-157/index.html)\n\nThe course is structured in different sections that can be studied independently or as a whole. If you have a particular question you can consult only the section related to your question, but if you are new to the framework and have time, you can do the course from start to end.\n\n\n.. toctree::\n   :maxdepth: 1\n   :caption: Contents\n\n   0-introduction\n   1-nparray\n   2-create-nn\n   3-autograd\n   4-components\n   5-datasets\n   6-train-nn\n   7-use-gpus"
  },
  {
    "path": "docs/python_docs/python/tutorials/getting-started/crash-course/prepare_dataset.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\n\nimport shutil, random, glob, os, logging\nfrom pathlib import Path\n\nlogging.basicConfig(level=logging.INFO)\nlogger = logging.getLogger()\n\nsplits = ('train', 'validation', 'test')\ntargets = ('healthy', 'diseased')\n\ndef split_file_list(file_list, train_split=0.7, val_split=0.2, test_split=0.2):\n    random.shuffle(file_list)\n    files = len(file_list)\n    train_items = round(files * train_split)\n    validation_items = round(files * val_split)\n    train = file_list[:train_items]\n    validation = file_list[train_items: train_items + validation_items]\n    test = file_list[train_items + validation_items:]\n\n    return train, validation, test\n\ndef process_dataset(root_directory, splits=splits, classes=targets, train=0.7, val=0.2, test=0.2):\n\n    # Get healthy and diseased file lists\n    for target in targets:\n        file_list = glob.glob(f\"{root_directory}/**/{target}/*.JPG\")\n        dataset_splits = split_file_list(file_list, train, val, test)\n        logger.info(f\"Starting transferring files from the {target} class\")\n        for idx, split in enumerate(dataset_splits):\n            new_path = os.path.join(\"datasets\", splits[idx], target)\n            logger.info(f\"Moving {splits[idx]} files\")\n            Path(new_path).mkdir(parents=True, exist_ok=True)\n            for file_path in split:\n                shutil.move(file_path, new_path)\n            logger.info(f\"Finished moving {splits[idx]} files\")\n    logger.info(f\"Finished moving files\")\n    logger.info(\"Removing old folders\")\n    shutil.rmtree(root_directory)\n    logger.info(\"Finished!\")\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/getting-started/gluon_from_experiment_to_deployment.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Gluon: from experiment to deployment\n\n## Overview\nMXNet Gluon API comes with a lot of great features, and it can provide you everything you need: from experimentation to deploying the model. In this tutorial, we will walk you through a common use case on how to build a model using gluon, train it on your data, and deploy it for inference.\n\nLet's say you need to build a service that provides flower species recognition. A common problem is that you don't have enough data to train a good model. In such cases, a technique called Transfer Learning can be used to make a more robust model.\nIn Transfer Learning we make use of a pre-trained model that solves a related task, and was trained on a very large standard dataset, such as ImageNet. ImageNet is from a different domain, but we can utilize the knowledge in this pre-trained model to perform the new task at hand.\n\nGluon provides State of the Art models for many of the standard tasks such as Classification, Object Detection, Segmentation, etc. In this tutorial we will use the pre-trained model [ResNet50 V2](https://arxiv.org/abs/1603.05027) trained on ImageNet dataset. This model achieves 77.11% top-1 accuracy on ImageNet. We seek to transfer as much knowledge as possible for our task of recognizing different species of flowers.\n\n\n\n\n## Prerequisites\n\nTo complete this tutorial, you need:\n\n- [Build MXNet from source](https://mxnet.apache.org/get_started/build_from_source) with Python(Gluon) and C++ Packages\n- Learn the basics about Gluon with [A 60-minute Gluon Crash Course](https://gluon-crash-course.mxnet.io/)\n\n\n## The Data\n\nWe will use the [Oxford 102 Category Flower Dataset](http://www.robots.ox.ac.uk/~vgg/data/flowers/102/) as an example to show you the steps.\nWe have prepared a utility file to help you download and organize your data into train, test, and validation sets. Run the following Python code to download and prepare the data:\n\n\n```{.python .input}\nimport mxnet as mx\ndata_util_file = \"oxford_102_flower_dataset.py\"\nbase_url = \"https://raw.githubusercontent.com/apache/mxnet/master/docs/tutorial_utils/data/{}?raw=true\"\nmx.test_utils.download(base_url.format(data_util_file), fname=data_util_file)\nimport oxford_102_flower_dataset\n\n# download and move data to train, test, valid folders\npath = './data'\noxford_102_flower_dataset.get_data(path)\n```\n\nNow your data will be organized into train, test, and validation sets, images belong to the same class are moved to the same folder.\n\n## Training using Gluon\n\n### Define Hyper-parameters\n\nNow let's first import necessary packages:\n\n\n```{.python .input}\nimport math\nimport os\nimport time\n\nfrom mxnet import autograd\nfrom mxnet import gluon, init\nfrom mxnet.gluon import nn\nfrom mxnet.gluon.data.vision import transforms\nfrom mxnet.gluon.model_zoo.vision import resnet50_v2\n```\n\nNext, we define the hyper-parameters that we will use for fine-tuning. We will use the [MXNet learning rate scheduler](../packages/gluon/training/learning_rates/learning_rate_schedules.ipynb) to adjust learning rates during training.\nHere we set the `epochs` to 1 for quick demonstration, please change to 40 for actual training.\n\n```{.python .input}\nclasses = 102\nepochs = 1\nlr = 0.001\nper_device_batch_size = 32\nmomentum = 0.9\nwd = 0.0001\n\nlr_factor = 0.75\n# learning rate change at following epochs\nlr_epochs = [10, 20, 30]\n\nnum_gpus = mx.device.num_gpus()\n# you can replace num_workers with the number of cores on you device\nnum_workers = 8\ndevice = [mx.gpu(i) for i in range(num_gpus)] if num_gpus > 0 else [mx.cpu()]\nbatch_size = per_device_batch_size * max(num_gpus, 1)\n```\n\nNow we will apply data augmentations on training images. This makes minor alterations on the training images, and our model will consider them as distinct images. This can be very useful for fine-tuning on a relatively small dataset, and it will help improve the model. We can use the Gluon [DataSet API](../../api/gluon/data/index.rst#mxnet.gluon.data.Dataset), [DataLoader API](../../api/gluon/data/index.rst#mxnet.gluon.data.DataLoader), and [Transform API](../../api/gluon/data/index.rst#mxnet.gluon.data.Dataset.transform) to load the images and apply the following data augmentations:\n1. Randomly crop the image and resize it to 224x224\n2. Randomly flip the image horizontally\n3. Randomly jitter color and add noise\n4. Transpose the data from `[height, width, num_channels]` to `[num_channels, height, width]`, and map values from [0, 255] to [0, 1]\n5. Normalize with the mean and standard deviation from the ImageNet dataset.\n\nFor validation and inference, we only need to apply step 1, 4, and 5. We also need to save the mean and standard deviation values for inference using other language bindings.\n\n```{.python .input}\njitter_param = 0.4\nlighting_param = 0.1\n\n# mean and std for normalizing image value in range (0,1)\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\n\ntraining_transformer = transforms.Compose([\n    transforms.RandomResizedCrop(224),\n    transforms.RandomFlipLeftRight(),\n    transforms.RandomColorJitter(brightness=jitter_param, contrast=jitter_param,\n                                 saturation=jitter_param),\n    transforms.RandomLighting(lighting_param),\n    transforms.ToTensor(),\n    transforms.Normalize(mean, std)\n])\n\nvalidation_transformer = transforms.Compose([\n    transforms.Resize(256),\n    transforms.CenterCrop(224),\n    transforms.ToTensor(),\n    transforms.Normalize(mean, std)\n])\n\n# save mean and std NDArray values for inference\nmean_img = mx.np.stack([mx.np.full((224, 224), m) for m in mean])\nstd_img = mx.np.stack([mx.np.full((224, 224), s) for s in std])\nmx.npx.savez('mean_std_224.np', **{\"mean_img\": mean_img, \"std_img\": std_img})\n\ntrain_path = os.path.join(path, 'train')\nval_path = os.path.join(path, 'valid')\ntest_path = os.path.join(path, 'test')\n\n# loading the data and apply pre-processing(transforms) on images\ntrain_data = gluon.data.DataLoader(\n    gluon.data.vision.ImageFolderDataset(train_path).transform_first(training_transformer),\n    batch_size=batch_size, shuffle=True, num_workers=num_workers)\n\nval_data = gluon.data.DataLoader(\n    gluon.data.vision.ImageFolderDataset(val_path).transform_first(validation_transformer),\n    batch_size=batch_size, shuffle=False, num_workers=num_workers)\n\ntest_data = gluon.data.DataLoader(\n    gluon.data.vision.ImageFolderDataset(test_path).transform_first(validation_transformer),\n    batch_size=batch_size, shuffle=False, num_workers=num_workers)\n```\n\n### Loading pre-trained model\n\n\nWe will use pre-trained ResNet50_v2 model which was pre-trained on the [ImageNet Dataset](http://www.image-net.org/) with 1000 classes. To match the classes in the Flower dataset, we must redefine the last softmax (output) layer to be 102, then initialize the parameters.\n\nBefore we go to training, one unique Gluon feature you should be aware of is hybridization. It allows you to convert your imperative code to a static symbolic graph, which is much more efficient to execute. There are two main benefits of hybridizing your model: better performance and easier serialization for deployment. The best part is that it's as simple as just calling `net.hybridize()`. To know more about Gluon hybridization, please follow the [hybridization tutorial](../packages/gluon/blocks/hybridize.rst).\n\n\n\n```{.python .input}\n# load pre-trained resnet50_v2 from model zoo\nfinetune_net = resnet50_v2(pretrained=True, device=device)\n\n# change last softmax layer since number of classes are different\nfinetune_net.output = nn.Dense(classes)\nfinetune_net.output.initialize(init.Xavier(), device=device)\n# hybridize for better performance\nfinetune_net.hybridize()\n\nnum_batch = len(train_data)\n\n# setup learning rate scheduler\niterations_per_epoch = math.ceil(num_batch)\n# learning rate change at following steps\nlr_steps = [epoch * iterations_per_epoch for epoch in lr_epochs]\nschedule = mx.lr_scheduler.MultiFactorScheduler(step=lr_steps, factor=lr_factor, base_lr=lr)\n\n# setup optimizer with learning rate scheduler, metric, and loss function\nsgd_optimizer = mx.optimizer.SGD(learning_rate=lr, lr_scheduler=schedule, momentum=momentum, wd=wd)\nmetric = mx.gluon.metric.Accuracy()\nsoftmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()\n```\n\n### Fine-tuning model on your custom dataset\n\nNow let's define the test metrics and start fine-tuning.\n\n\n\n```{.python .input}\ndef test(net, val_data, device):\n    metric = mx.gluon.metric.Accuracy()\n    for i, (data, label) in enumerate(val_data):\n        data = gluon.utils.split_and_load(data, device, even_split=False)\n        label = gluon.utils.split_and_load(label, device, even_split=False)\n        outputs = [net(x) for x in data]\n        metric.update(label, outputs)\n    return metric.get()\n\ntrainer = gluon.Trainer(finetune_net.collect_params(), optimizer=sgd_optimizer)\n\n# start with epoch 1 for easier learning rate calculation\nfor epoch in range(1, epochs + 1):\n\n    tic = time.time()\n    train_loss = 0\n    metric.reset()\n\n    for i, (data, label) in enumerate(train_data):\n        # get the images and labels\n        data = gluon.utils.split_and_load(data, device, even_split=False)\n        label = gluon.utils.split_and_load(label, device, even_split=False)\n        with autograd.record():\n            outputs = [finetune_net(x) for x in data]\n            loss = [softmax_cross_entropy(yhat, y) for yhat, y in zip(outputs, label)]\n        for l in loss:\n            l.backward()\n\n        trainer.step(batch_size)\n        train_loss += sum([l.mean().item() for l in loss]) / len(loss)\n        metric.update(label, outputs)\n\n    _, train_acc = metric.get()\n    train_loss /= num_batch\n    _, val_acc = test(finetune_net, val_data, device)\n\n    print('[Epoch %d] Train-acc: %.3f, loss: %.3f | Val-acc: %.3f | learning-rate: %.3E | time: %.1f' %\n          (epoch, train_acc, train_loss, val_acc, trainer.learning_rate, time.time() - tic))\n\n_, test_acc = test(finetune_net, test_data, device)\nprint('[Finished] Test-acc: %.3f' % (test_acc))\n```\n\nFollowing is the training result:\n```text\n[Epoch 40] Train-acc: 0.945, loss: 0.354 | Val-acc: 0.955 | learning-rate: 4.219E-04 | time: 17.8\n[Finished] Test-acc: 0.952\n```\nIn the previous example output, we trained the model using an [AWS p3.8xlarge instance](https://aws.amazon.com/ec2/instance-types/p3/) with 4 Tesla V100 GPUs. We were able to reach a test accuracy of 95.5% with 40 epochs in around 12 minutes. This was really fast because our model was pre-trained on a much larger dataset, ImageNet, with around 1.3 million images. It worked really well to capture features on our small dataset.\n\n\n### Save the fine-tuned model\n\n\nWe now have a trained our custom model. This can be serialized into model files using the export function. The export function will export the model architecture into a `.json` file and model parameters into a `.params` file.\n\n\n\n```{.python .input}\nfinetune_net.export(\"flower-recognition\", epoch=epochs)\n\n```\n\n`export` creates `flower-recognition-symbol.json` and `flower-recognition-0040.params` (`0040` is for 40 epochs we ran) in the current directory. These files can be used for model deployment using the `HybridBlock.import` API.\n\n## What's next\n\nYou can find more ways to run inference and deploy your models here:\n1. [MXNet Model Server Examples](https://github.com/awslabs/mxnet-model-server/tree/master/examples)\n\n## References\n\n1. [Transfer Learning for Oxford102 Flower Dataset](https://github.com/Arsey/keras-transfer-learning-for-oxford102)\n2. [Gluon book on fine-tuning](https://www.d2l.ai/chapter_computer-vision/fine-tuning.html)\n3. [Gluon CV transfer learning tutorial](https://cv.gluon.ai/build/examples_classification/transfer_learning_minc.html)\n4. [Gluon crash course](https://gluon-crash-course.mxnet.io/)\n5. [Gluon CPP inference example](https://github.com/apache/mxnet/blob/master/cpp-package/example/inference/)\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/getting-started/gluon_migration_guide.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Gluon2.0: Migration Guide\n\n## Overview\nSince the introduction of the Gluon API in MXNet 1.x, it has superseded commonly used symbolic, module and model APIs for model development. In fact, Gluon was the first in the deep learning community to unify the flexibility of imperative programming with the performance benefits of symbolic programming, through just-in-time compilation.\n\nIn Gluon2.0, we extend the support to MXNet NumPy and NumPy extension with simplified interface and new functionalities:\n\n- **Simplified hybridization with deferred compute and tracing**: Deferred compute allows the imperative execution to be used for graph construction, which allows us to unify the historic divergence of NDArray and Symbol. Hybridization now works in a simplified hybrid forward interface. Users only need to specify the computation through imperative programming. Hybridization also works through tracing, i.e. tracing the data flow of the first input data to create a graph.\n\n- **Data 2.0**: The new design for data loading in Gluon allows hybridizing and deploying data processing pipeline in the same way as model hybridization. The new C++ data loader improves data loading efficiency on CIFAR 10 by 50%.\n\n- **Distributed 2.0**: The new distributed-training design in Gluon 2.0 provides a unified distributed data parallel interface across native Parameter Server, BytePS, and Horovod, and is extensible for supporting custom distributed training libraries.\n\n- **Gluon Probability**: parameterizable probability distributions and sampling functions to facilitate more areas of research such as Baysian methods and AutoML.\n\n- **Gluon Metrics** and **Optimizers**: refactored with MXNet NumPy interface and addressed legacy issues.\n\nAdopting these new functionalities may or may not require modifications on your models. But don't worry, this migration guide will go through a high-level mapping from old functionality to new APIs and make Gluon2.0 migration a hassle-free experience.\n\n## Data Pipeline\n**What's new**: In Gluon2.0, `MultithreadingDataLoader` is introduced to speed up the data loading pipeline. It will use the pure MXNet C++ implementation of dataloader, datasets and batchify functions. So, you can use either MXNet internal multithreading mode dataloader or python multiprocessing mode dataloader in Gluon2.0.\n\n**Migration Guide**: Users can continue with the traditional gluon.data.Dataloader and the C++ backend will be applied automatically.\n\n[Gluon2.0 dataloader](../../api/gluon/data/index.rst#mxnet.gluon.data.DataLoader) will provide a new parameter called `try_nopython`. This parameter takes a default value of None; when set to `True` the dataloader will compile the python dataloading pipeline into pure MXNet C++ implementation. The compilation is not guaranteed to support all use cases, but it will fallback to python in case of failure:\n\n- The dataset is not fully [supported by the backend](../../api/gluon/data/index.rst#mxnet.gluon.data.Dataset) (e.g., there are custom python datasets).\n\n- Transform is not fully hybridizable.\n\n- Bachify is not fully [supported by the backend](https://github.com/apache/incubator-mxnet/blob/master/python/mxnet/gluon/data/batchify.py).\n\n\nYou can refer to [Step 5 in Crash Course](https://mxnet.apache.org/versions/master/api/python/docs/tutorials/getting-started/crash-course/5-datasets.html#New-in-MXNet-2.0:-faster-C++-backend-dataloaders) for a detailed performance increase with C++ backend.\n\n## Modeling\nIn Gluon2.0, users will have a brand new modeling experience with NumPy-compatible APIs and the deferred compute mechanism.\n\n- **NumPy-compatible programing experience**: users can build their models with MXNet implementation with NumPy array library, NumPy-compatible math operators and some neural network extension operators.\n\n- **Imperative-only coding experience**: with the deferred compute and tracing being introduced, users only need to specify the computation through imperative coding but can still make hybridization work. Users will no longer need to interact with symbol APIs.\n\nTo help users migrate smoothly to use these simplified interfaces, we will provide the following guidance on how to replace legacy operators with NumPy-compatible operators, how to build models with `forward` instead of `hybrid_forward` and how to use `Parameter` class to register your parameters.\n\n\n### NumPy-compatible Programming Experience\n#### NumPy Arrays\nMXNet [NumPy ndarray (i.e. mx.np.ndarray)](../../api/np/arrays.ndarray.rst) is a multidimensional container of items of the same type and size. Most of its properties and attributes are the same as legacy NDArrays (i.e. `mx.nd.ndarray`), so users can use the NumPy array library just as they did with legacy NDArrays. But, there are still some changes and deprecations that need attention, as mentioned below.\n\n**Migration Guide**:\n\n1. Currently, NumPy ndarray only supports `default` storage type, other storage types, like `row_sparse`, `csr` are not supported. Also, `tostype()` attribute is deprecated.\n\n2. Users can use `as_np_ndarray` attribute to switch from a legacy NDArray to NumPy ndarray just like this:\n\n```{.python}\nimport mxnet as mx\nnd_array = mx.ones((5,3))\nnp_array = nd_array.as_np_ndarray()\n```\n\n3. Compared with legacy NDArray, some attributes are deprecated in NumPy ndarray. Listed below are some of the deprecated APIs and their corresponding replacements in NumPy ndarray, others can be found in [**Appendix/NumPy Array Deprecated Attributes**](#NumPy-Array-Deprecated-Attributes).\n\n|                   Deprecated Attributes               |    NumPy ndarray Equivalent    |\n| ----------------------------------------------------- | ------------------------------ |\n|                   `a.asscalar()`                      |         `a.item()`             |\n|                 `a.as_in_context()`                   |      `a.to_device()`           |\n|                    `a.context`                        |          `a.device`            |\n|                   `a.reshape_like(b)`                 |    `a.reshape(b.shape)`        |\n|                    `a.zeros_like(b)`                  |   `mx.np.zeros_like(b)`        |\n|                    `a.ones_like(b)`                   |   `mx.np.ones_like(b)`         |\n\n\n**NOTE**\n\n`Context` class has also been deprecated in MXNet2.0, it is renamed to `Device` and some related methods and attributes are also renamed as above. All the creation functions inside MXNet NumPy package will take `device` as keyword instead of `ctx`.\n\n\n4. Compared with legacy NDArray, some attributes will have different behaviors and take different inputs. \n\n+--------------------------------------------------+--------------------------------------------------------------+------------------------------------------------------------------+\n|                       Attribute                  |                       Legacy Inputs                          |                    NumPy Inputs                                  |\n+==================================================+==============================================================+==================================================================+\n|            a.reshape(*args, **kwargs)            | **shape**: Some dimensions of the shape can take special     | **shape**: shape parameter will be **positional argument** rather|\n|                                                  | values from the set {0, -1, -2, -3, -4}.                     |            than key-word argument. Some dimensions of the shape  |\n|                                                  | The significance of each is explained below:                 |            can take special values from the set {-1, -2, -3, -4, |\n|                                                  | 0  copy this dimension from the input to the output shape.   |            -5, -6}.                                              |\n|                                                  | -1 infers the dimension of the output shape by using the     | The significance of each is explained below:                     |\n|                                                  |    remainder of the input dimensions.                        | -1 infers the dimension of the output shape by using the         |\n|                                                  | -2 copy all/remainder of the input dimensions to the         |    remainder  of the input dimensions.                           |\n|                                                  |    output shape.                                             | -2 copy this dimension from the input to the output shape.       |\n|                                                  | -3 use the product of two consecutive dimensions of the      | -3 skip the current dimension if and only if the current dim size|\n|                                                  |    input shape as the output dimension.                      |    is one.                                                       |\n|                                                  | -4 split one dimension of the input into two dimensions      | -4 copy all the remaining the input dimensions to the output     |\n|                                                  |    passed subsequent to -4 in shape (can contain -1).        |    shape.                                                        |\n|                                                  | **reverse**: If set to 1, then the special values are        | -5 use the product of two consecutive dimensions of the input    |\n|                                                  |              inferred from right to left                     |    shape as the output.                                          |\n|                                                  |                                                              | -6 split one dimension of the input into two dimensions passed   |\n|                                                  |                                                              |    subsequent to -6 in the new shape.                            |\n|                                                  |                                                              | **reverse**: No **reverse** parameter for `np.reshape` but for   |\n|                                                  |                                                              |              `npx.reshape`.                                      |\n|                                                  |                                                              | **order**: Read the elements of `a` using this index order, and  |\n|                                                  |                                                              |            place the elements into the reshaped array using this |\n|                                                  |                                                              |            index order.                                          |\n+--------------------------------------------------+--------------------------------------------------------------+------------------------------------------------------------------+\n\n\n\n#### NumPy and NumPy-extension Operators\nMost of the legacy NDArray operators (`mx.nd.op`) have the equivalent ones in np/npx namespace. Users can just replace them with `mx.np.op` or `mx.npx.op` to migrate. Some of the operators will have different inputs and behaviors as listed in the table below.\n\n**Migration Guide**:\n\n1. Operators migration with name/inputs changes\n\n+--------------------------------------------------+---------------------------------------------------------+-----------------------------------------------------------------------+\n|                   Legacy Operators               |               NumPy Operators Equivalent                |                              Changes                                  |\n+==================================================+=========================================================+=======================================================================+\n|        mx.nd.flatten(*args, **kwargs)            |        mx.npx.batch_flatten(*args, **kwargs)            |     moved to npx namespace with new name batch_flatten                |\n+--------------------------------------------------+---------------------------------------------------------+-----------------------------------------------------------------------+\n|       mx.nd.concat(a, b, c)                      |            mx.np.concatenate([a, b, c])                 |       - moved to np namespace with new name concatenate.              |\n|                                                  |                                                         |       - use list of ndarrays as input rather than positional ndarrays |\n+--------------------------------------------------+---------------------------------------------------------+-----------------------------------------------------------------------+\n|        mx.nd.stack(a, b, c)                      |            mx.np.stack([a, b, c])                       |       - moved to np namespace.                                        |\n|                                                  |                                                         |       - use list of ndarrays as input rather than positional ndarrays |\n+--------------------------------------------------+---------------------------------------------------------+-----------------------------------------------------------------------+\n|      mx.nd.SliceChannel(*args, **kwargs)         |            mx.npx.slice_channel(*args, **kwargs)        |         moved to npx namespace with new name slice_channel.           |\n+--------------------------------------------------+---------------------------------------------------------+-----------------------------------------------------------------------+\n|      mx.nd.FullyConnected(*args, **kwargs)       |        mx.npx.fully_connected(*args, **kwargs)          |         moved to npx namespace with new name fully_connected.         |\n+--------------------------------------------------+---------------------------------------------------------+-----------------------------------------------------------------------+\n|      mx.nd.Activation(*args, **kwargs)           |            mx.npx.activation(*args, **kwargs)           |         moved to npx namespace with new name activation.              |\n+--------------------------------------------------+---------------------------------------------------------+-----------------------------------------------------------------------+\n|      mx.nd.elemwise_add(a, b)                    |            a + b                                        |         Just use ndarray python operator.                             |\n+--------------------------------------------------+---------------------------------------------------------+-----------------------------------------------------------------------+\n|      mx.nd.elemwise_mul(a, b)                    |            mx.np.multiply(a, b)                         |              Use multiply operator in np namespace.                   |\n+--------------------------------------------------+---------------------------------------------------------+-----------------------------------------------------------------------+\n\n2. Operators migration with multiple steps: `mx.nd.mean` -> `mx.np.mean`:\n\n```{.python}\nimport mxnet as mx\n# Legacy: calculate mean value with reduction on axis 1\n#         with `exclude` option on \nnd_mean = mx.nd.mean(data, axis=1, exclude=1)\n\n# Numpy: no exclude option to users, but user can perform steps as follow\naxes = list(range(data.ndim))\ndel axes[1]\nnp_mean = mx.np.mean(data, axis=axes)\n```\n\n3. Random Operators\n\n+--------------------------------------------------+---------------------------------------------------------+-----------------------------------------------------------------------+\n|                   Legacy Operators               |               NumPy Operators Equivalent                |                              Changes                                  |\n+==================================================+=========================================================+=======================================================================+\n|   `mx.random.uniform(-1.0, 1.0, shape=(2, 3))`   |       `mx.np.random.uniform(-1.0, 1.0, size=(2, 3))`    |   For all the NumPy random operators, use **size** keyword instead of |\n|  `mx.nd.random.uniform(-1.0, 1.0, shape=(2, 3))` |                                                         |   **shape**                                                           |\n+--------------------------------------------------+---------------------------------------------------------+-----------------------------------------------------------------------+\n|    `mx.nd.random.multinomial(*args, **kwargs)`   |       `mx.npx.random.categorical(*args, **kwargs)`      |   use `npx.random.categorical` to have the behavior of drawing 1 sample from multiple distributions.  |\n+--------------------------------------------------+---------------------------------------------------------+-----------------------------------------------------------------------+\n\n4. Control Flow Operators\n\n+----------------------------------------------------------------------------+---------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------+\n|                               Legacy Operators                             |                NumPy Operators Equivalent                                 |                             Changes                                                                           |\n+============================================================================+===========================================================================+===============================================================================================================+\n|          `mx.nd.contrib.foreach(body, data, init_states, name)`            |    `mx.npx.foreach(body, data, init_states, name)`                        | - moved to `npx` namespace.                                                                        |\n|                                                                            |                                                                           | - Will not support global variables as body's inputs(body's inputs must be either data or states or both)   |\n+----------------------------------------------------------------------------+---------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------+\n|  `mx.nd.contrib.while_loop(cond, func, loop_vars, max_iterations, name)`   |    `mx.npx.while_loop(cond, func, loop_vars, max_iterations, name)`       | - moved to `npx` namespace.                                                                        |\n|                                                                            |                                                                           | - Will not support global variables as cond or func's inputs(cond or func's inputs must be in loop_vars)    |\n+----------------------------------------------------------------------------+---------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------+\n|       `mx.nd.contrib.cond(pred, then_func, else_func, inputs, name)`       |        `mx.npx.cond(pred, then_func, else_func, name)`                    | - moved to `npx` namespace.                                                                        |\n|                                                                            |                                                                           | - users needs to provide the inputs of pred, then_func and else_func as inputs                             |\n|                                                                            |                                                                           | - Will not support global variables as pred, then_func or else_func's                                       |\n|                                                                            |                                                                           | inputs(pred, then_func or else_func's inputs must be in inputs)                                             |\n+----------------------------------------------------------------------------+---------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------+\n\n5. Functionalities\n\n+--------------------------------------------------+---------------------------------------------------------+-----------------------------------------------------------------------+\n|                   Legacy Operators               |               NumPy Operators Equivalent                |                              Changes                                  |\n+==================================================+=========================================================+=======================================================================+\n|       `mx.nd.save(*args, **kwargs)`              |            `mx.npx.savez(*args, **kwargs)`              |  - moved to `npx` namespace.                                          |\n|                                                  |                                                         |  - Only accept positional arguments, try to flatten the list/dict     |\n|                                                  |                                                         |    before feed in                                                     |\n+--------------------------------------------------+---------------------------------------------------------+-----------------------------------------------------------------------+\n|       `mx.nd.load(*args, **kwargs)`              |            `mx.npx.load(*args, **kwargs)`               |  - moved to `npx` namespace.                                          |\n+--------------------------------------------------+---------------------------------------------------------+-----------------------------------------------------------------------+\n|       `mx.nd.waitall()`                          |            `mx.npx.waitall()`                           |  - moved to `npx` namespace.                                          |\n+--------------------------------------------------+---------------------------------------------------------+-----------------------------------------------------------------------+\n\nOther operator changes are included in [**Appendix/NumPy and NumPy-extension Operators**](#NumPy-and-NumPy-extension-Operators1) \n\n\n\n### Layers and Blocks\nWith the deferred compute and tracing being introduced in Gluon2.0, users do not need to interact with symbols any more. There are a lot of changes in building a model with Gluon API, including parameter management and naming, forward pass computing and parameter shape inferencing. We provide step-by-step migration guidance on how to build a model with new APIs. \n\n#### Parameter Management and Block Naming\nIn Gluon, each Parameter or Block has a name (and prefix). Parameter names are specified by users and Block names can be either specified by users or automatically created. In Gluon 1.x, parameters are accessed via the `params` variable of the `ParameterDict` in `Block`. Users will need to manually use `with self.name_scope():` for children blocks and specify prefix for the top level block. Otherwise, it will lead to wrong name scopes and can return parameters of children blocks that are not in the current name scope. An example for initializing the Block and Parameter in Gluon 1.x: \n```{.python}\nfrom mxnet.gluon import Parameter, Constant, HybridBlock\nclass SampleBlock(HybridBlock):\n    def __init__(self):\n        super(SampleBlock, self).__init__()\n        with self.name_scope():\n            # Access parameters, which are iterated during training\n            self.weight = self.params.get('weight')\n            # Access constant parameters, which are not iterated during training\n            self.weight = self.params.get_constant('const', const_arr)\n```\nNow in Gluon 2.0, Block/HybridBlock objects will not maintain the parameter dictionary (`ParameterDict`). Instead, users can access these parameters via `Parameter` class and `Constant` class. These parameters will be registered automatically as part of the Block. Users will no longer need to manage the name scope for children blocks and hence can remove `with self.name_scope():` this statement. For example:\n```{.python}\nclass SampleBlock(HybridBlock):\n    def __init__(self):\n        super(SampleBlock, self).__init__()\n        # Access parameters, which are iterated during training\n        self.weight = Parameter('weight')\n        # Access constant parameters, which are not iterated during training\n        self.weight = Constant('const', const_arr)\n```\nAlso, there will be new mechanisms for parameter loading, sharing and setting device. \n\n1. Parameter loading in Gluon 1.x vs Gluon 2.0:\n\n```{.python}\n# in Gluon 1.x\nnet = nn.Dense(8, activation='relu')\nnet.collect_params().load_dict(arg_dict, ctx=ctx)\n# in Gluon 2.0\nnet = nn.Dense(8, activation='relu')\nnet.load_dict(arg_dict, device=device)\n```\n\n2. Parameter sharing in Gluon 1.x vs Gluon 2.0:\n\n```{.python}\n# in Gluon 1.x\nshared = nn.Dense(8, activation='relu')\nnet = nn.Dense(8, activation='relu', params=shared.params)\n# in Gluon 2.0\nshared = nn.Dense(8, activation='relu')\nnet = nn.Dense(8, activation='relu').share_parameters(shared.params)\n```\n\n3. Parameter setting device in Gluon 1.x vs Gluon 2.0:\n\n```{.python}\n# in Gluon 1.x\nnet = nn.Dense(8, activation='relu')\nnet.collect_params().reset_ctx(devices)\n# in Gluon 2.0\nnet = nn.Dense(8, activation='relu')\nnet.reset_device(devices)\n```\n\n#### Forward Interface\n`hybrid_forward` interface in Gluon1.x provides the user with a unified imperative and symbolic programming interface to do graph construction and imperative execution. For the inputs of `hybrid_forward`, `F` can be either mx.symbol or mx.ndarray depending on the running mode(symbolic or imperative) of variable recording. Apart from `F` and input arrays, the parameters registered when Block is initialized are also required as part of the inputs. Take `nn.Dense` as an example:\n\n```{.python}\n# hybrid_forward interface, F can be either symbol or ndarray, weights\n# and bias are part of inputs\ndef hybrid_forward(self, F, x, weight, bias=None):\n    fc = F.npx.fully_connected if is_np_array() else F.FullyConnected\n    act = fc(x, weight, bias, no_bias=bias is None, num_hidden=self._units,\n             flatten=self._flatten, name='fwd')\n    if self.act is not None:\n        act = self.act(act)\n    return act\n```\n\nNow, in deferred computation mode of Gluon2.0, the divergence of NDArray and Symbol is unified, which means users no longer need to define `F` with specific running mode. One can easily specify the computation through imperative programming, hybridization will work through the tracing mechanism(data flow of the first input batch). What's more, users can implement the forward interface with `npx/npx` operators instead of `nd` and `symbol`. \n\n```{.python}\n# forward interface, no F any more\ndef forward(self, x):\n    # get the device information of input array and make parameters run on the same device\n    device = x.device\n    # use np/npx interfaces instead of F\n    act = npx.fully_connected(x, self.weight.data(device),\n                              self.bias.data(device) if self.bias is not None else None,\n                              no_bias=self.bias is None,\n                              num_hidden=self._units, flatten=self._flatten, name='fwd')\n    if self.act is not None:\n        act = self.act(act)\n    return act\n```\n\n#### Implement Infer Shape\nIn Gluon1.x, parameter shape inference happens in MXNet backend. Now in Gluon2.0, shape inference is disabled in the case of deferred parameter initialization. So, users should now always implement `infer_shape` method to set the parameter shapes if the parameter shape was not set during HybridBlock initialization. \n\n```{.python}\ndef infer_shape(self, x, *args):\n    # if true, self.weight.shape[1] will be flattened of input's shape\n    if self._flatten:\n        num_input = 1\n        for i in range(1, x.ndim):\n            num_input *= x.shape[i]\n        self.weight.shape = (self.weight.shape[0], num_input)\n    # if false, self.weight.shape[1] = x.shape[-1]\n    else:\n        self.weight.shape = (self.weight.shape[0], x.shape[x.ndim - 1])\n```\n\nNow, in Gluon2.0, users can implement a Dense Block like this: \n\n```{.python}\nclass Dense(HybridBlock):\n    def __init__(self, units, activation=None, use_bias=True, flatten=True,\n                 dtype='float32', weight_initializer=None, bias_initializer='zeros',\n                 in_units=0, **kwargs):\n        super(Dense, self).__init__(**kwargs)\n        self._flatten = flatten\n        self._units = units\n        self._in_units = in_units\n        self.weight = Parameter('weight', shape=(units, in_units),\n                                init=weight_initializer, dtype=dtype,\n                                allow_deferred_init=True)\n        if use_bias:\n            self.bias = Parameter('bias', shape=(units,),\n                                  init=bias_initializer, dtype=dtype,\n                                  allow_deferred_init=True)\n        else:\n            self.bias = None\n        if activation is not None:\n            self.act = Activation(activation)\n        else:\n            self.act = None\n\n    def forward(self, x):\n        device = x.device\n        act = npx.fully_connected(x, self.weight.data(device),\n                                  self.bias.data(device) if self.bias is not None else None,\n                                  no_bias=self.bias is None,\n                                  num_hidden=self._units, flatten=self._flatten, name='fwd')\n        if self.act is not None:\n            act = self.act(act)\n        return act\n\n    def infer_shape(self, x, *args):\n        if self._flatten:\n            num_input = 1\n            for i in range(1, x.ndim):\n                num_input *= x.shape[i]\n            self.weight.shape = (self.weight.shape[0], num_input)\n        else:\n            self.weight.shape = (self.weight.shape[0], x.shape[x.ndim - 1])\n```\n\n## Optimizers\nOptimizer module in MXNet provides a lot of optimization algorithms to reduce the training error. In Gluon 2.0, optimizers will also switch to use MXNet NumPy-compatible interface. Some important changes that needs attention are: \n\n1. AdaGrad: \n    - use `epsilon` instead of `eps`\n    - e.g. `adagrad_optimizer = optimizer.AdaGrad(learning_rate=0.1, epsilon=1e-07)`\n\n2. RMSProp:\n    - use `rho` instead of `gamma1` and use `momentum` instead of `gamma2`\n    - e.g. `rmsprop_optimizer = optimizer.RMSProp(learning_rate=0.001, rho=0.9, momentum=0.9, epsilon=1e-07, centered=False)`\n\n3. `optimizer.ccSGD` and `optimizer.LBSGD` are deprecated.\n\n## Metrics\nMetrics module in MXNet provides different methods for users to judge the performance of models. In Gluon 2.0, metrics will use MXNet NumPy-compatible interface and also introduce a lot of new evaluation metrics.\n**Changes**:\n1. metric module has been moved to gluon namespace\n    - `mxnet.metric` -> `mxnet.gluon.metric`\n\n2. Add new evaluation metrics: \n    - `Class BinaryAccuracy(threshold=0.5)`\n    - `Class MeanCosineSimilarity(axis=-1, eps=1e-12)`\n    - `Class MeanPairwiseDistance(p=2)`\n    - `Class Fbeta(class_type=\"binary\", beta=1, threshold=0.5, average=\"micro\")`\n\n3. Improve Class F1\n    - `Class F1(name='f1',output_names=None, label_names=None, average=\"macro\")` to\n      `Class F1(name='f1',output_names=None, label_names=None, class_type=\"binary\", threshold=0.5, average=\"micro\")`\n    - **average**: Strategy to be used for aggregating across mini-batches.\n        - \"macro\": Calculate metrics for each label and return unweighted mean of f1.\n        - \"micro\": Calculate metrics globally by counting the total TP, FN and FP.\n        - None: Return f1 scores for each class (numpy.ndarray).\n    - **class_type**:\n        - \"binary\": f1 for binary classification.\n        - \"multiclass\": f1 for multiclassification problem.\n        - \"multilabel\": f1 for multilabel classification.\n    - **threshold**: threshold for postive confidence value.\n\n\n## Key-Value Store\nGluon 2.0 will provide a new and unified low level API for data parallel training. These unified APIs can support different communication backends, including native Parameter Server, Horovod and BytePS. \nExample: \n\n```{.python}\nimport mxnet as mx\n# create key-value store with horovod backend\nkv = mx.kv.create('horovod') # or choose 'kvstore', 'byteps' as backend\ndevice = mx.gpu(kv.local_rank) if mx.device.num_gpus() > 0 else mx.cpu(kv.local_rank)\nval = mx.np.zeros((2, 3), device=device)\n# broadcast the value at rank 0 to all ranks\nkv.broadcast('0', mx.np.zeros((2, 3), device=device), out=val)\nscale = kv.rank + 1\n# performs allreduce on a single array\nkv.pushpull('3', val * scale)\n```\n\n## Probability\nA new module called `mxnet.gluon.probability` has been introduced in Gluon 2.0. It is analogous to pytorch distribution and the main difference is that `mxnet.gluon.probability` will use MXNet NumPy compatible operators and will allow hybridization. It has three parts: \n\n1. [Distribution Objects](https://github.com/apache/incubator-mxnet/tree/master/python/mxnet/gluon/probability/distributions): `gluon.probability.Bernoulli`, `gluon.probability.Beta` ...\n\n2. [StochasticBlock](https://github.com/apache/incubator-mxnet/tree/master/python/mxnet/gluon/probability/block): support accumulating loss in the forward phase, which is useful in building Bayesian Neural Network. \n\n3. [Transformation](https://github.com/apache/incubator-mxnet/tree/master/python/mxnet/gluon/probability/transformation): implement invertible transformation with computable log det jacobians.\n\n##  oneDNN Integration\n### Operator Fusion\nIn versions 1.x of MXNet pattern fusion in execution graph was enabled by default when using MXNet built with oneDNN library support and could have been disabled by setting 'MXNET_SUBGRAPH_BACKEND' environment flag to `None`. MXNet 2.0 introduced changes in forward inference flow which led to refactor of fusion mechanism. To fuse model in MXNet 2.0 there are two requirements:\n\n - the model must be defined as a subclass of HybridBlock or Symbol,\n\n - the model must have specific operator patterns which can be fused.\n\nBoth HybridBlock and Symbol classes provide API to easily run fusion of operators. Adding only one line of code is needed to run fusion passes on model:\n```{.python}\n# on HybridBlock\nnet.optimize_for(data, backend='ONEDNN')\n# on Symbol\noptimized_symbol = sym.optimize_for(backend='ONEDNN')\n```\n\nControling which patterns should be fused still can be done by setting proper environment variables. See [**oneDNN Environment Variables**](#oneDNN-Environment-Variables)\n\n### INT8 Quantization / Precision reduction\nQuantization API was also refactored to be consistent with other new features and mechanisms. In comparison to MXNet 1.x releases, in MXNet 2.0 `quantize_net_v2` function has been removed and development focused mainly on `quantize_net` function to make it easier to use for end user and ultimately give him more flexibility.\nQuantization can be performed on either subclass of HybridBlock with `quantize_net` or Symbol with deprecated `quantize_model` (`quantize_model` is left only to provide backward compatibility and its usage is strongly discouraged).\n\n```{.python}\nimport mxnet as mx\nfrom mxnet.contrib.quantization import quantize_net\nfrom mxnet.gluon.model_zoo.vision import resnet50_v1\n\n# load model\nnet = resnet50_v1(pretrained=True)\n\n# prepare calibration data\ndummy_data = mx.nd.random.uniform(-1.0, 1.0, (batch_size, 3, 224, 224))\ncalib_data_loader = mx.gluon.data.DataLoader(dummy_data, batch_size=batch_size)\n\n# quantization\nqnet = quantize_net(net, calib_mode='naive', calib_data=calib_data_loader)\n```\n`quantize_net` can be much more complex - all function attributes can be found in the [API](../../api/contrib/quantization/index.rst).\n\n### oneDNN Environment Variables\nIn version 2.0 of MXNet all references to MKLDNN (former name of oneDNN) were replaced by ONEDNN. Below table lists all environment variables:\n\n|               MXNet 1.x              |                MXNet 2.0               |\n| ------------------------------------ | ---------------------------------------|\n|         MXNET_MKLDNN_ENABLED         |          MXNET_ONEDNN_ENABLED          |\n|         MXNET_MKLDNN_CACHE_NUM       |         MXNET_ONEDNN_CACHE_NUM         |\n|    MXNET_MKLDNN_FORCE_FC_AB_FORMAT   |     MXNET_ONEDNN_FORCE_FC_AB_FORMAT    |\n|         MXNET_MKLDNN_ENABLED         |          MXNET_ONEDNN_ENABLED          |\n|         MXNET_MKLDNN_DEBUG           |           MXNET_ONEDNN_DEBUG           |\n|         MXNET_USE_MKLDNN_RNN         |          MXNET_USE_ONEDNN_RNN          |\n|     MXNET_DISABLE_MKLDNN_CONV_OPT    |      MXNET_DISABLE_ONEDNN_CONV_OPT     |\n|    MXNET_DISABLE_MKLDNN_FUSE_CONV_BN |    MXNET_DISABLE_ONEDNN_FUSE_CONV_BN   |\n|  MXNET_DISABLE_MKLDNN_FUSE_CONV_RELU |   MXNET_DISABLE_ONEDNN_FUSE_CONV_RELU  |\n|  MXNET_DISABLE_MKLDNN_FUSE_CONV_SUM  |   MXNET_DISABLE_ONEDNN_FUSE_CONV_SUM   |\n|      MXNET_DISABLE_MKLDNN_FC_OPT     |       MXNET_DISABLE_ONEDNN_FC_OPT      |\n| MXNET_DISABLE_MKLDNN_FUSE_FC_ELTWISE |  MXNET_DISABLE_ONEDNN_FUSE_FC_ELTWISE  |\n| MXNET_DISABLE_MKLDNN_TRANSFORMER_OPT |  MXNET_DISABLE_ONEDNN_TRANSFORMER_OPT  |\n|                  n/a                 |   MXNET_DISABLE_ONEDNN_BATCH_DOT_FUSE  |\n|                  n/a                 |      MXNET_ONEDNN_FUSE_REQUANTIZE      |\n|                  n/a                 |      MXNET_ONEDNN_FUSE_DEQUANTIZE      |\n\n## Appendix\n### NumPy Array Deprecated Attributes\n|                   Deprecated Attributes               |    NumPy ndarray Equivalent    |\n| ----------------------------------------------------- | ------------------------------ |\n|                   `a.abs()`                           |             `mx.np.abs(a)`           |\n|                   `a.sign()`                          |             `mx.np.sign(a)`          |\n|              `a.split_v2(2, axis=1)`                  |   `mx.np.split(a, 2, axis=1)`  |\n|            `a.flip(*args, **kwargs)`                  |    `mx.np.flip(a, *args, **kwargs)`  |\n|            `a.diag(*args, **kwargs)`                  |    `mx.np.diag(a, *args, **kwargs)`  |\n|           `a.nansum(*args, **kwargs)`                 | `mx.np.nan_to_num(a, *args, **kwargs).sum()`  |\n|           `a.nanprod(*args, **kwargs)`                | `mx.np.nan_to_num(a, *args, **kwargs).prod()` |\n|            `a.diag(*args, **kwargs)`                  |    `mx.np.diag(a, *args, **kwargs)`  |\n|                  `a.norm()`                           |           `mx.npx.norm(a)`           |\n|            `a.rint(*args, **kwargs)`                  |    `mx.np.rint(a, *args, **kwargs)`  |\n|            `a.fix(*args, **kwargs)`                   |    `mx.np.fix(a, *args, **kwargs)`   |\n|            `a.floor(*args, **kwargs)`                 |    `mx.np.floor(a, *args, **kwargs)`  |\n|            `a.ceil(*args, **kwargs)`                  |    `mx.np.ceil(a, *args, **kwargs)`   |\n|            `a.trunc(*args, **kwargs)`                 |    `mx.np.trunc(a, *args, **kwargs)`  |\n|            `a.sin(*args, **kwargs)`                   |    `mx.np.sin(a, *args, **kwargs)`    |\n|            `a.cos(*args, **kwargs)`                   |    `mx.np.cos(a, *args, **kwargs)`    |\n|            `a.tan(*args, **kwargs)`                   |    `mx.np.tan(a, *args, **kwargs)`    |\n|            `a.arcsin(*args, **kwargs)`                |    `mx.np.arcsin(a, *args, **kwargs)`  |\n|            `a.arccos(*args, **kwargs)`                |    `mx.np.arccos(a, *args, **kwargs)`  |\n|            `a.arctan(*args, **kwargs)`                |    `mx.np.arctan(a, *args, **kwargs)`  |\n|            `a.degrees(*args, **kwargs)`               |    `mx.np.degrees(a, *args, **kwargs)`  |\n|            `a.radians(*args, **kwargs)`               |    `mx.np.radians(a, *args, **kwargs)`  |\n|            `a.sinh(*args, **kwargs)`                  |    `mx.np.sinh(a, *args, **kwargs)`  |\n|            `a.cosh(*args, **kwargs)`                  |    `mx.np.cosh(a, *args, **kwargs)`  |\n|            `a.tanh(*args, **kwargs)`                  |    `mx.np.tanh(a, *args, **kwargs)`  |\n|            `a.arcsinh(*args, **kwargs)`               |    `mx.np.arcsinh(a, *args, **kwargs)`  |\n|            `a.arccosh(*args, **kwargs)`               |    `mx.np.arccosh(a, *args, **kwargs)`  |\n|            `a.arctanh(*args, **kwargs)`               |    `mx.np.arctanh(a, *args, **kwargs)`  |\n|            `a.exp(*args, **kwargs)`                   |    `mx.np.exp(a, *args, **kwargs)`  |\n|            `a.expm1(*args, **kwargs)`                 |    `mx.np.expm1(a, *args, **kwargs)`  |\n|            `a.log(*args, **kwargs)`                   |    `mx.np.log(a, *args, **kwargs)`  |\n|            `a.log10(*args, **kwargs)`                 |    `mx.np.log10(a, *args, **kwargs)`  |\n|            `a.log2(*args, **kwargs)`                  |    `mx.np.log2(a, *args, **kwargs)`  |\n|            `a.log1p(*args, **kwargs)`                 |    `mx.np.log1p(a, *args, **kwargs)`  |\n|            `a.sqrt(*args, **kwargs)`                  |    `mx.np.sqrt(a, *args, **kwargs)`  |\n|            `a.rsqrt(*args, **kwargs)`                 |    `1 / mx.np.sqrt(a, *args, **kwargs)`  |\n|            `a.cbrt(*args, **kwargs)`                  |    `mx.np.cbrt(a, *args, **kwargs)`  |\n|            `a.rcbrt(*args, **kwargs)`                 |    `1 / mx.np.cbrt(a, *args, **kwargs)`  |\n|            `a.square(*args, **kwargs)`                |    `mx.np.square(a, *args, **kwargs)`  |\n|                `a.pad(*args, **kwargs)`               |   `mx.npx.pad(a, *args, **kwargs)`   |\n|          `a.split(axis=1, num_outputs=2)`             |   `mx.np.split(a, 2, axis=1)`  |\n|            `a.slice(*args, **kwargs)`                 |   `mx.npx.slice(a, *args, **kwargs)`  |\n|          `a.one_hot(*args, **kwargs)`                 |   `mx.npx.one_hot(a, *args, **kwargs)`  |\n|           `a.pick(*args, **kwargs)`                   |   `mx.npx.pick(a, *args, **kwargs)`  |\n|           `a.topk(*args, **kwargs)`                   |   `mx.npx.topk(a, *args, **kwargs)`  |\n|               `a.shape_array()`                       |         `mx.np.array(a.shape)`       |\n|               `a.size_array()`                        |         `mx.np.array(a.size)`        |\n|         `a.expand_dims(*args, **kwargs)`              | `mx.np.expand_dims(a, *args, **kwargs)`  |\n|            `a.relu(*args, **kwargs)`                  |    `mx.npx.relu(a, *args, **kwargs)`  |\n|            `a.sigmoid(*args, **kwargs)`               |    `mx.npx.sigmoid(a, *args, **kwargs)`  |\n|            `a.softmax(*args, **kwargs)`               |    `mx.npx.softmax(a, *args, **kwargs)`  |\n|            `a.log_softmax(*args, **kwargs)`           |    `mx.npx.log_softmax(a, *args, **kwargs)`  |\n|        `a.broadcast_like(*args, **kwargs)`            |  `mx.npx.broadcast_like(a, *args, **kwargs)`  |\n|            `a.reciprocal(*args, **kwargs)`            |    `mx.np.reciprocal(a, *args, **kwargs)`  |\n\n### NumPy and NumPy-extension Operators\n|                   Legacy Operators               |    NumPy Operators Equivalent    |   Changes  |\n| ----------------------------------------------------- | ------------------------------ | ------------------- |\n|       `mx.nd.softmax(*args, **kwargs)`                |            `mx.npx.softmax(*args, **kwargs)`                    |                moved to `npx` namespace            |\n|       `mx.nd.log_softmax(*args, **kwargs)`                |            `mx.npx.log_softmax(*args, **kwargs)`                    |                moved to `npx` namespace            |\n|       `mx.nd.masked_softmax(*args, **kwargs)`                |            `mx.npx.masked_softmax(*args, **kwargs)`                    |                moved to `npx` namespace            |\n|       `mx.nd.masked_log_softmax(*args, **kwargs)`                |            `mx.npx.masked_log_softmax(*args, **kwargs)`                    |                moved to `npx` namespace            |\n|       `mx.nd.pick(*args, **kwargs)`                |            `mx.npx.pick(*args, **kwargs)`                    |                moved to `npx` namespace            |\n|       `mx.nd.topk(*args, **kwargs)`                |            `mx.npx.topk(*args, **kwargs)`                    |                moved to `npx` namespace            |\n|       `mx.nd.batch_dot(*args, **kwargs)`                |            `mx.npx.batch_dot(*args, **kwargs)`                    |                moved to `npx` namespace            |\n|       `mx.nd.broadcast_like(*args, **kwargs)`                |            `mx.npx.broadcast_like(*args, **kwargs)`                    |                moved to `npx` namespace            |\n|       `mx.nd.arange_like(*args, **kwargs)`                |            `mx.npx.arange_like(*args, **kwargs)`                    |                moved to `npx` namespace            |\n|      `mx.nd.BatchNorm(*args, **kwargs)`              |            `mx.npx.batch_norm(*args, **kwargs)`                 |              - moved to `npx` namespace with new name `batch_norm`.          |\n|      `mx.nd.Convolution(*args, **kwargs)`              |            `mx.npx.convolution(*args, **kwargs)`                 |              - moved to `npx` namespace with new name `convolution`.          |\n|      `mx.nd.Deconvolution(*args, **kwargs)`              |            `mx.npx.deconvolution(*args, **kwargs)`                 |              - moved to `npx` namespace with new name `deconvolution`.          |\n|      `mx.nd.Pooling(*args, **kwargs)`              |            `mx.npx.pooling(*args, **kwargs)`                 |              - moved to `npx` namespace with new name `pooling`.          |\n|      `mx.nd.Dropout(*args, **kwargs)`              |            `mx.npx.dropout(*args, **kwargs)`                 |              - moved to `npx` namespace with new name `dropout`.          |\n|      `mx.nd.RNN(*args, **kwargs)`              |            `mx.npx.rnn(*args, **kwargs)`                 |              - moved to `npx` namespace with new name `rnn`.          |\n|      `mx.nd.Embedding(*args, **kwargs)`              |            `mx.npx.embedding(*args, **kwargs)`                 |              - moved to `npx` namespace with new name `embedding`.          |\n|      `mx.nd.LayerNorm(*args, **kwargs)`              |            `mx.npx.layer_norm(*args, **kwargs)`                 |              - moved to `npx` namespace with new name `layer_norm`.          |\n|      `mx.nd.LeakyReLU(*args, **kwargs)`              |            `mx.npx.leaky_relu(*args, **kwargs)`                 |              - moved to `npx` namespace with new name `leaky_relu`.          |\n|      `mx.nd.GroupNorm(*args, **kwargs)`              |            `mx.npx.group_norm(*args, **kwargs)`                 |              - moved to `npx` namespace with new name `group_norm`.          |\n\n## Reference\n\n1. [Next Generation of GluonNLP](https://github.com/dmlc/gluon-nlp/tree/master)\n2. [MXNet NumPy-compatible coding experience](https://github.com/apache/incubator-mxnet/issues/14253)\n3. [Gluon Data API Extension](https://github.com/apache/incubator-mxnet/issues/17269)\n4. [Simplifying MXNet Gluon APIs](https://github.com/apache/incubator-mxnet/issues/18412)\n5. [Deferred Compute and Tracing](https://github.com/apache/incubator-mxnet/issues/16376)\n6. [MXNet Metrics Improvements](https://github.com/apache/incubator-mxnet/issues/18046)\n7. [Gluon Distribution Module](https://github.com/apache/incubator-mxnet/issues/17240)"
  },
  {
    "path": "docs/python_docs/python/tutorials/getting-started/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nGetting Started\n===============\n\nThe following tutorials teach how to use MXNet.\n\n.. container:: cards\n\n   .. card::\n      :title: A 60-minute Gluon crash course\n      :link: crash-course/index.html\n\n      A quick overview of the core concepts of MXNet using the Gluon API.\n\n   .. card::\n      :title: Moving from other frameworks\n      :link: to-mxnet/index.html\n\n      Guides that ease your transition to MXNet from other framework.\n\n   .. card::\n      :title: Logistic Regression Explained\n      :link: logistic_regression_explained.html\n\n      Logistic Regression with MXNet Gluon Explained\n\n   .. card::\n      :title: MNIST Training\n      :link: https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/image/mnist.html\n\n      MNIST Training with MXNet\n\n   .. card::\n      :title: Gluon From Experiment To Deployment\n      :link: gluon_from_experiment_to_deployment.html\n\n      A tutorial on implementing linear regression using MXNet APIs.\n\n   .. card::\n      :title: Gluon2.0: Migration Guide\n      :link: gluon_migration_guide.html\n\n      Migrate from MXNet v1.x to v2.x.\n\n.. toctree::\n   :hidden:\n   :maxdepth: 2\n\n   crash-course/index\n   to-mxnet/index\n   gluon_from_experiment_to_deployment\n   gluon_migration_guide\n   logistic_regression_explained.md\n   MNIST <https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/image/mnist.html>\n\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/getting-started/logistic_regression_explained.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Logistic regression explained\n\nLogistic Regression is one of the first models newcomers to Deep Learning are implementing. The focus of this tutorial is to show how to do logistic regression using Gluon API.\n\nBefore anything else, let's import required packages for this tutorial.\n\n\n```{.python .input}\nimport numpy as onp\nimport mxnet as mx\nfrom mxnet import np, npx, autograd, gluon\nfrom mxnet.gluon import nn, Trainer\nfrom mxnet.gluon.data import DataLoader, ArrayDataset\n\nmx.np.random.seed(12345)  # Added for reproducibility\n```\n\nIn this tutorial we will use fake dataset, which contains 10 features drawn from a normal distribution with mean equals to 0 and standard deviation equals to 1, and a class label, which can be either 0 or 1. The size of the dataset is an arbitrary value. The function below helps us to generate a dataset. Class label `y` is generated via a non-random logic, so the network would have a pattern to look for. Boundary of 3 is selected to make sure that number of positive examples smaller than negative, but not too small\n\n\n```{.python .input}\ndef get_random_data(size, device):\n    x = np.random.normal(0, 1, size=(size, 10), device=device)\n    y = x.sum(axis=1) > 3\n    return x, y\n```\n\nAlso, let's define a set of hyperparameters, that we are going to use later. Since our model is simple and dataset is small, we are going to use CPU for calculations. Feel free to change it to GPU for a more advanced scenario.\n\n\n```{.python .input}\ndevice = mx.cpu()\ntrain_data_size = 1000\nval_data_size = 100\nbatch_size = 10\n```\n\n## Working with data\n\nTo work with data, Apache MXNet provides [Dataset](../../api/gluon/data/index.rst#mxnet.gluon.data.Dataset) and [DataLoader](../../api/gluon/data/index.rst#mxnet.gluon.data.DataLoader) classes. The former is used to provide an indexed access to the data, the latter is used to shuffle and batchify the data. To learn more about working with data in Gluon, please refer to [Gluon Datasets and Dataloaders](../../api/gluon/data/index.rst).\n\nBelow we define training and validation datasets, which we are going to use in the tutorial.\n\n\n```{.python .input}\ntrain_x, train_ground_truth_class = get_random_data(train_data_size, device)\ntrain_dataset = ArrayDataset(train_x, train_ground_truth_class)\ntrain_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n\nval_x, val_ground_truth_class = get_random_data(val_data_size, device)\nval_dataset = ArrayDataset(val_x, val_ground_truth_class)\nval_dataloader = DataLoader(val_dataset, batch_size=batch_size, shuffle=True)\n```\n\n## Defining and training the model\n\nThe only requirement for the logistic regression is that the last layer of the network must be a single neuron. Apache MXNet allows us to do so by using [Dense](../../api/gluon/nn/index.rst#mxnet.gluon.nn.Dense) layer and specifying the number of units to 1. The rest of the network can be arbitrarily complex.\n\nBelow, we define a model which has an input layer of 10 neurons, a couple of inner layers of 10 neurons each, and output layer of 1 neuron. We stack the layers using [HybridSequential](../../api/gluon/nn/index.rst#mxnet.gluon.nn.HybridSequential) block and initialize parameters of the network using [Xavier](../../api/initializer/index.rst#mxnet.initializer.Xavier) initialization.\n\n\n```{.python .input}\nnet = nn.HybridSequential()\n\nnet.add(nn.Dense(units=10, activation='relu'))  # input layer\nnet.add(nn.Dense(units=10, activation='relu'))   # inner layer 1\nnet.add(nn.Dense(units=10, activation='relu'))   # inner layer 2\nnet.add(nn.Dense(units=1))   # output layer: notice, it must have only 1 neuron\n\nnet.initialize(mx.init.Xavier())\n```\n\nAfter defining the model, we need to define a few more things: our loss, our trainer and our metric.\n\nLoss function is used to calculate how the output of the network differs from the ground truth. Because classes  of the logistic regression are either 0 or 1, we are using [SigmoidBinaryCrossEntropyLoss](../../api/gluon/loss/index.rst#mxnet.gluon.loss.SigmoidBinaryCrossEntropyLoss). Notice that we do not specify `from_sigmoid` attribute in the code, which means that the output of the neuron doesn't need to go through sigmoid, but at inference we'd have to pass it through sigmoid. You can learn more about cross entropy on [wikipedia](https://en.wikipedia.org/wiki/Cross_entropy).\n\nTrainer object allows to specify the method of training to be used. For our tutorial we use [Stochastic Gradient Descent (SGD)](../../api/optimizer/index.rst#mxnet.optimizer.SGD). For more information on SGD refer to [the following tutorial](https://d2l.ai/chapter_optimization/sgd.html). We also need to parametrize it with learning rate value, which defines the weight updates, and weight decay, which is used for regularization.\n\nMetric helps us to estimate how good our model is in terms of a problem we are trying to solve. Where loss function has more importance for the training process, a metric is usually the thing we are trying to improve and reach maximum value. We also can use more than one metric, to measure various aspects of our model. In our example, we are using [Accuracy](../../api/gluon/metric/index.rst#mxnet.gluon.metric.Accuracy) and [F1 score](../../api/gluon/metric/index.rst#mxnet.gluon.metric.F1) as measurements of success of our model.\n\nBelow we define these objects.\n\n\n```{.python .input}\nloss = gluon.loss.SigmoidBinaryCrossEntropyLoss()\ntrainer = Trainer(params=net.collect_params(), optimizer='sgd',\n                  optimizer_params={'learning_rate': 0.1})\naccuracy = mx.gluon.metric.Accuracy()\nf1 = mx.gluon.metric.F1()\n```\n\nThe next step is to define the training function in which we iterate over all batches of training data, execute the forward pass on each batch and calculate training loss. On line 19, we sum losses of every batch per epoch into a single variable, because we calculate loss per single batch, but want to display it per epoch.\n\n\n```{.python .input}\ndef train_model():\n    cumulative_train_loss = 0\n\n    for i, (data, label) in enumerate(train_dataloader):\n        with autograd.record():\n            # Do forward pass on a batch of training data\n            output = net(data)\n\n            # Calculate loss for the training data batch\n            loss_result = loss(output, label)\n\n        # Calculate gradients\n        loss_result.backward()\n\n        # Update parameters of the network\n        trainer.step(batch_size)\n\n        # sum losses of every batch\n        cumulative_train_loss += np.sum(loss_result).item()\n\n    return cumulative_train_loss\n```\n\n## Validating the model\n\nOur validation function is very similar to the training one. The main difference is that we want to calculate accuracy of the model. We use [Accuracy metric](../../api/gluon/metric/index.rst#mxnet.gluon.metric.Accuracy) to do so. \n\n`Accuracy` metric requires 2 arguments: 1) a vector of ground-truth classes and 2) A vector or matrix of predictions. When predictions are of the same shape as the vector of ground-truth classes, `Accuracy` class assumes that prediction vector contains predicted classes. So, it converts the vector to `Int32` and compare each item of ground-truth classes to prediction vector.\n\nBecause of the behaviour above, you will get an unexpected result if you just apply [Sigmoid](https://mxnet.apache.org/versions/master/api/python/docs/api/npx/generated/mxnet.npx.sigmoid.html) function to the network result and pass it to `Accuracy` metric. As mentioned before, we need to apply `Sigmoid` function to the output of the neuron to get a probability of belonging to the class 1. But `Sigmoid` function produces output in range [0; 1], and all numbers in that range are going to be casted to 0, even if it is as high as 0.99. To avoid this we write a custom bit of code on line 12, that:\n\n1. Calculates sigmoid using `Sigmoid` function\n\n2. Subtracts a threshold from the original sigmoid output. Usually, the threshold is equal to 0.5, but it can be higher, if you want to increase certainty of an item to belong to class 1.\n\n3. Uses [mx.np.ceil](https://mxnet.apache.org/versions/master/api/python/docs/api/np/generated/mxnet.np.ceil.html#mxnet-np-ceil) function, which converts all negative values to 0 and all positive values to 1\n\nAfter these transformations we can pass the result to `Accuracy.update()` method and expect it to behave in a proper way.\n\nFor `F1` metric to work, instead of one number per class, we must pass probabilities of belonging to both classes. Because of that, on lines 21-22 we:\n\n1. Reshape predictions to a single vector\n\n2. We stack together two vectors: probabilities of belonging to class 0 (1 - `prediction`) and probabilities of belonging to class 1.\n\nThen we pass this stacked matrix to `F1` score.\n\n\n```{.python .input}\ndef validate_model(threshold):\n    cumulative_val_loss = 0\n\n    for i, (val_data, val_ground_truth_class) in enumerate(val_dataloader):\n        # Do forward pass on a batch of validation data\n        output = net(val_data)\n\n        # Similar to cumulative training loss, calculate cumulative validation loss\n        cumulative_val_loss += np.sum(loss(output, val_ground_truth_class)).item()\n\n        # getting prediction as a sigmoid\n        prediction = npx.sigmoid(net(val_data))\n\n        # Converting neuron outputs to classes\n        predicted_classes = mx.np.ceil(prediction - threshold)\n\n        # Update validation accuracy\n        accuracy.update(val_ground_truth_class, predicted_classes.reshape(-1))\n\n        # calculate probabilities of belonging to different classes. F1 metric works only with this notation\n        prediction = prediction.reshape(-1)\n        probabilities = mx.np.stack([1 - prediction, prediction], axis=1)\n\n        f1.update(val_ground_truth_class, probabilities)\n\n    return cumulative_val_loss\n```\n\n## Putting it all together\n\nBy using the defined above functions, we can finally write our main training loop.\n\n\n```{.python .input}\nepochs = 10\nthreshold = 0.5\n\nfor e in range(epochs):\n    avg_train_loss = train_model() / train_data_size\n    avg_val_loss = validate_model(threshold) / val_data_size\n\n    print(\"Epoch: %s, Training loss: %.2f, Validation loss: %.2f, Validation accuracy: %.2f, F1 score: %.2f\" %\n          (e, avg_train_loss, avg_val_loss, accuracy.get()[1], f1.get()[1]))\n\n    # we reset accuracy, so the new epoch's accuracy would be calculated from the blank state\n    accuracy.reset()\n```\n\nOutput:\n\n```bash\nEpoch: 0, Training loss: 0.43, Validation loss: 0.36, Validation accuracy: 0.85, F1 score: 0.00 <!--notebook-skip-line-->\n\nEpoch: 1, Training loss: 0.22, Validation loss: 0.14, Validation accuracy: 0.96, F1 score: 0.35 <!--notebook-skip-line-->\n\nEpoch: 2, Training loss: 0.09, Validation loss: 0.11, Validation accuracy: 0.97, F1 score: 0.48 <!--notebook-skip-line-->\n\nEpoch: 3, Training loss: 0.07, Validation loss: 0.09, Validation accuracy: 0.96, F1 score: 0.53 <!--notebook-skip-line-->\n\nEpoch: 4, Training loss: 0.06, Validation loss: 0.09, Validation accuracy: 0.97, F1 score: 0.58 <!--notebook-skip-line-->\n\nEpoch: 5, Training loss: 0.04, Validation loss: 0.12, Validation accuracy: 0.97, F1 score: 0.59 <!--notebook-skip-line-->\n\nEpoch: 6, Training loss: 0.05, Validation loss: 0.09, Validation accuracy: 0.99, F1 score: 0.62 <!--notebook-skip-line-->\n\nEpoch: 7, Training loss: 0.05, Validation loss: 0.10, Validation accuracy: 0.97, F1 score: 0.62 <!--notebook-skip-line-->\n\nEpoch: 8, Training loss: 0.05, Validation loss: 0.12, Validation accuracy: 0.95, F1 score: 0.63 <!--notebook-skip-line-->\n\nEpoch: 9, Training loss: 0.04, Validation loss: 0.09, Validation accuracy: 0.98, F1 score: 0.65 <!--notebook-skip-line-->\n```\n\nIn our case we hit the accuracy of 0.98 and F1 score of 0.65.\n\n## Tip 1: Use only one neuron in the output layer\n\nDespite that there are 2 classes, there should be only one output neuron, because `SigmoidBinaryCrossEntropyLoss` accepts only one feature as an input.\n\n## Tip 2: Encode classes as 0 and 1\n\nFor `SigmoidBinaryCrossEntropyLoss` to work it is required that classes were encoded as 0 and 1. In some datasets the class encoding might be different, like -1 and 1 or 1 and 2. If this is how your dataset looks like, then you need to re-encode the data before using `SigmoidBinaryCrossEntropyLoss`.\n\n## Tip 3: Use SigmoidBinaryCrossEntropyLoss\n\nGluon API has an options to calculate logistic regression loss: [SigmoidBinaryCrossEntropyLoss](../../api/gluon/loss/index.rst#mxnet.gluon.loss.SigmoidBinaryCrossEntropyLoss).\n\n## Conclusion\n\nIn this tutorial I explained some potential pitfalls to be aware of. When doing logistic regression using Gluon API remember to:\n1. Use only one neuron in the output layer\n1. Encode class labels as 0 or 1\n1. Use `SigmoidBinaryCrossEntropyLoss`\n1. Convert probabilities to classes before calculating Accuracy\n\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/getting-started/to-mxnet/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nMoving to MXNet from Other Frameworks\n=====================================\n\nComparison Guides\n-----------------\n\n.. container:: cards\n\n   .. card::\n      :title: PyTorch to MXNet (MNIST)\n      :link: pytorch.html\n\n      This guide compares PyTorch and MXNet when implementing MNIST.\n\n.. toctree::\n   :hidden:\n   :maxdepth: 1\n\n   pytorch\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/getting-started/to-mxnet/pytorch.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# PyTorch vs Apache MXNet\n\n[PyTorch](https://pytorch.org/) is a popular deep learning framework due to its easy-to-understand API and its completely imperative approach. Apache MXNet includes the Gluon API which gives you the simplicity and flexibility of PyTorch and allows you to hybridize your network to leverage performance optimizations of the symbolic graph. As of April 2019, [NVidia performance benchmarks](https://developer.nvidia.com/deep-learning-performance-training-inference) show that Apache MXNet outperforms PyTorch by ~77% on training ResNet-50: 10,925 images per second vs. 6,175.\n\nIn the next 10 minutes, we'll do a quick comparison between the two frameworks and show how small the learning curve can be when switching from PyTorch to Apache MXNet.\n\n## Installation\n\nPyTorch uses conda for installation by default, for example:\n\n```{.python .input}\n# !conda install pytorch-cpu -c pytorch, torchvision\n```\n\nFor MXNet we use pip:\n\n```{.python .input}\n# !pip install mxnet\n```\n\nTo install Apache MXNet with GPU support, you need to specify CUDA version. For example, the snippet below will install Apache MXNet with CUDA 10.2 support:\n\n```{.python .input}\n# !pip install mxnet-cu102\n```\n\n## Data manipulation\n\nBoth PyTorch and Apache MXNet relies on multidimensional matrices as a data sources. While PyTorch follows Torch's naming convention and refers to multidimensional matrices as \"tensors\", Apache MXNet follows NumPy's conventions and refers to them as \"NDArrays\".\n\nIn the code snippets below, we create a two-dimensional matrix where each element is initialized to 1. We show how to add 1 to each element of matrices and print the results.\n\n**PyTorch:**\n\n```{.python .input}\nimport torch\n\nx = torch.ones(5,3)\ny = x + 1\ny\n```\n\n**MXNet:**\n\n```{.python .input}\nfrom mxnet import np\n\nx = np.ones((5,3))\ny = x + 1\ny\n```\n\nThe main difference apart from the package name is that the MXNet's shape input parameter needs to be passed as a tuple enclosed in parentheses as in NumPy.\n\nBoth frameworks support multiple functions to create and manipulate tensors / NDArrays. You can find more of them in the documentation.\n\n## Model training\n\nAfter covering the basics of data creation and manipulation, let's dive deep and compare how model training is done in both frameworks. In order to do so, we are going to solve image classification task on MNIST data set using Multilayer Perceptron (MLP) in both frameworks. We divide the task in 4 steps.\n\n### 1. Read data\n\nThe first step is to obtain the data. We download the MNIST data set from the web and load it into memory so that we can read batches one by one.\n\n**PyTorch:**\n\n```{.python .input}\nfrom torchvision import datasets, transforms\n\ntrans = transforms.Compose([transforms.ToTensor(),\n                            transforms.Normalize((0.13,), (0.31,))])\npt_train_data = torch.utils.data.DataLoader(datasets.MNIST(\n    root='.', train=True, download=True, transform=trans),\n    batch_size=128, shuffle=True, num_workers=4)\n```\n\n**MXNet:**\n\n```{.python .input}\nfrom mxnet import gluon\nfrom mxnet.gluon.data.vision import datasets, transforms\n\ntrans = transforms.Compose([transforms.ToTensor(),\n                            transforms.Normalize(0.13, 0.31)])\nmx_train_data = gluon.data.DataLoader(\n    datasets.MNIST(train=True).transform_first(trans),\n    batch_size=128, shuffle=True, num_workers=4)\n```\n\nBoth frameworks allows you to download MNIST data set from their sources and specify that only training part of the data set is required.\n\nThe main difference between the code snippets is that MXNet uses [transform_first](../../../api/gluon/data/index.rst#mxnet.gluon.data.Dataset.transform_first) method to indicate that the data transformation is done on the first element of the data batch, the MNIST picture, rather than the second element, the label.\n\n### 2. Creating the model\n\nBelow we define a Multilayer Perceptron (MLP) with a single hidden layer\nand 10 units in the output layer.\n\n**PyTorch:**\n\n```{.python .input}\nimport torch.nn as pt_nn\n\npt_net = pt_nn.Sequential(\n    pt_nn.Linear(28*28, 256),\n    pt_nn.ReLU(),\n    pt_nn.Linear(256, 10))\n```\n\n**MXNet:**\n\n```{.python .input}\nimport mxnet.gluon.nn as mx_nn\n\nmx_net = mx_nn.Sequential()\nmx_net.add(mx_nn.Dense(256, activation='relu'),\n           mx_nn.Dense(10))\nmx_net.initialize()\n```\n\nWe used the Sequential container to stack layers one after the other in order to construct the neural network. Apache MXNet differs from PyTorch in the following ways:\n\n* In PyTorch you have to specify the input size as the first argument of the `Linear` object. Apache MXNet provides an extra flexibility to network structure by automatically inferring the input size after the first forward pass.\n\n* In Apache MXNet you can specify activation functions directly in fully connected and convolutional layers.\n\n* After the model structure is defined, Apache MXNet requires you to explicitly call the model initialization function.\n\nWith a Sequential block, layers are executed one after the other. To have a different execution model, with PyTorch you can inherit from `nn.Module` and then customize how the `.forward()` function is executed. Similarly, in Apache MXNet you can inherit from [gluon.Block](../../../api/gluon/block.rst#mxnet.gluon.Block) to achieve similar results.\n\n### 3. Loss function and optimization algorithm\n\nThe next step is to define the loss function and pick an optimization algorithm. Both PyTorch and Apache MXNet provide multiple options to chose from, and for our particular case we are going to use the cross-entropy loss function and the Stochastic Gradient Descent (SGD) optimization algorithm.\n\n**PyTorch:**\n\n```{.python .input}\npt_loss_fn = pt_nn.CrossEntropyLoss()\npt_trainer = torch.optim.SGD(pt_net.parameters(), lr=0.1)\n```\n\n**MXNet:**\n\n```{.python .input}\nmx_loss_fn = gluon.loss.SoftmaxCrossEntropyLoss()\nmx_trainer = gluon.Trainer(mx_net.collect_params(),\n                           'sgd', {'learning_rate': 0.1})\n```\n\nThe code difference between frameworks is small. The main difference is that in Apache MXNet we use [Trainer](../../../api/gluon/trainer.rst) class, which accepts optimization algorithm as an argument. We also use [.collect_params()](../../../api/gluon/block.rst#mxnet.gluon.Block.collect_params) method to get parameters of the network.\n\n### 4. Training\n\nFinally, we implement the training algorithm. Note that the results for each run\nmay vary because the weights will get different initialization values and the\ndata will be read in a different order due to shuffling.\n\n**PyTorch:**\n\n```{.python .input}\nimport time\n\nfor epoch in range(5):\n    total_loss = .0\n    tic = time.time()\n    for X, y in pt_train_data:\n        pt_trainer.zero_grad()\n        loss = pt_loss_fn(pt_net(X.view(-1, 28*28)), y)\n        loss.backward()\n        pt_trainer.step()\n        total_loss += loss.mean()\n    print('epoch %d, avg loss %.4f, time %.2f' % (\n        epoch, total_loss/len(pt_train_data), time.time()-tic))\n```\n\n**MXNet:**\n\n```{.python .input}\nfrom mxnet import autograd\n\nfor epoch in range(5):\n    total_loss = .0\n    tic = time.time()\n    for X, y in mx_train_data:\n        with autograd.record():\n            loss = mx_loss_fn(mx_net(X), y)\n        loss.backward()\n        mx_trainer.step(batch_size=128)\n        total_loss += loss.mean().item()\n    print('epoch %d, avg loss %.4f, time %.2f' % (\n        epoch, total_loss/len(mx_train_data), time.time()-tic))\n```\n\nSome of the differences in Apache MXNet when compared to PyTorch are as follows:\n\n* In Apache MXNet, you don't need to flatten the 4-D input into 2-D when feeding the data into forward pass.\n\n* In Apache MXNet, you need to perform the calculation within the [autograd.record()](../../../api/autograd/index.rst#mxnet.autograd.record) scope so that it can be automatically differentiated in the backward pass.\n\n* It is not necessary to clear the gradient every time as with PyTorch's `trainer.zero_grad()` because by default the new gradient is written in, not accumulated.\n\n* You need to specify the update step size (usually batch size) when performing [step()](../../../api/gluon/trainer.rst#mxnet.gluon.Trainer.step) on the trainer.\n\n* You need to call [.item()](../../../api/np/arrays.ndarray.rst#the-n-dimensional-array-ndarray) to turn a multidimensional array into a scalar.\n\n* In this sample, Apache MXNet is twice as fast as PyTorch. Though you need to be cautious with such toy comparisons.\n\n## Conclusion\n\nAs we saw above, Apache MXNet Gluon API and PyTorch have many similarities. The main difference lies in terminology (Tensor vs. NDArray) and behavior of accumulating gradients: gradients are accumulated in PyTorch and overwritten in Apache MXNet. The rest of the code is very similar, and it is quite straightforward to move code from one framework to the other.\n\n## Recommended Next Steps\n\nWhile Apache MXNet Gluon API is very similar to PyTorch, there are some extra functionality that can make your code even faster.\n\n* Check out [Hybridize tutorial](../../packages/gluon/blocks/hybridize.ipynb) to learn how to write imperative code which can be converted to symbolic one.\n\n* Also, check out how to extend Apache MXNet with your own [custom layers](../../packages/gluon/blocks/custom-layer.ipynb).\n\n## Appendix\n\nBelow you can find a detailed comparison of various PyTorch functions and their equivalent in Gluon API of Apache MXNet.\n\n### Tensor operation\n\nHere is the list of function names in PyTorch Tensor that are different from Apache MXNet NDArray.\n\n| Function                      | PyTorch                                   | MXNet Gluon                                               |\n|-------------------------------|-------------------------------------------|-----------------------------------------------------------|\n| Element-wise inverse cosine   | `x.acos()` or `torch.acos(x)`             | `nd.arccos(x)`                                            |\n| Batch Matrix product and accumulation| `torch.addbmm(M, batch1, batch2)`  | `nd.linalg_gemm(M, batch1, batch2)` Leading n-2 dim are reduced |\n| Element-wise division of t1, t2, multiply v, and add t | `torch.addcdiv(t, v, t1, t2)` | `t + v*(t1/t2)`                              |\n| Matrix product and accumulation| `torch.addmm(M, mat1, mat2)`             | `nd.linalg_gemm(M, mat1, mat2)`                           |\n| Outer-product of two vector add a matrix | `m.addr(vec1, vec2)`           | Not available                                             |\n| Element-wise applies function | `x.apply_(calllable)`                     | Not available, but there is `nd.custom(x, 'op')`          |\n| Element-wise inverse sine     | `x.asin()` or `torch.asin(x)`             | `nd.arcsin(x)`                                            |\n| Element-wise inverse tangent  | `x.atan()` or `torch.atan(x)`             | `nd.arctan(x)`                                            |\n| Tangent of two tensor         | `x.atan2(y)` or `torch.atan2(x, y)`       | Not available                                             |\n| batch matrix product          | `x.bmm(y)` or `torch.bmm(x, x)`           | `nd.linalg_gemm2(x, y)`                                   |\n| Draws a sample from bernoulli distribution | `x.bernoulli()`              | Not available                                             |\n| Fills a tensor with number drawn from Cauchy distribution | `x.cauchy_()` | Not available                                             |\n| Splits a tensor in a given dim| `x.chunk(num_of_chunk)`                   | `nd.split(x, num_outputs=num_of_chunk)`                   |\n| Limits the values of a tensor to between min and max | `x.clamp(min, max)`| `nd.clip(x, min, max)`                                    |\n| Returns a copy of the tensor  | `x.clone()`                               | `x.copy()`                                                |\n| Cross product                 | `x.cross(y)`                              | Not available                                             |\n| Cumulative product along an axis| `x.cumprod(1)`                          | Not available                                             |\n| Cumulative sum along an axis  | `x.cumsum(1)`                             | Not available                                             |\n| Address of the first element  | `x.data_ptr()`                            | Not available                                             |\n| Creates a diagonal tensor     | `x.diag()`                                | Not available                                             |\n| Computes norm of a tensor     | `x.dist()`                                | `nd.norm(x)` Only calculate L2 norm                       |\n| Computes Gauss error function | `x.erf()`                                 | Not available                                             |\n| Broadcasts/Expands tensor to new shape | `x.expand(3,4)`                  | `x.broadcast_to([3, 4])`                                  |\n| Fills a tensor with samples drawn from exponential distribution | `x.exponential_()` | `nd.random_exponential()`                      |\n| Element-wise mod              | `x.fmod(3)`                               | `nd.module(x, 3)`                                         |\n| Fractional portion of a tensor| `x.frac()`                                | `x - nd.trunc(x)`                                         |\n| Gathers values along an axis specified by dim | `torch.gather(x, 1,  torch.LongTensor([[0,0],[1,0]]))` | `nd.gather_nd(x, nd.array([[[0,0],[1,1]],[[0,0],[1,0]]]))`  |\n| Solves least square & least norm | `B.gels(A)`                            | Not available                                             |\n| Draws from geometirc distribution | `x.geometric_(p)`                     | Not available                                             |\n| Device context of a tensor    | `print(x)` will print which device x is on| `x.context`                                               |\n| Repeats tensor                | `x.repeat(4,2)`                           | `x.tile(4,2)`                                             |\n| Data type of a tensor         | `x.type()`                                | `x.dtype`                                                 |\n| Scatter                       | `torch.zeros(2, 4).scatter_(1, torch.LongTensor([[2], [3]]), 1.23)` | `nd.scatter_nd(nd.array([1.23,1.23]), nd.array([[0,1],[2,3]]), (2,4))` |\n| Returns the shape of a tensor | `x.size()`                                | `x.shape`                                                 |\n| Number of elements in a tensor| `x.numel()`                               | `x.size`                                                  |\n| Returns this tensor as a NumPy ndarray | `x.numpy()`                      | `x.asnumpy()`                                             |\n| Eigendecomposition for symmetric matrix | `e, v = a.symeig()`             | `v, e = nd.linalg.syevd(a)`                               |\n| Transpose                     | `x.t()`                                   | `x.T`                                                     |\n| Sample uniformly              | `torch.uniform_()`                        | `nd.sample_uniform()`                                     |\n| Inserts a new dimesion        | `x.unsqueeze()`                           | `nd.expand_dims(x)`                                       |\n| Reshape                       | `x.view(16)`                              | `x.reshape((16,))`                                          |\n| Veiw as a specified tensor    | `x.view_as(y)`                            | `x.reshape_like(y)`                                       |\n| Returns a copy of the tensor after casting to a specified type | `x.type(type)` | `x.astype(dtype)`                                   |\n| Copies the value of one tensor to another | `dst.copy_(src)`              | `src.copyto(dst)`                                         |\n| Returns a zero tensor with specified shape | `x = torch.zeros(2,3)`       | `x = nd.zeros((2,3))`                                     |\n| Returns a one tensor with specified shape | `x = torch.ones(2,3)`         | `x = nd.ones((2,3)`                                       |\n| Returns a Tensor filled with the scalar value 1, with the same size as input | `y = torch.ones_like(x)` | `y = nd.ones_like(x)`       |\n\n### Functional\n\n### GPU\n\nJust like Tensor, MXNet NDArray can be copied to and operated on GPU. This is done by specifying context.\n\n| Function               | PyTorch                           | MXNet Gluon                                                                |\n|------------------------|-----------------------------------|----------------------------------------------------------------------------|\n| Copy to GPU            | `y = torch.FloatTensor(1).cuda()` | `y = mx.nd.ones((1,), ctx=mx.gpu(0))`                                      |\n| Convert to numpy array | `x = y.cpu().numpy()`             | `x = y.asnumpy()`                                                          |\n| Context scope          | `with torch.cuda.device(1):`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`y= torch.cuda.FloatTensor(1)`                    | `with mx.gpu(1):`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`y = mx.nd.ones((3,5))`      |\n\n### Cross-device\n\nJust like Tensor, MXNet NDArray can be copied across multiple GPUs.\n\n| Function               | PyTorch                           | MXNet Gluon                                                                |\n|------------------------|-----------------------------------|----------------------------------------------------------------------------|\n| Copy from GPU 0 to GPU 1           | `x = torch.cuda.FloatTensor(1)`<br/>`y=x.cuda(1)`| `x = mx.nd.ones((1,), ctx=mx.gpu(0))`<br/>`y=x.as_in_context(mx.gpu(1))`                                      |\n| Copy Tensor/NDArray on different GPUs | `y.copy_(x)`             | `x.copyto(y)`                                                          |\n\n## Autograd\n\n### Variable wrapper vs autograd scope\n\nAutograd package of PyTorch/MXNet enables automatic differentiation of Tensor/NDArray.\n\n| Function               | PyTorch                           | MXNet Gluon                                                                |\n|------------------------|-----------------------------------|----------------------------------------------------------------------------|\n| Recording computation       | `x = Variable(torch.FloatTensor(1), requires_grad=True)`<br/>`y = x * 2`<br/>`y.backward()`  | `x = mx.nd.ones((1,))`<br/>`x.attach_grad()`<br/>`with mx.autograd.record():`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`y = x * 2`<br/>`y.backward()`                                   |\n\n### Scope override (pause, train_mode, predict_mode)\n\nSome operators (Dropout, BatchNorm, etc) behave differently in training and making predictions. This can be controlled with `train_mode` and `predict_mode` scope in MXNet.\nPause scope is for code that does not need gradients to be calculated.\n\n| Function               | PyTorch                           | MXNet Gluon                                                                |\n|------------------------|-----------------------------------|----------------------------------------------------------------------------|\n| Scope override   | Not available | `x = mx.nd.ones((1,))`<br/>`with autograd.train_mode():`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`y = mx.nd.Dropout(x)`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`with autograd.predict_mode():`<br/>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;`z = mx.nd.Dropout(y)`<br/><br/>`w = mx.nd.ones((1,))`<br/>`w.attach_grad()`<br/>`with autograd.record():`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`y = x * w`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`y.backward()`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`with autograd.pause():`<br/>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;`w += w.grad`   |\n\n### Batch-end synchronization is needed\n\nApache MXNet uses lazy evaluation to achieve superior performance. The Python thread just pushes the operations into the backend engine and then returns. In training phase batch-end synchronization is needed, e.g, `asnumpy()`, `wait_to_read()`, `metric.update(...)`.\n\n| Function               | PyTorch                           | MXNet Gluon                                                                |\n|------------------------|-----------------------------------|----------------------------------------------------------------------------|\n| Batch-end synchronization    |  Not available  | `for (data, label) in train_data:`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`with autograd.record():`<br/>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;`output = net(data)`<br/>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;`L = loss(output, label)`<br/>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;`L.backward()`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`trainer.step(data.shape[0])`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`metric.update([label], [output])` |\n\n## PyTorch module and Gluon blocks\n\n### For new block definition, gluon is similar to PyTorch\n\n\n| Function               | PyTorch                           | MXNet Gluon                                                                |\n|------------------------|-----------------------------------|----------------------------------------------------------------------------|\n| New block definition   | `class Net(torch.nn.Module):`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`def __init__(self, D_in, D_out):`<br/>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;`super(Net, self).__init__()`<br/>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;`self.linear = torch.nn.Linear(D_in, D_out)`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`def forward(self, x):`<br/>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;`return self.linear(x)`       |    `class Net(mx.gluon.Block):`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`def __init__(self, D_in, D_out):`<br/>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;`super(Net, self).__init__()`<br/>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;`self.dense=mx.gluon.nn.Dense(D_out, in_units=D_in)`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`def forward(self, x):`<br/>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;`return self.dense(x)`      |\n\n### Parameter and Initializer\n\nWhen creating new layers in PyTorch, you do not need to specify its parameter initializer, and different layers have different default initializer. When you create new layers in Gluon API, you can specify its initializer or just leave it none. The parameters will finish initializing after calling `net.initialize(<init method>)` and all parameters will be initialized in `init method` except those layers whose initializer specified.\n\n| Function       | PyTorch           | MXNet Gluon        |\n|----------------|-------------------|--------------------|\n| Get all parameters |  `net.parameters()` | `net.collect_params()` |\n| Initialize network |  Not Available | `net.initialize(mx.init.Xavier())` |\n| Specify layer initializer | `layer = torch.nn.Linear(20, 10)`<br/> `torch.nn.init.normal(layer.weight, 0, 0.01)` | `layer = mx.gluon.nn.Dense(10, weight_initializer=mx.init.Normal(0.01))` |\n\n### Usage of existing blocks look alike\n\n| Function               | PyTorch                           | MXNet Gluon                                                                |\n|------------------------|-----------------------------------|----------------------------------------------------------------------------|\n| Usage of existing blocks    |  `y=net(x)`  |  `y=net(x)`   |\n\n### HybridBlock can be hybridized, and allows partial-shape info\n\nHybridBlock supports forwarding with both Symbol and NDArray. After hybridized, HybridBlock will create a symbolic graph representing the forward computation and cache it. Most of the built-in blocks (Dense, Conv2D, MaxPool2D, BatchNorm, etc.) are HybridBlocks.\n\nInstead of explicitly declaring the number of inputs to a layer, we can simply state the number of outputs. The shape will be inferred on the fly once the network is provided with some input.\n\n| Function               | PyTorch                           | MXNet Gluon                                                                |\n|------------------------|-----------------------------------|----------------------------------------------------------------------------|\n| partial-shape  <br/> hybridized    |  Not Available   |  `net = mx.gluon.nn.HybridSequential()`<br/>`net.add(mx.gluon.nn.Dense(10))`<br/>`net.hybridize()`   |\n\n### SymbolBlock\n\nSymbolBlock can construct block from symbol. This is useful for using pre-trained models as feature extractors.\n\n| Function               | PyTorch                           | MXNet Gluon                                                                |\n|------------------------|-----------------------------------|----------------------------------------------------------------------------|\n|  SymbolBlock    |  Not Available   |  `alexnet = mx.gluon.model_zoo.vision.alexnet(pretrained=True)`<br/>`out = alexnet(inputs)`<br/>`internals = out.get_internals()`<br/>`outputs = [internals['model_dense0_relu_fwd_output']]`<br/>`feat_model = gluon.SymbolBlock(outputs, inputs, params=alexnet.collect_params())`   |\n\n## PyTorch optimizer vs Gluon Trainer\n### For Gluon API calling zero_grad is not necessary most of the time\n`zero_grad` in optimizer (PyTorch) or Trainer (Gluon API) clears the gradients of all parameters. In Gluon API, there is no need to clear the gradients every batch if `grad_req = 'write'`(default).\n\n| Function               | PyTorch                           | MXNet Gluon                              |\n|------------------------|-----------------------------------|------------------------------------------|\n| clear the gradients |   `optm = torch.optim.SGD(model.parameters(), lr=0.1)`<br/>`optm.zero_grad()`<br/>`loss_fn(model(input), target).backward()`<br/>`optm.step()`    | `trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1})`<br/>`with autograd.record():`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`loss = loss_fn(net(data), label)`<br/>`loss.backward()`<br/>`trainer.step(batch_size)`      |\n\n### Multi-GPU training\n\n| Function               | PyTorch                           | MXNet Gluon                              |\n|------------------------|-----------------------------------|------------------------------------------|\n| data parallelism |   `net = torch.nn.DataParallel(model, device_ids=[0, 1, 2])`<br/>`output = net(data)`    | `ctx = [mx.gpu(i) for i in range(3)]`<br/>`data = gluon.utils.split_and_load(data, ctx)`<br/>`label = gluon.utils.split_and_load(label, ctx)`<br/>`with autograd.record():`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`losses = [loss(net(X), Y) for X, Y in zip(data, label)]`<br/>`for l in losses:`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`l.backward()`      |\n\n### Distributed training\n\n| Function               | Pytorch                           | MXNet Gluon                              |\n|------------------------|-----------------------------------|------------------------------------------|\n| distributed data parallelism |   `torch.distributed.init_process_group(...)`<br/>`model = torch.nn.parallel.distributedDataParallel(model, ...)`    | `store = kv.create('dist')`<br/>`trainer = gluon.Trainer(net.collect_params(), ..., kvstore=store)`  |\n\n## Monitoring\n\n### Apache MXNet has pre-defined metrics\n\nGluon provide several predefined metrics which can online evaluate the performance of a learned model.\n\n| Function               | PyTorch                           | MXNet Gluon                              |\n|------------------------|-----------------------------------|------------------------------------------|\n| metric |  Not available   | `metric = mx.metric.Accuracy()`<br/>`with autograd.record():`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`output = net(data)`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`L = loss(ouput, label)`<br/>&nbsp;&nbsp;&nbsp;&nbsp;`loss(ouput, label).backward()`<br/>`trainer.step(batch_size)`<br/>`metric.update(label, output)`  |\n\n### Data visualization\n\nTensorboardX (PyTorch) and [MXBoard](https://github.com/awslabs/mxboard) (MXNet) can be used to visualize your network and plot quantitative metrics about the execution of your graph.\n\n| PyTorch                                        | MXNet                                          |\n| ---------------------------------------------- | ---------------------------------------------- |\n| `sw = tensorboardX.SummaryWriter()`            | `sw = mxboard.SummaryWriter()`                 |\n| `...`                                          | `...`                                          |\n| `for name, param in model.named_parameters():` | `for name, param in net.collect_params():`     |\n| `    grad = param.clone().cpu().data.numpy()`  | `    grad = param.grad.asnumpy().flatten()`    |\n| `    sw.add_histogram(name, grad, n_iter)`     | `    sw.add_histogram(tag=str(param),`         |\n| `...`                                          | `       values=grad,`                          |\n| `sw.close()`                                   | `       bins=200,`                             |\n|                                                | `       global_step=i)`                        |\n|                                                | `...`                                          |\n|                                                | `sw.close()`                                   |\n\n## I/O and deploy\n\n### Data loading\n\n`Dataset` and `DataLoader` are the basic components for loading data.\n\n| Class               | PyTorch                           | MXNet Gluon                              |\n|------------------------|-----------------------------------|------------------------------------------|\n| Dataset holding arrays | `torch.utils.data.TensorDataset(data_tensor, label_tensor)`| `gluon.data.ArrayDataset(data_array, label_array)`                        |\n| Data loader | `torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False, sampler=None, batch_sampler=None, num_workers=0, collate_fn=<function default_collate>, drop_last=False)` | `gluon.data.DataLoader(dataset, batch_size=None, shuffle=False, sampler=None, last_batch='keep', batch_sampler=None, batchify_fn=None, num_workers=0)`|\n| Sequentially applied sampler | `torch.utils.data.sampler.SequentialSampler(data_source)` | `gluon.data.SequentialSampler(length)` |\n| Random order sampler | `torch.utils.data.sampler.RandomSampler(data_source)` | `gluon.data.RandomSampler(length)`|\n\nSome commonly used datasets for computer vision are provided in `mx.gluon.data.vision` package.\n\n| Class               | PyTorch                           | MXNet Gluon                              |\n|------------------------|-----------------------------------|------------------------------------------|\n| MNIST handwritten digits dataset. | `torchvision.datasets.MNIST`| `mx.gluon.data.vision.MNIST` |\n| CIFAR10 Dataset. | `torchvision.datasets.CIFAR10` | `mx.gluon.data.vision.CIFAR10`|\n| CIFAR100 Dataset. | `torchvision.datasets.CIFAR100` | `mx.gluon.data.vision.CIFAR100` |\n| A generic data loader where the images are arranged in folders. | `torchvision.datasets.ImageFolder(root, transform=None, target_transform=None, loader=<function default_loader>)` | `mx.gluon.data.vision.ImageFolderDataset(root, flag, transform=None)`|\n\n### Serialization\n\nSerialization and deserialization are achieved by calling `save_parameters` and `load_parameters`.\n\n| Class               | PyTorch                           | MXNet Gluon                              |\n|------------------------|-----------------------------------|------------------------------------------|\n| Save model parameters | `torch.save(the_model.state_dict(), filename)`| `model.save_parameters(filename)`|\n| Load parameters | `the_model.load_state_dict(torch.load(PATH))` | `model.load_parameters(filename, ctx, allow_missing=False, ignore_extra=False)` |\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nPython Tutorials\n================\n\nGetting started\n---------------\n\n.. container:: cards\n\n   .. card::\n      :title: A 60-minute Gluon crash course\n      :link: getting-started/crash-course/index.html\n\n      A quick overview of the core concepts of MXNet using the Gluon API.\n\n   .. card::\n      :title: Moving from other frameworks\n      :link: getting-started/to-mxnet/index.html\n\n      Guides that ease your transition to MXNet from other framework.\n\n\nPackages & Modules\n------------------\n\n.. container:: cards\n\n   .. card::\n      :title: Gluon\n      :link: packages/gluon/index.html\n\n      MXNet's imperative interface for Python. If you're new to MXNet, start here!\n\n   .. card::\n      :title: NP and NPX\n      :link: packages/np/index.html\n\n      This section contains the `mxnet.np` and `mxnet.npx` usage hints.\n\n   .. card::\n      :title: Autograd API\n      :link: /api/python/docs/tutorials/packages/autograd/index.html\n\n      How to use Automatic Differentiation with the Autograd API.\n\n\n\nPerformance\n-----------\n.. container:: cards\n\n   .. card::\n      :title: Improving Performance\n      :link: performance/index.html\n\n      How to get the best performance from MXNet.\n\n   .. card::\n      :title: Profiler\n      :link: performance/backend/profiler.html\n\n      How to profile MXNet models.\n\n   .. card::\n      :title: Compression: int8\n      :link: performance/compression/int8.html\n\n      How to use int8 in your model to boost training speed.\n\n   .. card::\n      :title: oneDNN\n      :link: performance/backend/dnnl/index.html\n\n      How to get the most from your CPU by using oneDNN.\n\n   .. card::\n      :title: TVM\n      :link: performance/backend/tvm.html\n\n      How to use TVM to boost performance.\n\n\nDeployment\n----------\n.. container:: cards\n\n   .. card::\n      :title: MXNet on EC2\n      :link: deploy/run-on-aws/use_ec2.html\n\n      How to deploy MXNet on an Amazon EC2 instance.\n\n   .. card::\n      :title: MXNet on SageMaker\n      :link: deploy/run-on-aws/use_sagemaker.html\n\n      How to run MXNet using Amazon SageMaker.\n\n      ..\n         PLACEHOLDER\n         .. card::\n            :title: Export\n            :link: deploy/export/index.html\n\n            How to export MXNet models.\n\n         .. card::\n            :title: C++\n            :link: deploy/inference/cpp.html\n\n            How to use MXNet models in a C++ environment.\n\n         .. card::\n            :title: Scala and Java\n            :link: deploy/inference/scala.html\n\n            How to use MXNet models in a Scala or Java environment.\n\n         PLACEHOLDER\n      ..\n\n\nCustomization\n-------------\n\nComing Soon (CustomOps and Custom Operators)\n\nNext steps\n----------\n\n- To learn more about using MXNet to implement various deep learning algorithms\n  from scratch, we recommend the `Dive into Deep Learning\n  <https://d2l.ai>`_ book.\n\n- Check out the `API Reference docs <../api/index.html>`_.\n\n.. raw:: html\n\n   <style> h1 {display: none;} </style>\n   <style>.localtoc { display: none; }</style>\n\n\n.. toctree::\n   :hidden:\n   :maxdepth: 3\n\n   getting-started/index\n   packages/index\n   performance/index\n   deploy/index\n   extend/index\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/autograd/index.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Automatic Differentiation\n\n## Why do we need to calculate gradients?\n\n### Short Answer:\n\nGradients are fundamental to the process of training neural networks, and tell us how to change the parameters of the network to improve its performance.\n\n![auto-gradient](/_static/autograd_gradient.png)\n\n### Long Answer:\n\nUnder the hood, neural networks are composed of operators (e.g. sums, products, convolutions, etc) some of which use parameters (e.g. the weights in convolution kernels) for their computation, and it's our job to find the optimal values for these parameters. Gradients lead us to the solution!\n\nGradients tell us how much a given variable increases or decreases when we change a variable it depends on. What we're interested in is the effect of changing a each parameter on the performance of the network. We usually define performance using a loss metric that we try to minimize, i.e. a metric that tells us how bad the predictions of a network are given ground truth. As an example, for regression we might try to minimize the [L2 loss](../../../api/gluon/loss/index.rst#mxnet.gluon.loss.L2Loss) (also known as the Euclidean distance) between our predictions and true values, and for classification we minimize the [cross entropy loss](../../../api/gluon/loss/index.rst#mxnet.gluon.loss.SoftmaxCrossEntropyLoss).\n\nAssuming we've calculated the gradient of each parameter with respect to the loss (details in next section), we can then use an optimizer such as [stochastic gradient descent](https://en.wikipedia.org/wiki/Stochastic_gradient_descent) to shift the parameters slightly in the *opposite direction* of the gradient. See [Optimizers](../../../api/optimizer/index.rst) for more information on these methods. We repeat the process of calculating gradients and updating parameters over and over again, until the parameters of the network start to stabilize and converge to a good solution.\n\n## How do we calculate gradients?\n\n### Short Answer:\n\nWe differentiate. [MXNet Gluon](../gluon/index.ipynb) uses Reverse Mode Automatic Differentiation (`autograd`) to backprogate gradients from the loss metric to the network parameters.\n\n![forward-backward](http://mxnet.incubator.apache.org/api/python/docs/_static/autograd_images/autograd_forward_backward.png)\n\n### Long Answer:\n\nOne option would be to get out our calculus books and work out the gradients by hand. Who wants to do this though? It's time consuming and error prone for starters. Another option is [symbolic differentiation](https://www.cs.utexas.edu/users/novak/asg-symdif.html), which calculates the formulas for each gradient, but this quickly leads to incredibly long formulas as networks get deeper and operators get more complex. We could use finite differencing, and try slight differences on each parameter and see how the loss metric responds, but this is computationally expensive and can have poor [numerical precision](https://en.wikipedia.org/wiki/Finite_difference_coefficient).\n\nWhat's the solution? Use automatic differentiation to backpropagate the gradients from the loss metric back to each of the parameters. With [backpropagation](https://en.wikipedia.org/wiki/Backpropagation), a dynamic programming approach is taken to efficently calculate gradients. Sometimes this is called reverse mode automatic differentiation, and it's very efficient in 'fan-in' situations where many parameters effect a single loss metric. Although forward mode automatic differentiation methods exist, they're suited to 'fan-out' situations where few parameters effect many metrics, which isn't the case for training neural networks.\n\n## How does Automatic Differentiation (`autograd`) work?\n\n### Short Answer:\n\nStage 1. Create a record of the operators used by the network to make predictions and calculate the loss metric. Called the 'forward pass' of training.\nStage 2. Work backwards through this record and evaluate the partial derivatives of each operator, all the way back to the network parameters. Called the 'backward pass' of training.\n\n<p style=\"text-align:center\">\n    <video width=\"600\" controls playsinline autoplay muted loop>\n        <source src=\"/api/python/docs/_static/autograd_images/autograd_graph.mp4\" type=\"video/mp4\">\n    </video>\n</p>\n\n### Long Answer:\n\nAll operators in MXNet have two methods defined: a `forward` method for executing the operator as expected, and a `backward` method that returns the partial derivative (the derivative of the output with respect to the input). On the vary rare occasion you need to implement your own custom operator, you'll define the same two methods.\n\nAutomatic differentiation creates a record of the operators used (i.e. the `forward` method calls) by the network to make predictions and calculate the loss metric. A graph structure is used to record this, capturing the inputs (including their value) and outputs for each operator and how the operators are related. We call this the 'forward pass' of training.\n\nAutomatic differentiation then works backwards through each operator of the graph, calling the `backward` method on each operator to calculate the partial derivative and calculate the gradient of the loss metric with respect to the operator's input (which could be parameters). Usually we work backwards from the loss metric, and hence calculate the gradients of the loss metric, but this can be done from any output. We call this the 'backward pass' of training.\n\n## What are the advantages of Automatic Differentiation (`autograd`)?\n\n### Short Answer:\n\nIt's flexible, automatic and efficient. You can use native Python control flow operators such as `if` conditions and `while` loops and `autograd` will still be able to backpropogate the gradients correctly.\n\n### Long Answer:\n\nA huge benefit of using `autograd` is the flexibility it gives you when defining your network. You can change the operations on every iteration, and `autograd` will still be able to backpropogate the gradients correctly. You'll sometimes hear these called 'dynamic graphs', and are much more complex to implement in frameworks that require static graphs, such as TensorFlow.\n\nAs suggested by the name, `autograd` is automatic and so the complexities of the backpropogation procedure are taken care of for you. All you have to do is tell `autograd` when you're interested in recording gradients, and specify what gradients you're interested in calculating: this will nearly always just be the gradient of the loss metric. And these gradient calculations will be performed efficiently too.\n\n## How do I use `autograd` in MXNet Gluon?\n\nStep one is to import the `autograd` package.\n\n```{.python .input}\nfrom mxnet import autograd\n```\n\nAs a simple example, we'll implement the regression model shown in the diagrams above, and later use `autograd` to automatically calculate the gradient of the loss with respect to each of the weight parameters.\n\n```{.python .input}\nimport mxnet as mx\nfrom mxnet.gluon.nn import HybridSequential, Dense\nfrom mxnet.gluon.loss import L2Loss\n\n\n# Define network\nnet = HybridSequential()\nnet.add(Dense(units=3))\nnet.add(Dense(units=1))\nnet.initialize()\n\n# Define loss\nloss_fn = L2Loss()\n\n# Create dummy data\nx = mx.np.array([[0.3, 0.5]])\ny = mx.np.array([[1.5]])\n```\n\nWe're ready for our first forward pass through the network, and we want `autograd` to record the computational graph so we can calculate gradients. One of the simplest ways to do this is by running the network (and loss) code in the scope of an `autograd.record` context.\n\n```{.python .input}\nwith autograd.record():\n    y_hat = net(x)\n    loss = loss_fn(y_hat, y)\n```\n\nOnly operations that we want recorded are in the scope of the `autograd.record` context (since there is a computational overhead), and `autograd` should now have constructed a graph of these operations ready for the backward pass. We start the backward pass by calling the `backward` method on the quantity of interest, which in this case is `loss` since were trying to calculate the gradient of the loss with respect to the parameters.\n\nRemember: if `loss` isn't a single scalar value (e.g. could be a loss for each sample, rather than for whole batch) a `sum` operation will be applied implicitly before starting the backward propagation, and the gradients calculated will be of this `sum` with respect to the parameters.\n\n```{.python .input}\nloss.backward()\n```\n\nAnd that's it! All the `autograd` magic is complete. We should now have gradients for each parameter of the network, which will be used by the optimizer to update the parameter values for improved performance. Check out the gradients of the first layer for example:\n\n```{.python .input}\nnet[0].weight.grad()\n```\n\n## Advanced: Switching between training vs inference modes\n\nSome neural network layers behave differently depending on whether you're training the network or running it for inference. One example is `Dropout`, where activations are set to 0 at random during training, but remain unchanged during inference. Another is `BatchNorm`, where local batch statistics are used to normalize while training, but global statistics are used during inference.\n\nWith MXNet Gluon, `autograd` is critical for switching between training and inference modes. As the default, networks will run in inference mode. While `autograd` is recording though, networks will run in training mode. Operations under the `autograd.record()` context scope are an example of this.\n\nCreating a network of a single `Dropout` block will demonstrate this.\n\n```{.python .input}\ndropout = mx.gluon.nn.Dropout(rate=0.5)\ndata = mx.np.ones(shape=(3,3))\n\noutput = dropout(data)\nis_training = autograd.is_training()\nprint('is_training:', is_training, output)\n```\n\nWe called `dropout` when `autograd` wasn't recording, so our network was in inference mode and thus we didn't see any dropout of the input (i.e. it's still ones). We can confirm the current mode by calling `autograd.is_training()`.\n\n```{.python .input}\nwith autograd.record():\n    output = dropout(data)\nprint('is_training:', is_training, output)\n```\n\nWe called `dropout` while `autograd` was recording this time, so our network was in training mode and we see dropout of the input this time. Since the probability of dropout was 50%, the output is automatically scaled by 1/0.5=2 to preserve the average activation.\n\nWe can force some operators to behave as they would during training, even in inference mode. One example is setting `mode='always'` on the [Dropout](../../../api/legacy/ndarray/ndarray.rst#mxnet.ndarray.Dropout) operator, but this usage is uncommon.\n\n## Advanced: Skipping the calculation of parameter gradients\n\nWhen creating neural networks with MXNet Gluon it is assumed that you're interested in the gradients of the loss with respect to each of the network's parameters. We're usually training the whole network, so this is exactly what we want. When we call `net.initialize()`, the network parameters get (lazily) initalized and memory is also allocated for the gradients, esentially doubling the space required for each parameter. After performing a forward and backward pass through the network, we will have gradients for all of the parameters.\n\nSometimes we don't need the gradients for all of the parameters though. One example would be 'freezing' the values of the parameters in certain layers. Since we don't need to update the values, we don't need the gradients. Using the `grad_req` property of a parameter and setting it to `'null'`, we can indicate this to `autograd`, saving us computation time and memory.\n\n```{.python .input}\nnet[0].weight.grad_req = 'null'\n```\n\n<p style=\"text-align:center\">\n    <video width=\"600\" controls playsinline autoplay muted loop>\n        <source src=\"/api/python/docs/_static/autograd_images/autograd_grad_req.mp4\" type=\"video/mp4\">\n    </video>\n</p>\n\n## Advanced: Calculating non-parameter gradients\n\nAlthough it's most common to deal with network parameters (with `Parameter` being an MXNet Gluon abstraction), there are cases when you need to calculate the gradient with respect to thing that are not parameters (i.e. standard `ndarray`s). One example would be finding the gradient of the loss with respect to the input data to generate adversarial examples.\n\nWith `autograd` it's simple, but there's one key difference compared to parameters: parameters are assumed to require gradients by default, non-parameters are not. We need to explicitly state that we require the gradient, and we do that by calling `.attach_grad()` on the `ndarray`. We can then access the gradient using `.grad` after the `backward` pass.\n\nAs a simple example, let's take the case where $y=2x^2$ and use `autograd` to calculate gradient of $y$ with respect to $x$ at three different values of $x$. We could obviously work out the gradient by hand in this case as $dy/dx=4x$, but let's use this knowledge to check `autograd`. Given $x$ is an `ndarray` and not a `Parameter`, we need to call `x.attach_grad()`.\n\n```{.python .input}\nx = mx.np.array([1, 2, 3])\nx.attach_grad()\nwith autograd.record():\n    y = 2 * x ** 2\ny.backward()\nprint(x.grad)\n```\n\n## Advanced: Using Python control flow\n\nAs mentioned before, one of the main advantages of `autograd` is the ability to automatically calculate gradients of dynamic graphs (i.e. graphs where the operators could be different on every forward pass). One example of this would be applying a tree structured recurrent network to parse a sentence using its parse tree. And we can use Python control flow operators to create a dynamic flow that depends on the data, rather than using MXNet's control flow operators.\n\nWe'll write a function as a toy example of a dynamic network. We'll add an `if` condition and a loop with a variable number of iterations, both of which will depend on the input data. Although these can now be used in static graphs (with conditional operators) it's still much more natural to use native control flow.\n\n```{.python .input}\nimport math\n\n\ndef f(x):\n    y = x  # going to change y but still want to use x\n    if x < 0.75:  # variable num_loops because it depends on x\n        num_loops = math.floor(1/(1-x.item()))\n        for i in range(num_loops):\n            y = y * x  # increase polynomial degree\n    else:  # otherwise flatline\n        y = y * 0\n    return y\n```\n\nWe can plot the resultant function for $x$ between 0 and 1, and we should recognise certain functions in segments of $x$. Starting with a quadratic curve from 0 to 1/2, we have a cubic curve from 1/2 to 2/3, a quartic from 2/3 to 3/4 and finally a flatline.\n\n![control-flow](https://mxnet.incubator.apache.org/api/python/docs/_static/autograd_images/autograd_control_flow.png)\n\nUsing `autograd`, let's now find the gradient of this arbritrary function. We don't have a vectorized function in this case, because of the control flow, so let's also create a function to calculate the gradient using `autograd`.\n\n```{.python .input}\ndef get_grad(f, x):\n    x.attach_grad()\n    with autograd.record():\n        y = f(x)\n    y.backward()\n    return x.grad\n\nxs = mx.np.arange(0.0, 1.0, step=0.1)\ngrads = [get_grad(f, x).item() for x in xs]\nprint(grads)\n```\n\n![flow-grad](https://mxnet.incubator.apache.org/api/python/docs/_static/autograd_images/autograd_control_flow_grad.png)\n\nWe can calculate the gradients by hand in this situation (since it's a toy example), and for the four segments discussed before we'd expect $2x$, $3x^2$, $4x^3$ and 0. As a spot check, for $x=0.6$ the hand calculated gradient would be $3x^2=1.08$, which equals `1.08` as computed by `autograd`.\n\n\n## Advanced: Custom head gradients\n\nMost of the time `autograd` will be aware of the complete computational graph, and be able to calculate the gradients automatically. On a few rare occasions, you might have external post processing components (outside of MXNet Gluon) but still want to compute gradients with respect to MXNet Gluon network parameters.\n\n`autograd` enables this functionality by letting you pass in custom head gradients to `.backward()`. When nothing is specified (for the majority of cases), `autograd` will just used ones by default. Say we're interested in calculating $dz/dx$ but only calculate an intermediate variable $y$ using MXNet Gluon. We need to first calculate the head gradient $dz/dy$ (manually or otherwise), and then pass this to `.backward()`. `autograd` will then use this to calculate $dz/dx$, applying the chain rule.\n\n<p style=\"text-align:center\">\n    <video width=\"600\" controls playsinline autoplay muted loop>\n        <source src=\"/api/python/docs/_static/autograd_images/autograd_head_grad.mp4\" type=\"video/mp4\">\n    </video>\n</p>\n\nAs an example, let's take $y=x^3$ (calculated with `mxnet`) and $z=y^2$. (calculated with `numpy`). We can manually calculate $dz/dy=2y$ (once again with `numpy`), and use this as the head gradient for `autograd` to automatically calculate $dz/dx$. Applying the chain rule by hand we could calculate $dz/dx=6x^5$, so for $x=2$ we expect $dz/dx=192$. Let's check to see whether `autograd` calculates the same.\n\n```{.python .input}\nx = mx.np.array([2,])\nx.attach_grad()\n# compute y inside of mxnet (with `autograd`)\nwith autograd.record():\n    y = x**3\n# compute dz/dy outside of mxnet\ny_np = y.asnumpy()\nz_np = y_np**2\ndzdy_np = 2*y_np\n# compute dz/dx inside of mxnet (given dz/dy)\ndzdy = mx.np.array(dzdy_np)\ny.backward(dzdy)\nprint(x.grad)\n```\n\nAnd as expected, we get a gradient of 192 for `x`.\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/blocks/activations/activations.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Activation Blocks\n\nDeep neural networks are a way to express a nonlinear function with lots of parameters from input data to outputs. The nonlinearities that allow neural networks to capture complex patterns in data are referred to as activation functions. Over the course of the development of neural networks, several nonlinear activation functions have been introduced to make gradient-based deep learning tractable.\n\nIf you are looking to answer the question, 'which activation function should I use for my neural network model?', you should probably go with *ReLU*. Unless you're trying to implement something like a gating mechanism, like in LSTMs or GRU cells, then you should opt for sigmoid and/or tanh in those cells. However, if you have a working model architecture and you're trying to improve its performance by swapping out activation functions or treating the activation function as a hyperparameter, then you may want to try hand-designed activations like SELU, SiLU, or GELU. This guide describes these activation functions and others implemented in MXNet in detail.\n\n## Visualizing Activations\nIn order to compare the various activation functions and to understand the nuances of their differences we have a snippet of code to plot the activation functions (used in the forward pass) and their gradients (used in the backward pass).\n\n\n```{.python .input}\nimport numpy as np\nimport mxnet as mx\nfrom matplotlib import pyplot as plt\n%matplotlib inline\n\ndef visualize_activation(activation_fn):\n    data = np.linspace(-10, 10, 501)\n    x = mx.np.array(data)\n    x.attach_grad()\n    with mx.autograd.record():\n        y = activation_fn(x)\n    y.backward()\n\n    plt.figure()\n    plt.plot(data, y.asnumpy())\n    plt.plot(data, x.grad.asnumpy())\n    activation = activation_fn.__class__.__name__[:-1]\n    plt.legend([\"{} activation\".format(activation), \"{} gradient\".format(activation)])\n\n```\n\n## Sigmoids\n\n### Sigmoid\n\nThe sigmoid activation function, also known as the logistic function or logit function, is perhaps the most widely known activation owing to its [long history](http://www.cs.toronto.edu/~hinton/absps/pdp8.pdf) in neural network training and appearance in logistic regression and kernel methods for classification.\n\nThe sigmoid activation is a non-linear function that transforms any real valued input to a value between 0 and 1, giving it a natural probabilistic interpretation. The sigmoid takes the form of the function below.\n\n$$ \\sigma(x) = \\dfrac{1}{1 + e^x} $$ or alternatively\n\n$$ \\sigma(x) = \\dfrac{e^x}{e^x + 1} $$\n\nWarning: the term sigmoid is overloaded and can be used to refer to the class of 's' shaped functions or particularly to the logistic function that we've just described. In MxNet the sigmoid activation specifically refers to logistic function sigmoid.\n\n\n```{.python .input}\nvisualize_activation(mx.gluon.nn.Activation('sigmoid'))\n```\n\n\n![sigmoid activation and gradient](/_static/sigmoid.png)\n\n\nThe sigmoid activation has since fallen out of use as the preferred activation function in designing neural networks due to some of its properties, shown in the plot above, like not being zero-centered and inducing vanishing gradients, that leads to poor performance during neural network training. Vanishing gradients here refers to the tendency of the gradient of the sigmoid function to be nearly zero for most input values.\n\n### tanh\nThe tanh, or hyperbolic tangent, activation function is also an s shaped curve albeit one whose output values range from -1 to 1. It is defined by the mathematical equation:\n\n$$ tanh(x) = \\dfrac{e^x - e^{-x}}{e^x + e^{-x}}$$\n\ntanh addresses the issues of not being zero centered associated with the sigmoid activation function but still retains the vanishing gradient problems due to the gradient being asymptotically zero for values outside a narrow range of inputs.\n\nIn fact, the tanh can be rewritten as,\n\n$$tanh(x) = \\dfrac{e^{2x} - 1}{e^{2x} + 1}$$\n\nwhich shows its direct relation to sigmoid by the following equation:\n\n\n\n$$ tanh(x) = 2\\sigma(2x) - 1$$\n\n\n```{.python .input}\nvisualize_activation(mx.gluon.nn.Activation('tanh'))\n```\n\n\n![tanh activation and gradient](/_static/tanh.png)\n\n\nThe use of tanh as activation functions in place of the logistic function was popularized by the success of the [LeNet architecture](http://yann.lecun.com/exdb/publis/pdf/lecun-01a.pdf) and the [methods paper](http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf) by LeCun et al.\n\n### SoftSign\n\nThe SoftSign activation is an alternative to tanh that is also centered at zero but converges asymptotically to -1 and 1 polynomially instead of exponentially. This means that the SoftSign activation does not saturate as quickly as tanh. As such, there are a greater range of input values for which the softsign assigns an output of strictly between -1 and 1.\n\n$$ softsign(x) = \\dfrac{x}{abs(x) + 1} $$\n\n\n```{.python .input}\nvisualize_activation(mx.gluon.nn.Activation('softsign'))\n```\n\n\n![softsign activation and gradient](/_static/softsign.png)\n\n\nThe softsign function is not a commonly used activation with most neural networks and still suffers from the vanishing gradient problem as seen in the graph above.\n\n## Rectifiers\n\n### ReLU\nReLU, or Rectified Linear Unit is the most common activation function in convolutional neural networks and introduces a simple nonlinearity. When the value of the input into ReLU is positive, then it retains the same value. When the value is negative then it becomes zero. In equation form, the ReLU function is given as:\n\n$$ ReLU(x) = \\mathtt{max}(0, x) $$\n\nReLU was introduced to neural networks in the [paper by Hahnloser et al](https://papers.nips.cc/paper/1793-permitted-and-forbidden-sets-in-symmetric-threshold-linear-networks.pdf) and gained widespread popularity after it was shown in the [paper](https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf) by Alex Krizhevsky et al to perform much better than sigmoid and tanh. This paper also introduced the AlexNet CNN that won the ILSVRC challenge in 2012.\n\nReLU is the most widely used activation due to its simplicity and performance across multiple datasets and although there have been efforts to introduce activation functions, many of them described in this tutorial, that improve on ReLU, they have not gained as much widespread adoption.\n\n\n```{.python .input}\nvisualize_activation(mx.gluon.nn.Activation('relu'))\n```\n\n\n![relu activation and gradient](/_static/relu.png)\n\n\nAs shown above, the ReLU activation mitigates the vanishing gradient problem associated with the sigmoid family of activations, by having a larger (infinite) range of values where its gradient is non-zero. However, one drawback of ReLU as an activation function is a phenomenon referred to as the 'Dying ReLU', where gradient-based parameter updates can happen in such a way that the gradient flowing through a ReLU unit is always zero and the connection is never activated. This can largely be addressed by ensuring that the tuning the learning rate to ensure that it's not set too large when training ReLU networks.\n\n### SoftReLU\n\nSoftReLU also known as SmoothReLU or SoftPlus is a nonlinear activation function that takes the form\n\n$$ SoftReLU(x) = log(1 + e^x)$$\n\nThe SoftReLU can be seen as a smooth version of the ReLU by observing that its derivative is the sigmoid, seen below, which is a smooth version of the gradient of the ReLU shown above.\n\n\n```{.python .input}\nvisualize_activation(mx.gluon.nn.Activation('softrelu'))\n```\n\n\n![softrelu activation and gradient](/_static/softrelu.png)\n\n\n### Leaky ReLU\n\nLeaky ReLUs are a variant of ReLU that multiply the input by a small positive parameter $\\alpha$ when the value is negative. Unlike the ReLU which sets the activation and gradient for negative values to zero, the LeakyReLU allows a small gradient. The equation for the LeakyReLU is:\n\n$$ LeakyReLU(\\alpha, x) = \\begin{cases}\n    x,& \\text{if } x\\geq 0\\\\\n    \\alpha x,              & \\text{otherwise}\n\\end{cases}$$\n\nwhere $\\alpha > 0$ is small positive number. In MXNet, by default the $\\alpha$ parameter is set to 0.01.\n\nHere is a visualization for the LeakyReLU with $\\alpha = 0.05$\n\n\n```{.python .input}\nvisualize_activation(mx.gluon.nn.LeakyReLU(0.05))\n```\n\n\n![leakyrelu activation and gradient](/_static/leakyrelu.png)\n\n\nAs shown in the graph, the LeakyReLU's gradient is non-zero everywhere, in an attempt to address the ReLU's gradient being zero for all negative values.\n\n### PReLU\nThe PReLU activation function, or Parametric Leaky ReLU introduced by [He et al](https://arxiv.org/pdf/1502.01852.pdf), is a version of LeakyReLU that learns the parameter $\\alpha$ during training. An initialization parameter is passed into the PreLU activation layer and this is treated as a learnable parameter that is updated via gradient descent during training. This is in contrast to LeakyReLU where $\\alpha$ is a hyperparameter.\n\n\n```{.python .input}\nprelu = mx.gluon.nn.PReLU(mx.init.Normal(0.05))\nprelu.initialize()\nvisualize_activation(prelu)\n```\n\n\n![prelu activation and gradient](/_static/prelu.png)\n\n\nThe activation function and activation gradient of PReLU have the same shape as LeakyRELU.\n\n### ELU\n\nThe ELU or exponential linear unit introduced by [Clevert et al](https://arxiv.org/abs/1511.07289) also addresses the vanishing gradient problem like ReLU and its variants but unlike the ReLU family, ELU allows negative values which may allow them to push mean unit activations closer to zero like batch normalization.\n\nThe ELU function has the form\n\n$$ ELU(\\alpha, x) = \\begin{cases}\n    x,& \\text{if } x\\geq 0\\\\\n    \\alpha (e^x - 1),              & \\text{otherwise}\n\\end{cases}$$\n\n\n```{.python .input}\nvisualize_activation(mx.gluon.nn.ELU())\n```\n\n\n![elu activation and gradient](/_static/elu.png)\n\n\n### SELU\nSELU stands for Scaled Exponential Linear Unit and was introduced by [Klambuer et al](https://arxiv.org/abs/1706.02515) and is a modification of the ELU that improves the normalization of its outputs towards a zero mean and unit variance.\n\nThe SELU function has the form\n\n$$ SELU(\\alpha, x) = \\lambda \\cdot\\begin{cases}\n    x,& \\text{if } x\\geq 0\\\\\n    \\alpha (e^x - 1),              & \\text{otherwise}\n\\end{cases}$$\n\nIn SELU, unlike ELU, the parameters $\\alpha$ and $\\lambda$ are fixed parameters calculated from the data. For standard scaled inputs, these values are $$\\alpha=1.6732, \\lambda=1.0507$$ as calculated in the paper.\n\n\n```{.python .input}\nvisualize_activation(mx.gluon.nn.SELU())\n```\n\n\n![selu activation and gradient](/_static/selu.png)\n\n\n### SiLU\nThe SiLU is an activation function that attempts to address the shortcomings of ReLU by combining ideas from ReLU and sigmoid. The SiLU serves as a smooth approximation to the ReLU and was originally introduced in [Hendrycks et al](https://arxiv.org/abs/1606.08415).\n\nThe silu function is given as \n\n$$ silu(x) = x\\cdot\\sigma(x)$$\n\nwhere $\\sigma$ is the sigmoid activation function $\\sigma(x) = \\frac{1}{1 + e^{-x}}$ described above.\n\n\n```{.python .input}\nvisualize_activation(mx.gluon.nn.SiLU())\n```\n\n\n![silu activation and gradient](/_static/silu.png)\n\n### GELU\nThe GELU is a smooth approximation to the ReLU and was introduced in [Hendrycks et al](https://arxiv.org/abs/1606.08415). It is a common activation function in architectures such as Transformers, BERT, and GPT.\n\nThe gelu function is given as \n\n$$ gelu(x) = x\\cdot\\Phi(x),$$\n\nwhereas the ReLU can be written as $x\\cdot\\mathbf{1}(x>0)$, so $Phi(x)$ serves as a smooth approximation to the ReLU's indicator function.\n\nNote $\\Phi(x) = \\frac{1}{\\sqrt{2 \\pi}} \\exp\\left\\{-\\frac{x^2}{2}\\right\\}$ is the standard normal cumulative distribution.\n\n\n```{.python .input}\nvisualize_activation(mx.gluon.nn.GELU())\n```\n\n![gelu activation and gradient](/_static/gelu.png)\n\n## Summary\n\n* Activation functions introduce non-linearities to deep neural network that allow the models to capture complex interactions between features of the data.\n* ReLU is the activation function that is commonly used in many neural network architectures because of its simplicity and performance.\n* Sigmoids like the logistic (sigmoid) function and tanh where the first kinds of activation functions used in neural networks. They have since fallen out of use because of their tendency to saturate and have vanishing gradients.\n* Rectifiers like ReLU do not saturate like the Sigmoids and so address the vanishing gradient problem making them the de facto activation functions. ReLU however is still plagued by the dying ReLU problem.\n* LeakyReLU and PReLU are two similar approaches to improve ReLU and address the dying ReLU by introducing a parameter $\\alpha$ (learned in PReLU) that leaks to the gradient of negative inputs\n* MXNet also implements custom state-of-the-art activations like ELU, SELU, SiLU, and GELU.\n\n\n\n## Next Steps\n\nActivations are just one component of neural network architectures. Here are a few MXNet resources to learn more about activation functions and how they they combine with other components of neural nets.\n* Learn how to create a Neural Network with these activation layers and other neural network layers in the [Gluon crash course](../../../../getting-started/crash-course/index.ipynb).\n* Check out the guide to MXNet [gluon layers and blocks](../nn.ipynb) to learn about the other neural network layers in implemented in MXNet and how to create custom neural networks with these layers.\n* Also check out the [guide to normalization layers](../../training/normalization/index.ipynb) to learn about neural network layers that normalize their inputs.\n* Finally take a look at the [Custom Layer guide](../custom-layer.ipynb) to learn how to implement your own custom activation layer.\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/blocks/custom-layer.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Custom Layers\n\nWhile Gluon API for Apache MxNet comes with [a decent number of pre-defined layers](https://mxnet.apache.org/versions/master/api/python/docs/api/gluon/nn/index.html), at some point one may find that a new layer is needed. Adding a new layer in Gluon API is straightforward, yet there are a few things that one needs to keep in mind.\n\nIn this article, I will cover how to create a new layer from scratch, how to use it, what are possible pitfalls and how to avoid them.\n\n## The simplest custom layer\n\nTo create a new layer in Gluon API, one must create a class that inherits from [Block](https://github.com/apache/incubator-mxnet/blob/c9818480680f84daa6e281a974ab263691302ba8/python/mxnet/gluon/block.py#L128) class. This class provides the most basic functionality, and all pre-defined layers inherit from it directly or via other subclasses. Because each layer in Apache MxNet inherits from `Block`, words \"layer\" and \"block\" are used interchangeable inside of the Apache MxNet community.\n\nThe only instance method needed to be implemented is [forward(self, x)](https://github.com/apache/incubator-mxnet/blob/c9818480680f84daa6e281a974ab263691302ba8/python/mxnet/gluon/block.py#L909), which defines what exactly your layer is going to do during forward propagation. Notice, that it doesn't require to provide what the block should do during back propogation. Back propogation pass for blocks is done by Apache MxNet for you. \n\nIn the example below, we define a new layer and implement `forward()` method to normalize input data by fitting it into a range of [0, 1].\n\n\n```{.python .input}\n# Do some initial imports used throughout this tutorial \nfrom __future__ import print_function\nimport mxnet as mx\nfrom mxnet import np, npx, gluon, autograd\nfrom mxnet.gluon.nn import Dense\nmx.np.random.seed(1)                      # Set seed for reproducable results\n```\n\n\n```{.python .input}\nclass NormalizationLayer(gluon.Block):\n    def __init__(self):\n        super(NormalizationLayer, self).__init__()\n\n    def forward(self, x):\n        return (x - np.min(x)) / (np.max(x) - np.min(x))\n```\n\nThe rest of methods of the `Block` class are already implemented, and majority of them are used to work with parameters of a block. There is one very special method named [hybridize()](https://github.com/apache/incubator-mxnet/blob/master/python/mxnet/gluon/block.py#L384), though, which I am going to cover before moving to a more complex example of a custom layer.\n\n## Hybridization and the difference between Block and HybridBlock\n\nLooking into implementation of [existing layers](https://mxnet.apache.org/versions/master/api/python/docs/api/gluon/nn/index.html), one may find that more often a block inherits from a [HybridBlock](https://github.com/apache/incubator-mxnet/blob/master/python/mxnet/gluon/block.py#L428), instead of directly inheriting from `Block`.\n\nThe reason for that is that `HybridBlock` allows to write custom layers in imperative programming style, while computing in a symbolic way. It unifies the flexibility of imperative programming with the performance benefits of symbolic programming. You can learn more about the difference between symbolic and imperative programming from [this article](https://mxnet.apache.org/api/architecture/overview.html).\n\nHybridization is a process that Apache MxNet uses to create a symbolic graph of a forward computation. This allows to increase computation performance by optimizing the computational symbolic graph. Once the symbolic graph is created, Apache MxNet caches and reuses it for subsequent computations.\n\nHybridization of HybridBlock.forward is based on a deferred computation mode in the MXNet backend, which enables recording computation via tracing in the mxnet.nd and mxnet.np interfaces. The recorded computation can be exported to a symbolic representation and is used for optimized execution with the CachedOp.\n\nAs tracing is based on the imperative APIs, users can access shape information of the arrays. As x.shape for some array x is a python tuple, any use of that shape will be a constant in the recorded graph and may limit the recorded graph to be used with inputs of the same shape only.\n\nKnowing this, we can rewrite our example layer, using HybridBlock:\n\n\n```{.python .input}\nclass NormalizationHybridLayer(gluon.HybridBlock):\n    def __init__(self):\n        super(NormalizationHybridLayer, self).__init__()\n\n    def forward(self, x):\n        return (x - np.min(x)) / (np.max(x) - np.min(x))\n```\n\nThanks to inheriting from HybridBlock, one can easily do forward pass on a given ndarray, either on CPU or GPU:\n\n\n```{.python .input}\nlayer = NormalizationHybridLayer()\nlayer(np.array([1, 2, 3], device=mx.cpu()))\n```\n\nOutput:\n\n```bash\n[0.  0.5 1. ]\n```\n\n\nAs a rule of thumb, one should always implement custom layers by inheriting from `HybridBlock`. This allows to have more flexibility, and doesn't affect execution speed once hybridization is done. \n\nUnfortunately, at the moment of writing this tutorial, NLP related layers such as [RNN](../../../../api/gluon/rnn/index.rst#mxnet.gluon.rnn.RNN), [GRU](../../../../api/gluon/rnn/index.rst#mxnet.gluon.rnn.GRU) and [LSTM](../../../../api/gluon/rnn/index.rst#mxnet.gluon.rnn.LSTM) are directly inhereting from the `Block` class via common `_RNNLayer` class. That means that networks with such layers cannot be hybridized. But this might change in the future, so stay tuned.\n\nIt is important to notice that hybridization has nothing to do with computation on GPU. One can train both hybridized and non-hybridized networks on both CPU and GPU, though hybridized networks would work faster. Though, it is hard to say in advance how much faster it is going to be.\n\n## Adding a custom layer to a network\n\nWhile it is possible, custom layers are rarely used separately. Most often they are used with predefined layers to create a neural network. Output of one layer is used as an input of another layer.\n\nDepending on which class you used as a base one, you can use either [Sequential](../../../../api/gluon/nn/index.rst#mxnet.gluon.nn.Sequential) or [HybridSequential](../../../../api/gluon/nn/index.rst#mxnet.gluon.nn.HybridSequential) container to form a sequential neural network. By adding layers one by one, one adds dependencies of one layer's input from another layer's output. It is worth noting, that both `Sequential` and `HybridSequential` containers inherit from `Block` and `HybridBlock` respectively. \n\nBelow is an example of how to create a simple neural network with a custom layer. In this example, `NormalizationHybridLayer` gets as an input the output from `Dense(5)` layer and pass its output as an input to `Dense(1)` layer.\n\n\n```{.python .input}\nnet = gluon.nn.HybridSequential()                         # Define a Neural Network as a sequence of hybrid blocks\nnet.add(Dense(5))                                     # Add Dense layer with 5 neurons\nnet.add(NormalizationHybridLayer())                   # Add a custom layer\nnet.add(Dense(1))                                     # Add Dense layer with 1 neurons\n\n\nnet.initialize(mx.init.Xavier(magnitude=2.24))            # Initialize parameters of all layers\nnet.hybridize()                                           # Create, optimize and cache computational graph\ninput = np.random.uniform(low=-10, high=10, size=(5, 2))  # Create 5 random examples with 2 feature each in range [-10, 10]\nnet(input)\n```\n\n\nOutput:\n\n```bash\n[[-0.13601446]\n [ 0.26103732]\n [-0.05046433]\n [-1.2375476 ]\n [-0.15506986]]\n```\n\n\n## Parameters of a custom layer\n\nUsually, a layer has a set of associated parameters, sometimes also referred as weights. This is an internal state of a layer. Most often, these parameters are the ones, that we want to learn during backpropogation step, but sometimes these parameters might be just constants we want to use during forward pass. The parameters are usually represented as [Parameter](../../../../api/gluon/parameter.rst#gluon-parameter) class inside of Apache MXNet neural network.\n\n\n```{.python .input}\nclass NormalizationHybridLayer(gluon.HybridBlock):\n    def __init__(self, hidden_units, scales):\n        super(NormalizationHybridLayer, self).__init__()\n        self.hidden_units = hidden_units\n        self.weights = gluon.Parameter('weights',\n                                       shape=(hidden_units, -1),\n                                       allow_deferred_init=True)\n\n        self.scales = gluon.Parameter('scales',\n                                      shape=scales.shape,\n                                      init=mx.init.Constant(scales), # Convert to regular list to make this object serializable\n                                      differentiable=False)\n            \n    def forward(self, x):\n        normalized_data = (x - np.min(x)) / (np.max(x) - np.min(x))\n        weighted_data = npx.fully_connected(normalized_data, self.weights.data(), num_hidden=self.hidden_units, no_bias=True)\n        scaled_data = np.multiply(self.scales.data(), weighted_data)\n        return scaled_data\n    \n    def infer_shape(self, x, *args):\n        self.weights.shape = (self.hidden_units, x.shape[x.ndim-1])\n```\n\nIn the example above 2 set of parameters are defined:\n1. Parameter `weights` is trainable. Its shape is unknown during construction phase and will be infered on the first run of forward propogation; \n1. Parameter `scale` is a constant that doesn't change. Its shape is defined during construction.\n\nNotice a few aspects of this code:\n* Shape is not provided when creating `weights`. Instead it is going to be infered from the shape of the input by `infer_shape` method.\n* `Scales` parameter is initialized and marked as `differentiable=False`.\n\nRunning forward pass on this network is very similar to the previous example, so instead of just doing one forward pass, let's run whole training for a few epochs to show that `scales` parameter doesn't change during the training while `weights` parameter is changing.\n\n\n```{.python .input}\ndef print_params(title, net):\n    \"\"\"\n    Helper function to print out the state of parameters of NormalizationHybridLayer\n    \"\"\"\n    print(title)\n    hybridlayer_params = {k: v for k, v in net.collect_params().items()}\n    \n    for key, value in hybridlayer_params.items():\n        print('{} = {}\\n'.format(key, value.data()))\n\nnet = gluon.nn.HybridSequential()                             # Define a Neural Network as a sequence of hybrid blocks\nnet.add(Dense(5))                                         # Add Dense layer with 5 neurons\nnet.add(NormalizationHybridLayer(hidden_units=5, \n                                 scales = np.array([2]))) # Add a custom layer\nnet.add(Dense(1))                                         # Add Dense layer with 1 neurons\n\n\nnet.initialize(mx.init.Xavier(magnitude=2.24))                # Initialize parameters of all layers\nnet.hybridize()                                               # Create, optimize and cache computational graph\n\ninput = np.random.uniform(low=-10, high=10, size=(5, 2))      # Create 5 random examples with 2 feature each in range [-10, 10]\nlabel = np.random.uniform(low=-1, high=1, size=(5, 1))\n\nmse_loss = gluon.loss.L2Loss()                                # Mean squared error between output and label\ntrainer = gluon.Trainer(net.collect_params(),                 # Init trainer with Stochastic Gradient Descent (sgd) optimization method and parameters for it\n                        'sgd', \n                        {'learning_rate': 0.1, 'momentum': 0.9 })\n                        \nwith autograd.record():                                       # Autograd records computations done on NDArrays inside \"with\" block \n    output = net(input)                                       # Run forward propogation\n    \n    print_params(\"=========== Parameters after forward pass ===========\\n\", net)    \n    loss = mse_loss(output, label)                            # Calculate MSE\n    \nloss.backward()                                               # Backward computes gradients and stores them as a separate array within each NDArray in .grad field\ntrainer.step(input.shape[0])                                  # Trainer updates parameters of every block, using .grad field using oprimization method (sgd in this example)\n                                                              # We provide batch size that is used as a divider in cost function formula\nprint_params(\"=========== Parameters after backward pass ===========\\n\", net)\n```\n\nOutput:\n\n```bash\n=========== Parameters after forward pass ===========\n\nhybridsequential94_normalizationhybridlayer0_weights = \n[[-0.3983642  -0.505708   -0.02425683 -0.3133553  -0.35161012]\n [ 0.6467543   0.3918715  -0.6154656  -0.20702496 -0.4243446 ]\n [ 0.6077331   0.03922009  0.13425875  0.5729856  -0.14446527]\n [-0.3572498   0.18545026 -0.09098256  0.5106366  -0.35151464]\n [-0.39846328  0.22245121  0.13075739  0.33387476 -0.10088372]]\n\nhybridsequential94_normalizationhybridlayer0_scales = \n[2.]\n\n=========== Parameters after backward pass ===========\n\nhybridsequential94_normalizationhybridlayer0_weights = \n[[-0.29839832 -0.47213346  0.08348035 -0.2324698  -0.27368504]\n [ 0.76268613  0.43080837 -0.49052125 -0.11322092 -0.3339738 ]\n [ 0.48665082 -0.00144657  0.00376363  0.47501418 -0.23885089]\n [-0.22626656  0.22944227  0.05018325  0.6166192  -0.24941102]\n [-0.44946212  0.20532274  0.07579394  0.29261002 -0.14063817]]\n\nhybridsequential94_normalizationhybridlayer0_scales = \n[2.]\n``` \n\n\nAs it is seen from the output above, `weights` parameter has been changed by the training and `scales` not.\n\n## Conclusion\n\nOne important quality of a Deep learning framework is extensibility. Empowered by flexible abstractions, like `Block` and `HybridBlock`, one can easily extend Apache MxNet functionality to match its needs.\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/blocks/hybridize.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Hybridize\n\n<!-- adapted from diveintodeeplearning -->\n\n## A Hybrid of Imperative and Symbolic Programming\n\nImperative programming makes use of\nprogramming statements to change a program’s state. Consider the following\nexample of simple imperative programming code.\n\n```{.python .input}\ndef add(a, b):\n    return a + b\n\ndef fancy_func(a, b, c, d):\n    e = add(a, b)\n    f = add(c, d)\n    g = add(e, f)\n    return g\n\nfancy_func(1, 2, 3, 4)\n```\n\nAs expected, Python will perform an addition when running the statement `e = add(a, b)`, and will store the result as the variable `e`, thereby changing the program’s state. The next two statements `f = add(c, d)` and `g = add(e, f)` will similarly perform additions and store the results as variables.\n\nAlthough imperative programming is convenient, it may be inefficient. On the one hand, even if the `add` function is repeatedly called throughout the `fancy_func` function, Python will execute the three function calling statements individually, one after the other. On the other hand, we need to save the variable values of `e` and `f` until all the statements in `fancy_func` have been executed. This is because we do not know whether the variables `e` and `f` will be used by other parts of the program after the statements `e = add(a, b)` and `f = add(c, d)` have been executed.\n\nContrary to imperative programming, symbolic programming is usually performed after the computational process has been fully defined. Symbolic programming is used by multiple deep learning frameworks, including Theano and TensorFlow. The process of symbolic programming generally requires the following three steps:\n\n1. Define the computation process.\n2. Compile the computation process into an executable program.\n3. Provide the required inputs and call on the compiled program for execution.\n\nIn the example below, we utilize symbolic programming to re-implement the imperative programming code provided at the beginning of this section.\n\n```{.python .input}\ndef add_str():\n    return '''\ndef add(a, b):\n    return a + b\n'''\n\ndef fancy_func_str():\n    return '''\ndef fancy_func(a, b, c, d):\n    e = add(a, b)\n    f = add(c, d)\n    g = add(e, f)\n    return g\n'''\n\ndef evoke_str():\n    return add_str() + fancy_func_str() + '''\nprint(fancy_func(1, 2, 3, 4))\n'''\n\nprog = evoke_str()\nprint(prog)\ny = compile(prog, '', 'exec')\nexec(y)\n```\n\nThe three functions defined above will only return the results of the computation process as a string. Finally, the complete computation process is compiled and run using the `compile` function. This leaves more room to optimize computation, since the system is able to view the entire program during its compilation. For example, during compilation, the program can be rewritten as `print((1 + 2) + (3 + 4))` or even directly rewritten as `print(10)`. Apart from reducing the amount of function calls, this process also saves memory.\n\nA comparison of these two programming methods shows that\n\n- imperative programming is easier. When imperative programming is used in Python, the majority of the code is straightforward and easy to write. At the same time, it is easier to debug imperative programming code. This is because it is easier to obtain and print all relevant intermediate variable values, or make use of Python’s built-in debugging tools.\n\n- Symbolic programming is more efficient and easier to port. Symbolic programming makes it easier to better optimize the system during compilation, while also having the ability to port the program into a format independent of Python. This allows the program to be run in a non-Python environment, thus avoiding any potential performance issues related to the Python interpreter.\n\n## Hybrid programming provides the best of both worlds.\n\nMost deep learning frameworks choose either imperative or symbolic programming. For example, both Theano and TensorFlow (inspired by the latter) make use of symbolic programming, while Chainer and its predecessor PyTorch utilize imperative programming. When designing Gluon, developers considered whether it was possible to harness the benefits of both imperative and symbolic programming. The developers believed that users should be able to develop and debug using pure imperative programming, while having the ability to convert most programs into symbolic programming to be run when product-level computing performance and deployment are required This was achieved by Gluon through the introduction of hybrid programming.\n\nIn hybrid programming, we can build models using either the HybridBlock or the HybridSequential classes. By default, they are executed in the same way Block or Sequential classes are executed in imperative programming. When the `hybridize` function is called, Gluon will convert the program’s execution into the style used in symbolic programming. In fact, most models can make use of hybrid programming’s execution style.\n\nThrough the use of experiments, this section will demonstrate the benefits of hybrid programming.\n\n## Constructing Models Using the HybridSequential Class\n\nPreviously, we learned how to use the Sequential class to concatenate multiple layers. Next, we will replace the Sequential class with the HybridSequential class in order to make use of hybrid programming.\n\n```{.python .input}\nfrom mxnet import np, npx, sym\nfrom mxnet.gluon import nn\nimport time\n\ndef get_net():\n    net = nn.HybridSequential()  # Here we use the class HybridSequential.\n    net.add(nn.Dense(256, activation='relu'),\n            nn.Dense(128, activation='relu'),\n            nn.Dense(2))\n    net.initialize()\n    return net\n\nx = np.random.normal(size=(1, 512))\nnet = get_net()\nnet(x)\n```\n\nBy calling the `hybridize` function, we are able to compile and optimize the computation of the concatenation layer in the HybridSequential instance. The model’s computation result remains unchanged.\n\n```{.python .input}\nnet.hybridize()\nnet(x)\n```\n\nIt should be noted that only the layers inheriting the HybridBlock class will be optimized during computation. For example, the HybridSequential and `Dense` classes provided by Gluon are all subclasses of HybridBlock class, meaning they will both be optimized during computation. A layer will not be optimized if it inherits from the Block class rather than the HybridBlock class.\n\n### Computing Performance\n\nTo demonstrate the performance improvement gained by the use of symbolic programming, we will compare the computation time before and after calling the `hybridize` function. Here we time 1000 `net` model computations. The model computations are based on imperative and symbolic programming, respectively, before and after `net` has called the `hybridize` function.\n\n```{.python .input}\ndef benchmark(net, x):\n    start = time.time()\n    for i in range(1000):\n        _ = net(x)\n    npx.waitall()  # To facilitate timing, we wait for all computations to be completed.\n    return time.time() - start\n\nnet = get_net()\nprint('before hybridizing: %.4f sec' % (benchmark(net, x)))\nnet.hybridize()\nprint('after hybridizing: %.4f sec' % (benchmark(net, x)))\n```\n\nAs is observed in the above results, after a HybridSequential instance calls the `hybridize` function, computing performance is improved through the use of symbolic programming.\n\n### Achieving Symbolic Programming\n\nWe can save the symbolic program and model parameters to the hard disk through the use of the `export` function after the `net` model has finished computing the output based on the input, such as in the case of `net(x)` in the `benchmark` function.\n\n```{.python .input}\nnet.export('my_mlp')\n```\n\nThe .json and .params files generated during this process are a symbolic program and a model parameter, respectively. They can be read by other front-end languages supported by Python or MXNet, such as C++, R, Scala, and Perl. This allows us to deploy trained models to other devices and easily use other front-end programming languages. At the same time, because symbolic programming was used during deployment, the computing performance is often superior to that based on imperative programming.\n\nIn MXNet, a symbolic program refers to a program that makes use of the Symbol type. We know that, when the NDArray input `x` is provided to `net`, `net(x)` will directly calculate the model output and return a result based on `x`. For models that have called the `hybridize` function, we can also provide a Symbol-type input variable, and `net(x)` will return Symbol type results.\n\n```{.python}\nx = sym.var('data')\nnet(x)\n```\n\n## Constructing Models Using the HybridBlock Class\n\nSimilar to the correlation between the Sequential Block classes, the HybridSequential class is a HybridBlock subclass. \n\nEarlier, we demonstrated that, after calling the `hybridize` function, the model is able to achieve superior computing performance and portability. In addition, model flexibility can be affected after calling the `hybridize` function. We will demonstrate this by constructing a model using the HybridBlock class.\n\n```{.python .input}\nclass HybridNet(nn.HybridBlock):\n    def __init__(self, **kwargs):\n        super(HybridNet, self).__init__(**kwargs)\n        self.hidden = nn.Dense(10)\n        self.output = nn.Dense(2)\n\n    def forward(self, x):\n        print('x: ', x)\n        x = npx.relu(self.hidden(x))\n        print('hidden: ', x)\n        return self.output(x)\n```\n\n```{.python .input}\nnet = HybridNet()\nnet.initialize()\nx = np.random.normal(size=(1, 4))\nnet(x)\n```\n\nRepeating the forward computation will achieve the same results.\n\n```{.python .input}\nnet(x)\n```\n\nNext, we will see what happens after we call the `hybridize` function.\n\n```{.python .input}\nnet.hybridize()\nnet(x)\n```\n\nNow, we repeat the forward computation.\n\n```{.python .input}\nnet(x)\n```\n\nWe can see that the three lines of print statements defined in the `forward` function will not print anything. This is because a symbolic computing graph has been recorded since the last time `net(x)` was run by calling the `hybridize` function. Afterwards, when we run `net(x)` again, MXNet will no longer need to access Python code, but can directly perform symbolic programming at the C++ backend. This is another reason why model computing performance will be improve after the `hybridize` function is called. However, there is always the potential that any programs we write will suffer a loss in flexibility. If we want to use the three lines of print statements to debug the code in the above example, they will be skipped over and we would not be able to print when the symbolic program is executed. Additionally, in the case of a few functions not supported by Symbol (like `asnumpy`), and operations in-place like `a += b` and `a[:] = a + b` (must be rewritten as `a = a + b`). Therefore, we will not be able to use the `forward` function or perform forward computation after the `hybridize` function has been called.\n\n## Disabling Hybridization\n\nIf we want to disable the `hybridize` function, we can do that by using the following code:\n\n```{.python .input}\nnet.hybridize(active=False)\n```\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/blocks/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nBlocks\n======\n\n.. toctree::\n   :maxdepth: 1\n   :glob:\n\n   *\n   activations/activations\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/blocks/init.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Initialization\n\n<!-- adapted from diveintodeeplearning -->\n\nIn the [Neural Networks](./nn.ipynb) section we played fast and loose with setting\nup our networks. In particular we did the following things that *shouldn't*\nwork:\n\n* We defined the network architecture with no regard to the input\n  dimensionality.\n* We added layers without regard to the output dimension of the previous layer.\n* We even 'initialized' these parameters without knowing how many parameters\n  we were going to initialize.\n\nAll of those things sound impossible and indeed, they are. After all, there's\nno way MXNet (or any other framework for that matter) could predict what the\ninput dimensionality of a network would be. Later on, when working with\nconvolutional networks and images this problem will become even more pertinent,\nsince the input dimensionality (i.e. the resolution of an image) will affect\nthe dimensionality of subsequent layers. The ability to\ndetermine parameter dimensionality during run-time rather than at coding time\ngreatly simplifies the process of doing deep learning.\n\n## Instantiating a Network\n\nLet's see what happens when we instantiate a network. We start by defining a multi-layer perceptron.\n\n```{.python .input}\nfrom mxnet import init, np\nfrom mxnet.gluon import nn\n\n\ndef getnet():\n    net = nn.Sequential()\n    net.add(nn.Dense(256, activation='relu'))\n    net.add(nn.Dense(10))\n    return net\n\nnet = getnet()\n```\n\nAt this point the network doesn't really know yet what the dimensionalities of\nthe various parameters should be. All one could tell at this point is that each\nlayer needs weights and bias, albeit of unspecified dimensionality. If we try\naccessing the parameters, that's exactly what happens.\n\n```{.python .input}\nprint(net.collect_params())\n```\n\nYou'll notice `None` here in each `Dense` layer. This absence of value is how\nMXNet keeps track of unspecified dimensionality. In particular, trying to access\n`net[0].weight.data()` at this point would trigger a runtime error stating that\nthe network needs initializing before it can do anything.\n\nNote that if we did want to specify dimensionality, we could have done so by\nusing the kwarg `in_units`, e.g. `Dense(256, activiation='relu', in_units=20)`.\n\nLet's see whether anything changes after we initialize the parameters:\n\n\n```{.python .input}\nnet.initialize()\nnet.collect_params()\n```\n\nAs we can see, nothing really changed. Only once we provide the network with\nsome data do we see a difference. Let's try it out.\n\n```{.python .input}\nx = np.random.uniform(size=(2, 20))\nnet(x)  # Forward computation\nprint(net.collect_params())\n```\n\nWe see all the dimensions have been determined and the parameters initialized.\nThis is because shape inference and parameter initialization have been\nperformed in a lazy manner, so they are performed only when needed. In the\nabove case, they are performed as a prerequisite to the forward computation.\n\nDimensional inference works like this: as soon as we knew the input\ndimensionality, $\\mathbf{x} \\in \\mathbb{R}^{20}$ it was possible to define the\nweight matrix for the first layer, i.e. $\\mathbf{W}_1 \\in \\mathbb{R}^{256 \\times\n20}$. With that out of the way, we can progress to the second layer, define its\ndimensionality to be $10 \\times 256$ and so on through the computational graph\nand resolve all the dimensions as they become available. Once this is known, we\ncan proceed by initializing parameters. This is the solution to the three\nproblems outlined above.\n\n\n## Deferred Initialization in Practice\n\nNow that we know how it works in theory, let's see when the initialization is\nactually triggered. In order to do so, we mock up an initializer which does\nnothing but report a debug message stating when it was invoked and with which\nparameters.\n\n```{.python .input  n=22}\nclass MyInit(init.Initializer):\n    def _init_weight(self, name, data):\n        print('Init', name, data.shape)\n        # The actual initialization logic is omitted here.\n\nnet = getnet()\nnet.initialize(init=MyInit())\n```\n\nNote that, although `MyInit` will print information about the model parameters\nwhen it is called, the above `initialize` function does not print any\ninformation after it has been executed.  Therefore there is no actual\ninitialization when calling the `initialize` function - this\n+initialization is deferred until forward is called for the first time. Next,\nwe define the input and perform a forward calculation.\n\n```{.python .input  n=25}\nx = np.random.uniform(size=(2, 20))\ny = net(x)\n```\n\nAt this time, information on the model parameters is printed. When performing a\nforward calculation based on the input `x`, the system can automatically infer\nthe shape of the weight parameters of all layers based on the shape of the\ninput. Once the system has created these parameters, it calls the `MyInit`\ninstance to initialize them before proceeding to the forward calculation.\n\nOf course, this initialization will only be called when completing the initial\nforward calculation. After that, we will not re-initialize when we run the\nforward calculation `net(x)`, so the output of the `MyInit` instance will not be\ngenerated again.\n\n```{.python .input}\ny = net(x)\n```\n\nAs mentioned at the beginning of this section, deferred initialization can also\ncause confusion. Before the first forward calculation, we were unable to\ndirectly manipulate the model parameters, for example, we could not use the\n`data` and `set_data` functions to get and modify the parameters. Therefore, we\noften force initialization by sending a sample observation through the network.\n\n## Forced Initialization\n\nDeferred initialization does not occur if the system knows the shape of all\nparameters when calling the `initialize` function. This can occur in two cases:\n\n* We've already seen some data and we just want to reset the parameters.\n* We specified all input and output dimensions of the network or layer when\n  defining it.\n\nThe first case works just fine, as illustrated below.\n\n```{.python .input}\nnet.initialize(init=MyInit(), force_reinit=True)\n```\n\nThe second case requires us to specify the remaining set of parameters when\ncreating the layer. For instance, for dense layers we also need to specify the\n`in_units` so that initialization can occur immediately once `initialize` is\ncalled.\n\n```{.python .input}\nnet = nn.Sequential()\nnet.add(nn.Dense(256, in_units=20, activation='relu'))\nnet.add(nn.Dense(10, in_units=256))\n\nnet.initialize(init=MyInit())\n```\n\n## Parameter Initialization\n\nBy default, MXNet initializes the weight matrices uniformly by drawing random\nvalues with uniform-distribution between $-0.07$ and $0.07$ ($U[-0.07, 0.07]$)\nand updates the bias parameters by setting them all to $0$.  However, we often\nneed to use other methods to initialize the weights.  MXNet's `init` module\nprovides a variety of preset initialization methods, but if we want something\nout of the ordinary, we need a bit of extra work.\n\n### Built-in Initialization\n\nLet's begin with the built-in initializers. The code below initializes all\nparameters with Gaussian random variables.\n\n```{.python .input  n=9}\n# force_reinit ensures that the variables are initialized again, regardless of\n# whether they were already initialized previously.\nnet.initialize(init=init.Normal(sigma=0.01), force_reinit=True)\nprint(net[0].weight.data()[0])\n```\n\nIf we wanted to initialize all parameters to $1$, we could do this simply by\nchanging the initializer to `Constant(1)`.\n\n```{.python .input  n=10}\nnet.initialize(init=init.Constant(1), force_reinit=True)\nnet[0].weight.data()[0]\n```\n\nIf we want to initialize only a specific parameter in a different manner, we\ncan simply set the initializer only for the appropriate subblock (or\nparameter). For instance, below we initialize the second layer to a constant\nvalue of $42$ and we use the `Xavier` initializer for the weights of the\nfirst layer.\n\n```{.python .input  n=11}\nnet[0].weight.initialize(init=init.Xavier(), force_reinit=True)\nnet[1].initialize(init=init.Constant(42), force_reinit=True)\n\n# First layer\nprint(net[0].weight.data()[0])\nprint(net[0].bias.data()[0])  # initialized to 0\n\n# Second layer\nprint(net[1].weight.data()[0,0])\nprint(net[1].bias.data()[0])  # initialized to 0\n```\n\n### Custom Initialization\n\nSometimes, the initialization methods we need are not provided in the `init`\nmodule. At this point, we can implement a subclass of the `Initializer` class\nso that we can use it like any other initialization method. Usually, we only\nneed to implement the `_init_weight` function to suit our needs. In the example\nbelow, we pick a decidedly bizarre and nontrivial distribution, just to prove\nthe point. We draw the coefficients from the following distribution:\n\n$$\n\\begin{aligned}\n    w \\sim \\begin{cases}\n        U[5, 10] & \\text{ with probability } \\frac{1}{4} \\\\\n            0    & \\text{ with probability } \\frac{1}{2} \\\\\n        U[-10, -5] & \\text{ with probability } \\frac{1}{4}\n    \\end{cases}\n\\end{aligned}\n$$\n\n```{.python .input  n=12}\nclass MyInit(init.Initializer):\n    def _init_weight(self, name, data):\n        print('Init', name, data.shape)\n        data[:] = np.random.uniform(low=-10, high=10, size=data.shape)\n        data *= np.abs(data) >= 5\n\nnet.initialize(MyInit(), force_reinit=True)\nnet[0].weight.data()[0]\n```\n\nIf this functionality is insufficient, we can even set parameters directly.\nSince `data()` returns an `NDArray` we can access it just like any other matrix.\nA note for advanced users - if you want to adjust parameters within an\n`autograd` scope you need to use `set_data` to avoid confusing the automatic\ndifferentiation mechanics.\n\n```{.python .input  n=13}\nnet[0].weight.data()[:] += 1\nnet[0].weight.data()[0,0] = 42\nnet[0].weight.data()[0]\n```\n\n## Tied Parameters\n\nIn some cases, we want to share model parameters across multiple layers. For\ninstance when we want to find good word embeddings we may decide to use the\nsame parameters both for encoding and decoding of words. Let's see how to do\nthis a bit more elegantly. In the following we construct a dense layer and then\nuse its parameters specifically to set those of another layer.\n\n```{.python .input  n=14}\nnet = nn.Sequential()\n# We need to give the shared layer a name such that we can reference its\n# parameters.\nshared = nn.Dense(8, activation='relu')\nnet.add(nn.Dense(8, activation='relu'),\n        shared,\n        nn.Dense(8, activation='relu').share_parameters(shared.params),\n        nn.Dense(10))\nnet.initialize()\n\nx = np.random.uniform(size=(2, 20))\nnet(x)\n\n# Check whether the parameters are the same.\nprint(net[1].weight.data()[0] == net[2].weight.data()[0])\nnet[1].weight.data()[0,0] = 100\n# And make sure that they're actually the same object rather than just having\n# the same value.\nprint(net[1].weight.data()[0] == net[2].weight.data()[0])\n```\n\nThe above example shows that the parameters of the second and third layer are\ntied. As Python objects, they are identical rather than just being equal.\nThat is, by changing one of the parameters the other one changes too. What\nhappens to the gradients is quite ingenious. Since the model parameters contain\ngradients, the gradients of the second hidden layer and the third hidden layer\nare accumulated in `shared.params.grad` during backpropagation.\n\n## Conclusion\n\nIn this tutorial you learnt how to initialize a neural network, and should now\nunderstand the difference between deferred and forced initialization. Some more advanced\ncases you should now be aware of include custom initialization and tied parameters.\n\n## Recommended Next Steps\n\n* Check out the [API Docs](../../../../api/optimizer/index.rst) on initialization for a list of available initialization methods.\n* See [this tutorial](./naming.ipynb) for more information on Gluon Parameters.\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/blocks/naming.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Parameter and Block Naming\n\nIn gluon, each Parameter or Block has a name. Parameter names and Block names can be automatically created.\n\nIn this tutorial we talk about the best practices on naming. First, let's import MXNet and Gluon:\n\n\n```{.python .input}\nfrom __future__ import print_function\nimport mxnet as mx\nfrom mxnet import gluon\n```\n\n## Naming Blocks\n\nWhen creating a block, you can simply do as follows:\n\n\n```{.python .input}\nmydense = gluon.nn.Dense(100)\nprint(mydense.__class__.__name__)\n```\n\nWhen you create more Blocks of the same kind, they will be named with incrementing suffixes to avoid collision:\n\n\n```{.python .input}\ndense1 = gluon.nn.Dense(100)\nprint(dense1.__class__.__name__)\n```\n\n## Naming Parameters\n\nParameters will be named automatically by a unique name in the format of `param_{uuid4}_{name}`:\n\n\n```{.python .input}\nparam = gluon.Parameter(name = 'bias')\nprint(param.name)\n```\n\n`param.name` is used as the name of a parameter's symbol representation. And it can not be changed once the parameter is created.\n\nWhen getting parameters within a Block, you should use the structure based name as the key:\n\n\n```{.python .input}\nprint(dense1.collect_params())\n```\n\n## Nested Blocks\n\nIn MXNet 2, we don't have to define children blocks within a `name_scope` any more. Let's demonstrate this by defining and initiating a simple neural net:\n\n\n```{.python .input}\nclass Model(gluon.HybridBlock):\n    def __init__(self):\n        super(Model, self).__init__()\n        self.dense0 = gluon.nn.Dense(20)\n        self.dense1 = gluon.nn.Dense(20)\n        self.mydense = gluon.nn.Dense(20)\n\n    def forward(self, x):\n        x = mx.npx.relu(self.dense0(x))\n        x = mx.npx.relu(self.dense1(x))\n        return mx.npx.relu(self.mydense(x))\n\nmodel0 = Model()\nmodel0.initialize()\nmodel0.hybridize()\nmodel0(mx.np.zeros((1, 20)))\n```\n\nThe same principle also applies to container blocks like Sequential. We can simply do as follows:\n\n\n```{.python .input}\nnet = gluon.nn.Sequential()\nnet.add(gluon.nn.Dense(20))\nnet.add(gluon.nn.Dense(20))\n```\n\n\n## Saving and loading\n\n\nFor `HybridBlock`, we use `save_parameters`/`load_parameters`, which uses model structure, instead of parameter name, to match parameters.\n\n\n```{.python .input}\nmodel1 = Model()\nmodel0.save_parameters('model.params')\nmodel1.load_parameters('model.params')\nprint(mx.npx.load('model.params').keys())\n```\n\nFor `SymbolBlock.imports`, we use `export`, which uses parameter name `param.name`, to save parameters.\n\n```{.python .input}\nmodel0.export('model0')\nmodel2 = gluon.SymbolBlock.imports('model0-symbol.json', ['data'], 'model0-0000.params')\n```\n\n## Replacing Blocks from networks and fine-tuning\n\nSometimes you may want to load a pretrained model, and replace certain Blocks in it for fine-tuning.\n\nFor example, the alexnet in model zoo has 1000 output dimensions, but maybe you only have 100 classes in your application.\n\nTo see how to do this, we first load a pretrained ResNet.\n\n- In Gluon model zoo, all image classification models follow the format where the feature extraction layers are named `features` while the output layer is named `output`.\n- Note that the output layer is a dense block with 1000 dimension outputs.\n\n\n```{.python .input}\nresnet = gluon.model_zoo.vision.resnet50_v2()\nprint(resnet.output)\n```\n\n\nTo change the output to 100 dimension, we replace it with a new block.\n\n\n```{.python .input}\nresnet.output = gluon.nn.Dense(100)\nresnet.output.initialize()\n```\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/blocks/nn.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Layers and Blocks\n\n<!-- adapted from diveintodeeplearning -->\n\nAs network complexity increases, we move from designing single to entire layers\nof neurons.\n\nNeural network designs like\n[ResNet-152](https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/He_Deep_Residual_Learning_CVPR_2016_paper.pdf)\nhave a fair degree of regularity. They consist of *blocks* of repeated (or at\nleast similarly designed) layers; these blocks then form the basis of more\ncomplex network designs.\n\nIn this section, we'll talk about how to write code that makes such blocks on\ndemand, just like a Lego factory generates blocks which can be combined to\nproduce terrific artifacts.\n\nWe start with a very simple block, namely the block for a multilayer\nperceptron. A common strategy would be to construct a two-layer network as\nfollows:\n\n```{.python .input  n=1}\nimport mxnet as mx\nfrom mxnet import np, npx\nfrom mxnet.gluon import nn, Block, Parameter, Constant\n\n\nx = np.random.uniform(size=(2, 20))\n\nnet = nn.Sequential()\nnet.add(nn.Dense(256, activation='relu'))\nnet.add(nn.Dense(10))\nnet.initialize()\nnet(x)\n```\n\nThis generates a network with a hidden layer of $256$ units, followed by a ReLU\nactivation and another $10$ units governing the output. In particular, we used\nthe [nn.Sequential](../../../../api/gluon/nn/index.rst#mxnet.gluon.nn.Sequential)\nconstructor to generate an empty network into which we then inserted both\nlayers. What exactly happens inside `nn.Sequential`\nhas remained rather mysterious so far. In the following we will see that this\nreally just constructs a block that is a container for other blocks. These\nblocks can be combined into larger artifacts, often recursively. The diagram\nbelow shows how:\n\n![Blocks can be used recursively to form larger artifacts](/_static/blocks.svg)\n\nIn the following we will explain the various steps needed to go from defining\nlayers to defining blocks (of one or more layers):\n\n1. Blocks take data as input.\n1. Blocks store state in the form of parameters that are inherent to the block.\n   For instance, the block above contains two hidden layers, and we need a\n   place to store parameters for it.\n1. Blocks produce meaningful output. This is typically encoded in what\n   we will call the `forward` function. It allows us to invoke a block via\n   `net(X)` to obtain the desired output. What happens behind the scenes is\n   that it invokes `forward` to perform forward propagation (also called\n   forward computation).\n1. Blocks initialize the parameters in a lazy fashion as part of the first\n   `forward` call.\n1. Blocks calculate a gradient with regard to their input when invoking\n   `backward`. Typically this is automatic.\n\n## A Sequential Block\n\nThe [Block](../../../../api/gluon/block.rst#mxnet.gluon.Block) class is a\ngeneric component describing data flow. When the data flows through a sequence\nof blocks, each block applied to the output of the one before with the first\nblock being applied on the input data itself, we have a special kind of block,\nnamely the `Sequential` block.\n\n`Sequential` has helper methods to manage the sequence, with `add` being the\nmain one of interest allowing you to append blocks in sequence. Once the\noperations have been added, the forward computation of the model applies the\nblocks on the input data in the order they were added.  Below, we implement a\n`MySequential` class that has the same functionality as the `Sequential` class.\nThis may help you understand more clearly how the `Sequential` class works.\n\n```{.python .input  n=3}\nclass MySequential(Block):\n    def __init__(self):\n        super(MySequential, self).__init__()\n        self._layers = []\n\n    def add(self, block):\n        # Here, block is an instance of a Block subclass, and we assume it has a unique name. We save it in the\n        # member variable _layers of the Block class, and its type is List. When the MySequential instance\n        # calls the initialize function, the system automatically initializes all members of _layers.\n        self._layers.append(block)\n        self.register_child(block)\n\n    def forward(self, x):\n        # OrderedDict guarantees that members will be traversed in the order they were added.\n        for block in self._children.values():\n            x = block()(x)\n        return x\n```\n\nAt its core is the `add` method. It adds any block to the ordered dictionary of\nchildren. These are then executed in sequence when forward propagation is\ninvoked. Let's see what the MLP looks like now.\n\n```{.python .input  n=4}\nnet = MySequential()\nnet.add(nn.Dense(256, activation='relu'))\nnet.add(nn.Dense(10))\nnet.initialize()\nnet(x)\n```\n\nIndeed, it is no different than It can observed here that the use of the\n`MySequential` class is no different from the use of the Sequential class.\n\n\n## A Custom Block\n\nIt is easy to go beyond simple concatenation with `Sequential`. The\n`Block` class provides the functionality required to make such customizations.\n`Block` has a model constructor provided in the `nn` module, which we can\ninherit to define the model we want. The following inherits the `Block` class to\nconstruct the multilayer perceptron mentioned at the beginning of this section.\nThe `MLP` class defined here overrides the `__init__` and `forward` functions\nof the Block class. They are used to create model parameters and define forward\ncomputations, respectively. Forward computation is also forward propagation.\n\n```{.python .input  n=1}\nclass MLP(nn.Block):\n    # Declare a layer with model parameters. Here, we declare two fully\n    # connected layers.\n\n    def __init__(self, **kwargs):\n        # Call the constructor of the MLP parent class Block to perform the\n        # necessary initialization. In this way, other function parameters can\n        # also be specified when constructing an instance, such as the model\n        # parameter, params, described in the following sections.\n        super(MLP, self).__init__(**kwargs)\n        self.hidden = nn.Dense(256, activation='relu')  # Hidden layer\n        self.output = nn.Dense(10)  # Output layer\n\n    # Define the forward computation of the model, that is, how to return the\n    # required model output based on the input x.\n\n    def forward(self, x):\n        hidden_out = self.hidden(x)\n        return self.output(hidden_out)\n```\n\nLet's look at it a bit more closely. The `forward` method invokes a network\nsimply by evaluating the hidden layer `self.hidden(x)` and subsequently by\nevaluating the output layer `self.output( ... )`. This is what we expect in the\nforward pass of this block.\n\nIn order for the block to know what it needs to evaluate, we first need to\ndefine the layers. This is what the `__init__` method does. It first\ninitializes all of the Block-related parameters and then constructs the\nrequisite layers. This attaches the coresponding layers and the required\nparameters to the class. Note that there is no need to define a backpropagation\nmethod in the class. The system automatically generates the `backward` method\nneeded for back propagation by automatically finding the gradient (see the tutorial\non [autograd](../../autograd/index.ipynb)). The same applies to the\n[initialize](../../../../api/gluon/nn/index.rst#mxnet.gluon.nn.Block.initialize)\nmethod, which is generated automatically. Let's try\nthis out:\n\n```{.python .input  n=2}\nnet = MLP()\nnet.initialize()\nnet(x)\n```\n\nAs explained above, the `Block` class can be quite versatile in terms of what it\ndoes. For instance, its subclass can be a layer (such as the `Dense` class\nprovided by Gluon), it can be a model (such as the `MLP` class we just derived),\nor it can be a part of a model (this is what typically happens when designing\nvery deep networks). Throughout this chapter we will see how to use this with\ngreat flexibility.\n\n\n## Coding with `Blocks`\n\n### Blocks\nThe [Sequential](../../../../api/gluon/nn/index.rst#mxnet.gluon.nn.Sequential) class\ncan make model construction easier and does not require you to define the\n`forward` method; however, directly inheriting from\nits parent class, [Block](../../../../api/gluon/block.rst#mxnet.gluon.Block), can greatly\nexpand the flexibility of model construction. For example, implementing the\n`forward` method means you can introduce control flow in the network.\n\n### Constant parameters\nNow we'd like to introduce the notation of a *constant* parameter. These are\nparameters that are not used when invoking backpropagation. This sounds very\nabstract but here's what's really going on.\nAssume that we have some function\n\n$$f(\\mathbf{x},\\mathbf{w}) = 3 \\cdot \\mathbf{w}^\\top \\mathbf{x}.$$\n\nIn this case $3$ is a constant parameter. We could change $3$ to something else,\nsay $c$ via\n\n$$f(\\mathbf{x},\\mathbf{w}) = c \\cdot \\mathbf{w}^\\top \\mathbf{x}.$$\n\nNothing has really changed, except that we can adjust the value of $c$. It is\nstill a constant as far as $\\mathbf{w}$ and $\\mathbf{x}$ are concerned. However,\nGluon doesn't know about this unless we create it with `get_constant`\n(this makes the code go faster, too, since we're not sending the Gluon engine\non a wild goose chase after a parameter that doesn't change).\n\n```{.python .input  n=5}\nclass FancyMLP(nn.Block):\n    def __init__(self, **kwargs):\n        super(FancyMLP, self).__init__(**kwargs)\n\n        # Random weight parameters created with the get_constant are not\n        # iterated during training (i.e. constant parameters).\n        self.rand_weight = Constant(np.random.uniform(size=(20, 20)))\n        self.dense = nn.Dense(20, activation='relu')\n\n    def forward(self, x):\n        x = self.dense(x)\n        # Use the constant parameters created, as well as the ReLU and dot\n        # functions of NDArray.\n\n        x = npx.relu(np.dot(x, self.rand_weight.data()) + 1)\n        # Re-use the fully connected layer. This is equivalent to sharing\n        # parameters with two fully connected layers.\n        x = self.dense(x)\n        # Here in the control flow, we need to call `item` to return the\n        # scalar for comparison.\n\n        while npx.norm(x).item() > 1:\n            x /= 2\n        if npx.norm(x).item() < 0.8:\n            x *= 10\n        return x.sum()\n```\n\nIn this `FancyMLP` model, we used constant weight `rand_weight` (note that it is\nnot a model parameter), performed a matrix multiplication operation (`nd.dot`),\nand reused the *same* `Dense` layer. Note that this is very different from using\ntwo dense layers with different sets of parameters. Instead, we used the same\nnetwork twice. Quite often in deep networks one also says that the parameters\nare *tied* when one wants to express that multiple parts of a network share the\nsame parameters. Let's see what happens if we construct it and feed data through\nit.\n\n```{.python .input  n=6}\nnet = FancyMLP()\nnet.initialize()\nnet(x)\n```\n\nThere's no reason why we couldn't mix and match these ways of building a\nnetwork. Obviously the example below resembles a [Rube Goldberg\nMachine](https://en.wikipedia.org/wiki/Rube_Goldberg_machine). That said, it\ncombines examples for building a block from individual blocks,\nwhich in turn, may be blocks themselves. Furthermore, we can even combine\nmultiple strategies inside the same forward function. To demonstrate this,\nhere's the network.\n\n```{.python .input  n=7}\nclass NestMLP(nn.Block):\n    def __init__(self, **kwargs):\n        super(NestMLP, self).__init__(**kwargs)\n        self.net = nn.Sequential()\n        self.net.add(nn.Dense(64, activation='relu'),\n                     nn.Dense(32, activation='relu'))\n        self.dense = nn.Dense(16, activation='relu')\n\n    def forward(self, x):\n        return self.dense(self.net(x))\n\nchimera = nn.Sequential()\nchimera.add(NestMLP(), nn.Dense(20), FancyMLP())\n\nchimera.initialize()\nchimera(x)\n```\n\n## Hybridization\n\nThe reader may be starting to think about the efficiency of this Python code.\nAfter all, we have lots of dictionary lookups, code execution, and lots of\nother Pythonic things going on in what is supposed to be a high performance\ndeep learning library. The problems of Python's [Global Interpreter\nLock](https://wiki.python.org/moin/GlobalInterpreterLock) are well\nknown.\n\nIn the device of deep learning, we often have highly performant GPUs that\ndepend on CPUs running Python to tell them what to do. This mismatch can\nmanifest in the form of GPU starvation when the CPUs can not provide\ninstruction fast enough. We can improve this situation by deferring to a more\nperformant language instead of Python when possible.\n\nGluon does this by allowing for [Hybridization](hybridize.ipynb). In it, the\nPython interpreter executes the block the first time it's invoked. The Gluon\nruntime records what is happening and the next time around it short circuits\nany calls to Python. This can accelerate things considerably in some cases but\ncare needs to be taken with [control flow](../../autograd/index.ipynb#Advanced:-Using-Python-control-flow).\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/blocks/parameters.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Parameter Management\n\nThe ultimate goal of training deep neural networks is finding good parameter values for a given architecture. The [nn.Sequential](../../../../api/gluon/nn/index.rst#mxnet.gluon.nn.Sequential) class is a perfect tool to work with standard models. However, very few models are entirely standard, and most scientists want to build novel things, which requires working with model parameters.\n\nThis section shows how to manipulate parameters. In particular we will cover the following aspects:\n\n* How to access parameters in order to debug, diagnose, visualize or save them. It is the first step to understand how to work with custom models.\n* We will learn how to set parameters to specific values, e.g. how to initialize them. We will discuss the structure of parameter initializers.\n* We will show how this knowledge can be used to build networks that share some parameters.\n\nAs always, we start with a Multilayer Perceptron with a single hidden layer. We will use it to demonstrate the aspects mentioned above.\n\n```{.python .input  n=1}\nfrom mxnet import init, np\nfrom mxnet.gluon import nn\n\n\nnet = nn.Sequential()\nnet.add(nn.Dense(256, activation='relu'))\nnet.add(nn.Dense(10))\nnet.initialize()  # Use the default initialization method\n\nx = np.random.uniform(size=(2, 20))\nnet(x)            # Forward computation\n```\n\n## Parameter Access\n\nIn case of a Sequential class we can access the parameters simply by indexing each layer of the network. The `params` variable contains the required data. Let's try this out in practice by inspecting the parameters of the first layer.\n\n```{.python .input  n=2}\nprint(net.collect_params())\n```\n\nFrom the output we can see that the layer consists of two sets of parameters: `0.weight` and `0.bias`. They are both single precision and they have the necessary shapes that we would expect from the first layer, given that the input dimension is 20 and the output dimension 256. The names of the parameters are very useful, because they allow us to identify parameters *uniquely* even in a network of hundreds of layers and with nontrivial structure. The second layer is structured in a similar way.\n\n### Targeted Parameters\n\nIn order to do something useful with the parameters we need to access them. There are several ways to do this, ranging from simple to general. Let's look at some of them.\n\n```{.python .input  n=3}\nprint(net[1].bias)\nprint(net[1].bias.data())\n```\n\nThe first line returns the bias of the second layer. Since this is an object containing data, gradients, and additional information, we need to request the data explicitly. To request the data, we call `data` method on the parameter on the second line. Note that the bias is all 0 since we initialized the bias to contain all zeros.\n\nWe can also access the parameter by name, such as `0.weight`. This is possible since each layer comes with its own parameter dictionary that can be accessed directly. Both methods are entirely equivalent, but the first method leads to more readable code.\n\n```{.python .input  n=4}\nprint(net[0].params['weight'])\nprint(net[0].params['weight'].data())\n```\n\nNote that the weights are nonzero as they were randomly initialized when we constructed the network.\n\n[data](../../../../api/gluon/parameter.rst#mxnet.gluon.Parameter.data) is not the only method that we can invoke. For instance, we can compute the gradient with respect to the parameters. It has the same shape as the weight. However, since we did not invoke backpropagation yet, the values are all 0.\n\n```{.python .input  n=5}\nnet[0].weight.grad()\n```\n\n### All Parameters at Once\n\nAccessing parameters as described above can be a bit tedious, in particular if we have more complex blocks, or blocks of blocks (or even blocks of blocks of blocks), since we need to walk through the entire tree in reverse order to learn how the blocks were constructed. To avoid this, blocks come with a method [collect_params](../../../../api/gluon/block.rst#mxnet.gluon.Block.collect_params) which grabs all parameters of a network in one dictionary such that we can traverse it with ease. It does so by iterating over all constituents of a block and calls `collect_params` on sub-blocks as needed. To see the difference, consider the following:\n\n```{.python .input  n=6}\n# Parameters only for the first layer\nprint(net[0].collect_params())\n# Parameters of the entire network\nprint(net.collect_params())\n```\n\nThis provides us with the third way of accessing the parameters of the network. If we want to get the value of the bias term of the second layer we could simply use this:\n\n```{.python .input  n=7}\nnet.collect_params()['1.bias'].data()\n```\n\nBy adding a regular expression as an argument to `collect_params` method, we can select only a particular set of parameters whose names are matched by the regular expression.\n\n```{.python .input  n=8}\nprint(net.collect_params('.*weight'))\nprint(net.collect_params('0.*'))\n```\n\n### Rube Goldberg strikes again\n\nLet's see how the parameter naming conventions work if we nest multiple blocks inside each other. For that we first define a function that produces blocks (a block factory, so to speak) and then we combine these inside yet larger blocks.\n\n```{.python .input  n=20}\ndef block1():\n    net = nn.Sequential()\n    net.add(nn.Dense(32, activation='relu'))\n    net.add(nn.Dense(16, activation='relu'))\n    return net\n\ndef block2():\n    net = nn.Sequential()\n    for i in range(4):\n        net.add(block1())\n    return net\n\nrgnet = nn.Sequential()\nrgnet.add(block2())\nrgnet.add(nn.Dense(10))\nrgnet.initialize()\nrgnet(x)\n```\n\nNow that we are done designing the network, let's see how it is organized. `collect_params` provides us with this information, both in terms of naming and in terms of logical structure.\n\n```{.python .input}\nprint(rgnet.collect_params)\nprint(rgnet.collect_params())\n```\n\nWe can access layers following the hierarchy in which they are structured. For instance, if we want to access the bias of the first layer of the second subblock of the first major block, we could perform the following:\n\n```{.python .input}\nrgnet[0][1][0].bias.data()\n```\n\n### Saving and loading parameters\n\nIn order to save parameters, we can use [save_parameters](../../../../api/gluon/block.rst#mxnet.gluon.Block.save_parameters) method on the whole network or a particular subblock. The only parameter that is needed is the `file_name`. In a similar way, we can load parameters back from the file. We use [load_parameters](../../../../api/gluon/block.rst#mxnet.gluon.Block.load_parameters) method for that:\n\n```{.python .input}\nrgnet.save_parameters('model.params')\nrgnet.load_parameters('model.params')\n```\n\n## Parameter Initialization\n\nNow that we know how to access the parameters, let's look at how to initialize them properly. By default, MXNet initializes the weight matrices uniformly by drawing from $U[-0.07, 0.07]$ and the bias parameters are all set to $0$. However, we often need to use other methods to initialize the weights. MXNet's [init](../../../../api/initializer/index.rst) module provides a variety of preset initialization methods, but if we want something unusual, we need to do a bit of extra work.\n\n### Built-in Initialization\n\nLet's begin with the built-in initializers. The code below initializes all parameters with Gaussian random variables.\n\n```{.python .input  n=9}\n# force_reinit ensures that the variables are initialized again,\n# regardless of whether they were already initialized previously\nnet.initialize(init=init.Normal(sigma=0.01), force_reinit=True)\nnet[0].weight.data()[0]\n```\n\nIf we wanted to initialize all parameters to 1, we could do this simply by changing the initializer to [Constant(1)](../../../../api/initializer/index.rst#mxnet.initializer.Constant).\n\n```{.python .input  n=10}\nnet.initialize(init=init.Constant(1), force_reinit=True)\nnet[0].weight.data()[0]\n```\n\nIf we want to initialize only a specific parameter in a different manner, we can simply set the initializer only for the appropriate subblock (or parameter) for that matter. For instance, below we initialize the second layer to a constant value of 42 and we use the [Xavier](../../../../api/initializer/index.rst#mxnet.initializer.Xavier) initializer for the weights of the first layer.\n\n```{.python .input  n=11}\nnet[1].initialize(init=init.Constant(42), force_reinit=True)\nnet[0].weight.initialize(init=init.Xavier(), force_reinit=True)\nprint(net[1].weight.data()[0,0])\nprint(net[0].weight.data()[0])\n```\n\n### Custom Initialization\n\nSometimes, the initialization methods we need are not provided in the `init` module. If this is the case, we can implement a subclass of the [Initializer](../../../../api/initializer/index.rst#mxnet.initializer.Initializer) class so that we can use it like any other initialization method. Usually, we only need to implement the `_init_weight` method and modify the incoming NDArray according to the initial result. In the example below, we pick a nontrivial distribution, just to prove the point. We draw the coefficients from the following distribution:\n\n$$\n\\begin{aligned}\n    w \\sim \\begin{cases}\n        U[5, 10] & \\text{ with probability } \\frac{1}{4} \\\\\n            0    & \\text{ with probability } \\frac{1}{2} \\\\\n        U[-10, -5] & \\text{ with probability } \\frac{1}{4}\n    \\end{cases}\n\\end{aligned}\n$$\n\n```{.python .input  n=12}\nclass MyInit(init.Initializer):\n    def _init_weight(self, name, data):\n        print('Init', name, data.shape)\n        data[:] = np.random.uniform(low=-10, high=10, size=data.shape)\n        data *= np.abs(data) >= 5\n\nnet.initialize(MyInit(), force_reinit=True)\nnet[0].weight.data()[0]\n```\n\nIf even this functionality is insufficient, we can set parameters directly. Since `data()` returns an NDArray we can access it just like any other matrix. A note for advanced users - if you want to adjust parameters within an [autograd](../../../../api/autograd/index.rst) scope you need to use [set_data](../../../../api/gluon/parameter.rst#mxnet.gluon.Parameter.set_data) to avoid confusing the automatic differentiation mechanics.\n\n```{.python .input  n=13}\nnet[0].weight.data()[:] += 1\nnet[0].weight.data()[0,0] = 42\nnet[0].weight.data()[0]\n```\n\n## Tied Parameters\n\nIn some cases, we want to share model parameters across multiple layers. For instance, when we want to find good word embeddings we may decide to use the same parameters both for encoding and decoding of words. In the code below, we allocate a dense layer and then use its parameters specifically to set those of another layer.\n\n```{.python .input  n=14}\nnet = nn.Sequential()\n# We need to give the shared layer a name such that we can reference\n# its parameters\nshared = nn.Dense(8, activation='relu')\nnet.add(nn.Dense(8, activation='relu'),\n        shared,\n        nn.Dense(8, activation='relu').share_parameters(shared.params),\n        nn.Dense(10))\nnet.initialize()\n\nx = np.random.uniform(size=(2, 20))\nnet(x)\n\n# Check whether the parameters are the same\nprint(net[1].weight.data()[0] == net[2].weight.data()[0])\nnet[1].weight.data()[0,0] = 100\n# And make sure that they're actually the same object rather\n# than just having the same value\nprint(net[1].weight.data()[0] == net[2].weight.data()[0])\n```\n\nThe above example shows that the parameters of the second and third layer are tied. They are identical rather than just being equal. That is, by changing one of the parameters the other one changes, too. What happens to the gradients is quite ingenious. Since the model parameters contain gradients, the gradients of the second hidden layer and the third hidden layer are accumulated in the [shared.params.grad()](../../../../api/gluon/parameter.rst#mxnet.gluon.Parameter.grad) during backpropagation.\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/blocks/save_load_params.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Saving and Loading Gluon Models\n\nTraining large models take a lot of time and it is a good idea to save the trained models to files to avoid training them again and again. There are a number of reasons to do this. For example, you might want to do inference on a machine that is different from the one where the model was trained. Sometimes model's performance on validation set decreases towards the end of the training because of overfitting. If you saved your model parameters after every epoch, at the end you can decide to use the model that performs best on the validation set. Another reason would be to train your model using one language (like Python that has a lot of tools for training) and run inference using a different language (like Scala probably because your application is built on Scala).\n\nIn this tutorial, we will learn ways to save and load Gluon models. There are two ways to save/load Gluon models:\n\n**1. Save/load model parameters only**\n\nParameters of any Gluon model can be saved using the `save_parameters` and `load_parameters` method. This does not save model architecture. This method is used to save parameters of dynamic (non-hybrid) models. Model architecture cannot be saved for dynamic models because model architecture changes during execution.\n\n**2. Save/load model parameters AND architecture**\n\nThe Model architecture of `Hybrid` models stays static and don't change during execution. Therefore both model parameters AND architecture can be saved and loaded using `export`, `imports` methods.\n\nLet's look at the above methods in more detail. Let's start by importing the modules we'll need.\n\n```{.python .input}\nfrom __future__ import print_function\n\nimport mxnet as mx\nfrom mxnet import np, npx, autograd, gluon\nfrom mxnet.gluon.data.vision import transforms\n\nimport numpy as onp\n```\n\n## Setup: build and train a simple model\n\nWe need a trained model before we can save it to a file. So let's go ahead and build a very simple convolutional network and train it on MNIST data.\n\nLet's define a helper function to build a LeNet model and another helper to train LeNet with MNIST.\n\n```{.python .input}\n# Use GPU if one exists, else use CPU\ndevice = mx.gpu() if mx.device.num_gpus() else mx.cpu()\n\n# MNIST images are 28x28. Total pixels in input layer is 28x28 = 784\nnum_inputs = 784\n# Clasify the images into one of the 10 digits\nnum_outputs = 10\n# 64 images in a batch\nbatch_size = 64\n\n# Load the training data\ntrain_data = gluon.data.DataLoader(gluon.data.vision.MNIST(train=True).transform_first(transforms.ToTensor()),\n                                   batch_size, shuffle=True)\n\n# Build a simple convolutional network\ndef build_lenet(net):    \n    # First convolution\n    net.add(gluon.nn.Conv2D(channels=20, kernel_size=5, activation='relu'))\n    net.add(gluon.nn.MaxPool2D(pool_size=2, strides=2))\n    # Second convolution\n    net.add(gluon.nn.Conv2D(channels=50, kernel_size=5, activation='relu'))\n    net.add(gluon.nn.MaxPool2D(pool_size=2, strides=2))\n    # Flatten the output before the fully connected layers\n    net.add(gluon.nn.Flatten())\n    # First fully connected layers with 512 neurons\n    net.add(gluon.nn.Dense(512, activation=\"relu\"))\n    # Second fully connected layer with as many neurons as the number of classes\n    net.add(gluon.nn.Dense(num_outputs))\n\n    return net\n\n# Train a given model using MNIST data\ndef train_model(model):\n    # Initialize the parameters with Xavier initializer\n    model.initialize(mx.init.Xavier(), device=device)\n    # Use cross entropy loss\n    softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()\n    # Use Adam optimizer\n    trainer = gluon.Trainer(model.collect_params(), 'adam', {'learning_rate': .001})\n\n    # Train for one epoch\n    for epoch in range(1):\n        # Iterate through the images and labels in the training data\n        for batch_num, (data, label) in enumerate(train_data):\n            # get the images and labels\n            data = data.to_device(device)\n            label = label.to_device(device)\n            # Ask autograd to record the forward pass\n            with autograd.record():\n                # Run the forward pass\n                output = model(data)\n                # Compute the loss\n                loss = softmax_cross_entropy(output, label)\n            # Compute gradients\n            loss.backward()\n            # Update parameters\n            trainer.step(data.shape[0])\n\n            # Print loss once in a while\n            if batch_num % 50 == 0:\n                curr_loss = np.mean(loss).item()\n                print(\"Epoch: %d; Batch %d; Loss %f\" % (epoch, batch_num, curr_loss))\n```\n\nLet's build a model and train it. After training, we will save and restore this model from a file.\n\n```{.python .input}\nnet = build_lenet(gluon.nn.Sequential())\ntrain_model(net)\n```\n<pre>Epoch: 0; Batch 0; Loss 2.288904 <!--notebook-skip-line-->\nEpoch: 0; Batch 50; Loss 0.269372 <!--notebook-skip-line-->\nEpoch: 0; Batch 100; Loss 0.238990 <!--notebook-skip-line-->\nEpoch: 0; Batch 150; Loss 0.320592 <!--notebook-skip-line-->\nEpoch: 0; Batch 200; Loss 0.048619 <!--notebook-skip-line-->\nEpoch: 0; Batch 250; Loss 0.121555 <!--notebook-skip-line-->\nEpoch: 0; Batch 300; Loss 0.083645 <!--notebook-skip-line-->\nEpoch: 0; Batch 350; Loss 0.040627 <!--notebook-skip-line-->\nEpoch: 0; Batch 400; Loss 0.195946 <!--notebook-skip-line-->\nEpoch: 0; Batch 450; Loss 0.155514 <!--notebook-skip-line-->\nEpoch: 0; Batch 500; Loss 0.031762 <!--notebook-skip-line-->\nEpoch: 0; Batch 550; Loss 0.056516 <!--notebook-skip-line-->\nEpoch: 0; Batch 600; Loss 0.095174 <!--notebook-skip-line-->\nEpoch: 0; Batch 650; Loss 0.054901 <!--notebook-skip-line-->\nEpoch: 0; Batch 700; Loss 0.030067 <!--notebook-skip-line-->\nEpoch: 0; Batch 750; Loss 0.102611 <!--notebook-skip-line-->\nEpoch: 0; Batch 800; Loss 0.010036 <!--notebook-skip-line-->\nEpoch: 0; Batch 850; Loss 0.051853 <!--notebook-skip-line-->\nEpoch: 0; Batch 900; Loss 0.008402 <!--notebook-skip-line-->\n</pre> <!--notebook-skip-line-->\n\n## Saving model parameters to file\n\nOkay, we now have a model (`net`) that we can save to a file. Let's save the parameters of this model to a file using the `save_parameters` function.\n\n```{.python .input}\nfile_name = \"net.params\"\nnet.save_parameters(file_name)\n```\n\nWe have successfully saved the parameters of the model into a file.\n\n## Loading model parameters from file\n\nLet's now create a network with the parameters we saved into the file. We build the network again using the helper first and then load the weights from the file we saved using the `load_parameters` function.\n\n```{.python .input}\nnew_net = build_lenet(gluon.nn.Sequential())\nnew_net.load_parameters(file_name, device=device)\n```\n\nNote that to do this, we need the definition of the network as Python code. If we want to recreate this network on a different machine using the saved weights, we need the same Python code (`build_lenet`) that created the network to create the `new_net` object shown above. This means Python code needs to be copied over to any machine where we want to run this network.\n\nIf our network is [Hybrid](./hybridize.ipynb), we can even save the network architecture into files and we won't need the network definition in a Python file to load the network. We'll see how to do it in the next section.\n\nLet's test the model we just loaded from file.\n\n```{.python .input}\nimport matplotlib.pyplot as plt\n\ndef verify_loaded_model(net):\n    \"\"\"Run inference using ten random images.\n    Print both input and output of the model\"\"\"\n\n    def transform(data, label):\n        return data.astype(np.float32)/255, label.astype(np.float32)\n\n    # Load ten random images from the test dataset\n    sample_data = mx.gluon.data.DataLoader(mx.gluon.data.vision.MNIST(train=False).transform(transform),\n                                  10, shuffle=True)\n\n    for data, label in sample_data:\n\n        # Display the images\n        img = np.transpose(data, (1,0,2,3))\n        img = npx.reshape(img, (28,10*28,1))\n        imtiles = np.tile(img, (1,1,3))\n        plt.imshow(imtiles.asnumpy())\n        plt.show()\n\n        # Display the predictions\n        data = np.transpose(data, (0, 3, 1, 2))\n        out = net(data.to_device(device))\n        predictions = np.argmax(out, axis=1)\n        print('Model predictions: ', predictions.asnumpy())\n\n        break\n\nverify_loaded_model(new_net)\n```\n![Model inputs mnist in 1](https://raw.githubusercontent.com/indhub/web-data/4a9c100aa996df3dff0e7f493029d411c2b526c3/mxnet/tutorials/gluon/save_load_params/mnist_in_1.png) <!--notebook-skip-line-->\n\nModel predictions:  [1. 1. 4. 5. 0. 5. 7. 0. 3. 6.] <!--notebook-skip-line-->\n\n## Saving model parameters AND architecture to file\n\n[Hybrid](./hybridize.ipynb) models can be serialized as JSON files using the `export` function. Once serialized, these models can be loaded from other language bindings like C++ or Scala for faster inference or inference in different environments.\n\nNote that the network we created above is not a Hybrid network and therefore cannot be serialized into a JSON file. So, let's create a Hybrid version of the same network and train it.\n\n```{.python .input}\nnet = build_lenet(gluon.nn.HybridSequential())\nnet.hybridize()\ntrain_model(net)\n```\n\n<pre>Epoch: 0; Batch 0; Loss 2.323284 <!--notebook-skip-line-->\nEpoch: 0; Batch 50; Loss 0.444733 <!--notebook-skip-line-->\nEpoch: 0; Batch 100; Loss 0.103407 <!--notebook-skip-line-->\nEpoch: 0; Batch 150; Loss 0.166772 <!--notebook-skip-line-->\nEpoch: 0; Batch 200; Loss 0.227569 <!--notebook-skip-line-->\nEpoch: 0; Batch 250; Loss 0.069515 <!--notebook-skip-line-->\nEpoch: 0; Batch 300; Loss 0.074086 <!--notebook-skip-line-->\nEpoch: 0; Batch 350; Loss 0.074382 <!--notebook-skip-line-->\nEpoch: 0; Batch 400; Loss 0.026569 <!--notebook-skip-line-->\nEpoch: 0; Batch 450; Loss 0.097248 <!--notebook-skip-line-->\nEpoch: 0; Batch 500; Loss 0.059895 <!--notebook-skip-line-->\nEpoch: 0; Batch 550; Loss 0.053194 <!--notebook-skip-line-->\nEpoch: 0; Batch 600; Loss 0.076294 <!--notebook-skip-line-->\nEpoch: 0; Batch 650; Loss 0.047274 <!--notebook-skip-line-->\nEpoch: 0; Batch 700; Loss 0.007898 <!--notebook-skip-line-->\nEpoch: 0; Batch 750; Loss 0.039478 <!--notebook-skip-line-->\nEpoch: 0; Batch 800; Loss 0.031342 <!--notebook-skip-line-->\nEpoch: 0; Batch 850; Loss 0.059289 <!--notebook-skip-line-->\nEpoch: 0; Batch 900; Loss 0.037809 <!--notebook-skip-line-->\n</pre> <!--notebook-skip-line-->\n\nWe now have a trained hybrid network. This can be exported into files using the `export` function. The `export` function will export the model architecture into a `.json` file and model parameters into a `.params` file.\n\n```{.python .input}\nnet.export(\"lenet\", epoch=1)\n```\n\n`export` in this case creates `lenet-symbol.json` and `lenet-0001.params` in the current directory.\n\n## Loading model parameters AND architecture from file\n\n\n### From Python\n\nSerialized Hybrid networks (saved as .JSON and .params file) can be loaded and used inside Python frontend using `gluon.nn.SymbolBlock`. To demonstrate that, let's load the network we serialized above.\n\n```{.python .input}\nimport warnings\nwith warnings.catch_warnings():\n    warnings.simplefilter(\"ignore\")\n    deserialized_net = gluon.nn.SymbolBlock.imports(\"lenet-symbol.json\", ['data'], \"lenet-0001.params\", device=device)\n```\n\n`deserialized_net` now contains the network we deserialized from files. Let's test the deserialized network to make sure it works.\n\n```{.python .input}\nverify_loaded_model(deserialized_net)\n```\n\n![Model inputs mnist in 2](https://raw.githubusercontent.com/indhub/web-data/4a9c100aa996df3dff0e7f493029d411c2b526c3/mxnet/tutorials/gluon/save_load_params/mnist_in_2.png) <!--notebook-skip-line-->\n\nModel predictions:  [4. 8. 0. 1. 5. 5. 8. 8. 1. 9.] <!--notebook-skip-line-->\n\nThat's all! We learned how to save and load Gluon networks from files. Parameters of any Gluon network can be persisted into files. For hybrid networks, both the architecture of the network and the parameters can be saved to and loaded from files.\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/image/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nImage Tutorials\n===============\n\nThese tutorials will help you learn how to create and use models that work with\nimages and other computer vision tasks.\nMost of these tutorials use the `MXNet GluonCV toolkit <https://gluon-cv.mxnet.io/>`__.\n\nBasic Image Tutorials\n---------------------\n\n.. container:: cards\n\n   .. card::\n      :title: MNIST\n      :link: mnist.html\n\n      How to create a convolutional neural network for handwritten digit recognition.\n\n\nGluonCV Toolkit Tutorials\n-------------------------\n\nThese tutorials link to the MXNet GluonCV Toolkit website.\n\n.. container:: cards\n\n   .. card::\n      :title: Prepare Datasets\n      :link: https://gluon-cv.mxnet.io/build/examples_datasets/index.html\n\n      How to use built-in MXNet GluonCV features for loading and preparing both common & custom datasets.\n\n   .. card::\n      :title: Image Classification\n      :link: https://gluon-cv.mxnet.io/build/examples_classification/index.html\n\n      Pretrained models for inference, fine-tune models, train your own model\n      on ImageNet, and more.\n\n   .. card::\n      :title: Object Detection\n      :link: https://gluon-cv.mxnet.io/build/examples_detection/index.html\n\n      Learn how to use Single shot detector (SSD), RCNN, and YOLO models.\n\n   .. card::\n      :title: Semantic Segmentation\n      :link: https://gluon-cv.mxnet.io/build/examples_segmentation/index.html\n\n      Learn how to use and train models that can identify and segment objects in an image.\n\n   .. card::\n      :title: Instance Segmentation\n      :link: https://gluon-cv.mxnet.io/build/examples_instance/index.html\n\n      Learn how to use and train models the perform a variation of semantic\n      segmentation that also classifies similar objects into discrete entities.\n\n   .. card::\n      :title: Pose Estimation\n      :link: https://gluon-cv.mxnet.io/build/examples_pose/index.html\n\n      Learn how to use a simple Pose network that predicts the heatmap for each\n      joint then map it to the coordinates on the original image.\n\n\n.. toctree::\n   :hidden:\n   :maxdepth: 1\n   :glob:\n\n   *\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/image/info_gan.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Image similarity search with InfoGAN\n\nThis notebook shows how to implement an InfoGAN based on Gluon. InfoGAN is an extension of GANs, where the generator input is split in 2 parts: random noise and a latent code (see [InfoGAN Paper](https://arxiv.org/pdf/1606.03657.pdf)).\nThe codes are made meaningful by maximizing the mutual information between code and generator output. InfoGAN learns a disentangled representation in a completely unsupervised manner. It can be used for many applications such as image similarity search. This notebook uses the DCGAN example and extends it to create an InfoGAN.\n\n\n```{.python .input}\nfrom __future__ import print_function\nfrom datetime import datetime\nimport logging\nimport multiprocessing\nimport os\nimport sys\nimport tarfile\nimport time\n\nimport numpy as onp\nfrom matplotlib import pyplot as plt\nimport mxnet as mx\nfrom mxnet import gluon\nfrom mxnet import np, npx\nfrom mxnet.gluon import nn, utils\nfrom mxnet import autograd\n\n```\n\nThe latent code vector can contain several variables, which can be categorical and/or continuous. We set `n_continuous` to 2 and `n_categories` to 10.\n\n\n```{.python .input}\nbatch_size   = 64\nz_dim        = 100\nn_continuous = 2\nn_categories = 10\ndevice = mx.gpu() if mx.device.num_gpus() else mx.cpu()\n```\n\nSome functions to load and normalize images.\n\n\n```{.python .input}\nlfw_url = 'http://vis-www.cs.umass.edu/lfw/lfw-deepfunneled.tgz'\ndata_path = 'lfw_dataset'\nif not os.path.exists(data_path):\n    os.makedirs(data_path)\n    data_file = utils.download(lfw_url)\n    with tarfile.open(data_file) as tar:\n        tar.extractall(path=data_path)\n\n```\n\n\n```{.python .input}\ndef transform(data, width=64, height=64):\n    data = mx.image.imresize(data, width, height)\n    data = np.transpose(data, (2,0,1))\n    data = data.astype(onp.float32)/127.5 - 1\n    if data.shape[0] == 1:\n        data = np.tile(data, (3, 1, 1))\n    return data.reshape((1,) + data.shape)\n```\n\n\n```{.python .input}\ndef get_files(data_dir):\n    images    = []\n    filenames = []\n    for path, _, fnames in os.walk(data_dir):\n        for fname in fnames:\n            if not fname.endswith('.jpg'):\n                continue\n            img = os.path.join(path, fname)\n            img_arr = mx.image.imread(img)\n            img_arr = transform(img_arr)\n            images.append(img_arr)\n            filenames.append(path + \"/\" + fname)\n    return images, filenames\n```\n\nLoad the dataset `lfw_dataset` which contains images of celebrities.\n\n\n```{.python .input}\ndata_dir = 'lfw_dataset'\nimages, filenames = get_files(data_dir)\nsplit = int(len(images)*0.8)\ntest_images = images[split:]\ntest_filenames = filenames[split:]\ntrain_images = images[:split]\ntrain_filenames = filenames[:split]\n\ntrain_data = gluon.data.ArrayDataset(np.concatenate(train_images))\ntrain_dataloader = gluon.data.DataLoader(train_data, batch_size=batch_size, shuffle=True, last_batch='rollover', num_workers=multiprocessing.cpu_count()-1)\n```\n\n## Generator\nDefine the Generator model. The Generator consist of  4 layers where each layer involves a strided convolution, batch normalization, and rectified nonlinearity. It takes as input random noise and the latent code and produces an `(64,64,3)` output image.\n\n\n```{.python .input}\nclass Generator(gluon.HybridBlock):\n    def __init__(self, **kwargs):\n        super(Generator, self).__init__(**kwargs)\n        self.prev = nn.HybridSequential()\n        self.prev.add(nn.Dense(1024, use_bias=False), nn.BatchNorm(), nn.Activation(activation='relu'))\n        self.G = nn.HybridSequential()\n\n        self.G.add(nn.Conv2DTranspose(64 * 8, 4, 1, 0, use_bias=False))\n        self.G.add(nn.BatchNorm())\n        self.G.add(nn.Activation('relu'))\n        self.G.add(nn.Conv2DTranspose(64 * 4, 4, 2, 1, use_bias=False))\n        self.G.add(nn.BatchNorm())\n        self.G.add(nn.Activation('relu'))\n        self.G.add(nn.Conv2DTranspose(64 * 2, 4, 2, 1, use_bias=False))\n        self.G.add(nn.BatchNorm())\n        self.G.add(nn.Activation('relu'))\n        self.G.add(nn.Conv2DTranspose(64, 4, 2, 1, use_bias=False))\n        self.G.add(nn.BatchNorm())\n        self.G.add(nn.Activation('relu'))\n        self.G.add(nn.Conv2DTranspose(3, 4, 2, 1, use_bias=False))\n        self.G.add(nn.Activation('tanh'))\n\n    def forward(self, x):\n        x = self.prev(x)\n        x = np.reshape(x, (-2, -1, 1, 1))\n        return self.G(x)\n```\n\n## Discriminator\nDefine the Discriminator and Q model. The Q model shares many layers with the Discriminator. Its task is to estimate the code `c` for a given fake image.  It is used to maximize the lower bound to the mutual information.\n\n\n```{.python .input}\nclass Discriminator(gluon.HybridBlock):\n    def __init__(self, **kwargs):\n        super(Discriminator, self).__init__(**kwargs)\n        self.D = nn.HybridSequential()\n        self.D.add(nn.Conv2D(64, 4, 2, 1, use_bias=False))\n        self.D.add(nn.LeakyReLU(0.2))\n        self.D.add(nn.Conv2D(64 * 2, 4, 2, 1, use_bias=False))\n        self.D.add(nn.BatchNorm())\n        self.D.add(nn.LeakyReLU(0.2))\n        self.D.add(nn.Conv2D(64 * 4, 4, 2, 1, use_bias=False))\n        self.D.add(nn.BatchNorm())\n        self.D.add(nn.LeakyReLU(0.2))\n        self.D.add(nn.Conv2D(64 * 8, 4, 2, 1, use_bias=False))\n        self.D.add(nn.BatchNorm())\n        self.D.add(nn.LeakyReLU(0.2))\n\n        self.D.add(nn.Dense(1024, use_bias=False), nn.BatchNorm(), nn.Activation(activation='relu'))\n\n        self.prob = nn.Dense(1)\n        self.feat = nn.HybridSequential()\n        self.feat.add(nn.Dense(128, use_bias=False), nn.BatchNorm(), nn.Activation(activation='relu'))\n        self.category_prob = nn.Dense(n_categories)\n        self.continuous_mean = nn.Dense(n_continuous)\n        self.Q = nn.HybridSequential()\n        self.Q.add(self.feat, self.category_prob, self.continuous_mean)\n\n    def forward(self, x):\n        x               = self.D(x)\n        prob            = self.prob(x)\n        feat            = self.feat(x)\n        category_prob   = self.category_prob(feat)\n        continuous_mean = self.continuous_mean(feat)\n\n        return prob, category_prob, continuous_mean\n```\n\nThe InfoGAN has the following layout.\n<img src=\"https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/info_gan/InfoGAN.png\" style=\"width:800px;height:250px;\">\n\nDiscriminator and Generator are the same as in the DCGAN example. On top of the Disciminator is the Q model, which is estimating the code `c` for given fake images. The Generator's input is random noise and the latent code `c`.\n\n## Training Loop\nInitialize Generator and Discriminator and define correspoing trainer function.\n\n\n```{.python .input}\ngenerator = Generator()\ngenerator.hybridize()\ngenerator.initialize(mx.init.Normal(0.002), device=device)\n\ndiscriminator = Discriminator()\ndiscriminator.hybridize()\ndiscriminator.initialize(mx.init.Normal(0.002), device=device)\n\nlr   = 0.0001\nbeta = 0.5\n\ng_trainer = gluon.Trainer(generator.collect_params(), 'adam', {'learning_rate': lr, 'beta1': beta})\nd_trainer = gluon.Trainer(discriminator.collect_params(), 'adam', {'learning_rate': lr, 'beta1': beta})\nq_trainer = gluon.Trainer(discriminator.Q.collect_params(), 'adam', {'learning_rate': lr, 'beta1': beta})\n```\n\nCreate vectors with real (=1) and fake labels (=0).\n\n\n```{.python .input}\nreal_label = np.ones((batch_size,), device=device)\nfake_label = np.zeros((batch_size,),device=device)\n```\n\nLoad a pretrained model.\n\n\n```{.python .input}\nif os.path.isfile('infogan_d_latest.params') and os.path.isfile('infogan_g_latest.params'):\n    discriminator.load_parameters('infogan_d_latest.params', device=device, allow_missing=True, ignore_extra=True)\n    generator.load_parameters('infogan_g_latest.params', device=device, allow_missing=True, ignore_extra=True)\n```\nThere are 2 differences between InfoGAN and DCGAN: the extra latent code and the Q network to estimate the code.\nThe latent code is part of the Generator input and it contains mutliple variables (continuous, categorical) that can represent different distributions. In order to make sure that the Generator uses the latent code, mutual information is introduced into the GAN loss term. Mutual information measures how much X is known given Y or vice versa. It is defined as:\n\n![infogan entropy](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/info_gan/entropy.gif)\n\nThe InfoGAN loss is:\n\n![infogan loss](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/info_gan/loss.gif)\n\nwhere `V(D,G)` is the GAN loss and the mutual information `I(c, G(z, c))` goes in as regularization. The goal is to reach high mutual information, in order to learn meaningful codes for the data.\n\n\nDefine the loss functions. `SoftmaxCrossEntropyLoss` for the categorical code,  `L2Loss` for the continious code and `SigmoidBinaryCrossEntropyLoss` for the normal GAN loss.\n\n\n```{.python .input}\nloss1 = gluon.loss.SigmoidBinaryCrossEntropyLoss()\nloss2 = gluon.loss.L2Loss()\nloss3 = gluon.loss.SoftmaxCrossEntropyLoss()\n```\n\nThis function samples `c`, `z`, and concatenates them to create the generator input.\n\n\n```{.python .input}\ndef create_generator_input():\n\n    #create random noise\n    z      = np.random.normal(0, 1, size=(batch_size, z_dim), device=device)\n    label  = np.array(onp.random.randint(n_categories, size=batch_size)).to_device(device)\n    c1     = npx.one_hot(label, depth=n_categories).to_device(device)\n    c2     = np.random.uniform(-1, 1, size=(batch_size, n_continuous)).to_device(device)\n\n    # concatenate random noise with c which will be the input of the generator\n    return np.concatenate([z, c1, c2], axis=1), label, c2\n```\n\nDefine the training loop.\n1. The discriminator receives `real_data` and `loss1` measures how many real images have been identified as real\n2. The discriminator receives `fake_image` from the Generator and `loss1` measures how many fake images have been identified as fake\n3. Update Discriminator. Currently, it is updated every second iteration in order to avoid that the Discriminator becomes too strong. You may want to change that.\n4. The updated discriminator receives `fake_image` and `loss1` measures how many fake images have been been identified as real, `loss2` measures the difference between the sampled continuous latent code `c` and the output of the Q model and `loss3` measures the difference between the sampled categorical latent code `c` and the output of the Q model.\n4. Update Generator and Q\n\n\n```{.python .input}\nepochs = 1\ncounter = 0\nfor epoch in range(epochs):\n    print(\"Epoch\", epoch)\n    starttime = time.time()\n\n    d_error_epoch = np.zeros((1,), device=device)\n    g_error_epoch = np.zeros((1,), device=device)\n\n    for idx, data in enumerate(train_dataloader):\n\n        #get real data and generator input\n        real_data = data.to_device(device)\n        g_input, label, c2 = create_generator_input()\n\n\n        #Update discriminator: Input real data and fake data\n        with autograd.record():\n            output_real,_,_ = discriminator(real_data)\n            d_error_real    = loss1(output_real, real_label)\n\n            # create fake image and input it to discriminator\n            fake_image      = generator(g_input)\n            output_fake,_,_ = discriminator(fake_image.detach())\n            d_error_fake    = loss1(output_fake, fake_label)\n\n            # total discriminator error\n            d_error         = d_error_real + d_error_fake\n\n        d_error_epoch += d_error.mean()\n\n        #Update D every second iteration\n        if (counter+1) % 2 == 0:\n            d_error.backward()\n            d_trainer.step(batch_size)\n\n        #Update generator: Input random noise and latent code vector\n        with autograd.record():\n            fake_image = generator(g_input)\n            output_fake, category_prob, continuous_mean = discriminator(fake_image)\n            g_error = loss1(output_fake, real_label) + loss3(category_prob, label) + loss2(c2, continuous_mean)\n\n        g_error.backward()\n        g_error_epoch += g_error.mean()\n\n        g_trainer.step(batch_size)\n        q_trainer.step(batch_size)\n\n        # logging\n        if idx % 10 == 0:\n            count = idx + 1\n            logging.info('speed: {} samples/s'.format(batch_size / (time.time() - starttime)))\n            logging.info('discriminator loss = %f, generator loss = %f at iter %d epoch %d'\n                        %(d_error_epoch.item()/count,g_error_epoch.item()/count, count, epoch))\n\n            g_input,_,_ = create_generator_input()\n\n    discriminator.save_parameters(\"infogan_d_latest.params\")\n    generator.save_parameters(\"infogan_g_latest.params\")\n```\n\n## Image similarity\nOnce the InfoGAN is trained, we can use the Discriminator to do an image similarity search. The idea is that the network learned meaningful features from the images based on the mutual information e.g. pose of people in an image.\n\nLoad the trained discriminator and retrieve one of its last layers.\n\n\n```{.python .input}\ndiscriminator = Discriminator()\ndiscriminator.load_parameters(\"infogan_d_latest.params\", device=device, ignore_extra=True)\n\ndiscriminator = discriminator.D[:11]\nprint (discriminator)\n\ndiscriminator.hybridize()\n```\n\nNearest neighbor function, which takes a matrix of features and an input feature vector. It returns the 3 closest features.\n\n\n```{.python .input}\ndef get_knn(features, input_vector, k=3):\n    dist = (np.square(features - input_vector).sum(axis=1))/features.shape[0]\n    indices = dist.asnumpy().argsort()[:k]\n    return [(index, dist[index].item()) for index in indices]\n```\n\nA helper function to visualize image data.\n\n\n```{.python .input}\ndef visualize(img_array):\n    plt.imshow(((img_array.asnumpy().transpose(1, 2, 0) + 1.0) * 127.5).astype(onp.uint8))\n    plt.axis('off')\n```\n\nTake some images from the test data, obtain its feature vector from `discriminator.D[:11]` and plot images of the corresponding closest vectors in the feature space.\n\n\n```{.python .input}\nfeature_size = 8192\n\nfeatures = np.zeros((len(test_images), feature_size), device=device)\n\nfor idx, image in enumerate(test_images):\n\n    feature = discriminator(np.array(image, device=device))\n    feature = feature.reshape(feature_size,)\n    features[idx,:] = feature.copyto(device)\n\n\nfor image in test_images[:100]:\n\n    feature = discriminator(np.array(image, device=device))\n    feature = feature.reshape((feature_size,))\n    image   = image.reshape((3,64,64))\n\n\n    indices = get_knn(features, feature, k=10)\n    fig = plt.figure(figsize=(15,12))\n    plt.subplot(1,10,1)\n\n    visualize(image)\n    for i in range(2,9):\n        if indices[i-1][1] < 1.5:\n            plt.subplot(1,10,i)\n            sim = test_images[indices[i-1][0]].reshape(3,64,64)\n            visualize(sim)\n    plt.show()\n    plt.clf()\n```\n![png](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/info_gan/output.png)<!--notebook-skip-line-->\n\n## How the Generator learns\nWe trained the Generator for a couple of epochs and stored a couple of fake images per epoch. Check the video.\n                    ![infogan infogan](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/info_gan/infogan.gif)\n\n\nThe following function computes the TSNE on the feature matrix and stores the result in a json-file. This file can be loaded with [TSNEViewer](https://ml4a.github.io/guides/ImageTSNEViewer/)\n\n\n```{.python}\nimport json\n\nfrom sklearn.manifold import TSNE\nfrom scipy.spatial import distance\n\ntsne = TSNE(n_components=2, learning_rate=150, perplexity=30, verbose=2).fit_transform(features.asnumpy())\n\n# save data to json\ndata = []\ncounter = 0\nfor i,f in enumerate(test_filenames):\n\n    point = [float((tsne[i,k] - onp.min(tsne[:,k]))/(onp.max(tsne[:,k]) - onp.min(tsne[:,k]))) for k in range(2) ]\n    data.append({\"path\": os.path.abspath(os.path.join(os.getcwd(),f)), \"point\": point})\n\nwith open(\"imagetsne.json\", 'w') as outfile:\n    json.dump(data, outfile)\n```\n\nLoad the file with TSNEViewer. You can now inspect whether similiar looking images are grouped nearby or not.\n\n<img src=\"https://raw.githubusercontent.com/NRauschmayr/web-data/master/mxnet/doc/tutorials/info_gan/tsne.png\" style=\"width:800px;height:600px;\">\n\n<!-- INSERT SOURCE DOWNLOAD BUTTONS -->\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/image/mnist.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Handwritten Digit Recognition\n\nIn this tutorial, we'll give you a step by step walk-through of how to build a hand-written digit classifier using the [MNIST](https://en.wikipedia.org/wiki/MNIST_database) dataset.\n\nMNIST is a widely used dataset for the hand-written digit classification task. It consists of 70,000 labeled 28x28 pixel grayscale images of hand-written digits. The dataset is split into 60,000 training images and 10,000 test images. There are 10 classes (one for each of the 10 digits). The task at hand is to train a model using the 60,000 training images and subsequently test its classification accuracy on the 10,000 test images.\n\n![mnist mnist](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/mnist.png)\n\n**Figure 1:** Sample images from the MNIST dataset.\n\nThis tutorial uses MXNet's new high-level interface, Gluon package to implement MLP using\nimperative fashion.\n\n## Prerequisites\nTo complete this tutorial, we need:\n\n- MXNet. See the instructions for your operating system in [Setup and Installation](https://mxnet.apache.org/get_started).\n\n- [Python Requests](https://requests.readthedocs.io/en/latest/) and [Jupyter Notebook](http://jupyter.org/index.html).\n\n```\n$ pip install requests jupyter\n```\n\n## Loading Data\n\nBefore we define the model, let's first fetch the [MNIST](http://yann.lecun.com/exdb/mnist/) dataset.\n\nThe following source code downloads and loads the images and the corresponding labels into memory.\n\n```{.python .input}\nimport os\nimport mxnet as mx\nfrom mxnet import gluon\nfrom mxnet.gluon.data.vision import transforms\n\n# Fixing the random seed\nmx.np.random.seed(42)\n\nmnist = mx.test_utils.get_mnist()\n```\n\nAfter running the above source code, the entire MNIST dataset should be fully loaded into memory. Note that for large datasets it is not feasible to pre-load the entire dataset first like we did here. What is needed is a mechanism by which we can quickly and efficiently stream data directly from the source. MXNet Data iterators come to the rescue here by providing exactly that. Data iterator is the mechanism by which we feed input data into an MXNet training algorithm and they are very simple to initialize and use and are optimized for speed. During training, we typically process training samples in small batches and over the entire training lifetime will end up processing each training example multiple times. In this tutorial, we'll configure the data iterator to feed examples in batches of 100. Keep in mind that each example is a 28x28 grayscale image and the corresponding label.\n\nImage batches are commonly represented by a 4-D array with shape `(batch_size, num_channels, width, height)`. For the MNIST dataset, since the images are grayscale, there is only one color channel. Also, the images are 28x28 pixels, and so each image has width and height equal to 28. Therefore, the shape of input is `(batch_size, 1, 28, 28)`. Another important consideration is the order of input samples. When feeding training examples, it is critical that we don't feed samples with the same label in succession. Doing so can slow down training.\nData iterators take care of this by randomly shuffling the inputs. Note that we only need to shuffle the training data. The order does not matter for test data.\n\nThe following source code initializes the data iterators for the MNIST dataset. Note that we initialize two iterators: one for train data and one for test data.\n\n```{.python .input}\ndef transform(data, label):\n    return data.astype(np.float32)/255, label.astype(np.float32)\n\nbatch_size = 100\nnum_workers = 8\ntrain_data = gluon.data.DataLoader(\n    gluon.data.vision.MNIST(train=True).transform_first(transforms.ToTensor()),\n    batch_size=batch_size, shuffle=True, num_workers=num_workers)\n\nval_data = gluon.data.DataLoader(\n    gluon.data.vision.MNIST(train=False).transform(transform),\n    batch_size=batch_size, shuffle=False, num_workers=num_workers)\n```\n\n## Approaches\n\nWe will cover a couple of approaches for performing the hand written digit recognition task. The first approach makes use of a traditional deep neural network architecture called Multilayer Perceptron (MLP). We'll discuss its drawbacks and use that as a motivation to introduce a second more advanced approach called Convolution Neural Network (CNN) that has proven to work very well for image classification tasks.\n\nNow, let's import required nn modules\n\n```{.python .input}\nfrom __future__ import print_function\nimport mxnet as mx\nfrom mxnet import gluon\nfrom mxnet.gluon import nn\nfrom mxnet import autograd as ag\n```\n\n### Define a network: Multilayer Perceptron\n\nThe first approach makes use of a [Multilayer Perceptron](https://en.wikipedia.org/wiki/Multilayer_perceptron) to solve this problem. We'll define the MLP using MXNet's imperative approach.\n\nMLPs consist of several fully connected layers. A fully connected layer or FC layer for short, is one where each neuron in the layer is connected to every neuron in its preceding layer. From a linear algebra perspective, an FC layer applies an [affine transform](https://en.wikipedia.org/wiki/Affine_transformation) to the *n x m* input matrix *X* and outputs a matrix *Y* of size *n x k*, where *k* is the number of neurons in the FC layer. *k* is also referred to as the hidden size. The output *Y* is computed according to the equation *Y = W X + b*. The FC layer has two learnable parameters, the *m x k* weight matrix *W* and the *m x 1* bias vector *b*.\n\nIn an MLP, the outputs of most FC layers are fed into an activation function, which applies an element-wise non-linearity. This step is critical and it gives neural networks the ability to classify inputs that are not linearly separable. Common choices for activation functions are sigmoid, tanh, and [rectified linear unit](https://en.wikipedia.org/wiki/Rectifier_%28neural_networks%29) (ReLU). In this example, we'll use the ReLU activation function which has several desirable properties and is typically considered a default choice.\n\nThe following code declares three fully connected layers with 128, 64 and 10 neurons each.\nThe last fully connected layer often has its hidden size equal to the number of output classes in the dataset. Furthermore, these FC layers uses ReLU activation for performing an element-wise ReLU transformation on the FC layer output.\n\nTo do this, we will use [Sequential layer](../../../../api/gluon/nn/index.rst#mxnet.gluon.nn.Sequential) type. This is simply a linear stack of neural network layers. `nn.Dense` layers are nothing but the fully connected layers we discussed above.\n\n```{.python .input}\n# define network\nnet = nn.Sequential()\nnet.add(nn.Dense(128, activation='relu'))\nnet.add(nn.Dense(64, activation='relu'))\nnet.add(nn.Dense(10))\n```\n\n#### Initialize parameters and optimizer\n\nThe following source code initializes all parameters received from parameter dict using [Xavier](../../../../api/initializer/index.rst#mxnet.initializer.Xavier) initializer\nto train the MLP network we defined above.\n\nFor our training, we will make use of the stochastic gradient descent (SGD) optimizer. In particular, we'll be using mini-batch SGD. Standard SGD processes train data one example at a time. In practice, this is very slow and one can speed up the process by processing examples in small batches. In this case, our batch size will be 100, which is a reasonable choice. Another parameter we select here is the learning rate, which controls the step size the optimizer takes in search of a solution. We'll pick a learning rate of 0.02, again a reasonable choice. Settings such as batch size and learning rate are what are usually referred to as hyper-parameters. What values we give them can have a great impact on training performance.\n\nWe will use [Trainer](../../../../api/gluon/trainer.rst) class to apply the\n[SGD optimizer](../../../../api/optimizer/index.rst#mxnet.optimizer.SGD) on the\ninitialized parameters.\n\n```{.python .input}\ngpus = mx.test_utils.list_gpus()\ndevice =  mx.gpu() if gpus else [mx.cpu(0), mx.cpu(1)]\nnet.initialize(mx.init.Xavier(magnitude=2.24), device=device)\ntrainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.02})\n```\n\n#### Train the network\n\nTypically, one runs the training until convergence, which means that we have learned a good set of model parameters (weights + biases) from the train data. For the purpose of this tutorial, we'll run training for 10 epochs and stop. An epoch is one full pass over the entire train data.\n\nWe will take following steps for training:\n\n- Define [Accuracy evaluation metric](../../../../api/gluon/metric/index.rst#mxnet.gluon.metric.Accuracy) over training data.\n- Loop over inputs for every epoch.\n- Forward input through network to get output.\n- Compute loss with output and label inside record scope.\n- Backprop gradient inside record scope.\n- Update evaluation metric and parameters with gradient descent.\n\nLoss function takes (output, label) pairs and computes a scalar loss for each sample in the mini-batch. The scalars measure how far each output is from the label.\nThere are many predefined loss functions in gluon.loss. Here we use\n[softmax_cross_entropy_loss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.SoftmaxCrossEntropyLoss) for digit classification. We will compute loss and do backward propagation inside\ntraining scope which is defined by `autograd.record()`.\n\n```{.python .input}\n%%time\nepoch = 10\n# Use Accuracy as the evaluation metric.\nmetric = mx.gluon.metric.Accuracy()\nsoftmax_cross_entropy_loss = gluon.loss.SoftmaxCrossEntropyLoss()\nfor i in range(epoch):\n    # Loop over the train data iterator.\n    for batch_num, (data, label) in enumerate(train_data):\n        outputs = []\n        # Inside training scope\n        with ag.record():\n            for x, y in zip(data, label):\n                z = net(x.to_device(device))\n                # Computes softmax cross entropy loss.\n                loss = softmax_cross_entropy_loss(z, y.to_device(device))\n                # Backpropagate the error for one iteration.\n                loss.backward()\n                outputs.append(z)\n        # Updates internal evaluation\n        metric.update(label, outputs)\n        # Make one step of parameter update. Trainer needs to know the\n        # batch size of data to normalize the gradient by 1/batch_size.\n        trainer.step(data.shape[0])\n    # Gets the evaluation result.\n    name, acc = metric.get()\n    # Reset evaluation result to initial state.\n    metric.reset()\n    print('training acc at epoch %d: %s=%f'%(i, name, acc))\n```\n\n#### Prediction\n\nAfter the above training completes, we can evaluate the trained model by running predictions on validation dataset. Since the dataset also has labels for all test images, we can compute the accuracy metric over validation data as follows:\n\n```{.python .input}\n# Use Accuracy as the evaluation metric.\nmetric = mx.gluon.metric.Accuracy()\n# Loop over the validation data iterator.\nfor batch_num, (data, label) in enumerate(val_data):\n    outputs = []\n    for x in data:\n        outputs.append(net(x.to_device(device)))\n    # Updates internal evaluation\n    metric.update(label, outputs)\nprint('validation acc: %s=%f'%metric.get())\nassert metric.get()[1] > 0.94\n```\n\nIf everything went well, we should see an accuracy value that is around 0.96, which means that we are able to accurately predict the digit in 96% of test images. This is a pretty good result. But as we will see in the next part of this tutorial, we can do a lot better than that.\n\n### Convolutional Neural Network\n\nEarlier, we briefly touched on a drawback of MLP when we said we need to discard the input image's original shape and flatten it as a vector before we can feed it as input to the MLP's first fully connected layer. Turns out this is an important issue because we don't take advantage of the fact that pixels in the image have natural spatial correlation along the horizontal and vertical axes. A convolutional neural network (CNN) aims to address this problem by using a more structured weight representation. Instead of flattening the image and doing a simple matrix-matrix multiplication, it employs one or more convolutional layers that each performs a 2-D convolution on the input image.\n\nA single convolution layer consists of one or more filters that each play the role of a feature detector. During training, a CNN learns appropriate representations (parameters) for these filters. Similar to MLP, the output from the convolutional layer is transformed by applying a non-linearity. Besides the convolutional layer, another key aspect of a CNN is the pooling layer. A pooling layer serves to make the CNN translation invariant: a digit remains the same even when it is shifted left/right/up/down by a few pixels. A pooling layer reduces a *n x m* patch into a single value to make the network less sensitive to the spatial location. Pooling layer is always included after each conv (+ activation) layer in the CNN.\n\nThe following source code defines a convolutional neural network architecture called LeNet. LeNet is a popular network known to work well on digit classification tasks. We will use a slightly different version from the original LeNet implementation, replacing the sigmoid activations with tanh activations for the neurons.\n\nA typical way to write your network is creating a new class inherited from `gluon.Block`\nclass. We can define the network by composing and inheriting Block class as follows:\n\n```{.python .input}\nfrom mxnet import np, npx\n\nclass Net(gluon.Block):\n    def __init__(self, **kwargs):\n        super(Net, self).__init__(**kwargs)\n        self.conv1 = nn.Conv2D(20, kernel_size=(5,5))\n        self.pool1 = nn.MaxPool2D(pool_size=(2,2), strides = (2,2))\n        self.conv2 = nn.Conv2D(50, kernel_size=(5,5))\n        self.pool2 = nn.MaxPool2D(pool_size=(2,2), strides = (2,2))\n        self.fc1 = nn.Dense(500)\n        self.fc2 = nn.Dense(10)\n\n    def forward(self, x):\n        x = self.pool1(np.tanh(self.conv1(x)))\n        x = self.pool2(np.tanh(self.conv2(x)))\n        # 0 means copy over size from corresponding dimension.\n        # -1 means infer size from the rest of dimensions.\n        x = x.reshape((-2, -1))\n        x = np.tanh(self.fc1(x))\n        x = np.tanh(self.fc2(x))\n        return x\n```\n\nWe just defined the forward function here, and the backward function to compute gradients\nis automatically defined for you using autograd.\nWe also imported `mxnet.ndarray` package to use activation functions from `ndarray` API.\n\nNow, We will create the network as follows:\n\n```{.python .input}\nnet = Net()\n```\n\n![mnist conv mnist](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/image/conv_mnist.png){ width=500px }\n\n**Figure 3:** First conv + pooling layer in LeNet.\n\nNow we train LeNet with similar hyper-parameters as before. Note that, if a GPU is available, we recommend using it. This greatly speeds up computation given that LeNet is more complex and compute-intensive than the previous multilayer perceptron. To do so, we only need to change `mx.cpu()` to `mx.gpu()` and MXNet takes care of the rest. Just like before, we'll stop training after 10 epochs.\n\nTraining and prediction can be done in the similar way as we did for MLP.\n\n#### Initialize parameters and optimizer\n\nWe will initialize the network parameters as follows:\n\n```{.python .input}\n# set the device on GPU is available otherwise CPU\ndevice = [mx.gpu() if mx.test_utils.list_gpus() else mx.cpu()]\nnet.initialize(mx.init.Xavier(magnitude=2.24), device=device)\ntrainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.03})\n```\n\n#### Training\n\n```{.python .input}\n# Use Accuracy as the evaluation metric.\nmetric = mx.gluon.metric.Accuracy()\nsoftmax_cross_entropy_loss = gluon.loss.SoftmaxCrossEntropyLoss()\n\nfor i in range(epoch):\n    # Loop over the train data iterator.\n    for batch_num, (data, label) in enumerate(train_data):\n        outputs = []\n        # Inside training scope\n        with ag.record():\n            for x, y in zip(data, label):\n                z = net(x.to_device(device))\n                # Computes softmax cross entropy loss.\n                loss = softmax_cross_entropy_loss(z, y.to_device(device))\n                # Backpropogate the error for one iteration.\n                loss.backward()\n                outputs.append(z)\n        # Updates internal evaluation\n        metric.update(label, outputs)\n        # Make one step of parameter update. Trainer needs to know the\n        # batch size of data to normalize the gradient by 1/batch_size.\n        trainer.step(data.shape[0])\n    # Gets the evaluation result.\n    name, acc = metric.get()\n    # Reset evaluation result to initial state.\n    metric.reset()\n    print('training acc at epoch %d: %s=%f'%(i, name, acc))\n```\n\n#### Prediction\n\nFinally, we'll use the trained LeNet model to generate predictions for the test data.\n\n```{.python .input}\n# Use Accuracy as the evaluation metric.\nmetric = mx.gluon.metric.Accuracy()\n# Loop over the validation data iterator.\nfor batch_num, (data, label) in enumerate(val_data):\n    outputs = []\n    for x in data:\n        outputs.append(net(x.to_device(device)))\n    # Updates internal evaluation\n    metric.update(label, outputs)\nprint('validation acc: %s=%f'%metric.get())\nassert metric.get()[1] > 0.98\n```\n\nIf all went well, we should see a higher accuracy metric for predictions made using LeNet. With CNN we should be able to correctly predict around 98% of all test images.\n\n## Summary\n\nIn this tutorial, we have learned how to use MXNet to solve a standard computer vision problem: classifying images of hand written digits. You have seen how to quickly and easily build, train and evaluate models such as MLP and CNN with MXNet Gluon package.\n\n<!-- INSERT SOURCE DOWNLOAD BUTTONS -->\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nGluon\n=====\n\nGetting started\n---------------\n\n.. container:: cards\n\n   .. card::\n      :title: A 60-minute Gluon crash course\n      :link: ../../getting-started/crash-course/index.html\n\n      Six 10-minute tutorials covering the core concepts of MXNet using the Gluon API.\n\n   .. card::\n      :title: Gluon - Neural network building blocks\n      :link: blocks/nn.html\n\n      An introduction to defining and training neural networks with Gluon.\n\n   .. card::\n      :title: Gluon: from experiment to deployment\n      :link: ../../getting-started/gluon_from_experiment_to_deployment.html\n\n      An end to end tutorial on working with the MXNet Gluon API.\n\n   .. card::\n      :title: Custom Layers for Beginners\n      :link: blocks/custom_layer_beginners.html\n\n      A guide to implementing custom layers for beginners.\n\n   .. card::\n      :title: Logistic regression using Gluon API explained\n      :link: ../../getting-started/logistic_regression_explained.html\n\n      Implementing logistic regression using the Gluon API.\n\n   .. card::\n      :title: Saving and Loading Gluon Models\n      :link: blocks/save_load_params.html\n\n      Saving and loading trained models.\n\nData\n----\n\n.. container:: cards\n\n   .. card::\n      :title: Data Augmentation\n      :link: data/data_augmentation.html\n\n      Boost your training dataset with image augmentation.\n\n   .. card::\n      :title: Gluon Datasets and DataLoader\n      :link: data/datasets.html\n\n      A guide to loading data using the Gluon API.\n\nTraining\n--------\n\n.. container:: cards\n\n   .. card::\n      :title: Neural Networks\n      :link: blocks/nn.html\n\n      How to use Layers and Blocks.\n\n   .. card::\n      :title: Normalization Blocks\n      :link: training/normalization/index.html\n\n      Understand usage of normalization layers (such as BatchNorm).\n\n   .. card::\n      :title: Activation Blocks\n      :link: blocks/activations/activations.html\n\n      Understand usage of activation layers (such as ReLU).\n\n   .. card::\n      :title: Loss Functions\n      :link: loss/loss.html\n\n      How to use loss functions for predicting outputs.\n\n   .. card::\n      :title: Initializing Parameters\n      :link: blocks/init.html\n\n      How to use the init function.\n\n   .. card::\n      :title: Parameter Management\n      :link: blocks/parameters.html\n\n      How to manage parameters.\n\n   .. card::\n      :title: Fit API Tutorial\n      :link: training/fit_api_tutorial.html\n\n      How to use the fit API\n\n   .. card::\n      :title: Learning Rate Finder\n      :link: training/learning_rates/learning_rate_finder.html\n\n      How to use the Learning Rate Finder to find a good learning rate.\n\n   .. card::\n      :title: Learning Rate Schedules\n      :link: training/learning_rates/learning_rate_schedules.html\n\n      How to schedule Learning Rate change over time.\n\n   .. card::\n      :title: Trainer\n      :link: training/trainer.html\n\n      How to update neural network parameters using an optimization method.\n\n   .. card::\n      :title: Autograd API\n      :link: ../autograd/index.html\n\n      How to use Automatic Differentiation with the Autograd API.\n\nAdvanced Topics\n---------------\n\n.. container:: cards\n\n   .. card::\n      :title: Naming\n      :link: blocks/naming.html\n\n      Best practices for the naming of things.\n\n   .. card::\n      :title: Custom Layers\n      :link: blocks/custom-layer.html\n\n      A guide to implementing custom layers.\n\n   .. card::\n      :title: Custom Operators\n      :link: ../../extend/customop.html\n\n      Building custom operators with numpy.\n\n   .. card::\n      :title: Custom Loss\n      :link: loss/custom-loss.html\n\n      A guide to implementing custom losses.\n\n   .. card::\n      :title: Hybridize\n      :link: blocks/hybridize.html\n\n      Speed up training with hybrid networks.\n\n   .. card::\n      :title: Learning Rate Schedules (Advanced)\n      :link: training/learning_rates/learning_rate_schedules_advanced.html\n\n      How to schedule Learning Rate change over time (advanced)\n\nApplications Topics\n-------------------\n\n.. container:: cards\n\n   .. card::\n      :title: Image Tutorials\n      :link: image/index.html\n\n      How to create deep learning models for images.\n\n   .. card::\n      :title: Text Tutorials\n      :link: text/index.html\n\n      How to create deep learning models for text.\n\n\n.. toctree::\n   :hidden:\n   :maxdepth: 3\n   :glob:\n\n   */index*\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/loss/custom-loss.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Custom Loss Blocks\n\nAll neural networks need a loss function for training. A loss function is a quantitive measure of how bad the predictions of the network are when compared to ground truth labels. Given this score, a network can improve by iteratively updating its weights to minimise this loss. Some tasks use a combination of multiple loss functions, but often you'll just use one. MXNet Gluon provides a number of the most commonly used loss functions, and you'll choose certain functions depending on your network and task. Some common task and loss function pairs include:\n\n- Regression: [L1Loss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.L1Loss), [L2Loss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.L2Loss)\n- Classification: [SigmoidBinaryCrossEntropyLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.SigmoidBinaryCrossEntropyLoss), [SoftmaxCrossEntropyLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.SoftmaxCrossEntropyLoss)\n- Embeddings: [HingeLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.HingeLoss)\n\nHowever, we may sometimes want to solve problems that require customized loss functions; this tutorial shows how we can do that in Gluon. We will implement contrastive loss which is typically used in Siamese networks.\n\n```{.python .input}\nimport matplotlib.pyplot as plt\nimport mxnet as mx\nfrom mxnet import autograd, gluon, np, npx\nfrom mxnet.gluon.loss import Loss\nimport random\n```\n\n### What is Contrastive Loss\n\n[Contrastive loss](http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf) is a distance-based loss function. During training, pairs of images are fed into a model. If the images are similar, the loss function will return 0, otherwise 1.\n\n![contrastive loss](/_static/contrastive_loss.jpeg)\n\n*Y* is a binary label indicating similarity between training images. Contrastive loss uses the Euclidean distance *D* between images and is the sum of 2 terms:\n - the loss for a pair of similar points\n - the loss for a pair of dissimilar points\n\nThe loss function uses a margin *m* which is has the effect that dissimlar pairs only contribute if their loss is within a certain margin.\n\nIn order to implement such a customized loss function in Gluon, we just need to define a new class that is inheriting from the [Loss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.Loss) base class. We then define the contrastive loss logic in the [forward](../../../../api/gluon/hybrid_block.rst#mxnet.gluon.HybridBlock.forward) method. This method takes the images `image1`, `image2` and the label which defines whether  `image1` and `image2` are similar (=0) or  dissimilar (=1). Gluon's `Loss` base class is in fact a [HybridBlock](../../../../api/gluon/hybrid_block.rst#mxnet.gluon.HybridBlock), and we hybridize our custom loss function, we can get performance speedups.\n\n\n```{.python .input}\nclass ContrastiveLoss(Loss):\n    def __init__(self, margin=6., weight=None, batch_axis=0, **kwargs):\n        super(ContrastiveLoss, self).__init__(weight, batch_axis, **kwargs)\n        self.margin = margin\n\n    def forward(self, image1, image2, label):\n        distances = image1 - image2\n        distances_squared = np.sum(np.square(distances), 1, keepdims=True)\n        euclidean_distances = np.sqrt(distances_squared + 0.0001)\n        d = np.clip(self.margin - euclidean_distances, 0, self.margin)\n        loss = (1 - label) * distances_squared + label * np.square(d)\n        loss = 0.5*loss\n        return loss\nloss = ContrastiveLoss(margin=6.0)\n```\n\n### Define the Siamese network\nA [Siamese network](https://papers.nips.cc/paper/769-signature-verification-using-a-siamese-time-delay-neural-network.pdf) consists of 2 identical networks, that share the same weights. They are trained on pairs of images and each network processes one image. The label defines whether the pair of images is similar or not. The Siamese network learns to differentiate between two input images.\n\nOur network consists of 2 convolutional and max pooling layers that downsample the input image. The output is then fed through a fully connected layer with 256 hidden units and another fully connected layer with 2 hidden units.\n\n\n```{.python .input}\nclass Siamese(gluon.HybridBlock):\n    def __init__(self, **kwargs):\n        super(Siamese, self).__init__(**kwargs)\n        self.cnn = gluon.nn.HybridSequential()\n        self.cnn.add(gluon.nn.Conv2D(64, 5, activation='relu'))\n        self.cnn.add(gluon.nn.MaxPool2D(2, 2))\n        self.cnn.add(gluon.nn.Conv2D(64, 5, activation='relu'))\n        self.cnn.add(gluon.nn.MaxPool2D(2, 2))\n        self.cnn.add(gluon.nn.Dense(256, activation='relu'))\n        self.cnn.add(gluon.nn.Dense(2, activation='softrelu'))\n\n    def forward(self, input0, input1):\n        out0 = self.cnn(input0)\n        out1 = self.cnn(input1)\n        return out0, out1\n\n```\n\n### Prepare the training data\n\nWe train our network on the [Ominglot](http://www.omniglot.com/) dataset which is a collection of 1623 hand drawn characters from 50 alphabets. You can download it from [here](https://github.com/brendenlake/omniglot/tree/master/python). We need to create a dataset that contains a random set of similar and dissimilar images. We use Gluon's `ImageFolderDataset` where we overwrite `__getitem__` and randomly return similar and dissimilar pairs of images.\n\n\n```{.python .input}\nclass GetImagePairs(mx.gluon.data.vision.ImageFolderDataset):\n    def __init__(self, root):\n        super(GetImagePairs, self).__init__(root, flag=0)\n        self.root = root\n\n    def __getitem__(self, index):\n        items_with_index = list(enumerate(self.items))\n        image0_index, image0_tuple = random.choice(items_with_index)\n        should_get_same_class = random.randint(0, 1)\n        if should_get_same_class:\n            while True:\n                image1_index, image1_tuple = random.choice(items_with_index)\n                if image0_tuple[1] == image1_tuple[1]:\n                    break\n        else:\n            image1_index, image1_tuple = random.choice(items_with_index)\n        image0 = super().__getitem__(image0_index)\n        image1 = super().__getitem__(image1_index)\n        label = mx.np.array([int(image1_tuple[1] != image0_tuple[1])])\n        return image0[0], image1[0], label\n\n    def __len__(self):\n        return super().__len__()\n\n```\n\nWe train the network on a subset of the data, the  [*Tifinagh*](https://www.omniglot.com/writing/tifinagh.htm) alphabet. Once the model is trained we test it on the [*Inuktitut*](https://www.omniglot.com/writing/inuktitut.htm) alphabet.\n\n\n```{.python .input}\ndef transform(img0, img1, label):\n    normalized_img0 = nd.transpose(img0.astype('float32'), (2, 0, 1))/255.0\n    normalized_img1 = nd.transpose(img1.astype('float32'), (2, 0, 1))/255.0\n    return normalized_img0, normalized_img1, label\n\ntraining_dir = \"images_background/Tifinagh\"\ntesting_dir = \"images_background/Inuktitut_(Canadian_Aboriginal_Syllabics)\"\ntrain = GetImagePairs(training_dir)\ntest = GetImagePairs(testing_dir)\ntrain_dataloader = gluon.data.DataLoader(train.transform(transform),\n                                        shuffle=True, batch_size=16)\ntest_dataloader = gluon.data.DataLoader(test.transform(transform),\n                                        shuffle=False, batch_size=1)\n```\n\nFollowing code plots some examples from the test dataset.\n\n\n```{.python .input}\nimg1, img2, label = test[0]\nprint(\"Same: {}\".format(int(label.item()) == 0))\nfig, (ax0, ax1) = plt.subplots(ncols=2, figsize=(10, 5))\nax0.imshow(img1.asnumpy()[:,:,0], cmap='gray')\nax0.axis('off')\nax1.imshow(img2.asnumpy()[:,:,0], cmap='gray')\nax1.axis(\"off\")\nplt.show()\n\n```\n\n![example1](/_static/inuktitut_1.png)\n\n\n### Train the Siamese network\n\nBefore we can start training, we need to instantiate the custom constrastive loss function and initialize the model.\n\n\n```{.python .input}\nmodel = Siamese()\nmodel.initialize(init=mx.init.Xavier())\ntrainer = gluon.Trainer(model.collect_params(), 'adam', {'learning_rate': 0.001})\nloss = ContrastiveLoss(margin=6.0)\n```\n\nStart the training loop:\n\n\n```{.python .input}\nfor epoch in range(10):\n    for i, data in enumerate(train_dataloader):\n        image1, image2, label = data\n        with autograd.record():\n            output1, output2 = model(image1, image2)\n            loss_contrastive = loss(output1, output2, label)\n        loss_contrastive.backward()\n        trainer.step(image1.shape[0])\n        loss_mean = loss_contrastive.mean().item()\n        print(\"Epoch number {}\\n Current loss {}\\n\".format(epoch, loss_mean))\n\n```\n\n### Test the trained Siamese network\nDuring inference we compute the Euclidean distance between the output vectors of the Siamese network. High distance indicates dissimilarity, low values indicate similarity.\n\n\n```{.python .input}\nfor i, data in enumerate(test_dataloader):\n    img1, img2, label = data\n    output1, output2 = model(img1, img2)\n    dist_sq = mx.np.sum(mx.np.square(output1 - output2))\n    dist = mx.np.sqrt(dist_sq).item()\n    print(\"Euclidean Distance:\", dist, \"Test label\", label[0].item())\n    fig, (ax0, ax1) = plt.subplots(ncols=2, figsize=(10, 5))\n    ax0.imshow(img1.asnumpy()[0, 0, :, :], cmap='gray')\n    ax0.axis('off')\n    ax1.imshow(img2.asnumpy()[0, 0, :, :], cmap='gray')\n    ax1.axis(\"off\")\n    plt.show()\n\n```\n\n![example2](/_static/inuktitut_2.png)\n\n\n### Common pitfalls with custom loss functions\n\nWhen customizing loss functions, we may encounter certain pitfalls. If the loss is not decreasing as expected or if forward/backward pass is crashing, then one should check the following:\n\n#### Activation function in the last layer\nVerify whether the last network layer uses the correct activation function: for instance in binary classification tasks we need to apply a sigmoid on the output data. If we use this activation in the last layer and define a loss function like Gluon's SigmoidBinaryCrossEntropy, we would basically apply sigmoid twice and the loss would not converge as expected. If we don't define any activation function, Gluon will per default apply a linear activation.\n\n####  Intermediate loss values\nIn our example, we computed the square root of squared distances between 2 images: `F.sqrt(distances_squared)`. If images are very similar we take the sqare root of a value close to 0, which can lead to *NaN* values. Adding a small epsilon to `distances_squared` avoids this problem.\n\n#### Shape of intermediate loss vectors\nIn most cases having the wrong tensor shape will lead to an error, as soon as we compare data with labels. But in some cases, we may be able to normally run the training, but it does not converge. For instance, if we don't set `keepdims=True` in our customized loss function, the shape of the tensor changes. The example still runs fine but does not converge.\n\nIf you encounter a similar problem, then it is useful to check the tensor shape after each computation step in the loss function.\n\n#### Differentiable\nBackprogration requires the loss function to be differentiable. If the customized loss function cannot be differentiated the backward pass will crash.\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/loss/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nLosses\n======\n\n.. toctree::\n   :maxdepth: 1\n   :glob:\n\n   *"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/loss/kl_divergence.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Kullback-Leibler (KL) Divergence\n\nKullback-Leibler (KL) Divergence is a measure of how one probability distribution is different from a second, reference probability distribution. Smaller KL Divergence values indicate more similar distributions and, since this loss function is differentiable, we can use gradient descent to minimize the KL divergence between network outputs and some target distribution. As an example, this can be used in Variational Autoencoders (VAEs), and reinforcement learning policy networks such as [Trust Region Policy Optimization (TRPO)](https://arxiv.org/abs/1502.05477).\n\nIn MXNet Gluon, we can use [KLDivLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.KLDivLoss) to compare categorical distributions. One important thing to note is that the KL Divergence is an asymmetric measure (i.e. `KL(P,Q) != KL(Q,P)`): order matters and we should compare our predicted distribution with our target distribution in that order. Another thing to note is that there are two ways to use [KLDivLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.KLDivLoss) that depend on how we set `from_logits` (which has a default value of true). \n\nAs an example, let's compare a few categorical distributions (`dist_1`, `dist_2` and `dist_3`), each with 4 categories.\n\n```{.python .input}\nfrom matplotlib import pyplot as plt\nimport mxnet as mx\nimport numpy as np\n\nidx = np.array([1, 2, 3, 4])\ndist_1 = np.array([0.2, 0.5, 0.2, 0.1])\ndist_2 = np.array([0.3, 0.4, 0.1, 0.2])\ndist_3 = np.array([0.1, 0.1, 0.1, 0.7])\n\nplt.figure(figsize=(10,5))\nplt.subplot(1,2,1)\nplt.ylim(top=1)\nplt.bar(idx, dist_1, alpha=0.5, color='black')\nplt.bar(idx, dist_2, alpha=0.5, color='aqua')\nplt.title('Distributions 1 & 2')\nplt.subplot(1,2,2)\nplt.ylim(top=1)\nplt.bar(idx, dist_1, alpha=0.5, color='black')\nplt.bar(idx, dist_3, alpha=0.5, color='aqua')\nplt.title('Distributions 1 & 3')\n```\n\nWe can see visually that distributions 1 and 2 are more similar than distributions 1 and 3. We'll confirm this result using [KLDivLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.KLDivLoss). When using [KLDivLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.KLDivLoss) with the default `from_logits=True` we need:\n\n1. our predictions to be parameters of a logged probability distribution.\n2. our targets to be parameters of a probability distribution (i.e. not logged).\n\nWe often apply a [softmax](../../../../api/npx/generated/mxnet.npx.softmax.rst) operation to the output of our network to get a distribution, but this can have a numerically unstable gradient calculation. As as stable alternative, we use [log_softmax](../../../../api/npx/generated/mxnet.npx.log_softmax.rst) and so this is what is expected by [KLDivLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.KLDivLoss) when `from_logits=True`. We also usually work with batches of predictions, so the predictions and targets need to have a batch dimension (the first axis by default).\n\nSince we're already working with distributions in this example, we don't need to apply the softmax and only need to apply [log](../../../../api/np/generated/mxnet.np.log.rst). And we'll create batch dimensions even though we're working with single distributions.\n\n```{.python .input}\ndef kl_divergence(dist_a, dist_b):\n    # add batch dimension\n    pred_batch = mx.np.expand_dims(mx.np.array(dist_a), axis=0)\n    target_batch = mx.np.expand_dims(mx.np.array(dist_b), axis=0)\n    # log the distribution\n    pred_batch = mx.np.log(pred_batch)\n    # create loss (assuming we have a logged prediction distribution)\n    loss_fn = mx.gluon.loss.KLDivLoss(from_logits=True)\n    divergence = loss_fn(pred_batch, target_batch)\n    return divergence.item()\n```\n\n```{.python .input}\nprint(\"Distribution 1 compared with Distribution 2: {}\".format(\n        kl_divergence(dist_1, dist_2)))\nprint(\"Distribution 1 compared with Distribution 3: {}\".format(\n        kl_divergence(dist_1, dist_3)))\nprint(\"Distribution 1 compared with Distribution 1: {}\".format(\n        kl_divergence(dist_1, dist_1)))\n```\n\nAs expected we see a smaller KL Divergence for distributions 1 & 2 than 1 & 3. And we also see the KL Divergence of a distribution with itself is 0.\n\n#### `from_logits=False`\n\nAlternatively, instead of manually applying the [log_softmax](../../../../api/npx/generated/mxnet.npx.log_softmax.rst) to our network outputs, we can leave that to the loss function. When setting `from_logits=False` on [KLDivLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.KLDivLoss), the [log_softmax](../../../../api/npx/generated/mxnet.npx.log_softmax.rst) is applied to the first argument passed to `loss_fn`. As an example, let's assume our network outputs us the values below (favorably chosen so that when we [softmax](../../../../api/npx/generated/mxnet.npx.softmax.rst) these values we get the same distribution parameters as `dist_1`).\n\n```{.python .input}\noutput = mx.np.array([0.39056206, 1.3068528, 0.39056206, -0.30258512])\n```\n\nWe can pass this to our [KLDivLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.KLDivLoss) loss function (with `from_logits=False`) and get the same KL Divergence between `dist_1` and `dist_2` as before, because the [log_softmax](../../../../api/npx/generated/mxnet.npx.log_softmax.rst) is applied within the loss function.\n\n```{.python .input}\ndef kl_divergence_not_from_logits(dist_a, dist_b):\n    # add batch dimension\n    pred_batch = mx.np.expand_dims(mx.np.array(dist_a), axis=0)\n    target_batch = mx.np.expand_dims(mx.np.array(dist_b), axis=0)\n    # create loss (assuming we have a logged prediction distribution)\n    loss_fn = mx.gluon.loss.KLDivLoss(from_logits=False)\n    divergence = loss_fn(pred_batch, target_batch)\n    return divergence.item()\n```\n\n```{.python .input}\nprint(\"Distribution 1 compared with Distribution 2: {}\".format(\n        kl_divergence_not_from_logits(output, dist_2)))\n```\n\n### Advanced: Common Support\n\nOccasionally, you might have issues with [KLDivLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.KLDivLoss). One common issue arises when the support of the distributions being compared are not the same. 'Support' here is referring to the values of the distribution which have a non-zero probability. Conveniently, all our examples above had the same support, but we might have a case where some categories have a probability of 0.\n\n\n```{.python .input}\ndist_4 = np.array([0, 0.9, 0, 0.1])\n```\n\n```{.python .input}\nprint(\"Distribution 4 compared with Distribution 1: {}\".format(\n        kl_divergence(dist_4, dist_1)))\n```\n\nWe can see that the result is `nan`, which will obviously cause issues when calculating the gradient. One option is to add a small value `epsilon` to all of the probabilities, and this is already done for the target distribution (using the value of 1e-12).\n\n### Advanced: Aggregation\n\nOne minor difference between the true definition of KL Divergence and the result from [KLDivLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.KLDivLoss) is how the aggregation of category contributions is performed. Although the true definition sums up these contributions, the default behaviour in MXNet Gluon is to average terms along the batch dimension. As a result, the [KLDivLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.KLDivLoss) output will be smaller than the true definition by a factor of the number of categories.\n\n```{.python .input}\ntrue_divergence = (dist_2*(np.log(dist_2)-np.log(dist_1))).sum()\nprint('true_divergence: {}'.format(true_divergence))\n```\n\n```{.python .input}\nnum_categories = dist_1.shape[0]\ndivergence = kl_divergence(dist_1, dist_2)\nprint('divergence: {}'.format(divergence))\nprint('divergence * num_categories: {}'.format(divergence * num_categories))\n``` \n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/loss/loss.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Loss functions\n\nLoss functions are used to train neural networks and to compute the difference between output and target variable. A critical component of training neural networks is the loss function. A loss function is a quantative measure of how bad the predictions of the network are when compared to ground truth labels. Given this score, a network can improve by iteratively updating its weights to minimise this loss. Some tasks use a combination of multiple loss functions, but often you'll just use one. MXNet Gluon provides a number of the most commonly used loss functions, and you'll choose certain loss functions depending on your network and task. Some common task and loss function pairs include:\n\n- Regression: [L1Loss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.L1Loss), [L2Loss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.L2Loss)\n- Classification: [SigmoidBinaryCrossEntropyLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.SigmoidBinaryCrossEntropyLoss), [SoftmaxCrossEntropyLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.SoftmaxCrossEntropyLoss)\n- Embeddings: [HingeLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.HingeLoss)\n\nWe'll first import the modules, where the `mxnet.gluon.loss` module is imported as `gloss` to avoid the commonly used name `loss`.\n\n```{.python .input}\nfrom IPython import display\nfrom matplotlib import pyplot as plt\nimport mxnet as mx\nfrom mxnet import np, npx, autograd\nfrom mxnet.gluon import nn, loss as gloss\n```\n\n## Basic Usages\n\nNow let's create an instance of the $\\ell_2$ loss, which is commonly used in regression tasks.\n\n```{.python .input}\nloss = gloss.L2Loss()\n```\n\nAnd then feed two inputs to compute the elementwise loss values.\n\n```{.python .input}\nx = np.ones((2,))\ny = np.ones((2,)) * 2\nloss(x, y)\n```\n\nThese values should be equal to the math definition: $0.5\\|x-y\\|^2$.\n\n```{.python .input}\n.5 * (x - y)**2\n```\n\nNext we show how to use a loss function to compute gradients.\n\n```{.python .input}\nX = np.random.uniform(size=(2, 4))\nnet = nn.Dense(1)\nnet.initialize()\nwith autograd.record():\n    l =  loss(net(X), y)\nprint(l)\n```\n\nWe can compute the gradients w.r.t. the loss function.\n\n```{.python .input}\nl.backward()\nprint(net.weight.grad())\n```\n\n## Loss functions\n\nMost commonly used loss functions can be divided into 2 categories: regression and classification.\n\nLet's first visualize several regression losses. We visualize the loss values versus the predicted values with label values fixed to be 0.\n\n```{.python .input}\ndef plot(x, y):\n    display.set_matplotlib_formats('svg')\n    plt.plot(x.asnumpy(), y.asnumpy())\n    plt.xlabel('x')\n    plt.ylabel('loss')\n    plt.show()\n\ndef show_regression_loss(loss):\n    x = np.arange(-5, 5, .1)\n    y = loss(x, np.zeros_like(x))\n    plot(x, y)\n\n```\n\nThen plot the classification losses with label values fixed to be 1.\n\n```{.python .input}\ndef show_classification_loss(loss):\n    x = np.arange(-5, 5, .1)\n    y = loss(x, np.ones_like(x))\n    plot(x, y)\n```\n\n#### [L1 Loss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.L1Loss)\n\nL1 Loss, also called Mean Absolute Error, computes the sum of absolute distance between target values and the output of the neural network. It is defined as:\n\n$$ L = \\sum_i \\vert {label}_i - {pred}_i \\vert. $$\n\nIt is a non-smooth function that can lead to non-convergence. It creates the same gradient for small and large loss values, which can be problematic for the learning process.\n\n```{.python .input}\nshow_regression_loss(gloss.L1Loss())\n```\n\n#### [L2 Loss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.L2Loss)\n\nL2Loss, also called Mean Squared Error, is a regression loss function that computes the squared distances between the target values and the output of the neural network. It is defined as:\n\n$$ L = \\frac{1}{2} \\sum_i \\vert {label}_i - {pred}_i \\vert^2. $$\n\nCompared to L1, L2 loss it is a smooth function and it creates larger gradients for large loss values. However due to the squaring it puts high weight on outliers.\n\n```{.python .input}\nshow_regression_loss(gloss.L2Loss())\n```\n\n#### [Huber Loss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.HuberLoss)\n\nHuberLoss  combines advantages of L1 and L2 loss. It calculates a smoothed L1 loss that is equal to L1 if the absolute error exceeds a threshold $$\\rho$$, otherwise it is equal to L2. It is defined as:\n$$\n\\begin{split}L = \\sum_i \\begin{cases} \\frac{1}{2 {rho}} ({label}_i - {pred}_i)^2 &\n                   \\text{ if } |{label}_i - {pred}_i| < {rho} \\\\\n                   |{label}_i - {pred}_i| - \\frac{{rho}}{2} &\n                   \\text{ otherwise }\n    \\end{cases}\\end{split}\n$$\n\n```{.python .input}\nshow_regression_loss(gloss.HuberLoss(rho=1))\n```\n\nAn example of where Huber Loss is used can be found in [Deep Q Network](https://openai.com/blog/openai-baselines-dqn/).\n\n#### [Cross Entropy Loss with Sigmoid](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.SigmoidBinaryCrossEntropyLoss)\n\nBinary Cross Entropy is a loss function used for binary classification problems e.g. classifying images into 2 classes. Cross entropy measures the difference between two probability distributions and it is defined as:\n$$\\sum_i -{(y\\log(p) + (1 - y)\\log(1 - p))} $$\nBefore the loss is computed a sigmoid activation is applied per default. If your network has `sigmoid` activation as last layer, then you need set ```from_sigmoid``` to False, to avoid applying the sigmoid function twice.\n\n```{.python .input}\nshow_classification_loss(gloss.SigmoidBinaryCrossEntropyLoss())\n```\n\n#### [Cross Entropy Loss with Softmax](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.SoftmaxCrossEntropyLoss)\n\nIn classification, we often apply the\nsoftmax operator to the predicted outputs to obtain prediction probabilities,\nand then apply the cross entropy loss against the true labels:\n\n$$ \\begin{align}\\begin{aligned}p = \\text{softmax}({pred})\\\\L = -\\sum_i \\sum_j {label}_j \\log p_{ij}\\end{aligned}\\end{align}\n$$\n\nRunning these two steps one-by-one, however, may lead to numerical instabilities. The `loss` module provides a single operators with softmax and cross entropy fused to avoid such problem.\n\n```{.python .input}\nloss = gloss.SoftmaxCrossEntropyLoss()\nx = np.array([[1, 10], [8, 2]])\ny = np.array([0, 1])\nloss(x, y)\n```\n\n#### [Hinge Loss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.HingeLoss)\n\nCommonly used in Support Vector Machines (SVMs), Hinge Loss is used to additionally penalize predictions that are correct but fall within a margin between classes (the region around a decision boundary). Unlike `SoftmaxCrossEntropyLoss`, it's rarely used for neural network training. It is defined as:\n\n$$\nL = \\sum_i max(0, {margin} - {pred}_i \\cdot {label}_i)\n$$\n\n```{.python .input}\nshow_classification_loss(gloss.HingeLoss())\n```\n\n#### [Logistic Loss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.LogisticLoss)\n\nThe Logistic Loss function computes the performance of binary classification models.\n$$\nL = \\sum_i \\log(1 + \\exp(- {pred}_i \\cdot {label}_i))\n$$\nThe log loss decreases the closer the prediction is to the actual label. It is sensitive to outliers, because incorrectly classified points are penalized more.\n\n```{.python .input}\nshow_classification_loss(gloss.LogisticLoss())\n```\n\n#### [Kullback-Leibler Divergence Loss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.KLDivLoss)\n\nThe Kullback-Leibler divergence loss measures the divergence between two probability distributions by calculating the difference between cross entropy and entropy. It takes as input the probability of predicted label and the probability of true label.\n\n$$\nL = \\sum_i {label}_i * \\big[\\log({label}_i) - {pred}_i\\big]\n$$\n\nThe loss is large, if the predicted probability distribution is far from the ground truth probability distribution. KL divergence is an asymmetric measure. KL divergence loss can be used in Variational Autoencoders (VAEs), and reinforcement learning policy networks such as Trust Region Policy Optimization (TRPO)\n\n\nFor instance, in the following example we get a KL divergence of 0.02. We set ```from_logits=False```, so the loss functions will apply ```log_softmax``` on the network output, before computing the KL divergence.\n\n```{.python .input}\noutput = mx.np.array([[0.39056206, 1.3068528, 0.39056206, -0.30258512]])\nprint('output.softmax(): {}'.format(npx.softmax(output).asnumpy().tolist()))\ntarget_dist = mx.np.array([[0.3, 0.4, 0.1, 0.2]])\nloss_fn = gloss.KLDivLoss(from_logits=False)\nloss = loss_fn(output, target_dist)\nprint('loss (kl divergence): {}'.format(loss.asnumpy().tolist()))\n```\n\n#### [Triplet Loss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.TripletLoss)\n\nTriplet loss takes three input arrays and measures the relative similarity. It takes a positive and negative input and the anchor.\n\n$$\nL = \\sum_i \\max(\\Vert {pos_i}_i - {pred} \\Vert_2^2 -\n    \\Vert {neg_i}_i - {pred} \\Vert_2^2 + {margin}, 0)\n$$\n\nThe loss function minimizes the distance between similar inputs and maximizes the distance  between dissimilar ones.\nIn the case of learning embeddings for images of characters, the network may get as input the following 3 images:\n\n![triplet_loss](/_static/triplet_loss.png)\n\nThe network would learn to minimize the distance between the two `A`'s and maximize the distance between `A` and `Z`.\n\n#### [CTC Loss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.CTCLoss)\n\nCTC Loss is the [connectionist temporal classification loss](https://distill.pub/2017/ctc/) . It is used to train recurrent neural networks with variable time dimension. It learns the alignment and labelling of input sequences. It takes a sequence as input and gives probabilities for each timestep. For instance, in the following image the word is not well aligned with the 5 timesteps because of the different sizes of characters. CTC Loss finds for each timestep the highest probability e.g. `t1` presents with high probability a `C`. It combines the highest probapilities and returns the best path decoding.\n\n![ctc_loss](/_static/ctc_loss.png)\n\n#### [Cosine Embedding Loss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.CosineEmbeddingLoss)\nThe cosine embedding loss computes the cosine distance between two input vectors.\n\n$$\n\\begin{split}L = \\sum_i \\begin{cases} 1 - {cos\\_sim({input1}_i, {input2}_i)} & \\text{ if } {label}_i = 1\\\\\n                 {cos\\_sim({input1}_i, {input2}_i)} & \\text{ if } {label}_i = -1 \\end{cases}\\\\\ncos\\_sim(input1, input2) = \\frac{{input1}_i.{input2}_i}{||{input1}_i||.||{input2}_i||}\\end{split}\n$$\n\nCosine distance measures the similarity between two arrays given a label and is typically used for learning nonlinear embeddings.\nFor instance, in the following code example we measure the similarity between the input vectors `x` and `y`. Since they are the same the label equals `1`. The loss function returns $$ \\sum_i 1 - {cos\\_sim({input1}_i, {input2}_i)} $$ which is equal `0`.\n\n```{.python .input}\nx = mx.np.array([1,0,1,0,1,0])\ny = mx.np.array([1,0,1,0,1,0])\nlabel = mx.np.array([1])\nloss = gloss.CosineEmbeddingLoss()\nprint(loss(x,y,label))\n```\n\nNow let's make `y` the opposite of `x`, so we set the label `-1` and the function will return  $$ \\sum_i cos\\_sim(input1, input2) $$\n\n```{.python .input}\nx = mx.np.array([1,0,1,0,1,0])\ny = mx.np.array([0,1,0,1,0,1])\nlabel = mx.np.array([-1])\nloss = gloss.CosineEmbeddingLoss()\nprint(loss(x,y,label))\n```\n\n#### [PoissonNLLLoss](../../../../api/gluon/loss/index.rst#mxnet.gluon.loss.PoissonNLLLoss)\nPoisson distribution is widely used for modelling count data. It is defined as:\n\n$$\nf(x) = \\frac{\\mu ^ {\\kern 0.08 em x} e ^ {-\\mu}} {x!} \\qquad \\qquad x = 0,1,2 , \\ldots \\,.\n$$\n\n\nFor instance, the count of cars in road traffic approximately follows a Poisson distribution. Using an ordinary least squares model for Poisson distributed data would not work well because of two reasons:\n - count data cannot be negative\n - variance may not be constant\n\nInstead we can use a Poisson regression model, also known as log-linear model. Thereby the Poisson incident rate $$\\mu$$ is\nmodelled by a linear combination of unknown parameters.\nWe can then use the PoissonNLLLoss which calculates the negative log likelihood for a target that follows a Poisson distribution.\n\n$$ L = \\text{pred} - \\text{target} * \\log(\\text{pred}) +\\log(\\text{target!}) $$\n\n## Advanced: Weighted Loss\n\nSome examples in a batch may be more important than others. We can apply weights to individual examples during the forward pass of the loss function using the `sample_weight` argument. All examples are weighted equally by default.\n\n```{.python .input}\nx = np.ones((2,))\ny = np.ones((2,)) * 2\nloss = gloss.L2Loss()\nloss(x, y, np.array([1, 2]))\n```\n\n## Conclusion\n\nIn this tutorial we saw an example of how to evaluate model performance using loss functions (during the forward pass). Crucially, we then saw how calculate parameter gradients (using `backward`) which would minimise this loss. You should now have a better understanding of when to apply different loss functions, especially for regression vs classification tasks.\n\n## Recommended Next Steps\n\nIn addition to loss functions, which are used for explicit optimization, you might want to look at metrics that give useful evaluation feedback even if they're not explicitly optimized for in the same way as the loss. You might also want to learn more about the mechanics of the backpropagation stage in the autograd tutorial.\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/text/gnmt.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nGoogle Neural Machine Translation\n=================================\n\nIn this notebook, we are going to train Google NMT on IWSLT 2015\nEnglish-Vietnamese Dataset. The building process includes four steps: 1)\nload and process dataset, 2) create sampler and DataLoader, 3) build\nmodel, and 4) write training epochs.\n\nLoad MXNET and Gluon\n--------------------\n\n.. code:: python\n\n    import warnings\n    warnings.filterwarnings('ignore')\n\n    import argparse\n    import time\n    import random\n    import os\n    import logging\n    import numpy as np\n    import mxnet as mx\n    from mxnet import gluon\n    import gluonnlp as nlp\n    import nmt\n\nHyper-parameters\n----------------\n\n.. code:: python\n\n    np.random.seed(100)\n    random.seed(100)\n    mx.random.seed(10000)\n    device = mx.gpu(0)\n\n    # parameters for dataset\n    dataset = 'IWSLT2015'\n    src_lang, tgt_lang = 'en', 'vi'\n    src_max_len, tgt_max_len = 50, 50\n\n    # parameters for model\n    num_hidden = 512\n    num_layers = 2\n    num_bi_layers = 1\n    dropout = 0.2\n\n    # parameters for training\n    batch_size, test_batch_size = 128, 32\n    num_buckets = 5\n    epochs = 1\n    clip = 5\n    lr = 0.001\n    lr_update_factor = 0.5\n    log_interval = 10\n    save_dir = 'gnmt_en_vi_u512'\n\n    #parameters for testing\n    beam_size = 10\n    lp_alpha = 1.0\n    lp_k = 5\n\n    nmt.utils.logging_config(save_dir)\n\nLoad and Preprocess Dataset\n---------------------------\n\nThe following shows how to process the dataset and cache the processed\ndataset for future use. The processing steps include: 1) clip the source\nand target sequences, 2) split the string input to a list of tokens, 3)\nmap the string token into its integer index in the vocabulary, and 4)\nappend end-of-sentence (EOS) token to source sentence and add BOS and\nEOS tokens to target sentence.\n\n.. code:: python\n\n    def cache_dataset(dataset, prefix):\n        \"\"\"Cache the processed npy dataset  the dataset into a npz\n\n        Parameters\n        ----------\n        dataset : gluon.data.SimpleDataset\n        file_path : str\n        \"\"\"\n        if not os.path.exists(nmt._constants.CACHE_PATH):\n            os.makedirs(nmt._constants.CACHE_PATH)\n        src_data = np.array([ele[0] for ele in dataset])\n        tgt_data = np.array([ele[1] for ele in dataset])\n        np.savez(os.path.join(nmt._constants.CACHE_PATH, prefix + '.npz'), src_data=src_data, tgt_data=tgt_data)\n\n\n    def load_cached_dataset(prefix):\n        cached_file_path = os.path.join(nmt._constants.CACHE_PATH, prefix + '.npz')\n        if os.path.exists(cached_file_path):\n            print('Load cached data from {}'.format(cached_file_path))\n            dat = np.load(cached_file_path)\n            return gluon.data.ArrayDataset(np.array(dat['src_data']), np.array(dat['tgt_data']))\n        else:\n            return None\n\n\n    class TrainValDataTransform(object):\n        \"\"\"Transform the machine translation dataset.\n\n        Clip source and the target sentences to the maximum length. For the source sentence, append the\n        EOS. For the target sentence, append BOS and EOS.\n\n        Parameters\n        ----------\n        src_vocab : Vocab\n        tgt_vocab : Vocab\n        src_max_len : int\n        tgt_max_len : int\n        \"\"\"\n        def __init__(self, src_vocab, tgt_vocab, src_max_len, tgt_max_len):\n            self._src_vocab = src_vocab\n            self._tgt_vocab = tgt_vocab\n            self._src_max_len = src_max_len\n            self._tgt_max_len = tgt_max_len\n\n        def __call__(self, src, tgt):\n            if self._src_max_len > 0:\n                src_sentence = self._src_vocab[src.split()[:self._src_max_len]]\n            else:\n                src_sentence = self._src_vocab[src.split()]\n            if self._tgt_max_len > 0:\n                tgt_sentence = self._tgt_vocab[tgt.split()[:self._tgt_max_len]]\n            else:\n                tgt_sentence = self._tgt_vocab[tgt.split()]\n            src_sentence.append(self._src_vocab[self._src_vocab.eos_token])\n            tgt_sentence.insert(0, self._tgt_vocab[self._tgt_vocab.bos_token])\n            tgt_sentence.append(self._tgt_vocab[self._tgt_vocab.eos_token])\n            src_npy = np.array(src_sentence, dtype=np.int32)\n            tgt_npy = np.array(tgt_sentence, dtype=np.int32)\n            return src_npy, tgt_npy\n\n\n    def process_dataset(dataset, src_vocab, tgt_vocab, src_max_len=-1, tgt_max_len=-1):\n        start = time.time()\n        dataset_processed = dataset.transform(TrainValDataTransform(src_vocab, tgt_vocab,\n                                                                    src_max_len,\n                                                                    tgt_max_len), lazy=False)\n        end = time.time()\n        print('Processing time spent: {}'.format(end - start))\n        return dataset_processed\n\n\n    def load_translation_data(dataset, src_lang='en', tgt_lang='vi'):\n        \"\"\"Load translation dataset\n\n        Parameters\n        ----------\n        dataset : str\n        src_lang : str, default 'en'\n        tgt_lang : str, default 'vi'\n\n        Returns\n        -------\n        data_train_processed : Dataset\n            The preprocessed training sentence pairs\n        data_val_processed : Dataset\n            The preprocessed validation sentence pairs\n        data_test_processed : Dataset\n            The preprocessed test sentence pairs\n        val_tgt_sentences : list\n            The target sentences in the validation set\n        test_tgt_sentences : list\n            The target sentences in the test set\n        src_vocab : Vocab\n            Vocabulary of the source language\n        tgt_vocab : Vocab\n            Vocabulary of the target language\n        \"\"\"\n        common_prefix = 'IWSLT2015_{}_{}_{}_{}'.format(src_lang, tgt_lang,\n                                                       src_max_len, tgt_max_len)\n        data_train = nlp.data.IWSLT2015('train', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_val = nlp.data.IWSLT2015('val', src_lang=src_lang, tgt_lang=tgt_lang)\n        data_test = nlp.data.IWSLT2015('test', src_lang=src_lang, tgt_lang=tgt_lang)\n        src_vocab, tgt_vocab = data_train.src_vocab, data_train.tgt_vocab\n        data_train_processed = load_cached_dataset(common_prefix + '_train')\n        if not data_train_processed:\n            data_train_processed = process_dataset(data_train, src_vocab, tgt_vocab,\n                                                   src_max_len, tgt_max_len)\n            cache_dataset(data_train_processed, common_prefix + '_train')\n        data_val_processed = load_cached_dataset(common_prefix + '_val')\n        if not data_val_processed:\n            data_val_processed = process_dataset(data_val, src_vocab, tgt_vocab)\n            cache_dataset(data_val_processed, common_prefix + '_val')\n        data_test_processed = load_cached_dataset(common_prefix + '_test')\n        if not data_test_processed:\n            data_test_processed = process_dataset(data_test, src_vocab, tgt_vocab)\n            cache_dataset(data_test_processed, common_prefix + '_test')\n        fetch_tgt_sentence = lambda src, tgt: tgt.split()\n        val_tgt_sentences = list(data_val.transform(fetch_tgt_sentence))\n        test_tgt_sentences = list(data_test.transform(fetch_tgt_sentence))\n        return data_train_processed, data_val_processed, data_test_processed, \\\n               val_tgt_sentences, test_tgt_sentences, src_vocab, tgt_vocab\n\n\n    def get_data_lengths(dataset):\n        return list(dataset.transform(lambda srg, tgt: (len(srg), len(tgt))))\n\n\n    data_train, data_val, data_test, val_tgt_sentences, test_tgt_sentences, src_vocab, tgt_vocab\\\n        = load_translation_data(dataset=dataset, src_lang=src_lang, tgt_lang=tgt_lang)\n    data_train_lengths = get_data_lengths(data_train)\n    data_val_lengths = get_data_lengths(data_val)\n    data_test_lengths = get_data_lengths(data_test)\n\n    with open(os.path.join(save_dir, 'val_gt.txt'), 'w', encoding='utf-8') as of:\n        for ele in val_tgt_sentences:\n            of.write(' '.join(ele) + '\\n')\n\n    with open(os.path.join(save_dir, 'test_gt.txt'), 'w', encoding='utf-8') as of:\n        for ele in test_tgt_sentences:\n            of.write(' '.join(ele) + '\\n')\n\n\n    data_train = data_train.transform(lambda src, tgt: (src, tgt, len(src), len(tgt)), lazy=False)\n    data_val = gluon.data.SimpleDataset([(ele[0], ele[1], len(ele[0]), len(ele[1]), i)\n                                         for i, ele in enumerate(data_val)])\n    data_test = gluon.data.SimpleDataset([(ele[0], ele[1], len(ele[0]), len(ele[1]), i)\n                                          for i, ele in enumerate(data_test)])\n\nCreate Sampler and DataLoader\n-----------------------------\n\nNow, we have obtained ``data_train``, ``data_val``, and ``data_test``.\nThe next step is to construct sampler and DataLoader. The first step is\nto construct batchify function, which pads and stacks sequences to form\nmini-batch.\n\n.. code:: python\n\n    train_batchify_fn = nlp.data.batchify.Tuple(nlp.data.batchify.Pad(),\n                                                nlp.data.batchify.Pad(),\n                                                nlp.data.batchify.Stack(dtype='float32'),\n                                                nlp.data.batchify.Stack(dtype='float32'))\n    test_batchify_fn = nlp.data.batchify.Tuple(nlp.data.batchify.Pad(),\n                                               nlp.data.batchify.Pad(),\n                                               nlp.data.batchify.Stack(dtype='float32'),\n                                               nlp.data.batchify.Stack(dtype='float32'),\n                                               nlp.data.batchify.Stack())\n\nWe can then construct bucketing samplers, which generate batches by\ngrouping sequences with similar lengths. Here, the bucketing scheme is\nempirically determined.\n\n.. code:: python\n\n    bucket_scheme = nlp.data.ExpWidthBucket(bucket_len_step=1.2)\n    train_batch_sampler = nlp.data.FixedBucketSampler(lengths=data_train_lengths,\n                                                      batch_size=batch_size,\n                                                      num_buckets=num_buckets,\n                                                      shuffle=True,\n                                                      bucket_scheme=bucket_scheme)\n    logging.info('Train Batch Sampler:\\n{}'.format(train_batch_sampler.stats()))\n    val_batch_sampler = nlp.data.FixedBucketSampler(lengths=data_val_lengths,\n                                                    batch_size=test_batch_size,\n                                                    num_buckets=num_buckets,\n                                                    shuffle=False)\n    logging.info('Valid Batch Sampler:\\n{}'.format(val_batch_sampler.stats()))\n    test_batch_sampler = nlp.data.FixedBucketSampler(lengths=data_test_lengths,\n                                                     batch_size=test_batch_size,\n                                                     num_buckets=num_buckets,\n                                                     shuffle=False)\n    logging.info('Test Batch Sampler:\\n{}'.format(test_batch_sampler.stats()))\n\nGiven the samplers, we can create DataLoader, which is iterable.\n\n.. code:: python\n\n    train_data_loader = gluon.data.DataLoader(data_train,\n                                              batch_sampler=train_batch_sampler,\n                                              batchify_fn=train_batchify_fn,\n                                              num_workers=4)\n    val_data_loader = gluon.data.DataLoader(data_val,\n                                            batch_sampler=val_batch_sampler,\n                                            batchify_fn=test_batchify_fn,\n                                            num_workers=4)\n    test_data_loader = gluon.data.DataLoader(data_test,\n                                             batch_sampler=test_batch_sampler,\n                                             batchify_fn=test_batchify_fn,\n                                             num_workers=4)\n\nBuild GNMT Model\n----------------\n\nAfter obtaining DataLoader, we can build the model. The GNMT encoder and\ndecoder can be easily constructed by calling\n``get_gnmt_encoder_decoder`` function. Then, we feed the encoder and\ndecoder to ``NMTModel`` to construct the GNMT model. ``model.hybridize``\nallows computation to be done using the symbolic backend.\n\n.. code:: python\n\n    encoder, decoder = nmt.gnmt.get_gnmt_encoder_decoder(hidden_size=num_hidden,\n                                                         dropout=dropout,\n                                                         num_layers=num_layers,\n                                                         num_bi_layers=num_bi_layers)\n    model = nmt.translation.NMTModel(src_vocab=src_vocab, tgt_vocab=tgt_vocab, encoder=encoder, decoder=decoder,\n                                     embed_size=num_hidden, prefix='gnmt_')\n    model.initialize(init=mx.init.Uniform(0.1), device=device)\n    static_alloc = True\n    model.hybridize(static_alloc=static_alloc)\n    logging.info(model)\n\n    # Due to the paddings, we need to mask out the losses corresponding to padding tokens.\n    loss_function = nmt.loss.SoftmaxCEMaskedLoss()\n    loss_function.hybridize(static_alloc=static_alloc)\n\nWe also build the beam search translator.\n\n.. code:: python\n\n    translator = nmt.translation.BeamSearchTranslator(model=model, beam_size=beam_size,\n                                                      scorer=nlp.model.BeamSearchScorer(alpha=lp_alpha,\n                                                                                        K=lp_k),\n                                                      max_length=tgt_max_len + 100)\n    logging.info('Use beam_size={}, alpha={}, K={}'.format(beam_size, lp_alpha, lp_k))\n\nWe define evaluation function as follows. The ``evaluate`` function use\nbeam search translator to generate outputs for the validation and\ntesting datasets.\n\n.. code:: python\n\n    def evaluate(data_loader):\n        \"\"\"Evaluate given the data loader\n\n        Parameters\n        ----------\n        data_loader : gluon.data.DataLoader\n\n        Returns\n        -------\n        avg_loss : float\n            Average loss\n        real_translation_out : list of list of str\n            The translation output\n        \"\"\"\n        translation_out = []\n        all_inst_ids = []\n        avg_loss_denom = 0\n        avg_loss = 0.0\n        for _, (src_seq, tgt_seq, src_valid_length, tgt_valid_length, inst_ids) \\\n                in enumerate(data_loader):\n            src_seq = src_seq.to_device(device)\n            tgt_seq = tgt_seq.to_device(device)\n            src_valid_length = src_valid_length.to_device(device)\n            tgt_valid_length = tgt_valid_length.to_device(device)\n            # Calculating Loss\n            out, _ = model(src_seq, tgt_seq[:, :-1], src_valid_length, tgt_valid_length - 1)\n            loss = loss_function(out, tgt_seq[:, 1:], tgt_valid_length - 1).mean().asscalar()\n            all_inst_ids.extend(inst_ids.asnumpy().astype(np.int32).tolist())\n            avg_loss += loss * (tgt_seq.shape[1] - 1)\n            avg_loss_denom += (tgt_seq.shape[1] - 1)\n            # Translate\n            samples, _, sample_valid_length =\\\n                translator.translate(src_seq=src_seq, src_valid_length=src_valid_length)\n            max_score_sample = samples[:, 0, :].asnumpy()\n            sample_valid_length = sample_valid_length[:, 0].asnumpy()\n            for i in range(max_score_sample.shape[0]):\n                translation_out.append(\n                    [tgt_vocab.idx_to_token[ele] for ele in\n                     max_score_sample[i][1:(sample_valid_length[i] - 1)]])\n        avg_loss = avg_loss / avg_loss_denom\n        real_translation_out = [None for _ in range(len(all_inst_ids))]\n        for ind, sentence in zip(all_inst_ids, translation_out):\n            real_translation_out[ind] = sentence\n        return avg_loss, real_translation_out\n\n\n    def write_sentences(sentences, file_path):\n        with open(file_path, 'w', encoding='utf-8') as of:\n            for sent in sentences:\n                of.write(' '.join(sent) + '\\n')\n\nTraining Epochs\n---------------\n\nBefore entering the training stage, we need to create trainer for\nupdating the parameters. In the following example, we create a trainer\nthat uses ADAM optimzier.\n\n.. code:: python\n\n    trainer = gluon.Trainer(model.collect_params(), 'adam', {'learning_rate': lr})\n\nWe can then write the training loop. During the training, we evaluate on\nthe validation and testing datasets every epoch, and record the\nparameters that give the hightest BLEU score on the validation dataset.\nBefore performing forward and backward, we first use ``to_device``\nfunction to copy the mini-batch to GPU. The statement\n``with mx.autograd.record()`` tells Gluon backend to compute the\ngradients for the part inside the block.\n\n.. code:: python\n\n    best_valid_bleu = 0.0\n    for epoch_id in range(epochs):\n        log_avg_loss = 0\n        log_avg_gnorm = 0\n        log_wc = 0\n        log_start_time = time.time()\n        for batch_id, (src_seq, tgt_seq, src_valid_length, tgt_valid_length)\\\n                in enumerate(train_data_loader):\n            # logging.info(src_seq.context) Context suddenly becomes GPU.\n            src_seq = src_seq.to_device(device)\n            tgt_seq = tgt_seq.to_device(device)\n            src_valid_length = src_valid_length.to_device(device)\n            tgt_valid_length = tgt_valid_length.to_device(device)\n            with mx.autograd.record():\n                out, _ = model(src_seq, tgt_seq[:, :-1], src_valid_length, tgt_valid_length - 1)\n                loss = loss_function(out, tgt_seq[:, 1:], tgt_valid_length - 1).mean()\n                loss = loss * (tgt_seq.shape[1] - 1) / (tgt_valid_length - 1).mean()\n                loss.backward()\n            grads = [p.grad(device) for p in model.collect_params().values()]\n            gnorm = gluon.utils.clip_global_norm(grads, clip)\n            trainer.step(1)\n            src_wc = src_valid_length.sum().asscalar()\n            tgt_wc = (tgt_valid_length - 1).sum().asscalar()\n            step_loss = loss.asscalar()\n            log_avg_loss += step_loss\n            log_avg_gnorm += gnorm\n            log_wc += src_wc + tgt_wc\n            if (batch_id + 1) % log_interval == 0:\n                wps = log_wc / (time.time() - log_start_time)\n                logging.info('[Epoch {} Batch {}/{}] loss={:.4f}, ppl={:.4f}, gnorm={:.4f}, '\n                             'throughput={:.2f}K wps, wc={:.2f}K'\n                             .format(epoch_id, batch_id + 1, len(train_data_loader),\n                                     log_avg_loss / log_interval,\n                                     np.exp(log_avg_loss / log_interval),\n                                     log_avg_gnorm / log_interval,\n                                     wps / 1000, log_wc / 1000))\n                log_start_time = time.time()\n                log_avg_loss = 0\n                log_avg_gnorm = 0\n                log_wc = 0\n        valid_loss, valid_translation_out = evaluate(val_data_loader)\n        valid_bleu_score, _, _, _, _ = nmt.bleu.compute_bleu([val_tgt_sentences], valid_translation_out)\n        logging.info('[Epoch {}] valid Loss={:.4f}, valid ppl={:.4f}, valid bleu={:.2f}'\n                     .format(epoch_id, valid_loss, np.exp(valid_loss), valid_bleu_score * 100))\n        test_loss, test_translation_out = evaluate(test_data_loader)\n        test_bleu_score, _, _, _, _ = nmt.bleu.compute_bleu([test_tgt_sentences], test_translation_out)\n        logging.info('[Epoch {}] test Loss={:.4f}, test ppl={:.4f}, test bleu={:.2f}'\n                     .format(epoch_id, test_loss, np.exp(test_loss), test_bleu_score * 100))\n        write_sentences(valid_translation_out,\n                        os.path.join(save_dir, 'epoch{:d}_valid_out.txt').format(epoch_id))\n        write_sentences(test_translation_out,\n                        os.path.join(save_dir, 'epoch{:d}_test_out.txt').format(epoch_id))\n        if valid_bleu_score > best_valid_bleu:\n            best_valid_bleu = valid_bleu_score\n            save_path = os.path.join(save_dir, 'valid_best.params')\n            logging.info('Save best parameters to {}'.format(save_path))\n            model.save_parameters(save_path)\n        if epoch_id + 1 >= (epochs * 2) // 3:\n            new_lr = trainer.learning_rate * lr_update_factor\n            logging.info('Learning rate change to {}'.format(new_lr))\n            trainer.set_learning_rate(new_lr)\n\nSummary\n-------\n\nIn this notebook, we have shown how to train a GNMT model on IWSLT 2015\nEnglish-Vietnamese using Gluon NLP toolkit. The complete training script\ncan be found\n`here <https://github.com/dmlc/gluon-nlp/blob/v0.x/scripts/machine_translation/train_gnmt.py>`__.\nThe command to reproduce the result can be seen in the `nmt scripts\npage <http://gluon-nlp.mxnet.io/scripts/index.html#machine-translation>`__.\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/text/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nText Tutorials\n==============\n\nThese tutorials will help you learn how to create and use models that work with text and other natural language processing tasks.\n\nWord Embedding\n--------------\n\n.. container:: cards\n\n   .. card::\n      :title: Pre-trained Word Embeddings\n      :link: https://gluon-nlp.mxnet.io/examples/word_embedding/word_embedding.html\n\n      Basics on how to use word embedding with vocab in GluonNLP and apply it on word similarity and analogy problems.\n\n   .. card::\n      :title: Word Embeddings Training and Evaluation\n      :link: https://gluon-nlp.mxnet.io/examples/word_embedding/word_embedding_training.html\n\n      Learn how to train fastText and word2vec embeddings on your own dataset, and determine embedding quality through intrinsic evaluation.\n\nLanguage Model\n--------------\n\n\n.. container:: cards\n\n   .. card::\n      :title: LSTM-based Language Models\n      :link: https://gluon-nlp.mxnet.io/examples/language_model/language_model.html\n\n      Learn what a language model is, what it can do, and how to train a word-level language model with truncated back-propagation-through-time (BPTT).\n\nMachine Translation\n-------------------\n\n.. container:: cards\n\n   .. card::\n      :title: Google Neural Machine Translation\n      :link: https://gluon-nlp.mxnet.io/examples/machine_translation/gnmt.html\n\n      Learn how to train Google Neural Machine Translation, a seq2seq with attention model.\n\n   .. card::\n      :title: Machine Translation with Transformer\n      :link: https://gluon-nlp.mxnet.io/examples/machine_translation/transformer.html\n\n      Learn how to use a pre-trained transformer translation model for English to German translation.\n\nSentence Embedding\n---------------------\n\n.. container:: cards\n\n   .. card::\n      :title: ELMo: Deep Contextualized Word Representations\n      :link: https://gluon-nlp.mxnet.io/examples/sentence_embedding/elmo_sentence_representation.html\n\n      See how to use GluonNLP’s model API to automatically download the pre-trained ELMo model from NAACL2018 best paper, and extract features with it.\n\n   .. card::\n      :title: A Structured Self-attentive Sentence Embedding\n      :link: https://gluon-nlp.mxnet.io/examples/sentence_embedding/self_attentive_sentence_embedding.html\n\n      See how to use GluonNLP to build more advanced model structure for extracting sentence embeddings to predict Yelp review rating.\n\n   .. card::\n      :title: BERT: Bidirectional Encoder Representations from Transformers\n      :link: https://gluon-nlp.mxnet.io/examples/sentence_embedding/bert.html\n\n      See how to use GluonNLP to fine-tune a sentence pair classification model with pre-trained BERT parameters.\n\nSentiment Analysis\n------------------\n\n.. container:: cards\n\n   .. card::\n      :title: Sentiment Analysis by Fine-tuning Word Language Model\n      :link: https://gluon-nlp.mxnet.io/examples/sentiment_analysis/sentiment_analysis.html\n\n      See how to fine-tune a pre-trained language model to perform sentiment analysis on movie reviews.\n\nSequence Sampling\n-----------------\n\n.. container:: cards\n\n   .. card::\n      :title: Sequence Generation with Sampling and Beam Search\n      :link: https://gluon-nlp.mxnet.io/examples/sequence_sampling/sequence_sampling.html\n\n      Learn how to generate sentence from pre-trained language model through sampling and beam search. \n\n.. toctree::\n   :hidden:\n   :maxdepth: 1\n   :glob:\n\n   *\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/text/transformer.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nMachine Translation with Transformer\n====================================\n\nIn this notebook, we will show how to train Transformer introduced in\n[1] and evaluate the pretrained model using GluonNLP. The model is both\nmore accurate and lighter to train than previous seq2seq models. We will\ntogether go through:\n\n1) Use the state-of-the-art pretrained Transformer model: we will\n   evaluate the pretrained SOTA Transformer model and translate a few\n   sentences ourselves with the ``BeamSearchTranslator`` using the SOTA\n   model;\n\n2) Train the Transformer yourself: including loading and processing\n   dataset, define the Transformer model, write train script and\n   evaluate the trained model. Note that in order to obtain the\n   state-of-the-art results on WMT 2014 English-German dataset, it will\n   take around 1 day to have the model. In order to let you run through\n   the Transformer quickly, we suggest you to start with the ``TOY``\n   dataset sampled from the WMT dataset (by default in this notebook).\n\nPreparation\n-----------\n\nLoad MXNet and GluonNLP\n~~~~~~~~~~~~~~~~~~~~~~~\n\n.. code:: python\n\n    import warnings\n    warnings.filterwarnings('ignore')\n\n    import random\n    import numpy as np\n    import mxnet as mx\n    from mxnet import gluon\n    import gluonnlp as nlp\n\nSet Environment\n~~~~~~~~~~~~~~~\n\n.. code:: python\n\n    np.random.seed(100)\n    random.seed(100)\n    mx.random.seed(10000)\n    ctx = mx.gpu(0)\n\nUse the SOTA Pretrained Transformer model\n-----------------------------------------\n\nIn this subsection, we first load the SOTA Transformer model in GluonNLP\nmodel zoo; and secondly we load the full WMT 2014 English-German test\ndataset; and finally evaluate the model.\n\nGet the SOTA Transformer\n~~~~~~~~~~~~~~~~~~~~~~~~\n\nNext, we load the pretrained SOTA Transformer using the model API in\nGluonNLP. In this way, we can easily get access to the SOTA machine\ntranslation model and use it in your own application.\n\n.. code:: python\n\n    import nmt\n\n    wmt_model_name = 'transformer_en_de_512'\n\n    wmt_transformer_model, wmt_src_vocab, wmt_tgt_vocab = \\\n        nmt.transformer.get_model(wmt_model_name,\n                                  dataset_name='WMT2014',\n                                  pretrained=True,\n                                  ctx=ctx)\n\n    print(wmt_src_vocab)\n    print(wmt_tgt_vocab)\n\nThe Transformer model architecture is shown as below:\n\n.. raw:: html\n\n   <div style=\"width: 500px;\">\n\n|transformer|\n\n.. raw:: html\n\n   </div>\n\n.. code:: python\n\n    print(wmt_transformer_model)\n\nLoad and Preprocess WMT 2014 Dataset\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nWe then load the WMT 2014 English-German test dataset for evaluation\npurpose.\n\nThe following shows how to process the dataset and cache the processed\ndataset for the future use. The processing steps include:\n\n-  \n\n   1) clip the source and target sequences\n\n-  \n\n   2) split the string input to a list of tokens\n\n-  \n\n   3) map the string token into its index in the vocabulary\n\n-  \n\n   4) append EOS token to source sentence and add BOS and EOS tokens to\n      target sentence.\n\nLet's first look at the WMT 2014 corpus.\n\n.. code:: python\n\n    import hyperparameters as hparams\n\n    wmt_data_test = nlp.data.WMT2014BPE('newstest2014',\n                                        src_lang=hparams.src_lang,\n                                        tgt_lang=hparams.tgt_lang,\n                                        full=False)\n    print('Source language %s, Target language %s' % (hparams.src_lang, hparams.tgt_lang))\n\n    wmt_data_test[0]\n\n.. code:: python\n\n    wmt_test_text = nlp.data.WMT2014('newstest2014',\n                                     src_lang=hparams.src_lang,\n                                     tgt_lang=hparams.tgt_lang,\n                                     full=False)\n    wmt_test_text[0]\n\nWe then generate the target gold translations.\n\n.. code:: python\n\n    wmt_test_tgt_sentences = list(wmt_test_text.transform(lambda src, tgt: tgt))\n    wmt_test_tgt_sentences[0]\n\n.. code:: python\n\n    import dataprocessor\n\n    print(dataprocessor.TrainValDataTransform.__doc__)\n\n.. code:: python\n\n    wmt_transform_fn = dataprocessor.TrainValDataTransform(wmt_src_vocab, wmt_tgt_vocab, -1, -1)\n    wmt_dataset_processed = wmt_data_test.transform(wmt_transform_fn, lazy=False)\n    print(*wmt_dataset_processed[0], sep='\\n')\n\nCreate Sampler and DataLoader for WMT 2014 Dataset\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. code:: python\n\n    wmt_data_test_with_len = gluon.data.SimpleDataset([(ele[0], ele[1], len(\n        ele[0]), len(ele[1]), i) for i, ele in enumerate(wmt_dataset_processed)])\n\nNow, we have obtained data\\_train, data\\_val, and data\\_test. The next\nstep is to construct sampler and DataLoader. The first step is to\nconstruct batchify function, which pads and stacks sequences to form\nmini-batch.\n\n.. code:: python\n\n    wmt_test_batchify_fn = nlp.data.batchify.Tuple(\n        nlp.data.batchify.Pad(),\n        nlp.data.batchify.Pad(),\n        nlp.data.batchify.Stack(dtype='float32'),\n        nlp.data.batchify.Stack(dtype='float32'),\n        nlp.data.batchify.Stack())\n\nWe can then construct bucketing samplers, which generate batches by\ngrouping sequences with similar lengths.\n\n.. code:: python\n\n    wmt_bucket_scheme = nlp.data.ExpWidthBucket(bucket_len_step=1.2)\n\n.. code:: python\n\n    wmt_test_batch_sampler = nlp.data.FixedBucketSampler(\n        lengths=wmt_dataset_processed.transform(lambda src, tgt: len(tgt)),\n        use_average_length=True,\n        bucket_scheme=wmt_bucket_scheme,\n        batch_size=256)\n    print(wmt_test_batch_sampler.stats())\n\nGiven the samplers, we can create DataLoader, which is iterable.\n\n.. code:: python\n\n    wmt_test_data_loader = gluon.data.DataLoader(\n        wmt_data_test_with_len,\n        batch_sampler=wmt_test_batch_sampler,\n        batchify_fn=wmt_test_batchify_fn,\n        num_workers=8)\n    len(wmt_test_data_loader)\n\nEvaluate Transformer\n~~~~~~~~~~~~~~~~~~~~\n\nNext, we generate the SOTA results on the WMT test dataset. As we can\nsee from the result, we are able to achieve the SOTA number 27.35 as the\nBLEU score.\n\nWe first define the ``BeamSearchTranslator`` to generate the actual\ntranslations.\n\n.. code:: python\n\n    wmt_translator = nmt.translation.BeamSearchTranslator(\n        model=wmt_transformer_model,\n        beam_size=hparams.beam_size,\n        scorer=nlp.model.BeamSearchScorer(alpha=hparams.lp_alpha, K=hparams.lp_k),\n        max_length=200)\n\nThen we caculate the ``loss`` as well as the ``bleu`` score on the WMT\n2014 English-German test dataset. Note that the following evalution\nprocess will take ~13 mins to complete.\n\n.. code:: python\n\n    import time\n    import utils\n\n    eval_start_time = time.time()\n\n    wmt_test_loss_function = nmt.loss.SoftmaxCEMaskedLoss()\n    wmt_test_loss_function.hybridize()\n\n    wmt_detokenizer = nlp.data.SacreMosesDetokenizer()\n\n    wmt_test_loss, wmt_test_translation_out = utils.evaluate(wmt_transformer_model,\n                                                             wmt_test_data_loader,\n                                                             wmt_test_loss_function,\n                                                             wmt_translator,\n                                                             wmt_tgt_vocab,\n                                                             wmt_detokenizer,\n                                                             ctx)\n\n    wmt_test_bleu_score, _, _, _, _ = nmt.bleu.compute_bleu([wmt_test_tgt_sentences],\n                                                            wmt_test_translation_out,\n                                                            tokenized=False,\n                                                            tokenizer=hparams.bleu,\n                                                            split_compound_word=False,\n                                                            bpe=False)\n\n    print('WMT14 EN-DE SOTA model test loss: %.2f; test bleu score: %.2f; time cost %.2fs'\n          %(wmt_test_loss, wmt_test_bleu_score * 100, (time.time() - eval_start_time)))\n\n.. code:: python\n\n    print('Sample translations:')\n    num_pairs = 3\n\n    for i in range(num_pairs):\n        print('EN:')\n        print(wmt_test_text[i][0])\n        print('DE-Candidate:')\n        print(wmt_test_translation_out[i])\n        print('DE-Reference:')\n        print(wmt_test_tgt_sentences[i])\n        print('========')\n\nTranslation Inference\n~~~~~~~~~~~~~~~~~~~~~\n\nWe herein show the actual translation example (EN-DE) when given a\nsource language using the SOTA Transformer model.\n\n.. code:: python\n\n    import utils\n\n    print('Translate the following English sentence into German:')\n\n    sample_src_seq = 'We love each other'\n\n    print('[\\'' + sample_src_seq + '\\']')\n\n    sample_tgt_seq = utils.translate(wmt_translator,\n                                     sample_src_seq,\n                                     wmt_src_vocab,\n                                     wmt_tgt_vocab,\n                                     wmt_detokenizer,\n                                     ctx)\n\n    print('The German translation is:')\n    print(sample_tgt_seq)\n\nTrain Your Own Transformer\n--------------------------\n\nIn this subsection, we will go though the whole process about loading\ntranslation dataset in a more unified way, and create data sampler and\nloader, as well as define the Transformer model, finally writing\ntraining script to train the model yourself.\n\nLoad and Preprocess TOY Dataset\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nNote that we use demo mode (``TOY`` dataset) by default, since loading\nthe whole WMT 2014 English-German dataset ``WMT2014BPE`` for the later\ntraining will be slow (~1 day). But if you really want to train to have\nthe SOTA result, please set ``demo = False``. In order to make the data\nprocessing blocks execute in a more efficient way, we package them in\nthe ``load_translation_data`` (``transform`` etc.) function used as\nbelow. The function also returns the gold target sentences as well as\nthe vocabularies.\n\n.. code:: python\n\n    demo = True\n    if demo:\n        dataset = 'TOY'\n    else:\n        dataset = 'WMT2014BPE'\n\n    data_train, data_val, data_test, val_tgt_sentences, test_tgt_sentences, src_vocab, tgt_vocab = \\\n        dataprocessor.load_translation_data(\n            dataset=dataset,\n            src_lang=hparams.src_lang,\n            tgt_lang=hparams.tgt_lang)\n\n    data_train_lengths = dataprocessor.get_data_lengths(data_train)\n    data_val_lengths = dataprocessor.get_data_lengths(data_val)\n    data_test_lengths = dataprocessor.get_data_lengths(data_test)\n\n    data_train = data_train.transform(lambda src, tgt: (src, tgt, len(src), len(tgt)), lazy=False)\n    data_val = gluon.data.SimpleDataset([(ele[0], ele[1], len(ele[0]), len(ele[1]), i)\n                              for i, ele in enumerate(data_val)])\n    data_test = gluon.data.SimpleDataset([(ele[0], ele[1], len(ele[0]), len(ele[1]), i)\n                               for i, ele in enumerate(data_test)])\n\nCreate Sampler and DataLoader for TOY Dataset\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nNow, we have obtained ``data_train``, ``data_val``, and ``data_test``.\nThe next step is to construct sampler and DataLoader. The first step is\nto construct batchify function, which pads and stacks sequences to form\nmini-batch.\n\n.. code:: python\n\n    train_batchify_fn = nlp.data.batchify.Tuple(\n        nlp.data.batchify.Pad(),\n        nlp.data.batchify.Pad(),\n        nlp.data.batchify.Stack(dtype='float32'),\n        nlp.data.batchify.Stack(dtype='float32'))\n    test_batchify_fn = nlp.data.batchify.Tuple(\n        nlp.data.batchify.Pad(),\n        nlp.data.batchify.Pad(),\n        nlp.data.batchify.Stack(dtype='float32'),\n        nlp.data.batchify.Stack(dtype='float32'),\n        nlp.data.batchify.Stack())\n\n    target_val_lengths = list(map(lambda x: x[-1], data_val_lengths))\n    target_test_lengths = list(map(lambda x: x[-1], data_test_lengths))\n\nWe can then construct bucketing samplers, which generate batches by\ngrouping sequences with similar lengths.\n\n.. code:: python\n\n    bucket_scheme = nlp.data.ExpWidthBucket(bucket_len_step=1.2)\n    train_batch_sampler = nlp.data.FixedBucketSampler(lengths=data_train_lengths,\n                                                 batch_size=hparams.batch_size,\n                                                 num_buckets=hparams.num_buckets,\n                                                 ratio=0.0,\n                                                 shuffle=True,\n                                                 use_average_length=True,\n                                                 num_shards=1,\n                                                 bucket_scheme=bucket_scheme)\n    print('Train Batch Sampler:')\n    print(train_batch_sampler.stats())\n\n\n    val_batch_sampler = nlp.data.FixedBucketSampler(lengths=target_val_lengths,\n                                           batch_size=hparams.test_batch_size,\n                                           num_buckets=hparams.num_buckets,\n                                           ratio=0.0,\n                                           shuffle=False,\n                                           use_average_length=True,\n                                           bucket_scheme=bucket_scheme)\n    print('Validation Batch Sampler:')\n    print(val_batch_sampler.stats())\n\n    test_batch_sampler = nlp.data.FixedBucketSampler(lengths=target_test_lengths,\n                                            batch_size=hparams.test_batch_size,\n                                            num_buckets=hparams.num_buckets,\n                                            ratio=0.0,\n                                            shuffle=False,\n                                            use_average_length=True,\n                                            bucket_scheme=bucket_scheme)\n    print('Test Batch Sampler:')\n    print(test_batch_sampler.stats())\n\nGiven the samplers, we can create DataLoader, which is iterable. Note\nthat the data loader of validation and test dataset share the same\nbatchifying function ``test_batchify_fn``.\n\n.. code:: python\n\n    train_data_loader = nlp.data.ShardedDataLoader(data_train,\n                                          batch_sampler=train_batch_sampler,\n                                          batchify_fn=train_batchify_fn,\n                                          num_workers=8)\n    print('Length of train_data_loader: %d' % len(train_data_loader))\n    val_data_loader = gluon.data.DataLoader(data_val,\n                                 batch_sampler=val_batch_sampler,\n                                 batchify_fn=test_batchify_fn,\n                                 num_workers=8)\n    print('Length of val_data_loader: %d' % len(val_data_loader))\n    test_data_loader = gluon.data.DataLoader(data_test,\n                                  batch_sampler=test_batch_sampler,\n                                  batchify_fn=test_batchify_fn,\n                                  num_workers=8)\n    print('Length of test_data_loader: %d' % len(test_data_loader))\n\nDefine Transformer Model\n~~~~~~~~~~~~~~~~~~~~~~~~\n\nAfter obtaining DataLoader, we then start to define the Transformer. The\nencoder and decoder of the Transformer can be easily obtained by calling\n``get_transformer_encoder_decoder`` function. Then, we use the encoder\nand decoder in ``NMTModel`` to construct the Transformer model.\n``model.hybridize`` allows computation to be done using symbolic\nbackend. We also use ``label_smoothing``.\n\n.. code:: python\n\n    encoder, decoder = nmt.transformer.get_transformer_encoder_decoder(units=hparams.num_units,\n                                                       hidden_size=hparams.hidden_size,\n                                                       dropout=hparams.dropout,\n                                                       num_layers=hparams.num_layers,\n                                                       num_heads=hparams.num_heads,\n                                                       max_src_length=530,\n                                                       max_tgt_length=549,\n                                                       scaled=hparams.scaled)\n    model = nmt.translation.NMTModel(src_vocab=src_vocab, tgt_vocab=tgt_vocab, encoder=encoder, decoder=decoder,\n                     share_embed=True, embed_size=hparams.num_units, tie_weights=True,\n                     embed_initializer=None, prefix='transformer_')\n    model.initialize(init=mx.init.Xavier(magnitude=3.0), ctx=ctx)\n    model.hybridize()\n\n    print(model)\n\n    label_smoothing = nmt.loss.LabelSmoothing(epsilon=hparams.epsilon, units=len(tgt_vocab))\n    label_smoothing.hybridize()\n\n    loss_function = nmt.loss.SoftmaxCEMaskedLoss(sparse_label=False)\n    loss_function.hybridize()\n\n    test_loss_function = nmt.loss.SoftmaxCEMaskedLoss()\n    test_loss_function.hybridize()\n\n    detokenizer = nlp.data.SacreMosesDetokenizer()\n\nHere, we build the translator using the beam search\n\n.. code:: python\n\n    translator = nmt.translation.BeamSearchTranslator(model=model,\n                                                      beam_size=hparams.beam_size,\n                                                      scorer=nlp.model.BeamSearchScorer(alpha=hparams.lp_alpha,\n                                                                                        K=hparams.lp_k),\n                                                      max_length=200)\n    print('Use beam_size=%d, alpha=%.2f, K=%d' % (hparams.beam_size, hparams.lp_alpha, hparams.lp_k))\n\nTraining Loop\n~~~~~~~~~~~~~\n\nBefore conducting training, we need to create trainer for updating the\nparameter. In the following example, we create a trainer that uses ADAM\noptimzier.\n\n.. code:: python\n\n    trainer = gluon.Trainer(model.collect_params(), hparams.optimizer,\n                            {'learning_rate': hparams.lr, 'beta2': 0.98, 'epsilon': 1e-9})\n    print('Use learning_rate=%.2f'\n          % (trainer.learning_rate))\n\nWe can then write the training loop. During the training, we perform the\nevaluation on validation and testing dataset every epoch, and record the\nparameters that give the hightest BLEU score on validation dataset.\nBefore performing forward and backward, we first use ``as_in_context``\nfunction to copy the mini-batch to GPU. The statement\n``with mx.autograd.record()`` will locate Gluon backend to compute the\ngradients for the part inside the block. For ease of observing the\nconvergence of the update of the ``Loss`` in a quick fashion, we set the\n``epochs = 3``. Notice that, in order to obtain the best BLEU score, we\nwill need more epochs and large warmup steps following the original\npaper as you can find the SOTA results in the first subsection. Besides,\nwe use Averaging SGD [2] to update the parameters, since it is more\nrobust for the machine translation task.\n\n.. code:: python\n\n    best_valid_loss = float('Inf')\n    step_num = 0\n    #We use warmup steps as introduced in [1].\n    warmup_steps = hparams.warmup_steps\n    grad_interval = hparams.num_accumulated\n    model.setattr('grad_req', 'add')\n    #We use Averaging SGD [2] to update the parameters.\n    average_start = (len(train_data_loader) // grad_interval) * \\\n        (hparams.epochs - hparams.average_start)\n    average_param_dict = {k: mx.nd.array([0]) for k, v in\n                                          model.collect_params().items()}\n    update_average_param_dict = True\n    model.zero_grad()\n    for epoch_id in range(hparams.epochs):\n        utils.train_one_epoch(epoch_id, model, train_data_loader, trainer,\n                              label_smoothing, loss_function, grad_interval,\n                              average_param_dict, update_average_param_dict,\n                              step_num, ctx)\n        mx.nd.waitall()\n        # We define evaluation function as follows. The `evaluate` function use beam search translator\n        # to generate outputs for the validation and testing datasets.\n        valid_loss, _ = utils.evaluate(model, val_data_loader,\n                                       test_loss_function, translator,\n                                       tgt_vocab, detokenizer, ctx)\n        print('Epoch %d, valid Loss=%.4f, valid ppl=%.4f'\n              % (epoch_id, valid_loss, np.exp(valid_loss)))\n        test_loss, _ = utils.evaluate(model, test_data_loader,\n                                      test_loss_function, translator,\n                                      tgt_vocab, detokenizer, ctx)\n        print('Epoch %d, test Loss=%.4f, test ppl=%.4f'\n              % (epoch_id, test_loss, np.exp(test_loss)))\n        if valid_loss < best_valid_loss:\n            best_valid_loss = valid_loss\n            model.save_parameters('{}.{}'.format(hparams.save_dir, 'valid_best.params'))\n        model.save_parameters('{}.epoch{:d}.params'.format(hparams.save_dir, epoch_id))\n    mx.nd.save('{}.{}'.format(hparams.save_dir, 'average.params'), average_param_dict)\n\n    if hparams.average_start > 0:\n        for k, v in model.collect_params().items():\n            v.set_data(average_param_dict[k])\n    else:\n        model.load_parameters('{}.{}'.format(hparams.save_dir, 'valid_best.params'), ctx)\n    valid_loss, _ = utils.evaluate(model, val_data_loader,\n                                   test_loss_function, translator,\n                                   tgt_vocab, detokenizer, ctx)\n    print('Best model valid Loss=%.4f, valid ppl=%.4f'\n          % (valid_loss, np.exp(valid_loss)))\n    test_loss, _ = utils.evaluate(model, test_data_loader,\n                                  test_loss_function, translator,\n                                  tgt_vocab, detokenizer, ctx)\n    print('Best model test Loss=%.4f, test ppl=%.4f'\n          % (test_loss, np.exp(test_loss)))\n\nConclusion\n----------\n\n-  Showcase with Transformer, we are able to support the deep neural\n   networks for seq2seq task. We have already achieved SOTA results on\n   the WMT 2014 English-German task.\n-  Gluon NLP Toolkit provides high-level APIs that could drastically\n   simplify the development process of modeling for NLP tasks sharing\n   the encoder-decoder structure.\n-  Low-level APIs in NLP Toolkit enables easy customization.\n\nDocumentation can be found at https://gluon-nlp.mxnet.io/index.html\n\nCode is here https://github.com/dmlc/gluon-nlp\n\nReferences\n----------\n\n[1] Vaswani, Ashish, et al. \"Attention is all you need.\" Advances in\nNeural Information Processing Systems. 2017.\n\n[2] Polyak, Boris T, and Anatoli B. Juditsky. \"Acceleration of\nstochastic approximation by averaging.\" SIAM Journal on Control and\nOptimization. 1992.\n\n.. |transformer| image:: /_static/transformer.png\n\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/training/fit_api_tutorial.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# MXNet Gluon Fit API\n\nIn this tutorial, you will learn how to use the [Gluon Fit API](https://cwiki.apache.org/confluence/display/MXNET/Gluon+Fit+API+-+Tech+Design) which is the easiest way to train deep learning models using the [Gluon API](../index.rst) in Apache MXNet.\n\nWith the Fit API, you can train a deep learning model with a minimal amount of code. Just specify the network, loss function and the data you want to train on. You don't need to worry about the boiler plate code to loop through the dataset in batches (often called as 'training loop'). Advanced users can train with bespoke training loops, and many of these use cases will be covered by the Fit API.\n\nTo demonstrate the Fit API, you will train an image classification model using the [ResNet-18](https://arxiv.org/abs/1512.03385) neural network architecture. The model will be trained using the [Fashion-MNIST dataset](https://github.com/zalandoresearch/fashion-mnist).\n\n## Prerequisites\n\nTo complete this tutorial, you will need:\n\n- [MXNet](https://mxnet.apache.org/get_started) (The version of MXNet will be >= 1.5.0, you can use `pip install mxnet` to get 1.5.0 release pip package or build from source with master, refer to [MXNet installation](https://mxnet.apache.org/get_started?version=master&platform=linux&language=python&environ=pip&processor=cpu)\n- [Jupyter Notebook](https://jupyter.org/index.html) (For interactively running the provided .ipynb file)\n\n\n```{.python .input}\nimport mxnet as mx\nfrom mxnet import gluon\nfrom mxnet.gluon.model_zoo import vision\nfrom mxnet.gluon.contrib.estimator import estimator\nfrom mxnet.gluon.contrib.estimator.event_handler import TrainBegin, TrainEnd, EpochEnd, CheckpointHandler\n\ngpu_count = mx.device.num_gpus()\ndevice = [mx.gpu(i) for i in range(gpu_count)] if gpu_count > 0 else mx.cpu()\n```\n\n## Dataset\n\n[Fashion-MNIST](https://github.com/zalandoresearch/fashion-mnist) dataset consists of fashion items divided into ten categories: t-shirt/top, trouser, pullover, dress, coat, sandal, shirt, sneaker, bag and ankle boot.\n\n- It has 60,000 grayscale images of size 28 * 28 for training.\n- It has 10,000 grayscale images of size 28 * 28 for testing/validation.\n\nWe will use the ```gluon.data.vision``` package to directly import the Fashion-MNIST dataset and perform pre-processing on it.\n\n\n```{.python .input}\n# Get the training data\nfashion_mnist_train = gluon.data.vision.FashionMNIST(train=True)\n\n# Get the validation data\nfashion_mnist_val = gluon.data.vision.FashionMNIST(train=False)\n```\n\n\n```{.python .input}\ntransforms = [gluon.data.vision.transforms.Resize(224), # We pick 224 as the model we use takes an input of size 224.\n                gluon.data.vision.transforms.ToTensor()]\n\n# Now we will stack all these together.\ntransforms = gluon.data.vision.transforms.Compose(transforms)\n```\n\n\n```{.python .input}\n# Apply the transformations\nfashion_mnist_train = fashion_mnist_train.transform_first(transforms)\nfashion_mnist_val = fashion_mnist_val.transform_first(transforms)\n```\n\n\n```{.python .input}\nbatch_size = 256 # Batch size of the images\nnum_workers = 4 # The number of parallel workers for loading the data using Data Loaders.\n\ntrain_data_loader = gluon.data.DataLoader(fashion_mnist_train, batch_size=batch_size,\n                                          shuffle=True, num_workers=num_workers)\nval_data_loader = gluon.data.DataLoader(fashion_mnist_val, batch_size=batch_size,\n                                        shuffle=False, num_workers=num_workers)\n```\n\n## Model and Optimizers\n\nLet's load the resnet-18 model architecture from [Gluon Model Zoo](../../../../api/gluon/model_zoo/index.rst) and initialize its parameters. The Gluon Model Zoo contains a repository of pre-trained models as well the model architecture definitions. We are using the model architecture from the model zoo in order to train it from scratch.\n\n\n```{.python .input}\nresnet_18_v1 = vision.resnet18_v1(pretrained=False, classes = 10)\nresnet_18_v1.initialize(init = mx.init.Xavier(), device=device)\n```\n\nWe will be using `SoftmaxCrossEntropyLoss` as the loss function since this is a multi-class classification problem. We will be using `sgd` (Stochastic Gradient Descent) as the optimizer.\nYou can experiment with a [different loss](../../../../api/gluon/loss/index.rst) or [optimizer](../../../../api/optimizer/index.rst) as well.\n\n\n```{.python .input}\nloss_fn = gluon.loss.SoftmaxCrossEntropyLoss()\n```\n\nLet's define the trainer object for training the model.\n\n\n```{.python .input}\nlearning_rate = 0.04 # You can experiment with your own learning rate here\nnum_epochs = 2 # You can run training for more epochs\ntrainer = gluon.Trainer(resnet_18_v1.collect_params(),\n                        'sgd', {'learning_rate': learning_rate})\n```\n\n## Train using Fit API\n\nAs stated earlier, the Fit API greatly simplifies the boiler plate code and complexity for training using MXNet Gluon.\n\nIn the basic usage example, with just 2 lines of code, we will set up our model for training.\n\n### Basic Usage\n\n\n```{.python .input}\ntrain_acc = mx.gluon.metric.Accuracy() # Metric to monitor\n\n# Define the estimator, by passing to it the model, loss function, metrics, trainer object and device\nest = estimator.Estimator(net=resnet_18_v1,\n                          loss=loss_fn,\n                          train_metrics=train_acc,\n                          trainer=trainer,\n                          device=device)\n\n# ignore warnings for nightly test on CI only\nimport warnings\nwith warnings.catch_warnings():\n    warnings.simplefilter(\"ignore\")\n    # Magic line\n    est.fit(train_data=train_data_loader,\n        epochs=num_epochs)\n```\n\n```text\n    Training begin: using optimizer SGD with current learning rate 0.0400 <!--notebook-skip-line-->\n    Train for 2 epochs. <!--notebook-skip-line-->\n\n    [Epoch 0] finished in 25.110s: train_accuracy : 0.7877 train_softmaxcrossentropyloss0 : 0.5905 <!--notebook-skip-line-->\n\n    [Epoch 1] finished in 23.595s: train_accuracy : 0.8823 train_softmaxcrossentropyloss0 : 0.3197 <!--notebook-skip-line-->\n    Train finished using total 48s at epoch 1. train_accuracy : 0.8823 train_softmaxcrossentropyloss0 : 0.3197 <!--notebook-skip-line-->\n```\n\n### Advanced Usage\n\nThe Fit API is also customizable with several `Event Handlers` which give a fine grained control over the steps in training and exposes callback methods that provide control over the stages involved in training. Available callback methods are: `train_begin`, `train_end`, `batch_begin`, `batch_end`, `epoch_begin` and `epoch_end`.\n\nYou can use built-in event handlers such as `LoggingHandler`, `CheckpointHandler` or `EarlyStoppingHandler` to log and save the model at certain time-steps during training. You can also stop the training when the model's performance plateaus.\nThere are also some default utility handlers that will be added to your estimator by default. For example, `StoppingHandler` is used to control when the training ends, based on number of epochs or number of batches trained.\n`MetricHandler` is used to calculate training metrics at end of each batch and epoch.\n`ValidationHandler` is used to validate your model on test data at each epoch's end and then calculate validation metrics.\nYou can create these utility handlers with different configurations and pass to estimator. This will override the default handler configuration.\nYou can create a custom handler by inheriting one or multiple\n[base event handlers](https://github.com/apache/mxnet/blob/master/python/mxnet/gluon/contrib/estimator/event_handler.py#L32)\n including: `TrainBegin`, `TrainEnd`, `EpochBegin`, `EpochEnd`, `BatchBegin`, `BatchEnd`.\n\n\n### Custom Event Handler\n\nHere we will showcase an example custom event handler that inherits features from a few base handler classes.\nOur custom event handler is a simple one: record the loss values at the end of every epoch in our training phase.\n\nNote: For each of the method, the `Estimator` object is passed along, so you can access training metrics.\n\n```{.python .input}\nclass LossRecordHandler(TrainBegin, TrainEnd, EpochEnd):\n    def __init__(self):\n        super(LossRecordHandler, self).__init__()\n        self.loss_history = {}\n\n    def train_begin(self, estimator, *args, **kwargs):\n        print(\"Training begin\")\n\n    def train_end(self, estimator, *args, **kwargs):\n        # Print all the losses at the end of training\n        print(\"Training ended\")\n        for loss_name in self.loss_history:\n            for i, loss_val in enumerate(self.loss_history[loss_name]):\n                print(\"Epoch: {}, Loss name: {}, Loss value: {}\".format(i, loss_name, loss_val))\n\n    def epoch_end(self, estimator, *args, **kwargs):\n        for metric in estimator.train_metrics:\n            # look for train Loss in training metrics\n            # we wrapped loss value as a metric to record it\n            if isinstance(metric, mx.gluon.metric.Loss):\n                loss_name, loss_val = metric.get()\n                # append loss value for this epoch\n                self.loss_history.setdefault(loss_name, []).append(loss_val)\n```\n\n\n```{.python .input}\n# Let's reset the model, trainer and accuracy objects from above\n\nresnet_18_v1.initialize(force_reinit=True, init = mx.init.Xavier(), device=device)\ntrainer = gluon.Trainer(resnet_18_v1.collect_params(),\n                        'sgd', {'learning_rate': learning_rate})\ntrain_acc = mx.gluon.metric.Accuracy()\n```\n\n\n```{.python .input}\n# Define the estimator, by passing to it the model, loss function, metrics, trainer object and device\nest = estimator.Estimator(net=resnet_18_v1,\n                          loss=loss_fn,\n                          train_metrics=train_acc,\n                          trainer=trainer,\n                          device=device)\n\n# Define the handlers, let's say in built Checkpointhandler\ncheckpoint_handler = CheckpointHandler(model_dir='./',\n                                       model_prefix='my_model',\n                                       monitor=train_acc,  # Monitors a metric\n                                       save_best=True)  # Save the best model in terms of\n# Let's instantiate another handler which we defined above\nloss_record_handler = LossRecordHandler()\n# ignore warnings for nightly test on CI only\nimport warnings\nwith warnings.catch_warnings():\n    warnings.simplefilter(\"ignore\")\n    # Magic line\n    est.fit(train_data=train_data_loader,\n            val_data=val_data_loader,\n            epochs=num_epochs,\n            event_handlers=[checkpoint_handler, loss_record_handler]) # Add the event handlers\n```\n\n```text\n    Training begin: using optimizer SGD with current learning rate 0.0400 <!--notebook-skip-line-->\n    Train for 2 epochs. <!--notebook-skip-line-->\n\n    [Epoch 0] finished in 25.236s: train_accuracy : 0.7917 train_softmaxcrossentropyloss0 : 0.5741 val_accuracy : 0.6612 val_softmaxcrossentropyloss0 : 0.8627 <!--notebook-skip-line-->\n\n    [Epoch 1] finished in 24.892s: train_accuracy : 0.8826 train_softmaxcrossentropyloss0 : 0.3229 val_accuracy : 0.8474 val_softmaxcrossentropyloss0 : 0.4262 <!--notebook-skip-line-->\n\n    Train finished using total 50s at epoch 1. train_accuracy : 0.8826 train_softmaxcrossentropyloss0 : 0.3229 val_accuracy : 0.8474 val_softmaxcrossentropyloss0 : 0.4262 <!--notebook-skip-line-->\n\n    Training begin <!--notebook-skip-line-->\n    Epoch 1, loss 0.5741 <!--notebook-skip-line-->\n    Epoch 2, loss 0.3229 <!--notebook-skip-line-->\n```\n\nYou can load the saved model, by using the `load_parameters` API in Gluon. For more details refer to the [Loading model parameters from file tutorial](../blocks/save_load_params.ipynb#Loading-model-parameters-from-file)\n\n\n```{.python .input}\nresnet_18_v1 = vision.resnet18_v1(pretrained=False, classes=10)\nresnet_18_v1.load_parameters('./my_model-best.params', device=device)\n```\n\n## Next Steps\n\n- For more hands on learning about deep learning, check out [Dive into Deep Learning](https://d2l.ai)\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/training/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nTraining\n========\n\n.. toctree::\n   :maxdepth: 1\n   :glob:\n\n   *\n   */index*"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/training/learning_rates/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\n\nLearning Rates\n==============\n\n.. toctree::\n   :maxdepth: 1\n   :glob:\n\n   *\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/training/learning_rates/learning_rate_finder.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Learning Rate Finder\n\nSetting the learning rate for stochastic gradient descent (SGD) is crucially important when training neural network because it controls both the speed of convergence and the ultimate performance of the network. Set the learning too low and you could be twiddling your thumbs for quite some time as the parameters update very slowly. Set it too high and the updates will skip over optimal solutions, or worse the optimizer might not converge at all!\n\nLeslie Smith from the U.S. Naval Research Laboratory presented a method for finding a good learning rate in a paper called [\"Cyclical Learning Rates for Training Neural Networks\"](https://arxiv.org/abs/1506.01186). We implement this method in MXNet (with the Gluon API) and create a 'Learning Rate Finder' which you can use while training your own networks. We take a look at the central idea of the paper, cyclical learning rate schedules, in the ['Advanced Learning Rate Schedules'](./learning_rate_schedules_advanced.ipynb) tutorial.\n\n## Simple Idea\n\nGiven an initialized network, a defined loss and a training dataset we take the following steps:\n\n1. Train one batch at a time (a.k.a. an iteration)\n2. Start with a very small learning rate (e.g. 0.000001) and slowly increase it every iteration\n3. Record the training loss and continue until we see the training loss diverge\n\nWe then analyse the results by plotting a graph of the learning rate against the training loss as seen below (taking note of the log scales).\n\n<img src=\"https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/lr_finder/finder_plot_w_annotations.png\" width=\"500px\"/> <!--notebook-skip-line-->\n\nAs expected, for very small learning rates we don't see much change in the loss as the parameter updates are negligible. At a learning rate of 0.001, we start to see the loss fall. Setting the initial learning rate here is reasonable, but we still have the potential to learn faster. We observe a drop in the loss up until 0.1 where the loss appears to diverge. We want to set the initial learning rate as high as possible before the loss becomes unstable, so we choose a learning rate of 0.05.\n\n## Epoch to Iteration\n\nUsually, our unit of work is an epoch (a full pass through the dataset) and the learning rate would typically be held constant throughout the epoch. With the Learning Rate Finder (and cyclical learning rate schedules) we are required to vary the learning rate every iteration. As such we structure our training code so that a single iteration can be run with a given learning rate. You can implement Learner as you wish. Just initialize the network, define the loss and trainer in `__init__` and keep your training logic for a single batch in `iteration`.\n\n\n```{.python .input}\nimport mxnet as mx\n\n# Set seed for reproducibility\nmx.np.random.seed(42)\n\nclass Learner():\n    def __init__(self, net, data_loader, device):\n        \"\"\"\n        :param net: network (mx.gluon.Block)\n        :param data_loader: training data loader (mx.gluon.data.DataLoader)\n        :param device: device (mx.gpu or mx.cpu)\n        \"\"\"\n        self.net = net\n        self.data_loader = data_loader\n        self.device = device\n        # So we don't need to be in `for batch in data_loader` scope\n        # and can call for next batch in `iteration`\n        self.data_loader_iter = iter(self.data_loader)\n        self.net.initialize(mx.init.Xavier(), device=self.device)\n        self.loss_fn = mx.gluon.loss.SoftmaxCrossEntropyLoss()\n        self.trainer = mx.gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': .001})\n\n    def iteration(self, lr=None, take_step=True):\n        \"\"\"\n        :param lr: learning rate to use for iteration (float)\n        :param take_step: take trainer step to update weights (boolean)\n        :return: iteration loss (float)\n        \"\"\"\n        # Update learning rate if different this iteration\n        if lr and (lr != self.trainer.learning_rate):\n            self.trainer.set_learning_rate(lr)\n        # Get next batch, and move device (e.g. to GPU if set)\n        data, label = next(self.data_loader_iter)\n        data = data.to_device(self.device)\n        label = label.to_device(self.device)\n        # Standard forward and backward pass\n        with mx.autograd.record():\n            output = self.net(data)\n            loss = self.loss_fn(output, label)\n        loss.backward()\n        # Update parameters\n        if take_step: self.trainer.step(data.shape[0])\n        # Set and return loss.\n        self.iteration_loss = mx.np.mean(loss).item()\n        return self.iteration_loss\n\n    def close(self):\n        # Close open iterator and associated workers\n        self.data_loader_iter.shutdown()\n```\n\nWe also adjust our `DataLoader` so that it continuously provides batches of data and doesn't stop after a single epoch. We can then call `iteration` as many times as required for the loss to diverge as part of the Learning Rate Finder process. We implement a custom `BatchSampler` for this, that keeps returning random indices of samples to be included in the next batch. We use the CIFAR-10 dataset for image classification to test our Learning Rate Finder.\n\n\n```{.python .input}\nfrom mxnet.gluon.data.vision import transforms\n\ntransform = transforms.Compose([\n    # Switches HWC to CHW, and converts to `float32`\n    transforms.ToTensor(),\n    # Channel-wise, using pre-computed means and stds\n    transforms.Normalize(mean=[0.4914, 0.4822, 0.4465],\n                         std=[0.2023, 0.1994, 0.2010])\n])\n\ndataset = mx.gluon.data.vision.datasets.CIFAR10(train=True).transform_first(transform)\n\nclass ContinuousBatchSampler():\n    def __init__(self, sampler, batch_size):\n        self._sampler = sampler\n        self._batch_size = batch_size\n\n    def __iter__(self):\n        batch = []\n        while True:\n            for i in self._sampler:\n                batch.append(i)\n                if len(batch) == self._batch_size:\n                    yield batch\n                    batch = []\n\nsampler = mx.gluon.data.RandomSampler(len(dataset))\nbatch_sampler = ContinuousBatchSampler(sampler, batch_size=128)\ndata_loader = mx.gluon.data.DataLoader(dataset, batch_sampler=batch_sampler)\n```\n\n## Implementation\n\nWith preparation complete, we're ready to write our Learning Rate Finder that wraps the `Learner` we defined above. We implement a `find` method for the procedure, and `plot` for the visualization. Starting with a very low learning rate as defined by `lr_start` we train one iteration at a time and keep multiplying the learning rate by `lr_multiplier`. We analyse the loss and continue until it diverges according to `LRFinderStoppingCriteria` (which is defined later on). You may also notice that we save the parameters and state of the optimizer before the process and restore afterwards. This is so the Learning Rate Finder process doesn't impact the state of the model, and can be used at any point during training.\n\n\n```{.python .input}\nfrom matplotlib import pyplot as plt\n\nclass LRFinder():\n    def __init__(self, learner):\n        \"\"\"\n        :param learner: able to take single iteration with given learning rate and return loss\n           and save and load parameters of the network (Learner)\n        \"\"\"\n        self.learner = learner\n\n    def find(self, lr_start=1e-6, lr_multiplier=1.1, smoothing=0.3):\n        \"\"\"\n        :param lr_start: learning rate to start search (float)\n        :param lr_multiplier: factor the learning rate is multiplied by at each step of search (float)\n        :param smoothing: amount of smoothing applied to loss for stopping criteria (float)\n        :return: learning rate and loss pairs (list of (float, float) tuples)\n        \"\"\"\n        # Used to initialize weights; pass data, but don't take step.\n        # Would expect for new model with lazy weight initialization\n        self.learner.iteration(take_step=False)\n        # Used to initialize trainer (if no step has been taken)\n        if not self.learner.trainer._kv_initialized:\n            self.learner.trainer._init_kvstore()\n        # Store params and optimizer state for restore after lr_finder procedure\n        # Useful for applying the method partway through training, not just for initialization of lr.\n        self.learner.net.save_parameters(\"lr_finder.params\")\n        self.learner.trainer.save_states(\"lr_finder.state\")\n        lr = lr_start\n        self.results = [] # List of (lr, loss) tuples\n        stopping_criteria = LRFinderStoppingCriteria(smoothing)\n        while True:\n            # Run iteration, and block until loss is calculated.\n            loss = self.learner.iteration(lr)\n            self.results.append((lr, loss))\n            if stopping_criteria(loss):\n                break\n            lr = lr * lr_multiplier\n        # Restore params (as finder changed them)\n        self.learner.net.load_parameters(\"lr_finder.params\", device=self.learner.device)\n        self.learner.trainer.load_states(\"lr_finder.state\")\n        return self.results\n\n    def plot(self):\n        lrs = [e[0] for e in self.results]\n        losses = [e[1] for e in self.results]\n        plt.figure(figsize=(6,8))\n        plt.scatter(lrs, losses)\n        plt.xlabel(\"Learning Rate\")\n        plt.ylabel(\"Loss\")\n        plt.xscale('log')\n        plt.yscale('log')\n        axes = plt.gca()\n        axes.set_xlim([lrs[0], lrs[-1]])\n        y_lower = min(losses) * 0.8\n        y_upper = losses[0] * 4\n        axes.set_ylim([y_lower, y_upper])\n        plt.show()\n```\n\n\nYou can define the `LRFinderStoppingCriteria` as you wish, but empirical testing suggests using a smoothed average gives a more consistent stopping rule (see `smoothing`). We stop when the smoothed average of the loss exceeds twice the initial loss, assuming there have been a minimum number of iterations (see `min_iter`).\n\n\n```{.python .input}\nclass LRFinderStoppingCriteria():\n    def __init__(self, smoothing=0.3, min_iter=20):\n        \"\"\"\n        :param smoothing: applied to running mean which is used for thresholding (float)\n        :param min_iter: minimum number of iterations before early stopping can occur (int)\n        \"\"\"\n        self.smoothing = smoothing\n        self.min_iter = min_iter\n        self.first_loss = None\n        self.running_mean = None\n        self.counter = 0\n\n    def __call__(self, loss):\n        \"\"\"\n        :param loss: from single iteration (float)\n        :return: indicator to stop (boolean)\n        \"\"\"\n        self.counter += 1\n        if self.first_loss is None:\n            self.first_loss = loss\n        if self.running_mean is None:\n            self.running_mean = loss\n        else:\n            self.running_mean = ((1 - self.smoothing) * loss) + (self.smoothing * self.running_mean)\n        return (self.running_mean > self.first_loss * 2) and (self.counter >= self.min_iter)\n```\n\n## Usage\n\nUsing a Pre-activation ResNet-18 from the Gluon model zoo, we instantiate our Learner and fire up our Learning Rate Finder!\n\n\n```{.python .input}\ndevice = mx.gpu() if mx.device.num_gpus() else mx.cpu()\nnet = mx.gluon.model_zoo.vision.resnet18_v2(classes=10)\nlearner = Learner(net=net, data_loader=data_loader, device=device)\nlr_finder = LRFinder(learner)\nlr_finder.find(lr_start=1e-6)\nlr_finder.plot()\n```\n\n\n![png](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/lr_finder/finder_plot.png) <!--notebook-skip-line-->\n\n\nAs discussed before, we should select a learning rate where the loss is falling (i.e. from 0.001 to 0.05) but before the loss starts to diverge (i.e. 0.1). We prefer higher learning rates where possible, so we select an initial learning rate of 0.05. Just as a test, we will run 500 epochs using this learning rate and evaluate the loss on the final batch. As we're working with a single batch of 128 samples, the variance of the loss estimates will be reasonably high, but it will give us a general idea. We save the initialized parameters for a later comparison with other learning rates.\n\n\n```{.python .input}\nlearner.net.save_parameters(\"net.params\")\nlr = 0.05\n\nfor iter_idx in range(300):\n    learner.iteration(lr=lr)\n    if ((iter_idx % 100) == 0):\n        print(\"Iteration: {}, Loss: {:.5g}\".format(iter_idx, learner.iteration_loss))\nprint(\"Final Loss: {:.5g}\".format(learner.iteration_loss))\n```\n\nIteration: 0, Loss: 2.785 <!--notebook-skip-line-->\n\nIteration: 100, Loss: 1.6653 <!--notebook-skip-line-->\n\nIteration: 200, Loss: 1.4891 <!--notebook-skip-line-->\n\n\nFinal Loss: 1.1812 <!--notebook-skip-line-->\n\n\nWe see a sizable drop in the loss from approx. 2.7 to 1.2.\n\nAnd now we have a baseline, let's see what happens when we train with a learning rate that's higher than advisable at 0.5.\n\n\n```{.python .input}\nnet = mx.gluon.model_zoo.vision.resnet18_v2(classes=10)\nlearner = Learner(net=net, data_loader=data_loader, device=device)\nlearner.net.load_parameters(\"net.params\", device=device)\nlr = 0.5\n\nfor iter_idx in range(300):\n    learner.iteration(lr=lr)\n    if ((iter_idx % 100) == 0):\n        print(\"Iteration: {}, Loss: {:.5g}\".format(iter_idx, learner.iteration_loss))\nprint(\"Final Loss: {:.5g}\".format(learner.iteration_loss))\n```\n\nIteration: 0, Loss: 2.6469 <!--notebook-skip-line-->\n\nIteration: 100, Loss: 1.9666 <!--notebook-skip-line-->\n\nIteration: 200, Loss: 1.6919 <!--notebook-skip-line-->\n\n\nFinal Loss: 1.366 <!--notebook-skip-line-->\n\n\nWe still observe a fall in the loss but aren't able to reach as low as before.\n\nAnd lastly, we see how the model trains with a more conservative learning rate of 0.005.\n\n\n```{.python .input}\nnet = mx.gluon.model_zoo.vision.resnet18_v2(classes=10)\nlearner = Learner(net=net, data_loader=data_loader, device=device)\nlearner.net.load_parameters(\"net.params\", device=device)\nlr = 0.005\n\nfor iter_idx in range(300):\n    learner.iteration(lr=lr)\n    if ((iter_idx % 100) == 0):\n        print(\"Iteration: {}, Loss: {:.5g}\".format(iter_idx, learner.iteration_loss))\nprint(\"Final Loss: {:.5g}\".format(learner.iteration_loss))\n```\n\nIteration: 0, Loss: 2.605 <!--notebook-skip-line-->\n\nIteration: 100, Loss: 1.8621 <!--notebook-skip-line-->\n\nIteration: 200, Loss: 1.6316 <!--notebook-skip-line-->\n\n\nFinal Loss: 1.2919 <!--notebook-skip-line-->\n\n\nAlthough we get quite similar results to when we set the learning rate at 0.05 (because we're still in the region of falling loss on the Learning Rate Finder plot), we can still optimize our network faster using a slightly higher rate.\n\n## Wrap Up\n\nGive Learning Rate Finder a try on your current projects, and experiment with the different learning rate schedules found in the [basic learning rate tutorial](./learning_rate_schedules.ipynb) and the [advanced learning rate tutorial](./learning_rate_schedules_advanced.ipynb).\n\n<!-- INSERT SOURCE DOWNLOAD BUTTONS -->\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/training/learning_rates/learning_rate_schedules.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Learning Rate Schedules\n\nSetting the learning rate for stochastic gradient descent (SGD) is crucially important when training neural networks because it controls both the speed of convergence and the ultimate performance of the network. One of the simplest learning rate strategies is to have a fixed learning rate throughout the training process. Choosing a small learning rate allows the optimizer find good solutions, but this comes at the expense of limiting the initial speed of convergence. Changing the learning rate over time can overcome this tradeoff.\n\nSchedules define how the learning rate changes over time and are typically specified for each epoch or iteration (i.e. batch) of training. Schedules differ from adaptive methods (such as AdaDelta and Adam) because they:\n\n* change the global learning rate for the optimizer, rather than parameter-wise learning rates\n* don't take feedback from the training process and are specified beforehand\n\nIn this tutorial, we visualize the schedules defined in `mx.lr_scheduler`, show how to implement custom schedules and see an example of using a schedule while training models. Since schedules are passed to `mx.optimizer.Optimizer` classes, these methods work with both Module and Gluon APIs.\n\n\n```{.python .input}\nfrom __future__ import print_function\nimport math\nimport matplotlib.pyplot as plt\nimport mxnet as mx\nfrom mxnet.gluon import nn\nfrom mxnet.gluon.data.vision import transforms\nimport numpy as np\n%matplotlib inline\n```\n\n```{.python .input}\ndef plot_schedule(schedule_fn, iterations=1500):\n    # Iteration count starting at 1\n    iterations = [i+1 for i in range(iterations)]\n    lrs = [schedule_fn(i) for i in iterations]\n    plt.scatter(iterations, lrs)\n    plt.xlabel(\"Iteration\")\n    plt.ylabel(\"Learning Rate\")\n    plt.show()\n```\n\n## Schedules\n\nIn this section, we take a look at the schedules in `mx.lr_scheduler`. All of these schedules define the learning rate for a given iteration, and it is expected that iterations start at 1 rather than 0. So to find the learning rate for the 100th iteration, you can call `schedule(100)`.\n\n### Stepwise Decay Schedule\n\nOne of the most commonly used learning rate schedules is called stepwise decay, where the learning rate is reduced by a factor at certain intervals. MXNet implements a `FactorScheduler` for equally spaced intervals, and `MultiFactorScheduler` for greater control. We start with an example of halving the learning rate every 250 iterations. More precisely, the learning rate will be multiplied by `factor` _after_ the `step` index and multiples thereafter. So in the example below the learning rate of the 250th iteration will be 1 and the 251st iteration will be 0.5.\n\n\n```{.python .input}\nschedule = mx.lr_scheduler.FactorScheduler(step=250, factor=0.5)\nschedule.base_lr = 1\nplot_schedule(schedule)\n```\n\n\n![lr factor](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/lr_schedules/factor.png) <!--notebook-skip-line-->\n\n\nNote: the `base_lr` is used to determine the initial learning rate. It takes a default value of 0.01 since we inherit from `mx.lr_scheduler.LRScheduler`, but it can be set as a property of the schedule. We will see later in this tutorial that `base_lr` is set automatically when providing the `lr_schedule` to `Optimizer`. Also be aware that the schedules in `mx.lr_scheduler` have state (i.e. counters, etc) so calling the schedule out of order may give unexpected results.\n\nWe can define non-uniform intervals with `MultiFactorScheduler` and in the example below we halve the learning rate _after_ the 250th, 750th (i.e. a step length of 500 iterations) and 900th (a step length of 150 iterations). As before, the learning rate of the 250th iteration will be 1 and the 251th iteration will be 0.5.\n\n\n```{.python .input}\nschedule = mx.lr_scheduler.MultiFactorScheduler(step=[250, 750, 900], factor=0.5)\nschedule.base_lr = 1\nplot_schedule(schedule)\n```\n\n\n![lr multifactor](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/lr_schedules/multifactor.png) <!--notebook-skip-line-->\n\n\n### Polynomial Schedule\n\nStepwise schedules and the discontinuities they introduce may sometimes lead to instability in the optimization, so in some cases smoother schedules are preferred. `PolyScheduler` gives a smooth decay using a polynomial function and reaches a learning rate of 0 after `max_update` iterations. In the example below, we have a quadratic function (`pwr=2`) that falls from 0.998 at iteration 1 to 0 at iteration 1000. After this the learning rate stays at 0, so nothing will be learnt from `max_update` iterations onwards.\n\n\n```{.python .input}\nschedule = mx.lr_scheduler.PolyScheduler(max_update=1000, base_lr=1, pwr=2)\nplot_schedule(schedule)\n```\n\n\n![lr poly](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/lr_schedules/polynomial.png) <!--notebook-skip-line-->\n\n\nNote: unlike `FactorScheduler`, the `base_lr` is set as an argument when instantiating the schedule.\n\nAnd we don't evaluate at `iteration=0` (to get `base_lr`) since we are working with schedules starting at `iteration=1`.\n\n### Custom Schedules\n\nYou can implement your own custom schedule with a function or callable class, that takes an integer denoting the iteration index (starting at 1) and returns a float representing the learning rate to be used for that iteration. We implement the Cosine Annealing Schedule in the example below as a callable class (see `__call__` method).\n\n\n```{.python .input}\nclass CosineAnnealingSchedule():\n    def __init__(self, min_lr, max_lr, cycle_length):\n        self.min_lr = min_lr\n        self.max_lr = max_lr\n        self.cycle_length = cycle_length\n\n    def __call__(self, iteration):\n        if iteration <= self.cycle_length:\n            unit_cycle = (1 + math.cos(iteration * math.pi / self.cycle_length)) / 2\n            adjusted_cycle = (unit_cycle * (self.max_lr - self.min_lr)) + self.min_lr\n            return adjusted_cycle\n        else:\n            return self.min_lr\n\n\nschedule = CosineAnnealingSchedule(min_lr=0, max_lr=1, cycle_length=1000)\nplot_schedule(schedule)\n```\n\n\n![lr cosine](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/lr_schedules/cosine.png) <!--notebook-skip-line-->\n\n\n## Using Schedules\n\nWhile training a simple handwritten digit classifier on the MNIST dataset, we take a look at how to use a learning rate schedule during training. Our demonstration model is a basic convolutional neural network. We start by preparing our `DataLoader` and defining the network.\n\nAs discussed above, the schedule should return a learning rate given an (1-based) iteration index.\n\n\n```{.python .input}\n# Use GPU if one exists, else use CPU\ndevice = mx.gpu() if mx.device.num_gpus() else mx.cpu()\n\n# MNIST images are 28x28. Total pixels in input layer is 28x28 = 784\nnum_inputs = 784\n# Clasify the images into one of the 10 digits\nnum_outputs = 10\n# 64 images in a batch\nbatch_size = 64\n\n# Load the training data\ntrain_dataset = mx.gluon.data.vision.MNIST(train=True).transform_first(transforms.ToTensor())\ntrain_dataloader = mx.gluon.data.DataLoader(train_dataset, batch_size, shuffle=True, num_workers=5)\n\n# Build a simple convolutional network\ndef build_cnn():\n    net = nn.HybridSequential()\n    # First convolution\n    net.add(nn.Conv2D(channels=10, kernel_size=5, activation='relu'))\n    net.add(nn.MaxPool2D(pool_size=2, strides=2))\n    # Second convolution\n    net.add(nn.Conv2D(channels=20, kernel_size=5, activation='relu'))\n    net.add(nn.MaxPool2D(pool_size=2, strides=2))\n    # Flatten the output before the fully connected layers\n    net.add(nn.Flatten())\n    # First fully connected layers with 512 neurons\n    net.add(nn.Dense(512, activation=\"relu\"))\n    # Second fully connected layer with as many neurons as the number of classes\n    net.add(nn.Dense(num_outputs))\n    return net\n\nnet = build_cnn()\n```\n\nWe then initialize our network (technically deferred until we pass the first batch) and define the loss.\n\n\n```{.python .input}\n# Initialize the parameters with Xavier initializer\nnet.initialize(mx.init.Xavier(), device=device)\n# Use cross entropy loss\nsoftmax_cross_entropy = mx.gluon.loss.SoftmaxCrossEntropyLoss()\n```\n\nWe're now ready to create our schedule, and in this example we opt for a stepwise decay schedule using `MultiFactorScheduler`. Since we're only training a demonstration model for a limited number of epochs (10 in total) we will exaggerate the schedule and drop the learning rate by 90% after the 4th, 7th and 9th epochs. We call these steps, and the drop occurs _after_ the step index. Schedules are defined for iterations (i.e. training batches), so we must represent our steps in iterations too.\n\n\n```{.python .input}\nsteps_epochs = [4, 7, 9]\n# assuming we keep partial batches, see `last_batch` parameter of DataLoader\niterations_per_epoch = math.ceil(len(train_dataset) / batch_size)\n# iterations just before starts of epochs (iterations are 1-indexed)\nsteps_iterations = [s*iterations_per_epoch for s in steps_epochs]\nprint(\"Learning rate drops after iterations: {}\".format(steps_iterations))\n```\n\n\n```\nLearning rate drops after iterations: [3752, 6566, 8442]\n```\n\n\n```{.python .input}\nschedule = mx.lr_scheduler.MultiFactorScheduler(step=steps_iterations, factor=0.1)\n```\n\n**We create our `Optimizer` and pass the schedule via the `lr_scheduler` parameter.** In this example we're using Stochastic Gradient Descent.\n\n\n```{.python .input}\nsgd_optimizer = mx.optimizer.SGD(learning_rate=0.03, lr_scheduler=schedule)\n```\n\nAnd we use this optimizer (with schedule) in our `Trainer` and train for 10 epochs. Alternatively, we could have set the `optimizer` to the string `sgd`, and pass a dictionary of the optimizer parameters directly to the trainer using `optimizer_params`.\n\n\n```{.python .input}\ntrainer = mx.gluon.Trainer(params=net.collect_params(), optimizer=sgd_optimizer)\n```\n\n\n```{.python .input}\nnum_epochs = 10\n# epoch and batch counts starting at 1\nfor epoch in range(1, num_epochs+1):\n    # Iterate through the images and labels in the training data\n    for batch_num, (data, label) in enumerate(train_dataloader, start=1):\n        # get the images and labels\n        data = data.to_device(device)\n        label = label.to_device(device)\n        # Ask autograd to record the forward pass\n        with mx.autograd.record():\n            # Run the forward pass\n            output = net(data)\n            # Compute the loss\n            loss = softmax_cross_entropy(output, label)\n        # Compute gradients\n        loss.backward()\n        # Update parameters\n        trainer.step(data.shape[0])\n\n        # Show loss and learning rate after first iteration of epoch\n        if batch_num == 1:\n            curr_loss = mx.np.mean(loss).item()\n            curr_lr = trainer.learning_rate\n            print(\"Epoch: %d; Batch %d; Loss %f; LR %f\" % (epoch, batch_num, curr_loss, curr_lr))\n```\n\nEpoch: 1; Batch 1; Loss 2.304071; LR 0.030000 <!--notebook-skip-line-->\n\nEpoch: 2; Batch 1; Loss 0.059640; LR 0.030000 <!--notebook-skip-line-->\n\nEpoch: 3; Batch 1; Loss 0.072601; LR 0.030000 <!--notebook-skip-line-->\n\nEpoch: 4; Batch 1; Loss 0.042228; LR 0.030000 <!--notebook-skip-line-->\n\nEpoch: 5; Batch 1; Loss 0.025745; LR 0.003000 <!--notebook-skip-line-->\n\nEpoch: 6; Batch 1; Loss 0.027391; LR 0.003000 <!--notebook-skip-line-->\n\nEpoch: 7; Batch 1; Loss 0.048237; LR 0.003000 <!--notebook-skip-line-->\n\nEpoch: 8; Batch 1; Loss 0.024213; LR 0.000300 <!--notebook-skip-line-->\n\nEpoch: 9; Batch 1; Loss 0.008892; LR 0.000300 <!--notebook-skip-line-->\n\nEpoch: 10; Batch 1; Loss 0.006875; LR 0.000030 <!--notebook-skip-line-->\n\n\nWe see that the learning rate starts at 0.03, and falls to 0.00003 by the end of training as per the schedule we defined.\n\n### Manually setting the learning rate: Gluon API only\n\nWhen using the method above you don't need to manually keep track of iteration count and set the learning rate, so this is the recommended approach for most cases. Sometimes you might want more fine-grained control over setting the learning rate though, so Gluon's `Trainer` provides the `set_learning_rate` method for this.\n\nWe replicate the example above, but now keep track of the `iteration_idx`, call the schedule and set the learning rate appropriately using `set_learning_rate`. We also use `schedule.base_lr` to set the initial learning rate for the schedule since we are calling the schedule directly and not using it as part of the `Optimizer`.\n\n\n```{.python .input}\nnet = build_cnn()\nnet.initialize(mx.init.Xavier(), device=device)\n\nschedule = mx.lr_scheduler.MultiFactorScheduler(step=steps_iterations, factor=0.1)\nschedule.base_lr = 0.03\nsgd_optimizer = mx.optimizer.SGD()\ntrainer = mx.gluon.Trainer(params=net.collect_params(), optimizer=sgd_optimizer)\n\niteration_idx = 1\nnum_epochs = 10\n# epoch and batch counts starting at 1\nfor epoch in range(1, num_epochs + 1):\n    # Iterate through the images and labels in the training data\n    for batch_num, (data, label) in enumerate(train_dataloader, start=1):\n        # get the images and labels\n        data = data.to_device(device)\n        label = label.to_device(device)\n        # Ask autograd to record the forward pass\n        with mx.autograd.record():\n            # Run the forward pass\n            output = net(data)\n            # Compute the loss\n            loss = softmax_cross_entropy(output, label)\n        # Compute gradients\n        loss.backward()\n        # Update the learning rate\n        lr = schedule(iteration_idx)\n        trainer.set_learning_rate(lr)\n        # Update parameters\n        trainer.step(data.shape[0])\n        # Show loss and learning rate after first iteration of epoch\n        if batch_num == 1:\n            curr_loss = mx.np.mean(loss).item()\n            curr_lr = trainer.learning_rate\n            print(\"Epoch: %d; Batch %d; Loss %f; LR %f\" % (epoch, batch_num, curr_loss, curr_lr))\n        iteration_idx += 1\n```\n\nEpoch: 1; Batch 1; Loss 2.334119; LR 0.030000 <!--notebook-skip-line-->\n\nEpoch: 2; Batch 1; Loss 0.178930; LR 0.030000 <!--notebook-skip-line-->\n\nEpoch: 3; Batch 1; Loss 0.142640; LR 0.030000 <!--notebook-skip-line-->\n\nEpoch: 4; Batch 1; Loss 0.041116; LR 0.030000 <!--notebook-skip-line-->\n\nEpoch: 5; Batch 1; Loss 0.051049; LR 0.003000 <!--notebook-skip-line-->\n\nEpoch: 6; Batch 1; Loss 0.027170; LR 0.003000 <!--notebook-skip-line-->\n\nEpoch: 7; Batch 1; Loss 0.083776; LR 0.003000 <!--notebook-skip-line-->\n\nEpoch: 8; Batch 1; Loss 0.082553; LR 0.000300 <!--notebook-skip-line-->\n\nEpoch: 9; Batch 1; Loss 0.027984; LR 0.000300 <!--notebook-skip-line-->\n\nEpoch: 10; Batch 1; Loss 0.030896; LR 0.000030 <!--notebook-skip-line-->\n\n\nOnce again, we see the learning rate start at 0.03, and fall to 0.00003 by the end of training as per the schedule we defined.\n\n## Advanced Schedules\n\nWe have a related tutorial on Advanced Learning Rate Schedules that shows reference implementations of schedules that give state-of-the-art results. We look at cyclical schedules applied to a variety of cycle shapes, and many other techniques such as warm-up and cool-down.\n\n<!-- INSERT SOURCE DOWNLOAD BUTTONS -->\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/training/learning_rates/learning_rate_schedules_advanced.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Advanced Learning Rate Schedules\n\nGiven the importance of learning rate and the learning rate schedule for training neural networks, there have been a number of research papers published recently on the subject. Although many practitioners are using simple learning rate schedules such as stepwise decay, research has shown that there are other strategies that work better in most situations. We implement a number of different schedule shapes in this tutorial and introduce cyclical schedules.\n\nSee the \"Learning Rate Schedules\" tutorial for a more basic overview of learning rates, and an example of how to use them while training your own models.\n\n\n```{.python .input}\n%matplotlib inline\nimport copy\nimport math\nimport mxnet as mx\nimport numpy as np\nimport matplotlib.pyplot as plt\n```\n\n```{.python .input}\ndef plot_schedule(schedule_fn, iterations=1500):\n    # Iteration count starting at 1\n    iterations = [i+1 for i in range(iterations)]\n    lrs = [schedule_fn(i) for i in iterations]\n    plt.scatter(iterations, lrs)\n    plt.xlabel(\"Iteration\")\n    plt.ylabel(\"Learning Rate\")\n    plt.show()\n```\n\n## Custom Schedule Shapes\n\n### (Slanted) Triangular\n\nWhile trying to push the boundaries of batch size for faster training, [Priya Goyal et al. (2017)](https://arxiv.org/abs/1706.02677) found that having a smooth linear warm up in the learning rate at the start of training improved the stability of the optimizer and lead to better solutions. It was found that a smooth increases gave improved performance over stepwise increases.\n\nWe look at \"warm-up\" in more detail later in the tutorial, but this could be viewed as a specific case of the **\"triangular\"** schedule that was proposed by [Leslie N. Smith (2015)](https://arxiv.org/abs/1506.01186). Quite simply, the schedule linearly increases then decreases between a lower and upper bound. Originally it was suggested this schedule be used as part of a cyclical schedule but more recently researchers have been using a single cycle.\n\nOne adjustment proposed by [Jeremy Howard, Sebastian Ruder (2018)](https://arxiv.org/abs/1801.06146) was to change the ratio between the increasing and decreasing stages, instead of the 50:50 split. Changing the increasing fraction (`inc_fraction!=0.5`) leads to a **\"slanted triangular\"** schedule. Using `inc_fraction<0.5` tends to give better results.\n\n\n```{.python .input}\nclass TriangularSchedule():\n    def __init__(self, min_lr, max_lr, cycle_length, inc_fraction=0.5):\n        \"\"\"\n        min_lr: lower bound for learning rate (float)\n        max_lr: upper bound for learning rate (float)\n        cycle_length: iterations between start and finish (int)\n        inc_fraction: fraction of iterations spent in increasing stage (float)\n        \"\"\"\n        self.min_lr = min_lr\n        self.max_lr = max_lr\n        self.cycle_length = cycle_length\n        self.inc_fraction = inc_fraction\n\n    def __call__(self, iteration):\n        if iteration <= self.cycle_length*self.inc_fraction:\n            unit_cycle = iteration * 1 / (self.cycle_length * self.inc_fraction)\n        elif iteration <= self.cycle_length:\n            unit_cycle = (self.cycle_length - iteration) * 1 / (self.cycle_length * (1 - self.inc_fraction))\n        else:\n            unit_cycle = 0\n        adjusted_cycle = (unit_cycle * (self.max_lr - self.min_lr)) + self.min_lr\n        return adjusted_cycle\n```\n\nWe look an example of a slanted triangular schedule that increases from a learning rate of 1 to 2, and back to 1 over 1000 iterations. Since we set `inc_fraction=0.2`, 200 iterations are used for the increasing stage, and 800 for the decreasing stage. After this, the schedule stays at the lower bound indefinitely.\n\n\n```{.python .input}\nschedule = TriangularSchedule(min_lr=1, max_lr=2, cycle_length=1000, inc_fraction=0.2)\nplot_schedule(schedule)\n```\n\n\n![lr adv triangular](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/lr_schedules/adv_triangular.png) <!--notebook-skip-line-->\n\n\n### Cosine\n\nContinuing with the idea that smooth decay profiles give improved performance over stepwise decay, [Ilya Loshchilov, Frank Hutter (2016)](https://arxiv.org/abs/1608.03983) used **\"cosine annealing\"** schedules to good effect. As with triangular schedules, the original idea was that this should be used as part of a cyclical schedule, but we begin by implementing the cosine annealing component before the full Stochastic Gradient Descent with Warm Restarts (SGDR) method later in the tutorial.\n\n\n```{.python .input}\nclass CosineAnnealingSchedule():\n    def __init__(self, min_lr, max_lr, cycle_length):\n        \"\"\"\n        min_lr: lower bound for learning rate (float)\n        max_lr: upper bound for learning rate (float)\n        cycle_length: iterations between start and finish (int)\n        \"\"\"\n        self.min_lr = min_lr\n        self.max_lr = max_lr\n        self.cycle_length = cycle_length\n\n    def __call__(self, iteration):\n        if iteration <= self.cycle_length:\n            unit_cycle = (1 + math.cos(iteration * math.pi / self.cycle_length)) / 2\n            adjusted_cycle = (unit_cycle * (self.max_lr - self.min_lr)) + self.min_lr\n            return adjusted_cycle\n        else:\n            return self.min_lr\n```\n\nWe look at an example of a cosine annealing schedule that smoothing decreases from a learning rate of 2 to 1 across 1000 iterations. After this, the schedule stays at the lower bound indefinietly.\n\n\n```{.python .input}\nschedule = CosineAnnealingSchedule(min_lr=1, max_lr=2, cycle_length=1000)\nplot_schedule(schedule)\n```\n\n\n![lr adv cosine](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/lr_schedules/adv_cosine.png) <!--notebook-skip-line-->\n\n\n## Custom Schedule Modifiers\n\nWe now take a look some adjustments that can be made to existing schedules. We see how to add linear warm-up and its compliment linear cool-down, before using this to implement the \"1-Cycle\" schedule used by [Leslie N. Smith, Nicholay Topin (2017)](https://arxiv.org/abs/1708.07120) for \"super-convergence\". We then look at cyclical schedules and implement the original cyclical schedule from [Leslie N. Smith (2015)](https://arxiv.org/abs/1506.01186) before finishing with a look at [\"SGDR: Stochastic Gradient Descent with Warm Restarts\" by Ilya Loshchilov, Frank Hutter (2016)](https://arxiv.org/abs/1608.03983).\n\nUnlike the schedules above and those implemented in `mx.lr_scheduler`, these classes are designed to modify existing schedules so they take the argument `schedule` (for initialized schedules) or `schedule_class` when being initialized.\n\n### Warm-Up\n\nUsing the idea of linear warm-up of the learning rate proposed in [\"Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour\" by Priya Goyal et al. (2017)](https://arxiv.org/abs/1706.02677), we implement a wrapper class that adds warm-up to an existing schedule. Going from `start_lr` to the initial learning rate of the `schedule` over `length` iterations, this adjustment is useful when training with large batch sizes.\n\n\n```{.python .input}\nclass LinearWarmUp():\n    def __init__(self, schedule, start_lr, length):\n        \"\"\"\n        schedule: a pre-initialized schedule (e.g. TriangularSchedule(min_lr=0.5, max_lr=2, cycle_length=500))\n        start_lr: learning rate used at start of the warm-up (float)\n        length: number of iterations used for the warm-up (int)\n        \"\"\"\n        self.schedule = schedule\n        self.start_lr = start_lr\n        # calling mx.lr_scheduler.LRScheduler effects state, so calling a copy\n        self.finish_lr = copy.copy(schedule)(0)\n        self.length = length\n\n    def __call__(self, iteration):\n        if iteration <= self.length:\n            return iteration * (self.finish_lr - self.start_lr)/(self.length) + self.start_lr\n        else:\n            return self.schedule(iteration - self.length)\n```\n\nAs an example, we add a linear warm-up of the learning rate (from 0 to 1 over 250 iterations) to a stepwise decay schedule. We first create the `MultiFactorScheduler` (and set the `base_lr`) and then pass it to `LinearWarmUp` to add the warm-up at the start. We can use `LinearWarmUp` with any other schedule including `CosineAnnealingSchedule`.\n\n\n```{.python .input}\nschedule = mx.lr_scheduler.MultiFactorScheduler(step=[250, 750, 900], factor=0.5)\nschedule.base_lr = 1\nschedule = LinearWarmUp(schedule, start_lr=0, length=250)\nplot_schedule(schedule)\n```\n\n\n![lr adv warmup](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/lr_schedules/adv_warmup.png) <!--notebook-skip-line-->\n\n\n### Cool-Down\n\nSimilarly, we could add a linear cool-down period to our schedule and this is used in the \"1-Cycle\" schedule proposed by [Leslie N. Smith, Nicholay Topin (2017)](https://arxiv.org/abs/1708.07120) to train neural networks very quickly in certain circumstances (coined \"super-convergence\"). We reduce the learning rate from its value at `start_idx` of `schedule` to `finish_lr` over a period of `length`, and then maintain `finish_lr` thereafter.\n\n\n```{.python .input}\nclass LinearCoolDown():\n    def __init__(self, schedule, finish_lr, start_idx, length):\n        \"\"\"\n        schedule: a pre-initialized schedule (e.g. TriangularSchedule(min_lr=0.5, max_lr=2, cycle_length=500))\n        finish_lr: learning rate used at end of the cool-down (float)\n        start_idx: iteration to start the cool-down (int)\n        length: number of iterations used for the cool-down (int)\n        \"\"\"\n        self.schedule = schedule\n        # calling mx.lr_scheduler.LRScheduler effects state, so calling a copy\n        self.start_lr = copy.copy(self.schedule)(start_idx)\n        self.finish_lr = finish_lr\n        self.start_idx = start_idx\n        self.finish_idx = start_idx + length\n        self.length = length\n\n    def __call__(self, iteration):\n        if iteration <= self.start_idx:\n            return self.schedule(iteration)\n        elif iteration <= self.finish_idx:\n            return (iteration - self.start_idx) * (self.finish_lr - self.start_lr) / (self.length) + self.start_lr\n        else:\n            return self.finish_lr\n```\n\nAs an example, we apply learning rate cool-down to a `MultiFactorScheduler`. Starting the cool-down at iteration 1000, we reduce the learning rate linearly from 0.125 to 0.001 over 500 iterations, and hold the learning rate at 0.001 after this.\n\n\n```{.python .input}\nschedule = mx.lr_scheduler.MultiFactorScheduler(step=[250, 750, 900], factor=0.5)\nschedule.base_lr = 1\nschedule = LinearCoolDown(schedule, finish_lr=0.001, start_idx=1000, length=500)\nplot_schedule(schedule)\n```\n\n\n![lr adv cooldown](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/lr_schedules/adv_cooldown.png) <!--notebook-skip-line-->\n\n\n#### 1-Cycle: for \"Super-Convergence\"\n\nSo we can implement the \"1-Cycle\" schedule proposed by [Leslie N. Smith, Nicholay Topin (2017)](https://arxiv.org/abs/1708.07120) we use a single and symmetric cycle of the triangular schedule above (i.e. `inc_fraction=0.5`), followed by a cool-down period of `cooldown_length` iterations.\n\n\n```{.python .input}\nclass OneCycleSchedule():\n    def __init__(self, start_lr, max_lr, cycle_length, cooldown_length=0, finish_lr=None):\n        \"\"\"\n        start_lr: lower bound for learning rate in triangular cycle (float)\n        max_lr: upper bound for learning rate in triangular cycle (float)\n        cycle_length: iterations between start and finish of triangular cycle: 2x 'stepsize' (int)\n        cooldown_length: number of iterations used for the cool-down (int)\n        finish_lr: learning rate used at end of the cool-down (float)\n        \"\"\"\n        if (cooldown_length > 0) and (finish_lr is None):\n            raise ValueError(\"Must specify finish_lr when using cooldown_length > 0.\")\n        if (cooldown_length == 0) and (finish_lr is not None):\n            raise ValueError(\"Must specify cooldown_length > 0 when using finish_lr.\")\n\n        finish_lr = finish_lr if (cooldown_length > 0) else start_lr\n        schedule = TriangularSchedule(min_lr=start_lr, max_lr=max_lr, cycle_length=cycle_length)\n        self.schedule = LinearCoolDown(schedule, finish_lr=finish_lr, start_idx=cycle_length, length=cooldown_length)\n\n    def __call__(self, iteration):\n        return self.schedule(iteration)\n```\n\nAs an example, we linearly increase and then decrease the learning rate from 0.1 to 0.5 and back over 500 iterations (i.e. single triangular cycle), before reducing the learning rate further to 0.001 over the next 750 iterations (i.e. cool-down).\n\n\n```{.python .input}\nschedule = OneCycleSchedule(start_lr=0.1, max_lr=0.5, cycle_length=500, cooldown_length=750, finish_lr=0.001)\nplot_schedule(schedule)\n```\n\n\n![lr adv onecycle](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/lr_schedules/adv_onecycle.png) <!--notebook-skip-line-->\n\n\n### Cyclical\n\nOriginally proposed by [Leslie N. Smith (2015)](https://arxiv.org/abs/1506.01186), the idea of cyclically increasing and decreasing the learning rate has been shown to give faster convergence and more optimal solutions. We implement a wrapper class that loops existing cycle-based schedules such as `TriangularSchedule` and `CosineAnnealingSchedule` to provide infinitely repeating schedules. We pass the schedule class (rather than an instance) because one feature of the `CyclicalSchedule` is to vary the `cycle_length` over time as seen in [Ilya Loshchilov, Frank Hutter (2016)](https://arxiv.org/abs/1608.03983) using `cycle_length_decay`. Another feature is the ability to decay the cycle magnitude over time with `cycle_magnitude_decay`.\n\n\n```{.python .input}\nclass CyclicalSchedule():\n    def __init__(self, schedule_class, cycle_length, cycle_length_decay=1, cycle_magnitude_decay=1, **kwargs):\n        \"\"\"\n        schedule_class: class of schedule, expected to take `cycle_length` argument\n        cycle_length: iterations used for initial cycle (int)\n        cycle_length_decay: factor multiplied to cycle_length each cycle (float)\n        cycle_magnitude_decay: factor multiplied learning rate magnitudes each cycle (float)\n        kwargs: passed to the schedule_class\n        \"\"\"\n        self.schedule_class = schedule_class\n        self.length = cycle_length\n        self.length_decay = cycle_length_decay\n        self.magnitude_decay = cycle_magnitude_decay\n        self.kwargs = kwargs\n\n    def __call__(self, iteration):\n        cycle_idx = 0\n        cycle_length = self.length\n        idx = self.length\n        while idx <= iteration:\n            cycle_length = math.ceil(cycle_length * self.length_decay)\n            cycle_idx += 1\n            idx += cycle_length\n        cycle_offset = iteration - idx + cycle_length\n\n        schedule = self.schedule_class(cycle_length=cycle_length, **self.kwargs)\n        return schedule(cycle_offset) * self.magnitude_decay**cycle_idx\n```\n\nAs an example, we implement the triangular cyclical schedule presented in [\"Cyclical Learning Rates for Training Neural Networks\" by Leslie N. Smith (2015)](https://arxiv.org/abs/1506.01186). We use slightly different terminology to the paper here because we use `cycle_length` that is twice the 'stepsize' used in the paper. We repeat cycles, each with a length of 500 iterations and lower and upper learning rate bounds of 0.5 and 2 respectively.\n\n\n```{.python .input}\nschedule = CyclicalSchedule(TriangularSchedule, min_lr=0.5, max_lr=2, cycle_length=500)\nplot_schedule(schedule)\n```\n\n\n![lr adv cyclical](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/lr_schedules/adv_cyclical.png) <!--notebook-skip-line-->\n\n\nAnd lastly, we implement the scheduled used in [\"SGDR: Stochastic Gradient Descent with Warm Restarts\" by Ilya Loshchilov, Frank Hutter (2016)](https://arxiv.org/abs/1608.03983). We repeat cosine annealing schedules, but each time we halve the magnitude and double the cycle length.\n\n\n```{.python .input}\nschedule = CyclicalSchedule(CosineAnnealingSchedule, min_lr=0.01, max_lr=2,\n                            cycle_length=250, cycle_length_decay=2, cycle_magnitude_decay=0.5)\nplot_schedule(schedule)\n```\n\n\n![lr adv sgdr](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/lr_schedules/adv_sgdr.png) <!--notebook-skip-line-->\n\n\n**_Want to learn more?_** Checkout the \"Learning Rate Schedules\" tutorial for a more basic overview of learning rates found in `mx.lr_scheduler`, and an example of how to use them while training your own models.\n\n<!-- INSERT SOURCE DOWNLOAD BUTTONS -->\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/training/normalization/index.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Normalization Blocks\n\nWhen training deep neural networks there are a number of techniques that are thought to be essential for model convergence. One important area is deciding how to initialize the parameters of the network. Using techniques such as [Xavier](../../../../../api/initializer/index.rst#mxnet.initializer.Xavier) initialization, we can can improve the gradient flow through the network at the start of training. Another important technique is normalization: i.e. scaling and shifting certain values towards a distribution with a mean of 0 (i.e. zero-centered) and a standard distribution of 1 (i.e. unit variance). Which values you normalize depends on the exact method used as we'll see later on.\n\n<p align=\"center\">\n    <img src=\"./imgs/data_normalization.jpeg\" alt=\"drawing\" width=\"500\"/>\n    <p align=\"center\">Figure 1: Data Normalization\n        <a href=\"http://cs231n.github.io/neural-networks-2/\">(Source)</a>\n    </p>\n</p>\n\nWhy does this help? [Some research](https://papers.nips.cc/paper/7515-how-does-batch-normalization-help-optimization.pdf) has found that networks with normalization have a loss function that's easier to optimize using stochastic gradient descent. Other reasons are that it prevents saturation of activations and prevents certain features from dominating due to differences in scale.\n\n### Data Normalization\n\nOne of the first applications of normalization is on the input data to the network. You can do this with the following steps:\n\n* **Step 1** is to calculate the mean and standard deviation of the entire training dataset. You'll usually want to do this for each channel separately. Sometimes you'll see normalization on images applied per pixel, but per channel is more common.\n* **Step 2** is to use these statistics to normalize each batch for training and for inference too.\n\nTip: A `BatchNorm` layer at the start of your network can have a similar effect (see 'Beta and Gamma' section for details on how this can be achieved). You won't need to manually calculate and keep track of the normalization statistics.\n\nWarning: You should calculate the normalization means and standard deviations using the training dataset only. Any leakage of information from you testing dataset will effect the reliability of your testing metrics.\n\nWhen using pre-trained models from the [Gluon Model Zoo](https://mxnet.apache.org/versions/master/api/python/docs/api/gluon/model_zoo/index.html) you'll usually see the normalization statistics used for training (i.e. statistics from step 1). You'll want to use these statistics to normalize your own input data for fine-tuning or inference with these models. Using `transforms.Normalize` is one way of applying the normalization, and this should be used in the `Dataset`.\n\n```{.python .input}\nimport mxnet as mx\nfrom mxnet.gluon.data.vision.transforms import Normalize\n\nimage_int = mx.np.random.randint(low=0, high=256, size=(1,3,2,2))\nimage_float = image_int.astype('float32')/255\n# the following normalization statistics are taken from gluon model zoo\nnormalizer = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\nimage = normalizer(image_float)\nimage\n```\n\n### Activation Normalization\n\nWe don't have to limit ourselves to normalizing the inputs to the network either. A similar idea can be applied inside the network too, and we can normalize activations between certain layer operations. With deep neural networks most of the convergence benefits described are from this type of normalization.\n\nMXNet Gluon has 3 of the most commonly used normalization blocks: `BatchNorm`, `LayerNorm` and `InstanceNorm`. You can use them in networks just like any other MXNet Gluon Block, and are often used after `Activation` Blocks.\n\nWatch Out: Check the architecture of models carefully because sometimes the normalization is applied before the `Activation`.\n\nAdvanced: all of the following methods begin by normalizing certain input distribution (i.e. zero-centered with unit variance), but then shift by (a trainable parameter) beta and scale by (a trainable parameter) gamma. Overall the effect is changing the input distribution to have a mean of beta and a variance of gamma, also allowing to the network to 'undo' the effect of the normalization if necessary.\n\n## Batch Normalization\n\nFigure 1: `BatchNorm` on NCHW data | Figure 2: `BatchNorm` on NTC data\n- | -\n![normalization nchw bn](/_static/NCHW_BN.png) | ![normalization ntc bn](/_static/NTC_BN.png)\n(e.g. batch of images) using the default of `axis=1` | (e.g. batch of sequences) overriding the default with `axis=2` (or `axis=-1`)\n\nOne of the most popular normalization techniques is Batch Normalization, usually called BatchNorm for short. We normalize the activations **across all samples in a batch** for each of the channels independently. See Figure 1. We calculate two batch (or local) statistics for every channel to perform the normalization: the mean and variance of the activations in that channel for all samples in a batch. And we use these to shift and scale respectively.\n\nTip: we can use this at the start of a network to perform data normalization, although this is not exactly equivalent to the data normalization example seen above (that had fixed normalization statistics). With `BatchNorm` the normalization statistics depend on the batch, so could change each batch, and there can also be a post-normalization shift and scale.\n\nWarning: the estimates for the batch mean and variance can themselves have high variance when the batch size is small (or when the spatial dimensions of samples are small). This can lead to instability during training, and unreliable estimates for the global statistics.\n\nWarning: it seems that `BatchNorm` is better suited to convolutional networks (CNNs) than recurrent networks (RNNs). We expect the input distribution to the recurrent cell to change over time, so normalization over time doesn't work well. `LayerNorm` is better suited for this case. When you do *need* to use `BatchNorm` on sequential data, make sure the `axis` parameter is set correctly. With data in NTC format you should set `axis=2` (or `axis=-1` equivalently). See Figure 2.\n\nAs an example, we'll apply `BatchNorm` to a batch of 2 samples, each with 2 channels, and both height and width of 2 (in NCHW format).\n\n\n```{.python .input}\ndata = mx.np.arange(start=0, stop=2*2*2*2).reshape(2, 2, 2, 2)\nprint(data)\n```\n\nWith MXNet Gluon we can apply batch normalization with the `mx.gluon.nn.BatchNorm` block. It can be created and used just like any other MXNet Gluon block (such as `Conv2D`). Its input will typically be unnormalized activations from the previous layer, and the output will be the normalized activations ready for the next layer. Since we're using data in NCHW format we can use the default axis.\n\n\n```{.python .input}\nnet = mx.gluon.nn.BatchNorm()\n```\n\nWe still need to initialize the block because it has a number of trainable parameters, as we'll see later on.\n\n\n```{.python .input}\nnet.initialize()\n```\n\nWe can now run the network as we would during training (under `autograd.record` context scope).\n\nRemember: `BatchNorm` runs differently during training and inference. When training, the batch statistics are used for normalization. During inference, a exponentially smoothed average of the batch statistics that have been observed during training is used instead.\n\nWarning: `BatchNorm` assumes the channel dimension is the 2nd in order (i.e. `axis=1`). You need to ensure your data has a channel dimension, and change the `axis` parameter of `BatchNorm` if it's not the 2nd dimension. A batch of greyscale images of shape `(100,32,32)` would not work, since the 2nd dimension is height and not channel. You'd need to add a channel dimension using `data.expand_dims(1)` in this case to give shape `(100,1,32,32)`.\n\n\n```{.python .input}\nwith mx.autograd.record():\n    output = net(data)\n    loss = mx.np.abs(output)\nloss.backward()\nprint(output)\n```\n\nWe can immediately see the activations have been scaled down and centered around zero. Activations are the same for each channel, because each channel was normalized independently. We can do a quick sanity check on these results, by manually calculating the batch mean and variance for each channel.\n\n\n```{.python .input}\naxes = list(range(data.ndim))\ndel axes[1]\nbatch_means = mx.np.mean(data, axis=axes)\nbatch_square = mx.np.square(data - batch_means.reshape(1, -1, 1, 1))\naxes = list(range(batch_square.ndim))\ndel axes[1]\nbatch_vars = mx.np.mean(batch_square, axis=axes)\nprint('batch_means:', batch_means.asnumpy())\nprint('batch_vars:', batch_vars.asnumpy())\n```\n\nAnd use these to scale the first entry in `data`, to confirm the `BatchNorm` calculation of `-1.324` was correct.\n\n\n```{.python .input}\nprint(\"manually calculated:\", ((data[0][0][0][0] - batch_means[0])/mx.np.sqrt(batch_vars[0])).asnumpy())\nprint(\"automatically calculated:\", output[0][0][0][0].asnumpy())\n```\n\nAs mentioned before, `BatchNorm` has a number of parameters that update throughout training. 2 of the parameters are not updated in the typical fashion (using gradients), but instead are updated deterministically using exponential smoothing. We need to keep track of the average mean and variance of batches during training, so that we can use these values for normalization during inference.\n\nWhy are global statistics needed? Often during inference, we have a batch size of 1 so batch variance would be impossible to calculate. We can just use global statistics instead. And we might get a data distribution shift between training and inference data, which shouldn't just be normalized away.\n\nAdvanced: when using a pre-trained model inside another model (e.g. a pre-trained ResNet as a image feature extractor inside an instance segmentation model) you might want to use global statistics of the pre-trained model *during training*. Setting `use_global_stats=True` is a method of using the global running statistics during training, and preventing the global statistics from updating. It has no effect on inference mode.\n\nAfter a single step (specifically after the `backward` call) we can see the `running_mean` and `running_var` have been updated.\n\n```{.python .input}\nprint('running_mean:', net.running_mean.data().asnumpy())\nprint('running_var:', net.running_var.data().asnumpy())\n```\n\nYou should notice though that these running statistics do not match the batch statistics we just calculated. And instead they are just 10% of the value we'd expect. We see this because of the exponential average process, and because the `momentum` parameter of `BatchNorm` is equal to 0.9 : i.e. 10% of the new value, 90% of the old value (which was initialized to 0). Over time the running statistics will converge to the statistics of the input distribution, while still being flexible enough to adjust to shifts in the input distribution. Using the same batch another 100 times (which wouldn't happen in practice), we can see the running statistics converge to the batch statsitics calculated before.\n\n\n```{.python .input}\nfor i in range(100):\n    with mx.autograd.record():\n        output = net(data)\n        loss = mx.np.abs(output)\n    loss.backward()\nprint('running_means:', net.running_mean.data().asnumpy())\nprint('running_vars:', net.running_var.data().asnumpy())\n```\n\n#### Beta and Gamma\n\nAs mentioned previously, there are two additional parameters in `BatchNorm` which are trainable in the typical fashion (with gradients). `beta` is used to shift and `gamma` is used to scale the normalized distribution, which allows the network to 'undo' the effects of normalization if required.\n\nAdvanced: Sometimes used for input normalization, you can prevent `beta` shifting and `gamma` scaling by setting the learning rate multipler (i.e. `lr_mult`) of these parameters to 0. Zero centering and scaling to unit variance will still occur, only post normalization shifting and scaling will prevented. See [this discussion post](https://discuss.mxnet.io/t/mxnet-use-batch-norm-for-input-scaling/3581/3) for details.\n\nWe haven't updated these parameters yet, so they should still be as initialized. You can see the default for `beta` is 0 (i.e. not shift) and `gamma` is 1 (i.e. not scale), so the initial behaviour is to keep the distribution unit normalized.\n\n\n```{.python .input}\nprint('beta:', net.beta.data().asnumpy())\nprint('gamma:', net.gamma.data().asnumpy())\n```\n\nWe can also check the gradient on these parameters. Since we were finding the gradient of the sum of absolute values, we would expect the gradient of `gamma` to be equal to the number of points in the data (i.e. 16). So to minimize the loss we'd decrease the value of `gamma`, which would happen as part of a `trainer.step`.\n\n\n```{.python .input}\nprint('beta gradient:', net.beta.grad().asnumpy())\nprint('gamma gradient:', net.gamma.grad().asnumpy())\n```\n\n#### Inference Mode\n\nWhen it comes to inference, `BatchNorm` uses the global statistics that were calculated during training. Since we're using the same batch of data over and over again (and our global running statistics have converged), we get a very similar result to using training mode. `beta` and `gamma` are also applied by default (unless explicitly removed).\n\n\n```{.python .input}\noutput = net(data)\nprint(output)\n```\n\n## Layer Normalization\n\nAn alternative to `BatchNorm` that is better suited to recurrent networks (RNNs) is called `LayerNorm`. Unlike `BatchNorm` which normalizes across all samples of a batch per channel, `LayerNorm` normalizes **across all channels of a single sample**.\n\nSome of the disadvantages of `BatchNorm` no longer apply. Small batch sizes are no longer an issue, since normalization statistics are calculated on single samples. And confusion around training and inference modes disappears because `LayerNorm` is the same for both modes.\n\nWarning: similar to having a small batch sizes in `BatchNorm`, you may have issues with `LayerNorm` if the input channel size is small. Using embeddings with a large enough dimension size avoids this (approx >20).\n\nWarning: currently MXNet Gluon's implementation of `LayerNorm` is applied along a single axis (which should be the channel axis). Other frameworks have the option to apply normalization across multiple axes, which leads to differences in `LayerNorm` on NCHW input by default. See Figure 3. Other frameworks can normalize over C, H and W, not just C as with MXNet Gluon.\n\nRemember: `LayerNorm` is intended to be used with data in NTC format so the default normalization axis is set to -1 (corresponding to C for channel). Change this to `axis=1` if you need to apply `LayerNorm` to data in NCHW format.\n\nFigure 3: `LayerNorm` on NCHW data | Figure 4: `LayerNorm` on NTC data\n- | -\n![normalization nchw ln](/_static/NCHW_LN.png) | ![normalization ntc ln](/_static/NTC_LN.png)\n(e.g. batch of images) overriding the default with `axis=1` | (e.g. batch of sequences) using the default of `axis=-1`\n\nAs an example, we'll apply `LayerNorm` to a batch of 2 samples, each with 4 time steps and 2 channels (in NTC format).\n\n\n```{.python .input}\ndata = mx.np.arange(start=0, stop=2*4*2).reshape(2, 4, 2)\nprint(data)\n```\n\nWith MXNet Gluon we can apply layer normalization with the `mx.gluon.nn.LayerNorm` block. We need to call `initialize` because `LayerNorm` has two learnable parameters by default: `beta` and `gamma` that are used for post normalization shifting and scaling of each channel.\n\n\n```{.python .input}\nnet = mx.gluon.nn.LayerNorm()\nnet.initialize()\noutput = net(data)\nprint(output)\n```\n\nWe can see that normalization has been applied across all channels for each time step and each sample.\n\nWe can also check the parameters `beta` and `gamma` and see that they are per channel (i.e. 2 of each in this example).\n\n\n```{.python .input}\nprint('beta:', net.beta.data().asnumpy())\nprint('gamma:', net.gamma.data().asnumpy())\n```\n\n##  Instance Normalization\n\nAnother less common normalization technique is called `InstanceNorm`, which can be useful for certain tasks such as image stylization. Unlike `BatchNorm` which normalizes across all samples of a batch per channel, `InstanceNorm` normalizes **across all spatial dimensions per channel per sample** (i.e. each sample of a batch is normalized independently).\n\nWatch out: `InstanceNorm` is better suited to convolutional networks (CNNs) than recurrent networks (RNNs). We expect the input distribution to the recurrent cell to change over time, so normalization over time doesn't work well. LayerNorm is better suited for this case.\n\nFigure 3: `InstanceNorm` on NCHW data | Figure 4: `InstanceNorm` on NTC data\n- | -\n![normalization nchw in](/_static/NCHW_IN.png) | ![normalization ntc in](/_static/NTC_IN.png)\n(e.g. batch of images) using the default `axis=1` | (e.g. batch of sequences) overiding the default with `axis=2` (or `axis=-1` equivalently)\n\nAs an example, we'll apply `InstanceNorm` to a batch of 2 samples, each with 2 channels, and both height and width of 2 (in NCHW format).\n\n\n```{.python .input}\ndata = mx.np.arange(start=0, stop=2*2*2*2).reshape(2, 2, 2, 2)\nprint(data)\n```\n\nWith MXNet Gluon we can apply instance normalization with the `mx.gluon.nn.InstanceNorm` block. We need to call `initialize` because InstanceNorm has two learnable parameters by default: `beta` and `gamma` that are used for post normalization shifting and scaling of each channel.\n\n\n```{.python .input}\nnet = mx.gluon.nn.InstanceNorm()\nnet.initialize()\noutput = net(data)\nprint(output)\n```\n\nWe can also check the parameters `beta` and `gamma` and see that they are per channel (i.e. 2 of each in this example).\n\n\n```{.python .input}\nprint('beta:', net.beta.data().asnumpy())\nprint('gamma:', net.gamma.data().asnumpy())\n```\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/gluon/training/trainer.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Trainer\n\nTraining a neural network model consists of iteratively performing three simple steps.\n\nThe first step is the forward step which computes the loss.  In MXNet Gluon, this first step is achieved by doing a forward pass by calling `net.forward(X)` or simply `net(X)` and then calling the loss function with the result of the forward pass and the labels. For example `l = loss_fn(net(X), y)`.\n\nThe second step is the backward step which computes the gradient of the loss with respect to the parameters. In Gluon, this step is  achieved by doing the first step in an [autograd.record()](../../../../api/autograd/index.rst#mxnet.autograd.record) scope to record the computations needed to calculate the loss, and then calling `l.backward()` to compute the gradient of the loss with respect to the parameters.\n\nThe final step is to update the neural network model parameters using an optimization algorithm. In Gluon, this step is performed by the [gluon.Trainer](../../../../api/gluon/trainer.rst#mxnet.gluon.Trainer) and is the subject of this guide. When creating a  Gluon `Trainer` you must provide a collection of parameters that need to be learnt. You also provide an `Optimizer` that will be used to update the parameters every training iteration when `trainer.step` is called.\n\n## Basic Usage\n\n### Network and Trainer\n\nTo illustrate how to use the Gluon `Trainer` we will create a simple perceptron model and create a `Trainer ` instance using the perceptron model parameters and a simple optimizer - `sgd` with learning rate as 1.\n\n```{.python .input}\nfrom mxnet import np, autograd, optimizer, gluon\n\nnet = gluon.nn.Dense(1)\nnet.initialize()\n\ntrainer = gluon.Trainer(net.collect_params(),\n                        optimizer='sgd', optimizer_params={'learning_rate':1})\n\n```\n\n### Forward and Backward Pass\n\nBefore we can use the `trainer` to update model parameters, we must first run the forward and backward passes. Here we implement a function to compute the first two steps (forward step and backward step) of training the perceptron on a random dataset.\n\n```{.python .input}\nbatch_size = 8\nX = np.random.uniform(size=(batch_size, 4))\ny = np.random.uniform(size=(batch_size,))\n\nloss = gluon.loss.L2Loss()\n\ndef forward_backward():\n    with autograd.record():\n        l = loss(net(X), y)\n    l.backward()\n\nforward_backward()\n```\n\n**Warning**: It is extremely important that the gradients of the loss function with respect to your model parameters are computed before running `trainer step`. A common way to introduce bugs to your model training code is to omit the `loss.backward()`before the update step.\n\n\n\nBefore updating, let's check the current network parameters.\n\n```{.python .input}\ncurr_weight = net.weight.data().copy()\nprint(curr_weight)\n```\n\n### `Trainer` step\n\nNow we will call the `step` method to perform one update. We provide the `batch_size` as an argument to normalize the size of the gradients and make it independent of the batch size. Otherwise we'd get larger gradients with larger batch sizes. We can see the network parameters have now changed.\n\n```{.python .input}\ntrainer.step(batch_size)\nprint(net.weight.data())\n```\n\nSince we used plain SGD, the update rule is $w = w - \\eta/b \\nabla \\ell$, where $b$ is the batch size and $\\nabla\\ell$ is the gradient of the loss function with respect to the weights and $\\eta$ is the learning rate.\n\nWe can verify it by running the following code snippet which is explicitly performing the SGD update.\n\n```{.python .input}\nprint(curr_weight - net.weight.grad() * 1 / batch_size)\n```\n\n\n\n## Advanced Usage\n\n### Using Optimizer Instance\n\nIn the previous example, we use the string argument `sgd` to select the optimization method, and `optimizer_params` to specify the optimization method arguments.\n\nAll pre-defined optimization methods can be passed in this way and the complete list of implemented optimizers is provided in the [mxnet.optimizer](../../../../api/optimizer/index.rst) module.\n\nHowever we can also pass an optimizer instance directly to the `Trainer` constructor.\n\nFor example:\n\n```{.python .input}\noptim = optimizer.Adam(learning_rate = 1)\ntrainer = gluon.Trainer(net.collect_params(), optim)\n```\n\n```{.python .input}\nforward_backward()\ntrainer.step(batch_size)\nnet.weight.data()\n```\n\nFor reference and implementation details about each optimizer, please refer to the [guide](../../optimizer/index.ipynb) and [API doc](../../../../api/optimizer/index.rst) for the `optimizer` module.\n\n### KVStore Options\n\nThe `Trainer` constructor also accepts the following keyword arguments for :\n\n- `kvstore` – how key value store  should be created for multi-gpu and distributed training. Check out  [mxnet.kvstore.KVStore](../../../../api/kvstore/index.rst) for more information. String options are any of the following ['local', 'device', 'dist_device_sync', 'dist_device_async'].\n- `compression_params` – Specifies type of gradient compression and additional arguments depending on the type of compression being used. See [mxnet.KVStore.set_gradient_compression_method](../../../../api/kvstore/generated/mxnet.kvstore.KVStore.rst) for more details on gradient compression.\n- `update_on_kvstore` – Whether to perform parameter updates on KVStore. If None, then the `Trainer` instance  will choose the more suitable option depending on the type of KVStore.\n\n### Changing the Learning Rate\n\nWe set the initial learning rate when creating a trainer by passing the learning rate as an `optimizer_param`. However, sometimes we may need to change the learning rate during training, for example when doing an explicit learning rate warmup schedule.  The trainer instance provides an easy way to achieve this.\n\nThe current training rate can be accessed through the `learning_rate` attribute.\n\n```{.python .input}\ntrainer.learning_rate\n```\n\nWe can change it through the `set_learning_rate` method.\n\n```{.python .input}\ntrainer.set_learning_rate(0.1)\ntrainer.learning_rate\n```\n\n\n\nIn addition, there are multiple pre-defined learning rate scheduling methods that are already implemented in the [mxnet.lr_scheduler](../../../../api/lr_scheduler/index.rst) module. The learning rate schedulers can be incorporated into your trainer by passing them in as an `optimizer_param` entry. Please refer to the [LR scheduler guide](./learning_rates/learning_rate_schedules.ipynb) to learn more.\n\n\n\n## Summary\n\n* The MXNet Gluon `Trainer` API is used to update the parameters of a network with a particular optimization algorithm.\n* After the forward and backward pass, the model update step is done in Gluon using `trainer.step()`.\n* A Gluon `Trainer` can be instantiated by passing in the name of the optimizer to use and the `optimizer_params` for that optimizer or alternatively by passing in an instance of `mxnet.optimizer.Optimizer`.\n* You can change the learning rate for a Gluon `Trainer` by setting the member variable but Gluon also provides a module for learning rate scheduling.\n\n\n\n## Next Steps\n\nWhile optimization and optimizers play a significant role in deep learning model training, there are still other important components to model training. Here are a few suggestions about where to look next.\n\n* The [Optimizer API](../../../../api/optimizer/index.rst) and [optimizer guide](../../optimizer/index.ipynb) have information about all the different optimizers implemented in MXNet and their update steps. The [Dive into Deep Learning](http://d2l.ai/chapter_optimization/index.html) book also has a chapter dedicated to optimization methods and explains various key optimizers in great detail.\n\n- Take a look at the [guide to parameter initialization](../blocks/init.ipynb) in MXNet to learn about what initialization schemes are already implemented, and how to implement your custom initialization schemes.\n- Also check out this  [guide on parameter management](../blocks/parameters.ipynb) to learn about how to manage model parameters in gluon.\n- Make sure to take a look at the [guide to scheduling learning rates](./learning_rates/learning_rate_schedules.ipynb) to learn how to create learning rate schedules to make your training converge faster.\n- Finally take a look at the [KVStore API](../../../../api/kvstore/index.rst) to learn how parameter values are synchronized over multiple devices.\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nPackages\n========\n\nThe documents in this unit dive into the details how each MXNet module works.\n\nHigh Level APIs\n---------------\n\n.. container:: cards\n\n   .. card::\n      :title: Gluon\n      :link: gluon/index.html\n\n      MXNet's imperative interface for Python. If you're new to MXNet, start here!\n\n\nShared APIs\n-----------\n\n.. container:: cards\n\n   .. card::\n      :title: NP API\n      :link: np/index.html\n\n      Hints on MXNet NP and NPX modules, an array library that provides NumPy-compatible interfaces.\n\n   .. card::\n      :title: Autograd API\n      :link: autograd/index.html\n\n      How to use Automatic Differentiation with the Autograd API.\n\n   .. card::\n      :title: Learning Rate\n      :link: gluon/training/learning_rates/learning_rate_schedules.html\n\n      How to use the Learning Rate Scheduler.\n\n   .. card::\n      :title: KVStore API\n      :link: kvstore/index.html\n\n      How to use the KVStore API for distributed training.\n\n   .. card::\n      :title: Data APIs\n      :link: gluon/data/index.html\n\n      How to use MXNet's data APIs.\n\n   .. card::\n      :title: Visualizations\n      :link: viz/index.html\n\n      How to use MXNet's visualization features.\n\n   .. card::\n      :title: ONNX\n      :link: onnx/index.html\n\n      How to use Open Neural Network Exchange (ONNX) with MXNet.\n\n   .. card::\n      :title: Optimizer\n      :link: optimizer/index.html\n\n      How to use the optimizers.\n\n   .. card::\n      :title: Legacy\n      :link: legacy/index.html\n\n      Legacy modules from MXNet 1.x.\n\n.. toctree::\n   :hidden:\n   :glob:\n\n   */index*\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/kvstore/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nKVStore\n=======\n\n.. container:: cards\n\n   .. card::\n      :title: Distributed Training Using the KVStore API\n      :link: kvstore.html\n\n      How to use the KVStore API to use multiple GPUs when training a model.\n\n\nReferences\n-----------------\n\n- `KVStore API. </api/python/docs/api/kvstore/index.html>`_\n\n.. toctree::\n   :hidden:\n   :glob:\n\n   *\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/kvstore/kvstore.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Distributed Key-Value Store\n\nKVStore is a place for data sharing. Think of it as a single object shared\nacross different devices (GPUs and computers), where each device can push data in\nand pull data out.\n\n## Initialization\n\nLet's consider a simple example: initializing\na (`int`, `NDArray`) pair into the store, and then pulling the value out:\n\n```{.python .input}\nimport mxnet as mx\n\nkv = mx.kv.create('local') # create a local kv store.\nshape = (2,3)\nkv.init(3, mx.np.ones(shape)*2)\na = mx.np.zeros(shape)\nkv.pull(3, out = a)\nprint(a.asnumpy())\n```\n\n`[[ 2.  2.  2.],[ 2.  2.  2.]]`<!--notebook-skip-line-->\n\n## Push, Aggregate, and Update\n\nFor any key that has been initialized, you can push a new value with the same shape to the key:\n\n```{.python .input}\nkv.push(3, mx.np.ones(shape)*8)\nkv.pull(3, out = a) # pull out the value\nprint(a.asnumpy())\n```\n\n`[[ 8.  8.  8.],[ 8.  8.  8.]]`<!--notebook-skip-line-->\n\nThe data for pushing can be stored on any device. Furthermore, you can push multiple\nvalues into the same key, where KVStore will first sum all of these\nvalues and then push the aggregated value. Here we will just demonstrate pushing a list of values on CPU.\nPlease note summation only happens if the value list is longer than one\n\n```{.python .input}\ndevices = [mx.cpu(i) for i in range(4)]\nb = [mx.np.ones(shape=shape, device=device) for device in devices]\nkv.push(3, b)\nkv.pull(3, out = a)\nprint(a.asnumpy())\n```\n\n`[[ 4.  4.  4.],[ 4.  4.  4.]]`<!--notebook-skip-line-->\n\nFor each push, KVStore combines the pushed value with the value stored using an\n`updater`. The default updater is `ASSIGN`. You can replace the default to\ncontrol how data is merged:\n\n```{.python .input}\ndef update(key, input, stored):\n    print(\"update on key: %d\" % key)\n    stored += input * 2\nkv._set_updater(update)\nkv.pull(3, out=a)\nprint(a.asnumpy())\n```\n\n`[[ 4.  4.  4.],[ 4.  4.  4.]]`<!--notebook-skip-line-->\n\n```{.python .input}\nkv.push(3, mx.np.ones(shape))\nkv.pull(3, out=a)\nprint(a.asnumpy())\n```\n\n`update on key: 3`<!--notebook-skip-line-->\n\n`[[ 6.  6.  6.],[ 6.  6.  6.]]`<!--notebook-skip-line-->\n\n\n## Pull\n\nYou've already seen how to pull a single key-value pair. Similarly, to push, you can\npull the value onto several devices with a single call:\n\n```{.python .input}\nb = [mx.np.ones(shape=shape, device=device) for device in devices]\nkv.pull(3, out = b)\nprint(b[1].asnumpy())\n```\n\n`[ 6.  6.  6.]],[[ 6.  6.  6.]`<!--notebook-skip-line-->\n\n## Handle a List of Key-Value Pairs\n\nAll operations introduced so far involve a single key. KVStore also provides\nan interface for a list of key-value pairs.\n\nFor a single device:\n\n```{.python .input}\nkeys = [5, 7, 9]\nkv.init(keys, [mx.np.ones(shape)]*len(keys))\nkv.push(keys, [mx.np.ones(shape)]*len(keys))\nb = [mx.np.zeros(shape)]*len(keys)\nkv.pull(keys, out = b)\nprint(b[1].asnumpy())\n```\n\n`update on key: 5`<!--notebook-skip-line-->\n\n`update on key: 7`<!--notebook-skip-line-->\n\n`update on key: 9`<!--notebook-skip-line-->\n\n`[[ 3.  3.  3.],[ 3.  3.  3.]]`<!--notebook-skip-line-->\n\nFor multiple devices:\n\n```{.python .input}\nb = [[mx.np.ones(shape=shape, device=device) for device in devices]] * len(keys)\nkv.push(keys, b)\nkv.pull(keys, out = b)\nprint(b[1][1].asnumpy())\n```\n\n`update on key: 5`<!--notebook-skip-line-->\n\n`update on key: 7`<!--notebook-skip-line-->\n\n`update on key: 9`<!--notebook-skip-line-->\n\n`[[ 11.  11.  11.],[ 11.  11.  11.]]`<!--notebook-skip-line-->\n\n## Run on Multiple Machines\nBased on parameter server, the `updater` runs on the server nodes.\nWhen the distributed version is ready, we will update this section.\n\n\n<!-- ## How to Choose Between APIs -->\n\n<!-- You can mix APIs as much as you like. Here are some guidelines -->\n<!-- * Use the Symbolic API and a coarse-grained operator to create  an established structure. -->\n<!-- * Use a fine-grained operator to extend parts of a more flexible symbolic graph. -->\n<!-- * Do some dynamic NDArray tricks, which are even more flexible, between the calls of forward and backward executors. -->\n\n<!-- Different approaches offer you different levels of flexibility and -->\n<!-- efficiency. Normally, you do not need to be flexible in all parts of the -->\n<!-- network, so use the parts optimized for speed, and compose it -->\n<!-- flexibly with a fine-grained operator or a dynamic NDArray. Such a -->\n<!-- mixture allows you to build the deep learning architecture both efficiently and -->\n<!-- flexibly as your choice.  -->\n\n## Next Steps\n* [MXNet tutorials index](../../index.rst)\n\n<!-- INSERT SOURCE DOWNLOAD BUTTONS -->\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/legacy/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nLegacy\n======\n\nThis document hosts documents for legacy modules that are being deprecated in MXNet 2.x.\n\n.. container:: cards\n\n   .. card::\n      :title: NDArray API\n      :link: ndarray/index.html\n\n      MXNet NDArray API has been deprecated. Tutorials for NDArray are kept here for reference.\n\n   .. card::\n      :title: Symbol API\n      :link: ../../../api/symbol/\n\n      MXNet Symbol API has been deprecated. API documentation is still available for reference.\n\n.. toctree::\n   :hidden:\n   :glob:\n\n   */index*\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/legacy/ndarray/01-ndarray-intro.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# An Intro: Manipulate Data the MXNet Way with NDArray\n\n## Overview\nThis guide\nwill introduce you to how data is handled with MXNet. You will learn the basics\nabout MXNet's multi-dimensional array format, `ndarray`.\n\nThis content was extracted and simplified from the gluon tutorials in\n[Dive Into Deep Learning](https://d2l.ai/).\n\n## Prerequisites\n* [MXNet installed in a Python environment](https://mxnet.apache.org/get_started?version=master&platform=linux&language=python&environ=pip&processor=cpu).\n* Python 2.7.x or Python 3.x\n\n\n## Getting started\n\nIn this chapter, we'll get\nyou going with the basic functionality. Don't worry if you don't understand any\nof the basic math, like element-wise operations or normal distributions. In the\nnext two chapters we'll take another pass at `NDArray`, teaching you both the math\nyou'll need and how to realize it in code.\n\nTo get started, let's import\n`mxnet`. We'll also import `ndarray` from `mxnet` for convenience. We’ll make a\nhabit of setting a random seed so that you always get the same results that we\ndo.\n\n```{.python .input}\nimport mxnet as mx\nfrom mxnet import nd\n```\n\nLet's start with a very simple 1-dimensional array with a python list.\n\n```{.python .input}\nx = nd.array([1,2,3])\nprint(x)\n```\n\nNow a 2-dimensional array.\n\n```{.python .input}\ny = nd.array([[1,2,3,4], [1,2,3,4], [1,2,3,4]])\nprint(y)\n```\n\nNext, let's see how to create an `NDArray`, without any values initialized.\nSpecifically, we'll create a 2D array (also called a *matrix*) with 3 rows and 4\ncolumns using the `.empty` function. We'll also try out `.full` which takes an\nadditional parameter for what value you want to fill in the array.\n\n```{.python .input}\nx = nd.empty((3, 3))\nprint(x)\nx = nd.full((3,3), 7)\nprint(x)\n```\n\n`empty` just grabs some memory and hands us back a matrix without setting the\nvalues of any of its entries. This means that the entries can have any form of\nvalues, including very big ones! Typically, we'll want our matrices initialized\nand very often we want a matrix of all zeros, so we can use the `.zeros`\nfunction.\n\n<!-- showing something\ndifferent here (3,10) since the zeros may not produce anything different from\nempty... or use the two demonstrations to show something interesting or\nunique... when would I use one over the other?-->\n\n```{.python .input}\nx = nd.zeros((3, 10))\nprint(x)\n```\n\nSimilarly, `ndarray` has a function to create a matrix of all ones aptly named\n[ones](../../../../api/legacy/ndarray/ndarray.rst#mxnet.ndarray.ones).\n\n```{.python .input}\nx = nd.ones((3, 4))\nprint(x)\n```\n\nOften, we'll want to create arrays whose values are sampled randomly. This is\nespecially common when we intend to use the array as a parameter in a neural\nnetwork. In this snippet, we initialize with values drawn from a standard normal\ndistribution with zero mean and unit variance using\n[random_normal](../../../../api/legacy/ndarray/ndarray.rst#mxnet.ndarray.random_normal).\n\n<!--\nIs it that important to introduce zero mean and unit variance right now?\nDescribe more? Or how about explain which is which for the 0 and the 1 and what\nthey're going to do... if it actually matters at this point. -->\n\n```{.python .input}\ny = nd.random_normal(0, 1, shape=(3, 4))\nprint(y)\n```\n\nSometimes you will want to copy an array by its shape but not its contents. You\ncan do this with `.zeros_like`.\n\n```{.python .input}\nz = nd.zeros_like(y)\nprint(z)\n```\n\nAs in NumPy, the dimensions of each `NDArray` are accessible via the `.shape`\nattribute.\n\n```{.python .input}\ny.shape\n```\n\nWe can also query its `.size`, which is equal to the product of the components\nof the shape. Together with the precision of the stored values, this tells us\nhow much memory the array occupies.\n<!-- is there a function for that or do you\njust do it manually? Should we show that? -->\n\n```{.python .input}\ny.size\n```\n\nWe can query the data type using `.dtype`.\n\n```{.python .input}\ny.dtype\n```\n\n`float32` is the default data type. Performance can be improved with less\nprecision, or you might want to use a different data type. You can force the\ndata type when you create the array using a numpy type. This requires you to\nimport numpy first.\n\n```{.python .input}\nimport numpy as np\na = nd.array([1,2,3])\nb = nd.array([1,2,3], dtype=np.int32)\nc = nd.array([1.2, 2.3], dtype=np.float16)\n(a.dtype, b.dtype, c.dtype)\n```\n\nAs you will come to learn in detail later, operations and memory storage will\nhappen on specific devices that you can set. You can compute on CPU(s), GPU(s), a\nspecific GPU, or all of the above depending on your situation and preference.\nUsing `.context` reveals the location of the variable.\n\n```{.python .input}\ny.context\n```\n\n## Next Up\n\n[NDArray Operations](02-ndarray-operations.md)\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/legacy/ndarray/02-ndarray-operations.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# NDArray Operations\n\n## Overview\nThis guide will introduce you to MXNet's array operations.\n\nThis content was extracted and simplified from the gluon tutorials in\n[Dive Into Deep Learning](https://d2l.ai/).\n\n## Prerequisites\n* [MXNet installed in a Python environment](https://mxnet.apache.org/get_started).\n* Python 2.7.x or Python 3.x\n\n\n## Operations\n\nNDArray supports a large number of standard mathematical operations.\nSuch as element-wise addition:\n<!-- keeping it\neasy -->\n\n```{.python .input}\nimport mxnet as mx\nfrom mxnet import nd\n```\n\n```{.python .input}\nx = nd.ones((3, 4))\ny = nd.random_normal(0, 1, shape=(3, 4))\nprint('x=', x)\nprint('y=', y)\nx = x + y\nprint('x = x + y, x=', x)\n```\n\nMultiplication:\n\n```{.python .input}\nx = nd.array([1, 2, 3])\ny = nd.array([2, 2, 2])\nx * y\n```\n\nAnd exponentiation:\n<!-- with these next ones we'll just have to take your word\nfor it... -->\n\n```{.python .input}\nnd.exp(x)\n```\n\nWe can also grab a matrix's transpose to compute a proper matrix-matrix product.\n<!-- because we need to do that before we have coffee every day... and you know\nhow those dirty, improper matrixeses can be... -->\n\n```{.python .input}\nnd.dot(x, y.T)\n```\n\n\n## In-place operations\n\nIn the previous\nexample, every time we ran an operation, we allocated new memory to host its\nresults. For example, if we write `y = x + y`, we will dereference the matrix\nthat `y` used to point to and instead point it at the newly allocated memory. We\ncan show this using Python's `id()` function, which tells us precisely which\nobject a variable refers to.\n\n<!-- dereference is something C++ people would\nknow but everyone else... not so much. What's the point? ;) get it? Put it in\nmore context as to why you care about this and why this is in front of so much\nother material. Seems like an optimization topic best suited for later...\n###edit### we just talked about this, so I have better context. Now I\nunderstand, but your new reader will not. This should be covered in much more\ndetail, and quite possibily in its own notebook since I think it will help to\nshow some gotchas like you mentioned verbally. I am still leaning toward\ndelaying the introduction of this topic....-->\n\n```{.python .input}\nprint('y=', y)\nprint('id(y):', id(y))\ny = y + x\nprint('after y=y+x, y=', y)\nprint('id(y):', id(y))\n```\n\nWe can assign the result to a previously allocated array with slice notation,\ne.g., `result[:] = ...`.\n\n```{.python .input}\nprint('x=', x)\nz = nd.zeros_like(x)\nprint('z is zeros_like x, z=', z)\nprint('id(z):', id(z))\nprint('y=', y)\nz[:] = x + y\nprint('z[:] = x + y, z=', z)\nprint('id(z) is the same as before:', id(z))\n```\n\nHowever, `x+y` here will still allocate a temporary buffer to store the result\nbefore copying it to z. To make better use of memory, we can perform operations\nin place, avoiding temporary buffers. To do this we specify the `out` keyword\nargument every operator supports:\n\n```{.python .input}\nprint('x=', x, 'is in id(x):', id(x))\nprint('y=', y, 'is in id(y):', id(y))\nprint('z=', z, 'is in id(z):', id(z))\nnd.elemwise_add(x, y, out=z)\nprint('after nd.elemwise_add(x, y, out=z), x=', x, 'is in id(x):', id(x))\nprint('after nd.elemwise_add(x, y, out=z), y=', y, 'is in id(y):', id(y))\nprint('after nd.elemwise_add(x, y, out=z), z=', z, 'is in id(z):', id(z))\n```\n\nIf we're not planning to re-use ``x``, then we can assign the result to ``x``\nitself. There are two ways to do this in MXNet.\n1. By using slice notation x[:]\n= x op y\n2. By using the op-equals operators like `+=`\n\n```{.python .input}\nprint('x=', x, 'is in id(x):', id(x))\nx += y\nprint('x=', x, 'is in id(x):', id(x))\n```\n\n## Slicing\nMXNet NDArrays support slicing in all the ridiculous ways you might\nimagine accessing your data. For a quick review:\n\n* items start through end-1: a[start:end]\n* items start through the rest of the\narray: a[start:]\n* items from the beginning through end-1: a[:end]\n* a copy of\nthe whole array: a[:]\n\nHere's an example of reading the second and third rows from `x`.\n\n```{.python .input}\nx = nd.array([1, 2, 3])\nprint('1D complete array, x=', x)\ns = x[1:3]\nprint('slicing the 2nd and 3rd elements, s=', s)\nx = nd.array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]])\nprint('multi-D complete array, x=', x)\ns = x[1:3]\nprint('slicing the 2nd and 3rd elements, s=', s)\n```\n\nNow let's try writing to a specific element.\n\n```{.python .input}\nprint('original x, x=', x)\nx[2] = 9.0\nprint('replaced entire row with x[2] = 9.0, x=', x)\nx[0,2] = 9.0\nprint('replaced specific element with x[0,2] = 9.0, x=', x)\nx[1:2,1:3] = 5.0\nprint('replaced range of elements with x[1:2,1:3] = 5.0, x=', x)\n```\n\nMulti-dimensional slicing is also supported.\n\n```{.python .input}\nx = nd.array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]])\nprint('original x, x=', x)\ns = x[1:2,1:3]\nprint('plucking specific elements with x[1:2,1:3]', s)\ns = x[:,:1]\nprint('first column with x[:,:1]', s)\ns = x[:1,:]\nprint('first row with x[:1,:]', s)\ns = x[:,3:]\nprint('last column with x[:,3:]', s)\ns = x[2:,:]\nprint('last row with x[2:,:]', s)\n```\n\n## Broadcasting\n\nYou might wonder, what happens if you add a vector `y` to a\nmatrix `X`? These operations, where we compose a low dimensional array `y` with\na high-dimensional array `X` invoke a functionality called broadcasting. First\nwe'll introduce `.arange` which is useful for filling out an array with evenly\nspaced data. Then we can take the low-dimensional array and duplicate it along\nany axis with dimension $1$ to match the shape of the high dimensional array.\nConsider the following example.\n\nComment (visible to demonstrate with font):\ndimension one(1)? Or L(elle) or l(lil elle) or I(eye) or... ? We don't even use\nthe notation later, so did it need to be introduced here?\n\n<!--Also, if you use\na shape like (3,3) you lose some of the impact and miss some errors if people\nplay with the values. Better to have a distinct shape so that it is more obvious\nwhat is happening and what can break.-->\n\n```{.python .input}\nx = nd.ones(shape=(3,6))\nprint('x = ', x)\ny = nd.arange(6)\nprint('y = ', y)\nprint('x + y = ', x + y)\n```\n\nWhile `y` is initially of shape $6$,\nMXNet infers its shape to be (1,6),\nand then broadcasts along the rows to form a (3,6) matrix).\nYou might wonder, why did MXNet choose to interpret `y` as a (1,6) matrix and not (6,1).\nThat's because broadcasting prefers to duplicate along the left most axis.\nWe can alter this behavior by explicitly giving `y` a $2$D shape using `.reshape`.\nYou can also chain `.arange` and `.reshape` to do this in one step.\n\n```{.python .input}\ny = y.reshape((3,1))\nprint('y = ', y)\nprint('x + y = ', x+y)\ny = nd.arange(6).reshape((3,1))\nprint('y = ', y)\n```\n\n## Converting from MXNet NDArray to NumPy\nConverting MXNet NDArrays to and from\nNumPy is easy. The converted arrays do not share memory.\n\n```{.python .input}\na = x.asnumpy()\ntype(a)\n```\n\n```{.python .input}\ny = nd.array(a)\nprint('id(a)=', id(a), 'id(x)=', id(x), 'id(y)=', id(y))\n```\n\n## Next Up\n\n[NDArray Contexts](03-ndarray-contexts.md)\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/legacy/ndarray/03-ndarray-contexts.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# NDArray Contexts\n\n## Overview\nThis guide will introduce you to managing CPU versus GPU contexts for handling data.\n\nThis content was extracted and simplified from the gluon tutorials in\n[Dive Into Deep Learning](https://d2l.ai/).\n\n## Prerequisites\n* [MXNet installed (with GPU support) in a Python environment](https://mxnet.apache.org/get_started).\n* Python 2.7.x or Python 3.x\n* **One or more GPUs**\n\n\n## Managing Context\n\nIn MXNet, every array has a context.\nOne context could be the CPU. Other contexts might be various GPUs.\nThings can get even hairier when we deploy jobs across multiple servers.\nBy assigning arrays to contexts intelligently, we can minimize\nthe time spent transferring data between devices.\nFor example, when training neural networks on a server with a GPU,\nwe typically prefer for the model's parameters to live on the GPU.\nIf you have a GPU, let's try initializing an array on the first GPU.\nOtherwise, use `ctx=mx.cpu()` in place of `ctx=gpu(0)`.\n\n```{.python .input}\nfrom mxnet import gpu\nfrom mxnet import nd\nz = nd.ones(shape=(3,3), ctx=gpu(0))\nprint(z)\n```\n\nGiven an NDArray on a given context, we can copy it to another context by using\nthe copyto() method. Skip this if you don't have a GPU at the moment.\n\n```{.python .input}\nx_gpu = x.copyto(gpu(0))\nprint(x_gpu)\n```\n\nThe result of an operator will have the same context as the inputs.\n\n```{.python .input}\nx_gpu + z\n```\n\n## Watch out!\n\nImagine that your variable z already lives on your second GPU\n(`gpu(0)`). What happens if we call `z.copyto(gpu(0))`? It will make a copy and\nallocate new memory, even though that variable already lives on the desired\ndevice!\n<!-- wouldn't the second GPU be gpu(1)? -->\n\nOften, we only want to make\na copy if the variable currently lives in the wrong context. In these cases, we\ncan call `as_in_context()`. If the variable is already on `gpu(0)` then this is\na no-op.\n\n```{.python .input}\nprint('id(z):', id(z))\nz = z.copyto(gpu(0))\nprint('id(z):', id(z))\nz = z.as_in_context(gpu(0))\nprint('id(z):', id(z))\nprint(z)\n```\n\n## Next Up\n\n[Back to NDArray API Guides](../../../../api/legacy/ndarray/index.rst)\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/legacy/ndarray/gotchas_numpy_in_mxnet.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Gotchas using NumPy in Apache MXNet\n\nThe goal of this tutorial is to explain some common misconceptions about using [NumPy](http://www.numpy.org/) arrays in Apache MXNet. We are going to explain why you need to minimize or completely remove usage of NumPy from your Apache MXNet code. We also going to show how to minimize NumPy performance impact, when you have to use NumPy.\n\nWarning: The latest MXNet offers NumPy-compatible array class `mx.np.ndarray` and NDArray is now a legacy array class in MXNet 1.x. This tutorial is just for reference for the legacy NDArray.\n\n## Asynchronous and non-blocking nature of Apache MXNet\n\nInstead of using NumPy arrays Apache MXNet offers its own array implementation named [NDArray](../../../../api/legacy/ndarray/ndarray.rst). `NDArray API` was intentionally designed to be similar to `NumPy`, but there are differences.\n\nOne key difference is in the way calculations are executed. Every `NDArray` manipulation in Apache MXNet is done in asynchronous, non-blocking way. That means, that when we write code like `c = a * b`, where both `a` and `b` are `NDArrays`, the function is pushed to the [Execution Engine](https://mxnet.apache.org/api/architecture/overview.html#execution-engine), which starts the calculation. The function immediately returns back, and the  user thread can continue execution, despite the fact that the calculation may not have been completed yet.\n\n`Execution Engine` builds the computation graph which may reorder or combine some calculations, but it honors dependency order: if there are other manipulation with `c` done later in the code, the `Execution Engine` will start doing them once the result of `c` is available. We don't need to write callbacks to start execution of subsequent code - the `Execution Engine` is going to do it for us.\n\nTo get the result of the computation we only need to access the resulting variable, and the flow of the code will be blocked until the computation results are assigned to the resulting variable. This behavior allows to increase code performance while still supporting imperative programming mode.\n\nRefer to the [intro tutorial to NDArray](./index.ipynb), if you are new to Apache MXNet and would like to learn more how to manipulate NDArrays.\n\n## Converting NDArray to NumPy Array blocks calculation\n\nMany people are familiar with NumPy and flexible doing tensor manipulations using it. `NDArray API` offers  a convinient [.asnumpy() method](../../../../api/legacy/ndarray/ndarray.rst#mxnet.ndarray.NDArray.asnumpy) to cast `nd.array` to `np.array`. However, by doing this cast and using `np.array` for calculation, we cannot use all the goodness of `Execution Engine`. All manipulations done on `np.array` are blocking. Moreover, the cast to `np.array` itself is a blocking operation (same as [.asscalar()](../../../../api/legacy/ndarray/ndarray.rst#mxnet.ndarray.NDArray.asscalar), [.wait_to_read()](../../../../api/legacy/ndarray/ndarray.rst#mxnet.ndarray.NDArray.wait_to_read) and [.waitall()](../../../../api/legacy/ndarray/ndarray.rst#mxnet.ndarray.waitall)).\n\nThat means that if we have a long computation graph and, at some point, we want to cast the result to `np.array`, it may feel like the casting takes a lot of time. But what really takes this time is `Execution Engine`, which finishes all the async calculations we have pushed into it to get the final result, which then will be converted to `np.array`.\n\nBecause of the blocking nature of [.asnumpy() method](../../../../api/legacy/ndarray/ndarray.rst#mxnet.ndarray.NDArray.asnumpy), using it reduces the execution performance, especially if the calculations are done on GPU: Apache MXNet has to copy data from GPU to CPU to return `np.array`.\n\nThe best solution is to **make manipulations directly on NDArrays by methods provided in [NDArray API](../../../../api/legacy/ndarray/ndarray.rst)**.\n\n## NumPy operators vs. NDArray operators\n\nDespite the fact that [NDArray API](../../../../api/legacy/ndarray/ndarray.rst) was specifically designed to be similar to `NumPy`, sometimes it is not easy to replace existing `NumPy` computations. The main reason is that not all operators, that are available in `NumPy`, are available in `NDArray API`. The list of currently available operators is available on [NDArray class page](../../../../api/legacy/ndarray/ndarray.rst).\n\nIf a required operator is missing from `NDArray API`, there are few things you can do.\n\n### Combine a higher level operator using a few lower level operators\n\nThere are a situation, when you can assemble a higher level operator using existing operators. An example for that is the [np.full_like()](https://docs.scipy.org/doc/numpy-1.14.0/reference/generated/numpy.full_like.html) operator. This operator doesn't exist in `NDArray API`, but can be easily replaced with a combination of existing operators.\n\n\n```{.python .input}\nfrom mxnet import nd\nimport numpy as np\n\n# NumPy has full_like() operator\nnp_y = np.full_like(a=np.arange(6, dtype=int), fill_value=10)\n\n# NDArray doesn't have it, but we can replace it with\n# creating an array of ones and then multiplying by fill_value\nnd_y = nd.ones(shape=(6,)) * 10\n\n# To compare results we had to convert NDArray to NumPy\n# But this is okay for that particular case\nnp.array_equal(np_y, nd_y.asnumpy())\n```\n\n```True``` <!--notebook-skip-line-->\n\n### Find similar operator with different name and/or signature\n\nSome operators may have slightly different name, but are similar in terms of functionality. For example [nd.ravel_multi_index()](../../../../api/legacy/ndarray/ndarray.rst#mxnet.ndarray.ravel_multi_index) is similar to [np.ravel()](https://docs.scipy.org/doc/numpy-1.14.0/reference/generated/numpy.ma.ravel.html#numpy.ma.ravel). In other cases some operators may have similar names, but different signatures. For example [np.split()](https://docs.scipy.org/doc/numpy-1.14.0/reference/generated/numpy.split.html#numpy.split) and [nd.split()](../../../../api/legacy/ndarray/ndarray.rst#mxnet.ndarray.split) are similar, but the former works with indices and the latter requires the number of splits to be provided.\n\nOne particular example of different input requirements is [nd.pad()](../../../../api/legacy/ndarray/ndarray.rst#mxnet.ndarray.pad). The trick is that it can only work with 4-dimensional tensors. If your input has less dimensions, then you need to expand its number before using `nd.pad()` as it is shown in the code block below:\n\n\n```{.python .input}\ndef pad_array(data, max_length):\n    # expand dimensions to 4, because nd.pad can work only with 4 dims\n    data_expanded = data.reshape(1, 1, 1, data.shape[0])\n\n    # pad all 4 dimensions with constant value of 0\n    data_padded = nd.pad(data_expanded,\n                             mode='constant',\n                             pad_width=[0, 0, 0, 0, 0, 0, 0, max_length - data.shape[0]],\n                             constant_value=0)\n\n    # remove temporary dimensions\n    data_reshaped_back = data_padded.reshape(max_length)\n    return data_reshaped_back\n\npad_array(nd.array([1, 2, 3]), max_length=10)\n```\n\n`[ 1.  2.  3.  0.  0.  0.  0.  0.  0.  0.]` <!--notebook-skip-line-->\n\n\n`<NDArray 10 @cpu(0)>` <!--notebook-skip-line-->\n\n\n### Search for an operator on [Github](https://github.com/apache/mxnet/labels/Operator)\n\nApache MXNet community is responsive to requests, and everyone is welcomed to contribute new operators. Have in mind, that there is always a lag between new operators being merged into the codebase and release of a next stable version. For example, [nd.diag()](https://github.com/apache/mxnet/pull/11643) operator was recently introduced to Apache MXNet, but on the moment of writing this tutorial, it is not in any stable release. You can always get all latest implementations by installing the [master version](https://mxnet.apache.org/get_started?version=master&platform=linux&language=python&environ=pip&processor=cpu#) of Apache MXNet.\n\n## How to minimize the impact of blocking calls\n\nThere are cases, when you have to use either `.asnumpy()` or `.asscalar()` methods. As it is explained before, this will force Apache MXNet to block the execution until the result can be retrieved. One common use case is printing a metric or a value of a loss function.\n\nYou can minimize the impact of a blocking call by calling `.asnumpy()` or `.asscalar()` in the moment, when you think the calculation of this value is already done. In the example below, we introduce the `LossBuffer` class. It is used to cache the previous value of a loss function. By doing so, we delay printing by one iteration in hope that the `Execution Engine` would finish the previous iteration and blocking time would be minimized.\n\n\n```{.python .input}\nfrom __future__ import print_function\n\nimport mxnet as mx\nfrom mxnet import gluon, nd, autograd\nfrom mxnet.ndarray import NDArray\nfrom mxnet.gluon import HybridBlock\nimport numpy as np\n\nclass LossBuffer(object):\n    \"\"\"\n    Simple buffer for storing loss value\n    \"\"\"\n    def __init__(self):\n        self._loss = None\n\n    def new_loss(self, loss):\n        ret = self._loss\n        self._loss = loss\n        return ret\n\n    @property\n    def loss(self):\n        return self._loss\n\n\nnet = gluon.nn.Dense(10)\nce = gluon.loss.SoftmaxCELoss()\nnet.initialize()\n\ndata = nd.random.uniform(shape=(1024, 100))\nlabel = nd.array(np.random.randint(0, 10, (1024,)), dtype='int32')\ntrain_dataset = gluon.data.ArrayDataset(data, label)\ntrain_data = gluon.data.DataLoader(train_dataset, batch_size=128, shuffle=True, num_workers=2)\n\ntrainer = gluon.Trainer(net.collect_params(), optimizer='sgd')\nloss_buffer = LossBuffer()\n\nfor data, label in train_data:\n    with autograd.record():\n        out = net(data)\n        # This call saves new loss and returns previous loss\n        prev_loss = loss_buffer.new_loss(ce(out, label))\n\n    loss_buffer.loss.backward()\n    trainer.step(data.shape[0])\n\n    if prev_loss is not None:\n        print(\"Loss: {}\".format(np.mean(prev_loss.asnumpy())))\n```\n\n```text\n    Loss: 2.310760974884033 <!--notebook-skip-line-->\n\n    Loss: 2.334498643875122 <!--notebook-skip-line-->\n\n    Loss: 2.3244147300720215 <!--notebook-skip-line-->\n\n    Loss: 2.332686424255371 <!--notebook-skip-line-->\n\n    Loss: 2.321366310119629 <!--notebook-skip-line-->\n\n    Loss: 2.3236165046691895 <!--notebook-skip-line-->\n\n    Loss: 2.3178648948669434 <!--notebook-skip-line-->\n```\n\n## Conclusion\n\nFor performance reasons, it is better to use native `NDArray API` methods and avoid using NumPy altogether. In case when you must use NumPy, you can use convenient method `.asnumpy()` on `NDArray` to get NumPy representation. By doing so, you block the whole computational process, and force data to be synced between CPU and GPU. If it is a necessary evil to do that, try to minimize the blocking time by calling `.asnumpy()` in time, when you expect the value to be already computed.\n\n<!-- INSERT SOURCE DOWNLOAD BUTTONS -->\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/legacy/ndarray/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nNDArray\n========\n\n.. container:: cards\n\n\n   .. card::\n      :title: Introduction to NDArray - Part 1\n      :link: 01-ndarray-intro.html\n\n      Learn how to manipulate data with MXNet's multi-dimensional data format, NDArray.\n\n   .. card::\n      :title: Introduction to NDArray - Part 2: Operations\n      :link: 02-ndarray-operations.html\n\n      Learn basic array operations like math and slicing.\n\n   .. card::\n      :title: Introduction to NDArray - Part 3: Contexts\n      :link: 03-ndarray-contexts.html\n\n      This guide will introduce you to how CPU and GPU contexts are handled with MXNet.\n\n   .. card::\n      :title: Sparse NDArray\n      :link: sparse/index.html\n\n      For Sparse NDArray tutorials\n\n      This section contains the mxnet.np API reference documentation\n\n   .. card::\n      :title: Gotchas using NumPy in Apache MXNet\n      :link: gotchas_numpy_in_mxnet.md\n\n      Common misconceptions when using NumPy in Apache MXNet.\n\n.. toctree::\n   :hidden:\n   :glob:\n\n   *\n   sparse/index\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/legacy/ndarray/sparse/csr.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# CSRNDArray - NDArray in Compressed Sparse Row Storage Format\n\nMany real world datasets deal with high dimensional sparse feature vectors. Take for instance a recommendation system where the number of categories and users is on the order of millions. The purchase data for each category by user would show that most users only make a few purchases, leading to a dataset with high sparsity (i.e. most of the elements are zeros).\n\nStoring and manipulating such large sparse matrices in the default dense structure results in wasted memory and processing on the zeros. To take advantage of the sparse structure of the matrix, the `CSRNDArray` in MXNet stores the matrix in [compressed sparse row (CSR)](https://en.wikipedia.org/wiki/Sparse_matrix#Compressed_sparse_row_.28CSR.2C_CRS_or_Yale_format.29) format and uses specialized algorithms in operators.\n**The format is designed for 2D matrices with a large number of columns,\nand each row is sparse (i.e. with only a few nonzeros).**\n\n## Advantages of Compressed Sparse Row NDArray (CSRNDArray)\nFor matrices of high sparsity (e.g. ~1% non-zeros = ~1% density), there are two primary advantages of `CSRNDArray` over the existing `NDArray`:\n\n- memory consumption is reduced significantly\n- certain operations are much faster (e.g. matrix-vector multiplication)\n\nYou may be familiar with the CSR storage format in [SciPy](https://www.scipy.org/) and will note the similarities in MXNet's implementation. However there are some additional competitive features in `CSRNDArray` inherited from `NDArray`, such as non-blocking asynchronous evaluation and automatic parallelization that are not available in SciPy's flavor of CSR. You can find further explanations for evaluation and parallelization strategy in MXNet in the [NDArray tutorial](../gotchas_numpy_in_mxnet.ipynb).\n\nThe introduction of `CSRNDArray` also brings a new attribute, `stype` as a holder for storage type info, to `NDArray`. You can query **ndarray.stype** now in addition to the oft-queried attributes such as **ndarray.shape**, **ndarray.dtype**, and **ndarray.context**. For a typical dense NDArray, the value of `stype` is **\"default\"**. For a `CSRNDArray`, the value of stype is **\"csr\"**.\n\n## Prerequisites\n\nTo complete this tutorial, you will need:\n\n- MXNet. See the instructions for your operating system in [Setup and Installation](https://mxnet.apache.org/get_started)\n- [Jupyter](http://jupyter.org/)\n    ```\n    pip install jupyter\n    ```\n- Basic knowledge of NDArray in MXNet. See the detailed tutorial for NDArray in [NDArray - Imperative tensor operations on CPU/GPU](../01-ndarray-intro.rst).\n- SciPy - A section of this tutorial uses SciPy package in Python. If you don't have SciPy, the example in that section will be ignored.\n- GPUs - A section of this tutorial uses GPUs. If you don't have GPUs on your machine, simply set the variable `gpu_device` (set in the GPUs section of this tutorial) to `mx.cpu()`.\n\n## Compressed Sparse Row Matrix\n\nA CSRNDArray represents a 2D matrix as three separate 1D arrays: **data**, **indptr** and **indices**, where the column indices for row `i` are stored in `indices[indptr[i]:indptr[i+1]]` in ascending order, and their corresponding values are stored in `data[indptr[i]:indptr[i+1]]`.\n\n- **data**: CSR format data array of the matrix\n- **indices**: CSR format index array of the matrix\n- **indptr**: CSR format index pointer array of the matrix\n\n### Example Matrix Compression\n\nFor example, given the matrix:\n```\n[[7, 0, 8, 0]\n [0, 0, 0, 0]\n [0, 9, 0, 0]]\n```\n\nWe can compress this matrix using CSR, and to do so we need to calculate `data`, `indices`, and `indptr`.\n\nThe `data` array holds all the non-zero entries of the matrix in row-major order. Put another way, you create a data array that has all of the zeros removed from the matrix, row by row, storing the numbers in that order. Your result:\n\n```\ndata = [7, 8, 9]\n```\n\nThe `indices` array stores the column index for each non-zero element in `data`. As you cycle through the data array, starting with 7, you can see it is in column 0. Then looking at 8, you can see it is in column 2. Lastly 9 is in column 1. Your result:\n\n```\nindices = [0, 2, 1]\n```\n\nThe `indptr` array is what will help identify the rows where the data appears. It stores the offset into `data` of the first non-zero element number of each row of the matrix. This array always starts with 0 (reasons can be explored later), so indptr[0] is 0. Each subsequent value in the array is the aggregate number of non-zero elements up to that row. Looking at the first row of the matrix you can see two non-zero values, so indptr[1] is 2. The next row contains all zeros, so the aggregate is still 2, so indptr[2] is 2. Finally, you see the last row contains one non-zero element bring the aggregate to 3, so indptr[3] is 3. To reconstruct the dense matrix, you will use `data[0:2]` and `indices[0:2]` for the first row, `data[2:2]` and `indices[2:2]` for the second row (which contains all zeros), and `data[2:3]` and `indices[2:3]` for the third row. Your result:\n\n```text\nindptr = [0, 2, 2, 3]\n```\n\nNote that in MXNet, the column indices for a given row are always sorted in ascending order,\nand duplicated column indices for the same row are not allowed.\n\n## Array Creation\n\nThere are a few different ways to create a `CSRNDArray`, but first let's recreate the matrix we just discussed using the `data`, `indices`, and `indptr` we calculated in the previous example.\n\nYou can create a CSRNDArray with data, indices and indptr by using the `csr_matrix` function:\n\n\n```{.python .input}\nimport mxnet as mx\n# Create a CSRNDArray with python lists\nshape = (3, 4)\ndata_list = [7, 8, 9]\nindices_list = [0, 2, 1]\nindptr_list = [0, 2, 2, 3]\na = mx.nd.sparse.csr_matrix((data_list, indices_list, indptr_list), shape=shape)\n# Inspect the matrix\na.asnumpy()\n```\n\n\n\n```\narray([[ 7.,  0.,  8.,  0.],\n       [ 0.,  0.,  0.,  0.],\n       [ 0.,  9.,  0.,  0.]], dtype=float32)\n```\n\n\n\n```{.python .input}\nimport numpy as np\n# Create a CSRNDArray with numpy arrays\ndata_np = np.array([7, 8, 9])\nindptr_np = np.array([0, 2, 2, 3])\nindices_np = np.array([0, 2, 1])\nb = mx.nd.sparse.csr_matrix((data_np, indices_np, indptr_np), shape=shape)\nb.asnumpy()\n```\n\n\n\n```\narray([[7, 0, 8, 0],\n       [0, 0, 0, 0],\n       [0, 9, 0, 0]])\n```\n\n\n\n```{.python .input}\n# Compare the two. They are exactly the same.\n{'a':a.asnumpy(), 'b':b.asnumpy()}\n```\n\n\n\n```\n{'a': array([[ 7.,  0.,  8.,  0.],\n        [ 0.,  0.,  0.,  0.],\n        [ 0.,  9.,  0.,  0.]], dtype=float32), 'b': array([[7, 0, 8, 0],\n        [0, 0, 0, 0],\n        [0, 9, 0, 0]])}\n```\n\n\nYou can create an MXNet CSRNDArray from a `scipy.sparse.csr.csr_matrix` object by using the `array` function:\n\n\n```{.python .input}\ntry:\n    import scipy.sparse as spsp\n    # generate a csr matrix in scipy\n    c = spsp.csr.csr_matrix((data_np, indices_np, indptr_np), shape=shape)\n    # create a CSRNDArray from a scipy csr object\n    d = mx.nd.sparse.array(c)\n    print('d:{}'.format(d.asnumpy()))\nexcept ImportError:\n    print(\"scipy package is required\")\n```\n\n```\nd:[[7 0 8 0]\n [0 0 0 0]\n [0 9 0 0]]\n```\n\n\nWhat if you have a big set of data and you haven't calculated indices or indptr yet? Let's try a simple CSRNDArray from an existing array of data and derive those values with some built-in functions. We can mockup a \"big\" dataset with a random amount of the data being non-zero, then compress it by using the `tostype` function, which is explained further in the [Storage Type Conversion](#storage-type-conversion) section:\n\n\n```{.python .input}\nbig_array = mx.nd.round(mx.nd.random.uniform(low=0, high=1, shape=(1000, 100)))\nprint(big_array)\nbig_array_csr = big_array.tostype('csr')\n# Access indices array\nindices = big_array_csr.indices\n# Access indptr array\nindptr = big_array_csr.indptr\n# Access data array\ndata = big_array_csr.data\n# The total size of `data`, `indices` and `indptr` arrays is much lesser than the dense big_array!\n```\n\n```\n[[ 1.  1.  0. ...,  0.  1.  1.]\n [ 0.  0.  0. ...,  0.  0.  1.]\n [ 1.  0.  0. ...,  1.  0.  0.]\n ..., \n [ 0.  1.  1. ...,  0.  0.  0.]\n [ 1.  1.  0. ...,  1.  0.  1.]\n [ 1.  0.  1. ...,  1.  0.  0.]]\n<NDArray 1000x100 @cpu(0)>\n```\n\nYou can also create a CSRNDArray from another using the `array` function specifying the element data type with the option `dtype`,\nwhich accepts a numpy type. By default, `float32` is used.\n\n\n```{.python .input}\n# Float32 is used by default\ne = mx.nd.sparse.array(a)\n# Create a 16-bit float array\nf = mx.nd.array(a, dtype=np.float16)\n(e.dtype, f.dtype)\n```\n\n\n\n```\n(numpy.float32, numpy.float16)\n```\n\n\n\n## Inspecting Arrays\n\nA variety of methods are available for you to use for inspecting CSR arrays:\n* **.asnumpy()**\n* **.data**\n* **.indices**\n* **.indptr**\n\nAs you have seen already, we can inspect the contents of a `CSRNDArray` by filling\nits contents into a dense `numpy.ndarray` using the `asnumpy` function.\n\n\n```{.python .input}\na.asnumpy()\n```\n\n\n\n```\narray([[ 7.,  0.,  8.,  0.],\n       [ 0.,  0.,  0.,  0.],\n       [ 0.,  9.,  0.,  0.]], dtype=float32)\n```\n\n\nYou can also inspect the internal storage of a CSRNDArray by accessing attributes such as `indptr`, `indices` and `data`:\n\n\n```{.python .input}\n# Access data array\ndata = a.data\n# Access indices array\nindices = a.indices\n# Access indptr array\nindptr = a.indptr\n{'a.stype': a.stype, 'data':data, 'indices':indices, 'indptr':indptr}\n```\n\n\n\n```\n{'a.stype': 'csr', 'data': \n [ 7.  8.  9.]\n <NDArray 3 @cpu(0)>, 'indices': \n [0 2 1]\n <NDArray 3 @cpu(0)>, 'indptr': \n [0 2 2 3]\n <NDArray 4 @cpu(0)>}\n```\n\n\n## Storage Type Conversion\n\nYou can also convert storage types with:\n* **tostype**\n* **cast_storage**\n\nTo convert an NDArray to a CSRNDArray and vice versa by using the ``tostype`` function:\n\n\n```{.python .input}\n# Create a dense NDArray\nones = mx.nd.ones((2,2))\n# Cast the storage type from `default` to `csr`\ncsr = ones.tostype('csr')\n# Cast the storage type from `csr` to `default`\ndense = csr.tostype('default')\n{'csr':csr, 'dense':dense}\n```\n\n\n\n```\n{'csr': \n <CSRNDArray 2x2 @cpu(0)>, 'dense': \n [[ 1.  1.]\n  [ 1.  1.]]\n <NDArray 2x2 @cpu(0)>}\n```\n\n\nTo convert the storage type by using the `cast_storage` operator:\n\n\n```{.python .input}\n# Create a dense NDArray\nones = mx.nd.ones((2,2))\n# Cast the storage type to `csr`\ncsr = mx.nd.sparse.cast_storage(ones, 'csr')\n# Cast the storage type to `default`\ndense = mx.nd.sparse.cast_storage(csr, 'default')\n{'csr':csr, 'dense':dense}\n```\n\n\n\n```\n{'csr': \n <CSRNDArray 2x2 @cpu(0)>, 'dense': \n [[ 1.  1.]\n  [ 1.  1.]]\n <NDArray 2x2 @cpu(0)>}\n```\n\n\n## Copies\n\nYou can use the `copy` method which makes a deep copy of the array and its data, and returns a new array.\nYou can also use the `copyto` method or the slice operator `[]` to deep copy to an existing array.\n\n\n```{.python .input}\na = mx.nd.ones((2,2)).tostype('csr')\nb = a.copy()\nc = mx.nd.sparse.zeros('csr', (2,2))\nc[:] = a\nd = mx.nd.sparse.zeros('csr', (2,2))\na.copyto(d)\n{'b is a': b is a, 'b.asnumpy()':b.asnumpy(), 'c.asnumpy()':c.asnumpy(), 'd.asnumpy()':d.asnumpy()}\n```\n\n\n\n```\n{'b is a': False, 'b.asnumpy()': array([[ 1.,  1.],\n        [ 1.,  1.]], dtype=float32), 'c.asnumpy()': array([[ 1.,  1.],\n        [ 1.,  1.]], dtype=float32), 'd.asnumpy()': array([[ 1.,  1.],\n        [ 1.,  1.]], dtype=float32)}\n```\n\n\nIf the storage types of source array and destination array do not match,\nthe storage type of destination array will not change when copying with `copyto` or\nthe slice operator `[]`.\n\n\n```{.python .input}\ne = mx.nd.sparse.zeros('csr', (2,2))\nf = mx.nd.sparse.zeros('csr', (2,2))\ng = mx.nd.ones(e.shape)\ne[:] = g\ng.copyto(f)\n{'e.stype':e.stype, 'f.stype':f.stype, 'g.stype':g.stype}\n```\n\n\n\n```\n{'e.stype': 'csr', 'f.stype': 'csr', 'g.stype': 'default'}\n```\n\n\n## Indexing and Slicing\nYou can slice a CSRNDArray on axis 0 with operator `[]`, which copies the slices and returns a new CSRNDArray.\n\n\n```{.python .input}\na = mx.nd.array(np.arange(6).reshape(3,2)).tostype('csr')\nb = a[1:2].asnumpy()\nc = a[:].asnumpy()\n{'a':a, 'b':b, 'c':c}\n```\n\n\n\n```\n{'a': \n <CSRNDArray 3x2 @cpu(0)>,\n 'b': array([[ 2.,  3.]], dtype=float32),\n 'c': array([[ 0.,  1.],\n        [ 2.,  3.],\n        [ 4.,  5.]], dtype=float32)}\n```\n\n\nNote that multi-dimensional indexing or slicing along a particular axis is currently not supported for a CSRNDArray.\n\n## Sparse Operators and Storage Type Inference\n\nOperators that have specialized implementation for sparse arrays can be accessed in `mx.nd.sparse`. You can read the [mxnet.ndarray.sparse API documentation](../../../../../api/legacy/ndarray/sparse/index.rst) to find what sparse operators are available.\n\n\n```{.python .input}\nshape = (3, 4)\ndata = [7, 8, 9]\nindptr = [0, 2, 2, 3]\nindices = [0, 2, 1]\na = mx.nd.sparse.csr_matrix((data, indices, indptr), shape=shape) # a csr matrix as lhs\nrhs = mx.nd.ones((4, 1))      # a dense vector as rhs\nout = mx.nd.sparse.dot(a, rhs)  # invoke sparse dot operator specialized for dot(csr, dense)\n{'out':out}\n```\n\n\n\n```\n{'out': \n [[ 15.]\n  [  0.]\n  [  9.]]\n <NDArray 3x1 @cpu(0)>}\n```\n\n\nFor any sparse operator, the storage type of output array is inferred based on inputs. You can either read the documentation or inspect the `stype` attribute of the output array to know what storage type is inferred:\n\n\n```{.python .input}\nb = a * 2  # b will be a CSRNDArray since zero multiplied by 2 is still zero\nc = a + mx.nd.ones(shape=(3, 4))  # c will be a dense NDArray\n{'b.stype':b.stype, 'c.stype':c.stype}\n```\n\n\n\n```\n{'b.stype': 'csr', 'c.stype': 'default'}\n```\n\n\nFor operators that don't specialize in sparse arrays, we can still use them with sparse inputs with some performance penalty. In MXNet, dense operators require all inputs and outputs to be in the dense format.\n\nIf sparse inputs are provided, MXNet will convert sparse inputs into dense ones temporarily, so that the dense operator can be used.\n\nIf sparse outputs are provided, MXNet will convert the dense outputs generated by the dense operator into the provided sparse format.\n\n\n```{.python .input}\ne = mx.nd.sparse.zeros('csr', a.shape)\nd = mx.nd.log(a) # dense operator with a sparse input\ne = mx.nd.log(a, out=e) # dense operator with a sparse output\n{'a.stype':a.stype, 'd.stype':d.stype, 'e.stype':e.stype} # stypes of a and e will be not changed\n```\n\n\n\n```\n{'a.stype': 'csr', 'd.stype': 'default', 'e.stype': 'csr'}\n```\n\n\nNote that warning messages will be printed when such a storage fallback event happens. If you are using jupyter notebook, the warning message will be printed in your terminal console.\n\n## Data Loading\n\nYou can load data in batches from a CSRNDArray using `mx.io.NDArrayIter`:\n\n\n```{.python .input}\n# Create the source CSRNDArray\ndata = mx.nd.array(np.arange(36).reshape((9,4))).tostype('csr')\nlabels = np.ones([9, 1])\nbatch_size = 3\ndataiter = mx.io.NDArrayIter(data, labels, batch_size, last_batch_handle='discard')\n# Inspect the data batches\n[batch.data[0] for batch in dataiter]\n```\n\n\n\n```\n[\n <CSRNDArray 3x4 @cpu(0)>, \n <CSRNDArray 3x4 @cpu(0)>, \n <CSRNDArray 3x4 @cpu(0)>]\n```\n\n\nYou can also load data stored in the [libsvm file format](https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/) using `mx.io.LibSVMIter`, where the format is: ``<label> <col_idx1>:<value1> <col_idx2>:<value2> ... <col_idxN>:<valueN>``. Each line in the file records the label and the column indices and data for non-zero entries. For example, for a matrix with 6 columns, ``1 2:1.5 4:-3.5`` means the label is ``1``, the data is ``[[0, 0, 1,5, 0, -3.5, 0]]``. More detailed examples of `mx.io.LibSVMIter` are available in the [API documentation](../../../../../api/legacy/io/index.rst#mxnet.io.LibSVMIter).\n\n\n```{.python .input}\n# Create a sample libsvm file in current working directory\nimport os\ncwd = os.getcwd()\ndata_path = os.path.join(cwd, 'data.t')\nwith open(data_path, 'w') as fout:\n    fout.write('1.0 0:1 2:2\\n')\n    fout.write('1.0 0:3 5:4\\n')\n    fout.write('1.0 2:5 8:6 9:7\\n')\n    fout.write('1.0 3:8\\n')\n    fout.write('-1 0:0.5 9:1.5\\n')\n    fout.write('-2.0\\n')\n    fout.write('-3.0 0:-0.6 1:2.25 2:1.25\\n')\n    fout.write('-3.0 1:2 2:-1.25\\n')\n    fout.write('4 2:-1.2\\n')\n\n# Load CSRNDArrays from the file\ndata_train = mx.io.LibSVMIter(data_libsvm=data_path, data_shape=(10,), label_shape=(1,), batch_size=3)\nfor batch in data_train:\n    print(data_train.getdata())\n    print(data_train.getlabel())\n```\n\n```\n<CSRNDArray 3x10 @cpu(0)>\n\n[ 1.  1.  1.]\n<NDArray 3 @cpu(0)>\n\n<CSRNDArray 3x10 @cpu(0)>\n\n[ 1. -1. -2.]\n<NDArray 3 @cpu(0)>\n\n<CSRNDArray 3x10 @cpu(0)>\n\n[-3. -3.  4.]\n<NDArray 3 @cpu(0)>\n```\n\n\nNote that in the file the column indices are expected to be sorted in ascending order per row, and be zero-based instead of one-based.\n\n## Advanced Topics\n\n### GPU Support\n\nBy default, `CSRNDArray` operators are executed on CPU. To create a `CSRNDArray` on a GPU, we need to explicitly specify the context:\n\n**Note** If a GPU is not available, an error will be reported in the following section. In order to execute it a cpu, set `gpu_device` to `mx.cpu()`.\n\n\n```{.python .input}\nimport sys\ngpu_device=mx.gpu() # Change this to mx.cpu() in absence of GPUs.\ntry:\n    a = mx.nd.sparse.zeros('csr', (100, 100), ctx=gpu_device)\n    a\nexcept mx.MXNetError as err:\n    sys.stderr.write(str(err))\n```\n\n\n\n<!-- INSERT SOURCE DOWNLOAD BUTTONS -->\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/legacy/ndarray/sparse/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nTutorials\n=========\n\n.. toctree::\n   :glob:\n\n   *"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/legacy/ndarray/sparse/row_sparse.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# RowSparseNDArray - NDArray for Sparse Gradient Updates\n\n## Motivation\n\nMany real world datasets deal with high dimensional sparse feature vectors. When learning\nthe weights of models with sparse datasets, the derived gradients of the weights could be sparse.\n\nLet's say we perform a matrix multiplication of ``X``  and ``W``, where ``X`` is a 1x2 matrix, and ``W`` is a 2x3 matrix. Let ``Y`` be the matrix multiplication of the two matrices:\n\n\n```{.python .input}\nimport mxnet as mx\nX = mx.nd.array([[1,0]])\nW = mx.nd.array([[3,4,5], [6,7,8]])\nY = mx.nd.dot(X, W)\n{'X': X, 'W': W, 'Y': Y}\n```\n\n```\n{'W':\n [[ 3.  4.  5.]\n  [ 6.  7.  8.]]\n <NDArray 2x3 @cpu(0)>, 'X':\n [[ 1.  0.]]\n <NDArray 1x2 @cpu(0)>, 'Y':\n [[ 3.  4.  5.]]\n <NDArray 1x3 @cpu(0)>}\n```\n\nAs you can see,\n\n```\nY[0][0] = X[0][0] * W[0][0] + X[0][1] * W[1][0] = 1 * 3 + 0 * 6 = 3\nY[0][1] = X[0][0] * W[0][1] + X[0][1] * W[1][1] = 1 * 4 + 0 * 7 = 4\nY[0][2] = X[0][0] * W[0][2] + X[0][1] * W[1][2] = 1 * 5 + 0 * 8 = 5\n```\n\nWhat about dY / dW, the gradient for ``W``? Let's call it ``grad_W``. To start with, the shape of ``grad_W`` is the same as that of ``W`` as we are taking the derivatives with respect to ``W``, which is 2x3. Then we calculate each entry in ``grad_W`` as follows:\n\n```\ngrad_W[0][0] = X[0][0] = 1\ngrad_W[0][1] = X[0][0] = 1\ngrad_W[0][2] = X[0][0] = 1\ngrad_W[1][0] = X[0][1] = 0\ngrad_W[1][1] = X[0][1] = 0\ngrad_W[1][2] = X[0][1] = 0\n```\n\nAs a matter of fact, you can calculate ``grad_W`` by multiplying the transpose of ``X`` with a matrix of ones:\n\n\n```{.python .input}\ngrad_W = mx.nd.dot(X, mx.nd.ones_like(Y), transpose_a=True)\ngrad_W\n```\n\n\n\n\n```\n[[ 1.  1.  1.]\n [ 0.  0.  0.]]\n<NDArray 2x3 @cpu(0)>\n```\n\n\n\nAs you can see, row 0 of ``grad_W`` contains non-zero values while row 1 of ``grad_W`` does not. Why did that happen?\nIf you look at how ``grad_W`` is calculated, notice that since column 1 of ``X`` is filled with zeros, row 1 of ``grad_W`` is filled with zeros too.\n\nIn the real world, gradients for parameters that interact with sparse inputs ususally have gradients where many row slices are completely zeros. Storing and manipulating such sparse matrices with many row slices of all zeros in the default dense structure results in wasted memory and processing on the zeros. More importantly, many gradient based optimization methods such as SGD, [AdaGrad](https://stanford.edu/~jduchi/projects/DuchiHaSi10_colt.pdf) and [Adam](https://arxiv.org/pdf/1412.6980.pdf)\ntake advantage of sparse gradients and prove to be efficient and effective.\n**In MXNet, the ``RowSparseNDArray`` stores the matrix in ``row sparse`` format, which is designed for arrays of which most row slices are all zeros.**\nIn this tutorial, we will describe what the row sparse format is and how to use RowSparseNDArray for sparse gradient updates in MXNet.\n\n## Prerequisites\n\nTo complete this tutorial, we need:\n\n- MXNet. See the instructions for your operating system in [Setup and Installation](https://mxnet.apache.org/get_started)\n- [Jupyter](http://jupyter.org/)\n    ```\n    pip install jupyter\n    ```\n- Basic knowledge of NDArray in MXNet. See the detailed tutorial for NDArray in [NDArray - Imperative tensor operations on CPU/GPU](../01-ndarray-intro.rst)\n- Understanding of [automatic differentiation with autograd](../../../autograd/index.ipynb)\n- GPUs - A section of this tutorial uses GPUs. If you don't have GPUs on your\nmachine, simply set the variable `gpu_device` (set in the GPUs section of this\ntutorial) to `mx.cpu()`\n\n## Row Sparse Format\n\nA RowSparseNDArray represents a multidimensional NDArray of shape `[LARGE0, D1, .. , Dn]` using two separate 1D arrays:\n`data` and `indices`.\n\n- data: an NDArray of any dtype with shape `[D0, D1, ..., Dn]`.\n- indices: a 1D int64 NDArray with shape `[D0]` with values sorted in ascending order.\n\nThe ``indices`` array stores the indices of the row slices with **non-zeros**,\nwhile the values are stored in ``data`` array. The corresponding NDArray `dense` represented by RowSparseNDArray `rsp` has\n\n``dense[rsp.indices[i], :, :, :, ...] = rsp.data[i, :, :, :, ...]``\n\nA RowSparseNDArray is typically used to represent non-zero row slices of a large NDArray of shape `[LARGE0, D1, .. , Dn]` where LARGE0 >> D0 and most row slices are zeros.\n\nGiven this two-dimension matrix:\n\n\n```{.python .input}\n[[ 1, 2, 3],\n [ 0, 0, 0],\n [ 4, 0, 5],\n [ 0, 0, 0],\n [ 0, 0, 0]]\n```\n\nThe row sparse representation would be:\n- `data` array holds all the non-zero row slices of the array.\n- `indices` array stores the row index for each row slice with non-zero elements.\n\n\n```{.python .input}\ndata = [[1, 2, 3], [4, 0, 5]]\nindices = [0, 2]\n```\n\n`RowSparseNDArray` supports multidimensional arrays. Given this 3D tensor:\n\n\n```{.python .input}\n[[[1, 0],\n  [0, 2],\n  [3, 4]],\n\n [[5, 0],\n  [6, 0],\n  [0, 0]],\n\n [[0, 0],\n  [0, 0],\n  [0, 0]]]\n```\n\nThe row sparse representation would be (with `data` and `indices` defined the same as above):\n\n\n```{.python .input}\ndata = [[[1, 0], [0, 2], [3, 4]], [[5, 0], [6, 0], [0, 0]]]\nindices = [0, 1]\n```\n\n``RowSparseNDArray`` is a subclass of ``NDArray``. If you query **stype** of a RowSparseNDArray,\nthe value will be **\"row_sparse\"**.\n\n## Array Creation\n\nYou can create a `RowSparseNDArray` with data and indices by using the `row_sparse_array` function:\n\n\n```{.python .input}\nimport mxnet as mx\nimport numpy as np\n# Create a RowSparseNDArray with python lists\nshape = (6, 2)\ndata_list = [[1, 2], [3, 4]]\nindices_list = [1, 4]\na = mx.nd.sparse.row_sparse_array((data_list, indices_list), shape=shape)\n# Create a RowSparseNDArray with numpy arrays\ndata_np = np.array([[1, 2], [3, 4]])\nindices_np = np.array([1, 4])\nb = mx.nd.sparse.row_sparse_array((data_np, indices_np), shape=shape)\n{'a':a, 'b':b}\n```\n\n\n\n\n`{'a':  <RowSparseNDArray 6x2 @cpu(0)>, 'b':   <RowSparseNDArray 6x2 @cpu(0)>}`<!--notebook-skip-line-->\n\n\n\n## Function Overview\n\nSimilar to `CSRNDArray`, the are several functions with `RowSparseNDArray` that behave the same way. In the code blocks below you can try out these common functions:\n\n- **.dtype** - to set the data type\n- **.asnumpy** - to cast as a numpy array for inspecting it\n- **.data** - to access the data array\n- **.indices** - to access the indices array\n- **.tostype** - to set the storage type\n- **.cast_storage** - to convert the storage type\n- **.copy** - to copy the array\n- **.copyto** - to copy to deep copy an existing array\n\n\n## Setting Type\n\nYou can create a `RowSparseNDArray` from another specifying the element data type with the option `dtype`, which accepts a numpy type. By default, `float32` is used.\n\n\n```{.python .input}\n# Float32 is used by default\nc = mx.nd.sparse.array(a)\n# Create a 16-bit float array\nd = mx.nd.array(a, dtype=np.float16)\n(c.dtype, d.dtype)\n```\n\n\n\n\n`(numpy.float32, numpy.float16)`<!--notebook-skip-line-->\n\n\n\n## Inspecting Arrays\n\nAs with `CSRNDArray`, you can inspect the contents of a `RowSparseNDArray` by filling\nits contents into a dense `numpy.ndarray` using the `asnumpy` function.\n\n\n```{.python .input}\na.asnumpy()\n```\n\n\n\n```\narray([[ 0.,  0.],\n       [ 1.,  2.],\n       [ 0.,  0.],\n       [ 0.,  0.],\n       [ 3.,  4.],\n       [ 0.,  0.]], dtype=float32)\n```\n\n\nYou can inspect the internal storage of a RowSparseNDArray by accessing attributes such as `indices` and `data`:\n\n\n```{.python .input}\n# Access data array\ndata = a.data\n# Access indices array\nindices = a.indices\n{'a.stype': a.stype, 'data':data, 'indices':indices}\n```\n\n\n\n```\n{'a.stype': 'row_sparse', 'data':\n [[ 1.  2.]\n  [ 3.  4.]]\n <NDArray 2x2 @cpu(0)>, 'indices':\n [1 4]\n <NDArray 2 @cpu(0)>}\n```\n\n\n## Storage Type Conversion\n\nYou can convert an NDArray to a RowSparseNDArray and vice versa by using the `tostype` function:\n\n\n```{.python .input}\n# Create a dense NDArray\nones = mx.nd.ones((2,2))\n# Cast the storage type from `default` to `row_sparse`\nrsp = ones.tostype('row_sparse')\n# Cast the storage type from `row_sparse` to `default`\ndense = rsp.tostype('default')\n{'rsp':rsp, 'dense':dense}\n```\n\n\n\n```\n{'dense':\n [[ 1.  1.]\n  [ 1.  1.]]\n <NDArray 2x2 @cpu(0)>, 'rsp':\n <RowSparseNDArray 2x2 @cpu(0)>}\n```\n\n\nYou can also convert the storage type by using the `cast_storage` operator:\n\n\n```{.python .input}\n# Create a dense NDArray\nones = mx.nd.ones((2,2))\n# Cast the storage type to `row_sparse`\nrsp = mx.nd.sparse.cast_storage(ones, 'row_sparse')\n# Cast the storage type to `default`\ndense = mx.nd.sparse.cast_storage(rsp, 'default')\n{'rsp':rsp, 'dense':dense}\n```\n\n\n\n```\n{'dense':\n [[ 1.  1.]\n  [ 1.  1.]]\n <NDArray 2x2 @cpu(0)>, 'rsp':\n <RowSparseNDArray 2x2 @cpu(0)>}\n```\n\n\n## Copies\n\nYou can use the `copy` method which makes a deep copy of the array and its data, and returns a new array.\nWe can also use the `copyto` method or the slice operator `[]` to deep copy to an existing array.\n\n\n```{.python .input}\na = mx.nd.ones((2,2)).tostype('row_sparse')\nb = a.copy()\nc = mx.nd.sparse.zeros('row_sparse', (2,2))\nc[:] = a\nd = mx.nd.sparse.zeros('row_sparse', (2,2))\na.copyto(d)\n{'b is a': b is a, 'b.asnumpy()':b.asnumpy(), 'c.asnumpy()':c.asnumpy(), 'd.asnumpy()':d.asnumpy()}\n```\n\n\n\n```\n{'b is a': False, 'b.asnumpy()': array([[ 1.,  1.],\n        [ 1.,  1.]], dtype=float32), 'c.asnumpy()': array([[ 1.,  1.],\n        [ 1.,  1.]], dtype=float32), 'd.asnumpy()': array([[ 1.,  1.],\n        [ 1.,  1.]], dtype=float32)}\n```\n\n\nIf the storage types of source array and destination array do not match,\nthe storage type of destination array will not change when copying with `copyto` or the slice operator `[]`. The source array will be temporarily converted to desired storage type before the copy.\n\n\n```{.python .input}\ne = mx.nd.sparse.zeros('row_sparse', (2,2))\nf = mx.nd.sparse.zeros('row_sparse', (2,2))\ng = mx.nd.ones(e.shape)\ne[:] = g\ng.copyto(f)\n{'e.stype':e.stype, 'f.stype':f.stype, 'g.stype':g.stype}\n```\n\n\n\n\n`{'e.stype': 'row_sparse', 'f.stype': 'row_sparse', 'g.stype': 'default'}`<!--notebook-skip-line-->\n\n\n\n## Retain Row Slices\n\nYou can retain a subset of row slices from a RowSparseNDArray specified by their row indices.\n\n\n```{.python .input}\ndata = [[1, 2], [3, 4], [5, 6]]\nindices = [0, 2, 3]\nrsp = mx.nd.sparse.row_sparse_array((data, indices), shape=(5, 2))\n# Retain row 0 and row 1\nrsp_retained = mx.nd.sparse.retain(rsp, mx.nd.array([0, 1]))\n{'rsp.asnumpy()': rsp.asnumpy(), 'rsp_retained': rsp_retained, 'rsp_retained.asnumpy()': rsp_retained.asnumpy()}\n```\n\n\n\n```\n{'rsp.asnumpy()': array([[ 1.,  2.],\n        [ 0.,  0.],\n        [ 3.,  4.],\n        [ 5.,  6.],\n        [ 0.,  0.]], dtype=float32), 'rsp_retained':\n <RowSparseNDArray 5x2 @cpu(0)>, 'rsp_retained.asnumpy()': array([[ 1.,  2.],\n        [ 0.,  0.],\n        [ 0.,  0.],\n        [ 0.,  0.],\n        [ 0.,  0.]], dtype=float32)}\n```\n\n\n## Sparse Operators and Storage Type Inference\n\nOperators that have specialized implementation for sparse arrays can be accessed in ``mx.nd.sparse``. You can read the [mxnet.ndarray.sparse API documentation](https://mxnet.apache.org/versions/master/api/python/docs/api/legacy/ndarray/sparse/index.html) to find what sparse operators are available.\n\n\n```{.python .input}\nshape = (3, 5)\ndata = [7, 8, 9]\nindptr = [0, 2, 2, 3]\nindices = [0, 2, 1]\n# A csr matrix as lhs\nlhs = mx.nd.sparse.csr_matrix((data, indices, indptr), shape=shape)\n# A dense matrix as rhs\nrhs = mx.nd.ones((3, 2))\n# row_sparse result is inferred from sparse operator dot(csr.T, dense) based on input stypes\ntranspose_dot = mx.nd.sparse.dot(lhs, rhs, transpose_a=True)\n{'transpose_dot': transpose_dot, 'transpose_dot.asnumpy()': transpose_dot.asnumpy()}\n```\n\n\n\n```\n{'transpose_dot':\n <RowSparseNDArray 5x2 @cpu(0)>, 'transpose_dot.asnumpy()': array([[ 7.,  7.],\n        [ 9.,  9.],\n        [ 8.,  8.],\n        [ 0.,  0.],\n        [ 0.,  0.]], dtype=float32)}\n```\n\n\nFor any sparse operator, the storage type of output array is inferred based on inputs. You can either read the documentation or inspect the `stype` attribute of output array to know what storage type is inferred:\n\n\n```{.python .input}\na = transpose_dot.copy()\nb = a * 2  # b will be a RowSparseNDArray since zero multiplied by 2 is still zero\nc = a + mx.nd.ones((5, 2))  # c will be a dense NDArray\n{'b.stype':b.stype, 'c.stype':c.stype}\n```\n\n\n\n\n`{'b.stype': 'row_sparse', 'c.stype': 'default'}`<!--notebook-skip-line-->\n\n\n\nFor operators that don't specialize in sparse arrays, you can still use them with sparse inputs with some performance penalty.\nIn MXNet, dense operators require all inputs and outputs to be in the dense format.\n\nIf sparse inputs are provided, MXNet will convert sparse inputs into dense ones temporarily so that the dense operator can be used.\n\nIf sparse outputs are provided, MXNet will convert the dense outputs generated by the dense operator into the provided sparse format.\n\nFor operators that don't specialize in sparse arrays, you can still use them with sparse inputs with some performance penalty.\n\n\n```{.python .input}\ne = mx.nd.sparse.zeros('row_sparse', a.shape)\nd = mx.nd.log(a) # dense operator with a sparse input\ne = mx.nd.log(a, out=e) # dense operator with a sparse output\n{'a.stype':a.stype, 'd.stype':d.stype, 'e.stype':e.stype} # stypes of a and e will be not changed\n```\n\n\n\n\n`{'a.stype': 'row_sparse', 'd.stype': 'default', 'e.stype': 'row_sparse'}` <!--notebook-skip-line-->\n\n\n\nNote that warning messages will be printed when such a storage fallback event happens. If you are using jupyter notebook, the warning message will be printed in your terminal console.\n\n## Sparse Optimizers\n\nIn MXNet, sparse gradient updates are applied when gradient is in `row_sparse` storage and the optimizer is created with `lazy_update=True`.\nThe sparse optimizers only update the row slices of the weight and the states whose indices appear\nin `gradient.indices`. For example, the default update rule for SGD optimizer is:\n\n```\nrescaled_grad = learning_rate * rescale_grad * clip(grad, clip_gradient) + weight_decay * weight\nstate = momentum * state + rescaled_grad\nweight = weight - state\n```\n\nHowever, with sparse gradient the SGD optimizer uses the following lazy update by default:\n\n```\nfor row in grad.indices:\n    rescaled_grad[row] = learning_rate * rescale_grad * clip(grad[row], clip_gradient) + weight_decay * weight[row]\n    state[row] = momentum[row] * state[row] + rescaled_grad[row]\n    weight[row] = weight[row] - state[row]\n```\n\nThis means that the lazy update leads to different optimization results if `weight_decay` or `momentum` is non-zero.\nTo disable lazy update, please set `lazy_update` to be False when creating the optimizer.\n\n\n```{.python .input}\n# Create weight\nshape = (4, 2)\nweight = mx.nd.ones(shape).tostype('row_sparse')\n# Create gradient\ndata = [[1, 2], [4, 5]]\nindices = [1, 2]\ngrad = mx.nd.sparse.row_sparse_array((data, indices), shape=shape)\nsgd = mx.optimizer.SGD(learning_rate=0.01, momentum=0.01)\n# Create momentum\nmomentum = sgd.create_state(0, weight)\n# Before the update\n{\"grad.asnumpy()\":grad.asnumpy(), \"weight.asnumpy()\":weight.asnumpy(), \"momentum.asnumpy()\":momentum.asnumpy()}\n```\n\n\n\n```\n{'grad.asnumpy()': array([[ 0.,  0.],\n        [ 1.,  2.],\n        [ 4.,  5.],\n        [ 0.,  0.]], dtype=float32), 'momentum.asnumpy()': array([[ 0.,  0.],\n        [ 0.,  0.],\n        [ 0.,  0.],\n        [ 0.,  0.]], dtype=float32), 'weight.asnumpy()': array([[ 1.,  1.],\n        [ 1.,  1.],\n        [ 1.,  1.],\n        [ 1.,  1.]], dtype=float32)}\n```\n\n\n\n\n```{.python .input}\nsgd.update(0, weight, grad, momentum)\n# Only row 0 and row 2 are updated for both weight and momentum\n{\"weight.asnumpy()\":weight.asnumpy(), \"momentum.asnumpy()\":momentum.asnumpy()}\n```\n\n\n\n```\n{'momentum.asnumpy()': array([[ 0.  ,  0.  ],\n        [-0.01, -0.02],\n        [-0.04, -0.05],\n        [ 0.  ,  0.  ]], dtype=float32),\n 'weight.asnumpy()': array([[ 1.        ,  1.        ],\n        [ 0.99000001,  0.98000002],\n        [ 0.95999998,  0.94999999],\n        [ 1.        ,  1.        ]], dtype=float32)}\n```\n\n\nNote that only [mxnet.optimizer.SGD](../../../../../api/optimizer/index.rst#mxnet.optimizer.SGD), [mxnet.optimizer.Adam](../../../../../api/optimizer/index.rst#mxnet.optimizer.Adam), and\n[mxnet.optimizer.AdaGrad](../../../../../api/optimizer/index.rst#mxnet.optimizer.AdaGrad) support sparse updates in MXNet.\n\n## Advanced Topics\n\n### GPU Support\n\nBy default, RowSparseNDArray operators are executed on CPU. To create a RowSparseNDArray on gpu, we need to explicitly specify the context:\n\n**Note** If a GPU is not available, an error will be reported in the following section. In order to execute it on a cpu, set gpu_device to mx.cpu().\n\n\n```{.python .input}\nimport sys\ngpu_device=mx.gpu() # Change this to mx.cpu() in absence of GPUs.\ntry:\n    a = mx.nd.sparse.zeros('row_sparse', (100, 100), ctx=gpu_device)\n    a\nexcept mx.MXNetError as err:\n    sys.stderr.write(str(err))\n```\n\n\n\n<!-- INSERT SOURCE DOWNLOAD BUTTONS -->\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/np/cheat-sheet.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# The NP on MXNet cheat sheet\n\nTo begin, import the `np` and `npx` module and update MXNet to run in\nNumPy-like mode.\n\n```{.python .input}\nfrom mxnet import np, npx\nnpx.set_np()  # Change MXNet to the numpy-like mode.\n```\n\nNDArray figure (TODO)\n\n## Creating arrays\n\n```{.python .input}\nnp.array([1, 2, 3])  # default datatype is float32\n```\n\n```{.python .input}\nnp.array([(1.5, 2, 3), (4, 5, 6)], dtype='float16')\n```\n\n```{.python .input}\nnp.array([[(15,2,3), (4,5,6)], [(3,2,1), (4,5,6)]], dtype='int32')\n```\n\n### Initial placeholders\n\n```{.python .input}\nnp.zeros((3, 4))  # Create an array of zeros\n```\n\n```{.python .input}\nnp.ones((2, 3, 4), dtype='int8')  # Create an array of ones\n```\n\n```{.python .input}\nnp.arange(10, 25, 5)  # Create an array of evenly spaced values (step value)\n```\n\n```{.python .input}\n# Create an array of evenly spaced values (number of samples)\n# np.linspace(0, 2, 9)\n```\n\n```{.python .input}\n# np.full((2, 2), 7)  # Create a constant array\n```\n\n```{.python .input}\n# np.eye(2)  # Create a 2X2 identity matrix\n```\n\n```{.python .input}\n# np.random.random((2, 2))  # Create an array with random values\n```\n\n```{.python .input}\nnp.empty((3,2))  # Create an empty array\n```\n\n## I/O\n\n### Saving and loading on disk\n\n```{.python .input}\n# Save one array\na = np.array([1, 2, 3])\nnpx.save('my_array', a)\nnpx.load('my_array')\n```\n\n```{.python .input}\n# Save a list of arrays\nb = np.array([4, 6, 8])\nnpx.savez('my_arrays', *[a, b])\nnpx.load('my_arrays')\n```\n\n### Saving and loading text files\n\n```{.python .input}\n# np.loadtxt(\"myfile.txt\")\n# np.genfromtxt(\"my_file.csv\", delimiter=',')\n# np.savetxt(\"myarray.txt\", a, delimiter=\" \")\n```\n\n## Data types\n\n```{.python .input}\n# np.int64    # Signed 64-bit integer types\n# np.float32  # Standard double-precision floating point\n# np.complex  # Complex numbers represented by 128 floats\n# np.bool     # Boolean type storing TRUE and FALSE values\n# np.object   # Python object type\n# np.string_  # Fixed-length string type\n# np.unicode_ # Fixed-length unicode type\n```\n\n## Inspecting your array\n\n```{.python .input}\na.shape # Array dimensions\n```\n\n```{.python .input}\nlen(a) # Length of array\n```\n\n```{.python .input}\nb.ndim # Number of array dimensions\n```\n\n```{.python .input}\nb.size # Number of array elements\n```\n\n```{.python .input}\nb.dtype # Data type of array elements\n```\n\n```{.python .input}\n# b.dtype.name # Name of data type\n```\n\n```{.python .input}\nb.astype('int') # Convert an array to a different type\n```\n\n## Asking For Help\n\n```{.python .input}\n# np.info(np.ndarray.dtype)\n```\n\n## Array mathematics\n\n### Arithmetic operations\n\n```{.python .input}\na - b # Subtraction\n```\n\n```{.python .input}\nnp.subtract(a, b) # Subtraction\n```\n\n```{.python .input}\nb + a # Addition\n```\n\n```{.python .input}\nnp.add(b, a) # Addition\n```\n\n```{.python .input}\na / b # Division\n```\n\n```{.python .input}\nnp.divide(a,b) # Division\n```\n\n```{.python .input}\na * b # Multiplication\n```\n\n```{.python .input}\nnp.multiply(a, b) # Multiplication\n```\n\n```{.python .input}\nnp.exp(b) # Exponentiation\n```\n\n```{.python .input}\nnp.sqrt(b) # Square root\n```\n\n```{.python .input}\nnp.sin(a) # Sines of an array\n```\n\n```{.python .input}\nnp.cos(b) # Element-wise cosine\n```\n\n```{.python .input}\nnp.log(a) # Element-wise natural logarithm\n```\n\n```{.python .input}\na.dot(b) # Dot product\n```\n\n### Comparison\n\n### Aggregate functions\n\n```{.python .input}\na.sum() # Array-wise sum\n```\n\n```{.python .input}\n# a.min() # Array-wise minimum value\n```\n\n```{.python .input}\nc = np.array(([[1,2,3], [2,3,4]]))\n# c.max(axis=0) # Maximum value of an array row\n```\n\n```{.python .input}\n# c.cumsum(axis=1) # Cumulative sum of the elements\n```\n\n```{.python .input}\na.mean() # Mean\n```\n\n```{.python .input}\n# b.median() # Median\n```\n\n```{.python .input}\n# a.corrcoef() # Correlation coefficient\n```\n\n```{.python .input}\n# np.std(b) # Standard deviation\n```\n\n## Copying arrays\n\n```{.python .input}\n# a.view() # Create a view of the array with the same data\n```\n\n```{.python .input}\nnp.copy(a) # Create a copy of the array\n```\n\n```{.python .input}\na.copy() # Create a deep copy of the array\n```\n\n## Sorting Arrays\n\n```{.python .input}\n# a.sort() # Sort an array\n```\n\n```{.python .input}\n# c.sort(axis=0) # Sort the elements of an array's axis\n```\n\n## Subsetting, slicing, indexing\n\n### Subsetting\n\n```{.python .input}\na[2] # Select the element at the 2nd index 3\n```\n\n```{.python .input}\nc[0,1] # Select the element at row 1 column 2\n```\n\n### Slicing\n\n```{.python .input}\na[0:2] # Select items at index 0 and 1\n```\n\n```{.python .input}\nc[0:2,1] # Select items at rows 0 and 1 in column 1\n```\n\n```{.python .input}\nc[:1] # Select all items at row 0\n```\n\n```{.python .input}\n# c[1,...] # Same as [1,:,:]\n```\n\n```{.python .input}\na[ : :-1] #Reversed array a array([3, 2, 1])\n```\n\n### Boolean Indexing\n\n```{.python .input}\n# a[a<2] # Select elements from a less than 2\n```\n\n### Fancy indexing\n\n```{.python .input}\nc[[1,0,1,0], [0,1,2,0]] # Select elements (1,0),(0,1),(1,2) and (0,0)\n```\n\n```{.python .input}\nc[[1,0,1,0]][:,[0,1,2,0]] # Select a subset of the matrix’s rows\n```\n\n## Array manipulation\n\n### Transposing array\n\n```{.python .input}\nnp.transpose(c) # Permute array dimensions\n```\n\n```{.python .input}\nc.T # Permute array dimensions\n```\n\n### Changing array shape\n\n```{.python .input}\n# b.ravel() # Flatten the array\n```\n\n```{.python .input}\n# c.reshape(3,-2) # Reshape, but don’t change data\n```\n\n### Adding and removing elements\n\n```{.python .input}\n# c.resize((6,2)) # Return a new array with shape (6, 2)\n```\n\n```{.python .input}\n# np.append(h,g) # Append items to an array\n```\n\n```{.python .input}\n# np.insert(a, 1, 5) # Insert items in an array\n```\n\n```{.python .input}\n# np.delete(a, [1]) # Delete items from an array\n```\n\n### Combining arrays\n\n```{.python .input}\nnp.concatenate((a,b),axis=0) # Concatenate arrays\n```\n\n```{.python .input}\n# np.vstack((a,b)) # Stack arrays vertically (row-wise)\n```\n\n```{.python .input}\n# np.r_[e,f] # Stack arrays vertically (row-wise)\n```\n\n```{.python .input}\n# np.hstack((e,f)) # Stack arrays horizontally (column-wise)\n```\n\n```{.python .input}\n# np.column_stack((a,d)) # Create stacked column-wise arrays\n```\n\n```{.python .input}\n# np.c_[a,d] # Create stacked column-wise arrays\n```\n\n### Splitting arrays\n\n```{.python .input}\n# np.hsplit(a,3) # Split the array horizontally at the 3rd index\n```\n\n```{.python .input}\n# np.vsplit(c,2) # Split the array vertically at the 2nd index\n```\n\n## Use GPUs\n\nPrerequisites: A GPU exists and GPU-enabled MXNet is installed.\n\n```{.python .input}\nnpx.num_gpus()  # Query number of GPUs\n```\n\n```{.python .input}\nnpx.gpu(0), npx.gpu(1)  # Context for the first and second GPUs\n```\n\n```{.python .input}\ngpu_0 = npx.gpu(0) if npx.num_gpus() > 1 else npx.cpu()\ng0 = np.zeros((2,3), device=gpu_0)  # Create array on GPU 0\ng0\n```\n\n```{.python .input}\ngpu_1 = npx.gpu(1) if npx.num_gpus() > 2 else npx.cpu()\ng1 = np.random.uniform(size=(2,3), device=gpu_1)  # Create array on GPU 1\ng1\n```\n\n```{.python .input}\n# Copy to another GPU\ng1.copyto(gpu_0)\n```\n\n```{.python .input}\n# Return itself if matching the device, otherwise copy\ng1.copyto(gpu_0), g1.copyto(gpu_0)\n```\n\n```{.python .input}\ng1.device  # Query the device an array is on\n```\n\n```{.python .input}\n## The computation is performed by the devices on which the input arrays are\ng0 + g1.copyto(gpu_0)\n```\n\n## Auto differentiation\n\n```{.python .input}\na.attach_grad() # Allocate gradient for a variable\na.grad # access the gradient\n```\n\nCompute the $\\nabla_a b=\\exp(2a)^T a$\n\n```{.python .input}\nfrom mxnet import autograd\n\nwith autograd.record():\n    b = np.exp(2*a).dot(a)\nb.backward()\na.grad\n```\n\n**Acknowledgement**\n\nAdapted from www.datacamp.com.\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/np/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nWhat is NP on MXNet\n===================\n\nNP on MXNet provides a NumPy-like interface with extensions\nfor deep learning. It contains two modules, ``mxnet.np``, which is similar to\nNumPy, and ``mxnet.npx``, which contains extended operators that are useful for deep\nlearning. \n\nIf this is your first time using NP on MXNet, we recommend that you review the following topics in this section:\n\n.. toctree::\n   :maxdepth: 1\n\n   cheat-sheet\n   np-vs-numpy\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/np/np-vs-numpy.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Differences between NP on MXNet and NumPy\n\nThis topic lists known differences between `mxnet.np` and `numpy`. With this quick reference, NumPy users can more easily adopt  the MXNet NumPy-like API.\n\n```{.python .input}\nimport numpy as onp  # o means original\nfrom mxnet import np, npx\nnpx.set_np()  # Configue MXNet to be NumPy-like\n```\n\n## Missing operators\n\nMany, but not all, operators in NumPy are supported in MXNet. You can find the missing operators in [NP on MXNet reference](../../../api/np/index.rst). They're displayed in gray blocks instead of having links to their documents.\n\nIn addition, an operator might not contain all arguments available in NumPy. For example, MXNet does not support stride. Check the operator document for more details.\n\n## Extra functionalities\n\nThe `mxnet.np` module aims to mimic NumPy.  Most extra functionalities that enhance NumPy for deep learning use are available on other modules, such as `npx` for operators used in deep learning and `autograd` for automatic differentiation. The `np` module API is not complete. One notable change is GPU support. Creating routines accepts a `device` argument:\n\n```{.python .input}\ngpu = npx.gpu() if npx.num_gpus() > 0 else npx.cpu()\na = np.array(1, device=gpu)\nb = np.random.uniform(device=gpu)\n(a, b.device)\n```\n\nMethods to move data across devices.\n\n```{.python .input}\na.copyto(npx.cpu()), b.to_device(npx.cpu())\n```\n\n## Default data types\n\nNumPy uses 64-bit floating numbers or 64-bit integers by default.\n\n```{.python .input}\nonp.array([1,2]).dtype, onp.array([1.2,2.3]).dtype\n```\n\nMXNet uses 32-bit floating points as the default date type. It's the default data type for deep learning.\n\n```{.python .input}\nnp.array([1,2]).dtype, np.array([1.2,2.3]).dtype\n```\n\n## Scalars\n\nNumPy has classes for scalars, whose base class is 'numpy.generic'. The return values of selecting an element and reduce operators are scalars.\n\n```{.python .input}\na = onp.array([1,2])\ntype(a[0]), type(a.sum())\n```\n\nA scalar is almost identical to a 0-rank tensor (TODO, there may be subtle difference), but it has a different class. You can check the data type with `isinstance`\n\n```{.python .input}\nb = a[0]\n(b.ndim, b.size, isinstance(b, onp.generic), isinstance(b, onp.integer),\n isinstance(b, onp.int64), isinstance(b, onp.ndarray))\n```\n\nMXNet returns 0-rank `ndarray` for scalars. (TODO, may consider to add scalar classes later.)\n\n```{.python .input}\na = np.array([1,2])\ntype(a[0]), type(a.sum())\n```\n\n```{.python .input}\nb = a[0]\nb.ndim, b.size, isinstance(b, np.ndarray)\n```\n\n## Save and load\n\nUsers can use the `npx.save`, `npx.savez` and `npx.load` methods respectively to\nsave and load arrays. `npx.save` saves single, dense arrays to the `.npy`\nformat, whereas `npx.savez` can save a collection of both dense and sparse\narrays to the `.npz` format.\n\n```{.python .input}\na = np.array(1, device=gpu)\nnpx.save('a', a)\nnpx.load('a')\nnpx.savez('a', a=a, b=a*2)\nnpx.load('a')\n```\n\n## Matplotlib\n\nSometimes the MXNet ndarray cannot used by other libraries that accept NumPy input, for example matplotlib. The best practice is converting to NumPy with `asnumpy()`.\n\n```{.python .input}\n%matplotlib inline\nimport matplotlib.pyplot as plt\n\nplt.plot(np.array([1,2]).asnumpy());\n```\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/onnx/fine_tuning_gluon.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Fine-tuning an ONNX model\n\nFine-tuning is a common practice in Transfer Learning. One can take advantage of the pre-trained weights of a network, and use them as an initializer for their own task. Indeed, quite often it is difficult to gather a dataset large enough that it would allow training from scratch deep and complex networks such as ResNet152 or VGG16. For example in an image classification task, using a network trained on a large dataset like ImageNet gives a good base from which the weights can be slightly updated, or fine-tuned, to predict accurately the new classes. We will see in this tutorial that this can be achieved even with a relatively small number of new training examples.\n\n\n[Open Neural Network Exchange (ONNX)](https://github.com/onnx/onnx) provides an open source format for AI models. It defines an extensible computation graph model, as well as definitions of built-in operators and standard data types.\n\nIn this tutorial we will:\n\n- learn how to pick a specific layer from a pre-trained .onnx model file\n- learn how to load this model in Gluon and fine-tune it on a different dataset\n\n## Pre-requisite\n\nTo run the tutorial you will need to have installed the following python modules:\n- [MXNet > 1.1.0](https://mxnet.apache.org/get_started)\n- [onnx](https://github.com/onnx/onnx)\n- matplotlib\n\nWe recommend that you have first followed this tutorial:\n- [Inference using an ONNX model on MXNet Gluon](./inference_on_onnx_model.ipynb)\n\n\n```{.python .input}\nimport json\nimport logging\nimport multiprocessing\nimport os\nimport tarfile\n\nlogging.basicConfig(level=logging.INFO)\n\nimport matplotlib.pyplot as plt\nimport mxnet as mx\nfrom mxnet import gluon, np, npx, autograd\nfrom mxnet.gluon.data.vision.datasets import ImageFolderDataset\nfrom mxnet.gluon.data import DataLoader\nimport mxnet.contrib.onnx as onnx_mxnet\nimport numpy as onp\n\n%matplotlib inline\n```\n\n\n### Downloading supporting files\nThese are images and a vizualisation script:\n\n\n```{.python .input}\nimage_folder = \"images\"\nutils_file = \"utils.py\" # contain utils function to plot nice visualization\nimages = ['wrench.jpg', 'dolphin.jpg', 'lotus.jpg']\nbase_url = \"https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/onnx/{}?raw=true\"\n\n\nfor image in images:\n    mx.test_utils.download(base_url.format(\"{}/{}\".format(image_folder, image)), fname=image,dirname=image_folder)\nmx.test_utils.download(base_url.format(utils_file), fname=utils_file)\n\nfrom utils import *\n```\n\n## Downloading a model from the ONNX model zoo\n\nWe download a pre-trained model, in our case the [GoogleNet](https://arxiv.org/abs/1409.4842) model, trained on [ImageNet](http://www.image-net.org/) from the [ONNX model zoo](https://github.com/onnx/models). The model comes packaged in an archive `tar.gz` file containing an `model.onnx` model file.\n\n\n```{.python .input}\nbase_url = \"https://s3.amazonaws.com/download.onnx/models/opset_3/\"\ncurrent_model = \"bvlc_googlenet\"\nmodel_folder = \"model\"\narchive_file = \"{}.tar.gz\".format(current_model)\narchive_path = os.path.join(model_folder, archive_file)\nurl = \"{}{}\".format(base_url, archive_file)\nonnx_path = os.path.join(model_folder, current_model, 'model.onnx')\n\n# Download the zipped model\nmx.test_utils.download(url, dirname = model_folder)\n\n# Extract the model\nif not os.path.isdir(os.path.join(model_folder, current_model)):\n    print('Extracting {} in {}...'.format(archive_path, model_folder))\n    tar = tarfile.open(archive_path, \"r:gz\")\n    tar.extractall(model_folder)\n    tar.close()\n    print('Model extracted.')\n```\n\n## Downloading the Caltech101 dataset\n\nThe [Caltech101 dataset](https://data.caltech.edu/records/20086) is made of pictures of objects belonging to 101 categories. About 40 to 800 images per category. Most categories have about 50 images.\n\n*L. Fei-Fei, R. Fergus and P. Perona. Learning generative visual models from few training examples: an incremental Bayesian approach tested on 101 object categories. IEEE. CVPR 2004, Workshop on Generative-Model\nBased Vision. 2004*\n\n\n```{.python .input}\ndata_folder = \"data\"\ndataset_name = \"101_ObjectCategories\"\narchive_file = \"{}.tar.gz\".format(dataset_name)\narchive_path = os.path.join(data_folder, archive_file)\ndata_url = \"https://s3.us-east-2.amazonaws.com/mxnet-public/\"\n\nif not os.path.isfile(archive_path):\n    mx.test_utils.download(\"{}{}\".format(data_url, archive_file), dirname = data_folder)\n    print('Extracting {} in {}...'.format(archive_file, data_folder))\n    tar = tarfile.open(archive_path, \"r:gz\")\n    tar.extractall(data_folder)\n    tar.close()\n    print('Data extracted.')\n```\n\n\n```{.python .input}\ntraining_path = os.path.join(data_folder, dataset_name)\ntesting_path = os.path.join(data_folder, \"{}_test\".format(dataset_name))\n```\n\n### Load the data using an ImageFolderDataset and a DataLoader\n\nWe need to transform the images to a format accepted by the network\n\n\n```{.python .input}\nEDGE = 224\nSIZE = (EDGE, EDGE)\nBATCH_SIZE = 32\nNUM_WORKERS = 6\n```\n\nWe transform the dataset images using the following operations:\n- resize the shorter edge to 224, the longer edge will be greater or equal to 224\n- center and crop an area of size (224,224)\n- transpose the channels to be (3,224,224)\n\n\n```{.python .input}\ndef transform(image, label):\n    resized = mx.image.resize_short(image, EDGE)\n    cropped, crop_info = mx.image.center_crop(resized, SIZE)\n    transposed = np.transpose(cropped, (2,0,1))\n    return transposed, label\n```\n\nThe train and test dataset are created automatically by passing the root of each folder. The labels are built using the sub-folders names as label.\n```\ntrain_root\n__label1\n____image1\n____image2\n__label2\n____image3\n____image4\n```\n\n\n```{.python .input}\ndataset_train = ImageFolderDataset(root=training_path)\ndataset_test = ImageFolderDataset(root=testing_path)\n```\n\nWe use several worker processes, which means the dataloading and pre-processing is going to be distributed across multiple processes. This will help preventing our GPU from starving and waiting for the data to be copied across\n\n\n```{.python .input}\ndataloader_train = DataLoader(dataset_train.transform(transform, lazy=False), batch_size=BATCH_SIZE, last_batch='rollover',\n                              shuffle=True, num_workers=NUM_WORKERS)\ndataloader_test = DataLoader(dataset_test.transform(transform, lazy=False), batch_size=BATCH_SIZE, last_batch='rollover',\n                             shuffle=False, num_workers=NUM_WORKERS)\nprint(\"Train dataset: {} images, Test dataset: {} images\".format(len(dataset_train), len(dataset_test)))\n```\n\n\n`Train dataset: 6996 images, Test dataset: 1681 images`<!--notebook-skip-line-->\n\n\n\n```{.python .input}\ncategories = dataset_train.synsets\nNUM_CLASSES = len(categories)\nBATCH_SIZE = 32\n```\n\nLet's plot the 1000th image to test the dataset\n\n\n```{.python .input}\nN = 1000\nplt.imshow((transform(dataset_train[N][0], 0)[0].asnumpy().transpose((1,2,0))))\nplt.axis('off')\nprint(categories[dataset_train[N][1]])\n```\n\n\n`Motorbikes`<!--notebook-skip-line-->\n\n\n\n![onnx motorbike](https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/onnx/motorbike.png?raw=true)<!--notebook-skip-line-->\n\n\n## Fine-Tuning the ONNX model\n\n### Getting the last layer\n\nLoad the ONNX model\n\n\n```{.python .input}\nsym, arg_params, aux_params = onnx_mxnet.import_model(onnx_path)\n```\n\nThis function get the output of a given layer\n\n\n```{.python .input}\ndef get_layer_output(symbol, arg_params, aux_params, layer_name):\n    all_layers = symbol.get_internals()\n    net = all_layers[layer_name+'_output']\n    net = mx.symbol.Flatten(data=net)\n    new_args = dict({k:arg_params[k] for k in arg_params if k in net.list_arguments()})\n    new_aux = dict({k:aux_params[k] for k in aux_params if k in net.list_arguments()})\n    return (net, new_args, new_aux)\n```\n\nHere we print the different layers of the network to make it easier to pick the right one\n\n\n```{.python .input}\nsym.get_internals()\n```\n\n\n\n\n```<Symbol group [data_0, pad0, conv1/7x7_s2_w_0, conv1/7x7_s2_b_0, convolution0, relu0, pad1, pooling0, lrn0, pad2, conv2/3x3_reduce_w_0, conv2/3x3_reduce_b_0, convolution1, relu1, pad3, conv2/3x3_w_0, conv2/3x3_b_0, convolution2, relu2, lrn1, pad4, pooling1, pad5, inception_3a/1x1_w_0, inception_3a/1x1_b_0, convolution3, relu3, pad6, .................................................................................inception_5b/pool_proj_b_0, convolution56, relu56, concat8, pad70, pooling13, dropout0, flatten0, loss3/classifier_w_0, linalg_gemm20, loss3/classifier_b_0, _mulscalar0, broadcast_add0, softmax0]>```<!--notebook-skip-line-->\n\n\n\nWe get the network until the output of the `flatten0` layer\n\n\n```{.python .input}\nnew_sym, new_arg_params, new_aux_params = get_layer_output(sym, arg_params, aux_params, 'flatten0')\n```\n\n### Fine-tuning in gluon\n\n\nWe can now take advantage of the features and pattern detection knowledge that our network learnt training on ImageNet, and apply that to the new Caltech101 dataset.\n\n\nWe pick a device, fine-tuning on CPU will be **WAY** slower.\n\n\n```{.python .input}\ndevice = mx.gpu() if mx.device.num_gpus() > 0 else mx.cpu()\n```\n\nWe create a symbol block that is going to hold all our pre-trained layers, and assign the weights of the different pre-trained layers to the newly created SymbolBlock\n\n\n```{.python .input}\nimport warnings\nwith warnings.catch_warnings():\n    warnings.simplefilter(\"ignore\")\n    pre_trained = gluon.nn.SymbolBlock(outputs=new_sym, inputs=mx.sym.var('data_0'))\nnet_params = pre_trained.collect_params()\nfor param in new_arg_params:\n    if param in net_params:\n        net_params[param]._load_init(new_arg_params[param], device=device)\nfor param in new_aux_params:\n    if param in net_params:\n        net_params[param]._load_init(new_aux_params[param], device=device)\n\n```\n\nWe create the new dense layer with the right new number of classes (101) and initialize the weights\n\n\n```{.python .input}\ndense_layer = gluon.nn.Dense(NUM_CLASSES)\ndense_layer.initialize(mx.init.Xavier(magnitude=2.24), device=device)\n```\n\nWe add the SymbolBlock and the new dense layer to a HybridSequential network\n\n\n```{.python .input}\nnet = gluon.nn.HybridSequential()\nnet.add(pre_trained)\nnet.add(dense_layer)\n```\n\n### Loss\nSoftmax cross entropy for multi-class classification\n\n\n```{.python .input}\nsoftmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()\n```\n\n### Trainer\nInitialize trainer with common training parameters\n\n\n```{.python .input}\nLEARNING_RATE = 0.0005\nWDECAY = 0.00001\nMOMENTUM = 0.9\n```\n\nThe trainer will retrain and fine-tune the entire network. If we use `dense_layer` instead of `net` in the cell below, the gradient updates would only be applied to the new last dense layer. Essentially we would be using the pre-trained network as a featurizer.\n\n\n```{.python .input}\ntrainer = gluon.Trainer(net.collect_params(), 'sgd',\n                        {'learning_rate': LEARNING_RATE,\n                         'wd':WDECAY,\n                         'momentum':MOMENTUM})\n```\n\n### Evaluation loop\n\nWe measure the accuracy in a non-blocking way, using `np.array` to take care of the parallelisation that MXNet and Gluon offers.\n\n\n```{.python .input}\n def evaluate_accuracy_gluon(data_iterator, net):\n    num_instance = 0\n    sum_metric = np.zeros(1,device=device, dtype=np.int32)\n    for i, (data, label) in enumerate(data_iterator):\n        data = data.astype(np.float32).to_device(device)\n        label = label.astype(np.int32).to_device(device)\n        output = net(data)\n        prediction = np.argmax(output, axis=1).astype(np.int32)\n        num_instance += len(prediction)\n        sum_metric += (prediction==label).sum()\n    accuracy = (sum_metric.astype(np.float32)/num_instance)\n    return accuracy.item()\n```\n\n\n```{.python .input}\n%%time\nprint(\"Untrained network Test Accuracy: {0:.4f}\".format(evaluate_accuracy_gluon(dataloader_test, net)))\n```\n\n`Untrained network Test Accuracy: 0.0192`<!--notebook-skip-line-->\n\n\n\n### Training loop\n\n\n```{.python .input}\nval_accuracy = 0\nfor epoch in range(5):\n    for i, (data, label) in enumerate(dataloader_train):\n        data = data.astype(np.float32).to_device(device)\n        label = label.to_device(device)\n\n        if i%20==0 and i >0:\n            print('Batch [{0}] loss: {1:.4f}'.format(i, loss.mean().item()))\n\n        with autograd.record():\n            output = net(data)\n            loss = softmax_cross_entropy(output, label)\n        loss.backward()\n        trainer.step(data.shape[0])\n\n    npx.waitall() # wait at the end of the epoch\n    new_val_accuracy = evaluate_accuracy_gluon(dataloader_test, net)\n    print(\"Epoch [{0}] Test Accuracy {1:.4f} \".format(epoch, new_val_accuracy))\n\n    # We perform early-stopping regularization, to prevent the model from overfitting\n    if val_accuracy > new_val_accuracy:\n        print('Validation accuracy is decreasing, stopping training')\n        break\n    val_accuracy = new_val_accuracy\n```\n\n`Epoch 4, Test Accuracy 0.8942307829856873`<!--notebook-skip-line-->\n\n\n## Testing\nIn the previous tutorial, we saw that the network trained on ImageNet couldn't classify correctly `wrench`, `dolphin`, `lotus` because these are not categories of the ImageNet dataset.\n\nLet's see if our network fine-tuned on Caltech101 is up for the task:\n\n\n```{.python .input}\n# Number of predictions to show\nTOP_P = 3\n```\n\n\n```{.python .input}\n# Convert img to format expected by the network\ndef transform(img):\n    return np.array(np.expand_dims(np.transpose(img, (2,0,1)),axis=0).astype(np.float32), device=device)\n```\n\n\n```{.python .input}\n# Load and transform the test images\ncaltech101_images_test = [plt.imread(os.path.join(image_folder, \"{}\".format(img))) for img in images]\ncaltech101_images_transformed = [transform(img) for img in caltech101_images_test]\n```\n\nHelper function to run batches of data\n\n\n```{.python .input}\ndef run_batch(net, data):\n    results = []\n    for batch in data:\n        outputs = net(batch)\n        results.extend([o for o in outputs.asnumpy()])\n    return np.array(results)\n```\n\n\n```{.python .input}\nresult = run_batch(net, caltech101_images_transformed)\n```\n\n\n```{.python .input}\nplot_predictions(caltech101_images_test, result, categories, TOP_P)\n```\n\n\n![onnx caltech101 correct](https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/onnx/caltech101_correct.png?raw=true)<!--notebook-skip-line-->\n\n\n**Great!** The network classified these images correctly after being fine-tuned on a dataset that contains images of `wrench`, `dolphin` and `lotus`\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/onnx/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nONNX\n======\n\n.. container:: cards\n\n   .. card::\n      :title: Fine-tuning an ONNX model with MXNet/Gluon\n      :link: fine_tuning_gluon.html\n\n      A tutorial on loading a model in Gluon and fine-tuning it on a dataset.\n\n   .. card::\n      :title: Running inference on MXNet/Gluon from an ONNX model\n      :link: inference_on_onnx_model.html\n\n      A tutorial on running inference from an ONNX model.\n\n   .. card::\n      :title: Importing an ONNX model into MXNet\n      :link: super_resolution.html\n\n      How to load a pre-trained ONNX model file into MXNet.\n\n   .. card::\n      :title: Export ONNX Models\n      :link: https://mxnet.apache.org/api/python/docs/tutorials/deploy/export/onnx.html\n\n      How to export an MXNet model to the ONNX model format.\n\n\n.. toctree::\n   :hidden:\n   :glob:\n\n   *\n   Export ONNX Models <https://mxnet.apache.org/api/python/docs/tutorials/deploy/export/onnx.html>\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/onnx/inference_on_onnx_model.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Running inference on MXNet/Gluon from an ONNX model\n\n[Open Neural Network Exchange (ONNX)](https://github.com/onnx/onnx) provides an open source format for AI models. It defines an extensible computation graph model, as well as definitions of built-in operators and standard data types.\n\nIn this tutorial we will:\n\n- learn how to load a pre-trained .onnx model file into MXNet/Gluon\n- learn how to test this model using the sample input/output\n- learn how to test the model on custom images\n\n## Pre-requisite\n\nTo run the tutorial you will need to have installed the following python modules:\n- [MXNet > 1.1.0](https://mxnet.apache.org/get_started)\n- [onnx](https://github.com/onnx/onnx) (follow the install guide)\n- matplotlib\n\n\n```{.python .input}\nimport numpy as np\nimport mxnet as mx\nfrom mxnet.contrib import onnx as onnx_mxnet\nfrom mxnet import gluon, nd\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport tarfile, os\nimport json\nimport logging\nlogging.basicConfig(level=logging.INFO)\n```\n\n### Downloading supporting files\nThese are images and a vizualisation script\n\n\n```{.python .input}\nimage_folder = \"images\"\nutils_file = \"utils.py\" # contain utils function to plot nice visualization\nimage_net_labels_file = \"image_net_labels.json\"\nimages = ['apron.jpg', 'hammerheadshark.jpg', 'dog.jpg', 'wrench.jpg', 'dolphin.jpg', 'lotus.jpg']\nbase_url = \"https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/onnx/{}?raw=true\"\n\nfor image in images:\n    mx.test_utils.download(base_url.format(\"{}/{}\".format(image_folder, image)), fname=image,dirname=image_folder)\nmx.test_utils.download(base_url.format(utils_file), fname=utils_file)\nmx.test_utils.download(base_url.format(image_net_labels_file), fname=image_net_labels_file)\n\nfrom utils import *\n```\n\n## Downloading a model from the ONNX model zoo\n\nWe download a pre-trained model, in our case the [GoogleNet](https://arxiv.org/abs/1409.4842) model, trained on [ImageNet](http://www.image-net.org/) from the [ONNX model zoo](https://github.com/onnx/models). The model comes packaged in an archive `tar.gz` file containing an `model.onnx` model file.\n\n\n```{.python .input}\nbase_url = \"https://s3.amazonaws.com/download.onnx/models/opset_3/\"\ncurrent_model = \"bvlc_googlenet\"\nmodel_folder = \"model\"\narchive = \"{}.tar.gz\".format(current_model)\narchive_file = os.path.join(model_folder, archive)\nurl = \"{}{}\".format(base_url, archive)\n```\n\nDownload and extract pre-trained model\n\n\n```{.python .input}\nmx.test_utils.download(url, dirname = model_folder)\nif not os.path.isdir(os.path.join(model_folder, current_model)):\n    print('Extracting model...')\n    tar = tarfile.open(archive_file, \"r:gz\")\n    tar.extractall(model_folder)\n    tar.close()\n    print('Extracted')\n```\n\nThe models have been pre-trained on ImageNet, let's load the label mapping of the 1000 classes.\n\n\n```{.python .input}\ncategories = json.load(open(image_net_labels_file, 'r'))\n```\n\n## Loading the model into MXNet Gluon\n\n\n```{.python .input}\nonnx_path = os.path.join(model_folder, current_model, \"model.onnx\")\n```\n\nWe get the symbol and parameter objects\n\n\n```{.python .input}\nsym, arg_params, aux_params = onnx_mxnet.import_model(onnx_path)\n```\n\nWe pick a device, CPU is fine for inference, switch to mx.gpu() if you want to use your GPU.\n\n\n```{.python .input}\ndevice = mx.cpu()\n```\n\nWe obtain the data names of the inputs to the model by using the model metadata API:\n\n```{.python .input}\nmodel_metadata = onnx_mxnet.get_model_metadata(onnx_path)\nprint(model_metadata)\n```\n\n```\n{'output_tensor_data': [(u'gpu_0/softmax_1', (1L, 1000L))],\n 'input_tensor_data': [(u'gpu_0/data_0', (1L, 3L, 224L, 224L))]}\n```\n\n```{.python .input}\ndata_names = [inputs[0] for inputs in model_metadata.get('input_tensor_data')]\nprint(data_names)\n```\n\nAnd load them into a MXNet Gluon symbol block.\n\n```{.python .input}\nimport warnings\nwith warnings.catch_warnings():\n    warnings.simplefilter(\"ignore\")\n    net = gluon.nn.SymbolBlock(outputs=sym, inputs=mx.sym.var('data_0'))\nnet_params = net.collect_params()\nfor param in arg_params:\n    if param in net_params:\n        net_params[param]._load_init(arg_params[param], device=device)\nfor param in aux_params:\n    if param in net_params:\n        net_params[param]._load_init(aux_params[param], device=device)\n```\n\nWe can now cache the computational graph through [hybridization](https://mxnet.apache.org/versions/master/api/python/docs/tutorials/packages/gluon/blocks/hybridize.html) to gain some performance\n\n\n\n```{.python .input}\nnet.hybridize()\n```\n\nWe can visualize the network (requires graphviz installed)\n\n\n```{.python .input}\nmx.visualization.plot_network(sym,  node_attrs={\"shape\":\"oval\",\"fixedsize\":\"false\"})\n```\n\n\n![network2](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/onnx/network2.png?raw=true)<!--notebook-skip-line-->\n\n\n\nThis is a helper function to run M batches of data of batch-size N through the net and collate the outputs into an array of shape (K, 1000) where K=MxN is the total number of examples (mumber of batches x batch-size) run through the network.\n\n\n```{.python .input}\ndef run_batch(net, data):\n    results = []\n    for batch in data:\n        outputs = net(batch)\n        results.extend([o for o in outputs.asnumpy()])\n    return np.array(results)\n```\n\n## Test using real images\n\n\n```{.python .input}\nTOP_P = 3 # How many top guesses we show in the visualization\n```\n\n\nTransform function to set the data into the format the network expects, (N, 3, 224, 224) where N is the batch size.\n\n\n```{.python .input}\ndef transform(img):\n    return np.expand_dims(np.transpose(img, (2,0,1)),axis=0).astype(np.float32)\n```\n\n\nWe load two sets of images in memory\n\n\n```{.python .input}\nimage_net_images = [plt.imread('{}/{}.jpg'.format(image_folder, path)) for path in ['apron', 'hammerheadshark','dog']]\ncaltech101_images = [plt.imread('{}/{}.jpg'.format(image_folder, path)) for path in ['wrench', 'dolphin','lotus']]\nimages = image_net_images + caltech101_images\n```\n\nAnd run them as a batch through the network to get the predictions\n\n```{.python .input}\nbatch = nd.array(np.concatenate([transform(img) for img in images], axis=0), device=device)\nresult = run_batch(net, [batch])\n```\n\n\n```{.python .input}\nplot_predictions(image_net_images, result[:3], categories, TOP_P)\n```\n\n\n![imagenet](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/onnx/imagenet.png?raw=true)<!--notebook-skip-line-->\n\n\n**Well done!** Looks like it is doing a pretty good job at classifying pictures when the category is a ImageNet label\n\nLet's now see the results on the 3 other images\n\n\n```{.python .input}\nplot_predictions(caltech101_images, result[3:7], categories, TOP_P)\n```\n\n\n![png](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/onnx/caltech101.png?raw=true)<!--notebook-skip-line-->\n\n\n**Hmm, not so good...**  Even though predictions are close, they are not accurate, which is due to the fact that the ImageNet dataset does not contain `wrench`, `dolphin`, or `lotus` categories and our network has been trained on ImageNet.\n\nLucky for us, the [Caltech101 dataset](https://data.caltech.edu/records/20086) has them, let's see how we can fine-tune our network to classify these categories correctly.\n\nWe show that in our next tutorial:\n\n\n- [Fine-tuning an ONNX Model using the modern imperative MXNet/Gluon](https://mxnet.apache.org/versions/master/api/python/docs/tutorials/packages/onnx/fine_tuning_gluon.html)\n\n<!-- INSERT SOURCE DOWNLOAD BUTTONS -->\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/optimizer/index.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Optimizers\n\nDeep learning models are comprised of a model architecture and the model parameters. The model architecture is chosen based on the task - for example Convolutional Neural Networks (CNNs) are very successful in handling image based tasks and Recurrent Neural Networks (RNNs) are better suited for sequential prediction tasks. However, the values of the model parameters are learned by solving an optimization problem during model training.\n\nTo learn the parameters, we start with an initialization scheme and iteratively refine the parameter initial values by moving them along a direction that is opposite to the (approximate) gradient of the loss function. The extent to which the parameters are updated in this direction is governed by a hyperparameter called the learning rate. This process, known as gradient descent, is the backbone of optimization algorithms in deep learning. In MXNet, this functionality is abstracted by the [Optimizer API](../../../api/optimizer/index.rst).\n\nWhen training a deep learning model using the MXNet [Gluon API](../gluon/index.ipynb), a Gluon [Trainer](../gluon/training/trainer.ipynb) is initialized with the all the learnable parameters and the optimizer to be used to learn those parameters. A single step of iterative refinement of model parameters in MXNet is achieved by calling [Trainer.step](../../../api/gluon/trainer.rst#mxnet.gluon.Trainer.step) which in turn uses the gradient (and perhaps some state information) to update the parameters by calling `optimizer.update`.\n\nHere is an example of how a trainer with an optimizer is created for, a simple Linear (Dense) Network.\n\n\n```{.python .input}\nfrom mxnet import gluon, optimizer\n\nnet = gluon.nn.Dense(1)\nnet.initialize()\noptim = optimizer.SGD(learning_rate=0.1)\ntrainer = gluon.Trainer(net.collect_params(), optimizer=optim)\n```\n\nIn model training, the code snippet above would be followed by a training loop which, at every iteration performs a forward pass (to compute the loss), a backward pass (to compute the gradient of the loss with respect to the parameters) and a trainer step (which updates the parameters using the gradient). See the [Gluon Trainer guide](../gluon/training/trainer.ipynb) for a complete example.\n\nWe can also create the trainer by passing in the optimizer name and optimizer params into the trainer constructor directly, as shown below.\n\n\n```{.python .input}\ntrainer = gluon.Trainer(net.collect_params(), optimizer='adam', optimizer_params={'learning_rate':1})\n```\n\n### What should I use?\nFor many deep learning model architectures, the `sgd` and `adam` optimizers are a really good place to start. If you are implementing a deep learning model and trying to pick an optimizer, start with [SGD](../../../api/optimizer/index.rst#mxnet.optimizer.SGD) as you will often get good enough results as long as your learning problem is tractable. If you already have a trainable model and you want to improve the convergence then you can try [Adam](../../../api/optimizer/index.rst#mxnet.optimizer.Adam). If you would like to improve your model training process further, there are a number of specialized optimizers out there with many of them already implemented in MXNet. This guide walks through these optimizers in some detail.\n\n## Stochastic Gradient Descent\n[Gradient descent](https://en.wikipedia.org/wiki/Gradient_descent) is a general purpose algorithm for minimizing a function using information from the gradient of the function with respect to its parameters. In deep learning, the function we are interested in minimizing is the [loss function](../gluon/loss/loss.ipynb). Our model accepts training data as inputs and the loss function tells us how good our model predictions are. Since the training data can routinely consist of millions of examples, computing the loss gradient on the full batch of training data is very computationally expensive. Luckily, we can effectively approximate the full gradient with the gradient of the loss function on randomly chosen minibatches of our training data. This variant of gradient descent is [stochastic gradient descent](https://en.wikipedia.org/wiki/Stochastic_gradient_descent).\n\nTechnically, stochastic gradient descent (SGD) refers to an online approximation of the gradient descent algorithm that computes the gradient of the loss function applied to a *single datapoint*, instead of your entire dataset, and uses this approximate gradient to update the model parameter values. However, in MXNet, and other deep learning frameworks, the SGD optimizer is agnostic to how many datapoints the loss function is applied to, and it is more effective to use a mini-batch loss gradient, as described earlier, instead of a single datapoint loss gradient.\n\n### [SGD optimizer](../../../api/optimizer/index.rst#mxnet.optimizer.SGD)\n\nFor an SGD optimizer initialized with learning rate $lr$, the update function accepts parameters (weights) $w_i$, and their gradients $grad(w_i)$, and performs the single update step:\n\n$$w_{i+1} = w_i + lr\\cdot -grad(w_i)$$\n\nvisualized in the diagram shown below.\n\n<p align=\"center\">\n    <img src=\"images/sgd_animation.gif\" alt=\"drawing\"/>\n</p>\n\n\n### Weight decay\nThe SGD update step can be modified by introducing an extra term that enforces a penalty on the size of the parameters. This is achieved by subtracting a fraction of the weight $\\delta\\cdot w$ during the weight update as shown below.\n\n$$w_{i+1} = w_i + lr\\cdot (-grad(w_i) -\\delta\\cdot w_i)$$\n\nIntroducing weight decay modifies the objective of the optimization problem by adding an implicit regularization term to penalizes large weights. Weight decay is discussed more extensively in this [paper](https://papers.nips.cc/paper/563-a-simple-weight-decay-can-improve-generalization.pdf).\n\n### Momentum\nThe convergence of the  SGD optimizer can be accelerated by incorporating momentum. Originally proposed by [Polyak (1964)](https://www.sciencedirect.com/science/article/abs/pii/0041555364901375), SGD with momentum improves the approximation of the gradient term by incorporating the gradients from previous update steps. To achieve this, SGD with momentum stores and 'remembers' the update at each iteration to be included in the next iteration. In the equations below we denote the momentum history as $v$.\n\nFor the first update the SGD optimizer with momentum performs the single update step:\n\n$$ v_1= lr\\cdot -grad(w_0)$$\n$$ w_1= w_0 + v_1 $$\n\nFor subsequent updates, SGD with momentum, with momentum parameter $\\gamma$, performs the update step:\n\n$$ v_{i+1} = \\gamma \\cdot v_{i} + lr\\cdot -grad(w_{i}) $$\n$$ w_{i+1} = w_i + v_{i+1} $$\n\nThis is also shown in the diagram below.\n\n<p align=\"center\">\n    <img src=\"images/momentum_sgd_animation.gif\" alt=\"drawing\"/>\n</p>\n\n\nThe use of SGD with momentum for learning in neural networks was introduced by Rumelhart, Hinton and Williams in [Learning Internal Representations by Error Propagation](https://dl.acm.org/citation.cfm?id=104279.104293).\n\nTo create an SGD optimizer with momentum $\\gamma$ and weight decay in MXNet simply use the following code.\n\n\n```{.python .input}\nsgd_optimizer = optimizer.SGD(learning_rate=0.1, wd=0., momentum=0.8)\n```\n\n### [Nesterov Accelerated Stochastic Gradient Descent](../../../api/optimizer/index.rst#mxnet.optimizer.NAG)\n\nThe momentum method of [Nesterov] is a modification to SGD with momentum that allows for even faster convergence in practice. With Nesterov accelerated gradient (NAG) descent, the update term is derived from the gradient of the loss function with respect to *refined parameter values*. These refined parameter values are computed by performing a SGD update step using the momentum history as the gradient term.\n\nAlternatively, you can think of the NAG optimizer as performing two update steps:\n* The first (internal) update step approximates uses the current momentum history $v_i$ to calculate the refined parameter values $(w_i + \\gamma \\cdot v_i)$. This is also known as the lookahead step.\n* The second (actual) step uses the gradient of the loss function with respect to the lookahead parameter values from the first step and the current momentum history $v_i$ to obtain a new direction to update our original parameter values, like classical momentum.\n\nThe NAG optimizer with momentum parameter $\\gamma$ performs the update step:\n\n$$ v_{i+1} = \\gamma \\cdot v_{i} + lr\\cdot -grad(w_{i} + \\gamma \\cdot v_i) $$\n$$ w_{i+1} = w_i + v_{i+1} $$\n\n<p align=\"center\">\n    <img src=\"images/nesterov_momentum_animation.gif\" alt=\"drawing\"/>\n</p>\n\n\nThe effects of using NAG over SGD and classical momentum are discussed in this [paper](http://proceedings.mlr.press/v28/sutskever13.pdf) by Sutskever et al.\n\nThe NAG optimizer can be initialized in MXNet by using the code snippet below or by creating a trainer with argument `optimizer='nag'`.\n\n\n```{.python .input}\nnag_optimizer = optimizer.NAG(learning_rate=0.1, momentum=0.8)\n```\n\n## Adaptive Learning Rate Methods\n\nThe gradient methods implemented by the optimizers described above use a global learning rate hyperparameter for all parameter updates. This has a well-documented shortcoming in that it makes the training process and convergence of the optimization algorithm really sensitive to the choice of the global learning rate. Adaptive learning rate methods avoid this pitfall by incorporating some history of the gradients observed in earlier iterations to scale step sizes (learning rates) to each learnable parameter in the model.\n\n### [AdaGrad](../../../api/optimizer/index.rst#mxnet.optimizer.AdaGrad)\n\nThe AdaGrad optimizer, which implements the optimization method originally described by [Duchi et al](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf), multiplies the global learning rate by the $L_2$ norm of the preceeding gradient estimates for each paramater to obtain the per-parameter learning rate. To achieve this, AdaGrad introduces a new term which we'll denote as $g^2$ - the accumulated square of the gradient of the loss function with respect to the parameters.\n\nThus the AdaGrad optimizer update function performs the update steps below to obtain $i+1$th refinement.\n\n$$ g^2_{i+1} = g^2_{i} + grad(w_i)^2 $$\n$$ w_{i+1} = w_i + \\dfrac{lr}{\\sqrt{g^2 + \\epsilon}}\\cdot -grad(w_i)$$\n\nThe $\\epsilon$ term is a tiny positive value introduced to avoid division by zero due to floating point issues.\n\nThe overaching benefit of AdaGrad over SGD is that it ensures the overall convergence is more resilient to the choice of the global learning rate $lr$ especially in tasks, such as natural language processing some data is sparse but the parameters influenced by the sparse data are quite informative.\n\nTo instantiate the Adagrad optimizer in MXNet you can use the following line of code.\n\n\n```{.python .input}\nadagrad_optimizer = optimizer.AdaGrad(learning_rate=0.1, epsilon=1e-07)\n```\n\n### [RMSProp](../../../api/optimizer/index.rst#mxnet.optimizer.RMSProp)\n\nRMSProp, introduced by [Tielemen and Hinton](http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf), is similar to AdaGrad described above, but, instead of accumulating the sum of historical square gradients, maintains an exponential decaying average of the historical square gradients, in order to give more weighting to more recent gradients.\n\nFor rmsprop, we introduce the term $\\mathbb{E}[g^2]$ - the decaying average over past squared gradients and $\\beta$ as the forgetting factor. The rmsprop optimizer performs the update given below.\n\n\n$$ \\mathbb{E}[g^2]_{i+1} = \\beta\\cdot\\mathbb{E}[g^2]_{i} + (1-\\beta)\\cdot [grad(w_{i})]^2 $$\n$$ w_{i+1} = w_i + \\dfrac{lr}{\\sqrt{\\mathbb{E}[g^2]_{i+1} + \\epsilon}}\\cdot -grad(w_i) $$\n\nThe $\\epsilon$ term is included, as in AdaGrad, for numerical stability.\n\nRMSProp was derived independently of AdaGrad and the name RMSProp derives from a combination of [RProp](https://en.wikipedia.org/wiki/Rprop) and the RMS, root mean square, operation in the denominator of the weight update.\n\n\n### RMSProp (Centered)\nThe MXNet RMSProp optimizer with the `centered=True` argument implements a variant of the RMSProp update described by [Alex Graves](https://arxiv.org/pdf/1308.0850v5.pdf), which centres the second moment $\\mathbb{E}[g^2]$ or decaying average of square gradients by subtracting the square of decaying average of gradients. It also adds an explicit momentum term to weight past update steps. Representing the decaying average of gradients as $\\mathbb{E}[g]$ and momentum parameter as $\\gamma$, we add another equation to the non-centered rmsprop update described above.\n\nThe centered RMSProp optimizer performs the update step:\n\n$$ \\mathbb{E}[g]_{i+1} = \\beta\\cdot\\mathbb{E}[g]_{i} + (1-\\beta)\\cdot [grad(w_{i})] $$\n$$ \\mathbb{E}[g^2]_{i+1} = \\beta\\cdot\\mathbb{E}[g^2]_{i} + (1-\\beta)\\cdot [grad(w_{i})]^2 $$\n$$ v_{i+1} = \\gamma \\cdot v_{i} + \\dfrac{lr}{\\sqrt{\\mathbb{E}[g^2]_{i+1} - \\mathbb{E}[g]^2_{i+1}+ \\epsilon}}\\cdot -grad(w_{i}) $$\n$$ w_{i+1} = w_i + v_{i+1} $$\n\nHere is an example snippet creating the RMSProp optimizer in MXNet.\n\n\n```{.python .input}\nrmsprop_optimizer = optimizer.RMSProp(learning_rate=0.001, rho=0.9, momentum=0.9, epsilon=1e-07, centered=False)\n```\n\nIn the code snippet above, `rho` is $\\beta$ in the equations above and `momentum` is $\\gamma$, which is only used where `centered=True`.\n\n### [AdaDelta](../../../api/optimizer/index.rst#mxnet.optimizer.AdaDelta)\n\nAdaDelta was introduced to address some remaining lingering issues with AdaGrad and RMSProp - the selection of a global learning rate. AdaGrad and RMSProp assign each parameter its own learning rate but the per-parameter learning rate are still calculated using the global learning rate. In contrast, AdaDelta does not require a global learning rate, instead, it tracks the square of previous update steps, represented below as $\\mathbb{E}[\\Delta w^2]$ and uses the root mean square of the previous update steps as an estimate of the learning rate.\n\nThe AdaDelta optimizer performs the following equations in its update step:\n\n$$ \\mathbb{E}[\\Delta w^2]_{i+1} = \\beta\\cdot\\mathbb{E}[\\Delta w^2]_i + (1 - \\beta) \\cdot (w_i - w_{i-1})^2 $$\n$$ \\mathbb{E}[g^2]_{i+1} = \\beta\\cdot\\mathbb{E}[g^2]_{i} + (1-\\beta)\\cdot [grad(w_{i})]^2 $$\n$$ w_{i+1} = w_i + \\dfrac{\\sqrt{\\mathbb{E}[\\Delta w^2] + \\epsilon}}{\\sqrt{\\mathbb{E}[g^2]_{i+1} + \\epsilon}} \\cdot -grad(w_i)$$\n\nAs evident from the above equations, AdaDelta is similar to RMSProp but does not require you to specify $lr$ and instead uses $\\sqrt{\\mathbb{E}[\\Delta w^2] + \\epsilon}$ as the estimated learning rate. AdaDelta was introduced by Zeiler in this [paper](https://arxiv.org/abs/1212.5701).\n\nHere is the code snippet creating the AdaDelta optimizer in MXNet. The argument `rho` in the code is $\\beta$ in the update equations. Notice there is no learning rate argument in the code.\n\n\n```{.python .input}\nadadelta_optimizer = optimizer.AdaDelta(rho=0.9, epsilon=1e-07)\n```\n\n### [Adam](../../../api/optimizer/index.rst#mxnet.optimizer.Adam)\nAdam, introduced by [Kingma and Ba](https://arxiv.org/abs/1412.6980), is one of the popular adaptive algorithms for deep learning. It combines elements of RMSProp with momentum SGD. Like RMSProp, Adam uses the RootMeanSquare of decaying average of historical gradients but also explicitly keeps track of a decaying average of momentum and uses that for the update step direction. Thus, Adam accepts two hyperparameters $\\beta_1$ and $\\beta_2$ for momentum weighting and gradient RMS weighting respectively. Adam also accepts a global learning rate that's adaptively tuned to each parameter with the gradient RootMeanSquare. Finally, Adam also includes bias correction steps within the update that transform the biased estimates of first and second order moments, $v_{i+1}$ and $\\mathbb{E}[g^2]_{i+1}$ to their unbiased counterparts $\\tilde{v}_{i+1}$ and $\\tilde{\\mathbb{E}[g^2]}_{i+1}$\n\nThe Adam optimizer performs the update step described the following equations:\n\n$$ v_{i+1} = \\beta_1 \\cdot v_{i} + (1 - \\beta_1) \\cdot grad(w_i) $$\n$$ \\mathbb{E}[g^2]_{i+1} = \\beta_2\\cdot\\mathbb{E}[g^2]_{i} + (1-\\beta_2)\\cdot [grad(w_{i})]^2 $$\n$$ \\tilde{v}_{i+1} = \\dfrac{v_{i+1}}{1 - (\\beta_1)^{i+1}} $$\n$$ \\tilde{\\mathbb{E}[g^2]}_{i+1} = \\dfrac{\\mathbb{E}[g^2]_{i+1}}{1 - (\\beta_2)^{i+1}} $$\n$$ w_{i+1} = w_i + \\dfrac{lr}{\\sqrt{\\tilde{\\mathbb{E}[g^2]}_{i+1}} + \\epsilon} \\cdot -\\tilde{v}_{i+1} $$\n\nIn MXNet, you can construct the Adam optimizer with the following line of code.\n\n\n```{.python .input}\nadam_optimizer = optimizer.Adam(learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-08)\n```\n\n### [Adamax](../../../api/optimizer/index.rst#mxnet.optimizer.Adamax)\nAdamax is a variant of Adam also included in the original paper by [Kingma and Ba](https://arxiv.org/abs/1412.6980). Like Adam, Adamax maintains a moving average for first and second moments but Adamax uses the $L_{\\infty}$ norm for the exponentially weighted average of the gradients, instead of the $L_2$ norm used in Adam used to keep track of the gradient second moment. The $L_{\\infty}$ norm of a vector is equivalent to take the maximum absolute value of elements in that vector.\n\n$$ v_{i+1} = \\beta_1 \\cdot v_{i} + (1 - \\beta_1) \\cdot grad(w_i) $$\n$$ g^\\infty_{i+1} = \\mathtt{max}(\\beta_2\\cdot g^\\infty_{i},  |{grad(w_i)}|) $$\n$$ \\tilde{v}_{i+1} = \\dfrac{v_{i+1}}{1 - \\beta_1^{i+1}} $$\n$$ w_{i+1} = w_i + \\dfrac{lr}{g^\\infty_{i+1} + \\epsilon} \\cdot - \\tilde{v}_{i+1} $$\n\nSee the code snippet below for how to construct Adamax in MXNet.\n\n\n```{.python .input}\nadamax_optimizer = optimizer.Adamax(learning_rate=0.002, beta1=0.9, beta2=0.999)\n```\n\n### [Nadam](../../../api/optimizer/index.rst#mxnet.optimizer.Nadam)\nNadam is also a variant of Adam and draws from the perspective that Adam can be viewed as a combination of RMSProp and classical Momentum (or Polyak Momentum). Nadam replaces the classical Momentum component of Adam with Nesterov Momentum (See [paper](http://cs229.stanford.edu/proj2015/054_report.pdf) by Dozat). The consequence of this is that the gradient used to update the weighted average of the momentum term is a lookahead gradient as is the case with NAG.\n\nThe Nadam optimizer performs the update step:\n\n$$ v_{i+1} = \\beta_1 \\cdot v_{i} + (1 - \\beta_1) \\cdot grad(w_i + \\beta_1 \\cdot v_{i}) $$\n$$ \\mathbb{E}[g^2]_{i+1} = \\beta_2\\cdot\\mathbb{E}[g^2]_{i} + (1-\\beta_2)\\cdot [grad(w_{i})]^2 $$\n$$ \\tilde{v}_{i+1} = \\dfrac{v_{i+1}}{1 - \\beta_1^{i+1}} $$\n$$ \\tilde{\\mathbb{E}[g^2]}_{i+1} = \\dfrac{\\mathbb{E}[g^2]_{i+1}}{1 - \\beta_2^{i+1}} $$\n$$ w_{i+1} = w_i + \\dfrac{lr}{\\sqrt{\\tilde{\\mathbb{E}[g^2]}_{i+1}} + \\epsilon}\\cdot - \\tilde{v}_{i+1} $$\n\nHere is the line of code to create the NAdam optimizer in MXNet.\n\n\n```{.python .input}\nnadam_optimizer = optimizer.Nadam(learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-08)\n```\n\n## SGD optimized for large scale distributed training\n\nTraining very deep neural networks can be time consuming and as such it is very common now to see practitioners turn to distributed training on multiple processors on the same machine or even across a fleet of machines to parallelize network training because this can reduce neural network training time from days to minutes.\n\nWhile all the preceding optimizers, from SGD to Adam, can be readily used in the distributed setting, the following optimizers in MXNet provide extra features targeted at alleviating some of the problems associated with distributed training.\n\n### [Signum](../../../api/optimizer/index.rst#mxnet.optimizer.Signum)\nIn distributed training, communicating gradients across multiple worker nodes can be expensive and create a performance bottleneck. The Signum optimizer addresses this problem by transmitting just the sign of each minibatch gradient instead of the full precision gradient. In MXNet, the signum optimizer implements two variants of compressed gradients described in the paper by [Bernstein et al](https://arxiv.org/pdf/1802.04434.pdf).\n\nThe first variant, achieved by constructing the Signum optimizer with `momentum=0`, implements SignSGD update which performs the update below.\n\n$$ w_{i+1} =  w_i - lr \\cdot sign(grad(w_i)) $$\n\nThe second variant, achieved by passing a non-zero momentum parameter implements the Signum update which is equivalent to SignSGD and momentum.\nFor momentum parameter $\\gamma \\in [0, 1]$, the Signum optimizer performs the following update:\n\n$$ v_{i+1} = \\gamma \\cdot v_i + (1 - \\gamma) \\cdot grad(w_i) $$\n$$ w_{i+1} =  w_i - lr \\cdot sign(v_{i+1}) $$\n\nHere is how to create the signum optimizer in MXNet.\n\n\n```{.python .input}\nsignum_optimizer = optimizer.Signum(learning_rate=0.01, momentum=0.9, wd_lh=0.0)\n```\n\n\n### [DCASGD](../../../api/optimizer/index.rst#mxnet.optimizer.DCASGD)\n\nThe DCASGD optimizer implements Delay Compensated Asynchronous Stochastic Gradient Descent by [Zheng et al](https://arxiv.org/pdf/1609.08326.pdf). In asynchronous distributed SGD, it is possible that a training worker node add its gradients too late to the global (parameter) server resulting in a delayed gradient being used to update the current parameters. DCASGD addresses this issue of delayed gradients by compensating for this delay in the parameter update steps.\n\nIf $grad(w_i)$ denotes the delayed gradient, $w_{i+\\tau}$ denotes the parameter values at the current iteration, and $\\lambda$ is the delay scale factor, the DCASGD optimizer update function performs the update:\n\n$$ w_{i+\\tau+1} = w_{i+\\tau} − lr \\cdot (grad(w_i) + \\lambda \\cdot grad(w_i)^2 \\cdot (w_{i+\\tau} − w_i)) $$\n\nThe DCASGD optimizer in MXNet can be initialized using the code below.\n\n\n```{.python .input}\ndcasgd_optimizer = optimizer.DCASGD(momentum=0.0, lamda=0.04)\n```\n\n## Online Learning Algorithms\nBefore deep neural networks became popular post 2012, people were already solving large scale optimization problems to train (shallow) machine learning models. One particular area this was done was active or online learning where the model is continually learning and updating its parameters after it is deployed to production. In online learning, the model has to make predictions on new inputs but moments later may become aware of the true value of what it tried to predict and use this information to update its parameters.\n\nThe class of optimization algorithms designed to tackle online learning problems have also seen some success in offline training of deep neural models. The following optimizers are algorithms taken from online learning that have been implemented in MXNet.\n\n### [FTRL](../../../api/optimizer/index.rst#mxnet.optimizer.Ftrl)\n\nFTRL stands for Follow the Regularized Leader and describes a family of algorithms originally designed for online learning tasks.\n\nFor each iteration, FTRL algorithms finds the next parameter by solving the following optimization problem which minimizes the total regret i.e the sum of the inner product all preceding gradients and next parameter. The optimization objective is regularized so that the next parameter is close (proximal) in $L2$ norm to the preceding parameter values and is sparse which is enforced by the $L1$ norm.\n\n$$ w_{i+1} = \\texttt{argmin}_{w} \\left[\\sum_{j=1}^{i} grad(w_i)\\cdot w + \\dfrac{1}{2}\\sum_{j=1}^{i} \\sigma_j \\cdot ||w - w_j||_2^2 + \\lambda ||w||_1\\right]$$\n\nDue to the similarity of online learning and neural network training, there is an [equivalence](https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/37013.pdf) between variants of gradient descent and FTRL algorithms. In fact, the $w$ that minimizes FTRL with only $L_2$ regularization (i.e $\\lambda$ in the equation above is set to 0) is exactly the $w$ derived from stochastic gradient descent update.\n\nThe version of FTRL implemented as an MXNet optimizer is from [McMahan et al](https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/41159.pdf) and encourages sparse parameters due to $L_1$ regularization. It performs the following update:\n\n$$ z_{i+1} = z_i + \\dfrac{\\left(\\sqrt{\\eta_i + grad(w_i)^2} - \\sqrt{\\eta_i}\\right) \\cdot w_i}{lr}$$\n$$ \\eta_{i+1} = \\eta_i + grad(w_i)^2$$\n$$ w_{i+1} = (|z_{i+1}| > \\lambda) \\cdot \\left[ \\dfrac{-lr}{\\beta + \\sqrt{\\eta_{i+1}}} (z_{i+1} - \\lambda \\cdot sign(z_{i+1}))\\right] $$\n\nHere is how to initialize the FTRL optimizer in MXNet\n\n\n```{.python .input}\nftrl_optimizer = optimizer.Ftrl(lamda1=0.01, learning_rate=0.1, beta=1)\n```\n\n### [FTML](../../../api/optimizer/index.rst#mxnet.optimizer.FTML)\n\nFTML stands for Follow the Moving Leader and is a variant of the FTRL family of algorithms adapted specifically to deep learning. Regular FTRL algorithms, described above, solve an optimization problem every update that involves the sum of all previous gradients. This is not well suited for the non-convex loss functions in deep learning. In the non-convex settings, older gradients are likely uninformative as the parameter updates can move to converge towards different local minima at different iterations. FTML addresses this problem by reweighing the learning subproblems in each iteration as shown below.\n\n\n\\begin{equation*}\nw_{i+1} = \\texttt{argmin}_{w} \\left[\\sum_{j=1}^{i} (1 − \\beta_1)\\beta_1^{i−j} grad(w_i)\\cdot w + \\dfrac{1}{2}\\sum_{j=1}^{i} \\sigma_j \\cdot ||w - w_j||_2^2 \\right]\n\\end{equation*}\n\n$\\beta_1$ is introduced to compute the exponential moving average of the previous accumulated gradient. The improvements of FTML over FTRL can be compared to the improvements of RMSProp/Adam to AdaGrad. According to [Zheng et al](http://proceedings.mlr.press/v70/zheng17a/zheng17a.pdf), FTML enjoys some of the nice properties of RMSProp and Adam while avoiding their pitfalls.\n\nThe FTML optimizer performs the following update:\n\n$$ v_{i+1} = \\beta_2 \\cdot v_i + (1 - \\beta_2) \\cdot grad(w_i)^2$$\n$$ d_{i+1} = \\dfrac{1 - \\beta_1^{i+1}}{lr} \\big(\\sqrt{\\dfrac{v_{i+1}}{1 - \\beta_2^{i+1}}} + \\epsilon\\big)$$\n$$ z_{i+1} = \\beta_1 \\cdot z_i + (1 - \\beta_1)\\cdot grad(w_i) - (d_{i+1} - \\beta_1 \\cdot d_i) \\cdot w_i$$\n$$ w_{i+1} = \\dfrac{-z_{i+1}}{d_{i+1}} $$\n\nIn MXNet, you can initialize the FTML optimizer using\n\n\n```{.python .input}\nftml_optimizer = optimizer.FTML(beta1=0.6, beta2=0.999, epsilon=1e-08)\n```\n\nHere `beta1` and `beta2` are similar to the arguments in the Adam optimizer.\n\n## Bayesian SGD\nA notable shortcoming of deep learning is that the model parameters learned after training are only point estimates, therefore deep learning model predictions have no information about uncertainty or confidence bounds. This is in contrast to a fully Bayesian approach which incorporates prior distributions on the model parameters and estimates the model parameters as belonging to a posterior distribution. This approach allows the predictions of a bayesian model to have information about uncertainty, as you can sample different values from the posterior distribution to obtain different model parameters. One approach to close the bayesian gap in deep learning is to incorporate the gradient descent algorithm with properties that allow the model parameters to converge to a distribution instead of a single value or point estimate.\n\n### [SGLD](../../../api/optimizer/index.rst#mxnet.optimizer.SGLD)\nStochastic Gradient Langevin Dynamics or SGLD was introduced to allow uncertainties around model parameters to be captured directly during model training. With every update in SGLD, the learning rate decreases to zero and a gaussian noise of known variances is injected into the SGD step. This has the effect of having the training parameters converge to a sufficient statistic for a posterior distribution instead of simply a point estimate of the model parameters.\n\nSGLD performs the parameter update:\n\n$$ w_{i+1} = w_i + \\dfrac{lr_{i+1}}{2}\\cdot -grad(w_i) + \\eta_{i+1}$$\n\nwhere $ \\eta_{i+1} \\sim N(0, lr_{i+1})$ i.e $\\eta_{i+1}$ is drawn from a zero centered gaussian with variance $lr_{i+1}$\n\nSGLD was introduced by [Patterson and Teh](https://papers.nips.cc/paper/4883-stochastic-gradient-riemannian-langevin-dynamics-on-the-probability-simplex.pdf) and the optimizer can be created in MXNet with the following line of code.\n\n\n```{.python .input}\nsgld_optimizer = optimizer.SGLD()\n```\n\n## Custom Optimizer\n\nIf you would like to use a particular optimizer that is not yet implemented in MXNet or you have a custom optimization algorithm of your own that you would like to use to train your model, it is very straightforward to create a custom optimizer.\n\nStep 1: First create a function that is able to perform your desired updates given the weights, gradients and other state information.\n\nStep 2: You will have to write your own optimizer class that extends the [base optimizer class](../../../api/optimizer/index.rst#mxnet.optimizer.Optimizer) and override the following functions\n* `__init__`: accepts the parameters of your optimizer algorithm as inputs as saves them as member variables.\n* `create_state`: If your custom optimizer uses some additional state information besides the gradient, then you should implement a function that accepts the weights and returns the state.\n* `update`: Implement your optimizer update function using the function in Step 1\n\nStep 3: Register your optimizer with `@register` decorator on your optimizer class.\n\nSee the [source code](../../../api/optimizer/index.rst#mxnet.optimizer.NAG) for the NAG optimizer for a concrete example.\n\n## Summary\n* MXNet implements many state-of-the-art optimizers which can be passed directly into a gluon trainer object. Calling `trainer.step` during model training uses the optimizers to update the model parameters.\n* Gradient descent algorithms minimize the loss function by using information from the gradient of the loss function and a learning rate hyperparameter.\n* Stochastic Gradient Descent is the backbone of deep learning optimization algorithms and simple SGD optimizers can be made really powerful by incorporating momentum, for example `sgd` with momentum and `nag`.\n* Adaptive learning rate methods compute per-parameter learning rates to make optimization less sensitive to the choice of global learning rate. `adam` is a popular adaptive learning rate optimizer.\n* Certain MXNet optimizers like `Signum` and Large Batch SGD are well suited for large scale distributed training as they consider challenges specific these tasks.\n* MXNet also implements optimizers from active learning like `FTML`, `FTRL`, and optimizers for bayesian learning like `SGLD`.\n* Finally, it is easy to create a custom optimizer by following the patterns in the source code implementation for the optimizers that already exist in MXNet.\n\n## Next Steps\nWhile optimization and optimizers play a significant role in deep learning model training, there are still other important components to model training. Here are a few suggestions about where to look next.\n* The [trainer API](../../../api/gluon/trainer.rst) and [guide](../gluon/training/trainer.ipynb) have information about how to construct the trainer that encapsulate the optimizers and will actually be used in your model training loop.\n* Check out the guide to MXNet gluon [Loss functions](../gluon/loss/loss.ipynb) and [custom losses](../gluon/loss/custom-loss.ipynb) to learn about the loss functions optimized by these optimizers, see what loss functions are already implemented in MXNet and understand how to write your own custom loss functions.\n* Take a look at the [guide to parameter initialization](../gluon/blocks/init.ipynb) in MXNet to learn about what initialization schemes are already implemented, and how to implement your custom initialization schemes.\n* Also check out the [autograd guide](../autograd/index.ipynb) to learn about automatic differentiation and how gradients are automatically computed in MXNet.\n* Make sure to take a look at the [guide to scheduling learning rates](../gluon/training/learning_rates/learning_rate_schedules.ipynb) to learn how to create learning rate schedules to supercharge the convergence of your optimizer.\n* Finally take a look at the [KVStore API](../kvstore/index.ipynb) to learn how parameter values are synchronized over multiple devices.\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/packages/viz/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nVisualization\n=============\n\n.. container:: cards\n\n   .. card::\n      :title: How to Visualize Neural Networks as Computation Graph\n      :link: https://mxnet.apache.org/api/faq/visualize_graph\n\n      A demonstration how to use ``mx.viz.plot_network`` for visualizing your neural networks.\n\nReferences\n----------\n\n- `mxnet.viz </api/python/docs/api/mxnet/visualization/index.html>`_\n\n.. toctree::\n   :hidden:\n\n   Visualize networks <https://mxnet.apache.org/api/faq/visualize_graph>\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/performance/backend/amp.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Using AMP: Automatic Mixed Precision\n\nTraining Deep Learning networks is a very computationally intensive task. Novel model architectures tend to have increasing number of layers and parameters, which slows down training. Fortunately, new generations of training hardware as well as software optimizations, make it a feasible task.\n\nHowever, where most of the (both hardware and software) optimization opportunities exists is in exploiting lower precision (like FP16) to, for example, utilize Tensor Cores available on new Volta and Turing GPUs. While training in FP16 showed great success in image classification tasks, other more complicated neural networks typically stayed in FP32 due to difficulties in applying the FP16 training guidelines.\n\nThat is where AMP (Automatic Mixed Precision) comes into play. It automatically applies the guidelines of FP16 training, using FP16 precision where it provides the most benefit, while conservatively keeping in full FP32 precision operations unsafe to do in FP16.\n\nThis tutorial shows how to get started with mixed precision training using AMP for MXNet. As an example of a network we will use SSD network from GluonCV.\n\n## Data loader and helper functions\n\nFor demonstration purposes we will use synthetic data loader.\n\n\n```{.python .input}\nimport os\nimport logging\nimport warnings\nimport time\nimport numpy as np\nimport mxnet as mx\nimport mxnet.gluon as gluon\nfrom mxnet import autograd\nimport gluoncv as gcv\nfrom gluoncv.model_zoo import get_model\n\ndata_shape = 512\nbatch_size = 8\nlr = 0.001\nwd = 0.0005\nmomentum = 0.9\n\n# training devices\ndevice = [mx.gpu(0)]\n\n# set up logger\nlogging.basicConfig()\nlogger = logging.getLogger()\nlogger.setLevel(logging.INFO)\n\nce_metric = mx.metric.Loss('CrossEntropy')\nsmoothl1_metric = mx.metric.Loss('SmoothL1')\n```\n\n\n```{.python .input}\nclass SyntheticDataLoader(object):\n    def __init__(self, data_shape, batch_size):\n        super(SyntheticDataLoader, self).__init__()\n        self.counter = 0\n        self.epoch_size = 200\n        shape = (batch_size, 3, data_shape, data_shape)\n        cls_targets_shape = (batch_size, 6132)\n        box_targets_shape = (batch_size, 6132, 4)\n        self.data = mx.np.random.uniform(-1, 1, size=shape, device=mx.cpu_pinned())\n        self.cls_targets = mx.np.random.uniform(0, 1, size=cls_targets_shape, device=mx.cpu_pinned())\n        self.box_targets = mx.np.random.uniform(0, 1, size=box_targets_shape, device=mx.cpu_pinned())\n    \n    def next(self):\n        if self.counter >= self.epoch_size:\n            self.counter = self.counter % self.epoch_size\n            raise StopIteration\n        self.counter += 1\n        return [self.data, self.cls_targets, self.box_targets]\n    \n    __next__ = next\n    \n    def __iter__(self):\n        return self\n    \ntrain_data = SyntheticDataLoader(data_shape, batch_size)\n```\n\n\n```{.python .input}\ndef get_network():\n    # SSD with RN50 backbone\n    net_name = 'ssd_512_resnet50_v1_coco'\n    with warnings.catch_warnings(record=True) as w:\n        warnings.simplefilter(\"ignore\")\n        net = get_model(net_name, pretrained_base=True, norm_layer=gluon.nn.BatchNorm)\n        net.initialize()\n        net.reset_device(device)\n\n    return net\n```\n\n## Training in FP32\n\nFirst, let us create the network.\n\n\n```{.python .input}\nnet = get_network()\nnet.hybridize(static_alloc=True, static_shape=True)\n```\n\n\n\nNext, we need to create a Gluon Trainer.\n\n\n```{.python .input}\ntrainer = gluon.Trainer(\n    net.collect_params(), 'sgd',\n    {'learning_rate': lr, 'wd': wd, 'momentum': momentum})\n```\n\n\n```{.python .input}\nmbox_loss = gcv.loss.SSDMultiBoxLoss()\n\nfor epoch in range(1):\n    ce_metric.reset()\n    smoothl1_metric.reset()\n    tic = time.time()\n    btic = time.time()\n\n    for i, batch in enumerate(train_data):\n        batch_size = batch[0].shape[0]\n        data = gluon.utils.split_and_load(batch[0], ctx_list=device, batch_axis=0)\n        cls_targets = gluon.utils.split_and_load(batch[1], ctx_list=device, batch_axis=0)\n        box_targets = gluon.utils.split_and_load(batch[2], ctx_list=device, batch_axis=0)\n        with autograd.record():\n            cls_preds = []\n            box_preds = []\n            for x in data:\n                cls_pred, box_pred, _ = net(x)\n                cls_preds.append(cls_pred)\n                box_preds.append(box_pred)\n            sum_loss, cls_loss, box_loss = mbox_loss(\n                cls_preds, box_preds, cls_targets, box_targets)\n            autograd.backward(sum_loss)\n        trainer.step(1)\n        ce_metric.update(0, [l * batch_size for l in cls_loss])\n        smoothl1_metric.update(0, [l * batch_size for l in box_loss])\n        if not (i + 1) % 50:\n            name1, loss1 = ce_metric.get()\n            name2, loss2 = smoothl1_metric.get()\n            logger.info('[Epoch {}][Batch {}], Speed: {:.3f} samples/sec, {}={:.3f}, {}={:.3f}'.format(\n                epoch, i, batch_size/(time.time()-btic), name1, loss1, name2, loss2))\n        btic = time.time()\n```\n\noutput \n\n```text\nINFO:root:[Epoch 0][Batch 49], Speed: 58.105 samples/sec, CrossEntropy=1.190, SmoothL1=0.688\nINFO:root:[Epoch 0][Batch 99], Speed: 58.683 samples/sec, CrossEntropy=0.693, SmoothL1=0.536\nINFO:root:[Epoch 0][Batch 149], Speed: 58.915 samples/sec, CrossEntropy=0.500, SmoothL1=0.453\nINFO:root:[Epoch 0][Batch 199], Speed: 58.422 samples/sec, CrossEntropy=0.396, SmoothL1=0.399\n```\n\n## Training with AMP\n\n### AMP initialization\n\nIn order to start using AMP, we need to import and initialize it. This has to happen before we create the network.\n\n\n```{.python .input}\nfrom mxnet import amp\n\namp.init()\n```\noutput:\n```text\nINFO:root:Using AMP\n```\n\n\nAfter that, we can create the network exactly the same way we did in FP32 training.\n\n\n```{.python .input}\nnet = get_network()\nnet.hybridize(static_alloc=True, static_shape=True)\n```\n\nFor some models that may be enough to start training in mixed precision, but the full FP16 recipe recommends using dynamic loss scaling to guard against over- and underflows of FP16 values. Therefore, as a next step, we create a trainer and initialize it with support for AMP's dynamic loss scaling. Currently, support for dynamic loss scaling is limited to trainers created with `update_on_kvstore=False` option, and so we add it to our trainer initialization.\n\n\n```{.python .input}\ntrainer = gluon.Trainer(\n    net.collect_params(), 'sgd',\n    {'learning_rate': lr, 'wd': wd, 'momentum': momentum},\n    update_on_kvstore=False)\n\namp.init_trainer(trainer)\n```\n\n### Dynamic loss scaling in the training loop\n\nThe last step is to apply the dynamic loss scaling during the training loop and . We can achieve that using the `amp.scale_loss` function.\n\n\n```{.python .input}\nmbox_loss = gcv.loss.SSDMultiBoxLoss()\n\nfor epoch in range(1):\n    ce_metric.reset()\n    smoothl1_metric.reset()\n    tic = time.time()\n    btic = time.time()\n\n    for i, batch in enumerate(train_data):\n        batch_size = batch[0].shape[0]\n        data = gluon.utils.split_and_load(batch[0], ctx_list=device, batch_axis=0)\n        cls_targets = gluon.utils.split_and_load(batch[1], ctx_list=device, batch_axis=0)\n        box_targets = gluon.utils.split_and_load(batch[2], ctx_list=device, batch_axis=0)\n        with autograd.record():\n            cls_preds = []\n            box_preds = []\n            for x in data:\n                cls_pred, box_pred, _ = net(x)\n                cls_preds.append(cls_pred)\n                box_preds.append(box_pred)\n            sum_loss, cls_loss, box_loss = mbox_loss(\n                cls_preds, box_preds, cls_targets, box_targets)\n            with amp.scale_loss(sum_loss, trainer) as scaled_loss:\n                autograd.backward(scaled_loss)\n        trainer.step(1)\n        ce_metric.update(0, [l * batch_size for l in cls_loss])\n        smoothl1_metric.update(0, [l * batch_size for l in box_loss])\n        if not (i + 1) % 50:\n            name1, loss1 = ce_metric.get()\n            name2, loss2 = smoothl1_metric.get()\n            logger.info('[Epoch {}][Batch {}], Speed: {:.3f} samples/sec, {}={:.3f}, {}={:.3f}'.format(\n                epoch, i, batch_size/(time.time()-btic), name1, loss1, name2, loss2))\n        btic = time.time()\n```\n\noutput\n\n```bash\nINFO:root:[Epoch 0][Batch 49], Speed: 93.585 samples/sec, CrossEntropy=1.166, SmoothL1=0.684\nINFO:root:[Epoch 0][Batch 99], Speed: 93.773 samples/sec, CrossEntropy=0.682, SmoothL1=0.533\nINFO:root:[Epoch 0][Batch 149], Speed: 93.399 samples/sec, CrossEntropy=0.493, SmoothL1=0.451\nINFO:root:[Epoch 0][Batch 199], Speed: 93.674 samples/sec, CrossEntropy=0.391, SmoothL1=0.397\n```\n\nWe got 60% speed increase from 3 additional lines of code!\n\n## Inference with AMP\n\nTo do inference with mixed precision for a trained model in FP32, you can use the conversion API `amp.convert_hybrid_block` for gluon models. The conversion APIs will take the FP32 model as input and will return a mixed precision model, which can be used to run inference.\nBelow, we demonstrate for a gluon model:\n- Conversion from FP32 model to mixed precision model.\n- Run inference on the mixed precision model.\n\n```{.python .input}\nwith mx.Context(mx.gpu(0)):\n    # Below is an example of converting a gluon hybrid block to a mixed precision block\n    with warnings.catch_warnings(record=True) as w:\n        warnings.simplefilter(\"ignore\")\n        model = get_model(\"resnet50_v1\")\n        model.initialize(device=mx.current_device())\n        model.hybridize()\n        model(mx.np.zeros((1, 3, 224, 224)))\n        converted_model = amp.convert_hybrid_block(model)\n\n    # Run dummy inference with the converted gluon model\n    result = converted_model.forward(mx.np.random.uniform(size=(1, 3, 224, 224),\n                                                          dtype=np.float32))\n\n    print(\"Conversion and Inference completed successfully\")\n```\n\nYou can also customize the operators to run in FP16 versus the operator to run in FP32 or to conditionally run in FP32.\nAlso, you can force cast the params wherever possible to FP16. \n\n## Current limitations of AMP\n\n- AMP's dynamic loss scaling currently supports only Gluon trainer with `update_on_kvstore=False` option set\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/performance/backend/dnnl/dnnl_quantization.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# oneDNN Quantization\n\n## Introduction\n\nAfter successful model building and achieving desired accuracy on the test data, often the next step is to optimize inference to deploy the model to production. One of the key features of usable model is to have as small latency as possible to be able to provide services to large number of customers simultaneously. In addition to customer satisfaction, with well optimized model, hardware load is reduced which also reduces energy costs needed to perform inference.\n\nTwo main types of software optimizations can be characterized as:\n- memory-bound optimizations - main objective of these optimizations is to reduce the amount of memory operations (reads and writes) - it is done by e.g. chaining operations which can be performed one after another immediately, where input of every subsequent operation is the output of the previous one (example: ReLU activation after convolution),\n- compute-bound optimizations - these optimizations are mainly made on operations which require large number of CPU cycles to complete, like FullyConnected and Convolution. One of the methods to speedup compute-bound operations is to lower computation precision - this type of optimization is called quantization.\n\nIn version 2.0 of the Apache MXNet (incubating) GluonAPI2.0 replaced Symbolic API known from versions 1.x, thus there are some differences between API to perform graph fusion and quantization.\n\n## Operator Fusion\n\nModels are often represented as a directed graph of operations (represented by nodes) and data flow (represented as edges). This way of visualizing helps a lot when searching for common patterns in whole model which can be optimized by fusion. Example:\n![base_model](https://github.com/dmlc/web-data/blob/main/mxnet/tutorials/onednn/quantization_2_0/sample_net.png?raw=true)\n\n\nThe simplest way to explain what fusion is and how it works is to present an example. Image above depicts a sequence of popular operations taken from ResNet architecture. This type of architecture is built with many similar blocks called residual blocks. Some possible fusion patterns are:\n\n- Conv2D + BatchNorm => Fusing BatchNorm with Convolution can be performed by modifing weights and bias of Convolution - this way BatchNorm is completely contained within Convolution which makes BatchNorm zero time operation. Only cost of fusing is time needed to prepare weights and bias in Convolution based on BatchNorm parameters.\n- Conv2D + ReLU => this type of fusion is very popular also with other layers (e.g. FullyConnected + Activation). It is very simple idea where before writing data to output, activation is performed on that data. Main benefit of this fusion is that, there is no need to read and write back data in other layer only to perform simple activation function.\n- Conv2D + Add => even simpler idea than the previous ones - instead of overwriting the output memory, results are added to it. In the simplest terms: `out_mem = conv_result` is replaced by `out_mem += conv_result`.\n\nAbove examples are presented as atomic ones, but often they can be combined together, thus two patterns that can be fused in above example are:\n- Conv2D + BatchNorm + ReLU\n- Conv2D + BatchNorm + Add + ReLU\n\nAfter fusing all patterns, computational graph will be changed to the following one:\n![fused_fp32_model](https://github.com/dmlc/web-data/blob/main/mxnet/tutorials/onednn/quantization_2_0/fused_f32.png?raw=true)\n\n\n\n### Operator fusion in MXNet\nSince the version 1.6 of MXNet built with oneDNN support, operator fusion had been enabled by default for executing model with Module API, however in version 2.0 it has been decided to remove setting this feature by environment flag and replace it by aware user API call.\n\nTo fuse model in MXNet 2.0 there are two requirements:\n- the model must be defined as a subclass of HybridBlock or Symbol,\n- the model must have specific operator patterns which can be fused.\n\nAs an example we define sample block taken from ResNet architecture:\n\n```\nimport mxnet as mx\nfrom mxnet.gluon import nn\n\nclass SampleBlock(nn.HybridBlock):\n    def __init__(self):\n        super(SampleBlock, self).__init__()\n        self.conv1 = nn.Conv2D(channels=64, kernel_size=3, strides=1, padding=1,\n                               use_bias=False, in_channels=64)\n        self.bn1 = nn.BatchNorm()\n        self.conv2 = nn.Conv2D(channels=64, kernel_size=3, strides=1, padding=1,\n                               use_bias=False, in_channels=64)\n        self.bn2 = nn.BatchNorm()\n\n    def forward(self, x):\n        out = mx.npx.activation(self.bn1(self.conv1(x)), 'relu')\n        out = self.bn2(self.conv2(out))\n        out = mx.npx.activation(out + x, 'relu')\n        return out\n\nnet = SampleBlock()\nnet.initialize()\n\ndata = mx.np.zeros(shape=(1,64,224,224))\n# run fusion\nnet.optimize_for(data, backend='ONEDNN')\n\n# We can check fusion by plotting current symbol of our optimized network\nsym, _ = net.export(None)\ngraph = mx.viz.plot_network(sym, save_format='png')\ngraph.view()\n```\nBoth HybridBlock and Symbol classes provide API to easily run fusion of operators. Single line of code is enabling fusion passes on model:\n```\nnet.optimize_for(data, backend='ONEDNN')\n```\n\n*optimize_for* function is available also as Symbol class method. Example call to this API is shown below. Notice that Symbol’s *optimize_for* method is not done in-place, so assigning it to a new variable is required:\n\n```\noptimized_symbol = sym.optimize_for(backend='ONEDNN')\n```\n\nFor the above model definition in a naive benchmark with artificial data, we can gain up to *10.8x speedup* without any accuracy loss on our testing machine with Intel(R) Xeon(R) Platinum 8375C CPU.\n\n\n## Quantization\n\nAs mentioned in the introduction, precision reduction is another very popular method of improving performance of workloads and, what is important, in most cases is combined together with operator fusion which improves performance even more. In training precision reduction utilizes 16 bit data types like bfloat or float16, but for inference great results can be achieved using int8.\n\nModel quantization helps on both memory-bound and compute-bound operations. In quantized model IO operations are reduced as int8 data type is 4x smaller than float32, and also computational throughput is increased as more data can be SIMD'ed. On modern Intel architectures using int8 data type can bring even more speedup by utilizing special VNNI instruction set.\n\n![before_quant](https://github.com/dmlc/web-data/blob/main/mxnet/tutorials/onednn/quantization_2_0/before_quant.png?raw=true)\n\nFirstly quantization performs operator fusion on floating-point model as mentioned in paragraph earlier. Next, all operators which support int8 data type are marked as quantized and if needed additional operators are injected into graph surrounding quantizable operator - the goal of this additional operators is to quantize, dequantize or requantize data to keep data type between operators compatible.\n\n![quant_not_calib](https://github.com/dmlc/web-data/blob/main/mxnet/tutorials/onednn/quantization_2_0/quant_not_calib.png?raw=true)\n\n\nAfter injection step it is important to perform calibration of the model, however this step is optional. Quantizing without calibration is not recommended in terms of performance. It will result in calculating data minimum and maximum in quantize and requantize nodes during each inference pass. Calibrating a model greatly improves performance as minimum and maximum values are collected offline and are saved inside node - this way there is no need to search for these values during inference pass.\n\n![quant_calib](https://github.com/dmlc/web-data/blob/main/mxnet/tutorials/onednn/quantization_2_0/quant_calib.png?raw=true)\n\n\nCurrently, there are three supported calibration methods:\n- naive — min/max values from the calibration run,\n- entropy — uses KL divergence to determine the best symmetrical quantization thresholds for a given histogram of values,\n- custom — uses user-defined CalibrationCollector to control the calibration process.\n\nLast stage of quantization flow is to perform additional operator fusion. Second fusion is about merging requantize and dequantize operators into preceding node - oneDNN kernels can perform needed scaling before writing result to output which results in model execution speed-up. Notice that last Convolution does not need minimum and maximum values as it is not requantizing int32 to int8, but dequantizing directly to float32 and scale is calculated basing on minimum and maximum of input and weights.\n\n![quant_calib_fused](https://github.com/dmlc/web-data/blob/main/mxnet/tutorials/onednn/quantization_2_0/quant_calib_fused.png?raw=true)\n\nIn MXNet 2.0, the quantization procedure has been adjusted to work well with Gluon models since it is the main API now. The goal was to allow the user to quantize fp32 HybridBlock model in just a few lines of code.\n\n### Quantization in MXNet\n\nAs an example of a quantization procedure, pretrained *resnet50_v1* from *model_zoo.vision* package can be used. To get it simply run the following code:\n\n```\nimport mxnet as mx\nfrom mxnet.gluon.model_zoo.vision import resnet50_v1\n\nnet = resnet50_v1(pretrained=True)\n```\n\nNow, to get a ready-to-deploy quantized model two steps are required:\n\n1. Prepare data loader with calibration data - this data will be used as input to the network. All necessary layers will be observed with layer collector to calculate minimum and maximum value of that layer. This flow is internal mechanism and all what user needs to do is to provide data loader.\n2. Call `quantize_net` function from `contrib.quantize` package - both operator fusion calls will be called inside this API.\n\n```\ncalib_data_loader = mx.gluon.data.DataLoader(dummy_data, batch_size=batch_size)\nqnet = quantize_net(net, calib_mode='naive', calib_data=calib_data_loader)\n```\n\nFollowing function, which calculates total inference time on the model with an artificial data, can be used to compare the performance:\n\n```\ndef benchmark_net(net, batch_size=32, batches=100, warmup_batches=5):\n  import time\n  data = mx.np.random.uniform(-1.0, 1.0, (batch_size, 3, 224, 224))\n\n  for i in range(batches + warmup_batches):\n    if i == warmup_batches:\n      tic = time.time()\n    out = net(data)\n    out.wait_to_read()\n\n  total_time = time.time() - tic\n  return total_time\n```\n\n\nComparing fused float32 network to quantized network on Intel(R) Xeon(R) Platinum 8375C CPU shows *4.2x speedup* - measurment was done on 32 cores and this machine utilizes VNNI instruction set.\n\n\nThe other aspect of lowering the precision of a model is a difference in its accuracy. We will check that on previously tested resnet50_v1 with ImageNet dataset. To run this example you will need ImageNet dataset prepared with this tutorial and stored in path_to_imagenet. Let’s compare top1 and top5 accuracy of standard fp32 model with quantized int8 model calibrated using naive and entropy calibration mode. We will use only 10 batches of the validation dataset to calibrate quantized model.\n\n```\nimport mxnet as mx\nfrom mxnet.gluon.model_zoo.vision import resnet50_v1\nfrom mxnet.gluon.data.vision import transforms\nfrom mxnet.contrib.quantization import quantize_net\n\ndef test_accuracy(net, data_loader, description):\n  acc_top1 = mx.gluon.metric.Accuracy()\n  acc_top5 = mx.gluon.metric.TopKAccuracy(5)\n  count = 0\n  tic = time.time()\n  for x, label in data_loader:\n    count += 1\n    output = net(x)\n    acc_top1.update(label, output)\n    acc_top5.update(label, output)\n  time_spend = time.time() - tic\n  _, top1 = acc_top1.get()\n  _, top5 = acc_top5.get()\n  print('{:12} Top1 Accuracy: {:.4f} Top5 Accuracy: {:.4f} from {:4} batches in {:8.2f}s'\n        .format(description, top1, top5, count, time_spend))\n\nrgb_mean = (0.485, 0.456, 0.406)\nrgb_std = (0.229, 0.224, 0.225)\nbatch_size = 64\n\ndataset = mx.gluon.data.vision.ImageRecordDataset('path_to_imagenet/val.rec')\ntransformer = transforms.Compose([transforms.Resize(256),\n                                  transforms.CenterCrop(224),\n                                  transforms.ToTensor(),\n                                  transforms.Normalize(mean=rgb_mean, std=rgb_std)])\nval_data = mx.gluon.data.DataLoader(dataset.transform_first(transformer), batch_size, shuffle=True)\n\nnet = resnet50_v1(pretrained=True)\nnet.hybridize(static_alloc=True, static_shape=True)\ntest_accuracy(net, val_data, \"FP32\")\n\ndummy_data = mx.np.random.uniform(-1.0, 1.0, (batch_size, 3, 224, 224))\nnet.optimize_for(dummy_data, backend='ONEDNN', static_alloc=True, static_shape=True)\ntest_accuracy(net, val_data, \"FP32 fused\")\n\nqnet = quantize_net(net, calib_mode='naive', calib_data=val_data, num_calib_batches=10)\nqnet.hybridize(static_alloc=True, static_shape=True)\ntest_accuracy(qnet, val_data, 'INT8 Naive')\n\nqnet = quantize_net(net, calib_mode='entropy', calib_data=val_data, num_calib_batches=10)\nqnet.hybridize(static_alloc=True, static_shape=True)\ntest_accuracy(qnet, val_data, 'INT8 Entropy')\n```\n\n#### Output:\n> ``FP32         Top1 Accuracy: 0.7636 Top5 Accuracy: 0.9309 from 782 batches in 1560.97s``  \n> ``FP32 fused   Top1 Accuracy: 0.7636 Top5 Accuracy: 0.9309 from 782 batches in  281.03s``  \n> ``INT8 Naive   Top1 Accuracy: 0.7631 Top5 Accuracy: 0.9309 from 782 batches in  184.87s``  \n> ``INT8 Entropy Top1 Accuracy: 0.7617 Top5 Accuracy: 0.9298 from 782 batches in  185.23s``  \n\n\nWith quantized model there is a tiny accuracy drop, however this is the cost of great performance optimization and memory footprint reduction. The difference between calibration methods is dependent on the model itself, used activation layers and the size of calibration data.\n\n### Custom layer collectors and calibrating the model\nIn MXNet 2.0 new interface for creating custom calibration collector has been added. Main goal of this interface is to give the user as much flexibility as possible in almost every step of quantization. Creating own layer collector is pretty easy, however computing effective min/max values can be not a trivial task.\n\nLayer collectors are responsible for collecting statistics of each node in the graph — it means that the input/output data of every operator executed can be observed. Collector utilizes the register_op_hook method of HybridBlock class.\n\nCustom layer collector has to inherit from the CalibrationCollector class, which is provided in `contrib.quantization` package. This inheritance allows API to be consistent. Below is an example implementation of CalibrationCollector:\n\n```\nclass ExampleNaiveCollector(CalibrationCollector):\n  \"\"\"Saves layer output min and max values in a dict with layer names as keys.\n  The collected min and max values will be directly used as thresholds for quantization.\n  \"\"\"\n  def __init__(self, logger=None):\n    # important! initialize base class attributes\n    super(ExampleNaiveCollector, self).__init__()\n    self.logger = logger\n\ndef collect(self, name, op_name, arr):\n  \"\"\"Callback function for collecting min and max values from an NDArray.\"\"\"\n  if name not in self.include_layers: # include_layers is populated by quantization API\n    return\n  arr = arr.copyto(cpu()).asnumpy()\n\n  min_range = np.min(arr)\n  max_range = np.max(arr)\n\n  if name in self.min_max_dict: # min_max_dict is by default empty dict\n    cur_min_max = self.min_max_dict[name]\n    self.min_max_dict[name] = (min(cur_min_max[0], min_range),\n    max(cur_min_max[1], max_range))\n  else:\n    self.min_max_dict[name] = (min_range, max_range)\n\n  if self.logger:\n    self.logger.debug(\"Collecting layer %s min_range=%f, max_range=%f\"\n                       % (name, min_range, max_range))\n\ndef post_collect(self):\n  # we're using min_max_dict and don't process any collected statistics so we don't\n  # need to override this function, however we are doing this for the sake of this article\n  return self.min_max_dict\n```\nQuantization API ‘injects’ names of nodes which require calibration into the include_layers attribute of custom collector — list of included layers allows to avoid unnecessary collecting on nodes which are not relevant in terms of quantization. Using this attribute is fully optional and user can implement his own logic.\n\nAfter collecting all statistic data post_collect function is called. In post_collect additional processing logic can be implemented, but it must return dictionary of nodes names as key and tuple of minimum and maximum values which should be used to calculate data scaling factors.\n\n### Example of usage with quantization API:\n```\nfrom mxnet.contrib.quantization import *\nimport logging\nlogging.getLogger().setLevel(logging.DEBUG) \n\n#…\n\ncalib_data_loader = mx.gluon.data.DataLoader(…)\nmy_collector = ExampleNaiveCollector(logger=logging.getLogger())\n\nqnet = quantize_net(net, calib_mode='custom', calib_data=calib_data_loader, LayerOutputCollector=my_collector)\n```\n\n## Performance and accuracy results\n\n### Performance\nPerformance results of CV models. Chart presents three different runs: base float32 model without optimizations, fused float32 model with optimizations and quantized model.\n![performance](https://github.com/dmlc/web-data/blob/main/mxnet/tutorials/onednn/quantization_2_0/speedup.png?raw=true)\n**Figure 1.**  Relative Inference Performance (img/sec) for Batch Size 128\n\n### Accuracy\nAccuracy results of CV models. Chart presents three different runs: base float32 model without optimizations, fused float32 model with optimizations and fused quantized model.\n![accuracy](https://github.com/dmlc/web-data/blob/main/mxnet/tutorials/onednn/quantization_2_0/accuracy.png?raw=true)\n**Figure 2.**  ImageNet(ILSVRC2012) TOP1 validation accuracy\n\n## Notes\nAccuracy and speedup tested on:  \n- AWS c6i.16xlarge EC2 instance with Ubuntu 20.04 LTS (ami-0558cee5b20db1f9c)  \n- MXNet SHA: 9fa75b470b8f0238a98635f20f5af941feb60929 / oneDNN v2.6 SHA: 52b5f107dd9cf10910aaa19cb47f3abf9b349815  \n- with following enviroment variables were set: ``OMP_NUM_THREADS=32 OMP_PROC_BIND=TRUE OMP_PLACES={0}:32:1`` (by [benchmark/python/dnnl/run.sh](https://github.com/apache/incubator-mxnet/blob/102388a0557c530741ed8e9b31296416a1c23925/benchmark/python/dnnl/run.sh))  \n"
  },
  {
    "path": "docs/python_docs/python/tutorials/performance/backend/dnnl/dnnl_quantization_inc.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Improving accuracy with Intel® Neural Compressor\n\nThe accuracy of a model can decrease as a result of quantization. When the accuracy drop is significant, we can try to manually find a better quantization configuration (exclude some layers, try different calibration methods, etc.), but for bigger models this might prove to be a difficult and time consuming task. [Intel® Neural Compressor](https://github.com/intel/neural-compressor) (INC) tries to automate this process using several tuning heuristics, which aim to find the quantization configuration that satisfies the specified accuracy requirement.\n\n**NOTE:**\n\nMost tuning strategies will try different configurations on an evaluation dataset in order to find out how each layer affects the accuracy of the model. This means that for larger models, it may take a long time to find a solution (as the tuning space is usually larger and the evaluation itself takes longer).\n\n## Installation and Prerequisites\n\n- Install MXNet with oneDNN enabled as described in the [Get started](https://mxnet.apache.org/versions/master/get_started?platform=linux&language=python&processor=cpu&environ=pip&). (Until the 2.0 release you can use the nightly build version: `pip install --pre mxnet -f https://dist.mxnet.io/python`)\n\n- Install Intel® Neural Compressor:\n\n  Use one of the commands below to install INC (supported python versions are: 3.6, 3.7, 3.8, 3.9):\n\n  ```bash\n  # install stable version from pip\n  pip install neural-compressor\n\n  # install nightly version from pip\n  pip install -i https://test.pypi.org/simple/ neural-compressor\n\n  # install stable version from conda\n  conda install neural-compressor -c conda-forge -c intel\n  ```\n  If you get into trouble with dependencies on `cv2` library you can run: `apt-get update && apt-get install -y python3-opencv`\n\n## Configuration file\n\nQuantization tuning process can be customized in the yaml configuration file. Below is a simple example:\n\n```yaml\n# cnn.yaml\n\nversion: 1.0\n\nmodel:\n  name: cnn\n  framework: mxnet\n\nquantization:\n  calibration:\n    sampling_size: 160 # number of samples for calibration\n\ntuning:\n  strategy:\n    name: basic\n  accuracy_criterion:\n    relative: 0.01\n  exit_policy:\n    timeout: 0\n  random_seed: 9527\n```\n\nWe are using the `basic` strategy, but you could also try out different ones. [Here](https://github.com/intel/neural-compressor/blob/master/docs/tuning_strategies.md) you can find a list of strategies available in INC and details of how they work. You can also add your own strategy if the existing ones do not suit your needs.\n\nSince the value of `timeout` in the example above is 0, INC will run until it finds a configuration that satisfies the accuracy criterion and then exit. Depending on the strategy this may not be ideal, as sometimes it would be better to further explore the tuning space to find a superior configuration both in terms of accuracy and speed. To achieve this, we can set a specific `timeout` value, which will tell INC how long (in seconds) it should run.\n\nFor more information about the configuration file, see the [template](https://github.com/intel/neural-compressor/blob/master/neural_compressor/template/ptq.yaml) from the official INC repo. Keep in mind that only the `post training quantization` is currently supported for MXNet.\n\n## Model quantization and tuning\n\nIn general, Intel® Neural Compressor requires 4 elements in order to run:  \n1. Configuration file - like the example above  \n2. Model to be quantized  \n3. Calibration dataloader  \n4. Evaluation function - a function that takes a model as an argument and returns the accuracy it achieves on a certain evaluation dataset. \n\n### Quantizing ResNet\n\nThe [quantization](https://mxnet.apache.org/versions/master/api/python/docs/tutorials/performance/backend/dnnl/dnnl_quantization.html#Quantization) sections described how to quantize ResNet using the native MXNet quantization. This example shows how we can achieve the similar results (with the auto-tuning) using INC.\n\n1. Get the model\n\n```python\nimport logging\nimport mxnet as mx\nfrom mxnet.gluon.model_zoo import vision\n\nlogging.basicConfig()\nlogger = logging.getLogger('logger')\nlogger.setLevel(logging.INFO)\n\nbatch_shape = (1, 3, 224, 224)\nresnet18 = vision.resnet18_v1(pretrained=True)\n```\n\n2. Prepare the dataset:\n\n```python\nmx.test_utils.download('http://data.mxnet.io/data/val_256_q90.rec', 'data/val_256_q90.rec')\n\nbatch_size = 16\nmean_std = {'mean_r': 123.68, 'mean_g': 116.779, 'mean_b': 103.939,\n            'std_r': 58.393, 'std_g': 57.12, 'std_b': 57.375}\n\ndata = mx.io.ImageRecordIter(path_imgrec='data/val_256_q90.rec',\n                             batch_size=batch_size,\n                             data_shape=batch_shape[1:],\n                             rand_crop=False,\n                             rand_mirror=False,\n                             shuffle=False,\n                             **mean_std)\ndata.batch_size = batch_size\n```\n\n3. Prepare the evaluation function:\n\n```python\neval_samples = batch_size*10\n\ndef eval_func(model):\n    data.reset()\n    metric = mx.metric.Accuracy()\n    for i, batch in enumerate(data):\n        if i * batch_size >= eval_samples:\n            break\n        x = batch.data[0].as_in_context(mx.cpu())\n        label = batch.label[0].as_in_context(mx.cpu())\n        outputs = model.forward(x)\n        metric.update(label, outputs)\n    return metric.get()[1]\n```\n\n4. Run Intel® Neural Compressor:\n\n```python\nfrom neural_compressor.experimental import Quantization\nquantizer = Quantization(\"./cnn.yaml\")\nquantizer.model = resnet18\nquantizer.calib_dataloader = data\nquantizer.eval_func = eval_func\nqnet = quantizer.fit().model\n```\n\nSince this model already achieves good accuracy using native quantization (less than 1% accuracy drop), for the given configuration file, INC will end on the first configuration, quantizing all layers using `naive` calibration mode for each. To see the true potential of INC, we need a model which suffers from a larger accuracy drop after quantization.\n\n### Quantizing ResNet50v2\n\nThis example shows how to use INC to quantize ResNet50 v2. In this case, the native MXNet quantization introduce a huge accuracy drop (70% using `naive` calibration mode) and INC allows to automatically find better solution.\n\nThis is the [INC configuration file](https://github.com/apache/incubator-mxnet/blob/master/example/quantization_inc/resnet50v2_mse.yaml) for this example: \n```yaml\nversion: 1.0\n\nmodel:\n  name: resnet50_v2\n  framework: mxnet\n\nquantization:\n  calibration:\n    sampling_size: 192 # number of samples for calibration\n\ntuning:\n  strategy:\n    name: mse\n  accuracy_criterion:\n    relative: 0.015\n  exit_policy:\n    timeout: 0\n    max_trials: 500\n  random_seed: 9527\n```\n\nIt could be used with script below \n([resnet_mse.py](https://github.com/apache/incubator-mxnet/blob/master/example/quantization_inc/resnet_mse.py))\nto find operator, which caused the most significant accuracy drop and disable it from quantization. \nYou can find description of MSE strategy \n[here](https://github.com/intel/neural-compressor/blob/master/docs/tuning_strategies.md#user-content-mse).\n\n```python\nimport mxnet as mx\nfrom mxnet.gluon.model_zoo.vision import resnet50_v2\nfrom mxnet.gluon.data.vision import transforms\nfrom mxnet.contrib.quantization import quantize_net\n\n# Preparing input data\nrgb_mean = (0.485, 0.456, 0.406)\nrgb_std = (0.229, 0.224, 0.225)\nbatch_size = 64\nnum_calib_batches = 9\n# set proper path to ImageNet data set below\ndataset = mx.gluon.data.vision.ImageRecordDataset('../imagenet/rec/val.rec')\n# Tuning with INC on whole data set takes a lot of time. Therefore, we take only a part of the data set\n# as representative part of it:\ndataset = dataset.take(num_calib_batches * batch_size)\ntransformer = transforms.Compose([transforms.Resize(256),\n                                  transforms.CenterCrop(224),\n                                  transforms.ToTensor(),\n                                  transforms.Normalize(mean=rgb_mean, std=rgb_std)])\n# Note: as input data is used many times during tuning, it is better to have it prepared earlier.\n#       Therefore, lazy parameter for transform_first is set to False.\nval_data = mx.gluon.data.DataLoader(\n    dataset.transform_first(transformer, lazy=False), batch_size, shuffle=False)\nval_data.batch_size = batch_size\n\nnet = resnet50_v2(pretrained=True)\n\ndef eval_func(model):\n  metric = mx.gluon.metric.Accuracy()\n  for x, label in val_data:\n    output = model(x)\n    metric.update(label, output)\n  accuracy = metric.get()[1]\n  return accuracy\n\n\nfrom neural_compressor.experimental import Quantization\nquantizer = Quantization(\"resnet50v2_mse.yaml\")\nquantizer.model = net\nquantizer.calib_dataloader = val_data\nquantizer.eval_func = eval_func\nqnet_inc = quantizer.fit().model\nprint(\"INC finished\")\n# You can save optimized model for the later use:\nqnet_inc.export(\"__quantized_with_inc\")\n# You can see which configuration was applied by INC and which nodes were excluded from quantization,\n# to achieve given accuracy loss against floating point calculation.\nprint(quantizer.strategy.best_qmodel.q_config['quant_cfg'])\n```\n\n#### Results:\nResnet50 v2 model could be prepared to achieve better performance with various calibration and tuning methods.  \nIt is done by \n[resnet_tuning.py](https://github.com/apache/incubator-mxnet/blob/master/example/quantization_inc/resnet_tuning.py) \nscript on a small part of data set to reduce time required for tuning (9 batches). \nLater saved models are validated on a whole data set by \n[resnet_measurement.py](https://github.com/apache/incubator-mxnet/blob/master/example/quantization_inc/resnet_measurement.py)\nscript.\nAccuracy results on the whole validation dataset (782 batches) are shown below.\n\n| Optimization method  | Top 1 accuracy | Top 5 accuracy | Top 1 relative accuracy loss [%] | Top 5 relative accuracy loss [%] | Cost = one-time optimization on 9 batches [s] | Validation time [s] | Speedup |\n|----------------------|-------:|-------:|------:|------:|-------:|--------:|------:|\n| fp32 no optimization | 0.7699 | 0.9340 |  0.00 |  0.00 |   0.00 | 316.50 | 1.0 |\n| fp32 fused           | 0.7699 | 0.9340 |  0.00 |  0.00 |   0.03 | 147.77 | 2.1 |\n| int8 full naive      | 0.2207 | 0.3912 | 71.33 | 58.12 |  11.29 |  45.81 | **6.9** |\n| int8 full entropy    | 0.6933 | 0.8917 |  9.95 |  4.53 |  80.23 |  46.39 | 6.8 |\n| int8 smart naive     | 0.2210 | 0.3905 | 71.29 | 58.19 |  11.15 |  46.02 | 6.9 |\n| int8 smart entropy   | 0.6928 | 0.8910 | 10.01 |  4.60 |  79.75 |  45.98 | 6.9 |\n| int8 INC basic       | 0.7692 | 0.9331 | **0.09** |  0.10 | 266.50 |  48.32 | **6.6** |\n| int8 INC mse         | 0.7692 | 0.9337 | **0.09** |  0.03 | 106.50 |  49.76 | **6.4** |\n| int8 INC mycustom    | 0.7699 | 0.9338 | **0.00** |  0.02 | 370.29 |  70.07 | **4.5** |\n\n\nEnvironment:  \n- Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz (c6i.16xlarge Amazon EC2 instance)  \n- Ubuntu 20.04.4 LTS (GNU/Linux Ubuntu 20.04.4 LTS 5.15.0-1017-aws ami-0558cee5b20db1f9c)  \n- MXNet 2.0.0b20220902 (commit 3a19f0e50d75fedb05eb558a9c835726b57df4cf)  \n- INC 1.13.1  \n- scripts above were run as parameter for [run.sh](https://github.com/apache/incubator-mxnet/blob/master/benchmark/python/dnnl/run.sh) \nscript to properly setup parallel computation parameters.  \n\nFor this model INC `basic`, `mse` and `mycustom` strategies found configurations meeting the 1.5% relative accuracy \nloss criterion. Only the `bayesian` strategy didn't find solution within 500 attempts limit. \nAlthough these results may suggest that the `mse` strategy is the best compromise between time spent\nto find the optimized model and final model performance efficiency, different strategies may give \nbetter results for specific models and tasks. For example for ALBERT model there is no solution \ngiven by build-in INC strategies. For such situation you can create your custom strategy, similar \nto this one: \n[custom_strategy.py](https://github.com/apache/incubator-mxnet/blob/master/example/quantization_inc/custom_strategy.py). \nYou can notice, that the most important thing done by INC\nwas to find the operator, which had the most significant impact on the loss of accuracy and disable it from quantization if needed. \nYou can see below which operator was excluded by `mse` strategy in last print given by \n[resnet_mse.py](https://github.com/apache/incubator-mxnet/blob/master/example/quantization_inc/resnet_mse.py) \nscript:  \n\n{'excluded_symbols': ['**sg_onednn_conv_bn_act_0**'], 'quantized_dtype': 'auto', 'quantize_mode': 'smart', 'quantize_granularity': 'tensor-wise'}\n\n\n## Tips\n- In order to get a solution that generalizes well, evaluate the model (in eval_func) on a representative dataset.\n- With `history.snapshot` file (generated by INC) you can recover any model that was generated during the tuning process:\n  ```python\n  from neural_compressor.utils.utility import recover\n\n  quantized_model = recover(f32_model, 'nc_workspace/<tuning date>/history.snapshot', configuration_idx).model\n  ```\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/performance/backend/dnnl/dnnl_readme.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Install MXNet with oneDNN\n\nA better training and inference performance is expected to be achieved on Intel-Architecture CPUs with MXNet built with [oneDNN](https://github.com/oneapi-src/oneDNN) on multiple operating system, including Linux, Windows and MacOS.\nIn the following sections, you will find build instructions for MXNet with oneDNN on Linux, MacOS and Windows.\n\nThe detailed performance data collected on Intel Xeon CPU with MXNet built with oneDNN can be found [here](https://mxnet.apache.org/api/faq/perf#intel-cpu).\n\n\n<h2 id=\"0\">Contents</h2>\n\n* [1. Linux](#1)\n* [2. MacOS](#2)\n* [3. Windows](#3)\n* [4. Verify MXNet with python](#4)\n* [5. Enable MKL BLAS](#5)\n* [6. Enable graph optimization](#6)\n* [7. Quantization](#7)\n* [8. Support](#8)\n\n<h2 id=\"1\">Linux</h2>\n\n### Prerequisites\n\n```\nsudo apt-get update\nsudo apt-get install -y build-essential git\nsudo apt-get install -y libopenblas-dev liblapack-dev\nsudo apt-get install -y libopencv-dev\nsudo apt-get install -y graphviz\n```\n\n### Clone MXNet sources\n\n```\ngit clone --recursive https://github.com/apache/mxnet.git\ncd mxnet\n```\n\n### Build MXNet with oneDNN\n\nTo achieve better performance, the Intel OpenMP and llvm OpenMP are recommended as below instruction. Otherwise, default GNU OpenMP will be used and you may get the sub-optimal performance. If you don't have the full [MKL](https://software.intel.com/en-us/intel-mkl) library installation, you might use OpenBLAS as the blas library, by setting USE_BLAS=Open.\n\n```\n# build with llvm OpenMP and Intel MKL/OpenBlas\nmkdir build && cd build\ncmake -DUSE_CUDA=OFF -DUSE_ONEDNN=ON -DUSE_OPENMP=ON -DUSE_OPENCV=ON ..\nmake -j $(nproc)\n```\n\n```\n# build with Intel MKL and Intel OpenMP\nmkdir build && cd build\ncmake -DUSE_CUDA=OFF -DUSE_ONEDNN=ON -DUSE_BLAS=mkl ..\nmake -j $(nproc)\n```\n\n```\n# build with openblas and GNU OpenMP (sub-optimal performance)\nmkdir build && cd build\ncmake -DUSE_CUDA=OFF -DUSE_ONEDNN=ON -DUSE_BLAS=Open ..\nmake -j $(nproc)\n```\n\n<h2 id=\"2\">MacOS</h2>\n\n### Prerequisites\n\nInstall the dependencies, required for MXNet, with the following commands:\n\n- [Homebrew](https://brew.sh/)\n- llvm (clang in macOS does not support OpenMP)\n- OpenCV (for computer vision operations)\n\n```\n# Paste this command in Mac terminal to install Homebrew\n/usr/bin/ruby -e \"$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)\"\n\n# install dependency\nbrew update\nbrew install pkg-config\nbrew install graphviz\nbrew tap homebrew/core\nbrew install opencv\nbrew tap homebrew/versions\nbrew install llvm\n```\n\n### Clone MXNet sources\n\n```\ngit clone --recursive https://github.com/apache/mxnet.git\ncd mxnet\n```\n\n### Build MXNet with oneDNN\n\n```\nLIBRARY_PATH=$(brew --prefix llvm)/lib/ make -j $(sysctl -n hw.ncpu) CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ USE_OPENCV=1 USE_OPENMP=1 USE_ONEDNN=1 USE_BLAS=apple\n```\n\n<h2 id=\"3\">Windows</h2>\n\nOn Windows, you can use [Micrsoft Visual Studio 2015](https://www.visualstudio.com/vs/older-downloads/) and [Microsoft Visual Studio 2017](https://www.visualstudio.com/downloads/) to compile MXNet with oneDNN.\n[Micrsoft Visual Studio 2015](https://www.visualstudio.com/vs/older-downloads/) is recommended.\n\n**Visual Studio 2015**\n\nTo build and install MXNet yourself, you need the following dependencies. Install the required dependencies:\n\n1. If [Microsoft Visual Studio 2015](https://www.visualstudio.com/vs/older-downloads/) is not already installed, download and install it. You can download and install the free community edition.\n2. Download and Install [CMake 3](https://cmake.org/files/v3.14/cmake-3.14.0-win64-x64.msi) if it is not already installed.\n3. Download [OpenCV 3](https://sourceforge.net/projects/opencvlibrary/files/3.4.5/opencv-3.4.5-vc14_vc15.exe/download), and unzip the OpenCV package, set the environment variable ```OpenCV_DIR``` to point to the ```OpenCV build directory``` (e.g.,```OpenCV_DIR = C:\\opencv\\build ```). Also, add the OpenCV bin directory (```C:\\opencv\\build\\x64\\vc14\\bin``` for example) to the ``PATH`` variable.\n4. If you have Intel Math Kernel Library (Intel MKL) installed, set ```MKLROOT``` environment variable to point to ```MKL``` directory that contains the ```include``` and ```lib```. If you want to use MKL blas, you should set ```-DUSE_BLAS=mkl``` when cmake. Typically, you can find the directory in ```C:\\Program Files (x86)\\IntelSWTools\\compilers_and_libraries\\windows\\mkl```.\n5. If you don't have the Intel Math Kernel Library (MKL) installed, download and install [OpenBLAS](http://sourceforge.net/projects/openblas/files/v0.2.14/), or build the latest version of OpenBLAS from source. Note that you should also download ```mingw64.dll.zip``` along with openBLAS and add them to PATH.\n6. Set the environment variable ```OpenBLAS_HOME``` to point to the ```OpenBLAS``` directory that contains the ```include``` and ```lib``` directories. Typically, you can find the directory in ```C:\\Downloads\\OpenBLAS\\```.\n\nAfter you have installed all of the required dependencies, build the MXNet source code:\n\n1. Start a Visual Studio command prompt by click windows Start menu>>Visual Studio 2015>>VS2015 X64 Native Tools Command Prompt, and download the MXNet source code from [GitHub](https://github.com/apache/mxnet) by the command:\n```\ngit clone --recursive https://github.com/apache/mxnet.git\ncd C:\\mxent\n```\n2. Enable oneDNN by -DUSE_ONEDNN=1. Use [CMake 3](https://cmake.org/) to create a Visual Studio solution in ```./build```. Make sure to specify the architecture in the\ncommand:\n```\n>mkdir build\n>cd build\n>cmake -G \"Visual Studio 14 Win64\" .. -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=Open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DUSE_ONEDNN=1 -DCMAKE_BUILD_TYPE=Release\n```\n3. Enable oneDNN and Intel MKL as BLAS library by the command:\n```\n>\"C:\\Program Files (x86)\\IntelSWTools\\compilers_and_libraries\\windows\\mkl\\bin\\mklvars.bat\" intel64\n>cmake -G \"Visual Studio 14 Win64\" .. -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=mkl -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DUSE_ONEDNN=1 -DCMAKE_BUILD_TYPE=Release\n```\n4. After the CMake successfully completed, in Visual Studio, open the solution file ```.sln``` and compile it, or compile the MXNet source code by using following command:\n```r\nmsbuild mxnet.sln /p:Configuration=Release;Platform=x64 /maxcpucount\n```\n   These commands produce mxnet library called ```libmxnet.dll``` in the ```./build/Release/``` or ```./build/Debug``` folder. Also ```libmkldnn.dll``` with be in the ```./build/3rdparty/onednn/src/Release/```\n\n5. Make sure that all the dll files used above(such as `libmkldnn.dll`, `libmklml*.dll`, `libiomp5.dll`, `libopenblas*.dll`, etc) are added to the system PATH. For convinence, you can put all of them to ```\\windows\\system32```. Or you will come across `Not Found Dependencies` when loading MXNet.\n\n**Visual Studio 2017**\n\nUser can follow the same steps of Visual Studio 2015 to build MXNET with oneDNN, but change the version related command, for example,```C:\\opencv\\build\\x64\\vc15\\bin``` and build command is as below:\n\n```\n>cmake -G \"Visual Studio 15 Win64\" .. -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=mkl -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DUSE_ONEDNN=1 -DCMAKE_BUILD_TYPE=Release\n\n```\n\n<h2 id=\"4\">Verify MXNet with python</h2>\n\nPreinstall python and some dependent modules:\n```\npip install numpy graphviz\nset PYTHONPATH=[workdir]\\mxnet\\python\n```\nor install mxnet\n```\ncd python\nsudo python setup.py install\npython -c \"import mxnet as mx;print((mx.nd.ones((2, 3))*2).asnumpy());\"\n```\nExpected Output:\n```\n[[ 2.  2.  2.]\n [ 2.  2.  2.]]\n```\n### Verify whether oneDNN works\n\nAfter MXNet is installed, you can verify if oneDNN backend works well with a single Convolution layer.\n```\nfrom mxnet import np\nfrom mxnet.gluon import nn\n\nnum_filter = 32\nkernel = (3, 3)\npad = (1, 1)\nshape = (32, 32, 256, 256)\n\nconv_layer = nn.Conv2D(channels=num_filter, kernel_size=kernel, padding=pad)\nconv_layer.initialize()\n\ndata = np.random.normal(size=shape)\no = conv_layer(data)\nprint(o)\n```\n\nMore detailed debugging and profiling information can be logged by setting the environment variable 'DNNL_VERBOSE':\n```\nexport DNNL_VERBOSE=1\n```\nFor example, by running above code snippet, the following debugging logs providing more insights on oneDNN primitives `convolution` and `reorder`. That includes: Memory layout, infer shape and the time cost of primitive execution.\n```\ndnnl_verbose,info,oneDNN v2.3.2 (commit e2d45252ae9c3e91671339579e3c0f0061f81d49)\ndnnl_verbose,info,cpu,runtime:OpenMP\ndnnl_verbose,info,cpu,isa:Intel AVX-512 with Intel DL Boost\ndnnl_verbose,info,gpu,runtime:none\ndnnl_verbose,info,prim_template:operation,engine,primitive,implementation,prop_kind,memory_descriptors,attributes,auxiliary,problem_desc,exec_time\ndnnl_verbose,exec,cpu,reorder,jit:uni,undef,src_f32::blocked:abcd:f0 dst_f32::blocked:acdb:f0,,,32x32x256x256,8.34912\ndnnl_verbose,exec,cpu,reorder,jit:uni,undef,src_f32::blocked:abcd:f0 dst_f32::blocked:Acdb32a:f0,,,32x32x3x3,0.0229492\ndnnl_verbose,exec,cpu,convolution,brgconv:avx512_core,forward_inference,src_f32::blocked:acdb:f0 wei_f32::blocked:Acdb32a:f0 bia_f32::blocked:a:f0 dst_f32::blocked:acdb:f0,,alg:convolution_direct,mb32_ic32oc32_ih256oh256kh3sh1dh0ph1_iw256ow256kw3sw1dw0pw1,10.5898\n```\n\nYou can find step-by-step guidance to do profiling for oneDNN primitives in [Profiling oneDNN Operators](https://mxnet.apache.org/api/python/docs/tutorials/performance/backend/profiler.html#Profiling-MKLDNN-Operators).\n\n<h2 id=\"5\">Enable MKL BLAS</h2>\n\nWith MKL BLAS, the performace is expected to furtherly improved with variable range depending on the computation load of the models.\nYou can redistribute not only dynamic libraries but also headers, examples and static libraries on accepting the license [Intel Simplified license](https://software.intel.com/en-us/license/intel-simplified-software-license).\nInstalling the full MKL installation enables MKL support for all operators under the linalg namespace.\n\n  1. Download and install the latest full MKL version following instructions on the [intel website.](https://software.intel.com/en-us/mkl) You can also install MKL through [YUM](https://software.intel.com/content/www/us/en/develop/documentation/installation-guide-for-intel-oneapi-toolkits-linux/top/installation/install-using-package-managers/yum-dnf-zypper.html) or [APT](https://software.intel.com/content/www/us/en/develop/documentation/installation-guide-for-intel-oneapi-toolkits-linux/top/installation/install-using-package-managers/apt.html) Repository.\n\n  2. Create and navigate to build directory `mkdir build && cd build`\n\n  3. Run `cmake -DUSE_CUDA=OFF -DUSE_BLAS=mkl ..`\n\n  4. Run `make -j`\n\n  5. Navigate into the python directory\n\n  6. Run `sudo python setup.py install`\n\n### Verify whether MKL works\n\nAfter MXNet is installed, you can verify if MKL BLAS works well with a linear matrix solver.\n\n```\nfrom mxnet import np\ncoeff = np.array([[7, 0], [5, 2]])\ny = np.array([14, 18])\nx = np.linalg.solve(coeff, y)\nprint(x)\n```\n\nYou can get the verbose log output from mkl library by setting environment variable:\n```\nexport MKL_VERBOSE=1\n```\nThen by running above code snippet, you should get the similar output to message below (`SGESV` primitive from MKL was executed). Layout information and primitive execution performance are also demonstrated in the log message.\n```\nmkl-service + Intel(R) MKL: THREADING LAYER: (null)\nmkl-service + Intel(R) MKL: setting Intel(R) MKL to use INTEL OpenMP runtime\nmkl-service + Intel(R) MKL: preloading libiomp5.so runtime\nIntel(R) MKL 2020.0 Update 1 Product build 20200208 for Intel(R) 64 architecture Intel(R) Advanced Vector Extensions 512 (Intel(R) AVX-512) with support of Vector Neural Network Instructions enabled processors, Lnx 2.70GHz lp64 intel_thread\nMKL_VERBOSE SGESV(2,1,0x7f74d4002780,2,0x7f74d4002798,0x7f74d4002790,2,0) 77.58us CNR:OFF Dyn:1 FastMM:1 TID:0  NThr:56\n```\n\n<h2 id=\"6\">Graph optimization</h2>\n\nTo better utilise oneDNN potential, using graph optimizations is recommended. There are few limitations of this feature:\n\n- It works only for inference.\n- Only subclasses of HybridBlock and Symbol can call optimize_for API.\n- This feature will only run on the CPU, even if you're using a GPU-enabled build of MXNet.\n\nIf your use case met above conditions, graph optimizations can be enabled by just simple call `optimize_for` API. Example below:\n```\nfrom mxnet import np\nfrom mxnet.gluon import nn\n\ndata = np.random.normal(size=(32,3,224,224))\n\nnet = nn.HybridSequential()\nnet.add(nn.Conv2D(channels=64, kernel_size=(3,3)))\nnet.add(nn.Activation('relu'))\nnet.initialize()\nprint(\"=\" * 5, \" Not optimized \", \"=\" * 5)\no = net(data)\nprint(o)\n\nnet.optimize_for(data, backend='ONEDNN')\nprint(\"=\" * 5, \" Optimized \", \"=\" * 5)\no = net(data)\nprint(o)\n\n```\n\nAbove code snippet should produce similar output to the following one (printed tensors are omitted) :\n```\n===== Not optimized =====\n[15:05:43] ../src/storage/storage.cc:202: Using Pooled (Naive) StorageManager for CPU\ndnnl_verbose,info,oneDNN v2.3.2 (commit e2d45252ae9c3e91671339579e3c0f0061f81d49)\ndnnl_verbose,info,cpu,runtime:OpenMP\ndnnl_verbose,info,cpu,isa:Intel AVX-512 with AVX512BW, AVX512VL, and AVX512DQ extensions\ndnnl_verbose,info,gpu,runtime:none\ndnnl_verbose,info,prim_template:operation,engine,primitive,implementation,prop_kind,memory_descriptors,attributes,auxiliary,problem_desc,exec_time\ndnnl_verbose,exec,cpu,reorder,jit:uni,undef,src_f32::blocked:abcd:f0 dst_f32::blocked:acdb:f0,,,32x3x224x224,8.87793\ndnnl_verbose,exec,cpu,reorder,jit:uni,undef,src_f32::blocked:abcd:f0 dst_f32::blocked:Acdb64a:f0,,,64x3x3x3,0.00708008\ndnnl_verbose,exec,cpu,convolution,brgconv:avx512_core,forward_inference,src_f32::blocked:acdb:f0 wei_f32::blocked:Acdb64a:f0 bia_f32::blocked:a:f0 dst_f32::blocked:acdb:f0,,alg:convolution_direct,mb32_ic3oc64_ih224oh222kh3sh1dh0ph0_iw224ow222kw3sw1dw0pw0,91.511\ndnnl_verbose,exec,cpu,reorder,jit:uni,undef,src_f32::blocked:abcd:f0 dst_f32::blocked:Acdb64a:f0,,,64x3x3x3,0.00610352\ndnnl_verbose,exec,cpu,eltwise,jit:avx512_common,forward_inference,data_f32::blocked:acdb:f0 diff_undef::undef::f0,,alg:eltwise_relu alpha:0 beta:0,32x64x222x222,85.4392\n===== Optimized =====\ndnnl_verbose,exec,cpu,reorder,jit:uni,undef,src_f32::blocked:Acdb64a:f0 dst_f32::blocked:abcd:f0,,,64x3x3x3,0.00610352\ndnnl_verbose,exec,cpu,reorder,jit:uni,undef,src_f32::blocked:abcd:f0 dst_f32::blocked:Acdb64a:f0,,,64x3x3x3,0.00585938\ndnnl_verbose,exec,cpu,reorder,jit:uni,undef,src_f32::blocked:abcd:f0 dst_f32::blocked:acdb:f0,,,32x3x224x224,3.98999\ndnnl_verbose,exec,cpu,convolution,brgconv:avx512_core,forward_inference,src_f32::blocked:acdb:f0 wei_f32::blocked:Acdb64a:f0 bia_f32::blocked:a:f0 dst_f32::blocked:acdb:f0,attr-post-ops:eltwise_relu:0:1 ,alg:convolution_direct,mb32_ic3oc64_ih224oh222kh3sh1dh0ph0_iw224ow222kw3sw1dw0pw0,20.46\n```\nAfter optimization of Convolution + ReLU oneDNN executes both operations within single convolution primitive.\n\n<h2 id=\"7\">Quantization and Inference with INT8</h2>\n\nMXNet built with oneDNN brings outstanding performance improvement on quantization and inference with INT8 Intel CPU Platform on Intel Xeon Scalable Platform.\n\n- [CNN Quantization Examples](https://github.com/apache/mxnet/tree/master/example/quantization).\n\n- [Model Quantization for Production-Level Neural Network Inference](https://cwiki.apache.org/confluence/display/MXNET/MXNet+Graph+Optimization+and+Quantization+based+on+subgraph+and+MKL-DNN).\n\n<h2 id=\"8\">Next Steps and Support</h2>\n\n- For questions or support specific to MKL, visit the [Intel MKL](https://software.intel.com/en-us/mkl) website.\n\n- For questions or support specific to oneDNN, visit the [oneDNN](https://github.com/oneapi-src/oneDNN) website.\n\n- If you find bugs, please open an issue on GitHub for [MXNet with MKL](https://github.com/apache/mxnet/labels/MKL) or [MXNet with oneDNN](https://github.com/apache/mxnet/labels/MKLDNN).\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/performance/backend/dnnl/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\noneDNN\n=============\n\n.. container:: cards\n\n   .. card::\n      :title: oneDNN Installation and Verification\n      :link: dnnl_readme.html\n\n      A guide on using oneDNN with MXNet.\n\n   .. card::\n      :title: oneDNN Quantization\n      :link: dnnl_quantization.html\n\n      How to perform quantization with oneDNN\n\n   .. card::\n      :title: Intel® Neural Compressor\n      :link: dnnl_quantization_inc.html\n\n      How to improve accuracy of quantization with oneDNN\n\n.. toctree::\n   :hidden:\n   :maxdepth: 1\n   :glob:\n\n   dnnl_readme\n   dnnl_quantization\n   dnnl_quantization_inc"
  },
  {
    "path": "docs/python_docs/python/tutorials/performance/backend/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nAccelerated Backend Tools\n=========================\nThe following tutorials will help you learn how to use backend tools to boost performance.\n\n.. container:: cards\n\n  .. card::\n     :title: oneDNN\n     :link: dnnl/index.html\n\n     How to get the most from your CPU by using oneDNN.\n\n  .. card::\n     :title: TVM\n     :link: tvm.html\n\n     How to use TVM to boost performance.\n\n  .. card::\n     :title: Automatic Mixed Precision (AMP)\n     :link: amp.html\n\n     How to use Automatic Mixed Precision to boost performance.\n\n  .. card::\n     :title: MXNet Operator Profiler\n     :link: profiler.html\n\n     Use the profiler to monitor the performance of individual operators\n..\n\n.. toctree::\n   :hidden:\n   :maxdepth: 1\n\n   dnnl/index\n   tvm\n   profiler\n   amp\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/performance/backend/profiler.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Profiling MXNet Models\n\nIt is often helpful to check the execution time of each operation in a neural network. You can then determine where to focus your effort to speed up model training or inference. In this tutorial, we will learn how to profile MXNet models to measure their running time and memory consumption using the MXNet profiler.\n\n## The incorrect way to profile\n\nIf you have just started to use MXNet, you might be tempted to measure the execution time of your model using Python's `time` module like shown below:\n\n```{.python .input}\nfrom time import time\nfrom mxnet import autograd, np\nimport mxnet as mx\n\nstart = time()\nx = np.random.uniform(size=(2000,2000))\ny = np.dot(x, x)\nprint('Time for matrix multiplication: %f sec\\n' % (time() - start))\n\nstart = time()                                \ny_np = y.asnumpy()                             \nprint('Time for converting to numpy: %f sec' % (time() - start))\n```\n\n**Time for matrix multiplication: 0.005051 sec**<!--notebook-skip-line-->\n\n**Time for converting to numpy: 0.167693 sec**<!--notebook-skip-line-->\n\nFrom the timings above, it seems as if converting to numpy takes lot more time than multiplying two large matrices. That doesn't seem right.\n\nThis is because, in MXNet, all operations are executed asynchronously. So, when `nd.dot(x, x)` returns, the matrix multiplication is not complete, it has only been queued for execution. However, [asnumpy](../../../api/legacy/ndarray/ndarray.rst#mxnet.ndarray.NDArray.asnumpy) has to wait for the result to be calculated in order to convert it to numpy array on CPU, hence taking a longer time. Other examples of 'blocking' operations include [asscalar](../../../api/legacy/ndarray/ndarray.rst#mxnet.ndarray.NDArray.asscalar) and [wait_to_read](../../../api/legacy/ndarray/ndarray.rst#mxnet.ndarray.NDArray.wait_to_read).\n\nWhile it is possible to use [NDArray.waitall()](../../../api/legacy/ndarray/ndarray.rst#mxnet.ndarray.waitall) before and after operations to get running time of operations, it is not a scalable method to measure running time of multiple sets of operations, especially in a [Sequential](../../../api/gluon/nn/index.rst#mxnet.gluon.nn.Sequential) or hybridized network.\n\n## The correct way to profile\n\nThe correct way to measure running time of MXNet models is to use MXNet profiler. In the rest of this tutorial, we will learn how to use the MXNet profiler to measure the running time and memory consumption of MXNet models. You can import the profiler and configure it from Python code.\n\n```{.python .input}\nfrom mxnet import profiler\n\nprofiler.set_config(profile_all=True,\n                    aggregate_stats=True,\n                    continuous_dump=True,\n                    filename='profile_output.json')\n```\n\n`profile_all` enables all types of profiling. You can also individually enable the following types of profiling:\n\n- `profile_symbolic` (boolean): whether to profile symbolic operators\n- `profile_imperative` (boolean): whether to profile imperative operators\n- `profile_memory` (boolean): whether to profile memory usage\n- `profile_api` (boolean): whether to profile the C API\n\n`aggregate_stats` aggregates statistics in memory which can then be printed to console by calling `profiler.dumps()`.\n\n### Setup: Build a model\n\nLet's build a small convolutional neural network that we can use to demonstrate profiling.\n\n```{.python .input}\nfrom mxnet import gluon\n\nnet = gluon.nn.HybridSequential()\nnet.add(gluon.nn.Conv2D(channels=20, kernel_size=5, activation='relu'))\nnet.add(gluon.nn.MaxPool2D(pool_size=2, strides=2))\nnet.add(gluon.nn.Conv2D(channels=50, kernel_size=5, activation='relu'))\nnet.add(gluon.nn.MaxPool2D(pool_size=2, strides=2))\nnet.add(gluon.nn.Flatten())\nnet.add(gluon.nn.Dense(512, activation=\"relu\"))\nnet.add(gluon.nn.Dense(10))\n```\n\nWe need data that we can run through the network for profiling. We'll use the MNIST dataset.\n\n```{.python .input}\nfrom mxnet.gluon.data.vision import transforms\n\ndataset = gluon.data.vision.MNIST(train=True)\ndataset = dataset.transform_first(transforms.ToTensor())\ndataloader = gluon.data.DataLoader(dataset, batch_size=64, shuffle=True)\n```\n\nLet's define a function that will run a single training iteration given `data` and `label`.\n\n```{.python .input}\n# Use GPU if available\nif mx.device.num_gpus():\n    device=mx.gpu()\nelse:\n    device=mx.cpu()\n\n# Initialize the parameters with random weights\nnet.initialize(mx.init.Xavier(), device=device)\n\n# Use SGD optimizer\ntrainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1})\n\n# Softmax Cross Entropy is a frequently used loss function for multi-class classification\nsoftmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()\n\n# A helper function to run one training iteration\ndef run_training_iteration(data, label):\n    # Load data and label is the right device\n    data = data.to_device(device)\n    label = label.to_device(device)\n    # Run the forward pass\n    with autograd.record():\n        output = net(data)\n        loss = softmax_cross_entropy(output, label)\n    # Run the backward pass\n    loss.backward()\n    # Apply changes to parameters\n    trainer.step(data.shape[0])\n```\n\n### Starting and stopping the profiler from Python\n\nWhen the first forward pass is run on a network, MXNet does a number of housekeeping tasks including inferring the shapes of various parameters, allocating memory for intermediate and final outputs, etc. For these reasons, profiling the first iteration doesn't provide representative results for the rest of training. We will, therefore, skip the first iteration.\n\n```{.python .input}\n# Run the first iteration without profiling\nitr = iter(dataloader)\nrun_training_iteration(*next(itr))\n```\n\nWe'll run the next iteration with the profiler turned on.\n\n```{.python .input}\ndata, label = next(itr)\n\n# Ask the profiler to start recording\nprofiler.set_state('run')\n\nrun_training_iteration(*next(itr))\n\n# Make sure all operations have completed\nmx.npx.waitall()\n# Ask the profiler to stop recording\nprofiler.set_state('stop')\n# Dump all results to log file before download\nprofiler.dump()\n```\n\nBetween running and stopping the profiler, you can also pause and resume the profiler using `profiler.pause()` and `profiler.resume()` respectively to profile only parts of the code you want to profile.\n\n### Starting the profiler automatically using an environment variable\n\nThe method described above requires code changes to start and stop the profiler. You can also start the profiler automatically and profile the entire code without any code changes using the `MXNET_PROFILER_AUTOSTART` environment variable.\n\n`$ MXNET_PROFILER_AUTOSTART=1 python my_script.py`\n\nMXNet will start the profiler automatically if you run your code with the environment variable `MXNET_PROFILER_AUTOSTART` set to `1`. The profiler output is stored in `profile.json` inside the current directory.\n\nNote that the profiler output could be large depending on your code. It might be helpful to profile only sections of your code using the `set_state` API described in the previous section.\n\n### Increasing granularity of the profiler output\n\nMXNet executes computation graphs in 'bulk mode' which reduces kernel launch gaps in between symbolic operators for faster execution. This could reduce the granularity of the profiler output. If you need profiling result of every operator, please set the environment variables `MXNET_EXEC_BULK_EXEC_INFERENCE` and `MXNET_EXEC_BULK_EXEC_TRAIN` to `0` to disable the bulk execution mode.\n\nWhen working with networks created using the Gluon API, you will get a more granular profiling outputs if you profile networks that haven't been hybridized. Operations can appear fused together in the profiling outputs after hybridization, which can make debugging tricky.\n\n### Viewing profiler output\n\nThere are a few ways to view the information collected by the profiler. You can view it in the console, you can view a more graphical version in a browser, or you can use a vendor tool such as Intel VTune or Nvidia NVProf to view output. For most scenarios the information you need can be obtained with MXNet's built in profiler support, but if you want to investigate the performance of operators alongside extra device about your hardware (e.g. cache hit rates, or CUDA kernel timings) then profiling jointly with vendor tools is recommended.\n\n#### 1. View in console\n\nYou can use the `profiler.dumps()` method to view the information collected by the profiler in the console. The collected information contains time taken by each operator, time taken by each C API and memory consumed in both CPU and GPU.\n\n```{.python .input}\nprofiler.set_state('run')\nprofiler.set_state('stop')\nprint(profiler.dumps())\n```\n\n![Profile Statistics](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tutorials/python/profiler/profile_stats.png)<!--notebook-skip-line-->\n\n#### 2. View in browser\n\nYou can also dump the information collected by the profiler into a `json` file using the `profiler.dump()` function and view it in a browser.\n\n```{.python .input}\nprofiler.dump(finished=False)\n```\n\n`dump()` creates a `json` file which can be viewed using a trace consumer like `chrome://tracing` in the Chrome browser. Here is a snapshot that shows the output of the profiling we did above. Note that setting the `finished` parameter to `False` will prevent the profiler from finishing dumping to file. If you just use `profiler.dump()`, you will no longer be able to profile the remaining sections of your model. \n\n![Tracing Screenshot](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tutorials/python/profiler/profiler_output_chrome.png)\n\nLet's zoom in to check the time taken by operators\n\n![Operator profiling](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tutorials/python/profiler/profile_operators.png)\n\nThe above picture visualizes the sequence in which the operators were executed and the time taken by each operator.\n\n### Profiling oneDNN Operators\nReagrding oneDNN operators, the library has already provided the internal profiling tool. Firstly, you need set `DNNL_VERBOSE=1` to enable internal profiler.\n\n`$ DNNL_VERBOSE=1 python my_script.py > dnnl_verbose.log`\n\nNow, the detailed profiling insights of each oneDNN prmitive are saved into `dnnl_verbose.log` (like below).\n\n```\ndnnl_verbose,info,DNNL v1.1.2 (commit cb2cc7ac17ff4e2ef50805c7048d33256d82be4d)\ndnnl_verbose,info,Detected ISA is Intel AVX-512 with Intel DL Boost\ndnnl_verbose,exec,cpu,convolution,jit:avx512_common,forward_inference,src_f32::blocked:aBcd16b:f0 wei_f32::blocked:ABcd16b16a:f0 bia_undef::undef::f0 dst_f32::blocked:aBcd16b:f0,,alg:convolution_direct,mb32_ic32oc32_ih256oh256kh3sh1dh0ph1_iw256ow256kw3sw1dw0pw1,20.7539\n```\n\nFor example, if you want to calculate the total executing time of `convolution` primitive, you can just run:\n\n`$ cat dnnl_verbose.log | grep \"exec,cpu,convolution\" | awk 'BEGIN{FS=\",\"} {SUM+=$11} END {print SUM}'`\n\nMoreover, you can set `DNNL_VERBOSE=2` to collect both creating and executing time of each primitive.\n\n`$ cat dnnl_verbose.log | grep \"create,cpu,convolution\" | awk 'BEGIN{FS=\",\"} {SUM+=$11} END {print SUM}'`\n\n`$ cat dnnl_verbose.log | grep \"exec,cpu,convolution\" | awk 'BEGIN{FS=\",\"} {SUM+=$11} END {print SUM}'`\n\n\n### Profiling Custom Operators\nShould the existing NDArray operators fail to meet all your model's needs, MXNet supports [Custom Operators](../../extend/customop.ipynb) that you can define in Python. In `forward()` and `backward()` of a custom operator, there are two kinds of code: \"pure Python\" code (NumPy operators included) and \"sub-operators\" (NDArray operators called within `forward()` and `backward()`). With that said, MXNet can profile the execution time of both kinds without additional setup. Specifically, the MXNet profiler will break a single custom operator call into a pure Python event and several sub-operator events if there are any. Furthermore, all of those events will have a prefix in their names, which is, conveniently, the name of the custom operator you called.\n\nLet's try profiling custom operators with the following code example:\n\n```{.python .input}\nclass MyAddOne(mx.operator.CustomOp):\n    def forward(self, is_train, req, in_data, out_data, aux):  \n        self.assign(out_data[0], req[0], in_data[0]+1)\n\n    def backward(self, req, out_grad, in_data, out_data, in_grad, aux):\n        self.assign(in_grad[0], req[0], out_grad[0])\n\n@mx.operator.register('MyAddOne')\nclass CustomAddOneProp(mx.operator.CustomOpProp):\n    def __init__(self):\n        super(CustomAddOneProp, self).__init__(need_top_grad=True)\n\n    def list_arguments(self):\n        return ['data']\n\n    def list_outputs(self):\n        return ['output']\n\n    def infer_shape(self, in_shape):\n        return [in_shape[0]], [in_shape[0]], []\n\n    def create_operator(self, device, shapes, dtypes):\n        return MyAddOne()\n\n\ninp = mx.np.zeros(shape=(500, 500))\n\nprofiler.set_config(profile_all=True, continuous_dump=True, \\\n                    aggregate_stats=True)\nprofiler.set_state('run')\n\nw = nd.Custom(inp, op_type=\"MyAddOne\")\n\nmx.npx.waitall()\n\nprofiler.set_state('stop')\nprint(profiler.dumps())\nprofiler.dump(finished=False)\n```\n\nHere, we have created a custom operator called `MyAddOne`, and within its `forward()` function, we simply add one to the input. We can visualize the dump file in `chrome://tracing/`:\n\n![Custom Operator Profiling Screenshot](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tutorials/python/profiler/profiler_output_custom_operator_chrome.png)\n\nAs shown by the screenshot, in the **Custom Operator** domain where all the custom operator-related events fall into, we can easily visualize the execution time of each segment of `MyAddOne`. We can tell that `MyAddOne::pure_python` is executed first. We also know that `CopyCPU2CPU` and `_plus_scalr` are two \"sub-operators\" of `MyAddOne` and the sequence in which they are executed.\n\nPlease note that: to be able to see the previously described information, you need to set `profile_imperative` to `True` even when you are using custom operators in [symbolic mode](https://mxnet.apache.org/versions/master/api/python/docs/api/legacy/symbol/index.html) (refer to the code snippet below, which is the symbolic-mode equivelent of the code example above). The reason is that within custom operators, pure python code and sub-operators are still called imperatively. \n\n```{.python .input} \n# Set profile_all to True\nprofiler.set_config(profile_all=True, aggregate_stats=True, continuous_dump=True)\n# OR, Explicitly Set profile_symbolic and profile_imperative to True\nprofiler.set_config(profile_symbolic=True, profile_imperative=True, \\\n                    aggregate_stats=True, continuous_dump=True)\n\nprofiler.set_state('run')\n# Use Symbolic Mode\na = mx.symbol.Variable('a')\nb = mx.symbol.Custom(data=a, op_type='MyAddOne')\nc = b.bind(mx.cpu(), {'a': inp})\ny = c.forward()\nmx.npx.waitall()\nprofiler.set_state('stop')\nprint(profiler.dumps())\nprofiler.dump()\n```\n\n### Some Rules to Pay Attention to\n1. Always use `profiler.dump(finished=False)` if you do not intend to finish dumping to file. Otherwise, calling `profiler.dump()` in the middle of your model may lead to unexpected behaviors; and if you subsequently call `profiler.set_config()`, the program will error out.\n\n2. You can only dump to one file. Do not change the target file by calling `profiler.set_config(filename='new_name.json')` in the middle of your model. This will lead to incomplete dump outputs.\n\n## Advanced: Using NVIDIA Profiling Tools\n\nMXNet's Profiler is the recommended starting point for profiling MXNet code, but NVIDIA also provides a couple of tools for low-level profiling of CUDA code: [NVProf](https://devblogs.nvidia.com/cuda-pro-tip-nvprof-your-handy-universal-gpu-profiler/), [Visual Profiler](https://developer.nvidia.com/nvidia-visual-profiler) and [Nsight Compute](https://developer.nvidia.com/nsight-compute). You can use these tools to profile all kinds of executables, so they can be used for profiling Python scripts running MXNet. And you can use these in conjunction with the MXNet Profiler to see high-level information from MXNet alongside the low-level CUDA kernel information.\n\n### NVProf and Visual Profiler\n\nNVProf and Visual Profiler are available in CUDA 9 and CUDA 10 toolkits. You can get a timeline view of CUDA kernel executions, and also analyse the profiling results to get automated recommendations. It is useful for profiling end-to-end training but the interface can sometimes become slow and unresponsive.\n\nYou can initiate the profiling directly from inside Visual Profiler or from the command line with `nvprof` which wraps the execution of your Python script. If it's not on your path already, you can find `nvprof` inside your CUDA directory. See [this discussion post](https://discuss.mxnet.io/t/using-nvidia-profiling-tools-visual-profiler-and-nsight-compute/) for more details on setup.\n\n`$ nvprof -o my_profile.nvvp python my_profiler_script.py`\n\n`==11588== NVPROF is profiling process 11588, command: python my_profiler_script.py`\n\n`==11588== Generated result file: /home/user/Development/mxnet/ci/my_profile.nvvp`\n\nWe specified an output file called `my_profile.nvvp` and this will be annotated with NVTX ranges (for MXNet operations) that will be displayed alongside the standard NVProf timeline. This can be very useful when you're trying to find patterns between operators run by MXNet, and their associated CUDA kernel calls.\n\nYou can open this file in Visual Profiler to visualize the results.\n\n![Operator profiling nvprof](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tutorials/python/profiler/profiler_nvprof.png)\n\nAt the top of the plot we have CPU tasks such as driver operations, memory copy calls, MXNet engine operator invocations, and imperative MXNet API calls.  Below we see the kernels active on the GPU during the same time period.\n\n![Operator profiling nvprof zoomed](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tutorials/python/profiler/profiler_nvprof_zoomed.png)\n\nZooming in on a backwards convolution operator we can see that it is in fact made up of a number of different GPU kernel calls, including a cuDNN winograd convolution call, and a fast-fourier transform call.\n\n![Operator profiling winograd](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tutorials/python/profiler/profiler_winograd.png)\n\nSelecting any of these kernel calls (the winograd convolution call shown here) will get you some interesting GPU performance information such as occupancy rates (vs theoretical), shared memory usage and execution duration.\n\n### Nsight Compute\n\nNsight Compute is available in CUDA 10 toolkit, but can be used to profile code running CUDA 9. You don't get a timeline view, but you get many low level statistics about each individual kernel executed and can compare multiple runs (i.e. create a baseline).\n\n![Nsight Compute](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tutorials/python/profiler/profile_nsight_compute.png)\n\n## Further reading\n\n- [Examples using MXNet profiler.](https://github.com/apache/mxnet/tree/master/example/profiler)\n- [Some tips for improving MXNet performance.](https://mxnet.apache.org/api/faq/perf)\n\n<!-- INSERT SOURCE DOWNLOAD BUTTONS -->\n\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/performance/backend/tvm.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nUse TVM\n=======\n\nContributions welcome!\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/performance/compression/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nCompression\n===========\nThe following tutorials will help you learn how to use compression techniques with MXNet.\n\n.. container:: cards\n\n   .. card::\n      :title: Compression: float16\n      :link: https://mxnet.apache.org/api/faq/float16\n\n      How to use float16 in your model to boost training speed.\n\n   .. card::\n      :title: Gradient Compression\n      :link: https://mxnet.apache.org/api/faq/gradient_compression\n\n      How to use gradient compression to reduce communication bandwidth and increase speed.\n\n   .. card::\n      :title: Inference with Quantized Models\n      :link: https://gluon-cv.mxnet.io/build/examples_deployment/int8_inference.html\n\n      How to use quantized GluonCV models for inference on Intel Xeon Processors to gain higher performance.\n\n   .. card::\n      :title: Compression: int8\n      :link: int8.html\n\n      How to use int8 in your model to boost training speed.\n\n\n.. toctree::\n   :hidden:\n   :glob:\n\n   *\n   Float16 <https://mxnet.apache.org/api/faq/float16>\n   Gradient Compression  <https://mxnet.apache.org/api/faq/gradient_compression>\n   GluonCV with Quantized Models <https://gluon-cv.mxnet.io/build/examples_deployment/int8_inference.html>\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/performance/compression/int8.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nDeploy with int-8\n=================\n\nContributions welcome!\n"
  },
  {
    "path": "docs/python_docs/python/tutorials/performance/index.rst",
    "content": ".. Licensed to the Apache Software Foundation (ASF) under one\n   or more contributor license agreements.  See the NOTICE file\n   distributed with this work for additional information\n   regarding copyright ownership.  The ASF licenses this file\n   to you under the Apache License, Version 2.0 (the\n   \"License\"); you may not use this file except in compliance\n   with the License.  You may obtain a copy of the License at\n\n     http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing,\n   software distributed under the License is distributed on an\n   \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n   KIND, either express or implied.  See the License for the\n   specific language governing permissions and limitations\n   under the License.\n\nPerformance\n===========\nThe following tutorials will help you learn how to tune MXNet or use tools that will improve training and inference performance.\n\nEssential\n---------\n\n.. container:: cards\n\n   .. card::\n      :title: Improving Performance\n      :link: /api/faq/perf\n\n      How to get the best performance from MXNet.\n\n   .. card::\n      :title: Profiler\n      :link: backend/profiler.html\n\n      How to profile MXNet models.\n\n\nCompression\n-----------\n\n.. container:: cards\n\n   .. card::\n      :title: Compression: float16\n      :link: /api/faq/float16\n\n      How to use float16 in your model to boost training speed.\n\n   .. card::\n      :title: Gradient Compression\n      :link: /api/faq/gradient_compression\n\n      How to use gradient compression to reduce communication bandwidth and increase speed.\n   ..\n      .. card::\n         :title: Compression: int8\n         :link: compression/int8.html\n\n         How to use int8 in your model to boost training speed.\n   ..\n\n\nAccelerated Backend\n-------------------\n\n.. container:: cards\n\n   .. card::\n      :title: TensorRT\n      :link: backend/tensorrt/index.html\n\n      How to use NVIDIA's TensorRT to boost inference performance.\n\n   ..\n      TBD Content\n      .. card::\n         :title: oneDNN\n         :link: backend/dnnl/dnnl_readme\n\n         How to get the most from your CPU by using oneDNN.\n\n      .. card::\n         :title: TVM\n         :link: backend/tvm.html\n\n         How to use TVM to boost performance.\n   ..\n\n\nDistributed Training\n--------------------\n\n.. container:: cards\n\n   .. card::\n      :title: Distributed Training Using the KVStore API\n      :link: /api/faq/distributed_training.html\n\n      How to use the KVStore API to use multiple GPUs when training a model.\n\n   .. card::\n      :title: Training with Multiple GPUs Using Model Parallelism\n      :link: /api/faq/model_parallel_lstm.html\n\n      An overview of using multiple GPUs when training an LSTM.\n\n   .. card::\n      :title: Distributed training in MXNet\n      :link: /api/faq/distributed_training\n\n      An overview of distributed training strategies.\n\n   .. card::\n      :title: MXNet with Horovod\n      :link: https://github.com/apache/mxnet/tree/master/example/distributed_training-horovod\n\n      A set of example scripts demonstrating MNIST and ImageNet training with Horovod as the distributed training backend.\n\n.. toctree::\n   :hidden:\n   :maxdepth: 1\n\n   compression/index\n   backend/index\n"
  },
  {
    "path": "docs/python_docs/requirements",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nnumpy>=1.17,<1.20.0\njupyter\nJinja2==3.0.3\nsphinx==2.4.0\nmatplotlib\nnotebook\nnbconvert==5.6.1\njupyter-client<=6.1.12\nnbsphinx==0.4.3\nrecommonmark==0.6.0\nnotedown==1.5.1\npypandoc==1.4\nbreathe==4.13.1\nmock==3.0.5\nawscli==1.16.266\nautodocsumm==0.1.12"
  },
  {
    "path": "docs/python_docs/themes/.babelrc",
    "content": "{\n    \"presets\": [\"env\"]\n}"
  },
  {
    "path": "docs/python_docs/themes/.circleci/config.yml",
    "content": "version: 2\njobs:\n  build:\n    working_directory: ~/sphinx_materialdesign_theme\n    docker:\n      - image: circleci/python:3.6.4\n    steps:\n      - checkout\n      - run: sudo chown -R circleci:circleci /usr/local/bin\n      - run: sudo chown -R circleci:circleci /usr/local/lib/python3.6/site-packages\n      - run: \n          name: install dependencies\n          command: pip install -r requirements.txt\n      - run:\n          name: build\n          command: sphinx-build -b html ./example ./_build\n      - run:\n          name: deploy\n          command: |\n            remote=$(git config remote.origin.url)\n            pushd _build > /dev/null\n            git config --global user.email \"$GH_EMAIL\" > /dev/null 2>&1\n            git config --global user.name \"$GH_NAME\" > /dev/null 2>&1\n            touch .nojekyll\n            git init\n            git add .\n            git commit -m \"Deploy to GitHub Pages. [skip ci]\"\n            git push --force --quiet $remote master:gh-pages\n            popd > /dev/null\nworkflows:\n  version: 2\n  build_flow:\n    jobs:\n      - build:\n          filters:\n            branches:\n              only: master"
  },
  {
    "path": "docs/python_docs/themes/.gitignore",
    "content": ".idea/\n.cache/\n.vscode/\n*.egg-info/\ndist/\n_build/\nbuild/\nexample/_build/\nnode_modules/\n*.log\n**/*.pyc\n**/__pycache__\n.sass-cache/\n"
  },
  {
    "path": "docs/python_docs/themes/.sassrc",
    "content": "{\r\n    \"includePaths\": [\r\n        \"node_modules\"\r\n    ]\r\n}"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/LICENSE",
    "content": "MIT License\n\nCopyright (c) 2016 myyasuda\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/MANIFEST.in",
    "content": "recursive-include mxtheme *\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/README.md",
    "content": "# Material Design HTML Theme for Sphinx\r\n\r\n## How to use\r\n\r\n- Install the theme by\r\n\r\n```bash\r\npip install mxtheme\r\n```\r\n\r\n- Modify the `conf.py` for your sphinx project by\r\n\r\ncreate a submodule of this repo on the same folder with `conf.py` for your sphinx project. then modify the following three lines in `conf.py`:\r\n\r\n```python\r\nhtml_theme = 'mxtheme'\r\n```\r\n\r\nIn addition, to use the `card` directive in rst, you can and add the following two lines into your `def setup(app)` function:\r\n\r\n```python\r\ndef setup(app):\r\n    ...\r\n    import mxtheme\r\n    app.add_directive('card', mxtheme.CardDirective)\r\n```\r\n\r\n## How to build\r\n\r\n\r\nInstall `npm` first,\r\n\r\non ubuntu:\r\n\r\n```\r\nwget -qO- https://deb.nodesource.com/setup_8.x | sudo -E bash -\r\nsudo apt-get install -y nodejs\r\n```\r\n\r\non macos\r\n\r\n```\r\nbrew install nodejs\r\n```\r\n\r\nThen install packages\r\n\r\n```\r\nnpm install\r\n```\r\n\r\nLast, build css and js\r\n\r\n\r\n```\r\nnpm run build\r\n```\r\n\r\n## Acknowledgment\r\n\r\n\r\nThis is fork of\r\n[sphinx_materialdesign_theme](https://github.com/myyasuda/sphinx_materialdesign_theme). With\r\nsome CSS/JS modifications. Please refer to the original project for more\r\ndocuments.\r\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/__init__.py",
    "content": "from os import path\r\nfrom .card import CardDirective\r\n\r\n__version__ = '0.3.9'\r\n__version_full__ = __version__\r\n\r\npackage_dir = path.dirname(path.abspath(__file__))\r\n\r\ndef get_path():\r\n    return package_dir\r\n\r\ndef setup(app):\r\n    app.add_html_theme('mxtheme', package_dir)\r\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/card.py",
    "content": "from sphinx.locale import _\nfrom docutils import nodes\nfrom docutils.parsers.rst import Directive, directives\n\nclass card(nodes.General, nodes.Element):\n    pass\n\nclass CardDirective(Directive):\n\n    # defines the parameter the directive expects\n    # directives.unchanged means you get the raw value from RST\n    required_arguments = 0\n    optional_arguments = 0\n    final_argument_whitespace = True\n    option_spec = {'title': directives.unchanged,\n                   'link': directives.unchanged,\n                   'is_head': directives.unchanged}\n    has_content = True\n    add_index = False\n\n    def run(self):\n        # gives you access to the options of the directive\n        options = self.options\n\n        cid = nodes.make_id(\"card-{}\".format(options['title']))\n\n        classes = ['mx-card']\n        if options.get('is_head', 'False').lower() == 'true':\n            classes.append('head-card')\n        container = nodes.container(ids=[cid], classes=classes)\n\n        container += nodes.inline('', options['title'], classes=['mx-card-title'])\n        link = options.get('link')\n        if link:\n            container += nodes.inline('', link, classes=['mx-card-link'])\n\n        para = nodes.paragraph(classes=['mx-card-text'])\n        self.state.nested_parse(self.content, self.content_offset, para)\n        container += para\n\n        # we return the result\n        return [container]\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/drawer.html",
    "content": "<header class=\"mdl-layout__drawer\">      \r\n    {% block menu %}\r\n      <div class=\"globaltoc\">\r\n        <span class=\"mdl-layout-title toc\">{{ _('Table Of Contents') }}</span>\r\n        {% set toctree = toctree(maxdepth=6, collapse=False, includehidden=True, titles_only=True) %}\r\n        {% if toctree %}\r\n            {% set lines = toctree.split('\\n') %}\r\n            <nav class=\"mdl-navigation\">\r\n                {{ toctree }}\r\n            </nav>\r\n        {% else %}\r\n        <!-- Local TOC -->\r\n        <nav class=\"mdl-navigation\">{{ toc }}</nav>\r\n        {% endif %}\r\n        </div>\r\n    {% endblock %}\r\n</header>\r\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/feedback.html",
    "content": "<hr class=\"feedback-hr-top\" />\n<div class=\"feedback-container\">\n    <div class=\"feedback-question\">Did this page help you?</div>\n    <div class=\"feedback-answer-container\">\n        <div class=\"feedback-answer yes-link\" data-response=\"yes\">Yes</div>\n        <div class=\"feedback-answer no-link\" data-response=\"no\">No</div>\n    </div>\n    <div class=\"feedback-thank-you\">Thanks for your feedback!</div>\n</div>\n<hr class=\"feedback-hr-bottom\" />\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/footer.html",
    "content": "<footer class=\"site-footer h-card\">\r\n    <div class=\"wrapper\">\r\n        <div class=\"row\">\r\n            <div class=\"col-4\">\r\n                <h4 class=\"footer-category-title\">Resources</h4>\r\n                <ul class=\"contact-list\">\r\n                    <li><a href=\"https://lists.apache.org/list.html?dev@mxnet.apache.org\">Mailing list</a> <a class=\"u-email\" href=\"mailto:dev-subscribe@mxnet.apache.org\">(subscribe)</a></li>\r\n                    <li><a href=\"https://discuss.mxnet.io\">MXNet Discuss forum</a></li>\r\n                    <li><a href=\"https://github.com/apache/mxnet/issues\">Github Issues</a></li>\r\n                    <li><a href=\"https://github.com/apache/mxnet/projects\">Projects</a></li>\r\n                    <li><a href=\"https://cwiki.apache.org/confluence/display/MXNET/Apache+MXNet+Home\">Developer Wiki</a></li>\r\n                    <li><a href=\"/community\">Contribute To MXNet</a></li>\r\n                </ul>\r\n            </div>\r\n\r\n            <div class=\"col-4\"><ul class=\"social-media-list\"><li><a href=\"https://github.com/apache/mxnet\"><svg class=\"svg-icon\"><use xlink:href=\"{{pathto('_static/minima-social-icons.svg#github', 1)}}\"></use></svg> <span class=\"username\">apache/mxnet</span></a></li><li><a href=\"https://www.twitter.com/apachemxnet\"><svg class=\"svg-icon\"><use xlink:href=\"{{pathto('_static/minima-social-icons.svg#twitter', 1)}}\"></use></svg> <span class=\"username\">apachemxnet</span></a></li><li><a href=\"https://youtube.com/apachemxnet\"><svg class=\"svg-icon\"><use xlink:href=\"{{pathto('_static/minima-social-icons.svg#youtube', 1)}}\"></use></svg> <span class=\"username\">apachemxnet</span></a></li></ul>\r\n</div>\r\n\r\n            <div class=\"col-4 footer-text\">\r\n                <p>A flexible and efficient library for deep learning.</p>\r\n            </div>\r\n        </div>\r\n    </div>\r\n</footer>\r\n\r\n<footer class=\"site-footer2\">\r\n    <div class=\"wrapper\">\r\n        <div class=\"row\">\r\n            <div class=\"col-3\">\r\n                <img src=\"{{pathto('_static/apache_incubator_logo.png', 1)}}\" class=\"footer-logo col-2\">\r\n            </div>\r\n            <div class=\"footer-bottom-warning col-9\">\r\n                <p>Apache MXNet is an effort undergoing incubation at <a href=\"http://www.apache.org/\">The Apache Software Foundation</a> (ASF), <span style=\"font-weight:bold\">sponsored by the <i>Apache Incubator</i></span>. Incubation is required\r\n                    of all newly accepted projects until a further review indicates that the infrastructure,\r\n                    communications, and decision making process have stabilized in a manner consistent with other\r\n                    successful ASF projects. While incubation status is not necessarily a reflection of the completeness\r\n                    or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.\r\n                </p><p>\"Copyright © 2017-2018, The Apache Software Foundation Apache MXNet, MXNet, Apache, the Apache\r\n                    feather, and the Apache MXNet project logo are either registered trademarks or trademarks of the\r\n                    Apache Software Foundation.\"</p>\r\n            </div>\r\n        </div>\r\n    </div>\r\n</footer>\r\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/header.html",
    "content": "<header class=\"mdl-layout__header {% if theme_header_waterfall|tobool %}mdl-layout__header--waterfall{% endif %} {% if theme_header_scroll|tobool %}mdl-layout__header--scroll{% endif %}\">\r\n    <div class=\"mdl-layout__header-row\">\r\n        {% if theme_show_header_title|tobool %}\r\n        <!-- Title -->\r\n        <span class=\"mdl-layout-title\">\r\n            <a class=\"brand\" href=\"{{ pathto(master_doc) }}\">\r\n                {%- if logo %}\r\n                <img class=\"logo\" src=\"{{ pathto('_static/' + logo, 1) }}\" alt=\"{{ project }}\"/>\r\n                {%- else %}\r\n                {{ project }}\r\n                {%- endif %}\r\n            </a>\r\n        </span>\r\n        {% endif %}\r\n        <nav class=\"mdl-navigation breadcrumb\">\r\n            {%- for parent in parents %}\r\n            <a class=\"mdl-navigation__link\" href=\"{{ parent.link|e }}\">{{ parent.title }}</a><i class=\"material-icons\">navigate_next</i>\r\n            {%- endfor %}\r\n            <a class=\"mdl-navigation__link is-active\">{{ title }}</a>\r\n        </nav>\r\n        <div class=\"mdl-layout-spacer\"></div>\r\n        <nav class=\"mdl-navigation\">\r\n        {% include \"header_search.html\" %}\r\n        {% include \"header_sourcelink.html\" %}\r\n        </nav>\r\n    </div>\r\n    <div class=\"mdl-layout__header-row header-links\">\r\n      <div class=\"mdl-layout-spacer\"></div>\r\n      <nav class=\"mdl-navigation\">\r\n      {%- for title, href, isExternal, icon in theme_header_links %}\r\n          {% if isExternal %}\r\n              <a  class=\"mdl-navigation__link\" href=\"{{ href }}\">\r\n                  {% if icon %}<i class=\"{{ icon }}\"></i>{% endif %}\r\n                  {{ title }}\r\n              </a>\r\n          {%- else -%}\r\n              <a  class=\"mdl-navigation__link\" href=\"{{ pathto(href) }}\">\r\n                  {% if icon %}<i class=\"{{ icon }}\"></i>{% endif %}\r\n                  {{ title }}\r\n              </a>\r\n          {%- endif -%}\r\n      {%- endfor %}\r\n      </nav>\r\n    </div>\r\n</header>\r\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/header_search.html",
    "content": "{%- if pagename != \"search\" %}\r\n<form class=\"form-inline pull-sm-right\" action=\"{{ pathto('search') }}\" method=\"get\">\r\n      <div class=\"mdl-textfield mdl-js-textfield mdl-textfield--expandable mdl-textfield--floating-label mdl-textfield--align-right\">\r\n        <label id=\"quick-search-icon\" class=\"mdl-button mdl-js-button mdl-button--icon\"  for=\"waterfall-exp\">\r\n          <i class=\"material-icons\">search</i>\r\n        </label>\r\n        <div class=\"mdl-textfield__expandable-holder\">\r\n          <input class=\"mdl-textfield__input\" type=\"text\" name=\"q\"  id=\"waterfall-exp\" placeholder=\"Search\" />\r\n          <input type=\"hidden\" name=\"check_keywords\" value=\"yes\" />\r\n          <input type=\"hidden\" name=\"area\" value=\"default\" />\r\n        </div>\r\n      </div>\r\n      <div class=\"mdl-tooltip\" data-mdl-for=\"quick-search-icon\">\r\n      {{ _('Quick search') }}\r\n      </div>\r\n</form>\r\n{%- endif %}\r\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/header_sourcelink.html",
    "content": "{%- if display_github %}\r\n<a id=\"button-show-github\"\r\n    href=\"https://{{ github_host|default(\"github.com\") }}/{{ github_user }}/{{ github_repo }}/{{ theme_vcs_pageview_mode|default(\"edit\") }}/{{ github_version }}{{ conf_py_path }}{{ pagename }}{{ page_source_suffix }}\" class=\"mdl-button mdl-js-button mdl-button--icon\">\r\n<i class=\"material-icons\">edit</i>\r\n</a>\r\n<div class=\"mdl-tooltip\" data-mdl-for=\"button-show-github\">\r\n{{ _('Edit on Github') }}\r\n</div>\r\n{%- elif show_source and has_source and sourcename %}\r\n<a id=\"button-show-source\"\r\n    class=\"mdl-button mdl-js-button mdl-button--icon\"\r\n    href=\"{{ pathto('_sources/' + sourcename, true)|e }}\" rel=\"nofollow\">\r\n  <i class=\"material-icons\">code</i>\r\n</a>\r\n<div class=\"mdl-tooltip\" data-mdl-for=\"button-show-source\">\r\n{{ _('Show Source') }}\r\n</div>\r\n{%- endif %}\r\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/header_top.html",
    "content": "<header class=\"site-header\" role=\"banner\">\n  <div class=\"wrapper\">\n      <a class=\"site-title\" rel=\"author\" href=\"{{theme_relative_url}}\"><img\n            src=\"{{pathto('_static/mxnet_logo.png', 1)}}\" class=\"site-header-logo\"></a>\n    <nav class=\"site-nav\">\n      <input type=\"checkbox\" id=\"nav-trigger\" class=\"nav-trigger\"/>\n      <label for=\"nav-trigger\">\n          <span class=\"menu-icon\">\n            <svg viewBox=\"0 0 18 15\" width=\"18px\" height=\"15px\">\n              <path d=\"M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z\"/>\n            </svg>\n          </span>\n      </label>\n\n      <div class=\"trigger\">\n        <a class=\"page-link\" href=\"{{theme_relative_url}}get_started\">Get Started</a>\n        <a class=\"page-link\" href=\"{{theme_relative_url}}features\">Features</a>\n        <a class=\"page-link\" href=\"{{theme_relative_url}}ecosystem\">Ecosystem</a>\n        <a class=\"page-link page-current\" href=\"{{theme_relative_url}}api\">Docs & Tutorials</a>\n        <a class=\"page-link\" href=\"{{theme_relative_url}}trusted_by\">Trusted By</a>\n        <a class=\"page-link\" href=\"https://github.com/apache/mxnet\">GitHub</a>\n        <div class=\"dropdown\" style=\"min-width:100px\">\n          <span class=\"dropdown-header\">Apache\n            <svg class=\"dropdown-caret\" viewBox=\"0 0 32 32\" class=\"icon icon-caret-bottom\" aria-hidden=\"true\"><path class=\"dropdown-caret-path\" d=\"M24 11.305l-7.997 11.39L8 11.305z\"></path></svg>\n          </span>\n          <div class=\"dropdown-content\" style=\"min-width:250px\">\n            <a href=\"https://www.apache.org/foundation/\">Apache Software Foundation</a>\n            <a href=\"https://incubator.apache.org/\">Apache Incubator</a>\n            <a href=\"https://www.apache.org/licenses/\">License</a>\n            <a href=\"/versions/1.9.1/api/faq/security.html\">Security</a>\n            <a href=\"https://privacy.apache.org/policies/privacy-policy-public.html\">Privacy</a>\n            <a href=\"https://www.apache.org/events/current-event\">Events</a>\n            <a href=\"https://www.apache.org/foundation/sponsorship.html\">Sponsorship</a>\n            <a href=\"https://www.apache.org/foundation/thanks.html\">Thanks</a>\n          </div>\n        </div>\n        <div class=\"dropdown\">\n          <span class=\"dropdown-header\">master\n            <svg class=\"dropdown-caret\" viewBox=\"0 0 32 32\" class=\"icon icon-caret-bottom\" aria-hidden=\"true\"><path class=\"dropdown-caret-path\" d=\"M24 11.305l-7.997 11.39L8 11.305z\"></path></svg>\n          </span>\n          <div class=\"dropdown-content\">\n            <a class=\"dropdown-option-active\" href=\"/versions/master/\">master</a><br>\n            <a class=\"dropdown-option\" href=\"/versions/1.9.1/\">1.9.1</a><br>\n            <a class=\"dropdown-option\" href=\"/versions/1.8.0/\">1.8.0</a><br>\n            <a class=\"dropdown-option\" href=\"/versions/1.7.0/\">1.7.0</a><br>\n            <a class=\"dropdown-option\" href=\"/versions/1.6.0/\">1.6.0</a><br>\n            <a class=\"dropdown-option\" href=\"/versions/1.5.0/\">1.5.0</a><br>\n            <a class=\"dropdown-option\" href=\"/versions/1.4.1/\">1.4.1</a><br>\n            <a class=\"dropdown-option\" href=\"/versions/1.3.1/\">1.3.1</a><br>\n            <a class=\"dropdown-option\" href=\"/versions/1.2.1/\">1.2.1</a><br>\n            <a class=\"dropdown-option\" href=\"/versions/1.1.0/\">1.1.0</a><br>\n            <a class=\"dropdown-option\" href=\"/versions/1.0.0/\">1.0.0</a><br>\n            <a class=\"dropdown-option\" href=\"/versions/0.12.1/\">0.12.1</a><br>\n            <a class=\"dropdown-option\" href=\"/versions/0.11.0/\">0.11.0</a>\n          </div>\n        </div>\n      </div>\n    </nav>\n  </div>\n</header>\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/layout.html",
    "content": "{% extends \"basic/layout.html\" %}\n\n{%- block doctype -%}\n<!DOCTYPE html>\n{%- endblock %}\n{% block htmltitle %}\n    <meta charset=\"utf-8\">\n    <meta name=\"viewport\" content=\"width=device-width, initial-scale=1, shrink-to-fit=no\">\n    <meta http-equiv=\"x-ua-compatible\" content=\"ie=edge\">\n    <style>\n    .dropdown {\n        position: relative;\n        display: inline-block;\n    }\n\n    .dropdown-content {\n        display: none;\n        position: absolute;\n        background-color: #f9f9f9;\n        min-width: 160px;\n        box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2);\n        padding: 12px 16px;\n        z-index: 1;\n        text-align: left;\n    }\n\n    .dropdown:hover .dropdown-content {\n        display: block;\n    }\n\n    .dropdown-option:hover {\n        color: #FF4500;\n    }\n\n    .dropdown-option-active {\n        color: #FF4500;\n        font-weight: lighter;\n    }\n\n    .dropdown-option {\n        color: #000000;\n        font-weight: lighter;\n    }\n\n    .dropdown-header {\n        color: #FFFFFF;\n        display: inline-flex;\n    }\n\n    .dropdown-caret {\n        width: 18px;\n        height: 54px;\n    }\n\n    .dropdown-caret-path {\n        fill: #FFFFFF;\n    }\n    </style>\n    {{ super() }}\n{% endblock %}\n\n\n{% set css_files = css_files + [\n    '_static/material-design-lite-1.3.0/material.' + theme_primary_color|e + '-' + theme_accent_color|e + '.min.css',\n    '_static/sphinx_materialdesign_theme.css',\n    '_static/fontawesome/all.css',\n    '_static/fonts.css',\n    '_static/feedback.css',\n] %}\n\n{% set script_files = script_files + [\n    '_static/sphinx_materialdesign_theme.js'\n  ]\n%}\n\n{%- block header %}{% endblock %}\n{%- block relbar1 %}{% endblock %}\n{%- block relbar2 %}{% include \"relations.html\" %}{% endblock %}\n{%- block sidebar2 %}{% endblock %}\n\n{%- block body_tag %}\n<body>\n    {%- block header_top %}{% include \"header_top.html\" %}{% endblock %}\n    <div class=\"mdl-layout mdl-js-layout {% if theme_fixed_header|tobool %}mdl-layout--fixed-header{% endif %} {% if theme_fixed_drawer|tobool %}mdl-layout--fixed-drawer{% endif %}\">\n        {%- block md_header %}{% include \"header.html\" %}{% endblock %}\n        {%- block sidebar1 %}{% include \"drawer.html\" %}{% endblock %}\n        <main class=\"mdl-layout__content\" tabIndex=\"0\">\n{% endblock %}\n\n{%- block document %}\n        <div class=\"page-content\" role=\"main\">\n        {% block body %} {% endblock %}\n        {% include \"feedback.html\" %}\n        </div>\n        <div class=\"side-doc-outline\">\n            <div class=\"side-doc-outline--content\">\n                {%- block right_sidebar %} {% include \"localtoc.html\" %}{% endblock %}\n            </div>\n        </div>                    \n{% endblock %}\n        {%- block footer %}\n            {% if theme_show_footer|tobool %}{% include \"footer.html\" %}{% endif %}\n        {% endblock %}\n        </main>\n    </div>\n</body>\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/localtoc.html",
    "content": "{%- if display_toc %}\n<div class=\"localtoc\">\n    <p class=\"caption\">\n      <span class=\"caption-text\">{{ _('Table Of Contents') }}</span>\n    </p>\n    {{ toc }}\n</div>\n{%- endif %}"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/relations.html",
    "content": "<div class=\"pagenation\">\r\n  {%- if prev %}\r\n     <a id=\"button-prev\" href=\"{{ prev.link|e }}\" class=\"mdl-button mdl-js-button mdl-js-ripple-effect mdl-button--colored\" role=\"botton\" {{ accesskey(\"P\") }}>\r\n         <i class=\"pagenation-arrow-L fas fa-arrow-left fa-lg\"></i>\r\n         <div class=\"pagenation-text\">\r\n            <span class=\"pagenation-direction\">Previous</span>\r\n            <div>{{ prev.title|striptags }}</div>\r\n         </div>\r\n     </a>\r\n  {%- endif %}\r\n  {%- if next %}\r\n     <a id=\"button-next\" href=\"{{ next.link|e }}\" class=\"mdl-button mdl-js-button mdl-js-ripple-effect mdl-button--colored\" role=\"botton\" {{ accesskey(\"N\") }}>\r\n         <i class=\"pagenation-arrow-R fas fa-arrow-right fa-lg\"></i>\r\n        <div class=\"pagenation-text\">\r\n            <span class=\"pagenation-direction\">Next</span>\r\n            <div>{{ next.title|striptags }}</div>\r\n        </div>\r\n     </a>\r\n  {%- endif %}\r\n  </div>\r\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/search.html",
    "content": "{%- extends \"layout.html\" %}\n{% set title = _('Search') %}\n{% block extrahead %}\n  <script type=\"text/javascript\" src=\"{{ pathto('_static/searchtools.js', 1) }} \"></script>\n  <script type=\"text/javascript\">\n    jQuery(function() { Search.loadIndex(\"{{ pathto('searchindex.js', 1) }}\"); });\n  </script>\n  {# this is used when loading the search index using $.ajax fails,\n     such as on Chrome for documents on localhost #}\n  <script type=\"text/javascript\" id=\"searchindexloader\"></script>\n  {{ super() }}\n{% endblock %}\n{% block body %}\n  <h1 id=\"search-documentation\">{{ _('Search') }}</h1>\n  <div id=\"fallback\" class=\"admonition warning\">\n  <script type=\"text/javascript\">$('#fallback').hide();</script>\n  <p>\n    {% trans %}Please activate JavaScript to enable the search\n    functionality.{% endtrans %}\n  </p>\n  </div>\n  <p>\n    {% trans %}From here you can search these documents. Enter your search\n    words into the box below and click \"search\". Note that the search\n    function will automatically search for all of the words. Pages\n    containing fewer words won't appear in the result list.{% endtrans %}\n  </p>\n  <form action=\"\" method=\"get\">\n    <div class=\"mdl-textfield mdl-js-textfield mdl-textfield--floating-label\">\n      <input class=\"mdl-textfield__input\" type=\"text\" name=\"q\" id=\"search-input\">\n      <label class=\"mdl-textfield__label\" for=\"search-input\">{{ _('Search') }}...</label>\n    </div>\n    <button type=\"submit\" class=\"mdl-button mdl-js-button mdl-button--fab mdl-button--mini-fab mdl-button--colored\">\n      <i class=\"material-icons\">search</i>\n    </button>\n    <span id=\"search-progress\" style=\"padding-left: 10px\"></span>\n  </form>\n  {% if search_performed %}\n    <h2>{{ _('Search Results') }}</h2>\n    {% if not search_results %}\n      <p>{{ _('Your search did not match any documents. Please make sure that all words are spelled correctly and that you\\'ve selected enough categories.') }}</p>\n    {% endif %}\n  {% endif %}\n  <div id=\"search-results\">\n  {% if search_results %}\n    <ul class=\"search\">\n    {% for href, caption, context in search_results %}\n      <li>\n        <a href=\"{{ pathto(item.href) }}\">{{ caption }}</a>\n        <div class=\"context\">{{ context|e }}</div>\n      </li>\n    {% endfor %}\n    </ul>\n  {% endif %}\n  </div>\n{% endblock %}\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/static/fontawesome/all.css",
    "content": "/*!\n * Font Awesome Free 5.5.0 by @fontawesome - https://fontawesome.com\n * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)\n */\n.fa,.fab,.fal,.far,.fas{-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;display:inline-block;font-style:normal;font-variant:normal;text-rendering:auto;line-height:1}.fa-lg{font-size:1.33333em;line-height:.75em;vertical-align:-.0667em}.fa-xs{font-size:.75em}.fa-sm{font-size:.875em}.fa-1x{font-size:1em}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-6x{font-size:6em}.fa-7x{font-size:7em}.fa-8x{font-size:8em}.fa-9x{font-size:9em}.fa-10x{font-size:10em}.fa-fw{text-align:center;width:1.25em}.fa-ul{list-style-type:none;margin-left:2.5em;padding-left:0}.fa-ul>li{position:relative}.fa-li{left:-2em;position:absolute;text-align:center;width:2em;line-height:inherit}.fa-border{border:.08em solid #eee;border-radius:.1em;padding:.2em .25em .15em}.fa-pull-left{float:left}.fa-pull-right{float:right}.fa.fa-pull-left,.fab.fa-pull-left,.fal.fa-pull-left,.far.fa-pull-left,.fas.fa-pull-left{margin-right:.3em}.fa.fa-pull-right,.fab.fa-pull-right,.fal.fa-pull-right,.far.fa-pull-right,.fas.fa-pull-right{margin-left:.3em}.fa-spin{animation:fa-spin 2s infinite linear}.fa-pulse{animation:fa-spin 1s infinite steps(8)}@keyframes fa-spin{0%{transform:rotate(0deg)}to{transform:rotate(1turn)}}.fa-rotate-90{-ms-filter:\"progid:DXImageTransform.Microsoft.BasicImage(rotation=1)\";transform:rotate(90deg)}.fa-rotate-180{-ms-filter:\"progid:DXImageTransform.Microsoft.BasicImage(rotation=2)\";transform:rotate(180deg)}.fa-rotate-270{-ms-filter:\"progid:DXImageTransform.Microsoft.BasicImage(rotation=3)\";transform:rotate(270deg)}.fa-flip-horizontal{-ms-filter:\"progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)\";transform:scaleX(-1)}.fa-flip-vertical{transform:scaleY(-1)}.fa-flip-horizontal.fa-flip-vertical,.fa-flip-vertical{-ms-filter:\"progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)\"}.fa-flip-horizontal.fa-flip-vertical{transform:scale(-1)}:root .fa-flip-horizontal,:root .fa-flip-vertical,:root .fa-rotate-90,:root .fa-rotate-180,:root .fa-rotate-270{-webkit-filter:none;filter:none}.fa-stack{display:inline-block;height:2em;line-height:2em;position:relative;vertical-align:middle;width:2.5em}.fa-stack-1x,.fa-stack-2x{left:0;position:absolute;text-align:center;width:100%}.fa-stack-1x{line-height:inherit}.fa-stack-2x{font-size:2em}.fa-inverse{color:#fff}.fa-500px:before{content:\"\\f26e\"}.fa-accessible-icon:before{content:\"\\f368\"}.fa-accusoft:before{content:\"\\f369\"}.fa-acquisitions-incorporated:before{content:\"\\f6af\"}.fa-ad:before{content:\"\\f641\"}.fa-address-book:before{content:\"\\f2b9\"}.fa-address-card:before{content:\"\\f2bb\"}.fa-adjust:before{content:\"\\f042\"}.fa-adn:before{content:\"\\f170\"}.fa-adversal:before{content:\"\\f36a\"}.fa-affiliatetheme:before{content:\"\\f36b\"}.fa-air-freshener:before{content:\"\\f5d0\"}.fa-algolia:before{content:\"\\f36c\"}.fa-align-center:before{content:\"\\f037\"}.fa-align-justify:before{content:\"\\f039\"}.fa-align-left:before{content:\"\\f036\"}.fa-align-right:before{content:\"\\f038\"}.fa-alipay:before{content:\"\\f642\"}.fa-allergies:before{content:\"\\f461\"}.fa-amazon:before{content:\"\\f270\"}.fa-amazon-pay:before{content:\"\\f42c\"}.fa-ambulance:before{content:\"\\f0f9\"}.fa-american-sign-language-interpreting:before{content:\"\\f2a3\"}.fa-amilia:before{content:\"\\f36d\"}.fa-anchor:before{content:\"\\f13d\"}.fa-android:before{content:\"\\f17b\"}.fa-angellist:before{content:\"\\f209\"}.fa-angle-double-down:before{content:\"\\f103\"}.fa-angle-double-left:before{content:\"\\f100\"}.fa-angle-double-right:before{content:\"\\f101\"}.fa-angle-double-up:before{content:\"\\f102\"}.fa-angle-down:before{content:\"\\f107\"}.fa-angle-left:before{content:\"\\f104\"}.fa-angle-right:before{content:\"\\f105\"}.fa-angle-up:before{content:\"\\f106\"}.fa-angry:before{content:\"\\f556\"}.fa-angrycreative:before{content:\"\\f36e\"}.fa-angular:before{content:\"\\f420\"}.fa-ankh:before{content:\"\\f644\"}.fa-app-store:before{content:\"\\f36f\"}.fa-app-store-ios:before{content:\"\\f370\"}.fa-apper:before{content:\"\\f371\"}.fa-apple:before{content:\"\\f179\"}.fa-apple-alt:before{content:\"\\f5d1\"}.fa-apple-pay:before{content:\"\\f415\"}.fa-archive:before{content:\"\\f187\"}.fa-archway:before{content:\"\\f557\"}.fa-arrow-alt-circle-down:before{content:\"\\f358\"}.fa-arrow-alt-circle-left:before{content:\"\\f359\"}.fa-arrow-alt-circle-right:before{content:\"\\f35a\"}.fa-arrow-alt-circle-up:before{content:\"\\f35b\"}.fa-arrow-circle-down:before{content:\"\\f0ab\"}.fa-arrow-circle-left:before{content:\"\\f0a8\"}.fa-arrow-circle-right:before{content:\"\\f0a9\"}.fa-arrow-circle-up:before{content:\"\\f0aa\"}.fa-arrow-down:before{content:\"\\f063\"}.fa-arrow-left:before{content:\"\\f060\"}.fa-arrow-right:before{content:\"\\f061\"}.fa-arrow-up:before{content:\"\\f062\"}.fa-arrows-alt:before{content:\"\\f0b2\"}.fa-arrows-alt-h:before{content:\"\\f337\"}.fa-arrows-alt-v:before{content:\"\\f338\"}.fa-assistive-listening-systems:before{content:\"\\f2a2\"}.fa-asterisk:before{content:\"\\f069\"}.fa-asymmetrik:before{content:\"\\f372\"}.fa-at:before{content:\"\\f1fa\"}.fa-atlas:before{content:\"\\f558\"}.fa-atom:before{content:\"\\f5d2\"}.fa-audible:before{content:\"\\f373\"}.fa-audio-description:before{content:\"\\f29e\"}.fa-autoprefixer:before{content:\"\\f41c\"}.fa-avianex:before{content:\"\\f374\"}.fa-aviato:before{content:\"\\f421\"}.fa-award:before{content:\"\\f559\"}.fa-aws:before{content:\"\\f375\"}.fa-backspace:before{content:\"\\f55a\"}.fa-backward:before{content:\"\\f04a\"}.fa-balance-scale:before{content:\"\\f24e\"}.fa-ban:before{content:\"\\f05e\"}.fa-band-aid:before{content:\"\\f462\"}.fa-bandcamp:before{content:\"\\f2d5\"}.fa-barcode:before{content:\"\\f02a\"}.fa-bars:before{content:\"\\f0c9\"}.fa-baseball-ball:before{content:\"\\f433\"}.fa-basketball-ball:before{content:\"\\f434\"}.fa-bath:before{content:\"\\f2cd\"}.fa-battery-empty:before{content:\"\\f244\"}.fa-battery-full:before{content:\"\\f240\"}.fa-battery-half:before{content:\"\\f242\"}.fa-battery-quarter:before{content:\"\\f243\"}.fa-battery-three-quarters:before{content:\"\\f241\"}.fa-bed:before{content:\"\\f236\"}.fa-beer:before{content:\"\\f0fc\"}.fa-behance:before{content:\"\\f1b4\"}.fa-behance-square:before{content:\"\\f1b5\"}.fa-bell:before{content:\"\\f0f3\"}.fa-bell-slash:before{content:\"\\f1f6\"}.fa-bezier-curve:before{content:\"\\f55b\"}.fa-bible:before{content:\"\\f647\"}.fa-bicycle:before{content:\"\\f206\"}.fa-bimobject:before{content:\"\\f378\"}.fa-binoculars:before{content:\"\\f1e5\"}.fa-birthday-cake:before{content:\"\\f1fd\"}.fa-bitbucket:before{content:\"\\f171\"}.fa-bitcoin:before{content:\"\\f379\"}.fa-bity:before{content:\"\\f37a\"}.fa-black-tie:before{content:\"\\f27e\"}.fa-blackberry:before{content:\"\\f37b\"}.fa-blender:before{content:\"\\f517\"}.fa-blender-phone:before{content:\"\\f6b6\"}.fa-blind:before{content:\"\\f29d\"}.fa-blogger:before{content:\"\\f37c\"}.fa-blogger-b:before{content:\"\\f37d\"}.fa-bluetooth:before{content:\"\\f293\"}.fa-bluetooth-b:before{content:\"\\f294\"}.fa-bold:before{content:\"\\f032\"}.fa-bolt:before{content:\"\\f0e7\"}.fa-bomb:before{content:\"\\f1e2\"}.fa-bone:before{content:\"\\f5d7\"}.fa-bong:before{content:\"\\f55c\"}.fa-book:before{content:\"\\f02d\"}.fa-book-dead:before{content:\"\\f6b7\"}.fa-book-open:before{content:\"\\f518\"}.fa-book-reader:before{content:\"\\f5da\"}.fa-bookmark:before{content:\"\\f02e\"}.fa-bowling-ball:before{content:\"\\f436\"}.fa-box:before{content:\"\\f466\"}.fa-box-open:before{content:\"\\f49e\"}.fa-boxes:before{content:\"\\f468\"}.fa-braille:before{content:\"\\f2a1\"}.fa-brain:before{content:\"\\f5dc\"}.fa-briefcase:before{content:\"\\f0b1\"}.fa-briefcase-medical:before{content:\"\\f469\"}.fa-broadcast-tower:before{content:\"\\f519\"}.fa-broom:before{content:\"\\f51a\"}.fa-brush:before{content:\"\\f55d\"}.fa-btc:before{content:\"\\f15a\"}.fa-bug:before{content:\"\\f188\"}.fa-building:before{content:\"\\f1ad\"}.fa-bullhorn:before{content:\"\\f0a1\"}.fa-bullseye:before{content:\"\\f140\"}.fa-burn:before{content:\"\\f46a\"}.fa-buromobelexperte:before{content:\"\\f37f\"}.fa-bus:before{content:\"\\f207\"}.fa-bus-alt:before{content:\"\\f55e\"}.fa-business-time:before{content:\"\\f64a\"}.fa-buysellads:before{content:\"\\f20d\"}.fa-calculator:before{content:\"\\f1ec\"}.fa-calendar:before{content:\"\\f133\"}.fa-calendar-alt:before{content:\"\\f073\"}.fa-calendar-check:before{content:\"\\f274\"}.fa-calendar-minus:before{content:\"\\f272\"}.fa-calendar-plus:before{content:\"\\f271\"}.fa-calendar-times:before{content:\"\\f273\"}.fa-camera:before{content:\"\\f030\"}.fa-camera-retro:before{content:\"\\f083\"}.fa-campground:before{content:\"\\f6bb\"}.fa-cannabis:before{content:\"\\f55f\"}.fa-capsules:before{content:\"\\f46b\"}.fa-car:before{content:\"\\f1b9\"}.fa-car-alt:before{content:\"\\f5de\"}.fa-car-battery:before{content:\"\\f5df\"}.fa-car-crash:before{content:\"\\f5e1\"}.fa-car-side:before{content:\"\\f5e4\"}.fa-caret-down:before{content:\"\\f0d7\"}.fa-caret-left:before{content:\"\\f0d9\"}.fa-caret-right:before{content:\"\\f0da\"}.fa-caret-square-down:before{content:\"\\f150\"}.fa-caret-square-left:before{content:\"\\f191\"}.fa-caret-square-right:before{content:\"\\f152\"}.fa-caret-square-up:before{content:\"\\f151\"}.fa-caret-up:before{content:\"\\f0d8\"}.fa-cart-arrow-down:before{content:\"\\f218\"}.fa-cart-plus:before{content:\"\\f217\"}.fa-cat:before{content:\"\\f6be\"}.fa-cc-amazon-pay:before{content:\"\\f42d\"}.fa-cc-amex:before{content:\"\\f1f3\"}.fa-cc-apple-pay:before{content:\"\\f416\"}.fa-cc-diners-club:before{content:\"\\f24c\"}.fa-cc-discover:before{content:\"\\f1f2\"}.fa-cc-jcb:before{content:\"\\f24b\"}.fa-cc-mastercard:before{content:\"\\f1f1\"}.fa-cc-paypal:before{content:\"\\f1f4\"}.fa-cc-stripe:before{content:\"\\f1f5\"}.fa-cc-visa:before{content:\"\\f1f0\"}.fa-centercode:before{content:\"\\f380\"}.fa-certificate:before{content:\"\\f0a3\"}.fa-chair:before{content:\"\\f6c0\"}.fa-chalkboard:before{content:\"\\f51b\"}.fa-chalkboard-teacher:before{content:\"\\f51c\"}.fa-charging-station:before{content:\"\\f5e7\"}.fa-chart-area:before{content:\"\\f1fe\"}.fa-chart-bar:before{content:\"\\f080\"}.fa-chart-line:before{content:\"\\f201\"}.fa-chart-pie:before{content:\"\\f200\"}.fa-check:before{content:\"\\f00c\"}.fa-check-circle:before{content:\"\\f058\"}.fa-check-double:before{content:\"\\f560\"}.fa-check-square:before{content:\"\\f14a\"}.fa-chess:before{content:\"\\f439\"}.fa-chess-bishop:before{content:\"\\f43a\"}.fa-chess-board:before{content:\"\\f43c\"}.fa-chess-king:before{content:\"\\f43f\"}.fa-chess-knight:before{content:\"\\f441\"}.fa-chess-pawn:before{content:\"\\f443\"}.fa-chess-queen:before{content:\"\\f445\"}.fa-chess-rook:before{content:\"\\f447\"}.fa-chevron-circle-down:before{content:\"\\f13a\"}.fa-chevron-circle-left:before{content:\"\\f137\"}.fa-chevron-circle-right:before{content:\"\\f138\"}.fa-chevron-circle-up:before{content:\"\\f139\"}.fa-chevron-down:before{content:\"\\f078\"}.fa-chevron-left:before{content:\"\\f053\"}.fa-chevron-right:before{content:\"\\f054\"}.fa-chevron-up:before{content:\"\\f077\"}.fa-child:before{content:\"\\f1ae\"}.fa-chrome:before{content:\"\\f268\"}.fa-church:before{content:\"\\f51d\"}.fa-circle:before{content:\"\\f111\"}.fa-circle-notch:before{content:\"\\f1ce\"}.fa-city:before{content:\"\\f64f\"}.fa-clipboard:before{content:\"\\f328\"}.fa-clipboard-check:before{content:\"\\f46c\"}.fa-clipboard-list:before{content:\"\\f46d\"}.fa-clock:before{content:\"\\f017\"}.fa-clone:before{content:\"\\f24d\"}.fa-closed-captioning:before{content:\"\\f20a\"}.fa-cloud:before{content:\"\\f0c2\"}.fa-cloud-download-alt:before{content:\"\\f381\"}.fa-cloud-meatball:before{content:\"\\f73b\"}.fa-cloud-moon:before{content:\"\\f6c3\"}.fa-cloud-moon-rain:before{content:\"\\f73c\"}.fa-cloud-rain:before{content:\"\\f73d\"}.fa-cloud-showers-heavy:before{content:\"\\f740\"}.fa-cloud-sun:before{content:\"\\f6c4\"}.fa-cloud-sun-rain:before{content:\"\\f743\"}.fa-cloud-upload-alt:before{content:\"\\f382\"}.fa-cloudscale:before{content:\"\\f383\"}.fa-cloudsmith:before{content:\"\\f384\"}.fa-cloudversify:before{content:\"\\f385\"}.fa-cocktail:before{content:\"\\f561\"}.fa-code:before{content:\"\\f121\"}.fa-code-branch:before{content:\"\\f126\"}.fa-codepen:before{content:\"\\f1cb\"}.fa-codiepie:before{content:\"\\f284\"}.fa-coffee:before{content:\"\\f0f4\"}.fa-cog:before{content:\"\\f013\"}.fa-cogs:before{content:\"\\f085\"}.fa-coins:before{content:\"\\f51e\"}.fa-columns:before{content:\"\\f0db\"}.fa-comment:before{content:\"\\f075\"}.fa-comment-alt:before{content:\"\\f27a\"}.fa-comment-dollar:before{content:\"\\f651\"}.fa-comment-dots:before{content:\"\\f4ad\"}.fa-comment-slash:before{content:\"\\f4b3\"}.fa-comments:before{content:\"\\f086\"}.fa-comments-dollar:before{content:\"\\f653\"}.fa-compact-disc:before{content:\"\\f51f\"}.fa-compass:before{content:\"\\f14e\"}.fa-compress:before{content:\"\\f066\"}.fa-concierge-bell:before{content:\"\\f562\"}.fa-connectdevelop:before{content:\"\\f20e\"}.fa-contao:before{content:\"\\f26d\"}.fa-cookie:before{content:\"\\f563\"}.fa-cookie-bite:before{content:\"\\f564\"}.fa-copy:before{content:\"\\f0c5\"}.fa-copyright:before{content:\"\\f1f9\"}.fa-couch:before{content:\"\\f4b8\"}.fa-cpanel:before{content:\"\\f388\"}.fa-creative-commons:before{content:\"\\f25e\"}.fa-creative-commons-by:before{content:\"\\f4e7\"}.fa-creative-commons-nc:before{content:\"\\f4e8\"}.fa-creative-commons-nc-eu:before{content:\"\\f4e9\"}.fa-creative-commons-nc-jp:before{content:\"\\f4ea\"}.fa-creative-commons-nd:before{content:\"\\f4eb\"}.fa-creative-commons-pd:before{content:\"\\f4ec\"}.fa-creative-commons-pd-alt:before{content:\"\\f4ed\"}.fa-creative-commons-remix:before{content:\"\\f4ee\"}.fa-creative-commons-sa:before{content:\"\\f4ef\"}.fa-creative-commons-sampling:before{content:\"\\f4f0\"}.fa-creative-commons-sampling-plus:before{content:\"\\f4f1\"}.fa-creative-commons-share:before{content:\"\\f4f2\"}.fa-creative-commons-zero:before{content:\"\\f4f3\"}.fa-credit-card:before{content:\"\\f09d\"}.fa-critical-role:before{content:\"\\f6c9\"}.fa-crop:before{content:\"\\f125\"}.fa-crop-alt:before{content:\"\\f565\"}.fa-cross:before{content:\"\\f654\"}.fa-crosshairs:before{content:\"\\f05b\"}.fa-crow:before{content:\"\\f520\"}.fa-crown:before{content:\"\\f521\"}.fa-css3:before{content:\"\\f13c\"}.fa-css3-alt:before{content:\"\\f38b\"}.fa-cube:before{content:\"\\f1b2\"}.fa-cubes:before{content:\"\\f1b3\"}.fa-cut:before{content:\"\\f0c4\"}.fa-cuttlefish:before{content:\"\\f38c\"}.fa-d-and-d:before{content:\"\\f38d\"}.fa-d-and-d-beyond:before{content:\"\\f6ca\"}.fa-dashcube:before{content:\"\\f210\"}.fa-database:before{content:\"\\f1c0\"}.fa-deaf:before{content:\"\\f2a4\"}.fa-delicious:before{content:\"\\f1a5\"}.fa-democrat:before{content:\"\\f747\"}.fa-deploydog:before{content:\"\\f38e\"}.fa-deskpro:before{content:\"\\f38f\"}.fa-desktop:before{content:\"\\f108\"}.fa-dev:before{content:\"\\f6cc\"}.fa-deviantart:before{content:\"\\f1bd\"}.fa-dharmachakra:before{content:\"\\f655\"}.fa-diagnoses:before{content:\"\\f470\"}.fa-dice:before{content:\"\\f522\"}.fa-dice-d20:before{content:\"\\f6cf\"}.fa-dice-d6:before{content:\"\\f6d1\"}.fa-dice-five:before{content:\"\\f523\"}.fa-dice-four:before{content:\"\\f524\"}.fa-dice-one:before{content:\"\\f525\"}.fa-dice-six:before{content:\"\\f526\"}.fa-dice-three:before{content:\"\\f527\"}.fa-dice-two:before{content:\"\\f528\"}.fa-digg:before{content:\"\\f1a6\"}.fa-digital-ocean:before{content:\"\\f391\"}.fa-digital-tachograph:before{content:\"\\f566\"}.fa-directions:before{content:\"\\f5eb\"}.fa-discord:before{content:\"\\f392\"}.fa-discourse:before{content:\"\\f393\"}.fa-divide:before{content:\"\\f529\"}.fa-dizzy:before{content:\"\\f567\"}.fa-dna:before{content:\"\\f471\"}.fa-dochub:before{content:\"\\f394\"}.fa-docker:before{content:\"\\f395\"}.fa-dog:before{content:\"\\f6d3\"}.fa-dollar-sign:before{content:\"\\f155\"}.fa-dolly:before{content:\"\\f472\"}.fa-dolly-flatbed:before{content:\"\\f474\"}.fa-donate:before{content:\"\\f4b9\"}.fa-door-closed:before{content:\"\\f52a\"}.fa-door-open:before{content:\"\\f52b\"}.fa-dot-circle:before{content:\"\\f192\"}.fa-dove:before{content:\"\\f4ba\"}.fa-download:before{content:\"\\f019\"}.fa-draft2digital:before{content:\"\\f396\"}.fa-drafting-compass:before{content:\"\\f568\"}.fa-dragon:before{content:\"\\f6d5\"}.fa-draw-polygon:before{content:\"\\f5ee\"}.fa-dribbble:before{content:\"\\f17d\"}.fa-dribbble-square:before{content:\"\\f397\"}.fa-dropbox:before{content:\"\\f16b\"}.fa-drum:before{content:\"\\f569\"}.fa-drum-steelpan:before{content:\"\\f56a\"}.fa-drumstick-bite:before{content:\"\\f6d7\"}.fa-drupal:before{content:\"\\f1a9\"}.fa-dumbbell:before{content:\"\\f44b\"}.fa-dungeon:before{content:\"\\f6d9\"}.fa-dyalog:before{content:\"\\f399\"}.fa-earlybirds:before{content:\"\\f39a\"}.fa-ebay:before{content:\"\\f4f4\"}.fa-edge:before{content:\"\\f282\"}.fa-edit:before{content:\"\\f044\"}.fa-eject:before{content:\"\\f052\"}.fa-elementor:before{content:\"\\f430\"}.fa-ellipsis-h:before{content:\"\\f141\"}.fa-ellipsis-v:before{content:\"\\f142\"}.fa-ello:before{content:\"\\f5f1\"}.fa-ember:before{content:\"\\f423\"}.fa-empire:before{content:\"\\f1d1\"}.fa-envelope:before{content:\"\\f0e0\"}.fa-envelope-open:before{content:\"\\f2b6\"}.fa-envelope-open-text:before{content:\"\\f658\"}.fa-envelope-square:before{content:\"\\f199\"}.fa-envira:before{content:\"\\f299\"}.fa-equals:before{content:\"\\f52c\"}.fa-eraser:before{content:\"\\f12d\"}.fa-erlang:before{content:\"\\f39d\"}.fa-ethereum:before{content:\"\\f42e\"}.fa-etsy:before{content:\"\\f2d7\"}.fa-euro-sign:before{content:\"\\f153\"}.fa-exchange-alt:before{content:\"\\f362\"}.fa-exclamation:before{content:\"\\f12a\"}.fa-exclamation-circle:before{content:\"\\f06a\"}.fa-exclamation-triangle:before{content:\"\\f071\"}.fa-expand:before{content:\"\\f065\"}.fa-expand-arrows-alt:before{content:\"\\f31e\"}.fa-expeditedssl:before{content:\"\\f23e\"}.fa-external-link-alt:before{content:\"\\f35d\"}.fa-external-link-square-alt:before{content:\"\\f360\"}.fa-eye:before{content:\"\\f06e\"}.fa-eye-dropper:before{content:\"\\f1fb\"}.fa-eye-slash:before{content:\"\\f070\"}.fa-facebook:before{content:\"\\f09a\"}.fa-facebook-f:before{content:\"\\f39e\"}.fa-facebook-messenger:before{content:\"\\f39f\"}.fa-facebook-square:before{content:\"\\f082\"}.fa-fantasy-flight-games:before{content:\"\\f6dc\"}.fa-fast-backward:before{content:\"\\f049\"}.fa-fast-forward:before{content:\"\\f050\"}.fa-fax:before{content:\"\\f1ac\"}.fa-feather:before{content:\"\\f52d\"}.fa-feather-alt:before{content:\"\\f56b\"}.fa-female:before{content:\"\\f182\"}.fa-fighter-jet:before{content:\"\\f0fb\"}.fa-file:before{content:\"\\f15b\"}.fa-file-alt:before{content:\"\\f15c\"}.fa-file-archive:before{content:\"\\f1c6\"}.fa-file-audio:before{content:\"\\f1c7\"}.fa-file-code:before{content:\"\\f1c9\"}.fa-file-contract:before{content:\"\\f56c\"}.fa-file-csv:before{content:\"\\f6dd\"}.fa-file-download:before{content:\"\\f56d\"}.fa-file-excel:before{content:\"\\f1c3\"}.fa-file-export:before{content:\"\\f56e\"}.fa-file-image:before{content:\"\\f1c5\"}.fa-file-import:before{content:\"\\f56f\"}.fa-file-invoice:before{content:\"\\f570\"}.fa-file-invoice-dollar:before{content:\"\\f571\"}.fa-file-medical:before{content:\"\\f477\"}.fa-file-medical-alt:before{content:\"\\f478\"}.fa-file-pdf:before{content:\"\\f1c1\"}.fa-file-powerpoint:before{content:\"\\f1c4\"}.fa-file-prescription:before{content:\"\\f572\"}.fa-file-signature:before{content:\"\\f573\"}.fa-file-upload:before{content:\"\\f574\"}.fa-file-video:before{content:\"\\f1c8\"}.fa-file-word:before{content:\"\\f1c2\"}.fa-fill:before{content:\"\\f575\"}.fa-fill-drip:before{content:\"\\f576\"}.fa-film:before{content:\"\\f008\"}.fa-filter:before{content:\"\\f0b0\"}.fa-fingerprint:before{content:\"\\f577\"}.fa-fire:before{content:\"\\f06d\"}.fa-fire-extinguisher:before{content:\"\\f134\"}.fa-firefox:before{content:\"\\f269\"}.fa-first-aid:before{content:\"\\f479\"}.fa-first-order:before{content:\"\\f2b0\"}.fa-first-order-alt:before{content:\"\\f50a\"}.fa-firstdraft:before{content:\"\\f3a1\"}.fa-fish:before{content:\"\\f578\"}.fa-fist-raised:before{content:\"\\f6de\"}.fa-flag:before{content:\"\\f024\"}.fa-flag-checkered:before{content:\"\\f11e\"}.fa-flag-usa:before{content:\"\\f74d\"}.fa-flask:before{content:\"\\f0c3\"}.fa-flickr:before{content:\"\\f16e\"}.fa-flipboard:before{content:\"\\f44d\"}.fa-flushed:before{content:\"\\f579\"}.fa-fly:before{content:\"\\f417\"}.fa-folder:before{content:\"\\f07b\"}.fa-folder-minus:before{content:\"\\f65d\"}.fa-folder-open:before{content:\"\\f07c\"}.fa-folder-plus:before{content:\"\\f65e\"}.fa-font:before{content:\"\\f031\"}.fa-font-awesome:before{content:\"\\f2b4\"}.fa-font-awesome-alt:before{content:\"\\f35c\"}.fa-font-awesome-flag:before{content:\"\\f425\"}.fa-font-awesome-logo-full:before{content:\"\\f4e6\"}.fa-fonticons:before{content:\"\\f280\"}.fa-fonticons-fi:before{content:\"\\f3a2\"}.fa-football-ball:before{content:\"\\f44e\"}.fa-fort-awesome:before{content:\"\\f286\"}.fa-fort-awesome-alt:before{content:\"\\f3a3\"}.fa-forumbee:before{content:\"\\f211\"}.fa-forward:before{content:\"\\f04e\"}.fa-foursquare:before{content:\"\\f180\"}.fa-free-code-camp:before{content:\"\\f2c5\"}.fa-freebsd:before{content:\"\\f3a4\"}.fa-frog:before{content:\"\\f52e\"}.fa-frown:before{content:\"\\f119\"}.fa-frown-open:before{content:\"\\f57a\"}.fa-fulcrum:before{content:\"\\f50b\"}.fa-funnel-dollar:before{content:\"\\f662\"}.fa-futbol:before{content:\"\\f1e3\"}.fa-galactic-republic:before{content:\"\\f50c\"}.fa-galactic-senate:before{content:\"\\f50d\"}.fa-gamepad:before{content:\"\\f11b\"}.fa-gas-pump:before{content:\"\\f52f\"}.fa-gavel:before{content:\"\\f0e3\"}.fa-gem:before{content:\"\\f3a5\"}.fa-genderless:before{content:\"\\f22d\"}.fa-get-pocket:before{content:\"\\f265\"}.fa-gg:before{content:\"\\f260\"}.fa-gg-circle:before{content:\"\\f261\"}.fa-ghost:before{content:\"\\f6e2\"}.fa-gift:before{content:\"\\f06b\"}.fa-git:before{content:\"\\f1d3\"}.fa-git-square:before{content:\"\\f1d2\"}.fa-github:before{content:\"\\f09b\"}.fa-github-alt:before{content:\"\\f113\"}.fa-github-square:before{content:\"\\f092\"}.fa-gitkraken:before{content:\"\\f3a6\"}.fa-gitlab:before{content:\"\\f296\"}.fa-gitter:before{content:\"\\f426\"}.fa-glass-martini:before{content:\"\\f000\"}.fa-glass-martini-alt:before{content:\"\\f57b\"}.fa-glasses:before{content:\"\\f530\"}.fa-glide:before{content:\"\\f2a5\"}.fa-glide-g:before{content:\"\\f2a6\"}.fa-globe:before{content:\"\\f0ac\"}.fa-globe-africa:before{content:\"\\f57c\"}.fa-globe-americas:before{content:\"\\f57d\"}.fa-globe-asia:before{content:\"\\f57e\"}.fa-gofore:before{content:\"\\f3a7\"}.fa-golf-ball:before{content:\"\\f450\"}.fa-goodreads:before{content:\"\\f3a8\"}.fa-goodreads-g:before{content:\"\\f3a9\"}.fa-google:before{content:\"\\f1a0\"}.fa-google-drive:before{content:\"\\f3aa\"}.fa-google-play:before{content:\"\\f3ab\"}.fa-google-plus:before{content:\"\\f2b3\"}.fa-google-plus-g:before{content:\"\\f0d5\"}.fa-google-plus-square:before{content:\"\\f0d4\"}.fa-google-wallet:before{content:\"\\f1ee\"}.fa-gopuram:before{content:\"\\f664\"}.fa-graduation-cap:before{content:\"\\f19d\"}.fa-gratipay:before{content:\"\\f184\"}.fa-grav:before{content:\"\\f2d6\"}.fa-greater-than:before{content:\"\\f531\"}.fa-greater-than-equal:before{content:\"\\f532\"}.fa-grimace:before{content:\"\\f57f\"}.fa-grin:before{content:\"\\f580\"}.fa-grin-alt:before{content:\"\\f581\"}.fa-grin-beam:before{content:\"\\f582\"}.fa-grin-beam-sweat:before{content:\"\\f583\"}.fa-grin-hearts:before{content:\"\\f584\"}.fa-grin-squint:before{content:\"\\f585\"}.fa-grin-squint-tears:before{content:\"\\f586\"}.fa-grin-stars:before{content:\"\\f587\"}.fa-grin-tears:before{content:\"\\f588\"}.fa-grin-tongue:before{content:\"\\f589\"}.fa-grin-tongue-squint:before{content:\"\\f58a\"}.fa-grin-tongue-wink:before{content:\"\\f58b\"}.fa-grin-wink:before{content:\"\\f58c\"}.fa-grip-horizontal:before{content:\"\\f58d\"}.fa-grip-vertical:before{content:\"\\f58e\"}.fa-gripfire:before{content:\"\\f3ac\"}.fa-grunt:before{content:\"\\f3ad\"}.fa-gulp:before{content:\"\\f3ae\"}.fa-h-square:before{content:\"\\f0fd\"}.fa-hacker-news:before{content:\"\\f1d4\"}.fa-hacker-news-square:before{content:\"\\f3af\"}.fa-hackerrank:before{content:\"\\f5f7\"}.fa-hammer:before{content:\"\\f6e3\"}.fa-hamsa:before{content:\"\\f665\"}.fa-hand-holding:before{content:\"\\f4bd\"}.fa-hand-holding-heart:before{content:\"\\f4be\"}.fa-hand-holding-usd:before{content:\"\\f4c0\"}.fa-hand-lizard:before{content:\"\\f258\"}.fa-hand-paper:before{content:\"\\f256\"}.fa-hand-peace:before{content:\"\\f25b\"}.fa-hand-point-down:before{content:\"\\f0a7\"}.fa-hand-point-left:before{content:\"\\f0a5\"}.fa-hand-point-right:before{content:\"\\f0a4\"}.fa-hand-point-up:before{content:\"\\f0a6\"}.fa-hand-pointer:before{content:\"\\f25a\"}.fa-hand-rock:before{content:\"\\f255\"}.fa-hand-scissors:before{content:\"\\f257\"}.fa-hand-spock:before{content:\"\\f259\"}.fa-hands:before{content:\"\\f4c2\"}.fa-hands-helping:before{content:\"\\f4c4\"}.fa-handshake:before{content:\"\\f2b5\"}.fa-hanukiah:before{content:\"\\f6e6\"}.fa-hashtag:before{content:\"\\f292\"}.fa-hat-wizard:before{content:\"\\f6e8\"}.fa-haykal:before{content:\"\\f666\"}.fa-hdd:before{content:\"\\f0a0\"}.fa-heading:before{content:\"\\f1dc\"}.fa-headphones:before{content:\"\\f025\"}.fa-headphones-alt:before{content:\"\\f58f\"}.fa-headset:before{content:\"\\f590\"}.fa-heart:before{content:\"\\f004\"}.fa-heartbeat:before{content:\"\\f21e\"}.fa-helicopter:before{content:\"\\f533\"}.fa-highlighter:before{content:\"\\f591\"}.fa-hiking:before{content:\"\\f6ec\"}.fa-hippo:before{content:\"\\f6ed\"}.fa-hips:before{content:\"\\f452\"}.fa-hire-a-helper:before{content:\"\\f3b0\"}.fa-history:before{content:\"\\f1da\"}.fa-hockey-puck:before{content:\"\\f453\"}.fa-home:before{content:\"\\f015\"}.fa-hooli:before{content:\"\\f427\"}.fa-hornbill:before{content:\"\\f592\"}.fa-horse:before{content:\"\\f6f0\"}.fa-hospital:before{content:\"\\f0f8\"}.fa-hospital-alt:before{content:\"\\f47d\"}.fa-hospital-symbol:before{content:\"\\f47e\"}.fa-hot-tub:before{content:\"\\f593\"}.fa-hotel:before{content:\"\\f594\"}.fa-hotjar:before{content:\"\\f3b1\"}.fa-hourglass:before{content:\"\\f254\"}.fa-hourglass-end:before{content:\"\\f253\"}.fa-hourglass-half:before{content:\"\\f252\"}.fa-hourglass-start:before{content:\"\\f251\"}.fa-house-damage:before{content:\"\\f6f1\"}.fa-houzz:before{content:\"\\f27c\"}.fa-hryvnia:before{content:\"\\f6f2\"}.fa-html5:before{content:\"\\f13b\"}.fa-hubspot:before{content:\"\\f3b2\"}.fa-i-cursor:before{content:\"\\f246\"}.fa-id-badge:before{content:\"\\f2c1\"}.fa-id-card:before{content:\"\\f2c2\"}.fa-id-card-alt:before{content:\"\\f47f\"}.fa-image:before{content:\"\\f03e\"}.fa-images:before{content:\"\\f302\"}.fa-imdb:before{content:\"\\f2d8\"}.fa-inbox:before{content:\"\\f01c\"}.fa-indent:before{content:\"\\f03c\"}.fa-industry:before{content:\"\\f275\"}.fa-infinity:before{content:\"\\f534\"}.fa-info:before{content:\"\\f129\"}.fa-info-circle:before{content:\"\\f05a\"}.fa-instagram:before{content:\"\\f16d\"}.fa-internet-explorer:before{content:\"\\f26b\"}.fa-ioxhost:before{content:\"\\f208\"}.fa-italic:before{content:\"\\f033\"}.fa-itunes:before{content:\"\\f3b4\"}.fa-itunes-note:before{content:\"\\f3b5\"}.fa-java:before{content:\"\\f4e4\"}.fa-jedi:before{content:\"\\f669\"}.fa-jedi-order:before{content:\"\\f50e\"}.fa-jenkins:before{content:\"\\f3b6\"}.fa-joget:before{content:\"\\f3b7\"}.fa-joint:before{content:\"\\f595\"}.fa-joomla:before{content:\"\\f1aa\"}.fa-journal-whills:before{content:\"\\f66a\"}.fa-js:before{content:\"\\f3b8\"}.fa-js-square:before{content:\"\\f3b9\"}.fa-jsfiddle:before{content:\"\\f1cc\"}.fa-kaaba:before{content:\"\\f66b\"}.fa-kaggle:before{content:\"\\f5fa\"}.fa-key:before{content:\"\\f084\"}.fa-keybase:before{content:\"\\f4f5\"}.fa-keyboard:before{content:\"\\f11c\"}.fa-keycdn:before{content:\"\\f3ba\"}.fa-khanda:before{content:\"\\f66d\"}.fa-kickstarter:before{content:\"\\f3bb\"}.fa-kickstarter-k:before{content:\"\\f3bc\"}.fa-kiss:before{content:\"\\f596\"}.fa-kiss-beam:before{content:\"\\f597\"}.fa-kiss-wink-heart:before{content:\"\\f598\"}.fa-kiwi-bird:before{content:\"\\f535\"}.fa-korvue:before{content:\"\\f42f\"}.fa-landmark:before{content:\"\\f66f\"}.fa-language:before{content:\"\\f1ab\"}.fa-laptop:before{content:\"\\f109\"}.fa-laptop-code:before{content:\"\\f5fc\"}.fa-laravel:before{content:\"\\f3bd\"}.fa-lastfm:before{content:\"\\f202\"}.fa-lastfm-square:before{content:\"\\f203\"}.fa-laugh:before{content:\"\\f599\"}.fa-laugh-beam:before{content:\"\\f59a\"}.fa-laugh-squint:before{content:\"\\f59b\"}.fa-laugh-wink:before{content:\"\\f59c\"}.fa-layer-group:before{content:\"\\f5fd\"}.fa-leaf:before{content:\"\\f06c\"}.fa-leanpub:before{content:\"\\f212\"}.fa-lemon:before{content:\"\\f094\"}.fa-less:before{content:\"\\f41d\"}.fa-less-than:before{content:\"\\f536\"}.fa-less-than-equal:before{content:\"\\f537\"}.fa-level-down-alt:before{content:\"\\f3be\"}.fa-level-up-alt:before{content:\"\\f3bf\"}.fa-life-ring:before{content:\"\\f1cd\"}.fa-lightbulb:before{content:\"\\f0eb\"}.fa-line:before{content:\"\\f3c0\"}.fa-link:before{content:\"\\f0c1\"}.fa-linkedin:before{content:\"\\f08c\"}.fa-linkedin-in:before{content:\"\\f0e1\"}.fa-linode:before{content:\"\\f2b8\"}.fa-linux:before{content:\"\\f17c\"}.fa-lira-sign:before{content:\"\\f195\"}.fa-list:before{content:\"\\f03a\"}.fa-list-alt:before{content:\"\\f022\"}.fa-list-ol:before{content:\"\\f0cb\"}.fa-list-ul:before{content:\"\\f0ca\"}.fa-location-arrow:before{content:\"\\f124\"}.fa-lock:before{content:\"\\f023\"}.fa-lock-open:before{content:\"\\f3c1\"}.fa-long-arrow-alt-down:before{content:\"\\f309\"}.fa-long-arrow-alt-left:before{content:\"\\f30a\"}.fa-long-arrow-alt-right:before{content:\"\\f30b\"}.fa-long-arrow-alt-up:before{content:\"\\f30c\"}.fa-low-vision:before{content:\"\\f2a8\"}.fa-luggage-cart:before{content:\"\\f59d\"}.fa-lyft:before{content:\"\\f3c3\"}.fa-magento:before{content:\"\\f3c4\"}.fa-magic:before{content:\"\\f0d0\"}.fa-magnet:before{content:\"\\f076\"}.fa-mail-bulk:before{content:\"\\f674\"}.fa-mailchimp:before{content:\"\\f59e\"}.fa-male:before{content:\"\\f183\"}.fa-mandalorian:before{content:\"\\f50f\"}.fa-map:before{content:\"\\f279\"}.fa-map-marked:before{content:\"\\f59f\"}.fa-map-marked-alt:before{content:\"\\f5a0\"}.fa-map-marker:before{content:\"\\f041\"}.fa-map-marker-alt:before{content:\"\\f3c5\"}.fa-map-pin:before{content:\"\\f276\"}.fa-map-signs:before{content:\"\\f277\"}.fa-markdown:before{content:\"\\f60f\"}.fa-marker:before{content:\"\\f5a1\"}.fa-mars:before{content:\"\\f222\"}.fa-mars-double:before{content:\"\\f227\"}.fa-mars-stroke:before{content:\"\\f229\"}.fa-mars-stroke-h:before{content:\"\\f22b\"}.fa-mars-stroke-v:before{content:\"\\f22a\"}.fa-mask:before{content:\"\\f6fa\"}.fa-mastodon:before{content:\"\\f4f6\"}.fa-maxcdn:before{content:\"\\f136\"}.fa-medal:before{content:\"\\f5a2\"}.fa-medapps:before{content:\"\\f3c6\"}.fa-medium:before{content:\"\\f23a\"}.fa-medium-m:before{content:\"\\f3c7\"}.fa-medkit:before{content:\"\\f0fa\"}.fa-medrt:before{content:\"\\f3c8\"}.fa-meetup:before{content:\"\\f2e0\"}.fa-megaport:before{content:\"\\f5a3\"}.fa-meh:before{content:\"\\f11a\"}.fa-meh-blank:before{content:\"\\f5a4\"}.fa-meh-rolling-eyes:before{content:\"\\f5a5\"}.fa-memory:before{content:\"\\f538\"}.fa-menorah:before{content:\"\\f676\"}.fa-mercury:before{content:\"\\f223\"}.fa-meteor:before{content:\"\\f753\"}.fa-microchip:before{content:\"\\f2db\"}.fa-microphone:before{content:\"\\f130\"}.fa-microphone-alt:before{content:\"\\f3c9\"}.fa-microphone-alt-slash:before{content:\"\\f539\"}.fa-microphone-slash:before{content:\"\\f131\"}.fa-microscope:before{content:\"\\f610\"}.fa-microsoft:before{content:\"\\f3ca\"}.fa-minus:before{content:\"\\f068\"}.fa-minus-circle:before{content:\"\\f056\"}.fa-minus-square:before{content:\"\\f146\"}.fa-mix:before{content:\"\\f3cb\"}.fa-mixcloud:before{content:\"\\f289\"}.fa-mizuni:before{content:\"\\f3cc\"}.fa-mobile:before{content:\"\\f10b\"}.fa-mobile-alt:before{content:\"\\f3cd\"}.fa-modx:before{content:\"\\f285\"}.fa-monero:before{content:\"\\f3d0\"}.fa-money-bill:before{content:\"\\f0d6\"}.fa-money-bill-alt:before{content:\"\\f3d1\"}.fa-money-bill-wave:before{content:\"\\f53a\"}.fa-money-bill-wave-alt:before{content:\"\\f53b\"}.fa-money-check:before{content:\"\\f53c\"}.fa-money-check-alt:before{content:\"\\f53d\"}.fa-monument:before{content:\"\\f5a6\"}.fa-moon:before{content:\"\\f186\"}.fa-mortar-pestle:before{content:\"\\f5a7\"}.fa-mosque:before{content:\"\\f678\"}.fa-motorcycle:before{content:\"\\f21c\"}.fa-mountain:before{content:\"\\f6fc\"}.fa-mouse-pointer:before{content:\"\\f245\"}.fa-music:before{content:\"\\f001\"}.fa-napster:before{content:\"\\f3d2\"}.fa-neos:before{content:\"\\f612\"}.fa-network-wired:before{content:\"\\f6ff\"}.fa-neuter:before{content:\"\\f22c\"}.fa-newspaper:before{content:\"\\f1ea\"}.fa-nimblr:before{content:\"\\f5a8\"}.fa-nintendo-switch:before{content:\"\\f418\"}.fa-node:before{content:\"\\f419\"}.fa-node-js:before{content:\"\\f3d3\"}.fa-not-equal:before{content:\"\\f53e\"}.fa-notes-medical:before{content:\"\\f481\"}.fa-npm:before{content:\"\\f3d4\"}.fa-ns8:before{content:\"\\f3d5\"}.fa-nutritionix:before{content:\"\\f3d6\"}.fa-object-group:before{content:\"\\f247\"}.fa-object-ungroup:before{content:\"\\f248\"}.fa-odnoklassniki:before{content:\"\\f263\"}.fa-odnoklassniki-square:before{content:\"\\f264\"}.fa-oil-can:before{content:\"\\f613\"}.fa-old-republic:before{content:\"\\f510\"}.fa-om:before{content:\"\\f679\"}.fa-opencart:before{content:\"\\f23d\"}.fa-openid:before{content:\"\\f19b\"}.fa-opera:before{content:\"\\f26a\"}.fa-optin-monster:before{content:\"\\f23c\"}.fa-osi:before{content:\"\\f41a\"}.fa-otter:before{content:\"\\f700\"}.fa-outdent:before{content:\"\\f03b\"}.fa-page4:before{content:\"\\f3d7\"}.fa-pagelines:before{content:\"\\f18c\"}.fa-paint-brush:before{content:\"\\f1fc\"}.fa-paint-roller:before{content:\"\\f5aa\"}.fa-palette:before{content:\"\\f53f\"}.fa-palfed:before{content:\"\\f3d8\"}.fa-pallet:before{content:\"\\f482\"}.fa-paper-plane:before{content:\"\\f1d8\"}.fa-paperclip:before{content:\"\\f0c6\"}.fa-parachute-box:before{content:\"\\f4cd\"}.fa-paragraph:before{content:\"\\f1dd\"}.fa-parking:before{content:\"\\f540\"}.fa-passport:before{content:\"\\f5ab\"}.fa-pastafarianism:before{content:\"\\f67b\"}.fa-paste:before{content:\"\\f0ea\"}.fa-patreon:before{content:\"\\f3d9\"}.fa-pause:before{content:\"\\f04c\"}.fa-pause-circle:before{content:\"\\f28b\"}.fa-paw:before{content:\"\\f1b0\"}.fa-paypal:before{content:\"\\f1ed\"}.fa-peace:before{content:\"\\f67c\"}.fa-pen:before{content:\"\\f304\"}.fa-pen-alt:before{content:\"\\f305\"}.fa-pen-fancy:before{content:\"\\f5ac\"}.fa-pen-nib:before{content:\"\\f5ad\"}.fa-pen-square:before{content:\"\\f14b\"}.fa-pencil-alt:before{content:\"\\f303\"}.fa-pencil-ruler:before{content:\"\\f5ae\"}.fa-penny-arcade:before{content:\"\\f704\"}.fa-people-carry:before{content:\"\\f4ce\"}.fa-percent:before{content:\"\\f295\"}.fa-percentage:before{content:\"\\f541\"}.fa-periscope:before{content:\"\\f3da\"}.fa-person-booth:before{content:\"\\f756\"}.fa-phabricator:before{content:\"\\f3db\"}.fa-phoenix-framework:before{content:\"\\f3dc\"}.fa-phoenix-squadron:before{content:\"\\f511\"}.fa-phone:before{content:\"\\f095\"}.fa-phone-slash:before{content:\"\\f3dd\"}.fa-phone-square:before{content:\"\\f098\"}.fa-phone-volume:before{content:\"\\f2a0\"}.fa-php:before{content:\"\\f457\"}.fa-pied-piper:before{content:\"\\f2ae\"}.fa-pied-piper-alt:before{content:\"\\f1a8\"}.fa-pied-piper-hat:before{content:\"\\f4e5\"}.fa-pied-piper-pp:before{content:\"\\f1a7\"}.fa-piggy-bank:before{content:\"\\f4d3\"}.fa-pills:before{content:\"\\f484\"}.fa-pinterest:before{content:\"\\f0d2\"}.fa-pinterest-p:before{content:\"\\f231\"}.fa-pinterest-square:before{content:\"\\f0d3\"}.fa-place-of-worship:before{content:\"\\f67f\"}.fa-plane:before{content:\"\\f072\"}.fa-plane-arrival:before{content:\"\\f5af\"}.fa-plane-departure:before{content:\"\\f5b0\"}.fa-play:before{content:\"\\f04b\"}.fa-play-circle:before{content:\"\\f144\"}.fa-playstation:before{content:\"\\f3df\"}.fa-plug:before{content:\"\\f1e6\"}.fa-plus:before{content:\"\\f067\"}.fa-plus-circle:before{content:\"\\f055\"}.fa-plus-square:before{content:\"\\f0fe\"}.fa-podcast:before{content:\"\\f2ce\"}.fa-poll:before{content:\"\\f681\"}.fa-poll-h:before{content:\"\\f682\"}.fa-poo:before{content:\"\\f2fe\"}.fa-poo-storm:before{content:\"\\f75a\"}.fa-poop:before{content:\"\\f619\"}.fa-portrait:before{content:\"\\f3e0\"}.fa-pound-sign:before{content:\"\\f154\"}.fa-power-off:before{content:\"\\f011\"}.fa-pray:before{content:\"\\f683\"}.fa-praying-hands:before{content:\"\\f684\"}.fa-prescription:before{content:\"\\f5b1\"}.fa-prescription-bottle:before{content:\"\\f485\"}.fa-prescription-bottle-alt:before{content:\"\\f486\"}.fa-print:before{content:\"\\f02f\"}.fa-procedures:before{content:\"\\f487\"}.fa-product-hunt:before{content:\"\\f288\"}.fa-project-diagram:before{content:\"\\f542\"}.fa-pushed:before{content:\"\\f3e1\"}.fa-puzzle-piece:before{content:\"\\f12e\"}.fa-python:before{content:\"\\f3e2\"}.fa-qq:before{content:\"\\f1d6\"}.fa-qrcode:before{content:\"\\f029\"}.fa-question:before{content:\"\\f128\"}.fa-question-circle:before{content:\"\\f059\"}.fa-quidditch:before{content:\"\\f458\"}.fa-quinscape:before{content:\"\\f459\"}.fa-quora:before{content:\"\\f2c4\"}.fa-quote-left:before{content:\"\\f10d\"}.fa-quote-right:before{content:\"\\f10e\"}.fa-quran:before{content:\"\\f687\"}.fa-r-project:before{content:\"\\f4f7\"}.fa-rainbow:before{content:\"\\f75b\"}.fa-random:before{content:\"\\f074\"}.fa-ravelry:before{content:\"\\f2d9\"}.fa-react:before{content:\"\\f41b\"}.fa-reacteurope:before{content:\"\\f75d\"}.fa-readme:before{content:\"\\f4d5\"}.fa-rebel:before{content:\"\\f1d0\"}.fa-receipt:before{content:\"\\f543\"}.fa-recycle:before{content:\"\\f1b8\"}.fa-red-river:before{content:\"\\f3e3\"}.fa-reddit:before{content:\"\\f1a1\"}.fa-reddit-alien:before{content:\"\\f281\"}.fa-reddit-square:before{content:\"\\f1a2\"}.fa-redo:before{content:\"\\f01e\"}.fa-redo-alt:before{content:\"\\f2f9\"}.fa-registered:before{content:\"\\f25d\"}.fa-renren:before{content:\"\\f18b\"}.fa-reply:before{content:\"\\f3e5\"}.fa-reply-all:before{content:\"\\f122\"}.fa-replyd:before{content:\"\\f3e6\"}.fa-republican:before{content:\"\\f75e\"}.fa-researchgate:before{content:\"\\f4f8\"}.fa-resolving:before{content:\"\\f3e7\"}.fa-retweet:before{content:\"\\f079\"}.fa-rev:before{content:\"\\f5b2\"}.fa-ribbon:before{content:\"\\f4d6\"}.fa-ring:before{content:\"\\f70b\"}.fa-road:before{content:\"\\f018\"}.fa-robot:before{content:\"\\f544\"}.fa-rocket:before{content:\"\\f135\"}.fa-rocketchat:before{content:\"\\f3e8\"}.fa-rockrms:before{content:\"\\f3e9\"}.fa-route:before{content:\"\\f4d7\"}.fa-rss:before{content:\"\\f09e\"}.fa-rss-square:before{content:\"\\f143\"}.fa-ruble-sign:before{content:\"\\f158\"}.fa-ruler:before{content:\"\\f545\"}.fa-ruler-combined:before{content:\"\\f546\"}.fa-ruler-horizontal:before{content:\"\\f547\"}.fa-ruler-vertical:before{content:\"\\f548\"}.fa-running:before{content:\"\\f70c\"}.fa-rupee-sign:before{content:\"\\f156\"}.fa-sad-cry:before{content:\"\\f5b3\"}.fa-sad-tear:before{content:\"\\f5b4\"}.fa-safari:before{content:\"\\f267\"}.fa-sass:before{content:\"\\f41e\"}.fa-save:before{content:\"\\f0c7\"}.fa-schlix:before{content:\"\\f3ea\"}.fa-school:before{content:\"\\f549\"}.fa-screwdriver:before{content:\"\\f54a\"}.fa-scribd:before{content:\"\\f28a\"}.fa-scroll:before{content:\"\\f70e\"}.fa-search:before{content:\"\\f002\"}.fa-search-dollar:before{content:\"\\f688\"}.fa-search-location:before{content:\"\\f689\"}.fa-search-minus:before{content:\"\\f010\"}.fa-search-plus:before{content:\"\\f00e\"}.fa-searchengin:before{content:\"\\f3eb\"}.fa-seedling:before{content:\"\\f4d8\"}.fa-sellcast:before{content:\"\\f2da\"}.fa-sellsy:before{content:\"\\f213\"}.fa-server:before{content:\"\\f233\"}.fa-servicestack:before{content:\"\\f3ec\"}.fa-shapes:before{content:\"\\f61f\"}.fa-share:before{content:\"\\f064\"}.fa-share-alt:before{content:\"\\f1e0\"}.fa-share-alt-square:before{content:\"\\f1e1\"}.fa-share-square:before{content:\"\\f14d\"}.fa-shekel-sign:before{content:\"\\f20b\"}.fa-shield-alt:before{content:\"\\f3ed\"}.fa-ship:before{content:\"\\f21a\"}.fa-shipping-fast:before{content:\"\\f48b\"}.fa-shirtsinbulk:before{content:\"\\f214\"}.fa-shoe-prints:before{content:\"\\f54b\"}.fa-shopping-bag:before{content:\"\\f290\"}.fa-shopping-basket:before{content:\"\\f291\"}.fa-shopping-cart:before{content:\"\\f07a\"}.fa-shopware:before{content:\"\\f5b5\"}.fa-shower:before{content:\"\\f2cc\"}.fa-shuttle-van:before{content:\"\\f5b6\"}.fa-sign:before{content:\"\\f4d9\"}.fa-sign-in-alt:before{content:\"\\f2f6\"}.fa-sign-language:before{content:\"\\f2a7\"}.fa-sign-out-alt:before{content:\"\\f2f5\"}.fa-signal:before{content:\"\\f012\"}.fa-signature:before{content:\"\\f5b7\"}.fa-simplybuilt:before{content:\"\\f215\"}.fa-sistrix:before{content:\"\\f3ee\"}.fa-sitemap:before{content:\"\\f0e8\"}.fa-sith:before{content:\"\\f512\"}.fa-skull:before{content:\"\\f54c\"}.fa-skull-crossbones:before{content:\"\\f714\"}.fa-skyatlas:before{content:\"\\f216\"}.fa-skype:before{content:\"\\f17e\"}.fa-slack:before{content:\"\\f198\"}.fa-slack-hash:before{content:\"\\f3ef\"}.fa-slash:before{content:\"\\f715\"}.fa-sliders-h:before{content:\"\\f1de\"}.fa-slideshare:before{content:\"\\f1e7\"}.fa-smile:before{content:\"\\f118\"}.fa-smile-beam:before{content:\"\\f5b8\"}.fa-smile-wink:before{content:\"\\f4da\"}.fa-smog:before{content:\"\\f75f\"}.fa-smoking:before{content:\"\\f48d\"}.fa-smoking-ban:before{content:\"\\f54d\"}.fa-snapchat:before{content:\"\\f2ab\"}.fa-snapchat-ghost:before{content:\"\\f2ac\"}.fa-snapchat-square:before{content:\"\\f2ad\"}.fa-snowflake:before{content:\"\\f2dc\"}.fa-socks:before{content:\"\\f696\"}.fa-solar-panel:before{content:\"\\f5ba\"}.fa-sort:before{content:\"\\f0dc\"}.fa-sort-alpha-down:before{content:\"\\f15d\"}.fa-sort-alpha-up:before{content:\"\\f15e\"}.fa-sort-amount-down:before{content:\"\\f160\"}.fa-sort-amount-up:before{content:\"\\f161\"}.fa-sort-down:before{content:\"\\f0dd\"}.fa-sort-numeric-down:before{content:\"\\f162\"}.fa-sort-numeric-up:before{content:\"\\f163\"}.fa-sort-up:before{content:\"\\f0de\"}.fa-soundcloud:before{content:\"\\f1be\"}.fa-spa:before{content:\"\\f5bb\"}.fa-space-shuttle:before{content:\"\\f197\"}.fa-speakap:before{content:\"\\f3f3\"}.fa-spider:before{content:\"\\f717\"}.fa-spinner:before{content:\"\\f110\"}.fa-splotch:before{content:\"\\f5bc\"}.fa-spotify:before{content:\"\\f1bc\"}.fa-spray-can:before{content:\"\\f5bd\"}.fa-square:before{content:\"\\f0c8\"}.fa-square-full:before{content:\"\\f45c\"}.fa-square-root-alt:before{content:\"\\f698\"}.fa-squarespace:before{content:\"\\f5be\"}.fa-stack-exchange:before{content:\"\\f18d\"}.fa-stack-overflow:before{content:\"\\f16c\"}.fa-stamp:before{content:\"\\f5bf\"}.fa-star:before{content:\"\\f005\"}.fa-star-and-crescent:before{content:\"\\f699\"}.fa-star-half:before{content:\"\\f089\"}.fa-star-half-alt:before{content:\"\\f5c0\"}.fa-star-of-david:before{content:\"\\f69a\"}.fa-star-of-life:before{content:\"\\f621\"}.fa-staylinked:before{content:\"\\f3f5\"}.fa-steam:before{content:\"\\f1b6\"}.fa-steam-square:before{content:\"\\f1b7\"}.fa-steam-symbol:before{content:\"\\f3f6\"}.fa-step-backward:before{content:\"\\f048\"}.fa-step-forward:before{content:\"\\f051\"}.fa-stethoscope:before{content:\"\\f0f1\"}.fa-sticker-mule:before{content:\"\\f3f7\"}.fa-sticky-note:before{content:\"\\f249\"}.fa-stop:before{content:\"\\f04d\"}.fa-stop-circle:before{content:\"\\f28d\"}.fa-stopwatch:before{content:\"\\f2f2\"}.fa-store:before{content:\"\\f54e\"}.fa-store-alt:before{content:\"\\f54f\"}.fa-strava:before{content:\"\\f428\"}.fa-stream:before{content:\"\\f550\"}.fa-street-view:before{content:\"\\f21d\"}.fa-strikethrough:before{content:\"\\f0cc\"}.fa-stripe:before{content:\"\\f429\"}.fa-stripe-s:before{content:\"\\f42a\"}.fa-stroopwafel:before{content:\"\\f551\"}.fa-studiovinari:before{content:\"\\f3f8\"}.fa-stumbleupon:before{content:\"\\f1a4\"}.fa-stumbleupon-circle:before{content:\"\\f1a3\"}.fa-subscript:before{content:\"\\f12c\"}.fa-subway:before{content:\"\\f239\"}.fa-suitcase:before{content:\"\\f0f2\"}.fa-suitcase-rolling:before{content:\"\\f5c1\"}.fa-sun:before{content:\"\\f185\"}.fa-superpowers:before{content:\"\\f2dd\"}.fa-superscript:before{content:\"\\f12b\"}.fa-supple:before{content:\"\\f3f9\"}.fa-surprise:before{content:\"\\f5c2\"}.fa-swatchbook:before{content:\"\\f5c3\"}.fa-swimmer:before{content:\"\\f5c4\"}.fa-swimming-pool:before{content:\"\\f5c5\"}.fa-synagogue:before{content:\"\\f69b\"}.fa-sync:before{content:\"\\f021\"}.fa-sync-alt:before{content:\"\\f2f1\"}.fa-syringe:before{content:\"\\f48e\"}.fa-table:before{content:\"\\f0ce\"}.fa-table-tennis:before{content:\"\\f45d\"}.fa-tablet:before{content:\"\\f10a\"}.fa-tablet-alt:before{content:\"\\f3fa\"}.fa-tablets:before{content:\"\\f490\"}.fa-tachometer-alt:before{content:\"\\f3fd\"}.fa-tag:before{content:\"\\f02b\"}.fa-tags:before{content:\"\\f02c\"}.fa-tape:before{content:\"\\f4db\"}.fa-tasks:before{content:\"\\f0ae\"}.fa-taxi:before{content:\"\\f1ba\"}.fa-teamspeak:before{content:\"\\f4f9\"}.fa-teeth:before{content:\"\\f62e\"}.fa-teeth-open:before{content:\"\\f62f\"}.fa-telegram:before{content:\"\\f2c6\"}.fa-telegram-plane:before{content:\"\\f3fe\"}.fa-temperature-high:before{content:\"\\f769\"}.fa-temperature-low:before{content:\"\\f76b\"}.fa-tencent-weibo:before{content:\"\\f1d5\"}.fa-terminal:before{content:\"\\f120\"}.fa-text-height:before{content:\"\\f034\"}.fa-text-width:before{content:\"\\f035\"}.fa-th:before{content:\"\\f00a\"}.fa-th-large:before{content:\"\\f009\"}.fa-th-list:before{content:\"\\f00b\"}.fa-the-red-yeti:before{content:\"\\f69d\"}.fa-theater-masks:before{content:\"\\f630\"}.fa-themeco:before{content:\"\\f5c6\"}.fa-themeisle:before{content:\"\\f2b2\"}.fa-thermometer:before{content:\"\\f491\"}.fa-thermometer-empty:before{content:\"\\f2cb\"}.fa-thermometer-full:before{content:\"\\f2c7\"}.fa-thermometer-half:before{content:\"\\f2c9\"}.fa-thermometer-quarter:before{content:\"\\f2ca\"}.fa-thermometer-three-quarters:before{content:\"\\f2c8\"}.fa-think-peaks:before{content:\"\\f731\"}.fa-thumbs-down:before{content:\"\\f165\"}.fa-thumbs-up:before{content:\"\\f164\"}.fa-thumbtack:before{content:\"\\f08d\"}.fa-ticket-alt:before{content:\"\\f3ff\"}.fa-times:before{content:\"\\f00d\"}.fa-times-circle:before{content:\"\\f057\"}.fa-tint:before{content:\"\\f043\"}.fa-tint-slash:before{content:\"\\f5c7\"}.fa-tired:before{content:\"\\f5c8\"}.fa-toggle-off:before{content:\"\\f204\"}.fa-toggle-on:before{content:\"\\f205\"}.fa-toilet-paper:before{content:\"\\f71e\"}.fa-toolbox:before{content:\"\\f552\"}.fa-tooth:before{content:\"\\f5c9\"}.fa-torah:before{content:\"\\f6a0\"}.fa-torii-gate:before{content:\"\\f6a1\"}.fa-tractor:before{content:\"\\f722\"}.fa-trade-federation:before{content:\"\\f513\"}.fa-trademark:before{content:\"\\f25c\"}.fa-traffic-light:before{content:\"\\f637\"}.fa-train:before{content:\"\\f238\"}.fa-transgender:before{content:\"\\f224\"}.fa-transgender-alt:before{content:\"\\f225\"}.fa-trash:before{content:\"\\f1f8\"}.fa-trash-alt:before{content:\"\\f2ed\"}.fa-tree:before{content:\"\\f1bb\"}.fa-trello:before{content:\"\\f181\"}.fa-tripadvisor:before{content:\"\\f262\"}.fa-trophy:before{content:\"\\f091\"}.fa-truck:before{content:\"\\f0d1\"}.fa-truck-loading:before{content:\"\\f4de\"}.fa-truck-monster:before{content:\"\\f63b\"}.fa-truck-moving:before{content:\"\\f4df\"}.fa-truck-pickup:before{content:\"\\f63c\"}.fa-tshirt:before{content:\"\\f553\"}.fa-tty:before{content:\"\\f1e4\"}.fa-tumblr:before{content:\"\\f173\"}.fa-tumblr-square:before{content:\"\\f174\"}.fa-tv:before{content:\"\\f26c\"}.fa-twitch:before{content:\"\\f1e8\"}.fa-twitter:before{content:\"\\f099\"}.fa-twitter-square:before{content:\"\\f081\"}.fa-typo3:before{content:\"\\f42b\"}.fa-uber:before{content:\"\\f402\"}.fa-uikit:before{content:\"\\f403\"}.fa-umbrella:before{content:\"\\f0e9\"}.fa-umbrella-beach:before{content:\"\\f5ca\"}.fa-underline:before{content:\"\\f0cd\"}.fa-undo:before{content:\"\\f0e2\"}.fa-undo-alt:before{content:\"\\f2ea\"}.fa-uniregistry:before{content:\"\\f404\"}.fa-universal-access:before{content:\"\\f29a\"}.fa-university:before{content:\"\\f19c\"}.fa-unlink:before{content:\"\\f127\"}.fa-unlock:before{content:\"\\f09c\"}.fa-unlock-alt:before{content:\"\\f13e\"}.fa-untappd:before{content:\"\\f405\"}.fa-upload:before{content:\"\\f093\"}.fa-usb:before{content:\"\\f287\"}.fa-user:before{content:\"\\f007\"}.fa-user-alt:before{content:\"\\f406\"}.fa-user-alt-slash:before{content:\"\\f4fa\"}.fa-user-astronaut:before{content:\"\\f4fb\"}.fa-user-check:before{content:\"\\f4fc\"}.fa-user-circle:before{content:\"\\f2bd\"}.fa-user-clock:before{content:\"\\f4fd\"}.fa-user-cog:before{content:\"\\f4fe\"}.fa-user-edit:before{content:\"\\f4ff\"}.fa-user-friends:before{content:\"\\f500\"}.fa-user-graduate:before{content:\"\\f501\"}.fa-user-injured:before{content:\"\\f728\"}.fa-user-lock:before{content:\"\\f502\"}.fa-user-md:before{content:\"\\f0f0\"}.fa-user-minus:before{content:\"\\f503\"}.fa-user-ninja:before{content:\"\\f504\"}.fa-user-plus:before{content:\"\\f234\"}.fa-user-secret:before{content:\"\\f21b\"}.fa-user-shield:before{content:\"\\f505\"}.fa-user-slash:before{content:\"\\f506\"}.fa-user-tag:before{content:\"\\f507\"}.fa-user-tie:before{content:\"\\f508\"}.fa-user-times:before{content:\"\\f235\"}.fa-users:before{content:\"\\f0c0\"}.fa-users-cog:before{content:\"\\f509\"}.fa-ussunnah:before{content:\"\\f407\"}.fa-utensil-spoon:before{content:\"\\f2e5\"}.fa-utensils:before{content:\"\\f2e7\"}.fa-vaadin:before{content:\"\\f408\"}.fa-vector-square:before{content:\"\\f5cb\"}.fa-venus:before{content:\"\\f221\"}.fa-venus-double:before{content:\"\\f226\"}.fa-venus-mars:before{content:\"\\f228\"}.fa-viacoin:before{content:\"\\f237\"}.fa-viadeo:before{content:\"\\f2a9\"}.fa-viadeo-square:before{content:\"\\f2aa\"}.fa-vial:before{content:\"\\f492\"}.fa-vials:before{content:\"\\f493\"}.fa-viber:before{content:\"\\f409\"}.fa-video:before{content:\"\\f03d\"}.fa-video-slash:before{content:\"\\f4e2\"}.fa-vihara:before{content:\"\\f6a7\"}.fa-vimeo:before{content:\"\\f40a\"}.fa-vimeo-square:before{content:\"\\f194\"}.fa-vimeo-v:before{content:\"\\f27d\"}.fa-vine:before{content:\"\\f1ca\"}.fa-vk:before{content:\"\\f189\"}.fa-vnv:before{content:\"\\f40b\"}.fa-volleyball-ball:before{content:\"\\f45f\"}.fa-volume-down:before{content:\"\\f027\"}.fa-volume-mute:before{content:\"\\f6a9\"}.fa-volume-off:before{content:\"\\f026\"}.fa-volume-up:before{content:\"\\f028\"}.fa-vote-yea:before{content:\"\\f772\"}.fa-vr-cardboard:before{content:\"\\f729\"}.fa-vuejs:before{content:\"\\f41f\"}.fa-walking:before{content:\"\\f554\"}.fa-wallet:before{content:\"\\f555\"}.fa-warehouse:before{content:\"\\f494\"}.fa-water:before{content:\"\\f773\"}.fa-weebly:before{content:\"\\f5cc\"}.fa-weibo:before{content:\"\\f18a\"}.fa-weight:before{content:\"\\f496\"}.fa-weight-hanging:before{content:\"\\f5cd\"}.fa-weixin:before{content:\"\\f1d7\"}.fa-whatsapp:before{content:\"\\f232\"}.fa-whatsapp-square:before{content:\"\\f40c\"}.fa-wheelchair:before{content:\"\\f193\"}.fa-whmcs:before{content:\"\\f40d\"}.fa-wifi:before{content:\"\\f1eb\"}.fa-wikipedia-w:before{content:\"\\f266\"}.fa-wind:before{content:\"\\f72e\"}.fa-window-close:before{content:\"\\f410\"}.fa-window-maximize:before{content:\"\\f2d0\"}.fa-window-minimize:before{content:\"\\f2d1\"}.fa-window-restore:before{content:\"\\f2d2\"}.fa-windows:before{content:\"\\f17a\"}.fa-wine-bottle:before{content:\"\\f72f\"}.fa-wine-glass:before{content:\"\\f4e3\"}.fa-wine-glass-alt:before{content:\"\\f5ce\"}.fa-wix:before{content:\"\\f5cf\"}.fa-wizards-of-the-coast:before{content:\"\\f730\"}.fa-wolf-pack-battalion:before{content:\"\\f514\"}.fa-won-sign:before{content:\"\\f159\"}.fa-wordpress:before{content:\"\\f19a\"}.fa-wordpress-simple:before{content:\"\\f411\"}.fa-wpbeginner:before{content:\"\\f297\"}.fa-wpexplorer:before{content:\"\\f2de\"}.fa-wpforms:before{content:\"\\f298\"}.fa-wpressr:before{content:\"\\f3e4\"}.fa-wrench:before{content:\"\\f0ad\"}.fa-x-ray:before{content:\"\\f497\"}.fa-xbox:before{content:\"\\f412\"}.fa-xing:before{content:\"\\f168\"}.fa-xing-square:before{content:\"\\f169\"}.fa-y-combinator:before{content:\"\\f23b\"}.fa-yahoo:before{content:\"\\f19e\"}.fa-yandex:before{content:\"\\f413\"}.fa-yandex-international:before{content:\"\\f414\"}.fa-yelp:before{content:\"\\f1e9\"}.fa-yen-sign:before{content:\"\\f157\"}.fa-yin-yang:before{content:\"\\f6ad\"}.fa-yoast:before{content:\"\\f2b1\"}.fa-youtube:before{content:\"\\f167\"}.fa-youtube-square:before{content:\"\\f431\"}.fa-zhihu:before{content:\"\\f63f\"}.sr-only{border:0;clip:rect(0,0,0,0);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.sr-only-focusable:active,.sr-only-focusable:focus{clip:auto;height:auto;margin:0;overflow:visible;position:static;width:auto}@font-face{font-family:\"Font Awesome 5 Brands\";font-style:normal;font-weight:normal;src:url(../webfonts/fa-brands-400.eot);src:url(../webfonts/fa-brands-400.eot?#iefix) format(\"embedded-opentype\"),url(../webfonts/fa-brands-400.woff2) format(\"woff2\"),url(../webfonts/fa-brands-400.woff) format(\"woff\"),url(../webfonts/fa-brands-400.ttf) format(\"truetype\"),url(../webfonts/fa-brands-400.svg#fontawesome) format(\"svg\")}.fab{font-family:\"Font Awesome 5 Brands\"}@font-face{font-family:\"Font Awesome 5 Free\";font-style:normal;font-weight:400;src:url(../webfonts/fa-regular-400.eot);src:url(../webfonts/fa-regular-400.eot?#iefix) format(\"embedded-opentype\"),url(../webfonts/fa-regular-400.woff2) format(\"woff2\"),url(../webfonts/fa-regular-400.woff) format(\"woff\"),url(../webfonts/fa-regular-400.ttf) format(\"truetype\"),url(../webfonts/fa-regular-400.svg#fontawesome) format(\"svg\")}.far{font-weight:400}@font-face{font-family:\"Font Awesome 5 Free\";font-style:normal;font-weight:900;src:url(../webfonts/fa-solid-900.eot);src:url(../webfonts/fa-solid-900.eot?#iefix) format(\"embedded-opentype\"),url(../webfonts/fa-solid-900.woff2) format(\"woff2\"),url(../webfonts/fa-solid-900.woff) format(\"woff\"),url(../webfonts/fa-solid-900.ttf) format(\"truetype\"),url(../webfonts/fa-solid-900.svg#fontawesome) format(\"svg\")}.fa,.far,.fas{font-family:\"Font Awesome 5 Free\"}.fa,.fas{font-weight:900}"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/static/fonts.css",
    "content": "@font-face {\n    font-family: 'Roboto';\n    src: url('font/Roboto/Roboto-Regular.eot');\n    src: url('font/Roboto/Roboto-Regular.eot?#iefix') format('embedded-opentype'),\n         url('font/Roboto/Roboto-Regular.woff') format('woff'),\n         url('font/Roboto/Roboto-Regular.ttf') format('truetype');\n    font-weight: normal;\n    font-style: normal;\n}\n/*\n* Noto Sans SC Sliced (Chinese Simplified)\n */\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.0.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.0.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.0.otf')  format('opentype');unicode-range:U+2252,U+30c3,U+5c83-5c94,U+5c9c-5ca8,U+5caa-5cac,U+5cae-5cb2,U+5cb4-5cb7,U+5cb9-5cc0,U+5cc2-5cc3,U+5cc5-5cd8,U+5cdb-5ce0,U+5ce2-5ce7,U+5ce9-5cec,U+5cee-5cef,U+5cf1-5cfa,U+5cfc-5d06,U+5d08-5d0d,U+5d0f-5d13,U+5d15,U+5d17-5d1a,U+5d1c-5d28,U+5d2a-5d2c,U+5d2e-5d4b,U+5d4d-5dc4,U+5dc6-5dcc,U+5dce-5ddc,U+5ddf-5de0,U+5de3-5de4,U+5dea,U+5dec-5ded,U+5def-5df0,U+5df5-5df6,U+5df8-5dfd,U+5dff-5e00,U+5e04,U+5e07,U+5e09-5e0b,U+5e0d-5e0f,U+5e12-5e14,U+5e17,U+5e19-5e1b,U+5e1e-5e25,U+5e28-5e2c,U+5e2f-5e36,U+5e39-5e3c,U+5e3e-5e44,U+5e46-5e54,U+5e56-5e5e,U+67d1,U+6cba,U+9569-956b,U+958a-958b;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.0.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.0.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.0.otf')  format('opentype');unicode-range:U+2252,U+30c3,U+5c83-5c94,U+5c9c-5ca8,U+5caa-5cac,U+5cae-5cb2,U+5cb4-5cb7,U+5cb9-5cc0,U+5cc2-5cc3,U+5cc5-5cd8,U+5cdb-5ce0,U+5ce2-5ce7,U+5ce9-5cec,U+5cee-5cef,U+5cf1-5cfa,U+5cfc-5d06,U+5d08-5d0d,U+5d0f-5d13,U+5d15,U+5d17-5d1a,U+5d1c-5d28,U+5d2a-5d2c,U+5d2e-5d4b,U+5d4d-5dc4,U+5dc6-5dcc,U+5dce-5ddc,U+5ddf-5de0,U+5de3-5de4,U+5dea,U+5dec-5ded,U+5def-5df0,U+5df5-5df6,U+5df8-5dfd,U+5dff-5e00,U+5e04,U+5e07,U+5e09-5e0b,U+5e0d-5e0f,U+5e12-5e14,U+5e17,U+5e19-5e1b,U+5e1e-5e25,U+5e28-5e2c,U+5e2f-5e36,U+5e39-5e3c,U+5e3e-5e44,U+5e46-5e54,U+5e56-5e5e,U+67d1,U+6cba,U+9569-956b,U+958a-958b;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.0.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.0.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.0.otf')  format('opentype');unicode-range:U+2252,U+30c3,U+5c83-5c94,U+5c9c-5ca8,U+5caa-5cac,U+5cae-5cb2,U+5cb4-5cb7,U+5cb9-5cc0,U+5cc2-5cc3,U+5cc5-5cd8,U+5cdb-5ce0,U+5ce2-5ce7,U+5ce9-5cec,U+5cee-5cef,U+5cf1-5cfa,U+5cfc-5d06,U+5d08-5d0d,U+5d0f-5d13,U+5d15,U+5d17-5d1a,U+5d1c-5d28,U+5d2a-5d2c,U+5d2e-5d4b,U+5d4d-5dc4,U+5dc6-5dcc,U+5dce-5ddc,U+5ddf-5de0,U+5de3-5de4,U+5dea,U+5dec-5ded,U+5def-5df0,U+5df5-5df6,U+5df8-5dfd,U+5dff-5e00,U+5e04,U+5e07,U+5e09-5e0b,U+5e0d-5e0f,U+5e12-5e14,U+5e17,U+5e19-5e1b,U+5e1e-5e25,U+5e28-5e2c,U+5e2f-5e36,U+5e39-5e3c,U+5e3e-5e44,U+5e46-5e54,U+5e56-5e5e,U+67d1,U+6cba,U+9569-956b,U+958a-958b;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.1.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.1.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.1.otf')  format('opentype');unicode-range:U+8f7e,U+987c,U+c557-c57b,U+c57d-c5b3,U+c5b5-c5c5,U+c5c9-c5cf,U+c5d1-c5eb,U+c5ed-c600,U+c602-c623,U+c625-c63f,U+c641-c693,U+c695-c6af,U+c6b1-c6b3,U+c6b5-c6b7,U+c6b9-c6c2,U+c6c4-c6cf,U+c6d1-c6fe,U+c843-c845,U+c873-c874;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.1.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.1.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.1.otf')  format('opentype');unicode-range:U+8f7e,U+987c,U+c557-c57b,U+c57d-c5b3,U+c5b5-c5c5,U+c5c9-c5cf,U+c5d1-c5eb,U+c5ed-c600,U+c602-c623,U+c625-c63f,U+c641-c693,U+c695-c6af,U+c6b1-c6b3,U+c6b5-c6b7,U+c6b9-c6c2,U+c6c4-c6cf,U+c6d1-c6fe,U+c843-c845,U+c873-c874;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.1.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.1.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.1.otf')  format('opentype');unicode-range:U+8f7e,U+987c,U+c557-c57b,U+c57d-c5b3,U+c5b5-c5c5,U+c5c9-c5cf,U+c5d1-c5eb,U+c5ed-c600,U+c602-c623,U+c625-c63f,U+c641-c693,U+c695-c6af,U+c6b1-c6b3,U+c6b5-c6b7,U+c6b9-c6c2,U+c6c4-c6cf,U+c6d1-c6fe,U+c843-c845,U+c873-c874;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.2.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.2.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.2.otf')  format('opentype');unicode-range:U+54fb,U+573f,U+574b,U+5f89,U+647d-6486,U+6488-6490,U+6493-6494,U+6496-649d,U+649f-64a3,U+64a5-64ac,U+64b2-64bb,U+64bd-64c4,U+64c6-64cc,U+64cf-64d1,U+64d3-64e5,U+64e7-64ff,U+6501-6511,U+6513-651d,U+651f-652e,U+6530-6535,U+6537-6538,U+653c-653d,U+6540-6544,U+6546-6547,U+6549-654b,U+654d-654e,U+6550,U+6552-6558,U+655a,U+655c-655d,U+655f-6561,U+6564-6565,U+6567-656b,U+656d-656f,U+6571,U+6573,U+6575-6576,U+6578-6586,U+6588-658a,U+658d-658f,U+6592-6596,U+659a-659b,U+659d-65a3,U+65a6,U+65aa-65ac,U+65ae,U+65b1-65b8,U+65ba-65bb,U+65be-65c0,U+65c2-65c4,U+65c6-65ca,U+65cc-65ce,U+65d0-65d1,U+65d3-65d6,U+65d8-65df,U+65e1,U+65e3-65e4,U+65ea-65eb,U+65ee-65f0,U+65f2-65f5,U+65f8-65f9,U+65fb-6601,U+6603-6605,U+6607-660b,U+660d,U+6610-6612,U+6615-661e,U+6621-6624,U+6626,U+6629-662c,U+662e,U+6630-6639,U+663b,U+663f-6642,U+6644-664a,U+664c-6651,U+6657-6659,U+665b-6665,U+6667,U+6669-666d,U+6671-6673,U+6675,U+6677-6679,U+667b-667d,U+667f-6681,U+73c8-73c9,U+78f1,U+7a3e,U+8866-8867,U+957d-957e,U+95e5-95e7;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.2.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.2.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.2.otf')  format('opentype');unicode-range:U+54fb,U+573f,U+574b,U+5f89,U+647d-6486,U+6488-6490,U+6493-6494,U+6496-649d,U+649f-64a3,U+64a5-64ac,U+64b2-64bb,U+64bd-64c4,U+64c6-64cc,U+64cf-64d1,U+64d3-64e5,U+64e7-64ff,U+6501-6511,U+6513-651d,U+651f-652e,U+6530-6535,U+6537-6538,U+653c-653d,U+6540-6544,U+6546-6547,U+6549-654b,U+654d-654e,U+6550,U+6552-6558,U+655a,U+655c-655d,U+655f-6561,U+6564-6565,U+6567-656b,U+656d-656f,U+6571,U+6573,U+6575-6576,U+6578-6586,U+6588-658a,U+658d-658f,U+6592-6596,U+659a-659b,U+659d-65a3,U+65a6,U+65aa-65ac,U+65ae,U+65b1-65b8,U+65ba-65bb,U+65be-65c0,U+65c2-65c4,U+65c6-65ca,U+65cc-65ce,U+65d0-65d1,U+65d3-65d6,U+65d8-65df,U+65e1,U+65e3-65e4,U+65ea-65eb,U+65ee-65f0,U+65f2-65f5,U+65f8-65f9,U+65fb-6601,U+6603-6605,U+6607-660b,U+660d,U+6610-6612,U+6615-661e,U+6621-6624,U+6626,U+6629-662c,U+662e,U+6630-6639,U+663b,U+663f-6642,U+6644-664a,U+664c-6651,U+6657-6659,U+665b-6665,U+6667,U+6669-666d,U+6671-6673,U+6675,U+6677-6679,U+667b-667d,U+667f-6681,U+73c8-73c9,U+78f1,U+7a3e,U+8866-8867,U+957d-957e,U+95e5-95e7;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.2.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.2.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.2.otf')  format('opentype');unicode-range:U+54fb,U+573f,U+574b,U+5f89,U+647d-6486,U+6488-6490,U+6493-6494,U+6496-649d,U+649f-64a3,U+64a5-64ac,U+64b2-64bb,U+64bd-64c4,U+64c6-64cc,U+64cf-64d1,U+64d3-64e5,U+64e7-64ff,U+6501-6511,U+6513-651d,U+651f-652e,U+6530-6535,U+6537-6538,U+653c-653d,U+6540-6544,U+6546-6547,U+6549-654b,U+654d-654e,U+6550,U+6552-6558,U+655a,U+655c-655d,U+655f-6561,U+6564-6565,U+6567-656b,U+656d-656f,U+6571,U+6573,U+6575-6576,U+6578-6586,U+6588-658a,U+658d-658f,U+6592-6596,U+659a-659b,U+659d-65a3,U+65a6,U+65aa-65ac,U+65ae,U+65b1-65b8,U+65ba-65bb,U+65be-65c0,U+65c2-65c4,U+65c6-65ca,U+65cc-65ce,U+65d0-65d1,U+65d3-65d6,U+65d8-65df,U+65e1,U+65e3-65e4,U+65ea-65eb,U+65ee-65f0,U+65f2-65f5,U+65f8-65f9,U+65fb-6601,U+6603-6605,U+6607-660b,U+660d,U+6610-6612,U+6615-661e,U+6621-6624,U+6626,U+6629-662c,U+662e,U+6630-6639,U+663b,U+663f-6642,U+6644-664a,U+664c-6651,U+6657-6659,U+665b-6665,U+6667,U+6669-666d,U+6671-6673,U+6675,U+6677-6679,U+667b-667d,U+667f-6681,U+73c8-73c9,U+78f1,U+7a3e,U+8866-8867,U+957d-957e,U+95e5-95e7;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.3.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.3.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.3.otf')  format('opentype');unicode-range:U+25bd,U+51fc,U+6bd3,U+881d,U+962f-9630,U+bd34-bd7f,U+bd81-bd83,U+bd85-be43,U+be45-be5a,U+be5c-bed2;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.3.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.3.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.3.otf')  format('opentype');unicode-range:U+25bd,U+51fc,U+6bd3,U+881d,U+962f-9630,U+bd34-bd7f,U+bd81-bd83,U+bd85-be43,U+be45-be5a,U+be5c-bed2;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.3.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.3.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.3.otf')  format('opentype');unicode-range:U+25bd,U+51fc,U+6bd3,U+881d,U+962f-9630,U+bd34-bd7f,U+bd81-bd83,U+bd85-be43,U+be45-be5a,U+be5c-bed2;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.4.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.4.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.4.otf')  format('opentype');unicode-range:U+3050-3051,U+5e85,U+c7b6-c7bc,U+21ca2,U+249a9-25d30,U+25db9-25ee8,U+25f4b-26412,U+26488-26cc0,U+28eac;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.4.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.4.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.4.otf')  format('opentype');unicode-range:U+3050-3051,U+5e85,U+c7b6-c7bc,U+21ca2,U+249a9-25d30,U+25db9-25ee8,U+25f4b-26412,U+26488-26cc0,U+28eac;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.4.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.4.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.4.otf')  format('opentype');unicode-range:U+3050-3051,U+5e85,U+c7b6-c7bc,U+21ca2,U+249a9-25d30,U+25db9-25ee8,U+25f4b-26412,U+26488-26cc0,U+28eac;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.5.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.5.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.5.otf')  format('opentype');unicode-range:U+88ac,U+95fc,U+20e4c-20f4c,U+20fad-21088,U+2109d-21c56,U+21ca5-22c38,U+249a4,U+26cd1,U+2808a,U+2b36f,U+2f945;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.5.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.5.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.5.otf')  format('opentype');unicode-range:U+88ac,U+95fc,U+20e4c-20f4c,U+20fad-21088,U+2109d-21c56,U+21ca5-22c38,U+249a4,U+26cd1,U+2808a,U+2b36f,U+2f945;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.5.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.5.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.5.otf')  format('opentype');unicode-range:U+88ac,U+95fc,U+20e4c-20f4c,U+20fad-21088,U+2109d-21c56,U+21ca5-22c38,U+249a4,U+26cd1,U+2808a,U+2b36f,U+2f945;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.6.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.6.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.6.otf')  format('opentype');unicode-range:U+36e1-387f,U+2b300;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.6.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.6.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.6.otf')  format('opentype');unicode-range:U+36e1-387f,U+2b300;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.6.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.6.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.6.otf')  format('opentype');unicode-range:U+36e1-387f,U+2b300;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.7.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.7.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.7.otf')  format('opentype');unicode-range:U+339f-353e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.7.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.7.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.7.otf')  format('opentype');unicode-range:U+339f-353e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.7.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.7.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.7.otf')  format('opentype');unicode-range:U+339f-353e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.8.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.8.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.8.otf')  format('opentype');unicode-range:U+94cb-94ce,U+95ab-95ad,U+95d5-95d6,U+b02a-b097,U+b099-b09b,U+b09d-b09f,U+b0a1-b0a7,U+b0a9-b0b3,U+b0b5-b107,U+b109-b10f,U+b111-b123,U+b125-b154,U+b156-b1c2,U+c728-c72b,U+c761-c762;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.8.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.8.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.8.otf')  format('opentype');unicode-range:U+94cb-94ce,U+95ab-95ad,U+95d5-95d6,U+b02a-b097,U+b099-b09b,U+b09d-b09f,U+b0a1-b0a7,U+b0a9-b0b3,U+b0b5-b107,U+b109-b10f,U+b111-b123,U+b125-b154,U+b156-b1c2,U+c728-c72b,U+c761-c762;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.8.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.8.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.8.otf')  format('opentype');unicode-range:U+94cb-94ce,U+95ab-95ad,U+95d5-95d6,U+b02a-b097,U+b099-b09b,U+b09d-b09f,U+b0a1-b0a7,U+b0a9-b0b3,U+b0b5-b107,U+b109-b10f,U+b111-b123,U+b125-b154,U+b156-b1c2,U+c728-c72b,U+c761-c762;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.9.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.9.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.9.otf')  format('opentype');unicode-range:U+3bc2-3d62,U+65a8,U+8d32-8d33;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.9.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.9.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.9.otf')  format('opentype');unicode-range:U+3bc2-3d62,U+65a8,U+8d32-8d33;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.9.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.9.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.9.otf')  format('opentype');unicode-range:U+3bc2-3d62,U+65a8,U+8d32-8d33;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.10.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.10.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.10.otf')  format('opentype');unicode-range:U+4c0c-4da8,U+4e23,U+95c3-95c6,U+95fe-95ff;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.10.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.10.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.10.otf')  format('opentype');unicode-range:U+4c0c-4da8,U+4e23,U+95c3-95c6,U+95fe-95ff;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.10.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.10.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.10.otf')  format('opentype');unicode-range:U+4c0c-4da8,U+4e23,U+95c3-95c6,U+95fe-95ff;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.11.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.11.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.11.otf')  format('opentype');unicode-range:U+c726-c727,U+cd79-cd93,U+cd95-ce57,U+ce59-ce73,U+ce75-cf16,U+2b37d;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.11.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.11.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.11.otf')  format('opentype');unicode-range:U+c726-c727,U+cd79-cd93,U+cd95-ce57,U+ce59-ce73,U+ce75-cf16,U+2b37d;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.11.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.11.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.11.otf')  format('opentype');unicode-range:U+c726-c727,U+cd79-cd93,U+cd95-ce57,U+ce59-ce73,U+ce75-cf16,U+2b37d;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.12.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.12.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.12.otf')  format('opentype');unicode-range:U+519a,U+b851-b85b,U+b85d-b973,U+b975-b977,U+b979-b97b,U+b97d-b983,U+b985-b9ab,U+b9ad-b9af,U+b9b1-b9c7,U+b9c9-b9cb,U+b9cd-b9cf,U+b9d1-b9ee,U+c7ad-c7b5,U+c83c-c83d;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.12.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.12.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.12.otf')  format('opentype');unicode-range:U+519a,U+b851-b85b,U+b85d-b973,U+b975-b977,U+b979-b97b,U+b97d-b983,U+b985-b9ab,U+b9ad-b9af,U+b9b1-b9c7,U+b9c9-b9cb,U+b9cd-b9cf,U+b9d1-b9ee,U+c7ad-c7b5,U+c83c-c83d;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.12.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.12.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.12.otf')  format('opentype');unicode-range:U+519a,U+b851-b85b,U+b85d-b973,U+b975-b977,U+b979-b97b,U+b97d-b983,U+b985-b9ab,U+b9ad-b9af,U+b9b1-b9c7,U+b9c9-b9cb,U+b9cd-b9cf,U+b9d1-b9ee,U+c7ad-c7b5,U+c83c-c83d;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.13.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.13.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.13.otf')  format('opentype');unicode-range:U+00c5,U+03b6-03ba,U+03bf,U+03c5,U+500b,U+537b,U+5834,U+6703,U+6727,U+6c7a,U+752a,U+767c,U+8f24-8f65,U+8f6a-8f6b,U+8f6d,U+8f71-8f73,U+8f75-8f7a,U+8f80-8f82,U+8f8a-8f8f,U+8f92,U+8f94-8f95,U+8fa0-8fa2,U+8fa4-8fa7,U+8faa-8faf,U+8fb2-8fb8,U+8fba-8fbc,U+8fbf-8fc0,U+8fc2-8fc3,U+8fc9-8fcd,U+8fcf,U+8fd2-8fd3,U+8fd5-8fd7,U+8fda,U+8fe0-8fe4,U+8fe7-8fe9,U+8fec,U+8fee-8fef,U+8ff1-8ff6,U+8ff8,U+8ffa-8ffc,U+8ffe-8fff,U+9007-9008,U+900b-900c,U+9011,U+9013,U+9015-9016,U+9018-9019,U+901c,U+901e,U+9021,U+9023-902d,U+902f-9037,U+9039-903a,U+903d,U+903f-9041,U+9043-9046,U+9048-904c,U+904e-904f,U+9051-9052,U+9054-9056,U+9058-9062,U+9064,U+9066-906c,U+906f-9074,U+9076-907e,U+9081,U+9084-9090,U+9092,U+9094-90a2,U+90a4-90a5,U+90a7-90a9,U+90ab-90ad,U+90af-90b0,U+90b2-90b4,U+90b6-90b8,U+90ba,U+90bc-90c0,U+90c2-90c4,U+90c6-90c9,U+90cb-90cd,U+90cf-90d0,U+90d2-90dd,U+90df-90e0,U+90e2-90e7,U+90e9-90ec,U+90ee-90fc,U+90fe-9101,U+9103-9118,U+911a-9130,U+9611,U+c838-c83a,U+ff14,U+28482;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.13.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.13.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.13.otf')  format('opentype');unicode-range:U+00c5,U+03b6-03ba,U+03bf,U+03c5,U+500b,U+537b,U+5834,U+6703,U+6727,U+6c7a,U+752a,U+767c,U+8f24-8f65,U+8f6a-8f6b,U+8f6d,U+8f71-8f73,U+8f75-8f7a,U+8f80-8f82,U+8f8a-8f8f,U+8f92,U+8f94-8f95,U+8fa0-8fa2,U+8fa4-8fa7,U+8faa-8faf,U+8fb2-8fb8,U+8fba-8fbc,U+8fbf-8fc0,U+8fc2-8fc3,U+8fc9-8fcd,U+8fcf,U+8fd2-8fd3,U+8fd5-8fd7,U+8fda,U+8fe0-8fe4,U+8fe7-8fe9,U+8fec,U+8fee-8fef,U+8ff1-8ff6,U+8ff8,U+8ffa-8ffc,U+8ffe-8fff,U+9007-9008,U+900b-900c,U+9011,U+9013,U+9015-9016,U+9018-9019,U+901c,U+901e,U+9021,U+9023-902d,U+902f-9037,U+9039-903a,U+903d,U+903f-9041,U+9043-9046,U+9048-904c,U+904e-904f,U+9051-9052,U+9054-9056,U+9058-9062,U+9064,U+9066-906c,U+906f-9074,U+9076-907e,U+9081,U+9084-9090,U+9092,U+9094-90a2,U+90a4-90a5,U+90a7-90a9,U+90ab-90ad,U+90af-90b0,U+90b2-90b4,U+90b6-90b8,U+90ba,U+90bc-90c0,U+90c2-90c4,U+90c6-90c9,U+90cb-90cd,U+90cf-90d0,U+90d2-90dd,U+90df-90e0,U+90e2-90e7,U+90e9-90ec,U+90ee-90fc,U+90fe-9101,U+9103-9118,U+911a-9130,U+9611,U+c838-c83a,U+ff14,U+28482;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.13.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.13.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.13.otf')  format('opentype');unicode-range:U+00c5,U+03b6-03ba,U+03bf,U+03c5,U+500b,U+537b,U+5834,U+6703,U+6727,U+6c7a,U+752a,U+767c,U+8f24-8f65,U+8f6a-8f6b,U+8f6d,U+8f71-8f73,U+8f75-8f7a,U+8f80-8f82,U+8f8a-8f8f,U+8f92,U+8f94-8f95,U+8fa0-8fa2,U+8fa4-8fa7,U+8faa-8faf,U+8fb2-8fb8,U+8fba-8fbc,U+8fbf-8fc0,U+8fc2-8fc3,U+8fc9-8fcd,U+8fcf,U+8fd2-8fd3,U+8fd5-8fd7,U+8fda,U+8fe0-8fe4,U+8fe7-8fe9,U+8fec,U+8fee-8fef,U+8ff1-8ff6,U+8ff8,U+8ffa-8ffc,U+8ffe-8fff,U+9007-9008,U+900b-900c,U+9011,U+9013,U+9015-9016,U+9018-9019,U+901c,U+901e,U+9021,U+9023-902d,U+902f-9037,U+9039-903a,U+903d,U+903f-9041,U+9043-9046,U+9048-904c,U+904e-904f,U+9051-9052,U+9054-9056,U+9058-9062,U+9064,U+9066-906c,U+906f-9074,U+9076-907e,U+9081,U+9084-9090,U+9092,U+9094-90a2,U+90a4-90a5,U+90a7-90a9,U+90ab-90ad,U+90af-90b0,U+90b2-90b4,U+90b6-90b8,U+90ba,U+90bc-90c0,U+90c2-90c4,U+90c6-90c9,U+90cb-90cd,U+90cf-90d0,U+90d2-90dd,U+90df-90e0,U+90e2-90e7,U+90e9-90ec,U+90ee-90fc,U+90fe-9101,U+9103-9118,U+911a-9130,U+9611,U+c838-c83a,U+ff14,U+28482;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.14.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.14.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.14.otf')  format('opentype');unicode-range:U+527d-527e,U+5280-5287,U+5289-529a,U+52a2,U+52a4-52a7,U+52ac-52b0,U+52b4-52be,U+52c0-52c2,U+52c4-52c6,U+52c8,U+52ca,U+52cc-52d1,U+52d3-52d4,U+52d6-52d7,U+52d9-52de,U+52e0-52e3,U+52e5-52f9,U+52fb-52fd,U+5301-5304,U+5307-5315,U+5318,U+531a-531f,U+5321-5329,U+532b-5338,U+533c-533d,U+5340,U+5342,U+5344-5346,U+534b-534d,U+5350,U+5354,U+5358-5359,U+535b,U+535d-535f,U+5363,U+5365,U+5368-536a,U+536c-536e,U+5372,U+5376,U+5379-537a,U+537c-537e,U+5380-5381,U+5383,U+5387-5388,U+538a,U+538d-5394,U+5396-5397,U+5399,U+539b-539e,U+53a0-53a1,U+53a3-53a4,U+53a7,U+53a9-53ba,U+53bc-53be,U+53c0-53c1,U+53c3-53c7,U+53ce-53d0,U+53d2-53d3,U+53d5,U+53da,U+53dc-53df,U+53e1-53e2,U+53f1,U+53f4-53f5,U+53fa-5400,U+5402,U+5405-5407,U+540b,U+5412,U+5414,U+5416,U+5418-541a,U+5420-5425,U+5429-542a,U+542d-542e,U+5430-5433,U+5436-5437,U+543a,U+543d,U+543f,U+5441-5445,U+5447,U+5449,U+544b-544f,U+5451-5454,U+5456-5457,U+5459-5461,U+5463-5467,U+5469-5472,U+5474,U+5476-547b,U+547e-547f,U+5481-548a,U+548d-548e,U+5493-5495,U+5498-54a7,U+54ad-54ae,U+54b0,U+54b2,U+54b4-54b7,U+54b9-54bc,U+54be-54bf,U+54c2-54c3,U+54c5-54c6,U+54ca-54cc,U+8df4,U+8e1a-8e1c,U+9004,U+94e0-94e3,U+95a4-95aa,U+9641-9643,U+989e,U+280bb,U+2b4e7;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.14.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.14.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.14.otf')  format('opentype');unicode-range:U+527d-527e,U+5280-5287,U+5289-529a,U+52a2,U+52a4-52a7,U+52ac-52b0,U+52b4-52be,U+52c0-52c2,U+52c4-52c6,U+52c8,U+52ca,U+52cc-52d1,U+52d3-52d4,U+52d6-52d7,U+52d9-52de,U+52e0-52e3,U+52e5-52f9,U+52fb-52fd,U+5301-5304,U+5307-5315,U+5318,U+531a-531f,U+5321-5329,U+532b-5338,U+533c-533d,U+5340,U+5342,U+5344-5346,U+534b-534d,U+5350,U+5354,U+5358-5359,U+535b,U+535d-535f,U+5363,U+5365,U+5368-536a,U+536c-536e,U+5372,U+5376,U+5379-537a,U+537c-537e,U+5380-5381,U+5383,U+5387-5388,U+538a,U+538d-5394,U+5396-5397,U+5399,U+539b-539e,U+53a0-53a1,U+53a3-53a4,U+53a7,U+53a9-53ba,U+53bc-53be,U+53c0-53c1,U+53c3-53c7,U+53ce-53d0,U+53d2-53d3,U+53d5,U+53da,U+53dc-53df,U+53e1-53e2,U+53f1,U+53f4-53f5,U+53fa-5400,U+5402,U+5405-5407,U+540b,U+5412,U+5414,U+5416,U+5418-541a,U+5420-5425,U+5429-542a,U+542d-542e,U+5430-5433,U+5436-5437,U+543a,U+543d,U+543f,U+5441-5445,U+5447,U+5449,U+544b-544f,U+5451-5454,U+5456-5457,U+5459-5461,U+5463-5467,U+5469-5472,U+5474,U+5476-547b,U+547e-547f,U+5481-548a,U+548d-548e,U+5493-5495,U+5498-54a7,U+54ad-54ae,U+54b0,U+54b2,U+54b4-54b7,U+54b9-54bc,U+54be-54bf,U+54c2-54c3,U+54c5-54c6,U+54ca-54cc,U+8df4,U+8e1a-8e1c,U+9004,U+94e0-94e3,U+95a4-95aa,U+9641-9643,U+989e,U+280bb,U+2b4e7;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.14.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.14.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.14.otf')  format('opentype');unicode-range:U+527d-527e,U+5280-5287,U+5289-529a,U+52a2,U+52a4-52a7,U+52ac-52b0,U+52b4-52be,U+52c0-52c2,U+52c4-52c6,U+52c8,U+52ca,U+52cc-52d1,U+52d3-52d4,U+52d6-52d7,U+52d9-52de,U+52e0-52e3,U+52e5-52f9,U+52fb-52fd,U+5301-5304,U+5307-5315,U+5318,U+531a-531f,U+5321-5329,U+532b-5338,U+533c-533d,U+5340,U+5342,U+5344-5346,U+534b-534d,U+5350,U+5354,U+5358-5359,U+535b,U+535d-535f,U+5363,U+5365,U+5368-536a,U+536c-536e,U+5372,U+5376,U+5379-537a,U+537c-537e,U+5380-5381,U+5383,U+5387-5388,U+538a,U+538d-5394,U+5396-5397,U+5399,U+539b-539e,U+53a0-53a1,U+53a3-53a4,U+53a7,U+53a9-53ba,U+53bc-53be,U+53c0-53c1,U+53c3-53c7,U+53ce-53d0,U+53d2-53d3,U+53d5,U+53da,U+53dc-53df,U+53e1-53e2,U+53f1,U+53f4-53f5,U+53fa-5400,U+5402,U+5405-5407,U+540b,U+5412,U+5414,U+5416,U+5418-541a,U+5420-5425,U+5429-542a,U+542d-542e,U+5430-5433,U+5436-5437,U+543a,U+543d,U+543f,U+5441-5445,U+5447,U+5449,U+544b-544f,U+5451-5454,U+5456-5457,U+5459-5461,U+5463-5467,U+5469-5472,U+5474,U+5476-547b,U+547e-547f,U+5481-548a,U+548d-548e,U+5493-5495,U+5498-54a7,U+54ad-54ae,U+54b0,U+54b2,U+54b4-54b7,U+54b9-54bc,U+54be-54bf,U+54c2-54c3,U+54c5-54c6,U+54ca-54cc,U+8df4,U+8e1a-8e1c,U+9004,U+94e0-94e3,U+95a4-95aa,U+9641-9643,U+989e,U+280bb,U+2b4e7;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.15.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.15.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.15.otf')  format('opentype');unicode-range:U+353f-36e0;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.15.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.15.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.15.otf')  format('opentype');unicode-range:U+353f-36e0;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.15.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.15.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.15.otf')  format('opentype');unicode-range:U+353f-36e0;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.16.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.16.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.16.otf')  format('opentype');unicode-range:U+2609,U+273f-2ffb,U+3003-3007,U+3016-303f,U+959f-95a0;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.16.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.16.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.16.otf')  format('opentype');unicode-range:U+2609,U+273f-2ffb,U+3003-3007,U+3016-303f,U+959f-95a0;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.16.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.16.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.16.otf')  format('opentype');unicode-range:U+2609,U+273f-2ffb,U+3003-3007,U+3016-303f,U+959f-95a0;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.17.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.17.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.17.otf')  format('opentype');unicode-range:U+54a9,U+62c8,U+6c82,U+8099,U+809b-809c,U+809e-809f,U+80a3,U+80a6-80a8,U+80ab-80ae,U+80b0-80b1,U+80b5-80b9,U+80bb-80bd,U+80c2,U+80c4-80c5,U+80c7-80cb,U+80cd,U+80cf-80d5,U+80d7-80d9,U+80db,U+80dd,U+80df-80e0,U+80e2-80e6,U+80e8-80f0,U+80f2,U+80f9,U+80fb-80fc,U+80fe-8101,U+8103-8105,U+8107-8108,U+810b-810e,U+8110,U+8112-8115,U+8117-8119,U+811b-8130,U+8132-8137,U+8139-813d,U+813f-8149,U+814d-814f,U+8152-8153,U+8156-8164,U+8166-816f,U+8171-8178,U+817c-817d,U+8181-8189,U+818b-818e,U+8190-819b,U+819e-81a7,U+81a9-81b2,U+81b4-81bf,U+81c1,U+81c3-81e2,U+81e4-81e9,U+81eb-81ec,U+81ee-81f2,U+81f5-8205,U+8207-820b,U+820e-8211,U+8213-821d,U+8220-8229,U+822b,U+822d-822f,U+8232,U+8234-8238,U+823a-8246,U+8248-824e,U+8250-8257,U+8259-826e,U+8271,U+8274-8277,U+827b-827d,U+827f-8281,U+8283-8286,U+886d-886f,U+9496-9498,U+9522,U+95a1-95a3,U+95ae-95b3,U+95be-95bf,U+95f3;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.17.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.17.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.17.otf')  format('opentype');unicode-range:U+54a9,U+62c8,U+6c82,U+8099,U+809b-809c,U+809e-809f,U+80a3,U+80a6-80a8,U+80ab-80ae,U+80b0-80b1,U+80b5-80b9,U+80bb-80bd,U+80c2,U+80c4-80c5,U+80c7-80cb,U+80cd,U+80cf-80d5,U+80d7-80d9,U+80db,U+80dd,U+80df-80e0,U+80e2-80e6,U+80e8-80f0,U+80f2,U+80f9,U+80fb-80fc,U+80fe-8101,U+8103-8105,U+8107-8108,U+810b-810e,U+8110,U+8112-8115,U+8117-8119,U+811b-8130,U+8132-8137,U+8139-813d,U+813f-8149,U+814d-814f,U+8152-8153,U+8156-8164,U+8166-816f,U+8171-8178,U+817c-817d,U+8181-8189,U+818b-818e,U+8190-819b,U+819e-81a7,U+81a9-81b2,U+81b4-81bf,U+81c1,U+81c3-81e2,U+81e4-81e9,U+81eb-81ec,U+81ee-81f2,U+81f5-8205,U+8207-820b,U+820e-8211,U+8213-821d,U+8220-8229,U+822b,U+822d-822f,U+8232,U+8234-8238,U+823a-8246,U+8248-824e,U+8250-8257,U+8259-826e,U+8271,U+8274-8277,U+827b-827d,U+827f-8281,U+8283-8286,U+886d-886f,U+9496-9498,U+9522,U+95a1-95a3,U+95ae-95b3,U+95be-95bf,U+95f3;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.17.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.17.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.17.otf')  format('opentype');unicode-range:U+54a9,U+62c8,U+6c82,U+8099,U+809b-809c,U+809e-809f,U+80a3,U+80a6-80a8,U+80ab-80ae,U+80b0-80b1,U+80b5-80b9,U+80bb-80bd,U+80c2,U+80c4-80c5,U+80c7-80cb,U+80cd,U+80cf-80d5,U+80d7-80d9,U+80db,U+80dd,U+80df-80e0,U+80e2-80e6,U+80e8-80f0,U+80f2,U+80f9,U+80fb-80fc,U+80fe-8101,U+8103-8105,U+8107-8108,U+810b-810e,U+8110,U+8112-8115,U+8117-8119,U+811b-8130,U+8132-8137,U+8139-813d,U+813f-8149,U+814d-814f,U+8152-8153,U+8156-8164,U+8166-816f,U+8171-8178,U+817c-817d,U+8181-8189,U+818b-818e,U+8190-819b,U+819e-81a7,U+81a9-81b2,U+81b4-81bf,U+81c1,U+81c3-81e2,U+81e4-81e9,U+81eb-81ec,U+81ee-81f2,U+81f5-8205,U+8207-820b,U+820e-8211,U+8213-821d,U+8220-8229,U+822b,U+822d-822f,U+8232,U+8234-8238,U+823a-8246,U+8248-824e,U+8250-8257,U+8259-826e,U+8271,U+8274-8277,U+827b-827d,U+827f-8281,U+8283-8286,U+886d-886f,U+9496-9498,U+9522,U+95a1-95a3,U+95ae-95b3,U+95be-95bf,U+95f3;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.18.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.18.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.18.otf')  format('opentype');unicode-range:U+b50f-b52f,U+b531-b6ac,U+c7c9-c7cb;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.18.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.18.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.18.otf')  format('opentype');unicode-range:U+b50f-b52f,U+b531-b6ac,U+c7c9-c7cb;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.18.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.18.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.18.otf')  format('opentype');unicode-range:U+b50f-b52f,U+b531-b6ac,U+c7c9-c7cb;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.19.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.19.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.19.otf')  format('opentype');unicode-range:U+48cb-4a6b;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.19.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.19.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.19.otf')  format('opentype');unicode-range:U+48cb-4a6b;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.19.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.19.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.19.otf')  format('opentype');unicode-range:U+48cb-4a6b;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.20.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.20.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.20.otf')  format('opentype');unicode-range:U+6a56-6a57,U+6a5a-6a60,U+6a62-6a70,U+6a72-6a7f,U+6a81-6a8f,U+6a91-6aab,U+6aad-6b1f,U+6b24-6b26,U+6b28-6b31,U+6b33-6b37,U+6b39,U+6b3b-6b3d,U+6b3f-6b46,U+6b48,U+6b4a-6b4b,U+6b4d-6b61,U+6b68-6b69,U+6b6b-6b76,U+6b7d-6b89,U+6b8c-6b95,U+6b97-6ba9,U+6bab-6bb4,U+6bb6,U+6bb8-6bbe,U+6bc0,U+6bc2-6bc4,U+6bc6-6bca,U+6bcc,U+6bce,U+6bd0-6bd1,U+6bd6,U+6bd8,U+6bda,U+6bdc-6bea,U+6bec-6bee,U+6bf0-6c0e,U+6c10,U+6c12,U+6c15-6c1a,U+6c1c-6c21,U+6e3b,U+8716-8717,U+900d-900e,U+94e7-94e9,U+9592-9593,U+95c0-95c2,U+9a6e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.20.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.20.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.20.otf')  format('opentype');unicode-range:U+6a56-6a57,U+6a5a-6a60,U+6a62-6a70,U+6a72-6a7f,U+6a81-6a8f,U+6a91-6aab,U+6aad-6b1f,U+6b24-6b26,U+6b28-6b31,U+6b33-6b37,U+6b39,U+6b3b-6b3d,U+6b3f-6b46,U+6b48,U+6b4a-6b4b,U+6b4d-6b61,U+6b68-6b69,U+6b6b-6b76,U+6b7d-6b89,U+6b8c-6b95,U+6b97-6ba9,U+6bab-6bb4,U+6bb6,U+6bb8-6bbe,U+6bc0,U+6bc2-6bc4,U+6bc6-6bca,U+6bcc,U+6bce,U+6bd0-6bd1,U+6bd6,U+6bd8,U+6bda,U+6bdc-6bea,U+6bec-6bee,U+6bf0-6c0e,U+6c10,U+6c12,U+6c15-6c1a,U+6c1c-6c21,U+6e3b,U+8716-8717,U+900d-900e,U+94e7-94e9,U+9592-9593,U+95c0-95c2,U+9a6e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.20.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.20.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.20.otf')  format('opentype');unicode-range:U+6a56-6a57,U+6a5a-6a60,U+6a62-6a70,U+6a72-6a7f,U+6a81-6a8f,U+6a91-6aab,U+6aad-6b1f,U+6b24-6b26,U+6b28-6b31,U+6b33-6b37,U+6b39,U+6b3b-6b3d,U+6b3f-6b46,U+6b48,U+6b4a-6b4b,U+6b4d-6b61,U+6b68-6b69,U+6b6b-6b76,U+6b7d-6b89,U+6b8c-6b95,U+6b97-6ba9,U+6bab-6bb4,U+6bb6,U+6bb8-6bbe,U+6bc0,U+6bc2-6bc4,U+6bc6-6bca,U+6bcc,U+6bce,U+6bd0-6bd1,U+6bd6,U+6bd8,U+6bda,U+6bdc-6bea,U+6bec-6bee,U+6bf0-6c0e,U+6c10,U+6c12,U+6c15-6c1a,U+6c1c-6c21,U+6e3b,U+8716-8717,U+900d-900e,U+94e7-94e9,U+9592-9593,U+95c0-95c2,U+9a6e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.21.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.21.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.21.otf')  format('opentype');unicode-range:U+6ed2-6ed3,U+8dde,U+d73a-d787,U+d789-f9ea;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.21.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.21.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.21.otf')  format('opentype');unicode-range:U+6ed2-6ed3,U+8dde,U+d73a-d787,U+d789-f9ea;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.21.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.21.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.21.otf')  format('opentype');unicode-range:U+6ed2-6ed3,U+8dde,U+d73a-d787,U+d789-f9ea;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.22.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.22.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.22.otf')  format('opentype');unicode-range:U+7ea3,U+7ea5,U+cf17-d06b,U+d06d-d0b4;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.22.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.22.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.22.otf')  format('opentype');unicode-range:U+7ea3,U+7ea5,U+cf17-d06b,U+d06d-d0b4;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.22.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.22.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.22.otf')  format('opentype');unicode-range:U+7ea3,U+7ea5,U+cf17-d06b,U+d06d-d0b4;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.23.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.23.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.23.otf')  format('opentype');unicode-range:U+00e0,U+00ec,U+00f2-00f3,U+00f9,U+0101,U+01ce-01d0,U+01d4,U+0261-02cb,U+24fe,U+2501,U+3012-3013,U+3041-3043,U+3045-304a,U+304e,U+3053-3056,U+3058-305e,U+3064-3066,U+3069,U+306f-307d,U+3080-3088,U+308c-309b,U+309e-30ae,U+30b8-30c1,U+30c4-30c7,U+30cc-30e8,U+30ec-30f2,U+30f4-30fb,U+30fd-31fd,U+55ab,U+5739,U+5b6c,U+6242-6243,U+6f32,U+758e,U+7ebe,U+86a8,U+8bee,U+8c27,U+8deb-8dec,U+91d0,U+94e4-94e5,U+9504,U+9572-9575,U+95d7-95d8,U+9698,U+989f,U+9c90,U+9ddf;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.23.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.23.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.23.otf')  format('opentype');unicode-range:U+00e0,U+00ec,U+00f2-00f3,U+00f9,U+0101,U+01ce-01d0,U+01d4,U+0261-02cb,U+24fe,U+2501,U+3012-3013,U+3041-3043,U+3045-304a,U+304e,U+3053-3056,U+3058-305e,U+3064-3066,U+3069,U+306f-307d,U+3080-3088,U+308c-309b,U+309e-30ae,U+30b8-30c1,U+30c4-30c7,U+30cc-30e8,U+30ec-30f2,U+30f4-30fb,U+30fd-31fd,U+55ab,U+5739,U+5b6c,U+6242-6243,U+6f32,U+758e,U+7ebe,U+86a8,U+8bee,U+8c27,U+8deb-8dec,U+91d0,U+94e4-94e5,U+9504,U+9572-9575,U+95d7-95d8,U+9698,U+989f,U+9c90,U+9ddf;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.23.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.23.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.23.otf')  format('opentype');unicode-range:U+00e0,U+00ec,U+00f2-00f3,U+00f9,U+0101,U+01ce-01d0,U+01d4,U+0261-02cb,U+24fe,U+2501,U+3012-3013,U+3041-3043,U+3045-304a,U+304e,U+3053-3056,U+3058-305e,U+3064-3066,U+3069,U+306f-307d,U+3080-3088,U+308c-309b,U+309e-30ae,U+30b8-30c1,U+30c4-30c7,U+30cc-30e8,U+30ec-30f2,U+30f4-30fb,U+30fd-31fd,U+55ab,U+5739,U+5b6c,U+6242-6243,U+6f32,U+758e,U+7ebe,U+86a8,U+8bee,U+8c27,U+8deb-8dec,U+91d0,U+94e4-94e5,U+9504,U+9572-9575,U+95d7-95d8,U+9698,U+989f,U+9c90,U+9ddf;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.24.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.24.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.24.otf')  format('opentype');unicode-range:U+3880-3a21,U+966b-966c;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.24.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.24.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.24.otf')  format('opentype');unicode-range:U+3880-3a21,U+966b-966c;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.24.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.24.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.24.otf')  format('opentype');unicode-range:U+3880-3a21,U+966b-966c;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.25.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.25.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.25.otf')  format('opentype');unicode-range:U+039b,U+03bd,U+223d,U+5575,U+569d-56a2,U+56a4-56bb,U+56bd-56c9,U+56cb-56d9,U+56dc-56dd,U+56df,U+56e1,U+56e3-56ec,U+56ee-56ef,U+56f6-56f9,U+56fb-56fc,U+56ff-5705,U+5707,U+5709-570a,U+570c-571e,U+5720-5722,U+5724-5727,U+5729-572f,U+5731-5732,U+5734-5738,U+573b-573d,U+5743-5746,U+5748-5749,U+5752-5756,U+5758-5759,U+5762-5763,U+5765,U+576b-5776,U+5778-5781,U+5785-578a,U+578c-5791,U+5793-57a1,U+57a7-57aa,U+57ac-57ad,U+57af-57c2,U+57c4-57ca,U+57d0-57d3,U+57d5-57de,U+57e0-57f8,U+57fb-5801,U+5803-5805,U+5807-5814,U+5816-5820,U+5822-5823,U+5825-5829,U+582b-5833,U+5836-584b,U+584d-5850,U+5852-5853,U+5855-5857,U+5859-585d,U+585f-586a,U+586c-587a,U+59ab,U+651e,U+663a,U+6a50,U+6d65,U+7395,U+7817,U+78f2,U+7c9d,U+7f17,U+8233,U+8347,U+83f9,U+8764,U+8fc6,U+9506-9507,U+96e0,U+979e-979f,U+9899,U+9935,U+9e32,U+9f2f,U+fe31,U+ff1e,U+28090;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.25.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.25.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.25.otf')  format('opentype');unicode-range:U+039b,U+03bd,U+223d,U+5575,U+569d-56a2,U+56a4-56bb,U+56bd-56c9,U+56cb-56d9,U+56dc-56dd,U+56df,U+56e1,U+56e3-56ec,U+56ee-56ef,U+56f6-56f9,U+56fb-56fc,U+56ff-5705,U+5707,U+5709-570a,U+570c-571e,U+5720-5722,U+5724-5727,U+5729-572f,U+5731-5732,U+5734-5738,U+573b-573d,U+5743-5746,U+5748-5749,U+5752-5756,U+5758-5759,U+5762-5763,U+5765,U+576b-5776,U+5778-5781,U+5785-578a,U+578c-5791,U+5793-57a1,U+57a7-57aa,U+57ac-57ad,U+57af-57c2,U+57c4-57ca,U+57d0-57d3,U+57d5-57de,U+57e0-57f8,U+57fb-5801,U+5803-5805,U+5807-5814,U+5816-5820,U+5822-5823,U+5825-5829,U+582b-5833,U+5836-584b,U+584d-5850,U+5852-5853,U+5855-5857,U+5859-585d,U+585f-586a,U+586c-587a,U+59ab,U+651e,U+663a,U+6a50,U+6d65,U+7395,U+7817,U+78f2,U+7c9d,U+7f17,U+8233,U+8347,U+83f9,U+8764,U+8fc6,U+9506-9507,U+96e0,U+979e-979f,U+9899,U+9935,U+9e32,U+9f2f,U+fe31,U+ff1e,U+28090;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.25.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.25.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.25.otf')  format('opentype');unicode-range:U+039b,U+03bd,U+223d,U+5575,U+569d-56a2,U+56a4-56bb,U+56bd-56c9,U+56cb-56d9,U+56dc-56dd,U+56df,U+56e1,U+56e3-56ec,U+56ee-56ef,U+56f6-56f9,U+56fb-56fc,U+56ff-5705,U+5707,U+5709-570a,U+570c-571e,U+5720-5722,U+5724-5727,U+5729-572f,U+5731-5732,U+5734-5738,U+573b-573d,U+5743-5746,U+5748-5749,U+5752-5756,U+5758-5759,U+5762-5763,U+5765,U+576b-5776,U+5778-5781,U+5785-578a,U+578c-5791,U+5793-57a1,U+57a7-57aa,U+57ac-57ad,U+57af-57c2,U+57c4-57ca,U+57d0-57d3,U+57d5-57de,U+57e0-57f8,U+57fb-5801,U+5803-5805,U+5807-5814,U+5816-5820,U+5822-5823,U+5825-5829,U+582b-5833,U+5836-584b,U+584d-5850,U+5852-5853,U+5855-5857,U+5859-585d,U+585f-586a,U+586c-587a,U+59ab,U+651e,U+663a,U+6a50,U+6d65,U+7395,U+7817,U+78f2,U+7c9d,U+7f17,U+8233,U+8347,U+83f9,U+8764,U+8fc6,U+9506-9507,U+96e0,U+979e-979f,U+9899,U+9935,U+9e32,U+9f2f,U+fe31,U+ff1e,U+28090;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.26.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.26.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.26.otf')  format('opentype');unicode-range:U+59aa,U+6683-6686,U+6688-6690,U+6692-6695,U+6698-66a6,U+66a9-66ad,U+66af-66b3,U+66b5-66dc,U+66de-66ef,U+66f1,U+66f5-66f8,U+66fa-66fb,U+66fd,U+6701-6702,U+6704-6707,U+670a,U+670c,U+670e-6713,U+6715-6716,U+6718-671a,U+671c,U+671e,U+6720-6725,U+6729,U+672d-672e,U+6730,U+6732-6733,U+6736-6739,U+673b-673c,U+673e-673f,U+6741,U+6744-6745,U+6747-6748,U+674a-674d,U+6752-6755,U+6757-675b,U+675d-675e,U+6762-6764,U+6766-6767,U+6769-676c,U+676e,U+6771-677d,U+6780,U+6782-6783,U+6785-6788,U+678a-678f,U+6791-6794,U+6796,U+6798-6799,U+679b,U+679e-67a1,U+67a4-67a9,U+67ac-67ae,U+67b0-67b5,U+67b7-67c3,U+67c5-67cd,U+67d2,U+67d5-67db,U+67dd-67df,U+67e1-67e4,U+67e6-67ee,U+67f0,U+67f2,U+67f5-67fe,U+6800-6804,U+6806,U+6809-680a,U+680c-680e,U+6810,U+6812,U+6814-6815,U+6818-6820,U+6822-6829,U+682b-6836,U+683a-683b,U+683e-6841,U+6844-6845,U+6847,U+6849-684b,U+684d-684f,U+6852,U+6855-6862,U+6864,U+6866-6868,U+686a-6875,U+6877-6880,U+6882-6884,U+6886-6892,U+7c08,U+94aa-94ad,U+9511-9512,U+9e22,U+27870;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.26.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.26.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.26.otf')  format('opentype');unicode-range:U+59aa,U+6683-6686,U+6688-6690,U+6692-6695,U+6698-66a6,U+66a9-66ad,U+66af-66b3,U+66b5-66dc,U+66de-66ef,U+66f1,U+66f5-66f8,U+66fa-66fb,U+66fd,U+6701-6702,U+6704-6707,U+670a,U+670c,U+670e-6713,U+6715-6716,U+6718-671a,U+671c,U+671e,U+6720-6725,U+6729,U+672d-672e,U+6730,U+6732-6733,U+6736-6739,U+673b-673c,U+673e-673f,U+6741,U+6744-6745,U+6747-6748,U+674a-674d,U+6752-6755,U+6757-675b,U+675d-675e,U+6762-6764,U+6766-6767,U+6769-676c,U+676e,U+6771-677d,U+6780,U+6782-6783,U+6785-6788,U+678a-678f,U+6791-6794,U+6796,U+6798-6799,U+679b,U+679e-67a1,U+67a4-67a9,U+67ac-67ae,U+67b0-67b5,U+67b7-67c3,U+67c5-67cd,U+67d2,U+67d5-67db,U+67dd-67df,U+67e1-67e4,U+67e6-67ee,U+67f0,U+67f2,U+67f5-67fe,U+6800-6804,U+6806,U+6809-680a,U+680c-680e,U+6810,U+6812,U+6814-6815,U+6818-6820,U+6822-6829,U+682b-6836,U+683a-683b,U+683e-6841,U+6844-6845,U+6847,U+6849-684b,U+684d-684f,U+6852,U+6855-6862,U+6864,U+6866-6868,U+686a-6875,U+6877-6880,U+6882-6884,U+6886-6892,U+7c08,U+94aa-94ad,U+9511-9512,U+9e22,U+27870;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.26.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.26.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.26.otf')  format('opentype');unicode-range:U+59aa,U+6683-6686,U+6688-6690,U+6692-6695,U+6698-66a6,U+66a9-66ad,U+66af-66b3,U+66b5-66dc,U+66de-66ef,U+66f1,U+66f5-66f8,U+66fa-66fb,U+66fd,U+6701-6702,U+6704-6707,U+670a,U+670c,U+670e-6713,U+6715-6716,U+6718-671a,U+671c,U+671e,U+6720-6725,U+6729,U+672d-672e,U+6730,U+6732-6733,U+6736-6739,U+673b-673c,U+673e-673f,U+6741,U+6744-6745,U+6747-6748,U+674a-674d,U+6752-6755,U+6757-675b,U+675d-675e,U+6762-6764,U+6766-6767,U+6769-676c,U+676e,U+6771-677d,U+6780,U+6782-6783,U+6785-6788,U+678a-678f,U+6791-6794,U+6796,U+6798-6799,U+679b,U+679e-67a1,U+67a4-67a9,U+67ac-67ae,U+67b0-67b5,U+67b7-67c3,U+67c5-67cd,U+67d2,U+67d5-67db,U+67dd-67df,U+67e1-67e4,U+67e6-67ee,U+67f0,U+67f2,U+67f5-67fe,U+6800-6804,U+6806,U+6809-680a,U+680c-680e,U+6810,U+6812,U+6814-6815,U+6818-6820,U+6822-6829,U+682b-6836,U+683a-683b,U+683e-6841,U+6844-6845,U+6847,U+6849-684b,U+684d-684f,U+6852,U+6855-6862,U+6864,U+6866-6868,U+686a-6875,U+6877-6880,U+6882-6884,U+6886-6892,U+7c08,U+94aa-94ad,U+9511-9512,U+9e22,U+27870;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.27.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.27.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.27.otf')  format('opentype');unicode-range:U+2105,U+22a0,U+51fe,U+58ef,U+5cc4,U+62ee,U+6391,U+6c24-6c26,U+6c29-6c2d,U+6c30-6c33,U+6c36-6c37,U+6c39-6c3f,U+6c43-6c46,U+6c48,U+6c4a-6c56,U+6c58-6c5c,U+6c5e,U+6c62-6c63,U+6c65-6c69,U+6c6b-6c6f,U+6c73-6c75,U+6c77-6c78,U+6c7b-6c7c,U+6c7e-6c80,U+6c84-6c87,U+6c8a-6c8f,U+6c91-6c97,U+6c9c-6c9e,U+6ca0,U+6ca2-6ca4,U+6ca8-6ca9,U+6cac-6cb2,U+6cb4-6cb7,U+6cbd,U+6cc0-6cc3,U+6cc5-6cc8,U+6ccb,U+6ccd-6cd4,U+6cd6-6cda,U+6cdc-6ce0,U+6ce4,U+6ce6-6ce7,U+6ce9,U+6ceb-6cef,U+6cf1-6cf2,U+6cf4,U+6cf6-6cfa,U+6cfe-6d00,U+6d02-6d0a,U+6d0c-6d0d,U+6d0f-6d11,U+6d13-6d16,U+6d18-6d1a,U+6d1c-6d1d,U+6d1f-6d24,U+6d26-6d29,U+6d2b-6d31,U+6d33-6d3a,U+6d3c,U+6d3f-6d40,U+6d42-6d43,U+6d48-6d49,U+6d4c-6d4d,U+6d50,U+6d52,U+6d54-6d58,U+6d5a-6d64,U+6d67-6d68,U+6d6b-6d6d,U+6d6f-6d73,U+6d75-6d76,U+6d79-6d81,U+6d83-6d84,U+6d86-6d87,U+6d8a-6d8b,U+6d8d-6d94,U+6d96-6d9a,U+6d9c-6da0,U+6da2-6da3,U+6da5,U+6daa-6dae,U+6db0-6db1,U+6db3-6db4,U+6db6-6dbf,U+6dc1-6dc5,U+6dc8-6dca,U+6dcd-6dd0,U+6dd2-6dd7,U+6dd9-6de0,U+6de2-6de3,U+6de5-6dea,U+6dec-6ded,U+6def-6df0,U+6df2,U+6df4-6df6,U+6df8,U+6dfa,U+6dfc-6e04,U+6e06-6e09,U+6e0b-6e0f,U+6e11-6e13,U+6e15-6e16,U+6e18-6e1c,U+6e1e-6e1f,U+6e22,U+6e24-6e28,U+6e2a-6e2e,U+6e30-6e31,U+6e33,U+6e35-6e37,U+6e39,U+7a9e,U+8e1d-8e1e,U+8f9d,U+94d6-94db,U+951a-9520,U+960b-960d,U+9e20,U+26cc3;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.27.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.27.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.27.otf')  format('opentype');unicode-range:U+2105,U+22a0,U+51fe,U+58ef,U+5cc4,U+62ee,U+6391,U+6c24-6c26,U+6c29-6c2d,U+6c30-6c33,U+6c36-6c37,U+6c39-6c3f,U+6c43-6c46,U+6c48,U+6c4a-6c56,U+6c58-6c5c,U+6c5e,U+6c62-6c63,U+6c65-6c69,U+6c6b-6c6f,U+6c73-6c75,U+6c77-6c78,U+6c7b-6c7c,U+6c7e-6c80,U+6c84-6c87,U+6c8a-6c8f,U+6c91-6c97,U+6c9c-6c9e,U+6ca0,U+6ca2-6ca4,U+6ca8-6ca9,U+6cac-6cb2,U+6cb4-6cb7,U+6cbd,U+6cc0-6cc3,U+6cc5-6cc8,U+6ccb,U+6ccd-6cd4,U+6cd6-6cda,U+6cdc-6ce0,U+6ce4,U+6ce6-6ce7,U+6ce9,U+6ceb-6cef,U+6cf1-6cf2,U+6cf4,U+6cf6-6cfa,U+6cfe-6d00,U+6d02-6d0a,U+6d0c-6d0d,U+6d0f-6d11,U+6d13-6d16,U+6d18-6d1a,U+6d1c-6d1d,U+6d1f-6d24,U+6d26-6d29,U+6d2b-6d31,U+6d33-6d3a,U+6d3c,U+6d3f-6d40,U+6d42-6d43,U+6d48-6d49,U+6d4c-6d4d,U+6d50,U+6d52,U+6d54-6d58,U+6d5a-6d64,U+6d67-6d68,U+6d6b-6d6d,U+6d6f-6d73,U+6d75-6d76,U+6d79-6d81,U+6d83-6d84,U+6d86-6d87,U+6d8a-6d8b,U+6d8d-6d94,U+6d96-6d9a,U+6d9c-6da0,U+6da2-6da3,U+6da5,U+6daa-6dae,U+6db0-6db1,U+6db3-6db4,U+6db6-6dbf,U+6dc1-6dc5,U+6dc8-6dca,U+6dcd-6dd0,U+6dd2-6dd7,U+6dd9-6de0,U+6de2-6de3,U+6de5-6dea,U+6dec-6ded,U+6def-6df0,U+6df2,U+6df4-6df6,U+6df8,U+6dfa,U+6dfc-6e04,U+6e06-6e09,U+6e0b-6e0f,U+6e11-6e13,U+6e15-6e16,U+6e18-6e1c,U+6e1e-6e1f,U+6e22,U+6e24-6e28,U+6e2a-6e2e,U+6e30-6e31,U+6e33,U+6e35-6e37,U+6e39,U+7a9e,U+8e1d-8e1e,U+8f9d,U+94d6-94db,U+951a-9520,U+960b-960d,U+9e20,U+26cc3;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.27.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.27.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.27.otf')  format('opentype');unicode-range:U+2105,U+22a0,U+51fe,U+58ef,U+5cc4,U+62ee,U+6391,U+6c24-6c26,U+6c29-6c2d,U+6c30-6c33,U+6c36-6c37,U+6c39-6c3f,U+6c43-6c46,U+6c48,U+6c4a-6c56,U+6c58-6c5c,U+6c5e,U+6c62-6c63,U+6c65-6c69,U+6c6b-6c6f,U+6c73-6c75,U+6c77-6c78,U+6c7b-6c7c,U+6c7e-6c80,U+6c84-6c87,U+6c8a-6c8f,U+6c91-6c97,U+6c9c-6c9e,U+6ca0,U+6ca2-6ca4,U+6ca8-6ca9,U+6cac-6cb2,U+6cb4-6cb7,U+6cbd,U+6cc0-6cc3,U+6cc5-6cc8,U+6ccb,U+6ccd-6cd4,U+6cd6-6cda,U+6cdc-6ce0,U+6ce4,U+6ce6-6ce7,U+6ce9,U+6ceb-6cef,U+6cf1-6cf2,U+6cf4,U+6cf6-6cfa,U+6cfe-6d00,U+6d02-6d0a,U+6d0c-6d0d,U+6d0f-6d11,U+6d13-6d16,U+6d18-6d1a,U+6d1c-6d1d,U+6d1f-6d24,U+6d26-6d29,U+6d2b-6d31,U+6d33-6d3a,U+6d3c,U+6d3f-6d40,U+6d42-6d43,U+6d48-6d49,U+6d4c-6d4d,U+6d50,U+6d52,U+6d54-6d58,U+6d5a-6d64,U+6d67-6d68,U+6d6b-6d6d,U+6d6f-6d73,U+6d75-6d76,U+6d79-6d81,U+6d83-6d84,U+6d86-6d87,U+6d8a-6d8b,U+6d8d-6d94,U+6d96-6d9a,U+6d9c-6da0,U+6da2-6da3,U+6da5,U+6daa-6dae,U+6db0-6db1,U+6db3-6db4,U+6db6-6dbf,U+6dc1-6dc5,U+6dc8-6dca,U+6dcd-6dd0,U+6dd2-6dd7,U+6dd9-6de0,U+6de2-6de3,U+6de5-6dea,U+6dec-6ded,U+6def-6df0,U+6df2,U+6df4-6df6,U+6df8,U+6dfa,U+6dfc-6e04,U+6e06-6e09,U+6e0b-6e0f,U+6e11-6e13,U+6e15-6e16,U+6e18-6e1c,U+6e1e-6e1f,U+6e22,U+6e24-6e28,U+6e2a-6e2e,U+6e30-6e31,U+6e33,U+6e35-6e37,U+6e39,U+7a9e,U+8e1d-8e1e,U+8f9d,U+94d6-94db,U+951a-9520,U+960b-960d,U+9e20,U+26cc3;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.28.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.28.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.28.otf')  format('opentype');unicode-range:U+b36a-b36f,U+b371-b3c3,U+b3c5-b3d8,U+b3da-b417,U+b419-b44f,U+b451-b4db,U+b4dd-b4df,U+b4e1-b4e3,U+b4e5-b50e,U+c7a6-c7a7;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.28.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.28.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.28.otf')  format('opentype');unicode-range:U+b36a-b36f,U+b371-b3c3,U+b3c5-b3d8,U+b3da-b417,U+b419-b44f,U+b451-b4db,U+b4dd-b4df,U+b4e1-b4e3,U+b4e5-b50e,U+c7a6-c7a7;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.28.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.28.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.28.otf')  format('opentype');unicode-range:U+b36a-b36f,U+b371-b3c3,U+b3c5-b3d8,U+b3da-b417,U+b419-b44f,U+b451-b4db,U+b4dd-b4df,U+b4e1-b4e3,U+b4e5-b50e,U+c7a6-c7a7;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.29.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.29.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.29.otf')  format('opentype');unicode-range:U+529c,U+5b84,U+6026,U+6598,U+20b1d,U+20f90,U+246d4,U+26d22-26deb,U+26e00-26e12,U+26e42-27785,U+2789d-28048,U+280bd-28473,U+28501-28e97;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.29.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.29.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.29.otf')  format('opentype');unicode-range:U+529c,U+5b84,U+6026,U+6598,U+20b1d,U+20f90,U+246d4,U+26d22-26deb,U+26e00-26e12,U+26e42-27785,U+2789d-28048,U+280bd-28473,U+28501-28e97;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.29.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.29.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.29.otf')  format('opentype');unicode-range:U+529c,U+5b84,U+6026,U+6598,U+20b1d,U+20f90,U+246d4,U+26d22-26deb,U+26e00-26e12,U+26e42-27785,U+2789d-28048,U+280bd-28473,U+28501-28e97;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.30.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.30.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.30.otf')  format('opentype');unicode-range:U+c7ab-c7ac,U+c88c-c8fb,U+c8fd-c8ff,U+c901-c910,U+c912-c9bf,U+c9c2-c9c3,U+c9c5-c9d0,U+c9d2-ca36,U+2b363;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.30.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.30.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.30.otf')  format('opentype');unicode-range:U+c7ab-c7ac,U+c88c-c8fb,U+c8fd-c8ff,U+c901-c910,U+c912-c9bf,U+c9c2-c9c3,U+c9c5-c9d0,U+c9d2-ca36,U+2b363;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.30.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.30.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.30.otf')  format('opentype');unicode-range:U+c7ab-c7ac,U+c88c-c8fb,U+c8fd-c8ff,U+c901-c910,U+c912-c9bf,U+c9c2-c9c3,U+c9c5-c9d0,U+c9d2-ca36,U+2b363;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.31.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.31.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.31.otf')  format('opentype');unicode-range:U+57cf,U+6300,U+6ed4-6ed5,U+8287-8291,U+8293-8298,U+829a-829b,U+829e-82a4,U+82a7-82ab,U+82ae,U+82b0,U+82b2,U+82b4-82b7,U+82ba-82bc,U+82be-82c6,U+82c8-82cc,U+82ce,U+82d0,U+82d2-82d3,U+82d5-82d6,U+82d8-82da,U+82dc-82de,U+82e0-82e4,U+82e7-82ee,U+82f0,U+82f2-82f8,U+82fa-8301,U+8306-830d,U+830f-8327,U+8329-832a,U+832c-8334,U+833a-8345,U+8348,U+834a-834f,U+8351,U+8353-835e,U+8360,U+8362,U+8364-8366,U+8368-836a,U+836c-836e,U+8370-8376,U+8378-8388,U+838a-838d,U+838f-8392,U+8394-839d,U+839f-83aa,U+83ac-83b0,U+83b3-83b6,U+83b8,U+83ba-83c6,U+83c8-83c9,U+83cb,U+83cd-83db,U+83dd-83df,U+83e1-83e8,U+83ea-83f0,U+83f3-83f7,U+83fa-8402,U+8405-840b,U+840f-841c,U+841e-8424,U+8426,U+8429-843c,U+843e-8456,U+8458-845a,U+845c-8460,U+8462,U+8464-8468,U+846a,U+8869-886a,U+9527-9528,U+9606-9608,U+960f,U+9637-963a;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.31.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.31.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.31.otf')  format('opentype');unicode-range:U+57cf,U+6300,U+6ed4-6ed5,U+8287-8291,U+8293-8298,U+829a-829b,U+829e-82a4,U+82a7-82ab,U+82ae,U+82b0,U+82b2,U+82b4-82b7,U+82ba-82bc,U+82be-82c6,U+82c8-82cc,U+82ce,U+82d0,U+82d2-82d3,U+82d5-82d6,U+82d8-82da,U+82dc-82de,U+82e0-82e4,U+82e7-82ee,U+82f0,U+82f2-82f8,U+82fa-8301,U+8306-830d,U+830f-8327,U+8329-832a,U+832c-8334,U+833a-8345,U+8348,U+834a-834f,U+8351,U+8353-835e,U+8360,U+8362,U+8364-8366,U+8368-836a,U+836c-836e,U+8370-8376,U+8378-8388,U+838a-838d,U+838f-8392,U+8394-839d,U+839f-83aa,U+83ac-83b0,U+83b3-83b6,U+83b8,U+83ba-83c6,U+83c8-83c9,U+83cb,U+83cd-83db,U+83dd-83df,U+83e1-83e8,U+83ea-83f0,U+83f3-83f7,U+83fa-8402,U+8405-840b,U+840f-841c,U+841e-8424,U+8426,U+8429-843c,U+843e-8456,U+8458-845a,U+845c-8460,U+8462,U+8464-8468,U+846a,U+8869-886a,U+9527-9528,U+9606-9608,U+960f,U+9637-963a;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.31.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.31.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.31.otf')  format('opentype');unicode-range:U+57cf,U+6300,U+6ed4-6ed5,U+8287-8291,U+8293-8298,U+829a-829b,U+829e-82a4,U+82a7-82ab,U+82ae,U+82b0,U+82b2,U+82b4-82b7,U+82ba-82bc,U+82be-82c6,U+82c8-82cc,U+82ce,U+82d0,U+82d2-82d3,U+82d5-82d6,U+82d8-82da,U+82dc-82de,U+82e0-82e4,U+82e7-82ee,U+82f0,U+82f2-82f8,U+82fa-8301,U+8306-830d,U+830f-8327,U+8329-832a,U+832c-8334,U+833a-8345,U+8348,U+834a-834f,U+8351,U+8353-835e,U+8360,U+8362,U+8364-8366,U+8368-836a,U+836c-836e,U+8370-8376,U+8378-8388,U+838a-838d,U+838f-8392,U+8394-839d,U+839f-83aa,U+83ac-83b0,U+83b3-83b6,U+83b8,U+83ba-83c6,U+83c8-83c9,U+83cb,U+83cd-83db,U+83dd-83df,U+83e1-83e8,U+83ea-83f0,U+83f3-83f7,U+83fa-8402,U+8405-840b,U+840f-841c,U+841e-8424,U+8426,U+8429-843c,U+843e-8456,U+8458-845a,U+845c-8460,U+8462,U+8464-8468,U+846a,U+8869-886a,U+9527-9528,U+9606-9608,U+960f,U+9637-963a;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.32.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.32.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.32.otf')  format('opentype');unicode-range:U+4e4a,U+54ab,U+66f3,U+775c-7760,U+7762,U+7764-7765,U+7767-7778,U+777a-777e,U+7780-7783,U+7785-778a,U+778c-778d,U+778f-7791,U+7793-77a6,U+77a8,U+77ab,U+77ad-77af,U+77b1-77ba,U+77bc-77da,U+77dc-77e1,U+77e4,U+77e6-77e8,U+77ea,U+77ec,U+77ef-77f2,U+77f4-77f5,U+77f7-77fe,U+7800,U+7803-780b,U+780e-7813,U+7815,U+7818-7833,U+7835-7837,U+7839-783f,U+7841-7844,U+7846-7854,U+7856-785c,U+785e-786a,U+786d,U+786f-788b,U+788f-7890,U+7892-7896,U+7898-789e,U+78a0-78a6,U+78a8-78af,U+78b2,U+78b4-78c0,U+78c2-78c4,U+78c6-78c9,U+78cb-78e7,U+78e9-78f0,U+78f3-78f6,U+78f8-7900,U+7902-7930,U+795f,U+963c-963e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.32.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.32.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.32.otf')  format('opentype');unicode-range:U+4e4a,U+54ab,U+66f3,U+775c-7760,U+7762,U+7764-7765,U+7767-7778,U+777a-777e,U+7780-7783,U+7785-778a,U+778c-778d,U+778f-7791,U+7793-77a6,U+77a8,U+77ab,U+77ad-77af,U+77b1-77ba,U+77bc-77da,U+77dc-77e1,U+77e4,U+77e6-77e8,U+77ea,U+77ec,U+77ef-77f2,U+77f4-77f5,U+77f7-77fe,U+7800,U+7803-780b,U+780e-7813,U+7815,U+7818-7833,U+7835-7837,U+7839-783f,U+7841-7844,U+7846-7854,U+7856-785c,U+785e-786a,U+786d,U+786f-788b,U+788f-7890,U+7892-7896,U+7898-789e,U+78a0-78a6,U+78a8-78af,U+78b2,U+78b4-78c0,U+78c2-78c4,U+78c6-78c9,U+78cb-78e7,U+78e9-78f0,U+78f3-78f6,U+78f8-7900,U+7902-7930,U+795f,U+963c-963e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.32.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.32.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.32.otf')  format('opentype');unicode-range:U+4e4a,U+54ab,U+66f3,U+775c-7760,U+7762,U+7764-7765,U+7767-7778,U+777a-777e,U+7780-7783,U+7785-778a,U+778c-778d,U+778f-7791,U+7793-77a6,U+77a8,U+77ab,U+77ad-77af,U+77b1-77ba,U+77bc-77da,U+77dc-77e1,U+77e4,U+77e6-77e8,U+77ea,U+77ec,U+77ef-77f2,U+77f4-77f5,U+77f7-77fe,U+7800,U+7803-780b,U+780e-7813,U+7815,U+7818-7833,U+7835-7837,U+7839-783f,U+7841-7844,U+7846-7854,U+7856-785c,U+785e-786a,U+786d,U+786f-788b,U+788f-7890,U+7892-7896,U+7898-789e,U+78a0-78a6,U+78a8-78af,U+78b2,U+78b4-78c0,U+78c2-78c4,U+78c6-78c9,U+78cb-78e7,U+78e9-78f0,U+78f3-78f6,U+78f8-7900,U+7902-7930,U+795f,U+963c-963e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.33.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.33.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.33.otf')  format('opentype');unicode-range:U+03c0-03c1,U+2030,U+2042,U+2109-210a,U+4da9-4db5,U+4e02,U+4e04-4e06,U+4e0f-4e10,U+4e12,U+4e15,U+4e17,U+4e1f-4e21,U+4e2c,U+4e2e-4e2f,U+4e31,U+4e33,U+4e35-4e37,U+4e3c,U+4e3f-4e42,U+4e44,U+4e46-4e47,U+4e51-4e53,U+4e55,U+4e5a-4e5c,U+4e62-4e65,U+4e67-4e6f,U+4e72,U+4e74-4e7d,U+4e7f-4e85,U+4e87,U+4e90,U+4e96-4e99,U+4e9c-4ea0,U+4ea3,U+4eaa,U+4eaf-4eb1,U+4eb3-4eb9,U+4ebc-4ebe,U+4ec2-4ec4,U+4ec8-4ec9,U+4ecc,U+4ecf-4ed0,U+4ed2,U+4eda-4ee2,U+4ee6-4ee9,U+4eeb,U+4eed-4eef,U+4ef1,U+4ef3-4ef5,U+4ef8-4efa,U+4efc,U+4efe,U+4f00,U+4f02-4f09,U+4f0b-4f0c,U+4f0e,U+4f12-4f16,U+4f1b-4f1d,U+4f21-4f23,U+4f25,U+4f27-4f29,U+4f2b-4f2e,U+4f31-4f33,U+4f37,U+4f39,U+4f3b,U+4f3e-4f45,U+4f47-4f4c,U+4f52,U+4f54,U+4f56-4f58,U+4f5a,U+4f5d-4f5f,U+4f61-4f62,U+4f64-4f68,U+4f6a-4f6b,U+4f6d-4f72,U+4f74-4f7e,U+4f80-4f82,U+4f85-4f87,U+4f89-4f8a,U+4f8e-4f9a,U+4f9c,U+4f9e-4f9f,U+4fa1-4fa2,U+4fa4-4fa5,U+4fa9-4fad,U+4fb0-4fb4,U+4fb6-4fbe,U+4fc0-4fc2,U+4fc5-4fc9,U+4fcb-4fce,U+4fd0-4fd6,U+4fd9-4fdc,U+4fdf-4fe0,U+4fe2-4fe7,U+4fea-4fec,U+4ff0,U+4ff2-4ff9,U+4ffb-500a,U+500c,U+500e-5011,U+5013,U+5015-5017,U+501b-501e,U+5020,U+5022-5025,U+5027-5028,U+502b-5039,U+503b,U+503d,U+503f-5046,U+5048-504e,U+5050-5054,U+5056-5059,U+58ed,U+5b82,U+5cda,U+6f31,U+76d3-76d5,U+79bc,U+9580-9582,U+9615-9616,U+9670-9671,U+28083,U+2b4ef;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.33.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.33.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.33.otf')  format('opentype');unicode-range:U+03c0-03c1,U+2030,U+2042,U+2109-210a,U+4da9-4db5,U+4e02,U+4e04-4e06,U+4e0f-4e10,U+4e12,U+4e15,U+4e17,U+4e1f-4e21,U+4e2c,U+4e2e-4e2f,U+4e31,U+4e33,U+4e35-4e37,U+4e3c,U+4e3f-4e42,U+4e44,U+4e46-4e47,U+4e51-4e53,U+4e55,U+4e5a-4e5c,U+4e62-4e65,U+4e67-4e6f,U+4e72,U+4e74-4e7d,U+4e7f-4e85,U+4e87,U+4e90,U+4e96-4e99,U+4e9c-4ea0,U+4ea3,U+4eaa,U+4eaf-4eb1,U+4eb3-4eb9,U+4ebc-4ebe,U+4ec2-4ec4,U+4ec8-4ec9,U+4ecc,U+4ecf-4ed0,U+4ed2,U+4eda-4ee2,U+4ee6-4ee9,U+4eeb,U+4eed-4eef,U+4ef1,U+4ef3-4ef5,U+4ef8-4efa,U+4efc,U+4efe,U+4f00,U+4f02-4f09,U+4f0b-4f0c,U+4f0e,U+4f12-4f16,U+4f1b-4f1d,U+4f21-4f23,U+4f25,U+4f27-4f29,U+4f2b-4f2e,U+4f31-4f33,U+4f37,U+4f39,U+4f3b,U+4f3e-4f45,U+4f47-4f4c,U+4f52,U+4f54,U+4f56-4f58,U+4f5a,U+4f5d-4f5f,U+4f61-4f62,U+4f64-4f68,U+4f6a-4f6b,U+4f6d-4f72,U+4f74-4f7e,U+4f80-4f82,U+4f85-4f87,U+4f89-4f8a,U+4f8e-4f9a,U+4f9c,U+4f9e-4f9f,U+4fa1-4fa2,U+4fa4-4fa5,U+4fa9-4fad,U+4fb0-4fb4,U+4fb6-4fbe,U+4fc0-4fc2,U+4fc5-4fc9,U+4fcb-4fce,U+4fd0-4fd6,U+4fd9-4fdc,U+4fdf-4fe0,U+4fe2-4fe7,U+4fea-4fec,U+4ff0,U+4ff2-4ff9,U+4ffb-500a,U+500c,U+500e-5011,U+5013,U+5015-5017,U+501b-501e,U+5020,U+5022-5025,U+5027-5028,U+502b-5039,U+503b,U+503d,U+503f-5046,U+5048-504e,U+5050-5054,U+5056-5059,U+58ed,U+5b82,U+5cda,U+6f31,U+76d3-76d5,U+79bc,U+9580-9582,U+9615-9616,U+9670-9671,U+28083,U+2b4ef;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.33.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.33.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.33.otf')  format('opentype');unicode-range:U+03c0-03c1,U+2030,U+2042,U+2109-210a,U+4da9-4db5,U+4e02,U+4e04-4e06,U+4e0f-4e10,U+4e12,U+4e15,U+4e17,U+4e1f-4e21,U+4e2c,U+4e2e-4e2f,U+4e31,U+4e33,U+4e35-4e37,U+4e3c,U+4e3f-4e42,U+4e44,U+4e46-4e47,U+4e51-4e53,U+4e55,U+4e5a-4e5c,U+4e62-4e65,U+4e67-4e6f,U+4e72,U+4e74-4e7d,U+4e7f-4e85,U+4e87,U+4e90,U+4e96-4e99,U+4e9c-4ea0,U+4ea3,U+4eaa,U+4eaf-4eb1,U+4eb3-4eb9,U+4ebc-4ebe,U+4ec2-4ec4,U+4ec8-4ec9,U+4ecc,U+4ecf-4ed0,U+4ed2,U+4eda-4ee2,U+4ee6-4ee9,U+4eeb,U+4eed-4eef,U+4ef1,U+4ef3-4ef5,U+4ef8-4efa,U+4efc,U+4efe,U+4f00,U+4f02-4f09,U+4f0b-4f0c,U+4f0e,U+4f12-4f16,U+4f1b-4f1d,U+4f21-4f23,U+4f25,U+4f27-4f29,U+4f2b-4f2e,U+4f31-4f33,U+4f37,U+4f39,U+4f3b,U+4f3e-4f45,U+4f47-4f4c,U+4f52,U+4f54,U+4f56-4f58,U+4f5a,U+4f5d-4f5f,U+4f61-4f62,U+4f64-4f68,U+4f6a-4f6b,U+4f6d-4f72,U+4f74-4f7e,U+4f80-4f82,U+4f85-4f87,U+4f89-4f8a,U+4f8e-4f9a,U+4f9c,U+4f9e-4f9f,U+4fa1-4fa2,U+4fa4-4fa5,U+4fa9-4fad,U+4fb0-4fb4,U+4fb6-4fbe,U+4fc0-4fc2,U+4fc5-4fc9,U+4fcb-4fce,U+4fd0-4fd6,U+4fd9-4fdc,U+4fdf-4fe0,U+4fe2-4fe7,U+4fea-4fec,U+4ff0,U+4ff2-4ff9,U+4ffb-500a,U+500c,U+500e-5011,U+5013,U+5015-5017,U+501b-501e,U+5020,U+5022-5025,U+5027-5028,U+502b-5039,U+503b,U+503d,U+503f-5046,U+5048-504e,U+5050-5054,U+5056-5059,U+58ed,U+5b82,U+5cda,U+6f31,U+76d3-76d5,U+79bc,U+9580-9582,U+9615-9616,U+9670-9671,U+28083,U+2b4ef;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.34.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.34.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.34.otf')  format('opentype');unicode-range:U+4e28-4e29,U+4e8a,U+62f0,U+7eb4,U+88da-88db,U+20b0d,U+20e1d,U+21096,U+23781,U+25f1a,U+2644a,U+26484,U+28e99,U+28eb2-2b138,U+2b410-2b413,U+2b4f6-2cb73,U+2cb78-2f921,U+2f96c-2f9df;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.34.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.34.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.34.otf')  format('opentype');unicode-range:U+4e28-4e29,U+4e8a,U+62f0,U+7eb4,U+88da-88db,U+20b0d,U+20e1d,U+21096,U+23781,U+25f1a,U+2644a,U+26484,U+28e99,U+28eb2-2b138,U+2b410-2b413,U+2b4f6-2cb73,U+2cb78-2f921,U+2f96c-2f9df;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.34.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.34.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.34.otf')  format('opentype');unicode-range:U+4e28-4e29,U+4e8a,U+62f0,U+7eb4,U+88da-88db,U+20b0d,U+20e1d,U+21096,U+23781,U+25f1a,U+2644a,U+26484,U+28e99,U+28eb2-2b138,U+2b410-2b413,U+2b4f6-2cb73,U+2cb78-2f921,U+2f96c-2f9df;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.35.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.35.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.35.otf')  format('opentype');unicode-range:U+2243,U+4e26,U+7617,U+9665-9668,U+9672-9674,U+984a-9874,U+9878,U+9880,U+9883,U+988b-988f,U+9892,U+9894-9895,U+989a-989b,U+98a1-98a3,U+98a5-98cd,U+98cf-98d7,U+98da-98dd,U+98e0-990f,U+9911-9934,U+9936-9964,U+9966-9969,U+996b-996c,U+996f,U+9973-9975,U+9977-9978,U+997a-997b,U+997d-997e,U+9980-9984,U+9987,U+9989-9995,U+9997-9998,U+999a-99a7,U+99a9-9a16;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.35.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.35.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.35.otf')  format('opentype');unicode-range:U+2243,U+4e26,U+7617,U+9665-9668,U+9672-9674,U+984a-9874,U+9878,U+9880,U+9883,U+988b-988f,U+9892,U+9894-9895,U+989a-989b,U+98a1-98a3,U+98a5-98cd,U+98cf-98d7,U+98da-98dd,U+98e0-990f,U+9911-9934,U+9936-9964,U+9966-9969,U+996b-996c,U+996f,U+9973-9975,U+9977-9978,U+997a-997b,U+997d-997e,U+9980-9984,U+9987,U+9989-9995,U+9997-9998,U+999a-99a7,U+99a9-9a16;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.35.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.35.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.35.otf')  format('opentype');unicode-range:U+2243,U+4e26,U+7617,U+9665-9668,U+9672-9674,U+984a-9874,U+9878,U+9880,U+9883,U+988b-988f,U+9892,U+9894-9895,U+989a-989b,U+98a1-98a3,U+98a5-98cd,U+98cf-98d7,U+98da-98dd,U+98e0-990f,U+9911-9934,U+9936-9964,U+9966-9969,U+996b-996c,U+996f,U+9973-9975,U+9977-9978,U+997a-997b,U+997d-997e,U+9980-9984,U+9987,U+9989-9995,U+9997-9998,U+999a-99a7,U+99a9-9a16;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.36.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.36.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.36.otf')  format('opentype');unicode-range:U+574c-574d,U+7b15-7b1a,U+7b1c-7b25,U+7b27,U+7b29-7b2b,U+7b2d-7b3b,U+7b3d-7b48,U+7b4a,U+7b4c-7b50,U+7b53,U+7b55,U+7b57-7b5a,U+7b5c-7b76,U+7b78,U+7b7a-7b7d,U+7b81-7b96,U+7b98-7ba0,U+7ba2-7bac,U+7bae-7bb0,U+7bb2-7bc5,U+7bc8-7be0,U+7be2-7bed,U+7bef-7bf6,U+7bf8-7c06,U+7c09-7c3e,U+7c40-7c4c,U+7c4e-7c72,U+7c74-7c7a,U+7c7c,U+7c7e-7c88,U+7c8a-7c91,U+7c93-7c96,U+7c99-7c9c,U+7c9e,U+7ca0-7ca3,U+7ca6-7ca9,U+7cab-7cad,U+7caf-7cb8,U+7cba-7cbd,U+7cbf-7cc9,U+7ccb-7cd4,U+7cd7-7cd8,U+7cda-7cde,U+948c-948f;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.36.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.36.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.36.otf')  format('opentype');unicode-range:U+574c-574d,U+7b15-7b1a,U+7b1c-7b25,U+7b27,U+7b29-7b2b,U+7b2d-7b3b,U+7b3d-7b48,U+7b4a,U+7b4c-7b50,U+7b53,U+7b55,U+7b57-7b5a,U+7b5c-7b76,U+7b78,U+7b7a-7b7d,U+7b81-7b96,U+7b98-7ba0,U+7ba2-7bac,U+7bae-7bb0,U+7bb2-7bc5,U+7bc8-7be0,U+7be2-7bed,U+7bef-7bf6,U+7bf8-7c06,U+7c09-7c3e,U+7c40-7c4c,U+7c4e-7c72,U+7c74-7c7a,U+7c7c,U+7c7e-7c88,U+7c8a-7c91,U+7c93-7c96,U+7c99-7c9c,U+7c9e,U+7ca0-7ca3,U+7ca6-7ca9,U+7cab-7cad,U+7caf-7cb8,U+7cba-7cbd,U+7cbf-7cc9,U+7ccb-7cd4,U+7cd7-7cd8,U+7cda-7cde,U+948c-948f;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.36.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.36.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.36.otf')  format('opentype');unicode-range:U+574c-574d,U+7b15-7b1a,U+7b1c-7b25,U+7b27,U+7b29-7b2b,U+7b2d-7b3b,U+7b3d-7b48,U+7b4a,U+7b4c-7b50,U+7b53,U+7b55,U+7b57-7b5a,U+7b5c-7b76,U+7b78,U+7b7a-7b7d,U+7b81-7b96,U+7b98-7ba0,U+7ba2-7bac,U+7bae-7bb0,U+7bb2-7bc5,U+7bc8-7be0,U+7be2-7bed,U+7bef-7bf6,U+7bf8-7c06,U+7c09-7c3e,U+7c40-7c4c,U+7c4e-7c72,U+7c74-7c7a,U+7c7c,U+7c7e-7c88,U+7c8a-7c91,U+7c93-7c96,U+7c99-7c9c,U+7c9e,U+7ca0-7ca3,U+7ca6-7ca9,U+7cab-7cad,U+7caf-7cb8,U+7cba-7cbd,U+7cbf-7cc9,U+7ccb-7cd4,U+7cd7-7cd8,U+7cda-7cde,U+948c-948f;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.37.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.37.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.37.otf')  format('opentype');unicode-range:U+6c71,U+9131-914b,U+914e-9151,U+9153-9164,U+9166-9169,U+916d,U+9170,U+9172-9174,U+9179-917e,U+9180-9186,U+9188,U+918a,U+918c-9191,U+9193-91c6,U+91c8-91c9,U+91d2-9273,U+9275-92ea,U+22c4c;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.37.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.37.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.37.otf')  format('opentype');unicode-range:U+6c71,U+9131-914b,U+914e-9151,U+9153-9164,U+9166-9169,U+916d,U+9170,U+9172-9174,U+9179-917e,U+9180-9186,U+9188,U+918a,U+918c-9191,U+9193-91c6,U+91c8-91c9,U+91d2-9273,U+9275-92ea,U+22c4c;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.37.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.37.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.37.otf')  format('opentype');unicode-range:U+6c71,U+9131-914b,U+914e-9151,U+9153-9164,U+9166-9169,U+916d,U+9170,U+9172-9174,U+9179-917e,U+9180-9186,U+9188,U+918a,U+918c-9191,U+9193-91c6,U+91c8-91c9,U+91d2-9273,U+9275-92ea,U+22c4c;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.38.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.38.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.38.otf')  format('opentype');unicode-range:U+1186-2012,U+201e-2021,U+2025,U+2039-203c,U+2047-20de,U+210f,U+2116-213b,U+2162-2191,U+2193-21f5,U+2202-2229,U+222b-222e,U+2236-2237,U+2245-2248,U+2260-2299,U+22a5-2307,U+2329-23c8,U+2502,U+252c,U+309c,U+4e8d,U+4ebb,U+5155,U+55c4,U+570b,U+575c,U+5cc1,U+5e11,U+5f73,U+5fbc,U+6042,U+6206,U+6265-6266,U+6426,U+6b38,U+6c35,U+6d44,U+72b4-72b5,U+755a-755b,U+778b,U+7f2f,U+83f8,U+88fd,U+898b,U+89dc,U+8ba0,U+8bf3,U+8df6,U+9577,U+961d,U+9b2f,U+9fa2,U+ff3e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.38.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.38.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.38.otf')  format('opentype');unicode-range:U+1186-2012,U+201e-2021,U+2025,U+2039-203c,U+2047-20de,U+210f,U+2116-213b,U+2162-2191,U+2193-21f5,U+2202-2229,U+222b-222e,U+2236-2237,U+2245-2248,U+2260-2299,U+22a5-2307,U+2329-23c8,U+2502,U+252c,U+309c,U+4e8d,U+4ebb,U+5155,U+55c4,U+570b,U+575c,U+5cc1,U+5e11,U+5f73,U+5fbc,U+6042,U+6206,U+6265-6266,U+6426,U+6b38,U+6c35,U+6d44,U+72b4-72b5,U+755a-755b,U+778b,U+7f2f,U+83f8,U+88fd,U+898b,U+89dc,U+8ba0,U+8bf3,U+8df6,U+9577,U+961d,U+9b2f,U+9fa2,U+ff3e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.38.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.38.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.38.otf')  format('opentype');unicode-range:U+1186-2012,U+201e-2021,U+2025,U+2039-203c,U+2047-20de,U+210f,U+2116-213b,U+2162-2191,U+2193-21f5,U+2202-2229,U+222b-222e,U+2236-2237,U+2245-2248,U+2260-2299,U+22a5-2307,U+2329-23c8,U+2502,U+252c,U+309c,U+4e8d,U+4ebb,U+5155,U+55c4,U+570b,U+575c,U+5cc1,U+5e11,U+5f73,U+5fbc,U+6042,U+6206,U+6265-6266,U+6426,U+6b38,U+6c35,U+6d44,U+72b4-72b5,U+755a-755b,U+778b,U+7f2f,U+83f8,U+88fd,U+898b,U+89dc,U+8ba0,U+8bf3,U+8df6,U+9577,U+961d,U+9b2f,U+9fa2,U+ff3e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.39.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.39.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.39.otf')  format('opentype');unicode-range:U+d0b5-d0bf,U+d0c1-d0db,U+d0dd-d255;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.39.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.39.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.39.otf')  format('opentype');unicode-range:U+d0b5-d0bf,U+d0c1-d0db,U+d0dd-d255;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.39.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.39.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.39.otf')  format('opentype');unicode-range:U+d0b5-d0bf,U+d0c1-d0db,U+d0dd-d255;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.40.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.40.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.40.otf')  format('opentype');unicode-range:U+2113,U+4f8c,U+663d,U+9f30-9f3a,U+9f3c-9f4f,U+9f51-9f7e,U+9f80-9f83,U+9f85-9f98,U+9f9a-9f9e,U+9fa0-9fa1,U+9fa3-a97c,U+ac02-ac03,U+ac05-ac0f,U+ac11-ac18,U+ac1a-ac6f,U+ac71-ac77,U+ac79-ac82,U+ac84-ac8b,U+ac8d-ac9f,U+aca1-acaf,U+acb1-acbc,U+acbe-acc3,U+acc5-acdf,U+ace1-ace7,U+c6ff-c700,U+c7cc-c7d0,U+c841,U+c875-c878;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.40.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.40.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.40.otf')  format('opentype');unicode-range:U+2113,U+4f8c,U+663d,U+9f30-9f3a,U+9f3c-9f4f,U+9f51-9f7e,U+9f80-9f83,U+9f85-9f98,U+9f9a-9f9e,U+9fa0-9fa1,U+9fa3-a97c,U+ac02-ac03,U+ac05-ac0f,U+ac11-ac18,U+ac1a-ac6f,U+ac71-ac77,U+ac79-ac82,U+ac84-ac8b,U+ac8d-ac9f,U+aca1-acaf,U+acb1-acbc,U+acbe-acc3,U+acc5-acdf,U+ace1-ace7,U+c6ff-c700,U+c7cc-c7d0,U+c841,U+c875-c878;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.40.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.40.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.40.otf')  format('opentype');unicode-range:U+2113,U+4f8c,U+663d,U+9f30-9f3a,U+9f3c-9f4f,U+9f51-9f7e,U+9f80-9f83,U+9f85-9f98,U+9f9a-9f9e,U+9fa0-9fa1,U+9fa3-a97c,U+ac02-ac03,U+ac05-ac0f,U+ac11-ac18,U+ac1a-ac6f,U+ac71-ac77,U+ac79-ac82,U+ac84-ac8b,U+ac8d-ac9f,U+aca1-acaf,U+acb1-acbc,U+acbe-acc3,U+acc5-acdf,U+ace1-ace7,U+c6ff-c700,U+c7cc-c7d0,U+c841,U+c875-c878;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.41.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.41.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.41.otf')  format('opentype');unicode-range:U+55ac,U+6347-6348,U+6c23,U+6c98,U+751b,U+94bc-94bd,U+94ee-94ef,U+9a17-9a6b,U+9a72,U+9a75,U+9a77-9a7a,U+9a80-9a81,U+9a85,U+9a88-9a8b,U+9a8d-9a8e,U+9a90,U+9a92-9a96,U+9a98-9a99,U+9a9b-9aa3,U+9aa5-9aa7,U+9aa9-9abb,U+9abd-9ad2,U+9ad4-9ad7,U+9ad9-9ae5,U+9ae7-9b2e,U+9b30-9b3b,U+9b3d-9b40,U+9b43,U+9b46-9b4e,U+9b50-9b53,U+9b55-9bd1;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.41.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.41.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.41.otf')  format('opentype');unicode-range:U+55ac,U+6347-6348,U+6c23,U+6c98,U+751b,U+94bc-94bd,U+94ee-94ef,U+9a17-9a6b,U+9a72,U+9a75,U+9a77-9a7a,U+9a80-9a81,U+9a85,U+9a88-9a8b,U+9a8d-9a8e,U+9a90,U+9a92-9a96,U+9a98-9a99,U+9a9b-9aa3,U+9aa5-9aa7,U+9aa9-9abb,U+9abd-9ad2,U+9ad4-9ad7,U+9ad9-9ae5,U+9ae7-9b2e,U+9b30-9b3b,U+9b3d-9b40,U+9b43,U+9b46-9b4e,U+9b50-9b53,U+9b55-9bd1;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.41.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.41.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.41.otf')  format('opentype');unicode-range:U+55ac,U+6347-6348,U+6c23,U+6c98,U+751b,U+94bc-94bd,U+94ee-94ef,U+9a17-9a6b,U+9a72,U+9a75,U+9a77-9a7a,U+9a80-9a81,U+9a85,U+9a88-9a8b,U+9a8d-9a8e,U+9a90,U+9a92-9a96,U+9a98-9a99,U+9a9b-9aa3,U+9aa5-9aa7,U+9aa9-9abb,U+9abd-9ad2,U+9ad4-9ad7,U+9ad9-9ae5,U+9ae7-9b2e,U+9b30-9b3b,U+9b3d-9b40,U+9b43,U+9b46-9b4e,U+9b50-9b53,U+9b55-9bd1;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.42.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.42.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.42.otf')  format('opentype');unicode-range:U+9e3e,U+b1c3-b207,U+b209-b293,U+b295-b2c7,U+b2c9-b2e3,U+b2e5-b2e7,U+b2e9-b2eb,U+b2ed-b2f8,U+b2fa-b2ff,U+b301-b369;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.42.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.42.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.42.otf')  format('opentype');unicode-range:U+9e3e,U+b1c3-b207,U+b209-b293,U+b295-b2c7,U+b2c9-b2e3,U+b2e5-b2e7,U+b2e9-b2eb,U+b2ed-b2f8,U+b2fa-b2ff,U+b301-b369;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.42.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.42.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.42.otf')  format('opentype');unicode-range:U+9e3e,U+b1c3-b207,U+b209-b293,U+b295-b2c7,U+b2c9-b2e3,U+b2e5-b2e7,U+b2e9-b2eb,U+b2ed-b2f8,U+b2fa-b2ff,U+b301-b369;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.43.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.43.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.43.otf')  format('opentype');unicode-range:U+6894-6896,U+6898-68a1,U+68a3-68a5,U+68a9-68ac,U+68ae,U+68b1-68b2,U+68b4,U+68b6-68bf,U+68c1-68c8,U+68ca,U+68cc,U+68ce-68d1,U+68d3-68d4,U+68d6-68d7,U+68d9,U+68db-68df,U+68e1-68ed,U+68ef-68f0,U+68f2-68f4,U+68f6-68f9,U+68fb-6904,U+6906-690c,U+690f-6911,U+6913-692c,U+692e-692f,U+6931-6959,U+695b-695f,U+6961-6976,U+6978-697b,U+697d-6981,U+6983,U+6985-699b,U+699d-69a7,U+69a9-69b3,U+69b5-69da,U+69dc-69fc,U+69fe-6a09,U+6a0b-6a20,U+6a22-6a29,U+6a2b-6a30,U+6a32-6a43,U+6a45-6a4f,U+6a51-6a55,U+710b,U+7eb6,U+9494-9495,U+9515-9518,U+95f5-95f6,U+96e1;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.43.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.43.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.43.otf')  format('opentype');unicode-range:U+6894-6896,U+6898-68a1,U+68a3-68a5,U+68a9-68ac,U+68ae,U+68b1-68b2,U+68b4,U+68b6-68bf,U+68c1-68c8,U+68ca,U+68cc,U+68ce-68d1,U+68d3-68d4,U+68d6-68d7,U+68d9,U+68db-68df,U+68e1-68ed,U+68ef-68f0,U+68f2-68f4,U+68f6-68f9,U+68fb-6904,U+6906-690c,U+690f-6911,U+6913-692c,U+692e-692f,U+6931-6959,U+695b-695f,U+6961-6976,U+6978-697b,U+697d-6981,U+6983,U+6985-699b,U+699d-69a7,U+69a9-69b3,U+69b5-69da,U+69dc-69fc,U+69fe-6a09,U+6a0b-6a20,U+6a22-6a29,U+6a2b-6a30,U+6a32-6a43,U+6a45-6a4f,U+6a51-6a55,U+710b,U+7eb6,U+9494-9495,U+9515-9518,U+95f5-95f6,U+96e1;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.43.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.43.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.43.otf')  format('opentype');unicode-range:U+6894-6896,U+6898-68a1,U+68a3-68a5,U+68a9-68ac,U+68ae,U+68b1-68b2,U+68b4,U+68b6-68bf,U+68c1-68c8,U+68ca,U+68cc,U+68ce-68d1,U+68d3-68d4,U+68d6-68d7,U+68d9,U+68db-68df,U+68e1-68ed,U+68ef-68f0,U+68f2-68f4,U+68f6-68f9,U+68fb-6904,U+6906-690c,U+690f-6911,U+6913-692c,U+692e-692f,U+6931-6959,U+695b-695f,U+6961-6976,U+6978-697b,U+697d-6981,U+6983,U+6985-699b,U+699d-69a7,U+69a9-69b3,U+69b5-69da,U+69dc-69fc,U+69fe-6a09,U+6a0b-6a20,U+6a22-6a29,U+6a2b-6a30,U+6a32-6a43,U+6a45-6a4f,U+6a51-6a55,U+710b,U+7eb6,U+9494-9495,U+9515-9518,U+95f5-95f6,U+96e1;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.44.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.44.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.44.otf')  format('opentype');unicode-range:U+31fe-339e,U+94e6,U+9612-9613;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.44.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.44.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.44.otf')  format('opentype');unicode-range:U+31fe-339e,U+94e6,U+9612-9613;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.44.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.44.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.44.otf')  format('opentype');unicode-range:U+31fe-339e,U+94e6,U+9612-9613;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.45.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.45.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.45.otf')  format('opentype');unicode-range:U+60ea,U+6e3c-6e55,U+6e57,U+6e59-6e5a,U+6e5c-6e7d,U+6e80-6e82,U+6e84-6e8f,U+6e91-6e9b,U+6e9d-6ea1,U+6ea3-6ea9,U+6eab-6eae,U+6eb0-6eb5,U+6eb7-6eb8,U+6ebb-6eca,U+6ecc-6ed0,U+6ed6-6ed9,U+6edb-6edd,U+6edf-6ee0,U+6ee2-6ee3,U+6ee6-6ee7,U+6eea-6ef3,U+6ef5-6f01,U+6f03-6f05,U+6f07-6f0e,U+6f10-6f12,U+6f15-6f1f,U+6f21-6f2a,U+6f2c-6f30,U+6f33-6f46,U+6f48-6f57,U+6f59-6f5b,U+6f5d-6f6c,U+6f6f-6f83,U+6f85-6f87,U+6f89-6f9b,U+6f9d-6fa0,U+6fa2-6fb2,U+6fb4-6fbf,U+6fc1-6fd1,U+6fd3-7008,U+8088,U+9570-9571;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.45.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.45.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.45.otf')  format('opentype');unicode-range:U+60ea,U+6e3c-6e55,U+6e57,U+6e59-6e5a,U+6e5c-6e7d,U+6e80-6e82,U+6e84-6e8f,U+6e91-6e9b,U+6e9d-6ea1,U+6ea3-6ea9,U+6eab-6eae,U+6eb0-6eb5,U+6eb7-6eb8,U+6ebb-6eca,U+6ecc-6ed0,U+6ed6-6ed9,U+6edb-6edd,U+6edf-6ee0,U+6ee2-6ee3,U+6ee6-6ee7,U+6eea-6ef3,U+6ef5-6f01,U+6f03-6f05,U+6f07-6f0e,U+6f10-6f12,U+6f15-6f1f,U+6f21-6f2a,U+6f2c-6f30,U+6f33-6f46,U+6f48-6f57,U+6f59-6f5b,U+6f5d-6f6c,U+6f6f-6f83,U+6f85-6f87,U+6f89-6f9b,U+6f9d-6fa0,U+6fa2-6fb2,U+6fb4-6fbf,U+6fc1-6fd1,U+6fd3-7008,U+8088,U+9570-9571;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.45.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.45.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.45.otf')  format('opentype');unicode-range:U+60ea,U+6e3c-6e55,U+6e57,U+6e59-6e5a,U+6e5c-6e7d,U+6e80-6e82,U+6e84-6e8f,U+6e91-6e9b,U+6e9d-6ea1,U+6ea3-6ea9,U+6eab-6eae,U+6eb0-6eb5,U+6eb7-6eb8,U+6ebb-6eca,U+6ecc-6ed0,U+6ed6-6ed9,U+6edb-6edd,U+6edf-6ee0,U+6ee2-6ee3,U+6ee6-6ee7,U+6eea-6ef3,U+6ef5-6f01,U+6f03-6f05,U+6f07-6f0e,U+6f10-6f12,U+6f15-6f1f,U+6f21-6f2a,U+6f2c-6f30,U+6f33-6f46,U+6f48-6f57,U+6f59-6f5b,U+6f5d-6f6c,U+6f6f-6f83,U+6f85-6f87,U+6f89-6f9b,U+6f9d-6fa0,U+6fa2-6fb2,U+6fb4-6fbf,U+6fc1-6fd1,U+6fd3-7008,U+8088,U+9570-9571;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.46.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.46.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.46.otf')  format('opentype');unicode-range:U+5769,U+71d6-71e4,U+71e6-7205,U+7207-7229,U+722b,U+722d-7230,U+7232-7234,U+723a-723c,U+723e-7246,U+7249-724b,U+724d-7258,U+725a,U+7260,U+7263-7266,U+7268,U+726a-7271,U+7273-7274,U+7276-7278,U+727b-727f,U+7281-72ab,U+72ad-72ae,U+72b0-72b3,U+72b7-72b8,U+72ba-72c1,U+72c3,U+72c5-72cf,U+72d1-72d6,U+72d8-72df,U+72e2-72eb,U+72ef-72f0,U+72f2-72f7,U+72f9-72fb,U+72fd-730d,U+730f-731a,U+731d-7329,U+732c-732d,U+732f-7333,U+7335-7383,U+7385-7386,U+7388,U+738a,U+738c-7394,U+7396-739a,U+739c-73a8,U+73aa,U+73ac-73ae,U+73b1,U+916b,U+20158;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.46.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.46.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.46.otf')  format('opentype');unicode-range:U+5769,U+71d6-71e4,U+71e6-7205,U+7207-7229,U+722b,U+722d-7230,U+7232-7234,U+723a-723c,U+723e-7246,U+7249-724b,U+724d-7258,U+725a,U+7260,U+7263-7266,U+7268,U+726a-7271,U+7273-7274,U+7276-7278,U+727b-727f,U+7281-72ab,U+72ad-72ae,U+72b0-72b3,U+72b7-72b8,U+72ba-72c1,U+72c3,U+72c5-72cf,U+72d1-72d6,U+72d8-72df,U+72e2-72eb,U+72ef-72f0,U+72f2-72f7,U+72f9-72fb,U+72fd-730d,U+730f-731a,U+731d-7329,U+732c-732d,U+732f-7333,U+7335-7383,U+7385-7386,U+7388,U+738a,U+738c-7394,U+7396-739a,U+739c-73a8,U+73aa,U+73ac-73ae,U+73b1,U+916b,U+20158;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.46.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.46.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.46.otf')  format('opentype');unicode-range:U+5769,U+71d6-71e4,U+71e6-7205,U+7207-7229,U+722b,U+722d-7230,U+7232-7234,U+723a-723c,U+723e-7246,U+7249-724b,U+724d-7258,U+725a,U+7260,U+7263-7266,U+7268,U+726a-7271,U+7273-7274,U+7276-7278,U+727b-727f,U+7281-72ab,U+72ad-72ae,U+72b0-72b3,U+72b7-72b8,U+72ba-72c1,U+72c3,U+72c5-72cf,U+72d1-72d6,U+72d8-72df,U+72e2-72eb,U+72ef-72f0,U+72f2-72f7,U+72f9-72fb,U+72fd-730d,U+730f-731a,U+731d-7329,U+732c-732d,U+732f-7333,U+7335-7383,U+7385-7386,U+7388,U+738a,U+738c-7394,U+7396-739a,U+739c-73a8,U+73aa,U+73ac-73ae,U+73b1,U+916b,U+20158;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.47.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.47.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.47.otf')  format('opentype');unicode-range:U+3052,U+653a,U+6654,U+70d9,U+7edb,U+8b47-8b65,U+8b67-8b6b,U+8b6d-8b9f,U+8ba5-8ba7,U+8baa-8bac,U+8bb1,U+8bb4-8bb5,U+8bb7,U+8bb9,U+8bcb-8bcc,U+8bce,U+8bd2-8bd4,U+8bd6,U+8bd8-8bd9,U+8bdc,U+8bdf,U+8be4,U+8be7-8bea,U+8bf0,U+8bf6,U+8bf9,U+8bfc-8bfd,U+8bff-8c00,U+8c02,U+8c04,U+8c06-8c07,U+8c09,U+8c0c,U+8c11-8c12,U+8c14-8c1a,U+8c1d-8c21,U+8c29-8c2b,U+8c2e-8c30,U+8c32-8c33,U+8c35-8c36,U+8c38-8c40,U+8c42-8c45,U+8c47-8c60,U+8c62-8c69,U+8c6c-8c78,U+8c7a-8c8b,U+8c8d-8d1c,U+8d20,U+8d30,U+8d36,U+8d3b,U+8d3d,U+8d40-8d43,U+8d45-8d4a,U+8d51-8d53,U+8d55,U+8d57,U+8d59,U+8d5c-8d5d,U+8d5f,U+9609-960a,U+962a-962d,U+964e-964f,U+97f4,U+c7bd-c7c0,U+c879;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.47.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.47.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.47.otf')  format('opentype');unicode-range:U+3052,U+653a,U+6654,U+70d9,U+7edb,U+8b47-8b65,U+8b67-8b6b,U+8b6d-8b9f,U+8ba5-8ba7,U+8baa-8bac,U+8bb1,U+8bb4-8bb5,U+8bb7,U+8bb9,U+8bcb-8bcc,U+8bce,U+8bd2-8bd4,U+8bd6,U+8bd8-8bd9,U+8bdc,U+8bdf,U+8be4,U+8be7-8bea,U+8bf0,U+8bf6,U+8bf9,U+8bfc-8bfd,U+8bff-8c00,U+8c02,U+8c04,U+8c06-8c07,U+8c09,U+8c0c,U+8c11-8c12,U+8c14-8c1a,U+8c1d-8c21,U+8c29-8c2b,U+8c2e-8c30,U+8c32-8c33,U+8c35-8c36,U+8c38-8c40,U+8c42-8c45,U+8c47-8c60,U+8c62-8c69,U+8c6c-8c78,U+8c7a-8c8b,U+8c8d-8d1c,U+8d20,U+8d30,U+8d36,U+8d3b,U+8d3d,U+8d40-8d43,U+8d45-8d4a,U+8d51-8d53,U+8d55,U+8d57,U+8d59,U+8d5c-8d5d,U+8d5f,U+9609-960a,U+962a-962d,U+964e-964f,U+97f4,U+c7bd-c7c0,U+c879;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.47.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.47.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.47.otf')  format('opentype');unicode-range:U+3052,U+653a,U+6654,U+70d9,U+7edb,U+8b47-8b65,U+8b67-8b6b,U+8b6d-8b9f,U+8ba5-8ba7,U+8baa-8bac,U+8bb1,U+8bb4-8bb5,U+8bb7,U+8bb9,U+8bcb-8bcc,U+8bce,U+8bd2-8bd4,U+8bd6,U+8bd8-8bd9,U+8bdc,U+8bdf,U+8be4,U+8be7-8bea,U+8bf0,U+8bf6,U+8bf9,U+8bfc-8bfd,U+8bff-8c00,U+8c02,U+8c04,U+8c06-8c07,U+8c09,U+8c0c,U+8c11-8c12,U+8c14-8c1a,U+8c1d-8c21,U+8c29-8c2b,U+8c2e-8c30,U+8c32-8c33,U+8c35-8c36,U+8c38-8c40,U+8c42-8c45,U+8c47-8c60,U+8c62-8c69,U+8c6c-8c78,U+8c7a-8c8b,U+8c8d-8d1c,U+8d20,U+8d30,U+8d36,U+8d3b,U+8d3d,U+8d40-8d43,U+8d45-8d4a,U+8d51-8d53,U+8d55,U+8d57,U+8d59,U+8d5c-8d5d,U+8d5f,U+9609-960a,U+962a-962d,U+964e-964f,U+97f4,U+c7bd-c7c0,U+c879;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.48.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.48.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.48.otf')  format('opentype');unicode-range:U+627d,U+75b0-75b1,U+80b3,U+862c-8637,U+8639-864d,U+8652-8653,U+8655-8659,U+865b-865d,U+865f-866a,U+866c-8678,U+867a-867c,U+867f,U+8682-8689,U+868b-8694,U+8696-86a7,U+86a9-86c6,U+86c8-86ca,U+86cc-86da,U+86dc-86dd,U+86df-86ed,U+86ef-86ff,U+8701,U+8703-8711,U+8713-8714,U+8719-871b,U+871d-8720,U+8722-8746,U+8748,U+874a-8752,U+8755-8763,U+8765-8773,U+8775,U+8777-878c,U+878e-87b9,U+87bb-87e1,U+8bc2-8bc3,U+8d4d,U+94bf-94c0,U+9529-952d,U+961a-961b,U+9717;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.48.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.48.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.48.otf')  format('opentype');unicode-range:U+627d,U+75b0-75b1,U+80b3,U+862c-8637,U+8639-864d,U+8652-8653,U+8655-8659,U+865b-865d,U+865f-866a,U+866c-8678,U+867a-867c,U+867f,U+8682-8689,U+868b-8694,U+8696-86a7,U+86a9-86c6,U+86c8-86ca,U+86cc-86da,U+86dc-86dd,U+86df-86ed,U+86ef-86ff,U+8701,U+8703-8711,U+8713-8714,U+8719-871b,U+871d-8720,U+8722-8746,U+8748,U+874a-8752,U+8755-8763,U+8765-8773,U+8775,U+8777-878c,U+878e-87b9,U+87bb-87e1,U+8bc2-8bc3,U+8d4d,U+94bf-94c0,U+9529-952d,U+961a-961b,U+9717;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.48.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.48.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.48.otf')  format('opentype');unicode-range:U+627d,U+75b0-75b1,U+80b3,U+862c-8637,U+8639-864d,U+8652-8653,U+8655-8659,U+865b-865d,U+865f-866a,U+866c-8678,U+867a-867c,U+867f,U+8682-8689,U+868b-8694,U+8696-86a7,U+86a9-86c6,U+86c8-86ca,U+86cc-86da,U+86dc-86dd,U+86df-86ed,U+86ef-86ff,U+8701,U+8703-8711,U+8713-8714,U+8719-871b,U+871d-8720,U+8722-8746,U+8748,U+874a-8752,U+8755-8763,U+8765-8773,U+8775,U+8777-878c,U+878e-87b9,U+87bb-87e1,U+8bc2-8bc3,U+8d4d,U+94bf-94c0,U+9529-952d,U+961a-961b,U+9717;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.49.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.49.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.49.otf')  format('opentype');unicode-range:U+2100,U+6b7a,U+92eb-946a,U+946c-9487,U+959c-959e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.49.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.49.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.49.otf')  format('opentype');unicode-range:U+2100,U+6b7a,U+92eb-946a,U+946c-9487,U+959c-959e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.49.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.49.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.49.otf')  format('opentype');unicode-range:U+2100,U+6b7a,U+92eb-946a,U+946c-9487,U+959c-959e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.50.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.50.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.50.otf')  format('opentype');unicode-range:U+541d,U+94a1,U+94c2,U+94c4,U+94c6-94ca,U+955d-9563,U+9647,U+967b-9685,U+9687-968a,U+968c-968e,U+9691-9693,U+9695-9697,U+969a-969b,U+969d-96a6,U+96a8-96b5,U+96b7-96bd,U+96bf,U+96c2-96c3,U+96c8-96cb,U+96ce,U+96d0-96d4,U+96d6-96df,U+96e2-96e7,U+96e9,U+96eb-96f5,U+96f8-96fd,U+96ff,U+9701-9703,U+9705,U+970a-970c,U+970e-9715,U+9718-971b,U+971d,U+971f-9731,U+9733-9737,U+9739-9751,U+9754-9755,U+9757-9758,U+975a-975d,U+975f,U+9763-9768,U+976a-9773,U+9775-978a,U+978c,U+978e-979d,U+97a0-97ac,U+97ae-97e5,U+97e8,U+97ea-97f2,U+97f7-983b,U+983d-9849;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.50.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.50.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.50.otf')  format('opentype');unicode-range:U+541d,U+94a1,U+94c2,U+94c4,U+94c6-94ca,U+955d-9563,U+9647,U+967b-9685,U+9687-968a,U+968c-968e,U+9691-9693,U+9695-9697,U+969a-969b,U+969d-96a6,U+96a8-96b5,U+96b7-96bd,U+96bf,U+96c2-96c3,U+96c8-96cb,U+96ce,U+96d0-96d4,U+96d6-96df,U+96e2-96e7,U+96e9,U+96eb-96f5,U+96f8-96fd,U+96ff,U+9701-9703,U+9705,U+970a-970c,U+970e-9715,U+9718-971b,U+971d,U+971f-9731,U+9733-9737,U+9739-9751,U+9754-9755,U+9757-9758,U+975a-975d,U+975f,U+9763-9768,U+976a-9773,U+9775-978a,U+978c,U+978e-979d,U+97a0-97ac,U+97ae-97e5,U+97e8,U+97ea-97f2,U+97f7-983b,U+983d-9849;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.50.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.50.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.50.otf')  format('opentype');unicode-range:U+541d,U+94a1,U+94c2,U+94c4,U+94c6-94ca,U+955d-9563,U+9647,U+967b-9685,U+9687-968a,U+968c-968e,U+9691-9693,U+9695-9697,U+969a-969b,U+969d-96a6,U+96a8-96b5,U+96b7-96bd,U+96bf,U+96c2-96c3,U+96c8-96cb,U+96ce,U+96d0-96d4,U+96d6-96df,U+96e2-96e7,U+96e9,U+96eb-96f5,U+96f8-96fd,U+96ff,U+9701-9703,U+9705,U+970a-970c,U+970e-9715,U+9718-971b,U+971d,U+971f-9731,U+9733-9737,U+9739-9751,U+9754-9755,U+9757-9758,U+975a-975d,U+975f,U+9763-9768,U+976a-9773,U+9775-978a,U+978c,U+978e-979d,U+97a0-97ac,U+97ae-97e5,U+97e8,U+97ea-97f2,U+97f7-983b,U+983d-9849;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.51.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.51.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.51.otf')  format('opentype');unicode-range:U+57a4-57a6,U+9bd2-9c7b,U+9c7d-9c80,U+9c82-9c8c,U+9c8e-9c8f,U+9c91-9c9b,U+9c9d-9ca2,U+9ca4-9cb4,U+9cb6-9cdd,U+9cdf-9d72,U+c7a8-c7aa;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.51.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.51.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.51.otf')  format('opentype');unicode-range:U+57a4-57a6,U+9bd2-9c7b,U+9c7d-9c80,U+9c82-9c8c,U+9c8e-9c8f,U+9c91-9c9b,U+9c9d-9ca2,U+9ca4-9cb4,U+9cb6-9cdd,U+9cdf-9d72,U+c7a8-c7aa;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.51.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.51.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.51.otf')  format('opentype');unicode-range:U+57a4-57a6,U+9bd2-9c7b,U+9c7d-9c80,U+9c82-9c8c,U+9c8e-9c8f,U+9c91-9c9b,U+9c9d-9ca2,U+9ca4-9cb4,U+9cb6-9cdd,U+9cdf-9d72,U+c7a8-c7aa;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.52.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.52.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.52.otf')  format('opentype');unicode-range:U+c75c-c75d,U+d598-d5a4,U+d5a6-d603,U+d605-d653,U+d655-d68b,U+d68d-d6c3,U+d6c5-d739;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.52.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.52.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.52.otf')  format('opentype');unicode-range:U+c75c-c75d,U+d598-d5a4,U+d5a6-d603,U+d605-d653,U+d655-d68b,U+d68d-d6c3,U+d6c5-d739;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.52.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.52.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.52.otf')  format('opentype');unicode-range:U+c75c-c75d,U+d598-d5a4,U+d5a6-d603,U+d605-d653,U+d655-d68b,U+d68d-d6c3,U+d6c5-d739;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.53.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.53.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.53.otf')  format('opentype');unicode-range:U+c75e-c760,U+d3f5-d557,U+d55a-d55b,U+d55d-d55f,U+d561-d567,U+d56a-d573,U+d575-d597;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.53.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.53.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.53.otf')  format('opentype');unicode-range:U+c75e-c760,U+d3f5-d557,U+d55a-d55b,U+d55d-d55f,U+d561-d567,U+d56a-d573,U+d575-d597;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.53.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.53.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.53.otf')  format('opentype');unicode-range:U+c75e-c760,U+d3f5-d557,U+d55a-d55b,U+d55d-d55f,U+d561-d567,U+d56a-d573,U+d575-d597;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.54.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.54.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.54.otf')  format('opentype');unicode-range:U+53e7,U+c3b6-c543,U+c545-c547,U+c54b,U+c54d-c556,U+c763,U+c846;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.54.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.54.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.54.otf')  format('opentype');unicode-range:U+53e7,U+c3b6-c543,U+c545-c547,U+c54b,U+c54d-c556,U+c763,U+c846;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.54.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.54.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.54.otf')  format('opentype');unicode-range:U+53e7,U+c3b6-c543,U+c545-c547,U+c54b,U+c54d-c556,U+c763,U+c846;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.55.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.55.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.55.otf')  format('opentype');unicode-range:U+306c,U+4589-4729;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.55.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.55.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.55.otf')  format('opentype');unicode-range:U+306c,U+4589-4729;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.55.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.55.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.55.otf')  format('opentype');unicode-range:U+306c,U+4589-4729;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.56.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.56.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.56.otf')  format('opentype');unicode-range:U+90de,U+ac00-ac01,U+ac04,U+ac10,U+ac19,U+ac70,U+ac78,U+ac83,U+ac8c,U+aca0,U+acb0,U+acbd,U+acc4,U+ace0,U+acfc,U+ad50,U+ad6c-ad6d,U+adf8,U+ae30,U+ae38,U+ae40,U+ae4c,U+aed8,U+b098,U+b09c,U+b0a0,U+b0a8,U+b0b4,U+b108,U+b110,U+b124,U+b155,U+b208,U+b294,U+b2c8,U+b2e4,U+b2e8,U+b2ec,U+b2f9,U+b300,U+b370,U+b3c4,U+b3d9,U+b418,U+b450,U+b4dc,U+b4e0,U+b4e4,U+b530,U+b77c,U+b78c,U+b791,U+b798,U+b824,U+b85c,U+b974,U+b978,U+b97c,U+b984,U+b9ac,U+b9b0,U+b9c8,U+b9cc,U+b9d0,U+ba70,U+ba74,U+ba85,U+baa8-baa9,U+babb,U+bb34,U+bb38,U+bb3c,U+bbf8,U+bbfc,U+bc00,U+bc14-bc15,U+bc1b-bc1c,U+bc31,U+bc84,U+bcf4,U+bd80,U+bd84,U+be44,U+be5b,U+c0ac,U+c0b4,U+c0c1,U+c0dd,U+c11c,U+c120,U+c131,U+c138,U+c18c-c18d,U+c190,U+c218,U+c2a4,U+c2b5,U+c2dc-c2dd,U+c2e0,U+c2ec,U+c544,U+c548-c54a,U+c54c,U+c57c,U+c5b4,U+c5c6-c5c8,U+c5d0,U+c5ec,U+c601,U+c624,U+c640,U+c694,U+c6b0,U+c6b4,U+c6b8,U+c6c3,U+c6d0,U+c704-c725,U+c72c-c75a,U+c764-c7a5,U+c7db-c815,U+c81c-c837,U+c847-c870,U+c87a-c88b,U+c8fc,U+c900,U+c911,U+c9c0-c9c1,U+c9c4,U+c9d1,U+cc28,U+cc44,U+cc9c,U+ccb4,U+cd94,U+ce58,U+ce74,U+d06c,U+d0c0,U+d0dc,U+d2b8,U+d30c,U+d3ec,U+d558-d559,U+d55c,U+d560,U+d568-d569,U+d574,U+d5a5,U+d604,U+d654,U+d68c,U+d6c4,U+d788;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.56.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.56.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.56.otf')  format('opentype');unicode-range:U+90de,U+ac00-ac01,U+ac04,U+ac10,U+ac19,U+ac70,U+ac78,U+ac83,U+ac8c,U+aca0,U+acb0,U+acbd,U+acc4,U+ace0,U+acfc,U+ad50,U+ad6c-ad6d,U+adf8,U+ae30,U+ae38,U+ae40,U+ae4c,U+aed8,U+b098,U+b09c,U+b0a0,U+b0a8,U+b0b4,U+b108,U+b110,U+b124,U+b155,U+b208,U+b294,U+b2c8,U+b2e4,U+b2e8,U+b2ec,U+b2f9,U+b300,U+b370,U+b3c4,U+b3d9,U+b418,U+b450,U+b4dc,U+b4e0,U+b4e4,U+b530,U+b77c,U+b78c,U+b791,U+b798,U+b824,U+b85c,U+b974,U+b978,U+b97c,U+b984,U+b9ac,U+b9b0,U+b9c8,U+b9cc,U+b9d0,U+ba70,U+ba74,U+ba85,U+baa8-baa9,U+babb,U+bb34,U+bb38,U+bb3c,U+bbf8,U+bbfc,U+bc00,U+bc14-bc15,U+bc1b-bc1c,U+bc31,U+bc84,U+bcf4,U+bd80,U+bd84,U+be44,U+be5b,U+c0ac,U+c0b4,U+c0c1,U+c0dd,U+c11c,U+c120,U+c131,U+c138,U+c18c-c18d,U+c190,U+c218,U+c2a4,U+c2b5,U+c2dc-c2dd,U+c2e0,U+c2ec,U+c544,U+c548-c54a,U+c54c,U+c57c,U+c5b4,U+c5c6-c5c8,U+c5d0,U+c5ec,U+c601,U+c624,U+c640,U+c694,U+c6b0,U+c6b4,U+c6b8,U+c6c3,U+c6d0,U+c704-c725,U+c72c-c75a,U+c764-c7a5,U+c7db-c815,U+c81c-c837,U+c847-c870,U+c87a-c88b,U+c8fc,U+c900,U+c911,U+c9c0-c9c1,U+c9c4,U+c9d1,U+cc28,U+cc44,U+cc9c,U+ccb4,U+cd94,U+ce58,U+ce74,U+d06c,U+d0c0,U+d0dc,U+d2b8,U+d30c,U+d3ec,U+d558-d559,U+d55c,U+d560,U+d568-d569,U+d574,U+d5a5,U+d604,U+d654,U+d68c,U+d6c4,U+d788;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.56.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.56.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.56.otf')  format('opentype');unicode-range:U+90de,U+ac00-ac01,U+ac04,U+ac10,U+ac19,U+ac70,U+ac78,U+ac83,U+ac8c,U+aca0,U+acb0,U+acbd,U+acc4,U+ace0,U+acfc,U+ad50,U+ad6c-ad6d,U+adf8,U+ae30,U+ae38,U+ae40,U+ae4c,U+aed8,U+b098,U+b09c,U+b0a0,U+b0a8,U+b0b4,U+b108,U+b110,U+b124,U+b155,U+b208,U+b294,U+b2c8,U+b2e4,U+b2e8,U+b2ec,U+b2f9,U+b300,U+b370,U+b3c4,U+b3d9,U+b418,U+b450,U+b4dc,U+b4e0,U+b4e4,U+b530,U+b77c,U+b78c,U+b791,U+b798,U+b824,U+b85c,U+b974,U+b978,U+b97c,U+b984,U+b9ac,U+b9b0,U+b9c8,U+b9cc,U+b9d0,U+ba70,U+ba74,U+ba85,U+baa8-baa9,U+babb,U+bb34,U+bb38,U+bb3c,U+bbf8,U+bbfc,U+bc00,U+bc14-bc15,U+bc1b-bc1c,U+bc31,U+bc84,U+bcf4,U+bd80,U+bd84,U+be44,U+be5b,U+c0ac,U+c0b4,U+c0c1,U+c0dd,U+c11c,U+c120,U+c131,U+c138,U+c18c-c18d,U+c190,U+c218,U+c2a4,U+c2b5,U+c2dc-c2dd,U+c2e0,U+c2ec,U+c544,U+c548-c54a,U+c54c,U+c57c,U+c5b4,U+c5c6-c5c8,U+c5d0,U+c5ec,U+c601,U+c624,U+c640,U+c694,U+c6b0,U+c6b4,U+c6b8,U+c6c3,U+c6d0,U+c704-c725,U+c72c-c75a,U+c764-c7a5,U+c7db-c815,U+c81c-c837,U+c847-c870,U+c87a-c88b,U+c8fc,U+c900,U+c911,U+c9c0-c9c1,U+c9c4,U+c9d1,U+cc28,U+cc44,U+cc9c,U+ccb4,U+cd94,U+ce58,U+ce74,U+d06c,U+d0c0,U+d0dc,U+d2b8,U+d30c,U+d3ec,U+d558-d559,U+d55c,U+d560,U+d568-d569,U+d574,U+d5a5,U+d604,U+d654,U+d68c,U+d6c4,U+d788;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.57.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.57.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.57.otf')  format('opentype');unicode-range:U+6b77-6b78,U+79d5,U+899f-89c0,U+89ca-89d1,U+89d3-89db,U+89dd-89e2,U+89e4-89e5,U+89e7-89ff,U+8a01-8a78,U+8a7a-8a88,U+8a8a-8a92,U+8a94-8aab,U+8aad-8b46,U+9620-9625;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.57.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.57.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.57.otf')  format('opentype');unicode-range:U+6b77-6b78,U+79d5,U+899f-89c0,U+89ca-89d1,U+89d3-89db,U+89dd-89e2,U+89e4-89e5,U+89e7-89ff,U+8a01-8a78,U+8a7a-8a88,U+8a8a-8a92,U+8a94-8aab,U+8aad-8b46,U+9620-9625;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.57.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.57.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.57.otf')  format('opentype');unicode-range:U+6b77-6b78,U+79d5,U+899f-89c0,U+89ca-89d1,U+89d3-89db,U+89dd-89e2,U+89e4-89e5,U+89e7-89ff,U+8a01-8a78,U+8a7a-8a88,U+8a8a-8a92,U+8a94-8aab,U+8aad-8b46,U+9620-9625;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.58.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.58.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.58.otf')  format('opentype');unicode-range:U+7ce0-7cee,U+7cf0-7cfa,U+7cfc-7d09,U+7d0b-7d1f,U+7d21,U+7d23-7d26,U+7d28-7d2a,U+7d2c-7d2e,U+7d30-7d4f,U+7d51-7d6d,U+7d6f-7d92,U+7d94-7e40,U+7e42-7e4a,U+7e4c-7e84,U+7ea1,U+9490-9492,U+94f7,U+9513-9514;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.58.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.58.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.58.otf')  format('opentype');unicode-range:U+7ce0-7cee,U+7cf0-7cfa,U+7cfc-7d09,U+7d0b-7d1f,U+7d21,U+7d23-7d26,U+7d28-7d2a,U+7d2c-7d2e,U+7d30-7d4f,U+7d51-7d6d,U+7d6f-7d92,U+7d94-7e40,U+7e42-7e4a,U+7e4c-7e84,U+7ea1,U+9490-9492,U+94f7,U+9513-9514;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.58.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.58.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.58.otf')  format('opentype');unicode-range:U+7ce0-7cee,U+7cf0-7cfa,U+7cfc-7d09,U+7d0b-7d1f,U+7d21,U+7d23-7d26,U+7d28-7d2a,U+7d2c-7d2e,U+7d30-7d4f,U+7d51-7d6d,U+7d6f-7d92,U+7d94-7e40,U+7e42-7e4a,U+7e4c-7e84,U+7ea1,U+9490-9492,U+94f7,U+9513-9514;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.59.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.59.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.59.otf')  format('opentype');unicode-range:U+7980,U+bed3-c072;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.59.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.59.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.59.otf')  format('opentype');unicode-range:U+7980,U+bed3-c072;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.59.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.59.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.59.otf')  format('opentype');unicode-range:U+7980,U+bed3-c072;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.60.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.60.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.60.otf')  format('opentype');unicode-range:U+51a3,U+51ee,U+604c,U+75bd,U+8002,U+20e16,U+22c9b-23766,U+237bc-246a5,U+247e0-24994,U+249a7,U+25d43,U+26469,U+26df0,U+277cc-27858,U+2b372,U+2cb76;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.60.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.60.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.60.otf')  format('opentype');unicode-range:U+51a3,U+51ee,U+604c,U+75bd,U+8002,U+20e16,U+22c9b-23766,U+237bc-246a5,U+247e0-24994,U+249a7,U+25d43,U+26469,U+26df0,U+277cc-27858,U+2b372,U+2cb76;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.60.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.60.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.60.otf')  format('opentype');unicode-range:U+51a3,U+51ee,U+604c,U+75bd,U+8002,U+20e16,U+22c9b-23766,U+237bc-246a5,U+247e0-24994,U+249a7,U+25d43,U+26469,U+26df0,U+277cc-27858,U+2b372,U+2cb76;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.61.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.61.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.61.otf')  format('opentype');unicode-range:U+0251,U+52d5,U+6427,U+65d2,U+6d0e,U+86de,U+8753-8754,U+87e2-87f8,U+87fa-881a,U+881e-8821,U+8823-883f,U+8841-884b,U+884e-8853,U+8855-8856,U+8858-8860,U+8871-8876,U+8878-8880,U+8882-888a,U+888c,U+888e-8895,U+8897-889b,U+889d-88aa,U+88ae-88c0,U+88c3-88c4,U+88c6-88d3,U+88d6-88d8,U+88dc-88e3,U+88e5-88f2,U+88f5-88f7,U+88fa-88fc,U+88fe-890f,U+8911,U+8913-8929,U+892b-8943,U+8945-895e,U+8960-897e,U+8980,U+8982-8985,U+8987-898a,U+898c-899e,U+8d61,U+8f98,U+958c-958d,U+95b8-95ba,U+95cd-95d4,U+9649-964a,U+965c-965e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.61.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.61.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.61.otf')  format('opentype');unicode-range:U+0251,U+52d5,U+6427,U+65d2,U+6d0e,U+86de,U+8753-8754,U+87e2-87f8,U+87fa-881a,U+881e-8821,U+8823-883f,U+8841-884b,U+884e-8853,U+8855-8856,U+8858-8860,U+8871-8876,U+8878-8880,U+8882-888a,U+888c,U+888e-8895,U+8897-889b,U+889d-88aa,U+88ae-88c0,U+88c3-88c4,U+88c6-88d3,U+88d6-88d8,U+88dc-88e3,U+88e5-88f2,U+88f5-88f7,U+88fa-88fc,U+88fe-890f,U+8911,U+8913-8929,U+892b-8943,U+8945-895e,U+8960-897e,U+8980,U+8982-8985,U+8987-898a,U+898c-899e,U+8d61,U+8f98,U+958c-958d,U+95b8-95ba,U+95cd-95d4,U+9649-964a,U+965c-965e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.61.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.61.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.61.otf')  format('opentype');unicode-range:U+0251,U+52d5,U+6427,U+65d2,U+6d0e,U+86de,U+8753-8754,U+87e2-87f8,U+87fa-881a,U+881e-8821,U+8823-883f,U+8841-884b,U+884e-8853,U+8855-8856,U+8858-8860,U+8871-8876,U+8878-8880,U+8882-888a,U+888c,U+888e-8895,U+8897-889b,U+889d-88aa,U+88ae-88c0,U+88c3-88c4,U+88c6-88d3,U+88d6-88d8,U+88dc-88e3,U+88e5-88f2,U+88f5-88f7,U+88fa-88fc,U+88fe-890f,U+8911,U+8913-8929,U+892b-8943,U+8945-895e,U+8960-897e,U+8980,U+8982-8985,U+8987-898a,U+898c-899e,U+8d61,U+8f98,U+958c-958d,U+95b8-95ba,U+95cd-95d4,U+9649-964a,U+965c-965e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.62.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.62.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.62.otf')  format('opentype');unicode-range:U+c073-c0ab,U+c0ad-c0b3,U+c0b5-c0c0,U+c0c2-c0dc,U+c0de-c11b,U+c11d-c11f,U+c121-c130,U+c132-c137,U+c139-c18b,U+c18e-c18f,U+c191-c214,U+c816-c81b,U+c842,U+c871-c872;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.62.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.62.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.62.otf')  format('opentype');unicode-range:U+c073-c0ab,U+c0ad-c0b3,U+c0b5-c0c0,U+c0c2-c0dc,U+c0de-c11b,U+c11d-c11f,U+c121-c130,U+c132-c137,U+c139-c18b,U+c18e-c18f,U+c191-c214,U+c816-c81b,U+c842,U+c871-c872;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.62.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.62.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.62.otf')  format('opentype');unicode-range:U+c073-c0ab,U+c0ad-c0b3,U+c0b5-c0c0,U+c0c2-c0dc,U+c0de-c11b,U+c11d-c11f,U+c121-c130,U+c132-c137,U+c139-c18b,U+c18e-c18f,U+c191-c214,U+c816-c81b,U+c842,U+c871-c872;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.63.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.63.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.63.otf')  format('opentype');unicode-range:U+8862,U+8864,U+bb8d-bbf7,U+bbf9-bbfb,U+bbfd-bbff,U+bc01-bc13,U+bc16-bc1a,U+bc1d-bc30,U+bc32-bc83,U+bc85-bcf3,U+bcf5-bd33;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.63.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.63.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.63.otf')  format('opentype');unicode-range:U+8862,U+8864,U+bb8d-bbf7,U+bbf9-bbfb,U+bbfd-bbff,U+bc01-bc13,U+bc16-bc1a,U+bc1d-bc30,U+bc32-bc83,U+bc85-bcf3,U+bcf5-bd33;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.63.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.63.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.63.otf')  format('opentype');unicode-range:U+8862,U+8864,U+bb8d-bbf7,U+bbf9-bbfb,U+bbfd-bbff,U+bc01-bc13,U+bc16-bc1a,U+bc1d-bc30,U+bc32-bc83,U+bc85-bcf3,U+bcf5-bd33;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.64.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.64.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.64.otf')  format('opentype');unicode-range:U+846d-8470,U+8472-8474,U+8476-8481,U+8483-848a,U+848c-8498,U+849a-849b,U+849d-84b1,U+84b3-84b7,U+84b9-84c3,U+84c5-84c8,U+84ca-84dc,U+84de-84eb,U+84ed-8512,U+8514-8519,U+851b-8520,U+8522-852b,U+852d-853c,U+853e-8548,U+854b-8573,U+8575-857d,U+857f-8583,U+8585-8586,U+8588-859a,U+859c-85a9,U+85ab-85ae,U+85b0-85c8,U+85ca-85ce,U+85d0-85e3,U+85e5-85e8,U+85ea-85fa,U+85fc-8610,U+8612-862b,U+8bd0;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.64.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.64.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.64.otf')  format('opentype');unicode-range:U+846d-8470,U+8472-8474,U+8476-8481,U+8483-848a,U+848c-8498,U+849a-849b,U+849d-84b1,U+84b3-84b7,U+84b9-84c3,U+84c5-84c8,U+84ca-84dc,U+84de-84eb,U+84ed-8512,U+8514-8519,U+851b-8520,U+8522-852b,U+852d-853c,U+853e-8548,U+854b-8573,U+8575-857d,U+857f-8583,U+8585-8586,U+8588-859a,U+859c-85a9,U+85ab-85ae,U+85b0-85c8,U+85ca-85ce,U+85d0-85e3,U+85e5-85e8,U+85ea-85fa,U+85fc-8610,U+8612-862b,U+8bd0;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.64.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.64.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.64.otf')  format('opentype');unicode-range:U+846d-8470,U+8472-8474,U+8476-8481,U+8483-848a,U+848c-8498,U+849a-849b,U+849d-84b1,U+84b3-84b7,U+84b9-84c3,U+84c5-84c8,U+84ca-84dc,U+84de-84eb,U+84ed-8512,U+8514-8519,U+851b-8520,U+8522-852b,U+852d-853c,U+853e-8548,U+854b-8573,U+8575-857d,U+857f-8583,U+8585-8586,U+8588-859a,U+859c-85a9,U+85ab-85ae,U+85b0-85c8,U+85ca-85ce,U+85d0-85e3,U+85e5-85e8,U+85ea-85fa,U+85fc-8610,U+8612-862b,U+8bd0;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.65.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.65.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.65.otf')  format('opentype');unicode-range:U+3a22-3bc1;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.65.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.65.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.65.otf')  format('opentype');unicode-range:U+3a22-3bc1;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.65.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.65.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.65.otf')  format('opentype');unicode-range:U+3a22-3bc1;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.66.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.66.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.66.otf')  format('opentype');unicode-range:U+011a-011b,U+ae8c-aed7,U+aed9-b029,U+2b404;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.66.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.66.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.66.otf')  format('opentype');unicode-range:U+011a-011b,U+ae8c-aed7,U+aed9-b029,U+2b404;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.66.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.66.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.66.otf')  format('opentype');unicode-range:U+011a-011b,U+ae8c-aed7,U+aed9-b029,U+2b404;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.67.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.67.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.67.otf')  format('opentype');unicode-range:U+2032-2033,U+2312,U+57cc-57cd,U+7575-7577,U+7b7f,U+f9eb-fe30,U+fe32-fe35,U+fe37-fe6b,U+ff02-ff04,U+ff06-ff07,U+ff0a,U+ff10,U+ff12-ff13,U+ff15-ff19,U+ff1c-ff1d,U+ff20-ff2c,U+ff2e-ff3d,U+ff3f-ff5d,U+ff5f-ff61,U+ff64-ffe4,U+ffe6-ffee;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.67.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.67.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.67.otf')  format('opentype');unicode-range:U+2032-2033,U+2312,U+57cc-57cd,U+7575-7577,U+7b7f,U+f9eb-fe30,U+fe32-fe35,U+fe37-fe6b,U+ff02-ff04,U+ff06-ff07,U+ff0a,U+ff10,U+ff12-ff13,U+ff15-ff19,U+ff1c-ff1d,U+ff20-ff2c,U+ff2e-ff3d,U+ff3f-ff5d,U+ff5f-ff61,U+ff64-ffe4,U+ffe6-ffee;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.67.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.67.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.67.otf')  format('opentype');unicode-range:U+2032-2033,U+2312,U+57cc-57cd,U+7575-7577,U+7b7f,U+f9eb-fe30,U+fe32-fe35,U+fe37-fe6b,U+ff02-ff04,U+ff06-ff07,U+ff0a,U+ff10,U+ff12-ff13,U+ff15-ff19,U+ff1c-ff1d,U+ff20-ff2c,U+ff2e-ff3d,U+ff3f-ff5d,U+ff5f-ff61,U+ff64-ffe4,U+ffe6-ffee;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.68.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.68.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.68.otf')  format('opentype');unicode-range:U+5b6d,U+67ce,U+8d65,U+8d67-8d6a,U+8d6c-8d6f,U+8d71-8d73,U+8d78-8d80,U+8d82-8d84,U+8d86-8d89,U+8d8c-8d9e,U+8da0-8da2,U+8da4-8db2,U+8db5-8dc2,U+8dc4-8dca,U+8dcd-8dd0,U+8dd2-8ddc,U+8de0-8de7,U+8de9,U+8ded-8dee,U+8df0-8df2,U+8df7-8dfa,U+8dfc-8e0e,U+8e10-8e19,U+8e1f-8e21,U+8e23-8e28,U+8e2b-8e43,U+8e45-8e47,U+8e49-8e65,U+8e67-8e71,U+8e73-8e80,U+8e82-8eaa,U+8ead-8eae,U+8eb0-8eb1,U+8eb3-8eb9,U+8ebb-8f23,U+94b6-94b8,U+94d4-94d5,U+9536-953a,U+9559-955b,U+958e-9591,U+95e9;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.68.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.68.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.68.otf')  format('opentype');unicode-range:U+5b6d,U+67ce,U+8d65,U+8d67-8d6a,U+8d6c-8d6f,U+8d71-8d73,U+8d78-8d80,U+8d82-8d84,U+8d86-8d89,U+8d8c-8d9e,U+8da0-8da2,U+8da4-8db2,U+8db5-8dc2,U+8dc4-8dca,U+8dcd-8dd0,U+8dd2-8ddc,U+8de0-8de7,U+8de9,U+8ded-8dee,U+8df0-8df2,U+8df7-8dfa,U+8dfc-8e0e,U+8e10-8e19,U+8e1f-8e21,U+8e23-8e28,U+8e2b-8e43,U+8e45-8e47,U+8e49-8e65,U+8e67-8e71,U+8e73-8e80,U+8e82-8eaa,U+8ead-8eae,U+8eb0-8eb1,U+8eb3-8eb9,U+8ebb-8f23,U+94b6-94b8,U+94d4-94d5,U+9536-953a,U+9559-955b,U+958e-9591,U+95e9;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.68.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.68.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.68.otf')  format('opentype');unicode-range:U+5b6d,U+67ce,U+8d65,U+8d67-8d6a,U+8d6c-8d6f,U+8d71-8d73,U+8d78-8d80,U+8d82-8d84,U+8d86-8d89,U+8d8c-8d9e,U+8da0-8da2,U+8da4-8db2,U+8db5-8dc2,U+8dc4-8dca,U+8dcd-8dd0,U+8dd2-8ddc,U+8de0-8de7,U+8de9,U+8ded-8dee,U+8df0-8df2,U+8df7-8dfa,U+8dfc-8e0e,U+8e10-8e19,U+8e1f-8e21,U+8e23-8e28,U+8e2b-8e43,U+8e45-8e47,U+8e49-8e65,U+8e67-8e71,U+8e73-8e80,U+8e82-8eaa,U+8ead-8eae,U+8eb0-8eb1,U+8eb3-8eb9,U+8ebb-8f23,U+94b6-94b8,U+94d4-94d5,U+9536-953a,U+9559-955b,U+958e-9591,U+95e9;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.69.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.69.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.69.otf')  format('opentype');unicode-range:U+75c6,U+1f100-20118,U+20164-20acd,U+20b8f-20e11,U+20f5f-20f8d,U+21c70,U+22c55,U+25f23,U+2f9f4;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.69.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.69.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.69.otf')  format('opentype');unicode-range:U+75c6,U+1f100-20118,U+20164-20acd,U+20b8f-20e11,U+20f5f-20f8d,U+21c70,U+22c55,U+25f23,U+2f9f4;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.69.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.69.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.69.otf')  format('opentype');unicode-range:U+75c6,U+1f100-20118,U+20164-20acd,U+20b8f-20e11,U+20f5f-20f8d,U+21c70,U+22c55,U+25f23,U+2f9f4;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.70.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.70.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.70.otf')  format('opentype');unicode-range:U+d256-d2b7,U+d2b9-d30b,U+d30d-d3eb,U+d3ed-d3f4,U+24706-2472f;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.70.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.70.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.70.otf')  format('opentype');unicode-range:U+d256-d2b7,U+d2b9-d30b,U+d30d-d3eb,U+d3ed-d3f4,U+24706-2472f;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.70.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.70.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.70.otf')  format('opentype');unicode-range:U+d256-d2b7,U+d2b9-d30b,U+d30d-d3eb,U+d3ed-d3f4,U+24706-2472f;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.71.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.71.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.71.otf')  format('opentype');unicode-range:U+224c,U+2606,U+306b,U+30c8-30cb,U+4e93,U+5e5f-5e61,U+5e63-5e71,U+5e75,U+5e77,U+5e79-5e7a,U+5e7e,U+5e80-5e83,U+5e88-5e89,U+5e8b-5e8e,U+5e91-5e92,U+5e96,U+5e98,U+5e9b,U+5e9d,U+5ea0-5ea5,U+5ea8-5eac,U+5eae-5eb4,U+5eb9-5ec8,U+5ecb-5ed2,U+5ed4-5ed5,U+5ed7-5ef5,U+5ef8-5ef9,U+5efb-5eff,U+5f01,U+5f05-5f07,U+5f0b-5f0e,U+5f10-5f12,U+5f14,U+5f16,U+5f19-5f1a,U+5f1c-5f1e,U+5f21-5f24,U+5f28-5f2e,U+5f30,U+5f32-5f38,U+5f3b-5f51,U+5f54,U+5f56-5f61,U+5f63-5f65,U+5f67-5f68,U+5f6b,U+5f6e-5f6f,U+5f72,U+5f74-5f78,U+5f7a,U+5f7d-5f7f,U+5f82-5f83,U+5f86-5f87,U+5f8d-5f8f,U+5f91,U+5f93-5f96,U+5f9a-5f9d,U+5f9f-5fa0,U+5fa2-5fa8,U+5fab-5fad,U+5faf-5fb6,U+5fb8-5fbb,U+5fbe-5fc2,U+5fc4,U+5fc7-5fcb,U+5fce,U+5fd0-5fd6,U+5fda-5fdf,U+5fe1-5fe6,U+5fe8-5fea,U+5fec-5ff4,U+5ff6-5ffc,U+5ffe-5fff,U+6002-600d,U+600f-6011,U+6013-6014,U+6017-601b,U+601e-601f,U+6022-6024,U+6029,U+602b-603a,U+603c-6041,U+6043-604a,U+604e-604f,U+6051,U+6053-6054,U+6056-6061,U+6063,U+6065-6067,U+606a-606b,U+606e,U+6071-6072,U+711b-711c,U+7f07-7f09,U+8bbb,U+8bc7,U+94cf-94d3,U+95f0-95f1,U+9617-9618,U+965f-9660,U+26e40;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.71.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.71.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.71.otf')  format('opentype');unicode-range:U+224c,U+2606,U+306b,U+30c8-30cb,U+4e93,U+5e5f-5e61,U+5e63-5e71,U+5e75,U+5e77,U+5e79-5e7a,U+5e7e,U+5e80-5e83,U+5e88-5e89,U+5e8b-5e8e,U+5e91-5e92,U+5e96,U+5e98,U+5e9b,U+5e9d,U+5ea0-5ea5,U+5ea8-5eac,U+5eae-5eb4,U+5eb9-5ec8,U+5ecb-5ed2,U+5ed4-5ed5,U+5ed7-5ef5,U+5ef8-5ef9,U+5efb-5eff,U+5f01,U+5f05-5f07,U+5f0b-5f0e,U+5f10-5f12,U+5f14,U+5f16,U+5f19-5f1a,U+5f1c-5f1e,U+5f21-5f24,U+5f28-5f2e,U+5f30,U+5f32-5f38,U+5f3b-5f51,U+5f54,U+5f56-5f61,U+5f63-5f65,U+5f67-5f68,U+5f6b,U+5f6e-5f6f,U+5f72,U+5f74-5f78,U+5f7a,U+5f7d-5f7f,U+5f82-5f83,U+5f86-5f87,U+5f8d-5f8f,U+5f91,U+5f93-5f96,U+5f9a-5f9d,U+5f9f-5fa0,U+5fa2-5fa8,U+5fab-5fad,U+5faf-5fb6,U+5fb8-5fbb,U+5fbe-5fc2,U+5fc4,U+5fc7-5fcb,U+5fce,U+5fd0-5fd6,U+5fda-5fdf,U+5fe1-5fe6,U+5fe8-5fea,U+5fec-5ff4,U+5ff6-5ffc,U+5ffe-5fff,U+6002-600d,U+600f-6011,U+6013-6014,U+6017-601b,U+601e-601f,U+6022-6024,U+6029,U+602b-603a,U+603c-6041,U+6043-604a,U+604e-604f,U+6051,U+6053-6054,U+6056-6061,U+6063,U+6065-6067,U+606a-606b,U+606e,U+6071-6072,U+711b-711c,U+7f07-7f09,U+8bbb,U+8bc7,U+94cf-94d3,U+95f0-95f1,U+9617-9618,U+965f-9660,U+26e40;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.71.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.71.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.71.otf')  format('opentype');unicode-range:U+224c,U+2606,U+306b,U+30c8-30cb,U+4e93,U+5e5f-5e61,U+5e63-5e71,U+5e75,U+5e77,U+5e79-5e7a,U+5e7e,U+5e80-5e83,U+5e88-5e89,U+5e8b-5e8e,U+5e91-5e92,U+5e96,U+5e98,U+5e9b,U+5e9d,U+5ea0-5ea5,U+5ea8-5eac,U+5eae-5eb4,U+5eb9-5ec8,U+5ecb-5ed2,U+5ed4-5ed5,U+5ed7-5ef5,U+5ef8-5ef9,U+5efb-5eff,U+5f01,U+5f05-5f07,U+5f0b-5f0e,U+5f10-5f12,U+5f14,U+5f16,U+5f19-5f1a,U+5f1c-5f1e,U+5f21-5f24,U+5f28-5f2e,U+5f30,U+5f32-5f38,U+5f3b-5f51,U+5f54,U+5f56-5f61,U+5f63-5f65,U+5f67-5f68,U+5f6b,U+5f6e-5f6f,U+5f72,U+5f74-5f78,U+5f7a,U+5f7d-5f7f,U+5f82-5f83,U+5f86-5f87,U+5f8d-5f8f,U+5f91,U+5f93-5f96,U+5f9a-5f9d,U+5f9f-5fa0,U+5fa2-5fa8,U+5fab-5fad,U+5faf-5fb6,U+5fb8-5fbb,U+5fbe-5fc2,U+5fc4,U+5fc7-5fcb,U+5fce,U+5fd0-5fd6,U+5fda-5fdf,U+5fe1-5fe6,U+5fe8-5fea,U+5fec-5ff4,U+5ff6-5ffc,U+5ffe-5fff,U+6002-600d,U+600f-6011,U+6013-6014,U+6017-601b,U+601e-601f,U+6022-6024,U+6029,U+602b-603a,U+603c-6041,U+6043-604a,U+604e-604f,U+6051,U+6053-6054,U+6056-6061,U+6063,U+6065-6067,U+606a-606b,U+606e,U+6071-6072,U+711b-711c,U+7f07-7f09,U+8bbb,U+8bc7,U+94cf-94d3,U+95f0-95f1,U+9617-9618,U+965f-9660,U+26e40;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.72.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.72.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.72.otf')  format('opentype');unicode-range:U+95b6-95b7,U+c83b,U+cbd8-cc27,U+cc29-cc43,U+cc45-cc9b,U+cc9d-ccb3,U+ccb5-cd78;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.72.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.72.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.72.otf')  format('opentype');unicode-range:U+95b6-95b7,U+c83b,U+cbd8-cc27,U+cc29-cc43,U+cc45-cc9b,U+cc9d-ccb3,U+ccb5-cd78;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.72.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.72.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.72.otf')  format('opentype');unicode-range:U+95b6-95b7,U+c83b,U+cbd8-cc27,U+cc29-cc43,U+cc45-cc9b,U+cc9d-ccb3,U+ccb5-cd78;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.73.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.73.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.73.otf')  format('opentype');unicode-range:U+3f03-40a6,U+c83e-c840;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.73.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.73.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.73.otf')  format('opentype');unicode-range:U+3f03-40a6,U+c83e-c840;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.73.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.73.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.73.otf')  format('opentype');unicode-range:U+3f03-40a6,U+c83e-c840;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.74.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.74.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.74.otf')  format('opentype');unicode-range:U+541c,U+ace8-acfb,U+acfd-ad4f,U+ad51-ad6b,U+ad6e-adf7,U+adf9-ae2f,U+ae31-ae37,U+ae39-ae3f,U+ae41-ae4b,U+ae4d-ae8b,U+c7c3-c7c5;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.74.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.74.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.74.otf')  format('opentype');unicode-range:U+541c,U+ace8-acfb,U+acfd-ad4f,U+ad51-ad6b,U+ad6e-adf7,U+adf9-ae2f,U+ae31-ae37,U+ae39-ae3f,U+ae41-ae4b,U+ae4d-ae8b,U+c7c3-c7c5;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.74.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.74.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.74.otf')  format('opentype');unicode-range:U+541c,U+ace8-acfb,U+acfd-ad4f,U+ad51-ad6b,U+ad6e-adf7,U+adf9-ae2f,U+ae31-ae37,U+ae39-ae3f,U+ae41-ae4b,U+ae4d-ae8b,U+c7c3-c7c5;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.75.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.75.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.75.otf')  format('opentype');unicode-range:U+b6ad-b77b,U+b77d-b78b,U+b78d-b790,U+b792-b797,U+b799-b823,U+b825-b850;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.75.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.75.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.75.otf')  format('opentype');unicode-range:U+b6ad-b77b,U+b77d-b78b,U+b78d-b790,U+b792-b797,U+b799-b823,U+b825-b850;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.75.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.75.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.75.otf')  format('opentype');unicode-range:U+b6ad-b77b,U+b77d-b78b,U+b78d-b790,U+b792-b797,U+b799-b823,U+b825-b850;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.76.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.76.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.76.otf')  format('opentype');unicode-range:U+24ff,U+587b-5882,U+5884,U+5886-5892,U+5894-5898,U+589a-589d,U+58a0-58a7,U+58aa-58c0,U+58c2-58e3,U+58e5-58ea,U+58f1-58f2,U+58f4-58f5,U+58f7-5903,U+5905-5906,U+5908-590c,U+590e,U+5910-5914,U+5917-5919,U+591b,U+591d-591e,U+5920-5926,U+5928,U+592c,U+592f-5930,U+5932-5933,U+5935-5936,U+593b-5946,U+594a,U+594c-594d,U+5950,U+5952-5953,U+5958-595f,U+5961,U+5963-5964,U+5966-5972,U+5975,U+5977,U+597a-597c,U+597e-5981,U+5985,U+5989-5991,U+5994-5995,U+5997-5998,U+599a-59a4,U+59a6-59a7,U+59ac-59ad,U+59af-59b8,U+59ba,U+59bc-59bd,U+59bf-59c5,U+59c7-59c9,U+59cc-59cf,U+59d2,U+59d5-59d9,U+59db,U+59dd-59e7,U+59e9-59eb,U+59ed-59fa,U+59fc-59fe,U+5a00,U+5a02,U+5a04,U+5a06,U+5a08-5a17,U+5a19-5a1b,U+5a1d-5a1e,U+5a20-5a30,U+5a32-5a33,U+5a35,U+5a37-5a45,U+5a47-5a48,U+5a4a-5a59,U+5a5b-5a69,U+5a6b-5a73,U+5a75-5a76,U+7ae4,U+8f67,U+94ea-94ec,U+95b4-95b5,U+95c7-95cc;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.76.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.76.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.76.otf')  format('opentype');unicode-range:U+24ff,U+587b-5882,U+5884,U+5886-5892,U+5894-5898,U+589a-589d,U+58a0-58a7,U+58aa-58c0,U+58c2-58e3,U+58e5-58ea,U+58f1-58f2,U+58f4-58f5,U+58f7-5903,U+5905-5906,U+5908-590c,U+590e,U+5910-5914,U+5917-5919,U+591b,U+591d-591e,U+5920-5926,U+5928,U+592c,U+592f-5930,U+5932-5933,U+5935-5936,U+593b-5946,U+594a,U+594c-594d,U+5950,U+5952-5953,U+5958-595f,U+5961,U+5963-5964,U+5966-5972,U+5975,U+5977,U+597a-597c,U+597e-5981,U+5985,U+5989-5991,U+5994-5995,U+5997-5998,U+599a-59a4,U+59a6-59a7,U+59ac-59ad,U+59af-59b8,U+59ba,U+59bc-59bd,U+59bf-59c5,U+59c7-59c9,U+59cc-59cf,U+59d2,U+59d5-59d9,U+59db,U+59dd-59e7,U+59e9-59eb,U+59ed-59fa,U+59fc-59fe,U+5a00,U+5a02,U+5a04,U+5a06,U+5a08-5a17,U+5a19-5a1b,U+5a1d-5a1e,U+5a20-5a30,U+5a32-5a33,U+5a35,U+5a37-5a45,U+5a47-5a48,U+5a4a-5a59,U+5a5b-5a69,U+5a6b-5a73,U+5a75-5a76,U+7ae4,U+8f67,U+94ea-94ec,U+95b4-95b5,U+95c7-95cc;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.76.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.76.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.76.otf')  format('opentype');unicode-range:U+24ff,U+587b-5882,U+5884,U+5886-5892,U+5894-5898,U+589a-589d,U+58a0-58a7,U+58aa-58c0,U+58c2-58e3,U+58e5-58ea,U+58f1-58f2,U+58f4-58f5,U+58f7-5903,U+5905-5906,U+5908-590c,U+590e,U+5910-5914,U+5917-5919,U+591b,U+591d-591e,U+5920-5926,U+5928,U+592c,U+592f-5930,U+5932-5933,U+5935-5936,U+593b-5946,U+594a,U+594c-594d,U+5950,U+5952-5953,U+5958-595f,U+5961,U+5963-5964,U+5966-5972,U+5975,U+5977,U+597a-597c,U+597e-5981,U+5985,U+5989-5991,U+5994-5995,U+5997-5998,U+599a-59a4,U+59a6-59a7,U+59ac-59ad,U+59af-59b8,U+59ba,U+59bc-59bd,U+59bf-59c5,U+59c7-59c9,U+59cc-59cf,U+59d2,U+59d5-59d9,U+59db,U+59dd-59e7,U+59e9-59eb,U+59ed-59fa,U+59fc-59fe,U+5a00,U+5a02,U+5a04,U+5a06,U+5a08-5a17,U+5a19-5a1b,U+5a1d-5a1e,U+5a20-5a30,U+5a32-5a33,U+5a35,U+5a37-5a45,U+5a47-5a48,U+5a4a-5a59,U+5a5b-5a69,U+5a6b-5a73,U+5a75-5a76,U+7ae4,U+8f67,U+94ea-94ec,U+95b4-95b5,U+95c7-95cc;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.77.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.77.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.77.otf')  format('opentype');unicode-range:U+30b0-30b6,U+4f35,U+5c80,U+70da,U+7600-7602,U+7931-7933,U+7935-7939,U+793b,U+793d,U+793f,U+7941-7947,U+7949-7955,U+7957-795a,U+795c,U+7961-7964,U+7966-7967,U+7969-796c,U+796e-7976,U+7979-797f,U+7982-7983,U+7986-798e,U+7990-79a6,U+79a8-79b8,U+79ba,U+79bf,U+79c2-79c8,U+79ca,U+79cc,U+79ce-79d0,U+79d3-79d4,U+79d6-79d7,U+79d9-79de,U+79e0-79e3,U+79e5,U+79e7-79e8,U+79ea-79ee,U+79f1-79fa,U+79fc,U+79fe-79ff,U+7a01-7a0a,U+7a0c,U+7a0f-7a19,U+7a1b-7a1f,U+7a21-7a22,U+7a24-7a32,U+7a34-7a3a,U+7a3c,U+7a40-7a45,U+7a47-7a56,U+7a58-7a73,U+7a75,U+7a78,U+7a7b-7a7e,U+7a80,U+7a82,U+7a85-7a8c,U+7a8e-7a90,U+7a94-7a96,U+7a98-7a9b,U+7aa0-7aa4,U+7aa7-7aca,U+7acc-7ad5,U+7ad7-7ad8,U+7ada-7add,U+7ae1-7ae2,U+7ae6-7aec,U+7aee,U+7af0-7af8,U+7afa-7afe,U+7b00-7b02,U+7b04-7b0a,U+7b0c-7b10,U+7b12-7b13,U+8338,U+94af-94b0,U+9583-9589,U+9889,U+c7c6-c7c8,U+2b461;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.77.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.77.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.77.otf')  format('opentype');unicode-range:U+30b0-30b6,U+4f35,U+5c80,U+70da,U+7600-7602,U+7931-7933,U+7935-7939,U+793b,U+793d,U+793f,U+7941-7947,U+7949-7955,U+7957-795a,U+795c,U+7961-7964,U+7966-7967,U+7969-796c,U+796e-7976,U+7979-797f,U+7982-7983,U+7986-798e,U+7990-79a6,U+79a8-79b8,U+79ba,U+79bf,U+79c2-79c8,U+79ca,U+79cc,U+79ce-79d0,U+79d3-79d4,U+79d6-79d7,U+79d9-79de,U+79e0-79e3,U+79e5,U+79e7-79e8,U+79ea-79ee,U+79f1-79fa,U+79fc,U+79fe-79ff,U+7a01-7a0a,U+7a0c,U+7a0f-7a19,U+7a1b-7a1f,U+7a21-7a22,U+7a24-7a32,U+7a34-7a3a,U+7a3c,U+7a40-7a45,U+7a47-7a56,U+7a58-7a73,U+7a75,U+7a78,U+7a7b-7a7e,U+7a80,U+7a82,U+7a85-7a8c,U+7a8e-7a90,U+7a94-7a96,U+7a98-7a9b,U+7aa0-7aa4,U+7aa7-7aca,U+7acc-7ad5,U+7ad7-7ad8,U+7ada-7add,U+7ae1-7ae2,U+7ae6-7aec,U+7aee,U+7af0-7af8,U+7afa-7afe,U+7b00-7b02,U+7b04-7b0a,U+7b0c-7b10,U+7b12-7b13,U+8338,U+94af-94b0,U+9583-9589,U+9889,U+c7c6-c7c8,U+2b461;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.77.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.77.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.77.otf')  format('opentype');unicode-range:U+30b0-30b6,U+4f35,U+5c80,U+70da,U+7600-7602,U+7931-7933,U+7935-7939,U+793b,U+793d,U+793f,U+7941-7947,U+7949-7955,U+7957-795a,U+795c,U+7961-7964,U+7966-7967,U+7969-796c,U+796e-7976,U+7979-797f,U+7982-7983,U+7986-798e,U+7990-79a6,U+79a8-79b8,U+79ba,U+79bf,U+79c2-79c8,U+79ca,U+79cc,U+79ce-79d0,U+79d3-79d4,U+79d6-79d7,U+79d9-79de,U+79e0-79e3,U+79e5,U+79e7-79e8,U+79ea-79ee,U+79f1-79fa,U+79fc,U+79fe-79ff,U+7a01-7a0a,U+7a0c,U+7a0f-7a19,U+7a1b-7a1f,U+7a21-7a22,U+7a24-7a32,U+7a34-7a3a,U+7a3c,U+7a40-7a45,U+7a47-7a56,U+7a58-7a73,U+7a75,U+7a78,U+7a7b-7a7e,U+7a80,U+7a82,U+7a85-7a8c,U+7a8e-7a90,U+7a94-7a96,U+7a98-7a9b,U+7aa0-7aa4,U+7aa7-7aca,U+7acc-7ad5,U+7ad7-7ad8,U+7ada-7add,U+7ae1-7ae2,U+7ae6-7aec,U+7aee,U+7af0-7af8,U+7afa-7afe,U+7b00-7b02,U+7b04-7b0a,U+7b0c-7b10,U+7b12-7b13,U+8338,U+94af-94b0,U+9583-9589,U+9889,U+c7c6-c7c8,U+2b461;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.78.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.78.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.78.otf')  format('opentype');unicode-range:U+5a78-5a91,U+5a93-5a99,U+5a9c-5ab1,U+5ab4-5ac0,U+5ac3-5ac8,U+5aca-5acb,U+5acd-5ae8,U+5aea-5b08,U+5b0a-5b4f,U+5b51-5b53,U+5b56,U+5b5a-5b5b,U+5b5e,U+5b60-5b62,U+5b65,U+5b67-5b68,U+5b6a-5b6b,U+5b6e-5b77,U+5b79-5b7c,U+5b7e-5b80,U+5b86,U+5b8a,U+5b8d-5b8e,U+5b90-5b96,U+5b9f,U+5ba5-5ba9,U+5bac-5baf,U+5bb1-5bb2,U+5bb7-5bb8,U+5bba-5bbc,U+5bc0-5bc1,U+5bc3,U+5bc8-5bcb,U+5bcd-5bcf,U+5bd1,U+5bd4-5bdc,U+5be0,U+5be2-5be4,U+5be6-5be7,U+5be9-5bf7,U+5bfd-5bfe,U+5c00,U+5c02-5c03,U+5c05,U+5c07-5c08,U+5c0b-5c0e,U+5c10,U+5c12-5c13,U+5c15,U+5c17,U+5c19,U+5c1b-5c1c,U+5c1e-5c23,U+5c25-5c26,U+5c28-5c2b,U+5c2d-5c30,U+5c32-5c33,U+5c35-5c37,U+5c3b,U+5c43-5c44,U+5c46-5c47,U+5c49,U+5c4c-5c4d,U+5c52-5c54,U+5c56-5c5d,U+5c5f,U+5c62-5c64,U+5c66-5c6e,U+5c70,U+5c72-5c7e,U+5c98-5c99,U+5f09,U+6da7,U+8ba3,U+94a3-94a4,U+94a8,U+9542-9546,U+9550-9555,U+9598-959b;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.78.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.78.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.78.otf')  format('opentype');unicode-range:U+5a78-5a91,U+5a93-5a99,U+5a9c-5ab1,U+5ab4-5ac0,U+5ac3-5ac8,U+5aca-5acb,U+5acd-5ae8,U+5aea-5b08,U+5b0a-5b4f,U+5b51-5b53,U+5b56,U+5b5a-5b5b,U+5b5e,U+5b60-5b62,U+5b65,U+5b67-5b68,U+5b6a-5b6b,U+5b6e-5b77,U+5b79-5b7c,U+5b7e-5b80,U+5b86,U+5b8a,U+5b8d-5b8e,U+5b90-5b96,U+5b9f,U+5ba5-5ba9,U+5bac-5baf,U+5bb1-5bb2,U+5bb7-5bb8,U+5bba-5bbc,U+5bc0-5bc1,U+5bc3,U+5bc8-5bcb,U+5bcd-5bcf,U+5bd1,U+5bd4-5bdc,U+5be0,U+5be2-5be4,U+5be6-5be7,U+5be9-5bf7,U+5bfd-5bfe,U+5c00,U+5c02-5c03,U+5c05,U+5c07-5c08,U+5c0b-5c0e,U+5c10,U+5c12-5c13,U+5c15,U+5c17,U+5c19,U+5c1b-5c1c,U+5c1e-5c23,U+5c25-5c26,U+5c28-5c2b,U+5c2d-5c30,U+5c32-5c33,U+5c35-5c37,U+5c3b,U+5c43-5c44,U+5c46-5c47,U+5c49,U+5c4c-5c4d,U+5c52-5c54,U+5c56-5c5d,U+5c5f,U+5c62-5c64,U+5c66-5c6e,U+5c70,U+5c72-5c7e,U+5c98-5c99,U+5f09,U+6da7,U+8ba3,U+94a3-94a4,U+94a8,U+9542-9546,U+9550-9555,U+9598-959b;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.78.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.78.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.78.otf')  format('opentype');unicode-range:U+5a78-5a91,U+5a93-5a99,U+5a9c-5ab1,U+5ab4-5ac0,U+5ac3-5ac8,U+5aca-5acb,U+5acd-5ae8,U+5aea-5b08,U+5b0a-5b4f,U+5b51-5b53,U+5b56,U+5b5a-5b5b,U+5b5e,U+5b60-5b62,U+5b65,U+5b67-5b68,U+5b6a-5b6b,U+5b6e-5b77,U+5b79-5b7c,U+5b7e-5b80,U+5b86,U+5b8a,U+5b8d-5b8e,U+5b90-5b96,U+5b9f,U+5ba5-5ba9,U+5bac-5baf,U+5bb1-5bb2,U+5bb7-5bb8,U+5bba-5bbc,U+5bc0-5bc1,U+5bc3,U+5bc8-5bcb,U+5bcd-5bcf,U+5bd1,U+5bd4-5bdc,U+5be0,U+5be2-5be4,U+5be6-5be7,U+5be9-5bf7,U+5bfd-5bfe,U+5c00,U+5c02-5c03,U+5c05,U+5c07-5c08,U+5c0b-5c0e,U+5c10,U+5c12-5c13,U+5c15,U+5c17,U+5c19,U+5c1b-5c1c,U+5c1e-5c23,U+5c25-5c26,U+5c28-5c2b,U+5c2d-5c30,U+5c32-5c33,U+5c35-5c37,U+5c3b,U+5c43-5c44,U+5c46-5c47,U+5c49,U+5c4c-5c4d,U+5c52-5c54,U+5c56-5c5d,U+5c5f,U+5c62-5c64,U+5c66-5c6e,U+5c70,U+5c72-5c7e,U+5c98-5c99,U+5f09,U+6da7,U+8ba3,U+94a3-94a4,U+94a8,U+9542-9546,U+9550-9555,U+9598-959b;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.79.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.79.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.79.otf')  format('opentype');unicode-range:U+03a3,U+03be,U+222a,U+23c9-23db,U+2466-24fd,U+2503-252b,U+252d-25b7,U+25c0-25ce,U+25d0-2603,U+260e-273d,U+5b78,U+5f9e,U+7d93,U+91cb,U+953c-953f,U+95dc,U+961e,U+fe36,U+ff11;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.79.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.79.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.79.otf')  format('opentype');unicode-range:U+03a3,U+03be,U+222a,U+23c9-23db,U+2466-24fd,U+2503-252b,U+252d-25b7,U+25c0-25ce,U+25d0-2603,U+260e-273d,U+5b78,U+5f9e,U+7d93,U+91cb,U+953c-953f,U+95dc,U+961e,U+fe36,U+ff11;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.79.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.79.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.79.otf')  format('opentype');unicode-range:U+03a3,U+03be,U+222a,U+23c9-23db,U+2466-24fd,U+2503-252b,U+252d-25b7,U+25c0-25ce,U+25d0-2603,U+260e-273d,U+5b78,U+5f9e,U+7d93,U+91cb,U+953c-953f,U+95dc,U+961e,U+fe36,U+ff11;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.80.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.80.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.80.otf')  format('opentype');unicode-range:U+43e6-4588;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.80.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.80.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.80.otf')  format('opentype');unicode-range:U+43e6-4588;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.80.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.80.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.80.otf')  format('opentype');unicode-range:U+43e6-4588;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.81.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.81.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.81.otf')  format('opentype');unicode-range:U+0391,U+2200,U+2234-2235,U+2423,U+30c2,U+4e0c,U+4e57,U+51f7,U+5497,U+56f1-56f3,U+5c95-5c96,U+5fa9,U+625c-6262,U+6264,U+6271-6275,U+6277-6278,U+627a-627b,U+6281-6283,U+6285-6289,U+628b-6290,U+6294,U+6299,U+629c-62a1,U+62a3,U+62a6-62aa,U+62ad-62b0,U+62b2-62b4,U+62b6-62b8,U+62ba-62bb,U+62be-62c1,U+62c3-62c4,U+62ca-62cb,U+62ce-62cf,U+62d1,U+62d5,U+62d7,U+62da,U+62dd-62de,U+62e0-62e1,U+62e3-62e4,U+62ea-62eb,U+62f2,U+62f4-62f6,U+62f8-62fb,U+6303-6306,U+6308,U+630a-6310,U+6312-6315,U+6317-6319,U+631b-631e,U+6322,U+6326-6327,U+6329,U+632c-632e,U+6330-6339,U+633b-633c,U+633e-6345,U+634a-634d,U+6351-6354,U+6356-635d,U+6360,U+6364-6366,U+6368-636d,U+636f-6376,U+6378-637f,U+6381-6387,U+638a-638b,U+638d-638e,U+6393-6397,U+6399-639f,U+63a3-63a4,U+63a6,U+63ab-63b6,U+63b8-63b9,U+63bb-63c8,U+63ca-63ce,U+63d1,U+63d3-63e0,U+63e2,U+63e4-63e9,U+63eb-63ec,U+63ee-63f3,U+63f5-63fc,U+63fe-6400,U+6402-6404,U+6406-640e,U+6410-6412,U+6414-641b,U+641d,U+641f-6425,U+6428-642b,U+642e-6439,U+643b-6443,U+6445,U+6448-6449,U+644b-6453,U+6455-6457,U+6459-6466,U+6468,U+646a-6477,U+647a-647c,U+73fe,U+7481-7482,U+7573,U+7a92-7a93,U+7e4b,U+80f4-80f5,U+8aac,U+9678-967a,U+983c,U+ff62-ff63,U+25d99,U+2f947;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.81.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.81.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.81.otf')  format('opentype');unicode-range:U+0391,U+2200,U+2234-2235,U+2423,U+30c2,U+4e0c,U+4e57,U+51f7,U+5497,U+56f1-56f3,U+5c95-5c96,U+5fa9,U+625c-6262,U+6264,U+6271-6275,U+6277-6278,U+627a-627b,U+6281-6283,U+6285-6289,U+628b-6290,U+6294,U+6299,U+629c-62a1,U+62a3,U+62a6-62aa,U+62ad-62b0,U+62b2-62b4,U+62b6-62b8,U+62ba-62bb,U+62be-62c1,U+62c3-62c4,U+62ca-62cb,U+62ce-62cf,U+62d1,U+62d5,U+62d7,U+62da,U+62dd-62de,U+62e0-62e1,U+62e3-62e4,U+62ea-62eb,U+62f2,U+62f4-62f6,U+62f8-62fb,U+6303-6306,U+6308,U+630a-6310,U+6312-6315,U+6317-6319,U+631b-631e,U+6322,U+6326-6327,U+6329,U+632c-632e,U+6330-6339,U+633b-633c,U+633e-6345,U+634a-634d,U+6351-6354,U+6356-635d,U+6360,U+6364-6366,U+6368-636d,U+636f-6376,U+6378-637f,U+6381-6387,U+638a-638b,U+638d-638e,U+6393-6397,U+6399-639f,U+63a3-63a4,U+63a6,U+63ab-63b6,U+63b8-63b9,U+63bb-63c8,U+63ca-63ce,U+63d1,U+63d3-63e0,U+63e2,U+63e4-63e9,U+63eb-63ec,U+63ee-63f3,U+63f5-63fc,U+63fe-6400,U+6402-6404,U+6406-640e,U+6410-6412,U+6414-641b,U+641d,U+641f-6425,U+6428-642b,U+642e-6439,U+643b-6443,U+6445,U+6448-6449,U+644b-6453,U+6455-6457,U+6459-6466,U+6468,U+646a-6477,U+647a-647c,U+73fe,U+7481-7482,U+7573,U+7a92-7a93,U+7e4b,U+80f4-80f5,U+8aac,U+9678-967a,U+983c,U+ff62-ff63,U+25d99,U+2f947;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.81.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.81.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.81.otf')  format('opentype');unicode-range:U+0391,U+2200,U+2234-2235,U+2423,U+30c2,U+4e0c,U+4e57,U+51f7,U+5497,U+56f1-56f3,U+5c95-5c96,U+5fa9,U+625c-6262,U+6264,U+6271-6275,U+6277-6278,U+627a-627b,U+6281-6283,U+6285-6289,U+628b-6290,U+6294,U+6299,U+629c-62a1,U+62a3,U+62a6-62aa,U+62ad-62b0,U+62b2-62b4,U+62b6-62b8,U+62ba-62bb,U+62be-62c1,U+62c3-62c4,U+62ca-62cb,U+62ce-62cf,U+62d1,U+62d5,U+62d7,U+62da,U+62dd-62de,U+62e0-62e1,U+62e3-62e4,U+62ea-62eb,U+62f2,U+62f4-62f6,U+62f8-62fb,U+6303-6306,U+6308,U+630a-6310,U+6312-6315,U+6317-6319,U+631b-631e,U+6322,U+6326-6327,U+6329,U+632c-632e,U+6330-6339,U+633b-633c,U+633e-6345,U+634a-634d,U+6351-6354,U+6356-635d,U+6360,U+6364-6366,U+6368-636d,U+636f-6376,U+6378-637f,U+6381-6387,U+638a-638b,U+638d-638e,U+6393-6397,U+6399-639f,U+63a3-63a4,U+63a6,U+63ab-63b6,U+63b8-63b9,U+63bb-63c8,U+63ca-63ce,U+63d1,U+63d3-63e0,U+63e2,U+63e4-63e9,U+63eb-63ec,U+63ee-63f3,U+63f5-63fc,U+63fe-6400,U+6402-6404,U+6406-640e,U+6410-6412,U+6414-641b,U+641d,U+641f-6425,U+6428-642b,U+642e-6439,U+643b-6443,U+6445,U+6448-6449,U+644b-6453,U+6455-6457,U+6459-6466,U+6468,U+646a-6477,U+647a-647c,U+73fe,U+7481-7482,U+7573,U+7a92-7a93,U+7e4b,U+80f4-80f5,U+8aac,U+9678-967a,U+983c,U+ff62-ff63,U+25d99,U+2f947;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.82.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.82.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.82.otf')  format('opentype');unicode-range:U+472a-48ca;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.82.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.82.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.82.otf')  format('opentype');unicode-range:U+472a-48ca;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.82.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.82.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.82.otf')  format('opentype');unicode-range:U+472a-48ca;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.83.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.83.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.83.otf')  format('opentype');unicode-range:U+c215-c217,U+c219-c2a3,U+c2a5-c2b4,U+c2b6-c2db,U+c2de-c2df,U+c2e1-c2eb,U+c2ed-c3b5,U+c701-c703,U+c7c1-c7c2;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.83.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.83.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.83.otf')  format('opentype');unicode-range:U+c215-c217,U+c219-c2a3,U+c2a5-c2b4,U+c2b6-c2db,U+c2de-c2df,U+c2e1-c2eb,U+c2ed-c3b5,U+c701-c703,U+c7c1-c7c2;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.83.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.83.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.83.otf')  format('opentype');unicode-range:U+c215-c217,U+c219-c2a3,U+c2a5-c2b4,U+c2b6-c2db,U+c2de-c2df,U+c2e1-c2eb,U+c2ed-c3b5,U+c701-c703,U+c7c1-c7c2;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.84.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.84.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.84.otf')  format('opentype');unicode-range:U+40a7-4245,U+94de-94df;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.84.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.84.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.84.otf')  format('opentype');unicode-range:U+40a7-4245,U+94de-94df;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.84.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.84.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.84.otf')  format('opentype');unicode-range:U+40a7-4245,U+94de-94df;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.85.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.85.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.85.otf')  format('opentype');unicode-range:U+2016,U+5741-5742,U+6074-6075,U+6077-607b,U+607d-6083,U+6085-6088,U+608a-608c,U+608e-6093,U+6095-609e,U+60a1-60a2,U+60a4-60a5,U+60a7,U+60a9-60ab,U+60ad-60ae,U+60b0-60b1,U+60b3-60bb,U+60bd-60c4,U+60c6-60c9,U+60cb-60d0,U+60d2-60d4,U+60d6-60db,U+60dd-60de,U+60e1-60e6,U+60ee,U+60f0-60f2,U+60f4-60f8,U+60fa-6100,U+6102-6107,U+610a-610e,U+6110-6119,U+611b-611e,U+6120-6123,U+6125-6126,U+6128-613e,U+6140-6147,U+6149-614b,U+614d,U+614f-6154,U+6156-6161,U+6163-6166,U+6169-616f,U+6171-6176,U+6178-618d,U+618f-61a8,U+61aa-61bd,U+61bf-61c1,U+61c3-61c7,U+61c9-61d1,U+61d3-6205,U+6207,U+6209,U+620b,U+6213-6215,U+6217,U+6219,U+621b-6229,U+622b-6232,U+6235-6236,U+6238-623e,U+6244-6246,U+6248-624a,U+624c,U+624f-6250,U+6255-6257,U+6259-625a,U+6268,U+626a,U+725c-725e,U+79be,U+949a-949c,U+9509-950a,U+22c62,U+237a2;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.85.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.85.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.85.otf')  format('opentype');unicode-range:U+2016,U+5741-5742,U+6074-6075,U+6077-607b,U+607d-6083,U+6085-6088,U+608a-608c,U+608e-6093,U+6095-609e,U+60a1-60a2,U+60a4-60a5,U+60a7,U+60a9-60ab,U+60ad-60ae,U+60b0-60b1,U+60b3-60bb,U+60bd-60c4,U+60c6-60c9,U+60cb-60d0,U+60d2-60d4,U+60d6-60db,U+60dd-60de,U+60e1-60e6,U+60ee,U+60f0-60f2,U+60f4-60f8,U+60fa-6100,U+6102-6107,U+610a-610e,U+6110-6119,U+611b-611e,U+6120-6123,U+6125-6126,U+6128-613e,U+6140-6147,U+6149-614b,U+614d,U+614f-6154,U+6156-6161,U+6163-6166,U+6169-616f,U+6171-6176,U+6178-618d,U+618f-61a8,U+61aa-61bd,U+61bf-61c1,U+61c3-61c7,U+61c9-61d1,U+61d3-6205,U+6207,U+6209,U+620b,U+6213-6215,U+6217,U+6219,U+621b-6229,U+622b-6232,U+6235-6236,U+6238-623e,U+6244-6246,U+6248-624a,U+624c,U+624f-6250,U+6255-6257,U+6259-625a,U+6268,U+626a,U+725c-725e,U+79be,U+949a-949c,U+9509-950a,U+22c62,U+237a2;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.85.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.85.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.85.otf')  format('opentype');unicode-range:U+2016,U+5741-5742,U+6074-6075,U+6077-607b,U+607d-6083,U+6085-6088,U+608a-608c,U+608e-6093,U+6095-609e,U+60a1-60a2,U+60a4-60a5,U+60a7,U+60a9-60ab,U+60ad-60ae,U+60b0-60b1,U+60b3-60bb,U+60bd-60c4,U+60c6-60c9,U+60cb-60d0,U+60d2-60d4,U+60d6-60db,U+60dd-60de,U+60e1-60e6,U+60ee,U+60f0-60f2,U+60f4-60f8,U+60fa-6100,U+6102-6107,U+610a-610e,U+6110-6119,U+611b-611e,U+6120-6123,U+6125-6126,U+6128-613e,U+6140-6147,U+6149-614b,U+614d,U+614f-6154,U+6156-6161,U+6163-6166,U+6169-616f,U+6171-6176,U+6178-618d,U+618f-61a8,U+61aa-61bd,U+61bf-61c1,U+61c3-61c7,U+61c9-61d1,U+61d3-6205,U+6207,U+6209,U+620b,U+6213-6215,U+6217,U+6219,U+621b-6229,U+622b-6232,U+6235-6236,U+6238-623e,U+6244-6246,U+6248-624a,U+624c,U+624f-6250,U+6255-6257,U+6259-625a,U+6268,U+626a,U+725c-725e,U+79be,U+949a-949c,U+9509-950a,U+22c62,U+237a2;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.86.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.86.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.86.otf')  format('opentype');unicode-range:U+0000-0020,U+00a1-00af,U+00b1-00b6,U+00b8-00c4,U+00c6-00d6,U+00d8-00df,U+00e2-00e7,U+00ea-00eb,U+00ed-00f1,U+00f4-00f8,U+00fa-0100,U+0102-0113,U+0128-01cd,U+01d1-01d3,U+01d5-01f9,U+02d9,U+0392-039a,U+039c-03a1,U+03a4-03b5,U+03bb-03bc,U+03c3-03c4,U+03c6-1185,U+201a,U+2035,U+2160,U+3060,U+634e,U+6baa,U+6eb9,U+824f,U+948a-948b,U+9564-9568,U+95bb-95bd,U+9a83;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.86.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.86.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.86.otf')  format('opentype');unicode-range:U+0000-0020,U+00a1-00af,U+00b1-00b6,U+00b8-00c4,U+00c6-00d6,U+00d8-00df,U+00e2-00e7,U+00ea-00eb,U+00ed-00f1,U+00f4-00f8,U+00fa-0100,U+0102-0113,U+0128-01cd,U+01d1-01d3,U+01d5-01f9,U+02d9,U+0392-039a,U+039c-03a1,U+03a4-03b5,U+03bb-03bc,U+03c3-03c4,U+03c6-1185,U+201a,U+2035,U+2160,U+3060,U+634e,U+6baa,U+6eb9,U+824f,U+948a-948b,U+9564-9568,U+95bb-95bd,U+9a83;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.86.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.86.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.86.otf')  format('opentype');unicode-range:U+0000-0020,U+00a1-00af,U+00b1-00b6,U+00b8-00c4,U+00c6-00d6,U+00d8-00df,U+00e2-00e7,U+00ea-00eb,U+00ed-00f1,U+00f4-00f8,U+00fa-0100,U+0102-0113,U+0128-01cd,U+01d1-01d3,U+01d5-01f9,U+02d9,U+0392-039a,U+039c-03a1,U+03a4-03b5,U+03bb-03bc,U+03c3-03c4,U+03c6-1185,U+201a,U+2035,U+2160,U+3060,U+634e,U+6baa,U+6eb9,U+824f,U+948a-948b,U+9564-9568,U+95bb-95bd,U+9a83;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.87.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.87.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.87.otf')  format('opentype');unicode-range:U+4246-43e5;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.87.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.87.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.87.otf')  format('opentype');unicode-range:U+4246-43e5;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.87.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.87.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.87.otf')  format('opentype');unicode-range:U+4246-43e5;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.88.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.88.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.88.otf')  format('opentype');unicode-range:U+309d,U+5491,U+64ae-64af,U+89c7,U+9d73-9dde,U+9de0-9e1e,U+9e24-9e25,U+9e27-9e2c,U+9e2e-9e31,U+9e34-9e3c,U+9e40-9e44,U+9e46-9e49,U+9e4b-9e4e,U+9e50-9e63,U+9e65-9e6f,U+9e71-9e7e,U+9e80-9e92,U+9e94-9e9e,U+9ea0-9ea5,U+9ea7-9eba,U+9ebc-9ec3,U+9ec5-9ecd,U+9ed0,U+9ed2-9ed7,U+9ed9-9eda,U+9edc-9eee,U+9ef0-9f0d,U+9f0f-9f12,U+9f14-9f1f,U+9f21-9f2e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.88.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.88.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.88.otf')  format('opentype');unicode-range:U+309d,U+5491,U+64ae-64af,U+89c7,U+9d73-9dde,U+9de0-9e1e,U+9e24-9e25,U+9e27-9e2c,U+9e2e-9e31,U+9e34-9e3c,U+9e40-9e44,U+9e46-9e49,U+9e4b-9e4e,U+9e50-9e63,U+9e65-9e6f,U+9e71-9e7e,U+9e80-9e92,U+9e94-9e9e,U+9ea0-9ea5,U+9ea7-9eba,U+9ebc-9ec3,U+9ec5-9ecd,U+9ed0,U+9ed2-9ed7,U+9ed9-9eda,U+9edc-9eee,U+9ef0-9f0d,U+9f0f-9f12,U+9f14-9f1f,U+9f21-9f2e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.88.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.88.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.88.otf')  format('opentype');unicode-range:U+309d,U+5491,U+64ae-64af,U+89c7,U+9d73-9dde,U+9de0-9e1e,U+9e24-9e25,U+9e27-9e2c,U+9e2e-9e31,U+9e34-9e3c,U+9e40-9e44,U+9e46-9e49,U+9e4b-9e4e,U+9e50-9e63,U+9e65-9e6f,U+9e71-9e7e,U+9e80-9e92,U+9e94-9e9e,U+9ea0-9ea5,U+9ea7-9eba,U+9ebc-9ec3,U+9ec5-9ecd,U+9ed0,U+9ed2-9ed7,U+9ed9-9eda,U+9edc-9eee,U+9ef0-9f0d,U+9f0f-9f12,U+9f14-9f1f,U+9f21-9f2e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.89.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.89.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.89.otf')  format('opentype');unicode-range:U+3d63-3f02;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.89.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.89.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.89.otf')  format('opentype');unicode-range:U+3d63-3f02;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.89.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.89.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.89.otf')  format('opentype');unicode-range:U+3d63-3f02;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.90.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.90.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.90.otf')  format('opentype');unicode-range:U+2027,U+54cf-54d0,U+54d2-54d6,U+54d8-54de,U+54e0-54e4,U+54e7,U+54e9,U+54eb-54ec,U+54ef-54f1,U+54f3-54f9,U+54fd-5506,U+5508,U+550a-550f,U+5511-5523,U+5525-552d,U+5530,U+5532-5545,U+5547-5549,U+554b-5560,U+5562-5563,U+5567-5574,U+5576-5577,U+5579-557b,U+557d-557f,U+5581,U+5585-5586,U+5588,U+558b-5597,U+5599-559b,U+559e-55a6,U+55a8-55aa,U+55ad-55b6,U+55b8-55ba,U+55bc-55c3,U+55c6-55d2,U+55d4-55db,U+55dd-55e2,U+55e4-55ee,U+55f0-55fc,U+55fe-5608,U+560a-560d,U+560f-5617,U+5619-561a,U+561c-5630,U+5633,U+5635-563a,U+563c-563e,U+5640-5667,U+5669,U+566b,U+566d-569c,U+5767-5768,U+7fe8,U+80f7,U+8bc5,U+8d63,U+8f87,U+9502-9503,U+9530-9534,U+9548-9549;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.90.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.90.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.90.otf')  format('opentype');unicode-range:U+2027,U+54cf-54d0,U+54d2-54d6,U+54d8-54de,U+54e0-54e4,U+54e7,U+54e9,U+54eb-54ec,U+54ef-54f1,U+54f3-54f9,U+54fd-5506,U+5508,U+550a-550f,U+5511-5523,U+5525-552d,U+5530,U+5532-5545,U+5547-5549,U+554b-5560,U+5562-5563,U+5567-5574,U+5576-5577,U+5579-557b,U+557d-557f,U+5581,U+5585-5586,U+5588,U+558b-5597,U+5599-559b,U+559e-55a6,U+55a8-55aa,U+55ad-55b6,U+55b8-55ba,U+55bc-55c3,U+55c6-55d2,U+55d4-55db,U+55dd-55e2,U+55e4-55ee,U+55f0-55fc,U+55fe-5608,U+560a-560d,U+560f-5617,U+5619-561a,U+561c-5630,U+5633,U+5635-563a,U+563c-563e,U+5640-5667,U+5669,U+566b,U+566d-569c,U+5767-5768,U+7fe8,U+80f7,U+8bc5,U+8d63,U+8f87,U+9502-9503,U+9530-9534,U+9548-9549;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.90.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.90.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.90.otf')  format('opentype');unicode-range:U+2027,U+54cf-54d0,U+54d2-54d6,U+54d8-54de,U+54e0-54e4,U+54e7,U+54e9,U+54eb-54ec,U+54ef-54f1,U+54f3-54f9,U+54fd-5506,U+5508,U+550a-550f,U+5511-5523,U+5525-552d,U+5530,U+5532-5545,U+5547-5549,U+554b-5560,U+5562-5563,U+5567-5574,U+5576-5577,U+5579-557b,U+557d-557f,U+5581,U+5585-5586,U+5588,U+558b-5597,U+5599-559b,U+559e-55a6,U+55a8-55aa,U+55ad-55b6,U+55b8-55ba,U+55bc-55c3,U+55c6-55d2,U+55d4-55db,U+55dd-55e2,U+55e4-55ee,U+55f0-55fc,U+55fe-5608,U+560a-560d,U+560f-5617,U+5619-561a,U+561c-5630,U+5633,U+5635-563a,U+563c-563e,U+5640-5667,U+5669,U+566b,U+566d-569c,U+5767-5768,U+7fe8,U+80f7,U+8bc5,U+8d63,U+8f87,U+9502-9503,U+9530-9534,U+9548-9549;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.91.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.91.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.91.otf')  format('opentype');unicode-range:U+ca37-cbd7,U+26cdd;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.91.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.91.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.91.otf')  format('opentype');unicode-range:U+ca37-cbd7,U+26cdd;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.91.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.91.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.91.otf')  format('opentype');unicode-range:U+ca37-cbd7,U+26cdd;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.92.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.92.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.92.otf')  format('opentype');unicode-range:U+3067,U+505b,U+505d-5064,U+5066-5075,U+5078-507e,U+5080-5084,U+5086-508c,U+508e-50a7,U+50a9-50ab,U+50ad-50b1,U+50b3-50ba,U+50bc-50ce,U+50d0-50e6,U+50e8-50f4,U+50f6-50fa,U+50fc-5111,U+5113-513e,U+5142,U+5147,U+514a,U+514c,U+514e-5150,U+5152-5153,U+5156-5159,U+515b,U+515d-5164,U+5166-5167,U+5169-516a,U+516f,U+5172,U+517a,U+517e-517f,U+5181-5184,U+5186-5187,U+5189-518b,U+518e-5191,U+5193-5194,U+5196-5198,U+519d-519f,U+51a1,U+51a6-51ab,U+51ad-51ae,U+51b1,U+51b4,U+51b8-51ba,U+51bc-51bf,U+51c1-51c3,U+51c5,U+51c7-51c8,U+51ca,U+51cd-51ce,U+51d0,U+51d2-51da,U+51dc,U+51de-51df,U+51e2-51e3,U+51e5-51ec,U+51f1-51f2,U+51f4-51f5,U+5202,U+5204-5205,U+5208-5209,U+520b-5210,U+5213-5216,U+521c,U+521e-521f,U+5221-5223,U+5225-5228,U+522a,U+522c-522d,U+522f,U+5231-5232,U+5234-5235,U+523c-5241,U+5243-5249,U+524b-524c,U+524e-5250,U+5252-5253,U+5255,U+5257-5264,U+5266,U+5268,U+526b-526e,U+5270-5271,U+5273-527c,U+8c24-8c25,U+94b9-94ba,U+9602-9604,U+22c51,U+2478f,U+25da1;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.92.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.92.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.92.otf')  format('opentype');unicode-range:U+3067,U+505b,U+505d-5064,U+5066-5075,U+5078-507e,U+5080-5084,U+5086-508c,U+508e-50a7,U+50a9-50ab,U+50ad-50b1,U+50b3-50ba,U+50bc-50ce,U+50d0-50e6,U+50e8-50f4,U+50f6-50fa,U+50fc-5111,U+5113-513e,U+5142,U+5147,U+514a,U+514c,U+514e-5150,U+5152-5153,U+5156-5159,U+515b,U+515d-5164,U+5166-5167,U+5169-516a,U+516f,U+5172,U+517a,U+517e-517f,U+5181-5184,U+5186-5187,U+5189-518b,U+518e-5191,U+5193-5194,U+5196-5198,U+519d-519f,U+51a1,U+51a6-51ab,U+51ad-51ae,U+51b1,U+51b4,U+51b8-51ba,U+51bc-51bf,U+51c1-51c3,U+51c5,U+51c7-51c8,U+51ca,U+51cd-51ce,U+51d0,U+51d2-51da,U+51dc,U+51de-51df,U+51e2-51e3,U+51e5-51ec,U+51f1-51f2,U+51f4-51f5,U+5202,U+5204-5205,U+5208-5209,U+520b-5210,U+5213-5216,U+521c,U+521e-521f,U+5221-5223,U+5225-5228,U+522a,U+522c-522d,U+522f,U+5231-5232,U+5234-5235,U+523c-5241,U+5243-5249,U+524b-524c,U+524e-5250,U+5252-5253,U+5255,U+5257-5264,U+5266,U+5268,U+526b-526e,U+5270-5271,U+5273-527c,U+8c24-8c25,U+94b9-94ba,U+9602-9604,U+22c51,U+2478f,U+25da1;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.92.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.92.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.92.otf')  format('opentype');unicode-range:U+3067,U+505b,U+505d-5064,U+5066-5075,U+5078-507e,U+5080-5084,U+5086-508c,U+508e-50a7,U+50a9-50ab,U+50ad-50b1,U+50b3-50ba,U+50bc-50ce,U+50d0-50e6,U+50e8-50f4,U+50f6-50fa,U+50fc-5111,U+5113-513e,U+5142,U+5147,U+514a,U+514c,U+514e-5150,U+5152-5153,U+5156-5159,U+515b,U+515d-5164,U+5166-5167,U+5169-516a,U+516f,U+5172,U+517a,U+517e-517f,U+5181-5184,U+5186-5187,U+5189-518b,U+518e-5191,U+5193-5194,U+5196-5198,U+519d-519f,U+51a1,U+51a6-51ab,U+51ad-51ae,U+51b1,U+51b4,U+51b8-51ba,U+51bc-51bf,U+51c1-51c3,U+51c5,U+51c7-51c8,U+51ca,U+51cd-51ce,U+51d0,U+51d2-51da,U+51dc,U+51de-51df,U+51e2-51e3,U+51e5-51ec,U+51f1-51f2,U+51f4-51f5,U+5202,U+5204-5205,U+5208-5209,U+520b-5210,U+5213-5216,U+521c,U+521e-521f,U+5221-5223,U+5225-5228,U+522a,U+522c-522d,U+522f,U+5231-5232,U+5234-5235,U+523c-5241,U+5243-5249,U+524b-524c,U+524e-5250,U+5252-5253,U+5255,U+5257-5264,U+5266,U+5268,U+526b-526e,U+5270-5271,U+5273-527c,U+8c24-8c25,U+94b9-94ba,U+9602-9604,U+22c51,U+2478f,U+25da1;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.93.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.93.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.93.otf')  format('opentype');unicode-range:U+306a,U+620d,U+7009-7010,U+7012-7019,U+701b-704b,U+704d-706a,U+706c,U+706e,U+7071-7074,U+7077-707b,U+7080-7088,U+708a-708d,U+708f-7091,U+7093-7095,U+7097-7098,U+709a-70aa,U+70af-70b7,U+70ba-70bb,U+70bd-70c0,U+70c3-70c7,U+70c9-70d7,U+70dc-70de,U+70e0-70e3,U+70e5,U+70e8-70ea,U+70ec,U+70ee,U+70f0-70f8,U+70fa-7108,U+710c-7114,U+7116-7118,U+711d-7125,U+7127-712f,U+7131-7135,U+7137-714b,U+714d,U+714f-715d,U+715f-7163,U+7165-7166,U+7168-716d,U+716f-7171,U+7173-717c,U+717e-7183,U+7185-7189,U+718b-718e,U+7190-7193,U+7195-7198,U+719a-719e,U+71a0-71ab,U+71ad-71c2,U+71c4-71d4,U+7d50,U+8337,U+8f9a,U+94f0-94f1,U+94f9,U+94fb-94fd,U+94ff,U+950d-950f,U+9535,U+9556-9558,U+966d-966f,U+ff2d;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.93.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.93.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.93.otf')  format('opentype');unicode-range:U+306a,U+620d,U+7009-7010,U+7012-7019,U+701b-704b,U+704d-706a,U+706c,U+706e,U+7071-7074,U+7077-707b,U+7080-7088,U+708a-708d,U+708f-7091,U+7093-7095,U+7097-7098,U+709a-70aa,U+70af-70b7,U+70ba-70bb,U+70bd-70c0,U+70c3-70c7,U+70c9-70d7,U+70dc-70de,U+70e0-70e3,U+70e5,U+70e8-70ea,U+70ec,U+70ee,U+70f0-70f8,U+70fa-7108,U+710c-7114,U+7116-7118,U+711d-7125,U+7127-712f,U+7131-7135,U+7137-714b,U+714d,U+714f-715d,U+715f-7163,U+7165-7166,U+7168-716d,U+716f-7171,U+7173-717c,U+717e-7183,U+7185-7189,U+718b-718e,U+7190-7193,U+7195-7198,U+719a-719e,U+71a0-71ab,U+71ad-71c2,U+71c4-71d4,U+7d50,U+8337,U+8f9a,U+94f0-94f1,U+94f9,U+94fb-94fd,U+94ff,U+950d-950f,U+9535,U+9556-9558,U+966d-966f,U+ff2d;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.93.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.93.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.93.otf')  format('opentype');unicode-range:U+306a,U+620d,U+7009-7010,U+7012-7019,U+701b-704b,U+704d-706a,U+706c,U+706e,U+7071-7074,U+7077-707b,U+7080-7088,U+708a-708d,U+708f-7091,U+7093-7095,U+7097-7098,U+709a-70aa,U+70af-70b7,U+70ba-70bb,U+70bd-70c0,U+70c3-70c7,U+70c9-70d7,U+70dc-70de,U+70e0-70e3,U+70e5,U+70e8-70ea,U+70ec,U+70ee,U+70f0-70f8,U+70fa-7108,U+710c-7114,U+7116-7118,U+711d-7125,U+7127-712f,U+7131-7135,U+7137-714b,U+714d,U+714f-715d,U+715f-7163,U+7165-7166,U+7168-716d,U+716f-7171,U+7173-717c,U+717e-7183,U+7185-7189,U+718b-718e,U+7190-7193,U+7195-7198,U+719a-719e,U+71a0-71ab,U+71ad-71c2,U+71c4-71d4,U+7d50,U+8337,U+8f9a,U+94f0-94f1,U+94f9,U+94fb-94fd,U+94ff,U+950d-950f,U+9535,U+9556-9558,U+966d-966f,U+ff2d;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.94.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.94.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.94.otf')  format('opentype');unicode-range:U+b9ef-ba6f,U+ba71-ba73,U+ba75-ba84,U+ba86-baa7,U+baaa-baba,U+babc-bb33,U+bb35-bb37,U+bb39-bb3b,U+bb3d-bb8c,U+c75b,U+c7d1-c7da;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.94.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.94.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.94.otf')  format('opentype');unicode-range:U+b9ef-ba6f,U+ba71-ba73,U+ba75-ba84,U+ba86-baa7,U+baaa-baba,U+babc-bb33,U+bb35-bb37,U+bb39-bb3b,U+bb3d-bb8c,U+c75b,U+c7d1-c7da;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.94.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.94.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.94.otf')  format('opentype');unicode-range:U+b9ef-ba6f,U+ba71-ba73,U+ba75-ba84,U+ba86-baa7,U+baaa-baba,U+babc-bb33,U+bb35-bb37,U+bb39-bb3b,U+bb3d-bb8c,U+c75b,U+c7d1-c7da;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.95.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.95.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.95.otf')  format('opentype');unicode-range:U+4a6c-4c0b;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.95.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.95.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.95.otf')  format('opentype');unicode-range:U+4a6c-4c0b;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.95.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.95.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.95.otf')  format('opentype');unicode-range:U+4a6c-4c0b;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.96.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.96.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.96.otf')  format('opentype');unicode-range:U+3061-3063,U+64b1,U+7e85-7e9f,U+7ea8-7ea9,U+7eab,U+7ead-7eae,U+7eb0,U+7ebb-7ebc,U+7ec0-7ec2,U+7ec9-7ecc,U+7ed0,U+7ed4,U+7ed6-7ed7,U+7ee0-7ee2,U+7ee4-7ee6,U+7ee8,U+7eeb-7eec,U+7ef1-7ef2,U+7ef6,U+7ef9-7efb,U+7efe,U+7f01-7f04,U+7f0a-7f12,U+7f19,U+7f1b-7f1c,U+7f1e-7f1f,U+7f21-7f23,U+7f25-7f28,U+7f2b-7f2c,U+7f2e,U+7f30-7f33,U+7f35-7f37,U+7f39,U+7f3b-7f4f,U+7f52-7f54,U+7f56,U+7f58-7f59,U+7f5b-7f61,U+7f63-7f68,U+7f6b-7f6d,U+7f6f-7f71,U+7f73-7f80,U+7f82-7f89,U+7f8b-7f8d,U+7f8f-7f9d,U+7f9f-7fa0,U+7fa2-7fa3,U+7fa5-7fbc,U+7fbe-7fc0,U+7fc2-7fc4,U+7fc6-7fd3,U+7fd5-7fd7,U+7fd9-7fdf,U+7fe1-7fe7,U+7fea-7fef,U+7ff1-7ffa,U+7ffd-7fff,U+8004,U+8006-800b,U+800e-800f,U+8011-8014,U+8016,U+8018-8032,U+8034-8035,U+8037,U+8039-803a,U+803c,U+803e,U+8040-8045,U+8047-8049,U+804d-8053,U+8055-8057,U+8059,U+805b-8069,U+806b-8082,U+8084-8085,U+808a,U+808d-8095,U+8097,U+89c3,U+8bec,U+94b2-94b4,U+94f3-94f5,U+954a-954f,U+9578-957c,U+9626-9629,U+9708;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.96.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.96.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.96.otf')  format('opentype');unicode-range:U+3061-3063,U+64b1,U+7e85-7e9f,U+7ea8-7ea9,U+7eab,U+7ead-7eae,U+7eb0,U+7ebb-7ebc,U+7ec0-7ec2,U+7ec9-7ecc,U+7ed0,U+7ed4,U+7ed6-7ed7,U+7ee0-7ee2,U+7ee4-7ee6,U+7ee8,U+7eeb-7eec,U+7ef1-7ef2,U+7ef6,U+7ef9-7efb,U+7efe,U+7f01-7f04,U+7f0a-7f12,U+7f19,U+7f1b-7f1c,U+7f1e-7f1f,U+7f21-7f23,U+7f25-7f28,U+7f2b-7f2c,U+7f2e,U+7f30-7f33,U+7f35-7f37,U+7f39,U+7f3b-7f4f,U+7f52-7f54,U+7f56,U+7f58-7f59,U+7f5b-7f61,U+7f63-7f68,U+7f6b-7f6d,U+7f6f-7f71,U+7f73-7f80,U+7f82-7f89,U+7f8b-7f8d,U+7f8f-7f9d,U+7f9f-7fa0,U+7fa2-7fa3,U+7fa5-7fbc,U+7fbe-7fc0,U+7fc2-7fc4,U+7fc6-7fd3,U+7fd5-7fd7,U+7fd9-7fdf,U+7fe1-7fe7,U+7fea-7fef,U+7ff1-7ffa,U+7ffd-7fff,U+8004,U+8006-800b,U+800e-800f,U+8011-8014,U+8016,U+8018-8032,U+8034-8035,U+8037,U+8039-803a,U+803c,U+803e,U+8040-8045,U+8047-8049,U+804d-8053,U+8055-8057,U+8059,U+805b-8069,U+806b-8082,U+8084-8085,U+808a,U+808d-8095,U+8097,U+89c3,U+8bec,U+94b2-94b4,U+94f3-94f5,U+954a-954f,U+9578-957c,U+9626-9629,U+9708;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.96.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.96.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.96.otf')  format('opentype');unicode-range:U+3061-3063,U+64b1,U+7e85-7e9f,U+7ea8-7ea9,U+7eab,U+7ead-7eae,U+7eb0,U+7ebb-7ebc,U+7ec0-7ec2,U+7ec9-7ecc,U+7ed0,U+7ed4,U+7ed6-7ed7,U+7ee0-7ee2,U+7ee4-7ee6,U+7ee8,U+7eeb-7eec,U+7ef1-7ef2,U+7ef6,U+7ef9-7efb,U+7efe,U+7f01-7f04,U+7f0a-7f12,U+7f19,U+7f1b-7f1c,U+7f1e-7f1f,U+7f21-7f23,U+7f25-7f28,U+7f2b-7f2c,U+7f2e,U+7f30-7f33,U+7f35-7f37,U+7f39,U+7f3b-7f4f,U+7f52-7f54,U+7f56,U+7f58-7f59,U+7f5b-7f61,U+7f63-7f68,U+7f6b-7f6d,U+7f6f-7f71,U+7f73-7f80,U+7f82-7f89,U+7f8b-7f8d,U+7f8f-7f9d,U+7f9f-7fa0,U+7fa2-7fa3,U+7fa5-7fbc,U+7fbe-7fc0,U+7fc2-7fc4,U+7fc6-7fd3,U+7fd5-7fd7,U+7fd9-7fdf,U+7fe1-7fe7,U+7fea-7fef,U+7ff1-7ffa,U+7ffd-7fff,U+8004,U+8006-800b,U+800e-800f,U+8011-8014,U+8016,U+8018-8032,U+8034-8035,U+8037,U+8039-803a,U+803c,U+803e,U+8040-8045,U+8047-8049,U+804d-8053,U+8055-8057,U+8059,U+805b-8069,U+806b-8082,U+8084-8085,U+808a,U+808d-8095,U+8097,U+89c3,U+8bec,U+94b2-94b4,U+94f3-94f5,U+954a-954f,U+9578-957c,U+9626-9629,U+9708;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.97.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.97.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.97.otf')  format('opentype');unicode-range:U+0060,U+00e1,U+00e8,U+2015,U+2161,U+2464-2465,U+2500,U+25bc,U+25cf,U+2605,U+3008-3009,U+3014-3015,U+3044,U+304b-304d,U+304f,U+3057,U+305f,U+3068,U+307e-307f,U+3089-308b,U+30af,U+30b7,U+30e9-30eb,U+30f3,U+30fc,U+4e1e,U+4e4d,U+4ea2,U+4f36,U+4f3a,U+4f6c,U+4f84,U+4fae,U+4fde,U+5014,U+5018,U+5029-502a,U+5055,U+5140,U+516e,U+5180,U+5195,U+51a2,U+51cb,U+51db,U+51f3,U+5201,U+527f,U+533e,U+5364,U+536f,U+53a5,U+53e8-53e9,U+53ed,U+5480,U+5490,U+54aa,U+54d7,U+54e8,U+54ee,U+54fa,U+54fc,U+557c,U+5580,U+5583,U+55c5,U+55e3,U+55fd,U+560e,U+5618,U+563b,U+566c,U+5777,U+57a2-57a3,U+57ae,U+57d4,U+589f,U+58a9,U+58ec,U+592d,U+594e,U+5955,U+5993,U+59a9,U+59be,U+59ca,U+5a05,U+5a1f,U+5a34,U+5a6a,U+5a77,U+5a9b,U+5ab2,U+5ac2,U+5b09,U+5b5c,U+5b7d,U+5bc7,U+5bd0,U+5be5,U+5c09,U+5c27,U+5c4e,U+5c7f,U+5c9a,U+5cd9,U+5ce8,U+5ced,U+5dcd,U+5df3,U+5e37,U+5e62,U+5e87,U+5eb5-5eb6,U+5ed6,U+5f08,U+5f1b,U+5f6a,U+5f6c,U+5f8a,U+5f98-5f99,U+5fcf,U+6020,U+6055,U+6064,U+606c,U+60af,U+60bc,U+60eb-60ed,U+6177,U+618e,U+61a9,U+620a,U+620c,U+620e,U+6252,U+625b,U+627c,U+62c2,U+62c7,U+62e7,U+62ed,U+62f7,U+62fd,U+631f-6320,U+6346,U+6390,U+63b7,U+63ba,U+63ea,U+6479,U+6487,U+64d2,U+6590,U+6656,U+6666,U+667e,U+66a7,U+6714,U+6726,U+6789,U+67ff,U+6805,U+6808,U+6813,U+6853-6854,U+6893,U+68a2,U+68a7,U+68d8,U+68e0,U+68fa,U+6930,U+6960,U+69a8,U+69b4,U+6a0a,U+6a71,U+6a80,U+6aac,U+6b79,U+6b7c,U+6bcb,U+6bd9,U+6c13,U+6c2e-6c2f,U+6c40,U+6c72,U+6c76,U+6c79,U+6c81,U+6cbc,U+6cf5,U+6d95,U+6da1,U+6da4,U+6dc6-6dc7,U+6dcc,U+6de4,U+6e1d,U+6e3a,U+6eba,U+6fd2,U+701a,U+7076,U+7099,U+70ac,U+70c1,U+70ef,U+710a,U+7119,U+7172,U+717d,U+7184,U+725f,U+7280,U+72c4,U+72e1,U+72f8,U+73c0,U+73d1,U+7405,U+7409,U+7426,U+7436,U+745a,U+7480,U+74a7-74a8,U+7504,U+7578,U+759a,U+75a1,U+75b9,U+75d8,U+75f0,U+762b,U+7656,U+7678,U+76ce-76cf,U+7729,U+7738,U+773a,U+7766,U+7784,U+77aa,U+77b0,U+785d,U+78ca,U+7901,U+7934,U+7960,U+79a7,U+79b9,U+79e4,U+79fd,U+7a23,U+7a57,U+7a79,U+7a91,U+7a9c,U+7a9f,U+7aa6,U+7ae3,U+7aff,U+7b03,U+7b0b,U+7b77,U+7bc6,U+7be1,U+7bf7,U+7c07,U+7c3f,U+7c7d,U+7c9f,U+7caa,U+7cef,U+7d0a,U+7ec5,U+7ede,U+7eee-7ef0,U+7ef7,U+7f06,U+7f24,U+7f2a,U+7f2d,U+7f81,U+7fe9,U+803f,U+8046,U+804b,U+8087,U+8098,U+80b4,U+80da,U+80e7,U+80f1,U+80f3,U+80fa,U+814b-814c,U+8151,U+818a,U+81b3,U+8231,U+829c,U+82a5,U+82b8-82b9,U+82c7,U+82d4,U+82ef,U+8335,U+8339,U+835f,U+8367,U+839e,U+83e0,U+83f1,U+8469,U+846b,U+854a,U+85e9,U+85fb,U+8611,U+8638,U+864f,U+865e,U+8681,U+868a,U+8695,U+86db,U+8712,U+8715,U+8718,U+8747,U+8749,U+889c,U+88f3-88f4,U+8912,U+892a,U+8944,U+895f,U+8bcf,U+8bdb,U+8be3,U+8beb,U+8bf2,U+8c0d,U+8c0f,U+8c1b,U+8c2c,U+8c34,U+8d3f,U+8d4e,U+8d58,U+8d66,U+8db4,U+8dcb,U+8dfb,U+8e66,U+8eac,U+8f7c,U+8f84,U+8f97,U+8f99,U+8fe5,U+9005,U+9050,U+9082-9083,U+9091,U+90b1,U+90b5,U+90b9,U+9102,U+914c,U+916a,U+916e-916f,U+946b,U+949e,U+94a0,U+94a7,U+94b5,U+94be,U+94f2,U+950c,U+9523,U+952f,U+9540-9541,U+95f8,U+95fd,U+960e,U+9619,U+961c,U+962e,U+9631,U+9661,U+96a7,U+96cf,U+9704,U+9706,U+9716,U+9774,U+978d,U+97f6,U+9885,U+988a,U+9890,U+9893,U+98d9,U+996a,U+9a6d,U+9a6f,U+9a74,U+9a7c,U+9a7f,U+9a86-9a87,U+9a8f,U+9abc,U+9ae6,U+9e33,U+9e3d,U+9e4a,U+9e93,U+9eef,U+ff0b,U+ff0f,U+ffe5;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.97.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.97.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.97.otf')  format('opentype');unicode-range:U+0060,U+00e1,U+00e8,U+2015,U+2161,U+2464-2465,U+2500,U+25bc,U+25cf,U+2605,U+3008-3009,U+3014-3015,U+3044,U+304b-304d,U+304f,U+3057,U+305f,U+3068,U+307e-307f,U+3089-308b,U+30af,U+30b7,U+30e9-30eb,U+30f3,U+30fc,U+4e1e,U+4e4d,U+4ea2,U+4f36,U+4f3a,U+4f6c,U+4f84,U+4fae,U+4fde,U+5014,U+5018,U+5029-502a,U+5055,U+5140,U+516e,U+5180,U+5195,U+51a2,U+51cb,U+51db,U+51f3,U+5201,U+527f,U+533e,U+5364,U+536f,U+53a5,U+53e8-53e9,U+53ed,U+5480,U+5490,U+54aa,U+54d7,U+54e8,U+54ee,U+54fa,U+54fc,U+557c,U+5580,U+5583,U+55c5,U+55e3,U+55fd,U+560e,U+5618,U+563b,U+566c,U+5777,U+57a2-57a3,U+57ae,U+57d4,U+589f,U+58a9,U+58ec,U+592d,U+594e,U+5955,U+5993,U+59a9,U+59be,U+59ca,U+5a05,U+5a1f,U+5a34,U+5a6a,U+5a77,U+5a9b,U+5ab2,U+5ac2,U+5b09,U+5b5c,U+5b7d,U+5bc7,U+5bd0,U+5be5,U+5c09,U+5c27,U+5c4e,U+5c7f,U+5c9a,U+5cd9,U+5ce8,U+5ced,U+5dcd,U+5df3,U+5e37,U+5e62,U+5e87,U+5eb5-5eb6,U+5ed6,U+5f08,U+5f1b,U+5f6a,U+5f6c,U+5f8a,U+5f98-5f99,U+5fcf,U+6020,U+6055,U+6064,U+606c,U+60af,U+60bc,U+60eb-60ed,U+6177,U+618e,U+61a9,U+620a,U+620c,U+620e,U+6252,U+625b,U+627c,U+62c2,U+62c7,U+62e7,U+62ed,U+62f7,U+62fd,U+631f-6320,U+6346,U+6390,U+63b7,U+63ba,U+63ea,U+6479,U+6487,U+64d2,U+6590,U+6656,U+6666,U+667e,U+66a7,U+6714,U+6726,U+6789,U+67ff,U+6805,U+6808,U+6813,U+6853-6854,U+6893,U+68a2,U+68a7,U+68d8,U+68e0,U+68fa,U+6930,U+6960,U+69a8,U+69b4,U+6a0a,U+6a71,U+6a80,U+6aac,U+6b79,U+6b7c,U+6bcb,U+6bd9,U+6c13,U+6c2e-6c2f,U+6c40,U+6c72,U+6c76,U+6c79,U+6c81,U+6cbc,U+6cf5,U+6d95,U+6da1,U+6da4,U+6dc6-6dc7,U+6dcc,U+6de4,U+6e1d,U+6e3a,U+6eba,U+6fd2,U+701a,U+7076,U+7099,U+70ac,U+70c1,U+70ef,U+710a,U+7119,U+7172,U+717d,U+7184,U+725f,U+7280,U+72c4,U+72e1,U+72f8,U+73c0,U+73d1,U+7405,U+7409,U+7426,U+7436,U+745a,U+7480,U+74a7-74a8,U+7504,U+7578,U+759a,U+75a1,U+75b9,U+75d8,U+75f0,U+762b,U+7656,U+7678,U+76ce-76cf,U+7729,U+7738,U+773a,U+7766,U+7784,U+77aa,U+77b0,U+785d,U+78ca,U+7901,U+7934,U+7960,U+79a7,U+79b9,U+79e4,U+79fd,U+7a23,U+7a57,U+7a79,U+7a91,U+7a9c,U+7a9f,U+7aa6,U+7ae3,U+7aff,U+7b03,U+7b0b,U+7b77,U+7bc6,U+7be1,U+7bf7,U+7c07,U+7c3f,U+7c7d,U+7c9f,U+7caa,U+7cef,U+7d0a,U+7ec5,U+7ede,U+7eee-7ef0,U+7ef7,U+7f06,U+7f24,U+7f2a,U+7f2d,U+7f81,U+7fe9,U+803f,U+8046,U+804b,U+8087,U+8098,U+80b4,U+80da,U+80e7,U+80f1,U+80f3,U+80fa,U+814b-814c,U+8151,U+818a,U+81b3,U+8231,U+829c,U+82a5,U+82b8-82b9,U+82c7,U+82d4,U+82ef,U+8335,U+8339,U+835f,U+8367,U+839e,U+83e0,U+83f1,U+8469,U+846b,U+854a,U+85e9,U+85fb,U+8611,U+8638,U+864f,U+865e,U+8681,U+868a,U+8695,U+86db,U+8712,U+8715,U+8718,U+8747,U+8749,U+889c,U+88f3-88f4,U+8912,U+892a,U+8944,U+895f,U+8bcf,U+8bdb,U+8be3,U+8beb,U+8bf2,U+8c0d,U+8c0f,U+8c1b,U+8c2c,U+8c34,U+8d3f,U+8d4e,U+8d58,U+8d66,U+8db4,U+8dcb,U+8dfb,U+8e66,U+8eac,U+8f7c,U+8f84,U+8f97,U+8f99,U+8fe5,U+9005,U+9050,U+9082-9083,U+9091,U+90b1,U+90b5,U+90b9,U+9102,U+914c,U+916a,U+916e-916f,U+946b,U+949e,U+94a0,U+94a7,U+94b5,U+94be,U+94f2,U+950c,U+9523,U+952f,U+9540-9541,U+95f8,U+95fd,U+960e,U+9619,U+961c,U+962e,U+9631,U+9661,U+96a7,U+96cf,U+9704,U+9706,U+9716,U+9774,U+978d,U+97f6,U+9885,U+988a,U+9890,U+9893,U+98d9,U+996a,U+9a6d,U+9a6f,U+9a74,U+9a7c,U+9a7f,U+9a86-9a87,U+9a8f,U+9abc,U+9ae6,U+9e33,U+9e3d,U+9e4a,U+9e93,U+9eef,U+ff0b,U+ff0f,U+ffe5;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.97.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.97.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.97.otf')  format('opentype');unicode-range:U+0060,U+00e1,U+00e8,U+2015,U+2161,U+2464-2465,U+2500,U+25bc,U+25cf,U+2605,U+3008-3009,U+3014-3015,U+3044,U+304b-304d,U+304f,U+3057,U+305f,U+3068,U+307e-307f,U+3089-308b,U+30af,U+30b7,U+30e9-30eb,U+30f3,U+30fc,U+4e1e,U+4e4d,U+4ea2,U+4f36,U+4f3a,U+4f6c,U+4f84,U+4fae,U+4fde,U+5014,U+5018,U+5029-502a,U+5055,U+5140,U+516e,U+5180,U+5195,U+51a2,U+51cb,U+51db,U+51f3,U+5201,U+527f,U+533e,U+5364,U+536f,U+53a5,U+53e8-53e9,U+53ed,U+5480,U+5490,U+54aa,U+54d7,U+54e8,U+54ee,U+54fa,U+54fc,U+557c,U+5580,U+5583,U+55c5,U+55e3,U+55fd,U+560e,U+5618,U+563b,U+566c,U+5777,U+57a2-57a3,U+57ae,U+57d4,U+589f,U+58a9,U+58ec,U+592d,U+594e,U+5955,U+5993,U+59a9,U+59be,U+59ca,U+5a05,U+5a1f,U+5a34,U+5a6a,U+5a77,U+5a9b,U+5ab2,U+5ac2,U+5b09,U+5b5c,U+5b7d,U+5bc7,U+5bd0,U+5be5,U+5c09,U+5c27,U+5c4e,U+5c7f,U+5c9a,U+5cd9,U+5ce8,U+5ced,U+5dcd,U+5df3,U+5e37,U+5e62,U+5e87,U+5eb5-5eb6,U+5ed6,U+5f08,U+5f1b,U+5f6a,U+5f6c,U+5f8a,U+5f98-5f99,U+5fcf,U+6020,U+6055,U+6064,U+606c,U+60af,U+60bc,U+60eb-60ed,U+6177,U+618e,U+61a9,U+620a,U+620c,U+620e,U+6252,U+625b,U+627c,U+62c2,U+62c7,U+62e7,U+62ed,U+62f7,U+62fd,U+631f-6320,U+6346,U+6390,U+63b7,U+63ba,U+63ea,U+6479,U+6487,U+64d2,U+6590,U+6656,U+6666,U+667e,U+66a7,U+6714,U+6726,U+6789,U+67ff,U+6805,U+6808,U+6813,U+6853-6854,U+6893,U+68a2,U+68a7,U+68d8,U+68e0,U+68fa,U+6930,U+6960,U+69a8,U+69b4,U+6a0a,U+6a71,U+6a80,U+6aac,U+6b79,U+6b7c,U+6bcb,U+6bd9,U+6c13,U+6c2e-6c2f,U+6c40,U+6c72,U+6c76,U+6c79,U+6c81,U+6cbc,U+6cf5,U+6d95,U+6da1,U+6da4,U+6dc6-6dc7,U+6dcc,U+6de4,U+6e1d,U+6e3a,U+6eba,U+6fd2,U+701a,U+7076,U+7099,U+70ac,U+70c1,U+70ef,U+710a,U+7119,U+7172,U+717d,U+7184,U+725f,U+7280,U+72c4,U+72e1,U+72f8,U+73c0,U+73d1,U+7405,U+7409,U+7426,U+7436,U+745a,U+7480,U+74a7-74a8,U+7504,U+7578,U+759a,U+75a1,U+75b9,U+75d8,U+75f0,U+762b,U+7656,U+7678,U+76ce-76cf,U+7729,U+7738,U+773a,U+7766,U+7784,U+77aa,U+77b0,U+785d,U+78ca,U+7901,U+7934,U+7960,U+79a7,U+79b9,U+79e4,U+79fd,U+7a23,U+7a57,U+7a79,U+7a91,U+7a9c,U+7a9f,U+7aa6,U+7ae3,U+7aff,U+7b03,U+7b0b,U+7b77,U+7bc6,U+7be1,U+7bf7,U+7c07,U+7c3f,U+7c7d,U+7c9f,U+7caa,U+7cef,U+7d0a,U+7ec5,U+7ede,U+7eee-7ef0,U+7ef7,U+7f06,U+7f24,U+7f2a,U+7f2d,U+7f81,U+7fe9,U+803f,U+8046,U+804b,U+8087,U+8098,U+80b4,U+80da,U+80e7,U+80f1,U+80f3,U+80fa,U+814b-814c,U+8151,U+818a,U+81b3,U+8231,U+829c,U+82a5,U+82b8-82b9,U+82c7,U+82d4,U+82ef,U+8335,U+8339,U+835f,U+8367,U+839e,U+83e0,U+83f1,U+8469,U+846b,U+854a,U+85e9,U+85fb,U+8611,U+8638,U+864f,U+865e,U+8681,U+868a,U+8695,U+86db,U+8712,U+8715,U+8718,U+8747,U+8749,U+889c,U+88f3-88f4,U+8912,U+892a,U+8944,U+895f,U+8bcf,U+8bdb,U+8be3,U+8beb,U+8bf2,U+8c0d,U+8c0f,U+8c1b,U+8c2c,U+8c34,U+8d3f,U+8d4e,U+8d58,U+8d66,U+8db4,U+8dcb,U+8dfb,U+8e66,U+8eac,U+8f7c,U+8f84,U+8f97,U+8f99,U+8fe5,U+9005,U+9050,U+9082-9083,U+9091,U+90b1,U+90b5,U+90b9,U+9102,U+914c,U+916a,U+916e-916f,U+946b,U+949e,U+94a0,U+94a7,U+94b5,U+94be,U+94f2,U+950c,U+9523,U+952f,U+9540-9541,U+95f8,U+95fd,U+960e,U+9619,U+961c,U+962e,U+9631,U+9661,U+96a7,U+96cf,U+9704,U+9706,U+9716,U+9774,U+978d,U+97f6,U+9885,U+988a,U+9890,U+9893,U+98d9,U+996a,U+9a6d,U+9a6f,U+9a74,U+9a7c,U+9a7f,U+9a86-9a87,U+9a8f,U+9abc,U+9ae6,U+9e33,U+9e3d,U+9e4a,U+9e93,U+9eef,U+ff0b,U+ff0f,U+ffe5;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.98.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.98.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.98.otf')  format('opentype');unicode-range:U+2318,U+306d,U+4fe8,U+707d,U+7583-7585,U+7587-758d,U+7590,U+7592-7596,U+7598-7599,U+759b-75a0,U+75a2-75aa,U+75ac-75ad,U+75b3-75b8,U+75ba-75bb,U+75bf-75c4,U+75c8-75d1,U+75d3-75d4,U+75d6-75d7,U+75d9-75da,U+75dc-75ef,U+75f1-75f3,U+75f5-75ff,U+7603-7616,U+7618-7623,U+7625,U+7627-762a,U+762c-763d,U+763f-764b,U+764d-7655,U+7657-7677,U+7679-767a,U+767f-7681,U+7683,U+7685,U+7688-76ad,U+76af-76b0,U+76b2-76c5,U+76c7,U+76c9,U+76cb-76cd,U+76d9-76da,U+76dc-76de,U+76e0-76ed,U+76f0-76f1,U+76f3,U+76f5-76f7,U+76f9-76fb,U+76fd,U+76ff-7700,U+7702-7708,U+770a,U+770c-771e,U+7721-7728,U+772a-7736,U+7739,U+773b,U+773d-773f,U+7742-774f,U+7751-775a,U+956c-956f,U+9594-9597,U+9651-9654,U+9656-965b,U+9663,U+9979,U+20ad3,U+22c88,U+2b1ed;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.98.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.98.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.98.otf')  format('opentype');unicode-range:U+2318,U+306d,U+4fe8,U+707d,U+7583-7585,U+7587-758d,U+7590,U+7592-7596,U+7598-7599,U+759b-75a0,U+75a2-75aa,U+75ac-75ad,U+75b3-75b8,U+75ba-75bb,U+75bf-75c4,U+75c8-75d1,U+75d3-75d4,U+75d6-75d7,U+75d9-75da,U+75dc-75ef,U+75f1-75f3,U+75f5-75ff,U+7603-7616,U+7618-7623,U+7625,U+7627-762a,U+762c-763d,U+763f-764b,U+764d-7655,U+7657-7677,U+7679-767a,U+767f-7681,U+7683,U+7685,U+7688-76ad,U+76af-76b0,U+76b2-76c5,U+76c7,U+76c9,U+76cb-76cd,U+76d9-76da,U+76dc-76de,U+76e0-76ed,U+76f0-76f1,U+76f3,U+76f5-76f7,U+76f9-76fb,U+76fd,U+76ff-7700,U+7702-7708,U+770a,U+770c-771e,U+7721-7728,U+772a-7736,U+7739,U+773b,U+773d-773f,U+7742-774f,U+7751-775a,U+956c-956f,U+9594-9597,U+9651-9654,U+9656-965b,U+9663,U+9979,U+20ad3,U+22c88,U+2b1ed;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.98.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.98.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.98.otf')  format('opentype');unicode-range:U+2318,U+306d,U+4fe8,U+707d,U+7583-7585,U+7587-758d,U+7590,U+7592-7596,U+7598-7599,U+759b-75a0,U+75a2-75aa,U+75ac-75ad,U+75b3-75b8,U+75ba-75bb,U+75bf-75c4,U+75c8-75d1,U+75d3-75d4,U+75d6-75d7,U+75d9-75da,U+75dc-75ef,U+75f1-75f3,U+75f5-75ff,U+7603-7616,U+7618-7623,U+7625,U+7627-762a,U+762c-763d,U+763f-764b,U+764d-7655,U+7657-7677,U+7679-767a,U+767f-7681,U+7683,U+7685,U+7688-76ad,U+76af-76b0,U+76b2-76c5,U+76c7,U+76c9,U+76cb-76cd,U+76d9-76da,U+76dc-76de,U+76e0-76ed,U+76f0-76f1,U+76f3,U+76f5-76f7,U+76f9-76fb,U+76fd,U+76ff-7700,U+7702-7708,U+770a,U+770c-771e,U+7721-7728,U+772a-7736,U+7739,U+773b,U+773d-773f,U+7742-774f,U+7751-775a,U+956c-956f,U+9594-9597,U+9651-9654,U+9656-965b,U+9663,U+9979,U+20ad3,U+22c88,U+2b1ed;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.99.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.99.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.99.otf')  format('opentype');unicode-range:U+5233,U+56f5,U+5c50,U+63a1,U+6c9a,U+73b3-73ba,U+73bc-73bf,U+73c1-73c7,U+73cb-73cc,U+73ce-73d0,U+73d2-73df,U+73e1-73ec,U+73ee-73fd,U+73ff-7402,U+7404,U+7407-7408,U+740a-740f,U+7411-7421,U+7423-7425,U+7427-7429,U+742b-7432,U+7435,U+7437-743b,U+743d-7454,U+7456-7459,U+745b,U+745d,U+7460-746f,U+7471-7475,U+7477-747f,U+7484-74a6,U+74a9-74db,U+74dd-74e2,U+74e4-74e5,U+74e7-74f5,U+74f8-7503,U+7505-7517,U+7519,U+751d-751e,U+7520-7527,U+752c-752f,U+7534,U+7536,U+7539-753a,U+753c-7544,U+7546-754b,U+754d-754e,U+7550-7553,U+7555-7558,U+755d-7564,U+7566-7569,U+756b-7572,U+7579-7582,U+8278-8279,U+881b-881c,U+90c5,U+95d9-95db,U+95dd-95e4,U+95eb-95ec,U+9a7d,U+9ca3,U+9cb5;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.99.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.99.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.99.otf')  format('opentype');unicode-range:U+5233,U+56f5,U+5c50,U+63a1,U+6c9a,U+73b3-73ba,U+73bc-73bf,U+73c1-73c7,U+73cb-73cc,U+73ce-73d0,U+73d2-73df,U+73e1-73ec,U+73ee-73fd,U+73ff-7402,U+7404,U+7407-7408,U+740a-740f,U+7411-7421,U+7423-7425,U+7427-7429,U+742b-7432,U+7435,U+7437-743b,U+743d-7454,U+7456-7459,U+745b,U+745d,U+7460-746f,U+7471-7475,U+7477-747f,U+7484-74a6,U+74a9-74db,U+74dd-74e2,U+74e4-74e5,U+74e7-74f5,U+74f8-7503,U+7505-7517,U+7519,U+751d-751e,U+7520-7527,U+752c-752f,U+7534,U+7536,U+7539-753a,U+753c-7544,U+7546-754b,U+754d-754e,U+7550-7553,U+7555-7558,U+755d-7564,U+7566-7569,U+756b-7572,U+7579-7582,U+8278-8279,U+881b-881c,U+90c5,U+95d9-95db,U+95dd-95e4,U+95eb-95ec,U+9a7d,U+9ca3,U+9cb5;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.99.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.99.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.99.otf')  format('opentype');unicode-range:U+5233,U+56f5,U+5c50,U+63a1,U+6c9a,U+73b3-73ba,U+73bc-73bf,U+73c1-73c7,U+73cb-73cc,U+73ce-73d0,U+73d2-73df,U+73e1-73ec,U+73ee-73fd,U+73ff-7402,U+7404,U+7407-7408,U+740a-740f,U+7411-7421,U+7423-7425,U+7427-7429,U+742b-7432,U+7435,U+7437-743b,U+743d-7454,U+7456-7459,U+745b,U+745d,U+7460-746f,U+7471-7475,U+7477-747f,U+7484-74a6,U+74a9-74db,U+74dd-74e2,U+74e4-74e5,U+74e7-74f5,U+74f8-7503,U+7505-7517,U+7519,U+751d-751e,U+7520-7527,U+752c-752f,U+7534,U+7536,U+7539-753a,U+753c-7544,U+7546-754b,U+754d-754e,U+7550-7553,U+7555-7558,U+755d-7564,U+7566-7569,U+756b-7572,U+7579-7582,U+8278-8279,U+881b-881c,U+90c5,U+95d9-95db,U+95dd-95e4,U+95eb-95ec,U+9a7d,U+9ca3,U+9cb5;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.100.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.100.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.100.otf')  format('opentype');unicode-range:U+0021-0023,U+0025-005b,U+005d,U+005f,U+0061-007a,U+007e-00a0,U+00b7,U+2014,U+2018-2019,U+201c-201d,U+2026,U+3000-3002,U+300a-300d,U+3010-3011,U+4e00-4e01,U+4e03,U+4e07-4e0b,U+4e0d-4e0e,U+4e11,U+4e13-4e14,U+4e16,U+4e1a-4e1d,U+4e22,U+4e24-4e25,U+4e27,U+4e2a,U+4e2d,U+4e30,U+4e32,U+4e34,U+4e39-4e3b,U+4e3d-4e3e,U+4e43,U+4e45,U+4e48-4e49,U+4e4b-4e4c,U+4e4e-4e50,U+4e54,U+4e58,U+4e5d,U+4e5f-4e61,U+4e66,U+4e70-4e71,U+4e73,U+4e86,U+4e88-4e89,U+4e8b-4e8c,U+4e8e-4e8f,U+4e91-4e92,U+4e94-4e95,U+4e9a-4e9b,U+4ea1,U+4ea4,U+4ea6-4ea7,U+4eab-4eac,U+4eae,U+4eb2,U+4eba,U+4ebf-4ec1,U+4ec5,U+4eca-4ecb,U+4ecd-4ece,U+4ed3-4ed4,U+4ed6,U+4ed8-4ed9,U+4ee3-4ee5,U+4eea,U+4eec,U+4ef0,U+4ef6-4ef7,U+4efb,U+4efd,U+4eff,U+4f01,U+4f0a,U+4f0d,U+4f0f,U+4f11,U+4f17-4f1a,U+4f1f-4f20,U+4f24,U+4f26,U+4f2a,U+4f2f-4f30,U+4f34,U+4f38,U+4f3c,U+4f46,U+4f4d-4f4f,U+4f53,U+4f55,U+4f59,U+4f5b-4f5c,U+4f60,U+4f69,U+4f73,U+4f7f,U+4f8b,U+4f9b,U+4f9d,U+4fa7,U+4fb5,U+4fbf,U+4fc3-4fc4,U+4fca,U+4fd7,U+4fdd,U+4fe1,U+4fe9,U+4fee,U+4ff1,U+500d,U+5012,U+5019,U+501f,U+503c,U+503e,U+5047,U+504f,U+505a,U+505c,U+5065,U+5076-5077,U+507f,U+50a8,U+50b2,U+50bb,U+50cf,U+513f,U+5141,U+5143-5145,U+5148-5149,U+514b,U+514d,U+515a,U+5165,U+5168,U+516b-516d,U+5170-5171,U+5173-5178,U+517b-517d,U+5185,U+518c-518d,U+5192,U+5199,U+519b-519c,U+51a0,U+51ac,U+51b0,U+51b2-51b3,U+51b5,U+51b7,U+51c0,U+51c6,U+51c9,U+51cc,U+51cf,U+51dd,U+51e0-51e1,U+51e4,U+51ed,U+51ef,U+51f6,U+51fa-51fb,U+5200,U+5206-5207,U+520a,U+5211-5212,U+5217-521b,U+521d,U+5220,U+5224,U+5229,U+522b,U+5230,U+5236-5237,U+523a-523b,U+5242,U+524d,U+5251,U+5267,U+5269-526a,U+526f,U+5272,U+529b,U+529d-52a1,U+52a3,U+52a8-52aa,U+52b1-52b3,U+52bf,U+52c7,U+52d2,U+52e4,U+52fe,U+5300,U+5305,U+5316-5317,U+5339-533b,U+5341,U+5343,U+5347-5348,U+534a,U+534e-534f,U+5353,U+5355-5357,U+535a,U+5360-5361,U+5367,U+536b,U+5370-5371,U+5373-5374,U+5377,U+5382,U+5385-5386,U+5389,U+538b-538c,U+5398,U+539a,U+539f,U+53bb,U+53bf,U+53c2,U+53c8,U+53ca-53cd,U+53d1,U+53d6-53d8,U+53e0,U+53e3-53e6,U+53ea-53ec,U+53ef-53f0,U+53f2-53f3,U+53f6-53f9,U+5403-5404,U+5408-5409,U+540c-540e,U+5410-5411,U+5413,U+5417,U+541b,U+5426-5427,U+542b-542c,U+542f,U+5434,U+5438-5439,U+5440,U+5446,U+5448,U+544a,U+5458,U+5462,U+5468,U+5473,U+5475,U+547c-547d,U+548c,U+5496,U+54a8,U+54c1,U+54c8,U+54cd,U+54e5-54e6,U+54ea,U+54ed,U+54f2,U+5510,U+552e-552f,U+5531,U+5546,U+554a,U+5565-5566,U+5584,U+558a,U+559c-559d,U+55b7,U+5609,U+561b,U+5634,U+5668,U+56db,U+56de,U+56e0,U+56e2,U+56ed,U+56f0,U+56f4,U+56fa,U+56fd-56fe,U+5706,U+5708,U+571f,U+5723,U+5728,U+5730,U+573a,U+5740,U+5747,U+574f-5751,U+5757,U+575a-575b,U+5761,U+5766,U+5782,U+578b,U+57c3,U+57cb,U+57ce,U+57df,U+57f9-57fa,U+5802,U+5806,U+5821,U+582a,U+5851,U+5854,U+585e,U+586b,U+5883,U+5899,U+589e,U+58a8,U+58c1,U+58eb,U+58ee,U+58f0,U+58f3,U+5904,U+5907,U+590d,U+590f,U+5915-5916,U+591a,U+591c,U+591f,U+5927,U+5929-592b,U+592e,U+5931,U+5934,U+5938-593a,U+5947-5949,U+594b,U+594f,U+5954,U+5956-5957,U+5965,U+5973,U+5976,U+5979,U+597d,U+5982,U+5986-5988,U+5999,U+59b9,U+59bb,U+59c6,U+59cb,U+59d0-59d1,U+59d3-59d4,U+59dc,U+59fb,U+59ff,U+5a01,U+5a18,U+5a31,U+5a46,U+5a5a,U+5a92,U+5ac1,U+5acc,U+5ae9,U+5b50,U+5b54-5b55,U+5b57-5b59,U+5b63-5b64,U+5b66,U+5b69,U+5b81,U+5b83,U+5b85,U+5b87-5b89,U+5b8b-5b8c,U+5b8f,U+5b97-5b98,U+5b9a,U+5b9c-5b9e,U+5ba1-5ba4,U+5bab,U+5bb3,U+5bb6,U+5bb9,U+5bbd-5bbf,U+5bc2,U+5bc4,U+5bc6,U+5bcc,U+5bd2-5bd3,U+5bdf,U+5bf8-5bfc,U+5bff,U+5c01,U+5c04,U+5c06,U+5c0a,U+5c0f,U+5c11,U+5c14,U+5c16,U+5c18,U+5c1a,U+5c1d,U+5c24,U+5c31,U+5c3a,U+5c3c-5c40,U+5c42,U+5c45,U+5c48,U+5c4a-5c4b,U+5c4f,U+5c55,U+5c5e,U+5c71,U+5c81,U+5c97,U+5c9b,U+5ca9,U+5cb8,U+5cf0,U+5d07,U+5ddd-5dde,U+5de1,U+5de5-5de8,U+5dee,U+5df1-5df2,U+5df4,U+5e01-5e03,U+5e05,U+5e08,U+5e0c,U+5e1d,U+5e26,U+5e2d-5e2e,U+5e38,U+5e45,U+5e55,U+5e72-5e74,U+5e76,U+5e78,U+5e7b-5e7d,U+5e7f,U+5e84,U+5e86,U+5e8a,U+5e8f,U+5e93-5e95,U+5e97,U+5e99,U+5e9c,U+5e9f,U+5ea6-5ea7,U+5ead,U+5eb7,U+5ec9,U+5ef6-5ef7,U+5efa,U+5f00,U+5f02-5f04,U+5f0f,U+5f15,U+5f1f-5f20,U+5f25,U+5f2f,U+5f31,U+5f39-5f3a,U+5f52-5f53,U+5f55,U+5f62,U+5f69,U+5f71,U+5f79,U+5f7b-5f7c,U+5f80-5f81,U+5f84-5f85,U+5f88,U+5f8b,U+5f90,U+5f92,U+5f97,U+5fa1,U+5faa,U+5fae,U+5fb7,U+5fbd,U+5fc3,U+5fc5-5fc6,U+5fcc-5fcd,U+5fd7-5fd9,U+5fe0,U+5fe7,U+5feb,U+5ff5,U+5ffd,U+6000-6001,U+600e,U+6012,U+6015,U+601d,U+6025,U+6027-6028,U+602a,U+603b,U+604b,U+6050,U+6052,U+6062,U+6068-6069,U+606f-6070,U+6076,U+607c,U+6089,U+6094,U+609f-60a0,U+60a3,U+60a6,U+60a8,U+60ac,U+60b2,U+60c5,U+60ca,U+60d1,U+60dc,U+60e0,U+60e7-60e8,U+60ef,U+60f3,U+6108,U+610f,U+611f,U+6124,U+613f,U+6148,U+614e,U+6155,U+6162,U+6167,U+6170,U+61c2,U+61d2,U+620f-6212,U+6216,U+6218,U+622a,U+6234,U+6237,U+623f-6240,U+624b,U+624d-624e,U+6253,U+6258,U+6263,U+6267,U+6269,U+626b-626c,U+626e,U+6270,U+6276,U+6279,U+627e-6280,U+628a,U+6291,U+6293,U+6295,U+6297-6298,U+629b,U+62a2,U+62a4-62a5,U+62ab,U+62b1,U+62b5,U+62b9,U+62bd,U+62c5-62c6,U+62c9,U+62cd,U+62d2-62d4,U+62d6,U+62db-62dc,U+62df,U+62e5,U+62e8-62e9,U+62ec,U+62fc,U+62ff,U+6301-6302,U+6307,U+6309,U+6311,U+6316,U+6321,U+6324-6325,U+632f,U+633a,U+6355,U+635f,U+6362,U+636e,U+6377,U+6388-6389,U+638c,U+6392,U+63a2,U+63a5,U+63a7-63aa,U+63cf-63d0,U+63d2,U+63e1,U+63ed,U+63f4,U+641c,U+641e,U+642c-642d,U+643a,U+6444,U+6446-6447,U+6458,U+6469,U+6478,U+6491-6492,U+649e,U+64ad,U+64c5,U+64cd,U+64e6,U+652f,U+6536,U+6539,U+653b,U+653e-653f,U+6545,U+6548,U+654c,U+654f,U+6551,U+6559,U+6562-6563,U+656c,U+6570,U+6574,U+6587,U+6591,U+6597,U+6599,U+659c,U+65a4,U+65ad,U+65af-65b0,U+65b9,U+65bd,U+65c1,U+65c5,U+65cb,U+65cf,U+65d7,U+65e0,U+65e2,U+65e5-65e9,U+65f6,U+65fa,U+6602,U+660c,U+660e,U+6613,U+661f-6620,U+6625,U+6628,U+662f,U+663e,U+664b,U+6653,U+665a,U+6668,U+666e-6670,U+6676,U+667a,U+6682,U+6696-6697,U+66b4,U+66f0,U+66f2,U+66f4,U+66fc,U+66fe-6700,U+6708-6709,U+670b,U+670d,U+6717,U+671b,U+671d,U+671f,U+6728,U+672a-672c,U+672f,U+6731,U+6734-6735,U+673a,U+6740,U+6742-6743,U+6746,U+674e,U+6750-6751,U+675c,U+675f,U+6761,U+6765,U+6768,U+676f-6770,U+677e-677f,U+6781,U+6784,U+6790,U+6797,U+679c-679d,U+67aa,U+67b6,U+67d0,U+67d3-67d4,U+67e5,U+67f1,U+67f3,U+6807,U+680f,U+6811,U+6821,U+6837-6839,U+683c,U+6843,U+6846,U+6848,U+684c,U+6851,U+6863,U+6865,U+6881,U+6885,U+68a6,U+68af-68b0,U+68c0,U+68d2,U+68ee,U+690d,U+695a,U+697c,U+6982,U+699c,U+69fd,U+6a21,U+6a2a,U+6b21-6b23,U+6b27,U+6b32,U+6b3a,U+6b3e,U+6b4c,U+6b62-6b66,U+6b7b,U+6b8a-6b8b,U+6b96,U+6bb5,U+6bbf,U+6bc1,U+6bcd,U+6bcf,U+6bd2,U+6bd4-6bd5,U+6bdb,U+6beb,U+6c0f,U+6c11,U+6c14,U+6c1b,U+6c27,U+6c34,U+6c38,U+6c41-6c42,U+6c47,U+6c49,U+6c57,U+6c5f-6c61,U+6c64,U+6c7d,U+6c88-6c89,U+6c99,U+6c9f,U+6ca1,U+6cb3,U+6cb9,U+6cbb,U+6cbf,U+6cc4,U+6cc9,U+6cd5,U+6cdb,U+6ce1-6ce2,U+6ce5,U+6ce8,U+6cea,U+6cf0,U+6cfd,U+6d01,U+6d0b,U+6d12,U+6d17,U+6d1b,U+6d1e,U+6d25,U+6d2a,U+6d32,U+6d3b,U+6d3e,U+6d41,U+6d45,U+6d4b,U+6d4e,U+6d53,U+6d59,U+6d69-6d6a,U+6d6e,U+6d77-6d78,U+6d82,U+6d88-6d89,U+6d8c,U+6da6,U+6da8,U+6daf,U+6db2,U+6db5,U+6dcb,U+6dd8,U+6de1,U+6df1,U+6df7,U+6dfb,U+6e05,U+6e10,U+6e20-6e21,U+6e29,U+6e2f,U+6e34,U+6e38,U+6e56,U+6e7e-6e7f,U+6e90,U+6eaa,U+6ecb,U+6ed1,U+6eda,U+6ee1,U+6ef4,U+6f02,U+6f0f,U+6f14,U+6f2b,U+6f5c,U+6f6e,U+6fb3,U+6fc0,U+704c,U+706b,U+706d,U+706f-7070,U+7075,U+707e,U+7089,U+708e,U+7092,U+70ae,U+70b8-70b9,U+70bc,U+70c2,U+70c8,U+70df,U+70e6-70e7,U+70ed,U+7126,U+7136,U+7167,U+716e,U+718a,U+719f,U+71c3,U+71d5,U+71e5,U+7206,U+722c,U+7231,U+7236-7238,U+723d,U+7247-7248,U+724c,U+7259,U+725b,U+7262,U+7269,U+7275,U+7279,U+72af,U+72b6,U+72b9,U+72c2,U+72d7,U+72ec,U+72f1,U+731b-731c,U+732a-732b,U+732e,U+7384,U+7387,U+7389,U+738b,U+739b,U+73a9,U+73af-73b0,U+73bb,U+73cd,U+73e0,U+73ed,U+7403,U+7406,U+7434,U+745e,U+7483,U+74dc,U+74e6,U+74f6,U+7518,U+751a,U+751c,U+751f,U+7528,U+7530-7533,U+7535,U+7537,U+753b,U+7545,U+754c,U+7559,U+7565,U+756a,U+758f,U+7591,U+7597,U+75af,U+75b2,U+75bc,U+75be,U+75c5,U+75c7,U+75d5,U+75db,U+7626,U+767b,U+767d-767e,U+7684,U+7686-7687,U+76ae,U+76c6,U+76c8,U+76ca,U+76d0-76d2,U+76d6-76d8,U+76db,U+76df,U+76ee,U+76f4,U+76f8,U+76fe,U+7701,U+7709,U+770b,U+771f-7720,U+773c,U+7740,U+775b,U+7761,U+7763,U+77ac,U+77db,U+77e5,U+77ed,U+77f3,U+77ff,U+7801,U+7814,U+7834,U+7840,U+7855,U+786c,U+786e,U+788d-788e,U+7891,U+7897,U+78b0,U+78e8,U+793a,U+793c,U+793e,U+7956,U+795d-795e,U+7965,U+7968,U+7981,U+798f,U+79bb,U+79c0-79c1,U+79cb,U+79cd,U+79d1-79d2,U+79d8,U+79df,U+79e6,U+79ef-79f0,U+79fb,U+7a00,U+7a0b,U+7a0d-7a0e,U+7a33,U+7a3f,U+7a76-7a77,U+7a7a,U+7a7f,U+7a81,U+7a97,U+7acb,U+7ad9,U+7ade-7ae0,U+7ae5,U+7aef,U+7af9,U+7b11,U+7b14,U+7b26,U+7b2c,U+7b49,U+7b51,U+7b54,U+7b56,U+7b79,U+7b7e,U+7b80,U+7b97,U+7ba1,U+7bb1,U+7bc7,U+7c4d,U+7c73,U+7c7b,U+7c89,U+7c92,U+7c97-7c98,U+7cae,U+7cbe,U+7cca,U+7cd6,U+7cfb,U+7d20,U+7d22,U+7d27,U+7d2b,U+7d2f,U+7e41,U+7ea0,U+7ea2,U+7ea4,U+7ea6-7ea7,U+7eaa,U+7eaf,U+7eb3,U+7eb5,U+7eb7-7eb9,U+7ebd,U+7ebf,U+7ec3-7ec4,U+7ec6-7ec8,U+7ecd,U+7ecf,U+7ed3,U+7ed5,U+7ed8-7ed9,U+7edc-7edd,U+7edf,U+7ee7,U+7ee9-7eea,U+7eed,U+7ef4-7ef5,U+7efc,U+7eff,U+7f13,U+7f16,U+7f18,U+7f1d,U+7f29,U+7f3a,U+7f51,U+7f57,U+7f5a,U+7f62,U+7f6a,U+7f6e,U+7f72,U+7f8a,U+7f8e,U+7fa4,U+7fbd,U+7ffb-7ffc,U+8000-8001,U+8003,U+8005,U+800c,U+8010,U+8017,U+8033,U+804a,U+804c,U+8054,U+8058,U+805a,U+806a,U+8083,U+8089,U+808c,U+809a,U+80a0-80a1,U+80a4-80a5,U+80a9,U+80af,U+80b2,U+80bf,U+80c3,U+80c6,U+80cc,U+80ce,U+80dc,U+80de,U+80e1,U+80f6,U+80f8,U+80fd,U+8102,U+8106,U+8109,U+810f,U+8111,U+811a,U+8131,U+8138,U+8150,U+8170,U+8179,U+817b,U+817e-817f,U+819c,U+81e3,U+81ea,U+81f3-81f4,U+820d,U+8212,U+821e,U+822a,U+822c,U+8239,U+826f-8270,U+8272-8273,U+827a,U+827e,U+8282,U+82b1,U+82b3,U+82cd,U+82cf,U+82e5-82e6,U+82f1,U+8303,U+8336,U+8349,U+8350,U+8352,U+8361,U+8363,U+836f,U+8377,U+83ab,U+83b1-83b2,U+83b7,U+83dc,U+83f2,U+8425,U+8428,U+843d,U+8457,U+8463,U+8499,U+84b8,U+84dd,U+8584,U+85cf,U+864e,U+8651,U+865a,U+866b,U+867d,U+86cb,U+86ee,U+871c,U+878d,U+8840,U+884c,U+8857,U+8861,U+8863,U+8865,U+8868,U+8870,U+888b,U+8896,U+88ab,U+88ad,U+88c1-88c2,U+88c5,U+897f,U+8981,U+8986,U+89c1-89c2,U+89c4,U+89c6,U+89c8-89c9,U+89d2,U+89e3,U+89e6,U+8a00,U+8a89,U+8b66,U+8ba1-8ba2,U+8ba4,U+8ba8-8ba9,U+8bad-8bb0,U+8bb2,U+8bb8,U+8bba,U+8bbe-8bbf,U+8bc1,U+8bc4,U+8bc6,U+8bc9-8bca,U+8bcd,U+8bd1,U+8bd5,U+8bd7,U+8bda,U+8bdd-8bde,U+8be2,U+8be5-8be6,U+8bed,U+8bef,U+8bf1,U+8bf4,U+8bf7-8bf8,U+8bfa-8bfb,U+8bfe,U+8c01,U+8c03,U+8c08,U+8c0b,U+8c10,U+8c13,U+8c22,U+8c28,U+8c31,U+8c37,U+8c46,U+8c61,U+8c6a,U+8c8c,U+8d1d,U+8d1f,U+8d21-8d28,U+8d2a-8d2b,U+8d2d,U+8d2f,U+8d34-8d35,U+8d38-8d39,U+8d44,U+8d4b,U+8d4f,U+8d56,U+8d5a-8d5b,U+8d5e,U+8d60,U+8d62,U+8d64,U+8d6b,U+8d70,U+8d74-8d77,U+8d85,U+8d8a-8d8b,U+8da3,U+8db3,U+8dc3,U+8dcc,U+8dd1,U+8ddd,U+8ddf,U+8de8,U+8def,U+8df3,U+8df5,U+8e0f,U+8e2a,U+8eab,U+8f66,U+8f68,U+8f6c,U+8f6e-8f6f,U+8f7b,U+8f7d,U+8f83,U+8f85-8f86,U+8f88-8f89,U+8f91,U+8f93,U+8f9b,U+8f9e,U+8fa3,U+8fa8-8fa9,U+8fb9,U+8fbe,U+8fc1,U+8fc5,U+8fc7-8fc8,U+8fce,U+8fd0-8fd1,U+8fd4,U+8fd8-8fd9,U+8fdb-8fdf,U+8fea-8feb,U+8ff0,U+8ff7,U+8ff9,U+8ffd,U+9000-9003,U+9006,U+9009,U+900f-9010,U+9012,U+9014,U+901a,U+901d,U+901f-9020,U+9022,U+903b-903c,U+9047,U+904d,U+9053,U+9057,U+9065,U+906d,U+9075,U+907f-9080,U+90a3,U+90a6,U+90aa,U+90ae,U+90bb,U+90c1,U+90ce,U+90d1,U+90e8,U+90ed,U+90fd,U+914d,U+9152,U+9177-9178,U+9189,U+9192,U+91c7,U+91ca,U+91cc-91cf,U+91d1,U+9274,U+9488,U+949f,U+94a2,U+94b1,U+94bb,U+94c1,U+94dc,U+94f6,U+94fa,U+94fe,U+9500-9501,U+9505,U+950b,U+9510,U+9519,U+9526,U+952e,U+9547,U+955c,U+957f,U+95e8,U+95ea,U+95ed-95ee,U+95f2,U+95f4,U+95f7,U+95f9,U+95fb,U+9605,U+9614,U+961f,U+9632-9636,U+963b,U+963f,U+9644-9646,U+9648,U+964d,U+9650,U+9662,U+9664,U+9669-966a,U+9675-9677,U+9686,U+968f-9690,U+9694,U+969c,U+96be,U+96c4-96c6,U+96d5,U+96e8,U+96ea,U+96f6-96f7,U+96fe,U+9700,U+9707,U+9732,U+9738,U+9752,U+9759,U+975e,U+9760,U+9762,U+9769,U+978b,U+97e9,U+97f3,U+97f5,U+9875-9876,U+9879-987b,U+987e-987f,U+9884,U+9886-9887,U+9891,U+9897-9898,U+989c-989d,U+98ce,U+98d8,U+98de-98df,U+9910,U+996d-996e,U+9970-9971,U+9986,U+9996,U+9999,U+9a6c,U+9a71,U+9a7b,U+9a7e,U+9a82,U+9a8c,U+9a91,U+9a97,U+9aa4,U+9aa8,U+9ad8,U+9b3c,U+9b42,U+9b45,U+9b54,U+9c7c,U+9c81,U+9c9c,U+9e1f,U+9e21,U+9e23,U+9ea6,U+9ebb,U+9ec4,U+9ece,U+9ed1,U+9ed8,U+9f13,U+9f20,U+9f3b,U+9f50,U+9f7f,U+9f84,U+9f99,U+ff01,U+ff08-ff09,U+ff0c-ff0e,U+ff1a-ff1b,U+ff1f,U+ff5e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.100.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.100.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.100.otf')  format('opentype');unicode-range:U+0021-0023,U+0025-005b,U+005d,U+005f,U+0061-007a,U+007e-00a0,U+00b7,U+2014,U+2018-2019,U+201c-201d,U+2026,U+3000-3002,U+300a-300d,U+3010-3011,U+4e00-4e01,U+4e03,U+4e07-4e0b,U+4e0d-4e0e,U+4e11,U+4e13-4e14,U+4e16,U+4e1a-4e1d,U+4e22,U+4e24-4e25,U+4e27,U+4e2a,U+4e2d,U+4e30,U+4e32,U+4e34,U+4e39-4e3b,U+4e3d-4e3e,U+4e43,U+4e45,U+4e48-4e49,U+4e4b-4e4c,U+4e4e-4e50,U+4e54,U+4e58,U+4e5d,U+4e5f-4e61,U+4e66,U+4e70-4e71,U+4e73,U+4e86,U+4e88-4e89,U+4e8b-4e8c,U+4e8e-4e8f,U+4e91-4e92,U+4e94-4e95,U+4e9a-4e9b,U+4ea1,U+4ea4,U+4ea6-4ea7,U+4eab-4eac,U+4eae,U+4eb2,U+4eba,U+4ebf-4ec1,U+4ec5,U+4eca-4ecb,U+4ecd-4ece,U+4ed3-4ed4,U+4ed6,U+4ed8-4ed9,U+4ee3-4ee5,U+4eea,U+4eec,U+4ef0,U+4ef6-4ef7,U+4efb,U+4efd,U+4eff,U+4f01,U+4f0a,U+4f0d,U+4f0f,U+4f11,U+4f17-4f1a,U+4f1f-4f20,U+4f24,U+4f26,U+4f2a,U+4f2f-4f30,U+4f34,U+4f38,U+4f3c,U+4f46,U+4f4d-4f4f,U+4f53,U+4f55,U+4f59,U+4f5b-4f5c,U+4f60,U+4f69,U+4f73,U+4f7f,U+4f8b,U+4f9b,U+4f9d,U+4fa7,U+4fb5,U+4fbf,U+4fc3-4fc4,U+4fca,U+4fd7,U+4fdd,U+4fe1,U+4fe9,U+4fee,U+4ff1,U+500d,U+5012,U+5019,U+501f,U+503c,U+503e,U+5047,U+504f,U+505a,U+505c,U+5065,U+5076-5077,U+507f,U+50a8,U+50b2,U+50bb,U+50cf,U+513f,U+5141,U+5143-5145,U+5148-5149,U+514b,U+514d,U+515a,U+5165,U+5168,U+516b-516d,U+5170-5171,U+5173-5178,U+517b-517d,U+5185,U+518c-518d,U+5192,U+5199,U+519b-519c,U+51a0,U+51ac,U+51b0,U+51b2-51b3,U+51b5,U+51b7,U+51c0,U+51c6,U+51c9,U+51cc,U+51cf,U+51dd,U+51e0-51e1,U+51e4,U+51ed,U+51ef,U+51f6,U+51fa-51fb,U+5200,U+5206-5207,U+520a,U+5211-5212,U+5217-521b,U+521d,U+5220,U+5224,U+5229,U+522b,U+5230,U+5236-5237,U+523a-523b,U+5242,U+524d,U+5251,U+5267,U+5269-526a,U+526f,U+5272,U+529b,U+529d-52a1,U+52a3,U+52a8-52aa,U+52b1-52b3,U+52bf,U+52c7,U+52d2,U+52e4,U+52fe,U+5300,U+5305,U+5316-5317,U+5339-533b,U+5341,U+5343,U+5347-5348,U+534a,U+534e-534f,U+5353,U+5355-5357,U+535a,U+5360-5361,U+5367,U+536b,U+5370-5371,U+5373-5374,U+5377,U+5382,U+5385-5386,U+5389,U+538b-538c,U+5398,U+539a,U+539f,U+53bb,U+53bf,U+53c2,U+53c8,U+53ca-53cd,U+53d1,U+53d6-53d8,U+53e0,U+53e3-53e6,U+53ea-53ec,U+53ef-53f0,U+53f2-53f3,U+53f6-53f9,U+5403-5404,U+5408-5409,U+540c-540e,U+5410-5411,U+5413,U+5417,U+541b,U+5426-5427,U+542b-542c,U+542f,U+5434,U+5438-5439,U+5440,U+5446,U+5448,U+544a,U+5458,U+5462,U+5468,U+5473,U+5475,U+547c-547d,U+548c,U+5496,U+54a8,U+54c1,U+54c8,U+54cd,U+54e5-54e6,U+54ea,U+54ed,U+54f2,U+5510,U+552e-552f,U+5531,U+5546,U+554a,U+5565-5566,U+5584,U+558a,U+559c-559d,U+55b7,U+5609,U+561b,U+5634,U+5668,U+56db,U+56de,U+56e0,U+56e2,U+56ed,U+56f0,U+56f4,U+56fa,U+56fd-56fe,U+5706,U+5708,U+571f,U+5723,U+5728,U+5730,U+573a,U+5740,U+5747,U+574f-5751,U+5757,U+575a-575b,U+5761,U+5766,U+5782,U+578b,U+57c3,U+57cb,U+57ce,U+57df,U+57f9-57fa,U+5802,U+5806,U+5821,U+582a,U+5851,U+5854,U+585e,U+586b,U+5883,U+5899,U+589e,U+58a8,U+58c1,U+58eb,U+58ee,U+58f0,U+58f3,U+5904,U+5907,U+590d,U+590f,U+5915-5916,U+591a,U+591c,U+591f,U+5927,U+5929-592b,U+592e,U+5931,U+5934,U+5938-593a,U+5947-5949,U+594b,U+594f,U+5954,U+5956-5957,U+5965,U+5973,U+5976,U+5979,U+597d,U+5982,U+5986-5988,U+5999,U+59b9,U+59bb,U+59c6,U+59cb,U+59d0-59d1,U+59d3-59d4,U+59dc,U+59fb,U+59ff,U+5a01,U+5a18,U+5a31,U+5a46,U+5a5a,U+5a92,U+5ac1,U+5acc,U+5ae9,U+5b50,U+5b54-5b55,U+5b57-5b59,U+5b63-5b64,U+5b66,U+5b69,U+5b81,U+5b83,U+5b85,U+5b87-5b89,U+5b8b-5b8c,U+5b8f,U+5b97-5b98,U+5b9a,U+5b9c-5b9e,U+5ba1-5ba4,U+5bab,U+5bb3,U+5bb6,U+5bb9,U+5bbd-5bbf,U+5bc2,U+5bc4,U+5bc6,U+5bcc,U+5bd2-5bd3,U+5bdf,U+5bf8-5bfc,U+5bff,U+5c01,U+5c04,U+5c06,U+5c0a,U+5c0f,U+5c11,U+5c14,U+5c16,U+5c18,U+5c1a,U+5c1d,U+5c24,U+5c31,U+5c3a,U+5c3c-5c40,U+5c42,U+5c45,U+5c48,U+5c4a-5c4b,U+5c4f,U+5c55,U+5c5e,U+5c71,U+5c81,U+5c97,U+5c9b,U+5ca9,U+5cb8,U+5cf0,U+5d07,U+5ddd-5dde,U+5de1,U+5de5-5de8,U+5dee,U+5df1-5df2,U+5df4,U+5e01-5e03,U+5e05,U+5e08,U+5e0c,U+5e1d,U+5e26,U+5e2d-5e2e,U+5e38,U+5e45,U+5e55,U+5e72-5e74,U+5e76,U+5e78,U+5e7b-5e7d,U+5e7f,U+5e84,U+5e86,U+5e8a,U+5e8f,U+5e93-5e95,U+5e97,U+5e99,U+5e9c,U+5e9f,U+5ea6-5ea7,U+5ead,U+5eb7,U+5ec9,U+5ef6-5ef7,U+5efa,U+5f00,U+5f02-5f04,U+5f0f,U+5f15,U+5f1f-5f20,U+5f25,U+5f2f,U+5f31,U+5f39-5f3a,U+5f52-5f53,U+5f55,U+5f62,U+5f69,U+5f71,U+5f79,U+5f7b-5f7c,U+5f80-5f81,U+5f84-5f85,U+5f88,U+5f8b,U+5f90,U+5f92,U+5f97,U+5fa1,U+5faa,U+5fae,U+5fb7,U+5fbd,U+5fc3,U+5fc5-5fc6,U+5fcc-5fcd,U+5fd7-5fd9,U+5fe0,U+5fe7,U+5feb,U+5ff5,U+5ffd,U+6000-6001,U+600e,U+6012,U+6015,U+601d,U+6025,U+6027-6028,U+602a,U+603b,U+604b,U+6050,U+6052,U+6062,U+6068-6069,U+606f-6070,U+6076,U+607c,U+6089,U+6094,U+609f-60a0,U+60a3,U+60a6,U+60a8,U+60ac,U+60b2,U+60c5,U+60ca,U+60d1,U+60dc,U+60e0,U+60e7-60e8,U+60ef,U+60f3,U+6108,U+610f,U+611f,U+6124,U+613f,U+6148,U+614e,U+6155,U+6162,U+6167,U+6170,U+61c2,U+61d2,U+620f-6212,U+6216,U+6218,U+622a,U+6234,U+6237,U+623f-6240,U+624b,U+624d-624e,U+6253,U+6258,U+6263,U+6267,U+6269,U+626b-626c,U+626e,U+6270,U+6276,U+6279,U+627e-6280,U+628a,U+6291,U+6293,U+6295,U+6297-6298,U+629b,U+62a2,U+62a4-62a5,U+62ab,U+62b1,U+62b5,U+62b9,U+62bd,U+62c5-62c6,U+62c9,U+62cd,U+62d2-62d4,U+62d6,U+62db-62dc,U+62df,U+62e5,U+62e8-62e9,U+62ec,U+62fc,U+62ff,U+6301-6302,U+6307,U+6309,U+6311,U+6316,U+6321,U+6324-6325,U+632f,U+633a,U+6355,U+635f,U+6362,U+636e,U+6377,U+6388-6389,U+638c,U+6392,U+63a2,U+63a5,U+63a7-63aa,U+63cf-63d0,U+63d2,U+63e1,U+63ed,U+63f4,U+641c,U+641e,U+642c-642d,U+643a,U+6444,U+6446-6447,U+6458,U+6469,U+6478,U+6491-6492,U+649e,U+64ad,U+64c5,U+64cd,U+64e6,U+652f,U+6536,U+6539,U+653b,U+653e-653f,U+6545,U+6548,U+654c,U+654f,U+6551,U+6559,U+6562-6563,U+656c,U+6570,U+6574,U+6587,U+6591,U+6597,U+6599,U+659c,U+65a4,U+65ad,U+65af-65b0,U+65b9,U+65bd,U+65c1,U+65c5,U+65cb,U+65cf,U+65d7,U+65e0,U+65e2,U+65e5-65e9,U+65f6,U+65fa,U+6602,U+660c,U+660e,U+6613,U+661f-6620,U+6625,U+6628,U+662f,U+663e,U+664b,U+6653,U+665a,U+6668,U+666e-6670,U+6676,U+667a,U+6682,U+6696-6697,U+66b4,U+66f0,U+66f2,U+66f4,U+66fc,U+66fe-6700,U+6708-6709,U+670b,U+670d,U+6717,U+671b,U+671d,U+671f,U+6728,U+672a-672c,U+672f,U+6731,U+6734-6735,U+673a,U+6740,U+6742-6743,U+6746,U+674e,U+6750-6751,U+675c,U+675f,U+6761,U+6765,U+6768,U+676f-6770,U+677e-677f,U+6781,U+6784,U+6790,U+6797,U+679c-679d,U+67aa,U+67b6,U+67d0,U+67d3-67d4,U+67e5,U+67f1,U+67f3,U+6807,U+680f,U+6811,U+6821,U+6837-6839,U+683c,U+6843,U+6846,U+6848,U+684c,U+6851,U+6863,U+6865,U+6881,U+6885,U+68a6,U+68af-68b0,U+68c0,U+68d2,U+68ee,U+690d,U+695a,U+697c,U+6982,U+699c,U+69fd,U+6a21,U+6a2a,U+6b21-6b23,U+6b27,U+6b32,U+6b3a,U+6b3e,U+6b4c,U+6b62-6b66,U+6b7b,U+6b8a-6b8b,U+6b96,U+6bb5,U+6bbf,U+6bc1,U+6bcd,U+6bcf,U+6bd2,U+6bd4-6bd5,U+6bdb,U+6beb,U+6c0f,U+6c11,U+6c14,U+6c1b,U+6c27,U+6c34,U+6c38,U+6c41-6c42,U+6c47,U+6c49,U+6c57,U+6c5f-6c61,U+6c64,U+6c7d,U+6c88-6c89,U+6c99,U+6c9f,U+6ca1,U+6cb3,U+6cb9,U+6cbb,U+6cbf,U+6cc4,U+6cc9,U+6cd5,U+6cdb,U+6ce1-6ce2,U+6ce5,U+6ce8,U+6cea,U+6cf0,U+6cfd,U+6d01,U+6d0b,U+6d12,U+6d17,U+6d1b,U+6d1e,U+6d25,U+6d2a,U+6d32,U+6d3b,U+6d3e,U+6d41,U+6d45,U+6d4b,U+6d4e,U+6d53,U+6d59,U+6d69-6d6a,U+6d6e,U+6d77-6d78,U+6d82,U+6d88-6d89,U+6d8c,U+6da6,U+6da8,U+6daf,U+6db2,U+6db5,U+6dcb,U+6dd8,U+6de1,U+6df1,U+6df7,U+6dfb,U+6e05,U+6e10,U+6e20-6e21,U+6e29,U+6e2f,U+6e34,U+6e38,U+6e56,U+6e7e-6e7f,U+6e90,U+6eaa,U+6ecb,U+6ed1,U+6eda,U+6ee1,U+6ef4,U+6f02,U+6f0f,U+6f14,U+6f2b,U+6f5c,U+6f6e,U+6fb3,U+6fc0,U+704c,U+706b,U+706d,U+706f-7070,U+7075,U+707e,U+7089,U+708e,U+7092,U+70ae,U+70b8-70b9,U+70bc,U+70c2,U+70c8,U+70df,U+70e6-70e7,U+70ed,U+7126,U+7136,U+7167,U+716e,U+718a,U+719f,U+71c3,U+71d5,U+71e5,U+7206,U+722c,U+7231,U+7236-7238,U+723d,U+7247-7248,U+724c,U+7259,U+725b,U+7262,U+7269,U+7275,U+7279,U+72af,U+72b6,U+72b9,U+72c2,U+72d7,U+72ec,U+72f1,U+731b-731c,U+732a-732b,U+732e,U+7384,U+7387,U+7389,U+738b,U+739b,U+73a9,U+73af-73b0,U+73bb,U+73cd,U+73e0,U+73ed,U+7403,U+7406,U+7434,U+745e,U+7483,U+74dc,U+74e6,U+74f6,U+7518,U+751a,U+751c,U+751f,U+7528,U+7530-7533,U+7535,U+7537,U+753b,U+7545,U+754c,U+7559,U+7565,U+756a,U+758f,U+7591,U+7597,U+75af,U+75b2,U+75bc,U+75be,U+75c5,U+75c7,U+75d5,U+75db,U+7626,U+767b,U+767d-767e,U+7684,U+7686-7687,U+76ae,U+76c6,U+76c8,U+76ca,U+76d0-76d2,U+76d6-76d8,U+76db,U+76df,U+76ee,U+76f4,U+76f8,U+76fe,U+7701,U+7709,U+770b,U+771f-7720,U+773c,U+7740,U+775b,U+7761,U+7763,U+77ac,U+77db,U+77e5,U+77ed,U+77f3,U+77ff,U+7801,U+7814,U+7834,U+7840,U+7855,U+786c,U+786e,U+788d-788e,U+7891,U+7897,U+78b0,U+78e8,U+793a,U+793c,U+793e,U+7956,U+795d-795e,U+7965,U+7968,U+7981,U+798f,U+79bb,U+79c0-79c1,U+79cb,U+79cd,U+79d1-79d2,U+79d8,U+79df,U+79e6,U+79ef-79f0,U+79fb,U+7a00,U+7a0b,U+7a0d-7a0e,U+7a33,U+7a3f,U+7a76-7a77,U+7a7a,U+7a7f,U+7a81,U+7a97,U+7acb,U+7ad9,U+7ade-7ae0,U+7ae5,U+7aef,U+7af9,U+7b11,U+7b14,U+7b26,U+7b2c,U+7b49,U+7b51,U+7b54,U+7b56,U+7b79,U+7b7e,U+7b80,U+7b97,U+7ba1,U+7bb1,U+7bc7,U+7c4d,U+7c73,U+7c7b,U+7c89,U+7c92,U+7c97-7c98,U+7cae,U+7cbe,U+7cca,U+7cd6,U+7cfb,U+7d20,U+7d22,U+7d27,U+7d2b,U+7d2f,U+7e41,U+7ea0,U+7ea2,U+7ea4,U+7ea6-7ea7,U+7eaa,U+7eaf,U+7eb3,U+7eb5,U+7eb7-7eb9,U+7ebd,U+7ebf,U+7ec3-7ec4,U+7ec6-7ec8,U+7ecd,U+7ecf,U+7ed3,U+7ed5,U+7ed8-7ed9,U+7edc-7edd,U+7edf,U+7ee7,U+7ee9-7eea,U+7eed,U+7ef4-7ef5,U+7efc,U+7eff,U+7f13,U+7f16,U+7f18,U+7f1d,U+7f29,U+7f3a,U+7f51,U+7f57,U+7f5a,U+7f62,U+7f6a,U+7f6e,U+7f72,U+7f8a,U+7f8e,U+7fa4,U+7fbd,U+7ffb-7ffc,U+8000-8001,U+8003,U+8005,U+800c,U+8010,U+8017,U+8033,U+804a,U+804c,U+8054,U+8058,U+805a,U+806a,U+8083,U+8089,U+808c,U+809a,U+80a0-80a1,U+80a4-80a5,U+80a9,U+80af,U+80b2,U+80bf,U+80c3,U+80c6,U+80cc,U+80ce,U+80dc,U+80de,U+80e1,U+80f6,U+80f8,U+80fd,U+8102,U+8106,U+8109,U+810f,U+8111,U+811a,U+8131,U+8138,U+8150,U+8170,U+8179,U+817b,U+817e-817f,U+819c,U+81e3,U+81ea,U+81f3-81f4,U+820d,U+8212,U+821e,U+822a,U+822c,U+8239,U+826f-8270,U+8272-8273,U+827a,U+827e,U+8282,U+82b1,U+82b3,U+82cd,U+82cf,U+82e5-82e6,U+82f1,U+8303,U+8336,U+8349,U+8350,U+8352,U+8361,U+8363,U+836f,U+8377,U+83ab,U+83b1-83b2,U+83b7,U+83dc,U+83f2,U+8425,U+8428,U+843d,U+8457,U+8463,U+8499,U+84b8,U+84dd,U+8584,U+85cf,U+864e,U+8651,U+865a,U+866b,U+867d,U+86cb,U+86ee,U+871c,U+878d,U+8840,U+884c,U+8857,U+8861,U+8863,U+8865,U+8868,U+8870,U+888b,U+8896,U+88ab,U+88ad,U+88c1-88c2,U+88c5,U+897f,U+8981,U+8986,U+89c1-89c2,U+89c4,U+89c6,U+89c8-89c9,U+89d2,U+89e3,U+89e6,U+8a00,U+8a89,U+8b66,U+8ba1-8ba2,U+8ba4,U+8ba8-8ba9,U+8bad-8bb0,U+8bb2,U+8bb8,U+8bba,U+8bbe-8bbf,U+8bc1,U+8bc4,U+8bc6,U+8bc9-8bca,U+8bcd,U+8bd1,U+8bd5,U+8bd7,U+8bda,U+8bdd-8bde,U+8be2,U+8be5-8be6,U+8bed,U+8bef,U+8bf1,U+8bf4,U+8bf7-8bf8,U+8bfa-8bfb,U+8bfe,U+8c01,U+8c03,U+8c08,U+8c0b,U+8c10,U+8c13,U+8c22,U+8c28,U+8c31,U+8c37,U+8c46,U+8c61,U+8c6a,U+8c8c,U+8d1d,U+8d1f,U+8d21-8d28,U+8d2a-8d2b,U+8d2d,U+8d2f,U+8d34-8d35,U+8d38-8d39,U+8d44,U+8d4b,U+8d4f,U+8d56,U+8d5a-8d5b,U+8d5e,U+8d60,U+8d62,U+8d64,U+8d6b,U+8d70,U+8d74-8d77,U+8d85,U+8d8a-8d8b,U+8da3,U+8db3,U+8dc3,U+8dcc,U+8dd1,U+8ddd,U+8ddf,U+8de8,U+8def,U+8df3,U+8df5,U+8e0f,U+8e2a,U+8eab,U+8f66,U+8f68,U+8f6c,U+8f6e-8f6f,U+8f7b,U+8f7d,U+8f83,U+8f85-8f86,U+8f88-8f89,U+8f91,U+8f93,U+8f9b,U+8f9e,U+8fa3,U+8fa8-8fa9,U+8fb9,U+8fbe,U+8fc1,U+8fc5,U+8fc7-8fc8,U+8fce,U+8fd0-8fd1,U+8fd4,U+8fd8-8fd9,U+8fdb-8fdf,U+8fea-8feb,U+8ff0,U+8ff7,U+8ff9,U+8ffd,U+9000-9003,U+9006,U+9009,U+900f-9010,U+9012,U+9014,U+901a,U+901d,U+901f-9020,U+9022,U+903b-903c,U+9047,U+904d,U+9053,U+9057,U+9065,U+906d,U+9075,U+907f-9080,U+90a3,U+90a6,U+90aa,U+90ae,U+90bb,U+90c1,U+90ce,U+90d1,U+90e8,U+90ed,U+90fd,U+914d,U+9152,U+9177-9178,U+9189,U+9192,U+91c7,U+91ca,U+91cc-91cf,U+91d1,U+9274,U+9488,U+949f,U+94a2,U+94b1,U+94bb,U+94c1,U+94dc,U+94f6,U+94fa,U+94fe,U+9500-9501,U+9505,U+950b,U+9510,U+9519,U+9526,U+952e,U+9547,U+955c,U+957f,U+95e8,U+95ea,U+95ed-95ee,U+95f2,U+95f4,U+95f7,U+95f9,U+95fb,U+9605,U+9614,U+961f,U+9632-9636,U+963b,U+963f,U+9644-9646,U+9648,U+964d,U+9650,U+9662,U+9664,U+9669-966a,U+9675-9677,U+9686,U+968f-9690,U+9694,U+969c,U+96be,U+96c4-96c6,U+96d5,U+96e8,U+96ea,U+96f6-96f7,U+96fe,U+9700,U+9707,U+9732,U+9738,U+9752,U+9759,U+975e,U+9760,U+9762,U+9769,U+978b,U+97e9,U+97f3,U+97f5,U+9875-9876,U+9879-987b,U+987e-987f,U+9884,U+9886-9887,U+9891,U+9897-9898,U+989c-989d,U+98ce,U+98d8,U+98de-98df,U+9910,U+996d-996e,U+9970-9971,U+9986,U+9996,U+9999,U+9a6c,U+9a71,U+9a7b,U+9a7e,U+9a82,U+9a8c,U+9a91,U+9a97,U+9aa4,U+9aa8,U+9ad8,U+9b3c,U+9b42,U+9b45,U+9b54,U+9c7c,U+9c81,U+9c9c,U+9e1f,U+9e21,U+9e23,U+9ea6,U+9ebb,U+9ec4,U+9ece,U+9ed1,U+9ed8,U+9f13,U+9f20,U+9f3b,U+9f50,U+9f7f,U+9f84,U+9f99,U+ff01,U+ff08-ff09,U+ff0c-ff0e,U+ff1a-ff1b,U+ff1f,U+ff5e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.100.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.100.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.100.otf')  format('opentype');unicode-range:U+0021-0023,U+0025-005b,U+005d,U+005f,U+0061-007a,U+007e-00a0,U+00b7,U+2014,U+2018-2019,U+201c-201d,U+2026,U+3000-3002,U+300a-300d,U+3010-3011,U+4e00-4e01,U+4e03,U+4e07-4e0b,U+4e0d-4e0e,U+4e11,U+4e13-4e14,U+4e16,U+4e1a-4e1d,U+4e22,U+4e24-4e25,U+4e27,U+4e2a,U+4e2d,U+4e30,U+4e32,U+4e34,U+4e39-4e3b,U+4e3d-4e3e,U+4e43,U+4e45,U+4e48-4e49,U+4e4b-4e4c,U+4e4e-4e50,U+4e54,U+4e58,U+4e5d,U+4e5f-4e61,U+4e66,U+4e70-4e71,U+4e73,U+4e86,U+4e88-4e89,U+4e8b-4e8c,U+4e8e-4e8f,U+4e91-4e92,U+4e94-4e95,U+4e9a-4e9b,U+4ea1,U+4ea4,U+4ea6-4ea7,U+4eab-4eac,U+4eae,U+4eb2,U+4eba,U+4ebf-4ec1,U+4ec5,U+4eca-4ecb,U+4ecd-4ece,U+4ed3-4ed4,U+4ed6,U+4ed8-4ed9,U+4ee3-4ee5,U+4eea,U+4eec,U+4ef0,U+4ef6-4ef7,U+4efb,U+4efd,U+4eff,U+4f01,U+4f0a,U+4f0d,U+4f0f,U+4f11,U+4f17-4f1a,U+4f1f-4f20,U+4f24,U+4f26,U+4f2a,U+4f2f-4f30,U+4f34,U+4f38,U+4f3c,U+4f46,U+4f4d-4f4f,U+4f53,U+4f55,U+4f59,U+4f5b-4f5c,U+4f60,U+4f69,U+4f73,U+4f7f,U+4f8b,U+4f9b,U+4f9d,U+4fa7,U+4fb5,U+4fbf,U+4fc3-4fc4,U+4fca,U+4fd7,U+4fdd,U+4fe1,U+4fe9,U+4fee,U+4ff1,U+500d,U+5012,U+5019,U+501f,U+503c,U+503e,U+5047,U+504f,U+505a,U+505c,U+5065,U+5076-5077,U+507f,U+50a8,U+50b2,U+50bb,U+50cf,U+513f,U+5141,U+5143-5145,U+5148-5149,U+514b,U+514d,U+515a,U+5165,U+5168,U+516b-516d,U+5170-5171,U+5173-5178,U+517b-517d,U+5185,U+518c-518d,U+5192,U+5199,U+519b-519c,U+51a0,U+51ac,U+51b0,U+51b2-51b3,U+51b5,U+51b7,U+51c0,U+51c6,U+51c9,U+51cc,U+51cf,U+51dd,U+51e0-51e1,U+51e4,U+51ed,U+51ef,U+51f6,U+51fa-51fb,U+5200,U+5206-5207,U+520a,U+5211-5212,U+5217-521b,U+521d,U+5220,U+5224,U+5229,U+522b,U+5230,U+5236-5237,U+523a-523b,U+5242,U+524d,U+5251,U+5267,U+5269-526a,U+526f,U+5272,U+529b,U+529d-52a1,U+52a3,U+52a8-52aa,U+52b1-52b3,U+52bf,U+52c7,U+52d2,U+52e4,U+52fe,U+5300,U+5305,U+5316-5317,U+5339-533b,U+5341,U+5343,U+5347-5348,U+534a,U+534e-534f,U+5353,U+5355-5357,U+535a,U+5360-5361,U+5367,U+536b,U+5370-5371,U+5373-5374,U+5377,U+5382,U+5385-5386,U+5389,U+538b-538c,U+5398,U+539a,U+539f,U+53bb,U+53bf,U+53c2,U+53c8,U+53ca-53cd,U+53d1,U+53d6-53d8,U+53e0,U+53e3-53e6,U+53ea-53ec,U+53ef-53f0,U+53f2-53f3,U+53f6-53f9,U+5403-5404,U+5408-5409,U+540c-540e,U+5410-5411,U+5413,U+5417,U+541b,U+5426-5427,U+542b-542c,U+542f,U+5434,U+5438-5439,U+5440,U+5446,U+5448,U+544a,U+5458,U+5462,U+5468,U+5473,U+5475,U+547c-547d,U+548c,U+5496,U+54a8,U+54c1,U+54c8,U+54cd,U+54e5-54e6,U+54ea,U+54ed,U+54f2,U+5510,U+552e-552f,U+5531,U+5546,U+554a,U+5565-5566,U+5584,U+558a,U+559c-559d,U+55b7,U+5609,U+561b,U+5634,U+5668,U+56db,U+56de,U+56e0,U+56e2,U+56ed,U+56f0,U+56f4,U+56fa,U+56fd-56fe,U+5706,U+5708,U+571f,U+5723,U+5728,U+5730,U+573a,U+5740,U+5747,U+574f-5751,U+5757,U+575a-575b,U+5761,U+5766,U+5782,U+578b,U+57c3,U+57cb,U+57ce,U+57df,U+57f9-57fa,U+5802,U+5806,U+5821,U+582a,U+5851,U+5854,U+585e,U+586b,U+5883,U+5899,U+589e,U+58a8,U+58c1,U+58eb,U+58ee,U+58f0,U+58f3,U+5904,U+5907,U+590d,U+590f,U+5915-5916,U+591a,U+591c,U+591f,U+5927,U+5929-592b,U+592e,U+5931,U+5934,U+5938-593a,U+5947-5949,U+594b,U+594f,U+5954,U+5956-5957,U+5965,U+5973,U+5976,U+5979,U+597d,U+5982,U+5986-5988,U+5999,U+59b9,U+59bb,U+59c6,U+59cb,U+59d0-59d1,U+59d3-59d4,U+59dc,U+59fb,U+59ff,U+5a01,U+5a18,U+5a31,U+5a46,U+5a5a,U+5a92,U+5ac1,U+5acc,U+5ae9,U+5b50,U+5b54-5b55,U+5b57-5b59,U+5b63-5b64,U+5b66,U+5b69,U+5b81,U+5b83,U+5b85,U+5b87-5b89,U+5b8b-5b8c,U+5b8f,U+5b97-5b98,U+5b9a,U+5b9c-5b9e,U+5ba1-5ba4,U+5bab,U+5bb3,U+5bb6,U+5bb9,U+5bbd-5bbf,U+5bc2,U+5bc4,U+5bc6,U+5bcc,U+5bd2-5bd3,U+5bdf,U+5bf8-5bfc,U+5bff,U+5c01,U+5c04,U+5c06,U+5c0a,U+5c0f,U+5c11,U+5c14,U+5c16,U+5c18,U+5c1a,U+5c1d,U+5c24,U+5c31,U+5c3a,U+5c3c-5c40,U+5c42,U+5c45,U+5c48,U+5c4a-5c4b,U+5c4f,U+5c55,U+5c5e,U+5c71,U+5c81,U+5c97,U+5c9b,U+5ca9,U+5cb8,U+5cf0,U+5d07,U+5ddd-5dde,U+5de1,U+5de5-5de8,U+5dee,U+5df1-5df2,U+5df4,U+5e01-5e03,U+5e05,U+5e08,U+5e0c,U+5e1d,U+5e26,U+5e2d-5e2e,U+5e38,U+5e45,U+5e55,U+5e72-5e74,U+5e76,U+5e78,U+5e7b-5e7d,U+5e7f,U+5e84,U+5e86,U+5e8a,U+5e8f,U+5e93-5e95,U+5e97,U+5e99,U+5e9c,U+5e9f,U+5ea6-5ea7,U+5ead,U+5eb7,U+5ec9,U+5ef6-5ef7,U+5efa,U+5f00,U+5f02-5f04,U+5f0f,U+5f15,U+5f1f-5f20,U+5f25,U+5f2f,U+5f31,U+5f39-5f3a,U+5f52-5f53,U+5f55,U+5f62,U+5f69,U+5f71,U+5f79,U+5f7b-5f7c,U+5f80-5f81,U+5f84-5f85,U+5f88,U+5f8b,U+5f90,U+5f92,U+5f97,U+5fa1,U+5faa,U+5fae,U+5fb7,U+5fbd,U+5fc3,U+5fc5-5fc6,U+5fcc-5fcd,U+5fd7-5fd9,U+5fe0,U+5fe7,U+5feb,U+5ff5,U+5ffd,U+6000-6001,U+600e,U+6012,U+6015,U+601d,U+6025,U+6027-6028,U+602a,U+603b,U+604b,U+6050,U+6052,U+6062,U+6068-6069,U+606f-6070,U+6076,U+607c,U+6089,U+6094,U+609f-60a0,U+60a3,U+60a6,U+60a8,U+60ac,U+60b2,U+60c5,U+60ca,U+60d1,U+60dc,U+60e0,U+60e7-60e8,U+60ef,U+60f3,U+6108,U+610f,U+611f,U+6124,U+613f,U+6148,U+614e,U+6155,U+6162,U+6167,U+6170,U+61c2,U+61d2,U+620f-6212,U+6216,U+6218,U+622a,U+6234,U+6237,U+623f-6240,U+624b,U+624d-624e,U+6253,U+6258,U+6263,U+6267,U+6269,U+626b-626c,U+626e,U+6270,U+6276,U+6279,U+627e-6280,U+628a,U+6291,U+6293,U+6295,U+6297-6298,U+629b,U+62a2,U+62a4-62a5,U+62ab,U+62b1,U+62b5,U+62b9,U+62bd,U+62c5-62c6,U+62c9,U+62cd,U+62d2-62d4,U+62d6,U+62db-62dc,U+62df,U+62e5,U+62e8-62e9,U+62ec,U+62fc,U+62ff,U+6301-6302,U+6307,U+6309,U+6311,U+6316,U+6321,U+6324-6325,U+632f,U+633a,U+6355,U+635f,U+6362,U+636e,U+6377,U+6388-6389,U+638c,U+6392,U+63a2,U+63a5,U+63a7-63aa,U+63cf-63d0,U+63d2,U+63e1,U+63ed,U+63f4,U+641c,U+641e,U+642c-642d,U+643a,U+6444,U+6446-6447,U+6458,U+6469,U+6478,U+6491-6492,U+649e,U+64ad,U+64c5,U+64cd,U+64e6,U+652f,U+6536,U+6539,U+653b,U+653e-653f,U+6545,U+6548,U+654c,U+654f,U+6551,U+6559,U+6562-6563,U+656c,U+6570,U+6574,U+6587,U+6591,U+6597,U+6599,U+659c,U+65a4,U+65ad,U+65af-65b0,U+65b9,U+65bd,U+65c1,U+65c5,U+65cb,U+65cf,U+65d7,U+65e0,U+65e2,U+65e5-65e9,U+65f6,U+65fa,U+6602,U+660c,U+660e,U+6613,U+661f-6620,U+6625,U+6628,U+662f,U+663e,U+664b,U+6653,U+665a,U+6668,U+666e-6670,U+6676,U+667a,U+6682,U+6696-6697,U+66b4,U+66f0,U+66f2,U+66f4,U+66fc,U+66fe-6700,U+6708-6709,U+670b,U+670d,U+6717,U+671b,U+671d,U+671f,U+6728,U+672a-672c,U+672f,U+6731,U+6734-6735,U+673a,U+6740,U+6742-6743,U+6746,U+674e,U+6750-6751,U+675c,U+675f,U+6761,U+6765,U+6768,U+676f-6770,U+677e-677f,U+6781,U+6784,U+6790,U+6797,U+679c-679d,U+67aa,U+67b6,U+67d0,U+67d3-67d4,U+67e5,U+67f1,U+67f3,U+6807,U+680f,U+6811,U+6821,U+6837-6839,U+683c,U+6843,U+6846,U+6848,U+684c,U+6851,U+6863,U+6865,U+6881,U+6885,U+68a6,U+68af-68b0,U+68c0,U+68d2,U+68ee,U+690d,U+695a,U+697c,U+6982,U+699c,U+69fd,U+6a21,U+6a2a,U+6b21-6b23,U+6b27,U+6b32,U+6b3a,U+6b3e,U+6b4c,U+6b62-6b66,U+6b7b,U+6b8a-6b8b,U+6b96,U+6bb5,U+6bbf,U+6bc1,U+6bcd,U+6bcf,U+6bd2,U+6bd4-6bd5,U+6bdb,U+6beb,U+6c0f,U+6c11,U+6c14,U+6c1b,U+6c27,U+6c34,U+6c38,U+6c41-6c42,U+6c47,U+6c49,U+6c57,U+6c5f-6c61,U+6c64,U+6c7d,U+6c88-6c89,U+6c99,U+6c9f,U+6ca1,U+6cb3,U+6cb9,U+6cbb,U+6cbf,U+6cc4,U+6cc9,U+6cd5,U+6cdb,U+6ce1-6ce2,U+6ce5,U+6ce8,U+6cea,U+6cf0,U+6cfd,U+6d01,U+6d0b,U+6d12,U+6d17,U+6d1b,U+6d1e,U+6d25,U+6d2a,U+6d32,U+6d3b,U+6d3e,U+6d41,U+6d45,U+6d4b,U+6d4e,U+6d53,U+6d59,U+6d69-6d6a,U+6d6e,U+6d77-6d78,U+6d82,U+6d88-6d89,U+6d8c,U+6da6,U+6da8,U+6daf,U+6db2,U+6db5,U+6dcb,U+6dd8,U+6de1,U+6df1,U+6df7,U+6dfb,U+6e05,U+6e10,U+6e20-6e21,U+6e29,U+6e2f,U+6e34,U+6e38,U+6e56,U+6e7e-6e7f,U+6e90,U+6eaa,U+6ecb,U+6ed1,U+6eda,U+6ee1,U+6ef4,U+6f02,U+6f0f,U+6f14,U+6f2b,U+6f5c,U+6f6e,U+6fb3,U+6fc0,U+704c,U+706b,U+706d,U+706f-7070,U+7075,U+707e,U+7089,U+708e,U+7092,U+70ae,U+70b8-70b9,U+70bc,U+70c2,U+70c8,U+70df,U+70e6-70e7,U+70ed,U+7126,U+7136,U+7167,U+716e,U+718a,U+719f,U+71c3,U+71d5,U+71e5,U+7206,U+722c,U+7231,U+7236-7238,U+723d,U+7247-7248,U+724c,U+7259,U+725b,U+7262,U+7269,U+7275,U+7279,U+72af,U+72b6,U+72b9,U+72c2,U+72d7,U+72ec,U+72f1,U+731b-731c,U+732a-732b,U+732e,U+7384,U+7387,U+7389,U+738b,U+739b,U+73a9,U+73af-73b0,U+73bb,U+73cd,U+73e0,U+73ed,U+7403,U+7406,U+7434,U+745e,U+7483,U+74dc,U+74e6,U+74f6,U+7518,U+751a,U+751c,U+751f,U+7528,U+7530-7533,U+7535,U+7537,U+753b,U+7545,U+754c,U+7559,U+7565,U+756a,U+758f,U+7591,U+7597,U+75af,U+75b2,U+75bc,U+75be,U+75c5,U+75c7,U+75d5,U+75db,U+7626,U+767b,U+767d-767e,U+7684,U+7686-7687,U+76ae,U+76c6,U+76c8,U+76ca,U+76d0-76d2,U+76d6-76d8,U+76db,U+76df,U+76ee,U+76f4,U+76f8,U+76fe,U+7701,U+7709,U+770b,U+771f-7720,U+773c,U+7740,U+775b,U+7761,U+7763,U+77ac,U+77db,U+77e5,U+77ed,U+77f3,U+77ff,U+7801,U+7814,U+7834,U+7840,U+7855,U+786c,U+786e,U+788d-788e,U+7891,U+7897,U+78b0,U+78e8,U+793a,U+793c,U+793e,U+7956,U+795d-795e,U+7965,U+7968,U+7981,U+798f,U+79bb,U+79c0-79c1,U+79cb,U+79cd,U+79d1-79d2,U+79d8,U+79df,U+79e6,U+79ef-79f0,U+79fb,U+7a00,U+7a0b,U+7a0d-7a0e,U+7a33,U+7a3f,U+7a76-7a77,U+7a7a,U+7a7f,U+7a81,U+7a97,U+7acb,U+7ad9,U+7ade-7ae0,U+7ae5,U+7aef,U+7af9,U+7b11,U+7b14,U+7b26,U+7b2c,U+7b49,U+7b51,U+7b54,U+7b56,U+7b79,U+7b7e,U+7b80,U+7b97,U+7ba1,U+7bb1,U+7bc7,U+7c4d,U+7c73,U+7c7b,U+7c89,U+7c92,U+7c97-7c98,U+7cae,U+7cbe,U+7cca,U+7cd6,U+7cfb,U+7d20,U+7d22,U+7d27,U+7d2b,U+7d2f,U+7e41,U+7ea0,U+7ea2,U+7ea4,U+7ea6-7ea7,U+7eaa,U+7eaf,U+7eb3,U+7eb5,U+7eb7-7eb9,U+7ebd,U+7ebf,U+7ec3-7ec4,U+7ec6-7ec8,U+7ecd,U+7ecf,U+7ed3,U+7ed5,U+7ed8-7ed9,U+7edc-7edd,U+7edf,U+7ee7,U+7ee9-7eea,U+7eed,U+7ef4-7ef5,U+7efc,U+7eff,U+7f13,U+7f16,U+7f18,U+7f1d,U+7f29,U+7f3a,U+7f51,U+7f57,U+7f5a,U+7f62,U+7f6a,U+7f6e,U+7f72,U+7f8a,U+7f8e,U+7fa4,U+7fbd,U+7ffb-7ffc,U+8000-8001,U+8003,U+8005,U+800c,U+8010,U+8017,U+8033,U+804a,U+804c,U+8054,U+8058,U+805a,U+806a,U+8083,U+8089,U+808c,U+809a,U+80a0-80a1,U+80a4-80a5,U+80a9,U+80af,U+80b2,U+80bf,U+80c3,U+80c6,U+80cc,U+80ce,U+80dc,U+80de,U+80e1,U+80f6,U+80f8,U+80fd,U+8102,U+8106,U+8109,U+810f,U+8111,U+811a,U+8131,U+8138,U+8150,U+8170,U+8179,U+817b,U+817e-817f,U+819c,U+81e3,U+81ea,U+81f3-81f4,U+820d,U+8212,U+821e,U+822a,U+822c,U+8239,U+826f-8270,U+8272-8273,U+827a,U+827e,U+8282,U+82b1,U+82b3,U+82cd,U+82cf,U+82e5-82e6,U+82f1,U+8303,U+8336,U+8349,U+8350,U+8352,U+8361,U+8363,U+836f,U+8377,U+83ab,U+83b1-83b2,U+83b7,U+83dc,U+83f2,U+8425,U+8428,U+843d,U+8457,U+8463,U+8499,U+84b8,U+84dd,U+8584,U+85cf,U+864e,U+8651,U+865a,U+866b,U+867d,U+86cb,U+86ee,U+871c,U+878d,U+8840,U+884c,U+8857,U+8861,U+8863,U+8865,U+8868,U+8870,U+888b,U+8896,U+88ab,U+88ad,U+88c1-88c2,U+88c5,U+897f,U+8981,U+8986,U+89c1-89c2,U+89c4,U+89c6,U+89c8-89c9,U+89d2,U+89e3,U+89e6,U+8a00,U+8a89,U+8b66,U+8ba1-8ba2,U+8ba4,U+8ba8-8ba9,U+8bad-8bb0,U+8bb2,U+8bb8,U+8bba,U+8bbe-8bbf,U+8bc1,U+8bc4,U+8bc6,U+8bc9-8bca,U+8bcd,U+8bd1,U+8bd5,U+8bd7,U+8bda,U+8bdd-8bde,U+8be2,U+8be5-8be6,U+8bed,U+8bef,U+8bf1,U+8bf4,U+8bf7-8bf8,U+8bfa-8bfb,U+8bfe,U+8c01,U+8c03,U+8c08,U+8c0b,U+8c10,U+8c13,U+8c22,U+8c28,U+8c31,U+8c37,U+8c46,U+8c61,U+8c6a,U+8c8c,U+8d1d,U+8d1f,U+8d21-8d28,U+8d2a-8d2b,U+8d2d,U+8d2f,U+8d34-8d35,U+8d38-8d39,U+8d44,U+8d4b,U+8d4f,U+8d56,U+8d5a-8d5b,U+8d5e,U+8d60,U+8d62,U+8d64,U+8d6b,U+8d70,U+8d74-8d77,U+8d85,U+8d8a-8d8b,U+8da3,U+8db3,U+8dc3,U+8dcc,U+8dd1,U+8ddd,U+8ddf,U+8de8,U+8def,U+8df3,U+8df5,U+8e0f,U+8e2a,U+8eab,U+8f66,U+8f68,U+8f6c,U+8f6e-8f6f,U+8f7b,U+8f7d,U+8f83,U+8f85-8f86,U+8f88-8f89,U+8f91,U+8f93,U+8f9b,U+8f9e,U+8fa3,U+8fa8-8fa9,U+8fb9,U+8fbe,U+8fc1,U+8fc5,U+8fc7-8fc8,U+8fce,U+8fd0-8fd1,U+8fd4,U+8fd8-8fd9,U+8fdb-8fdf,U+8fea-8feb,U+8ff0,U+8ff7,U+8ff9,U+8ffd,U+9000-9003,U+9006,U+9009,U+900f-9010,U+9012,U+9014,U+901a,U+901d,U+901f-9020,U+9022,U+903b-903c,U+9047,U+904d,U+9053,U+9057,U+9065,U+906d,U+9075,U+907f-9080,U+90a3,U+90a6,U+90aa,U+90ae,U+90bb,U+90c1,U+90ce,U+90d1,U+90e8,U+90ed,U+90fd,U+914d,U+9152,U+9177-9178,U+9189,U+9192,U+91c7,U+91ca,U+91cc-91cf,U+91d1,U+9274,U+9488,U+949f,U+94a2,U+94b1,U+94bb,U+94c1,U+94dc,U+94f6,U+94fa,U+94fe,U+9500-9501,U+9505,U+950b,U+9510,U+9519,U+9526,U+952e,U+9547,U+955c,U+957f,U+95e8,U+95ea,U+95ed-95ee,U+95f2,U+95f4,U+95f7,U+95f9,U+95fb,U+9605,U+9614,U+961f,U+9632-9636,U+963b,U+963f,U+9644-9646,U+9648,U+964d,U+9650,U+9662,U+9664,U+9669-966a,U+9675-9677,U+9686,U+968f-9690,U+9694,U+969c,U+96be,U+96c4-96c6,U+96d5,U+96e8,U+96ea,U+96f6-96f7,U+96fe,U+9700,U+9707,U+9732,U+9738,U+9752,U+9759,U+975e,U+9760,U+9762,U+9769,U+978b,U+97e9,U+97f3,U+97f5,U+9875-9876,U+9879-987b,U+987e-987f,U+9884,U+9886-9887,U+9891,U+9897-9898,U+989c-989d,U+98ce,U+98d8,U+98de-98df,U+9910,U+996d-996e,U+9970-9971,U+9986,U+9996,U+9999,U+9a6c,U+9a71,U+9a7b,U+9a7e,U+9a82,U+9a8c,U+9a91,U+9a97,U+9aa4,U+9aa8,U+9ad8,U+9b3c,U+9b42,U+9b45,U+9b54,U+9c7c,U+9c81,U+9c9c,U+9e1f,U+9e21,U+9e23,U+9ea6,U+9ebb,U+9ec4,U+9ece,U+9ed1,U+9ed8,U+9f13,U+9f20,U+9f3b,U+9f50,U+9f7f,U+9f84,U+9f99,U+ff01,U+ff08-ff09,U+ff0c-ff0e,U+ff1a-ff1b,U+ff1f,U+ff5e;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:300;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.101.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.101.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Light.101.otf')  format('opentype');unicode-range:U+0024,U+005c,U+005e,U+007b-007d,U+00b0,U+00d7,U+00e9,U+2013,U+2022,U+2103,U+2192,U+2460-2463,U+300e-300f,U+306e,U+4e18-4e19,U+4e2b,U+4e38,U+4e56,U+4e59,U+4e5e,U+4e7e,U+4ea5,U+4ea8-4ea9,U+4ead,U+4ec6-4ec7,U+4ed1,U+4ed5,U+4ed7,U+4ef2,U+4f10,U+4f1e,U+4f3d,U+4f50-4f51,U+4f63,U+4f83,U+4f88,U+4f8d,U+4fa0,U+4fa3,U+4fa6,U+4fa8,U+4faf,U+4fcf,U+4fd8,U+4fed,U+4fef,U+4ffa,U+501a,U+5021,U+5026,U+503a,U+5085,U+508d,U+50ac,U+50e7,U+50f5,U+50fb,U+5112,U+5146,U+5151,U+5154,U+515c,U+5179,U+5188,U+51a4-51a5,U+51af,U+51b6,U+51bb,U+51c4,U+51d1,U+51f0,U+51f8-51f9,U+51fd,U+51ff,U+5203,U+522e,U+5238-5239,U+524a,U+5254,U+5256,U+5265,U+5288,U+52ab,U+52c3,U+52c9,U+52cb,U+52d8,U+52df,U+52fa,U+52ff,U+5306,U+5319,U+5320,U+532a,U+533f,U+5349,U+5351-5352,U+535c,U+5362,U+5366,U+5375,U+5378,U+537f,U+5384,U+5395,U+53a2,U+53a6,U+53a8,U+53c9,U+53d4,U+53d9,U+53db,U+53ee,U+5401,U+540a,U+540f,U+5415,U+541e-541f,U+5428,U+5435,U+543b-543c,U+543e,U+5450,U+5455,U+548b,U+548f,U+5492,U+54ac,U+54af,U+54b1,U+54b3,U+54b8,U+54bd,U+54c0,U+54c4,U+54c7,U+54c9,U+54ce,U+54d1,U+54df,U+5507,U+5509,U+5524,U+5561,U+5564,U+5578,U+5582,U+5587,U+5589,U+5598,U+55a7,U+55bb,U+55d3,U+55dc,U+55ef,U+5631-5632,U+563f,U+566a,U+56a3,U+56bc,U+56ca,U+56da,U+5733,U+573e,U+574a,U+574e,U+575d-5760,U+5764,U+576a,U+5783-5784,U+5792,U+57ab,U+5815,U+5824,U+5835,U+584c,U+5858,U+5885,U+5893,U+58e4,U+58f6,U+5937,U+5951,U+5960,U+5962,U+5974,U+5978,U+5983-5984,U+5992,U+5996,U+59a5,U+59a8,U+59ae,U+59da,U+59e8,U+59ec,U+5a03,U+5a07,U+5a1c,U+5a36,U+5a49,U+5a74,U+5a9a,U+5ab3,U+5ac9,U+5b5d,U+5b5f,U+5b99,U+5b9b,U+5ba0,U+5baa,U+5bb0,U+5bb4-5bb5,U+5bc5,U+5bdd-5bde,U+5be1,U+5be8,U+5c2c,U+5c34,U+5c38-5c39,U+5c41,U+5c51,U+5c60-5c61,U+5c65,U+5c6f,U+5c82,U+5cad,U+5cb3,U+5ce1,U+5cfb,U+5d0e,U+5d14,U+5d16,U+5d1b,U+5d29,U+5d2d,U+5d4c,U+5dc5,U+5de2,U+5de9,U+5deb,U+5df7,U+5dfe,U+5e06,U+5e10,U+5e15-5e16,U+5e18,U+5e1c,U+5e27,U+5e3d,U+5e90,U+5e9a,U+5e9e,U+5eb8,U+5eca,U+5ed3,U+5f0a,U+5f13,U+5f17-5f18,U+5f26-5f27,U+5f66,U+5f6d,U+5f70,U+5f8c,U+6016,U+601c,U+6021,U+604d,U+606d,U+6073,U+6084,U+608d,U+60d5,U+60df,U+60e9,U+60f9,U+6101,U+6109,U+611a,U+6127,U+614c,U+6168,U+61be,U+61c8,U+6208,U+621a,U+6233,U+6241,U+6247,U+6251,U+6254,U+626d,U+626f,U+6284,U+6292,U+6296,U+629a,U+62ac,U+62bc,U+62cc,U+62d0,U+62d8-62d9,U+62e2,U+62e6,U+62ef,U+62f1,U+62f3,U+62fe,U+631a,U+6323,U+6328,U+632a-632b,U+633d,U+6349,U+634f-6350,U+635e,U+6361,U+6363,U+6367,U+6380,U+638f,U+6398,U+63a0,U+63c9,U+63e3,U+63fd,U+6401,U+6405,U+640f,U+6413,U+644a,U+6454,U+6467,U+6495,U+64a4,U+64b0,U+64bc,U+64ce,U+6500,U+6512,U+655b,U+655e,U+6566,U+6572,U+6577,U+658b-658c,U+65a5,U+65a7,U+65a9,U+65bc,U+65ec-65ed,U+65f1,U+65f7,U+6606,U+660f,U+6614,U+6627,U+662d,U+663c,U+6643,U+6652,U+6655,U+6674,U+6687,U+6691,U+66a8,U+66ae,U+66dd,U+66f9,U+673d,U+6749,U+674f,U+6756,U+6760,U+676d,U+6795,U+679a,U+67a2-67a3,U+67ab,U+67af,U+67c4,U+67cf,U+67dc,U+67e0,U+67ef,U+67f4,U+680b,U+6816-6817,U+682a,U+683d,U+6842,U+6850,U+6869,U+6876,U+6897,U+68a8,U+68ad,U+68b3,U+68b5,U+68c9,U+68cb,U+68cd,U+68d5,U+68da,U+68f1,U+68f5,U+6905,U+690e,U+6912,U+692d,U+6977,U+6984,U+69db,U+6a31,U+6a44,U+6a58-6a59,U+6a61,U+6a90,U+6b20,U+6b47,U+6b49,U+6b67,U+6b6a,U+6bb7,U+6bc5,U+6bd7,U+6bef,U+6c22,U+6c28,U+6c5d,U+6c6a,U+6c70,U+6c83,U+6c90,U+6c9b,U+6ca5-6ca7,U+6caa-6cab,U+6cb8,U+6cbe,U+6cca,U+6ccc,U+6ce3,U+6cf3,U+6cfb-6cfc,U+6d3d,U+6d46-6d47,U+6d4a,U+6d4f,U+6d51,U+6d66,U+6d74,U+6d85,U+6d9b,U+6da9,U+6dc0,U+6dd1,U+6deb,U+6dee,U+6df3,U+6df9,U+6e0a,U+6e14,U+6e17,U+6e23,U+6e32,U+6e58,U+6e5b,U+6e83,U+6e9c,U+6ea2,U+6eaf,U+6eb6,U+6ede,U+6ee4-6ee5,U+6ee8-6ee9,U+6f06,U+6f13,U+6f20,U+6f47,U+6f58,U+6f6d,U+6f84,U+6f88,U+6f9c,U+6fa1,U+7011,U+707c,U+707f,U+7096,U+70ab,U+70ad,U+70d8,U+70db,U+70e4,U+70eb,U+70f9,U+7109,U+7115,U+711a,U+7130,U+714c,U+714e,U+715e,U+7164,U+718f,U+7194,U+7199,U+71ac,U+722a,U+7235,U+7239,U+7261,U+7267,U+7272,U+727a,U+72ac,U+72d0,U+72e0,U+72ed-72ee,U+72fc,U+730e,U+7334,U+73ab,U+73b2,U+73ca,U+7410,U+7422,U+742a,U+7433,U+743c,U+7455,U+745c,U+745f,U+7470,U+7476,U+74e3,U+74f7,U+7529,U+752b,U+7538,U+754f,U+7554,U+755c,U+7574,U+7586,U+75ab,U+75ae,U+75d2,U+75f4,U+7624,U+763e,U+764c,U+7682,U+76b1,U+76ef,U+76f2,U+76fc,U+7737,U+7741,U+7750,U+7779,U+777f,U+778e,U+7792,U+77a7,U+77a9,U+77bb,U+77e2-77e3,U+77e9,U+77eb,U+77ee,U+77f6,U+7802,U+780c-780d,U+7816,U+7838,U+7845,U+786b,U+788c,U+789f,U+78a7,U+78b1,U+78b3,U+78c1,U+78c5,U+78f7,U+7940,U+7948,U+795b,U+796d,U+7977-7978,U+7984-7985,U+79bd,U+79c9,U+79e9,U+7a1a,U+7a20,U+7a3b,U+7a3d,U+7a46,U+7a74,U+7a83-7a84,U+7a8d,U+7a9d,U+7aa5,U+7ad6,U+7aed,U+7b1b,U+7b28,U+7b3c,U+7b4b,U+7b52,U+7b5b,U+7bad,U+7bee,U+7ca4-7ca5,U+7cb9,U+7cd5,U+7cd9,U+7cdf,U+7d6e,U+7eac,U+7eb1-7eb2,U+7eba,U+7ece,U+7ed1-7ed2,U+7eda,U+7ee3,U+7ef3,U+7ef8,U+7efd,U+7f00,U+7f05,U+7f14-7f15,U+7f1a,U+7f20,U+7f34,U+7f38,U+7f50,U+7f55,U+7f69,U+7f9e,U+7fa1,U+7fc1,U+7fc5,U+7fd4,U+7fd8,U+7fe0,U+7ff0,U+800d,U+8015,U+8036,U+8038,U+803b,U+803d,U+8086,U+808b,U+8096,U+809d,U+80a2,U+80aa,U+80ba,U+80be,U+80c0-80c1,U+80d6,U+810a,U+8116,U+813e,U+814a,U+8154-8155,U+8165,U+817a,U+8180,U+818f,U+819d,U+81a8,U+81c0,U+81c2,U+81ed,U+8206,U+820c,U+821f,U+8230,U+8247,U+8258,U+8292,U+8299,U+829d,U+82a6,U+82ac-82ad,U+82af,U+82bd,U+82d1,U+82d7,U+82db,U+82df,U+82f9,U+8302,U+8304-8305,U+830e,U+8328,U+832b,U+8346,U+836b,U+8389,U+838e,U+8393,U+83b9,U+83c7,U+83ca,U+83cc,U+83e9,U+8403-8404,U+840c-840e,U+841d,U+8427,U+845b,U+8461,U+846c,U+8471,U+8475,U+8482,U+848b,U+849c,U+84b2,U+84c4,U+84c9,U+84ec,U+8513,U+851a,U+8521,U+852c,U+853d,U+8549,U+8574,U+857e,U+8587,U+859b,U+85aa,U+85af,U+85c9,U+85e4,U+8650,U+8654,U+8679,U+867e,U+8680,U+86c7,U+8700,U+8702,U+8721,U+8774,U+8776,U+87ba,U+87f9,U+8822,U+884d,U+8854,U+886b-886c,U+8877,U+8881,U+888d,U+88d4-88d5,U+88d9,U+88e4,U+88f8-88f9,U+8910,U+89c5,U+8a79,U+8a93,U+8b6c,U+8bb3,U+8bb6,U+8bbc-8bbd,U+8bc0,U+8bc8,U+8be0-8be1,U+8bf5,U+8c05,U+8c0a,U+8c0e,U+8c1c,U+8c23,U+8c26,U+8c2d,U+8c41,U+8c6b,U+8c79,U+8d1e,U+8d29,U+8d2c,U+8d2e,U+8d31,U+8d37,U+8d3a,U+8d3c,U+8d3e,U+8d4c,U+8d50,U+8d54,U+8d81,U+8d9f,U+8dea,U+8e22,U+8e29,U+8e44,U+8e48,U+8e72,U+8e81,U+8eaf,U+8eb2,U+8eba,U+8f69,U+8f70,U+8f74,U+8f7f,U+8f90,U+8f96,U+8f9c,U+8f9f,U+8fb0-8fb1,U+8fbd,U+8fc4,U+8fe6,U+8fed,U+900a,U+9017,U+901b,U+902e,U+9038,U+903e,U+9042,U+9063,U+906e,U+9093,U+90ca,U+90e1,U+9119,U+9165,U+916c,U+9171,U+9175-9176,U+917f,U+9187,U+918b,U+9489,U+9493,U+9499,U+949d,U+94a5-94a6,U+94a9,U+94ae,U+94c3,U+94c5,U+94dd,U+94ed,U+94f8,U+9508,U+9521,U+9524-9525,U+953b,U+9576,U+95ef,U+95fa,U+9600-9601,U+9610,U+9640,U+964b-964c,U+9655,U+968b,U+9699,U+96b6,U+96c0-96c1,U+96c7,U+96cc-96cd,U+9709,U+970d,U+971c,U+971e,U+9753,U+9756,U+9761,U+97ad,U+97e6-97e7,U+9877,U+987d,U+9881-9882,U+9888,U+9896,U+98a0,U+98a4,U+9965,U+9972,U+9976,U+997c,U+997f,U+9985,U+9988,U+99a8,U+9a70,U+9a73,U+9a76,U+9a84,U+9a9a,U+9ad3,U+9b41,U+9b44,U+9b4f,U+9c8d,U+9cde,U+9e26,U+9e2d,U+9e3f,U+9e45,U+9e4f,U+9e64,U+9e70,U+9e7f,U+9e9f,U+9ecf,U+9edb,U+9f0e,U+9f9f,U+ff05;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:400;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.101.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.101.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Regular.101.otf')  format('opentype');unicode-range:U+0024,U+005c,U+005e,U+007b-007d,U+00b0,U+00d7,U+00e9,U+2013,U+2022,U+2103,U+2192,U+2460-2463,U+300e-300f,U+306e,U+4e18-4e19,U+4e2b,U+4e38,U+4e56,U+4e59,U+4e5e,U+4e7e,U+4ea5,U+4ea8-4ea9,U+4ead,U+4ec6-4ec7,U+4ed1,U+4ed5,U+4ed7,U+4ef2,U+4f10,U+4f1e,U+4f3d,U+4f50-4f51,U+4f63,U+4f83,U+4f88,U+4f8d,U+4fa0,U+4fa3,U+4fa6,U+4fa8,U+4faf,U+4fcf,U+4fd8,U+4fed,U+4fef,U+4ffa,U+501a,U+5021,U+5026,U+503a,U+5085,U+508d,U+50ac,U+50e7,U+50f5,U+50fb,U+5112,U+5146,U+5151,U+5154,U+515c,U+5179,U+5188,U+51a4-51a5,U+51af,U+51b6,U+51bb,U+51c4,U+51d1,U+51f0,U+51f8-51f9,U+51fd,U+51ff,U+5203,U+522e,U+5238-5239,U+524a,U+5254,U+5256,U+5265,U+5288,U+52ab,U+52c3,U+52c9,U+52cb,U+52d8,U+52df,U+52fa,U+52ff,U+5306,U+5319,U+5320,U+532a,U+533f,U+5349,U+5351-5352,U+535c,U+5362,U+5366,U+5375,U+5378,U+537f,U+5384,U+5395,U+53a2,U+53a6,U+53a8,U+53c9,U+53d4,U+53d9,U+53db,U+53ee,U+5401,U+540a,U+540f,U+5415,U+541e-541f,U+5428,U+5435,U+543b-543c,U+543e,U+5450,U+5455,U+548b,U+548f,U+5492,U+54ac,U+54af,U+54b1,U+54b3,U+54b8,U+54bd,U+54c0,U+54c4,U+54c7,U+54c9,U+54ce,U+54d1,U+54df,U+5507,U+5509,U+5524,U+5561,U+5564,U+5578,U+5582,U+5587,U+5589,U+5598,U+55a7,U+55bb,U+55d3,U+55dc,U+55ef,U+5631-5632,U+563f,U+566a,U+56a3,U+56bc,U+56ca,U+56da,U+5733,U+573e,U+574a,U+574e,U+575d-5760,U+5764,U+576a,U+5783-5784,U+5792,U+57ab,U+5815,U+5824,U+5835,U+584c,U+5858,U+5885,U+5893,U+58e4,U+58f6,U+5937,U+5951,U+5960,U+5962,U+5974,U+5978,U+5983-5984,U+5992,U+5996,U+59a5,U+59a8,U+59ae,U+59da,U+59e8,U+59ec,U+5a03,U+5a07,U+5a1c,U+5a36,U+5a49,U+5a74,U+5a9a,U+5ab3,U+5ac9,U+5b5d,U+5b5f,U+5b99,U+5b9b,U+5ba0,U+5baa,U+5bb0,U+5bb4-5bb5,U+5bc5,U+5bdd-5bde,U+5be1,U+5be8,U+5c2c,U+5c34,U+5c38-5c39,U+5c41,U+5c51,U+5c60-5c61,U+5c65,U+5c6f,U+5c82,U+5cad,U+5cb3,U+5ce1,U+5cfb,U+5d0e,U+5d14,U+5d16,U+5d1b,U+5d29,U+5d2d,U+5d4c,U+5dc5,U+5de2,U+5de9,U+5deb,U+5df7,U+5dfe,U+5e06,U+5e10,U+5e15-5e16,U+5e18,U+5e1c,U+5e27,U+5e3d,U+5e90,U+5e9a,U+5e9e,U+5eb8,U+5eca,U+5ed3,U+5f0a,U+5f13,U+5f17-5f18,U+5f26-5f27,U+5f66,U+5f6d,U+5f70,U+5f8c,U+6016,U+601c,U+6021,U+604d,U+606d,U+6073,U+6084,U+608d,U+60d5,U+60df,U+60e9,U+60f9,U+6101,U+6109,U+611a,U+6127,U+614c,U+6168,U+61be,U+61c8,U+6208,U+621a,U+6233,U+6241,U+6247,U+6251,U+6254,U+626d,U+626f,U+6284,U+6292,U+6296,U+629a,U+62ac,U+62bc,U+62cc,U+62d0,U+62d8-62d9,U+62e2,U+62e6,U+62ef,U+62f1,U+62f3,U+62fe,U+631a,U+6323,U+6328,U+632a-632b,U+633d,U+6349,U+634f-6350,U+635e,U+6361,U+6363,U+6367,U+6380,U+638f,U+6398,U+63a0,U+63c9,U+63e3,U+63fd,U+6401,U+6405,U+640f,U+6413,U+644a,U+6454,U+6467,U+6495,U+64a4,U+64b0,U+64bc,U+64ce,U+6500,U+6512,U+655b,U+655e,U+6566,U+6572,U+6577,U+658b-658c,U+65a5,U+65a7,U+65a9,U+65bc,U+65ec-65ed,U+65f1,U+65f7,U+6606,U+660f,U+6614,U+6627,U+662d,U+663c,U+6643,U+6652,U+6655,U+6674,U+6687,U+6691,U+66a8,U+66ae,U+66dd,U+66f9,U+673d,U+6749,U+674f,U+6756,U+6760,U+676d,U+6795,U+679a,U+67a2-67a3,U+67ab,U+67af,U+67c4,U+67cf,U+67dc,U+67e0,U+67ef,U+67f4,U+680b,U+6816-6817,U+682a,U+683d,U+6842,U+6850,U+6869,U+6876,U+6897,U+68a8,U+68ad,U+68b3,U+68b5,U+68c9,U+68cb,U+68cd,U+68d5,U+68da,U+68f1,U+68f5,U+6905,U+690e,U+6912,U+692d,U+6977,U+6984,U+69db,U+6a31,U+6a44,U+6a58-6a59,U+6a61,U+6a90,U+6b20,U+6b47,U+6b49,U+6b67,U+6b6a,U+6bb7,U+6bc5,U+6bd7,U+6bef,U+6c22,U+6c28,U+6c5d,U+6c6a,U+6c70,U+6c83,U+6c90,U+6c9b,U+6ca5-6ca7,U+6caa-6cab,U+6cb8,U+6cbe,U+6cca,U+6ccc,U+6ce3,U+6cf3,U+6cfb-6cfc,U+6d3d,U+6d46-6d47,U+6d4a,U+6d4f,U+6d51,U+6d66,U+6d74,U+6d85,U+6d9b,U+6da9,U+6dc0,U+6dd1,U+6deb,U+6dee,U+6df3,U+6df9,U+6e0a,U+6e14,U+6e17,U+6e23,U+6e32,U+6e58,U+6e5b,U+6e83,U+6e9c,U+6ea2,U+6eaf,U+6eb6,U+6ede,U+6ee4-6ee5,U+6ee8-6ee9,U+6f06,U+6f13,U+6f20,U+6f47,U+6f58,U+6f6d,U+6f84,U+6f88,U+6f9c,U+6fa1,U+7011,U+707c,U+707f,U+7096,U+70ab,U+70ad,U+70d8,U+70db,U+70e4,U+70eb,U+70f9,U+7109,U+7115,U+711a,U+7130,U+714c,U+714e,U+715e,U+7164,U+718f,U+7194,U+7199,U+71ac,U+722a,U+7235,U+7239,U+7261,U+7267,U+7272,U+727a,U+72ac,U+72d0,U+72e0,U+72ed-72ee,U+72fc,U+730e,U+7334,U+73ab,U+73b2,U+73ca,U+7410,U+7422,U+742a,U+7433,U+743c,U+7455,U+745c,U+745f,U+7470,U+7476,U+74e3,U+74f7,U+7529,U+752b,U+7538,U+754f,U+7554,U+755c,U+7574,U+7586,U+75ab,U+75ae,U+75d2,U+75f4,U+7624,U+763e,U+764c,U+7682,U+76b1,U+76ef,U+76f2,U+76fc,U+7737,U+7741,U+7750,U+7779,U+777f,U+778e,U+7792,U+77a7,U+77a9,U+77bb,U+77e2-77e3,U+77e9,U+77eb,U+77ee,U+77f6,U+7802,U+780c-780d,U+7816,U+7838,U+7845,U+786b,U+788c,U+789f,U+78a7,U+78b1,U+78b3,U+78c1,U+78c5,U+78f7,U+7940,U+7948,U+795b,U+796d,U+7977-7978,U+7984-7985,U+79bd,U+79c9,U+79e9,U+7a1a,U+7a20,U+7a3b,U+7a3d,U+7a46,U+7a74,U+7a83-7a84,U+7a8d,U+7a9d,U+7aa5,U+7ad6,U+7aed,U+7b1b,U+7b28,U+7b3c,U+7b4b,U+7b52,U+7b5b,U+7bad,U+7bee,U+7ca4-7ca5,U+7cb9,U+7cd5,U+7cd9,U+7cdf,U+7d6e,U+7eac,U+7eb1-7eb2,U+7eba,U+7ece,U+7ed1-7ed2,U+7eda,U+7ee3,U+7ef3,U+7ef8,U+7efd,U+7f00,U+7f05,U+7f14-7f15,U+7f1a,U+7f20,U+7f34,U+7f38,U+7f50,U+7f55,U+7f69,U+7f9e,U+7fa1,U+7fc1,U+7fc5,U+7fd4,U+7fd8,U+7fe0,U+7ff0,U+800d,U+8015,U+8036,U+8038,U+803b,U+803d,U+8086,U+808b,U+8096,U+809d,U+80a2,U+80aa,U+80ba,U+80be,U+80c0-80c1,U+80d6,U+810a,U+8116,U+813e,U+814a,U+8154-8155,U+8165,U+817a,U+8180,U+818f,U+819d,U+81a8,U+81c0,U+81c2,U+81ed,U+8206,U+820c,U+821f,U+8230,U+8247,U+8258,U+8292,U+8299,U+829d,U+82a6,U+82ac-82ad,U+82af,U+82bd,U+82d1,U+82d7,U+82db,U+82df,U+82f9,U+8302,U+8304-8305,U+830e,U+8328,U+832b,U+8346,U+836b,U+8389,U+838e,U+8393,U+83b9,U+83c7,U+83ca,U+83cc,U+83e9,U+8403-8404,U+840c-840e,U+841d,U+8427,U+845b,U+8461,U+846c,U+8471,U+8475,U+8482,U+848b,U+849c,U+84b2,U+84c4,U+84c9,U+84ec,U+8513,U+851a,U+8521,U+852c,U+853d,U+8549,U+8574,U+857e,U+8587,U+859b,U+85aa,U+85af,U+85c9,U+85e4,U+8650,U+8654,U+8679,U+867e,U+8680,U+86c7,U+8700,U+8702,U+8721,U+8774,U+8776,U+87ba,U+87f9,U+8822,U+884d,U+8854,U+886b-886c,U+8877,U+8881,U+888d,U+88d4-88d5,U+88d9,U+88e4,U+88f8-88f9,U+8910,U+89c5,U+8a79,U+8a93,U+8b6c,U+8bb3,U+8bb6,U+8bbc-8bbd,U+8bc0,U+8bc8,U+8be0-8be1,U+8bf5,U+8c05,U+8c0a,U+8c0e,U+8c1c,U+8c23,U+8c26,U+8c2d,U+8c41,U+8c6b,U+8c79,U+8d1e,U+8d29,U+8d2c,U+8d2e,U+8d31,U+8d37,U+8d3a,U+8d3c,U+8d3e,U+8d4c,U+8d50,U+8d54,U+8d81,U+8d9f,U+8dea,U+8e22,U+8e29,U+8e44,U+8e48,U+8e72,U+8e81,U+8eaf,U+8eb2,U+8eba,U+8f69,U+8f70,U+8f74,U+8f7f,U+8f90,U+8f96,U+8f9c,U+8f9f,U+8fb0-8fb1,U+8fbd,U+8fc4,U+8fe6,U+8fed,U+900a,U+9017,U+901b,U+902e,U+9038,U+903e,U+9042,U+9063,U+906e,U+9093,U+90ca,U+90e1,U+9119,U+9165,U+916c,U+9171,U+9175-9176,U+917f,U+9187,U+918b,U+9489,U+9493,U+9499,U+949d,U+94a5-94a6,U+94a9,U+94ae,U+94c3,U+94c5,U+94dd,U+94ed,U+94f8,U+9508,U+9521,U+9524-9525,U+953b,U+9576,U+95ef,U+95fa,U+9600-9601,U+9610,U+9640,U+964b-964c,U+9655,U+968b,U+9699,U+96b6,U+96c0-96c1,U+96c7,U+96cc-96cd,U+9709,U+970d,U+971c,U+971e,U+9753,U+9756,U+9761,U+97ad,U+97e6-97e7,U+9877,U+987d,U+9881-9882,U+9888,U+9896,U+98a0,U+98a4,U+9965,U+9972,U+9976,U+997c,U+997f,U+9985,U+9988,U+99a8,U+9a70,U+9a73,U+9a76,U+9a84,U+9a9a,U+9ad3,U+9b41,U+9b44,U+9b4f,U+9c8d,U+9cde,U+9e26,U+9e2d,U+9e3f,U+9e45,U+9e4f,U+9e64,U+9e70,U+9e7f,U+9e9f,U+9ecf,U+9edb,U+9f0e,U+9f9f,U+ff05;}\n@font-face{font-family:'Noto Sans SC Sliced';font-weight:500;src:url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.101.woff2') format('woff2'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.101.woff') format('woff'),url('//fonts.gstatic.com/ea/notosansscsliced/v2/NotoSansSCSliced-Medium.101.otf')  format('opentype');unicode-range:U+0024,U+005c,U+005e,U+007b-007d,U+00b0,U+00d7,U+00e9,U+2013,U+2022,U+2103,U+2192,U+2460-2463,U+300e-300f,U+306e,U+4e18-4e19,U+4e2b,U+4e38,U+4e56,U+4e59,U+4e5e,U+4e7e,U+4ea5,U+4ea8-4ea9,U+4ead,U+4ec6-4ec7,U+4ed1,U+4ed5,U+4ed7,U+4ef2,U+4f10,U+4f1e,U+4f3d,U+4f50-4f51,U+4f63,U+4f83,U+4f88,U+4f8d,U+4fa0,U+4fa3,U+4fa6,U+4fa8,U+4faf,U+4fcf,U+4fd8,U+4fed,U+4fef,U+4ffa,U+501a,U+5021,U+5026,U+503a,U+5085,U+508d,U+50ac,U+50e7,U+50f5,U+50fb,U+5112,U+5146,U+5151,U+5154,U+515c,U+5179,U+5188,U+51a4-51a5,U+51af,U+51b6,U+51bb,U+51c4,U+51d1,U+51f0,U+51f8-51f9,U+51fd,U+51ff,U+5203,U+522e,U+5238-5239,U+524a,U+5254,U+5256,U+5265,U+5288,U+52ab,U+52c3,U+52c9,U+52cb,U+52d8,U+52df,U+52fa,U+52ff,U+5306,U+5319,U+5320,U+532a,U+533f,U+5349,U+5351-5352,U+535c,U+5362,U+5366,U+5375,U+5378,U+537f,U+5384,U+5395,U+53a2,U+53a6,U+53a8,U+53c9,U+53d4,U+53d9,U+53db,U+53ee,U+5401,U+540a,U+540f,U+5415,U+541e-541f,U+5428,U+5435,U+543b-543c,U+543e,U+5450,U+5455,U+548b,U+548f,U+5492,U+54ac,U+54af,U+54b1,U+54b3,U+54b8,U+54bd,U+54c0,U+54c4,U+54c7,U+54c9,U+54ce,U+54d1,U+54df,U+5507,U+5509,U+5524,U+5561,U+5564,U+5578,U+5582,U+5587,U+5589,U+5598,U+55a7,U+55bb,U+55d3,U+55dc,U+55ef,U+5631-5632,U+563f,U+566a,U+56a3,U+56bc,U+56ca,U+56da,U+5733,U+573e,U+574a,U+574e,U+575d-5760,U+5764,U+576a,U+5783-5784,U+5792,U+57ab,U+5815,U+5824,U+5835,U+584c,U+5858,U+5885,U+5893,U+58e4,U+58f6,U+5937,U+5951,U+5960,U+5962,U+5974,U+5978,U+5983-5984,U+5992,U+5996,U+59a5,U+59a8,U+59ae,U+59da,U+59e8,U+59ec,U+5a03,U+5a07,U+5a1c,U+5a36,U+5a49,U+5a74,U+5a9a,U+5ab3,U+5ac9,U+5b5d,U+5b5f,U+5b99,U+5b9b,U+5ba0,U+5baa,U+5bb0,U+5bb4-5bb5,U+5bc5,U+5bdd-5bde,U+5be1,U+5be8,U+5c2c,U+5c34,U+5c38-5c39,U+5c41,U+5c51,U+5c60-5c61,U+5c65,U+5c6f,U+5c82,U+5cad,U+5cb3,U+5ce1,U+5cfb,U+5d0e,U+5d14,U+5d16,U+5d1b,U+5d29,U+5d2d,U+5d4c,U+5dc5,U+5de2,U+5de9,U+5deb,U+5df7,U+5dfe,U+5e06,U+5e10,U+5e15-5e16,U+5e18,U+5e1c,U+5e27,U+5e3d,U+5e90,U+5e9a,U+5e9e,U+5eb8,U+5eca,U+5ed3,U+5f0a,U+5f13,U+5f17-5f18,U+5f26-5f27,U+5f66,U+5f6d,U+5f70,U+5f8c,U+6016,U+601c,U+6021,U+604d,U+606d,U+6073,U+6084,U+608d,U+60d5,U+60df,U+60e9,U+60f9,U+6101,U+6109,U+611a,U+6127,U+614c,U+6168,U+61be,U+61c8,U+6208,U+621a,U+6233,U+6241,U+6247,U+6251,U+6254,U+626d,U+626f,U+6284,U+6292,U+6296,U+629a,U+62ac,U+62bc,U+62cc,U+62d0,U+62d8-62d9,U+62e2,U+62e6,U+62ef,U+62f1,U+62f3,U+62fe,U+631a,U+6323,U+6328,U+632a-632b,U+633d,U+6349,U+634f-6350,U+635e,U+6361,U+6363,U+6367,U+6380,U+638f,U+6398,U+63a0,U+63c9,U+63e3,U+63fd,U+6401,U+6405,U+640f,U+6413,U+644a,U+6454,U+6467,U+6495,U+64a4,U+64b0,U+64bc,U+64ce,U+6500,U+6512,U+655b,U+655e,U+6566,U+6572,U+6577,U+658b-658c,U+65a5,U+65a7,U+65a9,U+65bc,U+65ec-65ed,U+65f1,U+65f7,U+6606,U+660f,U+6614,U+6627,U+662d,U+663c,U+6643,U+6652,U+6655,U+6674,U+6687,U+6691,U+66a8,U+66ae,U+66dd,U+66f9,U+673d,U+6749,U+674f,U+6756,U+6760,U+676d,U+6795,U+679a,U+67a2-67a3,U+67ab,U+67af,U+67c4,U+67cf,U+67dc,U+67e0,U+67ef,U+67f4,U+680b,U+6816-6817,U+682a,U+683d,U+6842,U+6850,U+6869,U+6876,U+6897,U+68a8,U+68ad,U+68b3,U+68b5,U+68c9,U+68cb,U+68cd,U+68d5,U+68da,U+68f1,U+68f5,U+6905,U+690e,U+6912,U+692d,U+6977,U+6984,U+69db,U+6a31,U+6a44,U+6a58-6a59,U+6a61,U+6a90,U+6b20,U+6b47,U+6b49,U+6b67,U+6b6a,U+6bb7,U+6bc5,U+6bd7,U+6bef,U+6c22,U+6c28,U+6c5d,U+6c6a,U+6c70,U+6c83,U+6c90,U+6c9b,U+6ca5-6ca7,U+6caa-6cab,U+6cb8,U+6cbe,U+6cca,U+6ccc,U+6ce3,U+6cf3,U+6cfb-6cfc,U+6d3d,U+6d46-6d47,U+6d4a,U+6d4f,U+6d51,U+6d66,U+6d74,U+6d85,U+6d9b,U+6da9,U+6dc0,U+6dd1,U+6deb,U+6dee,U+6df3,U+6df9,U+6e0a,U+6e14,U+6e17,U+6e23,U+6e32,U+6e58,U+6e5b,U+6e83,U+6e9c,U+6ea2,U+6eaf,U+6eb6,U+6ede,U+6ee4-6ee5,U+6ee8-6ee9,U+6f06,U+6f13,U+6f20,U+6f47,U+6f58,U+6f6d,U+6f84,U+6f88,U+6f9c,U+6fa1,U+7011,U+707c,U+707f,U+7096,U+70ab,U+70ad,U+70d8,U+70db,U+70e4,U+70eb,U+70f9,U+7109,U+7115,U+711a,U+7130,U+714c,U+714e,U+715e,U+7164,U+718f,U+7194,U+7199,U+71ac,U+722a,U+7235,U+7239,U+7261,U+7267,U+7272,U+727a,U+72ac,U+72d0,U+72e0,U+72ed-72ee,U+72fc,U+730e,U+7334,U+73ab,U+73b2,U+73ca,U+7410,U+7422,U+742a,U+7433,U+743c,U+7455,U+745c,U+745f,U+7470,U+7476,U+74e3,U+74f7,U+7529,U+752b,U+7538,U+754f,U+7554,U+755c,U+7574,U+7586,U+75ab,U+75ae,U+75d2,U+75f4,U+7624,U+763e,U+764c,U+7682,U+76b1,U+76ef,U+76f2,U+76fc,U+7737,U+7741,U+7750,U+7779,U+777f,U+778e,U+7792,U+77a7,U+77a9,U+77bb,U+77e2-77e3,U+77e9,U+77eb,U+77ee,U+77f6,U+7802,U+780c-780d,U+7816,U+7838,U+7845,U+786b,U+788c,U+789f,U+78a7,U+78b1,U+78b3,U+78c1,U+78c5,U+78f7,U+7940,U+7948,U+795b,U+796d,U+7977-7978,U+7984-7985,U+79bd,U+79c9,U+79e9,U+7a1a,U+7a20,U+7a3b,U+7a3d,U+7a46,U+7a74,U+7a83-7a84,U+7a8d,U+7a9d,U+7aa5,U+7ad6,U+7aed,U+7b1b,U+7b28,U+7b3c,U+7b4b,U+7b52,U+7b5b,U+7bad,U+7bee,U+7ca4-7ca5,U+7cb9,U+7cd5,U+7cd9,U+7cdf,U+7d6e,U+7eac,U+7eb1-7eb2,U+7eba,U+7ece,U+7ed1-7ed2,U+7eda,U+7ee3,U+7ef3,U+7ef8,U+7efd,U+7f00,U+7f05,U+7f14-7f15,U+7f1a,U+7f20,U+7f34,U+7f38,U+7f50,U+7f55,U+7f69,U+7f9e,U+7fa1,U+7fc1,U+7fc5,U+7fd4,U+7fd8,U+7fe0,U+7ff0,U+800d,U+8015,U+8036,U+8038,U+803b,U+803d,U+8086,U+808b,U+8096,U+809d,U+80a2,U+80aa,U+80ba,U+80be,U+80c0-80c1,U+80d6,U+810a,U+8116,U+813e,U+814a,U+8154-8155,U+8165,U+817a,U+8180,U+818f,U+819d,U+81a8,U+81c0,U+81c2,U+81ed,U+8206,U+820c,U+821f,U+8230,U+8247,U+8258,U+8292,U+8299,U+829d,U+82a6,U+82ac-82ad,U+82af,U+82bd,U+82d1,U+82d7,U+82db,U+82df,U+82f9,U+8302,U+8304-8305,U+830e,U+8328,U+832b,U+8346,U+836b,U+8389,U+838e,U+8393,U+83b9,U+83c7,U+83ca,U+83cc,U+83e9,U+8403-8404,U+840c-840e,U+841d,U+8427,U+845b,U+8461,U+846c,U+8471,U+8475,U+8482,U+848b,U+849c,U+84b2,U+84c4,U+84c9,U+84ec,U+8513,U+851a,U+8521,U+852c,U+853d,U+8549,U+8574,U+857e,U+8587,U+859b,U+85aa,U+85af,U+85c9,U+85e4,U+8650,U+8654,U+8679,U+867e,U+8680,U+86c7,U+8700,U+8702,U+8721,U+8774,U+8776,U+87ba,U+87f9,U+8822,U+884d,U+8854,U+886b-886c,U+8877,U+8881,U+888d,U+88d4-88d5,U+88d9,U+88e4,U+88f8-88f9,U+8910,U+89c5,U+8a79,U+8a93,U+8b6c,U+8bb3,U+8bb6,U+8bbc-8bbd,U+8bc0,U+8bc8,U+8be0-8be1,U+8bf5,U+8c05,U+8c0a,U+8c0e,U+8c1c,U+8c23,U+8c26,U+8c2d,U+8c41,U+8c6b,U+8c79,U+8d1e,U+8d29,U+8d2c,U+8d2e,U+8d31,U+8d37,U+8d3a,U+8d3c,U+8d3e,U+8d4c,U+8d50,U+8d54,U+8d81,U+8d9f,U+8dea,U+8e22,U+8e29,U+8e44,U+8e48,U+8e72,U+8e81,U+8eaf,U+8eb2,U+8eba,U+8f69,U+8f70,U+8f74,U+8f7f,U+8f90,U+8f96,U+8f9c,U+8f9f,U+8fb0-8fb1,U+8fbd,U+8fc4,U+8fe6,U+8fed,U+900a,U+9017,U+901b,U+902e,U+9038,U+903e,U+9042,U+9063,U+906e,U+9093,U+90ca,U+90e1,U+9119,U+9165,U+916c,U+9171,U+9175-9176,U+917f,U+9187,U+918b,U+9489,U+9493,U+9499,U+949d,U+94a5-94a6,U+94a9,U+94ae,U+94c3,U+94c5,U+94dd,U+94ed,U+94f8,U+9508,U+9521,U+9524-9525,U+953b,U+9576,U+95ef,U+95fa,U+9600-9601,U+9610,U+9640,U+964b-964c,U+9655,U+968b,U+9699,U+96b6,U+96c0-96c1,U+96c7,U+96cc-96cd,U+9709,U+970d,U+971c,U+971e,U+9753,U+9756,U+9761,U+97ad,U+97e6-97e7,U+9877,U+987d,U+9881-9882,U+9888,U+9896,U+98a0,U+98a4,U+9965,U+9972,U+9976,U+997c,U+997f,U+9985,U+9988,U+99a8,U+9a70,U+9a73,U+9a76,U+9a84,U+9a9a,U+9ad3,U+9b41,U+9b44,U+9b4f,U+9c8d,U+9cde,U+9e26,U+9e2d,U+9e3f,U+9e45,U+9e4f,U+9e64,U+9e70,U+9e7f,U+9e9f,U+9ecf,U+9edb,U+9f0e,U+9f9f,U+ff05;}\n\n@font-face {\n  font-family: 'Material Icons';\n  font-style: normal;\n  font-weight: 400;\n  src: url(material-design-icons-3.0.1/iconfont/MaterialIcons-Regular.eot); /* For IE6-8 */\n  src: local('Material Icons'),\n       local('MaterialIcons-Regular'),\n       url(material-design-icons-3.0.1/iconfont/MaterialIcons-Regular.woff2) format('woff2'),\n       url(material-design-icons-3.0.1/iconfont/MaterialIcons-Regular.woff) format('woff'),\n       url(material-design-icons-3.0.1/iconfont/MaterialIcons-Regular.ttf) format('truetype');\n}\n\n/*\n* Noto Sans Japanese (japanese) http://www.google.com/fonts/earlyaccess\n */\n@font-face {\n  font-family: 'Noto Sans Japanese';\n  font-style: normal;\n  font-weight: 100;\n  src: url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Thin.woff2) format('woff2'),\n       url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Thin.woff) format('woff'),\n       url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Thin.otf) format('opentype');\n}\n@font-face {\n  font-family: 'Noto Sans Japanese';\n  font-style: normal;\n  font-weight: 200;\n  src: url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Light.woff2) format('woff2'),\n       url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Light.woff) format('woff'),\n       url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Light.otf) format('opentype');\n}\n@font-face {\n   font-family: 'Noto Sans Japanese';\n   font-style: normal;\n   font-weight: 300;\n   src: url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-DemiLight.woff2) format('woff2'),\n        url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-DemiLight.woff) format('woff'),\n        url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-DemiLight.otf) format('opentype');\n}\n@font-face {\n   font-family: 'Noto Sans Japanese';\n   font-style: normal;\n   font-weight: 400;\n   src: url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Regular.woff2) format('woff2'),\n        url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Regular.woff) format('woff'),\n        url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Regular.otf) format('opentype');\n }\n@font-face {\n   font-family: 'Noto Sans Japanese';\n   font-style: normal;\n   font-weight: 500;\n   src: url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Medium.woff2) format('woff2'),\n        url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Medium.woff) format('woff'),\n        url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Medium.otf) format('opentype');\n }\n@font-face {\n   font-family: 'Noto Sans Japanese';\n   font-style: normal;\n   font-weight: 700;\n   src: url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Bold.woff2) format('woff2'),\n        url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Bold.woff) format('woff'),\n        url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Bold.otf) format('opentype');\n }\n@font-face {\n   font-family: 'Noto Sans Japanese';\n   font-style: normal;\n   font-weight: 900;\n   src: url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Black.woff2) format('woff2'),\n        url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Black.woff) format('woff'),\n        url(//fonts.gstatic.com/ea/notosansjapanese/v6/NotoSansJP-Black.otf) format('opentype');\n }\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/static/sphinx_materialdesign_theme.css",
    "content": ".admonition,.mdl-shadow--2dp,.page-content pre:hover,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list){box-shadow:0 2px 2px 0 rgba(0,0,0,.14),0 3px 1px -2px rgba(0,0,0,.2),0 1px 5px 0 rgba(0,0,0,.12)}.mdl-shadow--3dp{box-shadow:0 3px 4px 0 rgba(0,0,0,.14),0 3px 3px -2px rgba(0,0,0,.2),0 1px 8px 0 rgba(0,0,0,.12)}.mdl-shadow--4dp{box-shadow:0 4px 5px 0 rgba(0,0,0,.14),0 1px 10px 0 rgba(0,0,0,.12),0 2px 4px -1px rgba(0,0,0,.2)}.mdl-shadow--6dp{box-shadow:0 6px 10px 0 rgba(0,0,0,.14),0 1px 18px 0 rgba(0,0,0,.12),0 3px 5px -1px rgba(0,0,0,.2)}.mdl-shadow--8dp{box-shadow:0 8px 10px 1px rgba(0,0,0,.14),0 3px 14px 2px rgba(0,0,0,.12),0 5px 5px -3px rgba(0,0,0,.2)}.mdl-shadow--16dp{box-shadow:0 16px 24px 2px rgba(0,0,0,.14),0 6px 30px 5px rgba(0,0,0,.12),0 8px 10px -5px rgba(0,0,0,.2)}.mdl-shadow--24dp{box-shadow:0 9px 46px 8px rgba(0,0,0,.14),0 11px 15px -7px rgba(0,0,0,.12),0 24px 38px 3px rgba(0,0,0,.2)}.mdl-data-table,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list){position:relative;border:1px solid rgba(0,0,0,.12);border-collapse:collapse;white-space:nowrap;font-size:13px;background-color:#fff}.mdl-data-table thead,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) thead{padding-bottom:3px}.mdl-data-table thead .mdl-data-table__select,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) thead .mdl-data-table__select{margin-top:0}.mdl-data-table tbody tr,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) tbody tr{position:relative;height:48px;transition-duration:.28s;transition-timing-function:cubic-bezier(.4,0,.2,1);transition-property:background-color}.mdl-data-table tbody tr.is-selected,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) tbody tr.is-selected{background-color:#e0e0e0}.mdl-data-table tbody tr:hover,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) tbody tr:hover{background-color:#eee}.mdl-data-table td,.mdl-data-table th,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) td,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) th{padding:0 18px 12px;text-align:right}.mdl-data-table td:first-of-type,.mdl-data-table th:first-of-type,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) td:first-of-type,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) th:first-of-type{padding-left:24px}.mdl-data-table td:last-of-type,.mdl-data-table th:last-of-type,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) td:last-of-type,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) th:last-of-type{padding-right:24px}.mdl-data-table td,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) td{position:relative;vertical-align:middle;height:48px;border-top:1px solid rgba(0,0,0,.12);border-bottom:1px solid rgba(0,0,0,.12);padding-top:12px;box-sizing:border-box}.mdl-data-table td .mdl-data-table__select,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) td .mdl-data-table__select{vertical-align:middle}.mdl-data-table th,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) th{position:relative;vertical-align:bottom;text-overflow:ellipsis;font-size:14px;font-weight:700;line-height:24px;letter-spacing:0;height:48px;font-size:12px;color:rgba(0,0,0,.54);padding-bottom:8px;box-sizing:border-box}.mdl-data-table th.mdl-data-table__header--sorted-ascending,.mdl-data-table th.mdl-data-table__header--sorted-descending,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) th.mdl-data-table__header--sorted-ascending,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) th.mdl-data-table__header--sorted-descending{color:rgba(0,0,0,.87)}.mdl-data-table th.mdl-data-table__header--sorted-ascending:before,.mdl-data-table th.mdl-data-table__header--sorted-descending:before,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) th.mdl-data-table__header--sorted-ascending:before,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) th.mdl-data-table__header--sorted-descending:before{font-family:Material Icons;font-weight:400;font-style:normal;font-size:24px;line-height:1;letter-spacing:normal;text-transform:none;display:inline-block;word-wrap:normal;font-feature-settings:\"liga\";-webkit-font-feature-settings:\"liga\";-webkit-font-smoothing:antialiased;font-size:16px;content:\"\\e5d8\";margin-right:5px;vertical-align:sub}.mdl-data-table th.mdl-data-table__header--sorted-ascending:hover,.mdl-data-table th.mdl-data-table__header--sorted-descending:hover,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) th.mdl-data-table__header--sorted-ascending:hover,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) th.mdl-data-table__header--sorted-descending:hover{cursor:pointer}.mdl-data-table th.mdl-data-table__header--sorted-ascending:hover:before,.mdl-data-table th.mdl-data-table__header--sorted-descending:hover:before,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) th.mdl-data-table__header--sorted-ascending:hover:before,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) th.mdl-data-table__header--sorted-descending:hover:before{color:rgba(0,0,0,.26)}.mdl-data-table th.mdl-data-table__header--sorted-descending:before,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) th.mdl-data-table__header--sorted-descending:before{content:\"\\e5db\"}.mdl-data-table__select{width:16px}.mdl-data-table__cell--non-numeric.mdl-data-table__cell--non-numeric,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) td,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) th{text-align:left}.mdl-mini-footer{display:flex;flex-flow:row wrap;justify-content:space-between;padding:32px 16px;color:#9e9e9e;background-color:#424242}.mdl-mini-footer:after{content:\"\";display:block}.mdl-mini-footer .mdl-logo{line-height:36px}.mdl-mini-footer--link-list,.mdl-mini-footer__link-list,footer.mdl-mini-footer>div.mdl-mini-footer__left-section ul{display:flex;flex-flow:row nowrap;list-style:none;margin:0;padding:0}.mdl-mini-footer--link-list li,.mdl-mini-footer__link-list li,footer.mdl-mini-footer>div.mdl-mini-footer__left-section ul li{margin-bottom:0;margin-right:16px}@media screen and (min-width:760px){.mdl-mini-footer--link-list li,.mdl-mini-footer__link-list li,footer.mdl-mini-footer>div.mdl-mini-footer__left-section ul li{line-height:36px}}.mdl-mini-footer--link-list a,.mdl-mini-footer__link-list a,footer.mdl-mini-footer>div.mdl-mini-footer__left-section ul a{color:inherit;text-decoration:none;white-space:nowrap}.mdl-mini-footer--left-section,.mdl-mini-footer__left-section{display:inline-block;order:0}.mdl-mini-footer--right-section,.mdl-mini-footer__right-section{display:inline-block;order:1}.mdl-mini-footer--social-btn,.mdl-mini-footer__social-btn{width:36px;height:36px;padding:0;margin:0;background-color:#9e9e9e;border:none}.mdl-card{display:flex;flex-direction:column;font-size:16px;font-weight:400;min-height:200px;overflow:hidden;width:330px;z-index:1;position:relative;background:#fff;border-radius:2px;box-sizing:border-box}.mdl-card__media{background-color:#ff6e40;background-repeat:repeat;background-position:50% 50%;background-size:cover;background-origin:padding-box;background-attachment:scroll;box-sizing:border-box}.mdl-card__title{align-items:center;color:#000;display:block;display:flex;justify-content:stretch;line-height:normal;padding:16px;perspective-origin:165px 56px;transform-origin:165px 56px;box-sizing:border-box}.mdl-card__title.mdl-card--border{border-bottom:1px solid rgba(0,0,0,.1)}.mdl-card__title-text{align-self:flex-end;color:inherit;display:block;display:flex;font-size:24px;font-weight:300;line-height:normal;overflow:hidden;transform-origin:149px 48px;margin:0}.mdl-card__subtitle-text{font-size:14px;color:rgba(0,0,0,.54);margin:0}.mdl-card__supporting-text{color:rgba(0,0,0,.54);font-size:1rem;line-height:18px;overflow:hidden;padding:16px;width:90%}.mdl-card__supporting-text.mdl-card--border{border-bottom:1px solid rgba(0,0,0,.1)}.mdl-card__actions{font-size:16px;line-height:normal;width:100%;background-color:transparent;padding:8px;box-sizing:border-box}.mdl-card__actions.mdl-card--border{border-top:1px solid rgba(0,0,0,.1)}.mdl-card--expand{flex-grow:1}.mdl-card__menu{position:absolute;right:16px;top:16px}.mdl-button{background:transparent;border:none;border-radius:2px;color:#000;position:relative;height:36px;margin:0;min-width:64px;padding:0 16px;display:inline-block;font-family:Roboto,Helvetica,Arial,sans-serif;font-size:14px;font-weight:500;text-transform:uppercase;line-height:1;letter-spacing:0;overflow:hidden;will-change:box-shadow;transition:box-shadow .2s cubic-bezier(.4,0,1,1),background-color .2s cubic-bezier(.4,0,.2,1),color .2s cubic-bezier(.4,0,.2,1);outline:none;cursor:pointer;text-decoration:none;text-align:center;line-height:36px;vertical-align:middle}.mdl-button::-moz-focus-inner{border:0}.mdl-button:hover{background-color:hsla(0,0%,62%,.2)}.mdl-button:focus:not(:active){background-color:rgba(0,0,0,.12)}.mdl-button:active{background-color:hsla(0,0%,62%,.4)}.mdl-button.mdl-button--colored{color:#2196f3}.mdl-button.mdl-button--colored:focus:not(:active){background-color:rgba(0,0,0,.12)}input.mdl-button[type=submit]{-webkit-appearance:none}.mdl-button--raised{background:hsla(0,0%,62%,.2);box-shadow:0 2px 2px 0 rgba(0,0,0,.14),0 3px 1px -2px rgba(0,0,0,.2),0 1px 5px 0 rgba(0,0,0,.12)}.mdl-button--raised:active{box-shadow:0 4px 5px 0 rgba(0,0,0,.14),0 1px 10px 0 rgba(0,0,0,.12),0 2px 4px -1px rgba(0,0,0,.2);background-color:hsla(0,0%,62%,.4)}.mdl-button--raised:focus:not(:active){box-shadow:0 0 8px rgba(0,0,0,.18),0 8px 16px rgba(0,0,0,.36);background-color:hsla(0,0%,62%,.4)}.mdl-button--raised.mdl-button--colored{background:#2196f3;color:#fff}.mdl-button--raised.mdl-button--colored:active,.mdl-button--raised.mdl-button--colored:focus:not(:active),.mdl-button--raised.mdl-button--colored:hover{background-color:#2196f3}.mdl-button--raised.mdl-button--colored .mdl-ripple{background:#fff}.mdl-button--fab{border-radius:50%;font-size:24px;height:56px;margin:auto;min-width:56px;width:56px;padding:0;overflow:hidden;background:hsla(0,0%,62%,.2);box-shadow:0 1px 1.5px 0 rgba(0,0,0,.12),0 1px 1px 0 rgba(0,0,0,.24);position:relative;line-height:normal}.admonition.attention .mdl-button--fab .admonition-title:before,.admonition.caution .mdl-button--fab .admonition-title:before,.admonition.danger .mdl-button--fab .admonition-title:before,.admonition.error .mdl-button--fab .admonition-title:before,.admonition.hint .mdl-button--fab .admonition-title:before,.admonition.important .mdl-button--fab .admonition-title:before,.admonition.note .mdl-button--fab .admonition-title:before,.admonition.seealso .mdl-button--fab .admonition-title:before,.admonition.tip .mdl-button--fab .admonition-title:before,.admonition.warning .mdl-button--fab .admonition-title:before,.mdl-button--fab .admonition.attention .admonition-title:before,.mdl-button--fab .admonition.caution .admonition-title:before,.mdl-button--fab .admonition.danger .admonition-title:before,.mdl-button--fab .admonition.error .admonition-title:before,.mdl-button--fab .admonition.hint .admonition-title:before,.mdl-button--fab .admonition.important .admonition-title:before,.mdl-button--fab .admonition.note .admonition-title:before,.mdl-button--fab .admonition.seealso .admonition-title:before,.mdl-button--fab .admonition.tip .admonition-title:before,.mdl-button--fab .admonition.warning .admonition-title:before,.mdl-button--fab .material-icons,.mdl-button--fab a.download:before{position:absolute;top:50%;left:50%;transform:translate(-12px,-12px);line-height:24px;width:24px}.mdl-button--fab.mdl-button--mini-fab{height:40px;min-width:40px;width:40px}.mdl-button--fab .mdl-button__ripple-container{border-radius:50%;-webkit-mask-image:-webkit-radial-gradient(circle,#fff,#000)}.mdl-button--fab:active{box-shadow:0 4px 5px 0 rgba(0,0,0,.14),0 1px 10px 0 rgba(0,0,0,.12),0 2px 4px -1px rgba(0,0,0,.2);background-color:hsla(0,0%,62%,.4)}.mdl-button--fab:focus:not(:active){box-shadow:0 0 8px rgba(0,0,0,.18),0 8px 16px rgba(0,0,0,.36);background-color:hsla(0,0%,62%,.4)}.mdl-button--fab.mdl-button--colored{background:#ff6e40;color:#fff}.mdl-button--fab.mdl-button--colored:active,.mdl-button--fab.mdl-button--colored:focus:not(:active),.mdl-button--fab.mdl-button--colored:hover{background-color:#ff6e40}.mdl-button--fab.mdl-button--colored .mdl-ripple{background:#fff}.mdl-button--icon{border-radius:50%;font-size:24px;height:32px;margin-left:0;margin-right:0;min-width:32px;width:32px;padding:0;overflow:hidden;color:inherit;line-height:normal}.admonition.attention .mdl-button--icon .admonition-title:before,.admonition.caution .mdl-button--icon .admonition-title:before,.admonition.danger .mdl-button--icon .admonition-title:before,.admonition.error .mdl-button--icon .admonition-title:before,.admonition.hint .mdl-button--icon .admonition-title:before,.admonition.important .mdl-button--icon .admonition-title:before,.admonition.note .mdl-button--icon .admonition-title:before,.admonition.seealso .mdl-button--icon .admonition-title:before,.admonition.tip .mdl-button--icon .admonition-title:before,.admonition.warning .mdl-button--icon .admonition-title:before,.mdl-button--icon .admonition.attention .admonition-title:before,.mdl-button--icon .admonition.caution .admonition-title:before,.mdl-button--icon .admonition.danger .admonition-title:before,.mdl-button--icon .admonition.error .admonition-title:before,.mdl-button--icon .admonition.hint .admonition-title:before,.mdl-button--icon .admonition.important .admonition-title:before,.mdl-button--icon .admonition.note .admonition-title:before,.mdl-button--icon .admonition.seealso .admonition-title:before,.mdl-button--icon .admonition.tip .admonition-title:before,.mdl-button--icon .admonition.warning .admonition-title:before,.mdl-button--icon .material-icons,.mdl-button--icon a.download:before{position:absolute;top:50%;left:50%;transform:translate(-12px,-12px);line-height:24px;width:24px}.mdl-button--icon.mdl-button--mini-icon{height:24px;min-width:24px;width:24px}.admonition.attention .mdl-button--icon.mdl-button--mini-icon .admonition-title:before,.admonition.caution .mdl-button--icon.mdl-button--mini-icon .admonition-title:before,.admonition.danger .mdl-button--icon.mdl-button--mini-icon .admonition-title:before,.admonition.error .mdl-button--icon.mdl-button--mini-icon .admonition-title:before,.admonition.hint .mdl-button--icon.mdl-button--mini-icon .admonition-title:before,.admonition.important .mdl-button--icon.mdl-button--mini-icon .admonition-title:before,.admonition.note .mdl-button--icon.mdl-button--mini-icon .admonition-title:before,.admonition.seealso .mdl-button--icon.mdl-button--mini-icon .admonition-title:before,.admonition.tip .mdl-button--icon.mdl-button--mini-icon .admonition-title:before,.admonition.warning .mdl-button--icon.mdl-button--mini-icon .admonition-title:before,.mdl-button--icon.mdl-button--mini-icon .admonition.attention .admonition-title:before,.mdl-button--icon.mdl-button--mini-icon .admonition.caution .admonition-title:before,.mdl-button--icon.mdl-button--mini-icon .admonition.danger .admonition-title:before,.mdl-button--icon.mdl-button--mini-icon .admonition.error .admonition-title:before,.mdl-button--icon.mdl-button--mini-icon .admonition.hint .admonition-title:before,.mdl-button--icon.mdl-button--mini-icon .admonition.important .admonition-title:before,.mdl-button--icon.mdl-button--mini-icon .admonition.note .admonition-title:before,.mdl-button--icon.mdl-button--mini-icon .admonition.seealso .admonition-title:before,.mdl-button--icon.mdl-button--mini-icon .admonition.tip .admonition-title:before,.mdl-button--icon.mdl-button--mini-icon .admonition.warning .admonition-title:before,.mdl-button--icon.mdl-button--mini-icon .material-icons,.mdl-button--icon.mdl-button--mini-icon a.download:before{top:0;left:0}.mdl-button--icon .mdl-button__ripple-container{border-radius:50%;-webkit-mask-image:-webkit-radial-gradient(circle,#fff,#000)}.mdl-button__ripple-container{display:block;height:100%;left:0;position:absolute;top:0;width:100%;z-index:0;overflow:hidden}.mdl-button.mdl-button--disabled .mdl-button__ripple-container .mdl-ripple,.mdl-button[disabled] .mdl-button__ripple-container .mdl-ripple{background-color:transparent}.mdl-button--primary.mdl-button--primary{color:#2196f3}.mdl-button--primary.mdl-button--primary .mdl-ripple{background:#fff}.mdl-button--primary.mdl-button--primary.mdl-button--fab,.mdl-button--primary.mdl-button--primary.mdl-button--raised{color:#fff;background-color:#2196f3}.mdl-button--accent.mdl-button--accent{color:#ff6e40}.mdl-button--accent.mdl-button--accent .mdl-ripple{background:#fff}.mdl-button--accent.mdl-button--accent.mdl-button--fab,.mdl-button--accent.mdl-button--accent.mdl-button--raised{color:#fff;background-color:#ff6e40}.mdl-button.mdl-button--disabled.mdl-button--disabled,.mdl-button[disabled][disabled]{color:rgba(0,0,0,.26);cursor:default;background-color:transparent}.mdl-button--fab.mdl-button--disabled.mdl-button--disabled,.mdl-button--fab[disabled][disabled]{background-color:rgba(0,0,0,.12);color:rgba(0,0,0,.26)}.mdl-button--raised.mdl-button--disabled.mdl-button--disabled,.mdl-button--raised[disabled][disabled]{background-color:rgba(0,0,0,.12);color:rgba(0,0,0,.26);box-shadow:none}.mdl-button--colored.mdl-button--disabled.mdl-button--disabled,.mdl-button--colored[disabled][disabled]{color:rgba(0,0,0,.26)}.admonition.attention .mdl-button .admonition-title:before,.admonition.caution .mdl-button .admonition-title:before,.admonition.danger .mdl-button .admonition-title:before,.admonition.error .mdl-button .admonition-title:before,.admonition.hint .mdl-button .admonition-title:before,.admonition.important .mdl-button .admonition-title:before,.admonition.note .mdl-button .admonition-title:before,.admonition.seealso .mdl-button .admonition-title:before,.admonition.tip .mdl-button .admonition-title:before,.admonition.warning .mdl-button .admonition-title:before,.mdl-button .admonition.attention .admonition-title:before,.mdl-button .admonition.caution .admonition-title:before,.mdl-button .admonition.danger .admonition-title:before,.mdl-button .admonition.error .admonition-title:before,.mdl-button .admonition.hint .admonition-title:before,.mdl-button .admonition.important .admonition-title:before,.mdl-button .admonition.note .admonition-title:before,.mdl-button .admonition.seealso .admonition-title:before,.mdl-button .admonition.tip .admonition-title:before,.mdl-button .admonition.warning .admonition-title:before,.mdl-button .material-icons,.mdl-button a.download:before{vertical-align:middle}.font-light{font-weight:300}.font-regular{font-weight:400}.font-heavy{font-weight:700}.left{text-align:left}.right{text-align:right}.center{text-align:center;margin-left:auto;margin-right:auto}.justify{text-align:justify}.hidden-sm{display:none}.container{width:100%;margin-left:auto;margin-right:auto}.row{position:relative;width:100%}.row [class^=col]{float:left;margin:.5rem 1%;min-height:.125rem}.row:after{content:\"\";display:table;clear:both}.col-1,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-10,.col-11,.col-12{width:98%}.col-1-sm{width:6.33333%}.col-2-sm{width:14.66667%}.col-3-sm{width:23%}.col-4-sm{width:31.33333%}.col-5-sm{width:39.66667%}.col-6-sm{width:48%}.col-7-sm{width:56.33333%}.col-8-sm{width:64.66667%}.col-9-sm{width:73%}.col-10-sm{width:81.33333%}.col-11-sm{width:89.66667%}.col-12-sm{width:98%}@media only screen and (min-width:45em){.col-1{width:6.33333%}.col-2{width:14.66667%}.col-3{width:23%}.col-4{width:31.33333%}.col-5{width:39.66667%}.col-6{width:48%}.col-7{width:56.33333%}.col-8{width:64.66667%}.col-9{width:73%}.col-10{width:81.33333%}.col-11{width:89.66667%}.col-12{width:98%}.hidden-sm{display:block}}.row{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;flex-wrap:wrap}.row>[class*=col-]{display:flex;flex-direction:column}.admonition.attention .admonition-title:before,.admonition.caution .admonition-title:before,.admonition.danger .admonition-title:before,.admonition.error .admonition-title:before,.admonition.hint .admonition-title:before,.admonition.important .admonition-title:before,.admonition.note .admonition-title:before,.admonition.seealso .admonition-title:before,.admonition.tip .admonition-title:before,.admonition.warning .admonition-title:before,.material-icons,a.download:before{font-family:Material Icons;font-weight:400;font-style:normal;font-size:24px;display:inline-block;line-height:1;text-transform:none;letter-spacing:normal;word-wrap:normal;white-space:nowrap;direction:ltr;-webkit-font-smoothing:antialiased;text-rendering:optimizeLegibility;-moz-osx-font-smoothing:grayscale;font-feature-settings:\"liga\"}html{font-size:16px}body{display:block!important;background-color:#fafafa;font-size:1rem;line-height:1.5rem;font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Arial,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol}.mdl-layout__content:focus{outline:none}.mdl-layout__content header.mdl-layout__drawer{display:none}.mdl-layout__container{height:calc(100% - 76px);margin-top:76px}.mdl-layout__header{position:fixed;transition:transform .5s}.mdl-layout--fixed-drawer>.mdl-layout__content{margin-left:300px}@media screen and (max-width:1024px){.mdl-layout--fixed-drawer>.mdl-layout__content{margin-left:0}}a.download>code.download,blockquote,h1,h2,h3,h4,h5,h6,span.mdl-layout-title{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Arial,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol}.contents,.contents a,.globaltoc a.current,.toc-backref,.toctree-wrapper,.toctree-wrapper a,h1,h2,h3,h4,h5,h6{color:#048ccc!important}a{text-decoration:none}.page-content,.page-content dd,.page-content dl,.page-content dt,.page-content ol,.page-content p,.page-content table,.page-content td,.page-content th,.page-content ul{font-size:1rem}.brand{color:inherit;text-decoration:none}.section{overflow-x:auto}img{max-width:100%;display:block;margin-left:auto;margin-right:auto}div.figure p.caption{text-align:center;margin-top:.75rem}div.figure p.caption span.caption-number{font-style:normal}div.figure p.caption .caption-number:after{content:\"\\00a0\"}.svg-icon{width:16px;height:16px;display:inline-block;fill:#f5f5f5;padding-right:5px;padding-top:4px;vertical-align:text-top}.admonition.attention a.download>i.admonition-title:before,.admonition.caution a.download>i.admonition-title:before,.admonition.danger a.download>i.admonition-title:before,.admonition.error a.download>i.admonition-title:before,.admonition.hint a.download>i.admonition-title:before,.admonition.important a.download>i.admonition-title:before,.admonition.note a.download>i.admonition-title:before,.admonition.seealso a.download>i.admonition-title:before,.admonition.tip a.download>i.admonition-title:before,.admonition.warning a.download>i.admonition-title:before,a.download>i.material-icons{position:relative;top:5px}a.download{text-decoration:none}.wrapper:after{content:\"\";display:table;clear:both}.wrapper{max-width:1090px;margin-right:auto;margin-left:auto;padding-right:45px;padding-left:30px}@media screen and (max-width:1024px){.wrapper{max-width:1120px;padding-right:15px;padding-left:15px}}.document{width:100%;margin:84px auto;display:flex}@media (min-width:1795px){.document{width:100%}}.document .page-content{width:100%;margin:0 auto;padding:0 12px}@media (min-width:992px){.document .page-content{width:90%;padding:0 5%}}@media (min-width:1200px){.document .page-content{width:calc(90% - 230px);padding:0 5%}}.document .side-doc-outline{width:230px}@media (max-width:1199px){.document .side-doc-outline{display:none}}.document .side-doc-outline--content{position:sticky;overflow-x:auto;overflow-y:auto;width:inherit;right:0;top:80px}.document .side-doc-outline--content::-webkit-scrollbar{width:6px}.document .side-doc-outline--content::-webkit-scrollbar-track{border-radius:6px}.document .side-doc-outline--content::-webkit-scrollbar-thumb{background-color:rgba(0,0,0,.3);border-radius:6px;box-shadow:0 0 0 1px hsla(0,0%,100%,.3)}@keyframes float-in{0%{transform:translateY(.5rem);opacity:0}to{transform:translateY(0);opacity:1}}@keyframes float-out{0%{transform:translateY(0);opacity:1}to{transform:translateY(.5rem);opacity:0}}.page-content .headerlink{display:inline-block;text-decoration:none;margin-left:.8rem;color:inherit;opacity:0}.page-content .headerlink:hover{animation:float-in .2s cubic-bezier(.4,0,.2,1) 0s forwards}.page-content h1 .toc-backref,.page-content h2 .toc-backref,.page-content h3 .toc-backref,.page-content h4 .toc-backref,.page-content h5 .toc-backref,.page-content h6 .toc-backref{text-decoration:none}.page-content h1:hover .headerlink,.page-content h2:hover .headerlink,.page-content h3:hover .headerlink,.page-content h4:hover .headerlink,.page-content h5:hover .headerlink,.page-content h6:hover .headerlink{animation:float-in .2s cubic-bezier(.4,0,.2,1) 0s forwards}.page-content h1{font-size:2rem;line-height:2.25rem}.page-content h2{font-size:1.75rem;line-height:2rem;padding-top:1.5rem;margin-top:0;margin-bottom:1rem}.page-content h3{font-size:1.5rem;line-height:1.75rem;padding-top:1rem;margin-top:0;margin-bottom:.75rem}.page-content h4{font-size:1.25rem;line-height:1.5rem;padding-top:.75rem;margin-top:0;margin-bottom:.5rem}.page-content div.page-content h5{font-size:1.1rem;line-height:1.5rem;padding-top:2rem;margin-top:0;margin-bottom:1rem}.page-content div.page-content h6{font-size:1rem;line-height:1.5rem;padding-top:2rem;margin-top:0;margin-bottom:1rem}.admonition{padding:12px 20px;margin-top:10px;margin-bottom:10px}.admonition p.last{margin:16px}.admonition .admonition-title{font-size:16px;font-weight:700;color:#555;text-transform:uppercase;margin-top:7px}.admonition.note{border-left:4px solid #00bcd4;background-color:rgba(0,188,212,.1)}.admonition.note .admonition-title{font-size:16px;font-weight:700;color:#00bcd4;margin-top:4px;margin-bottom:8px}.admonition.note .admonition-title:before{position:relative;margin-right:5px;top:3px;content:\"info_outline\";font-size:18px}.admonition.seealso{border-left:4px solid #00bcd4;background-color:rgba(0,188,212,.1)}.admonition.seealso .admonition-title{font-size:16px;font-weight:700;color:#00bcd4;margin-top:4px;margin-bottom:8px}.admonition.seealso .admonition-title:before{position:relative;margin-right:5px;top:3px;content:\"search\";font-size:18px}.admonition.hint{border-left:4px solid #00bcd4;background-color:rgba(0,188,212,.1)}.admonition.hint .admonition-title{font-size:16px;font-weight:700;color:#00bcd4;margin-top:4px;margin-bottom:8px}.admonition.hint .admonition-title:before{position:relative;margin-right:5px;top:3px;content:\"help_outline\";font-size:18px}.admonition.warning{border-left:4px solid #ffc107;background-color:rgba(255,193,7,.1)}.admonition.warning .admonition-title{font-size:16px;font-weight:700;color:#ffc107;margin-top:4px;margin-bottom:8px}.admonition.warning .admonition-title:before{position:relative;margin-right:5px;top:3px;content:\"warning\";font-size:18px}.admonition.attention{border-left:4px solid #ffc107;background-color:rgba(255,193,7,.1)}.admonition.attention .admonition-title{font-size:16px;font-weight:700;color:#ffc107;margin-top:4px;margin-bottom:8px}.admonition.attention .admonition-title:before{position:relative;margin-right:5px;top:3px;content:\"warning\";font-size:18px}.admonition.tip{border-left:4px solid #8bc34a;background-color:rgba(139,195,74,.1)}.admonition.tip .admonition-title{font-size:16px;font-weight:700;color:#8bc34a;margin-top:4px;margin-bottom:8px}.admonition.tip .admonition-title:before{position:relative;margin-right:5px;top:3px;content:\"lightbulb_outline\";font-size:18px}.admonition.important{border-left:4px solid #8bc34a;background-color:rgba(139,195,74,.1)}.admonition.important .admonition-title{font-size:16px;font-weight:700;color:#8bc34a;margin-top:4px;margin-bottom:8px}.admonition.important .admonition-title:before{position:relative;margin-right:5px;top:3px;content:\"check_circle\";font-size:18px}.admonition.error{border-left:4px solid #f44336;background-color:rgba(244,67,54,.1)}.admonition.error .admonition-title{font-size:16px;font-weight:700;color:#f44336;margin-top:4px;margin-bottom:8px}.admonition.error .admonition-title:before{position:relative;margin-right:5px;top:3px;content:\"error_outline\";font-size:18px}.admonition.caution{border-left:4px solid #f44336;background-color:rgba(244,67,54,.1)}.admonition.caution .admonition-title{font-size:16px;font-weight:700;color:#f44336;margin-top:4px;margin-bottom:8px}.admonition.caution .admonition-title:before{position:relative;margin-right:5px;top:3px;content:\"error_outline\";font-size:18px}.admonition.danger{border-left:4px solid #f44336;background-color:rgba(244,67,54,.1)}.admonition.danger .admonition-title{font-size:16px;font-weight:700;color:#f44336;margin-top:4px;margin-bottom:8px}.admonition.danger .admonition-title:before{position:relative;margin-right:5px;top:3px;content:\"error_outline\";font-size:18px}.page-content .highlight{margin:1px 0}.page-content .highlight pre{background:rgba(0,0,0,.05);color:rgba(0,0,0,.87);font-family:Menlo,DejaVu Sans Mono,Liberation Mono,Consolas,Ubuntu Mono,Courier New,andale mono,lucida console,monospace;padding:.75rem;overflow:auto;overflow-y:hidden}.page-content .highlight pre .nd,.page-content .highlight pre .o{color:rgba(0,0,0,.87)}.page-content div.highlight-console div.highlight{background:none}.page-content .output .highlight pre{color:rgba(0,0,0,.87);background:#fafafa;border:1px solid #999;padding:.75rem}.page-content .code,.page-content code:not(.download){margin:0;border-radius:2px}.page-content .code,.page-content .code span.pre,.page-content code:not(.download),.page-content code:not(.download) span.pre{font-family:Menlo,DejaVu Sans Mono,Liberation Mono,Consolas,Ubuntu Mono,Courier New,andale mono,lucida console,monospace}.page-content .viewcode-link{padding-left:2em;font-size:80%}.page-content .class>dt,.page-content .function>dt,.page-content .method>dt,.page-content .rubric{display:table;margin:10px 0;font-size:100%;line-height:normal;background:#e7f2fa;color:#2b98f0;border-top:3px solid #55adf3;padding:10px;position:relative}.page-content .class>dt .descclassname,.page-content .class>dt .descname,.page-content .function>dt .descclassname,.page-content .function>dt .descname,.page-content .method>dt .descclassname,.page-content .method>dt .descname,.page-content .rubric .descclassname,.page-content .rubric .descname{color:rgba(0,0,0,.87);background:#e7f2fa;padding:3px}.page-content .class>dt em,.page-content .function>dt em,.page-content .method>dt em,.page-content .rubric em{padding:0 2px}.page-content .rubric{margin:30px 0 10px}.page-content .field-body{padding-left:40px}.page-content .field-body ul{padding:0 0 0 16px;margin:0}.page-content .seealso .docutils>dt{float:left;clear:left;padding:0 6px}.page-content .seealso .docutils>dd{padding-left:6em}.page-content .nblast{padding-bottom:1em}.page-content pre{font-size:90%;background:#eee;color:#455a64;padding:16px 32px;width:auto;border-radius:4px;word-wrap:break-word}.page-content pre:hover:before{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Arial,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol;padding:0 .5rem;content:attr(click-to-copy);color:rgba(0,0,0,.5);border-radius:4px;position:relative;float:right;top:-.5rem;right:-.5rem;background:#c8c8c8;font-size:.8rem;cursor:pointer}.page-content blockquote{font-size:1rem;padding:0 1rem;border-left:3px solid rgba(0,0,0,.05)}.page-content blockquote:after{content:\"\"!important;margin-left:0}.page-content blockquote:before{content:\"\"!important}.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list){margin:1.5rem 0;table-layout:fixed;max-width:100%;min-width:70%}.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) td,.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) th{white-space:normal;overflow-wrap:break-word}.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) caption{font-size:16px;margin:1rem 0 .8rem;white-space:normal}.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) caption .caption-number{font-style:normal}.page-content table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) caption .caption-number:after{content:\"\\00a0\"}.globaltoc .caption,.globaltoc .toc{display:none}.globaltoc ul{list-style-type:none;padding:0;margin:0}.globaltoc ul li{min-height:18px}.globaltoc ul li .link-wrapper{display:flex;justify-content:space-between}.globaltoc ul li .link-wrapper>a{padding:4px 0;display:block;width:100%;font-size:1rem;text-decoration:none;color:#757575}.globaltoc ul li .link-wrapper>a.current{font-weight:700}.globaltoc .nav-toggle{padding:0;float:right;display:flex;align-items:center;justify-content:center;height:36px}.globaltoc .nav-toggle>a{padding:0;margin-left:0;margin-right:4px;cursor:pointer}.globaltoc .nav-toggle>a>i{font-size:18px}.globaltoc .nav-toggle.show{transform:rotate(180deg)}.globaltoc .nav-toggle.show>a{margin-right:0;margin-left:4px}.globaltoc nav>ul>li>span.link-wrapper{padding-left:8px}.globaltoc nav>ul>li>ul>li>span.link-wrapper{padding-left:16px}.globaltoc nav>ul>li>ul>li>ul>li>span.link-wrapper{padding-left:24px}.globaltoc nav>ul>li>ul>li>ul>li>ul>li>span.link-wrapper{padding-left:32px}.globaltoc nav>ul>li>ul>li>ul>li>ul>li>ul>li>span.link-wrapper{padding-left:40px}.globaltoc nav>ul>li>ul>li>ul>li>ul>li>ul>li>ul>li>span.link-wrapper{padding-left:48px}.localtoc{font-size:.75rem;padding-top:1rem}.localtoc .caption{padding-left:12px}.localtoc .caption-text{font-size:.9rem;font-weight:700}.localtoc>ul>li>a{display:none}.localtoc ul{padding:0;list-style-type:none}.localtoc li{padding-left:6px}.localtoc a{display:block;text-decoration:none;color:inherit;margin-top:8px;padding-left:8px;line-height:1.1rem}.localtoc a.current{padding-left:5px;border-left:3px solid;font-weight:700}.contents.topic,.toctree-wrapper{border-left:5px solid}.contents.topic>p.topic-title,.toctree-wrapper>p.caption{color:#757575;font-size:1rem;padding-left:14px}.contents.topic ul,.toctree-wrapper ul{padding-left:14px;list-style:none;line-height:30px}.contents.topic a,.toctree-wrapper a{font-size:1.2rem;text-decoration:none}.contents.topic a .pre,.toctree-wrapper a .pre{font-size:1rem}.contents.topic>ul>li>a,.toctree-wrapper>ul>li>a{font-size:1.3rem}.contents.topic>ul>li>a .pre,.toctree-wrapper>ul>li>a .pre{font-size:1.1rem}.page-content ul li{margin:.3rem 0}.page-content ul li p{margin:0}.page-content .option-list .option{font-family:Menlo,DejaVu Sans Mono,Liberation Mono,Consolas,Ubuntu Mono,Courier New,andale mono,lucida console,monospace}.page-content .option-list td{padding:.5rem;border:none}.mdl-layout__drawer{background-color:#fff}.mdl-layout__drawer::-webkit-scrollbar{width:6px}.mdl-layout__drawer::-webkit-scrollbar-track{border-radius:6px}.mdl-layout__drawer::-webkit-scrollbar-thumb{background-color:rgba(0,0,0,.3);border-radius:6px;box-shadow:0 0 0 1px hsla(0,0%,100%,.3)}.mdl-layout__drawer>.mdl-layout-title{font-weight:700;text-align:right;margin:0;padding:0;line-height:32px;border-bottom:1px solid rgba(0,0,0,.1);min-height:64px}.mdl-layout__drawer>.mdl-layout-title .title{color:inherit;display:block;height:100%;width:100%;text-decoration:none}.mdl-layout__drawer>.mdl-layout-title .title>img.logo{width:100%;margin:0;padding:0}.mdl-layout__drawer>.mdl-layout-title .title-text{font-weight:700;text-align:right;padding:0 10px;margin:16px 0 8px;line-height:32px;font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Arial,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol;color:inherit;display:block}nav.breadcrumb>a.mdl-navigation__link{padding:0 8px;font-size:18px}@media (max-width:1199px){nav.breadcrumb{width:calc(100% - 64px)}nav.breadcrumb a.mdl-navigation__link.is-active{overflow-x:hidden;width:100%;overflow:hidden;white-space:nowrap;text-overflow:ellipsis}.admonition.attention nav.breadcrumb i.admonition-title:before,.admonition.caution nav.breadcrumb i.admonition-title:before,.admonition.danger nav.breadcrumb i.admonition-title:before,.admonition.error nav.breadcrumb i.admonition-title:before,.admonition.hint nav.breadcrumb i.admonition-title:before,.admonition.important nav.breadcrumb i.admonition-title:before,.admonition.note nav.breadcrumb i.admonition-title:before,.admonition.seealso nav.breadcrumb i.admonition-title:before,.admonition.tip nav.breadcrumb i.admonition-title:before,.admonition.warning nav.breadcrumb i.admonition-title:before,nav.breadcrumb .admonition.attention i.admonition-title:before,nav.breadcrumb .admonition.caution i.admonition-title:before,nav.breadcrumb .admonition.danger i.admonition-title:before,nav.breadcrumb .admonition.error i.admonition-title:before,nav.breadcrumb .admonition.hint i.admonition-title:before,nav.breadcrumb .admonition.important i.admonition-title:before,nav.breadcrumb .admonition.note i.admonition-title:before,nav.breadcrumb .admonition.seealso i.admonition-title:before,nav.breadcrumb .admonition.tip i.admonition-title:before,nav.breadcrumb .admonition.warning i.admonition-title:before,nav.breadcrumb a.mdl-navigation__link:not(.is-active),nav.breadcrumb i.material-icons{display:none}}div.mdl-layout__header{margin-top:77px}.mdl-layout__drawer-button{top:13px!important}div.mdl-layout__header-row.header-links{background:hsla(0,0%,100%,.2);width:100%;overflow-x:auto;overflow-y:hidden}div.mdl-layout__header-row.header-links a.mdl-navigation__link{font-size:1rem}div.mdl-layout__header-row.header-links a.mdl-navigation__link i{font-size:1.2rem;margin:0 8px;position:relative;bottom:-.1rem}div.mdl-layout__header-row.header-links a.mdl-navigation__link:hover{background-color:#2196f3;color:#eee}div.mdl-layout__header-row.header-links a.mdl-navigation__link[href=\"#\"]{background-color:#2196f3;opacity:1;color:#fff}.site-title{font-weight:300!important;line-height:57px;letter-spacing:-1px;margin-bottom:0;float:left;color:#fff}.site-title,.site-title:visited{color:#424242}.site-header{position:fixed;top:0;width:100%;min-height:55px;padding-top:10px;padding-bottom:10px;background-color:#048ccc;z-index:10;font-weight:300;font-size:17px;border-bottom:1px solid #fff}.site-header-logo{width:120px;display:initial}.site-nav{float:right;line-height:57px}.site-nav .menu-icon,.site-nav .nav-trigger{display:none}.site-nav .page-link{color:#fff;line-height:1.5;font-weight:300}.site-nav .page-link:not(:last-child){margin-right:40px}.site-nav .page-link:hover{color:#fff;text-shadow:-.06ex 0 #fff,.06ex 0 #fff}.site-nav .page-link.page-current{color:#fff;text-decoration:underline}@media screen and (max-width:1024px){.site-nav{position:absolute;top:9px;right:15px;background-color:#178dc9;border-radius:2px;text-align:right}.site-nav label[for=nav-trigger]{display:block;float:right;width:36px;height:36px;z-index:2;cursor:pointer}.site-nav .menu-icon{display:block;float:right;width:36px;height:26px;line-height:0;padding-top:20px;text-align:center}.site-nav .menu-icon>svg{fill:#fff}.site-nav input~.trigger{clear:both;display:none}.site-nav input:checked~.trigger{display:block;padding-bottom:5px}.site-nav .page-link{padding:5px 10px;display:block;margin-left:20px}.site-nav .page-link:not(:last-child){margin-right:0}}footer.mdl-mini-footer{background-color:#212121}footer.mdl-mini-footer>div.mdl-mini-footer__left-section{margin-bottom:20px;display:flex;flex-direction:column}footer.mdl-mini-footer>div.mdl-mini-footer__left-section .mdl-logo{font-size:1.1rem}footer.mdl-mini-footer>div.mdl-mini-footer__right-section{font-size:.9rem;display:flex;flex-direction:column;justify-content:flex-end}footer.mdl-mini-footer>div.mdl-mini-footer__right-section a{color:inherit;font-weight:700;text-decoration:none}footer.mdl-mini-footer p.caption{display:none}.pagenation{width:100%;margin-top:80px;height:92px;background-color:#424242;display:flex}.pagenation #button-next,.pagenation #button-prev,.pagenation .button-common{text-transform:none;padding:0;height:92px;display:flex;justify-content:center;align-items:center;color:#fff}.pagenation #button-prev{margin-right:auto}.pagenation #button-prev .pagenation-text{text-align:left}.pagenation #button-next{margin-left:auto;flex-direction:row-reverse}.pagenation #button-next .pagenation-text{text-align:right}.pagenation-arrow-L{margin-right:20px}.pagenation-arrow-R{margin-left:20px}.pagenation-text{line-height:30px;font-size:20px}.pagenation-direction{opacity:.7;font-size:18px}@media screen and (max-width:1024px){.pagenation #button-prev{width:20%}.pagenation #button-next{width:80%}.pagenation #button-prev .pagenation-text{display:none}}@media screen and (min-width:1025px){.pagenation #button-next,.pagenation #button-prev{width:50%}.pagenation #button-prev .pagenation-text{display:block}}.site-footer{border-top:1px solid #f5f5f5;padding:30px 0;background-color:#424242;position:relative;z-index:10}.site-footer .footer-category-title{color:#048ccc}.site-footer a,.site-footer a:visited{color:#f5f5f5!important}.site-footer2{background-color:#424242;padding-top:40px;padding-bottom:10px;position:relative;z-index:10}.footer-heading{margin-bottom:15px}.contact-list,.social-media-list{list-style:none;margin-left:0}.footer-bottom-warning{font-size:80%;color:#fff;float:left}.footer-logo{width:200px;margin-bottom:30px;margin-top:30px}.footer-col{float:left;margin-bottom:15px;padding-left:15px}.footer-text{color:#f5f5f5}#waterfall-exp::-webkit-input-placeholder{color:#ccc}#waterfall-exp:-ms-input-placeholder{color:#ccc}#waterfall-exp::-moz-placeholder{color:#ccc}ul.search span.highlighted{font-weight:700}ul.search>li{margin-bottom:24px}#search-results ul{list-style:none;padding:0}#search-results ul li>a{text-decoration:none;font-size:1.2rem}a.download:before{content:\"file_download\";position:relative;top:5px;margin-right:5px}button.download{position:sticky;margin-left:1em}.mdl-card{margin:1em 1.5em 1em 0;display:inline-block;width:250px;min-height:140px;padding:18px}.mdl-card:hover{box-shadow:0 10px 20px rgba(0,0,0,.25),0 6px 6px rgba(0,0,0,.22);color:#000;cursor:pointer}.mdl-card__title{padding:0 0 1em;font-size:18px;color:#444}.mdl-card__supporting-text{line-height:1.5rem;padding:0;width:100%}.head-card.mdl-card{width:auto;display:block;max-width:800px;padding:24px}.head-card>.mdl-card__title{padding-bottom:0;height:60px;font-weight:700;text-transform:uppercase}.head-card>.mdl-card__menu{color:#fff}.head-card>.mdl-card__actions{padding:0}.cards{display:flex;flex-direction:row;flex-wrap:wrap}\n/*# sourceMappingURL=/sphinx_materialdesign_theme.css.map */"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/static/sphinx_materialdesign_theme.js",
    "content": "parcelRequire=function(e,r,t,n){var i,o=\"function\"==typeof parcelRequire&&parcelRequire,u=\"function\"==typeof require&&require;function f(t,n){if(!r[t]){if(!e[t]){var i=\"function\"==typeof parcelRequire&&parcelRequire;if(!n&&i)return i(t,!0);if(o)return o(t,!0);if(u&&\"string\"==typeof t)return u(t);var c=new Error(\"Cannot find module '\"+t+\"'\");throw c.code=\"MODULE_NOT_FOUND\",c}p.resolve=function(r){return e[t][1][r]||r},p.cache={};var l=r[t]=new f.Module(t);e[t][0].call(l.exports,p,l,l.exports,this)}return r[t].exports;function p(e){return f(p.resolve(e))}}f.isParcelRequire=!0,f.Module=function(e){this.id=e,this.bundle=f,this.exports={}},f.modules=e,f.cache=r,f.parent=o,f.register=function(r,t){e[r]=[function(e,r){r.exports=t},{}]};for(var c=0;c<t.length;c++)try{f(t[c])}catch(e){i||(i=e)}if(t.length){var l=f(t[t.length-1]);\"object\"==typeof exports&&\"undefined\"!=typeof module?module.exports=l:\"function\"==typeof define&&define.amd?define(function(){return l}):n&&(this[n]=l)}if(parcelRequire=f,i)throw i;return f}({\"BS4D\":[function(require,module,exports) {\n\n},{}],\"dMzA\":[function(require,module,exports) {\n$(document).ready(function(){$(\".feedback-answer\").on(\"click\",function(){$(\".feedback-question\").remove(),$(\".feedback-answer-container\").remove(),$(\".feedback-thank-you\").show(),ga(\"send\",{hitType:\"event\",eventCategory:\"Did this page help you?\",eventAction:$(this).attr(\"data-response\"),eventLabel:window.location.pathname||\"unknown\",eventValue:\"yes\"===$(this).attr(\"data-response\")?1:0})})});\n},{}],\"vKy7\":[function(require,module,exports) {\nfunction e(t){return(e=\"function\"==typeof Symbol&&\"symbol\"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&\"function\"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?\"symbol\":typeof e})(t)}!function(){\"use strict\";function t(e,t){if(e){if(t.element_.classList.contains(t.CssClasses_.MDL_JS_RIPPLE_EFFECT)){var s=document.createElement(\"span\");s.classList.add(t.CssClasses_.MDL_RIPPLE_CONTAINER),s.classList.add(t.CssClasses_.MDL_JS_RIPPLE_EFFECT);var i=document.createElement(\"span\");i.classList.add(t.CssClasses_.MDL_RIPPLE),s.appendChild(i),e.appendChild(s)}e.addEventListener(\"click\",function(s){if(\"#\"===e.getAttribute(\"href\").charAt(0)){s.preventDefault();var i=e.href.split(\"#\")[1],n=t.element_.querySelector(\"#\"+i);t.resetTabState_(),t.resetPanelState_(),e.classList.add(t.CssClasses_.ACTIVE_CLASS),n.classList.add(t.CssClasses_.ACTIVE_CLASS)}})}}function s(e,t,s,i){function n(){var n=e.href.split(\"#\")[1],a=i.content_.querySelector(\"#\"+n);i.resetTabState_(t),i.resetPanelState_(s),e.classList.add(i.CssClasses_.IS_ACTIVE),a.classList.add(i.CssClasses_.IS_ACTIVE)}if(i.tabBar_.classList.contains(i.CssClasses_.JS_RIPPLE_EFFECT)){var a=document.createElement(\"span\");a.classList.add(i.CssClasses_.RIPPLE_CONTAINER),a.classList.add(i.CssClasses_.JS_RIPPLE_EFFECT);var l=document.createElement(\"span\");l.classList.add(i.CssClasses_.RIPPLE),a.appendChild(l),e.appendChild(a)}i.tabBar_.classList.contains(i.CssClasses_.TAB_MANUAL_SWITCH)||e.addEventListener(\"click\",function(t){\"#\"===e.getAttribute(\"href\").charAt(0)&&(t.preventDefault(),n())}),e.show=n}var i={upgradeDom:function(e,t){},upgradeElement:function(e,t){},upgradeElements:function(e){},upgradeAllRegistered:function(){},registerUpgradedCallback:function(e,t){},register:function(e){},downgradeElements:function(e){}};(i=function(){function t(e,t){for(var s=0;s<r.length;s++)if(r[s].className===e)return void 0!==t&&(r[s]=t),r[s];return!1}function s(e){var t=e.getAttribute(\"data-upgraded\");return null===t?[\"\"]:t.split(\",\")}function i(e,t){return-1!==s(e).indexOf(t)}function n(e,t,s){if(\"CustomEvent\"in window&&\"function\"==typeof window.CustomEvent)return new CustomEvent(e,{bubbles:t,cancelable:s});var i=document.createEvent(\"Events\");return i.initEvent(e,t,s),i}function a(e,s){if(void 0===e&&void 0===s)for(var i=0;i<r.length;i++)a(r[i].className,r[i].cssClass);else{var n=e;if(void 0===s){var o=t(n);o&&(s=o.cssClass)}for(var _=document.querySelectorAll(\".\"+s),d=0;d<_.length;d++)l(_[d],n)}}function l(a,l){if(!(\"object\"==e(a)&&a instanceof Element))throw new Error(\"Invalid argument provided to upgrade MDL element.\");var o=n(\"mdl-componentupgrading\",!0,!0);if(a.dispatchEvent(o),!o.defaultPrevented){var h=s(a),c=[];if(l)i(a,l)||c.push(t(l));else{var p=a.classList;r.forEach(function(e){p.contains(e.cssClass)&&-1===c.indexOf(e)&&!i(a,e.className)&&c.push(e)})}for(var C,u=0,E=c.length;u<E;u++){if(!(C=c[u]))throw new Error(\"Unable to find a registered component for the given class.\");h.push(C.className),a.setAttribute(\"data-upgraded\",h.join(\",\"));var m=new C.classConstructor(a);m[d]=C,_.push(m);for(var L=0,I=C.callbacks.length;L<I;L++)C.callbacks[L](a);C.widget&&(a[C.className]=m);var f=n(\"mdl-componentupgraded\",!0,!1);a.dispatchEvent(f)}}}function o(e){if(e){var t=_.indexOf(e);_.splice(t,1);var s=e.element_.getAttribute(\"data-upgraded\").split(\",\"),i=s.indexOf(e[d].classAsString);s.splice(i,1),e.element_.setAttribute(\"data-upgraded\",s.join(\",\"));var a=n(\"mdl-componentdowngraded\",!0,!1);e.element_.dispatchEvent(a)}}var r=[],_=[],d=\"mdlComponentConfigInternal_\";return{upgradeDom:a,upgradeElement:l,upgradeElements:function e(t){Array.isArray(t)||(t=t instanceof Element?[t]:Array.prototype.slice.call(t));for(var s,i=0,n=t.length;i<n;i++)(s=t[i])instanceof HTMLElement&&(l(s),s.children.length>0&&e(s.children))},upgradeAllRegistered:function(){for(var e=0;e<r.length;e++)a(r[e].className)},registerUpgradedCallback:function(e,s){var i=t(e);i&&i.callbacks.push(s)},register:function(e){var s=!0;void 0===e.widget&&void 0===e.widget||(s=e.widget||e.widget);var i={classConstructor:e.constructor||e.constructor,className:e.classAsString||e.classAsString,cssClass:e.cssClass||e.cssClass,widget:s,callbacks:[]};if(r.forEach(function(e){if(e.cssClass===i.cssClass)throw new Error(\"The provided cssClass has already been registered: \"+e.cssClass);if(e.className===i.className)throw new Error(\"The provided className has already been registered\")}),e.constructor.prototype.hasOwnProperty(d))throw new Error(\"MDL component classes must not have \"+d+\" defined as a property.\");t(e.classAsString,i)||r.push(i)},downgradeElements:function(e){var t=function(e){_.filter(function(t){return t.element_===e}).forEach(o)};if(e instanceof Array||e instanceof NodeList)for(var s=0;s<e.length;s++)t(e[s]);else{if(!(e instanceof Node))throw new Error(\"Invalid argument provided to downgrade MDL nodes.\");t(e)}}}}()).ComponentConfigPublic,i.ComponentConfig,i.Component,i.upgradeDom=i.upgradeDom,i.upgradeElement=i.upgradeElement,i.upgradeElements=i.upgradeElements,i.upgradeAllRegistered=i.upgradeAllRegistered,i.registerUpgradedCallback=i.registerUpgradedCallback,i.register=i.register,i.downgradeElements=i.downgradeElements,window.componentHandler=i,window.componentHandler=i,window.addEventListener(\"load\",function(){\"classList\"in document.createElement(\"div\")&&\"querySelector\"in document&&\"addEventListener\"in window&&Array.prototype.forEach?(document.documentElement.classList.add(\"mdl-js\"),i.upgradeAllRegistered()):(i.upgradeElement=function(){},i.register=function(){})}),Date.now||(Date.now=function(){return(new Date).getTime()},Date.now=Date.now);for(var n=[\"webkit\",\"moz\"],a=0;a<n.length&&!window.requestAnimationFrame;++a){var l=n[a];window.requestAnimationFrame=window[l+\"RequestAnimationFrame\"],window.cancelAnimationFrame=window[l+\"CancelAnimationFrame\"]||window[l+\"CancelRequestAnimationFrame\"],window.requestAnimationFrame=window.requestAnimationFrame,window.cancelAnimationFrame=window.cancelAnimationFrame}if(/iP(ad|hone|od).*OS 6/.test(window.navigator.userAgent)||!window.requestAnimationFrame||!window.cancelAnimationFrame){var o=0;window.requestAnimationFrame=function(e){var t=Date.now(),s=Math.max(o+16,t);return setTimeout(function(){e(o=s)},s-t)},window.cancelAnimationFrame=clearTimeout,window.requestAnimationFrame=window.requestAnimationFrame,window.cancelAnimationFrame=window.cancelAnimationFrame}var r=function(e){this.element_=e,this.init()};window.MaterialButton=r,r.prototype.Constant_={},r.prototype.CssClasses_={RIPPLE_EFFECT:\"mdl-js-ripple-effect\",RIPPLE_CONTAINER:\"mdl-button__ripple-container\",RIPPLE:\"mdl-ripple\"},r.prototype.blurHandler_=function(e){e&&this.element_.blur()},r.prototype.disable=function(){this.element_.disabled=!0},r.prototype.disable=r.prototype.disable,r.prototype.enable=function(){this.element_.disabled=!1},r.prototype.enable=r.prototype.enable,r.prototype.init=function(){if(this.element_){if(this.element_.classList.contains(this.CssClasses_.RIPPLE_EFFECT)){var e=document.createElement(\"span\");e.classList.add(this.CssClasses_.RIPPLE_CONTAINER),this.rippleElement_=document.createElement(\"span\"),this.rippleElement_.classList.add(this.CssClasses_.RIPPLE),e.appendChild(this.rippleElement_),this.boundRippleBlurHandler=this.blurHandler_.bind(this),this.rippleElement_.addEventListener(\"mouseup\",this.boundRippleBlurHandler),this.element_.appendChild(e)}this.boundButtonBlurHandler=this.blurHandler_.bind(this),this.element_.addEventListener(\"mouseup\",this.boundButtonBlurHandler),this.element_.addEventListener(\"mouseleave\",this.boundButtonBlurHandler)}},i.register({constructor:r,classAsString:\"MaterialButton\",cssClass:\"mdl-js-button\",widget:!0});var _=function(e){this.element_=e,this.init()};window.MaterialCheckbox=_,_.prototype.Constant_={TINY_TIMEOUT:.001},_.prototype.CssClasses_={INPUT:\"mdl-checkbox__input\",BOX_OUTLINE:\"mdl-checkbox__box-outline\",FOCUS_HELPER:\"mdl-checkbox__focus-helper\",TICK_OUTLINE:\"mdl-checkbox__tick-outline\",RIPPLE_EFFECT:\"mdl-js-ripple-effect\",RIPPLE_IGNORE_EVENTS:\"mdl-js-ripple-effect--ignore-events\",RIPPLE_CONTAINER:\"mdl-checkbox__ripple-container\",RIPPLE_CENTER:\"mdl-ripple--center\",RIPPLE:\"mdl-ripple\",IS_FOCUSED:\"is-focused\",IS_DISABLED:\"is-disabled\",IS_CHECKED:\"is-checked\",IS_UPGRADED:\"is-upgraded\"},_.prototype.onChange_=function(e){this.updateClasses_()},_.prototype.onFocus_=function(e){this.element_.classList.add(this.CssClasses_.IS_FOCUSED)},_.prototype.onBlur_=function(e){this.element_.classList.remove(this.CssClasses_.IS_FOCUSED)},_.prototype.onMouseUp_=function(e){this.blur_()},_.prototype.updateClasses_=function(){this.checkDisabled(),this.checkToggleState()},_.prototype.blur_=function(){window.setTimeout(function(){this.inputElement_.blur()}.bind(this),this.Constant_.TINY_TIMEOUT)},_.prototype.checkToggleState=function(){this.inputElement_.checked?this.element_.classList.add(this.CssClasses_.IS_CHECKED):this.element_.classList.remove(this.CssClasses_.IS_CHECKED)},_.prototype.checkToggleState=_.prototype.checkToggleState,_.prototype.checkDisabled=function(){this.inputElement_.disabled?this.element_.classList.add(this.CssClasses_.IS_DISABLED):this.element_.classList.remove(this.CssClasses_.IS_DISABLED)},_.prototype.checkDisabled=_.prototype.checkDisabled,_.prototype.disable=function(){this.inputElement_.disabled=!0,this.updateClasses_()},_.prototype.disable=_.prototype.disable,_.prototype.enable=function(){this.inputElement_.disabled=!1,this.updateClasses_()},_.prototype.enable=_.prototype.enable,_.prototype.check=function(){this.inputElement_.checked=!0,this.updateClasses_()},_.prototype.check=_.prototype.check,_.prototype.uncheck=function(){this.inputElement_.checked=!1,this.updateClasses_()},_.prototype.uncheck=_.prototype.uncheck,_.prototype.init=function(){if(this.element_){this.inputElement_=this.element_.querySelector(\".\"+this.CssClasses_.INPUT);var e=document.createElement(\"span\");e.classList.add(this.CssClasses_.BOX_OUTLINE);var t=document.createElement(\"span\");t.classList.add(this.CssClasses_.FOCUS_HELPER);var s=document.createElement(\"span\");if(s.classList.add(this.CssClasses_.TICK_OUTLINE),e.appendChild(s),this.element_.appendChild(t),this.element_.appendChild(e),this.element_.classList.contains(this.CssClasses_.RIPPLE_EFFECT)){this.element_.classList.add(this.CssClasses_.RIPPLE_IGNORE_EVENTS),this.rippleContainerElement_=document.createElement(\"span\"),this.rippleContainerElement_.classList.add(this.CssClasses_.RIPPLE_CONTAINER),this.rippleContainerElement_.classList.add(this.CssClasses_.RIPPLE_EFFECT),this.rippleContainerElement_.classList.add(this.CssClasses_.RIPPLE_CENTER),this.boundRippleMouseUp=this.onMouseUp_.bind(this),this.rippleContainerElement_.addEventListener(\"mouseup\",this.boundRippleMouseUp);var i=document.createElement(\"span\");i.classList.add(this.CssClasses_.RIPPLE),this.rippleContainerElement_.appendChild(i),this.element_.appendChild(this.rippleContainerElement_)}this.boundInputOnChange=this.onChange_.bind(this),this.boundInputOnFocus=this.onFocus_.bind(this),this.boundInputOnBlur=this.onBlur_.bind(this),this.boundElementMouseUp=this.onMouseUp_.bind(this),this.inputElement_.addEventListener(\"change\",this.boundInputOnChange),this.inputElement_.addEventListener(\"focus\",this.boundInputOnFocus),this.inputElement_.addEventListener(\"blur\",this.boundInputOnBlur),this.element_.addEventListener(\"mouseup\",this.boundElementMouseUp),this.updateClasses_(),this.element_.classList.add(this.CssClasses_.IS_UPGRADED)}},i.register({constructor:_,classAsString:\"MaterialCheckbox\",cssClass:\"mdl-js-checkbox\",widget:!0});var d=function(e){this.element_=e,this.init()};window.MaterialIconToggle=d,d.prototype.Constant_={TINY_TIMEOUT:.001},d.prototype.CssClasses_={INPUT:\"mdl-icon-toggle__input\",JS_RIPPLE_EFFECT:\"mdl-js-ripple-effect\",RIPPLE_IGNORE_EVENTS:\"mdl-js-ripple-effect--ignore-events\",RIPPLE_CONTAINER:\"mdl-icon-toggle__ripple-container\",RIPPLE_CENTER:\"mdl-ripple--center\",RIPPLE:\"mdl-ripple\",IS_FOCUSED:\"is-focused\",IS_DISABLED:\"is-disabled\",IS_CHECKED:\"is-checked\"},d.prototype.onChange_=function(e){this.updateClasses_()},d.prototype.onFocus_=function(e){this.element_.classList.add(this.CssClasses_.IS_FOCUSED)},d.prototype.onBlur_=function(e){this.element_.classList.remove(this.CssClasses_.IS_FOCUSED)},d.prototype.onMouseUp_=function(e){this.blur_()},d.prototype.updateClasses_=function(){this.checkDisabled(),this.checkToggleState()},d.prototype.blur_=function(){window.setTimeout(function(){this.inputElement_.blur()}.bind(this),this.Constant_.TINY_TIMEOUT)},d.prototype.checkToggleState=function(){this.inputElement_.checked?this.element_.classList.add(this.CssClasses_.IS_CHECKED):this.element_.classList.remove(this.CssClasses_.IS_CHECKED)},d.prototype.checkToggleState=d.prototype.checkToggleState,d.prototype.checkDisabled=function(){this.inputElement_.disabled?this.element_.classList.add(this.CssClasses_.IS_DISABLED):this.element_.classList.remove(this.CssClasses_.IS_DISABLED)},d.prototype.checkDisabled=d.prototype.checkDisabled,d.prototype.disable=function(){this.inputElement_.disabled=!0,this.updateClasses_()},d.prototype.disable=d.prototype.disable,d.prototype.enable=function(){this.inputElement_.disabled=!1,this.updateClasses_()},d.prototype.enable=d.prototype.enable,d.prototype.check=function(){this.inputElement_.checked=!0,this.updateClasses_()},d.prototype.check=d.prototype.check,d.prototype.uncheck=function(){this.inputElement_.checked=!1,this.updateClasses_()},d.prototype.uncheck=d.prototype.uncheck,d.prototype.init=function(){if(this.element_){if(this.inputElement_=this.element_.querySelector(\".\"+this.CssClasses_.INPUT),this.element_.classList.contains(this.CssClasses_.JS_RIPPLE_EFFECT)){this.element_.classList.add(this.CssClasses_.RIPPLE_IGNORE_EVENTS),this.rippleContainerElement_=document.createElement(\"span\"),this.rippleContainerElement_.classList.add(this.CssClasses_.RIPPLE_CONTAINER),this.rippleContainerElement_.classList.add(this.CssClasses_.JS_RIPPLE_EFFECT),this.rippleContainerElement_.classList.add(this.CssClasses_.RIPPLE_CENTER),this.boundRippleMouseUp=this.onMouseUp_.bind(this),this.rippleContainerElement_.addEventListener(\"mouseup\",this.boundRippleMouseUp);var e=document.createElement(\"span\");e.classList.add(this.CssClasses_.RIPPLE),this.rippleContainerElement_.appendChild(e),this.element_.appendChild(this.rippleContainerElement_)}this.boundInputOnChange=this.onChange_.bind(this),this.boundInputOnFocus=this.onFocus_.bind(this),this.boundInputOnBlur=this.onBlur_.bind(this),this.boundElementOnMouseUp=this.onMouseUp_.bind(this),this.inputElement_.addEventListener(\"change\",this.boundInputOnChange),this.inputElement_.addEventListener(\"focus\",this.boundInputOnFocus),this.inputElement_.addEventListener(\"blur\",this.boundInputOnBlur),this.element_.addEventListener(\"mouseup\",this.boundElementOnMouseUp),this.updateClasses_(),this.element_.classList.add(\"is-upgraded\")}},i.register({constructor:d,classAsString:\"MaterialIconToggle\",cssClass:\"mdl-js-icon-toggle\",widget:!0});var h=function(e){this.element_=e,this.init()};window.MaterialMenu=h,h.prototype.Constant_={TRANSITION_DURATION_SECONDS:.3,TRANSITION_DURATION_FRACTION:.8,CLOSE_TIMEOUT:150},h.prototype.Keycodes_={ENTER:13,ESCAPE:27,SPACE:32,UP_ARROW:38,DOWN_ARROW:40},h.prototype.CssClasses_={CONTAINER:\"mdl-menu__container\",OUTLINE:\"mdl-menu__outline\",ITEM:\"mdl-menu__item\",ITEM_RIPPLE_CONTAINER:\"mdl-menu__item-ripple-container\",RIPPLE_EFFECT:\"mdl-js-ripple-effect\",RIPPLE_IGNORE_EVENTS:\"mdl-js-ripple-effect--ignore-events\",RIPPLE:\"mdl-ripple\",IS_UPGRADED:\"is-upgraded\",IS_VISIBLE:\"is-visible\",IS_ANIMATING:\"is-animating\",BOTTOM_LEFT:\"mdl-menu--bottom-left\",BOTTOM_RIGHT:\"mdl-menu--bottom-right\",TOP_LEFT:\"mdl-menu--top-left\",TOP_RIGHT:\"mdl-menu--top-right\",UNALIGNED:\"mdl-menu--unaligned\"},h.prototype.init=function(){if(this.element_){var e=document.createElement(\"div\");e.classList.add(this.CssClasses_.CONTAINER),this.element_.parentElement.insertBefore(e,this.element_),this.element_.parentElement.removeChild(this.element_),e.appendChild(this.element_),this.container_=e;var t=document.createElement(\"div\");t.classList.add(this.CssClasses_.OUTLINE),this.outline_=t,e.insertBefore(t,this.element_);var s=this.element_.getAttribute(\"for\")||this.element_.getAttribute(\"data-mdl-for\"),i=null;s&&((i=document.getElementById(s))&&(this.forElement_=i,i.addEventListener(\"click\",this.handleForClick_.bind(this)),i.addEventListener(\"keydown\",this.handleForKeyboardEvent_.bind(this))));var n=this.element_.querySelectorAll(\".\"+this.CssClasses_.ITEM);this.boundItemKeydown_=this.handleItemKeyboardEvent_.bind(this),this.boundItemClick_=this.handleItemClick_.bind(this);for(var a=0;a<n.length;a++)n[a].addEventListener(\"click\",this.boundItemClick_),n[a].tabIndex=\"-1\",n[a].addEventListener(\"keydown\",this.boundItemKeydown_);if(this.element_.classList.contains(this.CssClasses_.RIPPLE_EFFECT))for(this.element_.classList.add(this.CssClasses_.RIPPLE_IGNORE_EVENTS),a=0;a<n.length;a++){var l=n[a],o=document.createElement(\"span\");o.classList.add(this.CssClasses_.ITEM_RIPPLE_CONTAINER);var r=document.createElement(\"span\");r.classList.add(this.CssClasses_.RIPPLE),o.appendChild(r),l.appendChild(o),l.classList.add(this.CssClasses_.RIPPLE_EFFECT)}this.element_.classList.contains(this.CssClasses_.BOTTOM_LEFT)&&this.outline_.classList.add(this.CssClasses_.BOTTOM_LEFT),this.element_.classList.contains(this.CssClasses_.BOTTOM_RIGHT)&&this.outline_.classList.add(this.CssClasses_.BOTTOM_RIGHT),this.element_.classList.contains(this.CssClasses_.TOP_LEFT)&&this.outline_.classList.add(this.CssClasses_.TOP_LEFT),this.element_.classList.contains(this.CssClasses_.TOP_RIGHT)&&this.outline_.classList.add(this.CssClasses_.TOP_RIGHT),this.element_.classList.contains(this.CssClasses_.UNALIGNED)&&this.outline_.classList.add(this.CssClasses_.UNALIGNED),e.classList.add(this.CssClasses_.IS_UPGRADED)}},h.prototype.handleForClick_=function(e){if(this.element_&&this.forElement_){var t=this.forElement_.getBoundingClientRect(),s=this.forElement_.parentElement.getBoundingClientRect();this.element_.classList.contains(this.CssClasses_.UNALIGNED)||(this.element_.classList.contains(this.CssClasses_.BOTTOM_RIGHT)?(this.container_.style.right=s.right-t.right+\"px\",this.container_.style.top=this.forElement_.offsetTop+this.forElement_.offsetHeight+\"px\"):this.element_.classList.contains(this.CssClasses_.TOP_LEFT)?(this.container_.style.left=this.forElement_.offsetLeft+\"px\",this.container_.style.bottom=s.bottom-t.top+\"px\"):this.element_.classList.contains(this.CssClasses_.TOP_RIGHT)?(this.container_.style.right=s.right-t.right+\"px\",this.container_.style.bottom=s.bottom-t.top+\"px\"):(this.container_.style.left=this.forElement_.offsetLeft+\"px\",this.container_.style.top=this.forElement_.offsetTop+this.forElement_.offsetHeight+\"px\"))}this.toggle(e)},h.prototype.handleForKeyboardEvent_=function(e){if(this.element_&&this.container_&&this.forElement_){var t=this.element_.querySelectorAll(\".\"+this.CssClasses_.ITEM+\":not([disabled])\");t&&t.length>0&&this.container_.classList.contains(this.CssClasses_.IS_VISIBLE)&&(e.keyCode===this.Keycodes_.UP_ARROW?(e.preventDefault(),t[t.length-1].focus()):e.keyCode===this.Keycodes_.DOWN_ARROW&&(e.preventDefault(),t[0].focus()))}},h.prototype.handleItemKeyboardEvent_=function(e){if(this.element_&&this.container_){var t=this.element_.querySelectorAll(\".\"+this.CssClasses_.ITEM+\":not([disabled])\");if(t&&t.length>0&&this.container_.classList.contains(this.CssClasses_.IS_VISIBLE)){var s=Array.prototype.slice.call(t).indexOf(e.target);if(e.keyCode===this.Keycodes_.UP_ARROW)e.preventDefault(),s>0?t[s-1].focus():t[t.length-1].focus();else if(e.keyCode===this.Keycodes_.DOWN_ARROW)e.preventDefault(),t.length>s+1?t[s+1].focus():t[0].focus();else if(e.keyCode===this.Keycodes_.SPACE||e.keyCode===this.Keycodes_.ENTER){e.preventDefault();var i=new MouseEvent(\"mousedown\");e.target.dispatchEvent(i),i=new MouseEvent(\"mouseup\"),e.target.dispatchEvent(i),e.target.click()}else e.keyCode===this.Keycodes_.ESCAPE&&(e.preventDefault(),this.hide())}}},h.prototype.handleItemClick_=function(e){e.target.hasAttribute(\"disabled\")?e.stopPropagation():(this.closing_=!0,window.setTimeout(function(e){this.hide(),this.closing_=!1}.bind(this),this.Constant_.CLOSE_TIMEOUT))},h.prototype.applyClip_=function(e,t){this.element_.classList.contains(this.CssClasses_.UNALIGNED)?this.element_.style.clip=\"\":this.element_.classList.contains(this.CssClasses_.BOTTOM_RIGHT)?this.element_.style.clip=\"rect(0 \"+t+\"px 0 \"+t+\"px)\":this.element_.classList.contains(this.CssClasses_.TOP_LEFT)?this.element_.style.clip=\"rect(\"+e+\"px 0 \"+e+\"px 0)\":this.element_.classList.contains(this.CssClasses_.TOP_RIGHT)?this.element_.style.clip=\"rect(\"+e+\"px \"+t+\"px \"+e+\"px \"+t+\"px)\":this.element_.style.clip=\"\"},h.prototype.removeAnimationEndListener_=function(e){e.target.classList.remove(h.prototype.CssClasses_.IS_ANIMATING)},h.prototype.addAnimationEndListener_=function(){this.element_.addEventListener(\"transitionend\",this.removeAnimationEndListener_),this.element_.addEventListener(\"webkitTransitionEnd\",this.removeAnimationEndListener_)},h.prototype.show=function(e){if(this.element_&&this.container_&&this.outline_){var t=this.element_.getBoundingClientRect().height,s=this.element_.getBoundingClientRect().width;this.container_.style.width=s+\"px\",this.container_.style.height=t+\"px\",this.outline_.style.width=s+\"px\",this.outline_.style.height=t+\"px\";for(var i=this.Constant_.TRANSITION_DURATION_SECONDS*this.Constant_.TRANSITION_DURATION_FRACTION,n=this.element_.querySelectorAll(\".\"+this.CssClasses_.ITEM),a=0;a<n.length;a++){var l;l=this.element_.classList.contains(this.CssClasses_.TOP_LEFT)||this.element_.classList.contains(this.CssClasses_.TOP_RIGHT)?(t-n[a].offsetTop-n[a].offsetHeight)/t*i+\"s\":n[a].offsetTop/t*i+\"s\",n[a].style.transitionDelay=l}this.applyClip_(t,s),window.requestAnimationFrame(function(){this.element_.classList.add(this.CssClasses_.IS_ANIMATING),this.element_.style.clip=\"rect(0 \"+s+\"px \"+t+\"px 0)\",this.container_.classList.add(this.CssClasses_.IS_VISIBLE)}.bind(this)),this.addAnimationEndListener_();var o=function(t){t===e||this.closing_||t.target.parentNode===this.element_||(document.removeEventListener(\"click\",o),this.hide())}.bind(this);document.addEventListener(\"click\",o)}},h.prototype.show=h.prototype.show,h.prototype.hide=function(){if(this.element_&&this.container_&&this.outline_){for(var e=this.element_.querySelectorAll(\".\"+this.CssClasses_.ITEM),t=0;t<e.length;t++)e[t].style.removeProperty(\"transition-delay\");var s=this.element_.getBoundingClientRect(),i=s.height,n=s.width;this.element_.classList.add(this.CssClasses_.IS_ANIMATING),this.applyClip_(i,n),this.container_.classList.remove(this.CssClasses_.IS_VISIBLE),this.addAnimationEndListener_()}},h.prototype.hide=h.prototype.hide,h.prototype.toggle=function(e){this.container_.classList.contains(this.CssClasses_.IS_VISIBLE)?this.hide():this.show(e)},h.prototype.toggle=h.prototype.toggle,i.register({constructor:h,classAsString:\"MaterialMenu\",cssClass:\"mdl-js-menu\",widget:!0});var c=function(e){this.element_=e,this.init()};window.MaterialProgress=c,c.prototype.Constant_={},c.prototype.CssClasses_={INDETERMINATE_CLASS:\"mdl-progress__indeterminate\"},c.prototype.setProgress=function(e){this.element_.classList.contains(this.CssClasses_.INDETERMINATE_CLASS)||(this.progressbar_.style.width=e+\"%\")},c.prototype.setProgress=c.prototype.setProgress,c.prototype.setBuffer=function(e){this.bufferbar_.style.width=e+\"%\",this.auxbar_.style.width=100-e+\"%\"},c.prototype.setBuffer=c.prototype.setBuffer,c.prototype.init=function(){if(this.element_){var e=document.createElement(\"div\");e.className=\"progressbar bar bar1\",this.element_.appendChild(e),this.progressbar_=e,(e=document.createElement(\"div\")).className=\"bufferbar bar bar2\",this.element_.appendChild(e),this.bufferbar_=e,(e=document.createElement(\"div\")).className=\"auxbar bar bar3\",this.element_.appendChild(e),this.auxbar_=e,this.progressbar_.style.width=\"0%\",this.bufferbar_.style.width=\"100%\",this.auxbar_.style.width=\"0%\",this.element_.classList.add(\"is-upgraded\")}},i.register({constructor:c,classAsString:\"MaterialProgress\",cssClass:\"mdl-js-progress\",widget:!0});var p=function(e){this.element_=e,this.init()};window.MaterialRadio=p,p.prototype.Constant_={TINY_TIMEOUT:.001},p.prototype.CssClasses_={IS_FOCUSED:\"is-focused\",IS_DISABLED:\"is-disabled\",IS_CHECKED:\"is-checked\",IS_UPGRADED:\"is-upgraded\",JS_RADIO:\"mdl-js-radio\",RADIO_BTN:\"mdl-radio__button\",RADIO_OUTER_CIRCLE:\"mdl-radio__outer-circle\",RADIO_INNER_CIRCLE:\"mdl-radio__inner-circle\",RIPPLE_EFFECT:\"mdl-js-ripple-effect\",RIPPLE_IGNORE_EVENTS:\"mdl-js-ripple-effect--ignore-events\",RIPPLE_CONTAINER:\"mdl-radio__ripple-container\",RIPPLE_CENTER:\"mdl-ripple--center\",RIPPLE:\"mdl-ripple\"},p.prototype.onChange_=function(e){for(var t=document.getElementsByClassName(this.CssClasses_.JS_RADIO),s=0;s<t.length;s++){t[s].querySelector(\".\"+this.CssClasses_.RADIO_BTN).getAttribute(\"name\")===this.btnElement_.getAttribute(\"name\")&&void 0!==t[s].MaterialRadio&&t[s].MaterialRadio.updateClasses_()}},p.prototype.onFocus_=function(e){this.element_.classList.add(this.CssClasses_.IS_FOCUSED)},p.prototype.onBlur_=function(e){this.element_.classList.remove(this.CssClasses_.IS_FOCUSED)},p.prototype.onMouseup_=function(e){this.blur_()},p.prototype.updateClasses_=function(){this.checkDisabled(),this.checkToggleState()},p.prototype.blur_=function(){window.setTimeout(function(){this.btnElement_.blur()}.bind(this),this.Constant_.TINY_TIMEOUT)},p.prototype.checkDisabled=function(){this.btnElement_.disabled?this.element_.classList.add(this.CssClasses_.IS_DISABLED):this.element_.classList.remove(this.CssClasses_.IS_DISABLED)},p.prototype.checkDisabled=p.prototype.checkDisabled,p.prototype.checkToggleState=function(){this.btnElement_.checked?this.element_.classList.add(this.CssClasses_.IS_CHECKED):this.element_.classList.remove(this.CssClasses_.IS_CHECKED)},p.prototype.checkToggleState=p.prototype.checkToggleState,p.prototype.disable=function(){this.btnElement_.disabled=!0,this.updateClasses_()},p.prototype.disable=p.prototype.disable,p.prototype.enable=function(){this.btnElement_.disabled=!1,this.updateClasses_()},p.prototype.enable=p.prototype.enable,p.prototype.check=function(){this.btnElement_.checked=!0,this.onChange_(null)},p.prototype.check=p.prototype.check,p.prototype.uncheck=function(){this.btnElement_.checked=!1,this.onChange_(null)},p.prototype.uncheck=p.prototype.uncheck,p.prototype.init=function(){if(this.element_){this.btnElement_=this.element_.querySelector(\".\"+this.CssClasses_.RADIO_BTN),this.boundChangeHandler_=this.onChange_.bind(this),this.boundFocusHandler_=this.onChange_.bind(this),this.boundBlurHandler_=this.onBlur_.bind(this),this.boundMouseUpHandler_=this.onMouseup_.bind(this);var e=document.createElement(\"span\");e.classList.add(this.CssClasses_.RADIO_OUTER_CIRCLE);var t,s=document.createElement(\"span\");if(s.classList.add(this.CssClasses_.RADIO_INNER_CIRCLE),this.element_.appendChild(e),this.element_.appendChild(s),this.element_.classList.contains(this.CssClasses_.RIPPLE_EFFECT)){this.element_.classList.add(this.CssClasses_.RIPPLE_IGNORE_EVENTS),(t=document.createElement(\"span\")).classList.add(this.CssClasses_.RIPPLE_CONTAINER),t.classList.add(this.CssClasses_.RIPPLE_EFFECT),t.classList.add(this.CssClasses_.RIPPLE_CENTER),t.addEventListener(\"mouseup\",this.boundMouseUpHandler_);var i=document.createElement(\"span\");i.classList.add(this.CssClasses_.RIPPLE),t.appendChild(i),this.element_.appendChild(t)}this.btnElement_.addEventListener(\"change\",this.boundChangeHandler_),this.btnElement_.addEventListener(\"focus\",this.boundFocusHandler_),this.btnElement_.addEventListener(\"blur\",this.boundBlurHandler_),this.element_.addEventListener(\"mouseup\",this.boundMouseUpHandler_),this.updateClasses_(),this.element_.classList.add(this.CssClasses_.IS_UPGRADED)}},i.register({constructor:p,classAsString:\"MaterialRadio\",cssClass:\"mdl-js-radio\",widget:!0});var C=function(e){this.element_=e,this.isIE_=window.navigator.msPointerEnabled,this.init()};window.MaterialSlider=C,C.prototype.Constant_={},C.prototype.CssClasses_={IE_CONTAINER:\"mdl-slider__ie-container\",SLIDER_CONTAINER:\"mdl-slider__container\",BACKGROUND_FLEX:\"mdl-slider__background-flex\",BACKGROUND_LOWER:\"mdl-slider__background-lower\",BACKGROUND_UPPER:\"mdl-slider__background-upper\",IS_LOWEST_VALUE:\"is-lowest-value\",IS_UPGRADED:\"is-upgraded\"},C.prototype.onInput_=function(e){this.updateValueStyles_()},C.prototype.onChange_=function(e){this.updateValueStyles_()},C.prototype.onMouseUp_=function(e){e.target.blur()},C.prototype.onContainerMouseDown_=function(e){if(e.target===this.element_.parentElement){e.preventDefault();var t=new MouseEvent(\"mousedown\",{target:e.target,buttons:e.buttons,clientX:e.clientX,clientY:this.element_.getBoundingClientRect().y});this.element_.dispatchEvent(t)}},C.prototype.updateValueStyles_=function(){var e=(this.element_.value-this.element_.min)/(this.element_.max-this.element_.min);0===e?this.element_.classList.add(this.CssClasses_.IS_LOWEST_VALUE):this.element_.classList.remove(this.CssClasses_.IS_LOWEST_VALUE),this.isIE_||(this.backgroundLower_.style.flex=e,this.backgroundLower_.style.webkitFlex=e,this.backgroundUpper_.style.flex=1-e,this.backgroundUpper_.style.webkitFlex=1-e)},C.prototype.disable=function(){this.element_.disabled=!0},C.prototype.disable=C.prototype.disable,C.prototype.enable=function(){this.element_.disabled=!1},C.prototype.enable=C.prototype.enable,C.prototype.change=function(e){void 0!==e&&(this.element_.value=e),this.updateValueStyles_()},C.prototype.change=C.prototype.change,C.prototype.init=function(){if(this.element_){if(this.isIE_){var e=document.createElement(\"div\");e.classList.add(this.CssClasses_.IE_CONTAINER),this.element_.parentElement.insertBefore(e,this.element_),this.element_.parentElement.removeChild(this.element_),e.appendChild(this.element_)}else{var t=document.createElement(\"div\");t.classList.add(this.CssClasses_.SLIDER_CONTAINER),this.element_.parentElement.insertBefore(t,this.element_),this.element_.parentElement.removeChild(this.element_),t.appendChild(this.element_);var s=document.createElement(\"div\");s.classList.add(this.CssClasses_.BACKGROUND_FLEX),t.appendChild(s),this.backgroundLower_=document.createElement(\"div\"),this.backgroundLower_.classList.add(this.CssClasses_.BACKGROUND_LOWER),s.appendChild(this.backgroundLower_),this.backgroundUpper_=document.createElement(\"div\"),this.backgroundUpper_.classList.add(this.CssClasses_.BACKGROUND_UPPER),s.appendChild(this.backgroundUpper_)}this.boundInputHandler=this.onInput_.bind(this),this.boundChangeHandler=this.onChange_.bind(this),this.boundMouseUpHandler=this.onMouseUp_.bind(this),this.boundContainerMouseDownHandler=this.onContainerMouseDown_.bind(this),this.element_.addEventListener(\"input\",this.boundInputHandler),this.element_.addEventListener(\"change\",this.boundChangeHandler),this.element_.addEventListener(\"mouseup\",this.boundMouseUpHandler),this.element_.parentElement.addEventListener(\"mousedown\",this.boundContainerMouseDownHandler),this.updateValueStyles_(),this.element_.classList.add(this.CssClasses_.IS_UPGRADED)}},i.register({constructor:C,classAsString:\"MaterialSlider\",cssClass:\"mdl-js-slider\",widget:!0});var u=function(e){if(this.element_=e,this.textElement_=this.element_.querySelector(\".\"+this.cssClasses_.MESSAGE),this.actionElement_=this.element_.querySelector(\".\"+this.cssClasses_.ACTION),!this.textElement_)throw new Error(\"There must be a message element for a snackbar.\");if(!this.actionElement_)throw new Error(\"There must be an action element for a snackbar.\");this.active=!1,this.actionHandler_=void 0,this.message_=void 0,this.actionText_=void 0,this.queuedNotifications_=[],this.setActionHidden_(!0)};window.MaterialSnackbar=u,u.prototype.Constant_={ANIMATION_LENGTH:250},u.prototype.cssClasses_={SNACKBAR:\"mdl-snackbar\",MESSAGE:\"mdl-snackbar__text\",ACTION:\"mdl-snackbar__action\",ACTIVE:\"mdl-snackbar--active\"},u.prototype.displaySnackbar_=function(){this.element_.setAttribute(\"aria-hidden\",\"true\"),this.actionHandler_&&(this.actionElement_.textContent=this.actionText_,this.actionElement_.addEventListener(\"click\",this.actionHandler_),this.setActionHidden_(!1)),this.textElement_.textContent=this.message_,this.element_.classList.add(this.cssClasses_.ACTIVE),this.element_.setAttribute(\"aria-hidden\",\"false\"),setTimeout(this.cleanup_.bind(this),this.timeout_)},u.prototype.showSnackbar=function(e){if(void 0===e)throw new Error(\"Please provide a data object with at least a message to display.\");if(void 0===e.message)throw new Error(\"Please provide a message to be displayed.\");if(e.actionHandler&&!e.actionText)throw new Error(\"Please provide action text with the handler.\");this.active?this.queuedNotifications_.push(e):(this.active=!0,this.message_=e.message,e.timeout?this.timeout_=e.timeout:this.timeout_=2750,e.actionHandler&&(this.actionHandler_=e.actionHandler),e.actionText&&(this.actionText_=e.actionText),this.displaySnackbar_())},u.prototype.showSnackbar=u.prototype.showSnackbar,u.prototype.checkQueue_=function(){this.queuedNotifications_.length>0&&this.showSnackbar(this.queuedNotifications_.shift())},u.prototype.cleanup_=function(){this.element_.classList.remove(this.cssClasses_.ACTIVE),setTimeout(function(){this.element_.setAttribute(\"aria-hidden\",\"true\"),this.textElement_.textContent=\"\",Boolean(this.actionElement_.getAttribute(\"aria-hidden\"))||(this.setActionHidden_(!0),this.actionElement_.textContent=\"\",this.actionElement_.removeEventListener(\"click\",this.actionHandler_)),this.actionHandler_=void 0,this.message_=void 0,this.actionText_=void 0,this.active=!1,this.checkQueue_()}.bind(this),this.Constant_.ANIMATION_LENGTH)},u.prototype.setActionHidden_=function(e){e?this.actionElement_.setAttribute(\"aria-hidden\",\"true\"):this.actionElement_.removeAttribute(\"aria-hidden\")},i.register({constructor:u,classAsString:\"MaterialSnackbar\",cssClass:\"mdl-js-snackbar\",widget:!0});var E=function(e){this.element_=e,this.init()};window.MaterialSpinner=E,E.prototype.Constant_={MDL_SPINNER_LAYER_COUNT:4},E.prototype.CssClasses_={MDL_SPINNER_LAYER:\"mdl-spinner__layer\",MDL_SPINNER_CIRCLE_CLIPPER:\"mdl-spinner__circle-clipper\",MDL_SPINNER_CIRCLE:\"mdl-spinner__circle\",MDL_SPINNER_GAP_PATCH:\"mdl-spinner__gap-patch\",MDL_SPINNER_LEFT:\"mdl-spinner__left\",MDL_SPINNER_RIGHT:\"mdl-spinner__right\"},E.prototype.createLayer=function(e){var t=document.createElement(\"div\");t.classList.add(this.CssClasses_.MDL_SPINNER_LAYER),t.classList.add(this.CssClasses_.MDL_SPINNER_LAYER+\"-\"+e);var s=document.createElement(\"div\");s.classList.add(this.CssClasses_.MDL_SPINNER_CIRCLE_CLIPPER),s.classList.add(this.CssClasses_.MDL_SPINNER_LEFT);var i=document.createElement(\"div\");i.classList.add(this.CssClasses_.MDL_SPINNER_GAP_PATCH);var n=document.createElement(\"div\");n.classList.add(this.CssClasses_.MDL_SPINNER_CIRCLE_CLIPPER),n.classList.add(this.CssClasses_.MDL_SPINNER_RIGHT);for(var a=[s,i,n],l=0;l<a.length;l++){var o=document.createElement(\"div\");o.classList.add(this.CssClasses_.MDL_SPINNER_CIRCLE),a[l].appendChild(o)}t.appendChild(s),t.appendChild(i),t.appendChild(n),this.element_.appendChild(t)},E.prototype.createLayer=E.prototype.createLayer,E.prototype.stop=function(){this.element_.classList.remove(\"is-active\")},E.prototype.stop=E.prototype.stop,E.prototype.start=function(){this.element_.classList.add(\"is-active\")},E.prototype.start=E.prototype.start,E.prototype.init=function(){if(this.element_){for(var e=1;e<=this.Constant_.MDL_SPINNER_LAYER_COUNT;e++)this.createLayer(e);this.element_.classList.add(\"is-upgraded\")}},i.register({constructor:E,classAsString:\"MaterialSpinner\",cssClass:\"mdl-js-spinner\",widget:!0});var m=function(e){this.element_=e,this.init()};window.MaterialSwitch=m,m.prototype.Constant_={TINY_TIMEOUT:.001},m.prototype.CssClasses_={INPUT:\"mdl-switch__input\",TRACK:\"mdl-switch__track\",THUMB:\"mdl-switch__thumb\",FOCUS_HELPER:\"mdl-switch__focus-helper\",RIPPLE_EFFECT:\"mdl-js-ripple-effect\",RIPPLE_IGNORE_EVENTS:\"mdl-js-ripple-effect--ignore-events\",RIPPLE_CONTAINER:\"mdl-switch__ripple-container\",RIPPLE_CENTER:\"mdl-ripple--center\",RIPPLE:\"mdl-ripple\",IS_FOCUSED:\"is-focused\",IS_DISABLED:\"is-disabled\",IS_CHECKED:\"is-checked\"},m.prototype.onChange_=function(e){this.updateClasses_()},m.prototype.onFocus_=function(e){this.element_.classList.add(this.CssClasses_.IS_FOCUSED)},m.prototype.onBlur_=function(e){this.element_.classList.remove(this.CssClasses_.IS_FOCUSED)},m.prototype.onMouseUp_=function(e){this.blur_()},m.prototype.updateClasses_=function(){this.checkDisabled(),this.checkToggleState()},m.prototype.blur_=function(){window.setTimeout(function(){this.inputElement_.blur()}.bind(this),this.Constant_.TINY_TIMEOUT)},m.prototype.checkDisabled=function(){this.inputElement_.disabled?this.element_.classList.add(this.CssClasses_.IS_DISABLED):this.element_.classList.remove(this.CssClasses_.IS_DISABLED)},m.prototype.checkDisabled=m.prototype.checkDisabled,m.prototype.checkToggleState=function(){this.inputElement_.checked?this.element_.classList.add(this.CssClasses_.IS_CHECKED):this.element_.classList.remove(this.CssClasses_.IS_CHECKED)},m.prototype.checkToggleState=m.prototype.checkToggleState,m.prototype.disable=function(){this.inputElement_.disabled=!0,this.updateClasses_()},m.prototype.disable=m.prototype.disable,m.prototype.enable=function(){this.inputElement_.disabled=!1,this.updateClasses_()},m.prototype.enable=m.prototype.enable,m.prototype.on=function(){this.inputElement_.checked=!0,this.updateClasses_()},m.prototype.on=m.prototype.on,m.prototype.off=function(){this.inputElement_.checked=!1,this.updateClasses_()},m.prototype.off=m.prototype.off,m.prototype.init=function(){if(this.element_){this.inputElement_=this.element_.querySelector(\".\"+this.CssClasses_.INPUT);var e=document.createElement(\"div\");e.classList.add(this.CssClasses_.TRACK);var t=document.createElement(\"div\");t.classList.add(this.CssClasses_.THUMB);var s=document.createElement(\"span\");if(s.classList.add(this.CssClasses_.FOCUS_HELPER),t.appendChild(s),this.element_.appendChild(e),this.element_.appendChild(t),this.boundMouseUpHandler=this.onMouseUp_.bind(this),this.element_.classList.contains(this.CssClasses_.RIPPLE_EFFECT)){this.element_.classList.add(this.CssClasses_.RIPPLE_IGNORE_EVENTS),this.rippleContainerElement_=document.createElement(\"span\"),this.rippleContainerElement_.classList.add(this.CssClasses_.RIPPLE_CONTAINER),this.rippleContainerElement_.classList.add(this.CssClasses_.RIPPLE_EFFECT),this.rippleContainerElement_.classList.add(this.CssClasses_.RIPPLE_CENTER),this.rippleContainerElement_.addEventListener(\"mouseup\",this.boundMouseUpHandler);var i=document.createElement(\"span\");i.classList.add(this.CssClasses_.RIPPLE),this.rippleContainerElement_.appendChild(i),this.element_.appendChild(this.rippleContainerElement_)}this.boundChangeHandler=this.onChange_.bind(this),this.boundFocusHandler=this.onFocus_.bind(this),this.boundBlurHandler=this.onBlur_.bind(this),this.inputElement_.addEventListener(\"change\",this.boundChangeHandler),this.inputElement_.addEventListener(\"focus\",this.boundFocusHandler),this.inputElement_.addEventListener(\"blur\",this.boundBlurHandler),this.element_.addEventListener(\"mouseup\",this.boundMouseUpHandler),this.updateClasses_(),this.element_.classList.add(\"is-upgraded\")}},i.register({constructor:m,classAsString:\"MaterialSwitch\",cssClass:\"mdl-js-switch\",widget:!0});var L=function(e){this.element_=e,this.init()};window.MaterialTabs=L,L.prototype.Constant_={},L.prototype.CssClasses_={TAB_CLASS:\"mdl-tabs__tab\",PANEL_CLASS:\"mdl-tabs__panel\",ACTIVE_CLASS:\"is-active\",UPGRADED_CLASS:\"is-upgraded\",MDL_JS_RIPPLE_EFFECT:\"mdl-js-ripple-effect\",MDL_RIPPLE_CONTAINER:\"mdl-tabs__ripple-container\",MDL_RIPPLE:\"mdl-ripple\",MDL_JS_RIPPLE_EFFECT_IGNORE_EVENTS:\"mdl-js-ripple-effect--ignore-events\"},L.prototype.initTabs_=function(){this.element_.classList.contains(this.CssClasses_.MDL_JS_RIPPLE_EFFECT)&&this.element_.classList.add(this.CssClasses_.MDL_JS_RIPPLE_EFFECT_IGNORE_EVENTS),this.tabs_=this.element_.querySelectorAll(\".\"+this.CssClasses_.TAB_CLASS),this.panels_=this.element_.querySelectorAll(\".\"+this.CssClasses_.PANEL_CLASS);for(var e=0;e<this.tabs_.length;e++)new t(this.tabs_[e],this);this.element_.classList.add(this.CssClasses_.UPGRADED_CLASS)},L.prototype.resetTabState_=function(){for(var e=0;e<this.tabs_.length;e++)this.tabs_[e].classList.remove(this.CssClasses_.ACTIVE_CLASS)},L.prototype.resetPanelState_=function(){for(var e=0;e<this.panels_.length;e++)this.panels_[e].classList.remove(this.CssClasses_.ACTIVE_CLASS)},L.prototype.init=function(){this.element_&&this.initTabs_()},i.register({constructor:L,classAsString:\"MaterialTabs\",cssClass:\"mdl-js-tabs\"});var I=function(e){this.element_=e,this.maxRows=this.Constant_.NO_MAX_ROWS,this.init()};window.MaterialTextfield=I,I.prototype.Constant_={NO_MAX_ROWS:-1,MAX_ROWS_ATTRIBUTE:\"maxrows\"},I.prototype.CssClasses_={LABEL:\"mdl-textfield__label\",INPUT:\"mdl-textfield__input\",IS_DIRTY:\"is-dirty\",IS_FOCUSED:\"is-focused\",IS_DISABLED:\"is-disabled\",IS_INVALID:\"is-invalid\",IS_UPGRADED:\"is-upgraded\",HAS_PLACEHOLDER:\"has-placeholder\"},I.prototype.onKeyDown_=function(e){var t=e.target.value.split(\"\\n\").length;13===e.keyCode&&t>=this.maxRows&&e.preventDefault()},I.prototype.onFocus_=function(e){this.element_.classList.add(this.CssClasses_.IS_FOCUSED)},I.prototype.onBlur_=function(e){this.element_.classList.remove(this.CssClasses_.IS_FOCUSED)},I.prototype.onReset_=function(e){this.updateClasses_()},I.prototype.updateClasses_=function(){this.checkDisabled(),this.checkValidity(),this.checkDirty(),this.checkFocus()},I.prototype.checkDisabled=function(){this.input_.disabled?this.element_.classList.add(this.CssClasses_.IS_DISABLED):this.element_.classList.remove(this.CssClasses_.IS_DISABLED)},I.prototype.checkDisabled=I.prototype.checkDisabled,I.prototype.checkFocus=function(){Boolean(this.element_.querySelector(\":focus\"))?this.element_.classList.add(this.CssClasses_.IS_FOCUSED):this.element_.classList.remove(this.CssClasses_.IS_FOCUSED)},I.prototype.checkFocus=I.prototype.checkFocus,I.prototype.checkValidity=function(){this.input_.validity&&(this.input_.validity.valid?this.element_.classList.remove(this.CssClasses_.IS_INVALID):this.element_.classList.add(this.CssClasses_.IS_INVALID))},I.prototype.checkValidity=I.prototype.checkValidity,I.prototype.checkDirty=function(){this.input_.value&&this.input_.value.length>0?this.element_.classList.add(this.CssClasses_.IS_DIRTY):this.element_.classList.remove(this.CssClasses_.IS_DIRTY)},I.prototype.checkDirty=I.prototype.checkDirty,I.prototype.disable=function(){this.input_.disabled=!0,this.updateClasses_()},I.prototype.disable=I.prototype.disable,I.prototype.enable=function(){this.input_.disabled=!1,this.updateClasses_()},I.prototype.enable=I.prototype.enable,I.prototype.change=function(e){this.input_.value=e||\"\",this.updateClasses_()},I.prototype.change=I.prototype.change,I.prototype.init=function(){if(this.element_&&(this.label_=this.element_.querySelector(\".\"+this.CssClasses_.LABEL),this.input_=this.element_.querySelector(\".\"+this.CssClasses_.INPUT),this.input_)){this.input_.hasAttribute(this.Constant_.MAX_ROWS_ATTRIBUTE)&&(this.maxRows=parseInt(this.input_.getAttribute(this.Constant_.MAX_ROWS_ATTRIBUTE),10),isNaN(this.maxRows)&&(this.maxRows=this.Constant_.NO_MAX_ROWS)),this.input_.hasAttribute(\"placeholder\")&&this.element_.classList.add(this.CssClasses_.HAS_PLACEHOLDER),this.boundUpdateClassesHandler=this.updateClasses_.bind(this),this.boundFocusHandler=this.onFocus_.bind(this),this.boundBlurHandler=this.onBlur_.bind(this),this.boundResetHandler=this.onReset_.bind(this),this.input_.addEventListener(\"input\",this.boundUpdateClassesHandler),this.input_.addEventListener(\"focus\",this.boundFocusHandler),this.input_.addEventListener(\"blur\",this.boundBlurHandler),this.input_.addEventListener(\"reset\",this.boundResetHandler),this.maxRows!==this.Constant_.NO_MAX_ROWS&&(this.boundKeyDownHandler=this.onKeyDown_.bind(this),this.input_.addEventListener(\"keydown\",this.boundKeyDownHandler));var e=this.element_.classList.contains(this.CssClasses_.IS_INVALID);this.updateClasses_(),this.element_.classList.add(this.CssClasses_.IS_UPGRADED),e&&this.element_.classList.add(this.CssClasses_.IS_INVALID),this.input_.hasAttribute(\"autofocus\")&&(this.element_.focus(),this.checkFocus())}},i.register({constructor:I,classAsString:\"MaterialTextfield\",cssClass:\"mdl-js-textfield\",widget:!0});var f=function(e){this.element_=e,this.init()};window.MaterialTooltip=f,f.prototype.Constant_={},f.prototype.CssClasses_={IS_ACTIVE:\"is-active\",BOTTOM:\"mdl-tooltip--bottom\",LEFT:\"mdl-tooltip--left\",RIGHT:\"mdl-tooltip--right\",TOP:\"mdl-tooltip--top\"},f.prototype.handleMouseEnter_=function(e){var t=e.target.getBoundingClientRect(),s=t.left+t.width/2,i=t.top+t.height/2,n=this.element_.offsetWidth/2*-1,a=this.element_.offsetHeight/2*-1;this.element_.classList.contains(this.CssClasses_.LEFT)||this.element_.classList.contains(this.CssClasses_.RIGHT)?(s=t.width/2,i+a<0?(this.element_.style.top=\"0\",this.element_.style.marginTop=\"0\"):(this.element_.style.top=i+\"px\",this.element_.style.marginTop=a+\"px\")):s+n<0?(this.element_.style.left=\"0\",this.element_.style.marginLeft=\"0\"):(this.element_.style.left=s+\"px\",this.element_.style.marginLeft=n+\"px\"),this.element_.classList.contains(this.CssClasses_.TOP)?this.element_.style.top=t.top-this.element_.offsetHeight-10+\"px\":this.element_.classList.contains(this.CssClasses_.RIGHT)?this.element_.style.left=t.left+t.width+10+\"px\":this.element_.classList.contains(this.CssClasses_.LEFT)?this.element_.style.left=t.left-this.element_.offsetWidth-10+\"px\":this.element_.style.top=t.top+t.height+10+\"px\",this.element_.classList.add(this.CssClasses_.IS_ACTIVE)},f.prototype.hideTooltip_=function(){this.element_.classList.remove(this.CssClasses_.IS_ACTIVE)},f.prototype.init=function(){if(this.element_){var e=this.element_.getAttribute(\"for\")||this.element_.getAttribute(\"data-mdl-for\");e&&(this.forElement_=document.getElementById(e)),this.forElement_&&(this.forElement_.hasAttribute(\"tabindex\")||this.forElement_.setAttribute(\"tabindex\",\"0\"),this.boundMouseEnterHandler=this.handleMouseEnter_.bind(this),this.boundMouseLeaveAndScrollHandler=this.hideTooltip_.bind(this),this.forElement_.addEventListener(\"mouseenter\",this.boundMouseEnterHandler,!1),this.forElement_.addEventListener(\"touchend\",this.boundMouseEnterHandler,!1),this.forElement_.addEventListener(\"mouseleave\",this.boundMouseLeaveAndScrollHandler,!1),window.addEventListener(\"scroll\",this.boundMouseLeaveAndScrollHandler,!0),window.addEventListener(\"touchstart\",this.boundMouseLeaveAndScrollHandler))}},i.register({constructor:f,classAsString:\"MaterialTooltip\",cssClass:\"mdl-tooltip\"});var b=function(e){this.element_=e,this.init()};window.MaterialLayout=b,b.prototype.Constant_={MAX_WIDTH:\"(max-width: 1024px)\",TAB_SCROLL_PIXELS:100,RESIZE_TIMEOUT:100,MENU_ICON:\"&#xE5D2;\",CHEVRON_LEFT:\"chevron_left\",CHEVRON_RIGHT:\"chevron_right\"},b.prototype.Keycodes_={ENTER:13,ESCAPE:27,SPACE:32},b.prototype.Mode_={STANDARD:0,SEAMED:1,WATERFALL:2,SCROLL:3},b.prototype.CssClasses_={CONTAINER:\"mdl-layout__container\",HEADER:\"mdl-layout__header\",DRAWER:\"mdl-layout__drawer\",CONTENT:\"mdl-layout__content\",DRAWER_BTN:\"mdl-layout__drawer-button\",ICON:\"material-icons\",JS_RIPPLE_EFFECT:\"mdl-js-ripple-effect\",RIPPLE_CONTAINER:\"mdl-layout__tab-ripple-container\",RIPPLE:\"mdl-ripple\",RIPPLE_IGNORE_EVENTS:\"mdl-js-ripple-effect--ignore-events\",HEADER_SEAMED:\"mdl-layout__header--seamed\",HEADER_WATERFALL:\"mdl-layout__header--waterfall\",HEADER_SCROLL:\"mdl-layout__header--scroll\",FIXED_HEADER:\"mdl-layout--fixed-header\",OBFUSCATOR:\"mdl-layout__obfuscator\",TAB_BAR:\"mdl-layout__tab-bar\",TAB_CONTAINER:\"mdl-layout__tab-bar-container\",TAB:\"mdl-layout__tab\",TAB_BAR_BUTTON:\"mdl-layout__tab-bar-button\",TAB_BAR_LEFT_BUTTON:\"mdl-layout__tab-bar-left-button\",TAB_BAR_RIGHT_BUTTON:\"mdl-layout__tab-bar-right-button\",TAB_MANUAL_SWITCH:\"mdl-layout__tab-manual-switch\",PANEL:\"mdl-layout__tab-panel\",HAS_DRAWER:\"has-drawer\",HAS_TABS:\"has-tabs\",HAS_SCROLLING_HEADER:\"has-scrolling-header\",CASTING_SHADOW:\"is-casting-shadow\",IS_COMPACT:\"is-compact\",IS_SMALL_SCREEN:\"is-small-screen\",IS_DRAWER_OPEN:\"is-visible\",IS_ACTIVE:\"is-active\",IS_UPGRADED:\"is-upgraded\",IS_ANIMATING:\"is-animating\",ON_LARGE_SCREEN:\"mdl-layout--large-screen-only\",ON_SMALL_SCREEN:\"mdl-layout--small-screen-only\"},b.prototype.contentScrollHandler_=function(){if(!this.header_.classList.contains(this.CssClasses_.IS_ANIMATING)){var e=!this.element_.classList.contains(this.CssClasses_.IS_SMALL_SCREEN)||this.element_.classList.contains(this.CssClasses_.FIXED_HEADER);this.content_.scrollTop>0&&!this.header_.classList.contains(this.CssClasses_.IS_COMPACT)?(this.header_.classList.add(this.CssClasses_.CASTING_SHADOW),this.header_.classList.add(this.CssClasses_.IS_COMPACT),e&&this.header_.classList.add(this.CssClasses_.IS_ANIMATING)):this.content_.scrollTop<=0&&this.header_.classList.contains(this.CssClasses_.IS_COMPACT)&&(this.header_.classList.remove(this.CssClasses_.CASTING_SHADOW),this.header_.classList.remove(this.CssClasses_.IS_COMPACT),e&&this.header_.classList.add(this.CssClasses_.IS_ANIMATING))}},b.prototype.keyboardEventHandler_=function(e){e.keyCode===this.Keycodes_.ESCAPE&&this.drawer_.classList.contains(this.CssClasses_.IS_DRAWER_OPEN)&&this.toggleDrawer()},b.prototype.screenSizeHandler_=function(){this.screenSizeMediaQuery_.matches?this.element_.classList.add(this.CssClasses_.IS_SMALL_SCREEN):(this.element_.classList.remove(this.CssClasses_.IS_SMALL_SCREEN),this.drawer_&&(this.drawer_.classList.remove(this.CssClasses_.IS_DRAWER_OPEN),this.obfuscator_.classList.remove(this.CssClasses_.IS_DRAWER_OPEN)))},b.prototype.drawerToggleHandler_=function(e){if(e&&\"keydown\"===e.type){if(e.keyCode!==this.Keycodes_.SPACE&&e.keyCode!==this.Keycodes_.ENTER)return;e.preventDefault()}this.toggleDrawer()},b.prototype.headerTransitionEndHandler_=function(){this.header_.classList.remove(this.CssClasses_.IS_ANIMATING)},b.prototype.headerClickHandler_=function(){this.header_.classList.contains(this.CssClasses_.IS_COMPACT)&&(this.header_.classList.remove(this.CssClasses_.IS_COMPACT),this.header_.classList.add(this.CssClasses_.IS_ANIMATING))},b.prototype.resetTabState_=function(e){for(var t=0;t<e.length;t++)e[t].classList.remove(this.CssClasses_.IS_ACTIVE)},b.prototype.resetPanelState_=function(e){for(var t=0;t<e.length;t++)e[t].classList.remove(this.CssClasses_.IS_ACTIVE)},b.prototype.toggleDrawer=function(){var e=this.element_.querySelector(\".\"+this.CssClasses_.DRAWER_BTN);this.drawer_.classList.toggle(this.CssClasses_.IS_DRAWER_OPEN),this.obfuscator_.classList.toggle(this.CssClasses_.IS_DRAWER_OPEN),this.drawer_.classList.contains(this.CssClasses_.IS_DRAWER_OPEN)?(this.drawer_.setAttribute(\"aria-hidden\",\"false\"),e.setAttribute(\"aria-expanded\",\"true\")):(this.drawer_.setAttribute(\"aria-hidden\",\"true\"),e.setAttribute(\"aria-expanded\",\"false\"))},b.prototype.toggleDrawer=b.prototype.toggleDrawer,b.prototype.init=function(){if(this.element_){var e=document.createElement(\"div\");e.classList.add(this.CssClasses_.CONTAINER);var t=this.element_.querySelector(\":focus\");this.element_.parentElement.insertBefore(e,this.element_),this.element_.parentElement.removeChild(this.element_),e.appendChild(this.element_),t&&t.focus();for(var i=this.element_.childNodes,n=i.length,a=0;a<n;a++){var l=i[a];l.classList&&l.classList.contains(this.CssClasses_.HEADER)&&(this.header_=l),l.classList&&l.classList.contains(this.CssClasses_.DRAWER)&&(this.drawer_=l),l.classList&&l.classList.contains(this.CssClasses_.CONTENT)&&(this.content_=l)}window.addEventListener(\"pageshow\",function(e){e.persisted&&(this.element_.style.overflowY=\"hidden\",requestAnimationFrame(function(){this.element_.style.overflowY=\"\"}.bind(this)))}.bind(this),!1),this.header_&&(this.tabBar_=this.header_.querySelector(\".\"+this.CssClasses_.TAB_BAR));var o=this.Mode_.STANDARD;if(this.header_&&(this.header_.classList.contains(this.CssClasses_.HEADER_SEAMED)?o=this.Mode_.SEAMED:this.header_.classList.contains(this.CssClasses_.HEADER_WATERFALL)?(o=this.Mode_.WATERFALL,this.header_.addEventListener(\"transitionend\",this.headerTransitionEndHandler_.bind(this)),this.header_.addEventListener(\"click\",this.headerClickHandler_.bind(this))):this.header_.classList.contains(this.CssClasses_.HEADER_SCROLL)&&(o=this.Mode_.SCROLL,e.classList.add(this.CssClasses_.HAS_SCROLLING_HEADER)),o===this.Mode_.STANDARD?(this.header_.classList.add(this.CssClasses_.CASTING_SHADOW),this.tabBar_&&this.tabBar_.classList.add(this.CssClasses_.CASTING_SHADOW)):o===this.Mode_.SEAMED||o===this.Mode_.SCROLL?(this.header_.classList.remove(this.CssClasses_.CASTING_SHADOW),this.tabBar_&&this.tabBar_.classList.remove(this.CssClasses_.CASTING_SHADOW)):o===this.Mode_.WATERFALL&&(this.content_.addEventListener(\"scroll\",this.contentScrollHandler_.bind(this)),this.contentScrollHandler_())),this.drawer_){var r=this.element_.querySelector(\".\"+this.CssClasses_.DRAWER_BTN);if(!r){(r=document.createElement(\"div\")).setAttribute(\"aria-expanded\",\"false\"),r.setAttribute(\"role\",\"button\"),r.setAttribute(\"tabindex\",\"0\"),r.classList.add(this.CssClasses_.DRAWER_BTN);var _=document.createElement(\"i\");_.classList.add(this.CssClasses_.ICON),_.innerHTML=this.Constant_.MENU_ICON,r.appendChild(_)}this.drawer_.classList.contains(this.CssClasses_.ON_LARGE_SCREEN)?r.classList.add(this.CssClasses_.ON_LARGE_SCREEN):this.drawer_.classList.contains(this.CssClasses_.ON_SMALL_SCREEN)&&r.classList.add(this.CssClasses_.ON_SMALL_SCREEN),r.addEventListener(\"click\",this.drawerToggleHandler_.bind(this)),r.addEventListener(\"keydown\",this.drawerToggleHandler_.bind(this)),this.element_.classList.add(this.CssClasses_.HAS_DRAWER),this.element_.classList.contains(this.CssClasses_.FIXED_HEADER)?this.header_.insertBefore(r,this.header_.firstChild):this.element_.insertBefore(r,this.content_);var d=document.createElement(\"div\");d.classList.add(this.CssClasses_.OBFUSCATOR),this.element_.appendChild(d),d.addEventListener(\"click\",this.drawerToggleHandler_.bind(this)),this.obfuscator_=d,this.drawer_.addEventListener(\"keydown\",this.keyboardEventHandler_.bind(this)),this.drawer_.setAttribute(\"aria-hidden\",\"true\")}if(this.screenSizeMediaQuery_=window.matchMedia(this.Constant_.MAX_WIDTH),this.screenSizeMediaQuery_.addListener(this.screenSizeHandler_.bind(this)),this.screenSizeHandler_(),this.header_&&this.tabBar_){this.element_.classList.add(this.CssClasses_.HAS_TABS);var h=document.createElement(\"div\");h.classList.add(this.CssClasses_.TAB_CONTAINER),this.header_.insertBefore(h,this.tabBar_),this.header_.removeChild(this.tabBar_);var c=document.createElement(\"div\");c.classList.add(this.CssClasses_.TAB_BAR_BUTTON),c.classList.add(this.CssClasses_.TAB_BAR_LEFT_BUTTON);var p=document.createElement(\"i\");p.classList.add(this.CssClasses_.ICON),p.textContent=this.Constant_.CHEVRON_LEFT,c.appendChild(p),c.addEventListener(\"click\",function(){this.tabBar_.scrollLeft-=this.Constant_.TAB_SCROLL_PIXELS}.bind(this));var C=document.createElement(\"div\");C.classList.add(this.CssClasses_.TAB_BAR_BUTTON),C.classList.add(this.CssClasses_.TAB_BAR_RIGHT_BUTTON);var u=document.createElement(\"i\");u.classList.add(this.CssClasses_.ICON),u.textContent=this.Constant_.CHEVRON_RIGHT,C.appendChild(u),C.addEventListener(\"click\",function(){this.tabBar_.scrollLeft+=this.Constant_.TAB_SCROLL_PIXELS}.bind(this)),h.appendChild(c),h.appendChild(this.tabBar_),h.appendChild(C);var E=function(){this.tabBar_.scrollLeft>0?c.classList.add(this.CssClasses_.IS_ACTIVE):c.classList.remove(this.CssClasses_.IS_ACTIVE),this.tabBar_.scrollLeft<this.tabBar_.scrollWidth-this.tabBar_.offsetWidth?C.classList.add(this.CssClasses_.IS_ACTIVE):C.classList.remove(this.CssClasses_.IS_ACTIVE)}.bind(this);this.tabBar_.addEventListener(\"scroll\",E),E();var m=function(){this.resizeTimeoutId_&&clearTimeout(this.resizeTimeoutId_),this.resizeTimeoutId_=setTimeout(function(){E(),this.resizeTimeoutId_=null}.bind(this),this.Constant_.RESIZE_TIMEOUT)}.bind(this);window.addEventListener(\"resize\",m),this.tabBar_.classList.contains(this.CssClasses_.JS_RIPPLE_EFFECT)&&this.tabBar_.classList.add(this.CssClasses_.RIPPLE_IGNORE_EVENTS);for(var L=this.tabBar_.querySelectorAll(\".\"+this.CssClasses_.TAB),I=this.content_.querySelectorAll(\".\"+this.CssClasses_.PANEL),f=0;f<L.length;f++)new s(L[f],L,I,this)}this.element_.classList.add(this.CssClasses_.IS_UPGRADED)}},window.MaterialLayoutTab=s,i.register({constructor:b,classAsString:\"MaterialLayout\",cssClass:\"mdl-js-layout\"});var S=function(e){this.element_=e,this.init()};window.MaterialDataTable=S,S.prototype.Constant_={},S.prototype.CssClasses_={DATA_TABLE:\"mdl-data-table\",SELECTABLE:\"mdl-data-table--selectable\",SELECT_ELEMENT:\"mdl-data-table__select\",IS_SELECTED:\"is-selected\",IS_UPGRADED:\"is-upgraded\"},S.prototype.selectRow_=function(e,t,s){return t?function(){e.checked?t.classList.add(this.CssClasses_.IS_SELECTED):t.classList.remove(this.CssClasses_.IS_SELECTED)}.bind(this):s?function(){var t;if(e.checked)for(t=0;t<s.length;t++)s[t].querySelector(\"td\").querySelector(\".mdl-checkbox\").MaterialCheckbox.check(),s[t].classList.add(this.CssClasses_.IS_SELECTED);else for(t=0;t<s.length;t++)s[t].querySelector(\"td\").querySelector(\".mdl-checkbox\").MaterialCheckbox.uncheck(),s[t].classList.remove(this.CssClasses_.IS_SELECTED)}.bind(this):void 0},S.prototype.createCheckbox_=function(e,t){var s=document.createElement(\"label\"),n=[\"mdl-checkbox\",\"mdl-js-checkbox\",\"mdl-js-ripple-effect\",this.CssClasses_.SELECT_ELEMENT];s.className=n.join(\" \");var a=document.createElement(\"input\");return a.type=\"checkbox\",a.classList.add(\"mdl-checkbox__input\"),e?(a.checked=e.classList.contains(this.CssClasses_.IS_SELECTED),a.addEventListener(\"change\",this.selectRow_(a,e))):t&&a.addEventListener(\"change\",this.selectRow_(a,null,t)),s.appendChild(a),i.upgradeElement(s,\"MaterialCheckbox\"),s},S.prototype.init=function(){if(this.element_){var e=this.element_.querySelector(\"th\"),t=Array.prototype.slice.call(this.element_.querySelectorAll(\"tbody tr\")),s=Array.prototype.slice.call(this.element_.querySelectorAll(\"tfoot tr\")),i=t.concat(s);if(this.element_.classList.contains(this.CssClasses_.SELECTABLE)){var n=document.createElement(\"th\"),a=this.createCheckbox_(null,i);n.appendChild(a),e.parentElement.insertBefore(n,e);for(var l=0;l<i.length;l++){var o=i[l].querySelector(\"td\");if(o){var r=document.createElement(\"td\");if(\"TBODY\"===i[l].parentNode.nodeName.toUpperCase()){var _=this.createCheckbox_(i[l]);r.appendChild(_)}i[l].insertBefore(r,o)}}this.element_.classList.add(this.CssClasses_.IS_UPGRADED)}}},i.register({constructor:S,classAsString:\"MaterialDataTable\",cssClass:\"mdl-js-data-table\"});var y=function(e){this.element_=e,this.init()};window.MaterialRipple=y,y.prototype.Constant_={INITIAL_SCALE:\"scale(0.0001, 0.0001)\",INITIAL_SIZE:\"1px\",INITIAL_OPACITY:\"0.4\",FINAL_OPACITY:\"0\",FINAL_SCALE:\"\"},y.prototype.CssClasses_={RIPPLE_CENTER:\"mdl-ripple--center\",RIPPLE_EFFECT_IGNORE_EVENTS:\"mdl-js-ripple-effect--ignore-events\",RIPPLE:\"mdl-ripple\",IS_ANIMATING:\"is-animating\",IS_VISIBLE:\"is-visible\"},y.prototype.downHandler_=function(e){if(!this.rippleElement_.style.width&&!this.rippleElement_.style.height){var t=this.element_.getBoundingClientRect();this.boundHeight=t.height,this.boundWidth=t.width,this.rippleSize_=2*Math.sqrt(t.width*t.width+t.height*t.height)+2,this.rippleElement_.style.width=this.rippleSize_+\"px\",this.rippleElement_.style.height=this.rippleSize_+\"px\"}if(this.rippleElement_.classList.add(this.CssClasses_.IS_VISIBLE),\"mousedown\"===e.type&&this.ignoringMouseDown_)this.ignoringMouseDown_=!1;else{if(\"touchstart\"===e.type&&(this.ignoringMouseDown_=!0),this.getFrameCount()>0)return;this.setFrameCount(1);var s,i,n=e.currentTarget.getBoundingClientRect();if(0===e.clientX&&0===e.clientY)s=Math.round(n.width/2),i=Math.round(n.height/2);else{var a=void 0!==e.clientX?e.clientX:e.touches[0].clientX,l=void 0!==e.clientY?e.clientY:e.touches[0].clientY;s=Math.round(a-n.left),i=Math.round(l-n.top)}this.setRippleXY(s,i),this.setRippleStyles(!0),window.requestAnimationFrame(this.animFrameHandler.bind(this))}},y.prototype.upHandler_=function(e){e&&2!==e.detail&&window.setTimeout(function(){this.rippleElement_.classList.remove(this.CssClasses_.IS_VISIBLE)}.bind(this),0)},y.prototype.init=function(){if(this.element_){var e=this.element_.classList.contains(this.CssClasses_.RIPPLE_CENTER);this.element_.classList.contains(this.CssClasses_.RIPPLE_EFFECT_IGNORE_EVENTS)||(this.rippleElement_=this.element_.querySelector(\".\"+this.CssClasses_.RIPPLE),this.frameCount_=0,this.rippleSize_=0,this.x_=0,this.y_=0,this.ignoringMouseDown_=!1,this.boundDownHandler=this.downHandler_.bind(this),this.element_.addEventListener(\"mousedown\",this.boundDownHandler),this.element_.addEventListener(\"touchstart\",this.boundDownHandler),this.boundUpHandler=this.upHandler_.bind(this),this.element_.addEventListener(\"mouseup\",this.boundUpHandler),this.element_.addEventListener(\"mouseleave\",this.boundUpHandler),this.element_.addEventListener(\"touchend\",this.boundUpHandler),this.element_.addEventListener(\"blur\",this.boundUpHandler),this.getFrameCount=function(){return this.frameCount_},this.setFrameCount=function(e){this.frameCount_=e},this.getRippleElement=function(){return this.rippleElement_},this.setRippleXY=function(e,t){this.x_=e,this.y_=t},this.setRippleStyles=function(t){if(null!==this.rippleElement_){var s,i,n=\"translate(\"+this.x_+\"px, \"+this.y_+\"px)\";t?(i=this.Constant_.INITIAL_SCALE,this.Constant_.INITIAL_SIZE):(i=this.Constant_.FINAL_SCALE,this.rippleSize_+\"px\",e&&(n=\"translate(\"+this.boundWidth/2+\"px, \"+this.boundHeight/2+\"px)\")),s=\"translate(-50%, -50%) \"+n+i,this.rippleElement_.style.webkitTransform=s,this.rippleElement_.style.msTransform=s,this.rippleElement_.style.transform=s,t?this.rippleElement_.classList.remove(this.CssClasses_.IS_ANIMATING):this.rippleElement_.classList.add(this.CssClasses_.IS_ANIMATING)}},this.animFrameHandler=function(){this.frameCount_-- >0?window.requestAnimationFrame(this.animFrameHandler.bind(this)):this.setRippleStyles(!1)})}},i.register({constructor:y,classAsString:\"MaterialRipple\",cssClass:\"mdl-js-ripple-effect\",widget:!1})}();\n},{}],\"QiIT\":[function(require,module,exports) {\n\nvar e=module.exports=\"undefined\"!=typeof window&&window.Math==Math?window:\"undefined\"!=typeof self&&self.Math==Math?self:Function(\"return this\")();\"number\"==typeof __g&&(__g=e);\n},{}],\"kOQz\":[function(require,module,exports) {\nvar r={}.hasOwnProperty;module.exports=function(e,n){return r.call(e,n)};\n},{}],\"BI7s\":[function(require,module,exports) {\nmodule.exports=function(r){try{return!!r()}catch(t){return!0}};\n},{}],\"jVdc\":[function(require,module,exports) {\nmodule.exports=!require(\"./_fails\")(function(){return 7!=Object.defineProperty({},\"a\",{get:function(){return 7}}).a});\n},{\"./_fails\":\"BI7s\"}],\"DcE6\":[function(require,module,exports) {\nvar e=module.exports={version:\"2.6.11\"};\"number\"==typeof __e&&(__e=e);\n},{}],\"tZ11\":[function(require,module,exports) {\nmodule.exports=function(o){return\"object\"==typeof o?null!==o:\"function\"==typeof o};\n},{}],\"AIrJ\":[function(require,module,exports) {\nvar r=require(\"./_is-object\");module.exports=function(e){if(!r(e))throw TypeError(e+\" is not an object!\");return e};\n},{\"./_is-object\":\"tZ11\"}],\"cz6Q\":[function(require,module,exports) {\nvar e=require(\"./_is-object\"),r=require(\"./_global\").document,t=e(r)&&e(r.createElement);module.exports=function(e){return t?r.createElement(e):{}};\n},{\"./_is-object\":\"tZ11\",\"./_global\":\"QiIT\"}],\"kIpn\":[function(require,module,exports) {\nmodule.exports=!require(\"./_descriptors\")&&!require(\"./_fails\")(function(){return 7!=Object.defineProperty(require(\"./_dom-create\")(\"div\"),\"a\",{get:function(){return 7}}).a});\n},{\"./_descriptors\":\"jVdc\",\"./_fails\":\"BI7s\",\"./_dom-create\":\"cz6Q\"}],\"S7GM\":[function(require,module,exports) {\nvar t=require(\"./_is-object\");module.exports=function(r,e){if(!t(r))return r;var o,n;if(e&&\"function\"==typeof(o=r.toString)&&!t(n=o.call(r)))return n;if(\"function\"==typeof(o=r.valueOf)&&!t(n=o.call(r)))return n;if(!e&&\"function\"==typeof(o=r.toString)&&!t(n=o.call(r)))return n;throw TypeError(\"Can't convert object to primitive value\")};\n},{\"./_is-object\":\"tZ11\"}],\"gGgn\":[function(require,module,exports) {\nvar e=require(\"./_an-object\"),r=require(\"./_ie8-dom-define\"),t=require(\"./_to-primitive\"),i=Object.defineProperty;exports.f=require(\"./_descriptors\")?Object.defineProperty:function(o,n,u){if(e(o),n=t(n,!0),e(u),r)try{return i(o,n,u)}catch(c){}if(\"get\"in u||\"set\"in u)throw TypeError(\"Accessors not supported!\");return\"value\"in u&&(o[n]=u.value),o};\n},{\"./_an-object\":\"AIrJ\",\"./_ie8-dom-define\":\"kIpn\",\"./_to-primitive\":\"S7GM\",\"./_descriptors\":\"jVdc\"}],\"zQQJ\":[function(require,module,exports) {\nmodule.exports=function(e,r){return{enumerable:!(1&e),configurable:!(2&e),writable:!(4&e),value:r}};\n},{}],\"nCfi\":[function(require,module,exports) {\nvar r=require(\"./_object-dp\"),e=require(\"./_property-desc\");module.exports=require(\"./_descriptors\")?function(t,u,o){return r.f(t,u,e(1,o))}:function(r,e,t){return r[e]=t,r};\n},{\"./_object-dp\":\"gGgn\",\"./_property-desc\":\"zQQJ\",\"./_descriptors\":\"jVdc\"}],\"jLFM\":[function(require,module,exports) {\nvar o=0,t=Math.random();module.exports=function(n){return\"Symbol(\".concat(void 0===n?\"\":n,\")_\",(++o+t).toString(36))};\n},{}],\"dG4y\":[function(require,module,exports) {\nmodule.exports=!1;\n},{}],\"k492\":[function(require,module,exports) {\n\nvar r=require(\"./_core\"),e=require(\"./_global\"),o=\"__core-js_shared__\",i=e[o]||(e[o]={});(module.exports=function(r,e){return i[r]||(i[r]=void 0!==e?e:{})})(\"versions\",[]).push({version:r.version,mode:require(\"./_library\")?\"pure\":\"global\",copyright:\"© 2019 Denis Pushkarev (zloirock.ru)\"});\n},{\"./_core\":\"DcE6\",\"./_global\":\"QiIT\",\"./_library\":\"dG4y\"}],\"it4f\":[function(require,module,exports) {\nmodule.exports=require(\"./_shared\")(\"native-function-to-string\",Function.toString);\n},{\"./_shared\":\"k492\"}],\"jDrK\":[function(require,module,exports) {\n\nvar e=require(\"./_global\"),r=require(\"./_hide\"),t=require(\"./_has\"),i=require(\"./_uid\")(\"src\"),n=require(\"./_function-to-string\"),o=\"toString\",u=(\"\"+n).split(o);require(\"./_core\").inspectSource=function(e){return n.call(e)},(module.exports=function(n,o,c,l){var s=\"function\"==typeof c;s&&(t(c,\"name\")||r(c,\"name\",o)),n[o]!==c&&(s&&(t(c,i)||r(c,i,n[o]?\"\"+n[o]:u.join(String(o)))),n===e?n[o]=c:l?n[o]?n[o]=c:r(n,o,c):(delete n[o],r(n,o,c)))})(Function.prototype,o,function(){return\"function\"==typeof this&&this[i]||n.call(this)});\n},{\"./_global\":\"QiIT\",\"./_hide\":\"nCfi\",\"./_has\":\"kOQz\",\"./_uid\":\"jLFM\",\"./_function-to-string\":\"it4f\",\"./_core\":\"DcE6\"}],\"QKlW\":[function(require,module,exports) {\nmodule.exports=function(o){if(\"function\"!=typeof o)throw TypeError(o+\" is not a function!\");return o};\n},{}],\"W8bf\":[function(require,module,exports) {\nvar r=require(\"./_a-function\");module.exports=function(n,t,u){if(r(n),void 0===t)return n;switch(u){case 1:return function(r){return n.call(t,r)};case 2:return function(r,u){return n.call(t,r,u)};case 3:return function(r,u,e){return n.call(t,r,u,e)}}return function(){return n.apply(t,arguments)}};\n},{\"./_a-function\":\"QKlW\"}],\"Vobs\":[function(require,module,exports) {\n\nvar e=require(\"./_global\"),r=require(\"./_core\"),o=require(\"./_hide\"),i=require(\"./_redefine\"),u=require(\"./_ctx\"),n=\"prototype\",t=function(c,f,l){var q,_,a,d,p=c&t.F,v=c&t.G,F=c&t.S,x=c&t.P,y=c&t.B,B=v?e:F?e[f]||(e[f]={}):(e[f]||{})[n],G=v?r:r[f]||(r[f]={}),P=G[n]||(G[n]={});for(q in v&&(l=f),l)a=((_=!p&&B&&void 0!==B[q])?B:l)[q],d=y&&_?u(a,e):x&&\"function\"==typeof a?u(Function.call,a):a,B&&i(B,q,a,c&t.U),G[q]!=a&&o(G,q,d),x&&P[q]!=a&&(P[q]=a)};e.core=r,t.F=1,t.G=2,t.S=4,t.P=8,t.B=16,t.W=32,t.U=64,t.R=128,module.exports=t;\n},{\"./_global\":\"QiIT\",\"./_core\":\"DcE6\",\"./_hide\":\"nCfi\",\"./_redefine\":\"jDrK\",\"./_ctx\":\"W8bf\"}],\"nxhn\":[function(require,module,exports) {\nvar e=require(\"./_uid\")(\"meta\"),r=require(\"./_is-object\"),t=require(\"./_has\"),n=require(\"./_object-dp\").f,i=0,u=Object.isExtensible||function(){return!0},f=!require(\"./_fails\")(function(){return u(Object.preventExtensions({}))}),o=function(r){n(r,e,{value:{i:\"O\"+ ++i,w:{}}})},s=function(n,i){if(!r(n))return\"symbol\"==typeof n?n:(\"string\"==typeof n?\"S\":\"P\")+n;if(!t(n,e)){if(!u(n))return\"F\";if(!i)return\"E\";o(n)}return n[e].i},c=function(r,n){if(!t(r,e)){if(!u(r))return!0;if(!n)return!1;o(r)}return r[e].w},E=function(r){return f&&a.NEED&&u(r)&&!t(r,e)&&o(r),r},a=module.exports={KEY:e,NEED:!1,fastKey:s,getWeak:c,onFreeze:E};\n},{\"./_uid\":\"jLFM\",\"./_is-object\":\"tZ11\",\"./_has\":\"kOQz\",\"./_object-dp\":\"gGgn\",\"./_fails\":\"BI7s\"}],\"I5XL\":[function(require,module,exports) {\nvar e=require(\"./_shared\")(\"wks\"),r=require(\"./_uid\"),o=require(\"./_global\").Symbol,u=\"function\"==typeof o,i=module.exports=function(i){return e[i]||(e[i]=u&&o[i]||(u?o:r)(\"Symbol.\"+i))};i.store=e;\n},{\"./_shared\":\"k492\",\"./_uid\":\"jLFM\",\"./_global\":\"QiIT\"}],\"IBDH\":[function(require,module,exports) {\nvar e=require(\"./_object-dp\").f,r=require(\"./_has\"),o=require(\"./_wks\")(\"toStringTag\");module.exports=function(t,u,i){t&&!r(t=i?t:t.prototype,o)&&e(t,o,{configurable:!0,value:u})};\n},{\"./_object-dp\":\"gGgn\",\"./_has\":\"kOQz\",\"./_wks\":\"I5XL\"}],\"Jnk4\":[function(require,module,exports) {\nexports.f=require(\"./_wks\");\n},{\"./_wks\":\"I5XL\"}],\"ZenZ\":[function(require,module,exports) {\n\nvar r=require(\"./_global\"),e=require(\"./_core\"),o=require(\"./_library\"),i=require(\"./_wks-ext\"),l=require(\"./_object-dp\").f;module.exports=function(u){var a=e.Symbol||(e.Symbol=o?{}:r.Symbol||{});\"_\"==u.charAt(0)||u in a||l(a,u,{value:i.f(u)})};\n},{\"./_global\":\"QiIT\",\"./_core\":\"DcE6\",\"./_library\":\"dG4y\",\"./_wks-ext\":\"Jnk4\",\"./_object-dp\":\"gGgn\"}],\"DrRY\":[function(require,module,exports) {\nvar r={}.toString;module.exports=function(t){return r.call(t).slice(8,-1)};\n},{}],\"sUp0\":[function(require,module,exports) {\nvar e=require(\"./_cof\");module.exports=Object(\"z\").propertyIsEnumerable(0)?Object:function(r){return\"String\"==e(r)?r.split(\"\"):Object(r)};\n},{\"./_cof\":\"DrRY\"}],\"V0RG\":[function(require,module,exports) {\nmodule.exports=function(o){if(null==o)throw TypeError(\"Can't call method on  \"+o);return o};\n},{}],\"zakI\":[function(require,module,exports) {\nvar e=require(\"./_iobject\"),r=require(\"./_defined\");module.exports=function(i){return e(r(i))};\n},{\"./_iobject\":\"sUp0\",\"./_defined\":\"V0RG\"}],\"ubM9\":[function(require,module,exports) {\nvar o=Math.ceil,r=Math.floor;module.exports=function(t){return isNaN(t=+t)?0:(t>0?r:o)(t)};\n},{}],\"KLzx\":[function(require,module,exports) {\nvar e=require(\"./_to-integer\"),r=Math.min;module.exports=function(t){return t>0?r(e(t),9007199254740991):0};\n},{\"./_to-integer\":\"ubM9\"}],\"tPLG\":[function(require,module,exports) {\nvar e=require(\"./_to-integer\"),r=Math.max,t=Math.min;module.exports=function(n,a){return(n=e(n))<0?r(n+a,0):t(n,a)};\n},{\"./_to-integer\":\"ubM9\"}],\"ntLR\":[function(require,module,exports) {\nvar e=require(\"./_to-iobject\"),r=require(\"./_to-length\"),t=require(\"./_to-absolute-index\");module.exports=function(n){return function(i,o,u){var f,l=e(i),a=r(l.length),c=t(u,a);if(n&&o!=o){for(;a>c;)if((f=l[c++])!=f)return!0}else for(;a>c;c++)if((n||c in l)&&l[c]===o)return n||c||0;return!n&&-1}};\n},{\"./_to-iobject\":\"zakI\",\"./_to-length\":\"KLzx\",\"./_to-absolute-index\":\"tPLG\"}],\"UE8F\":[function(require,module,exports) {\nvar e=require(\"./_shared\")(\"keys\"),r=require(\"./_uid\");module.exports=function(u){return e[u]||(e[u]=r(u))};\n},{\"./_shared\":\"k492\",\"./_uid\":\"jLFM\"}],\"tBLI\":[function(require,module,exports) {\nvar r=require(\"./_has\"),e=require(\"./_to-iobject\"),u=require(\"./_array-includes\")(!1),i=require(\"./_shared-key\")(\"IE_PROTO\");module.exports=function(o,a){var n,s=e(o),t=0,h=[];for(n in s)n!=i&&r(s,n)&&h.push(n);for(;a.length>t;)r(s,n=a[t++])&&(~u(h,n)||h.push(n));return h};\n},{\"./_has\":\"kOQz\",\"./_to-iobject\":\"zakI\",\"./_array-includes\":\"ntLR\",\"./_shared-key\":\"UE8F\"}],\"qGBL\":[function(require,module,exports) {\nmodule.exports=\"constructor,hasOwnProperty,isPrototypeOf,propertyIsEnumerable,toLocaleString,toString,valueOf\".split(\",\");\n},{}],\"huXi\":[function(require,module,exports) {\nvar e=require(\"./_object-keys-internal\"),r=require(\"./_enum-bug-keys\");module.exports=Object.keys||function(u){return e(u,r)};\n},{\"./_object-keys-internal\":\"tBLI\",\"./_enum-bug-keys\":\"qGBL\"}],\"vSss\":[function(require,module,exports) {\nexports.f=Object.getOwnPropertySymbols;\n},{}],\"NRj4\":[function(require,module,exports) {\nexports.f={}.propertyIsEnumerable;\n},{}],\"BDXu\":[function(require,module,exports) {\nvar e=require(\"./_object-keys\"),r=require(\"./_object-gops\"),o=require(\"./_object-pie\");module.exports=function(t){var u=e(t),i=r.f;if(i)for(var c,f=i(t),a=o.f,l=0;f.length>l;)a.call(t,c=f[l++])&&u.push(c);return u};\n},{\"./_object-keys\":\"huXi\",\"./_object-gops\":\"vSss\",\"./_object-pie\":\"NRj4\"}],\"JI5q\":[function(require,module,exports) {\nvar r=require(\"./_cof\");module.exports=Array.isArray||function(e){return\"Array\"==r(e)};\n},{\"./_cof\":\"DrRY\"}],\"XMZs\":[function(require,module,exports) {\nvar e=require(\"./_defined\");module.exports=function(r){return Object(e(r))};\n},{\"./_defined\":\"V0RG\"}],\"L4n9\":[function(require,module,exports) {\nvar e=require(\"./_object-dp\"),r=require(\"./_an-object\"),t=require(\"./_object-keys\");module.exports=require(\"./_descriptors\")?Object.defineProperties:function(o,i){r(o);for(var u,c=t(i),n=c.length,s=0;n>s;)e.f(o,u=c[s++],i[u]);return o};\n},{\"./_object-dp\":\"gGgn\",\"./_an-object\":\"AIrJ\",\"./_object-keys\":\"huXi\",\"./_descriptors\":\"jVdc\"}],\"HDWL\":[function(require,module,exports) {\nvar e=require(\"./_global\").document;module.exports=e&&e.documentElement;\n},{\"./_global\":\"QiIT\"}],\"EH8e\":[function(require,module,exports) {\nvar e=require(\"./_an-object\"),r=require(\"./_object-dps\"),t=require(\"./_enum-bug-keys\"),n=require(\"./_shared-key\")(\"IE_PROTO\"),o=function(){},i=\"prototype\",u=function(){var e,r=require(\"./_dom-create\")(\"iframe\"),n=t.length;for(r.style.display=\"none\",require(\"./_html\").appendChild(r),r.src=\"javascript:\",(e=r.contentWindow.document).open(),e.write(\"<script>document.F=Object<\\/script>\"),e.close(),u=e.F;n--;)delete u[i][t[n]];return u()};module.exports=Object.create||function(t,c){var a;return null!==t?(o[i]=e(t),a=new o,o[i]=null,a[n]=t):a=u(),void 0===c?a:r(a,c)};\n},{\"./_an-object\":\"AIrJ\",\"./_object-dps\":\"L4n9\",\"./_enum-bug-keys\":\"qGBL\",\"./_shared-key\":\"UE8F\",\"./_dom-create\":\"cz6Q\",\"./_html\":\"HDWL\"}],\"HNVq\":[function(require,module,exports) {\nvar e=require(\"./_object-keys-internal\"),r=require(\"./_enum-bug-keys\").concat(\"length\",\"prototype\");exports.f=Object.getOwnPropertyNames||function(t){return e(t,r)};\n},{\"./_object-keys-internal\":\"tBLI\",\"./_enum-bug-keys\":\"qGBL\"}],\"NpQ8\":[function(require,module,exports) {\nvar e=require(\"./_to-iobject\"),t=require(\"./_object-gopn\").f,o={}.toString,r=\"object\"==typeof window&&window&&Object.getOwnPropertyNames?Object.getOwnPropertyNames(window):[],n=function(e){try{return t(e)}catch(o){return r.slice()}};module.exports.f=function(c){return r&&\"[object Window]\"==o.call(c)?n(c):t(e(c))};\n},{\"./_to-iobject\":\"zakI\",\"./_object-gopn\":\"HNVq\"}],\"EGJe\":[function(require,module,exports) {\nvar e=require(\"./_object-pie\"),r=require(\"./_property-desc\"),i=require(\"./_to-iobject\"),t=require(\"./_to-primitive\"),o=require(\"./_has\"),c=require(\"./_ie8-dom-define\"),u=Object.getOwnPropertyDescriptor;exports.f=require(\"./_descriptors\")?u:function(p,q){if(p=i(p),q=t(q,!0),c)try{return u(p,q)}catch(_){}if(o(p,q))return r(!e.f.call(p,q),p[q])};\n},{\"./_object-pie\":\"NRj4\",\"./_property-desc\":\"zQQJ\",\"./_to-iobject\":\"zakI\",\"./_to-primitive\":\"S7GM\",\"./_has\":\"kOQz\",\"./_ie8-dom-define\":\"kIpn\",\"./_descriptors\":\"jVdc\"}],\"rGq9\":[function(require,module,exports) {\n\n\"use strict\";var e=require(\"./_global\"),r=require(\"./_has\"),t=require(\"./_descriptors\"),i=require(\"./_export\"),n=require(\"./_redefine\"),o=require(\"./_meta\").KEY,u=require(\"./_fails\"),s=require(\"./_shared\"),f=require(\"./_set-to-string-tag\"),c=require(\"./_uid\"),a=require(\"./_wks\"),l=require(\"./_wks-ext\"),p=require(\"./_wks-define\"),b=require(\"./_enum-keys\"),y=require(\"./_is-array\"),h=require(\"./_an-object\"),_=require(\"./_is-object\"),q=require(\"./_to-object\"),g=require(\"./_to-iobject\"),m=require(\"./_to-primitive\"),v=require(\"./_property-desc\"),d=require(\"./_object-create\"),S=require(\"./_object-gopn-ext\"),j=require(\"./_object-gopd\"),O=require(\"./_object-gops\"),w=require(\"./_object-dp\"),k=require(\"./_object-keys\"),P=j.f,F=w.f,E=S.f,N=e.Symbol,J=e.JSON,x=J&&J.stringify,I=\"prototype\",T=a(\"_hidden\"),C=a(\"toPrimitive\"),M={}.propertyIsEnumerable,D=s(\"symbol-registry\"),G=s(\"symbols\"),K=s(\"op-symbols\"),Q=Object[I],W=\"function\"==typeof N&&!!O.f,Y=e.QObject,z=!Y||!Y[I]||!Y[I].findChild,A=t&&u(function(){return 7!=d(F({},\"a\",{get:function(){return F(this,\"a\",{value:7}).a}})).a})?function(e,r,t){var i=P(Q,r);i&&delete Q[r],F(e,r,t),i&&e!==Q&&F(Q,r,i)}:F,B=function(e){var r=G[e]=d(N[I]);return r._k=e,r},H=W&&\"symbol\"==typeof N.iterator?function(e){return\"symbol\"==typeof e}:function(e){return e instanceof N},L=function(e,t,i){return e===Q&&L(K,t,i),h(e),t=m(t,!0),h(i),r(G,t)?(i.enumerable?(r(e,T)&&e[T][t]&&(e[T][t]=!1),i=d(i,{enumerable:v(0,!1)})):(r(e,T)||F(e,T,v(1,{})),e[T][t]=!0),A(e,t,i)):F(e,t,i)},R=function(e,r){h(e);for(var t,i=b(r=g(r)),n=0,o=i.length;o>n;)L(e,t=i[n++],r[t]);return e},U=function(e,r){return void 0===r?d(e):R(d(e),r)},V=function(e){var t=M.call(this,e=m(e,!0));return!(this===Q&&r(G,e)&&!r(K,e))&&(!(t||!r(this,e)||!r(G,e)||r(this,T)&&this[T][e])||t)},X=function(e,t){if(e=g(e),t=m(t,!0),e!==Q||!r(G,t)||r(K,t)){var i=P(e,t);return!i||!r(G,t)||r(e,T)&&e[T][t]||(i.enumerable=!0),i}},Z=function(e){for(var t,i=E(g(e)),n=[],u=0;i.length>u;)r(G,t=i[u++])||t==T||t==o||n.push(t);return n},$=function(e){for(var t,i=e===Q,n=E(i?K:g(e)),o=[],u=0;n.length>u;)!r(G,t=n[u++])||i&&!r(Q,t)||o.push(G[t]);return o};W||(n((N=function(){if(this instanceof N)throw TypeError(\"Symbol is not a constructor!\");var e=c(arguments.length>0?arguments[0]:void 0),i=function(t){this===Q&&i.call(K,t),r(this,T)&&r(this[T],e)&&(this[T][e]=!1),A(this,e,v(1,t))};return t&&z&&A(Q,e,{configurable:!0,set:i}),B(e)})[I],\"toString\",function(){return this._k}),j.f=X,w.f=L,require(\"./_object-gopn\").f=S.f=Z,require(\"./_object-pie\").f=V,O.f=$,t&&!require(\"./_library\")&&n(Q,\"propertyIsEnumerable\",V,!0),l.f=function(e){return B(a(e))}),i(i.G+i.W+i.F*!W,{Symbol:N});for(var ee=\"hasInstance,isConcatSpreadable,iterator,match,replace,search,species,split,toPrimitive,toStringTag,unscopables\".split(\",\"),re=0;ee.length>re;)a(ee[re++]);for(var te=k(a.store),ie=0;te.length>ie;)p(te[ie++]);i(i.S+i.F*!W,\"Symbol\",{for:function(e){return r(D,e+=\"\")?D[e]:D[e]=N(e)},keyFor:function(e){if(!H(e))throw TypeError(e+\" is not a symbol!\");for(var r in D)if(D[r]===e)return r},useSetter:function(){z=!0},useSimple:function(){z=!1}}),i(i.S+i.F*!W,\"Object\",{create:U,defineProperty:L,defineProperties:R,getOwnPropertyDescriptor:X,getOwnPropertyNames:Z,getOwnPropertySymbols:$});var ne=u(function(){O.f(1)});i(i.S+i.F*ne,\"Object\",{getOwnPropertySymbols:function(e){return O.f(q(e))}}),J&&i(i.S+i.F*(!W||u(function(){var e=N();return\"[null]\"!=x([e])||\"{}\"!=x({a:e})||\"{}\"!=x(Object(e))})),\"JSON\",{stringify:function(e){for(var r,t,i=[e],n=1;arguments.length>n;)i.push(arguments[n++]);if(t=r=i[1],(_(r)||void 0!==e)&&!H(e))return y(r)||(r=function(e,r){if(\"function\"==typeof t&&(r=t.call(this,e,r)),!H(r))return r}),i[1]=r,x.apply(J,i)}}),N[I][C]||require(\"./_hide\")(N[I],C,N[I].valueOf),f(N,\"Symbol\"),f(Math,\"Math\",!0),f(e.JSON,\"JSON\",!0);\n},{\"./_global\":\"QiIT\",\"./_has\":\"kOQz\",\"./_descriptors\":\"jVdc\",\"./_export\":\"Vobs\",\"./_redefine\":\"jDrK\",\"./_meta\":\"nxhn\",\"./_fails\":\"BI7s\",\"./_shared\":\"k492\",\"./_set-to-string-tag\":\"IBDH\",\"./_uid\":\"jLFM\",\"./_wks\":\"I5XL\",\"./_wks-ext\":\"Jnk4\",\"./_wks-define\":\"ZenZ\",\"./_enum-keys\":\"BDXu\",\"./_is-array\":\"JI5q\",\"./_an-object\":\"AIrJ\",\"./_is-object\":\"tZ11\",\"./_to-object\":\"XMZs\",\"./_to-iobject\":\"zakI\",\"./_to-primitive\":\"S7GM\",\"./_property-desc\":\"zQQJ\",\"./_object-create\":\"EH8e\",\"./_object-gopn-ext\":\"NpQ8\",\"./_object-gopd\":\"EGJe\",\"./_object-gops\":\"vSss\",\"./_object-dp\":\"gGgn\",\"./_object-keys\":\"huXi\",\"./_object-gopn\":\"HNVq\",\"./_object-pie\":\"NRj4\",\"./_library\":\"dG4y\",\"./_hide\":\"nCfi\"}],\"v5CS\":[function(require,module,exports) {\nvar e=require(\"./_export\");e(e.S,\"Object\",{create:require(\"./_object-create\")});\n},{\"./_export\":\"Vobs\",\"./_object-create\":\"EH8e\"}],\"pS46\":[function(require,module,exports) {\nvar e=require(\"./_export\");e(e.S+e.F*!require(\"./_descriptors\"),\"Object\",{defineProperty:require(\"./_object-dp\").f});\n},{\"./_export\":\"Vobs\",\"./_descriptors\":\"jVdc\",\"./_object-dp\":\"gGgn\"}],\"sbXv\":[function(require,module,exports) {\nvar e=require(\"./_export\");e(e.S+e.F*!require(\"./_descriptors\"),\"Object\",{defineProperties:require(\"./_object-dps\")});\n},{\"./_export\":\"Vobs\",\"./_descriptors\":\"jVdc\",\"./_object-dps\":\"L4n9\"}],\"gG9K\":[function(require,module,exports) {\nvar e=require(\"./_export\"),r=require(\"./_core\"),t=require(\"./_fails\");module.exports=function(c,i){var o=(r.Object||{})[c]||Object[c],u={};u[c]=i(o),e(e.S+e.F*t(function(){o(1)}),\"Object\",u)};\n},{\"./_export\":\"Vobs\",\"./_core\":\"DcE6\",\"./_fails\":\"BI7s\"}],\"xCvV\":[function(require,module,exports) {\nvar r=require(\"./_to-iobject\"),e=require(\"./_object-gopd\").f;require(\"./_object-sap\")(\"getOwnPropertyDescriptor\",function(){return function(t,o){return e(r(t),o)}});\n},{\"./_to-iobject\":\"zakI\",\"./_object-gopd\":\"EGJe\",\"./_object-sap\":\"gG9K\"}],\"dlIw\":[function(require,module,exports) {\nvar t=require(\"./_has\"),e=require(\"./_to-object\"),o=require(\"./_shared-key\")(\"IE_PROTO\"),r=Object.prototype;module.exports=Object.getPrototypeOf||function(c){return c=e(c),t(c,o)?c[o]:\"function\"==typeof c.constructor&&c instanceof c.constructor?c.constructor.prototype:c instanceof Object?r:null};\n},{\"./_has\":\"kOQz\",\"./_to-object\":\"XMZs\",\"./_shared-key\":\"UE8F\"}],\"Dkc5\":[function(require,module,exports) {\nvar e=require(\"./_to-object\"),r=require(\"./_object-gpo\");require(\"./_object-sap\")(\"getPrototypeOf\",function(){return function(t){return r(e(t))}});\n},{\"./_to-object\":\"XMZs\",\"./_object-gpo\":\"dlIw\",\"./_object-sap\":\"gG9K\"}],\"RpZ9\":[function(require,module,exports) {\nvar e=require(\"./_to-object\"),r=require(\"./_object-keys\");require(\"./_object-sap\")(\"keys\",function(){return function(t){return r(e(t))}});\n},{\"./_to-object\":\"XMZs\",\"./_object-keys\":\"huXi\",\"./_object-sap\":\"gG9K\"}],\"mVnl\":[function(require,module,exports) {\nrequire(\"./_object-sap\")(\"getOwnPropertyNames\",function(){return require(\"./_object-gopn-ext\").f});\n},{\"./_object-sap\":\"gG9K\",\"./_object-gopn-ext\":\"NpQ8\"}],\"bkZb\":[function(require,module,exports) {\nvar e=require(\"./_is-object\"),r=require(\"./_meta\").onFreeze;require(\"./_object-sap\")(\"freeze\",function(n){return function(t){return n&&e(t)?n(r(t)):t}});\n},{\"./_is-object\":\"tZ11\",\"./_meta\":\"nxhn\",\"./_object-sap\":\"gG9K\"}],\"LEG2\":[function(require,module,exports) {\nvar e=require(\"./_is-object\"),r=require(\"./_meta\").onFreeze;require(\"./_object-sap\")(\"seal\",function(n){return function(t){return n&&e(t)?n(r(t)):t}});\n},{\"./_is-object\":\"tZ11\",\"./_meta\":\"nxhn\",\"./_object-sap\":\"gG9K\"}],\"OeTo\":[function(require,module,exports) {\nvar e=require(\"./_is-object\"),r=require(\"./_meta\").onFreeze;require(\"./_object-sap\")(\"preventExtensions\",function(n){return function(t){return n&&e(t)?n(r(t)):t}});\n},{\"./_is-object\":\"tZ11\",\"./_meta\":\"nxhn\",\"./_object-sap\":\"gG9K\"}],\"Lm2M\":[function(require,module,exports) {\nvar r=require(\"./_is-object\");require(\"./_object-sap\")(\"isFrozen\",function(e){return function(n){return!r(n)||!!e&&e(n)}});\n},{\"./_is-object\":\"tZ11\",\"./_object-sap\":\"gG9K\"}],\"Lrni\":[function(require,module,exports) {\nvar e=require(\"./_is-object\");require(\"./_object-sap\")(\"isSealed\",function(r){return function(i){return!e(i)||!!r&&r(i)}});\n},{\"./_is-object\":\"tZ11\",\"./_object-sap\":\"gG9K\"}],\"ypI7\":[function(require,module,exports) {\nvar e=require(\"./_is-object\");require(\"./_object-sap\")(\"isExtensible\",function(r){return function(i){return!!e(i)&&(!r||r(i))}});\n},{\"./_is-object\":\"tZ11\",\"./_object-sap\":\"gG9K\"}],\"v89L\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_descriptors\"),r=require(\"./_object-keys\"),t=require(\"./_object-gops\"),o=require(\"./_object-pie\"),i=require(\"./_to-object\"),c=require(\"./_iobject\"),n=Object.assign;module.exports=!n||require(\"./_fails\")(function(){var e={},r={},t=Symbol(),o=\"abcdefghijklmnopqrst\";return e[t]=7,o.split(\"\").forEach(function(e){r[e]=e}),7!=n({},e)[t]||Object.keys(n({},r)).join(\"\")!=o})?function(n,u){for(var s=i(n),a=arguments.length,f=1,b=t.f,j=o.f;a>f;)for(var l,q=c(arguments[f++]),_=b?r(q).concat(b(q)):r(q),p=_.length,g=0;p>g;)l=_[g++],e&&!j.call(q,l)||(s[l]=q[l]);return s}:n;\n},{\"./_descriptors\":\"jVdc\",\"./_object-keys\":\"huXi\",\"./_object-gops\":\"vSss\",\"./_object-pie\":\"NRj4\",\"./_to-object\":\"XMZs\",\"./_iobject\":\"sUp0\",\"./_fails\":\"BI7s\"}],\"av62\":[function(require,module,exports) {\nvar e=require(\"./_export\");e(e.S+e.F,\"Object\",{assign:require(\"./_object-assign\")});\n},{\"./_export\":\"Vobs\",\"./_object-assign\":\"v89L\"}],\"wc34\":[function(require,module,exports) {\nmodule.exports=Object.is||function(e,t){return e===t?0!==e||1/e==1/t:e!=e&&t!=t};\n},{}],\"OI80\":[function(require,module,exports) {\nvar e=require(\"./_export\");e(e.S,\"Object\",{is:require(\"./_same-value\")});\n},{\"./_export\":\"Vobs\",\"./_same-value\":\"wc34\"}],\"IC1x\":[function(require,module,exports) {\nvar t=require(\"./_is-object\"),e=require(\"./_an-object\"),r=function(r,o){if(e(r),!t(o)&&null!==o)throw TypeError(o+\": can't set as prototype!\")};module.exports={set:Object.setPrototypeOf||(\"__proto__\"in{}?function(t,e,o){try{(o=require(\"./_ctx\")(Function.call,require(\"./_object-gopd\").f(Object.prototype,\"__proto__\").set,2))(t,[]),e=!(t instanceof Array)}catch(c){e=!0}return function(t,c){return r(t,c),e?t.__proto__=c:o(t,c),t}}({},!1):void 0),check:r};\n},{\"./_is-object\":\"tZ11\",\"./_an-object\":\"AIrJ\",\"./_ctx\":\"W8bf\",\"./_object-gopd\":\"EGJe\"}],\"xZ9m\":[function(require,module,exports) {\nvar e=require(\"./_export\");e(e.S,\"Object\",{setPrototypeOf:require(\"./_set-proto\").set});\n},{\"./_export\":\"Vobs\",\"./_set-proto\":\"IC1x\"}],\"pLtw\":[function(require,module,exports) {\nvar e=require(\"./_cof\"),t=require(\"./_wks\")(\"toStringTag\"),n=\"Arguments\"==e(function(){return arguments}()),r=function(e,t){try{return e[t]}catch(n){}};module.exports=function(u){var o,c,i;return void 0===u?\"Undefined\":null===u?\"Null\":\"string\"==typeof(c=r(o=Object(u),t))?c:n?e(o):\"Object\"==(i=e(o))&&\"function\"==typeof o.callee?\"Arguments\":i};\n},{\"./_cof\":\"DrRY\",\"./_wks\":\"I5XL\"}],\"zmtK\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_classof\"),r={};r[require(\"./_wks\")(\"toStringTag\")]=\"z\",r+\"\"!=\"[object z]\"&&require(\"./_redefine\")(Object.prototype,\"toString\",function(){return\"[object \"+e(this)+\"]\"},!0);\n},{\"./_classof\":\"pLtw\",\"./_wks\":\"I5XL\",\"./_redefine\":\"jDrK\"}],\"Grvq\":[function(require,module,exports) {\nmodule.exports=function(e,r,l){var a=void 0===l;switch(r.length){case 0:return a?e():e.call(l);case 1:return a?e(r[0]):e.call(l,r[0]);case 2:return a?e(r[0],r[1]):e.call(l,r[0],r[1]);case 3:return a?e(r[0],r[1],r[2]):e.call(l,r[0],r[1],r[2]);case 4:return a?e(r[0],r[1],r[2],r[3]):e.call(l,r[0],r[1],r[2],r[3])}return e.apply(l,r)};\n},{}],\"s1yo\":[function(require,module,exports) {\n\"use strict\";var n=require(\"./_a-function\"),t=require(\"./_is-object\"),r=require(\"./_invoke\"),e=[].slice,i={},o=function(n,t,r){if(!(t in i)){for(var e=[],o=0;o<t;o++)e[o]=\"a[\"+o+\"]\";i[t]=Function(\"F,a\",\"return new F(\"+e.join(\",\")+\")\")}return i[t](n,r)};module.exports=Function.bind||function(i){var u=n(this),c=e.call(arguments,1),a=function(){var n=c.concat(e.call(arguments));return this instanceof a?o(u,n.length,n):r(u,n,i)};return t(u.prototype)&&(a.prototype=u.prototype),a};\n},{\"./_a-function\":\"QKlW\",\"./_is-object\":\"tZ11\",\"./_invoke\":\"Grvq\"}],\"qI6I\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.P,\"Function\",{bind:require(\"./_bind\")});\n},{\"./_export\":\"Vobs\",\"./_bind\":\"s1yo\"}],\"z3jV\":[function(require,module,exports) {\nvar r=require(\"./_object-dp\").f,t=Function.prototype,e=/^\\s*function ([^ (]*)/,n=\"name\";n in t||require(\"./_descriptors\")&&r(t,n,{configurable:!0,get:function(){try{return(\"\"+this).match(e)[1]}catch(r){return\"\"}}});\n},{\"./_object-dp\":\"gGgn\",\"./_descriptors\":\"jVdc\"}],\"owRX\":[function(require,module,exports) {\n\"use strict\";var t=require(\"./_is-object\"),e=require(\"./_object-gpo\"),r=require(\"./_wks\")(\"hasInstance\"),i=Function.prototype;r in i||require(\"./_object-dp\").f(i,r,{value:function(r){if(\"function\"!=typeof this||!t(r))return!1;if(!t(this.prototype))return r instanceof this;for(;r=e(r);)if(this.prototype===r)return!0;return!1}});\n},{\"./_is-object\":\"tZ11\",\"./_object-gpo\":\"dlIw\",\"./_wks\":\"I5XL\",\"./_object-dp\":\"gGgn\"}],\"Pm3s\":[function(require,module,exports) {\nmodule.exports=\"\\t\\n\\v\\f\\r   ᠎             　\\u2028\\u2029\\ufeff\";\n},{}],\"JIX2\":[function(require,module,exports) {\nvar r=require(\"./_export\"),e=require(\"./_defined\"),i=require(\"./_fails\"),n=require(\"./_string-ws\"),t=\"[\"+n+\"]\",u=\"​…\",o=RegExp(\"^\"+t+t+\"*\"),p=RegExp(t+t+\"*$\"),a=function(e,t,o){var p={},a=i(function(){return!!n[e]()||u[e]()!=u}),f=p[e]=a?t(c):n[e];o&&(p[o]=f),r(r.P+r.F*a,\"String\",p)},c=a.trim=function(r,i){return r=String(e(r)),1&i&&(r=r.replace(o,\"\")),2&i&&(r=r.replace(p,\"\")),r};module.exports=a;\n},{\"./_export\":\"Vobs\",\"./_defined\":\"V0RG\",\"./_fails\":\"BI7s\",\"./_string-ws\":\"Pm3s\"}],\"UD3M\":[function(require,module,exports) {\nvar r=require(\"./_global\").parseInt,e=require(\"./_string-trim\").trim,t=require(\"./_string-ws\"),i=/^[-+]?0[xX]/;module.exports=8!==r(t+\"08\")||22!==r(t+\"0x16\")?function(t,n){var s=e(String(t),3);return r(s,n>>>0||(i.test(s)?16:10))}:r;\n},{\"./_global\":\"QiIT\",\"./_string-trim\":\"JIX2\",\"./_string-ws\":\"Pm3s\"}],\"nPGY\":[function(require,module,exports) {\nvar r=require(\"./_export\"),e=require(\"./_parse-int\");r(r.G+r.F*(parseInt!=e),{parseInt:e});\n},{\"./_export\":\"Vobs\",\"./_parse-int\":\"UD3M\"}],\"tlHn\":[function(require,module,exports) {\nvar r=require(\"./_global\").parseFloat,e=require(\"./_string-trim\").trim;module.exports=1/r(require(\"./_string-ws\")+\"-0\")!=-1/0?function(t){var i=e(String(t),3),a=r(i);return 0===a&&\"-\"==i.charAt(0)?-0:a}:r;\n},{\"./_global\":\"QiIT\",\"./_string-trim\":\"JIX2\",\"./_string-ws\":\"Pm3s\"}],\"yexh\":[function(require,module,exports) {\nvar r=require(\"./_export\"),e=require(\"./_parse-float\");r(r.G+r.F*(parseFloat!=e),{parseFloat:e});\n},{\"./_export\":\"Vobs\",\"./_parse-float\":\"tlHn\"}],\"IxAU\":[function(require,module,exports) {\nvar t=require(\"./_is-object\"),o=require(\"./_set-proto\").set;module.exports=function(r,e,p){var u,n=e.constructor;return n!==p&&\"function\"==typeof n&&(u=n.prototype)!==p.prototype&&t(u)&&o&&o(r,u),r};\n},{\"./_is-object\":\"tZ11\",\"./_set-proto\":\"IC1x\"}],\"F74v\":[function(require,module,exports) {\n\n\"use strict\";var e=require(\"./_global\"),r=require(\"./_has\"),t=require(\"./_cof\"),i=require(\"./_inherit-if-required\"),a=require(\"./_to-primitive\"),n=require(\"./_fails\"),o=require(\"./_object-gopn\").f,u=require(\"./_object-gopd\").f,s=require(\"./_object-dp\").f,c=require(\"./_string-trim\").trim,f=\"Number\",_=e[f],I=_,N=_.prototype,p=t(require(\"./_object-create\")(N))==f,l=\"trim\"in String.prototype,q=function(e){var r=a(e,!1);if(\"string\"==typeof r&&r.length>2){var t,i,n,o=(r=l?r.trim():c(r,3)).charCodeAt(0);if(43===o||45===o){if(88===(t=r.charCodeAt(2))||120===t)return NaN}else if(48===o){switch(r.charCodeAt(1)){case 66:case 98:i=2,n=49;break;case 79:case 111:i=8,n=55;break;default:return+r}for(var u,s=r.slice(2),f=0,_=s.length;f<_;f++)if((u=s.charCodeAt(f))<48||u>n)return NaN;return parseInt(s,i)}}return+r};if(!_(\" 0o1\")||!_(\"0b1\")||_(\"+0x1\")){_=function(e){var r=arguments.length<1?0:e,a=this;return a instanceof _&&(p?n(function(){N.valueOf.call(a)}):t(a)!=f)?i(new I(q(r)),a,_):q(r)};for(var g,h=require(\"./_descriptors\")?o(I):\"MAX_VALUE,MIN_VALUE,NaN,NEGATIVE_INFINITY,POSITIVE_INFINITY,EPSILON,isFinite,isInteger,isNaN,isSafeInteger,MAX_SAFE_INTEGER,MIN_SAFE_INTEGER,parseFloat,parseInt,isInteger\".split(\",\"),E=0;h.length>E;E++)r(I,g=h[E])&&!r(_,g)&&s(_,g,u(I,g));_.prototype=N,N.constructor=_,require(\"./_redefine\")(e,f,_)}\n},{\"./_global\":\"QiIT\",\"./_has\":\"kOQz\",\"./_cof\":\"DrRY\",\"./_inherit-if-required\":\"IxAU\",\"./_to-primitive\":\"S7GM\",\"./_fails\":\"BI7s\",\"./_object-gopn\":\"HNVq\",\"./_object-gopd\":\"EGJe\",\"./_object-dp\":\"gGgn\",\"./_string-trim\":\"JIX2\",\"./_object-create\":\"EH8e\",\"./_descriptors\":\"jVdc\",\"./_redefine\":\"jDrK\"}],\"Kwjt\":[function(require,module,exports) {\nvar r=require(\"./_cof\");module.exports=function(e,o){if(\"number\"!=typeof e&&\"Number\"!=r(e))throw TypeError(o);return+e};\n},{\"./_cof\":\"DrRY\"}],\"Lz3r\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_to-integer\"),e=require(\"./_defined\");module.exports=function(t){var i=String(e(this)),n=\"\",o=r(t);if(o<0||o==1/0)throw RangeError(\"Count can't be negative\");for(;o>0;(o>>>=1)&&(i+=i))1&o&&(n+=i);return n};\n},{\"./_to-integer\":\"ubM9\",\"./_defined\":\"V0RG\"}],\"qGBb\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_to-integer\"),i=require(\"./_a-number-value\"),t=require(\"./_string-repeat\"),n=1..toFixed,o=Math.floor,u=[0,0,0,0,0,0],f=\"Number.toFixed: incorrect invocation!\",a=\"0\",c=function(r,e){for(var i=-1,t=e;++i<6;)t+=r*u[i],u[i]=t%1e7,t=o(t/1e7)},l=function(r){for(var e=6,i=0;--e>=0;)i+=u[e],u[e]=o(i/r),i=i%r*1e7},v=function(){for(var r=6,e=\"\";--r>=0;)if(\"\"!==e||0===r||0!==u[r]){var i=String(u[r]);e=\"\"===e?i:e+t.call(a,7-i.length)+i}return e},x=function(r,e,i){return 0===e?i:e%2==1?x(r,e-1,i*r):x(r*r,e/2,i)},d=function(r){for(var e=0,i=r;i>=4096;)e+=12,i/=4096;for(;i>=2;)e+=1,i/=2;return e};r(r.P+r.F*(!!n&&(\"0.000\"!==8e-5.toFixed(3)||\"1\"!==.9.toFixed(0)||\"1.25\"!==1.255.toFixed(2)||\"1000000000000000128\"!==(0xde0b6b3a7640080).toFixed(0))||!require(\"./_fails\")(function(){n.call({})})),\"Number\",{toFixed:function(r){var n,o,u,s,F=i(this,f),g=e(r),b=\"\",h=a;if(g<0||g>20)throw RangeError(f);if(F!=F)return\"NaN\";if(F<=-1e21||F>=1e21)return String(F);if(F<0&&(b=\"-\",F=-F),F>1e-21)if(o=(n=d(F*x(2,69,1))-69)<0?F*x(2,-n,1):F/x(2,n,1),o*=4503599627370496,(n=52-n)>0){for(c(0,o),u=g;u>=7;)c(1e7,0),u-=7;for(c(x(10,u,1),0),u=n-1;u>=23;)l(1<<23),u-=23;l(1<<u),c(1,1),l(2),h=v()}else c(0,o),c(1<<-n,0),h=v()+t.call(a,g);return h=g>0?b+((s=h.length)<=g?\"0.\"+t.call(a,g-s)+h:h.slice(0,s-g)+\".\"+h.slice(s-g)):b+h}});\n},{\"./_export\":\"Vobs\",\"./_to-integer\":\"ubM9\",\"./_a-number-value\":\"Kwjt\",\"./_string-repeat\":\"Lz3r\",\"./_fails\":\"BI7s\"}],\"bLBB\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),i=require(\"./_fails\"),e=require(\"./_a-number-value\"),n=1..toPrecision;r(r.P+r.F*(i(function(){return\"1\"!==n.call(1,void 0)})||!i(function(){n.call({})})),\"Number\",{toPrecision:function(r){var i=e(this,\"Number#toPrecision: incorrect invocation!\");return void 0===r?n.call(i):n.call(i,r)}});\n},{\"./_export\":\"Vobs\",\"./_fails\":\"BI7s\",\"./_a-number-value\":\"Kwjt\"}],\"oSwj\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Number\",{EPSILON:Math.pow(2,-52)});\n},{\"./_export\":\"Vobs\"}],\"Iwqp\":[function(require,module,exports) {\nvar e=require(\"./_export\"),r=require(\"./_global\").isFinite;e(e.S,\"Number\",{isFinite:function(e){return\"number\"==typeof e&&r(e)}});\n},{\"./_export\":\"Vobs\",\"./_global\":\"QiIT\"}],\"tjYZ\":[function(require,module,exports) {\nvar e=require(\"./_is-object\"),r=Math.floor;module.exports=function(i){return!e(i)&&isFinite(i)&&r(i)===i};\n},{\"./_is-object\":\"tZ11\"}],\"XPnJ\":[function(require,module,exports) {\nvar e=require(\"./_export\");e(e.S,\"Number\",{isInteger:require(\"./_is-integer\")});\n},{\"./_export\":\"Vobs\",\"./_is-integer\":\"tjYZ\"}],\"PMgb\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Number\",{isNaN:function(r){return r!=r}});\n},{\"./_export\":\"Vobs\"}],\"EvBV\":[function(require,module,exports) {\nvar e=require(\"./_export\"),r=require(\"./_is-integer\"),i=Math.abs;e(e.S,\"Number\",{isSafeInteger:function(e){return r(e)&&i(e)<=9007199254740991}});\n},{\"./_export\":\"Vobs\",\"./_is-integer\":\"tjYZ\"}],\"fOC8\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Number\",{MAX_SAFE_INTEGER:9007199254740991});\n},{\"./_export\":\"Vobs\"}],\"yvVo\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Number\",{MIN_SAFE_INTEGER:-9007199254740991});\n},{\"./_export\":\"Vobs\"}],\"a09l\":[function(require,module,exports) {\nvar r=require(\"./_export\"),e=require(\"./_parse-float\");r(r.S+r.F*(Number.parseFloat!=e),\"Number\",{parseFloat:e});\n},{\"./_export\":\"Vobs\",\"./_parse-float\":\"tlHn\"}],\"fCj1\":[function(require,module,exports) {\nvar r=require(\"./_export\"),e=require(\"./_parse-int\");r(r.S+r.F*(Number.parseInt!=e),\"Number\",{parseInt:e});\n},{\"./_export\":\"Vobs\",\"./_parse-int\":\"UD3M\"}],\"rR7R\":[function(require,module,exports) {\nmodule.exports=Math.log1p||function(e){return(e=+e)>-1e-8&&e<1e-8?e-e*e/2:Math.log(1+e)};\n},{}],\"o78V\":[function(require,module,exports) {\nvar a=require(\"./_export\"),r=require(\"./_math-log1p\"),t=Math.sqrt,h=Math.acosh;a(a.S+a.F*!(h&&710==Math.floor(h(Number.MAX_VALUE))&&h(1/0)==1/0),\"Math\",{acosh:function(a){return(a=+a)<1?NaN:a>94906265.62425156?Math.log(a)+Math.LN2:r(a-1+t(a-1)*t(a+1))}});\n},{\"./_export\":\"Vobs\",\"./_math-log1p\":\"rR7R\"}],\"xkGF\":[function(require,module,exports) {\nvar t=require(\"./_export\"),a=Math.asinh;function i(t){return isFinite(t=+t)&&0!=t?t<0?-i(-t):Math.log(t+Math.sqrt(t*t+1)):t}t(t.S+t.F*!(a&&1/a(0)>0),\"Math\",{asinh:i});\n},{\"./_export\":\"Vobs\"}],\"Pmrp\":[function(require,module,exports) {\nvar a=require(\"./_export\"),t=Math.atanh;a(a.S+a.F*!(t&&1/t(-0)<0),\"Math\",{atanh:function(a){return 0==(a=+a)?a:Math.log((1+a)/(1-a))/2}});\n},{\"./_export\":\"Vobs\"}],\"ZIrZ\":[function(require,module,exports) {\nmodule.exports=Math.sign||function(n){return 0==(n=+n)||n!=n?n:n<0?-1:1};\n},{}],\"Giui\":[function(require,module,exports) {\nvar r=require(\"./_export\"),t=require(\"./_math-sign\");r(r.S,\"Math\",{cbrt:function(r){return t(r=+r)*Math.pow(Math.abs(r),1/3)}});\n},{\"./_export\":\"Vobs\",\"./_math-sign\":\"ZIrZ\"}],\"HsTu\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Math\",{clz32:function(r){return(r>>>=0)?31-Math.floor(Math.log(r+.5)*Math.LOG2E):32}});\n},{\"./_export\":\"Vobs\"}],\"xEUq\":[function(require,module,exports) {\nvar r=require(\"./_export\"),e=Math.exp;r(r.S,\"Math\",{cosh:function(r){return(e(r=+r)+e(-r))/2}});\n},{\"./_export\":\"Vobs\"}],\"sm22\":[function(require,module,exports) {\nvar e=Math.expm1;module.exports=!e||e(10)>22025.465794806718||e(10)<22025.465794806718||-2e-17!=e(-2e-17)?function(e){return 0==(e=+e)?e:e>-1e-6&&e<1e-6?e+e*e/2:Math.exp(e)-1}:e;\n},{}],\"aBEU\":[function(require,module,exports) {\nvar e=require(\"./_export\"),r=require(\"./_math-expm1\");e(e.S+e.F*(r!=Math.expm1),\"Math\",{expm1:r});\n},{\"./_export\":\"Vobs\",\"./_math-expm1\":\"sm22\"}],\"lqkS\":[function(require,module,exports) {\nvar r=require(\"./_math-sign\"),t=Math.pow,n=t(2,-52),a=t(2,-23),u=t(2,127)*(2-a),e=t(2,-126),o=function(r){return r+1/n-1/n};module.exports=Math.fround||function(t){var h,i,f=Math.abs(t),s=r(t);return f<e?s*o(f/e/a)*e*a:(i=(h=(1+a/n)*f)-(h-f))>u||i!=i?s*(1/0):s*i};\n},{\"./_math-sign\":\"ZIrZ\"}],\"IjCR\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Math\",{fround:require(\"./_math-fround\")});\n},{\"./_export\":\"Vobs\",\"./_math-fround\":\"lqkS\"}],\"HXfT\":[function(require,module,exports) {\nvar r=require(\"./_export\"),t=Math.abs;r(r.S,\"Math\",{hypot:function(r,a){for(var e,h,n=0,o=0,u=arguments.length,M=0;o<u;)M<(e=t(arguments[o++]))?(n=n*(h=M/e)*h+1,M=e):n+=e>0?(h=e/M)*h:e;return M===1/0?1/0:M*Math.sqrt(n)}});\n},{\"./_export\":\"Vobs\"}],\"m2OX\":[function(require,module,exports) {\nvar r=require(\"./_export\"),e=Math.imul;r(r.S+r.F*require(\"./_fails\")(function(){return-5!=e(4294967295,5)||2!=e.length}),\"Math\",{imul:function(r,e){var t=+r,u=+e,i=65535&t,n=65535&u;return 0|i*n+((65535&t>>>16)*n+i*(65535&u>>>16)<<16>>>0)}});\n},{\"./_export\":\"Vobs\",\"./_fails\":\"BI7s\"}],\"E567\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Math\",{log10:function(r){return Math.log(r)*Math.LOG10E}});\n},{\"./_export\":\"Vobs\"}],\"ymfv\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Math\",{log1p:require(\"./_math-log1p\")});\n},{\"./_export\":\"Vobs\",\"./_math-log1p\":\"rR7R\"}],\"hUIM\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Math\",{log2:function(r){return Math.log(r)/Math.LN2}});\n},{\"./_export\":\"Vobs\"}],\"d1Y4\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Math\",{sign:require(\"./_math-sign\")});\n},{\"./_export\":\"Vobs\",\"./_math-sign\":\"ZIrZ\"}],\"dhHM\":[function(require,module,exports) {\nvar e=require(\"./_export\"),r=require(\"./_math-expm1\"),t=Math.exp;e(e.S+e.F*require(\"./_fails\")(function(){return-2e-17!=!Math.sinh(-2e-17)}),\"Math\",{sinh:function(e){return Math.abs(e=+e)<1?(r(e)-r(-e))/2:(t(e-1)-t(-e-1))*(Math.E/2)}});\n},{\"./_export\":\"Vobs\",\"./_math-expm1\":\"sm22\",\"./_fails\":\"BI7s\"}],\"cxv8\":[function(require,module,exports) {\nvar r=require(\"./_export\"),e=require(\"./_math-expm1\"),t=Math.exp;r(r.S,\"Math\",{tanh:function(r){var a=e(r=+r),h=e(-r);return a==1/0?1:h==1/0?-1:(a-h)/(t(r)+t(-r))}});\n},{\"./_export\":\"Vobs\",\"./_math-expm1\":\"sm22\"}],\"xO7u\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Math\",{trunc:function(r){return(r>0?Math.floor:Math.ceil)(r)}});\n},{\"./_export\":\"Vobs\"}],\"DdG0\":[function(require,module,exports) {\nvar r=require(\"./_export\"),o=require(\"./_to-absolute-index\"),e=String.fromCharCode,n=String.fromCodePoint;r(r.S+r.F*(!!n&&1!=n.length),\"String\",{fromCodePoint:function(r){for(var n,t=[],i=arguments.length,a=0;i>a;){if(n=+arguments[a++],o(n,1114111)!==n)throw RangeError(n+\" is not a valid code point\");t.push(n<65536?e(n):e(55296+((n-=65536)>>10),n%1024+56320))}return t.join(\"\")}});\n},{\"./_export\":\"Vobs\",\"./_to-absolute-index\":\"tPLG\"}],\"KDcE\":[function(require,module,exports) {\nvar r=require(\"./_export\"),e=require(\"./_to-iobject\"),t=require(\"./_to-length\");r(r.S,\"String\",{raw:function(r){for(var n=e(r.raw),i=t(n.length),o=arguments.length,u=[],g=0;i>g;)u.push(String(n[g++])),g<o&&u.push(String(arguments[g]));return u.join(\"\")}});\n},{\"./_export\":\"Vobs\",\"./_to-iobject\":\"zakI\",\"./_to-length\":\"KLzx\"}],\"DDrZ\":[function(require,module,exports) {\n\"use strict\";require(\"./_string-trim\")(\"trim\",function(r){return function(){return r(this,3)}});\n},{\"./_string-trim\":\"JIX2\"}],\"j93N\":[function(require,module,exports) {\nvar e=require(\"./_to-integer\"),r=require(\"./_defined\");module.exports=function(t){return function(n,i){var o,u,c=String(r(n)),d=e(i),a=c.length;return d<0||d>=a?t?\"\":void 0:(o=c.charCodeAt(d))<55296||o>56319||d+1===a||(u=c.charCodeAt(d+1))<56320||u>57343?t?c.charAt(d):o:t?c.slice(d,d+2):u-56320+(o-55296<<10)+65536}};\n},{\"./_to-integer\":\"ubM9\",\"./_defined\":\"V0RG\"}],\"H5RD\":[function(require,module,exports) {\nmodule.exports={};\n},{}],\"gj4O\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_object-create\"),r=require(\"./_property-desc\"),t=require(\"./_set-to-string-tag\"),i={};require(\"./_hide\")(i,require(\"./_wks\")(\"iterator\"),function(){return this}),module.exports=function(o,u,s){o.prototype=e(i,{next:r(1,s)}),t(o,u+\" Iterator\")};\n},{\"./_object-create\":\"EH8e\",\"./_property-desc\":\"zQQJ\",\"./_set-to-string-tag\":\"IBDH\",\"./_hide\":\"nCfi\",\"./_wks\":\"I5XL\"}],\"MKcl\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_library\"),r=require(\"./_export\"),t=require(\"./_redefine\"),i=require(\"./_hide\"),n=require(\"./_iterators\"),u=require(\"./_iter-create\"),o=require(\"./_set-to-string-tag\"),s=require(\"./_object-gpo\"),a=require(\"./_wks\")(\"iterator\"),c=!([].keys&&\"next\"in[].keys()),f=\"@@iterator\",l=\"keys\",q=\"values\",y=function(){return this};module.exports=function(_,p,h,k,v,w,d){u(h,p,k);var x,b,g,j=function(e){if(!c&&e in I)return I[e];switch(e){case l:case q:return function(){return new h(this,e)}}return function(){return new h(this,e)}},m=p+\" Iterator\",A=v==q,F=!1,I=_.prototype,O=I[a]||I[f]||v&&I[v],P=O||j(v),z=v?A?j(\"entries\"):P:void 0,B=\"Array\"==p&&I.entries||O;if(B&&(g=s(B.call(new _)))!==Object.prototype&&g.next&&(o(g,m,!0),e||\"function\"==typeof g[a]||i(g,a,y)),A&&O&&O.name!==q&&(F=!0,P=function(){return O.call(this)}),e&&!d||!c&&!F&&I[a]||i(I,a,P),n[p]=P,n[m]=y,v)if(x={values:A?P:j(q),keys:w?P:j(l),entries:z},d)for(b in x)b in I||t(I,b,x[b]);else r(r.P+r.F*(c||F),p,x);return x};\n},{\"./_library\":\"dG4y\",\"./_export\":\"Vobs\",\"./_redefine\":\"jDrK\",\"./_hide\":\"nCfi\",\"./_iterators\":\"H5RD\",\"./_iter-create\":\"gj4O\",\"./_set-to-string-tag\":\"IBDH\",\"./_object-gpo\":\"dlIw\",\"./_wks\":\"I5XL\"}],\"WN4F\":[function(require,module,exports) {\n\"use strict\";var i=require(\"./_string-at\")(!0);require(\"./_iter-define\")(String,\"String\",function(i){this._t=String(i),this._i=0},function(){var t,e=this._t,n=this._i;return n>=e.length?{value:void 0,done:!0}:(t=i(e,n),this._i+=t.length,{value:t,done:!1})});\n},{\"./_string-at\":\"j93N\",\"./_iter-define\":\"MKcl\"}],\"gGid\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),t=require(\"./_string-at\")(!1);r(r.P,\"String\",{codePointAt:function(r){return t(this,r)}});\n},{\"./_export\":\"Vobs\",\"./_string-at\":\"j93N\"}],\"r5g1\":[function(require,module,exports) {\nvar e=require(\"./_is-object\"),r=require(\"./_cof\"),i=require(\"./_wks\")(\"match\");module.exports=function(o){var u;return e(o)&&(void 0!==(u=o[i])?!!u:\"RegExp\"==r(o))};\n},{\"./_is-object\":\"tZ11\",\"./_cof\":\"DrRY\",\"./_wks\":\"I5XL\"}],\"dpxX\":[function(require,module,exports) {\nvar e=require(\"./_is-regexp\"),r=require(\"./_defined\");module.exports=function(i,t,n){if(e(t))throw TypeError(\"String#\"+n+\" doesn't accept regex!\");return String(r(i))};\n},{\"./_is-regexp\":\"r5g1\",\"./_defined\":\"V0RG\"}],\"Z7lT\":[function(require,module,exports) {\nvar r=require(\"./_wks\")(\"match\");module.exports=function(t){var c=/./;try{\"/./\"[t](c)}catch(e){try{return c[r]=!1,!\"/./\"[t](c)}catch(a){}}return!0};\n},{\"./_wks\":\"I5XL\"}],\"PmIB\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_export\"),t=require(\"./_to-length\"),i=require(\"./_string-context\"),r=\"endsWith\",n=\"\"[r];e(e.P+e.F*require(\"./_fails-is-regexp\")(r),\"String\",{endsWith:function(e){var s=i(this,e,r),g=arguments.length>1?arguments[1]:void 0,h=t(s.length),l=void 0===g?h:Math.min(t(g),h),u=String(e);return n?n.call(s,u,l):s.slice(l-u.length,l)===u}});\n},{\"./_export\":\"Vobs\",\"./_to-length\":\"KLzx\",\"./_string-context\":\"dpxX\",\"./_fails-is-regexp\":\"Z7lT\"}],\"qgIv\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_export\"),i=require(\"./_string-context\"),r=\"includes\";e(e.P+e.F*require(\"./_fails-is-regexp\")(r),\"String\",{includes:function(e){return!!~i(this,e,r).indexOf(e,arguments.length>1?arguments[1]:void 0)}});\n},{\"./_export\":\"Vobs\",\"./_string-context\":\"dpxX\",\"./_fails-is-regexp\":\"Z7lT\"}],\"ZAbm\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.P,\"String\",{repeat:require(\"./_string-repeat\")});\n},{\"./_export\":\"Vobs\",\"./_string-repeat\":\"Lz3r\"}],\"U3MC\":[function(require,module,exports) {\n\"use strict\";var t=require(\"./_export\"),r=require(\"./_to-length\"),e=require(\"./_string-context\"),i=\"startsWith\",n=\"\"[i];t(t.P+t.F*require(\"./_fails-is-regexp\")(i),\"String\",{startsWith:function(t){var s=e(this,t,i),g=r(Math.min(arguments.length>1?arguments[1]:void 0,s.length)),h=String(t);return n?n.call(s,h,g):s.slice(g,g+h.length)===h}});\n},{\"./_export\":\"Vobs\",\"./_to-length\":\"KLzx\",\"./_string-context\":\"dpxX\",\"./_fails-is-regexp\":\"Z7lT\"}],\"OaTR\":[function(require,module,exports) {\nvar r=require(\"./_export\"),e=require(\"./_fails\"),t=require(\"./_defined\"),n=/\"/g,i=function(r,e,i,u){var o=String(t(r)),a=\"<\"+e;return\"\"!==i&&(a+=\" \"+i+'=\"'+String(u).replace(n,\"&quot;\")+'\"'),a+\">\"+o+\"</\"+e+\">\"};module.exports=function(t,n){var u={};u[t]=n(i),r(r.P+r.F*e(function(){var r=\"\"[t]('\"');return r!==r.toLowerCase()||r.split('\"').length>3}),\"String\",u)};\n},{\"./_export\":\"Vobs\",\"./_fails\":\"BI7s\",\"./_defined\":\"V0RG\"}],\"eRhq\":[function(require,module,exports) {\n\"use strict\";require(\"./_string-html\")(\"anchor\",function(n){return function(r){return n(this,\"a\",\"name\",r)}});\n},{\"./_string-html\":\"OaTR\"}],\"HLSM\":[function(require,module,exports) {\n\"use strict\";require(\"./_string-html\")(\"big\",function(t){return function(){return t(this,\"big\",\"\",\"\")}});\n},{\"./_string-html\":\"OaTR\"}],\"RtH9\":[function(require,module,exports) {\n\"use strict\";require(\"./_string-html\")(\"blink\",function(n){return function(){return n(this,\"blink\",\"\",\"\")}});\n},{\"./_string-html\":\"OaTR\"}],\"efe7\":[function(require,module,exports) {\n\"use strict\";require(\"./_string-html\")(\"bold\",function(t){return function(){return t(this,\"b\",\"\",\"\")}});\n},{\"./_string-html\":\"OaTR\"}],\"v3Ez\":[function(require,module,exports) {\n\"use strict\";require(\"./_string-html\")(\"fixed\",function(t){return function(){return t(this,\"tt\",\"\",\"\")}});\n},{\"./_string-html\":\"OaTR\"}],\"RECM\":[function(require,module,exports) {\n\"use strict\";require(\"./_string-html\")(\"fontcolor\",function(t){return function(r){return t(this,\"font\",\"color\",r)}});\n},{\"./_string-html\":\"OaTR\"}],\"l7OI\":[function(require,module,exports) {\n\"use strict\";require(\"./_string-html\")(\"fontsize\",function(t){return function(n){return t(this,\"font\",\"size\",n)}});\n},{\"./_string-html\":\"OaTR\"}],\"uJlj\":[function(require,module,exports) {\n\"use strict\";require(\"./_string-html\")(\"italics\",function(t){return function(){return t(this,\"i\",\"\",\"\")}});\n},{\"./_string-html\":\"OaTR\"}],\"vYww\":[function(require,module,exports) {\n\"use strict\";require(\"./_string-html\")(\"link\",function(r){return function(t){return r(this,\"a\",\"href\",t)}});\n},{\"./_string-html\":\"OaTR\"}],\"AiXZ\":[function(require,module,exports) {\n\"use strict\";require(\"./_string-html\")(\"small\",function(t){return function(){return t(this,\"small\",\"\",\"\")}});\n},{\"./_string-html\":\"OaTR\"}],\"MhVl\":[function(require,module,exports) {\n\"use strict\";require(\"./_string-html\")(\"strike\",function(t){return function(){return t(this,\"strike\",\"\",\"\")}});\n},{\"./_string-html\":\"OaTR\"}],\"DFMN\":[function(require,module,exports) {\n\"use strict\";require(\"./_string-html\")(\"sub\",function(t){return function(){return t(this,\"sub\",\"\",\"\")}});\n},{\"./_string-html\":\"OaTR\"}],\"X3LC\":[function(require,module,exports) {\n\"use strict\";require(\"./_string-html\")(\"sup\",function(t){return function(){return t(this,\"sup\",\"\",\"\")}});\n},{\"./_string-html\":\"OaTR\"}],\"Sydr\":[function(require,module,exports) {\nvar e=require(\"./_export\");e(e.S,\"Date\",{now:function(){return(new Date).getTime()}});\n},{\"./_export\":\"Vobs\"}],\"GNUn\":[function(require,module,exports) {\n\"use strict\";var t=require(\"./_export\"),e=require(\"./_to-object\"),r=require(\"./_to-primitive\");t(t.P+t.F*require(\"./_fails\")(function(){return null!==new Date(NaN).toJSON()||1!==Date.prototype.toJSON.call({toISOString:function(){return 1}})}),\"Date\",{toJSON:function(t){var i=e(this),n=r(i);return\"number\"!=typeof n||isFinite(n)?i.toISOString():null}});\n},{\"./_export\":\"Vobs\",\"./_to-object\":\"XMZs\",\"./_to-primitive\":\"S7GM\",\"./_fails\":\"BI7s\"}],\"wk7G\":[function(require,module,exports) {\n\"use strict\";var t=require(\"./_fails\"),e=Date.prototype.getTime,i=Date.prototype.toISOString,n=function(t){return t>9?t:\"0\"+t};module.exports=t(function(){return\"0385-07-25T07:06:39.999Z\"!=i.call(new Date(-5e13-1))})||!t(function(){i.call(new Date(NaN))})?function(){if(!isFinite(e.call(this)))throw RangeError(\"Invalid time value\");var t=this,i=t.getUTCFullYear(),r=t.getUTCMilliseconds(),a=i<0?\"-\":i>9999?\"+\":\"\";return a+(\"00000\"+Math.abs(i)).slice(a?-6:-4)+\"-\"+n(t.getUTCMonth()+1)+\"-\"+n(t.getUTCDate())+\"T\"+n(t.getUTCHours())+\":\"+n(t.getUTCMinutes())+\":\"+n(t.getUTCSeconds())+\".\"+(r>99?r:\"0\"+n(r))+\"Z\"}:i;\n},{\"./_fails\":\"BI7s\"}],\"fPZl\":[function(require,module,exports) {\nvar t=require(\"./_export\"),r=require(\"./_date-to-iso-string\");t(t.P+t.F*(Date.prototype.toISOString!==r),\"Date\",{toISOString:r});\n},{\"./_export\":\"Vobs\",\"./_date-to-iso-string\":\"wk7G\"}],\"FKfL\":[function(require,module,exports) {\nvar e=Date.prototype,t=\"Invalid Date\",a=\"toString\",r=e[a],i=e.getTime;new Date(NaN)+\"\"!=t&&require(\"./_redefine\")(e,a,function(){var e=i.call(this);return e==e?r.call(this):t});\n},{\"./_redefine\":\"jDrK\"}],\"EnIA\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_an-object\"),e=require(\"./_to-primitive\"),t=\"number\";module.exports=function(i){if(\"string\"!==i&&i!==t&&\"default\"!==i)throw TypeError(\"Incorrect hint\");return e(r(this),i!=t)};\n},{\"./_an-object\":\"AIrJ\",\"./_to-primitive\":\"S7GM\"}],\"nktC\":[function(require,module,exports) {\nvar e=require(\"./_wks\")(\"toPrimitive\"),i=Date.prototype;e in i||require(\"./_hide\")(i,e,require(\"./_date-to-primitive\"));\n},{\"./_wks\":\"I5XL\",\"./_hide\":\"nCfi\",\"./_date-to-primitive\":\"EnIA\"}],\"XjkF\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Array\",{isArray:require(\"./_is-array\")});\n},{\"./_export\":\"Vobs\",\"./_is-array\":\"JI5q\"}],\"RG8K\":[function(require,module,exports) {\nvar r=require(\"./_an-object\");module.exports=function(t,e,o,a){try{return a?e(r(o)[0],o[1]):e(o)}catch(n){var c=t.return;throw void 0!==c&&r(c.call(t)),n}};\n},{\"./_an-object\":\"AIrJ\"}],\"TuHS\":[function(require,module,exports) {\nvar r=require(\"./_iterators\"),e=require(\"./_wks\")(\"iterator\"),t=Array.prototype;module.exports=function(o){return void 0!==o&&(r.Array===o||t[e]===o)};\n},{\"./_iterators\":\"H5RD\",\"./_wks\":\"I5XL\"}],\"g07e\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_object-dp\"),r=require(\"./_property-desc\");module.exports=function(t,i,o){i in t?e.f(t,i,r(0,o)):t[i]=o};\n},{\"./_object-dp\":\"gGgn\",\"./_property-desc\":\"zQQJ\"}],\"um4Z\":[function(require,module,exports) {\nvar r=require(\"./_classof\"),e=require(\"./_wks\")(\"iterator\"),t=require(\"./_iterators\");module.exports=require(\"./_core\").getIteratorMethod=function(o){if(null!=o)return o[e]||o[\"@@iterator\"]||t[r(o)]};\n},{\"./_classof\":\"pLtw\",\"./_wks\":\"I5XL\",\"./_iterators\":\"H5RD\",\"./_core\":\"DcE6\"}],\"zP7t\":[function(require,module,exports) {\nvar r=require(\"./_wks\")(\"iterator\"),t=!1;try{var n=[7][r]();n.return=function(){t=!0},Array.from(n,function(){throw 2})}catch(e){}module.exports=function(n,u){if(!u&&!t)return!1;var o=!1;try{var c=[7],a=c[r]();a.next=function(){return{done:o=!0}},c[r]=function(){return a},n(c)}catch(e){}return o};\n},{\"./_wks\":\"I5XL\"}],\"WZRw\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_ctx\"),r=require(\"./_export\"),t=require(\"./_to-object\"),i=require(\"./_iter-call\"),o=require(\"./_is-array-iter\"),u=require(\"./_to-length\"),n=require(\"./_create-property\"),a=require(\"./core.get-iterator-method\");r(r.S+r.F*!require(\"./_iter-detect\")(function(e){Array.from(e)}),\"Array\",{from:function(r){var l,c,f,q,_=t(r),h=\"function\"==typeof this?this:Array,v=arguments.length,y=v>1?arguments[1]:void 0,d=void 0!==y,s=0,g=a(_);if(d&&(y=e(y,v>2?arguments[2]:void 0,2)),null==g||h==Array&&o(g))for(c=new h(l=u(_.length));l>s;s++)n(c,s,d?y(_[s],s):_[s]);else for(q=g.call(_),c=new h;!(f=q.next()).done;s++)n(c,s,d?i(q,y,[f.value,s],!0):f.value);return c.length=s,c}});\n},{\"./_ctx\":\"W8bf\",\"./_export\":\"Vobs\",\"./_to-object\":\"XMZs\",\"./_iter-call\":\"RG8K\",\"./_is-array-iter\":\"TuHS\",\"./_to-length\":\"KLzx\",\"./_create-property\":\"g07e\",\"./core.get-iterator-method\":\"um4Z\",\"./_iter-detect\":\"zP7t\"}],\"URTo\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_create-property\");r(r.S+r.F*require(\"./_fails\")(function(){function r(){}return!(Array.of.call(r)instanceof r)}),\"Array\",{of:function(){for(var r=0,t=arguments.length,n=new(\"function\"==typeof this?this:Array)(t);t>r;)e(n,r,arguments[r++]);return n.length=t,n}});\n},{\"./_export\":\"Vobs\",\"./_create-property\":\"g07e\",\"./_fails\":\"BI7s\"}],\"TiCE\":[function(require,module,exports) {\n\"use strict\";var l=require(\"./_fails\");module.exports=function(n,u){return!!n&&l(function(){u?n.call(null,function(){},1):n.call(null)})};\n},{\"./_fails\":\"BI7s\"}],\"BTDR\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_to-iobject\"),i=[].join;r(r.P+r.F*(require(\"./_iobject\")!=Object||!require(\"./_strict-method\")(i)),\"Array\",{join:function(r){return i.call(e(this),void 0===r?\",\":r)}});\n},{\"./_export\":\"Vobs\",\"./_to-iobject\":\"zakI\",\"./_iobject\":\"sUp0\",\"./_strict-method\":\"TiCE\"}],\"Ui7t\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_html\"),i=require(\"./_cof\"),t=require(\"./_to-absolute-index\"),u=require(\"./_to-length\"),a=[].slice;r(r.P+r.F*require(\"./_fails\")(function(){e&&a.call(e)}),\"Array\",{slice:function(r,e){var l=u(this.length),n=i(this);if(e=void 0===e?l:e,\"Array\"==n)return a.call(this,r,e);for(var s=t(r,l),c=t(e,l),h=u(c-s),o=new Array(h),f=0;f<h;f++)o[f]=\"String\"==n?this.charAt(s+f):this[s+f];return o}});\n},{\"./_export\":\"Vobs\",\"./_html\":\"HDWL\",\"./_cof\":\"DrRY\",\"./_to-absolute-index\":\"tPLG\",\"./_to-length\":\"KLzx\",\"./_fails\":\"BI7s\"}],\"TqUy\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),t=require(\"./_a-function\"),i=require(\"./_to-object\"),e=require(\"./_fails\"),o=[].sort,u=[1,2,3];r(r.P+r.F*(e(function(){u.sort(void 0)})||!e(function(){u.sort(null)})||!require(\"./_strict-method\")(o)),\"Array\",{sort:function(r){return void 0===r?o.call(i(this)):o.call(i(this),t(r))}});\n},{\"./_export\":\"Vobs\",\"./_a-function\":\"QKlW\",\"./_to-object\":\"XMZs\",\"./_fails\":\"BI7s\",\"./_strict-method\":\"TiCE\"}],\"TVdo\":[function(require,module,exports) {\nvar r=require(\"./_is-object\"),e=require(\"./_is-array\"),o=require(\"./_wks\")(\"species\");module.exports=function(i){var t;return e(i)&&(\"function\"!=typeof(t=i.constructor)||t!==Array&&!e(t.prototype)||(t=void 0),r(t)&&null===(t=t[o])&&(t=void 0)),void 0===t?Array:t};\n},{\"./_is-object\":\"tZ11\",\"./_is-array\":\"JI5q\",\"./_wks\":\"I5XL\"}],\"M6RC\":[function(require,module,exports) {\nvar r=require(\"./_array-species-constructor\");module.exports=function(e,n){return new(r(e))(n)};\n},{\"./_array-species-constructor\":\"TVdo\"}],\"tMyS\":[function(require,module,exports) {\nvar e=require(\"./_ctx\"),r=require(\"./_iobject\"),t=require(\"./_to-object\"),i=require(\"./_to-length\"),u=require(\"./_array-species-create\");module.exports=function(n,c){var s=1==n,a=2==n,o=3==n,f=4==n,l=6==n,q=5==n||l,_=c||u;return function(u,c,h){for(var v,p,b=t(u),d=r(b),g=e(c,h,3),j=i(d.length),x=0,m=s?_(u,j):a?_(u,0):void 0;j>x;x++)if((q||x in d)&&(p=g(v=d[x],x,b),n))if(s)m[x]=p;else if(p)switch(n){case 3:return!0;case 5:return v;case 6:return x;case 2:m.push(v)}else if(f)return!1;return l?-1:o||f?f:m}};\n},{\"./_ctx\":\"W8bf\",\"./_iobject\":\"sUp0\",\"./_to-object\":\"XMZs\",\"./_to-length\":\"KLzx\",\"./_array-species-create\":\"M6RC\"}],\"vDWP\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_array-methods\")(0),t=require(\"./_strict-method\")([].forEach,!0);r(r.P+r.F*!t,\"Array\",{forEach:function(r){return e(this,r,arguments[1])}});\n},{\"./_export\":\"Vobs\",\"./_array-methods\":\"tMyS\",\"./_strict-method\":\"TiCE\"}],\"O0lf\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_array-methods\")(1);r(r.P+r.F*!require(\"./_strict-method\")([].map,!0),\"Array\",{map:function(r){return e(this,r,arguments[1])}});\n},{\"./_export\":\"Vobs\",\"./_array-methods\":\"tMyS\",\"./_strict-method\":\"TiCE\"}],\"PXKF\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_array-methods\")(2);r(r.P+r.F*!require(\"./_strict-method\")([].filter,!0),\"Array\",{filter:function(r){return e(this,r,arguments[1])}});\n},{\"./_export\":\"Vobs\",\"./_array-methods\":\"tMyS\",\"./_strict-method\":\"TiCE\"}],\"wD6H\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_array-methods\")(3);r(r.P+r.F*!require(\"./_strict-method\")([].some,!0),\"Array\",{some:function(r){return e(this,r,arguments[1])}});\n},{\"./_export\":\"Vobs\",\"./_array-methods\":\"tMyS\",\"./_strict-method\":\"TiCE\"}],\"n6bP\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_array-methods\")(4);r(r.P+r.F*!require(\"./_strict-method\")([].every,!0),\"Array\",{every:function(r){return e(this,r,arguments[1])}});\n},{\"./_export\":\"Vobs\",\"./_array-methods\":\"tMyS\",\"./_strict-method\":\"TiCE\"}],\"fXgB\":[function(require,module,exports) {\nvar r=require(\"./_a-function\"),e=require(\"./_to-object\"),i=require(\"./_iobject\"),o=require(\"./_to-length\");module.exports=function(t,n,u,a,f){r(n);var c=e(t),l=i(c),h=o(c.length),q=f?h-1:0,_=f?-1:1;if(u<2)for(;;){if(q in l){a=l[q],q+=_;break}if(q+=_,f?q<0:h<=q)throw TypeError(\"Reduce of empty array with no initial value\")}for(;f?q>=0:h>q;q+=_)q in l&&(a=n(a,l[q],q,c));return a};\n},{\"./_a-function\":\"QKlW\",\"./_to-object\":\"XMZs\",\"./_iobject\":\"sUp0\",\"./_to-length\":\"KLzx\"}],\"OWmJ\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_array-reduce\");r(r.P+r.F*!require(\"./_strict-method\")([].reduce,!0),\"Array\",{reduce:function(r){return e(this,r,arguments.length,arguments[1],!1)}});\n},{\"./_export\":\"Vobs\",\"./_array-reduce\":\"fXgB\",\"./_strict-method\":\"TiCE\"}],\"k5ri\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_array-reduce\");r(r.P+r.F*!require(\"./_strict-method\")([].reduceRight,!0),\"Array\",{reduceRight:function(r){return e(this,r,arguments.length,arguments[1],!0)}});\n},{\"./_export\":\"Vobs\",\"./_array-reduce\":\"fXgB\",\"./_strict-method\":\"TiCE\"}],\"HB9A\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_array-includes\")(!1),i=[].indexOf,t=!!i&&1/[1].indexOf(1,-0)<0;r(r.P+r.F*(t||!require(\"./_strict-method\")(i)),\"Array\",{indexOf:function(r){return t?i.apply(this,arguments)||0:e(this,r,arguments[1])}});\n},{\"./_export\":\"Vobs\",\"./_array-includes\":\"ntLR\",\"./_strict-method\":\"TiCE\"}],\"tgt4\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_export\"),r=require(\"./_to-iobject\"),t=require(\"./_to-integer\"),i=require(\"./_to-length\"),n=[].lastIndexOf,u=!!n&&1/[1].lastIndexOf(1,-0)<0;e(e.P+e.F*(u||!require(\"./_strict-method\")(n)),\"Array\",{lastIndexOf:function(e){if(u)return n.apply(this,arguments)||0;var a=r(this),o=i(a.length),s=o-1;for(arguments.length>1&&(s=Math.min(s,t(arguments[1]))),s<0&&(s=o+s);s>=0;s--)if(s in a&&a[s]===e)return s||0;return-1}});\n},{\"./_export\":\"Vobs\",\"./_to-iobject\":\"zakI\",\"./_to-integer\":\"ubM9\",\"./_to-length\":\"KLzx\",\"./_strict-method\":\"TiCE\"}],\"QXjR\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_to-object\"),t=require(\"./_to-absolute-index\"),i=require(\"./_to-length\");module.exports=[].copyWithin||function(r,o){var n=e(this),u=i(n.length),h=t(r,u),l=t(o,u),d=arguments.length>2?arguments[2]:void 0,s=Math.min((void 0===d?u:t(d,u))-l,u-h),a=1;for(l<h&&h<l+s&&(a=-1,l+=s-1,h+=s-1);s-- >0;)l in n?n[h]=n[l]:delete n[h],h+=a,l+=a;return n};\n},{\"./_to-object\":\"XMZs\",\"./_to-absolute-index\":\"tPLG\",\"./_to-length\":\"KLzx\"}],\"ke6T\":[function(require,module,exports) {\nvar e=require(\"./_wks\")(\"unscopables\"),r=Array.prototype;null==r[e]&&require(\"./_hide\")(r,e,{}),module.exports=function(o){r[e][o]=!0};\n},{\"./_wks\":\"I5XL\",\"./_hide\":\"nCfi\"}],\"c9DC\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.P,\"Array\",{copyWithin:require(\"./_array-copy-within\")}),require(\"./_add-to-unscopables\")(\"copyWithin\");\n},{\"./_export\":\"Vobs\",\"./_array-copy-within\":\"QXjR\",\"./_add-to-unscopables\":\"ke6T\"}],\"hOOH\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_to-object\"),t=require(\"./_to-absolute-index\"),r=require(\"./_to-length\");module.exports=function(o){for(var i=e(this),u=r(i.length),n=arguments.length,d=t(n>1?arguments[1]:void 0,u),l=n>2?arguments[2]:void 0,s=void 0===l?u:t(l,u);s>d;)i[d++]=o;return i};\n},{\"./_to-object\":\"XMZs\",\"./_to-absolute-index\":\"tPLG\",\"./_to-length\":\"KLzx\"}],\"ZBH0\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.P,\"Array\",{fill:require(\"./_array-fill\")}),require(\"./_add-to-unscopables\")(\"fill\");\n},{\"./_export\":\"Vobs\",\"./_array-fill\":\"hOOH\",\"./_add-to-unscopables\":\"ke6T\"}],\"wTIB\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_array-methods\")(5),i=\"find\",n=!0;i in[]&&Array(1)[i](function(){n=!1}),r(r.P+r.F*n,\"Array\",{find:function(r){return e(this,r,arguments.length>1?arguments[1]:void 0)}}),require(\"./_add-to-unscopables\")(i);\n},{\"./_export\":\"Vobs\",\"./_array-methods\":\"tMyS\",\"./_add-to-unscopables\":\"ke6T\"}],\"ksrS\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_array-methods\")(6),n=\"findIndex\",i=!0;n in[]&&Array(1)[n](function(){i=!1}),r(r.P+r.F*i,\"Array\",{findIndex:function(r){return e(this,r,arguments.length>1?arguments[1]:void 0)}}),require(\"./_add-to-unscopables\")(n);\n},{\"./_export\":\"Vobs\",\"./_array-methods\":\"tMyS\",\"./_add-to-unscopables\":\"ke6T\"}],\"YBdf\":[function(require,module,exports) {\n\n\"use strict\";var e=require(\"./_global\"),r=require(\"./_object-dp\"),i=require(\"./_descriptors\"),t=require(\"./_wks\")(\"species\");module.exports=function(u){var s=e[u];i&&s&&!s[t]&&r.f(s,t,{configurable:!0,get:function(){return this}})};\n},{\"./_global\":\"QiIT\",\"./_object-dp\":\"gGgn\",\"./_descriptors\":\"jVdc\",\"./_wks\":\"I5XL\"}],\"Adki\":[function(require,module,exports) {\nrequire(\"./_set-species\")(\"Array\");\n},{\"./_set-species\":\"YBdf\"}],\"PECj\":[function(require,module,exports) {\nmodule.exports=function(e,n){return{value:n,done:!!e}};\n},{}],\"ZCkT\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_add-to-unscopables\"),r=require(\"./_iter-step\"),t=require(\"./_iterators\"),i=require(\"./_to-iobject\");module.exports=require(\"./_iter-define\")(Array,\"Array\",function(e,r){this._t=i(e),this._i=0,this._k=r},function(){var e=this._t,t=this._k,i=this._i++;return!e||i>=e.length?(this._t=void 0,r(1)):r(0,\"keys\"==t?i:\"values\"==t?e[i]:[i,e[i]])},\"values\"),t.Arguments=t.Array,e(\"keys\"),e(\"values\"),e(\"entries\");\n},{\"./_add-to-unscopables\":\"ke6T\",\"./_iter-step\":\"PECj\",\"./_iterators\":\"H5RD\",\"./_to-iobject\":\"zakI\",\"./_iter-define\":\"MKcl\"}],\"BaNd\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_an-object\");module.exports=function(){var i=e(this),r=\"\";return i.global&&(r+=\"g\"),i.ignoreCase&&(r+=\"i\"),i.multiline&&(r+=\"m\"),i.unicode&&(r+=\"u\"),i.sticky&&(r+=\"y\"),r};\n},{\"./_an-object\":\"AIrJ\"}],\"lK2M\":[function(require,module,exports) {\n\nvar e=require(\"./_global\"),r=require(\"./_inherit-if-required\"),i=require(\"./_object-dp\").f,t=require(\"./_object-gopn\").f,n=require(\"./_is-regexp\"),o=require(\"./_flags\"),u=e.RegExp,c=u,s=u.prototype,f=/a/g,a=/a/g,g=new u(f)!==f;if(require(\"./_descriptors\")&&(!g||require(\"./_fails\")(function(){return a[require(\"./_wks\")(\"match\")]=!1,u(f)!=f||u(a)==a||\"/a/i\"!=u(f,\"i\")}))){u=function(e,i){var t=this instanceof u,f=n(e),a=void 0===i;return!t&&f&&e.constructor===u&&a?e:r(g?new c(f&&!a?e.source:e,i):c((f=e instanceof u)?e.source:e,f&&a?o.call(e):i),t?this:s,u)};for(var p=function(e){e in u||i(u,e,{configurable:!0,get:function(){return c[e]},set:function(r){c[e]=r}})},q=t(c),_=0;q.length>_;)p(q[_++]);s.constructor=u,u.prototype=s,require(\"./_redefine\")(e,\"RegExp\",u)}require(\"./_set-species\")(\"RegExp\");\n},{\"./_global\":\"QiIT\",\"./_inherit-if-required\":\"IxAU\",\"./_object-dp\":\"gGgn\",\"./_object-gopn\":\"HNVq\",\"./_is-regexp\":\"r5g1\",\"./_flags\":\"BaNd\",\"./_descriptors\":\"jVdc\",\"./_fails\":\"BI7s\",\"./_wks\":\"I5XL\",\"./_redefine\":\"jDrK\",\"./_set-species\":\"YBdf\"}],\"N1Dl\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_flags\"),l=RegExp.prototype.exec,t=String.prototype.replace,r=l,a=\"lastIndex\",n=function(){var e=/a/,t=/b*/g;return l.call(e,\"a\"),l.call(t,\"a\"),0!==e[a]||0!==t[a]}(),o=void 0!==/()??/.exec(\"\")[1],c=n||o;c&&(r=function(r){var c,i,g,u,p=this;return o&&(i=new RegExp(\"^\"+p.source+\"$(?!\\\\s)\",e.call(p))),n&&(c=p[a]),g=l.call(p,r),n&&g&&(p[a]=p.global?g.index+g[0].length:c),o&&g&&g.length>1&&t.call(g[0],i,function(){for(u=1;u<arguments.length-2;u++)void 0===arguments[u]&&(g[u]=void 0)}),g}),module.exports=r;\n},{\"./_flags\":\"BaNd\"}],\"f98m\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_regexp-exec\");require(\"./_export\")({target:\"RegExp\",proto:!0,forced:e!==/./.exec},{exec:e});\n},{\"./_regexp-exec\":\"N1Dl\",\"./_export\":\"Vobs\"}],\"S072\":[function(require,module,exports) {\nrequire(\"./_descriptors\")&&\"g\"!=/./g.flags&&require(\"./_object-dp\").f(RegExp.prototype,\"flags\",{configurable:!0,get:require(\"./_flags\")});\n},{\"./_descriptors\":\"jVdc\",\"./_object-dp\":\"gGgn\",\"./_flags\":\"BaNd\"}],\"jkaB\":[function(require,module,exports) {\n\n\"use strict\";require(\"./es6.regexp.flags\");var e=require(\"./_an-object\"),r=require(\"./_flags\"),i=require(\"./_descriptors\"),n=\"toString\",t=/./[n],a=function(e){require(\"./_redefine\")(RegExp.prototype,n,e,!0)};require(\"./_fails\")(function(){return\"/a/b\"!=t.call({source:\"a\",flags:\"b\"})})?a(function(){var n=e(this);return\"/\".concat(n.source,\"/\",\"flags\"in n?n.flags:!i&&n instanceof RegExp?r.call(n):void 0)}):t.name!=n&&a(function(){return t.call(this)});\n},{\"./es6.regexp.flags\":\"S072\",\"./_an-object\":\"AIrJ\",\"./_flags\":\"BaNd\",\"./_descriptors\":\"jVdc\",\"./_redefine\":\"jDrK\",\"./_fails\":\"BI7s\"}],\"Js7k\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_string-at\")(!0);module.exports=function(t,e,n){return e+(n?r(t,e).length:1)};\n},{\"./_string-at\":\"j93N\"}],\"DcMJ\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_classof\"),r=RegExp.prototype.exec;module.exports=function(t,o){var c=t.exec;if(\"function\"==typeof c){var n=c.call(t,o);if(\"object\"!=typeof n)throw new TypeError(\"RegExp exec method returned something other than an Object or null\");return n}if(\"RegExp\"!==e(t))throw new TypeError(\"RegExp#exec called on incompatible receiver\");return r.call(t,o)};\n},{\"./_classof\":\"pLtw\"}],\"SCKl\":[function(require,module,exports) {\n\"use strict\";require(\"./es6.regexp.exec\");var e=require(\"./_redefine\"),r=require(\"./_hide\"),n=require(\"./_fails\"),t=require(\"./_defined\"),u=require(\"./_wks\"),i=require(\"./_regexp-exec\"),c=u(\"species\"),o=!n(function(){var e=/./;return e.exec=function(){var e=[];return e.groups={a:\"7\"},e},\"7\"!==\"\".replace(e,\"$<a>\")}),a=function(){var e=/(?:)/,r=e.exec;e.exec=function(){return r.apply(this,arguments)};var n=\"ab\".split(e);return 2===n.length&&\"a\"===n[0]&&\"b\"===n[1]}();module.exports=function(l,f,p){var s=u(l),v=!n(function(){var e={};return e[s]=function(){return 7},7!=\"\"[l](e)}),x=v?!n(function(){var e=!1,r=/a/;return r.exec=function(){return e=!0,null},\"split\"===l&&(r.constructor={},r.constructor[c]=function(){return r}),r[s](\"\"),!e}):void 0;if(!v||!x||\"replace\"===l&&!o||\"split\"===l&&!a){var d=/./[s],q=p(t,s,\"\"[l],function(e,r,n,t,u){return r.exec===i?v&&!u?{done:!0,value:d.call(r,n,t)}:{done:!0,value:e.call(n,r,t)}:{done:!1}}),g=q[0],_=q[1];e(String.prototype,l,g),r(RegExp.prototype,s,2==f?function(e,r){return _.call(e,this,r)}:function(e){return _.call(e,this)})}};\n},{\"./es6.regexp.exec\":\"f98m\",\"./_redefine\":\"jDrK\",\"./_hide\":\"nCfi\",\"./_fails\":\"BI7s\",\"./_defined\":\"V0RG\",\"./_wks\":\"I5XL\",\"./_regexp-exec\":\"N1Dl\"}],\"Iomp\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_an-object\"),e=require(\"./_to-length\"),n=require(\"./_advance-string-index\"),t=require(\"./_regexp-exec-abstract\");require(\"./_fix-re-wks\")(\"match\",1,function(i,a,u,l){return[function(r){var e=i(this),n=null==r?void 0:r[a];return void 0!==n?n.call(r,e):new RegExp(r)[a](String(e))},function(i){var a=l(u,i,this);if(a.done)return a.value;var c=r(i),o=String(this);if(!c.global)return t(c,o);var s=c.unicode;c.lastIndex=0;for(var v,d=[],g=0;null!==(v=t(c,o));){var x=String(v[0]);d[g]=x,\"\"===x&&(c.lastIndex=n(o,e(c.lastIndex),s)),g++}return 0===g?null:d}]});\n},{\"./_an-object\":\"AIrJ\",\"./_to-length\":\"KLzx\",\"./_advance-string-index\":\"Js7k\",\"./_regexp-exec-abstract\":\"DcMJ\",\"./_fix-re-wks\":\"SCKl\"}],\"weWA\":[function(require,module,exports) {\nvar global = arguments[3];\nvar r=arguments[3],e=require(\"./_an-object\"),t=require(\"./_to-object\"),n=require(\"./_to-length\"),i=require(\"./_to-integer\"),a=require(\"./_advance-string-index\"),u=require(\"./_regexp-exec-abstract\"),c=Math.max,l=Math.min,o=Math.floor,v=/\\$([$&`']|\\d\\d?|<[^>]*>)/g,s=/\\$([$&`']|\\d\\d?)/g,g=function(r){return void 0===r?r:String(r)};require(\"./_fix-re-wks\")(\"replace\",2,function(r,d,f,h){return[function(e,t){var n=r(this),i=null==e?void 0:e[d];return void 0!==i?i.call(e,n,t):f.call(String(n),e,t)},function(r,t){var o=h(f,r,this,t);if(o.done)return o.value;var v=e(r),s=String(this),d=\"function\"==typeof t;d||(t=String(t));var x=v.global;if(x){var b=v.unicode;v.lastIndex=0}for(var q=[];;){var S=u(v,s);if(null===S)break;if(q.push(S),!x)break;\"\"===String(S[0])&&(v.lastIndex=a(s,n(v.lastIndex),b))}for(var _=\"\",$=0,k=0;k<q.length;k++){S=q[k];for(var m=String(S[0]),A=c(l(i(S.index),s.length),0),I=[],M=1;M<S.length;M++)I.push(g(S[M]));var j=S.groups;if(d){var w=[m].concat(I,A,s);void 0!==j&&w.push(j);var y=String(t.apply(void 0,w))}else y=p(m,s,A,I,j,t);A>=$&&(_+=s.slice($,A)+y,$=A+m.length)}return _+s.slice($)}];function p(r,e,n,i,a,u){var c=n+r.length,l=i.length,g=s;return void 0!==a&&(a=t(a),g=v),f.call(u,g,function(t,u){var v;switch(u.charAt(0)){case\"$\":return\"$\";case\"&\":return r;case\"`\":return e.slice(0,n);case\"'\":return e.slice(c);case\"<\":v=a[u.slice(1,-1)];break;default:var s=+u;if(0===s)return t;if(s>l){var g=o(s/10);return 0===g?t:g<=l?void 0===i[g-1]?u.charAt(1):i[g-1]+u.charAt(1):t}v=i[s-1]}return void 0===v?\"\":v})}});\n},{\"./_an-object\":\"AIrJ\",\"./_to-object\":\"XMZs\",\"./_to-length\":\"KLzx\",\"./_to-integer\":\"ubM9\",\"./_advance-string-index\":\"Js7k\",\"./_regexp-exec-abstract\":\"DcMJ\",\"./_fix-re-wks\":\"SCKl\"}],\"EA9T\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_an-object\"),r=require(\"./_same-value\"),n=require(\"./_regexp-exec-abstract\");require(\"./_fix-re-wks\")(\"search\",1,function(t,i,a,u){return[function(e){var r=t(this),n=null==e?void 0:e[i];return void 0!==n?n.call(e,r):new RegExp(e)[i](String(r))},function(t){var i=u(a,t,this);if(i.done)return i.value;var s=e(t),l=String(this),c=s.lastIndex;r(c,0)||(s.lastIndex=0);var v=n(s,l);return r(s.lastIndex,c)||(s.lastIndex=c),null===v?-1:v.index}]});\n},{\"./_an-object\":\"AIrJ\",\"./_same-value\":\"wc34\",\"./_regexp-exec-abstract\":\"DcMJ\",\"./_fix-re-wks\":\"SCKl\"}],\"othv\":[function(require,module,exports) {\nvar r=require(\"./_an-object\"),e=require(\"./_a-function\"),u=require(\"./_wks\")(\"species\");module.exports=function(n,o){var i,t=r(n).constructor;return void 0===t||null==(i=r(t)[u])?o:e(i)};\n},{\"./_an-object\":\"AIrJ\",\"./_a-function\":\"QKlW\",\"./_wks\":\"I5XL\"}],\"d289\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_is-regexp\"),r=require(\"./_an-object\"),i=require(\"./_species-constructor\"),n=require(\"./_advance-string-index\"),t=require(\"./_to-length\"),u=require(\"./_regexp-exec-abstract\"),l=require(\"./_regexp-exec\"),s=require(\"./_fails\"),c=Math.min,a=[].push,o=\"split\",g=\"length\",h=\"lastIndex\",d=4294967295,f=!s(function(){RegExp(d,\"y\")});require(\"./_fix-re-wks\")(\"split\",2,function(s,v,p,x){var q;return q=\"c\"==\"abbc\"[o](/(b)*/)[1]||4!=\"test\"[o](/(?:)/,-1)[g]||2!=\"ab\"[o](/(?:ab)*/)[g]||4!=\".\"[o](/(.?)(.?)/)[g]||\".\"[o](/()()/)[g]>1||\"\"[o](/.?/)[g]?function(r,i){var n=String(this);if(void 0===r&&0===i)return[];if(!e(r))return p.call(n,r,i);for(var t,u,s,c=[],o=(r.ignoreCase?\"i\":\"\")+(r.multiline?\"m\":\"\")+(r.unicode?\"u\":\"\")+(r.sticky?\"y\":\"\"),f=0,v=void 0===i?d:i>>>0,x=new RegExp(r.source,o+\"g\");(t=l.call(x,n))&&!((u=x[h])>f&&(c.push(n.slice(f,t.index)),t[g]>1&&t.index<n[g]&&a.apply(c,t.slice(1)),s=t[0][g],f=u,c[g]>=v));)x[h]===t.index&&x[h]++;return f===n[g]?!s&&x.test(\"\")||c.push(\"\"):c.push(n.slice(f)),c[g]>v?c.slice(0,v):c}:\"0\"[o](void 0,0)[g]?function(e,r){return void 0===e&&0===r?[]:p.call(this,e,r)}:p,[function(e,r){var i=s(this),n=null==e?void 0:e[v];return void 0!==n?n.call(e,i,r):q.call(String(i),e,r)},function(e,l){var s=x(q,e,this,l,q!==p);if(s.done)return s.value;var a=r(e),o=String(this),g=i(a,RegExp),h=a.unicode,v=(a.ignoreCase?\"i\":\"\")+(a.multiline?\"m\":\"\")+(a.unicode?\"u\":\"\")+(f?\"y\":\"g\"),_=new g(f?a:\"^(?:\"+a.source+\")\",v),b=void 0===l?d:l>>>0;if(0===b)return[];if(0===o.length)return null===u(_,o)?[o]:[];for(var m=0,y=0,w=[];y<o.length;){_.lastIndex=f?y:0;var E,I=u(_,f?o:o.slice(y));if(null===I||(E=c(t(_.lastIndex+(f?0:y)),o.length))===m)y=n(o,y,h);else{if(w.push(o.slice(m,y)),w.length===b)return w;for(var R=1;R<=I.length-1;R++)if(w.push(I[R]),w.length===b)return w;y=m=E}}return w.push(o.slice(m)),w}]});\n},{\"./_is-regexp\":\"r5g1\",\"./_an-object\":\"AIrJ\",\"./_species-constructor\":\"othv\",\"./_advance-string-index\":\"Js7k\",\"./_to-length\":\"KLzx\",\"./_regexp-exec-abstract\":\"DcMJ\",\"./_regexp-exec\":\"N1Dl\",\"./_fails\":\"BI7s\",\"./_fix-re-wks\":\"SCKl\"}],\"Qz2Q\":[function(require,module,exports) {\nmodule.exports=function(o,n,r,i){if(!(o instanceof n)||void 0!==i&&i in o)throw TypeError(r+\": incorrect invocation!\");return o};\n},{}],\"L3cZ\":[function(require,module,exports) {\nvar e=require(\"./_ctx\"),r=require(\"./_iter-call\"),t=require(\"./_is-array-iter\"),i=require(\"./_an-object\"),o=require(\"./_to-length\"),n=require(\"./core.get-iterator-method\"),u={},a={},f=module.exports=function(f,l,c,q,_){var h,s,d,g,p=_?function(){return f}:n(f),v=e(c,q,l?2:1),x=0;if(\"function\"!=typeof p)throw TypeError(f+\" is not iterable!\");if(t(p)){for(h=o(f.length);h>x;x++)if((g=l?v(i(s=f[x])[0],s[1]):v(f[x]))===u||g===a)return g}else for(d=p.call(f);!(s=d.next()).done;)if((g=r(d,v,s.value,l))===u||g===a)return g};f.BREAK=u,f.RETURN=a;\n},{\"./_ctx\":\"W8bf\",\"./_iter-call\":\"RG8K\",\"./_is-array-iter\":\"TuHS\",\"./_an-object\":\"AIrJ\",\"./_to-length\":\"KLzx\",\"./core.get-iterator-method\":\"um4Z\"}],\"fNEO\":[function(require,module,exports) {\n\n\nvar e,t,n,i=require(\"./_ctx\"),o=require(\"./_invoke\"),r=require(\"./_html\"),s=require(\"./_dom-create\"),a=require(\"./_global\"),c=a.process,u=a.setImmediate,p=a.clearImmediate,f=a.MessageChannel,l=a.Dispatch,d=0,m={},h=\"onreadystatechange\",g=function(){var e=+this;if(m.hasOwnProperty(e)){var t=m[e];delete m[e],t()}},v=function(e){g.call(e.data)};u&&p||(u=function(t){for(var n=[],i=1;arguments.length>i;)n.push(arguments[i++]);return m[++d]=function(){o(\"function\"==typeof t?t:Function(t),n)},e(d),d},p=function(e){delete m[e]},\"process\"==require(\"./_cof\")(c)?e=function(e){c.nextTick(i(g,e,1))}:l&&l.now?e=function(e){l.now(i(g,e,1))}:f?(n=(t=new f).port2,t.port1.onmessage=v,e=i(n.postMessage,n,1)):a.addEventListener&&\"function\"==typeof postMessage&&!a.importScripts?(e=function(e){a.postMessage(e+\"\",\"*\")},a.addEventListener(\"message\",v,!1)):e=h in s(\"script\")?function(e){r.appendChild(s(\"script\"))[h]=function(){r.removeChild(this),g.call(e)}}:function(e){setTimeout(i(g,e,1),0)}),module.exports={set:u,clear:p};\n},{\"./_ctx\":\"W8bf\",\"./_invoke\":\"Grvq\",\"./_html\":\"HDWL\",\"./_dom-create\":\"cz6Q\",\"./_global\":\"QiIT\",\"./_cof\":\"DrRY\"}],\"m7QH\":[function(require,module,exports) {\n\n\nvar e=require(\"./_global\"),t=require(\"./_task\").set,r=e.MutationObserver||e.WebKitMutationObserver,n=e.process,o=e.Promise,a=\"process\"==require(\"./_cof\")(n);module.exports=function(){var i,c,s,v=function(){var e,t;for(a&&(e=n.domain)&&e.exit();i;){t=i.fn,i=i.next;try{t()}catch(r){throw i?s():c=void 0,r}}c=void 0,e&&e.enter()};if(a)s=function(){n.nextTick(v)};else if(!r||e.navigator&&e.navigator.standalone)if(o&&o.resolve){var u=o.resolve(void 0);s=function(){u.then(v)}}else s=function(){t.call(e,v)};else{var f=!0,l=document.createTextNode(\"\");new r(v).observe(l,{characterData:!0}),s=function(){l.data=f=!f}}return function(e){var t={fn:e,next:void 0};c&&(c.next=t),i||(i=t,s()),c=t}};\n},{\"./_global\":\"QiIT\",\"./_task\":\"fNEO\",\"./_cof\":\"DrRY\"}],\"hTzn\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_a-function\");function e(e){var o,t;this.promise=new e(function(r,e){if(void 0!==o||void 0!==t)throw TypeError(\"Bad Promise constructor\");o=r,t=e}),this.resolve=r(o),this.reject=r(t)}module.exports.f=function(r){return new e(r)};\n},{\"./_a-function\":\"QKlW\"}],\"X7pO\":[function(require,module,exports) {\nmodule.exports=function(e){try{return{e:!1,v:e()}}catch(r){return{e:!0,v:r}}};\n},{}],\"KrKR\":[function(require,module,exports) {\n\nvar e=require(\"./_global\"),r=e.navigator;module.exports=r&&r.userAgent||\"\";\n},{\"./_global\":\"QiIT\"}],\"FQFX\":[function(require,module,exports) {\nvar r=require(\"./_an-object\"),e=require(\"./_is-object\"),i=require(\"./_new-promise-capability\");module.exports=function(o,t){if(r(o),e(t)&&t.constructor===o)return t;var u=i.f(o);return(0,u.resolve)(t),u.promise};\n},{\"./_an-object\":\"AIrJ\",\"./_is-object\":\"tZ11\",\"./_new-promise-capability\":\"hTzn\"}],\"lGTj\":[function(require,module,exports) {\nvar r=require(\"./_redefine\");module.exports=function(e,n,i){for(var o in n)r(e,o,n[o],i);return e};\n},{\"./_redefine\":\"jDrK\"}],\"MWl4\":[function(require,module,exports) {\n\n\n\"use strict\";var e,r,t,i,n=require(\"./_library\"),o=require(\"./_global\"),c=require(\"./_ctx\"),s=require(\"./_classof\"),u=require(\"./_export\"),a=require(\"./_is-object\"),_=require(\"./_a-function\"),h=require(\"./_an-instance\"),f=require(\"./_for-of\"),l=require(\"./_species-constructor\"),v=require(\"./_task\").set,d=require(\"./_microtask\")(),p=require(\"./_new-promise-capability\"),m=require(\"./_perform\"),q=require(\"./_user-agent\"),y=require(\"./_promise-resolve\"),j=\"Promise\",w=o.TypeError,g=o.process,x=g&&g.versions,b=x&&x.v8||\"\",k=o[j],P=\"process\"==s(g),F=function(){},S=r=p.f,E=!!function(){try{var e=k.resolve(1),r=(e.constructor={})[require(\"./_wks\")(\"species\")]=function(e){e(F,F)};return(P||\"function\"==typeof PromiseRejectionEvent)&&e.then(F)instanceof r&&0!==b.indexOf(\"6.6\")&&-1===q.indexOf(\"Chrome/66\")}catch(t){}}(),O=function(e){var r;return!(!a(e)||\"function\"!=typeof(r=e.then))&&r},R=function(e,r){if(!e._n){e._n=!0;var t=e._c;d(function(){for(var i=e._v,n=1==e._s,o=0,c=function(r){var t,o,c,s=n?r.ok:r.fail,u=r.resolve,a=r.reject,_=r.domain;try{s?(n||(2==e._h&&H(e),e._h=1),!0===s?t=i:(_&&_.enter(),t=s(i),_&&(_.exit(),c=!0)),t===r.promise?a(w(\"Promise-chain cycle\")):(o=O(t))?o.call(t,u,a):u(t)):a(i)}catch(h){_&&!c&&_.exit(),a(h)}};t.length>o;)c(t[o++]);e._c=[],e._n=!1,r&&!e._h&&C(e)})}},C=function(e){v.call(o,function(){var r,t,i,n=e._v,c=G(e);if(c&&(r=m(function(){P?g.emit(\"unhandledRejection\",n,e):(t=o.onunhandledrejection)?t({promise:e,reason:n}):(i=o.console)&&i.error&&i.error(\"Unhandled promise rejection\",n)}),e._h=P||G(e)?2:1),e._a=void 0,c&&r.e)throw r.v})},G=function(e){return 1!==e._h&&0===(e._a||e._c).length},H=function(e){v.call(o,function(){var r;P?g.emit(\"rejectionHandled\",e):(r=o.onrejectionhandled)&&r({promise:e,reason:e._v})})},T=function(e){var r=this;r._d||(r._d=!0,(r=r._w||r)._v=e,r._s=2,r._a||(r._a=r._c.slice()),R(r,!0))},U=function(e){var r,t=this;if(!t._d){t._d=!0,t=t._w||t;try{if(t===e)throw w(\"Promise can't be resolved itself\");(r=O(e))?d(function(){var i={_w:t,_d:!1};try{r.call(e,c(U,i,1),c(T,i,1))}catch(n){T.call(i,n)}}):(t._v=e,t._s=1,R(t,!1))}catch(i){T.call({_w:t,_d:!1},i)}}};E||(k=function(r){h(this,k,j,\"_h\"),_(r),e.call(this);try{r(c(U,this,1),c(T,this,1))}catch(t){T.call(this,t)}},(e=function(e){this._c=[],this._a=void 0,this._s=0,this._d=!1,this._v=void 0,this._h=0,this._n=!1}).prototype=require(\"./_redefine-all\")(k.prototype,{then:function(e,r){var t=S(l(this,k));return t.ok=\"function\"!=typeof e||e,t.fail=\"function\"==typeof r&&r,t.domain=P?g.domain:void 0,this._c.push(t),this._a&&this._a.push(t),this._s&&R(this,!1),t.promise},catch:function(e){return this.then(void 0,e)}}),t=function(){var r=new e;this.promise=r,this.resolve=c(U,r,1),this.reject=c(T,r,1)},p.f=S=function(e){return e===k||e===i?new t(e):r(e)}),u(u.G+u.W+u.F*!E,{Promise:k}),require(\"./_set-to-string-tag\")(k,j),require(\"./_set-species\")(j),i=require(\"./_core\")[j],u(u.S+u.F*!E,j,{reject:function(e){var r=S(this);return(0,r.reject)(e),r.promise}}),u(u.S+u.F*(n||!E),j,{resolve:function(e){return y(n&&this===i?k:this,e)}}),u(u.S+u.F*!(E&&require(\"./_iter-detect\")(function(e){k.all(e).catch(F)})),j,{all:function(e){var r=this,t=S(r),i=t.resolve,n=t.reject,o=m(function(){var t=[],o=0,c=1;f(e,!1,function(e){var s=o++,u=!1;t.push(void 0),c++,r.resolve(e).then(function(e){u||(u=!0,t[s]=e,--c||i(t))},n)}),--c||i(t)});return o.e&&n(o.v),t.promise},race:function(e){var r=this,t=S(r),i=t.reject,n=m(function(){f(e,!1,function(e){r.resolve(e).then(t.resolve,i)})});return n.e&&i(n.v),t.promise}});\n},{\"./_library\":\"dG4y\",\"./_global\":\"QiIT\",\"./_ctx\":\"W8bf\",\"./_classof\":\"pLtw\",\"./_export\":\"Vobs\",\"./_is-object\":\"tZ11\",\"./_a-function\":\"QKlW\",\"./_an-instance\":\"Qz2Q\",\"./_for-of\":\"L3cZ\",\"./_species-constructor\":\"othv\",\"./_task\":\"fNEO\",\"./_microtask\":\"m7QH\",\"./_new-promise-capability\":\"hTzn\",\"./_perform\":\"X7pO\",\"./_user-agent\":\"KrKR\",\"./_promise-resolve\":\"FQFX\",\"./_wks\":\"I5XL\",\"./_redefine-all\":\"lGTj\",\"./_set-to-string-tag\":\"IBDH\",\"./_set-species\":\"YBdf\",\"./_core\":\"DcE6\",\"./_iter-detect\":\"zP7t\"}],\"yRub\":[function(require,module,exports) {\nvar r=require(\"./_is-object\");module.exports=function(e,i){if(!r(e)||e._t!==i)throw TypeError(\"Incompatible receiver, \"+i+\" required!\");return e};\n},{\"./_is-object\":\"tZ11\"}],\"I9w7\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_object-dp\").f,r=require(\"./_object-create\"),t=require(\"./_redefine-all\"),i=require(\"./_ctx\"),n=require(\"./_an-instance\"),_=require(\"./_for-of\"),o=require(\"./_iter-define\"),u=require(\"./_iter-step\"),f=require(\"./_set-species\"),s=require(\"./_descriptors\"),l=require(\"./_meta\").fastKey,c=require(\"./_validate-collection\"),v=s?\"_s\":\"size\",a=function(e,r){var t,i=l(r);if(\"F\"!==i)return e._i[i];for(t=e._f;t;t=t.n)if(t.k==r)return t};module.exports={getConstructor:function(o,u,f,l){var h=o(function(e,t){n(e,h,u,\"_i\"),e._t=u,e._i=r(null),e._f=void 0,e._l=void 0,e[v]=0,null!=t&&_(t,f,e[l],e)});return t(h.prototype,{clear:function(){for(var e=c(this,u),r=e._i,t=e._f;t;t=t.n)t.r=!0,t.p&&(t.p=t.p.n=void 0),delete r[t.i];e._f=e._l=void 0,e[v]=0},delete:function(e){var r=c(this,u),t=a(r,e);if(t){var i=t.n,n=t.p;delete r._i[t.i],t.r=!0,n&&(n.n=i),i&&(i.p=n),r._f==t&&(r._f=i),r._l==t&&(r._l=n),r[v]--}return!!t},forEach:function(e){c(this,u);for(var r,t=i(e,arguments.length>1?arguments[1]:void 0,3);r=r?r.n:this._f;)for(t(r.v,r.k,this);r&&r.r;)r=r.p},has:function(e){return!!a(c(this,u),e)}}),s&&e(h.prototype,\"size\",{get:function(){return c(this,u)[v]}}),h},def:function(e,r,t){var i,n,_=a(e,r);return _?_.v=t:(e._l=_={i:n=l(r,!0),k:r,v:t,p:i=e._l,n:void 0,r:!1},e._f||(e._f=_),i&&(i.n=_),e[v]++,\"F\"!==n&&(e._i[n]=_)),e},getEntry:a,setStrong:function(e,r,t){o(e,r,function(e,t){this._t=c(e,r),this._k=t,this._l=void 0},function(){for(var e=this._k,r=this._l;r&&r.r;)r=r.p;return this._t&&(this._l=r=r?r.n:this._t._f)?u(0,\"keys\"==e?r.k:\"values\"==e?r.v:[r.k,r.v]):(this._t=void 0,u(1))},t?\"entries\":\"values\",!t,!0),f(r)}};\n},{\"./_object-dp\":\"gGgn\",\"./_object-create\":\"EH8e\",\"./_redefine-all\":\"lGTj\",\"./_ctx\":\"W8bf\",\"./_an-instance\":\"Qz2Q\",\"./_for-of\":\"L3cZ\",\"./_iter-define\":\"MKcl\",\"./_iter-step\":\"PECj\",\"./_set-species\":\"YBdf\",\"./_descriptors\":\"jVdc\",\"./_meta\":\"nxhn\",\"./_validate-collection\":\"yRub\"}],\"J5Ss\":[function(require,module,exports) {\n\n\"use strict\";var e=require(\"./_global\"),r=require(\"./_export\"),t=require(\"./_redefine\"),n=require(\"./_redefine-all\"),i=require(\"./_meta\"),u=require(\"./_for-of\"),o=require(\"./_an-instance\"),c=require(\"./_is-object\"),a=require(\"./_fails\"),s=require(\"./_iter-detect\"),l=require(\"./_set-to-string-tag\"),f=require(\"./_inherit-if-required\");module.exports=function(d,h,q,_,p,g){var v=e[d],w=v,y=p?\"set\":\"add\",x=w&&w.prototype,E={},b=function(e){var r=x[e];t(x,e,\"delete\"==e?function(e){return!(g&&!c(e))&&r.call(this,0===e?0:e)}:\"has\"==e?function(e){return!(g&&!c(e))&&r.call(this,0===e?0:e)}:\"get\"==e?function(e){return g&&!c(e)?void 0:r.call(this,0===e?0:e)}:\"add\"==e?function(e){return r.call(this,0===e?0:e),this}:function(e,t){return r.call(this,0===e?0:e,t),this})};if(\"function\"==typeof w&&(g||x.forEach&&!a(function(){(new w).entries().next()}))){var m=new w,j=m[y](g?{}:-0,1)!=m,C=a(function(){m.has(1)}),D=s(function(e){new w(e)}),F=!g&&a(function(){for(var e=new w,r=5;r--;)e[y](r,r);return!e.has(-0)});D||((w=h(function(e,r){o(e,w,d);var t=f(new v,e,w);return null!=r&&u(r,p,t[y],t),t})).prototype=x,x.constructor=w),(C||F)&&(b(\"delete\"),b(\"has\"),p&&b(\"get\")),(F||j)&&b(y),g&&x.clear&&delete x.clear}else w=_.getConstructor(h,d,p,y),n(w.prototype,q),i.NEED=!0;return l(w,d),E[d]=w,r(r.G+r.W+r.F*(w!=v),E),g||_.setStrong(w,d,p),w};\n},{\"./_global\":\"QiIT\",\"./_export\":\"Vobs\",\"./_redefine\":\"jDrK\",\"./_redefine-all\":\"lGTj\",\"./_meta\":\"nxhn\",\"./_for-of\":\"L3cZ\",\"./_an-instance\":\"Qz2Q\",\"./_is-object\":\"tZ11\",\"./_fails\":\"BI7s\",\"./_iter-detect\":\"zP7t\",\"./_set-to-string-tag\":\"IBDH\",\"./_inherit-if-required\":\"IxAU\"}],\"ksBa\":[function(require,module,exports) {\n\"use strict\";var t=require(\"./_collection-strong\"),e=require(\"./_validate-collection\"),r=\"Map\";module.exports=require(\"./_collection\")(r,function(t){return function(){return t(this,arguments.length>0?arguments[0]:void 0)}},{get:function(n){var i=t.getEntry(e(this,r),n);return i&&i.v},set:function(n,i){return t.def(e(this,r),0===n?0:n,i)}},t,!0);\n},{\"./_collection-strong\":\"I9w7\",\"./_validate-collection\":\"yRub\",\"./_collection\":\"J5Ss\"}],\"jPMF\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_collection-strong\"),t=require(\"./_validate-collection\"),r=\"Set\";module.exports=require(\"./_collection\")(r,function(e){return function(){return e(this,arguments.length>0?arguments[0]:void 0)}},{add:function(i){return e.def(t(this,r),i=0===i?0:i,i)}},e);\n},{\"./_collection-strong\":\"I9w7\",\"./_validate-collection\":\"yRub\",\"./_collection\":\"J5Ss\"}],\"y1p1\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_redefine-all\"),t=require(\"./_meta\").getWeak,r=require(\"./_an-object\"),i=require(\"./_is-object\"),n=require(\"./_an-instance\"),u=require(\"./_for-of\"),o=require(\"./_array-methods\"),s=require(\"./_has\"),a=require(\"./_validate-collection\"),c=o(5),f=o(6),_=0,h=function(e){return e._l||(e._l=new l)},l=function(){this.a=[]},d=function(e,t){return c(e.a,function(e){return e[0]===t})};l.prototype={get:function(e){var t=d(this,e);if(t)return t[1]},has:function(e){return!!d(this,e)},set:function(e,t){var r=d(this,e);r?r[1]=t:this.a.push([e,t])},delete:function(e){var t=f(this.a,function(t){return t[0]===e});return~t&&this.a.splice(t,1),!!~t}},module.exports={getConstructor:function(r,o,c,f){var l=r(function(e,t){n(e,l,o,\"_i\"),e._t=o,e._i=_++,e._l=void 0,null!=t&&u(t,c,e[f],e)});return e(l.prototype,{delete:function(e){if(!i(e))return!1;var r=t(e);return!0===r?h(a(this,o)).delete(e):r&&s(r,this._i)&&delete r[this._i]},has:function(e){if(!i(e))return!1;var r=t(e);return!0===r?h(a(this,o)).has(e):r&&s(r,this._i)}}),l},def:function(e,i,n){var u=t(r(i),!0);return!0===u?h(e).set(i,n):u[e._i]=n,e},ufstore:h};\n},{\"./_redefine-all\":\"lGTj\",\"./_meta\":\"nxhn\",\"./_an-object\":\"AIrJ\",\"./_is-object\":\"tZ11\",\"./_an-instance\":\"Qz2Q\",\"./_for-of\":\"L3cZ\",\"./_array-methods\":\"tMyS\",\"./_has\":\"kOQz\",\"./_validate-collection\":\"yRub\"}],\"Y0Wb\":[function(require,module,exports) {\n\n\"use strict\";var e,t=require(\"./_global\"),r=require(\"./_array-methods\")(0),i=require(\"./_redefine\"),n=require(\"./_meta\"),o=require(\"./_object-assign\"),u=require(\"./_collection-weak\"),c=require(\"./_is-object\"),s=require(\"./_validate-collection\"),a=require(\"./_validate-collection\"),l=!t.ActiveXObject&&\"ActiveXObject\"in t,f=\"WeakMap\",_=n.getWeak,h=Object.isExtensible,q=u.ufstore,v=function(e){return function(){return e(this,arguments.length>0?arguments[0]:void 0)}},d={get:function(e){if(c(e)){var t=_(e);return!0===t?q(s(this,f)).get(e):t?t[this._i]:void 0}},set:function(e,t){return u.def(s(this,f),e,t)}},g=module.exports=require(\"./_collection\")(f,v,d,u,!0,!0);a&&l&&(o((e=u.getConstructor(v,f)).prototype,d),n.NEED=!0,r([\"delete\",\"has\",\"get\",\"set\"],function(t){var r=g.prototype,n=r[t];i(r,t,function(r,i){if(c(r)&&!h(r)){this._f||(this._f=new e);var o=this._f[t](r,i);return\"set\"==t?this:o}return n.call(this,r,i)})}));\n},{\"./_global\":\"QiIT\",\"./_array-methods\":\"tMyS\",\"./_redefine\":\"jDrK\",\"./_meta\":\"nxhn\",\"./_object-assign\":\"v89L\",\"./_collection-weak\":\"y1p1\",\"./_is-object\":\"tZ11\",\"./_validate-collection\":\"yRub\",\"./_collection\":\"J5Ss\"}],\"oeIc\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_collection-weak\"),t=require(\"./_validate-collection\"),i=\"WeakSet\";require(\"./_collection\")(i,function(e){return function(){return e(this,arguments.length>0?arguments[0]:void 0)}},{add:function(r){return e.def(t(this,i),r,!0)}},e,!1,!0);\n},{\"./_collection-weak\":\"y1p1\",\"./_validate-collection\":\"yRub\",\"./_collection\":\"J5Ss\"}],\"zl6z\":[function(require,module,exports) {\n\nfor(var r,a=require(\"./_global\"),t=require(\"./_hide\"),e=require(\"./_uid\"),y=e(\"typed_array\"),i=e(\"view\"),A=!(!a.ArrayBuffer||!a.DataView),o=A,p=0,l=9,n=\"Int8Array,Uint8Array,Uint8ClampedArray,Int16Array,Uint16Array,Int32Array,Uint32Array,Float32Array,Float64Array\".split(\",\");p<l;)(r=a[n[p++]])?(t(r.prototype,y,!0),t(r.prototype,i,!0)):o=!1;module.exports={ABV:A,CONSTR:o,TYPED:y,VIEW:i};\n},{\"./_global\":\"QiIT\",\"./_hide\":\"nCfi\",\"./_uid\":\"jLFM\"}],\"dyWK\":[function(require,module,exports) {\nvar r=require(\"./_to-integer\"),e=require(\"./_to-length\");module.exports=function(t){if(void 0===t)return 0;var n=r(t),o=e(n);if(n!==o)throw RangeError(\"Wrong length!\");return o};\n},{\"./_to-integer\":\"ubM9\",\"./_to-length\":\"KLzx\"}],\"hFSM\":[function(require,module,exports) {\n\n\"use strict\";var t=require(\"./_global\"),n=require(\"./_descriptors\"),r=require(\"./_library\"),e=require(\"./_typed\"),i=require(\"./_hide\"),o=require(\"./_redefine-all\"),u=require(\"./_fails\"),f=require(\"./_an-instance\"),s=require(\"./_to-integer\"),c=require(\"./_to-length\"),a=require(\"./_to-index\"),h=require(\"./_object-gopn\").f,l=require(\"./_object-dp\").f,g=require(\"./_array-fill\"),_=require(\"./_set-to-string-tag\"),q=\"ArrayBuffer\",v=\"DataView\",w=\"prototype\",I=\"Wrong length!\",b=\"Wrong index!\",y=t[q],p=t[v],d=t.Math,U=t.RangeError,N=t.Infinity,x=y,A=d.abs,F=d.pow,W=d.floor,V=d.log,j=d.LN2,B=\"buffer\",E=\"byteLength\",L=\"byteOffset\",m=n?\"_b\":B,D=n?\"_l\":E,M=n?\"_o\":L;function O(t,n,r){var e,i,o,u=new Array(r),f=8*r-n-1,s=(1<<f)-1,c=s>>1,a=23===n?F(2,-24)-F(2,-77):0,h=0,l=t<0||0===t&&1/t<0?1:0;for((t=A(t))!=t||t===N?(i=t!=t?1:0,e=s):(e=W(V(t)/j),t*(o=F(2,-e))<1&&(e--,o*=2),(t+=e+c>=1?a/o:a*F(2,1-c))*o>=2&&(e++,o/=2),e+c>=s?(i=0,e=s):e+c>=1?(i=(t*o-1)*F(2,n),e+=c):(i=t*F(2,c-1)*F(2,n),e=0));n>=8;u[h++]=255&i,i/=256,n-=8);for(e=e<<n|i,f+=n;f>0;u[h++]=255&e,e/=256,f-=8);return u[--h]|=128*l,u}function R(t,n,r){var e,i=8*r-n-1,o=(1<<i)-1,u=o>>1,f=i-7,s=r-1,c=t[s--],a=127&c;for(c>>=7;f>0;a=256*a+t[s],s--,f-=8);for(e=a&(1<<-f)-1,a>>=-f,f+=n;f>0;e=256*e+t[s],s--,f-=8);if(0===a)a=1-u;else{if(a===o)return e?NaN:c?-N:N;e+=F(2,n),a-=u}return(c?-1:1)*e*F(2,a-n)}function k(t){return t[3]<<24|t[2]<<16|t[1]<<8|t[0]}function z(t){return[255&t]}function C(t){return[255&t,t>>8&255]}function G(t){return[255&t,t>>8&255,t>>16&255,t>>24&255]}function H(t){return O(t,52,8)}function J(t){return O(t,23,4)}function K(t,n,r){l(t[w],n,{get:function(){return this[r]}})}function P(t,n,r,e){var i=a(+r);if(i+n>t[D])throw U(b);var o=t[m]._b,u=i+t[M],f=o.slice(u,u+n);return e?f:f.reverse()}function Q(t,n,r,e,i,o){var u=a(+r);if(u+n>t[D])throw U(b);for(var f=t[m]._b,s=u+t[M],c=e(+i),h=0;h<n;h++)f[s+h]=c[o?h:n-h-1]}if(e.ABV){if(!u(function(){y(1)})||!u(function(){new y(-1)})||u(function(){return new y,new y(1.5),new y(NaN),y.name!=q})){for(var S,T=(y=function(t){return f(this,y),new x(a(t))})[w]=x[w],X=h(x),Y=0;X.length>Y;)(S=X[Y++])in y||i(y,S,x[S]);r||(T.constructor=y)}var Z=new p(new y(2)),$=p[w].setInt8;Z.setInt8(0,2147483648),Z.setInt8(1,2147483649),!Z.getInt8(0)&&Z.getInt8(1)||o(p[w],{setInt8:function(t,n){$.call(this,t,n<<24>>24)},setUint8:function(t,n){$.call(this,t,n<<24>>24)}},!0)}else y=function(t){f(this,y,q);var n=a(t);this._b=g.call(new Array(n),0),this[D]=n},p=function(t,n,r){f(this,p,v),f(t,y,v);var e=t[D],i=s(n);if(i<0||i>e)throw U(\"Wrong offset!\");if(i+(r=void 0===r?e-i:c(r))>e)throw U(I);this[m]=t,this[M]=i,this[D]=r},n&&(K(y,E,\"_l\"),K(p,B,\"_b\"),K(p,E,\"_l\"),K(p,L,\"_o\")),o(p[w],{getInt8:function(t){return P(this,1,t)[0]<<24>>24},getUint8:function(t){return P(this,1,t)[0]},getInt16:function(t){var n=P(this,2,t,arguments[1]);return(n[1]<<8|n[0])<<16>>16},getUint16:function(t){var n=P(this,2,t,arguments[1]);return n[1]<<8|n[0]},getInt32:function(t){return k(P(this,4,t,arguments[1]))},getUint32:function(t){return k(P(this,4,t,arguments[1]))>>>0},getFloat32:function(t){return R(P(this,4,t,arguments[1]),23,4)},getFloat64:function(t){return R(P(this,8,t,arguments[1]),52,8)},setInt8:function(t,n){Q(this,1,t,z,n)},setUint8:function(t,n){Q(this,1,t,z,n)},setInt16:function(t,n){Q(this,2,t,C,n,arguments[2])},setUint16:function(t,n){Q(this,2,t,C,n,arguments[2])},setInt32:function(t,n){Q(this,4,t,G,n,arguments[2])},setUint32:function(t,n){Q(this,4,t,G,n,arguments[2])},setFloat32:function(t,n){Q(this,4,t,J,n,arguments[2])},setFloat64:function(t,n){Q(this,8,t,H,n,arguments[2])}});_(y,q),_(p,v),i(p[w],e.VIEW,!0),exports[q]=y,exports[v]=p;\n},{\"./_global\":\"QiIT\",\"./_descriptors\":\"jVdc\",\"./_library\":\"dG4y\",\"./_typed\":\"zl6z\",\"./_hide\":\"nCfi\",\"./_redefine-all\":\"lGTj\",\"./_fails\":\"BI7s\",\"./_an-instance\":\"Qz2Q\",\"./_to-integer\":\"ubM9\",\"./_to-length\":\"KLzx\",\"./_to-index\":\"dyWK\",\"./_object-gopn\":\"HNVq\",\"./_object-dp\":\"gGgn\",\"./_array-fill\":\"hOOH\",\"./_set-to-string-tag\":\"IBDH\"}],\"VqD6\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_export\"),r=require(\"./_typed\"),i=require(\"./_typed-buffer\"),t=require(\"./_an-object\"),u=require(\"./_to-absolute-index\"),n=require(\"./_to-length\"),s=require(\"./_is-object\"),o=require(\"./_global\").ArrayBuffer,f=require(\"./_species-constructor\"),c=i.ArrayBuffer,a=i.DataView,q=r.ABV&&o.isView,_=c.prototype.slice,l=r.VIEW,y=\"ArrayBuffer\";e(e.G+e.W+e.F*(o!==c),{ArrayBuffer:c}),e(e.S+e.F*!r.CONSTR,y,{isView:function(e){return q&&q(e)||s(e)&&l in e}}),e(e.P+e.U+e.F*require(\"./_fails\")(function(){return!new c(2).slice(1,void 0).byteLength}),y,{slice:function(e,r){if(void 0!==_&&void 0===r)return _.call(t(this),e);for(var i=t(this).byteLength,s=u(e,i),o=u(void 0===r?i:r,i),q=new(f(this,c))(n(o-s)),l=new a(this),y=new a(q),b=0;s<o;)y.setUint8(b++,l.getUint8(s++));return q}}),require(\"./_set-species\")(y);\n},{\"./_export\":\"Vobs\",\"./_typed\":\"zl6z\",\"./_typed-buffer\":\"hFSM\",\"./_an-object\":\"AIrJ\",\"./_to-absolute-index\":\"tPLG\",\"./_to-length\":\"KLzx\",\"./_is-object\":\"tZ11\",\"./_global\":\"QiIT\",\"./_species-constructor\":\"othv\",\"./_fails\":\"BI7s\",\"./_set-species\":\"YBdf\"}],\"q3b2\":[function(require,module,exports) {\nvar e=require(\"./_export\");e(e.G+e.W+e.F*!require(\"./_typed\").ABV,{DataView:require(\"./_typed-buffer\").DataView});\n},{\"./_export\":\"Vobs\",\"./_typed\":\"zl6z\",\"./_typed-buffer\":\"hFSM\"}],\"sXGm\":[function(require,module,exports) {\nvar global = arguments[3];\nvar e=arguments[3];if(require(\"./_descriptors\")){var r=require(\"./_library\"),t=(e=require(\"./_global\"),require(\"./_fails\")),n=require(\"./_export\"),i=require(\"./_typed\"),o=require(\"./_typed-buffer\"),u=require(\"./_ctx\"),c=require(\"./_an-instance\"),f=require(\"./_property-desc\"),a=require(\"./_hide\"),l=require(\"./_redefine-all\"),s=require(\"./_to-integer\"),h=require(\"./_to-length\"),d=require(\"./_to-index\"),g=require(\"./_to-absolute-index\"),_=require(\"./_to-primitive\"),v=require(\"./_has\"),p=require(\"./_classof\"),y=require(\"./_is-object\"),q=require(\"./_to-object\"),w=require(\"./_is-array-iter\"),b=require(\"./_object-create\"),S=require(\"./_object-gpo\"),E=require(\"./_object-gopn\").f,m=require(\"./core.get-iterator-method\"),x=require(\"./_uid\"),L=require(\"./_wks\"),P=require(\"./_array-methods\"),j=require(\"./_array-includes\"),T=require(\"./_species-constructor\"),F=require(\"./es6.array.iterator\"),O=require(\"./_iterators\"),A=require(\"./_iter-detect\"),R=require(\"./_set-species\"),B=require(\"./_array-fill\"),I=require(\"./_array-copy-within\"),M=require(\"./_object-dp\"),W=require(\"./_object-gopd\"),N=M.f,Y=W.f,k=e.RangeError,D=e.TypeError,V=e.Uint8Array,C=\"ArrayBuffer\",U=\"Shared\"+C,G=\"BYTES_PER_ELEMENT\",z=\"prototype\",H=Array[z],J=o.ArrayBuffer,K=o.DataView,Q=P(0),X=P(2),Z=P(3),$=P(4),ee=P(5),re=P(6),te=j(!0),ne=j(!1),ie=F.values,oe=F.keys,ue=F.entries,ce=H.lastIndexOf,fe=H.reduce,ae=H.reduceRight,le=H.join,se=H.sort,he=H.slice,de=H.toString,ge=H.toLocaleString,_e=L(\"iterator\"),ve=L(\"toStringTag\"),pe=x(\"typed_constructor\"),ye=x(\"def_constructor\"),qe=i.CONSTR,we=i.TYPED,be=i.VIEW,Se=\"Wrong length!\",Ee=P(1,function(e,r){return je(T(e,e[ye]),r)}),me=t(function(){return 1===new V(new Uint16Array([1]).buffer)[0]}),xe=!!V&&!!V[z].set&&t(function(){new V(1).set({})}),Le=function(e,r){var t=s(e);if(t<0||t%r)throw k(\"Wrong offset!\");return t},Pe=function(e){if(y(e)&&we in e)return e;throw D(e+\" is not a typed array!\")},je=function(e,r){if(!(y(e)&&pe in e))throw D(\"It is not a typed array constructor!\");return new e(r)},Te=function(e,r){return Fe(T(e,e[ye]),r)},Fe=function(e,r){for(var t=0,n=r.length,i=je(e,n);n>t;)i[t]=r[t++];return i},Oe=function(e,r,t){N(e,r,{get:function(){return this._d[t]}})},Ae=function(e){var r,t,n,i,o,c,f=q(e),a=arguments.length,l=a>1?arguments[1]:void 0,s=void 0!==l,d=m(f);if(null!=d&&!w(d)){for(c=d.call(f),n=[],r=0;!(o=c.next()).done;r++)n.push(o.value);f=n}for(s&&a>2&&(l=u(l,arguments[2],2)),r=0,t=h(f.length),i=je(this,t);t>r;r++)i[r]=s?l(f[r],r):f[r];return i},Re=function(){for(var e=0,r=arguments.length,t=je(this,r);r>e;)t[e]=arguments[e++];return t},Be=!!V&&t(function(){ge.call(new V(1))}),Ie=function(){return ge.apply(Be?he.call(Pe(this)):Pe(this),arguments)},Me={copyWithin:function(e,r){return I.call(Pe(this),e,r,arguments.length>2?arguments[2]:void 0)},every:function(e){return $(Pe(this),e,arguments.length>1?arguments[1]:void 0)},fill:function(e){return B.apply(Pe(this),arguments)},filter:function(e){return Te(this,X(Pe(this),e,arguments.length>1?arguments[1]:void 0))},find:function(e){return ee(Pe(this),e,arguments.length>1?arguments[1]:void 0)},findIndex:function(e){return re(Pe(this),e,arguments.length>1?arguments[1]:void 0)},forEach:function(e){Q(Pe(this),e,arguments.length>1?arguments[1]:void 0)},indexOf:function(e){return ne(Pe(this),e,arguments.length>1?arguments[1]:void 0)},includes:function(e){return te(Pe(this),e,arguments.length>1?arguments[1]:void 0)},join:function(e){return le.apply(Pe(this),arguments)},lastIndexOf:function(e){return ce.apply(Pe(this),arguments)},map:function(e){return Ee(Pe(this),e,arguments.length>1?arguments[1]:void 0)},reduce:function(e){return fe.apply(Pe(this),arguments)},reduceRight:function(e){return ae.apply(Pe(this),arguments)},reverse:function(){for(var e,r=Pe(this).length,t=Math.floor(r/2),n=0;n<t;)e=this[n],this[n++]=this[--r],this[r]=e;return this},some:function(e){return Z(Pe(this),e,arguments.length>1?arguments[1]:void 0)},sort:function(e){return se.call(Pe(this),e)},subarray:function(e,r){var t=Pe(this),n=t.length,i=g(e,n);return new(T(t,t[ye]))(t.buffer,t.byteOffset+i*t.BYTES_PER_ELEMENT,h((void 0===r?n:g(r,n))-i))}},We=function(e,r){return Te(this,he.call(Pe(this),e,r))},Ne=function(e){Pe(this);var r=Le(arguments[1],1),t=this.length,n=q(e),i=h(n.length),o=0;if(i+r>t)throw k(Se);for(;o<i;)this[r+o]=n[o++]},Ye={entries:function(){return ue.call(Pe(this))},keys:function(){return oe.call(Pe(this))},values:function(){return ie.call(Pe(this))}},ke=function(e,r){return y(e)&&e[we]&&\"symbol\"!=typeof r&&r in e&&String(+r)==String(r)},De=function(e,r){return ke(e,r=_(r,!0))?f(2,e[r]):Y(e,r)},Ve=function(e,r,t){return!(ke(e,r=_(r,!0))&&y(t)&&v(t,\"value\"))||v(t,\"get\")||v(t,\"set\")||t.configurable||v(t,\"writable\")&&!t.writable||v(t,\"enumerable\")&&!t.enumerable?N(e,r,t):(e[r]=t.value,e)};qe||(W.f=De,M.f=Ve),n(n.S+n.F*!qe,\"Object\",{getOwnPropertyDescriptor:De,defineProperty:Ve}),t(function(){de.call({})})&&(de=ge=function(){return le.call(this)});var Ce=l({},Me);l(Ce,Ye),a(Ce,_e,Ye.values),l(Ce,{slice:We,set:Ne,constructor:function(){},toString:de,toLocaleString:Ie}),Oe(Ce,\"buffer\",\"b\"),Oe(Ce,\"byteOffset\",\"o\"),Oe(Ce,\"byteLength\",\"l\"),Oe(Ce,\"length\",\"e\"),N(Ce,ve,{get:function(){return this[we]}}),module.exports=function(o,u,f,l){var s=o+((l=!!l)?\"Clamped\":\"\")+\"Array\",g=\"get\"+o,_=\"set\"+o,v=e[s],q=v||{},w=v&&S(v),m=!v||!i.ABV,x={},L=v&&v[z],P=function(e,r){N(e,r,{get:function(){return function(e,r){var t=e._d;return t.v[g](r*u+t.o,me)}(this,r)},set:function(e){return function(e,r,t){var n=e._d;l&&(t=(t=Math.round(t))<0?0:t>255?255:255&t),n.v[_](r*u+n.o,t,me)}(this,r,e)},enumerable:!0})};m?(v=f(function(e,r,t,n){c(e,v,s,\"_d\");var i,o,f,l,g=0,_=0;if(y(r)){if(!(r instanceof J||(l=p(r))==C||l==U))return we in r?Fe(v,r):Ae.call(v,r);i=r,_=Le(t,u);var q=r.byteLength;if(void 0===n){if(q%u)throw k(Se);if((o=q-_)<0)throw k(Se)}else if((o=h(n)*u)+_>q)throw k(Se);f=o/u}else f=d(r),i=new J(o=f*u);for(a(e,\"_d\",{b:i,o:_,l:o,e:f,v:new K(i)});g<f;)P(e,g++)}),L=v[z]=b(Ce),a(L,\"constructor\",v)):t(function(){v(1)})&&t(function(){new v(-1)})&&A(function(e){new v,new v(null),new v(1.5),new v(e)},!0)||(v=f(function(e,r,t,n){var i;return c(e,v,s),y(r)?r instanceof J||(i=p(r))==C||i==U?void 0!==n?new q(r,Le(t,u),n):void 0!==t?new q(r,Le(t,u)):new q(r):we in r?Fe(v,r):Ae.call(v,r):new q(d(r))}),Q(w!==Function.prototype?E(q).concat(E(w)):E(q),function(e){e in v||a(v,e,q[e])}),v[z]=L,r||(L.constructor=v));var j=L[_e],T=!!j&&(\"values\"==j.name||null==j.name),F=Ye.values;a(v,pe,!0),a(L,we,s),a(L,be,!0),a(L,ye,v),(l?new v(1)[ve]==s:ve in L)||N(L,ve,{get:function(){return s}}),x[s]=v,n(n.G+n.W+n.F*(v!=q),x),n(n.S,s,{BYTES_PER_ELEMENT:u}),n(n.S+n.F*t(function(){q.of.call(v,1)}),s,{from:Ae,of:Re}),G in L||a(L,G,u),n(n.P,s,Me),R(s),n(n.P+n.F*xe,s,{set:Ne}),n(n.P+n.F*!T,s,Ye),r||L.toString==de||(L.toString=de),n(n.P+n.F*t(function(){new v(1).slice()}),s,{slice:We}),n(n.P+n.F*(t(function(){return[1,2].toLocaleString()!=new v([1,2]).toLocaleString()})||!t(function(){L.toLocaleString.call([1,2])})),s,{toLocaleString:Ie}),O[s]=T?j:F,r||T||a(L,_e,F)}}else module.exports=function(){};\n},{\"./_descriptors\":\"jVdc\",\"./_library\":\"dG4y\",\"./_global\":\"QiIT\",\"./_fails\":\"BI7s\",\"./_export\":\"Vobs\",\"./_typed\":\"zl6z\",\"./_typed-buffer\":\"hFSM\",\"./_ctx\":\"W8bf\",\"./_an-instance\":\"Qz2Q\",\"./_property-desc\":\"zQQJ\",\"./_hide\":\"nCfi\",\"./_redefine-all\":\"lGTj\",\"./_to-integer\":\"ubM9\",\"./_to-length\":\"KLzx\",\"./_to-index\":\"dyWK\",\"./_to-absolute-index\":\"tPLG\",\"./_to-primitive\":\"S7GM\",\"./_has\":\"kOQz\",\"./_classof\":\"pLtw\",\"./_is-object\":\"tZ11\",\"./_to-object\":\"XMZs\",\"./_is-array-iter\":\"TuHS\",\"./_object-create\":\"EH8e\",\"./_object-gpo\":\"dlIw\",\"./_object-gopn\":\"HNVq\",\"./core.get-iterator-method\":\"um4Z\",\"./_uid\":\"jLFM\",\"./_wks\":\"I5XL\",\"./_array-methods\":\"tMyS\",\"./_array-includes\":\"ntLR\",\"./_species-constructor\":\"othv\",\"./es6.array.iterator\":\"ZCkT\",\"./_iterators\":\"H5RD\",\"./_iter-detect\":\"zP7t\",\"./_set-species\":\"YBdf\",\"./_array-fill\":\"hOOH\",\"./_array-copy-within\":\"QXjR\",\"./_object-dp\":\"gGgn\",\"./_object-gopd\":\"EGJe\"}],\"FrGE\":[function(require,module,exports) {\nrequire(\"./_typed-array\")(\"Int8\",1,function(r){return function(n,t,e){return r(this,n,t,e)}});\n},{\"./_typed-array\":\"sXGm\"}],\"jLcZ\":[function(require,module,exports) {\nrequire(\"./_typed-array\")(\"Uint8\",1,function(r){return function(n,t,e){return r(this,n,t,e)}});\n},{\"./_typed-array\":\"sXGm\"}],\"dFjM\":[function(require,module,exports) {\nrequire(\"./_typed-array\")(\"Uint8\",1,function(r){return function(n,t,e){return r(this,n,t,e)}},!0);\n},{\"./_typed-array\":\"sXGm\"}],\"XAXm\":[function(require,module,exports) {\nrequire(\"./_typed-array\")(\"Int16\",2,function(r){return function(n,t,e){return r(this,n,t,e)}});\n},{\"./_typed-array\":\"sXGm\"}],\"Vod2\":[function(require,module,exports) {\nrequire(\"./_typed-array\")(\"Uint16\",2,function(r){return function(n,t,e){return r(this,n,t,e)}});\n},{\"./_typed-array\":\"sXGm\"}],\"Mnlj\":[function(require,module,exports) {\nrequire(\"./_typed-array\")(\"Int32\",4,function(r){return function(n,t,e){return r(this,n,t,e)}});\n},{\"./_typed-array\":\"sXGm\"}],\"JJCv\":[function(require,module,exports) {\nrequire(\"./_typed-array\")(\"Uint32\",4,function(r){return function(n,t,e){return r(this,n,t,e)}});\n},{\"./_typed-array\":\"sXGm\"}],\"Asas\":[function(require,module,exports) {\nrequire(\"./_typed-array\")(\"Float32\",4,function(r){return function(t,n,e){return r(this,t,n,e)}});\n},{\"./_typed-array\":\"sXGm\"}],\"ZKGF\":[function(require,module,exports) {\nrequire(\"./_typed-array\")(\"Float64\",8,function(r){return function(t,n,e){return r(this,t,n,e)}});\n},{\"./_typed-array\":\"sXGm\"}],\"sL26\":[function(require,module,exports) {\nvar e=require(\"./_export\"),r=require(\"./_a-function\"),n=require(\"./_an-object\"),i=(require(\"./_global\").Reflect||{}).apply,u=Function.apply;e(e.S+e.F*!require(\"./_fails\")(function(){i(function(){})}),\"Reflect\",{apply:function(e,a,l){var t=r(e),c=n(l);return i?i(t,a,c):u.call(t,a,c)}});\n},{\"./_export\":\"Vobs\",\"./_a-function\":\"QKlW\",\"./_an-object\":\"AIrJ\",\"./_global\":\"QiIT\",\"./_fails\":\"BI7s\"}],\"n0sj\":[function(require,module,exports) {\nvar e=require(\"./_export\"),r=require(\"./_object-create\"),n=require(\"./_a-function\"),t=require(\"./_an-object\"),u=require(\"./_is-object\"),c=require(\"./_fails\"),i=require(\"./_bind\"),o=(require(\"./_global\").Reflect||{}).construct,a=c(function(){function e(){}return!(o(function(){},[],e)instanceof e)}),l=!c(function(){o(function(){})});e(e.S+e.F*(a||l),\"Reflect\",{construct:function(e,c){n(e),t(c);var f=arguments.length<3?e:n(arguments[2]);if(l&&!a)return o(e,c,f);if(e==f){switch(c.length){case 0:return new e;case 1:return new e(c[0]);case 2:return new e(c[0],c[1]);case 3:return new e(c[0],c[1],c[2]);case 4:return new e(c[0],c[1],c[2],c[3])}var p=[null];return p.push.apply(p,c),new(i.apply(e,p))}var s=f.prototype,q=r(u(s)?s:Object.prototype),_=Function.apply.call(e,q,c);return u(_)?_:q}});\n},{\"./_export\":\"Vobs\",\"./_object-create\":\"EH8e\",\"./_a-function\":\"QKlW\",\"./_an-object\":\"AIrJ\",\"./_is-object\":\"tZ11\",\"./_fails\":\"BI7s\",\"./_bind\":\"s1yo\",\"./_global\":\"QiIT\"}],\"XoPA\":[function(require,module,exports) {\nvar e=require(\"./_object-dp\"),r=require(\"./_export\"),t=require(\"./_an-object\"),i=require(\"./_to-primitive\");r(r.S+r.F*require(\"./_fails\")(function(){Reflect.defineProperty(e.f({},1,{value:1}),1,{value:2})}),\"Reflect\",{defineProperty:function(r,u,f){t(r),u=i(u,!0),t(f);try{return e.f(r,u,f),!0}catch(n){return!1}}});\n},{\"./_object-dp\":\"gGgn\",\"./_export\":\"Vobs\",\"./_an-object\":\"AIrJ\",\"./_to-primitive\":\"S7GM\",\"./_fails\":\"BI7s\"}],\"YgqD\":[function(require,module,exports) {\nvar e=require(\"./_export\"),r=require(\"./_object-gopd\").f,t=require(\"./_an-object\");e(e.S,\"Reflect\",{deleteProperty:function(e,o){var u=r(t(e),o);return!(u&&!u.configurable)&&delete e[o]}});\n},{\"./_export\":\"Vobs\",\"./_object-gopd\":\"EGJe\",\"./_an-object\":\"AIrJ\"}],\"CKoQ\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_export\"),t=require(\"./_an-object\"),i=function(e){this._t=t(e),this._i=0;var i,r=this._k=[];for(i in e)r.push(i)};require(\"./_iter-create\")(i,\"Object\",function(){var e,t=this._k;do{if(this._i>=t.length)return{value:void 0,done:!0}}while(!((e=t[this._i++])in this._t));return{value:e,done:!1}}),e(e.S,\"Reflect\",{enumerate:function(e){return new i(e)}});\n},{\"./_export\":\"Vobs\",\"./_an-object\":\"AIrJ\",\"./_iter-create\":\"gj4O\"}],\"Jr0s\":[function(require,module,exports) {\nvar e=require(\"./_object-gopd\"),r=require(\"./_object-gpo\"),t=require(\"./_has\"),i=require(\"./_export\"),o=require(\"./_is-object\"),u=require(\"./_an-object\");function a(i,c){var v,g,l=arguments.length<3?i:arguments[2];return u(i)===l?i[c]:(v=e.f(i,c))?t(v,\"value\")?v.value:void 0!==v.get?v.get.call(l):void 0:o(g=r(i))?a(g,c,l):void 0}i(i.S,\"Reflect\",{get:a});\n},{\"./_object-gopd\":\"EGJe\",\"./_object-gpo\":\"dlIw\",\"./_has\":\"kOQz\",\"./_export\":\"Vobs\",\"./_is-object\":\"tZ11\",\"./_an-object\":\"AIrJ\"}],\"rsHl\":[function(require,module,exports) {\nvar e=require(\"./_object-gopd\"),r=require(\"./_export\"),t=require(\"./_an-object\");r(r.S,\"Reflect\",{getOwnPropertyDescriptor:function(r,o){return e.f(t(r),o)}});\n},{\"./_object-gopd\":\"EGJe\",\"./_export\":\"Vobs\",\"./_an-object\":\"AIrJ\"}],\"mTTK\":[function(require,module,exports) {\nvar e=require(\"./_export\"),r=require(\"./_object-gpo\"),t=require(\"./_an-object\");e(e.S,\"Reflect\",{getPrototypeOf:function(e){return r(t(e))}});\n},{\"./_export\":\"Vobs\",\"./_object-gpo\":\"dlIw\",\"./_an-object\":\"AIrJ\"}],\"VxVc\":[function(require,module,exports) {\nvar e=require(\"./_export\");e(e.S,\"Reflect\",{has:function(e,r){return r in e}});\n},{\"./_export\":\"Vobs\"}],\"lQ3X\":[function(require,module,exports) {\nvar e=require(\"./_export\"),r=require(\"./_an-object\"),t=Object.isExtensible;e(e.S,\"Reflect\",{isExtensible:function(e){return r(e),!t||t(e)}});\n},{\"./_export\":\"Vobs\",\"./_an-object\":\"AIrJ\"}],\"yE4E\":[function(require,module,exports) {\nvar e=require(\"./_object-gopn\"),r=require(\"./_object-gops\"),o=require(\"./_an-object\"),t=require(\"./_global\").Reflect;module.exports=t&&t.ownKeys||function(t){var c=e.f(o(t)),n=r.f;return n?c.concat(n(t)):c};\n},{\"./_object-gopn\":\"HNVq\",\"./_object-gops\":\"vSss\",\"./_an-object\":\"AIrJ\",\"./_global\":\"QiIT\"}],\"vOF6\":[function(require,module,exports) {\nvar e=require(\"./_export\");e(e.S,\"Reflect\",{ownKeys:require(\"./_own-keys\")});\n},{\"./_export\":\"Vobs\",\"./_own-keys\":\"yE4E\"}],\"hWQ0\":[function(require,module,exports) {\nvar e=require(\"./_export\"),r=require(\"./_an-object\"),t=Object.preventExtensions;e(e.S,\"Reflect\",{preventExtensions:function(e){r(e);try{return t&&t(e),!0}catch(n){return!1}}});\n},{\"./_export\":\"Vobs\",\"./_an-object\":\"AIrJ\"}],\"AiN1\":[function(require,module,exports) {\nvar e=require(\"./_object-dp\"),r=require(\"./_object-gopd\"),t=require(\"./_object-gpo\"),i=require(\"./_has\"),u=require(\"./_export\"),f=require(\"./_property-desc\"),o=require(\"./_an-object\"),a=require(\"./_is-object\");function c(u,l,n){var q,s,_=arguments.length<4?u:arguments[3],b=r.f(o(u),l);if(!b){if(a(s=t(u)))return c(s,l,n,_);b=f(0)}if(i(b,\"value\")){if(!1===b.writable||!a(_))return!1;if(q=r.f(_,l)){if(q.get||q.set||!1===q.writable)return!1;q.value=n,e.f(_,l,q)}else e.f(_,l,f(0,n));return!0}return void 0!==b.set&&(b.set.call(_,n),!0)}u(u.S,\"Reflect\",{set:c});\n},{\"./_object-dp\":\"gGgn\",\"./_object-gopd\":\"EGJe\",\"./_object-gpo\":\"dlIw\",\"./_has\":\"kOQz\",\"./_export\":\"Vobs\",\"./_property-desc\":\"zQQJ\",\"./_an-object\":\"AIrJ\",\"./_is-object\":\"tZ11\"}],\"EPEE\":[function(require,module,exports) {\nvar e=require(\"./_export\"),r=require(\"./_set-proto\");r&&e(e.S,\"Reflect\",{setPrototypeOf:function(e,t){r.check(e,t);try{return r.set(e,t),!0}catch(c){return!1}}});\n},{\"./_export\":\"Vobs\",\"./_set-proto\":\"IC1x\"}],\"gMo0\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_array-includes\")(!0);r(r.P,\"Array\",{includes:function(r){return e(this,r,arguments.length>1?arguments[1]:void 0)}}),require(\"./_add-to-unscopables\")(\"includes\");\n},{\"./_export\":\"Vobs\",\"./_array-includes\":\"ntLR\",\"./_add-to-unscopables\":\"ke6T\"}],\"M1I7\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_is-array\"),e=require(\"./_is-object\"),i=require(\"./_to-length\"),t=require(\"./_ctx\"),o=require(\"./_wks\")(\"isConcatSpreadable\");function u(s,a,n,c,f,l,q,_){for(var d,h,p=f,v=0,b=!!q&&t(q,_,3);v<c;){if(v in n){if(d=b?b(n[v],v,a):n[v],h=!1,e(d)&&(h=void 0!==(h=d[o])?!!h:r(d)),h&&l>0)p=u(s,a,d,i(d.length),p,l-1)-1;else{if(p>=9007199254740991)throw TypeError();s[p]=d}p++}v++}return p}module.exports=u;\n},{\"./_is-array\":\"JI5q\",\"./_is-object\":\"tZ11\",\"./_to-length\":\"KLzx\",\"./_ctx\":\"W8bf\",\"./_wks\":\"I5XL\"}],\"zKV8\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_flatten-into-array\"),t=require(\"./_to-object\"),a=require(\"./_to-length\"),i=require(\"./_a-function\"),u=require(\"./_array-species-create\");r(r.P,\"Array\",{flatMap:function(r){var n,o,c=t(this);return i(r),n=a(c.length),o=u(c,0),e(o,c,c,n,0,1,r,arguments[1]),o}}),require(\"./_add-to-unscopables\")(\"flatMap\");\n},{\"./_export\":\"Vobs\",\"./_flatten-into-array\":\"M1I7\",\"./_to-object\":\"XMZs\",\"./_to-length\":\"KLzx\",\"./_a-function\":\"QKlW\",\"./_array-species-create\":\"M6RC\",\"./_add-to-unscopables\":\"ke6T\"}],\"GDMJ\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_export\"),r=require(\"./_flatten-into-array\"),t=require(\"./_to-object\"),i=require(\"./_to-length\"),a=require(\"./_to-integer\"),n=require(\"./_array-species-create\");e(e.P,\"Array\",{flatten:function(){var e=arguments[0],u=t(this),o=i(u.length),q=n(u,0);return r(q,u,u,o,0,void 0===e?1:a(e)),q}}),require(\"./_add-to-unscopables\")(\"flatten\");\n},{\"./_export\":\"Vobs\",\"./_flatten-into-array\":\"M1I7\",\"./_to-object\":\"XMZs\",\"./_to-length\":\"KLzx\",\"./_to-integer\":\"ubM9\",\"./_array-species-create\":\"M6RC\",\"./_add-to-unscopables\":\"ke6T\"}],\"K4uP\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),t=require(\"./_string-at\")(!0);r(r.P,\"String\",{at:function(r){return t(this,r)}});\n},{\"./_export\":\"Vobs\",\"./_string-at\":\"j93N\"}],\"m0x4\":[function(require,module,exports) {\nvar e=require(\"./_to-length\"),r=require(\"./_string-repeat\"),t=require(\"./_defined\");module.exports=function(i,n,l,g){var u=String(t(i)),a=u.length,h=void 0===l?\" \":String(l),o=e(n);if(o<=a||\"\"==h)return u;var c=o-a,d=r.call(h,Math.ceil(c/h.length));return d.length>c&&(d=d.slice(0,c)),g?d+u:u+d};\n},{\"./_to-length\":\"KLzx\",\"./_string-repeat\":\"Lz3r\",\"./_defined\":\"V0RG\"}],\"hmYY\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_string-pad\"),t=require(\"./_user-agent\"),i=/Version\\/10\\.\\d+(\\.\\d+)?( Mobile\\/\\w+)? Safari\\//.test(t);r(r.P+r.F*i,\"String\",{padStart:function(r){return e(this,r,arguments.length>1?arguments[1]:void 0,!0)}});\n},{\"./_export\":\"Vobs\",\"./_string-pad\":\"m0x4\",\"./_user-agent\":\"KrKR\"}],\"RIKd\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_string-pad\"),i=require(\"./_user-agent\"),t=/Version\\/10\\.\\d+(\\.\\d+)?( Mobile\\/\\w+)? Safari\\//.test(i);r(r.P+r.F*t,\"String\",{padEnd:function(r){return e(this,r,arguments.length>1?arguments[1]:void 0,!1)}});\n},{\"./_export\":\"Vobs\",\"./_string-pad\":\"m0x4\",\"./_user-agent\":\"KrKR\"}],\"hNft\":[function(require,module,exports) {\n\"use strict\";require(\"./_string-trim\")(\"trimLeft\",function(t){return function(){return t(this,1)}},\"trimStart\");\n},{\"./_string-trim\":\"JIX2\"}],\"uLyC\":[function(require,module,exports) {\n\"use strict\";require(\"./_string-trim\")(\"trimRight\",function(t){return function(){return t(this,2)}},\"trimEnd\");\n},{\"./_string-trim\":\"JIX2\"}],\"dRqM\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_export\"),r=require(\"./_defined\"),t=require(\"./_to-length\"),i=require(\"./_is-regexp\"),n=require(\"./_flags\"),s=RegExp.prototype,g=function(e,r){this._r=e,this._s=r};require(\"./_iter-create\")(g,\"RegExp String\",function(){var e=this._r.exec(this._s);return{value:e,done:null===e}}),e(e.P,\"String\",{matchAll:function(e){if(r(this),!i(e))throw TypeError(e+\" is not a regexp!\");var u=String(this),a=\"flags\"in s?String(e.flags):n.call(e),l=new RegExp(e.source,~a.indexOf(\"g\")?a:\"g\"+a);return l.lastIndex=t(e.lastIndex),new g(l,u)}});\n},{\"./_export\":\"Vobs\",\"./_defined\":\"V0RG\",\"./_to-length\":\"KLzx\",\"./_is-regexp\":\"r5g1\",\"./_flags\":\"BaNd\",\"./_iter-create\":\"gj4O\"}],\"enid\":[function(require,module,exports) {\nrequire(\"./_wks-define\")(\"asyncIterator\");\n},{\"./_wks-define\":\"ZenZ\"}],\"Oxke\":[function(require,module,exports) {\nrequire(\"./_wks-define\")(\"observable\");\n},{\"./_wks-define\":\"ZenZ\"}],\"ovdg\":[function(require,module,exports) {\nvar e=require(\"./_export\"),r=require(\"./_own-keys\"),t=require(\"./_to-iobject\"),o=require(\"./_object-gopd\"),i=require(\"./_create-property\");e(e.S,\"Object\",{getOwnPropertyDescriptors:function(e){for(var u,c,n=t(e),p=o.f,q=r(n),_={},a=0;q.length>a;)void 0!==(c=p(n,u=q[a++]))&&i(_,u,c);return _}});\n},{\"./_export\":\"Vobs\",\"./_own-keys\":\"yE4E\",\"./_to-iobject\":\"zakI\",\"./_object-gopd\":\"EGJe\",\"./_create-property\":\"g07e\"}],\"HVWH\":[function(require,module,exports) {\nvar e=require(\"./_descriptors\"),r=require(\"./_object-keys\"),t=require(\"./_to-iobject\"),o=require(\"./_object-pie\").f;module.exports=function(u){return function(i){for(var c,n=t(i),s=r(n),f=s.length,l=0,p=[];f>l;)c=s[l++],e&&!o.call(n,c)||p.push(u?[c,n[c]]:n[c]);return p}};\n},{\"./_descriptors\":\"jVdc\",\"./_object-keys\":\"huXi\",\"./_to-iobject\":\"zakI\",\"./_object-pie\":\"NRj4\"}],\"exYH\":[function(require,module,exports) {\nvar r=require(\"./_export\"),e=require(\"./_object-to-array\")(!1);r(r.S,\"Object\",{values:function(r){return e(r)}});\n},{\"./_export\":\"Vobs\",\"./_object-to-array\":\"HVWH\"}],\"jLAB\":[function(require,module,exports) {\nvar r=require(\"./_export\"),e=require(\"./_object-to-array\")(!0);r(r.S,\"Object\",{entries:function(r){return e(r)}});\n},{\"./_export\":\"Vobs\",\"./_object-to-array\":\"HVWH\"}],\"Se8n\":[function(require,module,exports) {\n\"use strict\";module.exports=require(\"./_library\")||!require(\"./_fails\")(function(){var e=Math.random();__defineSetter__.call(null,e,function(){}),delete require(\"./_global\")[e]});\n},{\"./_library\":\"dG4y\",\"./_fails\":\"BI7s\",\"./_global\":\"QiIT\"}],\"y7i0\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_export\"),r=require(\"./_to-object\"),t=require(\"./_a-function\"),i=require(\"./_object-dp\");require(\"./_descriptors\")&&e(e.P+require(\"./_object-forced-pam\"),\"Object\",{__defineGetter__:function(e,u){i.f(r(this),e,{get:t(u),enumerable:!0,configurable:!0})}});\n},{\"./_export\":\"Vobs\",\"./_to-object\":\"XMZs\",\"./_a-function\":\"QKlW\",\"./_object-dp\":\"gGgn\",\"./_descriptors\":\"jVdc\",\"./_object-forced-pam\":\"Se8n\"}],\"vFGQ\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_export\"),r=require(\"./_to-object\"),t=require(\"./_a-function\"),i=require(\"./_object-dp\");require(\"./_descriptors\")&&e(e.P+require(\"./_object-forced-pam\"),\"Object\",{__defineSetter__:function(e,u){i.f(r(this),e,{set:t(u),enumerable:!0,configurable:!0})}});\n},{\"./_export\":\"Vobs\",\"./_to-object\":\"XMZs\",\"./_a-function\":\"QKlW\",\"./_object-dp\":\"gGgn\",\"./_descriptors\":\"jVdc\",\"./_object-forced-pam\":\"Se8n\"}],\"urEd\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_export\"),r=require(\"./_to-object\"),t=require(\"./_to-primitive\"),i=require(\"./_object-gpo\"),o=require(\"./_object-gopd\").f;require(\"./_descriptors\")&&e(e.P+require(\"./_object-forced-pam\"),\"Object\",{__lookupGetter__:function(e){var u,_=r(this),c=t(e,!0);do{if(u=o(_,c))return u.get}while(_=i(_))}});\n},{\"./_export\":\"Vobs\",\"./_to-object\":\"XMZs\",\"./_to-primitive\":\"S7GM\",\"./_object-gpo\":\"dlIw\",\"./_object-gopd\":\"EGJe\",\"./_descriptors\":\"jVdc\",\"./_object-forced-pam\":\"Se8n\"}],\"qicQ\":[function(require,module,exports) {\n\"use strict\";var e=require(\"./_export\"),r=require(\"./_to-object\"),t=require(\"./_to-primitive\"),i=require(\"./_object-gpo\"),o=require(\"./_object-gopd\").f;require(\"./_descriptors\")&&e(e.P+require(\"./_object-forced-pam\"),\"Object\",{__lookupSetter__:function(e){var u,_=r(this),c=t(e,!0);do{if(u=o(_,c))return u.set}while(_=i(_))}});\n},{\"./_export\":\"Vobs\",\"./_to-object\":\"XMZs\",\"./_to-primitive\":\"S7GM\",\"./_object-gpo\":\"dlIw\",\"./_object-gopd\":\"EGJe\",\"./_descriptors\":\"jVdc\",\"./_object-forced-pam\":\"Se8n\"}],\"VUTp\":[function(require,module,exports) {\nvar r=require(\"./_for-of\");module.exports=function(e,o){var u=[];return r(e,!1,u.push,u,o),u};\n},{\"./_for-of\":\"L3cZ\"}],\"NEML\":[function(require,module,exports) {\nvar r=require(\"./_classof\"),e=require(\"./_array-from-iterable\");module.exports=function(t){return function(){if(r(this)!=t)throw TypeError(t+\"#toJSON isn't generic\");return e(this)}};\n},{\"./_classof\":\"pLtw\",\"./_array-from-iterable\":\"VUTp\"}],\"gCox\":[function(require,module,exports) {\nvar e=require(\"./_export\");e(e.P+e.R,\"Map\",{toJSON:require(\"./_collection-to-json\")(\"Map\")});\n},{\"./_export\":\"Vobs\",\"./_collection-to-json\":\"NEML\"}],\"CwpA\":[function(require,module,exports) {\nvar e=require(\"./_export\");e(e.P+e.R,\"Set\",{toJSON:require(\"./_collection-to-json\")(\"Set\")});\n},{\"./_export\":\"Vobs\",\"./_collection-to-json\":\"NEML\"}],\"rIFj\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\");module.exports=function(e){r(r.S,e,{of:function(){for(var r=arguments.length,e=new Array(r);r--;)e[r]=arguments[r];return new this(e)}})};\n},{\"./_export\":\"Vobs\"}],\"bPOJ\":[function(require,module,exports) {\nrequire(\"./_set-collection-of\")(\"Map\");\n},{\"./_set-collection-of\":\"rIFj\"}],\"swmI\":[function(require,module,exports) {\nrequire(\"./_set-collection-of\")(\"Set\");\n},{\"./_set-collection-of\":\"rIFj\"}],\"Kb3C\":[function(require,module,exports) {\nrequire(\"./_set-collection-of\")(\"WeakMap\");\n},{\"./_set-collection-of\":\"rIFj\"}],\"HgXJ\":[function(require,module,exports) {\nrequire(\"./_set-collection-of\")(\"WeakSet\");\n},{\"./_set-collection-of\":\"rIFj\"}],\"sb9z\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_a-function\"),u=require(\"./_ctx\"),i=require(\"./_for-of\");module.exports=function(t){r(r.S,t,{from:function(r){var t,n,o,s,f=arguments[1];return e(this),(t=void 0!==f)&&e(f),null==r?new this:(n=[],t?(o=0,s=u(f,arguments[2],2),i(r,!1,function(r){n.push(s(r,o++))})):i(r,!1,n.push,n),new this(n))}})};\n},{\"./_export\":\"Vobs\",\"./_a-function\":\"QKlW\",\"./_ctx\":\"W8bf\",\"./_for-of\":\"L3cZ\"}],\"mnJw\":[function(require,module,exports) {\nrequire(\"./_set-collection-from\")(\"Map\");\n},{\"./_set-collection-from\":\"sb9z\"}],\"Wc9c\":[function(require,module,exports) {\nrequire(\"./_set-collection-from\")(\"Set\");\n},{\"./_set-collection-from\":\"sb9z\"}],\"RABC\":[function(require,module,exports) {\nrequire(\"./_set-collection-from\")(\"WeakMap\");\n},{\"./_set-collection-from\":\"sb9z\"}],\"irWo\":[function(require,module,exports) {\nrequire(\"./_set-collection-from\")(\"WeakSet\");\n},{\"./_set-collection-from\":\"sb9z\"}],\"DjhA\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.G,{global:require(\"./_global\")});\n},{\"./_export\":\"Vobs\",\"./_global\":\"QiIT\"}],\"zQTI\":[function(require,module,exports) {\nvar e=require(\"./_export\");e(e.S,\"System\",{global:require(\"./_global\")});\n},{\"./_export\":\"Vobs\",\"./_global\":\"QiIT\"}],\"sx2w\":[function(require,module,exports) {\nvar r=require(\"./_export\"),e=require(\"./_cof\");r(r.S,\"Error\",{isError:function(r){return\"Error\"===e(r)}});\n},{\"./_export\":\"Vobs\",\"./_cof\":\"DrRY\"}],\"duUS\":[function(require,module,exports) {\nvar a=require(\"./_export\");a(a.S,\"Math\",{clamp:function(a,r,t){return Math.min(t,Math.max(r,a))}});\n},{\"./_export\":\"Vobs\"}],\"Nayo\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Math\",{DEG_PER_RAD:Math.PI/180});\n},{\"./_export\":\"Vobs\"}],\"pK3L\":[function(require,module,exports) {\nvar e=require(\"./_export\"),r=180/Math.PI;e(e.S,\"Math\",{degrees:function(e){return e*r}});\n},{\"./_export\":\"Vobs\"}],\"ZVag\":[function(require,module,exports) {\nmodule.exports=Math.scale||function(e,t,n,a,l){return 0===arguments.length||e!=e||t!=t||n!=n||a!=a||l!=l?NaN:e===1/0||e===-1/0?e:(e-t)*(l-a)/(n-t)+a};\n},{}],\"cNya\":[function(require,module,exports) {\nvar r=require(\"./_export\"),e=require(\"./_math-scale\"),a=require(\"./_math-fround\");r(r.S,\"Math\",{fscale:function(r,t,u,i,n){return a(e(r,t,u,i,n))}});\n},{\"./_export\":\"Vobs\",\"./_math-scale\":\"ZVag\",\"./_math-fround\":\"lqkS\"}],\"JpQg\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Math\",{iaddh:function(r,a,e,t){var i=r>>>0,n=e>>>0;return(a>>>0)+(t>>>0)+((i&n|(i|n)&~(i+n>>>0))>>>31)|0}});\n},{\"./_export\":\"Vobs\"}],\"kYRl\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Math\",{isubh:function(r,e,t,u){var a=r>>>0,i=t>>>0;return(e>>>0)-(u>>>0)-((~a&i|~(a^i)&a-i>>>0)>>>31)|0}});\n},{\"./_export\":\"Vobs\"}],\"iMz3\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Math\",{imulh:function(r,e){var t=+r,u=+e,a=65535&t,i=65535&u,n=t>>16,h=u>>16,o=(n*i>>>0)+(a*i>>>16);return n*h+(o>>16)+((a*h>>>0)+(65535&o)>>16)}});\n},{\"./_export\":\"Vobs\"}],\"Xbc5\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Math\",{RAD_PER_DEG:180/Math.PI});\n},{\"./_export\":\"Vobs\"}],\"YSH8\":[function(require,module,exports) {\nvar r=require(\"./_export\"),a=Math.PI/180;r(r.S,\"Math\",{radians:function(r){return r*a}});\n},{\"./_export\":\"Vobs\"}],\"gu1X\":[function(require,module,exports) {\nvar e=require(\"./_export\");e(e.S,\"Math\",{scale:require(\"./_math-scale\")});\n},{\"./_export\":\"Vobs\",\"./_math-scale\":\"ZVag\"}],\"dDqv\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Math\",{umulh:function(r,u){var e=+r,t=+u,a=65535&e,n=65535&t,h=e>>>16,i=t>>>16,o=(h*n>>>0)+(a*n>>>16);return h*i+(o>>>16)+((a*i>>>0)+(65535&o)>>>16)}});\n},{\"./_export\":\"Vobs\"}],\"Q8U8\":[function(require,module,exports) {\nvar r=require(\"./_export\");r(r.S,\"Math\",{signbit:function(r){return(r=+r)!=r?r:0==r?1/r==1/0:r>0}});\n},{\"./_export\":\"Vobs\"}],\"q6pY\":[function(require,module,exports) {\n\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_core\"),t=require(\"./_global\"),n=require(\"./_species-constructor\"),i=require(\"./_promise-resolve\");r(r.P+r.R,\"Promise\",{finally:function(r){var o=n(this,e.Promise||t.Promise),u=\"function\"==typeof r;return this.then(u?function(e){return i(o,r()).then(function(){return e})}:r,u?function(e){return i(o,r()).then(function(){throw e})}:r)}});\n},{\"./_export\":\"Vobs\",\"./_core\":\"DcE6\",\"./_global\":\"QiIT\",\"./_species-constructor\":\"othv\",\"./_promise-resolve\":\"FQFX\"}],\"aULC\":[function(require,module,exports) {\n\"use strict\";var r=require(\"./_export\"),e=require(\"./_new-promise-capability\"),i=require(\"./_perform\");r(r.S,\"Promise\",{try:function(r){var t=e.f(this),o=i(r);return(o.e?t.reject:t.resolve)(o.v),t.promise}});\n},{\"./_export\":\"Vobs\",\"./_new-promise-capability\":\"hTzn\",\"./_perform\":\"X7pO\"}],\"Qewb\":[function(require,module,exports) {\nvar e=require(\"./es6.map\"),r=require(\"./_export\"),t=require(\"./_shared\")(\"metadata\"),n=t.store||(t.store=new(require(\"./es6.weak-map\"))),i=function(r,t,i){var o=n.get(r);if(!o){if(!i)return;n.set(r,o=new e)}var u=o.get(t);if(!u){if(!i)return;o.set(t,u=new e)}return u},o=function(e,r,t){var n=i(r,t,!1);return void 0!==n&&n.has(e)},u=function(e,r,t){var n=i(r,t,!1);return void 0===n?void 0:n.get(e)},a=function(e,r,t,n){i(t,n,!0).set(e,r)},s=function(e,r){var t=i(e,r,!1),n=[];return t&&t.forEach(function(e,r){n.push(r)}),n},f=function(e){return void 0===e||\"symbol\"==typeof e?e:String(e)},c=function(e){r(r.S,\"Reflect\",e)};module.exports={store:n,map:i,has:o,get:u,set:a,keys:s,key:f,exp:c};\n},{\"./es6.map\":\"ksBa\",\"./_export\":\"Vobs\",\"./_shared\":\"k492\",\"./es6.weak-map\":\"Y0Wb\"}],\"zkDQ\":[function(require,module,exports) {\nvar e=require(\"./_metadata\"),a=require(\"./_an-object\"),t=e.key,r=e.set;e.exp({defineMetadata:function(e,i,n,d){r(e,i,a(n),t(d))}});\n},{\"./_metadata\":\"Qewb\",\"./_an-object\":\"AIrJ\"}],\"fy5i\":[function(require,module,exports) {\nvar e=require(\"./_metadata\"),t=require(\"./_an-object\"),r=e.key,a=e.map,i=e.store;e.exp({deleteMetadata:function(e,d){var n=arguments.length<3?void 0:r(arguments[2]),u=a(t(d),n,!1);if(void 0===u||!u.delete(e))return!1;if(u.size)return!0;var l=i.get(d);return l.delete(n),!!l.size||i.delete(d)}});\n},{\"./_metadata\":\"Qewb\",\"./_an-object\":\"AIrJ\"}],\"KBrn\":[function(require,module,exports) {\nvar e=require(\"./_metadata\"),r=require(\"./_an-object\"),t=require(\"./_object-gpo\"),a=e.has,n=e.get,u=e.key,i=function(e,r,u){if(a(e,r,u))return n(e,r,u);var o=t(r);return null!==o?i(e,o,u):void 0};e.exp({getMetadata:function(e,t){return i(e,r(t),arguments.length<3?void 0:u(arguments[2]))}});\n},{\"./_metadata\":\"Qewb\",\"./_an-object\":\"AIrJ\",\"./_object-gpo\":\"dlIw\"}],\"y0Gk\":[function(require,module,exports) {\nvar e=require(\"./es6.set\"),r=require(\"./_array-from-iterable\"),t=require(\"./_metadata\"),a=require(\"./_an-object\"),n=require(\"./_object-gpo\"),u=t.keys,i=t.key,o=function(t,a){var i=u(t,a),c=n(t);if(null===c)return i;var l=o(c,a);return l.length?i.length?r(new e(i.concat(l))):l:i};t.exp({getMetadataKeys:function(e){return o(a(e),arguments.length<2?void 0:i(arguments[1]))}});\n},{\"./es6.set\":\"jPMF\",\"./_array-from-iterable\":\"VUTp\",\"./_metadata\":\"Qewb\",\"./_an-object\":\"AIrJ\",\"./_object-gpo\":\"dlIw\"}],\"sn4U\":[function(require,module,exports) {\nvar e=require(\"./_metadata\"),t=require(\"./_an-object\"),a=e.get,r=e.key;e.exp({getOwnMetadata:function(e,n){return a(e,t(n),arguments.length<3?void 0:r(arguments[2]))}});\n},{\"./_metadata\":\"Qewb\",\"./_an-object\":\"AIrJ\"}],\"bQoJ\":[function(require,module,exports) {\nvar e=require(\"./_metadata\"),t=require(\"./_an-object\"),a=e.keys,r=e.key;e.exp({getOwnMetadataKeys:function(e){return a(t(e),arguments.length<2?void 0:r(arguments[1]))}});\n},{\"./_metadata\":\"Qewb\",\"./_an-object\":\"AIrJ\"}],\"jR0d\":[function(require,module,exports) {\nvar e=require(\"./_metadata\"),r=require(\"./_an-object\"),t=require(\"./_object-gpo\"),a=e.has,n=e.key,u=function(e,r,n){if(a(e,r,n))return!0;var i=t(r);return null!==i&&u(e,i,n)};e.exp({hasMetadata:function(e,t){return u(e,r(t),arguments.length<3?void 0:n(arguments[2]))}});\n},{\"./_metadata\":\"Qewb\",\"./_an-object\":\"AIrJ\",\"./_object-gpo\":\"dlIw\"}],\"tWeA\":[function(require,module,exports) {\nvar e=require(\"./_metadata\"),a=require(\"./_an-object\"),t=e.has,r=e.key;e.exp({hasOwnMetadata:function(e,n){return t(e,a(n),arguments.length<3?void 0:r(arguments[2]))}});\n},{\"./_metadata\":\"Qewb\",\"./_an-object\":\"AIrJ\"}],\"rYHV\":[function(require,module,exports) {\nvar e=require(\"./_metadata\"),t=require(\"./_an-object\"),a=require(\"./_a-function\"),r=e.key,n=e.set;e.exp({metadata:function(e,i){return function(u,o){n(e,i,(void 0!==o?t:a)(u),r(o))}}});\n},{\"./_metadata\":\"Qewb\",\"./_an-object\":\"AIrJ\",\"./_a-function\":\"QKlW\"}],\"kvVj\":[function(require,module,exports) {\n\nvar r=require(\"./_export\"),e=require(\"./_microtask\")(),i=require(\"./_global\").process,o=\"process\"==require(\"./_cof\")(i);r(r.G,{asap:function(r){var a=o&&i.domain;e(a?a.bind(r):r)}});\n},{\"./_export\":\"Vobs\",\"./_microtask\":\"m7QH\",\"./_global\":\"QiIT\",\"./_cof\":\"DrRY\"}],\"iOLx\":[function(require,module,exports) {\n\n\"use strict\";var r=require(\"./_export\"),t=require(\"./_global\"),n=require(\"./_core\"),e=require(\"./_microtask\")(),i=require(\"./_wks\")(\"observable\"),o=require(\"./_a-function\"),u=require(\"./_an-object\"),c=require(\"./_an-instance\"),f=require(\"./_redefine-all\"),s=require(\"./_hide\"),a=require(\"./_for-of\"),v=a.RETURN,h=function(r){return null==r?void 0:o(r)},l=function(r){var t=r._c;t&&(r._c=void 0,t())},_=function(r){return void 0===r._o},b=function(r){_(r)||(r._o=void 0,l(r))},y=function(r,t){u(r),this._c=void 0,this._o=r,r=new p(this);try{var n=t(r),e=n;null!=n&&(\"function\"==typeof n.unsubscribe?n=function(){e.unsubscribe()}:o(n),this._c=n)}catch(i){return void r.error(i)}_(this)&&l(this)};y.prototype=f({},{unsubscribe:function(){b(this)}});var p=function(r){this._s=r};p.prototype=f({},{next:function(r){var t=this._s;if(!_(t)){var n=t._o;try{var e=h(n.next);if(e)return e.call(n,r)}catch(i){try{b(t)}finally{throw i}}}},error:function(r){var t=this._s;if(_(t))throw r;var n=t._o;t._o=void 0;try{var e=h(n.error);if(!e)throw r;r=e.call(n,r)}catch(i){try{l(t)}finally{throw i}}return l(t),r},complete:function(r){var t=this._s;if(!_(t)){var n=t._o;t._o=void 0;try{var e=h(n.complete);r=e?e.call(n,r):void 0}catch(i){try{l(t)}finally{throw i}}return l(t),r}}});var w=function(r){c(this,w,\"Observable\",\"_f\")._f=o(r)};f(w.prototype,{subscribe:function(r){return new y(r,this._f)},forEach:function(r){var e=this;return new(n.Promise||t.Promise)(function(t,n){o(r);var i=e.subscribe({next:function(t){try{return r(t)}catch(e){n(e),i.unsubscribe()}},error:n,complete:t})})}}),f(w,{from:function(r){var t=\"function\"==typeof this?this:w,n=h(u(r)[i]);if(n){var o=u(n.call(r));return o.constructor===t?o:new t(function(r){return o.subscribe(r)})}return new t(function(t){var n=!1;return e(function(){if(!n){try{if(a(r,!1,function(r){if(t.next(r),n)return v})===v)return}catch(e){if(n)throw e;return void t.error(e)}t.complete()}}),function(){n=!0}})},of:function(){for(var r=0,t=arguments.length,n=new Array(t);r<t;)n[r]=arguments[r++];return new(\"function\"==typeof this?this:w)(function(r){var t=!1;return e(function(){if(!t){for(var e=0;e<n.length;++e)if(r.next(n[e]),t)return;r.complete()}}),function(){t=!0}})}}),s(w.prototype,i,function(){return this}),r(r.G,{Observable:w}),require(\"./_set-species\")(\"Observable\");\n},{\"./_export\":\"Vobs\",\"./_global\":\"QiIT\",\"./_core\":\"DcE6\",\"./_microtask\":\"m7QH\",\"./_wks\":\"I5XL\",\"./_a-function\":\"QKlW\",\"./_an-object\":\"AIrJ\",\"./_an-instance\":\"Qz2Q\",\"./_redefine-all\":\"lGTj\",\"./_hide\":\"nCfi\",\"./_for-of\":\"L3cZ\",\"./_set-species\":\"YBdf\"}],\"pUQh\":[function(require,module,exports) {\n\nvar e=require(\"./_global\"),t=require(\"./_export\"),n=require(\"./_user-agent\"),r=[].slice,u=/MSIE .\\./.test(n),i=function(e){return function(t,n){var u=arguments.length>2,i=!!u&&r.call(arguments,2);return e(u?function(){(\"function\"==typeof t?t:Function(t)).apply(this,i)}:t,n)}};t(t.G+t.B+t.F*u,{setTimeout:i(e.setTimeout),setInterval:i(e.setInterval)});\n},{\"./_global\":\"QiIT\",\"./_export\":\"Vobs\",\"./_user-agent\":\"KrKR\"}],\"uORE\":[function(require,module,exports) {\nvar e=require(\"./_export\"),r=require(\"./_task\");e(e.G+e.B,{setImmediate:r.set,clearImmediate:r.clear});\n},{\"./_export\":\"Vobs\",\"./_task\":\"fNEO\"}],\"kCWy\":[function(require,module,exports) {\n\nfor(var e=require(\"./es6.array.iterator\"),t=require(\"./_object-keys\"),i=require(\"./_redefine\"),r=require(\"./_global\"),s=require(\"./_hide\"),L=require(\"./_iterators\"),a=require(\"./_wks\"),o=a(\"iterator\"),l=a(\"toStringTag\"),S=L.Array,n={CSSRuleList:!0,CSSStyleDeclaration:!1,CSSValueList:!1,ClientRectList:!1,DOMRectList:!1,DOMStringList:!1,DOMTokenList:!0,DataTransferItemList:!1,FileList:!1,HTMLAllCollection:!1,HTMLCollection:!1,HTMLFormElement:!1,HTMLSelectElement:!1,MediaList:!0,MimeTypeArray:!1,NamedNodeMap:!1,NodeList:!0,PaintRequestList:!1,Plugin:!1,PluginArray:!1,SVGLengthList:!1,SVGNumberList:!1,SVGPathSegList:!1,SVGPointList:!1,SVGStringList:!1,SVGTransformList:!1,SourceBufferList:!1,StyleSheetList:!0,TextTrackCueList:!1,TextTrackList:!1,TouchList:!1},u=t(n),T=0;T<u.length;T++){var c,g=u[T],M=n[g],y=r[g],f=y&&y.prototype;if(f&&(f[o]||s(f,o,S),f[l]||s(f,l,g),L[g]=S,M))for(c in e)f[c]||i(f,c,e[c],!0)}\n},{\"./es6.array.iterator\":\"ZCkT\",\"./_object-keys\":\"huXi\",\"./_redefine\":\"jDrK\",\"./_global\":\"QiIT\",\"./_hide\":\"nCfi\",\"./_iterators\":\"H5RD\",\"./_wks\":\"I5XL\"}],\"y1LN\":[function(require,module,exports) {\nrequire(\"./modules/es6.symbol\"),require(\"./modules/es6.object.create\"),require(\"./modules/es6.object.define-property\"),require(\"./modules/es6.object.define-properties\"),require(\"./modules/es6.object.get-own-property-descriptor\"),require(\"./modules/es6.object.get-prototype-of\"),require(\"./modules/es6.object.keys\"),require(\"./modules/es6.object.get-own-property-names\"),require(\"./modules/es6.object.freeze\"),require(\"./modules/es6.object.seal\"),require(\"./modules/es6.object.prevent-extensions\"),require(\"./modules/es6.object.is-frozen\"),require(\"./modules/es6.object.is-sealed\"),require(\"./modules/es6.object.is-extensible\"),require(\"./modules/es6.object.assign\"),require(\"./modules/es6.object.is\"),require(\"./modules/es6.object.set-prototype-of\"),require(\"./modules/es6.object.to-string\"),require(\"./modules/es6.function.bind\"),require(\"./modules/es6.function.name\"),require(\"./modules/es6.function.has-instance\"),require(\"./modules/es6.parse-int\"),require(\"./modules/es6.parse-float\"),require(\"./modules/es6.number.constructor\"),require(\"./modules/es6.number.to-fixed\"),require(\"./modules/es6.number.to-precision\"),require(\"./modules/es6.number.epsilon\"),require(\"./modules/es6.number.is-finite\"),require(\"./modules/es6.number.is-integer\"),require(\"./modules/es6.number.is-nan\"),require(\"./modules/es6.number.is-safe-integer\"),require(\"./modules/es6.number.max-safe-integer\"),require(\"./modules/es6.number.min-safe-integer\"),require(\"./modules/es6.number.parse-float\"),require(\"./modules/es6.number.parse-int\"),require(\"./modules/es6.math.acosh\"),require(\"./modules/es6.math.asinh\"),require(\"./modules/es6.math.atanh\"),require(\"./modules/es6.math.cbrt\"),require(\"./modules/es6.math.clz32\"),require(\"./modules/es6.math.cosh\"),require(\"./modules/es6.math.expm1\"),require(\"./modules/es6.math.fround\"),require(\"./modules/es6.math.hypot\"),require(\"./modules/es6.math.imul\"),require(\"./modules/es6.math.log10\"),require(\"./modules/es6.math.log1p\"),require(\"./modules/es6.math.log2\"),require(\"./modules/es6.math.sign\"),require(\"./modules/es6.math.sinh\"),require(\"./modules/es6.math.tanh\"),require(\"./modules/es6.math.trunc\"),require(\"./modules/es6.string.from-code-point\"),require(\"./modules/es6.string.raw\"),require(\"./modules/es6.string.trim\"),require(\"./modules/es6.string.iterator\"),require(\"./modules/es6.string.code-point-at\"),require(\"./modules/es6.string.ends-with\"),require(\"./modules/es6.string.includes\"),require(\"./modules/es6.string.repeat\"),require(\"./modules/es6.string.starts-with\"),require(\"./modules/es6.string.anchor\"),require(\"./modules/es6.string.big\"),require(\"./modules/es6.string.blink\"),require(\"./modules/es6.string.bold\"),require(\"./modules/es6.string.fixed\"),require(\"./modules/es6.string.fontcolor\"),require(\"./modules/es6.string.fontsize\"),require(\"./modules/es6.string.italics\"),require(\"./modules/es6.string.link\"),require(\"./modules/es6.string.small\"),require(\"./modules/es6.string.strike\"),require(\"./modules/es6.string.sub\"),require(\"./modules/es6.string.sup\"),require(\"./modules/es6.date.now\"),require(\"./modules/es6.date.to-json\"),require(\"./modules/es6.date.to-iso-string\"),require(\"./modules/es6.date.to-string\"),require(\"./modules/es6.date.to-primitive\"),require(\"./modules/es6.array.is-array\"),require(\"./modules/es6.array.from\"),require(\"./modules/es6.array.of\"),require(\"./modules/es6.array.join\"),require(\"./modules/es6.array.slice\"),require(\"./modules/es6.array.sort\"),require(\"./modules/es6.array.for-each\"),require(\"./modules/es6.array.map\"),require(\"./modules/es6.array.filter\"),require(\"./modules/es6.array.some\"),require(\"./modules/es6.array.every\"),require(\"./modules/es6.array.reduce\"),require(\"./modules/es6.array.reduce-right\"),require(\"./modules/es6.array.index-of\"),require(\"./modules/es6.array.last-index-of\"),require(\"./modules/es6.array.copy-within\"),require(\"./modules/es6.array.fill\"),require(\"./modules/es6.array.find\"),require(\"./modules/es6.array.find-index\"),require(\"./modules/es6.array.species\"),require(\"./modules/es6.array.iterator\"),require(\"./modules/es6.regexp.constructor\"),require(\"./modules/es6.regexp.exec\"),require(\"./modules/es6.regexp.to-string\"),require(\"./modules/es6.regexp.flags\"),require(\"./modules/es6.regexp.match\"),require(\"./modules/es6.regexp.replace\"),require(\"./modules/es6.regexp.search\"),require(\"./modules/es6.regexp.split\"),require(\"./modules/es6.promise\"),require(\"./modules/es6.map\"),require(\"./modules/es6.set\"),require(\"./modules/es6.weak-map\"),require(\"./modules/es6.weak-set\"),require(\"./modules/es6.typed.array-buffer\"),require(\"./modules/es6.typed.data-view\"),require(\"./modules/es6.typed.int8-array\"),require(\"./modules/es6.typed.uint8-array\"),require(\"./modules/es6.typed.uint8-clamped-array\"),require(\"./modules/es6.typed.int16-array\"),require(\"./modules/es6.typed.uint16-array\"),require(\"./modules/es6.typed.int32-array\"),require(\"./modules/es6.typed.uint32-array\"),require(\"./modules/es6.typed.float32-array\"),require(\"./modules/es6.typed.float64-array\"),require(\"./modules/es6.reflect.apply\"),require(\"./modules/es6.reflect.construct\"),require(\"./modules/es6.reflect.define-property\"),require(\"./modules/es6.reflect.delete-property\"),require(\"./modules/es6.reflect.enumerate\"),require(\"./modules/es6.reflect.get\"),require(\"./modules/es6.reflect.get-own-property-descriptor\"),require(\"./modules/es6.reflect.get-prototype-of\"),require(\"./modules/es6.reflect.has\"),require(\"./modules/es6.reflect.is-extensible\"),require(\"./modules/es6.reflect.own-keys\"),require(\"./modules/es6.reflect.prevent-extensions\"),require(\"./modules/es6.reflect.set\"),require(\"./modules/es6.reflect.set-prototype-of\"),require(\"./modules/es7.array.includes\"),require(\"./modules/es7.array.flat-map\"),require(\"./modules/es7.array.flatten\"),require(\"./modules/es7.string.at\"),require(\"./modules/es7.string.pad-start\"),require(\"./modules/es7.string.pad-end\"),require(\"./modules/es7.string.trim-left\"),require(\"./modules/es7.string.trim-right\"),require(\"./modules/es7.string.match-all\"),require(\"./modules/es7.symbol.async-iterator\"),require(\"./modules/es7.symbol.observable\"),require(\"./modules/es7.object.get-own-property-descriptors\"),require(\"./modules/es7.object.values\"),require(\"./modules/es7.object.entries\"),require(\"./modules/es7.object.define-getter\"),require(\"./modules/es7.object.define-setter\"),require(\"./modules/es7.object.lookup-getter\"),require(\"./modules/es7.object.lookup-setter\"),require(\"./modules/es7.map.to-json\"),require(\"./modules/es7.set.to-json\"),require(\"./modules/es7.map.of\"),require(\"./modules/es7.set.of\"),require(\"./modules/es7.weak-map.of\"),require(\"./modules/es7.weak-set.of\"),require(\"./modules/es7.map.from\"),require(\"./modules/es7.set.from\"),require(\"./modules/es7.weak-map.from\"),require(\"./modules/es7.weak-set.from\"),require(\"./modules/es7.global\"),require(\"./modules/es7.system.global\"),require(\"./modules/es7.error.is-error\"),require(\"./modules/es7.math.clamp\"),require(\"./modules/es7.math.deg-per-rad\"),require(\"./modules/es7.math.degrees\"),require(\"./modules/es7.math.fscale\"),require(\"./modules/es7.math.iaddh\"),require(\"./modules/es7.math.isubh\"),require(\"./modules/es7.math.imulh\"),require(\"./modules/es7.math.rad-per-deg\"),require(\"./modules/es7.math.radians\"),require(\"./modules/es7.math.scale\"),require(\"./modules/es7.math.umulh\"),require(\"./modules/es7.math.signbit\"),require(\"./modules/es7.promise.finally\"),require(\"./modules/es7.promise.try\"),require(\"./modules/es7.reflect.define-metadata\"),require(\"./modules/es7.reflect.delete-metadata\"),require(\"./modules/es7.reflect.get-metadata\"),require(\"./modules/es7.reflect.get-metadata-keys\"),require(\"./modules/es7.reflect.get-own-metadata\"),require(\"./modules/es7.reflect.get-own-metadata-keys\"),require(\"./modules/es7.reflect.has-metadata\"),require(\"./modules/es7.reflect.has-own-metadata\"),require(\"./modules/es7.reflect.metadata\"),require(\"./modules/es7.asap\"),require(\"./modules/es7.observable\"),require(\"./modules/web.timers\"),require(\"./modules/web.immediate\"),require(\"./modules/web.dom.iterable\"),module.exports=require(\"./modules/_core\");\n},{\"./modules/es6.symbol\":\"rGq9\",\"./modules/es6.object.create\":\"v5CS\",\"./modules/es6.object.define-property\":\"pS46\",\"./modules/es6.object.define-properties\":\"sbXv\",\"./modules/es6.object.get-own-property-descriptor\":\"xCvV\",\"./modules/es6.object.get-prototype-of\":\"Dkc5\",\"./modules/es6.object.keys\":\"RpZ9\",\"./modules/es6.object.get-own-property-names\":\"mVnl\",\"./modules/es6.object.freeze\":\"bkZb\",\"./modules/es6.object.seal\":\"LEG2\",\"./modules/es6.object.prevent-extensions\":\"OeTo\",\"./modules/es6.object.is-frozen\":\"Lm2M\",\"./modules/es6.object.is-sealed\":\"Lrni\",\"./modules/es6.object.is-extensible\":\"ypI7\",\"./modules/es6.object.assign\":\"av62\",\"./modules/es6.object.is\":\"OI80\",\"./modules/es6.object.set-prototype-of\":\"xZ9m\",\"./modules/es6.object.to-string\":\"zmtK\",\"./modules/es6.function.bind\":\"qI6I\",\"./modules/es6.function.name\":\"z3jV\",\"./modules/es6.function.has-instance\":\"owRX\",\"./modules/es6.parse-int\":\"nPGY\",\"./modules/es6.parse-float\":\"yexh\",\"./modules/es6.number.constructor\":\"F74v\",\"./modules/es6.number.to-fixed\":\"qGBb\",\"./modules/es6.number.to-precision\":\"bLBB\",\"./modules/es6.number.epsilon\":\"oSwj\",\"./modules/es6.number.is-finite\":\"Iwqp\",\"./modules/es6.number.is-integer\":\"XPnJ\",\"./modules/es6.number.is-nan\":\"PMgb\",\"./modules/es6.number.is-safe-integer\":\"EvBV\",\"./modules/es6.number.max-safe-integer\":\"fOC8\",\"./modules/es6.number.min-safe-integer\":\"yvVo\",\"./modules/es6.number.parse-float\":\"a09l\",\"./modules/es6.number.parse-int\":\"fCj1\",\"./modules/es6.math.acosh\":\"o78V\",\"./modules/es6.math.asinh\":\"xkGF\",\"./modules/es6.math.atanh\":\"Pmrp\",\"./modules/es6.math.cbrt\":\"Giui\",\"./modules/es6.math.clz32\":\"HsTu\",\"./modules/es6.math.cosh\":\"xEUq\",\"./modules/es6.math.expm1\":\"aBEU\",\"./modules/es6.math.fround\":\"IjCR\",\"./modules/es6.math.hypot\":\"HXfT\",\"./modules/es6.math.imul\":\"m2OX\",\"./modules/es6.math.log10\":\"E567\",\"./modules/es6.math.log1p\":\"ymfv\",\"./modules/es6.math.log2\":\"hUIM\",\"./modules/es6.math.sign\":\"d1Y4\",\"./modules/es6.math.sinh\":\"dhHM\",\"./modules/es6.math.tanh\":\"cxv8\",\"./modules/es6.math.trunc\":\"xO7u\",\"./modules/es6.string.from-code-point\":\"DdG0\",\"./modules/es6.string.raw\":\"KDcE\",\"./modules/es6.string.trim\":\"DDrZ\",\"./modules/es6.string.iterator\":\"WN4F\",\"./modules/es6.string.code-point-at\":\"gGid\",\"./modules/es6.string.ends-with\":\"PmIB\",\"./modules/es6.string.includes\":\"qgIv\",\"./modules/es6.string.repeat\":\"ZAbm\",\"./modules/es6.string.starts-with\":\"U3MC\",\"./modules/es6.string.anchor\":\"eRhq\",\"./modules/es6.string.big\":\"HLSM\",\"./modules/es6.string.blink\":\"RtH9\",\"./modules/es6.string.bold\":\"efe7\",\"./modules/es6.string.fixed\":\"v3Ez\",\"./modules/es6.string.fontcolor\":\"RECM\",\"./modules/es6.string.fontsize\":\"l7OI\",\"./modules/es6.string.italics\":\"uJlj\",\"./modules/es6.string.link\":\"vYww\",\"./modules/es6.string.small\":\"AiXZ\",\"./modules/es6.string.strike\":\"MhVl\",\"./modules/es6.string.sub\":\"DFMN\",\"./modules/es6.string.sup\":\"X3LC\",\"./modules/es6.date.now\":\"Sydr\",\"./modules/es6.date.to-json\":\"GNUn\",\"./modules/es6.date.to-iso-string\":\"fPZl\",\"./modules/es6.date.to-string\":\"FKfL\",\"./modules/es6.date.to-primitive\":\"nktC\",\"./modules/es6.array.is-array\":\"XjkF\",\"./modules/es6.array.from\":\"WZRw\",\"./modules/es6.array.of\":\"URTo\",\"./modules/es6.array.join\":\"BTDR\",\"./modules/es6.array.slice\":\"Ui7t\",\"./modules/es6.array.sort\":\"TqUy\",\"./modules/es6.array.for-each\":\"vDWP\",\"./modules/es6.array.map\":\"O0lf\",\"./modules/es6.array.filter\":\"PXKF\",\"./modules/es6.array.some\":\"wD6H\",\"./modules/es6.array.every\":\"n6bP\",\"./modules/es6.array.reduce\":\"OWmJ\",\"./modules/es6.array.reduce-right\":\"k5ri\",\"./modules/es6.array.index-of\":\"HB9A\",\"./modules/es6.array.last-index-of\":\"tgt4\",\"./modules/es6.array.copy-within\":\"c9DC\",\"./modules/es6.array.fill\":\"ZBH0\",\"./modules/es6.array.find\":\"wTIB\",\"./modules/es6.array.find-index\":\"ksrS\",\"./modules/es6.array.species\":\"Adki\",\"./modules/es6.array.iterator\":\"ZCkT\",\"./modules/es6.regexp.constructor\":\"lK2M\",\"./modules/es6.regexp.exec\":\"f98m\",\"./modules/es6.regexp.to-string\":\"jkaB\",\"./modules/es6.regexp.flags\":\"S072\",\"./modules/es6.regexp.match\":\"Iomp\",\"./modules/es6.regexp.replace\":\"weWA\",\"./modules/es6.regexp.search\":\"EA9T\",\"./modules/es6.regexp.split\":\"d289\",\"./modules/es6.promise\":\"MWl4\",\"./modules/es6.map\":\"ksBa\",\"./modules/es6.set\":\"jPMF\",\"./modules/es6.weak-map\":\"Y0Wb\",\"./modules/es6.weak-set\":\"oeIc\",\"./modules/es6.typed.array-buffer\":\"VqD6\",\"./modules/es6.typed.data-view\":\"q3b2\",\"./modules/es6.typed.int8-array\":\"FrGE\",\"./modules/es6.typed.uint8-array\":\"jLcZ\",\"./modules/es6.typed.uint8-clamped-array\":\"dFjM\",\"./modules/es6.typed.int16-array\":\"XAXm\",\"./modules/es6.typed.uint16-array\":\"Vod2\",\"./modules/es6.typed.int32-array\":\"Mnlj\",\"./modules/es6.typed.uint32-array\":\"JJCv\",\"./modules/es6.typed.float32-array\":\"Asas\",\"./modules/es6.typed.float64-array\":\"ZKGF\",\"./modules/es6.reflect.apply\":\"sL26\",\"./modules/es6.reflect.construct\":\"n0sj\",\"./modules/es6.reflect.define-property\":\"XoPA\",\"./modules/es6.reflect.delete-property\":\"YgqD\",\"./modules/es6.reflect.enumerate\":\"CKoQ\",\"./modules/es6.reflect.get\":\"Jr0s\",\"./modules/es6.reflect.get-own-property-descriptor\":\"rsHl\",\"./modules/es6.reflect.get-prototype-of\":\"mTTK\",\"./modules/es6.reflect.has\":\"VxVc\",\"./modules/es6.reflect.is-extensible\":\"lQ3X\",\"./modules/es6.reflect.own-keys\":\"vOF6\",\"./modules/es6.reflect.prevent-extensions\":\"hWQ0\",\"./modules/es6.reflect.set\":\"AiN1\",\"./modules/es6.reflect.set-prototype-of\":\"EPEE\",\"./modules/es7.array.includes\":\"gMo0\",\"./modules/es7.array.flat-map\":\"zKV8\",\"./modules/es7.array.flatten\":\"GDMJ\",\"./modules/es7.string.at\":\"K4uP\",\"./modules/es7.string.pad-start\":\"hmYY\",\"./modules/es7.string.pad-end\":\"RIKd\",\"./modules/es7.string.trim-left\":\"hNft\",\"./modules/es7.string.trim-right\":\"uLyC\",\"./modules/es7.string.match-all\":\"dRqM\",\"./modules/es7.symbol.async-iterator\":\"enid\",\"./modules/es7.symbol.observable\":\"Oxke\",\"./modules/es7.object.get-own-property-descriptors\":\"ovdg\",\"./modules/es7.object.values\":\"exYH\",\"./modules/es7.object.entries\":\"jLAB\",\"./modules/es7.object.define-getter\":\"y7i0\",\"./modules/es7.object.define-setter\":\"vFGQ\",\"./modules/es7.object.lookup-getter\":\"urEd\",\"./modules/es7.object.lookup-setter\":\"qicQ\",\"./modules/es7.map.to-json\":\"gCox\",\"./modules/es7.set.to-json\":\"CwpA\",\"./modules/es7.map.of\":\"bPOJ\",\"./modules/es7.set.of\":\"swmI\",\"./modules/es7.weak-map.of\":\"Kb3C\",\"./modules/es7.weak-set.of\":\"HgXJ\",\"./modules/es7.map.from\":\"mnJw\",\"./modules/es7.set.from\":\"Wc9c\",\"./modules/es7.weak-map.from\":\"RABC\",\"./modules/es7.weak-set.from\":\"irWo\",\"./modules/es7.global\":\"DjhA\",\"./modules/es7.system.global\":\"zQTI\",\"./modules/es7.error.is-error\":\"sx2w\",\"./modules/es7.math.clamp\":\"duUS\",\"./modules/es7.math.deg-per-rad\":\"Nayo\",\"./modules/es7.math.degrees\":\"pK3L\",\"./modules/es7.math.fscale\":\"cNya\",\"./modules/es7.math.iaddh\":\"JpQg\",\"./modules/es7.math.isubh\":\"kYRl\",\"./modules/es7.math.imulh\":\"iMz3\",\"./modules/es7.math.rad-per-deg\":\"Xbc5\",\"./modules/es7.math.radians\":\"YSH8\",\"./modules/es7.math.scale\":\"gu1X\",\"./modules/es7.math.umulh\":\"dDqv\",\"./modules/es7.math.signbit\":\"Q8U8\",\"./modules/es7.promise.finally\":\"q6pY\",\"./modules/es7.promise.try\":\"aULC\",\"./modules/es7.reflect.define-metadata\":\"zkDQ\",\"./modules/es7.reflect.delete-metadata\":\"fy5i\",\"./modules/es7.reflect.get-metadata\":\"KBrn\",\"./modules/es7.reflect.get-metadata-keys\":\"y0Gk\",\"./modules/es7.reflect.get-own-metadata\":\"sn4U\",\"./modules/es7.reflect.get-own-metadata-keys\":\"bQoJ\",\"./modules/es7.reflect.has-metadata\":\"jR0d\",\"./modules/es7.reflect.has-own-metadata\":\"tWeA\",\"./modules/es7.reflect.metadata\":\"rYHV\",\"./modules/es7.asap\":\"kvVj\",\"./modules/es7.observable\":\"iOLx\",\"./modules/web.timers\":\"pUQh\",\"./modules/web.immediate\":\"uORE\",\"./modules/web.dom.iterable\":\"kCWy\",\"./modules/_core\":\"DcE6\"}],\"VuXv\":[function(require,module,exports) {\nvar global = arguments[3];\nvar t=arguments[3];!function(t){\"use strict\";var r,e=Object.prototype,n=e.hasOwnProperty,o=\"function\"==typeof Symbol?Symbol:{},i=o.iterator||\"@@iterator\",a=o.asyncIterator||\"@@asyncIterator\",c=o.toStringTag||\"@@toStringTag\",u=\"object\"==typeof module,h=t.regeneratorRuntime;if(h)u&&(module.exports=h);else{(h=t.regeneratorRuntime=u?module.exports:{}).wrap=w;var s=\"suspendedStart\",f=\"suspendedYield\",l=\"executing\",p=\"completed\",y={},v={};v[i]=function(){return this};var d=Object.getPrototypeOf,g=d&&d(d(P([])));g&&g!==e&&n.call(g,i)&&(v=g);var m=E.prototype=x.prototype=Object.create(v);b.prototype=m.constructor=E,E.constructor=b,E[c]=b.displayName=\"GeneratorFunction\",h.isGeneratorFunction=function(t){var r=\"function\"==typeof t&&t.constructor;return!!r&&(r===b||\"GeneratorFunction\"===(r.displayName||r.name))},h.mark=function(t){return Object.setPrototypeOf?Object.setPrototypeOf(t,E):(t.__proto__=E,c in t||(t[c]=\"GeneratorFunction\")),t.prototype=Object.create(m),t},h.awrap=function(t){return{__await:t}},j(_.prototype),_.prototype[a]=function(){return this},h.AsyncIterator=_,h.async=function(t,r,e,n){var o=new _(w(t,r,e,n));return h.isGeneratorFunction(r)?o:o.next().then(function(t){return t.done?t.value:o.next()})},j(m),m[c]=\"Generator\",m[i]=function(){return this},m.toString=function(){return\"[object Generator]\"},h.keys=function(t){var r=[];for(var e in t)r.push(e);return r.reverse(),function e(){for(;r.length;){var n=r.pop();if(n in t)return e.value=n,e.done=!1,e}return e.done=!0,e}},h.values=P,N.prototype={constructor:N,reset:function(t){if(this.prev=0,this.next=0,this.sent=this._sent=r,this.done=!1,this.delegate=null,this.method=\"next\",this.arg=r,this.tryEntries.forEach(G),!t)for(var e in this)\"t\"===e.charAt(0)&&n.call(this,e)&&!isNaN(+e.slice(1))&&(this[e]=r)},stop:function(){this.done=!0;var t=this.tryEntries[0].completion;if(\"throw\"===t.type)throw t.arg;return this.rval},dispatchException:function(t){if(this.done)throw t;var e=this;function o(n,o){return c.type=\"throw\",c.arg=t,e.next=n,o&&(e.method=\"next\",e.arg=r),!!o}for(var i=this.tryEntries.length-1;i>=0;--i){var a=this.tryEntries[i],c=a.completion;if(\"root\"===a.tryLoc)return o(\"end\");if(a.tryLoc<=this.prev){var u=n.call(a,\"catchLoc\"),h=n.call(a,\"finallyLoc\");if(u&&h){if(this.prev<a.catchLoc)return o(a.catchLoc,!0);if(this.prev<a.finallyLoc)return o(a.finallyLoc)}else if(u){if(this.prev<a.catchLoc)return o(a.catchLoc,!0)}else{if(!h)throw new Error(\"try statement without catch or finally\");if(this.prev<a.finallyLoc)return o(a.finallyLoc)}}}},abrupt:function(t,r){for(var e=this.tryEntries.length-1;e>=0;--e){var o=this.tryEntries[e];if(o.tryLoc<=this.prev&&n.call(o,\"finallyLoc\")&&this.prev<o.finallyLoc){var i=o;break}}i&&(\"break\"===t||\"continue\"===t)&&i.tryLoc<=r&&r<=i.finallyLoc&&(i=null);var a=i?i.completion:{};return a.type=t,a.arg=r,i?(this.method=\"next\",this.next=i.finallyLoc,y):this.complete(a)},complete:function(t,r){if(\"throw\"===t.type)throw t.arg;return\"break\"===t.type||\"continue\"===t.type?this.next=t.arg:\"return\"===t.type?(this.rval=this.arg=t.arg,this.method=\"return\",this.next=\"end\"):\"normal\"===t.type&&r&&(this.next=r),y},finish:function(t){for(var r=this.tryEntries.length-1;r>=0;--r){var e=this.tryEntries[r];if(e.finallyLoc===t)return this.complete(e.completion,e.afterLoc),G(e),y}},catch:function(t){for(var r=this.tryEntries.length-1;r>=0;--r){var e=this.tryEntries[r];if(e.tryLoc===t){var n=e.completion;if(\"throw\"===n.type){var o=n.arg;G(e)}return o}}throw new Error(\"illegal catch attempt\")},delegateYield:function(t,e,n){return this.delegate={iterator:P(t),resultName:e,nextLoc:n},\"next\"===this.method&&(this.arg=r),y}}}function w(t,r,e,n){var o=r&&r.prototype instanceof x?r:x,i=Object.create(o.prototype),a=new N(n||[]);return i._invoke=function(t,r,e){var n=s;return function(o,i){if(n===l)throw new Error(\"Generator is already running\");if(n===p){if(\"throw\"===o)throw i;return S()}for(e.method=o,e.arg=i;;){var a=e.delegate;if(a){var c=O(a,e);if(c){if(c===y)continue;return c}}if(\"next\"===e.method)e.sent=e._sent=e.arg;else if(\"throw\"===e.method){if(n===s)throw n=p,e.arg;e.dispatchException(e.arg)}else\"return\"===e.method&&e.abrupt(\"return\",e.arg);n=l;var u=L(t,r,e);if(\"normal\"===u.type){if(n=e.done?p:f,u.arg===y)continue;return{value:u.arg,done:e.done}}\"throw\"===u.type&&(n=p,e.method=\"throw\",e.arg=u.arg)}}}(t,e,a),i}function L(t,r,e){try{return{type:\"normal\",arg:t.call(r,e)}}catch(n){return{type:\"throw\",arg:n}}}function x(){}function b(){}function E(){}function j(t){[\"next\",\"throw\",\"return\"].forEach(function(r){t[r]=function(t){return this._invoke(r,t)}})}function _(r){function e(t,o,i,a){var c=L(r[t],r,o);if(\"throw\"!==c.type){var u=c.arg,h=u.value;return h&&\"object\"==typeof h&&n.call(h,\"__await\")?Promise.resolve(h.__await).then(function(t){e(\"next\",t,i,a)},function(t){e(\"throw\",t,i,a)}):Promise.resolve(h).then(function(t){u.value=t,i(u)},a)}a(c.arg)}var o;\"object\"==typeof t.process&&t.process.domain&&(e=t.process.domain.bind(e)),this._invoke=function(t,r){function n(){return new Promise(function(n,o){e(t,r,n,o)})}return o=o?o.then(n,n):n()}}function O(t,e){var n=t.iterator[e.method];if(n===r){if(e.delegate=null,\"throw\"===e.method){if(t.iterator.return&&(e.method=\"return\",e.arg=r,O(t,e),\"throw\"===e.method))return y;e.method=\"throw\",e.arg=new TypeError(\"The iterator does not provide a 'throw' method\")}return y}var o=L(n,t.iterator,e.arg);if(\"throw\"===o.type)return e.method=\"throw\",e.arg=o.arg,e.delegate=null,y;var i=o.arg;return i?i.done?(e[t.resultName]=i.value,e.next=t.nextLoc,\"return\"!==e.method&&(e.method=\"next\",e.arg=r),e.delegate=null,y):i:(e.method=\"throw\",e.arg=new TypeError(\"iterator result is not an object\"),e.delegate=null,y)}function k(t){var r={tryLoc:t[0]};1 in t&&(r.catchLoc=t[1]),2 in t&&(r.finallyLoc=t[2],r.afterLoc=t[3]),this.tryEntries.push(r)}function G(t){var r=t.completion||{};r.type=\"normal\",delete r.arg,t.completion=r}function N(t){this.tryEntries=[{tryLoc:\"root\"}],t.forEach(k,this),this.reset(!0)}function P(t){if(t){var e=t[i];if(e)return e.call(t);if(\"function\"==typeof t.next)return t;if(!isNaN(t.length)){var o=-1,a=function e(){for(;++o<t.length;)if(n.call(t,o))return e.value=t[o],e.done=!1,e;return e.value=r,e.done=!0,e};return a.next=a}}return{next:S}}function S(){return{value:r,done:!0}}}(\"object\"==typeof t?t:\"object\"==typeof window?window:\"object\"==typeof self?self:this);\n},{}],\"dUxS\":[function(require,module,exports) {\nmodule.exports=function(n,r){var t=r===Object(r)?function(n){return r[n]}:r;return function(r){return String(r).replace(n,t)}};\n},{}],\"AoXz\":[function(require,module,exports) {\nvar e=require(\"./_export\"),r=require(\"./_replacer\")(/[\\\\^$*+?.()|[\\]{}]/g,\"\\\\$&\");e(e.S,\"RegExp\",{escape:function(e){return r(e)}});\n},{\"./_export\":\"Vobs\",\"./_replacer\":\"dUxS\"}],\"Rlym\":[function(require,module,exports) {\nrequire(\"../../modules/core.regexp.escape\"),module.exports=require(\"../../modules/_core\").RegExp.escape;\n},{\"../../modules/core.regexp.escape\":\"AoXz\",\"../../modules/_core\":\"DcE6\"}],\"zUFY\":[function(require,module,exports) {\nvar global = arguments[3];\n\nvar e=arguments[3];if(require(\"core-js/shim\"),require(\"regenerator-runtime/runtime\"),require(\"core-js/fn/regexp/escape\"),e._babelPolyfill)throw new Error(\"only one instance of babel-polyfill is allowed\");e._babelPolyfill=!0;var r=\"defineProperty\";function i(e,i,n){e[i]||Object[r](e,i,{writable:!0,configurable:!0,value:n})}i(String.prototype,\"padLeft\",\"\".padStart),i(String.prototype,\"padRight\",\"\".padEnd),\"pop,reverse,shift,keys,values,entries,indexOf,every,some,forEach,map,filter,find,findIndex,includes,join,slice,concat,push,splice,unshift,sort,lastIndexOf,reduce,reduceRight,copyWithin,fill\".split(\",\").forEach(function(e){[][e]&&i(Array,e,Function.call.bind([][e]))});\n},{\"core-js/shim\":\"y1LN\",\"regenerator-runtime/runtime\":\"VuXv\",\"core-js/fn/regexp/escape\":\"Rlym\"}],\"sKvN\":[function(require,module,exports) {\n\"use strict\";function t(t,e){if(!(t instanceof e))throw new TypeError(\"Cannot call a class as a function\")}function e(t,e){for(var n=0;n<e.length;n++){var s=e[n];s.enumerable=s.enumerable||!1,s.configurable=!0,\"value\"in s&&(s.writable=!0),Object.defineProperty(t,s.key,s)}}function n(t,n,s){return n&&e(t.prototype,n),s&&e(t,s),t}Object.defineProperty(exports,\"__esModule\",{value:!0}),exports.default=void 0;var s=function(){function e(n){t(this,e),this.doc=document,this.nav=this.doc.querySelectorAll(n.navSelector),0!==!this.nav.length&&(this.win=window,this.winHeight=this.win.innerHeight,this.scrollElement=this.doc.querySelector(n.scrollSelector),this.className=n.className,this.offsetTop=n.offsetTop||0,this.contents=[],this.contents=this.getContents(n.contentSelector),this.attachEvent())}return n(e,[{key:\"attachEvent\",value:function(){var t,e,n=this;this.scrollElement.addEventListener(\"scroll\",function(){t&&clearTimeout(t),t=setTimeout(function(){n.spy()},1)}),this.scrollElement.addEventListener(\"resize\",function(){e&&clearTimeout(e),e=setTimeout(function(){n.spy()},1)}),this.scrollElement.addEventListener(\"click\",function(t){var e=t.target;if(\"A\"===e.tagName){window.onclickToc=!0;for(var s=0,i=n.nav.length;s<i;s++){var o=n.nav[s];o.href===e.href?(o.classList.add(n.className),o.classList.add(\"mdl-color-text--primary\")):(o.classList.remove(n.className),o.classList.remove(\"mdl-color-text--primary\"))}}})}},{key:\"getContents\",value:function(t){for(var e=[],n=0,s=this.nav.length;n<s;n++){var i=this.nav[n].href;e.push(this.doc.getElementById(i.split(\"#\")[1]))}return e}},{key:\"spy\",value:function(){var t=this.getViewState();this.toggleNavClass(t)}},{key:\"getViewState\",value:function(){for(var t=[],e=0,n=this.contents.length;e<n;e++){var s=this.contents[e];s&&this.isView(s)&&t.push(s)}return t}},{key:\"isView\",value:function(t){var e=this.scrollElement.scrollTop,n=document.querySelector(\".mdl-layout__header-row\").getBoundingClientRect(),s=n.top+n.height,i=e+window.innerHeight-s,o=t.getBoundingClientRect().top+e,a=o+t.offsetHeight;return o<i-30&&a>e+s+30}},{key:\"toggleNavClass\",value:function(t){if(window.onclickToc)window.onclickToc=!1;else{for(var e=0,n=$(),s=0,i=t.length;s<i;s++){var o=t[s],a=this.getTagDepth(o);e<a&&(e=a,n=o)}for(var r=0,l=this.nav.length;r<l;r++){var c=this.nav[r];c.href.split(\"#\")[1]===n.id?(c.classList.add(this.className),c.classList.add(\"mdl-color-text--primary\")):(c.classList.remove(this.className),c.classList.remove(\"mdl-color-text--primary\"))}}}},{key:\"getTagDepth\",value:function(t){return parseInt($(t).find(\"h1,h2,h3,h4,h5,h6\").get(0).tagName.split(\"H\")[1])}}]),e}();exports.default=s;\n},{}],\"brfV\":[function(require,module,exports) {\n\"use strict\";require(\"../scss/sphinx_materialdesign_theme.scss\"),require(\"./feedback\"),require(\"material-design-lite\"),require(\"babel-polyfill\");var a=t(require(\"./scrollspy\"));function t(a){return a&&a.__esModule?a:{default:a}}$(function(){var t,e;t=$(\".mdl-layout__drawer nav\").find(\"li\"),$.each(t,function(a,t){var e=$(t),n=$('<span class=\"link-wrapper\"></span>'),l=e.children(\"a\");e.append(n.append(l));var o=e.hasClass(\"current\")&&!l.hasClass(\"current\"),d=e.children(\"ul\");if(d.length){var s=\"globalnav-\".concat(a);d.attr(\"id\",s),d.addClass(\"collapse\");var i=$('<span class=\"nav-toggle\"></span>');o?(d.addClass(\"show\"),i.addClass(\"show\")):d.hide(),e.append(n.append(i.append($('<a class=\"mdl-button mdl-js-button mdl-button--icon\" data-toggle=\"#'.concat(s,'\"><span style=\"color: #888\"><i class=\"material-icons\">keyboard_arrow_down</i></span></span>'))))).append(d)}}),$(\".mdl-layout__drawer nav .nav-toggle a\").click(function(){var a=$(this),t=a.attr(\"data-toggle\");$(\"ul\".concat(t)).toggleClass(\"show\").animate({height:\"toggle\",opacity:\"toggle\"}),a.parent().toggleClass(\"show\")}),e=$(\".breadcrumb\"),$(\"#waterfall-exp\").focus(function(){$(window).width()<=1024&&e.hide()}).blur(function(){$(window).width()<=1024&&e.show()});new a.default({contentSelector:\".page-content .section\",navSelector:\".localtoc a\",scrollSelector:\"main\",className:\"current\",offsetTop:64});$(\".mdl-layout__content\").focus(),$(\".mx-card\").each(function(){$(this).addClass(\"mdl-card mdl-shadow--2dp\")}),$(\".mx-card .mx-card-title\").each(function(){$(this).addClass(\"mdl-card__title\")}),$(\".mx-card .mx-card-text\").each(function(){$(this).addClass(\"mdl-card__supporting-text\")}),$(\".mx-card-link\").each(function(){$(this).hide()}),$(\".mdl-card\").each(function(){$(this).click(function(){var a=$(this).find(\".mx-card-link\").text();return a&&(window.location=a),!0})}),$(\"a.download\").each(function(){var a=document.createElement(\"button\");a.className=\"download mdl-button mdl-js-button mdl-button--fab mdl-js-ripple-effect\";var t=document.createElement(\"i\");t.className=\"material-icons\";var e=document.createTextNode(\"file_download\");t.appendChild(e),a.appendChild(t);var n=$(this).attr(\"href\");a.onclick=function(){window.location=n};var l=n.split(\"/\").slice(-1).pop();a.id=l?l.replace(\".\",\"-\"):\"download-button-\"+$(this).index();var o=document.createElement(\"div\");o.className=\"mdl-tooltip\",o.setAttribute(\"data-mdl-for\",a.id);var d=$(this).find(\"span.pre\").map(function(){return $(this).text()}).get().join(\" \");o.innerHTML=d,componentHandler.upgradeElement(a),$(this).remove();var s=$(\".section h1\").first();s.append(a),s.append(o)}),$(\".mdl-layout\").css(\"visibility\",\"visible\");!function(){var a,t=0,e=$(\"main.mdl-layout__content\");e.focus();var n=$(\"header.mdl-layout__header\"),l=n.height();e.scroll(function(){t=e.scrollTop(),a<t&&t>l?n.addClass(\"scrollUp\"):a>t&&!(t<=l)&&n.removeClass(\"scrollUp\"),a=t})}()});\n},{\"../scss/sphinx_materialdesign_theme.scss\":\"BS4D\",\"./feedback\":\"dMzA\",\"material-design-lite\":\"vKy7\",\"babel-polyfill\":\"zUFY\",\"./scrollspy\":\"sKvN\"}]},{},[\"brfV\"], null)\n//# sourceMappingURL=/sphinx_materialdesign_theme.js.map"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/mxtheme/theme.conf",
    "content": "[theme]\ninherit = basic\nhtml5_doctype = true\npygments_style = friendly\n# stylesheet = material-icons.css\n\n[options]\nheader_links =\nrelative_url = /\nprimary_color = blue\naccent_color = deep_orange\n\nfixed_drawer = True\nfixed_header = True\nheader_waterfall = True\nheader_scroll = False\n\nshow_header_title = False\nshow_drawer_title = True\nshow_footer = True\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/setup.py",
    "content": "from setuptools import setup\r\nfrom mxtheme import __version__\r\n\r\nsetup(\r\n    name = 'mxtheme',\r\n    version = __version__,\r\n    author = 'Mu Li',\r\n    author_email= '',\r\n    url=\"https://github.com/mli/mx-theme\",\r\n    description='A Sphinx theme based on Material Design, adapted from sphinx_materialdesign_theme',\r\n    packages = ['mxtheme'],\r\n    include_package_data=True,\r\n    license= 'MIT License',\r\n    entry_points = {\r\n        'sphinx.html_themes': [\r\n            'mxtheme = mxtheme',\r\n        ]\r\n    },\r\n)\r\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/js/adjust-height.js",
    "content": "export default class AdjustHeight {\n    \n    constructor() {\n        this.header = $('header.mdl-layout__header');\n        this.pagenation = $('div.pagenation');\n        this.footer = $('footer.mdl-mini-footer');\n        this.win = $(window);\n        this.scrollElement = $('main');\n        this.content = $('.page-content');\n        this.outline = $('.side-doc-outline--content');\n\n        this.attachEvent();\n        this.adjust();\n    }\n\n    adjust() {\n        this.setPageContentMinHeight();\n        this.setLocaltocHeight();\n    }\n\n    setPageContentMinHeight() {\n        const winH = this.win.innerHeight();\n        const headerHeight = this.header.outerHeight();\n        const footerHeight = this.footer.outerHeight(true) + this.pagenation.outerHeight(true);\n\n        this.content.css('min-height', this.win.innerHeight() - headerHeight - footerHeight);\n    }\n\n    setLocaltocHeight() {\n        const pagenationPotisionTop = this.pagenation.position().top + parseInt(this.pagenation.css('margin-top'), 10);\n        const outlineBottom = this.scrollElement.scrollTop() + this.content.outerHeight();\n        const winH = this.win.innerHeight() - this.header.outerHeight();\n\n        let min = 0;\n        min = outlineBottom > pagenationPotisionTop ? pagenationPotisionTop : outlineBottom;\n        min = min > winH ? winH : min;\n        this.outline.css('height', min);\n    }\n\n    attachEvent() {\n        let scrollTimer;\n        this.scrollElement.on('scroll', () => {\n            if (scrollTimer) {\n                clearTimeout(scrollTimer);\n            }\n\n            scrollTimer = setTimeout(() => {\n                this.adjust();\n            }, 1);\n        });\n\n        let resizeTimer;\n        this.win.on('resize', () => {\n            if (resizeTimer) {\n                clearTimeout(resizeTimer);\n            }\n\n            resizeTimer = setTimeout(() => {\n                this.adjust();\n            }, 1);\n        });\n    }\n}"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/js/feedback.js",
    "content": "/*!\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n$(document).ready(function() {\n  $(\".feedback-answer\").on(\"click\", function () {\n    $(\".feedback-question\").remove();\n    $(\".feedback-answer-container\").remove();\n    $(\".feedback-thank-you\").show();\n    ga(\"send\", {\n      hitType: \"event\",\n      eventCategory: \"Did this page help you?\",\n      eventAction: $(this).attr(\"data-response\"),\n      eventLabel: window.location.pathname || \"unknown\",\n      eventValue: $(this).attr(\"data-response\") === \"yes\" ? 1 : 0\n    });\n  });\n});\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/js/scrollspy.js",
    "content": "export default class ScrollSpy {\n    constructor(args) {\n\n        this.doc = document;\n        this.nav = this.doc.querySelectorAll(args.navSelector);\n\n        if(!this.nav.length === 0) { return }\n\n        this.win = window;\n        this.winHeight = this.win.innerHeight;\n\n        this.scrollElement = this.doc.querySelector(args.scrollSelector);\n        this.className = args.className;\n        this.offsetTop = args.offsetTop || 0;\n\n        this.contents = [];\n        this.contents = this.getContents(args.contentSelector);\n\n        this.attachEvent();\n    }\n\n    attachEvent() {\n        let scrollTimer;\n        this.scrollElement.addEventListener('scroll', () => {\n            if (scrollTimer) {\n                clearTimeout(scrollTimer);\n            }\n\n            scrollTimer = setTimeout(() => {\n                this.spy();\n            }, 1);\n        });\n\n        let resizeTimer;\n        this.scrollElement.addEventListener('resize', () => {\n            if (resizeTimer) {\n                clearTimeout(resizeTimer);\n            }\n\n            resizeTimer = setTimeout(() => {\n                this.spy();\n            }, 1);\n        });\n\n        this.scrollElement.addEventListener(\"click\", (e) => {\n            const target = e.target;\n            if (target.tagName !== \"A\") return;\n            window.onclickToc = true;\n            for (let i = 0, max = this.nav.length; i < max; i++) {\n                const navElement = this.nav[i];\n                if (navElement.href === target.href) {\n                    navElement.classList.add(this.className);\n                    navElement.classList.add('mdl-color-text--primary');\n                } else {\n                    navElement.classList.remove(this.className);\n                    navElement.classList.remove('mdl-color-text--primary');\n                }\n            }\n        });\n    }\n\n    getContents(contentSelector) {\n        const targets = [];\n        for (let i = 0, max = this.nav.length; i < max; i++) {\n            const href = this.nav[i].href;\n            targets.push(this.doc.getElementById(href.split('#')[1]));\n        }\n        return targets;\n    }\n\n    spy() {\n        let elements = this.getViewState();\n        this.toggleNavClass(elements);\n    }\n\n    getViewState() {\n        const elementListInView = [];\n        for (let i = 0, max = this.contents.length; i < max; i++) {\n            const current = this.contents[i];\n            if (current && this.isView(current)) {\n                elementListInView.push(current);\n            }\n        }\n\n        return elementListInView;\n    }\n\n    isView(element) {\n        const scrollTop = this.scrollElement.scrollTop;\n        const subHeaderRect = document.querySelector(\".mdl-layout__header-row\").getBoundingClientRect();\n        const headerHeight = subHeaderRect.top + subHeaderRect.height;\n        const scrollBottom = scrollTop + window.innerHeight - headerHeight;\n        const rect = element.getBoundingClientRect();\n        const elementTop = rect.top + scrollTop;\n        const elementBottom = elementTop + element.offsetHeight;\n\n        return elementTop < scrollBottom - 30 && elementBottom > scrollTop + headerHeight + 30;\n    }\n\n    toggleNavClass(elements) {\n        if (window.onclickToc) {\n            window.onclickToc = false;\n            return;\n          }\n        let maxDepth = 0;\n        let maxDepthElement = $();\n\n        for (let i = 0, max = elements.length; i < max; i++) {\n            const el = elements[i];\n            const tempDepth = this.getTagDepth(el);\n            if (maxDepth < tempDepth) {\n                maxDepth = tempDepth;\n                maxDepthElement = el;\n            }\n        }\n\n        for (let i = 0, max = this.nav.length; i < max; i++) {\n            const navElement = this.nav[i];\n            if (navElement.href.split('#')[1] === maxDepthElement.id) {\n                navElement.classList.add(this.className);\n                navElement.classList.add('mdl-color-text--primary');\n            } else {\n                navElement.classList.remove(this.className);\n                navElement.classList.remove('mdl-color-text--primary');\n            }\n        }\n    }\n\n    getTagDepth(element) {\n        return parseInt($(element).find('h1,h2,h3,h4,h5,h6').get(0).tagName.split('H')[1]);\n    }\n}\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/js/sphinx_materialdesign_theme.js",
    "content": "import \"../scss/sphinx_materialdesign_theme.scss\";\r\nimport \"./feedback\";\r\nimport \"material-design-lite\";\r\nimport \"babel-polyfill\";\r\nimport ScrollSpy from \"./scrollspy\";\r\n\r\n$(function() {\r\n\r\n    function reconstructionDrawerGlobalToc() {\r\n        const $globaltoc = $('.mdl-layout__drawer nav');\r\n        const $lists = $globaltoc.find('li');\r\n        $.each($lists, function(index, li) {\r\n            const $li = $(li);\r\n            const $linkWrapper = $('<span class=\"link-wrapper\"></span>');\r\n            const $link = $li.children('a');\r\n            $li.append($linkWrapper.append($link));\r\n\r\n            const isCurrent = $li.hasClass('current') && !$link.hasClass('current');\r\n            const $ul = $li.children('ul');\r\n            if ($ul.length) {\r\n                const ulId = `globalnav-${index}`;\r\n                $ul.attr('id', ulId);\r\n                $ul.addClass('collapse');\r\n                const $toggleWrapper = $('<span class=\"nav-toggle\"></span>');\r\n                if (isCurrent) {\r\n                    $ul.addClass('show');\r\n                    $toggleWrapper.addClass('show');\r\n                } else {\r\n                    $ul.hide();\r\n                }\r\n\r\n                $li.append(\r\n                    $linkWrapper.append(\r\n                        $toggleWrapper.append(\r\n                            $(`<a class=\"mdl-button mdl-js-button mdl-button--icon\" data-toggle=\"#${ulId}\"><span style=\"color: #888\"><i class=\"material-icons\">keyboard_arrow_down</i></span></span>`)\r\n                        )\r\n                    )\r\n                ).append($ul);\r\n            }\r\n        });\r\n    }\r\n\r\n    function collapse() {\r\n        $('.mdl-layout__drawer nav .nav-toggle a').click(function() {\r\n            const $toggle = $(this);\r\n            const id = $toggle.attr('data-toggle');\r\n            $(`ul${id}`).toggleClass('show').animate({height: \"toggle\", opacity: \"toggle\"});\r\n            $toggle.parent().toggleClass('show');\r\n        });\r\n    }\r\n\r\n    function styleMdlCodeBlock() {\r\n        $('pre').hover(function() {\r\n            $(this).attr('click-to-copy', 'click to copy...');\r\n        });\r\n        $('pre').click(function(){\r\n            var result = copyClipboard(this);\r\n            if (result) {\r\n                $(this).attr('click-to-copy', 'copied!');\r\n            }\r\n        });\r\n    }\r\n\r\n    function copyClipboard(selector) {\r\n        var body = document.body;\r\n        if(!body) return false;\r\n\r\n        var $target = $(selector);\r\n        if ($target.length === 0) { return false; }\r\n\r\n        var text = $target.text();\r\n        var textarea = document.createElement('textarea');\r\n        textarea.value = text;\r\n        document.body.appendChild(textarea);\r\n        textarea.select();\r\n        var result = document.execCommand('copy');\r\n        document.body.removeChild(textarea);\r\n        return result;\r\n    }\r\n\r\n    function quickSearchClickEvent() {\r\n        const $breadcrumb = $('.breadcrumb');\r\n\r\n        $('#waterfall-exp').focus(() => {\r\n            if ($(window).width() <= 1024) {\r\n                $breadcrumb.hide();\r\n            }\r\n        }).blur(() => {\r\n            if ($(window).width() <= 1024) {\r\n                $breadcrumb.show();\r\n            }\r\n        });\r\n    }\r\n\r\n    // styleMdlCodeBlock();\r\n\r\n    reconstructionDrawerGlobalToc();\r\n    collapse();\r\n    quickSearchClickEvent();\r\n\r\n\r\n    const spy = new ScrollSpy({\r\n        contentSelector: '.page-content .section',\r\n        navSelector: '.localtoc a',\r\n        scrollSelector: 'main' ,\r\n        className: 'current',\r\n        offsetTop: 64});\r\n\r\n    $('.mdl-layout__content').focus();\r\n\r\n    $('.mx-card').each(function(){\r\n        $(this).addClass('mdl-card mdl-shadow--2dp');\r\n    });\r\n    $('.mx-card .mx-card-title').each(function(){\r\n        $(this).addClass('mdl-card__title');\r\n    });\r\n    $('.mx-card .mx-card-text').each(function(){\r\n        $(this).addClass('mdl-card__supporting-text');\r\n    });\r\n    $('.mx-card-link').each(function(){\r\n        $(this).hide();\r\n    });\r\n    $('.mdl-card').each(function(){\r\n        $(this).click(function() {\r\n            var url = $(this).find('.mx-card-link').text();\r\n            if (url) {\r\n                window.location = url;\r\n            }\r\n            return true;\r\n        });\r\n    });\r\n\r\n    $('a.download').each(function() {\r\n        // button\r\n        var button = document.createElement('button');\r\n        button.className = 'download mdl-button mdl-js-button mdl-button--fab mdl-js-ripple-effect';\r\n\r\n        // icon\r\n        var icon = document.createElement('i');\r\n        icon.className = 'material-icons';\r\n        var text = document.createTextNode('file_download');\r\n        icon.appendChild(text);\r\n        button.appendChild(icon);\r\n\r\n        // link\r\n        var link = $(this).attr('href');\r\n        button.onclick = function() {\r\n            window.location = link;\r\n        };\r\n        var fileName = link.split(\"/\").slice(-1).pop();\r\n        if (fileName) {\r\n            button.id = fileName.replace('.', '-');\r\n        } else {\r\n            button.id = 'download-button-' + $(this).index();\r\n        }\r\n\r\n        // hint\r\n        var hint = document.createElement('div');\r\n        hint.className = 'mdl-tooltip';\r\n        hint.setAttribute('data-mdl-for', button.id);\r\n        var hintText = $(this).find('span.pre').map(function() {\r\n            return $(this).text();\r\n        }).get().join(' ');\r\n        hint.innerHTML = hintText;\r\n\r\n        componentHandler.upgradeElement(button);\r\n        $(this).remove();\r\n        var header = $('.section h1').first();\r\n        header.append(button);\r\n        header.append(hint);\r\n    });\r\n\r\n    $('.mdl-layout').css('visibility', 'visible');\r\n\r\n    const addScrollAwareHeaderAnimation = function() {\r\n        let preScrollTop, curScrollTop = 0;\r\n        const scrollContent = $(\"main.mdl-layout__content\");\r\n        scrollContent.focus();\r\n        const navBar = $('header.mdl-layout__header');\r\n        const navBarHeight = navBar.height();\r\n    \r\n        scrollContent.scroll(function () {\r\n            curScrollTop = scrollContent.scrollTop();\r\n            if (preScrollTop < curScrollTop && curScrollTop > navBarHeight) {\r\n                navBar.addClass(\"scrollUp\");\r\n            } else if (preScrollTop > curScrollTop && !(curScrollTop <= navBarHeight)) {\r\n                navBar.removeClass(\"scrollUp\");\r\n            }\r\n            preScrollTop = curScrollTop;\r\n        });\r\n    }\r\n    addScrollAwareHeaderAnimation();\r\n});\r\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/_root.scss",
    "content": "html {\n    font-size: $font_size;\n}\n\nbody {\n    display: block !important;\n    background-color: $background_color;\n    font-size: 1rem;\n    line-height: 1.5rem;\n    font-family: $body_font_family;\n}\n\n.mdl-layout__content:focus {\n    outline: none;\n }\n\n.mdl-layout__content header.mdl-layout__drawer {\n    display: none;\n}\n\n.mdl-layout__container {\n    height: calc(100% - 76px);\n    margin-top: 76px;\n}\n.mdl-layout__header {\n    position: fixed;\n    transition: transform 0.5s;\n}\n.mdl-layout--fixed-drawer>.mdl-layout__content {\n    margin-left: 300px;    \n}\n\n@media screen and (max-width: 1024px) {\n    .mdl-layout--fixed-drawer>.mdl-layout__content {\n        margin-left:0\n    }\n}\n\nh1, h2, h3, h4, h5, h6, blockquote, span.mdl-layout-title,\na.download > code.download {\n    font-family: $body_font_family;\n}\n\nh1, h2, h3, h4, h5, h6, .toc-backref, .contents, .toctree-wrapper, .contents a, .toctree-wrapper a, .globaltoc a.current {\n    color: $color-mxnet !important;\n}\n\na {\n    text-decoration: none;\n}\n\n.page-content {\n    font-size: 1rem;\n    p, ul, ol, dl, dd, dt, table, th, td {\n        font-size: 1rem;\n    }\n}\n\n.brand {\n    color: inherit;\n    text-decoration: none;\n}\n\n.section {\n    overflow-x: auto;\n}\n\n\n/*\n *  Figure Directive Styles\n */\n img {\n    max-width: 100%;\n    display: block;\n    margin-left: auto;\n    margin-right: auto;\n }\n\ndiv.figure {\n    p.caption {\n        text-align: center;\n        margin-top: .75rem;\n\n        span.caption-number {\n            font-style: normal;\n        }\n        .caption-number::after {\n            content: \"\\00a0\";\n        }\n    }\n}\n\n.svg-icon {\n  width: 16px;\n  height: 16px;\n  display: inline-block;\n  fill: $grey-color-light;\n  padding-right: 5px;\n  padding-top: 4px;\n  vertical-align: text-top;\n}\n\n/*\n * Download Link Styles\n */\na.download > i.material-icons {\n    position: relative;\n    top: 5px;\n}\n\na.download {\n    text-decoration: none;\n}\n\n%clearfix:after {\n  content: \"\";\n  display: table;\n  clear: both;\n}\n\n.wrapper {\n  max-width: -webkit-calc(#{$content-width} - (#{$spacing-unit} * 2));\n  max-width: calc(#{$content-width} - (#{$spacing-unit} * 2));\n  margin-right: auto;\n  margin-left: auto;\n  padding-right: calc(#{$spacing-unit}+15px);\n  padding-left: $spacing-unit;\n  @extend %clearfix;\n\n  @media screen and (max-width: $on-laptop) {\n    max-width: -webkit-calc(#{$content-width} - (#{$spacing-unit}));\n    max-width: calc(#{$content-width} - (#{$spacing-unit}));\n    padding-right: $spacing-unit / 2;\n    padding-left: $spacing-unit / 2;\n  }\n}\n\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/_variables.scss",
    "content": "/*\nVariables\n*/\n$font_size: 16px;\n\n$background_color: #fafafa;\n$code_background: rgba(0,0,0,.05);\n\n$code_font_family: \"Menlo\", \"DejaVu Sans Mono\", \"Liberation Mono\", \"Consolas\", \"Ubuntu Mono\", \"Courier New\", \"andale mono\", \"lucida console\", monospace !default;\n$body_font_family: -apple-system,BlinkMacSystemFont,\"Segoe UI\",Roboto,\"Helvetica Neue\",Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\",\"Segoe UI Symbol\" !default;\n$base-font-size:   17px !default;\n\n$xl-breakpoint: 1795px;\n$lg-breakpoint: 1200px;\n$md-breakpoint: 992px;\n$sm-breakpoint: 768px;\n$xs-breakpoint: 576px;\n\n$color-primary: $palette-blue-500;\n$color-primary-dark: $palette-blue-700 !default;\n$color-accent: $palette-deep-orange-A200 !default;\n$color-primary-contrast: $color-white !default;\n$color-accent-contrast: $color-white !default;\n\n\n$base-line-height: 1.5 !default;\n$spacing-unit:     30px !default;\n\n$color-mxnet: rgb(4,140,204);\n$color-mxnet-dark: rgb(4,60,110);\n$grey-color:       #828282 !default;\n$grey-color-light: lighten($grey-color, 45%) !default;\n$grey-color-dark:  darken($grey-color, 25%) !default;\n\n$table-text-align: left !default;\n\n// Width of the content area\n$content-width:    1150px !default;\n\n$on-palm:          600px !default;\n$on-palm:          900px !default;\n$on-laptop:        1024px !default;"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/admonitions/_admonitions.scss",
    "content": "\n/*\n * Admonition Styles\n */\n $admonitions: (\n    hint: (\n        font-color: rgb(0, 188, 212),\n        background-color: rgba(0, 188, 212, 0.1),\n        icon-content: \"help_outline\"\n    ),\n    note: (\n        font-color: rgb(0, 188, 212),\n        background-color: rgba(0, 188, 212, 0.1),\n        icon-content: \"info_outline\"\n    ),\n    seealso: (\n        font-color: rgb(0, 188, 212),\n        background-color: rgba(0, 188, 212, 0.1),\n        icon-content: \"search\"\n    ),\n    warning: (\n        font-color: rgb(255, 193, 7),\n        background-color: rgba(255, 193, 7, 0.1),\n        icon-content: \"warning\"\n    ),\n    attention: (\n        font-color: rgb(255, 193, 7),\n        background-color: rgba(255, 193, 7, 0.1),\n        icon-content: \"warning\"\n    ),\n    tip: (\n        font-color: rgb(139, 195, 74),\n        background-color: rgba(139, 195, 74, 0.1),\n        icon-content: \"lightbulb_outline\"\n    ),\n    important: (\n        font-color: rgb(139, 195, 74),\n        background-color: rgba(139, 195, 74, 0.1),\n        icon-content:  \"check_circle\"\n    ),\n    error: (\n        font-color: rgb(244, 67, 54),\n        background-color: rgba(244, 67, 54, 0.1),\n        icon-content: \"error_outline\"\n    ),\n    caution: (\n        font-color: rgb(244, 67, 54),\n        background-color: rgba(244, 67, 54, 0.1),\n        icon-content: \"error_outline\"\n    ),\n    danger: (\n        font-color: rgb(244, 67, 54),\n        background-color: rgba(244, 67, 54, 0.1),\n        icon-content: \"error_outline\"\n    )\n);\n\n @mixin admonition-style($type) {\n    border-left: solid 4px map-get(map-get($admonitions, $type), font-color);\n    background-color: map-get(map-get($admonitions, $type), background-color);\n    .admonition-title {\n        font-size: 16px;\n        font-weight: bold;\n        color: map-get(map-get($admonitions, $type), font-color);\n\n        margin-top: 4px;\n        margin-bottom: 8px;\n        &::before {\n            @extend .material-icons;\n            position: relative;\n            margin-right: 5px;\n            top: 3px;\n            content: map-get(map-get($admonitions, $type), icon-content);\n            font-size: 18px;\n        }\n    }\n}\n\n.admonition {\n    @extend .mdl-shadow--2dp;\n\n    padding: 12px 20px;\n    margin-top: 10px;\n    margin-bottom: 10px;\n    p.last {\n        margin: 16px;\n    }\n    .admonition-title {\n        font-size: 16px;\n        font-weight: bold;\n        color: #555;\n        text-transform: uppercase;\n        margin-top: 7px;\n    }\n\n    @each $type in (note, seealso, hint, warning, attention, tip, important, error, caution, danger) {\n        &.#{$type} {\n            @include admonition-style($type);\n        }\n    }\n}\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/blockquote/_blockquote.scss",
    "content": "/*\n * Quotation Block Styles\n */\n .page-content {\n    blockquote {\n        font-size: 1rem;\n        padding: 0 1rem;\n        border-left: 3px solid $code_background;\n\n        &:after {\n            content: \"\" !important;\n            margin-left: 0;\n        }\n        &:before {\n            content: \"\" !important;\n        }\n    }\n }\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/card/_card.scss",
    "content": ".mdl-card {\n    margin: 1em 1.5em 1em 0;\n    display: inline-block;\n    width: 250px;\n    min-height: 140px;\n    padding: 18px;\n}\n.mdl-card:hover {\n    box-shadow: 0 10px 20px rgba(0,0,0,0.25), 0 6px 6px rgba(0,0,0,0.22);\n    color: #000;\n    cursor: pointer;\n}\n.mdl-card__title {\n    padding: 0 0 1em 0;\n    font-size: 18px;\n    color: #444;\n}\n\n.mdl-card__supporting-text {\n    line-height: 1.5rem;\n    padding: 0px;\n    width: 100%;\n}\n\n.head-card.mdl-card {\n    width: auto;\n    display: block;\n    max-width: 800px;\n    padding: 24px;\n}\n\n.head-card > .mdl-card__title {\n    padding-bottom: 0px;\n    height: 60px;\n    font-weight: 700;\n    text-transform: uppercase;\n}\n.head-card > .mdl-card__menu {\n    color: #fff;\n}\n.head-card > .mdl-card__actions {\n    padding: 0;\n}\n.cards {\n    display: flex;\n    flex-direction: row;\n    flex-wrap: wrap;\n}\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/code/_code.scss",
    "content": ".page-content {\n    .highlight {\n        margin: 1px 0;\n        pre {\n            background: $code_background;\n            color: rgba(0,0,0,.87);\n            font-family: $code_font_family;\n            padding: 0.75rem;\n            overflow: auto;\n            overflow-y: hidden;\n            .o, .nd {\n                color: rgba(0,0,0,.87);\n            }\n        }\n    }\n\n    div.highlight-console div.highlight {\n        background: none;\n    }\n\n    // for jupyter notebook output cell\n    .output {\n        .highlight {\n            pre {\n                color: rgba(0,0,0,.87);\n                background: $background_color;\n                border-width: 1px;\n                border-color: #999;\n                border-style: solid;\n                padding: 0.75rem;\n            }\n        }\n     }\n\n    .code, code:not(.download) {\n        margin: 0 0;\n        font-family: $code_font_family;\n        border-radius: 2px;\n        span.pre {\n            font-family: $code_font_family;\n        }\n    }\n\n    .viewcode-link {\n        padding-left: 2em;\n        font-size: 80%;\n    }\n\n    .rubric, .method > dt, .function > dt, .class > dt {\n        display: table;\n        margin: 10px 0;\n        font-size: 100%;\n        line-height: normal;\n        background: #e7f2fa;\n        color: #2B98F0;\n        border-top: solid 3px #55ADF3;\n        padding: 10px;\n        position: relative;\n        .descname, .descclassname {\n            color: rgba(0,0,0,.87);\n            background: #e7f2fa;\n            padding: 3px;\n        }\n        em {\n            padding: 0 2px;\n        }\n    }\n\n\n    .rubric {\n        margin: 30px 0 10px 0;\n     }\n\n\n    .field-body {\n        padding-left: 40px;\n        ul {\n            padding: 0 0 0 16px;\n            margin: 0;\n        }\n    }\n\n     // .docutils > dt {\n    //     padding: 6px;\n    //     display: table;\n    //     margin-bottom: 6px;\n    //     border: none;\n    //     border-left: solid 3px #ccc;\n    //     background: #f0f0f0;\n    //     color: #555;\n    // }\n\n    .seealso .docutils > dt {\n       float: left;\n       clear: left;\n       padding: 0 6px;\n     }\n\n    .seealso .docutils > dd {\n       padding-left: 6em;\n    }\n    .nblast {\n        padding-bottom: 1em;\n    }\n\n    pre {\n        font-size: 90%;\n        background: #eee;\n        color: #455A64;\n        padding: 16px 32px;\n        width: auto;\n        border-radius: 4px;\n        word-wrap: break-word;\n\n        &:hover {\n            @extend .mdl-shadow--2dp;\n\n            &:before {\n                font-family: $body_font_family;\n                padding: 0 0.5rem;\n                content: attr(click-to-copy);\n                color: rgba(0, 0, 0, 0.5);\n                border-radius: 4px;\n                position: relative;\n                float: right;\n                top: -0.5rem;\n                right: -0.5rem;\n                background: rgb(200, 200, 200);\n                font-size: 0.8rem;\n                cursor: pointer;\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/downloadlink/_downloadlink.scss",
    "content": "a.download {\n    &:before {\n        @extend .material-icons;\n        content: \"file_download\";\n        position: relative;\n        top: 5px;\n        margin-right: 5px;\n    }\n}\n\nbutton.download {\n    position: sticky;\n    margin-left: 1em;\n}\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/drawer/_drawer.scss",
    "content": "/*\r\n * Drawer Styles\r\n */\r\n.mdl-layout {\r\n    &__drawer {\r\n        background-color: #fff;\r\n\r\n        &::-webkit-scrollbar {\r\n            width: 6px;\r\n        }\r\n\r\n        &::-webkit-scrollbar-track {\r\n            border-radius: 6px;\r\n        }\r\n\r\n        &::-webkit-scrollbar-thumb {\r\n            background-color: rgba(0, 0, 0, .3);\r\n            border-radius: 6px;\r\n            box-shadow:0 0 0 1px rgba(255, 255, 255, .3);\r\n        }\r\n\r\n        > .mdl-layout-title {\r\n            font-weight: bold;\r\n            text-align: right;\r\n            margin: 0;\r\n            padding: 0;\r\n            line-height: 32px;\r\n            border-bottom: 1px solid rgba(0,0,0,.1);\r\n            min-height: 64px;\r\n            .title {\r\n                color: inherit;\r\n                display: block;\r\n                height: 100%;\r\n                width: 100%;\r\n                text-decoration: none;\r\n                > img.logo {\r\n                    width: 100%;\r\n                    margin: 0;\r\n                    padding: 0;\r\n                }\r\n\r\n                &-text {\r\n                    font-weight: bold;\r\n                    text-align: right;\r\n                    padding: 0 10px;\r\n                    margin: 16px 0 8px 0;\r\n                    line-height: 32px;\r\n                    font-family: $body_font_family;\r\n                    color: inherit;\r\n                    display: block;\r\n                }\r\n            }\r\n        }\r\n    }\r\n}\r\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/fonts/_material-icons.scss",
    "content": "\n/*\nMaterial Icons\n*/\n\n.material-icons {\n    font-family: 'Material Icons';\n    font-weight: normal;\n    font-style: normal;\n    font-size: 24px;  /* Preferred icon size */\n    display: inline-block;\n    line-height: 1;\n    text-transform: none;\n    letter-spacing: normal;\n    word-wrap: normal;\n    white-space: nowrap;\n    direction: ltr;\n  \n    /* Support for all WebKit browsers. */\n    -webkit-font-smoothing: antialiased;\n    /* Support for Safari and Chrome. */\n    text-rendering: optimizeLegibility;\n  \n    /* Support for Firefox. */\n    -moz-osx-font-smoothing: grayscale;\n  \n    /* Support for IE. */\n    font-feature-settings: 'liga';\n  }"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/footer/_footer.scss",
    "content": "/*\r\n * Footer Styles\r\n */\r\nfooter.mdl-mini-footer {\r\n    background-color: #212121;\r\n    > div.mdl-mini-footer__left-section {\r\n        margin-bottom: 20px;\r\n        display: flex;\r\n        flex-direction: column;\r\n        .mdl-logo {\r\n            font-size: 1.1rem;\r\n        }\r\n        ul {\r\n            @extend .mdl-mini-footer__link-list;\r\n        }\r\n    }\r\n    > div.mdl-mini-footer__right-section {\r\n        font-size: 0.9rem;\r\n        display: flex;\r\n        flex-direction: column;\r\n        justify-content: flex-end;\r\n\r\n        a {\r\n            color: inherit;\r\n            font-weight: bold;\r\n            text-decoration: none;\r\n        }\r\n    }\r\n    p.caption {\r\n        display: none;\r\n    }\r\n}\r\n\r\n/*\r\n * Pagenation Block Styles\r\n */\r\n .pagenation {\r\n    width: 100%;\r\n    margin-top: 80px;\r\n    height: 92px;\r\n    background-color: #424242;\r\n    display: flex;\r\n\r\n    .button-common {\r\n        text-transform: none;\r\n        padding: 0;\r\n        height: 92px;\r\n        display: flex;\r\n        justify-content: center;\r\n        align-items: center;\r\n        color: #ffffff;\r\n    }\r\n    #button-prev {\r\n        @extend .button-common;\r\n        margin-right: auto;\r\n        .pagenation-text {\r\n            text-align: left;\r\n        }\r\n        \r\n    }\r\n    #button-next {\r\n        @extend .button-common;\r\n        margin-left: auto;\r\n        flex-direction: row-reverse;\r\n        .pagenation-text {\r\n            text-align: right;\r\n        }\r\n    }\r\n\r\n    &-arrow {\r\n        &-L {\r\n            margin-right: 20px;\r\n        }\r\n        &-R {\r\n            margin-left: 20px;\r\n        }\r\n    }\r\n\r\n    &-text {\r\n        line-height: 30px;\r\n        font-size: 20px;\r\n    }\r\n\r\n    &-direction {\r\n        opacity: 0.7;\r\n        font-size: 18px;\r\n    }\r\n    @media screen and (max-width: 1024px) {\r\n        #button-prev {\r\n            width: 20%;\r\n        }\r\n        \r\n        #button-next {\r\n            width: 80%;\r\n        }\r\n    \r\n        #button-prev .pagenation-text {\r\n            display: none;\r\n        }\r\n    }\r\n    @media screen and (min-width: 1025px) {\r\n        #button-prev,\r\n        #button-next {\r\n            width: 50%;\r\n        }\r\n    \r\n        #button-prev .pagenation-text {\r\n            display: block;\r\n        }\r\n    }\r\n}\r\n\r\n\r\n\r\n/**\r\n * Site footer\r\n */\r\n.site-footer {\r\n  border-top: 1px solid $grey-color-light;\r\n  padding: $spacing-unit 0;\r\n  background-color: #424242;\r\n  position: relative;\r\n  z-index: 10;\r\n  .footer-category-title {\r\n    color: $color-mxnet;\r\n  }\r\n  a {\r\n    color: $grey-color-light !important;\r\n\r\n    &:visited {\r\n      color: $grey-color-light !important;\r\n    }\r\n  }\r\n\r\n}\r\n\r\n.site-footer2 {\r\n  background-color: #424242;\r\n  padding-top: 40px;\r\n  padding-bottom: 10px;\r\n  position: relative;\r\n  z-index: 10;\r\n}\r\n\r\n.footer-heading {\r\n  margin-bottom: $spacing-unit / 2;\r\n}\r\n\r\n.contact-list,\r\n.apache-list,\r\n.social-media-list {\r\n  list-style: none;\r\n  margin-left: 0;\r\n}\r\n\r\n\r\n.footer-bottom-warning {\r\n  font-size: 80%;\r\n  color: white;\r\n  float: left;\r\n}\r\n\r\n.footer-logo {\r\n  width: 200px;\r\n  margin-bottom: 30px;\r\n  margin-top: 30px;\r\n}\r\n\r\n.footer-col {\r\n  float: left;\r\n  margin-bottom: $spacing-unit / 2;\r\n  padding-left: $spacing-unit / 2;\r\n}\r\n\r\n.footer-text {\r\n  color: $grey-color-light;\r\n}\r\n\r\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/grid/_simplegrid.scss",
    "content": "// SIMPLE GRID - SASS/SCSS\n\n\n// fonts\n$font-weight-light: 300;\n$font-weight-regular: 400;\n$font-weight-heavy: 700;\n\n// colors\n$dark-grey: #333447;\n$dark-gray: #333447; // for the Americans\n\n\n.font-light {\n  font-weight: $font-weight-light;\n}\n\n.font-regular {\n  font-weight: $font-weight-regular;\n}\n\n.font-heavy {\n  font-weight: $font-weight-heavy;\n}\n\n// utility\n\n.left {\n  text-align: left;\n}\n\n.right {\n  text-align: right;\n}\n\n.center {\n  text-align: center;\n  margin-left: auto;\n  margin-right: auto;\n}\n\n.justify {\n  text-align: justify;\n}\n\n.hidden-sm {\n  display: none;\n}\n\n// grid\n\n$width: 98%;\n$gutter: 2%;\n$breakpoint-small: 33.75em; // 540px\n$breakpoint-med: 45em; // 720px\n$breakpoint-large: 60em; // 960px\n\n.container {\n  width: 100%;\n  margin-left: auto;\n  margin-right: auto;\n}\n\n.row {\n  position: relative;\n  width: 100%;\n}\n\n.row [class^=\"col\"] {\n  float: left;\n  margin: 0.5rem 1%;\n  min-height: 0.125rem;\n}\n\n.row::after {\n  content: \"\";\n  display: table;\n  clear: both;\n}\n\n.col-1,\n.col-2,\n.col-3,\n.col-4,\n.col-5,\n.col-6,\n.col-7,\n.col-8,\n.col-9,\n.col-10,\n.col-11,\n.col-12 {\n  width: $width;\n}\n\n.col-1-sm {\n  width: ($width / 12) - ($gutter * 11 / 12);\n}\n\n.col-2-sm {\n  width: ($width / 6) - ($gutter * 10 / 12);\n}\n\n.col-3-sm {\n  width: ($width / 4) - ($gutter * 9 / 12);\n}\n\n.col-4-sm {\n  width: ($width / 3) - ($gutter * 8 / 12);\n}\n\n.col-5-sm {\n  width: ($width / (12 / 5)) - ($gutter * 7 / 12);\n}\n\n.col-6-sm {\n  width: ($width / 2) - ($gutter * 6 / 12);\n}\n\n.col-7-sm {\n  width: ($width / (12 / 7)) - ($gutter * 5 / 12);\n}\n\n.col-8-sm {\n  width: ($width / (12 / 8)) - ($gutter * 4 / 12);\n}\n\n.col-9-sm {\n  width: ($width / (12 / 9)) - ($gutter * 3 / 12);\n}\n\n.col-10-sm {\n  width: ($width / (12 / 10)) - ($gutter * 2 / 12);\n}\n\n.col-11-sm {\n  width: ($width / (12 / 11)) - ($gutter * 1 / 12);\n}\n\n.col-12-sm {\n  width: $width;\n}\n\n@media only screen and (min-width: $breakpoint-med) {\n  .col-1 {\n    width: ($width / 12) - ($gutter * 11 / 12);\n  }\n  .col-2 {\n    width: ($width / 6) - ($gutter * 10 / 12);\n  }\n  .col-3 {\n    width: ($width / 4) - ($gutter * 9 / 12);\n  }\n  .col-4 {\n    width: ($width / 3) - ($gutter * 8 / 12);\n  }\n  .col-5 {\n    width: ($width / (12 / 5)) - ($gutter * 7 / 12);\n  }\n  .col-6 {\n    width: ($width / 2) - ($gutter * 6 / 12);\n  }\n  .col-7 {\n    width: ($width / (12 / 7)) - ($gutter * 5 / 12);\n  }\n  .col-8 {\n    width: ($width / (12 / 8)) - ($gutter * 4 / 12);\n  }\n  .col-9 {\n    width: ($width / (12 / 9)) - ($gutter * 3 / 12);\n  }\n  .col-10 {\n    width: ($width / (12 / 10)) - ($gutter * 2 / 12);\n  }\n  .col-11 {\n    width: ($width / (12 / 11)) - ($gutter * 1 / 12);\n  }\n  .col-12 {\n    width: $width;\n  }\n\n  .hidden-sm {\n    display: block;\n  }\n}\n\n.row {\n  display: -webkit-box;\n  display: -webkit-flex;\n  display: -ms-flexbox;\n  display: flex;\n  flex-wrap: wrap;\n}\n\n.row > [class*='col-'] {\n  display: flex;\n  flex-direction: column;\n}\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/header/_header.scss",
    "content": "/*\r\n * Header Styles\r\n */\r\n\r\nnav.breadcrumb {\r\n    > a.mdl-navigation__link {\r\n        padding: 0 8px;\r\n        font-size: 18px;\r\n    }\r\n    @media (max-width: $lg-breakpoint - 1) {\r\n        width: calc( 100% - 64px );\r\n        a.mdl-navigation__link.is-active {\r\n            overflow-x: hidden;\r\n            width: 100%;\r\n            overflow: hidden;\r\n            white-space: nowrap;\r\n            text-overflow: ellipsis;\r\n        }\r\n        a.mdl-navigation__link:not(.is-active),\r\n        i.material-icons {\r\n            display: none;\r\n        }\r\n    }\r\n}\r\n\r\ndiv.mdl-layout__header {\r\n    margin-top: 77px;\r\n}\r\n\r\n.mdl-layout__drawer-button {\r\n    top: 13px !important;\r\n}\r\n\r\ndiv.mdl-layout__header-row.header-links {\r\n    background: rgba(255,255,255,0.2);\r\n    width: 100%;\r\n    overflow-x: auto;\r\n    overflow-y: hidden;\r\n\r\n    a.mdl-navigation__link  {\r\n        font-size: 1rem;\r\n        i {\r\n            font-size: 1.2rem;\r\n            margin: 0 8px;\r\n            position: relative;\r\n            bottom: -0.1rem;\r\n        }\r\n    };\r\n\r\n    a.mdl-navigation__link:hover  {\r\n        background-color: unquote(\"rgb(#{$color-primary})\");\r\n        color: #eeeeee;\r\n    };\r\n    a.mdl-navigation__link[href=\"#\"]  {\r\n        background-color: unquote(\"rgb(#{$color-primary})\");\r\n        opacity: 1;\r\n        color: #ffffff;\r\n    };\r\n}\r\n\r\n/* mxnet-header */\r\n\r\n\r\n.site-title {\r\n  font-weight: 300 !important;\r\n  line-height: 57px;\r\n  letter-spacing: -1px;\r\n  margin-bottom: 0;\r\n  float: left;\r\n  color: white;\r\n\r\n  &,\r\n  &:visited {\r\n    color: $grey-color-dark;\r\n  }\r\n}\r\n\r\n\r\n.site-header {\r\n  position: fixed;\r\n  top: 0;\r\n  width: 100%;\r\n  min-height: 55px;\r\n  padding-top: 10px;\r\n  padding-bottom: 10px;\r\n  background-color: $color-mxnet;\r\n  z-index: 10;\r\n  font-weight: 300;\r\n  font-size: 17px;\r\n  border-bottom: 1px solid white;\r\n}\r\n\r\n.site-header-logo {\r\n  width: 120px;\r\n  display: initial;\r\n}\r\n\r\n.site-nav {\r\n  float: right;\r\n  line-height: 57px;\r\n\r\n  .nav-trigger {\r\n    display: none;\r\n  }\r\n\r\n  .menu-icon {\r\n    display: none;\r\n  }\r\n\r\n  .page-link {\r\n    color: white;\r\n    line-height: 1.5;\r\n    font-weight: 300;\r\n    // Gaps between nav items, but not on the last one\r\n    &:not(:last-child) {\r\n      margin-right: 40px;\r\n    }\r\n\r\n    &:hover {\r\n      color: white;\r\n      text-shadow: -0.06ex 0 white, 0.06ex 0 white;\r\n    }\r\n  }\r\n\r\n  .page-link.page-current {\r\n    color: white;\r\n    text-decoration: underline;\r\n  }\r\n\r\n  @media screen and (max-width: $on-laptop) {\r\n    position: absolute;\r\n    top: 9px;\r\n    right: 15px;\r\n    background-color: rgb(23,141,201);\r\n    border-radius: 2px;\r\n    text-align: right;\r\n\r\n    label[for=\"nav-trigger\"] {\r\n      display: block;\r\n      float: right;\r\n      width: 36px;\r\n      height: 36px;\r\n      z-index: 2;\r\n      cursor: pointer;\r\n    }\r\n\r\n    .menu-icon {\r\n      display: block;\r\n      float: right;\r\n      width: 36px;\r\n      height: 26px;\r\n      line-height: 0;\r\n      padding-top: 20px;\r\n      text-align: center;\r\n\r\n      > svg {\r\n        fill: white;\r\n      }\r\n    }\r\n\r\n    input ~ .trigger {\r\n      clear: both;\r\n      display: none;\r\n    }\r\n\r\n    input:checked ~ .trigger {\r\n      display: block;\r\n      padding-bottom: 5px;\r\n    }\r\n\r\n    .page-link {\r\n      padding: 5px 10px;\r\n      display: block;\r\n\r\n      &:not(:last-child) {\r\n        margin-right: 0;\r\n      }\r\n\r\n      margin-left: 20px;\r\n    }\r\n  }\r\n}"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/headerings/_headerings.scss",
    "content": "@keyframes float-in {\n    0% {\n        transform: translateY(0.5rem);\n        opacity: 0;\n    }\n\t100% {\n\t\ttransform: translateY(0);\n\t\topacity: 1;\n\t}\n}\n\n@keyframes float-out {\n    0% {\n        transform: translateY(0);\n        opacity: 1;\n    }\n\t100% {\n\t\ttransform: translateY(0.5rem);\n\t\topacity: 0;\n\t}\n}\n\n.page-content {\n    .headerlink {\n        display: inline-block;\n        text-decoration: none;\n        margin-left: 0.8rem;\n        color: inherit;\n        opacity: 0;\n        &:hover {\n            animation: float-in 0.2s $animation-curve-fast-out-slow-in 0s forwards;\n        }\n    }\n\n    h1, h2, h3, h4, h5, h6 {\n        .toc-backref {\n            text-decoration: none;\n        }\n        &:hover {\n            .headerlink {\n                animation: float-in 0.2s $animation-curve-fast-out-slow-in 0s forwards;\n            }\n        }\n    }\n\n    h1 {\n        font-size: 2rem;\n        line-height: 2.25rem;\n    }\n\n    h2 {\n        font-size: 1.75rem;\n        line-height: 2rem;\n        padding-top: 1.5rem;\n        margin-top: 0;\n        margin-bottom: 1rem;\n    }\n\n    h3 {\n        font-size: 1.5rem;\n        line-height: 1.75rem;\n        padding-top: 1rem;\n        margin-top: 0px;\n        margin-bottom: .75rem;\n    }\n\n    h4 {\n        font-size: 1.25rem;\n        line-height: 1.5rem;\n        padding-top: .75rem;\n        margin-top: 0px;\n        margin-bottom: .5rem;\n    }\n\n    div.page-content h5 {\n        font-size: 1.1rem;\n        line-height: 1.5rem;\n        padding-top: 2rem;\n        margin-top: 0px;\n        margin-bottom: 1rem;\n    }\n\n    div.page-content h6 {\n        font-size: 1rem;\n        line-height: 1.5rem;\n        padding-top: 2rem;\n        margin-top: 0px;\n        margin-bottom: 1rem;\n    }\n\n\n}\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/layout/_layout.scss",
    "content": "/**\n * Layout Styles\n */\n $layout: (\n    document: (\n        xl: (\n            width: 100%,\n        )\n    ),\n    drawer-container: (\n        width: $layout-drawer-width,\n    ),\n    side-doc-outline: (\n        width: 230px,\n    ),\n    page-content: (\n        md: (\n            width: 90%,\n            padding: 0 5%\n        ),\n        lg: (\n            width: calc( 90% - 230px ),\n            padding: 0 5%\n        )\n    )\n);\n\n.document {\n    width: 100%;\n    margin: 84px auto;\n    display: flex;\n\n    @media (min-width: $xl-breakpoint) {\n        width: map-get(map-get(map-get($layout, document), xl), width);\n    }\n    .page-content {\n        width: 100%;\n        margin: 0 auto;\n        padding: 0 12px;\n\n        @media (min-width: $md-breakpoint) {\n            width: map-get(map-get(map-get($layout, page-content), md), width);\n            padding: map-get(map-get(map-get($layout, page-content), md), padding);\n        }\n\n        @media (min-width: $lg-breakpoint) {\n            width: map-get(map-get(map-get($layout, page-content), lg), width);\n            padding: map-get(map-get(map-get($layout, page-content), lg), padding);\n        }\n    }\n\n    .side-doc-outline {\n        width: map-get(map-get($layout, side-doc-outline), width);\n\n        @media (max-width: $lg-breakpoint - 1) {\n            display: none;\n        } \n        &--content {\n            position: sticky;\n            overflow-x: auto;\n            overflow-y: auto;\n            width: inherit;\n            right: 0px;\n            top: 80px;\n            &::-webkit-scrollbar {\n                width: 6px;\n            }\n    \n            &::-webkit-scrollbar-track {\n                border-radius: 6px;\n            }\n    \n            &::-webkit-scrollbar-thumb {\n                background-color: rgba(0, 0, 0, .3);\n                border-radius: 6px;\n                box-shadow:0 0 0 1px rgba(255, 255, 255, .3);\n            }\n        }\n    }\n\n}"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/lists/_lists.scss",
    "content": ".page-content {\n    ul {\n        li {\n            margin: .3rem 0;\n            p {\n                margin: 0;\n            }\n        }\n    }\n    .option-list {\n        .option {\n            font-family: $code_font_family;\n        }\n        td {\n            padding: 0.5rem;\n            border: none;\n        }\n    }\n}\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/search/_search.scss",
    "content": " /*\r\n  * Search Styles\r\n  */\r\n#waterfall-exp::-webkit-input-placeholder {\r\n    color: #ccc;\r\n}\r\n#waterfall-exp:-ms-input-placeholder {\r\n    color: #ccc;\r\n}\r\n#waterfall-exp::-moz-placeholder {\r\n    color: #ccc;\r\n}\r\n\r\nul.search span.highlighted {\r\n    font-weight: bold;\r\n}\r\n\r\nul.search > li {\r\n    margin-bottom: 24px;\r\n}\r\n\r\n#search-results {\r\n    ul {\r\n        list-style: none;\r\n        padding: 0;\r\n        li {\r\n            > a {\r\n                text-decoration: none;\r\n                font-size: 1.2rem;\r\n            }\r\n        }\r\n    }\r\n}\r\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/sphinx_materialdesign_theme.scss",
    "content": "@import \"./material-design-lite/src/color-definitions\";\n@import \"./variables\";\n\n@import \"./material-design-lite/src/shadow/shadow\";\n@import \"./material-design-lite/src/data-table/data-table\";\n@import \"./material-design-lite/src/footer/mini_footer\";\n@import \"./material-design-lite/src/card/card\";\n@import \"./material-design-lite/src/button/button\";\n\n@import \"./grid/simplegrid\";\n@import \"./fonts/material-icons\";\n@import \"./root\";\n@import \"./layout/layout\";\n@import \"./headerings/headerings\";\n@import \"./admonitions/admonitions\";\n@import \"./code/code\";\n@import \"./blockquote/blockquote\";\n@import \"./tables/tables\";\n@import \"./toc/globaltoc\";\n@import \"./toc/localtoc\";\n@import \"./toc/toctree\";\n@import \"./lists/lists\";\n@import \"./drawer/drawer\";\n@import \"./header/header\";\n@import \"./footer/footer\";\n@import \"./search/search\";\n@import \"./downloadlink/downloadlink\";\n@import \"./card/card\";"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/tables/_tables.scss",
    "content": ".page-content {\n    table:not(.footnote):not(.indextable):not(.hlist):not(.option-list):not(.field-list) {\n        @extend .mdl-data-table;\n        @extend .mdl-shadow--2dp;\n\n        margin: 1.5rem 0;\n        table-layout: fixed;\n        max-width: 100%;\n        min-width: 70%;\n\n        th, td {\n            @extend .mdl-data-table__cell--non-numeric;\n            white-space: normal;\n            overflow-wrap: break-word;\n        }\n\n        caption {\n            font-size: $font_size;\n            margin: 1rem 0 0.8rem 0;\n            white-space: normal;\n            .caption-number {\n                font-style: normal;\n            }\n            .caption-number::after {\n                content: \"\\00a0\";\n            }\n        }\n\n    }\n}\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/toc/_globaltoc.scss",
    "content": ".globaltoc {\n    \n    .caption, .toc {\n        display: none;\n    }\n\n    ul {\n\n        list-style-type: none;\n        padding: 0;\n        margin: 0;\n\n        li {\n            min-height: 18px;\n            .link-wrapper {\n                display: flex;\n                justify-content: space-between;\n                > a {\n                    padding: 4px 0;\n                    display: block;\n                    width: 100%;\n                    font-size: 1rem;\n                    text-decoration: none;\n                    color: $layout-drawer-navigation-color;\n                    &.current {\n                        font-weight: bold;\n                    }\n                }\n            }\n        }\n    }\n\n    .nav-toggle {\n        padding: 0;\n        float: right;\n        display: flex;\n        align-items: center;\n        justify-content: center;\n        height: 36px;\n        > a {\n            padding: 0;\n            margin-left: 0;\n            margin-right: 4px;\n            cursor: pointer;\n            > i {\n                font-size: 18px;\n            }\n        }\n        &.show {\n            transform: rotateZ(180deg);\n            > a {\n                margin-right: 0;\n                margin-left: 4px;\n            }\n        }\n    }\n\n    nav {\n        > ul > li > span.link-wrapper {\n            padding-left: 8px;\n        }\n        > ul > li > ul > li > span.link-wrapper {\n            padding-left: 16px;\n        }\n        > ul > li > ul > li > ul > li > span.link-wrapper {\n            padding-left: 24px;\n        }\n        > ul > li > ul > li > ul > li > ul> li > span.link-wrapper {\n            padding-left: 32px;\n        }\n        > ul > li > ul > li > ul > li > ul > li > ul> li > span.link-wrapper {\n            padding-left: 40px;\n        }\n        > ul > li > ul > li > ul > li > ul > li > ul > li > ul> li > span.link-wrapper {\n            padding-left: 48px;\n        }\n    }\n}\n"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/toc/_localtoc.scss",
    "content": ".localtoc {\n    font-size: 0.75rem;\n    padding-top: 1rem;\n\n    .caption {\n        padding-left: 12px;\n        &-text {\n            font-size: 0.9rem;\n            font-weight: 700;\n        }\n    }\n\n    > ul > li > a {\n        display: none;\n    }\n\n    ul {\n        padding: 0;\n        list-style-type: none;\n    }\n\n    li {\n        padding-left: 6px;\n    }\n\n    a {\n        display: block;\n        text-decoration: none;\n        color: inherit;\n        margin-top: 8px;\n        padding-left: 8px;\n        line-height: 1.1rem;\n    \n        &.current {\n            padding-left: 5px;\n            border-left: 3px solid;\n            font-weight: bold;\n        }\n    }\n}"
  },
  {
    "path": "docs/python_docs/themes/mx-theme/src/scss/toc/_toctree.scss",
    "content": "/*\r\n *  Toctree and Contents Directive Styles\r\n */\r\n .toctree-wrapper,\r\n .contents.topic {\r\n     border-left: 5px solid;\r\n }\r\n\r\n .toctree-wrapper > p.caption,\r\n .contents.topic > p.topic-title {\r\n     color: rgb(117, 117, 117);\r\n     font-size: 1rem;\r\n     padding-left: 14px;\r\n }\r\n\r\n .toctree-wrapper ul,\r\n .contents.topic ul{\r\n     padding-left: 14px;\r\n     list-style: none;\r\n     line-height: 30px;\r\n }\r\n\r\n .toctree-wrapper a,\r\n .contents.topic a {\r\n     font-size: 1.2rem;\r\n     text-decoration: none;\r\n     .pre {\r\n         font-size: 1rem;\r\n     }\r\n }\r\n\r\n .toctree-wrapper > ul > li > a,\r\n .contents.topic > ul > li > a {\r\n     font-size: 1.3rem;\r\n     .pre {\r\n         font-size: 1.1rem;\r\n     }\r\n }\r\n"
  },
  {
    "path": "docs/static_site/.gitignore",
    "content": "_site\n.sass-cache\n.jekyll-metadata\n"
  },
  {
    "path": "docs/static_site/.nojekyll",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n\n#   http://www.apache.org/licenses/LICENSE-2.0\n\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n"
  },
  {
    "path": "docs/static_site/Makefile",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nall: html\n\nhtml:\n\tmkdir -p build\n\twget -O src/assets/js/jquery-3.3.1.min.js https://code.jquery.com/jquery-3.3.1.min.js\n\twget -O src/assets/img/mxnet-icon.png https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/image/mxnet-icon.png\n\twget -O src/assets/docsearch.min.css 'https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.css'\n\twget -O src/assets/js/docsearch.min.js 'https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.js'\n\twget -O src/assets/js/fontawesome.js 'https://use.fontawesome.com/releases/v5.0.12/js/all.js'\n\twget -O src/assets/js/buttons.js 'https://buttons.github.io/buttons.js'\n\twget -O src/assets/js/platform.js 'https://apis.google.com/js/platform.js'\n\tcd src && bundle install && JEKYLL_ENV=production bundle exec jekyll build --config _config_prod.yml -d ../build/html && cd ..\n\twget https://mxnet-website-static-artifacts.s3.us-east-2.amazonaws.com/versions.zip && unzip versions.zip -d build/html\n\tfind build/html/ -type d -name '__MACOSX' -exec rm -rf {} +\n\tfind build/html/ -type f -name '.DS_Store' -exec rm -rf {} +\n\trm versions.zip\n\nclean:\n\trm -rf build\n"
  },
  {
    "path": "docs/static_site/README.md",
    "content": "<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# MXNet.io v2\n\n1. Install Jekyll https://jekyllrb.com/docs/installation/\n\nThis is for hosting the mxnet.io beta website\n\nserve for test:\n```\ncd src && JEKYLL_ENV=development bundle exec jekyll serve\n```\n\nbuild for beta github pages:\n```\ncd src && JEKYLL_ENV=production bundle exec jekyll build --config _config_beta.yml -d ../docs && cd ..\n```\n\n\nbuild for release:\n```\ncd src && JEKYLL_ENV=production bundle exec jekyll build --config _config_prod.yml -d ../release && cd ..\n```\n\ntest:\n\nhttps://thomasdelteil.github.io/mxnet.io-v2/\n"
  },
  {
    "path": "docs/static_site/src/.asf.yaml",
    "content": "publish:\n  whoami: asf-site\n"
  },
  {
    "path": "docs/static_site/src/.gitignore",
    "content": "static_websites\nassets/docsearch.min.css\nassets/js/buttons.js\nassets/js/docsearch.min.js\nassets/js/fontawesome.js\nassets/js/jquery-3.3.1.min.js\nassets/js/platform.js\n"
  },
  {
    "path": "docs/static_site/src/.htaccess",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n\n#   http://www.apache.org/licenses/LICENSE-2.0\n\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n Options -Indexes\nDirectorySlash off\nRewriteEngine on\nRewriteOptions AllowNoSlash\n\n<IfModule mod_expires.c>\n  ExpiresActive on\n\n  # Images\n  ExpiresByType image/png                 \"access plus 7 days\"\n  ExpiresByType image/jpg                 \"access plus 7 days\"\n  ExpiresByType image/jpeg                \"access plus 7 days\"\n  ExpiresByType image/svg+xml             \"access plus 7 days\"\n\n  # CSS, Javascript, HTML\n  ExpiresByType text/css                  \"access plus 1 days\"\n  ExpiresByType application/javascript    \"access plus 1 days\"\n  ExpiresByType text/html                 \"access plus 0 seconds\"\n\n  # Web fonts\n  ExpiresByType application/font-woff     \"access plus 1 month\"\n\n</IfModule>\n\n# Set default website version to old stable (v1.9.1)\nRewriteCond %{REQUEST_URI} !^/versions/\nRewriteCond %{HTTP_REFERER} !mxnet.apache.org\nRewriteCond %{HTTP_REFERER} !mxnet.incubator.apache.org\nRewriteCond %{HTTP_REFERER} !mxnet.cdn.apache.org\nRewriteRule ^(.*)$ /versions/1.9.1/$1 [r=307,L]\n\n# Redirect Chinese visitors to Chinese CDN, temporary solution for slow site speed in China\nRewriteCond %{ENV:GEOIP_COUNTRY_CODE} ^CN$\nRewriteCond %{HTTP_HOST} !cdn\nRewriteRule ^(.*) https://mxnet.cdn.apache.org%{REQUEST_URI} [R,NC,L]\n\n# Show file instead of folder for example /api/docs/tutorials.html\n# instead of /api/docs/tutorials/\nRewriteCond %{REQUEST_FILENAME} -d\nRewriteCond %{REQUEST_FILENAME}\\.html -f\nRewriteRule ^(.*) $1.html [NC,L]\n\n# Prettify some files like tutorials/io to tutorials/io.html\nRewriteCond %{REQUEST_URI} !/julia/\nRewriteCond %{REQUEST_FILENAME}\\.html -f\nRewriteRule ^(.*) $1.html [NC,L]\n\n#API docs needs specific root\nRewriteCond %{REQUEST_URI} \\/docs\\/api$|\\/docs\\/tutorials$\nRewriteRule ^(.*) %{REQUEST_URI}/ [R,NC,L]\n\n# Prettify folders like /get_started\nRewriteCond %{REQUEST_URI} !\\/$\nRewriteCond %{REQUEST_FILENAME} -d\nRewriteCond %{REQUEST_FILENAME}/index.html -f\nRewriteRule ^(.*) $1/index.html [NC,L]\n\n# Redirect FAQ TODO: temporary fix for Github issue #18547\nRewriteRule ^versions/master/faq/(.*)$ /api/faq/$1 [R,NC,L]\n\n# 404\nErrorDocument 404 /404.html\n\n# Redirects\n# Python API\nRedirect 301 /versions/master/api/python/index.html /api/python/docs/api/index.html\nRedirect 301 /api/python/ndarray/ndarray.html /api/python/docs/api/ndarray/index.html\nRedirect 301 /api/python/ndarray/random.html /api/python/docs/api/ndarray/random/index.html\nRedirect 301 /api/python/ndarray/linalg.html /api/python/docs/api/ndarray/linalg/index.html\nRedirect 301 /api/python/ndarray/contrib.html /api/python/docs/api/ndarray/contrib/index.html\nRedirect 301 /api/python/ndarray/sparse.html /api/python/docs/api/ndarray/sparse/index.html\n\nRedirect 301 /api/python/autograd/autograd.html /api/python/docs/api/autograd/index.html\n\nRedirect 301 /api/python/gluon/gluon.html /api/python/docs/api/gluon/index.html\nRedirect 301 /api/python/gluon/nn.html /api/python/docs/api/gluon/nn/index.html\nRedirect 301 /api/python/gluon/rnn.html /api/python/docs/api/gluon/rnn/index.html\nRedirect 301 /api/python/gluon/loss.html /api/python/docs/api/gluon/loss/index.html\nRedirect 301 /api/python/gluon/data.html /api/python/docs/api/gluon/data/index.html\nRedirect 301 /api/python/gluon/model_zoo.html /api/python/docs/api/gluon/model_zoo/index.html\nRedirect 301 /api/python/gluon/contrib.html /api/python/docs/api/gluon/contrib/index.html\n\nRedirect 301 /api/python/kvstore/kvstore.html /api/python/docs/api/kvstore/index.html\nRedirect 301 /api/python/metric/metric.html /api/python/docs/api/metric/index.html\nRedirect 301 /api/python/optimization/optimization.html /api/python/docs/api/optimizer/index.html\nRedirect 301 /api/python/optimization/contrib.html /api/python/docs/api/optimizer/index.html\nRedirect 301 /api/python/profiler/profiler.html /api/python/docs/api/mxnet/profiler/index.html\nRedirect 301 /api/python/io/io.html /api/python/docs/api/mxnet/io/index.html\nRedirect 301 /api/python/contrib/contrib.html /api/python/docs/api/contrib/index.html\n\nRedirect 301 /api/python/symbol/symbol.html /api/python/docs/api/symbol/index.html\nRedirect 301 /api/python/symbol.html /api/python/docs/api/symbol/index.html\nRedirect 301 /api/python/symbol/linalg.html /api/python/docs/api/symbol/linalg/index.html\n\nRedirect 301 /api/python/module/module.html /api/python/docs/api/module/index.html\nRedirect 301 /api/python/callback/callback.html /api/python/docs/api/mxnet/callback/index.html\nRedirect 301 /api/python/tools/visualization.html /api/python/docs/api/mxnet/visualization/index.html\n\nRedirect 301 /api/python/executor/executor.html /api/python/docs/api/mxnet/executor/index.html\nRedirect 301 /api/python/rtc/rtc.html /api/python/docs/api/mxnet/rtc/index.html\nRedirect 301 /api/python/tools/test_utils.html /api/python/docs/api/mxnet/test_utils/index.html\n\n# Top Level Nav bar\nRedirect 301 /install/index.html /get_started\nRedirect 301 /test/get_started/install.html /get_started\nRedirect 301 /faq/index.html /api\nRedirect 301 /tutorials/index.html /api\nRedirect 301 /architecture/index.html /api/architecture/overview\nRedirect 301 /community/ecosystem.html /ecosystem\nRedirect 301 /community/powered_by.html /ecosystem\n"
  },
  {
    "path": "docs/static_site/src/.nojekyll",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n\n#   http://www.apache.org/licenses/LICENSE-2.0\n\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n"
  },
  {
    "path": "docs/static_site/src/404.html",
    "content": "---\n  # Licensed to the Apache Software Foundation (ASF) under one\n  # or more contributor license agreements.  See the NOTICE file\n  # distributed with this work for additional information\n  # regarding copyright ownership.  The ASF licenses this file\n  # to you under the Apache License, Version 2.0 (the\n  # \"License\"); you may not use this file except in compliance\n  # with the License.  You may obtain a copy of the License at\n  #   http://www.apache.org/licenses/LICENSE-2.0\n  # Unless required by applicable law or agreed to in writing,\n  # software distributed under the License is distributed on an\n  # \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  # KIND, either express or implied.  See the License for the\n  # specific language governing permissions and limitations\n  # under the License.\n\nlayout: default\n---\n\n<style type=\"text/css\" media=\"screen\">\n  .container {\n    margin: 10px auto;\n    max-width: 600px;\n    text-align: center;\n  }\n  h1 {\n    margin: 30px 0;\n    font-size: 4em;\n    line-height: 1;\n    letter-spacing: -1px;\n  }\n</style>\n\n<div class=\"container\">\n  <h1>404</h1>\n\n  <p><strong>Page not found :'(</strong></p>\n\n  <p><b>Due to a recent redesign of the website, some items have moved</b>\n    We're working towards adding redirects. Useful links:</p>\n  <div style=\"background-color:white;\">\n    <li><a href=\"/api/python/docs/tutorials/\">Python Tutorials</a></li>\n    <li><a href=\"/api/python/docs/api/\">Python API Documentation</a></li>\n  </div>\n</div>\n"
  },
  {
    "path": "docs/static_site/src/Gemfile",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nsource \"https://rubygems.org\"\nruby \"2.6.5\"\n\n# Hello! This is where you manage which Jekyll version is used to run.\n# When you want to use a different version, change it below, save the\n# file and run `bundle install`. Run Jekyll with `bundle exec`, like so:\n#\n#     bundle exec jekyll serve\n#\n# This will help ensure the proper Jekyll version is running.\n# Happy Jekylling!\ngem \"jekyll\", \"~> 4.0\"\n\n# This is the default theme for new Jekyll sites. You may change this to anything you like.\n# gem \"minima\", \"~> 2.0\"\n\n# If you want to use GitHub Pages, remove the \"gem \"jekyll\"\" above and\n# uncomment the line below. To upgrade, run `bundle update github-pages`.\n# gem \"github-pages\", group: :jekyll_plugins\n\n# If you have any plugins, put them here!\ngroup :jekyll_plugins do\n  gem \"jekyll-feed\", \"~> 0.6\"\n  gem \"jekyll-seo-tag\", \"~> 2.6.1\"\nend\n\n# Windows does not include zoneinfo files, so bundle the tzinfo-data gem\n# and associated library.\ninstall_if -> { RUBY_PLATFORM =~ %r!mingw|mswin|java! } do\n  gem \"tzinfo\", \"~> 1.2\"\n  gem \"tzinfo-data\"\nend\n\n# Performance-booster for watching directories on Windows\ngem \"wdm\", \"~> 0.1.0\", :install_if => Gem.win_platform?\n"
  },
  {
    "path": "docs/static_site/src/_config.yml",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# Welcome to Jekyll!\n#\n# This config file is meant for settings that affect your whole blog, values\n# which you are expected to set up once and rarely edit after that. If you find\n# yourself editing this file very often, consider using Jekyll's data files\n# feature for the data you need to update frequently.\n#\n# For technical reasons, this file is *NOT* reloaded automatically when you use\n# 'bundle exec jekyll serve'. If you change this file, please restart the server process.\n\n# Site settings\n# These are used to personalize your new site. If you look in the HTML files,\n# you will see them accessed via {{ site.title }}, {{ site.email }}, and so on.\n# You can create any custom variable you would like, and they will be accessible\n# in the templates via {{ site.myvariable }}.\ntitle: Apache MXNet\nemail: dev@mxnet.apache.org\ndescription: >- # this means to ignore newlines until \"baseurl:\"\n  A flexible and efficient library for deep learning.\ntwitter_username: apachemxnet\ngithub_username:  apache/mxnet\nyoutube_username: apachemxnet\nbaseurl: /versions/master\nversions:\n  - master\n  - 1.9.1\n  - 1.8.0\n  - 1.7.0\n  - 1.6.0\n  - 1.5.0\n  - 1.4.1\n  - 1.3.1\n  - 1.2.1\n  - 1.1.0\n  - 1.0.0\n  - 0.12.1\n  - 0.11.0\n\n# Build settings\nmarkdown: kramdown\n\n#redcarpet:\n#  extensions: [\"no_intra_emphasis\", \"fenced_code_blocks\", \"autolink\", \"tables\", \"with_toc_data\"]\nplugins:\n  - jekyll-feed\n  - jekyll-seo-tag\n\npermalink: pretty\n\n# Exclude from processing.\n# The following items will not be processed, by default. Create a custom list\n# to override the default setting.\n# exclude:\n#   - Gemfile\n#   - Gemfile.lock\n#   - node_modules\n#   - vendor/bundle/\n#   - vendor/cache/\n#   - vendor/gems/\n#   - vendor/ruby/\n"
  },
  {
    "path": "docs/static_site/src/_config_beta.yml",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# Welcome to Jekyll!\n#\n# This config file is meant for settings that affect your whole blog, values\n# which you are expected to set up once and rarely edit after that. If you find\n# yourself editing this file very often, consider using Jekyll's data files\n# feature for the data you need to update frequently.\n#\n# For technical reasons, this file is *NOT* reloaded automatically when you use\n# 'bundle exec jekyll serve'. If you change this file, please restart the server process.\n\n# Site settings\n# These are used to personalize your new site. If you look in the HTML files,\n# you will see them accessed via {{ site.title }}, {{ site.email }}, and so on.\n# You can create any custom variable you would like, and they will be accessible\n# in the templates via {{ site.myvariable }}.\ntitle: Apache MXNet\nemail: dev@mxnet.apache.org\ndescription: >- # this means to ignore newlines until \"baseurl:\"\n  A flexible and efficient library for deep learning.\nbaseurl: /mxnet.io-v2 # the subpath of your site, e.g. /blog\nurl: https://thomasdelteil.github.io\ntwitter_username: apachemxnet\ngithub_username:  apache/mxnet\nyoutube_username: apachemxnet\nbaseurl: /versions/master\nversions:\n  - master\n  - 1.9.1\n  - 1.8.0\n  - 1.7.0\n  - 1.6.0\n  - 1.5.0\n  - 1.4.1\n  - 1.3.1\n  - 1.2.1\n  - 1.1.0\n  - 1.0.0\n  - 0.12.1\n  - 0.11.0\n\n# Build settings\nmarkdown: kramdown\nplugins:\n  - jekyll-feed\n  - jekyll-seo-tag\n# Exclude from processing.\n# The following items will not be processed, by default. Create a custom list\n# to override the default setting.\n# exclude:\n#   - Gemfile\n#   - Gemfile.lock\n#   - node_modules\n#   - vendor/bundle/\n#   - vendor/cache/\n#   - vendor/gems/\n#   - vendor/ruby/\n"
  },
  {
    "path": "docs/static_site/src/_config_prod.yml",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n# Welcome to Jekyll!\n#\n# This config file is meant for settings that affect your whole blog, values\n# which you are expected to set up once and rarely edit after that. If you find\n# yourself editing this file very often, consider using Jekyll's data files\n# feature for the data you need to update frequently.\n#\n# For technical reasons, this file is *NOT* reloaded automatically when you use\n# 'bundle exec jekyll serve'. If you change this file, please restart the server process.\n\n# Site settings\n# These are used to personalize your new site. If you look in the HTML files,\n# you will see them accessed via {{ site.title }}, {{ site.email }}, and so on.\n# You can create any custom variable you would like, and they will be accessible\n# in the templates via {{ site.myvariable }}.\ntitle: Apache MXNet\nemail: dev@mxnet.apache.org\ndescription: >- # this means to ignore newlines until \"baseurl:\"\n  A flexible and efficient library for deep learning.\nurl: https://mxnet.apache.org\ntwitter_username: apachemxnet\ngithub_username:  apache/mxnet\nyoutube_username: apachemxnet\nbaseurl: /versions/master\nversions:\n  - master\n  - 1.9.1\n  - 1.8.0\n  - 1.7.0\n  - 1.6.0\n  - 1.5.0\n  - 1.4.1\n  - 1.3.1\n  - 1.2.1\n  - 1.1.0\n  - 1.0.0\n  - 0.12.1\n  - 0.11.0\n\n# Build settings\nmarkdown: kramdown\nplugins:\n  - jekyll-feed\n  - jekyll-seo-tag\n\n# Force include .asf.yaml\ninclude:\n  - .asf.yaml\n  - .htaccess\n\n# Exclude from processing.\n# The following items will not be processed, by default. Create a custom list\n# to override the default setting.\n# exclude:\n#   - Gemfile\n#   - Gemfile.lock\n#   - node_modules\n#   - vendor/bundle/\n#   - vendor/cache/\n#   - vendor/gems/\n#   - vendor/ruby/\n"
  },
  {
    "path": "docs/static_site/src/_includes/callout.html",
    "content": "<div markdown=\"span\" class=\"bs-callout bs-callout-{{include.type}}\">{{include.content}}</div>\n"
  },
  {
    "path": "docs/static_site/src/_includes/disqus_comments.html",
    "content": "{%- if page.comments != false and jekyll.environment == \"production\" -%}\n\n  <div id=\"disqus_thread\"></div>\n  <script>\n    var disqus_config = function () {\n      this.page.url = '{{ page.url | absolute_url }}';\n      this.page.identifier = '{{ page.url | absolute_url }}';\n    };\n\n    (function() {\n      var d = document, s = d.createElement('script');\n\n      s.src = 'https://{{ site.disqus.shortname }}.disqus.com/embed.js';\n\n      s.setAttribute('data-timestamp', +new Date());\n      (d.head || d.body).appendChild(s);\n    })();\n  </script>\n  <noscript>Please enable JavaScript to view the <a href=\"https://disqus.com/?ref_noscript\" rel=\"nofollow\">comments powered by Disqus.</a></noscript>\n{%- endif -%}\n"
  },
  {
    "path": "docs/static_site/src/_includes/feedback.html",
    "content": "<hr class=\"feedback-hr-top\" />\n<div class=\"feedback-container\">\n  <div class=\"feedback-question\">Did this page help you?</div>\n  <div class=\"feedback-answer-container\">\n    <div class=\"feedback-answer yes-link\" data-response=\"yes\">Yes</div>\n    <div class=\"feedback-answer no-link\" data-response=\"no\">No</div>\n  </div>\n  <div class=\"feedback-thank-you\">Thanks for your feedback!</div>\n</div>\n<hr class=\"feedback-hr-bottom\" />\n"
  },
  {
    "path": "docs/static_site/src/_includes/footer.html",
    "content": "<footer class=\"site-footer h-card\">\n    <div class=\"wrapper\">\n        <div class=\"row\">\n            <div class=\"col-4\">\n                <h4 class=\"footer-category-title\">Resources</h4>\n                <ul class=\"contact-list\">\n                    <li><a href=\"{{'community#stay-connected'|relative_url}}\">Mailing lists</a></li>\n                    <li><a href=\"{{'community#github-issues'|relative_url}}\">Github Issues</a></li>\n                    <li><a href=\"https://github.com/apache/mxnet/projects\">Projects</a></li>\n                    <li><a href=\"https://cwiki.apache.org/confluence/display/MXNET/Apache+MXNet+Home\">Developer Wiki</a></li>\n                    <li><a href=\"https://discuss.mxnet.io\">Forum</a></li>\n                    <li><a href=\"{{'community'|relative_url}}\">Contribute To MXNet</a></li>\n                </ul>\n            </div>\n\n            <div class=\"col-4\">\n                {%- include social.html -%}\n            </div>\n\n            <div class=\"col-4 footer-text\">\n                <p>{{- site.description | escape -}}</p>\n            </div>\n        </div>\n    </div>\n</footer>\n<footer class=\"site-footer2\">\n    <div class=\"wrapper\">\n        <div class=\"row\">\n            <div class=\"col-3\">\n                <img src=\"{{'/assets/img/asf_logo.svg' | relative_url}}\" class=\"footer-logo col-2\">\n            </div>\n            <div class=\"footer-bottom-warning col-9\">\n                </p><p>\"Copyright © 2017-2022, The Apache Software Foundation. Licensed under the Apache License, Version 2.0. Apache MXNet, MXNet, Apache, the Apache\n                    feather, and the Apache MXNet project logo are either registered trademarks or trademarks of the\n                    Apache Software Foundation.\"</p>\n            </div>\n        </div>\n    </div>\n</footer>\n\n\n\n\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/cloud/cpu.md",
    "content": "MXNet should work on any cloud provider's CPU-only instances. Follow the Python\npip install instructions, Docker instructions, or try the following preinstalled\noption.\n\n**WARNING**: the following cloud provider packages are provided for your convenience\nbut they point to packages that are *not* provided nor endorsed by the Apache\nSoftware Foundation. As such, they might contain software components with more\nrestrictive licenses than the Apache License and you'll need to decide whether\nthey are appropriate for your usage. Like all Apache Releases, the official\nApache MXNet releases consist of source code only and are found at\nthe [Download page](https://mxnet.apache.org/get_started/download).\n\n* **Amazon Web Services**\n- [AWS Deep Learning AMI](https://aws.amazon.com/machine-learning/amis/) - Preinstalled\nConda environments\nfor Python 2 or 3 with MXNet and oneDNN.\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/cloud/gpu.md",
    "content": "MXNet is available on several cloud providers with GPU support. You can also\nfind GPU/CPU-hybrid support for use cases like scalable inference, or even\nfractional GPU support with AWS Elastic Inference.\n\n**WARNING**: the following cloud provider packages are provided for your convenience\nbut they point to packages that are *not* provided nor endorsed by the Apache\nSoftware Foundation. As such, they might contain software components with more\nrestrictive licenses than the Apache License and you'll need to decide whether\nthey are appropriate for your usage. Like all Apache Releases, the official\nApache MXNet releases consist of source code only and are found at\nthe [Download page](https://mxnet.apache.org/get_started/download).\n\n* **Alibaba**\n- [NVIDIA\nVM](https://docs.nvidia.com/ngc/ngc-alibaba-setup-guide/launching-nv-cloud-vm-console.html#launching-nv-cloud-vm-console)\n* **Amazon Web Services**\n- [Amazon SageMaker](https://aws.amazon.com/sagemaker/) - Managed training and deployment of\nMXNet models\n- [AWS Deep Learning AMI](https://aws.amazon.com/machine-learning/amis/) - Preinstalled\nConda environments\nfor Python 2 or 3 with MXNet, CUDA, cuDNN, oneDNN, and AWS Elastic Inference\n- [Dynamic Training on\nAWS](https://github.com/awslabs/dynamic-training-with-apache-mxnet-on-aws) -\nexperimental manual EC2 setup or semi-automated CloudFormation setup\n- [NVIDIA VM](https://aws.amazon.com/marketplace/pp/B076K31M1S)\n* **Google Cloud Platform**\n- [NVIDIA\nVM](https://console.cloud.google.com/marketplace/details/nvidia-ngc-public/nvidia_gpu_cloud_image)\n* **Microsoft Azure**\n- [NVIDIA\nVM](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/nvidia.ngc_azure_17_11?tab=Overview)\n* **Oracle Cloud**\n- [NVIDIA VM](https://docs.cloud.oracle.com/iaas/Content/Compute/References/ngcimage.htm)\n\nAll NVIDIA VMs use the [NVIDIA MXNet Docker\ncontainer](https://ngc.nvidia.com/catalog/containers/nvidia:mxnet).\nFollow the [container usage\ninstructions](https://ngc.nvidia.com/catalog/containers/nvidia:mxnet) found in\n[NVIDIA's container repository](https://ngc.nvidia.com/).\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/devices/nvidia-jetson.md",
    "content": "# NVIDIA Jetson Devices\n\nTo install MXNet on a Jetson TX or Nano, please refer to the [Jetson installation\nguide](/get_started/jetson_setup)."
  },
  {
    "path": "docs/static_site/src/_includes/get_started/devices/raspberry_pi.md",
    "content": "MXNet supports running on ARM devices, such as the Raspberry PI.\n\nThese instructions will walk through how to build MXNet for the Raspberry Pi and\ninstall the Python bindings for the library.\n\nYou can do a cross compilation build on your local machine (faster) or a native\nbuild on-device (slower, but more foolproof).\n\nThe complete MXNet library and its requirements can take almost 200MB of RAM,\nand loading large models with the library can take over 1GB of RAM. Because of\nthis, we recommend running MXNet on the Raspberry Pi 3 or an equivalent device\nthat has more than 1 GB of RAM and a Secure Digital (SD) card that has at least\n4 GB of free memory.\n\n## Native build on the Raspberry Pi\n\nTo build MXNet directly on the Raspberry Pi device, you can mainly follow the\nstandard [Ubuntu setup]({{'/get_started/ubuntu_setup|relative_url}})\ninstructions. However, skip the step of copying the `config/linux.cmake` to\n`config.cmake` and instead run the `cmake` in the \"Build MXNet core shared\nlibrary\" step as follows:\n\n\n```\nrm -rf build\nmkdir -p build && cd build\ncmake \\\n  -DUSE_SSE=OFF \\\n  -DUSE_CUDA=OFF \\\n  -DUSE_BLAS=Open \\\n  -DUSE_OPENCV=ON \\\n  -DUSE_OPENMP=ON \\\n  -DUSE_SIGNAL_HANDLER=ON \\\n  -DCMAKE_BUILD_TYPE=Release \\\n  -GNinja ..\nninja -j$(nproc)\n```\n\nSome compilation units require memory close to 1GB, so it's recommended that you\nenable swap as explained below and be cautious about increasing the number of\njobs when building (-j). Executing these commands start the build process, which\ncan take up to a couple hours, and creates a file called `libmxnet.so` in the\nbuild directory.\n\nIf you are getting build errors in which the compiler is being killed, it is\nlikely that the compiler is running out of memory (especially if you are on\nRaspberry Pi 1, 2 or Zero, which have less than 1GB of RAM), this can often be\nrectified by increasing the swapfile size on the Pi by editing the file\n/etc/dphys-swapfile and changing the line CONF_SWAPSIZE=100 to\nCONF_SWAPSIZE=1024, then running:\n\n```\nsudo /etc/init.d/dphys-swapfile stop\nsudo /etc/init.d/dphys-swapfile start\nfree -m # to verify the swapfile size has been increased\n```\n\n## Cross-compiling on your local machine\n\n### Obtaining the toolchain\n\nYou first need to setup the cross-compilation toolchain on your local machine.\nOn Debian based systems, you can install `crossbuild-essential-armel` to obtain\na cross-toolchain for the ARMv4T, 5T, and 6, `crossbuild-essential-armhf` ARMv7\narchitecture and `crossbuild-essential-arm64` for ARMv8 (also called aarch64).\nSee for example\n[Wikipedia](https://en.wikipedia.org/wiki/Raspberry_Pi#Specifications) to\ndetermine the architecture of your Raspberry PI devices. If none of the Debian\ntoolchains works for you, you may like to refer to\n[toolchains.bootlin.com](https://toolchains.bootlin.com/) for a large number of\nready-to-use cross-compilation toolchains.\n\n### Cross-compiling MXNet dependencies\nBefore compiling MXNet, you need to cross-compile MXNet's dependencies. At the\nvery minimum, you'll need OpenBLAS. You can cross-compile it as follows,\nreplacing the `CC=aarch64-linux-gnu-gcc` and `PREFIX=/usr/aarch64-linux-gnu`\nbased on your architecture:\n\n```\ngit clone --recursive https://github.com/xianyi/OpenBLAS.git\ncd OpenBLAS\nmake NOFORTRAN=1 NO_SHARED=1 CC=aarch64-linux-gnu-gcc\nmake PREFIX=/usr/local/aarch64-linux-gnu NO_SHARED=1 install\n```\n\nIf you would like to compile MXNet with OpenCV support, enabling various image\ntransformation related features, you also need to cross-compile OpenCV.\n\n### Cross-compiling MXNet\n\nBefore you cross-compile MXNet, create a CMake toolchain file specifying all settings for your compilation. For example, `aarch64-linux-gnu-toolchain.cmake`:\n\n```\nset(CMAKE_SYSTEM_NAME Linux)\nset(CMAKE_SYSTEM_PROCESSOR \"aarch64\")\nset(CMAKE_C_COMPILER aarch64-linux-gnu-gcc)\nset(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++)\nset(CMAKE_CUDA_HOST_COMPILER aarch64-linux-gnu-gcc)\nset(CMAKE_FIND_ROOT_PATH \"/usr/aarch64-linux-gnu;/usr/local/aarch64-linux-gnu\")\n\nset(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)\nset(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)\nset(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)\nset(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)\n```\n\n`CMAKE_FIND_ROOT_PATH` should be a list of directories containing the\ncross-compilation toolchain and MXNet's cross-compiled dependencies. If you use\na toolchain from the bootlin site linked above, you can find the respective\nCMake toolchain file at `share/buildroot/toolchainfile.cmake`.\n\nYou can then cross-compile MXNet via\n\n```\nmkdir build; cd build\ncmake -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \\\n  -DUSE_CUDA=OFF \\\n  -DSUPPORT_F16C=OFF \\\n  -DUSE_BLAS=Open \\\n  -DUSE_OPENCV=OFF \\\n  -DUSE_OPENMP=ON \\\n  -DUSE_LAPACK=OFF \\\n  -DUSE_SIGNAL_HANDLER=ON \\\n  -DCMAKE_BUILD_TYPE=Release \\\n  -G Ninja ..\nninja\ncd ..\n```\n\nWe would like to simplify this setup by integrating the Conan C++ dependency\nmanager. Please send an email to the MXNet development mailinglist or open an\nissue on Github if you would like to help.\n\n### Building the Python wheel\n\nTo build the wheel, you can follow the following process\n\n```\nexport MXNET_LIBRARY_PATH=$(pwd)/build/libmxnet.so\n\ncd python\npython3 setup.py bdist_wheel\n\n\n# Fix pathing issues in the wheel.  We need to move libmxnet.so from the data folder to the\n# mxnet folder, then repackage the wheel.\nWHEEL=`readlink -f dist/*.whl`\nTMPDIR=`mktemp -d`\nunzip -d ${TMPDIR} ${WHEEL}\nrm ${WHEEL}\ncd ${TMPDIR}\nmv *.data/data/mxnet/libmxnet.so mxnet\nzip -r ${WHEEL} .\ncp ${WHEEL} ..\nrm -rf ${TMPDIR}\n```\n\nWe intend to fix the `setup.py` to avoid the repackaging step. If you would like\nto help, please send an email to the MXNet development mailinglist or open an\nissue on Github.\n\n\n### Final remarks\n\nYou are now ready to run MXNet on your Raspberry Pi device. You can get started\nby following the tutorial on [Real-time Object Detection with MXNet On The\nRaspberry\nPi](https://mxnet.io/api/python/docs/tutorials/deploy/inference/wine_detector.html).\n\n*Note - Because the complete MXNet library takes up a significant amount of the\nRaspberry Pi's limited RAM, when loading training data or large models into\nmemory, you might have to turn off the GUI and terminate running processes to\nfree RAM.*\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/get_started.html",
    "content": "<script>\n    /** Defaults **/\n    /** See options.js for the full ugly script **/\n    var versionSelect = defaultVersion = 'v1.9.1';\n    var platformSelect = 'linux';\n    var languageSelect = 'python';\n    var processorSelect = 'cpu';\n    var environSelect = 'pip';\n</script>\n<script src=\"{{'/assets/js/options.js'|relative_url}}\"></script>\n\n<div class=\"install-selector\">\n    <h2>Platform and use-case specific instructions for using Apache MXNet</h2>\n    <p>\n        Please indicate your preferred configuration below to see specific instructions.\n    </p>\n    <br>\n    <br>\n    <div class=\"install-widget\">\n        <div class=\"row\">\n            <div class=\"col-3 install-left\">\n                <span>MXNet Version</span>\n            </div>\n            <div class=\"col-9 install-right\">\n                <div class=\"dropdown\" id=\"version-dropdown-container\">\n                    <button class=\"current-version dropbtn btn\" type=\"button\" data-toggle=\"dropdown\">\n                        v1.9.1\n                        <svg class=\"dropdown-caret\" viewBox=\"0 0 32 32\" class=\"icon icon-caret-bottom\" aria-hidden=\"true\">\n                            <path class=\"dropdown-caret-path\" d=\"M24 11.305l-7.997 11.39L8 11.305z\"></path>\n                        </svg>\n                    </button>\n                    <ul class=\"opt-group version-dropdown\">\n                        <li class=\"opt active versions\"><a href=\"#\">v1.9.1</a></li>\n                        <li class=\"opt versions\"><a href=\"#\">v1.8.0</a></li>\n                        <li class=\"opt versions\"><a href=\"#\">v1.7.0</a></li>\n                        <li class=\"opt versions\"><a href=\"#\">v1.6.0</a></li>\n                        <li class=\"opt versions\"><a href=\"#\">v1.5.1</a></li>\n                        <li class=\"opt versions\"><a href=\"#\">v1.4.1</a></li>\n                        <li class=\"opt versions\"><a href=\"#\">v1.3.1</a></li>\n                        <li class=\"opt versions\"><a href=\"#\">v1.2.1</a></li>\n                        <li class=\"opt versions\"><a href=\"#\">v1.1.0</a></li>\n                        <li class=\"opt versions\"><a href=\"#\">v1.0.0</a></li>\n                        <li class=\"opt versions\"><a href=\"#\">v0.12.1</a></li>\n                        <li class=\"opt versions\"><a href=\"#\">v0.11.0</a></li>\n                    </ul>\n                </div>\n            </div>\n        </div>\n\n        <!-- START - OS Menu -->\n        <div class=\"row\">\n            <div class=\"col-3 install-left\">\n                <span>OS / Platform</span>\n            </div>\n            <div class=\"col-9 install-right\">\n                <div class=\"btn-group opt-group\" role=\"group\">\n                    <button type=\"button\" class=\"btn btn-default opt active platforms\">Linux</button>\n                    <button type=\"button\" class=\"btn btn-default opt platforms\">MacOS</button>\n                    <button type=\"button\" class=\"btn btn-default opt platforms\">Windows</button>\n                    <button type=\"button\" class=\"btn btn-default opt platforms\">Cloud</button>\n                    <button type=\"button\" class=\"btn btn-default opt platforms\">Devices</button>\n                </div>\n            </div>\n        </div>\n\n        <!-- START - Language Menu -->\n        <div class=\"linux macos windows\">\n            <div class=\"row\">\n                <div class=\"col-3 install-left\">\n                    <span>Language</span>\n                </div>\n                <div class=\"col-9 install-right\">\n                    <div class=\"btn-group opt-group\" role=\"group\">\n                        <button type=\"button\" class=\"btn btn-default opt active languages\">Python</button>\n                        <button type=\"button\" class=\"btn btn-default opt languages\">Scala</button>\n                        <button type=\"button\" class=\"btn btn-default opt languages\">Java</button>\n                        <button type=\"button\" class=\"btn btn-default opt languages\">Clojure</button>\n                        <button type=\"button\" class=\"btn btn-default opt languages\">R</button>\n                        <button type=\"button\" class=\"btn btn-default opt languages\">Julia</button>\n                        <button type=\"button\" class=\"btn btn-default opt languages\">Perl</button>\n                        <button type=\"button\" class=\"btn btn-default opt languages\">Cpp</button>\n                    </div>\n                </div>\n            </div>\n        </div>\n\n\n\n        <!-- No CPU GPU for other Devices -->\n        <div class=\"linux macos windows cloud\">\n            <div class=\"python cloud devices\">\n                <div class=\"row\">\n                    <div class=\"col-3 install-left\">\n                        <span>GPU / CPU</span>\n                    </div>\n                    <div class=\"col-9 install-right\">\n                        <div class=\"btn-group opt-group\" role=\"group\">\n                            <button type=\"button\" class=\"btn btn-default processors opt active\">GPU</button>\n                            <button type=\"button\" class=\"btn btn-default processors opt\">CPU</button>\n                        </div>\n                    </div>\n                </div>\n            </div>\n        </div>\n\n        <!-- other devices -->\n        <div class=\"devices\">\n            <div class=\"row\">\n                <div class=\"col-3 install-left\">\n                    <span>Device</span>\n                </div>\n                <div class=\"col-9 install-right\">\n                    <div class=\"btn-group opt-group\" role=\"group\">\n                        <button type=\"button\" class=\"btn btn-default iots opt active\">Raspberry Pi</button>\n                        <button type=\"button\" class=\"btn btn-default iots opt\">NVIDIA Jetson</button>\n                    </div>\n                </div>\n            </div>\n        </div>\n\n        <!-- Python Distribution -->\n        <div class=\"linux macos windows\">\n            <div class=\"python\">\n                <div class=\"cpu gpu\">\n                    <div class=\"row\">\n                        <div class=\"col-3 install-left\">\n                            <span>Distribution</span>\n                        </div>\n                        <div class=\"col-9 install-right\">\n                            <div class=\"btn-group opt-group\" role=\"group\">\n                                <button type=\"button\" class=\"btn btn-default environs opt active\">Pip</button>\n                                <button type=\"button\" class=\"btn btn-default environs opt\">Docker</button>\n                                <button type=\"button\" class=\"btn btn-default environs opt\">Build from Source</button>\n                            </div>\n                        </div>\n                    </div>\n                </div>\n            </div>\n        </div>\n\n    </div>\n\n\n<!-- END - Main Menu -->\n    <br>\n    <br>\n    <br>\n    <div class=\"install-content\">\n        <div class=\"linux\">\n            <div class=\"python\">\n                 <!-- START - Linux Python CPU Installation Instructions -->\n                <div class=\"cpu\">\n                    <div class=\"pip\">\n                        {% markdown %}{% include /get_started/linux/python/cpu/pip.md %}{% endmarkdown %}\n                    </div> <!-- End of pip -->\n\n                    <div class=\"docker\">\n                        {% markdown %}{% include /get_started/linux/python/cpu/docker.md %}{% endmarkdown %}\n                    </div> <!-- END of docker -->\n\n                    <div class=\"build-from-source\">\n                        {% markdown %}{% include /get_started/linux/python/cpu/build-from-source.md %}{% endmarkdown %}\n                    </div><!-- END of build from source -->\n\n                </div><!-- END of CPU -->\n                <!-- END - Linux Python CPU Installation Instructions -->\n\n                <!-- START - Linux Python GPU Installation Instructions -->\n                <div class=\"gpu\">\n                    <div class=\"pip\">\n                        {% markdown %}{% include /get_started/linux/python/gpu/pip.md %}{% endmarkdown %}\n                    </div> <!-- END of pip -->\n\n                    <div class=\"docker\">\n                        {% markdown %}{% include /get_started/linux/python/gpu/docker.md %}{% endmarkdown %}\n                    </div> <!-- END of docker -->\n\n                    <div class=\"build-from-source\">\n                         {% markdown %}{% include /get_started/linux/python/gpu/build-from-source.md %}{% endmarkdown %}\n                    </div> <!-- END of build from source -->\n                </div> <!-- END of GPU -->\n            </div> <!-- END of Python -->\n            <!-- END - Linux Python Installation Instructions -->\n\n\n            <div class=\"r\">\n                {% markdown %}{% include /get_started/linux/r/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- END of R -->\n\n\n            <div class=\"scala\">\n                {% markdown %}{% include /get_started/linux/scala/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- End of scala -->\n\n\n            <div class=\"clojure\">\n                {% markdown %}{% include /get_started/linux/clojure/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- End of clojure -->\n\n\n            <div class=\"java\">\n                {% markdown %}{% include /get_started/linux/java/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- End of java -->\n\n            <div class=\"julia\">\n                {% markdown %}{% include /get_started/linux/julia/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- End of julia -->\n\n            <div class=\"perl\">\n                {% markdown %}{% include /get_started/linux/perl/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- End of perl -->\n\n            <div class=\"cpp\">\n                {% markdown %}{% include /get_started/linux/cpp/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- END - C++-->\n        </div> <!-- END - Linux -->\n\n\n        <!-- START - MacOS Python CPU Installation Instructions -->\n\n        <div class=\"macos\">\n            <div class=\"python\">\n                 <!-- START - MacOS Python CPU Installation Instructions -->\n                <div class=\"cpu\">\n                    <div class=\"pip\">\n                        {% markdown %}{% include /get_started/macos/python/cpu/pip.md %}{% endmarkdown %}\n                    </div> <!-- End of pip -->\n\n                    <div class=\"docker\">\n                        {% markdown %}{% include /get_started/macos/python/cpu/docker.md %}{% endmarkdown %}\n                    </div> <!-- END of docker -->\n\n                    <div class=\"build-from-source\">\n                        {% markdown %}{% include /get_started/macos/python/cpu/build-from-source.md %}{% endmarkdown %}\n                    </div><!-- END of build from source -->\n\n                </div><!-- END of CPU -->\n                <!-- END - MacOS Python CPU Installation Instructions -->\n\n                <!-- START - MacOS Python GPU Installation Instructions -->\n                <div class=\"gpu\">\n                    <div class=\"build-from-source\">\n                         {% markdown %}{% include /get_started/macos/python/gpu/build-from-source.md %}{% endmarkdown %}\n                    </div> <!-- END of build from source -->\n                </div> <!-- END of GPU -->\n            </div> <!-- END of Python -->\n            <!-- END - MacOS Python Installation Instructions -->\n\n\n            <div class=\"r\">\n                {% markdown %}{% include /get_started/macos/r/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- END of R -->\n\n\n            <div class=\"scala\">\n                {% markdown %}{% include /get_started/macos/scala/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- End of scala -->\n\n\n            <div class=\"clojure\">\n                {% markdown %}{% include /get_started/macos/clojure/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- End of clojure -->\n\n\n            <div class=\"java\">\n                {% markdown %}{% include /get_started/macos/java/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- End of java -->\n\n            <div class=\"julia\">\n                {% markdown %}{% include /get_started/macos/julia/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- End of julia -->\n\n            <div class=\"perl\">\n                {% markdown %}{% include /get_started/macos/perl/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- End of perl -->\n\n            <div class=\"cpp\">\n                {% markdown %}{% include /get_started/macos/cpp/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- END - C++-->\n        </div> <!-- END - MacOS -->\n\n\n\n        <div class=\"windows\">\n            <div class=\"python\">\n                 <!-- START - Windows Python CPU Installation Instructions -->\n                <div class=\"cpu\">\n                    <div class=\"pip\">\n                        {% markdown %}{% include /get_started/windows/python/cpu/pip.md %}{% endmarkdown %}\n                    </div> <!-- End of pip -->\n\n                    <div class=\"build-from-source\">\n                        {% markdown %}{% include /get_started/windows/python/cpu/build-from-source.md %}{% endmarkdown %}\n                    </div><!-- END of build from source -->\n\n                </div><!-- END of CPU -->\n                <!-- END - Windows Python CPU Installation Instructions -->\n\n                <!-- START - Windows Python GPU Installation Instructions -->\n                <div class=\"gpu\">\n                    <div class=\"pip\">\n                        {% markdown %}{% include /get_started/windows/python/gpu/pip.md %}{% endmarkdown %}\n                    </div> <!-- END of pip -->\n\n                    <div class=\"build-from-source\">\n                         {% markdown %}{% include /get_started/windows/python/gpu/build-from-source.md %}{% endmarkdown %}\n                    </div> <!-- END of build from source -->\n                </div> <!-- END of GPU -->\n            </div> <!-- END of Python -->\n            <!-- END - Windows Python Installation Instructions -->\n\n\n            <div class=\"r\">\n                {% markdown %}{% include /get_started/windows/r/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- END of R -->\n\n\n            <div class=\"scala\">\n                {% markdown %}{% include /get_started/windows/scala/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- End of scala -->\n\n\n            <div class=\"clojure\">\n                {% markdown %}{% include /get_started/windows/clojure/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- End of clojure -->\n\n\n            <div class=\"java\">\n                {% markdown %}{% include /get_started/windows/java/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- End of java -->\n\n            <div class=\"julia\">\n                {% markdown %}{% include /get_started/windows/julia/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- End of julia -->\n\n            <div class=\"perl\">\n                {% markdown %}{% include /get_started/windows/perl/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- End of perl -->\n\n            <div class=\"cpp\">\n                {% markdown %}{% include /get_started/windows/cpp/build-from-source.md %}{% endmarkdown %}\n            </div> <!-- END - C++-->\n        </div> <!-- END - Windows -->\n\n\n        <!-- START - Cloud Python Installation Instructions -->\n\n        <div class=\"cloud\">\n            <div class=\"gpu\">\n                {% markdown %}{% include /get_started/cloud/gpu.md %}{% endmarkdown %}\n            </div> <!-- END gpu -->\n\n            <div class=\"cpu\">\n                {% markdown %}{% include /get_started/cloud/cpu.md %}{% endmarkdown %}\n\n            </div> <!-- end cpu -->\n        </div> <!-- END - Cloud Python Installation Instructions -->\n\n\n        <!-- DEVICES -->\n        <div class=\"devices\">\n            <div class=\"raspberry-pi\">\n                {% markdown %}{% include /get_started/devices/raspberry_pi.md %}{% endmarkdown %}\n            </div> <!-- End of raspberry pi -->\n\n\n            <div class=\"nvidia-jetson\">\n                {% markdown %}{% include /get_started/devices/nvidia-jetson.md %}{% endmarkdown %}\n            </div> <!-- End of jetson -->\n        </div> <!-- End of devices -->\n    </div>\n</div>\n\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/gpu_snippet.md",
    "content": "\n\n**Important:** Make sure your installed CUDA (CUDNN/NCCL if applicable) version matches the CUDA version in the pip package.\n\nCheck your CUDA version with the following command:\n\n{% highlight bash %}\nnvcc --version\n{% endhighlight %}\n\nYou can either upgrade your CUDA install or install the MXNet package that supports your CUDA version."
  },
  {
    "path": "docs/static_site/src/_includes/get_started/linux/clojure/build-from-source.md",
    "content": "Please refer to the [MXNet-Clojure setup guide](https://github.com/apache/incubator-mxnet/tree/master/contrib/clojure-package) for a detailed set of instructions to help you with the setup process that is required to use the Clojure dependency.\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/linux/cpp/build-from-source.md",
    "content": "To use the C++ package, build from source the `USE_CPP_PACKAGE=1` option. Please\nrefer to the build from source instructions linked above.\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/linux/java/build-from-source.md",
    "content": "Previously available binaries distributed via Maven have been removed as they\nredistributed Category-X binaries in violation of Apache Software Foundation\n(ASF) policies.\n\nAt this point in time, no third-party binary Java packages are available. Please\nfollow the build from source instructions linked above.\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/linux/julia/build-from-source.md",
    "content": "Please follow the build from source instructions linked above.\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/linux/perl/build-from-source.md",
    "content": "Please follow the build from source instructions linked above.\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/linux/python/cpu/build-from-source.md",
    "content": "Please follow the build from source instructions linked above.\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/linux/python/cpu/docker.md",
    "content": "**WARNING**: the following links and names of binary distributions are provided for\nyour convenience but they point to packages that are *not* provided nor endorsed\nby the Apache Software Foundation. As such, they might contain software\ncomponents with more restrictive licenses than the Apache License and you'll\nneed to decide whether they are appropriate for your usage. Like all Apache\nReleases, the official Apache MXNet releases consist of source code\nonly and are found at\nthe [Download page](https://mxnet.apache.org/get_started/download).\n    \n\nDocker images with *MXNet* are available at [DockerHub](https://hub.docker.com/r/mxnet/).\nAfter you installed Docker on your machine, you can use them via:\n\n{% highlight bash %}\n$ docker pull mxnet/python\n{% endhighlight %}\n\nYou can list docker images to see if mxnet/python docker image pull was successful.\n\n{% highlight bash %}\n$ docker images # Use sudo if you skip Step 2\n\nREPOSITORY TAG IMAGE ID CREATED SIZE\nmxnet/python latest 00d026968b3c 3 weeks ago 1.41 GB\n{% endhighlight %}\n\nYou can then <a href=\"/get_started/validate_mxnet.html\">validate the installation</a>.\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/linux/python/cpu/pip.md",
    "content": "**WARNING**: the following PyPI package names are provided for your convenience but\nthey point to packages that are *not* provided nor endorsed by the Apache\nSoftware Foundation. As such, they might contain software components with more\nrestrictive licenses than the Apache License and you'll need to decide whether\nthey are appropriate for your usage. The packages linked here contain GPL GCC\nRuntime Library components. Like all Apache Releases, the official Apache MXNet\nreleases consist of source code only and are found at the [Download\npage](https://mxnet.apache.org/get_started/download).\n\nRun the following command:\n\n<div class=\"v1-9-1\">\n{% highlight bash %}\npip install mxnet\n{% endhighlight %}\n\n</div> <!-- End of v1-9-1 -->\n\n<div class=\"v1-8-0\">\n{% highlight bash %}\npip install mxnet==1.8.0.post0\n{% endhighlight %}\n\nStart from 1.7.0 release, oneDNN(previously known as: MKL-DNN/DNNL) is enabled\nin pip packages by default.\n\noneAPI Deep Neural Network Library (oneDNN) is an open-source cross-platform\nperformance library of basic building blocks for deep learning applications.\nThe library is optimized for Intel Architecture Processors, Intel Processor\nGraphics and Xe architecture-based Graphics. Support for other architectures\nsuch as Arm* 64-bit Architecture (AArch64) and OpenPOWER* Power ISA (PPC64) is\nexperimental.\n\noneDNN is intended for deep learning applications and framework developers\ninterested in improving application performance on Intel CPUs and GPUs, more\ndetails can be found <a href=\"https://github.com/oneapi-src/oneDNN\">here</a>.\n\nYou can find performance numbers in the\n<a href=\"https://mxnet.apache.org/versions/1.8.0/api/faq/perf.html#intel-cpu\">\nMXNet tuning guide</a>.\n\nTo install native MXNet without oneDNN, run the following command:\n\n{% highlight bash %}\npip install mxnet-native==1.8.0.post0\n{% endhighlight %}\n\n</div> <!-- End of v1-8-0 -->\n\n<div class=\"v1-7-0\">\n{% highlight bash %}\npip install mxnet==1.7.0.post2\n{% endhighlight %}\n\nStart from 1.7.0 release, oneDNN(previously known as: MKL-DNN/DNNL) is enabled\nin pip packages by default.\n\noneAPI Deep Neural Network Library (oneDNN) is an open-source cross-platform\nperformance library of basic building blocks for deep learning applications.\nThe library is optimized for Intel Architecture Processors, Intel Processor\nGraphics and Xe architecture-based Graphics. Support for other architectures\nsuch as Arm* 64-bit Architecture (AArch64) and OpenPOWER* Power ISA (PPC64) is\nexperimental.\n\noneDNN is intended for deep learning applications and framework developers\ninterested in improving application performance on Intel CPUs and GPUs, more\ndetails can be found <a href=\"https://github.com/oneapi-src/oneDNN\">here</a>.\n\nYou can find performance numbers in the\n<a href=\"https://mxnet.apache.org/versions/1.7.0/api/faq/perf.html#intel-cpu\">\nMXNet tuning guide</a>.\n\nTo install native MXNet without oneDNN, run the following command:\n\n{% highlight bash %}\npip install mxnet-native==1.7.0\n{% endhighlight %}\n\n</div> <!-- End of v1-7-0 -->\n\n<div class=\"v1-6-0\">\n{% highlight bash %}\npip install mxnet==1.6.0\n{% endhighlight %}\n\nMKL-DNN enabled pip packages are optimized for Intel hardware. You can find\nperformance numbers in the\n<a href=\"https://mxnet.apache.org/versions/1.6/api/faq/perf.html#intel-cpu\">\nMXNet tuning guide</a>.\n\n{% highlight bash %}\npip install mxnet-mkl==1.6.0\n{% endhighlight %}\n\n</div> <!-- End of v1-6-0 -->\n\n<div class=\"v1-5-1\">\n{% highlight bash %}\npip install mxnet==1.5.1\n{% endhighlight %}\n\nMKL-DNN enabled pip packages are optimized for Intel hardware. You can find\nperformance numbers in the\n<a href=\"https://mxnet.apache.org/versions/1.6/api/faq/perf.html#intel-cpu\">\nMXNet tuning guide</a>.\n\n{% highlight bash %}\npip install mxnet-mkl==1.5.1\n{% endhighlight %}\n\n</div> <!-- End of v1-5-1 -->\n\n<div class=\"v1-4-1\">\n\n{% highlight bash %}\npip install mxnet==1.4.1\n{% endhighlight %}\n\nMKL-DNN enabled pip packages are optimized for Intel hardware. You can find\nperformance numbers in the\n<a href=\"https://mxnet.apache.org/versions/1.6/api/faq/perf.html#intel-cpu\">\nMXNet tuning guide</a>.\n\n{% highlight bash %}\npip install mxnet-mkl==1.4.1\n{% endhighlight %}\n\n</div> <!-- End of v1-4-1 -->\n<div class=\"v1-3-1\">\n\n{% highlight bash %}\npip install mxnet==1.3.1\n{% endhighlight %}\n\nMKL-DNN enabled pip packages are optimized for Intel hardware. You can find\nperformance numbers in the\n<a href=\"https://mxnet.apache.org/versions/1.6/api/faq/perf.html#intel-cpu\">\nMXNet tuning guide</a>.\n\n{% highlight bash %}\npip install mxnet-mkl==1.3.1\n{% endhighlight %}\n\n</div> <!-- End of v1-3-1 -->\n<div class=\"v1-2-1\">\n\n{% highlight bash %}\npip install mxnet==1.2.1\n{% endhighlight %}\n\nMKL-DNN enabled pip packages are optimized for Intel hardware. You can find\nperformance numbers in the\n<a href=\"https://mxnet.apache.org/versions/1.6/api/faq/perf.html#intel-cpu\">\nMXNet tuning guide</a>.\n\n{% highlight bash %}\npip install mxnet-mkl==1.2.1\n{% endhighlight %}\n\n</div> <!-- End of v1-2-1 -->\n\n<div class=\"v1-1-0\">\n\n{% highlight bash %}\npip install mxnet==1.1.0\n{% endhighlight %}\n\n</div> <!-- End of v1-1-0-->\n\n<div class=\"v1-0-0\">\n\n{% highlight bash %}\npip install mxnet==1.0.0\n{% endhighlight %}\n\n</div> <!-- End of v1-0-0-->\n\n\n<div class=\"v0-12-1\">\n\n{% highlight bash %}\npip install mxnet==0.12.1\n{% endhighlight %}\n\nFor MXNet 0.12.0:\n\n{% highlight bash %}\npip install mxnet==0.12.0\n{% endhighlight %}\n\n</div> <!-- End of v0-12-1-->\n\n<div class=\"v0-11-0\">\n\n{% highlight bash %}\npip install mxnet==0.11.0\n{% endhighlight %}\n\n</div> <!-- End of v0-11-0-->\n\n<br>\n\n\n{% include /get_started/pip_snippet.md %}\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/linux/python/gpu/build-from-source.md",
    "content": "Please follow the build from source instructions linked above.\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/linux/python/gpu/docker.md",
    "content": "**WARNING**: the following links and names of binary distributions are provided for\nyour convenience but they point to packages that are *not* provided nor endorsed\nby the Apache Software Foundation. As such, they might contain software\ncomponents with more restrictive licenses than the Apache License and you'll\nneed to decide whether they are appropriate for your usage. The packages linked\nhere contain proprietary parts of the NVidia CUDA SDK and GPL GCC Runtime\nLibrary components. Like all Apache Releases, the official Apache MXNet\nreleases consist of source code only and are found at the [Download\npage](https://mxnet.apache.org/get_started/download).\n\nDocker images with *MXNet* are available at [DockerHub](https://hub.docker.com/r/mxnet/).\n\nPlease follow the [NVidia Docker installation\ninstructions](https://github.com/NVIDIA/nvidia-docker/wiki) to enable the usage\nof GPUs from the docker containers.\n\nAfter you installed Docker on your machine, you can use them via:\n\n{% highlight bash %}\n$ docker pull mxnet/python:gpu # Use sudo if you skip Step 2\n{% endhighlight %}\n\nYou can list docker images to see if mxnet/python docker image pull was successful.\n\n{% highlight bash %}\n$ docker images # Use sudo if you skip Step 2\n\nREPOSITORY TAG IMAGE ID CREATED SIZE\nmxnet/python gpu 493b2683c269 3 weeks ago 4.77 GB\n{% endhighlight %}\n\nYou can then <a href=\"/get_started/validate_mxnet.html\">validate the installation</a>.\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/linux/python/gpu/pip.md",
    "content": "**WARNING**: the following PyPI package names are provided for your convenience but\nthey point to packages that are *not* provided nor endorsed by the Apache\nSoftware Foundation. As such, they might contain software components with more\nrestrictive licenses than the Apache License and you'll need to decide whether\nthey are appropriate for your usage. The packages linked here contain\nproprietary parts of the NVidia CUDA SDK and GPL GCC Runtime Library components.\nLike all Apache Releases, the official Apache MXNet releases\nconsist of source code only and are found at the [Download\npage](https://mxnet.apache.org/get_started/download).\n\n**PREREQUISITES**: [CUDA](https://developer.nvidia.com/cuda-downloads) should be installed first. Starting from version 1.8.0, [CUDNN](https://developer.nvidia.com/cudnn) and [NCCL](https://developer.nvidia.com/nccl) should be installed as well.\n\nRun the following command:\n\n<div class=\"v1-9-1\">\n{% highlight bash %}\n$ pip install mxnet-cu102\n{% endhighlight %}\n\n</div> <!-- End of v1-9-1 -->\n\n<div class=\"v1-8-0\">\n{% highlight bash %}\n$ pip install mxnet-cu102==1.8.0.post0\n{% endhighlight %}\n\n</div> <!-- End of v1-8-0 -->\n\n<div class=\"v1-7-0\">\n{% highlight bash %}\n$ pip install mxnet-cu102==1.7.0\n{% endhighlight %}\n\n</div> <!-- End of v1-7-0 -->\n\n<div class=\"v1-6-0\">\n{% highlight bash %}\n$ pip install mxnet-cu102==1.6.0.post0\n{% endhighlight %}\n\n</div> <!-- End of v1-6-0 -->\n\n<div class=\"v1-5-1\">\n{% highlight bash %}\n$ pip install mxnet-cu101==1.5.1\n{% endhighlight %}\n\n</div> <!-- End of v1-5-1 -->\n<div class=\"v1-4-1\">\n\n{% highlight bash %}\n$ pip install mxnet-cu101==1.4.1\n{% endhighlight %}\n\n</div> <!-- End of v1-4-1 -->\n<div class=\"v1-3-1\">\n\n{% highlight bash %}\n$ pip install mxnet-cu92==1.3.1\n{% endhighlight %}\n\n</div> <!-- End of v1-3-1-->\n<div class=\"v1-2-1\">\n\n{% highlight bash %}\n$ pip install mxnet-cu92==1.2.1\n{% endhighlight %}\n\n</div> <!-- End of v1-2-1-->\n\n<div class=\"v1-1-0\">\n\n{% highlight bash %}\n$ pip install mxnet-cu91==1.1.0\n{% endhighlight %}\n\n</div> <!-- End of v1-1-0-->\n\n<div class=\"v1-0-0\">\n\n{% highlight bash %}\n$ pip install mxnet-cu90==1.0.0\n{% endhighlight %}\n\n</div> <!-- End of v1-0-0-->\n\n<div class=\"v0-12-1\">\n\n{% highlight bash %}\n$ pip install mxnet-cu90==0.12.1\n{% endhighlight %}\n\n</div> <!-- End of v0-12-1-->\n\n<div class=\"v0-11-0\">\n\n{% highlight bash %}\n$ pip install mxnet-cu80==0.11.0\n{% endhighlight %}\n\n</div> <!-- End of v0-11-0-->\n\n<br>\n\n{% include /get_started/pip_snippet.md %}\n{% include /get_started/gpu_snippet.md %}\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/linux/r/build-from-source.md",
    "content": "You will need to R v3.4.4+ and build MXNet from source. Please follow the\ninstructions linked above.\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/linux/scala/build-from-source.md",
    "content": "Prebuilt binaries distributed via Maven have been removed as they redistributed\nCategory-X binaries in violation of Apache Software Foundation (ASF) policies.\nIf you would like to help re-do the binary releases in an ASF-compliant manner,\nplease reach out via one of the [developer communications\nchannels](https://mxnet.apache.org/community/contribute#mxnet-dev-communications).\nUntil then, please follow the build from source instructions linked below.\n"
  },
  {
    "path": "docs/static_site/src/_includes/get_started/pip_snippet.md",
    "content": "You can then <a href=\"/get_started/validate_mxnet.html\">validate your MXNet installation</a>.\n\n<div style=\"text-align: center\">\n    <img src=\"{{ \"/assets/img/pip-packages-1.9.1.png\" | relative_url }}\"\n    alt=\"pip packages\"/>\n</div>\n\n**NOTES:**\n\n*mxnet-cu101* means the package is built with CUDA/cuDNN and the CUDA version is\n10.1.\n\nAll MKL pip packages are experimental prior to version 1.3.0.\n"
  },
  {
    "path": "docs/static_site/src/_includes/head.html",
    "content": "<head>\n  <meta charset=\"utf-8\">\n  <meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\">\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n  <link href=\"{{ \"/assets/img/mxnet-icon.png\" | relative_url }}\" rel=\"icon\" type=\"image/png\">\n  {%- seo -%}\n  {%- if jekyll.environment == 'production' -%}\n    <link rel=\"stylesheet\" href=\"{{ \"/assets/docsearch.min.css\" | relative_url }}\" />\n  {%- else -%}\n    <link rel=\"stylesheet\" href=\"https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.css\" />\n  {%- endif -%}\n  <link rel=\"stylesheet\" href=\"{{ \"/assets/main.css\" | relative_url }}\">\n  {%- feed_meta -%}\n  {%- if jekyll.environment == 'production' -%}\n    {%- include matomo-analytics.html -%}\n  {%- endif -%}\n  {%- if jekyll.environment == 'production' -%}\n    <script src=\"{{'/assets/js/jquery-3.3.1.min.js'|relative_url}}\"></script>\n    <script src=\"{{ \"/assets/js/docsearch.min.js\" | relative_url }}\"></script>\n  {%- else -%}\n    <script src=\"https://code.jquery.com/jquery-3.3.1.min.js\" integrity=\"sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=\" crossorigin=\"anonymous\"></script>\n    <script src=\"https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.js\" defer></script>\n  {%- endif -%}\n  <script src=\"{{'/assets/js/globalSearch.js'|relative_url}}\" defer></script>\n  <script src=\"{{'/assets/js/clipboard.js'|relative_url}}\" defer></script>\n  <script src=\"{{'/assets/js/copycode.js'|relative_url}}\" defer></script>\n  {%- if page.feedback == true and jekyll.environment == \"production\" -%}\n    <script src=\"{{'/assets/js/feedback.js'|relative_url}}\" defer></script>\n  {%- endif -%}\n</head>\n"
  },
  {
    "path": "docs/static_site/src/_includes/header.html",
    "content": "<header class=\"site-header\" role=\"banner\">\n\n  <script>\n    $(document).ready(function () {\n\n      // HEADER OPACITY LOGIC\n\n      function opacity_header() {\n        var value = \"rgba(4,140,204,\" + ($(window).scrollTop() / 300 + 0.4) + \")\"\n        $('.site-header').css(\"background-color\", value)\n      }\n\n      $(window).scroll(function () {\n        opacity_header()\n      })\n      opacity_header();\n\n      // MENU SELECTOR LOGIC\n      $('.page-link').each( function () {\n        if (window.location.href.includes(this.href)) {\n          $(this).addClass(\"page-current\");\n        }\n      });\n    })\n  </script>\n  <div class=\"wrapper\">\n    <a class=\"site-title\" rel=\"author\" href=\"{{ '/' | relative_url }}\"><img\n            src=\"{{'/assets/img/mxnet_logo.png' | relative_url }}\" class=\"site-header-logo\"></a>\n    <nav class=\"site-nav\">\n      <input type=\"checkbox\" id=\"nav-trigger\" class=\"nav-trigger\"/>\n      <label for=\"nav-trigger\">\n          <span class=\"menu-icon\">\n            <svg viewBox=\"0 0 18 15\" width=\"18px\" height=\"15px\">\n              <path d=\"M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z\"/>\n            </svg>\n          </span>\n      </label>\n      <div class=\"gs-search-border\">\n        <div id=\"gs-search-icon\"></div>\n        <form id=\"global-search-form\">\n          <input id=\"global-search\" type=\"text\" title=\"Search\" placeholder=\"Search\" />\n          <div id=\"global-search-dropdown-container\">\n            <button class=\"gs-current-version btn\" type=\"button\" data-toggle=\"dropdown\">\n                <span id=\"gs-current-version-label\">{{site.versions[0]}}</span>\n                <svg class=\"gs-dropdown-caret\" viewBox=\"0 0 32 32\" class=\"icon icon-caret-bottom\" aria-hidden=\"true\">\n                    <path class=\"dropdown-caret-path\" d=\"M24 11.305l-7.997 11.39L8 11.305z\"></path>\n                </svg>\n            </button>\n            <ul class=\"gs-opt-group gs-version-dropdown\">\n              {% for version in site.versions %}\n                {% if version == site.versions[0] %}\n                  <li class=\"gs-opt gs-versions active\">{{version}}</li>\n                {% else %}\n                  <li class=\"gs-opt gs-versions\">{{version}}</li>\n                {% endif %}\n              {% endfor %}\n            </ul>\n        </div>\n          <span id=\"global-search-close\">x</span>\n        </form>\n      </div>\n      <div class=\"trigger\">\n        <div id=\"global-search-mobile-border\">\n          <div id=\"gs-search-icon-mobile\"></div>\n          <input id=\"global-search-mobile\" placeholder=\"Search...\" type=\"text\"/>\n          <div id=\"global-search-dropdown-container-mobile\">\n            <button class=\"gs-current-version-mobile btn\" type=\"button\" data-toggle=\"dropdown\">\n                <svg class=\"gs-dropdown-caret\" viewBox=\"0 0 32 32\" class=\"icon icon-caret-bottom\" aria-hidden=\"true\">\n                    <path class=\"dropdown-caret-path\" d=\"M24 11.305l-7.997 11.39L8 11.305z\"></path>\n                </svg>\n            </button>\n            <ul class=\"gs-opt-group gs-version-dropdown-mobile\">\n              {% for version in site.versions %}\n                {% if version == site.versions[0] %}\n                  <li class=\"gs-opt gs-versions active\">{{version}}</li>\n                {% else %}\n                  <li class=\"gs-opt gs-versions\">{{version}}</li>\n                {% endif %}\n              {% endfor %}\n            </ul>\n        </div>\n        </div>\n        <a class=\"page-link\" href=\"{{'/get_started' | relative_url }}\">Get Started</a>\n        <a class=\"page-link\" href=\"{{'/features' | relative_url }}\">Features</a>\n        <a class=\"page-link\" href=\"{{'/ecosystem' | relative_url }}\">Ecosystem</a>\n        <a class=\"page-link\" href=\"{{'/api' | relative_url }}\">Docs & Tutorials</a>\n        <a class=\"page-link\" href=\"{{'/trusted_by' | relative_url }}\">Trusted By</a>\n        <a class=\"page-link\" href=\"https://github.com/apache/mxnet\">GitHub</a>\n        <div class=\"dropdown\" style=\"min-width:100px\">\n          <span class=\"dropdown-header\">Apache\n            <svg class=\"dropdown-caret\" viewBox=\"0 0 32 32\" class=\"icon icon-caret-bottom\" aria-hidden=\"true\"><path class=\"dropdown-caret-path\" d=\"M24 11.305l-7.997 11.39L8 11.305z\"></path></svg>\n          </span>\n          <div class=\"dropdown-content\" style=\"min-width:250px\">\n            <a href=\"https://www.apache.org/foundation/\">Apache Software Foundation</a>\n            <a href=\"https://www.apache.org/licenses/\">License</a>\n            <a href=\"{{ '/api/faq/security.html' | relative_url }}\">Security</a>\n            <a href=\"https://privacy.apache.org/policies/privacy-policy-public.html\">Privacy</a>\n            <a href=\"https://www.apache.org/events/current-event\">Events</a>\n            <a href=\"https://www.apache.org/foundation/sponsorship.html\">Sponsorship</a>\n            <a href=\"https://www.apache.org/foundation/thanks.html\">Thanks</a>\n          </div>\n        </div>\n        <div class=\"dropdown\">\n          <span class=\"dropdown-header\">master\n            <svg class=\"dropdown-caret\" viewBox=\"0 0 32 32\" class=\"icon icon-caret-bottom\" aria-hidden=\"true\"><path class=\"dropdown-caret-path\" d=\"M24 11.305l-7.997 11.39L8 11.305z\"></path></svg>\n          </span>\n          <div class=\"dropdown-content\">\n            {% for version in site.versions %}\n              {% if version == site.versions[0] %}\n                <a class=\"dropdown-option-active\" href=\"/\">{{version}}</a>\n              {% else %}\n                <a href=\"/versions/{{version}}/\">{{version}}</a>\n              {% endif %}\n            {% endfor %}\n          </div>\n        </div>\n      </div>\n    </nav>\n  </div>\n</header>\n"
  },
  {
    "path": "docs/static_site/src/_includes/icon-github.html",
    "content": "<a href=\"https://github.com/{{ include.username }}\"><span class=\"icon icon--github\">{% include icon-github.svg %}</span><span class=\"username\">{{ include.username }}</span></a>\n"
  },
  {
    "path": "docs/static_site/src/_includes/icon-twitter.html",
    "content": "<a href=\"https://twitter.com/{{ include.username }}\"><span class=\"icon icon--twitter\">{% include icon-twitter.svg %}</span><span class=\"username\">{{ include.username }}</span></a>\n"
  },
  {
    "path": "docs/static_site/src/_includes/important.html",
    "content": "<div markdown=\"span\" class=\"alert alert-warning\" role=\"alert\"><i class=\"fa fa-warning\"></i> <b>Important:</b> {{include.content}}</div>"
  },
  {
    "path": "docs/static_site/src/_includes/matomo-analytics.html",
    "content": "<!-- Matomo -->\n<script>\n  var _paq = window._paq = window._paq || [];\n  /* tracker methods like \"setCustomDimension\" should be called before \"trackPageView\" */\n  /* We explicitly disable cookie tracking to avoid privacy issues */\n  _paq.push(['disableCookies']);\n  _paq.push(['trackPageView']);\n  _paq.push(['enableLinkTracking']);\n  (function() {\n    var u=\"https://analytics.apache.org/\";\n    _paq.push(['setTrackerUrl', u+'matomo.php']);\n    _paq.push(['setSiteId', '23']);\n    var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];\n    g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);\n  })();\n</script>\n<!-- End Matomo Code -->\n\n"
  },
  {
    "path": "docs/static_site/src/_includes/note.html",
    "content": "<div markdown=\"span\" class=\"alert alert-info\" role=\"alert\"><i class=\"fa fa-info-circle\"></i> <b>Note:</b> {{include.content}}</div>\n"
  },
  {
    "path": "docs/static_site/src/_includes/social.html",
    "content": "<ul class=\"social-media-list\">\n  {%- if site.dribbble_username -%}<li><a href=\"https://dribbble.com/{{ site.dribbble_username| cgi_escape | escape }}\"><svg class=\"svg-icon\"><use xlink:href=\"{{ '/assets/minima-social-icons.svg#dribbble' | relative_url }}\"></use></svg> <span class=\"username\">{{ site.dribbble_username| escape }}</span></a></li>{%- endif -%}\n  {%- if site.facebook_username -%}<li><a href=\"https://www.facebook.com/{{ site.facebook_username| cgi_escape | escape }}\"><svg class=\"svg-icon\"><use xlink:href=\"{{ '/assets/minima-social-icons.svg#facebook' | relative_url }}\"></use></svg> <span class=\"username\">{{ site.facebook_username| escape }}</span></a></li>{%- endif -%}\n  {%- if site.flickr_username -%}<li><a href=\"https://www.flickr.com/photos/{{ site.flickr_username| cgi_escape | escape }}\"><svg class=\"svg-icon\"><use xlink:href=\"{{ '/assets/minima-social-icons.svg#flickr' | relative_url }}\"></use></svg> <span class=\"username\">{{ site.flickr_username| escape }}</span></a></li>{%- endif -%}\n  {%- if site.github_username -%}<li><a href=\"https://github.com/{{ site.github_username }}\"><svg class=\"svg-icon\"><use xlink:href=\"{{ '/assets/minima-social-icons.svg#github' | relative_url }}\"></use></svg> <span class=\"username\">{{ site.github_username| escape }}</span></a></li>{%- endif -%}\n  {%- if site.instagram_username -%}<li><a href=\"https://instagram.com/{{ site.instagram_username| cgi_escape | escape }}\"><svg class=\"svg-icon\"><use xlink:href=\"{{ '/assets/minima-social-icons.svg#instagram' | relative_url }}\"></use></svg> <span class=\"username\">{{ site.instagram_username| escape }}</span></a></li>{%- endif -%}\n  {%- if site.linkedin_username -%}<li><a href=\"https://www.linkedin.com/in/{{ site.linkedin_username| cgi_escape | escape }}\"><svg class=\"svg-icon\"><use xlink:href=\"{{ '/assets/minima-social-icons.svg#linkedin' | relative_url }}\"></use></svg> <span class=\"username\">{{ site.linkedin_username| escape }}</span></a></li>{%- endif -%}\n  {%- if site.pinterest_username -%}<li><a href=\"https://www.pinterest.com/{{ site.pinterest_username| cgi_escape | escape }}\"><svg class=\"svg-icon\"><use xlink:href=\"{{ '/assets/minima-social-icons.svg#pinterest' | relative_url }}\"></use></svg> <span class=\"username\">{{ site.pinterest_username| escape }}</span></a></li>{%- endif -%}\n  {%- for mst in site.mastodon -%}{%- if mst.username and mst.instance -%}<li><a href=\"https://{{ mst.instance| cgi_escape | escape}}/@{{mst.username}}\"><svg class=\"svg-icon\"><use xlink:href=\"{{ '/assets/minima-social-icons.svg#mastodon' | relative_url }}\"></use></svg> <span class=\"username\">{{ mst.username|escape }}</span></a></li>{%- endif -%}{%- endfor -%}\n  {%- if site.twitter_username -%}<li><a href=\"https://www.twitter.com/{{ site.twitter_username| cgi_escape | escape }}\"><svg class=\"svg-icon\"><use xlink:href=\"{{ '/assets/minima-social-icons.svg#twitter' | relative_url }}\"></use></svg> <span class=\"username\">{{ site.twitter_username| escape }}</span></a></li>{%- endif -%}\n  {%- if site.youtube_username -%}<li><a href=\"https://youtube.com/{{ site.youtube_username| cgi_escape | escape }}\"><svg class=\"svg-icon\"><use xlink:href=\"{{ '/assets/minima-social-icons.svg#youtube' | relative_url }}\"></use></svg> <span class=\"username\">{{ site.youtube_username| escape }}</span></a></li>{%- endif -%}\n  {%- if site.googleplus_username -%}<li><a href=\"https://plus.google.com/{{ site.googleplus_username| escape }}\"><svg class=\"svg-icon\"><use xlink:href=\"{{ '/assets/minima-social-icons.svg#googleplus' | relative_url }}\"></use></svg> <span class=\"username\">{{ site.googleplus_username| escape }}</span></a></li>{%- endif -%}\n  {%- if site.rss -%}<li><a href=\"{{ 'feed.xml' | relative_url }}\"><svg class=\"svg-icon\"><use xlink:href=\"{{ '/assets/minima-social-icons.svg#rss' | relative_url }}\"></use></svg> <span>{{ site.rss | escape }}</span></a></li>{%- endif -%}\n</ul>\n"
  },
  {
    "path": "docs/static_site/src/_includes/tip.html",
    "content": "<div markdown=\"span\" class=\"alert alert-success\" role=\"alert\"><i class=\"fa fa-check-square-o\"></i> <b>Tip:</b> {{include.content}}</div>"
  },
  {
    "path": "docs/static_site/src/_includes/warning.html",
    "content": "<div markdown=\"span\" class=\"alert alert-danger\" role=\"alert\"><i class=\"fa fa-exclamation-circle\"></i> <b>Warning:</b> {{include.content}}</div>"
  },
  {
    "path": "docs/static_site/src/_layouts/default.html",
    "content": "<!DOCTYPE html>\n\n<!---\n  Licensed to the Apache Software Foundation (ASF) under one\n  or more contributor license agreements.  See the NOTICE file\n  distributed with this work for additional information\n  regarding copyright ownership.  The ASF licenses this file\n  to you under the Apache License, Version 2.0 (the\n  \"License\"); you may not use this file except in compliance\n  with the License.  You may obtain a copy of the License at\n    http://www.apache.org/licenses/LICENSE-2.0\n  Unless required by applicable law or agreed to in writing,\n  software distributed under the License is distributed on an\n  \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  KIND, either express or implied.  See the License for the\n  specific language governing permissions and limitations\n  under the License.\n-->\n\n<html lang=\"{{ page.lang | default: site.lang | default: \" en\" }}\">\n\n{%- include head.html -%}\n\n<body>\n\n{%- include header.html -%}\n\n<main class=\"page-content\" aria-label=\"Content\">\n    {{ content }}\n</main>\n\n{%- include footer.html -%}\n\n</body>\n\n</html>\n"
  },
  {
    "path": "docs/static_site/src/_layouts/home.html",
    "content": "<!DOCTYPE html>\n\n<!---\n  Licensed to the Apache Software Foundation (ASF) under one\n  or more contributor license agreements.  See the NOTICE file\n  distributed with this work for additional information\n  regarding copyright ownership.  The ASF licenses this file\n  to you under the Apache License, Version 2.0 (the\n  \"License\"); you may not use this file except in compliance\n  with the License.  You may obtain a copy of the License at\n    http://www.apache.org/licenses/LICENSE-2.0\n  Unless required by applicable law or agreed to in writing,\n  software distributed under the License is distributed on an\n  \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  KIND, either express or implied.  See the License for the\n  specific language governing permissions and limitations\n  under the License.\n-->\n\n<html lang=\"{{ page.lang | default: site.lang | default: \" en\" }}\">\n\n{%- include head.html -%}\n\n<body>\n\n{%- include header.html -%}\n\n<main class=\"page-content-home\" aria-label=\"Content\">\n    <div class=\"wrapper\">\n        <div class=\"home\">\n            <h1 style=\"margin-bottom:40px;\">APACHE MXNET:<br><span style=\"margin-left:75px\"> A FLEXIBLE AND EFFICIENT</span><br> <span\n                    style=\"margin-left:150px\"> LIBRARY FOR DEEP LEARNING</span>\n            </h1>\n\n            <p style=\"margin-bottom:40px;\">A truly open source deep learning framework suited<br>for flexible research\n                prototyping and\n                production.</p>\n            <div>\n                <a href=\"{{'/get_started' | relative_url }}\" class=\"btn\" style=\"float:left; margin-bottom:20px; margin-top:0px\">Get Started\n                    <span class=\"span-accented\">›</span></a>\n            </div>\n\n        </div>\n\n\n    </div>\n    </div>\n    <div class=\"key-features-section section\">\n        <div class=\"wrapper\">\n            <div class=\"row\">\n                <div class=\"col-8\">\n                    <h2>Key Features &amp;<br>Capabilities</h2>\n                </div>\n                <div class=\"col-4\">\n                    <div>\n                        <a href=\"{{'/features' | relative_url}}\" class=\"btn btn-action\">All Features <span class=\"span-accented\">›</span></a>\n                    </div>\n                </div>\n            </div>\n\n            <div class=\"row key-features\">\n                {{ layout.landing }}\n                {%- for feature in page.key_features -%}\n                <div class=\"col-3\">\n                    <div class=\"card\">\n                        <div class=\"card-text\">\n                            <h3>{{feature.title}}</h3>\n                            <p>{{feature.text}}</p>\n                        </div>\n                        <div class=\"key-feature-image\">\n                            <img width=50px src=\"{{feature.icon | relative_url}}\">\n                        </div>\n                    </div>\n                </div>\n                {%- endfor -%}\n            </div>\n\n        </div>\n    </div>\n    <div class=\"ecosystem-section section\">\n        <div class=\"wrapper\">\n            <div class=\"row\">\n                <div class=\"col-8\">\n                    <h2>Ecosystem </h2>\n                </div>\n                <div class=\"col-4\">\n                    <div>\n                        <a href=\"{{'/ecosystem' | relative_url}}\" class=\"btn btn-action\">All Projects <span\n                                class=\"span-accented\">›</span></a>\n                    </div>\n                </div>\n                <div class=\"col-8\">\n                    <p>Explore a rich ecosystem of libraries, tools, and more to support development.</p>\n                </div>\n            </div>\n\n            <div class=\"row\">\n                {%- for feature in page.ecosystem -%}\n                <div class=\"col-3\">\n                    <div class=\"card\">\n                        <a href=\"{{feature.link}}\">\n                            <div class=\"card-text\">\n                                <div class=\"card-header-title\">\n                                    <h3>{{feature.title}}</h3>\n                                    <img src=\"{{feature.icon | relative_url}}\">\n                                </div>\n                                <p class=\"card-summary\">{{feature.text}}</p>\n                            </div>\n                        </a>\n                    </div>\n                </div>\n                {%- endfor -%}\n            </div>\n        </div>\n    </div>\n\n    <div class=\"community-section section \">\n        <div class=\"wrapper\">\n            <div class=\"row\">\n                <div class=\"col-6\">\n                    <h2>Community</h2>\n                </div>\n                <div class=\"col-8\">\n                    <p>Join the Apache MXNet scientific community to contribute, learn, and get\n                        answers to your questions.</p>\n                </div>\n            </div>\n\n            <div class=\"row\">\n                {%- for feature in page.community -%}\n                <div class=\"col-4\">\n                    <div class=\"card\">\n                        <a href=\"{{feature.link}}\">\n                            <div>\n                                <div class=\"card-header-title\">\n                                    <h3>{{feature.title}}</h3>\n                                    <img src=\"{{feature.icon | relative_url}}\">\n                                </div>\n                                <p class=\"card-summary\">{{feature.text}}</p>\n                            </div>\n                        </a>\n                    </div>\n                </div>\n                {%- endfor -%}\n            </div>\n        </div>\n    </div>\n</main>\n\n{%- include footer.html -%}\n\n</body>\n\n</html>\n"
  },
  {
    "path": "docs/static_site/src/_layouts/page.html",
    "content": "---\n  # Licensed to the Apache Software Foundation (ASF) under one\n  # or more contributor license agreements.  See the NOTICE file\n  # distributed with this work for additional information\n  # regarding copyright ownership.  The ASF licenses this file\n  # to you under the Apache License, Version 2.0 (the\n  # \"License\"); you may not use this file except in compliance\n  # with the License.  You may obtain a copy of the License at\n  #   http://www.apache.org/licenses/LICENSE-2.0\n  # Unless required by applicable law or agreed to in writing,\n  # software distributed under the License is distributed on an\n  # \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  # KIND, either express or implied.  See the License for the\n  # specific language governing permissions and limitations\n  # under the License.\n\nlayout: default\n---\n<script>\n\n</script>\n<article class=\"post\">\n\n    <header class=\"post-header wrapper\">\n        <h1 class=\"post-title\">{{ page.title | escape }}</h1>\n        <h3>{{page.subtitle}}</h3>\n\n        {%- if page.action -%}\n        <a style=\"float:left; margin-top:20px\" href=\"{{page.action_url | relative_url}}\" class=\"btn btn-action\">{{page.action}}\n            <span class=\"span-accented\">›</span></a>\n        {%- endif -%}\n    </header>\n\n    <div class=\"post-content\">\n        <div class=\"wrapper\">\n            {{ content }}\n        </div>\n    </div>\n\n</article>\n"
  },
  {
    "path": "docs/static_site/src/_layouts/page_api.html",
    "content": "---\n  # Licensed to the Apache Software Foundation (ASF) under one\n  # or more contributor license agreements.  See the NOTICE file\n  # distributed with this work for additional information\n  # regarding copyright ownership.  The ASF licenses this file\n  # to you under the Apache License, Version 2.0 (the\n  # \"License\"); you may not use this file except in compliance\n  # with the License.  You may obtain a copy of the License at\n  #   http://www.apache.org/licenses/LICENSE-2.0\n  # Unless required by applicable law or agreed to in writing,\n  # software distributed under the License is distributed on an\n  # \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  # KIND, either express or implied.  See the License for the\n  # specific language governing permissions and limitations\n  # under the License.\n\nlayout: page\n---\n<div class=\"row\">\n    <div class=\"col-3 docs-side-bar\">\n\n        {% for p in site.pages %}\n        {% if p.tag == 'main_docs' %}\n        {% for doc in p.docs %}\n        {% if doc.tag == page.tag %}\n        <div class=\"docs-card docs-side\">\n            <ul>\n                <div class=\"docs-action-btn\">\n                    <a href=\"{{doc.guide_link | relative_url}}\"> <img src=\"{{'assets/img/compass.svg' | relative_url}}\"\n                                                                      class=\"docs-logo-docs\">{{doc.title}} Guide <span\n                            class=\"span-accented\">›</span></a>\n                </div>\n                <div class=\"docs-action-btn\">\n                    <a href=\"{{doc.tutorial_link | relative_url}}\"> <img\n                            src=\"{{'assets/img/video-tutorial.svg' | relative_url}}\" class=\"docs-logo-docs\">{{doc.title}}\n                        Tutorials <span class=\"span-accented\">›</span></a>\n                </div>\n                <div class=\"docs-action-btn\">\n                    <a href=\"{{doc.api_link | relative_url}}\"> <img src=\"{{'assets/img/api.svg' | relative_url}}\"\n                                                                    class=\"docs-logo-docs\">{{doc.title}} API Reference\n                        <span class=\"span-accented\">›</span></a>\n                </div>\n\n                <!-- Let's show the list of tutorials -->\n                <br>\n                {% if page.is_tutorial == true %}\n                <h3>Tutorials</h3>\n                {% for p in site.pages %}\n                {% if p.is_tutorial == true %}\n                {% if page.tag == p.tag %}\n                <li><a href=\"{{ p.url  | relative_url }}\">{{ p.title }}</a></li>\n                {% endif %}  <!-- page-category -->\n                {% endif %}   <!-- resource-p -->\n                {% endfor %}  <!-- page -->\n                {% endif %}\n            </ul>\n        </div>\n        {% endif %}\n        {% endfor %}\n        {% endif %}   <!-- resource-p -->\n        {% endfor %}  <!-- page -->\n        </ul>\n    </div>\n    <div class=\"col-9\">\n        {{ content }}\n        {%- if page.feedback == true and jekyll.environment == \"production\" -%}\n            {%- include feedback.html -%}\n        {%- endif -%}\n    </div>\n</div>\n"
  },
  {
    "path": "docs/static_site/src/_layouts/page_category.html",
    "content": "---\n  # Licensed to the Apache Software Foundation (ASF) under one\n  # or more contributor license agreements.  See the NOTICE file\n  # distributed with this work for additional information\n  # regarding copyright ownership.  The ASF licenses this file\n  # to you under the Apache License, Version 2.0 (the\n  # \"License\"); you may not use this file except in compliance\n  # with the License.  You may obtain a copy of the License at\n  #   http://www.apache.org/licenses/LICENSE-2.0\n  # Unless required by applicable law or agreed to in writing,\n  # software distributed under the License is distributed on an\n  # \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  # KIND, either express or implied.  See the License for the\n  # specific language governing permissions and limitations\n  # under the License.\n\nlayout: page\n---\n<div class=\"row\">\n    <div class=\"col-3 docs-side-bar\">\n        <h3 style=\"text-transform: capitalize; padding-left:10px\">{{page.category}}</h3>\n        <ul>\n            {% for p in site.pages %}\n            {% if p.category == page.category %}\n            <li><a href=\"{{ p.url  | relative_url }}\">{{ p.title }}</a></li>\n            {% endif %}  <!-- page-category -->\n            {% endfor %}   <!-- resource-p -->\n        </ul>\n    </div>\n    <div class=\"col-9\">\n        {{ content }}\n    </div>\n</div>\n"
  },
  {
    "path": "docs/static_site/src/_layouts/page_landing_tutorials.html",
    "content": "---\n  # Licensed to the Apache Software Foundation (ASF) under one\n  # or more contributor license agreements.  See the NOTICE file\n  # distributed with this work for additional information\n  # regarding copyright ownership.  The ASF licenses this file\n  # to you under the Apache License, Version 2.0 (the\n  # \"License\"); you may not use this file except in compliance\n  # with the License.  You may obtain a copy of the License at\n  #   http://www.apache.org/licenses/LICENSE-2.0\n  # Unless required by applicable law or agreed to in writing,\n  # software distributed under the License is distributed on an\n  # \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  # KIND, either express or implied.  See the License for the\n  # specific language governing permissions and limitations\n  # under the License.\n\nlayout: page_api\n---\n<h2>List of available tutorials</h2>\n<ul>\n    {% for p in site.pages %}\n    {% if p.is_tutorial == true %}\n    {% if page.tag == p.tag %}\n    <li><a href=\"{{ p.url | relative_url}}\">{{ p.title }}</a></li>\n    {% endif %}  <!-- page-category -->\n    {% endif %}   <!-- resource-p -->\n    {% endfor %}  <!-- page -->\n</ul>\n"
  },
  {
    "path": "docs/static_site/src/_layouts/post.html",
    "content": "---\n  # Licensed to the Apache Software Foundation (ASF) under one\n  # or more contributor license agreements.  See the NOTICE file\n  # distributed with this work for additional information\n  # regarding copyright ownership.  The ASF licenses this file\n  # to you under the Apache License, Version 2.0 (the\n  # \"License\"); you may not use this file except in compliance\n  # with the License.  You may obtain a copy of the License at\n  #   http://www.apache.org/licenses/LICENSE-2.0\n  # Unless required by applicable law or agreed to in writing,\n  # software distributed under the License is distributed on an\n  # \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  # KIND, either express or implied.  See the License for the\n  # specific language governing permissions and limitations\n  # under the License.\n\nlayout: default\n---\n<article class=\"post h-entry\" itemscope itemtype=\"http://schema.org/BlogPosting\">\n\n    <header class=\"post-header\">\n        <h1 class=\"post-title p-name\" itemprop=\"name headline\">{{ page.title | escape }}</h1>\n        <p class=\"post-meta\">\n            <time class=\"dt-published\" datetime=\"{{ page.date | date_to_xmlschema }}\" itemprop=\"datePublished\">\n                {%- assign date_format = site.minima.date_format | default: \"%b %-d, %Y\" -%}\n                {{ page.date | date: date_format }}\n            </time>\n            {%- if page.author -%}\n            • <span itemprop=\"author\" itemscope itemtype=\"http://schema.org/Person\"><span class=\"p-author h-card\"\n                                                                                          itemprop=\"name\">{{ page.author }}</span></span>\n            {%- endif -%}\n        </p>\n    </header>\n\n    <div class=\"post-content e-content\" itemprop=\"articleBody\">\n        {{ content }}\n    </div>\n\n    {%- if site.disqus.shortname -%}\n    {%- include disqus_comments.html -%}\n    {%- endif -%}\n\n    <a class=\"u-url\" href=\"{{ page.url | relative_url }}\" hidden></a>\n</article>\n"
  },
  {
    "path": "docs/static_site/src/_plugins/markdowner.rb",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nmodule Jekyll\n  class MarkdownBlock < Liquid::Block\n    def initialize(tag_name, text, tokens)\n      super\n    end\n    require \"kramdown\"\n    def render(context)\n      content = super\n      \"#{Kramdown::Document.new(content).to_html}\"\n    end\n  end\nend\nLiquid::Template.register_tag('markdown', Jekyll::MarkdownBlock)\n"
  },
  {
    "path": "docs/static_site/src/_sass/feedback.scss",
    "content": "/* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n * \n *   http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License. */\n\n.feedback-container {\n  text-align: center;\n}\n\n.feedback-answer-container {\n  display: inline-block;\n}\n\n.feedback-question {\n  display: inline-block;\n  padding: 0.5em 1em 0.5em 1em;\n}\n\n.feedback-answer {\n  display: inline-block;\n  padding: 0.5em 1em 0.5em 1em;\n  color: #048ccc;\n  cursor: pointer;\n}\n\n.feedback-answer:hover {\n  color: #ffffff;\n  background-color: #048ccc;\n}\n\n.feedback-thank-you {\n  display: none;\n  padding: 0.5em 1em 0.5em 1em;\n}\n\n.feedback-hr-top {\n  margin: 1em;\n  margin-top: 50px;\n}\n\n.feedback-hr-bottom {\n  margin: 1em;\n  margin-bottom: 30px;\n}\n"
  },
  {
    "path": "docs/static_site/src/_sass/generalVersionDropdown.scss",
    "content": "/* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n * \n *   http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License. */\n\n.dropdown-option-active {\n  color: #ff4500 !important;\n  font-weight: lighter;\n}\n\n.dropdown {\n  position: relative;\n  display: inline-block;\n}\n\n.dropdown-content {\n  display: none;\n  position: absolute;\n  background-color: #f9f9f9;\n  min-width: 160px;\n  box-shadow: 0px 8px 16px 0px rgba(0, 0, 0, 0.2);\n  padding: 12px 16px;\n  z-index: 1;\n  text-align: left;\n}\n\n.dropdown:hover .dropdown-content {\n  display: block;\n}\n\n.dropdown-header {\n  color: #ffffff;\n  display: inline-flex;\n}\n\n.dropdown-caret {\n  width: 18px;\n}\n\n.trigger .dropdown-caret {\n  height: 57px;\n}\n\n.dropdown-caret-path {\n  fill: #ffffff;\n}\n"
  },
  {
    "path": "docs/static_site/src/_sass/globalSearch.scss",
    "content": "/* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n * \n *   http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License. */\n\n.trigger {\n  float: right;\n}\n\n/* Search bar - wide screen */\n.gs-search-border {\n  padding-left: 25px;\n  float: right;\n}\n\n#global-search {\n  width: 0px;\n  border: 0px;\n  background-color: transparent;\n  border-bottom: 1px solid rgba(0, 0, 0, 0.12);\n  outline: none;\n  font-size: 1em;\n  color: #ffffff;\n  display: inline-block;\n  margin-bottom: 19px;\n}\n\n#gs-search-icon {\n  background: url(/assets/img/search-icon.svg) center no-repeat;\n  background-size: 1em;\n  height: 30px;\n  width: 30px;\n  position: relative;\n  top: 10px;\n  cursor: pointer;\n  display: inline-block;\n}\n\n#global-search-form {\n  display: none;\n  padding: 5px;\n  line-height: 1em;\n}\n\n#global-search-close {\n  cursor: pointer;\n}\n\n#global-search-dropdown-container {\n  display: inline-block;\n  position: relative;\n}\n\n#global-search-dropdown-container button {\n  background-color: inherit;\n  color: white;\n  font-size: 17px;\n  margin: 0px;\n  border: none;\n  min-width: 100%;\n  outline: 0;\n  height: 37px;\n  width: 90px;\n  padding: 0px;\n  cursor: pointer;\n}\n\n#global-search-dropdown-container-mobile\n  .gs-current-version-mobile\n  .gs-dropdown-caret,\n#global-search-dropdown-container .gs-current-version .gs-dropdown-caret {\n  position: relative;\n  top: 4px;\n  height: 18px;\n}\n\nul.gs-version-dropdown {\n  display: none;\n  position: absolute;\n  text-align: center;\n  background-color: whitesmoke;\n  box-shadow: none;\n  z-index: 1;\n  margin: 0px;\n  padding: 0px;\n  list-style-type: none;\n}\n\nli.gs-opt.gs-versions {\n  padding: 10px;\n  cursor: pointer;\n}\n\n.gs-version-dropdown li:hover {\n  color: #ff4500 !important;\n}\n\n/* Search bar - mobile */\n#global-search-mobile {\n  width: 120px;\n  margin-right: 10px;\n  background-color: transparent;\n  border: 0px;\n  border-bottom: 1px solid rgba(0, 0, 0, 0.12);\n  outline: none;\n  font-size: 1em;\n  color: white;\n}\n\n#gs-search-icon-mobile {\n  background: url(/assets/img/search-icon.svg) center no-repeat;\n  background-size: 1em;\n  height: 30px;\n  width: 30px;\n  position: relative;\n  top: 10px;\n  cursor: pointer;\n  display: inline-block;\n}\n\n#global-search-dropdown-container-mobile {\n  display: inline-block;\n  position: absolute;\n  right: 10px;\n}\n\n#global-search-dropdown-container-mobile button {\n  background-color: inherit;\n  margin: 0px;\n  border: none;\n  outline: 0;\n  height: 37px;\n  padding: 0px;\n  cursor: pointer;\n}\n\n.gs-version-dropdown-mobile {\n  display: none;\n  position: absolute;\n  text-align: center;\n  background-color: whitesmoke;\n  box-shadow: none;\n  z-index: 1;\n  margin: 0px;\n  padding: 0px;\n  left: -50px;\n  list-style-type: none;\n  width: 166px;\n  left: -138px;\n}\n\n#global-search-mobile-border {\n  line-height: 25px;\n}\n\n#global-search-mobile::placeholder,\n#global-search::placeholder {\n  color: #eeeeee;\n}\n\n.gs-version-dropdown-mobile li.active,\n.gs-version-dropdown li.active {\n  color: #ff4500 !important;\n  font-weight: lighter;\n}\n\n.gs-version-dropdown-mobile li,\n.gs-version-dropdown li {\n  color: #424242;\n  text-decoration: none;\n  display: block;\n  padding-left: 5px;\n  padding-right: 5px;\n  font-size: 17px;\n}\n\n/* Main dropdown wrapper */\n.algolia-autocomplete .ds-dropdown-menu > div {\n  max-height: 60vh;\n  overflow-y: scroll;\n}\n\n#global-search-mobile-border .algolia-autocomplete .ds-dropdown-menu {\n  min-width: 80vw;\n}\n\n#global-search-mobile-border .algolia-autocomplete .ds-dropdown-menu > div {\n  min-width: 80vw;\n  max-height: 90vh;\n}\n\n.gs-search-border .algolia-autocomplete .ds-dropdown-menu {\n  min-width: 680px;\n}\n\n/* Main category */\n.algolia-autocomplete .algolia-docsearch-suggestion--category-header {\n  color: #000000;\n  text-align: center;\n}\n\n/* Category */\n.algolia-autocomplete .algolia-docsearch-suggestion--subcategory-column {\n  color: gray;\n}\n\n/* Title */\n.algolia-autocomplete .algolia-docsearch-suggestion--title {\n  color: black;\n  font-weight: 300;\n}\n\n/* Description description */\n.algolia-autocomplete .algolia-docsearch-suggestion--text {\n  font-size: 0.8rem;\n  color: gray;\n}\n\n/* Highlighted text */\n.algolia-autocomplete .algolia-docsearch-suggestion--highlight {\n  color: blue;\n}\n"
  },
  {
    "path": "docs/static_site/src/_sass/minima/_base.scss",
    "content": "/* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n * \n *   http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License. */\n\n/**\n* Reset some basic elements\n*/\nbody, h1, h2, h3, h4, h5, h6,\np, blockquote, pre, hr,\ndl, dd, ol, ul, figure {\n  margin: 0;\n  padding: 0;\n}\n\nbody {\n  background-repeat: no-repeat;\n  background-size: contain;\n  min-height: 100%;\n  background-color: $grey-color-light;\n  @media screen and (min-width: $on-palm) and (max-width: $on-laptop) {\n    background-size: inherit;\n  }\n  @media screen and (max-width: $on-palm) {\n    background-size: cover;\n  }\n}\n\ntable {\n  color: $grey-color-dark !important;\n}\n\n/**\n * Basic styling\n */\nbody {\n  font-family: $base-font-family;\n  font-weight: $base-font-weight;\n  font-size: $base-font-size;\n  line-height: $base-line-height;\n  color: $text-color;\n  background-color: $color-mxnet;\n\n  display: flex;\n  min-height: 100vh;\n  flex-direction: column;\n}\n\n\n/**\n * Set `margin-bottom` to maintain vertical rhythm\n */\nh1, h2, h3, h4, h5, h6,\np, blockquote, pre,\nul, ol, dl, figure,\n%vertical-rhythm {\n  margin-bottom: $spacing-unit / 2;\n}\n\n\n/**\n * `main` element\n */\nmain {\n  display: block; /* Default value of `display` of `main` element is 'inline' in IE 11. */\n}\n\nheader {\n  z-index: 10;\n}\n\n/**\n * Images\n */\nimg {\n  max-width: 100%;\n  vertical-align: middle;\n}\n\n\n/**\n * Figures\n */\nfigure > img {\n  display: block;\n}\n\nfigcaption {\n  font-size: $small-font-size;\n}\n\n\n/**\n * Lists\n */\nul, ol {\n  margin-left: $spacing-unit;\n}\n\nli {\n  > ul,\n  > ol {\n    margin-bottom: 0;\n  }\n}\n\n\n/**\n * Headings\n */\nh1, h2, h3, h4, h5, h6 {\n  font-weight: $base-font-weight;\n}\n\n\nh1 {\n  font-weight: 200;\n  @include relative-font-size(3.5);\n  line-height: 120%;\n}\n\n\n/**\n * Links\n */\na {\n  color: $color-mxnet;\n  text-decoration: none;\n  font-weight: 300;\n\n  &:visited {\n    color: $color-mxnet;\n  }\n\n  &:hover {\n    color: $color-mxnet-dark;\n    text-decoration: none;\n  }\n\n  .social-media-list &:hover {\n\n  }\n}\n\n.clickable {\n  &:hover {\n    cursor: pointer;\n  }\n}\n\n/**\n * Blockquotes\n */\nblockquote {\n  color: $grey-color;\n  border-left: 4px solid $grey-color-light;\n  padding-left: $spacing-unit / 2;\n  @include relative-font-size(1.125);\n  letter-spacing: -1px;\n  font-style: italic;\n\n  > :last-child {\n    margin-bottom: 0;\n  }\n}\n\n\n/**\n * Code formatting\n */\npre,\ncode {\n  @include relative-font-size(0.9375);\n  border: 1px solid $grey-color-light;\n  border-radius: 3px;\n  background-color: #eef;\n}\n\ncode {\n  padding: 1px 5px;\n}\n\npre {\n  padding: 8px 12px;\n  overflow-x: auto;\n\n  > code {\n    border: 0;\n    padding-right: 0;\n    padding-left: 0;\n  }\n}\n\n.span-accented {\n  color: orangered;\n  float: none !important;\n  margin: 0 !important;\n  font-size: 120%;\n}\n\n/**\n * Wrapper\n */\n.wrapper {\n  max-width: -webkit-calc(#{$content-width} - (#{$spacing-unit} * 2));\n  max-width: calc(#{$content-width} - (#{$spacing-unit} * 2));\n  margin-right: auto;\n  margin-left: auto;\n  padding-right: $spacing-unit;\n  padding-left: $spacing-unit;\n  @extend %clearfix;\n\n  @include media-query($on-laptop) {\n    max-width: -webkit-calc(#{$content-width} - (#{$spacing-unit}));\n    max-width: calc(#{$content-width} - (#{$spacing-unit}));\n    padding-right: $spacing-unit / 2;\n    padding-left: $spacing-unit / 2;\n  }\n}\n\n\n/**\n * Clearfix\n */\n%clearfix:after {\n  content: \"\";\n  display: table;\n  clear: both;\n}\n\n\n/**\n * Icons\n */\n\n.svg-icon {\n  width: 16px;\n  height: 16px;\n  display: inline-block;\n  fill: $grey-color-light;\n  padding-right: 5px;\n  padding-top: 4px;\n  vertical-align: text-top;\n}\n\n.social-media-list {\n  li + li {\n    padding-top: 5px;\n  }\n}\n\n\n/**\n * Tables\n */\ntable {\n  margin-bottom: $spacing-unit;\n  width: 100%;\n  text-align: $table-text-align;\n  color: lighten($text-color, 18%);\n  border-collapse: collapse;\n  border: 1px solid $grey-color-light;\n\n  tr {\n    &:nth-child(even) {\n      background-color: lighten($grey-color-light, 6%);\n    }\n  }\n\n  th, td {\n    padding: ($spacing-unit / 3) ($spacing-unit / 2);\n  }\n\n  th {\n    background-color: lighten($grey-color-light, 3%);\n    border: 1px solid darken($grey-color-light, 4%);\n    border-bottom-color: darken($grey-color-light, 12%);\n  }\n\n  td {\n    border: 1px solid $grey-color-light;\n  }\n}\n"
  },
  {
    "path": "docs/static_site/src/_sass/minima/_blog.scss",
    "content": "/* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n * \n *   http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License. */\n\n.page-content-home div#rtb h2 {\n  font-size: 0.6em;\n}\n\n#rtb div.col-sm-4.rtb-col {\n\tborder: 1px solid $grey-color-light;\n\tmargin: 20px;\n}\n\n.blog-more {\n  margin-top: 40px;\n  margin-bottom: 40px;\n  height: 60px;\n  .btn {\n    background-color: $grey-color-light;\n  }\n}\n"
  },
  {
    "path": "docs/static_site/src/_sass/minima/_docs.scss",
    "content": "/* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n * \n *   http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License. */\n\n.docs-logo-docs {\n  width: 25px;\n  padding-bottom: 4px;\n  margin-right: 10px;\n}\n\n.docs-logo-container {\n  margin: auto;\n  text-align: center;\n  margin-bottom: 20px;\n\n  img.docs-logo-image {\n    height: 75px;\n    margin: auto;\n  }\n}\n\n.docs-side-bar {\n  margin-left: 0px !important;\n  padding-right: 10px;\n  background-color: $grey-color-light;\n}\n\n.docs-hero {\n  margin-bottom: 20px;\n}\n\n.docs-hero-left {\n\n}\n\n.docs-hero-right {\n\n}\n\n.docs-card {\n  background-color: $grey-color-light;\n  padding: 20px;\n  height: 100%;\n}\n\n.docs-card.docs-side {\n  padding: 5px;\n\n  ul {\n    margin-left: 0;\n  }\n\n  li {\n    margin-left: 20px;\n  }\n}\n\n.docs-action-btn {\n  a {\n    color: $grey-color-dark;\n\n    &:visited {\n      color: $color-mxnet-dark;\n    }\n\n    &:hover {\n      color: orangered;\n    }\n\n  }\n}\n\n.docs-faq {\n  background-color: $grey-color-light;\n  padding-top: 20px;\n  padding-bottom: 20px;\n}\n\n.docs-architecture {\n  background-color: $grey-color-light;\n  margin-top: 20px;\n  margin-bottom: 20px;\n  padding-top: 20px;\n  padding-bottom: 20px;\n}\n\n.docs-dev-guide {\n  background-color: white;\n  padding-top: 20px;\n  padding-bottom: 20px;\n}\n\n.language-binding-banner {\n  border: 1px solid transparent;\n  border-radius: .25rem;\n  color: #856404;\n  padding: .75rem 1.25rem;\n  margin-bottom: 1rem;\n  background-color: #fff3cd;\n  border-color: #ffeeba;\n}\n"
  },
  {
    "path": "docs/static_site/src/_sass/minima/_ecosystem.scss",
    "content": "/* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n * \n *   http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License. */\n\n.ecosystem-page {\n  .card {\n    background-color: $grey-color-light;\n  }\n\n  h4 {\n    float: left;\n  }\n}\n"
  },
  {
    "path": "docs/static_site/src/_sass/minima/_features.scss",
    "content": "/* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n * \n *   http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License. */\n\n.feature-image {\n  max-width: 100px;\n  filter: grayscale(1);\n  margin: auto;\n}\n\n.feature-paragraph {\n  margin: auto;\n}\n\n.feature-title {\n  float: left;\n  margin-bottom: 0px;\n}\n\n.highlight pre {\n  box-shadow: inset 0 -2px 0 0 $color-mxnet;\n  margin-bottom: 0 !important;\n}\n\nfigure {\n  margin-bottom: 0 !important;\n}\n"
  },
  {
    "path": "docs/static_site/src/_sass/minima/_getting_started.scss",
    "content": "/* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n * \n *   http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License. */\n\n.install-selector {\n  max-width: 800px;\n  margin: auto;\n  margin-bottom: 40px;\n  padding-bottom: 40px;\n  padding-top: 40px;\n\n  .highlight {\n    margin-left: 20px;\n    margin-top: 20px;\n    margin-bottom: 20px !important;\n  }\n}\n\n\n.install-content, .install-widget {\n  visibility: hidden;\n}\n\n.get-started-from-source {\n  background-color: $grey-color-light;\n  padding-top: 50px;\n  padding-bottom: 50px;\n}\n\n#lang-demo ul {\n  margin-top: 20px;\n  margin-bottom: 15px;\n}\n\n.option-title {\n  width: 100px;\n  float: left;\n  clear: none;\n  text-align: right;\n  font-size: 15px;\n  padding-top: 7px;\n  padding-bottom: 8px;\n  padding-right: 10px;\n  font-weight: bold;\n}\n\n.option-row {\n  padding-bottom: 8px;\n}\n\n.install-inst {\n}\n\n#setup-options {\n  margin-top: 15px;\n  margin-bottom: 15px;\n  margin-left: 30px;\n}\n\n/*\n * Drop down\n */\n\n.dropbtn {\n  background-color: $color-mxnet;\n  color: white;\n  font-size: $base-font-size;\n  margin: 0px;\n  border: none;\n  min-width: 100%;\n  padding: 10px;\n}\n\nli.opt.versions {\n  padding: 10px;\n  &:hover {\n    background-color: $color-mxnet;\n\n    & a {\n      color: whitesmoke;\n    }\n  }\n}\n\n.dropdown {\n  position: relative;\n  display: inline-block;\n  margin: 5px;\n  font-size: $base-font-size;\n\n}\n\nul.dropdown-content {\n  display: none;\n  position: absolute;\n  left: 40%;\n  text-align: center;\n  background-color: whitesmoke;\n  box-shadow: none;\n  z-index: 1;\n  margin: 0px;\n  padding: 0px;\n  list-style-type: none;\n}\n\nul.version-dropdown {\n  @extend ul.dropdown-content;\n  left: 0px;\n  width: 100%;\n}\n\n.current-version.dropbtn.btn:focus{\n  outline: 0;\n}\n\n.current-version .dropdown-caret {\n  position: relative;\n  top: 4px;\n}\n\n.version-dropdown .active a {\n  color: $grey-color-light;\n}\n\n.version-dropdown a,\n.dropdown-content a {\n  color: $grey-color-dark;\n  text-decoration: none;\n  display: block;\n  padding-left: 5px;\n  padding-right: 5px;\n  font-size: $base-font-size;\n\n  &:hover {\n    color: orangered;\n  }\n}\n\n.dropdown-content .active a {\n  color: $grey-color-light;\n}\n\n.dropdown:hover .dropdown-content {\n  display: block;\n}\n\n.dropdown:hover .dropbtn {\n  background-color: $color-mxnet;\n}\n\n/*\n * selector\n */\n\n.col-3.install-left {\n  margin: auto;\n\n  span {\n    padding-left: 20px;\n    border-left: 2px $color-mxnet dashed;\n    @media screen and (max-width: $on-palm) {\n      border: none;\n    }\n  }\n\n}\n\n.col-9.install-right {\n  margin: auto;\n}\n\n.install-selector {\n  .active {\n    background-color: $color-mxnet !important;\n    color: $grey-color-light !important;\n  }\n}\n\n.btn-group.opt-group {\n  margin-top: 5px;\n  margin-bottom: 5px;\n  display: flex;\n  justify-content: space-between;\n\n\n  .opt {\n    padding-bottom: 10px;\n    padding-top: 10px;\n    font-size: $base-font-size;\n    width: 100%;\n    border: none;\n    color: $dark-gray;\n    margin-left: 5px;\n    margin-right: 5px;\n    background-color: $grey-color-light;\n\n    @media screen and (max-width: $on-palm) {\n      font-size: 11px;\n      margin-left: 2px;\n      margin-right: 2px;\n    }\n\n    &:hover {\n      background-color: $color-mxnet;\n      color: $grey-color-light;\n    }\n\n    &:focus {\n      outline: 0;\n    }\n    \n  }\n\n\n}\n"
  },
  {
    "path": "docs/static_site/src/_sass/minima/_home.scss",
    "content": "/* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n * \n *   http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License. */\n\n.home {\n  margin-bottom: 40px;\n}\n\n\n.home .btn {\n  border: 0;\n}\n\na.btn {\n  padding: 10px;\n  padding-left: 20px;\n  padding-right: 20px;\n  margin-top: 20px;\n  background-color: white;\n  transition: box-shadow 0.3s linear, border 0.3s linear;\n\n  color: $grey-color-dark;\n\n  &:hover {\n    box-shadow: inset 0 -2px 0 0 $color-mxnet;\n    color: black;\n  }\n}\n\n.btn-action {\n  float: right;\n}\n\n.section {\n  padding-top: 20px;\n  padding-bottom: 30px;\n\n  h2 {\n    text-transform: uppercase;\n  }\n}\n\n// =====\n// Cards\n// =====\n\n\n.card {\n  color: $grey-color-dark;\n  padding: 10px;\n  background-color: white;\n  transition: box-shadow 0.3s linear;\n  height: 100%;\n\n\n  &:hover {\n    box-shadow: inset 0 -2px 0 0 $color-mxnet;\n    filter: none;\n\n    a h3, h4 {\n      color: orangered;\n    }\n  }\n\n  h3 {\n    color: black;\n  }\n\n  p {\n    font-size: 90%;\n  }\n\n  a {\n    color: $grey-color-dark;\n\n    &:hover {\n      color: black;\n    }\n  }\n}\n\n\n.card-header-title {\n  height: 50px;\n}\n\n.card-header-title h3 {\n  float: left\n}\n\n.card-header-title img {\n  float: right;\n  width: 30px;\n}\n\n\n// ==============\n// Key features\n// ==============\n\n.key-features-section {\n  background-color: white;\n  color: $grey-color-dark;\n\n  h3 {\n    color: orangered;\n  }\n\n  .btn {\n    background-color: $grey-color-light;\n  }\n\n  .card-text {\n    @media screen and (min-width: $on-laptop) {\n      min-height: 180px !important;\n    }\n  }\n\n  .card {\n    &:hover {\n      box-shadow: 0 0 0 0 !important;\n    }\n  }\n\n}\n\n.key-feature-image {\n  width: 100%;\n  text-align: center;\n  margin-top: 30px;\n\n  img {\n    width: 70px;\n    filter: grayscale(1);\n  }\n}\n\n// ====\n// ecosystem\n// ====\n\n.ecosystem-section {\n  background-color: $grey-color-light;\n  color: black;\n}\n\n// ===\n// community\n// ===\n\n.community-section {\n  background-color: $grey-color-light;\n  color: $grey-color-dark;\n}\n\n.community-section .card {\n  background-color: white;\n}\n\n// ===\n// news\n// ===\n.news-section {\n  color: $grey-color-dark;\n  min-height: 500px;\n  background-color: white;\n\n  .btn {\n    background-color: $grey-color-light;\n  }\n}\n"
  },
  {
    "path": "docs/static_site/src/_sass/minima/_layout.scss",
    "content": "/* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n * \n *   http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License. */\n\n/**\n * Site header\n */\n.site-header {\n  min-height: $spacing-unit * 1.865;\n\n  position: fixed;\n  top: 0;\n  width: 100%;\n  padding-top: 10px;\n  padding-bottom: 10px;\n}\n\n.site-header-logo {\n  width: 120px;\n}\n\n.site-title {\n  @include relative-font-size(1.625);\n  font-weight: 300;\n  line-height: $base-line-height * $base-font-size * 2.25;\n  letter-spacing: -1px;\n  margin-bottom: 0;\n  float: left;\n  color: white;\n\n  &,\n  &:visited {\n    color: $grey-color-dark;\n  }\n}\n\n.site-nav {\n  float: right;\n  line-height: $base-line-height * $base-font-size * 2.25;\n\n  .nav-trigger {\n    display: none;\n  }\n  \n  #global-search-mobile-border {\n    display: none;\n  }\n\n  .gs-search-border {\n    display: inline-block;\n  }\n\n  .menu-icon {\n    display: none;\n  }\n\n  .page-link {\n    color: $text-color;\n    line-height: $base-line-height;\n    //text-transform: uppercase;\n    //text-shadow: 1px 1px rgba(50,50,50,0.2);\n\n    // Gaps between nav items, but not on the last one\n    &:not(:last-child) {\n      margin-right: 40px;\n    }\n\n    &:hover {\n      color: white;\n      text-shadow: -0.06ex 0 white, 0.06ex 0 white;\n    }\n  }\n\n  .page-link.page-current {\n    color: white;\n    text-decoration: underline;\n  }\n\n  @media screen and (max-width: $on-laptop) {\n    position: absolute;\n    top: 9px;\n    right: $spacing-unit / 2;\n    background-color: $color-mxnet;\n    border-radius: 2px;\n    text-align: right;\n\n    label[for=\"nav-trigger\"] {\n      display: block;\n      float: right;\n      width: 36px;\n      height: 36px;\n      z-index: 2;\n      cursor: pointer;\n    }\n\n    .gs-search-border {\n      display: none;\n    }\n\n    #global-search-mobile-border {\n      display: block;\n    }\n\n    .menu-icon {\n      display: block;\n      float: right;\n      width: 36px;\n      height: 26px;\n      line-height: 0;\n      padding-top: 10px;\n      text-align: center;\n\n      > svg {\n        fill: $grey-color-light;\n      }\n    }\n\n    input ~ .trigger {\n      clear: both;\n      display: none;\n    }\n\n    input:checked ~ .trigger {\n      display: block;\n      padding-bottom: 5px;\n    }\n\n    .page-link {\n      padding: 5px 10px;\n      display: block;\n\n      &:not(:last-child) {\n        margin-right: 0;\n      }\n\n      margin-left: 20px;\n    }\n  }\n}\n\n\n/**\n * Site footer\n */\n.site-footer {\n  border-top: 1px solid $grey-color-light;\n  padding: $spacing-unit 0;\n  background-color: #424242;\n  .footer-category-title {\n    color: $color-mxnet;\n  }\n  a {\n    color: $grey-color-light !important;\n\n    &:visited {\n      color: $grey-color-light !important;\n    }\n  }\n\n}\n\n.site-footer2 {\n  background-color: #424242;\n  padding-top: 40px;\n  padding-bottom: 10px;\n}\n\n.footer-heading {\n  @include relative-font-size(1.125);\n  margin-bottom: $spacing-unit / 2;\n}\n\n.contact-list,\n.apache-list,\n.social-media-list {\n  list-style: none;\n  margin-left: 0;\n}\n\n.footer-col-wrapper {\n  @include relative-font-size(0.9375);\n\n  margin-left: -$spacing-unit / 2;\n  @extend %clearfix;\n}\n\n.footer-bottom-warning {\n  font-size: 80%;\n  color: white;\n  float: left;\n}\n\n.footer-logo {\n  width: 200px;\n  margin-bottom: 30px;\n  margin-top: 30px;\n}\n\n.footer-col {\n  float: left;\n  margin-bottom: $spacing-unit / 2;\n  padding-left: $spacing-unit / 2;\n}\n\n.footer-text {\n  color: $grey-color-light;\n}\n\n.footer-col-1 {\n  width: -webkit-calc(25% - (#{$spacing-unit} / 2));\n  width: calc(25% - (#{$spacing-unit} / 2));\n}\n\n.footer-col-2 {\n  width: -webkit-calc(30% - (#{$spacing-unit} / 2));\n  width: calc(30% - (#{$spacing-unit} / 2));\n}\n\n.footer-col-3 {\n  width: -webkit-calc(45% - (#{$spacing-unit} / 2));\n  width: calc(45% - (#{$spacing-unit} / 2));\n}\n\n@include media-query($on-laptop) {\n  .footer-col-1,\n  .footer-col-2 {\n    width: -webkit-calc(50% - (#{$spacing-unit} / 2));\n    width: calc(50% - (#{$spacing-unit} / 2));\n  }\n\n  .footer-col-3 {\n    width: -webkit-calc(100% - (#{$spacing-unit} / 2));\n    width: calc(100% - (#{$spacing-unit} / 2));\n  }\n}\n\n@include media-query($on-palm) {\n  .footer-col {\n    float: none;\n    width: -webkit-calc(100% - (#{$spacing-unit} / 2));\n    width: calc(100% - (#{$spacing-unit} / 2));\n  }\n}\n\n\n/**\n * Page content\n */\n\n.header-section {\n  .btn {\n    background-color: $grey-color-light;\n  }\n}\n\n.page-content {\n  height: 100%;\n  padding: $spacing-unit 0 0;\n  flex: 1;\n  margin-top: 100px;\n}\n\n.page-content-home {\n  height: 100%;\n  padding: $spacing-unit 0 0;\n  margin-top: 100px;\n}\n\n.page-heading {\n  @include relative-font-size(2);\n}\n\n.post-list-heading {\n  @include relative-font-size(1.75);\n}\n\n.post-list {\n  margin-left: 0;\n  list-style: none;\n\n  > li {\n    margin-bottom: $spacing-unit;\n  }\n}\n\n.post-meta {\n  font-size: $small-font-size;\n  color: $grey-color;\n}\n\n.post-link {\n  display: block;\n  @include relative-font-size(1.5);\n}\n\n\n/**\n * Posts\n */\n.post-header {\n  margin-bottom: 20px;\n  margin-top: 50px;\n}\n\n.post-title {\n  @include relative-font-size(2.625);\n  letter-spacing: -1px;\n  line-height: 1;\n\n  @include media-query($on-laptop) {\n    @include relative-font-size(2.25);\n  }\n}\n\n.post-content {\n  min-height: 700px;\n  background-color: white;\n  padding-top: 30px;\n  color: $grey-color-dark;\n\n  h2 {\n    @include relative-font-size(2);\n\n    @include media-query($on-laptop) {\n      @include relative-font-size(1.75);\n    }\n  }\n\n  h3 {\n    @include relative-font-size(1.625);\n\n    @include media-query($on-laptop) {\n      @include relative-font-size(1.375);\n    }\n  }\n\n  h4 {\n    @include relative-font-size(1.25);\n\n    @include media-query($on-laptop) {\n      @include relative-font-size(1.125);\n    }\n  }\n}\n\n.copy-btn {\n  display: none;\n  position: absolute;\n  right: 0;\n  width: 60px;\n  text-align: center;\n  padding: 2px;\n  padding-bottom: 5px;\n  font-family: monospace;\n  font-size: 15px;\n  background-color: $color-mxnet;\n  color: white;\n  border: none;\n\n  &:hover {\n    background-color: #eef;\n    color: $color-mxnet;\n    cursor: pointer;\n    border: 1px solid $color-mxnet;\n  }\n\n  &:active {\n    background-color: $color-mxnet;\n    color: $grey-color-light;\n  }\n}\n\n:target::before {\n  display: block;\n  content: \" \";\n  margin-top: -86px;\n  height: 86px;\n  visibility: hidden;\n  pointer-events: none;\n}\n"
  },
  {
    "path": "docs/static_site/src/_sass/minima/_syntax-highlighting.scss",
    "content": "/* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n * \n *   http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License. */\n\n/**\n * Syntax highlighting styles\n */\n.highlight {\n  background: #fff;\n  @extend %vertical-rhythm;\n\n  .highlighter-rouge & {\n    background: #eef;\n  }\n\n  .c {\n    color: #998;\n    font-style: italic\n  }\n\n  // Comment\n  .err {\n    color: #a61717;\n    background-color: #e3d2d2\n  }\n\n  // Error\n  .k {\n    font-weight: bold\n  }\n\n  // Keyword\n  .o {\n    font-weight: bold\n  }\n\n  // Operator\n  .cm {\n    color: #998;\n    font-style: italic\n  }\n\n  // Comment.Multiline\n  .cp {\n    color: #999;\n    font-weight: bold\n  }\n\n  // Comment.Preproc\n  .c1 {\n    color: #998;\n    font-style: italic\n  }\n\n  // Comment.Single\n  .cs {\n    color: #999;\n    font-weight: bold;\n    font-style: italic\n  }\n\n  // Comment.Special\n  .gd {\n    color: #000;\n    background-color: #fdd\n  }\n\n  // Generic.Deleted\n  .gd .x {\n    color: #000;\n    background-color: #faa\n  }\n\n  // Generic.Deleted.Specific\n  .ge {\n    font-style: italic\n  }\n\n  // Generic.Emph\n  .gr {\n    color: #a00\n  }\n\n  // Generic.Error\n  .gh {\n    color: #999\n  }\n\n  // Generic.Heading\n  .gi {\n    color: #000;\n    background-color: #dfd\n  }\n\n  // Generic.Inserted\n  .gi .x {\n    color: #000;\n    background-color: #afa\n  }\n\n  // Generic.Inserted.Specific\n  .go {\n    color: #888\n  }\n\n  // Generic.Output\n  .gp {\n    color: #555\n  }\n\n  // Generic.Prompt\n  .gs {\n    font-weight: bold\n  }\n\n  // Generic.Strong\n  .gu {\n    color: #aaa\n  }\n\n  // Generic.Subheading\n  .gt {\n    color: #a00\n  }\n\n  // Generic.Traceback\n  .kc {\n    font-weight: bold\n  }\n\n  // Keyword.Constant\n  .kd {\n    font-weight: bold\n  }\n\n  // Keyword.Declaration\n  .kp {\n    font-weight: bold\n  }\n\n  // Keyword.Pseudo\n  .kr {\n    font-weight: bold\n  }\n\n  // Keyword.Reserved\n  .kt {\n    color: #458;\n    font-weight: bold\n  }\n\n  // Keyword.Type\n  .m {\n    color: #099\n  }\n\n  // Literal.Number\n  .s {\n    color: #d14\n  }\n\n  // Literal.String\n  .na {\n    color: #008080\n  }\n\n  // Name.Attribute\n  .nb {\n    color: #0086B3\n  }\n\n  // Name.Builtin\n  .nc {\n    color: #458;\n    font-weight: bold\n  }\n\n  // Name.Class\n  .no {\n    color: #008080\n  }\n\n  // Name.Constant\n  .ni {\n    color: #800080\n  }\n\n  // Name.Entity\n  .ne {\n    color: #900;\n    font-weight: bold\n  }\n\n  // Name.Exception\n  .nf {\n    color: #900;\n    font-weight: bold\n  }\n\n  // Name.Function\n  .nn {\n    color: #555\n  }\n\n  // Name.Namespace\n  .nt {\n    color: #000080\n  }\n\n  // Name.Tag\n  .nv {\n    color: #008080\n  }\n\n  // Name.Variable\n  .ow {\n    font-weight: bold\n  }\n\n  // Operator.Word\n  .w {\n    color: #bbb\n  }\n\n  // Text.Whitespace\n  .mf {\n    color: #099\n  }\n\n  // Literal.Number.Float\n  .mh {\n    color: #099\n  }\n\n  // Literal.Number.Hex\n  .mi {\n    color: #099\n  }\n\n  // Literal.Number.Integer\n  .mo {\n    color: #099\n  }\n\n  // Literal.Number.Oct\n  .sb {\n    color: #d14\n  }\n\n  // Literal.String.Backtick\n  .sc {\n    color: #d14\n  }\n\n  // Literal.String.Char\n  .sd {\n    color: #d14\n  }\n\n  // Literal.String.Doc\n  .s2 {\n    color: #d14\n  }\n\n  // Literal.String.Double\n  .se {\n    color: #d14\n  }\n\n  // Literal.String.Escape\n  .sh {\n    color: #d14\n  }\n\n  // Literal.String.Heredoc\n  .si {\n    color: #d14\n  }\n\n  // Literal.String.Interpol\n  .sx {\n    color: #d14\n  }\n\n  // Literal.String.Other\n  .sr {\n    color: #009926\n  }\n\n  // Literal.String.Regex\n  .s1 {\n    color: #d14\n  }\n\n  // Literal.String.Single\n  .ss {\n    color: #990073\n  }\n\n  // Literal.String.Symbol\n  .bp {\n    color: #999\n  }\n\n  // Name.Builtin.Pseudo\n  .vc {\n    color: #008080\n  }\n\n  // Name.Variable.Class\n  .vg {\n    color: #008080\n  }\n\n  // Name.Variable.Global\n  .vi {\n    color: #008080\n  }\n\n  // Name.Variable.Instance\n  .il {\n    color: #099\n  }\n\n  // Literal.Number.Integer.Long\n}\n"
  },
  {
    "path": "docs/static_site/src/_sass/minima/colorful.scss",
    "content": "/* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n * \n *   http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License. */\n\n.highlight .hll {\n  background-color: #ffffcc\n}\n\n.highlight {\n  background: #f0f3f3;\n}\n\n.highlight .c {\n  color: #0099FF;\n  font-style: italic\n}\n\n/* Comment */\n.highlight .err {\n  color: #AA0000;\n  background-color: #FFAAAA\n}\n\n/* Error */\n.highlight .k {\n  color: #006699;\n  font-weight: bold\n}\n\n/* Keyword */\n.highlight .o {\n  color: #555555\n}\n\n/* Operator */\n.highlight .ch {\n  color: #0099FF;\n  font-style: italic\n}\n\n/* Comment.Hashbang */\n.highlight .cm {\n  color: #0099FF;\n  font-style: italic\n}\n\n/* Comment.Multiline */\n.highlight .cp {\n  color: #009999\n}\n\n/* Comment.Preproc */\n.highlight .cpf {\n  color: #0099FF;\n  font-style: italic\n}\n\n/* Comment.PreprocFile */\n.highlight .c1 {\n  color: #0099FF;\n  font-style: italic\n}\n\n/* Comment.Single */\n.highlight .cs {\n  color: #0099FF;\n  font-weight: bold;\n  font-style: italic\n}\n\n/* Comment.Special */\n.highlight .gd {\n  background-color: #FFCCCC;\n  border: 1px solid #CC0000\n}\n\n/* Generic.Deleted */\n.highlight .ge {\n  font-style: italic\n}\n\n/* Generic.Emph */\n.highlight .gr {\n  color: #FF0000\n}\n\n/* Generic.Error */\n.highlight .gh {\n  color: #003300;\n  font-weight: bold\n}\n\n/* Generic.Heading */\n.highlight .gi {\n  background-color: #CCFFCC;\n  border: 1px solid #00CC00\n}\n\n/* Generic.Inserted */\n.highlight .go {\n  color: #AAAAAA\n}\n\n/* Generic.Output */\n.highlight .gp {\n  color: #000099;\n  font-weight: bold\n}\n\n/* Generic.Prompt */\n.highlight .gs {\n  font-weight: bold\n}\n\n/* Generic.Strong */\n.highlight .gu {\n  color: #003300;\n  font-weight: bold\n}\n\n/* Generic.Subheading */\n.highlight .gt {\n  color: #99CC66\n}\n\n/* Generic.Traceback */\n.highlight .kc {\n  color: #006699;\n  font-weight: bold\n}\n\n/* Keyword.Constant */\n.highlight .kd {\n  color: #006699;\n  font-weight: bold\n}\n\n/* Keyword.Declaration */\n.highlight .kn {\n  color: #006699;\n  font-weight: bold\n}\n\n/* Keyword.Namespace */\n.highlight .kp {\n  color: #006699\n}\n\n/* Keyword.Pseudo */\n.highlight .kr {\n  color: #006699;\n  font-weight: bold\n}\n\n/* Keyword.Reserved */\n.highlight .kt {\n  color: #007788;\n  font-weight: bold\n}\n\n/* Keyword.Type */\n.highlight .m {\n  color: #FF6600\n}\n\n/* Literal.Number */\n.highlight .s {\n  color: #CC3300\n}\n\n/* Literal.String */\n.highlight .na {\n  color: #330099\n}\n\n/* Name.Attribute */\n.highlight .nb {\n  color: #336666\n}\n\n/* Name.Builtin */\n.highlight .nc {\n  color: #00AA88;\n  font-weight: bold\n}\n\n/* Name.Class */\n.highlight .no {\n  color: #336600\n}\n\n/* Name.Constant */\n.highlight .nd {\n  color: #9999FF\n}\n\n/* Name.Decorator */\n.highlight .ni {\n  color: #999999;\n  font-weight: bold\n}\n\n/* Name.Entity */\n.highlight .ne {\n  color: #CC0000;\n  font-weight: bold\n}\n\n/* Name.Exception */\n.highlight .nf {\n  color: #CC00FF\n}\n\n/* Name.Function */\n.highlight .nl {\n  color: #9999FF\n}\n\n/* Name.Label */\n.highlight .nn {\n  color: #00CCFF;\n  font-weight: bold\n}\n\n/* Name.Namespace */\n.highlight .nt {\n  color: #330099;\n  font-weight: bold\n}\n\n/* Name.Tag */\n.highlight .nv {\n  color: #003333\n}\n\n/* Name.Variable */\n.highlight .ow {\n  color: #000000;\n  font-weight: bold\n}\n\n/* Operator.Word */\n.highlight .w {\n  color: #bbbbbb\n}\n\n/* Text.Whitespace */\n.highlight .mb {\n  color: #FF6600\n}\n\n/* Literal.Number.Bin */\n.highlight .mf {\n  color: #FF6600\n}\n\n/* Literal.Number.Float */\n.highlight .mh {\n  color: #FF6600\n}\n\n/* Literal.Number.Hex */\n.highlight .mi {\n  color: #FF6600\n}\n\n/* Literal.Number.Integer */\n.highlight .mo {\n  color: #FF6600\n}\n\n/* Literal.Number.Oct */\n.highlight .sa {\n  color: #CC3300\n}\n\n/* Literal.String.Affix */\n.highlight .sb {\n  color: #CC3300\n}\n\n/* Literal.String.Backtick */\n.highlight .sc {\n  color: #CC3300\n}\n\n/* Literal.String.Char */\n.highlight .dl {\n  color: #CC3300\n}\n\n/* Literal.String.Delimiter */\n.highlight .sd {\n  color: #CC3300;\n  font-style: italic\n}\n\n/* Literal.String.Doc */\n.highlight .s2 {\n  color: #CC3300\n}\n\n/* Literal.String.Double */\n.highlight .se {\n  color: #CC3300;\n  font-weight: bold\n}\n\n/* Literal.String.Escape */\n.highlight .sh {\n  color: #CC3300\n}\n\n/* Literal.String.Heredoc */\n.highlight .si {\n  color: #AA0000\n}\n\n/* Literal.String.Interpol */\n.highlight .sx {\n  color: #CC3300\n}\n\n/* Literal.String.Other */\n.highlight .sr {\n  color: #33AAAA\n}\n\n/* Literal.String.Regex */\n.highlight .s1 {\n  color: #CC3300\n}\n\n/* Literal.String.Single */\n.highlight .ss {\n  color: #FFCC33\n}\n\n/* Literal.String.Symbol */\n.highlight .bp {\n  color: #336666\n}\n\n/* Name.Builtin.Pseudo */\n.highlight .fm {\n  color: #CC00FF\n}\n\n/* Name.Function.Magic */\n.highlight .vc {\n  color: #003333\n}\n\n/* Name.Variable.Class */\n.highlight .vg {\n  color: #003333\n}\n\n/* Name.Variable.Global */\n.highlight .vi {\n  color: #003333\n}\n\n/* Name.Variable.Instance */\n.highlight .vm {\n  color: #003333\n}\n\n/* Name.Variable.Magic */\n.highlight .il {\n  color: #FF6600\n}\n\n/* Literal.Number.Integer.Long */\n"
  },
  {
    "path": "docs/static_site/src/_sass/minima/simple-grid.scss",
    "content": "/* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n * \n *   http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License. */\n\n// SIMPLE GRID - SASS/SCSS\n\n\n// fonts\n$font-weight-light: 300;\n$font-weight-regular: 400;\n$font-weight-heavy: 700;\n\n// colors\n$dark-grey: #333447;\n$dark-gray: #333447; // for the Americans\n\n\n.font-light {\n  font-weight: $font-weight-light;\n}\n\n.font-regular {\n  font-weight: $font-weight-regular;\n}\n\n.font-heavy {\n  font-weight: $font-weight-heavy;\n}\n\n// utility\n\n.left {\n  text-align: left;\n}\n\n.right {\n  text-align: right;\n}\n\n.center {\n  text-align: center;\n  margin-left: auto;\n  margin-right: auto;\n}\n\n.justify {\n  text-align: justify;\n}\n\n.hidden-sm {\n  display: none;\n}\n\n// grid\n\n$width: 98%;\n$gutter: 2%;\n$breakpoint-small: 33.75em; // 540px\n$breakpoint-med: 45em; // 720px\n$breakpoint-large: 60em; // 960px\n\n.container {\n  width: 100%;\n  margin-left: auto;\n  margin-right: auto;\n\n  @media only screen and (min-width: $breakpoint-small) {\n    width: 80%;\n  }\n\n  @media only screen and (min-width: $breakpoint-large) {\n    width: 75%;\n    max-width: 60rem;\n  }\n}\n\n.row {\n  position: relative;\n  width: 100%;\n}\n\n.row [class^=\"col\"] {\n  float: left;\n  margin: 0.5rem 1%;\n  min-height: 0.125rem;\n}\n\n.row::after {\n  content: \"\";\n  display: table;\n  clear: both;\n}\n\n.col-1,\n.col-2,\n.col-3,\n.col-4,\n.col-5,\n.col-6,\n.col-7,\n.col-8,\n.col-9,\n.col-10,\n.col-11,\n.col-12 {\n  width: $width;\n}\n\n.col-1-sm {\n  width: ($width / 12) - ($gutter * 11 / 12);\n}\n\n.col-2-sm {\n  width: ($width / 6) - ($gutter * 10 / 12);\n}\n\n.col-3-sm {\n  width: ($width / 4) - ($gutter * 9 / 12);\n}\n\n.col-4-sm {\n  width: ($width / 3) - ($gutter * 8 / 12);\n}\n\n.col-5-sm {\n  width: ($width / (12 / 5)) - ($gutter * 7 / 12);\n}\n\n.col-6-sm {\n  width: ($width / 2) - ($gutter * 6 / 12);\n}\n\n.col-7-sm {\n  width: ($width / (12 / 7)) - ($gutter * 5 / 12);\n}\n\n.col-8-sm {\n  width: ($width / (12 / 8)) - ($gutter * 4 / 12);\n}\n\n.col-9-sm {\n  width: ($width / (12 / 9)) - ($gutter * 3 / 12);\n}\n\n.col-10-sm {\n  width: ($width / (12 / 10)) - ($gutter * 2 / 12);\n}\n\n.col-11-sm {\n  width: ($width / (12 / 11)) - ($gutter * 1 / 12);\n}\n\n.col-12-sm {\n  width: $width;\n}\n\n@media only screen and (min-width: $breakpoint-med) {\n  .col-1 {\n    width: ($width / 12) - ($gutter * 11 / 12);\n  }\n  .col-2 {\n    width: ($width / 6) - ($gutter * 10 / 12);\n  }\n  .col-3 {\n    width: ($width / 4) - ($gutter * 9 / 12);\n  }\n  .col-4 {\n    width: ($width / 3) - ($gutter * 8 / 12);\n  }\n  .col-5 {\n    width: ($width / (12 / 5)) - ($gutter * 7 / 12);\n  }\n  .col-6 {\n    width: ($width / 2) - ($gutter * 6 / 12);\n  }\n  .col-7 {\n    width: ($width / (12 / 7)) - ($gutter * 5 / 12);\n  }\n  .col-8 {\n    width: ($width / (12 / 8)) - ($gutter * 4 / 12);\n  }\n  .col-9 {\n    width: ($width / (12 / 9)) - ($gutter * 3 / 12);\n  }\n  .col-10 {\n    width: ($width / (12 / 10)) - ($gutter * 2 / 12);\n  }\n  .col-11 {\n    width: ($width / (12 / 11)) - ($gutter * 1 / 12);\n  }\n  .col-12 {\n    width: $width;\n  }\n\n  .hidden-sm {\n    display: block;\n  }\n}\n\n.row {\n  display: -webkit-box;\n  display: -webkit-flex;\n  display: -ms-flexbox;\n  display: flex;\n  flex-wrap: wrap;\n}\n\n.row > [class*='col-'] {\n  display: flex;\n  flex-direction: column;\n}\n"
  },
  {
    "path": "docs/static_site/src/_sass/minima.scss",
    "content": "/* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n * \n *   http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License. */\n\n@charset \"utf-8\";\n\n// import grid system\n@import  \"minima/simple-grid\";\n\n// Define defaults for each variable.\n\n$base-font-family: -apple-system,BlinkMacSystemFont,\"Segoe UI\",Roboto,\"Helvetica Neue\",Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\",\"Segoe UI Symbol\" !default;\n$base-font-size:   17px !default;\n$base-font-weight: 300 !default;\n$small-font-size:  $base-font-size * 0.875 !default;\n$base-line-height: 1.5 !default;\n\n$spacing-unit:     30px !default;\n\n$text-color:       white !default;\n$background-color: #fdfdfd !default;\n$brand-color:      #2a7ae2 !default;\n\n$color-mxnet: rgb(4,140,204);\n$color-mxnet-dark: rgb(4,60,110);\n$grey-color:       #828282 !default;\n$grey-color-light: lighten($grey-color, 45%) !default;\n$grey-color-dark:  darken($grey-color, 25%) !default;\n\n$table-text-align: left !default;\n\n// Width of the content area\n$content-width:    1150px !default;\n\n$on-palm:          600px !default;\n$on-palm:          900px !default;\n$on-laptop:        1024px !default;\n\n// Use media queries like this:\n// @include media-query($on-palm) {\n//   .wrapper {\n//     padding-right: $spacing-unit / 2;\n//     padding-left: $spacing-unit / 2;\n//   }\n// }\n@mixin media-query($device) {\n  @media screen and (max-width: $device) {\n    @content;\n  }\n}\n\n@mixin relative-font-size($ratio) {\n  font-size: $base-font-size * $ratio;\n}\n\n// Import partials.\n@import\n  \"minima/base\",\n  \"minima/layout\",\n  \"minima/syntax-highlighting\",\n  \"minima/home\",\n  \"minima/blog\",\n  \"minima/features\",\n  \"minima/ecosystem\",\n  \"minima/docs\",\n  \"minima/getting_started\",\n  \"minima/colorful\"\n;\n"
  },
  {
    "path": "docs/static_site/src/assets/js/clipboard.js",
    "content": "/*!\n * clipboard.js v2.0.6\n *\n * MIT License\n * Copyright (c) Zeno Rocha\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all\n * copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n * SOFTWARE.\n */\n!function(t,e){\"object\"==typeof exports&&\"object\"==typeof module?module.exports=e():\"function\"==typeof define&&define.amd?define([],e):\"object\"==typeof exports?exports.ClipboardJS=e():t.ClipboardJS=e()}(this,function(){return o={},r.m=n=[function(t,e){t.exports=function(t){var e;if(\"SELECT\"===t.nodeName)t.focus(),e=t.value;else if(\"INPUT\"===t.nodeName||\"TEXTAREA\"===t.nodeName){var n=t.hasAttribute(\"readonly\");n||t.setAttribute(\"readonly\",\"\"),t.select(),t.setSelectionRange(0,t.value.length),n||t.removeAttribute(\"readonly\"),e=t.value}else{t.hasAttribute(\"contenteditable\")&&t.focus();var o=window.getSelection(),r=document.createRange();r.selectNodeContents(t),o.removeAllRanges(),o.addRange(r),e=o.toString()}return e}},function(t,e){function n(){}n.prototype={on:function(t,e,n){var o=this.e||(this.e={});return(o[t]||(o[t]=[])).push({fn:e,ctx:n}),this},once:function(t,e,n){var o=this;function r(){o.off(t,r),e.apply(n,arguments)}return r._=e,this.on(t,r,n)},emit:function(t){for(var e=[].slice.call(arguments,1),n=((this.e||(this.e={}))[t]||[]).slice(),o=0,r=n.length;o<r;o++)n[o].fn.apply(n[o].ctx,e);return this},off:function(t,e){var n=this.e||(this.e={}),o=n[t],r=[];if(o&&e)for(var i=0,a=o.length;i<a;i++)o[i].fn!==e&&o[i].fn._!==e&&r.push(o[i]);return r.length?n[t]=r:delete n[t],this}},t.exports=n,t.exports.TinyEmitter=n},function(t,e,n){var d=n(3),h=n(4);t.exports=function(t,e,n){if(!t&&!e&&!n)throw new Error(\"Missing required arguments\");if(!d.string(e))throw new TypeError(\"Second argument must be a String\");if(!d.fn(n))throw new TypeError(\"Third argument must be a Function\");if(d.node(t))return s=e,f=n,(u=t).addEventListener(s,f),{destroy:function(){u.removeEventListener(s,f)}};if(d.nodeList(t))return a=t,c=e,l=n,Array.prototype.forEach.call(a,function(t){t.addEventListener(c,l)}),{destroy:function(){Array.prototype.forEach.call(a,function(t){t.removeEventListener(c,l)})}};if(d.string(t))return o=t,r=e,i=n,h(document.body,o,r,i);throw new TypeError(\"First argument must be a String, HTMLElement, HTMLCollection, or NodeList\");var o,r,i,a,c,l,u,s,f}},function(t,n){n.node=function(t){return void 0!==t&&t instanceof HTMLElement&&1===t.nodeType},n.nodeList=function(t){var e=Object.prototype.toString.call(t);return void 0!==t&&(\"[object NodeList]\"===e||\"[object HTMLCollection]\"===e)&&\"length\"in t&&(0===t.length||n.node(t[0]))},n.string=function(t){return\"string\"==typeof t||t instanceof String},n.fn=function(t){return\"[object Function]\"===Object.prototype.toString.call(t)}},function(t,e,n){var a=n(5);function i(t,e,n,o,r){var i=function(e,n,t,o){return function(t){t.delegateTarget=a(t.target,n),t.delegateTarget&&o.call(e,t)}}.apply(this,arguments);return t.addEventListener(n,i,r),{destroy:function(){t.removeEventListener(n,i,r)}}}t.exports=function(t,e,n,o,r){return\"function\"==typeof t.addEventListener?i.apply(null,arguments):\"function\"==typeof n?i.bind(null,document).apply(null,arguments):(\"string\"==typeof t&&(t=document.querySelectorAll(t)),Array.prototype.map.call(t,function(t){return i(t,e,n,o,r)}))}},function(t,e){if(\"undefined\"!=typeof Element&&!Element.prototype.matches){var n=Element.prototype;n.matches=n.matchesSelector||n.mozMatchesSelector||n.msMatchesSelector||n.oMatchesSelector||n.webkitMatchesSelector}t.exports=function(t,e){for(;t&&9!==t.nodeType;){if(\"function\"==typeof t.matches&&t.matches(e))return t;t=t.parentNode}}},function(t,e,n){\"use strict\";n.r(e);var o=n(0),r=n.n(o),i=\"function\"==typeof Symbol&&\"symbol\"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&\"function\"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?\"symbol\":typeof t};function a(t,e){for(var n=0;n<e.length;n++){var o=e[n];o.enumerable=o.enumerable||!1,o.configurable=!0,\"value\"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}function c(t){!function(t,e){if(!(t instanceof e))throw new TypeError(\"Cannot call a class as a function\")}(this,c),this.resolveOptions(t),this.initSelection()}var l=(function(t,e,n){return e&&a(t.prototype,e),n&&a(t,n),t}(c,[{key:\"resolveOptions\",value:function(t){var e=0<arguments.length&&void 0!==t?t:{};this.action=e.action,this.container=e.container,this.emitter=e.emitter,this.target=e.target,this.text=e.text,this.trigger=e.trigger,this.selectedText=\"\"}},{key:\"initSelection\",value:function(){this.text?this.selectFake():this.target&&this.selectTarget()}},{key:\"selectFake\",value:function(){var t=this,e=\"rtl\"==document.documentElement.getAttribute(\"dir\");this.removeFake(),this.fakeHandlerCallback=function(){return t.removeFake()},this.fakeHandler=this.container.addEventListener(\"click\",this.fakeHandlerCallback)||!0,this.fakeElem=document.createElement(\"textarea\"),this.fakeElem.style.fontSize=\"12pt\",this.fakeElem.style.border=\"0\",this.fakeElem.style.padding=\"0\",this.fakeElem.style.margin=\"0\",this.fakeElem.style.position=\"absolute\",this.fakeElem.style[e?\"right\":\"left\"]=\"-9999px\";var n=window.pageYOffset||document.documentElement.scrollTop;this.fakeElem.style.top=n+\"px\",this.fakeElem.setAttribute(\"readonly\",\"\"),this.fakeElem.value=this.text,this.container.appendChild(this.fakeElem),this.selectedText=r()(this.fakeElem),this.copyText()}},{key:\"removeFake\",value:function(){this.fakeHandler&&(this.container.removeEventListener(\"click\",this.fakeHandlerCallback),this.fakeHandler=null,this.fakeHandlerCallback=null),this.fakeElem&&(this.container.removeChild(this.fakeElem),this.fakeElem=null)}},{key:\"selectTarget\",value:function(){this.selectedText=r()(this.target),this.copyText()}},{key:\"copyText\",value:function(){var e=void 0;try{e=document.execCommand(this.action)}catch(t){e=!1}this.handleResult(e)}},{key:\"handleResult\",value:function(t){this.emitter.emit(t?\"success\":\"error\",{action:this.action,text:this.selectedText,trigger:this.trigger,clearSelection:this.clearSelection.bind(this)})}},{key:\"clearSelection\",value:function(){this.trigger&&this.trigger.focus(),document.activeElement.blur(),window.getSelection().removeAllRanges()}},{key:\"destroy\",value:function(){this.removeFake()}},{key:\"action\",set:function(t){var e=0<arguments.length&&void 0!==t?t:\"copy\";if(this._action=e,\"copy\"!==this._action&&\"cut\"!==this._action)throw new Error('Invalid \"action\" value, use either \"copy\" or \"cut\"')},get:function(){return this._action}},{key:\"target\",set:function(t){if(void 0!==t){if(!t||\"object\"!==(void 0===t?\"undefined\":i(t))||1!==t.nodeType)throw new Error('Invalid \"target\" value, use a valid Element');if(\"copy\"===this.action&&t.hasAttribute(\"disabled\"))throw new Error('Invalid \"target\" attribute. Please use \"readonly\" instead of \"disabled\" attribute');if(\"cut\"===this.action&&(t.hasAttribute(\"readonly\")||t.hasAttribute(\"disabled\")))throw new Error('Invalid \"target\" attribute. You can\\'t cut text from elements with \"readonly\" or \"disabled\" attributes');this._target=t}},get:function(){return this._target}}]),c),u=n(1),s=n.n(u),f=n(2),d=n.n(f),h=\"function\"==typeof Symbol&&\"symbol\"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&\"function\"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?\"symbol\":typeof t},p=function(t,e,n){return e&&y(t.prototype,e),n&&y(t,n),t};function y(t,e){for(var n=0;n<e.length;n++){var o=e[n];o.enumerable=o.enumerable||!1,o.configurable=!0,\"value\"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}var m=(function(t,e){if(\"function\"!=typeof e&&null!==e)throw new TypeError(\"Super expression must either be null or a function, not \"+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}(v,s.a),p(v,[{key:\"resolveOptions\",value:function(t){var e=0<arguments.length&&void 0!==t?t:{};this.action=\"function\"==typeof e.action?e.action:this.defaultAction,this.target=\"function\"==typeof e.target?e.target:this.defaultTarget,this.text=\"function\"==typeof e.text?e.text:this.defaultText,this.container=\"object\"===h(e.container)?e.container:document.body}},{key:\"listenClick\",value:function(t){var e=this;this.listener=d()(t,\"click\",function(t){return e.onClick(t)})}},{key:\"onClick\",value:function(t){var e=t.delegateTarget||t.currentTarget;this.clipboardAction&&(this.clipboardAction=null),this.clipboardAction=new l({action:this.action(e),target:this.target(e),text:this.text(e),container:this.container,trigger:e,emitter:this})}},{key:\"defaultAction\",value:function(t){return b(\"action\",t)}},{key:\"defaultTarget\",value:function(t){var e=b(\"target\",t);if(e)return document.querySelector(e)}},{key:\"defaultText\",value:function(t){return b(\"text\",t)}},{key:\"destroy\",value:function(){this.listener.destroy(),this.clipboardAction&&(this.clipboardAction.destroy(),this.clipboardAction=null)}}],[{key:\"isSupported\",value:function(t){var e=0<arguments.length&&void 0!==t?t:[\"copy\",\"cut\"],n=\"string\"==typeof e?[e]:e,o=!!document.queryCommandSupported;return n.forEach(function(t){o=o&&!!document.queryCommandSupported(t)}),o}}]),v);function v(t,e){!function(t,e){if(!(t instanceof e))throw new TypeError(\"Cannot call a class as a function\")}(this,v);var n=function(t,e){if(!t)throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\");return!e||\"object\"!=typeof e&&\"function\"!=typeof e?t:e}(this,(v.__proto__||Object.getPrototypeOf(v)).call(this));return n.resolveOptions(e),n.listenClick(t),n}function b(t,e){var n=\"data-clipboard-\"+t;if(e.hasAttribute(n))return e.getAttribute(n)}e.default=m}],r.c=o,r.d=function(t,e,n){r.o(t,e)||Object.defineProperty(t,e,{enumerable:!0,get:n})},r.r=function(t){\"undefined\"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(t,Symbol.toStringTag,{value:\"Module\"}),Object.defineProperty(t,\"__esModule\",{value:!0})},r.t=function(e,t){if(1&t&&(e=r(e)),8&t)return e;if(4&t&&\"object\"==typeof e&&e&&e.__esModule)return e;var n=Object.create(null);if(r.r(n),Object.defineProperty(n,\"default\",{enumerable:!0,value:e}),2&t&&\"string\"!=typeof e)for(var o in e)r.d(n,o,function(t){return e[t]}.bind(null,o));return n},r.n=function(t){var e=t&&t.__esModule?function(){return t.default}:function(){return t};return r.d(e,\"a\",e),e},r.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},r.p=\"\",r(r.s=6).default;function r(t){if(o[t])return o[t].exports;var e=o[t]={i:t,l:!1,exports:{}};return n[t].call(e.exports,e,e.exports,r),e.l=!0,e.exports}var n,o});"
  },
  {
    "path": "docs/static_site/src/assets/js/copycode.js",
    "content": "/*!\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/* Copy code to clipboard */\n\n$(document).ready(function () {\n  // Regex of prompts to be omitted when copy\n  const LANG_GP = {\n    default: [\">>> \", \"\\\\.\\\\.\\\\.\"],\n    python: [\">>> \", \"\\\\.\\\\.\\\\.\"],\n    scala: [\"scala>\"],\n    java: [],\n    julia: [\"julia> \"],\n    r: [\"> \"],\n    perl: [\"pdl>\"],\n    cpp: [\"\"],\n    bash: [\"\\\\$ \"],\n  };\n\n  /* Functions to get the language of a code block related to a copy button\n   * called one by one until a valid lang is returned \n   * new callbacks should be added before \"default\"\n   */\n  const LANG_GETTER = [\n    (copyBtn) => copyBtn.nextElementSibling.children[0].dataset.lang,\n    (copyBtn) => copyBtn.parentNode.parentNode.classList[0].split(\"-\")[1],\n    () => \"default\",\n  ];\n\n  // Append a copy button to each code block on the page\n  $(\"figure.highlight, div.highlight\").each(function () {\n    const copyBtn = $('<button type=\"button\" class=\"copy-btn\">copy</button>');\n    $(this)\n      .css(\"position\", \"relative\")\n      .prepend(copyBtn)\n      .hover(\n        () => copyBtn.show(),\n        () => copyBtn.hide()\n      );\n  });\n\n  // Clipboard feature based on Clipboard.js v2.0.6\n  const cleanPrompt = function (line, prompts) {\n    let res = line;\n    for (let i = 0; i < prompts.length; i++) {\n      let reg = new RegExp(\"(?:^\\\\s*)\" + prompts[i]);\n      if (reg.test(res)) {\n        res = res.replace(reg, \"\");\n        break;\n      }\n    }\n    return res + \"\\n\";\n  };\n\n  const getCodeBlockLang = function (copyBtn, langGetFunc) {\n    return langGetFunc.reduce((res, getter) => res || getter(copyBtn), \"\");\n  }\n\n  const clipboard = new ClipboardJS(\".copy-btn\", {\n    text: function (trigger) {\n      const lang = getCodeBlockLang(trigger, LANG_GETTER);\n      const langPrompts = LANG_GP[lang] || [];\n      const lines = trigger.parentNode.querySelector(\"code\").textContent.split(\"\\n\");\n      const cleanedCode = lines.reduce((content, line) => content.concat(cleanPrompt(line, langPrompts)), \"\");\n      return cleanedCode.replace(/\\n$/, \"\");\n    },\n  });\n\n  clipboard.on(\"success\", (e) => e.clearSelection());\n});\n"
  },
  {
    "path": "docs/static_site/src/assets/js/feedback.js",
    "content": "/*!\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n$(document).ready(function() {\n  $(\".feedback-answer\").on(\"click\", function () {\n    $(\".feedback-question\").remove();\n    $(\".feedback-answer-container\").remove();\n    $(\".feedback-thank-you\").show();\n    ga(\"send\", {\n      hitType: \"event\",\n      eventCategory: \"Did this page help you?\",\n      eventAction: $(this).attr(\"data-response\"),\n      eventLabel: window.location.pathname || \"unknown\",\n      eventValue: $(this).attr(\"data-response\") === \"yes\" ? 1 : 0\n    });\n  });\n});\n"
  },
  {
    "path": "docs/static_site/src/assets/js/globalSearch.js",
    "content": "/*!\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/* Installation page display functions for install selector.\n   This utility allows direct links to specific install instructions.\n*/\n\n$(document).ready(function () {\n  const default_version = $(\"#gs-current-version-label\").text() || \"master\";\n  // bind docsearch\n  const globalSearch = docsearch({\n    apiKey: \"500f8e78748bd043cc6e4ac130e8c0e7\",\n    indexName: \"apache_mxnet\",\n    inputSelector: \"#global-search\",\n    algoliaOptions: {\n      facetFilters: [\"version:\" + default_version],\n    },\n    debug: false, // Set debug to true if you want to inspect the dropdown\n  });\n\n  const globalSearchMobile = docsearch({\n    apiKey: \"500f8e78748bd043cc6e4ac130e8c0e7\",\n    indexName: \"apache_mxnet\",\n    inputSelector: \"#global-search-mobile\",\n    algoliaOptions: {\n      facetFilters: [\"version:\" + default_version],\n      hitsPerPage: 5,\n    },\n    debug: false, // Set debug to true if you want to inspect the dropdown\n  });\n\n  // search bar animation and event listeners for desktop \n  $(\"#gs-search-icon\").click(function () {\n    $(\".trigger\").fadeOut(\"fast\", function () {\n      $(\"#global-search-form\").css(\"display\", \"inline-block\");\n      $(\"#global-search-close\").show();\n      $(\"#global-search-dropdown-container\").show();\n      $(\"#global-search\")\n        .animate({\n          width: \"300px\",\n        })\n        .focus();\n    });\n  });\n\n  $(\"#global-search-close\").click(function () {\n    $(\"#global-search-dropdown-container\").hide();\n    $(\"#global-search\").animate(\n      {\n        width: \"0px\",\n      },\n      function () {\n        $(this).hide();\n        $(\"#global-search-form\").hide();\n        $(\".trigger\").fadeIn(\"fast\");\n      }\n    );\n  });\n\n  $(\"#global-search-dropdown-container\").click(function (e) {\n    $(\".gs-version-dropdown\").toggle();\n    e.stopPropagation();\n  });\n\n  $(\"ul.gs-version-dropdown li\").each(function () {\n    $(this).on(\"click\", function () {\n      $(\"#global-search\").val(\"\");\n      $(\".gs-version-dropdown li.gs-opt.active\").removeClass(\"active\");\n      $(this).addClass(\"active\");\n      $(\"#gs-current-version-label\").html(this.innerHTML);\n      globalSearch.algoliaOptions = {\n        facetFilters: [\"version:\" + this.innerHTML],\n      };\n    });\n  });\n\n  // search bar event listeners for mobile and tablet \n  $(\"#global-search-dropdown-container-mobile\").click(function (e) {\n    $(\".gs-version-dropdown-mobile\").toggle();\n    e.stopPropagation();\n  });\n\n  $(\"ul.gs-version-dropdown-mobile li\").each(function () {\n    $(this).on(\"click\", function () {\n      $(\"#global-search-mobile\")\n        .val(\"\")\n        .attr(\"placeholder\", \"v - \" + this.innerHTML);\n      $(\".gs-version-dropdown-mobile li.gs-opt.active\").removeClass(\"active\");\n      $(this).addClass(\"active\");\n      globalSearchMobile.algoliaOptions = {\n        facetFilters: [\"version:\" + this.innerHTML],\n        hitsPerPage: 5,\n      };\n    });\n  });\n\n  // Common logic\n  $(document).click(function () {\n    $(\".gs-version-dropdown\").hide();\n    $(\".gs-version-dropdown-mobile\").hide();\n  });\n});\n"
  },
  {
    "path": "docs/static_site/src/assets/js/options.js",
    "content": "/*!\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/* Installation page display functions for install selector.\n   This utility allows direct links to specific install instructions.\n*/\n\n$(document).ready(function () {\n    const dropdownVersions = $(\"#version-dropdown-container ul li\")\n        .toArray()\n        .map((li) => li.innerText);\n\n    function label(lbl) {\n        lbl = lbl.replace(/[ .]/g, '-').toLowerCase();\n\n        return lbl;\n    }\n\n    function urlSearchParams(searchString) {\n        let searchDict = new Map();\n        let searchParams = searchString.substring(1).split(\"&\");\n        searchParams.forEach(function (element) {\n            kvPair = element.split(\"=\");\n            if (kvPair[0] === 'version' && dropdownVersions.indexOf(kvPair[1]) == -1) {\n                kvPair[1] = dropdownVersions[0];\n            }\n            searchDict.set(kvPair[0], kvPair[1]);\n        });\n        return searchDict;\n    }\n\n    function is_a_match(elem, text) {\n\n        if (label(elem.text()).includes(label(text))) {\n            elem.addClass(('active'))\n        }\n    }\n\n    function setSelects(urlParams, dontPushState) {\n        let queryString = '?';\n        $('button.opt').removeClass('active');\n        if (urlParams.get('version')) {\n            versionSelect = urlParams.get('version');\n            $('li.versions').removeClass('active');\n            $('li.versions').each(function () { is_a_match($(this), versionSelect) });\n            $('.current-version').html(versionSelect + '<svg class=\"dropdown-caret\" viewBox=\"0 0 32 32\" class=\"icon icon-caret-bottom\" aria-hidden=\"true\"><path class=\"dropdown-caret-path\" d=\"M24 11.305l-7.997 11.39L8 11.305z\"></path></svg>');\n            queryString += 'version=' + versionSelect + '&';\n        }\n        if (urlParams.get('platform')) {\n            platformSelect = label(urlParams.get('platform'));\n            $('button.opt').each(function(){is_a_match($(this), platformSelect)});\n            queryString += 'platform=' + platformSelect + '&';\n        }\n        if (urlParams.get('language')) {\n            languageSelect = label(urlParams.get('language'));\n            $('button.opt').each(function(){\n                if (label($(this).text()) === label(languageSelect)) {\n                    $(this).addClass(('active'))\n                }\n            });\n            queryString += 'language=' + languageSelect + '&';\n        }\n        if (urlParams.get('processor')) {\n            processorSelect = label(urlParams.get('processor'));\n            $('button.opt').each(function(){is_a_match($(this), processorSelect)});\n            queryString += 'processor=' + processorSelect + '&';\n        }\n        if (urlParams.get('environ')) {\n            environSelect = label(urlParams.get('environ'));\n            $('button.opt').each(function(){is_a_match($(this), environSelect)});\n            queryString += 'environ=' + environSelect + '&';\n        }\n        if (urlParams.get('iot')) {\n            iotSelect = label(urlParams.get('iot'));\n            $('button.opt').each(function(){is_a_match($(this), iotSelect)});\n            queryString += 'iot=' + iotSelect + '&';\n        }\n\n        showContent();\n\n        if (window.location.href.indexOf(\"/get_started\") >= 0 && !dontPushState) {\n            history.pushState(null, null, queryString);\n        }\n    }\n\n    function showContent() {\n        $('.opt-group .opt').each(function () {\n            $('.' + label($(this).text())).hide();\n        });\n        $('.opt-group .active').each(function () {\n            $('.' + label($(this).text())).show();\n        });\n    }\n\n    setSelects(urlSearchParams(window.location.search));\n\n    function setContent() {\n        var el = $(this);\n        let urlParams = urlSearchParams(window.location.search);\n        el.siblings().removeClass('active');\n        el.addClass('active');\n        if ($(this).hasClass(\"versions\")) {\n            $('.current-version').html($(this).text());\n            urlParams.set(\"version\", $(this).text());\n        } else if ($(this).hasClass(\"platforms\")) {\n            urlParams.set(\"platform\", label($(this).text()));\n        } else if ($(this).hasClass(\"languages\")) {\n            urlParams.set(\"language\", label($(this).text()));\n        } else if ($(this).hasClass(\"processors\")) {\n            urlParams.set(\"processor\", label($(this).text()));\n        } else if ($(this).hasClass(\"environs\")) {\n            urlParams.set(\"environ\", label($(this).text()));\n        } else if ($(this).hasClass(\"iots\")) {\n            console.log($(this));\n            urlParams.set(\"iot\", label($(this).text()));\n        }\n        setSelects(urlParams);\n    }\n\n    $('.opt-group').on('click', '.opt', setContent);\n    $('.install-widget').css(\"visibility\", \"visible\");\n    $('.install-content').css(\"visibility\", \"visible\");\n    $(window).on('popstate', function(){\n        setSelects(urlSearchParams(window.location.search), true);\n    });\n\n    let timer;\n    const toggleDropdown = function(showContent) {\n        if (timer) clearTimeout(timer);\n        if (showContent) {\n            timer = setTimeout(function() {\n                $(\".version-dropdown\").show()\n            }, 250);  \n        } else {\n            $(\".version-dropdown\").hide()\n        }\n    }\n\n    $(\"#version-dropdown-container\")\n        .mouseenter(toggleDropdown.bind(null, true))\n        .mouseleave(toggleDropdown.bind(null, false))\n        .click(function() {$(\".version-dropdown\").toggle()});\n\n    $(\"ul.version-dropdown\").click(function(e) {\n        e.preventDefault();\n    });\n});\n"
  },
  {
    "path": "docs/static_site/src/assets/main.scss",
    "content": "---\n---\n/* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n * \n *   http://www.apache.org/licenses/LICENSE-2.0\n * \n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License. */\n\nbody {\n  background-image: url(\"{{'/assets/img/mxnet-background-compressed.jpeg' | relative_url}}\");\n}\n\n\n@import \"minima\";\n@import \"globalSearch\";\n@import \"generalVersionDropdown\";\n@import \"feedback\";\n"
  },
  {
    "path": "docs/static_site/src/index.html",
    "content": "---\n  # Licensed to the Apache Software Foundation (ASF) under one\n  # or more contributor license agreements.  See the NOTICE file\n  # distributed with this work for additional information\n  # regarding copyright ownership.  The ASF licenses this file\n  # to you under the Apache License, Version 2.0 (the\n  # \"License\"); you may not use this file except in compliance\n  # with the License.  You may obtain a copy of the License at\n  #   http://www.apache.org/licenses/LICENSE-2.0\n  # Unless required by applicable law or agreed to in writing,\n  # software distributed under the License is distributed on an\n  # \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  # KIND, either express or implied.  See the License for the\n  # specific language governing permissions and limitations\n  # under the License.\n\nlayout: home\n\nkey_features:\n- title: Hybrid Front-End\n  text: A hybrid front-end seamlessly transitions between Gluon eager imperative mode and symbolic mode to provide both flexibility and speed.\n  icon: /assets/img/circuit.svg\n- title: Distributed Training\n  text: Scalable distributed training and performance optimization in research and production is enabled by the dual Parameter Server and Horovod support.\n  icon: /assets/img/algorithm.svg\n- title: 8 Language Bindings\n  text: Deep integration into Python and support for Scala, Julia, Clojure, Java, C++, R and Perl.\n  icon: /assets/img/programming.svg\n- title: Tools &amp; Libraries\n  text: A thriving ecosystem of tools and libraries extends MXNet and enable use-cases in computer vision, NLP, time series and more.\n  icon: /assets/img/chip.svg\n\necosystem:\n- title: D2L.ai\n  text: An interactive deep learning book with code, math, and discussions. Used at Berkeley, University of Washington and more.\n  icon: /assets/img/textbook.svg\n  link: https://d2l.ai\n- title: GluonCV\n  text: GluonCV is a computer vision toolkit with rich model zoo. From object detection to pose estimation.\n  icon: /assets/img/visual.svg\n  link: https://gluon-cv.mxnet.io\n- title: GluonNLP\n  text: GluonNLP provides state-of-the-art deep learning models in NLP. For engineers and researchers to fast prototype research ideas and products.\n  icon: /assets/img/artificial-intelligence.svg\n  link: https://gluon-nlp.mxnet.io/\n- title: GluonTS\n  text: Gluon Time Series (GluonTS) is the Gluon toolkit for probabilistic time series modeling, focusing on deep learning-based models.\n  icon: /assets/img/line-graph.svg\n  link: https://gluon-ts.mxnet.io/\n\n\ncommunity:\n- title: GitHub\n  text: Report bugs, request features, discuss issues, and more.\n  icon: /assets/img/octocat.png\n  link: https://github.com/apache/mxnet\n- title: Discuss Forum\n  text: Browse and join discussions on deep learning with MXNet and Gluon.\n  icon: /assets/img/mxnet_m.png\n  link: https://discuss.mxnet.io/\n- title: Slack\n  text: Discuss advanced topics. Request access by mail dev@mxnet.apache.org\n  icon: /assets/img/slack-logo-icon.svg\n  link: mailto:dev@mxnet.apache.org\n\n---\n\n"
  },
  {
    "path": "docs/static_site/src/pages/api/api.html",
    "content": "---\nlayout: page\ntitle: Docs\nsubtitle: Documentation for the supported language bindings\naction: Get Started\naction_url: /get_started\npermalink: /api/\ntag: main_docs\n\nfaq_categories:\n- Deployment Environments\n- Model\n- Speed\n- Security\n- Extend and Contribute to Apache MXNet\n\ndocs:\n- title: Python\n  guide_link: /api/python.html\n  api_link: /api/python/docs/api\n  tutorial_link: /api/python/docs/tutorials\n  icon: /assets/img/python_logo.svg\n  tag: python\n- title: C/C++\n  guide_link: /api/cpp.html\n  api_link: /api/cpp/docs/api\n  tutorial_link: /api/cpp/docs/tutorials\n  description:\n  icon: /assets/img/cpp_logo.svg\n  tag: cpp\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n{%- for doc in page.docs -%}\n  {%- if doc.tag == 'python' -%}\n    <h2>Python API</h2>\n    <div class=\"row docs-hero\">\n      <div class=\"col-4 docs-hero-left\">\n        <div class=\"docs-card\">\n            <div class=\"docs-logo-container\">\n                <img class=\"docs-logo-image\" src=\"{{doc.icon | relative_url}}\">\n            </div>\n            <div class=\"docs-action-btn\">\n              <a href=\"{{doc.guide_link | relative_url}}\"> <img src=\"{{'assets/img/compass.svg' | relative_url}}\" class=\"docs-logo-docs\">{{doc.title}} Guide  <span class=\"span-accented\">›</span></a>\n            </div>\n            <div class=\"docs-action-btn\">\n              <a href=\"{{doc.tutorial_link | relative_url}}\"> <img src=\"{{'assets/img/video-tutorial.svg' | relative_url}}\" class=\"docs-logo-docs\">{{doc.title}} Tutorials  <span class=\"span-accented\">›</span></a>\n            </div>\n            <div class=\"docs-action-btn\">\n              <a href=\"{{doc.api_link | relative_url}}\"> <img src=\"{{'assets/img/api.svg' | relative_url}}\" class=\"docs-logo-docs\">{{doc.title}} API Reference  <span class=\"span-accented\">›</span></a>\n            </div>\n        </div>\n      </div>\n\n    <div class=\"col-8 docs-hero-right\">\n        <h4>Python-first API</h4>\n        <p>Apache MXNet provides a comprehensive and flexible Python API to serve a broad community of developers with different levels of experience and wide ranging requirements. Current efforts are focused on the\n        <a href=\"{{doc.api_link | relative_url}}\"></a>Gluon API. Gluon provides a clear, concise, and simple API for deep learning. It makes it easy to prototype, build, and train deep learning models without sacrificing training speed.</p>\n        <p>You can checkout the <a href=\"{{'/ecosystem'|relative_url}}\">rich ecosystem</a> built around Apache MXNet Gluon, including <a href=\"https://d2l.ai\">D2L.ai</a>, <a href=\"https://gluon-cv.mxnet.io\">GluonCV</a>,\n        <a href=\"https://gluon-nlp.mxnet.io\">GluonNLP</a> and <a href=\"https://gluon-ts.mxnet.io\">GluonTS</a>.</p>\n        <p>While most of the usability improvement around training are focused on the python API, the performance of Apache MXNet is accessible through a variety of different language bindings, checkout their respective API and guides below!</p>\n      </div>\n\n    </div>\n  {%- endif -%}\n{%- endfor -%}\n<h2>Other Bindings</h2>\n<div class=\"row\">\n{%- for doc in page.docs -%}\n  {%- if doc.tag != 'python' -%}\n  <div class=\"col-4\">\n      <div class=\"docs-card\">\n          <div class=\"docs-logo-container\">\n              <img class=\"docs-logo-image\" src=\"{{doc.icon | relative_url}}\">\n          </div>\n          <div class=\"docs-action-btn\">\n            <a href=\"{{doc.guide_link | relative_url}}\"> <img src=\"{{'assets/img/compass.svg' | relative_url}}\" class=\"docs-logo-docs\">{{doc.title}} Guide  <span class=\"span-accented\">›</span></a>\n          </div>\n          <div class=\"docs-action-btn\">\n            <a href=\"{{doc.tutorial_link | relative_url}}\"> <img src=\"{{'assets/img/video-tutorial.svg' | relative_url}}\" class=\"docs-logo-docs\">{{doc.title}} Tutorials  <span class=\"span-accented\">›</span></a>\n          </div>\n          <div class=\"docs-action-btn\">\n            <a href=\"{{doc.api_link | relative_url}}\"> <img src=\"{{'assets/img/api.svg' | relative_url}}\" class=\"docs-logo-docs\">{{doc.title}} API Reference  <span class=\"span-accented\">›</span></a>\n          </div>\n      </div>\n  </div>\n  {%- endif -%}\n{%- endfor -%}\n  <div class=\"language-binding-banner\">\n    <h4>Call for Contribution</h4>\n    The Clojure, Java, Julia, R, and Scala language bindings of <a href=\"/versions/{{site.versions[1]}}/api\">MXNet v1.x</a> were removed in v2.x due to some <a href=\"https://github.com/apache/incubator-mxnet/issues/17676\">C APIs being deprecated</a> and the bindings rely on the deprecated APIs. You can still use these language bindings in v1.x.\n    MXNet's new C APIs in v2.x can be used to reestablish your preferred language binding. Your contribution is welcome!\n  </div>\n</div>\n</div> <!-- closing outer wrapper -->\n<div class=\"docs-architecture\">\n    <div class=\"wrapper\">\n        <h2>Apache MXNet Architecture</h2>\n        <p>\n        Building a high-performance deep learning library\n        requires many systems-level design decisions.\n        In this design note, we share the rationale\n        for the specific choices made when designing MXNet.\n        We imagine that these insights may be useful\n        to both deep learning practitioners\n        and builders of other deep learning systems.\n        </p>\n        <h4>Deep Learning System Design Concepts</h4>\n        <p>\n        The following pages address general design concepts for deep learning systems.\n        Mainly, they focus on the following 3 areas:\n        abstraction, optimization, and trade-offs between efficiency and flexibility.\n        Additionally, we provide an overview of the complete MXNet system.\n        </p>\n        <ul>\n        {%- for p in site.pages -%}\n          {%- if p.category == 'architecture' -%}\n            <li><a href=\"{{p.url | relative_url}}\">{{p.title}}</a></li>\n          {%- endif -%}\n        {%- endfor -%}\n        </ul>\n    </div>\n</div>\n<div class=\"docs-dev-guide\">\n  <div class=\"wrapper\">\n      <h2>Developer Guide</h2>\n      <ul>\n        {%- for p in site.pages -%}\n          {%- if p.category == 'Developer Guide' -%}\n            <li><a href=\"{{p.url | relative_url}}\">{{p.title}}</a></li>\n          {%- endif -%}\n        {%- endfor -%}\n      </ul>\n  </div>\n</div>\n<div class=\"docs-faq\">\n    <div class=\"wrapper\">\n        <h2>FAQ</h2>\n        <ul>\n        {%- for faq_c in page.faq_categories -%}\n            <h3>{{faq_c}}</h3>\n            {%- for p in site.pages -%}\n                {%- if p.faq_c == faq_c -%}\n                  <li><a href=\"{{p.url | relative_url}}\">{{p.question}}</a></li>\n                {%- endif -%}\n            {%- endfor -%}\n            <br>\n        {%- endfor -%}\n        </ul>\n    </div>\n</div>\n<div> <!-- reopening to close wrapper -->\n"
  },
  {
    "path": "docs/static_site/src/pages/api/architecture/exception_handling.md",
    "content": "---\nlayout: page_category\ntitle:  Exception Handling in Apache MXNet\ncategory: architecture\npermalink: /api/architecture/exception_handling\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Exception Handling in Apache MXNet\n\nThis tutorial explains the exception handling support in Apache MXNet,\nand provides examples on how to throw and handle exceptions when in a multithreaded context.\nAlthough, the examples are in Python, they can be easily extended to MXNet\nlanguage bindings.\n\nMXNet exceptions can be thrown from two areas:\n- MXNet main thread. For eg. Infershape and InferType.\n- Spawned threads:\n    * By dependency engine for operator execution in parallel\n    * By the iterators, during the data loading, text parsing phase etc.\n\nIn the first case, the exception is thrown and can be handled in the main thread.\nIn the second case, the exception is thrown in a spawned thread, caught and transported to the\nmain thread, where it is rethrown. This tutorial will give more explanation and examples on how\nto handle exceptions for the second case.\n\n## Prerequisites\n\nTo complete this tutorial, we need:\n- MXNet [7b24137](https://github.com/apache/mxnet/commit/7b24137ed45df605defa4ce72ec91554f6e445f0). See Instructions in [Setup and Installation](https://mxnet.io/get_started).\n\n## Exception Handling for Iterators\n\nThe below example shows how to handle exceptions for iterators. In this example,\nwe populate files for data and labels with fewer number of labels compared to the\nnumber of samples. This should throw an exception.\n\nCSVIter uses PrefetcherIter for loading and parsing data.\nThe PrefetcherIter spawns a producer thread in the background which prefetches\nthe data while the main thread consumes the data. The exception is thrown in the spawned\nproducer thread during the prefetching, when the label is not found corresponding to a specific sample.\n\nThe exception is transported to the main thread, where it is rethrown when Next is\ncalled as part of the following line: `for batch in iter(data_train)`.\n\nIn general, Exception may be rethrown as part of `Next` and `BeforeFirst` calls which correspond to `reset()` and `next()` methods in `MXDataIter` for Python language bindings.\n\n```python\nimport os\nimport mxnet as mx\n\ncwd = os.getcwd()\ndata_path = os.path.join(cwd, \"data.csv\")\nlabel_path = os.path.join(cwd, \"label.csv\")\n\nwith open(data_path, \"w\") as fout:\n    for i in range(8):\n        fout.write(\"1,2,3,4,5,6,7,8,9,10\\n\")\n\nwith open(label_path, \"w\") as fout:\n    for i in range(7):\n        fout.write(\"label\"+str(i))\n\ntry:\n    data_train = mx.io.CSVIter(data_csv=data_path, label_csv=label_path, data_shape=(1, 10),\n                               batch_size=4)\n\n    for batch in iter(data_train):\n        print(data_train.getdata().asnumpy())\nexcept mx.base.MXNetError as ex:\n    print(\"Exception handled\")\n    print(ex)\n```\n\n### Limitation\n\nThere is a race condition when your last `next()` call doesnt reach the batch in your dataset where exception occurs. Exception may or may not be thrown in this case depending on which thread wins the race. To avoid this situation, you should try and iterate through your full dataset if you think it can throw exceptions which need to be handled.\n\n\n## Exception Handling for Operators\n\nThe below example shows how to handle exceptions for operators in the imperative mode.\n\nFor the operator case, the dependency engine spawns a number of threads if it is running in the `ThreadedEnginePool` or `ThreadedEnginePerDevice` mode. The final operator is executed in one of the spawned threads.\n\nIf an operator throws an exception during execution, this exception is propagated\ndown the dependency chain. Once there is a synchronizing call i.e. WaitToRead for a variable in the dependency chain, the propagated exception is rethrown.\n\nIn the below example, I illustrate how an exception that occured in the first line is propagated down the dependency chain, and finally is rethrown when we make a synchronizing call to WaitToRead.\n\n```python\nimport mxnet as mx\na = mx.nd.random.normal(0, 1, (2, 2))\nb = mx.nd.random.normal(0, 2, (2, 2))\nc = mx.nd.dot(a, b)\nd = mx.nd.random.normal(0, -1, (2, 2))\ne = mx.nd.dot(c, d)\ne.wait_to_read()\n```\n\nAlthough the above exception occurs when executing the operation which writes to the variable d in one of the child threads, it is thrown only when the synchronization happens as part of the line: `e.wait_to_read()`.\n\nLet us take another example. In the following case, we write to two variables and then `wait_to_read` for both. This example shows that any particular exception will not be thrown more than once.\n\n```python\nimport mxnet as mx\na = mx.nd.random.normal(0, 1, (2, 2))\nb = mx.nd.random.normal(0, -1, (2, 2))\nc, d  = mx.nd.dot(a, b)\ntry:\n    c.asnumpy()\nexcept mx.base.MXNetError as ex:\n    print(\"Exception handled\")\nd.asnumpy()\n```\n"
  },
  {
    "path": "docs/static_site/src/pages/api/architecture/note_data_loading.md",
    "content": "---\nlayout: page_category\ntitle:  Efficient Data Loaders\ncategory: architecture\npermalink: /api/architecture/note_data_loading\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Designing Efficient Data Loaders for Deep Learning\n\nData loading is an important component of any machine learning system.\nWhen we work with tiny datasets, we can get away with loading an entire dataset into GPU memory.\nWith larger datasets, we must store examples in main memory.\nAnd when datasets grow too large to fit into main memory,\ndata loading can become performance-critical.\nIn designing a data loader,\nwe aim to achieve more efficient data loading,\nto spend less effort on data preparation,\nand to present a clean and flexible interface.\n\nWe organize this design note as follows:\n\n* **IO Design Insight:**  Guiding principles in data loading design.\n* **Data Format:** Our solution using dmlc-core's binary recordIO implementation.\n* **Data Loading:** Our method to reduce IO cost by utilizing the threaded iterator provided by dmlc-core.\n* **Interface Design:** Our approach to facilitate writing MXNet data iterators in just a few lines of Python.\n* **Future Extension:** Prospective ideas for making data loading more flexible.\n\nOur analysis will motivate several requirements that an effective IO system should fulfill.\n\n***List of Key Requirements***\n- Small file size.\n- Parallel (distributed) packing of data.\n- Fast data loading and online augmentation.\n- Quick reads from arbitrary parts of the dataset in the distributed setting.\n\n## Design Insight\nTo design an IO system, we must address two kinds of tasks:\ndata preparation and data loading.\nData preparation is usually performed offline,\nwhereas data loading influences the online performance.\nIn this section, we will introduce our insight of IO design involving the two phases.\n\n### Data Preparation\nData preparation describes the process of packing data\ninto a desired format for later processing.\nWhen working with large datasets like ImageNet, this process can be time-consuming.\nIn these cases, there are several heuristics we ought to follow:\n\n- Pack the dataset into small numbers of files. A dataset may contain millions of data instances. Packed data distributes easily from machine to machine.\n- Do the packing once. We don't want to repack data every time run-time settings, like the number of machines, are changed.\n- Process the packing in parallel to save time.\n- Be able to access arbitrary parts of the data easily. This is crucial for distributed machine learning when data parallelism is introduced. Things may get tricky when the data has been packed into several physical data files. The desired behavior could be: the packed data can be logically separated into arbitrary numbers of partitions, no matter how many physical data files there are. For example, if we pack 1000 images into 4 physical files, then each file contains 250 images. If we then use 10 machines to train a DNN, we should be able to load approximately 100 images per machine. Some machines may need images from different physical files.\n\n### Data Loading\nThe next step to consider is how to load the packed data into RAM.\nOur goal is to load the data as quickly as possible.\nThere are several heuristics we try to follow:\n- **Read continuously:** We can read faster when reading from contiguous locations on disk.\n- **Reduce the bytes to be loaded:** We can achieve this by storing data in a compact way, e.g. saving images in JPEG format.\n- **Load and train in different threads:** This avoids computational bottlenecks while loading data.\n- **Save RAM:** Judiciously decide whether to load entire files into RAM.\n\n## Data Format\n\nSince the training of deep neural network often involves large amounts of data,\nthe format we choose should be both efficient and convenient.\nTo achieve our goals, we need to pack binary data into a splittable format.\nIn MXNet, we rely on the binary recordIO format implemented in dmlc-core.\n\n### Binary Record\n\n![baserecordio](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/io/baserecordio.jpg)\nIn MXNet's binary RecordIO, we store each data instance as a record.\n**kMagic** is a *magic number* indicating the start of a record.\n**Lrecord** encodes length and a continue flag.\nIn lrecord,\n- cflag == 0: this is a complete record\n- cflag == 1: start of a multiple-records\n- cflag == 2: middle of multiple-records\n- cflag == 3: end of multiple-records\n\n**Data** is the space to save data content.\n**Pad** is simply a padding space to make record align to 4 bytes.\n\nAfter we pack the data, each file contains multiple records.\nThen, loading can be continuous.\nThis avoids the low performance that can result\nfrom reading random locations on disk.\n\nOne advantage of storing data via records\nis that each record can vary in length.\nThis allows us to save data compactly\nwhen good compression algorithms are available for our data.\nFor example, we can use JPEG format to save image data.\nThe packed data will be much smaller\ncompared with storing uncompressed RGB values for each pixel.\n\nTake ImageNet_1K dataset as an example.\nIf we store the data as 3 * 256 * 256 array of raw RGB values,\nthe dataset would occupy more than **200G**.\nBut after compressing the images using JPEG,\nthey only occupy about **35G** of disk space.\nThis significantly reduces the cost owing to reading from disk.\n\nHere's an example of binary recordIO:\n![baserecordio](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/io/ImageRecordIO.jpg)\nWe first resize the image into 256 * 256,\nthen compress into JPEG format.\nAfter that, we save a header that indicates the index and label\nfor that image to be used when constructing the *Data* field for that record.\nWe then pack several images together into a file.\nYou may want to also review the [example using im2rec.py to create a RecordIO dataset](https://mxnet.apache.org/api/faq/recordio).\n\n### Access Arbitrary Parts Of Data\n\nOne desirable property for a data loader might be:\nThe packed data can be logically sliced into an arbitrary number of partitions,\nno matter how many physical packed data files there are.\nSince binary recordIO can easily locate\nthe start and end of a record using the Magic Number,\nwe can achieve the above goal using the InputSplit\nfunctionality provided by dmlc-core.\n\nInputSplit takes the following parameters:\n- FileSystem *filesys*: dmlc-core wrapper around the IO operations for different file systems, like hdfs, s3, local. User shouldn't need to worry about the difference between file systems anymore.\n- Char *uri*: The URI of files. Note that it could be a list of files because we may pack the data into several physical parts. File URIs are separated by ';'.\n- Unsigned *nsplit*: The number of logical splits. *nsplit* could be different from the number of physical files.\n- Unsigned *rank*: Which split to load in this process.\n\nThe splitting process is demonstrated below:\n- Determine the size of each partition.\n\n![beforepartition](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/io/beforepartition.jpg)\n\n- Approximately partition the records according to file size. Note that the boundary of each part may be located in the middle of a record.\n\n![approxipartition](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/io/approximatepartition.jpg)\n\n-  Set the beginning of partitions in such a way as to avoid splitting records across partitions.\n\n![afterpartition](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/io/afterpartition.jpg)\n\nBy conducting the above operations,\nwe now identify the records belong to each part,\nand the physical data files needed by each logical part.\nInputSplit greatly simplifies data parallelism,\nwhere each process only reads part of the data.\n\nSince our partitioning scheme does not depend on the number of physical data files,\nwe can process a huge dataset like ImageNet_22K in parallel fashion as illustrated below.\nWe don't need to consider distributed loading issue at the preparation time,\njust select the most efficient physical file number\naccording to the dataset size and computing resources available.\n![parallelprepare](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/io/parallelprepare.jpg)\n\n## Data Loading and Preprocessing\n\nWhen the speed of loading and preprocessing can't keep up\nwith the speed of training or evaluation,\nIO can bottleneck the speed of the whole system.\nIn this section, we will introduce a few tricks\nto achieve greater efficiency when loading\nand preprocessing data packed in binary recordIO format.\nWhen applied to the ImageNet dataset, our approach achieves\nthe IO speed of **3000** images/sec **with a normal HDD**.\n\n### Loading and preprocessing on the fly\n\nWhen training deep neural networks,\nwe sometimes must load and preprocess the data\nwhile simultaneously training for the following reasons:\n- When the whole size of the dataset exceeds available RAM size, we can't load it in advance;\n- Sometimes, to make models robust to things like translations, rotations, and small amounts of color shift of noise, we introduce randomness into the training process. In these cases we must re-preprocess the data each time we revisit an example.\n\nIn service of efficiency, we also address multi-threading techniques. Taking Imagenet training as an example, after loading a bunch of image records, we can start multiple threads to simultaneously perform image decoding and image augmentation. We depict this process in the following illustration:\n![process](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/io/process.jpg)\n\n### Hide IO Cost Using Threadediter\n\nOne way to lower IO cost is to pre-fetch the data for next batch on one thread,\nwhile the main thread performs the forward and backward passes for training.\nTo support more complicated training schemes,\nMXNet provides a more general IO processing pipeline\nusing *threadediter* provided by dmlc-core.\nThe key of *threadediter* is to start a stand-alone thread that acts as a data provider,\nwhile the main thread acts as a data consumer as illustrated below.\n\nThe threadediter maintains a buffer of a certain size\nand automatically fills the buffer when it's not full.\nAnd after the consumer finishes consuming part of the data in the buffer,\nthreadediter will reuse the space to save the next part of data.\n![threadediter](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/io/threadediter.png)\n\n## MXNet IO Python Interface\nWe make the IO object as an iterator in numpy.\nBy achieving that, the user can easily access the data\nusing a for-loop or calling next() function.\nDefining a data iterator is very similar to defining a symbolic operator in MXNet.\n\nThe following example code demonstrates a Cifar data iterator.\n\n```python\ndataiter = mx.io.ImageRecordIter(\n    # Dataset Parameter, indicating the data file, please check the data is already there\n    path_imgrec=\"data/cifar/train.rec\",\n    # Dataset Parameter, indicating the image size after preprocessing\n    data_shape=(3,28,28),\n    # Batch Parameter, tells how many images in a batch\n    batch_size=100,\n    # Augmentation Parameter, when offers mean_img, each image will subtract the mean value at each pixel\n    mean_img=\"data/cifar/cifar10_mean.bin\",\n    # Augmentation Parameter, randomly crop a patch of the data_shape from the original image\n    rand_crop=True,\n    # Augmentation Parameter, randomly mirror the image horizontally\n    rand_mirror=True,\n    # Augmentation Parameter, randomly shuffle the data\n    shuffle=False,\n    # Backend Parameter, preprocessing thread number\n    preprocess_threads=4,\n    # Backend Parameter, prefetch buffer size\n    prefetch_buffer=1,\n    # Optional, the device context which data loader optimized for, could be 'gpu' or 'cpu'\n    ctx=\"gpu\",\n    # The out data type, could be 'float32' 'int8' or 'uint8'\n    dtype=\"float32\")\n```\n\nGenerally, to create a data iterator, you need to provide five kinds of parameters:\n\n* **Dataset Param:** Information needed to access the dataset, e.g. file path, input shape.\n* **Batch Param:** Specifies how to form a batch, e.g. batch size.\n* **Augmentation Param:** Which augmentation operations (e.g. crop, mirror) should be taken on an input image.\n* **Backend Param:** Controls the behavior of the backend threads to hide data loading cost.\n* **Auxiliary Param:** Provides options to help with debugging.\n\nUsually, **Dataset Param** and **Batch Param** MUST be given,\notherwise the data batch can't be created.\nOther parameters can be given as needed.\nIdeally, we should separate the MX Data IO into modules,\nsome of which might be useful to expose to users, for example:\n\n* **Efficient prefetcher:** allows the user to write a data loader that reads their customized binary format that automatically gets multi-threaded prefetcher support.\n* **Data transformer:** image random cropping, mirroring, etc. Allows the users to use those tools, or plug in their own customized transformers (maybe they want to add some specific kind of coherent random noise to data, etc.)\n\n## Future Extensions\n\nIn the future, there are some extensions to our data IO\nthat we might consider adding.\nSpecifically, we might add specialized support\nfor applications including image segmentation, object localization, and speech recognition.\nMore detail will be provided when such applications have been running on MXNet.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/architecture/note_engine.md",
    "content": "---\nlayout: page_category\ntitle:  Dependency Engine\ncategory: architecture\npermalink: /api/architecture/note_engine\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Dependency Engine for Deep Learning\n\nWe always want deep learning libraries\nto run faster and scale to larger datasets.\nOne natural approach is to see if we can benefit\nfrom throwing more hardware at the problem,\nas by using multiple GPUs simultaneously.\n\nLibrary designers then ask:\nHow can we *parallelize* computation across devices?\nAnd, more importantly, how can we *synchronize* computation\nwhen we introduce multi-threading?\nA runtime dependency engine is a generic solution to these problems.\n\nIn this document, we examine approaches for using\nruntime dependency scheduling to accelerate deep learning.\nWe aim to explain how runtime dependency scheduling\ncan both speed up and simplify multi-device deep learning.\nWe also explore potential designs for a generic dependency engine\nthat could be both library- and operation-independent.\n\nMost of the discussion of on this page draws inspiration\nfrom the MXNet dependency engine.\nThe dependency tracking algorithm we discuss\nwas primarily developed by [Yutian Li](https://github.com/hotpxl)\nand [Mingjie Wang](https://github.com/jermainewang).\n\n## Dependency Scheduling\n\nAlthough most users want to take advantage of parallel computation,\nmost of us are more familiar with serial programs.\nSo one natural question is: how can we write serial programs\nand build a library to automatically parallelize our programs\nin an asynchronous way?\n\nFor example, in the following code, we can run `B = A + 1`\nand `C = A + 2` in any order, or in parallel:\n\n```python\n    A = 2\n    B = A + 1\n    C = A + 2\n    D = B * C\n```\n\nHowever, it's quite hard to code the sequence manually\nbecause the last operation, `D = B * C`, needs to wait\nfor both of the preceding operations to complete before it starts.\nThe following dependency graph/data flow graph illustrates this.\n\n![Dep Simple](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/engine/dep_simple.png)\n\n\nA dependency engine is a library that takes a sequence of operations\nand schedules them according to the dependency pattern,  potentially in parallel.\nSo in this example, a dependency library\ncould run ```B = A + 1``` and ```C = A + 2``` in parallel,\nand run ```D = B * C``` after those operations complete.\n\n## Problems in Dependency Scheduling\n\nA dependency engine relieves the burden of writing concurrent programs.\nHowever, as operations become parallelized,\nnew dependency tracking problems arise.\nIn this section, we discuss those problems.\n\n### Data Flow Dependency\nData flow dependency describes how the outcome of one computation\ncan be used in other computations.\nEvery dependency engine has to solve the data flow dependency problem.\n\n![Dep Simple](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/engine/dep_simple.png)\n\nBecause we discussed this issue in the preceding section,\nwe include the same figure here. Libraries that have\ndata flow tracking engines include Minerva and Purine2.\n\n### Memory Recycling\nWhen should we recycle the memory that we allocated to the arrays?\nIn serial processing, this is easy to determine.\nWe simply recycle the memory after the variable goes out of scope.\nHowever, as the following figure shows, this is a bit harder in parallel processing.\n\n![Dep Del](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/engine/dep_del.png)\n\nIn this example, because both computations need to use values from A,\nwe can't recycle the memory until both complete.\nThe engine must schedule the memory recycling operations according to the dependencies,\nand ensure that they are executed after both ```B = A + 1``` and ```C = A + 2``` complete.\n\n\n### Random Number Generation\nRandom number generators, which are commonly used in machine learning,\npose interesting challenges for dependency engines.\nConsider the following example:\n\n![Dep Rand](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/engine/dep_rand.png)\n\nIn this example, we are generating random numbers in a sequence.\nAlthough it seems that the two random number generations can be parallelized,\nthis is usually not the case. A pseudo-random number generator (PRNG)\nis not thread-safe because it might cause some internal state\nto mutate when generating a new number.\nEven if the PRNG is thread-safe,\nit is preferable to serialize number generation,\nso we can get reproducible random numbers.\n\n## Case Study: A Dependency Engine for a Multi-GPU Neural Network\n\nIn the last section, we discussed the problems\nwe might face in designing a dependency engine.\nBefore thinking about how to design a generic engine to solve those problems,\nlet's consider how a dependency engine can help in multi-GPU training of a neural network.\nThe following pseudocode Python program illustrates\ntraining one batch on a  two-layer neural network.\n\n```python\n    # Example of one iteration Two GPU neural Net\n    data = next_batch()\n    data[gpu0].copyfrom(data[0:50])\n    data[gpu1].copyfrom(data[50:100])\n    # forward, backprop on GPU 0\n    fc1[gpu0] = FullcForward(data[gpu0], fc1_weight[gpu0])\n    fc2[gpu0] = FullcForward(fc1[gpu0], fc2_weight[gpu0])\n    fc2_ograd[gpu0] = LossGrad(fc2[gpu0], label[0:50])\n    fc1_ograd[gpu0], fc2_wgrad[gpu0] =\n      FullcBackward(fc2_ograd[gpu0] , fc2_weight[gpu0])\n      _, fc1_wgrad[gpu0] = FullcBackward(fc1_ograd[gpu0] , fc1_weight[gpu0])\n    # forward, backprop on GPU 1\n    fc1[gpu1] = FullcForward(data[gpu1], fc1_weight[gpu1])\n    fc2[gpu1] = FullcForward(fc1[gpu1], fc2_weight[gpu1])\n    fc2_ograd[gpu1] = LossGrad(fc2[gpu1], label[50:100])\n    fc1_ograd[gpu1], fc2_wgrad[gpu1] =\n         FullcBackward(fc2_ograd[gpu1] , fc2_weight[gpu1])\n         _, fc1_wgrad[gpu1] = FullcBackward(fc1_ograd[gpu1] , fc1_weight[gpu1])\n    # aggregate gradient and update\n    fc1_wgrad[cpu]  = fc1_wgrad[gpu0] + fc1_wgrad[gpu1]\n    fc2_wgrad[cpu]  = fc2_wgrad[gpu0] + fc2_wgrad[gpu1]\n    fc1_weight[cpu] -= lr *  fc1_wgrad[cpu]\n    fc2_weight[cpu] -= lr *  fc2_wgrad[cpu]\n    fc1_weight[cpu].copyto(fc1_weight[gpu0] , fc1_weight[gpu1])\n    fc2_weight[cpu].copyto(fc2_weight[gpu0] , fc2_weight[gpu1])\n```\nIn this program, the data 0 to 50  is copied to GPU 0,\nand the data 50 to 100 is copied to GPU 1.\nThe calculated gradients are aggregated in the CPU,\nwhich then performs a simple SGD update,\nand copies the updated weight back to each GPU.\nThis is a common way to write a parallel program in a serial manner.\nThe following dependency graph shows how it can be parallelized:\n\n![Dep Net](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/engine/dep_net.png)\n\n***Notes:***\n\n- The gradient can be copied to the CPU as soon as we get the gradient of a layer.\n- The weight can be copied back soon as the weight is updated.\n- In the forward pass, we have a dependency on ```fc1_weight[cpu].copyto(fc1_weight[gpu0] , fc1_weight[gpu1])```\n  from the previous iteration.\n- There is a delay in computation between the last backward pass to layer k and the next forward call to layer k. We can synchronize the weight of layer k *in parallel* with other computation during this delay.\n\nThis approach to optimization is used by multi-GPU deep learning libraries, such as CXXNet.\nThe point is to overlap weight synchronization (communication) with computation.\nHowever, it's not easy to do that, because the copy operation needs to be triggered\nas soon as the backward pass of the layer completes,\nwhich then triggers the reduction, updates, etc.\n\nA dependency engine can schedule these operations and perform multi-threading\nand dependency tracking.\n\n## Designing a Generic Dependency Engine\n\nWe hope that you're convinced that a dependency engine is useful\nfor scaling deep learning programs to multiple devices.\nNow let's discuss how we can design and implement\na generic interface for a dependency engine.\nThis solution isn't the only possible design for a dependency engine.\nIt's an example that we think is useful in most cases.\n\nOur goal is to create a dependency engine that is *generic* and *lightweight*.\nIdeally, we'd like the engine that easily plugs into existing deep learning code,\nand that can scale up to multiple machines with minor modifications.\nTo do that, we need to focus only on dependency tracking,\nnot on assumptions about what users can or can't do.\n\nHere's a summary of goals for the engine:\n\n- The engine should not be aware of what operations it performs, so that users can perform any operations they define.\n- It should not be restricted in what type of objects it can schedule.\n\t- We should be able to schedule dependencies on GPU and CPU memory.\n\t- We should be able to track dependencies on the random number generator, etc.\n- The engine should not allocate resources. It should only track dependencies. Users can allocate their own memory, PRNG, etc.\n\nThe following Python snippet provides an engine interface that might help us reach our goal. Note that a real implementation will be closer to the metal, typically in C++.\n\n```python\n    class DepEngine(object):\n\t    def new_variable():\n\t\t    \"\"\"Return a new variable tag\n\t\t    Returns\n\t\t    -------\n\t\t    vtag : Variable Tag\n\t\t        The token of the engine to represent dependencies.\n\t\t    \"\"\"\n\t\t    pass\n\n\t    def push(exec_func, read_vars, mutate_vars):\n\t\t    \"\"\"Push the operation to the engine.\n\n\t\t    Parameters\n\t\t    ----------\n\t\t    exec_func : callable\n\t\t\t    The real operation to be performed.\n\n\t\t    read_vars : list of Variable Tags\n\t\t\t    The list of variables this operation will read from.\n\n\t\t    mutate_vars : list of Variable Tags\n\t\t\t    The list of variables this operation will mutate.\n\t\t    \"\"\"\n\t\t    pass\n```\n\nBecause we can't make assumptions about what objects we are scheduling, we ask the user to allocate a\n_virtual tag_ that is associated with each object to represent what we need to schedule.\nSo, at the beginning, the user can allocate the variable tag,\nand attach it to each of the objects that we want to schedule.\n\n![Dep Net](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/engine/tag_var.png)\n\nThe user then calls `push` to tell the engine about the function to execute.\nThe user also needs to specify the dependencies of the operation,\nusing `read_vars` and `write_vars`:\n\n- `read_vars` are variable tags for objects that the operation will _read from_, without changing their internal state.\n- `mutate_vars` are variable tags for objects whose internal states the operation will mutate.\n\n![Push Op](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/engine/push_var.png)\n\nThe preceding figure shows how to push operation `B = A + 1` to the dependency engine. `B.data` and\n`A.data` are the allocated space. Note that the engine is *only aware of variable tags*.\nAny execution function can be processed.\nThis interface is generic for the operations and resources we want to schedule.\n\nFor fun, let's look at how the engine internals work with the tags by considering the following code snippet:\n\n```\n    B = A + 1\n    C = A + 2\n    A = C * 2\n    D = A + 3\n```\n\nThe first line reads variable `A` and mutates variable `B`. The second line reads variable `A` and mutates variable `C`. And so on.\n\nThe engine maintains a queue for each variable, as the following animation shows for each of the four lines. Green blocks represents a read action, while red blocks represent mutations.\n\n![Dependency Queue](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/engine/dep_queue.gif)\n\nUpon building this queue, the engine sees that the first two green blocks at the beginning of `A`'s queue could actually be run in parallel because they are both read actions and won't conflict with each other. The following graph illustrates this point.\n\n![Dependency Parallelism](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/engine/dep_parallel.png)\n\nOne cool thing about all this scheduling is that it's not confined to numerical calculations.\nBecause everything that is scheduled is only a tag, the engine could schedule everything!\n\nThe following figure gives a complete push sequence of the programs we mentioned in previous sections.\n\n![Push Seq](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/engine/push_seq.png)\n\n### Porting Existing Code to the Dependency Engine\nBecause the generic interface doesn't control things like memory allocation and which operation to execute,\nmost existing code can be scheduled by the dependency engine in two steps:\n\n\n1. Allocate the variable tags associated with resources like memory blob, PRNGS.\n2. Call `push` with the execution function as the original code to execute, and put the variable tags of\n  corresponding resources correctly in `read_vars` and `mutate_vars`.\n\n## Implementing the Generic Dependency Engine\n\nWe have described the generic engine interface and\nhow it can be used to schedule various operations.\nIn this section, we provide a high-level discussion\nof how to implement such an engine.\n\nThe general idea is as follows:\n\n- Use a queue to track all of the pending dependencies on each variable tag.\n- Use a counter on each operation to track how many dependencies are yet to be fulfilled.\n- When operations are completed, update the state of the queue and dependency counters to schedule new operations.\n\nThe following figure illustrates the scheduling algorithm\nand might give you a better sense of what is going on in the engine.\n\n![Dep Tracking](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/engine/engine_queue_step.png)\n\nBelow, we show another example involving random number generators.\n\n![Dep Rand](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/engine/engine_queue_rand.png)\n\nAs you can see, the purpose of the algorithm is to update pending queues\nof operations and to make the right state transition when an operation has completed.\nMore care should be taken to make sure the state transitions\nare done in a way that's safe for threads.\n\n### Separate Dependency Tracking with Running Policy\nIf you're reading carefully, you might have noticed\nthat the preceding section shows only the algorithm\nfor deciding when an operation can be executed.\nWe didn't show how to actually run an operation.\nIn practice, there can be many different policies.\nFor example, we can either use a global thread-pool to run all operations,\nor use a specific thread to run operations on each device.\n\nThis running policy is usually independent of dependency tracking,\nand can be separated out as either an independent module\nor a virtual interface of base-dependency tracking modules.\nDeveloping an elegant runtime policy that is fair\nto all operations and schedules is an interesting systems problem itself.\n\n## Discussion\n\nThe design that we discussed in this article\nisn't the only solution to the dependency tracking problem.\nIt's just one example of how we might approach this.\nTo be sure, some of these design choices are debatable.\nWe'll discuss some of them in this section.\n\n### Dynamic vs. Static\nThe dependency engine interface discussed in this topic is somewhat dynamic\nin the sense that the user can push operations one by one,\ninstead of declaring the entire dependency graph (static).\nDynamic scheduling can require more overhead\nthan static declarations, in terms of data structure.\nHowever, it also enables more flexibility, such as supporting auto parallelism\nfor imperative programs or a mixture of imperative and symbolic programs.\nYou can also add some level of predeclared operations\nto the interface to enable data structure reuse.\n\n### Mutation vs. Immutable\nThe generic engine interface presented in this page\nsupports explicit scheduling for mutation.\nIn a typical data flow engine, the data are usually immutable.\nWorking with immutable data has a lot of benefits.\nFor example, immutable data is generally more suitable for parallelization,\nand facilitates better fault tolerance in a distributed setting (by way of re-computation).\n\nHowever, immutability presents several challenges:\n\n- It's harder to schedule resource contention problems, as arise when dealing with random numbers and deletion.\n- The engine usually needs to manage resources (memory, random number) to avoid conflicts. It's harder to plug in user-allocated space, etc.\n- Preallocated static memory isn't available, again because the usual pattern is to write to a preallocated layer space, which is not supported if data is immutable.\n\nAllowing mutation mitigates these issues.\n\n\n## Source Code of the Generic Dependency Engine\n[MXNet](https://github.com/apache/mxnet) provides an implementation\nof the generic dependency engine described in this page.\nWe welcome your contributions.\n\n## Next Steps\n\n* [Squeeze the Memory Consumption of Deep Learning](note_memory)\n* [Efficient Data Loading Module for Deep Learning](note_data_loading)\n"
  },
  {
    "path": "docs/static_site/src/pages/api/architecture/note_memory.md",
    "content": "---\nlayout: page_category\ntitle:  Memory Consumption\ncategory: architecture\npermalink: /api/architecture/note_memory\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Optimizing Memory Consumption in Deep Learning\n\nOver the last ten years, a constant trend in deep learning\nis towards deeper and larger networks.\nDespite rapid advances in hardware performance,\ncutting-edge deep learning models continue to push the limits of GPU RAM.\nSo even today, it's always desirable to find ways\nto train larger models while consuming less memory.\nDoing so enables us to train faster, using larger batch sizes,\nand consequently achieving a higher GPU utilization rate.\n\nIn this document, we explore techniques for optimizing\nmemory allocation for deep neural networks.\nWe discuss a few candidate solutions.\nWhile our proposals are by no means exhaustive,\nthese solutions are instructive and allow us to\nintroduce the major design issues at play.\n\n## Computation Graph\n\nFirst, let's revisit the idea of the computation graph.\nA computation graph describes the (data flow) dependencies\nbetween the operations in the deep network.\nThe operations performed in the graph\ncan be either fine-grained or coarse-grained.\nThe following figure shows two examples of computation graphs.\n\n![Comp Graph Example](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/memory/comp_graph_example.png)\n\nThe concept of a computation graph is explicitly encoded in packages like Theano and CGT.\nIn other libraries, computation graphs appear implicitly as network configuration files.\nThe major difference in these libraries comes down to how they calculate gradients.\nThere are mainly two ways: performing back-propagation on the _same_ graph\nor explicitly representing a _backwards path_ to calculate the required gradients.\n\n![Backward Graph](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/memory/back_graph.png)\n\nLibraries like Caffe, CXXNet, and Torch take the former approach,\nperforming back-prop on the original graph.\nLibraries like Theano and CGT take the latter approach,\nexplicitly representing the backward path.\nIn this discussion, we adopt the *explicit backward path* approach\nbecause it has several advantages for optimization.\n\nHowever, we should emphasize that choosing the explicit backward path approach doesn't restrict us\nto symbolic libraries, such as Theano and CGT. We can also use the explicit backward path for gradient calculation of\nlayer-based (which ties forward and backward together) libraries. The following graph shows how to do this.\nBasically, we introduce a backward node that links to the forward node of the graph and calls the ```layer.backward```\nin the backward operations.\n\n![Backward Layer](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/memory/explicit_back_layer.png)\n\nThis discussion applies to almost all existing deep learning libraries.\n(There are differences between libraries,  e.g., higher-order differentiation, which is beyond the scope of this topic.)\n\nWhy is the explicit backward path better? Let's explain it with two examples.\nThe first reason is that the explicit backward path\nclearly describes the dependency between computations.\nConsider the following case, where we want to get\nthe gradient of A and B. As we can see clearly from the graph,\nthe computation of the ```d(C)``` gradient doesn't depend on F.\nThis means that we can free the memory of ```F```\nright after the forward computation is done.\nSimilarly, the memory of ```C``` can be recycled.\n\n![Backward Prune](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/memory/back_dep_prune.png)\n\nAnother advantage of the explicit backward path\nis the ability to have a different backward path,\ninstead of a mirror of forward one.\nA common example is the split connection case,\nas shown in the following figure.\n\n![Backward Agg](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/memory/back_agg_grad.png)\n\nIn this example, the output of B is referenced by two operations.\nIf we want to do the gradient calculation in the same\nnetwork, we need to introduce an explicit split layer.\nThis means we need to do the split for the forward pass, too.\nIn this figure, the forward pass doesn't contain a split layer,\nbut the graph will automatically insert a gradient\naggregation node before passing the gradient back to B.\nThis helps us to save the memory cost of allocating the output of the split layer,\nand the operation cost of replicating the data in the forward pass.\n\nIf we adopt the explicit backward approach,\nthere's no difference between the forward pass and the backward pass.\nWe simply step through the computation graph in chronological order\nand carry out computations.\nThis makes the explicit backward approach easy to analyze.\nWe just need to answer the question:\nhow do we allocate memory for each output node of a computation graph?\n\n\n## What Can Be Optimized?\n\nAs you can see, the computation graph is a useful way\nto discuss memory allocation optimization techniques.\nAlready, we've shown how you can save some memory\nby using the explicit backward graph.\nNow let's explore further optimizations,\nand see how we might determine reasonable baselines for benchmarking.\n\nAssume that we want to build a neural network with `n` layers.\nTypically, when implementing a neural network,\nwe need to allocate node space for both the output of each layer\nand the gradient values used during back-propagation.\nThis means we need roughly `2 n` memory cells.\nWe face the same requirement when using the explicit backward graph approach\nbecause the number of nodes in a backward pass\nis roughly the same as in a forward pass.\n\n### In-place Operations\nOne of the simplest techniques we can employ\nis _in-place memory sharing_ across operations.\nFor neural networks, we can usually apply this technique\nfor the operations corresponding to activation functions.\nConsider the following case, where we want\nto compute the value of three chained sigmoid functions.\n\n![Inplace op](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/memory/alloc_inline.png)\n\nBecause we can compute sigmoid ```in-place```,\nusing the same memory for input and output,\nwe can compute an arbitrary-length chain\nof sigmoid functions using constant memory.\n\nNote: it's easy to make mistakes when implementing in-place optimization.\nConsider the following case, where the value of B is used not only by C, but also by F.\n\n![In-place trap](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/memory/alloc_inline_trap.png)\n\nWe can't perform in-place optimization because the value of B\nis still needed after ```C=sigmoid(B)``` is computed.\nAn algorithm that simply does in-place optimization\nfor every sigmoid operation might fall into such trap,\nso we need to be careful about when we can use it.\n\n### Standard Memory Sharing\nIn-place operations are not the only places where we can share memory.\nIn the following example, because the value of B is no longer needed\nafter we compute E, we can reuse B's memory to hold the result of E.\n\n![Normal Sharing](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/memory/alloc_normal.png)\n\n*Memory sharing doesn't necessarily require the same data shape*.\nNote that in the preceding example, the shapes of `B` and `E` can differ.\nTo handle such a situation, we can allocate a memory region\nof size equal to the maximum of that required by `B` and `E` and share it between them.\n\n### Example of Real Neural Network Allocation\nOf course, these are only toy examples and they address only the computation of the forward pass.\nBut the same ideas apply to real neural networks.\nThe following figure shows an allocation plan for a two-layer perceptron.\n\n![Net Alloc](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/memory/alloc_mlp.png)\n\nIn this example:\n\n- In-place optimization is applied when computing ```act1```, ```d(fc1)```, ```out``` and ```d(fc2)```.\n- Memory is shared between ```d(act1)``` and ```d(A)```.\n\n## Memory Allocation Algorithm\n\nSo far, we've discussed general techniques for optimizing memory allocation.\nWe've seen that there are traps to avoid,\nas demonstrated in the case of in-place memory optimization.\nSo, how can we allocate memory correctly?\nThis is not a new problem.\nFor example, it is very similar\nto the problem with register allocation in compilers.\nThere might be techniques that we can borrow.\nWe're not attempting to give a comprehensive review of techniques here,\nbut rather to introduce some simple\nbut useful tricks to attack the problem.\n\nThe key problem is that we need to place resources\nso that they don't conflict with each other.\nMore specifically, each variable has a *life time*\nbetween the time it gets computed until the last time it is used.\nIn the case of the multi-layer perceptron,\nthe *life time* of ```fc1``` ends after ```act1``` get computed.\n\n![Net Alloc](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/memory/alloc_mlp.png)\n\nThe principle is *to allow memory sharing only between variables whose lifetimes don't overlap*.\nThere are multiple ways to do this.\nYou can construct the conflicting graph\nwith each variable as a node and link the edge\nbetween variables with overlapping lifespans,\nand then run a graph-coloring algorithm.\nThis likely has ```$O(n^2)$``` complexity,\nwhere ```n``` is the number of nodes in the graph.\nThis might be too costly.\n\nLet's consider another simple heuristic.\nThe idea is to simulate the procedure of traversing the graph,\nand keep a count of future operations that depends on the node.\n\n![Alloc](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/memory/alloc_step.png)\n\n- An in-place optimization can be performed when only the current operation depends on the source (i.e., ```count==1```).\n- Memory can be recycled into the box on the upper right corner when the ```count``` goes to 0.\n- When we need new memory, we can either get it from the box or allocate a new one.\n\n***Note:*** During the simulation, no memory is allocated.\nInstead, we keep a record of how much memory each node needs,\nand allocate the maximum of the shared parts in the final memory plan.\n\n## Static vs. Dynamic Allocation\n\nThe preceding strategy exactly simulates\nthe dynamic memory allocation procedure\nin imperative languages, such as Python.\nThe ```count``` is the reference counter for each memory object,\nand the object gets garbage collected\nwhen the reference counter goes to 0.\nIn that sense,\nwe are simulating dynamic memory allocation once\nto create a static allocation plan.\nCan we simply use an imperative language\nthat dynamically allocates and deallocates memory?\n\nThe major difference is that static allocation is only done once,\nso we can afford to use more complicated algorithms.\nFor example, we can search for memory sizes\nthat are similar to the required memory block.\nThe Allocation can also be made graph aware.\nWe'll talk about that in the next section.\nDynamic allocation puts more pressure\non fast memory allocation and garbage collection.\n\nThere is also one takeaway for users\nwho want to rely on dynamic memory allocations:\n*do not unnecessarily reference objects*.\nFor example, if we organize all of the nodes in a list\nand store then in a Net object,\nthese nodes will never get dereferenced, and we gain no space.\nUnfortunately, this is a common way to organize code.\n\n\n## Memory Allocation for Parallel Operations\n\nIn the previous section, we discussed\nhow we can *simulate* running the procedure\nfor a computation graph to get a static allocation plan.\nHowever, optimizing for parallel computation presents other challenges\nbecause resource sharing and parallelization are on the two ends of a balance.\nLet's look at the following two allocation plans for the same graph:\n\n![Parallel Alloc](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/memory/parallel_alloc.png)\n\nBoth allocation plans are valid\nif we run the computation serially,\nfrom ```A[1]``` to ```A[8]```.\nHowever, the allocation plan on the left\nintroduces additional dependencies,\nwhich means we can't run computation of ```A[2]``` and ```A[5]``` in parallel.\nThe plan on the right can.\nTo parallelize computation, we need to take greater care.\n\n### Be Correct and Safe First\nBeing correct is our first principle.\nThis means to execute in a way that takes implicit dependency\nmemory sharing into consideration.\nYou can do this by adding the implicit dependency edge to the execution graph.\nOr, even simpler, if the execution engine is mutation aware,\nas described in [our discussion of dependency engine design](note_engine),\npush the operation in sequence\nand write to the same variable tag\nthat represents the same memory region.\n\nAlways produce a safe memory allocation plan.\nThis means never allocate the same memory\nto nodes that can be parallelized.\nThis might not be ideal when memory reduction is more desirable,\nand we don't gain too much when we can get benefit\nfrom multiple computing streams simultaneously executing on the same GPU.\n\n### Try to Allow More Parallelization\nNow we can safely perform some optimizations.\nThe general idea is to try and encourage memory sharing between nodes that can't be parallelized.\nYou can do this by creating an ancestor relationship\ngraph and querying it during allocation,\nwhich costs approximately ```$O(n^2)$``` in time to construct.\nWe can also use a heuristic here,\nfor example, color the path in the graph.\nAs shown in the following figure,\nwhen you try to find the longest paths in the graph,\ncolor them the same color and continue.\n\n![Path Color](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/memory/graph_color.png)\n\nAfter you get the color of the node,\nyou allow sharing (or encourage sharing)\nonly between nodes of the same color.\nThis is a stricter version of the ancestor relationship,\nbut it costs only `$O(n)$` of time\nif you search for only the first `k` path.\n\nThis is by no means the only solution.\nMore sophisticated approaches might exist:\n\n## How Much Can you Save?\n\nWe've discussed the techniques and algorithms you can use\nto squeeze memory usage for deep learning.\nHow much can you really save by using these techniques?\n\nOn coarse-grained operation graphs\nthat are already optimized for big operations,\nyou can reduce memory consumption roughly *by half*.\nYou can reduce memory usage even more\nif you are optimizing a fine-grained computation network\nused by symbolic libraries, such as Theano. Most of the ideas in this article inspired the design of _MXNet_.\n\nAlso, you will notice that memory cost, for forward pass only execution, is extremely low compared to running both forward and backward pass. This is simply because there's  more memory reuse if you run only the forward pass.\n\nSo here are two takeaways:\n\n- Use a computation graph to allocate memory.\n- For deep learning models, prediction consumes much less memory than training.\n\n\n## Next Steps\n\n* [Efficient Data Loading Module for Deep Learning](note_data_loading)\n"
  },
  {
    "path": "docs/static_site/src/pages/api/architecture/overview.md",
    "content": "---\nlayout: page_category\ntitle:  Apache MXNet System Architecture\ncategory: architecture\npermalink: /api/architecture/overview\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# MXNet System Architecture\n\n![System Overview](https://raw.githubusercontent.com/dmlc/dmlc.github.io/master/img/mxnet/system/overview.png)\n\nThis figure shows the major modules and components of the MXNet system and their interaction. The modules are:\n\n- Runtime Dependency Engine: Schedules and executes the\n  operations according to their read/write dependency.\n- Storage Allocator: Efficiently allocates and recycles memory blocks\n  on host (CPU) and devices (GPUs).\n- Resource Manager: Manages global resources, such as the random number generator\n  and temporal space.\n- NDArray: Dynamic, asynchronous n-dimensional arrays,\n  which provide flexible imperative programs for MXNet.\n- Symbolic Execution: Static symbolic graph executor,\n  which provides efficient symbolic graph execution and optimization.\n- Operator: Operators that define static forward and gradient\n  calculation (backprop).\n- SimpleOp: Operators that extend NDArray operators and symbolic operators\n  in a unified fashion.\n- Symbol Construction: Symbolic construction, which provides a way to construct\n  a computation graph (net configuration).\n- KVStore: Key-value store interface for efficient parameter synchronization.\n- Data Loading(IO): Efficient distributed data loading and augmentation.\n\n# MXNet System Components\n\n## Execution Engine\n\nYou can use MXNet's engine not only for deep learning,\nbut for any domain-specific problem.\nIt's designed to solve a general problem:\nexecute a bunch of functions following their dependencies.\nExecution of any two functions with dependencies should be serialized.\nTo boost performance, functions with no dependencies *can* be executed in parallel.\nFor a general discussion of this topic,\nsee our [notes on the dependency engine](note_engine).\n\n### Interface\n\nThe following API is the core interface for the execution engine:\n\n```c++\n    virtual void PushSync(Fn exec_fun, Context exec_ctx,\n                          std::vector<VarHandle> const& const_vars,\n                          std::vector<VarHandle> const& mutate_vars) = 0;\n```\nThis API allows you to push a function (`exec_fun`),\nalong with its context information and dependencies, to the engine.\n`exec_ctx` is the context information in which the `exec_fun` should be executed,\n`const_vars` denotes the variables that the function reads from,\nand `mutate_vars` are the variables to be modified.\nThe engine provides the following guarantee:\n\n>*The execution of any two functions\nthat modify a common variable\nis serialized in their push order.*\n\n### Function\n\nThe function type of the engine is:\n\n```c++\n    using Fn = std::function<void(RunContext)>;\n```\n`RunContext` contains runtime information, which is determined by the engine:\n\n```c++\n    struct RunContext {\n        // stream pointer which could be safely cast to\n        // cudaStream_t* type\n\t    void *stream;\n    };\n```\nAlternatively, you could use `mxnet::engine::DAGEngine::Fn`, which has the same type definition.\n\nAll of the functions are executed by the engine's internal threads.\nIn such a model, it's usually not a good idea to push *blocking* functions\nto the engine (usually for dealing with I/O tasks like disk, web service, UI, etc.)\nbecause it will occupy the execution thread and reduce total throughput.\nIn that case, we provide another *asynchronous* function type:\n\n```c++\n    using Callback = std::function<void()>;\n    using AsyncFn = std::function<void(RunContext, Callback)>;\n```\nIn the `AsyncFn` function, you can pass the heavy part to your own threads\nand safely exit the body of the function.\nThe engine doesn't consider the function finished\nuntil the `Callback` function is called.\n\n### Context\n\nYou can specify the `Context` of the function to be executed within.\nThis usually includes whether the function should be run on a CPU or a GPU,\nand if you specify a GPU, which GPU to use.\n`Context` is different from `RunContext`.\n`Context` contains device type (GPU/CPU) and device id,\n while `RunContext` contains information that can be decided only during runtime,\n for example, on which stream the function should be executed.\n\n### VarHandle\n\n`VarHandle` is used to specify the dependencies of functions.\nThe MXNet engine is designed to be decoupled from other MXNet modules.\nSo `VarHandle` is like an engine-provided token you use\nto represent the external resources the functions can use or modify.\nIt's designed to be lightweight, so creating,\ndeleting, or copying a variable incurs little overhead.\nUpon pushing the functions, you need to specify the variables\nthat will be used (immutable) in the `const_vars` vector,\nand the variables that will be modified (mutable) in the `mutate_vars` vector.\nThe engine uses one rule for resolving the dependencies among functions:\n\n>*The execution of any two functions when one of them modifies at least one common variable is serialized in their push order.*\n\nFor example, if `Fn1` and `Fn2` both mutate `V2` then `Fn2`\nis guaranteed to be executed after `Fn1`\nif `Fn2` is pushed after `Fn1`.\nOn the other hand, if `Fn1` and `Fn2` both use `V2`,\ntheir actual execution order could be random.\n\nThis design allows the engine to schedule *state-mutating* operations in a manner\nthat minimizes calls to allocate new memory.\nFor example, the weight update function in DNN\ncan now use the `+=` operator\nto update the weights in place,\nrather than generating a new weight array each time.\n\nTo create a variable, use the `NewVar()` API.\nTo delete a variable, use the `PushDelete` API.\n\n### Push and Wait\n\n*All `Push` APIs are asynchronous.* The API call returns immediately\nregardless of whether the pushed `Fn` is finished or not.\nThis allows the engine to start computing at the same time\nas the user thread is pushing functions.\n`Push` APIs are not thread-safe.\nTo be specific, only one thread should make engine API calls at a time.\n\nIf you want to wait for a specific `Fn` to finish,\ninclude a callback function in the closure,\nand call the function at the end of your `Fn`.\n\nIf you want to wait for all `Fn`s\nthat involve (use or mutate) a certain variable to finish,\nuse the `WaitForVar(var)` API.\n\nIf you want to wait for all pushed `Fn`s to finish,\nuse the `WaitForAll()` API.\n\n### Save Object Creation Cost\n\nIn some cases, you need to push several functions to the engine for a long period of time.\nIf the computation of these functions is light,\nthe overhead of copying lambdas and creating use/mutate variable lists becomes relatively high.\nWe provide an API to create an `OprHandle` beforehand:\n\n```c++\n    virtual OprHandle NewOperator(AsyncFn fn,\n                                  std::vector<VarHandle> const& const_vars,\n                                  std::vector<VarHandle> const& mutate_vars) = 0;\n```\nYou can keep pushing the `OprHandle` without repeatedly creating them:\n\n```c++\n    virtual void Push(OprHandle op, Context exec_ctx) = 0;\n```\nTo delete it, call the `DeleteOperator(OprHandle op)` API.\nEnsure that the operator has finished computing before calling this API.\n\n\n## Operators in MXNet\n\nIn MXNet, an operator is a class that contains both actual computation logic\nand auxiliary information that can aid the system in performing optimizations,\nlike in-place updates and auto-derivatives.\nTo understand the remainder of the document,\nwe recommend that you familiarize yourself with the `mshadow` library,\nbecause all operators compute on the tensor-like structure `mshadow::TBlob`\nprovided by the system during runtime.\n\nMXNet's operator interface allows you to:\n\n* Reduce memory allocation cost by specifying in-place updates.\n* Hide some internal arguments from Python to make it cleaner.\n* Define the relationships among input tensors and output tensors,\nwhich allows the system to perform shape checking for you.\n* Acquire additional temporary spaces from the system\nto perform computation (e.g., calling `cudnn` routines).\n\n### Operator Interface\n\n`Forward` is the core operator interface:\n\n```c++\n    virtual void Forward(const OpContext &ctx,\n                         const std::vector<TBlob> &in_data,\n                         const std::vector<OpReqType> &req,\n                         const std::vector<TBlob> &out_data,\n                         const std::vector<TBlob> &aux_states) = 0;\n```\nThe `OpContext` structure is:\n\n```c++\n           struct OpContext {\n             int is_train;\n             RunContext run_ctx;\n             std::vector<Resource> requested;\n           }\n```\nIt describes whether the operator is in the train or test phase,\nwhich device the operator should be run on (in `run_ctx`),\nand requested resources (covered in the following sections).\n\n- `in_data` and `out_data` represent the input and output tensors, respectively.\nAll of the tensor spaces have been allocated by the system.\n- `req` denotes how the computation results are written into the `out_data`.\nIn other words, `req.size() == out_data.size()` and `req[i]`\ncorrespond to the write type of `out_data[i]`.\n\n- The `OpReqType` is defined as:\n\n```c++\n           enum OpReqType {\n             kNullOp,\n             kWriteTo,\n             kWriteInplace,\n             kAddTo\n           };\n```\n  Normally, the types of all `out_data` should be `kWriteTo`,\n  meaning that the provided `out_data` tensor is a *raw* memory block,\n  so the operator should write results directly into it.\n  In some cases, for example when calculating the `gradient` tensor,\n  it would be great if we could accumulate the result,\n  rather than directly overwrite the tensor contents\n  so that  no extra space needs to be created each time.\n  In such a case, the corresponding `req` type is set as `kAddTo`,\n  indicating that a `+=` should be called.\n\n- `aux_states` is intentionally designed for auxiliary tensors used to help computation. Currently, it is useless.\n\nAside from the `Forward` operator, you could optionally implement the `Backward` interface:\n\n```c++\n    virtual void Backward(const OpContext &ctx,\n                          const std::vector<TBlob> &out_grad,\n                          const std::vector<TBlob> &in_data,\n                          const std::vector<TBlob> &out_data,\n                          const std::vector<OpReqType> &req,\n                          const std::vector<TBlob> &in_grad,\n                          const std::vector<TBlob> &aux_states);\n```\nThis interface follows the same design principle as the `Forward` interface,\nexcept that `out_grad`, `in_data`, and `out_data` are given,\nand the operator computes `in_grad` as the results.\n The naming strategy is similar to Torch's convention,\n and can be summarized in following figure:\n\n[input/output semantics figure]\n\nSome operators might not require all of the following:\n`out_grad`, `in_data` and `out_data`.\nYou can specify these dependencies with the `DeclareBackwardDependency` interface in `OperatorProperty`.\n\n### Operator Property\n\nOne convolution might have several implementations,\nand you might want to switch among them to achieve the best performance.\nTherefore, we separate the operator *semantic* interfaces\nfrom the implementation interface (`Operator` class)\ninto the `OperatorProperty` class.\nThe `OperatorProperty` interface consists of:\n\n* **InferShape:**\n\n```c++\n           virtual bool InferShape(mxnet::ShapeVector *in_shape,\n                                   mxnet::ShapeVector *out_shape,\n                                   mxnet::ShapeVector *aux_shape) const = 0;\n```\n\nThis interface has two purposes:\n* Tell the system the size of each input and output tensor,\n  so it can allocate space for them before the `Forward` and `Backward` call.\n* Perform a size check to make sure that there isn't an obvious error before running.\n  The shape in `in_shape` is set by the system\n  (from the `out_shape` of the previous operators).\n  It returns `false` when there is not enough information\n  to infer shapes or throws an error when the shape is inconsistent.\n\n* **Request Resources:** Operations like `cudnnConvolutionForward` need a work space for computation.\nIf the system can manage that, it could then perform optimizations,\nlike reuse the space, and so on.\nMXNet defines two interfaces to achieve this:\n\n```c++\n           virtual std::vector<ResourceRequest> ForwardResource(\n               const mxnet::ShapeVector &in_shape) const;\n           virtual std::vector<ResourceRequest> BackwardResource(\n               const mxnet::ShapeVector &in_shape) const;\n```\n  The `ResourceRequest` structure (in `resource.h`) currently contains only a type flag:\n\n```c++\n           struct ResourceRequest {\n             enum Type {\n               kRandom,  // get a mshadow::Random<xpu> object\n               kTempSpace,  // request temporary space\n             };\n             Type type;\n           };\n```\n  If `ForwardResource` and `BackwardResource` return non-empty arrays,\n  the system offers the corresponding resources through the `ctx` parameter\n  in the `Forward` and `Backward` interface of `Operator`.\n  Basically, to access those resources, simply write:\n\n```c++\n           auto tmp_space_res = ctx.requested[kTempSpace].get_space(some_shape, some_stream);\n           auto rand_res = ctx.requested[kRandom].get_random(some_stream);\n```\n  For an example, see `src/operator/cudnn_convolution-inl.h`.\n\n* **Backward dependency:** Let's look at two different operator signatures\n(we name all of the arguments for demonstration purposes):\n\n```c++\n           void FullyConnectedForward(TBlob weight, TBlob in_data, TBlob out_data);\n           void FullyConnectedBackward(TBlob weight, TBlob in_data, TBlob out_grad, TBlob in_grad);\n\n           void PoolingForward(TBlob in_data, TBlob out_data);\n           void PoolingBackward(TBlob in_data, TBlob out_data, TBlob out_grad, TBlob in_grad);\n```\n  Note that `out_data` in `FullyConnectedForward`\n  is not used by `FullyConnectedBackward`,\n  while `PoolingBackward` requires all of the arguments of `PoolingForward`.\n  Therefore, for `FullyConnectedForward`,\n  the `out_data` tensor once consumed could be safely freed\n  because the backward function will not need it.\n  This provides a chance for the system to collect some tensors\n  as garbage as soon as possible.\n  To specify this situation, we provide an interface:\n\n```c++\n          virtual std::vector<int> DeclareBackwardDependency(\n               const std::vector<int> &out_grad,\n               const std::vector<int> &in_data,\n               const std::vector<int> &out_data) const;\n```\n  The `int` element of the argument vector is an ID\n  to distinguish different arrays.\n  Let's see how this interface specifies different dependencies\n  for `FullyConnected` and `Pooling`:\n\n ```c++\n           std::vector<int> FullyConnectedProperty::DeclareBackwardDependency(\n               const std::vector<int> &out_grad,\n               const std::vector<int> &in_data,\n               const std::vector<int> &out_data) const {\n             return {out_grad[0], in_data[0]};  // NOTE: out_data[0] is NOT included\n           }\n           std::vector<int> PoolingProperty::DeclareBackwardDependency(\n               const std::vector<int> &out_grad,\n               const std::vector<int> &in_data,\n               const std::vector<int> &out_data) const {\n             return {out_grad[0], in_data[0], out_data[0]};\n           }\n```\n\n* **In place Option:** To further save the cost of memory allocation,\nyou can use in-place updates.\nThey are appropriate for element-wise operations\nwhen the input tensor and output tensor have the same shape.\nYou specify and in-place update with the following interface:\n\n```c++\n           virtual std::vector<std::pair<int, void*>>    ElewiseOpProperty::ForwardInplaceOption(\n               const std::vector<int> &in_data,\n               const std::vector<void*> &out_data) const {\n             return { {in_data[0], out_data[0]} };\n           }\n           virtual std::vector<std::pair<int, void*>> ElewiseOpProperty::BackwardInplaceOption(\n               const std::vector<int> &out_grad,\n               const std::vector<int> &in_data,\n               const std::vector<int> &out_data,\n               const std::vector<void*> &in_grad) const {\n             return { {out_grad[0], in_grad[0]} }\n           }\n```\n  This tells the system that the `in_data[0]` and `out_data[0]` tensors could share the same memory spaces during `Forward`, and so do `out_grad[0]` and `in_grad[0]` during `Backward`.\n\n  >**Important:** Even if you use the preceding specification, it's *not* guaranteed that the input and output tensors will share the same space. In fact, this is only a suggestion for the system, which makes the final decision. However, in either case, the decision is completely transparent to you, so the actual `Forward` and `Backward` implementation does not need to consider that.\n\n* **Expose Operator to Python:** Because of the restrictions of C++, you need user to implement following interfaces:\n\n```c++\n           // initial the property class from a list of key-value string pairs\n           virtual void Init(const vector<pair<string, string>> &kwargs) = 0;\n           // return the parameters in a key-value string map\n           virtual map<string, string> GetParams() const = 0;\n           // return the name of arguments (for generating signature in python)\n           virtual vector<string> ListArguments() const;\n           // return the name of output values\n           virtual vector<string> ListOutputs() const;\n           // return the name of auxiliary states\n           virtual vector<string> ListAuxiliaryStates() const;\n           // return the number of output values\n           virtual int NumOutputs() const;\n           // return the number of visible outputs\n           virtual int NumVisibleOutputs() const;\n```\n\n### Create an Operator from the Operator Property\n\n `OperatorProperty` includes all *semantic* attributes of an operation. It's also responsible for creating the `Operator` pointer for actual computation.\n\n#### Create Operator\nImplement the following interface in `OperatorProperty`:\n\n```c++\n    virtual Operator* CreateOperator(Context ctx) const = 0;\n```\nFor example:\n\n```c++\n    class ConvolutionOp {\n     public:\n      void Forward( ... ) { ... }\n      void Backward( ... ) { ... }\n    };\n    class ConvolutionOpProperty : public OperatorProperty {\n     public:\n      Operator* CreateOperator(Context ctx) const {\n        return new ConvolutionOp;\n      }\n    };\n```\n\n#### Parametrize Operator\nWhen implementing a convolution operator, you need to know the kernel size,\nthe stride size, padding size, and so on.\nThese parameters should be passed to the operator\nbefore any `Forward` or `Backward` interface is called.\nTo do so, you could define a `ConvolutionParam` structure, as follows:\n\n```c++\n    #include <dmlc/parameter.h>\n    struct ConvolutionParam : public dmlc::Parameter<ConvolutionParam> {\n      mxnet::TShape kernel, stride, pad;\n      uint32_t num_filter, num_group, workspace;\n      bool no_bias;\n    };\n```\nPut it in `ConvolutionOpProperty`, and pass it to the operator class during construction:\n\n```c++\n    class ConvolutionOp {\n     public:\n      ConvolutionOp(ConvolutionParam p): param_(p) {}\n      void Forward( ... ) { ... }\n      void Backward( ... ) { ... }\n     private:\n      ConvolutionParam param_;\n    };\n    class ConvolutionOpProperty : public OperatorProperty {\n     public:\n      void Init(const vector<pair<string, string>& kwargs) {\n        // initialize param_ using kwargs\n      }\n      Operator* CreateOperator(Context ctx) const {\n        return new ConvolutionOp(param_);\n      }\n     private:\n      ConvolutionParam param_;\n    };\n```\n\n#### Register the Operator Property Class and the Parameter Class to MXNet\nUse the following macros to register the parameter structure and the operator property class to MXNet:\n\n```c++\n    DMLC_REGISTER_PARAMETER(ConvolutionParam);\n    MXNET_REGISTER_OP_PROPERTY(Convolution, ConvolutionOpProperty);\n```\nThe first argument is the name string, the second is the property class name.\n\n### Interface Summary\n\nWe've almost covered the entire interface required to define a new operator. Let's do a recap:\n\n* Use the `Operator` interface to write your computation logic (`Forward` and `Backward`).\n* Use the `OperatorProperty` interface to:\n  - Pass the parameter to the operator class (you can use the `Init` interface).\n  - Create an operator using the `CreateOperator` interface.\n  - Correctly implement the operator description interface, such as the names of arguments, etc.\n  - Correctly implement the `InferShape` interface to set the output tensor shape.\n  - [Optional] If additional resources are needed, check `ForwardResource` and `BackwardResource`.\n  - [Optional] If `Backward` doesn't need all of the input and output of `Forward`, check `DeclareBackwardDependency`.\n  - [Optional] If in-place update is supported, check `ForwardInplaceOption` and `BackwardInplaceOption`.\n* Register the `OperatorProperty` class and the parameter class.\n\n## Unifying the NDArray Operator and Symbolic Operator\nNDArray operations are similar to symbolic operations,\nexcept that sometimes you can't write in place to the operands\nwithout a complete dependency graph.\nHowever, the logic underlying NDArray and symbolic operations are almost identical.\n*SimpleOp*, a new unified operator API,\nunifies different invoking processes\nand returns to the fundamental elements of operators.\nBecause most mathematical operators attend to one or two operands,\nand more operands make dependency-related optimization useful,\nthe unified operator is specifically designed for unary and binary operations.\n\nConsider the elements of an operation.\nIdeally, you need only functions and derivatives\nto describe an operation.\nLet's restrict that to the space of unary and binary operations.\nHow do we classify all operations to maximize the possibility\nof in-place write optimization?\nNote that you can separate functions by the number of operands.\nDerivatives are a bit more complex.\nTo construct a dependency graph, you need to know whether output value,\ninput data, or neither are needed alongside head gradient.\nGradient functions in the unified API are differentiated\nby the types of operands it takes for calculation.\n\nBefore you learn more about the SimpleOp interface,\n we recommend that you review the\n [mshadow library guide](https://github.com/dmlc/mshadow/tree/master/guide)\n because  calculations will be done in the `mshadow::TBlob` structure.\n\nIn the following example, we'll create an operator\nfunctioning as a smooth l1 loss,\nwhich is a mixture of l1 loss and l2 loss. The loss itself can be written as:\n\n```\n    loss = outside_weight .* f(inside_weight .* (data - label))\n    grad = outside_weight .* inside_weight .* f'(inside_weight .* (data - label))\n```\n `.*` stands for element-wise multiplication, and `f`, `f'` is the smooth l1 loss function,\nwhich we are assuming is in `mshadow` for now.\nAt first glance, it's impossible to implement\nthis particular loss as a unary or binary operator.\nBut we have automatic differentiation in symbolic execution.\nThat simplifies the loss to `f` and `f'` directly.\nThis loss is no more complex than a `sin` or an `abs` function,\nand can certainly be implemented as a unary operator.\n\n## SimpleOp: The Unified Operator API\n### Define Shapes\nThe `mshadow` library requires explicit memory allocation.\nAs a consequence, all data shapes\nmust be provided before any calculation occurs.\n Before we proceed with defining functions and gradient,\nlet's check input data shape consistency and provide output shape.\n\n```cpp\n    typedef mxnet::TShape (*UnaryShapeFunction)(const mxnet::TShape& src,\n                                         const EnvArguments& env);\n    typedef mxnet::TShape (*BinaryShapeFunction)(const mxnet::TShape& lhs,\n                                          const mxnet::TShape& rhs,\n                                          const EnvArguments& env);\n```\nYou can use `mshadow::TShape` to check input data shape and designate output data shape.\nIf you don't define this function, the default output shape is the same as the input shape.\nIn the case of a binary operator, the shape of `lhs` and `rhs` is checked as the same by default.\n\nYou can also use shape functions to check if any additional arguments and resources are present.\nRefer to the additional usages of `EnvArguments` to accomplish this.\n\nBefore we start on our smooth l1 loss example, we define a `XPU` to `cpu` or `gpu` in the header\n`smooth_l1_unary-inl.h` implementation so that we reuse the same code in `smooth_l1_unary.cc` and\n`smooth_l1_unary.cu`.\n\n```cpp\n    #include <mxnet/operator_util.h>\n    #if defined(__CUDACC__)\n    #define XPU gpu\n    #else\n    #define XPU cpu\n    #endif\n```\nIn our smooth l1 loss example, it's okay to use the default behavior whereby the output has the same shape as the source.\nWritten explicitly, it is:\n\n```cpp\n    inline mxnet::TShape SmoothL1Shape_(const mxnet::TShape& src,\n                                 const EnvArguments& env) {\n      return mxnet::TShape(src);\n    }\n```\n\n### Define Functions\nCreate a unary or binary function with one output: `mshadow::TBlob`.\n\n```cpp\n    typedef void (*UnaryFunction)(const TBlob& src,\n                                  const EnvArguments& env,\n                                  TBlob* ret,\n                                  OpReqType req,\n                                  RunContext ctx);\n    typedef void (*BinaryFunction)(const TBlob& lhs,\n                                   const TBlob& rhs,\n                                   const EnvArguments& env,\n                                   TBlob* ret,\n                                   OpReqType req,\n                                   RunContext ctx);\n```\n* Functions are differentiated by the types of input arguments.\n* `RunContext ctx` contains information needed during runtime for execution.\n\n```cpp\n        struct RunContext {\n          void *stream;  // the stream of the device, can be NULL or Stream<gpu>* in GPU mode\n          template<typename xpu> inline mshadow::Stream<xpu>* get_stream() // get mshadow stream from Context\n        }  // namespace mxnet\n```\n  `mshadow::stream<xpu> *s = ctx.get_stream<xpu>();` is an example of obtaining a stream from `ctx`.\n* `OpReqType req` denotes how computation results are written into `ret`.\n\n```cpp\n        enum OpReqType {\n          kNullOp,  // no operation, do not write anything\n          kWriteTo,  // write gradient to provided space\n          kWriteInplace,  // perform an in-place write\n          kAddTo  // add to the provided space\n        };\n```\n  A macro is defined in `operator_util.h` for a simplified use of `OpReqType`.\n  `ASSIGN_DISPATCH(out, req, exp)` checks `req` and performs an assignment.\n\nIn our smooth l1 loss example, we use `UnaryFunction` to define the function of this operator.\n\n```cpp\n    template<typename xpu>\n    void SmoothL1Forward_(const TBlob& src,\n                          const EnvArguments& env,\n                          TBlob *ret,\n                          OpReqType req,\n                          RunContext ctx) {\n      using namespace mshadow;\n      using namespace mshadow::expr;\n      mshadow::Stream<xpu> *s = ctx.get_stream<xpu>();\n      real_t sigma2 = env.scalar * env.scalar;\n      MSHADOW_TYPE_SWITCH(ret->type_flag_, DType, {\n        mshadow::Tensor<xpu, 2, DType> out = ret->get<xpu, 2, DType>(s);\n        mshadow::Tensor<xpu, 2, DType> in = src.get<xpu, 2, DType>(s);\n        ASSIGN_DISPATCH(out, req,\n                        F<mshadow_op::smooth_l1_loss>(in, ScalarExp<DType>(sigma2)));\n      });\n    }\n```\nAfter obtaining `mshadow::Stream` from `RunContext`, we get `mshadow::Tensor` from `mshadow::TBlob`.\n`mshadow::F` is a shortcut to initiate a `mshadow` expression. The macro `MSHADOW_TYPE_SWITCH(type, DType, ...)`\nhandles details on different types, and the macro `ASSIGN_DISPATCH(out, req, exp)` checks `OpReqType` and\nperforms actions accordingly. `sigma2` is a special parameter in this loss, which we will cover later.\n\n### Define Gradients (Optional)\nCreate a gradient function with various types of inputs.\n\n```cpp\n    // depending only on out_grad\n    typedef void (*UnaryGradFunctionT0)(const OutputGrad& out_grad,\n                                        const EnvArguments& env,\n                                        TBlob* in_grad,\n                                        OpReqType req,\n                                        RunContext ctx);\n    // depending only on out_value\n    typedef void (*UnaryGradFunctionT1)(const OutputGrad& out_grad,\n                                        const OutputValue& out_value,\n                                        const EnvArguments& env,\n                                        TBlob* in_grad,\n                                        OpReqType req,\n                                         RunContext ctx);\n    // depending only on in_data\n    typedef void (*UnaryGradFunctionT2)(const OutputGrad& out_grad,\n                                        const Input0& in_data0,\n                                        const EnvArguments& env,\n                                        TBlob* in_grad,\n                                        OpReqType req,\n                                        RunContext ctx);\n```\nGradient functions of binary operators have similar structures, except that `Input`, `TBlob`, and `OpReqType`\nare doubled.\n\n`GradFunctionArgument`\n\n  `Input0`, `Input`, `OutputValue`, and `OutputGrad` all share the structure of `GradFunctionArgument`,\n  which is defined as:\n\n  ```cpp\n      struct GradFunctionArgument {\n          TBlob data;\n      }\n  ```\n\nIn our smooth l1 loss example, note that it's an `f'(x)`,\nwhich utilizes input for the gradient calculation,\nso the `UnaryGradFunctionT2` is suitable.\nTo enable the chain rule of the gradient,\nwe also need to multiply `out_grad` from the top to the result of `in_grad`.\n\n```cpp\n    template<typename xpu>\n    void SmoothL1BackwardUseIn_(const OutputGrad& out_grad,\n                                const Input0& in_data0,\n                                const EnvArguments& env,\n                                TBlob *in_grad,\n                                OpReqType req,\n                                RunContext ctx) {\n      using namespace mshadow;\n      using namespace mshadow::expr;\n      mshadow::Stream<xpu> *s = ctx.get_stream<xpu>();\n      real_t sigma2 = env.scalar * env.scalar;\n      MSHADOW_TYPE_SWITCH(in_grad->type_flag_, DType, {\n        mshadow::Tensor<xpu, 2, DType> src = in_data0.data.get<xpu, 2, DType>(s);\n        mshadow::Tensor<xpu, 2, DType> ograd = out_grad.data.get<xpu, 2, DType>(s);\n        mshadow::Tensor<xpu, 2, DType> igrad = in_grad->get<xpu, 2, DType>(s);\n         ASSIGN_DISPATCH(igrad, req,\n                        ograd * F<mshadow_op::smooth_l1_gradient>(src, ScalarExp<DType>(sigma2)));\n      });\n    }\n```\n\n### Register SimpleOp to MXNet\nAfter creating the shape, function, and gradient, restore them into both an NDArray operator and\na symbolic operator. To simplify this process, use the registration macro defined in `operator_util.h`.\n\n```cpp\n    MXNET_REGISTER_SIMPLE_OP(Name, DEV)\n    .set_shape_function(Shape)\n    .set_function(DEV::kDevMask, Function<XPU>, SimpleOpInplaceOption)\n    .set_gradient(DEV::kDevMask, Gradient<XPU>, SimpleOpInplaceOption)\n    .describe(\"description\");\n```\n`SimpleOpInplaceOption` is defined as:\n\n```cpp\n    enum SimpleOpInplaceOption {\n      kNoInplace,  // do not allow inplace in arguments\n      kInplaceInOut,  // allow inplace in with out (unary)\n      kInplaceOutIn,  // allow inplace out_grad with in_grad (unary)\n      kInplaceLhsOut,  // allow inplace left operand with out (binary)\n      kInplaceOutLhs  // allow inplace out_grad with lhs_grad (binary)\n    };\n```\n\nIn our example, we have a gradient function that relies on input data, so the function can't be written in\nplace. The output gradient has no purpose after gradient computation, so the gradient can be written in place.\n\n```cpp\n    MXNET_REGISTER_SIMPLE_OP(smooth_l1, XPU)\n    .set_function(XPU::kDevMask, SmoothL1Forward_<XPU>, kNoInplace)\n    .set_gradient(XPU::kDevMask, SmoothL1BackwardUseIn_<XPU>, kInplaceOutIn)\n    .set_enable_scalar(true)\n    .describe(\"Calculate Smooth L1 Loss(lhs, scalar)\");\n```\nRemember from the discussion of shape functions that a default behavior without `set_shape_function` forces the inputs\n(if they're binary) to be the same shape and yield the same shape for output. We'll discuss `set_enable_scalar` later.\n\n### NDArray Operator Summary\n* Create a shape function for determining the output shape.\n* Create a function as the forward routine by choosing a suitable function type.\n* Create a gradient as the backward routine by choosing a suitable gradient type.\n* Register the operator using the registration process.\n\n## Additional Information on SimpleOp\n### Using SimpleOp on EnvArguments\nSome operations might need a scalar as input, such as a  gradient scale, a set of keyword arguments\ncontrolling behavior, or a temporary space to speed up calculations.`EnvArguments` provides additional arguments and resources to make calculations more scalable\nand efficient.\n\n```cpp\n    struct EnvArguments {\n      real_t scalar;  // scalar argument, if enabled\n      std::vector<std::pair<std::string, std::string> > kwargs;  // keyword arguments\n      std::vector<Resource> resource;  // pointer to the resources requested\n    };\n```\n\nMore registration parameters are required to enable these additional features. To prevent confusion on parameters, `scalar` and `kwargs`\ncan't be present at the same time. To enable `scalar`, use\n`set_enable_scalar(bool enable_scalar)` in registration. Then, in forward functions and gradients, the `scalar` can be accessed from `env.scalar` as in the function parameter `EnvArguments env`.\n\nTo enable `kwargs`, use `set_enable_kwargs(bool enable_kwargs)` in registration. Then, in forward\nfunctions and gradients, additional arguments are contained in `env.kwarg`, which is defined as\n`std::vector<std::pair<std::string, std::string> >`. Use the DMLC parameter structure to\nsimplify parsing keyword arguments. For more details, see the [guide on parameter structure](https://github.com/dmlc/dmlc-core/blob/master/doc/parameter.md).\n\nAdditional resources like `mshadow::Random<xpu>` and temporary memory space can also be requested and\naccessed from `EnvArguments.resource`. The registration routine is `set_resource_request(ResourceRequest req)`\nor `set_resource_request(const std::vector<ResourceRequest>)`, where `mxnet::ResourceRequest` is defined as:\n\n```cpp\n    struct ResourceRequest {\n      enum Type {  // Resource type, indicating what the pointer type is\n        kRandom,  // mshadow::Random<xpu> object\n        kTempSpace  // A dynamic temp space that can be arbitrary size\n      };\n      Type type;  // type of resources\n    };\n```\nRegistration will request the declared resource requests from `mxnet::ResourceManager`, and place resources\nin `std::vector<Resource> resource` in `EnvArguments`. To access resources, use the following:\n\n```cpp\n    auto tmp_space_res = env.resources[0].get_space(some_shape, some_stream);\n    auto rand_res = env.resources[0].get_random(some_stream);\n```\nFor an example, see `src/operator/loss_binary_op-inl.h`.\n\nIn our smooth l1 loss example, a scalar input is needed to mark the turning point of a loss function. Therefore,\nin the registration process, we use `set_enable_scalar(true)`, and use `env.scalar` in function and gradient\ndeclarations.\n\n### Crafting a Tensor Operation\nBecause computation utilizes the `mshadow` library and we sometimes don't have functions readily available, we\ncan craft tensor operations in operator implementations. If you define such functions as element-wise, you\ncan implement them as a `mxnet::op::mshadow_op`. `src/operator/mshadow_op.h` that contains a lot of `mshadow_op`,\nfor example. `mshadow_op` are expression mappers. They deal with the scalar case of desired functions. For details, see\n[mshadow expression API guide](https://github.com/dmlc/mshadow/tree/master/doc).\n\nIf an operation can't be done in an element-wise way, like the softmax loss and gradient, then you need to create a new tensor operation. You need to create as `mshadow` function and as `mshadow::cuda`\nfunction directly. For details, see the `mshadow` library. For an example, see `src/operator/roi_pooling.cc`.\n\nIn our smooth l1 loss example, we create two mappers, namely the scalar cases of smooth l1 loss and gradient.\n\n```cpp\n    namespace mshadow_op {\n    struct smooth_l1_loss {\n      // a is x, b is sigma2\n      MSHADOW_XINLINE static real_t Map(real_t a, real_t b) {\n        if (a > 1.0f / b) {\n          return a - 0.5f / b;\n        } else if (a < -1.0f / b) {\n          return -a - 0.5f / b;\n        } else {\n          return 0.5f * a * a * b;\n        }\n      }\n    };\n    }\n```\nThe gradient, which can be found in `src/operator/smooth_l1_unary-inl.h`, is similar.\n\n### Beyond Two Operands\nThe new unified API is designed to fulfill the fundamentals of an operation. For operators with more than two inputs,\nmore than one output, or that need more features, see the original [Operator API](overview#operators-in-mxnet).\n"
  },
  {
    "path": "docs/static_site/src/pages/api/architecture/program_model.md",
    "content": "---\nlayout: page_category\ntitle:  Deep Learning Programming Paradigm\ncategory: architecture\npermalink: /api/architecture/program_model\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Deep Learning Programming Paradigm\n\nHowever much we might ultimately care about performance,\nwe first need working code before we can start worrying about optimization.\nWriting clear, intuitive deep learning code can be challenging,\nand the first thing any practitioner must deal with is the language syntax itself.\nComplicating matters, of the many deep learning libraries out there,\neach has its own approach to programming style.\n\nIn this document, we focus on two of the most important high-level design decisions:\n1. Whether to embrace the _symbolic_ or _imperative_ paradigm for mathematical computation.\n2. Whether to build networks with bigger (more abstract) or more atomic operations.\n\nThroughout, we'll focus on the programming models themselves.\nWhen programming style decisions may impact performance, we point this out,\nbut we don't dwell on specific implementation details.\n\n\n## Symbolic vs. Imperative Programs\n\nIf you are a Python or C++ programmer, then you're already familiar with imperative programs.\nImperative-style programs perform computation as you run them.\nMost code you write in Python is imperative, as is the following NumPy snippet.\n\n```python\n    import numpy as np\n    a = np.ones(10)\n    b = np.ones(10) * 2\n    c = b * a\n    d = c + 1\n```\nWhen the program executes ```c = b * a```, it runs the actual numerical computation.\n\nSymbolic programs are a bit different. With symbolic-style programs,\nwe first define a (potentially complex) function abstractly.\nWhen defining the function, no actual numerical computation takes place.\nWe define the abstract function in terms of placeholder values.\nThen we can compile the function, and evaluate it given real inputs.\nIn the following example, we rewrite the imperative program from above\nas a symbolic-style program:\n\n```python\n    A = Variable('A')\n    B = Variable('B')\n    C = B * A\n    D = C + Constant(1)\n    # compiles the function\n    f = compile(D)\n    d = f(A=np.ones(10), B=np.ones(10)*2)\n```\nAs you can see, in the symbolic version, when ```C = B * A``` is executed, no computation occurs.\nInstead, this operation generates a _computation graph_ (also called a _symbolic graph_)\nthat represents the computation.\nThe following figure shows a computation graph to compute ```D```.\n\n![Comp Graph](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/prog_model/comp_graph.png)\n\nMost symbolic-style programs contain, either explicitly or implicitly, a *compile* step.\nThis converts the computation graph into a function that we can later call.\nIn the above example, numerical computation only occurs in the last line of code.\nThe defining characteristic of symbolic programs is their clear separation\nbetween building the computation graph and executing it.\nFor neural networks, we typically define the entire model as a single compute graph.\n\nAmong other popular deep learning libraries, Torch, Chainer, and Minerva embrace the imperative style.\nExamples of symbolic-style deep learning libraries include Theano, CGT, and TensorFlow.\nWe might also view libraries like CXXNet and Caffe, which rely on configuration files, as symbolic-style libraries.\nIn this interpretation, we'd consider the content of the configuration file as defining the computation graph.\n\nNow that you understand the difference between these two programming models, let's compare the advantages of each.\n\n\n### Imperative Programs Tend to be More Flexible\n\nWhen you're using an imperative-style library from Python, you are writing in Python.\nNearly anything that would be intuitive to write in Python, you could accelerate by calling down in the appropriate places to the imperative deep learning library.\nOn the other hand, when you write a symbolic program, you may not have access to all the familiar Python constructs, like iteration.\nConsider the following imperative program, and think about how you can translate this into a symbolic program.\n\n```python\n    a = 2\n    b = a + 1\n    d = np.zeros(10)\n    for i in range(d):\n        d += np.zeros(10)\n```\nThis wouldn't be so easy if the Python for-loop weren't supported by the symbolic API.\nWhen you write a symbolic program in Python, you're *not* writing in Python.\nInstead, you're writing in a domain-specific language (DSL) defined by the symbolic API.\nThe symbolic APIs found in deep learning libraries\nare powerful DSLs that generate callable computation graphs for neural networks.\n<!-- In that sense, config-file input libraries are all symbolic. -->\n\nIntuitively, you might say that imperative programs\nare more *native* than symbolic programs.\nIt's easier to use native language features.\nFor example, it's straightforward to print out the values\nin the middle of computation or to use native control flow and loops\nat any point in the flow of computation.\n\n### Symbolic Programs Tend to be More Efficient\n\nAs we've seen, imperative programs tend to be flexible\nand fit nicely into the programming flow of a host language.\nSo you might wonder, why do so many deep learning libraries\nembrace the symbolic paradigm?\nThe main reason is efficiency, both in terms of memory and speed.\nLet's revisit our toy example from before.\n\n```python\n    import numpy as np\n    a = np.ones(10)\n    b = np.ones(10) * 2\n    c = b * a\n    d = c + 1\n    ...\n```\n\n![Comp Graph](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/prog_model/comp_graph.png)\n\nAssume that each cell in the array occupies 8 bytes of memory.\nHow much memory do you need to execute this program in the Python console?\n\nAs an imperative program we need to allocate memory at each line.\nThat leaves us allocating 4 arrays of size 10.\nSo we'll need `4 * 10 * 8 = 320` bytes.\nOn the other hand, if we built a computation graph,\nand knew in advance that we only needed `d`,\nwe could reuse the memory originally allocated for intermediate values.\nFor example, by performing computations in-place,\nwe might recycle the bits allocated for ```b``` to store `c`.\nAnd we might recycle the bits allocated for `c` to store `d`.\nIn the end we could cut our memory requirement in half,\nrequiring just `2 * 10 * 8 = 160` bytes.\n\nSymbolic programs are more *restricted*.\nWhen we call `compile` on D, we tell the system\nthat only the value of `d` is needed.\nThe intermediate values of the computation,\nin this case ```c```, is then invisible to us.\n\nWe benefit because the symbolic programs\ncan then safely reuse the memory for in-place computation.\nBut on the other hand, if we later decide that we need to access `c`, we're out of luck.\nSo imperative programs are better prepared to encounter all possible demands.\nIf we ran the imperative version of the code in a Python console,\nwe could inspect any of the intermediate variables in the future.\n\n<!-- Of course, this is somewhat misleading, because garbage collection can occur in imperative programs and memory could then be reused.\nHowever, imperative programs do need to be \"prepared to encounter all possible demands,\" and this limits the optimization you can perform. This is true for non-trivial cases, such\nas gradient calculation, which we discuss in next section. -->\n\nSymbolic programs can also perform another kind of optimization, called operation folding.\nReturning to our toy example, the multiplication and addition operations\ncan be folded into one operation, as shown in the following graph.\nIf the computation runs on a GPU processor,\none GPU kernel will be executed, instead of two.\nIn fact, this is one way we hand-craft operations\nin optimized libraries, such as CXXNet and Caffe.\nOperation folding improves computation efficiency.\n\n![Comp Graph Folded](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/prog_model/comp_graph_fold.png)\n\nNote, you can't perform operation folding in imperative programs,\nbecause the intermediate values might be referenced in the future.\nOperation folding is possible in symbolic programs\nbecause you get the entire computation graph,\nand a clear specification of which values will be needed and which are not.\n\n\n### Case Study: Backprop and AutoDiff\n\nIn this section, we compare the two programming models\non the problem of auto differentiation, or backpropagation.\nDifferentiation is of vital importance in deep learning\nbecause it's the mechanism by which we train our models.\nIn any deep learning model, we define a _loss function_.\nA _loss function_ measures how far the model is from the desired output.\nWe then typically pass over training examples (pairs of inputs and ground-truth outputs).\nAt each step we update the model's _parameters_ to minimize the loss.\nTo determine the direction in which to update the parameters,\nwe need to take the derivative of the loss function with respect to the parameters.\n\nIn the past, whenever someone defined a new model,\nthey had to work out the derivative calculations by hand.\nWhile the math is reasonably straightforward,\nfor complex models, it can be time-consuming and tedious work.\nAll modern deep learning libraries make the practitioner/researcher's job\nmuch easier, by automatically solving the problem of gradient calculation.\n\nBoth imperative and symbolic programs can perform gradient calculation.\nSo let's take a look at how you might perform automatic differentiation with each.\n\nLet's start with imperative programs.\nThe following example Python code performs automatic differentiation using our toy example:\n\n```python\n    class array(object) :\n        \"\"\"Simple Array object that support autodiff.\"\"\"\n        def __init__(self, value, name=None):\n            self.value = value\n            if name:\n                self.grad = lambda g : {name : g}\n\n        def __add__(self, other):\n            assert isinstance(other, int)\n            ret = array(self.value + other)\n            ret.grad = lambda g : self.grad(g)\n            return ret\n\n        def __mul__(self, other):\n            assert isinstance(other, array)\n            ret = array(self.value * other.value)\n            def grad(g):\n                x = self.grad(g * other.value)\n                x.update(other.grad(g * self.value))\n                return x\n            ret.grad = grad\n            return ret\n\n    # some examples\n    a = array(1, 'a')\n    b = array(2, 'b')\n    c = b * a\n    d = c + 1\n    print d.value\n    print d.grad(1)\n    # Results\n    # 3\n    # {'a': 2, 'b': 1}\n```\n\nIn this code, each array object contains a grad function (it is actually a closure).\nWhen you run ```d.grad```, it recursively invokes the grad function of its inputs,\nbackprops the gradient value back, and\nreturns the gradient value of each input.\n\nThis might look a bit complicated, so let's consider\nthe gradient calculation for symbolic programs.\nThe following program performs symbolic gradient calculation for the same task.\n\n```python\n    A = Variable('A')\n    B = Variable('B')\n    C = B * A\n    D = C + Constant(1)\n    # get gradient node.\n    gA, gB = D.grad(wrt=[A, B])\n    # compiles the gradient function.\n    f = compile([gA, gB])\n    grad_a, grad_b = f(A=np.ones(10), B=np.ones(10)*2)\n```\n\nThe grad function of ```D``` generates a backward computation graph,\nand returns a gradient node, ```gA, gB```,\nwhich correspond to the red nodes in the following figure.\n\n![Comp Graph Folded](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/prog_model/comp_graph_backward.png)\n\nThe imperative program actually does the same thing as the symbolic program.\nIt implicitly saves a backward computation graph in the grad closure.\nWhen you invoked ```d.grad```, you start from ```d(D)```,\nbacktrack through the graph to compute the gradient, and collect the results.\n\nThe gradient calculations in both symbolic\nand imperative programming follow the same pattern.\nWhat's the difference then?\nRecall the *be prepared to encounter all possible demands* requirement of imperative programs.\nIf you are creating an array library that supports automatic differentiation,\nyou have to keep the grad closure along with the computation.\nThis means that none of the history variables can be\ngarbage-collected because they are referenced by variable `d` by way of function closure.\n\nWhat if you want to compute only the value of `d`,\nand don't want the gradient value?\nIn symbolic programming, you declare this with `f=compiled([D])`.\nThis also declares the boundary of computation,\ntelling the system that you want to compute only the forward pass.\nAs a result, the system can free the memory of previous results,\nand share the memory between inputs and outputs.\n\nImagine running a deep neural network with ```n``` layers.\nIf you are running only the forward pass,\nnot the backward(gradient) pass,\nyou need to allocate only two copies of\ntemporal space to store the values of the intermediate layers,\ninstead of ```n``` copies of them.\nHowever, because imperative programs need to be prepared\nto encounter all possible demands of getting the gradient,\nthey have to store the intermediate values,\nwhich requires ```n``` copies of temporal space.\n\nAs you can see, the level of optimization depends\non the restrictions on what you can do.\nSymbolic programs ask you to clearly specify\nthese restrictions when you compile the graph.\nOne the other hand, imperative programs\nmust be prepared for a wider range of demands.\nSymbolic programs have a natural advantage\nbecause they know more about what you do and don't want.\n\nThere are ways in which we can modify imperative programs\nto incorporate similar restrictions.\nFor example, one solution to the preceding\nproblem is to introduce a context variable.\nYou can introduce a no-gradient context variable\nto turn gradient calculation off.\n\n```python\n    with context.NoGradient():\n        a = array(1, 'a')\n        b = array(2, 'b')\n        c = b * a\n        d = c + 1\n```\n\n<!-- This provides an imperative program with the ability to impose some restrictions, but reduces efficiency. -->\n\nHowever, this example still must be prepared to encounter all possible demands,\nwhich means that you can't perform the in-place calculation\nto reuse memory in the forward pass (a trick commonly used to reduce GPU memory usage).\nThe techniques we've discussed generate an explicit backward pass.\nSome of the libraries such as Caffe and CXXNet perform backprop implicitly on the same graph.\nThe approach we've discussed in this section also applies to them.\n\nMost configuration-file-based libraries,\nsuch as CXXNet and Caffe are designed\nto meet one or two generic requirements:\nget the activation of each layer,\nor get the gradient of all of the weights.\nThese libraries have the same problem:\nthe more generic operations the library has to support,\nthe less optimization (memory sharing) you can do,\nbased on the same data structure.\n\nAs you can see, the trade-off between restriction\nand flexibility is the same for most cases.\n\n### Model Checkpoint\n\nIt's important to able to save a model and load it back later.\nThere are different ways to *save* your work.\nNormally, to save a neural network,\nyou need to save two things: a net configuration\nfor the structure of the neural network and the weights of the neural network.\n\nThe ability to check the configuration is a plus for symbolic programs.\nBecause the symbolic construction phase does not perform computation,\nyou can directly serialize the computation graph, and load it back later.\nThis solves the problem of saving the configuration\nwithout introducing an additional layer.\n\n```python\n    A = Variable('A')\n    B = Variable('B')\n    C = B * A\n    D = C + Constant(1)\n    D.save('mygraph')\n    ...\n    D2 = load('mygraph')\n    f = compile([D2])\n    # more operations\n    ...\n```\n\nBecause an imperative program executes as it describes the computation,\nyou have to save the code itself as the ```configuration```,\nor build another configuration layer on top of the imperative language.\n\n### Parameter Updates\n\nMost symbolic programs are data flow (computation) graphs.\nData flow graphs describe computation.\nBut it's not obvious how to use graphs to describe parameter updates.\nThat's because parameter updates introduce mutation,\nwhich is not a data flow concept.\nMost symbolic programs introduce a special update statement\nto update persistent state in the programs.\n\nIt's usually easier to write parameter updates in an imperative style,\nespecially when you need multiple updates that relate to each other.\nFor symbolic programs, the update statement is also executed as you call it.\nSo in that sense, most symbolic deep learning libraries\nfall back on the imperative approach to perform updates,\nwhile using the symbolic approach to perform gradient calculation.\n\n### There Is No Strict Boundary\n\nIn comparing the two programming styles,\nsome of our arguments might not be strictly true,\ni.e., it's possible to make an imperative program\nmore like a traditional symbolic program or vice versa.\nHowever, the two archetypes are useful abstractions,\nespecially for understanding the differences between deep learning libraries.\nWe might reasonably conclude that there is no clear boundary between programming styles.\nFor example, you can create a just-in-time (JIT) compiler in Python\nto compile imperative Python programs,\nwhich provides some of the advantages of global\ninformation held in symbolic programs.\n\n\n## Big vs. Small Operations\n\nWhen designing a deep learning library, another important programming model decision\nis precisely what operations to support.\nIn general, there are two families of operations supported by most deep learning libraries:\n\n- Big operations - typically for computing neural network layers (e.g. FullyConnected and BatchNormalize).\n- Small operations - mathematical functions like matrix multiplication and element-wise addition.\n\nLibraries like CXXNet and Caffe support layer-level operations.\nLibraries like Theano and Minerva support fine-grained operations.\n\n### Smaller Operations Can Be More Flexible\nIt's quite natural to use smaller operations to compose bigger operations.\nFor example, the sigmoid unit can simply be composed of division, addition and an exponentiation:\n\n```python\n    sigmoid(x) = 1.0 / (1.0 + exp(-x))\n```\nUsing smaller operations as building blocks, you can express nearly anything you want.\nIf you're more familiar with CXXNet- or Caffe-style layers,\nnote that these operations don't differ from a layer, except that they are smaller.\n\n```python\n    SigmoidLayer(x) = EWiseDivisionLayer(1.0, AddScalarLayer(ExpLayer(-x), 1.0))\n```\nThis expression composes three layers,\nwith each defining its forward and backward (gradient) function.\nUsing smaller operations gives you the advantage of building new layers quickly,\nbecause you only need to compose the components.\n\n### Big Operations Are More Efficient\nDirectly composing sigmoid layers requires three layers of operation, instead of one.\n\n```python\n    SigmoidLayer(x) = EWiseDivisionLayer(1.0, AddScalarLayer(ExpLayer(-x), 1.0))\n```\nThis code creates overhead for computation and memory (which could be optimized, with cost).\n\nLibraries like CXXNet and Caffe take a different approach.\nTo support coarse-grained operations,\nsuch as BatchNormalization and the SigmoidLayer directly,\nin each layer, the calculation kernel is hand crafted\nwith one or only some CUDA kernel launches.\nThis makes these implementations more efficient.\n\n### Compilation and Optimization\n\nCan small operations be optimized? Of course, they can.\nLet's look at the system optimization part of the compilation engine.\nTwo types of optimization can be performed on the computation graph:\n\n- Memory allocation optimization, to reuse the memory of the intermediate computations.\n- Operator fusion, to detect sub-graph patterns, such as the sigmoid, and fuse them into a bigger operation kernel.\n\nMemory allocation optimization isn't restricted to small operations graphs.\nYou can use it with bigger operations graph, too.\nHowever, optimization might not be essential\nfor bigger operation libraries like CXXNet and Caffe,\nbecause you can't find the compilation step in them.\nHowever, there's a (dumb) ```compilation step``` in these libraries,\nthat basically translates the layers into a fixed forward,\nbackprop execution plan, by running each operation one by one.\n\nFor computation graphs with smaller operations,\nthese optimizations are crucial to performance.\nBecause the operations are small,\nthere are many sub-graph patterns that can be matched.\nAlso, because the final, generated operations\nmight not be enumerable,\nan explicit recompilation of the kernels is required,\nas opposed to the fixed amount of precompiled kernels\nin the big operation libraries.\nThis creates compilation overhead for the symbolic libraries\nthat support small operations.\nRequiring compilation optimization also creates engineering overhead\nfor the libraries that solely support smaller operations.\n\nAs in the case of symbolic vs. imperative,\nthe bigger operation libraries \"cheat\"\nby asking you to provide restrictions (to the common layer),\nso that you actually perform the sub-graph matching.\nThis moves the compilation overhead to the real brain, which is usually not too bad.\n\n### Expression Template and Statically Typed Language\nYou always have a need to write small operations and compose them.\nLibraries like Caffe use hand-crafted kernels to build these bigger blocks.\nOtherwise, you would have to compose smaller operations using Python.\n\nThere's a third choice that works pretty well.\nThis is called the expression template.\nBasically, you use template programming to\ngenerate generic kernels from an expression tree at compile time.\nFor details, see [Expression Template Tutorial](https://github.com/dmlc/mshadow/blob/master/guide/exp-template/README.md).\nCXXNet makes extensive use of an expression template,\nwhich enables creating much shorter and more readable code that matches\nthe performance of hand-crafted kernels.\n\nThe difference between using an expression template and Python kernel generation\nis that expression evaluation is done at compile time for C++ with an existing type,\nso there is no additional runtime overhead.\nIn principle, this is also possible with other statically typed languages that support templates,\nbut we've seen this trick used only in C++.\n\nExpression template libraries create a middle ground between Python operations\nand hand-crafted big kernels by allowing C++ users to craft efficient big\noperations by composing smaller operations. It's an option worth considering.\n\n## Mix the Approaches\n\nNow that we've compared the programming models, which one should you choose?\nBefore delving into that, we should emphasize that depending on the problems you're trying to solve,\nour comparison might not necessarily have a big impact.\n\nRemember [Amdahl's law](https://en.wikipedia.org/wiki/Amdahl%27s_law):\nIf you are optimizing a non-performance-critical part of your problem,\nyou won't get much of a performance gain.\n\nAs you've seen, there usually is a trade-off between efficiency,\nflexibility, and engineering complexity.\nThe more suitable programming style depends on the problem you are trying to solve.\nFor example, imperative programs are better for parameter updates,\nand symbolic programs for gradient calculation.\n\nWe advocate *mixing* the approaches.\nSometimes the part that we want to be flexible\nisn't crucial to performance.\nIn these cases, it's okay to leave some efficiency on the table\nto support more flexible interfaces.\nIn machine learning, combining methods usually works better than using just one.\n\nIf you can combine the programming models correctly,\nyou can get better results than when using a single programming model.\nIn this section, we discuss how to do so.\n\n### Symbolic and Imperative Programs\nThere are two ways to mix symbolic and imperative programs:\n\n- Use imperative programs within symbolic programs as callbacks\n- Use symbolic programs as part of imperative programs\n\nWe've observed that it's usually helpful to write parameter updates imperatively,\nand perform gradient calculations in symbolic programs.\n\nSymbolic libraries already mix programs because Python itself is imperative.\nFor example, the following program mixes the symbolic approach with NumPy, which is imperative.\n\n```python\n    A = Variable('A')\n    B = Variable('B')\n    C = B * A\n    D = C + Constant(1)\n    # compiles the function\n    f = compile(D)\n    d = f(A=np.ones(10), B=np.ones(10)*2)\n    d = d + 1.0\n```\nThe symbolic graphs are compiled into a function that can be executed imperatively.\nThe internals are a black box to the user.\nThis is exactly like writing C++ programs and exposing them to Python, which we commonly do.\n\nBecause parameter memory resides on the GPU,\nyou might not want to use NumPy as an imperative component.\nSupporting a GPU-compatible imperative library\nthat interacts with symbolic compiled functions\nor provides a limited amount of updating syntax\nin the update statement in symbolic program execution\nmight be a better choice.\n\n### Small and Big Operations\n\nThere might be a good reason to combine small and big operations.\nConsider applications that perform tasks such as changing\na loss function or adding a few customized layers to an existing structure.\nUsually, you can use big operations to compose existing\ncomponents, and use smaller operations to build the new parts.\n\nRecall Amdahl's law. Often, the new components\nare not the cause of the computation bottleneck.\nBecause the performance-critical part is already optimized by\nthe bigger operations, it's okay to forego optimizing the additional small operations,\nor to do a limited amount of memory optimization instead\nof operation fusion and directly running them.\n\n### Choose Your Own Approach\n\nIn this document, we compared multiple approaches\nto developing programming environments for deep learning.\nWe compared both the usability and efficiency implications of each,\nfinding that many of these trade-offs (like imperative vs symbolic aren't necessarily black and white).\nYou can choose your approach, or combine the approaches\nto create more interesting and intelligent deep learning libraries.\n\n## Contribute to Apache MXNet\n\nThis document is part of our effort to provide [open-source system design notes](overview)\nfor deep learning libraries. If you're interested in contributing to Apache MXNet or its\ndocumentation, [fork us on GitHub](http://github.com/apache/mxnet).\n\n## Next Steps\n\n* [Dependency Engine for Deep Learning](note_engine)\n* [Squeeze the Memory Consumption of Deep Learning](note_memory)\n* [Efficient Data Loading Module for Deep Learning](note_data_loading)\n"
  },
  {
    "path": "docs/static_site/src/pages/api/clojure/docs/tutorials/index.md",
    "content": "---\nlayout: page_landing_tutorials\ntitle:  Clojure Tutorials\naction: Get Started\ntag: clojure\npermalink: /api/clojure/docs/tutorials\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n"
  },
  {
    "path": "docs/static_site/src/pages/api/clojure/docs/tutorials/kvstore.md",
    "content": "---\nlayout: page_api\ntitle: KVStore API\nis_tutorial: true\npermalink: /api/clojure/docs/tutorials/kvstore\ntag: clojure\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n# KVStore API\n\nTopics:\n\n* [Basic Push and Pull](#basic-push-and-pull)\n* [List Key-Value Pairs](#list-key-value-pairs)\n* [Clojure API Reference]({{'/api/clojure/docs/api'|relative_url}})\n\nTo follow along with this documentation, you can use this namespace to with the needed requires:\n\n```clojure\n(ns docs.kvstore\n  (:require [org.apache.clojure-mxnet.kvstore :as kvstore]\n            [org.apache.clojure-mxnet.ndarray :as ndarray]\n            [org.apache.clojure-mxnet.context :as context]))\n```\n\n## Basic Push and Pull\n\nProvides basic operation over multiple devices (GPUs) on a single device.\n\n### Initialization\n\nLet's consider a simple example. It initializes\na (`int`, `NDArray`) pair into the store, and then pulls the value out.\n\n```clojure\n(def kv (kvstore/create \"local\")) ;; create a local kvstore\n(def shape [2 3])\n;;; init the kvstore with a vector of keys (strings) and ndarrays\n(kvstore/init kv [\"3\"] [(ndarray/* (ndarray/ones shape) 2)])\n(def a (ndarray/zeros shape))\n(kvstore/pull kv [\"3\"] [a])\n(ndarray/->vec a) ;=> [2.0 2.0 2.0 2.0 2.0 2.0]\n```\n\n### Push, Aggregation, and Updater\n\nFor any key that's been initialized, you can push a new value with the same shape to the key, as follows:\n\n```clojure\n(kvstore/push kv [\"3\"] [(ndarray/* (ndarray/ones shape) 8)])\n(kvstore/pull kv [\"3\"] [a])\n(ndarray/->vec a);=>[8.0 8.0 8.0 8.0 8.0 8.0]\n```\n\nThe data that you want to push can be stored on any device. Furthermore, you can push multiple\nvalues into the same key, where KVStore first sums all of these\nvalues, and then pushes the aggregated value, as follows (Here we use multiple cpus):\n\n```clojure\n(def cpus [(context/cpu 0) (context/cpu 1) (context/cpu 2)])\n(def b [(ndarray/ones shape {:ctx (nth cpus 0)})\n        (ndarray/ones shape {:ctx (nth cpus 1)})\n        (ndarray/ones shape {:ctx (nth cpus 2)})])\n(kvstore/push kv [\"3\" \"3\" \"3\"] b)\n(kvstore/pull kv \"3\" a)\n(ndarray/->vec a) ;=> [3.0 3.0 3.0 3.0 3.0 3.0]\n```\n\n\n### Pull\n\nYou've already seen how to pull a single key-value pair. Similar to the way that you use the push command, you can\npull the value into several devices with a single call.\n\n```clojure\n(def b [(ndarray/ones shape {:ctx (context/cpu 0)})\n        (ndarray/ones shape {:ctx (context/cpu 1)})])\n(kvstore/pull kv [\"3\" \"3\"] b)\n(map ndarray/->vec b) ;=> ([3.0 3.0 3.0 3.0 3.0 3.0] [3.0 3.0 3.0 3.0 3.0 3.0])\n```\n\n## List Key-Value Pairs\n\nAll of the operations that we've discussed so far are performed on a single key. KVStore also provides\nthe interface for generating a list of key-value pairs. For a single device, use the following:\n\n```clojure\n(def ks [\"5\" \"7\" \"9\"])\n(kvstore/init kv ks [(ndarray/ones shape) (ndarray/ones shape) (ndarray/ones shape)])\n(kvstore/push kv ks [(ndarray/ones shape) (ndarray/ones shape) (ndarray/ones shape)])\n(def b [(ndarray/zeros shape) (ndarray/zeros shape) (ndarray/zeros shape)])\n(kvstore/pull kv ks b)\n(map ndarray/->vec b);=> ([1.0 1.0 1.0 1.0 1.0 1.0] [1.0 1.0 1.0 1.0 1.0 1.0] [1.0 1.0 1.0 1.0 1.0 1.0])\n```\n"
  },
  {
    "path": "docs/static_site/src/pages/api/clojure/docs/tutorials/module.md",
    "content": "---\nlayout: page_api\ntitle: Module API\nis_tutorial: true\ntag: clojure\npermalink: /api/clojure/docs/tutorials/module\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Module API\nThe module API provides an intermediate and high-level interface for performing computation with neural networks in MXNet. Module wraps a Symbol and one or more Executors. It has both a high level and intermediate level API.\n\n\nTopics:\n\n* [Prepare the Data](#prepare-the-data)\n* [List Key-Value Pairs](#list-key-value-pairs)\n* [Preparing a Module for Computation](#preparing-a-module-for-computation)\n* [Training and Predicting](#training-and-predicting)\n* [Saving and Loading](#saving-and-loading)\n* [Clojure API Reference]({{'/api/clojure/docs/api'|relative_url}})\n\n\nTo follow along with this documentation, you can use this namespace to with the needed requires:\n\n```clojure\n(ns docs.module\n  (:require [clojure.java.io :as io]\n            [clojure.java.shell :refer [sh]]\n            [org.apache.clojure-mxnet.eval-metric :as eval-metric]\n            [org.apache.clojure-mxnet.io :as mx-io]\n            [org.apache.clojure-mxnet.module :as m]\n            [org.apache.clojure-mxnet.symbol :as sym]\n            [org.apache.clojure-mxnet.ndarray :as ndarray]))\n```\n\n## Prepare the Data\n\nIn this example, we are going to use the MNIST data set. If you have cloned the MXNet repo and `cd contrib/clojure-package`, we can run some helper scripts to download the data for us.\n\n```clojure\n(def data-dir \"data/\")\n\n(when-not (.exists (io/file (str data-dir \"train-images-idx3-ubyte\")))\n  (sh \"../../scripts/get_mnist_data.sh\"))\n```\n\nMXNet provides function in the `io` namespace to load the MNIST datasets into training and test data iterators that we can use with our module.\n\n```clojure\n(def train-data (mx-io/mnist-iter {:image (str data-dir \"train-images-idx3-ubyte\")\n                                   :label (str data-dir \"train-labels-idx1-ubyte\")\n                                   :label-name \"softmax_label\"\n                                   :input-shape [784]\n                                   :batch-size 10\n                                   :shuffle true\n                                   :flat true\n                                   :silent false\n                                   :seed 10}))\n\n(def test-data (mx-io/mnist-iter {:image (str data-dir \"t10k-images-idx3-ubyte\")\n                                  :label (str data-dir \"t10k-labels-idx1-ubyte\")\n                                  :input-shape [784]\n                                  :batch-size 10\n                                  :flat true\n                                  :silent false}))\n```\n\n\n## Preparing a Module for Computation\n\nTo construct a module, we need to have a symbol as input. This symbol takes input data in the first layer and then has subsequent layers of fully connected and relu activation layers, ending up in a softmax layer for output.\n\n```clojure\n(let [data (sym/variable \"data\")\n      fc1 (sym/fully-connected \"fc1\" {:data data :num-hidden 128})\n      act1 (sym/activation \"relu1\" {:data fc1 :act-type \"relu\"})\n      fc2 (sym/fully-connected \"fc2\" {:data act1 :num-hidden 64})\n      act2 (sym/activation \"relu2\" {:data fc2 :act-type \"relu\"})\n      fc3 (sym/fully-connected \"fc3\" {:data act2 :num-hidden 10})\n      out (sym/softmax-output \"softmax\" {:data fc3})]\n  out)\n  ;=>#object[org.apache.mxnet.Symbol 0x1f43a406 \"org.apache.mxnet.Symbol@1f43a406\"]\n```\n\nYou can also write this with the `as->` threading macro.\n\n```clojure\n(def out (as-> (sym/variable \"data\") data\n           (sym/fully-connected \"fc1\" {:data data :num-hidden 128})\n           (sym/activation \"relu1\" {:data data :act-type \"relu\"})\n           (sym/fully-connected \"fc2\" {:data data :num-hidden 64})\n           (sym/activation \"relu2\" {:data data :act-type \"relu\"})\n           (sym/fully-connected \"fc3\" {:data data :num-hidden 10})\n           (sym/softmax-output \"softmax\" {:data data})))\n;=> #'tutorial.module/out\n```\n\n\nBy default, `context` is the CPU. If you need data parallelization, you can specify a GPU context or an array of GPU contexts like this `(m/module out {:contexts [(context/gpu)]})`\n\nBefore you can compute with a module, you need to call `bind` to allocate the device memory and `init-params` or `set-params` to initialize the parameters. If you simply want to fit a module, you don’t need to call `bind` and `init-params` explicitly, because the `fit` function automatically calls them if they are needed.\n\n```clojure\n(let [mod (m/module out)]\n  (-> mod\n      (m/bind {:data-shapes (mx-io/provide-data train-data)\n               :label-shapes (mx-io/provide-label train-data)})\n      (m/init-params)))\n```\n\nNow you can compute with the module using functions like `forward`, `backward`, etc.\n\n## Training and Predicting\n\nModules provide high-level APIs for training, predicting, and evaluating. To fit a module, call the `fit` function with some data iterators:\n\n```clojure\n(def mod (m/fit (m/module out) {:train-data train-data :eval-data test-data :num-epoch 1}))\n;; Epoch  0  Train- [accuracy 0.12521666]\n;; Epoch  0  Time cost- 8392\n;; Epoch  0  Validation-  [accuracy 0.2227]\n```\n\nYou can pass in batch-end callbacks using batch-end-callback and epoch-end callbacks using epoch-end-callback in the `fit-params`. You can also set parameters using functions like in the fit-params like optimizer and eval-metric. To learn more about the fit-params, see the fit-param function options. To predict with a module, call `predict` with a DataIter:\n\n```clojure\n(def results (m/predict mod {:eval-data test-data}))\n(first results) ;=>#object[org.apache.mxnet.NDArray 0x3540b6d3 \"org.apache.mxnet.NDArray@a48686ec\"]\n\n(first (ndarray/->vec (first results))) ;=>0.08261358\n```\n\nThe module collects and returns all of the prediction results. For more details about the format of the return values, see the documentation for the [`predict`]({{'/api/clojure/docs/api/org.apache.clojure-mxnet.module.html#var-predict'|relative_url}}) function.\n\nWhen prediction results might be too large to fit in memory, use the [`predict-every-batch`]({{'/api/clojure/docs/api/org.apache.clojure-mxnet.module.html#var-predict-every-batch'|relative_url}}) API.\n\n\n```clojure\n(let [preds (m/predict-every-batch mod {:eval-data test-data})]\n  (mx-io/reduce-batches test-data\n                        (fn [i batch]\n                          (println (str \"pred is \" (first (get preds i))))\n                          (println (str \"label is \" (mx-io/batch-label batch)))\n                          ;;; do something\n                          (inc i))))\n```\n\nIf you need to evaluate on a test set and don’t need the prediction output, call the `score` function with a data iterator and an eval metric:\n\n```clojure\n(m/score mod {:eval-data test-data :eval-metric (eval-metric/accuracy)}) ;=>[\"accuracy\" 0.2227]\n```\n\nThis runs predictions on each batch in the provided data iterator and computes the evaluation score using the provided eval metric. The evaluation results are stored in `eval-metric` object itself so that you can query later.\n\n\n## Saving and Loading\n\nTo save the module parameters in each training epoch, use the `save-checkpoint` function:\n\n```clojure\n(let [save-prefix \"my-model\"]\n  (doseq [epoch-num (range 3)]\n    (mx-io/do-batches train-data (fn [batch\n                                          ;; do something\n]))\n    (m/save-checkpoint mod {:prefix save-prefix :epoch epoch-num :save-opt-states true})))\n\n;; INFO  org.apache.mxnet.module.Module: Saved checkpoint to my-model-0000.params\n;; INFO  org.apache.mxnet.module.Module: Saved optimizer state to my-model-0000.states\n;; INFO  org.apache.mxnet.module.Module: Saved checkpoint to my-model-0001.params\n;; INFO  org.apache.mxnet.module.Module: Saved optimizer state to my-model-0001.states\n;; INFO  org.apache.mxnet.module.Module: Saved checkpoint to my-model-0002.params\n;; INFO  org.apache.mxnet.module.Module: Saved optimizer state to my-model-0002.states\n\n```\n\nTo load the saved module parameters, call the `load-checkpoint` function:\n\n```clojure\n(def new-mod (m/load-checkpoint {:prefix \"my-model\" :epoch 1 :load-optimizer-states true}))\n\nnew-mod ;=> #object[org.apache.mxnet.module.Module 0x5304d0f4 \"org.apache.mxnet.module.Module@5304d0f4\"]\n```\n\nTo initialize parameters, Bind the symbols to construct executors first with `bind` function. Then, initialize the parameters and auxiliary states by calling `init-params` function.\n\n```clojure\n(-> new-mod\n    (m/bind {:data-shapes (mx-io/provide-data train-data) :label-shapes (mx-io/provide-label train-data)})\n    (m/init-params))\n```\n\nTo get current parameters, use `params`\n\n```clojure\n\n(let [[arg-params aux-params] (m/params new-mod)]\n  {:arg-params arg-params\n   :aux-params aux-params})\n\n;; {:arg-params\n;;  {\"fc3_bias\"\n;;   #object[org.apache.mxnet.NDArray 0x39adc3b0 \"org.apache.mxnet.NDArray@49caf426\"],\n;;   \"fc2_weight\"\n;;   #object[org.apache.mxnet.NDArray 0x25baf623 \"org.apache.mxnet.NDArray@a6c8f9ac\"],\n;;   \"fc1_bias\"\n;;   #object[org.apache.mxnet.NDArray 0x6e089973 \"org.apache.mxnet.NDArray@9f91d6eb\"],\n;;   \"fc3_weight\"\n;;   #object[org.apache.mxnet.NDArray 0x756fd109 \"org.apache.mxnet.NDArray@2dd0fe3c\"],\n;;   \"fc2_bias\"\n;;   #object[org.apache.mxnet.NDArray 0x1dc69c8b \"org.apache.mxnet.NDArray@d128f73d\"],\n;;   \"fc1_weight\"\n;;   #object[org.apache.mxnet.NDArray 0x20abc769 \"org.apache.mxnet.NDArray@b8e1c5e8\"]},\n;;  :aux-params {}}\n\n```\n\nTo assign parameter and aux state values, use the `set-params` function.\n\n```clojure\n(m/set-params new-mod {:arg-params (m/arg-params new-mod) :aux-params (m/aux-params new-mod)})\n;=> #object[org.apache.mxnet.module.Module 0x5304d0f4 \"org.apache.mxnet.module.Module@5304d0f4\"]\n```\n\nTo resume training from a saved checkpoint, pass the loaded parameters to the `fit` function. This will prevent `fit` from initialzing randomly.\n\nCreate fit-params and then use it to set `begin-epoch` so that `fit` knows to resume from a saved epoch.\n\n```clojure\n;; reset the training data before calling fit or you will get an error\n(mx-io/reset train-data)\n(mx-io/reset test-data)\n\n(m/fit new-mod {:train-data train-data :eval-data test-data :num-epoch 2\n                :fit-params (-> (m/fit-params {:begin-epoch 1}))})\n\n```\n\n\n## Next Steps\n* See [Symbolic API](symbol) for operations on NDArrays that assemble neural networks from layers.\n* See [NDArray API](ndarray) for vector/matrix/tensor operations.\n* See [KVStore API](kvstore) for multi-GPU and multi-host distributed training.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/clojure/docs/tutorials/ndarray.md",
    "content": "---\nlayout: page_api\ntitle: NDArray\nis_tutorial: true\ntag: clojure\npermalink: /api/clojure/docs/tutorials/ndarray\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# NDArray API\n\n\nThe NDArray API contains tensor operations similar to `numpy.ndarray`. The syntax is also similar, except for some additional calls for dealing with I/O and multiple devices.\n\nTopics:\n\n* [Create NDArray](#create-ndarray)\n* [NDArray Operations](#ndarray-operations)\n* [NDArray API Reference]({{'/api/clojure/docs/api/org.apache.clojure-mxnet.ndarray-api.html'|relative_url}})\n\n\nTo follow along with this documentation, you can use this namespace with the needed requires:\n\n```clojure\n(ns docs.ndarray\n  (:require [org.apache.clojure-mxnet.ndarray :as ndarray]\n            [org.apache.clojure-mxnet.context :as context]))\n```\n\n\n## Create NDArray\n\nCreate `mxnet.ndarray` as follows:\n\n```clojure\n\n(def a (ndarray/zeros [100 50])) ;;all zero arrray of dimension 100 x 50\n(def b (ndarray/ones [256 32 128 1])) ;; all one array of dimension\n(def c (ndarray/array [1 2 3 4 5 6] [2 3])) ;; array with contents of a shape 2 x 3\n```\n\nThere are also ways to convert a NDArray to a vec or get the shape or the NDArray as an object or vec as follows:\n\n```clojure\n(ndarray/->vec c) ;=> [1.0 2.0 3.0 4.0 5.0 6.0]\n(ndarray/shape c) ;=> #object[org.apache.mxnet.Shape 0x583c865 \"(2,3)\"]\n(ndarray/shape-vec c) ;=> [2 3]\n```\n\n\n## NDArray Operations\n\nThere are some basic NDArray operations, like arithmetic and slice operations.\n\n### Arithmetic Operations\n\n```clojure\n(def a (ndarray/ones [1 5]))\n(def b (ndarray/ones [1 5]))\n(-> (ndarray/+ a b) (ndarray/->vec)) ;=>  [2.0 2.0 2.0 2.0 2.0]\n\n;; original ndarrays are unchanged\n(ndarray/->vec a) ;=> [1.0 1.0 1.0 1.0 1.0]\n(ndarray/->vec b) ;=> [1.0 1.0 1.0 1.0 1.0]\n\n;;inplace operators\n(ndarray/+= a b)\n(ndarray/->vec a) ;=>  [2.0 2.0 2.0 2.0 2.0]\n```\n\nOther arithmetic operations are similar.\n\n\n### Slice Operations\n\n```clojure\n(def a (ndarray/array [1 2 3 4 5 6] [3 2]))\n(def a1 (ndarray/slice a 1))\n(ndarray/shape-vec a1) ;=> [1 2]\n(ndarray/->vec a1) ;=> [3.0 4.0]\n\n(def a2 (ndarray/slice a 1 3))\n(ndarray/shape-vec a2) ;=>[2 2]\n(ndarray/->vec a2) ;=> [3.0 4.0 5.0 6.0]\n```\n\n### Dot Product\n\n```clojure\n(def arr1 (ndarray/array [1 2] [1 2]))\n(def arr2 (ndarray/array [3 4] [2 1]))\n(def res (ndarray/dot arr1 arr2))\n(ndarray/shape-vec res) ;=> [1 1]\n(ndarray/->vec res) ;=> [11.0]\n```\n\n### Save and Load NDArray\n\nYou can use MXNet functions to save and load a list or dictionary of NDArrays from file systems, as follows:\n\n```clojure\n(ndarray/save \"filename\" {\"arr1\" arr1 \"arr2\" arr2})\n;; you can also do \"s3://path\" or \"hdfs\"\n```\n\nTo load:\n\n```clojure\n(def from-file (ndarray/load \"filename\"))\nfrom-file\n;=>{\"arr1\" #object[\"org.apache.mxnet.NDArray@43d85753\"], \"arr2\" #object[\"org.apache.mxnet.NDArray@5c93def4\"]}\n```\n\nThe good thing about using the `save` and `load` interface is that you can use the format across all `mxnet` language bindings. They also already support Amazon S3 and HDFS.\n\n### Multi-Device Support\n\nDevice information is stored in the `mxnet.Context` structure. When creating NDArray in MXNet, you can use the context argument (the default is the CPU context) to create arrays on specific devices as follows:\n\n```clojure\n(def cpu-a (ndarray/zeros [100 200]))\n(ndarray/context cpu-a) ;=> #object[org.apache.mxnet.Context 0x3f376123 \"cpu(0)\"]\n\n(def gpu-b (ndarray/zeros [100 200] {:ctx (context/gpu 0)})) ;; to use with gpu\n\n```\n\n## Next Steps\n* See [KVStore API](kvstore) for multi-GPU and multi-host distributed training.\n\n"
  },
  {
    "path": "docs/static_site/src/pages/api/clojure/docs/tutorials/symbol.md",
    "content": "---\nlayout: page_api\ntitle: Symbolic API\nis_tutorial: true\ntag: clojure\npermalink: /api/clojure/docs/tutorials/symbol\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# MXNet Clojure Symbolic API\n\nTopics:\n\n* [How to Compose Symbols](#how-to-compose-symbols)\n* [More Complicated Compositions](#more-complicated-compositions)\n* [Group Multiple Symbols](#group-multiple-symbols)\n* [Serialization](#serialization)\n* [Executing Symbols](#executing-symbols)\n* [Symbol API Reference]({{'/api/clojure/docs/api/org.apache.clojure-mxnet.symbol'|relative_url}})\n\n\nWe also highly encourage you to read [Symbolic Configuration and Execution in Pictures](symbol_in_pictures).\n\n\nTo follow along with this documentation, you can use this namespace to with the following requirements:\n\n```clojure\n(ns docs.symbol\n  (:require [org.apache.clojure-mxnet.executor :as executor]\n            [org.apache.clojure-mxnet.ndarray :as ndarray]\n            [org.apache.clojure-mxnet.symbol :as sym]\n            [org.apache.clojure-mxnet.context :as context]))\n```\n\n\n## How to Compose Symbols\n\nThe Symbolic API provides a way to configure computation graphs.\nYou can configure the graphs either at the level of neural network layer operations or as fine-grained operations.\n\nThe following example configures a two-layer neural network.\n\n```clojure\n(def data (sym/variable \"data\"))\n(def fc1 (sym/fully-connected \"fc1\" {:data data :num-hidden 128}))\n(def act1 (sym/activation \"act1\" {:data fc1 :act-type \"relu\"}))\n(def fc2 (sym/fully-connected \"fc2\" {:data act1 :num-hidden 64}))\n(def net (sym/softmax-output \"out\" {:data fc2}))\n```\n\nThis can also be combined more dynamically with the `as->` Clojure threading form.\n\n```clojure\n(as-> (sym/variable \"data\") data\n  (sym/fully-connected \"fc1\" {:data data :num-hidden 128})\n  (sym/activation \"act1\" {:data data :act-type \"relu\"})\n  (sym/fully-connected \"fc2\" {:data data :num-hidden 64})\n  (sym/softmax-output \"out\" {:data data}))\n\nnet ;=> #object[org.apache.mxnet.Symbol 0x5c78c8c2 \"org.apache.mxnet.Symbol@5c78c8c2\"]\n```\n\nThe basic arithmetic operators (plus, minus, div, multiplication) work as expected.\n\nThe following example creates a computation graph that adds two inputs together.\n\n```clojure\n(def a (sym/variable \"a\"))\n(def b (sym/variable \"b\"))\n(def c (sym/+ a b))\n```\n\n## More Complicated Compositions\n\nMXNet provides well-optimized symbols for layers commonly used in deep learning (see src/operator). We can also define new operators in Python. The following example first performs an element-wise add between two symbols, then feeds them to the fully connected operator:\n\n```clojure\n(def lhs (sym/variable \"data1\"))\n(def rhs (sym/variable \"data2\"))\n(def net (sym/fully-connected \"fc1\" {:data (sym/+ lhs rhs) :num-hidden 128}))\n(sym/list-arguments net) ;=> [\"data1\" \"data2\" \"fc1_weight\" \"fc1_bias\"]\n```\n\n## Group Multiple Symbols\n\nTo construct neural networks with multiple loss layers, we can use `group` to group multiple symbols together. The following example groups two outputs:\n\n```clojure\n(def net (sym/variable \"data\"))\n(def fc1 (sym/fully-connected {:data net :num-hidden 128}))\n(def net2 (sym/activation {:data fc1 :act-type \"relu\"}))\n(def out1 (sym/softmax-output {:data net2}))\n(def out2 (sym/linear-regression-output {:data net2}))\n(def group (sym/group [out1 out2]))\n(sym/list-outputs group)\n;=> [\"softmaxoutput0_output\" \"linearregressionoutput0_output\"]\n```\n\n## Serialization\nYou can use the [`save`]({{'/api/clojure/docs/api/org.apache.clojure-mxnet.symbol.html#var-save'|relative_url}}) and [`load`]({{'/api/clojure/docs/api/org.apache.clojure-mxnet.symbol.html#var-load'|relative_url}}) functions to serialize the Symbol objects. The advantage of using save and load functions is that it is language agnostic and cloud friendly. The symbol is saved in JSON format. You can also get a JSON string directly using mxnet.Symbol.toJson. Refer to API documentation for more details.\n\n The following example shows how to save a symbol to a file, load it back, and compare two symbols using a JSON string. You can also save to S3 as well\n\n```clojure\n(def a (sym/variable \"a\"))\n(def b (sym/variable \"b\"))\n(def c (sym/+ a b))\n(sym/save c \"symbol-c.json\")\n(def c2 (sym/load \"symbol-c.json\"))\n(= (sym/to-json c) (sym/to-json c2)) ;=>true\n```\n\n\n## Executing Symbols\n\nTo execute symbols, first we need to define the data that they should run on. We can do it by using the bind method, which accepts device context and a dict mapping free variable names to NDArrays as arguments and returns an executor. The executor provides forward method for evaluation and an attribute outputs to get all the results.\n\n```clojure\n(def a (sym/variable \"a\"))\n(def b (sym/variable \"b\"))\n(def c (sym/+ a b))\n\n(def ex (sym/bind c {\"a\" (ndarray/ones [2 2]) \"b\" (ndarray/ones [2 2])}))\n(-> (executor/forward ex)\n    (executor/outputs)\n    (first)\n    (ndarray/->vec));=>  [2.0 2.0 2.0 2.0]\n```\n\nWe can evaluate the same symbol on GPU with different data.\n_To do this you must have the correct native library jar defined as a dependency_\n\n**Note In order to execute the following section on a cpu set gpu_device to (cpu)**\n\n\n```clojure\n(def ex (sym/bind c (context/gpu 0) {\"a\" (ndarray/ones [2 2]) \"b\" (ndarray/ones [2 2])}))\n```\n\n## Next Steps\n* See [NDArray API](ndarray) for vector/matrix/tensor operations.\n* See [KVStore API](kvstore) for multi-GPU and multi-host distributed training."
  },
  {
    "path": "docs/static_site/src/pages/api/clojure/docs/tutorials/symbol_in_pictures.md",
    "content": "---\nlayout: page_api\ntitle: Symbolic API with Pictures\nis_tutorial: true\ntag: clojure\npermalink: /api/clojure/docs/tutorials/symbol_in_pictures\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Symbolic Configuration and Execution in Pictures\n\nThis topic explains symbolic construction and execution in pictures.\n\nWe recommend that you read the [Symbolic API](symbol) as another useful reference.\n\n## Compose Symbols\n\nSymbols are a description of the computation that you want to perform. The symbolic construction API generates the computation\ngraph that describes the computation. The following picture shows how you compose symbols to describe basic computations.\n\n![Symbol Compose](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/compose_basic.png)\n\n- The ```mxnet.Symbol.Variable``` function creates argument nodes that represent input to the computation.\n- The symbol is overloaded with basic element-wise mathematical operations.\n\n## Configure Neural Networks\n\nIn addition to supporting fine-grained operations, MXNet provides a way to perform big operations that are analogous to layers in neural networks.\nYou can use operators to describe the configuration of a neural network.\n\n![Net Compose](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/compose_net.png)\n\n\n## Example of a Multi-Input Network\n\nThe following example shows how to configure multiple input neural networks.\n\n![Multi Input](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/compose_multi_in.png)\n\n\n## Bind and Execute Symbol\n\nWhen you need to execute a symbol graph, you call the bind function to bind ```NDArrays``` to the argument nodes\nin order to obtain an ```Executor```.\n\n![Bind](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/bind_basic.png)\n\nTo get the output results, given the bound NDArrays as input, you can call ```Executor.Forward```.\n\n![Forward](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/executor_forward.png)\n\n\n## Bind Multiple Outputs\n\nTo group symbols, then bind them to get outputs of both, use ```mx.symbol.Group```.\n\n![MultiOut](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/executor_multi_out.png)\n\nRemember: Bind only what you need, so that the system can perform more optimizations.\n\n\n## Calculate the Gradient\n\nIn the bind function, you can specify NDArrays that will hold gradients. Calling ```Executor.backward``` after ```Executor.forward``` gives you the corresponding gradients.\n\n![Gradient](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/executor_backward.png)\n\n\n## Simple Bind Interface for Neural Networks\n\nIt can be tedious to pass the argument NDArrays to the bind function, especially when you are binding a big\ngraph. ```Symbol.simple_bind``` provides a way to simplify\nthe procedure. You need to specify only input data shapes. The function allocates the arguments, and binds\nthe Executor for you.\n\n![SimpleBind](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/executor_simple_bind.png)\n\n## Auxiliary States\n\nAuxiliary states are just like arguments, except that you can't take the gradient of them. Although auxiliary states might not be part of the computation, they can be helpful for tracking. You can pass auxiliary states in the same way that you pass arguments.\n\n![SimpleBind](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/executor_aux_state.png)\n\n## Next Steps\n\nSee [Symbolic API](symbol) and [Python Documentation]({{'/api/python'|relative_url}}).\n"
  },
  {
    "path": "docs/static_site/src/pages/api/clojure/index.md",
    "content": "---\nlayout: page_api\ntitle: Clojure Guide\naction: Get Started\naction_url: /get_started\npermalink: /api/clojure\ntag: clojure\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# MXNet - Clojure API\n\nMXNet supports the Clojure programming language. The MXNet Clojure package brings flexible and efficient GPU\ncomputing and state-of-art deep learning to Clojure. It enables you to write seamless tensor/matrix computation with multiple GPUs in Clojure. It also lets you construct and customize the state-of-art deep learning models in Clojure, and apply them to tasks, such as image classification and data science challenges.\n\n\n## Tensor and Matrix Computations\nYou can perform tensor or matrix computation in pure Clojure:\n\n```clojure\n(def arr (ndarray/ones [2 3]))\n\narr ;=> #object[org.apache.mxnet.NDArray 0x597d72e \"org.apache.mxnet.NDArray@e35c3ba9\"]\n\n(ndarray/shape-vec arr) ;=>  [2 3]\n\n(-> (ndarray/* arr 2)\n    (ndarray/->vec)) ;=> [2.0 2.0 2.0 2.0 2.0 2.0]\n\n(ndarray/shape-vec (ndarray/* arr 2)) ;=> [2 3]\n\n```\n"
  },
  {
    "path": "docs/static_site/src/pages/api/cpp/docs/tutorials/basics.md",
    "content": "---\nlayout: page_api\ntitle: Basics\naction: Get Started\naction_url: /get_started\npermalink: /api/cpp/docs/tutorials/basics.html\nis_tutorial: true\ntag: cpp\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nBasics\n======\n\nThis tutorial provides basic usages of the C++ package through the classical handwritten digits\nidentification database--[MNIST](http://yann.lecun.com/exdb/mnist/).\n\nThe following contents assume that the working directory is `/path/to/mxnet/cpp-package/example`.\n\nLoad Data\n--------\nBefore going into codes, we need to fetch MNIST data. You can either use the script `/path/to/mxnet/cpp-package/example/get_data.sh`,\nor download mnist data by yourself from Lecun's [website](http://yann.lecun.com/exdb/mnist/)\nand decompress them into `data/mnist_data` folder.\n\nExcept linking the MXNet shared library, the C++ package itself is a header-only package,\nwhich means all you need to do is to include the header files. Among the header files,\n`op.h` is special since it is generated dynamically. The generation should be done when\n[building the C++ package]({{'/api/cpp/'|relative_url}}).\nIt is important to note that you need to **copy the shared library** (`libmxnet.so` in Linux and MacOS,\n`libmxnet.dll` in Windows) from `/path/to/mxnet/lib` to the working directory.\nWe do not recommend you to use pre-built binaries because MXNet is under heavy development,\nthe operator definitions in `op.h` may be incompatible with the pre-built version.\n\nIn order to use functionalities provides by the C++ package, first we include the general\nheader file `MxNetCpp.h` and specify the namespaces.\n\n```c++\n#include \"mxnet-cpp/MxNetCpp.h\"\n\nusing namespace std;\nusing namespace mxnet::cpp;\n```\n\nNext we can use the data iter to load MNIST data (separated to training sets and validation sets).\nThe digits in MNIST are 2-dimension arrays, so we should set `flat` to true to flatten the data.\n\n```c++\nauto train_iter = MXDataIter(\"MNISTIter\")\n    .SetParam(\"image\", \"./data/mnist_data/train-images-idx3-ubyte\")\n    .SetParam(\"label\", \"./data/mnist_data/train-labels-idx1-ubyte\")\n    .SetParam(\"batch_size\", batch_size)\n    .SetParam(\"flat\", 1)\n    .CreateDataIter();\nauto val_iter = MXDataIter(\"MNISTIter\")\n    .SetParam(\"image\", \"./data/mnist_data/t10k-images-idx3-ubyte\")\n    .SetParam(\"label\", \"./data/mnist_data/t10k-labels-idx1-ubyte\")\n    .SetParam(\"batch_size\", batch_size)\n    .SetParam(\"flat\", 1)\n    .CreateDataIter();\n```\n\nThe data have been successfully loaded. We can now easily construct various models to identify\nthe digits with the help of C++ package.\n\nGPU Support\n-----------\nIt's worth noting that changing context from `Context::cpu()` to `Context::gpu()` is not enough,\nbecause the data read by data iter are stored in memory, we cannot assign it directly to the\nparameters. To bridge this gap, NDArray provides data synchronization functionalities between\nGPU and CPU. We will illustrate it by making the mlp code run on GPU.\n\nIn the previous code, data are used like\n\n```c++\nargs[\"X\"] = data_batch.data;\nargs[\"label\"] = data_batch.label;\n```\n\nIt will be problematic if other parameters are created in the context of GPU. We can use\n`NDArray::CopyTo` to solve this problem.\n\n```c++\n// Data provided by DataIter are stored in memory, should be copied to GPU first.\ndata_batch.data.CopyTo(&args[\"X\"]);\ndata_batch.label.CopyTo(&args[\"label\"]);\n// CopyTo is imperative, need to wait for it to complete.\nNDArray::WaitAll();\n```\n\nBy replacing the former code to the latter one, we successfully port the code to GPU.\nYou can find the complete code in `mlp_gpu.cpp`. Compilation is similar to the cpu version.\nNote that the shared library must be built with GPU support enabled.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/cpp/docs/tutorials/index.md",
    "content": "---\nlayout: page_landing_tutorials\ntitle: C++ Tutorials\npermalink: /api/cpp/docs/tutorials/\ntag: cpp\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n"
  },
  {
    "path": "docs/static_site/src/pages/api/cpp/docs/tutorials/multi_threaded_inference.md",
    "content": "---\nlayout: page_api\ntitle: Multi Threaded Inference\naction: Get Started\naction_url: /get_started\npermalink: /api/cpp/docs/tutorials/multi_threaded_inference.html\nis_tutorial: true\ntag: cpp\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Multi Threaded Inference API\n\nA long standing request from MXNet users has been to invoke parallel inference on a model from multiple threads while sharing the parameters.\nWith this use case in mind, the threadsafe version of CachedOp was added to provide a way for customers to do multi-threaded inference for MXNet users.\nThis doc attempts to do the following:\n1. Discuss the current state of thread safety in MXNet\n2. Explain how one can use C API and thread safe version of cached op, along with CPP package to achieve multi threaded inference. This will be useful for end users as well as frontend developers of different language bindings\n3. Discuss the limitations of the above approach\n4. Future Work\n\n## Current state of Thread Safety in MXNet\n\nExamining the current state of thread safety in MXNet we can arrive to the following conclusion:\n\n1. MXNet Dependency Engine is thread safe (except for WaitToRead invoked inside a spawned thread. Please see Limitations section)\n2. Graph Executor which is Module/Symbolic/C Predict API backend is not thread safe\n3. Cached Op (Gluon Backend) is not thread safe\n\nThe CachedOpThreadSafe and corresponding C APIs were added to address point 3 above and provide a way\nfor MXNet users to do multi-threaded inference.\n\n```\n/*!\n * \\brief create cached operator, allows to choose thread_safe version\n * of cachedop\n */\nMXNET_DLL int MXCreateCachedOp(SymbolHandle handle,\n                               int num_flags,\n                               const char** keys,\n                               const char** vals,\n                               CachedOpHandle *out,\n                               bool thread_safe DEFAULT(false));\n```\n\n## Multithreaded inference in MXNet with C API and CPP Package\n\n### Prerequisites\nTo complete this tutorial you need to:\n- Learn the basics about [MXNet C++ API](/api/cpp)\n- Build MXNet from source with make/cmake\n- Build the multi-threaded inference example\n\n### Setup the MXNet C++ API\nTo use the C++ API in MXNet, you need to build MXNet from source with C++ package. Please follow the [built from source guide](/get_started/build_from_source.html), and [C++ Package documentation](/api/cpp.html)\nThe summary of those two documents is that you need to build MXNet from source with `USE_CPP_PACKAGE` flag set to 1.\nThis example requires a build with CUDA and CUDNN.\n\n### Get the example\nIf you have built mxnet from source with cmake, then do the following:\n\n```bash\n$ cp build/cpp-package/example/multi_threaded_inference .\n```\n\n### Run multi threaded inference example\nThe example is tested with models such as `imagenet1k-inception-bn`, `imagenet1k-resnet-50`,\n`imagenet1k-resnet-152`, `imagenet1k-resnet-18`\n\nTo run the multi threaded inference example:\n\nFirst export `LD_LIBRARY_PATH`:\n\n```bash\n$ export LD_LIBRARY_PATH=<MXNET_LIB_DIR>:$LD_LIBRARY_PATH\n```\n\n```bash\n$ ./multi_threaded_inference [model_name] [is_gpu] [file_names]\n```\ne.g.\n\n```bash\n./multi_threaded_inference imagenet1k-inception-bn 1 grace_hopper.jpg dog.jpg\n```\n\nThe above script spawns 2 threads, shares the same cachedop and params among two threads, and runs inference on GPU. It returns the inference results in the order in which files are provided.\n\nNOTE: This example is to demonstrate the multi-threaded-inference with cached op. The inference results work well only with specific models (e.g. imagenet1k-inception-bn). The results may not necessarily be very accurate because of different preprocessing step required etc.\n\n### Code walkthrough multi-threaded inference with CachedOp\n\nThe multi threaded inference example (`multi_threaded_inference.cc`) involves the following steps:\n\n1. Parse arguments and load input image into ndarray\n2. Prepare input data and load parameters, copying data to a specific context\n3. Preparing arguments to pass to the CachedOp and calling C API to **create cached op**\n4. Prepare lambda function which will run in spawned threads. Call C API to **invoke cached op** within the lambda function.\n5. Spawn multiple threads and wait for all threads to complete.\n6. Post process data to obtain inference results and cleanup.\n\n### Step 1: Parse arguments and load input image into ndarray\n\n[https://github.com/apache/mxnet/example/multi_threaded_inference/multi_threaded_inference.cc#L299-L341](multi_threaded_inference.cc#L299-L341)\n\nThe above code parses arguments, loads the image file into a ndarray with a specific shape. There are a few things that are set by default and not configurable. For example, `static_alloc` and `static_shape` are by default set to true.\n\n\n### Step 2: Prepare input data and load parameters, copying data to a specific context\n\n[https://github.com/apache/mxnet/example/multi_threaded_inference/multi_threaded_inference.cc#L147-L205](multi_threaded_inference.cc#L147-L205)\n\nThe above code loads params and copies input data and params to specific context.\n\n### Step 3: Preparing arguments to pass to the CachedOp and calling C API to create cached op\n\n[https://github.com/apache/mxnet/example/multi_threaded_inference/multi_threaded_inference.cc#L207-L233](multi_threaded_inference.cc#L207-233)\n\nThe above code prepares `flag_key_cstrs` and `flag_val_cstrs` to be passed the Cached op.\nThe C API call is made with `MXCreateCachedOp`. This will lead to creation of thread safe cached\nop since the `thread_safe` (which is the last parameter to `MXCreateCachedOp`) is set to\ntrue. When this is set to false, it will invoke CachedOp instead of CachedOpThreadSafe.\n\n\n### Step 4: Prepare lambda function which will run in spawned threads\n\n[https://github.com/apache/mxnet/example/multi_threaded_inference/multi_threaded_inference.cc#L248-L262](multi_threaded_inference.cc#L248-262)\n\nThe above creates the lambda function taking the thread number as the argument.\nIf `random_sleep` is set it will sleep for a random number (secs) generated between 0 to 5 seconds.\nFollowing this, it invokes `MXInvokeCachedOp`(from the hdl it determines whether to invoke cached op threadsafe version or not).\nWhen this is set to false, it will invoke CachedOp instead of CachedOpThreadSafe.\n\n### Step 5: Spawn multiple threads and wait for all threads to complete\n\n[https://github.com/anirudh2290/apache/mxnet/example/multi_threaded_inference/multi_threaded_inference.cc#L264-L276](multi_threaded_inference.cc#L264-L276)\n\nSpawns multiple threads, joins and waits to wait for all ops to complete.\nThe other alternative is to wait in the thread on the output ndarray and remove the WaitAll after join.\n\n### Step 6: Post process data to obtain inference results and cleanup\n\n[https://github.com/apache/mxnet/example/multi_threaded_inference/multi_threaded_inference.cc#L286-L293](multi_threaded_inference.cc#L286-293)\n\nThe above code outputs results for different threads and cleans up the thread safe cached op.\n\n## Current Limitations\n\n1. Only operators tested with the existing model coverage are supported. Other operators and operator types (stateful operators, custom operators are not supported. Existing model coverage is as follows (this list will keep growing as we test more models with different model types):\n\n|Models Tested|oneDNN|CUDNN|NO-CUDNN|\n| --- | --- | --- | --- |\n| imagenet1k-resnet-18 | Yes | Yes | Yes |\n| imagenet1k-resnet-152 | Yes | Yes | Yes |\n| imagenet1k-resnet-50 | Yes | Yes | Yes |\n\n2. Only dense storage types are supported currently.\n3. Multi GPU Inference not supported currently.\n4. Instantiating multiple instances of SymbolBlockThreadSafe is not supported. Can run parallel inference only on one model per process.\n5. dynamic shapes not supported in thread safe cached op.\n6. Bulking of ops is not supported.\n7. This only supports inference use cases currently, training use cases are not supported.\n8. Graph rewrites with subgraph API currently not supported.\n9. There is currently no frontend API support to run multi threaded inference. Users can use CreateCachedOp and InvokeCachedOp in combination with\nthe CPP frontend to run multi-threaded inference as of today.\n10. Multi threaded inference with threaded engine with Module/Symbolic API and C Predict API are not currently supported.\n11. Exception thrown with `wait_to_read` in individual threads can cause issues. Calling invoke from each thread and calling WaitAll after thread joins should still work fine.\n12. Tested only on environments supported by CI. This means that MacOS is not supported.\n\n## Future Work\n\nFuture work includes Increasing model coverage and addressing most of the limitations mentioned under Current Limitations except the training use case.\nFor more updates, please subscribe to discussion activity on RFC: https://github.com/apache/mxnet/issues/16431.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/cpp/docs/tutorials/mxnet_cpp_inference_tutorial.md",
    "content": "---\nlayout: page_api\ntitle: C++ API inference tutorial\naction: Get Started\naction_url: /get_started\npermalink: /api/cpp/docs/tutorials/cpp_inference.html\nis_tutorial: true\ntag: cpp\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# C++ API inference tutorial\n\n## Overview\nMXNet provides various useful tools and interfaces for deploying your model for inference. For example, you can use [MXNet Model Server](https://github.com/awslabs/mxnet-model-server) to start a service and host your trained model easily.\nBesides that, you can also use MXNet's different language APIs to integrate your model with your existing service. We provide [Python](/api/python/docs/api/), [Java](/api/java/docs/api/#package), [Scala](/api/scala/docs/api), and [C++](/api/cpp/docs/api/) APIs.\nWe will focus on the MXNet C++ API. We have slightly modified the code in [C++ Inference Example](https://github.com/apache/mxnet/tree/master/cpp-package/example/inference) for our use case.\n\n## Prerequisites\n\nTo complete this tutorial, you need to:\n- Complete the training part of [Gluon end to end tutorial](/api/python/docs/tutorials/getting-started/gluon_from_experiment_to_deployment.html).\n- Learn the basics about [MXNet C++ API](/api/cpp).\n\n\n## Setup the MXNet C++ API\n\nTo use the C++ API in MXNet, you need to build MXNet from source with C++ package. Please follow the [built from source guide](/get_started/ubuntu_setup.html), and [C++ Package documentation](/api/cpp).\nThe summary of those two documents is that you need to build MXNet from source with `USE_CPP_PACKAGE` flag set to 1. For example: `make -j USE_CPP_PACKAGE=1`.\n\n## Load the model and run inference\n\nAfter you complete [the previous tutorial](/api/python/docs/tutorials/getting-started/gluon_from_experiment_to_deployment.html), you will get the following output files:\n1. Model Architecture stored in `flower-recognition-symbol.json`\n2. Model parameter values stored in `flower-recognition-0040.params` (`0040` is for 40 epochs we ran)\n3. Label names stored in `synset.txt`\n4. Mean and standard deviation values stored in `mean_std_224` for image normalization.\n\n\nNow we need to write the C++ code to load them and run prediction on a test image.\nThe full code is available in the [C++ Inference Example](https://github.com/apache/mxnet/tree/master/cpp-package/example/inference), we will walk you through it and point out the necessary changes to make for our use case.\n\n\n\n### Write a predictor using the MXNet C++ API\n\nIn general, the C++ inference code should follow the 4 steps below. We can do that using a Predictor class.\n1. Load the pre-trained model\n2. Load the parameters of pre-trained model\n3. Load the image to be classified in to NDArray and apply image transformation we did in training\n4. Run the forward pass and predict the class of the input image\n\n```c++\nclass Predictor {\n public:\n    Predictor() {}\n    Predictor(const std::string& model_json_file,\n              const std::string& model_params_file,\n              const Shape& input_shape,\n              bool gpu_context_type = false,\n              const std::string& synset_file = \"\",\n              const std::string& mean_image_file = \"\");\n    void PredictImage(const std::string& image_file);\n    ~Predictor();\n\n private:\n    void LoadModel(const std::string& model_json_file);\n    void LoadParameters(const std::string& model_parameters_file);\n    void LoadSynset(const std::string& synset_file);\n    NDArray LoadInputImage(const std::string& image_file);\n    void LoadMeanImageData();\n    void LoadDefaultMeanImageData();\n    void NormalizeInput(const std::string& mean_image_file);\n    inline bool FileExists(const std::string& name) {\n        struct stat buffer;\n        return (stat(name.c_str(), &buffer) == 0);\n    }\n    NDArray mean_img;\n    std::map<std::string, NDArray> args_map;\n    std::map<std::string, NDArray> aux_map;\n    std::vector<std::string> output_labels;\n    Symbol net;\n    Executor *executor;\n    Shape input_shape;\n    NDArray mean_image_data;\n    NDArray std_dev_image_data;\n    Context global_ctx = Context::cpu();\n    std::string mean_image_file;\n};\n```\n\n### Load the model, synset file, and normalization values\n\nIn the Predictor constructor, you need to provide paths to saved json and param files. After that, add the following methods `LoadModel` and `LoadParameters` to load the network and its parameters. This part is the same as [the example](https://github.com/apache/mxnet/blob/master/cpp-package/example/inference/imagenet_inference.cpp).\n\nNext, we need to load synset file, and normalization values. We have made the following change since our synset file contains flower names and we used both mean and standard deviation for image normalization.\n\n```c++\n/*\n * The following function loads the synset file.\n * This information will be used later to report the label of input image.\n */\nvoid Predictor::LoadSynset(const std::string& synset_file) {\n  if (!FileExists(synset_file)) {\n    LG << \"Synset file \" << synset_file << \" does not exist\";\n    throw std::runtime_error(\"Synset file does not exist\");\n  }\n  LG << \"Loading the synset file.\";\n  std::ifstream fi(synset_file.c_str());\n  if (!fi.is_open()) {\n    std::cerr << \"Error opening synset file \" << synset_file << std::endl;\n    throw std::runtime_error(\"Error in opening the synset file.\");\n  }\n  std::string lemma;\n  while (getline(fi, lemma)) {\n    output_labels.push_back(lemma);\n  }\n  fi.close();\n}\n\n/*\n * The following function loads the mean and standard deviation values.\n * This data will be used for normalizing the image before running the forward\n * pass.\n * The output data has the same shape as that of the input image data.\n */\nvoid Predictor::LoadMeanImageData() {\n  LG << \"Load the mean image data that will be used to normalize \"\n     << \"the image before running forward pass.\";\n  mean_image_data = NDArray(input_shape, global_ctx, false);\n  mean_image_data.SyncCopyFromCPU(\n        NDArray::LoadToMap(mean_image_file)[\"mean_img\"].GetData(),\n        input_shape.Size());\n  NDArray::WaitAll();\n   std_dev_image_data = NDArray(input_shape, global_ctx, false);\n   std_dev_image_data.SyncCopyFromCPU(\n       NDArray::LoadToMap(mean_image_file)[\"std_img\"].GetData(),\n       input_shape.Size());\n    NDArray::WaitAll();\n}\n```\n\n\n\n### Load input image\n\nNow let's add a method to load the input image we want to predict and converts it to NDArray for prediction.\n```c++\nNDArray Predictor::LoadInputImage(const std::string& image_file) {\n  if (!FileExists(image_file)) {\n    LG << \"Image file \" << image_file << \" does not exist\";\n    throw std::runtime_error(\"Image file does not exist\");\n  }\n  LG << \"Loading the image \" << image_file << std::endl;\n  std::vector<float> array;\n  cv::Mat mat = cv::imread(image_file);\n  /*resize pictures to (224, 224) according to the pretrained model*/\n  int height = input_shape[2];\n  int width = input_shape[3];\n  int channels = input_shape[1];\n  cv::resize(mat, mat, cv::Size(height, width));\n  for (int c = 0; c < channels; ++c) {\n    for (int i = 0; i < height; ++i) {\n      for (int j = 0; j < width; ++j) {\n        array.push_back(static_cast<float>(mat.data[(i * height + j) * 3 + c]));\n      }\n    }\n  }\n  NDArray image_data = NDArray(input_shape, global_ctx, false);\n  image_data.SyncCopyFromCPU(array.data(), input_shape.Size());\n  NDArray::WaitAll();\n  return image_data;\n}\n```\n\n### Predict the image\n\nFinally, let's run the inference. It's basically using MXNet executor to do a forward pass. To run predictions on multiple images, you can load the images in a list of NDArrays and run prediction in batches. Note that the Predictor class may not be thread safe. Calling it in multi-threaded environments was not tested. To utilize multi-threaded prediction, you need to use the C predict API. Please follow the [C predict example](https://github.com/apache/mxnet/tree/master/example/image-classification/predict-cpp).\n\nAn additional step is to normalize the image NDArrays values to `(0, 1)` and apply mean and standard deviation we just loaded.\n\n```c++\n/*\n * The following function runs the forward pass on the model.\n * The executor is created in the constructor.\n *\n */\nvoid Predictor::PredictImage(const std::string& image_file) {\n  // Load the input image\n  NDArray image_data = LoadInputImage(image_file);\n\n  // Normalize the image\n  image_data.Slice(0, 1) /= 255.0;\n  image_data -= mean_image_data;\n  image_data /= std_dev_image_data;\n\n  LG << \"Running the forward pass on model to predict the image\";\n  /*\n   * The executor->arg_arrays represent the arguments to the model.\n   *\n   * Copying the image_data that contains the NDArray of input image\n   * to the arg map of the executor. The input is stored with the key \"data\" in the map.\n   *\n   */\n  image_data.CopyTo(&(executor->arg_dict()[\"data\"]));\n  NDArray::WaitAll();\n\n  // Run the forward pass.\n  executor->Forward(false);\n\n  // The output is available in executor->outputs.\n  auto array = executor->outputs[0].Copy(global_ctx);\n  NDArray::WaitAll();\n\n  /*\n   * Find out the maximum accuracy and the index associated with that accuracy.\n   * This is done by using the argmax operator on NDArray.\n   */\n  auto predicted = array.ArgmaxChannel();\n  NDArray::WaitAll();\n\n  int best_idx = predicted.At(0, 0);\n  float best_accuracy = array.At(0, best_idx);\n\n  if (output_labels.empty()) {\n    LG << \"The model predicts the highest accuracy of \" << best_accuracy << \" at index \"\n       << best_idx;\n  } else {\n    LG << \"The model predicts the input image to be a [\" << output_labels[best_idx]\n       << \" ] with Accuracy = \" << best_accuracy << std::endl;\n  }\n}\n```\n\n### Compile and run the inference code\n\nYou can find the [full code for the inference example](https://github.com/apache/mxnet/tree/master/cpp-package/example/inference) in the `cpp-package` folder of the project\n, and to compile it use this [Makefile](https://github.com/apache/mxnet/blob/master/cpp-package/example/inference/Makefile).\n\nMake a copy of the example code, rename it to `flower_inference` and apply the changes we mentioned above. Now you will be able to compile and run inference. Run `make all`. Once this is complete, run inference with the following parameters. Remember to set your `LD_LIBRARY_PATH` to point to MXNet library if you have not done so.\n\n```bash\nmake all\nexport LD_LIBRARY_PATH=$LD_LIBRARY_PATH=:path/to/mxnet/lib\n./flower_inference --symbol flower-recognition-symbol.json --params flower-recognition-0040.params --synset synset.txt --mean mean_std_224.nd --image ./data/test/lotus/image_01832.jpg\n```\n\nThen it will predict your image:\n\n```bash\n[17:38:51] resnet.cpp:150: Loading the model from flower-recognition-symbol.json\n\n[17:38:51] resnet.cpp:163: Loading the model parameters from flower-recognition-0040.params\n\n[17:38:52] resnet.cpp:190: Loading the synset file.\n[17:38:52] resnet.cpp:211: Load the mean image data that will be used to normalize the image before running forward pass.\n[17:38:52] resnet.cpp:263: Loading the image ./data/test/lotus/image_01832.jpg\n\n[17:38:52] resnet.cpp:299: Running the forward pass on model to predict the image\n[17:38:52] resnet.cpp:331: The model predicts the input image to be a [lotus ] with Accuracy = 8.63046\n```\n\n\n\n## What's next\n\nNow you can explore more ways to run inference and deploy your models:\n1. [Java Inference examples](https://github.com/apache/mxnet/tree/master/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer)\n2. [Scala Inference examples](https://github.com/apache/mxnet/tree/master/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer)\n3. [ONNX model inference examples](/api/python/docs/tutorials/packages/onnx/inference_on_onnx_model.html)\n4. [MXNet Model Server Examples](https://github.com/awslabs/mxnet-model-server/tree/master/examples)\n\n## References\n\n1. [Gluon end to end tutorial](/api/python/docs/tutorials/getting-started/gluon_from_experiment_to_deployment.html)\n2. [Gluon C++ inference example](https://github.com/apache/mxnet/blob/master/cpp-package/example/inference/)\n3. [Gluon C++ package](https://github.com/apache/mxnet/tree/master/cpp-package)\n"
  },
  {
    "path": "docs/static_site/src/pages/api/cpp/docs/tutorials/subgraphAPI.md",
    "content": "---\nlayout: page_api\ntitle: Subgraph API\naction: Get Started\naction_url: /get_started\npermalink: /api/cpp/docs/tutorials/subgraph_api.html\nis_tutorial: true\ntag: cpp\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n## Subgraph API\n\nThe subgraph API has been proposed and implemented as the default mechanism for integrating backend libraries to MXNet. The subgraph API is a very flexible interface. Although it was proposed as an integration mechanism, it has been used as a tool for manipulating NNVM graphs for graph-level optimizations, such as operator fusion.\n\nThe subgraph API works as the following steps:\n\n* Search for particular patterns in a graph.\n* Group the operators/nodes with particular patterns into a subgraph and shrink the subgraph into a single node.\n* Replace the subgraph in the original graph with the subgraph node.\n\nThe figure below illustrates the subgraph mechanism.\n\n![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tutorials/subgraph/subgraph.png)\n\nThe subgraph API allows the backend developers to customize the subgraph mechanism in two places:\n\n* Subgraph searching: define a subgraph selector to search for particular patterns in a computation graph.\n* Subgraph node creation: attach an operator to run the computation in the subgraph. We can potentially manipulate the subgraph here.\n\n\nThe following is a demonstration of how the subgraph API can be applied to a simple task. Refer to the previous figure for an overview of the process. That is, replacing `Convolution` and `BatchNorm` with the conv_bn.\n\nThe first step is to define a subgraph selector to find the required pattern. To find a pattern that has `Convolution` and `BatchNorm`, we can start the search on the node with `Convolution`. Then from the `Convolution` node, we search for `BatchNorm` along the outgoing edge.\n\n```c++\nclass SgSelector : public SubgraphSelector {\n public:\n  SgSelector() {\n    find_bn = false;\n  }\n  bool Select(const nnvm::Node &n) override {\n    // Here we start on the Convolution node to search for a subgraph.\n    return n.op() && n.op()->name == \"Convolution\";\n  }\n  bool SelectInput(const nnvm::Node &n, const nnvm::Node &new_node) override {\n    // We don't need to search on the incoming edge.\n    return false;\n  }\n  bool SelectOutput(const nnvm::Node &n, const nnvm::Node &new_node) override {\n    // We search on the outgoing edge. Once we find a BatchNorm node, we won't\n    // accept any more BatchNorm nodes.\n    if (new_node.op() && new_node.op()->name == \"BatchNorm\" && !find_bn) {\n      find_bn = true;\n      return true;\n    } else {\n      return false;\n    }\n  }\n  std::vector<nnvm::Node *> Filter(const std::vector<nnvm::Node *> &candidates) override {\n    // We might have found a Convolution node, but we might have failed to find a BatchNorm\n    // node that uses the output of the Convolution node. If we failed, we should skip\n    // the Convolution node as well.\n    if (find_bn)\n      return candidates;\n    else\n      return std::vector<nnvm::Node *>();\n  }\n private:\n  bool find_bn;\n};\n```\n\nThe second step is to define a subgraph property to use the subgraph selector above to customize the subgraph searching. By defining this class, we can also customize subgraph node creation. When customizing node creation, we can specify what operator to run the subgraph on the node. In this example, we use `CachedOp`, which itself is a graph executor, to run the subgraph with `Convolution` and `BatchNorm`. In practice, it's most likely that we use a single operator from a backend library to replace the two operators for execution.\n\n{% raw %}\n```c++\nclass SgProperty : public SubgraphProperty {\n public:\n  static SubgraphPropertyPtr Create() {\n    return std::make_shared<SgProperty>();\n  }\n  nnvm::ObjectPtr CreateSubgraphNode(\n      const nnvm::Symbol &sym, const int subgraph_id = 0) const override {\n    // We can use CachedOp to execute the subgraph.\n    nnvm::ObjectPtr n = nnvm::Node::Create();\n    n->attrs.op = Op::Get(\"_CachedOp\");\n    n->attrs.name = \"ConvBN\" + std::to_string(subgraph_id);\n    n->attrs.subgraphs.push_back(std::make_shared<nnvm::Symbol>(sym));\n    std::vector<std::pair<std::string, std::string> > flags{{\"static_alloc\", \"true\"}};\n    n->attrs.parsed = CachedOpPtr(new CachedOp(sym, flags));\n    return n;\n  }\n  SubgraphSelectorPtr CreateSubgraphSelector() const override {\n    auto property = std::make_shared<CreateSubgraphSelector>();\n    property->SetAttr<std::string>(\"property_name\", \"subgraph example pass\"); // Optional, better to have it.\n    property->SetAttr<bool>(\"inference_only\", true); // Optional, only for inference_only pass.\n    return property;\n  }\n};\n```\n{% endraw %}\n`SetAttr` is optional and developer can define their own attributes to control property behavior.\nThere're some built-in attributes that used by MXNet executor.\n\n`property_name`  : std::string, name of this property, used for diagnose.\n\n`disable` : bool, whther to disable this property.\n\n`inference_only` : bool, apply this property only for inference. Property will be skiped when need_grad=True. Default `false` if this attribute isn't defined.\n\nAfter defining the subgraph property, we need to register it under a backend in .cc file.\n\nFirstly, we need to register the backend\n\n```C++\nMXNET_REGISTER_SUBGRAPH_BACKEND(SgTest);\n```\n\nThen register the property under it.\n\n```c++\nMXNET_REGISTER_SUBGRAPH_PROPERTY(SgTest, SgProperty);\n```\n\nIt's possible to register multiple properties for same backend. In practice, we recommend to put each property definition into .h file, and register backend in single .cc file. Property will be executed according to the register order.\n\n```c++\n#include \"SgProperty.h\" // Define SgProperty class\n#include \"SgProperty2.h\" // Define SgProperty2 class\n#include \"SgProperty3.h\" // Define SgProperty3 class\n\nMXNET_REGISTER_SUBGRAPH_BACKEND(SgTest);\nMXNET_REGISTER_SUBGRAPH_PROPERTY(SgTest, SgProperty);  // Execution order 1.\nMXNET_REGISTER_SUBGRAPH_PROPERTY(SgTest, SgProperty2); // Execution order 2.\nMXNET_REGISTER_SUBGRAPH_PROPERTY(SgTest, SgProperty3); // Execution order 3.\n```\n\nAfter compiling this subgraph mechanism into MXNet you can use python symbol API `get_backend_symbol` to run all properties registered for this backend and get returned symbol.\n\n```python\nsym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch)\nsym = sym.get_backend_symbol('SgTest')\n```\n\nWhen `SgProperty` is activated, a message will be shown in terminal as\n\n```bash\nstart to execute subgraph example pass.\n```\n\nThis tutorial shows a simple example of how to use the subgraph API to search for patterns in an NNVM graph.\nIntested users can try different pattern matching rules (i.e., define their own `SubgraphSelector`) and\nattach different operators to execute the subgraphs.\n\n<!-- INSERT SOURCE DOWNLOAD BUTTONS -->\n"
  },
  {
    "path": "docs/static_site/src/pages/api/cpp/index.md",
    "content": "---\nlayout: page_api\ntitle: C++ Guide\naction: Get Started\naction_url: /get_started\npermalink: /api/cpp\ntag: cpp\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# MXNet - C++ API\n\nThe MXNet C++ Package provides C++ API bindings to the users of MXNet.  Currently, these bindings are not available as standalone package.\nThe users of these bindings are required to build this package as mentioned below.\n\n## Building C++ Package\n\nThe cpp-package directory contains the implementation of C++ API. Users are required to build this directory or package before using it. \n**The cpp-package is built while building the MXNet shared library, *libmxnet.so*, with *USE\\_CPP\\_PACKAGE* option turned on. Please follow the steps to build the C++ package**\n\n### Steps to build the C++ package:\n1.  Building the MXNet C++ package requires building MXNet from source.\n2.  Clone the MXNet GitHub repository **recursively** to ensure the code in submodules is available for building MXNet.\n\t```\n\tgit clone --recursive https://github.com/apache/mxnet\n\t```\n\n3.  Install the [recommended dependencies](https://mxnet.apache.org/versions/master/get_started/build_from_source.html#installing-mxnet's-recommended-dependencies) and [optional dependencies](https://mxnet.apache.org/versions/master/get_started/build_from_source.html#overview-of-optional-dependencies-and-optional-features) for building MXNet from source.\n4.  There is a configuration file for cmake, [config/*.cmake](<https://github.com/apache/mxnet/tree/master/config>) that contains all the compilation options. You can edit this file and set the appropriate options prior to running the **cmake** command.\n5.  Please refer to  [cmake configuration files](https://github.com/apache/mxnet/blob/970a2cfbe77d09ee610fdd70afca1a93247cf4fb/config/linux_gpu.cmake#L18-L37) for more details on how to configure and compile MXNet.\n6.  For enabling the build of C++ Package, set the **-DUSE\\_CPP\\_PACKAGE = 1** in cmake options.\n\n### Cross-Compilation steps:\n1.  Build the C++ package for the **host** platform to generate op.h file.\n2.  Remove the following line in [CMakeLists.txt](<https://github.com/apache/mxnet/blob/master/cpp-package/CMakeLists.txt#L15>).\n    ```\n\tCOMMAND python OpWrapperGenerator.py $<TARGET_FILE:mxnet>\n\t``` \n3.  Re-configure cmake for cross-compilation to build the **target** C++ package.\n\n## Usage\n\nIn order to consume the C++ API please follow the steps below.\n\n1. Ensure that the MXNet shared library is built from source with the **USE\\_CPP\\_PACKAGE = 1**.\n2. Include the [MxNetCpp.h](<https://github.com/apache/mxnet/blob/master/cpp-package/include/mxnet-cpp/MxNetCpp.h>) in the program that is going to consume MXNet C++ API.\n\t```c++\n\t#include <mxnet-cpp/MxNetCpp.h>\n\t```\n3. While building the program, ensure that the correct paths to the directories containing header files and MXNet shared library.\n4. The program links the MXNet shared library dynamically. Hence the library needs to be accessible to the program during runtime. This can be achieved by including the path to the shared library in the environment variable  **LD\\_LIBRARY\\_PATH** for Linux, Mac. and Ubuntu OS and **PATH** for Windows OS.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/developer_guide/1_github_contribution_and_PR_verification_tips.md",
    "content": "---\nlayout: page_category\ntitle:  GitHub contribution and PR verification tips \ncategory: Developer Guide\npermalink: /api/dev-guide/github_contribution_and_PR_verification_tips\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# GitHub contribution and PR verification tips \n\nUse this page for general git workflow tips. \n\n## Setup and configure\n\nIt is recommended that you fork the MXNet repo, and then set the original repo as an upstream remote repo. \n\nFork [https://github.com/apache/mxnet](https://github.com/apache/mxnet) then:\n\n```\ngit clone --recursive https://github.com/your_username/mxnet\ncd mxnet\ngit remote add upstream https://github.com/apache/mxnet\n```\n\nOnce `upstream` was added, then create a branch for your contribution.\n\n\n```\ngit branch your-contribution-branch\n```\n\nNote that you can incorporate the changes from `upstream` to any of your local branches during or after development via: \n\n```\ngit fetch upstream\ngit rebase upstream/master\n```\n\nSee [this stackoverflow discussion](https://stackoverflow.com/questions/3357122/git-pull-vs-git-fetch-vs-git-rebase) for more details about difference between `git pull`, `git rebase` and `git merge`.\n\nSince Apache MXNet 3rd party git submodules, to update their changes on your branch after rebase, you can run:\n\n```\ngit submodule update --recursive\n```\n\n## Save your local changes for future\n\nDuring development, you can save your current changes in your branch before committing anything. For example to go to another branch to do something else via:\n\n\n```\ngit stash save\n```\n\nTo restore the changes so that they can be added to a commit use:\n\n\n```\ngit stash pop\n```\n\n\nTo drop the changes, use:\n\n```\ngit stash drop\n```\n\n## Reset\n\nSometimes, if you want to wipe out the changes you have made you can use:\n\n```\ngit reset --hard\n```\n\nBe very careful since hard-reset removes any of the changes and you’ll be back to the HEAD commit. To remove all the changed before a commit given its commit-SHA you can use `git reset --hard commit-SHA` or `git reset --hard HEAD~2` to remove relative to the first two commits on top of HEAD.\n\nHowever, sometimes it’s useful to keep the files/changes staged when moving the HEAD which can be done via \n`git reset --soft`. All of the files changed between the original HEAD and the commit will be staged.\n\nIn [summary](https://stackoverflow.com/a/50022436),\n\n\n* **`--soft`**: **uncommit** changes, changes are left staged (*index*).\n* **`--mixed`** *(default)*: **uncommit + unstage** changes, changes are left in *working tree*.\n* **`--hard`**: **uncommit + unstage + delete** changes, nothing left.\n\n\n\n## Recover a previous commit after reset\n\nSometimes you might mistakenly reset a branch to a wrong commit. When that happens, you can use the following command to show the list of recent commits:\n\n\n```\ngit reflog\n```\n\nOnce you get the right hashtag, you can use git reset again to change the head to the right commit.\n\n\n## How to resolve conflict with master\n\nSometimes when rebasing to the most recent master as explained above, git may show you there are some conflicts which it cannot resolve. These changes will not be merged. For examples, your file `conflict.py` has some conflicts with the master branch. Here you need to:\n\n* manually modify the file to resolve the conflict.\n* After you resolved the conflict, mark it as resolved by:\n\n```\ngit add conflict.py\n```\n\n* Then you can continue rebase by:\n\n```\ngit rebase --continue\n```\n\n* Finally push to your fork, you may need to **force push** here:\n\n```\ngit push --force\n```\n\n**Note** that force push is okay when it’s on your branch and you are the only one who is using that branch. Otherwise, it can have bad consequences as it’s rewritten the history.\n\n\n## How to group multiple commits into one\n\nSometimes, you may have added a lot of related commits suitable to be grouped/combined together to create one meaningful atomic commit. For example, when later commits are only fixes to previous ones, in your PR. \nIf you haven’t configured your default git editor, do the following once:\n\n```\ngit config core.editor the-editor-you-like\n```\n\nAssume we want to merge the last 3 commits.\n\n```\ngit rebase -i HEAD~3\n```\n\n1. It will pop up an text editor. Set the **first commit as pick,** and **change later ones to squash**.\n2. After you saved the file, it will pop up another text editor to ask you modify the combined commit message.\n3. Push the changes to your fork, you need to force push.\n\n```\ngit push --force\n```\n\n**Note** that force push is okay when it’s on your branch and you are the only one who is using that branch. Otherwise, it can have bad consequences as it’s rewritten the history.\n\n\n## Apply only k-latest commits on to the master\n\nSometimes it is useful to only apply your k-latest changes on top of the master. This usually happens when you have other m-commits that are already merged before these k-commits. Directly rebase against the master might cause merge conflicts on these first m-commits (which can be safely discarded).\n\nYou can instead use the following command:\n\n\n```\n# k is the concrete number. Put HEAD~2 for the last 1 commit.\ngit rebase --onto upstream/master HEAD~k\n```\n\nYou can then force push to the master `git push --force`. Note that the above command will discard all the commits before the last k ones.\n\n\n## What is the consequence of force push\n\nThe last three tips require the force push, this is because we altered the path of the commits. **It is fine to force push to your own fork, as long as the commits changed are only yours.** In case there are multiple collaborators who use your branch there is a safer option `git push --force-with-lease.`\n\n\n## PR verification\n\nWhen sending a pull request, remember to add some tests. During the development, one can set `MXNET_TEST_COUNT=1000/10000` to test on some randomly selected test cases. This makes the testing and development cycle faster. Moreover, some test results might change due to the seed in pseudo-random number generator. To fix the seed during testing, set `MXNET_TEST_SEED=your seed number`.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/developer_guide/debugging_and_performance_optimization_tips.md",
    "content": "---\nlayout: page_category\ntitle:  Debugging and performance optimization tips\ncategory: Developer Guide\npermalink: /api/dev-guide/debugging_and_performance_optimization_tips\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Debugging and performance optimization tips\n\nThe general workflow when defining your network with Gluon API is either:\n\n* build sequentially using `nn.Sequential` or `nn.HybridSequential` \n\n* inherit from `nn.Block` or `nn.HybridBlock`\n\n## Debugging\n\nWhen debugging your MXNet code, remember the following:\n\n**Do NOT hybridize for debugging**\n\nThe difference between [imperative style (Gluon non-hybridized) and symbolic style (Gluon hybridized)]({{ \"/versions/1.2.1/architecture/program_model.html\" | relative_url }}) is:\n\n* *imperative style* is _define-by-run_\n* *symbolic style* is _define-then-run_\n\n\nBasically, that means the execution path changes when calling `hybridize` on your network inherited from `HybridBlock` or `HybridSequential` (note that inheriting directly from `Block` is the same as not hybridizing your network). For efficiency, symbolic code does not keep the intermediate results and so it would be hard to debug and examine the intermediate outputs. Therefore, if you want to *examine the intermediate results for debugging, do NOT hybridize*. Once everything is working as expected, then you can `hybridize` and enjoy the speed up.\n\nPlease checkout the [d2l](http://d2l.ai/chapter_computational-performance/hybridize.html?highlight=hybridize#hybrid-programming) for more details about the hybrid-programming model.\n\n## Use naive engine\n\nIt is also useful to set the environment variable `MXNET_ENGINE_TYPE='NaiveEngine'` prior to running your (end-to-end) code. This setting disables multi-threading and the execution engine will be synchronous, so you can examine the backtrace more easily. Remember to change it back to either the default `'ThreadedEnginePerDevice'` or `'ThreadedEngine'`.\n\nFor more details, here is a comprehensive tutorial on interactive debugging on [YouTube](https://www.youtube.com/watch?v=6-dOoJVw9_0).\n\n## Performance optimization\n\nFollowing up on using the environment variable `MXNET_ENGINE_TYPE` for debugging, here are the [available environment variables]({{ \"/api/faq/env_var\" | relative_url }})  that affect the performance of your code.\n\nPlease refer to [this presentation](https://www.slideshare.net/ThomasDelteil1/debugging-and-performance-tricks-for-mxnet-gluon) for more information on debugging and performance optimization.\n\n"
  },
  {
    "path": "docs/static_site/src/pages/api/developer_guide/examine_forward_results_with_hooks.md",
    "content": "---\nlayout: page_category\ntitle:  Examine forward results with hooks\ncategory: Developer Guide\npermalink: /api/dev-guide/examine_forward_results_with_hooks\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Examine forward results with hooks\n\nThere are currently three ways to register a function in an MXNet Gluon Block for execution:\n\n* before `forward` via [register_forward_pre_hook]({{\"/api/python/docs/api/gluon/block.html#mxnet.gluon.Block.register_forward_pre_hook\" | relative_url }})\n* after `forward` via [register_forward_hook]({{\"/api/python/docs/api/gluon/block.html#mxnet.gluon.Block.register_forward_hook\" | relative_url }})\n* as a callback via [register_op_hook]({{\"/api/python/docs/api/gluon/block.html#mxnet.gluon.Block.register_op_hook\" | relative_url }})\n\n## Pre-forward hook\n\nTo register a hook prior to forward execution, the requirement is that the registered operation **should not modify the input or output**. For example: `hook(block, input) -> None`. This is useful to get a summary before execution.\n\n```\nimport mxnet as mx\nfrom mxnet.gluon import nn\n\nblock = nn.Dense(10)\nblock.initialize()\nprint(\"{}\".format(block))\n# Dense(None -> 10, linear)\n\ndef pre_hook(block, input) -> None:  # notice it has two arguments, one block and one input\n    print(\"{}\".format(block))\n    return\n    \n# register\npre_handle = block.register_forward_pre_hook(pre_hook)\ninput = mx.nd.ones((3, 5))\nprint(block(input))\n\n# Dense(None -> 10, linear)\n# [[ 0.11254273  0.11162187  0.02200389 -0.04842059  0.09531345  0.00880495\n#  -0.07610667  0.1562067   0.14192852  0.04463106]\n# [ 0.11254273  0.11162187  0.02200389 -0.04842059  0.09531345  0.00880495\n#  -0.07610667  0.1562067   0.14192852  0.04463106]\n# [ 0.11254273  0.11162187  0.02200389 -0.04842059  0.09531345  0.00880495\n#  -0.07610667  0.1562067   0.14192852  0.04463106]]\n# <NDArray 3x10 @cpu(0)>\n```\n\nWe can `detach` a hook from a block:\n\n\n```\npre_handle.detach()\nprint(block(input))\n\n# [[ 0.11254273  0.11162187  0.02200389 -0.04842059  0.09531345  0.00880495\n#  -0.07610667  0.1562067   0.14192852  0.04463106]\n# [ 0.11254273  0.11162187  0.02200389 -0.04842059  0.09531345  0.00880495\n#  -0.07610667  0.1562067   0.14192852  0.04463106]\n# [ 0.11254273  0.11162187  0.02200389 -0.04842059  0.09531345  0.00880495\n#  -0.07610667  0.1562067   0.14192852  0.04463106]]\n# <NDArray 3x10 @cpu(0)>\n```\n\nNotice `Dense(None -> 10, linear)` is not displayed anymore.\n\n## Post-forward hook\n\nRegistering a hook after forward execution is very similar to pre-forward hook (as explained above) with the difference that the hook signature should be `hook(block, input, output) -> None` where **hook should not modify the input and output.** Continuing from the above example:\n\n\n```\ndef post_hook(block, intput, output) -> None:\n    print(\"{}\".format(block))\n    return\n    \npost_handle = block.register_forward_hook(post_hook)\nprint(block(input))\n\n# Dense(5 -> 10, linear)\n# [[ 0.11254273  0.11162187  0.02200389 -0.04842059  0.09531345  0.00880495\n#  -0.07610667  0.1562067   0.14192852  0.04463106]\n# [ 0.11254273  0.11162187  0.02200389 -0.04842059  0.09531345  0.00880495\n#  -0.07610667  0.1562067   0.14192852  0.04463106]\n# [ 0.11254273  0.11162187  0.02200389 -0.04842059  0.09531345  0.00880495\n#  -0.07610667  0.1562067   0.14192852  0.04463106]]\n# <NDArray 3x10 @cpu(0)>\n```\n\n\nNotice the difference between `pre_hook` and `post_hook` results due to shape inference after `forward` is done executing.\n\n## Callback hook\n\nWe can register a callback monitor to monitor all operators that are called by the `HybridBlock` **after hybridization** with `register_op_hook(callback, monitor_all=False) ` where the callback signature should be:\n\n\n```\ncallback(node_name: str,  opr_name: str, arr: NDArray) -> None\n```\n\nwhere `node_name` is the name of the tensor being inspected (str), `opr_name` is the name of the operator producing or consuming that tensor (str) and `arr` the tensor being inspected (NDArray).\n\n\n```\nimport mxnet as mx\nfrom mxnet.gluon import nn\n\ndef mon_callback(node_name, opr_name, arr):\n    print(\"{}\".format(node_name))\n    print(\"{}\".format(opr_name))\n    return\n    \nmodel = nn.HybridSequential(prefix=\"dense_\")\nwith model.name_scope():\n     model.add(mx.gluon.nn.Dense(2))\n\nmodel.initialize()\nmodel.hybridize()\nmodel.register_op_hook(mon_callback, monitor_all=True)\nprint(model(mx.nd.ones((2, 3, 4))))\n\n# b'dense_dense0_fwd_data'\n# b'FullyConnected'\n# b'dense_dense0_fwd_weight'\n# b'FullyConnected'\n# b'dense_dense0_fwd_bias'\n# b'FullyConnected'\n# b'dense_dense0_fwd_output'\n# b'FullyConnected'\n# [[-0.05979988 -0.16349721]\n#  [-0.05979988 -0.16349721]]\n# <NDArray 2x2 @cpu(0)>\n```\n\n\nSetting `monitor_all=False` will print only the output:\n\n\n```\n`# b'dense_dense0_fwd_output'`\n`# b'FullyConnected'``\n# [[-0.05979988 -0.16349721]\n#  [-0.05979988 -0.16349721]]\n# <NDArray 2x2 @cpu(0)`\n```\n\nNote that to get the internal operator node names, one can use `model.collect_params().items()`.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/developer_guide/exception_handing_and_custom_error_types.md",
    "content": "---\nlayout: page_category\ntitle:  Exception handing and custom error types\ncategory: Developer Guide\npermalink: /api/dev-guide/exception_handing_and_custom_error_types\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Exception handing and custom error types\n\n\nApache MXNet v1.7 has added the custom error type support and as a result `MXNetError` is inherited from `RuntimeError` so it is possible to register a custom error type in the backend and prepend its error message. Then in the frontend, one can throw the exception of the registered error type. \n\nFor example, we want the `transpose` operator defined in the C++ backend to throw `ValueError` type in the Python frontend. Therefore, in the C++ backend we can add this check:\n\n```\nCHECK_EQ(axes_set.size(), axes.ndim()) << \"ValueError: Repeated axis in transpose.\"\n                                       << \" param.axes = \"\n                                       << param.axes;\n```\n\nso that on the frontend, when a problematic `transpose` call is made such as:\n\n```\nfrom mxnet import np\n\ndat = np.random.normal(0, 1, (3, 4, 5))\ndat.transpose((0, 0, 1))\n```\n\nthe following traceback will be produced:\n\n\n```\nValueError                                Traceback (most recent call last)\n<ipython-input-3-3ad259b4e371> in <module>\n----> 1 dat.transpose((0, 0, 1))\n\n~/mxnet-distro/mxnet-build/python/mxnet/numpy/multiarray.py in transpose(self, *axes)\n   1460             elif axes[0] is None:\n   1461                 axes = None\n-> 1462         return _mx_np_op.transpose(self, axes=axes)\n   1463\n   1464     def flip(self, *args, **kwargs):\n~/mxnet-distro/mxnet-build/python/mxnet/ndarray/register.py in transpose(a, axes, out, name, **kwargs)\n\n~/mxnet-distro/mxnet-build/python/mxnet/_ctypes/ndarray.py in _imperative_invoke(handle, ndargs, keys, vals, out, is_np_op, output_is_list)\n    105         c_str_array(keys),\n    106         c_str_array([str(s) for s in vals]),\n--> 107         ctypes.byref(out_stypes)))\n    108\n    109     create_ndarray_fn = _np_ndarray_cls if is_np_op else _ndarray_cls\n    \n~/mxnet-distro/mxnet-build/python/mxnet/base.py in check_call(ret)\n    271     \"\"\"\n    272     if ret != 0:\n--> 273         raise get_last_ffi_error()\n    274\n    275\nValueError: Traceback (most recent call last):\n  File \"src/operator/numpy/np_matrix_op.cc\", line 77\n  \nValueError: Check failed: axes_set.size() == axes.ndim() (2 vs. 3) : Repeated axis in transpose. param.axes = [0,0,1]\n```\n\n\nNote that as of writing this document, the following Python error types are supported:\n\n\n* `ValueError`\n* `TypeError`\n* `AttributeError`\n* `IndexError`\n* `NotImplementedError`\n\nCheck [this](https://github.com/apache/mxnet/blob/master/python/mxnet/error.py) resource for more details\nabout Python supported error types that MXNet supports.\n\n## How to register a custom error type\n\nHere is the way to register a custom error type in Python frontend:\n\n\n```\nimport mxnet as mx\n\n@mx.error.register\nclass MyError(mx.MXNetError):\n    def __init__(self, msg):\n        super().__init__(msg)\n```\n\nThen in the C++ backend, you can refer to `MyError` via:\n\n`LOG(FATAL) << \"MyError: this is a custom error message\"`\n"
  },
  {
    "path": "docs/static_site/src/pages/api/developer_guide/profiling.md",
    "content": "---\nlayout: page_category\ntitle:  Profiling\ncategory: Developer Guide\npermalink: /api/dev-guide/profiling\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Profiling\n\nApache MXNet provides memory [profiler]({{\"/api/python/docs/api/mxnet/profiler/index.html\" | relative_url }}) which is a way to access what is happening under the hood during runtime. The common scenario is you want to use the profiler for your hybridized model and visualize the outputs via `chrome://tracing`. Here are the steps you need to do:\n\n1. Configure the profiler\n2. `set_state('run')` before the model is defined\n3. Add `mx.nd.waitall()` to enforce synchronization after you have done with some computation (maybe as part of training)\n4. Then add `set_state('stop')` \n5. Finally `dump` the profiling results\n\n\nHere is a simple example\n\n```\nimport mxnet as mx\nfrom mxnet.gluon import nn\nfrom mxnet import profiler\n\ndef enable_profiler(profile_filename, run=True, continuous_dump=False, aggregate_stats=False):\n    profiler.set_config(profile_symbolic=True,\n                        profile_imperative=True,\n                        profile_memory=True,\n                        profile_api=True,\n                        filename=profile_filename,\n                        continuous_dump=continuous_dump,\n                        aggregate_stats=aggregate_stats)\n    if run:\n        profiler.set_state('run')\n\nenable_profiler(profile_filename='test_profiler.json', run=True, continuous_dump=True)\nprofiler.set_state('run')\n\nmodel = nn.HybridSequential(prefix='net_')\nwith model.name_scope():\n    model.add(nn.Dense(128, activation='tanh'))\n    model.add(nn.Dropout(0.5))\n    model.add(nn.Dense(64, activation='tanh'),\n              nn.Dense(32, in_units=64))\n    model.add(nn.Activation('relu'))\nmodel.initialize(device=mx.cpu())\nmodel.hybridize()\n\ninputs = mx.sym.var('data')\n\nwith mx.autograd.record():\n    out = model(mx.nd.zeros((16, 10), device=mx.cpu()))\nout.backward()\nmx.nd.waitall()\nprofiler.set_state('stop')\nprofiler.dump(True)\n```\n\nAnd in `chrome://tracing` use the `load` and select `test_profiler.json`, then you will see something like this\n![dev_guide_profilling_1](/assets/img/dev_guide_profilling_1.png) To understand what is going on, we need to dive deep into the MXNet runtime.\n\n## Dive deep into MXNet runtime with the profiler\n\nLet's start with a simple example and explain as we go on. The following code creates a 3x3 tensor, computes the diagonal and then sum's along the diagonal (to compute the “trace”). Using the MXNet profiler, we capture internal MXNet behavior and dump it to a string and print it (`dumps()`) and also dump it to a file (`dump()`). Then we can import that file in `chrome://tracing` and view it graphically.\n\n```\nimport mxnet as mx\nimport numpy as np\n \nfrom mxnet import profiler\n \n#configure the profiler\nprofiler.set_config(profile_all=True, aggregate_stats=True, filename='trace_profile.json')\n#start the profiler collecting data\nprofiler.set_state('run')\n \n###########################################################\n#1. create our data\ndata = np.linspace(1,9,9).reshape((3,3))\n \n#2. create an MXNet ndarray\na = mx.nd.array(data)\n \n#3. compute on our data and produce results\nb = mx.nd.diag(a)\nc = mx.nd.sum(b,-1)\n \n#4. wait for computation to finish\nmx.nd.waitall()\n###########################################################\n \n#stop the profiler\nprofiler.set_state('stop')\n \n#dump the profiling data as a string\nprint(profiler.dumps())\n#dump the profiling data as a json file that can be viewed graphically\nprofiler.dump()\n```\n\nWhen running this code, the dumps function dumps the profiling data to a string and returns it (which we promptly print). This statistical info is shown below.\n\n```\nProfile Statistics:\n    Note the difference in units for different entries.\nDevice Storage\n=================\nName                          Total Count    Min Use  (kB)    Max Use  (kB)    Avg Use  (kB)\n----                          -----------    -------------    -------------    -------------\nMemory: cpu/0                           3          96.0600          96.0760           0.0080\n\nMXNET_C_API\n=================\nName                          Total Count        Time (ms)    Min Time (ms)    Max Time (ms)    Avg Time (ms)\n----                          -----------        ---------    -------------    -------------    -------------\nMXImperativeInvoke                      2           0.3360           0.0990           0.2370           0.1680\nMXNet C API Calls                      17           0.2320           0.2160           0.2320           0.0080\nMXNDArraySyncCopyFromCPU                1           0.1750           0.1750           0.1750           0.1750\nMXNDArrayCreate                         1           0.1050           0.1050           0.1050           0.1050\nMXNDArrayGetShape                      11           0.0210           0.0000           0.0160           0.0019\nMXNDArrayWaitAll                        1           0.0200           0.0200           0.0200           0.0200\nMXNDArrayGetDType                       1           0.0010           0.0010           0.0010           0.0010\nMXNet C API Concurrency                34           0.0000           0.0000           0.0010           0.0000\n\noperator\n=================\nName                          Total Count        Time (ms)    Min Time (ms)    Max Time (ms)    Avg Time (ms)\n----                          -----------        ---------    -------------    -------------    -------------\nsum                                     1           0.0520           0.0520           0.0520           0.0520\ndiag                                    1           0.0410           0.0410           0.0410           0.0410\nWaitForVar                              1           0.0220           0.0220           0.0220           0.0220\n```\n\nThe dump function writes out the same data in a format that can be opened in `chrome://tracing` and displayed visually. This can be seen in the diagram below.\n\n![dev_guide_profilling_2.png](/assets/img/dev_guide_profilling_2.png)\nThe profiling data has captured info about interesting functions that have executed while your program was running. Here are some explanations about what each one does.\n\n### **The functions in the C_API are:**\n\n|**Function Name**\t|**Description**\t|\n|---\t|---\t|\n|**MXImperativeInvoke**\t| invokes an operator to perform the computation |\n|**MXNDArrayCreate**\t| creates  an ndarray\t|\n| **MXNDArrayGetDType**\t| returns  the data type of the ndarray |\n| **MXNDArrayGetShape**\t| returns  the shape of the ndarray (as a tuple where each element is the size of a  dimension) |\n| **MXNDArraySyncCopyFromCPU** | called when data is initially residing outside of an MXNet data structure (ie.  numpy.ndarry rather than mxnet.numpy.ndarray). Data is copied into the MXNet  data structure   |\n| **MXNDArrayWaitAll**\t| wait for all asynchronous operations to finish in MXNet. This function is only  used in benchmarking to wait for work to happen. In a real program, there is no waiting and data dependencies are evaluated and computation executed as needed in a As Late As Possible (ALAP) way\t|\n\n### **The function in the Engine API are:**\n\n| **Function Name**\t| **Description**\t|\n|---\t|---\t|\n| **WaitForVar**\t| Takes a variable reference as input and waits until that variable has been computed before returning\t|\n\n### **Other API functions:**\n\n| **Function Name**\t| **Description**\t|\n|---\t|---\t|\n| **ResourceParallelRandomSetSeed**\t| sets the random number generator seed\t|\n\n### **Operators we intended to call in the code:**\n\n| **Operator Name**\t| **Description**\t|\n|---\t|---\t|\n| **sum**\t| sum  a tensor along a particular axis\t|\n| **diag**\t| compute the diagonal of the tensor\t|\n\n\n\n## Closer look\n\nFrom the code, we can identify the major events in our test application\n\n1. Initialize our input data\n2. Creating a new MXNet ndarray using our existing data values\n3. Compute on our data\n    1. produce the diagonal of the input data\n    2. sum along the diagonal to compute the “trace” of the matrix\n4. Wait for computation to finish (only needed when profiling)\n\nIn the following list, #1 uses regular numpy functions to initialize data. MXNet is not involved in this process. In #2, we create an MXNet ndarray and quite a few things happen under the hood. The screenshot below shows a zoomed in portion of the timeline.\n\n![dev_guide_profilling_3.png](/assets/img/dev_guide_profilling_3.png)\nHere, the four red arrows show the important events in this sequence.\n\n1. First, the `MXNDArrayCreate` is called to physically  allocate space to store the data and other necessary attributes in the `ndarray` class.\n2. Then some support functions are called (`MXNDArrayGetShape,` `MXNDArrayGetDType`) while initialing the data structure.\n3. Finally the data is copied from the non-MXNet ndarray into the newly prepared MXNet ndarray by the `MXNDArraySyncCopyFromCPU`  function.\n\nNext, #3 (in our code example) begins the computing process to produce our output data. The screenshot below shows this behavior.\n\n![dev_guide_profilling_4.png](/assets/img/dev_guide_profilling_4.png)\nHere you can see that the following sequence of events happen:\n\n1. `MXImperativeInvoke` is called the first time to launch the diagonal operator from #3 (in our code example).\n2. Soon after that the actual **`diag`**  operator begins executing in another thread.\n3. While that is happening, our main thread moves on and calls `MXImperativeInvoke` again to launch the **`sum`**  operator. Just like before, this returns without actually executing the operator  and continues.\n4. Lastly, the `MXNDArrayWaitAll` is called as the main thread has progressed to #4 in our app. It will wait here while all the  computation finishes.\n\nNext lets look at a view of the part of the timeline zoomed to the actual operator execution.\n\n![dev_guide_profilling_5.png](/assets/img/dev_guide_profilling_5.png)\nHere there are 3 main events happening:\n\n1. The **`diag`** operator is executing first.\n2. Then the `ResourceParallelRandomSetSeed` runs.\n3. And finally the `sum` operator executes  (for a very short time as shown by the big red arrow).\n\nThe `diag` operator running makes sense (although seems to take a little longer than we'd like). At the end, the sum operator runs (very quickly!). But the weird part in the middle is **`ResourceParallelRandomSetSeed`** running. This is part of the MXNet resource manager. The resource manager handles temporary space and random number generators needed by the operators. The **`sum`** operator requests temporary space in order to compute the sum, and therefore launches the resource manager (for the first time) here. As part of its startup sequence, the random number generator is initialized by setting the seed. So this is some initialization overhead. But let's try and run the app again, running the compute twice, and look at the 2nd run to try and remove this initialization from our profiling.\n\nHere is the modified code:\n\n```\nimport mxnet as mx\nimport numpy as np\n \nfrom mxnet import profiler\n \nprofiler.set_config(profile_all=True, aggregate_stats=True, filename='trace_profile.json')\nprofiler.set_state('run')\n \n################\n# first run\nsdata = np.linspace(1,9,9).reshape((3,3))\n \nsa = mx.nd.array(sdata)\nsb = mx.nd.diag(sa)\nsc = mx.nd.sum(sb,-1)\n \nmx.nd.waitall()\n################\n \n################\n# second run\ndata = np.linspace(1,9,9).reshape((3,3))\n \na = mx.nd.array(data)\nb = mx.nd.diag(a)\nc = mx.nd.sum(b,-1)\n \nmx.nd.waitall()\n################\n \nprofiler.set_state('stop')\n \nprint(profiler.dumps())\nprofiler.dump()\n```\n\nNotice that we renamed the variables and made another copy after the `waital` call. This is so that MXNet doesn’t have to worry about re-using variables, and to segment the 2nd half after the first time initialization.\n\nHere is an overview of the *new* timeline:\n\n![dev_guide_profilling_6.png](/assets/img/dev_guide_profilling_6.png)\nThe first red box is the first run, and the 2nd smaller one is the 2nd run. First off, we can see how much smaller the 2nd one is now without any of the initialization routines. Here is a zoomed in view of just the 2nd run. \n\n\n![dev_guide_profilling_7.png](/assets/img/dev_guide_profilling_7.png)\nWe still have the same sequence of events at the beginning to initialize the MXNet ndarray (`MXNDArrayCreate`, `MXNDArrayGetShape`, `MXNDArrayGetDType`, `MXNDArraySyncCopyFromCPU`). Then the **`diag`** operator runs, followed by the **`sum`** operator, and finally the `waitall`. When you look at this, be careful about the assumptions that you make. In this version of the timeline, it appears that the operator executes after the `MXImperativeInvoke` runs, and seems to imply an inherent ordering. But realize that there is no dependency between the **`diag`** operator finishing and the next **`MXImperativeInvoke`** launching the **`sum`** operator. In this case, it just-so-happens that the **`diag`** operator finishes so quickly that it appears that way. But in reality the main thread is launching the operators and not waiting for them to finish. Lastly, keep in mind that in this case by the time we hit the **`MXNDArrayWaitAll`** everything is already done and we return immediately, but in other circumstances it may sit here waiting for everything to finish (like we saw earlier in the first run). \n\n\n"
  },
  {
    "path": "docs/static_site/src/pages/api/faq/add_op_in_backend.md",
    "content": "---\nlayout: page_category\ntitle: A Beginner's Guide to Implementing Operators in MXNet Backend\ncategory: faq\nfaq_c: Extend and Contribute to MXNet\nquestion: How do I implement operators in MXNet backend?\npermalink: /api/faq/add_op_in_backend\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# A Beginner's Guide to Implementing Operators in MXNet Backend\n\n## Introduction\nOperators are essential elements for constructing neural networks. They define mathematical formulas\nof transforming input data (tensors) to outputs. MXNet has a rich set of operators from simple ones,\nsuch as element-wise sum, to complicated ones, such as convolution, that is\ncapable of constructing most of the popular neural networks. You may have noticed\nthat many operators implemented in MXNet have their equivalent forms in Numpy, such as\n[repeat](https://docs.scipy.org/doc/numpy/reference/generated/numpy.repeat.html),\n[tile](https://docs.scipy.org/doc/numpy/reference/generated/numpy.tile.html),\netc., and wonder why we could not simply use those Numpy operators in MXNet. One of the\nmajor reasons is that we need to support both CPU and GPU computing for the operators in MXNet,\nwhile Numpy operators do not possess GPU computing capability.\nIn addition, we have performed plenty of\noptimizations for various components in MXNet, such as tensor data structure (`NDArray`),\nexecution engine, computational graph and so on, for maximizing memory and runtime efficiency.\nAn operator implemented under the MXNet operator framework would greatly\nleverage those optimizations for exhaustive performance enhancement.\n\nIn this tutorial, we are going to practice implementing an operator using\nC++ in the MXNet backend. After finishing the implementation,\nwe will add unit tests using Python for the operator we just implemented.\n\n## Implementation\n\n### An Operator Example\n\nLet's take the [quadratic function](https://en.wikipedia.org/wiki/Quadratic_function)\nas an example: `f(x) = ax^2+bx+c`. We want to implement an operator called `quadratic`\ntaking `x`, which is a tensor, as an input and generating an output tensor `y`\nsatisfying `y.shape=x.shape` and each element of `y` is calculated by feeding the\ncorresponding element of `x` into the quadratic function `f`.\nHere variables `a`, `b`, and `c` are user input parameters.\nIn frontend, the operator works like this:\n\n```python\nx = [[1, 2], [3, 4]]\ny = quadratic(data=x, a=1, b=2, c=3)\ny = [[6, 11], [18, 27]]\n```\n\nTo implement this, we first create three files: `quadratic_op-inl.h`,\n`quadratic_op.cc`, and `quadratic_op.cu`. The header file's name\nis prefixed by the operator name and followed by `op` and `-inl`\nindicating that this is an operator implementation with inline\nfunctions shared by CPU and GPU computing. The CPU and GPU\nspecific implementations reside in their own `.cc` and `.cu` files,\nrespectively. We normally put pure tensor related operators\n(e.g. `tile`, `repeat`, etc.) under\nthe directory `src/operator/tensor`, and neural network operators\n(e.g. `Convolution`, `Pooling`, etc.) under `src/operator/nn`.\nYou may have noticed that many neural network operators including\n`Convolution` and `Pooling` are currently saved under `src/operator`.\nWe plan to move them to `src/operator/nn` for better file organization\nand clearer hierarchy in the future.\n\nNext, we are going to\n1. Define the parameter struct\nfor registering `a`, `b`, and `c` in `quadratic_op-inl.h`.\n2. Define type and shape inference functions in `quadratic_op-inl.h`.\n3. Define forward and backward functions in `quadratic_op-inl.h`.\n4. Register the operator using [nnvm](https://docs.tvm.ai/dev/nnvm_overview.html)\nin `quadratic_op.cc` and `quadratic_op.cu` for\nCPU and GPU computing, respectively.\n\nNow let's walk through the process step by step.\n\n### Parameter Registration\nWe first define `struct QuadraticParam` as a placeholder for the\nparameters `a`, `b`, and `c` in `quadratic_op-inl.h`.\nThe struct inherits from a base template\nstruct named `dmlc::Parameter`, where the template argument is the derived struct\n`QuadraticParam`. This technique, which is called [curiously recurring template\npattern](https://en.wikipedia.org/wiki/Curiously_recurring_template_pattern),\nachieves static polymorphism. It is similar to using a virtual function,\nbut without the cost associated with dynamic polymorphism.\n\n```cpp\nstruct QuadraticParam : public dmlc::Parameter<QuadraticParam> {\n  float a, b, c;\n  DMLC_DECLARE_PARAMETER(QuadraticParam) {\n    DMLC_DECLARE_FIELD(a)\n      .set_default(0.0)\n      .describe(\"Coefficient of the quadratic term in the quadratic function.\");\n    DMLC_DECLARE_FIELD(b)\n      .set_default(0.0)\n      .describe(\"Coefficient of the linear term in the quadratic function.\");\n    DMLC_DECLARE_FIELD(c)\n      .set_default(0.0)\n      .describe(\"Constant term in the quadratic function.\");\n  }\n};\n```\n\nThe function calls in the above parameter struct are self-explanatory by their names.\nNote that for each parameter, we set the default value to `0.0` such that users can\nskip passing 0-value parameters through the quadratic operator interface. You\ncan choose not to define the default value for a parameter if it is required\nat runtime. Meanwhile, adding brief descriptions to the parameters enables\nthe documentation engine to display them on\n[MXNet documentation web page]({{'/api/python/docs/api'|relative_url}}).\n\n\n### Attribute Inference\nAttribute inference is the process of deducing the properties of `NDArray`s\nin neural networks from user provided information. Two most common attributes\nof an `NDArray` are data shape and data type.\nLet's take a look at the following example.\nGiven an input `NDArray` called `data`, you invoke the `quadratic` operator\nlike this: `output = mx.nd.quadratic(data, a=1, b=2, c=3)`. Before calculating\nthe `output` values, its shape and data type are inferred from the input\n`data`'s shape and type following\nthe rules you defined in order to allocate memory space for the output tensor.\n\nOne important thing to note that inference functions should be capable of\nperforming **mutual inference**, i.e.\ninferring one argument's attribute from another argument's attribute if\npossible according to the definition of the operator.\nThis is very useful for a computational graph to deduce unknown attributes\nfor a neural network in symbolic programming. Users can view the computational\ngraph as a symbol with every element initialized for running data\nthroughout the neural network, including memory allocation for each tensor,\ndevice placement for each operator, etc. Users normally just need\nto provide minimum necessary information, such as input data shapes, etc.,\nto the computational graph, and the graph will fill up the unknown attributes\nusing the attribute inference functions defined in the operators building up\nthe neural network.\n\nLet's consider the following example.\n\n```python\n>>> import mxnet as mx\n>>> a = mx.sym.Variable('a', shape=(2, 0))\n>>> b = mx.sym.Variable('b')\n>>> c = mx.sym.Variable('c', shape=(0, 3))\n>>> d = a * b + b * c\n>>> print d.infer_shape()\n([(2L, 3L), (2L, 3L), (2L, 3L)], [(2L, 3L)], [])\n```\n\nThe last line of the above code snippet is a tuple of three lists returned\nby `d.infer_shape()`. The first list contains all the argument shapes\nof `a`, `b`, and `c`. The second contains the output shape of `d`. The\nthird one represents the shapes of auxiliary states, which is not used\nin this case, and thus is empty.\nIn this example, we only specified values for variable `a`'s first dimension\nand `c`'s second dimension. The `0` in shape `(2, 0)` indicates that the size\nof the second dimension is unknown, same meaning for shape `(0, 3)`.\nHowever, the symbol `d` still successfully inferred the shapes\nfor all the variables and final output. This is a result of mutual\ninference. In MXNet, the whole process can be interpreted as this:\n1. `a` and `b` are combined via an element-wise multiplication operator,\nso the shapes of `a` and `b` are same and `b`'s first dimension size is `2`.\n2. `b` and `c` are combined via an element-wise multiplication operator too,\nso the shapes of `b` and `c` are same and `b`'s second dimension size is `3`.\n3. Now `b`'s shape is completely known, so `a` and `c` missing dimension sizes\nare known as well.\n4. `d` is a result from adding `a * b` and `b * c`, so d should also\nhave the same shape as `b`.\n\nThe above four steps illustrate how shape inference logic works in MXNet.\nIt is actually implemented in the shape inference functions of the operators for\nelement-wise multiplication and addition.\n\nFor our `quadratic` operator, shape inference possesses quite similar logic.\n\n```cpp\ninline bool QuadraticOpShape(const nnvm::NodeAttrs& attrs,\n                             mxnet::ShapeVector* in_attrs,\n                             mxnet::ShapeVector* out_attrs) {\n  CHECK_EQ(in_attrs->size(), 1U);\n  CHECK_EQ(out_attrs->size(), 1U);\n\n  SHAPE_ASSIGN_CHECK(*out_attrs, 0, in_attrs->at(0));\n  SHAPE_ASSIGN_CHECK(*in_attrs, 0, out_attrs->at(0));\n  return out_attrs->at(0).ndim() != 0U && out_attrs->at(0).Size() != 0U;\n}\n```\n\nHere are a few things to note about the above function:\n\n1. `attrs` contains parameters `a`, `b`, and `c` from user input.\nIt's not used here since we don't rely on that information for shape inference.\n2. `in_attrs` is a vector containing all input shapes. Since there is\nonly one input argument for operator `quadratic`, we used macro `CHECK_EQ`\nto assert when the vector's size is wrong.\n3. `out_attrs` is a vector containing all output shapes. We also used\n`CHECK_EQ` to verify the size of the vector since there is only one output.\n4. We called macro `SHAPE_ASSIGN_CHECK` twice for mutual inference. One for\ninferring the output shape from the input shape, the other one is for inferring\nthe input shape from the output shape.\nIf there are any unequal non-zero values in the same\ndimension of two shapes, such as `(2, 3)` and `(3, 3)`, the macro would throw an\nexception with an error message for shape inference.\n5. At the end of the function body, we checked whether the output shape\nis completely known by testing whether the shape is not empty and\nthe shape's size is greater than `0`. Note that in MXNet, an empty shape\nmeans that the shape is unknown, and\na `0` in a shape means that the size of that dimension is unknown. In both\nsituations, the missing shape information must\nbe inferred from other shapes. If it cannot be inferred,\nthe function should return `false` to notify the caller about shape inference failure.\n6. MXNet provides a convenience function implementing the logic of mutual inference\nfor general element-wise operators with the following interface. Users can\ninstantiate this function with `n_in=1` and `n_out=1` to replace the above\nfunction `QuadraticOpShape` in operator registration (explained later).\nThe function `QuadraticOpShape` posted here is for the purpose of illustration only.\n\n```cpp\ntemplate<int n_in, int n_out>\ninline bool ElemwiseShape(const nnvm::NodeAttrs& attrs,\n                          mxnet::ShapeVector *in_attrs,\n                          mxnet::ShapeVector *out_attrs);\n```\n\nThe same logic goes for data type inference. We will leave the analysis of\nthe following code sample to users. Note that `-1` means the data type\nis unknown and must be inferred from other input or output data types.\n\n```cpp\ninline bool QuadraticOpType(const nnvm::NodeAttrs& attrs,\n                            std::vector<int>* in_attrs,\n                            std::vector<int>* out_attrs) {\n  CHECK_EQ(in_attrs->size(), 1U);\n  CHECK_EQ(out_attrs->size(), 1U);\n\n  TYPE_ASSIGN_CHECK(*out_attrs, 0, in_attrs->at(0));\n  TYPE_ASSIGN_CHECK(*in_attrs, 0, out_attrs->at(0));\n  return out_attrs->at(0) != -1;\n}\n```\n\nAgain, MXNet provides the following convenience function for mutual\ntype inference of element-wise operators. Users can use that\nin operator registration (explained later).\n\n```cpp\ntemplate<int n_in, int n_out>\ninline bool ElemwiseType(const nnvm::NodeAttrs& attrs,\n                         std::vector<int>* in_attrs,\n                         std::vector<int>* out_attrs);\n```\n\n### Forward Function\nForward function defines the operator's behavior in the forward pass\nof neural networks. For our `quadratic` operator, it simply implements\nthe logic of running a tensor through the quadratic function by performing\na few element-wise operations. The forward function's signature is fixed\nin MXNet as follows:\n\n```cpp\nvoid (const nnvm::NodeAttrs& attrs,\n      const OpContext& ctx,\n      const std::vector<TBlob>& inputs,\n      const std::vector<OpReqType>& req,\n      const std::vector<TBlob>& outputs);\n```\n\nWe first paste the whole forward function code here\nand then go through it line by line.\n\n\n{% raw %}\n\n```cpp\ntemplate<typename xpu>                                                        // 1\nvoid QuadraticOpForward(const nnvm::NodeAttrs& attrs,                         // 2\n                        const OpContext& ctx,                                 // 3\n                        const std::vector<TBlob>& inputs,                     // 4\n                        const std::vector<OpReqType>& req,                    // 5\n                        const std::vector<TBlob>& outputs) {                  // 6\n  CHECK_EQ(inputs.size(), 1U);                                                // 7\n  CHECK_EQ(outputs.size(), 1U);                                               // 8\n  CHECK_EQ(req.size(), 1U);                                                   // 9\n  mshadow::Stream<xpu> *s = ctx.get_stream<xpu>();                            // 10\n  const TBlob& in_data = inputs[0];                                           // 11\n  const TBlob& out_data = outputs[0];                                         // 12\n  const QuadraticParam& param = nnvm::get<QuadraticParam>(attrs.parsed);      // 13\n  using namespace mxnet_op;                                                   // 14\n  MSHADOW_TYPE_SWITCH(out_data.type_flag_, DType, {                           // 15\n    MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, {                               // 16\n      Kernel<quadratic_forward<req_type>, xpu>::Launch(                       // 17\n          s, out_data.Size(), out_data.dptr<DType>(), in_data.dptr<DType>(),  // 18\n          param.a, param.b, param.c);                                         // 19\n    });                                                                       // 20\n  });                                                                         // 21\n}                                                                             // 22\n```\n\n{% endraw %}\n\n- Line 1: `xpu` stands for a generic device type so that the function can be instantiated\nfor both CPU and GPU computing using concrete types `cpu` and `gpu`. The instantiation happens\nat the time when the operator is registered in `.cc` and `.cu` files.\n- Line 2: `attrs` is a node attribute containing the user input parameters `a`, `b`, and `c`.\nHere the node represents a placeholder for the operator in the whole computational graph for\nthe neural network.\n- Line 3: `ctx` holds something called `stream` for\nserializing asynchronous executions. Let's consider\nthis example for understanding the functionality of `stream`.\nWe want to launch several GPU kernels with the same `stream` from CPU.\nEven though the launching operation is non-blocking, the `stream` guarantees\nthat the kernels execute in the same order on GPU as they are launched from CPU.\n- Line 4: `inputs` is a vector of input tensors (only one input tensor\nfor the `quadratic` operator).\n- Line 5: `req` is a vector of `OpReqType` values. Each value defines\nthe way of writing calculated values to the output tensors.\nTherefore, the number of `req`s must be the same as the number of output tensors.\nMXNet currently supports three types of `req` in frontend: `null`, `write`, and `add`.\n`null` means skipping calculating the corresponding output tensor,\n`write` means overwriting the values in the output tensor with the ones\ncalculated by this operator, and `add` means adding the calculated values\nto the existing ones in the output tensor. Note that `null` and `add` are usually\nseen in backward passes. The former is for skipping calculating\nthe gradients of un-learnable parameters (such as index arrays),\nand the latter is for accumulating gradients throughout networks.\n- Line 6: `outputs` is a vector of output tensors (only one\noutput tensor for the `quadratic` operator).\n- Lines 7-9: Verify that the size of each vector is expected.\nOtherwise, stop moving forward and print error message.\n- Line 10: Get the `stream` from the `ctx` for launching kernels.\n- Lines 11-12: Define the references of the input and output tensors\nfor later coding convenience. Note that `TBlob` can be understood\nas a uniform data structure for tensors of various dimensions, such\nthat tensors of different dimensions can be put in a homogeneous container,\nsuch as `std::vector` and `std::list`. You can still\nget tensors of desired dimensions from a `TBlob` object through\nthe interface `get_with_shape`.\n- Line 13: Get user input parameters from the node attribute.\n- Lines 15-21: This is the place where the mathematical formula of the operator\nis implemented. The macros `MSHADOW_TYPE_SWITCH` and `MXNET_ASSIGN_REQ_SWITCH` enable\nthe code block to work for all the supported data types and `req` types in MXNet.\nInside the inner-most macro, we launch the kernel for calculating\nthe output tensor such that each thread takes an element from\nthe input tensor, feeds it into the quadratic function, and assigns\nthe output element to the output tensor based on `req` type. Note that\n`Kernel::Launch` serves as a universal interface for launching\nparallel computation on both CPU and GPU. This allows most of\nthe simple operators to share the same piece of code for CPU and GPU as\nparallelization approaches are often identical on both types of devices.\nThe kernel function is defined as the following, where the function\n`Map` is executed by each thread for each input element. The `out_data.Size()`,\nin the `Kernel::Launch` function corresponds to the factor by which the\nworkload will get parallelized among the different threads, which here\ncorresponds to the size of the output array. To explain a little\nbit more on the two macros used in the kernel struct: (1) `MSHADOW_XINLINE` is\na consolidated macro for inlining functions compiled by both CPU and GPU\ncompilers. It enables CPU and GPU computing to share the same piece of code.\n(2) `KERNEL_ASSIGN` is a macro for unifying the statements of different `req`s\ninto the same line of code. It's named `KERNEL_ASSIGN` because we call\nthe code blocks running parallel computation kernels.\nOn CPUs, the kernels are normally wrapped by the OpenMP `parallel` directive;\nwhile on GPUs, they are the kernel functions launched by CUDA library.\n\n```cpp\ntemplate<int req>\nstruct quadratic_forward {\n  template<typename DType>\n  MSHADOW_XINLINE static void Map(int i, DType* out_data, const DType* in_data,\n                                  const float a, const float b, const float c) {\n    KERNEL_ASSIGN(out_data[i], req, in_data[i] * (a * in_data[i] + b) + c);\n  }\n};\n```\n\n### Backward Function\nBackward functions play the role of propagating derivatives of loss function\nwith respect to the outputs of the last layer throughout the network to the first\nlayer. The whole process is often known as backward propagation. We are not\ngoing to delineate the principle of backward propagation here since users can find\ngreat details covered in other resources, such as\n[CS231n](https://cs231n.github.io/optimization-2/) and\n[How the backgropagation algorithm works](https://neuralnetworksanddeeplearning.com/chap2.html).\nThe problem we are going to solve here for the `quadratic` operator is that\ngiven a tensor representing the gradient of the loss function with respect\nto the output of the operator, calculate the gradient with respect to\nthe input of the operator. There is no need to calculate the derivatives\nof loss function with respect to user input parameters `a`, `b`, and `c`\nsince they are not learnable parameters in the network. To formulate the problem:\ngiven `dL/dy` and `y = a*x^2 + b*x + c`, where `L` represents the loss function and\n`y` stands for the output of the quadratic tensor, we need to solve for\n`dL/dx`. Using the chain-rule, it is obvious to find that\n\n```\ndL/dx = dL/dy * dy/dx = dL/dy * (2*a*x + b).\n```\n\nThe above equation indicates that `dL/dx` depends on the gradient\nof the output tensor and value of the input tensor.\nThe backward function's signature is the same as the forward function's.\nWith the aforementioned information in mind,\nlet's breakdown the following backward function line by line.\n\n{% raw %}\n\n```cpp\ntemplate<typename xpu>                                                       // 1\nvoid QuadraticOpBackward(const nnvm::NodeAttrs& attrs,                       // 2\n                         const OpContext& ctx,                               // 3\n                         const std::vector<TBlob>& inputs,                   // 4\n                         const std::vector<OpReqType>& req,                  // 5\n                         const std::vector<TBlob>& outputs) {                // 6\n  CHECK_EQ(inputs.size(), 2U);                                               // 7\n  CHECK_EQ(outputs.size(), 1U);                                              // 8\n  CHECK_EQ(req.size(), 1U);                                                  // 9\n  mshadow::Stream<xpu> *s = ctx.get_stream<xpu>();                           // 10\n  const TBlob& out_grad = inputs[0];                                         // 11\n  const TBlob& in_data = inputs[1];                                          // 12\n  const TBlob& in_grad = outputs[0];                                         // 13\n  const QuadraticParam& param = nnvm::get<QuadraticParam>(attrs.parsed);     // 14\n  using namespace mxnet_op;                                                  // 15\n  MSHADOW_TYPE_SWITCH(out_grad.type_flag_, DType, {                          // 16\n    MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, {                              // 17\n      Kernel<quadratic_backward<req_type>, xpu>::Launch(                     // 18\n          s, in_grad.Size(), in_grad.dptr<DType>(), out_grad.dptr<DType>(),  // 19\n          in_data.dptr<DType>(), param.a, param.b);                          // 20\n    });                                                                      // 21\n  });                                                                        // 22\n}                                                                            // 23\n```\n\n{% endraw %}\n\n- Lines 1-6: Backward function has the same signature as forward function.\n- Lines 7-9: Check the sizes of the function arguments. One thing to note\nthat since the gradient of the input depends on both the gradient of the output and\nthe input tensor itself, `inputs` must contain two `TBlob` objects.\n- Line 10: Get the `stream` of the context for serializing asynchronous executions.\n- Lines 11-13: Convenience reference variables for later use. We name `out_grad`\nas the gradient of the operator output, `in_data` as the input of the operator,\nand `in_grad` as the gradient of the operator input.\n- Line 14: Get the parameter object of `QuadraticParam`.\n- Lines 16-22: Same as in the forward function, this is where parallel\ncomputation for `in_grad` happens. The struct `quadratic_backward` implements\nthe formula of calculating each element of `in_grad` by one thread as the following.\n\n```cpp\ntemplate<int req>\nstruct quadratic_backward {\n  template<typename DType>\n  MSHADOW_XINLINE static void Map(int i, DType* in_grad, const DType* out_grad,\n                                  const DType* in_data, const float a, const float b) {\n    KERNEL_ASSIGN(in_grad[i], req, out_grad[i] * (2 * a * in_data[i] + b));\n  }\n};\n```\n\n### Operator Registration\nSo far, we have implemented necessary data structure and functions for the operator `quadratic`.\nNow let's register them using `nnvm` to expose the operator `quadratic`\nto frontend. Users can consider the registration process as creating the operator object\ninstance, saving it in the operator manager (a singleton),\nand setting attributes for the operator instance.\n\nThe following code is from `quadratic_op.cc`, which is responsible\nfor registering the operator working on CPU.\n\n{% raw %}\n\n```cpp\nDMLC_REGISTER_PARAMETER(QuadraticParam);                                           // 1\n\nNNVM_REGISTER_OP(quadratic)                                                        // 2\n.describe(R\"code(This operators implements the quadratic function:                 // 3\n.. math::\n\n    f(x) = ax^2+bx+c\n\nwhere :math:`x` is an input tensor and all operations\nin the function are element-wise.\n\nExample:\n\n  .. code-block:: python\n     :emphasize-lines: 1,3\n     x = [[1, 2], [3, 4]]\n     y = quadratic(data=x, a=1, b=2, c=3)\n     y = [[6, 11], [18, 27]]\n\n)code\" ADD_FILELINE)                                                               // 4\n.set_attr_parser(ParamParser<QuadraticParam>)                                      // 5\n.set_num_inputs(1)                                                                 // 6\n.set_num_outputs(1)                                                                // 7\n.set_attr<nnvm::FListInputNames>(\"FListInputNames\",                                // 8\n  [](const NodeAttrs& attrs) {                                                     // 9\n    return std::vector<std::string>{\"data\"};                                       // 10\n  })                                                                               // 11\n.set_attr<nnvm::FInferShape>(\"FInferShape\", QuadraticOpShape)                      // 12\n.set_attr<nnvm::FInferType>(\"FInferType\", QuadraticOpType)                         // 13\n.set_attr<FCompute>(\"FCompute<cpu>\", QuadraticOpForward<cpu>)                      // 14\n.set_attr<nnvm::FGradient>(\"FGradient\", ElemwiseGradUseIn{\"_backward_quadratic\"})  // 15\n.set_attr<nnvm::FInplaceOption>(\"FInplaceOption\",                                  // 16\n  [](const NodeAttrs& attrs) {                                                     // 17\n    return std::vector<std::pair<int, int> >{{0, 0}};                              // 18\n  })                                                                               // 19\n.add_argument(\"data\", \"NDArray-or-Symbol\", \"Input ndarray\")                        // 20\n.add_arguments(QuadraticParam::__FIELDS__());                                      // 21\n\nNNVM_REGISTER_OP(_backward_quadratic)                                              // 22\n.set_attr_parser(ParamParser<QuadraticParam>)                                      // 23\n.set_num_inputs(2)                                                                 // 24\n.set_num_outputs(1)                                                                // 25\n.set_attr<nnvm::TIsBackward>(\"TIsBackward\", true)                                  // 26\n.set_attr<FCompute>(\"FCompute<cpu>\", QuadraticOpBackward<cpu>);                    // 27\n```\n\n{% endraw %}\n\n- Line 1: Register the parameter struct.\n- Line 2: Register an operator named `quadratic` by creating an instance\nof `Op` type and save it in the operator manager and return a reference\nof the just created operator object.\n- Lines 3-4: Add description as an operator attribute\nincluding examples of the operator. The documentation engine will extract\nthis description and display it on the documentation web page.\n`emphasize-lines` is optional.\nFor more examples and troubleshooting with doc strings, refer to the [MXNet\ndeveloper wiki's Documentation Guide](https://cwiki.apache.org/confluence/display/MXNET/Documentation+Guide).\n- Line 5: Set parameter struct parser for the operator. It is used for parsing\nthe parameters `a`, `b`, and `c` input from frontend.\n- Line 6: Set the number of inputs for the operator.\n- Line 7: Set the number of outputs for the operator.\n- Lines 8-11: Defines a function generating a vector of names of\nthe operator input arguments. This function is used to add missing\narguments that users did not specify when creating a symbolic operator.\nFor example, `quad_func=mx.sym.quadratic()` is still a valid symbol\nsince we have added the attribute `FListInputNames` to the operator node\nin the computational graph. MXNet would\nadd the missing argument with name `quadratic0_data`, where the prefix\n`quadratic0` is the operator name appended with an index and the postfix\n`data` comes from the return value of the user defined `FListInputName` function.\nUsers still can generate an executor for the `quad_func` like the following:\n```python\nquad_exe = quad_func.simple_bind(ctx=mx.cpu(), quadratic0_data=(1,))\n```\n- Line 12: Register shape inference function.\n- Line 13: Register type inference function.\n- Line 14: Register forward function.\n- Line 15: Register the function for creating the node of the operator in\na backward pass. Note that we used a convenience functor struct `ElemwiseGradUseIn`.\nAs you can tell from the name, the registered functor creates the node for gradient computation\nwith dependencies on the output gradient node and input node. Similarly, there are\nother three functors defined as `ElemwiseGradUseOut`, `ElemwiseGradUseInOut`,\nand `ElemwiseGradUseNone` for developers' convenience. In order to add\nthis attribute, we also need to register a backward operator for `quadratic` with\nseveral basic attributes, as it can share attribute inference\nfunctions with the forward operator and is not exposed to frontend.\n- Lines 16-19: This registered function implies that which output tensor can reuse\nwhich input tensor's memory space instead of allocating a new memory space for the output.\nIn the operator `quadratic`, there is only one input and output, and the output can reuse\nthe input memory space, so we store a pair of zeros in the function return vector\nindicating that `inputs[0]`'s memory space can be reused by `outputs[0]`.\nNote that this function just provides a hint to the computational graph initializer.\nIf there are other nodes depending on the input tensor, the memory space\nof the input tensor will not be overwritten by the output.\n- Line 20: Define the input argument name as `data` for the operator.\n- Line 21: Add user input parameters `a`, `b`, and `c` as the attributes of the operator.\n- Line 22: Register an operator named `_backward_quadratic` for backward pass\nof the operator `quadratic`. The underscore prefix in the operator name indicates\nthat this is an operator not exposed to users. The convention\nof naming an internally used backward operator is prepending the prefix `_backward_`\nto the corresponding forward operator name.\n- Line 23: Set the parameter parser for the operator `_backward_quadratic`.\n- Line 24: Set the number of inputs.\n- Line 25: Set the number of outputs.\n- Line 26: Add `TIsBackward` attribute for the operator. The shape and type\ninference passes use this attribute to determine whether a node in the graph is a\nforward or backward node.\n- Line 27: Register backward function.\n\nSo far, we have acquired an operator working on CPU in frontend.\nIn order to register the operator working on GPUs, we just need to add the following\ncode to `quadratic_op.cu`. Note that forward and backward functions\nare registered with attribute key `FCompute<gpu>`, rather than `FCompute<cpu>`.\n\n```cpp\nNNVM_REGISTER_OP(quadratic)\n.set_attr<FCompute>(\"FCompute<gpu>\", QuadraticOpForward<gpu>);\n\nNNVM_REGISTER_OP(_backward_quadratic)\n.set_attr<FCompute>(\"FCompute<gpu>\", QuadraticOpBackward<gpu>);\n```\n\n### Unit Test\nNow we have finished implementing the operator `quadratic` in MXNet backend.\nIf you use python, when you type `import mxnet as mx`, two python\nfunctions for invoking your backend implementation are\ngenerated on the fly: one is for imperative programming\nregistered as `mxnet.ndarray.quadratic` or `mx.nd.quadratic` for short;\nthe other one is for symbolic programming registered under\nmodule `mxnet.symbol.quadratic` or `mx.sym.quadratic` for short.\n\nIn order to unit test it in frontend, we need to add the following code\nto the python file `test_operator.py`. A typical operator implementation\ntests for both the `symbol` API and the `ndarray` API. The following test\nhas both these tests. The imperative API test, tests for the `ndarray` API,\n`mx.nd.contrib.quadratic`. The `symbol` API test, tests for the complete\nfunctionality of the operator - the forward pass and the backward\npass. To facilitate the testing of these functionalities we use three\nhelper functions available in the `mxnet.test_utils` module:\n - `check_symbolic_forward`\n - `check_symbolic_backward`\n - `check_numeric_gradient`\n\n```python\ndef test_quadratic_function():\n    def f(x, a, b, c):\n        return a * x**2 + b * x + c\n\n    a = np.random.random_sample()\n    b = np.random.random_sample()\n    c = np.random.random_sample()\n    data = mx.symbol.Variable('data')\n    quad_sym = mx.sym.contrib.quadratic(data=data, a=a, b=b, c=c)\n    for dtype in [np.float16, np.float32, np.float64]:\n        for ndim in range(1, 6):\n            shape = rand_shape_nd(ndim, 5)\n            data_np = np.random.randn(*shape).astype(dtype)\n            expected = f(data_np, a, b, c)\n            backward_expected = 2 * a * data_np + b\n\n            # check imperative forward\n            output = mx.nd.contrib.quadratic(mx.nd.array(data_np), a=a, b=b, c=c)\n            assert_almost_equal(output.asnumpy(),expected,\n                                rtol=1e-2 if dtype is np.float16 else 1e-5,\n                                atol=1e-2 if dtype is np.float16 else 1e-5)\n            # check forward\n            check_symbolic_forward(quad_sym, [data_np], [expected],\n                                    rtol=1e-2 if dtype is np.float16 else 1e-5,\n                                    atol=1e-2 if dtype is np.float16 else 1e-5)\n            # check backward\n            check_symbolic_backward(quad_sym, [data_np], [np.ones(expected.shape)],\n                                        [backward_expected],\n                                        rtol=1e-2 if dtype is np.float16 else 1e-5,\n                                        atol=1e-2 if dtype is np.float16 else 1e-5)\n            # check backward using finite difference\n            check_numeric_gradient(quad_sym, [data_np], atol=0.001)\n```\n\nIn the above test we create a `quadratic` symbol and feed it into the three\nutility functions. The `check_symbolic_forward` and `check_symbolic_backward`\ntests the computed values against the expected values that we pass\nas an argument to the function. The `check_numeric_gradient` utility function\nperforms [gradient checking](http://ufldl.stanford.edu/tutorial/supervised/DebuggingGradientChecking/)\nto verify the implementation for the backward function of the operator.\nIt will perform a perturbation on the input and calculate the response\nrate of the output using the\n[finite difference method](https://en.wikipedia.org/wiki/Finite_difference_method).\nThen it will compare the gradient from the backward pass with the values\nfrom the finite difference method. All three of these tests will be successful\nonce the comparison satisfies user specified `rtol` and `atol` values. Here `rtol`\nand `atol` expand to relative tolerance and absolute tolerance respectively. They\nare used to specify how far the computed values can deviate from the expected values.\nThey are defined as follows\n\n```\nabs(Expected_Value - Computed_Value) < RTOL * abs(Expected_Value) + ATOL\n```\n\nFor example, if `rtol` is `1e-5` and `atol` is `1e-5` and the expected value is\n`1.5623145`, then the computed value should lie within the range of\n`(1.562288876855, 1.562340123145)` else the test will fail. Make sure you\ntune the `rtol` and `atol` values accordingly. Giving very low values for `rtol`\nand `atol` will likely make the test very flaky. It is recommended that you\nuse the flakiness checker tool to check if the test you have written is flaky\nor not. You can run the flakiness checker tool for the above test with the\nfollowing command -\n\n```bash\npython tools/flakiness_checker.py test_operator.test_quadratic_function\n```\n\nPlease note that for `check_symbolic_forward` and `check_symbolic_backward` we pass\nboth the operator symbols and expected results for comparison, for\n`check_numeric_gradient` we only pass the operator symbol, as the\n`check_numeric_gradient` computes the expected value using finite difference\nmethod. Which is why it is highly recommended to add `check_numeric_gradient`\ntest for every operator with backward function implemented as it eliminates\nthe possibility of passing incorrect expected results into `check_symbolic_backward`.\n\n\n## Summary\nIn this tutorial, we practiced implementing the operator `quadratic` in MXNet backend\nand unit testing the implementation in frontend. More specifically, we added parameter\nstruct for user-input parameters, walked through shape and type inference workflow,\nimplemented forward and backward functions, and registered the operator\nusing nnvm. Congratulations! You now know how to add operators.\nWe welcome your contributions to MXNet.\n\n**Note**: Source code in the tutorial can be found in\n[quadratic_op-inl.h](https://github.com/apache/mxnet/blob/master/src/operator/contrib/quadratic_op-inl.h),\n[quadratic_op.cc](https://github.com/apache/mxnet/blob/master/src/operator/contrib/quadratic_op.cc),\n[quadratic_op.cu](https://github.com/apache/mxnet/blob/master/src/operator/contrib/quadratic_op.cu),\nand\n[test_operator.py](https://github.com/apache/mxnet/blob/master/tests/python/unittest/test_operator.py#L6514).\n\n## Additional Resources\n- [Use TensorInspector to Help Debug Operators](./tensor_inspector_tutorial)\n- [Use RTC to write CUDA kernels](./using_rtc)\n"
  },
  {
    "path": "docs/static_site/src/pages/api/faq/cloud.md",
    "content": "---\r\nlayout: page_category\r\ntitle:  MXNet on the Cloud\r\ncategory: faq\r\nfaq_c: Deployment Environments\r\nquestion: How to run MXNet on AWS?\r\npermalink: /api/faq/cloud\r\n---\r\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\r\n<!--- or more contributor license agreements.  See the NOTICE file -->\r\n<!--- distributed with this work for additional information -->\r\n<!--- regarding copyright ownership.  The ASF licenses this file -->\r\n<!--- to you under the Apache License, Version 2.0 (the -->\r\n<!--- \"License\"); you may not use this file except in compliance -->\r\n<!--- with the License.  You may obtain a copy of the License at -->\r\n\r\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\r\n\r\n<!--- Unless required by applicable law or agreed to in writing, -->\r\n<!--- software distributed under the License is distributed on an -->\r\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\r\n<!--- KIND, either express or implied.  See the License for the -->\r\n<!--- specific language governing permissions and limitations -->\r\n<!--- under the License. -->\r\n\r\n# MXNet on the Cloud\r\n\r\nDeep learning can require extremely powerful hardware, often for unpredictable durations of time.\r\nMoreover, _MXNet_ can benefit from both multiple GPUs and multiple machines.\r\nAccordingly, cloud computing, as offered by AWS and others,\r\nis especially well suited to training deep learning models.\r\nUsing AWS, we can rapidly fire up multiple machines with multiple GPUs each at will\r\nand maintain the resources for precisely the amount of time needed.\r\n\r\n## Set Up an AWS GPU Cluster from Scratch\r\n\r\nIn this document, we provide a step-by-step guide that will teach you\r\nhow to set up an AWS cluster with _MXNet_. We show how to:\r\n\r\n- [Use Pre-installed EC2 GPU Instance](#use-pre-installed-ec2-gpu-instance)\r\n- [Build and run MXNet on a single computer](#build-and-run-mxnet-on-a-gpu-instance)\r\n- [Set up an EC2 GPU cluster for distributed training](#set-up-an-ec2-gpu-cluster-for-distributed-training)\r\n\r\n### Use Pre-installed EC2 GPU Instance\r\nThe [Deep Learning AMIs](https://aws.amazon.com/marketplace/search/results?x=0&y=0&searchTerms=Deep+Learning+AMI)\r\nare a series of images supported and maintained by Amazon Web Services for use\r\non Amazon Elastic Compute Cloud (Amazon EC2) and contain the latest MXNet release.\r\n\r\nNow you can launch _MXNet_ directly on an EC2 GPU instance.\r\nYou can also use [Jupyter](https://jupyter.org) notebook on EC2 machine.\r\nHere is a [good tutorial](https://github.com/dmlc/mxnet-notebooks)\r\non how to connect to a Jupyter notebook running on an EC2 instance.\r\n\r\n### Set Up an EC2 GPU Instance from Scratch\r\n\r\n[Deep Learning Base AMIs](https://aws.amazon.com/marketplace/search/results?x=0&y=0&searchTerms=Deep+Learning+Base+AMI)\r\nprovide a foundational image with NVIDIA CUDA, cuDNN, GPU drivers, oneDNN,\r\nDocker and Nvidia-Docker, etc. for deploying your own custom deep\r\nlearning environment. You may follow the [MXNet Build From Source\r\ninstructions](https://mxnet.apache.org/get_started/build_from_source) easily on\r\nthe Deep Learning Base AMIs.\r\n\r\n### Set Up an EC2 GPU Cluster for Distributed Training\r\n\r\nA cluster consists of multiple computers.\r\nYou can use one computer with _MXNet_ installed as the root computer for submitting jobs,and then launch several\r\nslave computers to run the jobs. For example, launch multiple instances using an\r\nAMI with dependencies installed. There are two options:\r\n\r\n- Make all slaves' ports accessible (same for the root) by setting type: All TCP,\r\n   Source: Anywhere in Configure Security Group.\r\n\r\n- Use the same `pem` as the root computer to access all slave computers, and\r\n   then copy the `pem` file into the root computer's `~/.ssh/id_rsa`. If you do this, all slave computers can be accessed with SSH from the root.\r\n\r\nNow, run the CNN on multiple computers. Assume that we are on a working\r\ndirectory of the root computer, such as `~/train`, and MXNet is built as `~/mxnet`.\r\n\r\n1. Pack the _MXNet_ Python library into this working directory for easy\r\n  synchronization:\r\n\r\n  ```bash\r\n  cp -r ~/mxnet/python/mxnet .\r\n  cp ~/mxnet/lib/libmxnet.so mxnet/\r\n  ```\r\n\r\n  And then copy the training program:\r\n\r\n  ```bash\r\n  cp ~/mxnet/example/image-classification/*.py .\r\n  cp -r ~/mxnet/example/image-classification/common .\r\n  ```\r\n\r\n2. Prepare a host file with all slaves private IPs. For example, `cat hosts`:\r\n\r\n  ```bash\r\n  172.30.0.172\r\n  172.30.0.171\r\n  ```\r\n\r\n3. Assuming that there are two computers, train the CNN using two workers:\r\n\r\n  ```bash\r\n  ../../tools/launch.py -n 2 -H hosts --sync-dir /tmp/mxnet python train_mnist.py --kv-store dist_sync\r\n  ```\r\n\r\n***Note:*** Sometimes the jobs linger at the slave computers even though you've pressed `Ctrl-c`\r\nat the root node. To terminate them, use the following command:\r\n\r\n```bash\r\ncat hosts | xargs -I{} ssh -o StrictHostKeyChecking=no {} 'uname -a; pgrep python | xargs kill -9'\r\n```\r\n\r\n***Note:*** The preceding example is very simple to train and therefore isn't a good\r\nbenchmark for distributed training. Consider using other [examples](https://github.com/apache/mxnet/tree/master/example/image-classification).\r\n\r\n### More Options\r\n#### Use Multiple Data Shards\r\nIt is common to pack a dataset into multiple files, especially when working in a distributed environment.\r\n_MXNet_ supports direct loading from multiple data shards.\r\nPut all of the record files into a folder, and point the data path to the folder.\r\n\r\n#### Use YARN and SGE\r\nAlthough using SSH can be simple when you don't have a cluster scheduling framework,\r\n_MXNet_ is designed to be portable to various platforms.\r\nWe provide scripts available in [tracker](https://github.com/dmlc/dmlc-core/tree/master/tracker)\r\nto allow running on other cluster frameworks, including Hadoop (YARN) and SGE.\r\nWe welcome contributions from the community of examples of running _MXNet_ on your favorite distributed platform.\r\n"
  },
  {
    "path": "docs/static_site/src/pages/api/faq/distributed_training.md",
    "content": "---\nlayout: page_category\ntitle:  Distributed Training in MXNet\ncategory: faq\nfaq_c: Deployment Environments\nquestion: How to do distributed training using MXNet on AWS?\npermalink: /api/faq/distributed_training\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Distributed Training in MXNet\nMXNet supports distributed training enabling us to leverage multiple machines for faster training.\nIn this document, we describe how it works, how to launch a distributed training job and\nsome environment variables which provide more control.\n\n## Types of Parallelism\nThere are two ways in which we can distribute the workload of training a neural network across multiple devices (can be either GPU or CPU).\nThe first way is *data parallelism*, which refers to the case where each device stores a complete copy of the model.\nEach device works with a different part of the dataset, and the devices collectively update a shared model.\nThese devices can be located on a single machine or across multiple machines.\nIn this document, we describe how to train a model with devices distributed across machines in a data parallel way.\n\nWhen models are so large that they don't fit into device memory, then a second way called *model parallelism* is useful.\nHere, different devices are assigned the task of learning different parts of the model.\nCurrently, MXNet supports Model parallelism in a single machine only. Refer [Training with multiple GPUs using model parallelism](model_parallel_lstm) for more on this.\n\n## How Does Distributed Training Work?\nThe following concepts are key to understanding distributed training in MXNet:\n### Types of Processes\nMXNet has three types of processes which communicate with each other to accomplish training of a model.\n- Worker: A worker node actually performs training on a batch of training samples.\nBefore processing each batch, the workers pull weights from servers.\nThe workers also send gradients to the servers after each batch.\nDepending on the workload for training a model, it might not be a good idea to run multiple worker processes on the same machine.\n- Server: There can be multiple servers which store the model's parameters, and communicate with workers.\nA server may or may not be co-located with the worker processes.\n- Scheduler: There is only one scheduler. The role of the scheduler is to set up the cluster. This includes waiting for messages that each node has come up and which port the node is listening on.\nThe scheduler then lets all processes know about every other node in the cluster, so that they can communicate with each other.\n\n### KV Store\nMXNet provides a key-value store, which is a critical component used for multi-device training. The communication of parameters across devices on a single machine, as well as across multiple machines, is relayed through one or more servers with a key-value store for the parameters. Each value in this store is represented by a key and value, where each parameter array in the network is assigned a key, and value refers to the weights of that parameter array. Workers `push` gradients after processing a batch, and `pull` updated weights before processing a new batch.\nWe can also pass in optimizers for the KVStore to use while updating each weight. Optimizers like Stochastic Gradient Descent define an update rule,\nessentially a mathematical formula to compute the new weight based on the old weight, gradient, and some parameters.\n\nIf you are using a Gluon Trainer object or the Module API,\nit uses a kvstore object internally to aggregate gradients from multiple devices on the same machine as well as across different machines.\n\nAlthough the API remains the same whether or not multiple machines are being used,\nthe notion of kvstore server exists only during distributed training.\nIn this case, each `push` and `pull` involves communication with the kvstore servers. When there are multiple devices on a single machine, gradients from these devices are first aggregated on the machine and then sent to the servers.\nNote that we need to compile MXNet with the build flag `USE_DIST_KVSTORE=1` to use distributed training.\n\nThe distributed mode of KVStore is enabled by calling `mxnet.kvstore.create` function\nwith a string argument which contains the word `dist` as follows:\n> kv = mxnet.kvstore.create('dist_sync')\n\nRefer [KVStore API]({{'/api/python/docs/api/kvstore/index.html#mxnet.kvstore.KVStore'|relative_url}}) for more information about KVStore.\n\n### Distribution of Keys\nEach server doesn't necessarily store all the keys or parameter arrays.\nParameters are distributed across different servers. The decision of which server stores a particular key is made at random.\nThis distribution of keys across different servers is handled transparently by the KVStore.\nIt ensures that when a key is pulled, that request is sent to the server which has the corresponding value.\nIf the value of some key is very large, it may be sharded across different servers. This means that different servers hold different parts of the value.\nAgain, this is handled transparently so that the worker does not have to do anything different.\nThe threshold for this sharding can be controlled with the environment variable `MXNET_KVSTORE_BIGARRAY_BOUND`.\nSee [environment variables](#environment-variables) for more details.\n\n### Split training data\nWhen running distributed training in data parallel mode, we want each machine to be working on different parts of the dataset.\n\nFor data parallel training on a single worker,\nwe can use `mxnet.gluon.utils.split_and_load` to split a batch of samples provided by the data iterator, and then load each part of the batch on the device which will process it.\n\nIn the case of distributed training though, we would need to divide the dataset into `n` parts at the beginning, so that each worker gets a different part. Each worker can then use `split_and_load` to again divide that part of the dataset across different devices on a single machine.\n\nTypically, this split of data for each worker happens through the data iterator,\non passing the number of parts and the index of parts to iterate over.\nSome iterators in MXNet that support this feature are [mxnet.io.MNISTIterator](/api/python/docs/api/mxnet/io/index.html?MNISTIter#mxnet.io.MNISTIter) and [mxnet.io.ImageRecordIter](api/python/docs/api/mxnet/io/index.html?imagerecorditer#mxnet.io.ImageRecordIter).\nIf you are using a different iterator, you can look at how the above iterators implement this.\nWe can use the kvstore object to get the number of workers (`kv.num_workers`) and rank of the current worker (`kv.rank`).\nThese can be passed as arguments to the iterator.\nYou can look at [example/gluon/image_classification.py](https://github.com/apache/mxnet/blob/master/example/gluon/image_classification.py)\nto see an example usage.\n\n### Updating weights\nKVStore server supports two modes, one which aggregates the gradients and updates the weights using those gradients, and second where the server only aggregates gradients. In the latter case, when a worker process pulls from kvstore, it gets the aggregated gradients. The worker then uses these gradients and applies the weights locally.\n\nWhen using Gluon there is an option to choose between these modes by passing `update_on_kvstore` variable when you create the [Trainer](/api/python/docs/api/gluon/trainer.html) object like this:\n\n```\ntrainer = gluon.Trainer(net.collect_params(), optimizer='sgd',\n                        optimizer_params={'learning_rate': opt.lr,\n                                          'wd': opt.wd,\n                                          'momentum': opt.momentum,\n                                          'multi_precision': True},\n                        kvstore=kv,\n                        update_on_kvstore=True)\n```\n\nWhen using the symbolic interface, it performs the weight updates on the server without the user having to do anything special.\n\n### Different Modes of Distributed Training\nDistributed training itself is enabled when kvstore creation string contains the word `dist`.\n\nDifferent modes of distributed training can be enabled by using different types of kvstore.\n\n- `dist_sync`: In synchronous distributed training, all workers use the same synchronized set of model parameters at the start of every batch.\nThis means that after each batch, the server waits to receive gradients from each worker before it updates the model parameters.\nThis synchronization comes at a cost because the worker pulling parameters would have to wait till the server finishes this process.\nIn this mode, if a worker crashes, then it halts the progress of all workers.\n\n- `dist_async`: In asynchronous distributed training, the server receives gradients from one worker and immediately updates its store, which it uses to respond to any future pulls.\nThis means that a worker who finishes processing a batch can pull the current parameters from server and start the next batch,\neven if other workers haven't finished processing the earlier batch.\nThis is faster than `dist_sync` because there is no cost of synchronization, but can take more epochs to converge.\nThe update of weights is atomic, meaning no two updates happen on the same weight at the same time. However, the order  of updates is not guaranteed.\nIn `async` mode, it is required to pass an optimizer because in the absence of an optimizer kvstore would replace the stored weights with received weights and this doesn't make sense for training in asynchronous mode. Hence, when using Gluon with `async` mode we need to set `update_on_kvstore` to `True`.\n\n- `dist_sync_device`: Same as `dist_sync` except that when there are multiple GPUs being used on each node,\nthis mode aggregates gradients and updates weights on GPU while dist_sync does so on CPU memory.\nThis is faster than `dist_sync` because it reduces expensive communication between GPU and CPU, but it increases memory usage on GPU.\n\n- `dist_async_device` : The analogue of `dist_sync_device` but in asynchronous mode.\n\n\n### Gradient Compression\nWhen communication is expensive, and the ratio of computation time to communication time is low, communication can become a bottleneck.\nIn such cases, gradient compression can be used to reduce the cost of communication, thereby speeding up training.\nRefer [Gradient compression]({{'/api/faq/gradient_compression'|relative_url}}) for more details.\n\nNote: For small models when the cost of computation is much lower than cost of communication,\ndistributed training might actually be slower than training on a single machine because of the overhead of communication and synchronization.\n\n## How to Start Distributed Training?\nMXNet provides a script tools/launch.py to make it easy to launch a distributed training job. This supports various types of cluster resource managers like `ssh`, `mpirun`, `yarn` and `sge`.\nIf you already have one of these clusters setup, you can skip the next section on setting up a cluster.\nIf you want to use a type of cluster not mentioned above, skip ahead to Manually launching jobs section.\n\n### Setting up the Cluster\nAn easy way to set up a cluster of EC2 instances for distributed deep learning is by using the [AWS CloudFormation template](https://github.com/awslabs/deeplearning-cfn).\nIf you can not use the above, this section will help you manually set up a cluster of instances\nto enable you to use `ssh` for launching a distributed training job.\nLet us denote one machine as the `master` of the cluster through which we will launch and monitor the distributed training on all machines.\n\nIf the machines in your cluster are a part of a cloud computing platform like AWS EC2, then your instances should be using key-based authentication already.\nEnsure that you create all instances using the same key, say `mxnet-key` and in the same security group.\nNext, we need to ensure that master has access to all other machines in the cluster through `ssh` by\nadding this key to [ssh-agent](https://en.wikipedia.org/wiki/Ssh-agent) and forwarding it to master when we log in. This will make `mxnet-key` the default key on master.\n\n```\nssh-add .ssh/mxnet-key\nssh -A user@MASTER_IP_ADDRESS\n```\n\n\nIf your machines use passwords for authentication, see [here](https://help.ubuntu.com/community/SSH/OpenSSH/Keys) for instructions on setting up password-less authentication between machines.\n\n\nIt is easier if all these machines have a shared file system so that they can access the training script. One way is to use [Amazon Elastic File System](https://aws.amazon.com/efs) to create your network file system.\nThe options in the following command are the recommended options when mounting an AWS Elastic File System.\n\n```\nsudo mkdir efs && sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 NETWORK_FILE_SYSTEM_IP:/ efs\n```\n\nTip: You might find it helpful to store large datasets on S3 for easy access from all machines in the cluster. Refer [Using data from S3 for training]({{'/api/faq/s3_integration'|relative_url}}) for more information.\n\n### Using Launch.py\nMXNet provides a script [tools/launch.py](https://github.com/apache/mxnet/blob/master/tools/launch.py) to make it easy to launch distributed training on a cluster with `ssh`, `mpi`, `sge` or `yarn`.\nYou can fetch this script by cloning the mxnet repository.\n\n```\ngit clone --recursive https://github.com/apache/mxnet\n```\n\n#### Example\nLet us consider training a VGG11 model on the CIFAR10 dataset using [example/gluon/image_classification.py](https://github.com/apache/mxnet/blob/master/tools/launch.py).\n```\ncd example/gluon/\n```\nOn a single machine, we can run this script as follows:\n```\npython image_classification.py --dataset cifar10 --model vgg11 --epochs 1\n```\n\nFor distributed training of this example, we would do the following:\n\nIf the mxnet directory which contains the script `image_classification.py` is accessible to all machines in the cluster (for example if they are on a network file system), we can run:\n```\n../../tools/launch.py -n 3 -H hosts --launcher ssh python image_classification.py --dataset cifar10 --model vgg11 --epochs 1 --kvstore dist_sync\n```\n\nIf the directory with the script is not accessible from the other machines in the cluster, then we can synchronize the current directory to all machines.\n```\n../../tools/launch.py -n 3 -H hosts --launcher ssh --sync-dst-dir /tmp/mxnet_job/ python image_classification.py --dataset cifar10 --model vgg11 --epochs 1 --kvstore dist_sync\n```\n\n> Tip: If you don't have a cluster ready and still want to try this out, pass the option `--launcher local` instead of `ssh`\n\n#### Options\nHere, launch.py is used to submit the distributed training job. It takes the following options:\n- `-n` denotes the number of worker nodes to be launched.\n- `-s` denotes the number of server nodes to be launched.\nIf it is not specified, it is taken to be equal to the number of worker nodes.\nThe script tries to cycle through the hosts file to launch the servers and workers.\nFor example, if you have 5 hosts in the hosts file and you passed `n` as 3 (and nothing for `s`).\nThe script will launch a total of 3 server processes,\none each for the first three hosts and launch a total of 3 worker processes, one each for the fourth, fifth and first host.\nIf the hosts file has exactly `n` number of worker nodes, it will launch a server process and a worker process on each of the `n` hosts.\n- `--launcher` denotes the mode of communication. The options are:\n    - `ssh` if machines can communicate through ssh without passwords. This is the default launcher mode.\n    - `mpi` if Open MPI is available\n    - `sge` for Sun Grid Engine\n    - `yarn` for Apache Yarn\n    - `local` for launching all processes on the same local machine. This can be used for debugging purposes.\n- `-H` requires the path of the hosts file\n  This file contains IPs of the machines in the cluster. These machines should be able to communicate with each other without using passwords.\n  This file is only applicable and required when the launcher mode is `ssh` or `mpi`.\n  An example of the contents of the hosts file would be:\n  ```\n  172.30.0.172\n  172.31.0.173\n  172.30.1.174\n  ```\n- `--sync-dst-dir` takes the path of a directory on all hosts to which the current working directory will be synchronized. This only supports `ssh` launcher mode.\nThis is necessary when the working directory is not accessible to all machines in the cluster. Setting this option synchronizes the current directory using rsync before the job is launched.\nIf you have not installed MXNet system-wide\nthen you have to copy the folder `python/mxnet` and the file `lib/libmxnet.so` into the current directory before running `launch.py`.\nFor example if you are in `example/gluon`, you can do this with `cp -r ../../python/mxnet ../../lib/libmxnet.so .`. This would work if your `lib` folder contains `libmxnet.so`, as would be the case when you use make. If you use CMake, this file would be in your `build` directory.\n\n- `python image_classification.py --dataset cifar10 --model vgg11 --epochs 1 --kvstore dist_sync`\nis the command for the training job on each machine. Note the use of `dist_sync` for the kvstore used in the script.\n\n#### Terminating Jobs\nIf the training job crashes due to an error or if we try to terminate the launch script while training is running,\njobs on all machines might not have terminated. In such a case, we would need to terminate them manually.\nIf we are using `ssh` launcher, this can be done by running the following command where `hosts` is the path of the hostfile.\n```\nwhile read -u 10 host; do ssh -o \"StrictHostKeyChecking no\" $host \"pkill -f python\" ; done 10<hosts\n```\n\n### Manually Launching Jobs\nIf for some reason, you do not want to use the script above to start distributed training, then this section will be helpful.\nMXNet uses environment variables to assign roles to different processes and to let different processes find the scheduler.\nThe environment variables are required to be set correctly as follows for the training to start:\n- `DMLC_ROLE`: Specifies the role of the process. This can be `server`, `worker` or `scheduler`. Note that there should only be one `scheduler`.\nWhen `DMLC_ROLE` is set to `server` or `scheduler`, these processes start when mxnet is imported.\n- `DMLC_PS_ROOT_URI`: Specifies the IP of the scheduler\n- `DMLC_PS_ROOT_PORT`: Specifies the port that the scheduler listens to\n- `DMLC_NUM_SERVER`: Specifies how many server nodes are in the cluster\n- `DMLC_NUM_WORKER`: Specifies how many worker nodes are in the cluster\n\nBelow is an example to start all jobs locally on Linux or Mac. Note that starting all jobs on the same machine is not a good idea.\nThis is only to make the usage clear.\n\n```bash\nexport COMMAND='python example/gluon/image_classification.py --dataset cifar10 --model vgg11 --epochs 1 --kvstore dist_sync'\nDMLC_ROLE=server DMLC_PS_ROOT_URI=127.0.0.1 DMLC_PS_ROOT_PORT=9092 DMLC_NUM_SERVER=2 DMLC_NUM_WORKER=2 $COMMAND &\nDMLC_ROLE=server DMLC_PS_ROOT_URI=127.0.0.1 DMLC_PS_ROOT_PORT=9092 DMLC_NUM_SERVER=2 DMLC_NUM_WORKER=2 $COMMAND &\nDMLC_ROLE=scheduler DMLC_PS_ROOT_URI=127.0.0.1 DMLC_PS_ROOT_PORT=9092 DMLC_NUM_SERVER=2 DMLC_NUM_WORKER=2 $COMMAND &\nDMLC_ROLE=worker DMLC_PS_ROOT_URI=127.0.0.1 DMLC_PS_ROOT_PORT=9092 DMLC_NUM_SERVER=2 DMLC_NUM_WORKER=2 $COMMAND &\nDMLC_ROLE=worker DMLC_PS_ROOT_URI=127.0.0.1 DMLC_PS_ROOT_PORT=9092 DMLC_NUM_SERVER=2 DMLC_NUM_WORKER=2 $COMMAND\n```\n\nFor an in-depth discussion of how the scheduler sets up the cluster, you can go [here](https://blog.kovalevskyi.com/mxnet-distributed-training-explained-in-depth-part-1-b90c84bda725).\n\n## Environment Variables\n### For tuning performance\n - `MXNET_KVSTORE_REDUCTION_NTHREADS`\n  Value type: Integer\n  Default value: 4\n  The number of CPU threads used for summing up big arrays on a single machine\n  This will also be used for `dist_sync` kvstore to sum up arrays from different contexts on a single machine.\n  This does not affect summing up of arrays from different machines on servers.\n  Summing up of arrays for `dist_sync_device` kvstore is also unaffected as that happens on GPUs.\n\n- `MXNET_KVSTORE_BIGARRAY_BOUND`\n  Value type: Integer\n  Default value: 1000000\n  The minimum size of a *big array*.\n  When the array size is bigger than this threshold, `MXNET_KVSTORE_REDUCTION_NTHREADS` threads are used for reduction.\n  This parameter is also used as a load balancer in kvstore.\n  It controls when to partition a single weight to all the servers.\n  If the size of a single weight matrix is less than this bound, then it is sent to a single randomly picked server; otherwise, it is partitioned to all the servers.\n\n- `MXNET_ENABLE_GPU_P2P` GPU Peer-to-Peer communication\n  Value type: 0(false) or 1(true)\n  Default value: 1\n  If true, MXNet tries to use GPU peer-to-peer communication, if available on your device. This is used only when kvstore has the type `device` in it.\n\n### Communication\n- `DMLC_INTERFACE` Using a particular network interface\n  Value type: Name of interface\n  Example: `eth0`\n  MXNet often chooses the first available network interface.\n  But for machines with multiple interfaces, we can specify which network interface to use for data communication using this environment variable.\n\n- `PS_VERBOSE` Logging communication\n  Value type: 1 or 2\n  Default value: (empty)\n    - `PS_VERBOSE=1` logs connection information like the IPs and ports of all nodes\n    - `PS_VERBOSE=2` logs all data communication information\n\n\nWhen the network is unreliable, messages being sent from one node to another might get lost.\nThe training process can hang when a critical message is not successfully delivered.\nIn such cases, an additional ACK can be sent for each message to track its delivery.\nThis can be done by setting `PS_RESEND` and `PS_RESEND_TIMEOUT`\n- `PS_RESEND` Retransmission for unreliable network\nValue type: 0(false) or 1(true)\nDefault value: 0\nWhether or not to enable retransmission of messages\n- `PS_RESEND_TIMEOUT` Timeout for ACK to be received\nValue type: Integer (in milliseconds)\nDefault value: 1000\nIf ACK is not received in `PS_RESEND_TIMEOUT` milliseconds, then the message will be resent.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/faq/env_var.md",
    "content": "---\nlayout: page_category\ntitle:  Environment Variables\ncategory: faq\nfaq_c: Deployment Environments\nquestion: What are MXNet environment variables?\npermalink: /api/faq/env_var\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nEnvironment Variables\n=====================\nMXNet has several settings that you can change with environment variables.\nTypically, you wouldn't need to change these settings, but they are listed here for reference.\n\nFor example, you can set these environment variables in Linux or macOS as follows:\n```\nexport MXNET_GPU_WORKER_NTHREADS=3\n```\n\nOr in powershell:\n```\n$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0\n```\n\n## Variables controlling the execution environment\n\n* MXNET_LIBRARY_PATH\n    Absolute path indicating where the mxnet dynamic library is to be located, this would be the absolute\n    path to `libmxnet.so` or `libmxnet.dll` depending on the platform. The logic for loading the\n    library is in `python/mxnet/libinfo.py`\n\n## Set the Number of Threads\n\n* MXNET_GPU_WORKER_NTHREADS\n  - Values: Int ```(default=2)```\n  - The maximum number of threads to use on each GPU. This parameter is used to parallelize the computation within a single GPU card.\n* MXNET_GPU_COPY_NTHREADS\n  - Values: Int ```(default=2)```\n  - The maximum number of concurrent threads that do the memory copy job on each GPU.\n* MXNET_CPU_WORKER_NTHREADS\n  - Values: Int ```(default=1)```\n  - The maximum number of scheduling threads on CPU. It specifies how many operators can be run in parallel. Note that most CPU operators are parallelized by OpenMP. To change the number of threads used by individual operators, please set `OMP_NUM_THREADS` instead.\n* MXNET_CPU_PRIORITY_NTHREADS\n  - Values: Int ```(default=4)```\n  - The number of threads given to prioritized CPU jobs.\n* MXNET_MP_WORKER_NTHREADS\n  - Values: Int ```(default=1)```\n  - The number of scheduling threads on CPU given to multiprocess workers. Enlarge this number allows more operators to run in parallel in individual workers but please consider reducing the overall `num_workers` to avoid thread contention (not available on Windows).\n* MXNET_MP_OPENCV_NUM_THREADS\n  - Values: Int ```(default=0)```\n  - The number of OpenCV execution threads given to multiprocess workers. OpenCV multithreading is disabled if `MXNET_MP_OPENCV_NUM_THREADS` < 1 (default). Enlarge this number may boost the performance of individual workers when executing underlying OpenCV functions but please consider reducing the overall `num_workers` to avoid thread contention (not available on Windows).\n\n## Memory Options\n\n* MXNET_EXEC_ENABLE_INPLACE\n  - Values: true or false ```(default=true)```\n    - Whether to enable in-place optimization in symbolic execution. Checkout [in-place optimization]({{'/api/architecture/note_memory#in-place-operations'|relative_url}}) to know more about it.\n* NNVM_EXEC_MATCH_RANGE\n  - Values: Int ```(default=16)```\n  - The approximate matching scale in the symbolic execution memory allocator.\n  - Set this to 0 if you don't want to enable memory sharing between graph nodes(for debugging purposes).\n  - This variable has impact on the result of memory planning. So, MXNet sweep between [1, NNVM_EXEC_MATCH_RANGE], and selects the best value.\n* MXNET_EXEC_NUM_TEMP\n  - Values: Int ```(default=1)```\n  - The maximum number of temporary workspaces to allocate to each device. This controls space replicas and in turn reduces the memory usage.\n  - Setting this to a small number can save GPU memory. It will also likely decrease the level of parallelism, which is usually acceptable.\n    - MXNet internally uses graph coloring algorithm to [optimize memory consumption]({{'/api/architecture/note_memory'|relative_url}}).\n  - This parameter is also used to get number of matching colors in graph and in turn how much parallelism one can get in each GPU. Color based match usually costs more memory but also enables more parallelism.\n* MXNET_GPU_MEM_POOL_TYPE\n  - Values: String ```(default=Naive)```\n  - The type of GPU memory pool.\n  - Choices:\n    - *Naive*: A simple memory pool that allocates memory for the requested size and cache memory buffers, when this memory is released. The size of memory chunk is defined by rounding the requested memory size to the nearest bigger multiple of MXNET_GPU_MEM_POOL_PAGE_SIZE (or MXNET_GPU_MEM_LARGE_ALLOC_ROUND_SIZE, when the result of rounding for MXNET_GPU_MEM_POOL_PAGE_SIZE is bigger than MXNET_GPU_MEM_LARGE_ALLOC_ROUND_SIZE) and allocates memory of the rounded size.\n    - *Round*: A memory pool that try to rounds the requested memory size to the nearest bigger power of 2. When this rounded number is bigger that 2**MXNET_GPU_MEM_POOL_ROUND_LINEAR_CUTOFF, the *Naive* rounding algorithm is used. Caching and allocating buffered memory works in the same way as the naive memory pool.\n    - *Unpooled*: No memory pool is used.\n* MXNET_GPU_MEM_POOL_RESERVE\n  - Values: Int ```(default=5)```\n  - The percentage of GPU memory to reserve for things other than the GPU array, such as kernel launch or cudnn handle space.\n  - The value is used only by the GPU memory pool. If it is not possible to allocate new memory AND still save this reserve, the memory pool will free the cached memory.\n  - If you see a strange out-of-memory error from the kernel launch, after multiple iterations, try setting this to a larger value.\n* MXNET_GPU_MEM_LARGE_ALLOC_ROUND_SIZE\n  - Values: Int ```(default=2097152)```\n  - When the rounded size of memory allocations calculated by the pool of *Naive* type is larger than this threshold, it will be rounded up to a multiple of this value.\n  - The default was chosen to minimize global memory fragmentation within the GPU driver. Set this to 1 to disable.\n* MXNET_GPU_MEM_POOL_ROUND_LINEAR_CUTOFF\n  - Values: Int ```(default=24)```\n  - The cutoff threshold used by *Round* strategy. Let's denote the threshold as T. If the memory size is smaller than `2 ** T` (by default, it's 2 ** 24 = 16MB), it rounds to the smallest `2 ** n` that is larger than the requested memory size; if the memory size is larger than `2 ** T`, it rounds to the next k * 2 ** T.\n* MXNET_CPU_MEM_POOL_TYPE\n  - Values: String ```(default=Naive)```\n  - The type of CPU memory pool.\n  - Choices:\n    - *Naive*: A simple memory pool that allocates memory for the requested size and cache memory buffers, when this memory is released. The size of memory chunk is defined by rounding the requested memory size to the nearest bigger multiple of MXNET_CPU_MEM_POOL_PAGE_SIZE (or MXNET_CPU_MEM_LARGE_ALLOC_ROUND_SIZE, when the result of rounding for MXNET_CPU_MEM_POOL_PAGE_SIZE is bigger than MXNET_CPU_MEM_LARGE_ALLOC_ROUND_SIZE) and allocates memory of the rounded size.\n    - *Round*: A memory pool that try to rounds the requested memory size to the nearest bigger power of 2. When this rounded number is bigger that 2**MXNET_CPU_MEM_POOL_ROUND_LINEAR_CUTOFF, the the *Naive* rounding algorithm is used. Caching and allocating buffered memory works in the same way as the naive memory pool.\n    - *Unpooled*: No memory pool is used.\n* MXNET_CPU_MEM_POOL_RESERVE\n  - Values: Int ```(default=5)```\n  - The percentage of CPU memory to reserve for things other than the CPU array.\n  - The value is used only by the CPU memory pool. If it is not possible to allocate new memory AND still save this reserve, the memory pool will free the cached memory.\n  - If you see a strange out-of-memory error from the kernel launch, after multiple iterations, try setting this to a larger value.\n* MXNET_CPU_MEM_LARGE_ALLOC_ROUND_SIZE\n  - Values: Int ```(default=2097152)```\n  - When the rounded size of memory allocations calculated by the pool of *Naive* type is larger than this threshold, it will be rounded up to a multiple of this value.\n  - Set this to 1 to disable.\n* MXNET_CPU_MEM_POOL_ROUND_LINEAR_CUTOFF\n  - Values: Int ```(default=24)```\n  - The cutoff threshold used by *Round* strategy. Let's denote the threshold as T. If the memory size is smaller than `2 ** T` (by default, it's 2 ** 24 = 16MB), it rounds to the smallest `2 ** n` that is larger than the requested memory size; if the memory size is larger than `2 ** T`, it rounds to the next k * 2 ** T.\n* MXNET_CPU_PINNED_MEM_POOL_TYPE\n  - Values: String ```(default=Naive)```\n  - The type of CPU_PINNED memory pool.\n  - Choices:\n    - *Naive*: A simple memory pool that allocates memory for the requested size and cache memory buffers, when this memory is released. The size of memory chunk is defined by rounding the requested memory size to the nearest bigger multiple of MXNET_CPU_PINNED_MEM_POOL_PAGE_SIZE (or MXNET_CPU_PINNED_MEM_LARGE_ALLOC_ROUND_SIZE, when the result of rounding for MXNET_CPU_PINNED_MEM_POOL_PAGE_SIZE is bigger than MXNET_CPU_PINNED_MEM_LARGE_ALLOC_ROUND_SIZE) and allocates memory of the rounded size.\n    - *Round*: A memory pool that try to rounds the requested memory size to the nearest bigger power of 2. When this rounded number is bigger that 2**MXNET_CPU_PINNED_MEM_POOL_ROUND_LINEAR_CUTOFF, the the *Naive* rounding algorithm is used. Caching and allocating buffered memory works in the same way as the naive memory pool.\n    - *Unpooled*: No memory pool is used.\n* MXNET_CPU_PINNED_MEM_POOL_RESERVE\n  - Values: Int ```(default=5)```\n  - The percentage of GPU memory to reserve for things other than the GPU array.\n  - The value is used only by the CPU memory pool. If it is not possible to allocate new memory AND still save this reserve, the memory pool will free the cached memory.\n  - If you see a strange out-of-memory error from the kernel launch, after multiple iterations, try setting this to a larger value.\n* MXNET_CPU_PINNED_MEM_LARGE_ALLOC_ROUND_SIZE\n  - Values: Int ```(default=2097152)```\n  - When the rounded size of memory allocations calculated by the pool of *Naive* type is larger than this threshold, it will be rounded up to a multiple of this value.\n  - Set this to 1 to disable.\n* MXNET_CPU_PINNED_MEM_POOL_ROUND_LINEAR_CUTOFF\n  - Values: Int ```(default=24)```\n  - The cutoff threshold used by *Round* strategy. Let's denote the threshold as T. If the memory size is smaller than `2 ** T` (by default, it's 2 ** 24 = 16MB), it rounds to the smallest `2 ** n` that is larger than the requested memory size; if the memory size is larger than `2 ** T`, it rounds to the next k * 2 ** T.\n* MXNET_USE_NAIVE_STORAGE_MANAGERS\n  - Values: Int ```(default=0)```\n  - When value is not 0, no memory pools will be used for any of the following three types of memory: GPU, CPU, CPU_PINNED.\n   \n## Engine Type\n\n* MXNET_ENGINE_TYPE\n  - Values: String ```(default=ThreadedEnginePerDevice)```\n  - The type of underlying execution engine of MXNet.\n  - Choices:\n    - NaiveEngine: A very simple engine that uses the master thread to do the computation synchronously. Setting this engine disables multi-threading. You can use this type for debugging in case of any error. Backtrace will give you the series of calls that lead to the error. Remember to set MXNET_ENGINE_TYPE back to empty after debugging.\n    - ThreadedEngine: A threaded engine that uses a global thread pool to schedule jobs.\n    - ThreadedEnginePerDevice: A threaded engine that allocates thread per GPU and executes jobs asynchronously.\n\n## Execution Options\n\n* MXNET_EXEC_BULK_EXEC_INFERENCE\n  - Values: 0(false) or 1(true) ```(default=1)```\n  - If set to `1`, during inference MXNet executes the entire computation graph in bulk mode, which reduces kernel launch gaps in between symbolic operators.\n* MXNET_EXEC_BULK_EXEC_TRAIN\n  - Values: 0(false) or 1(true) ```(default=1)```\n  - If set to `1`, during training MXNet executes the computation graph as several subgraphs in bulk mode.\n* MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN\n  - Values: Int ```(default=15)```\n  - The maximum number of nodes in the subgraph executed in bulk during training (not inference). Setting this to a larger number may reduce the degree of parallelism for multi-GPU training.\n* MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_FWD\n  - Values: Int ```(default=<value of MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN>)```\n  - The maximum number of nodes in the subgraph executed in bulk during training (not inference) in the forward pass.\n* MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_BWD\n  - Values: Int ```(default=<value of MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN>)```\n  - The maximum number of nodes in the subgraph executed in bulk during training (not inference) in the backward pass.\n* MXNET_ENABLE_CUDA_GRAPHS\n  - Values: 0(false) or 1(true) ```(default=0)```\n  - If set to `1`, MXNet will utilize CUDA graphs when executing models on the GPU when possible.\n  - For CUDA graphs execution, one needs to use either symbolic model or Gluon model hybridized with options `static_alloc` and `static_shape` set to True.\n* MXNET_CUDA_GRAPHS_VERBOSE\n  - Values: 0(false) or  1(true) ```(default=0)```\n  - If set to `1`, CUDA graphs executor will provide information about the graph being captured and executed.\n* MXNET_CUDA_GRAPHS_MAX_LOG_ENTRIES\n  - Values: Int ```(default=0)```\n  - The maximum number of log messages generated by CUDA graphs executor.\n* MXNET_CUDA_GRAPHS_DBG_FILE\n  - Values: String ```(default='', to indicate no debug dot files should be created)```\n  - The file prefix for '.dot' files for each graph created.  Full path is <prefix>-devN-{trn,inf}.<graphId>.dot .\n* MXNET_CUDA_GRAPHS_DBG_FILE_FLAGS\n  - Values: Int ```(default=<most verbose setting- includes all info>)```\n  - A bitmask to enable various types of info in the debug '.dot' files.  See cudaGraphDebugDotFlags in the CUDA runtime API doc for details.\n\n## Control the Data Communication\n\n* MXNET_KVSTORE_REDUCTION_NTHREADS\n  - Values: Int ```(default=4)```\n  - The number of CPU threads used for summing up big arrays on a single machine\n  - This will also be used for `dist_sync` kvstore to sum up arrays from different contexts on a single machine.\n  - This does not affect summing up of arrays from different machines on servers.\n  - Summing up of arrays for `dist_sync_device` kvstore is also unaffected as that happens on GPUs.\n\n* MXNET_KVSTORE_BIGARRAY_BOUND\n  - Values: Int ```(default=1000000)```\n  - The minimum size of a \"big array\".\n  - When the array size is bigger than this threshold, MXNET_KVSTORE_REDUCTION_NTHREADS threads are used for reduction.\n  - This parameter is also used as a load balancer in kvstore. It controls when to partition a single weight to all the servers. If the size of a single weight is less than MXNET_KVSTORE_BIGARRAY_BOUND then, it is sent to a single randomly picked server otherwise it is partitioned to all the servers.\n\n* MXNET_KVSTORE_USETREE\n  - Values: 0(false) or 1(true) ```(default=0)```\n  - If true, MXNet tries to use tree reduction for Push and Pull communication.\n  - Otherwise, MXNet uses the default Push and Pull implementation.\n  - Tree reduction technology has been shown to be faster than the standard ```--kv-store device``` Push/Pull and ```--kv-store nccl``` Push/Pull for small batch sizes.\n\n* MXNET_KVSTORE_LOGTREE\n  - Values: 0(false) or 1(true) ```(default=0)```\n  - If true and MXNET_KVSTORE_USETREE is set to 1, MXNet will log the reduction trees that have been generated.\n\n* MXNET_KVSTORE_TREE_ARRAY_BOUND\n  - Values: Int ```(default=10000000)```\n  - The minimum size of a \"big array\".\n  - When the array size is bigger than this threshold and MXNET_KVSTORE_USETREE is set to 1, multiple trees are used to load balance the big gradient being communicated in order to better saturate link bandwidth.\n  - Note: This environmental variable only takes effect if Tree KVStore is being used (MXNET_KVSTORE_USETREE=1).\n\n* MXNET_KVSTORE_TREE_BACKTRACK\n  - Values: 0(false) or 1(true) ```(default=0)\n  - If true and MXNET_KVSTORE_USETREE is set to 1, MXNet tries to use backtracking to generate the trees required for tree reduction.\n  - If false and MXNET_KVSTORE_USETREE is set to 1, MXNet tries to use Kernighan-Lin heuristic to generate the trees required for tree reduction.\n\n* MXNET_KVSTORE_TREE_LINK_USAGE_PENALTY\n  - Values: Float ```(default=0.7)```\n  - The multiplicative penalty term to a link being used once.\n\n* MXNET_ENABLE_GPU_P2P\n  - Values: 0(false) or 1(true) ```(default=1)```\n  - If true, MXNet tries to use GPU peer-to-peer communication, if available on your device,\n    when kvstore's type is `device`.\n\n* MXNET_UPDATE_ON_KVSTORE\n  - Values: 0(false) or 1(true) ```(default=1)```\n  - If true, weight updates are performed during the communication step, if possible.\n\n* MXNET_KVSTORE_SLICE_THRESHOLD\n  - Values: Int ```(default=40000)```\n  - The maximum size of an NDArray slice in terms of number of parameters.\n  - This parameter is used to slice an NDArray before synchronizing through P3Store (dist_p3).\n\n## Memory Optimizations\n\n* MXNET_BACKWARD_DO_MIRROR\n  - Values: 0(false) or 1(true) ```(default=0)```\n  - MXNet uses mirroring concept to save memory. Normally backward pass needs some forward input and it is stored in memory but you can choose to release this saved input and recalculate it in backward pass when needed. This basically trades off the computation for memory consumption.\n  - This parameter decides whether to do `mirror` during training for saving device memory.\n  - When set to `1`, during forward propagation, graph executor will `mirror` some layer's feature map and drop others, but it will re-compute this dropped feature maps when needed.\n  - `MXNET_BACKWARD_DO_MIRROR=1` will save 30%~50% of device memory, but retains about 95% of running speed.\n  - One extension of `mirror` in MXNet is called [memonger technology](https://arxiv.org/abs/1604.06174), it will only use O(sqrt(N)) memory at 75% running speed. Checkout the code [here](https://github.com/dmlc/mxnet-memonger).\n\n* MXNET_MEMORY_OPT\n  - Values: 0(no optimizations) or 1(highest optimization level) ```(default=0)```\n  - If set to '1', various optimizations on memory consumption will be enabled.\n\n## Control the profiler\n\nThe following environments can be used to profile the application without changing code. Execution options may affect the granularity of profiling result. If you need profiling result of every operator, please set `MXNET_EXEC_BULK_EXEC_INFERENCE`, `MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN` and `MXNET_EXEC_BULK_EXEC_TRAIN` to 0.\n\n* MXNET_PROFILER_AUTOSTART\n  - Values: 0(false) or 1(true) ```(default=0)```\n  - Set to 1, MXNet starts the profiler automatically. The profiling result is stored into profile.json in the working directory.\n\n* MXNET_PROFILER_MODE\n  - Values: 0(false) to 15(profile everything) ```(default=13)```\n  - If set to '0', turns off all profiling.\n  - If set to '1', profiler records the events of symbolic operators.\n  - If set to '2', profiler records the events of imperative operators.\n  - If set to '4', profiler records the C API events.\n  - If set to '8', profiler records the events of memory (i.e. storage alloc and free calls).\n  - You need to sum the values above for a custom combination. For example, for symbolic and imperative operators, set ```MXNET_PROFILER_MODE=3```(2 + 1).\n  - If set to '15', profiler records all the above listed events (API, Memory, Symbolic, Imperative).\n\n## Interface between Python and the C API\n\n* MXNET_ENABLE_CYTHON\n  - Values: 0(false), 1(true) ```(default=1)```\n  - If set to 0, MXNet uses the ctypes to interface with the C API.\n  - If set to 1, MXNet tries to use the cython modules for the ndarray and symbol. If it fails, the ctypes is used or an error occurs depending on MXNET_ENFORCE_CYTHON.\n\n* MXNET_ENFORCE_CYTHON\n  - Values: 0(false) or 1(true) ```(default=0)```\n  - This has an effect only if MXNET_ENABLE_CYTHON is 1.\n  - If set to 0, MXNet fallbacks to the ctypes if importing the cython modules fails.\n  - If set to 1, MXNet raises an error if importing the cython modules fails.\n\nIf cython modules are used, `mx.nd._internal.NDArrayBase` must be `mxnet._cy3.ndarray.NDArrayBase` for python 3 or `mxnet._cy2.ndarray.NDArrayBase` for python 2.\nIf ctypes is used, it must be `mxnet._ctypes.ndarray.NDArrayBase`.\n\n## Logging\n\n* DMLC_LOG_STACK_TRACE_DEPTH\n  - Values: Int ```(default=0)```\n  - The depth of stack trace information to log when exception happens.\n\n## Other Environment Variables\n\n* MXNET_GPU_WORKER_NSTREAMS\n  - Values: 1, or 2 ```(default=1)```\n  - Determines the number of GPU streams available to operators for their functions.\n  - Setting this to 2 may yield a modest performance increase, since ops like the cuDNN convolution op can then calculate their data- and weight-gradients in parallel.\n  - Setting this to 2 may also increase a model's demand for GPU global memory.\n\n* MXNET_CUDNN_AUTOTUNE_DEFAULT\n  - Values: 0, 1, or 2 ```(default=1)```\n  - The default value of cudnn auto tuning for convolution layers.\n  - Value of 0 means there is no auto tuning to pick the convolution algo\n  - Performance tests are run to pick the convolution algo when value is 1 or 2\n  - Value of 1 chooses the best algo in a limited workspace\n  - Value of 2 chooses the fastest algo whose memory requirements may be larger than the default workspace threshold\n\n* MXNET_CUDNN_HEUR_MODE\n  - Values: 0 or 1 (available since cuDNN 8.1) ```(default=1 for cuDNN 8.1 and later, otherwise 0)```\n  - Choose cuDNN heuristics mode.\n  - If set to '0', use fast decision tree based method.\n  - If set to '1', use neural network based method. It generalizes better for unknown or uncommon models.\n\n* MXNET_CUDNN_ALGO_VERBOSE_LEVEL\n  - Values: 0, 1, or 2 ```(default=0)```\n  - The level of printed output describing the \"convolution engine\" configurations\n  - Value of 0 produces no output\n  - Value of 1 outputs for the chosen config the engine number (\"algo\"), additional parameters (\"knobs\") and numerical notes\n  - Value of 2 outputs the same info as with a '1' setting, but for all configs considered\n  The output can be used to develop engine config filtering strategies to modify model behaviors.\n  Numerical accuracy may be improved by filtering out configs shown with 'rp', 'w' or 'fft' (i.e. reduced precision, winograd, or fft).\n  The configs are output with their list-index, as suggested by cuDNN, and with the chosen config flagged with a '*'.\n  If autotuning is enabled (MXNET_CUDNN_AUTOTUNE_DEFAULT != 0), the measured kernel times will be reported.\n\n* MXNET_CUDA_ALLOW_TENSOR_CORE\n  - 0(false) or 1(true) ```(default=1)```\n  - If set to '0', disallows Tensor Core use in CUDA ops.\n  - If set to '1', allows Tensor Core use in CUDA ops.\n  - This variable can only be set once in a session.\n  - Also controls filtering cuDNN engines with CUDNN_NUMERICAL_NOTE_TENSOR_CORE.\n\n* MXNET_CUDA_TENSOR_OP_MATH_ALLOW_CONVERSION\n  - 0(false) or 1(true) ```(default=0)```\n  - If set to '0', disallows implicit type conversions to Float16 to use Tensor Cores\n  - If set to '1', allows CUDA ops like RNN and Convolution to use TensorCores even with Float32 input data by using implicit type casting to Float16. Only has an effect if `MXNET_CUDA_ALLOW_TENSOR_CORE` is `1`.\n  - Also controls filtering cuDNN engines with CUDNN_NUMERICAL_NOTE_DOWN_CONVERT_INPUTS (such engines are disallowed if set to 0).\n\n* MXNET_CUDNN_ALLOW_REDUCED_PRECISION_REDUCTION\n  - 0(false) or 1(true) ```(default=1)```\n  - If set to '0', disallows cuDNN engines with CUDNN_NUMERICAL_NOTE_REDUCED_PRECISION_REDUCTION.\n  - If set to '1', allows cuDNN engines with CUDNN_NUMERICAL_NOTE_REDUCED_PRECISION_REDUCTION.\n\n* MXNET_CUDNN_ALLOW_FFT\n  - 0(false) or 1(true) ```(default=1)```\n  - If set to '0', disallows cuDNN engines with CUDNN_NUMERICAL_NOTE_FFT.\n  - If set to '1', allows cuDNN engines with CUDNN_NUMERICAL_NOTE_FFT.\n\n* MXNET_CUDNN_ALLOW_WINOGRAD\n  - 0(false) or 1(true) ```(default=1)```\n  - If set to '0', disallows cuDNN engines with CUDNN_NUMERICAL_NOTE_WINOGRAD.\n  - If set to '1', allows cuDNN engines with CUDNN_NUMERICAL_NOTE_WINOGRAD.\n\n* MXNET_CUDNN_DISABLED_CONV_FWD_ENGINES\n  - Comma-separated list of cuDNN convolution forward engine numbers to disable.\n  - Normally should be left alone, unless you know what you're doing.\n\n* MXNET_CUDNN_DISABLED_CONV_DGRAD_ENGINES\n  - Comma-separated list of cuDNN convolution dgrad engine numbers to disable.\n  - Normally should be left alone, unless you know what you're doing.\n\n* MXNET_CUDNN_DISABLED_CONV_WGRAD_ENGINES\n  - Comma-separated list of cuDNN convolution wgrad engine numbers to disable.\n  - Normally should be left alone, unless you know what you're doing.\n\n* MXNET_CUDA_LIB_CHECKING\n  - 0(false) or 1(true) ```(default=1)```\n  - If set to '0', disallows various runtime checks of the cuda library version and associated warning messages.\n  - If set to '1', permits these checks (e.g. compile vs. link mismatch, old version no longer CI-tested)\n\n* MXNET_CUDNN_LIB_CHECKING\n  - 0(false) or 1(true) ```(default=1)```\n  - If set to '0', disallows various runtime checks of the cuDNN library version and associated warning messages.\n  - If set to '1', permits these checks (e.g. compile vs. link mismatch, old version no longer CI-tested)\n\n* MXNET_GLUON_REPO\n  - Values: String ```(default='https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/'```\n  - The repository url to be used for Gluon datasets and pre-trained models.\n\n* MXNET_HOME\n  - Data directory in the filesystem for storage, for example when downloading gluon models.\n  - Default in *nix is .mxnet APPDATA/mxnet in windows.\n\n* MXNET_ONEDNN_ENABLED\n  - Values: 0, 1 ```(default=1)```\n  - Flag to enable or disable oneDNN accelerator. On by default.\n  - Only applies to mxnet that has been compiled with oneDNN (```pip install mxnet``` or built from source with ```USE_ONEDNN=1```)\n\n* MXNET_ONEDNN_CACHE_NUM\n  - Values: Int ```(default=-1)```\n  - Flag to set num of elements that oneDNN cache can hold. Default is -1 which means cache size is unbounded. Should only be set if your model has variable input shapes, as cache size may grow unbounded. The number represents the number of items in the cache and is proportional to the number of layers that use oneDNN and different input shape.\n\n* MXNET_ONEDNN_FORCE_FC_AB_FORMAT\n  - Values: 0, 1 ```(default=0)```\n  - If set to true, FullyConnected will use only AB format for weights, thus MXNet won't use BRGEMM implementation of FC on machines with AVX512-VNNI support which requires special weights format.\n\n* MXNET_ENFORCE_DETERMINISM\n  - Values: 0(false) or 1(true) ```(default=0)```\n  - If set to true, MXNet will only use deterministic algorithms in forward and backward computation.\n  If no such algorithm exists given other constraints, MXNet will error out. This variable affects the choice\n  of CUDNN convolution algorithms. Please see [CUDNN developer guide](https://docs.nvidia.com/deeplearning/sdk/cudnn-developer-guide/index.html) for more details.\n  - Also controls filtering cuDNN engines with CUDNN_NUMERICAL_NOTE_NONDETERMINISTIC (such engines are disallowed if set to 1).\n\n* MXNET_CPU_PARALLEL_SIZE\n  - Values: Int ```(default=200000)```\n  - The minimum size to call parallel operations by OpenMP for CPU context.\n  - When the array size is bigger than or equal to this threshold, the operation implemented by OpenMP is executed with the Recommended OMP Thread Count.\n  - When the array size is less than this threshold, the operation is implemented naively in single thread.\n\n* MXNET_OPTIMIZER_AGGREGATION_SIZE\n  - Values: Int ```(default=4)```\n  - Maximum value is 60.\n  - This variable controls how many weights will be updated in a single call to optimizer (for optimizers that support aggregation, currently limited to SGD).\n\n* MXNET_CPU_TEMP_COPY\n  - Values: Int ```(default=4)```\n  - This variable controls how many temporary memory resources to create for all CPU context for use in operator.\n\n* MXNET_GPU_TEMP_COPY\n  - Values: Int ```(default=1)```\n  - This variable controls how many temporary memory resources to create for each GPU context for use in operator.\n\n* MXNET_CPU_PARALLEL_RAND_COPY\n  - Values: Int ```(default=1)```\n  - This variable controls how many parallel random number generator resources to create for all CPU context for use in operator.\n\n* MXNET_GPU_PARALLEL_RAND_COPY\n  - Values: Int ```(default=4)```\n  - This variable controls how many parallel random number generator resources to create for each GPU context for use in operator.\n\n* MXNET_GPU_CUDNN_DROPOUT_STATE_COPY\n  - Values: Int ```(default=4)```\n  - This variable controls how many CuDNN dropout state resources to create for each GPU context for use in operator.\n\n* MXNET_SAFE_ACCUMULATION\n  - Values: Values: 0(false) or 1(true) ```(default=1)```\n  - If this variable is set, the accumulation will enter the safe mode, meaning accumulation is done in a data type of higher precision than\n    the input data type, leading to more accurate accumulation results with a possible performance loss and backward compatibility loss.\n    For example, when the variable is set to 1(true), if the input data type is float16, then the accumulation will be done\n    with float32.\n  - Model accuracies do not necessarily improve with this environment variable turned on.\n\n* MXNET_USE_FUSION\n  - Values: 0(false) or 1(true) ```(default=1)```\n  - If this variable is set, MXNet will try fusing some of the operations (pointwise operations only for now).\n  - It works in Symbolic execution as well as in Gluon models hybridized with ```static_alloc=True``` option.\n  - Only applies to MXNet that has been compiled with CUDA (```pip install mxnet-cuXX``` or built from source with ```USE_CUDA=1```) and running on GPU.\n\n* MXNET_RTC_VERBOSE\n  - Values: 0(false) or 1(true) ```(default=0)```\n  - Only applies to MXNet that has been compiled with CUDA.\n  - If this variable is set, MXNet will print the code for operators compiled at runtime.\n\n* MXNET_ELIMINATE_COMMON_EXPR\n  - Values: 0(false) or 1(true) ```(default=1)```\n  - If this variable is set, MXNet will simplify the computation graph, eliminating duplicated operations on the same inputs.\n\n* MXNET_USE_ONEDNN_RNN\n  - Values: 0(false) or 1(true) ```(default=1)```\n  - This variable controls whether to use the oneDNN backend in fused RNN operator for CPU context. There are two fusion implementations of RNN operator in MXNet. The oneDNN implementation has a better performance than the naive one, but the latter is more stable in the backward operation currently.\n\n* MXNET_FC_TRUE_FP16\n  - Values: 0(false) or 1(true) ```(default=0)```\n  - If this variable is set to true, MXNet will perform fp16 accumulation when using cuBLAS and input datatype is set to float16. This could increase the speed of the computation, but might result in loss of accuracy. This makes this setting useful mainly for inference usecases.\n\n* MXNET_RNN_USE_WEIGHT_CACHE\n  - Values: 0(false) or 1(true) ```(default=0)```\n  - If this variable is set, MXNet will ignore the altering of the version of NDArray which is the input parameter of the RNN operator. In Gluon API, there is a `_rnn_param_concat` operator concatenating the weights and bias of RNN into a single parameter tensor that changes the version number. Since the values of the parameters are invariant in inference pass, the RNN operator could ignore the altering of the version to escape much overhead from re-initializing the parameters.\n\nSettings for Minimum Memory Usage\n---------------------------------\n- Make sure ```min(MXNET_EXEC_NUM_TEMP, MXNET_GPU_WORKER_NTHREADS) = 1```\n  - The default setting satisfies this.\n\nSettings for More GPU Parallelism\n---------------------------------\n- Set ```MXNET_GPU_WORKER_NTHREADS``` to a larger number (e.g., 2)\n  - To reduce memory usage, consider setting ```MXNET_EXEC_NUM_TEMP```.\n  - This might not speed things up, especially for image applications, because GPU is usually fully utilized even with serialized jobs.\n\nSettings for controlling OMP tuning\n---------------------------------\n- Set ```MXNET_USE_OPERATOR_TUNING=0``` to disable Operator tuning code which decides whether to use OMP or not for operator\n   - Values: String representation of MXNET_ENABLE_OPERATOR_TUNING environment variable\n   -            0=disable all\n   -            1=enable all\n   -            float32, float16, float32=list of types to enable, and disable those not listed\n   - refer : https://github.com/apache/mxnet/blob/master/src/operator/operator_tune-inl.h#L444\n\n- Set ```MXNET_USE_NUM_CORES_OPERATOR_TUNING``` to define num_cores to be used by operator tuning code.\n  - This reduces operator tuning overhead when there are multiple instances of mxnet running in the system and we know that\n    each mxnet will take only partial num_cores available with system.\n  - refer: https://github.com/apache/mxnet/pull/13602\n"
  },
  {
    "path": "docs/static_site/src/pages/api/faq/float16.md",
    "content": "---\nlayout: page_category\ntitle:  Float16\ncategory: faq\nfaq_c: Speed\nquestion: How do I use mixed precision (float16) with MXNet or Gluon? \npermalink: /api/faq/float16\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Mixed precision training using float16\n\nIn this tutorial we will walk through how one can train deep learning neural networks with mixed precision on supported hardware. We will first see how to use float16 (both with Gluon and Symbolic APIs) and then some techniques on achieving good performance and accuracy.\n\n## Background\n\nThe computational resources required for training deep neural networks have been lately increasing because of growing complexity and model size. Mixed precision training allows us to reduce the utilization of the resources by using lower precision arithmetic which is computationally less expensive and less costly in terms of space utilization. In this approach you can train using 16 bit floating point (half precision) while using 32 bit floating point (single precision) for output buffers of float16 computation. This allows one to achieve the same accuracy as training with single precision, while decreasing the required memory and training or inference time.\n\nThe float16 data type is a 16 bit floating point representation according to the [IEEE 754 standard](https://ieeexplore.ieee.org/document/4610935). It has a dynamic range where the precision can go from 0.0000000596046 (highest, for values closest to 0) to 32 (lowest, for values in the range 32768-65536). Despite the inherent reduced precision when compared to single precision float (float32), using float16 has many advantages. The most obvious advantages are that you can reduce the size of the model by half allowing the training of larger models and using larger batch sizes. The reduced memory footprint also helps in reducing the pressure on memory bandwidth and lowering communication costs. On hardware with specialized support for float16 computation you can also greatly improve the speed of training and inference. The Volta range of Graphics Processing Units (GPUs) from Nvidia have [Tensor Cores](https://www.nvidia.com/en-us/data-center/tensorcore/) which perform efficient float16 computation. A tensor core allows accumulation of half precision products into single or half precision outputs. For the rest of this tutorial we assume that we are working with Nvidia's Tensor Cores on a Volta GPU.\n\n## Prerequisites\n\n- [Volta](https://www.nvidia.com/en-us/data-center/volta-gpu-architecture/) range of Nvidia GPUs (e.g. AWS P3 instance)\n- CUDA 9 or higher\n- cuDNN v7 or higher\n\nThis tutorial also assumes understanding of how to train a network with float32 (the default). Please refer to [logistic regression tutorial](/api/python/docs/tutorials/getting-started/logistic_regression_explained.html) to get started with Apache MXNet and Gluon API. This tutorial focuses on the changes needed to switch from float32 to mixed precision and tips on achieving the best performance with mixed precision.\n\n## Using the Gluon API\n\n### Training or Inference\n\nWith Gluon API, you need to take care of three things to convert a model to support computation with float16.\n\n1. Cast Gluon `Block`'s parameters and expected input type to float16 by calling the [cast](/api/python/docs/api/gluon/block.html?cast#mxnet.gluon.Block.cast) method of the `Block` representing the network.\n\n```python\nnet.cast('float16')\n```\n\n2. Ensure the data input to the network is of float16 type. If your `DataLoader` or `Iterator` produces output in another datatype, then you would have to cast your data. There are different ways you can do this. The easiest would be to use the [astype](/api/python/docs/api/ndarray/ndarray.html?astype#mxnet.ndarray.NDArray.astype) method of NDArrays.\n\n```python\ndata = data.astype('float16', copy=False)\n```\n\nIf you are using images and DataLoader, you can also use a [Cast transform](/api/python/docs/api/gluon/data/vision/transforms/index.html#mxnet.gluon.data.vision.transforms.Cast).\n\n3. It is preferable to use **multi_precision mode of optimizer** when training in float16. This mode of optimizer maintains a master copy of the weights in float32 even when the training (i.e. forward and backward pass) is in float16. This helps increase precision of the weight updates and can lead to faster convergence in some scenarios.\n\n```python\noptimizer = mx.optimizer.create('sgd', multi_precision=True, lr=0.01)\n```\n\nYou can play around with mixed precision using the image classification [example](https://github.com/apache/mxnet/blob/master/example/image-classification/train_imagenet.py). We suggest using the Caltech101 dataset option in that example and using a ResNet50V1 network so you can quickly see the performance improvement and how the accuracy is unaffected. Here's the starter command to run this example.\n\n```bash\npython image_classification.py --model resnet50_v1 --dataset caltech101 --gpus 0 --num-worker 30 --dtype float16\n```\n\n### Fine-tuning\n\nYou can also fine-tune a model, which was originally trained in float32, to use float16. Below is an example of how to fine-tune a pretrained model from the Model Zoo. You would first need to fetch the pretrained network and then cast that network to float16.\n\n```python\nimport numpy as np\nimport mxnet as mx\nfrom mxnet.gluon.model_zoo.vision import get_model\n\n\npretrained_net = get_model(name='resnet50_v2', ctx=mx.cpu(),\n                           pretrained=True, classes=1000)\npretrained_net.cast('float16')\n```\n\nThen, if you have another Resnet50V2 model you want to fine-tune, you can just assign the features to that network and then cast it.\n\n```python\nnet = get_model(name='resnet50_v2', ctx=mx.cpu(),\n                pretrained=False, classes=101)\nnet.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=mx.cpu())\nnet.features = pretrained_net.features\nnet.cast('float16')\n```\n\nYou can check the parameters of the model by calling [summary](/api/python/docs/api/gluon/block.html?block%20summary#mxnet.gluon.Block.summary) with some fake data. Notice the provided `dtype=np.float16` in the line below. As it was mentioned earlier, we have to provide data as float16 as well.\n\n```python\nnet.summary(mx.nd.uniform(shape=(1, 3, 224, 224), dtype=np.float16))\n```\n\n## Example training results\n\nLet us consider training a Resnet50-V1 model on the ImageNet 2012 dataset. For this model, the GPU memory usage is close to the capacity of V100 GPU with a batch size of 128 when using float32. Using float16 allows the use of 256 batch size. Shared below are results using 8 V100 GPUs on a an [AWS p3.16xlarge](https://aws.amazon.com/ec2/instance-types/p3/#Amazon_EC2_P3_Instance_Product_Details) instance.\n\nLet us compare the three scenarios that arise here: float32 with 1024 batch size, float16 with 1024 batch size and float16 with 2048 batch size. These jobs trained for 90 epochs using a learning rate of 0.4 for 1024 batch size and 0.8 for 2048 batch size. This learning rate was decayed by a factor of 0.1 at the 30th, 60th and 80th epochs. The only changes made for the float16 jobs when compared to the float32 job were that the network and data were cast to float16, and the multi-precision mode was used for optimizer. The final accuracy at 90th epoch and the time to train are tabulated below for these three scenarios. The top-1 validation errors at the end of each epoch are also plotted below.\n\nBatch size | Data type | Top 1 Validation accuracy | Time to train | Speedup |\n--- | --- | --- | --- | --- |\n1024 | float32 | 76.18% | 11.8 hrs | 1 |\n1024 | float16 | 76.34% | 7.3 hrs | 1.62x |\n2048 | float16 | 76.29% | 6.5 hrs | 1.82x |\n\n![Training curves of Resnet50V1 on Imagenet 2012](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tutorials/mixed-precision/resnet50v1b_imagenet_fp16_fp32_training.png)\n\nThe difference in accuracies above are within normal random variation, and there is no reason to expect float16 to have better accuracy than float32 in general. As the plot indicates, training behaves similarly for these cases, even though we didn't have to change any other hyperparameters. We can also see from the table that using float16 helps train faster through faster computation with float16 as well as allowing the use of larger batch sizes.\n\n## Things to keep in mind\n\n### For performance\n\nTypical performance gains seen for float16 typically range 1.6x-2x for convolutional networks like Resnet and even about 3x for networks with LSTMs. The performance gain you see can depend on certain things which this section will introduce.\n\n1. Nvidia Tensor Cores essentially perform the computation `D = A * B + C`, where A and B are half precision matrices, while C and D could be either half precision or full precision. The tensor cores are most efficient when dimensions of these matrices are multiples of 8. This means that Tensor Cores can not be used in all cases for fast float16 computation. When training models like Resnet50 on the Cifar10 dataset, the tensors involved are sometimes smaller, and Tensor Cores can not always be used. The computation in that case falls back to slower algorithms and using float16 turns out to be slower than float32 on a single GPU. Note that when using multiple GPUs, using float16 can still be faster than float32 because of reduction in communication costs.\n\n2. When you scale up the batch size ensure that IO and data pre-processing is not your bottleneck. If you see a slowdown this would be the first thing to check.\n\n3. It is advisable to use batch sizes that are multiples of 8 because of the above reason when training with float16. As always, batch sizes which are powers of 2 would be best when compared to those around it.\n\n4. You can check whether your program is using Tensor cores for fast float16 computation by profiling with `nvprof`. The operations with `s884cudnn` in their names represent the use of Tensor cores.\n\n5. When not limited by GPU memory, it can help to set the environment variable `MXNET_CUDNN_AUTOTUNE_DEFAULT` to `2`. This configures MXNet to run tuning tests and choose the fastest convolution algorithm whose memory requirements may exceed the default memory of CUDA workspace.\n\n6. Please note that float16 on CPU might not be supported for all operators, as in most cases float16 on CPU is much slower than float32.\n\n### For accuracy\n\n#### Multi precision mode\n\nWhen training in float16, it is advisable to still store the master copy of the weights in float32 for better accuracy. The higher precision of float32 helps overcome cases where gradient update can become 0 if represented in float16. This mode can be activated by setting the parameter `multi_precision` of optimizer params to `True` as in the above example. It has been found that this is not required for all networks to achieve the same accuracy as with float32, but nevertheless recommended. Note that for distributed training, this is currently slightly slower than without `multi_precision`, but still much faster than using float32 for training.\n\n#### Large reductions\n\nSince float16 has low precision for large numbers, it is best to leave layers which perform large reductions in float32. This includes BatchNorm and Softmax. Ensuring that Batchnorm performs reduction in float32 is handled by default in both Gluon and Module APIs. While Softmax is set to use float32 even during float16 training in Gluon, in the Module API it needs to be a cast to float32 before softmax as the above symbolic example code shows.\n\n#### Loss scaling\n\nFor some networks just switching the training to float16 mode was not found to be enough to reach the same accuracy as when training with float32. This is because the activation gradients computed are too small and could not be represented in float16 representable range. Such networks can be made to achieve the accuracy reached by float32 with a couple of changes.\n\nMost of the float16 representable range is not used by activation gradients generally. So you can shift the gradients into float16 range by scaling up the loss by a factor `S`. By the chain rule, this scales up the loss before backward pass, and then you can scale back the gradients before updating the weights. This ensures that training in float16 can use the same hyperparameters as used during float32 training.\n\nHere's how you can configure the loss to be scaled up by 128 and rescale the gradient down before updating the weights.\n\n*Gluon API*\n\n```python\nloss = gluon.loss.SoftmaxCrossEntropyLoss(weight=128)\noptimizer = mx.optimizer.create('sgd',\n                                multi_precision=True,\n                                rescale_grad=1.0/128)\n```\n\nNetworks like Multibox SSD, R-CNN, bigLSTM and Seq2seq were found to exhibit such behavior.\nYou can choose a constant scaling factor while ensuring that the absolute value of gradient when multiplied by this factor remains in the range of float16. Generally powers of 2 like 64, 128, 256, 512 are chosen. Refer to the linked articles below for more details on this.\n\n## References\n\n1. [Training with Mixed Precision User Guide](http://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html)\n2. [Mixed Precision Training at ICLR 2018](https://arxiv.org/pdf/1710.03740.pdf)\n3. [Mixed-Precision Training of Deep Neural Networks](https://devblogs.nvidia.com/mixed-precision-training-deep-neural-networks/)\n\n## Recommended Next Steps\n\n* Check out our video tutorial on [Using Mixed Precision with MXNet](https://www.youtube.com/watch?v=pR4KMh1lGC0)\n"
  },
  {
    "path": "docs/static_site/src/pages/api/faq/gradient_compression.md",
    "content": "---\nlayout: page_category\ntitle:  Gradient Compression\ncategory: Speed\nfaq_c: Speed\nquestion: How do I use gradient compression with distributed training?\npermalink: /api/faq/gradient_compression\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Gradient Compression\n\nGradient Compression reduces communication bandwidth, and in some scenarios, it can make training more scalable and efficient without significant loss in convergence rate or accuracy. Example implementations with GPUs, CPUs, and distributed training are provided in this document.\n\n\n## Benefits\n\n**Increased Speed**\n\nFor architectures with fully connected layers, the gradient compression capability is observed to speedup training by about 2x, depending on the size of the model and the network bandwidth of the instance. Bigger models see larger speedup with gradient compression.\n\n**Minimal Accuracy Loss**\n\nGradient compression uses the approach of delaying the synchronization of weight updates which are small. Although small weight updates might not be sent for that batch, this information is not discarded. Once the weight updates for this location accumulate to become a larger value, they will be propagated. Since there is no information loss, but only delayed updates, it does not lead to a significant loss in accuracy or convergence rate. In distributed training experiments[1], the accuracy loss observed due to gradient compression was as low as 1%\n\n\n## When to Use Gradient Compression\n\nWhen training models whose architectures include large fully connected components, it can be helpful to use gradient compression. For larger models, as well as recurrent neural networks, the communication cost becomes a major factor. Such models stand to benefit greatly with gradient compression.\n\n\n### GPU versus CPU\n\nThe greatest benefits from gradient compression are realized when using multi-node (single or multi-GPU) distributed training. Training on CPU would provide a lower compute density per compute node as compared to the massive compute density per compute node on a GPU. Due to this, the required communication bandwidth for CPU-based nodes during training is not as high as for GPU-based nodes. Hence, the benefits of gradient compression are lower for CPU-based nodes as compared to GPU-based nodes.\n\n\n### Network Latency\n\nBenefits of gradient compression can be found when using distributed training with network connected nodes. Depending on the network latency between nodes and the model's size, these can contribute to slow performance such that gradient compression may provide speed improvements.\n\nYou may not want to use gradient compression if you have low latency network communication.\n\n\n### Model Size\n\nDistributed training involves synchronization of weights after each batch. Larger models have much higher communication costs during training, hence such models stand to benefit much more from gradient compression.\nWhen running distributed training with gradient compression, the quantize and dequantize operations happen on CPU parallelized with OpenMP. For smaller models, when training on GPUs, it helps to set `OMP_NUM_THREADS=1` on each node, so that the overhead of launching OMP threads doesn't cause the compression and decompression to be slow.\n\n### Model Architecture\n\nThe communication bandwidth requirements during training vary across various neural network architectures and hence the benefits of gradient compression vary accordingly.\n\nIn networks which have significant fully connected components, since such layers have low compute cost on GPUs, communication becomes a bottleneck limiting the speed of distributed training. Gradient compression can help reduce the communication cost, and thus speed up training in such cases. We have observed speedup of about 2x on large fully connected neural networks. Models like AlexNet and VGG have large fully connected components as part of the network, hence stand to benefit from gradient compression. As with these models, Long Short-Term Memory architectures require more communication bandwidth, so they also exhibit speed improvements with gradient compression.\n\nArchitectures like Convolutional Neural Networks on the other hand have a higher compute cost, in which case some communication can be parallelized with computation. Since communication is not the bottleneck in such networks, gradient compression doesn't help much.\n\n\n### Single Node Gradient Compression\n\nWhen the training is configured to use device to device communication on a single node with multiple GPUs, gradient compression can be used to reduce the cost of communication. This can provide about 20% speedup for large models using older generation architectures. However, speed benefits may be negligible on a machine with a newer generation architecture where GPUs can communicate at low latency.\n\n\n## Approach\n\nThe idea behind gradient compression comes from two observations:\n\nFirst, when training large neural networks, the gradients of weights computed for a small mini-batch of training data are typically sparse. Only a small fraction of the weights have significant updates after each mini-batch. The synchronization of updates that are near zero can be safely delayed longer than the typical mini-batch size. This essentially means that the rate of weight-update can vary depending on the value of an individual weight.\n\nSecondly, gradients can be compressed significantly by considering only those gradient elements whose absolute values exceed a threshold, and then quantizing them to use lower bits per gradient value. By compressing the gradients, we can reduce communication bandwidth. The delayed gradient values, in the form of quantization error and values that don't meet the threshold, are aggregated into a gradient residual which is communicated when it reaches the threshold.\n\n## Technical Implementation\n\n### Two Bit Quantization\n\nCurrently the supported type of quantization uses two bits for each gradient value. Any positive value greater than or equal to the threshold sets two bits as `11`, any negative value whose absolute value is greater or equal to the threshold sets two bits as `10`, and others are set to `00`. This enables us to store 16 quantized gradients as one float. The error in quantization, which is `original_value - quantized_value` is stored in the form of a gradient residual.\n\n### Types of Kvstore\n\nSupported types of `kvstore` are `device` and all distributed kvstores such as `dist_sync`, `dist_async`, and `dist_sync_device`. When `kvstore` is `device`, the communication between GPUs is compressed. Please note that this increases the memory usage of GPUs because of the additional residual stored. When using a distributed kvstore, worker-to-server communication is compressed. In this case, compression and decompression happen on the CPU, and gradient residuals will be stored on the CPU. Server-to-worker communication and device-to-device communication are not compressed to avoid multiple levels of compression.\n\n## Enabling the Gradient Compression in MXNet\n\nGradient compression is a run-time configuration parameter to be enabled during training. Here are the MXNet APIs to enable gradient compression:\n\n**Gluon API**:\n\n```python\ntrainer = gluon.Trainer(..., compression_params={'type’:'2bit', 'threshold':0.5})\n```\nA reference `gluon` implementation with a gradient compression option can be found in the [train.py script from a word-level language modeling RNN example](https://github.com/apache/mxnet/blob/master/example/gluon/word_language_model/train.py).\n\n### Configuration Details\n\n**Threshold**\n\nA default `threshold` value of `0.5` is good for most use cases, but to get the most benefit from gradient compression for a particular scenario, it can be beneficial to experiment. If the threshold is set to a very large value, say `10.0`, then the updates become too infrequent and the training will converge slower. Setting the threshold automatically is expected in a future release.\n\n**Quantization**\n\nThis release supports 2-bit quantization for encoding of gradients to reduce the communication bandwidth during training. Future releases will support 1-bit quantization and other approaches for encoding of gradients based on experimental evidence of benefits and user demand.\n\n**Sparse Format**\n\nWe believe that the density of data will need to be really low (i.e. around > 90% zeros) to reap benefits of the sparse format. However, this is an area of experimentation that will be explored in a future release.\n\n\n## References\n\n1. [Nikko Storm, Amazon.com, Scalable Distributed Training using commodity GPU cloud computing.](https://s3-us-west-2.amazonaws.com/amazon.jobs-public-documents/strom_interspeech2015.pdf)\n"
  },
  {
    "path": "docs/static_site/src/pages/api/faq/large_tensor_support.md",
    "content": "---\nlayout: page_category\ntitle: Using MXNet with Large Tensor Support\ncategory: faq\nfaq_c: Extend and Contribute to MXNet\nquestion: How do I use MXNet built with Large Tensor Support\npermalink: /api/faq/large_tensor_support\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Using MXNet with Large Tensor Support\n\n## What is large tensor support?\nWhen creating a network that uses large amounts of data, as in a deep graph problem, you may need large tensor support. This means tensors are indexed using INT64, instead of INT32 indices.\n\nThis feature is enabled when MXNet is built with a flag *USE_INT64_TENSOR_SIZE=1*, which is now a default setting. You can also make MXNet use INT32 indices by changing this flag.\n\n## When do you need it?\n1. When you are creating NDArrays of size larger than 2^31 elements.\n2. When the input to your model requires tensors that have inputs larger than 2^31 (when you load them all at once in your code) or attributes greater than 2^31.\n\n## How to identify that you need to use large tensors ?\nWhen you see one of the following errors:\n\n\n1. OverflowError: unsigned int is greater than maximum\n2. Check failed: inp->shape().Size() < 1 >> 31 (4300000000 vs. 0) : Size of tensor you are trying to allocate is larger than 2^32 elements. Please build with flag USE_INT64_TENSOR_SIZE=1\n3. Invalid Parameter format for end expect int or None but value='2150000000', in operator slice_axis(name=\"\", end=\"2150000000\", begin=\"0\", axis=\"0\"). *_Basically input attribute was expected to be int32, which is less than 2^31 and the received value is larger than that so, operator's parmeter inference treats that as a string which becomes unexpected input.`_*\n\n## How to use it ?\nYou can create a large NDArray that requires large tensor enabled build to run as follows:\n\n```python\nLARGE_X=4300000000\na = mx.nd.arange(0, LARGE_X, dtype=“int64”)\nor\na = nd.ones(shape=LARGE_X)\nor\na = nd.empty(LARGE_X)\nor\na = nd.random.exponential(shape=LARGE_X)\nor\na = nd.random.gamma(shape=LARGE_X)\nor\na = nd.random.normal(shape=LARGE_X)\n```\n\n## Caveats\n1. Use `int64` as `dtype` whenever attempting to slice an NDArray when range is over maximum `int32` value\n2. Use `int64` as `dtype` when passing indices as parameters or expecting output as parameters to and from operators\n\nThe following are the cases for large tensor usage where you must specify `dtype` as `int64`:\n\n\n* _randint():_\n\n```python\nlow_large_value = 2**32\nhigh_large_value = 2**34\n# dtype is explicitly specified since default type is int32 for randint\na = nd.random.randint(low_large_value, high_large_value, dtype=np.int64)\n```\n\n* _ravel_multi_index()_ and _unravel_index()_:\n\n```python\nx1, y1 = rand_coord_2d((LARGE_X - 100), LARGE_X, 10, SMALL_Y)\nx2, y2 = rand_coord_2d((LARGE_X - 200), LARGE_X, 9, SMALL_Y)\nx3, y3 = rand_coord_2d((LARGE_X - 300), LARGE_X, 8, SMALL_Y)\nindices_2d = [[x1, x2, x3], [y1, y2, y3]]\n# dtype is explicitly specified for indices else they will default to float32\nidx = mx.nd.ravel_multi_index(mx.nd.array(indices_2d, dtype=np.int64),\n                                  shape=(LARGE_X, SMALL_Y))\nindices_2d = mx.nd.unravel_index(mx.nd.array(idx_numpy, dtype=np.int64),\n                                  shape=(LARGE_X, SMALL_Y))\n```\n\n* _argsort()_ and _topk()_\n\nThey both return indices which are specified by `dtype=np.int64`.\n\n```python\nb = create_2d_tensor(rows=LARGE_X, columns=SMALL_Y)\n# argsort\ns = nd.argsort(b, axis=0, is_ascend=False, dtype=np.int64)\n# topk\nk = nd.topk(b, k=10, axis=0, dtype=np.int64)\n```\n\n* _index_copy()_\n\nAgain whenever we are passing indices as arguments and using large tensor, the `dtype` of indices must be `int64`.\n\n```python\nx = mx.nd.zeros((LARGE_X, SMALL_Y))\nt = mx.nd.arange(1, SMALL_Y + 1).reshape((1, SMALL_Y))\n# explicitly specifying dtype of indices to np.int64\nindex = mx.nd.array([LARGE_X - 1], dtype=\"int64\")\nx = mx.nd.contrib.index_copy(x, index, t)\n```\n\n* _one_hot()_\n\nHere again array is used as indices that act as location of bits inside the large vector that need to be activated.\n\n```python\n# a is the index array here whose dtype should be int64.\na = nd.array([1, (VLARGE_X - 1)], dtype=np.int64)\nb = nd.one_hot(a, VLARGE_X)\n```\n\n## What platforms and version of MXNet are supported ?\nYou can use MXNet with large tensor support in the following configuration:\n\n*MXNet built for CPU on Linux (Ubuntu or Amazon Linux), and only for python bindings.*\n*Custom wheels are provided with this configuration.*\n\nThese flavors of MXNet are currently built with large tensor support:\n\n1. MXNet for linux-cpu\n2. MXNet for linux_cu100\n\nLarge tensor support only works for *forward pass*. \nBackward pass is partially supported and not completely tested, so it is considered experimental at best.\n\nNot supported:\n\n* GPU. \n* Windows, ARM or any operating system other than Ubuntu\n* Other language bindings like Scala, Java, R,  and Julia.\n\n\n## Other known Issues:\n* Randint operator is flaky: https://github.com/apache/mxnet/issues/16172.\n* dgemm operations using BLAS libraries currently don’t support int64.\n* linspace() is not supported.\n\n```python\na = mx.sym.Variable('a')\nb = mx.sym.Variable('b')\nc = 2 * a + b\ntexec = c.bind(mx.cpu(), {'a': nd.arange(0, LARGE_X * 2, dtype='int64').reshape(2, LARGE_X), 'b' : nd.arange(0, LARGE_X * 2, dtype='int64').reshape(2, LARGE_X)})\nnew_shape = {'a': (1, 2*LARGE_X), 'b': (1, 2*LARGE_X)}\ntexec.reshape(allow_up_sizing=True, **new_shape)\n\nTraceback (most recent call last):\n  File \"<stdin>\", line 1, in <module>\n  File \"/home/ubuntu/mxnet/python/mxnet/executor.py\", line 449, in reshape\n    py_array('i', provided_arg_shape_data)),\nOverflowError: signed integer is greater than maximum}\n```\n\nSymbolic reshape is not supported. Please see the following example.\n\n```python\na = mx.sym.Variable('a')\nb = mx.sym.Variable('b')\nc = 2 * a + b\ntexec = c.bind(mx.cpu(), {'a': nd.arange(0, LARGE_X * 2, dtype='int64').reshape(2, LARGE_X), 'b' : nd.arange(0, LARGE_X * 2, dtype='int64').reshape(2, LARGE_X)})\nnew_shape = {'a': (1, 2 * LARGE_X), 'b': (1, 2 * LARGE_X)}\ntexec.reshape(allow_up_sizing=True, **new_shape)\n\nTraceback (most recent call last):\n  File \"<stdin>\", line 1, in <module>\n  File \"/home/ubuntu/mxnet/python/mxnet/executor.py\", line 449, in reshape\n    py_array('i', provided_arg_shape_data)),\nOverflowError: signed integer is greater than maximum\n```\n\n## Working DGL Example(dgl.ai)\nThe following is a sample running code for DGL which works with int64 but not with int32.\n\n```python\nimport mxnet as mx\nfrom mxnet import gluon\nimport dgl\nimport dgl.function as fn\nimport numpy as np\nfrom scipy import sparse as spsp\n\nnum_nodes = 10000000\nnum_edges = 100000000\n\ncol1 = np.random.randint(0, num_nodes, size=(num_edges,))\nprint('create col1')\ncol2 = np.random.randint(0, num_nodes, size=(num_edges,))\nprint('create col2')\ndata = np.ones((num_edges,))\nprint('create data')\nspm = spsp.coo_matrix((data, (col1, col2)), shape=(num_nodes, num_nodes))\nprint('create coo')\nlabels = mx.nd.random.randint(0, 10, shape=(num_nodes,))\n\ng = dgl.DGLGraph(spm, readonly=True)\nprint('create DGLGraph')\ng.ndata['h'] = mx.nd.random.uniform(shape=(num_nodes, 200))\nprint('create node data')\n\nclass node_update(gluon.Block):\n    def __init__(self, out_feats):\n        super(node_update, self).__init__()\n        self.dense = gluon.nn.Dense(out_feats, 'relu')\n        self.dropout = 0.5\n\n    def forward(self, nodes):\n        h = mx.nd.concat(nodes.data['h'], nodes.data['accum'], dim=1)\n        h = self.dense(h)\n        return {'h': mx.nd.Dropout(h, p=self.dropout)}\nupdate_fn = node_update(200)\nupdate_fn.initialize(ctx=mx.cpu())\n\ng.update_all(fn.copy_src(src='h', out='m'), fn.sum(msg='m', out='accum'), update_fn)\nprint('update all')\n\nloss_fcn = gluon.loss.SoftmaxCELoss()\nloss = loss_fcn(g.ndata['h'], labels)\nprint('loss')\nloss = loss.sum()\nprint(loss)\n```\n\n## Performance Regression: \nRoughly 40 operators have shown performance regression in our preliminary analysis: Large Tensor Performance as shown in table below.\n\n|Operator                                |int32(msec)|int64(msec)  |int64/int32  |int32+mkl(msec)|int64+mkl(msec)|int64+mkl/int32+mkl|\n|----------------------------------------|-----------|-------------|-------------|---------------|---------------|-------------------|\n|topk                                    |12.81245198|42.2472195   |329.74%      |12.728027      |43.462353      |341.47%            |\n|argsort                                 |16.43896801|46.2231455   |281.18%      |17.200311      |46.7779985     |271.96%            |\n|sort                                    |16.57822751|46.5644815   |280.88%      |16.401236      |46.263803      |282.08%            |\n|flip                                    |0.221817521|0.535838     |241.57%      |0.2123705      |0.7950055      |374.35%            |\n|depth_to_space                          |0.250976998|0.534083     |212.80%      |0.2338155      |0.631252       |269.98%            |\n|space_to_depth                          |0.254336512|0.5368935    |211.10%      |0.2334405      |0.6343175      |271.73%            |\n|min_axis                                |0.685826526|1.4393255    |209.87%      |0.6266175      |1.3538925      |216.06%            |\n|sum_axis                                |0.720809505|1.5110635    |209.63%      |0.6566265      |0.8290575      |126.26%            |\n|nansum                                  |1.279337012|2.635434     |206.00%      |1.227156       |2.4305255      |198.06%            |\n|argmax                                  |4.765146994|9.682672     |203.20%      |4.6576605      |9.394067       |201.69%            |\n|swapaxes                                |0.667943008|1.3544455    |202.78%      |0.649036       |1.8293235      |281.85%            |\n|argmin                                  |4.774890491|9.545651     |199.91%      |4.666858       |9.5194385      |203.98%            |\n|sum_axis                                |0.540210982|1.0550705    |195.31%      |0.500895       |0.616179       |123.02%            |\n|max_axis                                |0.117824005|0.226481     |192.22%      |0.149085       |0.224334       |150.47%            |\n|argmax_channel                          |0.261897018|0.49573      |189.28%      |0.251171       |0.4814885      |191.70%            |\n|min_axis                                |0.147698505|0.2675355    |181.14%      |0.148424       |0.2874105      |193.64%            |\n|nansum                                  |1.142132009|2.058077     |180.20%      |1.042387       |1.263102       |121.17%            |\n|min_axis                                |0.56951947 |1.020972     |179.27%      |0.4722595      |0.998179       |211.36%            |\n|min                                     |1.154684491|2.0446045    |177.07%      |1.0534145      |1.9723065      |187.23%            |\n|sum                                     |1.121753477|1.959272     |174.66%      |0.9984095      |1.213339       |121.53%            |\n|sum_axis                                |0.158632494|0.2744115    |172.99%      |0.1573735      |0.2266315      |144.01%            |\n|nansum                                  |0.21418152 |0.3661335    |170.95%      |0.2162935      |0.269517       |124.61%            |\n|random_normal                           |1.229072484|2.093057     |170.30%      |1.222785       |2.095916       |171.41%            |\n|LeakyReLU                               |0.344101485|0.582337     |169.23%      |0.389167       |0.7003465      |179.96%            |\n|nanprod                                 |1.273265516|2.095068     |164.54%      |1.0906815      |2.054369       |188.36%            |\n|nanprod                                 |0.203272473|0.32792      |161.32%      |0.202548       |0.3288335      |162.35%            |\n|sample_gamma                            |8.079962019|12.7266385   |157.51%      |12.4216245     |12.7957475     |103.01%            |\n|sum                                     |0.21571602 |0.3396875    |157.47%      |0.1939995      |0.262942       |135.54%            |\n|argmin                                  |0.086381478|0.1354795    |156.84%      |0.0826235      |0.134886       |163.25%            |\n|argmax                                  |0.08664903 |0.135826     |156.75%      |0.082693       |0.1269225      |153.49%            |\n|sample_gamma                            |7.712843508|12.0266355   |155.93%      |11.8900915     |12.143009      |102.13%            |\n|sample_exponential                      |2.312778   |3.5953945    |155.46%      |3.0935085      |3.5656265      |115.26%            |\n|prod                                    |0.203170988|0.3113865    |153.26%      |0.180757       |0.264523       |146.34%            |\n|random_uniform                          |0.40893798 |0.6240795    |152.61%      |0.244613       |0.6319695      |258.35%            |\n|min                                     |0.205482502|0.3122025    |151.94%      |0.2023835      |0.33234        |164.21%            |\n|random_negative_binomial                |3.919228504|5.919488     |151.04%      |5.685851       |6.0220735      |105.91%            |\n|max                                     |0.212521001|0.3130105    |147.28%      |0.2039755      |0.2956105      |144.92%            |\n|LeakyReLU                               |2.813424013|4.1121625    |146.16%      |2.719118       |5.613753       |206.45%            |\n|mean                                    |0.242281501|0.344385     |142.14%      |0.209396       |0.313411       |149.67%            |\n|Deconvolution                           |7.43279251 |10.4240845   |140.24%      |2.9548925      |5.812926       |196.72%            |\n|abs                                     |0.273286481|0.38319      |140.22%      |0.3711615      |0.338064       |91.08%             |\n|arcsinh                                 |0.155792513|0.2090985    |134.22%      |0.113365       |0.1702855      |150.21%            |\n|sample_gamma                            |0.137634983|0.1842455    |133.87%      |0.1792825      |0.172175       |96.04%             |\n|sort                                    |0.864107016|1.1560165    |133.78%      |0.8239285      |1.1454645      |139.02%            |\n|argsort                                 |0.847259507|1.1320885    |133.62%      |0.842302       |1.1179105      |132.72%            |\n|cosh                                    |0.129947497|0.1727415    |132.93%      |0.1192565      |0.1217325      |102.08%            |\n|random_randint                          |0.822044531|1.085645     |132.07%      |0.6036805      |1.0953995      |181.45%            |\n|arctanh                                 |0.119817996|0.1576315    |131.56%      |0.115616       |0.111907       |96.79%             |\n|arccos                                  |0.185662502|0.2423095    |130.51%      |0.238534       |0.2351415      |98.58%             |\n|mean                                    |1.758513477|2.2908485    |130.27%      |1.5868465      |2.530801       |159.49%            |\n|erfinv                                  |0.142498524|0.184796     |129.68%      |0.1529025      |0.1538225      |100.60%            |\n|degrees                                 |0.12517249 |0.1576175    |125.92%      |0.1166425      |0.1199775      |102.86%            |\n|sample_exponential                      |0.07651851 |0.0960485    |125.52%      |0.0885775      |0.095597       |107.92%            |\n|arctan                                  |0.120863522|0.1496115    |123.79%      |0.1161245      |0.17206        |148.17%            |\n|prod                                    |1.147695002|1.408007     |122.68%      |1.0491025      |1.4065515      |134.07%            |\n|fix                                     |0.073436997|0.089991     |122.54%      |0.0390455      |0.099307       |254.34%            |\n|exp                                     |0.047701993|0.058272     |122.16%      |0.0397295      |0.0506725      |127.54%            |\n"
  },
  {
    "path": "docs/static_site/src/pages/api/faq/model_parallel_lstm.md",
    "content": "---\nlayout: page_category\ntitle:  Model Parallel\ncategory: faq\nfaq_c: Model\nquestion: How can I train using multiple GPUs with model parallelism?\npermalink: /api/faq/model_parallel_lstm\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Training with Multiple GPUs Using Model Parallelism\nTraining deep learning models can be resource intensive.\nEven with a powerful GPU, some models can take days or weeks to train.\nLarge long short-term memory (LSTM) recurrent neural networks\ncan be especially slow to train,\nwith each layer, at each time step, requiring eight matrix multiplications.\nFortunately, given cloud services like AWS,\nmachine learning practitioners often  have access\nto multiple machines and multiple GPUs.\nOne key strength of _MXNet_ is its ability to leverage\npowerful heterogeneous hardware environments to achieve significant speedups.\n\nThere are two primary ways that we can spread a workload across multiple devices.\nIn a previous document, [we addressed data parallelism](/api/faq/distributed_training),\nan approach in which samples within a batch are divided among the available devices.\nWith data parallelism, each device stores a complete copy of the model.\nHere, we explore _model parallelism_, a different approach.\nInstead of splitting the batch among the devices, we partition the model itself.\nMost commonly, we achieve model parallelism by assigning the parameters (and computation)\nof different layers of the network to different devices.\n\nIn particular, we will focus on LSTM recurrent networks.\nLSTMS are powerful sequence models, that have proven especially useful\nfor [natural language translation](https://arxiv.org/pdf/1409.0473.pdf), [speech recognition](https://arxiv.org/abs/1512.02595),\nand working with [time series data](https://arxiv.org/abs/1511.03677).\nFor a general high-level introduction to LSTMs,\nsee the excellent [tutorial](https://colah.github.io/posts/2015-08-Understanding-LSTMs/) by Christopher Olah.\n\n\n## Model Parallelism: Using Multiple GPUs As a Pipeline\nModel parallelism in deep learning was first proposed\nfor the _extraordinarily large_ convolutional layer in GoogleNet.\nFrom this implementation, we take the idea of placing each layer on a separate GPU.\nUsing model parallelism in such a layer-wise fashion\nprovides the benefit that no GPU has to maintain all of the model parameters in memory.\n\n<img width=\"517\" alt=\"screen shot 2016-05-06 at 10 13 16 pm\" src=\"https://cloud.githubusercontent.com/assets/5545640/15089697/d6f4fca0-13d7-11e6-9331-7f94fcc7b4c6.png\">\n\nIn the preceding figure, each LSTM layer is assigned to a different GPU.\nAfter GPU 1 finishes computing layer 1 for the first sentence, it passes its output to GPU 2.\nAt the same time, GPU 1 fetches the next sentence and starts training.\nThis differs significantly from data parallelism.\nHere, there is no contention to update the shared model at the end of each iteration,\nand most of the communication happens when passing intermediate results between GPUs.\n\n\n## Workload Partitioning\n\nImplementing model parallelism requires knowledge of the training task.\nHere are some general heuristics that we find useful:\n\n- To minimize communication time, place neighboring layers on the same GPUs.\n- Be careful to balance the workload between GPUs.\n- Remember that different kinds of layers have different computation-memory properties.\n\n<img width=\"449\" alt=\"screen shot 2016-05-07 at 1 51 02 am\" src=\"https://cloud.githubusercontent.com/assets/5545640/15090455/37a30ab0-13f6-11e6-863b-efe2b10ec2e6.png\">\n\nLet's take a quick look at the two pipelines in the preceding diagram.\nThey both have eight layers with a decoder and an encoder layer.\nBased on our first principle, it's unwise to place all neighboring layers on separate GPUs.\nWe also want to balance the workload across GPUs.\nAlthough the LSTM layers consume less memory than the decoder/encoder layers, they consume more computation time because of the dependency of the unrolled LSTM.\nThus, the partition on the left will be faster than the one on the right\nbecause the workload is more evenly distributed.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/faq/new_op.md",
    "content": "---\nlayout: page_category\ntitle:  Create New Operators\ncategory: faq\nfaq_c: Extend and Contribute to MXNet\nquestion: How do I create new operators in MXNet with Python?\npermalink: /api/faq/new_op\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# How to Create New Operators (Layers)\n\nThis tutorials walks you through the process of creating new MXNet operators (or layers).\nWe've done our best to provide high-speed operators for most common use cases.\nHowever, if you're engaged in research,\nthere's a good chance you'll want to define custom layers,\nlike a novel loss function. In these cases, you have two options:\n\n* Use CustomOp to write new operators using a front-end language (e.g., Python) that run on CPUs or GPUs.\nDepending on your implementation, this can range from very fast (if you only use operators under mx.nd) to very slow (if you copy out the data, using `.asnumpy()`).\n\n* Use C++/mshadow (CUDA). This provides the best performance, but can be difficult\nif you're not familiar with MXNet, mshadow, or Cuda.\n\n## CustomOp\nImplementing an operator in Python is simple.\nAs an example, let's create a softmax operator.\nStart by subclassing `mxnet.operator.CustomOp`,\nand then override a few methods:\n\n```python\nimport os\nimport mxnet as mx\nimport numpy as np\n\nclass Softmax(mx.operator.CustomOp):\n    def forward(self, is_train, req, in_data, out_data, aux):\n        x = in_data[0].asnumpy()\n        y = np.exp(x - x.max(axis=1).reshape((x.shape[0], 1)))\n        y /= y.sum(axis=1).reshape((x.shape[0], 1))\n        self.assign(out_data[0], req[0], mx.nd.array(y))\n```\n\nWe defined the computation for the forward pass of our operator.\nThe forward function takes a list of input and a list of output NDArrays.\nFor convenience, we called `.asnumpy()` on the first NDArray in input\nand convert it to a CPU-based NumPy array.\nThis can be very slow. If you want the best performance,\nkeep data in the NDArray format and use operators under mx.nd to do the computation.\n\nAt the end, we used CustomOp.assign to assign the resulting array y to out_data[0]. It handles assignment based on the value of req, which can be 'write', 'add', or 'null'.\n\nThen do the same for the backward pass:\n\n```python\ndef backward(self, req, out_grad, in_data, out_data, in_grad, aux):\n    l = in_data[1].asnumpy().ravel().astype(np.int)\n    y = out_data[0].asnumpy()\n    y[np.arange(l.shape[0]), l] -= 1.0\n    self.assign(in_grad[0], req[0], mx.nd.array(y))\n```\n\nSoftmax defines the computation of our custom operator,\nbut you still need to define its input/output format\nby subclassing mx.operator.CustomOpProp.\nFirst, register the new operator with the name 'softmax':\n\n```python\n@mx.operator.register(\"softmax\")\nclass SoftmaxProp(mx.operator.CustomOpProp):\n```\n\nThen, call the base constructor with `need_top_grad=False`\nbecause softmax is a loss layer and you don't need gradient input from preceding layers:\n\n```python\ndef __init__(self):\n    super(SoftmaxProp, self).__init__(need_top_grad=False)\n```\n\nThen declare the input and output:\n\n```python\ndef list_arguments(self):\n    return ['data', 'label']\n\ndef list_outputs(self):\n    return ['output']\n```\n\nNote that list_arguments declares both input and parameter.\nWe recommend ordering them as follows:  `['input1', 'input2', ... , 'weight1', 'weight2', ...]`\n\nNext, provide `infer_shape` to declare the shape of the output/weight\nand check the consistency of the input shapes:\n\n```python\ndef infer_shape(self, in_shape):\n    data_shape = in_shape[0]\n    label_shape = (in_shape[0][0],)\n    output_shape = in_shape[0]\n    return [data_shape, label_shape], [output_shape], []\n```\nThe first axis of an input/output tensor corresponds to different examples within the batch.\nThe label is a set of integers, one for each data entry,\nand the output has the same shape as the input.\nThe `infer_shape` function should always return three lists in this order:\ninputs, outputs, and auxiliary states (which we don't have here),\neven if one of them is empty.\n\nOptionally, you can also define `infer_type` to declare the input and output data type of your operator. Supported types are `np.float32`, `np.float64`, `np.float16`, `np.uint8`, and `np.int32`.\n\n```python\ndef infer_type(self, in_type):\n    dtype = in_type[0]\n    return [dtype, dtype], [dtype], []\n```\n\nFinally, define a create_operator function that will be called by the back end to create an instance of softmax:\n\n```python\ndef create_operator(self, ctx, shapes, dtypes):\n    return Softmax()\n```\n\nTo use the custom operator, create a mx.sym.Custom symbol with op_type as the registered name:\n\n```python\nmlp = mx.symbol.Custom(data=fc3, name='softmax', op_type='softmax')\n```\n\nPlease see the full code for this example [here](https://github.com/apache/mxnet/blob/master/example/numpy-ops/custom_softmax.py).\n\n## C++\nWith MXNet v0.9 (the NNVM refactor) or later, creating new operators has become easier.\nOperators are now registered with NNVM.\nThe following code is an example on how to register an operator (checkout [src/operator/tensor](https://github.com/apache/mxnet/tree/master/src/operator/tensor) for more examples):\n\n```c++\nNNVM_REGISTER_OP(abs)\n.MXNET_DESCRIBE(\"Take absolute value of the src\")\n.set_num_inputs(1)\n.set_num_outputs(1)\n.set_attr<nnvm::FInferShape>(\"FInferShape\", ElemwiseShape<1,1>);\n```\n\nThe syntax is quite simple, we register the operator with a name,\nthen set number of inputs and outputs.\nYou can register attributes with any key (`FInferShape` for example) to any operator,\nwithout having to modify a central class interface definition.\n\n### Operator Attribute System\n\nOne of the biggest improvements brought by NNVM is the operator attribute system.\nThis is like traits for types in common languages like C++.\nWe can register any attribute to any operator, with the syntax\n\n``` c++\nNNVM_REGISTER_OP(op-name)\n.set_attr<AttributeType>(\"AttributeKey\", CorrespondingAttributeObject);\n```\n\nThese attributes can be retrieved later for various purposes.\nFor example, `FInferShape` is used for shape inference, `FCompute<cpu>` is used for carrying out actual computation on CPU.\n\nAs long as all attributes registered with the same key have the same type,\nwe can register any attributes to operators.\nThe more attribute an operator provides,\nthe more information the system can use for optimization.\n\n### List of basic attributes\n\nIn this section, we will go through the basic attributes MXNet expect for all operators.\nYou can find the definition for them in the following two files:\n\n- [nnvm/op_attr_types.h](https://github.com/dmlc/nnvm/blob/master/include/nnvm/op_attr_types.h)\n- [mxnet/op_attr_types.h](https://github.com/apache/mxnet/blob/master/include/mxnet/op_attr_types.h)\n\n#### Descriptions (Optional)\n\n`.describe(comment)` adds a comment to the operator. Use `.MXNET_DESCRIBE(comment)` to add the current file name and line number to comment.\n\n#### Attribute Parser (Optional)\n\nSet attribute parser with `.set_attr_parser(PARSER)` where PARSER is a function with prototype `void(nnvm::NodeAttr* attrs)`. This function should parse the key-word arguments in `attrs->dict` and store the result in `attrs->parsed`.\n\nSimple arguments can be parsed like\n```c++\nNNVM_REGISTER_OP(scalar_op)\n.set_attr_parser(\n  [](NodeAttrs* attrs) {\n    attrs->parsed = std::stod(attrs->dict[\"scalar\"]);\n  })\n```\n\nThe parsed arguments can then be accessed in other attribute functions with\n```c++\ndouble alpha = nnvm::get<double>(attrs.parsed);\n```\n\nMore complex ops can use `dmlc::Parameters` and `ParamParser` (defined in operator_common.h) for parsing:\n\n```c++\n#include <dmlc/parameter.h>\n#include <operator_common.h>\nstruct ActivationParam : public dmlc::Parameter<ActivationParam> {\n  // use int for enumeration\n  int act_type;\n  DMLC_DECLARE_PARAMETER(ActivationParam) {\n    DMLC_DECLARE_FIELD(act_type)\n    .add_enum(\"relu\", activation::kReLU)\n    .add_enum(\"sigmoid\", activation::kSigmoid)\n    .add_enum(\"tanh\", activation::kTanh)\n    .add_enum(\"softrelu\", activation::kSoftReLU)\n    .describe(\"Activation function to be applied.\");\n  }\n};\nNNVM_REGISTER_OP(Activation)\n.set_attr_parser(ParamParser<ActivationParam>);\n// access with:\n// const ActivationParam& param = nnvm::get<ActivationParam>(attrs.parsed);\n```\n\n#### Inputs & Outputs\n\nNumber of inputs/outputs can be set with `.set_num_inputs(n_in)` and `.set_num_outputs(n_out)`\nwhere n_in and n_out are integers.\n\nAlternatively, if the number of inputs/outputs is variable and depends on arguments,\nyou can set `n_in`/`n_out` to functions with prototype `uint32_t(const nnvm::NodeAttrs& attrs)`\nthat return the number of inputs/outputs based on parsed arguments.\n\nOutputs can be made invisible to other operators by registering `FNumVisibleOutputs`\nand returning an integer smaller than `n_out`.\n\nInputs/outputs can be named by registering `FListInputNames` and `FListOutputNames` with prototype `std::vector<std::string>(const NodeAttrs& attrs)`.\n\n\n#### Argument Descriptions\n\nSet argument descriptions with `.add_argument(name, type, comment)`.\nThis is necessary for operators to be properly called imperatively.\n\nFirst, add NDArray arguments `num_inputs` times with type \"NDArray\"\nor one time with type \"NDArray[]\" for ops with variable length inputs.\n\nThen add key-word arguments with proper type (float, string, etc).\nOperators that parse key-word arguments with `dmlc::Parameter`\ncan add argument descriptions in bulk with `.add_arguments(ActivationParam::__FIELDS__())`\n(NDArray arguments still need to be manually added with type \"NDArray\").\n\n#### FInferShape or TIsBackward (for Backward Only Ops)\n\nNormally operators need to have `FInferShape` with prototype `bool(const nnvm::NodeAttrs& attrs, mxnet::ShapeVector *in_attrs, mxnet::ShapeVector *out_attrs)`. `FInferShape` fills unknown shapes (`shape.ndim() == 0`) in in_attrs/out_attrs based on known shapes in in_attrs/out_attrs. Use `ElemwiseShape<n_in, n_out>` for simple operators with uniform shapes.\n\nOperators that are only used for a backward pass can instead register `.set_attr<nnvm::TIsBackward>(\"TIsBackward\", true)`\nand their shapes with be copied from the corresponding forward operators.\n\n#### FInferType\n\nSimilar to `FInferShape`, `FInferType` fills unknown types (-1) based on known types. Use `ElemwiseType<n_in, n_out>` for simple operators with uniform types. Operators that registered `TIsBackward` don't need to register this.\n\n\n#### FInplaceOption (Optional)\n\n`FInplaceOption` with prototype `std::vector<std::pair<int, int> >(const NodeAttrs& attrs)`\nspecifies which input/output pairs can be computed in-place\nand share memory with each other.\nEach pair (i, j) in the returned list means\nthat the i-th input can share memory with the j-th output.\n\n\n#### FGradient (Optional for imperative use, required for symbolic use)\n\nIf an operator has gradient, it can be described with `FGradient` with prototype\n\n```c++\nstd::vector<nnvm::NodeEntry>(const nnvm::ObjectPtr& n,\n                             const std::vector<nnvm::NodeEntry>& ograds)\n```\n\nUse utility functions `ElemwiseGradUseIn{op_name}`, `ElemwiseGradUseOut{op_name}`, `ElemwiseGradUseNone{op_name}`  for ops that need corresponding forward op's input,\noutput or nothing to calculating gradient.\n\nFor more complicated patterns, use `MakeGradNode(op_name, n, heads, dict)` to create gradient entries,\nwhere heads are input entries to the backward op, composed from ograds and n->inputs.\n\nWhen assembling a return vector of `std::vector<nnvm::NodeEntry> ret;` a common pattern would be to\neither create nodes in place as in:\n\n```c++\nret.emplace_back(MakeNode(\"zeros_like\", n->attrs.name + \"_xyz_backward\",\n    {n->inputs[1]}, nullptr, &n))\n```\n\nOr create the node, modify and then move into NodeEntry's constructor if this node is not to be used\nagain. This avoids uneccessary copies of the shared_ptr.\n\n```c++\nfor (size_t i = 0; i < n->inputs.size(); ++i) {\n  nnvm::ObjectPtr node = nnvm::Node::Create();\n  node->attrs.op = copy_op;\n  node->inputs = {ograds[0]};\n  ret.emplace_back(std::move(node));\n}\n```\n\nThe first case uses RVO and the second in place construction.\n\n#### FCompute\\<xpu\\>\n\nSimple operators can register FCompute<xpu> with `.set_attr<FCompute>(\"FCompute<cpu>\", ...)` and `.set_attr<FCompute>(\"FCompute<gpu>\", ...)` for both CPU and (optionally) GPU computation.\n\nFCompute has prototype\n\n```c++\nvoid(const nnvm::NodeAttrs& attrs,\n     const OpContext& ctx,\n     const std::vector<TBlob>& inputs,\n     const std::vector<OpReqType>& req,\n     const std::vector<TBlob>& outputs)\n```\n\n`req` has the same length as `outputs`.\nEach entry of `req` specifies\nhow the corresponding `output` should be written to.\n`OpReqType` is defined as:\n\n```c++\nenum OpReqType {\n  kNullOp,\n  kWriteTo,\n  kWriteInplace,\n  kAddTo\n};\n```\n\nNormally, the `req` of all `outputs` should be `kWriteTo`,\nmeaning that the provided `outputs` tensor is a *raw* memory block,\nso the operator should write results directly into it.\nIn some cases, for example, when calculating the gradient tensor,\nit would be great if we could accumulate the result,\nrather than directly overwrite the tensor contents\nso that no extra space needs to be created each time.\nIn such cases, the corresponding `req` is set to `kAddTo`,\nindicating that a `+=` should be used.\n\n### Example: abs operator\n\n{% raw %}\n\n```c++\nNNVM_REGISTER_OP(abs)\n.MXNET_DESCRIBE(\"Take absolute value of the src\")\n.set_num_inputs(1)\n.set_num_outputs(1)\n.set_attr<nnvm::FInferShape>(\"FInferShape\", ElemwiseShape<1, 1>)\n.set_attr<nnvm::FInferType>(\"FInferType\", ElemwiseType<1, 1>)\n.set_attr<nnvm::FInplaceOption>(\"FInplaceOption\",\n[](const NodeAttrs& attrs){\n  return std::vector<std::pair<int, int> >{{0, 0}};\n})\n.set_attr<FCompute>(\"FCompute<cpu>\", UnaryCompute<cpu, mshadow_op::abs>)\n.set_attr<nnvm::FGradient>(\"FGradient\", ElemwiseGradUseIn{\"_backward_abs\"});\n.add_argument(\"data\", \"NDArray\", \"Source input\")\n\nNNVM_REGISTER_OP(_backward_abs)\n.set_num_inputs(2)\n.set_num_outputs(1)\n.set_attr<nnvm::FInferShape>(\"FInferShape\", ElemwiseShape<2, 1>)\n.set_attr<nnvm::FInferType>(\"FInferType\", ElemwiseType<2, 1>)\n.set_attr<nnvm::FInplaceOption>(\"FInplaceOption\",\n[](const NodeAttrs& attrs){\n  return std::vector<std::pair<int, int> >{{0, 0}, {1, 0}};\n})\n.set_attr<FCompute>(\"FCompute<cpu>\", BinaryCompute<cpu, backward_grad<mshadow_op::sign> >);\n```\n\n{% endraw %}\n\n### Legacy Operators\n\nFor the legacy (pre 0.9) way of defining operators with C++, please see:\n- [Developer Guide - Operators]({{'/api/architecture/overview.html#operators-in-mxnet'|relative_url}})\n- [Developer Guide - SimpleOp]({{'/api/architecture/overview.html#simpleop-the-unified-operator-api'|relative_url}})"
  },
  {
    "path": "docs/static_site/src/pages/api/faq/perf.md",
    "content": "---\nlayout: page_category\ntitle: Some Tips for Improving MXNet Performance\ncategory: faq\nfaq_c: Speed\nquestion: What are the best setup and data-handling tips and tricks for improving speed?\npermalink: /api/faq/perf\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Some Tips for Improving MXNet Performance\nEven after fixing the training or deployment environment and parallelization scheme,\na number of configuration settings and data-handling choices can impact the _MXNet_ performance.\nIn this document, we address some tips for improving _MXNet_ performance.\n\nPerformance is mainly affected by the following 4 factors:\n\n1. Implementation of operators (Convolution, Pooling, ..)\n   - [Intel CPU](#intel-cpu)\n   - [Nvidia GPU](#nvidia-gpu)\n2. Input data loading and augmentation\n   - [Input Data](#input-data)\n3. Workloads (computation graph) optimization and scheduling\n   - [Profiler](#profiler)\n4. Communication for multi-devices training\n   - [Multiple Devices](#multiple-devices)\n\n## Intel CPU\n\nWhen using Intel Xeon CPUs for training and inference, the `mxnet-mkl` package is recommended. Adding `--pre` installs a nightly build from master. Without it you will install the latest patched release of MXNet:\n\n```\n$ pip install mxnet-mkl [--pre]\n```\n\nOr build MXNet from source code with `USE_ONEDNN=1`. For Linux users, `USE_ONEDNN=1` will be turned on by default.\n\nWe also find that setting the following environment variables can help:\n\n\n| Variable  | Description |\n| :-------- | :---------- |\n| `OMP_NUM_THREADS`            | Suggested value: `vCPUs / 2` in which `vCPUs` is the number of virtual CPUs. For more information, please see the guide for [setting the number of threads using an OpenMP environment variable](https://software.intel.com/en-us/mkl-windows-developer-guide-setting-the-number-of-threads-using-an-openmp-environment-variable) |\n| `KMP_AFFINITY`               | Suggested value: `granularity=fine,compact,1,0`.  For more information, please see the guide for [Thread Affinity Interface (Linux* and Windows*)](https://software.intel.com/en-us/node/522691). |\n\nNote that _MXNet_ treats all CPUs on a single machine as a single device.\nSo whether you specify `cpu(0)` or `cpu()`, _MXNet_ will use all CPU cores on the machine.\n\n### Scoring results\nThe following table shows performance of MXNet-1.2.0.rc1,\nnamely number of images that can be predicted per second.\nWe used [example/image-classification/benchmark_score.py](https://github.com/apache/mxnet/blob/master/example/image-classification/benchmark_score.py)\nto measure the performance on different AWS EC2 machines.\n\nAWS EC2 C5.18xlarge:\n\n\n| Batch | Alexnet | VGG 16    | Inception-BN | Inception-v3 | Resnet 50 | Resnet 152 |\n|-------|---------|--------|--------------|--------------|-----------|------------|\n| 1     | 390.53  | 81.57  | 124.13       | 62.26        | 76.22     | 32.92      |\n| 2     | 596.45  | 100.84 | 206.58       | 93.36        | 119.55    | 46.80      |\n| 4     | 710.77  | 119.04 | 275.55       | 127.86       | 148.62    | 59.36      |\n| 8     | 921.40  | 120.38 | 380.82       | 157.11       | 167.95    | 70.78      |\n| 16    | 1018.43 | 115.30 | 411.67       | 168.71       | 178.54    | 75.13      |\n| 32    | 1290.31 | 107.19 | 483.34       | 179.38       | 193.47    | 85.86      |\n\n\n\nAWS EC2 C5.9xlarge:\n\n\n| Batch | Alexnet | VGG 16   | Inception-BN | Inception-v3 | Resnet 50 | Resnet 152 |\n|-------|---------|-------|--------------|--------------|-----------|------------|\n| 1     | 257.77  | 50.61 | 130.99       | 66.95        | 75.38     | 32.33      |\n| 2     | 410.60  | 63.02 | 195.14       | 87.84        | 102.67    | 41.57      |\n| 4     | 462.59  | 62.64 | 263.15       | 109.87       | 127.15    | 50.69      |\n| 8     | 573.79  | 63.95 | 309.99       | 121.36       | 140.84    | 59.01      |\n| 16    | 709.47  | 67.79 | 350.19       | 128.26       | 147.41    | 64.15      |\n| 32    | 831.46  | 69.58 | 354.91       | 129.92       | 149.18    | 64.25      |\n\n\nAWS EC2 C5.4xlarge:\n\n| Batch | Alexnet | VGG 16   | Inception-BN | Inception-v3 | Resnet 50 | Resnet 152 |\n|-------|---------|-------|--------------|--------------|-----------|------------|\n| 1     | 214.15  | 29.32 | 114.97       | 47.96        | 61.01     | 23.92      |\n| 2     | 310.04  | 34.81 | 150.09       | 60.89        | 71.16     | 27.92      |\n| 4     | 330.69  | 34.56 | 186.63       | 74.15        | 86.86     | 34.37      |\n| 8     | 378.88  | 35.46 | 204.89       | 77.05        | 91.10     | 36.93      |\n| 16    | 424.00  | 36.49 | 211.55       | 78.39        | 91.23     | 37.34      |\n| 32    | 481.95  | 37.23 | 213.71       | 78.23        | 91.68     | 37.26      |\n\n\nAWS EC2 C5.2xlarge:\n\n| Batch | Alexnet | VGG 16   | Inception-BN | Inception-v3 | Resnet 50 | Resnet 152 |\n|-------|---------|-------|--------------|--------------|-----------|------------|\n| 1     | 131.01  | 15.67 | 78.75        | 31.12        | 37.30     | 14.75      |\n| 2     | 182.29  | 18.01 | 98.59        | 39.13        | 45.98     | 17.84      |\n| 4     | 189.31  | 18.25 | 110.26       | 41.35        | 49.21     | 19.32      |\n| 8     | 211.75  | 18.57 | 115.46       | 42.53        | 49.98     | 19.81      |\n| 16    | 236.06  | 19.11 | 117.18       | 42.59        | 50.20     | 19.92      |\n| 32    | 261.13  | 19.46 | 116.20       | 42.72        | 49.95     | 19.80      |\n\n\nAWS EC2 C5.xlarge:\n\n| Batch | Alexnet | VGG 16  | Inception-BN | Inception-v3 | Resnet 50 | Resnet 152 |\n|-------|---------|------|--------------|--------------|-----------|------------|\n| 1     | 36.64   | 3.93 | 27.06        | 10.09        | 12.98     | 5.06       |\n| 2     | 49.21   | 4.49 | 29.67        | 10.80        | 12.94     | 5.14       |\n| 4     | 50.12   | 4.50 | 30.31        | 10.83        | 13.17     | 5.19       |\n| 8     | 54.71   | 4.58 | 30.22        | 10.89        | 13.19     | 5.20       |\n| 16    | 60.23   | 4.70 | 30.20        | 10.91        | 13.23     | 5.19       |\n| 32    | 66.37   | 4.76 | 30.10        | 10.90        | 13.22     | 5.15       |\n\n\n## Other CPU\n\nIf using CPUs (not just Intel CPUs -- ARMs also), NNPACK can improve the running performance with 2x~7x, please check [nnpack.md](nnpack) for details.\n\n## Nvidia GPU\n\n`cuDNN` typically accelerates _MXNet_ performance on NVIDIA GPUs significantly,\nespecially for convolution layers.\nWe suggest always checking to make sure that a recent cuDNN version is used.\n\nSetting the environment `export MXNET_CUDNN_AUTOTUNE_DEFAULT=1` sometimes also helps.\n\nWe show results when using various GPUs including K80 (EC2 p2.2xlarge), M60 (EC2 g3.4xlarge),\nand V100 (EC2 p3.2xlarge).\n\n### Scoring results\n\nBased on\n[example/image-classification/benchmark_score.py](https://github.com/apache/mxnet/blob/master/example/image-classification/benchmark_score.py)\nand  MXNet-1.2.0.rc1, with cuDNN 7.0.5\n\n- K80 (single GPU)\n\n| Batch | Alexnet | VGG 16    | Inception-BN | Inception-v3 | Resnet 50 | Resnet 152 |\n|-------|---------|--------|--------------|--------------|-----------|------------|\n| 1     | 243.93  | 43.59  | 68.62        | 35.52        | 67.41     | 23.65      |\n| 2     | 338.16  | 49.14  | 113.41       | 56.29        | 93.35     | 33.88      |\n| 4     | 478.92  | 53.44  | 159.61       | 74.43        | 119.18    | 45.23      |\n| 8     | 683.52  | 70.50  | 190.49       | 86.23        | 131.32    | 50.54      |\n| 16    | 1004.66 | 109.01 | 254.20       | 105.70       | 155.40    | 62.55      |\n| 32    | 1238.55 | 114.98 | 285.49       | 116.79       | 159.42    | 64.99      |\n| 64 | 1346.72 | 123.56 | 308.73 | 122.21 | 167.58 | 70.21 |\n| 128 | 1416.91 | OOM | 320.98 | 123.11 | 171.55 | 71.85 |\n| 256 | 1462.97 | OOM | 329.16 | 127.53 | 153.01 | 57.23 |\n\n- M60\n\n| Batch | Alexnet | VGG 16    | Inception-BN | Inception-v3 | Resnet 50 | Resnet 152 |\n|-------|---------|--------|--------------|--------------|-----------|------------|\n| 1     | 243.49  | 59.95  | 101.97       | 48.30        | 95.46     | 39.29      |\n| 2     | 491.04  | 69.14  | 170.35       | 80.27        | 142.61    | 60.17      |\n| 4     | 711.54  | 78.94  | 257.89       | 123.09       | 182.36    | 76.51      |\n| 8     | 1077.73 | 109.34 | 343.42       | 152.82       | 208.74    | 87.27      |\n| 16    | 1447.21 | 144.93 | 390.25       | 166.32       | 220.73    | 92.41      |\n| 32    | 1797.66 | 151.86 | 416.69       | 176.56       | 230.19    | 97.03      |\n| 64 | 1779.38 | 150.18 | 427.51 | 183.47 | 239.12 | 101.59 |\n| 128 | 1787.36 | OOM | 439.04 | 185.29 | 243.31 | 103.39 |\n| 256 | 1899.10 | OOM | 450.22 | 183.42 | 242.36 | 100.98 |\n\n\n- V100\n\n| Batch | Alexnet | VGG 16    | Inception-BN | Inception-v3 | Resnet 50 | Resnet 152 |\n|-------|---------|--------|--------------|--------------|-----------|------------|\n| 1     | 659.51  | 205.16 | 157.37 | 87.71 | 162.15    | 61.38      |\n| 2     | 1248.21 | 265.40 | 297.34 | 159.24 | 293.74    | 116.30     |\n| 4     | 2122.41 | 333.97 | 520.91 | 279.84 | 479.14    | 195.17     |\n| 8     | 3894.30 | 420.26 | 898.09 | 455.03 | 699.39    | 294.19     |\n| 16    | 5815.58 | 654.16 | 1430.97 | 672.54 | 947.45    | 398.79     |\n| 32    | 7906.09 | 708.43 | 1847.26 | 814.59 | 1076.81   | 451.82     |\n| 64 | 9486.26 | 701.59 | 2134.89 | 899.01 | 1168.37 | 480.44 |\n| 128 | 10177.84 | 703.30 | 2318.32 | 904.33 | 1233.15 | 511.79 |\n| 256 | 10990.46 | 473.62 | 2425.28 | 960.20 | 1155.07 | 449.35 |\n\nBelow is the performance result on V100 using float 16.\n\n| Batch | VGG 16  | Inception-BN | Inception-v3 | Resnet 50 | Resnet 152 |\n| ----- | ------- | ------------ | ------------ | --------- | ---------- |\n| 1     | 276.29  | 155.53       | 150.99       | 270.89    | 96.79      |\n| 2     | 476.91  | 296.45       | 282.02       | 493.99    | 176.88     |\n| 4     | 711.92  | 525.05       | 492.45       | 851.15    | 321.52     |\n| 8     | 1047.11 | 900.26       | 807.94       | 1282.36   | 517.66     |\n| 16    | 1299.88 | 1441.41      | 1192.21      | 1722.97   | 724.57     |\n| 32    | 1486.63 | 1854.30      | 1512.08      | 2085.51   | 887.34     |\n| 64    | 1219.65 | 2138.61      | 1687.35      | 2341.67   | 1002.90    |\n| 128   | 1169.81 | 2317.39      | 1818.26      | 2355.04   | 1046.98    |\n| 256   | 764.16  | 2425.16      | 1653.74      | 1991.88   | 976.73     |\n\n### Training results\n\nBased on\n[example/image-classification/train_imagenet.py](https://github.com/apache/mxnet/blob/master/example/image-classification/train_imagenet.py)\nand  MXNet-1.2.0.rc1, with CUDNN 7.0.5. The benchmark script is available at\n[here](https://github.com/mli/mxnet-benchmark/blob/master/run_vary_batch.sh),\nwhere the batch size for Alexnet is increased by 16x.\n\n- K80 (single GPU)\n\n| Batch | Alexnet(\\*16) | Inception-v3 | Resnet 50 |\n| --- | --- | --- | --- |\n|   1 | 300.30 | 10.48 | 15.61 |\n|   2 | 406.08 | 16.00 | 23.88 |\n|   4 | 461.01 | 22.10 | 32.26 |\n|   8 | 484.00 | 26.80 | 39.42 |\n|  16 | 490.45 | 31.62 | 46.69 |\n|  32 | 414.72 | 33.78 | 49.48 |\n\n- M60\n\n| Batch | Alexnet(\\*16) | Inception-v3 | Resnet 50 |\n| --- | --- | --- | --- |\n|   1 | 380.96 | 14.06 | 20.55 |\n|   2 | 530.53 | 21.90 | 32.65 |\n|   4 | 600.17 | 31.96 | 45.57 |\n|   8 | 633.60 | 40.58 | 54.92 |\n|  16 | 639.37 | 46.88 | 64.44 |\n|  32 | 576.54 | 50.05 | 68.34 |\n\n- V100\n\n| Batch | Alexnet(\\*16) | Inception-v3 | Resnet 50 |\n| --- | --- | --- | --- |\n|   1 | 1629.52 | 21.83 | 34.54 |\n|   2 | 2359.73 | 40.11 | 65.01 |\n|   4 | 2687.89 | 72.79 | 113.49 |\n|   8 | 2919.02 | 118.43 | 174.81 |\n|  16 | 2994.32 | 173.15 | 251.22 |\n|  32 | 2585.61 | 214.48 | 298.51 |\n| 64 | 1984.21 | 247.43 | 343.19 |\n| 128 | OOM | 253.68 | 363.69 |\n\n## Multiple Devices\n\nIf more than one GPU or machine are used, MXNet uses `kvstore` to communicate data.\nIt's critical to use the proper type of `kvstore` to get the best performance.\nRefer to [Distributed Training](https://mxnet.apache.org/api/faq/distributed_training.html) for more\ndetails.\n\nBesides, we can use [tools/bandwidth](https://github.com/apache/mxnet/tree/master/tools/bandwidth)\nto find the communication cost per batch.\nIdeally, the communication cost should be less than the time to compute a batch.\nTo reduce the communication cost, we can consider:\n\n- Exploring different `--kv-store` options.\n- Increasing the batch size to improve the computation to communication ratio.\n\nFinally, MXNet is integrated with other distributed training frameworks, including [horovod](https://github.com/apache/mxnet/tree/master/example/distributed_training-horovod) and [BytePS](https://github.com/bytedance/byteps#use-byteps-in-your-code).\n\n## Input Data\n\nTo make sure you're handling input data in a reasonable way consider the following:\n\n* Data format: If you are using the `rec` format, then everything should be fine.\n* Decoding: By default, _MXNet_ uses 4 CPU threads for decoding images.\nThis is often sufficient to decode more than 1K images per second.\nIf you are using a low-end CPU or your GPUs are very powerful, you can increase the number of threads.\n* Storage location. Any local or distributed file system (HDFS, Amazon S3) should be fine.\nIf multiple devices read the data from the shared network file system (NFS) at the same time, problems might occur.\n* Use a large batch size. We often choose the largest one that fits into GPU memory.\nA value that's too large can slow down convergence.\nFor example, the safe batch size for CIFAR 10 is approximately 200, while for ImageNet 1K, the batch size can exceed 1K.\n\n## Profiler\n\n_MXNet_ has a built-in profiler\nthat gives detailed information about execution time at the operator level.\nThis feature complements general profiling tools like _nvprof_ and _gprof_\nby summarizing at the operator level, instead of a function, kernel, or instruction level.\n\nThe profiler can be turned on with an [environment variable]({{'/api/faq/env_var#control-the-profiler' | relative_url}})\nfor an entire program run, or programmatically for just part of a run. Note that by default the profiler hides the details of each individual operator, and you can reveal the details by setting environment variables `MXNET_EXEC_BULK_EXEC_INFERENCE`, `MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN` and `MXNET_EXEC_BULK_EXEC_TRAIN` to 0.\nSee [example/profiler](https://github.com/apache/mxnet/tree/master/example/profiler)\nfor complete examples of how to use the profiler in code, or [this tutorial](https://mxnet.apache.org/api/python/docs/tutorials/performance/backend/profiler.html) on how to profile MXNet performance.\n\nBriefly, the Python code looks like:\n\n```python\n    # wait for previous operations to complete\n    mx.nd.waitall() \n    mx.profiler.set_config(profile_all=True, aggregate_stats=True, filename='profile_output.json')\n    mx.profiler.set_state('run')\n\n    # Code to be profiled goes here...\n\n    # wait for previous operations to complete\n    mx.nd.waitall() \n    mx.profiler.set_state('stop')\n```\n\nAfter the program finishes, navigate to your browser's tracing (Example - chrome://tracing in a Chrome browser) and load the `profile_output.json` file output by the profiler to inspect the results.\n\n![MLP Profile](https://cloud.githubusercontent.com/assets/17693755/18035938/0a43484a-6d93-11e6-80d4-241c6ca552ea.png)\n\nNote that the output file can grow extremely large, so this approach is not recommended for general use.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/faq/recordio.md",
    "content": "---\nlayout: page_category\ntitle: Create a Dataset Using RecordIO\ncategory: faq\nfaq_c: Speed\nquestion: How can I create a .rec dataset ?\npermalink: /api/faq/recordio\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n## Create a Dataset Using RecordIO\n\nRecordIO implements a file format for a sequence of records. We recommend storing images as records and packing them together. The benefits include:\n\n* Storing images in a compact format--e.g., JPEG, for records--greatly reduces the size of the dataset on the disk.\n* Packing data together allows continuous reading on the disk.\n* RecordIO has a simple way to partition, simplifying distributed setting. We provide an example later.\n\nWe provide two tools for creating a RecordIO dataset.\n\n* [im2rec.cc](https://github.com/apache/incubator-mxnet/blob/master/tools/im2rec.cc) - implements the tool using the C++ API.\n* [im2rec.py](https://github.com/apache/incubator-mxnet/blob/master/tools/im2rec.py) - implements the tool using the Python API.\n\nBoth provide the same output: a RecordIO dataset.\n\n### Prerequisites\n\nDownload the data. You don't need to resize the images manually. You can use ```im2rec``` to resize them automatically. For details, see the \"Extension: Using Multiple Labels for a Single Image,\" later in this topic.\n\n### Step 1. Make an Image List File\n\n* Note that the im2rec.py provides a param `--list` to generate the list for you, but im2rec.cc doesn't support it.\n\nAfter you download the data, you need to make an image list file.  The format is:\n\n```\ninteger_image_index \\t label_index \\t path_to_image\n```\nTypically, the program takes the list of names of all of the images, shuffles them, then separates them into two lists: a training filename list and a testing filename list. Write the list in the right format.\nThis is an example file:\n\n```bash\n95099  464.000000     n04467665_17283.JPEG\n10025081        412.000000     ILSVRC2010_val_00025082.JPEG\n74181   789.000000     n01915811_2739.JPEG\n10035553        859.000000     ILSVRC2010_val_00035554.JPEG\n10048727        929.000000     ILSVRC2010_val_00048728.JPEG\n94028   924.000000     n01980166_4956.JPEG\n1080682 650.000000     n11807979_571.JPEG\n972457  633.000000     n07723039_1627.JPEG\n7534    11.000000      n01630670_4486.JPEG\n1191261 249.000000     n12407079_5106.JPEG\n```\n\n### Step 2. Create the Binary File\n\nTo generate a binary image, use `im2rec` in the tool folder. `im2rec` takes the path of the `image list file` you generated, the `root path` of the images, and the `output file path` as input. This process usually takes several hours, so be patient.\n\nSample command:\n\n```bash\n./bin/im2rec image.lst image_root_dir output.bin resize=256\n```\nFor more details, run ```./bin/im2rec```.\n\n### Extension: Multiple Labels for a Single Image\n\nThe `im2rec` tool and `mx.io.ImageRecordIter` have multi-label support for a single image.\nFor example, if you have four labels for a single image, you can use the following procedure to use the RecordIO tools.\n\n1. Write the image list files as follows:\n\n```\ninteger_image_index \\t label_1 \\t label_2 \\t   label_3 \\t label_4 \\t path_to_image\n```\n\n2. Run `im2rec`, adding a 'label_width=4' to the command argument, for example:\n\n```bash\n./bin/im2rec image.lst image_root_dir output.bin resize=256 label_width=4\n```\n\n3. In the iterator generation code, set `label_width=4` and `path_imglist=<<The PATH TO YOUR image.lst>>`, for example:\n\n```python\ndataiter = mx.io.ImageRecordIter(\n  path_imgrec=\"data/cifar/train.rec\",\n  data_shape=(3,28,28),\n  path_imglist=\"data/cifar/image.lst\",\n  label_width=4\n)\n```\n"
  },
  {
    "path": "docs/static_site/src/pages/api/faq/s3_integration.md",
    "content": "---\nlayout: page_category\ntitle: Use data from S3 for training\ncategory: faq\nfaq_c: Deployment Environments\nquestion: How to use data from S3 for training?\npermalink: /api/faq/s3_integration\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Use data from S3 for training\n\nAWS S3 is a cloud-based object storage service that allows storage and retrieval of large amounts of data at a very low cost. This makes it an attractive option to store large training datasets. MXNet is deeply integrated with S3 for this purpose.\n\nAn S3 protocol URL (like `s3://bucket-name/training-data`) can be provided as a parameter for any data iterator that takes a file path as input. For example,\n\n```\ndata_iter = mx.io.ImageRecordIter(\n    path_imgrec=\"s3://bucket-name/training-data/caltech_train.rec\",\n    data_shape=(3, 227, 227),\n    batch_size=4,\n    resize=256)\n```\nFollowing are detailed instructions on how to use data from S3 for training.\n\n## Step 1: Build MXNet with S3 integration enabled\n\nFollow instructions [here]({{'/get_started'|relative_url}}) to install MXNet from source with the following additional steps to enable S3 integration.\n\n1. Install `libcurl4-openssl-dev` and `libssl-dev` before building MXNet. These packages are required to read/write from AWS S3.\n2. Set `USE_S3=1` in the configuration file.\n\n## Step 2: Configure S3 authentication tokens\n\nMXNet requires the S3 environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to be set. [Here](https://aws.amazon.com/blogs/security/wheres-my-secret-access-key/) are instructions to get the access keys from AWS console.\n\n```\nexport AWS_ACCESS_KEY_ID=<your-access-key-id>\nAWS_SECRET_ACCESS_KEY=<your-secret-access-key>\n```\n\n## Step 3: Upload data to S3\n\nThere are several ways to upload data to S3. One easy way is to use the AWS command line utility. For example, the following `sync` command will recursively copy contents from a local directory to a directory in S3.\n\n```\naws s3 sync ./training-data s3://bucket-name/training-data\n```\n\n## Step 4: Train with data from S3\n\nOnce the data is in S3, it is very straightforward to use it from MXNet. Any data iterator that can read/write data from a local drive can also read/write data from S3.\n\nLet's modify an existing example code in MXNet repository to read data from S3 instead of local disk. [`mxnet/tests/python/train/test_conv.py`](https://github.com/apache/mxnet/blob/master/tests/python/train/test_conv.py) trains a convolutional network using MNIST data from local disk. We'll do the following change to read the data from S3 instead.\n\n```\n~/mxnet$ sed -i -- 's/data\\//s3:\\/\\/bucket-name\\/training-data\\//g' ./tests/python/train/test_conv.py\n\n~/mxnet$ git diff ./tests/python/train/test_conv.py\ndiff --git a/tests/python/train/test_conv.py b/tests/python/train/test_conv.py\nindex 039790e..66a60ce 100644\n--- a/tests/python/train/test_conv.py\n+++ b/tests/python/train/test_conv.py\n@@ -39,14 +39,14 @@ def get_iters():\n\n     batch_size = 100\n     train_dataiter = mx.io.MNISTIter(\n-            image=\"data/train-images-idx3-ubyte\",\n-            label=\"data/train-labels-idx1-ubyte\",\n+            image=\"s3://bucket-name/training-data/train-images-idx3-ubyte\",\n+            label=\"s3://bucket-name/training-data/train-labels-idx1-ubyte\",\n             data_shape=(1, 28, 28),\n             label_name='sm_label',\n             batch_size=batch_size, shuffle=True, flat=False, silent=False, seed=10)\n     val_dataiter = mx.io.MNISTIter(\n-            image=\"data/t10k-images-idx3-ubyte\",\n-            label=\"data/t10k-labels-idx1-ubyte\",\n+            image=\"s3://bucket-name/training-data/t10k-images-idx3-ubyte\",\n+            label=\"s3://bucket-name/training-data/t10k-labels-idx1-ubyte\",\n             data_shape=(1, 28, 28),\n             label_name='sm_label',\n             batch_size=batch_size, shuffle=True, flat=False, silent=False)\n```\n\nAfter the above change `test_conv.py` will fetch data from S3 instead of the local disk.\n\n```\npython ./tests/python/train/test_conv.py\n[21:59:19] src/io/s3_filesys.cc:878: No AWS Region set, using default region us-east-1\n[21:59:21] src/io/iter_mnist.cc:94: MNISTIter: load 60000 images, shuffle=1, shape=(100,1,28,28)\n[21:59:21] src/io/iter_mnist.cc:94: MNISTIter: load 10000 images, shuffle=1, shape=(100,1,28,28)\nINFO:root:Start training with [cpu(0)]\nStart training with [cpu(0)]\nINFO:root:Epoch[0] Resetting Data Iterator\nEpoch[0] Resetting Data Iterator\nINFO:root:Epoch[0] Time cost=11.277\nEpoch[0] Time cost=11.277\nINFO:root:Epoch[0] Validation-accuracy=0.955100\nEpoch[0] Validation-accuracy=0.955100\nINFO:root:Finish fit...\nFinish fit...\nINFO:root:Finish predict...\nFinish predict...\nINFO:root:final accuracy = 0.955100\nfinal accuracy = 0.955100\n```\n"
  },
  {
    "path": "docs/static_site/src/pages/api/faq/security.md",
    "content": "---\nlayout: page_category\ntitle: MXNet Security Best Practices\ncategory: faq\nfaq_c: Security\nquestion: How to run MXNet securely?\npermalink: /api/faq/security\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Reporting a security vulnerability\nThe Apache Software Foundation takes a very active stance in eliminating security problems and denial of service attacks against its products.\n\nWe strongly encourage folks to report such problems to our private security mailing list first, before disclosing them in a public forum.\n\nPlease note that the security mailing list should only be used for reporting undisclosed security vulnerabilities and managing the process of fixing such vulnerabilities. We cannot accept regular bug reports or other queries at this address. All mail sent to this address that does not relate to an undisclosed security problem in our source code will be ignored.\n\n\nQuestions about:\n* if a vulnerability applies to your particular application\n* obtaining further information on a published vulnerability\n* availability of patches and/or new releases\nshould be addressed to the users mailing list. Please see the [mailing lists page](/community/contribute#mxnet-dev-communications) for details of how to subscribe.\n\nThe private security mailing address is: <a href=\"mailto:security@apache.org\">security@apache.org</a> <i class=\"far fa-envelope\">. Feel free to consult the general [Apache Security guide](http://www.apache.org/security/) for further details about the reporting process.\n\n\n# MXNet Security Best Practices\n\nMXNet framework has no built-in security protections. It assumes that the MXNet entities involved in model training and inferencing (hosting) are fully trusted. It also assumes that their communications cannot be eavesdropped or tampered with. MXNet consumers shall ensure that the above assumptions are met.\n\nIn particular the following threat-vectors exist when training using MXNet:\n\n* When running distributed training using MXNet there is no built-in support for authenticating cluster nodes participating in the training job.\n* Data exchange between cluster nodes happens is in plain-text.\n* Using `kvstore.set_optimizer` one can use a custom optimizer to combine gradients. This optimizer code is sent to the server nodes as a pickle file. A server does not perform any further validation of the pickle file and simply executes the code trusting the sender (worker).\n* Since there is no authentication between nodes, a malicious actor running on the same network can launch a Denial of Service (DoS) attack by sending data that can overwhelm/crash a scheduler or other server nodes.\n\nIt is highly recommended that the following best practices be followed when using MXNet:\n\n* Run MXNet with least privilege, i.e. not as root.\n* Run MXNet training jobs inside a secure and isolated environment. If you are using a cloud provider like Amazon AWS, running your training job inside a [private VPC](https://aws.amazon.com/vpc/) is a good way to accomplish this. Additionally, configure your network security settings so as to only allow connections that the cluster nodes require.\n* Make sure no unauthorized actors have physical or remote access to the nodes participating in MXNet training.\n* During training, one can configure MXNet to periodically save model checkpoints. To protect these model checkpoints from unauthorized access, make sure the checkpoints are written out to an encrypted storage volume, and have a provision to delete checkpoints that are no longer needed.\n* When sharing trained models, or when receiving trained models from other parties, ensure that model artifacts are authenticated and integrity protected using cryptographic signatures, thus ensuring that the data received comes from trusted sources and has not been maliciously (or accidentally) modified in transit.\n* By default, mx.random uses a static and fixed seed value. The random utilities in MXNet should therefore never be used to implement any type of security critical functionality where cryptographically secure pseudorandom number generation is required.\n\n# Deployment Considerations\nThe following are not MXNet framework specific threats but are applicable to Machine Learning models in general.\n\n* When deploying high-value, proprietary models for inference, care should be taken to prevent an adversary from stealing the model. The research paper [Stealing Machine Learning Models via Prediction APIs](https://arxiv.org/pdf/1609.02943.pdf) outlines experiments performed to show how an attacker can use a prediction API to leak the ML model or construct a nearly identical replica. A simple way to thwart such an attack is to not expose the prediction probabilities to a high degree of precision in the API response.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/faq/tensor_inspector_tutorial.md",
    "content": "---\nlayout: page_category\ntitle: Use TensorInspector to Help Debug Operators\ncategory: faq\npermalink: /api/faq/tensor_inspector_tutorial\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Use TensorInspector to Help Debug Operators\n\n## Introduction\n\nWhen developing new operators, developers need to deal with tensor objects extensively. This new utility, Tensor Inspector, mainly aims to help developers debug by providing unified interfaces to print, check, and dump the tensor value. To developers' convenience, this utility works for all the three data types: Tensors, TBlobs, and NDArrays. Also, it supports both CPU and GPU tensors.\n\n\n## Usage \n\nThis utility is located in `src/common/tensor_inspector.h`. To use it in any operator code, just include it using `#include \"{path}/tensor_inspector.h\"`, construct an `TensorInspector` object, and call the APIs on that object. You can run any script that uses the operator you just modified then.\n\nThe screenshot below shows a sample usage in `src/operator/nn/convolution-inl.h`.\n\n![tensor_inspector_example_usage](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/faq/tensor_inspector_tutorial/tensor_inspector_example_usage.png)\n\n\n## Functionalities/APIs\n\n### Create a TensorInspector Object from Tensor, TBlob, and NDArray Objects\n\nYou can create a `TensorInspector` object by passing in two things: 1) an object of type `Tensor`, `Tbob`, or `NDArray`, and 2) an `RunContext` object.\n\nEssentially, `TensorInspector` can be understood as a wrapper class around `TBlob`. Internally, the `Tensor`, `Tbob`, or `NDArray` object that you passed in will be converted to a `TBlob` object. The `RunContext` object is used when the tensor is a GPU tensor; in such a case, we need to use the context information to copy the data from GPU memory to CPU/main memory.\n\nFollowing are the three constructors:\n\n```c++\n// Construct from Tensor object\ntemplate<typename Device, int dimension, typename DType MSHADOW_DEFAULT_DTYPE>\nTensorInspector(const mshadow::Tensor<Device, dimension, DType>& ts, const RunContext& ctx);\n\n// Construct from TBlob object\nTensorInspector(const TBlob& tb, const RunContext& ctx);\n\n// Construct from NDArray object\nTensorInspector(const NDArray& arr, const RunContext& ctx):\n```\n\n### Print Tensor Value (Static) \n\nTo print out the tensor value in a nicely structured way, you can use this API:\n\n```c++\nvoid print_string();\n```\n\nThis API will print the entire tensor to `std::cout` and preserve the shape (it supports all dimensions from 1 and up). You can copy the output and interpret it with any `JSON` loader. You can find some useful information about the tensor on the last line of the output. Refer to the case below, we are able to know that this is a float-typed tensor with shape 20x1x5x5.\n\n![tensor_inspector_to_string](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/faq/tensor_inspector_tutorial/tensor_inspector_to_string.png)\n\nIf instead of printing the tensor to `std::cout`, you just need a `string`, you can use this API:\n```c++\nstd::string void to_string();\n```\n\n### Interactively Print Tensor Value (Dynamic) \n\nSometimes at compilation time, you may not know which part of a tensor to inspect. Also, it may be nice to pause the operator control flow to “zoom into” a specific, erroneous part of a tensor multiple times until you are satisfied. In this regard, you can use this API to interactively inspect the tensor:\n\n```c++\nvoid  interactive_print(std::string tag =  \"\") {\n```\n\nThis API will set a \"break point\" in your code. When that \"break point\" is reached, you will enter a loop that will keep asking you for further command input. In the API call, `tag` is an optional parameter to give the call a name, so that you can identify it when you have multiple `interactive_print()` calls in different parts of your code. A visit count will tell you how many times you stepped into this particular \"break point\", should this operator be called more than once. Note that all `interactive_print()` calls are properly locked, so you can use it in many different places without issues.\n\n![tensor_inspector_interactive_print](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/faq/tensor_inspector_tutorial/tensor_inspector_interactive_print.png)\n\nThere are many useful commands available, as described in the previous screenshot: you can type \"e\" to print out the entire tensor, \"d\" to dump the tensor to file (see below), \"b\" to break from this command loop, and \"s\" to skip all future `interactive_print()`. Most importantly, in this screen, you can specify a part of the tensor that you are particularly interested in and want to print out. For example, for this 64x20x24x24 tensor, you can type in \"0, 0\" and presss enter to check the sub-tensor with shape 24x24 at coordinate (0, 0). \n\n### Check Tensor Value\n\nSometimes, developers might want to check if the tensor contains unexpected values which could be negative values, NaNs, infinities or others. To facilitate that, you can use these APIs:\n\n```c++\ntemplate<typename ValueChecker>\nstd::vector<std::vector<int>> check_value(const ValueChecker& checker,\n\t\tbool interactive = false, std::string tag = \"\");\n// OR\nstd::vector<std::vector<int>> check_value(CheckerType ct,\n\t\tbool interactive = false, std::string tag =  \"\");\n```\n\nIn the first API, `ValueChecker checker` is a bool lambda function that takes in a single parameter which is of the same data type as the tensor.  For example:\n\n```c++\n// use the same DType as in the tensor object\n[] (DType x) {return x == 0};\n```\n\nThis checker is called on every value within the tensor. The return of the API is a `vector` of all the coordinates where the checker evaluates to `true`. The coordinates are themselves represented by `vector<int>`. If you set `interactive` to true, you will set a \"break point\" and enter a loop that asks for commands. This is similar to `interactive_print()`. You can type \"p\" to print the coordinates, \"b\" to break from the loop, and \"s\" to skip all future \"break points\" in `interactive_print()`. You can also specify a coordinate to print only a part of the tensor or type \"e\" to print out the entire tensor.  Just like `interactive_print()`, this this interactive screen is also properly locked.\n\n![tensor_inspector_value_check](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/faq/tensor_inspector_tutorial/tensor_inspector_value_check.png)\n\nAlso, there are a bunch of built-int value checkers. Refer to the Enum below:\n\n```c++\nenum  CheckerType {\n\tNegativeChecker, // check if negative\n\tPositiveChecker, // check if positive\n\tZeroChecker, // check for zero\n\tNaNChecker, // check if for NaN, will always return false if DType is not a float type\n\tInfChecker, // check for infinity, will always return false if DType is not a float type\n\tPositiveInfChecker, // check for positive infinity,\n\t\t\t\t\t\t// will always return false if DType is not a float type\n\tNegativeInfChecker, // check for nagative infinity,\n\t\t\t\t\t\t// will always return false if DType is not a float type\n\tFiniteChecker, // check if finite, will always return false if DType is not a float type\n\tNormalChecker, // check if it is neither infinity nor NaN\n\tAbnormalChecker, // chekck if it is infinity or nan\n};\n```\n\nRemember the second API?\n\n```c++\nstd::vector<std::vector<int>> check_value(CheckerType ct,\n\t\tbool interactive = false, std::string tag =  \"\");\n```\n\nYou can simply pass in a value from `CheckerType` where you would have passed in your own lambda if you were using the first API. Note that it's the developer's responsibility to pass in a valid value checker.\n\n### Dump Tensor Value\n\nSometimes, you might want to dump the tensor to a file in binary mode. Then, you might want to use a python script to further analyze the tensor value. Or, you might do that simply because a binary dump has better precision and is faster to load than the output copy-pasted from `print_string()` and loaded as a `JSON` string. Either way, you can use this API:\n\n```c++\nvoid dump_to_file(std::string tag);\n```\n\nThis API will create a file with name  \"{tag}_{visit_count}.npy\", where tag is the name that we give to the call, and visit is the visit count, should the operated be called more than once.\n\nThe output format is `.npy`, version 1.0. This is the Numpy format and we can easily load it with the following code:\n\n```\nimport numpy as np\na = np.load('abc_1.npy')\nprint(a)\n```\n\nLet's see how it runs:\n\n![tensor_inspector_dump_to_file](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/faq/tensor_inspector_tutorial/tensor_inspector_dump_to_file.png)\n\nNotice: in `interactive_print()`, you could also do value dumping with command \"d\". You will be prompted to enter the `tag` value:\n\n![tensor_inspector_interactive_print](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/faq/tensor_inspector_tutorial/tensor_inspector_interactive_print.png)\n\n### Test Coverage and Limitations\n\nThis utility has been tested on Mac and Ubuntu with and without CUDNN and oneDNN. Supports for `Tensor`, `TBlob`, and `NDArray`, as well as for CPU and GPU have been manually tested. \n\nCurrently, this utility only supports non-empty tensors and tensors with known shapes i.e. `tb_.ndim() > 0`. Also, this utility only supports dense `NDArray` objects, i.e. when the type is `kDefaultStorage`. \n\n"
  },
  {
    "path": "docs/static_site/src/pages/api/faq/using_rtc.md",
    "content": "---\nlayout: page_category\ntitle: Using runtime compilation (RTC) to write CUDA kernels in MXNet\ncategory: faq\nfaq_c: Extend and Contribute to MXNet\nquestion: How do I implement GPU functions in MXNet using RTC?\npermalink: /api/faq/using_rtc\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Using runtime compilation (RTC) to write CUDA kernels in MXNet\n\n## Introduction\n\nCUDA kernel is a function running on the GPU to perform computation. This tutorial assumes the\nreader has a basic knowledge about how to write such kernels.\n\nThere are currently 2 typical ways of writing and launching CUDA kernels in MXNet. The first one is\nto use the `Kernel<...>::Launch()` API, which is suitable for simple elementwise operations and\nenables writing only portion of the kernel, leaving the launch mechanism to MXNet. The\nother one is to write a kernel from scratch and launch it using the `<<<...>>>` method from CUDA.\nStarting from MXNet 2.0, there is a third option - runtime compilation (RTC). This differs from the\nprevious methods (which use kernels compiled ahead of time), as it compiles the needed kernels\nduring runtime of the user script.\n\nIn this tutorial we will cover the reasons for using RTC instead of the other methods, show how to\ndo it, as well as tips on what to keep in mind when doing it.\n\n## Why RTC?\n\n### Problems with kernels compiled ahead of time\n\nThe use of kernels compiled ahead of time in MXNet leads to a few problems, which unfortunately\nare mostly invisible in any single PR, but grow over the course of many contributions and result in\nserious issues.\n\nIn order to understand them, let us look at the typical way kernels are launched in MXNet. This\nexample shows a launch of the simple kernel, taking a single input of type `DType` and producing\nsingle output of type `OType`:\n\n```cpp\nMSHADOW_TYPE_SWITCH(inputs[0].type_flag_, DType, {\n  MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, OType, {\n    Kernel<...>::Launch(s, inputs[0].dptr<DType>(), outputs[0].dptr<OType>());\n  });\n});\n```\n\nThis launch mechanism uses the `MSHADOW_TYPE_SWITCH` macro, which produces a version of the kernel\nfor every possible type. In the case of nested usage (as is the case in the example shown) it\nproduces a version of the kernel for every combination of types. This results in a large number of\nkernels being generated.\n\nAnother factor that multiplies the number of kernels is that different GPU architectures require\ndifferent compiled binaries. Therefore for MXNet to support all of them with a single binary, that\nbinary needs to contain copies of those kernels for each architecure.\n\nThis proliferation of CUDA kernels in the binary leads to multiple issues. The first problem is the\nsize of the MXNet library - each compiled version of the kernel takes some space in the binary,\nwhich is small but multiplied by the number of all versions (which could reach thousands per\nGPU architecture) and GPU architectures. This increase in size led to multiple issues reported with\ndistribution of the MXNet package,\n[building the library](https://github.com/apache/incubator-mxnet/issues/17045) as well as\n[limiting the number of architectures natively\nsupported](https://github.com/apache/incubator-mxnet/pull/18205).\n\nThe second issue is the \"idle\" memory consumption of the MXNet library. In order to efficiently\nlaunch kernels when they are called, CUDA driver needs to transfer them to the GPU memory ahead of\ntime. Since it cannot anticipate which kernels will actually be used, all of the kernels are\ntransferred when the CUDA context is created on a GPU. This means that, even if a user never uses\ne.g. kernel which adds `int8` and `float16` tensors, that kernel still occupies memory on their GPU,\nreducing the amount of memory available for useful work.\n\nThe third issue, mostly affecting MXNet developers, is the compilation time of the MXNet library.\nThe more kernels versions need to be compiled, the more time and hardware resources is needed.\n\n### RTC to the rescue!\n\nAll of the issues mentioned in the previous paragraph are solved when using runtime compilation.\nUsing this paradigm, only the kernels actually invoked in the user script are compiled. They do not\noccupy space in the MXNet binary and there is no unused kernels stored in users' GPU memory.\n\nRTC also enables more features:\n\n - using more information about specific usage of the kernel when compiling it (e.g. using shape\n   information of the inputs) to optimize it better\n - writing kernels accepting any combinations of input and output types\n - (in the future) fusing more operations into the generated kernels.\n\n## RTC for kernel developers\n\n### Example: unary operators\n\nLet us start with an example of the simple kernel written using RTC: a kernel which performs unary\noperation (with a concrete example of sigmoid) on its input. It is not a toy example though: it is\na fully generic kernel, capable of operating on any combination of input and output types, as well\nas applying any unary operator:\n\n```cpp\nstruct UnaryRTCCompute {\n  std::string OP;\n\n  void operator()(const nnvm::NodeAttrs& attrs,\n                  const OpContext& ctx,\n                  const std::vector<TBlob>& inputs,\n                  const std::vector<OpReqType>& req,\n                  const std::vector<TBlob>& outputs);\n};\n\nconst char unary_kernel_fwd[] = R\"code(\n\n__launch_bounds__(kRTCMaxThreadsPerBlock)\n__global__ void unary_kernel(const InputType* input,\n                             const OutputType* output,\n                             const index_t N) {\n  using IType = AccType<InputType>;\n  using OType = AccType<OutputType>;\n\n  for (index_t tid = blockIdx.x * blockDim.x + threadIdx.x;\n       tid < N;\n       tid += gridDim.x * blockDim.x) {\n    const auto input = IType::from(input[i]);\n    const auto temp = OP(input);  // enables returning different type\n\n    if (req == OpReqType::kAddTo) {\n      // temp2 may have a wider type than either temp\n      // or OType\n      const auto temp2 = op::add(temp, OType::from(output[i]));\n      output[i] = OType::to(temp2);\n    } else {\n      output[i] = OType::to(temp);\n    }\n  }\n}\n\n)code\";\n\nvoid UnaryRTCCompute::operator()(const nnvm::NodeAttrs& attrs,\n                                 const OpContext& ctx,\n                                 const std::vector<TBlob>& inputs,\n                                 const std::vector<OpReqType>& req,\n                                 const std::vector<TBlob>& outputs) {\n  using namespace mxnet::common::cuda::rtc;\n  if (req[0] == kNullOp) return;\n  mshadow::Stream<gpu>* s = ctx.get_stream<gpu>();\n  CHECK_EQ(inputs.size(), 1U);\n  CHECK_EQ(outputs.size(), 1U);\n\n  const std::string code = std::string(\"const OpReqType req = \") +\n                           util::to_string(req[0]) +\n                           \";\\n\"\n                           \"#define OP op::\" +\n                           OP +\n                           \"\\n\"\n                           \"using InputType = \" +\n                           common::mshadow_type_info(inputs[0].type_flag_).name +\n                           \";\\n\"\n                           \"using OutputType = \" +\n                           common::mshadow_type_info(outputs[0].type_flag_).name +\n                           \";\\n\";\n\n  std::vector<const void*> args;\n  const index_t size = outputs[0].Size();\n  args.emplace_back(&(inputs[0].dptr_));\n  args.emplace_back(&(outputs[0].dptr_));\n  args.emplace_back(&size);\n\n  auto kernel = get_function(code, \"unary_kernel\", unary_kernel_fwd,\n                             ctx.run_ctx.get_ctx().dev_id);\n\n  const int n_threads = 512;\n  const index_t n_blocks = (size + n_threads - 1) / n_threads;\n  const int shared_memory_size = 0;\n  launch(kernel, {n_blocks, 1, 1}, {512, 1, 1},\n         shared_memory_size, s, &args);\n}\n\nNNVM_REGISTER_OP(sigmoid)\n.set_attr<FCompute>(\"FCompute<gpu>\", UnaryRTCCompute{\"sigmoid\"});\n```\n\n### Kernels are text...\n\nThe main difference when writing kernels using RTC is that the kernel code becomes the text string.\nThis means that it is possible to change or compose the code at runtime, as is done here:\n\n```cpp\n  const std::string code = std::string(\"const OpReqType req = \") +\n                           util::to_string(req[0]) +\n                           \";\\n\"\n                           \"#define OP op::\" +\n                           OP +\n                           \"\\n\"\n                           \"using InputType = \" +\n                           common::mshadow_type_info(inputs[0].type_flag_).name +\n                           \";\\n\"\n                           \"using OutputType = \" +\n                           common::mshadow_type_info(outputs[0].type_flag_).name +\n                           \";\\n\";\n```\n\nwhere the operation `OP` is also provided as a string in the operator declaration:\n\n```cpp\nNNVM_REGISTER_OP(sigmoid)\n.set_attr<FCompute>(\"FCompute<gpu>\", UnaryRTCCompute{\"sigmoid\"});\n```\n\n### and do not know MXNet source code\n\nHow does the kernel know what operation it should perform? The kernel's source code uses `OP`,\nwhich shows up in the `code` variable and is equal to `op::sigmoid`. Let us compare this to how the\nsame operator is defined for CPU:\n\n```cpp\nMXNET_OPERATOR_REGISTER_UNARY(sigmoid)\n.set_attr<FCompute>(\"FCompute<cpu>\", UnaryOp::Compute<cpu, mshadow_op::sigmoid>)\n```\n\nSince the kernel is compiled at runtime, it does not have access to the rest of the MXNet source\ncode, including `mshadow_op.h`, which defined `mshadow_op::sigmoid`. This means that we need to\nprovide the kernel with definitions of those functions (again, in text string form). Every\nRTC-compiled kernel is prepended with a common header, containing string found in\n`src/common/cuda/rtc/` directory. The `src/common/cuda/rtc/forward_functions-inl.h` file contains\nthe definition of `op::sigmoid`:\n\n```cpp\ntemplate <typename DType>\n__device__ inline DType sigmoid(const DType val) {\n  if (type_util::has_double_or_integral<DType>::value) {\n    return 1./(1 + ::exp(-val));\n  } else {\n    return 1.f/(1 + expf(-val));\n  }\n}\n```\n\n### Handling of data types\n\nMXNet has support for many datatypes. Some of those datatypes, like `float16`, `int8` or `bool` are\nuseful when storing the results, but in many computations they are too limiting as they can easily\noverflow in the intermediate stages. That is why in the example we use `AccType<T>` class - it\nprovides an accumulation type, that is potentially larger than the storage type - for example,\n`AccType<float16>::type` is `float32`. It also provides special loading and storing functions:\n`AccType<T>::from()` and `AccType<T>::to()`.\n\nOne of the features of RTC-enabled kernels is to be able to accommodate any combination of the\ninput and output datatypes. Using `auto` as the output type of the intermediate steps helps with,\nespecially since many binary operators return a mixed type:\n\n```cpp\ntemplate <typename DType, typename DType2>\n__device__ inline typename type_util::mixed_type<DType, DType2>::type\nadd(const DType a, const DType2 b) {\n  return a + b;\n}\n```\n\n`mixed_type<T, U>::type` is a type capable of storing value of the operation between 2 types `T` and\n`U` - e.g. `mixed_type<float64, float32>::type = float64` and `mixed_type<float32, int32>::type =\nfloat32`.\n\n### Compiling and launching RTC kernels\n\nThe kernel code stored in `unary_kernel_fwd` is generic and relies on multiple names to be defined,\nlike `req`, `OP` or `InputType`. This is handled in the specific operator using the kernel by\ndefining a set of parameters that will be concatenated to the code during compilation:\n\n```cpp\n  const std::string code = std::string(\"const OpReqType req = \") +\n                           util::to_string(req[0]) +\n                           \";\\n\"\n                           \"#define OP op::\" +\n                           OP +\n                           \"\\n\"\n                           \"using InputType = \" +\n                           common::mshadow_type_info(inputs[0].type_flag_).name +\n                           \";\\n\"\n                           \"using OutputType = \" +\n                           common::mshadow_type_info(outputs[0].type_flag_).name +\n                           \";\\n\";\n```\n\nIn order to compile the kernel, the `mxnet::common::cuda::rtc::get_function` method is used:\n\n```cpp\n  auto kernel = get_function(code, \"unary_kernel\", unary_kernel_fwd,\n                             ctx.run_ctx.get_ctx().dev_id);\n```\n\nIn order to eliminate overheads coming from the compilation, it uses cache of kernels, with a key\nbeing the name of the kernel (`\"unary_kernel\"` in our case) and the set of parameters (`code` in our\ncase). If the kernel is already in cache, it is returned, otherwise compilation takes place. If it\nfails, the full source code is saved to disk and the MXNet error with the compilation log is\ngenerated.\n\nTo launch the kernel, the `mxnet::common::cuda::rtc::launch` method is used:\n\n```cpp\n  launch(kernel, {n_blocks, 1, 1}, {512, 1, 1},\n         shared_memory_size, s, &args);\n```\n\nIt takes the kernel object, grid and block dimensions, size of dynamic shared memory, stream and\nkernel parameters.\n\n## Other features enabled by RTC\n\n### Vectorization\n\nThe actual kernel used for application of unary operator in MXNet looks slightly different compared\nto the simple example shown in the previous paragraph. Differences come from using vectorization.\nThis means, that instead of reading (or writing) 1 element at a time, kernel instead accesses\nmultiple array elements at once. This is beneficial, especially when dealing with smaller\ntypes like `float16` or `int8`. Accessing those small types one by one is inefficient and does not\nsaturate the memory bandwidth of the GPU, so using vector accesses improves achieved memory\nbandwidth.\n\n```cpp\n\n// excerpt from src/operator/tensor/elemwise_unary_op.h\nstruct UnaryRTCCompute {\n  std::string OP;\n\n  void operator()(const nnvm::NodeAttrs& attrs,\n                  const OpContext& ctx,\n                  const std::vector<TBlob>& inputs,\n                  const std::vector<OpReqType>& req,\n                  const std::vector<TBlob>& outputs);\n};\n\n// excerpt from src/operator/tensor/elemwise_unary_op.cc\nstruct unary_kernel_params {\n  const void *inputs[1];\n  void *outputs[1];\n};\n\nconst char unary_kernel_fwd[] = R\"code(\n\nstruct unary_kernel_params {\n  const void *inputs[1];\n  void *outputs[1];\n};\n\n__launch_bounds__(kRTCMaxThreadsPerBlock)\n__global__ void unary_kernel(const unary_kernel_params params,\n                             const index_t lead_dim,\n                             const index_t other_dim,\n                             const index_t N,\n                             const index_t num_aligned_elements) {\n  using namespace vector;\n  VectorizedLoader<InputType0, nvec, aligned> loader(\n    reinterpret_cast<const InputType0*>(params.inputs[0]), N);\n  VectorizedStorer<OutputType0, nvec, aligned> storer(\n    reinterpret_cast<OutputType0*>(params.outputs[0]), N);\n\n  using IType = AccType<InputType0>;\n  using OType = AccType<OutputType0>;\n\n  const index_t M = num_aligned_elements;\n\n  for (index_t tid = blockIdx.x * blockDim.x + threadIdx.x;\n       tid < M;\n       tid += gridDim.x * blockDim.x) {\n    loader.load(tid, N);\n    if (req == OpReqType::kAddTo) {\n      storer.load(tid, N);\n    }\n#pragma unroll\n    for (int i = 0; i < nvec; ++i) {\n      const auto input = IType::from(loader.separate()[i]);\n      const auto temp = OP(input);  // enables returning different type\n\n      if (req == OpReqType::kAddTo) {\n        // temp2 may have a wider type than either temp\n        // or OType\n        const auto temp2 = op::add(temp, OType::from(storer.separate()[i]));\n        storer.separate()[i] = OType::to(temp2);\n      } else {\n        storer.separate()[i] = OType::to(temp);\n      }\n    }\n    storer.store(tid, N);\n  }\n}\n\n)code\";\n\nvoid UnaryRTCCompute::operator()(const nnvm::NodeAttrs& attrs,\n                                 const OpContext& ctx,\n                                 const std::vector<TBlob>& inputs,\n                                 const std::vector<OpReqType>& req,\n                                 const std::vector<TBlob>& outputs) {\n  using namespace mxnet::common::cuda::rtc;\n  if (req[0] == kNullOp) return;\n  mshadow::Stream<gpu>* s = ctx.get_stream<gpu>();\n  CHECK_EQ(inputs.size(), 1U);\n  CHECK_EQ(outputs.size(), 1U);\n\n  const std::string code = std::string(\"const OpReqType req = \") +\n                           util::to_string(req[0]) +\n                           \";\\n\"\n                           \"#define OP op::\" +\n                           OP +\n                           \"\\n\";\n  const int nvec = outputs[0].type_flag_ == mshadow::kFloat64 ? 2 : 4;\n\n  const index_t size = outputs[0].Size();\n  unary_kernel_params params = { {inputs[0].dptr_},\n                                 {outputs[0].dptr_} };\n\n  VectorizedKernelRTCLauncher(code, \"unary_kernel\",\n                              unary_kernel_fwd, nvec,\n                              size, 1, s, params,\n                              inputs, outputs,\n                              ctx.run_ctx.get_ctx().dev_id);\n}\n\n// excerpt from src/operator/tensor/elemwise_unary_op_basic.cu\nNNVM_REGISTER_OP(sigmoid)\n.set_attr<FCompute>(\"FCompute<gpu>\", UnaryRTCCompute{\"sigmoid\"});\n```\n\nRTC implementation in MXNet provides a few useful helper functions and classes, which simplify the\nprocess of writing and launching kernels using vectorization. For accessing the memory using\nvectorization, 2 classes are provided, used in this kernel to access input and output array:\n\n```cpp\n  VectorizedLoader<InputType0, nvec, aligned> loader(\n    reinterpret_cast<const InputType0*>(params.inputs[0]), N);\n  VectorizedStorer<OutputType0, nvec, aligned> storer(\n    reinterpret_cast<OutputType0*>(params.outputs[0]), N);\n```\n\nThe `loader` object accesses `params.inputs[0]` pointer to array of N elements having type\n`InputType0` (which is the name assigned to the type of the first input by the\n`VectorizedKernelRTCLauncher`, which is the helper launcher function). It loads `nvec` elements at\na time and has additional `aligned` option, which is also set by the `VectorizedKernelRTCLauncher`.\nSimilarly `storer` object is used to write data of type `OutputType0` to `params.outputs[0]`.\n\nThe kernel using `VectorizedKernelRTCLauncher` needs to have specific parameters:\n\n```cpp\n__global__ void unary_kernel(const unary_kernel_params params,      // kernel-specific parameters\n                             const index_t lead_dim,                // lead dimension of the tensor\n                             const index_t other_dim,               // size of the other dimensions\n                             const index_t N,                       // total number of elements\n                             const index_t num_aligned_elements) {  // number of vector elements in\n                                                                    // lead dimension\n```\n"
  },
  {
    "path": "docs/static_site/src/pages/api/faq/why_mxnet.md",
    "content": "---\nlayout: page_category\ntitle: Why MXNet came to be?\ncategory: faq\nfaq_c: Extend and Contribute to MXNet\nquestion: Why was MXNet developed in the first place ?\npermalink: /api/faq/why_mxnet\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Why was MXNet developed in the first place ?\n\nProbably, if you've stumbled upon this page, you've heard of _deep learning_.\nDeep learning denotes the modern incarnation of neural networks,\nand it's the technology behind recent breakthroughs\nin self-driving cars, machine translation, speech recognition and more.\nWhile widespread interest in deep learning took off in 2012,\ndeep learning has become an indispensable tool for countless industries.\n\n![alt text](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/get-started/image-classification.png)\n\nIt might not come as a surprise that researchers\nhave investigated neural networks for decades.\nWarren McCulloch and Walter Pitts\nsuggested the forerunner of today's artificial neurons back in 1943.\nEach neuron is connected to other neurons along _edges_, analogous to the synapses that connect real neurons.\nAnd associated with each edge is a _weight_ that indicates whether the connection is excitatory or inhibitatory and the strength of the connection.\n\n![alt_text](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/get-started/artificial-neuron-2.png)\n\nIn the 1980s, the modern version of neural networks took shape.\nResearchers arranged artificial neurons into _layers_.\nNeurons in any layer get input from the neurons in the layers below them.\nAnd, in turn, their output feeds into the neurons in the layer above.\nTypically, the lowest layer represents the _input_ to a neural network.\nAfter computing the values of each layer, the _output_ values are read out from the topmost layer.\nThe behavior of the network is determined by the setting of the weights.\nAnd the process of _learning_ in neural networks\nis precisely the process of searching for good settings of these _weights_.\n\nAll that we need is an algorithm that tells us how to perform this search.\nAnd since David Rumelhart and colleagues\nintroduced the _backpropagation_ learning algorithm to train neural networks,\nnearly all the major ideas have been in place.\nStill, for many years neural networks took a backseat\nto classical statistical methods like logistic regression and support vector machines (SVMs).\nSo you might reasonably ask, what's changed to garner such interest?\n\n## Scale and Computation\nThe two biggest factors driving innovation in deep learning now are data and computation.\nWith distributed cloud computing and parallelism across GPU cores,\nwe can train models millions of times faster than researchers could in the 1980s.\nThe availability of large, high-quality datasets is another factor driving the field forward.\nIn the 1990s, the best datasets in computer vision had thousands of low-resolution images and ground truth assignments to a small number of classes.\nToday, researchers cut their teeth on ImageNet, a massive dataset containing millions of high-resolution images from a thousand distinct classes.\nThe falling price of storage and high network bandwidth\nmake it affordable to work with big data at will.\n\nIn this new world, with bigger datasets and abundant computation,\nneural networks dominate on most pattern recognition problems.\nOver the last five years, neural networks have come to dominate on nearly every problem in computer vision,\nreplacing classical models and hand-engineered features.\nSimilarly, nearly every production speech recognition system now relies on neural networks,\nwhere replacing the hidden Markov models that previously held sway.\n\n![alt text](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/get-started/nvidia-gpus.jpg)\n\nWhile GPUs and clusters present a huge opportunity for accelerating neural network training,\nadapting traditional machine learning code\nto take advantage of these resources can be challenging.\nThe familiar scientific computing stacks (Matlab, R, or NumPy & SciPy)\ngive no straight-forward way to exploit these distributed resources.\n\nAcceleration libraries like _MXNet_ offer powerful tools\nto help developers exploit the full capabilities of GPUs and cloud computing.\nWhile these tools are generally useful and applicable to any mathematical computation, _MXNet_ places a special emphasis on speeding up the development and deployment of large-scale deep neural networks. In particular, we offer the following capabilities:\n* __Device Placement:__ With _MXNet_, it's easy to specify where each data structures should live.\n* __Multi-GPU training__: _MXNet_ makes it easy to scale computation with number of available GPUs.\n* __Automatic differentiation__: _MXNet_ automates the derivative calculations that once bogged down neural network research.\n* __Optimized Predefined Layers__: While you can code up your own layers in _MXNet_, the predefined layers are optimized for speed, outperforming competing libraries.\n\n## Deep Nets on Fast Computers\nWhile MXNet can accelerate any numerical computation,\nwe developed the library with neural networks in mind.\nHowever you plan to use MXNet, neural networks make for a powerful motivating example to display MXNet's capabilities.\n\nNeural networks are just functions for transforming input arrays `X` into output arrays `Y`.\nIn the case of image classification, `X` might represent the pixel values of an image, and `Y` might represent the corresponding probabilities that the image belongs to each of `10` classes.\nFor language translation, `X` and `Y` both might denote sequences of words. We'll revisit the way you might represent sequences in subsequent tutorials - so for now it's safe to think of `X` and `Y` as fixed length vectors.\n\nTo perform this mapping, neural networks stack _layers_ of computation. Each layer consists of a linear function followed by a nonlinear transformation. In _MXNet_ we might express this as:\n```python\nhidden_linear = mx.sym.dot(X, W)\nhidden_activation = mx.sym.tanh(hidden_linear)\n```\nThe linear transformations consist of multiplication by parameter arrays (`W` above).\nWhen we talk about learning we mean finding the right set of values for `W`.\nWith just one layer, we can implement the familiar family of linear models,\nincluding linear and logistic regression, linear support vector machines (SVMs), and the perceptron algorithm.\nWith more layers and a few clever constraints, we can implement all of today's state-of-the-art deep learning techniques.\n\nOf course, tens or hundreds of matrix multiplications can be computationally taxing.\nGenerally, these linear operations are the computational bottleneck.\nFortunately, linear operators can be parallelized trivially across the thousands of cores on a GPU.\nBut low-level GPU programming requires specialized skills that are not common even among leading researchers in the ML community. Moreover, even for CUDA experts, implementing a new neural network architecture shouldn't require weeks of programming to implement low-level linear algebra operations. That's where _MXNet_ comes in.\n*  _MXNet_ provides optimized numerical computation for GPUs and distributed ecosystems, from the comfort of high-level environments like Python and R\n* _MXNet_ automates common workflows, so standard neural networks can be expressed concisely in just a few lines of code\n\nNow let's take a closer look at the computational demands of neural networks\nand give a sense of how _MXNet_ helps us to write better, faster, code.\nSay we have a neural network trained to recognize spam from the content of emails.\nThe emails may be streaming from an online service (at inference time),\nor from a large offline dataset __D__ (at training time).\nIn either case, the dataset typically must be managed by the CPU.\n\n![alt text](https://raw.githubusercontent.com/kevinthesun/web-data/master/mxnet/get-started/architecture.png)\n\nTo compute the transformation of a neural network quickly, we need both the parameters and data points to make it into GPU memory. For any example _X_, the parameters _W_ are the same. Moreover the size of the model tends to dwarf the size of an individual example. So we might arrive at the natural insight that parameters should always live on the GPU, even if the dataset itself must live on the CPU or stream in. This prevents IO from becoming the bottleneck during training or inference.\n\nFortunately, _MXNet_ makes this kind of assignment easy.\n\n```python\nimport mxnet.ndarray as nd\n\nX  = nd.zeros((10000, 40000), mx.cpu(0))           #Allocate an array to store 1000 datapoints (of 40k dimensions) that lives on the CPU\nW1 = nd.zeros(shape=(40000, 1024), mx.gpu(0))      #Allocate a 40k x 1024 weight matrix on GPU for the 1st layer of the net\nW2 = nd.zeros(shape=(1024, 10), mx.gpu(0))         #Allocate a 1024 x 1024 weight matrix on GPU for the 2nd layer of the net\n```\n\n<!-- * __Talk about how mxnet also makes it easy to assign a context (on which device the computation happens__ -->\nSimilarly, _MXNet_ makes it easy to specify the computing device\n\n```python\nwith mx.Context(mx.gpu()):          # Absent this statement, by default, MXNet will execute on CPU\n    h = nd.tanh(nd.dot(X, W1))\n    y = nd.sigmoid(nd.dot(h1, W2))\n```\n\nThus, with only a high-level understanding of how our numerical computation maps onto an execution environment, _MXNet_ allows us to exert fine-grained control when needed.\n\n## Nuts and Bolts\n\nMXNet supports two styles of programming: _imperative programming_ (supported by the _NDArray_ API) and _symbolic programming_ (supported by the _Symbol_ API). In short, imperative programming is the style that you're likely to be most familiar with. Here if A and B are variables denoting matrices, then `C = A + B` is a piece of code that _when executed_ sums the values referenced by `A` and `B` and stores their sum `C` in a new variable. Symbolic programming, on the other hand, allows functions to be defined abstractly through computation graphs. In the symbolic style, we first express complex functions in terms of placeholder values. Then, we can execute these functions by _binding them_ to real values.\n\n\n### Imperative Programming with _NDArray_\nIf you're familiar with NumPy, then the mechanics of _NDArray_ should be old hat. Like the corresponding `numpy.ndarray`, `mxnet.ndarray` (`mxnet.nd` for short) allows us to represent and manipulate multi-dimensional, homogenous arrays of fixed-size components. Converting between the two is effortless:\n\n```python\n# Create a numpy array from an mxnet NDArray\nA_np = np.array([[0,1,2,3,4],[5,6,7,8,9]])\nA_nd = nd.array(A)\n\n# Convert back to a numpy array\nA2_np = A_nd.asnumpy()\n```\n\nOther deep learning libraries tend to rely on NumPy exclusively for imperative programming and the syntax.\nSo you might reasonably wonder, why do we need to bother with _NDArray_?\nPut simply, other libraries only reap the advantages of GPU computing when executing symbolic functions. By using _NDArray_, _MXNet_ users can specify device context and run on GPUs. In other words, _MXNet_ gives you access to the high-speed computation for imperative operations that Tensorflow and Theano only give for symbolic operations.\n\n\n```python\nX = mx.nd.array([[1,2],[3,4]])\nY = mx.nd.array([[5,6],[7,8]])\nresult = X + Y\n```\n\n\n### Symbolic Programming in _MXNet_\n\nIn addition to providing fast math operations through NDArray, _MXNet_ provides an interface for defining operations abstractly via a computation graph.\nWith `mxnet.symbol`, we define operations abstractly in terms of place holders. For example, in the following code `a` and `b` stand in for real values that will be supplied at run time.\nWhen we call `c = a+b`, no numerical computation is performed. This operation simply builds a graph that defines the relationship between `a`, `b` and `c`. In order to perform a real calculation, we need to bind `c` to real values.\n\n```python\na = mx.sym.Variable('a')\nb = mx.sym.Variable('b')\nc = a + b\nexecutor = c.bind(mx.cpu(), {'a': X, 'b': Y})\nresult = executor.forward()\n```\n\nSymbolic computation is useful for several reasons. First, because we define a full computation graph before executing it, _MXNet_ can perform sophisticated optimizations to eliminate unnecessary or repeated work. This tends to give better performance than imperative programming. Second, because we store the relationships between different variables in the computation graph, _MXNet_ can then perform efficient auto-differentiation.\n\n**However** Symbolic programming is error-prone and very slow to iterate with, as the graph needs to be computed before it is processed.\n\n### Gluon for briding the gap between the two\n\n[MXNet Gluon]({{'/api/python'|relative_url}}) aims to bridge the gap between the imperative nature of MXNet and its symbolic capabilities and keep the advantages of both through [hybridization](https://d2l.ai/chapter_computational-performance/hybridize.html).\n\n## Conclusions\nGiven its combination of high performance, clean code, access to a high-level API, and low-level control, _MXNet_ stands out as a unique choice among deep learning frameworks.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/java/docs/tutorials/index.md",
    "content": "---\nlayout: page_landing_tutorials\ntitle: Java Tutorials\npermalink: /api/java/docs/tutorials\ntag: java\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n"
  },
  {
    "path": "docs/static_site/src/pages/api/java/docs/tutorials/ssd_inference.md",
    "content": "---\nlayout: page_api\ntitle: SSD Inference\npermalink: /api/java/docs/tutorials/ssd_inference\nis_tutorial: true\ntag: java\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Multi Object Detection using pre-trained SSD Model via Java Inference APIs\n\nThis tutorial shows how to use MXNet Java Inference APIs to run inference on a pre-trained Single Shot Detector (SSD) Model.\n\nThe SSD model is trained on the Pascal VOC 2012 dataset. The network is a SSD model built on Resnet50 as the base network to extract image features. The model is trained to detect the following entities (classes): ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor']. For more details about the model, you can refer to the [MXNet SSD example](https://github.com/apache/mxnet/tree/master/example/ssd).\n\n## Prerequisites\n\nTo complete this tutorial, you need the following:\n* [MXNet Java Setup on IntelliJ IDEA](mxnet_java_on_intellij) (Optional)\n* [wget](https://www.gnu.org/software/wget/) To download model artifacts\n* SSD Model artifacts\n    * Use the following script to get the SSD Model files :\n```bash\ndata_path=/tmp/resnet50_ssd\nmkdir -p \"$data_path\"\nwget https://s3.amazonaws.com/model-server/models/resnet50_ssd/resnet50_ssd_model-symbol.json -P $data_path\nwget https://s3.amazonaws.com/model-server/models/resnet50_ssd/resnet50_ssd_model-0000.params -P $data_path\nwget https://s3.amazonaws.com/model-server/models/resnet50_ssd/synset.txt -P $data_path\n```\n* Test images  : A few sample images to run inference on.\n    * Use the following script to download sample images :\n```bash\nimage_path=/tmp/resnet50_ssd/images\nmkdir -p \"$image_path\"\ncd $image_path\nwget https://cloud.githubusercontent.com/assets/3307514/20012567/cbb60336-a27d-11e6-93ff-cbc3f09f5c9e.jpg -O dog.jpg\nwget https://cloud.githubusercontent.com/assets/3307514/20012563/cbb41382-a27d-11e6-92a9-18dab4fd1ad3.jpg -O person.jpg\n```\n\nAlternately, you can get the entire SSD Model artifacts + images in one single script from the MXNet Repository by running [get_ssd_data.sh script](https://github.com/apache/mxnet/blob/master/scala-package/examples/scripts/infer/objectdetector/get_ssd_data.sh)\n\n## Time to code!\n1\\. Following the [MXNet Java Setup on IntelliJ IDEA](mxnet_java_on_intellij) tutorial, in the same project `JavaMXNet`, create a new empty class called : `ObjectDetectionTutorial.java`.\n\n2\\. In the `main` function of `ObjectDetectionTutorial.java` define the downloaded model path and the image data paths. This is the same path where we downloaded the model artifacts and images in a previous step.\n\n```java\nString modelPathPrefix = \"/tmp/resnet50_ssd/resnet50_ssd_model\";\nString inputImagePath = \"/tmp/resnet50_ssd/images/dog.jpg\";\n```\n\n3\\. We can run the inference code in this example on either CPU or GPU (if you have a GPU backed machine) by choosing the appropriate context.\n\n```java\n\nList<Context> context = getContext();\n...\n\nprivate static List<Context> getContext() {\nList<Context> ctx = new ArrayList<>();\nctx.add(Context.cpu()); // Choosing CPU Context here\n\nreturn ctx;\n}\n```\n\n4\\. To provide an input to the model, define the input shape to the model and the Input Data Descriptor (DataDesc) as shown below :\n\n```java\nShape inputShape = new Shape(new int[] {1, 3, 512, 512});\nList<DataDesc> inputDescriptors = new ArrayList<DataDesc>();\ninputDescriptors.add(new DataDesc(\"data\", inputShape, DType.Float32(), \"NCHW\"));\n```\n\nThe input shape can be interpreted as follows : The input has a batch size of 1, with 3 RGB channels in the image, and the height and width of the image is 512 each.\n\n5\\. To run an actual inference on the given image, add the following lines to the `ObjectDetectionTutorial.java` class :\n\n```java\nBufferedImage img = ObjectDetector.loadImageFromFile(inputImagePath);\nObjectDetector objDet = new ObjectDetector(modelPathPrefix, inputDescriptors, context, 0);\nList<List<ObjectDetectorOutput>> output = objDet.imageObjectDetect(img, 3); // Top 3 objects detected will be returned\n```\n\n6\\. Let's piece all of the above steps together by showing the final contents of the `ObjectDetectionTutorial.java`.\n\n```java\npackage mxnet;\n\nimport org.apache.mxnet.infer.javaapi.ObjectDetector;\nimport org.apache.mxnet.infer.javaapi.ObjectDetectorOutput;\nimport org.apache.mxnet.javaapi.Context;\nimport org.apache.mxnet.javaapi.DType;\nimport org.apache.mxnet.javaapi.DataDesc;\nimport org.apache.mxnet.javaapi.Shape;\n\nimport java.awt.image.BufferedImage;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\n\npublic class ObjectDetectionTutorial {\n\n    public static void main(String[] args) {\n\n        String modelPathPrefix = \"/tmp/resnet50_ssd/resnet50_ssd_model\";\n\n        String inputImagePath = \"/tmp/resnet50_ssd/images/dog.jpg\";\n\n        List<Context> context = getContext();\n\n        Shape inputShape = new Shape(new int[] {1, 3, 512, 512});\n\n        List<DataDesc> inputDescriptors = new ArrayList<DataDesc>();\n        inputDescriptors.add(new DataDesc(\"data\", inputShape, DType.Float32(), \"NCHW\"));\n\n        BufferedImage img = ObjectDetector.loadImageFromFile(inputImagePath);\n        ObjectDetector objDet = new ObjectDetector(modelPathPrefix, inputDescriptors, context, 0);\n        List<List<ObjectDetectorOutput>> output = objDet.imageObjectDetect(img, 3);\n\n        printOutput(output, inputShape);\n    }\n\n\n    private static List<Context> getContext() {\n        List<Context> ctx = new ArrayList<>();\n        ctx.add(Context.cpu());\n\n        return ctx;\n    }\n\n    private static void printOutput(List<List<ObjectDetectorOutput>> output, Shape inputShape) {\n\n        StringBuilder outputStr = new StringBuilder();\n\n        int width = inputShape.get(3);\n        int height = inputShape.get(2);\n\n        for (List<ObjectDetectorOutput> ele : output) {\n            for (ObjectDetectorOutput i : ele) {\n                outputStr.append(\"Class: \" + i.getClassName() + \"\\n\");\n                outputStr.append(\"Probabilties: \" + i.getProbability() + \"\\n\");\n\n                List<Float> coord = Arrays.asList(i.getXMin() * width,\n                        i.getXMax() * height, i.getYMin() * width, i.getYMax() * height);\n                StringBuilder sb = new StringBuilder();\n                for (float c: coord) {\n                    sb.append(\", \").append(c);\n                }\n                outputStr.append(\"Coord:\" + sb.substring(2)+ \"\\n\");\n            }\n        }\n        System.out.println(outputStr);\n\n    }\n}\n```\n\n7\\. To compile and run this code, change directories to this project's root folder, then run the following:\n```bash\nmvn clean install dependency:copy-dependencies\n```\n\nThe build generates a new jar file in the `target` folder called `javaMXNet-1.0-SNAPSHOT.jar`.\n\nTo run the ObjectDetectionTutorial.java use the following command from the project's root folder.\n```bash\njava -cp \"target/javaMXNet-1.0-SNAPSHOT.jar:target/dependency/*\" mxnet.ObjectDetectionTutorial\n```\n\nYou should see a similar output being generated for the dog image that we used:\n```bash\nClass: car\nProbabilties: 0.99847263\nCoord:312.21335, 72.02908, 456.01443, 150.66176\nClass: bicycle\nProbabilties: 0.9047381\nCoord:155.9581, 149.96365, 383.83694, 418.94516\nClass: dog\nProbabilties: 0.82268167\nCoord:83.82356, 179.14001, 206.63783, 476.78754\n```\n\n![dog_1](https://cloud.githubusercontent.com/assets/3307514/20012567/cbb60336-a27d-11e6-93ff-cbc3f09f5c9e.jpg)\n\nThe results returned by the inference call translate into the regions in the image where the model detected objects.\n\n![dog_2](https://cloud.githubusercontent.com/assets/3307514/19171063/91ec2792-8be0-11e6-983c-773bd6868fa8.png)\n\n## Next Steps\nFor more information about MXNet Java resources, see the following:\n\n* [Java Inference API]({{'/api/java'|relative_url}})\n* [Java Inference Examples](https://github.com/apache/mxnet/tree/master/scala-package/examples/src/main/java/org/apache/mxnetexamples/javaapi/infer)\n* [MXNet Tutorials Index]({{'/api'|relative_url}})\n"
  },
  {
    "path": "docs/static_site/src/pages/api/java/index.md",
    "content": "---\nlayout: page_api\ntitle: Java Guide\naction: Get Started\naction_url: /get_started\npermalink: /api/java\ntag: java\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# MXNet - Java Inference API\n\nMXNet supports Java for performing inference on a trained model. The MXNet Java Inference API is an extension of the [Scala Infer API]({{'/api/scala/docs/api/#org.apache.mxnet.infer.package'|relative_url}}) which provides model loading and inference functionality.\nThe goal of the MXNet Java package is to provide an efficient and easy to use inference API.\nThe MXNet Java package makes it easy to quickly deploy an existing model into a production level Java ecosystem.\n\n## Installation\nPlease see the [Get Started]({{'/get_started'|relative_url}}) page.\n\n## Tutorials\nSee the [Java tutorial page]({{'/api/java/docs/tutorials'|relative_url}}) for detailed tutorials and examples using the Java Inference API.\n\n## Java Inference API Reference\nThe [Java Infer API javadocs]({{'/api/java/docs/api/#org.apache.mxnet.infer.javaapi.package'|relative_url}}) provides detailed API information.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/julia/index.md",
    "content": "---\nlayout: page_api\ntitle: Julia Guide\naction: Get Started\naction_url: /get_started\npermalink: /api/julia\ntag: julia\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# MXNet - Julia API\n\n\nMXNet supports the Julia programming language. The MXNet Julia package brings flexible and efficient GPU\ncomputing and the state-of-art deep learning to Julia.\n\n- It enables you to write seamless tensor/matrix computation with multiple GPUs in Julia.\n- It also enables you to construct and customize the state-of-art deep learning models in Julia,\n  and apply them to tasks such as image classification and data science challenges.\n\n## Installation\n* [Ubuntu installation guide]({{'/get_started/ubuntu_setup.html'|relative_url}})\n* [maxOS installation guide]({{'/get_started/osx_setup.html'|relative_url}})\n* [Windows installation guide]({{'/get_started/windows_setup.html'|relative_url}})\n"
  },
  {
    "path": "docs/static_site/src/pages/api/perl/docs/tutorials/index.md",
    "content": "---\nlayout: page_landing_tutorials\ntitle:  Perl Tutorials\naction: Get Started\ntag: perl\npermalink: /api/perl/docs/tutorials\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->"
  },
  {
    "path": "docs/static_site/src/pages/api/perl/docs/tutorials/io.md",
    "content": "---\nlayout: page_api\ntitle: Data Loading API\nis_tutorial: true\ntag: perl\npermalink: /api/perl/docs/tutorials/io\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Data Loading API\n\n## Overview\n\nA data iterator reads data batch by batch.\n\n```perl\npdl> $data = mx->nd->ones([100,10])\npdl> $nd_iter = mx->io->NDArrayIter($data, batch_size=>25)\npdl> for my $batch (@{ $nd_iter }) { print $batch->data->[0],\"\\n\" }\n<AI::MXNet::NDArray 25x10 @cpu(0)>\n<AI::MXNet::NDArray 25x10 @cpu(0)>\n<AI::MXNet::NDArray 25x10 @cpu(0)>\n<AI::MXNet::NDArray 25x10 @cpu(0)>\n```\n\nIf `$nd_iter->reset()` is called, then reads the data again from beginning.\n\nIn addition, an iterator provides information about the batch, including the\nshapes and name.\n\n```perl\npdl> $nd_iter = mx->io->NDArrayIter(data=>{data => mx->nd->ones([100,10])}, label=>{softmax_label => mx->nd->ones([100])}, batch_size=>25)\npdl> print($nd_iter->provide_data->[0],\"\\n\")\nDataDesc[data,25x10,float32,NCHW]\npdl> print($nd_iter->provide_label->[0],\"\\n\")\nDataDesc[softmax_label,25,float32,NCHW]\n```\n\nSo this iterator can be used to train a symbol whose input data variable has\nname `data` and input label variable has name `softmax_label`.\n\n## Predefined Data iterators\n\n```perl\nmx->io->NDArrayIter\nmx->io->CSVIter\nmx->io->ImageRecordIter\nmx->io->ImageRecordInt8Iter\nmx->io->ImageRecordUInt8Iter\nmx->io->MNISTIter\nmx->recordio->MXRecordIO\nmx->recordio->MXIndexedRecordIO\nmx->image->ImageIter\n```\n\n## Helper classes and functions\n\nData structures and other iterators provided in the `AI::MXNet::IO` package.\n\n```perl\nAI::MXNet::DataDesc\nAI::MXNet::DataBatch\nAI::MXNet::DataIter\nAI::MXNet::ResizeIter\nAI::MXNet::MXDataIter\n```\n\nA list of image modification functions provided by `AI::MXNet::Image`.\n\n```perl\nmx->image->imdecode\nmx->image->scale_down\nmx->image->resize_short\nmx->image->fixed_crop\nmx->image->random_crop\nmx->image->center_crop\nmx->image->color_normalize\nmx->image->random_size_crop\nmx->image->ResizeAug\nmx->image->RandomCropAug\nmx->image->RandomSizedCropAug\nmx->image->CenterCropAug\nmx->image->RandomOrderAug\nmx->image->ColorJitterAug\nmx->image->LightingAug\nmx->image->ColorNormalizeAug\nmx->image->HorizontalFlipAug\nmx->image->CastAug\nmx->image->CreateAugmenter\n```\n\nFunctions to read and write RecordIO files.\n\n```perl\nmx->recordio->pack\nmx->recordio->unpack\nmx->recordio->unpack_img\n```\n\n## Develop a new iterator\n\nWriting a new data iterator in Perl is straightforward. Most MXNet\ntraining/inference programs accept an object with ``provide_data``\nand ``provide_label`` properties.\nPlease refer to AI-MXNet/examples for the examples of custom iterators.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/perl/docs/tutorials/kvstore.md",
    "content": "---\nlayout: page_api\ntitle: KVStore API\nis_tutorial: true\ntag: perl\npermalink: /api/perl/docs/tutorials/kvstore\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# KVStore API\n\nTopics:\n* [Basic Push and Pull](#basic-push-and-pull)\n* [List Key-Value Pairs](#list-key-value-pairs)\n\n## Basic Push and Pull\n\nProvides basic operation over multiple devices (GPUs) on a single device.\n\n### Initialization\n\nLet's consider a simple example. It initializes\na (int, NDArray) pair into the store, and then pulls the value out.\n\n```perl\npdl> $kv = mx->kv->create('local')\npdl> $shape = [2,3]\npdl> $kv->init(3, mx->nd->ones($shape)*2)\npdl> $a = mx->nd->zeros($shape)\npdl> $kv->pull(3, out => $a)\npdl> print $a->aspdl\n[\n [2 2 2]\n [2 2 2]\n]\n```\n\n### Push, Aggregation, and Updater\n\nFor any key that's been initialized, you can push a new value with the same shape to the key, as follows:\n\n```perl\npdl> $kv->push(3, mx->nd->ones($shape)*8)\npdl> $a = mx->nd->zeros($shape)\npdl> $kv->pull(3, out => $a)\npdl> print $a->aspdl\n[\n [8 8 8]\n [8 8 8]\n]\n```\n\nThe data that you want to push can be stored on any device. Furthermore, you can push multiple\nvalues into the same key, where KVStore first sums all of these\nvalues, and then you pull the aggregated value, as follows:\n\n```perl\npdl> $kv->push(3, [mx->nd->ones($shape, ctx=>mx->cpu(0)), mx->nd->ones($shape, ctx=>mx->cpu(1))])\npdl> $kv->pull(3, out => $a)\npdl> print $a->aspdl\n[\n [2 2 2]\n [2 2 2]\n]\n```\n\nFor each push command, KVStore applies the pushed value to the value stored by an\n`updater`. The default updater is `ASSIGN`. You can replace the default to\ncontrol how data is merged.\n\n```perl\npdl> $updater = sub { my ($key, $input, $stored) = @_; print \"update on key: $key\\n\"; $stored += $input * 3; }\npdl> $kv->_set_updater($updater)\npdl> $kv->push(3, [mx->nd->ones($shape, ctx=>mx->cpu(0)), mx->nd->ones($shape, ctx=>mx->cpu(1))])\nupdate on key: 3\npdl> $kv->pull(3, out => $a)\npdl> print $a->aspdl\n[\n [8 8 8]\n [8 8 8]\n]\n```\n\n### Pull\n\nYou've already seen how to pull a single key-value pair. Similar to the way that you use the push command, you can\npull the value into several devices with a single call.\n\n```perl\npdl> $b = [mx->nd->zeros($shape, ctx=>mx->cpu(0)), mx->nd->zeros($shape, ctx=>mx->cpu(1))]\npdl> $kv->pull(3, out => $b)\npdl> print $b->[1]->aspdl\n[\n [8 8 8]\n [8 8 8]\n]\n```\n\n## List Key-Value Pairs\n\nAll of the operations that we've discussed so far are performed on a single key. KVStore also provides\nthe interface for generating a list of key-value pairs. For a single device, use the following:\n\n```perl\npdl> $keys = [5,7,9]\npdl> $kv->init($keys, [map { mx->nd->ones($shape) } 0..@$keys-1])\npdl> $kv->push($keys, [map { mx->nd->ones($shape) } 0..@$keys-1])\nupdate on key: 5\nupdate on key: 7\nupdate on key: 9\npdl> $b = [map { mx->nd->ones($shape) } 0..@$keys-1]\npdl> $kv->pull($keys, out => $b)\npdl> print $b->[1]->aspdl\n[\n [4 4 4]\n [4 4 4]\n]\n```\n"
  },
  {
    "path": "docs/static_site/src/pages/api/perl/docs/tutorials/ndarray.md",
    "content": "---\nlayout: page_api\ntitle: NDArray API\nis_tutorial: true\ntag: perl\npermalink: /api/perl/docs/tutorials/ndarray\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# NDArray API\n\n## Overview\n\nA `AI::MXNet::NDArray` is a multidimensional container of items of the same type and\nsize. Various methods for data manipulation and computation are provided.\n\n```perl\npdl> $x = mx->nd->array([[1, 2, 3], [4, 5, 6]])\npdl> print $x->aspdl->shape\n[3, 2]\npdl> $y = $x + mx->nd->ones($x->shape)*3\npdl> print $y->aspdl\n[\n [4 5 6]\n [7 8 9]\n]\npdl> $z = $y->as_in_context(mx->gpu(0))\npdl> print $z,\"\\n\"\n<AI::MXNet::NDArray 2x3 @gpu(0)>\n```\n\nA detailed tutorial is available at\n[https://mxnet.io/tutorials/basic/ndarray.html](https://mxnet.io/tutorials/basic/ndarray.html).\n\nNote: AI::MXNet::NDarray is similar to numpy.ndarray in some aspects. But the difference is not negligible. For example\n\n- AI::MXNet::NDArray->T does real data transpose to return new a copied array, instead\n     of returning a view of the input array.\n- AI::MXNet::NDArray->dot performs dot between the last axis of the first input array\n     and the first axis of the second input, while numpy.dot uses the second\n     last axis of the input array.\n\nIn additional, NDArray supports GPU computation and various neural\nnetwork layers.\n\nAI::MXNet::NDarray also provides almost same routines as AI::MXNet::symbol. Most\nroutines between these two packages share the same C++ operator source\ncodes. But AI::MXNet::NDarray differs from AI::MXNet::Symbol in several aspects:\n\n- AI::MXNet::NDArray adopts imperative programming, namely sentences are executed\n     step-by-step so that the results can be obtained immediately.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/perl/docs/tutorials/symbol.md",
    "content": "---\nlayout: page_api\ntitle: Symbol API\nis_tutorial: true\ntag: perl\npermalink: /api/perl/docs/tutorials/symbol\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# MXNet Perl Symbolic API\n\nTopics:\n\n* [How to Compose Symbols](#how-to-compose-symbols) introduces operator overloading of symbols.\n* [Symbol Attributes](#symbol-attributes) describes how to attach attributes to symbols.\n* [Serialization](#serialization) explains how to save and load symbols.\n* [Executing Symbols](#executing-symbols) explains how to evaluate the symbols with data.\n* [Multiple Outputs](#multiple-outputs) explains how to configure multiple outputs.\n\n## How to Compose Symbols\n\nThe symbolic API provides a way to configure computation graphs.\nYou can configure the graphs either at the level of neural network layer operations or as fine-grained operations.\n\n\nThe basic arithmetic operators (plus, minus, div, multiplication) are overloaded for\n*element-wise operations* of symbols.\n\nThe following example creates a computation graph that adds two inputs together.\n\n```perl\npdl> use AI::MXNet qw(mx)\npdl> $a =  mx->symbol->Variable(\"a\")\npdl> $b =  mx->symbol->Variable(\"b\")\npdl> $c = $a + $b\n```\n\n## Symbol Attributes\n\nYou can add an attribute to a symbol by providing an attribute hash when you create a symbol.\n\n```perl\n$data =  mx->symbol->Variable(\"data\", attr => { mood => \"angry\" })\n$op   =  mx->symbol->Convolution(data => $data, kernel => [1, 1], num_filter => 1, attr => { mood => \"so so\" })\n```\n\nFor proper communication with the C++ backend, both the key and values of the attribute dictionary should be strings. To retrieve the attributes, use `->attr($key)`:\n\n```\n    $data->attr(\"mood\")\n```\n\nTo attach attributes, you can use ```AI::MXNet::AttrScope```. ```AI::MXNet::AttrScopeAttrScope``` automatically adds\nthe specified attributes to all of the symbols created within that scope.\nThe user can also inherit this object to change naming behavior. For example:\n\n```perl\nuse AI::MXNet qw(mx);\nuse Test::More tests => 3;\nmy ($data, $gdata);\n{\n    local($mx::AttrScope) = mx->AttrScope(group=>4, data=>'great');\n    $data = mx->sym->Variable(\"data\", attr => { dtype => \"data\", group => \"1\" });\n    $gdata = mx->sym->Variable(\"data2\");\n}\nok($gdata->attr(\"group\") == 4);\nok($data->attr(\"group\") == 1);\n\nmy $exceedScopeData = mx->sym->Variable(\"data3\");\nok((not defined $exceedScopeData->attr(\"group\")), \"No group attr in global attr scope\");\n```\n\n## Serialization\n\nThere are two ways to save and load the symbols. You can use the `mx->symbol->save` and `mxnet->symbol->load` functions to serialize the ```AI::MXNet::Symbol``` objects.\nThe advantage of using `save` and `load` functions is that it is language agnostic and cloud friendly.\nThe symbol is saved in JSON format. You can also get a JSON string directly using `$symbol->tojson`.\n\nThe following example shows how to save a symbol to an S3 bucket, load it back, and compare two symbols using a JSON string.\n\n```perl\npdl> use AI::MXNet qw(mx)\npdl> $a = mx->sym->Variable(\"a\")\npdl> $b = mx->sym->Variable(\"b\")\npdl> $c = $a + $b\npdl> $c->save(\"s3://my-bucket/symbol-c.json\")\npdl> $c2 = $c->load(\"s3://my-bucket/symbol-c.json\")\npdl> ok($c->tojson eq $c2->tojson)\nok 1\n```\n\n## Executing Symbols\n\nAfter you have assembled a set of symbols into a computation graph, the MXNet engine can evaluate them.\nIf you are training a neural network, this is typically\nhandled by the high-level [AI::MXNet::Module package](module) and the [`fit()`] function.\n\nFor neural networks used in \"feed-forward\", \"prediction\", or \"inference\" mode (all terms for the same\nthing: running a trained network), the input arguments are the\ninput data, and the weights of the neural network that were learned during training.\n\nTo manually execute a set of symbols, you need to create an [`AI::MXNet::Executor`] object,\nwhich is typically constructed by calling the [`simple_bind(<parameters>)`] method on a AI::MXNet::Symbol.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/perl/index.md",
    "content": "---\nlayout: page_api\ntitle: Perl Guide\naction: Get Started\naction_url: /get_started\npermalink: /api/perl\ntag: perl\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# MXNet - Perl API\n\nMXNet supports the Perl programming language. The MXNet Perl package brings flexible and efficient GPU\ncomputing and state-of-art deep learning to Perl. It enables you to write seamless tensor/matrix computation with multiple GPUs in Perl.\nIt also lets you construct and customize the state-of-art deep learning models in Perl,\n  and apply them to tasks, such as image classification and data science challenges.\n\nOne important thing to internalize is that Perl interface is written to be as close as possible to the Python's API,\nso most if not all of Python's documentation and examples should just work in Perl after making few\nchanges in order to make the code a bit more Perlish. In nutshell just add $ sigils and replace . = \\n with -> => ; and in 99% of cases\nthat's all that is needed there.\nIn addition please refer to [excellent metacpan doc interface](https://metacpan.org/release/AI-MXNet) and to very detailed\n[MXNet Python API Documentation]({{'/api/python' | relative_url}}).\n\nAI::MXNet supports new imperative PyTorch like Gluon MXNet interface. Please get acquainted with this new interface\nat [Dive into Deep Learning](https://www.d2l.ai/).\n\nFor specific Perl Gluon usage please refer to Perl examples and tests directories on github, but be assured that the Python and Perl usage\nare extremely close in order to make the use of the Python Gluon docs and examples as easy as possible.\n\nAI::MXNet is seamlessly glued with [PDL](https://metacpan.org/release/PDL), the C++ level state can be easily initialized from PDL and the results can be\ntransferred to PDL objects in order to allow you to use all the glory and power of the PDL!\n\nHere is how you can perform tensor or matrix computation in Perl with AI::MXNet and PDL:\n\n```perl\npdl> use AI::MXNet qw(mx); # creates 'mx' module on the fly with the interface close to the Python's API\n\npdl> print $arr = mx->nd->ones([2, 3])\n<AI::MXNet::NDArray 2x3 @cpu(0)>\n\npdl> print Data::Dumper::Dumper($arr->shape)\n$VAR1 = [\n          2,\n          3\n        ];\n\npdl> print (($arr*2)->aspdl) ## converts AI::MXNet::NDArray object to PDL object\n\n[\n [2 2 2]\n [2 2 2]\n]\n\npdl> print $arr = mx->nd->array([[1,2],[3,4]]) ## init the NDArray from Perl array ref given in PDL::pdl constructor format\n<AI::MXNet::NDArray 2x2 @cpu(0)>\npdl> print $arr->aspdl\n\n[\n [1 2]\n [3 4]\n]\n\n## init the NDArray from PDL but be aware that PDL methods expect the dimensions order in column major format\n## AI::MXNet::NDArray is row major\npdl> print mx->nd->array(sequence(2,3))->aspdl ## 3 rows, 2 columns\n\n[\n [0 1]\n [2 3]\n [4 5]\n]\n```\n\nExport/import to/from sparse MXNet tensors are supported via [PDL::CCS](https://metacpan.org/release/PDL-CCS).\nPlease check out the examples directory for the examples on how to use the sparse matrices.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/python/index.md",
    "content": "---\nlayout: page_api\ntitle: Python Guide\naction: Get Started\naction_url: /get_started\npermalink: /api/python\ntag: python\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n## Apache MXNet - Python API\n\nApache MXNet provides a comprehensive and flexible Python API to serve a broad community of developers with different levels of experience and wide ranging requirements. In this section, we provide an in-depth discussion of the functionality provided by various MXNet Python packages.\n\n\nApache MXNet’s Python API has two primary high-level packages*: the Gluon API and Module API. We recommend that new users start with the Gluon API as it’s more flexible and easier to debug. Underlying these high-level packages are the core packages of NDArray and Symbol.\n\n\nNDArray works with arrays in an imperative fashion, i.e. you define how arrays will be transformed to get to an end result. Symbol works with arrays in a declarative fashion, i.e. you define the end result that is required (via a symbolic graph) and the MXNet engine will use various optimizations to determine the steps required to obtain this. With NDArray you have a great deal of flexibility when composing operations (as you can use Python control flow), and you can easily step through your code and inspect the values of arrays, which helps with debugging. Unfortunately, this comes at a performance cost when compared to Symbol, which can perform optimizations on the symbolic graph.\n\n\nModule API is backed by Symbol, so, although it’s very performant, it’s also a little more restrictive. With the Gluon API, you can get the best of both worlds. You can develop and test your model imperatively using NDArray, a then switch to Symbol for faster model training and inference (if Symbol equivalents exist for your operations).\n\n\nCode examples are placed throughout the API documentation and these can be run after importing MXNet as follows:\n\n```python\n>>> import mxnet as mx\n```\n"
  },
  {
    "path": "docs/static_site/src/pages/api/r/docs/tutorials/char_rnn_model.md",
    "content": "---\nlayout: page_api\ntitle: Char RNN Model\nis_tutorial: true\ntag: r\npermalink: /api/r/docs/tutorials/char_rnn_model\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Character-level Language Model using RNN\n\nThis tutorial will demonstrate creating a language model using a character level RNN model using MXNet-R package. You will need the following R packages to run this tutorial -\n - readr\n - stringr\n - stringi\n - mxnet\n\nWe will use the [tinyshakespeare](https://github.com/dmlc/web-data/tree/master/mxnet/tinyshakespeare) dataset to build this model.\n\n\n```R\nlibrary(\"readr\")\nlibrary(\"stringr\")\nlibrary(\"stringi\")\nlibrary(\"mxnet\")\n```\n\n## Preprocess and prepare the data\n\nDownload the data:\n\n\n```R\ndownload.data <- function(data_dir) {\n    dir.create(data_dir, showWarnings = FALSE)\n    if (!file.exists(paste0(data_dir,'input.txt'))) {\n        download.file(url='https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tinyshakespeare/input.txt',\n                      destfile=paste0(data_dir,'input.txt'), method='wget')\n    }\n}\n```\n\nNext we transform the test into feature vectors that is fed into the RNN model. The `make_data` function reads the dataset, cleans it of any non-alphanumeric characters, splits it into individual characters and groups it into sequences of length `seq.len`.\n\n\n```R\nmake_data <- function(path, seq.len = 32, dic=NULL) {\n\n  text_vec <- read_file(file = path)\n  text_vec <- stri_enc_toascii(str = text_vec)\n  text_vec <- str_replace_all(string = text_vec, pattern = \"[^[:print:]]\", replacement = \"\")\n  text_vec <- strsplit(text_vec, '') %>% unlist\n\n  if (is.null(dic)) {\n    char_keep <- sort(unique(text_vec))\n  } else char_keep <- names(dic)[!dic == 0]\n\n  # Remove terms not part of dictionary\n  text_vec <- text_vec[text_vec %in% char_keep]\n\n  # Build dictionary\n  dic <- 1:length(char_keep)\n  names(dic) <- char_keep\n\n  # reverse dictionary\n  rev_dic <- names(dic)\n  names(rev_dic) <- dic\n\n  # Adjust by -1 to have a 1-lag for labels\n  num.seq <- (length(text_vec) - 1) %/% seq.len\n\n  features <- dic[text_vec[1:(seq.len * num.seq)]]\n  labels <- dic[text_vec[1:(seq.len*num.seq) + 1]]\n\n  features_array <- array(features, dim = c(seq.len, num.seq))\n  labels_array <- array(labels, dim = c(seq.len, num.seq))\n\n  return (list(features_array = features_array, labels_array = labels_array, dic = dic, rev_dic = rev_dic))\n}\n\n\nseq.len <- 100\ndata_prep <- make_data(path = \"input.txt\", seq.len = seq.len, dic=NULL)\n```\n\nFetch the features and labels for training the model, and split the data into training and evaluation in 9:1 ratio.\n\n\n```R\nX <- data_prep$features_array\nY <- data_prep$labels_array\ndic <- data_prep$dic\nrev_dic <- data_prep$rev_dic\nvocab <- length(dic)\n\nsamples <- tail(dim(X), 1)\ntrain.val.fraction <- 0.9\n\nX.train.data <- X[, 1:as.integer(samples * train.val.fraction)]\nX.val.data <- X[, -(1:as.integer(samples * train.val.fraction))]\n\nX.train.label <- Y[, 1:as.integer(samples * train.val.fraction)]\nX.val.label <- Y[, -(1:as.integer(samples * train.val.fraction))]\n\ntrain_buckets <- list(\"100\" = list(data = X.train.data, label = X.train.label))\neval_buckets <- list(\"100\" = list(data = X.val.data, label = X.val.label))\n\ntrain_buckets <- list(buckets = train_buckets, dic = dic, rev_dic = rev_dic)\neval_buckets <- list(buckets = eval_buckets, dic = dic, rev_dic = rev_dic)\n```\n\nCreate iterators for training and evaluation datasets.\n\n\n```R\nvocab <- length(eval_buckets$dic)\n\nbatch.size <- 32\n\ntrain.data <- mx.io.bucket.iter(buckets = train_buckets$buckets, batch.size = batch.size,\n                                data.mask.element = 0, shuffle = TRUE)\n\neval.data <- mx.io.bucket.iter(buckets = eval_buckets$buckets, batch.size = batch.size,\n                               data.mask.element = 0, shuffle = FALSE)\n```\n\n## Train the Model\n\n\nThis model is a multi-layer RNN for sampling from character-level language models. It has a one-to-one model configuration since for each character, we want to predict the next one. For a sequence of length 100, there are also 100 labels, corresponding the same sequence of characters but offset by a position of +1. The parameters output_last_state is set to TRUE in order to access the state of the RNN cells when performing inference.\n\n\n```R\nrnn_graph_one_one <- rnn.graph(num_rnn_layer = 3,\n                               num_hidden = 96,\n                               input_size = vocab,\n                               num_embed = 64,\n                               num_decode = vocab,\n                               dropout = 0.2,\n                               ignore_label = 0,\n                               cell_type = \"lstm\",\n                               masking = F,\n                               output_last_state = T,\n                               loss_output = \"softmax\",\n                               config = \"one-to-one\")\n\ngraph.viz(rnn_graph_one_one, type = \"graph\", direction = \"LR\",\n          graph.height.px = 180, shape=c(100, 64))\n\ndevices <- mx.cpu()\n\ninitializer <- mx.init.Xavier(rnd_type = \"gaussian\", factor_type = \"avg\", magnitude = 3)\n\noptimizer <- mx.opt.create(\"adadelta\", rho = 0.9, eps = 1e-5, wd = 1e-8,\n                           clip_gradient = 5, rescale.grad = 1/batch.size)\n\nlogger <- mx.metric.logger()\nepoch.end.callback <- mx.callback.log.train.metric(period = 1, logger = logger)\nbatch.end.callback <- mx.callback.log.train.metric(period = 50)\n\nmx.metric.custom_nd <- function(name, feval) {\n  init <- function() {\n    c(0, 0)\n  }\n  update <- function(label, pred, state) {\n    m <- feval(label, pred)\n    state <- c(state[[1]] + 1, state[[2]] + m)\n    return(state)\n  }\n  get <- function(state) {\n    list(name=name, value = (state[[2]] / state[[1]]))\n  }\n  ret <- (list(init = init, update = update, get = get))\n  class(ret) <- \"mx.metric\"\n  return(ret)\n}\n\nmx.metric.Perplexity <- mx.metric.custom_nd(\"Perplexity\", function(label, pred) {\n  label <- mx.nd.reshape(label, shape = -1)\n  label_probs <- as.array(mx.nd.choose.element.0index(pred, label))\n  batch <- length(label_probs)\n  NLL <- -sum(log(pmax(1e-15, as.array(label_probs)))) / batch\n  Perplexity <- exp(NLL)\n  return(Perplexity)\n})\n\nmodel <- mx.model.buckets(symbol = rnn_graph_one_one,\n                          train.data = train.data, eval.data = eval.data,\n                          num.round = 20, ctx = devices, verbose = TRUE,\n                          metric = mx.metric.Perplexity,\n                          initializer = initializer, optimizer = optimizer,\n                          batch.end.callback = NULL,\n                          epoch.end.callback = epoch.end.callback)\n\nmx.model.save(model, prefix = \"one_to_one_seq_model\", iteration = 20)\n```\n\n    Start training with 1 devices\n    [1] Train-Perplexity=13.7040474322178\n    [1] Validation-Perplexity=7.94617194460922\n    [2] Train-Perplexity=6.57039815554525\n    [2] Validation-Perplexity=6.60806110658011\n    [3] Train-Perplexity=5.65360504501481\n    [3] Validation-Perplexity=6.18932770630876\n    [4] Train-Perplexity=5.32547285727298\n    [4] Validation-Perplexity=6.02198756798859\n    [5] Train-Perplexity=5.14373631472579\n    [5] Validation-Perplexity=5.8095658243407\n    [6] Train-Perplexity=5.03077673487379\n    [6] Validation-Perplexity=5.72582993567431\n    [7] Train-Perplexity=4.94453383291536\n    [7] Validation-Perplexity=5.6445258528126\n    [8] Train-Perplexity=4.88635290100261\n    [8] Validation-Perplexity=5.6730024536433\n    [9] Train-Perplexity=4.84205646230548\n    [9] Validation-Perplexity=5.50960780230982\n    [10] Train-Perplexity=4.80441673535513\n    [10] Validation-Perplexity=5.57002263750006\n    [11] Train-Perplexity=4.77763413242626\n    [11] Validation-Perplexity=5.55152143269169\n    [12] Train-Perplexity=4.74937775290777\n    [12] Validation-Perplexity=5.44968305351486\n    [13] Train-Perplexity=4.72824849541467\n    [13] Validation-Perplexity=5.50889348298234\n    [14] Train-Perplexity=4.70980846981694\n    [14] Validation-Perplexity=5.51473225859859\n    [15] Train-Perplexity=4.69685776886122\n    [15] Validation-Perplexity=5.45391985233811\n    [16] Train-Perplexity=4.67837107034824\n    [16] Validation-Perplexity=5.46636764997829\n    [17] Train-Perplexity=4.66866961934873\n    [17] Validation-Perplexity=5.44267086113492\n    [18] Train-Perplexity=4.65611469144194\n    [18] Validation-Perplexity=5.4290169469462\n    [19] Train-Perplexity=4.64614689879405\n    [19] Validation-Perplexity=5.44221549833917\n    [20] Train-Perplexity=4.63764001963654\n    [20] Validation-Perplexity=5.42114250842862\n\n\n## Inference on the Model\n\nWe now use the saved model to do inference and sample text character by character that will look like the original training data.\n\n\n```R\nset.seed(0)\nmodel <- mx.model.load(prefix = \"one_to_one_seq_model\", iteration = 20)\n\ninternals <- model$symbol$get.internals()\nsym_state <- internals$get.output(which(internals$outputs %in% \"RNN_state\"))\nsym_state_cell <- internals$get.output(which(internals$outputs %in% \"RNN_state_cell\"))\nsym_output <- internals$get.output(which(internals$outputs %in% \"loss_output\"))\nsymbol <- mx.symbol.Group(sym_output, sym_state, sym_state_cell)\n\ninfer_raw <- c(\"Thou \")\ninfer_split <- dic[strsplit(infer_raw, '') %>% unlist]\ninfer_length <- length(infer_split)\n\ninfer.data <- mx.io.arrayiter(data = matrix(infer_split), label = matrix(infer_split),\n                              batch.size = 1, shuffle = FALSE)\n\ninfer <- mx.infer.rnn.one(infer.data = infer.data,\n                          symbol = symbol,\n                          arg.params = model$arg.params,\n                          aux.params = model$aux.params,\n                          input.params = NULL,\n                          ctx = devices)\n\npred_prob <- as.numeric(as.array(mx.nd.slice.axis(\n    infer$loss_output, axis = 0, begin = infer_length-1, end = infer_length)))\npred <- sample(length(pred_prob), prob = pred_prob, size = 1) - 1\npredict <- c(predict, pred)\n\nfor (i in 1:200) {\n\n  infer.data <- mx.io.arrayiter(data = as.matrix(pred), label = as.matrix(pred),\n                                batch.size = 1, shuffle = FALSE)\n\n  infer <- mx.infer.rnn.one(infer.data = infer.data,\n                            symbol = symbol,\n                            arg.params = model$arg.params,\n                            aux.params = model$aux.params,\n                            input.params = list(rnn.state = infer[[2]],\n                                                rnn.state.cell = infer[[3]]),\n                            ctx = devices)\n\n  pred_prob <- as.numeric(as.array(infer$loss_output))\n  pred <- sample(length(pred_prob), prob = pred_prob, size = 1, replace = T) - 1\n  predict <- c(predict, pred)\n}\n\npredict_txt <- paste0(rev_dic[as.character(predict)], collapse = \"\")\npredict_txt_tot <- paste0(infer_raw, predict_txt, collapse = \"\")\nprint(predict_txt_tot)\n```\n\n    [1] \"Thou NAknowledge thee my Comfort and his late she.FRIAR LAURENCE:Nothing a groats waterd forth. The lend he thank that;When she I am brother draw London: and not hear that know.BENVOLIO:How along, makes your \"\n\n\n<!-- INSERT SOURCE DOWNLOAD BUTTONS -->\n"
  },
  {
    "path": "docs/static_site/src/pages/api/r/docs/tutorials/classify_real_image_with_pretrained_model.md",
    "content": "---\nlayout: page_api\ntitle: Classify Images with a PreTrained Model\nis_tutorial: true\ntag: r\npermalink: /api/r/docs/tutorials/classify_real_image_with_pretrained_model\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nClassify Images with a PreTrained Model\n=================================================\nMXNet is a flexible and efficient deep learning framework. One of the interesting things that a deep learning\nalgorithm can do is classify real world images.\n\nIn this tutorial, we show how to use a pre-trained Inception-BatchNorm network to predict the class of an\nimage. For information about the network architecture, see  [1].\n\nThe pre-trained Inception-BatchNorm network is able to be downloaded from [this link](https://data.mxnet.io/mxnet/data/Inception.zip)\nThis model gives the recent state-of-art prediction accuracy on image net dataset.\n\nLoad the MXNet Package\n---------------\nTo get started, load the mxnet package:\n\n ```r\n    require(mxnet)\n ```\n\n ```\n    ## Loading required package: mxnet\n    ## Loading required package: methods\n ```\n\nNow load the imager package to load and preprocess the images in R:\n\n\n ```r\n    require(imager)\n ```\n\n ```\n    ## Loading required package: imager\n    ## Loading required package: plyr\n    ## Loading required package: magrittr\n    ## Loading required package: stringr\n    ## Loading required package: png\n    ## Loading required package: jpeg\n    ##\n    ## Attaching package: 'imager'\n    ##\n    ## The following object is masked from 'package:magrittr':\n    ##\n    ##     add\n    ##\n    ## The following object is masked from 'package:plyr':\n    ##\n    ##     liply\n    ##\n    ## The following objects are masked from 'package:stats':\n    ##\n    ##     convolve, spectrum\n    ##\n    ## The following object is masked from 'package:graphics':\n    ##\n    ##     frame\n    ##\n    ## The following object is masked from 'package:base':\n    ##\n    ##     save.image\n ```\n\nLoad the PreTrained Model\n-------------------------\nMake sure you unzip the pre-trained model in the current folder. Use the model\nloading function to load the model into R:\n\n ```r\n    model = mx.model.load(\"Inception/Inception_BN\", iteration=39)\n ```\n\nLoad in the mean image, which is used for preprocessing using:\n\n\n ```r\n    mean.img = as.array(mx.nd.load(\"Inception/mean_224.nd\")[[\"mean_img\"]])\n ```\n\nLoad and Preprocess the Image\n-----------------------------\nNow, we are ready to classify a real image. In this example, we simply take the parrots image\nfrom the imager package. You can use another image, if   you prefer.\n\nLoad and plot the image:\n\n\n```r\n    im <- load.image(system.file(\"extdata/parrots.png\", package=\"imager\"))\n    plot(im)\n ```\n\n![plot of chunk unnamed-chunk-5](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/knitr/classifyRealImageWithPretrainedModel-unnamed-chunk-5-1.png)\n\nBefore feeding the image to the deep network, we need to perform some preprocessing\nto make the image meet the deep network input requirements. Preprocessing\nincludes cropping  and subtracting the mean.\nBecause MXNet is deeply integrated with R, we can do all the processing in an R function:\n\n\n ```r\n    preproc.image <- function(im, mean.image) {\n      # crop the image\n      shape <- dim(im)\n      short.edge <- min(shape[1:2])\n      xx <- floor((shape[1] - short.edge) / 2)\n      yy <- floor((shape[2] - short.edge) / 2)\n      cropped <- crop.borders(im, xx, yy)\n      # resize to 224 x 224, needed by input of the model.\n      resized <- resize(cropped, 224, 224)\n      # convert to array (x, y, channel)\n      arr <- as.array(resized) * 255\n      dim(arr) <- c(224, 224, 3)\n      # subtract the mean\n      normed <- arr - mean.img\n      # Reshape to format needed by mxnet (width, height, channel, num)\n      dim(normed) <- c(224, 224, 3, 1)\n      return(normed)\n    }\n ```\n\nUse the defined preprocessing function to get the normalized image:\n\n\n ```r\n    normed <- preproc.image(im, mean.img)\n ```\n\nClassify the Image\n------------------\nNow we are ready to classify the image! Use the ```predict``` function\nto get the probability over classes:\n\n\n ```r\n    prob <- predict(model, X=normed)\n    dim(prob)\n ```\n\n ```\n    ## [1] 1000    1\n ```\n\nAs you can see, ```prob``` is a 1 times 1000 array, which gives the probability\nover the 1000 image classes of the input.\n\nUse the ```max.col``` on the transpose of ```prob``` to get the class index:\n\n ```r\n    max.idx <- max.col(t(prob))\n    max.idx\n ```\n\n ```\n    ## [1] 89\n ```\n\nThe index doesn't make much sense, so let's see what it really means.\nRead the names of the classes from the following file:\n\n\n ```r\n    synsets <- readLines(\"Inception/synset.txt\")\n ```\n\nLet's see what the image really is:\n\n\n ```r\n    print(paste0(\"Predicted Top-class: \", synsets  [[max.idx]]))\n ```\n\n ```\n    ## [1] \"Predicted Top-class: n01818515 macaw\"\n ```\n\nIt's a macaw!\n\nReference\n---------\n[1] Ioffe, Sergey, and Christian Szegedy. \"Batch normalization: Accelerating deep network training by reducing internal covariate shift.\" arXiv preprint arXiv:1502.03167 (2015).\n\n## Next Steps\n* [Handwritten Digits Classification Competition](https://mxnet.io/tutorials/r/mnistCompetition.html)\n* [Character Language Model using RNN](https://mxnet.io/tutorials/r/charRnnModel.html)\n"
  },
  {
    "path": "docs/static_site/src/pages/api/r/docs/tutorials/custom_iterator.md",
    "content": "---\nlayout: page_api\ntitle: Custom Iterator Tutorial\nis_tutorial: true\ntag: r\npermalink: /api/r/docs/tutorials/custom_iterator\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\nCustom Iterator Tutorial\n========================\n\nThis tutorial provides a guideline on how to use and write custom iterators, which can very useful when having a dataset that does not fit into memory.\n\nGetting the data\n----------\nThe data we are going to use is the [MNIST dataset](https://yann.lecun.com/exdb/mnist/) in CSV format, the data can be found in this [web](https://pjreddie.com/projects/mnist-in-csv/).\n\nTo download the data:\n\n```bash\nwget http://pjreddie.com/media/files/mnist_train.csv\nwget http://pjreddie.com/media/files/mnist_test.csv\n```\n\nYou'll get two files, `mnist_train.csv` that contains 60.000 examples of hand written numbers and `mxnist_test.csv` that contains 10.000 examples. The first element of each line in the CSV is the label, which is a number between 0 and 9. The rest of the line are 784 numbers between 0 and 255, corresponding to the levels of grey of a matrix of 28x28. Therefore, each line contains an image of 28x28 pixels of a hand written number and its true label.\n\nCustom CSV Iterator\n----------\nNext we are going to create a custom CSV Iterator based on the [C++ CSVIterator class](https://github.com/apache/mxnet/blob/master/src/io/iter_csv.cc).\n\nFor that we are going to use the R function `mx.io.CSVIter` as a base class. This class has as parameters `data.csv, data.shape, batch.size` and two main functions, `iter.next()` that calls the iterator in the next batch of data and `value()` that returns the train data and the label.\n\nThe R Custom Iterator needs to inherit from the C++ data iterator class, for that we used the class `Rcpp_MXArrayDataIter` extracted with RCPP. Also, it needs to have the same parameters: `data.csv, data.shape, batch.size`. Apart from that, we can also add the field `iter`, which is the CSV Iterator that we are going to expand.\n\n```r\nCustomCSVIter <- setRefClass(\"CustomCSVIter\",\n\t\t\t\t\t\t\t\tfields=c(\"iter\", \"data.csv\", \"data.shape\", \"batch.size\"),\n\t\t\t\t\t\t\t\tcontains = \"Rcpp_MXArrayDataIter\",\n\t\t\t\t\t\t\t\t#...\n                            )\n```\n\nThe next step is to initialize the class. For that we call the base `mx.io.CSVIter` and fill the rest of the fields.\n\n```r\nCustomCSVIter <- setRefClass(\"CustomCSVIter\",\n\t\t\t\t\t\t\t\tfields=c(\"iter\", \"data.csv\", \"data.shape\", \"batch.size\"),\n\t\t\t\t\t\t\t\tcontains = \"Rcpp_MXArrayDataIter\",\n\t\t\t\t\t\t\t\tmethods=list(\n\t                             \tinitialize=function(iter, data.csv, data.shape, batch.size){\n\t\t\t\t\t\t\t\t\t\tfeature_len <- data.shape*data.shape + 1\n\t\t\t\t\t\t\t\t\t\tcsv_iter <- mx.io.CSVIter(data.csv=data.csv, data.shape=c(feature_len), batch.size=batch.size)\n\t\t\t\t\t\t\t\t\t\t.self$iter <- csv_iter\n\t\t\t\t\t\t\t\t\t\t.self$data.csv <- data.csv\n\t\t\t\t\t\t\t\t\t\t.self$data.shape <- data.shape\n\t\t\t\t\t\t\t\t\t\t.self$batch.size <- batch.size\n\t\t\t\t\t\t\t\t\t\t.self\n\t                               \t},\n                             \t#...\n                             \t)\n                            )\n```\n\nSo far there is no difference between the original class and the custom class. Let's implement the function `value()`. In this case what we are going to do is transform the data that comes from the original class as an array of 785 numbers into a matrix of 28x28 and a label. We will also normalize the training data to be between 0 and 1.\n\n```r\nCustomCSVIter <- setRefClass(\"CustomCSVIter\",\n\t\t\t\t\t\t\t\tfields=c(\"iter\", \"data.csv\", \"data.shape\", \"batch.size\"),\n\t\t\t\t\t\t\t\tcontains = \"Rcpp_MXArrayDataIter\",\n\t\t\t\t\t\t\t\tmethods=list(\n\t                             \tinitialize=function(iter, data.csv, data.shape, batch.size){\n\t\t\t\t\t\t\t\t\t\tfeature_len <- data.shape*data.shape + 1\n\t\t\t\t\t\t\t\t\t\tcsv_iter <- mx.io.CSVIter(data.csv=data.csv, data.shape=c(feature_len), batch.size=batch.size)\n\t\t\t\t\t\t\t\t\t\t.self$iter <- csv_iter\n\t\t\t\t\t\t\t\t\t\t.self$data.csv <- data.csv\n\t\t\t\t\t\t\t\t\t\t.self$data.shape <- data.shape\n\t\t\t\t\t\t\t\t\t\t.self$batch.size <- batch.size\n\t\t\t\t\t\t\t\t\t\t.self\n\t                               \t},\n\t\t\t\t\t\t\t\t\tvalue=function(){\n\t\t\t\t\t\t\t\t\t\tval <- as.array(.self$iter$value()$data)\n\t\t\t\t\t\t\t\t\t\tval.x <- val[-1,]\n\t\t\t\t\t\t\t\t\t\tval.y <- val[1,]\n\t\t\t\t\t\t\t\t\t\tval.x <- val.x/255\n\t\t\t\t\t\t\t\t\t\tdim(val.x) <- c(data.shape, data.shape, 1, ncol(val.x))\n\t\t\t\t\t\t\t\t\t\tval.x <- mx.nd.array(val.x)\n\t\t\t\t\t\t\t\t\t\tval.y <- mx.nd.array(val.y)\n\t\t\t\t\t\t\t\t\t\tlist(data=val.x, label=val.y)\n\t\t\t\t\t\t\t\t\t},\n                             \t#...\n                             \t)\n                            )\n```\nFinally we are going to add the rest of the functions needed for the training to work correctly. The final `CustomCSVIter` looks like this:\n\n```r\nCustomCSVIter <- setRefClass(\"CustomCSVIter\",\n\t\t\t\t\t\t\t\tfields=c(\"iter\", \"data.csv\", \"data.shape\", \"batch.size\"),\n\t\t\t\t\t\t\t\tcontains = \"Rcpp_MXArrayDataIter\",\n\t\t\t\t\t\t\t\tmethods=list(\n\t                             \tinitialize=function(iter, data.csv, data.shape, batch.size){\n\t\t\t\t\t\t\t\t\t\tfeature_len <- data.shape*data.shape + 1\n\t\t\t\t\t\t\t\t\t\tcsv_iter <- mx.io.CSVIter(data.csv=data.csv, data.shape=c(feature_len), batch.size=batch.size)\n\t\t\t\t\t\t\t\t\t\t.self$iter <- csv_iter\n\t\t\t\t\t\t\t\t\t\t.self$data.csv <- data.csv\n\t\t\t\t\t\t\t\t\t\t.self$data.shape <- data.shape\n\t\t\t\t\t\t\t\t\t\t.self$batch.size <- batch.size\n\t\t\t\t\t\t\t\t\t\t.self\n\t                               \t},\n\t\t\t\t\t\t\t\t\tvalue=function(){\n\t\t\t\t\t\t\t\t\t\tval <- as.array(.self$iter$value()$data)\n\t\t\t\t\t\t\t\t\t\tval.x <- val[-1,]\n\t\t\t\t\t\t\t\t\t\tval.y <- val[1,]\n\t\t\t\t\t\t\t\t\t\tval.x <- val.x/255\n\t\t\t\t\t\t\t\t\t\tdim(val.x) <- c(data.shape, data.shape, 1, ncol(val.x))\n\t\t\t\t\t\t\t\t\t\tval.x <- mx.nd.array(val.x)\n\t\t\t\t\t\t\t\t\t\tval.y <- mx.nd.array(val.y)\n\t\t\t\t\t\t\t\t\t\tlist(data=val.x, label=val.y)\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\titer.next=function(){\n\t\t\t\t\t\t\t\t\t\t.self$iter$iter.next()\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\treset=function(){\n\t\t\t\t\t\t\t\t\t\t.self$iter$reset()\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\tnum.pad=function(){\n\t\t\t\t\t\t\t\t\t\t.self$iter$num.pad()\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\tfinalize=function(){\n\t\t\t\t\t\t\t\t\t\t.self$iter$finalize()\n\t\t\t\t\t\t\t\t\t}\n                             \t)\n                            )\n```\n\nTo call the class we can just do:\n\n```r\nbatch.size <- 100\ntrain.iter <- CustomCSVIter$new(iter = NULL, data.csv = \"mnist_train.csv\", data.shape = 28, batch.size = batch.size)\n```\n\n\nConclusion\n----------\n\nWe have shown how to create a custom CSV Iterator by extending the class `mx.io.CSVIter`. In our class, we iteratively read from a CSV file a batch of data that will be transformed and then processed in the stochastic gradient descent optimization. That way, we are able to manage CSV files that are bigger than the memory of the machine we are using.\n\nBased of this custom iterator, we can also create data loaders that internally transform or expand the data, allowing to manage files of any size.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/r/docs/tutorials/index.md",
    "content": "---\nlayout: page_landing_tutorials\ntitle:  R Tutorials\naction: Get Started\ntag: r\npermalink: /api/r/docs/tutorials\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n"
  },
  {
    "path": "docs/static_site/src/pages/api/r/docs/tutorials/multi_dim_lstm.md",
    "content": "---\nlayout: page_api\ntitle: LSTM Time Series\nis_tutorial: true\ntag: r\npermalink: /api/r/docs/tutorials/multi_dim_lstm\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\nLSTM Time Series Example\n========================\n\nThis tutorial shows how to use an LSTM model with multivariate data, and generate predictions from it. For demonstration purposes, we used an open source [pollution data](https://archive.ics.uci.edu/ml/datasets/Beijing+PM2.5+Data).\nThe tutorial is an illustration of how to use LSTM models with MXNet-R. We are forecasting the air pollution with data recorded at the US embassy in Beijing, China for five years.\n\nDataset Attribution:\n\"PM2.5 data of US Embassy in Beijing\"\nWe want to predict pollution levels(PM2.5 concentration) in the city given the above dataset.\n\n```r\nDataset description:\nNo: row number\nyear: year of data in this row\nmonth: month of data in this row\nday: day of data in this row\nhour: hour of data in this row\npm2.5: PM2.5 concentration\nDEWP: Dew Point\nTEMP: Temperature\nPRES: Pressure\ncbwd: Combined wind direction\nIws: Cumulated wind speed\nIs: Cumulated hours of snow\nIr: Cumulated hours of rain\n```\n\nWe use past PM2.5 concentration, dew point, temperature, pressure, wind speed, snow and rain to predict\nPM2.5 concentration levels.\n\nLoad and pre-process the data\n---------\nThe first step is to load in the data and preprocess it. It is assumed that the data has been downloaded in a .csv file: data.csv from the [pollution dataset](https://archive.ics.uci.edu/ml/datasets/Beijing+PM2.5+Data).\n\n ```r\n## Loading required packages\nlibrary(\"readr\")\nlibrary(\"dplyr\")\nlibrary(\"mxnet\")\nlibrary(\"abind\")\n ```\n\n\n\n ```r\n## Preprocessing steps\nData <- read.csv(file = \"/Users/khedia/Downloads/data.csv\",\n                 header = TRUE,\n                 sep = \",\")\n\n## Extracting specific features from the dataset as variables for time series We extract\n## pollution, temperature, pressue, windspeed, snowfall and rainfall information from dataset\ndf <- data.frame(Data$pm2.5,\n                 Data$DEWP,\n                 Data$TEMP,\n                 Data$PRES,\n                 Data$Iws,\n                 Data$Is,\n                 Data$Ir)\ndf[is.na(df)] <- 0\n\n## Now we normalise each of the feature set to a range(0,1)\ndf <- matrix(as.matrix(df),\n             ncol = ncol(df),\n             dimnames = NULL)\n\nrangenorm <- function(x) {\n    (x - min(x))/(max(x) - min(x))\n}\ndf <- apply(df, 2, rangenorm)\ndf <- t(df)\n  ```\nFor using multidimesional data with MXNet-R, we need to convert training data to the form\n(n_dim x seq_len x num_samples). For one-to-one RNN flavours labels should be of the form (seq_len x num_samples) while for many-to-one flavour, the labels should be of the form (1 x num_samples). Please note that MXNet-R currently supports only these two flavours of RNN.\nWe have used n_dim = 7, seq_len = 100,  and num_samples = 430 because the dataset has 430 samples, each the length of 100 timestamps, we have seven time series as input features so each input has dimesnion of seven at each time step.\n\n\n```r\nn_dim <- 7\nseq_len <- 100\nnum_samples <- 430\n\n## extract only required data from dataset\ntrX <- df[1:n_dim, 25:(24 + (seq_len * num_samples))]\n\n## the label data(next PM2.5 concentration) should be one time step\n## ahead of the current PM2.5 concentration\ntrY <- df[1, 26:(25 + (seq_len * num_samples))]\n\n## reshape the matrices in the format acceptable by MXNetR RNNs\ntrainX <- trX\ndim(trainX) <- c(n_dim, seq_len, num_samples)\ntrainY <- trY\ndim(trainY) <- c(seq_len, num_samples)\n```\n\n\n\nDefining and training the network\n---------\n\n```r\nbatch.size <- 32\n\n# take first 300 samples for training - remaining 100 for evaluation\ntrain_ids <- 1:300\neval_ids <- 301:400\n\n## The number of samples used for training and evaluation is arbitrary.  I have kept aside few\n## samples for testing purposes create dataiterators\ntrain.data <- mx.io.arrayiter(data = trainX[, , train_ids, drop = F],\n                              label = trainY[, train_ids],\n                              batch.size = batch.size, shuffle = TRUE)\n\neval.data <- mx.io.arrayiter(data = trainX[, , eval_ids, drop = F],\n                             label = trainY[, eval_ids],\n                             batch.size = batch.size, shuffle = FALSE)\n\n## Create the symbol for RNN\nsymbol <- rnn.graph(num_rnn_layer = 1,\n                    num_hidden = 5,\n                    input_size = NULL,\n                    num_embed = NULL,\n                    num_decode = 1,\n                    masking = F,\n                    loss_output = \"linear\",\n                    dropout = 0.2,\n                    ignore_label = -1,\n                    cell_type = \"lstm\",\n                    output_last_state = T,\n                    config = \"one-to-one\")\n\n\n\nmx.metric.mse.seq <- mx.metric.custom(\"MSE\", function(label, pred) {\n    label = mx.nd.reshape(label, shape = -1)\n    pred = mx.nd.reshape(pred, shape = -1)\n    res <- mx.nd.mean(mx.nd.square(label - pred))\n    return(as.array(res))\n})\n\n\n\nctx <- mx.cpu()\n\ninitializer <- mx.init.Xavier(rnd_type = \"gaussian\",\n                              factor_type = \"avg\",\n                              magnitude = 3)\n\noptimizer <- mx.opt.create(\"adadelta\",\n                           rho = 0.9,\n                           eps = 1e-05,\n                           wd = 1e-06,\n                           clip_gradient = 1,\n                           rescale.grad = 1/batch.size)\n\nlogger <- mx.metric.logger()\nepoch.end.callback <- mx.callback.log.train.metric(period = 10,\n                                                   logger = logger)\n\n## train the network\nsystem.time(model <- mx.model.buckets(symbol = symbol,\n                                      train.data = train.data,\n                                      eval.data = eval.data,\n                                      num.round = 100,\n                                      ctx = ctx,\n                                      verbose = TRUE,\n                                      metric = mx.metric.mse.seq,\n                                      initializer = initializer,\n                                      optimizer = optimizer,\n                                      batch.end.callback = NULL,\n                                      epoch.end.callback = epoch.end.callback))\n```\nOutput:\n```\nStart training with 1 devices\n[1] Train-MSE=0.197570244409144\n[1] Validation-MSE=0.0153861071448773\n[2] Train-MSE=0.0152517843060195\n[2] Validation-MSE=0.0128299412317574\n[3] Train-MSE=0.0124418652616441\n[3] Validation-MSE=0.010827143676579\n[4] Train-MSE=0.0105128229130059\n[4] Validation-MSE=0.00940261723008007\n[5] Train-MSE=0.00914482437074184\n[5] Validation-MSE=0.00830172537826002\n[6] Train-MSE=0.00813581114634871\n[6] Validation-MSE=0.00747016374953091\n[7] Train-MSE=0.00735094994306564\n[7] Validation-MSE=0.00679832429159433\n[8] Train-MSE=0.00672049634158611\n[8] Validation-MSE=0.00623159145470709\n[9] Train-MSE=0.00620287149213254\n[9] Validation-MSE=0.00577476259786636\n[10] Train-MSE=0.00577280316501856\n[10] Validation-MSE=0.00539038667920977\n..........\n..........\n[91] Train-MSE=0.00177705133100972\n[91] Validation-MSE=0.00154715491225943\n[92] Train-MSE=0.00177639147732407\n[92] Validation-MSE=0.00154592350008897\n[93] Train-MSE=0.00177577760769054\n[93] Validation-MSE=0.00154474508599378\n[94] Train-MSE=0.0017752077546902\n[94] Validation-MSE=0.0015436161775142\n[95] Train-MSE=0.00177468206966296\n[95] Validation-MSE=0.00154253660002723\n[96] Train-MSE=0.00177419915562496\n[96] Validation-MSE=0.00154150440357625\n[97] Train-MSE=0.0017737578949891\n[97] Validation-MSE=0.00154051734716631\n[98] Train-MSE=0.00177335749613121\n[98] Validation-MSE=0.00153957353904843\n[99] Train-MSE=0.00177299699280411\n[99] Validation-MSE=0.00153867155313492\n[100] Train-MSE=0.00177267640829086\n[100] Validation-MSE=0.00153781197150238\n\n   user  system elapsed\n 21.937   1.914  13.402\n```\nWe can see how mean squared error varies with epochs below.\n\n![png](https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/r/images/loss.png?raw=true)<!--notebook-skip-line-->\n\nInference on the network\n---------\nNow we have trained the network. Let's use it for inference.\n\n```r\n## We extract the state symbols for RNN\ninternals <- model$symbol$get.internals()\nsym_state <- internals$get.output(which(internals$outputs %in% \"RNN_state\"))\nsym_state_cell <- internals$get.output(which(internals$outputs %in% \"RNN_state_cell\"))\nsym_output <- internals$get.output(which(internals$outputs %in% \"loss_output\"))\nsymbol <- mx.symbol.Group(sym_output, sym_state, sym_state_cell)\n\n## We will predict 100 timestamps for 401st sample (first sample from the test samples)\npred_length <- 100\npredicted <- numeric()\n\n## We pass the 400th sample through the network to get the weights and use it for predicting next\n## 100 time stamps.\ndata <- mx.nd.array(trainX[, , 400, drop = F])\nlabel <- mx.nd.array(trainY[, 400, drop = F])\n\n\n## We create dataiterators for the input, please note that the label is required to create\n## iterator and will not be used in the inference. You can use dummy values too in the label.\ninfer.data <- mx.io.arrayiter(data = data,\n                              label = label,\n                              batch.size = 1,\n                              shuffle = FALSE)\n\ninfer <- mx.infer.rnn.one(infer.data = infer.data,\n                          symbol = symbol,\n                          arg.params = model$arg.params,\n                          aux.params = model$aux.params,\n                          input.params = NULL,\n                          ctx = ctx)\n## Once we get the weights for the above time series, we try to predict the next 100 steps for\n## this time series, which is technically our 401st time series.\n\nactual <- trainY[, 401]\n\n## Now we iterate one by one to generate each of the next timestamp pollution values\n\nfor (i in 1:pred_length) {\n\n    data <- mx.nd.array(trainX[, i, 401, drop = F])\n    label <- mx.nd.array(trainY[i, 401, drop = F])\n    infer.data <- mx.io.arrayiter(data = data,\n                                  label = label,\n                                  batch.size = 1,\n                                  shuffle = FALSE)\n    ## note that we use rnn state values from previous iterations here\n    infer <- mx.infer.rnn.one(infer.data = infer.data,\n                              symbol = symbol,\n                              ctx = ctx,\n                              arg.params = model$arg.params,\n                              aux.params = model$aux.params,\n                              input.params = list(rnn.state = infer[[2]],\n                                                  rnn.state.cell = infer[[3]]))\n\n    pred <- infer[[1]]\n    predicted <- c(predicted, as.numeric(as.array(pred)))\n\n}\n\n```\nNow predicted contains the predicted 100 values. We use ggplot to plot the actual and predicted values as shown below.\n\n![png](https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/r/images/sample_401.png?raw=true)<!--notebook-skip-line-->\n\nWe also repeated the above experiments to generate the next 100 samples to 301st time series and we got the following results.\n\n![png](https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/r/images/sample_301.png?raw=true)<!--notebook-skip-line-->\n\nThe above tutorial is just for demonstration purposes and has not been tuned extensively for accuracy.\n\nFor more tutorials on MXNet-R, head on to [MXNet-R tutorials](/api/r/docs/tutorials)\n"
  },
  {
    "path": "docs/static_site/src/pages/api/r/docs/tutorials/ndarray.md",
    "content": "---\nlayout: page_api\ntitle: NDArray\nis_tutorial: true\ntag: r\npermalink: /api/r/docs/tutorials/ndarray\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# NDArray: Vectorized Tensor Computations on CPUs and GPUs\n\n`NDArray` is the basic vectorized operation unit in MXNet for matrix and tensor computations.\nUsers can perform usual calculations as on an R\"s array, but with two additional features:\n\n\n\n- Multiple devices: All operations can be run on various devices including\nCPUs and GPUs.\n\n\n- Automatic parallelization: All operations are automatically executed in\n   parallel with each other.\n\n## Create and Initialize\n\nLet\"s create `NDArray` on either a GPU or a CPU:\n\n\n```r\nrequire(mxnet)\n```\n\n```\n## Loading required package: mxnet\n## Loading required package: methods\n```\n\n```r\na <- mx.nd.zeros(c(2, 3)) # create a 2-by-3 matrix on cpu\nb <- mx.nd.zeros(c(2, 3), mx.cpu()) # create a 2-by-3 matrix on cpu\n# c <- mx.nd.zeros(c(2, 3), mx.gpu(0)) # create a 2-by-3 matrix on gpu 0, if you have CUDA enabled.\n```\n\nTypically for CUDA-enabled devices, the device id of a GPU starts from 0.\nThat's why we passed in 0 to the GPU id.\n\nWe can initialize an `NDArray` object in various ways:\n\n\n```r\na <- mx.nd.ones(c(4, 4))\nb <- mx.rnorm(c(4, 5))\nc <- mx.nd.array(1:5)\n```\n\nTo check the numbers in an `NDArray`, we can simply run:\n\n\n```r\na <- mx.nd.ones(c(2, 3))\nb <- as.array(a)\nclass(b)\n```\n\n```\n## [1] \"matrix\"\n```\n\n```r\nb\n```\n\n```\n##      [,1] [,2] [,3]\n## [1,]    1    1    1\n## [2,]    1    1    1\n```\n\n## Performing Basic Operations\n\n### Elemental-wise Operations\n\nYou can perform elemental-wise operations on `NDArray` objects, as follows:\n\n\n```r\na <- mx.nd.ones(c(2, 4)) * 2\nb <- mx.nd.ones(c(2, 4)) / 8\nas.array(a)\n```\n\n```\n##      [,1] [,2] [,3] [,4]\n## [1,]    2    2    2    2\n## [2,]    2    2    2    2\n```\n\n```r\nas.array(b)\n```\n\n```\n##       [,1]  [,2]  [,3]  [,4]\n## [1,] 0.125 0.125 0.125 0.125\n## [2,] 0.125 0.125 0.125 0.125\n```\n\n```r\nc <- a + b\nas.array(c)\n```\n\n```\n##       [,1]  [,2]  [,3]  [,4]\n## [1,] 2.125 2.125 2.125 2.125\n## [2,] 2.125 2.125 2.125 2.125\n```\n\n```r\nd <- c / a - 5\nas.array(d)\n```\n\n```\n##         [,1]    [,2]    [,3]    [,4]\n## [1,] -3.9375 -3.9375 -3.9375 -3.9375\n## [2,] -3.9375 -3.9375 -3.9375 -3.9375\n```\n\nIf two `NDArray`s are located on different devices, we need to explicitly move them to the same one. For instance:\n\n\n```r\na <- mx.nd.ones(c(2, 3)) * 2\nb <- mx.nd.ones(c(2, 3), mx.gpu()) / 8\nc <- mx.nd.copyto(a, mx.gpu()) * b\nas.array(c)\n```\n\n### Loading and Saving\n\nYou can save a list of `NDArray` object to your disk with `mx.nd.save`:\n\n\n```r\na <- mx.nd.ones(c(2, 3))\nmx.nd.save(list(a), \"temp.ndarray\")\n```\n\nYou can load it back easily:\n\n\n```r\na <- mx.nd.load(\"temp.ndarray\")\nas.array(a[[1]])\n```\n\n```\n##      [,1] [,2] [,3]\n## [1,]    1    1    1\n## [2,]    1    1    1\n```\n\nWe can directly save data to and load it from a distributed file system, such as Amazon S3 and HDFS:\n\n\n```r\nmx.nd.save(list(a), \"s3://mybucket/mydata.bin\")\nmx.nd.save(list(a), \"hdfs///users/myname/mydata.bin\")\n```\n\n## Automatic Parallelization\n\n`NDArray` can automatically execute operations in parallel. Automatic parallelization is useful when\nusing multiple resources, such as CPU cards, GPU cards, and CPU-to-GPU memory bandwidth.\n\nFor example, if we write `a <- a + 1` followed by `b <- b + 1`, and `a` is on a CPU and\n`b` is on a GPU, executing them in parallel improves\nefficiency. Furthermore, because copying data between CPUs and GPUs are also expensive, running in parallel with other computations further increases efficiency.\n\nIt's hard to find the code that can be executed in parallel by eye. In the\nfollowing example, `a <- a + 1` and `c <- c * 3` can be executed in parallel, but `a <- a + 1` and\n`b <- b * 3` should be in sequential.\n\n\n```r\na <- mx.nd.ones(c(2,3))\nb <- a\nc <- mx.nd.copyto(a, mx.cpu())\na <- a + 1\nb <- b * 3\nc <- c * 3\n```\n\nLuckily, MXNet can automatically resolve the dependencies and\nexecute operations in parallel accurately. This allows us to write our program assuming there is only a single thread. MXNet will\nautomatically dispatch the program to multiple devices.\n\nMXNet achieves this with lazy evaluation. Each operation is issued to an\ninternal engine, and then returned. For example, if we run `a <- a + 1`, it\nreturns immediately after pushing the plus operator to the engine. This\nasynchronous processing allows us to push more operators to the engine. It determines\nthe read and write dependencies and the best way to execute them in\nparallel.\n\nThe actual computations are finished, allowing us to copy the results someplace else, such as `as.array(a)` or `mx.nd.save(a, \"temp.dat\")`. To write highly parallelized codes, we only need to postpone when we need\nthe results.\n\n## Next Steps\n* [Symbol](/api/r/docs/tutorials/symbol)\n* [Classify Real-World Images with Pre-trained Model](/api/r/docs/tutorials/classify_real_image_with_pretrained_model)\n* [Character Language Model using RNN](/api/r/docs/tutorials/char_rnn_model)\n"
  },
  {
    "path": "docs/static_site/src/pages/api/r/docs/tutorials/symbol.md",
    "content": "---\nlayout: page_api\ntitle: NDArray\nis_tutorial: true\ntag: r\npermalink: /api/r/docs/tutorials/symbol\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Symbol and Automatic Differentiation\n\nThe computational unit `NDArray` requires a way to construct neural networks. MXNet provides a symbolic interface, named Symbol, to do this. Symbol combines both flexibility and efficiency.\n\n## Basic Composition of Symbols\n\nThe following code creates a two-layer perceptron network:\n\n```\nrequire(mxnet)\n## [1] \"Rcpp_MXSymbol\"\n## attr(,\"package\")\n## [1] \"mxnet\"\n```\n\nEach symbol takes a (unique) string name. *Variable* often defines the inputs,\nor free variables. Other symbols take a symbol as the input (*data*),\nand may accept other hyper parameters, such as the number of hidden neurons (*num_hidden*)\nor the activation type (*act_type*).\n\nWe can also specify the names explicitly:\n\n```r\ndata <- mx.symbol.Variable(\"data\")\nw <- mx.symbol.Variable(\"myweight\")\nnet <- mx.symbol.FullyConnected(data=data, weight=w, name=\"fc1\", num_hidden=128)\narguments(net)\n```\n\n```\n## [1] \"data\"     \"myweight\" \"fc1_bias\"\n```\n\n## More Complicated Composition of Symbols\n\nMXNet provides well-optimized symbols for\ncommonly used layers in deep learning. You can also define new operators\nin Python. The following example first performs an element-wise add between two\nsymbols, then feeds them to the fully connected operator:\n\n\n```r\nlhs <- mx.symbol.Variable(\"data1\")\nrhs <- mx.symbol.Variable(\"data2\")\nnet <- mx.symbol.FullyConnected(data=lhs + rhs, name=\"fc1\", num_hidden=128)\narguments(net)\n```\n\n```\n## [1] \"data1\"      \"data2\"      \"fc1_weight\" \"fc1_bias\"\n```\n\nWe can construct a symbol more flexibly than by using the single\nforward composition, for example:\n\n\n```r\nnet <- mx.symbol.Variable(\"data\")\nnet <- mx.symbol.FullyConnected(data=net, name=\"fc1\", num_hidden=128)\nnet2 <- mx.symbol.Variable(\"data2\")\nnet2 <- mx.symbol.FullyConnected(data=net2, name=\"net2\", num_hidden=128)\ncomposed.net <- mx.apply(net, data=net2, name=\"compose\")\narguments(composed.net)\n```\n\n```\n## [1] \"data2\"       \"net2_weight\" \"net2_bias\"   \"fc1_weight\"  \"fc1_bias\"\n```\n\nIn the example, *net* is used as a function to apply to an existing symbol\n*net*. The resulting *composed.net* will replace the original argument *data* with\n*net2* instead.\n\n## Training a Neural Net\n\nThe [model API](https://github.com/apache/mxnet/blob/master/R-package/R/model.R) is a thin wrapper around the symbolic executors to support neural net training.\n\nWe encourage you to read [Symbolic Configuration and Execution in Pictures for python package](/api/python/symbol_in_pictures/symbol_in_pictures.md)for a detailed explanation of concepts in pictures.\n\n## How Efficient Is the Symbolic API?\n\nThe Symbolic API brings the efficient C++\noperations in powerful toolkits, such as CXXNet and Caffe, together with the\nflexible dynamic NDArray operations. All of the memory and computation resources are\nallocated statically during bind operations, to maximize runtime performance and memory\nutilization.\n\nThe coarse-grained operators are equivalent to CXXNet layers, which are\nextremely efficient.  We also provide fine-grained operators for more flexible\ncomposition. Because MXNet does more in-place memory allocation, it can\nbe more memory efficient than CXXNet and gets to the same runtime with\ngreater flexibility.\n\n## Next Steps\n* [Classify Real-World Images with Pre-trained Model](/api/r/docs/tutorials/classify_real_image_with_pretrained_model)\n* [Character Language Model using RNN](/api/r/docs/tutorials/char_rnn_model)\n"
  },
  {
    "path": "docs/static_site/src/pages/api/r/index.md",
    "content": "---\nlayout: page_api\ntitle: R Guide\naction: Get Started\naction_url: /get_started\npermalink: /api/r\ntag: r\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# MXNet - R API\n\nSee the [MXNet R Reference Manual](/api/r/docs/api/R-package/build/mxnet-r-reference-manual.pdf).\n\nMXNet supports the R programming language. The MXNet R package brings flexible and efficient GPU\ncomputing and state-of-art deep learning to R. It enables you to write seamless tensor/matrix computation with multiple GPUs in R. It also lets you construct and customize the state-of-art deep learning models in R,\n  and apply them to tasks, such as image classification and data science challenges.\n\nYou can perform tensor or matrix computation in R:\n\n```r\n   > require(mxnet)\n   Loading required package: mxnet\n   > a <- mx.nd.ones(c(2,3))\n   > a\n        [,1] [,2] [,3]\n   [1,]    1    1    1\n   [2,]    1    1    1\n   > a + 1\n        [,1] [,2] [,3]\n   [1,]    2    2    2\n   [2,]    2    2    2\n```\n"
  },
  {
    "path": "docs/static_site/src/pages/api/scala/docs/tutorials/index.md",
    "content": "---\nlayout: page_landing_tutorials\ntitle: Scala Tutorials\npermalink: /api/scala/docs/tutorials\ntag: scala\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n"
  },
  {
    "path": "docs/static_site/src/pages/api/scala/docs/tutorials/infer.md",
    "content": "---\nlayout: page_api\ntitle: Infer API\nis_tutorial: true\ntag: scala\npermalink: /api/scala/docs/tutorials/infer\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Infer API\nThe MXNet Scala Infer API provides you with model loading and inference functionality using the MXNet Scala package.\n\n\n## Prerequisites\nTo use the Infer API you must first install the MXNet Scala package. Instructions for this are provided in the following variations:\n* [Tutorial for setting up a project in the IntelliJ IDE](mxnet_scala_on_intellij)\n* [Installing the MXNet Scala Package for macOS]({{'get_started/ubuntu_setup.html#install-the-mxnet-package-for-scala'|relative_url}})\n* [Installing the MXNet Scala for Linux]({{'get_started/ubuntu_setup.html#install-the-mxnet-package-for-scala'|relative_url}})\n\n## Inference\nThe Scala Infer API includes both single image and batch modes. Here is an example of running inference on a single image by using the `ImageClassifier` class. A complete [image classification example](https://github.com/apache/mxnet/blob/master/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExample.scala) using ResNet-152 is provided in the [Scala package's example folder](https://github.com/apache/mxnet/tree/master/scala-package/examples/src/main/scala/org/apache/mxnetexamples). This example also demonstrates inference with batches of images.\n\n```scala\ndef runInferenceOnSingleImage(modelPathPrefix: String, inputImagePath: String,\n                              context: Array[Context]):\nIndexedSeq[IndexedSeq[(String, Float)]] = {\n  val dType = DType.Float32\n  val inputShape = Shape(1, 3, 224, 224)\n\n  val inputDescriptor = IndexedSeq(DataDesc(\"data\", inputShape, dType, \"NCHW\"))\n\n  // Create object of ImageClassifier class\n  val imgClassifier: ImageClassifier = new\n      ImageClassifier(modelPathPrefix, inputDescriptor, context)\n\n  // Loading single image from file and getting BufferedImage\n  val img = ImageClassifier.loadImageFromFile(inputImagePath)\n\n  // Running inference on single image\n  val output = imgClassifier.classifyImage(img, Some(5))\n\n  output\n}\n```\n\n\n## Related Resources\n* [Infer API Scaladocs]({{'/api/scala/docs/api/#org.apache.mxnet.infer.package'|relative_url}})\n* [Single Shot Detector Inference Example](https://github.com/apache/mxnet/tree/master/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/objectdetector)\n* [Image Classification Example](https://github.com/apache/mxnet/tree/master/scala-package/examples/src/main/scala/org/apache/mxnetexamples/infer/imageclassifier)\n"
  },
  {
    "path": "docs/static_site/src/pages/api/scala/docs/tutorials/io.md",
    "content": "---\nlayout: page_api\ntitle: Data Loading API\npermalink: /api/scala/docs/tutorials/io\nis_tutorial: true\ntag: scala\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# MXNet Scala Data Loading API\nThis topic introduces the data input method for MXNet. MXNet uses an iterator to provide data to the neural network.  Iterators do some preprocessing and generate batches for the neural network.\n\nMXNet provides basic iterators for MNIST and RecordIO images. To hide the cost of I/O, MXNet uses a prefetch strategy that enables parallelism for the learning process and data fetching. Data is automatically fetched by an independent thread.\n\nTopics:\n\n* [Data Iterator Parameters](#parameters-for-data-iterator) clarifies the different usages for dataiter parameters.\n* [Create a Data Iterator](#create-a-data-iterator) introduces how to create a data iterator in MXNet for Scala.\n* [How to Get Data](#how-to-get-data) introduces the data resource and data preparation tools.\n* [IO API Reference]({{'/api/scala/docs/api/#org.apache.mxnet.io.package'|relative_url}}) explains the IO API.\n\n\n## Data Iterator Parameters\n\nTo create a data iterator, you typically need to provide five parameters:\n\n* **Dataset Param** provides basic information about the dataset, e.g., file path, input shape.\n* **Batch Param** provides information required to form a batch, e.g., batch size.\n* **Augmentation Param** tells MXNet which augmentation operations (e.g., crop or mirror) to perform on an input image.\n* **Backend Param** controls the behavior of the back-end threads to hide the cost of data loading.\n* **Auxiliary Param** provides options for checking and debugging.\n\nYou *must* provide the **Dataset Param** and **Batch Param**, otherwise MXNet can't create the data batch. Provide other parameters as required by your algorithm and performance needs. We provide a detailed explanation and examples of the options later.\n\n## Create a Data Iterator\n\nThe IO API provides a simple way to create a data iterator in Scala.\nThe following example code shows how to create a CIFAR data iterator.\n\n```scala\nval dataiter = IO.ImageRecordIter(Map(\n    // Utility Parameter\n    // Optional\n    // Name of the data, should match the name of the data input of the network\n    // data_name='data',\n    // Utility Parameter\n    // Optional\n    // Name of the label, should match the name of the label parameter of the network\n    // Usually, if the loss layer is named 'foo', then the label input has the name\n    // 'foo_label', unless overwritten\n    // label_name='softmax_label',\n    // Dataset Parameter\n    // Impulsary\n    // indicating the data file, please check the data is already there\n    \"path_imgrec\" -> \"data/cifar/train.rec\",\n    // Dataset Parameter\n    // Impulsary\n    // indicating the image size after preprocessing\n    \"data_shape\" -> \"(3,28,28)\",\n    // Batch Parameter\n    // Impulsary\n    // tells how many images in a batch\n    \"batch_size\" -> \"100\",\n    // Augmentation Parameter\n    // Optional\n    // when offers mean_img, each image will subtract the mean value at each pixel\n    \"mean_img\" -> \"data/cifar/cifar10_mean.bin\",\n    // Augmentation Parameter\n    // Optional\n    // randomly crop a patch of the data_shape from the original image\n   \"rand_crop\" -> \"True\",\n    // Augmentation Parameter\n    // Optional\n    // randomly mirror the image horizontally\n    \"rand_mirror\" -> \"True\",\n    // Augmentation Parameter\n    // Optional\n    // randomly shuffle the data\n    \"shuffle\" -> \"False\",\n    // Backend Parameter\n    // Optional\n    // Preprocessing thread number\n    \"preprocess_threads\" -> \"4\",\n    // Backend Parameter\n    // Optional\n    // Prefetch buffer size\n    \"prefetch_buffer\" = \"1\"))\n```\n\nFirst, explicitly specify the kind of data (MNIST, ImageRecord, etc.) to fetch. Then, provide the options for the dataset, batching, image augmentation, multi-tread processing,  and prefetching operations. The code automatically validates the parameters. If a required parameter is missing, MXNet returns an error.\n\n## How to Get Data\n\n\nWe provide [scripts](https://github.com/apache/mxnet/tree/master/scala-package/core/scripts) to download MNIST data and CIFAR10 ImageRecord data. If you want to create your own dataset, we recommend using the Image RecordIO data format.\n\n## Create a Dataset Using RecordIO\n\nRecordIO implements a file format for a sequence of records. We recommend storing images as records and packing them together. The benefits include:\n\n* Storing images in a compact format--e.g., JPEG, for records--greatly reduces the size of the dataset on the disk.\n* Packing data together allows continuous reading on the disk.\n* RecordIO has a simple way to partition, simplifying distributed setting. We provide an example later.\n\nWe provide the [im2rec tool](https://github.com/apache/mxnet/blob/master/tools/im2rec.cc) so you can create an Image RecordIO dataset by yourself. The following walkthrough shows you how.\n\n### Prerequisites\nDownload the data. You don't need to resize the images manually. You can use `im2rec` to resize them automatically. For details, see \"Extension: Using Multiple Labels for a Single Image,\" later in this topic.\n\n### Step 1. Make an Image List File\nAfter you download the data, you need to make an image list file.  The format is:\n\n```\ninteger_image_index \\t label_index \\t path_to_image\n```\nTypically, the program takes the list of names of all of the images, shuffles them, then separates them into two lists: a training filename list and a testing filename list. Write the list in the right format.\n\nThis is an example file:\n\n```bash\n95099  464     n04467665_17283.JPEG\n10025081        412     ILSVRC2010_val_00025082.JPEG\n74181   789     n01915811_2739.JPEG\n10035553        859     ILSVRC2010_val_00035554.JPEG\n10048727        929     ILSVRC2010_val_00048728.JPEG\n94028   924     n01980166_4956.JPEG\n1080682 650     n11807979_571.JPEG\n972457  633     n07723039_1627.JPEG\n7534    11      n01630670_4486.JPEG\n1191261 249     n12407079_5106.JPEG\n```\n\n### Step 2. Create the Binary File\nTo generate a binary image, use `im2rec` in the tool folder. `im2rec` takes the path of the `_image list file_` you generated, the `_root path_` of the images, and the `_output file path_` as input. This process usually takes several hours, so be patient.\n\nA sample command:\n\n```bash\n./bin/im2rec image.lst image_root_dir output.bin resize=256\n```\nFor more details, run ```./bin/im2rec```.\n\n### Extension: Multiple Labels for a Single Image\n\nThe `im2rec` tool and `IO.ImageRecordIter` have multi-label support for a single image.\nFor example, if you have four labels for a single image, you can use the following procedure to use the RecordIO tools.\n\n1. Write the image list files as follows:\n\n ```\n integer_image_index \\t label_1 \\t label_2 \\t   label_3 \\t label_4 \\t path_to_image\n ```\n\n2. Run `im2rec`, adding a 'label_width=4' to the command argument, for example:\n\n ```bash\n ./bin/im2rec image.lst image_root_dir output.bin resize=256 label_width=4\n ```\n\n3. In the iterator generation code, set `label_width=4` and `path_imglist=<<The PATH TO YOUR image.lst>>`, for example:\n\n```scala\nval dataiter = IO.ImageRecordIter(Map(\n    \"path_imgrec\" -> \"data/cifar/train.rec\",\n    \"data_shape\" -> \"(3,28,28)\",\n    \"path_imglist\" -> \"data/cifar/image.lst\",\n    \"label_width\" -> \"4\"\n))\n```\n\n## Next Steps\n* [NDArray API](ndarray) for vector/matrix/tensor operations\n* [KVStore API](kvstore) for multi-GPU and multi-host distributed training\n"
  },
  {
    "path": "docs/static_site/src/pages/api/scala/docs/tutorials/kvstore.md",
    "content": "---\nlayout: page_api\ntitle: KVStore API\npermalink: /api/scala/docs/tutorials/kvstore\nis_tutorial: true\ntag: scala\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# KVStore API\n\nTopics:\n* [Basic Push and Pull](#basic-push-and-pull)\n* [List Key-Value Pairs](#list-key-value-pairs)\n* [API Reference]({{'/api/scala/docs/api/#org.apache.mxnet.KVStore'|relative_url}})\n\n\n## Basic Push and Pull\n\nProvides basic operation over multiple devices (GPUs) on a single device.\n\n### Initialization\n\nLet's consider a simple example. It initializes\na (`int`, `NDArray`) pair into the store, and then pulls the value out.\n\n```scala\nval kv = KVStore.create(\"local\") // create a local kv store.\nval shape = Shape(2,3)\nkv.init(3, NDArray.ones(shape)*2)\nval a = NDArray.zeros(shape)\nkv.pull(3, out = a)\na.toArray\n// Array[Float] = Array(2.0, 2.0, 2.0, 2.0, 2.0, 2.0)\n```\n\n### Push, Aggregation, and Updater\n\nFor any key that's been initialized, you can push a new value with the same shape to the key, as follows:\n\n```scala\nkv.push(3, NDArray.ones(shape)*8)\nkv.pull(3, out = a) // pull out the value\na.toArray\n// Array[Float] = Array(8.0, 8.0, 8.0, 8.0, 8.0, 8.0)\n```\n\nThe data that you want to push can be stored on any device. Furthermore, you can push multiple\nvalues into the same key, where KVStore first sums all of these\nvalues, and then pushes the aggregated value, as follows:\n\n```scala\nval gpus = Array(Context.gpu(0), Context.gpu(1), Context.gpu(2), Context.gpu(3))\nval b = Array(NDArray.ones(shape, gpus(0)), NDArray.ones(shape, gpus(1)), \\\nNDArray.ones(shape, gpus(2)), NDArray.ones(shape, gpus(3)))\nkv.push(3, b)\nkv.pull(3, out = a)\na.toArray\n// Array[Float] = Array(4.0, 4.0, 4.0, 4.0, 4.0, 4.0)\n```\n\nFor each push command, KVStore applies the pushed value to the value stored by an\n`updater`. The default updater is `ASSIGN`. You can replace the default to\ncontrol how data is merged.\n\n```scala\nval updater = new MXKVStoreUpdater {\n          override def update(key: Int, input: NDArray, stored: NDArray): Unit = {\n            println(s\"update on key $key\")\n            stored += input * 2\n          }\n          override def dispose(): Unit = {}\n       }\nkv.setUpdater(updater)\nkv.pull(3, a)\na.toArray\n// Array[Float] = Array(4.0, 4.0, 4.0, 4.0, 4.0, 4.0)\nkv.push(3, NDArray.ones(shape))\n// update on key 3\nkv.pull(3, a)\na.toArray\n// Array[Float] = Array(6.0, 6.0, 6.0, 6.0, 6.0, 6.0)\n```\n\n### Pull\n\nYou've already seen how to pull a single key-value pair. Similar to the way that you use the push command, you can\npull the value into several devices with a single call.\n\n```scala\nval b = Array(NDArray.ones(shape, gpus(0)), NDArray.ones(shape, gpus(1)),\\\nNDArray.ones(shape, gpus(2)), NDArray.ones(shape, gpus(3)))\nkv.pull(3, outs = b)\nb(1).toArray\n// Array[Float] = Array(6.0, 6.0, 6.0, 6.0, 6.0, 6.0)\n```\n\n## List Key-Value Pairs\n\nAll of the operations that we've discussed so far are performed on a single key. KVStore also provides\nthe interface for generating a list of key-value pairs. For a single device, use the following:\n\n```scala\nval keys = Array(5, 7, 9)\nkv.init(keys, Array.fill(keys.length)(NDArray.ones(shape)))\nkv.push(keys, Array.fill(keys.length)(NDArray.ones(shape)))\n// update on key: 5\n// update on key: 7\n// update on key: 9\nval b = Array.fill(keys.length)(NDArray.zeros(shape))\nkv.pull(keys, outs = b)\nb(1).toArray\n// Array[Float] = Array(3.0, 3.0, 3.0, 3.0, 3.0, 3.0)\n```\n"
  },
  {
    "path": "docs/static_site/src/pages/api/scala/docs/tutorials/ndarray.md",
    "content": "---\nlayout: page_api\ntitle: NDArray\npermalink: /api/scala/docs/tutorials/ndarray\nis_tutorial: true\ntag: scala\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# NDArray API\n\n\nThe NDArray package (`mxnet.ndarray`) contains tensor operations similar to `numpy.ndarray`. The syntax is also similar, except for some additional calls for dealing with I/O and multiple devices.\n\nTopics:\n\n* [Create NDArray](#create-ndarray)\n* [NDArray Operations](#ndarray-operations)\n* [NDArray API Reference]({{'/api/scala/docs/api/#org.apache.mxnet.NDArray'|relative_url}})\n\n## Create NDArray\n\nCreate `mxnet.ndarray` as follows:\n\n```scala\nimport org.apache.mxnet._\n// all-zero array of dimension 100x50\nval a = NDArray.zeros(100, 50)\n// all-one array of dimension 256x32x128x1\nval b = NDArray.ones(256, 32, 128, 1)\n// initialize array with contents, you can specify dimensions of array using Shape parameter while creating array.\nval c = NDArray.array(Array(1, 2, 3, 4, 5, 6), shape = Shape(2, 3))\n```\nThis is similar to the way you use `numpy`.\n## NDArray Operations\n\nWe provide some basic ndarray operations, like arithmetic and slice operations.\n\n### Arithmetic Operations\n\n```scala\nimport org.apache.mxnet._\nval a = NDArray.zeros(100, 50)\na.shape\n// org.apache.mxnet.Shape = (100,50)\nval b = NDArray.ones(100, 50)\n// c and d will be calculated in parallel here!\nval c = a + b\nval d = a - b\n// inplace operation, b's contents will be modified, but c and d won't be affected.\nb += d\n```\n\n### Multiplication/Division Operations\n\n```scala\nimport org.apache.mxnet._\n// Multiplication\nval ndones = NDArray.ones(2, 1)\nval ndtwos = ndones * 2\nndtwos.toArray\n// Array[Float] = Array(2.0, 2.0)\n(ndones * ndones).toArray\n// Array[Float] = Array(1.0, 1.0)\n(ndtwos * ndtwos).toArray\n// Array[Float] = Array(4.0, 4.0)\nndtwos *= ndtwos // inplace\nndtwos.toArray\n// Array[Float] = Array(4.0, 4.0)\n\n//Division\nval ndones = NDArray.ones(2, 1)\nval ndzeros = ndones - 1f\nval ndhalves = ndones / 2\nndhalves.toArray\n// Array[Float] = Array(0.5, 0.5)\n(ndhalves / ndhalves).toArray\n// Array[Float] = Array(1.0, 1.0)\n(ndones / ndones).toArray\n// Array[Float] = Array(1.0, 1.0)\n(ndzeros / ndones).toArray\n// Array[Float] = Array(0.0, 0.0)\nndhalves /= ndhalves\nndhalves.toArray\n// Array[Float] = Array(1.0, 1.0)\n```\n\n### Slice Operations\n\n```scala\nimport org.apache.mxnet._\nval a = NDArray.array(Array(1f, 2f, 3f, 4f, 5f, 6f), shape = Shape(3, 2))\nval a1 = a.slice(1)\nassert(a1.shape === Shape(1, 2))\nassert(a1.toArray === Array(3f, 4f))\n\nval a2 = arr.slice(1, 3)\nassert(a2.shape === Shape(2, 2))\nassert(a2.toArray === Array(3f, 4f, 5f, 6f))\n```\n\n### Dot Product\n\n```scala\nimport org.apache.mxnet._\nval arr1 = NDArray.array(Array(1f, 2f), shape = Shape(1, 2))\nval arr2 = NDArray.array(Array(3f, 4f), shape = Shape(2, 1))\nval res = NDArray.dot(arr1, arr2)\nres.shape\n// org.apache.mxnet.Shape = (1,1)\nres.toArray\n// Array[Float] = Array(11.0)\n```\n\n### Save and Load NDArray\n\nYou can use MXNet functions to save and load a list or dictionary of NDArrays from file systems, as follows:\n\n```scala\nimport org.apache.mxnet._\nval a = NDArray.zeros(100, 200)\nval b = NDArray.zeros(100, 200)\n// save list of NDArrays\nNDArray.save(\"/path/to/array/file\", Array(a, b))\n// save dictionary of NDArrays to AWS S3\nNDArray.save(\"s3://path/to/s3/array\", Map(\"A\" -> a, \"B\" -> b))\n// save list of NDArrays to hdfs.\nNDArray.save(\"hdfs://path/to/hdfs/array\", Array(a, b))\nval from_file = NDArray.load(\"/path/to/array/file\")\nval from_s3 = NDArray.load(\"s3://path/to/s3/array\")\nval from_hdfs = NDArray.load(\"hdfs://path/to/hdfs/array\")\n```\nThe good thing about using the `save` and `load` interface is that you can use the format across all `mxnet` language bindings. They also already support Amazon S3 and HDFS.\n\n### Multi-Device Support\n\nDevice information is stored in the `mxnet.Context` structure. When creating NDArray in MXNet, you can use the context argument (the default is the CPU context) to create arrays on specific devices as follows:\n\n```scala\nimport org.apache.mxnet._\nval cpu_a = NDArray.zeros(100, 200)\ncpu_a.context\n// org.apache.mxnet.Context = cpu(0)\nval ctx = Context.gpu(0)\nval gpu_b = NDArray.zeros(Shape(100, 200), ctx)\ngpu_b.context\n// org.apache.mxnet.Context = gpu(0)\n```\n\nCurrently, we *do not* allow operations among arrays from different contexts. To manually enable this, use the `copyto` member function to copy the content to different devices, and continue computation:\n\n```scala\nimport org.apache.mxnet._\nval x = NDArray.zeros(100, 200)\nval ctx = Context.gpu(0)\nval y = NDArray.zeros(Shape(100, 200), ctx)\nval z = x + y\n// mxnet.base.MXNetError: [13:29:12] src/ndarray/ndarray.cc:33:\n// Check failed: lhs.ctx() == rhs.ctx() operands context mismatch\nval cpu_y = NDArray.zeros(100, 200)\ny.copyto(cpu_y)\nval z = x + cpu_y\n```\n\n## Next Steps\n* See [KVStore API](kvstore) for multi-GPU and multi-host distributed training.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/scala/docs/tutorials/symbol.md",
    "content": "---\nlayout: page_api\ntitle: Symbol API\npermalink: /api/scala/docs/tutorials/symbol\nis_tutorial: true\ntag: scala\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# MXNet Scala Symbolic API\n\nTopics:\n\n* [How to Compose Symbols](#how-to-compose-symbols) introduces operator overloading of symbols.\n* [Symbol Attributes](#symbol-attributes) describes how to attach attributes to symbols.\n* [Serialization](#serialization) explains how to save and load symbols.\n* [Executing Symbols](#executing-symbols) explains how to evaluate the symbols with data.\n* [Execution API Reference]({{'/api/scala/docs/api/#org.apache.mxnet.Executor'|relative_url}}) documents the execution APIs.\n* [Multiple Outputs](#multiple-outputs) explains how to configure multiple outputs.\n* [Symbol Creation API Reference]({{'/api/scala/docs/api/#org.apache.mxnet.Symbol'|relative_url}}) documents functions.\n\nWe also highly encourage you to read [Symbolic Configuration and Execution in Pictures](symbol_in_pictures).\n\n## How to Compose Symbols\n\nThe symbolic API provides a way to configure computation graphs.\nYou can configure the graphs either at the level of neural network layer operations or as fine-grained operations.\n\nThe basic arithmetic operators (plus, minus, div, multiplication) are overloaded for\n*element-wise operations* of symbols.\n\nThe following example creates a computation graph that adds two inputs together.\n\n```scala\n    import org.apache.mxnet._\n    val a = Symbol.Variable(\"a\")\n    val b = Symbol.Variable(\"b\")\n    val c = a + b\n```\n\n## Symbol Attributes\n\nYou can add an attribute to a symbol by providing an attribute dictionary when you create a symbol.\n\n```scala\n    val data = Symbol.Variable(\"data\", Map(\"mood\"-> \"angry\"))\n    val op = Symbol.api.Convolution(Some(data), kernel = Shape(1, 1), num_filter = 1, attr = Map(\"mood\" -> \"so so\"))\n```\nFor proper communication with the C++ backend, both the key and values of the attribute dictionary should be strings. To retrieve the attributes, use `attr(key)`:\n\n```\n    data.attr(\"mood\")\n    // Option[String] = Some(angry)\n```\n\nTo attach attributes, you can use ```AttrScope```. ```AttrScope``` automatically adds the specified attributes to all of the symbols created within that scope. The user can also inherit this object to change naming behavior. For example:\n\n```scala\n    val (data, gdata) =\n    AttrScope(Map(\"group\" -> \"4\", \"data\" -> \"great\")).withScope {\n      val data = Symbol.Variable(\"data\", attr = Map(\"dtype\" -> \"data\", \"group\" -> \"1\"))\n      val gdata = Symbol.Variable(\"data2\")\n      (data, gdata)\n    }\n    assert(gdata.attr(\"group\").get === \"4\")\n    assert(data.attr(\"group\").get === \"1\")\n\n    val exceedScopeData = Symbol.Variable(\"data3\")\n    assert(exceedScopeData.attr(\"group\") === None, \"No group attr in global attr scope\")\n```\n\n## Serialization\n\nThere are two ways to save and load the symbols. You can use the `mxnet.Symbol.save` and `mxnet.Symbol.load` functions to serialize the ```Symbol``` objects.\nThe advantage of using `save` and `load` functions is that it is language agnostic and cloud friendly.\nThe symbol is saved in JSON format. You can also get a JSON string directly using `mxnet.Symbol.toJson`.\nRefer to [API documentation]({{'/api/scala/docs/api/#org.apache.mxnet.Symbol'|relative_url}}) for more details.\n\nThe following example shows how to save a symbol to an S3 bucket, load it back, and compare two symbols using a JSON string.\n\n```scala\n    import org.apache.mxnet._\n    val a = Symbol.Variable(\"a\")\n    val b = Symbol.Variable(\"b\")\n    val c = a + b\n    c.save(\"s3://my-bucket/symbol-c.json\")\n    val c2 = Symbol.load(\"s3://my-bucket/symbol-c.json\")\n    c.toJson == c2.toJson\n    // Boolean = true\n```\n\n## Executing Symbols\n\nAfter you have assembled a set of symbols into a computation graph, the MXNet engine can evaluate them.\nIf you are training a neural network, this is typically\nhandled by the high-level [Model class](model) and the [`fit()`] function.\n\nFor neural networks used in \"feed-forward\", \"prediction\", or \"inference\" mode (all terms for the same\nthing: running a trained network), the input arguments are the\ninput data, and the weights of the neural network that were learned during training.\n\nTo manually execute a set of symbols, you need to create an [`Executor`] object,\nwhich is typically constructed by calling the [`simpleBind(<parameters>)`] method on a symbol.\n\n## Next Steps\n* See [IO Data Loading API](io) for parsing and loading data.\n* See [NDArray API](ndarray) for vector/matrix/tensor operations.\n* See [KVStore API](kvstore) for multi-GPU and multi-host distributed training.\n"
  },
  {
    "path": "docs/static_site/src/pages/api/scala/docs/tutorials/symbol_in_pictures.md",
    "content": "---\nlayout: page_api\ntitle: Symbol in Pictures\npermalink: /api/scala/docs/tutorials/symbol_in_pictures\nis_tutorial: true\ntag: scala\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Symbolic Configuration and Execution in Pictures\n\nThis topic explains symbolic construction and execution in pictures.\nWe recommend that you also read [Symbolic API](symbol).\n\n## Compose Symbols\n\nSymbols are a description of the computation that you want to perform. The symbolic construction API generates the computation\ngraph that describes the computation. The following picture shows how you compose symbols to describe basic computations.\n\n![Symbol Compose](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/compose_basic.png)\n\n- The ```mxnet.Symbol.Variable``` function creates argument nodes that represent input to the computation.\n- The symbol is overloaded with basic element-wise mathematical operations.\n\n## Configure Neural Networks\n\nIn addition to supporting fine-grained operations, MXNet provides a way to perform big operations that are analogous to layers in neural networks.\nYou can use operators to describe the configuration of a neural network.\n\n![Net Compose](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/compose_net.png)\n\n\n## Example of a Multi-Input Network\n\nThe following example shows how to configure multiple input neural networks.\n\n![Multi Input](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/compose_multi_in.png)\n\n\n## Bind and Execute Symbol\n\nWhen you need to execute a symbol graph, you call the bind function to bind ```NDArrays``` to the argument nodes\nin order to obtain an ```Executor```.\n\n![Bind](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/bind_basic.png)\n\nTo get the output results, given the bound NDArrays as input, you can call ```Executor.Forward```.\n\n![Forward](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/executor_forward.png)\n\n\n## Bind Multiple Outputs\n\nTo group symbols, then bind them to get outputs of both, use ```mx.symbol.Group```.\n\n![MultiOut](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/executor_multi_out.png)\n\nRemember: Bind only what you need, so that the system can perform more optimizations.\n\n\n## Calculate the Gradient\n\nIn the bind function, you can specify NDArrays that will hold gradients. Calling ```Executor.backward``` after ```Executor.forward``` gives you the corresponding gradients.\n\n![Gradient](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/executor_backward.png)\n\n\n## Simple Bind Interface for Neural Networks\n\nIt can be tedious to pass the argument NDArrays to the bind function, especially when you are binding a big\ngraph. ```Symbol.simple_bind``` provides a way to simplify\nthe procedure. You need to specify only input data shapes. The function allocates the arguments, and binds\nthe Executor for you.\n\n![SimpleBind](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/executor_simple_bind.png)\n\n## Auxiliary States\n\nAuxiliary states are just like arguments, except that you can't take the gradient of them. Although auxiliary states might not be part of the computation, they can be helpful for tracking. You can pass auxiliary states in the same way that you pass arguments.\n\n![SimpleBind](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/symbol/executor_aux_state.png)\n\n## Next Steps\n\nSee [Symbolic API](symbol) and [Python Documentation]({{'/api/python'|relative_url}}).\n"
  },
  {
    "path": "docs/static_site/src/pages/api/scala/index.md",
    "content": "---\nlayout: page_api\ntitle: Scala Guide\naction: Get Started\naction_url: /get_started\npermalink: /api/scala\ntag: scala\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# MXNet - Scala API\n\nMXNet supports the Scala programming language. The MXNet Scala package brings flexible and efficient GPU\ncomputing and state-of-art deep learning to Scala. It enables you to write seamless tensor/matrix computation with multiple GPUs in Scala. It also lets you construct and customize the state-of-art deep learning models in Scala, and apply them to tasks, such as image classification and data science challenges.\n\n\n\n## Image Classification with the Scala Infer API\nThe Infer API can be used for single and batch image classification. More information can be found at the following locations:\n\n## Tensor and Matrix Computations\nYou can perform tensor or matrix computation in pure Scala:\n\n```scala\n   import org.apache.mxnet._\n\n   val arr = NDArray.ones(2, 3)\n   // arr: org.apache.mxnet.NDArray = org.apache.mxnet.NDArray@f5e74790\n\n   arr.shape\n   // org.apache.mxnet.Shape = (2,3)\n\n   (arr * 2).toArray\n   // Array[Float] = Array(2.0, 2.0, 2.0, 2.0, 2.0, 2.0)\n\n   (arr * 2).shape\n   // org.apache.mxnet.Shape = (2,3)\n```\n\n## Related Resources\n\n* [Neural Style in Scala on MXNet](https://github.com/apache/mxnet/blob/master/scala-package/examples/src/main/scala/org/apache/mxnetexamples/neuralstyle/NeuralStyle.scala)\n* [More Scala Examples](https://github.com/apache/mxnet/tree/master/scala-package/examples/src/main/scala/org/apache/mxnetexamples)\n"
  },
  {
    "path": "docs/static_site/src/pages/community/clang_format_guide.md",
    "content": "---\nlayout: page\ntitle: Clang format\nsubtitle: Clang format in MXNet codebase for reviewers and contributors.\naction: Contribute\naction_url: /community/index\npermalink: /community/clang_format_guide\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nClang-format Guide and Tips\n===================\n\nThis wiki page describes how to set up clang-format tool as a part of your worklow. Running the command given in the description will fix clang-format problem.\n\n\n- Add `tools/lint/git-clang-format-13 ` to your `$PATH`. Once its added to your `$PATH`, running `git clang-format` will invoke it.\n```bash\ngit clang-format\n```\n\n\n- To reformat chosen file just do: \n```bash\n# `_FILE_NAME_` is the name of a file to be formatted.\n# i - apply edits to files instead of displaying a diff\nclang-format -i _FILE_NAME_\n```\n\n- To reformat all the lines in the latest git commit, just do: \n```bash\ngit diff -U0 --no-color HEAD^ | clang-format-diff.py -i -p1\n\n```\n\n- If you want to apply clang-format only to the changed lines in each commit do the following:\n```bash\n# If it's a child of origin/master, the following command-line could be used:\n# If you want to run this command on another brnach, then origin/master needs to be replaced.\nexport COMMIT_SHA=$(git rev-list --ancestry-path origin/master..HEAD | tail -n 1)\n\ngit filter-branch --tree-filter 'git-clang-format $COMMIT_SHA^' -- $COMMIT_SHA..HEAD\n```"
  },
  {
    "path": "docs/static_site/src/pages/community/code_guide.md",
    "content": "---\nlayout: page\ntitle: Code Guide and Tips\nsubtitle: Tips in MXNet codebase for reviewers and contributors.\naction: Contribute\naction_url: /community/index\npermalink: /community/code_guide\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nCode Guide and Tips\n===================\n\nThis is a document used to record tips in MXNet codebase for reviewers and\ncontributors. Most of them are summarized through lessons during the\ncontributing and process.\n\nC++ Code Styles\n---------------\n\n-   Use the [clang-format]({% link pages/community/clang_format_guide.md %}) to reformat your code.\n-   The public facing functions are documented in [doxygen](https://www.doxygen.nl/manual/docblocks.html) format.\n-   Favor concrete type declaration over `auto` as long as it is short.\n-   Favor passing by const reference (e.g. `const Expr&`) over passing\n    by value. Except when the function consumes the value by copy\n    constructor or move, pass by value is better than pass by const\n    reference in such cases.\n-   Favor `const` member function when possible.\n-   Use [RAII](https://en.cppreference.com/w/cpp/language/raii) to manage resources, including smart pointers like shared_ptr and unique_ptr as well as allocating in constructors and deallocating in destructors. Avoid explicit calls to new and delete when possible. Use make_shared and make_unique instead.\n\nWe use [`cpplint`](https://github.com/cpplint/cpplint) to enforce the code style. Because\ndifferent version of `cpplint` might change by its version, it is\nrecommended to use the same version of the `cpplint` as the master.\nYou can also use the following command via docker.\n\n```bash\nci/build.py -R --docker-registry mxnetci --platform ubuntu_cpu --docker-build-retries 3 --shm-size 500m /work/runtime_functions.sh sanity_cpp\n```\n\n`cpplint` is also not perfect, when necessary, you can use disable\n`cpplint` on certain code regions.\n\nPython Code Styles\n------------------\n\n-   The functions and classes are documented in\n    [numpydoc](https://numpydoc.readthedocs.io/en/latest/) format.\n-   Check your code style using `make pylint`\n-   Stick to language features as in `python 3.6` and above.\n\nTesting\n-------\n\nOur tests are maintained in the [/tests](https://github.com/apache/incubator-mxnet/tree/master/tests) folder. We use the following testing tools:\n-   For Python, we use [pytest](https://pytest.org).\n    -   An example of setting up and running tests (tested on MacOS with Python 3.6):\n        -   follow the [build from source](https://mxnet.apache.org/get_started/build_from_source) guide to build MXNet\n        -   install python libraries\n            ```\n            python3 -m pip install opencv-python\n            python3 -m pip install -r ci/docker/install/requirements\n            ```\n        -   install MXNet Python bindings:\n            ```\n            python3 -m pip install -e ./python\n            ```\n        -   run tests in a specific module\n            ```\n            python3 -m pytest tests/python/unittest/test_smoke.py\n            ```\n        -   or run a specific test in a module\n            ```\n            python3 -m pytest tests/python/unittest/test_smoke.py::test_18927\n            ```\n        -   or run all the Python unittests\n            ```\n            python3 -m pytest tests/python/unittest/\n            ```\n-   For C++, we use [gtest](https://github.com/google/googletest).\n\nOur CI pipelines check for a wide variety of configuration on all platforms. To locate and reproduce\na test issue in PR, you can refer to the process described in [#18723](https://github.com/apache/incubator-mxnet/issues/18723)\n\n<script async defer src=\"https://buttons.github.io/buttons.js\"></script>\n<script src=\"https://apis.google.com/js/platform.js\"></script>\n"
  },
  {
    "path": "docs/static_site/src/pages/community/code_review.md",
    "content": "---\nlayout: page\ntitle: Perform Code Reviews\nsubtitle: General guideline for code reviewers.\naction: Contribute\naction_url: /community/index\npermalink: /community/code_review\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nPerform Code Reviews\n====================\n\nThis is a general guideline for code reviewers. First of all, while it\nis great to add new features to a project, we must also be aware that\neach line of code we introduce also brings **technical debt** that we\nmay have to eventually pay.\n\nOpen source code is maintained by a community with diverse background,\nand hence it is even more important to provide clear, documented and\nmaintainable code. Code reviews are a shepherding process to spot\npotential problems, improve quality of the code. We should, however, not\nrely on the code review process to get the code into a ready state.\nContributors are encouraged to polish the code to a ready state before\nrequesting reviews. This is especially expected for code owner and\ncommitter candidates.\n\nHere are some checklists for code reviews, it is also helpful reference\nfor contributors.\n\nHold the Highest Standard\n-------------------------\n\nThe first rule for code reviewers is to always keep the highest\nstandard, and do not approve code just to \"be friendly\". Good,\ninformative critics each other learn and prevent technical debt in early\nstages.\n\nDeliberate on API and Data Structures\n-------------------------------------\n\nA minimum and stable API is critical to the project's life. A good API\nmakes a huge difference. Always think very carefully about all the\naspects including naming, argument definitions and behavior.\n\nWhen possible, pay more attention still to the proposed API design\nduring code reviews. Remember, it is easier to improve code\nimplementation, but it is extremely hard to change an API once accepted.\nWe should treat data structures that are shared across modules(e.g. AST)\nin the same way. If/when uncertain, start a conversation with more\ndevelopers before committing.\n\nHere are some useful principles for designing APIs:\n\n-   Be consistent with existing well-known package's APIs if the\n    features overlap. For example, tensor operation APIs should always\n    be consistent with the numpy API.\n-   Be consistent with existing APIs in the same project. For example,\n    we should use the same argument ordering across all the optimization\n    passes, so there is no \"surprise\" when using them.\n-   Think about whether the API will change in the future. For example,\n    we will have more options like loop_unrolling and device placement\n    policy as we add more optimizations in build. We can package\n    optimization knobs into a build configuration object. In this way,\n    the build API is stable over time, even though it may be enriched.\n-   Write documentation. Documentation is mandatory for APIs and\n    sometimes writing documents helps us to think further about the\n    design as well as whether we need to add further clarifications.\n-   Minimum. Think about how many lines of code a user has to write to\n    use the API. Remove layers of abstraction when possible.\n\nEnsure Test Coverage\n--------------------\n\nEach new change of features should introduce test cases. Bug fixes\nshould include regression tests that prevent the problem from happening\nagain.\n\nDocumentation is Mandatory\n--------------------------\n\nDocumentation is often overlooked. When adding new functions or changing\nan existing function, the documentation should be directly updated. A\nnew feature is meaningless without documentation to make it accessible.\nSee more at [Write Document and Tutorials]({% link pages/community/document.md %}).\n\nMinimum Dependency\n------------------\n\nAlways be cautious in introducing dependencies. While it is important to\nreuse code and avoid reinventing the wheel, dependencies can increase\nburden of users in deployment. A good design principle is that a feature\nor function should only have a dependency if/when a user actually use it.\n\nEnsure Readability\n------------------\n\nWhile it is hard to implement a new feature, it is even harder to make\nothers understand and maintain the code you wrote. It is common for a\nPMC or committer to not be able to understand certain contributions. In\nsuch case, a reviewer should say \"I don't understand\" and ask the\ncontributor to clarify. We highly encourage code comments which explain\nthe code logic along with the code.\n\nConcise Implementation\n----------------------\n\nSome basic principles applied here: favor vectorized array code over\nloops, use existing APIs that solve the problem.\n\nDocument Lessons in Code Reviews\n--------------------------------\n\nWhen you find there are some common or recurring lessons that can be\nsummarized, add it to the [Code Guide and Tips]({% link pages/community/code_guide.md %}).\nIt is always good to refer to the guideline document when requesting\nchanges, so the lessons can be shared to all the community.\n\nRespect each other\n------------------\n\nThe code reviewers and contributors are paying the most precious\ncurrency in the world \\-\\- time. We are volunteers in the community to\nspend the time to build good code, help each other, learn and have fun\nhacking.\n\nLearn from other Code Reviews\n-----------------------------\n\nThere can be multiple reviewers reviewing the same changes. Many times\nother reviewers may spot things you did not find. Try to learn from\nother code reviews, when possible, document these lessons.\n\nApprove and Request Changes Explicitly\n--------------------------------------\n\nThe contributor and code owner can request code reviews from multiple\nreviewers. Remember to approve changes when your comments are addressed\nin a code review. To do so \\-\\- please click on changes tab in the pull\nrequest, then select approve, or comment on the code and click request\nchanges. Code owner can decide if the code can be merged in case by case\nif some of the reviewers did not respond in time(e.g. a week) and\nexisting reviews are sufficient.\n\nGet Started\n-----------\n\nCheckout the following [PRs that need review](https://github.com/apache/mxnet/labels/pr-awaiting-review)\"\n\n<script async defer src=\"https://buttons.github.io/buttons.js\"></script>\n<script src=\"https://apis.google.com/js/platform.js\"></script>\n"
  },
  {
    "path": "docs/static_site/src/pages/community/committer_guide.md",
    "content": "---\nlayout: page\ntitle: Committer Guide\nsubtitle: Tips for committers.\naction: Contribute\naction_url: /community/index\npermalink: /community/committer_guide\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nCommitter Guide\n===============\n\nThis is an evolving document to provide some helpful tips for\ncommitters. Most of them are lessons learned during development. We\nwelcome every committer to contribute to this document. See the\n[MXNet Community Guideline]({% link pages/community/community.md %})\nfor an overview of the committership and the general development process.\n\nCommunity First\n---------------\n\nThe collective effort of the community moves the project forward and\nmakes the project awesome for everyone. When we make a decision, it is\nalways helpful to keep the community in mind. Here are some example\nquestions that we can ask:\n\n-   How can I encourage new contributors to get more involved in the\n    project?\n-   Can I help to save my fellow committers\\' time?\n-   Have I enabled the rest of the community to participate the design\n    proposals?\n\nPublic Archive Principle\n------------------------\n\nWhile private channels such as face to face discussion are useful for\ndevelopment, they also create barriers for the broader community\\'s\nparticipation. The Apache way of development requires all decisions to\nbe made in public channels, which are archived and accessible to\neveryone. As a result, any contributor can keep up with the development\nby watching the archives and join the development anytime.\n\nWhile this principle applies to every contributor, it is especially\nimportant for committers. Here are some example applications of this\nprinciple:\n\n-   When getting a project-related question from a personal channel,\n    encourage the person to open a public thread in the discuss forum,\n    so others in the community can benefit from the answer.\n-   After an in-person discussion, send a summary to public channels (as\n    an RFC or a discuss thread).\n\nShepherd a Pull Request\n-----------------------\n\nHere are some tips to shepherd a pull request. You can also take a look\nat the [Perform Code Reviews]({% link pages/community/code_review.md %}).\n\n-   Assign the PR to yourself, so that other committers know that the PR\n    has already been tended to.\n-   Make use of the status label to indicate the current status.\n-   Check if an RFC needs to be sent.\n-   If the contributor has not requested a reviewer, kindly ask the\n    contributor to do so. If the PR comes from a new contributor, help\n    the contributor to request reviewers and ask the contributor to do\n    so next time.\n-   Moderate the reviews, ask reviewers to approve explicitly.\n-   Mark the PR as accepted and acknowledge the contributor/reviewers.\n-   Merge the PR :)\n\nTime Management\n---------------\n\nThere are many things that a committer can do, such as moderating\ndiscussions, pull request reviews and code contributions.\n\nWorking on an open source project can be rewarding, but also be a bit\noverwhelming sometimes. A little bit of time management might be helpful\nto alleviate the problem. For example, some committers have a\n\\\"community day\\\" in a week when they actively manage outstanding PRs,\nbut watch the community less frequently in the rest of the time.\n\nRemember that your merit will never go away, so please take your time\nand pace when contributing to the project :)\n\nBroad Collaboration\n-------------------\n\nSometimes, we tend to only interact with people we know. However, broad\ncollaborations are necessary to the success of the project. Try to keep\nthat in mind, shepherd PRs for, and request code reviews from community\nmembers who you do not interact physically.\n\n<script async defer src=\"https://buttons.github.io/buttons.js\"></script>\n<script src=\"https://apis.google.com/js/platform.js\"></script>\n"
  },
  {
    "path": "docs/static_site/src/pages/community/community.md",
    "content": "---\nlayout: page\ntitle: MXNet Community Guideline\nsubtitle: How MXNet community operates\naction: Contribute\naction_url: /community/index\npermalink: /community/community\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nMXNet Community Guideline\n=========================\n\nApache MXNet follows the [Apache Way](https://www.apache.org/theapacheway/) governs by merit.\nWe believe that\nit is important to create an inclusive community where everyone can use,\ncontribute to, and influence the direction of the project. See\n[CONTRIBUTORS.md](https://github.com/apache/incubator-mxnet/blob/master/CONTRIBUTORS.md)\nfor the current list of contributors.\n\nGeneral Development Process\n---------------------------\n\nEveryone in the community is welcome to send patches, documents, and\npropose new directions to the project. The key guideline here is to\nenable everyone in the community to get involved and participate the\ndecision and development. When major changes are proposed, an RFC should\nbe sent to allow discussion by the community. We encourage public\ndiscussion, archivable channels such as issues, discuss forum and\nmailing-list, so that everyone in the community can participate and\nreview the process later.\n\nCode reviews are one of the key ways to ensure the quality of the code.\nHigh-quality code reviews prevent technical debt for long-term and are\ncrucial to the success of the project. A pull request needs to be\nreviewed before it gets merged. A committer who has the expertise of the\ncorresponding area would moderate the pull request and the merge the\ncode when it is ready. The corresponding committer could request\nmultiple reviewers who are familiar with the area of the code. We\nencourage contributors to request code reviews themselves and help\nreview each other's code \\-\\- remember everyone is volunteering their\ntime to the community, high-quality code review itself costs as much as\nthe actual code contribution, you could get your code quickly reviewed\nif you do others the same favor.\n\nThe community should strive to reach a consensus on technical decisions\nthrough discussion. We expect committers and PMCs to moderate technical\ndiscussions in a diplomatic way, and provide suggestions with clear\ntechnical reasoning when necessary.\n\nCommitters\n----------\n\nCommitters are individuals who are granted the write access to the\nproject. A committer is usually responsible for a certain area or\nseveral areas of the code where they oversee the code review process.\nThe area of contribution can take all forms, including code\ncontributions and code reviews, documents, education, and outreach.\nCommitters are essential for a high quality and healthy project. The\ncommunity actively look for new committers from contributors. Here is a\nlist of useful traits that help the community to recognize potential\ncommitters:\n\n-   Sustained contribution to the project, demonstrated by discussion\n    over RFCs, code reviews and proposals of new features, and other\n    development activities. Being familiar with, and being able to take\n    ownership on one or several areas of the project.\n-   Quality of contributions: High-quality, readable code contributions\n    indicated by pull requests that can be merged without a substantial\n    code review. History of creating clean, maintainable code and\n    including good test cases. Informative code reviews to help other\n    contributors that adhere to a good standard.\n-   Community involvement: active participation in the discussion forum,\n    promote the projects via tutorials, talks and outreach. We encourage\n    committers to collaborate broadly, e.g. do code reviews and discuss\n    designs with community members that they do not interact physically.\n\nThe Project Management Committee(PMC) consists group of active\ncommitters that moderate the discussion, manage the project release, and\nproposes new committer/PMC members. Potential candidates are usually\nproposed via an internal discussion among PMCs, followed by a consensus\napproval, i.e. least 3 +1 votes, and no vetoes. Any veto must be\naccompanied by reasoning. PMCs should serve the community by upholding\nthe community practices and guidelines and make MXNet a better community for\neveryone. PMCs should strive to only nominate new candidates outside of\ntheir own organization.\n\nReviewers\n---------\n\nReviewers are individuals who actively contributed to the project and\nare willing to participate in the code review of new contributions. We\nidentify reviewers from active contributors. The committers should\nexplicitly solicit reviews from reviewers. High-quality code reviews\nprevent technical debt for long-term and are crucial to the success of\nthe project. A pull request to the project has to be reviewed by at\nleast one reviewer in order to be merged.\n\n<script async defer src=\"https://buttons.github.io/buttons.js\"></script>\n<script src=\"https://apis.google.com/js/platform.js\"></script>\n"
  },
  {
    "path": "docs/static_site/src/pages/community/document.md",
    "content": "---\nlayout: page\ntitle: Write Document and Tutorials\nsubtitle: Guidelines on documentation to help the community.\naction: Contribute\naction_url: /community/index\npermalink: /community/document\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nWrite Document and Tutorials\n============================\n\nWe use the [Sphinx](http://sphinx-doc.org) for the main documentation.\nSphinx support both the reStructuredText and markdown. When possible, we\nencourage to use reStructuredText as it has richer features. Note that\nthe python doc-string and tutorials allow you to embed reStructuredText\nsyntax.\n\nDocument Python\n---------------\n\nWe use [numpydoc](https://numpydoc.readthedocs.io/en/latest/) format to\ndocument the function and classes. The following snippet gives an\nexample docstring. We always document all the public functions, when\nnecessary, provide an usage example of the features we support(as shown\nbelow).\n\n```python\ndef myfunction(arg1, arg2, arg3=3):\n    \"\"\"Briefly describe my function.\n\n    Parameters\n    ----------\n    arg1 : Type1\n        Description of arg1\n\n    arg2 : Type2\n        Description of arg2\n\n    arg3 : Type3, optional\n        Description of arg3\n\n    Returns\n    -------\n    rv1 : RType1\n        Description of return type one\n\n    Examples\n    --------\n    .. code:: python\n\n        # Example usage of myfunction\n        x = myfunction(1, 2)\n    \"\"\"\n    return rv1\n```\n\nBe careful to leave blank lines between sections of your documents. In\nthe above case, there has to be a blank line before\n[Parameters]{.title-ref}, [Returns]{.title-ref} and\n[Examples]{.title-ref} in order for the doc to be built correctly. To\nadd a new function to the doc, we need to add the\n[sphinx.autodoc](http://www.sphinx-doc.org/en/master/ext/autodoc.html)\nrules to the\n[/docs/python_docs/python](https://github.com/apache/incubator-mxnet/tree/master/docs/python_docs/python)).\nYou can refer to the existing files under this folder on how to add the\nfunctions.\n\nDocument C++\n------------\n\nWe use the doxgen format to document c++ functions. The following\nsnippet shows an example of c++ docstring.\n\n```cpp\n/*!\n * \\brief Description of my function\n * \\param arg1 Description of arg1\n * \\param arg2 Description of arg2\n * \\returns describe return value\n */\nint myfunction(int arg1, int arg2) {\n  // When necessary, also add comment to clarify internal logic\n}\n```\n\nBesides documenting function usages, we also highly recommend\ncontributors to add comments about code logic to improve readability.\n\nWrite Tutorials\n---------------\n\nWe use the [notedown](https://github.com/aaren/notedown) to write Jupyter notebooks\nin Markdown as Python tutorials. You can find the source code under\n[/docs/python_docs/python/tutorials](https://github.com/apache/incubator-mxnet/tree/master/docs/python_docs/python/tutorials).\n\n\nThe tutorial code will run on our build server to generate the document\npage and the tutorial page will show the result of executing the Jupyter notebook.\n\n\nApplication Examples\n--------------------\n\nOur deep learning examples are maintained in [apache/incubator-mxnet-examples](http://github.com/apache/incubator-mxnet-examples)\nand are checked regularly by CI to ensure quality.\n\n<script async defer src=\"https://buttons.github.io/buttons.js\"></script>\n<script src=\"https://apis.google.com/js/platform.js\"></script>\n"
  },
  {
    "path": "docs/static_site/src/pages/community/error_handling.md",
    "content": "---\nlayout: page\ntitle: Error Handling Guide\nsubtitle: Utilize structured error types in MXNet for modern cross-language error handling.\naction: Contribute\naction_url: /community/index\npermalink: /community/error_handling\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nError Handling Guide\n====================\n\nMXNet contains structured error classes to indicate specific types of\nerror. Please raise a specific error type when possible, so that users\ncan write code to handle a specific error category if necessary. You can\ndirectly raise the specific error object in python. In other languages\nlike c++, you simply add `<ErrorType>:` prefix to the error message(see\nbelow).\n\n{% include note.html content=\"Please refer to [/python/mxnet/error.py](https://github.com/apache/incubator-mxnet/blob/master/python/mxnet/error.py) for the list of errors.\" %}\n\n\nRaise a Specific Error in C++\n-----------------------------\n\nYou can add `<ErrorType>:` prefix to your error message to raise an\nerror of the corresponding type. Note that you do not have to add a new\ntype `mxnet.base.MXNetError` will be\nraised by default when there is no error type prefix in the message.\nThis mechanism works for both `LOG(FATAL)` and `CHECK` macros. The\nfollowing code gives an example on how to do so.\n\n```cpp\n// Python frontend receives the following error type:\n// ValueError: Check failed: x == y (0 vs. 1) : expect x and y to be equal.\nCHECK_EQ(0, 1) << \"ValueError: expect x and y to be equal.\"\n\n\n// Python frontend receives the following error type:\n// InternalError: cannot reach here\nLOG(FATAL) << \"InternalError: cannot reach here\";\n```\n\nAs you can see in the above example, MXNet's ffi system combines both the\npython and C++'s stacktrace into a single message, and generate the\ncorresponding error class automatically.\n\nHow to choose an Error Type\n---------------------------\n\nYou can go through the error types are listed below, try to use common\nsense and also refer to the choices in the existing code. We try to keep\na reasonable amount of error types. If you feel there is a need to add a\nnew error type, do the following steps:\n\n-   Send a RFC proposal with a description and usage examples in the\n    current codebase.\n-   Add the new error type to `mxnet.error` with clear documents.\n-   Update the list in this file to include the new error type.\n-   Change the code to use the new error type.\n\nWe also recommend to use less abstraction when creating the short error\nmessages. The code is more readable in this way, and also opens path to\ncraft specific error messages when necessary.\n\n```python\ndef preferred():\n    # Very clear about what is being raised and what is the error message.\n    raise OpNotImplemented(\"Operator relu is not implemented in the MXNet frontend\")\n\ndef _op_not_implemented(op_name):\n    return OpNotImplemented(\"Operator {} is not implemented.\").format(op_name)\n\ndef not_preferred():\n    # Introduces another level of indirection.\n    raise _op_not_implemented(\"relu\")\n```\n\nIf we need to introduce a wrapper function that constructs multi-line\nerror messages, please put wrapper in the same file so other developers\ncan look up the implementation easily.\n\nSignal Handling\n---------------\n\nWhen not careful, some errors can occur in the form of a [signal](https://en.wikipedia.org/wiki/Signal_(IPC)),\nwhich is handled by the OS kernel. In MXNet, you can choose to handle certain signals in the form of\na catchable exception. This can be combined with the error type selection above so that it can be\ncaught in the Python frontend. Currently, the following signals are handled this way:\n\n-   `SIGFPE`: throws `FloatingPointError`\n-   `SIGBUS`: throws `IOError`\n\nTo extend this to other signals, you can modify the signal handler registration in\n[/src/initialize.cc](https://github.com/apache/incubator-mxnet/blob/72eff9b66ecc683c3e7f9ad2c0ba69efa8dd423b/src/initialize.cc#L347-L376).\n\n<script async defer src=\"https://buttons.github.io/buttons.js\"></script>\n<script src=\"https://apis.google.com/js/platform.js\"></script>\n"
  },
  {
    "path": "docs/static_site/src/pages/community/git_howto.md",
    "content": "---\nlayout: page\ntitle: Git Usage Tips\nsubtitle: Git 101\naction: Contribute\naction_url: /community/index\npermalink: /community/git_howto\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nGit Usage Tips\n==============\n\nHere are some tips for git workflow.\n\n\\#\\# How to resolve conflict with master\n\n-   First rebase to most recent master\n\n```bash\n# The first two steps can be skipped after you do it once.\ngit remote add upstream git@github.com:apache/incubator-mxnet.git\ngit fetch upstream\ngit rebase upstream/master\n```\n\n-   The git may show some conflicts it cannot merge, say\n    `conflicted.py`.\n    -   Manually modify the file to resolve the conflict.\n    -   After you resolved the conflict, mark it as resolved by\n\n```bash\ngit add conflicted.py\n```\n\n-   Then you can continue rebase by\n\n```bash\ngit rebase --continue\n```\n\n-   Finally push to your fork, you may need to force push here.\n\n```bash\ngit push --force\n```\n\nHow to manage branches\n----------------------\n\nWe recommend to always reserve the master branch for synchronizing with upstream.\nFor development of new features, create a new branch for each feature:\n\n```bash\ngit checkout -b fancy_new_feature\n```\n\nThe benefit of this practices is that you can easily rebase onto the latest master\nchanges with little effort\n\n```bash\ngit pull upstream master --rebase\n```\n\nHow to combine multiple commits into one\n----------------------------------------\n\nSometimes we want to combine multiple commits, especially when later\ncommits are only fixes to previous ones, to create a PR with set of\nmeaningful commits. You can do it by following steps.\n\n-   Before doing so, configure the default editor of git if you haven\\'t\n    done so before.\n\n```bash\ngit config core.editor [the-editor-you-like]\n```\n\n-   Assume we want to merge last 3 commits, type the following commands\n\n```bash\ngit rebase -i HEAD~3\n```\n\n-   It will pop up an text editor. Set the first commit as\n    [pick]{.title-ref}, and change later ones to [squash]{.title-ref}.\n-   After you saved the file, it will pop up another text editor to ask\n    you modify the combined commit message.\n-   Push the changes to your fork, you need to force push.\n\n```bash\ngit push --force\n```\n\nReset to the most recent master\n-------------------------------\n\nYou can always use git reset to reset your version to the most recent\nmaster. Note that all your **\\*local changes will get lost**\\*. So only\ndo it when you do not have local changes or when your pull request just\nget merged.\n\n```bash\ngit reset --hard [hash tag of master]\n```\n\nRecover a Previous Commit after Reset\n-------------------------------------\n\nSometimes we could mistakenly reset a branch to a wrong commit. When\nthat happens, you can use the following command to show the list of\nrecent commits\n\n```bash\ngit reflog\n```\n\nOnce you get the right hashtag, you can use git reset again to change\nthe head to the right commit.\n\nApply only k-Latest Commits on to the master\n--------------------------------------------\n\nSometimes it is useful to only apply your k-latest changes on top of the\nmaster. This usually happens when you have other m-commits that are\nalready merged before these k-commits. Directly rebase against the\nmaster might cause merge conflicts on these first m-commits(which are\ncan be safely discarded).\n\nYou can instead use the following command\n\n```bash\n# k is the concrete number\n# Put HEAD~2 for the last 1 commit.\ngit rebase --onto upstream/master HEAD~k\n```\n\nYou can then force push to the master. Note that the above command will\ndiscard all the commits before the last k ones.\n\nWhat is the consequence of force push\n-------------------------------------\n\nThe previous two tips requires force push, this is because we altered\nthe path of the commits. It is fine to force push to your own fork, as\nlong as the commits changed are only yours.\n\n<script async defer src=\"https://buttons.github.io/buttons.js\"></script>\n<script src=\"https://apis.google.com/js/platform.js\"></script>\n"
  },
  {
    "path": "docs/static_site/src/pages/community/index.md",
    "content": "---\nlayout: page\ntitle: Contribute\nsubtitle: Contribute to the Apache MXNet project\naction: Get Started\naction_url: /get_started\npermalink: /community/index\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Contributing to MXNet\n\nApache MXNet (incubating) is a community-led open-source deep learning project. We welcome new members and look forward to your contributions. Here you will find how to stay connected with the MXNet community, get started to contribute, and best practices and processes in MXNet.\n\n## Stay Connected\n\nIn MXNet, we have the following communication channels.\n\n| Channel | Purpose |\n|---|---|\n| [Follow MXNet Development on Github](#github-issues) | See what's going on in the MXNet project. |\n| [Check out the MXNet Confluence Wiki](https://cwiki.apache.org/confluence/display/MXNET/Apache+MXNet+Home) <i class=\"fas fa-external-link-alt\"> | MXNet developer wiki for information related to project development, maintained by contributors and developers. To request write access, send an email to [send request to the dev list](mailto:dev@mxnet.apache.org?subject=Requesting%20CWiki%20write%20access) <i class=\"far fa-envelope\"></i>. |\n| [dev@mxnet.apache.org mailing list](https://lists.apache.org/list.html?dev@mxnet.apache.org) | The \"dev list\". Discussions about the development of MXNet. To subscribe, send an email to [dev-subscribe@mxnet.apache.org](mailto:dev-subscribe@mxnet.apache.org) <i class=\"far fa-envelope\"></i>. |\n| [discuss.mxnet.io](https://discuss.mxnet.io) <i class=\"fas fa-external-link-alt\"></i> | Asking & answering MXNet usage questions. |\n| [Apache Slack #mxnet Channel](https://the-asf.slack.com/archives/C7FN4FCP9) <i class=\"fas fa-external-link-alt\"> | Connect with MXNet and other Apache developers. To join the MXNet slack channel [send request to the dev list](mailto:dev@mxnet.apache.org?subject=Requesting%20slack%20access) <i class=\"far fa-envelope\"></i>. |\n| [Follow MXNet on Social Media](#social-media) | Get updates about new features and events. |\n\n### Social Media\n\nKeep connected with the latest MXNet news and updates.\n\n<p>\n<a href=\"https://twitter.com/apachemxnet\"><img src=\"https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/social/twitter.svg?sanitize=true\" height=\"30px\"/> Apache MXNet on Twitter</a>\n</p>\n<p>\n<a href=\"https://medium.com/apache-mxnet\"><img src=\"https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/social/medium_black.svg?sanitize=true\" height=\"30px\"/> Contributor and user blogs about MXNet</a>\n</p>\n<p>\n<a href=\"https://reddit.com/r/mxnet\"><img src=\"https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/social/reddit_blue.svg?sanitize=true\" height=\"30px\" alt=\"reddit\"/> Discuss MXNet on r/mxnet</a>\n</p>\n<p>\n<a href=\"https://www.youtube.com/apachemxnet\"><img src=\"https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/social/youtube_red.svg?sanitize=true\" height=\"30px\"/> Apache MXNet YouTube channel</a>\n</p>\n<p>\n<a href=\"https://www.linkedin.com/company/apache-mxnet\"><img src=\"https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/social/linkedin.svg?sanitize=true\" height=\"30px\"/> Apache MXNet on LinkedIn</a>\n</p>\n\n## Start Contributing\n\nWe value all forms of contributions, including, but not limited to:\n\n- Code reviewing of the existing patches.\n- Documentation and usage examples\n- Community participation in forums and issues.\n- Code readability and developer guide\n  - We welcome contributions that add code comments\n    to improve readability\n  - We also welcome contributions to docs to explain the\n    design choices of the internal.\n\n- Test cases to make the codebase more robust.\n- [Tutorials]({% link pages/community/document.md %}), [blog posts](https://medium.com/apache-mxnet), talks that promote the project.\n- [Examples](http://github.com/apache/incubator-mxnet-examples) <i class=\"fab fa-github\"></i> for deep learning applications.\n\n\n{% include note.html content=\"Looking for ideas to start contributing? Check out the [good first issues](https://github.com/apache/incubator-mxnet/labels/good%20first%20issue), and [PRs that need review](https://github.com/apache/incubator-mxnet/labels/pr-awaiting-review)\" %}\n<br/>\n\n### Contribution Guides\n\n- [MXNet Community Guideline]({% link pages/community/community.md %})\n- [Write Document and Tutorials]({% link pages/community/document.md %})\n- [Committer Guide]({% link pages/community/committer_guide.md %})\n- [Submit a Pull Request]({% link pages/community/pull_request.md %})\n- [Perform Code Reviews]({% link pages/community/code_review.md %})\n- [Code Guide and Tips]({% link pages/community/code_guide.md %})\n- [Error Handling Guide]({% link pages/community/error_handling.md %})\n- [Git Usage Tips]({% link pages/community/git_howto.md %})\n\n\n#### RFC Process\n\nAny new features of improvements that are non-trivial should follow the [RFC](https://github.com/apache/incubator-mxnet/issues?q=label%3ARFC+) <i class=\"fab fa-github\"></i> process:\n\n1. [Create an RFC issue on GitHub](https://github.com/apache/incubator-mxnet/issues/new/choose): RFC issues will notify MXNet developer community through all channels, including dev@ list and Slack.\n1. [Create the PR on GitHub](https://github.com/apache/incubator-mxnet/pulls) and mention the RFC issue in description.\n\n#### Github Issues\n\nApache MXNet uses Github issues to track feature requests and bug reports. [Open a Github issue](https://github.com/apache/incubator-mxnet/issues/new/choose) <i class=\"fas fa-external-link-alt\"></i>.\n\nWe also use Github projects for tracking larger projects, and Github milestones for tracking releases.\n\n* [Github Projects](https://github.com/apache/incubator-mxnet/projects) <i class=\"fab fa-github\"></i>\n* [Github Milestones](https://github.com/apache/incubator-mxnet/milestones) <i class=\"fab fa-github\"></i>\n* [Roadmaps](https://github.com/apache/incubator-mxnet/labels/Roadmap) <i class=\"fab fa-github\"></i>\n\n\nThe process for setting up MXNet for development depends on several factors, and is constantly being improved and expanded for more development languages. Setup information is on the MXNet Confluence Wiki.\n\n* [MXNet Confluence Wiki: Development](https://cwiki.apache.org/confluence/display/MXNET/Development) <i class=\"fas fa-external-link-alt\"></i>\n\n<br/>\n## Contributors\n\nMXNet has been developed by and is used by a group of active community members. Contribute to improving it!\n\n[Contributors and Committers](https://github.com/apache/incubator-mxnet/blob/master/CONTRIBUTORS.md) <i class=\"fab fa-github\"></i>\n\n<br/>\n\n<script defer src=\"https://use.fontawesome.com/releases/v5.0.12/js/all.js\" integrity=\"sha384-Voup2lBiiyZYkRto2XWqbzxHXwzcm4A5RfdfG6466bu5LqjwwrjXCMBQBLMWh7qR\" crossorigin=\"anonymous\"></script>\n<script async defer src=\"https://buttons.github.io/buttons.js\"></script>\n<script src=\"https://apis.google.com/js/platform.js\"></script>\n"
  },
  {
    "path": "docs/static_site/src/pages/community/pull_request.md",
    "content": "---\nlayout: page\ntitle: Submit a Pull Request\nsubtitle: What to do to submit a pull request\naction: Contribute\naction_url: /community/index\npermalink: /community/pull_request\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\nSubmit a Pull Request\n=====================\n\nThis is a quick guide to submit a pull request, please also refer to the\ndetailed guidelines.\n\n-   Before submit, please rebase your code on the most recent version of\n    master, you can do it by\n\n    ```bash\n    git remote add upstream git@github.com:apache/incubator-mxnet.git\n    git fetch upstream\n    git rebase upstream/master\n    ```\n\n-   Make sure code style check pass by typing the following command, and\n    all the existing test-cases pass. Use the [clang-format]({% link pages/community/clang_format_guide.md %}) to reformat your code.\n\n    ```bash\n    # Reproduce the lint procedure in the CI.\n    ci/build.py -R --docker-registry mxnetci --platform ubuntu_cpu --docker-build-retries 3 --shm-size 500m /work/runtime_functions.sh sanity\n    ```\n\n-   Add test-cases to cover the new features or bugfix the patch\n    introduces.\n\n-   Document the code you wrote, see more at [Write Document and Tutorials]({% link pages/community/document.md %}).\n\n-   Send the pull request and fix the problems reported by automatic\n    checks.\n\n-   Request code reviews from other contributors and improves your patch\n    according to feedbacks.\n\n    -   To get your code reviewed quickly, we encourage you to help\n        review others\\' code so they can do the favor in return.\n    -   Code review is a shepherding process that helps to improve\n        contributor\\'s code quality. We should treat it proactively, to\n        improve the code as much as possible before the review. We\n        highly value patches that can get in without extensive reviews.\n    -   The detailed guidelines and summarizes useful lessons.\n\n-   The patch can be merged after the reviewers approve the pull\n    request.\n\nCI Environment\n--------------\n\nWe use docker containers to create stable CI environments that can be\ndeployed to multiple machines. Because we want a relatively stable CI\nenvironment and make use of pre-cached image, all of the CI images are\nbuilt and maintained by committers.\n\nUpgrade of CI base images are done automatically from the MXNet master branch\nCI builds, tracked in [restricted-docker-cache-refresh](https://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/restricted-docker-cache-refresh/activity).\nSometimes this can be broken and needs fixes to accommodate\nthe new env. When this happens, send a PR to fix the build script in the repo.\n\nTesting\n-------\n\nEven though we have hooks to run unit tests automatically for each pull\nrequest, It\\'s always recommended to run unit tests locally beforehand\nto reduce reviewers\\' burden and speedup review process.\n\n### C++\n\nC++ tests are maintained in [/tests/cpp](https://github.com/apache/incubator-mxnet/tree/master/tests/cpp) and requires [gtest](https://github.com/google/googletest) to build and run. Once you complete building the MXNet binary, tests are automatically built and generated in `/build/tests/mxnet_unit_tests`.\n\n### Python\n\nThe dependencies for testing pipelines can be found in [/ci/docker/install/requirements](https://github.com/apache/incubator-mxnet/blob/master/ci/docker/install/requirements). To install these dependencies:\n\n```bash\npip install --user -r ci/docker/install/requirements\n```\n\n<script defer src=\"https://use.fontawesome.com/releases/v5.0.12/js/all.js\" integrity=\"sha384-Voup2lBiiyZYkRto2XWqbzxHXwzcm4A5RfdfG6466bu5LqjwwrjXCMBQBLMWh7qR\" crossorigin=\"anonymous\"></script>\n<script async defer src=\"https://buttons.github.io/buttons.js\"></script>\n<script src=\"https://apis.google.com/js/platform.js\"></script>\n"
  },
  {
    "path": "docs/static_site/src/pages/ecosystem.html",
    "content": "---\nlayout: page\ntitle: Ecosystem\nsubtitle: Explore a rich ecosystem of libraries, tools, and more to support research and development of Deep Learning application across many fields and domains of application.\naction: Get Started\naction_url: /get_started\npermalink: /ecosystem/\n\necosystem_toolkits:\n- title: GluonCV\n  text: GluonCV is a computer vision toolkit with rich model zoo. From object detection to pose estimation.\n  icon: /assets/img/visual.svg\n  link: https://gluon-cv.mxnet.io\n- title: GluonNLP\n  text: GluonNLP provides state-of-the-art deep learning models in NLP. For engineers and researchers to fast prototype research ideas and products.\n  icon: /assets/img/artificial-intelligence.svg\n  link: https://gluon-nlp.mxnet.io/\n- title: GluonTS\n  text: Gluon Time Series (GluonTS) is the Gluon toolkit for probabilistic time series modeling, focusing on deep learning-based models.\n  icon: /assets/img/line-graph.svg\n  link: https://gluon-ts.mxnet.io/\n- title: AutoGluon\n  text: AutoGluon enables easy-to-use and easy-to-extend AutoML with a focus on deep learning and real-world applications spanning image, text, or tabular data.\n  icon: /assets/img/autogluon.png\n  link: https://autogluon.mxnet.io\n\necosystem_other:\n- title: Flower\n  text: Flower is an agnostic federated learning framework. Federate any workload, any machine learning framework, and any programming language.\n  icon: /assets/img/flower_icon.png\n  link: https://flower.dev/\n- title: InsightFace\n  text: State-of-the-art face detection and face recognition repository, including ArcFace loss and RetinaFace implementation\n  link: https://github.com/deepinsight/insightface\n- title: Kubeflow\n  text: Kubeflow training operator provides Kubernetes custom resources that makes it easy to run distributed or non-distributed model training jobs on Kubernetes for various frameworks, including Apache MXNet.\n  icon: /assets/img/kubeflow.png\n  link: https://github.com/kubeflow/training-operator\n- title: Sockeye\n  text: Sockeye is a sequence-to-sequence framework for Neural Machine Translation based on Apache MXNet. It implements state-of-the-art encoder-decoder architectures.\n  link: https://awslabs.github.io/sockeye/\n- title: Deep Graph Library\n  text: DGL is a Python package dedicated to deep learning on graphs supporting MXNet as a backend.\n  link: https://www.dgl.ai/\n- title: TensorLy\n  text: TensorLy is a high level API for tensor methods and deep tensorized neural networks in Python that aims to make tensor learning simple.\n  icon: /assets/img/tensorly_logo.png\n  link: http://tensorly.org/stable/home.html\n- title: Apache TVM\n  text: Apache TVM is an open deep learning compiler stack for CPUs, GPUs, and specialized accelerators. It supports a number of framework including Apache MXNet.\n  link: https://tvm.ai/about\n  icon: /assets/img/tvm.png\n- title: GluonFR\n  text: Community-driven toolkit for Face Recognition and Face Detection\n  link: https://gluon-face.readthedocs.io/en/latest/\n- title: Optuna\n  text: Optuna is a hyperparameter optimization framework that automates the search for good hyperparameters using Python conditionals, loops, and syntax.\n  link: https://optuna.org/\n  icon: /assets/img/optuna.png\n- title: Ray Tune\n  text: Tune is a Python library for experiment execution and hyperparameter tuning at any scale.\n  link: https://docs.ray.io/en/latest/tune.html\n  icon: /assets/img/tune.png\n- title: Coach RL\n  text: Coach is a python reinforcement learning framework containing implementation of many state-of-the-art algorithms, it supports MXNet as a back-end\n  icon: /assets/img/coach_logo.png\n  link: https://github.com/NervanaSystems/coach\n- title: XFer\n  text: Xfer is a library that allows quick and easy transfer of knowledge stored in deep neural networks implemented in Apache MXNet.\n  link: https://xfer.readthedocs.io/en/master/\n  icon: /assets/img/xfer.png\n- title: DJL\n  text: Deep Java Library is an open source library to build and deploy deep learning in Java\n  icon: /assets/img/djl.png\n  link: https://djl.ai/\n- title: Multi Model Server\n  text: Model Server for Apache MXNet (MMS) is a flexible and easy to use tool for serving deep learning models exported from MXNet or the Open Neural Network Exchange (ONNX).\n  link: https://github.com/awslabs/multi-model-server\n- title: MxNet Sharp\n  text: MxNet Sharp package brings efficient and flexible GPU computing and state-of-art deep learning to .NET. It covers all the Imperative, Symbolic and Gluon interface with API's written closer to Python syntax which makes is easier to port python code easily.\n  link: https://github.com/deepakkumar1984/MxNet.Sharp\n  icon: /assets/img/mxnet_sharp.png\n---\n\n<!---\n  Licensed to the Apache Software Foundation (ASF) under one\n  or more contributor license agreements.  See the NOTICE file\n  distributed with this work for additional information\n  regarding copyright ownership.  The ASF licenses this file\n  to you under the Apache License, Version 2.0 (the\n  \"License\"); you may not use this file except in compliance\n  with the License.  You may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing,\n  software distributed under the License is distributed on an\n  \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  KIND, either express or implied.  See the License for the\n  specific language governing permissions and limitations\n  under the License.\n-->\n\n<div class=\"ecosystem-page\">\n    <div class=\"row\">\n        <h2>D2L.ai</h2>\n        <div class=\"row\">\n            <div class=\"col-4\">\n                <a href=\"http://d2l.ai/\"><img src=\"{{'/assets/img/front.jpg' | relative_url}}\"></a>\n            </div>\n            <div class=\"col-8\">\n                <p>A <a href=\"https://d2l.ai\">deep learning book</a> with interactive jupyter notebooks, math formula,\n                    and a dedicated forum for discussions.</p>\n                <p>It offers an interactive learning experience with mathematics, figures, code, text, and discussions,\n                    where concepts and techniques are illustrated and implemented with experiments on real data\n                    sets.</p>\n                <p>Each section is an executable Jupyter notebook. You can modify the code and tune hyperparameters to\n                    get instant feedback to accumulate practical experiences in deep learning.</p>\n                <p>The book is authored by <a href=\"https://www.astonzhang.com/\">Aston Zhang</a>, Amazon Applied\n                    Scientist UIUC Ph.D., <a href=\"http://zacklipton.com/\">Zack C. Lipton</a>, CMU Assistant Professor\n                    UCSD Ph.D.,\n                    <a href=\"https://scholar.google.com/citations?user=Z_WrhK8AAAAJ&hl=en\">Mu Li</a> Amazon Principal\n                    Scientist CMU Ph.D. and <a href=\"https://alex.smola.org/\">Alex J. Smola</a> Amazon VP/Distinguished\n                    Scientist TU Berlin Ph.D.\n                <p>D2L is used as a textbook or a reference book at Carnegie Mellon University, Georgia Institute of\n                    Technology, the University of California Berkeley and many more university</p>\n            </div>\n        </div>\n    </div>\n    <br><br>\n    <h2>Toolkits</h2>\n    <div class=\"row\">\n        {%- for feature in page.ecosystem_toolkits -%}\n        <div class=\"col-4\">\n            <div class=\"card\">\n                <a href=\"{{feature.link}}\">\n                    <div class=\"card-text\">\n                        <div class=\"card-header-title\">\n                            <h4>{{feature.title}}</h4>\n                            <img src=\"{{feature.icon | relative_url}}\">\n                        </div>\n                        <p class=\"card-summary\">{{feature.text}}</p>\n                    </div>\n                </a>\n            </div>\n        </div>\n        {%- endfor -%}\n    </div>\n    <br><br>\n    <h2>Ecosystem</h2>\n    <div class=\"row\">\n        {%- for feature in page.ecosystem_other -%}\n        <div class=\"col-3\">\n            <div class=\"card\">\n                <a href=\"{{feature.link}}\">\n                    <div class=\"card-text\">\n                        <div class=\"card-header-title\">\n                            <h4>{{feature.title}}</h4>\n                            <img src=\"{{feature.icon | relative_url}}\">\n                        </div>\n                        <p class=\"card-summary\">{{feature.text}}</p>\n                    </div>\n                </a>\n            </div>\n        </div>\n        {%- endfor -%}\n    </div>\n    <br><br>\n</div>\n"
  },
  {
    "path": "docs/static_site/src/pages/features.html",
    "content": "---\nlayout: page\ntitle: Features\nsubtitle: Whether you are looking for a flexible library to quickly develop cutting-edge deep learning research or a robust framework to push production workload, Apache MXNet caters to all needs.\npermalink: /features/\naction: Get Started\naction_url: /get_started\n---\n\n<!---\n  Licensed to the Apache Software Foundation (ASF) under one\n  or more contributor license agreements.  See the NOTICE file\n  distributed with this work for additional information\n  regarding copyright ownership.  The ASF licenses this file\n  to you under the Apache License, Version 2.0 (the\n  \"License\"); you may not use this file except in compliance\n  with the License.  You may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing,\n  software distributed under the License is distributed on an\n  \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  KIND, either express or implied.  See the License for the\n  specific language governing permissions and limitations\n  under the License.\n-->\n\n<!-- HYBRID FRONTEND -->\n<div class=\"row\">\n    <div class=\"col-12\">\n        <h3 class=\"feature-title\">Hybrid Front-End</h3>\n    </div>\n    <div class=\"col-5\">\n        <p class=\"feature-paragraph\">The Gluon Python API lets you use Apache MXNet in a fully imperative manner. It also\n            allows you to simply switch to\n            symbolic mode by calling the <a\n                    href=\"/api/python/docs/tutorials/packages/gluon/blocks/hybridize.html\">hybridize</a>\n            functionality. The symbolic execution provides faster and more optimized\n            execution as well as the ability to export the network for inference in different language bindings like\n            java or C++.\n        </p>\n    </div>\n    <div class=\"col-1\">\n\n    </div>\n    <div class=\"col-6 code-block\">\n        {% highlight python %}\nnet = model_zoo.vision.resnet50_v2(pretrained=True)\nnet.hybridize()\n\ndummy_input = mx.nd.ones(shape=(1,3,224,224))\nnet(dummy_input)\n\nnet.export(\"symbolic_resnet50\")\n        {% endhighlight %}\n    </div>\n</div>\n\n<!-- DISTRIBUTED TRAINING -->\n<br>\n<br>\n<div class=\"row \">\n    <div class=\"col-7\"></div>\n    <div class=\"col-5\">\n        <h3 class=\"feature-title\">Distributed Training</h3>\n    </div>\n\n    <div class=\"col-6 code-block\">\n        {% highlight python %}\nimport horovod.mxnet as hvd\n\n# Horovod: initialize Horovod\nhvd.init()\n\n# Horovod: pin a GPU to be used to local rank\ncontext = mx.gpu(hvd.local_rank())\n        {% endhighlight %}\n    </div>\n\n    <div class=\"col-1\"></div>\n    <div class=\"col-5\">\n        <p class=\"feature-paragraph\">Apache MXNet allows you to make the most out of your hardware, whether it is multi-gpu or\n            multi-host training with near-linear scaling efficiency. Apache MXNet recently introduced support for\n            <a href=\"https://medium.com/apache-mxnet/distributed-training-using-apache-mxnet-with-horovod-44f98bf0e7b7\">Horovod</a>,\n            the distributed learning framework developed by Uber.\n        </p>\n    </div>\n</div>\n\n\n<!-- 8 Language Bindings -->\n<br><br>\n<div class=\"row\">\n    <div class=\"col-12\">\n        <h3 class=\"feature-title\">8 Language Bindings</h3>\n    </div>\n    <div class=\"col-5\">\n        <p class=\"feature-paragraph\">Deep integration into Python and support for Scala, Julia, Clojure, Java, C++, R\n            and Perl.\n            Combined with the hybridization feature, this allows a very smooth transition from Python training to\n            deployment\n            in the language of your choice to shorten the time to production.\n        </p>\n    </div>\n    <div class=\"col-1\">\n\n    </div>\n    <div class=\"col-6 code-block\">\n        {% highlight java %}\nimport org.apache.mxnet.javaapi.*;\n...\nList\n<DataDesc> inputDesc = new ArrayList<>();\nShape inputShape = new Shape(new int[]{1, 3, 224, 224});\ninputDesc.add(new DataDesc(\"data\", inputShape, DType.Float32(), \"NCHW\"));\nPredictor predictor = new Predictor(inst.modelPathPrefix, inputDesc, context,0);\n...\nfloat[][] result = predictor.predict(new float[][]{img.toArray()});\n            {% endhighlight %}\n    </div>\n</div>\n\n\n</div>\n"
  },
  {
    "path": "docs/static_site/src/pages/get_started/build_from_source.md",
    "content": "---\nlayout: page\ntitle: Building From Source\naction: Get Started\naction_url: /get_started\npermalink: /get_started/build_from_source\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Build Apache MXNet from Source\n\nBuilding and installing Apache MXNet from source is a three-step process. First, build\nthe shared `libmxnet` which provides the MXNet backend, then install your\npreferred language binding and finally validate that MXNet was installed\ncorrectly by running a small example.\n\n1. [Obtaining the source](#obtaining-the-source-code)\n2. [Installing MXNet's recommended dependencies](#installing-mxnet's-recommended-dependencies)\n3. [Overview of optional dependencies and optional features](#overview-of-optional-dependencies-and-optional-features)\n4. [Building MXNet](#building-mxnet)\n5. [Install the language API binding(s)](#installing-mxnet-language-bindings) you would like to use for MXNet.\n\nMXNet's newest and most popular API is Gluon. Gluon is built into the Python\nbinding. If Python isn't your preference, you still have more options. MXNet\nsupports several other language bindings. Please see the [API Documentation\npage](/api) for an overview of all supported languages and their APIs.\n\n\n## Obtaining the source code\n\nTo obtain the source code of the latest Apache MXNet release,\nplease access the [Download page](/get_started/download) and download the\n`.tar.gz` source archive corresponding to the release you wish to build.\n\nDevelopers can also obtain the unreleased development code from the git\nrepository via `git clone --recursive https://github.com/apache/mxnet`\n\nBuilding a MXNet 1.x release from source requires a C++11 compliant compiler.\n\nBuilding the development version of MXNet or any 2.x release from source\nrequires a C++17 compliant compiler. The oldest compiler versions tested during\nMXNet 2 development are GCC 7, Clang 6 and MSVC 2019.\n\n## Installing MXNet's recommended dependencies\nTo install the build tools and recommended dependencies, please run the\nfollowing commands respectively based on your Operating System. Please see the\nnext section for further explanations on the set of required and optional\ndependencies of MXNet.\n\n### Debian Linux derivatives (Debian, Ubuntu, ...)\n```bash\nsudo apt-get update\nsudo apt-get install -y build-essential git ninja-build ccache libopenblas-dev libopencv-dev cmake\n```\n\n### Red Hat Enterprise Linux derivatives (RHEL, CentOS, Fedora, ...)\n```bash\nsudo yum install epel-release centos-release-scl\nsudo yum install git make ninja-build automake autoconf libtool protobuf-compiler protobuf-devel \\\n    atlas-devel openblas-devel lapack-devel opencv-devel openssl-devel zeromq-devel python3 \\ \n    devtoolset-8\nsource /opt/rh/devtoolset-7/enable\n```\nHere `devtoolset-8` refers to the [Developer Toolset\n8](https://www.softwarecollections.org/en/scls/rhscl/devtoolset-8/) created by\nRed Hat for developers working on CentOS or Red Hat Enterprise Linux platform\nand providing the GNU Compiler Collection 9.\n\n### macOS\n```bash\n# Install OS X Developer Tools\nxcode-select --install\n\n# Install Homebrew\n/usr/bin/ruby -e \"$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)\"\n\n# Install dependencies\nbrew install cmake ninja ccache opencv\n```\n\nNote: the compiler provided by Apple on macOS does not support OpenMP. To use\nOpenMP on macOS you need to install for example the Clang compiler via `brew`:\n\n```bash\nbrew install llvm\n```\n\n### Windows\nYou can use Chocolatey software management solution to install some dependencies\non Windows.\n\n```bash\nchoco install python git 7zip cmake ninja opencv\n```\n\nCurrently OpenBLAS is not available from Chocolatey. You may download it from\nfrom [the OpenBLAS release page](https://github.com/xianyi/OpenBLAS/releases)\nand compile from source. Set the `OpenBLAS_HOME` environment variable to point\nto the OpenBLAS directory that contains the `include` and `lib` directories for\nexample by typing `set OpenBLAS_HOME=C:\\utils\\OpenBLAS`.\n\nIf you like to compile MXNet with Visual Studio compiler, please install at\nleast [VS2019](https://www.visualstudio.com/downloads/).\n\n## Overview of optional dependencies and optional features\n\n### Math Library Selection\nMXNet relies on the\n[BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) (Basic\nLinear Algebra Subprograms) library for numerical computations. In addition to\nBLAS, some operators in MXNet rely on the [LAPACK (Linear Algebra\nPackage)](https://github.com/Reference-LAPACK/lapack), an additional set of\nmathematical functions.\n\nSeveral BLAS and LAPACK implementations exist. Among them, MXNet is tested with:\n\n* [Apple Accelerate](https://developer.apple.com/documentation/accelerate)\n* [ATLAS](http://math-atlas.sourceforge.net/)\n* [Intel MKL](https://software.intel.com/en-us/intel-mkl)\n* [OpenBLAS](https://www.openblas.net/)\n\nApple Accelerate and MKL are proprietary. ATLAS and OpenBLAS are Open Source. If\nyou don't have any specific requirements, MXNet recommends OpenBLAS as it\ntypically outperforms ATLAS, is portable across many platforms, provides a\nLAPACK implementation and has a permissive license.\n\nPlease note that since MXNet 2.0 we are forcing static link to OpenBLAS\n`libopenblas.a` on non-Windows systems. In the case that the OpenBLAS library depends\non `gfortran`, be sure to install it too as a dependency. For example, on Debian\nsystems you can run:\n```bash\nsudo apt install gfortran\n```\nOr on Red Hat systems, run:\n```bash\nsudo yum install gcc-gfortran\n```\n\n### Optional GPU support\n\nMXNet optionally supports [NVDIA CUDA and\ncuDNN](https://developer.nvidia.com/cuda-downloads) for better performance on\nNVidia devices. MXNet releases in general are tested with the last two major\nCUDA versions available at the time of the release. For example, CUDA 9.2 and\n10.2.\n\nTo compile MXNet with CUDA support, define the `USE_CUDA` option. If you compile\nMXNet on a system with NVidia GPUs, the build system will automatically detect\nthe CUDA Architecture. If you are compiling on a system without NVidia GPUs,\nplease specify the `MXNET_CUDA_ARCH` option to select the CUDA Architecture and\navoid a lengthy build targeting all common CUDA Architectures. Please see the\nMXNet build configuration instructions in the next step.\n\nMXNet also supports [NCCL](https://developer.nvidia.com/nccl) - NVIDIA's\nCollective Communications Library. NCCL is useful when using MXNet on multiple\nGPUs that require communication. Instructions for installing NCCL are found in\nthe following [Build MXNet with NCCL](#build-mxnet-with-nccl) section.\n\nTo enable building MXNet with NCCL, install NCCL and define the `USE_NCCL`\noption in the MXNet build configuration in the next step.\n\nAfter building with NCCL, you may optionally use the tests in\n`tests/python/gpu/test_nccl.py` to ensure NCCL is enabled correctly. Please\nfirst delete the line containing `skip(reason=\"Test requires NCCL library\ninstalled and enabled during build\")` before running the test. In MXNet 2.x\nversions, the test can be run via `pytest --verbose\ntests/python/gpu/test_nccl.py`. In MXNet 1.x it is run via `python\ntests/python/gpu/test_nccl.py`.\n\nTo get the best performance out of NCCL it is recommended to set environment\nvariable `NCCL_LAUNCH_MODE=PARALLEL` when using NCCL version 2.1 or newer.\n\n### Optional OpenCV support\n\nMXNet's Image Loading and Augmentation features rely on\n[OpenCV](http://opencv.org/). Image Loading and Augmentation\n\n## Building MXNet\n\nMXNet 1.x can be built either with a classic Makefile setup or with the `cmake`\ncross platform build system. Starting with MXNet 1.7, MXNet recommends using the\n`cmake` cross platform build tool.\n\nNote: The `cmake` build requires CMake 3.13 or higher. If you are running an\nolder version of CMake, you will see an error message like `CMake 3.13 or higher\nis required. You are running version 3.10.2`. Please update CMake on your\nsystem. You can download and install latest CMake from https://cmake.org or via\nthe Python package manager `pip` with `python3 -m pip install --user --upgrade\n\"cmake>=3.13.2\"`. After installing cmake with `pip3`, it is usually available at\n`~/.local/bin/cmake` or directly as `cmake`.\n\nPlease see the [`cmake configuration\nfiles`](https://github.com/apache/mxnet/tree/v1.x/config) files for\ninstructions on how to configure and build MXNet with cmake.\n\nUp to the MXNet 1.6 release, please follow the instructions in the\n[`make/config.mk`](https://github.com/apache/mxnet/blob/v1.x/make/config.mk)\nfile on how to configure and compile MXNet. This method is supported on all 1.x\nreleases.\n\nTo enable the optional MXNet C++ package, please set the `USE_CPP_PACKAGE=1`\noption prior to compiling. See the [C++ guide](cpp_setup) for more information.\n\n\n## Installing MXNet Language Bindings\nAfter building MXNet's shared library, you can install other language bindings.\n\n**NOTE:** The C++ API binding must be built when you build MXNet from source. See [Build MXNet with C++]({{'/api/cpp.html'|relative_url}}).\n\n## Installing Language Packages for MXNet\n\nAfter you have installed the MXNet core library. You may install MXNet interface\npackages for the programming language of your choice:\n- [Python](#install-mxnet-for-python)\n- [C++](#install-the-mxnet-package-for-c&plus;&plus;)\n- [Clojure](#install-the-mxnet-package-for-clojure)\n- [Julia](#install-the-mxnet-package-for-julia)\n- [Perl](#install-the-mxnet-package-for-perl)\n- [R](#install-the-mxnet-package-for-r)\n- [Scala](#install-the-mxnet-package-for-scala)\n- [Java](#install-the-mxnet-package-for-java)\n\n\n### Install MXNet for Python\n\nTo install the MXNet Python binding navigate to the root of the MXNet folder then run the following:\n\n```bash\npython3 -m pip install --user -e ./python\n```\n\nNote that the `-e` flag is optional. It is equivalent to `--editable` and means\nthat if you edit the source files, these changes will be reflected in the\npackage installed.\n\nYou may optionally install ```graphviz``` library that is used for visualizing\nnetwork graphs you build on MXNet. You may also install [Jupyter\nNotebook](http://jupyter.readthedocs.io/) which is used for running MXNet\ntutorials and examples.\n\n```bash\npython3 -m pip install --user graphviz==0.8.4 jupyter\n```\n\nPlease also see the [MXNet Python API](/api/python) page.\n\n## Contributions\n\nYou are more than welcome to contribute easy installation scripts for other operating systems and programming languages.\nSee the [community contributions page]({{'/community/contribute'|relative_url}}) for further information.\n\n## Next Steps\n\n* [Tutorials]({{'/api'|relative_url}})\n* [How To]({{'/api/faq/add_op_in_backend'|relative_url}})\n* [Architecture]({{'/api/architecture/overview'|relative_url}})\n"
  },
  {
    "path": "docs/static_site/src/pages/get_started/download.md",
    "content": "---\nlayout: page\ntitle: Download Source Files\naction: Get Started\naction_url: /get_started\npermalink: /get_started/download\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Source Download\n\nThe source archives listed on this page are official Apache MXNet releases following\nthe [Apache Software Foundation Release\nPolicy](http://www.apache.org/legal/release-policy.html).\n\nIf you would like to actively participate in the Apache MXNet development, you are\nencouraged to contribute to our development version on\n[GitHub](https://github.com/apache/mxnet).\n\n| Version | Source                                                                                                      | PGP                                                                                                             | SHA                                                                                                                |\n|---------|-------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------|\n| 1.9.1   | [apache-mxnet-src-1.9.1-incubating.tar.gz](https://www.apache.org/dyn/closer.lua?filename=incubator/mxnet/1.9.1/apache-mxnet-src-1.9.1-incubating.tar.gz&action=download)   | [asc](https://downloads.apache.org/incubator/mxnet/1.9.1/apache-mxnet-src-1.9.1-incubating.tar.gz.asc)    |  [sha512](https://downloads.apache.org/incubator/mxnet/1.9.1/apache-mxnet-src-1.9.1-incubating.tar.gz.sha512)    |\n| 1.9.0   | [apache-mxnet-src-1.9.0-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/1.9.0/apache-mxnet-src-1.9.0-incubating.tar.gz)   | [asc](https://archive.apache.org/dist/incubator/mxnet/1.9.0/apache-mxnet-src-1.9.0-incubating.tar.gz.asc)    |  [sha512](https://archive.apache.org/dist/incubator/mxnet/1.9.0/apache-mxnet-src-1.9.0-incubating.tar.gz.sha512)    |\n| 1.8.0   | [apache-mxnet-src-1.8.0-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/1.8.0/apache-mxnet-src-1.8.0-incubating.tar.gz)   | [asc](https://archive.apache.org/dist/incubator/mxnet/1.8.0/apache-mxnet-src-1.8.0-incubating.tar.gz.asc)    |  [sha512](https://archive.apache.org/dist/incubator/mxnet/1.8.0/apache-mxnet-src-1.8.0-incubating.tar.gz.sha512)    |\n| 1.7.0   | [apache-mxnet-src-1.7.0-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/1.7.0/apache-mxnet-src-1.7.0-incubating.tar.gz)   | [asc](https://archive.apache.org/dist/incubator/mxnet/1.7.0/apache-mxnet-src-1.7.0-incubating.tar.gz.asc)    |  [sha512](https://archive.apache.org/dist/incubator/mxnet/1.7.0/apache-mxnet-src-1.7.0-incubating.tar.gz.sha512)    |\n| 1.6.0   | [apache-mxnet-src-1.6.0-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/1.6.0/apache-mxnet-src-1.6.0-incubating.tar.gz)   | [asc](https://archive.apache.org/dist/incubator/mxnet/1.6.0/apache-mxnet-src-1.6.0-incubating.tar.gz.asc)    |  [sha512](https://archive.apache.org/dist/incubator/mxnet/1.6.0/apache-mxnet-src-1.6.0-incubating.tar.gz.sha512)    |\n| 1.5.1   | [apache-mxnet-src-1.5.1-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/1.5.1/apache-mxnet-src-1.5.1-incubating.tar.gz)   | [asc](https://archive.apache.org/dist/incubator/mxnet/1.5.1/apache-mxnet-src-1.5.1-incubating.tar.gz.asc)    |  [sha512](https://archive.apache.org/dist/incubator/mxnet/1.5.1/apache-mxnet-src-1.5.1-incubating.tar.gz.sha512)     |\n| 1.5.0   | [apache-mxnet-src-1.5.0-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/1.5.0/apache-mxnet-src-1.5.0-incubating.tar.gz)   | [asc](https://archive.apache.org/dist/incubator/mxnet/1.5.0/apache-mxnet-src-1.5.0-incubating.tar.gz.asc)    |  [sha512](https://archive.apache.org/dist/incubator/mxnet/1.5.0/apache-mxnet-src-1.5.0-incubating.tar.gz.sha512)     |\n| 1.4.1   | [apache-mxnet-src-1.4.1-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/1.4.1/apache-mxnet-src-1.4.1-incubating.tar.gz)   | [asc](https://archive.apache.org/dist/incubator/mxnet/1.4.1/apache-mxnet-src-1.4.1-incubating.tar.gz.asc)    | [sha512](https://archive.apache.org/dist/incubator/mxnet/1.4.1/apache-mxnet-src-1.4.1-incubating.tar.gz.sha512)      |\n| 1.4.0   | [apache-mxnet-src-1.4.0-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/1.4.0/apache-mxnet-src-1.4.0-incubating.tar.gz)   | [asc](https://archive.apache.org/dist/incubator/mxnet/1.4.0/apache-mxnet-src-1.4.0-incubating.tar.gz.asc)    | [sha512](https://archive.apache.org/dist/incubator/mxnet/1.4.0/apache-mxnet-src-1.4.0-incubating.tar.gz.sha512)      |\n| 1.3.1   | [apache-mxnet-src-1.3.1-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/1.3.1/apache-mxnet-src-1.3.1-incubating.tar.gz)   | [asc](https://archive.apache.org/dist/incubator/mxnet/1.3.1/apache-mxnet-src-1.3.1-incubating.tar.gz.asc)    | [sha512](https://archive.apache.org/dist/incubator/mxnet/1.3.1/apache-mxnet-src-1.3.1-incubating.tar.gz.sha512)      |\n| 1.3.0   | [apache-mxnet-src-1.3.0-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/1.3.0/apache-mxnet-src-1.3.0-incubating.tar.gz)   | [asc](https://archive.apache.org/dist/incubator/mxnet/1.3.0/apache-mxnet-src-1.3.0-incubating.tar.gz.asc)    | [sha512](https://archive.apache.org/dist/incubator/mxnet/1.3.0/apache-mxnet-src-1.3.0-incubating.tar.gz.sha512)      |\n| 1.2.1   | [apache-mxnet-src-1.2.1-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/1.2.1/apache-mxnet-src-1.2.1-incubating.tar.gz)   | [asc](https://archive.apache.org/dist/incubator/mxnet/1.2.1/apache-mxnet-src-1.2.1-incubating.tar.gz.asc)    | [sha512](https://archive.apache.org/dist/incubator/mxnet/1.2.1/apache-mxnet-src-1.2.1-incubating.tar.gz.sha512)      |\n| 1.2.0   | [apache-mxnet-src-1.2.0-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/1.2.0/apache-mxnet-src-1.2.0-incubating.tar.gz)   | [asc](https://archive.apache.org/dist/incubator/mxnet/1.2.0/apache-mxnet-src-1.2.0-incubating.tar.gz.asc)    | [sha512](https://archive.apache.org/dist/incubator/mxnet/1.2.0/apache-mxnet-src-1.2.0-incubating.tar.gz.sha512)      |\n| 1.1.0   | [apache-mxnet-src-1.1.0-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/1.1.0/apache-mxnet-src-1.1.0-incubating.tar.gz)   | [asc](https://archive.apache.org/dist/incubator/mxnet/1.1.0/apache-mxnet-src-1.1.0-incubating.tar.gz.asc)    | [sha512](https://archive.apache.org/dist/incubator/mxnet/1.1.0/apache-mxnet-src-1.1.0-incubating.tar.gz.sha512)     |\n| 1.0.0   | [apache-mxnet-src-1.0.0-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/1.0.0/apache-mxnet-src-1.0.0-incubating.tar.gz)   | [asc](https://archive.apache.org/dist/incubator/mxnet/1.0.0/apache-mxnet-src-1.0.0-incubating.tar.gz.asc)    | [sha512](https://archive.apache.org/dist/incubator/mxnet/1.0.0/apache-mxnet-src-1.0.0-incubating.tar.gz.sha512)   |\n| 0.12.1  | [apache-mxnet-src-0.12.1-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/0.12.1/apache-mxnet-src-0.12.1-incubating.tar.gz) | [asc](https://archive.apache.org/dist/incubator/mxnet/0.12.1/apache-mxnet-src-0.12.1-incubating.tar.gz.asc)  | [sha512](https://archive.apache.org/dist/incubator/mxnet/0.12.1/apache-mxnet-src-0.12.1-incubating.tar.gz.sha512) |\n| 0.12.0  | [apache-mxnet-src-0.12.0-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/0.12.0/apache-mxnet-src-0.12.0-incubating.tar.gz) | [asc](https://archive.apache.org/dist/incubator/mxnet/0.12.0/apache-mxnet-src-0.12.0-incubating.tar.gz.asc)  | [sha512](https://archive.apache.org/dist/incubator/mxnet/0.12.0/apache-mxnet-src-0.12.0-incubating.tar.gz.sha512) |\n| 0.11.0  | [apache-mxnet-src-0.11.0-incubating.tar.gz](https://archive.apache.org/dist/incubator/mxnet/0.11.0/apache-mxnet-src-0.11.0-incubating.tar.gz) | [asc](https://archive.apache.org/dist/incubator/mxnet/0.11.0/apache-mxnet-src-0.11.0-incubating.tar.gz.asc)  | [sha512](https://archive.apache.org/dist/incubator/mxnet/0.11.0/apache-mxnet-src-0.11.0-incubating.tar.gz.sha512) |\n\n## Verify the Integrity of the Files\nIt is essential that you verify the integrity of the downloaded file using the PGP signature (.asc file) or a hash (.md5 or .sha* file). Please read [Verifying Apache Software Foundation Releases](https://www.apache.org/info/verification.html) for more information on why you should verify our releases.\n\nThe PGP signature can be verified using PGP or GPG. First download the [KEYS](https://apache.org/dist/incubator/mxnet/KEYS) as well as the .asc signature file for the relevant distribution. Make sure you get these files from the main distribution site, rather than from a mirror. Then verify the signatures using one of the following alternatives:\n\n```bash\n% gpg --import KEYS\n% gpg --verify downloaded_file.asc downloaded_file\n```\n\n```bash\n% pgpk -a KEYS\n% pgpv downloaded_file.asc\n```\n\n```bash\n% pgp -ka KEYS\n% pgp downloaded_file.asc\n```\n\nAlternatively, you can verify the hash on the file.\n\nHashes can be calculated using GPG:\n\n```bash\n% gpg --print-md SHA1 downloaded_file\n```\n\nThe output should be compared with the contents of the SHA1 file. Similarly for other hashes (SHA256 MD5 etc) which may be provided.\n\nWindows 7 and later systems should all now have `certUtil`:\n\n```bash\n% certUtil -hashfile pathToFileToCheck\n```\n\nHashAlgorithm choices: MD2 MD4 MD5 SHA1 SHA256 SHA384 SHA512\n\nUnix-like systems (and macOS) will have a utility called `md5`, `md5sum` or `shasum`.\n"
  },
  {
    "path": "docs/static_site/src/pages/get_started/index.html",
    "content": "---\nlayout: page\ntitle: Get Started\naction: Apache MXNet Tutorials\naction_url: /api/python/docs/tutorials\npermalink: /get_started/index.html\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n</div>\n<div class=\"get-started-from-source\">\n<div class=\"wrapper\">\n    <h2>Build and install Apache MXNet from source</h2>\n    <p>\n        To build and install Apache MXNet from the official Apache Software Foundation\n        signed source code please follow our <a href=\"/get_started/build_from_source\">Building From Source</a> guide.\n    </p>\n    <p>\n        The signed source releases are available <a href=\"/get_started/download\">here</a>\n    </p>\n</div>\n</div>\n\n{% include /get_started/get_started.html %}\n"
  },
  {
    "path": "docs/static_site/src/pages/get_started/jetson_setup.md",
    "content": "---\nlayout: page\ntitle: Jetson Setup\naction: Get Started\naction_url: /get_started\npermalink: /get_started/jetson_setup\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n\n# Install MXNet on a Jetson\n\nMXNet supports Ubuntu AArch64 based operating system so you can run MXNet on all [NVIDIA Jetson modules](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/), such as Jetson Nano, TX1, TX2, Xavier NX and AGX Xavier.\n\nThese instructions will walk through how to build MXNet and install MXNet's Python language binding.\n\nFor the purposes of this install guide we will assume that CUDA is already installed on your Jetson device. NVIDIA Jetpack comes with the latest OS image for Jetson mdoule, and developer tools for both host computer and developer kit, and this also includes CUDA. You should double check what versions are installed and which version you plan to use.\n\nAfter installing the prerequisites, you have several options for installing MXNet:\n1. Build MXNet from source\n   * On a faster Linux computer using cross-compilation\n   * On the Jetson itself (very slow and not recommended)\n2. Use a Jetson MXNet pip wheel for Python development and use a precompiled Jetson MXNet binary (not provided on this page as CUDA enabled wheels are not in accordance with ASF policy, users can download them from other 3rd party sources)\n\n## Prerequisites\nTo build from source or to use the Python wheel, you must install the following dependencies on your Jetson.\nCross-compiling will require dependencies installed on that machine as well.\n\n### Python Dependencies\n\nTo use the Python API you need the following dependencies:\n\n```bash\nsudo apt-get update\nsudo apt-get install -y \\\n                        build-essential \\\n                        git \\\n                        libopenblas-dev \\\n                        libopencv-dev \\\n                        python3-pip \\\n                        python-numpy\n\nsudo pip3 install --upgrade \\\n                        pip \\\n                        setuptools \\\n                        numpy\n```\n\nIf you plan to cross-compile you will need to install these dependencies on that computer as well.\n\n### Download the source & setup some environment variables:\n\nThese steps are optional, but some of the following instructions expect MXNet source files and the `MXNET_HOME` environment variable. Also, CUDA commands will not work out of the box without updating your path.\n\nClone the MXNet source code repository using the following `git` command in your home directory:\n\n```bash\ngit clone --recursive https://github.com/apache/mxnet.git mxnet\n```\n\nYou can also checkout a particular branch of MXNet. For example, to install MXNet v1.6:\n```bash\ngit clone --recursive -b v1.6.x https://github.com/apache/mxnet.git mxnet\n```\n\nSetup your environment variables for MXNet and CUDA in your `.profile` file in your home directory.\nAdd the following to the file.\n\n```bash\nexport PATH=/usr/local/cuda/bin:$PATH\nexport MXNET_HOME=$HOME/mxnet/\nexport PYTHONPATH=$MXNET_HOME/python:$PYTHONPATH\n```\n\nYou can then apply this change immediately with the following:\n```bash\nsource .profile\n```\n\n**Note:** Change the `~/.profile` steps according to how you prefer to use your shell. Otherwise, your environment variables will be gone after you logout.\n\n### Configure CUDA\n\nYou can check to see what version of CUDA is running with `nvcc`.\n\n```bash\nnvcc --version\n```\n\nTo switch CUDA versions on a device or computer that has more than one version installed, use the following and replace the symbolic link to the version you want. This one uses CUDA 10.2, which comes with Jetpack 4.4.\n\n```bash\nsudo rm /usr/local/cuda\nsudo ln -s /usr/local/cuda-10.2 /usr/local/cuda\n```\n\n**Note:** When cross-compiling, change the CUDA version on the host computer you're using to match the version you're running on your Jetson device.\n\n\n## Build MXNet from Source\n\nInstalling MXNet from source is a two-step process:\n\n1. Build the shared library from the MXNet C++ source code.\n2. Install the supported language-specific packages for MXNet.\n\nYou can use a Docker method or you can build from source manually.\n\n### Docker\n\nYou must have installed Docker and be able to run `docker` without `sudo`.\nFollow these [setup instructions to get to this point](https://docs.docker.com/install/linux/#manage-docker-as-a-non-root-user).\nThen run the following to execute cross-compilation via Docker.\n\n```bash\n$MXNET_HOME/ci/build.py -p jetson\n```\n\n### Manually on the Jetson module (Slow)\n\n**Step 1** Build the Shared Library\n\nUse the config_jetson.mk Makefile to install MXNet with CUDA bindings to leverage the GPU on the Jetson module.\n\n```bash\ncp $MXNET_HOME/make/config_jetson.mk config.mk\n```\n\nThe pre-existing Makefile builds for all Jetson architectures. Edit `config.mk` if you want to specifically build for a particular architecture or if you want to build without CUDA bindings (CPU only). You can make the following changes:\n\n1. Modify `CUDA_ARCH` to build for specific architectures. Currently, we have `CUDA_ARCH = -gencode arch=compute_53,code=sm_53 -gencode arch=compute_62,code=sm_62 -gencode arch=compute_72,code=sm_72`. Keep `-gencode arch=compute_53,code=sm_53` for Nano and TX1, `-gencode arch=compute_62,code=sm_62` for TX2, `-gencode arch=compute_72,code=sm_72` for Xavier NX and AGX Xavier.\n\n2. For CPU only builds, remove `USE_CUDA_PATH`, `CUDA_ARCH`, `USE_CUDNN` flags.\n\nNow you can build the complete MXNet library with the following command:\n\n```bash\ncd $MXNET_HOME\nmake -j $(nproc)\n```\n\nExecuting this command creates a file called `libmxnet.so` in the `mxnet/lib` directory.\n\n**Step 2** Install MXNet Python Bindings (optional)\n\nTo install Python bindings run the following commands in the MXNet directory:\n\n```bash\ncd $MXNET_HOME/python\nsudo pip3 install -e .\n```\n\nNote that the `-e` flag is optional. It is equivalent to `--editable` and means that if you edit the source files, these changes will be reflected in the package installed.\n\n**Step 3** Install the MXNet Java & Scala Bindings (optional)\n\nChange directories to `scala-package` and run `mvn install`.\n\n```bash\ncd $MXNET_HOME/scala-package\nmvn install\n```\n\nThis creates the required `.jar` file to use in your Java or Scala projects.\n\n## Conclusion and Next Steps\n\nYou are now ready to run MXNet on your NVIDIA module.\nYou can verify your MXNet Python installation with the following:\n\n```python\nimport mxnet\nmxnet.__version__\n```\n\nYou can also verify MXNet can use your GPU with the following test:\n\n```python\nimport mxnet as mx\na = mx.nd.ones((2, 3), mx.gpu())\nb = a * 2 + 1\nb.asnumpy()\n```\n\nIf everything is working, it will report the version number.\nFor assistance, head over to the [MXNet Forum](https://discuss.mxnet.io/).\n"
  },
  {
    "path": "docs/static_site/src/pages/get_started/validate_mxnet.md",
    "content": "---\nlayout: page\ntitle: Validate MXNet\naction: Get Started\naction_url: /get_started\npermalink: /get_started/validate_mxnet\n---\n<!--- Licensed to the Apache Software Foundation (ASF) under one -->\n<!--- or more contributor license agreements.  See the NOTICE file -->\n<!--- distributed with this work for additional information -->\n<!--- regarding copyright ownership.  The ASF licenses this file -->\n<!--- to you under the Apache License, Version 2.0 (the -->\n<!--- \"License\"); you may not use this file except in compliance -->\n<!--- with the License.  You may obtain a copy of the License at -->\n\n<!---   http://www.apache.org/licenses/LICENSE-2.0 -->\n\n<!--- Unless required by applicable law or agreed to in writing, -->\n<!--- software distributed under the License is distributed on an -->\n<!--- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->\n<!--- KIND, either express or implied.  See the License for the -->\n<!--- specific language governing permissions and limitations -->\n<!--- under the License. -->\n\n# Validate Your MXNet Installation\n\n- [Python](#python)\n- [Python with GPU](#python-with-gpu)\n- [Verify GPU training](#verify-gpu-training)\n- [Virtualenv](#virtualenv)\n- [Docker with CPU](#docker-with-cpu)\n- [Docker with GPU](#docker-with-gpu)\n- [Cloud](#cloud)\n- [C++](#alternative-language-bindings)\n- [Clojure](#clojure)\n- [Julia](#julia)\n- [Perl](#perl)\n- [R](#r)\n- [Scala](#scala)\n\n\n## Python\n\nStart the python terminal.\n\n```bash\n$ python\n```\n\nRun a short *MXNet* python program to create a 2X3 matrix of ones, multiply each element in the matrix by 2 followed by adding 1. We expect the output to be a 2X3 matrix with all elements being 3.\n\n```python\n>>> import mxnet as mx\n>>> a = mx.nd.ones((2, 3))\n>>> b = a * 2 + 1\n>>> b.asnumpy()\narray([[ 3.,  3.,  3.],\n       [ 3.,  3.,  3.]], dtype=float32)\n```\n\n\n## Python with GPU\n\nThis is similar to the previous example, but this time we use *mx.gpu()*, to set *MXNet* context to be GPUs.\n\n```python\n>>> import mxnet as mx\n>>> a = mx.nd.ones((2, 3), mx.gpu())\n>>> b = a * 2 + 1\n>>> b.asnumpy()\narray([[ 3.,  3.,  3.],\n       [ 3.,  3.,  3.]], dtype=float32)\n```\n\n\n## Alternative Language Bindings\n\n### C++\n\nPlease contribute an example!\n\n\n### Clojure\n\nPlease contribute an example!\n\n\n### Julia\n\nPlease contribute an example!\n\n\n### Perl\n\nStart the pdl2 terminal.\n\n```bash\n$ pdl2\n```\n\nRun a short *MXNet* Perl program to create a 2X3 matrix of ones, multiply each element in the matrix by 2 followed by adding 1. We expect the output to be a 2X3 matrix with all elements being 3.\n\n```perl\npdl> use AI::MXNet qw(mx)\npdl> $a = mx->nd->ones([2, 3])\npdl> $b = $a * 2 + 1\npdl> print $b->aspdl\n\n[\n [3 3 3]\n [3 3 3]\n]\n```\n\n### R\n\nRun a short *MXNet* R program to create a 2X3 matrix of ones, multiply each element in the matrix by 2 followed by adding 1. We expect the output to be a 2X3 matrix with all elements being 3.\n\n```r\nlibrary(mxnet)\na <- mx.nd.ones(c(2,3), ctx = mx.cpu())\nb <- a * 2 + 1\nb\n```\n\nYou should see the following output:\n\n```r\n[,1] [,2] [,3]\n[1,]    3    3    3\n[2,]    3    3    3\n```\n\n\n#### R with GPU\n\nThis is similar to the previous example, but this time we use *mx.gpu()*, to set *MXNet* context to be GPUs.\n\n```r\nlibrary(mxnet)\na <- mx.nd.ones(c(2,3), ctx = mx.gpu())\nb <- a * 2 + 1\nb\n```\n\nYou should see the following output:\n\n```r\n[,1] [,2] [,3]\n[1,]    3    3    3\n[2,]    3    3    3\n```\n\n\n### Scala\n\nRun the <a href=\"https://github.com/apache/mxnet/tree/master/scala-package/mxnet-demo\">MXNet-Scala demo project</a> to validate your Maven package installation.\n"
  },
  {
    "path": "docs/static_site/src/pages/trusted_by.html",
    "content": "---\nlayout: page\ntitle: Trusted By\nsubtitle: These are some of the organizations that use Apache MXNet.\naction: Get Started\naction_url: /get_started\npermalink: /trusted_by/\n---\n\n<!---\n  Licensed to the Apache Software Foundation (ASF) under one\n  or more contributor license agreements.  See the NOTICE file\n  distributed with this work for additional information\n  regarding copyright ownership.  The ASF licenses this file\n  to you under the Apache License, Version 2.0 (the\n  \"License\"); you may not use this file except in compliance\n  with the License.  You may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing,\n  software distributed under the License is distributed on an\n  \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  KIND, either express or implied.  See the License for the\n  specific language governing permissions and limitations\n  under the License.\n-->\n\n<div class=\"trusted-by-page\">\n    <div class=\"row\">\n        <img src=\"{{'/assets/img/logos.png' | relative_url}}\">\n    </div>\n    <br><br>\n</div>\n"
  },
  {
    "path": "docs/tutorial_utils/vision/cnn_visualization/gradcam.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nfrom __future__ import print_function\n\nimport mxnet as mx\n\nfrom mxnet import gluon, np, npx\nfrom mxnet import autograd\nfrom mxnet.gluon import nn\n\nimport numpy as onp\nimport cv2\n\nclass ReluOp(mx.operator.CustomOp):\n    \"\"\"Modified ReLU as described in section 3.4 in https://arxiv.org/abs/1412.6806.\n    This is used for guided backpropagation to get gradients of the image w.r.t activations.\n    This Operator will do a regular backpropagation if `guided_backprop` is set to False\n    and a guided packpropagation if `guided_backprop` is set to True. Check gradcam_demo.py\n    for an example usage.\"\"\"\n\n    guided_backprop = False\n\n    def forward(self, is_train, req, in_data, out_data, aux):\n        x = in_data[0]\n        y = np.maximum(x, np.zeros_like(x))\n        self.assign(out_data[0], req[0], y)\n\n    def backward(self, req, out_grad, in_data, out_data, in_grad, aux):\n        if ReluOp.guided_backprop:\n            # Get output and gradients of output\n            y = out_data[0]\n            dy = out_grad[0]\n            # Zero out the negatives in the gradients of the output\n            dy_positives = np.maximum(dy, np.zeros_like(dy))\n            # What output values were greater than 0?\n            y_ones = y.__gt__(0)\n            # Mask out the values for which at least one of dy or y is negative\n            dx = dy_positives * y_ones\n            self.assign(in_grad[0], req[0], dx)\n        else:\n            # Regular backward for ReLU\n            x = in_data[0]\n            x_gt_zero = x.__gt__(0)\n            dx = out_grad[0] * x_gt_zero\n            self.assign(in_grad[0], req[0], dx)\n\ndef set_guided_backprop(mode=True):\n    ReluOp.guided_backprop = mode\n\n@mx.operator.register(\"relu\")\nclass ReluProp(mx.operator.CustomOpProp):\n    def __init__(self):\n        super(ReluProp, self).__init__(True)\n\n    def infer_shape(self, in_shapes):\n        data_shape = in_shapes[0]\n        output_shape = data_shape\n        return (data_shape,), (output_shape,), ()\n\n    def create_operator(self, ctx, in_shapes, in_dtypes):\n        return ReluOp()  \n\nclass Activation(mx.gluon.HybridBlock):\n    @staticmethod\n    def set_guided_backprop(mode=False):\n        ReluOp.guided_backprop = mode\n\n    def __init__(self, act_type, **kwargs):\n        assert act_type == 'relu'\n        super(Activation, self).__init__(**kwargs)\n\n    def forward(self, x):\n        return npx.Custom(x, op_type='relu')\n\nclass Conv2D(mx.gluon.HybridBlock):\n    \"\"\"Wrapper on top of gluon.nn.Conv2D to capture the output and gradients of output of a Conv2D\n    layer in a network. Use `set_capture_layer_name` to select the layer\n    whose outputs and gradients of outputs need to be captured. After the backward pass,\n    `conv_output` will contain the output and `conv_output.grad` will contain the\n    output's gradients. Check gradcam_demo.py for example usage.\"\"\"\n\n    conv_output = None\n    capture_layer_name = None\n\n    def __init__(self, channels, kernel_size, strides=(1, 1), padding=(0, 0),\n                 dilation=(1, 1), groups=1, layout='NCHW',\n                 activation=None, use_bias=True, weight_initializer=None,\n                 bias_initializer='zeros', in_channels=0, **kwargs):\n        super(Conv2D, self).__init__(**kwargs)\n        self.conv = nn.Conv2D(channels, kernel_size, strides=strides, padding=padding,\n                             dilation=dilation, groups=groups, layout=layout,\n                             activation=activation, use_bias=use_bias, weight_initializer=weight_initializer,\n                             bias_initializer=bias_initializer, in_channels=in_channels)\n\n    def forward(self, x):\n        out = self.conv(x)\n        name = self._prefix[:-1]\n        if name == Conv2D.capture_layer_name:\n            out.attach_grad()\n            Conv2D.conv_output = out\n        return out\n\ndef set_capture_layer_name(name):\n    Conv2D.capture_layer_name = name\n\ndef _get_grad(net, image, class_id=None, conv_layer_name=None, image_grad=False):\n    \"\"\"This is an internal helper function that can be used for either of these\n    but not both at the same time:\n    1. Record the output and gradient of output of an intermediate convolutional layer.\n    2. Record the gradients of the image.\n\n    Parameters\n    ----------\n    image : NDArray\n        Image to visuaize. This is an NDArray with the preprocessed image.\n    class_id : int\n        Category ID this image belongs to. If not provided,\n        network's prediction will be used.\n    conv_layer_name: str\n        Name of the convolutional layer whose output and output's gradients need to be acptured.\n    image_grad: bool\n        Whether to capture gradients of the image.\"\"\"\n\n    if image_grad:\n        image.attach_grad()\n        Conv2D.capture_layer_name = None\n        Activation.set_guided_backprop(True)\n    else:\n        # Tell convviz.Conv2D which layer's output and gradient needs to be recorded\n        Conv2D.capture_layer_name = conv_layer_name\n        Activation.set_guided_backprop(False)\n    \n    # Run the network\n    with autograd.record(train_mode=False):\n        out = net(image)\n    \n    # If user didn't provide a class id, we'll use the class that the network predicted\n    if class_id == None:\n        model_output = out.asnumpy()\n        class_id = onp.argmax(model_output)\n\n    # Create a one-hot target with class_id and backprop with the created target\n    one_hot_target = mx.npx.one_hot(mx.np.array([class_id]), 1000)\n    out.backward(one_hot_target, train_mode=False)\n\n    if image_grad:\n        return image.grad[0].asnumpy()\n    else:\n        # Return the recorded convolution output and gradient\n        conv_out = Conv2D.conv_output\n        return conv_out[0].asnumpy(), conv_out.grad[0].asnumpy()\n\ndef get_conv_out_grad(net, image, class_id=None, conv_layer_name=None):\n    \"\"\"Get the output and gradients of output of a convolutional layer.\n\n    Parameters:\n    ----------\n    net: Block\n        Network to use for visualization.\n    image: NDArray\n        Preprocessed image to use for visualization.\n    class_id: int\n        Category ID this image belongs to. If not provided,\n        network's prediction will be used.\n    conv_layer_name: str\n        Name of the convolutional layer whose output and output's gradients need to be acptured.\"\"\"\n    return _get_grad(net, image, class_id, conv_layer_name, image_grad=False)\n\ndef get_image_grad(net, image, class_id=None):\n    \"\"\"Get the gradients of the image.\n\n    Parameters:\n    ----------\n    net: Block\n        Network to use for visualization.\n    image: NDArray\n        Preprocessed image to use for visualization.\n    class_id: int\n        Category ID this image belongs to. If not provided,\n        network's prediction will be used.\"\"\"\n    return _get_grad(net, image, class_id, image_grad=True)\n\ndef grad_to_image(gradient):\n    \"\"\"Convert gradients of image obtained using `get_image_grad`\n    into image. This shows parts of the image that is most strongly activating\n    the output neurons.\"\"\"\n    gradient = gradient - gradient.min()\n    gradient /= gradient.max()\n    gradient = onp.uint8(gradient * 255).transpose(1, 2, 0)\n    gradient = gradient[..., ::-1]\n    return gradient\n\ndef get_cam(imggrad, conv_out):\n    \"\"\"Compute CAM. Refer section 3 of https://arxiv.org/abs/1610.02391 for details\"\"\"\n    weights = onp.mean(imggrad, axis=(1, 2))\n    cam = onp.ones(conv_out.shape[1:], dtype=onp.float32)\n    for i, w in enumerate(weights):\n        cam += w * conv_out[i, :, :]\n    cam = cv2.resize(cam, (imggrad.shape[1], imggrad.shape[2]))\n    cam = onp.maximum(cam, 0)\n    cam = (cam - onp.min(cam)) / (onp.max(cam) - onp.min(cam)) \n    cam = onp.uint8(cam * 255)\n    return cam\n\ndef get_guided_grad_cam(cam, imggrad):\n    \"\"\"Compute Guided Grad-CAM. Refer section 3 of https://arxiv.org/abs/1610.02391 for details\"\"\"\n    return onp.multiply(cam, imggrad)\n\ndef get_img_heatmap(orig_img, activation_map):\n    \"\"\"Draw a heatmap on top of the original image using intensities from activation_map\"\"\"\n    heatmap = cv2.applyColorMap(activation_map, cv2.COLORMAP_COOL)\n    heatmap = cv2.cvtColor(heatmap, cv2.COLOR_BGR2RGB)\n    img_heatmap = onp.float32(heatmap) + onp.float32(orig_img)\n    img_heatmap = img_heatmap / onp.max(img_heatmap)\n    img_heatmap *= 255\n    return img_heatmap.astype(int)\n\ndef to_grayscale(cv2im):\n    \"\"\"Convert gradients to grayscale. This gives a saliency map.\"\"\"\n    # How strongly does each position activate the output\n    grayscale_im = onp.sum(onp.abs(cv2im), axis=0)\n\n    # Normalize between min and 99th percentile\n    im_max = onp.percentile(grayscale_im, 99)\n    im_min = onp.min(grayscale_im)\n    grayscale_im = onp.clip((grayscale_im - im_min) / (im_max - im_min), 0, 1)\n\n    grayscale_im = onp.expand_dims(grayscale_im, axis=0)\n    return grayscale_im\n\ndef visualize(net, preprocessed_img, orig_img, conv_layer_name):\n    # Returns grad-cam heatmap, guided grad-cam, guided grad-cam saliency\n    imggrad = get_image_grad(net, preprocessed_img)\n    conv_out, conv_out_grad = get_conv_out_grad(net, preprocessed_img, conv_layer_name=conv_layer_name)\n\n    cam = get_cam(conv_out_grad, conv_out)\n    cam = cv2.resize(cam, (imggrad.shape[1], imggrad.shape[2]))\n    ggcam = get_guided_grad_cam(cam, imggrad)\n    img_ggcam = grad_to_image(ggcam)\n    \n    img_heatmap = get_img_heatmap(orig_img, cam)\n    \n    ggcam_gray = to_grayscale(ggcam)\n    img_ggcam_gray = onp.squeeze(grad_to_image(ggcam_gray))\n    \n    return img_heatmap, img_ggcam, img_ggcam_gray\n\n"
  },
  {
    "path": "example/MXNetTutorialTemplate.ipynb",
    "content": "{\n \"cells\": [\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"# Licensed to the Apache Software Foundation (ASF) under one\\n\",\n    \"# or more contributor license agreements.  See the NOTICE file\\n\",\n    \"# distributed with this work for additional information\\n\",\n    \"# regarding copyright ownership.  The ASF licenses this file\\n\",\n    \"# to you under the Apache License, Version 2.0 (the\\n\",\n    \"# \\\"License\\\"); you may not use this file except in compliance\\n\",\n    \"# with the License.  You may obtain a copy of the License at\\n\",\n    \"#\\n\",\n    \"#   http://www.apache.org/licenses/LICENSE-2.0\\n\",\n    \"#\\n\",\n    \"# Unless required by applicable law or agreed to in writing,\\n\",\n    \"# software distributed under the License is distributed on an\\n\",\n    \"# \\\"AS IS\\\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\\n\",\n    \"# KIND, either express or implied.  See the License for the\\n\",\n    \"# specific language governing permissions and limitations\\n\",\n    \"# under the License.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"# Tutorial Title\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"A brief introduction to the tutorial that describes:\\n\",\n    \"\\n\",\n    \"- The problem that the tutorial addresses\\n\",\n    \"- Who the intended audience is\\n\",\n    \"- The expected experience level of that audience with a concept or tool \\n\",\n    \"- Which environment/language it runs in \\n\",\n    \"\\n\",\n    \"If there is another similar tutorial that's more appropriate for another audience, direct the reader there with a linked reference.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## How to Use This Tutorial\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"A brief explanation of how the reader can use the tutorial. Can the reader copy each code snippet into a Python or other environment? Or can the reader run `<filename>` before or after reading through the explanations to understand how the code works?\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"You can use this tutorial by *insert method(s) here*. \\n\",\n    \"\\n\",\n    \"A bulleted list of the tasks the reader will accomplish and skills he or she will learn. Begin each list item with a noun (Learn, Create, Use, etc.).\\n\",\n    \"\\n\",\n    \"You will accomplish the following:\\n\",\n    \"\\n\",\n    \"- First task or skill\\n\",\n    \"- Second task or skill\\n\",\n    \"- X task or skill\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Prerequisites\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"Provide a *complete* list of the software, hardware, knowledge, and skills required to be successful using the tutorial. For each item, link the item to installation instructions, specs, or skill development tools, as appropriate. If good installation instructions aren't available for required software, start the tutorial with instructions for installing it.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"To complete this tutorial, you need:\\n\",\n    \"\\n\",\n    \"- [MXNet](https://mxnet.apache.org/install/#overview)\\n\",\n    \"- [Language](https://mxnet.apache.org/tutorials/)\\n\",\n    \"- [Tool](https://mxnet.apache.org/api/python/index.html)\\n\",\n    \"- [Familiarity with concept or tool](https://gluon.mxnet.io/)\\n\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## The Data\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"Provide a link to where the data is hosted and explain how to download it. If it requires more than two steps, use a numbered list.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"You can download the data used in this tutorial from the [Site Name](http://) site. To download the data:\\n\",\n    \"\\n\",\n    \"1. At the `<language>` prompt, type:\\n\",\n    \"\\n\",\n    \"    `<command>`\\n\",\n    \"2. Second task.\\n\",\n    \"\\n\",\n    \"3. Last task.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"Briefly describe key aspects of the data. If there are two or more aspects of the data that require involved discussion, use subheads (### `<Concept or Sub-component Name>`). To include a graphic, introduce it with a brief description and use the image linking tool to include it. Store the graphic in GitHub and use the following format: <img width=\\\"517\\\" alt=\\\"screen shot 2016-05-06 at 10 13 16 pm\\\" src=\\\"https://cloud.githubusercontent.com/assets/5545640/15089697/d6f4fca0-13d7-11e6-9331-7f94fcc7b4c6.png\\\">. You do not need to provide a title for your graphics. \"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"The data *add description here. (optional)*\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## (Optional) Concept or Component Name\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"If concepts or components need further introduction, include this section. If there are two or more aspects of the concept or component that require involved discussion, use subheads (### Concept or Sub-component Name).\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Prepare the Data\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"If appropriate, summarize the tasks required to prepare the data, defining and explaining key concepts.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"To prepare the data, *provide explanation here.*\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"Use a numbered procedure to explain how to prepare the data. Add code snippets or blocks that show the code that the user must type or that is used for this task in the Jupyter Notebook. To include code snippets, precede each line of code with four spaces and two tick marks. Always introduce input or output with a description or context or result, followed by a colon.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"To prepare the data:\\n\",\n    \"\\n\",\n    \"1.\\n\",\n    \"\\n\",\n    \"2.\\n\",\n    \"\\n\",\n    \"3.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"If there are any aspects of data preparation that require elaboration, add it here.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Create the Model\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"If appropriate, summarize the tasks required to create the model, defining and explaining key concepts.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"To create the model, *provide explanation here.*\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"Use a numbered procedure to explain how to create the data. Add code snippets or blocks that show the code that the user must type or that is used for this task in the Jupyter Notebook. To include code snippets, precede each line of code with four spaces and two tick marks. Always introduce input or output with a description or context or result, followed by a colon.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"To create the model:\\n\",\n    \"\\n\",\n    \"1.\\n\",\n    \"\\n\",\n    \"2.\\n\",\n    \"\\n\",\n    \"3.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"If there are any aspects of model creation that require elaboration, add it here.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Fit the Model\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"If appropriate, summarize the tasks required to fit the model, defining and explaining key concepts.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"To fit the model, *provide explanation here.*\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"Use a numbered procedure to explain how to fit the model. Add code snippets or blocks that show the code that the user must type or that is used for this task in the Jupyter Notebook. To include code snippets, precede each line of code with four spaces and two tick marks. Always introduce input or output with a description or context or result, followed by a colon.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"To fit the model:\\n\",\n    \"\\n\",\n    \"1.\\n\",\n    \"\\n\",\n    \"2.\\n\",\n    \"\\n\",\n    \"3.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"If there are any aspects of model fitting that require elaboration, add it here.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Evaluate the Model\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"If appropriate, summarize the tasks required to evaluate the model, defining and explaining key concepts.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"To evaluate the model, *provide explanation here.*\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"Use a numbered procedure to explain how to evaluate the model. Add code snippets or blocks that show the code that the user must type or that is used for this task in the Jupyter Notebook. To include code snippets, precede each line of code with four spaces and two tick marks. Always introduce input or output with a description or context or result, followed by a colon.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"To evaluate the model:\\n\",\n    \"\\n\",\n    \"1.\\n\",\n    \"\\n\",\n    \"2.\\n\",\n    \"\\n\",\n    \"3.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"If there are any aspects of model evaluation that require elaboration, add it here.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## (Optional) Additional Tasks\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"If appropriate, summarize the tasks required to perform the task, defining and explaining key concepts.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"To *perform the task*, *provide explanation here.*\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"Use a numbered procedure to explain how to perform the task. Add code snippets or blocks that show the code that the user must type or that is used for this task in the Jupyter Notebook. To include code snippets, precede each line of code with four spaces and two tick marks. Always introduce input or output with a description or context or result, followed by a colon.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"To *perform the task*:\\n\",\n    \"\\n\",\n    \"1.\\n\",\n    \"\\n\",\n    \"2.\\n\",\n    \"\\n\",\n    \"3.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"If there are any aspects of model evaluation that require elaboration, add it here.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Summary\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"Briefly describe the end result of the tutorial and how the user can use it or modify it to customize it.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Next Steps\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"Provide a bulleted list of other documents, tools, or tutorials that further explain the concepts discussed in this tutorial or build on this tutorial. Start each list item with a brief description of a user task followed by the title of the destination site or topic that is formatted as a link.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"- For more information on *topic*, see [Site Name](http://).\\n\",\n    \"- To learn more about using *tool or task*, see [Topic Title](http://).\\n\",\n    \"- To experiment with *service*, *tool*, or *object*, see [Site Name](http://).\\n\",\n    \"- For a more advanced tutorial on *subject*, see [Tutorial Title](http://).\"\n   ],\n   \"metadata\": {}\n  }\n ],\n \"metadata\": {\n  \"kernelspec\": {\n   \"display_name\": \"Python 2\",\n   \"language\": \"python\",\n   \"name\": \"python2\"\n  },\n  \"language_info\": {\n   \"codemirror_mode\": {\n    \"name\": \"ipython\",\n    \"version\": 2\n   },\n   \"file_extension\": \".py\",\n   \"mimetype\": \"text/x-python\",\n   \"name\": \"python\",\n   \"nbconvert_exporter\": \"python\",\n   \"pygments_lexer\": \"ipython2\",\n   \"version\": \"2.7.12\"\n  }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}"
  },
  {
    "path": "example/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# MXNet Examples\n\nThis page contains a curated list of awesome MXNet examples, tutorials and blogs. It is inspired by [awesome-php](https://github.com/ziadoz/awesome-php) and [awesome-machine-learning](https://github.com/josephmisiti/awesome-machine-learning). See also [Awesome-MXNet](https://github.com/chinakook/Awesome-MXNet) for a similar list.\n\n  - [Contributing](#contributing)\n  - [List of examples](#list-of-examples)\n    - [Languages Binding Examples](#language-binding-examples)\n    - [Deep Learning Examples in the MXNet Project Repository](#deep-learning-examples-mxnet)\n    - [Other Deep Learning Examples with MXNet](#deep-learning-examples-other)\n    - [IPython Notebooks](#ipython-notebooks)\n    - [Mobile App Examples](#mobile-apps-examples)\n    - [Web Predictive Services](#web-predictive-services)\n  - [List of tutorials](#list-of-tutorials)\n    - [GPU Technology Conference 2016 Hands-on session](#gtc2016-hands-on)\n    - [Deep learning for hackers with MXnet](#deep-learning-for-hackers)\n    - [MXnet setup on AWS](#mxnet-aws)\n    - [Kaggle tutorials](#kaggle-tutorials)\n    - [Learning Note](#learning-note)\n  - [Machine Learning Challenge Winning Solutions](#winning-solutions)\n  - [Tools with MXnet](#tools-with-mxnet)\n\n## <a name=\"Contributing\"></a>Contributing\n\nIf you want to contribute to this list and the examples, please open a new pull request.\n\n\n### Examples\n\nExample applications or scripts should be submitted in this `example` folder.\n\n\n### Tutorials\n\nIf you have a tutorial idea for the website, download the [Jupyter notebook tutorial template](https://github.com/apache/mxnet/tree/master/example/MXNetTutorialTemplate.ipynb).\n\n#### Tutorial location\n\nNotebook tutorials should be submitted in the `docs/tutorials` folder, so that they maybe rendered in the [web site's tutorial section](https://mxnet.apache.org/tutorials/index.html).\n\nDo not forget to udpdate the `docs/tutorials/index.md` for your tutorial to show up on the website.\n\n#### Tutorial formatting\n\nThe site expects the format to be markdown, so export your notebook as a .md via the Jupyter web interface menu (File > Download As > Markdown). Then, to enable the download notebook button in the web site's UI ([example](https://mxnet.apache.org/tutorials/python/linear-regression.html)), add the following as the last line of the file ([example](https://github.com/apache/mxnet/blame/master/docs/tutorials/python/linear-regression.md#L194)):\n\n```\n<!-- INSERT SOURCE DOWNLOAD BUTTONS -->\n```\n\nIf you want some lines to show-up in the markdown but not in the generated notebooks, add  this comment `<!--notebook-skip-line-->` after your `![png](img_url)`. Like this:\n\n```\n![png](img_url.png)<!--notebook-skip-line-->\n```\n\nTypically when you have a `plt.imshow()` you want the image tag `[png](img.png)` in the `.md` but not in the downloaded notebook as the user will re-generate the plot at run-time.\n\n#### Tutorial tests\n\nAs part of making sure all our tutorials are running correctly with the latest version of MXNet, each tutorial is run automatically through a python2 and python3 jupyter notebook kernel in the CI, in a GPU environment, checking for errors and warnings.\n\nAdd your own test here `tests/tutorials/test_tutorials.py`. (If you forget, don't worry your PR will not pass the sanity check).\n\nIf your tutorial depends on specific packages, simply add them to this provisioning script: `ci/docker/install/ubuntu_tutorials.sh`\n\n## <a name=\"list-of-examples\"></a>List of examples\n\n### <a name=\"language-binding-examples\"></a>Languages Binding Examples\n------------------\n* [MXNet C++ API](https://mxnet.apache.org/api/c++/index.html)\n   - [C++ examples](https://github.com/apache/mxnet/tree/master/example/image-classification/predict-cpp) - Example code for using C++ interface, including NDArray, symbolic layer and models.\n* [MXNet Python API](https://mxnet.apache.org/api/python/index.html)\n* [MXNet Java API](https://mxnet.apache.org/api/java/index.html)\n* [MXNet Scala API](https://mxnet.apache.org/api/scala/index.html)\n* [MXNet R API](https://mxnet.apache.org/api/r/index.html)\n* [MXNet Julia API](https://mxnet.apache.org/api/julia/index.html)\n* [MXNet Perl API](https://mxnet.apache.org/api/perl/index.html)\n* [go-mxnet-predictor](https://github.com/songtianyi/go-mxnet-predictor) - Go binding for inference\n* [MXNet Javascript](https://github.com/dmlc/mxnet.js/) - MXNetJS: Javascript Package for Deep Learning in Browser (without server)\n\n### <a name=\"deep-learning-examples-mxnet\"></a>Deep Learning Examples in the MXNet Project Repository\n--------------\n* [Autoencoder](autoencoder) - unsupervised feature learning\n* [Gluon Examples](gluon) - several examples using the Gluon API\n  * [Style Transfer](gluon/style_transfer) - a style transfer example using gluon\n  * [Word Language Model](gluon/word_language_model) - an example that trains a multi-layer RNN on the Penn Treebank language modeling benchmark\n  * [SN-GAN](gluon/sn_gan) - an example that utilizes spectral normalization to train GAN(Generative adversarial network) using Gluon API\n* [Image Classification with R](image-classification) - image classification on MNIST,CIFAR,ImageNet-1k,ImageNet-Full, with multiple GPU and distributed training.\n* [Kaggle 2nd national data science bowl](kaggle-ndsb2) - a tutorial for Kaggle Second Nation Data Science Bowl\n* [Multi-task Learning](multi-task) - how to use MXNet for multi-task learning\n* [Profiling](profiler) - generate profiling results in json files\n* [Quantization and Calibration Examples](quantization) - examples of quantizing a FP32 model to INT8 and performing low-precision inference with oneDNN on CPU or cuDNN on GPU\n* [Recommender Systems](recommenders) - examples of how to build various kinds of recommender systems\n* [Restricted Boltzmann Machine](restricted-boltzmann-machine) - an example of the binary restricted Boltzmann machine learning MNIST\n* [Single Shot MultiBox Detector](ssd) - SSD object recognition example\n\n### <a name=\"deep-learning-examples-other\"></a>Other Deep Learning Examples with MXNet\n\n* [Face Recognition with ArcFace](https://github.com/onnx/models/tree/master/vision/body_analysis/arcface) - ONNX model for face recognition with notebooks for training, validating and running inference in MXNet by [abhinavs95](https://github.com/abhinavs95)\n* [Chinese plate recognition](https://github.com/imistyrain/mxnet-mr) - Recognize Chinese vehicle plate, by [imistyrain](https://github.com/imistyrain)\n* [Fast R-CNN](https://github.com/precedenceguo/mx-rcnn) by [Jian Guo](https://github.com/precedenceguo)\n* \"End2End Captcha Recognition (OCR)\" by [xlvector](https://github.com/xlvector) [github link](https://github.com/xlvector/learning-dl/tree/master/mxnet/ocr) [Blog in Chinese](http://blog.xlvector.net/2016-05/mxnet-ocr-cnn/)\n* \"Prediction step of xlvector's lstm ocr\" by [melody-rain](https://github.com/melody-rain) [github link](https://github.com/melody-rain/mxnet/commit/46002e31fc34c746c01bcaa7ade999187068ad3c) [Blog in Chinese](https://zhuanlan.zhihu.com/p/22698511)\n* \"Solving classification + regression with MXnet in Multi Input + Multi Obj\" by [xlvector](https://github.com/xlvector) [github link](https://gist.github.com/xlvector/c304d74f9dd6a3b68a3387985482baac) [Blog in Chinese](http://blog.xlvector.net/2016-05/mxnet-regression-classification-for-concret-continuous-features/)\n* \"Learn to sort by LSTM\" by [xlvector](https://github.com/xlvector) [github link](https://github.com/xlvector/learning-dl/tree/master/mxnet/lstm_sort) [Blog in Chinese](http://blog.xlvector.net/2016-05/mxnet-lstm-example/)\n* [Neural Art using extremely lightweight (<500K) neural network](https://github.com/pavelgonchar/neural-art-mini) Lightweight version of mxnet neural art implementation by [Pavel Gonchar](https://github.com/pavelgonchar)\n* [Neural Art with generative networks](https://github.com/zhaw/neural_style) by [zhaw](https://github.com/zhaw)\n* [Faster R-CNN in MXNet with distributed implementation and data parallelization](https://github.com/apache/mxnet/tree/master/example/rcnn)\n* [Asynchronous Methods for Deep Reinforcement Learning in MXNet](https://github.com/zmonoid/Asyn-RL-MXNet/blob/master/mx_asyn.py) by [zmonoid](https://github.com/zmonoid)\n* [Deep Q-learning in MXNet](https://github.com/zmonoid/DQN-MXNet) by [zmonoid](https://github.com/zmonoid)\n* [Face Detection with End-to-End Integration of a ConvNet and a 3D Model (ECCV16)](https://github.com/tfwu/FaceDetection-ConvNet-3D) by [tfwu](https://github.com/tfwu), source code for paper Yunzhu Li, Benyuan Sun, Tianfu Wu and Yizhou Wang, \"Face Detection with End-to-End Integration of a ConvNet and a 3D Model\", ECCV 2016 <https://arxiv.org/abs/1606.00850>\n* [End-to-End Chinese plate recognition base on MXNet](https://github.com/szad670401/end-to-end-for-chinese-plate-recognition) by [szad670401](https://github.com/szad670401)\n* [Reproduce ResNet-v2 (Identity Mappings in Deep Residual Networks) using MXNet](https://github.com/tornadomeet/ResNet) by [tornadomeet](https://github.com/tornadomeet)\n* [Learning similarity among images in MXNet](http://www.jianshu.com/p/70a66c8f73d3) by xlvector in Chinese. Github [link](https://github.com/xlvector/learning-dl/tree/master/mxnet/triple-loss)\n* [Matrix decomposition (SVD) with MXNet](http://www.jianshu.com/p/ebf7bf53ed3e) by xlvector in Chinese. Github [link](https://github.com/xlvector/mxnet/blob/svd/example/svd/svd.py)\n* [MultiGPU enabled image generative models (GAN and DCGAN)](https://github.com/tqchen/mxnet-gan) by [Tianqi Chen](https://github.com/tqchen)\n* [Deep reinforcement learning for playing flappybird by mxnet](https://github.com/li-haoran/DRL-FlappyBird) by LIHaoran\n* [Neural Style in Markov Random Field (MRF) and Perceptual Losses Realtime transfer](https://github.com/zhaw/neural_style) by [zhaw](https://github.com/zhaw)\n* [MTCNN Face keypoints detection and alignment](https://github.com/YYuanAnyVision/mxnet_mtcnn_face_detection) by [yuanyang](https://github.com/YYuanAnyVision), source code for [paper](https://kpzhang93.github.io/papers/spl.pdf) \"Joint Face Detection and Alignment using Multi-task Cascaded Convolutional Neural Networks\", [Kaipeng Zhang](https://github.com/kpzhang93), Zhanpeng Zhang, Zhifeng Li and Yu Qiao, IEEE Signal Processing Letters, 23(10), 2016\n* [SSD: Single Shot MultiBox Object Detector](https://github.com/zhreshold/mxnet-ssd) by [zhreshold](https://github.com/zhreshold)\n* [Fast Neural Style in Scala](https://github.com/Ldpe2G/DeepLearningForFun/tree/master/Mxnet-Scala/FastNeuralStyle) by [Ldpe2G](https://github.com/Ldpe2G)\n* [LSTM Human Activity Recognition](https://github.com/Ldpe2G/DeepLearningForFun/tree/master/Mxnet-Scala/HumanActivityRecognition) by [Ldpe2G](https://github.com/Ldpe2G)\n* [Visual Question Answering](https://github.com/liuzhi136/Visual-Question-Answering) by [liuzhi136](https://github.com/liuzhi136)\n* [Deformable ConvNets](https://arxiv.org/abs/1703.06211) ([github](https://github.com/msracver/Deformable-ConvNets)) by [MSRACVer](https://github.com/msracver)\n* [OCR with bi-LSTM and CTC Loss in Gluon](https://github.com/ThomasDelteil/Gluon_OCR_LSTM_CTC) by [ThomasDelteil](https://github.com/ThomasDelteil)\n* [Visual Search with Gluon and HNSWlib](https://github.com/ThomasDelteil/VisualSearch_MXNet), by [ThomasDelteil](https://github.com/ThomasDelteil), online demo [here](https://thomasdelteil.github.io/VisualSearch_MXNet/)\n* [MXNet-face](https://github.com/tornadomeet/mxnet-face) - Using MXNet for a face-related algorithm by [tornadomeet](https://github.com/tornadomeet) where the single model gets 97.13%+-0.88% accuracy on LFW, and with only 20MB size\n\n### <a name=\"ipython-notebooks\"></a>IPython Notebooks\n-----------------\n* [Predict with Pre-trained model](https://github.com/dmlc/mxnet-notebooks/blob/master/python/moved-from-mxnet/predict-with-pretrained-model.ipynb) - Notebook on how to predict with pretrained model.\n* [composite symbol](https://github.com/dmlc/mxnet-notebooks/blob/master/python/moved-from-mxnet/composite_symbol.ipynb) - A demo of how to composite a symbolic Inception-BatchNorm Network\n* [cifar-10 recipe](https://github.com/dmlc/mxnet-notebooks/blob/master/python/moved-from-mxnet/cifar10-recipe.ipynb) - A step by step demo of how to use MXNet\n* [cifar-100](https://github.com/dmlc/mxnet-notebooks/blob/master/python/moved-from-mxnet/cifar-100.ipynb) - A demo of how to train a 75.68% accuracy CIFAR-100 model\n* [simple bind](https://github.com/dmlc/mxnet-notebooks/blob/master/python/moved-from-mxnet/simple_bind.ipynb) - A demo of low level training API.\n* [Multi task tutorial](https://github.com/haria/mxnet-multi-task-example/blob/master/multi-task.ipynb) - A demo of how to train and predict multi-task network on both MNIST and your own dataset.\n* [class active maps](https://github.com/dmlc/mxnet-notebooks/blob/master/python/moved-from-mxnet/class_active_maps.ipynb) - A demo of how to localize the discriminative regions in an image using global average pooling (GAP) in CNNs.\n* [DMLC MXNet Notebooks](https://github.com/dmlc/mxnet-notebooks) DMLC's repo for various notebooks ranging from basic usages of MXNet to state-of-the-art deep learning applications.\n* [AWS Seoul Summit 2017 Demos](https://github.com/sxjscience/aws-summit-2017-seoul) The demo codes and ipython notebooks in AWS Seoul Summit 2017.\n* [Character-level CNN for text classification](https://github.com/ThomasDelteil/CNN_NLP_MXNet) Performing category classification on Amazon reviews using Gluon and character-level Convolutional Neural Networks. Online demo [here](https://thomasdelteil.github.io/CNN_NLP_MXNet/)\n\n### <a name=\"mobile-apps-examples\"></a>Mobile App Examples\n-------------------\n* [MXNet Android Classification App](https://github.com/Leliana/WhatsThis) - Image classification on Android with MXNet.\n* [MXNet iOS Classification App](https://github.com/pppoe/WhatsThis-iOS) - Image classification on iOS with MXNet.\n* [Compile MXnet on Xcode (in Chinese)](http://www.liuxiao.org/2015/12/ios-mxnet-%E7%9A%84-ios-%E7%89%88%E6%9C%AC%E7%BC%96%E8%AF%91/) - a step-by-step tutorial of compiling MXnet on Xcode for iOS app\n\n### <a name=\"web-predictive-services\"></a>Web Predictive Services\n-----------------------\n* [MXNet Shinny](https://github.com/thirdwing/mxnet_shiny) - Source code for quickly creating a Shiny R app to host online image classification.\n* [Machine Eye](http://rupeshs.github.io/machineye/) - Web service for local image file/image URL classification without uploading.\n\n## <a name=\"list-of-tutorials\"></a>List of tutorials\n\n### <a name=\"gtc2016-hands-on\"></a>GPU Technology Conference 2016 Hands-on session\n\n* [Video on GTC 2016 site](http://on-demand.gputechconf.com/gtc/2016/video/L6143.html)\n* [Video backup in Mainland China](http://pan.baidu.com/s/1eS58Gue)\n* [iPython Notebook](https://github.com/dmlc/mxnet-gtc-tutorial)\n\n### <a name=\"deep-learning-for-hackers\"></a>Deep learning for hackers with MXNet\n\n* Deep learning for hackers with MXNet (1) GPU installation and MNIST [English](https://no2147483647.wordpress.com/2015/12/07/deep-learning-for-hackers-with-mxnet-1/) [Chinese](http://phunter.farbox.com/post/mxnet-tutorial1) - a tutorial of installing MXnet with GPU and introduction to deep learning by MNIST example.\n* Deep learning for hackers with MXNet (2): Neural art [English](https://no2147483647.wordpress.com/2015/12/21/deep-learning-for-hackers-with-mxnet-2/) [Chinese](http://phunter.farbox.com/post/mxnet-tutorial2) - a tutorial of generating Van Gogh style cat paintings.\n\n### <a name=\"mxnet-aws\"></a>MXNet on the cloud\n* [Setup Amazon AWS GPU instance with MXnet](https://no2147483647.wordpress.com/2016/01/16/setup-amazon-aws-gpu-instance-with-mxnet/) - AWS GPU instance setup with GPU (CUDA with latest cuDNN and S3 support)\n* [Intro Guide to AWS (MXNet with Julia)](http://www.datasciencebowl.com/aws_guide/) - A step-by-step guide of using spot instances with Amazon Web Services (AWS) to help you save money when training DSB models on MXNet by [Mike Kim](http://www.datasciencebowl.com/author/mikekim/)\n* [Building Deep Neural Networks in the Cloud with Azure GPU VMs, MXNet and Microsoft R Server](https://blogs.technet.microsoft.com/machinelearning/2016/09/15/building-deep-neural-networks-in-the-cloud-with-azure-gpu-vms-mxnet-and-microsoft-r-server/) by [Cortana Intelligence and ML Blog Team](https://social.technet.microsoft.com/profile/Cortana+Intelligence+and+ML+Blog+Team) at Microsoft\n* [Applying Deep Learning at Cloud Scale, with Microsoft R Server & Azure Data Lake](https://blogs.technet.microsoft.com/machinelearning/2016/10/31/applying-cloud-deep-learning-at-scale-with-microsoft-r-server-azure-data-lake/) by [Cortana Intelligence and ML Blog Team](https://social.technet.microsoft.com/profile/Cortana+Intelligence+and+ML+Blog+Team) at Microsoft\n* [Training Deep Neural Neural Networks on ImageNet Using Microsoft R Server and Azure GPU VMs](https://blogs.technet.microsoft.com/machinelearning/2016/11/15/imagenet-deep-neural-network-training-using-microsoft-r-server-and-azure-gpu-vms/) by [Cortana Intelligence and ML Blog Team](https://social.technet.microsoft.com/profile/Cortana+Intelligence+and+ML+Blog+Team) at Microsoft\n* [Cloud-Scale Text Classification with Convolutional Neural Networks on Microsoft Azure](https://blogs.technet.microsoft.com/machinelearning/2017/02/13/cloud-scale-text-classification-with-convolutional-neural-networks-on-microsoft-azure/) by [Cortana Intelligence and ML Blog Team](https://social.technet.microsoft.com/profile/Cortana+Intelligence+and+ML+Blog+Team) at Microsoft\n* [Distributed Deep Learning Made Easy](https://aws.amazon.com/blogs/compute/distributed-deep-learning-made-easy/) at AWS/Amazon for deploying deep learning clusters using MXNet\n\n### <a name=\"kaggle-tutorials\"></a>Kaggle tutorials\n* [Kaggle 2nd Annual Data Science Bowl End-to-End Deep Learning Tutorial (Python)](https://www.kaggle.com/c/second-annual-data-science-bowl/forums/t/18079/end-to-end-deep-learning-tutorial-0-0392) - an end-to-end python tutorial for Kaggle heart disease diagnose competition (public leaderboard score 0.0392)\n* [Kaggle 2nd Annual Data Science Bowl End-to-End Deep Learning Tutorial (R)](https://www.kaggle.com/c/second-annual-data-science-bowl/forums/t/18122/deep-learning-model-in-r) - an end-to-end R tutorial for Kaggle heart disease diagnose competition\n* [Dogs vs. Cats classification with mxnet and R](https://statist-bhfz.github.io/cats_dogs_finetune) - end-to-end (not winning) tutorial with an example of fine-tuning in R\n\n### <a name=\"learning-note\"></a>Learning Note\n* [Learning Note in Chinese](https://github.com/zhubuntu/MXNet-Learning-Note) - MXNet learning note in Chinese.\n* [Getting Started with MXNet](https://indico.io/blog/getting-started-with-mxnet/) by [indico.io](https://indico.io) (Chinese Translation [MXNet实践](http://www.infoq.com/cn/articles/practise-of-mxnet) by [侠天](http://www.infoq.com/cn/author/%E4%BE%A0%E5%A4%A9) )\n* [{mxnet} R package from MXnet, an intuitive Deep Learning framework including CNN & RNN](http://tjo-en.hatenablog.com/entry/2016/03/30/233848) by [TJO](http://tjo-en.hatenablog.com/)\n* [MXnet with R: combined power of deep learning](http://cos.name/2016/04/mxnet-r/) in Chinese by Tong He\n* [Understand MXNet dependency engine](http://yuyang0.github.io/articles/mxnet-engine.html) in Chinese by [Yu Yang](https://github.com/yuyang0)\n\n## <a name=\"winning-solutions\"></a>Machine Learning Challenge Winning Solutions\n\n* Dmitrii Tsybulevskii, 1st place of the [Yelp Restaurant Photo Classification](https://www.kaggle.com/c/yelp-restaurant-photo-classification). Link to [the Kaggle interview](http://blog.kaggle.com/2016/04/28/yelp-restaurant-photo-classification-winners-interview-1st-place-dmitrii-tsybulevskii/).\n\n## <a name=\"tools-with-mxnet\"></a>Tools with MXnet\n* [TensorFuse](https://github.com/dementrock/tensorfuse) - Common interface for Theano, CGT, TensorFlow, and mxnet (experimental) by [dementrock](https://github.com/dementrock)\n* [MXnet-face](https://github.com/tornadomeet/mxnet-face) - Using mxnet for face-related algorithm by [tornadomeet](https://github.com/tornadomeet) where the single model get 97.13%+-0.88% accuracy on LFW, and with only 20MB size.\n* [MinPy](https://github.com/dmlc/minpy) - Pure numpy practice with third party operator Integration and MXnet as backend for GPU computing\n* [MXNet Model Server](https://github.com/awslabs/mxnet-model-server) - a flexible and easy to use tool for serving Deep Learning models\n"
  },
  {
    "path": "example/adversary/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# Adversarial examples\n\nThis demonstrates the concept of \"adversarial examples\" from [1] showing how to fool a well-trained CNN.\nAdversarial examples are samples where the input has been manipulated to confuse a model (i.e. confident in an incorrect prediction) but where the correct answer still appears obvious to a human.\nThis method for generating adversarial examples uses the gradient of the loss with respect to the input to craft the adversarial examples.\n\n[1] Goodfellow, Ian J., Jonathon Shlens, and Christian Szegedy. \"Explaining and harnessing adversarial examples.\" [arXiv preprint arXiv:1412.6572 (2014)](https://arxiv.org/abs/1412.6572)\n"
  },
  {
    "path": "example/adversary/adversary_generation.ipynb",
    "content": "{\n \"cells\": [\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"# Licensed to the Apache Software Foundation (ASF) under one\\n\",\n    \"# or more contributor license agreements.  See the NOTICE file\\n\",\n    \"# distributed with this work for additional information\\n\",\n    \"# regarding copyright ownership.  The ASF licenses this file\\n\",\n    \"# to you under the Apache License, Version 2.0 (the\\n\",\n    \"# \\\"License\\\"); you may not use this file except in compliance\\n\",\n    \"# with the License.  You may obtain a copy of the License at\\n\",\n    \"#\\n\",\n    \"#   http://www.apache.org/licenses/LICENSE-2.0\\n\",\n    \"#\\n\",\n    \"# Unless required by applicable law or agreed to in writing,\\n\",\n    \"# software distributed under the License is distributed on an\\n\",\n    \"# \\\"AS IS\\\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\\n\",\n    \"# KIND, either express or implied.  See the License for the\\n\",\n    \"# specific language governing permissions and limitations\\n\",\n    \"# under the License.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"# Fast Sign Adversary Generation Example\\n\",\n    \"\\n\",\n    \"This notebook demos finds adversary examples using MXNet Gluon and taking advantage of the gradient information\\n\",\n    \"\\n\",\n    \"[1] Goodfellow, Ian J., Jonathon Shlens, and Christian Szegedy. \\\"Explaining and harnessing adversarial examples.\\\" arXiv preprint arXiv:1412.6572 (2014).\\n\",\n    \"https://arxiv.org/abs/1412.6572\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 1,\n   \"source\": [\n    \"%matplotlib inline\\n\",\n    \"import mxnet as mx\\n\",\n    \"import numpy as np\\n\",\n    \"\\n\",\n    \"import matplotlib.pyplot as plt\\n\",\n    \"import matplotlib.cm as cm\\n\",\n    \"\\n\",\n    \"from mxnet import gluon\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {\n    \"collapsed\": false\n   }\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"Build simple CNN network for solving the MNIST dataset digit recognition task\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 17,\n   \"source\": [\n    \"ctx = mx.gpu() if mx.device.num_gpus() else mx.cpu()\\n\",\n    \"batch_size = 128\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {\n    \"collapsed\": true\n   }\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Data Loading\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 3,\n   \"source\": [\n    \"transform = lambda x,y: (x.transpose((2,0,1)).astype('float32')/255., y)\\n\",\n    \"\\n\",\n    \"train_dataset = gluon.data.vision.MNIST(train=True).transform(transform)\\n\",\n    \"test_dataset = gluon.data.vision.MNIST(train=False).transform(transform)\\n\",\n    \"\\n\",\n    \"train_data = gluon.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=5)\\n\",\n    \"test_data = gluon.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Create the network\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 4,\n   \"source\": [\n    \"net = gluon.nn.HybridSequential()\\n\",\n    \"with net.name_scope():\\n\",\n    \"    net.add(\\n\",\n    \"        gluon.nn.Conv2D(kernel_size=5, channels=20, activation='tanh'),\\n\",\n    \"        gluon.nn.MaxPool2D(pool_size=2, strides=2),\\n\",\n    \"        gluon.nn.Conv2D(kernel_size=5, channels=50, activation='tanh'),\\n\",\n    \"        gluon.nn.MaxPool2D(pool_size=2, strides=2),\\n\",\n    \"        gluon.nn.Flatten(),\\n\",\n    \"        gluon.nn.Dense(500, activation='tanh'),\\n\",\n    \"        gluon.nn.Dense(10)\\n\",\n    \"    )\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {\n    \"collapsed\": true\n   }\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Initialize training\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 5,\n   \"source\": [\n    \"net.initialize(mx.initializer.Uniform(), ctx=ctx)\\n\",\n    \"net.hybridize()\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {\n    \"collapsed\": true\n   }\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 6,\n   \"source\": [\n    \"loss = gluon.loss.SoftmaxCELoss()\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {\n    \"collapsed\": true\n   }\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 7,\n   \"source\": [\n    \"trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1, 'momentum':0.95})\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {\n    \"collapsed\": true\n   }\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Training loop\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 8,\n   \"source\": [\n    \"epoch = 3\\n\",\n    \"for e in range(epoch):\\n\",\n    \"    train_loss = 0.\\n\",\n    \"    acc = mx.gluon.metric.Accuracy()\\n\",\n    \"    for i, (data, label) in enumerate(train_data):\\n\",\n    \"        data = data.as_in_context(ctx)\\n\",\n    \"        label = label.as_in_context(ctx)\\n\",\n    \"        \\n\",\n    \"        with mx.autograd.record():\\n\",\n    \"            output = net(data)\\n\",\n    \"            l = loss(output, label)\\n\",\n    \"            \\n\",\n    \"        l.backward()\\n\",\n    \"        trainer.update(data.shape[0])\\n\",\n    \"        \\n\",\n    \"        train_loss += l.mean().item()\\n\",\n    \"        acc.update(label, output)\\n\",\n    \"    \\n\",\n    \"    print(\\\"Train Accuracy: %.2f\\\\t Train Loss: %.5f\\\" % (acc.get()[1], train_loss/(i+1)))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Train Accuracy: 0.92\\t Train Loss: 0.32142\\n\",\n      \"Train Accuracy: 0.97\\t Train Loss: 0.16773\\n\",\n      \"Train Accuracy: 0.97\\t Train Loss: 0.14660\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {\n    \"collapsed\": false\n   }\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Perturbation\\n\",\n    \"\\n\",\n    \"We first run a validation batch and measure the resulting accuracy.\\n\",\n    \"We then perturbate this batch by modifying the input in the opposite direction of the gradient.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 9,\n   \"source\": [\n    \"# Get a batch from the testing set\\n\",\n    \"for data, label in test_data:\\n\",\n    \"    data = data.as_in_context(ctx)\\n\",\n    \"    label = label.as_in_context(ctx)\\n\",\n    \"    break\\n\",\n    \"\\n\",\n    \"# Attach gradient to it to get the gradient of the loss with respect to the input\\n\",\n    \"data.attach_grad()\\n\",\n    \"with mx.autograd.record():\\n\",\n    \"    output = net(data)    \\n\",\n    \"    l = loss(output, label)\\n\",\n    \"l.backward()\\n\",\n    \"\\n\",\n    \"acc = mx.gluon.metric.Accuracy()\\n\",\n    \"acc.update(label, output)\\n\",\n    \"\\n\",\n    \"print(\\\"Validation batch accuracy {}\\\".format(acc.get()[1]))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Validation batch accuracy 0.96875\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"Now we perturb the input\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 10,\n   \"source\": [\n    \"data_perturbated = data + 0.15 * mx.np.sign(data.grad)\\n\",\n    \"\\n\",\n    \"output = net(data_perturbated)    \\n\",\n    \"\\n\",\n    \"acc = mx.gluon.metric.Accuracy()\\n\",\n    \"acc.update(label, output)\\n\",\n    \"\\n\",\n    \"print(\\\"Validation batch accuracy after perturbation {}\\\".format(acc.get()[1]))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Validation batch accuracy after perturbation 0.40625\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {\n    \"collapsed\": false\n   }\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Visualization\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"Let's visualize an example after pertubation.\\n\",\n    \"\\n\",\n    \"We can see that the prediction is often incorrect.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 16,\n   \"source\": [\n    \"from random import randint\\n\",\n    \"idx = randint(0, batch_size-1)\\n\",\n    \"\\n\",\n    \"plt.imshow(data_perturbated[idx, :].asnumpy().reshape(28,28), cmap=cm.Greys_r)\\n\",\n    \"print(\\\"true label: %d\\\" % label.asnumpy()[idx])\\n\",\n    \"print(\\\"predicted: %d\\\" % np.argmax(output.asnumpy(), axis=1)[idx])\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"true label: 1\\n\",\n      \"predicted: 3\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"display_data\",\n     \"data\": {\n      \"image/png\": \"iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADpJJREFUeJzt3V+IXeW5x/Hfc9JsNbbMmLbGkAQdgxwZAxoZY+EMJy1tgo2F2AuluSg5IE0vIrbQi4q9qJeh9A9eSHGqobG2ScVWDConsaFgS0p1FI/G8VRNSWmGJGOxpCnIjJk8vdgrZYx7r7Wz1989z/cDw+xZ715rPbMmv6y997vW+5q7C0A8/1F3AQDqQfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwT1sSp31mq1fNmyZaVs+/Tp06Vs97yhoaHa9p0lrbYmq/O41X3M0n73rNref//9rm1nz57V/Py89VJDrvCb2W2SHpS0RNIj7r4r7fnLli3T+Ph4nl129eyzz5ay3fPS6i5731nKOqZlq/O41X3M0n73rNqmpqa6tk1PT/dcQ98v+81siaSHJH1R0qikbWY22u/2AFQrz3v+DZLecfc/u/ucpH2SthZTFoCy5Qn/Kkl/XfDz8WTZh5jZDjObNLPJubm5HLsDUKTSP+139wl3H3P3sVarVfbuAPQoT/inJa1Z8PPqZBmAAZAn/C9Jus7MRsysJekrkvYXUxaAsvXd1efuZ83sHkkH1O7q2+3ubxRWWQd1dg3V3Z3XVE0+LrfffnvdJTRarn5+d39O0nMF1QKgQlzeCwRF+IGgCD8QFOEHgiL8QFCEHwiq0vv5szS5z7jJBvW4NbkfPuuYlll71r5HRkYK2Q9nfiAowg8ERfiBoAg/EBThB4Ii/EBQ5u7V7cysup1doMndSnmldQ1l/d51dhPW+TcZ1O5RKb2rb3p6WrOzsz0N3c2ZHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCatQtvVkWc199HoN6XOq8bbZueX63tFl6LwZnfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IKlc/v5kdk3RG0ryks+4+lvb8oaEhjY+P59klOshzb3revvQ8ffXPPPNM6rr79u1Lbd+7d29q+/z8fGp7HovhGoQiLvL5nLv/rYDtAKgQL/uBoPKG3yUdNLOXzWxHEQUBqEbel/3j7j5tZldKet7M/t/dX1j4hOQ/hR2SdNlll+XcHYCi5Drzu/t08n1G0lOSNnR4zoS7j7n7WKvVyrM7AAXqO/xmdrmZfeL8Y0mbJR0pqjAA5crzsn+FpKfM7Px2fuHu/1tIVQBKV+m4/cPDw74Y+/kHeQz4vIaHh1PbDx482LXtlltuybXvO++8M7X9ySef7HvbTe7HT7ufn3H7AWQi/EBQhB8IivADQRF+ICjCDwQ1UEN3lylyd10eGzduTG1fvnx517ajR4+mrjs7O5vanqcrr8my/i2mTdF9MTjzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQld7Sa2bV7QyFWLp0aWp7Vp/0tdde2/e+d+7cmdp+4MCBvrc9yNL6+bmlF0Amwg8ERfiBoAg/EBThB4Ii/EBQhB8IqtL7+bOm6Oae+s7KHEY665ivW7cutX3Tpk2p7Wn37J87dy513aj9+FXhzA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQWX285vZbklfkjTj7uuSZcsl/VLSNZKOSbrL3f+et5g8/dlNvkagzume8x6XLVu2pLZnjb2f5sUXX+x7XeTXy5n/p5Juu2DZfZIOuft1kg4lPwMYIJnhd/cXJL13weKtkvYkj/dIuqPgugCUrN/3/Cvc/UTy+KSkFQXVA6AiuT/w8/YggF3H5jOzHWY2aWaTc3NzeXcHoCD9hv+Uma2UpOT7TLcnuvuEu4+5+1ir1epzdwCK1m/490vanjzeLunpYsoBUJXM8JvZXkl/kPSfZnbczO6WtEvSJjN7W9IXkp8BDJDMfn5339al6fMF15JLnX3pdSvzGodbb7011/rz8/Nd23btynfOiPo3n5qaKmQ7XOEHBEX4gaAIPxAU4QeCIvxAUIQfCKrSobtRjjxdXuvXr09tHx0dTW1fu3Ztavvs7GzXtquvvjp13ax25MOZHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCqrSf//Tp06Xdfhr19s68brzxxtT2rH78LIcPH861PsrDmR8IivADQRF+ICjCDwRF+IGgCD8QFOEHguJ+/uCuv/76XOtnTcH2+OOP59p+Hkzbno4zPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8EldnPb2a7JX1J0oy7r0uWPSDpa5LeTZ52v7s/V1aR6N/NN9+c2n7DDTfk2n5WP/+pU6dybR/l6eXM/1NJt3VY/iN3vyn5IvjAgMkMv7u/IOm9CmoBUKE87/nvMbPXzGy3mV1RWEUAKtFv+H8saa2kmySdkPSDbk80sx1mNmlmk33uC0AJ+gq/u59y93l3PyfpJ5I2pDx3wt3H3H2s3yIBFK+v8JvZygU/flnSkWLKAVCVXrr69kr6rKRPmdlxSd+V9Fkzu0mSSzom6esl1gigBJnhd/dtHRY/WkItudR573YT7s3uZnh4OLXdzHJt/6233sq1fh6L9X79rN9rZGSk720vxBV+QFCEHwiK8ANBEX4gKMIPBEX4gaAYursAWV0zdXYFbt68ObX96NGjqe1r1qxJbX/iiScuuqaiZB3XtL9Lk7tnq8KZHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCqrSff2hoSOPj413bm3yLZh55rwPIWv+qq67q2nbppZemrpvl8OHDqe1HjqSP45Lnb0pffLk48wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUNzP3wB5r29Iu3Yi79DcBw4cSG0v89qMJo+TsBhw5geCIvxAUIQfCIrwA0ERfiAowg8ERfiBoDL7+c1sjaTHJK2Q5JIm3P1BM1su6ZeSrpF0TNJd7v73PMWUOa3xYjY0NNT3umfOnEltf/jhh/vedpa8/fR5/uZNvoYga99TU1OF7KeXM/9ZSd9y91FJn5G008xGJd0n6ZC7XyfpUPIzgAGRGX53P+HurySPz0h6U9IqSVsl7UmetkfSHWUVCaB4F/We38yukbRe0h8lrXD3E0nTSbXfFgAYED2H38w+LulXkr7p7v9Y2OburvbnAZ3W22Fmk2Y2OTc3l6tYAMXpKfxmtlTt4P/c3X+dLD5lZiuT9pWSZjqt6+4T7j7m7mOtVquImgEUIDP81r4t7FFJb7r7Dxc07Ze0PXm8XdLTxZcHoCy93NL7X5K+Kul1M3s1WXa/pF2SnjCzuyX9RdJd5ZSILGm39GZZvXp1avvGjRtT2z/44IO+951X3iHPmyqr7pGRkUL2kxl+d/+9pG43hX++kCoAVI4r/ICgCD8QFOEHgiL8QFCEHwiK8ANBLZqhuwd5GOesft0lS5aktqf1+65duzZ13ZMnT6a219mPn6XJw4YPwjUGnPmBoAg/EBThB4Ii/EBQhB8IivADQRF+IKhF088/yLL6jLP6+fMM3T0z03EApp4NQn92J3n76Qf1916IMz8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBGXtmbYq2plZ6s4G+Z78Ol155ZVd2+69997UdR966KHU9kceeaSvmlCetPEbpqenNTs7222o/Q/hzA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQWX285vZGkmPSVohySVNuPuDZvaApK9Jejd56v3u/lzGtqq7qKBCZV+fkGcM+cVw3/kgKvPfxNTUVNe2i+nn72Uwj7OSvuXur5jZJyS9bGbPJ20/cvfv97IjAM2SGX53PyHpRPL4jJm9KWlV2YUBKNdFvec3s2skrZf0x2TRPWb2mpntNrMruqyzw8wmzWwyV6UACtVz+M3s45J+Jemb7v4PST+WtFbSTWq/MvhBp/XcfcLdx9x9rIB6ARSkp/Cb2VK1g/9zd/+1JLn7KXefd/dzkn4iaUN5ZQIoWmb4zcwkPSrpTXf/4YLlKxc87cuSjhRfHoCy9NLVNy7pd5Jel3QuWXy/pG1qv+R3ScckfT35cDBtW4uyqw+4WHm6Aivr6nP330vqtLHUPn0AzcYVfkBQhB8IivADQRF+ICjCDwRF+IGgKh26+5JLLvFVqwbznqDR0dG+181zS27Zyr7lN8/txmlDVKMzhu4GkInwA0ERfiAowg8ERfiBoAg/EBThB4KqeorudyX9ZcGiT0n6W2UFXJym1tbUuiRq61eRtV3t7p/u5YmVhv8jOzebbOrYfk2tral1SdTWr7pq42U/EBThB4KqO/wTNe8/TVNra2pdErX1q5baan3PD6A+dZ/5AdSklvCb2W1m9icze8fM7qujhm7M7JiZvW5mr9Y9xVgyDdqMmR1ZsGy5mT1vZm8n3ztOk1ZTbQ+Y2XRy7F41sy011bbGzH5rZlNm9oaZfSNZXuuxS6mrluNW+ct+M1si6S1JmyQdl/SSpG3u3n0w8gqZ2TFJY+5ee5+wmf23pH9Keszd1yXLvifpPXfflfzHeYW7f7shtT0g6Z91z9ycTCizcuHM0pLukPQ/qvHYpdR1l2o4bnWc+TdIesfd/+zuc5L2SdpaQx2N5+4vSHrvgsVbJe1JHu9R+x9P5brU1gjufsLdX0ken5F0fmbpWo9dSl21qCP8qyT9dcHPx9WsKb9d0kEze9nMdtRdTAcrFsyMdFLSijqL6SBz5uYqXTCzdGOOXT8zXheND/w+atzdb5b0RUk7k5e3jeTt92xN6q7paebmqnSYWfrf6jx2/c54XbQ6wj8tac2Cn1cnyxrB3aeT7zOSnlLzZh8+dX6S1OT7TM31/FuTZm7uNLO0GnDsmjTjdR3hf0nSdWY2YmYtSV+RtL+GOj7CzC5PPoiRmV0uabOaN/vwfknbk8fbJT1dYy0f0pSZm7vNLK2aj13jZrx298q/JG1R+xP/o5K+U0cNXeq6VtL/JV9v1F2bpL1qvwz8QO3PRu6W9ElJhyS9Lek3kpY3qLafqT2b82tqB21lTbWNq/2S/jVJryZfW+o+dil11XLcuMIPCIoP/ICgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBPUv5DLnMbZADooAAAAASUVORK5CYII=\",\n      \"text/plain\": [\n       \"<Figure size 432x288 with 1 Axes>\"\n      ]\n     },\n     \"metadata\": {}\n    }\n   ],\n   \"metadata\": {\n    \"collapsed\": false\n   }\n  }\n ],\n \"metadata\": {\n  \"kernelspec\": {\n   \"display_name\": \"Python 3\",\n   \"language\": \"python\",\n   \"name\": \"python3\"\n  },\n  \"language_info\": {\n   \"codemirror_mode\": {\n    \"name\": \"ipython\",\n    \"version\": 3\n   },\n   \"file_extension\": \".py\",\n   \"mimetype\": \"text/x-python\",\n   \"name\": \"python\",\n   \"nbconvert_exporter\": \"python\",\n   \"pygments_lexer\": \"ipython3\",\n   \"version\": \"3.6.4\"\n  }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}"
  },
  {
    "path": "example/bi-lstm-sort/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# Bidirectionnal LSTM to sort an array.\n\nThis is an example of using bidirectionmal lstm to sort an array. Please refer to the notebook.\n\nWe train a bidirectionnal LSTM to sort an array of integer.\n\nFor example:\n\n`500 30 999 10 130` should give us `10 30 130 500 999`\n\n![](https://cdn-images-1.medium.com/max/1200/1*6QnPUSv_t9BY9Fv8_aLb-Q.png)\n\n\n([Diagram source](http://colah.github.io/posts/2015-09-NN-Types-FP/))"
  },
  {
    "path": "example/bi-lstm-sort/bi-lstm-sort.ipynb",
    "content": "{\n \"cells\": [\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"# Licensed to the Apache Software Foundation (ASF) under one\\n\",\n    \"# or more contributor license agreements.  See the NOTICE file\\n\",\n    \"# distributed with this work for additional information\\n\",\n    \"# regarding copyright ownership.  The ASF licenses this file\\n\",\n    \"# to you under the Apache License, Version 2.0 (the\\n\",\n    \"# \\\"License\\\"); you may not use this file except in compliance\\n\",\n    \"# with the License.  You may obtain a copy of the License at\\n\",\n    \"#\\n\",\n    \"#   http://www.apache.org/licenses/LICENSE-2.0\\n\",\n    \"#\\n\",\n    \"# Unless required by applicable law or agreed to in writing,\\n\",\n    \"# software distributed under the License is distributed on an\\n\",\n    \"# \\\"AS IS\\\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\\n\",\n    \"# KIND, either express or implied.  See the License for the\\n\",\n    \"# specific language governing permissions and limitations\\n\",\n    \"# under the License.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"# Using a bi-lstm to sort a sequence of integers\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 1,\n   \"source\": [\n    \"import random\\n\",\n    \"import string\\n\",\n    \"\\n\",\n    \"import mxnet as mx\\n\",\n    \"from mxnet import gluon, np\\n\",\n    \"import numpy as onp\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Data Preparation\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 2,\n   \"source\": [\n    \"max_num = 999\\n\",\n    \"dataset_size = 60000\\n\",\n    \"seq_len = 5\\n\",\n    \"split = 0.8\\n\",\n    \"batch_size = 512\\n\",\n    \"ctx = mx.gpu() if mx.device.num_gpus() > 0 else mx.cpu()\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"We are getting a dataset of **dataset_size** sequences of integers of length **seq_len** between **0** and **max_num**. We use **split*100%** of them for training and the rest for testing.\\n\",\n    \"\\n\",\n    \"\\n\",\n    \"For example:\\n\",\n    \"\\n\",\n    \"50 10 200 999 30\\n\",\n    \"\\n\",\n    \"Should return\\n\",\n    \"\\n\",\n    \"10 30 50 200 999\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 3,\n   \"source\": [\n    \"X = mx.np.random.uniform(low=0, high=max_num, size=(dataset_size, seq_len)).astype('int32').asnumpy()\\n\",\n    \"Y = X.copy()\\n\",\n    \"Y.sort() #Let's sort X to get the target\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 4,\n   \"source\": [\n    \"print(\\\"Input {}\\\\nTarget {}\\\".format(X[0].tolist(), Y[0].tolist()))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Input [548, 592, 714, 843, 602]\\n\",\n      \"Target [548, 592, 602, 714, 843]\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"For the purpose of training, we encode the input as characters rather than numbers\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 5,\n   \"source\": [\n    \"vocab = string.digits + \\\" \\\"\\n\",\n    \"print(vocab)\\n\",\n    \"vocab_idx = { c:i for i,c in enumerate(vocab)}\\n\",\n    \"print(vocab_idx)\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"0123456789 \\n\",\n      \"{'0': 0, '1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8, '9': 9, ' ': 10}\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"We write a transform that will convert our numbers into text of maximum length **max_len**, and one-hot encode the characters.\\n\",\n    \"For example:\\n\",\n    \"\\n\",\n    \"\\\"30 10\\\" corresponding indices are [3, 0, 10, 1, 0]\\n\",\n    \"\\n\",\n    \"We then one hot encode that and get a matrix representation of our input. We don't need to encode our target as the loss we are going to use support sparse labels\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 6,\n   \"source\": [\n    \"max_len = len(str(max_num))*seq_len+(seq_len-1)\\n\",\n    \"print(\\\"Maximum length of the string: %s\\\" % max_len)\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Maximum length of the string: 19\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 7,\n   \"source\": [\n    \"def transform(x, y):\\n\",\n    \"    x_string = ' '.join(map(str, x.tolist()))\\n\",\n    \"    x_string_padded = x_string + ' '*(max_len-len(x_string))\\n\",\n    \"    x = [vocab_idx[c] for c in x_string_padded]\\n\",\n    \"    y_string = ' '.join(map(str, y.tolist()))\\n\",\n    \"    y_string_padded = y_string + ' '*(max_len-len(y_string))\\n\",\n    \"    y = [vocab_idx[c] for c in y_string_padded]\\n\",\n    \"    return mx.npx.one_hot(mx.nd.array(x), len(vocab)), mx.np.array(y)\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 8,\n   \"source\": [\n    \"split_idx = int(split*len(X))\\n\",\n    \"train_dataset = gluon.data.ArrayDataset(X[:split_idx], Y[:split_idx]).transform(transform)\\n\",\n    \"test_dataset = gluon.data.ArrayDataset(X[split_idx:], Y[split_idx:]).transform(transform)\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 9,\n   \"source\": [\n    \"print(\\\"Input {}\\\".format(X[0]))\\n\",\n    \"print(\\\"Transformed data Input {}\\\".format(train_dataset[0][0]))\\n\",\n    \"print(\\\"Target {}\\\".format(Y[0]))\\n\",\n    \"print(\\\"Transformed data Target {}\\\".format(train_dataset[0][1]))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Input [548 592 714 843 602]\\n\",\n      \"Transformed data Input \\n\",\n      \"[[0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\\n\",\n      \" [0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\\n\",\n      \" [0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0.]\\n\",\n      \" [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1.]\\n\",\n      \" [0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\\n\",\n      \" [0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0.]\\n\",\n      \" [0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]\\n\",\n      \" [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1.]\\n\",\n      \" [0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\\n\",\n      \" [0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n\",\n      \" [0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\\n\",\n      \" [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1.]\\n\",\n      \" [0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0.]\\n\",\n      \" [0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\\n\",\n      \" [0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\\n\",\n      \" [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1.]\\n\",\n      \" [0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\\n\",\n      \" [1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n\",\n      \" [0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]]\\n\",\n      \"<NDArray 19x11 @cpu(0)>\\n\",\n      \"Target [548 592 602 714 843]\\n\",\n      \"Transformed data Target \\n\",\n      \"[ 5.  4.  8. 10.  5.  9.  2. 10.  6.  0.  2. 10.  7.  1.  4. 10.  8.  4.\\n\",\n      \"  3.]\\n\",\n      \"<NDArray 19 @cpu(0)>\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 10,\n   \"source\": [\n    \"train_data = gluon.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=20, last_batch='rollover')\\n\",\n    \"test_data = gluon.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=5, last_batch='rollover')\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Creating the network\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 11,\n   \"source\": [\n    \"net = gluon.nn.HybridSequential()\\n\",\n    \"net.add(\\n\",\n    \"    gluon.rnn.LSTM(hidden_size=128, num_layers=2, layout='NTC', bidirectional=True),\\n\",\n    \"    gluon.nn.Dense(len(vocab), flatten=False)\\n\",\n    \")\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 12,\n   \"source\": [\n    \"net.initialize(mx.init.Xavier(), ctx=ctx)\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 13,\n   \"source\": [\n    \"loss = gluon.loss.SoftmaxCELoss()\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"We use a learning rate schedule to improve the convergence of the model\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 14,\n   \"source\": [\n    \"schedule = mx.lr_scheduler.FactorScheduler(step=len(train_data)*10, factor=0.75)\\n\",\n    \"schedule.base_lr = 0.01\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 15,\n   \"source\": [\n    \"trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate':0.01, 'lr_scheduler':schedule})\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Training loop\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 16,\n   \"source\": [\n    \"epochs = 100\\n\",\n    \"for e in range(epochs):\\n\",\n    \"    epoch_loss = 0.\\n\",\n    \"    for i, (data, label) in enumerate(train_data):\\n\",\n    \"        data = data.as_in_context(ctx)\\n\",\n    \"        label = label.as_in_context(ctx)\\n\",\n    \"\\n\",\n    \"        with mx.autograd.record():\\n\",\n    \"            output = net(data)\\n\",\n    \"            l = loss(output, label)\\n\",\n    \"\\n\",\n    \"        l.backward()\\n\",\n    \"        trainer.step(data.shape[0])\\n\",\n    \"    \\n\",\n    \"        epoch_loss += l.mean()\\n\",\n    \"        \\n\",\n    \"    print(\\\"Epoch [{}] Loss: {}, LR {}\\\".format(e, epoch_loss.item()/(i+1), trainer.learning_rate))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Epoch [0] Loss: 1.6627886372227823, LR 0.01\\n\",\n      \"Epoch [1] Loss: 1.210370733382854, LR 0.01\\n\",\n      \"Epoch [2] Loss: 0.9692377131035987, LR 0.01\\n\",\n      \"Epoch [3] Loss: 0.7976046623067653, LR 0.01\\n\",\n      \"Epoch [4] Loss: 0.5714595343476983, LR 0.01\\n\",\n      \"Epoch [5] Loss: 0.4458411196444897, LR 0.01\\n\",\n      \"Epoch [6] Loss: 0.36039798817736035, LR 0.01\\n\",\n      \"Epoch [7] Loss: 0.32665719377233626, LR 0.01\\n\",\n      \"Epoch [8] Loss: 0.262064205702915, LR 0.01\\n\",\n      \"Epoch [9] Loss: 0.22285924059279422, LR 0.0075\\n\",\n      \"Epoch [10] Loss: 0.19018426854559717, LR 0.0075\\n\",\n      \"Epoch [11] Loss: 0.1718730723604243, LR 0.0075\\n\",\n      \"Epoch [12] Loss: 0.15736752171670237, LR 0.0075\\n\",\n      \"Epoch [13] Loss: 0.14579375246737866, LR 0.0075\\n\",\n      \"Epoch [14] Loss: 0.13546599733068587, LR 0.0075\\n\",\n      \"Epoch [15] Loss: 0.12490207590955368, LR 0.0075\\n\",\n      \"Epoch [16] Loss: 0.11803316300915133, LR 0.0075\\n\",\n      \"Epoch [17] Loss: 0.10653189395336395, LR 0.0075\\n\",\n      \"Epoch [18] Loss: 0.10514750379197141, LR 0.0075\\n\",\n      \"Epoch [19] Loss: 0.09590611559279422, LR 0.005625\\n\",\n      \"Epoch [20] Loss: 0.08146028108494256, LR 0.005625\\n\",\n      \"Epoch [21] Loss: 0.07707348782965477, LR 0.005625\\n\",\n      \"Epoch [22] Loss: 0.07206193436967566, LR 0.005625\\n\",\n      \"Epoch [23] Loss: 0.07001185417175293, LR 0.005625\\n\",\n      \"Epoch [24] Loss: 0.06797058351578252, LR 0.005625\\n\",\n      \"Epoch [25] Loss: 0.0649358110224947, LR 0.005625\\n\",\n      \"Epoch [26] Loss: 0.06219124286732775, LR 0.005625\\n\",\n      \"Epoch [27] Loss: 0.06075144828634059, LR 0.005625\\n\",\n      \"Epoch [28] Loss: 0.05711334495134251, LR 0.005625\\n\",\n      \"Epoch [29] Loss: 0.054747099572039666, LR 0.00421875\\n\",\n      \"Epoch [30] Loss: 0.0441775271233092, LR 0.00421875\\n\",\n      \"Epoch [31] Loss: 0.041551097910454936, LR 0.00421875\\n\",\n      \"Epoch [32] Loss: 0.04095017269093503, LR 0.00421875\\n\",\n      \"Epoch [33] Loss: 0.04045371045457556, LR 0.00421875\\n\",\n      \"Epoch [34] Loss: 0.038867686657195394, LR 0.00421875\\n\",\n      \"Epoch [35] Loss: 0.038131744303601854, LR 0.00421875\\n\",\n      \"Epoch [36] Loss: 0.039834817250569664, LR 0.00421875\\n\",\n      \"Epoch [37] Loss: 0.03669035941996473, LR 0.00421875\\n\",\n      \"Epoch [38] Loss: 0.03373505967728635, LR 0.00421875\\n\",\n      \"Epoch [39] Loss: 0.03164981273894615, LR 0.0031640625\\n\",\n      \"Epoch [40] Loss: 0.025532766055035336, LR 0.0031640625\\n\",\n      \"Epoch [41] Loss: 0.022659448867148543, LR 0.0031640625\\n\",\n      \"Epoch [42] Loss: 0.02307056112492338, LR 0.0031640625\\n\",\n      \"Epoch [43] Loss: 0.02236944056571798, LR 0.0031640625\\n\",\n      \"Epoch [44] Loss: 0.022204211963120328, LR 0.0031640625\\n\",\n      \"Epoch [45] Loss: 0.02262336903430046, LR 0.0031640625\\n\",\n      \"Epoch [46] Loss: 0.02253308448385685, LR 0.0031640625\\n\",\n      \"Epoch [47] Loss: 0.025286573044797207, LR 0.0031640625\\n\",\n      \"Epoch [48] Loss: 0.02439300988310127, LR 0.0031640625\\n\",\n      \"Epoch [49] Loss: 0.017976388018181983, LR 0.002373046875\\n\",\n      \"Epoch [50] Loss: 0.014343131095805067, LR 0.002373046875\\n\",\n      \"Epoch [51] Loss: 0.013039355582379281, LR 0.002373046875\\n\",\n      \"Epoch [52] Loss: 0.011884741885687715, LR 0.002373046875\\n\",\n      \"Epoch [53] Loss: 0.011438189668858305, LR 0.002373046875\\n\",\n      \"Epoch [54] Loss: 0.011447292693117832, LR 0.002373046875\\n\",\n      \"Epoch [55] Loss: 0.014212571560068334, LR 0.002373046875\\n\",\n      \"Epoch [56] Loss: 0.019900493724371797, LR 0.002373046875\\n\",\n      \"Epoch [57] Loss: 0.02102568301748722, LR 0.002373046875\\n\",\n      \"Epoch [58] Loss: 0.01346214400961044, LR 0.002373046875\\n\",\n      \"Epoch [59] Loss: 0.010107964911359422, LR 0.0017797851562500002\\n\",\n      \"Epoch [60] Loss: 0.008353193600972494, LR 0.0017797851562500002\\n\",\n      \"Epoch [61] Loss: 0.007678258292218472, LR 0.0017797851562500002\\n\",\n      \"Epoch [62] Loss: 0.007262124660167288, LR 0.0017797851562500002\\n\",\n      \"Epoch [63] Loss: 0.00705223578087827, LR 0.0017797851562500002\\n\",\n      \"Epoch [64] Loss: 0.006788556293774677, LR 0.0017797851562500002\\n\",\n      \"Epoch [65] Loss: 0.006473606571238091, LR 0.0017797851562500002\\n\",\n      \"Epoch [66] Loss: 0.006206096486842378, LR 0.0017797851562500002\\n\",\n      \"Epoch [67] Loss: 0.00584477313021396, LR 0.0017797851562500002\\n\",\n      \"Epoch [68] Loss: 0.005648705267137097, LR 0.0017797851562500002\\n\",\n      \"Epoch [69] Loss: 0.006481769871204458, LR 0.0013348388671875003\\n\",\n      \"Epoch [70] Loss: 0.008430448618341, LR 0.0013348388671875003\\n\",\n      \"Epoch [71] Loss: 0.006877245421105242, LR 0.0013348388671875003\\n\",\n      \"Epoch [72] Loss: 0.005671108281740578, LR 0.0013348388671875003\\n\",\n      \"Epoch [73] Loss: 0.004832422162624116, LR 0.0013348388671875003\\n\",\n      \"Epoch [74] Loss: 0.004441103402604448, LR 0.0013348388671875003\\n\",\n      \"Epoch [75] Loss: 0.004216198591475791, LR 0.0013348388671875003\\n\",\n      \"Epoch [76] Loss: 0.004041922989711967, LR 0.0013348388671875003\\n\",\n      \"Epoch [77] Loss: 0.003937713643337818, LR 0.0013348388671875003\\n\",\n      \"Epoch [78] Loss: 0.010251983049068046, LR 0.0013348388671875003\\n\",\n      \"Epoch [79] Loss: 0.01829354052848004, LR 0.0010011291503906252\\n\",\n      \"Epoch [80] Loss: 0.006723233448561802, LR 0.0010011291503906252\\n\",\n      \"Epoch [81] Loss: 0.004397524798170049, LR 0.0010011291503906252\\n\",\n      \"Epoch [82] Loss: 0.0038475305476087206, LR 0.0010011291503906252\\n\",\n      \"Epoch [83] Loss: 0.003591177945441388, LR 0.0010011291503906252\\n\",\n      \"Epoch [84] Loss: 0.003425112014175743, LR 0.0010011291503906252\\n\",\n      \"Epoch [85] Loss: 0.0032633850549129728, LR 0.0010011291503906252\\n\",\n      \"Epoch [86] Loss: 0.0031762316505959693, LR 0.0010011291503906252\\n\",\n      \"Epoch [87] Loss: 0.0030452777096565734, LR 0.0010011291503906252\\n\",\n      \"Epoch [88] Loss: 0.002950224184220837, LR 0.0010011291503906252\\n\",\n      \"Epoch [89] Loss: 0.002821172171450676, LR 0.0007508468627929689\\n\",\n      \"Epoch [90] Loss: 0.002725780961361337, LR 0.0007508468627929689\\n\",\n      \"Epoch [91] Loss: 0.002660556359493986, LR 0.0007508468627929689\\n\",\n      \"Epoch [92] Loss: 0.0026011724946319414, LR 0.0007508468627929689\\n\",\n      \"Epoch [93] Loss: 0.0025355776256703317, LR 0.0007508468627929689\\n\",\n      \"Epoch [94] Loss: 0.0024825221997626283, LR 0.0007508468627929689\\n\",\n      \"Epoch [95] Loss: 0.0024245587435174497, LR 0.0007508468627929689\\n\",\n      \"Epoch [96] Loss: 0.002365282145879602, LR 0.0007508468627929689\\n\",\n      \"Epoch [97] Loss: 0.0023112583984719946, LR 0.0007508468627929689\\n\",\n      \"Epoch [98] Loss: 0.002257173682780976, LR 0.0007508468627929689\\n\",\n      \"Epoch [99] Loss: 0.002162747085094452, LR 0.0005631351470947267\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Testing\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"We get a random element from the testing set\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 17,\n   \"source\": [\n    \"n = random.randint(0, len(test_data)-1)\\n\",\n    \"\\n\",\n    \"x_orig = X[split_idx+n]\\n\",\n    \"y_orig = Y[split_idx+n]\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 41,\n   \"source\": [\n    \"def get_pred(x):\\n\",\n    \"    x, _ = transform(x, x)\\n\",\n    \"    output = net(mx.np.expand_dims(x.to_device(ctx), axis=0))\\n\",\n    \"\\n\",\n    \"    # Convert output back to string\\n\",\n    \"    pred = ''.join([vocab[int(o)] for o in output[0].argmax(axis=1).asnumpy().tolist()])\\n\",\n    \"    return pred\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"Printing the result\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 43,\n   \"source\": [\n    \"x_ = ' '.join(map(str,x_orig))\\n\",\n    \"label = ' '.join(map(str,y_orig))\\n\",\n    \"print(\\\"X         {}\\\\nPredicted {}\\\\nLabel     {}\\\".format(x_, get_pred(x_orig), label))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"X         611 671 275 871 944\\n\",\n      \"Predicted 275 611 671 871 944\\n\",\n      \"Label     275 611 671 871 944\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"We can also pick our own example, and the network manages to sort it without problem:\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 66,\n   \"source\": [\n    \"print(get_pred(onp.array([500, 30, 999, 10, 130])))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"10 30 130 500 999  \\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"The model has even learned to generalize to examples not on the training set\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 64,\n   \"source\": [\n    \"print(\\\"Only four numbers:\\\", get_pred(onp.array([105, 302, 501, 202])))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Only four numbers: 105 202 302 501    \\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"However we can see it has trouble with other edge cases:\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 63,\n   \"source\": [\n    \"print(\\\"Small digits:\\\", get_pred(onp.array([10, 3, 5, 2, 8])))\\n\",\n    \"print(\\\"Small digits, 6 numbers:\\\", get_pred(onp.array([10, 33, 52, 21, 82, 10])))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Small digits: 8  0 42 28         \\n\",\n      \"Small digits, 6 numbers: 10 0 20 82 71 115  \\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"This could be improved by adjusting the training dataset accordingly\"\n   ],\n   \"metadata\": {}\n  }\n ],\n \"metadata\": {\n  \"kernelspec\": {\n   \"display_name\": \"Python 3\",\n   \"language\": \"python\",\n   \"name\": \"python3\"\n  },\n  \"language_info\": {\n   \"codemirror_mode\": {\n    \"name\": \"ipython\",\n    \"version\": 3\n   },\n   \"file_extension\": \".py\",\n   \"mimetype\": \"text/x-python\",\n   \"name\": \"python\",\n   \"nbconvert_exporter\": \"python\",\n   \"pygments_lexer\": \"ipython3\",\n   \"version\": \"3.6.4\"\n  }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}"
  },
  {
    "path": "example/distributed_training/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# Distributed Training using Gluon\n\nDeep learning models are usually trained using GPUs because GPUs can do a lot more computations in parallel that CPUs. But even with the modern GPUs, it could take several days to train big models. Training can be done faster by using multiple GPUs like described in [this](https://gluon.mxnet.io/chapter07_distributed-learning/multiple-gpus-gluon.html) tutorial. However only a certain number of GPUs can be attached to one host (typically 8 or 16). To make the training even faster, we can use multiple GPUs attached to multiple hosts.\n\nIn this tutorial, we will show how to train a model faster using multi-host distributed training.\n\n![Multiple GPUs connected to multiple hosts](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/distributed_training/distributed_training.png)\n\nWe will use data parallelism to distribute the training which involves splitting the training data across GPUs attached to multiple hosts. Since the hosts are working with different subset of the training data in parallel, the training completes a lot faster.\n\nIn this tutorial, we will train a ResNet18 network using CIFAR-10 dataset using two hosts each having four GPUs.\n\n## Distributed Training Architecture:\n\nMultihost distributed training involves working with three different types of processes - worker, parameter server and scheduler.\n\n![Distributed training architecture](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/distributed_training/dist_train_arch.png)\n\n### Parameter Server:\nThe parameters of the model needs to be shared with all hosts since multiple hosts are working together to train one model. To make this sharing efficient, the parameters are split across multiple hosts. A parameter server in each host stores a subset of parameters. In the figure above, parameters are split evenly between the two hosts. At the end of every iteration, each host communicates with every other host to update all parameters of the model.\n\n### Worker:\nEach host has a worker process which in each iteration fetches a batch of data, runs forward and backward pass on all GPUs in the host, computes the parameter updates and sends those updates to the parameter servers in each host. Since we have multiple workers to train the model, each worker only needs to process 1/N part of the training data where N is the number of workers.\n\n### Scheduler:\nScheduler is responsible for scheduling the workers and parameter servers. There is only one scheduler in the entire cluster.\n\n## Moving to distributed training:\n\n[cifar10_dist.py](cifar10_dist.py) contains code that trains a ResNet18 network using distributed training. In this section we'll walk through parts of that file that are unique to distributed training.\n\n### Step 1: Use a distributed key-value store:\n\nLike mentioned above, in distributed training, parameters are split into N parts and distributed across N hosts. This is done automatically by the [distributed key-value store](https://mxnet.apache.org/tutorials/python/kvstore.html). User only needs to create the distributed kv store and ask the `Trainer` to use the created store.\n\n```python\nstore = mxnet.kv.create('dist')\n```\n\nIt is the job of the trainer to take the gradients computed in the backward pass and update the parameters of the model. We'll tell the trainer to store and update the parameters in the distributed kv store we just created instead of doing it in GPU of CPU memory. For example,\n\n```python\ntrainer = gluon.Trainer(net.collect_params(),\n                        'adam', {'learning_rate': .001},\n                        kvstore=store)\n```\n\n## Step 2: Split the training data:\n\nIn distributed training (using data parallelism), training data is split into equal parts across all workers and each worker uses its subset of the training data for training. For example, if we had two machines, each running a worker, each worker managing four GPUs we'll split the data like shown below. Note that we don't split the data depending on the number of GPUs but split it depending on the number of workers.\n\n![Splitting data](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/distributed_training/split_data.png)\n\nEach worker can find out the total number of workers in the cluster and its own rank which is an integer between 0 and N-1 where N is the number of workers.\n\n```python\nstore = kv.create('dist')\nprint(\"Total number of workers: %d\" % store.num_workers)\nprint(\"This worker's rank: %d\" % store.rank)\n```\n\n```\nTotal number of workers: 2\nThis worker's rank: 0\n```\n\nKnowing the number of workers and a particular worker's rank, it is easy to split the dataset into partitions and pick one partition to train depending on the rank of the worker. Here is a sampler that does exactly that.\n\n```python\nclass SplitSampler(gluon.data.sampler.Sampler):\n    \"\"\" Split the dataset into `num_parts` parts and sample from the part with index `part_index`\n    Parameters\n    ----------\n    length: int\n      Number of examples in the dataset\n    num_parts: int\n      Partition the data into multiple parts\n    part_index: int\n      The index of the part to read from\n    \"\"\"\n    def __init__(self, length, num_parts=1, part_index=0):\n        # Compute the length of each partition\n        self.part_len = length // num_parts\n        # Compute the start index for this partition\n        self.start = self.part_len * part_index\n        # Compute the end index for this partition\n        self.end = self.start + self.part_len\n\n    def __iter__(self):\n        # Extract examples between `start` and `end`, shuffle and return them.\n        indices = list(range(self.start, self.end))\n        random.shuffle(indices)\n        return iter(indices)\n\n    def __len__(self):\n        return self.part_len\n```\n\nWe can then create a `DataLoader` using the `SplitSampler` like shown below:\n\n```python\n# Load the training data\ntrain_data = gluon.data.DataLoader(gluon.data.vision.CIFAR10(train=True).transform(transform),\n                                      batch_size,\n                                      sampler=SplitSampler(50000, store.num_workers, store.rank))\n```\n\n## Step 3: Training with multiple GPUs\n\nNote that we didn't split the dataset by the number of GPUs. We split it by the number of workers which usually translates to number of machines. It is the worker's responsibility to split the partition it has across multiple GPUs it might have and run the training in parallel across multiple GPUs.\n\nTo train with multiple GPUs, we first need to specify the list of GPUs we want to use for training:\n\n```python\nctx = [mx.gpu(i) for i in range(gpus_per_machine)]\n```\n\nWe can then train a batch like shown below:\n\n```python\n# Train a batch using multiple GPUs\ndef train_batch(batch, ctx, net, trainer):\n\n    # Split and load data into multiple GPUs\n    data = batch[0]\n    data = gluon.utils.split_and_load(data, ctx)\n\n    # Split and load label into multiple GPUs\n    label = batch[1]\n    label = gluon.utils.split_and_load(label, ctx)\n\n    # Run the forward and backward pass\n    forward_backward(net, data, label)\n\n    # Update the parameters\n    this_batch_size = batch[0].shape[0]\n    trainer.step(this_batch_size)\n```\n\nHere is the code that runs the forward (computing loss) and backward (computing gradients) pass on multiple GPUs:\n\n```python\n# We'll use cross entropy loss since we are doing multiclass classification\nloss = gluon.loss.SoftmaxCrossEntropyLoss()\n\n# Run one forward and backward pass on multiple GPUs\ndef forward_backward(net, data, label):\n\n    # Ask autograd to remember the forward pass\n    with autograd.record():\n        # Compute the loss on all GPUs\n        losses = [loss(net(X), Y) for X, Y in zip(data, label)]\n\n    # Run the backward pass (calculate gradients) on all GPUs\n    for l in losses:\n        l.backward()\n```\n\nGiven `train_batch`, training an epoch is simple:\n\n```python\nfor batch in train_data:\n    # Train the batch using multiple GPUs\n    train_batch(batch, ctx, net, trainer)\n```\n\n## Final Step: Launching the distributed training\n\nNote that there are several processes that needs to be launched on multiple machines to do distributed training. One worker and one parameter server needs to be launched on each host. Scheduler needs to be launched on one of the hosts. While this can be done manually, MXNet provides the [`launch.py`](https://github.com/apache/mxnet/blob/master/tools/launch.py) tool to make this easy.\n\nFor example, the following command launches distributed training on two machines:\n\n```\npython ~/mxnet/tools/launch.py -n 2 -s 2 -H hosts \\\n    --sync-dst-dir /home/ubuntu/cifar10_dist \\\n    --launcher ssh \\\n    \"python /home/ubuntu/cifar10_dist/cifar10_dist.py\"\n```\n\n- `-n 2` specifies the number of workers that must be launched\n- `-s 2` specifies the number of parameter servers that must be launched.\n- `--sync-dst-dir` specifies a destination location where the contents of the current directory will be rsync'd\n- `--launcher ssh` tells `launch.py` to use ssh to login on each machine in the cluster and launch processes.\n- `\"python /home/ubuntu/dist/dist.py\"` is the command that will get executed in each of the launched processes.\n- Finally, `-H hosts` specifies the list of hosts in the cluster to be used for distributed training.\n\nLet's take a look at the `hosts` file.\n\n```\n~/dist$ cat hosts\nd1\nd2\n```\n\n'd1' and 'd2' are the hostnames of the hosts we want to run distributed training using. `launch.py` should be able to ssh into these hosts by providing just the hostname on the command line. For example:\n\n```\n~/dist$ ssh d1\nWelcome to Ubuntu 16.04.3 LTS (GNU/Linux 4.4.0-1049-aws x86_64)\n\n * Documentation:  https://help.ubuntu.com\n * Management:     https://landscape.canonical.com\n * Support:        https://ubuntu.com/advantage\n\n  Get cloud support with Ubuntu Advantage Cloud Guest:\n    http://www.ubuntu.com/business/services/cloud\n\n0 packages can be updated.\n0 updates are security updates.\n\n\nLast login: Wed Jan 31 18:06:45 2018 from 72.21.198.67\n```\n\nNote that no authentication information was provided to login to the host. This can be done using multiple methods. One easy way is to specify the ssh certificates in `~/.ssh/config`. Example:\n\n```\n~$ cat ~/.ssh/config\nHost d1\n    HostName ec2-34-201-108-233.compute-1.amazonaws.com\n    port 22\n    user ubuntu\n    IdentityFile /home/ubuntu/my_key.pem\n    IdentitiesOnly yes\n\nHost d2\n    HostName ec2-34-238-232-97.compute-1.amazonaws.com\n    port 22\n    user ubuntu\n    IdentityFile /home/ubuntu/my_key.pem\n    IdentitiesOnly yes\n```\n\nA better way is to use ssh agent forwarding. Check [this](https://aws.amazon.com/blogs/security/securely-connect-to-linux-instances-running-in-a-private-amazon-vpc/) article for more details.\n\nHere is a sample output from running distributed training:\n\n```\n$ python ~/mxnet/tools/launch.py -n 2 -s 2 -H hosts --sync-dst-dir /home/ubuntu/cifar10_dist --launcher ssh \"python /home/ubuntu/cifar10_dist/cifar10_dist.py\"\n2018-06-03 05:30:05,609 INFO rsync /home/ubuntu/cifar10_dist/ -> a1:/home/ubuntu/cifar10_dist\n2018-06-03 05:30:05,879 INFO rsync /home/ubuntu/cifar10_dist/ -> a2:/home/ubuntu/cifar10_dist\nEpoch 0: Test_acc 0.467400\nEpoch 0: Test_acc 0.466800\nEpoch 1: Test_acc 0.568500\nEpoch 1: Test_acc 0.571300\nEpoch 2: Test_acc 0.586300\nEpoch 2: Test_acc 0.594000\nEpoch 3: Test_acc 0.659200\nEpoch 3: Test_acc 0.653300\nEpoch 4: Test_acc 0.681200\nEpoch 4: Test_acc 0.687900\n```\n\nNote that the output from all hosts are merged and printed to the console.\n"
  },
  {
    "path": "example/distributed_training/cifar10_dist.py",
    "content": "#!/usr/bin/env python\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"cifar10_dist.py contains code that trains a ResNet18 network using distributed training\"\"\"\n\nfrom __future__ import print_function\n\nimport sys\nimport random\nimport numpy as np\nimport mxnet as mx\nfrom mxnet import autograd, gluon, kv, nd\nfrom mxnet.gluon.model_zoo import vision\n\n# Create a distributed key-value store\nstore = kv.create('dist')\n\n# Clasify the images into one of the 10 digits\nnum_outputs = 10\n\n# 64 images in a batch\nbatch_size_per_gpu = 64\n# How many epochs to run the training\nepochs = 5\n\n# How many GPUs per machine\ngpus_per_machine = 4\n# Effective batch size across all GPUs\nbatch_size = batch_size_per_gpu * gpus_per_machine\n\n# Create the context (a list of all GPUs to be used for training)\nctx = [mx.gpu(i) for i in range(gpus_per_machine)]\n\n\n# Convert to float 32\n# Having channel as the first dimension makes computation more efficient. Hence the (2,0,1) transpose.\n# Dividing by 255 normalizes the input between 0 and 1\ndef transform(data, label):\n    return nd.transpose(data.astype(np.float32), (2, 0, 1))/255, label.astype(np.float32)\n\n\nclass SplitSampler(gluon.data.sampler.Sampler):\n    \"\"\" Split the dataset into `num_parts` parts and sample from the part with index `part_index`\n\n    Parameters\n    ----------\n    length: int\n      Number of examples in the dataset\n    num_parts: int\n      Partition the data into multiple parts\n    part_index: int\n      The index of the part to read from\n    \"\"\"\n    def __init__(self, length, num_parts=1, part_index=0):\n        # Compute the length of each partition\n        self.part_len = length // num_parts\n        # Compute the start index for this partition\n        self.start = self.part_len * part_index\n        # Compute the end index for this partition\n        self.end = self.start + self.part_len\n\n    def __iter__(self):\n        # Extract examples between `start` and `end`, shuffle and return them.\n        indices = list(range(self.start, self.end))\n        random.shuffle(indices)\n        return iter(indices)\n\n    def __len__(self):\n        return self.part_len\n\n\n# Load the training data\ntrain_data = gluon.data.DataLoader(gluon.data.vision.CIFAR10(train=True).transform(transform), batch_size,\n                                   sampler=SplitSampler(50000, store.num_workers, store.rank))\n\n# Load the test data\ntest_data = gluon.data.DataLoader(gluon.data.vision.CIFAR10(train=False).transform(transform),\n                                  batch_size, shuffle=False)\n\n# Use ResNet from model zoo\nnet = vision.resnet18_v1()\n\n# Initialize the parameters with Xavier initializer\nnet.collect_params().initialize(mx.init.Xavier(), ctx=ctx)\n\n# SoftmaxCrossEntropy is the most common choice of loss function for multiclass classification\nsoftmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()\n\n# Use Adam optimizer. Ask trainer to use the distributor kv store.\ntrainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': .001}, kvstore=store)\n\n\n# Evaluate accuracy of the given network using the given data\ndef evaluate_accuracy(data_iterator, network):\n    \"\"\" Measure the accuracy of ResNet\n\n    Parameters\n    ----------\n    data_iterator: Iter\n      examples of dataset\n    network:\n      ResNet\n\n    Returns\n    ----------\n    tuple of array element\n    \"\"\"\n    acc = mx.gluon.metric.Accuracy()\n\n    # Iterate through data and label\n    for i, (data, label) in enumerate(data_iterator):\n\n        # Get the data and label into the GPU\n        data = data.as_in_context(ctx[0])\n        label = label.as_in_context(ctx[0])\n\n        # Get network's output which is a probability distribution\n        # Apply argmax on the probability distribution to get network's classification.\n        output = network(data)\n        predictions = nd.argmax(output, axis=1)\n\n        # Give network's prediction and the correct label to update the metric\n        acc.update(preds=predictions, labels=label)\n\n    # Return the accuracy\n    return acc.get()[1]\n\n\n# We'll use cross entropy loss since we are doing multiclass classification\nloss = gluon.loss.SoftmaxCrossEntropyLoss()\n\n\n# Run one forward and backward pass on multiple GPUs\ndef forward_backward(network, data, label):\n\n    # Ask autograd to remember the forward pass\n    with autograd.record():\n        # Compute the loss on all GPUs\n        losses = [loss(network(X), Y) for X, Y in zip(data, label)]\n\n    # Run the backward pass (calculate gradients) on all GPUs\n    for l in losses:\n        l.backward()\n\n\n# Train a batch using multiple GPUs\ndef train_batch(batch_list, context, network, gluon_trainer):\n    \"\"\" Training with multiple GPUs\n\n    Parameters\n    ----------\n    batch_list: List\n      list of dataset\n    context: List\n      a list of all GPUs to be used for training\n    network:\n      ResNet\n    gluon_trainer:\n      rain module of gluon\n    \"\"\"\n    # Split and load data into multiple GPUs\n    data = batch_list[0]\n    data = gluon.utils.split_and_load(data, context)\n\n    # Split and load label into multiple GPUs\n    label = batch_list[1]\n    label = gluon.utils.split_and_load(label, context)\n\n    # Run the forward and backward pass\n    forward_backward(network, data, label)\n\n    # Update the parameters\n    this_batch_size = batch_list[0].shape[0]\n    gluon_trainer.step(this_batch_size)\n\n\n# Run as many epochs as required\nfor epoch in range(epochs):\n\n    # Iterate through batches and run training using multiple GPUs\n    batch_num = 1\n    for batch in train_data:\n\n        # Train the batch using multiple GPUs\n        train_batch(batch, ctx, net, trainer)\n\n        batch_num += 1\n\n    # Print test accuracy after every epoch\n    test_accuracy = evaluate_accuracy(test_data, net)\n    print(f\"Epoch {epoch}: Test_acc {test_accuracy}\")\n    sys.stdout.flush()\n"
  },
  {
    "path": "example/distributed_training/cifar10_kvstore_hvd.py",
    "content": "#!/usr/bin/env python\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"cifar10_dist_hvd.py contains code that runs distributed training of a\nResNet18 network using Horovod framework\"\"\"\n\nimport argparse\nimport logging\nimport time\nimport random\nimport types\nimport warnings\n\nimport numpy as np\nimport mxnet as mx\nfrom mxnet import autograd, gluon, kv, nd\nfrom mxnet.gluon.model_zoo import vision\n\nlogging.basicConfig(level=logging.INFO)\n\n# Training settings\nparser = argparse.ArgumentParser(description='MXNet CIFAR Example')\n\nparser.add_argument('--batch-size', type=int, default=64,\n                    help='training batch size per worker (default: 64)')\nparser.add_argument('--epochs', type=int, default=5,\n                    help='number of training epochs (default: 5)')\nparser.add_argument('--lr', type=float, default=0.01,\n                    help='learning rate (default: 0.01)')\nparser.add_argument('--no-cuda', action='store_true', default=False,\n                    help='disable training on GPU (default: False)')\nargs = parser.parse_args()\n\nif not args.no_cuda:\n    # Disable CUDA if there are no GPUs.\n    if mx.device.num_gpus() == 0:\n        args.no_cuda = True\n\n\n# Transform input data\ndef transform(data, label):\n    return nd.transpose(data.astype(np.float32), (2, 0, 1))/255,\\\n      label.astype(np.float32)\n\n\n# Train a batch using multiple GPUs\ndef train(batch_list, context, network, gluon_trainer, metric):\n    \"\"\" Training with multiple GPUs\n\n    Parameters\n    ----------\n    batch_list: List\n      list of dataset\n    context: List\n      a list of all GPUs to be used for training\n    network:\n      ResNet\n    gluon_trainer:\n      rain module of gluon\n    \"\"\"\n\n    # Run one forward and backward pass\n    def forward_backward(network, data, labels, metric):\n        with autograd.record():\n            # Compute outputs\n            outputs = [network(X) for X in data]\n            # Compute the loss\n            losses = [loss(yhat, y) for yhat, y in zip(outputs, labels)]\n\n        # Run the backward pass (calculate gradients)\n        for l in losses:\n            l.backward()\n\n        metric.update(preds=outputs, labels=labels)\n\n    # Use cross entropy loss\n    loss = gluon.loss.SoftmaxCrossEntropyLoss()\n\n    # Split and load data\n    data = batch_list[0]\n    data = gluon.utils.split_and_load(data, context)\n\n    # Split and load label\n    label = batch_list[1]\n    label = gluon.utils.split_and_load(label, context)\n\n    # Run the forward and backward pass\n    forward_backward(network, data, label, metric)\n\n    # Update the parameters\n    this_batch_size = batch_list[0].shape[0]\n    gluon_trainer.step(this_batch_size)\n\n\n# Evaluate accuracy of the given network using the given data\ndef evaluate(data_iterator, network, context):\n    \"\"\" Measure the accuracy of ResNet\n\n    Parameters\n    ----------\n    data_iterator: Iter\n      examples of dataset\n    network:\n      ResNet\n\n    Returns\n    ----------\n    tuple of array element\n    \"\"\"\n    acc = mx.gluon.metric.Accuracy()\n\n    # Iterate through data and label\n    for i, (data, label) in enumerate(data_iterator):\n\n        # Get the data and label into the GPU\n        data = data.as_in_context(context)\n        label = label.as_in_context(context)\n\n        # Get network's output which is a probability distribution\n        # Apply argmax on the probability distribution to get network's\n        # classification.\n        output = network(data)\n        predictions = nd.argmax(output, axis=1)\n\n        # Give network's prediction and the correct label to update the metric\n        acc.update(preds=predictions, labels=label)\n\n    # Return the accuracy\n    return acc.get()[1]\n\n\nclass SplitSampler(gluon.data.sampler.Sampler):\n    \"\"\" Split the dataset into `num_parts` parts and sample from the part with\n    index `part_index`\n\n    Parameters\n    ----------\n    length: int\n      Number of examples in the dataset\n    num_parts: int\n      Partition the data into multiple parts\n    part_index: int\n      The index of the part to read from\n    \"\"\"\n    def __init__(self, length, num_parts=1, part_index=0):\n        # Compute the length of each partition\n        self.part_len = length // num_parts\n        # Compute the start index for this partition\n        self.start = self.part_len * part_index\n        # Compute the end index for this partition\n        self.end = self.start + self.part_len\n\n    def __iter__(self):\n        # Extract examples between `start` and `end`, shuffle and return them.\n        indices = list(range(self.start, self.end))\n        random.shuffle(indices)\n        return iter(indices)\n\n    def __len__(self):\n        return self.part_len\n\n\n# Use Horovod as the KVStore\nstore = kv.create('horovod')\n\n# Get the number of workers\nnum_workers = store.num_workers\n\n# Create the context based on the local rank of the current process\nctx = mx.cpu(store.local_rank) if args.no_cuda else mx.gpu(store.local_rank)\n\n# Load the training data\ntrain_data = gluon.data.DataLoader(gluon.data.vision.CIFAR10(train=True,\n                                   transform=transform), args.batch_size,\n                                   sampler=SplitSampler(50000,\n                                                        num_workers,\n                                                        store.rank))\n\n# Load the test data\ntest_data = gluon.data.DataLoader(gluon.data.vision.CIFAR10(train=False,\n                                  transform=transform),\n                                  args.batch_size, shuffle=False)\n\n# Load ResNet18 model from GluonCV model zoo\nnet = vision.resnet18_v1()\n\n# Initialize the parameters with Xavier initializer\nnet.initialize(mx.init.Xavier(), ctx=ctx)\n\n# Use Adam optimizer. Ask trainer to use the distributor kv store.\ntrainer = gluon.Trainer(net.collect_params(), optimizer='adam',\n                        optimizer_params={'learning_rate': args.lr},\n                        kvstore=store)\n\ntrain_metric = mx.gluon.metric.Accuracy()\n\n# Run as many epochs as required\nfor epoch in range(args.epochs):\n    tic = time.time()\n    train_metric.reset()\n\n    # Iterate through batches and run training using multiple GPUs\n    batch_num = 1\n    btic = time.time()\n    for batch in train_data:\n        # Train the batch using multiple GPUs\n        train(batch, [ctx], net, trainer, train_metric)\n        if store.rank == 0 and batch_num % 100 == 0:\n            speed = args.batch_size / (time.time() - btic)\n            logging.info('Epoch[{}] Rank [{}] Batch[{}]\\tSpeed: {:.2f} samples/sec'\n                         .format(epoch, store.rank, batch_num, speed))\n            logging.info('{} = {:.2f}'.format(*train_metric.get()))\n\n        btic = time.time()\n        batch_num += 1\n\n    elapsed = time.time() - tic\n    # Print test accuracy after every epoch\n    test_accuracy = evaluate(test_data, net, ctx)\n    if store.rank == 0:\n        logging.info(f\"Epoch {epoch}: Test_acc {test_accuracy}\")"
  },
  {
    "path": "example/distributed_training-horovod/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# Distributed Training using MXNet with Horovod \n[Horovod](https://github.com/horovod/horovod) is a distributed training framework that demonstrates \nexcellent scaling efficiency for dense models running on a large number of nodes. It currently \nsupports mainstream deep learning frameworks such as MXNet, TensorFlow, Keras, and PyTorch. \nIt is created at Uber and currently hosted by the [Linux Foundation Deep Learning](https://lfdl.io)(LF DL). \n\nMXNet is supported starting from Horovod 0.16.0 [release](https://eng.uber.com/horovod-pyspark-apache-mxnet-support/).\n\n## What's New?\nCompared with the standard distributed training script in MXNet which uses parameter server to \ndistribute and aggregate parameters, Horovod uses ring allreduce and/or tree-based allreduce algorithm \nto communicate parameters between workers. There is no dedicated server and the communication data size \nbetween workers does not depend on the number of workers. Therefore, it scales well in the case where \nthere are a large number of workers and network bandwidth is the bottleneck.\n\n# Setup\n\n## Install MXNet\n```bash\n$ pip install mxnet\n```\n**Note**: The [known issue](https://github.com/horovod/horovod/issues/884) when running Horovod with MXNet on a Linux system with GCC version 5.X and above has been resolved. Please use MXNet 1.4.1 or later releases with Horovod 0.16.2 or later releases to avoid the GCC incompatibility issue. MXNet 1.4.0 release works with Horovod 0.16.0 and 0.16.1 releases with the GCC incompatibility issue unsolved.\n\n## Install Horovod\n```bash\n$ pip install horovod\n```\n\nThis basic installation is good for laptops and for getting to know Horovod.\nIf you're installing Horovod on a server with GPUs, read the [Horovod on GPU](https://github.com/horovod/horovod/blob/master/docs/gpus.rst) page.\nIf you want to use Docker, read the [Horovod in Docker](https://github.com/horovod/horovod/blob/master/docs/docker.rst) page.\n\n## Install MPI\nMPI is required to run distributed training with Horovod. Install [Open MPI](https://www.open-mpi.org/) or another MPI implementation.\nSteps to install Open MPI are listed [here](https://www.open-mpi.org/faq/?category=building#easy-build).\n\n**Note**: Open MPI 3.1.3 has an issue that may cause hangs.  It is recommended\nto downgrade to Open MPI 3.1.2 or upgrade to Open MPI 4.0.0.\n\n## On Kubernetes\n\nDistributed MXNet jobs with Horovod can be submitted to a Kubernetes cluster via [Kubeflow MPI Operator](https://github.com/kubeflow/mpi-operator). Please refer to [this example](https://github.com/kubeflow/mpi-operator/tree/master/examples/mxnet) for details, including the Dockerfile with all the dependencies mentioned in previous sections, distributed training Python script based on Horovod, and the YAML configuration file that can be used for submitting a job on a Kubernetes cluster.\n\n# Usage\n\nTo run MXNet with Horovod, make the following additions to your training script:\n\n1. Run `hvd.init()`.\n\n2. Pin the context to a processor using `hvd.local_rank()`.\n    Typically, each Horovod worker is associated with one process. The local rank is a unique ID specifically\n    for all processes running Horovod job on the same node.\n\n3. Scale the learning rate by number of workers. Effective batch size in synchronous distributed training is scaled by\n    the number of workers. An increase in learning rate compensates for the increased batch size.\n\n4. Create `hvd.DistributedTrainer` with optimizer when using Gluon API.  The distributed trainer or optimizer delegates gradient computation\n    to the original optimizer, averages gradients using *allreduce*, and then applies those averaged\n    gradients.\n\n5. Add `hvd.broadcast_parameters` to broadcast initial variable states from rank 0 to all other processes.\n    This is necessary to ensure consistent initialization of all workers when training is started with random weights or\n    restored from a checkpoint. \n\n# Example\n\nHere we provide the building blocks to train a model using MXNet with Horovod.\nThe full examples are in [MNIST](gluon_mnist.py) and [ImageNet](resnet50_imagenet.py).\n\n## Gluon API\n```python\nfrom mxnet import autograd, gluon\nimport mxnet as mx\nimport horovod.mxnet as hvd\n\n# Initialize Horovod\nhvd.init()\n\n# Set context to current process \ncontext = mx.cpu(hvd.local_rank()) if args.no_cuda else mx.gpu(hvd.local_rank())\n\nnum_workers = hvd.size()\n\n# Build model\nmodel = ...\nmodel.hybridize()\n\n\n# Create optimizer\noptimizer_params = ...\nopt = mx.optimizer.create('sgd', **optimizer_params)\n\n# Create DistributedTrainer, a subclass of gluon.Trainer\ntrainer = hvd.DistributedTrainer(params, opt)\n\n# Initialize parameters\nmodel.initialize(initializer, ctx=context)\n\n# Fetch and broadcast parameters\nparams = model.collect_params()\nif params is not None:\n    hvd.broadcast_parameters(params, root_rank=0)\n\n# Create loss function\nloss_fn = ...\n\n# Train model\nfor epoch in range(num_epoch):\n    train_data.reset()\n    for nbatch, batch in enumerate(train_data, start=1):\n        data = batch.data[0].as_in_context(context)\n        label = batch.label[0].as_in_context(context)\n        with autograd.record():\n            output = model(data.astype(dtype, copy=False))\n            loss = loss_fn(output, label)\n        loss.backward()\n        trainer.step(batch_size)\n```\n\n\n# Running Horovod\n\nThe example commands below show how to run distributed training. See the \n[Running Horovod](https://github.com/horovod/horovod/blob/master/docs/running.rst)\npage for more instructions.\n\n1. To run on a machine with 4 CPUs:\n\n```bash\n$ mpirun -np 4 \\\n    -H localhost:4 \\\n    -bind-to none -map-by slot \\\n    python train.py\n```\n\n2. To run on 2 machines with 4 GPUs each:\n\n```bash\n$ mpirun -np 8 \\\n    -H server1:4,server2:4 \\\n    -bind-to none -map-by slot \\\n    -x NCCL_DEBUG=INFO \\\n    -mca pml ob1 -mca btl ^openib \\\n    python train.py\n```\n\n## Tuning Horovod Performance\n\n1. To analyse horovod performance, [horovod timeline](https://github.com/horovod/horovod/blob/master/docs/timeline.rst) is a handy tool to trace and visualize the time spent on horovod operations. \n\n2. A few tuning knobs affect horovod runtime performance (explained [here](https://github.com/horovod/horovod/blob/master/docs/tensor-fusion.rst)). Apart from `HOROVOD_FUSION_THRESHOLD`, sometimes we find increasing `HOROVOD_CYCLE_TIME` (up to 100 ms), changing [`NCCL_ALGO`](https://docs.nvidia.com/deeplearning/sdk/nccl-developer-guide/docs/env.html#nccl-algo), and [`NCCL_MIN_NCHANNELS`](https://docs.nvidia.com/deeplearning/sdk/nccl-developer-guide/docs/env.html#nccl-min-nchannels) improves performance.\n\n3. If you are running horovod on AWS, you can potentially leverage [EFA](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa.html) if your instance supports 100 Gb/s networking. To use EFA, you can refer to the [official documentation](https://docs.aws.amazon.com/eu_us/AWSEC2/latest/UserGuide/efa-start-nccl-dlami.html) for the setup instructions, and the environment variables (`-x FI_PROVIDER`, `-x FI_EFA_TX_MIN_CREDITS`) to set. Besides, you need to make sure EFA library is included in the shared library path (`-x LD_LIBRARY_PATH`).\n"
  },
  {
    "path": "example/distributed_training-horovod/gluon_mnist.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport argparse\nimport logging\nimport os\nimport zipfile\nimport time\n\nimport mxnet as mx\nimport horovod.mxnet as hvd\nfrom mxnet import autograd, gluon, nd\nfrom mxnet.test_utils import download\n\n# Training settings\nparser = argparse.ArgumentParser(description='MXNet MNIST Example')\n\nparser.add_argument('--batch-size', type=int, default=64,\n                    help='training batch size (default: 64)')\nparser.add_argument('--dtype', type=str, default='float32',\n                    help='training data type (default: float32)')\nparser.add_argument('--epochs', type=int, default=5,\n                    help='number of training epochs (default: 5)')\nparser.add_argument('--lr', type=float, default=0.01,\n                    help='learning rate (default: 0.01)')\nparser.add_argument('--momentum', type=float, default=0.9,\n                    help='SGD momentum (default: 0.9)')\nparser.add_argument('--no-cuda', action='store_true', default=False,\n                    help='disable training on GPU (default: False)')\nargs = parser.parse_args()\n\nif not args.no_cuda:\n    # Disable CUDA if there are no GPUs.\n    if mx.device.num_gpus() == 0:\n        args.no_cuda = True\n\nlogging.basicConfig(level=logging.INFO)\nlogging.info(args)\n\n\n# Function to get mnist iterator given a rank\ndef get_mnist_iterator(rank):\n    data_dir = f\"data-{rank}\"\n    if not os.path.isdir(data_dir):\n        os.makedirs(data_dir)\n    zip_file_path = download('http://data.mxnet.io/mxnet/data/mnist.zip',\n                             dirname=data_dir)\n    with zipfile.ZipFile(zip_file_path) as zf:\n        zf.extractall(data_dir)\n\n    input_shape = (1, 28, 28)\n    batch_size = args.batch_size\n\n    train_iter = mx.io.MNISTIter(\n        image=f\"{data_dir}/train-images-idx3-ubyte\",\n        label=f\"{data_dir}/train-labels-idx1-ubyte\",\n        input_shape=input_shape,\n        batch_size=batch_size,\n        shuffle=True,\n        flat=False,\n        num_parts=hvd.size(),\n        part_index=hvd.rank()\n    )\n\n    val_iter = mx.io.MNISTIter(\n        image=f\"{data_dir}/t10k-images-idx3-ubyte\",\n        label=f\"{data_dir}/t10k-labels-idx1-ubyte\",\n        input_shape=input_shape,\n        batch_size=batch_size,\n        flat=False,\n    )\n\n    return train_iter, val_iter\n\n\n# Function to define neural network\ndef conv_nets():\n    net = gluon.nn.HybridSequential()\n    with net.name_scope():\n        net.add(gluon.nn.Conv2D(channels=20, kernel_size=5, activation='relu'))\n        net.add(gluon.nn.MaxPool2D(pool_size=2, strides=2))\n        net.add(gluon.nn.Conv2D(channels=50, kernel_size=5, activation='relu'))\n        net.add(gluon.nn.MaxPool2D(pool_size=2, strides=2))\n        net.add(gluon.nn.Flatten())\n        net.add(gluon.nn.Dense(512, activation=\"relu\"))\n        net.add(gluon.nn.Dense(10))\n    return net\n\n\n# Function to evaluate accuracy for a model\ndef evaluate(model, data_iter, context):\n    data_iter.reset()\n    metric = mx.gluon.metric.Accuracy()\n    for _, batch in enumerate(data_iter):\n        data = batch.data[0].as_in_context(context)\n        label = batch.label[0].as_in_context(context)\n        output = model(data.astype(args.dtype, copy=False))\n        metric.update([label], [output])\n\n    return metric.get()\n\n\n# Initialize Horovod\nhvd.init()\n\n# Horovod: pin context to local rank\ncontext = mx.cpu(hvd.local_rank()) if args.no_cuda else mx.gpu(hvd.local_rank())\nnum_workers = hvd.size()\n\n# Load training and validation data\ntrain_data, val_data = get_mnist_iterator(hvd.rank())\n\n# Build model\nmodel = conv_nets()\nmodel.cast(args.dtype)\nmodel.hybridize()\n\n# Create optimizer\noptimizer_params = {'momentum': args.momentum,\n                    'learning_rate': args.lr * hvd.size()}\nopt = mx.optimizer.create('sgd', **optimizer_params)\n\n# Initialize parameters\ninitializer = mx.init.Xavier(rnd_type='gaussian', factor_type=\"in\",\n                             magnitude=2)\nmodel.initialize(initializer, ctx=context)\n\n# Horovod: fetch and broadcast parameters\nparams = model.collect_params()\nif params is not None:\n    hvd.broadcast_parameters(params, root_rank=0)\n\n# Horovod: create DistributedTrainer, a subclass of gluon.Trainer\ntrainer = hvd.DistributedTrainer(params, opt)\n\n# Create loss function and train metric\nloss_fn = gluon.loss.SoftmaxCrossEntropyLoss()\nmetric = mx.gluon.metric.Accuracy()\n\n# Train model\nfor epoch in range(args.epochs):\n    tic = time.time()\n    train_data.reset()\n    metric.reset()\n    for nbatch, batch in enumerate(train_data, start=1):\n        data = batch.data[0].as_in_context(context)\n        label = batch.label[0].as_in_context(context)\n        with autograd.record():\n            output = model(data.astype(args.dtype, copy=False))\n            loss = loss_fn(output, label)\n        loss.backward()\n        trainer.step(args.batch_size)\n        metric.update([label], [output])\n\n        if nbatch % 100 == 0:\n            name, acc = metric.get()\n            logging.info(f'[Epoch {epoch} Batch {nbatch}] Training: {name}={acc}')\n\n    if hvd.rank() == 0:\n        elapsed = time.time() - tic\n        speed = nbatch * args.batch_size * hvd.size() / elapsed\n        logging.info('Epoch[%d]\\tSpeed=%.2f samples/s\\tTime cost=%f',\n                     epoch, speed, elapsed)\n\n    # Evaluate model accuracy\n    _, train_acc = metric.get()\n    name, val_acc = evaluate(model, val_data, context)\n    if hvd.rank() == 0:\n        logging.info('Epoch[%d]\\tTrain: %s=%f\\tValidation: %s=%f', epoch, name,\n                     train_acc, name, val_acc)\n\n    if hvd.rank() == 0 and epoch == args.epochs - 1:\n        assert val_acc > 0.96, f\"Achieved accuracy ({val_acc}) is lower than expected\\\n                                (0.96)\"\n"
  },
  {
    "path": "example/distributed_training-horovod/resnet50_imagenet.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport argparse\nimport logging\nimport math\nimport os\nimport time\n\nfrom gluoncv.model_zoo import get_model\nimport horovod.mxnet as hvd\nimport mxnet as mx\nimport numpy as np\nfrom mxnet import autograd, gluon, lr_scheduler\nfrom mxnet.io import DataBatch, DataIter\n\n\n# Training settings\nparser = argparse.ArgumentParser(description='MXNet ImageNet Example',\n                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)\nparser.add_argument('--use-rec', action='store_true', default=False,\n                    help='use image record iter for data input (default: False)')\nparser.add_argument('--data-nthreads', type=int, default=2,\n                    help='number of threads for data decoding (default: 2)')\nparser.add_argument('--rec-train', type=str, default='',\n                    help='the training data')\nparser.add_argument('--rec-train-idx', type=str, default='',\n                    help='the index of training data')\nparser.add_argument('--rec-val', type=str, default='',\n                    help='the validation data')\nparser.add_argument('--rec-val-idx', type=str, default='',\n                    help='the index of validation data')\nparser.add_argument('--batch-size', type=int, default=128,\n                    help='training batch size per device (default: 128)')\nparser.add_argument('--dtype', type=str, default='float32',\n                    help='data type for training (default: float32)')\nparser.add_argument('--num-epochs', type=int, default=90,\n                    help='number of training epochs (default: 90)')\nparser.add_argument('--lr', type=float, default=0.05,\n                    help='learning rate for a single GPU (default: 0.05)')\nparser.add_argument('--momentum', type=float, default=0.9,\n                    help='momentum value for optimizer (default: 0.9)')\nparser.add_argument('--wd', type=float, default=0.0001,\n                    help='weight decay rate (default: 0.0001)')\nparser.add_argument('--lr-mode', type=str, default='poly',\n                    help='learning rate scheduler mode. Options are step, \\\n                    poly and cosine (default: poly)')\nparser.add_argument('--lr-decay', type=float, default=0.1,\n                    help='decay rate of learning rate (default: 0.1)')\nparser.add_argument('--lr-decay-epoch', type=str, default='40,60',\n                    help='epoches at which learning rate decays (default: 40,60)')\nparser.add_argument('--warmup-lr', type=float, default=0.0,\n                    help='starting warmup learning rate (default: 0.0)')\nparser.add_argument('--warmup-epochs', type=int, default=10,\n                    help='number of warmup epochs (default: 10)')\nparser.add_argument('--last-gamma', action='store_true', default=False,\n                    help='whether to init gamma of the last BN layer in \\\n                    each bottleneck to 0 (default: False)')\nparser.add_argument('--model', type=str, default='resnet50_v1',\n                    help='type of model to use. see vision_model for options.')\nparser.add_argument('--use-pretrained', action='store_true', default=False,\n                    help='load pretrained model weights (default: False)')\nparser.add_argument('--no-cuda', action='store_true', default=False,\n                    help='disables CUDA training (default: False)')\nparser.add_argument('--eval-epoch', action='store_true', default=False,\n                    help='evaluate validation accuracy after each epoch \\\n                    when training in module mode (default: False)')\nparser.add_argument('--eval-frequency', type=int, default=0,\n                    help='frequency of evaluating validation accuracy \\\n                    when training with gluon mode (default: 0)')\nparser.add_argument('--log-interval', type=int, default=0,\n                    help='number of batches to wait before logging (default: 0)')\nparser.add_argument('--save-frequency', type=int, default=0,\n                    help='frequency of model saving (default: 0)')\n\n\nargs = parser.parse_args()\n\nlogging.basicConfig(level=logging.INFO)\nlogging.info(args)\n\n# Horovod: initialize Horovod\nhvd.init()\nnum_workers = hvd.size()\nrank = hvd.rank()\nlocal_rank = hvd.local_rank()\n\nnum_classes = 1000\nnum_training_samples = 1281167\nbatch_size = args.batch_size\nepoch_size = \\\n    int(math.ceil(int(num_training_samples // num_workers) / batch_size))\n\nif args.lr_mode == 'step':\n    lr_decay_epoch = [int(i) for i in args.lr_decay_epoch.split(',')]\n    steps = [epoch_size * x for x in lr_decay_epoch]\n    lr_sched = lr_scheduler.MultiFactorScheduler(\n        step=steps,\n        factor=args.lr_decay,\n        base_lr=(args.lr * num_workers),\n        warmup_steps=(args.warmup_epochs * epoch_size),\n        warmup_begin_lr=args.warmup_lr\n    )\nelif args.lr_mode == 'poly':\n    lr_sched = lr_scheduler.PolyScheduler(\n        args.num_epochs * epoch_size,\n        base_lr=(args.lr * num_workers),\n        pwr=2,\n        warmup_steps=(args.warmup_epochs * epoch_size),\n        warmup_begin_lr=args.warmup_lr\n    )\nelif args.lr_mode == 'cosine':\n    lr_sched = lr_scheduler.CosineScheduler(\n        args.num_epochs * epoch_size,\n        base_lr=(args.lr * num_workers),\n        warmup_steps=(args.warmup_epochs * epoch_size),\n        warmup_begin_lr=args.warmup_lr\n    )\nelse:\n    raise ValueError('Invalid lr mode')\n\n# Function for reading data from record file\n# For more details about data loading in MXNet, please refer to\n# https://mxnet.incubator.apache.org/tutorials/basic/data.html?highlight=imagerecorditer\ndef get_data_rec(rec_train, rec_train_idx, rec_val, rec_val_idx, batch_size,\n                 data_nthreads):\n    rec_train = os.path.expanduser(rec_train)\n    rec_train_idx = os.path.expanduser(rec_train_idx)\n    rec_val = os.path.expanduser(rec_val)\n    rec_val_idx = os.path.expanduser(rec_val_idx)\n    jitter_param = 0.4\n    lighting_param = 0.1\n    mean_rgb = [123.68, 116.779, 103.939]\n\n    def batch_fn(batch, ctx):\n        data = batch.data[0].as_in_context(ctx)\n        label = batch.label[0].as_in_context(ctx)\n        return data, label\n\n    train_data = mx.io.ImageRecordIter(\n        path_imgrec=rec_train,\n        path_imgidx=rec_train_idx,\n        preprocess_threads=data_nthreads,\n        shuffle=True,\n        batch_size=batch_size,\n        label_width=1,\n        data_shape=(3, 224, 224),\n        mean_r=mean_rgb[0],\n        mean_g=mean_rgb[1],\n        mean_b=mean_rgb[2],\n        rand_mirror=True,\n        rand_crop=False,\n        random_resized_crop=True,\n        max_aspect_ratio=4. / 3.,\n        min_aspect_ratio=3. / 4.,\n        max_random_area=1,\n        min_random_area=0.08,\n        verbose=False,\n        brightness=jitter_param,\n        saturation=jitter_param,\n        contrast=jitter_param,\n        pca_noise=lighting_param,\n        num_parts=num_workers,\n        part_index=rank,\n        device_id=local_rank\n    )\n    # Kept each node to use full val data to make it easy to monitor results\n    val_data = mx.io.ImageRecordIter(\n        path_imgrec=rec_val,\n        path_imgidx=rec_val_idx,\n        preprocess_threads=data_nthreads,\n        shuffle=False,\n        batch_size=batch_size,\n        resize=256,\n        label_width=1,\n        rand_crop=False,\n        rand_mirror=False,\n        data_shape=(3, 224, 224),\n        mean_r=mean_rgb[0],\n        mean_g=mean_rgb[1],\n        mean_b=mean_rgb[2],\n        device_id=local_rank\n    )\n\n    return train_data, val_data, batch_fn\n\n# Create data iterator for synthetic data\nclass SyntheticDataIter(DataIter):\n    def __init__(self, num_classes, data_shape, max_iter, dtype, ctx):\n        self.batch_size = data_shape[0]\n        self.cur_iter = 0\n        self.max_iter = max_iter\n        self.dtype = dtype\n        label = np.random.randint(0, num_classes, [self.batch_size, ])\n        data = np.random.uniform(-1, 1, data_shape)\n        self.data = mx.nd.array(data, dtype=self.dtype,\n                                ctx=ctx)\n        self.label = mx.nd.array(label, dtype=self.dtype,\n                                 ctx=ctx)\n\n    def __iter__(self):\n        return self\n\n    @property\n    def provide_data(self):\n        return [mx.io.DataDesc('data', self.data.shape, self.dtype)]\n\n    @property\n    def provide_label(self):\n        return [mx.io.DataDesc('softmax_label',\n                               (self.batch_size,), self.dtype)]\n\n    def next(self):\n        self.cur_iter += 1\n        if self.cur_iter <= self.max_iter:\n            return DataBatch(data=(self.data,),\n                             label=(self.label,),\n                             pad=0,\n                             index=None,\n                             provide_data=self.provide_data,\n                             provide_label=self.provide_label)\n        else:\n            raise StopIteration\n\n    def __next__(self):\n        return self.next()\n\n    def reset(self):\n        self.cur_iter = 0\n\n# Horovod: pin GPU to local rank\ncontext = mx.cpu(local_rank) if args.no_cuda else mx.gpu(local_rank)\n\nif args.use_rec:\n    # Fetch training and validation data if present\n    train_data, val_data, batch_fn = get_data_rec(args.rec_train,\n                                                  args.rec_train_idx,\n                                                  args.rec_val,\n                                                  args.rec_val_idx,\n                                                  batch_size,\n                                                  args.data_nthreads)\nelse:\n    # Otherwise use synthetic data\n    image_shape = (3, 224, 224)\n    data_shape = (batch_size,) + image_shape\n    train_data = SyntheticDataIter(num_classes, data_shape, epoch_size,\n                                   np.float32, context)\n    val_data = None\n\n\n# Get model from GluonCV model zoo\n# https://gluon-cv.mxnet.io/model_zoo/index.html\nkwargs = {'ctx': context,\n          'pretrained': args.use_pretrained,\n          'classes': num_classes}\nif args.last_gamma:\n    kwargs['last_gamma'] = True\nnet = get_model(args.model, **kwargs)\nnet.cast(args.dtype)\n\n# Create initializer\ninitializer = mx.init.Xavier(rnd_type='gaussian', factor_type=\"in\",\n                             magnitude=2)\n\n\ndef train_gluon():\n    def evaluate(epoch):\n        if not args.use_rec:\n            return\n\n        val_data.reset()\n        acc_top1 = mx.gluon.metric.Accuracy()\n        acc_top5 = mx.gluon.metric.TopKAccuracy(5)\n        for _, batch in enumerate(val_data):\n            data, label = batch_fn(batch, context)\n            output = net(data.astype(args.dtype, copy=False))\n            acc_top1.update([label], [output])\n            acc_top5.update([label], [output])\n\n        top1_name, top1_acc = acc_top1.get()\n        top5_name, top5_acc = acc_top5.get()\n        logging.info('Epoch[%d] Rank[%d]\\tValidation-%s=%f\\tValidation-%s=%f',\n                     epoch, rank, top1_name, top1_acc, top5_name, top5_acc)\n\n    # Hybridize and initialize model\n    net.hybridize()\n    net.initialize(initializer, ctx=context)\n\n    # Horovod: fetch and broadcast parameters\n    params = net.collect_params()\n    if params is not None:\n        hvd.broadcast_parameters(params, root_rank=0)\n\n    # Create optimizer\n    optimizer_params = {'wd': args.wd,\n                        'momentum': args.momentum,\n                        'lr_scheduler': lr_sched}\n    if args.dtype == 'float16':\n        optimizer_params['multi_precision'] = True\n    opt = mx.optimizer.create('sgd', **optimizer_params)\n\n    # Horovod: create DistributedTrainer, a subclass of gluon.Trainer\n    trainer = hvd.DistributedTrainer(params, opt)\n\n    # Create loss function and train metric\n    loss_fn = gluon.loss.SoftmaxCrossEntropyLoss()\n    metric = mx.gluon.metric.Accuracy()\n\n    # Train model\n    for epoch in range(args.num_epochs):\n        tic = time.time()\n        if args.use_rec:\n            train_data.reset()\n        metric.reset()\n\n        btic = time.time()\n        for nbatch, batch in enumerate(train_data, start=1):\n            data, label = batch_fn(batch, context)\n            with autograd.record():\n                output = net(data.astype(args.dtype, copy=False))\n                loss = loss_fn(output, label)\n            loss.backward()\n            trainer.step(batch_size)\n\n            metric.update([label], [output])\n            if args.log_interval and nbatch % args.log_interval == 0:\n                name, acc = metric.get()\n                logging.info('Epoch[%d] Rank[%d] Batch[%d]\\t%s=%f\\tlr=%f',\n                             epoch, rank, nbatch, name, acc, trainer.learning_rate)\n                if rank == 0:\n                    batch_speed = num_workers * batch_size * args.log_interval / (time.time() - btic)\n                    logging.info('Epoch[%d] Batch[%d]\\tSpeed: %.2f samples/sec',\n                                 epoch, nbatch, batch_speed)\n                btic = time.time()\n\n        # Report metrics\n        elapsed = time.time() - tic\n        _, acc = metric.get()\n        logging.info('Epoch[%d] Rank[%d] Batch[%d]\\tTime cost=%.2f\\tTrain-accuracy=%f',\n                     epoch, rank, nbatch, elapsed, acc)\n        if rank == 0:\n            epoch_speed = num_workers * batch_size * nbatch / elapsed\n            logging.info('Epoch[%d]\\tSpeed: %.2f samples/sec', epoch, epoch_speed)\n\n        # Evaluate performance\n        if args.eval_frequency and (epoch + 1) % args.eval_frequency == 0:\n            evaluate(epoch)\n\n        # Save model\n        if args.save_frequency and (epoch + 1) % args.save_frequency == 0:\n            net.export(f'{args.model}-{rank}', epoch=epoch)\n\n    # Evaluate performance at the end of training\n    evaluate(epoch)\n\n\n\nif __name__ == '__main__':\n    train_gluon()\n"
  },
  {
    "path": "example/extensions/lib_api/Makefile",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nall:\n\tg++ -std=c++11 -shared -fPIC init_lib.cc ../../../src/lib_api.cc -o libinit_lib.so -I ../../../include\n\ntest:\n\tg++ -std=c++11 -O3 -o libtest libtest.cc -ldl -I ../../../include/mxnet\n\nwindows:\n\tcl /LD init_lib.cc\n\nwin_test:\n\tcl libtest.cc\n\nclean:\n\trm -rf *.so libtest\n"
  },
  {
    "path": "example/extensions/lib_api/init_lib.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file init_lib.cc\n * \\brief Sample library file\n */\n\n#include <iostream>\n#include \"mxnet/lib_api.h\"\n\nusing namespace mxnet::ext;\n\nMXReturnValue initialize(int version) {\n  if (version >= 10700) {\n    std::cout << \"MXNet version \" << version << \" supported\" << std::endl;\n    return MX_SUCCESS;\n  } else {\n    MX_ERROR_MSG << \"MXNet version \" << version << \" not supported\";\n    return MX_FAIL;\n  }\n}\n"
  },
  {
    "path": "example/extensions/lib_api/libtest.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file libtest.cc\n * \\brief This test checks if the library is implemented correctly\n * and does not involve dynamic loading of library into MXNet\n * This test is supposed to be run before test.py\n */\n\n#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)\n#include <windows.h>\n#else\n#include <dlfcn.h>\n#endif\n\n#include <iostream>\n#include \"lib_api.h\"\n\n#define MXNET_VERSION 10500\n\nint main(void) {\n  // Get a handle to the library.\n#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)\n  HINSTANCE handle;\n  handle = LoadLibrary(TEXT(\"libinit_lib.dll\"));\n#else\n  void* handle;\n  handle   = dlopen(\"libinit_lib.so\", RTLD_LAZY);\n#endif\n\n  if (!handle) {\n    std::cerr << \"Unable to load library\" << std::endl;\n    return 1;\n  }\n\n  // get initialize function address from the library\n  initialize_t init_lib;\n#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)\n  init_lib = (initialize_t)GetProcAddress(handle, MXLIB_INITIALIZE_STR);\n#else\n  init_lib = (initialize_t)dlsym(handle, MXLIB_INITIALIZE_STR);\n#endif\n\n  if (!init_lib) {\n    std::cerr << \"Unable to get function 'intialize' from library\" << std::endl;\n    return 1;\n  }\n\n  // Call the function.\n  (init_lib)(MXNET_VERSION);\n\n  // Deallocate memory.\n#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)\n  FreeLibrary(handle);\n#else\n  dlclose(handle);\n#endif\n\n  return 0;\n}\n"
  },
  {
    "path": "example/extensions/lib_api/test_loading.py",
    "content": "#!/usr/bin/env python3\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=arguments-differ\n\n# This test checks if dynamic loading of library into MXNet is successful\n\nimport mxnet as mx\nimport os\n\n# test loading library\nif (os.name=='posix'):\n    path = os.path.abspath('libinit_lib.so')\n    mx.library.load(path)\nelif (os.name=='nt'):\n    path = os.path.abspath('libinit_lib.dll')\n    mx.library.load(path)\n\n# test loading library with verbose=False\nif (os.name=='posix'):\n    path = os.path.abspath('libinit_lib.so')\n    mx.library.load(path, False)\nelif (os.name=='nt'):\n    path = os.path.abspath('libinit_lib.dll')\n    mx.library.load(path, False)\n    \n"
  },
  {
    "path": "example/extensions/lib_custom_op/Makefile",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nall: gemm_lib relu_lib transposecsr_lib transposerowsp_lib\n\ngemm_lib:\n\tg++ -shared -fPIC -std=c++11 gemm_lib.cc ../../../src/lib_api.cc -o libgemm_lib.so -I ../../../include\n\nrelu_lib:\n\tg++ -fPIC -c -std=c++11 relu_lib.cc -o relu_lib.cc.o -I ../../../include\n\tg++ -fPIC -c -std=c++11 ../../../src/lib_api.cc -o lib_api.cc.o -I ../../../include\n\tnvcc -c -std=c++11 -Xcompiler -fPIC relu_lib.cu -o relu_lib.cu.o -I ../../../include\n\tnvcc -shared relu_lib.cc.o lib_api.cc.o relu_lib.cu.o -o librelu_lib.so\n\ntransposecsr_lib:\n\tg++ -shared -fPIC -std=c++11 transposecsr_lib.cc ../../../src/lib_api.cc -o libtransposecsr_lib.so -I ../../../include\n\ntransposerowsp_lib:\n\tg++ -shared -fPIC -std=c++11 transposerowsp_lib.cc ../../../src/lib_api.cc -o libtransposerowsp_lib.so -I ../../../include\n\nclean:\n\trm -rf libgemm_lib.so librelu_lib.so libtransposecsr_lib.so libtransposerowsp_lib.so\n"
  },
  {
    "path": "example/extensions/lib_custom_op/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\nC++ Custom Operator Example and Tutorial\n========================================\n\n## Introduction\n\nAdding new operators in MXNet requires understanding of MXNet backend operator registration and recompiling of MXNet with all its dependencies. Users can use the old Python custom operator to add new operators, but it is slow, complicated and has poor adoption rate. So our approach for adding custom operators is to enable dynamic loading of C++ custom operators compiled in external libraries at runtime.\n\nCustom operators (CustomOp) enable users to write new operators without compiling against all of MXNet header files and dependencies. When a library containing custom operators is loaded dynamically, the operators found in the library will be registered in MXNet so that users can call those operators natively just like other built-in operators.\n\n## Getting Started\n\n### Have MXNet Ready\n\nTo run the following example, the build type of MXNet doesn’t matter since the custom operator doesn’t interact with the execution of other native MXNet operators.\nNote that if you want to run GPU examples and write your custom operators running on GPU, you still need an MXNet CUDA build.\n\n### Run An Example\n\nYou can start getting familiar with custom operators by running some examples provided in the `example/extensions/lib_custom_op` directory. Start with a common linear algebra operator like `gemm` (Generalized Matrix Multiplication). Go to `lib_custom_op` directory and follow these steps:\n\n1. Run `make gemm_lib`. The Makefile will generate a dynamic library **libgemm_lib.so** compiled from `gemm_lib.cc`. This is the library you are going to load that contains everything for the custom gemm operator.\n2. Run `python test_gemm.py`. It’ll first load the library compiled from step 1, find the operators, register them in the MXNet backend, then invoke the operator like a regular MXNet operator and output the result.\nBelow is the output when running the python `test_gemm.py` command. Notice that it loads 2 operators: `my_gemm` and `state_gemm`.\n\n```\n[19:22:02] ../src/c_api/c_api.cc:286: Found 2 operators in library\n[19:22:02] ../src/c_api/c_api.cc:350: \tOp[0] my_gemm\n[19:22:02] ../src/c_api/c_api.cc:350: \tOp[1] state_gemm\n[19:22:02] ../src/c_api/c_api.cc:785: Found 0 partitioners in library\n--------start ndarray compute---------\n[[ 50.]\n [122.]]\n<NDArray 2x1 @cpu(0)>\n...\n```\n\nNote that you can safely ignore the `Found 0 partitioners` info as it is not related to the custom operator.\n\n### Basic Files For A GeMM Library\n\n* **lib_custom_op/gemm_lib.cc**: This file has a source code implementation of all required components of a custom operator, as well as the registration of the custom operator.\n\n* **lib_custom_op/Makefile**: This file compiles `gemm_lib.cc` to a dynamic shared library named `libgemm_lib.so`. It includes the header file `include/mxnet/lib_api.h` from MXNet source code. Currently the custom operator APIs require C++11 onwards.\n\n* **lib_custom_op/test_gemm.py**: This file calls `mx.library.load(‘libgemm_lib.so’)` to load the library containing the custom operator, invokes the operator using both NDArray and Symbol APIs, and prints outputs of the forward and backward passes. The outputs should be the same as the regular MXNet `gemm` operator.\n\n* **include/mxnet/lib_api.h**: This file from MXNet source code is the single header file needed to include all necessary data types and function prototypes for writing a custom operator library.\nYou can either specify the include path in the `Makefile`, or copy the header file over to `example/extensions/lib_custom_op` folder.\nNote that apart from this header, the custom operator library is independent of MXNet source.\n\n## Writing A Custom CPU Operator Library\n\nTo build your own library containing custom CPU operator, compose a C++ source file like `myop_lib.cc`, include `lib_api.h` header file, and write your custom operator implementation with these required functions:\n- `initialize` - Library Initialization Function\n- `REGISTER_OP` - Operator Registration Macro\n- `parseAttrs` - Attribute Parser\n- `inferType` - Type Inference\n- `inferShape` - Shape Inference\n- `forward` - Forward Computation (can be replaced with `createOpState`, see below for details)\n\nThen compile it to `libmyop_lib.so` dynamic library using the following command:\n\n```bash\ng++ -shared -fPIC -std=c++11 myop_lib.cc -o libmyop_lib.so -I ../../../include/mxnet\n```\n\nIf you don't want to download MXNet source and choose to only use `lib_api.h` header, you can copy the header over to the same folder of `myop_lib.cc` and run:\n\n```bash\ng++ -shared -fPIC -std=c++11 myop_lib.cc -o libmyop_lib.so\n```\n\nFinally, you can write some code to load the library by specifying its absolute path and run your custom operator in any language binding.\nHeres an example in Python (but C Predict API and C++ API work too):\n\n```python\nimport os\nimport mxnet as mx\npath = os.path.abspath('libmyop_lib.so')\nmx.library.load(path)\nmx.nd.my_op(...)\n```\n\n### Writing A Regular Custom Operator\n\nThere are several required building blocks for making a custom operator:\n\n* [initialize](./gemm_lib.cc#L227):\n    * This function is called when MXNet first loads the library. MXNet passes its version to this function when called.\n    This gives the library the ability to check which version of MXNet is being used. It also provides a place where library state can be initialized.\n\n```c++\n    MXReturnValue initialize(int version)\n```\n\n* [parseAttrs](./gemm_lib.cc#L118):\n    * This function specifies number of input and output tensors for the custom operator; also this is where a custom operator can validate the attributes (ie. options) specified by the user.\n\n```c++\n    MXReturnValue parseAttrs(\n        const std::unordered_map<std::string, std::string>& attrs,\n        int* num_in,\n        int* num_out)\n```\n\n\n* [inferType](./gemm_lib.cc#L124):\n    * This function specifies how the custom operator infers output data types using input data types.\n\n```c++\n    MXReturnValue inferType(\n        const std::unordered_map<std::string, std::string>& attrs,\n        std::vector<int>* intypes,\n        std::vector<int>* outtypes)\n```\n\n* [inferShape](./gemm_lib.cc#L143):\n    * This function specifies how the custom operator infers output tensor shape using input shape.\n\n```c++\n    MXReturnValue inferShape(\n        const std::unordered_map<std::string, std::string>& attrs,\n        std::vector<std::vector<unsigned int>>* inshapes,\n        std::vector<std::vector<unsigned int>>* outshapes)\n```\n\n* [forward](./gemm_lib.cc#L56):\n    * This function specifies the computation of the forward pass of the operator.\n\n```c++\n    MXReturnValue forward(\n        const std::unordered_map<std::string, std::string>& attrs,\n        std::vector<MXTensor>* inputs,\n        std::vector<MXTensor>* outputs,\n        const OpResource& res)\n```\n\nAlso there are some optional functions you can specify:\n\n* [backward](./gemm_lib.cc#L90) - Backward gradient function:\n    * This function specifies the computation of the backward pass of the operator.\n\n```c++\n    MXReturnValue backward(\n        const std::unordered_map<std::string, std::string>& attrs,\n        std::vector<MXTensor>* inputs,\n        std::vector<MXTensor>* outputs,\n        const OpResource& res)\n```\n\n* [inferSType](./transposecsr_lib.cc#168) - Storage Type Inference:\n    * This function specifies how the custom operator infers storage types for inputs and outputs.\n\n```c++\n    MXReturnValue inferSType(\n        const std::unordered_map<std::string, std::string>& attrs,\n        std::vector<MXTensor>* inputs,\n        std::vector<MXTensor>* outputs,\n        const OpResource& res)\n```\n\n* [mutateInputs](./gemm_lib.cc#L214) - Specify mutable input:\n    * This function allows you to mark some inputs to be mutable inputs. It is useful when using aux parameters for BatchNorm-like operators.\n\n```c++\n    MXReturnValue mutateInputs(\n        const std::unordered_map<std::string, std::string>& attrs,\n        std::vector<int>* input_indices)\n```\n\nAfter specifying those functions, register the custom opeartor with MXNet:\n\n* [REGISTER_OP(my_op_name)](./gemm_lib.cc#L169):\n    * This macro registers the custom operator with MXNet.\n    Note that you register functions for each context for `forward` and  `backward`, and here we show an example for CPU context.\n    These are the minimum required functions, but you can specify additional functions as needed.\n\n```c++\n    REGISTER_OP(my_op_name)\n    .setParseAttrs(parseAttrs)\n    .setInferType(inferType)\n    .setInferShape(inferShape)\n    .setForward(forward, \"cpu\");\n```\n\nLet’s take a closer look at those registry functions:\n\n* **parseAttrs**: This function takes three arguments. The 1st argument is an input, which is the attributes passed all the way from Python code. When user calls `mx.nd.my_op_name(s,t,keyword=1)`, the keyword is passed to the attributes as an entry of the map. The 2nd & 3rd arguments are outputs, and you need to set number of inputs and outputs values to those placeholders.\nIf the number of input and output tensors are fixed, you can use hard-coded numbers. Otherwise you can get the user-specified attributes to determine the number of inputs and outputs.\n\n* **inferType**: This function takes three arguments. The 1st argument is the attributes (same as above). The 2nd argument is the a list of input data types corresponding to the input tensors. The 3rd argument is the placeholder for output tensor data types you need to assign.\nFor example, if this operator has one input and one output, and data type doesn’t change, then you can do `outtypes[0] = intypes[0]` to populate the data type.\n\n* **inferSType**: This function takes three arguments. The 1st argument is the attributes (same as above). The 2nd argument is the a list of input storage types corresponding to the input tensors. The 3rd argument is the placeholder for output storage types you need to assign.\nFor example, if this operator has one input and one output, and data type doesn’t change, then you can do `outtypes[0] = intypes[0]` to populate the data type.\n\n* **inferShape**: This function is similar to the `inferType` function, except it is used for populating the output data shapes. You need to figure out the shapes of each output tensors for this computation.\nFor example, if the inputs are images with shape (224,224,3) and you write a padding operator to make 10px borders for the images, then your output shape will be (234,234,3).\n\n* **forward**: This function executes the main forward computation. It takes four arguments. The 1st argument is the attributes. The 2nd argument is the input `MXTensors` which stores all data and info of input ndarrays. The 3rd argument is the output `MXTensors`. The 4th argument is the `OpResource` object for memory allocation and other utilities. The details of `OpResource` are covered in the section below.\nYou can write different forward computations for each data type by doing `if(inputs[0].dtype == kFloat32)` to check the data types of tensors.\nAdditionally, you can use a `dltensor` tensor structure stored in the `MXTensor` as a more standardized data structure for computing.\n\n* **backward**: This function is doing the backward gradient computation. It will be similar to the forward function. And you need to figure out the formula of the backward gradient computation.\n\n* **mutateInputs**: This function is for marking mutable inputs. It takes two arguments. The 1st argument is the attributes. The 2nd argument is a list of input indices that are mutable among all input tensors.\nFor example, you can write `input_indices.push_back(1)` to mark the 2nd input tensor a mutable input.\nIt is useful when some inputs are auxiliary model parameters and might be altered during forward/backward computation. Remember, the index number of `input_indices` should not exceed the number of inputs.\n\n### Writing A Stateful Custom Operator\n\nA stateful custom operator is useful when a forward/backward call needs some data or ‘state’ from previous forward/backward calls. Normally we create a class, and make instance variables store the states used for computing or caching.\n\nMost of the building blocks for making a stateful custom operator is the same as regular custom operator, except it’ll register `createOpState` instead of a `forward` function for the computation.\n\n* [createOpState](./gemm_lib.cc#L204) - Create stateful operator instance:\n    * This function takes two arguments. The 1st argument is attributes. The 2nd argument is a placeholder for `CustomStatefulOp` object. You must [define a class that inherits CustomStatefulOp](./gemm_lib.cc#L178) and override the forward function (optionally the backward function).\n    Then you need to create an instance of your class and assign it to the placeholder. In this way, all of the forward/backward calls will use the same methods in that instance, and the instance is able to keep the state of the operator.\n\n```c++\n    MXReturnValue createOpState(\n        std::map<std::string, std::string> attrs,\n        CustomStatefulOp** op_inst)\n```\n\n* The operator registering function will look like this:\n\n```c++\n    REGISTER_OP(my_state_op)\n    ...\n    .setCreateOpState(createOpState, \"cpu\");\n```\n\n* Note that you will need to register each `createOpState` function specific for each context your operator supports.\n\n## Writing A Custom GPU Operator Library\n\nMost of the building blocks for registering GPU custom operators are the exactly same as CPU ones, except you need to specify the `\"gpu\"` context name when registering `forward`, `backward` or `createOpState` function.\n\n### Run A GPU Example\n\nFor illustration purposes, we provided a `ReLU` (Rectified Linear Unit) activation operator that can run on GPU. Make sure you have installed a CUDA compatible MXNet build. Go to `lib_custom_op` directory and follow these steps: \n\n1. Run `make relu_lib`. The Makefile will invoke `NVCC` compiler to compile the CUDA kernel along with regular custom operator functions from `relu_lib.cu` to generate `librelu_lib.so` library.\n2. Run `python test_relu.py`. It’ll register the GPU `ReLU` operator in the MXNet backend, then invoke the operator by passing an `NDArray` input with GPU context, and output the result tensor with GPU context.\n\n### Writing A Regular GPU Custom Operator\n\nSince most of the building blocks for registering GPU custom operators are the exactly same as CPU ones, the registering function for an operator supporting both GPU and CPU will look like this:\n\n```c++\n    REGISTER_OP(my_op)\n    ...\n    .setForward(forwardCPU, \"cpu\")\n    .setForward(forwardGPU, \"gpu\")\n    .setBackward(backwardCPU, \"cpu\")\n    .setBackward(backwardGPU, \"gpu\");\n```\n\nNote that operators don’t have to support both CPU and GPU functions (can be GPU only).\n\nAfter you register the forward or backward functions with `“gpu”` context, MXNet will call the appropriate forward or backward functions you just registered when the operator is invoked with GPU context.\n\nIn the `forwardGPU` function, you will specify the grid and block size and launch your CUDA kernel.\nMXNet pre-allocates the memory for input and output tensors on the GPU, just like for CPU operators tensors are pre-allocated on the CPU.\nAs a result, you don’t need to call `cudaMemcpy` to move the tensor data to the GPU device.\n\n```c++\n    MXReturnValue forwardGPU(std::map<std::string, std::string> attrs,\n                             std::vector<MXTensor> inputs,\n                             std::vector<MXTensor> outputs,\n                             OpResource op_res) {\n        float* in_data = inputs[0].data<float>();\n        float* out_data = outputs[0].data<float>();\n        mx_stream_t cuda_stream = op_res.get_cuda_stream();\n        ...\n        my_op_forward<<<grid,block,0,cuda_stream>>>(out_data, in_data);\n        ...\n    }\n```\n\nNote that the `cuda_stream` object used for launching kernels is passed from MXNet backend via `OpResource` object. See below for details of `Operator Resource`. You need to compile the `lib_api.h` header file with `nvcc` if you plan to create a custom GPU operator to enable the GPU support in the APIs.  \nAlso, `in_data` and `out_data` are pointers to the tensor data allocated on the GPU, so you can pass them directly to your CUDA kernel.\n\nAt this point all the attribute functions for each operator (`parseAttrs`, `inferShape`, etc.) run on the CPU, including the `forwardGPU` function. The only part that will actually run on the GPU is the launched CUDA kernel function.\n\n```c++\n    __global__ void my_op_forward(float* out, float* in) {\n        // code your CUDA kernel here\n    }\n```\n\n### Writing A Stateful GPU Custom Operator\n\nRecall that for stateful custom operators, you need to define a class that inherits `CustomStatefulOp` and overrides the `forward` and `backward` functions. Stateful operators are created context-aware, so you can create different classes for GPU and CPU stateful operators separately if desired. To do so, you register a createOpState function for each context separately like this\n\n```c++\n    REGISTER_OP(my_state_op_gpu)\n    ...\n    .setCreateOpState(createOpStateCPU, \"cpu\")\n    .setCreateOpState(createOpStateGPU, \"gpu\");\n```\n\nThen you can create different classes for CPU and GPU stateful operators. MXNet will create the stateful operator instance based on the running context when the operator is invoked, and call stateful `forward` or `backward` function from the instantiated stateful operator class.\n\n```c++\n    class MyStatefulOpCPU : public CustomStatefulOp {\n    public:\n        explicit MyStatefulOpCPU() {}\n        MXReturnValue Forward(...) {\n            // code your CPU forward computational logic here\n        }\n        MXReturnValue Backward(...) {\n            // code your CPU backward computational logic here\n        }\n        ~MyStatefulOpCPU() {}\n    };\n\n    class MyStatefulOpGPU : public CustomStatefulOp {\n    public:\n        explicit MyStatefulOpGPU() {}\n        MXReturnValue Forward(...) {\n            // code your GPU forward computational logic here\n        }\n        MXReturnValue Backward(...) {\n            // code your GPU backward computational logic here\n        }\n        ~MyStatefulOpGPU() {}\n    };\n\n    MXReturnValue createOpStateCPU(std::map<std::string,std::string> attrs,\n                                   CustomStatefulOp** op_inst) {\n        *op_inst = new MyStatefulOpCPU();\n        return MX_SUCCESS;\n    }\n\n    MXReturnValue createOpStateGPU(std::map<std::string,std::string> attrs,\n                                   CustomStatefulOp** op_inst) {\n        *op_inst = new MyStatefulOpGPU();\n        return MX_SUCCESS;\n    }\n```\n\nOptionally, you can use the same class for CPU and GPU, but you’ll need to check the `MXContext` type in the `MXTensors` to dispatch CPU or GPU `forward` or `backward` functions yourself to do the computation.\n\n## Operator Resource\n\nMost operators running in MXNet need some shared resources managed by MXNet. Custom operators also need `CPU memory allocation`, `GPU memory allocation`, and `CUDA stream` managed by MXNet backend to implement some functionalities. Those resources are provided in `OpResource` class in `forward` and `backward` functions.\n\n1. CPU memory allocation: MXNet manages memory very carefully to reduce the memory usage and risk of memory leak. Instead of using `malloc` to obtain a temporary workspace from heap memory, it is strongly recommended to use MXNet managed memory allocation function. The `alloc_cpu(int size)` function in `OpResource` class is an API to allocate a chunk of CPU memory through MXNet, and it is safe and easy to use.\n\n```c++\n    unsigned n = inputs[1].shape[0];\n    unsigned m = inputs[1].shape[1];\n    void *workspace = resource.alloc_cpu(n * m * sizeof(float));\n```\n\n2. GPU memory allocation: It is almost the same as CPU memory allocation, except the API name is `alloc_gpu(int size)` and the memory chunk is located in a GPU device.\n\n3. CUDA stream: The CUDA stream object, obtained from `get_cuda_stream()` API, helps custom operator reuse the existing MXNet CUDA stream in order to synchronize GPU running multiple kernels from multiple operators concurrently.\n\nWhen you write your own custom operators, you have the option to use any of the operator resources provided above.\n"
  },
  {
    "path": "example/extensions/lib_custom_op/gemm_lib.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file gemm_lib.cc\n * \\brief Sample 2D gemm custom operator implementation library file\n */\n\n#include <iostream>\n#include <utility>\n#include \"mxnet/lib_api.h\"\n\nusing namespace mxnet::ext;\n\n// main matrix multiplication routine\nvoid gemm(const float* A,\n          const float* B,\n          float* C,\n          const unsigned n,\n          const unsigned k,\n          const unsigned m) {\n  unsigned i, j, kk;\n  for (i = 0; i < n; i++) {\n    for (j = 0; j < m; j++) {\n      C[i * m + j] = 0;\n      for (kk = 0; kk < k; kk++) {\n        C[i * m + j] += A[i * k + kk] * B[kk * m + j];\n      }\n    }\n  }\n}\n\nvoid transpose(const float* A, float* At, const unsigned n, const unsigned m) {\n  unsigned i, j;\n  for (i = 0; i < n; i++) {\n    for (j = 0; j < m; j++) {\n      At[i * m + j] = A[j * n + i];\n    }\n  }\n}\n\n/*\n * Executes C = A * B\n * inputs[0] = A; inputs[1] = B; outputs[0] = C\n */\nMXReturnValue forward(const std::unordered_map<std::string, std::string>& attrs,\n                      std::vector<MXTensor>* inputs,\n                      std::vector<MXTensor>* outputs,\n                      const OpResource& res) {\n  // simple example of using runtime data type\n  if (inputs->at(0).dtype == kFloat32) {\n    typedef float DType;\n    // extract data pointers from tensors\n    // if using dltensor repr, below lines can be changed to something like\n    // DType* A = reinterpret_cast<DType*>(inputs[0].dltensor.data);\n    DType* A = inputs->at(0).data<DType>();\n    DType* B = inputs->at(1).data<DType>();\n    DType* C = outputs->at(0).data<DType>();\n    // set tensor shapes\n    unsigned n = inputs->at(0).shape[0];\n    unsigned k = inputs->at(0).shape[1];\n    unsigned m = inputs->at(1).shape[1];\n\n    gemm(A, B, C, n, k, m);\n  }\n  return MX_SUCCESS;\n}\n\n/*\n * Executes dA = dC * B.T; Executes dB = A.T * dC\n ***** gradient inputs\n * inputs[0] = dC\n ***** original inputs\n * inputs[1] = A; inputs[2] = B\n ***** original outputs\n * inputs[3] = C\n ***** gradient outputs\n * outputs[0] = dA; outputs[1] = dB\n */\nMXReturnValue backward(const std::unordered_map<std::string, std::string>& attrs,\n                       std::vector<MXTensor>* inputs,\n                       std::vector<MXTensor>* outputs,\n                       const OpResource& res) {\n  // extract data pointers from tensors\n  float* dC = inputs->at(0).data<float>();\n  float* A  = inputs->at(1).data<float>();\n  float* B  = inputs->at(2).data<float>();\n  float* dA = outputs->at(0).data<float>();\n  float* dB = outputs->at(1).data<float>();\n  // set tensor shapes\n  unsigned n = inputs->at(1).shape[0];\n  unsigned k = inputs->at(1).shape[1];\n  unsigned m = inputs->at(2).shape[1];\n  // allocate temporary workspace memory through resource manager\n  // for multiple arrays better to request a big memory pool\n  void* workspace = res.alloc_cpu((k * n + m * k) * sizeof(float));\n  float* At       = static_cast<float*>(workspace);\n  float* Bt       = static_cast<float*>(workspace) + (k * n);\n\n  transpose(A, At, k, n);\n  transpose(B, Bt, m, k);\n  gemm(dC, Bt, dA, n, m, k);\n  gemm(At, dC, dB, k, n, m);\n\n  return MX_SUCCESS;\n}\n\nMXReturnValue parseAttrs(const std::unordered_map<std::string, std::string>& attrs,\n                         int* num_in,\n                         int* num_out) {\n  *num_in  = 2;\n  *num_out = 1;\n  return MX_SUCCESS;\n}\n\nMXReturnValue inferType(const std::unordered_map<std::string, std::string>& attrs,\n                        std::vector<int>* intypes,\n                        std::vector<int>* outtypes) {\n  // validate inputs\n  if (intypes->size() != 2) {\n    MX_ERROR_MSG << \"Expected 2 inputs to inferType\";\n    return MX_FAIL;\n  }\n  for (unsigned i = 0; i < intypes->size(); i++) {\n    if (intypes->at(i) != kFloat32) {\n      MX_ERROR_MSG << \"Expected input \" << i << \" to have float32 type\";\n      return MX_FAIL;\n    }\n  }\n\n  outtypes->at(0) = intypes->at(0);\n  return MX_SUCCESS;\n}\n\nMXReturnValue inferShape(const std::unordered_map<std::string, std::string>& attrs,\n                         std::vector<std::vector<unsigned int>>* inshapes,\n                         std::vector<std::vector<unsigned int>>* outshapes) {\n  // validate inputs\n  if (inshapes->size() != 2) {\n    MX_ERROR_MSG << \"Expected 2 inputs to inferShape\";\n    return MX_FAIL;\n  }\n  if (inshapes->at(0).size() != 2 || inshapes->at(1).size() != 2) {\n    MX_ERROR_MSG << \"Expected 2D matrices for both inputs to inferShape\";\n    return MX_FAIL;\n  }\n\n  unsigned n  = inshapes->at(0)[0];\n  unsigned k  = inshapes->at(0)[1];\n  unsigned kk = inshapes->at(1)[0];\n  unsigned m  = inshapes->at(1)[1];\n  if (k != kk) {\n    MX_ERROR_MSG << \"Exected first input axis 1 equals to second input axis 0\";\n    return MX_FAIL;\n  }\n\n  outshapes->at(0) = {n, m};\n  return MX_SUCCESS;\n}\n\nREGISTER_OP(my_gemm)\n    .setForward(forward, \"cpu\")\n    .setBackward(backward, \"cpu\")\n    .setParseAttrs(parseAttrs)\n    .setInferType(inferType)\n    .setInferShape(inferShape);\n\n/* ------------------------------------------------------------------------- */\n\nclass MyStatefulGemm : public CustomStatefulOp {\n public:\n  explicit MyStatefulGemm(int count, std::unordered_map<std::string, std::string> attrs)\n      : count(count), attrs_(std::move(attrs)) {}\n\n  ~MyStatefulGemm() override {\n    std::cout << \"Info: destructing MyStatefulGemm\" << std::endl;\n  }\n\n  MXReturnValue Forward(std::vector<MXTensor>* inputs,\n                        std::vector<MXTensor>* outputs,\n                        const OpResource& op_res) override {\n    std::cout << \"Info: keyword + number of forward: \" << ++count << std::endl;\n    return forward(attrs_, inputs, outputs, op_res);\n  }\n\n  MXReturnValue Backward(std::vector<MXTensor>* inputs,\n                         std::vector<MXTensor>* outputs,\n                         const OpResource& op_res) override {\n    return backward(attrs_, inputs, outputs, op_res);\n  }\n\n private:\n  int count;\n  const std::unordered_map<std::string, std::string> attrs_;\n};\n\nMXReturnValue createOpState(const std::unordered_map<std::string, std::string>& attrs,\n                            const MXContext& ctx,\n                            const std::vector<std::vector<unsigned int>>& in_shapes,\n                            const std::vector<int> in_types,\n                            CustomStatefulOp** op_inst) {\n  // testing passing of keyword arguments\n  int count = attrs.count(\"test_kw\") > 0 ? std::stoi(attrs.at(\"test_kw\")) : 0;\n  // creating stateful operator instance\n  *op_inst = CustomStatefulOp::create<MyStatefulGemm>(count, attrs);\n  std::cout << \"Info: stateful operator created\" << std::endl;\n  return MX_SUCCESS;\n}\n\nMXReturnValue mutateInputs(const std::unordered_map<std::string, std::string>& attrs,\n                           std::vector<int>* input_indices) {\n  // input_indices.push_back(1);  // mark mutate input\n  return MX_SUCCESS;\n}\n\nREGISTER_OP(state_gemm)\n    .setParseAttrs(parseAttrs)\n    .setInferType(inferType)\n    .setInferShape(inferShape)\n    .setMutateInputs(mutateInputs)\n    .setCreateOpState(createOpState, \"cpu\");\n\nMXReturnValue initialize(int version) {\n  if (version >= 10700) {\n    std::cout << \"MXNet version \" << version << \" supported\" << std::endl;\n    return MX_SUCCESS;\n  } else {\n    MX_ERROR_MSG << \"MXNet version \" << version << \" not supported\";\n    return MX_FAIL;\n  }\n}\n"
  },
  {
    "path": "example/extensions/lib_custom_op/relu_lib.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file relu_lib.cu\n * \\brief simple custom relu and noisy relu operator implemented using CUDA function\n */\n\n#include <iostream>\n#include \"relu_lib.h\"\n\nusing namespace mxnet::ext;\n\nMXReturnValue parseAttrs(const std::unordered_map<std::string, std::string>& attrs,\n                         int* num_in,\n                         int* num_out) {\n  *num_in  = 1;\n  *num_out = 1;\n  return MX_SUCCESS;\n}\n\nMXReturnValue inferType(const std::unordered_map<std::string, std::string>& attrs,\n                        std::vector<int>* intypes,\n                        std::vector<int>* outtypes) {\n  outtypes->at(0) = intypes->at(0);\n  return MX_SUCCESS;\n}\n\nMXReturnValue inferShape(const std::unordered_map<std::string, std::string>& attrs,\n                         std::vector<std::vector<unsigned int>>* inshapes,\n                         std::vector<std::vector<unsigned int>>* outshapes) {\n  outshapes->at(0) = inshapes->at(0);\n  return MX_SUCCESS;\n}\n\nMXReturnValue forwardCPU(const std::unordered_map<std::string, std::string>& attrs,\n                         std::vector<MXTensor>* inputs,\n                         std::vector<MXTensor>* outputs,\n                         const OpResource& res) {\n  float* in_data  = inputs->at(0).data<float>();\n  float* out_data = outputs->at(0).data<float>();\n  for (int i = 0; i < inputs->at(0).size(); i++) {\n    out_data[i] = in_data[i] > 0 ? in_data[i] : 0;\n  }\n  return MX_SUCCESS;\n}\n\nMXReturnValue backwardCPU(const std::unordered_map<std::string, std::string>& attrs,\n                          std::vector<MXTensor>* inputs,\n                          std::vector<MXTensor>* outputs,\n                          const OpResource& res) {\n  float* out_grad = inputs->at(0).data<float>();\n  float* in_data  = inputs->at(1).data<float>();\n  float* in_grad  = outputs->at(0).data<float>();\n  for (int i = 0; i < inputs->at(1).size(); i++) {\n    in_grad[i] = in_data[i] > 0 ? 1 * out_grad[i] : 0;\n  }\n  return MX_SUCCESS;\n}\n\nREGISTER_OP(my_relu)\n    .setParseAttrs(parseAttrs)\n    .setInferType(inferType)\n    .setInferShape(inferShape)\n    .setForward(forwardCPU, \"cpu\")\n    .setForward(forwardGPU, \"gpu\")\n    .setBackward(backwardCPU, \"cpu\")\n    .setBackward(backwardGPU, \"gpu\");\n\nMyStatefulReluCPU::MyStatefulReluCPU(const std::unordered_map<std::string, std::string>& attrs)\n    : attrs_(attrs) {}\n\nMXReturnValue MyStatefulReluCPU::Forward(std::vector<MXTensor>* inputs,\n                                         std::vector<MXTensor>* outputs,\n                                         const OpResource& op_res) {\n  return forwardCPU(attrs_, inputs, outputs, op_res);\n}\n\nMXReturnValue MyStatefulReluCPU::Backward(std::vector<MXTensor>* inputs,\n                                          std::vector<MXTensor>* outputs,\n                                          const OpResource& op_res) {\n  return backwardCPU(attrs_, inputs, outputs, op_res);\n}\n\nMyStatefulReluGPU::MyStatefulReluGPU(const std::unordered_map<std::string, std::string>& attrs)\n    : attrs_(attrs) {}\n\nMXReturnValue MyStatefulReluGPU::Forward(std::vector<MXTensor>* inputs,\n                                         std::vector<MXTensor>* outputs,\n                                         const OpResource& op_res) {\n  return forwardGPU(attrs_, inputs, outputs, op_res);\n}\n\nMXReturnValue MyStatefulReluGPU::Backward(std::vector<MXTensor>* inputs,\n                                          std::vector<MXTensor>* outputs,\n                                          const OpResource& op_res) {\n  return backwardGPU(attrs_, inputs, outputs, op_res);\n}\n\nMXReturnValue createOpStateCPU(const std::unordered_map<std::string, std::string>& attrs,\n                               const MXContext& ctx,\n                               const std::vector<std::vector<unsigned int>>& in_shapes,\n                               const std::vector<int> in_types,\n                               CustomStatefulOp** op_inst) {\n  *op_inst = new MyStatefulReluCPU(attrs);\n  return MX_SUCCESS;\n}\n\nMXReturnValue createOpStateGPU(const std::unordered_map<std::string, std::string>& attrs,\n                               const MXContext& ctx,\n                               const std::vector<std::vector<unsigned int>>& in_shapes,\n                               const std::vector<int> in_types,\n                               CustomStatefulOp** op_inst) {\n  *op_inst = new MyStatefulReluGPU(attrs);\n  return MX_SUCCESS;\n}\n\nREGISTER_OP(my_state_relu)\n    .setParseAttrs(parseAttrs)\n    .setInferType(inferType)\n    .setInferShape(inferShape)\n    .setCreateOpState(createOpStateCPU, \"cpu\")\n    .setCreateOpState(createOpStateGPU, \"gpu\");\n\nMXReturnValue noisyForwardCPU(const std::unordered_map<std::string, std::string>& attrs,\n                              std::vector<MXTensor>* inputs,\n                              std::vector<MXTensor>* outputs,\n                              const OpResource& res) {\n  float* in_data  = inputs->at(0).data<float>();\n  float* out_data = outputs->at(0).data<float>();\n\n  mx_cpu_rand_t* states = res.get_cpu_rand_states();\n  std::normal_distribution<float> dist_normal;\n\n  for (int i = 0; i < inputs->at(0).size(); ++i) {\n    float noise = dist_normal(*states);\n    out_data[i] = in_data[i] + noise > 0 ? in_data[i] + noise : 0;\n  }\n  return MX_SUCCESS;\n}\n\nREGISTER_OP(my_noisy_relu)\n    .setParseAttrs(parseAttrs)\n    .setInferType(inferType)\n    .setInferShape(inferShape)\n    .setForward(noisyForwardCPU, \"cpu\")\n    .setForward(noisyForwardGPU, \"gpu\")\n    .setBackward(backwardCPU, \"cpu\")\n    .setBackward(backwardGPU, \"gpu\");\n\nMXReturnValue initialize(int version) {\n  if (version >= 20000) {\n    std::cout << \"MXNet version \" << version << \" supported\" << std::endl;\n    return MX_SUCCESS;\n  } else {\n    MX_ERROR_MSG << \"MXNet version \" << version << \" not supported\";\n    return MX_FAIL;\n  }\n}\n"
  },
  {
    "path": "example/extensions/lib_custom_op/relu_lib.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file relu_lib.cu\n * \\brief simple custom relu and noisy relu operator implemented using CUDA function\n */\n\n#include <iostream>\n#include \"relu_lib.h\"\n\nusing namespace mxnet::ext;\n\n__global__ void relu_gpu_forward(float* out, float* in, int64_t N) {\n  int tid = blockIdx.x * blockDim.x + threadIdx.x;\n  if (tid < N)\n    out[tid] = in[tid] > 0 ? in[tid] : 0;\n}\n\nMXReturnValue forwardGPU(const std::unordered_map<std::string, std::string>& attrs,\n                         std::vector<MXTensor>* inputs,\n                         std::vector<MXTensor>* outputs,\n                         const OpResource& res) {\n  float* in_data  = inputs->at(0).data<float>();\n  float* out_data = outputs->at(0).data<float>();\n\n  mx_stream_t cuda_stream = res.get_cuda_stream();\n  int64_t N               = inputs->at(0).size();\n  int num_block           = (N + NumThreadPerBlock - 1) / NumThreadPerBlock;\n\n  relu_gpu_forward<<<num_block, NumThreadPerBlock, 0, cuda_stream>>>(out_data, in_data, N);\n\n  return MX_SUCCESS;\n}\n\n__global__ void relu_gpu_backward(float* ingrad, float* outgrad, float* indata, int64_t N) {\n  int tid = blockIdx.x * blockDim.x + threadIdx.x;\n  if (tid < N)\n    ingrad[tid] = indata[tid] > 0 ? 1 * outgrad[tid] : 0;\n}\n\nMXReturnValue backwardGPU(const std::unordered_map<std::string, std::string>& attrs,\n                          std::vector<MXTensor>* inputs,\n                          std::vector<MXTensor>* outputs,\n                          const OpResource& res) {\n  float* out_grad = inputs->at(0).data<float>();\n  float* in_data  = inputs->at(1).data<float>();\n  float* in_grad  = outputs->at(0).data<float>();\n\n  mx_stream_t cuda_stream = res.get_cuda_stream();\n  int64_t N               = inputs->at(0).size();\n  int num_block           = (N + NumThreadPerBlock - 1) / NumThreadPerBlock;\n  relu_gpu_backward<<<num_block, NumThreadPerBlock, 0, cuda_stream>>>(\n      in_grad, out_grad, in_data, N);\n\n  return MX_SUCCESS;\n}\n\n__global__ void noisy_relu_gpu_forward(float* out,\n                                       float* in,\n                                       int64_t N,\n                                       mx_gpu_rand_t* states,\n                                       int step) {\n  // the launcher logic ensures tid less than NumGPURandomStates\n  int tid = blockIdx.x * blockDim.x + threadIdx.x;\n  // each thread generates unique sequence of random numbers\n  mx_gpu_rand_t thread_state = states[tid];\n  // each thread works on <step> number of calculation\n  int start = tid * step;\n  int end   = start + step;\n  for (int i = start; i < end && i < N; ++i) {\n    float noise = curand_normal(&thread_state);\n    out[i]      = in[i] + noise > 0 ? in[i] + noise : 0;\n  }\n}\n\nMXReturnValue noisyForwardGPU(const std::unordered_map<std::string, std::string>& attrs,\n                              std::vector<MXTensor>* inputs,\n                              std::vector<MXTensor>* outputs,\n                              const OpResource& res) {\n  float* in_data  = inputs->at(0).data<float>();\n  float* out_data = outputs->at(0).data<float>();\n\n  mx_stream_t cuda_stream = res.get_cuda_stream();\n  int64_t N               = inputs->at(0).size();\n\n  // below is mxnet recommended workflow to parallel random number generating\n  int nthread = (N + NumRandomPerThread - 1) / NumRandomPerThread;\n  // we should not launch more threads than mxnet supported random number GPU states\n  int num_thread_need = nthread < MX_NUM_GPU_RANDOM_STATES ? nthread : MX_NUM_GPU_RANDOM_STATES;\n  // each cuda thread processes [step * tid, step * id + step) snippet of input tensor\n  int step = (N + num_thread_need - 1) / num_thread_need;\n  // this can ensure number of parallel threads less than mxnet supported random number states\n  int num_block = (num_thread_need + NumThreadPerBlock - 1) / NumThreadPerBlock;\n\n  noisy_relu_gpu_forward<<<num_block, NumThreadPerBlock, 0, cuda_stream>>>(\n      out_data, in_data, N, res.get_gpu_rand_states(), step);\n\n  return MX_SUCCESS;\n}\n"
  },
  {
    "path": "example/extensions/lib_custom_op/relu_lib.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file relu_lib.cu\n * \\brief simple custom relu and noisy relu operator implemented using CUDA function\n */\n\n#ifndef __EXAMPLE__RELU_LIB_H__\n#define __EXAMPLE__RELU_LIB_H__\n\n#include <iostream>\n#include \"mxnet/lib_api.h\"\n\nusing namespace mxnet::ext;\n\n#define NumThreadPerBlock  256  // mxnet recommended cuda thread number per block\n#define NumRandomPerThread 64   // mxnet recommended random numbers generated per thread\n\nclass MyStatefulReluCPU : public CustomStatefulOp {\n public:\n  explicit MyStatefulReluCPU(const std::unordered_map<std::string, std::string>& attrs);\n\n  MXReturnValue Forward(std::vector<MXTensor>* inputs,\n                        std::vector<MXTensor>* outputs,\n                        const OpResource& op_res);\n  MXReturnValue Backward(std::vector<MXTensor>* inputs,\n                         std::vector<MXTensor>* outputs,\n                         const OpResource& op_res);\n\n private:\n  const std::unordered_map<std::string, std::string> attrs_;\n};\n\nclass MyStatefulReluGPU : public CustomStatefulOp {\n public:\n  explicit MyStatefulReluGPU(const std::unordered_map<std::string, std::string>& attrs);\n\n  MXReturnValue Forward(std::vector<MXTensor>* inputs,\n                        std::vector<MXTensor>* outputs,\n                        const OpResource& op_res);\n\n  MXReturnValue Backward(std::vector<MXTensor>* inputs,\n                         std::vector<MXTensor>* outputs,\n                         const OpResource& op_res);\n\n private:\n  const std::unordered_map<std::string, std::string> attrs_;\n};\n\nMXReturnValue forwardGPU(const std::unordered_map<std::string, std::string>& attrs,\n                         std::vector<MXTensor>* inputs,\n                         std::vector<MXTensor>* outputs,\n                         const OpResource& res);\n\nMXReturnValue backwardGPU(const std::unordered_map<std::string, std::string>& attrs,\n                          std::vector<MXTensor>* inputs,\n                          std::vector<MXTensor>* outputs,\n                          const OpResource& res);\n\n/*\n * Below is noisy ReLU operator example\n * noisy ReLU is made from ReLU extended to include Gaussian noise\n * forward - add Gaussian noise generated from normal distribution to each unit\n * backward - gradient doesn't need to change since noise is constant\n */\n\nMXReturnValue noisyForwardGPU(const std::unordered_map<std::string, std::string>& attrs,\n                              std::vector<MXTensor>* inputs,\n                              std::vector<MXTensor>* outputs,\n                              const OpResource& res);\n\n#endif\n"
  },
  {
    "path": "example/extensions/lib_custom_op/test_gemm.py",
    "content": "#!/usr/bin/env python3\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=arguments-differ\n\n# This test checks dynamic loading of custom library into MXNet\n# and checks end to end compute of a simple 2D gemm custom op\n\nimport mxnet as mx\nimport os\n\n#load library\nif (os.name=='posix'):\n    path = os.path.abspath('libgemm_lib.so')\n    mx.library.load(path)\nelif (os.name=='nt'):\n    path = os.path.abspath('libgemm_lib.dll')\n    mx.library.load(path)\n\na = mx.nd.array([[1,2,3],[4,5,6]])\nb = mx.nd.array([[7],[8],[9]])\n\nprint(\"--------start ndarray compute---------\")\nprint(mx.nd.my_gemm(a,b))\nprint(\"--------\")\nprint(mx.nd.state_gemm(a,b,test_kw=100))\n\nprint(\"--------start symbolic compute--------\")\ns = mx.sym.Variable('s')\nt = mx.sym.Variable('t')\nc = mx.sym.my_gemm(s,t)\nd = mx.sym.state_gemm(s,t,test_kw=200)\ne = mx.sym.linalg.gemm2(s,t)\n\nout_grad = mx.nd.ones((2,1))\n\n# stateless\nblock = mx.gluon.nn.SymbolBlock(c,[s,t])\nwith mx.autograd.record():\n    a_ = mx.nd.array([[1,2,3],[4,5,6]])\n    b_ = mx.nd.array([[7],[8],[9]])\n    a_.attach_grad()\n    b_.attach_grad()\n    # foward\n    out = block(a_,b_)\n    print(out)\n    print('+++++')\n    # backward\n    out.backward(out_grad)\n    print(a_.grad)\n    print(b_.grad)\n    print(\"-------\")\n\n# stateful\nblock2 = mx.gluon.nn.SymbolBlock(d,[s,t])\nblock2.hybridize(static_alloc=True, static_shape=True)\nout2 = block2(a,b)\nout2 = block2(a,b)\nprint(out2)\nwith mx.autograd.record():\n    a_ = mx.nd.array([[1,2,3],[4,5,6]])\n    b_ = mx.nd.array([[7],[8],[9]])\n    a_.attach_grad()\n    b_.attach_grad()\n    # forward\n    out2 = block2(a_,b_)\n    print('+++++')\n    # backward\n    out2.backward(out_grad)\n    print(a_.grad)\n    print(b_.grad)\n    print(\"-------\")\n\n# baseline\nblock3 = mx.gluon.nn.SymbolBlock(e,[s,t])\nwith mx.autograd.record():\n    a_ = mx.nd.array([[1,2,3],[4,5,6]])\n    b_ = mx.nd.array([[7],[8],[9]])\n    a_.attach_grad()\n    b_.attach_grad()\n    # forward\n    out3 = block3(a_,b_)\n    print(out3)\n    print('+++++')\n    # backward\n    out3.backward(out_grad)\n    print(a_.grad)\n    print(b_.grad)\n"
  },
  {
    "path": "example/extensions/lib_custom_op/test_relu.py",
    "content": "#!/usr/bin/env python3\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=arguments-differ\n\n# This test checks dynamic loading of custom library into MXNet\n# and checks end to end compute of a simple 2D gemm custom op\n\nimport mxnet as mx\nimport os\nimport time\n\n#load library\nif (os.name=='posix'):\n    path = os.path.abspath('librelu_lib.so')\n    mx.library.load(path)\n\na = mx.nd.array([[-2,-1],[1,2]], ctx=mx.cpu())\nb = mx.nd.array([[-2,-1],[1,2]], ctx=mx.gpu())\n\nprint(\"--------ndarray compute---------\")\nprint(mx.nd.my_relu(a))\nprint(mx.nd.my_relu(b))\nprint(mx.nd.my_state_relu(a))\nprint(mx.nd.my_state_relu(b))\n\nprint(\"--------symbolic compute--------\")\nc = mx.sym.Variable('c')\nd = mx.sym.Variable('d')\ne = mx.sym.my_relu(c)\nbase = mx.sym.relu(d)\n#in_grad = [mx.nd.empty((2,2), ctx=mx.gpu())]\n#in_grad_base = [mx.nd.empty((2,2), ctx=mx.gpu())]\nout_grad = mx.nd.ones((2,2), ctx=mx.gpu())\n#exe = e.bind(ctx=mx.gpu(), args={'c':b}, args_grad=in_grad)\nblock = mx.gluon.nn.SymbolBlock(e,[c])\n#exe_base = base.bind(ctx=mx.gpu(), args={'d':b}, args_grad=in_grad_base)\nblock_base = mx.gluon.nn.SymbolBlock(base,[d])\n\n# base\nwith mx.autograd.record():\n    b_ = mx.nd.array([[-2,-1],[1,2]], ctx=mx.gpu())\n    b_.attach_grad()\n    # foward\n    out_base = block_base(b_)\n    print(out_base)\n    print('+++++')\n    # backward\n    out_base.backward(out_grad)\n    print(b_.grad)\n    print(\"-------\")\n\n# custom relu\nwith mx.autograd.record():\n    b_ = mx.nd.array([[-2,-1],[1,2]], ctx=mx.gpu())\n    b_.attach_grad()\n    # foward\n    out = block(b_)\n    print(out)\n    print('+++++')\n    # backward\n    out.backward(out_grad)\n    print(b_.grad)\n    print(\"-------\")\n\nprint(\"--------test ndarray with size of 1 million---------\")\nb = mx.nd.uniform(shape=(100,100,100), ctx=mx.gpu())\nmx.nd.waitall()\nt1 = time.time()\nr1 = mx.nd.my_relu(b)\nmx.nd.waitall()\nt2 = time.time()\nr2 = mx.nd.relu(b)\nmx.nd.waitall()\nt3 = time.time()\nprint(\"Custom ReLU running time in ms:\")\nprint((t2 - t1) * 1000)\nprint(\"Native ReLU running time in ms:\")\nprint((t3 - t2) * 1000)\n\nprint(\"--------test noisy relu identical sequence---------\")\n\na = mx.nd.ones(shape=(13,5), ctx=mx.cpu())\nb = mx.nd.ones(shape=(13,5), ctx=mx.gpu())\n\nmx.random.seed(128, ctx=mx.cpu())\nprint(mx.nd.my_noisy_relu(a))\n\nmx.random.seed(128, ctx=mx.cpu())\nprint(mx.nd.my_noisy_relu(a))\n\nmx.random.seed(128, ctx=mx.gpu())\nprint(mx.nd.my_noisy_relu(b))\n\nmx.random.seed(128, ctx=mx.gpu())\nprint(mx.nd.my_noisy_relu(b))\n"
  },
  {
    "path": "example/extensions/lib_custom_op/test_transposecsr.py",
    "content": "#!/usr/bin/env python3\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=arguments-differ\n\n# This test checks dynamic loading of custom library into MXNet\n# and checks end to end compute of a simple 2D gemm custom op\n\nimport mxnet as mx\nimport os\n\n#load library\nif (os.name=='posix'):\n    path = os.path.abspath('libtransposecsr_lib.so')\n    mx.library.load(path)\nelif (os.name=='nt'):\n    path = os.path.abspath('libtransposecsr_lib.dll')\n    mx.library.load(path)\n\na = mx.nd.array([[1,3,0,2,1],[0,1,0,0,0],[0,2,4,5,3]])\na = a.tostype('csr')\nprint(\"--------Input CSR Array---------\")\nprint(\"data:\", a.data.asnumpy())\nprint(\"indices:\", a.indices.asnumpy())\nprint(\"indptr:\", a.indptr.asnumpy())\n\nprint(\"--------Start NDArray Compute---------\")\nb = mx.nd.my_transposecsr(a)\nprint(\"Compute Results:\")\nprint(\"data:\", b.data.asnumpy())\nprint(\"indices:\", b.indices.asnumpy())\nprint(\"indptr:\", b.indptr.asnumpy())\n\nprint(\"Stateful Compute Result:\")\nc = mx.nd.my_state_transposecsr(a, test_kw=100)\nprint(\"data:\", c.data.asnumpy())\nprint(\"indices:\", c.indices.asnumpy())\nprint(\"indptr:\", c.indptr.asnumpy())\n\nprint(\"--------start Gluon compute--------\")\nd = mx.sym.Variable('d')\ne = mx.sym.my_transposecsr(d)\nf = mx.sym.my_state_transposecsr(d, test_kw=200)\n\nblock = mx.gluon.nn.SymbolBlock(e, [d])\nout = block(a)\nprint(\"Compute Results:\")\nprint(\"data:\", out.data.asnumpy())\nprint(\"indices:\", out.indices.asnumpy())\nprint(\"indptr:\", out.indptr.asnumpy())\n\nblock2 = mx.gluon.nn.SymbolBlock(f,[d])\nout2 = block2(a)\nout2 = block2(a)\nprint(\"Stateful Compute Result:\")\nprint(\"data:\", out2.data.asnumpy())\nprint(\"indices:\", out2.indices.asnumpy())\nprint(\"indptr:\", out2.indptr.asnumpy())\n\nprint(\"--------Baseline(dense)--------\")\nprint(mx.nd.transpose(a.tostype('default')))\n"
  },
  {
    "path": "example/extensions/lib_custom_op/test_transposerowsp.py",
    "content": "#!/usr/bin/env python3\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=arguments-differ\n\n# This test checks dynamic loading of custom library into MXNet\n# and checks end to end compute of a simple 2D gemm custom op\n\nimport mxnet as mx\nimport os\n\n#load library\nif (os.name=='posix'):\n    path = os.path.abspath('libtransposerowsp_lib.so')\n    mx.library.load(path)\nelif (os.name=='nt'):\n    path = os.path.abspath('libtransposerowsp_lib.dll')\n    mx.library.load(path)\n\na = mx.nd.array([[1,2,3],[0,0,0],[4,0,5],[0,0,0],[0,0,0]])\na = a.tostype('row_sparse')\nprint(\"--------Input CSR Array---------\")\nprint(\"data:\", a.data.asnumpy())\nprint(\"indices:\", a.indices.asnumpy())\n\nprint(\"--------Start NDArray Compute---------\")\nb = mx.nd.my_transposerowsp(a)\nprint(\"Compute Results:\")\nprint(\"data:\", b.data.asnumpy())\nprint(\"indices:\", b.indices.asnumpy())\n\nprint(\"Stateful Compute Result:\")\nc = mx.nd.my_state_transposerowsp(a, test_kw=100)\nprint(\"data:\", c.data.asnumpy())\nprint(\"indices:\", c.indices.asnumpy())\n\nprint(\"--------start Gluon compute--------\")\nd = mx.sym.Variable('d')\ne = mx.sym.my_transposerowsp(d)\nf = mx.sym.my_state_transposerowsp(d, test_kw=200)\n\nblock = mx.gluon.nn.SymbolBlock(e,[d])\nout = block(a)\nprint(\"Compute Results:\")\nprint(out)\nprint(\"data:\", out.data.asnumpy())\nprint(\"indices:\", out.indices.asnumpy())\n\nblock2 = mx.gluon.nn.SymbolBlock(f,[d])\nout2 = block2(a)\nout2 = block2(a)\nprint(\"Stateful Compute Result:\")\nprint(\"data:\", out2.data.asnumpy())\nprint(\"indices:\", out2.indices.asnumpy())\n\nprint(\"--------Baseline(dense)--------\")\nprint(mx.nd.transpose(a.tostype('default')))\n"
  },
  {
    "path": "example/extensions/lib_custom_op/transposecsr_lib.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file transsparse_lib.cc\n * \\brief Sample 2D transpose custom operator.\n */\n\n#include <iostream>\n#include <utility>\n#include \"mxnet/lib_api.h\"\n\nusing namespace mxnet::ext;\n\nvoid transpose(MXTensor& src, MXTensor& dst, const OpResource& res) {\n  MXSparse* A                = src.data<MXSparse>();\n  MXSparse* B                = dst.data<MXSparse>();\n  std::vector<int64_t> shape = src.shape;\n  int64_t h                  = shape[0];\n  int64_t w                  = shape[1];\n  if (src.stype == kCSRStorage) {\n    float* Aval = (float*)(A->data);\n    // Here we need one more element to help calculate index(line 57).\n    std::vector<int64_t> rowPtr(w + 2, 0);\n    // count column\n    for (int i = 0; i < A->data_len; i++) {\n      rowPtr[A->indices[i] + 2]++;\n    }\n    // Accumulated sum. After this for loop, rowPtr[1:w+2) stores the correct\n    // result of transposed rowPtr.\n    for (int i = 2; i < rowPtr.size(); i++) {\n      rowPtr[i] += rowPtr[i - 1];\n    }\n\n    // Alloc memory for sparse data, where 0 is the index\n    // of B in output vector.\n    res.alloc_sparse(B, 0, A->data_len, w + 1);\n    float* Bval = (float*)(B->data);\n    for (int i = 0; i < h; i++) {\n      for (int j = A->indptr[i]; j < A->indptr[i + 1]; j++) {\n        // Helps calculate index and after that rowPtr[0:w+1) stores the\n        // correct result of transposed rowPtr.\n        int index         = rowPtr[A->indices[j] + 1]++;\n        Bval[index]       = Aval[j];\n        B->indices[index] = i;\n      }\n    }\n    memcpy(B->indptr, rowPtr.data(), sizeof(int64_t) * (w + 1));\n  }\n}\n\nMXReturnValue forward(const std::unordered_map<std::string, std::string>& attrs,\n                      std::vector<MXTensor>* inputs,\n                      std::vector<MXTensor>* outputs,\n                      const OpResource& res) {\n  // The data types and storage types of inputs and outputs should be the same.\n  if (inputs->at(0).dtype != outputs->at(0).dtype || inputs->at(0).stype != outputs->at(0).stype) {\n    MX_ERROR_MSG << \"Error! Expected all inputs and outputs to be the same type.\"\n                 << \"Found input storage type:\" << inputs->at(0).stype\n                 << \" Found output storage type:\" << outputs->at(0).stype\n                 << \" Found input data type:\" << inputs->at(0).dtype\n                 << \" Found output data type:\" << outputs->at(0).dtype;\n    return MX_FAIL;\n  }\n\n  transpose(inputs->at(0), outputs->at(0), res);\n  return MX_SUCCESS;\n}\n\nMXReturnValue backward(const std::unordered_map<std::string, std::string>& attrs,\n                       std::vector<MXTensor>* inputs,\n                       std::vector<MXTensor>* outputs,\n                       const OpResource& res) {\n  return MX_SUCCESS;\n}\n\nMXReturnValue parseAttrs(const std::unordered_map<std::string, std::string>& attrs,\n                         int* num_in,\n                         int* num_out) {\n  *num_in  = 1;\n  *num_out = 1;\n  return MX_SUCCESS;\n}\n\nMXReturnValue inferType(const std::unordered_map<std::string, std::string>& attrs,\n                        std::vector<int>* intypes,\n                        std::vector<int>* outtypes) {\n  // validate inputs\n  if (intypes->size() != 1) {\n    MX_ERROR_MSG << \"Expected 1 inputs to inferType\";\n    return MX_FAIL;\n  }\n  if (intypes->at(0) != kFloat32) {\n    MX_ERROR_MSG << \"Expected input to have float32 type\";\n    return MX_FAIL;\n  }\n\n  outtypes->at(0) = intypes->at(0);\n  return MX_SUCCESS;\n}\n\nMXReturnValue inferSType(const std::unordered_map<std::string, std::string>& attrs,\n                         std::vector<int>* instypes,\n                         std::vector<int>* outstypes) {\n  if (instypes->at(0) != kCSRStorage) {\n    MX_ERROR_MSG << \"Expected storage type is kCSRStorage\";\n    return MX_FAIL;\n  }\n  outstypes->at(0) = instypes->at(0);\n  return MX_SUCCESS;\n}\n\nMXReturnValue inferShape(const std::unordered_map<std::string, std::string>& attrs,\n                         std::vector<std::vector<unsigned int>>* inshapes,\n                         std::vector<std::vector<unsigned int>>* outshapes) {\n  // validate inputs\n  if (inshapes->size() != 1) {\n    MX_ERROR_MSG << \"Expected 1 inputs to inferShape\";\n    return MX_FAIL;\n  }\n\n  outshapes->at(0).push_back(inshapes->at(0)[1]);\n  outshapes->at(0).push_back(inshapes->at(0)[0]);\n  return MX_SUCCESS;\n}\n\nREGISTER_OP(my_transposecsr)\n    .setForward(forward, \"cpu\")\n    .setBackward(backward, \"cpu\")\n    .setParseAttrs(parseAttrs)\n    .setInferType(inferType)\n    .setInferSType(inferSType)\n    .setInferShape(inferShape);\n\n/* ------------------------------------------------------------------------- */\n\nclass MyStatefulTransposeCSR : public CustomStatefulOp {\n public:\n  explicit MyStatefulTransposeCSR(int count, std::unordered_map<std::string, std::string> attrs)\n      : count(count), attrs_(std::move(attrs)) {}\n\n  MXReturnValue Forward(std::vector<MXTensor>* inputs,\n                        std::vector<MXTensor>* outputs,\n                        const OpResource& op_res) override {\n    std::cout << \"Info: keyword + number of forward: \" << ++count << std::endl;\n    return forward(attrs_, inputs, outputs, op_res);\n  }\n\n  MXReturnValue Backward(std::vector<MXTensor>* inputs,\n                         std::vector<MXTensor>* outputs,\n                         const OpResource& op_res) override {\n    return backward(attrs_, inputs, outputs, op_res);\n  }\n\n private:\n  int count;\n  const std::unordered_map<std::string, std::string> attrs_;\n};\n\nMXReturnValue createOpState(const std::unordered_map<std::string, std::string>& attrs,\n                            const MXContext& ctx,\n                            const std::vector<std::vector<unsigned int>>& in_shapes,\n                            const std::vector<int> in_types,\n                            CustomStatefulOp** op_inst) {\n  // testing passing of keyword arguments\n  int count = attrs.count(\"test_kw\") > 0 ? std::stoi(attrs.at(\"test_kw\")) : 0;\n  // creating stateful operator instance\n  *op_inst = new MyStatefulTransposeCSR(count, attrs);\n  std::cout << \"Info: stateful operator created\" << std::endl;\n  return MX_SUCCESS;\n}\n\nREGISTER_OP(my_state_transposecsr)\n    .setParseAttrs(parseAttrs)\n    .setInferType(inferType)\n    .setInferSType(inferSType)\n    .setInferShape(inferShape)\n    .setCreateOpState(createOpState, \"cpu\");\n\nMXReturnValue initialize(int version) {\n  if (version >= 10700) {\n    std::cout << \"MXNet version \" << version << \" supported\" << std::endl;\n    return MX_SUCCESS;\n  } else {\n    MX_ERROR_MSG << \"MXNet version \" << version << \" not supported\";\n    return MX_FAIL;\n  }\n}\n"
  },
  {
    "path": "example/extensions/lib_custom_op/transposerowsp_lib.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file transsparse_lib.cc\n * \\brief Sample 2D transpose custom operator.\n */\n\n#include <iostream>\n#include <utility>\n#include \"mxnet/lib_api.h\"\n\nusing namespace mxnet::ext;\n\nvoid transpose(MXTensor& src, MXTensor& dst, const OpResource& res) {\n  MXSparse* A = src.data<MXSparse>();\n  MXSparse* B = dst.data<MXSparse>();\n\n  std::vector<int64_t> shape = src.shape;\n  int64_t h                  = shape[0];\n  int64_t w                  = shape[1];\n  if (src.stype == kRowSparseStorage) {\n    // Keys of the map is the row index of transposed tensors.\n    // Values of the map is the rows which have non-zero elements.\n    std::map<int, std::vector<float>> mp;\n    float* Aval = (float*)(A->data);\n    for (int i = 0; i < A->data_len; i++) {\n      int row = i / w;\n      int col = i % w;\n      row     = A->indices[row];\n      if (Aval[i] != 0) {\n        if (mp.find(col) == mp.end()) {\n          mp[col]      = std::vector<float>(h, 0);\n          mp[col][row] = Aval[i];\n        } else {\n          mp[col][row] = Aval[i];\n        }\n      }\n    }\n\n    // Alloc memory for output tensors.\n    res.alloc_sparse(B, 0, mp.size());\n    float* Bval = (float*)(B->data);\n    int didx = 0, iidx = 0;\n    for (const auto& i : mp) {\n      B->indices[iidx++] = i.first;\n      for (auto j : i.second) {\n        Bval[didx++] = j;\n      }\n    }\n  }\n}\n\nMXReturnValue forward(const std::unordered_map<std::string, std::string>& attrs,\n                      std::vector<MXTensor>* inputs,\n                      std::vector<MXTensor>* outputs,\n                      const OpResource& res) {\n  // The data types and storage types of inputs and outputs should be the same.\n  if (inputs->at(0).dtype != outputs->at(0).dtype || inputs->at(0).stype != outputs->at(0).stype) {\n    MX_ERROR_MSG << \"Error! Expected all inputs and outputs to be the same type.\"\n                 << \"Found input storage type:\" << inputs->at(0).stype\n                 << \" Found output storage type:\" << outputs->at(0).stype\n                 << \" Found input data type:\" << inputs->at(0).dtype\n                 << \" Found output data type:\" << outputs->at(0).dtype;\n    return MX_FAIL;\n  }\n  transpose(inputs->at(0), outputs->at(0), res);\n  return MX_SUCCESS;\n}\n\nMXReturnValue backward(const std::unordered_map<std::string, std::string>& attrs,\n                       std::vector<MXTensor>* inputs,\n                       std::vector<MXTensor>* outputs,\n                       const OpResource& res) {\n  return MX_SUCCESS;\n}\n\nMXReturnValue parseAttrs(const std::unordered_map<std::string, std::string>& attrs,\n                         int* num_in,\n                         int* num_out) {\n  *num_in  = 1;\n  *num_out = 1;\n  return MX_SUCCESS;\n}\n\nMXReturnValue inferType(const std::unordered_map<std::string, std::string>& attrs,\n                        std::vector<int>* intypes,\n                        std::vector<int>* outtypes) {\n  // validate inputs\n  if (intypes->size() != 1) {\n    MX_ERROR_MSG << \"Expected 1 inputs to inferType\";\n    return MX_FAIL;\n  }\n  if (intypes->at(0) != kFloat32) {\n    MX_ERROR_MSG << \"Expected input to have float32 type\";\n    return MX_FAIL;\n  }\n\n  outtypes->at(0) = intypes->at(0);\n  return MX_SUCCESS;\n}\n\nMXReturnValue inferSType(const std::unordered_map<std::string, std::string>& attrs,\n                         std::vector<int>* instypes,\n                         std::vector<int>* outstypes) {\n  if (instypes->at(0) != kRowSparseStorage) {\n    MX_ERROR_MSG << \"Expected storage type is kRowSparseStorage\";\n    return MX_FAIL;\n  }\n  outstypes->at(0) = instypes->at(0);\n  return MX_SUCCESS;\n}\n\nMXReturnValue inferShape(const std::unordered_map<std::string, std::string>& attrs,\n                         std::vector<std::vector<unsigned int>>* inshapes,\n                         std::vector<std::vector<unsigned int>>* outshapes) {\n  // validate inputs\n  if (inshapes->size() != 1) {\n    MX_ERROR_MSG << \"Expected 1 inputs to inferShape\";\n    return MX_FAIL;\n  }\n\n  outshapes->at(0).push_back(inshapes->at(0)[1]);\n  outshapes->at(0).push_back(inshapes->at(0)[0]);\n  return MX_SUCCESS;\n}\n\nREGISTER_OP(my_transposerowsp)\n    .setForward(forward, \"cpu\")\n    .setBackward(backward, \"cpu\")\n    .setParseAttrs(parseAttrs)\n    .setInferType(inferType)\n    .setInferSType(inferSType)\n    .setInferShape(inferShape);\n\n/* ------------------------------------------------------------------------- */\n\nclass MyStatefulTransposeRowSP : public CustomStatefulOp {\n public:\n  explicit MyStatefulTransposeRowSP(int count, std::unordered_map<std::string, std::string> attrs)\n      : count(count), attrs_(std::move(attrs)) {}\n\n  MXReturnValue Forward(std::vector<MXTensor>* inputs,\n                        std::vector<MXTensor>* outputs,\n                        const OpResource& op_res) override {\n    std::cout << \"Info: keyword + number of forward: \" << ++count << std::endl;\n    return forward(attrs_, inputs, outputs, op_res);\n  }\n\n  MXReturnValue Backward(std::vector<MXTensor>* inputs,\n                         std::vector<MXTensor>* outputs,\n                         const OpResource& op_res) override {\n    return backward(attrs_, inputs, outputs, op_res);\n  }\n\n private:\n  int count;\n  const std::unordered_map<std::string, std::string> attrs_;\n};\n\nMXReturnValue createOpState(const std::unordered_map<std::string, std::string>& attrs,\n                            const MXContext& ctx,\n                            const std::vector<std::vector<unsigned int>>& in_shapes,\n                            const std::vector<int> in_types,\n                            CustomStatefulOp** op_inst) {\n  // testing passing of keyword arguments\n  int count = attrs.count(\"test_kw\") > 0 ? std::stoi(attrs.at(\"test_kw\")) : 0;\n  // creating stateful operator instance\n  *op_inst                = new MyStatefulTransposeRowSP(count, attrs);\n  (*op_inst)->ignore_warn = true;\n  std::cout << \"Info: stateful operator created\" << std::endl;\n  return MX_SUCCESS;\n}\n\nREGISTER_OP(my_state_transposerowsp)\n    .setParseAttrs(parseAttrs)\n    .setInferType(inferType)\n    .setInferSType(inferSType)\n    .setInferShape(inferShape)\n    .setCreateOpState(createOpState, \"cpu\");\n\nMXReturnValue initialize(int version) {\n  if (version >= 10700) {\n    std::cout << \"MXNet version \" << version << \" supported\" << std::endl;\n    return MX_SUCCESS;\n  } else {\n    MX_ERROR_MSG << \"MXNet version \" << version << \" not supported\";\n    return MX_FAIL;\n  }\n}\n"
  },
  {
    "path": "example/extensions/lib_external_ops/CMakeLists.txt",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# specify CXX sources\nFILE(GLOB CXX_SRCS\n  # Required files\n  ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/lib_api.cc\n  # Your custom files\n  ${CMAKE_CURRENT_SOURCE_DIR}/init_lib.cc\n  ${CMAKE_CURRENT_SOURCE_DIR}/min_ex.cc\n  )\n\n# create library & set libraries\nadd_library(external_lib SHARED ${CXX_SRCS})\ntarget_link_libraries(external_lib PUBLIC mxnet)\n\nif(USE_CUDA)\n  # specify GPU sources (optional)\n  FILE(GLOB CU_SRCS \"*.cu\")\n  target_sources(external_lib PUBLIC ${CU_SRCS})\nendif(USE_CUDA)\n"
  },
  {
    "path": "example/extensions/lib_external_ops/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\nExternal Operators Example and Tutorial\n=======================================\n\n## Introduction\n\nExtending MXNet with custom components used to mean distributing a custom fork. This feature allows adding custom components to MXNet by dynamically loading external libraries at runtime. Currently it is only supported on Linux systems (Windows and Mac are __NOT__ supported). \n\n## Getting Started\n\n### Have MXNet Ready\n\nFor this tutorial, clone MXNet from source like:\n```\ngit clone https://github.com/apache/incubator-mxnet.git --recursive --init\n```\n\nBuild MXNet like:\n```\ncp config/linux.cmake config.cmake\nmkdir build\ncd build\ncmake ..\ncmake --build .\n```\n\n## Run An Example\n\nThis example shows compiling a custom backend operator and then dynamically loading it into MXNet at runtime. Go to the **lib_external_ops** directory and follow these steps:\n\n1. Touch or modify the **min_ex.cc** and/or **min_ex-inl.h** file(s)\n2. Go into the **build** directory that was created when building MXNet.\n3. Run `cmake .. -DBUILD_EXTENSION_PATH=$(pwd)/../example/extensions/lib_external_ops`\n4. Run `cmake --build .`\n5. Go to the **example/extensions/lib_external_ops** directory again\n6. Run `python test_loading.py` to execute the test program. You should see the following output:\n```\nOperator not registered yet\nMXNet version 20000 supported\n[]\nOperator executed successfully\n```\n\n## Writing an External Operator Library\nTo build your own library containing custom components, compose a C++ source file like `mycomp_lib.cc`, include the `lib_api.h` header file, compile the `lib_api.cc` file, and implement the following required function:\n- `initialize` - Library Initialization Function\n\nThen create a CMakeLists.txt file and set `mxnet` as a link library like:\n```\nadd_library(external_lib SHARED ${SRCS})\ntarget_link_libraries(external_lib PUBLIC mxnet)\n```\n\nNext, build MXNet and set the path to your directory with the CMakeLists.txt file via the `BUILD_EXTENSION_PATH` option. This will build your library with all of the MXNet includes. \n"
  },
  {
    "path": "example/extensions/lib_external_ops/init_lib.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file init_lib.cc\n * \\brief initialize function implementation library file\n */\n\n#include <iostream>\n#include \"mxnet/lib_api.h\"\n\nusing namespace mxnet::ext;\n\nMXReturnValue initialize(int version) {\n  if (version >= 10700) {\n    std::cout << \"MXNet version \" << version << \" supported\" << std::endl;\n    return MX_SUCCESS;\n  } else {\n    MX_ERROR_MSG << \"MXNet version \" << version << \" not supported\";\n    return MX_FAIL;\n  }\n}\n"
  },
  {
    "path": "example/extensions/lib_external_ops/min_ex-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file min_ex-inl.h\n * \\brief example external operator header file\n */\n\n#ifndef MXNET_OPERATOR_TENSOR_MIN_EX_OP_INL_H_\n#define MXNET_OPERATOR_TENSOR_MIN_EX_OP_INL_H_\n\n#include <dmlc/parameter.h>\n#include <vector>\n#include <algorithm>\n#include \"operator/mxnet_op.h\"\n#include \"operator/operator_common.h\"\n#include \"operator/elemwise_op_common.h\"\n\nnamespace mxnet {\nnamespace op {\n\ntemplate <typename xpu>\nvoid MinExForward(const nnvm::NodeAttrs& attrs,\n                  const OpContext& ctx,\n                  const std::vector<TBlob>& inputs,\n                  const std::vector<OpReqType>& req,\n                  const std::vector<TBlob>& outputs) {\n  // do nothing\n}\n\ninline bool MinExOpShape(const nnvm::NodeAttrs& attrs,\n                         mxnet::ShapeVector* in_attrs,\n                         mxnet::ShapeVector* out_attrs) {\n  // do nothing\n  return true;\n}\n\ninline bool MinExOpType(const nnvm::NodeAttrs& attrs,\n                        std::vector<int>* in_attrs,\n                        std::vector<int>* out_attrs) {\n  // do nothing\n  return true;\n}\n\n}  // namespace op\n}  // namespace mxnet\n\n#endif  // MXNET_OPERATOR_TENSOR_MIN_EX_OP_INL_H_\n"
  },
  {
    "path": "example/extensions/lib_external_ops/min_ex.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file min_ex.cc\n * \\brief example external operator source file\n */\n\n#include \"min_ex-inl.h\"\n\nnamespace mxnet {\nnamespace op {\n\nNNVM_REGISTER_OP(min_ex)\n    .describe(\"some description\")\n    .set_num_inputs(0)\n    .set_num_outputs(0)\n    .set_attr<mxnet::FInferShape>(\"FInferShape\", MinExOpShape)\n    .set_attr<nnvm::FInferType>(\"FInferType\", MinExOpType)\n    .set_attr<FCompute>(\"FCompute<cpu>\", MinExForward<cpu>);\n\n}  // namespace op\n}  // namespace mxnet\n"
  },
  {
    "path": "example/extensions/lib_external_ops/min_ex.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file min_ex.cu\n * \\brief example external operator CUDA source file\n */\n\n#include \"./min_ex-inl.h\"\n\nnamespace mxnet {\nnamespace op {\n\nNNVM_REGISTER_OP(min_ex).set_attr<FCompute>(\"FCompute<gpu>\", MinExForward<gpu>);\n\n}  // namespace op\n}  // namespace mxnet\n"
  },
  {
    "path": "example/extensions/lib_external_ops/test_loading.py",
    "content": "#!/usr/bin/env python3\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=arguments-differ\n\n# This test checks if dynamic loading of library into MXNet is successful\n# and checks the computation of an external operator\n\nimport mxnet as mx\nimport os\n\n# check if operator exists\nif hasattr(mx.nd, 'min_ex'):\n    raise Exception('Operator already loaded')\nelse:\n    print('Operator not registered yet')\n\n# test loading library\nif (os.name == 'posix'):\n    path = os.path.abspath('build/libexternal_lib.so')\n    mx.library.load(path, False)\n\n# execute operator\nprint(mx.nd.min_ex())\nprint('Operator executed successfully')\n"
  },
  {
    "path": "example/extensions/lib_pass/Makefile",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nall: pass_lib\n\npass_lib:\n\tg++ -shared -fPIC -std=c++11 pass_lib.cc ../../../src/lib_api.cc -o libpass_lib.so -I ../../../include\n\nclean:\n\trm -rf libpass_lib.so\n"
  },
  {
    "path": "example/extensions/lib_pass/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\nCustom Graph Pass Example and Tutorial\n=======================================\n\n## Introduction\n\nAdding custom graph passes in MXNet used to require deep understanding of the MXNet backend, including nnvm pass registration and other internal classes, followed by recompiling MXNet from source. This feature allows adding custom graph passes by dynamically loading external libraries at runtime.\n\nThis custom graph pass feature enables users to write custom model modification strategies without compiling against all of MXNet header files and dependencies. When a library containing custom passes is loaded dynamically, the components found in the library will be registered in MXNet so that users can use those natively just like other built-in components.\n\n## Getting Started\n\n### Have MXNet Ready\n\nTo run the following example, the build type of MXNet doesn’t matter since the custom pass doesn’t interact with the execution of other native MXNet features. Note that if you want to use your custom pass with models running on GPU, you still need an MXNet CUDA build. \n\n### Run An Example\n\nYou can start getting familiar with custom passes by running an example provided in the **example/extensions/lib_pass** directory. The `myPass` example just prints out the graph. Go to the **lib_pass** directory and follow these steps:\n\n1. Run `make`. The Makefile will generate the dynamic library **libpass_lib.so** which is compiled from the `pass_lib.cc` file. This is the library you are going to load that contains everything for the custom pass.\n2. Run `python test_pass.py`. It’ll first load the above library, find the components, register them in the MXNet backend, then execute the pass on the model and execute the operators like a regular MXNet operator and output the result. Below is the output when running the `python test_pass.py` command. Notice that it loads 1 pass: `myPass`.\n\n```\n[10:38:03] src/c_api/c_api.cc:286: Found 0 operators in library\n[10:38:03] src/c_api/c_api.cc:785: Found 0 partitioners in library\n[07:14:00] src/c_api/c_api.cc:887: Found 1 graph passes in library\n[07:14:00] src/c_api/c_api.cc:902:       Graph Pass [0] myPass\n```\n\n### Basic Files For Custom Pass Library\n* **lib_pass/pass_lib.cc**: This file has a source code implementation of all required components to make a custom pass, it also shows registration of them so that they can be loaded by MXNet.\n* **lib_pass/Makefile**: This file compiles the source code to a dynamic shared library, with a header file `include/mxnet/lib_api.h` from MXNet source code. Currently the custom pass is compatible with C++11 and above.\n* **lib_pass/test_pass.py**: This file calls `mx.library.load(‘libpass_lib.so’)` to load the library containing the custom components, executes the pass on the model using the `optimize_for` API, and prints outputs of the forward passes. The outputs should be the same as the regular MXNet forward pass without running the pass.\n* **include/mxnet/lib_api.h**: This file from MXNet source code is the single header file needed to include all necessary data types and function prototypes for writing a custom library. You can either specify the include path in the `Makefile`, or copy the header file over to `example/extensions/lib_pass` folder. Note that apart from this header, the custom library is independent of MXNet source.\n## Writing Custom Pass Library\nTo build your own library containing a custom pass, compose a C++ source file like `mypass_lib.cc`, include `lib_api.h` header file, and write your custom pass with these essential functions:\n- `initialize` - Library Initialization Function\n- `REGISTER_PASS` - Pass Registration Macro\n- `graphPass` - Pass Implementation\nThen compile it to the `mypass_lib.so` dynamic library using the following command:\n```bash\ng++ -shared -fPIC -std=c++11 mypass_lib.cc -o libmypass_lib.so -I ../../../include/mxnet\n```\n\nFinally, you can write a Python script to load the library and execute your pass on a model:\n\n```python\nimport mxnet as mx\nmx.library.load(‘libmypass_lib.so’)\nsym, _, _ = mx.model.load_checkpoint('mymodel', 0) \n# Symbol/Module flow\nsym2 = sym.optimize_for(\"myPass\")\n# Gluon flow 1\nsym_block = nn.SymbolBlock(sym, inputs)\nsym_block.hybridize(static_alloc=True, static_shape=True)\nsym_block.optimize_for(x, backend='myPass')\n# Gluon flow 2\nsym_block = nn.SymbolBlock(sym, inputs)\nsym_block.optimize_for(x, backend='myPass')\n```\n\n### Using a Custom Pass Library\n\nAPIs in MXNet are available in both Symbol and Gluon APIs. For the Symbol API, `optimize_for` can be called on Symbol objects to run the graph pass and return a new Symbol.\n\n```python\nsym.optimize_for(backend, args=None, aux=None, ctx=None, **kwargs)\n```\n\nThe `optimize_for` API takes at least 1 argument, `backend` which is a string that identifies which backend to use to optimize the model. The `args` and `aux` arguments are optional and take a list of NDArray or dict of str to NDArray. They are used to infer shapes and types and before executing the graph pass. The `ctx` argument is optional and takes a device context to infer storage types. It also takes any other user-specified options that will be passed to the backend APIs (in the `kwargs`).\n\n```python\nblock.optimize_for(x, backend=None, backend_opts=None, **kwargs)\n```\n\nWhen the `optimize_for` API is called on a HybridBlock it runs the graph pass immediately. This lets users export the modified model without running a complete forward pass.\n\n```python\nblock.optimize_for(x, backend='myPass')\nblock.export('optimized')\n```\n\nBut you can also use `optimize_for` and run inference immediately after too.\n\n```python\nblock.optimize_for(x, backend='myPass')\nblock(x)\n```\n\n### Writing A Custom Graph Pass\n\nThere are several essential building blocks for making a custom pass:\n\n* [initialize](./pass_lib.cc#44):\n    * This function is the library initialization function necessary for any dynamic libraries. It lets you check if the user is using a compatible version of MXNet. Note that this `version` parameter is passed from MXNet when library is loaded.\n```c++\n            MXReturnValue initialize(int version)\n```\n* [graphPass](./pass_lib.cc#31):\n    * This function provides a copy of the model graph, and any specific options from the user.\n```c++\n            MXReturnValue graphPass(\n                mxnet::ext::Graph *g,\n                const std::unordered_map<std::string, std::string>& options)\n```\n* [REGISTER_PASS(my_pass_name)](./pass_lib.cc#L41):\n    * This macro registers the custom pass and its properties to MXNet by its name. The argument to `setBody` is the `graphPass` function.\n```c++\n            REGISTER_PASS(my_pass_name)\n            .setBody(graphPass);\n```\nLet’s take a closer look at those registry functions:\n\n* **graphPass**: This function takes two arguments. The first argument is the Graph of the model architecture, where nodes are inputs/params/weights and edges are data dependencies. The second argument is the map of options specified by the user. Users can pass custom options to the pass and they are passed to this function in the `options` map.\n\n### Graph representation\n\nThe `Graph` class represents the model's architecture. Each `Node` in the graph represents an operator or weight (ie. args/aux param). Since an operator in MXNet can take multiple inputs and produce multiple outputs, each input/output is represented by a `NodeEntry`. A `Node` contains the following:\n- `op` - [string] operator name\n- `name` - [string] unique node name\n- `inputs` - [vector of NodeEntry] set of inputs to the node\n- `outputs` - [vector of NodeEntry] set of outputs from the node\n- `subgraph` - [vector of Graph] set of subgraphs in the node\n- `attrs` - [map of string to string] set of attributes for the node\n\nThe `inputs` are a set of `NodeEntry` where each contains a pointer to a `Node` that produces the data, and an `entry` that is the index of the output on the other `Node`. Conversely, the `outputs` are a set of `NodeEntry` where each contains a pointer to a`Node` that consumes the data, and and `entry` that is the index of the input on the other `Node`. This bidirectional dependency will enable you to easily traverse the graph. \n\nA `Graph` contains the following:\n- `nodes` - [vector of Node] set of nodes in the graph\n- `inputs` - [vector of Node] set of inputs to the graph\n- `outputs` - [vector of NodeEntry] set of outputs from the graph\n- `attrs` - [map of string to JSON object] set of attributes for the graph\n\nThe `nodes` are all the nodes in the graph (superset). The `inputs` are only those nodes that are model inputs (ie. input image) or weights (ie. arg/aux params). The `outputs` are the outputs from the operators in the model that are true outputs of the model (ie. prediction results). \n\nHeres an example creating a new node and adding it to the graph:\n```c++\ng->addNode(\"myConv\",\"Convolution\");\n```\nHeres an example creating an edge between two nodes:\n```c++\nn1->outputs.push_back({n2,1});\nn2->inputs.push_back({n1,0});\n```\nHere node `n1` produces an output at index 0 that is consumed by node `n2` on the input at index 1.\n\n![example connection](example_connection.png)\n\nSome graph passes require allocating new NDArrays to add/replace model params. The `alloc_arg` and `alloc_aux` APIs enable allocating new NDArrays and integrate them with the model args and aux params. Both APIs have the following signature:\n\n```c++\n    MXTensor* alloc_xxx(const std::vector<int64_t>& shapes,\n                        const MXContext &ctx,\n                        MXDType dtype)\n```\n\nThis function can be called on a node in the graph to allocate a tensor for that node like:\n\n```c++\nnode->alloc_arg({1},MXContext::CPU(0),kFloat32);\n```\nIt adds a new param to the appropriate arg/aux set when the graph pass returns. If you wish to remove an existing param, just remove the node in the graph corresponding to that param. It will be deleted after the pass completes and removed from the dictionary of args or aux (whichever it is a member of).\n\n### Parsing a JSON string\n\nTo simplify custom libraries, basic JSON parsing utility functions have been implemented in the `lib_api.h` header file. You create a `JsonParser` object and parse the string by calling the `parse_to_json` API like:\n\n```c++\nJsonVal json_val = JsonVal::parse(json);\n```\n\nA `JsonVal` is a class that represents the nodes in a JSON structure. You can check the type of a node (num, str, list, or map) by comparing the `JsonVal.type` to `STR`, `NUM`, `LIST`, or `MAP`. Then you can get that value from the node like:\n\n```c++\nswitch(json_val.type) {\n  case STR:\n    std::string str = json_val.str;\n    break;\n  case NUM:\n    int num = json_val.num;\n    break;\n  case LIST:\n    std::vector<JsonVal> list = json_val.list;\n    break;\n  case MAP:\n    std::map<JsonVal, JsonVal> map = json_val.map;\n    break;\n  default:\n    // error\n}\n```\n\nYou call the `dump` function on a `JsonVal` object like `json_val.dump()` to get a JSON-compatible string. There are also convenience constructors for creating `JsonVal` objects for strings and numbers like `JsonVal(\"myKey\")` or `JsonVal(42)`. This makes it easy to get specific keys from a map like `json_val.map[JsonVal(\"nodes\")]`.\n"
  },
  {
    "path": "example/extensions/lib_pass/pass_lib.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file subgraph_lib.cc\n * \\brief subgraph operator implementation library file\n */\n\n#include <cmath>\n#include <iostream>\n#include <algorithm>\n#include \"mxnet/lib_api.h\"\n\nusing namespace mxnet::ext;\n\n/* \\brief a basic pass that prints out the options and the graph */\nMXReturnValue myPass(mxnet::ext::Graph* g,\n                     const std::unordered_map<std::string, std::string>& options) {\n  for (auto kv : options) {\n    std::cout << \"option: \" << kv.first << \" ==> \" << kv.second << std::endl;\n  }\n  g->print();\n  return MX_SUCCESS;\n}\n\nREGISTER_PASS(myPass).setBody(myPass);\n\nMXReturnValue initialize(int version) {\n  if (version >= 10700) {\n    std::cout << \"MXNet version \" << version << \" supported\" << std::endl;\n    return MX_SUCCESS;\n  } else {\n    MX_ERROR_MSG << \"MXNet version \" << version << \" not supported\" << std::endl;\n    return MX_FAIL;\n  }\n}\n"
  },
  {
    "path": "example/extensions/lib_pass/test_pass.py",
    "content": "#!/usr/bin/env python3\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=arguments-differ\n\n# This test checks if dynamic loading of library into MXNet is successful\n# and checks the end of end computation of custom operator\n\nimport os, ctypes\nimport mxnet as mx\nfrom mxnet.gluon import nn\nfrom mxnet import nd\nfrom mxnet.base import _LIB, check_call, mx_uint, c_str, c_str_array, SymbolHandle\n\n# load library\nif (os.name=='posix'):\n    path = os.path.abspath('libpass_lib.so')\n    mx.library.load(path)\nelif (os.name=='nt'):\n    path = os.path.abspath('libpass_lib.dll')\n    mx.library.load(path)\n\n###############################################\n# Test with not consuming params\n###############################################\n# example model, ops do not have args (use outputs from other ops as inputs)\na = mx.sym.var('a')\nb = mx.sym.var('b')\nc = a + b\nd = mx.sym.exp(c)\nsym = mx.sym.log(d)\n\ndef test_model(pass_name):\n    args={'a':mx.nd.ones((3,2)), 'b':mx.nd.ones((3,2))}\n    # execute in MXNet\n    print('-------------------------------')\n    print('Testing regular MXNet execution')\n    inputs = [a,b]\n    sym_block = nn.SymbolBlock(sym, inputs)\n    sym_block.initialize()\n    out = sym_block(mx.nd.ones((3,2)),mx.nd.ones((3,2)))\n    print(out)\n\n    # Gluon optimize_for\n    print('-------------------------------')\n    print(f'Testing pass \"{pass_name}\" Gluon Hybridize with shapes/types without inference')\n    inputs = [a,b]\n    sym_block2 = nn.SymbolBlock(sym, inputs)\n    sym_block2.initialize()\n    sym_block2.optimize_for(mx.nd.ones((3,2)), mx.nd.ones((3,2)), backend=pass_name)\n    sym_block2.export('modified')\n\ntest_model('myPass')\n"
  },
  {
    "path": "example/extensions/lib_subgraph/Makefile",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nall: subgraph_lib\n\nsubgraph_lib:\n\tg++ -shared -fPIC -std=c++11 subgraph_lib.cc ../../../src/lib_api.cc -o libsubgraph_lib.so -I ../../../include\n\nclean:\n\trm -rf libsubgraph_lib.so\n"
  },
  {
    "path": "example/extensions/lib_subgraph/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\nCustom Partitioner Example and Tutorial\n=======================================\n\n## Introduction\n\nAdding custom model partitioners in MXNet used to require deep understanding of the MXNet backend, including operator registration and other internal classes, followed by recompiling MXNet from source. This feature allows adding custom partitioners by dynamically loading external libraries at runtime.\n\nThis custom partitioner feature enables users to write custom model partitioning strategies without compiling against all of MXNet header files and dependencies. When a library containing custom partitioners is loaded dynamically, the components found in the library will be registered in MXNet so that users can use those natively just like other built-in components.\n\n## Getting Started\n\n### Have MXNet Ready\n\nTo run the following example, the build type of MXNet doesn’t matter since the custom partitioner doesn’t interact with the execution of other native MXNet features. Note that if you want to use your custom partitioners with models running on GPU, you still need an MXNet CUDA build. \n\n### Run An Example\n\nYou can start getting familiar with custom partitioners by running an example provided in the **example/extensions/lib_subgraph** directory. This example partitions `exp` and `log` operators into subgraphs. Go to the **lib_subgraph** directory and follow these steps:\n\n1. Run `make`. The Makefile will generate the dynamic library **libsubgraph_lib.so** which is compiled from the `subgraph_lib.cc` file. This is the library you are going to load that contains everything for the custom partitioner.\n2. Run `python test_subgraph.py`. It’ll first load the above library, find the components, register them in the MXNet backend, then partition the model and execute the operators like a regular MXNet operator and output the result. Below is the output when running the `python test_subgraph.py` command. Notice that it loads 2 operators: my_gemm and state_gemm.\n\n```\n[02:01:18] src/c_api/c_api.cc:515: Found 1 operators in library\n[02:01:18] src/c_api/c_api.cc:580: \tOp[0] _custom_subgraph_op\n[02:01:18] src/c_api/c_api.cc:581: \t\tisSubgraphOp\n[02:01:18] src/c_api/c_api.cc:1121: Found 2 partitioners in library\n[02:01:18] src/c_api/c_api.cc:1137: \tPartitioner[0] myProp\n[02:01:18] src/c_api/c_api.cc:1159: \t\tStrategy[0] strategy1 subgraphOp: '_custom_subgraph_op'\n[02:01:18] src/c_api/c_api.cc:1137: \tPartitioner[1] mySelect\n[02:01:18] src/c_api/c_api.cc:1159: \t\tStrategy[0] strategy1 subgraphOp: '_custom_subgraph_op'\n[02:01:18] src/c_api/c_api.cc:1182: Found 1 graph passes in library\n[02:01:18] src/c_api/c_api.cc:1197: \tGraph Pass [0] addInputPass\n```\n\n### Basic Files For Custom Partitioner Library\n\n* **lib_subgraph/subgraph_lib.cc**: This file has a source code implementation of all required components to make a custom partitioner, it also shows registration of them so that they can be loaded by MXNet.\n\n* **lib_subgraph/Makefile**: This file compiles the source code to a dynamic shared library, with a header file `include/mxnet/lib_api.h` from MXNet source code. Currently the custom operator is compatible with C++11 onwards.\n\n* **lib_subgraph/test_subgraph.py**: This file calls `mx.library.load(‘libsubgraph_lib.so’)` to load the library containing the custom components, partitions the model using the `optimize_for` API, and prints outputs of the forward passes. The outputs should be the same as the regular MXNet forward pass without partitioning.\n\n* **include/mxnet/lib_api.h**: This file from MXNet source code is the single header file needed to include all necessary data types and function prototypes for writing a custom operator library. You can either specify the include path in the `Makefile`, or copy the header file over to `example/extensions/lib_subgraph` folder. Note that apart from this header, the custom operator library is independent of MXNet source.\n\n## Writing Custom Partitioner Library\n\nTo build your own library containing a custom partitioner, compose a C++ source file like `mypart_lib.cc`, include `lib_api.h` header file, and write your custom partitioner with these essential functions:\n- `initialize` - Library Initialization Function\n- `REGISTER_PARTITIONER ` - Partitioner Registration Macro\n- `mySupportedOps ` - Operator Support\n\nThen compile it to the `mypart_lib.so` dynamic library using the following command:\n\n```bash\ng++ -shared -fPIC -std=c++11 mypart_lib.cc -o libmypart_lib.so -I ../../../include/mxnet\n```\n\nFinally, you can write a Python script to load the library and partition a model with your custom partitioner:\n\n```python\nimport mxnet as mx\nmx.library.load(‘libmyop_lib.so’)\nsym, _, _ = mx.model.load_checkpoint('mymodel', 0) \n\n# Symbol/Module flow\nsym2 = sym.optimize_for(\"myPart\")\n\n# Gluon flow\nsym_block = nn.SymbolBlock(sym, inputs)\nsym_block.optimize_for(x, backend='myPart')\n```\n\n### Using a Custom Partitioner Library\n\nPartitioning APIs in MXNet are available in both Symbol and Gluon APIs. For the Symbol API, `optimize_for` can be called on Symbol objects to return a partitioned Symbol.\n\n```python\nsym.optimize_for(backend, args=None, aux=None, ctx=None, **kwargs)\n```\n\nThe `optimize_for` API takes at least 1 argument, `backend` which is a string that identifies which backend to partition the model for. The `args` and `aux` arguments are optional and take a list of NDArray or dict of str to NDArray. They are used to infer shapes and types and before partitioning, and passed to the backend to use during compilation. The `ctx` argument is optional and takes a device context to infer storage types. It also takes any other user-specified options that will be passed to the backend partitioning APIs. The backend options can be passed as kwargs.\n\nWhen the `optimize_for` API is called on a HybridBlock it partitions immediately. This lets users export the partitioned model without running a complete forward pass. Chaining multiple optimizations is as simple as calling `optimize_for` multiple times.\n\n```python\nblock.optimize_for(x, backend='myPart')\nblock.optimize_for(x, backend='myOtherPart')\nblock.export('partitioned')\n```\n\nFor the Gluon API, hybridization is needed, so calling `optimize_for` on a non-hybridized block will hybridize it.\nIf the users need to pass some hybridization parameters, they can either call `hybridize` explicitly, or directly pass the arguments to `optimize_for`.\n\nThis:\n```python\nblock.hybridize(static_shape=True, static_alloc=False)\nblock.optimize_for(x, backend='myPart')\n```\nis equivalent to:\n```python\nblock.optimize_for(x, backend='myPart', static_shape=True, static_alloc=False)\n```\n\nIt's important to note that `hybridize` clears the CachedOp and any previous optimization.\n\n```python\nblock.optimize_for(x, backend='myPart')\nblock.hybridize()\n# block is not optimized for myPart anymore!!\n```\n\n### Writing A Custom Partitioner\n\nThere are several essential building blocks for making a custom partitioner:\n\n* [initialize](./subgraph_lib.cc#L261):\n    * This function is the library initialization function necessary for any dynamic libraries. It lets you check if the user is using a compatible version of MXNet. Note that this `version` parameter is passed from MXNet when library is loaded.\n```c++\n            MXReturnValue initialize(int version)\n```\n* [supportedOps](./subgraph_lib.cc#L179):\n    * This function provides a copy of the model Graph, and an interface for identifying which operators should be partitioned into a subgraph. Also this is where a custom partitioner can validate the options specified by the user.\n```c++\n            MXReturnValue supportedOps(\n                const mxnet::ext::Graph* graph,\n                std::vector<int>* ids,\n                const std::unordered_map<std::string, std::string>& options)\n```\n* [REGISTER_PARTITIONER(my_part_name)](./subgraph_lib.cc#L257):\n    * This macro registers the custom partitioner and its properties to MXNet by its name. Notice that a partitioner can have multiple partitioning strategies. This enables multiple *passes* to be run in a single partitioning call from the user. The first argument to `addStrategy` is a user-specified name. The second argument is the name of the subgraph operator to create for each subgraph created during partitioning (see below for more info about subgraph operators). The `setSupportedOps` API registers the `supportedOps` function. The `setReviewSubgraph` API registers a callback function that is called for each subgraph created during partitioning (more on this below). Notice that the first argument to this function is the strategy to associate with and the second argument is the `reviewSubgraph` function.\n```c++\n            REGISTER_PARTITIONER(my_part_name)\n            .addStrategy(\"strategy1\", \"_custom_subgraph_op\")\n            .setSupportedOps(\"strategy1\", supportedOps)\n            .setReviewSubgraph(\"strategy1\", reviewSubgraph);\n```\nAlso there are some optional functions you can specify:\n\n* [reviewSubgraph](./subgraph_lib.cc#L219):\n    * This function provides an opportunity to accept/reject a subgraph after MXNet partitions it. It also allows specifying custom attributes on the subgraph (ie. user-generated IDs). If you do not register this function, subgraphs will be accepted by default. \n```c++\n            MXReturnValue reviewSubgraph(\n                const mxnet::ext::Graph* subgraph,\n                int subgraph_id,\n                bool* accept,\n                const std::unordered_map<std::string, std::string>& options)\n```\nLet’s take a closer look at those registry functions:\n\n* **supportedOps**: This function takes 3 arguments. The 1st argument is the model architecture graph, where nodes are inputs/params/weights and edges are data dependencies. The graph is pre-sorted in topological order. The 2nd argument is an array of integers, one for each operator in the model. When traversing the graph, operators to be partitioned into subgraphs are identified and an entry is set to a value for the index in the `ids` array corresponding to the node ID. Setting a non-negative value (ie. [0, MAX_INT]) indicates the operator should be partitioned into that specific subgraph. Setting a value of -1 indicates that the operator can be partitioned into any subgraph. The last argument is the map of options specified by the user. Users can pass custom options to the partitioner and they are passed to this function in the `options` map. \n\n* **reviewSubgraph**: This function takes four arguments. The 1st argument is the newly partitioned subgraph. The 2nd argument is the subgraph ID, this is just a number MXNet uses to identify this particular subgraph (it starts at zero and increments, unique for each subgraph in the model). The 3rd argument is an output to be set in this function to tell MXNet whether to accept (value: `true`) or reject (value: `false`) the subgraph. You might want to reject a subgraph if it doesnt include all the operators you want, for example. The `options` map is the same one passed to the `supportedOps` API. The 4th argument is the map of options specified by the user. Any custom attributes set on the Graph object will be available later at runtime, and provides a mechanisn to pass info from partition-time to runtime. For inputs to the subgraph that come directly from the params/weights of the model, you can access the raw tensor data directly from that node in the graph.\n\n### Writing a Custom Selector\nInstead of implementing the `supportedOps` API, you can choose to implement a custom selector class for more control over partitioning instead. \n\n* [createSelector](./subgraph_lib.cc#L321):\n    * This function provides a copy of the model graph as the first argument. The 2nd argument is a placeholder for CustomOpSelector object. You must define a class that inherits from the `CustomOpSelector` class and override the required functions. Then you need to create an instance of your class and assign it to the placeholder. The last argument is a map of user-specified options.\n```c++\n            MXReturnValue createSelector(\n                const mxnet::ext::Graph *graph,\n                CustomOpSelector** sel_inst,\n                const std::unordered_map<std::string, std::string>& options)\n```\nInstead of registering a `supportedOps` API, register the `setCreateSelector` API. \n```c++\n            REGISTER_PARTITIONER(my_part_name)\n            .addStrategy(\"strategy1\", \"_custom_subgraph_op\")\n            .setCreateSelector(\"strategy1\", createSelector)\n            .setReviewSubgraph(\"strategy1\", reviewSubgraph);\n```\nWhen implementing your own selector class, you must inherit from the `CustomOpSelector` class and define the following APIs:\n* [Select](./subgraph_lib.cc#L301):\n    * This function selects a node to include in a subgraph by the index of the node (`nodeID`) in the graph. Return `true` to include this node or `false` to reject this node. \n```c++\n            bool Select(\n                int nodeID)\n```\n* [SelectInput](./subgraph_lib.cc#L304):\n    * This function grows the subgraph from a node (`nodeID`) to a node that produces one of its inputs (`input_nodeID`). Return `true` to include this node (`input_nodeID`) or `false` to reject this node. \n```c++\n            bool SelectInput(\n                int nodeID,\n                int input_nodeID)\n```\n* [SelectOutput](./subgraph_lib.cc#L304):\n    * This function grows the subgraph from a node (`nodeID`) to a node that consumes one of its outputs (`output_nodeID`). Return `true` to include this node (`output_nodeID`) or `false` to reject this node. \n```c++\n            bool SelectOutput(\n                int nodeID,\n                int output_nodeID)\n```\nAll of these APIs refer to the model's graph that is provided to the `createSelector` API. When you implement your custom `createSelector` function, you can pass the graph and options to the constructor of your class like this:\n```c++\nMXReturnValue myCreateSelector(const mxnet::ext::Graph *graph,\n                               CustomOpSelector** sel_inst,\n                               const std::unordered_map<std::string, std::string>& options) {\n  *sel_inst = new MySelector(graph, options);\n  return MX_SUCCESS;\n}\n```\nIn addition to the 3 required APIs shown above, you can also implement the following optional APIs for your `CustomOpSelector` class:\n* [Filter](./subgraph_lib.cc#L310):\n    * This function enables reviewing the candidate nodes to include in subgraph. The `candidates` are the indices of nodes in the graph to be included in the subgraph. The 2nd argument `keep` is an empty vector to be filled with the indices of nodes you wish to keep in the subgraph. Any remaining candidate nodes not added to `keep` will be excluded from the subgraph. The following function body shows the default behavior when not overloaded, to keep all candidates:\n```c++\n            void Filter(\n                std::vector<int>& candidates,\n                std::vector<int>& keep) {\n              keep.insert(keep.end(), candidates.begin(), candidates.end());\n            }\n```\n* [Reset](./subgraph_lib.cc#L314):\n    * This function provides an opportunity to reset any selector state between subgraphs. It is called after growing subgraph, and before `Filter`. There is no default behavior.\n```c++\n            virtual void Reset() {}\n```\n\n### Writing A Custom Subgraph Operator\n\nA partitioning strategy specifies how to partition a model and isolate operators into subgraphs. In MXNet, subgraphs are just a [stateful operator](../lib_custom_op#writing-stateful-custom-operator). Subgraph operators have an extra attribute called `MX_STR_SUBGRAPH_SYM_JSON` that maps to a JSON string of the subgraph. The expectation is that when a subgraph operator executes a forward/backward call, it executes all of the operators in the subgraph. \n\nWhen registering a custom subgraph operator, all thats needed is to register a `createOpState` function and to set that the operator is a subgraph operator by calling the `setIsSubgraphOp` API like:\n\n```c++\nREGISTER_OP(my_subgraph_op)\n.setIsSubgraphOp()\n.setCreateOpState(createOpState, \"cpu\");\n```\n\n### Converting a JSON string encoded graph\n\nA Graph object can be created from a JSON string containing a graph/subgraph like:\n\n```c++\nmxnet::ext::Graph* g = mxnet::ext::Graph::fromString(json);\n```\n\nIt can be converted back to a JSON string just as easily:\n```c++\nstd::string json = g->toString();\n```\n\n### Parsing a JSON string\n\nTo simplify custom partitioner libraries, basic JSON parsing utility functions have been implemented in the `lib_api.h` header file. You create a `JsonParser` object and parse the string by calling the `parse_to_json` API like:\n\n```c++\nJsonVal json_val = JsonVal::parse(json);\n```\n\nA `JsonVal` is a class that represents the nodes in a JSON structure. You can check the type of a node (num, str, list, or map) by comparing the `JsonVal.type` to `STR`, `NUM`, `LIST`, or `MAP`. Then you can get that value from the node like:\n\n```c++\nswitch(json_val.type) {\n  case STR:\n    std::string str = json_val.str;\n    break;\n  case NUM:\n    int num = json_val.num;\n    break;\n  case LIST:\n    std::vector<JsonVal> list = json_val.list;\n    break;\n  case MAP:\n    std::map<JsonVal, JsonVal> map = json_val.map;\n    break;\n  default:\n    // error\n}\n```\n\nYou call the `dump` function on a `JsonVal` object like `json_val.dump()` to get a JSON-compatible string. There are also convenience constructors for creating `JsonVal` objects for strings and numbers like `JsonVal(\"myKey\")` or `JsonVal(42)`. This makes it easy to get specific keys from a map like `json_val.map[JsonVal(\"nodes\")]`.\n"
  },
  {
    "path": "example/extensions/lib_subgraph/subgraph_lib.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file subgraph_lib.cc\n * \\brief subgraph operator implementation library file\n */\n\n#include <cmath>\n#include <iostream>\n#include <algorithm>\n#include <utility>\n#include \"mxnet/lib_api.h\"\n\nusing namespace mxnet::ext;\n\n/* function to execute log operator on floats */\nvoid myLog(MXTensor* in, MXTensor* out) {\n  float* inp  = in->data<float>();\n  float* outp = out->data<float>();\n  for (int64_t i = 0; i < in->size(); i++) {\n    outp[i] = logf(inp[i]);\n  }\n}\n/* function to execute exp operator on floats */\nvoid myExp(MXTensor* in, MXTensor* out) {\n  float* inp  = in->data<float>();\n  float* outp = out->data<float>();\n  for (int64_t i = 0; i < in->size(); i++) {\n    outp[i] = expf(inp[i]);\n  }\n}\n\n/* function to execute ops in subgraph\n * In MXNet, subgraphs are sorted in topological order\n * so all we need to do is go through the ops in order\n * and execute each op.\n */\nMXReturnValue myExecutor(std::vector<MXTensor>* inputs,\n                         std::vector<MXTensor>* outputs,\n                         mxnet::ext::Graph* subgraph) {\n  std::cout << \"Info: subgraph is: \" << std::endl;\n  subgraph->print();\n\n  // counter for inputs\n  int input_cnt = 0;\n  // temporary tensor storage\n  std::vector<MXTensor> data;\n  // track memory allocations to free later\n  std::vector<void*> to_free;\n\n  // loop over nodes\n  for (int i = 0; i < subgraph->size(); i++) {\n    mxnet::ext::Node* node = subgraph->getNode(i);\n    // handle each op type\n    if (node->op.compare(\"null\") == 0) {\n      // set tensor for this input to the subgraph\n      node->tensor = &inputs->at(input_cnt++);\n    } else if (node->op.compare(\"log\") == 0) {\n      // get input tensor based on node ID inputs from data storage\n      MXTensor* input = node->inputs.at(0).node->tensor;\n      // create temporary storage\n      MXTensor tmp(malloc(input->size() * 4),\n                   input->shape,\n                   input->dtype,\n                   0,\n                   MXContext::CPU(0),\n                   kDefaultStorage);  // NOLINT\n      // save allocated ptr to free later\n      to_free.push_back(tmp.data_ptr);\n      // execute log operator\n      myLog(input, &tmp);\n      // add output tensor to data storage\n      data.push_back(tmp);\n      // set tensor for this node so we can read it later\n      node->tensor = &data.back();\n    } else if (node->op.compare(\"exp\") == 0) {\n      // get input tensor based on node ID inputs from data storage\n      MXTensor* input = node->inputs.at(0).node->tensor;\n      // create temporary storage\n      MXTensor tmp(malloc(input->size() * 4),\n                   input->shape,\n                   input->dtype,\n                   0,\n                   MXContext::CPU(0),\n                   kDefaultStorage);  // NOLINT\n      // save allocated ptr to free later\n      to_free.push_back(tmp.data_ptr);\n      // execute exp operator\n      myExp(input, &tmp);\n      // add output tensor to data storage\n      data.push_back(tmp);\n      // set tensor for this node so we can read it later\n      node->tensor = &data.back();\n    } else {\n      MX_ERROR_MSG << \"Error! Unsupported op '\" << node->op << \"' found in myExecutor\";\n      // free allocated temporary storage\n      for (void* ptr : to_free)\n        free(ptr);  // NOLINT\n      return MX_FAIL;\n    }\n  }\n\n  // copy all operator results to outputs of subgraph\n  for (int j = 0; j < subgraph->outputs.size(); j++) {\n    // get computed result\n    MXTensor* result = subgraph->outputs[j].node->tensor;\n    // get output tensor to pass to MX\n    MXTensor& out   = outputs->at(j);\n    float* out_data = out.data<float>();\n    float* res_data = result->data<float>();\n    // loop and copy data\n    for (int64_t i = 0; i < result->size(); i++) {\n      out_data[i] = res_data[i];\n    }\n  }\n\n  // free allocated temporary storage\n  for (void* ptr : to_free) {\n    free(ptr);  // NOLINT\n  }\n\n  return MX_SUCCESS;\n}\n\nclass MyStatefulOp : public CustomStatefulOp {\n public:\n  explicit MyStatefulOp(std::string json, const std::unordered_map<std::string, std::string>& attrs)\n      : attrs_(attrs) {\n    for (const auto& kv : attrs) {\n      std::cout << \"subgraphOp attributes: \" << kv.first << \" ==> \" << kv.second << std::endl;\n    }\n    subgraph_ = mxnet::ext::Graph::fromString(json);\n  }\n\n  MXReturnValue Forward(std::vector<MXTensor>* inputs,\n                        std::vector<MXTensor>* outputs,\n                        const OpResource& op_res) override {\n    if (attrs_.count(MX_STR_EXTRA_INPUTS) > 0 && std::stoi(attrs_.at(MX_STR_EXTRA_INPUTS)) > 0)\n      std::cout << \"forward::extra_inputs(\" << attrs_.at(MX_STR_EXTRA_INPUTS) << \")::inputs [\"\n                << inputs->size() << \"]\" << std::endl;\n    return myExecutor(inputs, outputs, subgraph_);\n  }\n\n private:\n  mxnet::ext::Graph* subgraph_;\n  const std::unordered_map<std::string, std::string> attrs_;\n};\n\nMXReturnValue createOpState(const std::unordered_map<std::string, std::string>& attrs,\n                            const MXContext& ctx,\n                            const std::vector<std::vector<unsigned int> >& in_shapes,\n                            const std::vector<int> in_types,\n                            CustomStatefulOp** op_inst) {\n  std::string serialized_subgraph = \"[empty]\";\n  // MXNet subgraph is stored as Symbol in operator node attrs subgraphs field\n  // custom subgraph is stored as json string in custom operator attrs map entry\n  if (attrs.count(MX_STR_SUBGRAPH_SYM_JSON)) {\n    // user can now parse json and run other custom ops inside subgraph\n    serialized_subgraph = attrs.at(MX_STR_SUBGRAPH_SYM_JSON);\n  }\n  *op_inst = new MyStatefulOp(serialized_subgraph, attrs);\n  std::cout << \"Info: stateful operator created\" << std::endl;\n  return MX_SUCCESS;\n}\n\nREGISTER_OP(_custom_subgraph_op).setIsSubgraphOp().setCreateOpState(createOpState, \"cpu\");\n\nconst std::vector<std::string> op_names({\"exp\", \"log\"});\n\nMXReturnValue mySupportedOps(const mxnet::ext::Graph* graph,\n                             std::vector<int>* ids,\n                             const std::unordered_map<std::string, std::string>& options) {\n  for (auto kv : options) {\n    std::cout << \"option: \" << kv.first << \" ==> \" << kv.second << std::endl;\n  }\n\n  // loop over nodes\n  for (int i = 0; i < graph->size(); i++) {\n    const mxnet::ext::Node* node = graph->getNode(i);\n\n    // get shape/type if available\n    std::string shape;\n    int dtype = -1;\n    if (node->attrs.count(\"shape\") > 0)\n      shape = node->attrs.at(\"shape\");\n    if (node->attrs.count(\"dtype\") > 0)\n      dtype = std::stoi(node->attrs.at(\"dtype\"));\n\n    // check if op dtype is float, and if option was specified to require float types\n    if ((dtype == kFloat32 && options.count(\"reqFloat\") > 0) || options.count(\"reqFloat\") == 0) {\n      // check if op is in allowlist\n      if (std::find(op_names.begin(), op_names.end(), node->op.c_str()) != op_names.end()) {\n        // found op in allowlist, set value to -1 to include op in any subgraph\n        ids->at(i) = -1;\n      }\n    }\n  }\n  return MX_SUCCESS;\n}\n\nMXReturnValue myReviewSubgraph(const mxnet::ext::Graph* subgraph,\n                               int subgraph_id,\n                               bool* accept,\n                               const std::unordered_map<std::string, std::string>& options,\n                               std::unordered_map<std::string, std::string>* attrs) {\n  for (auto kv : options) {\n    std::cout << \"option: \" << kv.first << \" ==> \" << kv.second << std::endl;\n  }\n\n  std::string sg = subgraph->toString();\n  std::cout << \"subgraph \" << subgraph_id << \": \" << std::endl;\n  std::cout << sg << std::endl;\n\n  // check if option `reject` was specified, and if so check if value is 'True'\n  if (options.count(\"reject\") > 0 && options.at(\"reject\").compare(\"True\") == 0) {\n    // if specified, reject the subgraph. this is only used for testing\n    *accept = false;\n    std::cout << \"rejecting subgraph\" << std::endl;\n  } else {\n    *accept = true;\n    std::cout << \"accepting subgraph\" << std::endl;\n  }\n\n  attrs->emplace(\"myKey\", \"myVal\");\n\n  return MX_SUCCESS;\n}\n\nREGISTER_PARTITIONER(myProp)\n    .addStrategy(\"strategy1\", \"_custom_subgraph_op\")\n    .setSupportedOps(\"strategy1\", mySupportedOps)\n    .setReviewSubgraph(\"strategy1\", myReviewSubgraph);\n\nclass MySelector : public CustomOpSelector {\n public:\n  MySelector(const mxnet::ext::Graph* graph,\n             const std::unordered_map<std::string, std::string>& options)\n      : graph_(graph), options_(options) {\n    for (auto kv : options) {\n      std::cout << \"selector options: \" << kv.first << \" ==> \" << kv.second << std::endl;\n    }\n  }\n  bool chooseNode(int nodeID) {\n    const mxnet::ext::Node* node = graph_->getNode(nodeID);\n\n    // get shape/type if available\n    std::string shape;\n    int dtype = -1;\n    if (node->attrs.count(\"shape\") > 0)\n      shape = node->attrs.at(\"shape\");\n    if (node->attrs.count(\"dtype\") > 0)\n      dtype = std::stoi(node->attrs.at(\"dtype\"));\n\n    // check if op dtype is float, and if option was specified to require float types\n    if ((dtype == kFloat32 && options_.count(\"reqFloat\") > 0) || options_.count(\"reqFloat\") == 0) {\n      // check if op is in allowlist\n      if (std::find(op_names.begin(), op_names.end(), node->op.c_str()) != op_names.end()) {\n        // found op in allowlist, return true to include op subgraph\n        return true;\n      }\n    }\n    return false;\n  }\n  bool Select(int nodeID) override {\n    return chooseNode(nodeID);\n  }\n  bool SelectInput(int nodeID, int input_nodeID) override {\n    return chooseNode(input_nodeID);\n  }\n  bool SelectOutput(int nodeID, int output_nodeID) override {\n    return chooseNode(output_nodeID);\n  }\n  virtual void Filter(std::vector<int>& candidates, std::vector<int>& keep) {\n    keep.insert(keep.end(), candidates.begin(), candidates.end());\n  }\n  void Reset() override {}\n\n private:\n  const mxnet::ext::Graph* graph_;\n  const std::unordered_map<std::string, std::string> options_;\n};\n\nMXReturnValue createSelector(const mxnet::ext::Graph* graph,\n                             CustomOpSelector** sel_inst,\n                             const std::unordered_map<std::string, std::string>& options) {\n  *sel_inst = new MySelector(graph, options);\n  std::cout << \"Info: selector created\" << std::endl;\n  return MX_SUCCESS;\n}\n\nREGISTER_PARTITIONER(mySelect)\n    .addStrategy(\"strategy1\", \"_custom_subgraph_op\")\n    .setCreateSelector(\"strategy1\", createSelector)\n    .setReviewSubgraph(\"strategy1\", myReviewSubgraph);\n\n/* \\brief a basic pass that adds a new input for subgraph ops */\nMXReturnValue addInputPass(mxnet::ext::Graph* graph,\n                           const std::unordered_map<std::string, std::string>& options) {\n  // find node with '_custom_subgraph_op' op type\n  for (int i = 0; i < graph->size(); i++) {\n    mxnet::ext::Node* n = graph->getNode(i);\n    if (n->op.compare(\"_custom_subgraph_op\") == 0) {\n      // set extra input\n      n->attrs[MX_STR_EXTRA_INPUTS] = std::to_string(1);\n\n      // create a new input Node\n      Node* input = graph->addNode(n->name + \"_input\", \"null\");\n      // set this node as an input in the graph\n      graph->inputs.push_back(input);\n      // connect new input to node\n      input->outputs.push_back({n, (int)(n->inputs.size())});\n      // connect node to new input\n      n->inputs.push_back({input, 0});\n      // add a corresponding tensor for this input\n      input->alloc_arg({1}, MXContext::CPU(0), kFloat32);\n    }\n  }\n\n  return MX_SUCCESS;\n}\n\nREGISTER_PASS(addInputPass).setBody(addInputPass);\n\nMXReturnValue initialize(int version) {\n  if (version >= 10700) {\n    std::cout << \"MXNet version \" << version << \" supported\" << std::endl;\n    return MX_SUCCESS;\n  } else {\n    MX_ERROR_MSG << \"MXNet version \" << version << \" not supported by custom library\" << std::endl;\n    return MX_FAIL;\n  }\n}\n"
  },
  {
    "path": "example/extensions/lib_subgraph/test_subgraph.py",
    "content": "#!/usr/bin/env python3\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=arguments-differ\n\n# This test checks if dynamic loading of library into MXNet is successful\n# and checks the end of end computation of custom operator\n\nimport os, ctypes\nimport mxnet as mx\nfrom mxnet.gluon import nn\nfrom mxnet import nd\nfrom mxnet.base import _LIB, check_call, mx_uint, c_str, c_str_array, SymbolHandle\n\n# load library\nif (os.name=='posix'):\n    path = os.path.abspath('libsubgraph_lib.so')\n    mx.library.load(path)\nelif (os.name=='nt'):\n    path = os.path.abspath('libsubgraph_lib.dll')\n    mx.library.load(path)\n\n# example model, ops to be partitioned do not have args (use outputs from other ops as inputs)\na = mx.sym.var('a')\nb = mx.sym.var('b')\nc = a + b\nd = mx.sym.exp(c)\nsym = mx.sym.log(d)\n\n# example model, ops to be partitioned have args\nd2 = mx.sym.exp(a)\nsym2 = mx.sym.log(d2)\n\ndef test(backend):\n    args = {'a':mx.nd.ones((3,2)), 'b':mx.nd.ones((3,2))}\n    ###############################################\n    # Test with subgraph not consuming params\n    ###############################################\n    #execute in MXNet\n    print('-------------------------------')\n    print('Testing regular Gluon execution')\n    inputs = [a,b]\n    sym_block = nn.SymbolBlock(sym, inputs)\n    sym_block.initialize()\n    out = sym_block(mx.nd.ones((3,2)),mx.nd.ones((3,2)))\n    print(out)\n\n    # Gluon Hybridize partitioning with shapes/types without inference\n    print('-------------------------------')\n    print(f'Testing {backend} Gluon Hybridize partitioning with shapes/types without inference')\n    inputs = [a,b]\n    sym_block2 = nn.SymbolBlock(sym, inputs)\n    sym_block2.initialize()\n    sym_block2.optimize_for(mx.nd.ones((3,2)), mx.nd.ones((3,2)), backend=backend)\n    sym_block2.export('partitioned')\n\n    # Test with additional input to subgraph op\n    print('-------------------------------')\n    print(f'Testing {backend} Gluon Hybridize partitioning with extra input')\n    sym_block2.optimize_for(mx.nd.ones((3,2)), mx.nd.ones((3,2)), backend=\"addInputPass\")\n    out3 = sym_block2(mx.nd.ones((3,2)),mx.nd.ones((3,2)))\n    print(out3)\n    \n    \n    ###############################################\n    # Test with subgraph directly consuming params\n    ###############################################\n    args = {'a':mx.nd.ones((3,2))}\n    #execute in MXNet\n    print('-------------------------------')\n    print('Testing regular MXNet execution')\n    inputs = [a]\n    sym2_block = nn.SymbolBlock(sym2, inputs)\n    sym2_block.initialize()\n    out5 = sym2_block(mx.nd.ones((3,2)))\n    print(out5)\n\n    # Gluon optimize_for partitioning with shapes/types\n    print('-------------------------------')\n    print(f'Testing {backend} Gluon optimize_for partitioning with shapes/types')\n    inputs = [a]\n    sym2_block = nn.SymbolBlock(sym2, inputs)\n    sym2_block.initialize()\n    sym2_block.optimize_for(mx.nd.ones((3,2)), backend=backend)\n    out8 = sym2_block(mx.nd.ones((3,2)))\n    print(out8)\n\ntest(\"myProp\")\ntest(\"mySelect\")\n"
  },
  {
    "path": "example/gluon/actor_critic/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# Actor Critic Model\n\nThis example shows an actor critic model that consists of a critic that measures how good an action taken is and an actor that controls the agent's behavior. \nIn our example actor and critic use the same model:\n\n```\nclass Policy(gluon.Block):\n    def __init__(self, **kwargs):\n        super(Policy, self).__init__(**kwargs)\n        with self.name_scope():\n            self.dense = nn.Dense(16, in_units=4, activation='relu')\n            self.action_pred = nn.Dense(2, in_units=16)\n            self.value_pred = nn.Dense(1, in_units=16)\n\n    def forward(self, x):\n        x = self.dense(x)\n        probs = self.action_pred(x)\n        values = self.value_pred(x)\n        return F.softmax(probs), values\n```\nThe example uses [Gym](https://gym.openai.com/docs/), which is a toolkit for developing and comparing reinforcement learning algorithms. The model is running an instance of [CartPole-v0](https://gym.openai.com/envs/CartPole-v0/) that simulates a pole that is attached by an un-actuated joint to a cart, which moves along a frictionless track. The goal is to prevent it from falling over. \n\n\nThe example provides the following commandline options:\n```\nMXNet actor-critic example\n\noptional arguments:\n  -h, --help        show this help message and exit\n  --gamma G         discount factor (default: 0.99)\n  --seed N          random seed (default: 1)\n  --render          render the environment\n  --log-interval N  interval between training status logs (default: 10)\n\n```\n\nTo run the model execute, type \n```\npython actor_critic.py --render\n```\n\nYou will get an output like the following:\n![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/gluon/actor_critic/actor_critic.gif)\n\n"
  },
  {
    "path": "example/gluon/actor_critic/actor_critic.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nfrom __future__ import print_function\n\nimport argparse\nimport gym\nfrom itertools import count\nimport numpy as onp\n\nimport mxnet as mx\nfrom mxnet import gluon\nfrom mxnet.gluon import nn\nfrom mxnet import autograd, npx\n\n\nparser = argparse.ArgumentParser(description='MXNet actor-critic example')\nparser.add_argument('--gamma', type=float, default=0.99, metavar='G',\n                    help='discount factor (default: 0.99)')\nparser.add_argument('--seed', type=int, default=543, metavar='N',\n                    help='random seed (default: 1)')\nparser.add_argument('--render', action='store_true',\n                    help='render the environment')\nparser.add_argument('--log-interval', type=int, default=10, metavar='N',\n                    help='interval between training status logs (default: 10)')\nargs = parser.parse_args()\n\n\nenv = gym.make('CartPole-v0')\nenv.seed(args.seed)\n\n\nclass Policy(gluon.Block):\n    def __init__(self, **kwargs):\n        super(Policy, self).__init__(**kwargs)\n        self.dense = nn.Dense(16, in_units=4, activation='relu')\n        self.action_pred = nn.Dense(2, in_units=16)\n        self.value_pred = nn.Dense(1, in_units=16)\n\n    def forward(self, x):\n        x = self.dense(x)\n        probs = self.action_pred(x)\n        values = self.value_pred(x)\n        return npx.softmax(probs), values\n\nnet = Policy()\nnet.initialize(mx.init.Uniform(0.02))\ntrainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': 3e-2})\nloss = gluon.loss.L1Loss()\n\nrunning_reward = 10\nfor epoch in count(1):\n    state = env.reset()\n    rewards = []\n    values = []\n    heads = []\n    actions = []\n    with autograd.record():\n        # Sample a sequence of actions\n        for t in range(10000):\n            state = mx.nd.array(onp.expand_dims(state, 0))\n            prob, value = net(state.as_np_ndarray())\n            action, logp = mx.nd.sample_multinomial(prob.as_nd_ndarray(), get_prob=True)\n            state, reward, done, _ = env.step(action.asnumpy()[0])\n            if args.render:\n                env.render()\n            rewards.append(reward)\n            values.append(value.as_np_ndarray())\n            actions.append(action.asnumpy()[0])\n            heads.append(logp)\n            if done:\n                break\n\n        # reverse accumulate and normalize rewards\n        running_reward = running_reward * 0.99 + t * 0.01\n        R = 0\n        for i in range(len(rewards)-1, -1, -1):\n            R = rewards[i] + args.gamma * R\n            rewards[i] = R\n        rewards = onp.array(rewards)\n        rewards -= rewards.mean()\n        rewards /= rewards.std() + onp.finfo(rewards.dtype).eps\n\n        # compute loss and gradient\n        L = sum([loss(value, mx.np.array([r])) for r, value in zip(rewards, values)])\n        final_nodes = [L]\n        for logp, r, v in zip(heads, rewards, values):\n            reward = r - v.asnumpy()[0,0]\n            # Here we differentiate the stochastic graph, corresponds to the\n            # first term of equation (6) in https://arxiv.org/pdf/1506.05254.pdf\n            # Optimizer minimizes the loss but we want to maximizing the reward,\n            # so use we use -reward here.\n            final_nodes.append(logp*(-reward))\n        autograd.backward(final_nodes)\n\n    trainer.step(t)\n\n    if epoch % args.log_interval == 0:\n        print('Episode {}\\tLast length: {:5d}\\tAverage length: {:.2f}'.format(\n            epoch, t, running_reward))\n    if running_reward > 200:\n        print(\"Solved! Running reward is now {} and \"\n              \"the last episode runs to {} time steps!\".format(running_reward, t))\n        break\n"
  },
  {
    "path": "example/gluon/data.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: skip-file\n\"\"\" data iterator for mnist \"\"\"\nimport os\nimport random\nimport tarfile\nimport logging\nimport tarfile\nlogging.basicConfig(level=logging.INFO)\n\nimport mxnet as mx\nfrom mxnet.test_utils import get_cifar10\nfrom mxnet.gluon.data.vision import ImageFolderDataset\nfrom mxnet.gluon.data import DataLoader\nfrom mxnet.contrib.io import DataLoaderIter\n\ndef get_cifar10_iterator(batch_size, data_shape, resize=-1, num_parts=1, part_index=0):\n    get_cifar10()\n\n    train = mx.io.ImageRecordIter(\n        path_imgrec = \"data/cifar/train.rec\",\n        # mean_img    = \"data/cifar/mean.bin\",\n        resize      = resize,\n        data_shape  = data_shape,\n        batch_size  = batch_size,\n        rand_crop   = True,\n        rand_mirror = True,\n        num_parts=num_parts,\n        part_index=part_index)\n\n    val = mx.io.ImageRecordIter(\n        path_imgrec = \"data/cifar/test.rec\",\n        # mean_img    = \"data/cifar/mean.bin\",\n        resize      = resize,\n        rand_crop   = False,\n        rand_mirror = False,\n        data_shape  = data_shape,\n        batch_size  = batch_size,\n        num_parts=num_parts,\n        part_index=part_index)\n\n    return train, val\n\ndef get_imagenet_transforms(data_shape=224, dtype='float32'):\n    def train_transform(image, label):\n        image, _ = mx.image.random_size_crop(image, (data_shape, data_shape), 0.08, (3/4., 4/3.))\n        image = mx.nd.image.random_flip_left_right(image)\n        image = mx.nd.image.to_tensor(image)\n        image = mx.nd.image.normalize(image, mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))\n        return mx.nd.cast(image, dtype), label\n\n    def val_transform(image, label):\n        image = mx.image.resize_short(image, data_shape + 32)\n        image, _ = mx.image.center_crop(image, (data_shape, data_shape))\n        image = mx.nd.image.to_tensor(image)\n        image = mx.nd.image.normalize(image, mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))\n        return mx.nd.cast(image, dtype), label\n    return train_transform, val_transform\n\ndef get_imagenet_iterator(root, batch_size, num_workers, data_shape=224, dtype='float32'):\n    \"\"\"Dataset loader with preprocessing.\"\"\"\n    train_dir = os.path.join(root, 'train')\n    train_transform, val_transform = get_imagenet_transforms(data_shape, dtype)\n    logging.info(\"Loading image folder %s, this may take a bit long...\", train_dir)\n    train_dataset = ImageFolderDataset(train_dir).transform_first(train_transform)\n    train_data = DataLoader(train_dataset, batch_size, shuffle=True,\n                            last_batch='discard', num_workers=num_workers)\n    val_dir = os.path.join(root, 'val')\n    if not os.path.isdir(os.path.expanduser(os.path.join(root, 'val', 'n01440764'))):\n        user_warning = 'Make sure validation images are stored in one subdir per category, a helper script is available at https://git.io/vNQv1'\n        raise ValueError(user_warning)\n    logging.info(\"Loading image folder %s, this may take a bit long...\", val_dir)\n    val_dataset = ImageFolderDataset(val_dir).transform(val_transform)\n    val_data = DataLoader(val_dataset, batch_size, last_batch='keep', num_workers=num_workers)\n    return DataLoaderIter(train_data, dtype), DataLoaderIter(val_data, dtype)\n\ndef get_caltech101_data():\n    url = \"https://s3.us-east-2.amazonaws.com/mxnet-public/101_ObjectCategories.tar.gz\"\n    dataset_name = \"101_ObjectCategories\"\n    data_folder = \"data\"\n    if not os.path.isdir(data_folder):\n        os.makedirs(data_folder)\n    tar_path = mx.gluon.utils.download(url, path=data_folder)\n    if (not os.path.isdir(os.path.join(data_folder, \"101_ObjectCategories\")) or\n        not os.path.isdir(os.path.join(data_folder, \"101_ObjectCategories_test\"))):\n        tar = tarfile.open(tar_path, \"r:gz\")\n        tar.extractall(data_folder)\n        tar.close()\n        print('Data extracted')\n    training_path = os.path.join(data_folder, dataset_name)\n    testing_path = os.path.join(data_folder, \"{}_test\".format(dataset_name))\n    return training_path, testing_path\n\ndef get_caltech101_iterator(batch_size, num_workers, dtype):\n    def transform(image, label):\n        # resize the shorter edge to 224, the longer edge will be greater or equal to 224\n        resized = mx.image.resize_short(image, 224)\n        # center and crop an area of size (224,224)\n        cropped, crop_info = mx.image.center_crop(resized, (224, 224))\n        # transpose the channels to be (3,224,224)\n        transposed = mx.nd.transpose(cropped, (2, 0, 1))\n        return transposed, label\n\n    training_path, testing_path = get_caltech101_data()\n    dataset_train = ImageFolderDataset(root=training_path).transform(transform)\n    dataset_test = ImageFolderDataset(root=testing_path).transform(transform)\n\n    train_data = DataLoader(dataset_train, batch_size, shuffle=True, num_workers=num_workers)\n    test_data = DataLoader(dataset_test, batch_size, shuffle=False, num_workers=num_workers)\n    return DataLoaderIter(train_data), DataLoaderIter(test_data)\n\nclass DummyIter(mx.io.DataIter):\n    def __init__(self, batch_size, data_shape, batches = 100):\n        super(DummyIter, self).__init__(batch_size)\n        self.data_shape = (batch_size,) + data_shape\n        self.label_shape = (batch_size,)\n        self.provide_data = [('data', self.data_shape)]\n        self.provide_label = [('softmax_label', self.label_shape)]\n        self.batch = mx.io.DataBatch(data=[mx.nd.zeros(self.data_shape)],\n                                     label=[mx.nd.zeros(self.label_shape)])\n        self._batches = 0\n        self.batches = batches\n\n    def next(self):\n        if self._batches < self.batches:\n            self._batches += 1\n            return self.batch\n        else:\n            self._batches = 0\n            raise StopIteration\n\ndef dummy_iterator(batch_size, data_shape):\n    return DummyIter(batch_size, data_shape), DummyIter(batch_size, data_shape)\n\nclass ImagePairIter(mx.io.DataIter):\n    def __init__(self, path, data_shape, label_shape, batch_size=64, flag=0, input_aug=None, target_aug=None):\n        super(ImagePairIter, self).__init__(batch_size)\n        self.data_shape = (batch_size,) + data_shape\n        self.label_shape = (batch_size,) + label_shape\n        self.input_aug = input_aug\n        self.target_aug = target_aug\n        self.provide_data = [('data', self.data_shape)]\n        self.provide_label = [('label', self.label_shape)]\n        is_image_file = lambda fn: any(fn.endswith(ext) for ext in [\".png\", \".jpg\", \".jpeg\"])\n        self.filenames = [os.path.join(path, x) for x in os.listdir(path) if is_image_file(x)]\n        self.count = 0\n        self.flag = flag\n        random.shuffle(self.filenames)\n\n    def next(self):\n        from PIL import Image\n        if self.count + self.batch_size <= len(self.filenames):\n            data = []\n            label = []\n            for i in range(self.batch_size):\n                fn = self.filenames[self.count]\n                self.count += 1\n                image = Image.open(fn).convert('YCbCr').split()[0]\n                if image.size[0] > image.size[1]:\n                    image = image.transpose(Image.TRANSPOSE)\n                image = mx.np.expand_dims(mx.np.array(image), axis=2)\n                target = image.copy()\n                for aug in self.input_aug:\n                    image = aug(image)\n                for aug in self.target_aug:\n                    target = aug(target)\n                data.append(image)\n                label.append(target)\n\n            data = mx.np.concatenate([mx.np.expand_dims(d, axis=0) for d in data], axis=0)\n            label = mx.np.concatenate([mx.np.expand_dims(d, axis=0) for d in label], axis=0)\n            data = [mx.np.transpose(data, axes=(0, 3, 1, 2)).astype('float32')/255]\n            label = [mx.np.transpose(label, axes=(0, 3, 1, 2)).astype('float32')/255]\n\n            return mx.io.DataBatch(data=data, label=label)\n        else:\n            raise StopIteration\n\n    def reset(self):\n        self.count = 0\n        random.shuffle(self.filenames)\n"
  },
  {
    "path": "example/gluon/house_prices/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# House Prices: Advanced Regression Techniques\n\nThis example shows how to predict house prices and it is based on the [House Price Kaggle challenge](https://www.kaggle.com/c/house-prices-advanced-regression-techniques#description)\n\nFirst you need to download train and test data set from here:\n```\nhttps://www.kaggle.com/c/house-prices-advanced-regression-techniques/download/train.csv\nhttps://www.kaggle.com/c/house-prices-advanced-regression-techniques/download/test.csv\n```\nAfterwards you can execute the script with  ```python kaggle_k_fold_cross_validation.py```\n\nFor a detailed explanation of the code, you can check out this [chapter](http://d2l.ai/chapter_deep-learning-basics/kaggle-house-price.html) of the Dive into Deep Learning book.\n"
  },
  {
    "path": "example/gluon/house_prices/kaggle_k_fold_cross_validation.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\n# This example provides an end-to-end pipeline for a common Kaggle competition.\n# The entire pipeline includes common utilities such as k-fold cross validation\n# and data pre-processing.\n#\n# Specifically, the example studies the `House Prices: Advanced Regression\n# Techniques` challenge as a case study.\n#\n# The link to the problem on Kaggle:\n# https://www.kaggle.com/c/house-prices-advanced-regression-techniques\n\nimport numpy as onp\nimport pandas as pd\nfrom mxnet import autograd\nfrom mxnet import gluon\nfrom mxnet import np\n\n# After logging in www.kaggle.com, the training and testing data sets can be downloaded at:\n# https://www.kaggle.com/c/house-prices-advanced-regression-techniques/download/train.csv\n# https://www.kaggle.com/c/house-prices-advanced-regression-techniques/download/test.csv\ntrain = pd.read_csv(\"train.csv\")\ntest = pd.read_csv(\"test.csv\")\nall_X = pd.concat((train.loc[:, 'MSSubClass':'SaleCondition'],\n                      test.loc[:, 'MSSubClass':'SaleCondition']))\n\n# Get all the numerical features and apply standardization.\nnumeric_feas = all_X.dtypes[all_X.dtypes != \"object\"].index\nall_X[numeric_feas] = all_X[numeric_feas].apply(lambda x:\n                                                (x - x.mean()) / (x.std()))\n# Convert categorical feature values to numerical (including N/A).\nall_X = pd.get_dummies(all_X, dummy_na=True)\n# Approximate N/A feature value by the mean value of the current feature.\nall_X = all_X.fillna(all_X.mean())\n\nnum_train = train.shape[0]\n\n# Convert data formats to NDArrays to feed into gluon.\nX_train = all_X[:num_train].as_matrix()\nX_test = all_X[num_train:].as_matrix()\ny_train = train.SalePrice.as_matrix()\n\nX_train = np.array(X_train)\ny_train = np.array(y_train)\ny_train.reshape((num_train, 1))\n\nX_test = np.array(X_test)\nsquare_loss = gluon.loss.L2Loss()\n\ndef get_rmse_log(net, X_train, y_train):\n    \"\"\"Gets root mse between the logarithms of the prediction and the truth.\"\"\"\n    num_train = X_train.shape[0]\n    clipped_preds = np.clip(net(X_train), 1, float('inf'))\n    return np.sqrt(2 * np.sum(square_loss(\n        np.log(clipped_preds), np.log(y_train))).item() / num_train)\n\ndef get_net():\n    \"\"\"Gets a neural network. Better results are obtained with modifications.\"\"\"\n    net = gluon.nn.Sequential()\n    net.add(gluon.nn.Dense(50, activation=\"relu\"))\n    net.add(gluon.nn.Dense(1))\n    net.initialize()\n    return net\n\ndef train(net, X_train, y_train, epochs, verbose_epoch, learning_rate,\n          weight_decay, batch_size):\n    \"\"\"Trains the model.\"\"\"\n    dataset_train = gluon.data.ArrayDataset(X_train, y_train)\n    data_iter_train = gluon.data.DataLoader(dataset_train, batch_size,\n                                            shuffle=True)\n    trainer = gluon.Trainer(net.collect_params(), 'adam',\n                            {'learning_rate': learning_rate,\n                             'wd': weight_decay})\n    net.initialize(force_reinit=True)\n    for epoch in range(epochs):\n        for data, label in data_iter_train:\n            with autograd.record():\n                output = net(data)\n                loss = square_loss(output, label)\n            loss.backward()\n            trainer.step(batch_size)\n            avg_loss = get_rmse_log(net, X_train, y_train)\n        if epoch > verbose_epoch:\n            print(f\"Epoch {epoch}, train loss: {avg_loss}\")\n    return avg_loss\n\ndef k_fold_cross_valid(k, epochs, verbose_epoch, X_train, y_train,\n                       learning_rate, weight_decay, batch_size):\n    \"\"\"Conducts k-fold cross validation for the model.\"\"\"\n    assert k > 1\n    fold_size = X_train.shape[0] // k\n\n    train_loss_sum = 0.0\n    test_loss_sum = 0.0\n    for test_idx in range(k):\n        X_val_test = X_train[test_idx * fold_size: (test_idx + 1) *\n                                                   fold_size, :]\n        y_val_test = y_train[test_idx * fold_size: (test_idx + 1) * fold_size]\n        val_train_defined = False\n        for i in range(k):\n            if i != test_idx:\n                X_cur_fold = X_train[i * fold_size: (i + 1) * fold_size, :]\n                y_cur_fold = y_train[i * fold_size: (i + 1) * fold_size]\n                if not val_train_defined:\n                    X_val_train = X_cur_fold\n                    y_val_train = y_cur_fold\n                    val_train_defined = True\n                else:\n                    X_val_train = np.concatenate([X_val_train, X_cur_fold], axis=0)\n                    y_val_train = np.concatenate([y_val_train, y_cur_fold], axis=0)\n        net = get_net()\n        train_loss = train(net, X_val_train, y_val_train, epochs, verbose_epoch,\n                           learning_rate, weight_decay, batch_size)\n        train_loss_sum += train_loss\n        test_loss = get_rmse_log(net, X_val_test, y_val_test)\n        print(f\"Test loss: {test_loss}\")\n        test_loss_sum += test_loss\n    return train_loss_sum / k, test_loss_sum / k\n\n# The sets of parameters. Better results are obtained with modifications.\n# These parameters can be fine-tuned with k-fold cross-validation.\nk = 5\nepochs = 100\nverbose_epoch = 95\nlearning_rate = 0.3\nweight_decay = 100\nbatch_size = 100\n\ntrain_loss, test_loss = \\\n    k_fold_cross_valid(k, epochs, verbose_epoch, X_train, y_train,\n                       learning_rate, weight_decay, batch_size)\nprint(f\"{k}-fold validation: Avg train loss: {train_loss}, Avg test loss: {test_loss}\")\n\ndef learn(epochs, verbose_epoch, X_train, y_train, test, learning_rate,\n          weight_decay, batch_size):\n    \"\"\"Trains the model and predicts on the test data set.\"\"\"\n    net = get_net()\n    _ = train(net, X_train, y_train, epochs, verbose_epoch, learning_rate,\n                 weight_decay, batch_size)\n    preds = net(X_test).asnumpy()\n    test['SalePrice'] = pd.Series(preds.reshape(1, -1)[0])\n    submission = pd.concat([test['Id'], test['SalePrice']], axis=1)\n    submission.to_csv('submission.csv', index=False)\n\nlearn(epochs, verbose_epoch, X_train, y_train, test, learning_rate,\n      weight_decay, batch_size)\n"
  },
  {
    "path": "example/gluon/image_classification.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nfrom __future__ import division\n\nimport argparse, time, os\nimport logging\n\nimport mxnet as mx\nfrom mxnet import gluon\nfrom mxnet import profiler\nfrom mxnet.gluon import nn\nfrom mxnet.gluon.model_zoo import vision as models\nfrom mxnet import autograd as ag\nfrom mxnet.test_utils import get_mnist_iterator\nfrom mxnet.gluon.metric import Accuracy, TopKAccuracy, CompositeEvalMetric\nimport numpy as np\n\nfrom data import (get_cifar10_iterator, get_imagenet_iterator,\n                  get_caltech101_iterator, dummy_iterator)\n\n# logging\nlogging.basicConfig(level=logging.INFO)\nfh = logging.FileHandler('image-classification.log')\nlogger = logging.getLogger()\nlogger.addHandler(fh)\nformatter = logging.Formatter('%(message)s')\nfh.setFormatter(formatter)\nfh.setLevel(logging.DEBUG)\nlogging.debug('\\n%s', '-' * 100)\nformatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')\nfh.setFormatter(formatter)\n\n# CLI\nparser = argparse.ArgumentParser(description='Train a model for image classification.')\nparser.add_argument('--dataset', type=str, default='cifar10',\n                    help='dataset to use. options are mnist, cifar10, caltech101, imagenet and dummy.')\nparser.add_argument('--data-dir', type=str, default='',\n                  help='training directory of imagenet images, contains train/val subdirs.')\nparser.add_argument('--num-worker', '-j', dest='num_workers', default=4, type=int,\n                    help='number of workers for dataloader')\nparser.add_argument('--batch-size', type=int, default=32,\n                    help='training batch size per device (CPU/GPU).')\nparser.add_argument('--gpus', type=str, default='',\n                    help='ordinates of gpus to use, can be \"0,1,2\" or empty for cpu only.')\nparser.add_argument('--epochs', type=int, default=120,\n                    help='number of training epochs.')\nparser.add_argument('--lr', type=float, default=0.1,\n                    help='learning rate. default is 0.1.')\nparser.add_argument('--momentum', type=float, default=0.9,\n                    help='momentum value for optimizer, default is 0.9.')\nparser.add_argument('--wd', type=float, default=0.0001,\n                    help='weight decay rate. default is 0.0001.')\nparser.add_argument('--seed', type=int, default=123,\n                    help='random seed to use. Default=123.')\nparser.add_argument('--mode', type=str,\n                    help='mode in which to train the model. options are imperative, hybrid')\nparser.add_argument('--model', type=str, required=True,\n                    help='type of model to use. see vision_model for options.')\nparser.add_argument('--use_thumbnail', action='store_true',\n                    help='use thumbnail or not in resnet. default is false.')\nparser.add_argument('--batch-norm', action='store_true',\n                    help='enable batch normalization or not in vgg. default is false.')\nparser.add_argument('--use-pretrained', action='store_true',\n                    help='enable using pretrained model from gluon.')\nparser.add_argument('--prefix', default='', type=str,\n                    help='path to checkpoint prefix, default is current working dir')\nparser.add_argument('--start-epoch', default=0, type=int,\n                    help='starting epoch, 0 for fresh training, > 0 to resume')\nparser.add_argument('--resume', type=str, default='',\n                    help='path to saved weight where you want resume')\nparser.add_argument('--lr-factor', default=0.1, type=float,\n                    help='learning rate decay ratio')\nparser.add_argument('--lr-steps', default='30,60,90', type=str,\n                    help='list of learning rate decay epochs as in str')\nparser.add_argument('--dtype', default='float32', type=str,\n                    help='data type, float32 or float16 if applicable')\nparser.add_argument('--save-frequency', default=10, type=int,\n                    help='epoch frequence to save model, best model will always be saved')\nparser.add_argument('--kvstore', type=str, default='device',\n                    help='kvstore to use for trainer/module.')\nparser.add_argument('--log-interval', type=int, default=50,\n                    help='Number of batches to wait before logging.')\nparser.add_argument('--profile', action='store_true',\n                    help='Option to turn on memory profiling for front-end, '\\\n                         'and prints out the memory usage by python function at the end.')\nparser.add_argument('--builtin-profiler', type=int, default=0, help='Enable built-in profiler (0=off, 1=on)')\nopt = parser.parse_args()\n\n# global variables\nlogger.info('Starting new image-classification task:, %s',opt)\nmx.random.seed(opt.seed)\nmodel_name = opt.model\ndataset_classes = {'mnist': 10, 'cifar10': 10, 'caltech101':101, 'imagenet': 1000, 'dummy': 1000}\nbatch_size, dataset, classes = opt.batch_size, opt.dataset, dataset_classes[opt.dataset]\ndevice = [mx.gpu(int(i)) for i in opt.gpus.split(',')] if opt.gpus.strip() else [mx.cpu()]\nnum_gpus = len(device)\nbatch_size *= max(1, num_gpus)\nlr_steps = [int(x) for x in opt.lr_steps.split(',') if x.strip()]\nmetric = CompositeEvalMetric([Accuracy(), TopKAccuracy(5)])\nkv = mx.kv.create(opt.kvstore)\n\ndef get_model(model, device, opt):\n    \"\"\"Model initialization.\"\"\"\n    kwargs = {'device': device, 'pretrained': opt.use_pretrained, 'classes': classes}\n    if model.startswith('resnet'):\n        kwargs['thumbnail'] = opt.use_thumbnail\n    elif model.startswith('vgg'):\n        kwargs['batch_norm'] = opt.batch_norm\n\n    net = models.get_model(model, **kwargs)\n    if opt.resume:\n        net.load_parameters(opt.resume)\n    elif not opt.use_pretrained:\n        if model in ['alexnet']:\n            net.initialize(mx.init.Normal())\n        else:\n            net.initialize(mx.init.Xavier(magnitude=2))\n    net.cast(opt.dtype)\n    return net\n\nnet = get_model(opt.model, device, opt)\n\ndef get_data_iters(dataset, batch_size, opt):\n    \"\"\"get dataset iterators\"\"\"\n    if dataset == 'mnist':\n        train_data, val_data = get_mnist_iterator(batch_size, (1, 28, 28),\n                                                  num_parts=kv.num_workers, part_index=kv.rank)\n    elif dataset == 'cifar10':\n        train_data, val_data = get_cifar10_iterator(batch_size, (3, 32, 32),\n                                                    num_parts=kv.num_workers, part_index=kv.rank)\n    elif dataset == 'imagenet':\n        shape_dim = 299 if model_name == 'inceptionv3' else 224\n\n        if not opt.data_dir:\n            raise ValueError('Dir containing raw images in train/val is required for imagenet.'\n                             'Please specify \"--data-dir\"')\n\n        train_data, val_data = get_imagenet_iterator(opt.data_dir, batch_size,\n                                                                opt.num_workers, shape_dim, opt.dtype)\n    elif dataset == 'caltech101':\n        train_data, val_data = get_caltech101_iterator(batch_size, opt.num_workers, opt.dtype)\n    elif dataset == 'dummy':\n        shape_dim = 299 if model_name == 'inceptionv3' else 224\n        train_data, val_data = dummy_iterator(batch_size, (3, shape_dim, shape_dim))\n    return train_data, val_data\n\ndef test(device, val_data):\n    metric.reset()\n    val_data.reset()\n    for batch in val_data:\n        data = gluon.utils.split_and_load(batch.data[0].astype(opt.dtype, copy=False),\n                                          device_list=device, batch_axis=0)\n        label = gluon.utils.split_and_load(batch.label[0].astype(opt.dtype, copy=False),\n                                           device_list=device, batch_axis=0)\n        outputs = [net(X) for X in data]\n        metric.update(label, outputs)\n    return metric.get()\n\ndef update_learning_rate(lr, trainer, epoch, ratio, steps):\n    \"\"\"Set the learning rate to the initial value decayed by ratio every N epochs.\"\"\"\n    new_lr = lr * (ratio ** int(np.sum(np.array(steps) < epoch)))\n    trainer.set_learning_rate(new_lr)\n    return trainer\n\ndef save_checkpoint(epoch, top1, best_acc):\n    if opt.save_frequency and (epoch + 1) % opt.save_frequency == 0:\n        fname = os.path.join(opt.prefix, f'{opt.model}_{epoch}_acc_{top1:.4f}.params')\n        net.save_parameters(fname)\n        logger.info(f'[Epoch {epoch}] Saving checkpoint to {fname} with Accuracy: {top1:.4f}')\n    if top1 > best_acc[0]:\n        best_acc[0] = top1\n        fname = os.path.join(opt.prefix, f'{opt.model}_best.params')\n        net.save_parameters(fname)\n        logger.info(f'[Epoch {epoch}] Saving checkpoint to {fname} with Accuracy: {top1:.4f}')\n\ndef train(opt, device):\n    if isinstance(device, mx.Device):\n        device = [device]\n\n    train_data, val_data = get_data_iters(dataset, batch_size, opt)\n    for p in net.collect_params().values():\n        p.reset_device(device)\n    trainer = gluon.Trainer(net.collect_params(), 'sgd',\n                            optimizer_params={'learning_rate': opt.lr,\n                                              'wd': opt.wd,\n                                              'momentum': opt.momentum,\n                                              'multi_precision': True},\n                            kvstore=kv)\n    loss = gluon.loss.SoftmaxCrossEntropyLoss()\n\n    total_time = 0\n    num_epochs = 0\n    best_acc = [0]\n    for epoch in range(opt.start_epoch, opt.epochs):\n        trainer = update_learning_rate(opt.lr, trainer, epoch, opt.lr_factor, lr_steps)\n        tic = time.time()\n        train_data.reset()\n        metric.reset()\n        btic = time.time()\n        for i, batch in enumerate(train_data):\n            data = gluon.utils.split_and_load(batch.data[0].astype(opt.dtype), device_list=device, batch_axis=0)\n            label = gluon.utils.split_and_load(batch.label[0].astype(opt.dtype), device_list=device, batch_axis=0)\n            outputs = []\n            Ls = []\n            with ag.record():\n                for x, y in zip(data, label):\n                    z = net(x)\n                    L = loss(z, y)\n                    # store the loss and do backward after we have done forward\n                    # on all GPUs for better speed on multiple GPUs.\n                    Ls.append(L)\n                    outputs.append(z)\n                ag.backward(Ls)\n            trainer.step(batch.data[0].shape[0])\n            metric.update(label, outputs)\n            if opt.log_interval and not (i+1)%opt.log_interval:\n                name, acc = metric.get()\n                logger.info('Epoch[%d] Batch [%d]\\tSpeed: %f samples/sec\\t%s=%f, %s=%f'%(\n                               epoch, i, batch_size/(time.time()-btic), name[0], acc[0], name[1], acc[1]))\n            btic = time.time()\n\n        epoch_time = time.time()-tic\n\n        # First epoch will usually be much slower than the subsequent epics,\n        # so don't factor into the average\n        if num_epochs > 0:\n          total_time = total_time + epoch_time\n        num_epochs = num_epochs + 1\n\n        name, acc = metric.get()\n        logger.info('[Epoch %d] training: %s=%f, %s=%f'%(epoch, name[0], acc[0], name[1], acc[1]))\n        logger.info('[Epoch %d] time cost: %f'%(epoch, epoch_time))\n        name, val_acc = test(device, val_data)\n        logger.info('[Epoch %d] validation: %s=%f, %s=%f'%(epoch, name[0], val_acc[0], name[1], val_acc[1]))\n\n        # save model if meet requirements\n        save_checkpoint(epoch, val_acc[0], best_acc)\n    if num_epochs > 1:\n        print('Average epoch time: {}'.format(float(total_time)/(num_epochs - 1)))\n\ndef main():\n    if opt.builtin_profiler > 0:\n        profiler.set_config(profile_all=True, aggregate_stats=True)\n        profiler.set_state('run')\n    if opt.mode == 'hybrid':\n        net.hybridize()\n    train(opt, device)\n    if opt.builtin_profiler > 0:\n        profiler.set_state('stop')\n        print(profiler.dumps())\n\nif __name__ == '__main__':\n    if opt.profile:\n        import hotshot, hotshot.stats\n        prof = hotshot.Profile(f'image-classifier-{opt.model}-{opt.mode}.prof')\n        prof.runcall(main)\n        prof.close()\n        stats = hotshot.stats.load(f'image-classifier-{opt.model}-{opt.mode}.prof')\n        stats.strip_dirs()\n        stats.sort_stats('cumtime', 'calls')\n        stats.print_stats()\n    else:\n        main()\n"
  },
  {
    "path": "example/gluon/mnist/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# MNIST classification example\n\nThis script shows a simple example how to do image classification with Gluon. \nThe model is trained on MNIST digits image dataset and the goal is to classify the digits ```0-9```.  The model has the following layout:\n```\nnet = nn.Sequential()\nnet.add(nn.Dense(128, activation='relu'))\nnet.add(nn.Dense(64, activation='relu'))\nnet.add(nn.Dense(10))\n```\n\nThe script provides the following commandline arguments: \n\n\n```\nMXNet Gluon MNIST Example\n\noptional arguments:\n  -h, --help            show this help message and exit\n  --batch-size BATCH_SIZE\n                        batch size for training and testing (default: 100)\n  --epochs EPOCHS       number of epochs to train (default: 10)\n  --lr LR               learning rate (default: 0.1)\n  --momentum MOMENTUM   SGD momentum (default: 0.9)\n  --cuda                Train on GPU with CUDA\n  --log-interval N      how many batches to wait before logging training\n                        status\n```\n\nAfter one epoch we get the following output vector for the given test image:\n\n<img src=\"https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/gluon/mnist/test_image.png\" width=\"250\" height=\"250\">\n\n[-5.461655  -4.745     -1.8203478 -0.5705207  8.923972  -2.2358544 -3.3020825 -2.409004   4.0074944 10.362008] \n\nAs we can see the highest activation is 10.362 which corresponds to label `9`.\n\n"
  },
  {
    "path": "example/gluon/mnist/mnist.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: skip-file\nfrom __future__ import print_function\n\nimport argparse\nimport logging\nlogging.basicConfig(level=logging.DEBUG)\n\nimport numpy as np\nimport mxnet as mx\nfrom mxnet import gluon, autograd\nfrom mxnet.gluon import nn\n\n# Parse CLI arguments\n\nparser = argparse.ArgumentParser(description='MXNet Gluon MNIST Example')\nparser.add_argument('--batch-size', type=int, default=100,\n                    help='batch size for training and testing (default: 100)')\nparser.add_argument('--epochs', type=int, default=10,\n                    help='number of epochs to train (default: 10)')\nparser.add_argument('--lr', type=float, default=0.1,\n                    help='learning rate (default: 0.1)')\nparser.add_argument('--momentum', type=float, default=0.9,\n                    help='SGD momentum (default: 0.9)')\nparser.add_argument('--cuda', action='store_true', default=False,\n                    help='Train on GPU with CUDA')\nparser.add_argument('--log-interval', type=int, default=100, metavar='N',\n                    help='how many batches to wait before logging training status')\nopt = parser.parse_args()\n\n\n# define network\n\nnet = nn.Sequential()\nnet.add(nn.Dense(128, activation='relu'))\nnet.add(nn.Dense(64, activation='relu'))\nnet.add(nn.Dense(10))\n\n# data\n\ndef transformer(data, label):\n    data = data.reshape((-1,)).astype(np.float32)/255\n    return data, label\n\ntrain_data = gluon.data.DataLoader(\n    gluon.data.vision.MNIST('./data', train=True).transform(transformer),\n    batch_size=opt.batch_size, shuffle=True, last_batch='discard')\n\nval_data = gluon.data.DataLoader(\n    gluon.data.vision.MNIST('./data', train=False).transform(transformer),\n    batch_size=opt.batch_size, shuffle=False)\n\n# train\n\ndef test(ctx):\n    metric = mx.gluon.metric.Accuracy()\n    for data, label in val_data:\n        data = data.to_device(ctx)\n        label = label.to_device(ctx)\n        output = net(data)\n        metric.update([label], [output])\n\n    return metric.get()\n\n\ndef train(epochs, ctx):\n    # Collect all parameters from net and its children, then initialize them.\n    net.initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx)\n    # Trainer is for updating parameters with gradient.\n    trainer = gluon.Trainer(net.collect_params(), 'sgd',\n                            {'learning_rate': opt.lr, 'momentum': opt.momentum})\n    metric = mx.gluon.metric.Accuracy()\n    loss = gluon.loss.SoftmaxCrossEntropyLoss()\n\n    for epoch in range(epochs):\n        # reset data iterator and metric at begining of epoch.\n        metric.reset()\n        for i, (data, label) in enumerate(train_data):\n            # Copy data to ctx if necessary\n            data = data.to_device(ctx)\n            label = label.to_device(ctx)\n            # Start recording computation graph with record() section.\n            # Recorded graphs can then be differentiated with backward.\n            with autograd.record():\n                output = net(data)\n                L = loss(output, label)\n                L.backward()\n            # take a gradient step with batch_size equal to data.shape[0]\n            trainer.step(data.shape[0])\n            # update metric at last.\n            metric.update([label], [output])\n\n            if i % opt.log_interval == 0 and i > 0:\n                name, acc = metric.get()\n                print(f'[Epoch {epoch} Batch {i}] Training: {name}={acc}')\n\n        name, acc = metric.get()\n        print(f'[Epoch {epoch}] Training: {name}={acc}')\n\n        name, val_acc = test(ctx)\n        print(f'[Epoch {epoch}] Validation: {name}={val_acc}')\n\n    net.save_parameters('mnist.params')\n\n\nif __name__ == '__main__':\n    if opt.cuda:\n        ctx = mx.gpu(0)\n    else:\n        ctx = mx.cpu()\n    train(opt.epochs, ctx)\n"
  },
  {
    "path": "example/gluon/super_resolution/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# Superresolution\n\nNote: this example use The BSDS500 Dataset which is copyright Berkeley Computer Vision Group.\nFor more details, see [dataset website](https://www2.eecs.berkeley.edu/Research/Projects/CS/vision/grouping/resources.html#bsds500)\n\nThis example trains a convolutional neural network to enhance the resolution of images (also known as superresolution). \nThe script takes the following commandline arguments:\n\n```\nSuper-resolution using an efficient sub-pixel convolution neural network.\n\noptional arguments:\n  -h, --help            show this help message and exit\n  --upscale_factor UPSCALE_FACTOR\n                        super resolution upscale factor. default is 3.\n  --batch_size BATCH_SIZE\n                        training batch size, per device. default is 4.\n  --test_batch_size TEST_BATCH_SIZE\n                        test batch size\n  --epochs EPOCHS       number of training epochs\n  --lr LR               learning Rate. default is 0.001.\n  --use-gpu             whether to use GPU.\n  --seed SEED           random seed to use. Default=123\n  --resolve_img RESOLVE_IMG\n                        input image to use\n```\n\nOnce the network is trained you can use the following command to increase the resolution of your image:\n```\npython  super_resolution.py --resolve_img myimage.jpg\n```\n\n## Citation\n<b>Contour Detection and Hierarchical Image Segmentation\nP. Arbelaez, M. Maire, C. Fowlkes and J. Malik.\nIEEE TPAMI, Vol. 33, No. 5, pp. 898-916, May 2011.\n[PDF](http://web.archive.org/web/20160306133802/http://www.eecs.berkeley.edu/Research/Projects/CS/vision/grouping/papers/amfm_pami2010.pdf)\n[BibTex](http://web.archive.org/web/20160306133802/http://www.eecs.berkeley.edu/Research/Projects/CS/vision/grouping/papers/amfm_pami2011.bib)\n</b>"
  },
  {
    "path": "example/gluon/super_resolution/super_resolution.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nfrom __future__ import print_function\n\nimport argparse\nimport math\nimport os\nimport shutil\nimport sys\nimport zipfile\nfrom os import path\n\nimport numpy as np\n\nimport mxnet as mx\nfrom mxnet import gluon, autograd as ag\nfrom mxnet.gluon import nn\nfrom mxnet.image import CenterCropAug, ResizeAug\nfrom mxnet.io import PrefetchingIter\nfrom mxnet.test_utils import download\n\nthis_dir = path.abspath(path.dirname(__file__))\nsys.path.append(path.join(this_dir, path.pardir))\n\nfrom data import ImagePairIter\n\n\n# CLI\nparser = argparse.ArgumentParser(description='Super-resolution using an efficient sub-pixel convolution neural network.')\nparser.add_argument('--upscale_factor', type=int, default=3, help=\"super resolution upscale factor. default is 3.\")\nparser.add_argument('--batch_size', type=int, default=4, help='training batch size, per device. default is 4.')\nparser.add_argument('--test_batch_size', type=int, default=100, help='test batch size')\nparser.add_argument('--epochs', type=int, default=30, help='number of training epochs')\nparser.add_argument('--lr', type=float, default=0.001, help='learning Rate. default is 0.001.')\nparser.add_argument('--use-gpu', action='store_true', help='whether to use GPU.')\nparser.add_argument('--seed', type=int, default=123, help='random seed to use. Default=123')\nparser.add_argument('--resolve_img', type=str, help='input image to use')\nopt = parser.parse_args()\n\nprint(opt)\n\nupscale_factor = opt.upscale_factor\nbatch_size, test_batch_size = opt.batch_size, opt.test_batch_size\ncolor_flag = 0\n\n# Get data from https://github.com/BIDS/BSDS500/\n# The BSDS500 Dataset is copyright Berkeley Computer Vision Group\n# For more details, see https://www2.eecs.berkeley.edu/Research/Projects/CS/vision/grouping/resources.html#bsds500\ndatasets_dir = path.expanduser(path.join(\"~\", \".mxnet\", \"datasets\"))\ndatasets_tmpdir = path.join(datasets_dir, \"tmp\")\ndataset_url = \"https://github.com/BIDS/BSDS500/archive/master.zip\"\ndata_dir = path.expanduser(path.join(datasets_dir, \"BSDS500\"))\ntmp_dir = path.join(data_dir, \"tmp\")\n\ndef get_dataset(prefetch=False):\n    \"\"\"Download the BSDS500 dataset and return train and test iters.\"\"\"\n\n    if path.exists(data_dir):\n        print(\n            \"Directory {} already exists, skipping.\\n\"\n            \"To force download and extraction, delete the directory and re-run.\"\n            \"\".format(data_dir),\n            file=sys.stderr,\n        )\n    else:\n        print(\"Downloading dataset...\", file=sys.stderr)\n        downloaded_file = download(dataset_url, dirname=datasets_tmpdir)\n        print(\"done\", file=sys.stderr)\n\n        print(\"Extracting files...\", end=\"\", file=sys.stderr)\n        os.makedirs(data_dir)\n        os.makedirs(tmp_dir)\n        with zipfile.ZipFile(downloaded_file) as archive:\n            archive.extractall(tmp_dir)\n        shutil.rmtree(datasets_tmpdir)\n\n        shutil.copytree(\n            path.join(tmp_dir, \"BSDS500-master\", \"BSDS500\", \"data\", \"images\"),\n            path.join(data_dir, \"images\"),\n        )\n        shutil.copytree(\n            path.join(tmp_dir, \"BSDS500-master\", \"BSDS500\", \"data\", \"groundTruth\"),\n            path.join(data_dir, \"groundTruth\"),\n        )\n        shutil.rmtree(tmp_dir)\n        print(\"done\", file=sys.stderr)\n\n    crop_size = 256\n    crop_size -= crop_size % upscale_factor\n    input_crop_size = crop_size // upscale_factor\n\n    input_transform = [CenterCropAug((crop_size, crop_size)), ResizeAug(input_crop_size)]\n    target_transform = [CenterCropAug((crop_size, crop_size))]\n\n    iters = (\n        ImagePairIter(\n            path.join(data_dir, \"images\", \"train\"),\n            (input_crop_size, input_crop_size),\n            (crop_size, crop_size),\n            batch_size,\n            color_flag,\n            input_transform,\n            target_transform,\n        ),\n        ImagePairIter(\n            path.join(data_dir, \"images\", \"test\"),\n            (input_crop_size, input_crop_size),\n            (crop_size, crop_size),\n            test_batch_size,\n            color_flag,\n            input_transform,\n            target_transform,\n        ),\n    )\n\n    return [PrefetchingIter(i) for i in iters] if prefetch else iters\n\ntrain_data, val_data = get_dataset()\n\nmx.np.random.seed(opt.seed)\ndevice = [mx.gpu(0)] if opt.use_gpu else [mx.cpu()]\n\n\nclass SuperResolutionNet(gluon.HybridBlock):\n    def __init__(self, upscale_factor):\n        super(SuperResolutionNet, self).__init__()\n        self.conv1 = nn.Conv2D(64, (5, 5), strides=(1, 1), padding=(2, 2), activation='relu')\n        self.conv2 = nn.Conv2D(64, (3, 3), strides=(1, 1), padding=(1, 1), activation='relu')\n        self.conv3 = nn.Conv2D(32, (3, 3), strides=(1, 1), padding=(1, 1), activation='relu')\n        self.conv4 = nn.Conv2D(upscale_factor ** 2, (3, 3), strides=(1, 1), padding=(1, 1))\n        self.pxshuf = nn.PixelShuffle2D(upscale_factor)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = self.conv3(x)\n        x = self.conv4(x)\n        x = self.pxshuf(x)\n        return x\n\nnet = SuperResolutionNet(upscale_factor)\nmetric = mx.gluon.metric.MSE()\n\ndef test(device):\n    val_data.reset()\n    avg_psnr = 0\n    batches = 0\n    for batch in val_data:\n        batches += 1\n        data = gluon.utils.split_and_load(batch.data[0], device_list=device, batch_axis=0)\n        label = gluon.utils.split_and_load(batch.label[0], device_list=device, batch_axis=0)\n        outputs = []\n        for x in data:\n            outputs.append(net(x))\n        metric.update(label, outputs)\n        avg_psnr += 10 * math.log10(1/metric.get()[1])\n        metric.reset()\n    avg_psnr /= batches\n    print(f'validation avg psnr: {avg_psnr}')\n\n\ndef train(epoch, device):\n    if isinstance(device, mx.Device):\n        device = [device]\n    net.initialize(mx.init.Orthogonal(), device=device)\n    # re-initialize conv4's weight to be Orthogonal\n    net.conv4.initialize(mx.init.Orthogonal(scale=1), force_reinit=True, device=device)\n    trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': opt.lr})\n    loss = gluon.loss.L2Loss()\n\n    for i in range(epoch):\n        train_data.reset()\n        for batch in train_data:\n            data = gluon.utils.split_and_load(batch.data[0], device_list=device, batch_axis=0)\n            label = gluon.utils.split_and_load(batch.label[0], device_list=device, batch_axis=0)\n            outputs = []\n            with ag.record():\n                for x, y in zip(data, label):\n                    z = net(x)\n                    L = loss(z, y)\n                    L.backward()\n                    outputs.append(z)\n            trainer.step(batch.data[0].shape[0])\n            metric.update(label, outputs)\n\n        name, acc = metric.get()\n        metric.reset()\n        print(f'training mse at epoch {i}: {name}={acc}')\n        test(device)\n\n    net.save_parameters(path.join(this_dir, 'superres.params'))\n\ndef resolve(device):\n    from PIL import Image\n\n    if isinstance(device, list):\n        device = [device[0]]\n\n    img_basename = path.splitext(path.basename(opt.resolve_img))[0]\n    img_dirname = path.dirname(opt.resolve_img)\n\n    net.load_parameters(path.join(this_dir, 'superres.params'), device=device)\n    img = Image.open(opt.resolve_img).convert('YCbCr')\n    y, cb, cr = img.split()\n    data = mx.np.expand_dims(mx.np.expand_dims(mx.np.array(y), axis=0), axis=0)\n    out_img_y = mx.np.reshape(net(data), shape=(-3, -2)).asnumpy()\n    out_img_y = out_img_y.clip(0, 255)\n    out_img_y = Image.fromarray(np.uint8(out_img_y[0]), mode='L')\n\n    out_img_cb = cb.resize(out_img_y.size, Image.BICUBIC)\n    out_img_cr = cr.resize(out_img_y.size, Image.BICUBIC)\n    out_img = Image.merge('YCbCr', [out_img_y, out_img_cb, out_img_cr]).convert('RGB')\n\n    out_img.save(path.join(img_dirname, '{}-resolved.png'.format(img_basename)))\n\nif opt.resolve_img:\n    resolve(device)\nelse:\n    train(opt.epochs, device)\n"
  },
  {
    "path": "example/multi-task/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# Mulit-task learning example\n \nThis is a simple example to show how to use mxnet for multi-task learning. It uses MNIST as an example, trying to predict jointly the digit and whether this digit is odd or even.\n\nFor example:\n\n![](https://camo.githubusercontent.com/ed3cf256f47713335dc288f32f9b0b60bf1028b7/68747470733a2f2f7777772e636c61737365732e63732e756368696361676f2e6564752f617263686976652f323031332f737072696e672f31323330302d312f70612f7061312f64696769742e706e67)\n\nShould be jointly classified as 4, and Even.\n\nIn this example we don't expect the tasks to contribute to each other much, but for example multi-task learning has been successfully applied to the domain of image captioning. In [A Multi-task Learning Approach for Image Captioning](https://www.ijcai.org/proceedings/2018/0168.pdf) by Wei Zhao, Benyou Wang, Jianbo Ye, Min Yang, Zhou Zhao, Ruotian Luo, Yu Qiao, they train a network to jointly classify images and generate text captions\n\nPlease refer to the notebook for a fully worked example.\n"
  },
  {
    "path": "example/multi-task/multi-task-learning.ipynb",
    "content": "{\n \"cells\": [\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"# Licensed to the Apache Software Foundation (ASF) under one\\n\",\n    \"# or more contributor license agreements.  See the NOTICE file\\n\",\n    \"# distributed with this work for additional information\\n\",\n    \"# regarding copyright ownership.  The ASF licenses this file\\n\",\n    \"# to you under the Apache License, Version 2.0 (the\\n\",\n    \"# \\\"License\\\"); you may not use this file except in compliance\\n\",\n    \"# with the License.  You may obtain a copy of the License at\\n\",\n    \"#\\n\",\n    \"#   http://www.apache.org/licenses/LICENSE-2.0\\n\",\n    \"#\\n\",\n    \"# Unless required by applicable law or agreed to in writing,\\n\",\n    \"# software distributed under the License is distributed on an\\n\",\n    \"# \\\"AS IS\\\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\\n\",\n    \"# KIND, either express or implied.  See the License for the\\n\",\n    \"# specific language governing permissions and limitations\\n\",\n    \"# under the License.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"# Multi-Task Learning Example\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"This is a simple example to show how to use mxnet for multi-task learning.\\n\",\n    \"\\n\",\n    \"The network is jointly going to learn whether a number is odd or even and to actually recognize the digit.\\n\",\n    \"\\n\",\n    \"\\n\",\n    \"For example\\n\",\n    \"\\n\",\n    \"- 1 : 1 and odd\\n\",\n    \"- 2 : 2 and even\\n\",\n    \"- 3 : 3 and odd\\n\",\n    \"\\n\",\n    \"etc\\n\",\n    \"\\n\",\n    \"In this example we don't expect the tasks to contribute to each other much, but for example multi-task learning has been successfully applied to the domain of image captioning. In [A Multi-task Learning Approach for Image Captioning](https://www.ijcai.org/proceedings/2018/0168.pdf) by Wei Zhao, Benyou Wang, Jianbo Ye, Min Yang, Zhou Zhao, Ruotian Luo, Yu Qiao, they train a network to jointly classify images and generate text captions\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 16,\n   \"source\": [\n    \"import logging\\n\",\n    \"import random\\n\",\n    \"import time\\n\",\n    \"\\n\",\n    \"import matplotlib.pyplot as plt\\n\",\n    \"import mxnet as mx\\n\",\n    \"from mxnet import gluon, np, npx, autograd\\n\",\n    \"import numpy as onp\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"### Parameters\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 99,\n   \"source\": [\n    \"batch_size = 128\\n\",\n    \"epochs = 5\\n\",\n    \"ctx = mx.gpu() if mx.device.num_gpus() > 0 else mx.cpu()\\n\",\n    \"lr = 0.01\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Data\\n\",\n    \"\\n\",\n    \"We get the traditionnal MNIST dataset and add a new label to the existing one. For each digit we return a new label that stands for Odd or Even\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"![](https://upload.wikimedia.org/wikipedia/commons/2/27/MnistExamples.png)\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 3,\n   \"source\": [\n    \"train_dataset = gluon.data.vision.MNIST(train=True)\\n\",\n    \"test_dataset = gluon.data.vision.MNIST(train=False)\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 4,\n   \"source\": [\n    \"def transform(x,y):\\n\",\n    \"    x = x.transpose((2,0,1)).astype('float32')/255.\\n\",\n    \"    y1 = y\\n\",\n    \"    y2 = y % 2 #odd or even\\n\",\n    \"    return x, onp.float32(y1), onp.float32(y2)\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"We assign the transform to the original dataset\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 5,\n   \"source\": [\n    \"train_dataset_t = train_dataset.transform(transform)\\n\",\n    \"test_dataset_t = test_dataset.transform(transform)\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"We load the datasets DataLoaders\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 6,\n   \"source\": [\n    \"train_data = gluon.data.DataLoader(train_dataset_t, shuffle=True, last_batch='rollover', batch_size=batch_size, num_workers=5)\\n\",\n    \"test_data = gluon.data.DataLoader(test_dataset_t, shuffle=False, last_batch='rollover', batch_size=batch_size, num_workers=5)\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 7,\n   \"source\": [\n    \"print(\\\"Input shape: {}, Target Labels: {}\\\".format(train_dataset[0][0].shape, train_dataset_t[0][1:]))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Input shape: (28, 28, 1), Target Labels: (5.0, 1.0)\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Multi-task Network\\n\",\n    \"\\n\",\n    \"The output of the featurization is passed to two different outputs layers\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 135,\n   \"source\": [\n    \"class MultiTaskNetwork(gluon.HybridBlock):\\n\",\n    \"    \\n\",\n    \"    def __init__(self):\\n\",\n    \"        super(MultiTaskNetwork, self).__init__()\\n\",\n    \"        \\n\",\n    \"        self.shared = gluon.nn.HybridSequential()\\n\",\n    \"        self.shared.add(\\n\",\n    \"            gluon.nn.Dense(128, activation='relu'),\\n\",\n    \"            gluon.nn.Dense(64, activation='relu'),\\n\",\n    \"            gluon.nn.Dense(10, activation='relu')\\n\",\n    \"        )\\n\",\n    \"        self.output1 = gluon.nn.Dense(10) # Digist recognition\\n\",\n    \"        self.output2 = gluon.nn.Dense(1) # odd or even\\n\",\n    \"\\n\",\n    \"        \\n\",\n    \"    def forward(self, x):\\n\",\n    \"        y = self.shared(x)\\n\",\n    \"        output1 = self.output1(y)\\n\",\n    \"        output2 = self.output2(y)\\n\",\n    \"        return output1, output2\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"We can use two different losses, one for each output\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 136,\n   \"source\": [\n    \"loss_digits = gluon.loss.SoftmaxCELoss()\\n\",\n    \"loss_odd_even = gluon.loss.SigmoidBCELoss()\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"We create and initialize the network\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 137,\n   \"source\": [\n    \"mx.np.random.seed(42)\\n\",\n    \"random.seed(42)\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 138,\n   \"source\": [\n    \"net = MultiTaskNetwork()\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 139,\n   \"source\": [\n    \"net.initialize(mx.init.Xavier(), ctx=ctx)\\n\",\n    \"net.hybridize() # hybridize for speed\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 140,\n   \"source\": [\n    \"trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate':lr})\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Evaluate Accuracy\\n\",\n    \"We need to evaluate the accuracy of each task separately\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 141,\n   \"source\": [\n    \"def evaluate_accuracy(net, data_iterator):\\n\",\n    \"    acc_digits = mx.gluon.metric.Accuracy(name='digits')\\n\",\n    \"    acc_odd_even = mx.gluon.metric.Accuracy(name='odd_even')\\n\",\n    \"    \\n\",\n    \"    for i, (data, label_digit, label_odd_even) in enumerate(data_iterator):\\n\",\n    \"        data = data.to_device(ctx)\\n\",\n    \"        label_digit = label_digit.to_device(ctx)\\n\",\n    \"        label_odd_even = label_odd_even.to_device(ctx).reshape(-1,1)\\n\",\n    \"\\n\",\n    \"        output_digit, output_odd_even = net(data)\\n\",\n    \"        \\n\",\n    \"        acc_digits.update(label_digit, npx.softmax(output_digit))\\n\",\n    \"        acc_odd_even.update(label_odd_even, npx.sigmoid(output_odd_even) > 0.5)\\n\",\n    \"    return acc_digits.get(), acc_odd_even.get()\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Training Loop\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"We need to balance the contribution of each loss to the overall training and do so by tuning this alpha parameter within [0,1].\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 142,\n   \"source\": [\n    \"alpha = 0.5 # Combine losses factor\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 143,\n   \"source\": [\n    \"for e in range(epochs):\\n\",\n    \"    # Accuracies for each task\\n\",\n    \"    acc_digits = mx.gluon.metric.Accuracy(name='digits')\\n\",\n    \"    acc_odd_even = mx.gluon.metric.Accuracy(name='odd_even')\\n\",\n    \"    # Accumulative losses\\n\",\n    \"    l_digits_ = 0.\\n\",\n    \"    l_odd_even_ = 0. \\n\",\n    \"    \\n\",\n    \"    for i, (data, label_digit, label_odd_even) in enumerate(train_data):\\n\",\n    \"        data = data.to_device(ctx)\\n\",\n    \"        label_digit = label_digit.to_device(ctx)\\n\",\n    \"        label_odd_even = label_odd_even.to_device(ctx).reshape(-1,1)\\n\",\n    \"        \\n\",\n    \"        with autograd.record():\\n\",\n    \"            output_digit, output_odd_even = net(data)\\n\",\n    \"            l_digits = loss_digits(output_digit, label_digit)\\n\",\n    \"            l_odd_even = loss_odd_even(output_odd_even, label_odd_even)\\n\",\n    \"\\n\",\n    \"            # Combine the loss of each task\\n\",\n    \"            l_combined = (1-alpha)*l_digits + alpha*l_odd_even\\n\",\n    \"            \\n\",\n    \"        l_combined.backward()\\n\",\n    \"        trainer.step(data.shape[0])\\n\",\n    \"        \\n\",\n    \"        l_digits_ += l_digits.mean()\\n\",\n    \"        l_odd_even_ += l_odd_even.mean()\\n\",\n    \"        acc_digits.update(label_digit, npx.softmax(output_digit))\\n\",\n    \"        acc_odd_even.update(label_odd_even, npx.sigmoid(output_odd_even) > 0.5)\\n\",\n    \"        \\n\",\n    \"    print(\\\"Epoch [{}], Acc Digits   {:.4f} Loss Digits   {:.4f}\\\".format(\\n\",\n    \"        e, acc_digits.get()[1], l_digits_.item()/(i+1)))\\n\",\n    \"    print(\\\"Epoch [{}], Acc Odd/Even {:.4f} Loss Odd/Even {:.4f}\\\".format(\\n\",\n    \"        e, acc_odd_even.get()[1], l_odd_even_.item()/(i+1)))\\n\",\n    \"    print(\\\"Epoch [{}], Testing Accuracies {}\\\".format(e, evaluate_accuracy(net, test_data)))\\n\",\n    \"        \"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Epoch [0], Acc Digits   0.8945 Loss Digits   0.3409\\n\",\n      \"Epoch [0], Acc Odd/Even 0.9561 Loss Odd/Even 0.1152\\n\",\n      \"Epoch [0], Testing Accuracies (('digits', 0.9487179487179487), ('odd_even', 0.9770633012820513))\\n\",\n      \"Epoch [1], Acc Digits   0.9576 Loss Digits   0.1475\\n\",\n      \"Epoch [1], Acc Odd/Even 0.9804 Loss Odd/Even 0.0559\\n\",\n      \"Epoch [1], Testing Accuracies (('digits', 0.9642427884615384), ('odd_even', 0.9826722756410257))\\n\",\n      \"Epoch [2], Acc Digits   0.9681 Loss Digits   0.1124\\n\",\n      \"Epoch [2], Acc Odd/Even 0.9852 Loss Odd/Even 0.0418\\n\",\n      \"Epoch [2], Testing Accuracies (('digits', 0.9580328525641025), ('odd_even', 0.9846754807692307))\\n\",\n      \"Epoch [3], Acc Digits   0.9734 Loss Digits   0.0961\\n\",\n      \"Epoch [3], Acc Odd/Even 0.9884 Loss Odd/Even 0.0340\\n\",\n      \"Epoch [3], Testing Accuracies (('digits', 0.9670472756410257), ('odd_even', 0.9839743589743589))\\n\",\n      \"Epoch [4], Acc Digits   0.9762 Loss Digits   0.0848\\n\",\n      \"Epoch [4], Acc Odd/Even 0.9894 Loss Odd/Even 0.0310\\n\",\n      \"Epoch [4], Testing Accuracies (('digits', 0.9652887658227848), ('odd_even', 0.9858583860759493))\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Testing\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 144,\n   \"source\": [\n    \"def get_random_data():\\n\",\n    \"    idx = random.randint(0, len(test_dataset))\\n\",\n    \"\\n\",\n    \"    img = test_dataset[idx][0]\\n\",\n    \"    data, _, _ = test_dataset_t[idx]\\n\",\n    \"    data = np.expand_dims(data.to_device(ctx), axis=0)\\n\",\n    \"\\n\",\n    \"    plt.imshow(img.squeeze().asnumpy(), cmap='gray')\\n\",\n    \"    \\n\",\n    \"    return data\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 152,\n   \"source\": [\n    \"data = get_random_data()\\n\",\n    \"\\n\",\n    \"digit, odd_even = net(data)\\n\",\n    \"\\n\",\n    \"digit = digit.argmax(axis=1)[0].asnumpy()\\n\",\n    \"odd_even = (npx.sigmoid(odd_even)[0] > 0.5).asnumpy()\\n\",\n    \"\\n\",\n    \"print(\\\"Predicted digit: {}, odd: {}\\\".format(digit, odd_even))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Predicted digit: [9.], odd: [1.]\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"display_data\",\n     \"data\": {\n      \"image/png\": \"iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADeVJREFUeJzt3X+MFPX9x/HXG6QGAQ3aiBdLpd9Ga6pBak5joqk01caaRuAfUhMbjE2viTUpEVFCNT31Dxu1rdWYJldLCk2/QhUb+KPWWuKP1jQNIKiotFJC00OEkjNBEiNyvPvHzdlTbz6zzs7uzPF+PpLL7e57Z+ad5V7M7H5m9mPuLgDxTKq7AQD1IPxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4I6oZsbMzNOJwQ6zN2tlee1tec3s6vM7O9mtsvMVrSzLgDdZWXP7TezyZL+IelKSYOSNku61t1fSyzDnh/osG7s+S+WtMvdd7v7EUlrJS1oY30Auqid8J8p6d9j7g9mj32ImfWZ2RYz29LGtgBUrOMf+Ln7gKQBicN+oEna2fPvlTR7zP3PZI8BmADaCf9mSWeb2efM7FOSvilpYzVtAei00of97n7UzG6S9JSkyZJWufurlXUGoKNKD/WV2hjv+YGO68pJPgAmLsIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCKj1FtySZ2R5J70galnTU3XuraApA57UV/sxX3P1gBesB0EUc9gNBtRt+l/RHM9tqZn1VNASgO9o97L/M3fea2emSnjazne7+/NgnZP8p8B8D0DDm7tWsyKxf0mF3vz/xnGo2BiCXu1srzyt92G9m08xsxuhtSV+TtKPs+gB0VzuH/bMk/c7MRtfz/+7+h0q6AtBxlR32t7QxDvuBjuv4YT+AiY3wA0ERfiAowg8ERfiBoAg/EFQVV/WhwaZPn56sL1++vK3lb7755mT97bffzq3deeedyWUffvjhZP3o0aPJOtLY8wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUFzSOwFMnTo1WV+xYkVurWgcftq0acl69n0NuTr591M0zr9s2bJk/ciRI1W2M2FwSS+AJMIPBEX4gaAIPxAU4QeCIvxAUIQfCIpx/i4oGqe//PLLk/Vbb701WZ8/f/4nballQ0NDbdWnTJmSWzvrrLNK9TTqySefTNafe+653NoDDzyQXHYinyPAOD+AJMIPBEX4gaAIPxAU4QeCIvxAUIQfCKpwnN/MVkn6hqQD7n5+9tipktZJmiNpj6TF7p7/Be3/W9dxOc5/0kknJesPPvhgsn7DDTdU2c6H7NixI1m/5557kvVt27Yl6zt37kzWZ8yYkVt76qmnkstecsklyXo7zjnnnGR9165dHdt2p1U5zv8rSVd95LEVkja5+9mSNmX3AUwgheF39+clffQ0rgWSVme3V0taWHFfADqs7Hv+We6+L7v9lqRZFfUDoEvanqvP3T31Xt7M+iT1tbsdANUqu+ffb2Y9kpT9PpD3RHcfcPded+8tuS0AHVA2/BslLcluL5G0oZp2AHRLYfjN7FFJf5X0BTMbNLNvS/qRpCvN7A1JV2T3AUwghe/53f3anNJXK+5lwrriiiuS9XbH8Q8ePJisr1u3Lrd2yy23JJd97733SvXUqp6entq2jTTO8AOCIvxAUIQfCIrwA0ERfiAowg8E1fbpvVGkprJevnx5R7f9yCOPJOsrV67s2LZPOCH9J7Jo0aJk/aGHHsqtnX766aV6atUzzzyTW9u7d29Htz0RsOcHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAY52/RHXfckVu79NJL21p30Tj+3Xff3db6U84999xkfenSpcl6X19zv6Ht3nvvza29++67XeykmdjzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQjPO3qJPXnq9ZsyZZLxqTTk03XTROv3jx4mT9tNNOS9aLpnjvpNR3BUjSs88+251GJij2/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QVOE4v5mtkvQNSQfc/fzssX5J35H0n+xpK939951qsgk2b96cW7v++uvbWveGDRuS9SNHjiTrU6dOza2dfPLJpXoa9f777yfr1113XbKemlNg7ty5pXoa9dhjjyXrTAGe1sqe/1eSrhrn8Z+6+7zs57gOPnA8Kgy/uz8vaagLvQDoonbe899kZi+b2Sozm1lZRwC6omz4fy7p85LmSdon6cd5TzSzPjPbYmZbSm4LQAeUCr+773f3YXc/JukXki5OPHfA3XvdvbdskwCqVyr8ZtYz5u4iSTuqaQdAt7Qy1PeopPmSPm1mg5J+KGm+mc2T5JL2SPpuB3sE0AHWzeuxzay+i7/bNGlS/kHS448/nlx24cKFVbdTmRdeeCFZv+uuu5L1ovMIisbiU4p6mz9/frI+PDxcetsTmbtbK8/jDD8gKMIPBEX4gaAIPxAU4QeCIvxAUHx1d4uOHTuWW7vxxhuTy+7fvz9ZL7osdufOncn6E088kVsr+nrrw4cPJ+snnnhisl40HGeWP+qUek0ladOmTcl61KG8qrDnB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGguKQXSWeccUay/uabb5Ze9/bt25P1Cy+8sPS6I+OSXgBJhB8IivADQRF+ICjCDwRF+IGgCD8QFNfzI6m/v7+t5VNTfK9du7atdaM97PmBoAg/EBThB4Ii/EBQhB8IivADQRF+IKjC6/nNbLakNZJmSXJJA+7+MzM7VdI6SXMk7ZG02N3fLlgX1/M3zKJFi5L11JwAklT093Pffffl1m677bbksiinyuv5j0pa5u5flHSJpO+Z2RclrZC0yd3PlrQpuw9ggigMv7vvc/cXs9vvSHpd0pmSFkhanT1ttaSFnWoSQPU+0Xt+M5sj6UuS/iZplrvvy0pvaeRtAYAJouVz+81suqT1kpa6+6Gxc7C5u+e9nzezPkl97TYKoFot7fnNbIpGgv8bdx/9BGi/mfVk9R5JB8Zb1t0H3L3X3XuraBhANQrDbyO7+F9Ket3dfzKmtFHSkuz2Ekkbqm8PQKe0MtR3maQ/S3pF0uicyis18r7/t5I+K+lfGhnqGypYF0N9DfPSSy8l63Pnzk3Wh4aS/+S64IILcmuDg4PJZVFOq0N9he/53f0vkvJW9tVP0hSA5uAMPyAowg8ERfiBoAg/EBThB4Ii/EBQfHX3ca7ostnzzjsvWR8eHk7Wb7/99mSdsfzmYs8PBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0EVXs9f6ca4nr8j5syZk1vbtm1bctlTTjklWd+6dWuyftFFFyXr6L4qv7obwHGI8ANBEX4gKMIPBEX4gaAIPxAU4QeC4nr+48DSpUtza0Xj+EX6+/vbWh7NxZ4fCIrwA0ERfiAowg8ERfiBoAg/EBThB4IqvJ7fzGZLWiNpliSXNODuPzOzfknfkfSf7Kkr3f33Beviev4SrrnmmmR9/fr1ubXJkye3te1Jk9g/TDStXs/fykk+RyUtc/cXzWyGpK1m9nRW+6m731+2SQD1KQy/u++TtC+7/Y6ZvS7pzE43BqCzPtExnZnNkfQlSX/LHrrJzF42s1VmNjNnmT4z22JmW9rqFEClWg6/mU2XtF7SUnc/JOnnkj4vaZ5Gjgx+PN5y7j7g7r3u3ltBvwAq0lL4zWyKRoL/G3d/QpLcfb+7D7v7MUm/kHRx59oEULXC8JuZSfqlpNfd/SdjHu8Z87RFknZU3x6ATmnl0/5LJX1L0itmtj17bKWka81snkaG//ZI+m5HOoR2796drB86dCi3NnPmuB/FfOD++xmsiaqVT/v/Imm8ccPkmD6AZuMMDiAowg8ERfiBoAg/EBThB4Ii/EBQTNENHGeYohtAEuEHgiL8QFCEHwiK8ANBEX4gKMIPBNXtKboPSvrXmPufzh5roqb21tS+JHorq8rezmr1iV09yedjGzfb0tTv9mtqb03tS6K3surqjcN+ICjCDwRVd/gHat5+SlN7a2pfEr2VVUtvtb7nB1Cfuvf8AGpSS/jN7Coz+7uZ7TKzFXX0kMfM9pjZK2a2ve4pxrJp0A6Y2Y4xj51qZk+b2RvZ7/R3c3e3t34z25u9dtvN7OqaepttZs+Y2Wtm9qqZfT97vNbXLtFXLa9b1w/7zWyypH9IulLSoKTNkq5199e62kgOM9sjqdfdax8TNrMvSzosaY27n589dq+kIXf/UfYf50x3v60hvfVLOlz3zM3ZhDI9Y2eWlrRQ0vWq8bVL9LVYNbxudez5L5a0y913u/sRSWslLaihj8Zz9+clDX3k4QWSVme3V2vkj6frcnprBHff5+4vZrffkTQ6s3Str12ir1rUEf4zJf17zP1BNWvKb5f0RzPbamZ9dTczjlnZtOmS9JakWXU2M47CmZu76SMzSzfmtSsz43XV+MDv4y5z9wslfV3S97LD20bykfdsTRquaWnm5m4ZZ2bpD9T52pWd8bpqdYR/r6TZY+5/JnusEdx9b/b7gKTfqXmzD+8fnSQ1+32g5n4+0KSZm8ebWVoNeO2aNON1HeHfLOlsM/ucmX1K0jclbayhj48xs2nZBzEys2mSvqbmzT68UdKS7PYSSRtq7OVDmjJzc97M0qr5tWvcjNfu3vUfSVdr5BP/f0r6QR095PT1f5Jeyn5erbs3SY9q5DDwfY18NvJtSadJ2iTpDUl/knRqg3r7taRXJL2skaD11NTbZRo5pH9Z0vbs5+q6X7tEX7W8bpzhBwTFB35AUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4L6L4bahh5ke9v1AAAAAElFTkSuQmCC\",\n      \"text/plain\": [\n       \"<Figure size 432x288 with 1 Axes>\"\n      ]\n     },\n     \"metadata\": {}\n    }\n   ],\n   \"metadata\": {}\n  }\n ],\n \"metadata\": {\n  \"kernelspec\": {\n   \"display_name\": \"Python 3\",\n   \"language\": \"python\",\n   \"name\": \"python3\"\n  },\n  \"language_info\": {\n   \"codemirror_mode\": {\n    \"name\": \"ipython\",\n    \"version\": 3\n   },\n   \"file_extension\": \".py\",\n   \"mimetype\": \"text/x-python\",\n   \"name\": \"python\",\n   \"nbconvert_exporter\": \"python\",\n   \"pygments_lexer\": \"ipython3\",\n   \"version\": \"3.6.4\"\n  }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}"
  },
  {
    "path": "example/probability/VAE/VAE.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n\n# VAE with Gluon.probability \n\nIn this example, we will demonstrate how you can implement a Variational Auto-encoder(VAE) with Gluon.probability and MXNet's latest NumPy API.\n\n\n```{.python .input}\nimport numpy as np\nimport mxnet as mx\nfrom mxnet import autograd, gluon, np, npx\nfrom mxnet.gluon import nn\nimport mxnet.gluon.probability as mgp\nimport matplotlib.pyplot as plt\n\n# Switch numpy-compatible semantics on.\nnpx.set_np()\n\n# Set context for model context, here we choose to use GPU. \nmodel_ctx = mx.gpu(0)\n```\n\n## Dataset\n\nWe will use MNIST here for simplicity purpose.\n\n\n```{.python .input}\ndef load_data(batch_size):\n    mnist_train = gluon.data.vision.MNIST(train=True)\n    mnist_test = gluon.data.vision.MNIST(train=False)\n    num_worker = 4\n    transformer = gluon.data.vision.transforms.ToTensor()\n    return (gluon.data.DataLoader(mnist_train.transform_first(transformer),\n                                batch_size, shuffle=True,\n                                num_workers=num_worker),\n          gluon.data.DataLoader(mnist_test.transform_first(transformer),\n                                batch_size, shuffle=False,\n                                num_workers=num_worker))\n                                 \n```\n\n## Model definition\n\n\n```{.python .input}\nclass VAE(gluon.HybridBlock):\n    def __init__(self, n_hidden=256, n_latent=2, n_layers=1, n_output=784, act_type='relu', **kwargs):\n        r\"\"\"\n        n_hidden : number of hidden units in each layer\n        n_latent : dimension of the latent space\n        n_layers : number of layers in the encoder and decoder network\n        n_output : dimension of the observed data\n        \"\"\"\n        self.soft_zero = 1e-10\n        self.n_latent = n_latent\n        self.output = None\n        self.mu = None\n        super(VAE, self).__init__(**kwargs)\n        self.encoder = nn.HybridSequential()\n        for _ in range(n_layers):\n            self.encoder.add(nn.Dense(n_hidden, activation=act_type))\n        self.encoder.add(nn.Dense(n_latent*2, activation=None))\n        self.decoder = nn.HybridSequential()\n        for _ in range(n_layers):\n            self.decoder.add(nn.Dense(n_hidden, activation=act_type))\n        self.decoder.add(nn.Dense(n_output, activation='sigmoid'))\n        \n    def encode(self, x):\n        r\"\"\"\n        Given a batch of x,\n        return the encoder's output\n        \"\"\"\n        # [loc_1, ..., loc_n, log(scale_1), ..., log(scale_n)]\n        h = self.encoder(x)\n\n        # Extract loc and log_scale from the encoder output.\n        loc_scale = np.split(h, 2, 1)\n        loc = loc_scale[0]\n        log_scale = loc_scale[1]\n\n        # Convert log_scale back to scale.\n        scale = np.exp(log_scale)\n\n        # Return a Normal object.\n        return mgp.Normal(loc, scale)\n    \n    def decode(self, z):\n        r\"\"\"\n        Given a batch of samples from z,\n        return the decoder's output\n        \"\"\"\n        return self.decoder(z)\n\n    def forward(self, x):\n        r\"\"\"\n        Given a batch of data x,\n        return the negative of Evidence Lower-bound,\n        i.e. an objective to minimize.\n        \"\"\"\n        # prior p(z)\n        pz = mgp.Normal(0, 1)\n        \n        # posterior q(z|x)\n        qz_x = self.encode(x) \n        \n        # Sampling operation qz_x.sample() is automatically reparameterized.\n        z = qz_x.sample() \n\n        # Reconstruction result\n        y = self.decode(z) \n        \n        # Gluon.probability can help you calculate the analytical kl-divergence\n        # between two distribution objects.\n        KL = mgp.kl_divergence(qz_x, pz).sum(1)\n        \n        # We assume p(x|z) ~ Bernoulli, therefore we compute the reconstruction\n        # loss with binary cross entropy.\n        logloss = np.sum(x * np.log(y + self.soft_zero) + (1 - x)\n                         * np.log(1 - y + self.soft_zero), axis=1)\n        loss = -logloss + KL\n        return loss\n```\n\n## Training\n\n\n```{.python .input}\ndef train(net, n_epoch, print_period, train_iter, test_iter):\n    net.initialize(mx.init.Xavier(), ctx=model_ctx)\n    net.hybridize()\n    trainer = gluon.Trainer(net.collect_params(), 'adam',\n                          {'learning_rate': .001})\n    training_loss = []\n    validation_loss = []\n    for epoch in range(n_epoch):\n        epoch_loss = 0\n        epoch_val_loss = 0\n\n        n_batch_train = 0\n        for batch in train_iter:\n            n_batch_train += 1\n            data = batch[0].as_in_context(model_ctx).reshape(-1, 28 * 28)\n            with autograd.record():\n                loss = net(data)\n            loss.backward()\n            trainer.step(data.shape[0])\n            epoch_loss += np.mean(loss)\n\n        n_batch_val = 0\n        for batch in test_iter:\n            n_batch_val += 1\n            data = batch[0].as_in_context(model_ctx).reshape(-1, 28 * 28)\n            loss = net(data)\n            epoch_val_loss += np.mean(loss)\n\n        epoch_loss /= n_batch_train\n        epoch_val_loss /= n_batch_val\n\n        training_loss.append(epoch_loss)\n        validation_loss.append(epoch_val_loss)\n\n        if epoch % max(print_period, 1) == 0:\n            print('Epoch{}, Training loss {:.2f}, Validation loss {:.2f}'.format(\n              epoch, float(epoch_loss), float(epoch_val_loss)))\n```\n\n\n```{.python .input}\nn_hidden = 128\nn_latent = 40\nn_layers = 3\nn_output = 784\nbatch_size = 128\nmodel_prefix = 'vae_gluon_{}d{}l{}h.params'.format(\n  n_latent, n_layers, n_hidden)\nnet = VAE(n_hidden=n_hidden, n_latent=n_latent, n_layers=n_layers,\n        n_output=n_output)\nnet.hybridize()\nn_epoch = 50\nprint_period = n_epoch // 10\ntrain_set, test_set = load_data(batch_size)\ntrain(net, n_epoch, print_period, train_set, test_set)\n```\n\n\n## Reconstruction visualiztion\n\nTo verify the effictiveness of our model, we first take a look at how well our model can reconstruct the data.\n\n\n```{.python .input}\n# Grab a batch from the test set\nqz_x = None\nfor batch in test_set:\n    data = batch[0].as_in_context(model_ctx).reshape(-1, 28 * 28)\n    qz_x = net.encode(data)\n    break\n```\n\n\n```{.python .input}\nnum_samples = 4\nfig, axes = plt.subplots(nrows=num_samples, ncols=2, figsize=(4, 6), subplot_kw={'xticks': [], 'yticks': []})\naxes[0, 0].set_title('Original image')\naxes[0, 1].set_title('reconstruction')\nfor i in range(num_samples):\n    axes[i, 0].imshow(data[i].squeeze().reshape(28, 28).asnumpy(), cmap='gray')\n    axes[i, 1].imshow(net.decode(qz_x.sample())[i].reshape(28, 28).asnumpy(), cmap='gray')\n```\n\n\n![png](./VAE_11_0.png)\n\n\n## Sample generation\n\nOne of the most important difference between Variational Auto-encoder and Auto-encoder is VAE's capabilities of generating new samples.\n\nTo achieve that, one simply needs to feed a random sample from $p(z) \\sim \\mathcal{N}(0,1)$ to the decoder network.\n\n\n```{.python .input}\ndef plot_samples(samples, h=5, w=10):\n    fig, axes = plt.subplots(nrows=h,\n                             ncols=w,\n                             figsize=(int(1.4 * w), int(1.4 * h)),\n                             subplot_kw={'xticks': [], 'yticks': []})\n    for i, ax in enumerate(axes.flatten()):\n        ax.imshow(samples[i], cmap='gray')\n```\n\n\n```{.python .input}\nn_samples = 20\nnoise = np.random.randn(n_samples, n_latent).as_in_context(model_ctx)\ndec_output = net.decode(noise).reshape(-1, 28, 28).asnumpy()\nplot_samples(dec_output, 4, 5)\n```\n\n\n![png](./VAE_14_0.png)\n\n"
  },
  {
    "path": "example/profiler/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# MXNet Profiler Examples\n\nThis folder contains examples of using MXNet profiler to generate profiling results in json files.\nPlease refer to [this link](https://mxnet.apache.org/api/faq/perf?highlight=profiler#profiler)\nfor visualizing profiling results.\n\n- profiler_executor.py. To run this example,\n    - clone mxnet-memonger (git clone https://github.com/dmlc/mxnet-memonger.git).\n    - Add mxnet-memonger folder to PYTHONPATH.\n    export PYTHONPATH=$PYTHONPATH:/path/to/mxnet-memonger\n    - type python profiler_executor.py in terminal.\n    It will generate a json file named `profile_executor_5iter.json`.\n\n- profiler_imageiter.py. You first need to create a file named `test.rec`,\nwhich is an image dataset file before running this example.\nPlease follow\n[this tutorial](https://mxnet.apache.org/faq/recordio.html?highlight=rec%20file#create-a-dataset-using-recordio)\non how to create `.rec` files using an existing tool in MXNet. After you created 'test.rec',\ntype `python profiler_imageiter.py` in terminal. It will generate `profile_imageiter.json`.\n\n- profiler_matmul.py. This example profiles matrix multiplications on GPU. Please make sure\nthat you have installed a GPU enabled version of MXNet before running this example. Type\n`python profiler_matmul.py` and it will generate `profile_matmul_20iter.json`.\n\n- profiler_ndarray.py. This examples profiles a series of `NDArray` operations. Simply type\n`python profiler_ndarray.py` in terminal and it will generate `profile_ndarray.json`.\n"
  },
  {
    "path": "example/profiler/profiler_imageiter.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nfrom __future__ import print_function\nimport os\n# uncomment to set the number of worker threads.\n# os.environ[\"MXNET_CPU_WORKER_NTHREADS\"] = \"4\"\nimport time\nimport mxnet as mx\n\n\ndef run_imageiter(path_rec, n, batch_size=32):\n\n    data = mx.img.ImageIter(batch_size=batch_size,\n                            data_shape=(3, 224, 224),\n                            path_imgrec=path_rec,\n                            rand_crop=True,\n                            rand_resize=True,\n                            rand_mirror=True)\n    data.reset()\n    tic = time.time()\n    for i in range(n):\n        data.next()\n    mx.nd.waitall()\n    print(batch_size*n/(time.time() - tic))\n\n\nif __name__ == '__main__':\n    mx.profiler.set_config(profile_all=True, filename='profile_imageiter.json')\n    mx.profiler.set_state('run')\n    run_imageiter('test.rec', 20)  # See https://mxnet.io/tutorials/python/image_io.html for how to create .rec files.\n    mx.profiler.set_state('stop')\n"
  },
  {
    "path": "example/profiler/profiler_matmul.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nfrom __future__ import print_function\nimport mxnet as mx\nimport argparse\nimport time\n\n\ndef parse_args():\n    parser = argparse.ArgumentParser(description='Set network parameters for benchmark test.')\n    parser.add_argument('--profile_filename', type=str, default='profile_matmul_20iter.json')\n    parser.add_argument('--iter_num', type=int, default=100)\n    parser.add_argument('--begin_profiling_iter', type=int, default=50)\n    parser.add_argument('--end_profiling_iter', type=int, default=70)\n    return parser.parse_args()\n\n\nargs = parse_args()\n\nif __name__ == '__main__':\n    mx.profiler.set_config(profile_symbolic=True, filename=args.profile_filename)\n    print('profile file save to {0}'.format(args.profile_filename))\n\n    A = mx.sym.Variable('A')\n    B = mx.sym.Variable('B')\n    C = mx.symbol.dot(A, B)\n\n    executor = C.simple_bind(mx.gpu(0), 'write', A=(4096, 4096), B=(4096, 4096))\n\n    a = mx.random.uniform(-1.0, 1.0, shape=(4096, 4096))\n    b = mx.random.uniform(-1.0, 1.0, shape=(4096, 4096))\n\n    a.copyto(executor.arg_dict['A'])\n    b.copyto(executor.arg_dict['B'])\n\n    flag = False\n    print(\"execution begin\")\n    for i in range(args.iter_num):\n        if i == args.begin_profiling_iter:\n            t0 = time.process_time()\n            mx.profiler.set_state('run')\n        if i == args.end_profiling_iter:\n            t1 = time.process_time()\n            mx.profiler.set_state('stop')\n        executor.forward()\n        c = executor.outputs[0]\n        c.wait_to_read()\n    print(\"execution end\")\n    duration = t1 - t0\n    print('duration: {0}s'.format(duration))\n    print('          {0}ms/operator'.format(duration*1000/args.iter_num))\n"
  },
  {
    "path": "example/profiler/profiler_ndarray.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport os\nimport mxnet as mx\nimport numpy as np\nimport pickle as pkl\n\n\ndef _np_reduce(dat, axis, keepdims, numpy_reduce_func):\n    if isinstance(axis, int):\n        axis = [axis]\n    else:\n        axis = list(axis) if axis is not None else range(len(dat.shape))\n    ret = dat\n    for i in reversed(sorted(axis)):\n        ret = numpy_reduce_func(ret, axis=i)\n    if keepdims:\n        keepdims_shape = list(dat.shape)\n        for i in axis:\n            keepdims_shape[i] = 1\n        ret = ret.reshape(tuple(keepdims_shape))\n    return ret\n\n\ndef reldiff(a, b):\n    diff = np.abs(a - b)\n    norm = np.abs(a)\n    reldiff = np.max(diff  / (norm + 1e-7))\n    return reldiff\n\n\ndef same(a, b):\n    return np.sum(a != b) == 0\n\n\ndef check_with_uniform(uf, arg_shapes, dim=None, npuf=None, rmin=-10, type_list=[np.float32]):\n    \"\"\"check function consistency with uniform random numbers\"\"\"\n    if isinstance(arg_shapes, int):\n        assert dim\n        shape = tuple(np.random.randint(1, int(1000**(1.0/dim)), size=dim))\n        arg_shapes = [shape] * arg_shapes\n    for dtype in type_list:\n        ndarray_arg = []\n        numpy_arg = []\n        for s in arg_shapes:\n            npy = np.random.uniform(rmin, 10, s).astype(dtype)\n            narr = mx.nd.array(npy, dtype=dtype)\n            ndarray_arg.append(narr)\n            numpy_arg.append(npy)\n        out1 = uf(*ndarray_arg)\n        if npuf is None:\n            out2 = uf(*numpy_arg).astype(dtype)\n        else:\n            out2 = npuf(*numpy_arg).astype(dtype)\n\n        assert out1.shape == out2.shape\n        if isinstance(out1, mx.nd.NDArray):\n            out1 = out1.asnumpy()\n        if dtype == np.float16:\n            assert reldiff(out1, out2) < 2e-3\n        else:\n            assert reldiff(out1, out2) < 1e-6\n\n\ndef random_ndarray(dim):\n    shape = tuple(np.random.randint(1, int(1000**(1.0/dim)), size=dim))\n    data = mx.nd.array(np.random.uniform(-10, 10, shape))\n    return data\n\n\ndef test_ndarray_elementwise():\n    np.random.seed(0)\n    nrepeat = 10\n    maxdim = 4\n    all_type = [np.float32, np.float64, np.float16, np.uint8, np.int32]\n    real_type = [np.float32, np.float64, np.float16]\n    for repeat in range(nrepeat):\n        for dim in range(1, maxdim):\n            check_with_uniform(lambda x, y: x + y, 2, dim, type_list=all_type)\n            check_with_uniform(lambda x, y: x - y, 2, dim, type_list=all_type)\n            check_with_uniform(lambda x, y: x * y, 2, dim, type_list=all_type)\n            check_with_uniform(lambda x, y: x / y, 2, dim, type_list=real_type)\n            check_with_uniform(lambda x, y: x / y, 2, dim, rmin=1, type_list=all_type)\n            check_with_uniform(mx.nd.sqrt, 1, dim, np.sqrt, rmin=0)\n            check_with_uniform(mx.nd.square, 1, dim, np.square, rmin=0)\n            check_with_uniform(lambda x: mx.nd.norm(x).asscalar(), 1, dim, np.linalg.norm)\n\n\ndef test_ndarray_negate():\n    npy = np.random.uniform(-10, 10, (2,3,4))\n    arr = mx.nd.array(npy)\n    assert reldiff(npy, arr.asnumpy()) < 1e-6\n    assert reldiff(-npy, (-arr).asnumpy()) < 1e-6\n\n    # a final check to make sure the negation (-) is not implemented\n    # as inplace operation, so the contents of arr does not change after\n    # we compute (-arr)\n    assert reldiff(npy, arr.asnumpy()) < 1e-6\n\n\ndef test_ndarray_choose():\n    shape = (100, 20)\n    npy = np.arange(np.prod(shape)).reshape(shape)\n    arr = mx.nd.array(npy)\n    nrepeat = 3\n    for repeat in range(nrepeat):\n        indices = np.random.randint(shape[1], size=shape[0])\n        assert same(npy[np.arange(shape[0]), indices],\n                    mx.nd.choose_element_0index(arr, mx.nd.array(indices)).asnumpy())\n\n\ndef test_ndarray_fill():\n    shape = (100, 20)\n    npy = np.arange(np.prod(shape)).reshape(shape)\n    arr = mx.nd.array(npy)\n    new_npy = npy.copy()\n    nrepeat = 3\n    for repeat in range(nrepeat):\n        indices = np.random.randint(shape[1], size=shape[0])\n        val = np.random.randint(shape[1], size=shape[0])\n        new_npy[:] = npy\n        new_npy[np.arange(shape[0]), indices] = val\n        assert same(new_npy,\n                    mx.nd.fill_element_0index(arr, mx.nd.array(val), mx.nd.array(indices)).asnumpy())\n\n\ndef test_ndarray_onehot():\n    shape = (100, 20)\n    npy = np.arange(np.prod(shape)).reshape(shape)\n    arr = mx.nd.array(npy)\n    nrepeat = 3\n    for repeat in range(nrepeat):\n        indices = np.random.randint(shape[1], size=shape[0])\n        npy[:] = 0.0\n        npy[np.arange(shape[0]), indices] = 1.0\n        mx.nd.onehot_encode(mx.nd.array(indices), out=arr)\n        assert same(npy, arr.asnumpy())\n\n\ndef test_ndarray_copy():\n    c = mx.nd.array(np.random.uniform(-10, 10, (10, 10)))\n    d = c.copyto(mx.Context('cpu', 0))\n    assert np.sum(np.abs(c.asnumpy() != d.asnumpy())) == 0.0\n\n\ndef test_ndarray_scalar():\n    c = mx.nd.empty((10,10))\n    d = mx.nd.empty((10,10))\n    c[:] = 0.5\n    d[:] = 1.0\n    d -= c * 2 / 3 * 6.0\n    c += 0.5\n    assert(np.sum(c.asnumpy()) - 100 < 1e-5)\n    assert(np.sum(d.asnumpy()) + 100 < 1e-5)\n    c[:] = 2\n    assert(np.sum(c.asnumpy()) - 200 < 1e-5)\n    d = -c + 2\n    assert(np.sum(d.asnumpy()) < 1e-5)\n\n\ndef test_ndarray_pickle():\n    np.random.seed(0)\n    maxdim = 5\n    nrepeat = 10\n    for repeat in range(nrepeat):\n        for dim in range(1, maxdim):\n            a = random_ndarray(dim)\n            b = mx.nd.empty(a.shape)\n            a[:] = np.random.uniform(-10, 10, a.shape)\n            b[:] = np.random.uniform(-10, 10, a.shape)\n            a = a + b\n            data = pkl.dumps(a)\n            a2 = pkl.loads(data)\n            assert np.sum(a.asnumpy() != a2.asnumpy()) == 0\n\n\ndef test_ndarray_saveload():\n    np.random.seed(0)\n    maxdim = 5\n    nrepeat = 10\n    fname = 'tmp_list.bin'\n    for repeat in range(nrepeat):\n        data = []\n        for i in range(10):\n            data.append(random_ndarray(np.random.randint(1, 5)))\n        mx.nd.save(fname, data)\n        data2 = mx.nd.load(fname)\n        assert len(data) == len(data2)\n        for x, y in zip(data, data2):\n            assert np.sum(x.asnumpy() != y.asnumpy()) == 0\n        dmap = {f'ndarray xx {i}': x for i, x in enumerate(data)}\n        mx.nd.save(fname, dmap)\n        dmap2 = mx.nd.load(fname)\n        assert len(dmap2) == len(dmap)\n        for k, x in dmap.items():\n            y = dmap2[k]\n            assert np.sum(x.asnumpy() != y.asnumpy()) == 0\n    os.remove(fname)\n\n\ndef test_ndarray_slice():\n    shape = (10,)\n    A = mx.nd.array(np.random.uniform(-10, 10, shape))\n    A2 = A.asnumpy()\n    assert same(A[3:8].asnumpy(), A2[3:8])\n    A2[3:8] *= 10;\n    A[3:8] = A2[3:8]\n    assert same(A[3:8].asnumpy(), A2[3:8])\n\n\ndef test_ndarray_slice_along_axis():\n    arr = mx.nd.array(np.random.uniform(-10, 10, (3, 4, 2, 3)))\n    sub_arr = arr.slice(begin=(None, 1), end=(None, 3))\n\n    # test we sliced correctly\n    assert same(arr.asnumpy()[:, 1:3, :, :], sub_arr.asnumpy())\n\n    # test that slice is copy, instead of shared memory\n    sub_arr[:] = 0\n    assert not same(arr.asnumpy()[:, 1:3, :, :], sub_arr.asnumpy())\n\n\ndef test_clip():\n    shape = (10,)\n    A = mx.random.uniform(-10, 10, shape)\n    B = mx.nd.clip(A, -2, 2)\n    B1 = B.asnumpy()\n    for i in range(shape[0]):\n        assert B1[i] >= -2\n        assert B1[i] <= 2\n\n\ndef test_dot():\n    a = np.random.uniform(-3, 3, (3, 4))\n    b = np.random.uniform(-3, 3, (4, 5))\n    c = np.dot(a, b)\n    A = mx.nd.array(a)\n    B = mx.nd.array(b)\n    C = mx.nd.dot(A, B)\n    assert reldiff(c, C.asnumpy()) < 1e-5\n\n\ndef test_reduce():\n    sample_num = 200\n\n    def test_reduce_inner(numpy_reduce_func, nd_reduce_func):\n        for i in range(sample_num):\n            ndim = np.random.randint(1, 6)\n            shape = np.random.randint(1, 11, size=ndim)\n            axis_flags = np.random.randint(0, 2, size=ndim)\n            axes = []\n            for (axis, flag) in enumerate(axis_flags):\n                if flag:\n                    axes.append(axis)\n            keepdims = np.random.randint(0, 2)\n            dat = np.random.rand(*shape) - 0.5\n            if 0 == len(axes):\n                axes = tuple(range(ndim))\n            else:\n                axes = tuple(axes)\n            numpy_ret = numpy_reduce_func(dat, axis=axes, keepdims=keepdims)\n\n            ndarray_ret = nd_reduce_func(mx.nd.array(dat), axis=axes, keepdims=keepdims)\n            if type(ndarray_ret) is mx.ndarray.NDArray:\n                ndarray_ret = ndarray_ret.asnumpy()\n            assert (ndarray_ret.shape == numpy_ret.shape) or \\\n                   (ndarray_ret.shape == (1,) and numpy_ret.shape == ()), \\\n                   f\"nd:{ndarray_ret.shape}, numpy:{numpy_ret.shape}\"\n            err = np.square(ndarray_ret - numpy_ret).mean()\n            assert err < 1E-4\n    test_reduce_inner(lambda data, axis, keepdims:_np_reduce(data, axis, keepdims, np.sum),\n                      mx.nd.sum)\n    test_reduce_inner(lambda data, axis, keepdims:_np_reduce(data, axis, keepdims, np.max),\n                      mx.nd.max)\n    test_reduce_inner(lambda data, axis, keepdims:_np_reduce(data, axis, keepdims, np.min),\n                      mx.nd.min)\n\n\ndef test_broadcast():\n    sample_num = 1000\n\n    def test_broadcast_to():\n        for i in range(sample_num):\n            ndim = np.random.randint(1, 6)\n            target_shape = np.random.randint(1, 11, size=ndim)\n            shape = target_shape.copy()\n            axis_flags = np.random.randint(0, 2, size=ndim)\n            axes = []\n            for (axis, flag) in enumerate(axis_flags):\n                if flag:\n                    shape[axis] = 1\n            dat = np.random.rand(*shape) - 0.5\n            numpy_ret = dat\n            ndarray_ret = mx.nd.array(dat).broadcast_to(shape=target_shape)\n            if type(ndarray_ret) is mx.ndarray.NDArray:\n                ndarray_ret = ndarray_ret.asnumpy()\n            assert (ndarray_ret.shape == target_shape).all()\n            err = np.square(ndarray_ret - numpy_ret).mean()\n            assert err < 1E-8\n    test_broadcast_to()\n\n\nif __name__ == '__main__':\n    mx.profiler.set_config(profile_all=True, filename='profile_ndarray.json')\n    mx.profiler.set_state('run')\n    test_ndarray_slice_along_axis()\n    test_broadcast()\n    test_ndarray_elementwise()\n    test_ndarray_slice()\n    test_ndarray_pickle()\n    test_ndarray_saveload()\n    test_ndarray_copy()\n    test_ndarray_negate()\n    test_ndarray_scalar()\n    test_clip()\n    test_dot()\n    test_ndarray_choose()\n    test_ndarray_onehot()\n    test_ndarray_fill()\n    test_reduce()\n    mx.profiler.set_state('stop')\n"
  },
  {
    "path": "example/quantization/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# Model Quantization with Calibration Examples\n\nThis folder contains examples of quantizing a FP32 model with oneAPI Deep Neural Network Library (oneDNN) to (U)INT8 model.\n\n<h2 id=\"1\">Model Quantization with oneDNN</h2>\n\noneDNN supports quantization with subgraph features on Intel® CPU Platform and can bring performance improvements on the [Intel® Xeon® Scalable Platform](https://www.intel.com/content/www/us/en/processors/xeon/scalable/xeon-scalable-platform.html).\n\n```\nusage: python imagenet_gen_qsym_onednn.py [-h] [--model MODEL] [--epoch EPOCH]\n                                          [--no-pretrained] [--batch-size BATCH_SIZE]\n                                          [--calib-dataset CALIB_DATASET]\n                                          [--image-shape IMAGE_SHAPE]\n                                          [--data-nthreads DATA_NTHREADS]\n                                          [--num-calib-batches NUM_CALIB_BATCHES]\n                                          [--exclude-first-conv] [--shuffle-dataset]\n                                          [--calib-mode CALIB_MODE]\n                                          [--quantized-dtype {auto,int8,uint8}]\n                                          [--quiet]\n\nGenerate a calibrated quantized model from a FP32 model with oneDNN support\n\noptional arguments:\n  -h, --help            show this help message and exit\n  --model MODEL         model to be quantized. If no-pretrained is set then\n                        model must be provided to `model` directory in the same path\n                        as this python script, default is `resnet50_v1`\n  --epoch EPOCH         number of epochs, default is `0`\n  --no-pretrained       If enabled, will not download pretrained model from\n                        MXNet or Gluon-CV modelzoo, default is `False`\n  --batch-size BATCH_SIZE\n                        batch size to be used when calibrating model, default is `32`\n  --calib-dataset CALIB_DATASET\n                        path of the calibration dataset, default is `data/val_256_q90.rec`\n  --image-shape IMAGE_SHAPE\n                        number of channels, height and width of input image separated by comma,\n                        default is `3,224,224`\n  --data-nthreads DATA_NTHREADS\n                        number of threads for data loading, default is `0`\n  --num-calib-batches NUM_CALIB_BATCHES\n                        number of batches for calibration, default is `10`\n  --exclude-first-conv  excluding quantizing the first conv layer since the\n                        input data may have negative value which doesn't\n                        support at moment\n  --shuffle-dataset     shuffle the calibration dataset\n  --calib-mode CALIB_MODE\n                        calibration mode used for generating calibration table\n                        for the quantized symbol; supports 1. none: no\n                        calibration will be used. The thresholds for\n                        quantization will be calculated on the fly. This will\n                        result in inference speed slowdown and loss of\n                        accuracy in general. 2. naive: simply take min and max\n                        values of layer outputs as thresholds for\n                        quantization. In general, the inference accuracy\n                        worsens with more examples used in calibration. It is\n                        recommended to use `entropy` mode as it produces more\n                        accurate inference results. 3. entropy: calculate KL\n                        divergence of the FP32 output and quantized output for\n                        optimal thresholds. This mode is expected to produce\n                        the best inference accuracy of all three kinds of\n                        quantized models if the calibration dataset is\n                        representative enough of the inference dataset.\n                        default is `entropy`\n  --quantized-dtype {auto,int8,uint8}\n                        quantization destination data type for input data,\n                        default is `auto`\n  --quiet               suppress most of log\n```\n\nA new benchmark script `launch_inference_onednn.sh` has been designed to launch performance benchmark for FP32 or INT8 image-classification models with oneDNN.\n```\nusage: bash ./launch_inference_onednn.sh -s symbol_file [-b batch_size] [-iter iteraton] [-ins instance] [-c cores/instance] [-h]\n\narguments:\n  -h, --help                show this help message and exit\n  -s, --symbol_file         symbol file for benchmark, required\n  -b, --batch_size          inference batch size\n                            default: 64\n  -iter, --iteration        inference iteration\n                            default: 500\n  -ins, --instance          launch multi-instance inference\n                            default: one instance per socket\n  -c, --core                number of cores per instance\n                            default: divide full physical cores\n\nexample: resnet INT8 performance benchmark on c5.24xlarge(duo sockets, 24 physical cores per socket).\n\n    bash ./launch_inference_onednn.sh -s ./model/resnet50_v1-quantized-5batches-naive-symbol.json\n\nwill launch two instances for throughput benchmark and each instance will use 24 physical cores.\n```\n\nThe following models have been tested on Linux systems. Accuracy is collected on Intel XEON Cascade Lake CPU. For CPU with Skylake Lake or eariler architecture, the accuracy may not be the same.\n| Model | Source | Dataset | FP32 Accuracy (top-1/top-5)| INT8 Accuracy (top-1/top-5)|\n|:---|:---|---|:---:|:---:|\n| ResNet18-V1  | [MXNet ModelZoo](https://github.com/apache/mxnet/tree/master/python/mxnet/gluon/model_zoo)  | [Validation Dataset](http://data.mxnet.io/data/val_256_q90.rec)  |70.45%/89.55%|70.22%/89.38%|\n| ResNet50-V1  | [MXNet ModelZoo](https://github.com/apache/mxnet/tree/master/python/mxnet/gluon/model_zoo)  | [Validation Dataset](http://data.mxnet.io/data/val_256_q90.rec)  |76.36%/93.49%|76.04%/93.30%|\n| ResNet101-V1  | [MXNet ModelZoo](https://github.com/apache/mxnet/tree/master/python/mxnet/gluon/model_zoo)  | [Validation Dataset](http://data.mxnet.io/data/val_256_q90.rec)  |78.23%/93.99%|77.85%/93.69%|\n| MobileNet v2 1.0  | [MXNet ModelZoo](https://github.com/apache/mxnet/tree/master/python/mxnet/gluon/model_zoo)  | [Validation Dataset](http://data.mxnet.io/data/val_256_q90.rec)  |71.72%/90.28%|71.22%/89.92%|\n| VGG16 | [MXNet ModelZoo](https://github.com/apache/mxnet/tree/master/python/mxnet/gluon/model_zoo)  | [Validation Dataset](http://data.mxnet.io/data/val_256_q90.rec)  |72.83%/91.11%|72.81%/91.10%|\n| VGG19  | [MXNet ModelZoo](https://github.com/apache/mxnet/tree/master/python/mxnet/gluon/model_zoo)  | [Validation Dataset](http://data.mxnet.io/data/val_256_q90.rec)  |73.67%/91.63%|73.67%/91.67%|\n*Measured on validation ImageNet (ILSVRC2012) with batch-size=64,  num-calib-batches=10 and calib-mode=entropy*\n\n<h3>Pre-trained Model</h3>\n\nThe following command is to download the pre-trained model from [MXNet ModelZoo](http://data.mxnet.io/models/imagenet/resnet/152-layers/) and transfer it into the symbolic model which would be finally quantized. The [validation dataset](http://data.mxnet.io/data/val_256_q90.rec) is available for testing the pre-trained models:\n\n```\npython imagenet_gen_qsym_onednn.py --model=resnet50_v1 --num-calib-batches=5 --calib-mode=naive\n```\n\nThe model would be automatically replaced in fusion and quantization format. It is then saved as the quantized symbol and parameter files in the `./model` directory. Set `--model` to one of above listed verified models to quantize them. The following command is to launch inference.\n\n```\n# Launch FP32 Inference\npython imagenet_inference.py --symbol-file=./model/resnet50_v1-symbol.json --param-file=./model/resnet50_v1-0000.params --rgb-mean=0.485,0.456,0.406 --rgb-std=0.229,0.224,0.225 --num-skipped-batches=50 --batch-size=64 --num-inference-batches=500 --dataset=./data/val_256_q90.rec\n\n# Launch INT8 Inference\npython imagenet_inference.py --symbol-file=./model/resnet50_v1-quantized-5batches-naive-symbol.json --param-file=./model/resnet50_v1-quantized-0000.params --rgb-mean=0.485,0.456,0.406 --rgb-std=0.229,0.224,0.225 --num-skipped-batches=50 --batch-size=64 --num-inference-batches=500 --dataset=./data/val_256_q90.rec\n\n# Launch dummy data Inference\nbash ./launch_inference_onednn.sh -s ./model/resnet50_v1-symbol.json\nbash ./launch_inference_onednn.sh -s ./model/resnet50_v1-quantized-5batches-naive-symbol.json\n```\n\n<h3 id='4'>Custom Model</h3>\n\nThis script also supports custom symbolic models. Quantization layer configs can easily be added in `imagenet_gen_qsym_onednn.py` like below:\n\n```\nif logger:\n    frameinfo = getframeinfo(currentframe())\n    logger.info(F'Please set proper RGB configs inside this script below {frameinfo.filename}:{frameinfo.lineno} for model {args.model}!')\n# add rgb mean/std of your model.\nrgb_mean = '0,0,0'\nrgb_std = '0,0,0'\n# add layer names that shouldn't be quantized.\nif logger:\n    frameinfo = getframeinfo(currentframe())\n    logger.info(F'Please set proper excluded_sym_names inside this script below {frameinfo.filename}:{frameinfo.lineno} for model {args.model} if required!')\nexcluded_sym_names += []\nif exclude_first_conv:\n    excluded_sym_names += []\n```\n\nSome tips on quantization configs:\n\n1. First, data, symbol file (custom-symbol.json) and parameter file (custom-0000.params) of FP32 symbolic model should be prepared.\n2. Then, following command should be run to verify that FP32 symbolic model runs inference as expected.\n\n```\n# Launch FP32 Inference\npython imagenet_inference.py --symbol-file=./model/custom-symbol.json --param-file=./model/custom-0000.params --rgb-mean=* --rgb-std=* --num-skipped-batches=* --batch-size=* --num-inference-batches=*--dataset=./data/*\n```\n\n3. Proper `rgb_mean`, `rgb_std` and `excluded_sym_names` should be added in `imagenet_gen_qsym_onednn.py` script.\n\n4. Run following command for quantization:\n\n```\npython imagenet_gen_qsym_onednn.py --model=custom --num-calib-batches=5 --calib-mode=naive\n```\n\n5. After quantization, the quantized symbol and parameter files will be saved in the `model/` directory.\n\n6. Finally, INT8 inference can be run:\n\n```\n# Launch INT8 Inference\npython imagenet_inference.py --symbol-file=./model/resnet50_v1-quantized-10batches-entropy-symbol.json --param-file=./model/resnet50_v1-quantized-10batches-entropy-0000.params --benchmark\n\n# Launch dummy data Inference\nbash ./launch_inference_onednn.sh -s ./model/*.json\n```\n"
  },
  {
    "path": "example/quantization/imagenet_gen_qsym_onednn.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport argparse\nimport logging\nimport os\nimport re\nimport sys\nfrom inspect import currentframe, getframeinfo\n\nimport mxnet as mx\nfrom mxnet import gluon\nfrom mxnet.contrib.quantization import quantize_net\nfrom mxnet.gluon.data import DataLoader\nfrom mxnet.gluon.data.vision import transforms\nfrom mxnet.gluon.model_zoo.vision import get_model\n\nSCRIPT_DIR = os.path.dirname(os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__))))\nsys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, '..', '..')))\nfrom tools.rec2idx import IndexCreator\n\n\ndef download_calib_dataset(dataset_url, calib_dataset, logger=None):\n    if logger is not None:\n        logger.info(f'Downloading calibration dataset from {dataset_url} to {calib_dataset}')\n    mx.test_utils.download(dataset_url, calib_dataset)\n\n\ndef get_from_gluon(model_name, classes=1000, logger=None):\n    dir_path = os.path.dirname(os.path.realpath(__file__))\n    model_path = os.path.join(dir_path, 'model')\n    if logger is not None:\n        logger.info(f'Converting model from Gluon-CV ModelZoo {model_name}... into path {model_path}')\n    net = get_model(name=model_name, classes=classes, pretrained=True)\n    prefix = os.path.join(model_path, model_name)\n    return net, prefix\n\n\ndef regex_find_excluded_symbols(patterns_dict, model_name):\n    for key, value in patterns_dict.items():\n        if re.search(key, model_name) is not None:\n            return value\n    return None\n\n\ndef get_exclude_symbols(model_name, exclude_first_conv):\n    \"\"\"Grouped supported models at the time of commit:\n    - alexnet\n    - densenet121, densenet161\n    - densenet169, densenet201\n    - inceptionv3\n    - mobilenet0.25, mobilenet0.5, mobilenet0.75, mobilenet1.0,\n    - mobilenetv2_0.25, mobilenetv2_0.5, mobilenetv2_0.75, mobilenetv2_1.0\n    - resnet101_v1, resnet152_v1, resnet18_v1, resnet34_v1, resnet50_v1\n    - resnet101_v2, resnet152_v2, resnet18_v2, resnet34_v2, resnet50_v2\n    - squeezenet1.0, squeezenet1.1\n    - vgg11, vgg11_bn, vgg13, vgg13_bn, vgg16, vgg16_bn, vgg19, vgg19_bn\n    \"\"\"\n    exclude_symbol_regex = {\n        'mobilenet[^v]': ['mobilenet_hybridsequential0_flatten0_flatten0', 'mobilenet_hybridsequential0_globalavgpool2d0_fwd'],\n        'mobilenetv2': ['mobilenetv2_hybridsequential1_flatten0_flatten0'],\n        # resnetv2_hybridsequential0_hybridsequential0_bottleneckv20_batchnorm0_fwd is excluded for the sake of accuracy\n        'resnet.*v2': ['resnetv2_hybridsequential0_flatten0_flatten0', 'resnetv2_hybridsequential0_hybridsequential0_bottleneckv20_batchnorm0_fwd'],\n        'squeezenet1': ['squeezenet_hybridsequential1_flatten0_flatten0'],\n    }\n    excluded_sym_names = regex_find_excluded_symbols(exclude_symbol_regex, model_name)\n    if excluded_sym_names is None:\n        excluded_sym_names = []\n    if exclude_first_conv:\n        first_conv_regex = {\n            'alexnet': ['alexnet_hybridsequential0_conv2d0_fwd'],\n            'densenet': ['densenet_hybridsequential0_conv2d0_fwd'],\n            'inceptionv3': ['inception3_hybridsequential0_hybridsequential0_conv2d0_fwd'],\n            'mobilenet[^v]': ['mobilenet_hybridsequential0_conv2d0_fwd'],\n            'mobilenetv2': ['mobilenetv2_hybridsequential0_conv2d0_fwd'],\n            'resnet.*v1': ['resnetv1_hybridsequential0_conv2d0_fwd'],\n            'resnet.*v2': ['resnetv2_hybridsequential0_conv2d0_fwd'],\n            'squeezenet1': ['squeezenet_hybridsequential0_conv2d0_fwd'],\n            'vgg': ['vgg_hybridsequential0_conv2d0_fwd'],\n        }\n        excluded_first_conv_sym_names = regex_find_excluded_symbols(first_conv_regex, model_name)\n        if excluded_first_conv_sym_names is None:\n            raise ValueError(f'Currently, model {model_name} is not supported in this script')\n        excluded_sym_names += excluded_first_conv_sym_names\n    return excluded_sym_names\n\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser(description='Generate a calibrated quantized model from a FP32 model with oneDNN support')\n    parser.add_argument('--model', type=str, default='resnet50_v1',\n                        help='model to be quantized. If no-pretrained is set then'\n                             'model must be provided to `model` directory in the same path'\n                             'as this python script')\n    parser.add_argument('--epoch', type=int, default=0,\n                        help='number of epochs, default is 0')\n    parser.add_argument('--no-pretrained', action='store_true', default=False,\n                        help='If enabled, will not download pretrained model from MXNet or Gluon-CV modelzoo.')\n    parser.add_argument('--batch-size', type=int, default=32)\n    parser.add_argument('--calib-dataset', type=str, default='data/val_256_q90.rec',\n                        help='path of the calibration dataset')\n    parser.add_argument('--image-shape', type=str, default='3,224,224',\n                        help='number of channels, height and width of input image separated by comma')\n    parser.add_argument('--data-nthreads', type=int, default=0,\n                        help='number of threads for data loading')\n    parser.add_argument('--num-calib-batches', type=int, default=10,\n                        help='number of batches for calibration')\n    parser.add_argument('--exclude-first-conv', action='store_true', default=False,\n                        help='excluding quantizing the first conv layer since the'\n                             ' input data may have negative value which doesn\\'t support at moment')\n    parser.add_argument('--shuffle-dataset', action='store_true',\n                        help='shuffle the calibration dataset')\n    parser.add_argument('--calib-mode', type=str, default='entropy',\n                        help='calibration mode used for generating calibration table for the quantized symbol; supports'\n                             ' 1. none: no calibration will be used. The thresholds for quantization will be calculated'\n                             ' on the fly. This will result in inference speed slowdown and loss of accuracy'\n                             ' in general.'\n                             ' 2. naive: simply take min and max values of layer outputs as thresholds for'\n                             ' quantization. In general, the inference accuracy worsens with more examples used in'\n                             ' calibration. It is recommended to use `entropy` mode as it produces more accurate'\n                             ' inference results.'\n                             ' 3. entropy: calculate KL divergence of the fp32 output and quantized output for optimal'\n                             ' thresholds. This mode is expected to produce the best inference accuracy of all three'\n                             ' kinds of calibration modes if the calibration dataset is representative enough of the'\n                             ' inference dataset.')\n    parser.add_argument('--quantized-dtype', type=str, default='auto',\n                        choices=['auto', 'int8', 'uint8'],\n                        help='quantization destination data type for input data')\n    parser.add_argument('--quiet', action='store_true', default=False,\n                        help='suppress most of log')\n    args = parser.parse_args()\n    ctx = mx.cpu(0)\n    logger = None\n\n    if not args.quiet:\n        logging.basicConfig()\n        logger = logging.getLogger('logger')\n        logger.setLevel(logging.INFO)\n\n    if logger:\n        logger.info(args)\n        logger.info(f'shuffle_dataset={args.shuffle_dataset}')\n        logger.info(f'calibration mode set to {args.calib_mode}')\n\n    calib_mode = args.calib_mode\n\n    # download calibration dataset\n    if calib_mode != 'none':\n        idx_file_name = os.path.splitext(args.calib_dataset)[0] + '.idx'\n        if not os.path.isfile(idx_file_name):\n            download_calib_dataset('http://data.mxnet.io/data/val_256_q90.rec', args.calib_dataset)\n            creator = IndexCreator(args.calib_dataset, idx_file_name)\n            creator.create_index()\n            creator.close()\n\n    # get image shape\n    image_shape = args.image_shape\n    data_shape = [(1,) + tuple(int(i) for i in image_shape.split(','))]\n\n    # check if directory for output model exists\n    dir_path = os.path.dirname(os.path.realpath(__file__))\n    dir_path = os.path.join(dir_path, 'model')\n    if not os.path.exists(dir_path):\n        os.mkdir(dir_path)  # without try catch block as we expect to finish script if it fail\n\n    # download model\n    if not args.no_pretrained:\n        if logger:\n            logger.info('Get pre-trained model from Gluon-CV modelzoo.')\n            logger.info('If you want to use custom model, please set --no-pretrained.')\n        net, prefix = get_from_gluon(model_name=args.model, classes=1000, logger=logger)\n        rgb_mean = '0.485,0.456,0.406'\n        rgb_std = '0.229,0.224,0.225'\n        epoch = 0\n        net.hybridize()\n        net(mx.np.zeros(data_shape[0])) # dummy forward pass to build graph\n        net.export(prefix) # save model\n        net.hybridize(active=False) # disable hybridization - it will be handled in quantization API\n    else:\n        prefix = os.path.join(dir_path, args.model)\n        epoch = args.epoch\n        net = gluon.SymbolBlock.imports(\"{}-symbol.json\".format(prefix), ['data'], \"{}-0000.params\".format(prefix))\n\n    # get batch size\n    batch_size = args.batch_size\n    if logger:\n        logger.info(f'batch size = {batch_size} for calibration')\n\n    # get number of batches for calibration\n    num_calib_batches = args.num_calib_batches\n    if logger:\n        if calib_mode == 'none':\n            logger.info('skip calibration step as calib_mode is none')\n        else:\n            logger.info(f'number of batches = {num_calib_batches} for calibration')\n\n    # get number of threads for decoding the dataset\n    data_nthreads = args.data_nthreads\n\n    exclude_first_conv = args.exclude_first_conv\n    if args.quantized_dtype == \"uint8\":\n        if logger:\n            logger.info('quantized dtype is set to uint8, will exclude first conv.')\n        exclude_first_conv = True\n    excluded_sym_names = []\n    if not args.no_pretrained:\n        excluded_sym_names += get_exclude_symbols(args.model, args.exclude_first_conv)\n    else:\n        if logger:\n            frameinfo = getframeinfo(currentframe())\n            logger.info(F'Please set proper RGB configs inside this script below {frameinfo.filename}:{frameinfo.lineno} for model {args.model}!')\n        # add rgb mean/std of your model.\n        rgb_mean = '0,0,0'\n        rgb_std = '0,0,0'\n        # add layer names you donnot want to quantize.\n        if logger:\n            frameinfo = getframeinfo(currentframe())\n            logger.info(F'Please set proper excluded_sym_names inside this script below {frameinfo.filename}:{frameinfo.lineno} for model {args.model} if required!')\n        excluded_sym_names += []\n        if exclude_first_conv:\n            excluded_sym_names += []\n\n    if logger:\n        logger.info(f'These layers have been excluded {excluded_sym_names}')\n        logger.info(f'Input data shape = {str(data_shape)}')\n        logger.info(f'rgb_mean = {rgb_mean}')\n        logger.info(f'rgb_std = {rgb_std}')\n\n    rgb_mean = [float(i) for i in rgb_mean.split(',')]\n    mean_args = {'mean_r': rgb_mean[0], 'mean_g': rgb_mean[1], 'mean_b': rgb_mean[2]}\n    rgb_std = [float(i) for i in rgb_std.split(',')]\n    std_args = {'std_r': rgb_std[0], 'std_g': rgb_std[1], 'std_b': rgb_std[2]}\n    if calib_mode == 'none':\n        if logger:\n            logger.info(f'Quantizing FP32 model {args.model}')\n        qsym = quantize_net(net, ctx=ctx, exclude_layers_match=excluded_sym_names, data_shapes=data_shape,\n                            calib_mode=calib_mode, quantized_dtype=args.quantized_dtype,\n                            logger=logger)\n        suffix = '-quantized'\n    else:\n        if logger:\n            logger.info('Creating DataLoader for reading calibration dataset')\n        dataset = mx.gluon.data.vision.ImageRecordDataset(args.calib_dataset)\n        transformer = transforms.Compose([transforms.Resize(256),\n                                          transforms.CenterCrop(224),\n                                          transforms.ToTensor(),\n                                          transforms.Normalize(mean=rgb_mean, std=rgb_std)])\n        data_loader = DataLoader(dataset.transform_first(transformer), batch_size, shuffle=args.shuffle_dataset, num_workers=data_nthreads)\n        qsym = quantize_net(net, ctx=ctx, exclude_layers_match=excluded_sym_names,\n                            calib_mode=calib_mode, calib_data=data_loader, num_calib_batches=num_calib_batches,\n                            quantized_dtype=args.quantized_dtype, logger=logger)\n        if calib_mode == 'entropy':\n            suffix = f'-quantized-{num_calib_batches}batches-entropy'\n        elif calib_mode == 'naive':\n            suffix = f'-quantized-{num_calib_batches}batches-naive'\n        else:\n            raise ValueError(f'unknown calibration mode {calib_mode} received, only supports `none`, `naive`, and `entropy`')\n    save_path = prefix + suffix\n    model_path, params_path = qsym.export(save_path, epoch)\n    if logger is not None:\n        logger.info(F'Saved quantized model into:\\n{model_path}\\n{params_path}')\n"
  },
  {
    "path": "example/quantization/imagenet_inference.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport argparse\nimport logging\nimport time\n\nimport mxnet as mx\nimport numpy as np\nfrom mxnet import gluon\nfrom mxnet.gluon.data import DataLoader\nfrom mxnet.gluon.data.vision import transforms\n\n\ndef download_dataset(dataset_url, dataset_dir, logger=None):\n    if logger is not None:\n        logger.info(f'Downloading dataset for inference from {dataset_url} to {dataset_dir}')\n    mx.test_utils.download(dataset_url, dataset_dir)\n\n\ndef score(symblock, data, ctx, max_num_examples, skip_num_batches, logger=None):\n    metrics = [gluon.metric.create('acc'),\n               gluon.metric.create('top_k_accuracy', top_k=5)]\n\n    # make sure that fp32 inference works on the same images as calibrated quantized model\n    logger.info(f'Skipping the first {skip_num_batches} batches')\n\n    tic = time.time()\n    num = 0\n    for i, input_data in enumerate(data):\n        if i < skip_num_batches:\n            continue\n        x = input_data[0].to_device(ctx)\n        label = input_data[1].to_device(ctx)\n        outputs = symblock.forward(x)\n        for m in metrics:\n            m.update(label, outputs)\n        num += batch_size\n        if max_num_examples is not None and num >= max_num_examples:\n            break\n\n    speed = num / (time.time() - tic)\n\n    if logger is not None:\n        logger.info(f'Finished inference with {num} images')\n        logger.info(f'Finished with {speed} images per second')\n        for m in metrics:\n            logger.info(m.get())\n\ndef initialize_block_params(block, initializer):\n    for _, param in block.collect_params('.*gamma|.*moving_var|.*running_var').items():\n        param.initialize(mx.init.Constant(1))\n    for _, param in block.collect_params('.*beta|.*moving_mean|.*running_mean|.*bias').items():\n        param.initialize(mx.init.Constant(0))\n    for _, param in block.collect_params('.*weight').items():\n        param.initialize(initializer)\n\ndef benchmark_score(symblock, ctx, batch_size, warmup_batches, num_batches, data_layer_type):\n    if data_layer_type == \"int8\":\n        dshape = mx.io.DataDesc(name='data', shape=(\n            batch_size,) + data_shape, dtype=np.int8)\n    elif data_layer_type == 'uint8':\n        dshape = mx.io.DataDesc(name='data', shape=(\n            batch_size,) + data_shape, dtype=np.uint8)\n    else:  # float32\n        dshape = mx.io.DataDesc(name='data', shape=(\n            batch_size,) + data_shape, dtype=np.float32)\n\n    # get data\n    if data_layer_type == \"float32\":\n        data = [mx.random.uniform(-1.0, 1.0, shape=shape, ctx=ctx, dtype=data_layer_type)\n                for _, shape in [dshape]]\n    else:\n        data = [mx.nd.full(shape=shape, val=127, ctx=ctx, dtype=data_layer_type)\n                for _, shape in [dshape]]\n\n    # run\n    for i in range(warmup_batches+num_batches):\n        if i == warmup_batches:\n            tic = time.time()\n        outputs = symblock.forward(*data)\n        for output in outputs:\n            output.wait_to_read()\n\n    # return num images per second\n    return num_batches * batch_size / (time.time() - tic)\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser(description='Score a model on a dataset')\n    parser.add_argument('--ctx', type=str, default='cpu')\n    parser.add_argument('--benchmark', type=bool, default=False, help='dummy data benchmark')\n    parser.add_argument('--symbol-file', type=str, required=True, help='symbol file path')\n    parser.add_argument('--param-file', type=str, required=False, help='param file path')\n    parser.add_argument('--batch-size', type=int, default=32)\n    parser.add_argument('--dataset', type=str, required=False, help='dataset path')\n    parser.add_argument('--rgb-mean', type=str, default='0,0,0')\n    parser.add_argument('--rgb-std', type=str, default='1,1,1')\n    parser.add_argument('--image-shape', type=str, default='3,224,224')\n    parser.add_argument('--data-nthreads', type=int, default=60, help='number of threads for data decoding')\n    parser.add_argument('--num-skipped-batches', type=int, default=0, help='skip the number of batches for inference')\n    parser.add_argument('--num-inference-batches', type=int, required=True, help='number of images used for inference')\n    parser.add_argument('--num-warmup-batches', type=int, default=5, help='number of warmup batches used for benchmark')\n    parser.add_argument('--shuffle-dataset', action='store_true', default=True,\n                        help='shuffle the score dataset')\n    parser.add_argument('--data-layer-type', type=str, default='float32',\n                        choices=['float32', 'int8', 'uint8'],\n                        help='data type for data layer (only with --benchmark)')\n\n    args = parser.parse_args()\n\n    logging.basicConfig()\n    logger = logging.getLogger('logger')\n    logger.setLevel(logging.INFO)\n\n    if args.device == 'cpu':\n        ctx = mx.cpu(0)\n    elif args.device == 'gpu':\n        ctx = mx.gpu(0)\n        logger.warning('Notice that oneDNN optimized and quantized model may not work with GPU context')\n    else:\n        raise ValueError(f'ctx {args.device} is not supported in this script')\n\n    symbol_file = args.symbol_file\n    param_file = args.param_file\n    data_nthreads = args.data_nthreads\n\n    batch_size = args.batch_size\n    logger.info(f'batch size = {batch_size} for inference')\n\n    rgb_mean = args.rgb_mean\n    logger.info(f'rgb_mean = {rgb_mean}')\n    rgb_mean = [float(i) for i in rgb_mean.split(',')]\n    rgb_std = args.rgb_std\n    logger.info(f'rgb_std = {rgb_std}')\n    rgb_std = [float(i) for i in rgb_std.split(',')]\n\n    image_shape = args.image_shape\n    data_shape = tuple([int(i) for i in image_shape.split(',')])\n    logger.info(f'Input data shape = {str(data_shape)}')\n\n    data_layer_type = args.data_layer_type\n\n    if not args.benchmark:\n        dataset = args.dataset\n        download_dataset('http://data.mxnet.io/data/val_256_q90.rec', dataset)\n        logger.info(f'Dataset for inference: {dataset}')\n\n        dataset = mx.gluon.data.vision.ImageRecordDataset(dataset)\n        transformer = transforms.Compose([transforms.Resize(256),\n                                          transforms.CenterCrop(224),\n                                          transforms.ToTensor(),\n                                          transforms.Normalize(mean=rgb_mean, std=rgb_std)])\n        data_loader = DataLoader(dataset.transform_first(\n            transformer), batch_size, shuffle=args.shuffle_dataset, num_workers=data_nthreads)\n\n        # loading model\n        symblock = gluon.SymbolBlock.imports(symbol_file, ['data'], param_file)\n\n        num_inference_images = args.num_inference_batches * batch_size\n        logger.info(f'Running model {symbol_file} for inference')\n        score(symblock, data_loader, ctx, max_num_examples=num_inference_images,\n              skip_num_batches=args.num_skipped_batches, logger=logger)\n    else:\n        # loading model\n        symblock = gluon.SymbolBlock.imports(symbol_file, ['data'])\n        initialize_block_params(symblock, mx.init.One())\n\n        logger.info(f'Running model {symbol_file} for inference.')\n        logger.info(f'Warmup batches: {args.num_warmup_batches}')\n        logger.info(f'Inference batches: {args.num_inference_batches}')\n        speed = benchmark_score(symblock, ctx, batch_size,\n                                args.num_warmup_batches, args.num_inference_batches, data_layer_type)\n        logger.info('batch size %2d, image/sec: %f', batch_size, speed)\n"
  },
  {
    "path": "example/quantization/launch_inference_onednn.sh",
    "content": "#!/bin/sh\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nusage()\n{\n    echo \"usage: bash ./launch_inference_onednn.sh [[[-s symbol_file ] [-b batch_size] [-iter iteraton] [-ins instance] [-c cores/instance]] | [-h]]\"\n}\n\nwhile [ $# -gt 0 ]; do\n  case \"$1\" in\n    --symbol | -s)\n      shift\n      SYMBOL=$1\n      ;;\n    --batch-size | -b)\n      shift\n      BS=$1\n      ;;\n    --iteration | -iter)\n      shift\n      ITERATIONS=$1\n      ;;\n    --instance | -ins)\n      shift\n      INS=$1\n      ;;\n    --core | -c)\n      shift\n      CORES=$1\n      ;;\n    --help | -h)\n      usage\n      exit 1\n      ;;\n    *)\n      usage\n      exit 1\n  esac\n  shift\ndone\n\nNUM_SOCKET=`lscpu | grep 'Socket(s)' | awk '{print $NF}'`\nNUM_NUMA_NODE=`lscpu | grep 'NUMA node(s)' | awk '{print $NF}'`\nCORES_PER_SOCKET=`lscpu | grep 'Core(s) per socket' | awk '{print $NF}'`\nNUM_CORES=$((CORES_PER_SOCKET * NUM_SOCKET))\nCORES_PER_NUMA=$((NUM_CORES / NUM_NUMA_NODE))\necho \"target machine has $NUM_CORES physical core(s) on $NUM_NUMA_NODE numa nodes of $NUM_SOCKET socket(s).\"\n\nif [ -z $SYMBOL ]; then\n  echo \"Error: Need a symbol file as input.\"\nfi\nif [ -z $INS ]; then\n  echo \"Default: launch one instance per socket.\"\n  INS=$NUM_SOCKET\nfi\nif [ -z $CORES ]; then\n  echo \"Default: divide full physical cores.\"\n  CORES=$((NUM_CORES / $INS))\nfi\nif [ -z $BS ]; then\n  echo \"Default: set batch size to 64.\"\n  BS=64\nfi\nif [ -z $ITERATIONS ]; then\n  echo \"Default: set iterations to 500.\"\n  ITERATIONS=500\nfi\n\necho \"  benchmark configs\"\necho \"  cores per instance: $CORES\"\necho \"  total instances: $INS\"\necho \"  batch size: $BS\"\necho \"  iterations: $ITERATIONS\"\necho \"\"\n\nrm BENCHMARK_*.log  || echo \"benchmarking...\"\n\ni=0\nwhile [ \"$i\" -lt $INS ]; do\n  a=$((i * CORES))\n  b=$((a + CORES - 1))\n  memid=$((b/CORES_PER_NUMA % NUM_NUMA_NODE))\n  LOG=BENCHMARK_$i.log\n  echo \"  Instance $i use $a-$b cores and mem $memid with $LOG\"\n  KMP_AFFINITY=granularity=fine,noduplicates,compact,1,0 \\\n  OMP_NUM_THREADS=$CORES \\\n  nohup numactl --physcpubind=$a-$b --membind=$memid python imagenet_inference.py --symbol-file=$SYMBOL --batch-size=$BS --num-inference-batches=$ITERATIONS --benchmark=True > $LOG 2>&1 &\n  i=$(( i + 1 ))\ndone\nwait\n\nfps=`grep image/sec BENCHMARK_*.log | awk '{ sum += $(NF) }; END { print sum }'`\nif [ -z \"$fps\" ]; then\n  echo \"FPS not found in benchmark log.\"\n  return 1\nfi\nlatency=$(awk \"BEGIN {printf \\\"%.2f\\\", 1000*${BS}*${INS}/${fps}}\")\necho \"overall throughput (image/sec): $fps\"\necho \"latency per batch per instance (ms): $latency\"\necho \"benchmark finish:)\"\n"
  },
  {
    "path": "example/quantization_inc/custom_strategy.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport copy\nimport numpy as np\nfrom collections import OrderedDict\nfrom neural_compressor.strategy.strategy import TuneStrategy, strategy_registry\n\nplot_operator_influence = False\n\ndef calc_approx_error(expected_tensor: np.ndarray, observed_tensor: np.ndarray) -> float:\n    '''\n    Calculating relative error for one tensor\n    '''\n    error = observed_tensor - expected_tensor\n    absolute_error = np.abs(error)\n    mean_absolute_error = absolute_error.mean()\n    mean_expected_value = np.abs(expected_tensor).mean()\n    error = mean_absolute_error / mean_expected_value\n    return error\n\n\ndef get_approx_errors(expected_tensors, observed_tensors):\n    '''\n    Calculating relative error for multiple tensors: Dict[tensors_name: str, tensor: np.ndarray]\n    '''\n    errors = {}\n    for node_name in observed_tensors.keys():\n        expected_tensor = expected_tensors[node_name][node_name]\n        observed_tensor = observed_tensors[node_name][node_name]\n        errors[node_name] = calc_approx_error(expected_tensor, observed_tensor)\n    return errors\n\n\n@strategy_registry\nclass MyCustomTuneStrategy(TuneStrategy):\n    '''INC Custom strategy definition'''\n    def __init__(self, model, conf, q_dataloader, q_func=None,\n                 eval_dataloader=None, eval_func=None, dicts=None, q_hooks=None):\n        super().__init__(\n            model,\n            conf,\n            q_dataloader,\n            q_func,\n            eval_dataloader,\n            eval_func,\n            dicts,\n            q_hooks)\n\n\n    def get_qtensors(self, quant_cfg, node_list):\n        '''\n        Generating quantized model based on configuration and capturing intermediate tensors\n        '''\n        qmodel = self.adaptor.quantize(quant_cfg, self.model, self.calib_dataloader)\n        tensors = self.adaptor.inspect_tensor(qmodel, self.calib_dataloader, node_list, [1]) # 1 is a batch index\n        return tensors['activation'][0] # we need to specify that we want activation (layer output) because INC stores also weight tensors\n                                        # 0 is the first batch\n    def next_tune_cfg(self):\n        FALLBACK_DTYPE = 'fp32'\n\n        # creating base configuration - all nodes are quantized and calibrated with minmax algorithm\n        best_cfg = {}\n        best_cfg['calib_iteration'] = int(self.calib_iter[0]) # number of batches for calibration\n        best_cfg['calib_sampling_size'] = int(self.calib_sampling_size[0]) # number of samples for calibration (multiplicity of batch)\n        nodes_cfg = OrderedDict()\n        nodes_cfg_idx = {}\n        for node_key, cfgs in self.opwise_tune_cfgs.items():\n            for i, cfg in enumerate(cfgs):\n                if cfg['activation']['algorithm'] == 'minmax':\n                    nodes_cfg_idx[node_key] = i\n                    break\n            nodes_cfg[node_key] = cfg\n        best_cfg['op'] = nodes_cfg\n\n        yield best_cfg\n\n        # If fully quantized model does not meet the requirements, we proceed to exclude some nodes\n\n        # Collecting tensors from the original model - expected tensors\n        node_list = [op_name for (op_name, op_type) in best_cfg['op'].keys()]\n        f32_tensors = self.adaptor.inspect_tensor(self.model, self.calib_dataloader, node_list, [1])\n        f32_tensors = f32_tensors['activation'][0]\n\n        # Collecting tensors from the fully quantized model\n        q_tensors = self.get_qtensors(best_cfg, node_list)\n        approx_errors = get_approx_errors(f32_tensors, q_tensors)\n\n        # best_cfg['op'] is an OrderedDict, which order of elements should correspond to their\n        # order in the computational graph\n        for node_key, cfg in best_cfg['op'].items():\n            # Node's key in INC is its name + its operator\n            node_name, node_op = node_key\n            # Checking what configuration options are available for this particular node\n            capabilities = self.opwise_tune_space[node_key]['activation']['dtype']\n            # If a particular node can be excluded from quanrtization ('fp32' in capabilities)\n            # and current error is bigger than threshold value, we check what accuracy improvement\n            # would be achieved by this exclusion\n            if FALLBACK_DTYPE in capabilities and approx_errors[node_name] > 0.06:\n                original_dtype = cfg['activation']['dtype']\n                cfg['activation']['dtype'] = FALLBACK_DTYPE # Exclude the node from quantization\n\n                # Collecting tensors for a new configuration with the current node excluded\n                q_tensors = self.get_qtensors(best_cfg, node_list)\n                # Calculating errors for the new configuration\n                new_approx_errors = get_approx_errors(f32_tensors, q_tensors)\n                # Calculating error differences for every node in a model\n                err_diffs = {}\n                for tensor_node_name in new_approx_errors.keys():\n                    diff = approx_errors[tensor_node_name] - new_approx_errors[tensor_node_name]\n                    err_diffs[tensor_node_name] = diff\n                err_diffs_arr = np.array(list(err_diffs.values()))\n\n                # If the sum of errors on the following layers is greater than the threshold value we\n                # keep the node excluded\n                threshold_sum_error_layers = err_diffs_arr.size * 0.007\n                if err_diffs_arr.sum() >= threshold_sum_error_layers:\n                    before = approx_errors\n                    after = approx_errors.copy()\n                    after.update(new_approx_errors)\n                    if plot_operator_influence:\n                        import matplotlib.pyplot as plt\n                        plt.figure()\n                        plt.plot(before.values(), marker='o', markersize=2.5, label='Before')\n                        plt.plot(after.values(), marker='o', markersize=2.5, label='After')\n                        plt.ylabel('Relative error')\n                        plt.xlabel('Layer')\n                        plt.legend()\n                        plt.savefig(f'{node_name}_error.png')\n\n                    approx_errors.update(new_approx_errors)\n                    nodes_cfg_idx.pop(node_key) # Mark node as not quantizable\n                else:\n                    cfg['activation']['dtype'] = original_dtype\n\n        yield best_cfg\n\n        # Choosing calibration algorithm (kl or minmax) for every node which was not excluded from quantization\n        for cfg in self.bayesian_configurations(best_cfg, nodes_cfg_idx):\n            yield cfg\n\n    def bayesian_params_to_tune_configs(self, params):\n        '''\n        Creating configuration from params - changing configurations' indexes for real configurations\n        '''\n        node_cfgs = {}\n        for node_key, configs in self.opwise_quant_cfgs.items():\n            if node_key in params:\n                value = int(params[node_key])\n                value = min(value, len(configs) - 1)\n                node_cfgs[node_key] = copy.deepcopy(configs[value])\n        return node_cfgs\n\n    def bayesian_configurations(self, cfg_base, params_base):\n        from neural_compressor.strategy.bayesian import BayesianOptimization\n\n        # For each node we specify the possible range of values (we treat them as a configurations' index)\n        pbounds = {}\n        for node_key, configs in self.opwise_quant_cfgs.items():\n            if node_key in params_base and len(configs) > 1:\n                pbounds[node_key] = (0, len(configs))\n\n        cfg = copy.deepcopy(cfg_base)\n        if len(pbounds) == 0: # if there is nothing to be optimized, we finish\n            cfg['op'].update(self.bayesian_params_to_tune_configs(params_base))\n            return\n\n        bayes_opt = BayesianOptimization(pbounds=pbounds, random_seed=self.cfg.tuning.random_seed)\n        bayes_opt._space.register(params_base, self.last_tune_result[0]) # registering the outcome of current configuration\n        while True:\n            # Generating next configuration\n            params = bayes_opt.gen_next_params()\n            cfg['op'].update(self.bayesian_params_to_tune_configs(params))\n            yield cfg\n            try:\n                # Registering the outcome\n                bayes_opt._space.register(params, self.last_tune_result[0])\n            except KeyError:\n                pass\n"
  },
  {
    "path": "example/quantization_inc/resnet50v2_mse.yaml",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nversion: 1.0\n\nmodel:\n  name: resnet50_v2\n  framework: mxnet\n\nquantization:\n  calibration:\n    sampling_size: 192 # number of samples for calibration\n\ntuning:\n  strategy:\n    name: mse\n  accuracy_criterion:\n    relative: 0.015\n  exit_policy:\n    timeout: 0\n    max_trials: 500\n  random_seed: 9527\n"
  },
  {
    "path": "example/quantization_inc/resnet_measurement.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport mxnet as mx\nfrom mxnet.gluon.data.vision import transforms\nimport time\nimport glob\n\n\ndef test_accuracy(net, data_loader, description):\n  count = 0\n  acc_top1 = mx.gluon.metric.Accuracy()\n  acc_top5 = mx.gluon.metric.TopKAccuracy(5)\n  start = time.time()\n  for x, label in data_loader:\n    output = net(x)\n    acc_top1.update(label, output)\n    acc_top5.update(label, output)\n    count += 1\n  time_spend = time.time() - start\n  _, top1 = acc_top1.get()\n  _, top5 = acc_top5.get()\n  print('{:21} Top1 Accuracy: {:.4f} Top5 Accuracy: {:.4f} from {:4} batches in {:8.2f}s'\n        .format(description, top1, top5, count, time_spend))\n\n# Preparing input data\nrgb_mean = (0.485, 0.456, 0.406)\nrgb_std = (0.229, 0.224, 0.225)\nbatch_size = 64\n\nstart = time.time()\n# Set proper path to ImageNet data set below\ndataset = mx.gluon.data.vision.ImageRecordDataset('../imagenet/rec/val.rec')\ntransformer = transforms.Compose([transforms.Resize(256),\n                                  transforms.CenterCrop(224),\n                                  transforms.ToTensor(),\n                                  transforms.Normalize(mean=rgb_mean, std=rgb_std)])\n# Note: as the input data is used many times it is better to prepare it once.\n#       Therefore, lazy parameter for transform_first is set to False.\nval_data = mx.gluon.data.DataLoader(\n    dataset.transform_first(transformer, lazy=False), batch_size, shuffle=False)\nval_data.batch_size = batch_size\ntime_consumed = time.time() - start\nprint(\"Input data prepared in {:8.2f}s\".format(time_consumed))\n\nprint(\"Measure accuracy on the whole data set could take a long time. Please wait...\")\nroot_path = '__resnet50_v2_'\nsymbol_part = '-symbol.json'\nfor symbol in glob.glob(root_path + '*' + symbol_part):\n  param = symbol.replace(symbol_part,'-0000.params')\n  net_name = symbol.replace(root_path,'').replace(symbol_part,'').replace('_', ' ')\n  net = mx.gluon.SymbolBlock.imports(symbol, ['data'], param)\n  net.hybridize(static_alloc=True, static_shape=True)\n  test_accuracy(net, val_data, net_name)\n"
  },
  {
    "path": "example/quantization_inc/resnet_mse.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport mxnet as mx\nfrom mxnet.gluon.model_zoo.vision import resnet50_v2\nfrom mxnet.gluon.data.vision import transforms\nfrom mxnet.contrib.quantization import quantize_net\n\n# Preparing input data\nrgb_mean = (0.485, 0.456, 0.406)\nrgb_std = (0.229, 0.224, 0.225)\nbatch_size = 64\nnum_calib_batches = 9\n# Set proper path to ImageNet data set below\ndataset = mx.gluon.data.vision.ImageRecordDataset('../imagenet/rec/val.rec')\n# Tuning with INC on the whole data set takes a lot of time. Therefore, we take only a part of the whole data set\n# as representative part of it:\ndataset = dataset.take(num_calib_batches * batch_size)\ntransformer = transforms.Compose([transforms.Resize(256),\n                                  transforms.CenterCrop(224),\n                                  transforms.ToTensor(),\n                                  transforms.Normalize(mean=rgb_mean, std=rgb_std)])\n# Note: as input data is used many times during tuning it is better to have it prepared earlier.\n#       Therefore, lazy parameter for transform_first is set to False.\nval_data = mx.gluon.data.DataLoader(\n    dataset.transform_first(transformer, lazy=False), batch_size, shuffle=False)\nval_data.batch_size = batch_size\n\nnet = resnet50_v2(pretrained=True)\n\ndef eval_func(model):\n  metric = mx.gluon.metric.Accuracy()\n  for x, label in val_data:\n    output = model(x)\n    metric.update(label, output)\n  accuracy = metric.get()[1]\n  return accuracy\n\n\nfrom neural_compressor.experimental import Quantization\nquantizer = Quantization(\"resnet50v2_mse.yaml\")\nquantizer.model = net\nquantizer.calib_dataloader = val_data\nquantizer.eval_func = eval_func\nqnet_inc = quantizer.fit().model\nprint(\"INC finished\")\n# You can save the optimized model for the later use:\nqnet_inc.export(\"__quantized_with_inc\")\n# You can see which configuration was applied by INC and which nodes were excluded from quantization\n# to achieve given accuracy loss against floating point calculation.\nprint(quantizer.strategy.best_qmodel.q_config['quant_cfg'])\n"
  },
  {
    "path": "example/quantization_inc/resnet_tuning.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Disable noisy logging from INC:\nimport logging\nlogging.disable(logging.INFO)\n\nimport time\nimport mxnet as mx\nfrom mxnet.gluon.model_zoo.vision import resnet50_v2\nfrom mxnet.gluon.data.vision import transforms\nfrom mxnet.contrib.quantization import quantize_net\nimport custom_strategy\n\n\ndef save_model(net, data_loader, description, time_spend):\n  save_model.count += 1\n  print( \"{:21s} tuned in {:8.2f}s\".format(description, time_spend))\n  net.export(\"__resnet50_v2_{:02}_\".format(save_model.count) + description.replace(' ', '_'))\n\nsave_model.count = 0\n\n# Preparing input data\nstart = time.time()\nrgb_mean = (0.485, 0.456, 0.406)\nrgb_std = (0.229, 0.224, 0.225)\nbatch_size = 64\nnum_calib_batches = 9\n# Set proper path to ImageNet data set below\ndataset = mx.gluon.data.vision.ImageRecordDataset('../imagenet/rec/val.rec')\n# Tuning with INC on the whole data set takes too much time. Therefore, we take only a part of the whole data set\n# as representative part of it:\ndataset = dataset.take(num_calib_batches * batch_size)\ntransformer = transforms.Compose([transforms.Resize(256),\n                                  transforms.CenterCrop(224),\n                                  transforms.ToTensor(),\n                                  transforms.Normalize(mean=rgb_mean, std=rgb_std)])\n# Note: as the input data is used many times during tuning it is better to have it prepared earlier.\n#       Therefore, lazy parameter in transform_first is set to False.\nval_data = mx.gluon.data.DataLoader(\n    dataset.transform_first(transformer, lazy=False), batch_size, shuffle=False)\nval_data.batch_size = batch_size\ntime_consumed = time.time() - start\nprint(\"Input data prepared in {:.2f}s\".format(time_consumed))\n\nnet = resnet50_v2(pretrained=True)\n\nstart = time.time()\nnet.hybridize(static_alloc=True, static_shape=True)\ntime_consumed = time.time() - start\n# Run forward path once to cache the graph - required to save the model\nnet(next(iter(val_data))[0])\nsave_model(net, val_data, \"fp32\", time_consumed)\n\nstart = time.time()\nnet.optimize_for(next(iter(val_data))[0], backend='ONEDNN', static_alloc=True, static_shape=True)\ntime_consumed = time.time() - start\nsave_model(net, val_data, \"fp32 fused\", time_consumed)\n\nstart = time.time()\nqnet = quantize_net(net, calib_mode='naive', calib_data=val_data)\nqnet.hybridize(static_alloc=True, static_shape=True)\ntime_consumed = time.time() - start\nsave_model(qnet, val_data, 'int8 full naive', time_consumed)\n\nstart = time.time()\nqnet = quantize_net(net, calib_mode='entropy', calib_data=val_data)\nqnet.hybridize(static_alloc=True, static_shape=True)\ntime_consumed = time.time() - start\nsave_model(qnet, val_data, 'int8 full entropy', time_consumed)\n\nstart = time.time()\nqnet = quantize_net(net, calib_mode='naive', quantize_mode='smart', calib_data=val_data)\nqnet.hybridize(static_alloc=True, static_shape=True)\ntime_consumed = time.time() - start\nsave_model(qnet, val_data, 'int8 smart naive', time_consumed)\n\nstart = time.time()\nqnet = quantize_net(net, calib_mode='entropy', quantize_mode='smart', calib_data=val_data)\nqnet.hybridize(static_alloc=True, static_shape=True)\ntime_consumed = time.time() - start\nsave_model(qnet, val_data, 'int8 smart entropy', time_consumed)\n\ndef eval_func(model):\n  metric = mx.gluon.metric.Accuracy()\n  for x, label in val_data:\n    output = model(x)\n    metric.update(label, output)\n  accuracy = metric.get()[1]\n  return accuracy\n\nfrom neural_compressor.experimental import Quantization\nquantizer = Quantization(\"resnet50v2_mse.yaml\")\nquantizer.model = net\nquantizer.calib_dataloader = val_data\nquantizer.eval_func = eval_func\nfor strategy in ['basic', 'mse', 'mycustom', 'bayesian']:\n  quantizer.cfg.tuning.strategy.name = strategy\n  start = time.time()\n  qnet_inc = quantizer.fit().model\n  time_consumed = time.time() - start\n  save_model(qnet_inc, val_data, \"INC \" + strategy, time_consumed)\n"
  },
  {
    "path": "example/recommenders/.gitignore",
    "content": "ml-100k.zip\nml-100k\n"
  },
  {
    "path": "example/recommenders/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# Recommender Systems\n\n\nThis directory has a set of examples of how to build various kinds of recommender systems\nusing MXNet. The sparsity of user / item data is handled through the embedding layers that accept\nindices as input rather than one-hot encoded vectors.\n\n\n## Examples\n\nThe examples are driven by notebook files.\n\n* [Matrix Factorization: linear and non-linear models](demo1-MF.ipynb)\n* [Deep Structured Semantic Model (DSSM) for content-based recommendations](demo2-dssm.ipynb)\n\n\n### Negative Sampling\n\n* A previous version of this example had an example of negative sampling. For example of negative sampling, please refer to:\n    [Gluon NLP Sampled Block](https://github.com/dmlc/gluon-nlp/blob/master/src/gluonnlp/model/sampled_block.py)\n    \n\n## Acknowledgements\n\nThanks to [xlvector](https://github.com/xlvector/) for the first Matrix Factorization example\nthat provided the basis for these examples.\n\n[MovieLens](http://grouplens.org/datasets/movielens/) data from [GroupLens](http://grouplens.org/).\nNote: MovieLens 100K and 10M dataset are copy right to GroupLens Research Group at the University of Minnesota,\nand licensed under their usage license. For full text of the usage license, see [ml-100k license](http://files.grouplens.org/datasets/movielens/ml-100k-README.txt)\n and [ml-10m license](http://files.grouplens.org/datasets/movielens/ml-10m-README.html). "
  },
  {
    "path": "example/recommenders/demo1-MF.ipynb",
    "content": "{\n \"cells\": [\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"# Licensed to the Apache Software Foundation (ASF) under one\\n\",\n    \"# or more contributor license agreements.  See the NOTICE file\\n\",\n    \"# distributed with this work for additional information\\n\",\n    \"# regarding copyright ownership.  The ASF licenses this file\\n\",\n    \"# to you under the Apache License, Version 2.0 (the\\n\",\n    \"# \\\"License\\\"); you may not use this file except in compliance\\n\",\n    \"# with the License.  You may obtain a copy of the License at\\n\",\n    \"#\\n\",\n    \"#   http://www.apache.org/licenses/LICENSE-2.0\\n\",\n    \"#\\n\",\n    \"# Unless required by applicable law or agreed to in writing,\\n\",\n    \"# software distributed under the License is distributed on an\\n\",\n    \"# \\\"AS IS\\\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\\n\",\n    \"# KIND, either express or implied.  See the License for the\\n\",\n    \"# specific language governing permissions and limitations\\n\",\n    \"# under the License.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"# Matrix Factorization (MF) Recommender Example\\n\",\n    \"Demonstrates matrix factorization with MXNet on the [MovieLens 100k](http://grouplens.org/datasets/movielens/100k/) dataset. We perform **collaborative filtering**, where the recommendations are based on previous rating of users.\\n\",\n    \"\\n\",\n    \"We are trying to learn embeddings for users and movies, based on user partial ratings of movies, to estimate future movie ratings\\n\",\n    \"\\n\",\n    \"![](https://i.imgur.com/twyWChh.png)\\n\",\n    \"\\n\",\n    \"\\n\",\n    \"For more deep learning based architecture for recommendation, refer to this survey: [Deep Learning based Recommender System: A Survey and New Perspectives](https://arxiv.org/pdf/1707.07435.pdf)\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 1,\n   \"source\": [\n    \"import matplotlib.pyplot as plt\\n\",\n    \"import mxnet as mx\\n\",\n    \"from mxnet import gluon, np, npx, autograd\\n\",\n    \"import numpy as onp\\n\",\n    \"\\n\",\n    \"from matrix_fact import train\\n\",\n    \"from movielens_data import get_dataset, max_id\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stderr\",\n     \"text\": [\n      \"DEBUG:matplotlib.backends:backend module://ipykernel.pylab.backend_inline version unknown\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {\n    \"collapsed\": false\n   }\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"### Config\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 2,\n   \"source\": [\n    \"ctx = [mx.gpu(0)] if mx.device.num_gpus() > 0 else [mx.cpu()]\\n\",\n    \"batch_size = 128\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {\n    \"collapsed\": true\n   }\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Data\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 3,\n   \"source\": [\n    \"train_dataset, test_dataset = get_dataset()\\n\",\n    \"max_user, max_item = max_id('./ml-100k/u.data')\\n\",\n    \"(max_user, max_item)\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"execute_result\",\n     \"data\": {\n      \"text/plain\": [\n       \"(944, 1683)\"\n      ]\n     },\n     \"metadata\": {},\n     \"execution_count\": 3\n    }\n   ],\n   \"metadata\": {\n    \"collapsed\": false\n   }\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 4,\n   \"source\": [\n    \"train_data = gluon.data.DataLoader(train_dataset, shuffle=True, last_batch='rollover', batch_size=batch_size, num_workers=0)\\n\",\n    \"test_data = gluon.data.DataLoader(test_dataset, shuffle=True, batch_size=batch_size, num_workers=0)\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 5,\n   \"source\": [\n    \"for user, item, score in test_data:\\n\",\n    \"    print(user[0], item[0], score[0])\\n\",\n    \"    break\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"\\n\",\n      \"[38.]\\n\",\n      \"<NDArray 1 @cpu(0)> \\n\",\n      \"[508.]\\n\",\n      \"<NDArray 1 @cpu(0)> \\n\",\n      \"[2.]\\n\",\n      \"<NDArray 1 @cpu(0)>\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Linear Matrix Factorization\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 6,\n   \"source\": [\n    \"class LinearMatrixFactorization(gluon.HybridBlock):\\n\",\n    \"    \\n\",\n    \"    def __init__(self, k, max_user=max_user, max_item=max_item):\\n\",\n    \"        super(LinearMatrixFactorization, self).__init__()\\n\",\n    \"        \\n\",\n    \"        # user feature lookup\\n\",\n    \"        self.user_embedding = gluon.nn.Embedding(input_dim=max_user, output_dim = k) \\n\",\n    \"\\n\",\n    \"        # item feature lookup\\n\",\n    \"        self.item_embedding = gluon.nn.Embedding(input_dim=max_item, output_dim = k) \\n\",\n    \"    \\n\",\n    \"    def forward(self, user, item):\\n\",\n    \"        user_embeddings = npx.relu(self.user_embedding(user))\\n\",\n    \"        items_embeddings = npx.relu(self.item_embedding(item))\\n\",\n    \"        \\n\",\n    \"        # predict by the inner product, which is elementwise product and then sum\\n\",\n    \"        pred = (user_embeddings * items_embeddings).sum(axis=1)\\n\",\n    \"        \\n\",\n    \"        return pred.flatten()\\n\",\n    \"\\n\",\n    \"net1 = LinearMatrixFactorization(64)\\n\",\n    \"net1.initialize(mx.init.Xavier(), ctx=ctx)\\n\",\n    \"mx.viz.plot_network(net1(mx.sym.var('user'), mx.sym.var('item')), node_attrs={\\\"fixedsize\\\":\\\"false\\\"})\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"execute_result\",\n     \"data\": {\n      \"image/svg+xml\": \"<?xml version=\\\"1.0\\\" encoding=\\\"UTF-8\\\" standalone=\\\"no\\\"?>\\n<!DOCTYPE svg PUBLIC \\\"-//W3C//DTD SVG 1.1//EN\\\"\\n \\\"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\\\">\\n<!-- Generated by graphviz version 2.38.0 (20140413.2041)\\n -->\\n<!-- Title: plot Pages: 1 -->\\n<svg width=\\\"340pt\\\" height=\\\"536pt\\\"\\n viewBox=\\\"0.00 0.00 339.50 536.00\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\" xmlns:xlink=\\\"http://www.w3.org/1999/xlink\\\">\\n<g id=\\\"graph0\\\" class=\\\"graph\\\" transform=\\\"scale(1 1) rotate(0) translate(4 532)\\\">\\n<title>plot</title>\\n<polygon fill=\\\"white\\\" stroke=\\\"none\\\" points=\\\"-4,4 -4,-532 335.5,-532 335.5,4 -4,4\\\"/>\\n<!-- user -->\\n<g id=\\\"node1\\\" class=\\\"node\\\"><title>user</title>\\n<ellipse fill=\\\"#8dd3c7\\\" stroke=\\\"black\\\" cx=\\\"77.5\\\" cy=\\\"-29\\\" rx=\\\"47\\\" ry=\\\"29\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"77.5\\\" y=\\\"-25.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">user</text>\\n</g>\\n<!-- linearMF_emb_user_fwd -->\\n<g id=\\\"node2\\\" class=\\\"node\\\"><title>linearMF_emb_user_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"155,-152 0,-152 0,-94 155,-94 155,-152\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"77.5\\\" y=\\\"-119.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">linearMF_emb_user_fwd</text>\\n</g>\\n<!-- linearMF_emb_user_fwd&#45;&gt;user -->\\n<g id=\\\"edge1\\\" class=\\\"edge\\\"><title>linearMF_emb_user_fwd&#45;&gt;user</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M77.5,-83.7443C77.5,-75.2043 77.5,-66.2977 77.5,-58.2479\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"77.5,-93.8971 73.0001,-83.897 77.5,-88.8971 77.5001,-83.8971 77.5001,-83.8971 77.5001,-83.8971 77.5,-88.8971 82.0001,-83.8971 77.5,-93.8971 77.5,-93.8971\\\"/>\\n</g>\\n<!-- linearMF_relu0 -->\\n<g id=\\\"node3\\\" class=\\\"node\\\"><title>linearMF_relu0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"142.5,-246 40.5,-246 40.5,-188 142.5,-188 142.5,-246\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"91.5\\\" y=\\\"-213.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">linearMF_relu0</text>\\n</g>\\n<!-- linearMF_relu0&#45;&gt;linearMF_emb_user_fwd -->\\n<g id=\\\"edge2\\\" class=\\\"edge\\\"><title>linearMF_relu0&#45;&gt;linearMF_emb_user_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M85.6785,-177.744C84.3789,-169.204 83.0236,-160.298 81.7986,-152.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"87.2235,-187.897 81.2702,-178.688 86.4712,-182.954 85.719,-178.011 85.719,-178.011 85.719,-178.011 86.4712,-182.954 90.1677,-177.334 87.2235,-187.897 87.2235,-187.897\\\"/>\\n</g>\\n<!-- item -->\\n<g id=\\\"node4\\\" class=\\\"node\\\"><title>item</title>\\n<ellipse fill=\\\"#8dd3c7\\\" stroke=\\\"black\\\" cx=\\\"252.5\\\" cy=\\\"-29\\\" rx=\\\"47\\\" ry=\\\"29\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"252.5\\\" y=\\\"-25.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">item</text>\\n</g>\\n<!-- linearMF_emb_item_fwd -->\\n<g id=\\\"node5\\\" class=\\\"node\\\"><title>linearMF_emb_item_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"331.5,-152 173.5,-152 173.5,-94 331.5,-94 331.5,-152\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"252.5\\\" y=\\\"-119.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">linearMF_emb_item_fwd</text>\\n</g>\\n<!-- linearMF_emb_item_fwd&#45;&gt;item -->\\n<g id=\\\"edge3\\\" class=\\\"edge\\\"><title>linearMF_emb_item_fwd&#45;&gt;item</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M252.5,-83.7443C252.5,-75.2043 252.5,-66.2977 252.5,-58.2479\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"252.5,-93.8971 248,-83.897 252.5,-88.8971 252.5,-83.8971 252.5,-83.8971 252.5,-83.8971 252.5,-88.8971 257,-83.8971 252.5,-93.8971 252.5,-93.8971\\\"/>\\n</g>\\n<!-- linearMF_relu1 -->\\n<g id=\\\"node6\\\" class=\\\"node\\\"><title>linearMF_relu1</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"289.5,-246 187.5,-246 187.5,-188 289.5,-188 289.5,-246\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"238.5\\\" y=\\\"-213.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">linearMF_relu1</text>\\n</g>\\n<!-- linearMF_relu1&#45;&gt;linearMF_emb_item_fwd -->\\n<g id=\\\"edge4\\\" class=\\\"edge\\\"><title>linearMF_relu1&#45;&gt;linearMF_emb_item_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M244.322,-177.744C245.621,-169.204 246.976,-160.298 248.201,-152.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"242.777,-187.897 239.832,-177.334 243.529,-182.954 244.281,-178.011 244.281,-178.011 244.281,-178.011 243.529,-182.954 248.73,-178.688 242.777,-187.897 242.777,-187.897\\\"/>\\n</g>\\n<!-- linearMF__mul0 -->\\n<g id=\\\"node7\\\" class=\\\"node\\\"><title>linearMF__mul0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"206.5,-340 96.5,-340 96.5,-282 206.5,-282 206.5,-340\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"151.5\\\" y=\\\"-307.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">linearMF__mul0</text>\\n</g>\\n<!-- linearMF__mul0&#45;&gt;linearMF_relu0 -->\\n<g id=\\\"edge5\\\" class=\\\"edge\\\"><title>linearMF__mul0&#45;&gt;linearMF_relu0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M127.649,-273.428C121.749,-264.383 115.519,-254.828 109.923,-246.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"133.172,-281.897 123.94,-275.979 130.441,-277.709 127.709,-273.521 127.709,-273.521 127.709,-273.521 130.441,-277.709 131.479,-271.063 133.172,-281.897 133.172,-281.897\\\"/>\\n</g>\\n<!-- linearMF__mul0&#45;&gt;linearMF_relu1 -->\\n<g id=\\\"edge6\\\" class=\\\"edge\\\"><title>linearMF__mul0&#45;&gt;linearMF_relu1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M185.028,-274.545C193.892,-265.172 203.338,-255.182 211.787,-246.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"178.076,-281.897 181.677,-271.539 181.511,-278.264 184.947,-274.631 184.947,-274.631 184.947,-274.631 181.511,-278.264 188.216,-277.723 178.076,-281.897 178.076,-281.897\\\"/>\\n</g>\\n<!-- linearMF_sum0 -->\\n<g id=\\\"node8\\\" class=\\\"node\\\"><title>linearMF_sum0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"203.5,-434 99.5,-434 99.5,-376 203.5,-376 203.5,-434\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"151.5\\\" y=\\\"-401.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">linearMF_sum0</text>\\n</g>\\n<!-- linearMF_sum0&#45;&gt;linearMF__mul0 -->\\n<g id=\\\"edge7\\\" class=\\\"edge\\\"><title>linearMF_sum0&#45;&gt;linearMF__mul0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M151.5,-365.744C151.5,-357.204 151.5,-348.298 151.5,-340.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"151.5,-375.897 147,-365.897 151.5,-370.897 151.5,-365.897 151.5,-365.897 151.5,-365.897 151.5,-370.897 156,-365.897 151.5,-375.897 151.5,-375.897\\\"/>\\n</g>\\n<!-- linearMF_flatten0 -->\\n<g id=\\\"node9\\\" class=\\\"node\\\"><title>linearMF_flatten0</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"209.5,-528 93.5,-528 93.5,-470 209.5,-470 209.5,-528\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"151.5\\\" y=\\\"-495.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">linearMF_flatten0</text>\\n</g>\\n<!-- linearMF_flatten0&#45;&gt;linearMF_sum0 -->\\n<g id=\\\"edge8\\\" class=\\\"edge\\\"><title>linearMF_flatten0&#45;&gt;linearMF_sum0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M151.5,-459.744C151.5,-451.204 151.5,-442.298 151.5,-434.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"151.5,-469.897 147,-459.897 151.5,-464.897 151.5,-459.897 151.5,-459.897 151.5,-459.897 151.5,-464.897 156,-459.897 151.5,-469.897 151.5,-469.897\\\"/>\\n</g>\\n</g>\\n</svg>\\n\",\n      \"text/plain\": [\n       \"<graphviz.dot.Digraph at 0x7f1e1d805588>\"\n      ]\n     },\n     \"metadata\": {},\n     \"execution_count\": 6\n    }\n   ],\n   \"metadata\": {\n    \"collapsed\": false\n   }\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 7,\n   \"source\": [\n    \"net1.summary(user.to_device(ctx[0]), item.to_device(ctx[0]))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"--------------------------------------------------------------------------------\\n\",\n      \"        Layer (type)                                Output Shape         Param #\\n\",\n      \"================================================================================\\n\",\n      \"               Input                              (128,), (128,)               0\\n\",\n      \"         Embedding-1                                   (128, 64)           60416\\n\",\n      \"         Embedding-2                                   (128, 64)          107712\\n\",\n      \"LinearMatrixFactorization-3                                    (128, 1)               0\\n\",\n      \"================================================================================\\n\",\n      \"Parameters in forward computation graph, duplicate included\\n\",\n      \"   Total params: 168128\\n\",\n      \"   Trainable params: 168128\\n\",\n      \"   Non-trainable params: 0\\n\",\n      \"Shared params in forward computation graph: 0\\n\",\n      \"Unique parameters in model: 168128\\n\",\n      \"--------------------------------------------------------------------------------\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 8,\n   \"source\": [\n    \"losses_1 = train(net1, train_data, test_data, epochs=15, learning_rate=1, ctx=ctx)\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Epoch [0], Training RMSE 6.1854, Test RMSE 5.2134\\n\",\n      \"Epoch [1], Training RMSE 2.9043, Test RMSE 2.1358\\n\",\n      \"Epoch [2], Training RMSE 1.3456, Test RMSE 1.3472\\n\",\n      \"Epoch [3], Training RMSE 0.9293, Test RMSE 1.0726\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stderr\",\n     \"text\": [\n      \"INFO:root:Update[3126]: Change learning rate to 2.00000e-01\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Epoch [4], Training RMSE 0.7585, Test RMSE 0.9467\\n\",\n      \"Epoch [5], Training RMSE 0.6742, Test RMSE 0.9301\\n\",\n      \"Epoch [6], Training RMSE 0.6587, Test RMSE 0.9139\\n\",\n      \"Epoch [7], Training RMSE 0.6449, Test RMSE 0.9023\\n\",\n      \"Epoch [8], Training RMSE 0.6324, Test RMSE 0.8886\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stderr\",\n     \"text\": [\n      \"INFO:root:Update[6251]: Change learning rate to 4.00000e-02\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Epoch [9], Training RMSE 0.6210, Test RMSE 0.8793\\n\",\n      \"Epoch [10], Training RMSE 0.6100, Test RMSE 0.8764\\n\",\n      \"Epoch [11], Training RMSE 0.6080, Test RMSE 0.8744\\n\",\n      \"Epoch [12], Training RMSE 0.6059, Test RMSE 0.8747\\n\",\n      \"Epoch [13], Training RMSE 0.6039, Test RMSE 0.8717\\n\",\n      \"Epoch [14], Training RMSE 0.6020, Test RMSE 0.8688\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {\n    \"collapsed\": false,\n    \"scrolled\": false\n   }\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 9,\n   \"source\": [\n    \"losses_1\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"execute_result\",\n     \"data\": {\n      \"text/plain\": [\n       \"[(6.185443237304687, 5.213418274168756),\\n\",\n       \" (2.9042590438842772, 2.1358377728492592),\\n\",\n       \" (1.345566930294037, 1.347152523554055),\\n\",\n       \" (0.9292820259094239, 1.0726493737500185),\\n\",\n       \" (0.7584892754554748, 0.9466582980884868),\\n\",\n       \" (0.6742267098426818, 0.9300613839914844),\\n\",\n       \" (0.6587229638576507, 0.9138735935186885),\\n\",\n       \" (0.6448600271701813, 0.9023025612922231),\\n\",\n       \" (0.632410079240799, 0.8885752661212994),\\n\",\n       \" (0.6209696002960206, 0.8793287337965267),\\n\",\n       \" (0.6100408156871796, 0.8763645693754695),\\n\",\n       \" (0.6079610646724701, 0.8743740775782591),\\n\",\n       \" (0.6059287497997284, 0.8747020732065675),\\n\",\n       \" (0.6039103961467743, 0.8717364558748378),\\n\",\n       \" (0.6019688241481781, 0.8687770996883417)]\"\n      ]\n     },\n     \"metadata\": {},\n     \"execution_count\": 9\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"The optimizer used for training and hyper-parameter influence greatly how fast the model converge.\\n\",\n    \"We can try with the [Adam optimizer](https://arxiv.org/abs/1412.6980) which will often converge much faster than SGD without momentum as we used before.  You should see this model over-fitting quickly. \"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 10,\n   \"source\": [\n    \"net1 = LinearMatrixFactorization(64)\\n\",\n    \"net1.initialize(mx.init.Xavier(), ctx=ctx)\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 11,\n   \"source\": [\n    \"losses_1_adam = train(net1, train_data, test_data, epochs=15, optimizer='adam', learning_rate=0.01, ctx=ctx)\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Epoch [0], Training RMSE 1.2345, Test RMSE 0.7134\\n\",\n      \"Epoch [1], Training RMSE 0.6484, Test RMSE 0.6597\\n\",\n      \"Epoch [2], Training RMSE 0.5852, Test RMSE 0.6618\\n\",\n      \"Epoch [3], Training RMSE 0.5195, Test RMSE 0.5936\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stderr\",\n     \"text\": [\n      \"INFO:root:Update[3126]: Change learning rate to 2.00000e-03\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Epoch [4], Training RMSE 0.4531, Test RMSE 0.5900\\n\",\n      \"Epoch [5], Training RMSE 0.2978, Test RMSE 0.4903\\n\",\n      \"Epoch [6], Training RMSE 0.2770, Test RMSE 0.4891\\n\",\n      \"Epoch [7], Training RMSE 0.2710, Test RMSE 0.4920\\n\",\n      \"Epoch [8], Training RMSE 0.2654, Test RMSE 0.4949\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stderr\",\n     \"text\": [\n      \"INFO:root:Update[6251]: Change learning rate to 4.00000e-04\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Epoch [9], Training RMSE 0.2613, Test RMSE 0.4922\\n\",\n      \"Epoch [10], Training RMSE 0.2311, Test RMSE 0.4868\\n\",\n      \"Epoch [11], Training RMSE 0.2284, Test RMSE 0.4876\\n\",\n      \"Epoch [12], Training RMSE 0.2278, Test RMSE 0.4886\\n\",\n      \"Epoch [13], Training RMSE 0.2274, Test RMSE 0.4898\\n\",\n      \"Epoch [14], Training RMSE 0.2272, Test RMSE 0.4899\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"### Visualizing embeddings\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 12,\n   \"source\": [\n    \"ratings = np.dot(net1.user_embedding.weight.data(ctx=ctx[0]), net1.item_embedding.weight.data(ctx=ctx[0]).T).asnumpy()\\n\",\n    \"ratings.shape\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"execute_result\",\n     \"data\": {\n      \"text/plain\": [\n       \"(944, 1683)\"\n      ]\n     },\n     \"metadata\": {},\n     \"execution_count\": 12\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 13,\n   \"source\": [\n    \"# Helper function to print the recommendation matrix\\n\",\n    \"# And the top 5 movies in several categories\\n\",\n    \"\\n\",\n    \"def evaluate_embeddings(ratings):\\n\",\n    \"    plt.figure(figsize=(15,15))\\n\",\n    \"    plt.xlabel('items')\\n\",\n    \"    plt.ylabel('users')\\n\",\n    \"    plt.title('Users estimated ratings of items sorted by mean ratings across users')\\n\",\n    \"    im = plt.imshow(((ratings[:, ratings.mean(axis=0).argsort()[::-1]])))\\n\",\n    \"    cb = plt.colorbar(im,fraction=0.026, pad=0.04, label=\\\"score\\\")\\n\",\n    \"    \\n\",\n    \"    top_5_movies = ratings.mean(axis=0).argsort()[::-1][:5] # Highest mean projected rating\\n\",\n    \"    worst_5_movies = ratings.mean(axis=0).argsort()[:5] # Lowest mean projected rating\\n\",\n    \"    top_5_controversial = ratings.std(axis=0).argsort()[::-1][:5] # With most variance\\n\",\n    \"    \\n\",\n    \"    with open('ml-100k/u.item', 'rb') as f:\\n\",\n    \"        movies = f.readlines()\\n\",\n    \"        \\n\",\n    \"    print(\\\"Top 5 movies:\\\")\\n\",\n    \"    for movie in top_5_movies:\\n\",\n    \"        print(\\\"{}, average rating {:.2f}\\\".format(str(movies[int(movie)-1]).split(\\\"|\\\")[1], ratings.mean(axis=0)[movie]))\\n\",\n    \"    print(\\\"\\\\nWorst 5 movies:\\\")\\n\",\n    \"    for movie in worst_5_movies:\\n\",\n    \"        print(\\\"{}, average rating {:.2f}\\\".format(str(movies[int(movie)-1]).split(\\\"|\\\")[1], ratings.mean(axis=0)[movie]))\\n\",\n    \"    print(\\\"\\\\n5 most controversial movies:\\\")\\n\",\n    \"    for movie in top_5_controversial:\\n\",\n    \"        print(\\\"{}, average rating {:.2f}\\\".format(str(movies[int(movie)-1]).split(\\\"|\\\")[1], ratings.mean(axis=0)[movie]))\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 14,\n   \"source\": [\n    \"evaluate_embeddings(ratings)\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stderr\",\n     \"text\": [\n      \"DEBUG:matplotlib.font_manager:findfont: Matching :family=sans-serif:style=normal:variant=normal:weight=normal:stretch=normal:size=10.0 to DejaVu Sans ('/home/ubuntu/anaconda3/envs/mxnet_p36/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSans.ttf') with score of 0.050000\\n\",\n      \"DEBUG:matplotlib.font_manager:findfont: Matching :family=sans-serif:style=normal:variant=normal:weight=normal:stretch=normal:size=12.0 to DejaVu Sans ('/home/ubuntu/anaconda3/envs/mxnet_p36/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSans.ttf') with score of 0.050000\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Top 5 movies:\\n\",\n      \"Schindler's List (1993), average rating 4.18\\n\",\n      \"L.A. Confidential (1997), average rating 4.16\\n\",\n      \"Star Wars (1977), average rating 4.16\\n\",\n      \"Titanic (1997), average rating 4.15\\n\",\n      \"Shawshank Redemption, The (1994), average rating 4.13\\n\",\n      \"\\n\",\n      \"Worst 5 movies:\\n\",\n      \"Homage (1995), average rating -0.00\\n\",\n      \"Bird of Prey (1996), average rating -0.00\\n\",\n      \"Promise, The (Versprechen, Das) (1994), average rating -0.00\\n\",\n      \"Fear, The (1995), average rating -0.00\\n\",\n      \"Window to Paris (1994), average rating -0.00\\n\",\n      \"\\n\",\n      \"5 most controversial movies:\\n\",\n      \"Pulp Fiction (1994), average rating 3.68\\n\",\n      \"Independence Day (ID4) (1996), average rating 3.15\\n\",\n      \"Clockwork Orange, A (1971), average rating 3.29\\n\",\n      \"Big Night (1996), average rating 3.27\\n\",\n      \"Apocalypse Now (1979), average rating 3.54\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"display_data\",\n     \"data\": {\n      \"image/png\": \"iVBORw0KGgoAAAANSUhEUgAAA5YAAAHzCAYAAABbrYK+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsvXm0b9lV1/uZa+3m15z+tnVv3brVpQLEhCaAAYMPnmAwgtijYoOKkNf5VBDF5hkYojyfA7F9Nu8pQpRGnjikcQQYNCIhIQTpQiWpSvXNbc49/a/ZzVrz/THX/p1fXe65VZUU1C3Y3zHOOOf89t5rzdXsved3ze9cP1FVevTo0aNHjx49evTo0aNHj48V7tU2oEePHj169OjRo0ePHj16vLbRE8sePXr06NGjR48ePXr06PFxoSeWPXr06NGjR48ePXr06NHj40JPLHv06NGjR48ePXr06NGjx8eFnlj26NGjR48ePXr06NGjR4+PCz2x7NGjR48ePXr06NGjR48eHxd6YtmjR49fN4jIfxGRP/Vq23EriMiPi8hX/BqW/zki8uFfq/I/FojI7xORp0XkSEQ+9RbHj0Tk/lfDttcCROSdIvKuE459rog88+tt02sNd+J90aNHjx49Pjb0xLJHjzsIIqIi8uBNn53ovN7JuJXdqvq7VPXf/hrUdW/qu+yVLvtjxc1jqao/qaqvfzVtugX+PvC/quqKqv73mw+mzx8DEJFvFZG//etu4a8hROQJEfn8V9uO30x4jdwXPXr06NHjY0BPLHv0+E2IO4mAvRbxG6j/LgMffLWN+PXGb6Dxu6Pwm61ff6O09zdKO3r06PHqoyeWPXq8hiAip0Xk+0VkT0R2ROQnRcSlYxdE5P8Tkesi8riI/Pml694pIt8jIu8SkQPgy0XkM0XkZ0XkQESuisg336beLxKRn0/1vkdE3rR07K+IyLMicigiHxaR3yEiXwj8NeBLk5zyF9K5C7mpiHy5iPyUiPyDVO5jIvLZ6fOnReTasmxWRH63iPz3ZO/TIvLOJRP/a/q9l+r7rHTNnxGRh0VkV0TeLSKXl8r7AhH5kIjsi8g/AeQ27T+p/3462f68iPwTESnS+Z09v5Ds+dKbpZEpWvY1IvKLyYbvEpHB0vGvTeU+JyJfsRzpEZG3i8ivpD5/VkS+5gS7nYj8DRF5MvXnt4nIuoiUInIE+GTjR0+4XkXkQRH5SuDLgK9N7fm+dPzF5tx/SH12KCK/JCIPicjXJVueFpHfuXT+l6c5cJjK+rITbDpx3orI7xGRD6Yx+XER+cSb+vuviMgvAhMR+Q7gHuD7Upu+Np33ljTH90TkF0Tkc5fKuE9EfiLZ+MPA6VvZeJO9f01EtlP9X5Y++4xku1867/dLuk9uUca3isg/E5OSH4ndN+dF5FvS3P6QLEmZX2RcTpy36biKyDtE5JF0zj8VkVveG/LavS8eEJEfFZEbaWz+nYhsLB2/JCL/MfXfDbHnw83PrBvAO+WEeyydP0h9cyP1x/tF5NxSWS9lvr9AKXCL/vpVz9/0uRORvyoiH031f7eIbKVjncLjz4rIU8CP3s7WHj169HjJUNX+p//pf+6QH0CBB2/67J3Au9Lffxf450Cefj4HI0QO+ADwfwAFcD/wGPC2pTIa4Pemc4fATwN/Ih1fAd5ygk2fClwDfitGRP4U8ARQAq8HngYupHPvBR642e6lsn4c+Ir095cDLfCnU7l/G3gK+Kep7N8JHAIr6fzPBd6Y7H8TcBX4vUv1KpAt1fUlwKPAJwIZ8DeA96Rjp1PZfzD1419MtnzFCX1wq/57M/CWVPa9wMPAXzhpLJP9zyz9/wTwM8AFYCtd/4507AuBK8AbgBHwruXygOeBz0l/bwKfdoLdfyb1wf1pjP8j8O23m28nzUfgW4G/vXTspcy5OfC21EffBjwO/PXU538OeDydOwYOgNen/+8C3nCCTbect8BDwAT4glT+16a2F0v9/fPAJWC49NnnL5V9EbgBvD217wvS/2eW6v5mbH7+dmwOvesEOz8Xm1Pd+f9Dsq9r468Av2vp/O8FvvqEsr4V2Mbm3AD40dSXf5Lje+fHXuK4vJR5+/3ABka8rwNf+BvsvngwjW0JnMEWpr4lHfPALwD/AJuXA+CtNz2z/rfUviG3uceArwK+L9nqU9+s8fLm+7fywvtu0V/c/vn7vwPvBe5O7fwXwHfc9Lz8tmTL8CRbX+yd1f/0P/1P/7P800cse/R4baHBnJDLqtqo5Scp8BmY8/sNqlqr5cX9K+CPLF3706r6n1Q1quoslfWgiJxW1SNVfe8JdX4l8C9U9X2qGtRyJCvMeQyY0/JJIpKr6hOqesvo1wl4XFX/jaoG4Lswp/8bVLVS1R8CaswJRFV/XFV/Kdn/i8B3YM76SXgH8HdV9WFVbYG/A3yKWNTy7cAHVfV7VLUBvgVzWG+HF/Sfqn5AVd+rqq2qPoE5brez51b4R6r6nKruYE7dp6TP/zDwb1T1g6o6xRz4ZTRYn6+p6q6q/twJ5X8Z8M2q+piqHgFfB/wReWWkby9lzv2kqr479f9/wJz4b0p9/p3AvUuRogj8FhEZqurzqnqSRPekefulwA+o6g+n8v8+5jB/9tK1/0hVn07z/1b448APquoPpnH+YeBngbeLyD2pzX8zzc//io3Zi6E7/yeAH8DGFuDfpvpIkaS3Af/+NuV8b5pzc4yEzlX125bunS5iedtxeYnz9ptUdU9VnwJ+jON5eSu85u4LVX00zZNKVa9j5L+z8TMxUvuXVXWiqnNV/W9Llz+nqv84tW/G7e+xBjiFEd+Q+uYglfNS5/vtcLvn7zuAv66qz6hqlfrqD950778ztbF7H5xka48ePXq8JPTEskePOwsBi7YsI8de+gD/F7Y6/kNJRvVX0+eXgQtJwrQnInuYFHVZyvT0TeX+WSzK86Eke/qiE2y6DHz1TWVfwlbJHwX+Aua0XBOR7xSRCy+jvVeX/p4BqOrNn60AiMhvFZEfS/K0fcxxup0U8TLwD5ds3sGiuxcxx3HRH4mc39w/N+MFx8Vknd8vIleSDPDvvIg9t8IymZ2S2nqzfbew7Q9g5PhJMWnmZ51Q/gXgyaX/n8QiLa+ExO2lzLmbx3I7EaHuf7CI9AQjhu8AnheRHxCRTzih3pPm7QvaqqoR67eLS9e+2BhfBv7QTW16K7aYcwHYTbZ2ePJWhSzhVud398e7gC8WkTFGmH5SVZ+/TVk39+Ut7xNeZFxe4rw9aV7eCq+5+0JEzqVn1bPJxnct2XgJeDIthtwKN9d5u3vs24F3A98pJt39e4kAvpz5fiJe5Pl7GfjepTnwMPZ+OemdcEtbX65NPXr0+M2Nnlj26HFn4SlMprSM+0iOi6oequpXq+r9wO8B/lLKqXkai/5tLP2squrbl8rR5UJV9RFV/aPAWeD/BL4nObk342ngG28qe6Sq35HK+feq+lbMkdFU1q+q7xXAvwf+M3BJVdcxSXCX+3Wrup4Gvuomu4eq+h5MMnepO1FEZPn/E3BzHf838CHgdaq6hjnvJ+Zpvkw8j0nYOrzANlV9v6p+CTZ2/wn47hPKeQ4blw73YFK+q7c+/ba4uf0vZc699MItsvkFGIn7EBZlu9V5J83bF7R1aUyfvU0bbtWmb7+pTWNV/SZsTDZvukfueZFm3er851I7nsWktb8f+BOYY/9K4MXG5ZWet6/F++LvYHa/Mdn4x5dsfBq45zZR/Zvbe+I9pqYq+XpV/SQscv5FmHz5Jc93TD49Wvr//AuMOfn5+zQmtV6eB4M0735VW25na48ePXq8VPTEskePOwvfBfwNEbk7bb7w+cAXA98Di010HkxO8z62Ah2xnKTDtJHDUES8iPwWEfmMkyoSkT8uImdSZGcvfRxvceq/At6RIoYiImOxjXRWReT1IvI/ikiJ5dPNlsq4ikkdX6nnzCqwo6pzEflM4I8tHbue6l3+zsV/DnydiLwhtXddRP5QOvYDwBvENkzJgD/PTQ7bS7TnADhK0Yb/6abjV2+y5+Xgu4E/LSKfKCIj4G92B0SkEJEvE5F1NcnnAbceNzC58F8U23RmBXOov+s20Zjb4eb2vOw5dxJSBOlLEgmrgCNOaNNt5u13A79bbPOoHPjqVNZ7Xkabuiji21J7BmKbpdytqk9istivT2PwVuzefDF0538O5qz/h6Vj34blgr4Ry817JfBi4/Ji8/bjxWvhvljF5ti+iFwE/vLSsZ/BCOw3pWfdQER+221sOvEeE5HPE5E3im3SdIApT+LLme9YXvDbRWRLRM5jEcquzbd7/v5z4BslbVgmImdE5EtOasRJtt6m3T169Ojxq9ATyx497ix8A+YI/zdgF/h7wJep6i+n468DfgRzRH4a+Geq+mNJXvhFWC7S49hGH/8PsH6bur4Q+KDY7qD/EPgjeovcM1X9WWyjlX+SbHoU28QCLL/nm1J9V7BIwdelY50DfUNETsoBfDn4n4FvEJFDbGOSRTQi5Vt9I/BTSfr1FlX9Xmz1/jvF5G6/DPyudP428IeS7Tewfv2pl2nP12Dk9hAj39910/F3Av822fOHeRlQ1f8C/CMsv+1RbBMOMCcULML1RGrXO7A8r1vhX2ORsP+KzYs5tvHIx4L/F8vl2hOR//QxzrmT4IC/hEV/drB8t5MIzy3nrap+GIs8/eNkyxcDX6yq9W3q/bvYQs6eiHyNqj6Nbfr017DFiqcx0tG9K/8YtonVDvC3MGJ4O1zB7pnngH+HbULzoaXj30uSLKY5/HHjJYzLi83bjxevhfvi64FPwxbnfoAlUp/674ux3O6ngGcw2epJuN09dh5bFDzApKg/kc59OfP927HNhJ4AfogX9uftnr//EFN4/FB6Zr4Xm7sn4SRbe/To0eMlQyy1qEePHj163KkQ+9qMXwbKjzHa2OMOhdhXvXyVqv7Iq23Law39fdGjR48edxb6iGWPHj163IEQkd8n9n2Tm1jk9ft65/k3FkTkD2B5bj/6atvyWkF/X/To0aPHnYs7iliKyBeKfcHvo3K822WPHj16/GbEV2HfH/pRLJf2lc6F6/EqQkR+HNvo5n9J+aI9Xhr6+6JHjx497lDcMVLYlDD+EexLi58B3g/8UVX9lVfVsB49evTo0aNHjx49evTocVvcSRHLzwQeVfuS4Rr78uwTdzDr0aNHjx49evTo0aNHjx53Bk76nqZXAxd54Zf1PsPtdzCj3BhodnYTFFbzilYdQQUvSqvGmTOJzIM1UwRUIUTHMG9ooyOqEKJDRAnBMSpq2ugpfUsVMwSlcIF5yGiqHHxEBDIfaIOnyFqa4CmygCooQtN6Mh9QFRBQFUZZzaQpWC/m7NUDnCiqgndxYZcTNZuiQ6OQZQFJ5zmnC9tjtP87qAreR9rKk5et1Qu0wUEQXG4qK+cUL5GqyRABkeMyYhQyb+cFFbT2lMOaqEIbrD2Zi8ybHG2FvGxp6owsD4Rphh+3tLW3wgQQtT8CoILkkcwHmiqnGDS0wREbh2QKomhwOB9RQJfaJ6LE6Oza1i/6qsgCdZ0xGDTMZzl46L6SK/ORdpZBEY/HPfVnW3tbTklNF69oFIiQFYEQHapms3WGgDcbiQKtkI8amiZD0pjkWaCpMmt3FMgUGoE8mi3B5mKRBeomg9bmBV5BQWpBM+szl0eYeHQUcS4SqgxXBGLjzCYFCcAwoo0Dp8lWfUFfSi1ooUhj7dAiInNn9ThNYwRE6wMqZ8Vkau1aKltS9pI6u0YCaGb1ae0QBfVme6w8eH1Bm4ipL33qdBUki2gQO28Qkdqh2fG5rgjo3L/gs25eSWNtI1p/vKAfCz3+ZjZJdYWl9hfWdj8VYpbaEcTanisEWbTPVVZvLNJcbJb6NFWNt7kkrdWnAhJB89R3SeAoDeBSH6qdg0IswKU9S2MBLvV1zNMcdWmMQrqlgJjxwm/v9NYXEpZsSOerLI2bWlmaK6629ks029WDlgqt2HUOsim0K6m/0xLkwr705pDA4n7yFYTCrhWsD/3s+Nrojx8LnR1+JouyNLO+UJds92a/a9NnwfrFNXaMVJak/gml2RwG6ZbwNoaL8sRslZDmsDc7YnHcX4vPsd/SWv1x6WviO1uskcmGZCPO/o6lHXPNcR917ddUdleuq5falaU2BzvfhTSe3ZzWpXmhx7Z3xySaHerSHEs2dmO/sCMe19G1WZ19tlyOBNJ9ksY+pls6zU8Jdm47BN8c2xLz42Pqjvu6s7vrE+tfpR2KjbOkfu/q1+Myu3tnMZfTPO3GiphuC11q/2LQjsvs0LV/cUpmY4Aej4fVoyBi4xHT3I9Wp68U9en5kfrahe7eUGIux/YkG2KW7pdGiZk9I6K3foi5LPrN5pUSSjunuy/VpbFQq9u1SvSS+kpTfwuuOa7f15GYOyQq6uyaUNixrl5X6+K58oJ7RuS43EzskdOd293PgDr7Q6IunkHqBVFdlGMPTU19n8rNBNeYfa7V9MwSXBtT38qiL2zOaDo3LuokXSNtelar2nUdbioTQJqIeuuTzma8W4y3Lu6d7qY9nr84gdi9z+z8RX90/3e/nSBNsDLd8eeLOpOBEs0eFipCWWqL9etyf3c2Lp4NMYJzx3937e/K6/pfl8au+7y7SZYRu0m4PNAKqi/494VG6dIvveWhm9GpJuXkU2510Us981fhkN1tVT3zMRfQ40VxJxHLlwQR+UrgKwH81gbnv+YvgVOkDMnRF9zVkrDZQOugiGw+m9OupAdjGckOPO2KPf391CGtmIMZhVgocRSQyrE+ccSBorkyziL5jcyc5fRS6Rw010A7VvxcaNYC+Z5HHcRSUW+OKwKr6SU5GCqD5z0IVFuRYtfRrEdcI0hympqtQLbnaTcC/tCj5+f4ZwaEUikOHPWand+9/F0APxOaFV28dIt9IQwgDOxB0mwEBlcyqtMBP3H2ok2Om6uF+qEZulMwes4TSnu5hIGa4+ug3BEmFyPDa47ZebM75kp+JMzPRPzc7HFNevnVQhgq42eEagvaoSanHfxcCKXZlR/YeZ0D3DkF6iHft6fS7K6Aa4TsyOHn1t/ZRGhWlcENoRl3zpHadYfW582qnUdyVIsDaEdWR7kD9bo5gX5uTnGzau3xlTmqg22oNjubj53bzrlXb/PGVXZOzCA/hPxImZ+x8Sh3zVmoNhXXCMOrVm82h3YAxaHZ1DmSANMLkfKGwzVWpoRunOx3s8qC8BUHSrUlC4dFoo1BKCE/Mkc7Zub4uSo5vZgT3owhDGFwDdoVmw+xsOP5odVXHCqTC2LtT3MrlPbju03+08uj2Ddb8kNzWGanZeHEZlNrL87qzSeQHyrzLSGbHTu2fq7sP6SMn3HE7Hg+SSIBfpZs7N6XMTm3IxheVZo1mw/Z1Potmyn1ulDsK/WGEHIY7CiuhWpTyI5s7NqhtUcCtGP7e7CtTC5a231lTp6rrb3Z1NqqYnWpTzZcs/qyqRILwVdKcaA043Tu3BzCZmz2uyZ95oRQ2FjFXBb3QjbVhbPtK6jXxJzauTlPYWhzc7ATmZ8yxyJmUO4qsYBQmAOXT63P5lvHhNfXdk+GIg2gS86pCMWRMj9lc9sFtbmYmx31qs3pck9pVoXBjYhrlWbkaFbMaYq59Xm3GNEOj59VKMxPC+WOEgZWTrUuZHMWZLgdC36uZHM4uiiMn9PFc8k1Vk47sD5xrZWfT5VmJIu51I3VgtTq0u9oc9glchO9UB5G5utuMdektT6KuTnjKMRMyKeRduiQoAsH3Mozh74Z25zOp9bR6m3edURp2cbBXmC+7slnSrVmz6p8psfESaA4jDQjc7yzmc0rdTZuzcieVwi4xvo7mynt4LiObJbsjRByO7fcj8TM7G2HsmhPPlMjRZlQ7LfMzuSEUiiOIiEXsrlSrzjymTn2vopUa55sHq1fBo52IBSTY2bXDN2iHzsyo17IppGY5p6fRZpVTyjsfgjp3lkmyOpS/0/igiTF3Pogq2wuu0bTs1jIJolILDm3EnSh1QpFIjMOmrGjOIwLZ9jVEV9bPSF3hKEnlEI2i2TTQLWRM9iuaccZMTM7smlAMyE/aGnHGerAzyNh6JAAfh4IpXuB028kzyXCbuPROc7ZPBAzRyjcghz6KixIUywc0qaF46C4KhBLT/RCftRQrxe4VvGzFs0d0irt0FPs14RRRkzEKhaO4vqMOMhwbSQMMiORTvDT9KJxQhh4G7fDZkHeYunJ9+Y0GwO7H6Li6kDMHbH0ZEcNAO3YVmfUCX7eIm1EFMIgIzuY064NcFULToiZszIKnwi14BrrCz9vibnH1S3q7TzrCzvXTWviMDfi5QU3b83OUY4/nKNFZsSubtHcI7XZopkD55AmEIsMsmOSJlUAL3aeN7LVkUVU7doIUjeQJ7e6adFBjlQtUtXgnBFHOB5/EaRpjRSHiObZghRK06Y2OCREaMMx8fUe6gYdFHZMBKYzGA2TAzGDLDNbZnP7rCysjBDs+pjuz2UyugSt04qn8xADkudGAqv00s8yqJv0IvFo2yLeoSEel7dMABPx1WZpn62XmVquHZmP4fYn3gY/ot/z5Md8cY+XhDuJWD4LXFr6/+702Qugqv8S+JcAKw+d1wv3bvP8tQ0euvsqjzx3lvHKnKNpRj5uCK3jM+5/kveFB3DDFvYK3LjB3fD49QaeGxC2GsQr+bDBPbyCf/CAEBxF3nJxfZ+HH7lItpMRVkDvnREax2i1YnJtDHkkBoFMWdmcMtkfIkcZ5998hd0fuYvpm+ZWZyXIvRMuntrn+d012idXaMeJ6G42VFuK1p7xmSNmv7JBGCn5rqcdR/yhJ44i/vkBw0/c4+jxdeYXGoiCNoK0QrnjaD/liObxMe1GwE0dcaNFs5yYKXEUkVagiFRnAkRoz9YMHysJpdKsKr6C7IkB9dmWMPD4GUwvBaQVwticlZg54kqgqoU4iFSnNJHrDD1b4R4dUt1X4a8VhEFk/SOeo7thfgaalYivbFU1XJ7R7BdGcu+rCMOcwTXH9IEad5jhKqFdixRnprQfWUEzjJwr1FsBBPzEHobNqZYw9JQ7jmacXrAXZxQ/Nebw3kgszZkZXHdMLra0K552NZDvemb3NQyfKKgfmCFXBoT1lnw7IwyUelPJD4R6zcirRCNo7SiSHzpCAcWeEEpo1wLDZz3tSGk2A/XlQPlEScyVdqtleC2nHZhT7ufmVHWLAvWmUu4Z0Z5dUvKJEAZKHERCIVRbirSJ2MyFel0ZXDciH1YCMvdMnOInDl+DRCF4W+RoV5VyT4gBqnNmt08OdjuEyRvmFE8MEIXpXfbAHmynOoPVNz+jFB9OUYuR2bv1y3D9zUqx5xar6PPzgfKaOR3VlvVdKIV2bES9WVGOHmjJ9jL83IhNM8acXw/u0Mg2FUzPgausze2W2etqmJ5X8kMhb2Fyt1LsCeWefe6CUOzD4f0p0tBAdcrONzJmjnIXEVInzM7YwklH6Jr1SLFnCxfN2IgGp22cmlXr23YERWuLDa5NJDA7JjExT/NkCPNTKYJ2Rth4xEgRCtN1RzuGtccju58g3PWelmbs2L/fMbymiOsWSQCB2VmhOLB6ZqfB13bcVbIgsypweNktoh8IuNbIrTojo/nU7uN2BCvPGGGeXoDTP6/Mt6AZ23wuDsUiNrtGTgZTu4easbU5mypH9wVWHvc0Y6FeBRVHNlXCwMhgR+5dYyQ+lMLsjDC6oszOCMWBkh8aAY+5LY40K/b35KKy/lFbnAmltS2bLUWcxPq5HXcLWUo7NKe8LWHvE5TNh4V2KORBj+dYYyR4eF1xQZmdOp6/rrG+2vkEz/C6kh8os7Ejr404tgNQ78hmSnGkTM95iGmRZ0VsoSIRyjAwQt2sgK+F6TlHuWv/L4hVk54pGTQjsyMUVpZrjaD62uqabwq+drYw4YVyX6lXrE0xE+ZnYPUJtSgSFgnav98x2Dbb6xFUa55yXwkFzE458iMlK82WZsXhGjumXqjWHaPtSLUuxDxPpFPIJ1BtCNlVpR3BcSaNs3kfhWacoQKD/YifK/WqwwVrV6iNnEIXkVOqDXtuNSPBl0JxaCRVopHytnQ0I3teZHO1hbuZzbN6xebfYMcIbjNKhD+RPF8rvo60I28LAnkXPReKw0DM7Rk1O+UY7BppVgftwFGvCcWhI59Eit0aSouGZVGpVz3NyBEzoTqV4+dGzGMugE/3pkuRRcXPW2Znh+RHER15QumoVxwrz9X2TBKIpaSouxH16ZnMFn0UQumIuZAfNsTcUW3k5JNjxzoMrU4JkAPt0JMfNMTSU69nlDsNEpV6lOGa2DlShNyhXij2G0Q9cZQT8yQ3cPaMzCYN0kbm54b4uZGqduSQNiPfnVOdHzG4MqVdL1Ev1OsZEqHcroilJxSO6tKI/CggqS8QIQwzXCLKMRf8PLOIXCKVftbQbAzIDmvCKCdmsohWhmGOhEgsM9ysNZtWywXBjsUQf1ShRYZUgTAucLUR/nZziJ8YGdLcG7mtWsK6ETLNHG4maOmRKiCJ5OkwhzaiQ4+b1miREcscUUWmFVrmuKpCiyVpw6gkDjKkyHAhoIMSHea4/akRwUTwdGCET51bRDJpGnQ0MCIZoh0rciOheYZM55BnSNUcR0bzHJrWCFyWQZFbHVlmv0OEzB9HU52DGNG2k6HocSRW4+JaWRmhh0fH5/lOmebs70QOJctAI5JIszoHTZPsao6vW4a4Y3K5/PcJkBShfq1vdfa2zxvrjZ2PnRyfhA/8YvVuVf3CV7zgl4k7afOeDNu853dghPL9wB9T1Q+edM2FN2zol77rbTx8cJ7MRc6UR3zK6tN8/5U3cvd4j/1mwH3jG7zv+r1c3VvlrZcfYzWfk0vgPdfu46GN60zagg9tn+ULLn2Yg3bIB67ezaeefZZJKHj4+jkub+6SSeC3n3qE9+7dz889dYnPue+jAGQukEvkfVcv8yfvex8/duMh7hvf4Eeefj2XNvaoQsZbT3+U/7b9AKVvOapLnnzsLOcv32BzMOPhRy7y4ANXODWYEFX40PZZMhe5e32fe0a7OIl8eP8cuQ+cGRzx3qfv5S2XnuAnP/ogMQh//s1/EzX+AAAgAElEQVQ/yg9d+yQ+ce0K77t+L2XW8vjDd/G1v+P7+c9XP5kqZKwXMx65cYZh0XB+5ZAP/ty9fOqnP8pHbpxhPitAlPGwpo2OzdEMJ8oz1ze57/w2z+2tsTaa40Wp2oyD/36K+mLDxqkjRmXNle11Puv+x/mZp+7h7MYRa+Wchx+/wP33XCN3gSpkPPHEWcgio7U5o7JBRLn+5CYX79/mDVvP88PvfROb9+0ynZdcPrXDM3sbTJ9b4a1vfpiDesiVySp16zk4GhIqz7lz++wcjDi7ccT+bEBdZ8Tg+NwHHuGJoy2uHKySuch0XrC1NqENnqNZybCs2b22CsDGmSPa6Di6ssLW3XvsPLuBX6txLhKDZzCsqT+8xvobb7D91AZbl/Y4mg7IssB8VuCzQFEEQnBsrEzZnwwJwRFaT5hmIEq+UuO9MhpUjIuGp588jRu1XDyzx7PXNsiKQEzyX4DPve8RPrx3jiefPQWVx6/VrK3M2D8Y4X2kmeRsnTtg76NbDO855MFT2/zCI5cYbsyZ3RgiZeTyxW12JiMOD4ZcOLvHwbykqnLaxhNnGYOnc3jjISEI3ivVc2PkVEWsPS4P6E4Jopx+YIftx7Yozk1pm4xwlNlLvXK4SnjLZ32In/rQg7g8IF75hAtXeWz7FLOdIW7YMl6dc7gzhsqclMGpGSJK+WNr7H1qzXhrRpk37FxZJ9/OiPfMCQc5MmrJypZ2nrOxdcTh0ZDQODjKIYCOA5cvbfPUc6fQxrFyesLkcIAeZVBGk4NOzSnVUzUc5rDaINsFcRyQYUBbhy8DedEyvzE81t4oZBs17UFhiob1lvGZKc2vrNHcXeOvFLSbLf7IE081uOsFen5OnGemltgu8JXQnGpNTtoKeJBaiBsNcpShZUQKkzDL1MNmjX++JJyrcds5cahoGTh/cZerj5w22W3lOPfQda5eX0dnnmI7o76rgdohgwCpXAB/4E22qyTJryKNKTLiMOAPMtQrca1F8kj+TGmLNWda1s4dMXt4g5grYbO1ftys4VqJhER2Bhb9ry805Fdzmk1TUywUD5uBfN8RRrZw0g7VFCJ7nljataFTBwwgbDW4Q1toIHbRWVMvtOOIFoqbOVwjFLvC7EIgO3SUe8LsjBJWTVmSHwquTfV5ZXDd0awp0RsB9zWEUhlsOyavr5BJRr7rCEMlbLUUz+fUZ1vyGxnFri3aZBNHfSrY+I0i+V5yiqIpUfJDt5A3x4yFwkEi1OsRCbJQTSzkms6is8WB0I6UdjVSblu50dui0vSC9eFgW2jWLGIahseS4mZdGV6xRZdYWJ31hvXt7HJD+XyGnwk+LXK0IyWbGmlpVk1hIwGqLWV43VQOroZ6XReLEcMryuy8LAh9taUUu+bQGUm0hatYQDtQin1J0kwWUe12nGTN3sYbsQh6vS60Qztvcm/L8LlsoRIwYgvlblJ0TJIKICkJXGNqi4WkWEz10A5Z1N/V1c0z6wMWigj19nc7TPf8vJOhGtk3pQGLCKlEXZxvfWgLVdlUqTaE4bYSchsL9Z3U0OSLzYr1b3FgiyQ2F5W2TNHGnIVkNQzs2jCE+Sll/VFTU+QHiq+hPIhMznuyqSkHwsDmImKR/5VnIm0qo1OclPuaFg5szLuFBxWLJlerthDlG1t88ZUdD4UR9GZkUWYXoFqzSLVJeDvJu51XrTlG1wLVhrPFuC4angvVqrD54RnzsyUSlHbgOLjXsfZkJJsnQpL4Sr3mbPHAm00hl2SLLhQV6mCwGxaLBBIhmwTCwBFKRyiE8bNzmvWcmAsH92RsfqQhpD7PjwLNil/Iq03JYQa0A5/Gwt5boXD4ecQFRVq1CK0TJudzhjcCvgpGjgtHs5LhZ4FsFiyCmaLD1VbB6MkDwri0+2vWQuZMyltZxBRYkFgtMpMst9Git6rEQW7EWMTKDZHZhTHjj+7Rbo5w04Y4yMj2Z7TrQ1Ms7M3MDu8X5JgYjVx2RNQ7pGqIK4M0DolEJ2Lo9g4t4tlFGLP0DOzKgOOy2tYilW1rRLZtjf2Js0hmF6H13iKhy7zjZsnuSViKqMb5/Pbn3gY/ot/zAVX99I+5gFcAn/7JA/2Zd9/zipfr73rkVW8b3EHEEkBE3g58C5a59K9V9Rtvd/7ggYv6wDd/BVWV4Zxyem3ClZ01iqKlmuc4p2yuT9g7GNFMcoYbc+aTwnLqanec84WRjVmVU+0M8asNcafAbdUWkdwtyM5NCa0nL1ra1hPmma1Yl/aE0t3CnLkyQOPw45ZwlOEPMnOg9jP0dI1sF2huUai40SDTDFmrYbskDrqVG+ylt17THuXIzCQesYxIygfUYUAqb/l6LmnZu1yqOr1gxpag42YpkjFQ3FwW0koJKYJYKpryMEWtfLdVo9dLpDEnpVmN+JlL0lhZyJNCqYuyYhlxlXlRvjKH1R95y2FzatLglZbiuYKYQRhHsiOT04ZRJDu0lWVfpRd1csiyqclmw0AXsmU/E9qVSL7v0KVFQnv5H/eTa4RsJjQrkTiwfsoOzalEIJtY7oxm5iy2Q5NJ5YfmqMYckJSTVijZxKFeF5LoRR6PkiKrqV+K1AdJdgxQ7DrqdSWMA8WuSb66vCKX8pzyA5NatiOoTkWyqdDlJFlOj0UBu5y+/FCYn44U+9YmPzc5dRhqkpwdy5NjzsLBh+RoViwk4NlUiLmRguM5YnaUuyb1AZMh54cW/QuFSYF9teRgJzVOZ2MYpPEamWx5fsrstMiTRR7rTSU7EnNck4x1IWNOMsOun0KSwbrWnOpyV6hXrUzfsMjTWuRjpXbU60px0GnizN5FW5JkUN1x3lU2Z5HX5WtzLjtJcRcV7HIlO+n28TiZg1juCM3K8bzopJi+snYOti36Wu6YDLReZSFl9lWyJ+/KS8faJL3tVFWJlHUOtq/s784eX3FMoLtEltTOMDRpsWtSfllXZjiOPtbrQn5kn7UjWfRtlqS1Me8idyafJvW/RcGSQ5h1Echj6WVxYBHEbJqkoykymk11QdgsJ80IaZev6IJFk00umc4prNwuZ0mzFDUc2/FmxciY5dFZO8pd+50fWiQMZxEjsyPNmTVJUltLCYjpvo55N1cshywUQjY7bp+k/mlWUr8KxzmOSd7dzZ12nGwo0hxvjse8WbFyu7EOhZBPdCFJ7p4f3Vzs+qwjMJ1MXJ09U1wDxZHZ241PO0qEDDuvPDDZbEfMs9lxekUol6S62Bwy2atSrZocdSHHT4SkPDgeJ9emZ2Od7r+Btacjfd190tndkciO7HQ5dfasOZYoh06q3EV8sHkj4fgd72uT30afJKcpT7H73DUm8fW1kk8i802/SE0IuZGletUtZNDFoR2sVzxZFZNcU5PaIKVfpHEzKbmpJpZh94IzkpXmsaVbCL6OhNKRTaPNqy5/MLBQX9icl6SCMSlxl7fqmrjIwYu5S/VFu4+H3ghTkq6GIkXLOklokujaYkG0aN/I42cxzfdUXmtETAJkRw1h4Be5nqE0abVvoklb20j0jmza0I5z/Kwllh4/a2lHuUVTVcHL8b3iHX7WIqrE3C+iln5/TrsxMDl07ixf0okRsNybrFTs3uxyIf2kJowLFjmPQCxNFuvqYNE7b2WR5K4LKa0XpLKyu7x3YCEHliZA5mwsppVFFVVNemobMSz6VZLcF+eO5a2dfDSEY2mqd0bqnLNI5bKMtos4LhE8aVojbi7JaLvjXR84sahhRxSX8zBTGS8oP0Y0kTkRWeRCEjpW7o4lqd1nsOhvnBjZXJbYLp+3qHZZLvsifCSFKhfR048BdwKxfPMnD/R97777FS83v+ujr3rb4M6SwqKqPwj84Es9f5A3vPHs80zbgjp6dmYjRqOKw50xFy/soMD1vRUAts4dMJmVlKMG7yProxlH8xInynReMJ0XZFmAPBIqDystGkFnnuzMnFPrEzYHM65PVpg3GaFoaVvH+sqc/cMhTaGMzkxo6oyQOYqyYTax7vWDQL5WkecthzPP6PSUpvG46AhOcV5pV1uk8qzfvc/B4ZCibKmujdi8tMf+/gjxynjQMJsUxMqT3cjtBXN5TmwcOvXIKG14Mo4WNSmjyVLP1qgoG5sT9p7asPzKyxOao7S0GYW1s0e0wdG2Ho2CPj0ibJnkNg6s3LhZw15BON3gy0CoPMWzBeM37XBwNITGEYJDJh4u1Dx4fptHnzhHfjWneP0h090hLos0qxF3qkaPMkIpxKGt8IdBRFdbWhX8jZx4rkIbRywy4kpLNmpp9wqTewroKOB2PM3IolVho7XNV2pBNxo0CBxmVAOFDYtg+amjXY9IJfjzM+pZbtfMHfPVhmy1IUahulaixTHhDgIkkhZHcfHic+OGOMnt/yKSX8sXG+TE3Gzxc6G+UBNmBbFQZNzSzh1xrSXOjHhLys+NmQMV4rkK/1xJO44WOdouLdpUCPFsjbtWWB5RrrhWmJ9vIVOaIAv5dLdxkhYRP3UMrzgmWyaxdbWQTYVm1doiAebnUt7xEnn2M7O/ev0M3S2MWCey064oMVNwSu2MfIfWHKp2lKIsKQ+5WTFJbLWlxELxlZHZ+rRJr2OpKd/pmPChQjuOJiXHSGyx76jOBPI9RxBoVyMSHOWuRbGYSMpRTFGV0pyjmEE2F6YXLBLmJ45saiRpftqiAp2zkE3NaWtHVme7orYosBUZPeeoNpVQWhvMiUttHlveprRCvWaENQyOoynNuvVJyJV6zRYFYgntmk0m3bcoULNiZLtZMVJe7liUJxbH+drTu9JCQMr91UR0q3WlvGFkQQLEobHrZcfdNTA/Y+MBx+S8za0vBokMu9qI0+ysSbEkmCy83DVSWK2bPC3kifAnuXeXCxoKecFnobB+ldaikqE06Wv09lszI1jqYXqXLPJ3Q8kix1jFIlrNmkWPwNptiw4WbYp5twggizxYX1kUrKt/cMPqqVfTAtCqkbByT6nXzJ6OcLcjiyzlh+bIz84aaW0HNhaI2T25Wyj2oM2PJcrZRJneZTJty50zu6qNRMxJY7fabZAiaUMeI90kUpwfGQk2Uswi11fUCK/Jzq2f80Nd5FzGlLNb7ivzkcPPoBnaefNNi8zEJA/3iWhWzuSnlmd43L9dnm8oZJFjHk2gwXToFoS1W4yJeYp4rspiQShmkkjv8ZxtB7bAGT3oIJFOD+0w5Viqkcj5hkW0mrHdb36xmNDlWCaSe2i/bcHD8jHbocPXgXpsOZXN0Bah8qnJdWNmZQz2ItW65Yg2Y1tU7chhtWHEocvF7CTMEpW2dLhgxDQUtujREdCYQYyCDk3eu8hn7uaut/zWas0xvBFQD9W6Q6KjOIomVU75y2AE2TWKYvLb+bpndL2lWbHcSnGJmAqEgT2vQymUu4F25NMilJHZLr/aBSU/tMgfQChsdS2bWX5oM3IUByFdmy3Is6sVPw/4KtKsWZQwmwRLT9mtqE6VxMKRH7Y0q9ki0qiZQ8Vks/VGiauj5aQmGa6ruvw/I3cx96m+SLOaGzFOG/1EL2StWm5p6dOCol8sKmQTI7JxmOGnLSH3tgePswhg9M7+9kJ2VBOHKQczWK5pl+8pPiaCm6UcZJMS+3lLLAvLw3QQ1oeW9xiUsDbATyp7twRFQkCHxTHhk8xIIFhEMJFHdc4ijd5b5LHIj2WqIRzncYa4kMcuy3A1N2lxtzkRnQq1u1YVohihvRVRFUFjRJzlnWrdIF3EsiOZIYDzlleZ8jDtuC4IpGTZcYRSErG9iVxKkshqCEiXY3oinNX9cRDLHr/2cC9+yp2LqMLjB1v8wsOXeXD1Olef2+DPve49XLp4g3Fe89xTp/i0S8+wvjrjcDIgBsc9W7v8lnPP87svfJDpRzZsR8+rQ+ppzsZ4xtqpCfmw4U9+2nuJRzm//9M/wGBYM6kKogp7v3yKo2tjVoYVze6AcVGzsTblwr3blHnL5z3wEeRayWx7xOkL+1x40xUQpW08R8+ska9XnF074sFz2wyGNZ/9usc4t3XAb/ukR/l9b3k/h5MBn3X/48QoDM5N+IzzT3Hm1CGx9sQojFfn9vAaKG/5vA+CKKfPHHDv664ieznjsxPytYo3v/ExVn6pZPTQHuvvL9nYnAAg6zXtqYby51bIRi2SKflaxWRacnlrlywLXD67YxGsUcsbHnqGlbUZZy7uEWsPUcifL9BrA1wWed1bn2D3+TV+5+s+hNaeUz+dk5+dEYMwzBrcQcaFn2qJ0XHv5eusvn/I6O4jRuM5btziamF8doKWwaRvWSS7njN44IDVtZmRua0amXraaYastLbR0pkKokUV83smPPjmp9g8d8DgSoZs1WjjyEdN2jwJ8kG6LgCrjUUyPzgmHzas/UqOq4Xz9+zgfOSz7n/coqyDAFG4cPkGUgbwSVY2bClPzyivZqytzbhw77ZNyMbISn4oRpTP1eQHjmYjsvGBkvquhmJf0NZx9mcBAT9zxEHk0qc/y+iZDFcLxZ6QPVeSHwlaRj7/9R/Cn59x+gOOOFTctYKw0XLXTyphLaAX5/i1huHmDL9q+SNuLgwvHiFbFQwCKBw90OK2atz5OWE1svqEEZWw0XL250y2OHjeclCH1xxbD+7QjiPtaoTtkmziGD0ryKUpg21oT6W6KkFHgThUVh8zsqcjy4Ud3n8AmPxvcEPTbotCs2ZOzen3ZYSN1ghoC81m4KFPf5KwEhk/Y6v3xYGRWy2UeiPiZo7RG3epN9OqcwaHD7QMdiwqql5ptsxBq883ZFOh2E+7256pkPWa8dNCfgDdLo7FXorADpXBDWXlaYuQn/4lJZaR6oE52WHaGbK2vOZuI6rqTGB4XRleE7Y+aPWrN+LtGiN+03taQqFUpwKuFtqtFsSIzeqjns2Hsdy/2jbJ0rTBiq+E6SfPmN0VKXct/FBt2ph0u9rOLgZmF1tGzynNRmByb6B93TSpHGDtiUizpuSHtlFUvZ42FWuNODUbMeUQGvkmQvGmPZpVZX7WosbVphGawbaRplAI89O6iJaFwhYTzvx8a6T6SAlDjsdjxXK5pbX6Yw5rv/0qYaQcvb6hWVPqdcXP4fA+2+irHSl85j7jZ5W7frpaEJVsZuQ7/+wdkwQmtEMgwuCGUuzB7Jwe554dKvkRzC62tunPUKg2heZNEyNvq9b+0fUWiSb/rDaNSLXjLlqqrD3ZLHZV7TaM6iKQ+QGL3WnDAGZnbCEom1i0NmbC6jNhYXOzBnXaGGx2xhzddmh9Odixvm1XMOe9VrKJRY9da1FEzRKhHFmUrv3Mw7SoYxGs6XlhvmUEae3RCc0YqlMwuh6Yn5ZFn3Wqi8N7rM/mp9ICQDQbbaxNllgcKu3Y8n6rTWG4o+QTpd6AZmR1o1CvC6PtaJt+JRWBa1PUNCkZ6jUjx0XaiKc8jJarmTYX80ki2skuZ2fEiFOEvU+yKOjwRmC+ZW5MteYWixE7b5st6sinkejh6C5PvSrks2gqnDHME4kMJQx34iLa2oyF6V1GXLO5svJsQ/Q294/uduRHLZPznuIg0IytjLYU6lXr+8mFTv9sJHaw05o0dN2c6OF2w3zTUey31Ct2rQSYb3pmW55sZhHD/KBlesbhZ3ERZZ2c8+x8Yo6rlWK/pdoS6lXPfMOnPGPh8JKn2vDMNy0C24yEYq9eRNSH1xuaoWN62jPYsef4/FRukcik9Og2NJqeyWiHjvLajGYtY3rGoqLNSMgntsdBvZFTrXvqVUezllFtevxhRTO2fM96PSeUjvmmT7sIO+an8zRG3qKh84BrokXShz7tWK2EUYaKRUzr9Rxf2cZArom0A0+5PaMdZ0llYRHObNLYBmmrmV3vLP83ZiYl7fIxCUp59ShtgJPIpneLjYvqjcwIb+Zwh7a5Ucw9YWDErl3J0cwRBhkSI25aH+/A3EbCyJRtWloOK41tJKTOIVVrn6+U4G3RIg5zI5KlR50jrg2Jq4MF4dM8QwclcVDc5AxHpD0mbBLSDrPdJj0xEtbHaFnYhjsxouPhC8voyO7Nn4kgmUdXx+jKKEUhdZFTqfO5kd6UNyqjYapX0bq26GKKjMpynmWKzGrT2oY+Ue3vYHmfi9/dT/d/t6nQaxpK0PiK/9wpuKOksC8Xd71hU//0d3weXiIfPjrHWzYe47HZGS6Wezw2O82pfMLZ4oCojndf/STevPUUj07OcLqccNAMFl9DctSU/NZTT7DfDpmFnMNmwLQtuDDaZ68ecnW2SulbDqoB47zm3OiAnWrMm9af5SNHZ7kyWeOe1V0A5iHjnvEuk7bkRjUic5HdasRhVXLP2i7TtuDxG1ucXTvi1GDC85M1Htq4zgdvnGe1rDiYD3hgc5sb8zH3r27zc9cuUWQthQ/sTEZsjadkLnJYlZwfH/Lk3iZb4ym5C9yYjokKn3b2WZ6ebLBWzFnNKt5/5RJelMPJgM21KZkP7ByMGQ8rZlXBxsqUSVUwnQwI04wLl25wajjlo9unaOqMtspY3bDr5nXO3Rv7PPr8GbY2Jlxc3efD184yPyxZ3ZrYrryihOioq4yH7rrGXcMDfuLxBwDI84AIrAwqtoZTogo7sxGFD0zrfJFTOBzWTKYlg0Fj+XjPbODXGsbjOZc3d/mlR+7m9PkDiqxlVueoClVjkmgR5WhvyNrmdBGFzbLAqGwWX/0RVZhOzYNpJzmrZ46YTUt4eki72dJ9JcfmmUPKvOXKc5v4QWAwrJkelQzHFdPrY4rNOW3tyQr7api2yVhfm7Ba1jz59GlOnzuwHNHrK/hRi88Ca+M50yonBEe1P2C4OWNtNGdnf0y7MyA7NaPZHTA8M6Wa58iVknimZmNzQhO8fc1K2XCwOwLg3ru3eeKxs+RrNc1hgQyCfV1I7XArDXK1JGy0lGsVqkK9XyKDwGBUM9segVc2zh5yOLGvwWkmOa4MxMMct9bYV520DoKQrZk83OeBZm8ARWTrzAG7N1bRIJSrFdXegHzVvqomtuaoZGWLPjFGc7XIcvo6k8HmnPDoCs2Zxr4aZ9wyGNZUT6wS1luTrDuM6Hul2Jyb/ZVFVWUUcDdywigaga68ScG7r4zIU+5l5SgvTGgfXzGn6LTlQgJItEWEbCcnFopmyuD8hNm+MQC/l1nO4NjKQhTZrBeRaj9zi7zLOIpoZptuhQ3LVdRxIBu2hOsDdGgLFH43g7sqso8OqC42Nt8qb3mTkwypzIHOjxzz861F4UcBguAPMotGpl2n8wOLPNVnW8rnM5o1Tc6MWhu9SeCBF+zI3O1YnU2EZt3yCqU1+fboqYzpPS3Z3vHGIBJtZ2lXOeLQcie7r+8IA5NQA2RHQr0Z087LLkWfbZfqWIA0LHZztugyVGdtA6x6M8mmx9HahbWzi0KXNyx/sotG+9SudqSL/MjR887y8mqrx8+E+dlAue3TLpTHUtdiT5JzawS6XUnR9Nyk2baxWeo7r0bQ9x2htJzImL6GQ53l0lWbJtfPj1Ie4SWLPrcj6+tYGLkeXRGmFyyqbTllxzmG2eRYtmr1WrS6y1v1lSwkzflhiqgleV4YsFjwiOnrXDQj7dKdIst5knOnXaTVs9hYrNthNwzs/CZFdLMjs6nbyKjctUULFbs2nxhRs93QobyRIv5Du5Ykx+/y6bryY9rl18/snFBY1DiUsti8ydVGZMs9I5qdpHr561ZcrYsUimbFNmgKJQup72In55kuIueuMvIqqb3FoTI7bRs0udpIKfz/7L3Jr2xbnh70rW430Z7+3O41WVn5ylmNH1W2QWLGhDFjBBNP8Mj/ARIzxIABExAj/ggkqhBC2ELgAW6oTpXO7uV7973bnD5ONLtZHYNvrbXPTTurrKq0sl7hkK7uvedE7Nh7x46I37e+LjGEI0rSbL0JOFxKzN6HxCzG4u1EBPTAcKVgyLgGzcWQXIOTJfalSkdlmTAl3llur8YssQ6wcwYkVVvOa95MNT25kkan5+JnWromtyElB8tUWRKfSIgBxJgCgyL6E1kWQPRAQFztY5HTRknmWPUBw5EqTLKdke1rbx36I1WOZVxI6CGifvAYjhTqe49QE7BRTRAwriTaK4txrT9IDVZ9gEvssnCA7nyR+YpIz6NdKPpGI0qyLy0pTOf1VWLrkjxadR5unkODREkmfirdFGk7MrGdwpLR8y2BkhooC5aDTzJhX8KEcnqrHBxTZZH3lfLikFJm5eiLPJmhQild2iY5rklsawJ6sdIlwVb2yZcZQknvjjkhV08JwTHJicUwkiEFKMHVZEDhfGEGs3Q8J9OWW2YynefjMnDNybRP2M7Y9wVMIqS03CehQAVnhACh1Ify1yKtFROw/XNYyxgjYk6m/Uvc/npIYev4f//By1/6dpsXX/zKjw34lgPLo791EX/3v//PcdrscVHv8NXhGL+xfI9//PbXcdIeEKPAj765wOnJDt87vsYfvnuB75zeoZIOM23xsn3A73/5fXx6fI+f3JziYrVD7zT+3vlX+J//n9+FWlmcHO1QpQ7Hk2aPrW3w5mGFF0eP+OZ+jZPFAbV2+PLqBALA89MNXn91hsuX99h2NY7mHXyQWNYDvrw+hhs1/tZH7/BnP32Bdt1DqYDd+wVm53sc7mYQtUfVWPz6xQ3+9Acf4cWnN+itxrIecX9oMatH3G9nGG5bzC72OJp3DAL6+hk+//hrfHF/iu2uBUREvGpgXuzR1hZSBvzW2Tv8ky9+DVIF/Nbzt3i7X+EwVOi6CkJGLGY9TmYdfvLNOUztsF50GKzGb5xd4c+uLwEAfVfB7QzmZwfs7yl/vfz4DqNT8EGyI7RiSM+n6zv8sy8+xvxftDD/0Q36kVKNoecHnR8JVhanB+xu5oCM+OzTd/jhj59DLS18p6FbB6kCjpcHvP/qBEfPH/HwzQqL5zuGD71u4ZYecmUhZKSVKQh8dHGPr2+OEL3A+ekW77864YC9p1QmHltgY/Dx99/hq7cniKOCelQwHzI5yCIAACAASURBVO9RGYftpkXVWgyPNRAEZqcHhCAxHAziQOb22Se3uL5bFSmyWnMSDDc1v4hPRsjrCn4eIBeU5prGwb6bEfC0Hqa18G9miCcW0Qkgha1EE4EqQFbJw3tbF0lrqCNi4yH2lBIJD4SLEXGvMb/cY+gNQhAIOwM5tzg53uPm3QqiS5KTGbfZflGh/6wHItD8sEF/wcoYdzEWQAcA0AFwEtVqgP9qDrzogTcN8LKDP2hUbw3sOiA2AfKg6LXtZUqmDYh1gLlm2m6sYpKZC8hHBrfYMwv1QOZSbTTUQWC8TODeSai9hG/p381VQLENgBcwDwr2iCm/vo6o7yW674zswbytpx691qP5xqB/RRBXvTWJ8YkISwdzbWAvCW7VRqG+leheeTRvFLpP+HO9JZsrO4nFVxL9KcNYwiygvlbon1vIvaKvVRGoMLkz+eUSSFEHgfEkFGZAWEoFhaPnt3vlCxD0xw716wp2TRluUBxQM1ubfc8AsP6XEo/fC9A7Sk3lmCSGSa5oNgRKbhEQdcTqX2qGhZxTbhwqDlWzNxL7TzyaK5UAEQqDofeC4OpRFN+qdASNOU3YzQkAs4d2PAqYvVGwC7LAaiCY8ksy5L6NZbv1LfcRUZSqIzWmhOA506XNTiJKArXsN9adKJ5cX1NZ4GZ8rqgo3/U1QdC4iph/w/Nr56Cnekf2ePF1xOGSwLA/J/DjMZLhlpaptfM34oMhHqA8ub0ScA3g03mrNgJPQ210xzTY7SeTFFoOKPuqDzzO5ZdkHPNz1PexABXXEBSoMabk1gkMVg8oXm/VM0DG7GMJnim9eUneigxu5CR3zX2aIdUziUBAR2BB9vXwjDVb2feMkCpzkkf08JyJzbP3HrsXCWxY1v2ofgryUUPye7qIxVuP7kQlCSr3nQsScZLsSlFYz/YmJLZRlN+bPetYuvPk+9vHcry5KioH+ZR+zlyj4yfvqwgEXNU2YDiSBejmPtPZtcfuuSLYrUR5bbM82uxjqZ9RA/fftkzFpQyVibXDKn0fZTCTPhPMIZKBTV7Z+ZWDr2SRAefamlzrQwDL10gPfIwe+H52KWE4BxS5WaoY0TzvOY1W9wE5sTjKpOAwIl2zMclY03UwxlK9kj2e3ZlCveGbIVfV2JWCPgTIMaA7N8XPqveeFScPFvuXNep7B99KqJ5AzLeKnZTJbxsSWMz1KqGSkGNIlTW++DGjBKqHAeMRF41DJfl7Adi5Rn0/0gfqI3yjIC19rGbrSpVIrjcBQCZ1CJAj2dSQJMJTPQg9maXyZfRwi4rpt7mfMQFA2ZNpizVZTrkfEJqKnsx6Yjazl5N1KpTIh0pD9iOiUpCHHmE1g+hGMoXOTz7MpiLgy2BNKUptQyBb6T1EPzIxNkagH1hDkgN5st8zh/0AlL5aC1ElltTaSQorJDAMEPMZYtfx/wAQyWrGvB9SIo4jGcsnoLFsJ+3rv86D+fQWc2LtXwG3/HUAlr/3eR3/rz948Uvf7uzFz37lxwZ8y4Fl/eqj+NF/+1/APlbQ9xpqYCy/XQLDqcfJH0rc/3YEzgbM/qhFd05pWg4vEYHSt6DTKvYsYPETjbM/tnj7H2oMFx6n/1TBLij7UgOHlvHEo7nSqFJ6XR6gRPL4jMcciMyWXqT9C4HmFuguYpGS2TkmucmKnZAi+bTqOw4z1YbDm2sojeou05CSlADtDf1Aw3HE/Btg/wqYfw0cnvO5d991aN5omB3v7xtg+WXA3e8INFcp3rxJkvj0hVnfA7uPI3wbsfyphNlzqMi+JcqcIhavBTbf95j/TOHwKmD1Q4n+nF92+sDV7/5sYiL0gQPI4RkT7x4+A6V9+XWoufquxojdx8DsLWsMcsCG7iIefw04+VNg95HA7G3E7iOB1U8jdh+LEpSh95TCuZlIlQ4eh3MFtwDMY5JWJanVuI5Y/4jnwM4p8xqXrA7ozjkA9RdkIObfcJvdZcTpH0U8fspqiP6Mq+P9GQNkqscpdfHoX7IzMA9+1YahJbuPeQ7GVZI/nXMglh5ornm9zN7xuMwemL2N2H4q0NwwiZC9f3wtZlcB248l9J4DQXPPL/ztJwKrLwIePhOQVmD5ZcTt71Bmafap3mE29cSZLuJwQcYqVxxID6y+dLj+XCOXwLfX9KzVtxH1NuLxE4nuMmD5U4nmIeDutwTO/0XA1d+RkCOfq7mOSWbIwTFKoL0POJxJHJ4JnPwgYPuKlQwu1Vec/5HFu3/fYP2TVLcwxMLg9GcC87eUwvVnAssvuQK9eyVw9CPPASslPdqZQH/K16i7YGCL2UXc/bZA+15g8Y3H4Zyr+e0tQzJ8RVbi5m8rSAusfxJQbzxufttg/bNpUB6XwPqLAFdTFog49TqaLRmTeRqsu0v673IHag4KOv0Th5u/TQZy+TqgP5KwS2D9U49cZD4uJYZjgXHN9377PjIsZc1ht72hv8rOJZp7SvP648yyCczfeXhDmZ6vBVZfUcYnUo9nVMDsXcTsyuFwoeEbDu03n0u8+j8G3PxOw55Kz6oK1Uesvhyx/bhKfsp8XQusfxQ4DLessBjWKZBpJrD+0pUexMO5wvy9Q3eiC/Cdv7XYfmxw/MMe15+3WH/h4GuB/oh9rnYusPyGrIjueb0e/diiO9fYfixw/v86HC4U1j8d0J8ZmK3H/rlh72Nim0QA6q3H7pnC/Crg/jOFl/9oh4fvzSEd97e5p49v+bXF7fcr6I6gZ0j9lrn30s7Se33Nzko1Tp2LzZ2DcBH75wbdOc9Ze8uBbVwIzN85dGcas/cWm18zOPrRiM2vVVh+7VDf9Lj7rQXmVw7jQiXwRwYoy27HFQfg0z/pIWKEa1gtwcqNiNmbHve/0WL5tUV/olE/emxfalS7iKM/3aB/vsDhXKO9ddi+0qgfp7qSKFDYpnEusHptoXcW3bOG19FK4PSP93Azg/7MkJm+d/CNxLBWECFi/saiuzBwtcDs2qG+G9A9a1DfWyAAD7/eYH7lsPmOgeoijn464HBRIUpg+WUHtzDQO1ZmHC6rIiWtHixljSuN7kRj/ZND6Vys3+0wXMwBKVKQjGZ/pCPTFLSAsvSUd5cVAds7C18z/VP1DuNxhSgFZl8+4ubvHqN+ZACOrwQWb0YcLiusfrKHbzRUZ7H53gLtrYN5HDlIGwU31xjWCu21pYe0VahvekQtMR5XaN53GE4bDEcai697/v76gMPHS7Rf72GPG0oxBdCfaKx/sIU7qpnCuhkAH+GOaoxrjfZdj/GohtkyOGdcabRXZHPs0qC+PgBSYvudOeZfd7CrCtVdj+G0QXPTAyFgOJ+h+XqL/tWyBB7pnoxfdT/ALRPj5SOqt4+AkhieLRGMQH03wC4rSBtgbnawZ4sEHC3sqoJ56MmkLyqYqy1iQ0AzPFug+WaLwycrzH50i+HjY1Tv9xgv51C9h77ZYXyxgt5ZyHsOMO5yDbXtMZ7PUb/dIsxriNEhtAbqsYdIgMsvagLBjnJY4QLgPIHXrIHoBkpDT5esI1nWiCmMyLzZMGin0pCbPSs/Rsuqj8c9Jauzhs+1nkPs+1TrERI7WBPoGU2AFgJENyCuFwSXuwPCYgZ5/0gWMET+nUCh2B0IsCoDdD1ijAR01rKjskvMoBCAVojbHUTTIHYdxHKBuO+A4CEWc16Puz3EasH7Pw3tSV2Y8dBNYC55OGOMTHFtG+7fMExAVAjkWpDY9RBtA/hABlEIspHOTSAxyWOjcxB1nbouP6wdyeBTCDF1YP55txi+9eE9v/d5Hf/PP3j2S9/u4sVXv/JjA77lwHLx2bN4/l/9Q6yWHZbNgO8fv8Pbbo3n7Qb//Ooj/N7Fa3x//hb/6OY38Ic/+JiR+r3BYjag0h63DwtUtU21EWTnvnN8i83YIkSB11cn+L1PvsI//+ojSgQ3NeqTDsNdi/qkwyen97jez4s8MQYBpQPOVzt01qDWDiftAT94ewHvFMKgcHy+xeanxxDnAy5PN3j77hhCB4RRwcxGeKfItv3xM/zm3/0Zfnx9hm5bo1mMGDqD6Chx1KsRq+UBd+/WgCd79u6LU3z03Wu8/uIc7dkB3X3L5FsrgCAYyvJsILP3dQt3QibGrAfYffry6BVi42FmFiFIsoqdQn3WwQ5cPg97g+qWbMbiOxvYf3qM/rmH7ChPi1XgduoAKFZv+Lct4okFdhpn37nD9ZfH9EseNORBQr88wPYaeKjIqEWQvfp4j3FXQfSUOMaTEXFUPCYVcf7qATc/O4GwCSjPA37js2/wwz95RcZtaRE7zcqE4wF+U8Ec97C7CuaGPZ/SCthjD9Qe8sEgzDzMrWaAy4kFnISwBA7tO4XDKwdUAc3rCsOZR5x5iE4x/TUC8jt7jPcNZCdRf7TD+OUCoYkQqxFhZ8jEBQGYAPWgU1AIWUrhRfKgRGBtgUcN4VKdwpLnRe9Yh9G8Mew0BaDvNfwysMO0InM3nHpUdwr1HbD9noe5l7Br9lnaVUD97AD/4wU9lCsL+b6GP6UvVm/JIMWZh7nWqO+5gGGPPVnCY+5L81ZD90B3GRCqiOZKYThNab06FiYkNJRetm8U7JILF3IQU/dmk4NNgFAzLRhikgOKIOCamMC3wPa7HtV9Dqbg4kS1ERhOOQjqA2WNekepJ2sKyFwFQ5nkuAb6S4fFFxrjiqwUg1AA970DFv9khlAB4wqFgezPA2bvCKabK0l5YACCTqxYk/YnAkNKBJYjgMT2Zcnh8quAm88FzE4URjB32y6+JOMDJHnlEashxjU+SJ2lny6dHwe4ZYTZTMwSkOR3cpI4xiQ3NDvg8fsOsy81E2YV0D1Lss2Wi1oAF8mqDX1uecHG15R8mi1ZqXHN13j5FRc+WEPB87J4HTEciwKo9YHyRwBlkazaRtz/FgOSqg0ZwZM/TecgoHQ8ZlCXqzbkMEkh9SHCt6L0XkbFxZMMJsc1ygJb9kYiAMuvp4UZIEk/O+6jeeQ226uAPgFqu5gCXUTgcdJPymqI4TilyNrEUi0ETn5AID+uUkWCR2ERc/JuZsTsnIwcBKbO27QAYLYR+jDJEnNAUZRky+gbJDAYjlOSb5Il2gUXVaIAbJJ4NrdMzc3pmbm6o97Ess3DBSsk3Gyqr8gLgSIAzR2TUgE+d72JhQ3TPRfqpAX2LwTmb2Kp9sjBNuZAr/Xjpwr1XerdXPFcZGau2gXsXijU91z4qR/ZOWkOAduXGuYQC/MpIplTb1AWZlwzpeNKF9EfS7S3gf8+kgSPSpSUX+l5nubvLIZjjaAFTArRqbYe/bFCfyIxf+tR3zvcf1ZD94mh3ofCHo4LifrRw9cSLqe6JpCqxojqMTAERwnYVmD23qI/M5A2otp6qI6P9S0XjRhkxARcZRm2058aVI8ew1qhuXcISqT9TMFBRsDsPMyjRX9RQ/XhScJ2Dkzj8UYtCgsoPEGEm0k0VwOikXAzxYTmSqB6JGhgZ2fAcKxhdqEkNgPsR603rOUYjgxUHwqDKF2Aa9mrGZWA6nx5/lCxjkN1ZCJdq2EexxRsxRRZ6QLsKs0sIRbvreod73OwTGEF4FtWeNTvD/CLiqE2WkA9jvCrCmo3Jhkq4I5byMEln2SAsGQPQ63hW00f6OghRvoGcwWI2g0INVfn5WApV81hPbXhe8x7uHULc7NjwI9jwI9I8ld5GCcAqFXp2ZzSbQNEPxZwHGsDcegZ1JP7LZNXkSm0STKbpb7ZVzmM9GP6QAB86PhxnABlxgQip9MCQFMT1OaP7kiPZQF4zgFSTTUj+ZZltj6l44ZQvJU/L3f9+ef/hbcQEe1f3mf574Dlv/3btxpYXvzmafyP/6f/BAvNi2ype2xdg6Xu8cX+FGvT4223wvP2Efdji+1Iz1StHbZjjeP6gDe7NU7aA2Z6xLv9CmftHp8ffY0/fXyOLzcneL58xHascbuf4XR+gBQRd/sZls2AbV/jbLEvnY13+xm0Cjid7bEbayyqAauqx+vtEWrlsR8rnM722I41Gu3gg8Sm4z5JGdAah03XwCiPh4c5vvvyGi5I3O5nMMpDyYjHfYOjRVf8ie+37HlczXrshwonsw7f3Bxhueiw2zdYzHv4IKFkQD8anCwO2PY1nJfQKiBEgRgF+q6CNg5tbbHvKhwtCbTbysIHiRAB6xUEgLayGKzGdt/g/HiLm/slewH3FdoFJ9qYt3vL+paqJkIYeoOPL+/w1bsTxCiwWHXoDjW08VAqIEagqSxGp9EdaqxXewzWwDmJsTM4Ot5jt2+wXHTYbGc4P95i19c47DmtKu1hjKc3dFQ4Pd1hs2vgBo3ZckDfVZAylP0zlYO1PK6mHTGOGkoFeC8Rg4Ab0oJB7Wl5cPQGAmDqbBClciZYCSEj1kcH7PZN8lwqxEFBNg4i1dv4TQVzNDBBV0aEUSF2iZlYWEQnIU2A32sGBrUObmegFxbeSUpxRaS/TUb6KQGIUQILC6Eiwl5jdnbA4XoO1PS1Ricoca0CZJP2eVRcXFgOsA8N/Zle0JtXBUgTEO+YxBtVBJYOsVeASVJOJ1FK5fL3ROuBLvnvRoG48MAosbjcYfd+wcWOXPXTc39ErxB1/gIEFwVi+pO/31IvozpIuFML0SvIXsAfu5Lsi6Xj/g8S0Lx/7nuLOjIxtw0QViA0AaL1EHcGLATnOY2GCyJqoyGsgFtxwBFWsOanU5DJ5xYM61ZiktpFE9MihwAkuxbhBWRPBjf7GgE+NswC6ivFJNx1gLmTECGl9QoGBbklwbwIKH65XB9Sqm8iQ5R8ywFXd6L430T+iE8yu6Aj1Ej1hXmUBdQXr1eWbR6Y5ltt+DfrdgT8LCI0EWqbrru06QzssyexPGd6DXN1BAEmjVu+mthbhlkJDGce7RsFt4jlWoiCktgoY5Lt4oPuSGmRElBZ2yFHpPM9SVWzbDxKFKmwGgjczS6FEj1J+4WcAHwO3HnanVjqY5KnUThMNUQRxR9och/j7Mn9s+Q0VbcQyCWP4Hyq48neVt+mFNl0PmSq6/BtPjcJEBsUr2JOV2XdChc+GLzEn+l+eo18w+371I1pdijXVU4Uzn5Ak2picg1PqQFK6b0Z5OX9z7JaNUzHm32BGYBnySgTUqek4yzfzP2IWYaZAXpOjlUjk32zHDeqD2We2YcHEIzntFnXTj7KD2qZjECTFAx5EcHOCGrtnCFCylJ2a2dTXVHuR8wsfknnVdMx+4bnQo2pfiS9J8xhktpm6XDQoiSpFrlrqvLRfSzHz1AeFJ+i8NNroMaYvJAEkQT9oUhb+Tz0m1LiGcs1WryIUpTEX2lT+nVkL6XuyQZH8fT58wJYLEC6VPF4QPUedqXJJKf7iydS4Czvza8lPwvJ+EsbWJ+SGbQcbCNZ0cJkWfolAcphERNrXdHPSCaTv5c2JKmvnFhtgL8PkWxmIPAGANl7QIoihxUhlm0WEIh0PCEUv6Uc6b2kxDX1WErJahNgChRKNSDZewkAcKFUm/C7iPJXymDF5BN9Os8ryfOTK0+Awo4WsAlMjOXPV31IQfCZa06ehAMhJ9bm/sryc8/tZHCZgeVTyevTbUg5yWX/ols5xm8/Y/m7n1fxH//+Lx9Yrl++/pUfG/AtB5aLz57F/+B//E/x5mGFYTAIo0I1GzHctfSFBQHZOoQ0qIdOM0yjCtNgKyNEpzgwHhT0nkyJPaKXi8Mph+bQhJTiGUvwR6yZUhkWHrACwkoydRHsZUSaueuAaCLMRrHIu/Gorhnb7eaBw0KfvsTi5GWi74pMUw6qCAowyUsUdOpcXPC+woPpo28N7AkZrDwQZCZH9fT0lIkw+VNyz6Wf8Tij4vCl94mFmMXSewnNwVL3rLow96qwS8A08NV3ArvfHFB9U5XnUj0HUL1n2qeb0xM1HoXyXCoVpecVcuGn/krfcN/lkEIxQqqISAOQ8EghLCh1EL7hQApMLFiWqOaB0C5iWvVMIComIBCQiq6TjFaQ/dKHLDdEGXry6yfstC95sC3dbbNpYM2DdqlMSPvmZhHzN1y9z7LocmxmGk7Nnv/OfX2hSttIA2F+3qiA9or+saevkdmn6zN1Q4YaqDYoPZYcwtPgu0c5frNLMuqAUnieAy4y65drNpDPi+PP1cCficSgyST3ZtplCglJ13Kop2NRY2Lm5um4+klaqtO/S78igFyUDvBxIZWm5+2FmvfNX95ulutDULoIc7l4CQop3ZxpHyPZMm6Tw2aWXZo9GRuzI2OUuwDzwPV0eMsBIvn1z0Nn7pIj6BFTJ2L+jpUow2IGkXLkPrmWssig6K8LhixeGcrnAip5/qSbzpV0oHwteddyR6u0sTA/SCCXyY1pwLexDNB8g/E1MQe+vtzXBAhTyEyuECkhOGq6bvJniGtFKZjP58bVojB35pB8hzYNu3X2vJG9yoyVrzObJQrwy/tfnjsN8zJVf4gQi98qs3jZU5eZZtZ5kCHKwIcMSupN1Bm4iMKuSBdLYE9mkPTAfaP/j/ugRtoUOPDG4rHk5waH+Qx2CmBNUt0MxjKoysEo5bMvA1sAZh9SqAqS5xGp35DPJdMclwGL6UJhzrwRYD1DfMJ2JWYugUXdheQ/nHon83BbbQPGlSrXcv5cjALls7feeNg5ZbZPPXd2wQOQGQw9IVn4mmVghMTMiXLMBDkJEMjMCk1AJioymGpIQCFGVmk0qjB1dslaBx5nuqZtYB1GAkD5GpK5a9ETsJCJnypLcsqpGkM5h/naY70FQ3Lyvj99/wMJTA9kEoPhHCNCRK64eerhFO7njjsS2MQn+5vlxDmBNRgJ6cO08AdQ3jn6qdYCgNoN7IuMgOx54YSWyay5K1PEXK2jpscnjyPBo2d9R6r0QIjJ08gXT6Tfh0VFQJaOmc9dQ1pfgGdmHTODWBizGKefhzABufDkGANf96gl8KQT9QOQ11s+V2YZbQJXSpJRrCtuU6sPAm8+6K98yizmfdPs4ywA+inDl/snc4jO021l32WuB3FJkirE9LskUYWURRpbmMqndSIAASQwAbswHR98ALT+AFDG0U77JyW3k/Y9+gChpjdpkd3+a/79592+7eE9f9OB5V+rHsu/zO3tZoVZbbGa9bh9WCBGgfaMwT2ZAesPFZT2iHWqiqgdRFrGd04itAr+0QArB9tKQAcIFRGjJoMUBVkaFeFlAjl1gGgdw2IkWR8RBeLCAUGwHuPMQjUO/qAhdAQ6BXvioOYOfmNgj5kgKQYJrCy8pmQzNB5Ba0BGhIVHaBSwcLAHVT7Q/SoBwyAQGnYVlnLzQWE891A7CT/3DJepA4QTGNahfLnACchBIhomYUJFhBkgekk2ZcHH2ksPsdOIFQNUVCcQcmT+gsDanjqoR6LX0AYGzWw0huOI+qsa40lifQaB8cxDP5KRsGsuAPia+zBc8HwEzclChNTzFxOQnEeoQwoaEYBbeai9YhIm0sA8i4CM0DsJt2AXpU7yT9UTuDKxUqC/CJBWkFhREaEmmxFTLyNi6mCrI/NsxgRAIwM6VJ+SGsdUbK0B88gvHIa18HrJx2d2sqyKF/YoeXelE7DrWEDY/hVZLQAlCTMvEMT0zt2/iJi9TcEliWHKTILwHCRtulbyoOna9KEtgCGlMta3Am7BbY/r9OYK9B6rnkN7d8lBVx0ojcshLmZHCaFrmZTp5iigIgNMfeB95IjSgal69vsdnscCwHOBuFvm4BVMATy55sGze5GeK2B4wVoH1XMgPTxLCaUigd0EXlVPHy6Q2CFFH7De8zz55CGm1FFAdQxAGY5FkXH6WT63BLN2AQxH9JKOS6Dacv/HNYEqh1hRAIi0aaCXAnbO56xvE9tmUNgR34qyWm92BCAldMSgBLREkcBvINjvT7mfXE3nvmVgqg+YAGjI4ELAJ6CbWahqEzEe07c8HHHxIVQ8jgKmDChFbRNQppoKUQmM8wnAS5/lkATX0qaKCw+4Nc9hZqSyrNi19JTnnkq9j+X199VUd8IgEgHfsE/P1wBDf+jJHldJGpuYx2AA76fjz6E642rqz8wsKIapBxHIixexSK8zU5fBuJ3JsjiTgWzQMklUOYRyUYVAx84FwshjrR4jA2IOMSWPpmstJX9OC1apt1FRPgsQZO4v+Zki8wKITkxTSkIl8BDT9hL4FCE+AW/039rZBNBFYDqpbQVsWgxTaeGjO1YJjMe0CMBrI+/DuJBc1EhS3f5YFXCkxg+ZtacLBlGyw1F3ObmU568/1mQYa1lCaaRlqA4A+CgKE2f2Af0xN2oOMQVoCbhGJvmnQL1hZ2Nm2cYFAV619QhmSvvNgDwza6GScDN6Mt1ClZ/bBT3F0ECUEiKm1NLkYae3n/8PmRkV4He2J8BEWojwlYTuUqXGGODmCqoPZMlSDUbUrCJxcw049lPK1OXoK1kWRoIWUL1P73f6TaMWVLoIINSSYLmWUBnIpkUyNzdk+0JEZrxcq7mYs3fl/U52TcBVBMWYV1PITTU9p2s1hAskwBwZQtU72IWBPrgSnIMQ4GYGKoOMiqBOps9lERhmI4xCMAoqA1KA4TYJaJYFoVrz3NWyLC5n4MOFkOTxcwGh1ul9QzAprC+9lbGhNBUhIEoFuICY5K9icIAkMA3zushaY+6MzKmq2XuZQZ9W9HAqhSjI4GVZK/fdFCCYE5WfhuqQ9UtAtQDQOAFJkd7gMT7prwwTWAQmUBkiz1MGdSnlVeR+y3yOQ2JJRULeOQU2XcvFa/mkExPZe/l0++XpM2BNyb4/z5z+/O2vUa3GX+UW8Bcc57f49q0GljEK7N/PsRfAycsHKB2SjFKhu2+h5xZCBoRBERyOEu1xB+cUnKXEERFQJsAcDbCbGicvH+C8wtlij59+cYnFskfXVfAiIgaBaFPNQevo9RMR9VGPtrbYPMygTYDbG8SZRzUfIUSEh4bQAXJXQX+yw3DXQq0tghfAKOnRi2Ay7NUcQhG4xCXrFsTRiNAroAmlPgACbulBFAAAIABJREFUrHm4WyAqxZ7F+wridEAIGs1Jj7GbAxUTICEjIiSa4x79XcNZd+YR50zoFAdNQNl6QAGhCpCtg1p52F0FfdHBPtT0zSmBULGYvvpkB/9nS/hLi7hnlHWcO8iNYRrjlcThOdlaqAgxKrRnBwzdAqEKgIkEklWSELYO4lBBvOjhBgVVBcRBQWw1awg+7eFjDZwP8Nc15MrCV+mDZkzSvJmHuDcpkMlDjBJuFgBF5WZmod1MwC88fBRQO6atqr2EO3HQDxq+DRAW7NKUBFlRCoSlh9wrDv5z9hxKqzhUaaB75WAeFPzKAVuFKAmSmMYJmE0C7vU0VJERDghNQPNOw7dkZ4OO8PMA7CXccUCsIkHykUd9reCXHrtWQHYS/XnS4AlAjRLDhUeV0jndLEB1CnbJGoiYjtXPAsQoYFdI8fcC/XOGPvXPHJn09xpqTyBuVxF2HQmE7SQzs8vAeoNBob/0mL1WCMnL111EDKcxDfKscohVRNxKMvPHHu03GsMxe+yiAvwsQH+jpmRPybClnG6aKyO4QgKMRwHrHwrc/z0LdUsfq5uTpc6Jpdk/GGqUagzfRCBwULMLLiZ4SwAg6gSUVnytpJMY10lSKMgSuzaB75Esz7hEYYmC4SJEMCntM9WGVY/0beoDE0qba9Bn2gL1HV+H4TgtlASy+9IK6K+4KDKuKb/NDJVdpsURmRZiFhMDTQ8XkPsVpQP604jmmr7J7FccjwV8Rd/k4RmVAM0NF3KaW4Jkd8zgILukl1SZ6TmCBrAVsIv0nDKDseRbbfiYYFI1yJbnFRuBmADnuGZwWX+WFAyrBMLTEKi7SKB7EAnUEaxVW57X4TgFaS25qDKcpEUDCWBLcHd4lsLFhsmH2J3Hcu4ymPU1Fx3aa3B2qlLvYmL5+nrybCICYQnIMbFWcQpFIhvF69cuEkBLXkizIzAORhSwLQKKJ5WsGEGerwjCDucK0lOK6Gr2S1YP9HgGR5+kr8nyNl1Ady7Tghw9lUFPx1E/pNCyHUF4lmWyh1UgzgUQuN9R0QsbJb2NYw4ysk+kwZEhRtUwAWQRgXFB0Fo9kqn2TQrZmwtUW8DOFfpjLs6IgNJbCaSFkLxfMj2+4mJf0JJAODHO4yJ1ECoGDNUbAkZfP1GpaAHTUcIpAjCuCC7NIYNaVlh0p5Kdo4kZD4aVHRDA/kLy+l0KrF47uEYWySzAbWUWcVjLsiAhIuBmEt7wPqony42YXq+7iGEtqeCYERxKI9CdalQ79ju6lo/XQ4Rwiiy5QpK5AmrgIuy4VNB94PM1Au27AcOqgj4E2LlC9egwHjE0TA6B/klDQOvrSTIblAAUATU/L2Va/NJQA8G5CBHjml7UeBBAo+hhHAKEi3ArBX3wPD9aYmwV6tsB41rDSG47OHZZVvcjok49kgm02oUGlSgyLfgxZKm6H4EY4VuDmPZV1qowrdlLGZMsNktifZPA8SMtVGRPJSCBUGuypzEStPYOQUvI0cHPc70Ut6/2OQBAIMxqQEuI0ZVqEACITUv/5G4EDEN3RGJKRTcyzEhzdYq9lhqx1mRZVSISlKJXM0YygFmiGiP9lqMFjKbvsqn4HIc+PX/Fb8gSvCOmFFityTrWT+4zDdf0ej4FoZnB1KlKRWtCI+cmBtsHhg15T+bziWQ1h/b8K9LXp0Dy/yegMgLwf4OB5bdaClt/51X83n/397F7mEHeGg7pSbIKFWHu+AESNQdGt2CoiEzskm9j8r5QmqrvNSDp85EWGE4CqkeZ5If07viaYEU/aH6QH8hG2AUH+moj0F0GmC1lnmrgwKQPAt0LD/MgJ/N8YlV8w5CRUHFf9T5F3Keo/XJcFVme8SigvpWlkyyvtLsZv+T7Ew6Iw3lAfcMeOTmmlegsLwRK+EVmkYLhwOdmQJTc5zw8+ApkuI5ikYiEOqK+kegvPRY/UxjXwHDpMP+ZJuiSYKjP8Yj6T1oWmyfWajzK/W4c0HKsvu7ABN7+SddYks6N64jZO4aY6ENiDvrkWUmeHDeLaN/xseMSxUvkkszQLqcBrn/msfyxwnCMcm4Apu32p8kzMpCVaq85fI3HEfOvRfG5uRmTaw/PCQIB7k+VmLz+jP16ALc1HvHc2WVEc5fYuRlrAkIa9LuLmK4flO6ycc1BXPVpMcTydV58LdCdp145zfsGLSb2JLFA+gDsPmIdQvY65bThKIH9S3b65Y617jKiuaJnyc4FhgRGosqsLQfV/oxgs7nioLx/BbRXAv1JYkXSOaUMkSyh2ZH9GdYC/XnE0Q840AtPwCUdC+3tKqX9zjmUybzfLbB8nV6PJbD4JsI1QHcuYA4pTMVkKQ6PUw3AcML0UxGA/UvWRzR3iZFUfBzAQVoNwHBMVnrxMybldueSAS0nkyy5euT7dP8yDcsLXkv1HT1FwfB11R3KeyIqlGqInMCbGcjsd2vuCMR1z365oJMEODIlOMuHRUTp0OvOBNqbmHxy02vFEKRYht9SC+ES25qkuos3AXYmMJwILF4HHJ5JzN8EdKcMOBkXopyb7EejLDp1BqYhvL5nv58aU0Lujsxac8dOvcxA148Bh3MOkWZLZupwITG7Cti/4HMDBEL7FwLte6B5CNg/kwwSaoEm1T10ZwLztwF2IVIKLP1w3TE/b91cQB+mZGHHRiZ4I9DeBgxH/HzsTwVm72MZ3r2Zwp5yWE79EItfstpReptrO7KEvNoFLug0Av0xv0OaBx6v7iPam4DDhcT8HZOAZ9ce+0sFPdC3t79kdQOBFmWMrp3ASzAEfbObNGAmiaWdkTXJjOAEuPjZIDwXZ4IiMGoeAsY5wQkrMFJlBnhuTBdLd6FdSNg5gVjzwNfGzvkcVQqpsXMBVzNRtzvTXFTpInTPLsbqkQzfuGTAjJ0RBC7eeoxzfqfVD7748QBgOCLTJ32EHKd6jf2lxuzKlfe56lkd0Z1qLF8P8DWvLTtXMDtf2C6m/6qkYuBj1JBer5ZhRfWDw7gmw6NsJFPaB9iZxOzKYjjSqDYOh2cG87c2XVMKqvdkMjv2HWZVgfSxgBsRWPkRkkSYqgSH7sygvmdwTHdmSkpvcz1gPK4gHOWyOgXddOcVqkefFq7IcCJJheVIya7eO8RKwqb+xlzP4WYKZucgrEd/2aK+TQE9iVWUnknD9W0PmxJpVe8hBw/VWQxnbWJrCbZV5yA7B3vSJMuNm8CdEpAuQD/0QIwIjWE67cOA8aRBdd3BrWuog4WfGUqOH3q4o4bH3FtEJckmDlNfYzSK0tdZBekCxOjgZxWfb/SsAXkicRXdiLBqIYZUC9YYbs8oMqxCQO2GxDQqiMHyeRID/bQnMvdLyscDotGIbQWRQntEl+o8AhnNXAsSE0CElASBubKkMkyU1QrC+SmEx5IFLb2RKWk1Gl2OAU9lpTpViwDlviXIRwpguyf7WFcTaEzHAqBIdJ9KYiEEovMEdElqK+qK98mBPTEgDiMlroLS1xLWk7dpHURiYf+VupF/09sT0PltD+/59z6v4v/++xe/9O2evfzmV35swLecsYQT2F4vIPcKwlK6hgioO83V79sEMNZcuZZWQngmGbKagl+gIgDxoKEPZELMnoPtuOY2s49tXBMIhUGjvkk+njT8ywdR5FZ6L1A9TJ6qKv0uvlNM+ksDmWspHdIHfjnHbgKboRIwW8rAfPIPZSlZVJKr9BHQSWKnO8r/pAPmb9KKfkMQ0tzwb4DeK8a1p3LtrIiIvH9OR4xSoL6LmRCivE4grZynQSutOLu5RP2QpE1vNLvOHrlqPR4LzP5ZC31IBdZaJJ+XoOfvGVe5hWPVRH2fPDlJRjV/5zEuJFd0b6bV++aagKB65OukDzxPwqWAhZmAUUxqZOpqGsLH5PuRQJQKzW1MwDVVRCQZXRRcQbZzUXxr1SaiPwPm73wZFrliH9FecSiNGqiuCaCqh4jqIcnKkv9H9wyBaG6nYR+B11swHJSHI8n6giEijphKq3dpMPL59eYQDkjIMfI8RrISNrI+5aB5TeQ+t/ohAZYuFrYiCiAKfnnKkaA1Sh5/9mo1N6mq4T4FPYwcqn3NxFqzj6g3EXbBffKVLMEVZps9ThNr194GCCchokCz8dADQWzfEUTN33tslUb9wEWaDLohUKo0qk1Mpe7JByeA+ZuUHLkJlAQmH9HsOkBEFoDTT8hV7ObBw7UKxqbjSUNr/cguNbMXmF/5tMIdyWzoiemrHziszd5xm+0V5cJmHzAuyDq4mtdFTl9lzQtfA3MIcA2Hg+ae/w6GoCsHhETBvkyzJ6CoH5NXyyaZcc//qyF53yqJ/jhJF4fp3EfF9061oz9OpvCT+jGi2kUmOI4S9YZBHqpLksGary8ljAQgAErKrHQRzhG81A9J+ikj6keP3lGipw8R1T4gdpPEt9o4+MrANcDs2iHLSOsNh+X6gfJ56SSaO5RewXHJcyCdQLXxcHOJcVAwXUAwiq+RoFTQzkySo/L6qB8CPyeCKL7S2fsR1VbBLlTqQgxokq+uP2YRfL0JMAcuHpl9gB7YKaqHqeez2nq4VsI1AtWWQ104MwRsPV/v5de+dPz5mkmZzQap4iGi2gVUGwcIQHcheeMkmluLw4XhED8wzVQPEc2tpYxSTDJxCEDvPfoTjerRQzpZvJGsPWI1CIRC9UCDuYiA6giugiIrqjueg+rRQfUekIYBNSFC7zyUDRDB0I/XebJdVYV6ILtW7fm3tBF67yAtj5dyboNq4+Baw2t45yEHJsqarYVsNdTBUdrZSJhHSiV1T4AYlEC1DzCHiRGRgwdgUO1YHSKcRDQScgjQB4fhtGZfoQ9TwM/IjkU10OMmBwVIAfMwYFwptDeWElIPmK1DMAZmM0CO7C5UqZtZb0eoTnIQTz5TNYQUhJMHeEAEBb23kKOBmytUDyOrVR4HVLWC2VkgRMyylNUIqM6iEgKqd5Qu24DQauhD4H6eNGTP5kwfNduxgAW1HxCdgTcSZmcRjIIcHEwgYBODR3XPFFW3rKfU00AgLEKE3qbajhCh7/ZAjNAHk35HP6W0AXKw0HvFz6KDTdsIUz+k9WmfRoRWQ3YW6sB+RtnoAiCl9RCHAWJeQe3H8jgRIsHjsoHa9gn8OSghILqRoMpHxEZD9I5MHgCR5aZCUKI6jAR3ShV/pBQCQUuI3kJYhzBrIPqB94tMaZWPPeITICccvYNitGTonIfoBdNYLXsshXWUttZV+v1IVrHrCfKEgAgEbSIlwooUthNHC0gBYQxBYgXEQw+R+ihjSI8xBrCW/07AMCe6Ruch0u8iwN/vk29BCp6XHP6TVAIlcOeDUKAnYLBPbKgPEMozvCcEgkmlSl9lTKxledwvCunJP081Jr/oFv8iNvNbdvubLIX9VjOW7a+/iB//N/8A/baGauh3nM0GdL1BDBLuoPHs5T3e/eyUXsIji9grVjLM6DlEABAF6hd79Dct6tMOUkZ09y3UnKX2uK6TTIJF90ICvlPQMwf3kEpjW18SN4WKkNcV/NqxqsIzzVC0HrFXEI7SS3fKNEuoCPWoEC8GiKsafuEBL2COB7i7BkjhNWHp6HVcOpgrA/d8gLxmYqc7dpC7qZw+zD3MnYabB6heUro4CvilhzxIFrUP3C83o7/PLSiLpM8h6el1LMl+5pFevgySQhMg5g7mdV18S/bIQx1SiMUgWGURATEK6I7gqHvhmcjYTX5LIDGm95SkulkaXseJIaF3Eghzj+a1gVvEclwlGEVygJ69k+guAvzSo3lrEEyEa1PwQx1R3SkMlw71e42oKZuMOsJsOESojuyk2Qh0H1tUVwxagkRilxmo058TqOVwmHwOxCBRPUgMl46VHB3Z3szCSk9JaE6QzN5PfUgySIHiFzXbKeETggsF3UUCMFuB/acOzTvNNMmRixPCUR5aPUgmJq5iCc6RI2thfB3RXMskrwwEEvdkhCl5BoQFmjsyg3kQrx7Z05nDpNyMwVEmeRXdLLHCMfk5qfZK/kLKTvWBLGHUXODICy1uEeHmAe0bhnnk8yAdt+sbPjZ7w3xNaapdpaE6TD4wes1SQE+g5LG5feLbTN95vp7Y+pKiuWIQTXslprClzDiKpBJILHn2K5K5RjqPEf15lglTVipdYl/T+ynvVxRkk4YjUTpqcyCUHNlzqvqJ6csMfw44yWxnDoLJ+8mkRS5eBZV8peZDht+na64oB6pJCZB9iFFlGaooIS75PrpL+xonxl1aLhK5lr/PacHZs5YZMemyXJjXVKldSV7XesPETbK2fO7mhgtKuiMbqnd4ksKIybcnU3jSmgsFw5FAtaEXVyYmNZ/rekP2WwS+PvVDGqzE5LnMHi5fc0EjP166WNjh9oZMLcNmYmEvS4BU4IJhMGT7D5e8ToDs3xRJEkp2Unf8bCkl8YYy2eZ2YlKzmqG+j6XiJTPV+fqwS1HCosqiYJt9lryPmyVVxRhRbbn9HFAFTN5JN8seYSTpZUr9tNMCSlDA/IqguD/OklS+jnlR1baigNRhRUZcjcCwFGg27IbVAxe12C3Lc5TTUnPfrBpjWeDIITvjklLOvADiGjEtPgElcKj4mVMYkXQRtpUp3TWWpFrp+LzjMrGMibnMwLnIdk3ywD7Zrh5SaElKW80JqNkrXG1Def/k7leRFljIiBPwujbVjaQAGV9PYTmulTC7UFJ8fS1h9r6oRSj/5OMzuykHMppPb1HLCZCOAeNKw2w9QiVTHY5HTlP1Df2fUZPBZuCgw3hk6GF94EKAnVMGK3yE7nxh1+3KwOwdXJNksprBQ6FWRe7qZjq93jwHdqGTvDawrmRmoHrWfuSbr1lHInyYQKDgNuAIyOXIxYkMqNR+hJ9XlKyOPGk5VEh0FrFW+CDE5wnDJ3rKQENjCNR9nIJzYoRbt9CPfaolobRVdinsx5G9jE1iNdM2IQXZz7T/sa4KyEUIZByHcQoWSz8XORiovKARpd4jP18O8CmLHROrWP4OP/e79O/4NBwImEDnMEw/AybWMgcAPUnuzcDzg9tfBliGXwBS/w1uf10Yy//t989/6du9ePnmV35swLccWDbffRlf/df/AHZT03doJaAj5FYhNAHmUcGu6bHL6a36ID4YygACjdAEqB1lsr6J0J2AXbOWoHqQTzx2QDRp0POAcKJIbYcTDvJ2xch94SmVtSmIxKbhPRekI3DQdgt2Cw4nHuZRMlCjiUWyGzW9Yn5GyWz3LJAp202smc/pmUmOKRwls2ZLc0cwBETjSUhhJfw2kCnx1s84xJidQH8aUzImv7CrjSh9eG42Ba0wll5gXAe0VzIBm/TlnQbTcR0RPtvD/OECWfabj0UlhlgfREnoLBUJefAXk4fHtRH1A0NPcsJqTkIVbhqYqgcOz6HiOSa7yqHKLgguQg6MGEBQYrktNZDV7Z4JSi9Tsmwe6u2SQCanszKYhoNrVMB4zA5G1X94rZZYfjvJU/Pg7hv+Hkl27OYoaZlPawmyp44hLIkB7egFK+xzTLLeNGAGPUkxt9/1WHyhysKAa1GY73HFfcgD6bhkmidlu+k49yjJo1GkMJzldN6CmY7RLlG6FJ8mfuZznmXYdklp53AkimRZxMTSphRL32RWPisHCDCCFkl2TK9aHnyrTZIGOjLQmbEe1xzoc2CLtBwy7XzqAswAIqdv9mfA4qspACR7SQpoAqC6iOGU14u0CTTuuU/SpuPI388CyNUb9X0sw6CbEVD4FFhSznGSuLtZ8k82fFwGPAxaydcmAYZdiAnMJaYq1x9wAOSg7Gb0RDZJtls9RtgZ2fv6PpY+QdeKMqTnJFVficJo0TMIREGAQU9lkh8fUTqagejTJGo1Ug6NyPe49BH7ZxKz96EEtyBSbsmBOEIPsUhhC7ixEd2ZRH2fqgtCYjc8SniMa7nfuuNz+1TrEDXS/okUXCUwe0/WO6ei5vCfpwmq/TEZeXYe8jps7kOSc+bPT0pT7ZyPN6mawlfA7DowSXdESrTldah7suLd8RR+w8UVXtPZv5r/VDsyqK6RKUk4A5pYAo7sTKZk11j2P3crmkMoQIwM+PS+y9+TuqcE1TeiVFaYfUhglwDP7HjOXMvPRpPY1gz+MpgyewIg21IKO6zpm2seqEzRfZySYX1MUlbKUEUConzhkMDNpKpRnYeb03dYbci6BSM/+OxUiRW1c1WqThBBps5HDMcmXY8e3alOjHQsnwm+Fqk/kmzw4cKgvXEl6TUnrTL1epIoPwWEagglpCcnsuaOR2nJcvpGFRCmOw+70ARmyN/zOZiHj1MDwRI/MwIX81ICa0mJBRKbbTEc1zAHBzEG+Bn7Gd2cFha9p9w0V3y4VkHZgKAk9G4k8F0YAkuBct/sbeQ+8jxDiCIRVkNi+FyAO2qgDg52WaG662CPGui9pSQ1REjr4RuCQKase7hFBdU7+NYQrElJqWulp15JLcnKWs/k2CSZRSTbGasUYCMlz0+SyubOSGHZURlrU4AV/Y55WMTUGen9VMGhud/QijJXJVHSZ5WcgntCoFzZsnIk70v+v0hgs4C73DVpHUoKq9GFdX0qhX0KaD8ICnJu2hcA0TkIKcl46rRPT2WrT9NhnSPgAyYwl9+DSnLbKjG0OSwpy3HzNhPIfMpePr39Qgzy5zCU33Yp7OefV/F//V/Ofunbff7q7a/82IBvObCcf/Y8nv+X/xAxCByf7rDdtThe73H9fk154ZzUhX2syBbumfIqVGQo6tYAOpI11Kwd0acd6tpBy4CHb1Zozjt4J2Efat7nwESGaCLkwiLsDYvuWwc/qCll+dYgXg44Odrj5vURg2s6hbhwELmzsBPwczKnce7Rrnq4Hy/hno0QdwZh7gtzyPvLxOJEiOMRpnYYr2YQTkBc9BBftXBH6c1fBbKaS7KQoWaia1h4wAmoPZlBYQVizZoRIA/wiYmaB8Q57w8vYDaKAS0dQz2qjUT/akTzukL/8YjqHb+UnwJyFl4/YQofJIZPB4h7hvuIkIbgJ585ek8mEwEp8EdC9hL1ncDhE4fqWmE89aivNYZLx1RdEaF6rp76Iwd9qzlALgJkJ0pojAhkG+UgSiqs8CLVWJDBG4+4ABANCqsxDaYE7HIUpW7CzSP0VrAbsCHb2lzJEroiPI+pvwyo7iXMNrEyKeBE9fScqn4C2q6lB9MukvQ0JceqntLh/pQA9/CCixlRTSvllPNNASxBExBXjwIupfnmhYT8XDnMRo5MZK3v6Ol084jFV7IwV3ZJoG2XfHx9T7AyrpNs7l6gv4iYvxYFELJSh4siMYFP35KJbd9HbD+lJ9Uuk7dWcP9WPwEOl6L4aXPHW7UBumdceGlu6EmFBNr3AocXqVYmITP6IJP0/ZDZJ4bk+JagLS8CuHl6vSPZZ/NIsNmdcyFk/pqBMNUmL6qQPePiywTmgZQ+mz5aKXdHCbRR6TkgmGB79ocBw0piOKEUXbrk8U3eabtk+m69IfDMqb3FK5sY4iy15eufFkQigYLP1SqBbFu1SczfnscxrhjwopLHNhgCzv1LYP1j4HCRA18oy+3ORZJrk/WUDmivA/oTWUCPbxMITnaB/pQgc0wAuD8jmM03u+D/dx8Byy8I8kSkBDmza9lvmwNdfEPWTvVcLEDk61jf0wurdzzmkuKaQJzueAyZKdYJ2GbQrMYEqPf5ReTjswfZzpOX00wLPTotRuS6mexrzKA8S7jzOQa4uFI9RgzHEvXDxHg2iYHMCwcASgWNCNymrwSGFUG3Scm51Y4/H1cC9UMoKa9coOPv+lPev7njdVdvKbsOBsUvWm8m9qM/5nM0dyElr0Yczghk6Z9kSI50lOlLy//Xj/TUBk3JdZW8tmqg5Lc/UphdO/hGYJzTC1vtAg5nTJwlaJ+YbTWS6az2IaXLRozzBGp6sqKUSEf0RwpmH1LIT1JHpIWILFuOQhQvMPMKyDAGxZ/P37ri3VQHh+5ZA7N1OFya0iHZ3lD+nDsjeW0ROAadPZQE4EFxnzPQ1Xt6HdUYMC4V6geH4Uina94hGAmzc+jPKpi9Z5COZNiR7gKq+xF2ZQrQzJJju+A2hGOirO4myaxwBFLmkRJcyrItQaUUSRbsS2ejNwkIa5GSjWWy6fgU3qMK0+gbSW8pv44hxwSWHaX6PndIagmz4b5nv2e1GRmWcxgRGlMqToQPGNcV04H7JIlNIFMfLPd5cAiNKZ8jcpz8faFShS0LFfdV+IBQKQLdGCEfO/oQtaSst8s+xQS0UpVKMKz/kCOvHbnvEeZNkeoCIFCVecGAPkwEAlYm1Uqyj3UFZD9lrv0YxsJG8ovOTGyfTSxsqmxBjAShT9lG5/FB56TR9F12A0oqLEAQmAdVIQgOgYlVTLfMUIrKEIzmGpJ0f2EM4jBOHtC07QwiYz6uzFqm//9CSexTOe4v+n3et78BwPL3/y0Ay5f/Dlj+1W/z7z2Pf+d/+M9wsAZKBqyqAY9jjeu7FZp2xKweUSmPq4cF7LaGmlss5z0e7ubQtS+l83VjUyJ0YBrsj5/ho+9co3ca11crCAEcnewwWA2lAmrtcfN2jZcf3eKb16dQM4dnpxvcbuewo8Z6tcfd2zVOnm9wf79gob0VOH61wf3NEubKwK480ATGfqcydVV7hLsace6ga48YBcJDBSwtopOAFdBLC7xu0Xy2wf6bJWLj+YVWeeC2RqgD0KQPYAEIFRC9BLaaYLiXmH+6wfb9AupRwx9bsgcPTF4LVYRwAuHIQT7owoaENsA8sC4FMsLcGNgzAnf1oOEXlLxWJz3sdUvAKoH21RaH6znUVjG85lnKWY+AaDwTafesMsngPqaOUbUaEd81iHWEfmQSqkgASDiBaJiQ6uYcPEMTEVWEWQ+Ir2eltB7gcQGgLPlZjzAoBjy9rWBPCU7Nlgywb1AYHz+j70hvCcT90qN+azAee8y+UVPa6UjwFBUQ5w7q3kAfBIZLB9lJmEd6KV5+AAAgAElEQVQJNw/FQ8teSFGqS6KK8GuH9osK3a+NEDuN2RuJ7lnA7K3E4VmSVFWRCwwmpvTGCDRcRIigTHY440BW3wv05wHmUcI3key9F9CJmQ8myZpTcBOCQHNLQBn1/8fem8XKlqVnQt8a9hTTiTPemzfvvZk1l+2qtFtuut1y0xJICAMS3RI8IzXwBBLDA1K/IHjjEfEAD61uWbwhBEiNZCEh8QCW6Da0i7ara/KQWZlZmXc6Uww79rAmHr611o6TLpfdUpXsbDmk1M1zTkTsISJ2rO//phhiZScgmaVVAhgvKSNmdQh7T8s7gsMgCRSZbBkgHJNmZx9RTpyqVVwVYBce9SsF18T3XRXZCk+2P8RaixSQJQ0Xyq5O9yPAMfMQS95jpcWcwNUXyEMDNXKIIS2BcJKXkjGkSoF1F6ljjgyX6mMXYdwPgOdDtyIDucTamkVAsefrqoap8iGlfBa7KQgrVeeIgAwiXYVckRI02ffhDDlsDJ7HLMcoXa2Q5dQEzARU44oDIN3F53QEaK5i+qzqp+5T3cX3fEhDDqC+AYYIYKt77n+xpye62E/Ho2LvqHR8new87vNp9OFuk5+ZkmLdIrO0KWDGq7QQpYR2OKWqgN2o3OfhnIm0lNfx2G0zvSbHrLQ0ExtM/3UcSgREWTXfO+OJiNLx9HkM6B5xuwmcJ2mstCC7KukZTgwyU2ujFDqC5qRiCJq+0CT5Tt2cKTArJcMiKxlSjQgls6qL6aqKgKk/ZcJpFQOLKCPnviQZaGJXpZ3CfkRkffWBAJv1OGSWEyuZwOe4YGplueW5ToFF0hKMuyjHT/2hw1rm4cBxoFl9R+YxhRCZWRxERGAlbUx/FUy2TZ78oiUD7AsywK5iGBCADObUGKWpC/a0Jpb/WBVRbRwTXiPLyJRmgi81TnLT9N5MQUGp75HdriHXkaS/DSuJeuPz8UpLRliNIQ8kfAR540KiueVFMwFPr8jaDicSRRdifQwy25g6Lcf5BNzV6GEWCvpA0JYCh3w1LcIJCkP02BP4uVpOCoxYX3LMdKZuyqAE1MjnTAoL1bm8yC+2I7pHNaQJKBJwdQSu7LaMva1x3xlK5NCfF6hvDVylciCQVwKuVlkaa+cKunW5FzKx1Ko1sEuG8riCQ4vk80zSY2mZOptYU90a2EURvZg+15TQ2hMimylR7EYedyEJNC292ccST+F8ltMGxeEFBEFsYjGFcfDNQ2ZTGEcGtCpYR1JIyAPXSaFQzFuIwTuhZFASYiqtSEzjsRw1+jAR/YvHwTe549JzCJRDf0JgIFAEhokZTccjUn1M6sA8Sm7lA458mpE9Dd5PtSPHHZqIjOM4lZgHayGUpAfzuBMz7f+PAZZ/KhySgKcPfwEs/5jbXwDLn8Kt+fKT8NX/+t9DoR2skxiMxjAUcLsCzXkH5yTGXQlReBS1xXgo6IOUgGws/KjIVspAGa0KvLK7dIGZJGyylwRtpYdoFSPkC4ZUAEBxr+gn9IjSWEkP4iARCp8ldAgAVIDcK1ZwxMVx+luaqvq4sHcLDziB0FDSm9i9sLBQN0X2oameQMEtPMo3iguSWOHhy5C3HzSBiW/81OnkeQ5SMmvy/CWvIADIMR5vEdnNWNGg94JdkTJKcmc+eyZDrMqYfaQxnPsYjsT9CjqgvJWThFcd+XgWHsW9nLxwCjDn9JCyU1HQOzoIymqbqQvSF7F/UiZAMDF5Ccy5mou59PzSxECRWFGhenpDi3v6D7NvycTXL75mQRIgJV+nJ0maWU6EKfjHzunZK7cip/QCEfB6MoxBIicPkx2J3lw7VRekBN9ij5xMmcBwksDSSzr5Tm3s2xzPWQPiqriAifIwLp4iuAABQn6+6Adkfx/BVPJaZhnZMDEwx116GUAfEBfZyCAodX4C8fxGyfEUNDN9HkSIct4WuQcxPY4Jl+m9Hf2C0WsmXVz0l8jsclq8yXECXMlXebwwFfEY1Ri9i4G/S+xJAiNp8AJwu9lzmPYvgg8T5c25lD4NbOK+QfL4kveNBfN8/iIm/aZjjG8t3i9tX0zy6XS/9Pr6CFqlYS2FHJCvBz56OfPrnIYxElkKnv2Ufcj1DsLRr5hk1dKG6f2kRPR08j4JOKaBAc/NFEqWfGkAcvCJ8MgMjTSRuUqv1dE5T3LXXPIeP6tJCp28uCIwNKmIgVNq5P0RwVB+nY9Y58lfiuzxYxgUsucx7QfwEJTk60SYXu/0e3H0+cj7fHQ86Za+B1JgT/JdBiVy0vG07emzx88h91W6CeiQ7Yzy6CgZViZkP58e6JP0sQ4jf/6i5PVYhhqUyH7AxFAffyZ8vJ7nDtp4no8l4Qx5ikObyBp6LeAK/py6MYF47rN/dnpeFTtQgbgdFf2U6T1gQgQlYL9jlL8mgOLVxATbmsFWtpbQAwOudBdD4dx0bMcya3G0fkrnOAFY7hNZSzuL2w7IYW62kdDdBFJT2m1iFVOabGLnde+Y7mp8vB6FnHyrkpczASDjow8TQJjkxezTJeslrI8DtCiVNj6/V46PNRyxSEFxYa+MZ9hNlPkmCXD+HAZ+r6XfJ3ZT+EAJbepeBMgMus+sQ6OXMwfJaBmBFHI6q3A+eh4BaaIUOP5MAOf4+ZYyX1fS9T8dR5LrJtmx8AGit9lXmRLwuT8+SlkBHHstj/ZdeA9fFQSZib10rA4RMUDoAauYuyiPLiQJ9KX/x/Q6iMHkxwetpmAoAA/8lMcAMAHQtN308/FjjgDrA8D4427HwPD4Ps4h+SsBPGQmQ/gjYT4/dht/moCe4B/UmPyz3v48AMv3frEIv/EzAJbPn778Mz82YII7n8tbegte3yzhvIRzEmVpcfZkg25TYzXvUSxG6NIhvD9HsxxQn/UolgOUpsSzOe2gGgt4YHHRoliMEKNEddoDOqBY99CrEeKqJxNoBcTpCD9zqB+1EDML0TjMf/4OYeagrzqgoGdzdtlCnA3snxRxRRGBqJ/HD9/CkdW56uFXXP26E4tQeYh3DuyZrHyWxIY5fZiwEm5tEc4MfO1hziwlnaOAeT7APDLsP3zeIVQEhPBAmFuylIHSVDdnwA1CXJg2js+3dlDt1FXmG092zAqCzOj7NM8HfqHPLf1K1wpuEeU/dUCxUQSTdcigEh4Qo8BwSf+EXTlABJhzC7N20DuJ8cLBLik5NWcW5Wsa+e08YDxz8GWAq7mIo+/OM4Qo1sq4MsBGWWoCJin8xtXI4UDwie0KEfCQNatuVJbCuiZkvyOiTHM8dZCOINHOfQaK6sBkWgQCXVcGhvFEr6dZEGAm6WpiMA9PHPpLD1cxCEcfBM6+4+hDXHmCcRlglgFeB/TnIXs67Zw9i64OGNcB0gkEFVDdEGSqCFCXf0CGdVwzMdXVDDRyMx6f1yH6JqNPriLQFpaMZrmlD7K/4peQLzggcM3E1tpZrNyI1/0UXjKsQwa+XlP6C8lFuzkJ+dzqmLZqllxoJk+pbievpS/5GFdPIDPV5tgZH5M8uCmwKIXZuCYyaIrvh7wADsnLOAFNYFowky2dmD2GbfA/JkyH7LlkfUfcrwjsEmhNj0+MXwLFZaywSKDD1siLWTOPLHdSMs1i9UVBabKdxfMSzz+rgxAZwpDPqRrJaPqSQMnO4mvkI+gskf3T9FNH5nGgrDWliqbAlLxQLRD9SohF8MfDGN5X9Xyf+NRvGEG/6iZPHwI/j+MJF+cZ6NqjYUSsK/Gxn87H4BgZk4yTbLTcJn8kGRo5IoaXIDNeZiFyCI6rpmO3tZgkrYrdk8IBw0pGxj1eB8qp/kMEZM9pqpVJqdz6QDCTQnzUEDKoTOc0+bWVmVi5BMgSuGI9DBmH9F5L76P8Xose6JQGbJsEulIXL+WkQdEHOcZgn2rnYzcoWcMExumlJ3NJRikdOzIYS/5JDjfitVKKB9UlaciS2Mf0vIlp9Jpy2AQabS2nL3gRAXZMBHcl3x9eC9iKFShkH+NzhxhupDlMYNAP+x8TS5kGBGRlCTxY4UM/p48/J1CvRnpKVR8TlSNwVB3vm7oj6aEkCEgBO2Yu4+crgs4IAlXqmayYJKtbymp9JXNQXAr8UQO9jpS2Sx5PTSYtBQgl4Ezvucx+WfY3IgNqs2QNiS8lZajbmIw6ev4+Dh107GlMwDG9Hq5W+Vy7msccFI83RFDrlXxQRZL2wTYKZqF5/1rFwQQBnS9V9qIClLSGUsI1ehqEVfRP+lrlsBppPbssFeWzkASBflbw94LH4GYaiMzwxGAqwHoG6oCgNSSfqJZkIAuyhkGwMiQUKoYY0d8JTbICWhL0evo6E0MJ5yl31WrySTrPnwH8Ea8kgNw5KeUEKo1FqEsErchO+nAUUCQmcJpYTgChiF2V8TEhyWElf/fgMem/tG0AQgiCQR8gtKYnU0hk3+XRfY9De6bOzYnlFLH/Mv39x4LKFKp0/N9f3D53t881Y1k9fRae/4f/aQ7EcFUMCVg6lG8m2V11J+FKSt3Sl3BikRC4CNctv0j6sxCZSKC6lpQAAtC7tHCiRGpc8TG+oKewupY5gMbWBG3qEMMW8n4waTQxaQAyAwREn1og01ZuuZhLMqtiBxyeBMw/EZM0rqI0sNgxSbRoBYZ1wPr3gPYJJWtyBNpn9O0VW6B9Tp9frn6IK7GUvpoCXcyCgEuAzOnxZD6l8BV7LmyrG0rgdDstcil74gLv9V+3qH9U8IvS0MNXv4kpjDNMcj07sXFmzgX/cBq9eXqSCuqW2yv2wO5LFrOPecFKrEOupgAyAynHxGQc1T7EknqziL7RWcjVL/pAWV2xiyEjp7Fu44aewPpGYFzTh5cWjPUNpXzdFatuzJLHJk08jgMvkuU9A3fSQlZ30f9UCeyfBpx9B+jPKB2sbwPaJwLV3cMAkiAmAJeSHl1M9CwOUdZ3yU4+hFgefx2DahQwrPmZoQcNGM4D5h8D/SVZpHLD+9o5w3VSuujuOUvoXZKndUyRPDySGJeUTY5L9kWaFcvbxxVfv903BlQfVSg3kcGQBDrdlcDsJdMtUyG8Tx1wamL1klxUmjAt2qMHUneUP26/RAlwqgtK3YUp3EcfKB0sd5Qt+pLHahsxdW1GH+jJBw67ZwqZUY8hUPV1yAXwib1hKMaULqp7Lm7tgu/F+QuHcTkx9KnUvbsSWH5IL9rhEbsk1Riwf6IIxmIyZdGG6Nvi66wGwM55TIlddiWinzOgufHoT/ihdVXsOIweRrMQqK95/EAKNOF7JPkhdR9Q3U+BMqYRLFq/8fT1lWQUVGQ1g5jSVXHEFgfNawAEMKTk1bgf0iL3RdpaYFjH9+hMYPmpRXulcoetV5Q9+iPfc9nyvAXBYJzDhUTRMshHd3z+csfaF2DaTlr4ixBQ3XtsvqhzDc+4EmiuKdvrTySaOwIJM5tSSnXvY9gPwZMy3Ocg+DcRgHEho0/VszakFBGkEjxJF9C8sbFYHjl9lGE/0V8bWWJpA7pzifqe/j1bi7wgH5bcL+GB+sZiXHHR358KLD9hII4yrCdZvLCwNaubhD/ybEbgoMYA1cUU0rmCHENOvR2XU6poc21xuKQ8o7pnv6UaeA7K1kP17Glk2jFrT3h9JuhMzKBwgO6YGLp9p0J9zyTT4USivncxUdRP7LgHfXtJGikBM5coWh/BrsyAHUDudswS2J6smZ1JVPeWFVczFcOqLFwVmaDIgM4+3qN/NJsYOwFUtwO2X5hR+m0Dk1IFKFXtfZZzlpsRrtaUlzqmupZbVur05wWqewvVWZhVCQT2W1bXA+xc55AlOTqYVQlXEXSqgRVIXgsULYfhSdrpZpoD28TqlTKGStFLqPcG42kFvUtg0TMMJ3oOXa2hDrHewoWciupmmp5DKXJiaxDsiVQD96/Y8lgJjil9DQVTY4WJ99lH4CYEZG8hBwNzOYfesK4kSUjlwBCbUCiodiTQ03xdxEAWMEtQfRwizEv6LAsFGVNaISXEaFlPctLws9oOCErlZFi5OSDMqomJNBZhXkf5LIGnaPscguPnNaAEH1fFFNm2J+DTiuCt1JD3ewI4yTAev5qx71IKoNAQ/Qg/byB37QTkEkuo+LMwlnLW2DcZygKijxc+JYF+AOpqYhuPPZbp/4EHzGcYDbsmlSLjF4HcjwvwAfAguEeUBUIbu+i0nu7rfPZfApj6K71H7tKMvsosi/0sK/kTkmAf3u3ocZ/zVNj33ivC//ozYCy/8OxPZiyFEGsAfw/AN8Cr5b8bQviHP839+FwDy9OvX4Z/6e//2zivWizUABMUfnRYwweJUllIEeCDwFyPuBtmuO1nOKsP2JkKjTbwQWBwGo022A41fBBY111+3Hao4YKAALAoB7ggYZyC9RJaehxMgYtZi84W+fc+CBiroJVHpS1cEHBeolQO7VBm2W5VcP+cl+jGAlo5jFZjXo3Y9xWMUSgKh0erHbZ9jd5orJoe+77CMGo8Xu/wertAXRo4L9GUvHCn5wOAcWSY0Greozcap7MO910NDuwEhAgwRkOIAKU8pAioCov7XYOydBAiQIiAddPjvqsxKw22hxrWSpSlQ9eWkNEPOZsNGEYNazR0YSEEsGgG3N7PETYllm9vj5VQaA8VlPKx9zfAOQGtPVazHpu2iWoOgbK08F7AWgUhAorCoTtUaGYDhp7jedsXEMpDlw7OKlyc7rDrKgxDgeAEdOlgBo2yNrBGwzsBpacWoaoy6LsS3kjUixF1aTAYjXHkgsi1BdScZirvBERknZWOq+e4D8ELBh1pMruijPKkAFTzEWbQHOr1CvAConRQhYfdM4AhVdWEvUZx1kOpgP6+xupyj+31PE4Z4klUAXAConbAfYGwZLWNrC28UTg+2bJ09PHuCnpZJQcq0J7fI15AHBQThat4TKMEtJ+kQKMk8z5GqZEVCDMHGJmTkVE5iINGSNU78SacABzZ9rydJAN0gj7heGyy5eTHz/k7yoBD9tQCYF3QMsq0nWAg1kEh1fqEMqoDnIAw1HSFMuTAqqBDlniHuYW612QvqsD91CF7Ucky8jwIK5ge3UrK1I+YbNUJuCWDooQXcA0Xu6qVDIuKcmtIytPJ9HGIpVtJj2scKgUNuJL7kBKnj2W6ZLJCltAHyfOpDlFmbQg4k3T0WEIKIMufg6LvNIBS7HFNZUSxo1e22MdgJ1YdcrgQA6qOJZt5WGcmFlal9PzIniMcPU5yUMALEVi9E+ttUpdwGvRIw9dURdl+qt9JlSk+VXlYstE6hlYlyalwPNdpe6lCJygOEVyF6FvD5P2rkGs2gkb2UQLIydMpBRmI7KyYJNJJAp4GCFNw0CRfNUsGPqVqFT4RQbMvpyFaen7h47mKQ70kLU7ydYKuifnMnlI/DWUgkJO2c2JzPAY58lh99DQnm0C6jiRZeLrv8fOn85reA6mqIyVUc+gbB2lRHaCiXxLgIMQ1IktF7XwaBB57YpMcPEvwRw4aErOfmK80jEiy2OPBUPJEJ8adnaZR2lpMScdJ6YCjz4xMNUbxdfY6+j5jVUti1vnZmyTRrkxMKs+NrQmSyzbAxQT0FEZ0bA/Ism4xscdUlpClT0nBSS6bZK/Hkn9bCaj4mhxLiNPjklw4nZMk03WlyInHSQKdzr/XIkuFMzuuwA7gcep1TecuAfs04OL7IDKxvX+Q7puOMUlX8/cPkNnIdGwphTdd1yih5z4fD8KPb/mYoqcTcVsqVaFEeS+fjx5JABCGgP3Yh8mBomACbUzf5b7zMTIC1uRrzAm11mdpbWI+j2tChHHwpY7vA5+l0cK4LK3NTOGRtzGDwgT6EhuZQKNS8fmPZKSxQ/PB4xImSGyjkg/B4LEMNTGP6e/e8f+DxwNZbAKbx3jjWH4b75PB50+6/XPgsfwzBpb/PYDfDCH8PSFECWAWQrj/ae7H5xpYrr72KPzLf//fQmsq/N4PH1MuavmFptYj3K5AuR4wHgqoysEdNPRNES8mAW7GRFB1kHCPB8hXVfQJxoXNyqO4jReWAAwXjt619QjxqoKvAvReQhguBsiGCQxXFsW9omRUhux3DI8GPk5PISOUpUU2NS5O9IELSlfyC0wNDD/RG3oMh8cGsw8JRtJCYjyhFLLYCphVTCW9YLVIktD5IqC64fEkOVqW6FWAWbKexEcvnSujjLKYQjrMysNXPG8IwPxjhe6RR3XLNMP+3RGz3y/RPfZoXkocnlmoE4PiB02WJCYpanUr0L3lUN0q6D0wnlJeO1w4zD9WsHNEEMNF43jhMPtQY7jwaF6xoHxckwlMjJ5ZBZR3aXFO5lgOky/Ql3GhPsZ+wVv6lfL5tkD9RqB7zPdAsSdLpQbuR38R0LwSD7yXwrGug4EUXOwULdmf4TTeP75OSarpC/ZgJrlmuUFmvDdfoX+0vEvA4TMLSnDb3WXA4mOB1EdY7AGIh12JZAW4XVeSUUyvf3cVMP8RH394MjGq9ZuAwxNg9ikys9M+Ze3MsA44+25A+5bMtRLjCdl4FRfqumfwixrJhJZ3IvsAgyTjR0kyq21O/oDpr8D0nlRDyJUgKa1V9chVLAz7mKok9IGvQXMdZb2xLy4l70pLlnj2ggvNLqbN6gMXGIlxT+xT6q5Mr43uYsroIQbg1LF/M3Bx2T4RaF5zn3WPHDrjiwhCNHJQS+qwIwscciVGnroEMtjHC2CzZDWI6pG7DxPrqwaCuf4yVp5E1hKYgEQKmxGxCoPycYL5Yh+4+IydgnYGLH/kMc4ZbtOfxQV3fB59YK9k6nvMQANcpJXbkBNa2SdIZYJK+9axB7DaMpU0yZhT/Ui5I6uauiATm1ltAop9QH8mUezDAy+r18gstO55HLNrH/sSQ5ZgNndkObszmY8pMXH0vfF9y9CUSY7qtaBsNP5tXIrM4if/JZBCcoBqy4oC0zChlceHvOCtdh79mucggSQzlzANg3qGE8lr/zhVoQRF76ONvZlBANUuxNcpoGw9hpWKFSNkMss9zzW9hUlSyWCY1BGaQ2cUpanjgkmxrpjCUsqdg6simyrJWqYanezhLBi0Y2uB5ScW/anK55feb4Fq48AKIwU1+FxXUt07jEsVJcW0UiRw74uHHtrjoJ08oIqACCFEeaeFbdQkzY3BLww1kvlaozuXk05TsE7ydvqC0tVy6/he8/yM6IND96hE82bE/u0Ks5cm+qUlyo2BWegMMvIxNhL6QNmsGjzMkq+TcJTZElArqM5DOg8bOyaDFKjuBvSXDM9Rg4NIvY5LpoYW25Fy0UCW08cuR9ZXeIgQMK6rXEkiRw7KVMt6ELMqUN5xoe5mmn8H2Ie5HeHmRfycBcAF6P0Is67JaNbR8+kD1KbHeDWHGhzkwcCtyuyTTGykHCyClDBnNcqbDsPVDMX9AN9o6M0As655Tg4jXFPweMdUf0LQ40sNvelg1w0B20C5qjBkM92yguoMX/PBTZJU6xGaArIdACHgmyKDO19qhgENJgMq0RtkD2JkUUVPVhfWsV/ysyCtLB6mrsbQGjEahLqcGEZjed/RRHbVPPRMfja8JwTKX5NPM9WPHAPLugK6fvr/FM6Tqk+6no9NDKVzgNYIff8QVCaQqPVDKWraVpS08g0aAaQZ+a8UCH2cWiUw6/2DVNnPgsc/FZjM5zhwG8PwJ9/3j7n9eQCW33yvCP/gZwAsv/QnAEshxAmAfwLgi+FnCP70n3yXP783LT2ez+9wO86weavGSdXj9Z4snhQB4gz4yvoNrqod/qfv/CU8e3aD6/Uc3gtIGWCNgpAB3kn82le+j398/gzGKtSlwZvbFdbLA9qLCnZUKGuLVWQZn63v8f7sHM5JhCtAyoCTZsDtZg5oh6+e3+GH12fQhUOhHHZtDddrPD7f4o1YUalRWHgnYQeNrz5/iT94cQVdWHijIUoLrTy+dn6D3399SfZS8AvTOoHZqsewKCGlh9mXkI2FUh6l8hiaGvVpj/FCYzHvEQDsXi8AHSBLh15V8EsyPKGIY/0iMmyVw6hKYD2iWAwwny4wNg6wkszYVkOcjlgue+yu55ifdQhPAbep0VVkqqrFgO4pk137nx+wWvJCtz+tuAipPMTcAl7gcCqwvGixFyuMXx7hO345Li5btCc1k3ADIPrIwKmAw5dGlMsRbdEAJwZhUHCPPPrHXMSImcX4SELfFtm7KUaJMHMQmgm5xWzEcCihaot2UTHgaObINA4SrpFwjYdrJIveTy3URsPXHjgx6EOF8cyheq0xPBuhbgq4MwM4AbVTCI8GWOVhPpxBvHPAftZEpseTWXQC8ALDY0D0ks/Z8aPYfKIRVgbWCbiKabnFRjGQ52xg4JQXSAFT+3ck/LmBuCvQASg3EuPKc19lZGw8oA4S4p0W3V2N8pbP52cerWCvpXk8whgJMUiMKwk/8zArgWJDZGJOHOCZwHvzDQF7OaJ4VTDV9UmL9tM5ynuJ8ZR+U7d0UDvW05gV32P1JxyGbL8Uga8KKL+wQ9uu0F/6zND52qN+pdFfWZS3CubEo9hI9FdTyEuxYChVCnrqLwDfeJgTsolMNyZYlwaQg0D/bIQcSlZfvG1Q3OnoYQuUqt9IiMBUUncxAoOCaiXGE4H6VqC7pLfNzulRlecidmty8IOg0V95At+NxHDlUGwkzNojiID6tYaLwVXCA+bUYv3tArt3PYTnIMHOCaaqOxE7OYH2qYduAXPqQQ+iYtJpHGal+hNXk2QeT5H9kkGxazYDpAqZkfSai/HuMZlSJgLTQyudRPskYP6pwLAGbC9iumhAuAJc5eEridS9CkRfbExvHM44PJFGQMRhXfNKROZIYVxRxtc+iYv4KqB5TTn1cJZqe2SWfI9nHnYuUV9zH10tclgSBBlA/xJonwKzTyXGNSAtE0d1x4oW1QHdlYrdr2Q+7QyYvQTaJ0B1L/j7LcFTfRMrWXoQ3M+5reqOMndgYoC8nu4vT+glkwY4PJ7YE+kmAG5uFfpzwL+S6M8mCTAZckAOldoAACAASURBVMlKkB0yQ5N8wbsZh115EBEHFa4C+nMdmVw+ZjgTMFtBL2kcUuqDnLo3GybPmuXU1ZnYpnGlKAnXPG/JZ2ljDyeg4QrANdxWlSTvcciljIatKa+ub6fQFFdQ6jmcyMjsT15W+rMFmtfxvR9Dq1IybdHGALT9JOmXI6L03GcfKQc1Oqbi8vFT0i+HCcOKclo1qsyIukqgu+B3TnPj0Z0z1MY0iUZE7CBV6M4khC/hSoH2SZGBqKs5HHBlrMo5TDUow1pz0BH7LSGA4uAxaJ37RosDBx9mJlDtyG5KF3C4IOhubmK/asXtCA+0j2fQfWACblnwuGaK5yf6ecelgjKRSYvsYVkpVqusFYQrMK4UEBlK3Xr0ZwpVrZA6fFMNSihqdBcFpGMCq624zbqUGNcarixQ3WmMJ5rprwcPXwnogwJQxddEAqFBf0YgaxasO2HiLK8h41pDDoF9m8ZjXJeUA5cEMOO6gOo9MCvi54uBQkGAHkzjEVZVZl31foRZlihCgK90rkoBkP2eqo9yXyWg9wq+omzWlwT6stSsKmlHso9awqXtj3wuoSV8pSPjqSAPI/yyJjj2gQA4MpIoVGQji6gMMshdlABBeVNmSW4QAqGKabKOSa2hLAhMm4qSVOcQZjXlsN5Typs6P6uS+2gsICjDFqnUN1WDpFsRfx/BdRZWlGUErQ6QRQSO0ScaAkRTk9VMwUtSZn9lgqniM5hGpO38hBt9m+In3ufzcxNw+Jkcy4UQ4h8f/fx3Qwh/9+jnLwB4A+DXhRC/COC3AfzHIYT2p7kTn2vG8ovfnIdv/Hd/G5/uV/jm+QuYIPGvn30bhXD49U9/Fb98+hH+39t30NkCl80eo9P4aLPG8xPSNjZILIoB27GG8xK///EjfOXZK/zKxQf44HCO3/wnX8ff/CvfQiUt/q+XX8JhKNEeKkjp8Veef4RvffoUX768xvs35/jX3v0ufn93hVeHBUIQqLVFO5ZYVAP2Q4Vnqzv84e0FfuXJD/F7myv8tYsP8A/e/ya08phVI1blgNEraOlx085w93qJd55fo9YGP7w+wxcvb/B7L64gpYdSlKgumgG1tvj4k3P8whc/wXc+eIKLyx3+lbe/j//z1Zdxt59hHApcnO5QKofbdoa/8ewP8b9965uYXx5QKId5NWLXV2hKg03b4OnZPe67Bjc/PMUvvfc+NmODN/s5ZqXBuye3+NbHT1FVFt+8eoFvffIUF6sWL65P8LW3X+F7P3wL67MWPgg8Xu7ww+sziO8v8Nf+1W/jOzePcVp3+L0fPMG/8N4f4lsfPocbFDBIzK5aHO4aLC9afOPyJX7rg3fx7OoO277CX7r6BD+4v8KmqzEMGovZgH4s8N5bn+K3/umX8Fe/8Yf43ptHBNCvFlhctdDSY7dv0MwGvH2ywYvtCidND+MlLmctvvPxW/CDgmgVHn3lGoPRuP9wjbe+8gYvXq+xXrfYbGdYLHpsXy2AIFCe9nBWIbyp8Mt/+ffx2x88x2rVYbtrMF/0OBwquF5Dlg7np3vcf/sCl7/0CtuuxqGtgABUDVfg4wdLrL52i/v7OYKRKN4UTL2tLX75Cx/hd/+Pr2F4YmI/J/DkS2/w8tuPoN/ZwzsJ8f4Ml7/8Cq9uV3wf7/kFUJ4MuDjZ424/Q/inK+AXdhheziCcQPGkRfV/L7H9eQNReqzP9jj0Jcau4HCljV9sKqA67TG0JerFiPXigDd3S+BFDWEF7BUBFzxQnvcYb2umDM88Lp/d4c3LExTzEe7FjAx+J1B+dRvl0xZ9V8Je11g+22L7ZgF9q1F9ZYvDiwXKywOG2wbCCpy+c4f2WxcY3h4phxIBqvTwVkDclvCNj7JSdnjCA6e/q1D+m2/w6qMzQAaouaV0+aYELgaoj2vYlUcoPMrXGq4JDJMSBIfV0z26XQXR0j+2eF+hfdsjXIwoawP//gL+eQd80jDx2QmEsxHqRQV9IJg1J57y3JXB4nsVul/sEF5XqG4lhlOPUARUbxTGU4/T7wjc/OqIs39YstPxr95i/71TSAOMj+yD+h1hBJpPmfZsFlQnSEM/uVoauJ5DGVgOLWYfaYwnAboX3O6CAw19q2EvDPR1AV8GLD6UsHPg8AUDPTfARw1srKU5+a5C+zf2KH5nwdqapy3Ed5dRMosYihVQv1YYTj3W3xfYfDVEL6zIKcgM3QH6x5bSYA0OIc4cpbcR+NoF/d/jqUf1RuU+XABwc4/mhUL3xKH5RGE8jYqHxx5nv0uQsfkycPE7Aa9+JfblPraoXhRR2kwliI8Jx+UO2H3RoflUof+FDvJTakqrW0ppXUWwe/J9AjOzYuhRktH6MqC6ldknzZAvMsf1bcD+ObdT7Ag4gxQ4PA5wDft/2ZkrMJ4QyA5n7Jy1M2Bc+1hPFNC8JpAbzqhOQGD9zPJD9m9uf3HA+rcr9Gdk7aleIUisrwPap3yMawLOvk3/6f4dj+pOorojyz//lEyxWdCf7aMvdlgT3B7e5vkrN+wftTXrXxDI2uuOvau6i0xvT0WAcNyPcYkpsTlW3egDYBfA4kce1Z3D7c8XWZlhUp9qlCaXG9ZuDKccPKghdpcOQPsW1Shy5ACivo01K4qeZ4Jk7tvstcfhSuZu0jL6cVMKqqvpCYcA9s8Ezr/j0D5WqO7pB+3PJBafOmyfRzZRke3XsbqHYUTcDsLE4IqoLEpSZrNkOvHstcXmCwWqjce4EJi/9tg9VTFwi4y5GgPax9xeOn++YEdptfE4XCqsPhrRvlVAWmD+YsDhqoxSVmQf+Pylwf7tkgx2Q49rd6ZYQ9R6mLnE4kcjxhMN1ZPFK3dkX9srjflrC1dKHK7oYy7agHJj0b5VYPbGwswVxqWE7gPqG4PusoDu2CnaXmlU29SBytTd+s6iP9MoWnqgq42DWSjUNwbDmtff+UcHdG812SO9+HTEsC4w/7jF9ssLLD7sMJ6W0AeH/qKIn18y6QwukhmgJR98sbUwS/pBq5sBrtFQnYXsLEIhYdYVJbegZ1cNPntHh7OKQLm1cLWC7sgAq95G/7sEFKtSIAS9wJ1F/3iG5kULXxKoqgNBaUqpBSi1lYOBPWkgnI8hbjIy8bH2ZtMhNBEMpuTgQkHuevhFBXkYEUqNICVk22fPpzgMURrLVNkHPs6mIqsZWdgssU09mcYiVEWuSOGCI7K0VUmvZ6oRkZLsp1QIfU8/pz8K6Tn2cQITeD5md4/TcX/cLYJP3/c/+X4/4fbng7Esw//yM2Asv/rsJ9eNCCH+MoB/BOBXQwi/JYT4bwBsQwj/+U9zPz7XwLJ+8iw8+i/+I6hWonktUewY6hFklP3dUfbVPqVMT3VRshSnk0ku6MooPYsei9kbh36tKMe6o5Sl3HOCmTq90gQ0qKnvjNH5QH/BL+AkL1KGU0uIKEuMXw7lZpJaBQXU95RHCcf9TL6FFM6QUgi9AmY3LOkdlzJLg/pzieaa4RrVhnLFYheOotkRu7F8fNwkr/QaebLXn0lUGz5ntXXozhTqO4/9ExW/4PnFGCTQXHtc/5LE5f/HUAnbUMrWr8WR/IlMRpJxFZ3H7m2N5ScWXgv0pxL1LXvaZtcOu7c1Fi9cPtem4TkYlhJ1LBEvWn5BFZ2HrRgaMS54TOWeHWZe83VL561oKQNLEfPbdxRWH3mUG4vhVMdJMTB/7bF/iwsKryl7bW6Ymtg+Ulh+amErTpdTBP64oEys2kTmQQKzaxc9RSzsru84DTxc8QuVUloGcDS30ZPXe9z8fIHmOqDc+Rj0wfdBUOxnMzOJcsdEwVTNYBuJcmNj2p6ErQTqO4f7LxZQQ8DsxmH3RGP+mmETKUa/2DGZtz/ll7nwQHVr0F0VLBgfA8xC4nDFL8L6nq+lbvmFa+aShfcbj9mLAdsvNlj8aMTmC2VO8mxuHCVLnu97ACjvRoynJdorhcWnFmYhUd1Z2Dlft+Z6RHdZctpdMHGw3DoECXTnGvWdiwmJInt2ds80Vj80cLWEbh0Qkxm7M4XFC4PuXKPaOqje43BVwFUCJx/06M9KLoJi3H5QlIsdHhXwhcD8paVM9ESxH29N1sVWAs0NQ0BcLdGfKtT308RXDiwIDyr2390xfXlYq8gweOjOYf+khDIJ8DguvG5NrlFgHYvIUsHqzsCVrBmAB8rNCDtj56yrYhqkYsBOCv0wS80C88Gj2BiYVZELzIMUKO9HQJAxaF60GM8ajGuN5tWA4bSENJER0nzO6tUe3dMlghQoDhb9WZmvQ7pL7zGHcV0wFCUA5ZbnMfnlir3BcFpRomgCit2I/rJGuTHorio0L3sWtyuBYmcwnpQotyamYyIGlxiYZQEzl5i9HDCuC1Rv+hwY4upY/G54Dso7Bp24SkL1PP7ytkcoFOxcZ7mgsAFqPyJUCnZeZHkgJKKH66h2pJRQvcuhL0HwNfFawlcKwykZnWJnyTQggXL26CVvoFkWqG56wAXYkyrXQfhSZlZHHxzGkwKuFtCtR31NGZtrGI5mZhq6c3khXN4NsMsS6mBzr5/eU2ZolwVU5xC0gDCev58VMKsySy5DIVHccRFnF2X8N+4nuPill9FBdAbjoznZod2IcV1BOqaNqk0Pe95AdhZydBjPGuj9CLvkcxb3fUzblNB3B/hZCUSPnWsKFLcH2HUDvenhS40Qg2RSwIuwPnvY3LyE2vU5mVMeRkDKmDpKP5uvNOAD9KbLj5GdJatTaojBwJzPyGjFVMygJdRh5AK+UBDdiP4LZyhve8jO0LMHkMVqCkorO4PQsH4iCAFI8PFVwfCcmLwq2x72fAH9ZktmqlD5OiIOAxftZZSkRhmkW1RQ9weEuoDoTQYdqVPRzyvIbQdIAXs2h369hV809OjJKZjGNwXktsuBNJBgCE7qRJzXQPT6wVgI6xCaCn7G0B0xWsAyBIifHUfZ5lEVhp/VkIf+wc+iHwhOYlCN3LXwJ3MC8P2BATjHvY8VuxnDrCazdyzfFILhNlLAL+cQ/cDtp2AbgPLVpgK2ewgp4c/XkIc+A61QFpCb/bSdQxdTUAXZwF1LKafWBEFFMYEja3P3Y5g3EG3HkBxjyFIu55SihgAUBUI/QGiFYB07H+PzhmNpazjyJKZtCokcdqMU9zF2RsrVEn674/tpMee2rYWoax7DOAIyvq+iRzG4KIk9rjyJ4E2U5VHQz/T3xD76wwGyqfkcRz7OLIUF8rEFYyFyCNYfAx6TrDb9/0+4fd7rRr7xXhn+59/46QPLrz//E4HlYwD/KITwbvz5XwTwd0II/8ZPcz8+18CyeudpeOvv/CeQ65HScCMBKzH1UALCSOiDgLmcpvQqBlD4meME/8xCbdlNaRcOUAH1iwJmMcnzKK9jNUb9UmNc+ehZmv6eYtFz2AIAV0fwWAZUrxTMysPNPMpblT1SyRcpYugC02bpu+wvKS30JRdk+iBincDU15jub+chetjICqie+9G+ayE7BpGEyqP5RKO/8AxasOIoYCHA1wHVa365DBesHHENvZepomPxocTuSw71S97PNon1QQ7eSGEnqmc1hC8YRpJCIopNLLpvJslcEPRwFjsZQVaqeRCUFd5SPqU7eiuLLc9FdUdgf5zAKzwDVew8Ttu3lDgWe5E9b6mEPrEHCDE191bCLsiCLT4SMCvkUnt9oF+x2E9yuqQREZ5yQN3Sf9Y+BaQR0C0ZBwZpCFT3MQXX8LnK+8nD5yomzQ7nIfr7KFus7tjvVt3Ts2QWIldt1LcE+8dDinKDHAThakSGSzw4V8n3CZDdSN7B4965VDejRrIbTKpErrZQIzB75XF4JNFfBCw/oASv2PPxZhGHGiMlnbNP+dqmCo5iT2ZE90c+RDAtVY5TSqtXPOch1mqk8616YFzz53JDZszM+LgUuOFjdH8qkw+SCb6sQeG2hWO1hm6Rvan1zXSehaPcbzilh67c8vG2iQOjA+V5kDF5thbwsf/RF0DzOuSkY2FjkusIDOcC1S0/P2bO941XlBCmIVS1CblDM9WAJH9Z6lMUlsOx4VRAdZQLpsoSsxRZrio8GZ3k07RzeiK95rmgXJLpxrPXHu0jma8P8KlPkPuR+1olt1+2AcNSwMQKi2LP1yB5YJPkkq/b1MvoyiQVJABlEXtMrRWRPTuRR95DBpSU24A+SlWlCRjWEosXZJTSvqWKkdzxGD+r0sb3tgsYVmSLhpXMQSRqCGRguskDWxy4SB3nHBamQZwyQL+m/zL5SocTprHO3rjcGeljBYYauK/VPQd86VhTMjR99lPHYwpVAYBy7zHOZZRZehzOVU6Eru+jX0whDvj4c/KX1hvui6um5GL2mHKAl6pX6luXvY/NtcF4otGfKOhhGip1l5qv8YHDGFdNnZFyDOjPFNQY8jAvncvks+a+8X3jY7hNuSErNqwVJaKraZvJq8haDfoe5egxnnDIk6plki829Wum3spUw5ErSTwHrGZOryO/NySKnYMao9wS7I50lcqeaF5PY2ejC9knmas/Yj9qGoCwfiLAzBVU7+OwhRJPNfhYTxL/HX0M8YlDHAGo3tFLKcGB0AmHXUUcsokASMt6D3ZE0m/oY/WGnXEIU9xzwKB39Ewm72xKeLWLAtV1l8+PLxVUZ+DrguxcfJwvZGTtDIcKhYTsHaRxsIsSuqW3MSh6Kl2t6fk8WPgoPU2MYJKU2kUJ1fO+aiDIcbMSaj/CJxkyQFaxUrmfMtV9iNHCNwUHSJ2FHC3DbyQI6Kso11SxCzOF0qRwIEQgHUL2XQJ4GOSTBgNKQHQj/LyG7Ecc+xJDoaMXODz0WRYaGEae85pA+oGv0rqpNuSzPZQhTHUjQ5TJ2pi8WiZje7yoGTtVmKRAnuP7p33NPZNkKnMqbLodMYkZ+H4msCfffKB/MgFfKSawGmW5UJT2Pui1PA4DSiBSyIf3+Um3z3kq7J8VsAQAIcRvAvj3Qwg/EEL8lwDmIYT/7Ke5H59rYLn++lX4tV//W9iaGtZLzGL6a295MUrproX0uDs0eLza4WAKCAC7voKSHlp59EZDS4+nJxt856O3UDWGiaBBYD7v4YN4kKiaZKN1aWAc00r3dzPM1x2ECBgGjeW8x66tYfsCzbJHt6v5eXQC9ckApTyqwmCznaMoLYauALzA/KTLj+9juusw8F+3L6AXBnZfYH5xQHvXQFYOZWUxtGX0EApIHeB6hflph74vKJNLUXFOQBQeQnmEwIRTIQPK0kLKgK4tUTUGSnnsb2dIyY1QgdelXlF612ruy7ZEddZhuKtRn/XorxugdhAyYL7q0XUl3C56EOYWi0WP/Q9PEIoAdTrA7gqg9CgaA9OWQEzxlJ2iTzDehBMIBWWGKDxgJdTSILypENYGIUoAhYmL07MBri0gughG3zrAXDeQMdQJhYdoFUIZKOVsPGQv2SXaS0osKw+9GrmPYQKQD1JZo4cxlB5ilBCG0r7hLQ4rAMCtLcHH6wJu7iHWI8JthVA7+kdPR4RBAUZAxjTVIAPCwkG0yV8KwAHCC/iFhborIltCwOhqDwR2jIoA2LWFvtOUNVoBtZdMMZ05DlFiyJWPIJygnIuw4l7moKNiK2CejcBe83kOAubMoXqtYBa8PwTreEJBUK9bAXPqEWSA3iqGQe0Eyp3AuKQcUMbtu5rhSnbtUNwrloIbAIG1NDKmgaY+SWkExjN6F4MCzIlDeauY7qk4OBAxZVR4dpKyn1NkqR2AXNGRuwIDoPfT0EaNfI5Ub6FGAbPgsMc2AamHVB8IflQXZY276BONaZVyJKM6nvL1SYvTohXoL2LA1TsWai9R3TBwSzgB1aUAkTgIq/jmS6msTI/la8/O0fjWDBNwTbcEsuySALW6o08x3d+V3O8knUuA3M74Wo6nfP10h9zPOXsV0F2IPETzGux9HSYJo+qjOqPCFFalkJ8nhT0VBw4DigNBcrkhQObnfkpTdeUE1IXn71KFUBpspeCjFLSTXufUn5n8iUn26UoOZsxM5JoiXzAkqrpjCJXwVKV4Pb1nEqhP3kUfk2RZNB+HKT2HRSmRVhoOkOibQ+zXoyRUdRyyqZ4STjuLao0lB1NB8djS8U41QwH6QJnocVLr7DUVNqm6QjgOfFRPQO6LeE4Kkfc1JbyK6AVlf2sc8HQhJ6WOaxHVQBwO+DTAia+L7mKAWKDKRh8IvtNrkK/pISaJDoGDqxOCtWI/pcimJFuVknkFjzFV4rBmJ/oto4pADxwm2lrkiiKv02tD/14aFqVaGlfGLsWYbCoNw5TS9UEaBnu5guqYVNOSAC1VRSEDwlTpkvap2vncnVl0rCUZF1TcuELkf9npyW15FUH0jnU1tmbYkhxZv5IUPUFwO8KzfiWBaw6iPUIE/qm+SDr6OauNi9dgmZUEHPrFOpoIeOm/ZQgh3yf0eNqa+2YbGQcK/L5mHVQaEnEAn6pcfCWg944s/xCrY0ZuR5oo74y+d2lCVgMI6+lzFFT5qIMFkvwUmJjlEBAK+SDNVg5UE0ibfLjxb1GuGpSg4kDQJ5kY8KDk9DghIEd+efhSQ5qYIqtkDiZ6kKiasJJhRYqvYt3a4Nh7aT1Crfn3BGSFgIgMa07FTSE9D4DYZ+SkQkzMbHh4vwddl/FvfK3D9Bj3Y0AlgNy5ae3EmB7fPgNAM9gzdvJCCkmmNN2OgOYfuf0Y1vTH3UIIn/vwnm+8V4b/8Tcuf+rP+wvPP/3TAMtfAutGSgDvA/jbIYS7n+Z+fK6BZfXu0/D2f/UfwI0SclMgyAB1kGTRVg76XsE1ATgxkNcFfBVQbCT9P5qL6pQ6Oj4xqH5Uwpch9mGKvFCVAxdzbkavkz23KN7o+DwspPex+1J1It6P+1js6ZsJMmA8d1CdRLGRcPXE8tkZg0PsnF8OuuUXolmwJB4+LmyXAeW9wOHZ1N2YvmQzcxEXekx9DDl9Ni9IxunvviSTkErnXc0F5rhmvUOxn5iVxHAFFTCe0gPkSjKBQQLllovIcRUwexUDF0pgPPGoX/NiUW65oEvno74R2H3FoX7BvjobOy27S/qn0pQ4Rfv3Fzx+s4p+pMjYJYbOVWlhmrwsmICKRK4RSAvf/pyBIvThEOzoQ5IzB5TbmNpac7siAIdH9DpxERpyyIZZxQXQEFmzOTtHxxPuT1ogmeW0SDx+HeobMmvslARSVH9aPIuYPunLtKDk+Zp/zB5INfLYjuPZ7Syen+gp2z8LmL0gY8uET7J4QQj0VxGABC6ihzX/1S0XS8Np7M+UZBXNYgIawylfj8SYsVeV21ADF80AstcoSdLHpUB/GXD6PXqdIKYqh3I7+bbGFc9h6rQEyCwyLTV6ywzPQ32NGCCTFsgid6T250yFDVrk40vH7CtK513Jx9jZtLit7tnV2T6WmbFOlQ7lJgbgXAmUO+QFa30T0F9MtQmunhjQrGpwPM5xKXLfI2spKJPnApOgy0c2yZcx2VZMAT0qbmM4IfvoamQ20Gsu7BEl9K7mzwzjSP9yIdfckAG0NZlTOwNmbzzaRwrVxsM29BuKyJzqPkTJK89VSowt2tgLGhfrqUe0uSbrlK5Z1Zbsm50l8BZ7Igf2Nlb3lIcnG0EQlIwPK5GvF/Vt7FttBOo7DzPnv+0VJev9iYpS8QjsosVgOJFobph829x5dKcSyvAc1nc+9mZSbq97Lp7T+0F3k00hJT0ndUgaKDSR9RtWEraO7PyBwCRIoN54HC5pXTBzifrWZca2OIQHzGwK//GxxzQxlBCU6APIzOa4iIv8yJrWd7QopPRW3bF3EwC6M1oLbC0yIEtAjR9YPm9976EP7KtMtoL6zkUGVmTvnCspwQ+Snr7ukr7f5BccF5SDBwUcLjTqDWXfphGYXfP5bM3k2LT4D0LAziR0R0uD7jwBtqYNpL7lAl+4kKXgrpbs44znLrGBQbJWIigBs1ARBPO4dOuzVDxtz8xVZktdRfDltUB5z/7RckdfIJNc46LdBpiliooZLrQTiKJaQOV6ESodaPlQA9nX1HU5npARdpVAdW9iqA2gO0umTQm4SgE+gS8fFTiS94k2ixQoM5xVKFrLz9dhYgeFcRhPKxRbA9fouK+Ox7HSKO8j86oEdO/IOPYW5rSO59Oy31IIlLcd+sdzCOuhD/QsZoDlaDVICa92VUHvKIXW9wPcggylW5VURuwG2JOK2+sMgpTsnxwsA3U6+0d+Fo7spC9k3hYwMY/CxITYyD6GmmwnhICblRAhQO2G/De5jV9cx92QKWHVOvhZBbnvKFOuNEQXGczj+xd6YgwTwIs1IUkqnMN3InuZwWIClXF7oakgumFiIY99kSFwW7H3MrOZNvoWpcgJtJlJBAiOLdlaSIUH8tskmwXituy0/8DkzQQo7VWKPydfZZLHep/TYNMtfIb1/Ge6Bf+5l8L+wntl+B9+4+qn/rzvPf/kz/zYgM85sJx95a3wi//tv4Ntx/G8lh7LeoCSHp9cr9E0I37u8hV+59O30e8ZRmKMglIezqrM6rWv52ThRgl4gcXZAUNfoKz45lXSox8KjLc1WamzAbp08I4XCrsvsLzaY38/gywd3K6AnFv4UWFxesD+bkY20Ur2+5UeQnuI6xJ+ZbG6aNH1BcNhYu+ivamhLzo4q+APmomlY6RWFP9fr0cGtzjBAJzrGcQocfHFW1xfLxGMnJi2xDzGBFh9q2HXjH6HJvuG2uUkTF8FMoQA4AUQuxtlR1YJ6xHqVQX/Vg/9YY3xykLtFMpbie6pJQv5uoAaBJ789R/hg+++BZwYqJesaQkqQF92MLc1xMwh9IrHFYDytcZ4wX0Tc4swKMiDIrN1Ei+IIkBvNFY/d4Ptd84ZSHJbQFjAPhkhrwu4hSe7GVlMCMTYTLKYxT0TO1XsEQwXI3BfMDlxZIhG/UrBLFlNow4S808ENt80THMViImyHnqrMjOSuuPGc8fz7QSlyJLMZnmtYs8aGSEI3lcdyMKt2U8pUAAAIABJREFUvyew+wJgLg3KFwXM2qO8UQiSDJXugfaZQ3knYVbhiN0C3IygPwHR6p6BFPt3gfX3gPuvx2HGnB2KxWZiOhhAwkVzeScRdIBtQFDtuaDdP/eo30hKiGcElGbOIY1d8tjGNetg7DzkFEzgmMkRGC4dyluJYs+0VSCyXIOAraMk+w1ZVkonCayGc4fqmhJsXwTAx+GHA5qXBKppEAREkHctMJ5OA5KUpGpnlHGn0JThjMMENXC4sfyAg4FU1yMsZd3NK5E91eMp2UN9iEzbWmSADCD3UBY7Hp+r+ZE8HkboA/d/9w73hQvkyPjF4UB1h9zdOJ7EAUkM5qD0WyB10UgjUL8hq8HpP8G5XTAMZjglE51qXPSBz5UHUWMaRgTsnwKzVwLjku+v+gYxHTSyU4f4uQKHFK4SefjRX1BiXuzZU5gAVnofuIYMp+4Cq08EWDu05vlJ20gyXt0RmCcwXrSx3qUFZtce+7eSlJWSbldRguxqqgooX42XNM3BVbllVYzw/P/klwcQ2SIyWiYPaUQ+XjVE8D9gsj5UQLUNlBA7DmbqjYeZCQJkIIPkVI/S3Hr0axGPIVZuHMhaqgj2hsgSJlZVBIJ3GzMFdB+mblAR738b0z/3BF7SsdfQLAXkGFBtCeDrO4bTdJcS5TagvnUw89i9F4D+hJUwxT4yjmI61uaGA9XhJMpt73hOxoXIuQUJ2KshDvocZbz9qcT8pUOxs9i+U+Vz42LtUpL/1jcO7Vs6Ph8yw9hcW+zf1rlnNMl7EyMnXMgMZ0iDgkrkepWim9Y+Lga+2IbBOGkwMy4lqi2B1rhSKPYew1rBzATKNqC6d7AzmcG4NAHN9Qiz0LA1Qb6ZRVbNEmC7UmSPdX+mUW55vps3Bu2TEsIDzWuDoOkNHlc6M6i65/Pp3sd0V4nZiw7d4xr64KH3BuNpSbnszlEK6wJUz4qTYu/gC7KAtlGwM/q5XaUyIPWlzOCMAFigvKVv2c6ZIl60Nkq1J9Y0aDl5kROhJsh8jusC+uDiUEk8CMCxjYLqmbyapLxBkpV0M50ZSmk9XM1aEt9oBCXpvd+MMKsSXgtUrzu4ecEqlJTWasjaBpGCdsgAytFmya1wDmKwsOf88GfAOk4ADFJOHZrWQ46WPt7oaUUK07lv6ZPVEuIwwC9nkLsDQl0hVApyPyDUBVInpXARhBmLMKsghgiIa3o4hfUM5Nm19JceV5IIMXlbo384+1wTUM1fwHICewBBp1bT830WCyTwmepFkoTWe8CHSSKr1OT/9DHw5whIBufycwfnCEDTLbGUPwZ8Ht8+i1M+74zlP+/A8nNdNyIAzAqDy6bFq8MCq3LAvBhQK4vqkcVmqPH//OG7+Pqzl7ie8YLRG43z+QHGKfzak+/if3/xc1i+0+Pli1N88flrvP/BI7xzeofvffwYB1Ph4nyHi1mL73/8mAufqw7mUGB12mKza/CNtz/F7378FMZoyMLjYr3HMNf40tk1vvX+c3zt4jV+gCu8c3qH7/zgKZaPd1jWAwrp8Xq2gHMS+12Ny/MdXn10htlli74rUT06wBqF9UmLrZxBaQdn+YH0XsCPErPZADnvMVqFbl9BLSzcoBCCgCw8ROFRVgbjUEAISl6rymD3ZoHFz91BiIDRavR9lHoCQOkhTzyJryCglMfYllCVw6PzDV78wSXO373D+azFD+xbqCqL6hv3cH9wAjzp4dsG6mREVRkcnIDfKbz/8SXUxYCytBifBoSbEvACZltBLCyqxqA3NdZXO+z2DcYnAUJ5qBcVrASKdQ83UxA/qoHKQRYewQv4mcW66XF7YTA77XAQAIxE8bKEf95BAfBWoj7p0bcldOmAD2dcaJ87FqpfdLBuhjC37Cg9G2GKAnJpIALQNRqicpAqwJYFNhce5XLE6GvU5x2G1zOImcX/z96b9Ni2JehB32p2e9ro4/b3vnzZVKUrTWHhAtkyMESCkpAQljwHJIb8ghwwsQDJAxCSjSxgwBzbPwGJCQOMbbLkqsqmXr737rtdRJxmd6tj8K219rkvs6pMkVLmK3Gkqxtx4py9127ixPrW19nCE3wbCXlQ8NcWsnJo/88Gxx8MwCJAaw87KbhWwl+yNX2qFeTIxYbQS+hHHfb9Anbp0W57jPea98JxCdd46IPMCXcApZGbfwl8+B3Wh9w8/4CvPjuDGBSDq75tcJwkhBPYfYvg1twYQASo9wStoSALoHYa8mkHc19jEohpqwL9K4PqiwLdcwfZS/SvJjQ/LWHXDkNQcI2HGiSwNBhbC/lQoHthAB1QflnANQHt56wLMVsHO0qEwsMueBBuRS9vWglt3ggcPvEYf6tHuCs5DoBS2TJgeMxVfdVJ6IOEW1n4UWL3PYfijsC23EnYhrLb3RWrTxA9jpR2CfRPHGQvcHjJFFFfefRPAooPCm7l0N/o3IMaVIBfAa7xOLwQaF5LHL/FBQY5KrgSuP9eIJOKKPGMdRa+iJ2UhqqDUARII2FWVAb0jyhXps9LYDr3UAPBq2tiJ+0ZGf/UP2tWAebSQt/HGHdLUGhWXLRxTZS7hlgDYgBzxs6+BFzLewLi4ZrjsG30aL+PiwUlpcTdDSf0buUBoeBqen69BoYrSlxtGyssamC8pJdWTgTm3aPYnxqA7jL2pcaFgfZL9jy6GtmHnbpjh0uP8p7nafNHwN33wI2EBPjpO3YlYFZcyCj2s9pDODKZ04YLCmoQURIaoDsCpbQAIByTTMPbWTZpVoC0TEv1mscwbvmHp30dQbROSo7I5h8DDo8lXPK3LgRsO6eIlg9kWM0KgBBRwcDrG6QimHZkz/trdiumhQ2mm8ZKk6hmsAtuc1qJLIPOyo/Y1coeUrLGSdrtqtivaTnGcUMgGCRwvFUYt3Nnp3AEwoen7Inl5w7fJw3/p78QON7yHHePA+QkcfYHHuNWZJnycBUXOqWCPgIPrzSqe4XxnMB5WnMhwmvAtjJWc2gU3eynpRc4YFopMsAi5JCzoRWRAQ4IJUFkCts73qqcAOtLAdtGX2tUA+iBxzmtJXuCL7lQcbjV+frymvPc2ErA3mhIx21Ue4/hXCLIMoPX4y3rbXSUoTIoTWL1uUV/yfoRQLGH9rLIcl9pPEyjMa0Kylb3DsKTkTZLAUACDcFqf1Pzvt4QKMopUAlQk3mdlgrYEhAyyCrAbTTGtWL4Xa0wnCuUx3hsqdfVC9ha0iNsS4znRax74XvMSqH6QA+u6sn0lj5A2IBxq1HdW/hKQhkfP4dUXhCTRsE1scu0IqNmFgoiKEp2a/4tMCsFOQaEZQF9tLCtQlB1DP+y8IWEXRRR6SHhFgWCljCtzmmqQSoEzU5SNViyvwEo7zxco+CqGsXBIKzqzBYGJeHqeXocCgl9PyA0BXxkNK2q6ClNi1UNgai9XNGzqSVBohLwm5bPWQ931nKh5zgyLMk7ekUjm+iXFT2YUW5Lea9ieJMQOTwJk0FoWacC74HRZjCKIfZdnkphhYh2IRtZzACME8KyjZ+BfmYjATKcpzJbFetorAVkZCmrkiFBWueKkTBNuaoknPhFU5flR/tI4UDAx32Wsa/yl7KZf06wzzfl4cMvB9F/GR7feMbyb/0P/zE6U+L9sYUxCuNdDUigPe8wdFE3J0AfY6wyCDKQNSw9QqfJYln640I5s3Qi+vaSnw4yINQeEAHyoOmz0vToSRt9XQHwNZkU+NkzAUk5GgJZKzHwl114+uJyz1ftoTsmlwYVJ6GDhFs5CCPoA2w8gmbIjm34BxUAvA45eEQfRf6ZL2JpfEN27tSnBcT9lmCFguVxq5EMBwIgR3qCbMOOQdVJ+CZAdhLlg8B44aFGsgLm0qJ6reFKsjVuQTYvecsoqeW+1SgoVQ5Jtgj4MiCUAcUHiXDCdpm1gzAyeuiQ/T5kZOgdcjUrC+TEiQ59dcknR5+a7gkSUshP+SAxbTxZv0AvnN5L2KVH9UHBK3rcbBOy/FGN7Obz0feGwP2EOGnzJcOJfAEkb5Vt4oQsBhilkCO7CFBjlNPGnjzbsk6iek9W1CwDiqPIgUy+4MRxPOc26nfAcDmH6qR7IElAzYK1E16TsUqMjK9mT1x+TxlQ7qOfsOMkMPUfApHhOkYmTcwyPWkjGxGlxq6az4U+YmbxJL8XHrB1nGTEQCdhyYqqkRLbJA1OY01esyRvTp5AFtTPTEeSy6bx2Ha+VxJblpgX4eI5jWFKSV6rBuQKg9RjmCavXnMsvuTz0tAn50qOX0TwmryrqkeW5akxJbbO3r8kMydLg7m3MJ4bacleJdl0Efv9sgctBYsE5O0CZE9sy2so+LGVJZNAlIfaWd7pqpnhlJYMmKtEvk/URFCUmFcVz4sIBJXShTltW2GeoFrKj9OYgPkc5YL7eGxBxGAcObNbruAYygdKlYPi5FhNyJ69xGTxdy2xIiEzeEEilt2H7PWTdg7RSiFCKVBNjfFeWDAIKV2zxLqlffJGizLY8oRhjR8NIsT7OH3vItsYA3WkO2E7C6DoQw77SZJ5rwE98j3lYU4YV9N8nzNcKGQglcJ0uP/UnYicEJ7Oa2LQhCebZhsJs6D8m6Fbc3jR6ZjUFCJbEnJnY/ob5hUl3LaO0lUghwqpzC6G7MGkn+80IV1k366aeOLUGL2p2cowhyclBjh9NqTPo/T+dMzZP6rT3z0eWwr2cZVAEdOuk0cxeQvTOUzXKt33riKQS7+HCRTJk/2mcB9pTvzsMfzn9D7Jn1nRa5hDWywBUHGw7EedKL0NkgngavBzbUUiDKNs9/Q5XvcYClVQMsvFIMXrPzKoSPVkNblYENk+mTyoEsIGSDf7FeUUWTAVw4CMzyxaULQmwYXsXZTWwxdzQm9mATFfv9SnCR/idaJEN9V6qKNBiFLgXM3hPITx8DUTaeU0B+5kgJRYu5P/vZb0T0bQk+SxHyXKJn+ii0BPJ4mqpy8yMZpxX8LG504DdayjRDUyfpkhPA3WOa3gyPuMwUM+zPLYX1bbESWz/IWJf5hPf34K0k49jM79ojT1pC8yA76vg7xTgHfi08wy1a8FAqVgno8A5Onja/v9sySywUx/6s/+vMdvCmP5v/yTm1/5dv+1Fz//tR8bAMg//yW/uQ8pAv7gs1t8/m6LdTNAKY8nL95jcdGh29W4OD9AqgDxusLF+gjIgOqmw+L2iPasx3Z7hF5PEDrS9CpANA5/7bd/gmI7QhiB7as7LJ/uIK8GVLcdpbBHDVyO7IarHfyZgb0w8CuH6uUexb1EKALCykI97RAah83zB4hRQF/2qLYDoAiIcDEymOPxAHczQk4C5sLy/VWAPJ/gr0cUmxGhcfCtR6i4Aih+6wDxpIddefjnPf091wPcwmN8atC8FTBbB7d2wPUItZlw9f23sBsPv7awT0aYc4fp0sKtHCCA8ukxhg4IiOsB5dMj3KMR0+MJvmWwyOJzCbmdUL8T6J8bFDsJ/emeHs0PGuONhX8yAALY/EjTF3lj4J4PDDY5M/S83Vi4MsA9HVj7svSon++x+ImCeTbBLj3s0sNsHUTryOI8GzkZfzqQubi2CDqge2FhFwSV5szBXFiMZx7TizEyNQF2TZaS4S5A/VZi+NYIt+Gxp8ATc2OgO4nh1jJwRIPnaetRvxPA9/ewiwDzaoAvgOnlQHD4wMmnPgg0bwOm7/RwdUB/4+ErwJx7TBufAZIvE3ME9LcEk/0zg/q9yKEwEEB4NsSJDCet1T3Q3xAMjxceZsVFgONzh+G3ewxXLoI6dspJB5iVx+qnnChPZwHDlYerAvonDv1Ty944yTCawycWwlEyeXxhyVwcEhsSMETpaveUzOHmxx7jWUD3nRHlQ8DxlUH7JVkhBCbcQrDHcDzzGC8C9r81wbUBF//Cofv2RDndecig7/jdEdU9x5pk3HZBX6x0wPG5Q38dmKx65XF46dB8FdDfBvRXAf21x3geMjs1ngV28j3yOD712PzEQ1qgv/WwC05odt+1OD7jJM2VrNJp3gYMlwGHlw7lLqB7RFZSGbIg/aOA7nHA4nPKCeGjXHVNGfG04cJG89Zn/6aKpeXlPuD43KN+H9B+FWKgDSf4/Q09jsdnlH8GwUCW+l3AcC7Q3wZMG/pGd991ODznNg8vPYYrnsPEcFHmSyAvJ2D3qWfgyhR7+GKi7f6lj0E5AfsX9EDuXwWonvvqr0WuIUohPr7iftLxCAt0t5xAT5sZ3FR3HuMG6C8F+msBVwvsPuX9keSzh+ec+B4fc6GovxU4PmJibP2edUXDFSce04bn5PiUrGT7xuP4REQGh8Bk/4pjnFZ8j+4C+iuB4VxkH2j1ELD7hGMIGli+5qRImoDdtwjkvCa7x4UPyp0TIE2Jx3mRJS4g7F8I7F/x+BdvLL2FkhP+8uAxbQWUAbpbAozdS3ocuyuJ3ScC9b3DtBJ5AYehMZR5Sge0bx2atxbdjUB/JTBt43FFL2J/ze0P5+zCSyDR1gL75xLdlUR5JJtY3zuoKaC7UTAtr6OM7Fb14GAWAsdHkmB9CGjeWexeKBwezfVb3ZXMQLK+dzg8IVuWfKrDGQNJ+guB/kqiOHiYVqJ9bbD4csRwRnlnuff0+VZkUYcz2him6O8cthz7sFEojqwF6y4VuiuJh5cqpuoK3H9bwSwkTJSpTksyb8drBdsweVUN6e8+WU6vOEbdMfioODLFtdg5VHcxgK1z2D3T8IXA7plGdWcAAfTnimO5phdSOC6qmKWKYUGSvk7LtFtpAuq3A6aNgi8Fjjca+mDQ3WjYpcoy1PLDgN2LkvVJAKaVwuFpiXFb5DAaNTi4WlHiOjq4RmI8L6AGh/FMY9pqlO87+id7Ju9W73p01yX66xLVVx2GMw1hA4rdRFAkYgWM5sKU6lir1F9o9FcFEGtkvBYZ5HQ3Fb/3AdOmhOoM7EKzdqdUMCsNsyr4s56SXX3XYbiuIDuDaVNA70eo0cGsNORg4GuF8azCeFEC3sOsS+j7AdNFHT9/mGhqVwXGyxq+YU+k8KzsccsSblHAtRr9owV8ITGd1bBLpr/6QkLtR8j9APXhAFdruEVJ5vpyAbco+b1SCErBrmtWE20aIAS4VYUgKYF1qyTnFjPbV2j4RQXRDRk4imGCbytAzRUyCSyGRY2wbCilXdTwqxZBK4S6IgBctQTjBWtQ/GaBUOhY0+IR1gv4s/jhHAKlswDgPBNp5Xy9UnJrWC2A1QJi0fJf/BoAWdTIPoqqhKjSqq3PtSOYImMpBZ+PoFakkJ5YQyKEIKh0EVj7wDGkf8H/4vfBRynt/HWwJ5Us39BHAOAgfuX/flMe32jGsnr1NDz7u/8ZzKGE3EW5Su0grAQcyCRVHrKXcUU8MHSmQmSAQl5lTp6s9nOJ8YzF6dKSXRKWkjCzirUbC/qmUliKjmzSeGPR/kzTd9XHVLjI2ngdMN1Y1D8vyEKtPKr3lEzZBX1xqaw9bTsUZJBU+j6uGidWS3ezxMi2DBFSAyfo1QemVFZ3nAQFQebFtoBXZMASU5CkY+UDJ4hjnMxX73mjJs+Uq4D6AyeOifVMNR7Vncjl19Jwwl4cJIQB+mcWzWcaKlUeWNaFyFFkJiet7qZEwJS6aFsyPsURuP9ti80f6I9CaXQfQ0xOmIKg+P503OWJX2vckiFMnsvkj0oMiRo5AU2eMBWl/GpI0riYBGm5qp+CMcyCY+TACIJUDLcRgfdBdUcGadpE5iJ61FyDjxIrpzW3lRInpZvZO2mQgVBKYHQ1GaXMDmJOmGQ1CtnW8YzHnbxwdsGakaDYvaqPMUBGcuKuRuRqE5cYw+jr09GflOR1qVg8MVauQa7PSQFMeiCAKfaRBYghOc1XAd0t711I5EAdllyLnAoKz5+7KtZ9FDPzQKZKzOxbPReK817heW/eRkB4zeOVkTUQIW4fPEazoNzNLBm6kxJMeU9y3HLkfZA8gvX7kKWRTGgUqD9QpuhLvh4iMc4hp3ume4/pjRxD+RDytm1DEDVcxYTOLuSQKDnNaZXDRZIRRiaqEvl6kmkMSJU5Sb7pY2pmiFUvlGkKtK8Z6pJ+B1PoV/7dPLHuFLFeJQVNJZYrs+eZ6ed51j3BhppmD24K3xnXMnsnq4fI7AmyeQDQfGBFR/59NTGJU7KndlpJlLGyRk0M4UmhMImVVRNTYBOjWnTs3U3MpB5iafuOATCmjSE9NYGiPmXPTlj77CuLgAwCsDUBFwII5FaUWTYfHMe69/l/08p4Hlz253lN5iZ9z20SbNtazCylCfl6Jbk3ABQHShQT6JU2ZHYxbZvnwOc6jtSNy6ClEBdRQk4INa1E89bCl3ONCjtBKaU0rUTzjj8fNgrVPrKWAigODOZxDRm8xLCqkT2xygQmmUrk47ctJan8e0VwHQTgGgk1pC7bAN07eCVysE4QiKmeIdd6kK0Tefv6mEAZg318KeFLMo7peHUXg2cCMsM4bTTqdxOBWQSdAMcXIkBNoM+29BL6Qp4wbCKzdUVn48KZhhpc/IyIfj4XoDsG5NhGodgZMnAAXMFjSoxgDn6P3sWgBcTkIUKAWRbZz5j6XZPf0BcSqjMER5WKbDIZYX2YYDYV74nYq0r/ogY8xwkAqreQg81VJsIy2CXJMIMWubOTf58KqNQHaj18paH2A1yUo6bxeC2hekPQVM2eQwBkJq2fQ3kmi1BGa8Bk4duS8tP069lPcJsG6jgBxsIvaspNJTL7J6xnr2fcZ6r/CFpCDGYO05nMHMwjBFnJ+JwYpvw5gLKA6EcErfi+GLwjjEWoS4hhohdynFhXkraXgoLS194DVUnm0ro5vOe097KugGEkkKsr/j8ZdoUCCP1AMCgleze9J9BL4T1fT5VNwNEHwLvZc1lVfM5G9lQpdnRG72SYpnk7SgHG/Jn+yfT4MwN+vv7ab7jH8rd+UIX/+Z88+pVv96+/+Nmv/diAbziwbD59HD75b/4ThCDQdyWEAKRy8E7BW4HFesDhQwsxKITKQVYOwUnIgrowNyoIFRB6BX2n4Z8OwNtqnigAZAfTJ7YKgKUcNeiA8qajvHaMenVBYIsp1lUU/NAXk6CkNUpm7bmBumd9g12zLoEVCooT09YjlB5qp3LEvWs9qncK46VD+V7BLmdwR4lmiJ2UAm7hCdosw27kwFVfaSgDTayo2qu8wg5Q4mnOHGQnc3emPlKuqjv6qFTa7pI+MP5BDPkPq+4oOXVLBrl4PXdIEkQGlHcSdhXgykDW8Maj2AlMZx6qlyjvOcEWgZPxaRNI3llkQCitgFccly95/BBRAhylpNKKLLlj+EuUMoU58de2DMRJINtsQmaeKGONcl0fAdJA+SrA8ak+yopTLYYC6jcMWPEVIAyyRyrVPiRp4an8xy65X9XP0kPpouQ0MpUZWNbIkipfANUHBsEkSV6SUVbvY/ek4cS8ekAOYVETcr8jgSKBbGJg5soJLrroo5j/bxiu4soIFLo58ESmaoXI4pjFLFFM0s0MvtxJtUR8T5LzTps5aVj3s8RTOD6XQFmSqyZJZ0qsTZJV+rn4s/Ta4hi3k9JdVeyA3PJ9KcU27SONNwXVpLGLMKfApiAW3fMeG84JtFiREmWmCnNSa0/JnI0LyqnD0Cx4vUQ8Z77gAsZpd6RN1Rh9BG4eeUHFNsgdoqdgMWie65TImh7Szu9N8kk1RUAeYgem5XjmNGC+JkjeiwloSlp3Y9VHmOWkS1YdJDkvwVC81+L8YE7xna9lOubyQGAzLWX0Z3G/eogLAEPIUlER5ns3XX8RkBM2pYmS6WGWlppWxH5OgfJAcCsjYE2yVRUBPH1ukXUG8jaBuBDk5ns2LdrJE/ArbeC+G35esIqC/Z96TECLqavTkmFeKUE4gfb0GuEIJhOgBLj/FCqUrouIwUVJ6usjMCk6spvKhBz0Mi1lrOsggEvA3CtuWw9cFMgVFElWGz2FKZzF1jyXrhB58SbJXBEAPcbKjhNpaWZVG5mrIlh9EaJkOMpyRTzOgaA5HY/wIX7WcLEsLV7oztF/V4q8SKeHed6TgnmUYaBOlssKzDUc8T4SdpappkTWHIwXFSbChsxSqtjD6Yv4+995yktjCFQC89LFPsxKRsBL2etcyeFgFzpPRaQNczhROoWC96srZU7JnSX8IYNiVnuE/NriYAlqO8tzXCkEKaCPJvsMU59jlrSnYwCAEHKwjrAeamQQTwIrcrSwy3Iea6DcNwfoLMtYCUI5a4h1G2qw8IXK7/uFQB3BZFy9G+Bbgic5WEqpjYNbVhF4Ely7pqBc1nler1hVAu+ZPFvQQxiUgBxslNQSYKeqEBH9jvAeYjD0NAL0Umo+H8qYEBvBpnCezGSq2ogS0VCVEMYSZEqBXLVxKokFCCQTwFWSfZbFSffk12W06XHqU0ydlqnfEvg45fXr4Thf31aqHAE+SpTN7z/ZXpjMfJxSzq8/Cej5f407viaL/aZLYf+yA8tvtBQWALwXCAFYLAcsFgMuNkdI6SF1wPGhwfriiKA9oAL8sUC9HFFWBgJAUVtI7QFP4LdYDvAbC+GA3/6rP0NoHPR6wuWTB1w9uwNUiB5NIBQBZtKoL3rIy5FS182E85sdRBBQVwNWj/fQ6wlh6SBXBr71rEFpLdyZBZ73EK3DdGWhthPEJ0f41gMLi3I9wl8YhKVlD2LlMZ176O1EZvJRB3s9QZyP8Jfch5oE7NYilD5LW1Fx5dZX3LfbWPjnA/S95urmysEvHIQTCC84o/a1B573CDpguuJ2pnMPd2ZZvt1GRmAQmG4sV61XFqEMKO8EE1T3Cub5iOIgssfUbii5nc49XM0U1ukswC8cxiuHoALspWFNyZnD+GrAcONht44s57WFOeeH6XjlYDYew7WHawLsiwFm5Vn1sQocazH3eZk19ym8mGsvRsCsA6aNx3hJ6WQAMF3SnO6agOGRJfBbe4YZLALMuYM543mdth5m48nK6Dm106wJbpmgt16dAAAgAElEQVS2Gcd8niajITONZknWfLpwufOPibScCNg4hmmTai0C7CLALkP0E8ZJVMPn+icOtg2wbcB4TqA8nAceT8lglenMM73ynmOZNpQwMimW+1Y90N+EODGO/s4+JrNeuugrJEDob3hcZhXZzlWSgc4T/DEej9c8D9MmwKwJVMYLD9vwe9ZWzPdYSpZ1BUHhcBlTKC8900kXQCi47+mMFSW+nFM8E7ttazLOvmTiq1lyom7aODGu5yTRJAeFjEmkay5umBW9rMmTOm54rsZzTvBZd8PJsoue3MSKc/GDstG0b9tGOXQdvWaO9RlyRD4PIUpa2cknYBdkXm0bfZ0VFwvGLVB/oJSViwHx/opMpa3JSgNko4dLgfGck/zj4znkxqwEw1y2ZC5tmxh8wC4+9u8liSnZb94b7BPlOZ/W0edtQz7+dDy+EBjPRD7fZiky45wWlforBqx0VzJPzNN5ClGe6qq4zSVfm0C2rbkfAoyZceyvuR9bcbL48IkkAx2vn/C8J4dziWlD2Sa7TQHTiOzP5KICv6enc/YPFgeC62kdF5viBNwsCB5tIzBuCYhcLTCuef5sFe+dAugvKOs0raDHNILkaSEj0GE6qKsIipOUn+MRaN47uELkYB0EoLuWMeQnZJVHArhmpTCuZWaxbS3gSolxIzEt47mO9SIIBHymlTALmWWr41rOHs+CTC/PkcgMnStFPJcS04L/eC9FNnTB+hHTyNxPSR9fyEEyQQr057FWZAzzfuLxCk/20zY8f7ZVTGYVIt8LrqSn0lVkRHM4T8kpEet9eA6Sv3BaSbhGYdwogtGtRnG0OTQnxHPimtiNGBlxsyDQ85rbc7XiokyjCEwRwZ6NfZWNZJVIfN6sFMGe4nkMWmA809G3Sl+jL5iQ6ioG1ZgVQcB4puHqyERGT6gvmfhqWwXXxOPVAnZZzGAx3rfjeQE5OobcNCozwq5SeR++5PjMKiahGrLArtG5UkOODq6UGM4LejqdJ+sp0vgp5RUuIJSUziJ+7Rruxy1L+ErBV5os6mgQCkmWEyD7WpORs+uastaYDBsKRWBuPINwKgUxmtiNyXvTVxx/UOzBhBDwtaaXU0qG8JTxNYUClIRbVpmxDKdsW1vRh1noXwSVgiBSxAqSoORcD5K8kWVBplFKSmFTtUhFOWvQKv+bAZznPpT6mPHMAUCarG+UwKKuCBbTe1JAj1IQUpI9jIBOlCVEUUAUxQxitcZHXlGA30fJa05/TQA4bRP4mL1Mstg/6x+AJIf9yxTe86v+95vy+EYzltvvXYe/+Q/+NpZ6xPthgd1UYV2O+NC3aAqDySncHxpslz2ECNjHWpKqMCiUh5Ye+6FCAHC4b7E9P6AfS0jp0e1qYFQozwZIGeC9QFlajKOG9xICgFSe1SVOYuoKSO1RVBbWzpUmAOCNhK4snJXwVqJdDxACaEqDd29XEJKJrWFXQp6NCF5gvepx/+UacmEQgoCQAX5SUJWDG+Iq3iQhF4aJpZ1GsTAwgyYLOyqIkgwtAiAKDyECvFEQB4Xipsd0KCErBz8pCB1X0SYJ3VrYY5FDjWRt4TuN6mzA9LqFvJjghvie+xKL5zscXi+hNxPsvoCoWKdSVhbDoQQGxUoTAKJ0wL4gs2slUEZGeJIE7jogOAG50/ANZcx+6SAmCdjIGgMIOgCFhzgyKTQHLAFkYh8PcLuC7289024fNHzlM+MsvIDfGMj7gtvzAqF1SLUsolcQ6wnhoUSQDFHyZwZiVwAqIMgAEQQZaRUiQyCgOnpbVSdR3kmMFz7vT46RUR4or4YXPDc2jr0XsFeG6bK95LEBEEZADWSDgzqpCYm+yPHcQR8l7NoDjgsfeqdgL0mnyAcN3/pce6J7ekKHa5/ZVtuEmDZJhsJXlHSbs7jwoAPUkRKt8l7CbOKijObrUtKnPgpMFzxGOYns3SToPpGAKsBsHNRRAbFKBYihS4JBTCKel1TbARBMJxl1XpE3glL1g5iTFSPrnoOeVGS9MYMfNfJ9qp8Z51QfMp2FzJAnwDOz3MihUOnhmhD7DAVDkHaCwUI9faZypJTVx3qQxLgPVywRL+9Y6xB0gIpMfJDcjllG5jZWwyS5dwo8crEOwqwi26dnlks45GsjYniUWVO+jpD8p/HcFif7ATLDmZgvX3CxoHlLptnFQBsXmcv0HjWchDBV8zi9QmZE1UQQla6N7k6Yy3Ler224CGKWs9w9sUOJTeIvSdx3BPKnoSWqJ6tfHMiiJzbVF7E/U4uZ4V0mWRxygm5i2xPjqYb5Pkr3s61j12sKnQpx4aCc91ccQl7EUMN8H5LN48+T6iCFr+T7ViRGb1ZYSEdm2EQWOzGZ5Y7HkSTLwiN3kOqen1VmKXKYEhDZaw+4Jh0/FxESQ52qXkwMw0q+0nTNkk1ATRyPCCGrNBIb6wtkC4mwBLYpFCfdRyloC0BmJ/UQMC1Elvl6zesmAjIgFz5kVjL5SVmTlAKbRD5nXsXxRcaevsv5XKiYrJrk3wzlij3TUZbLhSkZ2UPkvtO0gCajTYXe+CgXN2lRlvUhaSwqBgip0efPG1/E6xO4eJKAizRcsEyy5ZT0mj4T5BQZUkkvqSsllUN7nxdobCVRdMxqcFHinc5hYj75mTcvTFCx47LU15cCKoUzyaQ68VCDg1kVUJPPDLYafQTS9OGeBhf5MlbbuJPAIRdi4FTqwp0rQ3wZA4QmB18xzCgF3CBEqXEZ5z0u5CAhKAFhTkJmTmS6KaRIHQ18oSCNy4xo8rLKyQEukOkMgcC3opQ3yXNhPXxb5OAgYfgLSCDrICKjmcGWB8RkyCaeModJApu+14rBPVKS4dSzfzGFPKX3i5gES2bW87VI7HsCafHrX1Y1cgIEP/r+9BECvZKJEc09mYLjPH1pqhv5+na/tu1/JSwSQeZfBsbyf/zHj3/l2/03X/70135sAKB++MMf/rrH8Bd+/Nf/3d/94e/+R68wBY2HscHvXf0MvS+wLCd8e/0WWnv87vXP4YXAT99c4j/49J/jGGgSCgD2Q41/49Fn+OKwwSc37/Bv3fwUf3I8w/6rFX73uz/D33z1R3hrl5iswvdvX8MGhUebHZ6d3QEa+Ndvfw4jFKQKKCuLv/L4SwQp0DYTLldHnC07fHhY4Ppqh34s8R/+1j/Fe7ToxxLTWODJ2QNWqwFPzh9w17W4enSPurK4XB9xu9zj8fUd3uzX2Gw6TFOBi4sDNsseRWPgJaAbCyEDLrcHOCUgJFipAYHltsf3Hr1B2Vr0TsMbhdBriMLj4skDDrsGunbQhcNm0+Fs3eFwrCFkwPXlDutNh/1DC+iA0GvUFwPGDw3KywHPr+6w+9kGL7/9FcLCoe9LBAk8unrAFLuegpXQhcfvf+//wr88XEKWHmFSOL/eox8qfOeT15ALi36ghPns6oChr1B+UcJvHLBwKFoD5xSElQgy4MV3vsL9rsWn3/0SD3+ywfUnH3C8a1Ff92hXA1TDAminA8Kg0V4fMVkNYSXK7QirgGI9ob3qYLXAy0/e4P7LDUIVsHqyJ5D6oiIQNQS6jx7f4ebmAR+OCwYWtYZdqBsDdBryfIQXAsX5ALXkokLz4oAfvPo5vnh7hovffYuj1AiGgU5+5fB7v/PH+MIvsLjsMJoC20c7BjNVAd5LFNsR6ssK7sJCLizKz0uIl0eYUuDi+T3q7YB+IeCbAC+A7//1n+D1YYXzT+7RPzQQTqC86WCkhCg95EMB4YGLT+5Qng0YgoZTZHEROKFefPoAoyLrVAJ+y8TAUALF+QDUHmGSCI2HPJvgLw1w0Ag3I+TKwhaCPZxPRtgggZWF3BpYLRAg4B6P8F4R6K2YLGzOHdaP98CfNDDnBK/iWQ/1aIC/K2EvKREKmvovd0aZk68DhJVwrUf56gD5WQ2z8QhXE9AzPMMuIpO7SX/ABcLzAVYKCC+hPjlgggKcgL82cGuH4oOGawOggOGFQf3siNEWUL2AtGTn9UFhunAQj0fYJkBMCqEA/KseOGpKvG8mqJ3G8Jhgv3tuIa5HmEWAuzKQtyMmr+DWDJsSLztYryCtQPU79+gLCTkogjhHQORLoPjBA/pWwCFNiJKXlddw+qtHiHclzDJgejrBXVnYGjwmL6CPrCPRPZlEXxLwuoWHOfPQR/q8h++MEEcF870e5ecFum9NKHaKr7+1ZGZLieHaQyCOrwD8pz3k+4J+3ouAaQvYdYDXAWYdt58Yyo2H7gWmLdnM4ZGD7llJoyaB4zOP8l7EgKsAu+JrigdJT3IBdM8tynuJ8SJgvKb/vXvqsfgiBhd1kU0PAv0ThjL1tyEuaHARaNoG1B+A/QsgKIHDC0ro+0ceehDobkNO/AUE9t+yUD3HMJ0RJI4XZLUhuADQ3/jc/Xp8FjDeOkxnvGeHWwYpVXcCh1dcCOkfEeT3N9xOuQd2n3Dxxyx4nSFZ7XL41EEfJPobnt/ygQFG0lL2Pm0CXEsZ9uE5xzReEAT0t1GGPQDjeQoGEuiveD685uRbmoAhBiwN1wHKcEJvF2SRh6voAY8MsFky4Xs8F5jOBfavAhZfcHvH54B0InqxuW+7ZNAQFFny8Zyv6SJz3rwPmLaSCw81a1O6a9Zf0A/Hfe9fglVAkaWtDgHDBdntJIW3y1QJI7J1wNYCdinQX3JxyrYistBkqKc1LSX9DY8tFILsvACOTyV0H8cfRGTfkwSaftrhUpLB12RgmX7Nif9wLnOoUncr2Zsamf/DcxkXCSS6W4YP9VcSemB40bgVKAbKgseNiunAAofHiim8jcDupcK0VrEGRkZ1jsDxMT9fjrcK5YFyY68FBEBf8xCwf6phG4nxnEmph8cKeiAQ764UAf4UEAqB7ooBRsoC/aWGi9Lnw5MS3W0BKIH+QmPaKMgQfa87+gJdLXF4XKA8ehwflWje06sqPTBu5+0eH5VA9DzbRuF4W/I+37DXs7stoSagvyogQBa2e1zDNYqLYLWMDKRAiKmy01mZ02p9FRk9KTBc0r+oJjKpdlkAksA/KEGWUwpASUzbEtIF2BUlvAz4iRHwUSZt1xUgBcxFA32c4FY13LIEFNlYEYBQU5rrlxUEBHxTUEZdawJEgIsKdUEW+HxBkBuDf0T0bIampLy0KQFJJjXUBZlH6xBWDVDqzJSGNnokteY+kk9S6/lrFX/RUoqtUgS3SjHMR5MBF1XJcfQ9nxeS20my2SjhRXqtjhr9BC5jxQoiwymEIFv69X9A/Flc6Ejy2r/A48f4v7/84Q9/+Pf/whv4FTz+2//+v/zh7/+dNQLEr/TfP/x797/2YwO+4Yzlk+9vw9/7X7+Fe9fif3v4No62hBYeB1uhjJn6nS3xO9svsLc1fnR/A+cl2mLCw1ijLQz2Y4VPtu9hvcSXxzX6qcDFooMSHutywE/uL7A71lg0IzbNgONUoi0MBqvhg4CxClJ6rOsRzkv8/Ksz/ODF5/hst0GpHZyXCEFgUU646xos65G/F0Hgq7cbNIsRWnn4IFBqjvnYVzCjRllbbBY9+qnINHffVVgsBrw4u8Mfvb2EEJTkLhcD1vWI98cWzkl4L3C26jAajbY06I3GoatRlhabZsB+qOCDwNCXECJEVtRDKbKzV5sD3tytALCqxUwam1WfmWAlPe4OLaxRuDrb482HNQBgu+7wsG9QNxMOuwZSBTy9usOfvD4HAlC1Bs7GDxUREOL5CQGoaoN+X+Hyao+HfQNrFNrlyIU1J7Fd9nj7nmPSpYOdFIQElHaYPtRA7cj8RlCIIFAtRzirYA8FUHgUjUHwUX4Ux6G0hx0VpPbs+L0r0Tw6YOhLeCMhDhph4WZWN7HBRrKmxgpgbRBGhfa8wzgW8FYiDAqytfC7AsXZCHNXAZWH6BUB6V2F0FrIwsMfNUTlEQaF6rznmDvNlf0DWVm1MvBGIngBmMjeCgAjj1ftNFN3WwcUARgkIAG5MPDHgv/vC8hJwtcey5sDDl8tIUwcf6e5ymskxPmI8KFCqDz0vYJ/OsDHc4hJAoWH3GvuS5HdlZOAv5rISJfzSqQoPEKvIBoH7DVBXkV5OmSAutdwDaXGXGVmfYtwsRqmDFzZdYLPReYordD70qPYsV/Rl1zxV0cZGVcJ13iE2kOMEupAFsJcG4iRK7n6QUYGgKyp2RDkykHCX00QHwpO8lcE4tBkjdUxeuCMgK/m41WRFRae3mhhkkQzhZ3EYytYV1S9U5g2Hn7pUL7RZAtPQlgS+5gZOszeT+miD7gOJ1H9QLHjcxBkCQFEplySVZME33KkRzDoKG3feuhOQB8Y/qXGJB0PcyWIRAxCE5nBJGNKn3R5L5l0HcD7L9YKiTjWxGKpKfqno9JATrFCZ4ivi55N21JaXd7H3sV1rCuKFUNBIndrmmWUu0dPcmLVAN4v5Q4xnIr7VePMAjN8KmT5frmjzDiFjrnoTaZU/Rd9w5mxG5GrQJKE1RfzsaheoNzP+02dl2TLkSXZupvH7hrkELPhfPaz0sPL/ZZ7+liDQA6QUiOiZDygfpuky8i+4xxKlbyZZQzq6k+Y6hN/YxoHfZ+xXsbPHtbkvSQAiNVKTQwpi5UxAMFcek/at20peU2BO+ma2IYpvim8i4FrZG5TcNFH1xGsbTFNBI51qlPBXHMkGXxEnzDHkvy5AHJYHuTMkvsCqHaBnZNnEvW9x7gWKLr5WhddyNfcLAXqu5DDm1I1Sgr2Ko4cY3mcj1VN9PQGSWbUtBLlwcO0ia1ErvKhX5VA08ZQqsyATpTuqpHPAWQp0+eDtCEHH3kdw8Si1Dz5WYUlgD9l/hHoE/UlJ/imkageXGYW+ToyqXYhUd5bmLWOYw/RMxrl7IKAOyXo6qOL7CVyYmuqpUmhSCIGMaVjSs9z+4mBJZsnLb2fPlZ1SONhlhrFIdpblkVmYBMrmdlaGyCNi+NQEVzKzI6K0WUWMjOYkeEULtDrWUTWMC62U4Ybj8k4hCiHDUpkD2uuL4lez/w8QPZREdClGhPRDQz9MRahrSH6ka9bNHlMubZkmCjL9dFn6hzfe+w/em3CBKKpZ0ZTCmBM8oMIGk+wQ0ghQlLMQT5JsurnqpFfWpNycnx/Gpv59Ycfhj/1Z3/e4zeBsfzeD6rwD//xk1/5dv/Gy5/82o8N+IYDy/V3b8K//z/9Pj6MLUanMViNq+aIoy1hnIIQAYtiwroY8L//wbfw177zU/yzLx5j6go8eXSHz39+jhfP3+HtfoG2Mng41HhxdYc//tFjLJ/uIACcLzr89MfXgBMQrYOuLKra4HDf4Pxyj/2hQQgCl2d7vP7yDPVqRFFYHHYNqobsljsWgAxYnPWwVmJ60wIrQ8mqF5DaQ0oPc18DMkAvDcIXNa6+/xavvzgje1ZSXgovoEoH/6YGLkb4Y0FgszIIvSZ4sBJCBlxcHDAYjcPrJf+YdJSFlu/JhrjaAylJt1OcIC8c6vMBw0MFWElZqWAYDwoP1VgEJwg6Vha6trC7EiJ2R4aCfyTlSJaxvO0wfbngcR0kJ/+NJ3j6yRJ2ww8ctVeQTzuoHy0wPDWsdZkEqvcSw43Lk3WIgOrVHtMfrWG3FiLKbGVPiamYJELpUdwp2A0n9Tnts2SAkls57u/ZEepHS+geOD6jx1MOMWBp6VDcK5hbg+VZh/6PNmSPNgxbMluP5nOFMXo+hRewGwt5VAgXE4KVKN4WMJcG5ZcFJXNpgn8zQb6j1lBaAXNpoN8VOSTJbB3UQaF+JzCeh9jFOMuvVB8LuhecpE6XDnrHIKY08RAeUJ3A8Nih/kLRv7hgaIEaJN935lC9VwgqcEIeCBjKOxkntnNwUQ5DijI9V83Jx65h32r1RsGsmQZsFgRNwgo0bxneJKfYxVgR0PgqwFXs6/QlJ5/TJmC6dFj8REevp8iJrakn1DZMfS0O7BctDpS8MtlYwuuA9rXAeM5xMvRIYNpwbDJOQs3Go/xA8J2CZFKBvC9j0vIkmFRcAimxlinPBFvlTuTE3pRwDHBb05pgtTgKjFumLgfF86UPZFpUT+ZLHxja5KpAAJVCnyI4Yv0G7wF9FLm79lSWOW2YBh00fa7SiVz54vUsNU1BUOOW57S84/tT32eScZoVgYxZ8nkGBxF0wMevY9hS8iEWh5NxxU7S077SJIf1FVOSg4gpuw2BnNcxyfhIb6dZzhPbnDodJ/1mSU8qvXDIfaQpEOo0OThJdYt9TNmN/aHl/QyYAbJHaojSTkNGsjjM6c9JwpskkkmqWz6Q2ZlWHG/zlj5GE32WyX+dgEV17zFuKNdT40k/YgSptpklqLaOzHXL+6e6D6wTGU8CmcooLV0QnKkRmLbs8gwqXTtuM1W8pG7LtF/dI9/LKWhqOJdMFlbIsk7bIIcuZWAdw33YMwo07wmQbBtl0VHCm4KObBNDnGLqcwp/8gVQ34cI0CltTWFFEJTEBoEMYqWN0tKRAMs0SQJKQDUtZZapjmuJ8kjgl+5xZQjEKD/lvZS2l2SuXs/S0HReXMUkXdOSpWw+OEyLGHzkEWW0MXQogp8kh60efA4XKo4eZsGQodwRGrsz8/lNoBTIHsjUwQlEqWny2Z6EKLnUIxrAZN0YWOQLgeLIBF1I5O5JX/B4zYLjEpadngRMBN/pdQgJ2KZ7QqJ5M8KsiizNtI1EsbcoDgbjeYViZ+Banf2davBwlczP575bHwj4YmdmkremPs003vRwlYyf8T56pDXKhymDxCSRlYYBQ3K02UspnM/H5CsFOTiCTMFzLo8j02nbcu6mFIKSWCViGu/H8tJQRE9mTKwV3uf3ByEgnMuhQMK5HBqEEBgCpCRyH2V6CLKl4tgTQI6GlSOOwT7COqbLGhtBqfwYuCWAGZNs0+tyLcrXU2G9R7AOInk/Tx+xtzM4VosE5xnck+S4J0m1OazH2l/Y/ulr/pUeJwE+33Qp7Pd+UId/8I+e/sq3+7de/fGv/diAbziwbB49C8/+8/+Ck7cD/ziofp7ASzsnSKYUQwjkUvhUz+CL2ZNT7ALlOjVw/1c8Lv8PGYMl+Ic3hYGkSoQ0YUp+JxHiRCfWM6QEQJZ/cyLg0yQoTmamNSdBcuLEgBN5+m2KY8jemGkbvSQWWLx2GNcyBmLwD8vxqcDqJyFXLNhFXJFXcyVAdR9gFvTR6OMcUZ4S+9JKcerb0/08qXKViH+kkFmPcs9qgPoDJ0oi9qydpk+6Mv1R4jaWrx12zyjXkXaevNHjweCLckfwgPgcWQnBVfkaqO88umsF3Ydc+O4q7jdF8Cd/m22RvXVqQk6bdJVgdcMUMG4ZIiFjV51pU0gI5gTS6K+q72JJOXjuUlG7XQioOJ7+RmD9U49xFUFgO9c82GYuak/H7DWv+eIrj2EjkTxQagho38UV68CJhStnFqU4egxRNqbHkGsZ+N4EeGKNQkrOjL4jaUMOTPko3TFO6KqdY6hDLXKqqTRzqmdiHIZzifaNhxoD+nOJJtZGpGRPPXACqwdeY9OIfM36S4nFa5fvreQtqx8cjteUbSXGJBWqA3OhfZogqzFgXEssvrIwSxXL2ONEKk4c0wRNBGBcSVbG7FntkK4lV7l5PUwrUfQhru57HG81qgeuuLuCvig1kcFwMTk0PXTnczXLtJSxZiPWvFQCRc/rXz04HK81ipgmq3uPYavQfOAJsvW8Oj+uFYrOR7aCQSNq8pAjJ2TjRqK656RVWiY+pq9FlOrRS8VV/2LvYBcyflYGyDHALFl9ERQ786o7S2ldTMrk77zjpLCW0fPElfVxo1DuPYqjy1K7BKTU4POYpPGwC4Vix2N0NXv7pA2wrUJ5b3B8XKG+iz+vZK7c0J3Paco+sh1myYCV6p7XXvfRExV9WQCyX0za2UNW3htMa3rBXC2zH5G/05yk2lbl3w81epiVigwbq0ykY6JouYteqtgTJ1yAMp4hJ0v2FkoToA8GZlPQExfBDMDt2ZYpoCIEmIUmO+KRqzJcLePfOJ6/oIDqvaH0L4LEVEKfAIGcOIHjOeekXg1kmGzchy/oxcsT9hDgWh3rRoBiZ3LqJ89HQLHnB5BdFoAPuQqFKbJMGvWFnAFACAQKsbYjgYRpwy7IxCr5UkIf4mQ3MnwIQChnkMEFNpFZHTkxhVR1DGORk4MwntLD6DkD8FEFRgqKUYNlgAsA2Rm4JQGAOo6YrhaRuQvzsYQAvR/hFiXUYUSoipxCCg+CDSHgtSRrFUN48v0YAYuLiaMikL0S1sOuKuj9mGWXQcXrvRsBJZg2OsVqCCnhFiVkx+uQ6kF8o6GOUw7R4YHxtb/wfHyEQkF20wx+CgVYT0Yujo3diQryoePXleb3A0EaXCBYUXKuECk15GgAY5mA6j2ZttjBCGMp8xwMQluxIqQp6EUcJ4QFAVT2GlYlEJ+X+x6hqWZAdAqQymIOr3Ge4wJmD2ZMRg1lgdPgHWEsxGSyfzGDL8eqk7QdVmjYXB2S9sNzLeYTm8aTQGJ6faz7QOp5BHK4T7D2l9dypJ9JmWtCIASrPaQCvCOTmPyMKVjH2jkYyDlQkuXmsX3ND8lf1fj7EmXC+ZGAXfRVBmshtP4FNjJYC6EUgnNzAuxpQu3Xt5mYTSHxZwXzBB8ohQUIVP+Cj98EYPndH9Th7/+j57/y7f47r/7w135sAKB/3QP4//JYbHs8+Rs/x3EqsakGKOnx47cXAICm4gfutppQKAcpAn782RXOLg7YNAO+vFvj0dkOh7HC/a7FKgb8fPhiAxQBt4/uUO5bvP23S2zPj/h3n/wxfvRwi8/eb1GVFlo5fHi7xvbigLo0eHe3wrdu3+IPv7hG3UxY1BMOfYX7ty3Wj/aQIqAuDSar0PUVlPIotUM3lHBO4nAssLjo4L3AcKggtEe7GNE7iWks4EaFop1g+gLoFep/7wN2//QK4pMjnJOQ0sNOGvJ3RvQ/XyEsLWAk5YPrCd5K1MsJ+88XePS9N/j8i3OUiwnOSWtcwYAAACAASURBVCjlURQO1kqYSTNMCIAdFf2cPm5/KBAmifNHD7j78TlC6yB6ibBw2JcOftAQlUMYFIrVhEU7YrdvoH9aYzp3aK47PDu7x2d3WwyvFyiuerjPWrJAL3v4dxX01QClPe7fNyi2A4rCoXvXQowS5W2H+2NJBtcKoJr4f+khdgXCwkJWDnU7oT9WCIMig1l46JWBPRQc30iWU0gyxtuLA+6/XDMMaFBoHx3Qfb4EJFBdd/B/sIR5MSLJrIKTEJqeQ3jB93UacJRBypdHmHcNukcC4ZxhTPVywvjFIrIBnivFFVeTxSQZ/lM73DkBWU3wnUbSNn7QnsFETkAsLaQKDGL6qoa4niiPHcm8wQkII1h1owLkUUFeD8DPG7J2twbyQXMSWHG5mR69eHACZMB9oL/yjsAj1dCEkkykOki4tYcwEoDD/qWA7iSmcwc5JWkQYvgPWfLpgmnBvvKQI/186qrD7nWNoCl51bs4kRecOIcCs+z02QD1kxqqFxhuHD2D5xbFnUaxk5i2AXffV6zD2ToUH9ScEroIDBxa8ZjDzQD5ZQ27CVB7nov6dZTTVpSw1l/FJNY6QE3xvWCIUnjRA581AGKSaRvgVxb6XQHhAHMOtD/TkI5MohqYfptY3iBiWMy+iLIsQB/oa1ODwMOnDJQiqKZENRQeZey+VQOvEwM+WKXTvBV4+FQCUW6rppgmCuTKkBDDc8bzALMWqF8rmI1H/YYqBspfydq7KqB+V8KseCvqLjJ1tcz9uklSK2z8WSNQvynz8/ooYgKymtngyBbqTmE6o0RzuCAjrgZABA05ArtXJYoDcHxK1lwfBSdRIYWpIKbOJra2jOeG4WaJAUNk4JLHrogyVN0VGC6B5i1iom8MkKmY1tx+KdBfz/2yviCrKqcAu9AnIVGAvFW5eiaFByWZK4CcjixNvD/WQHXH1wxXQHkPjGesDhJh7rJN8sOZIWTHblrMfHjBCgo1UMaaFgnbtx7DeVzoMyc+Qp+YYy7occEuHsNE0GMX6R5FBHZFlACHvPCphwK6p+eSgV9k0NLiXJCxV1qAvZAxhIaLWnExU1OGuiglhrNYUbILgChyn2uSG6eAoCQRdSVB72kNifAVTAss3rCH1BWUxALAuBY4+8MJ3XWRA3+SXNjrFBzUoDhE1jm0qO8CxjX3A8mFXjKaDaqdw/hpCzVykTEt1lFqG+t2ppDTiPU4V6UATDFuPvg5VCjKPfVZCVcyXTaxrNWONxH9sbOcFABcVXPRIdaUcNGuQblzsfJEZJkrLrntovPozzXqezJn7O1suFAUWWAufHDRaPEnR3RPWt7rVzXvM5lky8jX1ld8vji4nKqrO1bCpMWv9Fx6uIodqGmBJXV8zswsA4MApugWnYVZagi3BALPxbTVKPYOqrPwNUE5z1UEPB5wjYI+2o8qUrgDJuiq0UN1EaxIXifVG5hNDd0Z/j1UMblXS6j9CLeucqhPqjQJSsy+QiADdTEaylidi+qbFKhz0sNpPWAdgvcIC/Z5in5CTooVguxgYHo9tKKMNbHZTQUxThxrXcbPC59DivJxC8EKkwTK++Ej8JcXYhIg9Z5A9bTSxPvM1AohEIYxMq0ElSh03g7GEdDxvDiX2coQ4v5U7Nl0Ln/90SO+PsHRECJg/v8fv7GPbzRjufrubXjxX/2nOHy+xqNvvcVXH9aoa8pPp2OJop1wvu5wHEscXi8hWovgRUxW1cAgobcT7LHAp5+8xo//2ROE8wlSz2mvq8WAh4c2y1qrf95i/H4P7wTEuxK4GuEHjfPbBwDAh8+3AAB1kAg3I1NZj5oyyyhT1W9KqFcHmM8XgAAWLx9weGigKwv/2QJu5dD+TEP/3h127xYQo4KYBPzSQT0ouDMLUXiUtYH5YkEvxLlF+WUBs/GobjqmsU6SklIj6NE7Sti1o6ToQeV00qABX/noKxPZixV0QGgokw1loCwWAGRcbfeAv56gv6hgzi3Ktzp3XPqCMsLmtcTxBaWsPiachrMJoeNr5SA50ZeArz3kJDmGxyNCnCCKXkJHb5xdUkZprwyKrwrYlZ+BR/LEXA8o/kWLaRvgNhbFBw2zdRCjRKg8ZbtBRBaSQU7SANOtRfmVhm0ZcJKSUfUxTsoAIFAOKyeZvWSuSv2ePD3TFbtGXRXgthbiqPPPpSVQSAms0hIUmJWHGiRc7VHeqXmbHpjOPPRe8Nq+Uwg6es4eBMZLHr/e8/wGSYmoNALDpUfzOvpJG3rQupu5vzOlw4ZYo5NkqtOG/rmUvFl9oN+uiL67BBDsgmmi09ZDGoFiJ+P58dAHgmU5kelMclbbUu47XlKGSyY+5O7MNPn0CmhfC0zreE1Hvk4fBap7VoaIlEg7cPKnBuD4zKN+I+duUUHlwLSlf1JY5E5MVmPwvLgqSos7ysNcSaCWAEcCJNOWx3x86tF8NXsp1YAsbXX1LKkNOvrfHpCTThPI0D3HsP6ZR38pKZ0VBBS2Qe4IPT4OqO4FzIL3JdNpkRUTrubXcuJ7p1X0AtpYFRL7MXUPZNmipU9QOKD5KmA8T/64ebL9kT8uSRzfBEwbgWnLfepjPJ4SqN9TolnsKEm1MeGzfeMxbmXs2GSvpu4Au0jSVo7R1TzH4zlQvwuxTgFx0n0ip5xmySIEj9FrAhj6wvhciF4zr9MElu9LrLRpZ0ksEGWxAVmhkZ6vPgRWrzSUocLTP5jBng9UMSwYUqO7EBUqc+KpcJjTZiMYSsoOALNctaEkGhGolLsQK2Mi61pzgSD5b1OqakqYTRUXSTbs6vl+qu5DrmXRfcgMOuLnWlLtsPc3nndPJYAIAdV99FGKuF/H/RYdgZyKoFL3lHemzs9TeX5a6Cl6j3FFCejiS4PdyxK2FqjvfN6HdDPbnthmVxAYpf7QBF71GDCuKEWVFpgWAvWD5zmKx6jHmKAaVTlq8hjX6iOVQpAEnNNSoH3n2XV69PmzHQIYN/Qu2lpS8VDJrLTQY0C5c3OaKuYFHd4rs+qi3Lv/h70367EtO9CEvjXt4Qwxx40735ycaZcxpbagaQEPLZCallC3aBASL7zwivgJvPCIBAIhQEI8wD+giwd+AK1+ANRdk6vKWbbTmc47x42IM+5hTTx8a619btrlVrcs2VnqI4Ui4gx7Puesb30T+zOTqqm+degvTNqeCJmAqp1L1LcO9kgV1YoaA4SLsEtVPI8ySVULIxQi3EJBDhHV3YjxtIIaQvHu+UqmZO1YgF3u6BQ+Qg6eTKlINSeK1S3s2fSF0SfrnVQfSqC6G1mPkhQgwpEtzwm1AKB7z8RVy/3guAIQY0CsqISQI+Ww3lC1IWxAqBT03sLNDPTecj8aXfyYOSlWby0QAnzqt4xKQK8GxFoVoMVOy0gWGECUEn5uoDrL50hBsJdlrzGWHs0si41aQu5H1njkW/YKHtZsjI5gMsldY62RLTrIjHdKfc3gLSZgJ7wn8+qoNEBd8bnJiylyb2R+nZKJPWYdicjeyEMm9JBF/GYybPZappRXoXWRvwJI4DZwnd5DGIPYdUBd01+ZV+E9hNGIh6yo978kfc21JNE6CPMNruvQZ5mOaYwRcRjwL3r7XWEs/+c/ePYbX+6/++Hnv/V9A77ljGUtHf7+B3+KfzT7GC9ujvDk8ha1crhodvjR2/u4Wm4w0yP2TYWfdBU+uf8Wf/n8HozhTFaQCu1swGaTPhTOByAK/GvPvsKfvn6Apra4Wm5gvcJu1UDsNfaPHU4WHfZ9hfknO2y2LYITGKxBCAKQEfVJT5N45XA07/E2MthmsewhRcRKRvjeMAQkJYnGIKB1QN8EVKc9Otvi907u8KObOQNMjj0wSvhFwMNHN3jx5TmOzje4ljPEBmiWA/wrhqYY4zBIA3MyYNaMuHuzJJPlAaSQkSgJUEKFUmkRZfI4SHZLyj23y2wk7ElAmAXE2qM96TGOGr7XWCx7dKaCviUgC7Wnv7KKqN4obL8/4OpqhesfXXLwOwg4KyFaMpthOTIExwqokxHy5y3BohcQnUKcO0CA95kI6ABx6YENPYmLx2vstg280lBrRd/e24ZdwMcOzWmPoZsTOJ0NkIq1LXGQ8AJQbzRVOU9GHJ3s0b09gRzpP1SdhN4LjOe+1GwEk0eC9MPlm3/cI64qBrW0DvGjAeFdA9V6hEFBv5PsvJwRtLdPN9g/X0DtJdzSAxJQdxwIukXyxSjA7MhgiSAQFx6jF2Sq6oDuSYC+1ZRcCgBRlOqGGIC4dAg3FfSOg/jt0wCzSSmK88jj8ectnAFO/tYbXP/5BfyJg7rVGC896tca44VHUGRDVc++TXOjMF7Qp+orILYBGDREAPorTlx4H+GOCOZziAoCWBtiJVQnCW41PZZ2GdPgm8BTJoCQfZgiBXGMZyGBA4GgE6CYR0DSqyhCYosEyBD2IoVJCHSPHBY/0xA9GTs/D6jfch8oi0+9hIsIfzXAHlUQAWjeSAa2HBNg+UsCELvk+qqVwPYjj9nXCv0l+zjrayaUIgLRRPhWorpjCikANG8ImoOJ2DxlHUB/j+9NP0usaEzMTqq08A2Pn1mL99gwBIK7/Scj2i+YCitW9LcFTcDm2ohQk23Kks9gIvRAOXF/SUAeBRNd29fv922qARgXwPYpmUz3eID5qmaFx1qkPk1RQCZ7THNhvcRwThAaVQoHiTzPLvVj5gmHnoIT9Bf0tkbBAT8DiATDay6SxF/nihUOmOTAntFqDfTnAu1b9pqaLc9dM5LxsQuB7pjPb97xOb4lQzOcRbRvUCTucqTU3814DrZPgPlzHpMMFnQnsHtAa0Hu+5y9iggqYv9AvFeV4mY8X8dfRKw+kmjfxALofc3tr2/T8lM9BgT3Rw2sSqnWgNVZZg509wSaa4K84YR+5fEYEJbHJ7N/2UOcl23nKcBGTNJ55TPYTO8lNYFh17J7FKClw1cCUkb0p0wuHY4YgmXnEovnBLEZ5AOczMjbsE/eR2WB1UcVP+NnwJjqcKICWa4EDO1SwGxQZNDS8XrLNhexnwB8d07LBL2/qbvSA/ulQHMXsb+gDF46mVhkUYJ1ohDl+OQeUGUlYOjXzMB180Rh8TygP1YEPWkyRnS5k1OkqpYAPz+oQElVLr4SZQID4H4Npzr1tNK6IGPEcEJpd3fPwNW0COQE25g6NsNhv2jFzs5qE5LFhICgv6rhagmxUDAbXySFAODmBKc51dYkX6a9MqhXfqo9AVKolYebU94dJTs1dRdhtg7dpYGbNTAbj1CzZ9NsPcZjjebaolqNGI8rStCHwLTYO7KQZufgjigtJhNZodTIaAmRrsWomPA6nNWsZwFtAbqjJNq1iqBYqAQGCcihBNw8BfZoTi5zokEitKlTU0v2WUoG+ojeITa6yKX5uclzHhsNte4RmooBPglQihRAJZC8qEIgtpTd+kZDr3tASkRDQBlaQ7tDWx2E+PjCCJLpDJT+WgJUYV3ysgKxqckEGgMxWN5fV8WrGeuKAHQYuY8p7IczRwfA8qDzEiGQscxgVCnAOXomc1Jsfo0+gBGVmdjEbODOIDtJuIvU9xDIAhCHvZv59ivCfIQQOHjGv7z9Dt6+1Yxl/fRJvP9f/hcwtxpuFqD3EnrLEvvseQxp0G22TAwkq0XmQwTK3JjeyPtyOMZwEUqCIT2YEQiCDM1pgN4yYt3XlM4BaSbaZUmVAMTE8lBKxpCRKDijbza5r4+DqijpXdOJNcnBHUEzAMTNchciv3h9zf2Ujvdl72YulRdJPhYFw1dEEGXWuAQYHNxyAqBvMqNDlkd3BCZIMqbxKKZEQlGYNYBsTV6vtGm2fJ22NQU0ZObL1ygdYtKhJH26Oe9DAnEEBskrmaRmObwjb7/Ps/MpTCN3yeWwG+EYRFKtUPyODIDI7AEHPDENYLPMLqoprCOvL0oUFiUHueTtP9zvfCwLKGgZkhJzwXpab5bnlbRPjxIuk1mdKNN5qfm4XRJc5F5BcQAWcjBLvj/KtO8poMWnrr2cXlnCNZJfMrMWJaVyAPQuor/gdYAk06s2vB5CzR7ESfKYfMtJ0WK2sRTJ624KWsmMC5CkaMmzzGsE7w0imZYZk4wrMTiJDbRzggMAKZCG2yEtJV45SfK9pNUE1gh6phAMSusSmzFkgMT3j0lBMs07epQzY6U6DiLMLqI/SwPa9D1brSOG4zTb7lGYM6aDch+rDdnnfK50xwGz6if2LJ8TnVIsKV2LKV1x6j3MSaH5WiGbw22jNHUKc8nddcCUilltyU7VqwA346C6ueFnUr5WESgvlDZXFkyfHVniyvWR0XINwZxrEsMkpx/1jU7IQx+snYvyXg85yX+f+9rS6RKTj9a2ZIuy/JWfcdxHsltJipj85vkzcFzkdNDU5RdRwIx03L5qOyV9muztNQQjmYEyHYEMQ3BSaEyIrIFIbFvuXaXcL71f3bTtUQD12qM/ZRG92YeS+JnDofJnT/YsBy3QvhnRXZqS6CltLJ2KMcn68muKBx4ogTsEJVNnIEvr32dVM/ACOIhn6uYUMpTPn7QEGONRAknpM7ra+Pe8z+yR5DZUm5A873JiNlM/oxrol1W9h2/o76SiQKYuy1COaWZfdUdmrIQTJcBIfy2ZyvFI0/fr6TdWnS+sW/7uDVUOOGIaKc+th2tV+W02DqHKfuIJrGVvqWuTf9jG6fqOiX1LCaL5ORlEhUrSP9374kszW7J0zIiYmER+foryuRWlKIyetImdjDkohuxhrOSUlJoYPuFikU1mRrN4UrWAN2l7chfkQeCL3FuERlMGqgTUdkRoDK+RzhF0pfernxvIMUD2FqHSxWMaDZlEOfrJz+qSvFTL6ZrM22gU1KoHtERoNGRnSz9kNAoIEXJvi080p6mK0fF5B4E7USmIEIr/VPgUwJOBTGZzhaBPNN8OuyaNLseDJz5/IYuJbdRq8nAClKMmr+R7MlMp0zXn32cWAwEfDlk9IQpzCR8m2Wx+bfZ1CjH5Og/ZR+cgjHkfzKVbLExmYicPk18BHIbp5PqPmMKECvBMIDXGSGYy/f3e63/V8n7dLYZvvcfy0x+08X/8gw9+48v9Ox/9xW993wBA/rOf8jt80/TO2WOPOPewxx7jCS9oe+rg5hEP/sYrNJ+uMJ4EuBNH0BYExpTm6eeM1w8Lx6j5Teq7qmNhCVQn6I3yBJihDuyuqyJL0TuRvojIAFJuygHceBQToyDQfTSWwXROwfNtRHjQYzgN7N4zkWmUAzBceQwXHtJyZl9aMYEVTbCs95SeDecBWb41nrD0PjMx9S1fG+rIlEgA7WsO6HSXJHsSJUHTzXKIzVSwnQf9lBIyxdJXHMjpLWfH1UgJW073VAmQsvibi6lvcoF9nimfUhlzObY9DtD7BIoVy9EzYDLb5N3qeN9wxtAj1RGIqQHoz7lunySPAEHqeBIxnBAI5SJ43ad0Sk15ZQZxGcQKjyn103EdMkmb8vp8PYGkfKx0iuR3c3BAskWRYMq0nJz2KZM/zez4vKjpudt8HAoDJyIwexNh1sDsOY9Lcx1hNskzlZQhuktl3fv0excxexFLcA7Pea4DSNI+cIY+A+lqzdeabURzE+DmDFeiBI5yyCzDNNsp7VU4HsscwiMc0F3wfFdrnjO9j6hXEaqLJW2yfRsxnMZyvKVFGdCqngN2NaAE/hCg8rFqwwFljrCvbxl45doUCBUZq1/dpWu65/q53XxetaEfqtry2p1dB3RXfE8370J5vuoYHKV6lCCuqBKYNEworTaxAKXhVKLapJTII3b4VRsGDS2/YghPtUkyu8BjxNqOWEKL2puA+o6yxeEkebEcUz/z88yOgWNMYWXolQhcl+4j6k0orwN4f3sd0F1wIs3sIxavfAJyEdUuwOwig8b0BILMLpaQITuTnJwYD853YKhWljAORwL1OtDjlWRx9Tpg/sol5ofezGobKO/dhwIU63VaV/rcye8T0wU+b5ySVEXgPtR3gaFTd6H40eq1R73iE6WLMPsIswsYFymcrGdC6xTcxGMtPUFe+y6guXGotgG6J6Cu1h7KRlTbUGTewkfUd7546KptQHPjUyhW2lcfYfaBrNAu7YOPaG881BhLvURz61FtQ9m/HLQlbYSyEbrjMfWJSRpOTUpBRQFSvpaUhu4D9BDQXNsD2XaAGtJxHAjqTApFki6DPc8JgQTW6M0kkLWtQPvWQo3cR9Wn7x5JhnB3nwxktaFktVqxXxiR99W3FjqtW0TAbCwBXJK1inS86lubvIuhgErpKcOULqK9thyAR4Irs/EwW0/gkq51OQZUtyP9i52DTvUSamSwkxwDqrUtsl81hMISF6/ezsNsXWHHqpVlQNHaIupJHqp6D721qG56BjaNAWbnCEzSfoVKFvYsA7tqbaEGppPSsxdQ3Y0wWwu9c6juBrg5VVV6ZxPIElApuCd7B1XvUsKpKnJN4QLUekyptgSVejVQMbCzrNDoPZNSpaBXcyQoiVUCWmNA9a7n9g6pWsQT0MuBwDFqCb0ZCCSFgNoOXA7ADsaZRlQSejNC7Ub4WQW1G2AXSe7pAsOQVPYs+hJ8JHoH0VkC2jz0u9kxrbXRkPvc7RMg9yOEC1OoUQ5VkunvGCF2/SRxBSg1FQKys8XnCBcoWVUEZlEIiH6A6EcCuJGhRFCSQG+0fMw6YBghhpGs4rYjQMwVIrMmTUiRVYyV4eRPCgyCD3x9qilhUJLh/Tl1FZiAY9fzsX4ArGW4UdcjWsvXDyNBmHPsmTQaMXscfaCMdRwRrUUcxvd+YEe+Tkjk3ktozX3M8lbvJ8ZSpTTcDC7zfYdDI8/gISEEGUopyv/0W8pf+ZMfgxS/tMxv6y1A/MZ/fldu327G8qPH8dP/7j+DVh6j01g0A0anEIJEiKTLu30NbTyGzkAbz77IysFZhap2cE4ieAm7qrG42mK/rSFVZO+hjFAzB609n7OpoBcWrtMQJkDIiNl8wPZmxkCZAFTnPWyvsTzusLmbQdeu9CWGHcvM5NwhDAqypkwSVgKSXYA4HQniUrCMUBGh04XFg4iQa40w94CKEDpysmlQgA6Ak2n6Nl1k5uBvOc2aFi2BIhOLIIAqUDI7MPAGXkBEkUBELNsYsx8zVXyUKpM2wKwU3DJJAFtfAoSiiFy+itBvKyAC7oJBMmGWZ7eQlpv8eVYU76XwAnIU7AV0ySvXRiYCrxR9j4Okn0UDYZ4qVNK+Rk0ApDoyHm6WQmtULLJKaQX8gh5M1dP7GLLcNQLCCagx9QMGsL6kz2mpfL60E/sVFdlcd1C9wVAASu/omyKAiQeflWZHJj3UsWyHdJMXNLNtqmeNR7WmZzMYILQBei0L28iaEpFkaNz23KNoNpPXMiffZjZVHk4IJhkrDkB3tRKwy1gALSLZ+Jyki8SiqWHyQeZzk8NeyCSklydvKAdg3O7MwKs+e1JFmfBhqAePjRpSd1+dJjzSczNzlxnYnFILMR0bLiv3TGLqZWxSlUieHE5AGpjAb06Izt5Zu+B5yiqHnBacg3Nyl2FQnNBwM06I9BdULGRp4OE6c4VHZpze+8me1IgyaWDnaZJBoshIARQ2eVJgTJMbvpp6BgEy+75BYdsPAVzpskvJ2ZkhLEnQ+mDZyBNWaaIlewTz211M2+1mia3PrKZACSnK6boAynWd+xKjEuWazc/LqcY51MY3B8E1bppwUmMsbF5WN+g9r+/s1yznPAEr3/CCZZALSmdgVh1kRYibJU91Cs8pHrs0UZcTn4MmK8b1ERwGNfkic2iN8Dyn5bM7K1rkdNxlYo2l5eSLm4kyUab7lGQtUjp1Oh75eo8aKdWcx8/ORZnMA6Zro1xvCcSqMaaeyPS+GslGZi+osglQJW9rloKqPl1bCZRn1jcnRucka2U5OZJTsZmSmydEQkkePkyzFoGsaK6aySDzPSVHOu9ZbRIVCrOuxlium5JifTDBSFlrSgV2qZbEp/3UAnpgwm5OWkfkhEDQooQclVTlg/ewHKeaETlO25wZd6p4RHkv5PejSOvOCe+QE1tPxZMovt7MBOeaEZESfX09JUZHmapcxgDfpqRiz2Tc/Hkt/ZTwDPB4uEYxMCeiLI9hPb4Aat+wC1JZhuDonS0JuT4xt5T0Uu7qUwWKHP3EpkYm9UYB5MoPAPQ7JjAbk30DPu1zYsiF9Qi1pr/SBfjE7Moh+yzTRIpPqbapegTABGxT8i9c4DoEJa05kZkLSucv/R+lJNuZk1M1g7UQAoGhVpMk9VASmgFc7qysDNlPo8vvaV3+fTY0s6DZt5lBaU5xTV2WqMz7bGHGBKkiJIYAcShJjanr8jBlNi9XyF9Kti1VI3m9OFzUwXoPE2N/HXv514Sx/B/+4MPf+HL/7kd//lvfN+DbDiw/fBwf/Ff/OXTlEYKEHyWEihDXFT2CnSKTOAuQcwv5ouGgOgChISiRg0ygCRBJ0plrFcLlCKwMcsl3ng0G+AGuctDNwLAUt2BoSVAoKZsy9Tv6GZ+HQDADJBBlKcNV+ynkhANVMoKhSsEqAXBHAWYlkySW23QoF4wJNHJAMAEduwwF8Ph5QPtCYTw6ADYgCMoDVpEYVrsMTJZMQMYtuP6SkLg6AGme2xxlLN/aoQLUnmAgGAIqgOxh1Cw8Z2hKSP2DKCAkD4jHk5BkyyiDPzVmtlQU+Wne/6hTUl3qSczHjl9WByE7gmyyGvKXLpcRNNMuc3l8kbxqlGOdmW3fTmxvZl3lSKmq7lKQSDt5yXIiIzANCmJKJJQWsDNuV7UCunscFOgdAVx9I9gruJ0AYB4kHxbAl4FGlke66X4AJZExM7bABBZycmZJhEQewL8vrdU9CpOVly9tCmuxEzjNg9Y8KC7HKe0z5dwoNR0czKQ3t3h/4JTL2n2NqWsR037nQJwssZaJBcxgI7Ph7wVpRD4nv6eDmQaaGVyXdMp0fwZN0qVycYMS7PEe0PPT//Rv5o42VkbZUwAAIABJREFUFJmw6pnwqTo+5lqy0AUQpG3LMrxcBJ4H0PnY5O5Dvu+5LNXxMy0olIEkN5I/ynJyxKdidXY0igTw00B/N1UXZWCbgaTueL4LWKsn9j77H3PAzqHEtMjTkwIg7yPDTyaAmm9Ffq4Bk8FvmMBgnojIKayZrS7XQZKrl2OVQJkcOTjOjFwOj1JjLFUyQYtJupkG5wX0i6w0iOX9kmW6ua9Q+inIJgOKfMvvnTy5IlOvnSyyu0niGAXPUZ5MiFLAm0kKq2wsXsNYjqWA7gm8skeuAGiZwJagqoXvr6n/MJ9vXlPxPXCTa3UOAQaAIm1lvUc+p9Nr8j6V3skk5c5gK29TPkZF9hu4jkOQn9dXuh+BIgstE6f573yLZGpj8SIKqC6k2pP3P0/yJFJeBzB9fk4XJgpYyxLUzLqKdGxlYkUPz3vZx8jX5moWXxNMiBDLoLpUx2g5AS4j0vs7lO2KigmoIdWTFPAoJgCdAWNmT1mLc3B/zNdfeA+05cAcAn2Zzs1UwQIAqnNFBgsgVd8ISJvAYNoGnv9QzlVmJwEU+arw3N98Hvg9MB0TXpeySGghBIJRieETBZwIn0BV9iwmAIgkpWYy6sG68vOASe5aJJt8vXChAMTyW0uI3hIYHoK3Q1lrDtgJSVKauyetIzDMUtac2JoAZu6pLOfosNsyg9EQ8J5U9vCWg3kOfYtZOpulqzFOlSD59s3l5b8Pq0C+KdE9eCz6gPdCdw7lsN9c1+E6/zlu3/Yey+/8oI3//R98/Btf7r//0Y9+6/sG4NsthdXa48HVHeazAX6rUc0sEIH2gw2qkwHRUFp6+eQWYWsw++wOUaei+AigDghHDoiAPusRL0aoXmD589T/llimzCipjsApNAGxolQppgARe+QR65A8UBG4GIp3zzcB5k4iXIwEcDoiLDzCFemeeDayW1FFuAUlsXYRYC8dqpUsM6vw9ImKq57gq+Lg0B4TfPkZt8meBj720Q7juS8ArL4RaF6qMugKFTCeevg2wi5TouhGwD0aMDzmsUDgfcIDsmdvpvrOFmYrIP7GCnYRYS8cqjsxyXklQaU99ugfOtgn3E/G7HNb3Ylj0X3LY9xfUBbVvBHono2FFTJrsofS0WtWrQSGMw/dCQz3HKKK2D913I4lZYCz52ICJYmtGq48pBMYjwNsSjZ19yykBeYvI8aTgP6BI5A2MTGAEcOHPfbPbGEuh/OA6o7ngVJfsnbdI4/hkscyKkD98I7nVk1yX2mBUAPdZwP6ewHdQ8p0th85ynA1BzSbDwOqVaqRqJmOOlyQwdw9Dlh/6hAMma76NqK7PzG+WfprjyKGy4DhLBbQMZ7wfzfjczPIkiPQX/LYdVeB12zL7TE7YPeBx3AeYBdMjO2uYpJmMyV198kIX1Gqu38YUK2B/VOP7pEr7JjP3aGZLUrgtL/yMOtY5MTbDzyTXa8jdo8okdzfjxiP4tQlO0dKvwS2z7jvwQD7B5wsqdaxsCHBMIxG75gYm1nX7bOA7l6SmKZjUq1jmhQiEN18wGOSJbS7x2k5jyPuvhuxfcxBpRojNh8HMl0G2D2aUh0J2IC73wvoz/ja/cOI8YRdp2YdsfmI2x4lcPt7QHeV+jErejjdXGA8Flh/EtBdUlZLWR29u7rj87dPKe30NTCcCuzvCwxnnAAJmmmx3VXq/DxliJNvge0Tgf29KSl0f58D3Lvvsqx+OOPgeDwS6C4FhhMOnoZTkR5jgM7t9/hbBBRmr7sHDGdcPtlSfpYMp4Iey4Z/d5ei+EurbfL09rx2u0sGodx9CgJzw+t2f18kOXxavgVWH0k0dwGuZcKoXYgyILdLUXyBu4f0SfqW6+nPBIZjge0jCTcDdg9kST8djiXsjPu/e8jJBF/RzzscSayfSdg5H7ez1Eu843G++0Rh80xg9RFTcX0D7B7SUzkeE7gMZ9yG8UgU5m3zVBbQ5Cv2vfZnEvsrCTtjgNHmqSygNwoBOxPYX/F5eojY3VdM262YgNtdSIzLKVV1/ZRerv2VRH/OflVlWbexu899Wj9TqQYkYFxKuJqP6Z6y42Ep0Z2RVXKNwP6ewvYRa366c4HtQwU7E6XnEyDoXT/VcA23aX9PYlxKbB8qbJ4ySMabadJ0nEvYhcS4kASwCQjvrlQBkWoIqNaOQT8zejPlSAbUNUnmP5fJ00457/6eLv9nhs81Av0Zk1TtTGLzWGHzSGF3X2FcSOwvFNnGiszb7kpjXKoibwxawM0VunOF8Vhjf6lhF4qMWQD6M10merYPDUSM6C5ZgdJdUO4btYRvee729w2ZtVpiOGPAjN758r9vFD2ZtYJvJfZXptw3nOhS1bF5UiNowXWFyG1KbO/ugQEiw4P29ypsnzSwS4PuooJdGvTnBuORgV0qqM5BrwfYpYZd8gLt79VMnN2O8EbCLg3cTKG/bEpaq09+UoDA1B5X3OeHLeTgsL/PGbz94xnskqExbk75rHCpU3ZZ8T055zF0yxq+NQz5iSjeSDc38E06zkqUJFh1s0U0iQ0UAr6lDNW3BvasgT1tIDz9nqGeZjFEZv0Ez1uYVRAxwh81ZCobQ9+mpPczaokwq6ZKEevgT2YIJ3Ne/8czQEmG8YTA8B0AqEwBmznZNXtKhfN8fgrfgWbiK6Sc7m9rgk2tWDeSQGysDeKsKf7L2NZMlq2r0ocppITQmr8VA3aEVgzq8Z6/lQKUgjCGvkwp+Dt3ZgoJiCxfTfueE1+zxxLgevKPUkUWK4wmUP1n/fzL2+/87VvNWJ597zL+vf/972Fja3y5OoPzEtYrGOVxMutws5thGJn6KSV7I7e7Bkp7CAEs2gHrXfPeMp1VCFayqqMOVBKYgLDTgI6QladUdlAwjYPdGUpNK05bS8PHvZWIg4KoOL0evYQ0HmFUkFWSwAIo3YMmsJakoZTEzC3s3hSZbHvWoXszg5ilL+g1+y9iHQAZIbdJHusFa1X2mumknvsCfSBzbQK3a6cRdQACv2BEw6RW1B7oFKW2Y5qltIJAfc7tgxeQMwe8rYGLAXhXI7Qe5miEXdUQjQc2GtXVHsNNCzGmZFMTAY8kI5Wl5kRYgdAyMTXMPOROIfcg+oYptWKQkAMBtNopsrlJkipGMsGxiqjeqSJzZBl35Pa6aaa9pPEFpD7AHOIUYO6SnDekmdXGQ20VEAT80sPcKQQ9ySgBkJVteK7lnl2KEEnCKg8YyjR762uGGAVF9lzYxI5awD4aId8ZsskxSUFnpMLkKIoMzrcR1WpKq82MRA4lUsPEVpd1blIYjEWa3Z4krLlSRabXmY1M6awE2pQKionVyIP8xGRJy+2sbwQTWj23S3WcGBAOsMf0IIogEBSBlxxTj2Yky5zZe9/EIskNOqW21gQb3A8GUuUZ5fE4YPZcwrfT+zkqwM4j2fEmQnoBYZMc1wPVRtAHjUldkAG6PSZTklnuLM21S55XJLYus5LBJKA5pPOWgqL0nr9ZmUGQnRk8NXB5rBlhpYj0KaAryROlB4YTsuul1y+rJzJDesCaZrannKPq/cdzGNUvsVMHLK/qmWSq+hSO1aGEDqkBhfnMkt+idsiT/Enum0OyMnuYpayHgVW5pgWS66xvE+jqJ6Yos+6sueExPew2zGFXuYsys5++RpH/Apw8sMtpsjAHOEUx9Rrm6zmqg88Jh8IA52NW0nV1ktUqUY5rZrpcM7HFqp/qLlizw2oS6UDZZmIbs4c4Ky18nQKQWj4/M8tqjOWYlrCyxD6VYDF7oBD4hqSYAUEx1WxkBo4yXd1FZOlpTMFbOYAps/8A3mPslSUDHNNEVg4Iy/Lawlym86X6NOlUCdQrSkujnAKSMruWJaeFWUvHMCgCdE5aTYyhrwSqTZiYwsQuuyb3bJItLYmyKWQKQAn3kj5NgmWprEQJUmpWPl3jAv0JJzKKTBwoyoLJmxyKNDtfByEFQDHIKDHlST4KZGmpTIqcWFjnYAias4+W7/H8+S0OZL+x+EN9o9I5iRNTGTkpDSR5qo3InZaFic3sb/6cyUrNgVUhfF8EhASyRUSRx5LllsW3GtW03mAYdmR2Dq5RJVQpHzvhY5HKMlCNfxcmFiCDm2mRAxkwv+9T2NJhIFUEQckBc5cDqjLLGKWAHByXkcKRZJaypmXDhUkKG+MkfQVKuA9TW8mmlgCiwuZOj78njz24lb5LmySkWYYqBfswZbovM6CHwT5Z9hoIxvN+FSY0+Ul53NK6lfzVrGdmHVNFSJHD5uCfwxCeGIoPNI6WzOSh1zLLYPM2Hi4ffwVz+qtueZ++9YzlLP63//CT3/hy//7Hf/Jb3zfgW85YCkT8fy+f4Iu7M3x48g5SBvxbj3+GfjR4t5vhYrHDs/NbeCcRo8B63aJpR4QgcLbcAQBmzYiq4jejfdPi+09e4t/53o9x/+kNdGPx2bNXWCx6qKWFah3CliOU+TGLvszcQi8t/vb3Poc0HmenOwhBf+Tlozv825/9JYSM+OTZa8Qo8OTxOwgJ1Ef8ZjathTAB5xcbnD9jdKeaObh3DWTlcfboDvOLPebNiOayA2SEbizkxYAPv/cSZjkCTuLR915D1B6Lqy3q1uLo/gbihqkvzXmH9qTHw2fv8ODTtxBVgDQB1eUeamkh5mRtVeVRnfbQbyrorYJaONz7+B3ah1vUT7eICwdV0zf57IO3iDcVzj99B/V1g/kHK4hewfYaYuagKk8w+uMFgfXcsyN0L/H4szcEkk1Ac9EREKoIfTTi7JMbVMcDLj+7hrjXwz/ugcuB+wnAPNtBBIH2kxXUIPDo916TxTix9DE2Hvq7a4Q2INYB8w9WqO7R8BXOLaJJADASSITLEebjDVnoqwEwrFLAkUWcOzTXEuadRrgYEaoIuZewVyMHZrMAd+YQn3Rkve/4ZRtOLMxGQD7ew554RBXhq4jq4zX8sUP7mt9Svo5wJx7VjUwywojx0nHS4EEP1RFIye+vCRCSpLi+FrBXFhDA+HFHmfRxgD0JEJ9uEe4NjK9/OpQkYnvPor7mjL2bBYz3HcYrh+Gegz13rNhoOdhw39nz9fccumcj5l8D9a0kK97Gsr7xgU0Dc8CeO4wf9KjWAt1nA6pbQbZ5ELBHEeNpQPfxCHc5kl3+TscQpr1A/4zHc7hyhYkPn21h1gS4dsmBynDpMV7x2HYfjqyzGIH+2Yjh0qO+lth+4tA9dOge8Pd4TPa3v0/mfjwO6J44svBBYPuhK3Lw/sMBw1mAW0TsnzjUNwLVnUB/5bH/ZIQcgd2HDnIk8JMW6C/JgDOYSBSgbY/IoA+XHnbBrkg1crBabfj5ZTbA9kOHsz+LmL9g6NTia6C+YfjVcBownkZ0V4nNf8fEXbtgEJVrySDvPrLYPfVkRC4DussJkEtHED2chzKY335sEcVhHQiBxu6xx3DGQd7+QUS1ArYfOzRvI7fjMmL2kgPh7XdHuAUAQTDoK4YXuRmXm8OwAKb2VquI7kGEPQL2DznQ234QSjWJWwDbp/Qj3vzQsZP0EZnd3P9Z30X0F3xtrv/IkvHmXUR3n/uzfxhhl2T+qRhIoCmS/cySZ3tE4Ln6DmtCYpJwuzkACewfEQDqXYRbkPHNQN1XKMm79GoKZJkpBCtAtk8YKtS+JQM/HqUKF5HY1pE1IghAf5kSgmfA+mMOUvvEFEdNFjQz4GYXMXvDgLPdY4HuHtlYCMpjoxTorhKbe8pKhfFkqiLaPOPy1BAxHAvUa6ouhlOyrlmG7msyr8OJwPpDCV+ziqO5Jau9eyCKLHk4FeysdLzGxiORApACXANsnpItHU5FCcyyc4HZW4/5a4/+VMLsA/QQsX2YnntCpjgk4OhmQHfOx3xKbN1d8f/MMnvDbdg+VLBzgkfbcttcS5Z5XHLCzOy5//2JLIx8lj7v7ktU24Bq41GvPGavLXwl0N54bB5rjAvJypGXDr4S2F+Sce1PZPLIeqghFrDPsCuylpnFba8thiX9mft7Cmbr0Z2z8sO17Pisby329xS7LwWZ5919DdcymMnVlMWy4gSo7iyyx1H1AbsHFcYjifrWwjUSaiBbXb8bYJcKw4mC2Xn2jkbAbDyqFfshVcdqJ06g8b3cnypsH1UJMLJ2xOwcgiFbyg8dAd9KqJEyTDcjYzucati5xnjENN3+rIJ0ZGzN1sI3CnrnUpq0Sr7RiP6swnisi1dXDR7DuSlhZCJEjCcG9tjQm7lnEFGoFOtXlgZurtHdqxEqBd9q2AVRf5QC+q6DHBz0ZoDPbKeSCLVC0GSDRQTc3MAd12Q5Z/Q2+tYASVIbGl3Y0CyxjUbBHTcQA0OcwqwiGJ9VENYh1IYsZ5ICh1mqLhECqCuE4xliYxBmNcRgEWYNw4JSqm+sNcFm9lpqVVJjo5IIixQW5ChLfi+FVoq0rPQaKQvjSc9nAoNtQ2YyMZZl/G3S+c4BPzpJd3OYTw7yAabXSYnoffoJ5Qchlt+/9icGvCfJ/ZbeIoAA+Rv/+V25fasZy6PPruLf/l//Y7zaLdGNBk9O7tA5g95pdKOBVgEXsx1ero+wWs1werrFYA3GQTPcxwso4xG8wsf33+LLd6cwxuOo7TE6jV1fwWiP0WoY49B3FbTxmDcjVpsWi3mPXVdDaw9rFSeoRoXzsy1W2xZKBYyjRgwcdJjawTsFtzGQcwdTOYydgTIBbl1BH41wg0I1s3CjwmwxYPtuVuC/0AREQkVEJ6EbC7utgCBgliPsbQ0x84ijhKg9l9snc+AoSxiPOBoZCOQFWck6T9NGMpbxgL0UYGDP0vG3Dji5v8HdzRxSB4YL3dYoYTXHI/y6YlCNjmQ/o4DYK3palzYdD4E4Sq5/pxBnDCOSK4OwcICTEG2aLt6kD8qWz8GgyvL1EScK4qqiv7Pih47oFGIVABMgNppMq5WAIcOLQUHMHOTbioEwrQdk5HNNhNonz2fry/KgI6IJkDuF0ATotWKQj4iIMw8x0NMrz0be96pBOHGQa26/8GREIUA2udOFbZN7WdimqOLBbCwgB1JB/tzyPAoATgCth7wzCDWZ3qjYqUffTUS4GKFeU2YTBQiog6AUec5tNu80Qwo/2mJ8OUesAtR+kuFFGaG67N2hH1Y4gVgTZAOAP3KQnYJZs4pHWPZGZr8sZbWJlfWZtUrhSFWEuWP/HbeRkwzmTiImViskP62vUghQYuQEUOpuIpBCe1DCfEogSfoesscBbZKC2+NQ2MgS3tGlap46wi8DZMf9YCqpgFvEwgC5pYfeqOT3nGqLsg9OdyjevqiSXzewWkh4sBt2nljrFKZEmTxSwjLv13t2TJqVhJ/FMiFS+krzTHqkTLu+SeckddNKLwpzLNLnUA5j8k06dpL7XN9xYO1nMYH6lJJ94NV1MwLPaHhMZPbbZSVAPKgGymzXbupxzD5MgH7h7h6PgZuT2S61GIUdIaDLYCdX2WTPXWY6Mzs2Hk3MW3NDiatO7Kveo+yLTWFB2XeZA4j6s4jmRkwBS0kKncNuoiKjOpxM1U3Zk6d3ZHej5HOCEhiPMdUjZTWB4+TCuCRYtkuB+oZAK/uSx2N6rbMP01cTG5x93YfsrBoIOn0zecbzsoCpViiHSOXjqXoyibxmJ/8fr51YQoiy/9LXCXi67D0lu1OtCZwBQLhYroFQibRclLTk7C3Pn4n01icmapyUJjkAys7fP1c6bbOdiZL0S4/3FCKUgRAiwViuWDlkTgFMNTL75KONlAIzMfwgTCgcsE7puJhtYqXTcxCRkn2Z+pvB4SGLnStvgqJ/Nq9D2YnxBDLjTzluvfFwTfIzjmQAgyGjdrisLBFGTOnDLZlIaSODgtL7lDYCBggJH+HmiiCynljUsiwAeh9gc+/oGJJaRpaJjsxEMtiN3xWqD3AzCb1LzGJEYZ0P/bmulTBbn9JoyWz6RrIHMzGhxdZwwMjKkfJg1sUoqN6X1N2oBFlTN52zKAX0dkRodQGkQBr2jAz0iYq+yyxHVr07YFLle4ynHBzDfUZXwFquTYELiEaSITz0dSamTgwWueIk153I0RW2tnguAcCH0jsZaw3RjYh1RUYzxMJAZsksPc6J2f1GtYiwDlAKJdUX4OOH3s18H6btzX/HkZP7IrOiOGAabY7fjxPwSyxo/CZT+k3McejH/DW3Q6wSh+HXPPPX334XGMtPfjCL/80//PQ3vtz/4OM/+q3vG/AtB5aPv38c/73/7R9g5ypIEbF3BqcVE0mGoPHHbx6g1h7daPD09BYv10f4mw++xE/Wl7jrGvwb97/CP71+BCMDOmtwOd9ib5msIUREZw32o8HlYofdWKFSHkoG+CBRawfrFVZdg3k94uFihTf7JWZmxNvdAr3VaIzDSdthbw1Gp7HtakgZcLkkq1lJj8FrfPXyDFf3VjAyYNU1GK3GDx68wE9vz7HZNTg72pd1+iiwGyrsNg3OTneotIORAdfbOQFWFFjOelTK48WbEyyPupRuzvO8Xrc4P9tCyYB3qzmaxiJGgW5f4+p8hf1QwXqFWT3ibtPi7GgPJQNutzP0uwqL4w7dvsbpMRnf/VCh0g4uSMwqi9Ep/vYKg9Xo+sTwtiMGq2FHDakCnFVQKqBuLEIQ6HY1Hty7w7prsNs0mC97LJoBb2+XmM8GbLcNohc4PtnDRwHvyUI7J3G86MuxWe1a9NsaqvIwlYP3EnbQECJSfqwiTMNUYKkpX65qy6LvrsJs3iMEif2mxsXFBu9uF1gsenS9gesNHj+4wS++uIRaWvhBoT3q4ayCGxWkjlDaY9xU0DMHbyXU6xr+/gD5poI/tzAtJwPEoAhSZ47pvzJCyIjgJOrWwlkFf11DXgyYzwbs9zXcuoI56RE8EWgMAmFDf4yYeUQnoFqPYCVU5RECC8xjDlN62wDH3IZxVUP0ErFN7O1OUW7cKeBkBNYGs0db7K5nELVHdJyYQBDc97VJIP3gDTlIyrA7zcfSfTCRUu+thrnsMG4qyK0mgFxJjFcO1RtNtlbGsh7RekQvuAwJTjy0DmFUJTUZfXpMRIjWAxtDwCsjQpO+/GoPsdOcaMjy28YhriqYO5nWi2ndEWi/MnD/yg7hecvliAQQk1TbXGvYEw9hJWJ7INOeB+iVKinEvk3bkCcLIsjWn9r3JgVUShcGAL8InDhQEbKT8AuP+dUO/c+WJQk3tAnAB1FAu1lLjKdkZvN2lOCqOk6vTfJztZOco0ieczmQPbFnDnAE1XrDgSKDiwhEQxVR3UrYZTqWKdAn1xDloCUAZcIgyx55X/Kug+B0eDJi/nnFPtS9SAApy6y4LL0X6O6HkkastwJumbqHgWnwnmTjWZptj1i9pPqpz1cNTIEOhse6ea2S7zOWxNn6lv3D0goM5x7NawV7FFGtpmAvyrjThEaq3JGOFUh6J0oPcA7woswVJWQqKoLpEj6VvKNmy+eEmpMMkFNqb5YQ5+erkQDuUG6dQVnucM59lVnaqXq+himuE0g/fF6+HYY1+SoFfgmUtOUpNAolKVn1KGDY7PLrEtOc0mbHYx67HP5W3YkyeWHWEfZIFCA5Hk+hViWoLQV/1TdxColKgMHN0rnIoWBqCgtCQJH/2sW0Lc11LDLtUg+05zaLQO+w7gj+1RBLR62IcZJgj9ymLOPNEw8MyUMJocrnLOgkRVZg32mLIrlWPdCfiZLwmycYsjzYJylvvYnoT8gAd5eyVEVlGWSeDIlSQHcEm4isw8nHTYRYEppzim35G2kiraH3WY2sRMlJuUWinBJxMwiUltUwQTFEihPPXN6h7FiOqW8YB3+n12dp+BTgJVLNUEyfHaEwsznkx7VkUF2rketvdEeQ6eYKeufL59lhABgiwbJZuxToxPClvEy9pwQkVARmWZ4rBw85eviZgd6OgBCU8CYQGioFOfgC5OTg4OcVZOcgR1e6PLNENBoFkcCqGB2ilJTeHqS7CusRWgO57lgfkvo3fylVNafCZpYSYIgQMlCW5f/3wF6WwyqJklR7eDsEgiEgZmB6mA57sKzofdkOoRQTXb8hg31/+X8FJomBzOhfg1TYT34wi//1//HZb3y5/9Enf/hb3zfgWw4sT757L/6b/8t/gje7Bb579ho/W11gbkbsbAUfBRbViKt2gz958wD7fQ1TOSgVUGmHi9keP319gauzNV5dHyPsNM4erjBYg901ewTU3EEbVyZamspis2uK9eB4ucd6M4MQEW5UUCZAaU9WstcwrYUbNIQOiE5CVh5+ZzjIFhFSxzINrbXHcNNCzi1MzTfN2BM0REtmD6OEWlrIL1u4hwOwNmT6vEB90tPL2LoCUBDIQqJTZPJcYuwEIGQEthqx9dCtg1QB9rqFOBohJICXNcLlCPGuQjyziIMEqgB5Z6C39Mz1DxyrQXSEOB0RRsWBvk4AwElIHWC+aDBcOQKFUZK9rCJgAvQ7A39vRHT8tFczB/lVw4HVLKC6VhjuOcheTgmYVYTaSYQmQm8otYwNj4NeK4hne9g70iLVjYI9CYgN60fo5QPkKFHdSozHlBmqjkmSoQ0EBT2rS/wRPaXCSSAAeidhTzxgAtSNQajIduobzb91RPtSIdQolSH0huWBNBmh6lZS8pkGf+6IX756w9Tf7DuMhoNYdkFKMlECxRNnjwlucvKqWXM/IID2tcD+QYDZSthFQHUn0V8GqJ6sZV5/VMDRT4HNB2lQkLbVzTjQHi88zC19p2ZFqZfu0qB9L9DcCOyvODAzaaAfDNnF8YjshepFGTDbI76f8mDLN+xurd8RJNpFSF5OsngZPNgjAopo0ralbll63zh4VyPZqcz2ZTAjHDB7GbH+BLx+PUpQT2FEGy5T75EGGhyg2wUQdES1EYWxHI9ZR+JrDlLywN/seOx3j+lRrd9x/VmyWa1F8ekBKBUReVDbn5ElPLzpDiXciJ5MpAFnAgvpOFYrDr5z5UdOGy7sVgIFqufzXIvkf0vyxTTYz2wTkBhJrYB2AAAgAElEQVS4BEDMBqVGotpE2EVmOydWrr6L6C5EOucEeMO5KFUoZpuY7yRPzRJa1m8kgKqZTBsVt9FsY/JLHlSLHHgv2ZM5sW5uJlCtyKBFjeJDDMkrmN+Hdpk8dwngENRFDKcM2qnvyD6JkNYhCUr6M7I1amTYkLRk7zLzGQxlq9LhoLaD+6N6TjB4I1Jlw8T2UX4KtNdpMkJOTKUa0t9poqCkkyeGXPgElEauq9py22XyUje3AftLWQbTauTvfG3M3nrsrhSiQuleFYGeQjunT013BCTdOT9v6k0sjJ9dMEVYBL6vXCvQ3AXsLyTaG7JyGYiqkWFTvmKwTrVLgLDOPkleV9WGnZ96iClYCBiWfL5tBXtEJaW1UfDc+CqFEx0sWw38n9eLKOm0ZCiZ3DsuZWFDAYIYVqCEsky9D+hPNcw+kDFrGH5k59z37IvUfYCdK0piU3prUFye7iPMzsO1fG3xH2K6ngmKA3wrUb+zlJKmVOL8nPzakEBQTo+OKQU4g62SuivoS4ya4MQ3srC3epfYwdS1mr2buvOlYqQE7wTALgnQYvK+FrBbpdflepF0vJkSi8JG1neW1SR7Jp6GigAt1KrsA7+jPXyj30uqjUamCSSHKChJ1ZuxVIhkL6UcfZGwytEjpK5NmQBY8VHKDGRTSmrxZEZEJeFnBmpvU4L4xGYKFyC6Ee58zt7N0aXvtEiwaD0lrS5AjBaxSn2dB5UjoTWQvQOGkZUfwMQe5pTZDBC1oucyxqlWRKkJGJYL6GA8LwT7LSuGIcXKcB/6kdLXwy7K/NpvJsjGSJbTuSJ9jcOBvzEmn2eIBJYh8rnZtwkQWOblZ4/lr2Ap/3l8lt92j+Vfd2D5uyPK/Re4daPBj/7iCe7WM/zh60e43XE0tOlrvPryHF/fnOAf/5PPsP3ZMaQK6F/N0XUV9n2Nn72+wP2zNV58dY4wKFTXGjdvj7B7O4M5GgATEG4qBC9xvtxh6Aw2PzlBuK2h/nIGv9e4eXEM12m4USEOCv62xrCrEJ+3kMbDe4nZUY/oBZrFAPlVyxCZyiPuNZd/U8N8PsOwrclAvGow3LQY384I6pwErISuCZz81pSBhTwfICveP9y0lNNZCfmygdhpzH5SQd4agtLUs1kv+ZroBGWeAnDrCv4XMzI6ApC/aDjQ6hXCsSNrJAAMCnJgl2P/yGL+BWcF9VYidBryTqN5o4EkTUWvsPx/2pQuS+mxXFNq2rzUkGtNALEyUCsNuVPwI784wixAbyTscfqCrEOa/Q+Y/0LBnTqoPUFl/U5CjBJ6oyiPfN0ChiE84zmTV/VbA7WXWPxUY/YLDb0RaK6BWEeyYz6HhRDs56CY5muD9iuDKCiPNSuCdX1tEFqCNdEpqEGgupMwKcVXbyl7rd9JshkR8AuPYID5VxK+icXb174WMLcS5k7CHQU012Tclj/j8kKVQnokGZPmmgP35ppA+vjH9D/qrcBwz5M52qeOy1sJu+QxnL+IaF/JlD4c0VwLQIK+04q9lqxRSQE7noPO5ecazbVA80pBb6eKl6DJJJk1ZZTSA2ZNoDh7KVCtgPnzPCvORFJ7RJA2XLA8fv41k5DbVzL1UAKLLyXG01CkcNIi1XzwOqxW6QNARNS3lHFKn6SmzcR2Mfqf5yJUcerynMVS2wLB2ptqhcJouTn3u7mOaN8wHdi3EXKgD1J4YPELLjcqENDvMvihJ89sJGYv6N0TATj+CdC85TU2e800X71nKE/7NmL+nIOvkx8TaGVWJiaGQ++B5VcRy69DCbXJEwjDCQf49d1UDzJ7FWE2ETqxM8ITgJptRH9JwFWvgOVXlARHSS8m2bNYAI6bcdsIFoD2DffTzRJ4sihJvEc/97DzPIGSGKLEts1ehwKc7JIg2C6YyBySlNIuU8CNYrVIf47SbRnU1B2pRm5DfZeOkyCbksNq7Dz5Nudg7Yrga6tNCunR2QvIc9BdccAfBVCvODht3hHs1XcxVYUk2WlKJa02vA7qVURzFwqIrzY8t3bBxNzZm4DjLy1m1wFqoGyzfUdvqdmmmhDLbW9uA6p1xP4qh9OgMG9RcNntjYfpIuavHNobD98QsATNazBvS1AC7U1I9S8Mp6k2MYXqCLgGqHaUGc5f+8Ke6T2vtxw+NH/toDsCkCxTXn7t4BuBYclBuUqBM1EAZhtQr0OaBBA4+gWTWhnKE+AbMla6J4BcvHKobx1cIzB/7VJwD/e7O5fls6Nekz3KQUdmH1GtfWGqRcyS11gSavP2+lqWc54ZMLMPqFc+Sd9lmhzgscr1O0c/79G8s6jWAaoLGI4V6juH/oT+PzuXaK5HAtqavsgsUZ29HsgKKlHYO2UjzMbBLhR0F9DcjBiPFFTn4WsBs3EF6Oidg96S6RpONVQXIC0Zx3GhiscREahWll2TIaK+HmC2BH16z/MatEB9mzIdNhahkpg93/P9KQCztQhGwKxH6M2I6naEWVsCttRDqfcOkALjsSZA7jx07wsoVp2DmytUqxGqJ2BRe8fOys7BrEeE1EnpWo3m+RZurqFvdnCtIvgzEqqzUDsL1yqo7UDGsFHlOSJEmLsebm6gdgOkCym8jPJTANA3O7KGOZRH8HPSzTXEQLmqbzRBnRBQnYVc7SH3I5k8T1YwKvZsApiqTSoyitHQhyhcgOws169UAWUiRojRMbinMhDDCLndA6MFLJlIuekJFLNkdrR4r8NSSv6fl7nvk28x2X1yXYkQBJojvZYxg8YQEBczgtVhhHC8nqK1XFfuotSaP9lDmf6PIUwyVoC/vSeAdA4lsKfrv/F4QLQOMR6wlZgAZrTuPX9ldI4MpPfl91/1w9d+Q7r7LbxFCHjI3/jP78rtW81Yzr7zIP7wf/pP8eL6BBenG1gvsfrxGfBgwNX5Cq/eHuM//P4f4o9vH+HF//UU3/8Hf4E/f3uFzfMjREVws/ybb7HrK3x2+QY/ubkAAMR/dIp7f/drLM2AP/rRM7QvNOLvb1AZh7ay+PT0Df7vH32K03sbrDczSBlgKocQJIbOYL7ssf/iCOHEof2iQvfBiOYXFc7+1iu8/MtL1NcK9iggXIzQlcfl6QYvXp4CAJNmew0MCk8+fIuXf3Qf7sShfmngvrOHFBF2VdNTl+QrYe4ht4p+vpVGPB8RvcD3PniJHz+/QtgY6JVCqCNmLyT2/2oHvK3pf2soq1NHFuGW9I4+77GY91j/+Axmy+f4hmDLPia7ePSnFdq/8wavX5wATnJ7zkbEvYbcE+iM5x4fffoKP//Th4gqon6nIH+wgv38CMc/eIfrl8cQA4FknNEDOf+5xu6Zg9pL+FOH5ssqJU5G+I96qJ81EJ9tIX60xPBxj9mfNdg9c2TUHg3AXQVxNqD+vEV/6dG+UoiSoSYiAmHpMD/tsFs1uH//Dv3/eYXNRwHhwkJem5LymkEJPtxjMe+x/8MzjCcBOLGY/3GD7XdHzH5aoXvsoddkGePcQV8buCMPfTLC/GgG8cMVuucLAMDiS4X9w4DHP3iF5//kAfyCfkVxr4e/qwAV0bwwGE8CzFrCfszz1LyR2H9ooW81+1DPB8RXDXBvgP6ygfjOFu6rOUITsfwJzYfrzxxmX2nsnzjMv9QICug+sBC1h35ew6wF9t8ZUT03UIPAcBEoL1xQnuobytNEBMbf38G95WRD/ZbXbtTA7IXE7lEATnneFz/V2D3hMe8e+uQtlJh9LdHfi6huCT7tHGjfsIpiPAk4+qnE5sOA5o1Efy8gzDyOf2Sw+oHF7AtDqdxawM6Bas2aFb2j5HJ4PGLx5xXcnFU2sxeyACk3R2KRKH/sHjs0LzR0B+yeek4UJCAdFdC+EgXQBQ3sn1mojcLJjznYW30CLH4hsH8QYY8CRBA4+lyiv6QcUm8EzJ4Aun0lsP6+xckfGqw/DpwoWSu4hYcIgp8By4D5LyTW33UwdwqzFwKr3x8h9gqLL1SRj/aXZGrt729hNzXqFwb13YE/LLF7q399wPH/WyNKYP1dD9kLqE6geZcAwEDP4ew1y993jyLslUX9tYHeCTTXlCCOx0B1B6y/63DvHytc/zBi+VMOCrdPgfod/WXdPbK7IUkst9+xOPmnrFDoL8hQ2zkZ0P4ion0tirRyPOGkQncpJsb1jjUo1Zqg8ugL+t1YLwMMp0Bzzc/+HNxz8pfA9jGZ6fZVRH8uMH/B5NT2XcD6QwnVJQmpJECtbyO6K4H2Dbfz7M887j5RqNYRq08jFj+XcHOC87vPOCGQWWMkWW8JBQqU0FYrHmOAEtajnxJIrD+QGI/J5DVv+dkyHnHfdg8F5i/5+/Rzj5vPFOq7iPZdxO2nEouvCfCDEqUGZjgPOPsTge0TTg6c/6nH9gG33bVkYbMccn81HQs1ROwf8Nguv/YYjhh6U60jNh8KzL+OhXmcvwrYPpKchLgQaF9H6NTD2Z/zfXj6OQe324cMHmtuA4Yj1rcEBZz/mUV/qrD+QOL0c3oEfQ0sXhKYMTyIIUaIwOJ5xP/P3pv0XJol6EHPmd7pjt8c8UVEZmRmDVljtxsDbVkYIXfDyiskhNoWSFgWXiAWLAGxQBawAYRYmAUL/gBu75BY2KjBaqqru+2uLtecQ0RGRsQ33/mdzsDiOee9X1RlqxurrKps9d18w733ve98z3OeafmeQPUqoLry6MYSxZLvaw5YN5MvwsAcBwVsHktMnvuB/cwXDrsTTVZwR6knfZjx3D9ggmtf8Ti0cwkTWddiQeawnXHiIF95bB6qmK7Nyp/5T3os3jOYfmK5fncOu2NGaZfXDMBRXUA75T0o24TBmysCkC0ddmcao1c96hONdipx9P0G24cZihgKVN5wErCZUYJuC4GDH7WwI0WQt/XQtUM/0aiPFIqFQzNTyFd83/qRQnnjkS8cPY41J0q2DzTKW4e+kihuLHZnBqOX3RCeU1122D6kysfmAqYm4z153sFWCt1Uwmw9ylc17CSG6fgA1QSyvQAmH++wfVLBFgLV6x71qUG2cgSTlUT1YgdXaPhMYf1WhvmPa6yeFjj4owW2702RLS26mYbsAoqrGut3RihuIyjNFLqZgVlbtEcG1bMtmvMK2W2Hbp7BbCzUpoOd5ejHGvlty5Cc6KkUzkNfLNG9dQjRe6hdj+6ohN72cIWGrThZkN0RzLrSQNU9RGMRcoXBQykl3IiyVzvLoVYdn/cBctcj5AqidXCzArLpASmhrpawpzPKXJ2H6B3kukGocoi6gx/nkJsWocwgts1ethoZxaAVK0sOJlC3K/jJKPolJUTdDs+HzOylrjHAR0Swyd7OyHQCBJhSQNTtm57K+x2eKZE2Abl7XZsIkZnUXLbfbCHyLCbNRhbTOUpXnUPoOoiyHHyaw2f8WR6fwW76pvmzvfczHr8MjOV73xiF/+4fvv9zX+6/94U//IVvG/A5B5b520/Cg//iP4W5i2mcUX7XzzxUTblOP3fIbhSyhcDmCwQgZkV2w6yB+ozv6x9wgNUeO1QvNLp5gNdkw3TNwaLPKU0MhUP2yiBo9i7uHjJwhHI/MkX5rYArgMnHAeun/EJfv2eR3SjonRg69oKMNQmvJZpDzsTLNprys1STgCH2X/QY6gAADi5TXUDQey+PywEb5Y/JB8MvWMrtstXeP3G/jD2Z1VO1gq75JW4LBk5sngSYrYBZccAre0Q/EZe/e8ti/KGGWXMgBBEGVjFbcqAZFAFFfiNjNyW9OWbDfZKCP9KALb/jOm7eIsvWjxmwYUsOgJtT/rQxOEPaxIikGVkGZCQfFX08As2Zx+QDye2I4SrZKkkOyVjIngClvOSgNUgyOd1UQNUc7JbXfC4VpfcTngv5HbD8Ro/p9w0HtAvuE59R6pqi7H3Gz7UlB7PNIdkFs2Wf3fhFwOZR7M+Lnj29FajPLaY/0tj+qzVG3y7Jhr706CaCcfbRB5U8UPUJAVd5FWIKIqWbZkOWxGxi6bqJz62j5FByED36NO6TjimWk485KE77vLrg9ultOkcpS3QZfWdmK1C95CBN1ayU6KcBsx/vPUf9hCE08x9wMCks2VTVEmTkd3x/tgoxZZKD9GGQP+cANIVksD+U29SPeV4A7D4sL0L0tTFVMl8EwFNyqeuA7aM4KK0ps+smAsUNz6V2RsZP1WSb2nkECQ95zlSXMczER6+cSxI2MlDtEVBccd/bKspPiygd82SMU7Lr+m2B6jVBWGJss3WUDxZkqlwmsHvIgXmQ3L7kD8sXUQoXGVoAg9+rn5BFdQUw+8ixl/BEoIrn++S5QzOXyNcB3Zgdltk6DKxT2i7ZMXE1JZbuTiQ7NTOG4HRTAbPeBxx5Q5Bx8xUD1VJqWiz4PlZaCExeWHQjCVsKrN8GJh8D2ZbHwWzpd8tWvJ67sUC+DHsg0RHYbc8kzCauR8v3lNcefSUH2W2+CqiPJIpbj9VTidGrAFuQjSQDxaCRvhJRVhqwO2X9QToPpY1hKyOC7Wzr4/FhImmQBBq7Uwm9C6iuPZq5RHXtUB8qjD8l2NANGVyCJjJOKaHVKwI/rj/vB9W1HySSANBNCCZk9Ce6jNdRNxHIF36QelJmK+I6R2bQ8ftH10lKyqRR3QRkG4cgRPQVJpbaYXumka8pE+0mZFuCBMbPanTzDOsnGsUdwZ8tJYpbS5lpIZFCX5q5QnVl0U4VVAybSZLtIMHORYFhHdgZSbBUXdn9Odh6QIro64syTimYQrp1ZPAMJXr9RA+eTJcL5HcWrpBwOSdsdb2X2Zqth8/I0LucUs7myCC/5XuEowy2H2uYjUVzaJCt3VC9kTosdW3RjzX0zqE5NDBbD9U4dHMNs3LoJwrZiqzl8mmG8o7Sy+KqRXOaQ7YBunYwiwb9QQFbKKgUYmMDZM9+TRHIFvKgCsjewY7MIFkVNkR2kIyaHRmCtygXdbmErh1sSQbS5Qp2pCN4axG0RD8vYFYd+kkG2TlACehFg/ZsBJdJ5LctuhkBXz/VyK8avi7ErslxBr1s0TyoUH6yRn9cQW97BEWPol41cOOc9R8uwBca3ijodUuWVAh4o6C2Ley0gF41vNeOM9aEdHYfhCPpQRR1CzcbMXAn9lCqNZlcN2LSrYhsq68M1OZeQExv4WcVJa8e7LtUCqLt2BuZG4hdi1AYyNUOflpBtBa+yiG3HKyFMiPz2VsCwQQIxxXEth7+ZgrknulDZshGlkyGDVUBsWsQioyy1gT2gD8ZrAlBcGfMz7wm+RUHGWry5yrJ56KvcfBbJhZVazKfAAGjZP8ljEZo2jdAaXAOIssIMn/Kg/lnftxLg/28eyzf+8Yo/De//dWf+3L//S/+/i9824DPObA8++ph+G//4VfwrD3Gx/URvjH+FLd2hNoZfH/1AIf5Du+NrnBqVvj7P/hr+Hff+2cAgG/fvo1Nl6MyHQplUekOf2n2HJfdFH+8OMdvnX8Lf/+jfxO/9dbv44PmBB9ujvGbx9/HP775Eq7rMf7WW9/Cq26Of7p4gkp3GOkOF80Ej6sFPlgd49PlDP/6+TN8vDnExLTQ0uHZ8hAPJyv8+PIEh5MtHo2X2PQ5amvwm2c/wP/5+iswyqF3CuOsxa/OX6D1Gj9enyJTFt95eY6vP3yFZVfim/NP8Wx3iHdGN/jdy3fw5fklvvXybZxN17hYTfDNs5d4vj7A1w5fofMatTP44O4YR9UWy7bAOOuw6TKcVFtIBNw2Fc7H1Bf+8PoURjv8x+/+3/h7v/M38IUvvMbz6wMczza4WoxRFj2kCHg0W0ILj7FpMdIt/vGHX8Tf/Mrv43/71l/F1770Aj96fYJHR0s4L/G33voW/pcf/xsY5x3emtzhn3z3i3j76RWOyw2erw7QW4WvHF/g2foAr69mAICvvfUKH94cwXsBIQK08kNw0ulkAyMdjosNvvX8KYIXeP/8AoumRGM1FusSk1GDR9MVVm2By9UYTw4WuNqO8NXjC3z/5hSPpitk0uKPPnmMo/kG86LGi8Ucs6rG1YKdAEXZYf1qgvyoRggCs3GNQltcLCbIMotMW4yyHi9eHeL8wR0WuxJNncFtNH7jV7+H33/9BEIEfPX4An/48jHq2xJ/6f2PUSiLm2aEkWnxo+tTPJyt8Hi0wHeuHtLjezHC6GyLx/MFfvjsAUxhoY3DwXiH6+UY46rBal1hPGqweDVFdtCgyHvkxsJ5gdtP5vhXvvEhni0P4QOwWI7gdxrT0w3KrMfduoLWDlp5WCfR1Bn+2hd+gm+/fAt9rzCuGuyaHH2n4RqF6dEWubG4/ugQIfd48tY1Xl7P4RqFw9MVTkZb/OT1CSt2DteoO4Nx0WK5K9E2BuNRg822gN0YTE436DrOpnaNwXy+xeJuxATjnYbQYUgLnh9vsLgdoZy0qNc5pPYYTRrMywYvfnQKddTivQdX+OEH5xC5w+nxClc3E/itgSgtpA7wTiDsNJB5fO3dT/HPf/AEonA4OFpjuRrBdRIq83CdhNhphNxhdFhDSo8QBM6nK/zkO49Zg/I2/5/nFptFieOTNa4/nUGNLSbjGovXE8iRRTVqsf10gre+dIHnPzrDV7/+HN979hChUzg4W8F5idVdhaePr/H8jx/i5P1r3CzGcLc5yocbzEc1Xn56CLnU8GPHTtiVwem7N1huS7S1AVYG2ekO7V0B0TDMqTjfor6qAE2ALEcW4S5jd6wI7J0VgG8U8mmL7qLC2ReucXVL06t8VsBOPPRxA/+iAs4bqI8KuHcauKWhT7uTyI9rOKvgOoXQKMg4iZe/s8buakQVggoIucfbb1/h+Y/PEHIHfW3gRkwddjFdWDjw99hz6yuH6iOD+isNxG3GicJ5TKQxAerGwE0c32sp/+4PPIIIEL1kONUnGfqZR3Gl0B4ReLkJfdBuwlRLfVJDfDBCd+BQfaKx+1ILET3naqGHSQk3cyjmDdrLaBWIMngAcCW3xWw4ySN6Cb2Lfakdq2raQ+57nweYFQFcd9ajfJaRiV8KbJ9aVM80+lmsJVrQAqAaMQTmpIRhPK4hPyphS/qC81uB3TnTgO2I/lxb8vPgOdEpLH3dQQCQZOa7GWtrZM8JSLWjv1lvOEFy+/XY87rkxKqwQHsU0I/J/pcXYkjg7Q49Rs8ltxVcV7NmirO0YkimRfSKdjPEfcQJDWkpc+8n9DhXr5m6q6I31uWALbltaV8gyqpVx0mN1EfaHnLCsbzia5rj+JyNk5erJMfmJGzqltVb7ovF+wxumjz3WL4ruQ6Cnu7RS5CJX3Mb8lugPQTyOwwy75TYe79/spsD40/ICLuC/lNdcwJRddEvvQvR80tWnaE/AdtzYPKM+64+pdogTSav3uVkSz8m+54Cicw2RAk0mdfmQL4xsSPc3oNtdgG7YzK57XSfnhsEg5aKGzLhlFfTVyv7gPpYDmFUq3cFJh8FZBtOSLRzTvIAQDsTKO44KVPeOlZ1ZCJOBHGCp7q2qA81dMPJiWztIS3QHEi4jAy77D3auYZ0IeYNONQnGsWdQzdW0E28zmNXaYgSZJftE3OzRY/mOIt+dAtXqBjOs+8LHby3scczCMp+vRbI7lr08xwuk4OPNFu0sCMz+HWDAPLLGt1RMUxc6G0PV2l4JSEjUyijjFmEQCC9buFLw2CeXEN2jp7QXYeQaQLeCPxTlYm628GPcwzJszoyjh5ATLiF9QPjmqS3flJA7hg0JJpuYDd58fo3QaoQBM9KDZ7MkEe/6LYGipzS3rqGMGYPTBPwU3IAnz/dk7nvwfyMgKDPwiXxdSGEz30q7LvfGIW/99tf/7kv929+8fd+4dsGfM6BZf70cXj8n/xncIUHphahUSiOajQ3JdRGIpy1kK8KyA7oTizMHZMcAQCZh6gVZOzNy29YHdHP2QW5fT5FfivRvN0BjYReUUqqt4L+tSP6U2QrOHALuFcQL+F1QD/zrHS4MrATj+qFQv2AN0DR78umi2uB+sxDxoGSHcd0RM/uPlVLuJyDEBZZ83PciHI+Dr48shsVZZRkOcfPBXaPuG6prNlNLbIrzYCPIkT/477+wY4DigvOenZzSmCDwFASD9ATuH0cMHopUJ+SVd09cpS69gw2AQCzobzO5QJ1DHdJgTLlpRwYRg66yES6Igzsrdnw2LicX6T5rUBzHJDfMCGyfC2HAYJL2xIY8gKQ1eun6UuF+yQlBTLcJmD5RW6brmOYi+KgJ/U/lpf0LzVHHFDlC/brmRWPgdkyPbK4JpPXTck+qjZg/Q5QvRJDWmM35XKLmz2bmsJL2gO+n6waWdL8loONbh4oo4teIr3jwEvvyByWl/feP6V3M7+LYSnbyIpPySi6nFK2JOmzFdnF/Cb6Axugne8Z1H5Cv6TwYehSS8l/3YTrOrrw2D6U6KYcQG0fRWmdALoZ95/sYmplZFDbA7LOk08crn5Forzi8TFr+qjWb9OXWJ9QdpnCN7whQ5dkmOVVwOYxB4jVa/oHXayOAGIYTRy0Va/JqAYJzH/i0BxKNIdkamUfZZl2P1hNrODujCqE+Q+B7TkHcWbHQePulOs1/tTTXyji9s0Z1NKPebxVH31fjgXwqRph9VTg+LsOzgjUJxJ5LFvfnLNrz2f0cqo2YHRJKVtzmJiUOHCdcOBcXFO6KPv9fhSe54E3YgjB2Z2S1WU6JFkhWwg0JwLZgtu2O5WYf2hx81WN6Uce9TFZkXzJY9GPyTontYS0DICpjxTlrMswBKrYgtK67ZkcOh/zlcf2XGL0kgEtquM+K689NucSkxcMiYEAqksf0yeB3bGE2ZGpzVcc4Jp1QHVtsXxqUNx67E4lyhuuc3FLr6LZ8Xz0isx4tvVopxLlrcPybY3RhY8+VY/mQFHSOBMYXTrYXA5Jt7phGqjq9myt2XnYPFb0RF9fPyJjXl15smGZRHOooBt6+Oojyhd3xxKjC4f1I43xK3pU26nE/IOOvYCBTGsKgpHdvqIBQmD9SN9LHw3Il8hQOk0AACAASURBVKx56MYC5Y1DO1WUhWqBbMPQmOZAkiG+dWgOyBZ6JdCP6RGkh9TFzsCAdq7QjQTydYDeeZitxeLdHLoJKO7o02/mCmYXoBp6AbfnGcorC9URECSG2xa8xoqbHnasUFy08IVCc2iQL/gFs31oIjjiPS9b8zO8EUNAjOo88rse9Uk2pJL2I4nilq9tZ5L7ugkxIIbex5S8ly0thA9oDwxDcOLzuvZQrUN9kqF61Q7VDbK1aE8KZLcddo8KmA1B0Ohli35i4GLSKROM6QV0uYJqybK6nMBd1QzlUbWHWXdoTgpKW0caxXWD9iiHajy7GBUrMnaPx8jWPeABWym4gqy0WbQMl2ks7MhAWg+9bGEnORnJ1qEfk5nNLraw8wKyc+inGfKrHbqjCj6TKF5v0R6XyJYdGTWAUtBdh/6ghLAequ4RlER3QCCTX+0AKYdt9Eaim2UorgleXGnYC1lGEOI8+mkG1XvABehFg/64Qna5Qf1kiuL1Fm6UQa1aeiBHBnrRIBQabsRl6JsafpxBLWvYg4rsaW4gNw38uCCYbGzsdSSLCOsHwOWqDGrTMWUVIPsoJcSuHZhCNxtBeE8fZU52L0TGE5EpFXWHUBiIbZSztjHl1EQGz+h9CE8IQJ7RH+k95arWMUjHuSGAJyjWk4Qig2jvJbdqRfAXZa/Is33ozv1Qn+ixhNYEcF1PMJgZeiDjeiB6LEUCj0nW+plAjoE8whgyhKmLMgFFJQFLv6RI2+Pf9FXel9mG3v4MuOR2fAawfOPpz1q3f3Gf5V8Ay3/5j881sCzOn4Tz/+HvIlwUUG38ElpQAtgeeYw+pa/DjsLgJzQrDiSTdDMlSDJ0Q8QZ3ehB+td2GP1eNQyKXQ50B55exRe8yJIMMA3g8gXlovkt+9Py2zgIroHVewRD2YrLtwUH6P04YPQC6KccFGcryl6Bfdrk6JXH3fuCMfgxFbK6CLGom4PJfsIQDZdz0Lp7SGAUJGecB2mgJaDV2zgwNCJGw3NGM8k687s4eBKcCWSCJ/eZ2RB0FFcC7REwfhbQzQX6itLQ1E929ysOxSuN4oayv/qMYHPzWKC8ipLSGE8vPDB67bF+Qn8PAw3CUEreHHJQv30kMPvAY/NEoryM0rgod6WMKURfFwfKSXoX1N5DFySw/lqH0//LoDmMVQBx8FneBKwfSxQ3lHpCELR0E4F+Csw+8DF2ngNc1XIg2R4RkKRKgNmHHrszyf9llCR6tZ/1BsAZ01H0t404i7s740x5iv8vrzy2D+QgY3M5B+7NMUHL6h2B8Qs+lxIddzF6vj3gzK3eAYsvM3TGsCkG3VSgiJ6ezbkc3qt3AduHEtNnbliWHfE4qYbAozmSqC4cdicK7QGln8Wdx/qxQnnlsXsgB8/ZALrAfSM7YHRpsXpLY/cQOP0Dh92pQnHr0RxK9GOB2Yd8Pl/se+0AyvWaI4nRaw+vgN0DrqfXAqu3JSafJI8If/hYLm+2PH+K2wBdE7R4IzB6TYCJwBCTvmJBeZAMqOknwPF3KBnbnNMD1cxlBIYBxR0HrrszenRSAml6Xb4mSFNtYPiMZvE9BK+v6prbKRxBXgo6KSLABChZDJLb6jJg9qGL25Z6/Agc1o80Zh/3sJXE9kwReEZQxXRLib4kc9CPOEu/+ILE+BN66MzaoZspni+1x/aBxvTjDosvZKiuGMjSHAj+HoM/UkJrkPSsZauA6sqiPmLVSfL3BSVQ3lAG6Y2AzQUmz1vsHmRDjYDZeuxONMobi80jjflPWthCoR9L9JWE6gPyBeWXxZKAtLroUZ8Y1IcSs2c96iON4pZhKMVNj27GfrpmxiqVbMOQk76SyJcO7Uxh8nGN5pRdvLsThfFLC1tKZEuLzaMM2dYPILIbS1QXlECmbbNllAV7DGxHfheBy9ygPuL1XN5wuS4TKG752eV1j/ZAo3zdYvOkQHFrKZU8yaB3njaEUqK46bE5z3jtb6LEdxsw+6hFP9ID8GYQDc/JbqqQ31n0ExXXk88V1x2CFqhPMpgN10N1gbUQAPpKxuAbnl/lqwaQAv3EoDnk9974kzam9qoY/OXgCol+JGFLidkHNbq5gS0l8jsLaT2awwzFNUNc+qlCtrDYPuTxH3/aoZ8ogqDbDiGTEJ1HyCRsqeM5QjDlcw2XSzTHBuVlD9U6uELBLFt0hwW6qcL44y1caZim6gMZnZRQKoB+YmiTWFuGyNQWamfRnBQcW1zWWH5xjOKW0lRX8HxoDzRGn+zQTzOYVYfmrES26qF2Fj5jeIvPFbwSMBsChmQvEZ7P6U0POzboR/QCulLD3DZozisUr3eAEGjOSk5AW4/spiHrFO9/ou0RcoPmpEB+26KfZsjuWngj4UqmpArn4TMdAaFAc1ohv2spJ1016A9L6FUL2Vj0hxX0skZ/WMXqC9CPrCXy1xu4UTYAWLnpIJyDm5XD+ngjKd+/2cCeTBCMhF623B+pisRI6Ks1AZB16M9nMJcbuEkBfb2GPZ5AbTvYGcGvXNXoz6bQqwai5mywmxNM2oMK+mrNvxc7+GkJ0VgCtDKDG+fQi5qgprcD0BNdD38woY8xBIRxCdFZ+NLQ/2g95LYFrEPIM8hdQ09iBI1ivdunt4ZAaepqC2SGMtboZxSWwT8ACBybdkiFHVjAzW6QuCbWD0YzxTX5E+MjeE8gmGcI2x3EqELYNRBa8TljCBazjJJUHyCKnOtY1xAlj1VoGnoih8qPALTtHjAm0JeAnrgXHpQ6MwGuJ4BQNxBlQcayI6splBpksUMCLIDQx6RY4GdBbAKOnwU69zvhzT8/51LYd74xDv/1P/j5A8v/4Evf+oVvG/A5B5aTLz8I3/if/0M8GK1Q6Q6vdjPkymLVFmisxnG1xcc3hyiyHo9nS/zg1SlO5hsAQKEtdr1B22vkxuLiaobRpEGZUerZ9hqbbYEs7zEqOqy2BSZVi22ToV4UKGYtvBcYlS06qxEC0DYZ8qJD2zAjPs97TMoW14sxZpMdbi+myKctlOJF0uwyKOPQbzKMDmvs1jl05tjPuOGNaHq6QV1nEDLAWUre3CpDdbJF1xpo49A1mt2HVkIIoG80hArsOCwcTGEBEWAMbwy7RQldWgSAia8AfM+by2heY3tXAr1EdsCOxqODDW4XY+RFh92yhNgpyIMO4TKHHztAe0wOdlhfjwArIUo7bId/XTAV9ekWCILBQ+MefmOQegPVrINbG4jCoRh1qO/KmGQrIAoHZTxcLxFqDTnqEW5z6NMa/V0O6ADRykHuJzoJddQiBMGuRRkgcg+sNULhIWoJKCBIVorgpIXfsPsw5AFi1rEj0Qn4TkGuNb0bUwtYCbWRcIcWiHJN0UqEykEu9eAfk72AbIHuxJHp1gF+xJl0uVFDnUcwAa70MEvFqhKFodfQVw5qw/j/MO8hbs0gK7vfh9ad9cg/zdAdO8idhBs5QGKQJ3qDIe03u1YMA6liB2Iv4CtKZ/IrhX7CwbErAnQtYCsPKLDv0AN24iFrShaFY9WJWSmYJSV/vvTIrxTaI4fsTjFwqcDQrZdCdLwBXOGhGobn1I8s5E5G3zMlVt2RQ3YryTK3QIhgHeCkSnfkobaSbPlj1tFkC8oBk7RSWrLxTGsNKC8k6ge89opLst12xGAc1Qm0h57namRV9ZaArj3y8EVA8ZpdhmZNfy19zBzQlxdkNVVDOVlzFFUHIJBmvP2+4y8F77RHHsUVmbluygmpoPl7KrnPFmJgUb1hXYvL9x2IQe/Dh+oH/D9rJ/bsssuj19ZFr3acDEuhNKqhj1w1TEvePqIiYfM4+pqnlF6azX7CI3nEk3Qvv+NEhq1YzcLKgJjO2lABIfs4+dMA3YSMf/IIuowTVs0JJ8S6KfdZtsDAdCbpYT8B9JZMvOrJ6rYH9L22B/TF2jF9wt5w3frRvmdPxyRP2VPOWFyHCPgicx297HrLyQBIIHUgpjoVIF2HBM7eYJB99lTTc5IteltTN6LqyHxXr8Mw+dUcRRYZ9L2WF/QgAxiUItKy/idJE9MkYOp2FJ7MNrD3CadaE4Q4CaA5YaLi5Fs/jkx2TFK1IypTzC7WJMVuyn4soLcBuuF6NAcEqbrh5ybfq7Rkyuvj1KlI4O0KMvhpvbJNGKpDpA1oJxLZlqCvOdz7fimTjUnERazqiZOQ+ZKsYTr/XMHzR/UB7ZT7TjcYajWEj95rQa8uwn4S1hb8f7Hk/aGdRkmmQ+yZDOhGMkpLyQLbghOB3TimzsZKDtUmGSWGYCF6B/eVKGnyJiXE9hXDcdL6JJ+p2VjUxwa65mSBreTgjdVbMoXCBficxy15I9Mksq14nLKl5aSDC+gmir5Rw0oWTj5omLUdsha8FtBbOwBw1Xp4w0kDr4H8zsJnnJzLlpww6mYaxW0f5aUaetPDZwo+Yz2HHSved1yAWffopwb5bYv6rEB+28dOSjKDttT0feZq8I2m5TEdV0FvbfR+9rAVQ8Nk58maKcFakHtA2eVM4E29nVD8fhDWMyFWkqkVAZCtjT7OmNLqw9BFKVz0bXaxkzLVjUQw5QszvB+eabHCe4gmAs/AyhK4AEhAtBYh16wyUYIMqFZDNUz6W9Tt4Kt8o4fyflfl/SAegIDXxqTWe4E+vHnxGvgT60ZUBNVKUdoaAegboC7VjGjN5YRAoPnT1SHevwGW+XGfATDv93D+SY8/Bz2Wf96Bpf7TX/LL+5CRxvh0M8Oj8RISAYXqsQglpAiorcHTo1u8Xk/w0e0hiqJHoS3udiUKbbFtMygRsK4LjKf00VWmR6F7bGSOttc4GNXYthnKvEdrFcHkYQ0hAgpjUXcGSnqMixZ3QWBUdLBWwXYKubHwQXBd2gyycJAyoKkzaOOQFRZCBGDUQ4iAvOrRNRpFxRk6k1k4LyGVh5QBzknkeY+6lGibDGXVom0MlHFQyqNeFMgmHbKyh9Ye251GVvXoawNpPIL26DsNmTt4J+CthJAB0nigB6Rx6FqC0iA8nOPzi3UJIQKaXQZ4IKgA30uEmUU+btFdVejHmmAQHtJ4eC/5mqmFujSwLdPzAEBpx9lMAGoUbxAmEEA6yUqW0kLkgGsUgvEIraJHR3vYecfX6QAY9k4KxeScILj89q6gJ0xE8Jx5COMRnIAoLdAqBBXYEQoQ+AEEr6Ul8FUxrCmLqTyB8mFYARjPXszCQxoHCM0SehvL15MPaOSZjrpV8BMHX/BLSTaUHgMMaXJjBzj6xoYv95yhUHACQu6ZddkJuIlDqCXQ0Y8VBIGdCPFmLiJzKgJkLeELVhz40gOW+wkm7I9nHghqW+7XIAUQJzKHGfeYmOt1gEjPSaA7CNwuxKoPwW0KgssGd12ULGPYbkRPinAEgd6w3sLne/l2UAEhyvNchqFKJQiGa/UTMJFY7WtGZFQvsKAeQJw86WYBqdvVxwlk9oUCVvHYJalcEARCqVNS9HwdEOXSZVy/uI0uF5COHZsuptK6PAyVBwDoIZzEjlLF/SD7CEgsYpdmBAlh79XqpwE9CDC9AXzapzoM64MQQ696rk8/ifUsvCTh9b3P1QEyUOqNQLDlJ1wGAsEaQEAB8DwSlvvBFvHcxP74p33WT1JpeuzSNICJITI+4/anAvgkzU/HVcTBX3rYEsP9gscq7v9AoBoEwVs6jxABo09pzlGlMShTRnsPVACZVJ+lbSPYCoJy3HRu+AwMi3FcHoOsxOD7Syymy9OKRrAdmVwBfq6uGcjDsCD+ZMqtiNdMqsLgOtErFo+rBxDPJ68pqydQSufUXonCCQOuH8BajqAAL6PfrBDDue2NgHdxMssg1mUQQNObFtU0URGT2PMkmx56CfOwr6speOxTR6ZXQMj2+4brGxnDGD5kc4FQcT37UgznclI2hChddhkGtYmwcYJlJAf7wv2ye2fEAIqD4Pre38/CA84AQcZgO7c//2whhve5TCDEr6d0vrZTOQQfpc8JEex6dQ+8qnjfFGK4fyJwe1OHoq3kAG6DBHzGYyAcB2YM1tM8L+I1gMDzOQUgse6DgFsoAgmvoppAYvAKutib6bQYjmNIE0QZJ7ZcoYaAHwDop2boyLSleqPLMr1H9gG2YmKt8EA31QTFAOzYxOMVWc04wRKUgB0TsPZjEyfQuC3ByIEBdYVGSD2dke0lyIzAMcqiffQchggEhd2zW14TeIt7oTQJVLJfU/A5LfcVJd7Dm6RI4/GCwAB6vCI4xL3XINOsKUG8d+s9Y5nO/VAYBCmBWF0CGSgRve9xjBUjIfkmJYZOy5AbSlmNJpPo/JDyChDwDqmvUcL9RiclgKHGJG4zbASFIfwssBwWLLgdKa02AkYRg7p4Q4nv94HMZfA/Cw4j2E3gN8lxB4D5ZwGVf44e/peoHuTn/fhcM5b5u4/Cu//930Gzy+BrDdFIDl56Qf9hLdntqMhi6BVnzIaBa07WismRHEiYVSxVFyHWeXDAqzrOxKcAAg4oOQOcitpdkeRQqYA6lkKbMFQYZCsyNPml4uy9xMAsDQxDZI1UzcGfqjmoSq8NOvqXtmJgdERgwIDsI0sT/VlJkuoNgxSAxCLEgvkktyu4XGEFgt77FdPAUW/5GvoC+eUjLfv/1k89ygu5r+gAl5/6G9P/U2hAkvcmBiXoPXjhzDQTaF2sMWB4AmK4BIZBYwogSNLTNIDXOzIaKaWVX8QYUjHTgCXN/ANxu0w8B7ZxkNTvByxpMJgGPOmeIFwCEfG4p20SccBi4wAt1i6k16b9nnxf0vJzAAZDAEBxEz2QKjIhgedVki27TOw9lwd7z65sAZ9Hpk/vJdou575JfrFhMBYH/mmWPSVukhXZJ8iqGJIBT68ZZ+K5fqrd9+XJPgZRxNcOQRZpkAkM3Xi2IKvAnkBeX9JxcJgG88Jzm1VLKW43F3sZdyzTVi3/ny3DMDhN5xZLwCk5lh3fZwsOmJKXS3ZhGKSzTDwCBsEZ9pRUquuw72rswjDQs4WAjx2W6XxL25UqB5J01BZkN1Ighs/2xzbdJ1KlBRCPRxyIJzY1PZfCONI50k3IZqRBtnDxuN7bl2ZHRietE68Znnxknu6tRwf0FX8O15NL4GMPxNO5nAb0IuyXpyKrlc43Ecg+Skfpr4jnU6pXMFsWzYs4wGAAB2JYSBiudwLJCBoySoldTuldkAK6DYM3WHiCGbMji5eOny3p9/QRIOpmPyBLx9EWZHqSrzJdJykxNtkh0j2GXZsYBmjdWELHfkSey2LYZxz48/PS4Dzt29SnmEru6X/cdzQKz23gNcDlJDbZbGL9RR9i2Iin/DnwOXY7xn1Q0DOqOsqlub+4gqoPg40iBa8A9JUi8PpK68fOwT3gSAnHACDbeP2LCORjgE1i1HRNBs1rAdV4Tj6YCIhzrn+Sw3rFewOv8Xi8AEgXu1PDm4BbtQ4uV28MmEMEX7L1DERJasC4v1TjYEeUhQd5bzs6StSl8whCoJ9o6K0b7hMEgRGMpHRNvz9npAsQMdSlH2uo1sfP9WTf1mSTfCaHTkvZewILLZFSXfkmAI6AS+0sQhYBCUBwpSI4AUGj3vI1CejJLkrqIwvIxFYJ0fs3tplJv+ma4Pp6LYeJGuEDZGsH+adwHkFx+a40rNgIewAoYw/kAACFiEydYz2IJ0AZ7A/R9xmMgujdUAUyHEcPCBfBWlzW/fcNTOD9n0IQWMX9KizPA9nYzwRWQYg905dAWpKJWrcHZDEVFW03rI/o+sFbyc9T+7oP5zEE5qQU1fvLSkmsiPeTCCjf+N8br0/hAvJnQeK9KpE9kxh/hp9a5r1t/5n/pfcFz9mL9P4E3K39zHV6A2vcB4//fwHlnwPG8unXx+G/+ge/+nNf7t/+8j/5hW8b8HlnLCVZtPl8i7rM4JyE7RSk9oCXkIcOrlfATQ4za2H7gn2JXgAqIBt16O4KmHmDbpVDtBLyaQ1jHLavR2TgosSyrxVv5MbDWgF4ATGywNLAzjxZtkaSNaos3MaQrckd0Cgg95BLjfqJhejYbRZGPcQujmRjkiME+DNw1t+NHZmiWY+w1VA7BkmE0sEtNdzIQa8V7MwRRBsP0VDuqdYMDLBzJhYGFSDHPXCVw1cOslbDZ8pewM4shJUwtxL9OMDOHGQtoxwwUEraSRSvFeq3e4x+YlCfMmSnfuCgdxL9jD19csebruoE2ncayKssgjKB5pwBQt2M26dXKgJjjzByMBcG2xgARFaF+yNbCmzfZsejHbFvcfvlFsXHOYIKEF6gH4VB3qm3ErYk41i+1Ng+YNKirvk6s2aiYjjqoD/NoWoBO+LUtc8DrAB0TVmrKyNb6BjcpDdkCrM7rkt2x5vj9gknNMbPBTZPQhzYMJgIMkoNrygdVTWTEPPbKLk85MQAJUz05gZN9iC/I7DoAwNzdg840UEJIpMQhY1yQwdkd4I+3cCB/vZRGIKEXM792s3oOQ4GULt9emN7EMFdR9llfs3C+r7iOlYvBepTTriYNSsf6mOB9iSguBDwOeto4CO7pAD4uN+2AqoR2J5zEFfcsK5HbwS6GUOHnIjdjdt9ZQhn9JmMmC2B+gGPS3EbsH0ECEff6vYRGePEjhoqsNHNWIPiI4uTx6TC7SMur5vwWOhtHKRaekM3jziz4nKC89V77F8MgoO+FPiTrTiorU/pHW4OuT+7Ofsb2fMnoW2g51AT6HQzhi/ZEatTyksCzvpM3GNmRAQEBLWpkieBmcTkZauA9gjwq1jDIYB+Jt6YeNE7YPOEUsvmiCCSA3wxVMVIR/CVL1m/Ub2OsswoKxYuoD2gv9lle5atuGWIE6suuP350g+JifUxf8peRFYjAryWMlFbEuC3B4r7/lzGjk6gK9mvuH2wT3Asb1hPYjb0iNYnEvmdx/otiXwR0BxHr7bZJ0O6TAyAqjkQKBYBi3cVRq8IWNoJJ/GKuxQ2FCdRoi8UcULRq70clVJKDAyf6gPqQ4GgJKpLh+rKwWcC9SGBgdkwfKu8jTUni4D6UKG4I8CtjxTyJX3Asudu6sZi6IvsR3KYCNmeyjg5JlAsfWTjBBZfMJi8YMehbgLWjxWqK4K35bsGZs0Uz24iB9lrO1P7RNPIQiLQO+sMga50GHyunByJvvy5ggj0X8s2oDmkbzNbe3RjhT7W36g4sZMv6MlULf2ru1ON8trCtB6rtwzyVRjk4tnaDz7QbqJZp9EGmDU9moPcVLLmJU0m2lIgW3m08wyyJ7B3uYiTZx6y9einOtaIRNCbg9UpM41sGcNGQMAYJK+NbqbhMhOlyR52pOKEBmXCsvOQnR+AKXxAO+d6662DPeD/i6sGu4clVOPRTxlk1JzlUA2XwdA8h35iEJRAtu7Rj8hgUtIJuLFEftOiPWToj9508CW7BeHpJRU+IL+s0TyokC069FNDAF1quEwiW3boJwbZsoOMNSUuJ/BzlYFsHaTzcKVGN+Ky9aYnyJtk0NsedpIP4FX29Bh24wyqtrCTDKqhD1Z2Hk5nUHVPgNhY+lWXLdyY4TrBSPgsBh8pATsuIGxgGNAog2z4XmljlYv18HlG/+hdDcRKDV9lkA3TXwHA5xpq18EXEeC2Fj7XQ7ek6CzcrISse8CFoZOS/ZMObl5BOM+k1tzQn1lkkE1Hj2SecX16R3ApJcS2Zv/kksEGflRCtB1CkdHLaXQMG9IEoKPyXjhOZB97G+tMGr4OluDSRKbR+X2IT57twW6Uy6LrCSqTz7ONtSPWAkXGn0H+jHxVSEnvZp4D3iOkQKJUT1IUCG23X9c0EzAkwqohoGiQ3Ca5630/5Z8ie/2Lx+fv8blmLL/yzTz8+v/6W5Ai4MfLE3z14DU+WB3j148/wh8vz1HpHmf5Cv/v1VN89eACF80Eq7bAPK9xXY9wWO5QqB7f/u57+Bt/+Z/i//jR1zCf7vDN45e4aCb43sfnePjgDg9HK3z/8gyTssXl9RQPTxfYtRneO7zGj25OoKVHZzW+efYS37k4R5n12NQ5jLHQ0mOzy2GMw3vHNwCA7370CAdHaxxWNRZ1iW2TodlmmM13WLye4OzJHa5uJ/gr736E3/2994GjFt946yX+6IMnqGY1jsY7fPLiCIenK9zdTKBzi0fHC3zyz85x8JUbHFVbPL89QN9pVFWL1e0IQnuIm4ySxcJBbDXCyELsNPRhQ/B8WwASODhf4u56AlN1CF7Ctory2F4CAdCVRXhZwJ92CL2EuTToH3RAL3HwYIXFYoSDgw1uX89w/HCJ64spgXjmEDqFX/nSc3x0d4jVXYXxnLLi9csJJudrNHUGPKvQn9HrqHML22nIqwxu7PD03UvUvcHFiwMgAPMHaxjtcLccwTYaKnesfrgacz0BeCsROglZOJwerXCzGKPfGsiNppzQCeCgg9Qe+R9X6L6xg90aoBc4fLLAts7Rva4QsoDJgzXWV2OI3CFsNEQQkAct5wMuCw5qThogCLilQXFSo90ZVN8r0Jx6uKmDyB0lvcssStcCzGEDe1kijBxUaREuCwQToNcSdhzZgdJB5pRQF1mPKu/w8vkRZblbjVA51jEc9dCXBu68hbjOUF5K7M4dzFqinxIU27GHmHdQLwq4IsDP+WVWTlq0L8ZkxXWAmnZwrYK+yMhMvrVFd1dAVJRxh1UGvZLoz3qgFwP1LHdkAYLxkJ1E0IHy3zIAluAy6IDsjucUJFA/7QiENxrZQkL09CAmqZvPPdS0h/lRieahhawl9FagO7FQKx5L4UHvaB5iH2mcvfb0hGbX9B/1h5zQ0DvWM7gx/aOqifUVFeXS2ZVCYDgnA5Uy/m6P6W1NUt/uvIN5lXEi5YBeTXfQQ18bSEvPqFmqwQ+m2qgSkOx47Q44sVR8eYn2B7MoVeNz7SGl0rKVEH309wURB8/8fNUINA/Z05smpMydQrbkpEY/85h8pNAehMG3WT/wrJPY0hMqPBOq1+/3KF6YQUGxZ6SBdk6gF57WkB+wO9Ws9ui6vgAAIABJREFUxdB7CxB8djN+zugTTnC42MGa+n29AYpr7gOXA82xh66Zip0Si1OHqnAEY9tHVIf4nB7KfsyJK0PbPCWd2R5oFzfsbs0XVAEUVxhAOUBVxegFsH3MbQgyMXAYwrXSciEoSc4WYpBJ57d7UF1ekv3sx9ymyTMmAq++7FC8jJI5zeXkNzyft48Cytd7lrI9EOhnAaMXDGhLAU+JIU+qBhE4GSAs5ce784DRJwRuLnYA7x4GmCX3jWr3rJ5uAhbvA9VLMSQOD2A5TjPbkl5Vs+Y6dXOus97FxOAzOTD1AAY/YJKc+hworrj9tuLkjjfYqwn8XpliohLGxd7d6pJBXymgLPlW2wMGuXVzgdFLVmikOgevBUYXDrsTiXwV2X1NBlt4gvLElGYbBsZlK4JRW0alQJTBzp71WL2lsXkCnP6Bx/aM6b0IZPqbAzmoTqSlx9Vs2Dur+gBdB3QTOShSpN2rmIJM/k6PbiQZJBWiBFgKmJ1n4nDLtF0bg8SoFOF2mW0YAK5ZOeweGIL3iYzsOc/j6pqeStV6qNqjm2v0Je8ZZsvkYLOmH9+WZISbA07wqi76XmOicLawaA/NECbFTmE5+CxdzsnnbO2weM+guvRxEoDbki8sbEVQKVyAzxjyZDYeQQvIjv5N2fvoByerKiJbS5ksoBp2dmYLsoHdjIBbtWTLVMvleSWQLVr00zxed9Ei08Wqj9j9qXc2st6eTGnn4QsF2bDqQzU2MqKRjR9nZFmFGABwUJHlRGRIJRAydm5K6wkaMz2kzHotobZMvvWZZqdmfIgokRVNT6AI7L2YALqzEfJntwSUsetSJHDa7cEp36cYOFSzTzIkdvUeAzkw0nXz5sA6YYIiJ/OaGYK/xHgmVlRrAs+yQFiu+TprI+sqB1DJoKB7701AE8Dg4Qxx+Uku+1mPe7Ukvmn+5Nf9KY9fDsZyEv7LfwmM5d/58v/zC9824HMOLPN3H4X3/6f/CFJ6LNcV3MbQ/wZAVA7YaohOwM8sdGE58Jf0zQlPQOBvcjJxO4VQOOSvDdmqkjc2s5ToJ7yhphCUFMiiOoF+Fis/akH5ZysGqa1qBWRHFuz+gNrOHETL96R6AhbKC3QzTw+ZYqdYe+ih2r1nI0mxIMhKdQecidcbgebcQa8lZMvwDjsmK+aqmIyakZGh/I6sjuw5IOmnPrKlrE9JATOy45dothLD4M8VYQjcKF9J7H61hv6o2A/KtgL9xMNnAaIX7Fgbh5iuG6C3kmzXhp/N52KIQhXgSlae+IyfEaK/sD+2KJ/HWTdBD1t+x/oM1UbfmwpxXbmtSRKcpJH7IBmgH1HiKC3ZIWnB99VcD1VTciXj/qeEkssfjkdkBL3hQNWsoswtHisZZWTdjHUx8AwVkS1fkzxe0g2YDO0Rj7neiv262z2TYMu9zFLXlIc2RwQcNg4svd4HofQHDuMPNdqDOBjWe2mjN+BAL6YKqyZKKufcp3rHZXYzsmzFNc+ffMGBoyu4bqpFZFr3n6vqeA71BAau5GeTxeVy7ShWmxRRYtnuB6zdDPd8VhyECrdfzyR1ReCgtzkRqF6RXRN+/95+TCl3SJJUz/9JFyXesfNN2r28N0lkbcXl6Jrbq3fcxkHuuqUUzBuB5oRBNwj83CQv9WZ/HNO5kpQJqqVsGOC+Fo7spdlEWR+A4oYVGkOYTxwUB81BfZJoE/jFHruU2urflG8GTVlwe8ifAINZ8jve77oZB/T0tIlBZq1iaEtKQi0WlLHaai/NTd4uppLGcyoyrLrZrweiTFHdS1BO14qtgHwZ0MzJeKZtTnLflJYMYO/7U2Q8y+swyI5TGu7AUPZR0ttikPebLYFCHpk+W96TygrKPduJHK7lNOC2pRyqM4AobY9+Tl1z2Qn0AJFt7HisZZTGdyOeo6onYEgTDfmCLIwzAqrn9jgjBum52Xm0c8lzp+d+lZbHwuv9MU/BNqplAq6JATHCBWRbssh9xdoKW1AyqxsG0QAYJJXShjfSqBEwrId0lI1S9s3vvFSjkyTIrErhtjgjYrgP96HqAmXCtR+SjVPFCYGeHN6XEn6TZHa4v/b76yTJ9tN5luTEA3CJibD0gIvI/PkBXAlH8GIr1rm0cx2lxoj7guBExG1AANoDzWoWJSgnlYLpuzYMqa7eUErNxGAMPtOguD9FBEX9WCNbdAhaUgobX6MaP3glEwADEPerhzMEkC6nT5GfHYbUaBEC+pGGWcXwG+fRVxpmQ9mnqzRUTXbQFWQKhyTdJC0V6TpliE0w9CQKT2mviOwhhIBq7OB1DEoOkuhBstwxYyHJaoOS8ViQiRTJrxe/R2UdvxDu14YkyWoCSp6sY1AKwUhAUlrL88IPclcIMopBCIQyMpvZXpYr2/1yRdMjFHwNpGQNiN/LUX2VQzjH98a6EggBX+WQ24avs26oEUEfl32vjuS+FHYI2rknl33D+5jkpZGNDEYzXTYxgwkIpmqRn6n1iH/fC+sJzgPe4Y1k1gQME6D8k1jD+xUkKTVWCtaKpEeS93q/Z0Lvh/rcfyQ280+TxP65kML+BbD8pX0U7z0KX/wf/zZWV2PIjWKJueEAWHZiSE68P5jTO87gqwawUfomWwKIEGfK7ThA7QSyNZMSXQYUNwwQAAjKVBN7DNdiMO8DiB5OQPTgF9YoxIRGJkkCwO7co7iUQ9mzcHtAIOMAhf6VMPgyu2kENIIDENXsB1yD9y9K57xBlLTtZ/EBDsz7cVp2BDfAkBqZPJypUJohBdwOs8YAJn0WEx03HDD1Yw6+WbvBwV9zRJlj8mYGtV9mWr8BNAUO0gdPpdx7H5OvzFZMh+Tgi7POerv3UqZ+yuQ96iYc5HWzPahIiZZJXplen+Rf3YTHPr8Vw4DWrHls03bpbRxot1yO2USmYkS5aErChEzrHGLQAoakw3xBmatZcdlvDMxGnEVHAKYfe7QzgX7CbbVjDLUkqmUYh9fA5BOH7QM1+LCG8I5sXz8jbUB15dGXEv0IQ32M2dD75BUGqZjZ7Jctu70XLR0nXacKF/qkkgc1+TeHczF6zlxGgNLN+TnZmoPN1IuY5I2Juck2/F+24ufSZ8XBtexihUUcdOsasdMzMpSRfQAwBHO4jNsEYAgrSRU6Zh2GzssBIBRJqkkA34/2bJHsCcSkDQP4S8xe8g1LSxBjtgRq/IyUBolh3YMQ6KaCYEKK4Vq7X5Wiuv3vuiHYaWb7/cUJlgjet+x11DUliGndbL4PpEnBKtIFlpSv9gN+P4SexBUQfK/qMcxumx2BEOWlfFlKAlWdRztRvM82HBD1FQEEgMi+cH8nUMF9vH+eB2kP8qUjSEXgfgtSDBJJ3TKpUzoyRfSSi8HzSH9cvCfE8yBfRiYoSh91s2eC2LdJoKUbdlNmGxawCx8GTyM9yXuQlQBnN+a2+igb5b5jofxQoC45EeQz8UY6qK1i8nJMJ9XRbyziYN5lkW1aWzRHBvmSPZPeCLQTAl3dxGMeQbiuCTZkF+L9dw8EIcj0JA+hitLLfswglnzBAa7eObRHhiCmJUjxhqyQCICq/R40CAzeRh9DV1THQWm639pSxvOQ57xZk1kTAQxm8fd8ij4M9zTZcfA9gKXPAGchgkrVOtiC52HyRBKMiuHeoNK+iqd6EGDh/V0HO2bhfX7XDeExZLkUVGPRzTKYdc9aEZ32neDnRyAnW4dulg2g0eUSqvOQbfriYdBMkALynmfOq1jdsSMIZAcnP9eVOnqTuRxhPdzIDFUnHL9Y9NOcbFvyroHnEFKYixCQnYvbTICXZKRuVhAICUGpqSDwS75oHyWygz+z1FA7C7mLVSF9BErR9xhiqA0k3khLFX2UmNY93CQfalF4QyGolk2PkJJIcwW56QafJQ+iGHyRPiP6l53lpIjkNg+Mn7oHzO4/XGCHpXODlJW5ABGM1pGxa1mnkcDi8FOrfe9kquEwGnKz24NeG6WxyUsZH6m7clin+2msbn+e7AGhf9Nj6RzX7af/n9JhvWcwTwKuqb9SCISU0qrUzwK9BPCSVzKtWwKc6by6/3cEliEEoO8x1I0Yw78BgtjPetxfXgKqP70uPwU2Q9/hX/TxywAs3/76JPzn//uv/dyX+3ff/51f+LYBn3OP5ThroaTHb3zze/irsx/jR/UD/OHdE4QgMDIt3hnd4A9u3sLdrsRXTi7w3YuH+Hfe/j4AYKobPK8P8Xx7gIfVEt9+8TaeHC7wa4ef4B99+iXMyxo+CLy8m+GvP/0xXBBY9wVu2wovFnMcj7dYtxnOxhs8Hd/iH330RYzKFmfjDT66PkJZ8PdPlzO8c3CLdZ/j5d0MWjvMlUP55R6/8fCH+J3LL+D1Yor5eIddm+Gdg1tc1SPM8gbf//Ac8+MNqrzDWbVG4wzGpsW3f/gOfuULn+B7r87wpQdX6J3C6/UEXzm5gA8C3339ENPxDi+fHQEqYH6ygRAB7x9d4oe3JyiNRWl65MqiUD2u6zGsl3h3do1Pt3NcrsfYrgo8fnAHHwSeTm9xVY9RW4Pr9Qj1ogC8AL7UYntTwBy0+PWnH+GD5TF6pzDJW7xeTvBv/+b38dv//FcRnMBXnr6Clh6vNxMU2uL17RQnB2tc3DKC8vxoiU9eHuKvfOlD3DQj/Oj5GUazBtZKPDpcwnmJ568OMZ7VuH01wfx8hbo1aNc5IAOEChhPa2Ta4cn0DoWy+N3vvwehA44e3GFZF1iuC4RGQZYWyji4lxXmX7zFtGjx4uoAdqcxP9ngi0dX+ONX53BW4vhoiWfPTvCld1/h4+tD9I1G6BQmJxu0rQHyHoWxWH06Ry0DzKiH/7Rkeuy0R/tpDvekRpZbwFisX0+wm/QQnxaov94w7fc6R8gC1KSHjwmmwUrUDxVmby/QbAsUZYfuuzO0v7aF93Jg56uzLV5ejTA6XWJ5MUJ+VA/fBVJ63N6VkLnD4cEWN98+Rvteg9GkweZqBH2nYf6tGzin4L51gM1fbhDuMoSxg740CE8auJ2GyBywNFDHLdxtTsZJB5SnOyxvSmRXGv2BByY91Cv2AbrKI79VaE8cZCPgyjjADAJbKxBGPeRGk619vIHt9CAPhg4AHD8XADYGQQSgoE9ZtoB6d4N2UQBWQE56hGUGtSG7ZE96FJMW7esYaxqAYAJGH2s036w5Af1pAdkFrL5q6aW2aacBov3/2HuTXdmyBEto7e401t72df7cPdrMyMqoSqQEhGDCiAFDfgAJJFRSiaZAUBJ/AGLIHzAriikgJARCJRiUVFSizMqsCHeP8OY1t7f2dLtjsPY+x55nQCYQKMJTadLTfXav2enNzl57dRLmKfnh9gLHHw9YXR2xe7eECAJ6K0eFgZtF+AsHfWfYk5tkoe56wOxnJUI5dZyGgnLgcsPak6iYGKx3rJup7uRYZ9K+8FQOLC3UN+zW0wdqFruXHrHyKG4M3IwKDARg9lahu6aPOEoJs6PcVR8T0Oq5bXYdEGoPtWf1jb3wWHyucfgha1/MQaD/yKJ8YzBcBpiNxPDSwbw3kFbC7BNLeEbVQ/eMXuNQKPhUm6OPAmZPsGRXSP5oXgsEAhLhZQfzi4oApOexZC9uhGp5LAB6acsNg576q4DFlxLdZYQa6AUPhvuepb3dM4/6rUL7ysNsCFoWXwP77wGyZ51A+UhAPqwTKN9zQnD2TuDp9wHhVQpPEyPzHjUlr/tP00Rjmowjkw+0zyLKJ7KJ7fOIUETU79U4UdddAasvIvavBfY/9jj/I4H99wSKrYJdcOLQ7Nn1a/ZcbkhKiVxZU2yo5BhWGt5QpgqAHvlaotiDA+MIqF7Sy3nHc5IDoHSTQGap6cMO9BM3z+UoE9atHEGiqzEGsFWPVHm012SxdSvhS5E6lJHqVNiDWz6xTsbOBdprgWJH/3b1EKEbmXzCCuUuYvcJu2ldSZ8ye5I5Kbh469G/MJOCIKk96rswVsj4ehqwL7+W2L+m5L3YknG1c9a5dOfslzZHsrNDmpTi5JhAca7GShM3q9g7fCHHShZlDbozgcVbgf1rhfltwPYH1MpnGS4nWTjZkQPW8rHPE6bS0hcMEIS7iiwwkJQwgeDWNAHthYLuCug+pIkUXnfmSL+tbotxUsMcuYxhzRCc7ozdoOWOTCsk0F7qMWzJzllzIiIgLw30wSMUBi6x2e2FRrXxUyBaoAe4vqdvWPgIVSm4j2fo1hKLt5ay3SV9pNnHWr/vsPt+hSgE5jcWcgiU1x7JDFdPZkx8lTbCzhWKLfuDh5WG6gLk3KSAKIYNSRfh5gr66MmClgqIJWTv4Wd6DEtiwrhAKDlx4CtNYK3JpgYtoBtPz2WhPkiRddfz1I3tPwgwipIM9XBeQh8s1HEYk2CjUbBXM+ijpQT2JKhIDA5hTomusB5BCMhjh1gXfF4ayGOHsJ4T3EpJgK7UyIKKNvViGg1xbEfZLIrEroYAaHZr5vAgxIh4eQk0PZ8XZvpbP5CQPk16BSBmM4JCYya/JoB4bCgPrmvEjp2goioJXH34kOkMESgSs/FBTckEMuMpoP5VbOWfY17/avgu/V+nwv52PqpXH8eP/u7f5eCsmeSJoYwoNhLtS4cX/1Di7g+B4lGi/dSiemvgUk+fcICfxdHnpnqRoufJtrmarJqb0ffSX6SZ15hkexZjDLoa+DzH8QMYZ84BnJQk55t1RLkhg9E+J3NaPcSxPN0cyKBxlnxi8VyNlI7IIJVcKWD29NXUd7z5Cp+YRJfYTU2mrHkJLH+BFL7Cbcp+It1SRugrkE2TmaFJ667o+Tl+BBT7HDVNFq7YYWS4clKoL8mwRclgkvlb9hpGIdBdAfUdQ0PcjOvLPX/DGQcUxY61HQz1IGMxJNlgZkDNjgyibjBKHWc3HrvvqdEjVD1yv8oH3lhz6qFdisQMerRXMjEAHBzZZWKYEyOUJY9RZ+lNZgX4+enPsn+F0sTDRwxGKbYRriYLlhk4u5gYPt1E2BW3n7I1Mcr0Vl9aHF6ZsZJhOGPnXX8usPw6oEvM6bAmm6CbHPSS2R6+T3iMEj8RJ0nW4p1n0fqBy4iJccjJm6qfWGS7EOO++sQm5PRW9sdFlDsyZmbP8+aNQLEn4w+Rmc4TZi4tK8vd8gBQDVP3H082Rsmx7hmMUhyyl4ryxZCYyPwwx+nc5TAPVrVgTCd1NQNq+jOJ6ok3KzubUlX7M8lBZ2KVo6JMc1hMLKXu8uc8jh6qYDjI1E3A4SMF3VCqWT0GuDolierpcxkVxv3J6gruE7+DdAtUT35M9bR1lv5Nr/UmfY4OcTxulK1OssysSMi+s9XXFnauUichvV+uJFvVXXKwksNjgk7L6/i68sni+LKALwSKQxgH5pkFFSFCdRHtpRq/B3UXKBO0SUooJmk0fYMhSVe5nGrL56on40u2UsAcySIqy/UMc0lPWi1hZxLlzifmcpptN4cAu5Aotx7tpWZirGcXYHeZuuscz9mwUmPAT7Fh+IevUqhWLUYGNifk+kKg2HvYORmmzBSOaa0gs6YGsleZ8WQ9BUZG3hzJ/vVretvGZM4A+Eqmz3VKSC3ExETaOF4bOUAmP9jVGRN7d1KvcLJM3SRmM9dEpHOUuzHlwEG8r+h987UcQQYioJInTQ70uCHEsT4iS3OLxwHDWQGflp1ZRrOzsEtDL10h4Ss59iJm2afZT32IAEa2FAmEAJRXurn+oBJDRFZhjAx8Omc5/ZbBQTF58MgE8vsp/c0F+JmmMuPoMZxpVHfDdF1JTOcoyUZzF2NMlRmuZsCK6gNZPymgWge172EvZny9DWNSqz7aBHb4ftk6+FUB/dDCXdbIvYs5eTWzOVlWChcALRm+4yPkwICaLDVFAPyqgGocfKVH2akIEbJziFqOqatZ3mrPKqjGjUnPshnIJGo5nqNsC8lsYqg0l5cZQOspPU3smWwGhHnJ/keAIToZQKRzKDsylPz8qPG5sB5hUXEZlWbwjvWs40j7jwzkvs0GnjBwYV5CHvtJWpq8kB88bJJ7asUQnMwgpg7JqE/8gkrxd3VJCWyWhDrP/knryHxW5cRefpvNPA3iOZH5IiRmU+tx+1kDkpjTXCGS9zFQLiqMSRLVDxNnow8TK6m/xS+NoTp+2qbTBNjxy4XS1+ynjKc+SXsSQJRTYb/NWH6bifxV6zh9nADK77oU9tOfLuPf+29+/Zvwd37yP//G9w34KwAsv/dv/4esiEgzvNkTqHqM9QoQnP0sNrzpDisO+rsLoHpM/qTkoSqfCEJmtyxAFp6Dwjwwtqf+p+Q7kC6i2HG5ud7D15x9ZmojJTIygdL+jAPuXBSeB85RTsESo9QtDRbcbJLzuRmSXCmOaYTCp6qBll6pYh/HXrDsO5I2rWOWBjQujnUYYy9a9gWmAbJM+2OOBBmshcA4a1rsA44vFAFE6kMzTdquNHPePGMZeJa+6paSvvZS8jgoATvjNrsZb0RFGiDbmRjrWEY5Yh7EJ5lbLqLO3iXdE4Dy+JycpxwYkQbiJpWk50Fklg1miV8eiBdHyg+BSaKWkxqL/SSDy37KLO3L3pygxPj7oAV0y5CGfK6yVywPsl0tUOx5o3GlQLXxGJZTMTZBWEi+oOm6UpbyuzwgzPs8JFAo3QQGs88sH7tcqSAdJUR2Jkff01g/IDjzW+zD2NWGSNDjk4zRlXm6k+cpaIFhzkqH7LkCMAZNBJNSSdP7pScAER70hCUAO8zpExPj9ZgG78lflQfP2WeVJYo5JdM0TGoMBQfFdi7H7ZOO16A5MKkyf+6LwwQWcjItgLHmxOynZMthNSVq5vOp+gkc5PNabn2S4cpx3T75vfLAisBejMvTbRh9WxnE5gdlgjGdSzECjlzLEEqRAP5U56DblCyaklIzOOF5SddqKl+nRSAf4wCfEhWD5vEK5qSEHIBdKugmYKxFSJJp1U3+NjmE0QuWX+8LCd16+ugaDurzwCOUBIzFPozHzhw99N5iOE9pSiBgsEsFmYCotIHAIpW7QwJma0c2gymdLIGXQxiPd5ZIqsZR7jj6xVLNh2MACb17ArrL0ksmVCobxv688XVJCusLCXNgmEmWiUZJHxuZWz9OULA+I4G5FMQiBw+7MiODIjz/Jl2EPlr4VNcQlBwTQvk5k2PBPL/nk3Q0vTbLK0OpRpCESPDtazI8Yy1EOt/S0pvlZ+YD6eYHQSHp+skMD3c4kjE6kViyg4+fzSyJzMmgsnfwcwPZujFkRQ6OqaSLcvwsZBmtCBGi93zP4Edwog49g1WKEzmlPKm5OLnm6B1kKmqUWRaawVlg4uhhgF9Xo7wTIUxVFkn6qY4D9+ekw0/4+IEMVFqfJJl5m5AG9HGUdIbaTM97j1wlEmsDuWtHxguRr5WtJdt1WjFR6JE54/Z3lJ2m2o0PpKMAmTKTGLy2R5hVPPc5MCYEhNIQ2GXwkpNDAUArHouU3MobhJ8AzmltBzBJOp3n/uQQG+dHhitfX2KwiLNqlGCO5z5Vf4wexXDS7dhbpqvmWgwlx20Qg51SVTOIO5GtAuA+d8N4PPnGBO66fmL2nB8BW+wHCCX5txE8pmOVakliCExgDWGUfo6SVef5fgDRWkCqCWBl0CYkQaBU/Al8+Dpges34XBAoG839Pt3P7IXUepLknv7+tMfyNIQnT6Kl94w9lRnEnizrg4eQQAyIIUJI8atfc7p9IX64L/8PH78NwPKTn67if/IPfv2b8O/+3v/0G9834DsuhZ2tO/zoX/sCn8yecD/MUUiH19UGf7x7hcvyiL0tmdjqFRpHOv5ZvcduqFEoh4dujnXRwkWFXV9h3xf4g+u3uOsX+PHiFv/tF7+P3312CwCotMVNs0RrDXqr8XK1gw0KEhFDUPib52/xj+8/xvdWj/jF7gIhCigR8c9ff4V/8vAapXL4/OYK1+d7BKuxrHo8NjWerXb44uYKf+v1G3y5vcBM80O+70pczBuUysGnEe39YY6Xqx3+2TfPsVq1eH62wdFyv2ptcRhKHIcCZ3WL3mnc3J3h4uyIVTGgsQafrJ7wj3/xCX7n9Q1+cXeJV0liuu9KHJsSP3l1g95pfP7+GuvVEd4raONwWbW4b2aIUaAuLO63CyxmHZ6eFnh2tcPm/Rmun28xWI22KbFc0EyqVMDjhi3m+l/oMC8H9FajDxJHLzGvBtxv5yhLC+cUXjy7x81hAecVyqrH5lijNCl9NApsDzWWixbb7Qwvrre42yywmPXYbmcoa4u6HLA/Vvj+9SPumxme2hJa88v1uKmhSg/fKZTLHiEI2EMBUQSUtUUIAs4q1LMBZ7MW7x9XKEuL9lhCyAilPbxLs6dBoKw5I3cIAmVp0Rwr+L0BCq5P7jTqT/boOoOytLCDRlUPsFaje6yglz1co2HmFvZoYBYDvFMoSofuocb82RHOSQw3M1x+f4OHhwXioJi8aiKgA2AlzHKA+GIG+9EAWXgIAN7SMKwKj+AFwtFAzimpdK96JvwGgWglhAnQxsNuS0BFzC4aeC8x3M4Qaw+hA+KgIHqJWHmIVkFfdbD7AmbVw24qiCpAFh5KBQxPFcqLFv1jzZF7EYBecXujAGQEnIBZDbCtQfHGwH5sERvFwK29hmoV5KdHDJuSAKUIiC09U2bdw7+voV72sK2BaDTMVYvhqQJKD9FoJrp6wX+BKgRxZhGOPD9CRogniWgALAfIe6bzusvIPhcRARUhj4o1KFcBkHweVg6iUTA7Abtm2i0UoHaUI46dtWW6DjoB4QWE4wSHW0fojYLqBYazgFgF6CfFFF6ZwrQAuKWHsBLQHKiajRzTWYfzwACygcuMKYDJbBTsGd+nejWCPUjAzwBhU5jYeYA8CECKMcxLRMAtA2QnoI8Sw0WqT5oFlLcK/ZWHPprRx+4LIBoB2U2VMLl8frgQUCmgq9jltMrI7txhKq8fLj1mXzFUSnjArYDZNwr9hUCxNeiuwqgiKR8F7FKx4ucVcFJ0AAAgAElEQVRR0pMYDXI/Z3VPSa5qszSSINzs0qRgqpEZzjRVIF1KgZUmKU4EomTAVndJ9YovqRbQrUgThPRfNy+nmh7hMfq73WySOtpFCn/bi9HjnUPGEA3cPKlS1rmvOAVJ1ZQ3V/di/FueqBCBjDT9jCZNUvL8k+EvxyAtSnwpSzcNknyd94pRZZO8+VRjYKwnysqYPCnrZtNzTjzxOYOk+F47r8hKN9zWgQ4HmAPGZGHuBMbaGzejouNUSiscJ3hzr7S00/H1FdU7rkaS4caUISBGVp2pqWIMOMuTGcw+mI0haGNA0PzEa54mQCA4CTR/5zEsJklwVujYVVIXnS9Q3we015xUDGnyUHVcp7QRrqpHhYT0SFJOXovFPo6dtnYmqIZYcWK12nJSKiiqUMwxjuFMp2FjnICYj35dldhgMutI0txUcZQYcm84QRr0fDxmLiXGSpvCfmqZJn9ZfZMnCm1N6XP15McJjXGydEXlQJ7AosqE4T5RTx2orqJSJMtw6XX3nGRKstzscw96WpZuOJGj2lRdYsmQqy7A1ZMP2uzJguccgNOJuDzJlIOWAE6YyTTJ48vkze3JOmcZreoyYJ3Y7fz9Sp8kCAhT1UrUkhMOaR3TpAFBfDCKjG7qDYVMn9HULxqMInua/JBRKciThNlQGtacZC/jaW8kMPotY2UI0HNoUAg47cHMYUGnfZji2x2aeR8yK+r8yASLBChjkg4jpEkGvgGIYQLyJ6FBp1LY8SaVwOqvepwSYEIBsf9/Dyz/+vH//+M7LfLV6Y772f4Kd+0CjSuw1i1WpsOfPj7HTA/YDRX+9OYFzsoW+6HEn9y/wF07x5vDGlJE7G2FrzdnaKzBbj/Dn22e4V+9/Bn+0f2n8E7i1WwLFyX+6d1zvHlY42GzQIgC980c267C7WGBu/0C/+ThNY59gc82V9gca3SDQWs17vsF7vZzAARa67KDCxL7vkDTlHizXcMNCl88XaLUDu8f1rjbLrB7RxD7brfCm6c1GmswOIU//cUrrNcNXJAopMPdYY4vvr6GFBFffvEM7WCw7So8HmcIvULTG9zuFji0Jf743Uu8fv6En799BtsabNsKbx/WOLYF7KHAn719jm82a/i9wePXZxishpIBn727hvMK290MD7s5hsbg+eIA9b5EiALm1mB3rLC/XcAdeBz3hxqP//QK+rMaP3p5i6YpsT3WeHqzRgTQvl/gcbPAfNajawsMTxW2fYX9ocbu3RKHrkR7KNF0BQ5Nhd2hRrit0FvOhbSDgbur8TuXd4hBYBg0Ht+cwbYGv7y/wNPjAkNT4GLeoOsM0Eso7YEg0N/XBJWdQnQS3gvYTkOpgOPtHG/eXsBvC7SHEuptCXFTwvUa4WCA+xJlbdG/n2HoNNybGdqmhN8bFPdphC7opQMAvzfo2gLuvsIwaPTHAtVbgxgFRKNgNyXMvYF/P0N8LCBEhHlUON7MIf5kCdkLtIOBeCwgBgnVSpRvDcFaEPA+fYT3Gv5gICSBYPFlieAFis9rVO81glUoNgL6bYmw46C0WAyIjYZtDGbXR4hWormbY+gM4AVEqyB1RHGjUd4qzD4vEE1E/JrexeAVzEah/GUJvy3gBoXiQXEC4hsN86Ch7gtKoXqFYtWjXPYQg4SQEegkJdW9hNkpxFbB7CRkD9heo/7GQAwS2BrIXkJ2EvZQsBKkNRB7bpftNETlUX5dQJ4NEAcN9cT3CitQPEnEDT2YcqOBrcHiKwnVSMSO6+TJSjdQx2qc2VsOLNSeg43qjs/LewXVEOwBgNpomK1E8UQAqfcEVrKVCDMP1QksvhYoNgLVO41iy4Hz/I0EgsD8jcD8m+Tp3AnUNwR1chCsQDmyFmX2jn4/dZSpKoXXA1SEHCTTmpNXtHgS7Bjd5oENE6lNqvOobyVUJ6DScmUCVuZAsGx2EqvPkcAWO2EB1lPIXgAftdAHdsKao4DuBBZf8zCqRtJq4OmPrO/oSVQDJe3Spu17oky42HE5sksD9CKifAT3tRVjCnP5BKiG+6Nb9sHKAahvBfs/82s1Q9pUK1A+ETzzd6xZMbvcaRoxf0sQKnvAHMQYllbfEhjO37A3VlpAePoqq7vE6jaAainZVwMwexdTgjCBrHD0CpabCLMn4KByIQGjhmnY1SOVI/VdZkEYEiZTtUxOmqXPD1R9HCN0k1QggWCvuouo72OSU0dUDzGBBfpHyyfK9N0i9d9u+Pf6IYyKlgwyq3uCkfIpjuAuS/fruzD6Ruv7AH2MqRKDwUTS8j3VY0T1FMZANt1MacHmwPCr5RuHxTufQGmkAiSFO0mbk3TjGJZF+TLfX6SQsRzeJuIkL8/L023aHs/jlZl503DbzG46N6qjH3MMZmsCimNEtfWoHz2kBWb3/DmsxLid5VNKRg0EsKblegnGCNyy1FoN6Vw2EdWDZU1PAs7lxhHotan/eAioHt3IyJsclJRUJuXWAwIotp4yaxtR3w1jkq85elokuojZu54Ko52H7vi6rIYqnyxESH9rA8zOozgEFDuH6onVIcXOp+TjJE0fIspHTq6ao0+TMzHVf/B4m8MJAxsn9UVxCCj2FspGmKNLQJ9BUqZxSVorYA4Ous2MKyXXIkQU2wGq88kXSVAoIgOxVJ9Cp46O18zeEwAGMuc5lTcYCbMbKLV+bKGOA9S+GxUVmaHWhwG69ZQap9uEPgwEl5ZsumzdFMIjBdSxh2oGdl62liCyGZgQrCVEricJgaAS3D7Z23R+BURr6QHVMnmmE6CzHlFKyG4gA6tlArxylM/GLHtND2EdZbiJlc3PRT8ktYQc/ZiQkkxoBqxZ2ioEl28d4BylqFIiOp+SZeMIQKMP/F3kzxEUJgAbk2w2/xsfIX7w+/+r1/yFqbHfkYeH/LX/+215fKelsOUPPoqf/Gd/G94phE3qlAMAQzYH4GAgqsifRYBo+XvZS3Y6hpR2ubYQRw29l7AvBogml3rxyyinyfLuxYEaAtLsloCfsbsOUSDUgZUflh15rAuJ47YAgK8DiicGBfgFmYLcJyZsqskwMSXQSkTFgQYCmYXiUcGuwjijnWeEhWPfWl5nrg2JWREqGQLi63yT54Ay+yJlzzTd7HeExOhBDQWPQ3/pEasAtUsrjWngKabeMV9N3XD9JTsUVSeSZzPddAfBfeh4rHyRmII6VRMEDvpUJyCdgF2QwQgqp1sK9jE6esnyoDrPhoeC22x2lFyGMnn3IiA918ebWmKa0vEvdgwSAThAPGUixpqRNJjL8kg/S5UpyRvoKz5XA0Z/rfRkLaLksqXjwDgk7x8TVDGm++bz6WsGurgaH9TNRDml1ubXZ4kfE/8wVjYgSa3HhMwyeQE3/L1d4oNKE28wynTHdUZun0rhGdkPaFO6cpbGSo8xAEOEk/23SZ7eTb9jnQvG/j1xuu9pvb4AlOXr3AwfJPyefpfmZaqe63Lz6W9yIItSPmH0/0qX2BAxndvsnc0eXldzearl8vK5yRJSkQbuIvK9wWCsGvG1GGtUyIpgTPzN3utJxkmZvU7r0Q3GeqFcu5Ol7PS/iXE7cjqsnefBG5JcU4yVDK4W43X2oVyX0upQpkqY9H/VJWYs5mNBmXpe3+gNLFK9SbrWxmszXZ/5ugkpGfhU0p737bS2J8vPh1VOIU3bms6LGjD6UvPv8/WWqzyCxlj9ka/D8Zina9LOpucfXD+C7xtlu6nOIvspg8JoD8hJp5k1zQAqH5/8+1yNATG9P5+THPCiEhhztZg+M/l7JrFcMtmWQpHYuiTNNQ3Tnvmhmd47yuMz+LK5UzFNNsQ4ft6zbP80CyBoMcqls2RbhGx3CGNS7Sjb1WLsaT2tFMnLG78HHD2hBI85zZmS3qDIpsWT88L06Q99oKdS/sy0+VJMCbuJHYxq8o4HM6U952Ou2oBQTis7Pad5fzO4cTUDbaIUsHOJYs+KDtV61oNkOXteXPpc5kRYXsO8EQuXGcSYKjwo+c7BS5MUWozJvaxiCWO1SVSTfzbvp+opu86D/Cyt9uVJgEy6pk6vfdpxGIyTU275nRMwrA10Q1AXCgIyNzMEOyfDR+ECmUQfR98pLT5JQu0SYEv1Ilk6yfovSZYwBeqoLqW7CkxgK/lKc1rt9D6mwopIOXEo9OQ5HS+iMLKF+TiM6bEpoZZS3ok9zFUisaBPdkyXzdJ/LUdWkQuMk/w4J8eGOHkmM3iL8UMPpWKVSZQiBdgkD60U4+uF86Okl9dp+HNM4wQG4yQvPk2V7foJNJ56FBN4+3Y67CjPzY/UkTn5PE/+5hxGSWuuUDlZfn4eY4RI+zj9+S+JQU4A5Xc9FfaTn67if/QP/sVf+3L/g9/7H3/j+wZ8x6WwdWHxb/2N/w1ftFf4R+8/gVYBndXQMuDFco9vtmsY5XE9P+Krx3M8X+/x5mENrQOcVSiMRwgCMQq8vtzgOBS4e1wCnUKsPS6ud3h8cwa97lFUFgUAozxmhcX9bg7bawgJuF6hXnWwg4a3EuWM0sqhoVRPPG/gjgWunu9w/34FVXuUxqGvSkBEzFYdmm0N9BKLFwccHmaYXR1x3FcQAJyT0KWH3RuU5x3EoNFXGmqjYS8d9NLCP5SoXx3QvlkwvfOhxOX3n3D/fgVheFeXhQfe1HA/aSiXPBjE2gODBEyAmVn0uwJ6aRGshG811F7BX1vIo0K8GOC8xB/++Jf437/8GK8/vcNXP3+O9estdp+fIV4OePl8g3efXUOsB7hPPH70/B6f315hKEuIXgIrB/FoIF90GLYFB1ulQDQBslVw64DnP7jH7Z9dI5xZoFe88Uvg5Q/vcPtHz+GfD8D7Av5FD/NNgeGlhT9oxNJDzBzClyXsmnJB2bFgPl4NwN4AC4voJNBLlFcthpsZbBCIZYCw/BkKBb+ipLB/Rlmk2ioOiq96mJ/X6K48VCvhFwHFg8TwzKE669C9myEuHaQJUF9V6F47iE4ilgHFvaZE7qJHcBJir4EoIC57soiSckb1wwO6p4rpqL1Eeatx/NgjLh2lnkWA3FOWGd8U6J85qKOE6slA9WcRfumhDhLhylIFc1NA/2iP9mZOFqxjEmd/zi9rVwN+4SFbiWIn0X1koR81io1A9yIAL3r4o4beaFhwcKIage6Vw49+/A5f/a+voRuB5nd6zH5eov3dDrFXgBecxCkC5l9qhDKiWwJ6L2CXEc/+uRts/pcXaD/yUEcJPw8Qg0B5L9F94qF3EvbMk83qBaKO8IsA/TON4TxiuHaYf07ZU/M7PWY/K2GXDMcyexbOuwWTTe157rMTUH+wxfGXS8zeSXTXEVFF1O8lfJm6VluB4w97xF5CbzWqO4HjJx6zbxSaj/wIaMt7BTkA7UtuY0iTFcVGJlaMEz32ysE8aASdJnE0O2/nX0vsfuIgvIB5kugvAbcImL1Ro0ec6aURfuWgDgrlQ5JopWCmcpPOX0nwmutkfBlH2SfA6qMogcVXAvsfBJQPElHwGEUBLH/BUK1QUop5+DRg9XOJ40cR1SMnWIY1k1erWyaDhoIA2s7zJBBlq8NqYsqCiQg6Yv5mSsZ0NRm0/adIfbpMCz2+Fpi/idj/IGD1OQ9y+zzCVxH6IFE+cYIgd7bm0LDuOmL5S6C7EFi8YfhWFEDzUUT5QKlpnogZZakd0D2LWP8MaF5yUijLen0dMf8GOL6SSb6KMQVWt1wGB/JIgWQTQCv2BObCA+1zgf6CbKrZp3MW6PdvXwjM3gLdpcDsPXB4zXCk6jHi8IlA+ZgnKnJwECbQrvidOP8m9SkvCATtktu0/oXH9nsK5gD051y3rxlIVu4ITprnDNpy9cTASjtNIITUZWr2ZELtQsDVDLFbf8ZraveJZMrqnt8jdslwrvn7gOaZGLMAyk1MtUAE1k1KvrVLiWCA83/mcXzBpNjVL8OYF5BlqYDEMCfrDfA49GuBxVtPaWkC6f1awRcC9YNHfyYnL36aXFMJxLbXBHHVJgVu6Wm5vgBmdwHNM5VAr0TOQbAzhfOfp/oEJXB8rlBuc/0RWfphyddDcOLGVQLFkefNl5SBNs812e+nxKDuPfYfaSy/dhiWcgTerhJYfjXg+NJA+ojiABQbh2Gtx55SiAlgDmsGGOWJgOoxwtUKzTOF+Y1Dv1JjrVT1SLC4/8hgfsPalyyJVckXrPrU51vrlHRLwDOcceioktffHDwUADdX4ySGXTBoKgNg1QMuhTDZuUT1YNE+KzB716M/Nyh2lLsHLVDfRNiVBiJgdgzNGS4KmINDe1nSY6zoMXYLw+7N1sGtSrgqhWg1fI3sPUQUCfjqsYrFnpWU1ZYMIYpaoH7fJjmsgGoG+GU1BSWFyK7NRQl16OFrAykEYsk0U5HksVGbNClIMCt6i1jXlMceBsRZAXmI9Lm2NlWyBPpKU71JEAKyYQBHrEtEkfymiRmMBohKQXSc8RaBr0NPpjTWJdnJ/H+RfJshIBYGwusJ8DlaaCCyfzqBvrKgBzqvo8y9dUyfjdbSh4kTcCg0hJjYSqH1FLQjBNeTWVGAIJoL+IsH/OIEjNq/+OW/zY8IIGRW4q/g47vNWH7ycXz9n/9thPcVGbF+ksH05xzIHD4JKJK8bjiLKJ94c3UzDmTaa0pa2mcB9S1nTGc3Ec3L5AUaUpfdIsuhGCtf33A2LbNjUZKJqx4ZRFNueSMpduwsLLYRh0/pBRp7NSPG5Fmznzw6cmBEv3AMzckelTwL7ioOINlBeCJPsZxlHdZTAmx1nzoZ08z97DZg/7HE7H1MMfIipdcJtM8iVr/ggFFaLgPA2JkZJQdglHHRx1PfRbRX9Jy4VIZebCJ0xzCh48dksxZf5RsfmYv9p+BAU09F5lEBs5uA40sG2/TnSeI2RPRnEt0lsP6MnhZz4ACo2DJsSR858EE65lHQP1NsObjuz1Mx/CXleSyDBy7+1KO5lik9kV/QxZ7sEdlfbuP8bUjHSLBfciVgmhwQFdFdyHHQmAeZxS5yAHUgw5BDWY6vuF1ZvtVdcZvMka9pXsjEYEayT56Dzzx48yVTUbtLgfm7gOMLOaaB1g8BQQH9irH0w4rvM23E4+/xvBd7Xiv9WU4qlXAVUgfelO5a3weYY8DhpUJ/LjC74fsyOy4t93s4owyw2HPAVt8FHD6S9PjMgcU3AXZBZqtfSRQHyuyGBTs6179wjNRPv4PggMunsCe7IHOWmU67FKjvA7wR6M8FFm88fCGw+57E+oswdSC6PCjiug8fcf+VjWgv2P03u+UA1BuB+omz+8Oc/qL9awVIYPENpXr9mp4gDvw4WKofmUjbnUuYJjNCvKaHhUC1CWPKqWmmQK1cITG7czi84s158c5h94mmZPMYxgFav6JM9viSA9b5+5A8Ysk/lboTuzOJakMmqV+T/VRDKr6fTzex4sCuRlcLdBcCsxvesHOyar+SqJ88js8VVl867F/rURJ3fKZQbRhs5guB4kh2whfcx/qB29ZdqJFFKvYB3ZnE7MEnJhRj6FG/UmO6su4CmmuN9Wcttj+qMX/v0vmT4/d6ufE4PtcojgH9UnJAXEm0lxKLtxw413cW/blGufXo14rhUDN6PZUlO5QH7u2lxMWftDh8XEH3+fok8Cr2Hs01E2R1x2MmLf9v5wo+M4daQKegJlcyoMrsKMXrLwy6M6buVhtep64SWH7T4/iiRH1v0Z9p1LcDjh+V0B0Td3efVKgfmKoZFaAPHt2VSaABaC/4+Vp+06dwMIFQyDHMSXVMu62e3Mh6DUt23ZaPPeyqoMftwLoH3YekWEjpryFC9kyANTsHZQO8kXALhWEhMX9nCbbriR0EADvn+SRYKDCsJGY3Fvpo0T6vUN0PCIVEd2lQbBzsUhMI3lt0FzxnqmUolOr5czhjoFMUQLEdEDSrbYa1xvzrI/zcICgJsx/g5gauVjAHx30pVWKymPCq96xEGM5L5J5N6UIKSCKQdAuF6rZH+7xKoUhkJM3eorsqUN32CKWCHDwOH9eoHizMboCfGeijhV0VU0WFQAqvSWxaraG3Pex5BVcrFE8DQqWgNz2G6xp6zxGznxEg+kqhvGngFwXVHa2D7AhuhnWBYjvA1zqlySbW1E/hV6KnH264rNM2aqijhVsUMJsOCAH2Ygbz2MAvygl4KTKH+pjCiioD2dixJzIsCiBEyN7BXtTQmx7y2MFdLyGSRNTPCgYT5e7MXUufX4zoXyxRvt9jeLZA8XYLfzGH2vfwc4IXtW3h1zXk4NkTKQT8ooTsHWKhGViU/INhUVJq2luEWYWwKKD2fUp0lSPTJfqBgT8JcIVFRf+h0QxHEgJy2xBYlQUTXPNPKSknzcFA1jHMJwXfRKMZKJR+F+uSgDIFDUFKArqUJCuajsvO3Y39wGNzEn4UU7JqBmeirhDbDqIqyR5mhhLgMT7tvwRG9vNX/g3A2Gd5AvxO/ZdjamtmJnNi67frPrRGbFqm4iqJmNNz8755ymWF0ZPP8v/LI8TvPGP58U/X8d//r/+lX/ty/+O/8T/8xvcN+I4zllARv/vqBp/pKwytQWw1zEbBScCfWxyURrywmP14j8evzyCXFk1VwhwE7FmAW0jYlafX6nJAKwpIK9BIyl6v/tYt7v/oGUMNQNYgaiBeDLBNBV/FUWLla0ofu8sIuw6AUBjOAspHAg5fCAzXFvmQ5xlh4QD3aYfwZYWoJ/leTnKNmsCmvgUOn3B9/sUA97aAm1MCaheA8GSAzAHwZUQoBfqPe9iVQXknR5/DdslBzu6HlHlKRzO9W0SYA+WW3TOycSLw781zAsn+gkDn+JMe9Rcl2h/0iLLEcBFgGon+jHLd4UygS+EQ/tmA+v8o0T7j7Gl/RgmKnwUMK4HmexbmkTcbfaR+bv8Dj8UvOXBzlUB/RrARCvajNa8Dzv9YoH1OcNldB5T3koxJS9kswGALgHUYwzJCnvO86paS3/aVw2HLTq/mZUzsj4AvBXY/HbD6kwL7H3jIgTHyugXaVx7VvUR3LRAfCE7NkbP4/YVH+aCgj8DuDztc/MOSfh1JYEsmIKJ9FqAbiWHNc9xdRhQbzu4jAsfXgXLlip6w2U1MExBilAFmdsTWrHLxFUF4ey7ha4H2Ok1WzIH2GpjdSAzf76Cb1BvYcTIhV024OVkmEXKFCX1stlZon3OSZfO7SF4/gvnqnmxN+7GFORgU+4juihLo9lmkV28vMSzFCIKki+jXAvP3DGLY/9CjelA4vhaobwl2fQ1UjwHHlwq6Ze1OsQcGTblWv6b/Lipe+7kqZlhzhj8nNoeUtNlfAItv+Fo3A2Iv0L5IqcytRPM8122QFQkFELRE8xHPy7CQ0H3E4WN6IbsLbqNqxeiPcgue49ktAzhyRQsg0V8INC8Czv5MYFgJDGf8rFF2p9Fe5yRfDrLtAtBfCdjVJH/sLqfexFw+T1mygPQew5xgt9gD/UpgSMwRIFA/IC2XXsPqyaP9SI5yuC6BbN9IdOcEp8Oc+zm/ESlFW6C9IhgVkfUsQQtW3nRkM5sXIvUfMmHVzrhOX/B9fidGBqlfSSzeBvRrMU6I6SbAzgTa5yW6S4FqI+AqiX4tyCBdSeiO58cPPB6+JKjszwVmdwLNM4lyK8dgkPZSotoQzHZnEusvLfqVQnslsXjjE/NLlqQvknTZZglemlAbmFAsHTAsOLjyht9j3Rn30RwIsvNxppc14vBCpc8SEKVCuScbZ+caw0pA9wRqutNoLyTqB6AM/FwRUHOyzhsBWwvYJSeR+nPKYBdv2c/nSwLnfsWwmWoDTsA1Mk2IEETT38nv2+5MjhLsKHNdCcF17mLsLiVWqSbFLRS6tYJdCMzfIdXLqFTD4+ELifaKTGCx07ALJiy7mZqSphOb1K8kdCtxeEmwW9/xut5/ZLD6MsKuFNQdQ1qCFrDzFJwmS5iDQ3+uYWcS6roeE4BDoeDqbM+IGNYGuS4lVAr9mWaycWFglwre8J4UtCKzlVKjma6scHwuUe6SDNkBvipga4nZ4DCcFSi7BF6r5HduUuKwEuguNIqtHKtNpA0QlcKw0qgHKif6NUPAYmKK7ELBbHpEwyRiCU76lQAgAVcpFC29baFUlN+DclBf6fRdp1E+0qfo5hrlfQtvNNorA9V52IWG7D3cXEE1CiJI2IWG3iu0L6qUAiwTc80gGbssuR9lheKhYRpvpREKieKRY5LhukbVWwL5QpJVKyQ9coqMoAYoHQ2cBIk6VfIoCbsiGLSrgpNlxwF2VaDY9Awp8zy/srHoLyuua1FClBpRSybl6gBoCbtIEzANr4NYKojOIhaGoLQbEI2GX1ZQAEJl4Of0XxQNPxuh0FBNR+AlJcK8gtoeecBzUq2SiFF9APCED/DnS8imZ/1JAqNhyaoYeE8ACRBg5p/RsJLk0I5AXBQFcujOyCgaM0lojUY8HCGqCrEj4IyDBYKHWHAgGY89RF2P0ujxvRlM+pTSmz2SYfI0iipF+0Pig1RZkba/HyCK5FtRagreEQJCKS4nB/6cyGNFZivT4zRF9rtMdP31Y3p8t7nYIPDl0zn6+xryvoA68uZKs7ZAdSdhvi6w+fwC9VsN3Jcwe4HyQaC8UQyz2JANEE/F6C80e/rybj67YjBFQw+g2ZMxiI0eQwjKTZIVJo+kdAJ6L1NSITsCETkwlgeGfhRbhmXQ/8SAFXMQ0AcGTphdirvvJ79YTuvTrYC6LVDfsnA6y3yqB8rXVEfAqA+AekigEhi9hcWGh04f+VqfwgOyDxER0AcyZvrIY1k9ki0qH1M1xwPDX8RRk8HSDFaQXsDNkxRwz8GRLt0IoIDshyGwK3ZA9cagvuEx4fsiiieVM3Bo8D/kAAlKqfQ+9d85MqXlA2fvhScYUy1vdsLzn9mx77N8SiEiLUuy5dJCH1lNojqB6oEeIWmB2ResMahuFYTne8cQhYaBJrnihEEa3B5pgeE8Qr8tRx9QKMhOqS550vTx8toAACAASURBVBKAzwy48BglYlHz2OiGgS8mJTMCmPxLiem0yzhWm6iWQCb7+4odGVBfUiqn+oh41PS8pO91BjtMLLhukSYmuK+s7Ejl8UVMwTJkhlWH0c8mj+rPJT8yoEOO1TTBJMBzlj8nnDRRDWsv6psE+B29lGNdio1pUJ6WnSSg0k/HtnpiAEeoeCykRQr+wFiQnR8x+dyiYFiLGmKSwiU5kUqerp5+XLuIY9qpOSRfnuT5yn7B7JcUgSz9eJ2ksBXhgNk7prpKT3AeFI93uQ9JMpeW5TECLZHCQXyVPHDi5BhoJB8YK16U5bUQJdlZym2ZUjl5xTh50q8ZRqTT5591QyLJ/oB+KcdBq0whKjKnn+ZrUU9S3PFcpcROXyTJa55UFtl/KRJASlJezevLzcTIKopIBjYfw7zsJk0e+IIgMic+qiFMNUplrvwR6fMfR3+rnYvUg6uSV41Ah0A5pnRLbkuelMgVP/0Zkyn7pUiSYzKEbsbjW+ymWpt8XwhGcMDextHnbFp2bnJ7eSxGr27yMYoQ4WuFU090FOy4jHr6Doo6r2fy9uV+2hycQk+rGP150uZ1kGkMhhN3rN/J7+U+saaK3bu5miR7OtVAEMR1Y0xLVUMYPebCs/pIWiSJaEzfAxJ2rnn9pA7a/kJQQhl5jFTrUlCMAiJ9v7lX0zQOdqUhPSfbTOPoT1QiSQ+B9lojGgnVcfLDzdSYUBoUQ2HGii8l4CuZAmA4eUfvdBhrsURIctx0nFwGWjUngnI/pVsWCFrCzhV0G0YfpAgRypIZVT2ZxCy9jZqMrJ8ZTiKkzklXyyQLBcKMjKy0ESH1YiLycxwNE5Kl/dCDys8O/x8LOfo2+bmTSbIpUycm60/00fPz7HjOVb639IHey561KFHK8ZwGQzCvGnZg5tqPKAUQwPqaGCF7D9mQ+YuGkxH0NQpAcvtDbUa/eP7sh0LRd1kZQAnESqeJAkPGtCErmL2PoVBQqVczH18OeNQot4xVMTGZmYxzqSYmAxtNhjEqRWDnybzmkJtoNEKVgF+MiGWBqBWiTumtCTyiLBLwPDknQgCJ9YR1yEmryLUcBWtL4jCQtTytcHHJm6nJboq65rKqioCuLBKQ5P6JXIGSwa9LLG4O6okRKMsP/omymGSvSnG78nrz7zJ7KQT/FgIBYupF5f+/5bWUcpTj5n/jezKgDf83/zII/jZj+h18ZCnsr/vfb8vjOy2FffX7Z/Hf+K/+dYQo0XqDp6HG28Ma67LD89kOt+2SNRV9hYu6wWM7gw88+Fp5PJ8d8PawwrrqsO0qOC9RaI9jX+BqcURrDfZtiaqwUDKitxqzckA3cJbGp28vJSJckFjXHR4PMygV4JyClPy5mndQMmBwCrPCQoqIm80SUkYUxqFpSijtMasGxCiwP9QoqwF9b1BV9MiVxiFGgbY3KAoHJSK2mxmq+QDvJbyXqCoLozx2+xpCRgQrEZ3E2dUB1nN7vJdQKkCJiLZnOmnwEqZwmFcDOqvRd0xumdcDQhQIUaBN23i1PuDuaYmq5nqbTY1iMUCICNtrvLze4t39GkXp0LcG9bxH3xsei0FD6YB61qNpSvhGwywG2KZgrYWTMLVFUTq0TQGlWA1he43n16xGub9bIQYBVTn4XkEavi/2EqL2KCpL9joI+vtKz/7eVkGWHsFJIAgIFWFqC9saVIse3aEEZITSAa5XMJVDiAK+0VzO3HE9nu+NnYJKXtQYBISavuykjvA7A1QeqgjwewO1GhCcROwVxCARSw84eltFqyCcYJhUESB04MAyCIZQ6Qikugp4MQVSzRwnOaxANGkwLiOgIhNvTYCwEnHm+X4kEFhERB0AFcfli5YTLNGkkKkZ/Zws5BYIM8+aEyugdhqhTmZ8HRlkFfP/T77cZPygFqN6eUS7LyG3humyKtJ3q/masPAQrYTqZKrboKeSvW6RKa8ujxgwsvChDFCtHIOvoiFbCgCyEwh1BDz3g71600THNFDPoFQw5GmgnyvoOIXk5OOeQY+bQGQoImLBgYrsiQikTRMgkWDSVymUKmLyGs6ZwOpL7mf+O9UQZL1UywkbmY5zDpERafuzukFEjBMjOZCK25aAi50Aaq5zQEyTTgBCiTTg5LpURw9lXmaW42cm7zTohEFI3E7VT/5OkcLPcgjVWJ3heG4ZDEa5ffaAjkFQPO0AADsny0tAye1jgiz3IYej5YAoAGOwzGlglerSeRdJNWII0nKaaE6vzLYDXhMYO32BaZk+TSzk12UFSgazBFuYQozyqtN6M2geA47MdI5i6gvOgVB5G8xx8lpmT2g+TqeWhSgx1l5klYNuMAZT5TChD5aRQG6+NsnoU84cU1gRO39T+JWdzn3e3w+OrU7vr+hjhDyZnEjAP7923IaTkB1XM9yM74uTUiNNyvhqugbHrl1M5ymoKWgsL3vsos32kLzdguDR7NPnPab9E0jdv3yfstN5Hz/buYdX5EkhgnMGrHESY1imgLkUKHQaUpRB36jqOFlflAxQykmwvhRj13CU/H7J/cinn5fceZsnA1jdgQ+Oz2moEYCRXc0duHzt1HWbO24Z6BbTdZzqPCKS7/jkebrOmNZKhYrs03FJ/sh8jESeAEzBOeN5SRMA+ZoQjn2n3PY4hmtlth3A2A8sPKszQupHnT6A6Weczkd+LS8m9nmKFEIUpWBAUKlHmXHUktUgrRtfA4AgF2B4UAayUkzpryfhP+K0JiSH8Jw+z9+Vuev11P/owygr/pX+xBx0M8pZ+V7hf8U6AYyVJXnZ+X358avWkwN5cmBQPNk24APp7QcS2NPuy289/lJ45K+AFPb1T9fx3/v7//Kvfbl/7/f/+9/4vgHfcWBZ/+hV/Jv/5b+Ju0eCNFM4tPsS1892uH9Yoqgsnq0PuN/P0d3MoS86uF6jWvTjAFc9b2G3JeTCQhuPYVtCdJSORh2BBABE6REbmslRBMBKiNoh5vTZo+YA3guo1QB/TJUNaeBsHiXTWjt+CdprC1V5xLsS8WKAeGCqbZx5yMojHAz0agC+qscBp/CAPQswTxLDJcNl8oAv1AHqkGaQLyxw0Azm8QLFrSZ7safk1S3D2Ivny0iQoCL0RkO1Av1Ly/1oJFQjIL2AncfxZuuriPnXEvsfe4ieN1TdMAFXH1Oqq0uD0ipArS3Mz2qmyZYB5Y1GKHgMym8K2GVAWHqoLfv9ZFpmMGl2vQTKJ5GSTCN8HVE8SQznTOLN/XuZBbbnDICRjiwwJNC9tChuNYZnDqLjKKbYig+CTcgKMy1WNxzEtt8foB/MOMARnjdku4qobgnGfJ1SdFsxDprcnDJKX0UM65j67Pj39qVnBUY6bv1VQCgi9F4y/XUWUexT6EcnYHZAfxnHDkG7iigekzRvHVOMvEgeJA5wh3WSNs8YXpLl3P1FRPE07bNuRAJnaXCj2NVHxoLvaV97VG/pNwzmZPCfMaRMA2pHqaluMcqiM7sKAOUjk0rdjAPMqFM/3hHorsi4+4rX5PIroL2apKY5jVcNZFdz2u6wjqjvBIblFKqiWg6O86Aagr5fuyTjrTpKeX1iaYPhILPYcl9yuqocctcekpeZA/uggOZlTGoFbl8GCsWGnurcXafbdAyfR+iDGGfk9ZFy49lNxPEVz0GxBZpXEbP3nO0XCfT5iv+3C4zJvfQRpxqNlCw6rKlcCJr7lgdlquUJsEuej2JHua6bYwQ/OvX/5bAOen0ZvJI78XSTB8gxSfDFyIC5On0PpGoFf5ICm7en2McRWFC2S59wfUef8Ow2qwcIsqXlsfYlWVi7mJaZwd9p2qoaeIzMMQ18IzAsqE6waV+L1AM5LEQamJIlo2d2ApfDmv7l9poBKbqdwFRmrkzD96k+hSQNZOmU5UBeuoj9xwrFdgIruuG2FAeeE+GR0oBjkhwj9QdS8pqTWbMcM3eOZkBijmlwLsg+5n5KmRhDNcTxWPkyfUccwwhKoiBLWz/Qm3s66W2OAc21ouqj4Xqko8928d6P22Dr6VgIj7GvERE4PldYvnXwhRwHshkAQFBiXO6SjPZCoNwlz7AWKHYeoSAY6S5VSh+m93VYkRXMIC3vpy/5GVM9GTwe4zix2MkPPE7ORAbn+FrC7D26C53YeQIWO6dUtdzTH0yWncmzZu/Rn2t4I1Du/AgEVB/QXTHgJPvqVfLgMjTHJuZWwRzpa9ZHP1lflIBO1Rqu1tAHi1CSGdSNh/QBdpYsNRGQA/dVdx5uptL3jh9fn9Nex87JxtMretcjajl6Uoe1GV8jIsNvMtAEEsgd+x6TdDqBKp88qOxv9Ailgq8UzN4CIcKuDIrtALvgT7cwkJ2nv/Rgx+dRCXpSbUDuhkSIDNJJ/ksIAf14hDufJTCeJJYuAD7CLwqoxI6O4E5LqMcjwnoGuJDk+gSHUUnIwY3XifBxTIeNRjE1NiXDCuvJvA2OgTtGIxoFeaB/NMrkqdQKUUrII28AYVlP73MeEILe0EOOm49p+anGI7GKwvkRtIrM2mXvpBBcVk6dLcyU3JpBZ/aJ5gqSfEwySyrE6OX8do2HKMzkj9QKMTOnWVabvataIbbt5Ke0bpTCjp7L9IjWQWQ2NwNLyc9p9P5DmeyvAJ48VBGx73/l3/4yj98WYPl3/v6/8mtf7n/6+//db3zfgO84sFz8zov4/f/i32GXovKoC4ufnN3irl/gZzfXeLY+YPAKt59dQl93sJ2GLj2U9gip/y8kBtO3CqIIOD8/4PnigM/eX8M7iU9ePOJ2t0B7LMk2JYCplhbr1RGbzZwMmglwrYaZWYTIO70pHNbzFrd3K0gT4DcF5Moiev49RgBBQO41wtJB1Q5+U0CfDXBHg2LVY9jzy1SWHqEje4YIqPWAuh5weJpBHBSKFw3clwvgVQe/MxBzB/W2hHtmIWRE9AJyp7H4/ha7xzkwSDIwJkAUAdhrsku1g7wtEcqAOPeAjJAbg7B0TI/VEcWdxvBqQPGmgP9+B/llBXvtyIYlcCxaBXXkzefyD25x88UVsLSQdwX8hYW5KWDPHfRWwy08t2XhgF6h/kaj/d5A1k3n2S1A7xXchYPaaPiFhzoohOsB8t7Arz30A1NH47kFdpqA52xAfCgJiFceoqN3FEhVInVEmAXIJpXdJxa62NKzVz5I2FUYB4bFExNT1VaPUf12GaAP9DblNNrqnUb30pGxGShtHS485CBR3Uq0LzzMViZmQsBVZNmiAspHyfTNjYRdRvg5JxPcgmyQ2dKnhwi4tYdJ0mE3IwuoOgmzS57bPQd77YuA2TsJu2Dg0ylA9BXZNvMkoQaB/pIez2C4/PqNHkvK83qjBoKOKDYS5YaBVvYsoH6j0HzisPiFRlAEZHbJ4xdNgD4ojDUyvUB1L9Bes5g+sxZRAnYVMHsjMZxTgpsraljonSTXNjGBNWVx1b1A+5yVNKoV8DPOqhfb5K09Th7VYpc9lzxGoQTsIrB+J80PzL8iWPN1hC+B2VuB7jqi2KUbvSMgVj2DcuQADGcp0CuxSm4WYbYC9X1Ed0GQMz4i0L6ImH/D5R1fEazqhsBVDkmunsC36smGZFYqM3Z2zhRbEcDKnoGBUUGnbTqfgL9uCHKrWwJLORDwtc/4WSgfk5y8JwDdfR9YfM200bEOpmA4Wn0rRvYNAObvAroLCTfDyPzJIQENR19gZhTLLY+/SlJsEVJSa8/Jj/VnIEB33L7uQqB8SgFhiTVTHcFm+UTJcXcpUGyYLFs+YiySj+rDBFmAYLy7oi+5eSlQPmC8vjIjZFPwFesSJmYt7zMnTehrH9aCgWJpAseX3M5yw1ArXxLMuhkl0lERlLuKctNhTVl/+4zrKjapLmZksMRY3QExTSb052KcHGC/ILfNzgkWu3OG60DSt+wqBjZJz6C0YUEw503y5MrUu5j2UbcRzTMJX/HYikiGrHlGwF1uU0VNkmYXB36H2ZqeU1fyw9Sfcf98yWNcPwSCyj23aUjhXsqCvt3+5BryqZIlJaxKz/OTw5YYyjQx0r7EKEnOAV65oidLJItjSMFY9HxmVi9PqtiZRP3gRjaPKaes4ujO+EUlXUT16Cmv1kjS7NQhefBoniUptI9wCYjpNozBUaoPGBbqJMHVoXlmuG0Dv+/MwaG9Lri/KYHVG8G+zY5VKaP8ODBoaDjTnIBKfkkRIqrbHt11CXMg+Cx2FsPScB17h+FMo9i6sQYlaEFP5tpwPYbLGZZJsnzwUL3HsDJQLUGxXaqxuzK/PktmCZJSKrdnz+SwMqjuWvSXFczBwc01zKZHKDV8rVhrMjeUKvdhDJAqNj18pZNEW0JaD7timJD6P9l7t1Db1gQ96Ptv4zbva+21b+fsc06dqjJtd5too+ZFJEHykhfFBx9EMCgoiAEVoX0QRVCJDz70o+2DGNCYQKKgPooKghHbRtJtqk111amqs+/rPm9jjP/qw/f/Y659uqpb0lV2nTYDNnutueYc9znn//3fzUXIkfLK0BnKkSUnl8s4ImU21twN8PMKejcSPGoJv2khbIQcPMLMQB0cPZohITQEa6p32YNpyWy6QMmsEkBIkEOW/GqJJElSFBlwbDTkbkDq6iwHxYkpzSAxCcGQn+xfFBkQiuNAiW0OxpkSX7PfUwyWr5GCPytJ76N1J5bwIVvpT35Pfh99BQv4LGctgPErQE9oTdD4UJoaIp/7QffkAwZTyp8c4PPA5/n7Limekmb/LpafB2D50S+t07/y1/6xn/p6/+1f/u//yI8N+JoDy/qzj9M/8p/+s3j5boPqB/U0S+9XrE2QY+6oNCz+Hp54dD9iTYHec5DXX2SpR+6MnH1ZZvxOYTksh84sTpWmQSr7FbNEaY/JS5gEAUb3ij4Wu2KKrF3xeeNZ/tLYZlbsaURzKXMMfvY0lX4wx8Gb7jH1Ek7sTnWSxenhgTeu4UD08ElgHULex/oGUxy9GoFxg6mLL1ZkGQtIgMDEbNV3meFoH0iWEiZ/1XARsfhCYlxh8qMVtsd3CatfuUL/P17wtYIJovuPGZld3fK4CjNU35G1KEs55tBkiV9P5onSudNAIikeM5MjI3afykkOxuAhDizHM24nVARJ3WuB0gFYzuXyB5EhMnn2OzSY/Kj1XcoJs6fHleWA8AOZVqkmGE7dhfNXTNC1qxN7Ud1z8F7fCdQ3eWAzsLz7eCEnuVnxT6qcUNy9pTdJHxnjX99xZtzsmHo6nMkP2Cc1MH223M8QvBe6NxzsHZ+TGU2SNRBkozhYtSvea+17DuzdDFP5um/FNKArDGRzzX2K1eleAzIzqDmwLoyXb4H5KyYBz14zPdauBBY/5KCX7zm+T6p7yrn2nwicfYchKP0jgdlr3hMFSOmeg05f54CXVmD2OmL7mURzQxahz+e2uk/TgLrI/0KDDBIEtt8ENt/hYLo/I6Nkc1Jw0qxZYMCQyImjBBn1Dd8zAAfqbpalcjbBrplMLR0wex9w9y01VT2UdGI1JNS7hGAwMYb9RfZdjjm5uMveyYHeMbvi4Nsc+BlS0oKLVPHwXKC+5zEDfGx4xITiJIF6S9lrkgLVIeL+GwqrL0IOzcmD61kJHToteuC+hFagvo2ZQeMg+/BUos5s8exNgJ1LVIdcZZBTWeu7eArbeSSnJNr2hv2MJ7aabNZwJrMfl9dyXDIgrbmJORTIob/Q0MPJ+xhqMYGRmAPRAO5ncxs+SLFt7rifxYcZNSs6ChOrMqApLJnpmRhbfJGl47EwaAyRArr3Hoen9DkvvrQY16xwGNYK1T6SoWpy8vCS2/etQLVnem5JhHUzvh/NgUmz+hAwnOspiE0GAOmUGlyYpSKXLHUVEwuawbT0CeaQwUvFpFsAMNucLLrM4VId03h9R6bNLhWqHaWOUWef9yFi3KgJpLfvRvRPako0VU4U3oYJtKshwq401BhRbT2ikdNAk3UQAqqPqLYObq5zbyRQ3zEwJmU2NVT0JppdgO6ZpKt7DuBLxyoiMJwp1NuI+oYzZqFWUINH/6SBshHN+wGHF12WkIqJgeZ3MoOM6luH/lGFahem2gqRgYLv1OTpjLUgaIsJUTHtlqnlBEd+plBfZ5CS+xKTkVOSLf2fmb3s/cRgublGdWspybRxuubFV5ikgLkdEFsN3+l8LyTI3sMvKqghPOgtzSE7VX6tEtD7nMZaK+ijmwb88ujgz9ssESU7qXoP2TvE1uRryn7J0jsZOg193SPVCrJ38OuWvkojofYj3FkHfSCTiwTIwSOZLCsdHAFhayaQIgePsKghB4/Y6FNH5+Dhz1uovYVw4cQ2AhCHAXHRQcQIYT09nll+Wno2hY8nltIoMowlhKZ0UYZIn2KITIAt/ZRaIXYV1Ps7QCkmwvYjUPGcoDITOBQhMrSnJMkOdgoFSm3N10nxYUpr2Q8lyURqTSCqFZlNJSEmVjEzlEUyKwTQ51nNzGymGOl5fOid/CpgEwLJWkDmv0lx6rH0no8DDA8q1SOFTYyR1STOnXoyczrsB8vDjksh8JOWh1jl685Y/nEHll/rVFjZC7z9zaeoB4HZy4ThgoOkca05oM1hEnYJzL5MEF6jfZ8Zh8SZ8O5tHmCds0Nr/iUTDe2CM4Hnv8UUS3NkRUL/SGL2MkvMBs5QlwQ9NVDeNa4kRJLo3keGYawEqm2EOXJWe/PdiNtvczbTHDjwXfwo1xac8zlJswoCibO0JsvZ+jOJ5ffJAHRXAVELHC+Y0ugbhniU2eLQKqx+N+ZQB1ZOJMFqBN0nNDeUipXS8WofUd0HJMFOxfYywhwTpE2otxLbTyWWPwi4/BWJ9e8Au08FVt+PqO4E1t+32H1sKM3Ki50LDBcC+//1AuuXcWImqm3AuDEw+4Rqz1oG3XPgqYcIuzRY/66DmysMazGlOo4rgcVLj2FDWdT+GasPXA7/KIXnzY2H9Bp2LrB46WCXChBMH0TiYKu6z6FEx4jZW4ftJxVKsXe1jwiVwuxdwOGJZLdcTsHUY8LsbcKwUWh/6GHnigMwkdMSX/OcXf+iwvnfDkyBPaMhrr3y0IPCbSux+JLgq7uM/OLuOfuvbJjKqfcvgIv/M6G+8xg2+uQ32/IY5q8S5l8OuP7lFouXjIGvdg52aZAE99U3QHvpCSTnmlK7Q5w60up7SrHMUXIQVAGrH9gHqYkeV79cY/ODMDEMSCmHljj0FxW2nyisv+8wrhT6c4nN7/S4/RNtrpFAlroJ1LeeJeCJs+TjmcH9NzRWf+eAYOZo7gKS1BA+YfZqAD5qoMeIYaVg+oRqGxAaAZEUZi8HmE0F6ThAbO4S+nOFxY84UPRzRTallhhWEtU+oHsnMHvjoGyE62o0dwnd6wF2w6Amc/CTtyg0NGDNvpRoLx2qrYNvWnSXHoCeJLbdZaDMTRl0lx7SJeyjQXfloaxCc+PhZwr7Fwqb34mUUG6B1Q8dDo812kuLw9MW0iV07wmMlCMTAgDV3YjhosmDVA1zTDDHCH0MsEsNcwgwW4tYK9x9s8HmO0f4hclMFweTxdcUDcO2lj8aMG4MhAfqnZiSZru3liE1Zwb1jUM7k2iuLUJVo9qGHNZhUN8HtG962LOGQUeHgMMzg2qfMHs9AinBrQykS6j2TPSUIaF7O6KaUdpXEh+la1DfWBye11h8cYD0M3Tfu8W4eoT2nUVdSdiVRve6x/FZi+bKor0S8C2TPOsbCzUYhFaie91D+gb1+x5JdTBbh/FRhe6dyzJEiWqbZYhzjeqO9QtmawHRQPgEt1Co7j2CMZi9HjFcVKju+d5Sg8dwUaPaejTXmEJxyncBEkGc2fIeFGPA/S8sUO94LfUQMH+dYJcK5mYAUgO9txCxQX05INYalRaoXt1Dfn4G3fsJTDB9PEGOkcnVjWRi6I2FvjlAjTOyQQsCEX0M7Ah8O5I9yoN7NQbo2x66bzCeVTA7Vn6Yvc+hORG6NVnentA/qaH3Fmo3QMQ5hE84PqvRfv8GcdFk3xkZKEjBFFcjofcOaQKyDnpv0cUEaQkmhyctqnt+Vpl9gN6NSKrjuvZkWOS2R1ySAhY+IswqqN0AdQ+ERY3YMDRG7S2lz4cBYTNjpdh9j9hUaIBcxSEQjYI6WsRKA7KirHT0ENZDDgpy8Ggzo6Mu72E2da4MCxCWEyaxVqje7jC8WMG8vkeSa57vux7VcUBcdoAQUL0mONISblVPxyQqNQEupSXM1RFVoyHvjwibGdTdkTUaxxEiJrinK+irPfyjOYx30LdHAJRoyqGC3g4EI/WJwUNgEmpSEup2B+E6JCOhr3uEJTsYk1HQd0wgDfOa9R6bjuxtoGcuKQnZE1Cq+56g6WYPOA8tgTCrYV7fIFysWDFyf4AYKsSuJjCrFfTlFnHZQfkIMVqIIUH4ACUlOxhXHeR+gOoqyLsDsKK8VV3dIy06grfR8hhtQyDXNRDbA1Q+bnXfE4iVa5fPZ5GSiqHIbRxUkZPGBCEZwgOjAUMljbjfUwIaud20WRIMzlqI3YGgymjgvgeaGhhGprEeeojKQB0N0jBCaE0gdThCiFlxyPB4RAD6AaJrWX2SuyJFUwPOQTgC+uJdFEqybqSu2SmZWT2hNdJ2B1FVSNYyHdZ7wHuG9miNeHsHMZ9BSIYB8csxB+CkhFTOB4AfmwrrPdddpK2lRgRAso7nqjCaAB4G9iTvIbLMFYGsbgoBctZ+wErGDBLLcfzY5Q9iMb+GS/g5Ctv5aS9fb8bykxfp+b/5r0EfBL1zVUDqNUQTgK1GqiPMrYY786jfGIwfWQamAEzw7FlozgcoBxRNgLijBCE21PkLKz8I+5iSNSMY2JE/NUoAi9ophEWAyAmxJZkTkf5EfaDXMZ1Z6Nc13LmHvqGMUwQgdBFylFnOQ6li805ieMKZ1xJc4Vb0FibFigcGW5AJsY/I2pYie7IrAscXHs07DbeIYOx3yomEgqmvBhg3cWJK6VkT3Jblc4ZzgvPhcUBq7jVnIgAAIABJREFUA8x7w3MiycKO5yxRHzeRMfz3rAYp0lMmgp4kcEmSMa3u5BRe4ueUOipLX6HqKTc8PuX1KpMG+ijgZ3G6pnoQEA6ApCyQEieBcRNR32ZWIcvZ7Irrai/JnEWdYA6UkLL+Axge85zrnsnAIntMIZBrFlL2sfE6Sc8JifFRRHUjJ4ZaBDGFdfg55Z12Q7lraHIwS77Hqh2Zt9CQndQHyiGLd5Opp5Q7Jgnsvu3RfqkROqbA1jcnlqf49PItDrvO/ZmBqbx2BcjxdE5EPLHHdk0WuyT5lvvcLunNLD5CXr/MWqqTfzJU7Dn1nZgeZ/UOphoWAFNqrnSYpKfNezFJPvXAY/AznoviB0X2WM5eiklWaddUBZS/FzamXPfiR2uuyRyKwImnImUtjCuTP0/HXEJPyn77ju9/nY9Z7/mYdMhyx+xJXJ2OgwxvYYceHHPe/kOpJQMukNMzuT41njyqiJlhz6EqJaGXfj4+p4T6FH+k7/I9JouCgY+FmtuT40k+6BaUDZrDifmyS8p0RUhobsk2lmAJPPjsKfJNft6VlE3uf+m65fs1Pz+mHPBBZt13ZEwKW4QE1PcRwzlTL9WAXAmT5afZ1+s6esPUgFwPkT+jNaaQGqYXk+kvCadFylmutfRkpekd52dENFQY2CUVBeW9EnP6rswhPcryeHx38iuqkc+THjmZNnsRsyol5endon4oCa2FGZ7OMbj95jbCtVmiGThRFGox+TuTBKodWeAiBy6J0tFQrlokqlM4TZWl0n3Kkkc5yWHNIftoGzGlUBe/ZmEgJ98ksme1zSEz+fOheF5d7hMFGF4zKXy6k3S2MNoilkqgBDcvoSgfMs4i5BCcfK59c6rombpHHyRDR8OE3SJxLZ9f9NEG9BcG5sBeSz/LtRuNhB4jU4fzcUpH+Wp9H+BmlNNKmzBuFOr7MPWJurmCGuMkxS3BMvQ0avq9F5xMKj2iADKDRhmtmyvUN449pfpUi1I8o9EwVRUAk7aHyC7SXO/kOwXhE/QQ8iQDmUu70tD7B97OzAqXlFzp+T0XMospXZqSbss/dWRaqFtqvjfHgGBYL2L2uYdVCkgbMvvJ/loRIitTSqiNEB8chwjZYyn4t0kmmcerhSVNgrUmam8ROqa0ipj/5dCj8t1XpKexy4mylvsuXKBkVWcPpBBMrhV5P/L+ycOIOKun8B7h4+TF5AnKry+LyFUy2c8JgCC5MJzW8R43+kM/odFkR/N5KKnPCNkTqVX2MCcm3GavpLAOKae5itFiqvlQctqfIpstSbPJOoiuwQdLuQeHkaA2RP4/he4U6S4TZtNoIXLK7uSrLPUiDwOHHvZuPlgmWeuP69p8+Lyy3QyK/26XnxfG8l/+q//4T329/+4/8N/+kR8b8DVnLIWJ+Ojvf4ebQwe3r/Htj97juy8f4x/91g/ww+0G1/czPP3sCu/v54g3Gi9eXOP11RoAmPa5sZAiIdxVaC56zNsRVy/XQB6QP/r4DlfvlxBzhxQk1LsK6UWP6CWkjpAyQqmE4b4mYJ17SB3RPtnDOYVxV8OZiHozwF520GcD0nUD22SJURXgnzFRNXQSUABWFiIKNE9GNJXD3XfPkEzC8YUH6kgf5mUDX0fAREQopC7A5dFJqhPVEzpCPR3hthU/sEdKh8y9gv1Wz3CgO4VUR6Q6IFoJv6QfMLURejnCv+kghwwwZYKIAsfnAVg7uFhDX/TwVsMvIsN0zgPchgBvqAMHab+4x+6yg7ASoYnA2kH+qCYYlGSdkQTiLGBoIhbf1egv+IUmwEEH5cQRUIkpnxcW9XcajM85+5zq/GEVgdhICA+YrYS9YLKo3kmkOmE8j9NAAgCwtqi/15ANfcwPtNAJeh8lMJ5FxDZg/viA4xdLIHv21ChgzwOa1wxFijLBnvP18khPhXp6RNrOsPtlx9TXkdJptwkQbYA/FxC9QqgS7IWHsBKyp88ztII+wT95D/e9JdQguC9NhOwl3FmA7BWGR8DslWTNjgLsJqB5r5EUMFwE1sZ4wPT0ZErHtE4kgVQn2HVCc5l9nC3l4W4dUV8r9E95rGbIYNSdJlP8MmapFgfFbh1QXbOWpf/Yo/uhxvgoIswikmJ0IsEF7yXfEWj7jr2wzRWBkEiAn8UcvCNw/NSjulTwC4L4qAhAWAmUPaDzCD9TlPO+CFh8VxGoZhCXJGBn9HAWybkagftvAwAl6rFKsDU7QyGA4zP6YSGA5i3DS9rLhO3nnIToH6cp4VVccYC3/yxPFqxykJMV2D8iEOufcQJI9wJ2xevYfalx/Cxi/R2B/nmkZN0Bxxce6ihRX8kJfAKcYNj+QkTzljPrpWtUWo5rVA8cn9DvmQSl2MU/6ltKPkNNUF7dEkj0j3OK68BJkeqWILBfCXRvebzme8Dus4T5lwR4/RN+ofsZFR7KFg8a97G5JBAdlxlMBeRuSKC9ykDGsZu0e0/WPLSUdibFLtX5q4jtZwLVloPr0ApsP+dE1erv5MqamL2nPUOJhkeUhw9nAlWiemXxo4j9c1YRHT6jJ7S8//snQPtOYFzRi9hf0IoQaqC5Irit7xL6i9PxASl7TTnQCzXXF2sCV7OnCsHPBJorSqT3H4vpvVPdsRfSd8Dyi4jDRxLtu4RxKbD4MmJYC+w+lVj+gLLt2etchVNxgBqaLL+3rEBRI/dx/1yh2vI8HZ8X6bnE/iP2qo5rAvDhEV+TjgTjSQrU9xHHxwSRyuZk3ZArYI4pB7xlkF8B/Zz7P3sdkUzuS23oIbbz/P55QrVOsRPUtwnIFT4pT5gM5/TBDo8EzI7bPj6S7Id9ydepkZ8Jh6f0eNZ3kec+JKRKZt9m5MSAIZhOHS0Zs/cBvqGMmcE2BK2mp0eSQXA5BEsJJBkRGgbx2KcKszdUWdgMznutUW8jjk8U5q/CJN0ezvRkRQEIUMeVyiFyWQovJESdpboZ0PYXFY6PKftOSkD0VFzU2zhNhsWWHa6zN/RN+prKi1hJTjA1lGOPC4nuigB4XEqqGg4Rdq3QXDsyz480qj0lxfWNQ6gUq4RCQv/IoNoxEdZ1rK9RjnJo6RL0MSK0/NLUhwA9BIwrk8GbwnBuUG0D9N5hPK+nTtrDRzWUTehe9xguGjSXA/fdSNhHFZorhviYewu7MZA2Yjw3eZsBvqEqobqzEFJgeFTRw5kAsx1hz3LvZiMngBpyqJHqA9PbJWtoohQwtxZ+1RAwZsAZWgPMqPAItYLZAYjszETKDLuivDjVuQ1gVrFnc9OgertDbGp2bPaegTxHh5TlubFSZKGXFTea18UQoHpi8qaEWEu28oMgHqWQRESaN5D3B/oUp3TZhLieQd4dThUnKU0/P/RlllAhSDmBPPGww1I8kN6mdOqoDPEDmazI1S3Jg/JXIQFn6e0UBIdCSc4LlG7KEjpUgofyNrg+nQGpwu8HF4UC9zWJP5TH8udhSQDiw6jwP2bL15qx3PzCRfpz/9k/DR8l3h6WAICQBD6a3+PNYQklEgav8dnqBr/99hmW3QAtI5b1gKvjDCGHuByHGueLA3yUuLqbo64dxtFgtTjifjvD2XqPwWmEIOGcgjEBTeWw27doWgslEqxXsKOBVBGbxREhStzvGzSNQwgSKQnY0UCIhJQEZjN+Ex0ODZaLI45DBTdqzBelmkTDe4nxUEHVgRUmTsFUHsEr+PsKZj0iJQFtApxTmHUjdncd6pllpYiMGEYDN2gG+EQB7A3Mox4hSEjBJN0QJGIUSFHCDxrVzMINGag6CaETZBUgRYI7GDx6usXV5QJPn97h6nYBbQLGQwWhIqRKDHOTcaLMlEoYdjVU45GCQHQKuvGIUSBaavSFjkijAmRCuxrQX+cUjLztODKVFAkMMtobiI4fLu1sxPG+BUSCkAm4N0gNgTciK0ekTPDbCjCR6xs0hJUMFHpfIcwi1NIh9IqTBJnZEQ3poOQkH9cR4qggH42ItzVSlQ34TeBzEge9ZmnhLxuImOsvRIJIAqmKZNaj4L7uDLBwZA8GxXCmJkDsMmAWgLnWcCsCHdEGCB0RjxpiZP1GzM+TPUFt0gmpCUAUEE5yssAkqKOEX3smFScOkJPM9SISE0AXIweaqSKQB4Awj1OVh/CZ+crTwakJBMaDRHw8Qr+q4c7I2AsvpkmJ6lYyfReYKiP8giFMSWe5ZhMBkaYEX3lQSHWC7MXEiAGYUofLPkpHZrtUipSFs868luBpyqxxhBwkzIFstrTi1EcZMO2ntGLy6tpHAdWVgp8nxHlgL+0g8uD/lCQc2gQ55r7SOd8PYRGht2Sn6UEW8HMCRbsuPjjkCoQEc8/nPozdpxeNxzlVbSAn9IrMhO8ehCAJINZpSvB1czKJKgNc3TPkyG4iu3iPDDxKEtPjzfvTNTvVb5zAUqx4rJBM85VWTKxuqSRJmus0e3Gq6mgIhkvQSqiZYlwST908JxaDwE1aPmZ2p0TPWOegpNzjTZDJ9QInFrewfw+rMR4yw4h83ZRgnE4APFZ8Tspv7Um5gtPrWatxOj/Ff87+TEzXlvcTTmnEmQH3HaZe2MLuukVmMEeex+IxL4qOwhCrvrAxfE35udTJfFVFwAAmWi1Kd22pSaEKg4C5vNdCw9eaPdnmUttS7QiSJra1gEZNpi5J/nxKSs3sV0OQa5enQJ0SVBNyh6Q5ZC/jg37Y0rfK/IMHXl+Z2fCc4sr7s3QpYnovlOtVKnnK48XDPlXMqOylz4FGbs5JGT2kKSRI2RwmZEtab/pgXfx8Exl45YTkzKoWbzEnTE8/K5eDnPaZZc19lK7LnmQF+FYSPMvTcUyfc4XVEmJ6bTQEeCm/Vo30qJe6kdINaucS7VWA7yTXKZh8G4w4VXFEQIaUWcsc4hNP6wAye509s6zPEbnPM00dmx/U0gAITa430TmRusk+8j5MQT/0FucuzVyJUpjyqJiaXNJqSwdtqR8p52ViTfGQFc7fl+J0nHIMgOTnlBrDgxCiwphHSowPFqE1kC4QVNowMZncSAYNmSWFLoylI0AtwT2Z1SO7eqoUSUpNoT4fKAJc7pMsSbH9iFQZyna7BmJ0BKtNzfX6QA+okvzbw3ulsInjh/LTSQprSudNIhAtYO6BvLUcZ3ogr52ksoXpLOxlWddDcFkee7j8hDTYh/v3dfdYPv+lTfoX/6s/81Nf77//J/+bP/JjA77mjGVIEpX0eHfc4NWX55CtR9wbvFVn7EfcVahWI37z7hOkBLzf11DvK7yuycCEeeQMbS9w93nE8dUc+iBhZYtYJ9yMCvpdhdtXDUooECIQVw7DlwvEKiK8aaGPAm4Zmax5p/D+qQF2GnovMeAkx0svBuB1C5GA3ayC7FmRcfeRgHpTA3XC7r4iS+gF/CpA36tTPx2AGAD/xEHvFNSbGU+EAESbcJQtzAgMFwr1e4XDE4/qSrPPFif2A9czmNwvlwAIDcR5hPACszuB/qmGdJSeiigesCYJjRW4UgvIO4O3/gzNKwO7jpi9ZuLo+Cig+1Kjfx5QXysMTzxWL+6A315A2gq+S6wvmWvoUcAtIqpbRUlyoly3f67QvVEIDT2s9W3FWe7HAe0bBbtK6C4F3FLBdwk21uhu86C+y7UOScEui0y1glsk6ERGUtwYVANgLwLa/7vhwMsJhF6i2UqYHUOd9CAQtZqqWlj5QBlwfNeh2oLF3hEYz3LnY8zBPXca3S3TSFWWJOsBcJ2CX2jKDPc8t+OZJFsGSun2n0joXiD1p8fNTrOu5q1GUhmUzRLaNwJuKaeKDX3MEkCoaVCohzxYXwDVraGvtwGG84TFFwKxEth9FlG/4brb9wLHJwnVa4a22AUQDjIHhyR0b5goavZkBY8fJ7SvFesedg2qHTi9mGWFapAPZIIC1V1h3AR6KXH2fwHbzwlyYsVz1Vwm6CMLs+2abGAZLCeVg4QaSQnsyPM+boDmhgFGbiEgRx6nnwH1LcO4Ft8DkIDd5xJ6z7Cq6l7mSpuUz1Opqsm1MldkXfZHzfCmtYTNsnDpgOYmYfeZRHWXAVVFr7ebUUIdNVOnqy0HMb7LnYLXCvUd3w++Bcwuy4Idk0JDLdC9Z6CN75jEKh3QvcEUupIk/cVJA8cnOXmz4mO+43EgZvAQOWhrrnmN7Yr3X5FOM72U12X+MmL/kUT7npUg1Y7rLXLfEiQVdfaVbyh5lw6obyPGtUSRWuojz2lzk6b+Pjs/pZYikSFrbpgq29xEjBuG+iRJdo7hTXxO/0iiOaTcF8j73rcC3Tuyb+016zrmrwPGJe+nEmxU3zFoyK7EBGDMMcHtRJZb89xHxcd9IybQUiTNZk8fve7TJFdWFhPoUpbS3aSA/lxO8sypnmQAZu8Cj+Mmwi4kmruIcSERGqC9ijg8UfS3Z8ll7Aso5D6KkZMo3fs4SXzb6xwilWW5UXM/3ExOklc1ZvVApCS32maPe5a4Sg/YOQOZ2NXJUKoi+fSNgIvA/JXP4IL36SS9zbU8yy899k+pnqi3aUrAnb/JSatJoTpE2Bnvk/o+YlhLyAA0N4GdkgfKLn0rJ+8+J0YizFFO4G7qLIwEM+NSYvbOUcKZARyAqXuRMn0JJB5XqOTUx5hkBoJDQppz4qnahxNIrAWqO4/+wqB973B4blDfR0jLe7VIRUu4UAHPwAnQC58Qa4E0Cph9yAmsEXKpoI8R+pilutnzX19b2HUFZRnUVICsb3M6bd7mVKsCsCokJ6vGWgFgiFGoJOrrEXZTobqzOSDH0CMLnfsiGSQltUB17+AWrE7hd2mA2lsk0U77USZczO0Ae95kL3Bg8NAYmJYaE8zeTXUddlOjuh0xnjeoL4+wmwbV3Qi7rulTvhvgFzWki5CjzyDJMJDHa+jdCL+qoX2RR7LjUiTAz5nmKhKmEB4AkPsBYdVOCa1Jywk0TQzl4AkCfYRwgay9EJA58ZVf0gLCeigAoreQUmCq8YiZjSwS1MIGAgwRiuD2nScgzCyiGCzBXwaCcrAncKokvaWVmepHxOCnbYrCbPowSWhLaiycp/IrJYb3SAlRV2RDvQeMOQHJwi4WyWkBiDnkp9SSFMCZct+lkJmxzFUh6WGPpRAEyp4VJFNNyo/t3owfsKk/aRFC/L7M5t9b/uiXrzVjWX/j4/T03/mL0K1HDAIpCVZ59ApQCaIN/B0ARnUqVwdONRb5PznKSaYVMouCmEvrJVgCH0EQ5wW7H/MMqijl9cDkkyo/h2WAPCrEJqJ+r+Dm2ctpM6ORPVehTWQMcg2DyEmufu2hdorMw16euvxmuVw8D9jdkvtT6hJKYm0SgD0PUNlPKm2uYmjz7KU9MQCQTMY1d1kW1iYIJxC7CLWT076pXsCuYx6Qko3yTZoSYQEOyPWO8j960jj4LTPYus9l854siRwpxwtthD6SMdN7geEx6yOKN7VUZNgN5bduwYqEUiMBAPaM0sMSSoTMwISagKx0CZod5XtJkR2Jmt5IvReT77G+5oW1KwIAswOGiywbXfL/WOV0yCzPLJ7E4Zw3l8pS1FLWbXY50TeXnpeuw1CRYapvCexCmzB7KXF8FjF7JZnGesNjcjNed7tKWHwB9E9OUfrF8wXk2elEAFjfYUpA9R0m72qSp/L08YwgbmJmRLlemX3J3rDixWQvJM8juz0F3BJMmFWndFxzBMY1k3kBrksfuV67zDP9IqfNWmD3DUxVKaEmU1DeEyoDRjIVlGdW95SdRkMGqywlkda3BIClb7H4ztyC50UESgXNlufPzYHZSw6afcfjVQOmZGeVWRnf5tTmzPQU5sctTqCK28sl2gKQNhGwL9m/2V6RHfH5nCSJqfIj1AQyhR0ze6aiApiYi3Lu1JDBpKNEskhQ7VJMjG3xaZo9gYU5Zo+1O6XOmn2amKXhkThNWvSUdxYWRyROBhUmqd4m2JmAmxPEyXx9QsW/P0waLmBS92nyS1P+mMHeARn4pskjq2zCuJFTl2V9T58nQ9ByJck9gVKs8kRNT9Z3YqYKy6i4b9InDJscftYK1FsCPBlOzy++Ppk9a3ZBCSJSBtb7CDuX0/WQnuBVjfzZ9Hw9fYdMtrULOYEx3WdwN2appKAP0s7EBz5QZdP0mgKojhecwEkSqHdp8h6OS4Fqfzp3SRLQx8xQ6TFN8lD6NAkcY+5FLb5ONUTYpYLr8r0ST4BTpHwtQW8jJ88i643OFZRNU0JuCZYrjHW1DdOxKxvhZpz9FCFN+xdMXucY6ZF9AAjNMUL17IssgDCqXPfRR4SG3YolzbUk4gL5PrOs6yhMXtKAayXquwDpItxSw+R02sJwAYCyTK8tY4eSvF0+J2UBuhlEsi+U96MaItQY6XkMCebg4Wb69/wvPQOMSsLrlNSaAUrIKbOFnTM5DIu1H5R0lnMZajmFK/mFgTp6gihgSgpWY0RoFfSOIUciJsRasWeyVkyX3VG2Gio5BVSFhudBjkwNDq2eAqBilX2AOQQoVRLBSKiBQFOOAVFLqMEjtAaqdwiNnhJzY61zrUcZTAmo3YDYVQRugkAvtprs4bwGYoI6MMSndFQWthIgQynLa0vlRk5FFb2DcJSyApgAcPFIIiQC5JSmMUypBBEpMWW1rk7VIZlZFNaxEqT4KfPP0IoATkn+XoDpw75JILOZZEmnBNnCWmbQ9hC8TqCw/P4Veevkr4yRgDFv9/eAywdLSXOdlvRgfWUbBZCmiFTA7cMlJdaMlDCiHwssHxzHj/vbw9W5nxDy8/9i+XlgLJ/90ib9C3/lz/7U1/sf/qn/+o/82ICfIWMphHgB4C8DeAJCsF9PKf2aEOIMwF8F8BmAHwD4Z1JKt4I5w78G4M8DOAL4Cyml3/x9tyETVOvxJ56/wxfXZ5AyYRwMvADqmYUdNGQdcbY6YHdsML7rINYWQiYYE9DWFreXC3TrHsebDu2mR3/VQXYen37zHb74nWdImoE+cZQQViI1EXJHb6IwEavNAXdvlhCdR+oVzNJi1o3YblvEJLBeHXH/cgW1tLBjQ9ngoFC9OGDsDVIQCEmgno8YbxsOAC880n0FBMCsRjhTQVYBzhjARIj3huFAg0TqAoKXEE5ABIEwixCthxsU1FYjLCknjBrA0iHuDPscj4oMUZ0QBeh523jIrYY7i8DcIR01hBETqDRPj0jfm8GuI1KdAyPmEWnh2XVZAfW1xPCMzK59FJHuFCVJMsvmMlMMqRA6ykFSlSjjfDQiHQy8JHvq24RkElwXoO81Qs1ORySF2CTYTUTYeCCZLDel/yiphLBICF5kjxnBitsEzL+vMZ4l+vy8hF1HyFHAdxH6IBGrCLc8DRT6Z2kC6gjA/rMAOQoMT+iL811C6BjCM24Y0hQrMBBHAt0bMQGf4SKiuhdTLYyfpxy+wsAgNYocNMR1MoAIiCsPt60gR2B4lGWBq4TqJfsxq0PEIXsPfe7UZOCLwPE5Jxz0nn6oEnhTBpwc4HNyoEgNQ3sKmFJ9/lKWBI5mKzCuEppLgm/hWT8STYKfJYznPG/DYw7KywQHfYpkFRFP7E5J8g0Nj59sBuW5scL0Je47BveM52kKiKqvMpMRCVTHs1PoUpHM+hkycwAcPmIoUZFBhoYSzGgEhCCocsuT5DCpHJqU5bR+XiSyAuoaGM6y9C5gAlz6eJLuAZRUqp49fmRvCfSqXZx6LQ/PMiu9z6BkJgjEZgxySiKD9X3C8Sk9onaJLNc6ATuzB4Zzjn9cR6Ym1NnnaIDYApACwxkBNsEZj6OwlSVYK9RkXt08S1azbLK/IKCQY7mGBEtuTuABmaWrXsBXHLz6NofRpCwtzUWhvqPPklVGp/sE+fNiXHNyQXqgu6QHsUgk3Vzg+FjkCQ6RmTqye6ERk+yVky/055bXFfYuGvqz1EA2LUlM/6eYJbYJAMTUxxkqbqu9TjkVFlNXZFI8L9O6unz/58mdcn5DJfI5kXAzATc7TUgpl4N1Ur4vAxAWIvdT8v3hcjCQSDlgaeBxl/5HNSaYI3I3JPLkY8KwJjssA8NykC9FFMC4JBB0Cwlfc/2mTxjWepIPjyuB5g4IOtfvnEvUkZ2VUzhVoyapc6zE5MENFSeHCos9bKj0ODyTqO/IMnbXmY1JyHK9/B5uJEwG7wUM+0Zj9iYnQGdQW+Slfac+kEAWaabrKKEsANfNqMIItZiCiUIrEWvu83BuYI4RCSdAl44EironQ+ky4xoVmc3ivQxzso9Fjmv2AW6uGKzVSlT3Ab7VfB+0egq5KRMiQAaGDQNvhE8ZAMocrKTJ5rmE8awmIF8ZmD0H9dFIxFbkJGc5MZqhUYCkhJQSbwHlIiegz6pcbcZQnFgr+E5BDRFuWeVANEHAPzNwcyYo+xmHkaHhJIc+ZFBcrqXAJCWNuRoGIRE0KgHX6Qm8x4pVHyKyN/JUOcLakNBqCJPDkkZWkqQCjpSYKkSSlgS0QAa3Yuq85MkVrD1ZNWQ6lUDo2il5HTiFyMTWIBoFc30gy6kJOGNn8r65k1suRsSmgsxy1om1qyt2PgKngB03ItUackhIlSLAVBlYlsCdDCoB0CspJR9r61OwUK4+gZKZbS1a5wJWy6yHOgFNpcggpnTyK8aE9IALfAgARdkfACngBC5/XD1Ikbvm8/FhgA/Z3YfnawKZPw5U/jFMg/3/w/IzYyyFEM8APEsp/aYQYgHg/wDwTwH4CwBuUkp/SQjxbwHYpJR+VQjx5wH8RRBY/mkAv5ZS+tO/3zbqTz9Oz/+Nfx1Jpcn7leoI4STMrYR96qCvDUJL039SiT6ySKmp3mcfUx54lxTI0CTEvB6o/MayBG71TS5Tn9ETFOoEP49Q/ak3UVkBNzsV3qtBwC8j2pcKw0WkPA0MC3FLDozLl19h2aTjYKKUs9O/lVC6uPwsIckEfZSYkg/zl2t1R4ZM9wKhSoiZqXvorQoNQZpwAnKkZM5s8+sO/DvAwRcy8GSKo8iJiwQdvgSjPInQx1xJMINlAAAgAElEQVR4//jE4DVXZD18R+masgBSPhYJqCMBxORLUkXuRQAR2pTDXABzz3NvdsDxORnLUKfpfHIGGVPKpQgEs0mRNRTh5H9LGmguWUSvjxz8FUZu8jqFzMiNyMck0FwB+xf5HLXcx1gDiEBzneWmd+y1jCVkRvF86V5MDE8JJSqdpX7GQalbJbRvxeT1OUlbMSW8lgGqGinZNftcGp/TXctgJknkWgSCnuqe+8frmrfb8ToWFjJUmf24Iwhw3SmVlaEemQXOx1H8V3Z12odQZ6lqHgAX9s3N8/E6yjqrLe+P3TeA+pYsrhq4nf5xwuwlQcHkkVOAPpAxmVI790D/lOe5uU44PD8lbkadWdGB/aElyATIrFwlJp+e6nnNdH9ikpmImrs6G953vj356aTjwI9y7TRdn1hhYjFEBksysNMuab6OiZsCwwWw+i5ZHbvOiZYRU0eqz8CuymyUOSbsP2IFTkkxjZkRBDIzp3kPlkTQkvBa2MLQ5GCWzHSW4y0D9MljeE/AqA+8vvqY0NyRSQtNBtWeAKgkuoYaU9cjshxTZxmnXfL+l54TTOOa8lj6sU4MZmHQS2+u2RNwkakXUy9oYV0BsG7mTE7slDnwWje37AJtbym7LL5Plf1e1T5i+0JP3Z96JNtYWMdql9mwzJ6pMX8ecX5hqvUpyabFS+e6zP72ZNd8Laf3XPFrNrcB/blmUuwsV0QZ3mvVNk3puWoki1fv4nTfmAN7E8elnDyCypKFLBVU7TW9efWWgTX1LsDXkvcmTmmyyjKgxc3VBMZKiqsIWQKdmT2RyG7253rq8SQzdmILhU84PjaotwHSJgxnBK0AGV4I5LAYAjSzdTg+q7NUN6I/06gOcQq6Ka9lAmrpcqVh2s0UGcKYptTW+tZj3Gje6/leLMcsEuBrwX0bI9xC5xTR02e+CAm+lWivLMFfyqynkajuRuw/6Rho0wiYfZzk78VXq8ZTeioZvuzxc/QChkay6sRF2JWZUlzN1k3dlrrPzF2l2HN5Y2FXZurq1D3rfyAIuqa+y8wSuqUhEM2MZXVPxlGOAW5hUN1buCUBjt6TLZQ2QPqIaFROZWYVlkgJ6uAQOkOAJshYFiZS5H7PWCuY+xHRKDKeY5g8iKUjU+a+SX3dw5+30Pcj3LqBGjyiYRVLAZXq4IBcE5OU4O+SbGJYNJTIPgA2UxdlTnlNRpLtM4osqlETEy9dQJHPFo8jnEfqagLQ3rEfM6RT8mtJdnUM9BGDO8ldgVNgTgnEAQiqtDoxlgU8ZVA3sY7luWUbxRuZEhnKzGQmo9lVWUBlSZQt6ylMZ2EyC+gs6y/ny/sPf//qktIkdZ2Wktgq5IesZWYqIViL8tVwnmnJILJ0Vf5E1vIPWlL8Q4X3/Lwwlv/8X/knfurr/Y/+1F//A49NCPEDADtw9sT/LM7Fz4yxTCm9AfAm/7wTQnwHwEcA/kkAfyY/7T8H8D8B+NX8+F9OvNv+phBiLYR4ltfz4xeVIJ4MiIPG2cUWx6HGNy+u8Op+hf2mwdPNDuZFxMvffQxsRiBIVK2D1gFCAIfLDmY1IkUJ21aASnj+4hoxCby/XiLdV1i/uMPt1QJiFRB6jf5JQNproIqoP+8x7mvo2iMGhXBvkJ45PLm4w5dfnqNZjRAiob/sYNYjxr5FemT5vDrCLwVQRahrg9hGJBPhJSAbD3cwQBUBS31VWAPqTiPMybr5TwZIleB6DXFU2Hx2i9sfbJDaAL+UlAFfGYRzB4yKd1AnkGYBdg1q/fcasY6ICw9cG7gFEB5b4B2/cNLHA+JVDqipItnMJw7mneEXqADcEwfVV4hrh+gq7D4PSFWEPKqJdXHrAH2Q8E8swkEjmQh9p+FXARUU3ILhKWEVACfQvWRYje9YYh+6CL1X9Fk+9XBzmQumCeqLJFj3An4OyE8P8K+6abBQBnIAB7Bxw2AbNUrEGhjmZBwLcA+5msXPE+yKwJMy1QgkiVgn2PoEDuw6wtxL2CUwXEQcPomYf6ExdEwq1Vt62Pp1gN5JtJdkjfSRKaLVllLicB5RXyrYNQGfPgr0K4LYaFL2UPJY7YagNn3riOF1C72nLLi65zrVSJA0nhHcjGf8UispoH6eJm9M1GkKZFGWKao2cTvjeUS6ZW2K7Qg4oibjK62AOZAVcXPkkBaB8SKi2slcL4ApKdXsBcQAHF7EEyBCnvzIUmbf8fECzqJmIuvwiNeA3qvMulrKNFVPxre9JKsqIlDf8FgJCCi3Fv7EXlZbglA3471hF1wHIJBmyGEeHAyWChR9EJM/jUwdj7sAAN+IqfrCd6f7rbnKjOeC8s0CVkslh8uSR7sCqjsC2vGcbGOZYADohxvOCCQKE8l1cVvmgMkH2D8me+Vnp/oTZHA5Zp/kcMZ7xOwJoopfldJbkaXRZIrLxMOwOVW0mH1mLWdkbOr7CLuiXzU0PD/NNf2SHKwDZpfDYRTX2T8WqG8ySyt4zQ/PE85/K6c95kms41P6PXlPiEnGF1qB9jJOgSx6DxyeUZYbKwGXr8PuY6ZgMlCE8tvdC4nZazH5IkXg8QG8PknynExVJQoTS+k6AltOvfPeNEeyryJiqk8heMrv2cUpJZX3h+L6t0BoBSwy8Gnpw4yG59xnUORaOUlR7UJN4FcPKU8IiZMc9Jglt2cSvslJyUFOzKN0rIxxuVLDLqgs8ZnFq3dk5EIt0Z+RSWuvElJiEIzr8uSNAnS5D2tOUErke80yjVRZ3recHBFZIh7gZgYhScSqypUe2Tveko3Vw6kKhmA6wXWUAUdN9tJ1pdKB29QDMJzpfI4otR1XWbosBKI8XcekyBZLn4NbckUNpdknpUY0EuLA7yM1EojGKjNyPmJ8ZMgMdmKSBicl0Z9rVPsIfYjwc/o4VZ5AtivNvtSGbKJv+FkaWqaylr5fJqk2BKBjZH+rZuiNuXdwKwO0ZGg1CijOuQyVhJ+RORXWwzcNTAG8bQ7ny8xiaPK9NQbAZMbYBYS1QX05wM+rzK5y+2qkVHVYV2jeHxHmFWtKvKFsVABuVUEN4eQxDYnsZAKUUVNoDvuhA8LcQN86xM6QGcwsY+nlLROyCpjYuqglpKWctlSAaB/5WqP4WEyIhkxtwTJFdhs69o2KYQCUROgMk2CFQKw0pGd2Z9ISdl2juhtZ/aMlRG8nhlT4DGALI6pkBrcKcnRkLHPvptz1SCVxVQjAgz2kvT29Np1AYTIaqAzEoc9AWQPFyiWzvAOAsJndFIK9q0o+8H7y/k9tDXEcTqmz+gEEeAg0U4KAJxCUkp2aoshzee6FMeydlAIQGuzopAyZhyYofy1LZip/j1S2gNeSHPv3WMr/L5Y/m1K6+lmtXP7BT/nDL0KIzwD8QwD+NwBPHoDFt6BUFiDo/PLBy17mx766rn9JCPEbQojfwGGP6CREr7Dbt/jmxRWetVuczY4IVuH99RJfvjxH96WCeF8jjRJu1DhezjB8sYBZjQhvOwQvoZcW8AKXtwtYr6B0QJIJQiTU85FflAcFcV1BWglhIoZjhXTQ8NctZrMBcuFQNw7v/+Yz1MsRw77CcKyAKsLd15SA3lZQB0k2FIDYsY5BrC1gEpovDWWwOkIahvdAAvVqQNg4qKXD/pse+ocNHq33kHUAJLD7rXMsv6sgBoX6vGcCrMjrrwNE9prKqsgkgCQTzGYAdgzr8c8s0qDY57cICDsDbCzWv62BkJnNe41qK6C/uYfvEqqZpd/zoOEXgTP5iRJNt4lorgSwdPCzCHlvyHSNJ6a4pI8iCcjWMwXWAWYzIs09QheZZnrG5FThBGavJNbfvgEApCpywD/3iJoSUveu5eCuThCPRwao1PQs1jcC5k4SyFZkZJMhUEKWe4aW6Zl2E9BcSoyPApJOUyJn7AKqWwmzfeBbHfnauPJkrxUQPx6gDhLu3KN7TZbcPbME25+OBDISmEJuDhLDU4/uDUGl2ZIRHJ6EqTMz6sJkBQzPApr/fYbmveQ123gcv20BQTBzfE5JsgwAZEL3nscwXgREQ6YTANxZgJ9FAhMF+E8H+C6h/9hn8JwZ6EPut3zmCHabzM7MGcJkz8Lk66UEi/5BASCZhPE8YP+LlgmuCz63e0tGHZnBdQte//higJvT87r/BFNyql1H2FVmU5cR/RPWILh5wv03Qb/sTuD4jIxGrBPGDcHZ9ltcz/EjdiKWcznpcj4/4Pipn9h6uyCrhgSILI8dz8lsisiqBLtKObBJTDLQWOHk124S7JLbcMuE3Wc8X4UNArieUIuJqTw+pmIgVgRvdh3RP2aIDhLXYzcEvG4B9J+6LOMkW1nA5XCRYHaUKA/nCcM5ZbfuH97B5jCdEvKz/WWH4UlAtUtwS4FxTaBsV3FiHodzAd8JHD4S+X3C80OvIoGFCOkDsH14RqarJMUen5W6FmD3eURzlZniIaF/ynvG7FgDsvssws8x1Xq4ucBwTlBrVwL9EwLUccXB+HhWjjvi8FxmWS+Z4NCclAG7T3MCaZar9k850TGcE2j4hmDl7lcs7FJgOBfoL+gdPTznz7Hm/gBZ9mnIyKYs9Rw3wP4Fr4nrRGZBi/yawGX/QsIuxRSKVOcQpHHDyQK75PERFGY2uD4xp6Hic8e1yJ5CMuPjisc7LiX6i5MUl5NIrHQ5fAQcHmeLhOBxHx/LPGGRMKwUDo+zR00TTNtF9mPuI9xCYPdCwi4E7EJiXDFMJxgBO1cYNgL33zAwPb2rJQ3VdcDhucD20wp99gofzxV2nwgEw+sSK3ZyDmuuc1wT/A5rhWCA/XMJ30r052qakIpV9izPBPYfKRweK0hXJkwoIR4Xgt7THVNIk+Z+hYrHVapVkhJZbps7KG1Cf2GQBHD799XYP1cYFxLHx6zOCBUlsXYu0J9LjEuF+nJAvQ3YPyPbNqWr+oTDU0pf3VzDNwLDRjEQbGFw+23Nc6AFfCcxnjfYfqrhW06m9o8U7FwiVBKHj5rJmwpB6W6pSnOz09Du8EQjdBVTXyvF+pWdxfGx5mO1YkWKj3CLCqGWPDetRhICdlPDd5TF2oXMrDf3TY0Rw9OOz5srBEOm0DeKgUZKQPqYvZ1MgLULiTAz8DOFWGvEWiLMK4wbjbCsKbvNnZVubmCXiudfCxw+pnR1PGOfJAQQWoPQKNiVphwXZHpjJeGWFWKtEFoNuzSQjq91SwPRO0AKBuUoidjxuRACft3AnlUIjYZf1UhSoLoeYM8axM5gPKvhHy0mNtSvGoSZQcq+09iaKQnWr1pgtJOUNXUNYldNQTupMTwWAHHWIKxaAr+YkBpDxjMzmKkyBIizBmKwcE+WSF0N/5itCP7JCu7JMvsVJWLXnBjVtoboxxPgzKAyni0Qz5dIqzniao64mSOuFwSoUk6yVtHUDP4py1d7Lb/CbqYQTs954McUSn7wDyGcQGUo6bK/91/K60x/DIBngkBMP/1/Py/Lzzy8RwgxB/A/A/gPUkp/Qwhxl1JaP/j7bUppI4T47wD8pZTS/5If/x8A/GpK6Td+0rrrzz9Kz/69fxUpSGCUZBgGAUQGqjRXEsO3BiSroLZkvBApTdU5gEbaDBYuLPSrmoPEJk3S1OqOqYJumU6M1tpDXxv6z0SCm58qBKQTsJsAfZTQh+zbmlMGaZdklCCA4RG3bXYMotFHkQN0MEkTS3x/SUxNhumkdsNAGyRuE+Dg351Rbms3KdcFBNQ3apLJ+VlC90rg+DxLNDMoQiIj45YR7VvFAf6c+1pCcyav2Hli3YEVCC07D/2MyZluzplzs+dg0835PPeLR9R/q2NAxwNJaNmn+oagSTrOTtsNE2VDw9frHnmWt+w3B0lka3it6hsOfqZ6hfweK6xlrLJ0N+9TSQc0W7J50hcfE2V4UxhQOvnY2H8Xsfi+hF1l6WjDwX+RdoY635wxB5fk8JoiXbXLk/+v1BjEiuf6Yf1ACakpNRPTsWSptj4KjGcJix+WDkB8UJfhZic5ZTTcx0lCmysICkOle0pPyYyenqss5bl2hRzYgCkVNFYnX2Fo+LzC1KkBGB7zupbaFsTCxjDAp8gwXcfEVrfEBLCTYPjM7jN6VIdzgtpJfqjyNcoeyJKWWVJw63syYNJmdkJz/8YzYPaanr/xDKhvcjjOIUs+jynfZ+KBZJcgXVkCoGqbAZzi8VR3/EwYzshEhYb3cHNTKhXwQXKoXfBYkuJ6Zm+47VBRThsNck0AgaW0wJBDdZLiMZs9GddS2aAHrt8uc5psCa5xaZKOJgHYdYny5zXxLab6geouob6nfNQuKBvuHwnMX0WylJMMO8u0m8JkpimgZzjn+dE9PwfYJSrQ3EaMKwmzPxXZRwOmwq4pzdRDSfXk/ROa03uvvxD58zWH2szFJA3WY5Fq8ryW9x/l3AQWJWiqvBeau4TjYzlJfNurOIXt9I8pqS7yZTvnZ5myDPmp7+jRs/MiBT71dKpShRNTlvyTYfNtlpPn4CDfUtqcBGXaw5lE957+vCJFPj7mtoqM0rf5c6io0RKPu9rFU9BQBOysBN9gApQlrCgpnmeVE0yHlUR1SBgXlOHSf8ntFSkvP1cSmtsAO1eTDLi5DYjVCZjV9wHDhsyZsuxRHFdk3fRQUmHpP0z6FP7jOrLZpqd3skh6+ZnJVNxxpeh1LJUQnnLGIgVn9QSm75RQS+ieEsdoZGbP01Rj4WfZg5nHPsEINDcM6vFdDsjL9Si+OYXbVbuYZd8RvpPQx8h928dpglANOZwnpUnimqRgmuoQ4JZmklULzx7JqAXrMqSY/IAxB93YtUb7zrLf0VIiWySVvlNk8kIC5MlPWV8PU58jIiBHj/G8YXWHT9BHhv3oPeWZhYnzHcEbA3SYbCvdqeYjCaC6J6tmV9UpFTYD5+p2zOebIT6h1dBHRxYtpUmmK2JiVcfoMT5iv6WfG+i9g5+TnVS9m54jAqWeSUuIMSB2Bvr2SHAWElKtII92quhIQnzwGDJjKncDUl3kqmmqBklSIrYasvcQIZzYR+u5zixtTTX7IBEoT43LFnLbnwakKSFVBvI4IHYNJawPxtepMafgoRygww9qBTGMBFbOn/yVhUEsKalS8u9F/gpwPT8uZbVIcEP8ELSVpUhmhUD6St3IJHOdPKISojJIw3B6Tkwn1lMKJOtOEtcCKEsKbA7uEUp9yGB+dflxIUQ/Yfm6h/c8/aWz9M/9l3/up77e//gf/Gs/BPCQifz1lNKvP3yOEOILALfgJ+Z/8tW//zSWn2ndiBDCAPjrAP6LlNLfyA+/KxLX7MN8nx9/BeDFg5d/nB/7iYvsJbq/3bAoe88v1jJ4Gc8k6puE9n2NUFPSVN+eQkuYKinRXmU50BfNg/Q/SrvUyEFE+fLiDL5A/bdM7khLp0Gu5helbxJmr2UOAMnekAQEk9BccpArfML8RyURNmH93eJ54/ZFiDn2O6f8CUx+qSQS6nsOisyxSKCA/4e9N4mVLkvQg74z3Ski3viPmVmZWZXdlGi6BbjUXhnLEhIyg4SQkJHYWEjsQGLBBrFkxQpYWGJhvKBlWsACC8QCqzESbWQ33eCmTXdmVWVVVmb+85tjvMOZWHznnPv+6qpy266iKlsO6dd7/4uIO8eN851vMgdG/EcRcfR58pw0/LbLdSUM4Ihor3NUf5xN8xpcUeQMM1MU0/ES9M5MS8nwE8kS5mAkC7YfSixfeUxLWQrHvRFYvIjYvSfw+H+o4askfzQC3kToMaLaBfRnCnrgc8rSdzOcKDS3vLlJxy8V17Aaor7j9t1PRwwqHzsOBLorh2nJZEc1RdiWA6ppmdMUObCaVkxO7C5Q/FbekMkYzmUZcAqPMrBRg8TqpQNeZmkaB/D+OVmeeh0xnDCdUrpYOsO84QDs5Pss7hY+lhANaelbyp1w6w80qxmSzy/72HLfWR5Y+YpeKxF5rgvYbgRWzxLgOZZwnUB34XH7TymcfeIotZtmX1GUAovXvOY5KCY70N54AqBrBlSMRwJHXzJEIvfPTUcSQXNgmUMs1BjR36jig6o2IflmOGiLEqjXHuOxgl1ItNe+gOco2b0GAGavUO08Fm94/vPgFuD5KGmNgdKw8Uhi8drCLhW6i1iWN60k2kuHYa1Qr7lse50qYJJXqr0GqjsH3yq4Gwmz57q6C3rRRGTFjXRMWjS7JI0bYvJKaQzHEt2lT+yu5zlLfXrBCL7WzX4vNXBbpiMFXwl0l66EatDnKKB3Hsc/iOgfVUkKJ7F4McItNLrLDKo5KJVTKBUEoZ6BQT6m4nNKHus7SvDIZs+JmSIwzCcPXNtrpGoHBdV7ytJayWqE1C/na/oaXSuxeEP/KCLSAJWDUdcpNDcWuvfJZ0omTI0e1ValMBR+UOxSo9pY2KUux6e7THK21FlXtxJm65PfjoOR8USjvrEJPDEYRY0sWZc2lGTN7Ds73nkO/FsJs3MQU4BbaCxe876rBoeoJMNLRt4vVl8QNEgXsXgVGEgSCWSQPLcIHLDqHVGmW+g0QOf+ihCREybtgvtYrVGknPW1BWJEdyHL9evr2X+npgC7nEvg6bUL8C2rHeoqySIPFm5hCGZW/FmSbS39V2ajoQ8OnZE87rf0zdmVRnOd5G4hQg703pmNLrJGlQBDSAmkcvCobgC3MohSwGwmqF5DDT6FS0VEI8lcGb5P7yzGsxpRCTQXPYFgY6D2E4NdJJljsxEEGl1VkkJzZ6A6WAKEPKgVgFtWMJvx7QF3ShCVk0d9ReAQjCRwcQFuVUPvLZ9Lg1u3rNC+nhB0Cr9J8lJz06MGvwflVMPcDWndIgFKk7x+AUalaoi0DDXOLE4wCuow8fNgPUJXQW0ZYx0qDREjqjv6/dRQQa4PiF1NwGMUzK2Y/XXpmoqVhpgczIEyTTFaRKXQ+AAxsnpDDhOE9azWcB61JZgyPhIATo4gtdKQw4QqyRtFTlK1Ds0hd5C5ORnVOojKQN/xXOr1vfeFQJCTtlfYGuIwottPwGRRHQxgHap98h56ynfF6GZgBIA1GjXgPOR+hBgSExgCpHX0IEoJ7OYajuJLTB2LIr9upKQ2VoasJUBGD0nE4gOEFCjJscM4J7hqBek9a0Ly/mkNrLfAooO8TfHnGRBqBbHd8/dU3fGW11BKQHoCNH0vVEfIMuEgVPIw9kjAUc9g7P7PGCC0nrseM8C7zyBmgBcSAMxgMv9NCp6vGBB9QDwcGPYDzBMgu4lgsU/1LXlfE+jNKbDl9f5Hg8HMQAopEO0/gOi6v53/5PGjHld/AtD852KML4QQjwD8lhDi2zHG3/5pbsTPMhVWAPhrAD6JMf5n9576nwD8ZQD/afr5P977+78vhPhvwfCe9U/0VwIIbUD35y9RKY+Xr0/x3tMbnDQ97oYWlVc4TAbb2w6IArLycMbDTQpKB8QoIFXAFsC0q6BuDL72z73EF99+UgZ2saXMVOgAqQP81qA6GbFZ15CdQ1U7jPsK2KU0k5WF1BG+V3j8zh3evDmGOGhg4RAPCu2jA4ZXC0QTARmhlhZhUsDaIFYecpCIpxNiBI5OD9g+O2LRPbh4MUrg2EK9qinztAL+2AEmAKOCSl1fvgtMsDUeovPAVhcZaVh6CBOAdQLHCw94yjRlz/qQ0BIk6DsFAST2UyCK5M87clB7CX/qIHYasbHYva/hHljIrcbml4BoPNB6dEcDXnyjS1JaiaiTj/A0IqwCxJ7JtmIgeBEO0NuI3YcpiXGS9BsOAtNHPdTzBm7FRNbhiUPzmmmx9oGDWiuoETg81ejfcYAEzA1vhrnKQY0SLhXPhyrNYh9SCNFGzWX3e2D3fkqYXXpUVzoluEbsPpAMQxoZ0iRCGnQYHv/qRmD7Icrf7SqgfS0pldUSIcmA5TQHSPlWQPXsr3RdxHhGnxVTSkUK8hGwx2kEGhIg3WqMD+gRHR6SKZcjv7DqO2D7DbK/4xkTbF/8hTTAHiSaS6aNum4OcjHbJN9ZRKydghroPYwqQvXA4YnEdAY0aeBrj1iTI52GXRLMisCJDOkTMx05+KzuBFNWI6AmVu+EChguNYZzMr52SWo4V8yIoEqdz8ZLuEWEryOaK538hxHNdZL+nUVMx+w8nVZkQsdTrn73rqFMVdHLFAxSXYmknNUJbtMiVc9Uc32EPnBfgyLz5FpABEXGz3D9wzlZ5+2HigEggb2euax+PE8M1UGUMKOgeY42HwWYjcDuPfatCi9KBY3eM1jm8Dj5oCKw/VrLazFNDOie+5IZaF9TgZCreaIC1KgKSx8UPXftG05YjGdkAzObnYOexrO0nQYwOw3hGXaleoX2ihNtOYAqajJ39ToxeCm0KCpgOgGqOwU1aNij1A+6YhhSTpR1rUF7yd+rjcJ4ygk+OQKQPJ8A2W67AtSgqRbYILG8gDmvYJciBUfxNaHipN94wmOd/anZUyldhJw0U1pjTOcOGM9q6D2BlV1xm6WjHJne0TlcR/jki0wsF3sJ+dWaK1ZCBdS3mjLoFF40nYgyESodJ5+272k0NwmQNylh2gF4mvYleURZS6ILs83JUlGSaZevNfaPZWHMq43C4bEsIUVBIXk4KwbGGSC+pxle5SLsQpfzp4Z0HJZzcFK9rmBT2m9WRmTW8PBEoL3U5bhUu5h8kUAUDdclAD1UPDenAs0DBqW4TqBeNyUEh7JQQISG9xZfFxaaFSsMZYoKWLxxGI8VhhOJxYUmG5rYY1bHxNJ9yQlO1ssEQ+lrc+spu3VAfedxeKihrJnDygJVAYs3GtnGYRcSeFyxvmQM7PZMQDoHDYmQql2auaJFuIjxVEG6tnS7MmiohZq4TXmSjuy0hHzSFsbWdRLeMDzIN7JMZAUtUvBOTAyqp0x0JdHcOE6kbBsyiynROQPioMVbvZsAIEIHhgsKqMFDWU6o+FqmCQwywKr3KfQugaUkZQ2VRHVnObkycltEmgg0W6bS5VoWAPPkR5LR+kZB711Kl1bpu24zNFYAACAASURBVMsDx21J0s29mwip8mSgdGc6qVCt2Z2ZU2JFiIU95cRYOl8J3MimQmjorYyKybhi4sSi7AlUQ63LcrDqUkosICfPoJ8QEDsm/MdKkym9F+wTjSrHVAycSIoAmUjnEYyGCAGiHxG1KnUjUQhASf491Y7EtiYAzOE9qcYkhkCmd7KITVUmXkpVCUD203ugrmZ29H6PZalOuddfmUF0jIBzEJr+zwxCM8sZfZiTX326qUr9x9nVkhobkyz2JwDHGAAoAtH4E5jPr8AjRsD/nKSrMcYX6eeFEOJvAPizAH6qwPJnmQr75wD8bQD/L5BLA/Afgz7L/x7A+wC+AOtGbhIQ/SsA/iJYN/Lv/CQZLADUH74Xn/xH/wElsKdTAXRXHz8AAHoaqwB1o8uXXlh41nCcMjE2mDm5NXurRACijtAPe9hDBX1p4LsAOcgUOZ9mHLsAvVaswAAAQZ9cliy6lYccJSWYK9ZZ5AqGLMt1XSwppEAKjenSAMcLRJ37JoFokiTnjnLYYDhwFAGwpwF6LcvAPqerZmmYmpCS25JXUGPuONPcJuGB6TjQO1inG+3E5NsoUcJGqjU9VllaqneUZUpLSep4nro4LQfi9IWlk5YG2rnXUY6pk3JN0KD3grUJZ6zu4IAplu3Lki+3SDLZIe1Xkv5TzpYSY1MXZR5835fhkd1Og/EhM7YosrKcrCpyEnedQO+B0tAcdJM7InONhV2ya3JaYZbgJQlbDo4ByKy7ThTp7XSSWNMN1zOdoHS3qZG/5xRW6djZV625LVkGmztOg07L2vJ3s+eA2Owoxc2DQHqu5vVmWaSv5/32TfpOsCipr65DkQarniy4XeYApXQMrmLyTM5JoyyUF6mzkMDC7FPgiQdZn4nvp0+Ug3qzjcjBLtxHetHUCKienkCeG/rBctVHTpPVPYHZ8jmPgxpZyD6tOAjL/Y45FTYnpkbFgT+BzyxvrTaxfH7tEfe7vYwpjj9dEx0H+tU6FhCY1Qf5OnUNgXZzxYF3/5BKhKhRPGm+St2HqXdTj5Rk5mOaX5MTLbOsNydpmh23OWjui/QEBlky62sG3biGvkY9RMpsE5t/eCRRrVMy9cBrL6d1Aii+QeESmIxJ0hhRUmxzIuZ4wuMrPKWzoQIWbxIwkDMwcZ1AcxPK+a7ShFkGW/lemwOgpEUJcimBOUMsFSZZ4lnYYKDUVughKR8SKJQ2Fk9nMLOEMq8vdzhC3E8r5bnJVSGZpdZDTMwuj0UGNVGm5UZKV6s9lRGUEHMDq20oAUIicrsy811tfVKjJE+iyYBgTlv1tSiftyxZDOkzb7uUnnpH1YAekiJlRTbF7H3Z/1zXwXAZHiM1UAKqUx9pBlTKxgJo7FKiubL0z9UyKXGokgCA+s5iPDGJuY6wy5n9tyuVElQJoPIxFI51IDIdd2mZZBtTv2A+H2ogQ11SXu/VjvD8yrlTslWljzX/3+wdXKfSfWFmboMmY+7auc6kWlsMDyqoIZRzpUbWpvgkJQXIdOZ15MkHvXdMaM1s9OhhO9aI6N7DtQRVuR8yVgQXriWLLuKsOOLFEIvsk6E2oSgKcupqZo71ZoRfVghKQu8mhFZzfw8TQpJ8Ck8QKZJ8NdQ8Jr6WqG4HhMYUkEbgA6ieTHZUZCpFYkP9oirAjtJgyxTYwZFJHqkQEDbJitNYgRPAGsInoJQYsagk5GDJyvoEhjN4mhwQgLCsSkosYiwdmFlJVrp4XaDc1QfErgZCSpY1ClGR4S3ryevIwC8DOwBvdWPmR+6NzMdJqwLuohT35KaiAEiReyardHyd/9Ey2PsgEGDSaw7jyb8nlrYwnFoh9gMBYQKAf6zmoyiBEnhLry3hPfcTWWOYWc/EgpJp/REA8t7jLdxxfx/+BIxkDHHetn+Exy+CFPbxr5zFf/s3/6Wf+nL/i3/+v/uJ+yaEWACQKVB1AeC3APwnMcb/5ae5HT/LVNj/A4D4MU//iz/i9RHAv/cPsw6hIp58/RpvLo7hR4VqNeHyeoX4cEIcidS64x79fkmQdjYBkyLLl9iU0AWYGwV7xgtVOLJ3oQqw+wrwAn4ZyOTJgDgyeERtFDsmQw0R5voMd+RZTRIEIFlj4rvI1M5Tj/pCYTqh9NMvA1nCKkLtGARjlwkQA4i1h7lMzGLyzIRVQOgl/JKgNjNNIvk17SoF0UQgnAQIL1Bdyzk1VaIwJ4IVkHBVJFgWgHQC4yPuQ3VHIFwNItWb8H3jgwQqU4UIFrEwG9k3GGWEO2JdAtlSrstXEe0bmRIII6ophdos5sqY7IN1SyafSkvv1vCA2+DbiPa1xHhO4JgrWdjhJiAnDiKiAZBZmFUkkF9GVGsOOnIya9C5n5DA3lcEZb4GhnMOhjKAn47TTPUqQl2Kss8++e6k5fuYUspQDrtiv6LZ8TX9Y7KclEKTBYqa68gBLqXeo07sQsOBPZkbkcB5Wl8Ch4E5UFA9WZ0c5iKn2T/qWq47vyYD7uFhYodAwJ0DYHi8ef6qbVpGRyCKwIH94YT7pqa5UuTwdJZXZhatvp39o2bP9w4NPWyuoyeOvW0E5zlgJVdbuBalpkKkSY7hnBJp3wDjKYGLGmbfFetA+Jrd1wTMloPE/qEswJjeXHrrfJ26NNP63EJA73mcpiMeR/YuzpMEIjJNVKbBLwS3HckzmAFc9iL6NtWxLMh87d4TqLaUyvaPRWHyEGegowb6AW3MgyeCOLKR/OlahrBkdikDGt2na3JJFtzsGfDiFryOhjSplas1CICA/lxCp6RTs+PnYzhNYTyZ5UpjAtfyM2d2aXJGIHnouI7xWKYaBh6fqAXqVBGSa27qNYFVc0twJDwABfRnBLl2IaAP6R4j0j1xQqmQ4HEgqAxmThWVnnL43JkqApNRhQcmI9Hc0utJv+R83Ko9a0DyxFTuYlQTgTdTLdM1kNjPKEV5r3QRw7EskzD3+yDbmwjb8FxNS3oPi19xE9CfK5g+TWB4FLA7rQTZ8vTe7F8UgezZeEypbnPjsX+soYcIQEBNAeOZgu4jqrQu1/EzMKXE0nzOfCULWB1Omayaa2CiBKZzjeaOsl67UPAVa0pcLSAS6NY9qzyGUx5fevQSQyaB/mHFfV8R2ApPWb20orB7aozlmPqW1SQyWQHGEwU1ZfaRn2lped6zfBlIoUFjZg/5HUBmlIBXJuldYcyMwHSkU/ibKMBWTvw8uqWA2XjYlSIYT55JvZ+9jkEJ+FaVapWoyOz5WpZakag0hoc1qrXDdGSgB0/2LNBi4tL2TCcVzNbCd5qS6L0r/xc2FqCpew9oBgKpKRS/pRo4rhnPagLmVKExnbdQBwchInxH6W6o+aUZaoIooQWCTl7PYwN9IEjUIcIek21Uo0/VJh6yd/BdVdg8X7NzUlqfAFMoNSRuWUEdLPyygtpN8MsKciTIjEZC9g4wc/KuHB3cqqY1RhGQRsPwnygF9HYsCa1+WZM9BAjYEkiUu6kAb6aegu9RChiBuGwgRktAK3j/FoJ9lFEpJsdWKZRn9DMTmD2Rzs++RpNBXShJrrFOwD5GguS6AsZpZvnugcpSOzLe67bMy8xJr0IAPoHNXElyH7ApyeWn44DsdU1yXCjF5NbSeylmABmTtNf9kBT5/kNJMrSVmcFm6cyU8+sS43lfJvu2DPg+TPgR6wHeAp9CAfEfA1j+ojx+TmE7jwH8jXT8NYDf/GmDSuBnyFj+//E4/6cfxH/hr/4lTEHjyAx4t72DFBF7V+NiXKJRFoM3WI8thIholIOWHjtbQ4oIGxRq5bAZGygZMHmFj46v8LDaYedr/M7LD/HPPn4BFxQ2tsHt0GJVjbgbWqzqsVwYtXKwQeFgDU6bHtd9h4fdHjd9h0fdFq/2R2i0w9VugY/OrvFiewwlA4SIMDLg9tCi0g6V9jiMFZYNl33a9Hi9XaHSHkJETG7+0E1O45sPLvBidwwjA5QMeH55itWy574ah5vtAst2LO/r6gm7oYb3EkoFHLWzGXvTN7BWYdWN2OwbSBnRNSNW9YSbQ4uzrseLqxMsFwNTc1XA0Ff42sNbfPbZYxw92sF5mbzmAf2hRowCVW3hnISUEYt2xO7Q4HjZ4zBW/FLzfM4ojxAFfJAYBoMHJzscJgMtAyanIUTE7nKB0ycbTE6jrSw2+wZPTze43neotMMwGSgVYJTHZttBGw8pA7yXCF5C6YBp0GgXE6xV5TgIEeGshjYOU2+wPO6xvV5AVh7RS6jKIwLQ2mPqDdrliOFQoW65b1XlMQ4G3knESWL1YI/9tkHd2rKOft3g5MEO1ivsb1tUS0qeY5BwOwPR8By3iwmHZyvEVZr8CGB1jJWQtUcMAtFJmNbCWYW6tbCTRrD8m+0NICNir9A9OOBw0wFWQCwdJ1tkhFARqvJwewPVOcQANN2Ew22L5nhECALTbQO5tFAqwDsJcVFzMmTFG7o5HuEmjXhIEp82Jdz1GvVZj+l1x4kCz25Z2TiEUUFUyTurIoITlGSfTcBVjbBMtKsXUCvLVGIRIaxE1BFQEaLywMYgLjwHAE5SCi4AudHAgxFhr4EoIDqHOEnW6iwdq3skABNgLgzsiWdsezrO0BEIHMCLhUPsFeReIdYRsQoQB8UqnYFSXZcngFL0u75TcCfzBFU0SboAQG2T97ELUPv5SzfLsVmzkJiZiKIYgADXuVXsn61jCvPKzB27ecUoEU2E2ik+LyNUL+GWHiIKiJFl9UHxHOqNKjqSqIDQBiAAei9hTzyEE6huFOxx4ORYDUTDDlgIzB24gyh/z+XiruMEUWjY70u/NjeZXag5qCupEpAmMLqA5o3CeM51ui49nyexi0xdzDLUgJLKPJ5FyDzBBdDG4PmPbBFTfKXlxFWoqbaYTmKR7uaeSyb9cjJOeKomokBRt6RDXxQYWdGQw7aYpMxjonezXDe/J9RxTuqekLz36XVVRH0rYFd87n73bVFfJHbcLTiJJ6fEovZcVq4ecu2sPlBJVmwXs+LCp0me8TRPyMzHmunO87bnEDOZJoI48ZEmPNNEFHMAsp8fhXXyFdeTA6PMnq+pb5MioME8aaBmhQcnymgVyAFgUaVjJufzxc9hOl5pcjMzwPnv8d5YN7ObruNyc/9nvhajArqLwKoXwW1mjgInv3IFTXPDpFsG2CAB33T52XkyLAeT5fCmfKyzUiYrAvJko0x4gP7luZs3qvlzlKW9JbQtxDKp4ytR5LXCM9PAthJ6yEmtBO88f3xO+pxBkY5NI5MXf2bhqEaIaTIgT2i+rSLInaZqyuuYmWqA4N3syMirBI4z613t6AXOagFEgv7c3RolX+urpM7yOfRLFDWTPvj5XGoxM5NI13N6zuxdkhsnljSxsdnvnG+xOfQos85zF2ag1zezpRIQU4C0nr2gLvmwAQLi5L+Wo0OoGRaEe0wrXCiy2sJE+1i2HYkBLhUnk0NoUyCQlOX9eVki+2/vJ7Nmz2MCk8IH9mXm5+89RPaTZiYWeJtdTMt/K/wnA76y/YnFzBLa1Mf51uMem/kT8ci9dX/Vw3se/cp5/Lf++l/8qS/3r3zrN3/u+wb8jMN7ftYPAUDLgI+Wr/G/PvsmXrbHWFYjPlzc4Nn2BN968By/88WHePd8jYWZ8MXdKXwU+PD0Fq+2R1hUE9ZDA6M81vsOSgb8nc8+wr/yzT/E77z8ELttg2eLUyzMhO+8fAzvJN6oiOAF+qMeIQoIETEMBjHwA/06HOP8dIcvbk8xWY1/5uwV/vDFOzhe9eg/O8JnIsI5hW88uMbHXz6FEBFKB3gjsd4YrJY9rFe4fH6C3aM9pIi4WTc4XvW4ec14TrOcYA8VvjBnuNt0CEHgaNXDO4ntroWpHPqxwrCrEIJA8BJCRGyuF6iWE6Z1DbVwGAeDEPjtHSYFoQJurYYf6InzXuAwUEv7/OoExnis1x2EiDg66mHvanwZz4AI7HYNQgJI0gQ07YTDqyXGscbTX7nAixdncE5BPGux+yhg/HKJeD5B6ggpA/aHFkfnewyDQZgUrm5XCLcVHn90hclp9H0FuVMYJoP+sgMe72GvWjQPrzH0FXZ9B7FXrC4xAbL2GO4aLM4PcFbD7zRcQ4A2yArBCciLGv7dHuGyQawDsAJ05bF9vQJMQAwC6jV1lf7EwwdwsP2RR9ga9JOC3Grsjy2kDtAva7hVwOHTE8Q6YAgC6nUN2xDM3V0vgQBUrwz8Rw7+roKwEjICsVeAB/ZOQo8CXigIL+hxPAeqS43pVEIOEtARbstZ9elBRP3tln2TQwUsCDTUTuJQNdA3moNuCSy/rzGd0KPozgQB6JsakMAAQPQKg2shrIDuJVwEQh1QvTSQjsAHKsJcGbgE9uoLBTkJTCeJRd8ojKJBfUdvLHtBBXBZQ5iIaCWqK4XpiYPcKbRvJA6ooEaBqBTUQcJsBYanAs2Fgj0KQCB4oRdWkWUeJCAj6iuJ/psjsNOobiWGVkOMkr2ijyPUmsdJjqbIqYNhMnEUCtHE1GFKGbgamJbspcLRtw36RxHRCWDPShd7KqB6nhd14PZJJ+kztYBIftJgItBrhMcj6s+aAm70K/ob1UA2PGigvuIgyZ541K/o38oMOgc59M22F2QG3QKIKklvpUDoZQE0UbNGpnvBjtbpiMtjfUmE3lOmVt3S69le5GRfBdeSwVcjZXHVmqx2fUPwJfYSIiUCI8oEXCL0WiJqYPllxO59KgyEBc/nNAOimLxq7ZZAcFqJJGMlE68GSWn1XqC5pK8bYe7LFImxU5ZApblK0mbL30Xg612HMlDUB/4uRw7G28skp5WA9QLLFxHX5xH6Tha2P4MoXyOx2ECoZtARNI9BdZd+rjn4z7UsBF6iLGf5krLr8ZggpVonFnhPkCJHqg+WL/j36UiguiMoMDuCleEBGfV6HXF4LNHc8vo4SE4YZJm9GgmQhgcR9S2tBtLxPC5fUHLrax5/s41QlUD3hqnkWW3hFkw4znLr6XhevtnR0yk896+7yExrBnMEbYfH/Lw0t6z2oQWE7GO1iZTAL5nIu3gdsX1XEeQ4yufVEAtAMfsIN6Ew2yr9Xm0Ddk8pwaUPWVJCnaXhAewurRK4lijAR/cEk36X2N5OQFsysxBI2+ZKD6caI/ZPFI4/m7B5v2La8U2AdASw41EKixsjzCEUFri7CKyvSdJZ6QERBMwhwGw9du8YNCmQrrnx6M8VpAeaK5vk5ALTsS5SXDEC04LhdboPsEuJ6s5hPGPPZX09kQndhZS2THVUfT0BT2p2aqZEW92zk9RsPaSL0DsP36rSGdncOEzHHCJWa0tf5rliN2cfEBQZ3ebWkY2tBfQ+FNm59DFZAdjPCTAszbUS9c0InNeobyb0j2vIKaC5jSWR1i0UqjsLXyv4lsqJ+mbEdFwxwXZVpfuaQn3VY3jU0oqTEnHVkIDgBAQjCxsvXITe5/A5D2Ek1GFiAJqPwHmLKAT0nkFgYvQIrS5Msho95bQRBHlKvCXLjUlubG4OCLWBuT3An3bsyhwtwqJmaFIKNYoRlO6CywuLGuqQQ4fY9Sksj4mYLGB0kvoSqKlhKtdWrNl1CYAy2pCY0yTVFYcBMGR4hQ9kOVM/5g9LVREjYAwZ0cwuDinYKHdwAgxDqgyic2Q/AwOIyhhdScptAUpkgTkEKD9UYnJTcuyPH/DfA6D2x7/snzx+/o+vNLDcjg1+74++gd8zX0d7NODNtMKtadE7g9fPzvC3dh1sb/D5548401QFnJ5v8fHzpzCVw/UVEyFk5RFua8TWozvu8T9//GvQxiNMCs8uziBlgDtoiIOC12QCNlbyfRNZoJOzPdbPjyFPJlw+O6Wk9mTCb3/xS7B3Na4GDRWB7c0CcqPx8asFByKth3zZYHx3QNgabF828CcOYlDYvUkSXhlxta0gegUcW4jvLyCWEddqyb7KUWHtJMSNgesCnNZQrYe6qmAfAvFAkCIEMPma8t61QQiJVQGBSzCAO3NQdxqhinA7hWBTX6EO8KGC7CXqa4n1kwrNtcLYelQXGtN7ZGLkrYJ94HC4qSBHymm3f/MJxEce3koYDwyvF2RPbiqwID7A3CpssABGieZCY3jiIK3A1R88gjt3MFca+iDQL1rIXmJ/sYDeSXz3k/egNxKiS12LVxLTA0DcGCgA/X6FKCPqWwW3VNB7BoVIybCY0TQwO4F4UMAtw3vEIsBcmcJY2FUoQUbSCUyvWkgJqDVvgtFo1C8VogGEp6cWULCDTHJJAX2QsEccdOteQP1Rl2byuR3DwyR9vKxQ3QqESqV6kojqQkN4ge6Zhm95/Y3nAWYr4camyGilTdKrXsHsBMR1zcHtBLidhNnGJOUVqF6a2UuqIxb/V8sqEUfJcXUnoA+awUl7kXyiAtW6ooT5hp13qheUtRpJsNUzuKR9M3frxVsCD2RZbMtjV60lulf0NnavKemSE1kYtZcpPEXCbJFmsbmfZg/s3uf+6x4wz6viE0U08InpMtcaesdBpD0CmkvKkNXEgBfpKFvLrAtrCRhgJN+w5/LoM6B/IGCPI7pXAn2QqK+T/7EWEFHC1xHLZxHTCftbzSFi/45kbctNw8HnAUW2TlmhQHMtMNnUJ3kAotKob7JskrJb9gqy2kMfInQPDDIVqvcJ5NQETvTcztJiaVmvEhQH69WaDMjuPQmzJ2ukBsrYfcN+xvqGktjdu5S/6gO3L2qCtmoz+08BlIodlZZdXxP46rRt3QVlpmYfcXgiSp+lHAFVA4sX9At3bwjOqnVMjAYlnFEB3ZsU5GF4HdR39CfnblX2ZJI1rTYRvRZoLiPG8ySdBWtKsh86B9gIRzalvZBoLglAqi1lwvpAybOyyROWKjBYH0XgBRAU0888J/4Gxc98MJT3ml1gpYFK2zslP+QmYtAC3RVn8oWPKXU71X9cznLr5TMmlActsHgVUugKUN/legwkrzwZzONPU7jXgaDL7HOVBwGxGiO6y4DhVMIcAvQIDMcEd93rQEZPcMAqooRdpJqbHeWo2/cUqi0QBUFOfy5Rr8mMCc97Wnvt03VOsMckZ14LegxAlInNYmUOXw9ESXmsOcyVLJlJk1bM40vB/cs1I9Ud35P9tfSG32NARKqh2kZUW24b2lSlsyMzpcYI1XvQRwG0V64wTe11QDBMRO8uWQ/TvZkwPDBo1slj6Vkz0limYGcAF8zshTUpFEpNAe2NT35bhuPokZLxzMLpg4dvCR6jyr5N+iaFC6hvA+ToYfacXMoe2aAEqp5AkX5Vj+rO8RptJZqLA8aHHaSNqG5H2FhD9x7VeoJbUBrLZGRWpeSwm/aa26UODgoMBSNzF9O9mhUmIaUTy8nDbDzc0jB1PPLaQwRU7yFHBzUYmM0EvzBQB65T+Ai1Z7BE1KZ4VaWLEINDPKphNgOEZ/BMtbbIwTJqMyVWsSLDnlKNWbEyFZ+nCBHRkXUUhxGxriBHyn2FC4Aj08kU5QBzm8IcU1K9mBxibZgO6yOEs8BhQqw003C9B6SA3E30VYbAACDrIHeBMtnUUZnlrLKXc2iP83x+ZPqqmCxltPdlryFARPo4xWQLCKQvUxBUptqT2HCSXPTjzJIe+pkl/KG6DwHLEB6tEPO2AHPaLPiZzN7PkgILFHbybZDJdNq3KkdSii1fIH9sciyAPxX9lfcf/sc6Bb/6j6+0FLb+xrvxl/7zfxcxCuxvWyAIyF1KHTshQPIdL1p9PCG8blJiW5Lu1GRYgolFzqUO9OwJJ+BXAdWVoiwtS5w0v9yytIUDffr63CKmqhJun+4p98oz2UCadW3Yx4hIORY7D9NrQVlcDqwJmjPxWTInvIA9Yo+l2dDPE3Us6y+BJYcUAhRESSv1FYN/IFGCbbJEyDexeJaylAiYfUF6T4bDLUJil7ie+lrg8A7DcqJJ+7YRJYhnPOXgNMu9slzK1yk0pyL7Iq0okiZfzwPxnDo5nXAW1awpbcoyHIDnM3u+csBM0CBTYWcvlq+zlC1tS/793v6GmozPcMafedY+y7KyZw1A6acDZhZDOJTwpuLpSp4t6d6edQdQ0klFRFmXTQmY42nA4rksgATgccl9lAS+lAPZhSgz+a6d9ydL2fRAT+niJVJy4Px+VlKghCBVa4b8VGsuwx7NrJPw93ooMXtL9f7eNeMpt8vPVxsCGLsQ5ZqSE1Ji5+wBzCFBMg2gQ42STOkTM+RbUbyRpeOvm+V53ZsUipQYCwY9kZ1xXeqqBAqroQYUCRdDKsg25eueryEQyV5PAMVLWdbREmAx5VMUCVuW4+UAFuEA37Kq5b50LEv3KHXjbH8eQKuRxzMY3ndyh+P9MKio0/nT9wJn8kPO13jU9IPlfQsVj0n+PGWmJ/cfSpvAah+LPDQfm/y6++FjeXt8heItzK/Jfkj6v0T5nNDbTZbDN0B7FYonM9c45fvvWxK+XMMjUxhUkpzmbslql+SNcWYaWdFE0Fdtk5dvyvs2B93ogcdoWsgStHP/mJZJmdSHGIUo94hcT5WlgBlQyOTDpO8TKWV6PpaseWKnYw4Iyt8b2Z+r+4hpRTCpE6Cm3xIpeVWkeygH53l/MyjlOYpFwqimOeQo11u5NvVAps9yfn1UTE+Vnh7Kcg/IXkWBEuyUq7NKwFHkPuk+hVMpPjctJbs1h8DJi/z5CrliaU7rZeJt+twIzBJpiTlUJaWxymkGD/naIigg81Q+8z6mChdVgGi+Zl2XQJ7nxIpwMbHfDAdSY8B4qmG2viSL5joiTkbMDEsO8sly8cyiqYmMovRULeXrL9epuFbB7FjRUgBRritR+T7B/9Pzm0KSUqDN/fAgEZKEc/L0Og6e11it3jouckwpqjJ3WlLKGWoFeILUkrLqA6KRlIC6wJqYBMb4nSHTOWFFjMzBOp2BHGwJ5mENikVoDNm0EBmYY31ZHw9kRGg09GaAX9UEeUXmSqAV6yQPzbUXpYsSiDWXWf4eqklswwAAIABJREFUCYIZDhRn5i4AIoSyDezQTL2UGZRpRVAlBOWdUtJ3KESpJRHWUWp6X5IqBISjp5LbzX3Ix3P+EIi3mcT8/5xIm1NUc0BQBql528I9+Snw9rK8fxtQ3g/Nyfuo1NvP/XAwUQ7+ya9JIT5xvFf1U5JmQ1mfuLeMeD+AKC/rH/SIobCf/yiPXxQp7L/51//ln/py/8tv/Tc/930DvuKMpVIBXz+7QYgCn+MMXT1htBq/fH6J27HDN371Gn/3xYf4C1/7Hv63L34Zv/pnP8XvfZfs5lE34OLiGM3jHYzyeOdog0+eP8HDB2vsxwrWK0yTQvvehLtXR/RPHTu888E1Lm9XcFYBmpHK06CxPD1gfL5COJ3wa197ie9cPEIIArY3iKMETISQEauTA/bfO8bqz1xje6hhbxssH+2xu1wAXuCDjy5wve8gf/cE8VsbVCpgd7mAWjj4jQFUhGwd5A8aHN7nTVodJOI3BkgZ6VOcJKwOqFuLsTfAdfKvOYnxqYX+vGENQBdSUFFMAwsCwuaDLfqXS4iTCfFVA7f0mB5GLB/t4foK7z++xvc+e4LF+QHDpODvGgwGqG8lzC9vMHy5QmgCpnc8FscD9nctMErUF5rR9b++xnjTwlwahA96+IOmD67zgBN49xtXePm9hww8ejwBo4K5UfDvD7CPFI7O9ui/fQJ75CEWDkJGVJ+2cF2EO/EQ4xyzH1r6zyApEZV7BfdLI/Cqhvlgj8NFh/aFQv+uoyfKsvIidAGAgntkARGh31TF5xQMpYrNlUT/OBQJYmw8fX6PRoStgbljxUf7WqJ/6inVi0A4coBN3ouDRDhy6L5flaTd4R0HOUh0zyX6J3niAZhOQpoASV6PlDDsvz5Af7+Bt5xosEesNxlPeTzUVmFQEbGK2NaUG/o2wh15tC94CxieeHq1eokoJcazALfgJEbzRsGuItOPIyCCQv+EkwmhjrBLToLk5Fg9CEwngfKzBHbcihUu6kAAJmtWlUxnHkff1ugfxZS4DPhTi+V3KkxHcziTGgQGzYmYYJBkraIEI/mGx2P9ywBERPdSYP/unLgctMB4HtBcyDlEJU16ADyn3WtgPOP/x44+t/5xhEhgKRimHasxe6FESRfOQOrwhBM6qy8IXG/+jMPyewa7bzgsf6AJeOMckFOCoAaBwztA94qDb9cKTEdcdjyZJ2T6R/ysuhSKc3jKdN1gyDQ310ifab7HriKaS1FAYV43JY9kWg+Pma7sUgKw2RH0Z7CPwOPnG8AuI9oLUf5ujzipMJ5xO1ybKkGWnARpbiM2HxIkZTYWCYS1V3N9R/8QWH3BwfL1rynIkWFPvk0MoEh9wanCIS7S8c9dwSFN7iRgOR2LxBKhhLcAgOnntOLpWKC+iTg8oWyTYTRk1PZPJeW1EWjuPHZPWZ8xnoo5jCdNwrl2DoWaTgS6V1zftBKYTsiOD2dpgkimiYqJ+5aTJkWIqHYRzZ3H+kOD7PHKAUTTMc+L6siK+gYAyMaffnfC+uuUejP9OR8TAbvkdojIiRlpAX86y16jkqg2EYdHlKX6ilLc+o5es8NjsrPmwHAeNUUcziXqu1Qlsk0TJ9U8WSM8z/u0Sv6xNPE3rSj/rXYBtiPTrwfKSHVKAB7OJOq7wAmbdL/N4Dj3EOdeZukAu6BCIANsk/qWi69XogQ5lYkMoFwX0gu4hSyJvTkxN4PS/pyAs956bD4wOPqcXbf7xwrLl45px6e6pDPn2pH+TBefYa5MUVZhOBFob7h/qmf/bU4UFg6ICVjapUFzZaFG9pZOR4osqxJlUkv3HtORhtl5TMcaqmfSrV1pSBuTzJUMYEigNQpANAz9k0amXln6DEOu/ehYHyJ9hG8UU3rHCHNwsMcGZifgDb+/fGvQXA7YfKND99oytdfG9D0pUa0tpqWm17h38JoMY6gUO14T+A1GwiQ/ZGgV9GEGDvakSaDUI1YEafZBB/iI2GrIiUBfuAh1mDA8aFDdceZWDg5+YSAi+1EzUPSLuvTpZs9j9k9mUK42A8RgEboa0Uj4sw56OxLMAvRJDrRwyP2A2JDljkYx2FYIxBAQli0nNaxn/UhvEZKcVISAsNDFExmFIFgFCATrFOufQKJwYMjQ7oBwvIBc7/kZ0wqxriDCSJBamdk/mbtDhwmxqQhePftvYTTQD4gelK3eA3pCpwRaaXDfSylyWFGtZoDnHKAJIsVigTgMlMZ6z05N5ziXk4OAEugUMc+o83HfB/1jH+FP8qJf7Afnof70MpZf6TPkrYILEh9/+RSHZytcvjzB4Tsn+PvP38UPXj7A3/o/fxX72xa/9dk34b67wieXjwEvMOwrXHzvHOZ5hf22webTU3zy7AnabsL1330C/7+fwX58BHvXYP9Hp5AHCbzfAwJ4+eIMIQrEvUZ43cDdVRBbjfE7x4gmImwM/uDjDzBct8CnCxz93zUQBRbfrRCthP+dUwgvcP35KaY3HcQosbtrodYa1ZXCF88fYP9yhf6bI4Z9heHTY5gbDfNpC3OngCCgP2+gBoHuCw21V/DLgOaPWuBZi/b/aYFeQb1oMPYGYWvQvpKoX1SoLxROfruBPQnQewG9S9KUgaym61j70W8bRBPRfNyifSNw9KlG/drA/f4JzHc7fO+zJ1h+1+CwrYHPFhCtQ3MhMZ57uG8fQe8F6kumDDgnoS8MqiuOqA8fONR/8whyo2FPPJq/36F6ZTiAuNY4/kTj4vcfY/Elw0WO/qBG96VGeyGAqxrLTypsXq+weMbk3bO/XUO8qQnsTETzUmP5hYTe05e2+q7G6jP2cy6+b8gUv2wgvKAH8wcKeg+YO4X2hYLZCjRXEme/r9DcCIi9AgaF+k4UNqa54g1BDYDeSbSv6emTW432QmLx9+hT7F4J1DcElWYrcfpHAvW1hDBkws2dQvdaAl7g8DXHAdgINK80jj6V9OL1AkefkfnrXioy6McWzaWE2bKLMlzVJQm3fQOc/iErX7rXAmonYXYCR9+TqK/IzksLLJ4LHH1CoG92QPtcobpRqG8lqjVQX9ND9+DvsX/TLwLaVwrH31YYzyOWn/N1CEDsPKo7geNPgVBFdC8jYmLa9Y7L6V5IPPz9gHrNQWZzRdmkHGSRui2eMZVXXxm0F7Gkx5oNU4HVBLRveK22F5SFuhOHo88DutcR03HE8afA8XfI7Jx8h4y6WQs0V3PwjOoZ9KL3BAEE6EB3GWA27EbsXgL7dwmkl8+Bs4+ZhHry3ZBqewjM6lsg114AQHPNfSD4Bc5/lx5PtZfwhoDersh8TMcR539oMT518DVw8m1g/17A9uuUSa6+jLzG9tz3DJRzuitAsLZ4HbB4ScDXXEb6AhvuV3VHmWl9F5Nfj167UAPdS4HDY1YHAcD5J44Sz8hQleE84uR7HtNxhBoi2jcMnJE24uhzV5hqaSO61xGHpxHtVYTZcRtyzUtm0ruLgPouorsIhWHLLFFzJVCvA9QY8fTvTOx93ERUd7F4+6ZjlOVS7gi0NwQB4wnlo66jN669iFi9cMnPR1WA7sma5wqS9jJieCjw+HcPpbKn2sQi4623AYdHAjffNASuFbejWnP/sgQ3+/XojUvKikNAdxmgBmD/lLUy7XVAexFmqSt4zYUKWL3wsAuB8Ujh+DMLaSMWrz19Y5uAo88pT7UrkZJeeX0df2bp5bsMmFLSrbTA2XcGsrKbiPYy4uxjW3yfD/5gj3rNa7vaRAynIlXscFJj+cIX1YO0QHMX0L2xZb3TCmiuPboLD28oUdY9wWdWwEjHc2QXBFJHn/VYvvKotpTSTkcEnPtHlNSbfcB4wr8tXlnokXJVc2CnY5bStpcWZh+QfZDLFxbVlizN8vmY7qF8b7XlddZee+g+wOw8qq1Hc20RpWDCcU9ZcbUNlJtKpuQuP99jPJJF2XF4qLB86bF7R0HvPY5/MMFsHQNuxojqzqG5stApYCYmsJjZ1dXzEfWdQ3MbsPziALNx2D8lKIQQqK8mTCsJ3bsyuB5PDfbvGJidQ3tl4VqJamPRvtoT9D+qYHYe/QOD6s4xnfqBQZYGSxthNhP0gd5N3TtELSAnVuzIKaDaWKoltpaTgEbAtRJuoWCXmqxzH4oUvrkYoDYTQiXhG4XqdsLhnRbthYVZD4XVV1OAtAGuU6ivB1ao3PYEbCNTdc3dyAmnywPVIi7CdayPEYEdk/aogrk5wNwcYI+YwhqM5LK0TOyqgL4boTcDosmJwIllTeymHCzErocYyVzq6x1kYkP9wiAqidAogslUSRKlRFg1EIFMc/X8lp2UlYYYbFFfRCMRFk0BnHI3Ug4rBGJTQ12tyUBXGupmR0A7jASb/QjRT5C7A6WxuwNlrEJAWAex2fM1u6Tpt45S2cpA7oYk005JtPfVh6lmROz7OQhIScpl4/z/eE8K+xZ7CMyM76FHHCcIJWdQCSDu9wDIQMbJkqUNEXEYyvJEVSFOE5eVUmjfDhNKjGpOwU2v+XH//vQ8BEKUP/V/vyiPr7QU9uibj+Of/6/+El7vVohRYHQKh0ON1WKACxIupZf2hwpSsoTWHzTgJFMnR8UEyoWFlAF2W2NxfiDDBkDIiOgFvY0RkOcjjPEY9xXioKCPJrheA1YygbJXnJHuHMJdhWgC5EEhNGlGpkrTsF5AjApRRUBGLl9F+h29gJwkoowp+ZLy3gychBXlZ32lMJ4F1ogsGdjiHlioO01pnObrwsJzuaNEaAKqa0VwuZGl25J9lhGxTUbvXIOy5brlKOAe2rTtkgExW5bX+0WAmATUKGAfOOgbDd+m6WEdycq1ATKlKYaGUuPqlt2VIQWOTOdMmhRWpOoKDjxzx2iO7A9VZNfmvQRHNZApioLpkmbLGXGRWCURgOmxI5vZczbenTpUbzTcKs7sWs19DVUsvaX22MPcqTkVMvnE1DTLjIUHt38UCG1AdcNrxq4iulcSwzlTGn1m3FKKZn6/2TCxUu/J7vk6pYVOAu4ooL5kuIq0lDn7NsItIsxaktEb+cXPVEXKkVkNQi+hbzjIUYMonqz+HY/2hUrsBxKLwWAbe5Sk2CrSO2qzpJZhI1memxMw5cTO1SxppnyV71U9j6c9CWifM7xG7ylRDRVl4Zy1JxORQ17uy8pjYvrUkMJZkHx8Kskw03dOlh3nBEomeqZzpriPb7FbCfRkOXDuA9UpxCXLbEWY5cdZ2nm/m5SAjwAm96NmX+l9GaUak/xbcFA/nsw1GCFdW9Kl5MiJ7JbrRJLQpvetk091QOmpzceIPtT5mKgkdc6MbpbKqzGif8jPWXvJDshQc3ulT9uSZJJRJWm64vHQ+7QvuUNT8u/tVeqARNqO5H/zjSgy2xl4JHn0SiQ2MlWd1JTNq4nXv2tTEmeqXIm5izIkuWVFaW/u7tQHYDhLaawHhuOU9FvNYx5Mvm5SYuW968p1ZPxyumnuds1dpLmzMqZ/99Mus6xZuliuzWwryK/LstR8PQQzV/5k1jt/fu5PaGcprMqSfoEiua52rFOpN5QQl6TPThSZee7LzdeZ9PM20T7AyZ0M+OpNTIE8osh39Zgk2ILnwOxDSSzNn618T86dnjmdNGjed0yfZbjJ+928LQPPzGNOOs3HIae25mCYbEvIPZX88HPQmiW4pd9ToqSlAij+xdz9GaUoSar1hifG17IwiTlwpoTSOLKB1dbDtTIB8lCWGUxKLq0lgaYWiQlF8ZTaBetKyEgq6EMoUmU1eLiFSvcLyh7tkoylT5LWfA8CUIJpMqAP9xjILF2UNhRfdJYFqyRnlj7CV5KS22oenMopyVm1ZC9oku7G+1JGTaDIBQO+JduZnxM2wK7MDPYm+h3dwkBmqXBMaamW4ChXklD+Lsp5zfvL+3ySyyaJbK4zkdbDHtUw24nHxQYmpRZZrCwpt/l4RCkge4fQUUIrbAAUa07YJ8maD+HolRbWl/RVSotndjVvj7C++CxjlQJ3hECoDeQwzb7KUhtyT1aaei3fksLmY56ZvmFC7BqI+4mswLwc4EdLYHMfZl52lvHewwFFsiolYgj8aS1K5Uj2SOZakrxdWZL9wzLVEGapcAaS/zBJsD9CIvtVT4V9+CsP4r/xG//qT325f/XXf+Pnvm/AVxxY1h+8F7/+l/9DDI885CRKimaOdPctZ8LtChieOHRfaIQKZYA+vWvRfr+CXXEwnQvpfUNJYnvJgZ9bcuCsU/deHtBluY6v7vn28uc/DSpj+qKNgl+8rksgIEn09AGlVD5KeuFCGgi2lxww+QqwxxH1FUGSXXEfmqskiYqpZ+949q+VQWSaac8Dl8y6MdkRc1y8yv42IPsC7ZLbWyUvZ5aXlt65Oy5jOuYgISqCQSZLppMUuS9mlzoMFc/L4jnlaFGShRhPGdzRPxRlncC9QZYiG2BXHPiNpyJJOlFm6aOgHHD5DOgfijLQtYt5wC4tz71dMujj8ITbUN+mxEPDnkQkH5+vE7NxLpL/LknSRl4barjn40xAw7VA95rP1+uYuueA8QSo15SkDeez77QMwNt0rldkorrLgMMjCbPjYMs3GVzNaZI5BIRMTIRbMLq/uY7Fb7f5iBJLCP59WiV5muXvvGY4IMrbe9/DVO1i2S9glqDl6yf70YLhQEANsYAFPaauQo/UETl7wXIgShSUyI1nBAlmz/3NAJMgNfXa+dmjC6RKgSR5zJ8j4QkQdB+LPFf4+fxQGpv8qs28Dt1TDuqbFFzT85jkgXyeUCjexykWT5OvKa3MYThB8/wNJ7LI8HzqWtRjQH+milyv2qXwk83sq3SNKP2RamTi5eqZLwmTGehkX58IQHPLQJEMAqLgclxLaSLEPJDnIBGo9gHTgt4qn4Oe2pTuaXke6k2gXDH1DUbJbWtuA2wriwdOeuDwQKLekJXLnYM5sMN2DERCBgrqXihOkujm+4yy7IBs1kzVtK1As6ZUMyhKHqWNGI8VzJ7SSpV8q64mu0WfIUoXYq4niIpAwC4oi/aG58BXInXHcvm2lekeOgOXXAUgU8ovgNIfe9+jF4x4y+9KBkame43DcKZh+lCAgq+47ObWoz/XJQQney+FB/QYoHce/SODakNGyK4U9D7AN/T8FT+gJzCwRwrCAdNKorkhYzUt51oHkWorXMcuzaiA8ViV3krdE1TpgcBJpeNujzhxVryEaf+L1LTm92m1tRhPKPtjQiv9j9InWWaSMvuGXsL8ez6GMn3GzJ4+Q19R0qn3fK3rVGGOfM0ApFxTkQGea9kPmcGWsvOg1tcKavQImstWI4+l8BH6wLCWXBmBCKjBFeBjV5QIms0EtzRA8mgjXSP5+mAIjiw+R9co6MFDr0fY06YAqBxcU4BUHqgn2Sl7Ig0DZUayam5hiidShAg5pB5GlWStkvJStR3hjxv6GGtdgIrwrMQooC1SGpol2tASYrSIlUaodfEnQgrIA6WlCEBoNCAIHHOoTf6sRC0hDxP8oi4eQrUbEbqKoF5LqMTyFTBXG8pDEzCDEKVjkumrTGLNn+ccqBOVgBgsYldDHEZKRDV9j1FLhucACK3h7zF5ObO/MgOvzMwVj2BEbAxEP7NvhXED6JmsK3oqa1PkrCLQO0lgmgCcTP7U0dILOYxAU5PxS7LW3IHJGpMErGIEnCMTmGWq92Slb3kg87ZVhstNz0Xn56TWGGYfJcD/3weLeTn3jwPwNshL3srZr3kfaCYwH2OSxSbQ+8N1I8AfX+9PeMQQgX+MHstfFGD5r//Gv/ZTX+5f+/X/+ue+b8BXXAoLL9B/zUJagWrNQYLZkTXwHZmQ/lGEXQUsPtewR5xJbq4F6jvKU6Xn/+1RACQH9PU1ExR379Mv1FyK5O/hQG54GAtoK2mUSMzZvXS+agu0bziLXm0on2uuCEb0wL+JAByecrvMjgNbNXFw1T+Ykw8Xz0QJfJEWWH3OxMHmOiagw3U2V4kB21M6lr1Aquf6utd8fX3Lkvb6NkLv5hAaleLZXcftX33OAXh7mdiynvtvDkD/aN7v9oIdcmbHWPwcSjMd86dwXJeagNUPkpyvByD4O8N1COb1gdJFyuyA+oaSuOwfGs4Futdphrvn+jOI6N4QKC1exgJGmut5tnM65qBYOvqH2gv6zHwNDOeM4l+8opdBeJ6L4QG9RLqnNLG54Q2wuaLcyyRZGzvLeC53X2N65XicCro1cPQ5B36Hx3NQUQao7FqjTLBac90X30pgys4MjtnxC9/sOIhur0ICAkxWXLwMWLxKoSeBx+j4uwSUruH1Va8DzJ5pod2bgPaKUic1URaX96/eBHRXAXbBc5Q9ONKyv23xmjKy7Qfz34JiKIpdCOyfSuyeqgLGmjvKA3UPLF956J4gfvHGw3U8z2S/CCKCESlcSBRJI0KqArgO0IfIkJd0zMdTSgLrdUR7GVBtYpkMqfY8F9Uuor4LcB0BY33Ha1KNBBfdVUD3hhK6/oGkZ3AbcfQl5armwGvbtQJ2MXfERQmYJKMkQCAYU0n2t39CX+y0FNi8rwuT2txRXhe0oJduKTCcEhgtXvsE0ihN3b2jKAnsA5q7AGUj6m1Ad+lg9oFVBTYSBKjEnvmI9obXyrQSKanVJ7BLVgsAEIHlSwdlI9rrgObOY1oILF5ZArxbhnhMK5lqJnwKoUnpnQoYVwLd5f/H3pv8WpLl932fM0XEHd+cLzNr7uqq6pYoUxQpWrKpBReGBQPyAPtfMLww4L/AS+8M/wGGJ8LwwtDOgOGtAcMwQEiERVHu7qru6soacn7DnWM6gxe/E3EzuylRBmiQRegCDy/z3XsjTgzvvvM93ylQruIIpqWKQnG4Esal2EcmN7KtyU1Pt5A/QZL2GlBBft4uNbOXfgRqRV5cKdchgy55T3WffWcpUWwC0cDi245oFdNXfpTcDp9TLlcltCeGciXjmL3oCaWMw1dy7qNRTF73+fhi9stFVEwCwBTZH8QIOHUG4igoVp7Z05ZoFO2Jpp8JaJHfd8v0VY8vNeVKxjh9IVLTUGmWX9WQZEFBpeP+o1W0506YxYWmPbOU9x7TxRFUdidWjiUl6gcOuwty/2+FGQOYvuwkLMcIKPRTPbL2uk8ZtMr++5kwc+Vtm+soNIdrR/W6o1z1+fpIeqmwTYpYKIqVJxaK+qrA1pFyJVLL8q7H1mEcS3vu8HOTqyQszYXD7j2mjVQ3HbYOmEZSTomJYttjd3K9+oXFtBG79yLrvO+wjXywuk0rYwJsEwT8KvElAnRLRz+zAlBzAI7N6ae2DrhNT7+0dAs3hmIlDd1pgV21+LmjvG0yMNG4TUex7nCb9sjEqQG0O/nczGMr1lIR0VxPMbVH9xF31xAmFlP3xwAXpegXDnsraXFh5rD7HntfQ0yEqUV3Ic8JpAbDL0vi1I1AymxbdOPpHszEWzgvMAepDjHbRsCVUehDPx5LmBbEyhKWuWprXo7AQrcec7eT/c/LUQqqO4+92cmiV+UEjA4sYuOJlcNuGsy6Rh96/OlEZKVWY19vCbMSfehkv7MSvWsEUGuN3tToTT2G7WAUZnUQgGmPMkq8nIN4Ms2g68geqrrD3GwE8HQ9en0YASdKEQsrgLDtxgAe1Yq8U7UCRvW2FimpMSNTqNoMBrV+62cSBhRFutpJUm5yVsaqlCSzDv7HSSXgb2AwfRBQGQL0Ii/FZ5bRGJTLdR/WCpOo1Fvf3+yVTLu97NNa0pAU+9YcOsi2vZd9Dv8eWMeB0RzGk5NuR4CodAan+Z4deioHgKv126AyCih+8wtj5DljZHt/xte/sJLkXz3+Ujy+14zl4rOH6ff/2/+I68mG23bGwrVcFjue1qfsfMm2K7nZzfjg7J59Lx+S19MthfY0wWFV5OALmmB5cnPO3//4p/wv//Q3WZwdcCYwK3qcCXw4v6MOjj9+8ZhZ1bFvCj65vOGL11f8zUdPWXUTfvHykpN5w8V0z6vdnN988Ix/dvOIwnrmrsMnzdO7E4I3fHh9y8x2NMFyV0+ZFR19MNzvJ3x8eUsXDMui4Y+fvsPj8zVdMKwPE6Zlx9T1PL05pax6Tqc1zgQmtufnzx9wujywq0u0jpQur5ipxO5QURSeWdlRd47eG2ZVR0xwPq059A6nI33UdF7CXCau53Y3xZpI0zpO5jUhKu5eLnGzHm0ixkS0Trx3uuLFdsGyavnmi2uuP77hbjPj4wc3/Oybh5AUJ2d7TiYNjbe8/PYMPfOcLA/c3yxYnu+ZFD03q7lIkmuHqzy+NyidmEw7utYSvMa4QPAG6wLzacNqPcO6wLsXK17vZuzWE1JtuX7/js4b1usp2iSWiwN3r5dS6KgTahJIjeH0esvq2RI970lRMZm39L2h35S4ZSsS6qSwNtB3ln5bcPZwQ9M5VEar3htC0ESvSQeLuzPwgz2hN3Bb4h7v6VuLGhdBE37vmF/u2a8noBPTecthV1JOesKXc/wiok46Yq/lM1on0sGiZz1F5Wm2JebWERYB3WjSWYervEi/7ybMr/bi3S0ixga6VYnZGnjUjuPWv5zgP2iIXqPXjnTekVqRZeuNRT1oCOsiS7iBXqEaw/yDNfUXp/gzCY9SvTBezPw4TlXlCc+tGxOa7V7kzmmaw4u8QnkFpz2pNahOy4r0JKBaA/Oe1MjPk5FIfgC17Ig7CTihiOiNJeYx6ia/NqijfHzYDxAneVwHYVxQEBcee+eILhEnIimHPNHuFH4ZMDtDOOvRWyudmnuNP/eQK17sxhByGFYqkvh49joHiGQWq4qoXPGja0OcBYoXlvBxDc8riKK28OciZ0dBmEXKVxICNaRR+4UwRGkSUbVGd1lCV0v4lsn/D5V4Xc1OGAm/iJhDBlDnPdNfFvhponunw9y5McjLbTT9IpKs9F6GKo2MlPaKUCQWTzT1dRrDWnSraC8Dxb3Bz47BTt2pXBdTa+xeqntMI+NEIT7vrKoYtlNs4XCdRln6IHXvlxEPKhOmAAAgAElEQVTdKdxWpNrSUyjycgnHkNdJKrFs3+0U/WxIdZVgLp1Dr+xBApzKOwmBSgYmL5QoNeZyvpOC7ixQ3hncNqfzFhLslAxj+vDAhLudLBh0pyJrV7nUfii4N60stg1qlWjJHlbprDwmFL+hMslhU0PgzpjUXct5063sU7ors9f6lSgPhrTe8j5xeCzjmb6URZY35aTRMfZ46h7qa1molEU7RrnzoNqxewluGoOshsTfeFxQlGoTkXIPC1nt2TFxedjv5EYWOEwr4UfFSsBqzMc+pLcOyazRqZw2LZ9jEkDzhox2UJD4LIOvM8uck521T3lR6I36nKxgkEXHRHWfOFwKO9oth45aWfwDWdTtc+XNcB7dflhIPKp0+pmwupNbkQ0PCcWTu8DhyrytLLJHebmr5XwN7yk3mWHvjym+ScuCQXMmVSihOEqPu5n0u5pW7ut+Kh7msWYjVxYNPucxRTwxSmsHaTdJEogHOeogww6lVM0M25GQJAEYvpIFiX5mKNaeUIlkeEjKNp1IhU0Tx0WdY2BVfq7NbPle2Pkwkc/sIQ3ctHEM5TJ1lK7MJIsH/cxmyStS2VLI/t3O46eyAGcaP7LOuovCBL8RBBQLYbGHxQqz70UWXJqjiiEi9Ew4nhudvZaDZFzXnjiRRQB96AjLCjK7nJwR36cZdNrqbda6FdaciLC3zhwZ2ZxWi1LH8J/coympuJk1HBJih/oRgCiLZHT9sYbkVxNZB2ZxSJ6FI3MK4rssnPgrtTqC35SOoDNF0hBGBL8mg32T2ZT//hl4JCfQft9TYS9/fJn+wf/4D/7ct/sHv/sHf+HHBt9zYLn87Dp98F/+J0zLjuvpDp80tXes6oqUFB+e3vGzV9fU64oP3rvh6+8uefz4DqcjMSlOq5qfPbtmPmtYrWa40nN9umVie2rv+O7lGbNFw+V8TxcMdefYHUp8aymnPdOqpQ8G7w2l8+zrgqvTHc+en2HKwNXZFq0Sr1dzHpzuePr8jHLW4VzgsCtRJlGWPYebKZOLGt8b+oPDTDzOBbrWMpnK6lPpera7Ca7wHF7NmD/c4b1hWrXsDpUsdDWOciYgrKx6wucL/AeSFqt0ZDbpcDaw2kwpq579tkJb0S+FnQWTOL3asXq6RAVF9WiPMZEQNCFoSbg9GOzGwPs16skE/26Lfl1w8ukddy9OUC5iqx5jEl1rMd9UuK2i/RtiPg87R3na4L+dEa860t5mf6qApgeXG15+cYV7eCD47DOxgRA06WVF8e6e/usZ5r0D/cvJmPQaTwXkoGB2VnPYlqS9RS9kpTzdlqSFp3ju6E+iVM3cW+xHO4LX+FcTmSAsPMW8o389IU0D9sZJDcxVS+wMZmVRjxrCfUmqAvQas+zQTybCaLtE0onJC8Phhx32xuHnAdVr1Lmkxc6+sbTnAj78aWDyraW9jJJgGxS6UYRJ9sKeR5j36FflmN5rGkV3EShuDd3jjuJpQX8qQEBNPCkq7OuCMI2SrHsawEZmPy/oThL9hRgJ3b2hv+rBJMpvZPt2r6g/7HCvnbxPJybfCohrzyKmU/hJZqMm0u9ZbBRNHv/0G8vhw575LxzR5rCa+RuTF5XP0SSgt5bpd5rDb9aYpxV+HiUEqVf0n9S4Lya0V4HytSFU4jtNLlG9khRf3WiqV5rmgTBIg5c1FuJX1u1QzC7AYfZUs38/gILpN4Z+Lr5Pu5MJSX8exN970DkwRuTvm08CySWm31iaq0h5pyk2MsmvH8ofxfnXmsMjkfdVd4rDw0j1WtOfJOxO+iHra1FEgMidTQeHh5HpC/EvhUqeD6X0baIkAGnwJc6/FeakuYR+IUFb0Sb6s4iuNadfwObjDK46xklsyNerWMvkZfd+pFhJEur0RaK5lEl8eyGAsLyH/buJ+deK9WeBxZdGlAfpWOnxZq1PmMjkvbqTdNEwyb5ZK9JvSYpN7N9BkpGRSfzhcWT+tR69rs2FBBAd3knMvpVUX9PKuZOu1sT+XSjvRN5rD9A8kH2VdwKcqtcScjR5raivBDQOIVBJiVQaBdPnWU7fSdLuIBUv76N0kG4Th0eKySs5j0P1je5FYWAPR0m3pG1mIJTl7e0ZJJNE8ZH9gc2FSKWLTWL9iaa8k9dNXokkf/pCJsz1Azj7WWT/SO5fAcgC7nUQkFCtIirA5kN99GUqAWlJi1S9XIm1wO1Equ32osxoTyUFePoy0p6I/N1XAjTCREBOeZ/G61ZfCXNfrrL6ZhdZ/0Ck3LoTFr89zYmvGagdHuoctCTMfXdy7B8FmL0IHC619H8GaE4Vs5cCjg4P9AhUVPaQJq1GoJWMnNNiK+B58LL6qSgbpP83v3cr4EJ8h0d7gsm1Jd0sM+bZGjIkzB4eaObPgkhxcz1TN5MOy8MDzexlYH9tOP1FR33lRpA5+EXdQeTZKCg2kX4uybPlOozpttWdp7lwwjQvNNNXnsMDm1UHIvGsbnt27xSUaxlLe2ayjD1R3ntJpO2PvlO3y75Po7AHYeaTgcWTmu6swNRRejdvepoLCeeZvuw5XDsmryWFdvBZai+JtG7n6eeismjOjChlXvVj2qw9yGfq/pFjchMEnA0y5KmAQdsEmnM3VgRNnzW05yWTlzX796ZMnje0lyXlTUsyin7pcJueWBr8VBQd1V2Hn1jcpqM7K3AbkUeb2kvCrBMw6tYNYeKkSzOHFQF0J5bytiNUwpDZ3ZFVHYJ//Ekp/u0+ECo71r7owUea5b2xNJhdJ8mwrfRKRqsxe5H46rofWWMAfRBfZarEx5lKkdWmSSHfKyeAcVJkYJZGsDn4NfWhIU4rCf4pCwGFZSEsqrMS6jMRj4iqczrspEQ1XZYXi1R3BIwDkHS5HGLwXA6PpmUM1ul7GJjSvn9b3tp3kl7rf33bY2flsA2Tk2T/RbUif4Yk9q+CFPavOrD8XteNhKRpOzmEta2oe8fE9XTe0tQFPw9XAqpc5PV2Bgl2TYkPwgK9Ws8BODQSxpMKz/PbE0DAjNKJw77kaSv7SFETOkNqNa2CENQIfrrOEjvD69UcoiIFxeu7JdoE+trxIi4hKtp9QRsVdBpMou41qtfyfq9RB0O0kWbnUL1mv7fgEgdbklpDb4RZ2N1PUTrRNZbYWMzUk3pNsykhKhqvYZ7Aa7yXSdO6yR8MKtHv5d+h03nFTUFQrO9n6FaYlnotLApBgY3g5bUK6FcF1kI6WKKF1WqGajVqZ+hnmj5Xd0QHzYNI3BTCYgVFezeRft9eCxtVKVSniNHxMp5ga0V3W8mHeK8IpVSGaK9obifYqOjuK0ynCFa8teogQUzawz5NoJPzmu4LYZg6RdwZ8ZI2itQb3EZR30iCq2ny6uKtoz8YYWq8rPArpYh3JTozJ92qEPbIW1kFjwWuE6ohOGFnANReJgG61cLSqALb6DExNBmwazOyFPqgs08YdCf3lW4U1CLZtith40yrJIDIK2EaNZh9niDVWsZZK9w2x8q3Fj879s6ZjUwSSGDWdmRURnnu2kq40K0AumE1vdhooknYKKv9IWhsLXJVYc3k5rBrYdsgr0AfZNVbeuSESfSdkp7QBOpVierBrfTYkcrzUiat9zqDJAmHikaClszWjL5Z0yghH6PsK3qI1uD2Ku8fubYduLWwoqYBlRTJSGouQCyE/XM79ZaXurwxhKlIjd1W4/Y5aCVIvytk9mCV+ypb6bBFSe/qIMcdrvkgb49WtqezH3nspvRQbIRlHXpctf+V59e5IsMr1J0kIPczsNuBWcnfewi9GuXIuodiJXURSYs03O0h1ZC0lrTZBMW9XDO3kUmw9gKyVQITIM7k+E0OHjIduRpCwKPu8z3ls389yLUZgJJuoXqppR8zZPZom0HnXvyZ5UrJtWoTMbO5bpu3Vct5cBu5/+Q52bbbSsqp22ZQVcn2ANgLANBZ5u52aeyPVNmX53Yp13MwVjSY3F2rvVgElB98loPcWu6PaEF1It8eakSG3zt7yAssRlHeCsOVjDCVxTrfb5WA4aTz8+rIaA1fRa4TiS7X0Riyb5HRoztIiO1BgJY7ZC9YHBKLj53HcNym6WUcOuQKqiAdsLoXMK2zJ9JtRUKuPdmPKAsNZLbLHlLuuhTmrNhK9UZSjH2exU4AlHhu1cjY2SaNHk+Q6x8KcO0bxxdE8u1LMypBdCcdoHLswjbq7CEdQPnAcgrTKK8nJWI/pAXLOMvMmo5BYl3CKZHMDnaEYivS5cH+MLCXthZFwVA3QpZJD9dvGONwnVQcEnUjbp9l1+2gBhCQKj2oIsMeak2S1SKTbgTAKS/9kUYrUW34iG2HACM1eubdPkhKa67e0SGOoUVybrK/MuT+UKWw+0AsRBIfqiEQL6FSRGefZrmO2Nrnz0Odr13ut7Ti4QUwjQAYYS1ziqtCQn+G+yfLknUbMEa896qPGCWhPvYQRGYLqBAxvdgydCshMboTFlC1RwDjdlnqnKtMdB+FcOx89opGVB9RXr4b5Nh1ZsfGaqM+jBJfYeBA9f2oFNG7TlhVo1BNFICYeyZVSMIAZiZR9eHIMKYkIT8jqyjVX/iAMpkx7kRmm9pOGMa2E3/nEJAzyGAHkNj18vOBvRy8m0a//Rrk3n/LSzkwjwiTOPSnpjw+6XvNvsyY5EtpqRZR6m32cQjvCUGee9OE96sgU2l+7fEG2FRa/cvaMf9SP/4ypbj+eT++10eWEuh/umB/KCl0kA7HqNlvhbHs+6wDWjv+zjtf41473jlZM6s6zmcHPr1+LWmy9xX6YPA3E85O9vzeD77kd9/7WmosguIHD2756OoOYwMpKopXltmiITyfMp21nCwPnCwOpNpwcbJn/rOC+aIhesXvvPctpgw8vljjXjqur9e4aQ9VQJWBk9MDZq+ZTDoWyxqWOYUmCtiaXR24ul7z6bsvKU8a3KQnzT2mDFxcbpnOW957/4brizWYxPz8ACZhS8/sW41aOXQVUC7y7/6NP4atsK1ouH68olh0FLMOikhx0XB9vZLJZK2YndX81mdP+Nf/xi/44P0b6e2ceMo7hV70UrthEuf/VPFbH34rqbYJAZBlQJ93lPeKky8U5XnN7//oc3St+fDjlzJRmXhSGQWIT0XS+Xd++BX9SWDxeMvses+nv/Ed7qTl8uM7UInLd1coD599+pRYJK4+uCe5xOTRDs46wonnBx++wp21YOUDa/loKz66y1YyaSbS6Th9mbh6/575+xuSSfgTj18Efu93fkp/EohLj2kVk1cK+6AmnvbYnUIve5JLPP7sFe6DPdcf3tHPhc0r7oX9KjZQXB9IVuSVScP7P3pJerfOnldhHcOjlupWJh/JJuwHO+bfQX/h6R56VK9YfnpPqNKYzBmKhHl0AJVwj/cUa4U/Cfgzj3rQCqNaJdxv30tQ0TISH3QsvhXvYDj1Iu/sFPpxjX10oLoFlPRZzj5e47YQPmpI1y3daWL2XADdIDHrzyLpYUvSwnzEKnH1oxuKFVSfrLE7OQfJymTbzxL9MtKdBbqzQPXBVhJzX0TKD7foXhF/UI8AbPHJStjOs4ifJcI00V4Ewse1AMXHDd2l1DFMfrTCn3nK20R3FuivesofbGg+aOlOo3ToPeyZvYj077Ysf+tWJi97iB80wkg5SNctfhFpzxLdUu7l0y8DzYcd8f0aP4H2YU90cPrLIEE300T7yI8MpK9EItgvI/U7IXecMibIJgPzZzGHq0B77UcvqMm1GipAcynvbx572vOEn4mf1u2FAW4vA/0i4ReJ/qOG+l2PrRPtZaQ7STlkKrF/V17vpyKt65bQvtPRnooEs9gmuqVI9pp3evwssf0o0v5GTbmJdOfie2zPI80DqdroZ3D4qKefC5PXXKScsiuT18P7nlDB7mMJI/GVBPO0Z4nuNHF4HOmX0P5GjZ8q2jPpwdz9uKW6jzQPAyrC7kPP4bHUaJhWxr/7gc/nQI7l8DhK3+c+sfswYOpEe57YvQ/1jxuaS+mALDYClLY/FD/v4Lvslor9h37se7SNeLhJwhpHp8aKl36RGc4r8aQPATMDoDhcCziJTmpj9u+K17VcRSY3sg9JO05sPpVjbC6FGWyu5DOgPVUcHgvgOTyUQKhQSh+lpM2KR3T2rMPtE82FMK7D+IcOwcNj2U+/PMphfSWgsr4S5jApCVDTXvyx9ZWiPRE2uJ+KNxSkm3X3QZKxOEW58vQLqarxOVDs8EgqX4bApsNDqa2o7j1+Avt3JHSpX8o4TS3BZOW9p1jJa9xevKLNhXiNd+8IyxutdJI2Z4r2XH7WT1UO6QI/k/N0eCSsa7GNbD9QtKeafir9l36aPZILOcYBbB2uNM2ZxleK9UfHjtvtB5ryvkd3EVuLj1eqRzS792ThdfOBpnzdyNguNM2ZjLs90ZIXcNPl8C/5inkZf/fYcLgyuE0ni4KteNvRMub6wuTgLVmokzEa8ZXONIcHhm6ucetOWFGQfU70mNjanBlsLXLbbm4we+mYlPAgYen6maZbaFQXaU4N0WlUiNhtR7KK9qKgz8Feg5ezOZMOz5QTbFVIRCOKi8OlVJTUVwV+ZjCHni77VHUnHvBQKtpTi9l3tCcGU/fU5xZde7qT7BlNifrSESorfZdTTf3AjWya7jx+YkY2cwhE8hMjPtVXK2JpiVbTnhf4mcVPDLvHBRI2WNCdFJkJzIA7RvR6LyynM2AlYKc7K2Sb25ZQGkJlCbOCMHPEiSNMrbzWmWMIkIY4FXlpmjj8spLJap/DmYyW530gTnL3ZSE3R5iX8vy8JE4LwqIkVY5YFaTCkQonTOR8IjUjQ9DPEMLj7NuBPoU7gsrcnTkCx8LJ660ZfaUYLf+3Jr9UtqmsPb4+RUhRvJ4pSZ1IEg9l8h41dHQaPUpW3wwbGmtN8lcKQZjNFMd//9pXTMevPy3853v2SChi+vP/+svy+F5LYcsP302P/ov/FAD7rMwrKDBo9XUQD090UqMg4T7iz9Bt9ra0apScjVH+lUyKD9cSJBOqYwn5WG2QfUGmZawwGDwo3emxMD7kfrshHMFtZRIFwqSY5o1kUy8TvCFhEeQ4hih8nQNl/MAWNLwVT9/PZWU9OllVb0/lOLplXt3f5tfk7Q2BPUSZIPazDAayd2hIiY2lJJm2Z2oMHRkK26tbSR2dP5E/8KHM6ZyzLJuzsPusZ/65w2YGyO4lvXT0H2VvQixEgtWeqjEIJkzkg/9ND9CwHJKUnDvdDQxN9nkcZBKk/DHFNVm55n4Odifb6BZyvVUSVkMCTyT4p7mSSeDQFzhInIY6geExMEPNuRorAdz+mNoqq7q5xmCTBGg5mby5LAPrluIrQst575cydvE4pSy/EXlbsZZwpXItkqxiLeEx/Uy8Pf1UXvsmS/OmR2s4FqmmUJQrYRUOD7IPykraZ3Omx1CipBkDj6KTa6TyNSHlgvjdG8eyEa/RUB8wVIeESn7HyrVsNzj5WbnOQTetnAPdyaR4kM2FQr4PgUX9TAKGhhTRoQKmvpLwp4GNGCZZ3SLv41S8SCSoryT5sVyncfsq5uN5o2KkX6jMzojsbkiklWChlNM6c/LvoJaKwkSYntFzJZ9HMrY+J9bKvZSvh87+ufxZ8mbgjO5ljH6Si94HtZETRsx0+Vzmqg/bJMp1oF0aTJ9Tc21mNBu5j4dJrqRkpvHYh4UD28Qs2ZOU1OE+TLnU2rTHahFbp3Fs3VxTbI+sinTzpWOarsq+rzyh7mdS4yCMUqJdGKqVyPnUG+drOBfuEOkWcq8MqbyhUFluF3NIjoynupPtDPdwdIrZ847m3Mm579NY9TJMuI+sWhoTSd+svxjYJ6mPOJ43FXNgViKzVfJ7lQyEwQeWr4WKMtZoh8qLY+m9n2rsIdItpY5iYHh9ZonsQWSVSUtw0lBzkZQE6PQLc2STMluXjBrP95t/I4ck2aRFLqj9cD4l6Vf3eeKXqyBCqWVfWWYpn3/ibRu6DqPL4LmLdCcW20g9hWkCYSKsmoopqwOEMTmmh+bzlIOFojn66FT2sA3nelB5DMoIucjCBodSU6w6/NSO99zwGNJjQa7jwGzFsYYCkXfuJUHXT2xmpWV8di8yPr9wuFVLe1nhtgOjFY/zD60IUytJr3a4XvEtL14shkqRHDozyCoh+zY1aIVu/dHb5+Wa6D4QJk5CYnwcX6/7QHQG3XjCzGH2PVhNqCw6A6lB3mkO4vELlYQGERKxsnIcWknoTgZauvWyXR9RdS8BQVqN21O9ePwGvx9WS1JtTpFNNjN7Uc5BckY8gbmSI1ZOkmJnWcrZh3E7wiSGMbwnWS3Js8Ycn88ptqr3xGmJCpKayxBYo7WweaUAOlISkDbeF3n7vT96ETNDmMribSYvSjDPWF0yMIFayzZjHLso6XoBawPYy68jRpLN+4OR0Rw9jH9a0msIsp2hHiTLSsdaEGtHv6PKjPFQF4LWpLZFwm+0gLvBB+n9kXmUN8l3m8OEBn/mMNYhPGd4nTHH54DYtsfOSedgAI4DM/lmgNCw7V/9968+/orVjVz8+Cr9O3/w7/25b/d/+jv/3V/4scH3XAqrbSTtLf/gb//fPPnggt89e0Kpe/5o/QFdMJwUDc/2J3y3OuHDsxWfP3nE1eM7nAlYHfls+Yr/6/mH/LXLl/zh//Fjfvvvfc6X95f8m49+yR/fvcPu9Rln5xvu9xNmk5ab+wUfPbzhy58+5oPPXlDowElZs+tLfvr5u5hFjzcB5wJ//foFT9bnfHr2ms/vHnA6qfny2wd0RcCYyPXplgfTLZ/fPKD+eom5rvGt5dP3X/B6P2N3qOAXM65++yV1byEY5lXL/W5KfTvhBx+/pPGWQ1swLTsWhejp1m3Fy19e0p839OuS2iQuH62530wp5jUTnei8YeY8ISra3lFYT4oaOguFp/n8lMWP77AmYnSkNAEfNX1T0jSOvnZU84762xn7T+QDZf9RJ37RFwtOH21o1lPssqb9ySmP37ulf2w4q2q+fHmJ1hH/csrigzVNXRCjZrk4sK9LiklLu5tQR0VaFzz++DU+atresnq5oDpr6L+ZcfnjG+63U37znaf8oz/5mPnDHbu7KcokHj2858z1fPXikhgUDx7d8vUXD9GnHfG2IM2CsKVPJyz+3mtWO+nR6A4FykTU3+o4rCZUy5bdfYW7saiP9/jeUPx8QvjRgX5VUl7UxKiwNtJ1FvPlhO4qYLaGcNpjZj3pZUWcBdyy5ex8w8v1gvbllFRGzh+u2Tcl3dMZh0cQqwTLnuqLivrTjvfeu+Xp61Mm0479qxn6oNn+KIg8uQzQGIrzBvtP5hze96CgPK/xvSUGhbGRcF+CV9jrA/afzcXLedIzWbTU64py0dJ3luWiZvPFGeGkZ53A3SvCOw2zRUN9KCn+ZMr2Uw8ussu9rqmI2JVl8VVi9RkUn2xonyzgYQsvSlQviyv9mYTazD9csXmxQHWKNA/QaspXltXf8jlUSUJwVAC/TOizhrAqjr/vrSZOA+7Osv7bPews868M6fdW7F7NKF849h9GYc4LYcLNSmS/61nC3Sd270nv6vSpALHD7+9IX8xlYebCi1IgKPRZy/QfT/EzWP8okoqYZdsBd+NYfJVYfyKMpSwaRapXhvYijj2zcelRe4M9aFSfvYiNyinKmtVv9gJCNgaSIpx4yucSIuTnefZpE8Wtxm1FIl1fSf1RMkh37V6Lp1SDoCsJ65k+d/gpHN7z0q2Ze02TgnAi59A0ksx8eDdS3hiaxz1mY0kmoXvD9IVi+3Fg8kzTXoj/lqRoH3r03mAPSsKNpsOinXjxunMob+WYkxWPpdtJT2uYBczegBYZXvVaYw+w+yCge83kpeLmtzUnP1Pc/03xGUcnkuJ+kdDBMPsO9p8pio38LDqp09m/I5J2P5f7TfeWIr8PZAy3v1GKZC3KAli5Sqw+hTJLfyWdmbFLtFzJAh0weiKTFW+qz/UpbpuoH0jw0fSF4nCdUFFWzMo7xfxZpF0q2rMje9idas5+Atv31Vg7M3ktlTuhNFQ3ifYjO4b07N+NzL/VlHdaFlI6AeV3PzaEQnqATXdMYG4uEouvGStrmnM1Ssf72SAhl7oOexgW7WQxp1zJ8djDcdGlX4pSorpVzF5Edg8d3Ykswu0fS4p6dyoLddFCdwbVa5g/h+27hsNjw/wJxEIqleZPI/WVlYWxJrH6oebsC/Ef7t5TzJ7K4qDpRIY6qMbS0Ie5T5TryPpjg9tmZnmQ93cJHttxUWisg8mqgd17cPpz+V3oZ+q4sJHl5qIesFR3Ut/TnuTUXAXz55LqHEpFsXHMXnhWP6zoTiVNvM8e0HKTaBcKV1uikWPWQRZhVp9olk8i7iALMbZJ1OeK6U3EVyXRKKY3XpJojaKfVIRSan9CoWiXinIjEtlhoWKo0CHJOU4K+qkm6YlIb730itomjf2doaiIVmU/aIltE3YfMkMvtpFQKIp9xB4sTfZzdnORxA4VP/YQ6JaSWL34uqF5UI4+UBWHxd5IfWGzrD4xeV5TX88o7jt27y+YvuzpzkrxRLaRfmFx+5B7NIffVwGpIm1O+JmVQJ4sbe1nFt1Fypua5tFcemo7Acl+qseqnCEYyG07urNSWOldh942+Mu5yLdrCepJRo/g2c+sVNXkxYFYaEn61UpqXkAA8rYhlgIwh3CfWDnszQ5/MUN3AT8vKF7tCKcz9L6VRNrcmamHfWdwqDsv9Si53kVva/Fa5rTWgRxKKcFsIttR6ui7NMJEM8npvr1IVUcAO4A2749g8g0AjNbZE5o9GwNjOPglux41nci/U0IP4NcYUtsKWwmkmNnMN6tJnDmG+/zzqkh+5ZH+Rf7M79Ej8s8B0X8FHt9rYDlzHaePN/yT23eJSfEP73+LRdXidOT5/ZJ3L1YANHXBXTWFCC/ulpRlz7Ts+d++++tcXOz44v4Kf+H5w3/2MYvrHf/r//47/I1t8ZoAACAASURBVO7f/ZxvXj7ixgZiVLzaLUmd4RdfPsTUmiffXbI4O/DFLx+hK8+HH7/k6e0J/aFgfrnlH/3kB5h5zxfA7c2CWz0ntZpq2XB4PufWBr77+QOYe6lLKT2zacs3d2ekpNA/mdNeBG63M9p9gS0922+XMulrNd++PsPfVrDo2T0/g7/5kuefP8A9PKDPWrSJ4CJ6bbmrZsTGcr9zIg/ttKRYPujElznrCY1FbS3tJMBJYP3lmaSE5sRPXQRSUqROUzx3NNeacqdopxp3bygv9rR/cgpXgfV3J6QyUH9+im0Uu6Zk+82S2/MO/bRi8ddu2dYzNi8W6EZjDorVoiCVkgibogKvwEU2Tcn2dobaG3RQNGGC6xQvn55RvLD8rHyArjX1L06wEZJJxGvFlz95jOoVGHjSX2G3Gm/Fr4pOhPuS4qB49eUFaRIoXjrSRcBsCg5Lh2403cZiO4U9KNpnUyAzzlliHX85J1SJfhbQWwmxCRNNmAUu/tBy91uaNIlUzxz9xvD1rsC9csxWin6u2b26ENlsl+VoZ4lgrfSori33nz/CniYOywJbizy5cwm31nTXCbs26EuZWLt7CWlQzxbEi8jil5p+CWWeJO/0hPMnkbuZwm0Kuplj/oMN7ecnpFmk/mWFUaCSlcnZQRFflmz3jtkT8ZJWzy3dqYTXNNcCoCcvhQ3cODisJkxvNI0pWTzRhCpPzm8MzQPPdjWluDW4rcjW7F4CS0LpcmKj1PF0JwkiLP7PKc0DiEY8fqFKTJ9J3cJ2bsWT2cD66QK305x8mXh9KR5P00C/kHNla+h6zeJrmVS6vZEqmw723804/VYm4ElZ6WwtoOsrTr4KrD4x2I0mTBTLLzWHR5rJC8X+MSyeJHbvivfT7bIqYC8r87uPIsUzBxrm32TW90RTvU65TzNx9keW3Yfw4B9H9g81u/csbgfTF7D+RCYVbqOzh04YFrdTxFJBC5MXhu4UogflBXjGUl5rmuyL/MJmVlk+M8tVoj1zzL9NvP6dyOlLxfwPFcFJN2F1k/vvyP2qQTF9LgzT5IVMSqfPLbsPJERHe9A5cffs88DmQ4NpzKiuOPkq4CeKdimM7PS5HdNTu6Uw7trD9JmAItOA8jB7Fel/aTOIEInxyS8Tu/fEO1fdqNw7q5h/nfIkVlHeRzYfadn+ae6TbQVgJCsKlPI+oj2sfiisvGmkIklFUQyohLBlDVK9ckvupYTJLwSkhUK2GQrxeC6/SlkFkDj5UsCbzwE6bh9IWibe7TnMv8uVSnee5swyfRU5PNQUOxnD4VoYZrfL1wCRYFY3idnLns17jmInPZumViyeSApuucngoU2QNKZJOC3+P9vA4huRdN79SFOsYfHMs3tomNxGio2Ag+g0/QyWTwKmS/iJpr5QhF4x+47Ri6dDolwJeLj4qQTZzJ4y1uP090dPna0T1SsBcGabmO0i5TqgomX+nUhC+ulEgMm3Hf1cQI66Z2SUh/TSbiYBO7aNcnwbLVVBW0ldre5zpcuJ/Hzo30xKQoe6uWbxtWJy61E+0c9NVnAIYFt8KzUpu0eG5Vc9oSw5+arDNIH2rGDybE99eUKxjaIq0uKZNL0ksU7uI/Yg+9S9orzv6edWJKdZBn3ypVgS5l9uqB/PM6NqmD5tOLxTCYCKSJ1L7akfT4itgDPdK6I1lPeB8rahOysp7lu2H82wh4BbNXRnFTokiq2mvrAUa0/1dMf+4yVuGwiVzqBN0VxIoI1p7NgNCkjw0c2e5tF87BedtZK2qktJVy1vAu1FSfViD8xESptg9u2BfiELgu6+oX0wIRqRKOvMWA9fyQqwdpsOP3PY204Ycp9w9zVJa5rHU/GE5rAcs2tpHs0pb2phZQ+e9rzE1gG7aYmVpbxtRgY4TJz4KZ14L92mlS7PGHGrVvowW6kFSQphYDuxoaAUYeIw+xY/s6guYrYtWI296YiLSjyXTWYmBzC4qUmlRbWeuJxgb3ZyXmNCbxtcH97o7RQgpzI4HatICif+zN6jY4SQmVBAr3ekqkAdmvGaUTjYHY4S2BBJ0wq1F4O4CsKSjkxo25HmU1ht5f0pZjY3+7GdE8azOgYAAaPHUlkrnsmqlLoTY6Rnc/SkhjekrxHl7K/7L0NA5fCg1HtUcVxIfqvC5I3kWAXE5vsvh/2r/Ph+S2Hfey89/M//M9xK/jiEqaR9YhP2tSOWiXTWMfuTShiJhaxeVjeK9lxWk9tzCXZRQeR0h3+tJt2KHGP6THN4L2C3EupRrMULM6QuHt73TL+x+Kl4wWZPDN2ZSDjdVibCYZKoXsuHja3h8DgxfSayND/Pk5rXIjMdUgX9XJIkuxNJNxxCO0wrMtgh8r2fJdxeJkC7DyJXfwSrTxWmVsQyF8CX8l4nn2tjfL3OXZn9Mk9K7oXREDmqTOSiY1zlDpVM+LpTSW7s5/L+5iJx/v8k9u8I83C4ThSbY2ri/lEGBM/F62RrkSBPXmXPy5Ixyr09zQmP9xI7H4osY82r8CJNlDFJQiPs3pNV9GECmgwcHso5jo5RRipJgVnKHGWiFQpJZGyu1DiptPUxKl4KvPM5r2GIvfdTNQZURCfnwu5l0hBKNSYohmqQvMrkqrkQ1sDtRLrq9rKtIUhjkH7aOqcXrlJOC1VjuXhSjIEkh0fCIlV3ecVSSXhKP5OEzvpS5ITzF4H1h5Zik8a491BAcyXArp8q/Ex6ObtlDuPxIu3zM5nsT26ivC4nL0rCpYxFe5nIhUqk0fUDWb0fPF3i0cty1u1RNpiM+DPry2NcvhTCyzUvV8dagEGCPkh6u4WcP92LLy1MpAdTglhkgrt/lIFLZoQEBMk1q+4Du0dmrIAQeasca3sm16m6k2NGC7MxfxalYzJLcqM5+qaGbsVhvAKqFSg5r+6Q5X3Zb2kbSeLs52o8h/WFgOBqdRybrUVmO7lLqJBozkSeShJppaR5Zn/nVGFqkcZGK88PLM1wXlydJIHzVWD72FJsE9Uq0JwZurli8dTTnBqZ7B0im/csi2de5LROPHaml3GU6zjKv6VSwRNKTbsQD5okd0ZhLUtNN1dMbwPtwuTzJb8vSZGljzL53z80nP28lwRKhOnxpTAn3Vz8XEMASrTSTan7yOHaUa7l/cLMvNFLmXtaZWIrTExzrll869m+a5nciR+yvtCU6zTe/6ZPmEbGGQqVA3JSlgcf5bntMrM4c5G8DYmxIr9tac8cvtKjJFXlz5KBzUlaJMb9zBzlqggwUkGu37C/Qf46bH+Q5ps6yqIDwla5vdxzKib6iaZcyWRs/8hS7KT300/ES5iMIlRSr2Aa8TqaxuOnlvbcERxUdxL6Il5Aiz2I/zBaRbIiQbUHkWZGJ1JhP9XYXaA7tdgMNCWwyBMqg1t3tBfl+LlkD4H23GIyeyZsVRorGIauzUE2F3KCqWkCfm5wW5EDS3+nSGmHZNP23I2pqeVtB0bRLZx0XGZ57hAa0y0c0+92+EUpQCNEwsxBglCJV9DuA27T0VxPcgKqnEsA1Ue6k2LsrOwXbmTcJBRIUklJ4GdWZM2blubBFLfNvYldQIVILCx+ZilWrSSVRpFHqhyaY3cd/WmFaXz+nRLJalJK0ksbjz50+LMJuhZZrc4hO4OUNFZOWLcImNxX3Hr8ssIcOpncpyTeRSf7VI0nLErZ15Bg6uMIXEJlpaKjNONzyYq/c2DEVOcFYLW9sHV7kcMqH8efhblUa+iDbEtvG1LhRBbrDPhInBfoxr+9PZPltoV9G6T4CBpJYy0L9O4wPheXU+mmHKSswNDlGBdTAZGdJL6q3pOGrsz8emE3g3RWwiilHV6bqhK92krFh1JHyewwDx96I2M6+h4HmeyQ6Fq4LLHN8trMKhLjW9LX4/HmQKUslwXG18qkcPBdvSG7TQm0gRjkO0j6Kxwls0a2nUIcpa+jxPaNMY2S3SFF9lcf/5JJPG8xld/zVNjzH1+lf/t/+A/+3Lf7P//d/+Yv/Njgew4sl59dp7//B/8+L/ZLpq7j0WTD63bOaXHg6f6Uq8mOgy/46dOHlFXP5XyPM4E+GPZdwemkxqhIHw1PXlzw2Tsv6aOhzV2OMSkmrmdiewrt+ZNnj3l0tiEmReMts6JjXVdMih4FbJqSq/mep3cnXCz3hKjpg2biPKvDBKVyuqECpRKV8/ig2R9KXOFZTFoKE1jXFaXzHNqCwsqHwnY3oSh7rIns9xWPL1fcHyZUzmdvtma3r6gmHW3jKKuepi6wLuCcp2kczgVOZzUxKVa7KSlBWXgKG9jsK2LQFGXP4X7C6dWOfV1QVT1dZ7E2sl9NsJUnAc4Fml3B6fme1fMlxVlDDJoYFNN5S30oCZ1GmcSDyw136xnVpCMlJVUrLypOfnRL2zvqQ8HJ8sBuX2FdoL6f4BYtxiT6zmKdp28tsTWUy5YYNMHLtk9yP+XpxY71WljFatoRo6KrHdrlyUwcPjAh9oZi1qFUInjDcnGg85b9ekIKCnRCrxzpoht7FMvrA11jUS8qzPt7uk1JsWzpa4etevpVJZNBk4Rt1eBOWvpVyeLRlr63+N4Qeo26L6RPMcs16TXohCoipgjwzUTkjUUiDR2SOqFqQyqzJLOIUu1Sekn4zem+etaPHY/Ki4RIeUWcRqpnluaxlzHaCL1GFZI8om0i3hfy2rlH76SvMbmIPhhJ0nWJMIljn2RSCVPrsZMunASpfll43EuHP4mYrZYFnyJhFj0pQtw72X9QmI0lPmhhJ34Xs5fk2zCTJOBUBXRtcuedeDCHnsrheqIgTQPm1hGnEdUK8A/TmOW4Im8l/wwkHbZfyPGZvRGJ5kFLsJNO6EZTrDXdUvxLYSbVKn4pNS+6V3SnInuFI7PoZxHdiizSHjT9MuA2kkDcLxM6gz4gBxvJsbi9GheKSNBehCz9jdi1zv5sWaQZ5Lf5Isgx52Tj6BLmIMccJrIwgRZ/8eAJf7MTc/ArKy+BQ8VKH9M4a0V3LhUkUvUinvVkhXGVSb7UzwwS3zDJIKdRI9M8JMT6KSSbxm2HKiccZ+Dfz2WhxC+SSF+Xst1oU5ZqZjlsr8bU1u5Etu02Uj1i6qO8yM9yl2SWMQ49i/m0jUFRfpb9x4Nnb1CGTXOibQZ0gw/WNEfZpEry/36eFy6yL705l3PrNpKI2s+OMtjBx1ve54W1DP7tXhYkhgW1WIp31jay0Dd8d4fj38BQMnrVh1TWaGShwbTCrhZr8ZYrL/dcUvJa08ii2uSVMK8py0DHbr4ozOQw9uEcmS570nP3Y8wkQ7TixY9WFnqGRNqhImQI/5IMANmnpK9KOvGYXqqOYxgSZ/t5TkLVImE37XGBQEVZUDJv2K5MK37WQeYKWd5cqfH5lM9T0oyZDP1UWM9oFa7OPm6TQX/u42xPpHbEV7B4GmhPNL4cQEiulxkWkeIbEts3chNsK4sX3VyP91a1CjQnJr8ne0phZKG1l3OmvdwXg1ezWgW6uXlr0WTwMg8+9GIfCYU+pgU3kW4hloZyHagvjFStdMdFC7cTz7I95M/AShaWUALYh9yIpGSBI2ko1kEWQI34Yge137DANdTzmEa8uW8uBPi5wTRy0ENdSSiPf1+0F2ZP+i1N9hG/seigc+IsjH5d4tGbO0hgQyWMqN1LCJCMx6P6QJgX2RPs8TOHacLIdCYjf2NVH4mFQffZa5lBfnLZM7vvjr7aNgPdzo+dkwP4JSTxnI7SUGQbRl4zgHnV5/7KfSMe0UGqCsRKQoiG9yTzxvtDBvmDZ/QNMKyGZFoYQTC/ggVG/+fgfXyjw/JNOWvqOgGfKR/LAFSVfitVVj6wwvi+4fGnYpA/o47k+95jef7jq/Rv/ff/4Z/7dv/hv/Ff/4UfG3zPgeX0k0fp4//qP+Zk0tBHjVGJ1WHCYVeibWQ5r9ntK3xrWZwe2LxYjJNbZRLTactuNUXZOAIOvGL5YMfm9Zzl1Y7N7Qwz8cRek6JCmSRgI0/IbRHoD24EByhk0pwUSks6a187JsuG+mYqaakuEndurF4YJzT6CEp0LX4yVUbZX+4npDbgEmbWy5gaI2Xr8/yLpkDtDRjQB02cRJj7cRsoUHcF6bSXuo9cKA/Ih2YVMC8KYpGIiyBjSoDX4o2zCbM3xKsO87IgVLl4epm9hRPpD4yzILUVOTkzVlHAUadJLlG8NvhFnvhakT6iIFUB1cgfPLPX+NMgAGnu0bdunOCHecRuNf2Fx91YARwg4z7x6I1FeUU49eAVbm3ozzzuVqR4SWeWZ5akKmAnk+c4jL+KY09iqBLtVRjBRnsepI6kTGOvo61V9tHlQKgk2x7K3UMl/Yq6VZS3Wgreg0yyqluVQ3vSCCDa85jrGgSkFBuNnybcJm97njIYSMy+Ew/cEHKETmMJ/fSZSNv6ZWTyQv5I+6mkrAr7N/wBlonjMPEcqgiSgyqz5gMwcFth05PO1Q61TJBDmZi+VBweJqrXObhkIhUKdi/HVKyFcfJTAVG6FSXBMLkqV8Jg1ddR6k1skiqTN+wIJr9nUA7UDyOmVRQrYUj95Dgp0528JlTil+vnwwRYftaei7SXxFtAprsMnPxE7pXuFEg5mOtE5Kgq11W0p0cwMZy7wZ9HnhBFK72JfpZBVD6/YZLZ5RsZ65seNZ9DmKIVQGFqcpm6JH1Gc5yo+uo4hlDJ66pbYbijfdtTB8L6F2sZLwjoRMk1FCZ+kCWL1LfY5n34zOiX0J6LvFEl+b/bCQBoLtV4fMPvmJwrUWUIWMlAdi7jGMBcvzgGf1V3aQyMcrs8qd+J784e5NiKtfQ0mk5ATL9QORVWjeFLUqlwvHmGMByiMLq6g/paVCNDxcwQ3hTKHGzkBRwN3YeDF3O4tgPjPYLNyBhOVq7TOMHu5nI8poPmQsK8wkSqFPqForqNtGfi26zuBLgA472s/FEaqr1IiA9XohQwDZnBTzl5UzG5EWXC8PfF7eVctPkcFnthXk0OMfJZYWEaAT6+UlSrQH1m8BNFsRM1TrkVhlZqc6Iknhr5HbX18VjdQeoxurmmOddU90dVQrmOOXBMqkW6pR77PtuleAH9RFQRtslhUL1s17QSjOX2gf21HcOhfKVEltqlERC6WjpuB0AzsLy2jpnxF1VHtAKmin0cg5xs9uQNrK+f6NzLeWTSi7UnVHoMiTKdXG/TCICBrLSZ6sx+C6AbvIl+anBbT7+0mFreIwxqlk32IlEfOlT9RAKUik3Ivj8JeQqFvE/3EV8J6JIANEkqHfZh90GY5SbgK0l3NXXMXtsglRKJEeAPTKRpA/3UjuNzO0+0wnAPTKyfGOwgUUzCUIbKoIMA7VAd96GbIGmrtR9DjmIpYx39jE0Y2VMAU8s+RxltDg8ybcDPXQ6w6oUZ1VIlJbUfAtDGsJ3MzOnuCBolgCjgTyfCSubtqwHgvMEoqj4I89rK+IBjIFEGfao/Mp7JKFQr/sM0KaSv0lmRiuY6kRH8GT0CuIGVFa9mehsU/mmPP2UuLz2d4VcY0TcA4MBswltg763tWTsyk28F7QwP72XcQw3Jm6BvYC6B1HW/XkPyq9v6sx7pT9nH/8fHvwKW//8/9J/9kr+8j5QUP756yXcvz3h9v+Dp8zPqQ8F8WeNcYL2ZMpu2VPOW/h+dsXy4Jd4XpKQIrWH/1Qm6CHBTcnq6Z3m+B2D73ZKzP7ICRKtA2MiSbDnrSK3h8fu3oBIpaPrGCvPTKwgKVQRmJ42wS60mPBdzU/+LBWbZ4Z4XxL0sdadpQLWa6YM9+qAxWyNXxEZOf3jH5Ttr0sFIZ9/LErWzqF6jtwb3xRR1V2C2RsDe2mJvHMUzR5pEkkrwuCG5SPlViWo15S8r3Dclbi01JNV3Dt0qYaA0qFZjXxSExy3aK9ytZfKkoPpG9lPeGapXlnDiKb4p8Fc9aRIIZ57ipR2ZGBUkkCSceLrHnXib1obyhXjOpl9bAUa9hH8Ut9LDp1tF9W0hPry1eDtPfmIpbwyTz0uKO035yuAve05+avDTxPInDj+PhEo8VCpC9U0h5fFFwt5bZk8s/mFH8dqOQSfaQ3flKW81s2/zKnGnsBvD5IVm+TNLca9pHkTCD2vK1wZTK/5f9t7s59Ysv+/6rLWecc/vfMY6dWrq6sHloe04trvdTpxABoUgQQhXXBCJK27h3wCJSYBBIC4REQHLRgkmsePEkd22u9t2ubumU2c+77znZ1xrcfFbz7Pf6rSJhRzZFWVLpXrP3vuZn733+q7vVO058pcGF3tGjwUk5acC2Jp9mVCwCdQHlsFLGSR2fYDjT4TpKu5ZbCpfpumVYv1mQ3nisKknPxUAlp9qkrnGZSKldrHHDqRKorxjpV/TQ3ahWb9b92xAeqXIT6Urc/BCU+57ULKO8tBT3Lbi6b3W5K802ZkiPw3djTEkCyVBJkoYk8FzRTOG7V3b9zaWR076A7fCim1/rMAbz+iZYv2akyCV+456L0gxEXB5+DuK5FrYruGLEBzzVs3gZQi0OZVKis3Dltn3AmDfhN7CjTCW0Ubk19mFgLTtXcvRNyE7C4PiAkZPYfwpjB7LYBwE2BXHAn7yU6losClMP5RtNmM599EW8jPF5IOI5TtWpNcljJ5I5cf0I2FoNq9btrc88SZ0UnasTSqMU7SF5HqX3Ly5K0Cq2hM5/PIdSz32nHyzZf2aVH4kc3mtuOVFPruCwancr9UerN50bO7K/ZSs6JnA4XPp3TO1SLvTuWd7R2o8XCwgJd6E78wIRk891b4A3HLfUx0IQB4/ccJAxZCdSyDT5JGnOPLk58I+VXuy3PQDemlyHDoQVw/lmmRX8t50LlUazRi2x+LVTBee7FLO/eSxo7gVgkQy2b5qYfxUKicGp554KaCyOvBs7ioGZw4bQN72loCn9Frk14NXAnJmHznyc8/ehxLvH23k/MpEike1AiKbgWL1QHH82xXlgTAe1SwwxvviHW5GiuJEBlPtUJJ5IVR3RASZ407CLkmiIgUfvrJUM8X8C4rNLd3XlmxvKfIzR3Es6cb1RDE4lW15DeNnlmqqQs8jfVhNO1CsXpO6jvU9zfq2Jr9wTB5b8ktHtafY3JbqjMPvVNQjkZe6RMDs9ljkyJMnrUii9wRUFoeazS0dQJuwUNVEvsdWdyRZdfKk7VN6r982vcy5Hglbly6lg7EZKNpUkV+58N6IqPLMPm56ljFZOdZ3DV4pVvcNywcRuvEU+5r1Xc3opXxoRy9akrWjzXTPTCYrR7J2VGPF8n7E4Ey8oPHWM3na9AnKydJJd6cXUBkFJiyqPMMXNdVUU+wbolK8knHhRMqcK4p9w/BlxeaWYXsU9f/Hw/pOxOC0IVk5ssuGai/CpiIzTha2Z12Lo0jqNRKRUGfXLcpJ7+XwRUVUONZ3Y0zlKI5FolscRgJIE015YCiOIqr9iPS6QVlPM9LEq5bh867iRFQei4dxsGFotseJ1Ip4CeBJrmt07djcTdEd02c925NUfIzLlmpPpMI+UtRTkT67RAeG0+GMYnssX3D5aUl2VlLtxdhUXm8HBq8V6WVJeRDjYo1LNM00FmBZCxjIX25kmdpRnqRE65p6lpBcbMW/2Mq66llCtBGgaWNNtKqJz7dU+6nUj4zivipEmFSR3JpClqn3s5BkLCyjLhuU88RnK3TZYDY1ZlNLKq8Hl0a0o4Ti3hizqYWRTCOZ0FEhRdeIhFeVEqSjWteDSh8b7DQT9pEg+U0ieX6YoFcFPk9w04HIb+NI2MCbqbFBEtt7JpVCr7eopXxxq7qRFNlSlveR6QG3z0K1SeefhB5Mq6KS8J04km3Ark4keCT7R5f22o+vBYD7qhZQ6V3fQ4lWAii1QqXJLlnWO5TRvTRWxRE00qWpIqlCUUr1/3XvUXEk6zL6//M/jOEH9lx+3h7/EqpG/izVjXyuw3v20i1/7eA7fH3vQ/7B+ZdITMvT1Yz3Dl7w3ugZ/+0HXyONW75+92PObo8B2O5d91LX/I2G2hkOH665KEfsp1s+iI84P52S/s0L3ogbPn15QDwrybKGQdIwPpjTWMN7rz/n/Zcn3NlfsixTJlnFJC355PKAe7M51XjNs/M9GDe8feucJ5M97u/NWe6laOV5Z3ZOpByfrvdZVSmzLxVYp/mZk0/4pU++zP3Jgu88useth5dkUcvVZsCb+xdcFCMGcS0hP5uE6d6Go+GGD5+eMNtfsylS3jm8YlmnfGnvlH/86E3y20tipxl/oeLF831mh2tmcUtiLLGxnK1GHAy3KOX59NkhP/HGYy7vDHn04pB4WGFbwzBtGGUVZRPx9mTJ6mHKzx5/xP/64Y/yVx++z9/L3+NH7j3nohixKDKs09yeLsmjhkhZLooRV5sBqXashgPu3Lrm9GrC3nTD+VMB/VUdUV7l/Bs/9nv8o0dvo4HtA7nWcdZwb7rgDz+4y8mdOVejAW8eXfPo8JA7BwtWZco4q9jWMcsP9/DHFaNxyepshHqtIo8t1UNHFLdEkaNpDO8cXvHx8IgsrxmYXQ9qVcYc7q1oNjmvTVcsy5T5G8JAv3v7jO+q+8QnBctZwsmdOacvZjx47YJlmZJElvPLMQ9OrnmcHKKXEe2DUnpVjcd7+NK9V3zw8pjpZMv1YsjrR9c8fnmArwzbr5bsT7ZcHQ4ZjUt8E1EeRNw6WvDik0PuvPeKNGr55OUhUWypnw2g0dI56WF7IKz78HCLexMmSUNRJZRFzO2jBeeLEeqOo5hnxMNGJkasAuNR2tM8bKkXKfG4Joocq3jI7S+eUbUR69sp20VKNGjZjmPMyhDd33Ay2fCiNNTHiruvXXJanfDejzzi8XyP1SbDXae0ueXsSJFMK5rTHB9pmv2Wwaxg9UaEm7bUBwazVxErz+LtnHvvveLx00NObs+Zr3OabUJ9aBjfWrEcjsF4klnF2U/l6MKTPFyxvhhg1non14w8g5MN12dD9u4umF8P8UXE7PaSuZPr6wAAIABJREFU+ekYm0b4WyVNbfA6pp1YMB5dGMb3l3AfVk8m1DNN8s6C68GE7J05rDLyewWrJxOUVUzfvOb6dAKNIt6rqE5lQkmXivawYXy4YXU9IBk0YDWR8tRZwqufjIgerDFvWdYfT9D3tiKr1wNc6jCFxqYOP2qJhyIl35zloL3IoitNci0TGM1RQ3QV4VKPGzXQaMykYXuWygz4QU06aCg+HmHvlJTrCCJPPK5pNjE2i2kmDnNUsj7PyO+tWC2nuAcFV8MUn1pRTpSG6iCi2RMAEM0N7X5LOitZPx32xe66UjQntcjyspbrYU47lmNqJy3FicEbT/16BfOYempoxo7JR5rtWzXtIMamYPdroosY/YU1V9kQH4lc2R1XuCSjPLGoWU07zLCpoxlrqkNHeilBWJKG62kmMjlhCul9ja809Z2Gq8uU4rZjk8g5TV9FtEPxCTYjT7Nvia9lMkk5YV3jDWwmIidtxz5Ie7vkWJH5icTT0x7X1KsIey4JrvWtFptK+u/qvmZ71+ISQ3koQTteGVZvWsojLZLmrKsMEXXIego2cygnYWGmDEmsUy9VWhZckrJ6wzJ8Zqj2hPVvhw5TKKLCUJwotvcsw08N27sOXcvkUhtAfDsSSbfXntFTTZsblm8JW28TYTqXr+8UENWBDNLLI7k2w+eGzR1PeiVANFmY3h/fjAzzr7SMP4jY3pHJuHZgxKsbwfpWhM1U8Ijvcgk29wz5qe/9pts7Hm+i4LVVZHNZptqH4TNRTLSDLj/B9xUznCQ0I5mIageRMOfzXWdpPQavMjZ3RX4t+Q2ewUtDNQXdJFIZVBqWD2H4HKqxAOhmIBMNXsmkULkfFBqZyEOrqWJzImms5SHoNqIeKfARqwcK0xjaVFFPZL9Xr0GbphTHWuqyMgU+Fn/2WDF8Kce4OYmCN11AfXsYUc4EkNtUsb6r8ToWP/u5E196LEFo2xMBouVM+j6jrccdalyiSOeGcl+jWshiBVpAzPZAQEh+7Sinkg6bLgRIr+7GUosUWNdqlhFvPTYbin91ElPsGZTNQyfmmGLfkF9ZVncN6dIDCeW+IS5Ehum1otwzZFdQzQwuHsj1aiUFt0t1rvYEZDejTCS8zuNHEShFFEsXaCeP7kBvtReFyidFvIyxuUwWJAuPSw3OJLJMYEFtatCRpjpMGDwvpNsy1agmwg5idG2l3qW2+FjjoxntQBJlvRlSHmXkrwphNQMAVZXGZTFqkEptWmIwC8DoIKFNsZMMvb5RkeI9vhYfKk2LH2ZQNf3fPgQC+SQWL23nzewqV7QSdVjHaJouejkwiMN8FxgUgKpsWKThaC3gNPgo0WbXXWkdqvNOxjE0zS6Y5waAVezYR5UkP1Aqe/OhguT5Xz/+bD8+11LY0Tu3/Nv/2d8hj1uWZUpsLEZ7vFeUTYRWHqMd2zIhSVpJNfWKttU4p/pZAwUMBhWr+YDBpGSUVWRRy4urCXFsaRpDW0fglHjwFKjIYSKHC16vJGupCjH3e6vFPzZoiZOWcpUSZS3tJiadllTLFJU4/DaSgVqrUZETqWsVJKvrGEYNvjbipcsamlWKSi1Ke7xVDMYVm6tcPHNeQa1RgxZfGVRi8W2Q54KAhw5AxA5fRPKaC8+VIlEldlL9kIjBnUb1fhuMeNw6bZbaRKhZDWcpbmRFxtt5AhstVQipFzlnFmbzKg3TRmTFW4PXsk5VK3zm0GuDTz0+s7K/Ohxb97eiT4xVhcGPWvEXetV7aZRTqEpkuz4XT6IuFHYkM9gSwqNQjbxH1wqX7bx1PpG/+/cYdn2D3bkOBhddi/fQazlOZRV2ZGUA6ulTfDtGsevDc6nHbERm28kGdS1+OV3KwMQNHPE8VHuEcCpd6tAh2XncQLeqT7NUTiSpupFaDbMOMpTIyzbXOkicwvK16l/vzocwuiFIZ+RIrmRw1PW4SmfazutmE7+T1Hp6v5eLkM+KD2mbbvd655PsWM9oEwbQeiejjDZKArhUWI8V5jNeyXnzCqIgM+7U5D4cW7SVGgxTyQDNJUGaWYRjDCFO0ger8LEAUeXEp9id8667U4X0U5f4IBsOvrFwvqMwABUWT0CVS4UtUg1o20lyZRu6kvXagSeZd2X1YX+Dd6m7V13aVX2EU3wjyEjZIGkN+9vVKqhwv3brcrHIO+UDEu5nFTxmCqpDCQzrvFDS1xiknnVYR7TzSvlItiGdobKOju3s+li7dYjvSzatG3lPJ6ntZJ7CLAVQZHaST2dECtzJxvtr0vUJh3u+kw3rdsce+3COTCXb6STRvS/XCMPdBZEpvztnnby1k592UlcBzbsQtJvnqwuZ6u/zzqta78KqenlwCPrqzp+cG9/7Ebv6i44Bc/ENHyhBJmlDd2mwEXTdlS6S8Lautxa6z5N8fzbDHcOqvLyvGUlAWBd21t17aHrVhW7COUe2q6wPx/fZ+7WTaNtkd/9191oXkkb4TJtK9q37fuiuo/q+cWV3X6jg+wR6+a10mErIVbwN51oHr2NEHzjV1W24SLyX3bnqezHZXcM2VaShqqTrA24zkbO6KCxnhQHt/LrOiBfSGT7TWdrJKrUN79Uq3Mey3psezG7fTfBUtpnaeYO7+zZcI1O7sP+EFFq/+35AtqPsrs/UpapncOO1DSmuYZkAFnr5Zzj/Lgn3RRgDdN2n3XF116DzMfcaOIdUhzT+hr9Z1ALtwGBKu/NhQi+x1N0xdfcf3e+ukm5YJdJXm5ndsnTXzn9mnVJ1Eo7x5v0UmLiud1QHaas3OgT2+f68qMpCpPtOUfFsBm+nQsBjF3gTPME+0qGSJHQph+fxHq9Dt2c3pgJhPm+8v2M+UUoAVidfNVoCfLq+S612LOTN3subj676A+R4u/d02/h+qe/Nv7+vu/MHSm27upIuTMf5f95X+Uct33k8/7gY5IZk9vPeY7n37rH/i//jv/snvt6/+zP/zZ/6scHnnLFsmoj9QcGzqxn182EfTqAsuMOa5HEq6YyzFjuPaEeW5FokjaobjOUeH3tWqfTulRcx+sWUxQjsviM+02Rl8BUdOhnoxo7kaYZLPNlSQg9s5okUUk1xIIEXysYStDCBZJmwfLdBf3tMZuhDLJSL+jAFrwLgKCOqPU90kREVnY8nRY9AuZh66hi+1Og6Y5SK/ylZirwv/SgKnpuYzQNLci2Jtt2ALF6Jl8p0nqzkxqDNyHo2dz26NgxO5QuzG9S0mQxAm5klfx5R3G8Y/F7O5p2a/d+MWd+XDrvRI/Gm2RT4yorq1YDRxxHZpcjXkg9Tlm86Rp9q6j2I1sggMjIMXnmuvwLDjxKaAQy6YAkDq4eOvfcVy4cwfmzY3IXJNxNWD8QH1gxlX2cfOsp98RbGazHn11NF8rF4j1wiA9v1Fyv2fyOhHSjqmeyHj8SftrmjiFfit2pGnskjmRlevWU5+XUojoxIDk8Ug1ewvq+pZ+KBbKwBBZOPpcZCBpIyWGgHis1dT3YtKbooxeauY/RYAF96pbn46Zbsaczge7K/g1fSv5jMZb0ugeGHsH5NtrF8Q5EtxGeWrGQbxaEi/USzeghRoRi8gPnPVqRPc0wpUkqbhEG7g+VDLamha0l9XT2QUI9kpSiOFOsHjvRc9+nC3ogscflQAmpGTyPGTy0XP2SYfOK5/rJ8FnAyc1/NlKTOTkWeN3rpWL4mctPbv6bY3FJy792We27/u5ardw2DU0mA7fxyXmuKE0kirkeK8gCO3pcB2MWPeg6+rVBO9V7HeiyyxvEjz+ILIhs1FVx9BdKFYu9Dy/I1AyVMH4l3rB5LqMjlDwuLsf8HMkhdvaaYvC/ded0AePRMQOjyzeCPNAAhFbeTTQ6kZiKZywCwDfdpsoDRdz3XX5BB6vQjCaNxkfT82USJ7G+iqaciWwV5n/K7AX9+Jb6p7ZGk67aZojhW6FL8k3GorHCxgIrxM8vmliFei4RUNzB8qpg+aoT5GCoGp5brLxhOfqvm6t2Ewbl05m1vh/tiKVJK5UWmWU005aF0MeaX8lq88WxPpPYBD+Onth9U21QxfNmwfD3GpiIJza4ti9djRi8t128bDv6gpR1oin2FS4SdmH3asngjYvjSURxrpp+0FPuG7S3F/nel7uTg9xuKg4jhy5rl6wnxNgTQKMgvBRhUU6kaqfYUt/9ZyfKBJIEXh3J8LobJY8vmWOwJXUKxhKzId1KbC5huB3J8fRCMhsmnNS7RbE4iNncUUSnXtBmIhHb8tGXxZszk05btkWH8tGbxRsLohSO9ali8mZBfigzRpuJl3B5rAUWNpGbrBg5/rxbfXfAO1iOR8A7OW1b3YvIrS7EvwSz1UPoPZx9V1NOI9R1DfuYoZxpTeLK5DPSqyc4b6GKRwcaLhu3tlGaoaQZyr6CgmkqK7eBMehfXtyOageLwDwqWr2W0uSRK52c1mzsp2aV09C0fRIxeWLZHcu2Of6ehOBB53/77W+r9hGhjwzmMya9a2kyTXTbYzOAixea2Ye+7Zc9CTT5pqA4yikPD5JMSlxhsLv5pXYduxFWLbj3lYYyLFMPThjbX6FqCa+qJdFTuPyqYv50Hz6hcs3Ql4TqzjwqacUy8rLn+woDRy5Z4JfulW0d5IDMbydphCgFwnQ+xGUehJiShOIgYPa9pRobhs4rlGznTj7e42FCcJH3P3ejxluogQzmP2bZE85L2IGd7nDA4qykOE/JTqcOQxFZLtG2ppwnZ2RaXxaxeyxg/Lqj2U/J5TTOIyE8LaB2b10cMn6ypDnN8It/ZphZJbnZWYQcRzdiQXjbkTwuUtZT3p+LdLGzvz4wvNzRHQ5Gvblpc8HqqELoTXW/xSURypSjuDsmfbyhuD8mfrqhuDUnPt1QnQ1TrSV+tqG6PiTYNZlnitaY5HBDPS5r9jPiqxA5jolWFHSSYokGtC+zhGBcbdNWK7NUhYTatQxUV7eEYM5dKjuZoRDQv8bmwlC7WZE8XImONNHqxhVjYThUbqQNxrmf+/DBDL7f4NBbWcS1gR0WGPhE1idDXG9x0CN6jV1v8IEUvNuKhLGt8nkrf5ChHr7ay/qbFl5XIP+NYPI7DXKStSbxjGUOlB9biRwOpFvFe/q4aaAvIM5HGlqGHMk2EudQaXzf0oTs96AwALolRWuPLauetBFQsrKkvS0hT8E7WA6go6utFsFb23VrxV3aM5R/BRv6Rj39Feiu///FnSbr6J/34XDOWky+c+J/7hb9F7Qwn+YrzcsQkLjktxmybmLdn5zzfzKjaiNoaWqs5GG4ZxyV7ScGTzR4az9PrGcVVzk99+SN++9l9rNUY49ifbGit4f7kmsRY3j8/YbPKMLElzxrGmXxQiyZivc2YjgoSY3l1OWU62RAbh/OKbZWQxg2L1QC8Ih9UPXMaRQ6lPLERT6gxjum4YJjUvLiakKYtTWPwXlFvYyZ7W1aPpxy8ecX1Ykia1UTGYZ2m2AQfxLCmaQz1Nma8t2V1McTkFpQniixNFZHmXdqrJY4tbWsolynZpMJaRbNMMcNWgo40qDCt7s4k/dTl4cshFWZxsFfQNAb3bIC+t5VE18ZgK0P+cUrxRk06qmjqCH+V4BPP8HhD8WRMem8tz3uF1g73PMdOJDjILCLU7ZJ2LdJLOwzb1R6Mx1yL9E8fVLjawDpC78kXvC2NBBlNWkxiaZeJBBoNHMSO+CzG3i9Rr1LwCpc6/NCijId16FbKhYmNxg3tNiI+j/EPCpxTuELeY4YNvMhkP2pF/lKzvefChIHC3qrwQXZoFhE+8iRzTT0TuZkC3P0S1ypYSoemN8Im2JHFrA0u8cRzjUuFLfRG/JXF6zV6FeFS189oK6vwqSN7HlPPXD8Ln51qtvdblBdGN7vQbF8TCnX8QcT6dUkiRQtD1gSJn0uEXbQDR7TWNDNJRq0PLPHCYLaKduR7mWHXu6i8hNjo4M1Mr6E8kHCddmKJloaoVJS3Wulh3Ircyw66GglhHbNzjUtkuY4JbYcStNOxQl1Ikikl2Mim4fVIGEMfCXhafMERFRL004x8kMyGlNJUmIt4GRJLSwEE1X7oCl2qXQLuIMgLB7JMfqooDyTYSHnYvN6SP436apB2KExqslASzNPuaoWacfCSevrk2O43x+ae/FTv0kgb2N6WbUZb9RmGN7sQP2NXV6HbHVsswSqyTheFjs+1LO9i8bBWe8K8JkvxUpoCykPfM1XJXM6nKQVoFccyCG0HArDkQggraAq55+INVFNJOW27EKHAErZDCXmKSpl46IJ+OmbUZvRBNqaSCYauCqiXVQ7kuLoqpi7VVDcC0rv9UmE/O/CXLDrG0PdeJRRMHrfM34okfXUTmMom3M8hYbIZSXAQIIFBpQ9JnYF1sTKBYoruvAuLZTNFfiHBItsTmcCpx4p04WUiaB4YDgWDM0kbtYkwSV36qgvJrd7IsTVDYZWUkyTa/CywiNUNlr6EZOMp9qXmQoX00HjjqWYCsLuKIxfJxE/PHpruOoivNZtLXU33Pm3lXisP5HjirRzr5raWyZFU9ddU2zDRVvge7OsW8kvL9tD0rORNVs5HkF1LZ6e2niaXEJ8OsDZD3Xe0dgx7dh2eD0m6yUr+3QUbdexfx/pGpWN7ILJLr0MAEdAMNfHWha5S31e72BCe04wN8cZSzmTfdXMj8OggkkTbKtxf3tMMNMna9RUyLlYCahv5Po8KR5tLmmh61dAODcmypZrFpPOGehrRBet0ya420X3AjG7ke8EmOiQe74KrTOn69zUj0y/rYkV2Kes2pUPXDpsL5ZksG6pZTFRIeIyPZALDpor0uhVAGepmgD6IKCoszTgW9jFWxMuGei/BJYo20yRLS7SV5eJlTXWQkSyl79PUsp/NMCKZ1+KntF58mt5jBxG6EeZS1xLak8wrbB71bCYIO9cOJB3dbMVr3Uxi4uWOxhcWV5jPaFVJum0WidKituK9HEZE6wZvNNGqCt/JMbqx6FVJezCU87ttgqS16atbfBKhyhZlrVSf3KgG8UbJ641FbSsJ9WlCmI4PrGeoRlGBpVRFJT7LpsXn6Q5cdswmfIZlFOZZuiuV8/hUAGlfcWKtAMwbFSSfqRu5EbzTg0StoJLxlW/Csj172sp90rYCiMPzvgkJt99fQ/IDHn8sPOL8vxKM5c/9D3/rT3y9//vX/us/9WODzzmwzN6863/mF/42rdM8fnlAlLS0dYTSnjhpaVuDnSfcenjJ2eWEOGmprnIZeEdB6hmkoum4olqmEsCTOthEpMdbyqsMXRiilaI+CBURxqOWu7zvaKNo9i16q0VmN7CSotqITNKsxYvi98OsznWMSx26lvfYWYsqJcBGWZHi6UZ8QFK/gEhGFahGsf/OFcvfPZCB/dyIlHCtsIMAAIYip7T7DThFdBmLdLGVxMtkqWlGDpdIXYSkTYrUL9rC5oFFNUqqBYJc0sU+eH0gub2hPh+g9yrsPAENw0cR1b6nnVrS0wibi8SPry7YXgxIziKySwmCqafC/MYLjXpnTX06QJcCaOKlpn67QL/IRJrnxcPTJTHqCuqTluxFTDN2wtoYT3IlPjOXeAYv5ZzVY7+TkKkgo1xLhYSpFM3Mkp5FJEvY3naoVhEVimgD6y/W5I8S2oGnnTqylyJJLe61jB5F2FgGnvVMBnjVvqeZWKK1+MbsrCV7mqBDGIYppQO0S4v1RsCWKaW2Ir2S6wCwfdCSnkYiXdTCdFV7AmZwgXleSxdqvNrJGKOtDCTbkQyEXRwAWBHSKH9sjvvNWQ9Getmkg819F6SNArqqAwkQ6gaFfYptJOE221vSsdoMobzbkL6KSefCqiVLJV4rI5LQZKF6hq7aE2CgWxkUb19r2fuOoTiW/a/2BNDFa6gOdsDLVAHwzEQGnJ+FBNgAalwExW3L4IUJ0i+5X2wnN72WQKHRE5ES9+s5FaAHsn/tUO6VauZ7mfL4U0mOLPcV8Ub2u5NNRoUApc196cV1sYCRwcvQTZsKsK0OHJNPZBDcyyrDte0CbOJVUAgE1txm4neyiXSlogRcD58LY9z1tJowwN8eK0bPBaR0susOeHcy1WYk6bgu2tVgwC4FVt4jIKGahU7Ziep7X5uJVGi4EFDU5gGkhc5SXQu724zUZ+SgAMNXjnoUpHtRSBmNAiPtIV6FJFgr98nglYAUH+R2Lpb3lAeiJrC5nKc6BM1kl57tbcX4iaM4Cl6yqe7TQFGQn8trUunhe0WIi+VzUE8Uycr3jLfXkKw91VTOU7WnyC538mwQgIAXxYDUXnjSpcfGUBzpPtQsnfteKpusPKvXNPmp7+WcHfjqPIPdvzvZbTtQvazQprLM8KX7TCJqm+0mC2wioLUZBvBrd2BRWyj35fqC1H0ky9BJGxjMjhXPrp2k+k4NLsiq8ytHM9glqkZlqA0ZC/DILxz1eDdZEJUSwmOC3LY40AzOLDZRbE+0sNlGUc4Uo5e2l0N28tfufJja99LrNtN9DYcpBcR4o6gmmvwiJJUORN5oGs/2UPehQl0PabKSeowuVdUm8v5kZdkeRX3oD8j5dDEMT1uK/Yj8sqXcN6RzJ764IEsFuR7d/dPVh0jNjEhJde2pZoZkHeSYrZzfZGmlTmMo/j9vBMx2AFQ5BFgZRTMymMr1UugeLLciCbWJ7sFZMwy9piosH/oktfWU+zHpdUs9C92LzhNtHfXEkF00lAcSLqQ84DzJvKIdJ7SZId62tLmwvNlZRT1LgsTZ9ZNjXbBQvG776ox2GGEC25ksauqpBPY0Ywk0irYN9TSRTk/PZ+o8bGoC8DOYQkKRXGrQAVR2abOqlU7Vdpxiiia8FmPWNT41uEhjyhasp52lwghvmn6iSVdt371pB6En82bYjVISxpMnu6fKRv7duh5IemP+OemrqkN4T9PuGMguxCcyUDfSh5kIO9j3YZaVMJ2BIRSQKkmxPk9RZS3bzFNhLJ3Dp7J/alsK0EtiYT5vSmLDuj4jWVVKgGMj2+o6KFWoPPFliUoSAZhtKwyElmPq+yw7oGqteD6V+mxIkPf4Tqp7MzH2X8BSft6B5ezdY/+NX/j3/sTX+398/b/6Uz82+JwDyztfnvm//j//W+SmweDQynMYrxnomj/c3uYkWfKs3GPRZCTa0nrN0NRo5XFecV6NGEUVpY3YtgnrOuXHj55wHK/4/dUdlk3GF8annNcjTrcTKhsRG8u2idnLChLd0nqDxjOIaj5ZHHA8XJOZhstyyEG24a3hOf/0/A1GScVlMeBksGbVpJ+hwa3TDOIa6zVFE3NntOCyHLKXbnm5mRBrR2pa1k1CrB2bOsFoxxf3T/l4cYjzinvjOZ8u9kmMxXrF3dGC710cc3e64GI7xGhhNRNjqdqIPG4ompgkaom143IzIE8aBnHDq/mYKHLkSUNkLFp5sqjlYj1kkNZcLoZMRyXXywFvnlzwwZMT3rh/zvl6iAL2h1uuNgOKQr7QxqOCuo2Y5CWvLqbcP77mepsTGUtsHFUTYb1if1CwKDI225Rb+0s2dZBcBG/s/HzE3XtXXG9yhlnNYp3zxVunPF1OiY1juZV8/r3RlsvlkChyjPOS1hqulwPyvMZaTRxZrNNs5jmT/Q11E+E9TIYl81VOmraURYLSDtsa+X6NxaPb1oZssPtSa1tDljYUZUxTxAwmpTC31hAnLUp5yk2CbzSHt5asi5TyIifZL2mqCBM52k2MGTZoJf7gdhMTj2qadYIZtLhWif/WK+K8oa0NOkip80HFdi0yPm+19HYqj2s1w3HJ+nQEHrLDgvr5EDdpUcYRJRbb6t2ARIPdRqjESkXOMoXYMZyWbC4G4sGNPGbaYFcxg6MNVRVjlwmqVjCTAYAvDeleSX06AKekh3LQynHWpvcMp8OauhL2Wh/UuMsE37Hg2hPlLfYqxedWvLiRh0x+gPR16Bh1SvpG92t8q9GLCI4rGaRvI8y4wa5j9NrgJq0wu6H2JrqOsCPXB+CgQE1qYaGDV1iVRhjavRai0COayf6IT1LRzuQ1at1X2qhKgwGvPGpgYR31XtebtSkopBfzXH5o6yOZSOp8ld4E0GE8ZikSfpdKcEvvS44kcAYbujq762mVhNRMpNeT4OFVLTR7YcJKg6mVpPYOxRutKyXXLfYymXZkyV5Fn5mI8sZLvdBCkhM7plo3qk9dlokcR7TVqAbQMrGja9V7IXuvbfDzukT8s+VJS/4iohn53hvZ+Vx9LPvoQ0WRD2C277hsoZ458eEG1ro7LgFkoVuzgmYqEw7FiUw0dQnCdEqDzPeTa67LzFDBhxu82Tt/JL2XtHuv11JHFK8EiDejEP7TvVYGmetKJoOitYDlLg3ZdX2WEbseSbWrjXGRLGcTYcpvepvrqSe7UrSZsLRd56Q3od6nlUmSzhvbDtl5VG/4asVXFyZRwnZNsevflGCc4F1FJv4AXCosuDBo9B2q3USfKQXwRlsBMtWe2Bm8ksmPqKQHTLoO5yf0fHa2EWUDc13vGO5ozQ7k+Z1vtZukk3MvkwqyLbUDftAzgi6RSYg2uwEWc/Fwtjm9ry5ddvJmFbx3u+vU+XF1EyYC2KkLlAsTMLnqfcFRIeekqzLCi4ezCZL6bj+6SaOoELAOYfKy9v2kROd7bDMB/VHhqMe6DzFKVi5IqEUq33V+dpObMjkhvZXJxvUAv7OlRIUP1o4AyEP3ZectNXXol/Q32O9wf0lqr5XgqcJJ2u0mVK0UcpB97QoygdB5Sr1SRIWVKpdwrUwpkunOU2oqF/b/xu+bD6x7WKfrPZv0MmWvEEa07tg/drUjracdRqG7MqgCSqn/6JbtvZ3+pgqiOwdSQeKSCF23wm7eYCj7fsvYSOIs9FUv4udUslxXbwK7SpOOpexAqfc7T6YOvsyO2QygtV9GKWEw4QcwliJhJo526+w6NJ37LOi82VN5sw7lBhD3Xb3JHxHQ0wPPH/D4fpziq+oHvu+P8/ga+FNwAAAgAElEQVTXwPJf/uNz7bG8XI355i/8CNfvOeKjgqaKODlacPbdI9JLTfbnLil+50AYhQcto08j6q+uZWBuPHcP5/zeR/dQxnP0/ySc/3zFL//2CdmX51R1hP9gxEfvHPIjd59zth5R/u6+SKs8vLxvUXs1fpnIACdxmJVh8SAn+adj1vcdT28X3Hp7yZMPT5jeW2B/dZ9vv3MAxhNdCoO3fqth8n7M4i9cszwbMfoo5vkbh5A4pt9K2P7UhrY2DMYVzfsTmokjvTC07275J4sh1mrcOuZ5sk98FtMOHBh4Odzj1v8V870/Nw1dmYrB/RWVdixPR8RT+XVua4MvIqKFYT22qGHLwT9KKQ8Vl69Z8jtrtudDSBzxaQynir1LT/nvNBz9YsaHP3OX2fuG870hxfdmTD6GT398hKo1h9/UIeAgZ/2e4sXxkOxZzONGc/SrCedfazALkffoRvE0n3Lw9iXxd4Y8/5LGXyeSinmrwlvN3m/HvKiOOPwtzeKvbhj86ojv/pwn/7Uxp+81JKcy0H1xNIDUwbOYV8dDVN6SfZCxPcg4+U0oZ4riAbzxf9c8/g8z7Dpi+vsxth7h3vFsUk98rbG55/6vWMoDw9mfT4ivNfuPYPvXa/y3phRvVCQvElZ3a8a/mzJdecr9lKPHlvy84eP/IOX4H8U0X4LBqaLYj7Efjnjz71ec//CINPhQX/vNihdfG4ik7H7D0a9HbO7E7D9xLB/GFA9rZr+ThBCQmNnHDU//UszBdzyn34h4+39qePxXRA9m3yxwVymjRwbdZBxsPYt3oFmPuP8PW9Z3YuZf9LQRzL6rmH9J5L+Hvx6zuasYPYk4/4Zm8gcxxS1PdZbw4FdbdGu5/GKCthE2Bf/xFPuw5fifGaYfbnn280OKhzV3fsXw8usD3v5ftlSHGcVBxPYkJr2WMvbtHcvoD2Pan64YfDvn3t+/5oP/JGPwwlDta279hpMo/v94i/mljPk7MdmFCkFAEfVEqj+ufiwmXmhu/4bl6V+OUQqOfwtWr+W0Q+kdVc9y8pVi/NizfCPhzj+uePKXE7yBe/+wYXscc/1lzfCp+AHPv6FRhWH4TKpZHvxyhUs0p19NKG5b7v8Dz/Ofizj+TRi+qigOEy5+WDzSr/+fa1799Jjhs5jhK8vzv+yYfSemnkSMnzr2fn/B1XtT8suWZN5IyubDhPkXNA9+eYvXissfypl+0rC5HbN8qPHa0w49h99SrO8qHvy9C+wo5dnPj6mnwgzjYfvAk51F3Pm1ihdfTzEl7H1gBfxYmL8pg7DZx5aocHz6bysmHykGF47hsy3r1wa0meb8JxzDJ5pk5Vm+qbj/KxWP/yPH8S/C2Y9H5Kee2Yc1mzsJmzsxo2cyqK5nmr3vtQw/XnLxE3ss34DJI0lvnH3SYkpP/mTB4795CMigefzE8uIveB7+XUszkZ+hi/cMsw8cV7Hh6FstT/4aTD6IOPx2RTM25KclT39+xPCFSE2HLx3X72rGn3qmnxS8+HrO7CPH1ZcM40eeq/dg9oF0H04fNbQD8aFqKz7D+Zspg/OW5yeak99ytKli77fPOf/aMflly+lPGPY+tKFkXpFfis9vc2KYPpJO4HagGTwvWb+WS09h4WjGhqsvCrq+9RsV+ccX2MMJlz80Il1aJt8649N//zazDy3ztwyHv9/w/OciTn6rZXsUsXwDXv+751z85GEPLuKN+PWirbA6+ceXADz/G7clNKaQ36Wjb20oTjI2tzSzDyuWryeMnzZ4o0jmNZt7OZdf1gxfeo5+a8H5V6cc/+or/Chn/qUJ9UgxemFl8J5qBh9ecvnnT1i9JnUx42cV8eWWZ//mPqaC0XNLet1y+ZWUwZlj9LjADiKe/nzC8Tdrso/PWPzEHVZ35V6YvxGRrDyH37zm+r0ZB//4OSjF+TfucvDtBT42nP7kmPzCUe7JcSUrR/TI4VJFsWdwCQzOHOM/uOD8a8d9sJhNFMMzy+B5yfmPDjGVZ3BpSRYtxVHSV8Qka8/wWYE3muI4Ib1u2dwRz+XgvCVet5T7CcMna8zVuu8/LN44IHu55vRn9jn6zpqL94Yc/JOXuMmAyx+ZhXAexeRJKeFMmRHGsRYGsh4bRo8Llm/kzN5foZ+8ZPlzbzN4UbJ8M2f/N885+/oxgwvL6LtXuFGKOVtw8Rfus/+dBXYQUx2kbG4ZRo9r4nlJvZeRPV1QPNxDN47se69o7h9SHabkLzas3hyjnGf0Tz6m+uHXSZ/OWX/xgPG3X7H46m1AMfndV8x//Bajx1v0uoTW4qYD9HxDc3dGNC/Riw3trRnLhwNcpBh/b45PTQ9QvNFc/tCI6Xc26GVBc2tM/GJBczIhWpa0k4zNvQxVixR8+L1zNu8eMXz/FRffuMfs1x+zfe+e3Ndas31rn8GjOXaa404yqVV6/4zyjUPSx1esv3LE6IM57TQnfvSK4iv3UB6S0w3q/AoOZjQHQ5F+Nw7l4eK9nONvFtjUoBJD+nKJHWdEL6+hafDbAvWVhwIei4b2IKfrE41WFWZdCwvZOuwkJXp2SXP/ELOpUNsKNxui5xvcdEB0tRbWcFPg9saoZ6egNGY0gLrBHU7Riw12b4x+dgazMWpbYm/vC5vpPUprqTbZFFLHcXaJOj6A0ws43IfLazjaRy3Wvb/ST4bgvHgtAbc3QW8K8UDOxnB2KYxhxwhqBXmYFapq8V+CyE3XG4giweZlicpljOHLMiwv4M7XNXo8klqSqvqMhLZnI5NEXlO6f657KK3wYX/+WDSXd//i93wOHv/aY/ln9DF799i/85//HbZ1zFuhiuPOaMHj5R5XyyFvHl+wrlOKJqa1Gq0djTUMkgalPNsqITIW7xUHwy0fPz9iOC750VvP+NbpXbxXfPn4Fe+fn1AWCSZylOuEwUSmbkXarnBOMRmWrAJjBlAVMQf7a5yHxUr6NNfrjDRrKNcp6bCmXKYo48nHsj7vVV8DwacD0ncXRNqx2mQY42gbg2s0cd7gneb2wYKXl1PaRcLszpLFfECUtsSxHFO5TYjTlqaM0LHDNZoHdy759OkROKRuYpFCJGwFTUiVbbSkuo4btJbtJllDU0W4jUzHJ3sl9umA/M0lmycT9FGJO8/QtcIeBkavMgweR5QnDjduwSmwivxwS/lqKGxRYtGRxy5i9KSRCo3LjOyooDobYPYqbKPFe1grovsbqnmGGba4ywR9WGEXQepRa5HFHm+prjPwinS/oN4m+EqjMitpuA7QEC0M7VGDKoQx8sbvPJjhPaqW5Fc7Fi9ptDL4+wXuIg0yZ4Pda9DLCFMr2rEDizBdx42k2TrFzXTe5MrQzFyfdGu2kp7beVfTM0O954g2wrr4vYb4WUJ9YEkuDLpVlHcakvOI+rjtfZs+8pKSWxjS84hm7CR1NQWXObIzI2yFg2Zm+zRgtEisdRm8MhPbp8/aWUv2JOnZHhcLQ4WSfU0upZtT5M2O7CyivCVSXm/Ctsa7lFZJUxWpcHwWkyyUdGS2MsseBd9fs2eFtRp6FPQpq16LZLQ8sZitJlqLlBgNw6ea9RviiTWFsG6qlRqFeib+UK9Fjjh4pSiOpb8yOw89pFMHTuR7zcgzfiQBUO3I0w5EYr29Z8mfm555qfY9phR5cD3dySfFDyfLRVtFspTkXhe8kt1vSrXvGT9S2FzCt/JT1UtxlVN9eq/0hnby5vAlE7xo7dBJwu4mJI9GgcHSO7bEheReXQc260JRz8TzqVphe6o92dfhS8/mtiK7gtXrjuFTTTOW7WXnAurqPU96EeSvYZyQzD3FkepZNRcL49ZJGptJYB2D97IeiywXOq9gFx7mGH8qYUC6EvZK18JyFSeq9xsmwZuYrOiDiJKFsF+mlPOUXtPXWEQb+nTbLuk1vfKsH0j/a5fC2vkC66m8biqRiuoWmUSM5HzFK99XHigvXtRmKF7EZqx6L2e6FIlqPRaZr6mCZLYSGW+y8H1vp02hPBQQ14wD0xaAk1RiyPmMCvm+2NzRsh/BB9gl33YBVM1YEa9l+/FaUoqLQ5GVp4sgd56H9NSUAISkP7PzujajIGVfyjZNHWo0tKxDtxKkBXJdo0pkp8lKmJo2lxoTE+Sy2oo/sjjQZPMwkXRL6jqUg3JPmDUxn4s/tOuxrMeq74ZNF471bROYOpExZ1cy6KzHAkq1DeFvSgLrOglzVArj1vk8bwR9Y2rP9lgzOHM94xUVTuouri2bW53XEUavRBbbhTaZxpMuXEiW1X366032yaYSyhUvLet7CenCUk0Mw5c1m9sJUSVSVGUlUGhzKyabS/BVm4kvU5ZvqacRycpSjw3R1oXj1cHCIoyibj3pVSMget5iM5HVNiODV4gMdhoRFcGHGQb5pnG0eRTSty1tLuBYt570upUQrtwQr2V5rxTRNkh58yDT7ZSQid6lxBpk32exeCuHEfFGJLXxspZKj9xIAFCsaYdib4iXIuuNlzXNJCFe1rjUgBUQ72JNsmx211KLUsMr1TOIpmjFKxqLDNZrJcE/VhjF+niIKcQr6BKDqWz/Xd0lx5ptjUsiTNHgEpHl4n3vmfRGCZsYrrfLY/RWxkRumKKXBW6Q9sxlB8RUUeNT6fPs1ue1RtdBSlHVfZhQnwbbJcUaIzLZNBEWsqykuzJNUOutAMU0DtsJYTzWCYMpg8/dwLoDnW2Ldw6VpTu2kh1bqOIYX9egb0hbuz7LDije9Gp2Hsvu+e5nrOvetO4zIUF/5ONfAY/l7N1j//X//m//ia/3F3/2v/xTPzb4nAPLW1/a9//p//ZVXtYzChtzVcuISwfNyFU1JFKOg3TDZTXEodg0idSQKEdmWlqvaawRyWIbU7YRRR2TJw2zrOByO2SSlWybGOs03iuGSU3rNNs65mC45WqbM8tLqjaiaCIOB1uWdUprDXVrMNoxzUu08mybGB30U0Ut6xxnFUUT0VhDFrcUQQIaaQn/aa3GWk2atHjAOR0CweS5qo5IkpYsblkXKdNhwWKTM8orijru12OdxmhH3UQSkuMEbHdhRZFxNK0hSxrKOkZrT1XGRLGlbQx70w2rbcZstGVTJSigKGOOZmtOryZoHWQmTjMYVDinqWtDktgQQARp2lJsU9Ks7oFvFDmKbUKchF688O/u0ak2srwWgG9kn5X2/d99DYwC10h9S1fLooy8r60NynhsGaGMI85a2trgaoNOLEnW9oFDw2lJWcZo7fBO6mlcbUiGNU0VoZTHOyXVMsqjI4cysj0I0tLCoCJ5zmQWW5oeZCrj8e1OukgIXMEqolEj8l8HrowE+HfVMY3egVRFAK6IHNMqkYs2QfPllFTXgADqSIAT7sYoqvPndOuLHKoyIkHdGPH8pg41kNGXtwoajS51LyvVeSvMfavw4xa1NvihDZMVgn5UpfGx+Fg7v4mPPOQWtRFgrGz3Cy5A16XS1SeMu8g7sSocK7vXoK+00UGq2ckfvQGfeJG3Qu8plXRn8T+7VKpW+mvQXY/utISAEpcIUPUGmRRplIQOVcGDPHCYdZikiIKcMUJqfLzsY1f7ohrVy1A7SahIzFTvgfPhmnQSOJuFihcrz7tOatdVooRr2ckEO0+yJC7LMZuik53u1uv5bKVIMpcKGBfT12lIrcxOdtj3KyqR04rMtUsa3g0OXHftggS08/XelF12z+ta/NQi+RWpqUgpBWR3y3ZSwm7nBcjfCI5qd6CoC4ABmRDpJKZd/UV/jAFwmFrOq492ksvunPaf1f7GCPdFqE/prklXlYKT13xEYD522+zvq+4CKPqAJhXkp7ol9PR99joAu3qhll1Fi915L6PtTlJ6M+BJPHa769kFAekQcNVVqfQ1KOGc3wyB6jzjhHu3/8yG4+0qR2xCX2ciDJ4KFS2+r+uweQDoUQABmr42o6+S8Tf2q9uO4bP3g+++U+gDi7rz1p3fm55jF+3On9RZhPva7s5Tf50cfQCaqW4ARS8eY2UFtPaySed39R2dtK/7n/P9PeRNSK+2vq/Q6eqo+tqasH2CvLf/9433SCUNoIIH94Y/rbtfbspTO8+xsmGCMJLX5Lyofh+1lTqRvuInEuDcnU9nus+k3+2TlqAgkabSBwr1/lP/fecpyFe7WhOXqPCd5gMT+tnKlpvP6Ua6ZjvfaHdu+q5K5/tqGqnxCL8V4bp0cmlvROoKoG/IOFX7ffvZ3njtxrVV1vXn2SsBoBIwJK/Jm25ITJWsq7tH+of30FWoBBkxLmyrA6udD9J5AYU/qG6k65iEnbz1pvS1acN5dzs5qvu+/bi5zzefu/l356H0flc74p2AyxsPf3M7f1QtSrev/4KH/z5w+//n8WcBWE7fPfFf++/+5IHlL33jv/hTPzb4zPDp8/dQynPdDnlZTrifXXE7W5DqllRbnqz2OclWAHy8FAnWvMy5PVhilGNVp2zahKKNyaOGt8YXXG9zvn7rY/7Kgz/kx46eMUlKfuLkCSf5itRY7oyWDJOa1LS0ThJmF2XGLC+5O5xTNBHDpOG6zIm1Yz/f8pfuf49RWlO2Ec+vppwM1sw3OfNNzmIxwGjHqkyZ5SWv711zcTkWgDjP+fLRK9K4ZZA2TIYlg1Q8grenS754fMrfeOv3SWP5EGdxy/nplMmgZBA37I+2AjLzknFWYbR4Jn/o+CWjXD6UeVpjjCNPBcis1xnDrGZ+PURrj1Kew70VdRVxvL/kajEkjlvO3z9if1CwfDHmrZMLXrzc497hnLaKSBJLFFlWV0M2p0OabdKDMK0lKfbe0TXWarJBHTyMniyvxRM5qNhcDHj7zhlp1jAZF+SDitl0w3aesz/b0KwSXju5wm5i4tgKkAygwESW27euMcbhreL4cIlWnnqVMJkU2G1EnDcMJ6UAUq84uT3HbWLKRYrWAjjrKsI1GvtqgK3FZzmYFtSLlOl0K4DSKyg1JnG4TYyrDSa27O9t0E8zxocbkkGNySxunmDylnRaotYR2aiSHlDj0etIPHLGk+8X2Asx5CQf5ahCy+9CpRnMClStUKVGJQ4VO/LDbQCEHj0KDGnw5AGoy0RA5f/L3nv8yrbd+X2ftdaOlU4ON737EvnIVieqA9o2JBtSwzIg2DDgiYM08djwwH+A/wIHyIO2rZntgeGRp4YHQgsyHBots6UOJF+Tj3zpphMr7rSCB7+1dp37mna3YAIkG13AxT2nTtWuvdfeVfX7/b6pdOh1JgilCqjKMbvYQOnJDnrJBe00WE19KZkiYeJQZx1mY8helJjXBWon280e7VC1RfUa80WFWvTos5b8Tc702Zryi4LsOsdsjDRthz16PhDmllA7issdamqpf1CKi2+AoAK6lWYtf7IlW0nOaXZvxiYu5F5MsmYOXzvRGeYeTjuKGwOnHeq4J5SecNHha49uFBwMkjVZBNxM6OSogDtwmEajrSIcDvjaoy46/MFAvkyiLHAzQXx9LShwcW3I76WJtdEt1+x0RL2kwOovB+yhwy0c1bWOxZCifmkwvRg32dOB+o2iuNfiGtxExPTQ4SspyoYjNyK1BHGKdUU0UyoD/ZMeV4kTro9NKnrfELhKvoyzbUTiHg3SPFaynfJOUO3+yI16OVfI6/WnTsyXMkCJ5hCg/0YzaiNtJTE71ZVEYNg6yL9oulXeiZ7P1ZL32p14QQEPPVkTC3cLzYVoo5oPu9HUyueiFfRFYPJCqMGuSvmCe81evhbzsqyF3VMbm5YgcVN1RMqivhOkQegXgWwLzRMnqKiVSKW0zeYsjIZbIds3wSDIc8rzzDeM+ZhJX9rPA+2p7M/0y0CxitfEQRDdZHRbTa7J/YEgoyETI62H2sqUITk2UZaI0op5VXsScLU8rn4tDrKbd+S1+7mcSxsbOG1h89zTHUUEEkZXWG/kmFFyn3JCdfW5oOmpCS3vAu2pILpyXclQxBfyGvkmxGY7yGAjCAqbGs9hJqZCwhyQ3MLUEE7eeIZoAORqiZkhCBIszr7qLd1jv1AS6xOguVBj89We7ItiO5HmqF8IcupKGQTkG0F0R6MrE9FtBcNCEM/JtadcBoptoNjI2mwfSyRRe6yo7uW9ZivYPBUnWuWh2PqxgVQ+7AcoTii7KMh3fmx0+5mYLzVnGldGA6ZMYmbaI2nIs6hLbU7kSUmvCKJDDEoeH4sjzBDopxpbKqYve4apOLLaSpGvHT7qIou1w8aMS9N7ydB0IQ4ofDRMkoZgd2ZoD8XILt84hloMk4KGfiauscPMjEitq3TMYVWj9lMcer240zaOEHM17VRT3g7iGBuNmACGmaY7zKJjrca0Pg7jFHrwmM7THefYqRHqqxcXWW2lMbaV3G+nBlsbhmkmKGhsdPP7jmzTYxrL7lwG2kEr7FROmivNaBA0LPJ4TeVou0fYglG4SSYaSb/P7gxG44uIQrqAr3JUK6hpahiDUYJcukAoDaGS1/WTAjcvZR1Sz2XF/CghlaEuxJCnLsBoQpETJqXsl1b4w+nbjZzRhIRSgpj/FDmhKiQ2pYw/Vyl4WAlK+vCWGs6HJjw+SMzKj2kYlVKo9NjkRuul2Qxv/fxnNZdf/feX6eZRP/F/Pyu3n2vEsvrgSfjN//bfZzfkXN/MKaqBflegCycoVO5xOynaUQFMGJEfNWiJkkhT2cbA3KKvCjGRWCn6S4veGkFmQFCaIhpnOKGp4ZHCtFPYmZfiJcYWqF6+AN3UY7YaNxXqYH/sUIMeC9A0mZcvzji9LILETuRSbPtCvsB0REf81FFcZ29NcV2ZEBWJEugPJD7ATj1mp8epq49FaXKZTfehAqaVx9nJnk43ToaNFKkqBsWbVpGvFbt3hPo4LMRGXfdqDG33ZaB4ssX+YCYGHrWnfilUwuSG6oqAj+YG2Wa/7RTtYGdhDM3Omr1xhqvEHEPF9XrIZ3J1GNc3mXJop3BFpGVOJR6jvNEMB3Ies2gAUtypaFiRqJuCdiSkJjmCJlQixU+kNcq2sm8JgfO57H+xlP2SMG15PT1ElKZLX1Cyryq5okdUJOSyP6aRbeRrOb96UBRLQSdS4Wl6KZrzrRqn77A38UgIhp2FMUKjP5B1SdN3OxFTkYRi2Vr+nvYvFZlpyZNhhWSHSiGLkrU3kc6XYgH2WZgR4YroSjLvUA7Q++D6FHSfRQrpECmUIW4zGV0Mc6ivHhxj3M7D10iZov0Bo/GITghaPIZ+IT9njbjc1ldSdPnigRFH2CMY8n7Zv09gvy7pnIwh6JF2qnvZ96xhRNBSY5Oa0xSNke3kmmmP5XrRwwNkJdJdxaJfrk2JJRHKnekEKRpjPIJQ5FwptMx+Lm64D51TXbV3WzXR4VW56CI7e3CdRpQR9hP8hPyIAU4Y4zhSgHowsk8JjRrRwDgYspPoRDuXwljZROWN12YrdE4C5NEN2BVSCKZj1TbEInuPRg1ToeEVm4AtBbVMhi5ZE8Zgejm4+HXRCVU0aHHnfWgEIk6damz60uuMyE63v8bT30wbG8l4/hKCklxVUzZmHqmfpguju2hqTLJWcmpBruOslf0Q59VI7UzoWUSnElo5XrMRlU2GM64QMxf5rFKxYZHHaStNuTfSlKdrJGWjmt7THpho8CL75UpF1khMR4ojGWueEe0iOsRK0auCoGoPjXQemsEImhzeQrmC3iM+Pl5XrpQmplg57ES0xUn7mNbblXKilQ8jOpqaGHlPSdzHMM/25zA2bYKWyfViWnFOzXd+RK8EzZTvhWROk65dFcKIghFfR66nEFFHtaeiahVRU8nAFBfoGBUTiGYyZnRfTcig8kKFzVqHreLfY2OXngcQMoWJTZ0rxKk2mc7I4zWmdW+dt2Bk3ZX1+MKM75dgFPnGyvvYKEHdlETbBKUwrcVVmVBPc41uLL7MBDFLSF90qU3Zl7joShwpq2qQ+A9lvdBRtx1uXo2IYFAKHemnvszQvVBY1eDByDHpdsBPCnGLHdxIbQ0m0k1TX6MZ0cFkqCP0Uz9CMcr6vQFPQu20JpQG3QwErVHe7yNBUjPngzSFCVXsB5K5z+juGv+XA3uAQI6OqQ8asAfIZSglFxMQp1nv36a72tjE5dl+v7wnJARUq7eQS5WMiRJF9aEuEwSZTDTYoSe5woa2e7ANI83jVx1oYWwURyrsX7QX+UtAhT34xkX4l//hv/sT3+7/8q/+Vz/1Y4Ofc/OeWdFxt6v5N9/9I36/foe/e/FH/MH6GVkU/bxsDvjjzx/x5OwerQKvl3PeOb6j94Y3qxnfevQlH9+d8duPv8f/9Me/xt/7pd/jf/zOr3F5uOF2M0E3OflBi/18inm641eefEmmPL/3o3f5+uPXXO+mXM7WfLlasFxPOJi2uKDYbSt+4elLvn91ijGe09mW18s5AN0s5/Roy6zsOCp3fLE+5Pa7J0w/WLK6m/D40R0vXh5RzzuaVcXloztB3GzGyWTL6/Wc1aomzx1uPnB2tKbKLD/69IzpcUNdDFy/WmAvLTooqmnLyXTHy7sFAH2bU087Lg/WrNoKpQKbpsQ5jbOG2axheT/h3/iFP+F3P/2QqhjYNiVZ7siNY1L2vHh5RDnt8V6x2xZ8+O5rPj045sPzG75cHoyZl9+6/IJ/8idfZxgMv/gvfcL33pxzWHdcTxao1lBdbGnqCflhi+syLs6XLLc1/bpE556wLCjOdkxL6bLWmxpnHEOT8/zJDdu+4OaTI8LUkU97oe7eV+iJ5fxkxc1yim1zlPFiBHTa4e5KhlOPmQ24NmN4KkhpCIr+synVe2uUCjTrisPDLXfXc3QR3Uh1oG+yPdUU0Zr2y1L0iltDftHQvpjAWSeRN5/OcAuLKj3NNOP0/VsGa9g2BTp3eK9RAZrrGvJANh1QP6oJFej3tgxNztNHt3zx8pjQGurTHZNi4P5uSlY4utsKn2n6C4sqHVnhaJocU1n6VUF+2DKsSnRt0S8r3GXHfNGwXtZkpaXzGttmmNIxvFQsSTcAACAASURBVCpxtUdNHKE1mF/e0rY5z89v+eT7l/QmYJaGp7/8ii/++SVu6lG1o/64ZPOBpT7d0d3VdA/otfl1RvOORQ2a5x+94kefnNNZLYOWSnSk/QH42mOnWhxGO9Gqzp6uWF/NUIUjOD2ipWFq6Z8D20yGLicDLHPIAv0RYNhTXwOEw4EQ9bej5rXw5K9yum9KjmkwoM47XGdEo7nM6A9FbypZZ6JlDJk4vc6/XbL6hV4GUla6SBPjhhLi5U8H1K1MuP3CMj/dsnw1RzmFbhT+TAIrs6iDpteo0stAbJWTLfcT4ZAHgpJBkSvD2LAkRrPc7/EzR/E6k9zPw4HsNsMeWnGbRTSzANlVLrrPApbfdDIo017ifh4P7O4yfKTzutpL/NGdoj33FE+3DD+akTWiRZVBCqA9+VLTPrIUt4bJS8XuImCPLGYVM1/f2TBc1eNQq7iNea5xWJWtBW3KGlid711d8xXs3h0wq4xiGYcqLjboU0HG0kBwOJChTnkjzWJ3KgOvfKlYfSAXhXJKhgq9FN3lnaBywYjekjjA0IOiO3OUV2bMpewOA/UbPQ4+TJ/ihjz1C0N7LusolGehCuOl2eyOZUAj+lNZN8kx1WSNIJ35Vo96TVfHY4pZp/lG9Lj5Cihg+1SNVF4ZJqaBSUTiBmnWhplcR1kjf+9O9vrjYQb52pA1okvNV4HmUo2xPflGGs3uODB5KXEszbGiP1Qoq+lOA9PPDK5mHCwEbXAFLD71rJ9pUIKm2qmin8t1m20F+XW5rE2xDnGAo8emXluJBpq8DFEbKL+Xyz3d2fRhRIt9ERHJnR6HQTIQfDBgyWVY4wuN7vc6VEHABHHdPCnEsXcX6A40zTkcfqxozmM8l92/brfQuEriWoaZIKOzLzz9QqG8lsc+oMqmzFJXQnUnuaAPKb8pC9Tniu5Qke3MqGd1ZTo+GQBNrmB3ZiiXgX66b8yzzrA701S3gWGiooPvftggjWK2pybnGVmThp5y/etBx9gSuXZk3/yo9RxmivLeo51kY0qEkBO3WSNDAdN6hkWJ7gLdUSVDpUUuje6DWBOI56hQ0sAnimypMK0Zm3XREksD6o0CLTmKIdOxuROTqu64FGfZVqigwUC+yRgmGcGIFjRFltjakG9M/EzyuMpgYlZmtpXYE9M6tPW4OhuHI8Mso7jrxybaNFY0n0hzm4YG+EDIDWbbE3KDqzI59rsdYVpJc6fUvoH14vGgu0Ea7kFyMEMtzaJe7giTMmoyi7GZVyHgq3Ic8inn3m5kk8srQB4be+f2aOJX6a6Rdqu+giSOt0z0nqFpIC/AS5Ot8phtaUzUYGpxj9Vaji9tN6LgIUanqDyTzMs/x5wn+L9gA/qzfAv8lXnPz+qtfP40PP2P/hMxCXGJdy9fJhCRozJ+kSdUYNjrGGwddTR6j9ANixADsffmF75E8ggzQQWFMrK3p3d11AMF+QJxJaO+B6LOJr62IGlS1IwaFx3DvmOhku1EewJRK5LtqVZZG9GRqK1KiEhC0BLykb5A862E0qfAehHix2NvGXUaQcXX9HtdzjAXhCxZvrtSkLNk+677iBZ5ea7poLkIVG/UqCUKap9HaDrZh2Eqf5t9EYuHaNWvvCAoJg6jHppuJMMS84Cylkw80v3yYR9t7pHfx2y4PobU6/3fXS3ndVgIajOiDpFiJ2jRfl3T9gT1EvQmIVauYtQkDTMxIQFBv+qriCr4/XlKmisJnI/6tFr0dVm7NxBJUQfZVgqgrJHnBgPdMUxfxMJhn/ss2rqIWAnFLSKNSl5LzEDUWGClokxQb/UWepWKs2wbovZLiqJ+Hh+XS9GcEBxt5eeEWqVr006kQMm3jCiBzwVlUF7OT3e0R4bSsYuGLV2DIRY88ThHZEAm+aaX19Ej0iXFTYjHSxA06GF8g7Z7JNXngq65UlEuvSBSMKLlCQFN7z0XKXQoxjB2V8pjTReRzXiOx4zRiFK6Utwp+7miuttnAGY7WdOEBiSb/ocaphTgnsLPk+bMTqKByyBaqESTE8Rpj4AGzZh5mO4fNWdOGrvyLoW7S6GZTFiM9ML7SArNuMYpjmBEzhJTYNgXrEm3pW3A9BIan3Racr2oMa9Pji3meE4V5SrsUdoHhisJ2QMpfLsDoerp+B7NdgEzyBpA3F+txuI9rWnQMEzkGPOdoFPe7B+fzvdoCKPlveozxcMaYYyu4AFS7SQ7MV13yktRZXqPLSVuwT6IlRgmMaNx2B9z0BGBTE1UagAiKmkaiWyQ61FR3gvSJYU+kpkIEUmMaIFjr90LeyRZebkOBd3WI6rmSkG3EsqVtHHyflTj+SaEUWeXjkmOPYwo1V6jGrcRNXkpszEhZIJyS7MxahIHzzDNxsiIRM9TPsR4BsYmytZG0DfkuT7XQt3zCCXSBrLWMcyySMOUnTadi5mJkpGZ4i6EfSHbDEaRtQ5vdPzckjXKtlHbnkuD6yp5vEShCGU0mduMKOLDeIlMUENl5Vh9odGdI8T7Et0y5HJ/am5w4c/oOLVNaHSMsniAQurBSaPjAtp6fBbROy0GN7qx+EkuzVFCLGOmpN4N+DobtYfKB9lWYMyr1L2sodkOEHWCQetRQxhyg24tvjCiVbRenFCjgU2K2gilQXVOmqVE7fTyeRxMiu5Ix2xQzu1jOzLJW036vqBU/Bzfv0/TfXhQnTRsakiC14CyDj+p9iikUiQDHGXdXouZZ5I5WRViqhP1jWqI0SRa77eb9I/J2Obhzw8RxIfRJYMdG7O3kMdEMU3aSuei+2tEJB/eF/8fcyOTNhIEdXz4s7XSmIKgq7DfH2Nkf/gxaOPDSBH9Z51g5W8/pvf4Czq+hq/oOP9Fbj8TiOVHF+G3/uG/9xPf7v/6r/2Dn/qxwc85YpkVjsO/dkOZWd7czbGD6PWUluzE+1vpRuwyx1zuGNYlD41O8nlPtylAB1RrGM68mI3EJsJsteSyzSz0kVNYeNTO0OcevBqpp7rX+DJSOzQxG0++LEMlI2XVCaKRH7a0V/VoPGK2Gl+J/ojKo9YZYW7JrnLskaA9zAe4zwWNCRCMOJL6Wl5TDUrcTAOYjZG/dzLBHg7deEzJQAQQqm2ke/oikJ01gm7tpDDzE4fqNaHwZMsMV3vJt1sZ7EJQjvxOi8NpRCA47diWpWjVejV+2A9zxIDDCTVUD4r180jTbYXeq6zCzfx+TXIpSM1UYc8GTG1Rn9exCQsMByoaSkTaaYCQyWR6mMnxuzKiE9GcJFEpgfiFL2YqKUjbHjjye4MvgiAg93o0ukhTZZ+lRlX2PTmmjrl0W0EBhkUsCo1QbwVZQgxxokunK2Mj0KpIDQ24ep/9FXcTdSIF6TCPhimdYph7to80+Raak/2HdJrky8AFMUKJGXIJpfJGaMT9QqjFxVphJ/J8ZWV/pViV/WrP0nAkjEHtIYWWt6CCYpiGuB5qpEwLJZiR8it5fXEfkplKbMBcJc/PGqHmahvpYKUcW7ZOFDJZ/0Q9dFW8/kvRr7k8fRHLWrs6jPl63dEeyUxDnbdMarzQuF2tR0OcfiHDKWAcDmgrJyY1fYmqGLI9PTQNcLKdNJ7JkMVV8ng7FerjMN1TzvuFGqnS6RjyTaTxdmngs9cK7gtIOQ+bZ4psK4OvYPbU2tHgJDYOrVZjQ9+cPxhSZYwNlx6i1quQ69NnjMWX6WUgI4Yjieonr9OevG20MxbMXg5IBnIy2EjDg6zd68qKNTHig7H5cLUMCkZjm2HfUCdKrfIR5YtUV+Xk+nGlDB6E5inTtaCluRqm8tlgK0FDTCvN/jDVo8SgV/v9l6GWGqnItlZ7qmie6JpqpJ7m8dx/1XBlbJKswWdIM6z215YrFa4lFtGMFPRiDd1CjdefrE/6nNX7UPoB/Im4laZB5zARXZnQS6URMl2iP+6vkWQik3IgXfmgaw7APDVR8f9oVjNShSMyl7SLkBquaCCj1DiQAHm/5rt905uQQ1vtDWeyRo3UWqH7x4FZJu6rqVlOAwdg1HkGo8ZjePg+SEY1Kii6QyMU2izSRl0Ycx5V7I47k437rHxgWGTR6ElhKz3qEl2p8YUeh3PikqpHxDRdB8PMSHZkdBV2ZUTYIGrviM2fxtYa05lIv39gvqOBKukn9p8ZDw2pHiKV3kgWY6KeBpNLA+sD2c5JA9wZXCHXvz8oxm2YXqJ0RqOc0ozrKfrHqOu3ARVzJV1pxgGYq0wcjOyzIdGiV0w04Ycurrp3IxoXco2KDrBpQP7QBEjlQmmVGsfjpvnYBI+3RKGOAyn/4LIet+k8oZIvwqR1RClUN4ir6uAgN9Icx+Y0xAYxxPMR6kIa3bLYD0eS/lApfJFLg5xuiWpqonYy7XNqNsf3XhCk0TrAEDKDepCDKR+G8fHhwb4/NOjxHvJ83J4azYAerFOIyGLan3S32v+c1nOksqf9jc1mSChobG5V/P+tJvOrDrA+SDP7593+EsSNBP4KsfyZvR1/8yx89A/+Q+ZFR+ey0f101UrsR50PbPucKrcMzhCCwnnF4AzOaYY+o6p76mIgN47r5YzZpEUr+Oj4Df/7n77P0fGGu7sZSgXOT1fcrqY4Kw6n9aSna3OU9tTVgA+KYTAY42nW1RgjsltVmNKhgNm0ZbWuR5fSrLTYPsNknrywNLc1Zmpxq5zypBE30uhaaxv5sNCFUB6DjzmUVpNPBoY2o5r1dNtC6HStQZfyOKUDvjOozBM6A16hp4O4juoQ6XxIELwOFJOeblOicze6nyodhLbX5HGyZ2A2SOh94WS7iW4YQ+OBMUoiNcTFcSturE18bYXQAGtH2MVtFNK468LJPg5KsinjY/VswK+F/oiK4fW5uG2GqYNBaFpi9mIEwZp5yNOHtpjghCyMX8jKyaDAl/Fxvd4X0k0cMuQyKVVOjfrUdMyilQ24Y4teZbIdHTCrGL0Rv7x85VGDjohlGBt+FaM4pCnXEVFTDAsxmEkGKkJbjMVjJa+dXBG1lWY1OZXuNXhKhhQ7g+kUQYuJTbYSy3k3k4iRVJSkb11fewmXj4wAVzxcr/26+TJEDZwaIzZcEffXSvSHL4QuOhbKOpBtZF1hX3TrnjEGZJhFJEsxul9mjaI/kqY+OYkSBLV1E7kGkhlHovP5XPS9xf0eRQEZUORLHZH0QNZKow2ylj661fqozR1miZ6ZGogQw+elyFWW8fhScz5GrfT7BiUF2XfHnskrHUPk5ThMJ5rWxIqws4BJ+tbYsICsk89BkPS4ToZYqAJBmnSfMbqmgjAwinvojmTQMbIR4v4nvbNpGVHmUQ8aWQhpGDA22IVsM5naJD1parJtLfueXDhNF/c1HktyHE0xIPlKtpka5oRgjyh2pDgmbayO5/gh+i7nMF0zxKicfcNLHFDYWpr3tH3lGVkBKuzZKol1Ekxs1sL+XKRb2r+Hg6wR5Y0siHy7R6iSm2tyIE0a3nR9JIQ+IcKC8qo9bTE6Wyamg2yYMapEkHZBbiX+Q5Bx5ffo/Yj+uj1rIu3jQwZDcg5NLAE9pCEXIyPi4fGi9tdLuqUYk5Gp00f0N9ah+fZBBIjer2HSP4/OomF/rokDr2GqRu2nLQU5T+6taViTIkfMIJ9XI6qf6uBIb+0X+xiWfCefo66QNehn+8/J0dUzIq/pvKT91G6PaD90nJXImD0qHrQi6wS9Ts6rSSeb9OLjtR+1p/pBk5saW2UZ3X0TS8B00hS6Usf1FPQVGLWiIVN7jWc8xw8bt9R8pvVMOuZg1LhtNaLgex1pthPEMrFLHmp/E4Mhub3qYY/OpjXVQ9pvoa/6QgsltRSkWTuhjbpSYwYf37dqdIxV1uMj6kwII3IsJ1uh+kj1jNrSEJslPQhqqJwfaa2phkkI8BijYf2efhoiLTdSU4m/jxEiuSCvaBjdWoFRSxkdYx9mQY6oZDLReYiaej+eJ4wZ0dCxkX3gHKucl/u9l8el1x8kN/Otm/dvm/M8bIRTM5rQVdgjiBJXELcRP+fSfV91g/0xmsu/qN7y590VdvHRRfiN/+Y/+Ilv9x/9rf/yp35s8HOOWO6WFVd/cMHybj9914N8EWwfK8IbyT1rfaRdniryVWDWyJdZ1gSa0wlsAnfPFCff97RHU9QO/ujglFkO6r7k0Vqmp92kYj7A7lIxfRHweUVJMvlQFG2g6mVif/HC088LinVgFqeP20cKdVdz3EnWl8sVwZRjlpmdKBZLL1NFG2jO5tRX4pqXiqKsgfakYPJaqFPlvdB0XJ5HalZOdxBNYWKmWh7d7LxRlGsfKXCKrE1FiexfH7UmPlNol0cKXE4ejRjynWf9pMTnUN5L1l19raNFvsEMsHpHc/BDh60ldP7u64b6KlCuPNnW0x0a+vkUO1HMv3BsHhvK+0DWBrqFoVx6dhdZzGaDYpWhXaBbaOwko7rxtEea+lY+dIdadEqTNzIVtqUi6xTzH7VsnlW4Yp+P5jOpJodaUa4DzYli/rnDF2rMmSuXgWIV2J3lFFuPywXVq689ysP2wrD43NLPNcXS0Z4Yslb2Q9ArFTUZgelrWL6bc/R9oZD0MxO/2GWam6bxPlOUKzdS4ppTafbyXSCYMKIetpRqTVuZzHcHiukbTz9TY4RActRzhdD6ypW4+rlSsT3Pmb1yMl2uZMpe3UrQ++Yyi68nBWo3V0yuXCzaPNvLjOrejc9ZP8mpbx3DRI+5eLOXA81JNha/0oAqyqU8Lm8CQx3Id7KWthKUc/rKMkx1RICEBpd1AVt5oVXWsUiMdEAz+Ghpr2S7W9HQdYeafBuEohbpfQnJcIWmPdTMv7T4TOiBysm+uCKMVDfdy5d0e2wEyd3IJD/bOdrTnPLe0h2YWNzJNQ3QnGZMriz9Qqb49dVAd5iN7zttA1kjToiEEBGhgO4CKrjRSMVWUkRV1wOukn3sDjO09REdU9RXFtP7kbKX6IHby5zZFz3Dwsi1tHNy/JFql28cdmJG0xJXaLmO156QQXnd0x8VdAea6cuB/iCjvB1oT3LKO4srNcNMi+nPVArgfCuujIBk6TWefDXQnpWRfhyobizdkeTtmc7hM40rNPnGYqcZttZkTSy8tIqGIorqZhA918TQzzV54ynvJPdO9+IEOXnd4UqDKzX52oqhSSOB9MVdj4tojqvjcXcSDTDMjNA8a01xbxkWhnzt2J3nVHeRNjlIHl950zEscuxEEJfifmBYSG6f0G1lGyDUSm0D5U1HUOIs2R/Ie0Led/Ler990bJ5VTL9o8aUhv2+xixJXGvL1QHNeUt4P6NZhpxkmUj99oSjvBvqFfN6XtwPDLCPfWTF0qc0YhyGmQE5yAtd2pFMmUxRf6DETMDUQehCa60O6uh4CppXj648KlA3kqwEiIqWdR/WiP0v01/rLDXZRYaeSTQjgS0O26vBFhptEl89SMgurqxY7zXGFZv7ZEBtWOQ/DvBBUykq2ox6EejgsCrKtlSZgNFBRgkwNbkTqfJEJ5XOaY1b9aOYi1MSoj9sOuDoXOmdpWPxwiLTOgG5S3mDA1RmzH7W4Oheq7CQju2sIxuDrDLOL23FS8PvCYHYx73A34GfFSP10k5z8eifNRgjYg5LJzoL3cRsBtCK7b7CLCj0IvVM1PRiNL3N8lYFWVK18x/giw2w70SfGXEUVAm5eUsR1SpmKhADWMVwekL9Z4+cVyaRGOYc9qsnfbPDTUo5j0+2potNKzGziLeQGc7/BXhyINvFmg1/U4ALFriNUOWrbRkSwkDzH+y32fEH+aomfVaIZnJSyj6sd7miKXjXksakKxkSaaU6xafGTCr1tCFVBPqJxMeex64V+Gn9XzkPXE6b12FSFTLbHYAmHc3k/bxuhj04qyXvseozW0qClhkopcF7McjY70RJOKnTT7RvFLCJ3dYnetfLc1Ow5H/MexelUVSWh7VBZNjZgwTpB5oxBZRmhH+Rxmy1qOiEMw0iTDQ/osm/pD7WK24vfEyljMi8I/iGl+KvTsYiaD1b0kunuGBuiHjScYwP4UEOZIkHeQmXdvlF9SL39yu3nF+b6q9vD2891Y1ktOn7zb36HRd5S6553qxt+o/6Ef7z9Bp80pywyQQy/bA55uVvw3mSN9ZpMezSB+77m/XLHzua82c25/DsSTfG3jr/L//zyW+TG8TdOvs/OF/xwd8LHd+cc1zsOi4bDouG+rymN5a6b8MO7Y+qqY1G23DYTfuP8U140B7w3veFle0DvDS824s8+K7oxM/O43tG5DOc1uXHshpyn0xXXzYxaBS4ma+7aCXdtzWm9Y92XPK63eBRfm73hn968Q2ksR9WO17s5ISgWxnLX1rg+52i2pTSWzklO5q+cfMk/v33Mpis4W6xorIzKtQo0Q85h1fCdLy5ZzBseLVZk2vNmO+N0smU7FDTRBOj85J6rz855+tuv+f7LMx6dLlm3JdtVzexvL1lta/LM0bc5X3/8ktt2yiTvuVseUBcDfsioZ1uuro84OblnN+T4IeNgvubFx0949u41r24XvH9xzaYvyYLi5uqAi6evuXl9ysXFNZ/eHPPe6Q3f/fySo8fXvLk+pCgcTw/v2Q0FJqyYZIJW3+1qqtyy63Kc0/SFZbuusBFVTtEjnQ6EAO2uoJr07K6mlMcNQ2GZVx33f3rGwb/zhtevjrk8W7LZ1WgtRs9NG/NHM0/f5dQnS+bAesjYNiWTquf9oxv+70/e4fnjG5ohZ9OW7NYlRT0w9BnPzu/Y/aPHbN8fuHx2y5vrBYtFw/2bOZdPb3l9dUDoDLPTLV2XYauBzaZC6cDBfAfA/XJK8Ipq0nN9W0PuyUqHXRXc/7onqyzvnN9yc3fAzWDIcses7rhaTtHaM5u2LFcT1oXlZLHlejnDvjAyPZ9a8smAc46itLT3FdmdYvOkJT9ZcnVzwMXJkjefnKKmltAbqsOWoc/IcsvQZ8KyySQbNdwX3B4JxBacIOOmcCjtGXaFUNe9ElQ4QH3Y0m5KQeEzR+g1ZmJxncHcavyJReeeLLc4a/BeEe4K1FFHWBbc/jqQO9RG0GE1dbDRqAHCcZz29prZyYbt53NCQsm1Qu0CHHq4N5TXmv7Y4xcRLVdOEPLKCeOgNcyO12yXNaZw8EWNnQcorRj+AKH06NLBjUCE5dMN7YupUOdzUDuhoWMs5i4DH52otVCxwtQJUj6x+G0G+cAbp2EA04BP9L+Y46mcbEOfdvjbgmyrGSKlPeQBTI7qFGqANzoXM5hzQ36tcZVc26ZTDBc9WelwNyXKapTXmF2in2rsQqJXXC1rZ7ZFjD1RBG3AgN5p3AJ0Y8i2iv7MC+q8Nfi5wyw17sBgVkaMg+ae/NZgZzoa7giiS6jEHbVRtKca0+/p8UFXQjHPxJm7fJ1hJ2HUOYLElDSPhJFgOhPRpEwQ5zpEmr5Y/iZHZmWrMavTG3ENbn9JonFMzE61E/lqzVeKchl15QW0p57yTuNNTTBw80u16PlNgWnElEiFjOpK0R8KSyFfiyFRvpQhWtZko4a3PzC4KpBtMrJGjZVZdxyobmRwk+0UKuRjJqatBW0XA5hi1N87Ifow/TIIndnu9f7dcSEU9jjctBOhSFbXoovXVrT/5V081t84eqAHz8Vte6XwWYXPxcinO1ZMXgnSePe1GVkjA7D2LKO8VSOamm0D/aGK1zQkJ3fxJShH7X5yAE9ux9kuonbRAVi0xPXoD5A10fm4A9NVMuyq45DOFTL88Yn6HEbktT8sSW7A5V1gmNV7tgJCVa3uA91C0EZXRWfkIJr25lQzuRJtaP+NMtLjFdWtZ3dWy3ZXnuQa2x9MxkEryD7mG5FQuCIZA1Xj/g71ZFy7h8jqMJUBqc9n4tsQHYCzLjC8X8nwwwkaKYNSRfaoxJYyoLTv1mPUi+keMEksMdpksdd2fzgh33kZCG19RHsXUWurxLTn6zOqG8vu8UWkPstjfaFwxUJo2mb2FsLrCk2xtLgnc1yt0cNi1LUm5FrcbvWoHYY9e0U0t1piVVoZHOVbK9fTbqB5Psc08rnlin0ephn8qI1Ngxuzs2AO38osDUaR37cSK+LiOWQuAwEfxuGGmxWYXS/NfxxCSBxJ/D0w0mhTo6ZaCweziGKLFjbpUUOm0c2ALzJhYjSDDCwS+vnjTHyiMU+oCt4y9ok3nZDMJAVIqOoDqm6i0gbvRbNpLUmbqQDCgxiRt5rLh9Tb8Gfv+zG3t1DM/x8ay5+FW0D9FRX2Z/U2/+gynP6n/zEAl2dLlruaX754wR+8fEJzW3N0ueJsuuXj7z3m8MmK5WcHXHx4TQgKoz27rqDpcoYuQ7+oCE9bLk+WBOCs3vJHv/8e0w+WZMbR9jnDYHAvJ0KtOe2ZLxq2W3FW/dqjN3z/9SknB1te/+kpoQhMz7d84+w1//Q777E437D7+BD1bIcPCrfLUDuDPukxP6qY/OId212J+v6E4Z1upF2wzmExUE87mi/m6LMW84Ma9dGGy8M1N9sJu03JYtGw/ZMjhsue07M196sJ2fcmmF9Zsr2ZSPC9Dujck+eOdiPfUjrzKBNQX4h2sX/Sk10VUhTOpUj3Vr5BQmtQvWL2I8Pmuad+pbG/usH84Qz/y2u625ryjcF92OAGTf5ZSXWr2DwXbaZa9HBV4qeO7C5DPd/hXtWEowE2GfqoJy8s2e/N6f66OKLq+4yDD++4/+ER1RstGXfXBfrZlvCjKfbQYTYa/aRhWJbonSZ/uqW7qzAbg5t6yDz5VS4UzW2M/MgDp7+vufpNJw6hV3ksVhTtux3FiwI7DWRrhekUzTsDeiM00uF8gEGj2xgzczow+14hRcChFKD5GlZ/bWDyw5zmGy3TP67YfDBQvs6o36gxdqA5D0y/UCx/qyVsMlRQ5Lca5RX1a9g8j3RWJxpDNwkU94rtM0f90rD7esfsOyXNhehrfRkIWaC4NVIIRuOp9txz+F3F+jnYmce0QrXtTxx4OPhORnsqPQgfBQAAIABJREFUtMfNu5bF9zJ2jwN27ihujezzROiiw6HE19i5o/4yo1xKbp6beGY/NKw/sBz9oeh0h4UUyz6HyUtFeyp5i+5koPii4OSPAy//zsD0uyXdibhOugK271mO/plh8xzql4phLnTO/iAw+0yz/mggu8uYfapYfU0Qy8UPFKsPhLo66gqd5Cg2Z4H5p7B+V9Z9/ik0p2qMXEHB7qnDbLU4VMbsxvIusH2i6M4tp79nWH0AkxeCNg9TRfNIsiAPvyfaLDHHCtx/I1BeyxpnbaQjzhWmleLSRSOXzTuew+9IU5bcUE0H26dhpI1mOylQD78vBdryAwg5+6iYQ6EFH34Mq3flWIoV4COz4IloXes3UqhsnkF1I3rgyStptLojxe6RZ/apHnMspy8CN3/dcf5/Gu6/Jvrc+krQ6P6AMYMyGYFNX3t2Z5r2TK5RWwtTRJgHgbtvaoo79nrQ54HjPxbkXTnoD6G6Dqw+hNM/CCw/EEfN+k2gWnq6uWb3SNBxV+zNwqZfQHUfuP9QU90IW+XgB4H7r0N1LcV5fSPsg/ZUXmv6ynP/oWb2RWDzjmL6pTQOk2vP8j1DcS9sl/J27xbaHyhmX3p2FzpmSYbRoTNRhVHS8HWHUjSUt4HJjcMVms0jcenMGy9MjTvZ19kXntX7mumXwlrp53DyJ5bNYzNSiVOOocSrKGYvLT5X3L+fRRMuoUDOXjj6qaY/UEzeSAZiMiDLGjFg2j5SZC0sPnXsTjXT18JM2J1IDFSxDiNDQFtYvWvEwfRKjsW0gduPcrSV82p6yV8sVoFiI0Xy8r2MxaeCWPRTTb9QzL+0bC/kmMqVp59p6lt5zOaxYXIljIHthRmP2WexyYo63fZQ3iPaBqpbx+ZxxrBQFPeSWVqsQmwshZE0ahq1mFslffD09YArNN2BHvMgTR8Ebb633H9QcPBJP2qqTefF3Kd1LN8tmb6xbC8y5p/1NGf5qCVWPlBfWVwVdZF9wDRCNR2mOjqeyudScd+zfVJR3Vl25xkHH2+5/2iGGQKzz1vsNCNbD2yf1VTXAyoE+kU2si0mb3r6eS4uqKcFykJ519MvcnwpzVt3YNAOZp+saR5NyRrH7rJg8YMtu8c1rlDMP9mwfWeGaT3FsicYMe7J71raxxPyVUTXck13mGMrxeyLLhp6RYRcK5qznOmLDt1ZhsMS07lRSxkyYR64iGjPPtvRnlVUr3c0j6ZMvtjQPJpSv9wSckN7WlG92eFmBf0iR/eyb8OioLhq6E8qijtB+YvXG/oLsT3WnSO7WuMPJrhpHs2I5JpsTwvKG3FxTccXSoPe9CPyOZzOUM6T3TcMxxO03Tf4otH0YyNnbja4oylqiPElVYZet4KSAmRatJm5EZRZa0KZCTJbF2Mdm5BatW0J0dUV7wlFBiGgOguZQW128veuF0R1vSPU5YiiKuvGjEoV6bOhyAWh9V4cZL1HdcI2GJvJB0ZE8mT1NqXVGGnkshi/M0QXV6WkaRx6cYW1FpyT5jI1kA+3EZ/31bzKH2sc9GNuIxob/J+l7v4L3H4WqLDzjy7Dr//XP3kq7O/+7f/ip35s8HPeWJ79wkn4t//7v8svTl/wWXfMaQzIK+M4+nevP6IwlqtmxjcPX/Pt6yd86/RLrrsp06znUbnkk90pH81e8z98+7f4t37pn3Hdzfjnbx7xN598wvdW5zRDzvsH11yUa/6vq3f55eMX/B+vnvOrZy942SzItOek3PLx/RnP5ve0Nue2nfDBwTUvdwterhb8+uXndD5jNVT0zvBses9VO+MXFi952R3wv33yAU9O7zHa882D13z75gn/+qPv8t/9wW/x/PENT6ZLfrg65oODa6w3/OGbR3zt5Io/evGIv/HuJ3gUf3T9iEfzFYW2fPtHz3hyfs/drkapwIfH16yHimnWY4Pm3ektn+2OsF7zbHrHTTdl3VeUmUyBbpoJJxEdLY3FBY0mcFA2LLuaq62YIs3KnrtdzXvHt6y6isvpirt2wjTv+HR5xK+evWA5VFw3M2ZFxyzvsF7z5eaA+82Ew9mOdSNj8tPZluvNlKeH99w2E6zTTIqBdVtyULf4oHg8W/Ld63NCUBSZ5Wy65Yv7Q4bB8LWLK7ZDQWszdl3B+XzDsq24uZvx6HTJsqno+0w+j4KirAaq3JIZx/PFHZ+ujri6mzOftkzKntv1lINpw+A026ZkPukw2uO80EqaPmdetyyKjlfrOV2foXXAGE/X5mgjDXzTFBzMdxgdKIxj1Za0TSHIcrx/uakJAapqYFr23NzP8F5zdLDFaM+mLTme7vj8xTHffPcln90doVRAq8BgDUOfkReWPHMyMGkLssyz25Scn66Ylx2v1zPKzNEOGbOqY91UbG8mnD5a0lvD+n5CXg/0u5xq1mMHw2K+Y72tMCagVCAERbsqmR3v2NxNUJmX7Ng3E0LtWJyI3e5mXVFNeoY+Y2gzstIRgNPDDXfrCd5pbJNRzjuMkaiX5rZGVVELogLBK6aLlrYpUFq+bIwJ2MGQF5a+y/eDzkY0uOawl4FN1NEGJ3rOrLaiVe4M1aLDDgaTObrrGkqPqcR5xjUZprZ4q6XJn1pM7gX1dNKwBStuFeYmJ1x0+MGgjKeoB+wgmu3dqsIUDrssRnMv5RTqsBe9cObRmccPBnpNtugJgFvnkq+be8iD6IQrh35TSjTJNiOUTrJ4c4/Ko146gBo0+b2mP7ViElbJ1F1FLbbaZIJKAuiAnljUqxJfB8rzHe1dJXpnGHXFWI3q1fg8NSjZrg4Ur3PRv2oZ0iinYGphk+11zV40wKFymPtMNL9RTysQmRKdauHRG0OIjtv5nZac2cpjGsn/JZfBiWqMaDWjHlm3YpoWStElm23UTYeYh9spXOVH/bQ7tOiNQXcqxiQEGSIURKMvxoxcHSNr7MJh1kYanTwIbfhGCxrYSHNvp4K4Fvcy/EkRDRIpIuhOtlWRdg0hD+QrLUOWiQywJJ9WtNfDPBpuReOdpCtMGcLwQKfYCCrrizBqDtMwRnSjcT1KcZ8u78RczEaHcmA0VMuaOPA6i+jVEJHCgnEwl143NdG+FA1w0nkn/WsacPSLvdY26UF9GV3YZ3Kn7iTypT9kzHFNxlOuiJEnlTTvw5TRcKmIWbvDBKpbcRlHRcfwqD9O8SBZI1rT5PqdkMykaR1RTy+RK66SRrFfqOiMzWhIVNzHTFctTtsSkxFiRIwMFBINXvkwIszJeTtpS0cjnujCW6zDaApFSMccnarvw7h/rhR9ZLkUKUS+jTppszf5SprMpEnOt3sn62T+lbTA4/UVHWWFKixrX0QpUL6Txruf6fE85k0YHaKHWlFsveTEZntHblsp6ls/DkWyZBilRNNqK3msj3rSJBFR0X1ZpCyBodajW7PLRZOqrKxVsZZmY5gbyjtLe5yhnTglp8+05Pqd8kCTDtNONFnrR3dgW5nRqdqVOiK0D5xfo1GVKzXZzo3n2bQuOt96cQOGsbElBDEjguikGyR+xKixUQ1Gjz4MwWhMM4x041A9cN+NzepIu00RJfAVum4QOnCZkRx0VWffairVYPd6zHhLOtsxTzO5zabXS68VHWpD0+ybT22k8Uu61eQk+9CJ9iFFNgT53Yd93Mj/1y14aTD9j3GZ/QveflYay1/7nb/3E9/uP/7t//ynfmzwc95YLj66CN/6nb/PSbXlB7enbLYVk0mHjbTG49mON/czjhc7qsxyu6vpunxsAEKAInPcX8/E/OaulC+lg5560rG+maJMoJz2dE1OWQ+0y1IMcYIiDBpTOVwjJjmpUFQTizYBt8zR84FwW3Lw7j3bXUlZDWyupqjCo69z3NyhaifUmutiRNiymxw39UKFC2DmA26bC/VtOqC/rLCHUhArq9GHwmcKbyrUWYdb5WDCvsgEzHTALYvR1ZUsRPdaCJUjv8rFQdZD/TKjeZKKVLGYyxY9fF7jLnqyasBe11I460CopDj0E0+2NIRnDf6qIuSefGWk8PJicOMOLPmiw72ayHNnjuLLnOHQx4I4yDq6B0Vtp1HHHeG2FNOO2pOtDfbAordGzGIqKfpMo7AzP9LCQhEwGx1jO4RWp3sptELpMfcZpoXh0EvBWopDb35vSEY1AUYX0uHUYpYmFlQau3DoTtDL4l7RXHqKSFtzZSDfRJfVWCCixEjG7DTKiuOhncjO2rmTNS091QuhH/ZHnvJaMywEXesPwt50KEB5q+kXYW/woKPbajI4yaWwKpaCOgr1KuxdAz2j0Y6vA2atxcdprbDzQHGn6I6jPf8kMHmp6Q/3xh0ShyP7n3IW843kvaXCPZmPDHMpKLtjKVzVgzgGV8XCehDkt1jFzMZ4TkHMdspbxTCV/c22iubCU78RR8zUICB/RndSLO8e7417QsxntFMpptUgBWt3JNRHbaG5EPTOTmKhbJEiMjmzRgdc0aTuCzU9xPWIRXtaI6HBSVGcoku0FcpevpXX6BdyzHYC2UYKRbRE4aQoHu2Q4j2PRXrMzBR0OBoLedmn0cgmCE3RNIzxKokml21igT1IA5AKU5Sgtc2pGg11kitqtkUQy22MSLKRDtfFKKRoHjTMhBqZqGwuXo+jUUq7N5zZXcjrlPfSHIxxI3ZPW9VWivzUQKRrO+nqbaXGNUkup+WtZCdmUT+cYn+KpTR4poP1O1KY59tAvg20x5KjmEx/iq0gjCkqw5WSWZhcbX0eaYwXojdPYfXJ8GZ//PJ7vg0jSitIojRGyaBmmCvqN57uSFPeeYapGq/nYhWjfuKa2BqJ/IiRKNlO3lebJ4r6zT62RYxsZP03TzXVdRgNjCTDUTTc+sGapjgVlyfHW8b4F1kHua9Yi+7ddEKJtJV6QD0kNh4xcsnKz/kuiL53I7rJ7iBpfxMdNYwoZR51zOncjcY4VraRsiyHiaLYeKFlzvT42LF56aWJs6U0QCbF6WSKfiafDdW9Z5iIxjf5EWRNGKOVipWjOc2iaVV4YKgktOWsDUITBYmSmejoGJwMkxzD1GA6T3k/sDsvyDeeYabHhkr3e+fUbOtE63wvmmciRbK8HQhGUMDirmc4yNGdH/XkPtdkjWOYie45NYq683sjnkyiN1SMXtGD39NHU/bjSnTQ8l70DHOJeDGNl8iXqO8FcHlEOndWGirrcbnE0thpNjYsPlMUywFbG7KdpT8oKFaDaImjBlv2KUiTZjSu1BQ37ej2aue5uNsqhWktwzwXjXznCEajO4uvM3AhHktqduNjHtBXH8as+DITHX8ziE61ytCtFUMeFeNSUmyKMdJIJrppbOb8JEe3D/K/tEbtxGwmOcaGIq7p4MRcJ9FUU2MYnxcyjV4LmorzI4V1dGN1HtpOHGjLQjSjCam0shbpfKouoqZJW/pwe6kPeEsj6VFFTsqhTL2CyjJBQIdBflZKUEylpeFzbkQlx0iTr8SPvEV7jSZFIWVq/jmUWADftn/uY/7fbj8rjeW3fufv/8S3+09++z/7qR8bkMIgfj5vgzP86LMzvv3pM85nG6q6p8wtmfEY41k2FbbPuPruKS9uDtisamaTFqUCs6oTpKTL+eb7LxjWBWpq+Vd+5WPOj2MI4aA5OV3jrCZ04vZ6fLHCNxk682S1pax6ykVHVjjUOuPdD16DV7jOUJ/vODzckl/usF7jv6zp2pzF+YbgFPnzLYeXa1jl1LOO6r01Zj5gakf5tRUh95SHLfMLefzh+Zr8oCM4jT1wnD69Z3GxQS16XGdwTUb9zhq3zDl6skRvDOqwJ5sO5LOeuu55/2uv8GXg+Ok95UFLebHDHAn1djixHD1eQuFpnliqk4bZ8yX5Qcf8co1tM+zZQP1xifeCkBRPthAUR5crwmmP6jTFhyv8dUmYOCafywfo/NmK4tEWX3vMbCB8NiVkQXRiVklTOXFUhy3F65zipJWYFx3ABLKLHeXHNdnFTjQEs0Ga44gGhQ+3uGOLLzzuaYs67AmFNI4UEsDOwQBq7/CaraTqzt/d0J9F59VDK9EuCobzAW9gOLbY0wF/1kuDHJ1s9XGPPbaQe3QrSELzkXyBKK/Qz7eUN3p06uSyw808plGU12LkY09E35E1e2Rj8qWheJPh6kB37jAXDe2Fw7SM/4M0fv5wkIiSqcOeDvCkwU09rhDKqyukibVPO7qTgKsC/Td39JcDbhKwj3r6RwP5WpFtNZPPDe5Rh2kVzfs97knL9l3L5KUUGvUrze6xw068RLLkMP0iFi2XrTTUMy8Fdi/6sP4w0J57dl/vcFVg90GPn3h8Hqmvjzp0rzBNzO7MIVy2DFOhGuZLaQJtLUX69omne9KLHs2Dcort+wPFCtrHgzT+vTi4dudOmtheXDK7c0t/aimWMmTojrzs32lgeNbTvDNgp4Hpl1rC5oM0x80zG6nLsqanfygGPL4IbJ86ipUU0P1BmmbEbR957DRgGqG3JsShPZMCtHlmGWbSfGVbQYdMK43wMA90J57NO14o0y+l2bFTaM88w0xoxdv3B9pzT3GvaM9c1PIFuuNAex7YXYZRa+Yq2D23o0tjfyANQnMuurzkjLp5z9Iey5AEhGbpKkG80v76UuI0XHSCtTEXdvPcS07nvTSr3ZGgS91JYPuOZ/PcYydw/62B1Xtw/3VBntpTabi6Y0F42rPA7nGITbQgSN2pNFa2liZ2WAjV1laK7bNAtgtsn8g+yxoKDXqYShbm9kmgO5S12F0odo8i6tRJ47O70OMwYP0+DDPF7TcNrmKMQRgWcr+r1BilsL3UpHD69lSxeUeovio28Kn5t7WiOdFsn0hDunlHHEg374jZ2jBVdEfSYNgaNs80w0yK4qChOdNvGTK1p/L6/YEanWWTk2/6e79QbJ5omhPN5qk0v825GqOTxERLYadCk3WFYpgo2hPFUMt+7C4UzZnC1orqztOeyONdAc2Jxk5lkGVLFQcmKmoFHcrB9pE0gP1CsX2iaY80tob2QLO9NGJgt/Fj5I6tZL+HqdBk20Mta3OgWL4nx9KcaPq5DPC2l0L/bU412wvD9lJM2vqZojkW5sLuVLN5rMe4l36u2Z0ZdmfRvTxAPxOTMZAs0cmraLw2F33d7iwbH7+9FKOubqGxtUSabB8Zto8Mw1SosLsTM8aQeAO20qNOsT0u0A6as4xhIue1OdL0C017aGiPDK7SbC8020fFSG3dnWpW75YMM3n95Yc1w0TTnub0BxkhU/QLja0N3cKwfio6PFvJfvcLQbuak4zuQFMsB5pTeS3T+dEFWNlAd5xjJ4Lq+VKe3x4ZXK2x0yw2bJphkrF9JLrb7jCnXwgq52rD7lGJaaWh9tE4yef/D3tv1mtLlucH/dYU057OeO+5N7MqszKzquk2dLcBuWUJhBACAwLEAxJ+YnjxGw/wBXhk8BvTF+CLgAR2W24sU93tpp2VVTnd4dwz7immNfHwWyvi3Orqdksuqyttb+nonrvP3hErVkTsvf7/3yRhVzqhspJjXrHjpDoHVysEJeErjeHMwK40xssadmXYcAg0w5Kjn8zI7ELT0Gk/INQaQUmEUk3usG6pIGzAcF7Crg2iklPzQo40aIomobGlRn+1AELEeF6nwlzAnlSISuL4gzWgJUJB7wG/LOGXJfM2bUBUClEp+CWNj2JlJmOhaJKB2+OB+1dyKlyjUQjLiq8BWCTW5YxKGs1CzLBQ9xdrhPMT+MsNxGjhz5bwF+sUSQIWm86TPluVLCq7YdZcCkHjotM14maFuF7y382KRSVA5DFGiKKAKApE5xCdp6mQDwi7PabMS5s0lgCEkojjODvCpqiT6D1iMi6KPiD0A0LPMYV+QBjtP/Lnnz9+tR/facSyef69+Ov/4X9NKsQustPfcuETNJ8b19Sa5NBogB3GLBbP9teumm3TM01FWoZK+1KgP6U+Kndic15dDvwG+DfTzh1d01JPMa4kO+rJul0mm3xv2CEWgeHu3tDZsnoIGFccv0joRO6cRiUYpl3RKTMYfgmrYaaN5IVHUALVg0f7TE37X1w77L+noVvqaHK+V1D8Ii92MYWMy6n7OuWHCS4YclD5uBCoHwKCYjfeF2LS9ojA4/MlFzeLV6kYS2He2elTujiFmlOsP1vw25SXWBzZje7PBDZf+ml+J1t3YEYCE70od5izbTvnUWBccL85k1K6iP5UotzG2V7d08GzP1UojoE0l9T1FgHQfcC4JF3GlVwQuXQ83pAmwzy7iMOVQnPLjrG0PI5M81EW7Dom+stTG/lgBPpT6pZ0T/dERBoP5OPpTyQ2X1rYpZqC1osDXWFdWgQoG6GPqXPdBzp3pqw028gpPw7I6Ga+Hub5sTWdQNVInZrukotqz1wzgMYKmSLFsHcJ3bJjz31xftQYodLxsIsfYfbsAtulmizrqWEkRck2vBbzHOkhQB883JKaHd2ye+4rOaFKoRDJeZddbV+KFPUQpkXl4u3IBYzJ2XV01XULOoYyLiFdg1sHu9bT2HNkie7Z5bcLLkK7SzM5wAaT7suK5z6kjDwaeRBZMEef9sEOum04h6qfg85josqNSwVlOX/Z8RKg/isqge5Co9gHqM4DksVIDrSP2TVQA3KMCIWYYzfA+6bcku61+77G5qcjfK1gDo6uph1jALK+K9/HwidULEa6jo6B1/maET8i5c4FzQgACJEy7+gKO66pbc45ecFImO0IuynoChrptDpsFHQXUezcdNwhRR3wfEtIG+ieW3HhrHqiPRmxebpfX/K6dbWEOfD+YGajTK8HzN7BrvSE5vi0j7x/CAHdOXjDhSE1iZGL3ifB9nGirNGNVgSg2FkEJaFbi+G0RHXTwS8MXK3pCvu8RPlgqU9Tgu62MUINYYq48CZFLAAT/c6bhNRFLryFD3ALw/um9XRUTQhO1HSJDZWadHBy8FCHAe605pgDiH61FsFwEc3mXEKpai5wZfc+QqQOI3x2m825iT5CtURN3ILFDhSvfTl6hLQoz2hVvidCzi9MuYQiAmJ0cKtycqsNRkH2DpBANNTNxdoAIcA3BRGoTCtMx/E0e1Nan2IlElK4HxClpMlKz+tF7wf4RQF1GOBXFdRhQKgN92VI0Za9g1+VkO1IF1OfkC0tIUbH/EMl4BcGsnNEtlJkRVgWk+lKSE6xQUuYmwPcxRLCesjeTmhZrA1/T/M06eVi5L6yeUuMCFUxjUWOROBEl9x6Nw3krktzoyFComYaRUQtXevC+imaIqwayGMPaIVYUDcI7xEXFY1l2gFhXUN0lsfeDnOxYzRiqSH3HcK6gTj2QGG4bc2CTPTD7PCaC4nCQPSJU501g4mm+R5S54lyTqY3WdOXkD3e+H4urLSaY1as4/NScr8Z8RtGxGVDvaJS/H9dskiTkq/rByA7ty4X3EeO+8jRH3k/jjTSyQV2HCGMQXaEFUpOER65gBNaMTMzucAKIRCHcaa2Ps3AzON+ikSORB2FppNwfsRfVKiFAFGVdLZNMSBTpEjSTwolZ+rqlE2ZzHx+vq7Ixj0/70AL4M+TSxl//n3fcSrs8kdX8bf/t//sl77d//vf/pt/4ccGfMddYX0JqP/kBrdfn6F+RQ2P6ti5tSu69Q0nEbsfRhSPpEKaHc1YYsrSyuHk4wnd+ugWxy5o9zygecUFjV0I9GcABFGj5luVFlQpYLtmAVhsBbrn1PYUjwrFnloMVwu0LyOKB8kA+DUmqs54QkOQvI2gScEbTmkC48tsIiIhB+DxRxKrr0jJ8iWSJTzSAoQmIMWjwP6TgOJRp4D1tBgrNILmcY9rMekpXJNMRmLE/W8oIACL1ywIMnXLVdSPbD+jKcZwFuFfKxw+jLj4MZ0Eg+F7mJ0nsP84QFqgfpdoWC8lmuuI9oWYNDZqVFMmX30b0T4XyQWP1K4xjd1XLKZy3Ev3jK+zK8bIxNTVr28YT9KfCzTvWDzbRQo/P8l5k0B7FXH++xzr9hN28s0xotgDd58YVHcRDy8ZGVG/i3AV0L4E1l/MtCfXACKyU56D26MgkrL5Calgx2csmosD5727JP2VOZURw6lAsU0Njy1w95do7OELomxB02SC4dF0ECzvI0LJ7nF3KWEOIrkCMs4gU+2GM4lix8L59b+qsfwKKA7s4HcXEou3LC4ef02iugWy0cnxhURznZCf50QXygcxhcbT6IRIQfcsYvUzhXIXsPuBwuIVsPs4uYWOQHVHpA0ROFwpmKNEc+vQn2rsP5J4/nsR+w8U6ruA7oIL4+Urh2Ej4I2CXSKdP95nu1ONzU9ZpHUXEs014AuFw4cCy2/jpMNhU0djOGGzY/sDic3PWOS1VxLt8wrrrzz6E6LKi2sPu1IY1kRX+nNe48//LvPTjs8lqns2OKKgMVLzlgVzeykxroqUiyjR3HB+zZEIUqZwBkNjF33ElC939xs0/1h+S9pjVIzPAQQbHCcKQWO6v05+kimtYi6kSzYiTBvQPjcTfVKNNKSC4L0jYkRzw8ZUlPwMMAfSqIudTwYjzF7bfaRx/ocO3ZlCfUdNUXcuYI5zwwaC90x3JmFXAs11gOnYDJAO6M4EigM/P1dfx1TwS4xLBd0yhibr9ap74Hilse49dh9prL9iQdqdqxRHJKF6ieMLjXIXYGuJxbVFf6bRnbPJsnteYP3ViP5co+5TLMWCcSXS0lymuywwrCSqbcC4kIzsaAz0EHF8JrF8k3RYQaM/VampQgQsKo3qgfMEzEhWRiuZ80djlSiAw8sCdpkMg6791IQwe4f2uUHzjs6uui+w/7BEfe+AkBp3az1dx9JF2EZNkTuZOnvyBRkS9oTmYa6RLPo9MHwgaSRTs7gdTjT0EFA8OvhKoX2mUd8xPkd4Nl1kqbD7pOZniedn5+I1aYH2nMYxrhLY/EMLtyoYQbOQqG8EEbITNmrWPwOOV2Wi73qY3Yj+WYnKkRbZXxjUNxbHFwZRCCxfjxhONaSNqN90cMsCZjfA1wYqaMJYAAAgAElEQVTj2kANbPAV25FI1qbAcKrRvE004RABJWDXJfoLg+WXvIZdQ0diSBbtunOIQmDcsGFQ3o8IpYSvNVRPWibAptr++xVMGyBHjVAKhFLBLhVqFzBuDKrRY/dJg/q2gHkcWBCmKBV/WsIcWOgGLaFaByE03LKA6hyCkhif1ygeFHyjUdx1GE9KZEsXtzJQnce4MdCHgtFCCU0DAL8oMZyXqG57jv3IgtTVCuZgIQoNX2mYuyPCskL/rErxOxrFXYvxvIF5kBDeY9wUKEeH4bKZjKpEYOZjedPCNwVdUHsHtR8gugGhMXCbEvowwi0LyMZA3+zhl2VqliapRcUlpnu2gnm3n9Zu9qxB4QLcSQXTDrDnC5j7FvasgXABuudz6mghU2ZjWJQQUsKvS+jbA/xpA7XtGFUyOprgVAZ2XUHvBxbVzs+mNSEgbBYsgrVCXPB9oSpIeXUBctuyOI2RPQfNbMh4toE4djOVNBoWw/3IKBStIPLfVksWtlqxqNWKhj65kFSK2zL0BohVQQpoYYCuh1jUgHWYSj8hILQCyoJ01sqwsE1RJRO1tK6ArmcxWVcsCvueRa4QiG0HKQRps33KgfR+RiZ/Pn5EKqDWLNqlgMjFaFnyNcMAUVcs6Luex6Uko1HwPrU1ItFiJf4E3TUmd9SJNvsLHkLhvaI0/mMUlv/88U/+8Z1GLMtPPohX/+1/BXiBHOguumSyUHLRI0aB6kaie+lhtilwXTFEPawc9XUDaXh2Tf2Xa1KQ9KlHeaNhV2GyqM9mB1GmfQRMuiy+LyEB6R7UvYCrGQ4vAlA8SoybAH2kcYOIczGStUfCMxS9upEpvzKjMEi6KFIDiy3pfllzwjdzIRk0pkUtLfJF0vnEJGRnMDxAzVlUCb2IpORFDUQ163LkQMpYLgBz0HixJ0Lsy6Q36bjgtssA3bJ4dIv5GKIGynuiQONphD5QRyeTyUmUdJ/MGp6s0fEl4wVi0iQBfD43CPJ4stlDnov8uow2S0daatYR2VV6aUINM2qdg8qFz8U/tYa0uE/7SppJt+B1g0hr/uZ1RH8pEsWVlL3sBJjzJhEwWeernvTGrL1j8Pn8nuz6mPV9ocBEqRSBGim75Ie2XSbNXkJ9zT4h8g2w+cJj/6GazTWAifoox/lY8znPdvJI14Wv0mv8bIShesyLnZJzm48jFHMRqntSGCGp9ctIge7YGMjIWdZguYaF97hiwyMH0yNkjWhiDFgW5tn5MmvfxhWfm7VE83aREMGQ0OUcXj7pCAekcZJJMG5yAwaTAyKdTanfkwMmFEk6jtGlEPuMpAvPQlBZLtb1kTRfn4Pc5fumGk81QOYYpyiI7ECadXXZDIRozmwMorvZqCO/N89vPi8iYHKezdpLETmv+b5yNabPl0nfhnSv5M8CjUl/Oc8TJmpmdjQdVmLScAadGkn3ZFIwP5Bjn+65pWCgvaSOb1izWJV+ZmXYRqS/z8wKX2KK0FAjWQTZHEX3fI9p4xTknscfJfWUuSjODJNc+KqBbJOJLZL0lUTCY6JR8v2cpMRcSEU4s4OJ5gc9u7RmAxc1RvQnEuUu8DhMzhnGxGzJFFhE7jsoNs1MGycdM4AJgckxDVGIqRgWERNTIl+fxSOR6Wxgw22wuMzMnvycdPxuzWYmviA6JD0mJN0bXutEukXKRk73jpx1qN6knNvl+xpWxnuIhLwmhN0hIe2J/TLGiWGR9atBi8R84fyYA108faUSIp5iNDoyDZj3KKfPUzmGGWFP9/qEfMVcbMuJuWAODr4mxdMcHOySCK7uPVytUx6xg13oxGIi+inSHNI0JyaTn/xZESf9ou490dKjQzBsGubXCJdMYtK4cnYv73Oiu9FIahbTNjMqzcaYSvMYJraBsD6hlRKMiiGaHKVIesLM6kksABcmnaJvNMQY0nNI80itJQD+3cWUMUpUUfYWflVBDsyWzK/ntczmAL/E0zmwRLnh44ROi5TvGUo9u7ZKHoPwAWLwE6Kc9Y3CegjrEUqDHM/BHST2iE86yLQf4SPRzOz4GiOilBAhTCY27zmspmKYSK+bdZEAi84c4ZHNcfK+czGatyXEvN+M+nUJ4U1F64RKjpbPA8AwAlpTBzlla+rZ0VVKop/4Mwo6n5BWqd5/PudhPnVxFXI6DkaOzPdNLiIn5PPn9/WLUMz8+NPyLv8x4kZ+VRDL3/pf//Nf+nb/1r/zP/6FHxvwHUcsl+WIq6tH3D0u8dvf+xYhCvxo+Q7Xwxq/f/cClXbwUeD18hxqYXH26R5GBtggcfe4RGk89IXHumL35tWrM/xH/9bfw6Nt8Krd4Hq/wr/xOz/B/3v/Ab65OcVHz+7xdstKZN30dNM81DjbHPHZyS1+/O4FSu2x3ddokt7zrG6xHSo8HhqMvYb4eIS0Csp4KAB21AgiYlGPOFu0+NnrCyzXHexjg9WvbbFrKyzLEYOlC+muraBVgI4C/+b3Psf/+e1nOBwqLJY9Drsazy52uN81+N7FI758c44YBT58/oCHtsbhocHv/NpP8YfvrhCCwKIiraS3GsNg4EeF5arH7maJ1eUB46hxsT5i35dYVwNevT6DqS38dY3Fx1vs367w6b/wDf7o73+Ek0/v8XC/hFARZydH3L5bAy8d5D9ocPI719h3FaxVCF5i/ds7vL3dQMiIbm8gFw5SRggZUFUW7U82+NG/8jW+eHeBs80B1/drSBXQv23w7Ie3ePfFOV788AavX5+h2XRoDyVOTo94uKPl+AcvHvD2boMYgQ8uH/H69gRDpyAM3TuHA3MDRaeApcWzyx2urzeQOmCwCqYZ4d80UFcd7L4AZISqPYYPaY60+HiL/fUScuEQegVZeQy9gmgVmg8POP6mxPhqgfMf3uFhu0BI+YyQEc1yQPvtEtXLI7p9SXfQTkOMdLNcXe3h/v4phjMPfZRwi4DyRYvjuwbqdIDbFZC9RDwbp/tAiIjQaZQnPYzxONw1kAcN8/KI7qslVB9hvz8g6Artx5bH0zgEKxFHBVF6RCdh3hrYM4/FsyO6r1cQz3qsFj0e75YovzUQXmA8CYjnI4SMCFZC7AzKB4n+owHNuof9fA35yQHhp0sEE6EPEsMzB3MyYNEMaPsC475AuR4wbCss/9ig+5db4E1F4yoAYpBYffSI/U83CAsPMXLxFEv+3TxoRr4AUA8afmMBJ1HcKrh/fYf2oYYYFB1UrYTZKthzB7lXiCYgLl2KkwkINR0/AaD+dIf9fQOMEnKQNC9aJJ3uysJ8XWJ8aaFvzeRyadcesYjQO4XyXqC/pAMpi84Iv3EQvUL1VmE8DfBnFsWrguyGRwH/l/cQ/9cK4xroX3iYRy7oxucWoidCFz/sIL+sEyVToH/uIHs5ua6qswHuYJhVabkQ0x0dSNUgYDfMqkRgA81dWjRfFDRgUmRzdB84QEUU1xp2FQHJyJrx3GPxtcJwGuHWAaufKBw/DPArD3lQdEUVmBxFg6bZlOoEVM/5Kx9YaIxnXDCGU4vymwLDhcfhkVpjc5DoPh5RfVOg/8Ci+alB/5wxKnIQEIGshuGcDSsW9hKupjlW1AK7TwIW30rsf+RQv9IY1xEmNfDUIKEPQH/JgrnY0oCq2AocP/Kov2XUhq85bjkC7ScW9VcmmQFFiCCnIju7qKrUkMkNHzUo6rI31AQvvpFTniI1oXSnlW42c9It/1bfRBxfAg8rgfJGznKL1OQyLabcwPKRxkr9M+pe5ShR7FIDYwH0ZxGLV9SU1u8UxpOI+i0bTP0FG6D1DTWSugXwQk1GTGrAZIRV3pMZks9jFBzn7lNABCLv2S01lHRTzc0Y4XmO7EpMDq6+4rE2b9iYqt9FtM8Uxg0dZHVLxopKgAodYtl0kCOPDWDDLCRToWw0lKUwABt81Q2ZBHYloA+piabZOCp3lGscX7CoXLxmoekaxcilDwQu/sBNpjxsQHF/x5dkmNgVsP5SoT/lPRe0IYK/j1i+oWawfSZRPurJ3XU4UejPJUyKdBmXyZ14IcgUeU7GQG4iqlGjvZKobk0yJeIYzDG5p6Yi3VWShfzIRXpufnSnRDmbdx79mUKxpztrsfM4vjCQNqJ68BhO+LfcQCDtmwV9UIZFpKccJGjmWqqeEpbiwIJ9WEnUdx7jmvNhjpxjV0uYvYevyJhwpUR1Z9E+N1i8oSmR6g2GE4Xm2mIsqbnMFP5hQ8ZGde8RSoHqZkR/WaDYunROiOjmZluxs/ClSuZ0YmoasBkWMJxoMoi2dood8csSkALDaUkzogDYtUZxP8ItNIrHAfZqieFUo77mc/pIWrDe9XDrivRiF6B3Pfxpk+jd1Lzquw6x0ghGQbUj6dlH0q1DkSngjBiJSpASrWWi+ZJCLbMb7LoBXICwDu5sAbUfEJYF1O0e4ZQ5l/rdDrE0LKL3ieasJNBUNPXJDrcZTc2Pp+6w+3Qz6YQw/jzNNiOf3rOwLAwwWrq7jpbopQ+JypsyLwGIp0Y+mBtZ0+NpoSm+0zYw/8w+vtOIZX31vfjDv/7fzA5yyYyifIhT6HB/xiy1/OGSH66iVbdd0OI76/HsIiFRSAHKq7RgSs6aysbJoGA4oQV4Rs4yWsMPfqIjRODi+/pOPyMTQSWrbU1toR5mBztf8Msh6wnVkJ3pAo7PFZTll4oaI7Y/UFi8TZ1Bk7V6mI0epu4ob2TThkkbahs60uUOsunSF0J2CdR8r/T5y47zqfoIu5QoHxy6Cz0dmy/SsfQR0ke0F6Q5jis56Z0AJFfGmDqfdPhTQ0SxD/CVSDpX/j0HJwct5u54iBgX1K+SDsxrub0k9TM7ngJzB9/WpIoR1eEXoi/EpMF8qg/M3XDp4vSavMgJWiTKJQ0ZdBeo1Vqp5CrH8fKYk1a1ZDc9684ATFbz2T2w2Hn4mrS7chumfek+THMhHfWHOdDbtGFCHnIXPyOS1Ps59GcFqpsB/WUJ1QdAAsOaiwmilmLSy4kn+jBXy0mzihjhFgqqT+ci6X2JNpFOqtuAca1Q7DykDRhOzNSRV8PcNfelhByTrk4RyYySiwlXc1Fj9kQApI8QjvsGAHNw6M8MdMfFzXiiITz1d65RE+pLVEBAH6mVUz33LdzsuugaNWnughFzuLYRKHYW8BF2ZRAKwWDuSkH1Hq5Rkx5TpGtRHyx8pSFtwHhCeqs5+DkLLT2iENP8+kKieKTtvG/4oaD3Fm7FzrO0gYjFwcJXKi3uFKQNDO1WEtSCzpqiqAR052dXRSNhFxq68zTGaAzU4NFfFNBdgN6njLxNMd1D0lKfN54WKB7p2KgPKYdM0XhDDQGqc/AJmcl2+1m7FpSA2Y3JqVXS/fFxnMYoArPxhGMenhqS9nelUb86wK3K9zrfGanKcyki4CsFvR8hbIBfcM6iIfoSjJyOIxg1aajl4BAVzUd05yekR6ZIgWgk4OOExohED/MLA70j5dFXmkiOkpN2T3WWYwYgBzcdt+wcoCXPb7r3M5ISGgN1HLm4jIBbGMYSZKQ7cN/RqPR8TNuISYtHgxLZZ9QD04I2xxvABUDTlEh4ag6l9ZDtCL8sIUcPue/hN9RV5lw+ESNCU3DcAVCHkewFSdMTMRD1gRLUJ0oxO2jGCNlZuNMG6jjC14bIlQsQ3kN0I8KmIYXRR/hVyc+fzqbfQ5oXBr5DiERbNNPzcrAIi3JGxdK2EcDjNYqZfZiRLQiOVbaW1MimmOYzzy0kEEsDuevo4tmNEKOl/tA6xKaatxcj53BIi/9+TOdLMyNx35MqmQxbsmYxFhpye0RsKlIFjZo0h4gRoqe+L2v4Ys3rCqOdUDPRj3QLtW5a1NPq3hAxCwGxKiHbftYkZiTMJ9Qw6QSn/MPRznTPfkRcNRDHDrGg/i9vUwwjC5TCcG5ULhoCsvPnexmKWS/5ZN9PETzRP9EKGmpdsw4ymkTHzMibD1OuYiyT7jK/J8bJ7TTW5XvjmYom6+axSDHNWzao4WvS/rSe9ZBGc0wA4BzRwLqmtvQp+uccdYz58z65ogqtOeauo54y00RDmDMhn0R1QOsntFTxfpwHN8xjyQ/JbUwIZUY58zaUYlE4Wv6eUMc4jvP2nj6yXjfnV+a/p8JQaM33GvOe62vM48+Pp4ViGvtTzWSm2MYQZ7rtL3i8p7P8zmssX8R/6X/55SOWv/vX/oe/8GMDvuOI5fnFDr/xn/4R/urJT7H1NSQiGjVgJXv8re1nODEtfvfmY/zl81eo1YhH2+Bl9YhSpLxGu8DBl/iseYf//Sd/BX/9k7+H33v8Pv6/d8/xH3/6Y7wZNji6AiEKnJdH/PjuJQBgt13gtz98hYMtUSmLpRmwG2sszYAQBf72H32Kv/rrX+DgShTSYWN6XPcrvN6tcbJocVkdcNMvoUTAs3qPv/P1x/gPPvsD/NHuCoPXMNLDR4nXj2tcro64WtCltncGlba4ble4MiNCFLioDjgxHf6Pbz/DZrWHUR7vjktcLXfYjhU+WGxx0y3hosTgNEav8KPTG9wPDX59/RY2Knx9PMPrwxqnVYdV0eNgSyzNgFeHDU7rFnddg03ZQ8uApRnwe199H6frFt1oUBcWddXjygxo9Ijf/fwT/Gu/9jn+4OYFinLEzW4JYxw+u7wGAHx+f4G2L7GsB7xY7fBHr65wujnCeYnSOFy/22Cx7lFoh340GAeDT57fQoiIz18/AwDUzYjvnz6gtQXubk8gVcDl5oBV2eNnt+ewVuH52Q6PxxohSJwsW9zvFlAqoNtWkIXHYtVjf7/A2eUWj48L5i5ahaqyWFUDrncLLOoB/ch4mkI7WK9w3FcwpcOqGSC1w3EoMFiNoTdQKmDRHPH4ag1zNuDZyQHXX53j8sNHPO5rVJVF3xXQxqM/lCgXI8ZeQxcel5sDdm2Nk6bDq6/PIY8KoQbKsxZ20Li63OL17z9H88NHHPcV8xM7BSzIrS1qCztoLFc9uq4ARIRtC6jSI1iJxWaP47crFM9bfHJ5hz9+9RzhCFx++IgYBe5/coaw9IAAqvWA/qGCrB2U8fCWlKlgJcQROP94i9ubFYrGwo0K4m2F8HyA0oEOyiNS1iKgqpHRNyLi5MUOh2MFNyisTg/YPzQw1wX0Z3t0dzUX0wUgpIc0AX5QkMYhHA3kwiJYAdEpxFrCpGxT32tIMyKMCrAaOaMxWAlYCagIBA1ZOeC2xObTBxyOFexDyczItYU4GEQdIRYOpnQY7yvonYJvWPAiANhYxIMGygAEBX2v4U4BRKA66zFcN4nDnOiLG8usSyWhBtLh49JB7kiJgwRCw/lGKCDbNMdLB3moERYe8qiAiwF4pyCiYhajB/z3aAscBkWUsgzAUUIMAqEmYio7jahSIWYSFd4pRFUgNB5i0CjvFWn+QrO5sHIQe8Pxrx3EsUAsA2RbIZ6OUNcNpkzOkHMQGYske4moNIRPEoNtWoxIFtj6KOA2AdEUs1zhYsTiDyscv+9R3hDRFS5pQuMJfBXhm4DqnYYcgP55QDQRxW0atyZq6BcKeqvga8YIiZBo1Oce1WsNt8wRNXGSE2TJA3XecaIhmz116iIyUka3jC5SraCztKhglwHFg2SkzJibT3QP1keiqW4ZEUxEeScnxJGaakwZisUDcPxeQHnfYNxwjNT7M1boKU0dmUqfqNa+ShT6ZfoyTDTKjBpLRyfeqLktaYkoesPXjJtM048pymc9UZl1DwwbTJE0qic6F3RNVPFIqn9M0ggRqCkv78XUWHUVsyfLh4jjhw10m6QHi/n7W9oUixOIJBePMx1cWmA4w5OoGf6r24hxQwp+NsYKKjdeE2rcURJQ7OlIXN5jyrTMiG9u0mV5BuLsDCwdG8O2WU0NXl9TchAU3W/LRxoD5mYzpQtJgtIminMfAbGepQFPIonY8NxMTWnu/3SiZRdHZqH6glTn6oFNjqDpl6BbLrJNl8wDBaZcSelIqRZ+biQXhzxGYNhIrL51aC+JBDY3YTrubFIHkH5+fK5QbsPclBegLnkfYGsB05FaLh33X986uIWazPmA1IhODVjp4iRLyKZs2dgt65OLHU23zMHTtC5Tf5M8gw6wkkZsKjXq3BPZTqL2ZnM4xnxF2KWCOXg2e1yiEisWmDrpXnPDUfg4NRFDweaX6t17+ZQisOEJiYkiTLptmAptaf1kJCQcXWzd8pLU5JEFrkhxJ5zfJxTS3FCzyaAo0ZRFbxFLnejPiVbrkqmXEuxJeRbZsdAQYzJcWlaQh57FcSq0RTYryqf3afGW3VzzeKrE9Gr7qYgXAOIwTAZDkAIi6S2nbSAfTpxQSvHk+HKupSzU+xmXwHtFqciM3Bj+PH4/v/KPrC39p/HxnUYsV792FT/+m38Dp02Hl4stxqDggsLRFdj2FX50eoMvtqSDahlwHApUhYWR81XZOw3nJQ7HCkJGnK5a/NbFK/zu649xOFT47MUNhIi43q/SawJikFguSIWtCosQZAqSZ8yJHTXKykLKgJfrHb59PMGyGnD96hTLiyO8lyxy2hJSecQgoY2HHTWk8qTVdwamttgsO+zbClp7DL2B9xJhVHj58h7daND2BYKXqOsRh0OFsrLwXsI7Bb83gAmo1gO84z5K47A/1NCG9FNnU6dSRvi8nbdL0jYvWhTaQ8qAfjQIQaLfl5BbDf2yRfhyAX81QNwXqD/ao92XwN5AbEbeNBGQtwUpKZcDc0APJepVj/augVo4+J2BaByEjBASqJsBh7dLmJMB9lCgOunhnYTSAfbLJaNBXi1w8uk97r89gTnt4e5qxNpTaysj1udHdF0BezSQFefTJyps7NOnUxGgbwr45wOEiogPBaKJUOsRUkTYLlELewndSriXA6mEbwuoTw4Y7muUZx2Gh2rKGJV7NSHDahRwVyPiQNMdWTkubLyAelvCnTrqglWEvtfwdeAi7nyA/uMGww8GxCCgHjTEix7xVQ1/4lBcaxo2NQFiEJBXPdTnDYZLGkXABI7bSUQTWABtGMvSfGkwbhg5Ehdp/xFAFJAd6aZhydeqewO/8EDtIVqN8p2atMYigGHsSwd9U6C8E2hfBmBjod4WcM8sqq8L6mrBxascBHzDoieqiFgHiFahvpZoPx0hd5qh8Y8S0gn0Hw0wbwtGXAxiyu1ktppIRYiAPgiMz5llWt5K9M+Y/SqCQNQBaqcZ3n0S0Hyj0J9HhDKgulZwiwh76lHcEi0azwJkJ5PmOKJ+w/M5nDMapbxXGE6ZUZpzQu0qLfC2AnZF6mmUEW7FLFTdcl6LLY228kIdYHEwJKqpLxlVYo6JRbAKQOBxq577Uz3nvr+Ic8GLWUOdNdfSkgqbF82ujklbnArCJQPtfc3CKaPb/bMAfeSc2jWplPsfMEPVLXhs5QPQnwG+YRGV96F6ZkMOp6RNqlZMz0OQzTGcUScdFI9zOCGN1exZ6LgFi5PxJKK+FuiuuA85zhTN9nlE+UhKZ3kH9BcsTsqHiMOHAtUtaJ7WM75HH+fCjxRWzqHuqMVmxmlEdcPzWTzShEz3c96nHDFlfWY6puoTXdQA5ZZsBNfM+txxzdfqlgt3WwsM5yzwdEsaa3XHIsnsaeCWKaW+Apq3nMusy86a3qxrVT21pocPkxut5XNqZKHRX3AbrmEREgoB1bEQG9cpn3PHv5NFQRZPMJhcs6kJJJ3ULVLh50m9zWMzh+TyvWKRVW6pUTy+SHTPpDG3S5pE5SzK5ppGXfVdgO5DMjhjkXa8Istjzv1kQZwZGtKTKqoG5pnmc+tq6pqnnNA469KzjjpKHmexj1CWplPmGFPUCCmeiHRnX772pDgbMWlH1cD50D2L58U7j35DGvVcmJMRQw0s761c5BSHgO5UoTjSDbs/VxzvmoZm7YWGshHllm7Rcojoz0k39cm92heJitrRlVt3ZLbI5Bhtlypd83FiI1V3dKwudollEWY3e3PwGE40qgfHosoF+FpNDAEAUF2ASyZYAGmpeW5Uz3M+nGiYvZ+KWdUHMkiSu7WvZ61ueWcxnhhU1x26FzXKu5HsiAei93ataDJVEh0PRqC6pYu37jx8xfEFTffnUKqkr005lcVcAGZnbeaCcptIv2dWQ0aw7Vk9of+kooLshdFPrAIk0zI5OCLnCS2PT8yVJmZBQqOF9RC9RViWkL1DqM3k3Ct7NyPwpZlYAJlaKyw1oqIdZuQ1u81qNSOc2bEWRJ6jVnjPMVfTLVs8RTkzgg3M28mPhHpDP0GDgdmtNqOsGWkGiKoCf6KgFMn59k9kWD55RB8gnhS5v/A1eQzD8Ge+7s96/Koglv/i//xf/NK3+3f+3f/+L/zYgO94YVl+8mH8+G/+DYy9Rtibme5pJeLKwVwb2DMuoGUnEaoAvVPJ8S6HrycK6IpaK2nZ8YEEoko6micZg0FH+GWAfqRboBxF+oClaYjuuCChDihOaIXuBLqXDmar6F7bi4mylwPlXRVT2Di769w+JlMdgFqm4dJDHem4KQcuEFUv4JZchAfD/Y0bap1EENOxCMcFnG6ffICkBbuv4pQZ6Eua6ORFSzbvGM4jgo4odhLDaUB5L+GaiOqWC0K7CqjeqamDfPy+A3TE8nOen2wGkhd9wykXk6FMC6MB6C75N1/xOHJA+7gJqK+pjdIHMZmQMF8vBaYXHAs7pJy7aZFoiQbkuc/j9uUTQxOXguGvBEQy2gk66YMKvr95wwVY1iEVj0D/LM1lyAt9FhPDGUPvsw4o63XoKsvn7IqoAs0jgP5ZMpGqOSflY0R/luiiCcGQI1GE8h4YTxl0zwVzWuQmI5bczS8fge1vjlj+wwKqn8ehBo65v0iL06Tp6i94Xkh3FnBL0KG44Lh9CZgdF8R2zUJAtxHdJRfKwxn3L1J3nkHiycm5TzTqNc//6mdceHKByjGZHR189TGZJUUiEaHkMTXXHJddJMfehgtmc5jnIMcCBcP3dlcRi2953Ifv0dk4G6jkogLAZErUP9F4NEkAACAASURBVON1X91yAd1fzBFGOZvUHPme7lJMZlki0AnXNWlhKfHefRQKpNB4bjcv6DPiEhSLFWoXuWi3CZmKgtt+ek1n8ypX0+gnH0/OUBTZVEWL5Fwdp3GOa+oEfQEsXwWMa7reLr9lgbV4E9BdMvbGF7O7dDbJUgMLpWBmJMUcaZ5EijfnzC4F6lvSnrP5jjlGdJdc+Ok2Qg8R3YVEdR/Qn0m+XvMc92cC5RaobwMOLyWqOzrNllu+ZjgVqG8Y05Rp98WBhkHKZiOgmMymIoYTCdXRfGnxNqA/zWZRAtVDhDdEf8bk6Jqdv3kvcd9sILEQjJJFpC84N8WBKM+wkROyVW45vyLwOI5XCou3dCWu7wPaZxJq4OsOLxSqhzDF3QCYjJIATHTgxbWfIppEIMJFp16iUGpkLFR2c1ZDhB4YnZILqXEhkwYQKTIpfSekBafpwhQdNC6oOawfSIG3NRGlYhdgF3KK9Wne0W3WNgLFkbID20iUW6JQw1qiOPA5VwPL1x7jiu9v3rlJL5ejgaSNCVFLVDrBrMb61lEegSxZoEttLpIYuZMMfTQgh1RErlgE6OQgnV2OXToec6B+kOZ6MRm68e/1jcW4ZoxNf0ZdIM8BCxu7zhmEmKnlY5KXNAkFKyVsI1HsSMVWXcC40Sh2PB5fE3H0hUDzdsBwWkAEFo6qJ/3c1cmQKOsRBRBKCZmyKLO5UBSA3RhKGIyE2VvYlSFNPUT0FxWKxxG+UjTxcYyqcbWC2ZMGz3lg0SY7B3tWTaY8Gb3TuwHjaTUVbW5BWUBeu+g9i4GoJOy6QPHQYzyvUb5rMZ7VMLsBbllAxAh1pOtw3h8kqEtM0S+cAwN1GJkd6cJElY4JeZQpiiZqORd265r0aCEmNC9qSSo3AHUYZqp3MgXKhj5Iz2XTnimyJVGdxZgos85PBVpUioWdVnxfQghFP7LQC4FuqqkQFNZNlOusaYylgRgs/33692n9llBFpWZqbKbBOk9qtBA0/QFm2nR+ZBrwzxnrZEptdKlgzUVjpvlmbWXw1FQKASg1G+s8QTyj9zNFd9rtn1J7/FlmPvm9dvxHvuZPe/wqFJaLH72If+l/+i9/6dv9u//ef/cXfmzAd5wKCw/0+xIQNM+gvoRf/k4yR808aKheJPoSC0fhwQKgBMxesjC5TR8sAxcBOVC82Iq0CE83WQ2Ye5WQibnwU32iVu1EMnoAzDFnxfGLq7hVpGkNs9GFCKRfyRF0qi0j9JHbLB9oegAkylJPzWN5k7LQtsmpM4p03CJ1yUWiEEku8k8xUbR0B3Lx5Wx0gNR5z4sTsxeTe6p+0iF3NRep5ZZ6uuJRoroBuis+T6QDsybVA3r/5Es2sS5ITUlOnjHNRUIGIDhWcwREEBPaEauEjAw0PZGONHuiCHx9MIDzAnKMNOEYMHUQyweeT7PjYisK7jsXH/qIaYEtIgu1vCgvRiRkL0VueEB5TIsxEVKhHgBIopWMrYlJm8eFte54Xn2J1HFPBcaQXUKTo+chZagmmpl0LJiimgvdjAToLjkRB44jX4863Qsz/Sqi+aKA2ccJAbGLtOCSbCJIx3nRbYRO3f9cEIYBia7FAiy7noaS0SmMe+E4zSGSEuNmalowaW46oiAIRHeqW44B4PFXt3EyxtFHjtuka9e0Ed6KNMcRMdHrIOkKGlN2bZRp7jXPo2tY5JR3nN8ouF9zjKnYi4hDQqdSE0P3EfVbvjdnNuojGwb6yCYTF+qcl1zQCpfcWn1ylU3NrnyfIQJo599FKqoyjUulc5ZpbCIA1X2g03OivBV77l93YioWfKARS0x/Fz4hJun8Cw8guWXqnnTWrP9WuXmVrhk1UutdbPM1ld1IBfyIdL7SNZmohXkcURJ1UjYVnYkOZ/acQxljQnZY4JSPicrmYzK0iclghOeKZiT8vDHpuIod0Sb1GKfr3hwBZYHqgcWISmOW6XyUO7qM5nzZ8jEkh1mOpdzy868Ary01AHqguyQpqBHKztEfxQGTJp6O1tSU8v/5viVCJa2YChRzFKlIium8RpiOWmNzYBHM44nTMXBhy887085OuJNLrI+Qkp97IlGxIRJ6GCKipJGJHmIq1gA1BBSR916BgNjO5ypTnaN8klea9OWmDSlnNSRDFOoTpWPervRzkadshL6P0/WnLM+fGgLKPVEwEQA9UHNsWqDcpdfkzGgXk0adBYrMObeDA2BIKQQmh1dpA4pDhGod7MpA9R4iUHML8P2+VDDHJ9sbWSQw6zMgO6UWeyBn4WYNv+oD5Ehttz46mDIhZkZCSAE1eIT+6f5yjq6E9AG6pfY5lDTQkS5C9h7CBmgjoDoHKQVEpL5OOBZFamRWL+mXYcoRlS5AtfNCPOeaysFDeDqMCoD7ze/3HHvW7OqWOnDRRgSnJhdY0xKpMxFJn5tzRCPnNVEzlaUTK88bEUs5OKisZ9ZEBREAEfg63ToI66EPlvdl53hcLaMt4OP0nPAeUSpmkY4OKuV0quTeCoupCMz7Ez6SHhoC4JJ7K+bXTVTOlLsqO8eIERcAESEs5iItO3Qn7aoAkTwhkmbVOhZ6MT7RNfI+EDnDcuR5mCM9yEgRzifNqJ/iRaZtZSpsP3IMPvCaGCyLvJ8vEAtQG5ofeT85P9TaqYh9r5h03N90FaWczJg1njFyf0/mY9p+nDWcMUaIGN5HJH+uePyFxeSfo5Dkm/8p4L/+M/L4TiOW1aeMG9mcHgEARnvc3q7w4vkj7vcLGONw2NbAwaC4bDE8Vjh7sUWhPQ59SXrq9Ql06eBuKpx/8oDdscJ4LGBqC9trwEr86NM36JzBN9+c003ztoQ4G6FUgBsVTOUw7kqIwk+/qwWpsCFIlCW1deFAaqowAfGoUZz1pKzuDOTSIgYBoXg+Xlxs8fYPnyFcjJAmQMpAauhmgP9iCfGDI+x1jeKqpRPoTbbLC0CnoDYW4hs6bcbas4gqPYSIuDzf4+ZuhWAVTG3hrELsFfTSwnUa4qDphFl6auUAiIJ6MHFXQB8F7EmA3km4VYCwAng2ILSaNMyFA3YG4nREfChQ3ir0H1rIylEL5wXkQUFc9fCPBQvIlYW8KSA/6ODf1ohFRKzSB3TOArECMBHFW43xIjtdAqqTcGcO8GKiWorKQzwYSCvg1p5jFCzK7SrArz3KNwbjD3pEJyF3GqoVcKuAWAUU7zTsmh/+qhfwSy5EzIPEeOU4FhWJjhcB+oGaNhEE6YWO7pjSJlfOIBAbB3VvoHox0eggiFYPF2mMkY2BUCS6qY5QR8l4mD1dYtkIoJunXbGoDQZTXAtpOhyHL+nmKZxAldBeXwfIQaK8F+heeMQ6YPNjg+6KqPa4JiXSLdjMUD0Xq+M6LfgMt89mB6b9Dxce+iBTHIlImXtgt7liUSZHAd9ERsyMzFvtPvDQOzk1MEQkCqqPCcnreE55IfL/dkV6lTkSoc8FunCAW0W4JkAf5NQACiaifscs2alBEJDietgMyjosObJRxHOB6T15X7oVU7SDXbNYy1RYadnkaF8ENK/kRB0WYW4oiICklcK0gAa4L0YCPRmXjnSPLYkYQ5Bm6eusjUPS+NBps71i42SKOGkSbTJr1xxpp3Tspbtlf8lx2RXnXLczNXM8YUGeNYI0aMJkRuVLUkOLHQtpu5wzgqOiq+hwlvJmF2LSkzFSiYh7ptPmWKBMpRtO074SQq5bItq6nampjn4znEfNxkV2GB1OMDWXWLyzUZddVRljJCa9IgRQPpCiSZMgTPo4NgrJ5BjXAvUNEdwoZ4olCzYeT3/Jc1PdxglBz1oy6YDjCzEhn+aYonuGhOTn+BCJydwrN91yM0pEFm9jQvqpCU3NoNRwY5YwpiieoJE0aUj5pkTL9cCi29Xz9Z+bGiI5gfpkKJe1feNaTEh5sScCnJsCUQm0lxLVAz8zo+R1okbAprxk3dMELxfP44oxK1EKZqUe5ripchdgm6Tf1HM0ke7ZJPKFmKKhlKUxXUZCy8RCyNpH2/B4ARrVZTM6XyYzsi4xBfwc7ZIpo5kSG4xA+eAxnCo0by2GMw1XChTHwGgWH6fok7yNqHj+i32YzNXUSORYxDhHoBhSNjNKmg3ydBemMdDMjxTU7O6Z41ikJSoJcPt2SSpqsXXcl6cJmrRx2pc5ehrWdWGKQQlGTqZy1bsBvqK7aVACrpEotg5IhbRPmsX+3KDYugktBTBRR4nk6hTlQefWnLk5nJcoHkaiqAcizb5WibaqpnnPTQU5eNh1QTQ1mX6Rlkqqp7QewSiijjFCdi7Raw2qdx1CkQ32LHxNNE/vekaUJPQzN3N8k5oXQ6J/PjEdEi4gVCy+JqOsTH1NGsfJAGdMCKQQgJJT8RuVgDz0NGeybjZDqsxkNgXrAE0jqqgVzZaqAtlMCWCTCdYxE9OHRIuV82tCMjOyDugHFqUhvl8E5jiT/JxSfyKuBABin5DPhFbyd5FMk9J7Q5yLyLy9n/8/nhSaqZj9cxWZMXzn40YWP3wRf+OfAGL5e//+rwZi+Z328i20w4urBxhN7aCSAX/lsy/hg4QxDi9We/zmx69w8r1HhCBQnfY4diW2xxpaBry7XeP0fI8QBdTFgIddA2M8Pnh5j+WCN8/HP3iHn92c4fXtCYQJ8K2GvuwhBLBZdSwkW4PlxRFCRXinUJ928INCWTpcnTH+IAYBvR6haofYaqjNCDcq+EEBlYeQgDQBoVcoSofXn19CftBC6gjfabhBAwLodyXciwHjoUD9wQHDtsLhzRJnL7aMZQBgTgfEALgzB3E6sjj0AvGgEYPA9atT4KYEegl3VyGmOAe3LSBawmJikBBt5l0B0UnEo0bUEeMLi6gj3CpAno2Qo0A4asijgmolYqtZzL4rgSDQfzxAtArxsYB61BCVJ/V2UFAnI2IREFsNvBhgdwVCSZ0cBonJlOJeQ/Y0YxkvUpFrBeoPDtQQegHZSsAJyIUFjtTsuQs7IYlRRvRXDv7EQYw03xB3BdSdQWg87IsR+igBKyZdIAT1dvCAOkpEA4hRQliJ4kZD9gIIRJpVzw/g4TRgOAtTQaMOCuZ+ns8oALdMnVGf9tULGoEIFg2+CdAHAXWU8E2YNGMAizrhiEaFeqYayTFRuhL6O26ICDTfqpkWGQGzk4gqMtt1L6HvNba/NcLVkRRsJ5DzUN06TAt93VGryBgJGqmokQUVdXsyxVuEVHBEhJImJkFhOtYcIaF6FtAmIeCZGh4VoFoxIbdRJ1RIsuiashItj9VXPL9IBakceYwAqeu647H0FyzSzIHzNKaiUHWAcKS8q47Nie7K0+G4JoIugNmAI5KSO+W+7jk2UmszE4LU8OGUBYxLuZs05JjpzG5JtE060sxtk5D9dC2ZHZHIbL6S6a9mTx0g0X+O5fAR58EtMOnKyApgEZuzaoUj6s/zwyJMjpyDnD+qD5iQY5WQcyAxFxrg8H1uE2kfruLvSPMUBRE9V7Oo9GVuEszIrD5iLtAXmFDg44epOEr7DUkLyYX2/LpMFdYtqcOczydU7UMqTDsWHxntlJaO3sNZzpDk6xGA4VROx8B5TsWUYDHlS2oFx3W6HsuZ3iVHHq9bCJT3McWJiKkADyUR8HFJFkPOzZUjx6gTjTn30XyRnJRdoiDHuZAWntuOUsAuSYU1h1TESGogo5qvjawtFWlbvmCREopp+MgZtSHdY2rkeHzFzwOyfARcJSbX9XKb8jmPnMdxSYqtObIZ4wsx7X863oSCQswoadDAsJJwVboGDQtBaZlDGlJRQyr+nHVqlzzOcpeosgNp1mwExemzi7EVPLf9ho7aMpvbpAzJHNUhIseCSBdxEalDZCYlb4TuUqdoF0WzG0F3deEZIxI054m5wCIh8IAv6f6dc28zpT4kV3tXyfQZHqFbIoEyu8abzKZKhZZL9F4Xp4JNuuSQnmjBmXHAOA5JZ+kxTqhrzurMjvO+Ig00inR/jRFuaeaMzITYx+wov6Brt12RGkw2VKJeR0xjIi2WiLKIYByIEvA1HYejkdC9T3R9IpwssDnfygZmWKbvOt05ZnKmPE+3LEhVTfmYIpvZ+IhQa4RCwexJnxUhXReGVGe97VjoJeOeYFSK8wgwDx1UotSGQk0ZlMIRmSOimhBQl1DZp4hodgFWCjAaYVURHVSKTsFCINZloto+MdFJuZ4xu9EKMe07NqSxidFCOE8EMwQgu+d6ai9j+oGULEyzG279JNjYaL43ayKTw20MYS4qk5stnOOPUkQ+tQKkosMtwNcIORWHQgi+P+tCw/toZi4qhRDUVoYIoeQv/IEU8082BvqOPwLEL/3nV+XxnS4srVd4/eUFbu9WKJTH9ljj8/sL3O8a9L3BV3en+OL+HN3/cw6lIuzXC+YligijPZQO2B9qhFGh/HED32v0vcGr12fohgLlNwW+en2OqrIQIiKOvKDVHy/gR4m72xXGQwFTORy/WSHsDOpmwPjVEhglDtdLvHtYJQOZCH9Twe8KVOcd/LZAcBKmZq5dDIAUEaII8F6gvGqh/4DCKll4IpmWXzrqTQkIoL1ZQC0soCIeHxf8oi498GWDGAWanxnoryqIQQJFwOLqyAWCT+6RCwesLXTjoJYOcmWhzgeiWQ8SWDjUpx3UwkGYMNFa1z8uIKxAca8Qb+gUdvnhI2KR3CcbBzEoalo7AXNdINYeF5/ccyEpmB8HK6G+qFHcsGg2n9fQK4vyVnHMZYAYJeRewz8fIa0ABonqTbK/7wXauwblOwW5sHS/9EBMCCgCoG8NYCKNbnoJ86hQvDXQB4n6rUDYOMSrHsU7DXVnYNcecmVR3tEUo36jUGwlYh3g1h7VO25bRMB+f4A/dZBLm8xV+MVf3Uqc/QHdMfUhFTgFgKWFHAXKR4H1T+YPx+pWQFrSZ0MZUL+V0DtJva+OiI2HWwY0rwVps+1c+MGEhBhG2BPPQlMgOWOKSccaNanTcmBBAQDFg4TdBLgTh9U/KKAGgWIn4KuA8p7a3Vyo6paFa3XNcRePkvE6Dnjxtz31QC8GNG+IRJb3wPqnfA9RT2A884iaaGUoaW6z+ipivOL8UZvIhaU99TM195B0ZoLU9Kg5T9GwQAFYxCzexAntdU2ckA3SkoHNT2hQ0196VHcRq68wFRXZ1CYUpEHWbxWatxHFY4RdB9iNh26JxJojUL+NjDWy1DYX+4RARxZMbkFEsXwgMrX6JiZ9IzWSEPyX5yiiuolYfk0tbLFNBksyYjiPE+rWvCE91TURds1iPRo2EIKJWH0JjCeBxVdEihvivKme48pU4ahYMAeVdLNtnJDfoIDuReACWXGOXMPtFTsWZ/rIhbLZx8mBE2CBN25Ir9XdvAhl7iC345acq+4qwDW8N6q7SJTWU+dtjhG+oqGRbnkd6j5yHGOcZARukVCRSKSyfsfxqIFIsxpYPOei6/AR52P5OtA9NOkPiz0pnc07Hnfel4gs0hCovV68oaaxugtE/UbSVoVn0VVuifocPxDoz4H6LsC0EfVtQPkQk6EN9brVY5zGP25m5NPVnDs1psgqwSKLVN8AlSip9W2gljbJLYbTJKnwLKqqR15n0pFOnYu5IhVh5S6hHZ4FU0aNs27YG+pV6cJJRFj3EfUD9ZtEZdJ38YpNoHIfUOxZeBfHgMW1g/Qp5sUnvXYtJq2k6gOKnYfugOUbx6xCw0JnSIU8jXF4LGqkDpZyAF7rwgPdGXWiviQq3J8zOzUooEyU0+oxbXskrTcbyWRqt22ILklLva/wQPPOQqaC0yREb1wJLF6PGFdiyoEkcphdXIng1Xd+0mYixlT8+9TMIi3W7H2KlJpRYgiwQVRLSE8092mElSvF1ChR3Yx6zsZxIbmmOthaTjFKaggwB5dMjQJ8zULebEe4RkJ1AcXWwjwO02uiFFCtgxwD9MHy+JcJEe08Y40cEchxpaAGPyHtxU0H1yj4UlJzWUuMG03Ka+8Yi3SwGNcK6mhhGw1hA9TAOKcc4wMArtapcIyTi2qm9krr034ZFaQfmduIRHMVNkB6RkCp3hF9lQKyTdEwKpnrHHvqSFMxGBqD4fkCvmF0z//P3psEWbLl6V2/c46Pd74xR2RGDi/fWFPPEy1kgBoM6BawYMUSwY4dC/YgtsLYscdMBjuMBTIZwkwmIYmeu6pe1Xv1psx8OUTGfCef/ZzD4u9+I19LamMorKuwvmbXMjLiDu7H/fo93/mm3lNK67BDYQxdaLrFbwWBhj5wpwOZLuqqYaoaWqn12QKvSuqsVFYI6OurTAKzZRtVX58CIpt1DlV2wTU9KI3DO8YzjvBJLNLa/vWaFupGpLx1A1W9rWTBOogjSXwNAogjVBKjoujOc9lLYcPwrlqkZwydxeeFAMqmFsbSmLuQnbq+81pqLf5V7yX0p69SattOjuu2P//5O9YK6G1a/Nv+0r+6/Uzefq6BpXMK1Si8VWy6xFeAJGlosog0lv83E0d5ncoKs9V4r1hlCW1taBsjtQ8zjw4tWsmXZRrXNI9LTOiIA8vefA1WEVyFtInHxBbfaMJBg/cKP24x04aiiIQtiRx61KC0l3TP0AqY8lBdDKS2oNG0jaHesduqBqU8TRZR3iZUO06SXRVSLWA8OrYEpSJIWqJ5KWA0drhWWCjbatp7cuFpB5524GFWQ63JlglR0mDGDWiPDjzpUBJjbWFkdRuZ4FS7ViTCjcFuAnTgsIeVfHn1zIRDwGOuKOpQJKGlEmDXKtAweKWw90ownqyM8LHHLyORRg1bTK2o91swXuSgWmSbPrWozMCkQR+UxMNaAogqSeQbn6yFPXIde1cbAbUehpMS32hUKyBJhQK2+qAmF0IzEzBEq3CV6SSLEnTkslDYnMHdxE2FffE96KCTeOUBZhFs31v8q8I+VHORynrjsfMGmzp8K2XtzbArKW9lIHUjx8omQORILyT5sjxshYWsJJnWdXKvPuHTlAg4LxEfV9MdGM822RRg8FqknsmNE0AXdXfTjUkkq8jRUibQPrqTo6hGSSl9dsfQNeNOjhsLSClnRhg2JYxMz0Q0w7vQop5RhM7vO2+6fZeT7o7dRBhF47fSPdf5Xk0hjKf4WjupUgR24LpKDQnC6vdfwfZ4FweuY01FOv02C2QqmbC2O802lVOAi0yIJTlVAG89ldcRSakSuXInWQs3bL2+vbxTugxFgik1FML0BHk3yR50YUtDRbEvyaLtQOSZuhEJsI07sJp2/tSOdewDeryR98wPlbDRSkrsfdAFysR9+JXsc7VnZbGgZOszbSaKZscKo9T5lpWTsZWKDAFBkjIrbGnPegWFgNEw460UzE562TEpyss54CLx1uLBTluSa79NK5XjLeew78LTer+f+EaF9Qsz7nyorYDVMHe4yG8Zuv4a5SLZFt0xu8FG2EPbgZUwEyDcs0vKCeNVj6ViQgKC5LrRpgJcmlHHFvYJyZ331QcCxqLNXW+yV+Klezu4TJgctn7ffsx05yeUa6swgaq9893eBbix7ZbtPxt9WFpYSOhbNVXYUMYzqKSeIVq7rZy0TdX2Gg4CeLae30KAVd9t3MuCxWcnwL5nI4X5k/2QFGBPtBFw0fZySCfHqP+8b/uJFfhAEV+X1FNFNTXbz7u8p3zGe6+3ct01QcnY6ebuet4fN9spCXq2XYCe5C+0iQQP9bUbppTn9D7ksPAd8Oy8n4V4JyXcrpOvVqJEKXfD7lrWUo+NzC1CJYsomduCVRf0tTGdLNbTsY/CJvYVGOLrb2i78KNw026PiVd3sldn5ByUWheNTfW2G1Z16aW9/1UApwBUk9ciq/ZdsFl3frlA4RKzXbjDeewgFBlvZXGR2gIrmwTU024RuAv1EYZP40yXTly0hKtazo9JRHxTi0e1S2kNM3vX7aqAVlKBfV+34RGWz4EpulCkUK5psj0aXbXo1m3ZTACbGNnfxnVjLSmt3gjT18uLXaBFpttLMLUSwFo2qLLumF3ThfNICJFq3J10tQOPsq0CasXkKxJYH2ho7TZRVueVsI59iE3X99p3s2KUAD4tfyfQIsnV0sPqIklk9YEWgPhWH2jPYG5vPSDVSn7u3zcMtt2nvMVgfuO5cMeY9v2exqD613ibJTSGbVKsNm+9drAFijI8Xh7bv0/HWCpjvslGdgyoPFb/hXdlzF/YdfnzcvMguOGnfP9Zuf1ceyyPvrXj/6e/N+F/vPkNik7T82yzwy/PX7AXrvl7b77D4WDFL0++5kW5Q6xbPl0fsp9seJHNGYUV07Dk45sjfvfej/h0c8SPLo/YGeY8fbnP733nB/zx1SnzpGAUVuxEOdYrZmHBokn5Bz/5kF975zmt03xycciTvWtuigFHwxXTsOSfvXxEsUr49juveLWc8lsnz/jD8wfspDknwyXjoORPrk75N48/5e9++qtMhiX/2snn/OM3T3gwuQXg1WaKdZrz1zPefXzOpo44HS/4+M0xTR2QDiqOx2teryY82bnmi+s9xmnJukj4zuEZP748pG0No7QiMpaLxQitPWFoSaOG1gqY/WDnks9v91luEn7nyWf8+PaI60yYz9mgoGoDlPLcLof87gcf8798/i3+7Xc/4R88+4DvHJ3xJ88fcH//llUZU9Yho7Til/Zf4rzmh9fH1K0hDlsOBxt++OKEo70liyzFGPGh7o83PHu5x3BaMh0UnN9M0MozHFQsrkYo4zk8XHC7HvDh4QXf/8kDvv3+S3787IQnpxcsipTWyoVxtU4ZDCumacnrN3NMZPnu/Vd8cn7Et4/O+Px6Hw/81skz/umrx1RVwHhYohWs8xhjHIFxFEXErz98zk9uDradnZOk5OnrPaKkIQgcSnmKPOZwd0lkLK+up4Sh5btHZ/z+p+8QpC2TcU4Strx+sUs8LfnO8Rk/enPMyXzJ69sp93cWnK9H5HmMd4o4aairkF988IIfvj7BtoY4qWlqQSrvH13y6etDchqWPQAAIABJREFU+V7OA7797is+OzsA5WnLEF9p5scrjO6lJp7LF3MevnPBbZ6yuhqKP7STLKvcoKY1YdwyTCtuXs0Ip9LJ6i8SvvtLT3mxmlJUEXUdSB3LUpjq0eGGk8mKz744RrWa8b0VqzdjvvvR11xkI27XA7xTNKsIVRr8wEo/5ouE5qDhtz/6gn/2+x8S3suo1jHJpCIILM0Pp4x/+ZqrNxPiSUW1iTGxxRaG4U5B+XSMHThOHl/x5scSyWuOC9qbZNvnqCuNG1rCUY19k2KOc5pFgrKK4cmasojwZwnxI/k5/DKlnnV1JVYxONlIxc9tjFlr0g8XZE+nsFeRDqX3tLlM0aVi/N5COjpXEcODjPKrMXYoKdScFswmOVfnE8JBQ5w0pFHD5dmUwVcR0W/c0FhD+dUYf1jJd+irROpaekA2aZnsZDStofp6hK7FO6xrTbjUtCOHHTrCW4ONPeYkp8kiwkGDe5WKDHmvwSSW+Ecp1Xdy3HWM3qlJ0pr8bETy2lAeSBck2uMPKsZ/mLL5zRz9dUqz10gQRqukd3LmtknbduTYe3TD9ee7Ig0OBIy6PVlAUMaR/DgVb6yGZqclfR5SfVhI1U/kiS+MVG28Vqy+V5N+FWFjkd4H1yHtXkP0JqRNRYJtY090K1Uo7U5L8iqknjlGzzXZiSe5llTmXnKMQ2TeDuzIMXxu2DxpmX/fsHq3Y+c7j3Q9cwxea4oD1wEVtQVZQS4LNV4LmHJdKJpNBDQr1/slYfPYwrSB24j4WpK88/uW8FbjA0guFZt3LKOnhs1jS3xlSN94Ng/pAps61rHp/NIjOT4u8WAVwxca3UlHq5nfgt30QlEcdP7ctPcKS0r49DNZHFi957YVPZIurIivFdWex0YC5nwAg1dakrL3RcVQzx0739cd+yiLYumFAO524NFWvLvVXIBvfCWsfbHfBapZkX2Ha0V+7IiWeusPtoln8qUsdPThY7Lw1o19BspLSrYPYPi6B3Cyj9VMEsbDtSwK9eMBogToA+iKg05CeyP7qGth2ZuhLOrEt578+M6fDaIsqacwfu63nuP8UBZWoqXfpgj31TPDM0c1FZl/ciOJyH3tigug3BOvbpsKM7y+r5k8tzRDLanKXhYjRq+krsamivjWEWWOfM9gky6leqyIFxJ4Vewr0qu7upgok9+vHmlGLxzNUDF63XLzUcjwtSMsJIF4cOkodkWmaxr5rLQDxeDSSaqwk/ClaCM1J5sT6bsNM9el9kJ605IdBndhV7XfgmBlIVpbbKIJVy3ZSURya2kGkrK7Po1IFlYWKWth5OuxxlSe5FpAaH4YktxYyrlhcNFQ7obCvMaaeNGCE/9o72ENcmHseml5fFPRjKNO0eNFxts4yp1guy993YkLBVTW84ho1QUMlS0u0LhYkn2bUUC4bsTbCR0LqzsWVMC7MKYWl3bhkJn4Ok3RSKhR2eKSoKscMei8EYAaanR9F5CE99+oJlHWIlUi7TaR1keBgPLusTjQ6ww3Hgp4Lmp6jye2l/LarWz3G52VIOmzvV+zrzXx/g6AVrUwmN7ji2ILLNFKGE0nj/VVvQWUfWAQ3Elh/2X1I38++Gd7cw7X+zz/H9x+FjyWg/eO/Yf/7d/6qb/un/7uf/2Xvm/wc85YKuX5s/IhGxtzU8uS6LenZzTecFFPmCc5h/Gai3rCPzl/h1Wb0nrDF6t9Khvw+fU+X2/mAhxNyapJ+Dfuf8bvHn3Mk9MLrqoRjyc3vF5N+Ox6ny9Xe6ybhNZpLssRv/DwJas64Syb8GTvmqyJ8MD7ows+XRxwMNnw2x9+wUU2QmvH63zKTppzmQ359PaAP7h4iPWKTzZHJHHDLC34h6/fYyfNWVYp8yhnVYgefrSbo/HkVUSgHGUW8bsffIz3iqINORhv+Ho5oypD5knBg/kttTU8mt+itaRGXq8l4CcM5YK73KRkRYxzmh+cn7DOY9o64B+9eMKqjKmqDkxmqYCTlzOOd5d8tdljmNZ8ne3w26dP+eJmD4CbPOVovCYILIt1yv/6yUecl2N+ef8lZR3y5s2M739+SpLWTGO5MHxr/1y2pUjYP1iRvRmyLmNOdpcMBwJukklFELdc3kxI44bSBoTjWsD06QVGOaxTOK8YxjVKQfZyzLqMOT66xXvFpxeHeA+fXBxyOlsAcFZMybOYJo/QCnbSnGoVk18OWV0PcV7xTz97Qho2ZFcDVlnCqkzweUBTB2QXQ5xTuEZTtwHLIiGOWz7Yv6B1GmU8u7MNi8WQsy/2iSYVcdTy8dkxs1HO2WJC2xjerMbUtSzFu0LGHOAHL+/JtfUspshidmcbosjy9HoHYxxh1KIiR2MNbW1wTkMWgPEsV/J5uLkdcvV0B5W2xKYlyxKwimjQEB/k4mP18CuPv6YuQm5vRlum3TUadVDy/U8fsFwPKM+HAioXMYcPbsBDnsXEQUs4rdCFomkNZtzwxeUeN8shrgP7e/eW8qEN5EukmYu8+/vnJ3jjqc+GUGuSqCEKWuqpLGLQaqpFAq3CloZwVFPkkbAMsePsfCasxcjSVgHBTolLJFDKTVrQAoJ96JmOSszaoBrF5mqIrYVpKDaxMLHTnk6C5Chjb5TdJeVp2CwGwjjUhnyVSNepFdayagLU8xQzqamrQIKEUos9rpiMc64uJuhlQFsbsnXC1dUYvPROLs4m5K9H2KnFZSG+Y7q8EVlx+mANtWZ1PaRtxR+GFgADbCXLw4NMti+RoC9VGJoy2II8vQywWUBxZHG3sVQVVYb85Qg9rySUqQHuFQJeioBi38NZIimJVqTXPnRUjyvcyG593TrXXD6fb8FZO5KALJSHZYjLQqkUqqXOSW86dmod4kMv/aEdq71+R57ng44FWwTS++rZ9lLSqSW2iaUbmUi5UUu52yXPAl53rFbits9VTp6fHzvChaHcU9hJF7B1r8RGImNuUwgy6VUVttVvt6OeOWHKGrbVRdGtEmnyzN0FHa01+iIS1jfyFEdOAplWEurlA4hu5JjGV7IoUO52DKiSwKje8+y1qAlMIR7lHvgVh37LJppKWNzsvtuG69QTR37kiG40wUaRH6kusEm8kvFN5+Ouuh7LTBGutfSnroXRa4ayDVIzpCn31Bbs9UnrPXNqE7/1AffMYbXTeYLfmkO6EOJrWRiodrokciuMvqQNy3FshxLa1O8v0EnPBWD2vkybdHLpLqAp6Dya9USOd3ZPfIrt8A74u+iuIsgr8VTbmK0fOFpBeq5IL9S2TswFUi/UV0aFGwHQxU7P0Mr+5wciiTVdUFEvzbeJ7KOoGYRprEdyjlUzAZV97VCQ3e13z9A2qVyzcbLt7bDzshoBxMLmC/h1gYxltPBbP3AzMoxeua03c1sn1gr7rBtRRvjudfrj+jbbCnyj1unteppqLseyTbSELnXeeUnWtsL2+l72LYm5faCSqf02QVmqiaSipU1Nd86I1NmFwj7r2gsL2m2XjfU2GElZj4s0baq33s12YDq2XZhoZT2jFyXRWl7DJvKd4CIJ2IlvarwSZreexXcsaNClO3dpsm1q3mJG7Zalq2dSf+ICLenAQ/F4utCgqq4qqA8ach0gNJKmixP21cXBndzWe1wSCIjsalN6memWGe1lpyB+TKP4Rn9lH+jTM4mdRHV7V289/m3vZ9Ol0b7NYmp1J3XdfrC7b03voG2/ASRVEOCtxVt757nsgW1/72Sv27939/55P89k2N1N5qs/7fvPyu3nm7H89o6P//P/gvnBmqPxmkFQE2nL62zK+XLMBwcXfP/LU+JRxb/68Cv+8fN32JtkPJrc8MVij50052w1QWvHYjkkCC1J3HA6W6CV54df3WO+uyEwjqyMOJqueXE1o1nGTA43RIHldjkkTmrisOVovOYyG3H9bE56vEFrj1aeuglI45pNHrM7lQTb2/WAtjVo7WivU5LDjDCwrM5HhJOa5jZGjxuipMUYxyCuuTybomORzaZpTVUHRFFLXQekcUPVBBjjqGv5t3oxQh2WW+ZvkyXMpxnrPME5xaO9G55fz2nqAFdIoquZ1rhG4ytDOK7ZmWYE2nG9Hgo7t0zkSyh0qOsIv9PAJmB0f8X6egiVJt4rmI0KLq4mqKsIN2sZzgpheC5SDt+94s3ZnNE8J1t1RnKv8KVheJCRXQ0kobYMOL13zU02IH8xlknQozX5KmE0K9icj1CDFt9ocCJrREEwbATwbULSvZy6CrGrECKHSYT1otVEV4b5L19ysxrQXqd444muDP5Jjj1LcbMWSk18ZahOGpG2LAPUQYVdhQTTmnYdohKLOY87T4XHDS3JWUj9boHbSOKv24h0SllFeCtprj4W2a/JNO1+A6VGTxr8dUyQS+F99rhjfLoJvE8c4XWAvVeizxLa/VqSfa9TvPaYSSM2h1cJ7VCSUduBvFf6IqA8sfhAZlG61AK+ai2eVjpf3lzSa13qiA9z2q9GMqkvRUYpYQte/J03obAuj6VrLLo21Pst6YtQ/Iupp521kp5rvAQwAW7colcB0VJT3mswiwC7V6NWoUge5w2jH8dkD6SzFQ3tSJKAkwtDuS+LI0GmaSeO+NIQrmH9rRoqjRpY1CLspK+Kes+SvA4oj1qUU4RLTXSr2LzbEl0ZkcglIh8Ou4TaaCETvHricNOW9GlEcdKSngW0Q098pcgeW1Tdh/7QSaEV9dyRXEgKr6m63tTOdxmuO09dDeWRZfZjTTVTlAeOcCUVQcWx27JUulG4wDN6IRPK4sh3nbOSEuxiCZqKrzXVTleNUkgPK0jaq3KShGsTaIZ3FS7JlbAibSqAKCjEm1vPHKOnmvU7junnms2p+FejlQT11F3/bDPsOv5aiK8V2QMBcLqSMRg97zoSM8ge+K10t++hjVZdcA4yPvGNYvNAnteDCd11oeoKyv27Pt4wh/xYKp7SC2GY7uSnkN0TJs0FkF5JCEw7YtsFmh8pJk8dV7+oGL6U7Ukv5Pe6k9gO3ohcUibvbLtK2wEk1z34kzEQ36FMjstdYd/iW6nQsYl4A70RmXKxLx2t1RwGF+LJTK5kHMo9GL7wFAeqS2+V90yu5X2qmfSLulCxfqC2/bNB3rGzA9newZknO5HzzcXyfBvC+pEwmrqR9xi9FC+npPX2E/u7VNZmpLb7qawE5SyeaKmb0ZK8Wu2IvLsPV6on4qs1tWf9UIvfthJQFW7E/1lNFeOvG6pZcJdo66SzVAKdZDyH59IzaqpOau9EChyvHPmBoe+g7KudZPy1AL6u9qqaK0YvHfVIjm1YiJS4GuutdLqvVDG1Z/3AsPNpQz0y25qUamoIc0exI/tTTxSjM4szXS9mz4yu5QcbCTDqa7yaodom2WoLw1cV64cxpoZqIkxiNTXbqhYXKMK1JTsOSRZWgn24kwInNw3FfsTgvCI7jqU6ZtXSjHrGzdEOBWglNzXlTtQxddIn2qYaGyoG5w3VPBCmcNOKTLdLLO1rVnQtUtxm0ieqOkl6HRuiVUs9CWgTYQ4l1VWYSRcKqAtySzWXgJ820aTnFcVhTLxoKXcC0itJeI2WDTYxNCNDvBCmsJ4KCxotG+n8zBrWj4ekV913Y5cW6yItXtLOR2lTs/Ve4gWghlm3fZEW8Nk4qVqphB1sZzIf0b2P08vrBpuGdhRuPaBegS4b7DhBV1KL4gN912XZz6u9p50khDe5AL9Ao6oGl4jCTvWhP10nphvF0r/Zs44d4ykX8i419u06Eq0FhGmNaq2E9YD8rBVEodSVOIcfJPJz93hau+3G/EYqa7/tqmMe++CeuEv6KisBiWEobGR/64Dk28mu3naMqDFbsPgX9lf+y2Sub6fF/v8gFXbw3ol/77/56TOWP/ibf/svfd/g5xxYTj449L/x3/1H3BQDWqtZLgfowDGf5ORVSFWF/Lvv/4hn2S4/+vqYKGlp6gBbGIJBS7uMMNMa7xRHe0teP98l2SlJooZpWvLiYk46qMluUlRlukAYxXx3zc3VGKU7jQUSsGNXERjP/HDF7ZsJOEW6l1O+GuEnDTSa0X5G9vVE/m9lWfrw3i2XN2NMYGlWMcGogZcpweON9HQ2SmSEkwa9CBk9XtL80ZzyvRIWshquTwpJeE0tOrKiuW40aI9ahfjQgZHtDZamqzUQD5MdWQhEWmXWBjuT6g6UlwTUVuESRziraC8T8eX1wSjDBn2WdOmbijb1nffSC8iYOfyoJTwX32K0VNSPKqk3WQTMvnXN9dUYtQhxQwEtqu58LYFHpVJdYnItPsXIyz7VGl0qTn/hjGfP9wmuw26C4LETS3QZ0MyEuRJvpsId1PjKoGKLzwN03rERQDuVACN3GxFsNM3MEi5Mt4rtiBa6q/SAdmzRlazoOwN2aKW/sBK/Y70v1SG6EjYrWsjqqg8EeKpuhV/ChtSWQTG1hPDgJN3U7JcEPxlQ7VviS2Ezen+djZFxaCSxFH+XJOpCCVepZ7Lin1woqh1hEkwpUf7Vrqc9qkm+iLGpBNSYXGpboougS4gUYAPie2xGnmbHMvs4ID+WCb6uZFtt7LsVc02bdOdAKExRuNDbffMd8HIRXU/eW3Iz1cn/QtnOPsHUDvwWuNVTAUVtKuebKe/YmXAlIFp6PxXFkSNcikwLJcBl+FLRDIV5KPY7dvJKtqevtAD52+CNoppLuiiebdJtfCtslnnrOxXe8nV1QCl/3GCWgXg0vYC4pgM28cLTjOX86SefLob4uku57RgoYcfuKk7agSe56pnKOwbIa3mcrroO1bIbu1RketVcjl+06oDWpWf1RFJn4wWo1tN2vsG+2kJX4gntfbI2km3VDZS7AgJ9cHc+hhuZ8Lddz6JNIFrIpL+aS+hRnz4pbMOdDLFnOYJCEmJ70GJT2Y5t/UbfzenZspUg7FafttkHvgSZMDAuFDCiu7AZ3QjQCHLx65a78hkMctmO5EYAmk0632LuKWfd+di9r42VMFEdI6MbAWRt55+VNFC+MZ427vdPJIw2Ybs9LpJrgOyM/Ny+VR/SjOT9+p5VXcvvwq5T9e3zD2DxoUiKo7WExvTXDdQdwOnPr7t9ujuX+uClZqC2Hk9T9768nhlUxGs5Lm2itiFYfbVL74eMsq6yJBJQ2vtPQc57SZ8VgBeUwt6Fud9W2kQrR7lzByDD/K6SpZjrrgfUUw+1hBBpkaMGhdvKYU0toTNh7qm75/aVItIz2X2PN9LHWewG287PZCFexzaVmo1iP6SaKobnDhvfVQ+hRE4pYUeaaC2T+z58J6jc1tcYVA5T3tVDiZfbUu4EhLm8hmo9NpHvQhd221dLBUjfGassxLc1xWFMkN0xRz0glBqTjo1L9RYQtqnp+jm1dG8aJV2eodqG9vT720zMtq7EddUiqnWg75jlZiyP6eWlpgOiunuPehpIFUkXuGMTAUBephndvslJ4Y2kwdrYYIp26780eY0dRRLa0zracSSgsHaSOms94XVOvTckXFXbsWhHobx2J/dUtgvXsR4fm60MVTUON5AaDl233wgP8p3XEbgDj9224kCXtbCGgZZAni4sR2cFdjZC5xUEhnaSECxySYntfZYg4NE6kZx2zKJ4L7u/9aCrkf5KlRXfBJlKiXS1bkSKG/cL2e4uyTYvBVz28lbXvV8HDre9lv1+dvvY+yv7mhHV1468jR3q5g6A9gmzbz3G17VIYa2V57/93E4K+7ZM9s/f3k6Q9d7jq+pf+Lj/K7e/Apb/399+rqWwHsWr5RSlPHujjNGk4HT/lt2BsIIPD274+19+xMfPTjjYW1FXAeNRwenpNbNJzsMnF8wnOdNJzptPDtAD+UC3TnO5HuKyAKMdv/LBM5LDTJJhc8PNqxmDSSnAUnuUkZ5JlbZEk4rll3MJphk37I0zhg9WHB0tJNAGmD2+xcQWlQXE05KLp7vszDKc0+i0xVmFPaipXwxJJyXxvCS9v5YgoHnDepliv7shHjSMHy5xe1I8pncqwkEjrF3UojYBWEV8kpEe5ESzio8+eEn4ZI16nDF774b03SXxbkE4qiFwpO8uUYUmWBqSnZL0ZEN8uiGcy0XFJ47o1jA+WhMsDN4LeJq9d0O91+IGDn0/J9wvaMaOcK2g0fhHBeOHS6oDYVzNMqDdr7l+Nsc3WkBWrdm7t0S1inC/EDmcU4QHBebxRkBlbAluA+LDHOUUr2+mmEWAPy2wIyt9kwraB6WE1+xWJMcZduTwhZFKlNcxyiriGw33C9SDjGAR4G4kvbbZFblbc9AQrhTRQlOd1jQHDcFGsf/gFq896YcL7FBSZPEd8NmzmLVh/KUherRBN4rquKU5aPD3C9zAkl4q2pndhgv1/Zn1zDH87k0XxKHgZUo79LJNY4cdSImxKRRu4MAp7ElFsJHnl4ct9bsF9UFLO/Db+oD1u5bqsCW9kG/xzZOWZq9BLUKKRzXN/Yr0zEhX31VAc6+mTaHacTSHDeFabUG12WjWj5x4hA5b2pFn+ELO6+Awx3ZywWilCNcSJmNTCZGq7jXU+y3N2NM8KTqGCtS31iIzHPdJkYr6wwIUVHsiOaxnjmpX2Lk29diBp9yXhEWCu9RbF0sdTX5q8dp34TieatcxegaLb7XkDyz1pJNsDjzVXHoji9OGzSMrgJOOKfMimdy812AaeX0XCCMGUO45qrkkt5b7nmpHnpuftgyehVv2Z/jSU81AahDE46UryB+0xNcSuFJPJJnTG/HDFYeezROLrjtQuIbBmaI49GT3PdXMUxw4Nk8ayj1Heq6258nWl+Zhc8r2fPBK+lWLA+kQDbtQoHJfvH71WABPfigLBdkD2+2np5l6wpXvOgO7DsukA7Q9U7enyO67bb2ETcCm4iUrDjybU2EZAVbvOmzSA6hu/GayeGBjYRzrjm0t9gT85cfik2vGcmyKQ0lv9Ur+5pXqWE1YfmglrGjWbUckyanlnnj+JHwJ8nvdNWOoGL1y1GPxwm3uC9janGhZBAil8qLYEwakr/AINzJWzUDSTNGwfqRYPbkDpm8nt0ZrYSjxkB8JYN2cCrisdhTrB2xlks1QSZhTF9IjSb8CjAaXjvxIUU0V5Y6i6RjRZixqhzDzFHvqzgda+rsQp4Ewnc1IwKtufFdDche41QyERWuG4iWsJgL+oo1nc6plLGeSUlrsyfUvXnbnf7dfyUK8e/VEtm1zosn3Nc4ISxdtnAQvBWyDi+qJohkqNvcM9VhRzYSp90ZY3809I2AwlH2rJorswFDNZRttrKROZipSTBcoqs6vV+yobW+sV1CNpS6kGahtCm2batpOVhuvXAeouuMfa8pdzfDCsTkRdjEsxLvYDGQ7mqEmKB31xFDNZBFX6lRMt7Ain0tdO/L9ABtril3ThSppqokwjs1Y+hA3xwF992s1M1Rj2Ufd92+GeivfDLK2Y891N16aYi8kyFoJ5Gk85V5IUFjyg5BiV8ay3AsxlSXIrVR9GEguK2zSp3/7rstYC8jvqkBcJMcmWrRkxxFBYTG1o54EIvGdhthEYypHPQ2opyHFQYwpLfUkIFzV4pUsWsr9Oxav3IswZUszCalnIeV+hEtCbGzQtaU4SQW0dmE1PtDY2NBOU4JNjR2GtCO5y8KBIT9J5Po3jahnMS4JRIYaGgkEqhoZS6PwoaHeG2BHMe1EKkJsGkqibGhop1IR4qJAZPKDiHaa4OIAn4b4KJTwnckAHxsBUq3FrCsBoLHZMq2qqIXNBHnOMMGnkQT5dKDODWJUUW0ZSN+xh246xMcRbiLWFzce4uajLWDtq0nwHj8Zduyku2MGncePh/jRADVI5d+h3O8CdUTuuv19D4Z7EGit9GK+PTf3XY9lH9gTRfKvUpL02staO2mrPNZvf/bWfePeM5beum8A15/n21+F9/yM3pIn9/zv/fd/k9ebKbdZyslsxRefHzM+kuz7tguMeTi75fufn/Lo4SXX2YBB1HC9GNEuI5K9gqoIGYwqvnXwhj/8+IkwVQ8KDndWvHq+i0os42lBlscMBhXeKzbXA4JBK9UlUUv1dIw5zanzkHgoPr8qizDnEe5eifk6ob1XYc5jbOJg2jCd5SwuR+wcrrj9ei5gZT+nbTVRZKlrQ3uV4gNPeGtoRw6zW+HPEsz9HPtyIIzYYU7zcoibtKgswA8sNIrxyZrNMsWcx3gtKaRBqajfLeBKfGc+lsRCFUqXZHRl0B9sGCQVy5/soOuOfRgIWxgsDPH7K9T/McX++orq6xHqqMQ8TWkflJgXCe3YYQqNOymZTTNuXs5EzqggfJCh/3RM8WFJMqgpXw/xoWd+b8nq8znJpSZ70jB4GlLNO+YrFtlfM5Vux3pfgjqaDwrCn6RU71SoW+miDJYBXnvsUMJbknOR4jVPCvTLhHZmJQXViGYuOIsIV4r8YSv+u867AgLuxr9yxdXZFLMS74UPPcm5oXhckzyPtv62ZioXu6Rj+/QvLeH3p9gEygeSyjv4OsBFMPq1K5Yf79LMW6KrAP8kwzYG9SYhWiqqHUd8qyneqTA3IcOvNasPW+JLQzPxhCcZ+odjQFjn+rAluAkwtUjcmiHkH1VEX0dUe5bo1hCuFdkjSd+NLgJMV1niImEe62lXN5A4Jp8E1LNu0ryG/Ndz3Bv5ggpyqSsJVx07O/XYo5rk85h4AcuPLJOfGJbfFn9jcBNIRYiB4SuZzNczSC6h3JWxmfwoYvVhw/BpSLnrcbGEg9x+xxPfdF6azn8VdPLH5FpSL4uHDeOfhNgYimPL+Cv5Iu77Hm0kjGp6ocjuOdILTZDB5oFM9pNrRT0V5nj6ZefZSu8ApSkUw1eS6Lj40DP9TJMfe+pdiy40g9e6Sw8VcDV4rdg8EGbehZCeC9iqPyrQzxNJDk0d6WtD/l7N3v8ecvs3StwiYvqJYfGdlmBlGJx1vrEuIROEmQ1y6TuNVp0PrBSJYz1W5CdSOWITRbEvY9amnuErSWYtO5Ax/cKxfFdS/YXcAAAgAElEQVSjGpEq29QzfKmJllIT0bNoq/cdu3+mWHwA8Y3IDDcPJXBm/MKyfGI6wNRNnHcdwxeaoLhj8poRXZKsZ/KUbf9ofqQYvJFtim9Exjp56ij2NMM3jtv3NcMzAfHVTDE4FzA2eC0TcxdKKM3gzLN+KNs8eiks7O7HAiCShSc70oSZsHbNWN7LBwJURy+FoZx/3rB4Em77E3UtcuPdjz03H4k0eVvBEQkjKYy72vZK6pZtT6RynmgjlSU3H0mHrEhGJcRlc08zuJB06GglIG383LG5rxm8kfqK5TsBybWjmsl43vnRBMhWcwGA889bSTsNodgVyXiQe9Jrx/KxYXDuOwbSs36gCXKYfVFTTwPyA40pRQ6bXkpCrgvkc9UHxaweaaZPXScBFV9sfqjY/34rQH0i19fkRs6dvgYmWouvsNxTjF9ISmrb+QqVg2JHEa88+YEcn2jtyY41uobRayvJw4XIkE3jO/mp33Zb9hLY9FbYNhC2MN8PiDLpaKzHBhsrkltLPepAceZo0g5wRrLdppHzI1o78oMAF8DkWc3ycdT5k+WYRxthO8evpFqjl6I2qWJw1WIKRzsw+ADyfUO0Fka0HmoGV8L41SPN6HWNjTXr+wGzL2uqWUBy05AfRAxfVzSTzkjqoNgLmDwrBXzuh1J/UlnqWUQ70EQrCfsJCvEb5/uB1Ks0kjg7OK9wRrM+jRi/FBAXLVs2JyHTZ8L69HLa9f1YAnciJYnFRlhfYW5ljONbYbfqWUgzlNcvdyM5FuclNg2wkdSk9Iynsl6CbjYt7TAgWlRUuzHRoqHaiUhfZuQPhiSXFdVOLOnIm4ZmHBKUlmBdo8qW6mhIuGmodmLS1xnVwWAr2Y0upWLEJQHNNCIoWlQtXZa9TNXcZLR7I5GuNhY7lu+0ZhR0igNPeFtuw3J02dJOEnQt/ZfhbSEAcxRjloWAPSfVI8GyW3jvg3a6IB1V1Ki6oT2aSWrvphT5aVHjxgk6E6ApfkqDzrvXiYU5VVWN7zoofRLf/d95+TcrhIXc5PjJCJUVUssxHgqQ2+Tyc+eZ9IGA3G0VSVXjm+abE+ueHUxT2ZeyRCXJHaAzRra3rqXD0hipHOmktL5thZW0FsIAX1ZbVlV+9xYI/fPBPX8RHnnrsT//4T0n/t2/85/81F/3h//ef/WXvm/wcw4spx8c+oO//Z/x3uElizKVsJbJis++PsK3isG8oHgj/rCDd645fznn9OEVjdNc3Y557/iCLy/2SOKG1eWIdF5Qvhnyq7/4BT86P6LIYkaTguPxmovNiKIKqa5SmZzPKmxrsFWXvpVYvFMEUYv+yQj3vjCQ3zt9yY/OjiVA5/MRzX7L6YMrXrzYJbwMaQ4bVGGY3l+yyRKZwB9UaO1pVxGz4xVFFXK6u+CLz48hkPqMeNCwP9lwsRzRNgE2D1CRVKCMdnM2i5TgPMKeVMRpQ1MLW6uUp7GGqgzZmWZcLUbYVYRKLMdHt1yvhrgvR/iHBe5NwuidJVke41oN6xAfW+Z/HMK/c0P5B7sU9wWsDHdzmsagfjKkeacUz6ZVDJ8GVL+Q025CwrF4R9EQLA3BOxuq8wFqVuNqA6Vmdm9F/oM5wbdW5OdDglmNbTRh0lLnISZyJD9MGf31Cy4/3cMNHPF5QPOkgIt42/04ee+WxbMZZq+iLQKC6xCbiqcxyBX1XsvRPzS8+Z2WaFRjvx7K7x9W+ELi6dWgZfRn8uWzfr9FtYro2tC+l8PrhPjxmuJ11zVaCbNZHlt2/0hjY1j+tZLgaUJ90BJdBtQ7Ek0+eB5Q7gtIsIknPdds3m0IlgHtyDJ6FlAcOoZfa6pdT3XcEp8FKKu2srp6KlLQeubY/yO4+E2PqhXqqMSfJ9sky6CQx5pSAFI1l8oMlIR2lPsWHzsGz0KasTyufFATXoYiXTVIj6cSUOdCARDBRtHMReY6/RxWT6B+UDP+s5jVd2sGX0RbFqoZC8ttSukujG819sMNPBuy/8eO1/+WwywNLvaMnhqGZ47Vf7iGP5pS7Yh8s56I9LI4tgQbmUijPekbGTuzCroETN/1UYq011TC3hQHjvFTTXbqaYeO4QvZns1DR3KpMQWsvtMQ3AYi684Uw9eS3OgCaEeO0TPN6kNL+tKIZy8WEOmNF4DZSUVNKR2KPWMcrgUYbe6L3DO5FHBU7QjY3/3jjjUZyRiX++Kh7EF8dCuT4uGZ+ATzY4/vg2O6oBzdKNI3ivUTS5BpBq/v/Ib1TBYgTCnnw+aRJX1jRELfdDLc8M4D6kORKadvFMsPLdPPDNm9zst45ckPRGadXqpt4E56IcC62BNGNbkSls0HIgE2tQDDfmyTG8/ttzw7H3cs3kL+Hm7ouhQhuy+dloM3UkeS3HSAuJNwDs4960cSNpNcy/PTcwGp8Q2UB15Y3KGMXZsIQ6ctBFknkfWyyDB5KnLO6bOW5TsBQeYpDsUH2Ut60ZI2WhwKyO493dHKU+50Us5cpKmbUznWk6+kx7JNFOVcE5QCUFaPBMg3I0V8K4B4/Ey2odqBvR9Y1qem6z7tgk2qO3nq+GVDMzIdqyd+U9XC5EVLvmfI7ilmnzvyfU286Oo6vGxHsS/nQXop7OzoTCaMm2NJGh2euY65g/lnFbfvxdJL2fVfJreO1QODtrLvpvKUcxnXeCkewtUjAcnptSU7NrSppKH2vsnxq5bs0DB+KX2G1VQqgKK1ZXUqi2T1RMBlkAswN7WnnEtnY7zwDC4asuOQetR5MkeKwYWA2PxA0kUlkEbqZXofpXKdZLUVRi/MHc3gzleYXkm6abxy2yoQkaELQ5kdGuKV9GymNw5de8q5+LSDystr176TfEp4TFAK8xlu7LZiJCgst+/HTJ63VDNDctWwuRcRFp70ot6+X7EXkF401NNAxmqsSBZO6mA8RIuaajdCN55wJYCsHXQ+xqMYZSG9qCj3I6JlSzUPSa5rqllIm2hGLwrKg5hoIcfC5MIUKg821rhAMTgraIehbIOG5LoRyXEif49vGrJ7MYM3tSxYGkW0qKhnwk66yFDNA6K17H96XlHPQtJXGfnpiPimphmHRIsK1ViK4yHRsumAeseGX+ZUeynRbUUziYSFjQ3RZUZ1OEI5T5A16LymnSbiibTiD1XOUxzEpOelpK82TtJaI0OwqaF1qKqmOp1LZUvWYCeR5AkEivCmwEeByHPLFhcagqs17cEEnTcoK35FWofqA2kQGasPDXqVC6iLwjv/JIg3MitFktpa3CgVWa1S2/oSvS63Ula3O0HfrgVMrjL8MEUVFT6JtvJbQLyUgTCmerHZymNVa0X6Gglw7d/3bU+ovIAEBfm2FfDYtneS2E6GqsJQeiqVAtWJH50VINkxsD1DqYIAX9f/vL/ybV+ltd+sNXn77+6bz/PNn/Oh/N+4/SwAy/TdE//u3/lPf+qv+/G//1/+pe8bQPCXvQH/b26JaTjZWaKV57cOnpLZmJ1QwmY2dcxeuiE4OuPzmz0Ohhuy/Yij4Yq8jfhwfsHLbMZ0VGC0Y3R6TdkEfPi9cz4av2FZpVyGQ47Ga5xXHIw2mInjTTKmqCJO5kuskw/PKKpYlLJ0PUsKvjSe2aBkEDbcVgOSuBHAW4TcO1hIeulOwez+DVUbkFchWjsOd1acOcV0klM1AbP7OdYpxoOKTR0xOtwQaIf1CqM8obGMBxVKlQwPal5dzTg5vuEmT9nZW3NTzdjb2chnM60w2hFox8VtwnAg/59PcupBRVFELPOUg+mGN480eMWD756hlSeNGtnXHSjqkNvvacIsoXmnZjgryG5SRknFTTWkeVgxGNSoYYVSnk0zwRcB44MNs7TkzE+ZjHOWk4F4z3cqjnZWXCxGmLFImr/Ym2C8IpjVzCY5dWvwQH2dsH+04PK7MNMOdVhhlKcetASBxR5UWKu4f3TL6+speq/CBJZ43pApGM4KqjLEK4+2mstfMYx2cvIsxu026NMa6oDhYUa+SgiilvX7LYROwKfV1C4mCiz1vtRNMG1IhxXFmxFlIsE4V7/pwCrCwNI+LtFAHXhM2uKtorivULWiOZULdK4j9KCl7Xyu9dhjR5biNwuaPCQa1VSNEo+sVcIG7zbYVQCzhvO/FkBsYezBKdRBhV9E+J2K5jrGlAp7WLMOQ9xIGGoCR5FKgivKUxwZfOhoQo+KHDb2uHmDDh3l2GAWAc1YGE0CTxNqmDQ0PmT1jqE6aAnjlvw4QoWOZuop96ULlNBRBR5Cjyo05f0a4zRu7Lj5lsEMK1xu8EPL5jFkDxQ+izFzjx1bKqPxgae4Bz52+EJL0E8uAFwPW2zocXlEfdiZ+rXHhh7bKlxgsDstWRvSjl0XXmQoDsSPWxgB4Sp0tCOp8Gimjk0ggLM6bCG2FHkkYzTWuEhAkh06vBIGaVsNsnPH8OtcUi/radep2gqAVFYWFVTbSx9FttsOJQW0SR3WKuzAUSae+NJQ7ggL6qKOadYK8LjES+LoQdfdGnmKQ7Y+2Xbgt4sE0kUpjFs78He+3Eh8tiiNriV4J1rIAku503mLh54s7HzA2ndy4k42OZYe1DZFUn4ndLUNauvHbYZdZ+bY046AwFPsiS+2msmYhRtNtefQrUjwmqknR4Jb1g8l9Cq+FmlwftR5um3XVziSxRYXe+q5jG89VVuWDUTa6gPf+fCEkYcuMMbD8nFA06Vs2vgOjFdzZMIcSCepDZVIo7teQUngZMvUeuPxXpEfSoiM19BMxJNVT+U1dK0ksdRLGJEAX0Qiu6+pO7lvPe37HOnSI2H5OEQ5kbX6zr+pQtgciYxSjpuwif0Y6LrvVJTHV1NNM4FVJFJLYWGF+dRWtvn6o4R2IMeybYTJaobCFvtalAHNQEk9SGd76vfXhZAdmW39hmr7GhoBdC5U5PvSi1hPBKy3SUAzUfjOYys+Ukc91LRpLz/upLcu3NZ5qLZb9JoIeG/TblyNlx72vv4BOY5e6a2f04ZS/SKeQKjHhnaoiDJFsdcV3Pe+2lrqQVzXEeo1bE5EzaItNFrhlYBkG6nt+7VdV6TXRt4zMoS5nBvl3HThRWbL/tfTQNjxLlG22A+7bko5tm2q0FbYUBfGtKnIsG0Sd+eDotyP5fEe2qEwie3AYENFNQ+xkYxdtRvTDDRehYSZpd6JxJuqBVjq1lPtxNhY08Zquz+uS4RVDqrdEK+gGct4iXRYrpPt0Gz7U5uREe9npGkTTbWXdj2+gXyO0gA77ZjiSG+DdpTzNNNEejPjAB9o2mEgCbUHQ1wk8mBvFCYSMIrzXUaCAecJcunidJHuZMPgQ007jTFZswV7PtCSvKpF/mwaRztJ5PwxChdJAqybyJzPpQG6keRX7ZzIWt9iIV0SbIGWSwJ0XuOSEF0L6GSYdH2/VuSvUedh7Dss0y40x2hoHX4o7+tnolrqPZQetp2SHgS0lgI2lXUQhXitUVrGVdmOAQzeAnNvA7/e89j/3fSpxx3jqLWAy7ZFGS0yVUAl8Zbd9NYJo9kFDf1zYk1DlxDrOyb0X0B0vZ0y+1e3n8pNKWWAPwJeee9/76f++j/PjOXuR3v+P/4f/nWeZrv88OyEXzh5xXfGr1nbhH/05gl//ehLAK6qEX/05pT/4PEP+J+ffZedYc7lZoi1mid71/z41RG/ePqSi3zMi9c7/K1f+Sf83c9+FaU8f+PhZ/xvz9+nzCNcJsmiR8e33K4HvLN/zavllLIK8R6Rr1YBv/zgBX/26h7jQcXp5JbvP7+PNp75NGOVJVTLhL3jJctNQlsFTGc5y+WAIGppq4DH96746rMjfu17X/KnX5/SFgEmbXGXCX7UMtmVGoR3d6740x8/BuD08SUvvtrHTDsjtlPszjeM44qvPjuCwDN4FpK/WxMNa5rzFN8FyeidGtv5MVWjYVYzm2csns8kQMdJb5vJJJb+l77zlD/95BHRrGI+zrl4ukuw0jRTS7Ax2MOK0bTA/sGc/L2Kh/euefUnx9ixVBWgQK8N6qjk/t6C5y/2UIHjwfENN3//RLrfFARL+eJrJ5bkPKAZOdx+zXSWs/5sjp20/Mq3nvJ0scPNxQS9kgmSD2SJ2Q8s+ycLrr7cgWmDzwP2ThdcX4/wrYZaE6wNw5eK/F+RKo/6JxNMobYSQTdqwUmokWpl0uoV+HFLeBESLiUkRu1UBF8n1Dsikdx5/4br53M5FqnF3AYS5hPLpCu6n1FepYS3Bv+owFklKa8ji5k2+MsYN22IXkcEG0W573AjS3ATYAeOINcyqfbCLoUrjbJ322cTmfwHa83wlaI4gHYoYx/dCrMa7BaYT4Rx/fbvfMYff/oYGkVyEVBPJdW0OHKkbzTNVFb/7TsF/qILcPIQLjT1QUuwCEiuFdmplfGqFO1Og14F25RcGwt46nsSdSOTnmipqGZeUmfnIj9NbiTYph0Ke2eqzjPSxd5vQ1s6wOJSObfMKpBeuYWi2u3DWoQJW3+7ZvbH8iVdHPR+WC3VDyORjArA8fh3ctSXAykrX/dhOeI/VC00xzXhubyWriQgKsgkddVFwu5VOx47EbmzCyUkSBJVHZMvNIvvNUQXMrEOMkW157Ajy+B5uC0s1w1bKXAz6hJioy6Rdcy2KgAlvs74UpiTcC19fvENHcgTJs2UClPI46tdCamJr6WfL8gRYGyE4ctO5Pgo30s87wKM6h1LdGMwlQQotSnEtwLAoiXdOIis1NTQDAQYmYqt3C5eOrIjTbT2bO53Mu6x7G9yJcfaawk1MqW89vBVl3S6EXmqjUTSGK7ls3DzXcfsE/H0Dc9EIhxmntVjRXoBwzeW7NCwfscRZJrdjy2b+4ZmJPUSzbh7by1BQ6YS72UvSW4HnZT4iSa+FaAeroVRTS9F5qocDF9LeMnifRi9lLAgU7ENuJl9btn8n+y9Sa9tWYIe9K1ud6e93etfNJmRWa1LBdgjZOEpJURn7BEzhG0YlpjSDRBQYH6ABQMLhgyZ8QsQsjxwVlVmVGZERvua259md6vz4Ftr7/siKyttyJQzSrWlp/vuuefs7uyzz/rW1z1VMG3yrDqep913ybpmhjB3AB5fCFSXPM+6m99z3VLKag6Uuw7bdz8n5sB+xnEjoLo4gRIRE2ubpL12QUZRD3FKTq2vPfYveH0W95QlhyKFD93HSUraXUjUV5Qx58UcYmLLuV3dk22NklLbUMz+UwD0aibfZ7Ejo1/dBfjkbexPBeor/l7feAxrNUtk+4jyniyrdEB3xv5FBu7QY8l7CRnN4xP2cubuQoBMcXXr0Z8qSkEHSpIPzzVUD9TXDodnGs0lv5v2zzWWX3v0JzLJ9PlemSQdbS802sdMop3CaSzloVmqOi4JutXAICGz97Arpq7WlyOEDeielNBH3rf7E4XqloFAOTTLNUxitUsF3dIbOawVynuPca0mBtEm4BYlj1kNIVUaCYKMyHVlBYIaI1THapDy3kMN9EQCQHk9YtwWUGOAa5gsG5XA4tUAVxMMiQj0JxrFwaO4GXF8UaG+sgSJPqbQIHZQSsuAnyKxrYiUOzPpV0K4CLMf4RaG8tyjhXABw1mVrCuRx6MFZO/hFhq6TYE8LiCUBEbF5RHDkyX0wc4eRC2grzvEBCRj8myq4whfG+ibI2KpYU8biMh6EUgB2Y5zr2RtIHr2STIki2xoLDTEoQOMRliU0zblcUAoDUKtoY7j1EOJALKdIOMY65LJrUbTG5nYxVgZiHZgMI8UiFUJ2faTZBYiH4d8pwokh/dASohhZICP0WQs8/LNUJ2c5loWiPsD11MkoJvXrzV7LKUCYkBsO4jsuVQMI4qpRiR6D5FZySxtzb/ntNqftzwI+fm2h/fUHz2L3/mHf++Xvt4/+ff/23+hYxNC/CGAvw5g/VfA8hvL4ntP43t/9PexqAimCuXRjga7fYPFomeZeMcOte3THe6/3AALhzhKiMqjrC36a3bP+fsCkBFqZVFVFsf7igXylWfoThD05pkA3Bv6GEkYAABTWxUH8/BiSllF6QnYdAQOLKSXpUc4sKYi9gr6XsGt/JTampNMVSsRLkbETkPUDtFJiFYhVtyP2CtARFZFpC8EbCxrQHREcaMYXrJNN63KI3oJ/daw3sJm+QJngWPFYzKXBAHxxEKoMFWRyFEiLB0lvE9GiL1OVQgET/KogPMBcVcg6gB1UBCex+KbABEEZM/OuOItB9u+oowvl1DHMkxJtIhAqFlWL08HiC9qls4fJIN13hjYCwt1ayaAFcrIIKCBzItfsaJCHSRCmaR+KYlu8o6qCHOnCCRXjgnAgeezvFYIJsKeBMiO/W7jllJQt5zTH1XPege74fGYncRw4WHuONAMBYGdcALVJVksBi7QP2oOZB9cEyefpfCp9mARYPb80s/ddrnKQwTA3AvYNUFUMATWZidZsfGavqhxTQmqcGSM8msRAUSRJJxiAqsiJZVm0MOeOEowzT1BSzCp5uIOZP9KYPG1QH8eoQ+ZCZt9nP2LkRUjO8n3O5LR6h4TlOU0WEp9CTBz/1kONIqCYMU33LbwDI9RY6qwSMmWPgFs3YkphVKOSOmvlEj6iiXuJvUbuia+U0pfvxEp7CRO58HXvIYyCLNLDvBkYkyCmcGg7oFxzWMrdgR4uk3SvLSfwzaiusmJrymN0yaGLDGgmQkqbzkwt6vkrxwwXQfSUraZE2T1kecsFGTCcmUHB5lMtgXmzwGASYobJV9bvyErqDskRofbdHVOCU37kOpGdJfYrDL/Pl9D2W8JYAJLLqW/AlzPsOHvvgbKG1ZYIHI9viLw89WcUmuOBI5qIJDpHjGUpzsXU3qt6ngO8qIGglERkdbN7TSvCcpUP9enZBaNic4MS8lT7nkATrCeAEwC0tJR4grB92xKJX6Q9puBY/YS2oVAeU+mzVes6uB+xokVFp7BObbhfYEySTmdU3Mge8trnB5Jm8rtfSEmoMawI4Hi8LMJrVHNxxQlaznGJbsipSNApBeRoS3VHX2H+ZyoMSWwrsnC1jcsvXdNqmoxvJayZ9W0PE5bc/9cJabJALtMfZNDTCnmIrF+DNbRfUS/JbCMiuAwB27xnMSUniqm3zPjlwEtjysxz4WgP3Ngqqs5BggXEUqCr6AF1BgYwmNTiNc+1XA0vOZyIEuxDw+SbgOGjZpCkoaNRHFIHaw+wpeSgUzHwOuhJ6jKYwtXyyS59qwj8YAaAhCAULL6IyTApjsPlxNXFY+Vk2MevpJTSqvuPLsh03XlaonylvLWDOikDVNarLRhSpaVNkJ1gfcJzcAg9kUqmIObzre0BHu+VNCdo5dxCFCWiaxZkpr9l8GQmcznRLeO3ZCpg9LsRvhKQ44zWOL3gUcoyUSq1kGODqHUPIeB28lyZoZi8VypztITGSLXGSNBok+Tx0pCOnZJisHPfZAP1i9ckr3GOAXtCO/TOshChqqAHFMvpWFoT9RkE6Ek02ABriNLWTPIyuArxIkFFA8qQWJVEBhKSQ9lToBN8taoJJnJPM5/mDybfY+5p/KbbGGu9Mhg7xsdkkIIMpLDAJQlYEdKYh8+L8tbM6h8kDo7Ldm7KeTESsYQIaRA/PMYTAAI/98DfH5dgOWH//MvH1j+6X/wi4GlEOIFgH8M4L8D8Id/BSy/sax/43H82//7H6DzBndjjdu+xtvrNcrK4veffoV/8uVLFIXD2aJFiAL7vsToFLpDieWmQ1NY3Nwv8OSUEY9fX2/w7OweX74+QdlYbBYd3rzeQsjICo8gsF51CFGgbUu4VsMs7JTIJJWH6w10ZVmeHgRg5SQ3bJoBw2Bge+awqyLdKERMzweK1YjxpoI56WFvKqAK0JWDazWWZy2G3qAoHYHvzgAbC106+Fc1zIsj7KARWq7fbHu4qxrqdIB3cpJsYmUhdYA/GIJZKyFqB1062F0JUTFeXaiIMCqo0sPv2cdY1SOkjOjakvUob2vUzw7TepfrDvvLJYHo0uLR+Q5X/+wR3DnZK3ExwA8KzabD8NMVwsUI3BZTpyMCsHi5x+HtgkC40xCjgH7UwVuFeE85IqLgfh40ok79iGcDTOkwvmkASTma3im40xlEiyAQBWWZ779/ic9+/AgoA4RmeBHKAAwS9UWL7rKBWlv4XgGDpJf0osXw8Zp9lSvWo6iDgl95VCc9+vsSZjnCDRripiArPArgYuBkghWIZQCcgIgCsXGAk/T2FemzeD4gBoGTkwNuPz1hn2ETABWh9vQiyk7Cbx3MJTsVAQ6GM7iETl88jUO1HDF+tcDpRze4/dMzRDUD4lxujZCAYB0gRoGwdtBXhvKuZUSsPMx6hH9dJ0kbgZ3beuj1iPCmguoF7Dn9oMNTVtYIJ1Dc0BNpUxIuAJSXCuNpQDwdob9mB2iUQKjIjJq9RP/UTZMuIrDvMyqyGeZAcB0MgZTwAsN7A6qflrArzvALx4FmZlTd2kPvUoXMivtSXip6TkEvadAE3cWdRPfCQVgBfZRkFM/mCYXMapeXEtFgYtPlVUFG+k4SUI8MRpJWvAMEo0xF9Aeu1zcB1SsmDLrkrwwmJgDEbeXrIwdShZKglb2iMwAFCIYzkJSWrKqvmKZb3HJiQ/VkaqUHEAjCfRFR7Ak+cvBP+zQm/yoQVIQaZ5YyV2MEEyfQbQ58PUFoujY9wXhQCaSK1M9Y8xpUPa9Hu2DNS/uMEywAr79cUSPHGVT7Eij2wLAF7Ipe2VCyZoRMIpIEM4HlBxUnCHxtdxHRvCGTHAquW3fcn2BmsM3UUkwVIexUnIH29HufgQ+Py5ecyJEjE3gBTAE3wXDdDPEB2icRzSuCgeGUx5EBeWaDdZ+lkjw/xX1Mg2WCcF9h6vx0S8DsyEpm9ZpwDNbxJUOR6isGVkWVko4jWduYrxsPFHcRuaYkFFy/6gjY3IL3HdXRf2lX3A+zjxjO6B8NiutyCwJmnlcypMMpAWixS8m5iucyT26IgHf8mVUGy54e5Q/SZC4AACAASURBVOZNIJjWBERkKHmMuXQ+aDFVl0jL6902YpoEynLOXC/jaoL0/kySPdVzRYxb0OvL91bANsnf6rnemGS95hinShbd8adKEyJBc3+jBMp7+jtNF9BvJeprJuTaRk77VN2GqbvSVRLFgZ2W+XW2liiOnDTI4Fh4Plf37Dt1pYDp6NHWfcC4IrhVXUB3zsAjX0h4Q9AOQbl3cSC49rVMEzG8V/qKYFsfPcFmBv+5tmWM9EaGmMKPmOwpfWR6a8NgH9eoJFGV03akpWx13OipHsVXKnmY0+N9AISAPjqMG8NQoRjhlmaSPRf3I0LJ1FsRI4RN1RsxphRZgmK7SmAuAmY38jpqDPQhsYk+IAoBmfokc/2JrzX03cB74AMQ6GvDGhTHfYQLU60JQqAMtrcQg0M0D9xoSjBoSAgC/t5i6qpUElFKMpP5NVIQKGZwmlNaQ5j3J78emAN7gJkd7Jk0O4FbIfi6h7Uf30xhzTLfFN4TXQKp73Rhhlkam3yW76zzAe6IGTCnOpJfuIT4l8Jj+SsClp8BuHrw0D+KMf6jh88RQvyfAP57ACsA/8VfActvLNV3n8e/+b/+Xdx0Da6+2JLhM2SsEAGYCHOtWWOwtMDOsDB+QWASU8AHwIGRaiW/HCTZDwjALVlvYHYM0sishW7F1EHFLybKReUophAUiFSG7NIAzrKr0FURAmR+MhNAudHssRGBkjOWfTN8pLzmzGWWwfDLOrFwkgNNaTmArK4EhhP+HhPIEk5MheOZpWCoAwd8romQPg0u7zmI0C2Zlsy4TLPbRUR1yYH77iPKJgEOeKobJm0GRZljZgcelm67BY8XmAdW3aM0oC7mQVWWn2XAZA6YqhRyXx8HaBx45IFhdzEnc2YpXx6wQHI/fBqESjefj+GUbI5dcN0AX5uPwTV8fu6MywOekMIh8rqkxTv9ZnZB9gKgD8tXTPN0DWVaKoEC4Xk+y1uJ7Z+FSSKmRmBcAcuvWK5eXbNgHILr6S4Su5ISHZHe53HD9RZ7vsfmOJeGZ89QUGn/0+y12XPAZw4zq3l8QTChhjTgHGdWJhgem+6AYcOB1LjmtZF7FEWa/HQLMXUZZrlnHlBSDsjkx3GZgG4anGQWJ3ubAKQycyTWigmjzSue09wlGDQHu+aQHk+9gK6i/LN5/S5rA3DQW+wZ9hKVmFgImY4hA/K8z/k9z2EjU8/ekcdR3eU+wFQDUs8sKuV687Uj8uStn7djOt4Hhi0H1Gp4dx9y4mZxCOhP5SQ3DYqJmlGk0Jo0iJaevq2oyNwETbbELvghqG5zeXr6rKcOPTXOg3Q+DvrSHAesxZHVElmqycASShnzsapUQJ+Zpyi5j65K/rU0ALb1PBDPhfI5XIWvI9uYz2UUYhpk6yHCG0wD6dxf6UuB8s6jPyHrMzGBIck0BWAOrGHI3YXcd243D8rJes3BRazgCKmvMLN2ZHuk5znjdUymSXcBdjn3+GVp8MNj0h0BIWJmh8ni1JcjDs8LXhctAU7USMeZWMM+dR2OYUqpDYWYEnwpfUzf+3G+DqVlqml3KrF8zcol1nzw+WRgOPjP0k/hYkqoFFA92bAokCZI6NXMQGE4MQy18Yn5TYy8SBJZEbK0P39niWn9uicTlz+LkPybtAF2pScmTo0ESNIGhJIgJTNQIgWx5EXaMB33cFIwFfRAZg0B0H3qd1S8xoMWE1ihyoe1GNKSKQyGjBmUQFD0kOrep/sYP1uqp58slGTrxrWB2duJZZ2WNEgX1vNnYviCkZCDh4iR61RikmdGISB7B78wDJgp1OwrBaZeRr8wEDZAjvQc8n5DBs/XhvfkECn3LDRy3yMA+EpDtRbCB4RC8zhGD3UYpu5Eed8iVmZi5+ACYqX5mUnrzgBKWM+/xfw+EVQBIEuoZ6+iHB1CpacOSihBz2DyNQrvGcizrCCGB0mnAWQbfQS0hOhGQAqERUkJqhD0IlZmAjxicDPAipHgy/mZQcxsn1bcVghkG5MnUvR8TAyW743R0/+hVAK3au6hTI+9k5IqEnB0bgZ/DxlOIQBr5/7HskBse14vRcHX5c5IIeb1OP8uq+jcu2xlpNd+YjIzaMxLAucTG5n3M70uDsN0HKIoCCrDDDR/ZvmLwnv+vOUvCbD84H/6+7/09f7wP/xv/sJjE0L8OwD+IMb4nwsh/hZ+RcDyWx3esywG/NtPfoCP2yf4pDmiUg43fYNaW9Ta4tPbU2ze6yFFxOAV1OOI22ONQgDOSZSFg1YeIUgsyhE+CtwdGvSHAmbdY1UPuN03aIyHUR4uPS9EgXYoMPQGdTMgBImuLXC6PeLmboHt5ggpgLtdA3cwWF4c4XuDJ6c7XO6WWFcDjl0JqAA3aigd0O9KiN/tIURE3xVYLHv4UcONGh2AMCjIDzq0hxJVM0KIiONVA1F7CBn599IzTfZoMF4Ai/MWx+sG1baHGzWZNxmhSo9gCVKlDkyzNR5ae3T7CtVygEz/74NAHBSG5xGy9Cgqh6YacHe3gPvNEe3lAqfP73BTbSFXFqZwOJxUgAD00kI3A56ud/jRT58CYIDO4apBddJjd5c6lrxA3zjEVsP1EvF0xLA3lC0HAdEqYG2x2bS4/3KDWHuITkFsRoQDfa9wApBxlipXAePzCIySUuGNRewVROMQDxowESePd7j/yQnCmtUYQlIvYzepssUL9oo6CQw8d7J2lPqqNHkh2O0YLka+D71Gte3hnYS7qiC2I+KuABYO/a1BNBHyZESwEq2VgBUQS8fEXRHJYlYe3VKgewnuuwBZThnRfkdAFB7tZQG/JQu8+x0P0ZPNkz1TM/3KT/sHETHsFeTzDru7EnLgF0ZUYWYBL0bgqBGLAH2r4bYW8kAPnd2wqzPelIgFJbrCkmnwKw9RO/RvSzIBpxbqXsOfjBw0dgqtFay2qT3fSy9gdoA9DdCnPcQnDSc17BxwYw7srpQjPYXRRMhBQrUi1bQoMoVbj+Ka1QPjY4vhVBF4JaBGWWAgu3piWe8iAPt4hGgV3ELODOI0mRQYeFRGhMbBXGvojsyi2XHiKNQBwgpKnSMDfsQgp4kcfcygi7454YFcKJ5fK4dU/7Mmw0qPJj3NciDLJxwH2FFFhgh5kRJ203M9oHpWstgNJbCIKSRF018KIPVOJun0Xk4st91wskyOEqojG2uXEvVrieE0YvE1GUv6TBkEJBwDhlxNXyxCeo+UnFJwXQNIqzCcBhQ7sq88vzMwLHZAfxYZ1BMI0u2aCbvDGRlH9m7Sm6s6yUmfVWIw00STL5lUXL+h5FK3YvJ8+ioxRCXoE+8NclCNbikdrl+LSabrFhLFvcD+PZ2uQUyTYgAQJWXroZiZSk5Kzd4g6QE5cJLSLnhMuSLG1TIlvaZ9OJCJrC+T7LjnY+1zTtwFPa/TVcD9d6oUSBIhgkZ5nfeL+2jXmHyKw0bPLCs4CSEtJ3pcTbZRt5QgS8dzmeWiN79ppkmOOYyHMmyAUmUROUETDLfnS04KBgXUV5z4CkVigDsNX1NqHAwn17iPSYbfRfhaTBNbeQJQuNS12Sfp7DEmTx8fr66Tzyz1dA4ptZRMY6o/SeA5Hw8EPb8icBIgV8xECcSnGq6mR3P3vp5AeFScnNp9YGAOnBhTA6XIxS5CWZWklhq2yXJWIMq0Dh/TBJSZ2OrMxLaPyBgOK4nqjveCYc3QHHo/wyThlS5NWmgB21AG7IsyVcVg9my6EsEQECMC45LsZp5EGldySq/tTxX7OsuKEyAZvKNONTqc1ECMKPZkD8cVr28ACMrAHAtKwBcK4lkKl1FkiKdJw57gW/czG+krMpfDqYE5eLiGzGiepBCBoJ1M+gL66OEaBXN0TGgdPOxSQx/9NDESjIS0JdQYpgkHCKQKkwIiVFMFCVDw2ABACJi95cSIEhBjmCdAEiCeliSxVb0DmmIC0aFQkL1FaIppIizLaBkO5SmDfQCsoSXgAkHvYGdADlCyqnKnkSQoVHL2R/rA75Wc8Jr6LaOixFdYNwF+0Y98vCwgrKOqP8l432Ub43Q+IAUwWgK+LI3V6aaUk2JDQIwjIDQgJEQO2cmBPBmsG/MucP4mqSXELLv9Zg3JgyXKn/unv1p+8fJvAvh3hRB/AKACsBZC/B8xxv/4l7mRbzVjWb7/Ir73D/4Q4xOLk4s9XJDo/2QLfNjCWQUEgfKTkob25yOqzwuI39shRoHhVcMqir3B8tER7p9u0b8/oloP6C9rqI2F/LzijO9vHOCsZprpfYPixzX65xayVcAZZYvYGcoxn/Uoflijf8KCdpgAWXmETqN4q2HXAfq8h39dA+cDgpOof1Kie98CVgBlgNxrhJVD9UWB/ombwJDaWPhOoXhjMF44iF6ienbE0BvozyqMj0mTFW817MsBzZ9UcDVYoTFKqKVF9ALBUt4KAEJEuL2BcBJRM2HTfFIxNGXroGomro6dAXaUcjZfSxxfMtRlOKd0Z/HeDscvVwyQWfgpnKf8QcMo+H/tANtrRM+7QvlFgeEZw0tcHRGqgPJKofn9Gxz++BT2JH1rFYEAr3LsyHw+YPHHFbrfbyG+rBFlki0+5wy77CXiKb2h5tOKaYZrj/KtIkv4loPe4SzA7AX8hz2qeoT94zVEEBieWsjGQX1ZwW48qte8ifZPHEQQKN+oWaL5eCTgjED9lU7F1GSjV58CN/+6w/LPDIazyDRSFSFGieqtTEEwEaGOqL+kD9I1EfbUY/1DzZTJgh49f2Kh7jTKaw72hedgSQ0C/VOL5U8M2mcEK+FihLw2UH1iQkdgOKeUdP2xQncRMT63kHca1aVE+x0LOIH1x3pOcvwe60LaDyzEIFHeKOgDpXVypLRPH1g5YfYC608jbn+bIG/9Q43ji4jlF8lrZ1Ny5CbCXtAXS8kUgeTqp8Dd3+yhP0s9mUd6+na/RYBK7xhZ9OyzlCPQX1BeWt4wpEdaYPk5sP8w+StNhN5JqJHeV7eMWHwhcP9bHrEIWP+JQdSpG/Io6AF96iA7ieKOHZ3Fjkxpf8YKk+qNhFtGFLesD+ke8VoaNwGLL9N74yh57B5T1uhqgqPlVxGHlylk5RjRn/F57bMZQIhI5ndcc5u6E+jPKfHUHVBdkQntz3mtmD0H7nbNY1h/Atx9H2hei8mLOJySuVfJJwkA+w8CijuJ6pqD7fYZk1b7C76vxQ7Yfd9j9WOF4Syi+Rrozzh4r64TgzYQCABkxatLMsjjhkqJ5jUHlb7ic8w+pbgued1XbyS6px7llYK0ZN33HwCbPwMO7zPA5vA+oPc8Z9KT2Tq+4HO6C4Iw1wC+YJ/l4X2g+RrYfwAsv6CctvlKoH8EbH9ENrc/J5hafhVS8itwfCaw+inloeUupkAWqgDqt6mTM/B9U2Pq4HzDHspcKcPPKsNaXCNweCHgFgGP/584MexjYo31EPH2rwOn/0ykahSy7dU1Wf/9S4HNpwH9iZxChHSf2M3EQBdHMqK799XkH5YuTsoHX5KN7s4kqlsyuGrk+QqK7HBmR9XA9Y5LAVeTrQ+GAHHxloAnV5SoPmL1lcPNb5oJbOuO50539BaqMeL+A436KqC+djg+MejPKFvtTyl5bK643vrWI2hWsSy/dohCTB2ex6cM2alvmQobJT8j/SlBqe4IHgloxKQSqK8DDk8VdE/1QGbCi0PA8RFBVLFjUI5t5HQ9u0qg3AVOCNQCzaWDcBHjVpNRT0z98bHC9hOL3Xsap3/aY9wY9lYe6TdVNmLxyiIqAddIyMQ2ByNg9p71IdduAiyukVQc7An8MstNtZLH8alBc+mgW4/2MTsj1UgWOPsZQ5Kg6lSP4guG47RPCCpWn/VwC81AnvOCrHk6p9W1g10plHcWYiTgCimJ1ReUi/pGU6paSXZ4XttJEmp2BCy7Dyp2btYKQQmU1z3cqoBdKdRfd+gf1xNgqt7y9/rLA44frlC/7jGclijuKEUdTksU9wR5bkFwbvaWDO3gMW5LblcKSBcQtEQ0EsIG6P0At63IUGbGUQnYRqO66mHXBVnfB8y5Og6UD28qekOVmCS0dmlQXraIRhFMthZ+UZBtVZL+SZD1nbyPiUGFSB7VdiRo1Io1IKWBut0zsXUYEauSAK0pyZ4aPakEZJu6L9ueXZaHlnUj1s2vL8y7dSOjJbNXs/sSPvA1bT/LYKUErGWv5EMG8oE/MsbIFFnnmfYaI+Iw4mFCa+x6iLoCnEP0IXkvGdITx/FdsCglos3Sn9lPCeAv9lQ+XGL4WVD6L7H8OjCW1UfP4wd/9MtnLH/0t//rf+Fj+yvG8ucsdT3i6d94hfOaCRCvj2v89r/1I9z2DW66Bh9ur/HD7SPsbxb4/vuv8eniDC+3O4QocPbkFXpvsBsqlNrhk5M1Ti726EeD9bM9tPK4OTFYP9njpOlwfWxw7EoUlUP/1KE5a7F+2WPfVfBeAo2FtQon6xZXHypI41GUDufrI673C2zPdngtT6Brx/7IIBGsxOqkRbs1ePz8Fle3K0jlUZ626LoCdqnx5P1r7NoKxabF/lBjcdrhGAREESAah7ocMY4a5rd2iINGVY84lDV04dCfB4Rzi3rVw3uJRT0gRoHdoUZR8MPtvUSxJTgOUeB0c8TlU4b1lKsBIdWd3AwaxZMW/W2F8USgfHbE4BbwiwBRO1TG4bi2CEcNsxqwetLjfreYQmgK7dFsR9xdLllnUUVAB4xnyaN3X8B+p0eMHJBBU66MAOiFpQ/1kYUyAd1j/s03AVg62FBA1OzwxAAIHVA3I45PNaAiTDPC9g3C0qHVemIAY8vwo3ZXIV7QL6mXFkp7DGcOsnLwFcN75NIidBpRKZRnHQZRw9QWdlAwJz36sSGLUkZUbyWGE0AuHMYNpdix8mRTrZli/bOULhRMChUeQOlxfK4QmsQmmkjP5lFPHiCAQCloAEWAHIBQBEhIQLKqRHjN1MBFhLCSjGcEogEwSoQqoHsWKelquJ829SGKXsFu4uQdDYYMkj1zUDuVvGoCfu2gBs3BquZ67FIj1AF2oSbQI62A6gSsZ+ei6iSvm3uC8uhT52FitXwJvm+HFGKUPHxyJEvnGtDfuldkzk4txKDQ9QpyjHArph27ZUBsJWTy9PkKZEESW29XSQqZmEHhyAr4miDWHMXU7xhqD+klfYqlwLDl3+yKLLcckyy14zr9IqA/ywM3hnvk95sBHRlwe5Q3GtIxyAchsY1lhEtss24z84okhWdIlXdI9RFkUMaVgNs6DKOC2fE8VlcEt+OTCNUKmBZQHcGEXVAiXV1l2XJMIT+stFFjxHgWYPYKbkk7QPa+jRuyS9m7l32AiNz3kJIipaWEW/W8ZoVjKjFluQLVFScsCN4ixm0KaBGUOrplRHlPabMvuP7hREyfhXFD9nIO1SGgna7JSFvCuBaJFeY1YBs+VuyzN4y+MXnDwaZIsmTdUbYPZOYrTimjDLR6IBkHWTRXk6nWrcCY2DZfpMmDJEctr9nHmGXpQfPYuvNZCkipMd+PPvn+TAd0p9SaymQVyKFRwtN+AWDah+z7UylQKgqe79hxv+xKwLRMMgXE5AEUHjDJL5q9p+gzK8gaGl+SmQwqbW+MyXdID7AvBCXVaYLJ1jxvMrwrg5dpkJiZI15XyZNsyLZJF+GMQJCzJN20Ae2FSnYCWi6k5fOjJpuaZc5BC/RbTuD1p1QZZFm37oH+JMs0OUHiN3KqxkAEih2BWHXr0J0TJEoLjFsz+QpdmdjXIWLcaB5rukZsncN/EnBMILM/NxOrqPoAe6ohegJKX/C4RQAQwLCdOFeNZFCpBiafZq8o+y7TvW4KCFLJfsD3To4BopFwhSDTVxkUQiCWTKV1tUJ5PWBca0RpoDuCWPZW8jKL4L3A10xjl46TKPpgMW7InAXN47TbckrtlWOEXxiCZ0NwXymZvp8kYgKfwge4paF8Oc7HoVqXrh8B32jIuwE+BRbpIQXjuAhXsweUwT3cd1/piXkUMQX7jAxCypJR32jo+wFuyR5L6eZQHxGSXDftXygUmLKa6kqCQKg19B17L4WdPYNRSQLAdL2HdTN7ITUjsKOUwANQGQ2rTaJRBGqlhuiTjH7yWkoC6AwK8/9lJLuZ5apCkOXU6Xk+zOzjw/XkRSmm1Go9S+eFYLVInJ8nqpJeyxiZ+GrdDCZN9lOE+WeWviKFHSlMUlvxi5SwMQBQlNd+y5dvL6X3i5dvNWPZfO9Z/N7/8p/AeoXj2wVDUhR4kxCA7Bke4rbpwx0pURJpQIaLAfLrakoGlYOEPpC9EAHonjomqy4YVJI9LG4doA7pSzBye/TYxSSh4//1kf17UWFKJMshGtlToVqRqiH4f4ADx1xroKbkS6ZkQpKtyJ5GSqIogcuMg1vEVLfAQT0Et8tBKgdldhmg0/ZyDcbwxKF8TbYsqojyJvlPdZKNbeLkCY0qQniB5rXA7rctlh8bjNuYZn8xffnIkftj9qmrLaWejhuGhIxrnjPhKRnLNRHltZiCHHwe2CSJYTCYAFn2SRV3BAqhiGheAd1jMV0HuSj+YeBHDuvI6ZluQQbNLVjlMK7oNwVmVkK3wOH9gOVn7FQDyAIIz237IibWicmoxS0HH6HgNsyO4MZX3J/MQuQQCabLAvvvepQ3CtXbOdwkFExUNAfK+syerNf6p2EeJO/4WPayqiGdI598X30aFBdzOEkOQ8nnN1cUDFsGb+R0TFZTiCm1VA5zpYRdRNSX9HgeXgosv4w4vKAvNCY5W7Gn52/c8PznQZyrUxKpwhTKkpM5x/X8mQtm7hx0dfK/xiRzTCEou48CTn8gpsTV/L4PJ2SD3EJAHwkc+jNec/VbekWz9zDoWdo3nIj0vvN9HVciBaUgfW7I1uTQCtfwMXPg51m3ad2B7E/ucgM4WPEV98euOOBSQ5yCPXTLz1r2RY4bgu9iH6f711QpkcJlgNnDbI70KWa/py9475CWFSM5hAbgNlRin3K1iXQR/alEeRd43GE+p/nz8DAQJ0+WuJrJrDk11tXpPpY8x1HOSaDSkU3qtxLlnkxcfyKw+SmrHfK5JQvE+wArNSg/lJ7XlC8JTOsr7qvueBzVXUB3ImE6joJtI7B469GdKjJ6xzTAb8k0jEv6M20tUN/mqgsCFNPGKUF08nomBtEXIqWAZnRLRklEgg1fzgDHHOMkV/QlZYfZX+gLHqfwEcNawrQp7KRI7G+ZEmSbJI8XQHXPIBfKOgNcJd+59lWubdAp2VRhkkS6iuAyKAE9hOm6GtYckNOHmiSohwC7TP5JpOPzQEjeUGnppRzXavKSDhvWY/hCoLxz6E81wXACR8U+YNjISeaaQdDkQY2Yj7sN9JwCqYoi30Po7wxaQHesr+C9lpJJrnh+z+SYWBKdPImWQTehlJNUNsr0+cwSyZiuc8tj1keHHFp1eFmivPMpTCY9N/tDfZz9kYlZ5fdQgFukjkItIIcA3Xn4SkG6MPk3g2HvpNnThxdKnhM1EOSNWwPVeYI+gQQu6TvNck7VOvhKMyTn6BCNTPcuP4GIUMjpvPhSQfq0Dz4Q6AmyhKFSUEf6K31TTOfA1wr6YKftyoHVG6HUkK0ls1eb5A+lpzGUGnJwcMsCqrUIpYZqR/hlATEGsoBS8jxmcJa8kiHVfGSJcig09K5PntzAao8QJk/p5N9UaUwAQISAUDMwMR9HNBLq+MC7F+MsPZWSktDc6ejDBBJFN/C9NJohO0qRrQRYN6IVmUqQSczM4dQvCcws4kNfZ2bvstcxJ7qmxNfpefln3ueH4PDhY9nDmT2dD7smp+c+9HjKmR18uI285P2TgutKctrQ9XOlSN5OrgrxD4ElfnZ9+W+/gLn8tnssq4+ex/d/BYzlx/8SjOWvcvlWq5W9k2jKEd2xAEREXHjofQJzC4dQ8eKUPZkc4UQCMWm2fW8QdERMf8sz8C7JFEWci6UFkHxMPnnh+EWYJX3VVapGyLPG6Ud5PYMTu/aTn4uDYqY0RgmyAZpeIuHzQIRAiUE5EnbN/YwyVyNgWrcIQHEr5qRIcL/NPvm7EpLtzyKEJTsjnGCnoOAAsHxlWKItAX2UqSqCMfEhxcTnoKC83aABMWSPVAqo6AR0l5P7OGDxKdEyz35LnwYkYwbTnOEXce4NjJKDSTUmsF2m6P/kFcrMQh7Y5uoMXxHEQiaJWBo0u0X2ufD5/fvs+Bq36T1JkrKQZntVR6YsJDA9bpGSNN9NLsx1BqHk+9+fpfcprScYgi9fzQAqlHECt0ACVCkls3qrIIe5csIuMQ3oMsAMhsxWfyonNibLFVm/gEn6aFdJwvV8/uy4moxVLiwfzsheZp+OrznocQ0ZL9XnHZiBnnQZsPO9zOEy4zpdoymhUrc8jv58Zt7MIcItEmuoeNy6nX1C+RqbfoqZ1clhTiJEhgPtI0SMKG9lCnOZ6wGinkvhbcN1Z7aBPjkCkwnA6/x+Jcnt+cwYiEDQO8kfy3QtjpiSOkVkd58vBIYTMhvjmte1r8TElI4bgkckJnVc5+tdTNfnFHCTQCmAqUpj+gxGTHLbccXjRmTJvV1SYuhqyvMQkyeuAHJ3nV0m5rXJ1xXX5QsyhlGQgVMJyNsFz6evMLHuwTA8JjNAUxVHJOALBhPYj5rXiS+RWCTeO2zD7UQ1s1xBi8lPl4OEcuF73qZ0SXqW6iKE5z1j8nEeCMz6M3ob+xOCSk4kpPOYBqj0RAqU+5lhdJWYQCmZR+6HtDxOn0KobCPJSguRXjv/DAaz3LLm43Yxs1hks8k4B5WYtFRn4UomeorIbbk6ya9X6TNWyimkKssno+Ax+TKtv2adxXTfSANPn4CabQT6jYKvxOQBzRUgTP5kRyRBvCB4FZhkzraWk+wzKEp+Y/ITZ1A9rvWU0uqqB8FHgu8xwPtuLqUng4qJiQ4lz5tOvjxXSwwnKlVH5M8uwTLrTdJAPnUiZpBKv51gAmqqEfGVS+z3/AAAIABJREFUmgKWghGwSzUxnLzn8hwCgF1K2LWGXRCsZTAaCp4DX0u4JRki6eIUfMVrNPVFLvRc6TEwfMhX3Gdv6GUcTgomoSoB1yhEw+qLUMh0TxNTgNK4NtM2fKXI3gmBoCUDf9K+hYJspOw9xrR+JAbOlyr9kwhKwi003NJMQUG+ISsZao1QaO5DrnFJn8FgJNyqgF+WBI6jh9uUGB8t4Et2SrIWgyqYKMUUBMRzqKfzFBXZxVyfEY1CmLom0/XbmAkwhuRzDIsSbmEwJ6Ey+CdUms8tFUFuaSZZbij1BLjJEBKsRqMQC4NYaLKgD5JXY10g1vQMRq3YOWmoaCJQTfeluqQ8NbGBseKNMAf9ICbmWcr5WAtDQKYVgVZOc801ITk8B2AYj1ZkIgvDf0bzXwakmaXMS/I6xhgRnYMoDIRW/GcMhNYQWnNSRGtA5gmaxHpKxX/e452qknw9aP2z4DYtQkkIIaZ/7yw/D3D+ZVwipjaJX+a/X5flW81Yrn/jcfzOP/xP8Wy9w21f4/nyHs/qe3zdbfDFfounix2+Pmxwu2+gtUeMAiEIOKuhtMeyoVzzZHOEEBFXl2v87odf4XfXX+P/+ux34JzCRxdXeHtcoh0NjocKRengrEJVM0Cna0uUlYVzEmNnIGSELhiEUxUWp02Hr+/WKLTHsSuwXXW4uVsiBkAZDzdqxMSMVcsR/b6ErhxDbY2HdxJSBRjjIUTE/maBi8f3cF5iWY746nLL19YjukOJzbbF7idbiCc9fM8b9enFDve7BXyrcfrkHoe2ohLCKSjtGS7WGejSQekA98kS7sJCFh5lZTF0BqZ0GHYllmctjrsKi3WP41crVE+OGD9dwZ+PgBe4eHqP69slqzU8k3K3z3a4u1lAqIjYK5w82eH2qw0ggOKkx3hdsfLDBMRBASqiXA4Y0z7ZY4H12RG71yuolUW4LqAfdbD3JS5e3OHy6y1lpkcNfdrTX7s3iDpg+2SP+/sG2BuI9YgwMjwGAActOkDqMPlO/d5Ms5qoAsxrA7cmow0doXYK4cxCXhn4DQN+hGPgjugYMrTcdNhfLyALj2Y5oPtkDd8EmO0AqQLij5Zofu8Wd69XgIoQRw1zEBhPPOTaAm9LhC2jaHNXqPAC5Uc7HG9r6EsDvwgMJGr1NEkQTUz7GaCvDcR7LcRPGk6WXFhglFB7RSnllr8Ly4mY5uKI/rMVQdjaAk4SaDcO9Q/qqb5iPPcwtwr+ZY+wN0xVDpwUCY1n2NDWwnxWwjdJhvjIQZiAzckRd6/WXO/KIvQaYpSIIqJ6qzGcesQiQt8puFMHdZ8qWFQEvGDNjASKK8XBb0W5rF3xcdkLdrZayfUWAbKTlCSeef7/KGC3gZNNAPzSw9yr5Hf1kJ1CqALUeoT8vEYE4LYespVTkJE6KgItzIoBX5LhH7/XQbwpGYSzYBcsDhrLTxXskpNJAKb0ZbcMWHypYBc8t9XXlOvZVIEih1Rrcp16THWc5Lu5fsQtKC8Ohr7MqIDiLvk7daQH80C2SYDhROWlmlhHaYH2PTedK7um39kcma5cXZMttauUwruJiCsH89ZADgwWUoOA2TPZOQcL5fCVPNFF5p7grLxLXaxVnPpJ6RWVsKuA5rXEuKECI2qge8+i/txMjFBOx/Vl9l9SskopIMOXVC8mUG6XVCJkllWk3tHqCth/GJl43AuMpwHFjYSy9DcXd2QphxOy/OOWnlhfEtQ/7B3VxxlAAwS7Zo+JnQwl0D3m+nOa9ZTSLcigHp/xsyR8kiynifmcUj3VY6RJkBymo0YG6viCwNPX9OSOG06yDecR65/wNe1THqvZ5/MY0T1+kPqcmLVQcPLDrgiOc49o/Tbi+FykdGhM/ldfzZMlOalXpfRoVzOsSfg4pS4HQ5XDuGaQUHmTqj3WAvrA+7O0iW1VyctZUXL7kInPigL2XsaUjJtu8YF+3PIuTtdC9p6T/U6ToUmazs5hAdVFLN56jKsk0VRpssnmRHFuq0rn3C4Iik1LefPilcfhmSLr7OaJCTVEHJ8oqD6iTPJa3dObaVItSU5t1wMBMcE8UKbU7ygZwAMBjAs5seO6p6Q5p/E+TCOuri2OTw3DdwqB8tZh3OpJJhyUQLHzE3vHCStOSJT3Hr7KkzwEzFQ+RLiFnNhq9mSyA1QlWbR0ZMSLvZ8YY7dQKG8t7JKeT18r/qwUmdtCzixzOgaqrtJxHh3sikBXd2Sms3IpSsFOzJQMDJCFRYzTdT15Q/eWacapViRKAV9rssbguvTBTiFC44YJwKp3yP2QsmMKsK8Ncr+k7BxiAtJ+YYCUuCt8RKg0ZDsiNAXUfpjAtHhQDfJOb2auC4lxSrCNqbMSAGJlyAhr+U59yfR3o6fkWTHaCcDmnktIOU0QcD8eMJ8ZxAIEjjmBNstnBZnKaB8k8GY8kbefEmujZxItjKGvEw9f8nOY0J+3xPD/Swr7a8FYfvd5fO+P/sEvfb1/9h/9V//Kjw34lgPL5ntP49/63/4OPrs9gbUKUkZIGdHuS0QnIUzAs0d36KzGzevN9Dq5Vxy4jxw8w0TohYW/rCDPB0jlYdsCQvHmLa4KxNwLaCLUZoTfFUDpuY7cGfmm4SB4zWqTWPIGpXYKfu0hjwrRsNcw6sRqjgy7EYPk4Lij/00fJOwjCwwyyXcF/NJDtcmfNsipLkV1HLyrnZpYWuEEQhEIXEbJAWDy2OhuDnbJCYNqEJNnQkTArTzUQU3sK/CANasCRBSAE4hlgNqr2fuXGNdoIvQ+sxuJRRWJsdRx6iG0J2k7APzWTX2O9Dhlpoc+KjkKjFt+66mWPhn72EJdGw4k+FbArTlwdouIYHg+VJ+SKyXPjRo52CzuZ4YoGLLPwpFRVD1l0UFniS8ZYTkw5ZLpl6lQ3RJE59qRUEaY+5TUmtYhknQzh65w1p/rUz0ZVd0SkPiS51GOfJ5uGZYjBw7KpxCf9JxQcnCUZbbSpXWP6XiK5E1KTLivgf4FA62kA/pHDHTJg4qJKUxex9w/6Ku5ekdabtuu+X/dkqXkBYgkLyfIyDUyk4QtMUbZY4aIJJ1OlS5JGpzZKd0lJirJd32ZmFCRrrmQWOrEYuue3r7MVuf6nupKzAz0hte9bjHX0ngkaWUKx8nSUMvjtylJM6sbZBrk5wFnloBSFoppkG0OmCSoUVIOPK6TZzYNXF3NdNBcxTL3+KVeRJdY6JjewxITM8+QF0yhK+aYJZR8jWsSiByRZLpMAzVteixi6kBUPWWQmbXTDyTUuuVgPB+TSLI9yjoxqwmQZLU58bMUSZXAwXIoZmAoAqa6kSgSyLid5bJuwTTRKMQkf5U+vvP6DDhznyNl85HXnEv+SUtA4kqB4ZSfuRxUk2XVvqY8OfdFTunCZr5WdUdWPku2M5tY7iiHjYJyXyAlfyZZctQzINTd/DkL6oEUOZEDWcI+VS4lOe7D6yID7Mxm55qWzNDn+p6oyIYWO97L8/Wf36dsF8hhMpkFRkwscMDksYry3boZ3c2y3SwFztJWbwT0QA9mZiFz+FC+XjI764tUedPIia3NICjLSHOisO7CVGGTt53rXXJ1Vj5ugKxvrnRRNk7VLA/DWx76ZzNbmv26+TmhEPQzaoFxJVHeB35uS+5zrnqRNiQZq4Q5+hkYjbP3mN9JYWIeoxRQvYdb8AJQQ2Ti6UJPj+suTN5CEfg5gEjgKvkJ8/e0TEE7cpilh7nfUfgIZZlKmveFn9c5EXb6u0z77gJczaoPmWpBgiGDxeRVnhuRGDg5Mj1WDh7RyJTUypRWMnT8LgoFmVj4JFctklS3cwhlBl2Raaoq74+AHFinkq/ZLCeXQ3pdBKT1ieEkCBQ2ENj1BIMT45oAKJSYmbYYISzrSyDIYArvKXMN3N/cO8mTFx6wlZif2w+sAUlBQKHQkH2aLXJJ/porRHJCK5BkyPqdQB1h3cRqTp2UWarrA5nJXF+SGdEQZu+ldWQuhaCsN8tTv5nA+kDiKjJLqRRiz9kaoTXBoPeAkIAdMclmgXeltSnAaJLd5moUYPr9HUntnwcqY/jZh/4yAMv/8VcALP/Orwew/FaH92gZ8PGrR1g0A0QR4ZyClAHff/kGP359gdWix1dfnEGUHsvzI477CkVlcf6SgTp1OWKwGtYq+FcNFh/scLhtENqCDNp9gbByePrbb/H2Zg3Xa2CQiK8ryEcDQjJRi8YhBAmcjFA6ILytEFYOCAKLsxZ2reD3BULDD5c5H+CsQtgZxLWH3GssP7zH7rbhbJ2KcCsHtLxRhZWDbhzk2wr+xAG9hDwf4A8a8lkH5yTQarI1o4TajvC9hmgVYp38F6uA2CsydE8H2F0JuRxhjIc7lrDHdCnoBEQ7CX9iCZQBAmgAYpTQBwW39AxhAOAXHvGJRTgaguazETEIWMF0UvF4wDhKSBMQbwrEIsI7Bf94gLqkjNkvAuSRj+FgEBKwghfAxsJpzQRJByBIuHMmpYq9RjQR49azuiMKiFGgf24ZNGMi2bSakmhhsxwporqS6F5aCCth7iQAAsawDDD3amJuhBfwq4CQAPJw4SCcgD2NEwOY+/CiJlipriUO37MovzboX45Qdxqhiqneg4Xuqk+gsIgYHnvIXsK+N0C+KVHeSOgWOL5kVUb3OKQ6CmA44WBNKMA1nmxUkg2Hgt48fRSpVJ5yUV9ELD+X6B7HBOYlqs8LDBcewgssP5PoLlLH4yqiuJVwTZyAWu5NBCgPVSMwNvTpmp3AeBLplx1ygqYAxnmQO24J8tVIJqm/8ChuOSnSvfDQ9zJ5TEVi2Qims291SME7+iAm5ms4JUjPATZ6EJSLB2A84T64JsnWnUB5LXB4n+cdgeE8BJFZ6p0mEHqeO2Bmu3wdUb+Vs7dQZXaM50ym485/Oz4P2PxIwq5n8GfXZLAyAyYcgXiWmosI9OdMcM1MVAb244aMmG6B44uYAGBiBiPPldnP9Q1ZgglgqpuQ2ZspgWGbwI1L24rzRItPNREysZPNKzF1zEpLD2OWgLsKEJFBK5RBZ3A6b9OuGXDTbSgBzfLoYTv7jFVPxgngdZzrMBDThEVKL+1POSnkwffOLjCllbpmDhcq7iMO77PWRkRKr4MChi2Z21zvoQaec8P8N3bCnpNhC4Ys0XAyd6/KGNGfpnOcZLGseEnAKgDaAu1jCV+krtokgQ5KAMm/258Q6I1bsnbtE4HqiseQpeK+IIDOXbj5HpO91iKw7iKDJNeQXQ0KqSYCGE4kdEtvp7LsU3XNDA7tgoBYJBmwdLPcWESCVYYF8ZjNgZ+RYUsWxzYEvP2pQHmfmKtC0Ae753PHFcE1BHB8oqawo6DZmepLrouBQwTHIjB9NTOLvhCTZ9XWin2oAz9741JAeAWI2fs5bOTsq7SJRRvoT2UdBwFmZkXZ65q8r4WYWEFf8f8ZhLqKCa71lUd3ppjO2hMQiwD4NVOVQ8HJOFfJqQN5XCd2MZDtEyGFVnkxTaJElSZGEOEadl32ZwamDXC1nIAefbwEdLEXyVsaUdyOcEsDV6dU14rnpdhZhAXloMOpQXlj4ZaKEyL3ZAl169OEIKWxZjdiOC3pv0zeTV9I+JrAUQ1hSruVLmI4Maguh3RfNImd1InhDHCVAlIAUe4LVUPAuDIo7i3sxkD1lN4OZxXK2wGuMZxE8ApqCIiaLGJ/UaO4G+FrDdVldk4iBjWBy1DqaTI8KgV3UsLsLEJN6asYZ8AiR4/ogVjyukGM8IsCuXtTDZ5spg3EOUZA9I7psBEQoyOTGPnBiUrz92AIJgeLKCXksU+SWQGYJA/O1SHO8yc4gQpglr0G3mMgBAGm0QSfNZNaYRIYrXhTyoxkTNUiABBXTUqFDXNVCSbeYGYbHy4ZEFoLUacamTEBY0UpNUxB8BfFO37K6TkPwXGS4D5cGAYU35HTvrs8kPGG+OcCzW/j8uskXf1lL99qxnL5/SfxD/7xv4ePFpf4wf0zBAj85OoMAPDXnrzCP/nsPfwb738OAOidwY+vz3G+POI4FhicQlNY3B1qfPToCl/db/B4tcfHnz8BAPzuh1/haX2P//v//T0gAtv37tCPBh9dXOGur7HvS+wPNU42R2zrDj/54TOsnu+wu17gr330JS7bBW52C7hXDRYf3GNdDYgA3t6sEbyALjxOVi2EiGiHArurBZZnLV5u7/DDL56gqCz8p0vgZYfVssPd3QIvHt/icreEHTXOT/Z489kpzEmPurI4HCr8jQ8/wz/98gWlpVXA8qTF4baZqkUuTnd4/dkZfuc3v8DHry8AUAIrVERRWRjjcbhtoCuHDx5f4/OrE/gE1rXxCEGgKByawuLN11usz4/Yf7nGi++9xVeX20km7J1EbDWWjw8otMP9boHgBKSOaBY9Dvc16iX7P72TsK2Zal5QBrx8cY0vPr0gCK78JBX+3ou3+PiTp0kyK1Ge9BhbAlPcFRCnA6KXVHjogOgl1OsC4v0W7rIis6OY8CpVRAQQrIQuHWKQ8JcVYpXkkBsLWIliM8C+rclUr5gY6z9fEHQ7Sda6V0BF9lpUHrFXKLYD/JcN/JKxjHo9wt0XgIrQS4v4qoJ574gYwWM4JmnPXsI9GxA7jZcfXuLt/RL2ywXUsxbhiwX8qYVZjIhBwu3SyNaEJP8U0K2EawL0ow7uqkZcOCy3HY5frHD2nVtcfb6dGOXipMe4LyCOmvJXL6CvKTd0Gw9hmQ7qto7fPipC7nUK7SAjOz5OSbGC7Gm4GKG/LmCfjhAHjVgFiE5S4rjxkAOlt+WVwnDugbVF9WcVuucOYkx+sZVD/VmB4fsdw7WWrFIRuQ/TRIhhDpQJJnVANgTEdh0QCnZ6Sk+pqPCCEteDpE/2JMDcsj5E9WSUyxsJV0dEA7gmAAqIMqK4UVCtQPfhCH3FSQ92JRL0+RJwJ6nHchAIdURxLTG+N6L+UYnhLMCveF3FdB7FKICLAeaTGsMTnvviRk3ManGbvIerOQ03GNbyFNfsFx3OWEWSWTBfR0jHgZBryDraEwdzq8ky12TwZEoIhaAyAbVH83EJ6YDjSw99lGn9rBxpnzMttrwR6B5nxjGivJITYywdMDzyKC7ZsRkNEFRiyBIjWNxTDipTmmv+XXgec3lNf3LzmkBWJSZ/3MSJNazfUnaIpGQodgLdkwBfByw/1VPwVw4Ry32eUYDgPrHW/XnE4it2dC6/YMCTtPQaV2/5npZ3cQoBy8CZzBPXlT3EGbgjEii7isBPpYArn8K7ckiVT+qCzIRPtSjnwOIVgVH7WKDYE3wLj4nt1B2BY5YAl7dzqBArix6EYWkeg10QrIwnnNhYfhXQnUrYFfezfRpR3qbk3ZLPdYuZFV98xTFCf5pBL7B4w9Awu2C3ZX1JSWtOss2AWg3JA6d5zupLMot2QRaxfSJQ3BPAkyHF7OtXWVmQ5K6Bxw9glqY6VrSMSwYzDWtKS+k/TcygJRgvjmRLh61I6yDr7EsCX6YJU/K5fO1ZWZIqWaLihEAOYbKNQLmLaM8lyl2YgGhUAu25JJAfyWgzHGmWtYYH42STpMPSMTyqeeMwrhXPnWWw1fKVx7AhIM4McJT0uxZ7vg9qJOiwNSW1IoU+ZeB7fKxR3tMrW9569GcK5X1IE0oS5Z3H8bEmYB95nrozifqa9TGZdTVJ0moXlMkWhzBNmpiDh68k7EKivPXwdapaSWoBORJgSh8nJnbcavpmC4nyzqI/M9DHAN17tI8K1FcWqiNTN24NyusB3eOK9STnJczewTUK5VUPSMEqkcyW2jDLWoWA6h1cY6A6B+ED3KpIEzJMzC3vLGWsht5dfT/AbUroo03saGJEC7Ke41mF8qqDrw3k6JL0lWFE0SjAMfxIthZ+VZIp1RJBS0pjuxExsbI5mEi4QID5MKQnyVFDVZDtzP5NoyH3R1aQ9AN/WgdYh9hUgJIQ+5bgEwSb0WjWjwBkEp2fJa7ZP5oCdkRVEnx6z37M/LocMpRVDD4AIYX3aM3/A/Tv5roRm7ow8+uBmTFN7KPQ+uczkd9gMr/14T3ffR5f/g//2S99vT/+u//lv/Jjwz9n701iLMvyNK/fGe70JpvdI9xjysiZrElUS7WGHQKEEItuNbChlyzYwxIQYgWsgCWwoFcgUIHUNDNdUHRXVVdkRWZkRGQM7uGDzWZvvOM5h8X/nPsssrKrKXUVnVnUlUzmbnbffXd8dr7zTfySA8vFdx+H7/yHf4PVtozmVajKngBs1yXKBCaTlq6ztHclk9MdfWcF0HRa/JDRw1cdNDS7HHz8Y5Q7htpiSod3Sn6uAmwy8fF1BmUCfiv1FWdP77m+nqNtwDVmv53JIL69ABReGLheQemFbSs8eipAZ2gsamMIUwe9FjCSwEO+Xx/AbDX+pBcWEiDzqJ1B9YrZ+8J+KoXs8zqTgCIvbJ6vxHsWikCIoT6qF6b0oVdP7wQEmEbjKg8+sjJ5IGReXjMdYJl0hMIMhVwCkexaZif9kwZ1UewlXAr8RDSXqhcPi+pj/+VOvF9uJr/XW0PIk6xUwpLGgvlOidcwAgzdSeKv8jIYdBMfB4LCRKpBjZLmFFwUsjB6QYNFknmjN8xVMqgfJuJdE+lOgCCvM7u9vHiUkhHlgINIa5P0rLgTD9xQhSib3fulXCmsr6kjo+pFWpvksYktTIOz1CWp+8TqRVmpIc7Ax/MxCONntzLQt9vogeuIxxIZQh3205ZReqq/boMAYvF8rDlJkf+6jx5Hr75WNp6usx4EtIVMwqWS/HSYJAaM0a+W7lGRAUfpsZWfJ7DiokQ5mDDKdZPUWA9yne12f11S0q5IGYUxTSmPKnrz5IZgvDZ6iOchSnfVg7916T1dGZ8nL9sxMSQqhbuke8BFSeN4DYufOT9J7sg+lCjJWlMdQz8P+2Csbr/N9D4Pi98fblcksYzBRcLoMHYaJvClgvxeD9FrNtkDJJBtuDJKDB+EViUpp2nk/+n+SYskrzJ2BCamJYXm7KsQ1MiMp+oPCWeK20ty1AcDcthfw8S+jbK+B88hMEqtk58sLcM0Hk+8T3S/P6bEqO2l1LHzsXuwfofI26z451I6eJrwED+nhCAlJkjkyPIMDmVkJN1+e8MksYVyEAnEpn1JzFYC2lLD8PXzkfoex0qPJBEf7zMV00MZw77knt4D6JQynCYGTJOug7B7yu/P70NG8eH1T/eJjpJcn47f7dlCW0fwZdUoI0+9kyLBJgZNhfhsJGZv/1ylXs6g2YMgohw03/sCffIHxmdDeUZpdZLCuuzrcllvpdYkhRHtQ8XS8YYoFY6S2z4IWC0fyP20wnRJeho9rFFmO543GP2JIvMOX3u2lQsMlf7az9P1FiAlO5Y8pollfXgfJLlo0Eq8jIWO+/GAAYpyY+X2nkRf6DGgJ+1LsHpMgU2eRTX4WPUhwMQbkcfq3jNMhHEU1tfIfR+Td027f/8kR04/S+m9KgSRsuYG00jSrWmdPF/x53JPO7zds9UC1oSB3J+TMG4TED+j1V+TBafXj17Hh+dHger9H9uO7HAQmWw6J70AcdU7fJmPCbcp0VZFWbFyTkDcwwRYpfby0CSTTTLSxFw+DPJ5sCSvZLrm4/IwdTaxhD+7/w/XhT2QfLDOiBsSy9j1pHTYr0lhYWQ9R0nsz25v3OwDcPsn4RIf/kIAy7f+vT97YPnZX/3FAJa/1FJYgCezFaeTLbf1hNw4eq/J9J7iP5rUtIWhKXq09mgdpHeyAO8VWgfyo5rMONRUfued5o3jFVermfQndhalPUOTYU9qtA6gDEXZ0+uAd4pdm6MzGdUV8xY3GJT2uMFALuDHFI7gwWMx5UDI3AhAvVfo3OEnYEqHGxRKe2GjIP5OofL4kM4DDEp+H0DZGA1eeNpORtW6GNDa0zdGwJD1hAJh/LwSUJn+6ETAGkyA0qEzj496jNALS4R1UaUTH34bgXLlJATGhOj3kh5BN/WEzGN0lBrG8JXxr50JBO3BxtCeLBDygNf7DxVfyR8jl4M7GOSY4366aVwvAVodCLnsl4odmMrtE0b9JIJQLccbMj2GnoQ8oKMPU6o3BFT2R8LcBaUIRcBVEvwiwEZFJiSMDFBi1ZyJoCqGz7hCGCcVwBcyCA1Bje+l+lgAjoBXV4aYkBj3PQJgbx8MLPQDUJbssMmH5+U1vgj00avYzyKIjgA02LBPH43bTx7M5IHzuchobR39pEmOF9N6XRnAqzGAREXAmnyUPgtjemzyw/kYTuLKMHrJhliTEVTYe/SylJAsnk7TqAh844AgJjAHEwen8X4eU3rjJEIa5IXYxZc62FwR0FoAvByvijLlEINJ1Oi1TAm76dybTo0yK4jsVfKQxgF3V0lljYoTEkn6nBJTk8/WT+R6JSltCu9I7FjqaUy+ULuRN1YBcIwJ1UkWGjS4CET6aQSvCeyo/b/TedJOWJ0EBNKSgLGL19PWsj0TAYnq9kAsdR+m7rwEsIRFUyMYfsiWPqx3CBpMvA8AdDwXe4/h3rsqD8keMAajxgmNoON19owdjimwJ6Vy4vd+Unkz2S89RDlpvJdTcrCOPtgRbEavstRIqeiPfSCXHfb3sDNyXInxSceTzoE3CowAnGEqzNxQ7gfT8iypCPASA6zGz5/kAUy1Pui9JzRVXCVfbrp3lY9VIuk+jPd28qxCBNSRQXzohfZWjZNb6fR5G73waRIlekgJ+/tw7JTso58weiCH8o8D0gS2Ht6LaYJhHMQH2X8/DrTT87n3ZPoIXqV7ML1eJlaSvNhlcd8t2Dbdyyp+nsRnKfr+0vOuXcBZxrCdNGmRJnZ8TEhWLoyTKOlvRQrTSf9OSxr86wTo4gSEH6tm9PgexImNNAmVwEYC3GliIsTwF+X24DKdp70nMwKcuN/ja7Xag17P1yS4CVSnChvi8ZkBeb9ABFdxbBNTacfezU6Y3XRtpM81hu0oOQfp98kvLdUjXdGXAAAgAElEQVRh8kyETI+VK3Ly1HgOgxIwiRMQ9jVA7OOxdn4Ei8FoQrkHh8mPiZf+ZOL5Tdde+bAHbbG+ZKxEGf2IfH3xSA+l9xLq00YmUu3re4IxAqoSKFNqDzJT5cfDOhLn9vuKnPev/X/0x6aTGFfL7N6fCQIGB//HgV4Iwj4mUPhzwKBS+7HVfp/N14Es8fPrZ4Hiz7CWf6rl/0/psb+kyy81sDzIxMiTa8c//eYnvG4PeLNY8uPVmxyXO55Mlvzw5gm3qym/+dZX7Iacs3LDcb7ltpvSeoNRgS9WJxwWNbfNhM4Z3j+44Y8u3yTPBr5/dsHb1R1fbE+Y2I7PlqcYFVg8aihNz7PlMWfTDbf1hPdPbpjZli9Wx4Sg6AbDyXRHO1iM9izrkt4Z8sMtISiUCmTGY7VnmnW8Wi2ojnqOqx0hKK53E8rDDc5rTidbPnn9iKen9zx7LXLfR2crpnnHqilZlA27PmPb5nzn5IplV/Hs6gilYPF4w6xsmWUdy7ZkXRcsHjcYFdAq0AyWedGy7XJy41jWJavzOcdv3dMNloOqYfCadV2wu5qCDszOtuw2BcZ6lA68c3rHzXbCalPx+HjFQdFwW0+4uDjAbSzzd5fMy5beGda7kjwbKLKB66sFB0dbdk1OCHD2ZMOrL04pT2pCgCIfqJsMPzewzDh8umJ5N6WYdrS1oC5tPUXRMwwG5zTTScvRpOZuJ6meISjKbODmZkZe9Qy9BRVw9zlv/vo5l8sZ7VL6TLX1lFVPczHFvr3FAn1nsW8MMhl5X3L2vWtuVxOOv7nj4tUhp2+suL5YQK/Rpz1+lVFeWtz3Nvhdzvx4S9Nk+M6CV8Iy516YcBWg00web6m3BSpzDMsCVbqxRiSbdgx3JWjxdLqpJMz2dwXqcc+wsajKMZ039L1haC2+tgL6A1KbUzhmH+fsfq3GbzMoHHqZ0b/RkZUDPJswHEk/m8odYWsZjsOYyNoXmlAKk66nPeE+J3tvR3sxwa40w5OOfNLRv5riZ5Loile4mWewHr0z+AG6swF0IJv2DDcl5YWh+16Nq41MeGwtyiv0cQuvS9xBj2qMeFqPxANktlq8xgFUDMQiQHZvcFXAnzjUZICoFNCNws0dxXlG/X4Hg6J6kREK6GaecNijbjPCaUeoDXZpGfKARYKudu8OkrR7neEqYUT7Rz35RUZ3OqAGTfXC0DyWACSfBboTR/naSshRJsxweyKTQAnAqkHK6lPwlIvhSCqB56AIE4dZmtE3m60Uu7ccIQuiOjBhPP781lC/IRMiibXUrWKwsT7JKXQrHtoE0pVX9JUMvLsTh90YJi8V6/c9xa2meTyQ3xlhkb0iv4+DrCCM31DJgMHWagQvwyQGPmUBtRRfsO731S0JNKQk3ZTQ2jwSGWp3IMyyyIDluO1OgEv9OJBtFd6EMcXVbhXllWL7NEiw1aFMBjRnnvJaj6BTOwlC8jYwfSXy1xGsbyWMZPFFYP2ODJZFkhylt0r8mLoV6ahyIg1Ox6a7vZzWFdL7qlyq/YkhV9PY4aoV9SORrA4TSUvdnkFxJ9vwVuSnzalMegQLdsPIuPhYq5KvpS82scDNcWD6KlW5yHu1c8iXEqiTPKzNqUhfTfOgr7aH7Ztq7IdVEWDaWrpxxcfLKA2tzxjZ/PI2UJ8pYacbSdhdvyOe0ZTWCnFiZKGilDawPZW0Vj3svaLShRlZMy/PSXXtaY/0KFVN25qce0lajkDOFbIvykO90HLvNCFOsEgSbXMk5zSxhInl7CfqQaAQtMeK6lK6TF0m20ns6FApqtvA+qlhcuXppjKATz7VfBVVHSiaeUzGnch9mG3EQ2q62OF6JAPxodDkW0mH1QMUK8/ukaG6EYnq7GVPc2JH4GvrMDKr3u6DrhIIChqKlaObCcrJtj5KptUYcpWYd5HTarKdj6m58izY2tNXeuwNTbLZoGV7pk2TNSK9rR+JF1TtpIc02zj6ucFuBZy0R+ItlARk8YdmW0d9llPe9NSnGeXtEP2iClNLLYmL9Tb5cpDtrh3diSW/Fxms3Qlz2c8lZVYPAV/ZkcUVdlbqdIrrjv6wAB++5mk1rZNezUkmk4FGM1RmTKo1dezftGqcUTGbjv6oxDQDymiRj/YOX2ToToBmyMT+YLadsKq9EwVCVew7YbeSSkvn8fNSWM4EvIx4OUNmxKdZWFTTEyaFvFcVWVCtUXWLn1ZyPXYNKB29myJJCFWxT2vNM0mJBZG5wh60pqWXLlEyKzLYuF6oU+SyAGGVZ5BkrMPwdeCXvJVK7VnIFCqUlnSsic1knP/Y71da7+d4NH8ZlzQu/Yu66H/4Kr+4y0074Yd/9B6fXJ1R6AGrPBftgqt6yo9fP6b3hmnWMSk7fnp/yo9fvUHtMp7tjrlpJ3y+POWT+zMWRcOHL56QGccb0zVD0Hz/7ILvn12wG3L+8O4tcu14uT1k3RRcr6dcbmds+gKjPRPbcX0/Y9vnfHx7xvnLI3ZdxjsH9/TO0DnDqim4v5uiVOD2csFmVzDNe9re8vr6gNfrOUeTmvvVhNt6wifP3uD2YsGuzTHa8+Pnb3Iwr7lczcAr/oXvfUBpB263E7ZNjo836fGk5g8+f4eb7YThusI7TQCulzNerRb8lbOv+MbJLU2XMc06lnVJ21u2Xc7l1YL7umR1NUNNBrZ1wbToqHtLaQempVAU2a1lczMh3OcoHegvKwav2TU5+kvxn3702RNul1PUxqJrgwK6wdL2lsPZjtX1FKM9YVBs65w8H+jvRXunIivZrgu63lKWPa41UDmaLkPdZeSZAIuwsfibgt2qpK8zXGt49+iOwWtWN1O2mxLnNDc3M3TmaTcFrhbfqNkazu/mLKYNeMjOc8IqJ7OOkHna+5K+tZgXJd4rjJGAodWuZOgst6sJBMWmLtBri10awjIXFtQGhs6iV5auswQv0uskXdb3VkCbArsy7FYloRcNW35jYGPRGytMtTNgAmYjqb9mY+hXOfmdwWbDGLC0Xcn5842AOrPTmK0RptgEuoOAbxODrvCFF1Cp4sDBKbIrizIBuzEwKLJJh2o1xYVB1Qa7NPh1BgraXYbqFQc/BTaWEBT5rUblnmypydYKs9bYO4tuY+XHzqDXlqG1qJRY6xTKBkKvyZYi53brDLNT6LUlu9OYnXgss3tDtox/fGqDqZPkSEuXrAYyj7GRye8U2UYGJtUVY8WK7kTa6xcD5jyXPtWl0DxuIh7N2VcyQMtuDWYykK8kBTlfKcqv8nh8IvdO8uJU4YCXDtf8Ps6mO2FY1aDIlyqmFIN71FHciCzbtIp8qZm81COLxiD7n99qAXVGmNRUDWIajYrXGiXsphqEfTJRaiz1I5rJK0W2lfOoW9kvV4rMtrhT7JMaibH+kAKTghEpc7YWQCHeVPFNPkziHY/VxxCkVoBzcRse1KPsU6KzdQzwsQKOXRErbdayL7oTIGKbQHEvElLdCjjPtpCtNamXGETi6ItAcSeS6yQVNYkpBvKVHLvdyUA/GPEiljd7Jh4v589V0Q/Zy5fphDFO/cSyvUA/k3/rThjD1M+bbWD62pNtY4iUC9hanrfiLoz7lPa/vI69yl72T5J45Tokls/WgXwdxtel5NvJawFqxb28l2kFVJk2hVJFuWgtdRr5KjJqVlHdeop76datrjx2F7DbKLePUmiZ8BCgB3JeytuwTySOUtZUw1Ld+NG72JzJz7OtnJ/ydiBfBfKNJ9+Kj89uA9WNk3Mepa2jFDlKhvFyHNkmjEmvKkq47U5AUXnnR4YZL2CYKJJJyczFymPrWHVSyH1qa1Gr2Fb2BaBcevQgcth84ynvXQSywhAW927PrqZbQkG+8ZhWjkfuvzBKvX1kSe1WQFC2k+tQ3g7YRs6JHuJrh8S2ynVPagYVwNSyXrZxsW4kYGuPbeV8Jk+nCpCve1EmND5eL7lHdA/ZxmHbKEutHdnGyWdaI9u3W4fdOWG6oxRax+sTrEwGDJXGW0W+HBj9cF4kvb7Qe7Y0BPKVI1/34jte9zHMyAtYbRzZOnruPGPSrN06dOvIVwPKy/+VD+N+KRfI1gN6kDoQ0wyyzfUgadK9VLEEq0dQqXuP8pK+q3qHXtVyz0WQaWtHth32DFwENKbuUYPsl1SMCDhUg5fU3BgmxOBRbY+KCbuqd4TcChCM0l7lI8hKMlcXUE0v68SUWuU9yjlU0+1fu2thcKhdi6rlixBQzf7fqZpkTIRNi/djoA/Oi4y17eSr66Fp5QsE7CX2Mp0DHx7Icz2h6wkJpLrotVQRWjyoHSFVjiRvZZTUBuekjsTv//21r4frPUyR/cvlF3b5pQaWpR1497vnPFpseNEcUbuMw2zH9w4vOTvcMLUtSgWKbCA3jqen97zYHPKoWNO4jCezJWfVllVbUk1aplnHR+eP6Zzl1eaAH12+wVmx4TeOXnDfVZwv5zRNhopM331TUWU91/UMBTSDJTOe48crymygHjKOyy3TvGNRtpTTjhAUj9+8lz7L7YRuMBwsdkyLjt5rirInN47JQc3RozWZdSx3FYeHW1bbUmT3medHyzc5LreUuTzQdZ+x2pUs65L5QY3zGhY9edFT7wqyzLFdlZw3c1ZtyeA1t/UEoz1Ge3xQVLOWEBR22qN0YFq1+KBouox1m7PcSBN7/7jDVI5gA3k+ECbCcvadZZh5vNeUBy1DZ7AbjT/o2e0K2sHQDwLM6DVtb1G5yJMBzLxn0xSE3OMfSIQBbDmgdhbvRWraD4bQGuZP1ujjjmreErxC6cBPr07ZtjnowGJeM/SG2UGN77X4WweNG8wYBHH1+gBMYHizIxSO9f0EBoUqZbQ3zDzGeNomR/eK07n0nioFqtF4LwDFF+I9TT2ENh/2NTWAsW5kHcNxL7LeTuPLgDIBei0gOg72gwn4xgqb12vcQmS5vhQtVVAy62WWlrCzhF4zDGYEmm4qM9Cq19BLaI3aGeg02aRHt5q+sQzxugH0hw5fW5E1Kwl30q1IelWvJOzFRs+fkQHw+l0BK92qYJgJePZZoDv04iU7kFla1UsFTsgCJneMtQg6jAFNw0S2TZTp+sLjqsAwjQyeBjcRljdJz5SXHtF+jviEa8Nwn4MTT+1QSf9oewhmF6t3DHSH4ktO1T9ywwnDGTJJBzWteG3dNpN962JYzzQI+xXrZ1xO9JumgZ+iuNO0J16uZQRf2kUvZZJybq2kn8JYj9PP5VyJdDHW3RSp8oY44FajnxP2MsfEVmSbJFFjlP02J2FMDA1aBro2MouSXqv2vslajR4+keWpuH8xOKYV0JU8o9rFGpbInqnoU+2nAjj6mRJP7QB2HathvCTSqoHY1SrHYZdGvHVR9j1M5PX9VACx7D9fAwvi14tSu1oxxPRZPewlvSlARzpQE7Mn4LM7iCm0KskLiaA6nt94/SRRdS8NTYN30woAG727QY7JZ9JhmCST3kb5Z9yWHvbrm1YYPe2EPYO9LFqYVkZPqsgQ07WR/fGFJLB2c9l+GvhLxZMaJcvynEXJaAumDjRHwhJlW2iO47MVz29KCE4gvZurfQVLqrR7IOVOEtp2oSVAR0FxK4DQx3Ce3SPLUMlx6lgz43PYvmlIcsvUZZh8zPpnvKLit2QPWuIzLCybvBbk/O/li7Kf/USkntkmkNVhZPFMtw/9eVjfgoL6yEQ5t4AJPcAw0Q88vOkcCQPZLjT1iaSyJglsks3arfgO03uBgOJRHpwknPEzTlKeBdCmLk4UscYnVZYgvsUoPXVllM+GgLd6L68fBLAleazPJIjH1tIx6WIPZpLrulJYN1dqSdXt9vu2vw4xMKiIEtsg2w0GqTxhv64v1F76b2Ufk1zWFQaf61iVJKwmyLZ8bhgqExN6I3uf7c/fMI3nOjP4wkQLgh5fT0B8oEZSdJPvEh9iwmoe72GRkbrSoAbxggYj7y/bMgSrBSzGcCBfWnxu8FVGyLRIcR/IlDEq/g2MrJxS4+g7GDMCQBUCocrFo6mUMHTRrzlKWxOLqDVhWhKmFX5WCpAsi/Hf8lxGFjKFAT1kJGMwUChzYTPLQljJqpQvYEx+hRjk44WRTPuiNMoaYWxhDyhTcmsM7BkrRhI7GX+mlEIZgzKGVG/yD/v6CyGFDcQ/1n/GX78gyy81sGyGjK/+8AnPP37MB9dP+J0ffoe/9ez7/C8/+S6vP37Ef/s7v8lXf/tdtv/zIy7vZ1z/7ae8vlvw2z/6VT76/Al/78fv88Gnb/P8/JjuRwd88dvv060KPvjkHQDajw743Zfv8t988qt88nvvUm8LwrMp5ncOuHh9yPrvPOKrD97k+etj3HXB60/OuLw8gN8+4fYnJ3z64Vv84efv8MXLU15eH5L9zoLN3YSL58dcf3ZM98mCelOg/rtjLm8W3CyndB8e8PrHj5j/13NWnx5x9dURu03B/bNDJmXH7tWMouz48v94l2fLI67u5rTPZ9x8eIb/4ACtPdlvH7K6mvHob+W4Dw8YVjnb+4rQGp4tj/nqizP6Hx7S9Jb7+yl3rw64+fSE9ou5ALmvKoqPK24vFxyVNW2bsd6W+GdTVK84+19z/E3O7HPL5npK9WVObh3FRxXVuWF7OaW5rgidYfYcHv9PAsa3m5Lh0zmLScP0mWX390+Y//2S7mKC+4NDeF2wXlZUX2U4p8lfZXTbnPWF0AHluXxITb+U7+XLjPrjQ4oPK/rP56iNIXtR0Oxy7l8tsJc5q1XFcFPi/u8j7KuC4z+w5FeG4qNK/lieTwRoXWboKxmpTw9qyktL2Frmf7fCrjX1usDtLIvP4OJ+Tv5phXsxIVtphs4w+3HO5JWmfJmhG8Xj/0sxnzZMnxvc5zPMFxXDxQTVaGZfKfLnOfbWolvN5JVm+kcls59anixWHHysKG40xx9oFj/KOHq0xqxl25PXmukzg1kbqgtFtyyYPdPSg+rkw8XcWY4+EPno5FyR32hUpzn+yFG9lnPX3xfScbm1KBWYvDQUV5bJV5Z80TL9SqMmA0oHHv9uoLpUTF9ojj8wqE6T32vCoKnONW//jzW+8mTzlpMPAtjA0ceB4kZYtvmnlslrxen7t5iDHrvS+IuS6kLz3n91jVJw8EFOdp1RXmuqSwGts6+EicuXmvJSk20EnBS3Cj3tUZ2iuhAAbpeW6UsBv6Fy5Lcmsi2RybQiNfQnPfrpjupSUk1lG5rqQsWAHKkaya8M8+demK9GRtmLzwRgFbfw5v/pmL4KFFdixpqcSy3H9KWkjpavo3x0EOnd6YcD2UaRrRWLz4V5OPoxqEXH4aeO+bNAtlQc/ziw+DwyWjd67Kk1O8XsZaC6CmQrYcSztYCxw7fvUQ6mLyWwKoseTLtB9vFeztvkXKSO/Rsd8y9hciH/n5wrqnOp5TG1ktL3txqqy4BdGWYvhd0obhWnH3aUN0ESeJfyHrqXWg+7C0xeiZ+4vA0jqCtvA7OXfvTj9O+22AaKdzYc/cTRnMLkdaD4/pL588CwcJS3AvaHSWD+lefok4HJpRe2s2b0Aete1qmuPd0iMDkXAO4z+fnspcdnMDkPTF8G6fNUwsplWyhvhZ2eP/PYHZT3nvImcPhTT3cgLF5KA7V14OjjgWBhcilsTHEfmJ47pucen4s8M18FhjLQf6fm4IuB+YuBxVcDh585qmvP/OUwMpbSaSky48WzAbuDbuGZvxjG47PbQHktxzi99BT3nsn1wPR1LxMtVUwrbWH2ykkS7EJYPZEdwuy1o7oNVNfha0m2PoN8Iywq0UOaGD7l4OgnO5GczmUd28DxTxq6eaA9CZGRlMmGfBWorh2zlx35Shi0s99fjr7stG7qRrVN4ODjtYTruMDiy57DT1vsDop7zzBh9A8XyyD1LbeBfh7Z9lzSXPN1kPUj06l8YPmDgepGrn1WBznG6IHNV4Hpq57qdhCp77HIUL2B6magXDqGqWLyqubwx2uqa0Gy5VLqR+5+w5GvPet3oLjryXYCgJojAcpDqZm+ajn4ssG0gWznsbWwncX9ICmxtSPbDJT3wq4O0uSAy6BYepFvxg7R5XehuOuYvqjxmZIJrz5gGk95K+xitpPOT9N6YRwRJnKI+5OtOvQQyJcyEV09X0ePMBQXO7k+64HyuiO/78mXwtjZ2mN3Dp9rJq+E0evmimw3oFsJNTKtI18OEZwOuFImhqYfXwmb13thPWtPvnZRLj1Ieu75knZuyJatpPOuO/KbWhjVQdJhJQFaUl31EMhudwLoB09x06BcoFvYeE9ospstuhH5qmncCD5DZABdJRUwZtuRrTrs9RrdDOh1PX5GJd+v2XSgIbvZkl9tcYXBrhoB4zOpGrG3W/E3+oCue3Tn0N2A3rT4KsNPMvRyi17tUI2wgz7TmJfX6G2Lajt0N8Dg8LkZPZuqEwms2rUxEEkLa9n1woRuawGkXf81wJaAq2o73KwgSVYxWhJhjRFG0nmRtWrZ95BZCQMiguvYWwlIuqvRso0UsNPLveSXq1EGG4YBjBEWsx8IbSt+0GEYpbYhrue7Xr43Lb5pCb38zLftz/+Kvwtt+6fGCr+IS8pP+rP8+kVZfqlTYc/+iZPwW//pX2fdFfTO8MZ0xW7IeW92ywc3T5hkPfOs4Siv+d8++zb/1Lc+4fcv3mLwmqYWvVdZ9hTW8S++8wH/5We/yTePb8jNwEdXjzmZ7gC421X4oPBeMS073prf82J9iPOKu7sZh4fCYOXWcXF5wKOzFeu6QClo6pyi7NhdTXn63jWrpmDzYsHsrRUn0x1fPj/jV7/1gg+/eEqZGEPraJuMp6f3fPnsDDsZ8E5hrCcExVtnd5zfLzDGs72rsJXIDbTxOKdZzGq2dcGvP33JHzx/mzwfaLY5rDLMSctk0gorpyQV1w2GatKxWxewzJi8JdP026s4CjNBpIBFT7PNefL4npfPT3jvG5dcb6Y0dY6/LMmebKmKntW6wq8z7EHHyeGGXz99yf/w4Q+g1WSHLcZ6Dmc7Li4OmSwattcTJic7dldTkQo+2tG38ociKwbauxK9Nag3GlxrqOYt9aqkWjTU64LD4y33NzPwMD2u2V5OZWBUOXTucPc50zfEE/or777ii9tj6l3BZNqwfjVHTRy27BmaDHOex9AheS6mj+R1+qLAHQ7MTrfsvlzg5wOqEbbLrgz63a10dzaWw2O5H+6/OKJ6a02zy/GNRW8MnLXMZg1V3nO7nNJvculYXVYihdVBwpvWGaoaCK2hfJXRPBo4fLpiW+cMTUZwiulRTf/RguN/8pKLz04JJmB2MYzIBkkhDiIZJcD8nRWbTYl6XYos8aBHX2cop7Dvb2iSFLkVtjtsrEjw5j2uMVIfsujRywxOWrgq8IuByVFN/WqG7hTlN9bUz+eYxzVu0HBT4CuHng6o16UkDisBaL7wzN9Z0f7wkO7ICwN+3NA3dqwgsWvDcNqLl9LKAJxBUV5pmjOPOm4pPqkYJtLPifXY20zOQQx/GYONHjXkH03wFoZv7/A3BcWNoXkk3tLy3DKUQapWBgWLHps7wvOJdDqeOsxOEpLzk4butiS7ixUh7zao8xJfecLEUX6Z0x/IrK07GlDWoy9zeLOVazsII5NfG4ZZrCPZGvxEakmESYRspWieOFSrxAM6yCSI3UhCbyDWeGTQfbtm+oeVMJP5PuRJDZJw2c8CofBUL6zIoqM01TSK4aSnfJFLHcPjgemXlvoNT/Va0y9E/tme+AcBMAJqk0+yX4QxzXfyStOc7fsXU4pxeSkDugRqpl8Flt+NSbQVVFdSN5JtFO2RZ/JamKft2x5fBMxGky8V7XGguJWqknwpwT7DJPorj2VfVRCZ7epbnvJS0x8IcE/Jxd1CfJwpobc5EWlv96Z0z7oSihtFP5P1JKAn4Cae6pWhPfFRiqwiIyhA3xUCBpNEdP1uHKSqeJ4nEjZ18LFi+xZMXsHuCcy+gu1TmYQwNay/5aheSY9jYm3VINfQ7mR/XCnVLCM7F5lXb/dhTCoC8H4WKG+ihzATqWpzIv5O8SmGOPEh7GE/l/0eJsLEltfi0RymxH5OOaZuLlLm4k78mq6U+2P62rN7rOkOxM8pfb2QRQlvexh9s6W8f3EP7bFMVJS38l7FnYDJYCNTXAvDlK/l3NZn8m/dx3TZyDr2M8X0XIBZXynaI2Emxz7TLPapRsbLtFFarIStdTlj12ZzokYptOkE1M5eelbvajnGR1Itkm33qbOJnRRpsPhCs228PnNht4eYwpxUDKYNdHNFHvtWh0pScduF1I3UJ8IiF/ex/iNTtHMt7GOfZMMiSdW9gNl+oimWkkraLiJzZ6UipDnUTG4cpvas386Yvxhojs1YUWJaqT8pl46+0mPQVb5ymM6zfTMjX3th4Cth46fnUhcyFIps5+nmmnztGSp5z3wp45ShMnRzQ75y1GeWxbOG7RuF9GkWwiZWVx3tUSY1Jb2ATFcZ8V3megwbMrWjX1iK2x7de/qZxVUa0wRslKS6TD538usd/XEl0lSrx/TZYBVD7Ne0tUN3Dl8YzLYfgRaJwXTCzJpdh6syTDOM8lrlPMFodOcYZrn4LCO7FrQamUzdOVTvwWpU72Q7u46gtQDMh4v3hCJDtb3IaOsOPy3R20Z8kk3L2BWZZ9BLlUrIrLCUvXhDMeLBDEYL8HyARFTzoH5kTIv1wlomiatSUhkCIosdpbvyvmG5gqLYh/2kmhJjBFQ6J3UiD3yU6fjG5WHqbfr/z1u8xz8AvX/a5RchFbZ4/2l4+u/+63/m2/3ir/9b/9iPDX7JgWXx/tPwvf/oXxM2rC6kky/34mNTSC9ep1CPG9xOfGcjBR3TVtXWoI46/CYjO2rwLyfkt9JvZ7+9ZvhkLjK0XAYEVA61jcEoUS4YSoe9yRgWUjwvEkSRB469ekUApyhuNe2x+DZ8EbAbzTD1mK2EF/RzP9ZiFLcmpoMyDiRdEfvfchcO7ScAACAASURBVJGUdUeSpGZ2Cv/NGvWsihUS0J4N4rEz0d/TR/lFkPc2uxj/Hgcfqe5hOJAYz+zWkG0U9WNPcadpHg3YlcHNpEuvO3FULw3hN1eEP1rQHXnyO+kCRMHwqEOtLdlS40rxtPVzGTBu33YU13Kd2mMvfrXoiernMgiSqHliwALs3vRU53oMUNk98Rx/qKjPVEzFFF+VraWg3u4U+UoxTOIA+E4GijoOFNsY/FFdKdojGbB1Cxkg9gcyg+9KYaxsLfszTGUf+jljOqRtojfsQLZhd7Kvs+daBp0G2hP5f1BScD/7StEeg+qFcQC5LtOXsPxeIFtJGX0/k8Fjd7D3eqUevJQCW78hTFO2kYFaUNAfBOafw/33ZbBfnWvqN3z0rMUBbCPhMSoyS91CBni6l8AO8SFCdwDtqTwv1WvNMIXySu6Z7kAG5vm9DI7qx4rqIrB5W6SJKVUzFdDbRiSm+UqOo34k56JbMFZsuBxmLwPbJwpTy/ZdIb+3jfy8uJMBt8tlkKwc1I9g9jzQHidpoiRt9rO9X1CulwyClRMmpD5T2FiL4SNLYLeB5kyeD7tjn7q6QySgJso/N4wJqN2hsKIkQBU79toD2QfT7s9zqvuYnAc27yjsRradkklTb98wUeTLEMGQACbTyP0yVDJgLZaBdqG+BqJdJoP8bBN9WTambToZRNenmuI+jGExs1cSVNIeKeZfOZpjTXugmFzGe6GVAawrFHYbWDzr2D7JRomnK+Q5A+k09Fa8at1c0S3E+5dqL0wrA+zqxtNP1Vj1Ud471m9ZJpee3al06LlCjfUoQyXsWp+SWyMT1s8UQwXTc09zpJmeuzEkZIiJwLaJzNQ69gAaYYbaA015J4Nfl6kxpGUoFJMbR7sw0ku5i2xbFiVpgRFkp3qUlEKrByTdM56vdiGMQL4JEpQyjb7gmBDbV9Jz2M2UMJWZojk05FsBB2n/kyxTKmjkHqluoly/1JImq+U6lTeO9tBgm9RzGWgPpOOxuhrwuaY9kLoJb8QTaJsITIYwnjeXKyaXQ/xcNsKQZkp8kY2nPZQBeVbLJFY305GpDuPrtRP2rFuYCFp8DI9R1MeabCe+wpTGmq2lDzElzHqjsDsvCamdSAyTJNO0+2qPbCVhMboXD10X9zdf+7FLMXlapdZDfI4ohSsV+f1Ad2glkOiyZvmd6VilYluZEDOtsH4+15jGsX1aYnfC6mnnRcppRO6pBgEq/dxgai+yzAD5fYcvRNYp9VCBbN3THudUr7a4WY7LND6T+ybbiG/RTewIbHyuGSaG/L7HFRobAVc/tdg6aYURcGMU7WFGednSLwSs+VxjNz0Yhcs0djswzDN0L52SdudQzjNMLCbWkXijya+3+DLDF5ZhKpUfACp2RupmkGTewoJS6DaF3iSfpUJ3DldadCugTPmAqyzZXUN/WIrHsRkIRcwjaAZU3TEcTzF1L4E5234EZz43mGUTU5QtvrJj5YluhrHiIxhFMAa9E8arfzSX6pLCSN+lD/sgHe9RTY9fVOhdJ4E8m3ovK42ALeQSIKiSJ1FrQpkJM2m0hNvULWExFU9k1wsgbDv5nroqlSJYI4xiqvDQWuSrMEpZE8gECGUOq83Y/6jKAtqO4DxqUsp2huHr/khrCNsaZeXchi6+X/B/XMZaFNFPGffhIegDAZ2ZHaWyYZAQo9DFzkovLCvRK6mMke8/s53Umyk32Z+AR9I+Bv8P7rv8f7H8wgDLf+fPAVj+y38JLP+Rl6PvPQr/7H/2z2NU4FGx5mV9SKYdy65Eq8DEdlSm5/P1Kb918iV/5/KbPJku+e7sguf1MX3QGBX45P6M5a7i1x6/4nc/+wbvvnlDrveRzN9eXHHZzrjczbnZTlAqcDLdse1ynsxWnG/n3G8qThdbplnHXVMxyXqawfLGdM12yCnMwMVmTjsYFmWL1Z6X14c8Pb1HqcBB3vByfcA07yQIp8uo25x3ju8kHTYbuN9WTIqe27spv/rOK54vD5kVHeumYF62rJuCbjA8OVxxu6vY1uKt3G0LFvOaadFho6fycj2jynt2bU6R9WTGs2kKiqyn7TN264LHj5asdiVaB8pswBrH+RcnkHumxzW71zPscYMbNO+8ccv5/YL+yxnz79xRZAMX54fQaCg9+bTjvdNbXq/nbO4noAPfeHLN518+YnosZse+N0yrlrtXB5hFR/CKLB+oil6kuL1G24DvNbbs6bc585Mtm2XFdNFQ70SuOj/bSCpvZ3GDoSg7hsGgVKBdFaChWjS4n8zJf7CU160zyDy2GnCDJnQaZQPKevw6Y/p4S9cZ/MsJT37lgpvNBGM861dzDp+uuH+1kEFL6bFXmYREvN/grwtm7y5pGvkDNFxX0V8ifkO0MITl0w31smRyWNN9tmBYOBZvrFndTjGFw+0s9tbKIH7mCZWjfJ7jvreFZxP6sx4VE3r9zoINmFvpF02doOXLjPaRk3RXgEFLOmzmyT+paN4c5I+/9Zi7jJAF2UcF2dLQH4osCR2w9xb3uENf5VJQfuyxJw3uvMJXHrMRf40vPLrV2I3CdIrmTJJdWfSwzChuDP13apQKDPc52UomU/yBMKM+l5Alog/XZ9I16RYO1WrsWtMfyPEUV1b8kFMvtSWtjkEVCld6qteG3Xs9emso7jT9PDDMnAQGeUY2Kb8zDBORtLpKfu5jH6grg0yeTAUodWcO1SkmL42kkF5pvIX6nZ7s1gqbFPs1gxGw7fIQ90nSWlOdiLeM9QSpasVNPdVLI6xhDONpzh4wh330v3oorzXNqY8dqDIxMEzjRIoNEYRHr10ux5RktXYnKarKw+y5ZvdEGL76iRO590TWNQ2jly55ZF0lx2B3QIDmLJDfq3GCKHlBfQJgsc6jO/Yil14Ke/bQA2kaSS/VTljb4l5AWnMiTGQ/l8mK7iC+dy3AM9syTlI0p4HpCzVKKlNHq3KynoBExo5IFWD20rN9Uwv4jF2oqefRtMJANcfiRdSdAPM0sZH8e/g42dMo8iWxEzAmom5lorA9VGOaar6UtNvqMu6rh+lLT32m9/UmYZ9wmrYjAVFqrB4aJsIkBrOf7ACZ2KluhV1zeWTKvMiAm2PpRrQ7aE4FMOuBryWBDpWKE4OMNR6J7UshN82ZBDslZnDzlqK6ikBOxUkNH6KkUSZchlLu99mrgd2ZHftzE8PochlslreefqbG9GCZaFJMLxzbR4bU85k6TW0t51fFoJ98G9nAtaSuBvvgmnZynN1cJttAgHk3VWO4DcgkiTfyvTnU5BtZp1x6umlMfN0KIC/vHe2BGTsnIZ7HUpFvBaDaRj5X27mhWDn6qaa4dzRHhqAV04uefmownac5MpS3sg5K0U2FVSzvBIDrXt7XRHZvKLVcm0ZSXW0bsDsvPZidAMeULuszmL4WptHGyQHlAv1M2M5+Kp5Ju3MME0N7oKOM240TdSm8qJsZ8o0E/fRTi905YQ9rT3cY/3blwmbarWyvuOtoTnPy1UC3sGRrR9DQz63Ia4u9l153XkB0I1Ujygf6qaW463ClERnsuheQFmtnglGyH4VmmBrKqxYfAavd9PSLXCZG1h0MnuGgEClrOzDMC5Fpd16kq/UwVrGAgHZXZSKB3fWEyIwGBbqJDGMEyLoZ9lUoIUharJH19abBlzm66eR7N+BLO9aHJLCu6m4PavNsz+gpJaC1H/ZAt+0IRf61oJ4wrWS9xHL2g4DftJ3UnZmWEIFplo2gFCBsYzhF7K1U8fcpoCe0HcraEQiGrhNQmeSwMSn2jy0JiP68nz34HkL4R5LD/iWw/PNf/tyBpVLKAL8HvAwh/HNKqW8AfxM4AX4f+FdDCJ1SqgD+c+A3gRvgr4YQvvyTtv3mD47Cv/Rf/DO8Xd7yRX3God3x4eoJ78+umeiOF80hC9titeMny8cA9N5wUm7ZDTlWea7rKY8ma96Z3vF8e8Qsa7nYLZhmLV/cnXA4qcf30yrQDpZtm3M4qWkGOwLI3Dgq21MPGfO85XI746BoMNqz63MuVzMeH6xZNQVtn/F4seYgr/lqdcTJZMu6KzAq0HvNLO/YdDmFcSgVcF6TGceulw+NTVPw9uE95+s5zmsBiF029nYu65LMOuouo20tee4w2tP1Fq093zy94SevHqN0IMvcmHh6UDW8vj5gMa85nW05X82x2jP4OGse+0HrNqNrMqbzBqUC3mv63mCMpyo66jbHOc03zm642U1pB0PTZASvKauOk+mOu10lAUNAvctZzGuaTo4vy+SDsO8ts6pluSnJMkeRDbS9ZXdfUR00aB2odznGyH5pI0FAZd6za3IJsglKPt9MQGsJFsoyh9aeYTAMvYl+eC9hQ8Ugn9OdQZnAdNJSNxn9JkcXDmUCrjVk5UC/yskOWlxv0CaIPzVAUfXUK4lRLGcd7S5DW3n/oTfYPEp0ekMIiqwYGDojia1OkR3Ih+a06ri/mqHLmNJn4z7mjqGVsB7VaULhwCv0ZMB3ibKSpFVlZADjG0M26+R1jQzGMAG1NTAfJNW18JGFl9frwuG3IpUNhRM1wKAx5YDbZCNjr3OHb42sX8sAws56htbAoFGNJpRxBNIlEwuiGhhEVRDKOArOPGpnpWM1C9Ar1HQgdGbsbVU7I/vqxRNJQFjCQwGraAiZF+mxBj9xqNqgzxrCeZTjVk62k8fajigLVa3Cz+L53GkJkMnDCJRDLoPlkHlRRMRKF195GWTFZF/l1NiNGtI+tJpULmh2muHQYe+lLsUdDRQvMrpjAeW+EJBbXhnaIwF87nCQ67IzsepBAGOS1ZrInAtIlePRwx54BCMhQLqLaoBClBIAePGqmk6NYHeYe+xaj5LibC1VIMEKuE+BTjpKYu0ulasrXBXQrby/q0Sym2SdqUsz9ZASFMPMo1sVw1Fit2AZxvqHFCqUZKBBJ/lf7J61Elg0TMLIBNqNgAgJv5H9S9tUQeTY/Twm1MZBclJ0pJCYfi4AGwQEgagCdAzwkV8I6/8QRKQ0Xknxjfc78Zi8MMAhdTzG5NnUP5u8pXYbOw8jaAo6guoFY2CM6cZbCm8ZAWDymdpG1h0qxh7RlO4KUZEwZww6SgoRSR2Nxxp7aFPwSwKnKUE3KEaJ8Qhm+33gULbdX8PENKd7ylWMvsukwkgAU5Jdw75PMQjgBR4kqwaRHzthUrWL68drlZJjtdtL48fjNAKuTCMsqMsi+C0U2W7fffnwmg5TFf1/6fWM4TkJmAa9nyBKkwJjd2nYn6eUFuuKFDS1By3Kh3gOhWkNRo1hRCowym6lZ1PtWeL4bJk2jBJVFeKEgIkKjFZYf9ukvkqRErtCxWspDFo698XS0c1FBmr6GA4Uz43pQgzAivsWQ3lUkIoVW+97OSXBWLY7drDG8yX3TFSA6RRGJJ9rKfAoeSVToFPq7EQxBgT5LALRyOyilXTFashWAlR12nYSCmm+JmHWTryjqo/d4BHEqmF/HZTzUh8y+BgmxL5HU6Xj8Xir0TEwKF0/qXnR6NR9OfgHIF1kqMEYVJKVxpCgYEVKmwJ5JGk2gsPBMfZeDlFumkDjQxZQCwgdZachyDYSS5rSXkGAZOrFTL8DRrlsksIaI37MFPQTYsjPQ4YxbSctPyuB/Qf97ucsfzGA5Vvhyb/9Zw8sv/xX/s1/7McG/9/0WP4bwEfAIv7/3wf+gxDC31RK/SfA3wD+4/j9LoTwLaXUX4vr/dU/acO3myn//Sc/wLWG4nnBMJEP9w8P3gUDxZWhn3n00xq+mODebuC64HknfWyukpS6Cx7z4Xtb/MuJsBK5zOjrQdHdH8uHd5mSKcEfDGyujsiXimXsnnOlyFr1AM+PPKZRrDcPpJ3XiufvzMjvNXj44mxGttToXnFzdiSy0CAMw/1OWI1giP1qwnBIyl+gfTzw7O8djv1s1ydRXvqmx58f0R3JwLD+Zkt2keO3ChdlY5OXgY++uyBbyaDLIYOiwcK1htkdbN4ouV8sWHxiGWxMgRygtRAy0L+yIv+yYPNUk59nDNPA4hPN7mngvgoUdxoT4MtsRncifrXyQmNaaI8rrjeHtEd7L9X0haY+qGL8P9z/Ws/kWYafBjbtnLyVAdX6XWFCzKnHfLSgOQsUa0X9ZKB6JWzVoMB+DNmZwsTCetMwerfqx56hlzCT4cAzfRnB7aNAsYsS0VZ8P7qH+qCSEItX0C1ydm87Hv1dzfqdgmoL3Txj8Sqw/I58zpbXin5RwtnA4qOcoSqoAAJMLgLrd0UOrJzCxtqE+u2e4twKcwW0x5byWrE9CsxuFOWN9NUN04AJ0C8c02eW+rHn6CPF6n1NcaPINhbbyOx7/YYAjsRmTV/A3a9ryldWWJSwZ2ryteb+O2A6Sf8rb4T1yTYyQGmPlMxg36rI8kh63vRVYP2ewn1/Q/VhxeQ8cP1XPIcfau6/bzC9sGhBQbCGyatAdyD3cXUdqE8tm28OnPyhpjk20sV3KgP/w48Dd99XTF4rdm8a8tUeIPUzOPxUsXukaY8D82dyfu+/JwFKdhfQbi8N7Oea8iawDCVHP1a4XLP8rZ7qdU51JdJTl8PBF472QNEdZNht4O4HwnrOPlVMLj3Xv6FYfKjYvAtJczr7Sgak27clVMjHFM7F54FuoWVwnUNzBtMX4jkjaIIO5NeGg5/C6n2F3WYUNwq7MwwVHP1kP7u/6YQRGboM3SuqC2FH2kORS1c30lFXP1bMvwgEK16x5KerrmRg15xofAGHH3tW72nsTkf/mgx2Hv/ewPptS3Osmb0M3H1P8/R/H7j7Tka+jJUaSqpRiqXIJl0u/rp+pqWH8krOVXsg10vkwYr2CI5+4sfBrs9h/rzj6jdyTAoQugnszjSTS8fd9wzHP4o+xbf1CFzmzz27My3ewLliei7vVZ8qDj/zLL+hOfrUsX2kmV44Vu8a8eoVsWLBi0x181RT3AW6WnHwuWP1nsE0wiBOzuV9q9vA7pGWAXWU76og+9DN1FizIemue4ZPOZhcOYKF5XtWpNg1lDexo1HB4ecDy/csB18M3H/LcvRJz+aplWqWtePuW5bJxb7wvp+rkZkSFlSA29EnA2oIDFMZANfHEshULIWhLJYCIrK1PDv5RpiufqLZPDFMLjymjd2MV3709Q2xaxEl53j6omH7tKSfiofx8NMeXyi2jyUka/ZqoFsY2gNhymcvO+6/nROUsKsHX4qPLl+LTPn+W1YCl+aS4Hv6w4712xnewvGnPf3MYHcyIG8PNFktnY35xkWLgmb3SDO9kFoJEysdXGnop5rypo+fpRl9pchqYRyLpcMbRX0mQ5/qWt4rX8kgvp9qXKEo7kRKnLyO3sp3FWDxZUt7nJHfD6zfzplc9eguWmu8fH6DgBUVk2Oryw4VAs1JLn2Mc0M/0VSXPcPMUFx37J6UTF/s6I5yhkpja083txx+2tCc5bhctmPXLd1RSb8QVm+ohNmUJFOp/Ph/2HvzWNuy/L7rs9ba45nufN/8qqq7q7ur7VY7HmLHtuwQYkcKmYgiEgQEkFAUgSIUM+U/AhJ/AxICKVIiLP5BDJESKSDAENtEOHG70+52z901vHr1hjufcY9r4I/f2vu+amI7kt3G7vhIT3Xr3nP2sPY+56zv+k5SsWEoX+zwWcLm9ZLyosdOY69kSCjPGnTnqB5OmDyraY8K+mlksGthECcve+zEkG49pvNkVzV6XVN9/Ajdyrj3swTdB/KzHc3dKT7TpBuLcgnZssenGt17knUj3sYipZ8lZDct9b2S6btrqscL8suG7lAknOWLHe3phGRnMZuGYAx2LydZtbhFhq4tbpKS7HrsNCW9rFDO4WcFrpBsgOSmJuSRwesknMftTcXHGAJ2v8RsW3yRYmcZaEX+fE3IpG9TbxoG6WuIKa2qd/giQa9r3MEUc7MTZlAj8tc0GRlJZT06BAnJKVLxUG5rwiRHrytCmqCsw08K9LYi5JmwiUNqbNsJyJyWqF0N+3PUtiaUOco6wa9tJ+xl18vvm45YAA6tFxCZJiKz3ewE/MXni9TWEqz7EAgcFwmKYmQgx9cAKs9u60Oi5DVUtbCUcNszObCMfU8IQf4+vOZVMPlPAJK/l5WU/9SP7+JT/I4ylkqph8DPAv8Z8DPAnwQugLshBKuU+kPAXw8h/DGl1P8Wf/4lpVQCvAROwm9wgJM374c//rN/ipe7BT4oHs6XfOnsHlli2VY5h4uKMu2Zph3vXR+O7GOiPau6IDGeLLFcraekqeMTx+d87itvQBpIy54Hx0ueXe1xtLfjzmTLF771iHTS8+jkhg+u9snzniKyaNtNQTntxvfLyXzH1W7C9mrC5ECSVbX29LsMvCJbtMwmDevNhIO9HZcXC5Lckhc9bZNiEqncCF7hrBYmKoCZWMJ5waNPveTJkxPMtB/Hw3eG0Gn272xYXk+5c2fFtsnZnU+l623REZyWQJt1BrkDqzETK8dWp2PATpZbqmcz0Egh/V1ZIdJnOe5QSusBQual2L4x6FlP8m5Bd9eid4Zw2AmrVktCqTppOdjbsfzyESgo3lyxPZuhSktaWLpVLquPOwE4JML4uD1L9iKlO4l+jolFXWb4gx5q8amQeeg0utH40qNnPb4W+aibetReh9/Jl0yyNNiTHnOd4nM/VoLkl0a6BYvYOWYV4XENLwrczKGsAAKSAImH1qBaTcg9uop9egGypaJ6ZMmu5TzC6zV2nUnnoQq4PUd2ntDveWG0EvAzK9diK7UgEBmIiZfAmKl0Q9pZwOfi4c0ujVR61AIgh5XSpFJ0h47yeUK374U12Ig8srkrATTKgZ17YTadsEe+lOMxtazwyuq1oj10JJVGRbaj2/dkNyJZRQEe0p2wRS4XAD+kHCa1YDBTiS+yPZKFFqlBEXmkqRXtqSW9MeQ3avSq9ouA6uUYQOShhFtWKmnk2PJLQ7/wURI5PDdK5rLY19bKOblMtu3TuD3NWO2SX0toycCAFVeKdl+8uS4TX/LAmtkyjFUbyom/tTmWBSSfQrcvizv5tbAy/TSgnSJo8UtmK0V96pk91Wxf86RrFeWjjD2QkoYovk1bMtYF2ImM8bCwlDSvsGdxJX+omni1Y9MVwh6JhFJYP+0E6ExeCuiX+ydOkA9u2SSQoJehggQYmUGfxe2XskilfGSy9C1zNyxiyPYlBEX8s9JNWd1RlJcClssLkZvKwhrYqYS+hESuwxAKlG4iK6Nk8Sup4/kOtRbpIO2UxQOQY9BdoF8o0rWExEzOA/1EtmlaKbkXH3Ggui9AqbgOsbaCsWMRJK1yqPsYeiAHsCle1ijrzAWIFtcBV4iUM93K84UFk21kyxDZt/he2xMvpTcidRyYJh/7LIf+wgHogryuuPHsTkWOaXoZg+qOLGpKYq+8tj6K/s91HMs4RkM3pi1jAml7ew3tBKZnTiSXRs51YOIGienw3KAZ6zykxuHDzJR4YwXo9rPYe/oKozYwVsFEn20RazFKTfqKB9WnxKAY8ZFqG8bAmrQSf2q+Fm+ny4i+0yGd1mMLTdJ4upmOftpAdWzitY6BNpFhM61IMoewHJfKPiCy6rWnOknI15KE3E81xbXDliqGDwm4JgjoNK10LJpG6j4kUMijW0+3l5DufAyP0iQ7H72CjCyYaUSqqe2tRzYYeR8Ii8Yr1SMynj6/ZQx1K6+3EwGzwch9YGqHzzW6FcWEnRp058cuTpdpsrWln5n4fnQCwHcWnxups7EB0zi6/XQcr2wlHt/Bz5luJUl2VApERlP5ECWoTryQk0RqaRJFuhYvpM+k9sM0lsG/aaepMJ5GmD7TuiiN1beBOl7Gxk1TfCIyV93GFFYXRlDkYoiPLxPMupXvxTxFdRZcIOSGkBrMusHnopZRvSPEmg2z2hESI+AuNcJOgshWy0z8n9bd1oG4yGJm6a38leHz1gtAtE62CePPagjpeWW6HIyW30d56fiarmdMin01rAdumUgQADh4GdOUMdV1CPEZGEytbkN+QKSvaXLLVg77g+jDjNKO34SZ/PbHq1Dgu4Kx/E+/A4zlv/bPBmP5XwD/ITCP/38ELEMIg/P2A+BB/PkB8BQggs5VfP7lqxtUSv0l4C8BmIMDPvfe41FCeL6YYb4yY/2Jmum04fx8D3WTyoS50uz2S/QqgeOWsM4EaAVFqBL6RvO5mwkkgfRlSnmRcTaZoiaBswcpL/tDVKOxqefdr98jqICrJmwOLel5Sr5V9EVB+GiFe1ny/G6C/uaEzEBlAuYqBaswqXjA/LtTNq+Du8q5flGQtIpkl9N8T0V4XuDvtbhtgq5iEFGUZrlOk20VVz93nzKF+pG8YSdPEup7DtNqVv0eptJcXh9jGkUaPVVOpSQbI114i0BoNSHzhLMcm8ZS7UpjfYE5N3DiSDYxROE8G7/c1VVKula0x550mUhQx05hZ4b+wJNeiF/G95p02qFeTnF5IJzlXPV6ZAardEHaKlgabJGRrzTtiSXZCbjRFik832b0C0mztNOAazTlmabxqVQxNAo7DwxhO9mVwTaafC1/673G9Tn5WsvEGuisrI4nWx0nA7Kq35yKh840MrmqlxkqDZh1EgumFT5KBpsTJ0CmMiQ7AUTyBRyYvS3Jm8VSUeUFKruV76SXCa4Iwlh3UZ70IhPmKKZLphtYfdJRXJjxuutW2CJXSG2FdqCtjqEoEjwUjAC16XtJDPpRpL0amZTiTAChbhXlS0N7dAuibGHo9yIQqwXwZktwuSZdi2wwu5GV8HQN3YEa++1AgGhxIcy0dNFFOV/cN1qAULoSX5orYgp6K+mo6U7AUnEhni479xQvzPjeTzdqTFMsz2OY0kRTXArr6hOpPBj8UkMvoJ0F0t0AuOL75UyYWOWgPQyjdE53UqvRz0TumG7E12iaIfJ+kCTeBioNoMC0jH7C7EaN6ZbJTq5JSBDpLnJfL97WTC48m4+K9yxbgSb6uwAAIABJREFUi9xL94wTap8LgPKZwmyJtSkyIU9q2a+PAMy0YaxgqO8osJJ+qrxi8iLQHEtq6QBcywuZZPezKD+zAlKH0CPTQLaS1+U3cmxBC0OZrzz1sR67BHUf6JS8xk6JYybjWt2TsKYi+vx8FvtICxkP3QfSrUz2J2eSaNktDIwhOINnSt6T6RaGHtCkCrhSkV/H0KPoqatjkmUaw36G/s9kJ6CyuJC6i7SScdO97L+8cShnpBs1kdTZbCVANVvLf/upsLKDnE95oLoNkipWjk2ZYAskbdQLw5zUAk5Mr9hN1Lj4UVxFQOoY2bHyWpI707WAq8mNsKTllYQskYZbJq329BMzBhzpNo5NBJ4gdRTpWkC4BOVIz2S6le330VOYrcMoEdVWvJmDvFXAniRfeqNivUUYk02TWio9glYCdJMPp5h6I6yvNzHQpxLvabsQH9/kwlEfGFIbyDeBbqpJK8/kRcvuYUGIx14f6hF0Fkvx5NWHZvwMEhAZRrCnXKC8FvDWFIbJubCrLlWkO5EyaiPHndQC1NKdHeWb9ZEwl9oF2rmmWEK2srT7CaaLQX1ewOgQzJSvJQV1dy+lvBTvetKA8l7A1sbSHqVka0c3H1YENMoGkkHumwkIc6X0KRaXPd2eJKDaqXQNu1xeq3sfpcZBgnxSCRcCaA9SAao2YGKwTz3JUBap44gdkknl0S6ImjGCS+K1VyGQrXtcKkBYWVmkMLUVqadWkZm8BbPisZTrkq76GJAjAUEoUL0n2/WgFcnO0x7lEhrUe0Iu56yDj2mr0u0ZIuhWPuC1GpliUUFIyqpprCwGNP5D1X6msajGYo8kGVa1wvYrHWT8rBegp5SA1lRLOFGiMVUvMlUH9A5fplJR0li81gIaE42uhDGUlOZXJKO9FQnr4I9MDKruogzVE3wi56mV9Fgqhd7UI1AMWQpNJ+E+Sskasvf42QTlYwps20ngDkDb3aa9Dl2SzgsAHCSyfS9BP/Pph6SxIziNXszQNHJOkXFUSgnIs/Fc2lZA5CDd1TpKX/2HAWv8OQzPGx6/WWDP8BgY1e+GR4Dwu6h38rf78R0DlkqpPwGchxA+p5T6w79d2w0h/A3gbwDsffJO+KMf/xpX7ZSvnt+h7xLcWzvyzOK8Jskt5mHH/YMVT84POZxX3JiZ+IkST6hipcVey6OP3fDseo/2usQ9atg+gsmspflgjr5KcVNPfrfC9ob83pbdssSl4lGzc497ICCV8xJ92tIvczh14mFqjJRyH/RgFarT8LjGvZgQSo86aunXGf1RQHUG8gBXGWQBP7fQa2H8Ys+ee6Nhu0tIrxOSpaS0Nnc87PeEq4yQeagV6l5Dt8rECwfoyuCzgD10AnJ3WhjMTuESYW98EWDW00wsai00SD+TlNL+UUc30RzeX3FzPeP4eMPVN47IH22x35zjJhLgwnaCO+pRW4OeB/xHKsLLUmS9NqOfBZrHPcllip15gg6YnaGfewlSeXOLfWcm6bKxQkP3ivQHb/BfOMAuPM2x+MWypaZ+rad8mtLP5DzTtYIYxiKBHgo79fQLT//RlmA1rBNYWMzLDG2hX3jsVGHnjmSXsPtoj2o06UrqILKlgNf2xDF5auj2AuULM7JH1UMnvr9Vgi8D3ZEjuzDsHnmSjUK6DhX1HeiPLaqVaxp0oD/tcVcpeMjWivrTNdVKrqOdCqAMBpr7jmQV5YszkWZPnml2Dz3ZUpPfSKhIfXJbS9DvS8pwfqXx37/Bvjsj2Ql4qe4LGwcSuOLTQHYtssPdY0e61KOvpXpsUb1CBblf+pmAzt0jj75fY746pbhS7B47Jh8YugMJd+j3wdSyOGHqGIjRMXqp2gcde1/MaI8EAPVzj5mJrDQ/Ey9heyTScuUiKMtE8t4fxlX/mTA8/d4QLBJBay5AxGXCvvULz+SZyNWvf9CSnSUUV0pk4dGaajphVbOVnJubyoJGthYWMr9W1HfCbfpnLwChXzB6HX0m41qfChsYFtDe7cnPolRICUDsO0U/12RXEExg+0iNvr5spWI3ZaA+jYnQp+KPLC+kNsPFnsMBPFX3RMZczdXYVZjfyM/1qSQMo2HyQkUwIeec1MJgzp4Kw9buKfKlAMRszVjj4FNoD4QJ8KnGFYp+KlUU1T0Bla6M1RMHKqZjyud2dS8AOoZwBFInAHH7ULr/TCtM2vp1xeRMU58GFu8wypnbA2Fby7NAdVeCgpqpsH5BQ3VXMX0u+x0Y2mwb2DySc4EILBthBHf3FEmjsAUU14zhMLsHiZx7CeV5vIciG9ztqXG8qzs61nxEP2Nkr7t9sBNDsRS2qj4WObQkkAojnexg9kIqKyZnws6Wl576SNMtBIyu3jAj06mtSHfjN+sYelOfSLepy2VRRRj6IdREJKv5UhYG2v1EJOJWAGIwSmowtgLITSPMZrsXGaGdMKvdnibdBIqlw2Wadl+PEuyk9rT7WljzyHTVh3KNyyvP7o5m9RFDcSVJxM2hYnrm8UaYUm0F0PZTxeFXLcuPpdH3J/dbvhaJ981bJeWVsInKy6IGQLMvycHBQHktk1XTC/upfLStpAqlBQT7CCT7maHZF3AqqcPiy9NWPkdsYURqeyyLjsUyjGOrLZjGs3mYMX3Zs32QMjl3KB9uWdQoS94+zCgvpcYjrQRs9FOR3e5el86dTHsB2RtHdSelvOhj4qt8ZnZzQ3nRU5+kdHuGbOOwU/kcGTyMQavY0Sgg2pYC1NvDlOKiI905+plB7Rz1aSYSZyDdCgDrZ4biUvbbz4yA5DYQEkmbrU8zkjoC8M5LMuusBA3dfib7tYF03dPvZfg0xXSefpGQri3toQTkZKuebNXhciPy2pWnvleSL3va/ZTyRU2/n+OmCeXLin4vH69j0IqQ3CbKKp8QElk8Uz7gc0OyaQmpodvLSLcWn0j4jU81uhUQ2J1MSW9EudYdT0g2HRqwsxSX5WQ3rXgnXcBsW2EeNbgyRcd+yWAUZiMdlVgv/1yQSpBEi+S17QlaE+Yi3QlGgKfqhFlU25owLcBoQpag1xV+VgjYbWPITQjCMiZGkmUnsadJK9nXpEBvdrdz49kEta3AB8KsFL9nCCLPVQq1qSJoTgTQJgmqMFDVIlUdAGMEempaSqptmhJcZB4RKWxc9xSwmaaEXSUgE1BWPJvBOVSaCMjsJSlWOSfs56uP38hv+f95XmRffwupsL//+M4/vpOM5Y8Bf0op9ceBAvFY/pfAvlIqiazlQ+BZfP4z4BHwQZTC7iEhPr/uIwR4VNzwc9/4JNk3SqaXgILdj+/wz0oOPnnNzdcO8ftr9JOS4z94if9fj9n7M8+5+Pv3Kc8D+s9eUv3iCbuf3NG9nPDG37Vcf6qMUrQS3nIkj3aENqF9OeHo85rmT3bSJbcbIsWhuCro/twN5f+ScfmnPdnzhHQjk45uX3H45cDLn9TsfynB1IH5X7ih/Tv3qO4k3P0zz3j3nceYVpH+wA2b7YLJM0P3fVvciwmz9zXJLjA9czT7hvawpDkOnH5OVm/txEiK49cKTn61p/0r19S/dofsrR3uF8oxkv/4Cy27eylXn06481nP9VvCzCU1NH+4ons65e4vBc7/Rc/R3ytksuXFB9hPYacy7v4jx8t/taD8WsG6KLj7JceLn5hy8B40h4a9/7tEhcDlp1ORZNYz/r0/+Xf57/72n6C6o2kPZLJ8+WNgDy33/3eNLQzrNwQ0n/yq5bmesf91aA81mzctR18URmG1PODur/bs7iZxwmdIqkC3l7B4R+Lq61OpUZAVS5lUTl86dn9xhf57B6xOwTuFWlhYpex/Xe4lO9XsHgYmTxOOvmypXgjjd/M9cPqPFNp6No81k+eG6l6IVSZw+vmYaPhDO7JfXDB77tg8NMKA7gLrNzT9NOALz+FXHNtVyuUPw/Eb12x++YTTz1me/csOewh7X8ik/+8sJ6kUxZXh6CsdNx8T8LsNmnu/ZHn/zzsOfy7HZYrlJwKzJ3qcIB7/agVGcfYDJcV1IL+RyoHFkw77xZLL71WcfNFSHxl0r7j72Zr2IOXZT2rpI9wLHP8/jqATFk88xXXP0z+SkqwNi3due9iG8IvpU816kTE/C0wuPNs3PflS0y9kslueRzZpqrjzD9e8/NEFx19s2N3LyFaK5hEcfbll80Oe4klJSBQnnxd2QXeB5sgQtHjqlIP9b3Vcfibj9JdXXPzAAlfA0Zd6tA2sX0tpjuDOZ1tWH82YPbfUR4lM7Hthd2YvLPlly+YNqeU5/EpHtzB0c83Rr22x05TVGzntgeLkc3D2Y4rTz/ekG8uLHys5+XzL+Q/m3P1jT3n284+YvAzMXvS8/CHxa+690/HyR3JsCfkVI1NynibM3pfPp24uE36fBva/1bF6I2P1Juy/E7j55xuSb5XM35dJ9eJJi2lzvIGbn2owX5+ge1g8sfRTkSIu3hP/2/otz/1/ENjdSdAujJ7U2bOO9jDl4vs0xUvFvb9/xcUfPGT5loDM/EZqG+ZPapSf4HI4+FrDs58smb50NIcJ2SowfdmzeZSy/3Yj79nXJDZ1eiYdc/OnjuZAc/DVmt3Dgj6mie69a3nxowl7b/ckjRtZBl31VHf2ePR3XvL0T99l/rSln+ScfG6DLRZMX/YktePqUwWTc095YUk3PWlVjkBq+lL67myhOPjcJec/ccLhF5bsXp+TX3WUl8LW1CcymZm/I2Xmk4uMds/Ql4q99zqe/lTG6Wf9WEp/9b2KB//XmtUn5tKvl2mUFU8fATaPEsprTzvXTM8t5ZMNN5/ZF3bxpUygCYHuh+YQBJDkS48rNNk6MH1ao1zB/Ftrth9dMH1/R/8Dc7J1YP5eTTebUNx4imuZQA3Hn60c69eSMRBn790Gs+3IDwp29zPSbWD/nYb6RH6ePe+YfyChIipIima6sySbjuUnZmRbT5UYjn5tSzAaN0m4fisn33jym57145y0DpRnDfnSUB9nXP94z91fkvAS0xmmT7bYWYZygWmiqe5mlC8bgi7Yvq45+YKleLGl//590rUlXXeUFyn9PGH2zJKdbfFlyurPlbz2PynSrRVQdBnHcD/DZ4r50xaz69FVj9srSJqMybtr3CLH7ITJcdOMAjDblv6wxOWGbNlS3y0o365RAfpFhk9SksqTbq3II4HmJCdbec5/IGPxrmfxvuXlj6QUy0B50aEbR7KKpfTMKd+5ZvP4joDdy4Y0TzBVR+Hh5tMLps87spuGdm/O5HmNTw1Zosie3pA8PABi+umyJ73c4tM9smWLTw3pLga1uEBysSb/QPPsX7jD4u2dWBSWFenxDFcIwEpWHabqwXv6ownZ2Zb+cEJ6tSP1geRoGtUFgexixwc/fcjiK9eE1FCf7FO8e4k7XtAcTCluRDEw+DlLGLsbfZGi6x7T5iTbDl8muFSTX1SoXYMrDyneucAdzEnWiBS17qkeTMie3RDKPCY1KZJNC0aRPl+hujlmVaN8oNvPUb0ju6zweSLXFkSm2llMatCbGnu6iF5Sh3Ja/I5FRv+wpPxgI0xglqBWPe3DPYovf0CqlIAxpTDTDLNpcHslycaSRsbSbCR8RnU9fjFBbTtUngkjua5wR3M4u8SUJaHIUCEVhtB5/N4UvWvAe2EVtUbtasJsIoAu1pIo52mPJmSXO6kPcQ69roTRdA4mpbCc1kkliFLCaHYdzKfQ9ZL8utnGuXBATQrCzUq2P6S4ag3bmjCfELY7VJGLTFVrQtPCfEI4F1Fg+DYp7ZD6irWg9IfBXAzoCdbSffp10s9+HX3nhHB9g68bSYHtLereKeHp87FiJIQYPvTKJF6fHgPgXpxh7t358AR/eLwaLgT4px/8RtDg98bj9z2Wv8WdCGP578dU2P8R+J9fCe/5Ygjhv1ZK/TvAp0MIfzmG9/zZEMK/9Bttd/+Tp+GH/pt/hV2foVVg16VsdgU/+Ogp//jZQx4dLbmuSuo2Q2tJ7Tye78bnlUVP06Y8Pr7hfDOj7RLuHaylkkN5XqwXOK/xXpEmjlms9NhtCkzqOFxUXFzNyQrL0XzH1WaKsxptPO0mJ5t1pKmjrjLSzNIuC+4+uma5ndCsZCk/mVjcdY7a7zCJo18WmEWHt5q87OmaRPwoXliv4BU6cwSrZTGq7OnrFJ14fGcoFi3NTUF5WFNfTNCznsmslUqN1mAKy3TSYp2ma1Oc1ZLsarUkpyYet0spD2rqywmkHlNaXJWQTnuy3LI7n5LutdjrArXXScVFrwmNwcwsPjJL6joTxjUg1RaJpI7m0472ukQVjtCLpGZMCfUKeo2e9igtclpUIPQanTuRLrexfqLXqNKKv7MyYzInhfgth5L70BjMKsEdd5jM41ojSaaNAa+YHlfsLieoXktXaUxFRYOudPQDITUdCjnGWH+haiPpqr14FJVTMOvl3EsPKqBrg59b8XQWHt1o3J7FrI2kdDYaP4kftg5YWNQyFYlvGcZkQV+KrM7sxHMZ0kHqJcxdupIaDADdRNaqlHRR5YUdVV6N/lF3t0MtU/GNapFBj3UmOpCsDXbuyG4M/UxkdSENJGvpEh2SKXUfk0TbWKsRpaIuEwZ8TK3cE18rXuGmEuqUVJp+4aQqpBAvl+4UzT1H8cLQz+X8fSKsnu5FDmynMtZDQM3g5/KFpJYmG01/4MguRQbkM25luzGBz04lvTWpiXUNUhHhikBSCyvT7YuEcKjqaI+djPMkkK2kqsOVUsXR7Xu0Fba13RdmzJa3VSBDf+LgNQwa+oXUTLgMugNRBkBUCWzVOM66FxbeZ4MHLiazpkPiYmQ794S9HrxUA7MZTGRYrTCVIhMOZGth//q5eE6VVaPHzjRyHFm8li4XT2ayE+ayuBQvootJs+V5oL4bPYeJvKfTrTC02kn6qGmF1cxWRI+Z/Dfdyd8BQhp7R4O8bvCamiZ6TYfQqdWtnPl2CV0Y38kLkf3aIsp+L4XFG7yrwoqJL3L3QPy/bpjzdrKvwVuq4rh/u/cVIvvuB1nvK995sRt47FOdqDG9dPCGDj2NupO/S+eisMZDmqsko96mVppWAoCSSl473Ee6E2mpNyJ/7/Yg2zAm8HZ7wkj3M6niSCpGdnBYKNJ2YG5vu09NK1LroSbEFVFqfSILRv0iyqQj0B9YYVuocTzs0L9pbwOUhnHVTvyiLjLbxY0fKy2G8TdtTM5tb8fYdCLZ7adqZI77qfhaB4+eWDdk3CSJk1FmPrB2Q1+nj4oF5eSYbK4obzw2V6S1eAFtLmFawkjGqhLHmJ5qS9k/ihi8E72VE41Lh47RKNedKKbnlnZhxnF49RoP1SA+SlKHcRoe2oWxdzXdWOqTVNjU1o/BPyhGz2W2kjAe5YnMqoQgoYWBdZm+3Yce7j3xNadrO0pabanj55GE+XSLZOwNtaX+UFdoMIqkdlEyHOJ95Uffpk81aWXp5ilJJZJmkebG3IT4GPo8Te0kgdeGGBjkMa0HLSxzspNFgn4/x1R2BGM+N+hWthuMEolwEJZTDRLi1uJTI78LQXo5ewGabppF5tNI/UcdAdZwPUIQSWwWFSmRIQ2pkQ7MaS7g0UMoU3TVEaLcdkx+HT47jJIQIC2dmyqE277IQUbrParpxFs5+BdDIBQ5qpJo4jCR4B3ybEyBVV0vr/FRluq8yGKb9sNhOQNjWeS3Uty+v+2fHDyS0Q8arAPvbo+jaQW8Dv2Wr3o4v10G+8rjnxqL+EDou9/8eb/O43eFx/KNh+Hef/JXftu3++Rf/2v/v58bDLzy7+zjPwJ+Rin1LcRD+Tfj7/8mcBR//zPAX/vNNhRQnG9n3J+tOCwqjA4c7e3Yz2r2ZzVvLi7Y7AqUCmSJpch6rrYT3jy85ON3L8gSx8G8oupTNk8X7M9qnNe88/yYJ1eHbDcFWnseHKxYn814ebnHdjnBb1KMCVyvpqS5RanAza6kuSmYTlq6OiWdduzNar73zguODrbMJy0kgZvNhCLrmRzUqNRLT+O9LZ95/AHTsmNxd8PevCI0huam4O7Jitms4bV7V8z2K7JpR3CKct5wfLJhPm1IC4tJHemkl3Pda9mb1qADygSqXU5Ryhvx+x8/ZVflWGsoJy13TlZkhYXWwCYli1UY/Ttzjh8umR7UpKnj4HRDXvTszqYULxOpyVhqykmH+qDkow8vUJ3GLzPKaUuSWXzpWHxJEtPuPL7GTCzlt3KclTAfkzlU6lG5fAnq1JNOO6mUMAG/zAgBZns1s6OK/GsleEVykzA/2mE2Ur1h1gl+T45bD3UWmUf1iiR1JLNewm2covxCCa3Gr1PyZxnlYU31bEayTGBqmZzs5Fg6AcL5dZRezC3Kao4+m5AdNAIuQYDjgYxtdmUIhSP9IKd8qZme7kivEpnc1Yb09a30JK6lMiIkoJtY1L3W6EbBwpK/m5NdaybPBMC6qUgys0sjPksrryufJajSomuRSdtpwC+sgPk4kSrONeULCVK68w+V+Dof1Lj7LWwT/MLCfs/0iUFZkRGHNPpZ9y2YQPlSMf1AfJa6GdJZA24iPYSLdyBkAXu/Jb+U8SpfKpHmrjT5TZTC5m4MdFETh+4Vi2+CXvR0h048Qxk09xz6oBVf8FSkvFITECeRXkKN3MyLT3Du6A8de98CX0gYk88CZidAW0fJcXmm6O719Pue6XPxg/qJhP4op/ALS3/oKC5iyMklLL6p6RcBe2BF8rfRJDvF4m0t/tNSUlizDex9Q40VEb4Ise4iMP1A5IJoSdItL6IUbSfnoXsBOEGJeiBbyXPtLNDPBqAO0xeBxbuB9tCPoUyugO7YRaArk1aXBSYvBKTbKaPMdwhU6Y6ceEmXEjo09EZ2hzJ26U5Ad1JBd6/H1ET/sKI4J6Y3q3GyT5DfZZsgctCFdAkmlXggpQQefB5oTm99T+tP9fK8GD4UUhmT9sCTbgPdfmD7mhcJZydj5DOYvS8Aqp8JIHEFLN4bQqoQAG5g+7qTsVbyOx0rIkIiUs2he9CVMsb5MrD3rh/BQnMamD9zpNtAspOAG90HXCnAN1uFMcxn+sKPILK4DLSHsHvkmL5wTM8ck3MvoAOigkSkuvVJoFgG2qPA0Zd7vBHJ/eTMM1RcJHUEk3Ug2wYm54HyyjM593R7sq1+IX5i6UqV/YhXVo6vuBJ/p/K34TxyzwlQzNfi8wSRHWZbuYbF0hEMtPtEaa2nWDoI4p0tzz1pJUCTIKE+ykuKbVoFyms3gsmkCbRHin6upBcygdkHHbMXvagfzqSfcOigtGX0fe48piPW2si2xTcbpayZnLtPYz2Kk05OF/2wyt2GHA2/N52E9qQ7PwYcEaSKI9t6kla8luWVLIzm1z3KB8rLjm4WJbVTRXHVy3FlIsFuDhXpNnoCvah4tJXj9MlQowGz5/1YkWE6CRpKKlEp2FKTbh26D8y/vqI+iiFEO0lVlj5STXnZYwuFy3XcX6B8fyPhRhMJtbGFop0bkm2PcpBfd7gMirOa+jhh9VqKqS3dniFdtuRXDaaR48ovawFxCwFxeKkQqY9vU2iDVuTXrfTSzjTpusd0AvaylSTCJrX0VvYzjc/lszW/qCRQqBWPaVJJYi1KkWw7XKFJrxsBhInCTqSew6eaZBVDGCtLP0uExe4D9d2ckBqy60Z8n4kipHGxtEgi+PbYaYKdyXmjFMlNhV5VmF0bfZQCgLsD6bH0WawciR5IgP6wkITYXGSmvkwlCKhIhAF3QWSz6RCyI32SI6gsU3TVile8tSKTjf5JP8sFpBnZbsgzgtYCIkOU3Bot4UKHC8KkwB/MUb3FH8zwh/KzitUhquslLGhaCoPp3K2XMgTCYgbDv/kU9mbyr5f9YbRIXvMMipxgrbCXr4BKlWXigQRUlhF6K7UgEUgqY2S/r/RUhth9GdwrdSQ+/Mb/nPvu8Vl+Fz9+RxjL79Rj/om74TP/1V+kcwalAp01WGdou4Qy76nblNePr3nnTKj22bShd4auS8gyK+Aq79hWubyPjae+kQJ7M7NkeU/XJQSnUCYQvKIoO0mCblJM4vBOGMquylDGk+aWLLNsV6V4+bxCl/LlZBJPv8qFqQuIaTz142K7axLSiXQN6tTjrSZ0+kOrdzpzhOtc2DOvUKUjVAY1E4DrqwSVeznmRHoHiQyimVrcJvYPemQWasTHqFNHcMIOAvKzB9XGUITEj4ye8mrsGVRdZPl0gNSTXGTYAxvTRoVlCTMBrqpw6NTDyxw3HdLRZIU0qDAuc+ha33ZbdQKuggGfe3Sr8RNHep3Q7wnrpXqFjz2IutP40o3bMithBYfkVwLoVvoJlROgZPechIhUkiirbDzuNES2Us55ZLsc8vogr3FTJwAxMnfpVlbIXRl9PoUEPAiLqHATT3oj1Q9BBwjx+H0ETlbhC5HPmq3GLRx6Z8hWkko7JJQqK9c1qSQp1U8kpGqYxKfrCKwig5RUwsDpGObjylhjo2IPnZFzNtWwSi5/a08s+aURxqUTP9kQbkSQa+TzgC3D6FcU5kJhC2HudCfJtUExrogPTJxPw7jtIV0WYpdf7C3UvRrZlwEImEaYOpcLuJMgp1tWb0gjVWFgkCKb26oRaEm5egSf6pa1GLY/3IfBML4Ph6L2V9+XA7MzgCSfCluZX8fAHB9fo+PbrrllWofzTGqpeRk67ob9JpVMqO1MkmaHjsVXk1blPXp7fEOC7NAzOHQsDs9XTlgskGPJl5LIOpxLSGIyqZfrrawcB4wfEbJPxW2C6Ezkv0N4zzCOQwfh2N9nhak0LSMz9mqabboTwONi96J0dApb5iPzOryXfcrYSzj097nIQgUVz8Xcdi/akpHtk/RWSWL12SsXMzCmSvqU2MUo72Vthb0bGIahq3LYt4QSyd+TOozeVPEVC2hv90SSmNSD35Gxw1Cudxj7PocuP9MJmJZFr7hP9+H9D0mnY19eDKORtNtAfaoxjSwYDNc5FdVAAAAgAElEQVTFp1KXU9x4+pkct+kj0zswwa+MCyoyvSth6cZOysgwvtovmNTxcyeGDIXbOeXIznUL2bePY21axhL6Qb4+nN/QFxiUGplRn8r/uwyynYDVoG67ElUQmXeIHY5Dwqzyct5Je/u9IP2Q3PaVxs83ueklddblEnwy9FTaXI2hOQNjq4JsO6093VSPtS2v9jkOvYyvJqACMdhIApFcLt5J094GKBEYey9lEUHYxgHoDWylMNFDN2MQENcFVAjo1tPPJXhIPi+G8VG3nsbk9mcYrq8wkK7Q43tEBUQmHsHY8Fy5TkAM2HG5vmVkO4+dGEztxwTcYNT4fNPKAoM3cWxjkqo3wlwGo0g2nVSXRD/kt48jRKbT+lfuBUZwqLzIjH0mIT2SrmvH7wC8FwZSK3QbP3BcnKMMKavDc60f5Z1BS4AQSbzZh7+FQMikimToqwyRAVRDEiwIi6hfpWsjABz+PrB/St3+TilhMXsrIT4gIDRWgYySUmPk3JvutqPy27f96sM5gvcCKl+Rr46psENqrDbCWA7HO4b4iHQ1DEB2AJfAb6li5LuFsfzr3wHG8t/43cFY/k70WH7HHl2X8OJqj3LSMs075nnH5XZKu8uwfUKSWl6sFwwl3DdXM3TiURra+H5a3kwhKPRVir3TjvJE12mKeU9zNiVMLFig17QK/GVOmFl8pigmHdW6QC9T/MTRdobWFahYak8vMdfZpKdvEszGkB1XdE9mwjjkfiw6142m7+IKm5YkUjd3EppipbjdNwlJq3D7DtYpwSqRUuqE2d0N2/UCs0ykSgEgCYTSg1O4KoHcSQF97kAFVGMIKuB7JeXuMythRaXIVoMKkASp1TDS6+cLYQOHig0389ApaKRP0eVGCuR7Yaxsp0S62WlcgKxW4A1uLgEx3bETSeJaSwVGiGmcVsJcJOxEiuyVh2SZiKyqktL64kLT7auRGekKheoEyAFy3I180CfVkKwqk/R+HkiXZtyH6tU4YdOtyCFBAnf0zpBd6BGcuTKuONdaUj+NyCV1K2ErulMReBnskYXGgId0qUU2mgugSTYKO9PjpCS/UtSnoPYdodLorSGp4qSvk5oGO5HKivY4jOmgeqVxkwB9lE0bSXG1kzCCjGytcYVMUrOllmL6Ts45aRXdvhyTcpHF2g/kF9KvN0wyki0jc6h6mYi6OG5ZTG5VViZQCUqAWzdIFGOXI5Kw6/JbEKR7kenqVuEmAvyVlTJ5FRjZTt0rAY+D17MkptXG8J1MwFq2jtuLUtpsGY8t3Mpi+3mI95oAImWFNdvdF6mnxPzLeY9grR2OQ9iisRQ8rhCZRiYd+VLAop1AWkFoBYRJeq0Au6G+RPcxbGc5yH7jvbwvAKW4hvZAkW4C7eEwEYhAMUp76W+B1lgfEWS7yW6YiMrkzqcS9OMK2U5SiUTWlVC8EDlmvpTzS6pbkGnaECeDwsIkTaDbF9ZLW5m8SVJuBGaRLc3WIlsdJmMmzguUh2R7W/QetAAD7QIu3CbkDkybslEuudBk20BzJGORbQb2SjEkIJNIn2O7p4WZzNRYpaJbKC+EtexjCJBPZDu2jAB2CJKJ16O4kb/prYAVW9wmu44gP4IMYcoga8I4BkOqqYvBR+nOYUstktKpgMABPCZCymCLQW4JqajcxroTkW8rXCJ1GEXs6yTej0HHhOdOEmSzZWQyIwAegnwIYexo9AaKG0e7Z0b5tHLCqA2ps/kyMoARtPYTNYL9pJFxdpnsR+9EVjoAv+H6pFUMVttCeSn7M+tAN9fYTJFvQgTKgXTtR5mlT4bZPORLy/Z+SraLwMEFSBTllcWWWkKOLKRbT3tgyNYelyv6qWwr615JDVUCeEjlOkhFjQAZl0qKrU8U2VqumQoiF052HlfGZPFKPLSm82RLj880SQzOG8F2fH5SOQGpCnTrsNNYDRME+JjWS3J2TH0VpjN+x1m5ptrJQl1W23g9A8oaqfEo5L8+MfF960WKauV40p1jqH8xnR/BYVKJ3NSnGnyQc6idMIsH4qNVDpKdxU4TlPVkS3kzu8KQX9a4aTbWiRDTXFEiVxXQHMivWgncqZzITzsBbS7VqN5LvUcjgEt5Yf9UaiS8ZyIJqsRrrqxHWy+/b53IVrMkhg1JsFJIhMFV1t9WijCAQwGVuuoimES2qWSMcEHOw7moVPLjNUHLvaO6WFUSAdMQ0oMdVgiDVIcMktY0GQEnvf2wvNQBWhOyFGWHZFctgT9FJsE7A0h0ktKsnAcria3E7kzpqXS32y1NfJ4dAeqHgOOrybAwsojBWtnutwPQoWLEdZAksRtTSWiPMbegcngMgPQV1vLV/bzqoZQ/eL6rkmD/GXr8nmYsD986CX/pv/8JZqblS5v7NC7hL9//eX5p9yZ/+93PcG+xpveG//xj/wN/4Vf+Lf78m/+Yq37K81qW6r/04h5/8ZO/zD+4+ijOa/7ona/y3379R5gWHZ8+esHG5rw1f8lNP+GL1w+YpB2TpKMwlkXa8PnLB8yyju87+ICvb+5wvpvxkb0rvvDyPv/2W7/Iz777I5xOt2y7nMYmnH/zmOnjNYuy4d/9yP/J3/rgx7muJ/zo3Xd52Sx4++aYH77zhNYn/Pzbb/JnPvkFfFB8aXmfb704YTZrsE7z6GDJ17/xgM+89YTn2wXTrCPRnnuTFV+6uIdSAaPDuN2vXtwhMeJP3Csb3v/SPXSv+Imf+DV2LmPb52gVeLFZkBjHT9x9m8t2xs9/801Oj9eEoDidbnk4WfJ/fPOT/MBr7/PO8ojjyY5Z2nKQVfzCex/j9eNr3vsHj5n+gSu2VcGfevPX+IUXH+Ofu/9Nfu6Dj/NgsebLT+8BsLeoCEGxPJ/zw596m7Nqzvl6Rt8l7M1rtnVOs834yKMLWpvw4qunmHsVeW750Qfv8itnj/jE4QW/8vQRRdFzZ77l5XrO9mwGieetjz7HB8U3n5/ie80f+vg7/Mr7jzk52JBqz9OzA44Ot+yajO7tBeZ1McF//M4F37o4pm1S8qKnXhccHG9YbyaYxNEtc04eLqnajKbOONjbsd4VTIqOXZWTf37K9nXL93zqKW9fHNNclRzcX9E7w+6DOQ8/fs5P3/sqf+sf/xg/9amv8HNf/yR/5OPf4Cs3d7hczTje23L1y3do71jxZmae198844OLA0JQ5EWHc5quTjEvcnhck2aWJHE4p2mqjLDMOHrjhsuXC/7Ax5/w+bcfy2LCxPLDH3+XL5/fpdrl+HUKuYdWs7i/4dH+kmVT8uz5Icl5yuMffMa7z47ZP9hRNRn+7Rk+C8w/tmT1dG9kzQFU7pjMW7xXdG3KfFazVzY8eXosbLZXlO9lNHddXHQQ6W/yaEe7zpkc1FRn0reaHDf4DyZwvyFc5OJLrTQhC4Tc8/GPvOAbb9+D1PORRxe8894pOne3HuOrkvJpQv3QMr2zo3oxk1X43INVkHrSaU+/zMcOS7MzUToaJ+1xwef0I1ecf/NYOlUPWul/NYH0KmH+PVfcvHtAmDr0KqG4kPAje7dDX6Vkr21FAeEUr33knCdPj1kc7dhVOW6ZUZzU9O9POfnUBWffOoa5Rd2khMMOlpmk+p70qNbIostMfKV+4jCrBH+nJThFcpaJAmDRE6oE1Unipos+1PxM+k6L1zZUF1PSazOy1bYMJKc16Ren2Gmgnwe5rlHNUDxLaU4tambhRgKX+nng8NcUV58R/2u61rSHnuROhfrWlO7IUZwl9DOPm8e0zplFPynweRiZxnQTWSQN/UmPuUnIbsS/2z3oMFepMPNOUZxp2mNh9bO1oj2Icr0rPXpi+5ksJuy94zn7YSJglkWIbBmlkfNAcSUpztm1FvbtWtGchMiyRZb+bs/kvVS6WYu4KDGNDE1UAwwqiHStaE6FdTYNpGtoTrldrIo+4KQS6XG3J8BWx4WA5tSTX2qyjXyvDSxwtwj0+569rxk2r8ek4fuOZKtHNjhdK+bve1ZvavqZJ9mJqiGLvmC0VDBNnim2j+X35Zl0lvYLaF5rKZ7kUT0g/xbvOa6+18hi0EL8qoPsOqlg+9izeFvTz0R63O0Jy+cjE5juZJFiYF3RsSd1cVuTMywolRdhDLOq7soYH/9qYHc3skKRmc02sjCRrWUBoT2AyQsZf5fJAofyjAB96EW9/j7P9D3zCuspiwDNiac801L1MlGxpgO2DyRdOK081R2DLeK9ur1doOn24gLgtTDdyg6yc1n8yzaBzWO5D22puPxhy51fFH9fcySLGMWNZ3dXuhtlUUYWLdIIxNNKPocGWXBSSYLy3ruStpsv5RrOnjv6UkcfqEhfdS8y5fxapNM+gd19LeOlGDtSTSesmY6As59IUm67J3Utw+LF5pFm8Z6XHs+4wGcLLTVE17KYoYLIy3UXaPc13VyxeM9KN2bl2d5PmFw6upkmjwC/OZAgK9PJwkpahVe8qLJ408005ZUVIFsKuygqE7lmrhBA3M00+cpjGgHpwFg5Mj4iYz0AaJdrfCb1Kf3MkOwcpvc0RxnFhQRH+UzALEZhKoudphH4O/ABN0lir6YkwhICybrBLorI4jp8Kp7RkGp0bQUIDvPuCHp1bWPFyi2j6jMjbKkH5X2U3sbkVe/lddUtc9cfTUkvtqAV9mCC2bYoF3DzXNQLq0YAdmrQq2rszlR9BJpaC8AcvJFFBlqjqka6OyNIDXkmQHmzkyTaridUDSoxhOH1bSsgWhtCVTHUlQRrUdkrqbDf7sP8NtB5+7wPA19fVb8ZPPh1H79rGMv/+DvAWP6bvzsYy9/TwLK4/yg8+Kt/FdPC5LnIN/qpfGll0WuTbcRXoXuRthRXYTTZ9xPp2rKlsArlhWzXpzB94alOhl4m+b2L0fe2FMlbuhYJnhQQy0pseenZ3ZOJXXEVJZzpID2ToAOCfEkU17LKvL0vXxTaQnUqX7KDr2ZYBR/kSUkjXzbFlR+DBwBZOZ4rZs8869c15ZnUF0yfSRm57sJYVq9clOtEqY9IimJ4xDZQnQhrmq98lI/Jl1u3kC/F3X1hYiRmXiYEh1911Md6lAMGLZ6cbB24+ema+S+WY5JhWgXqYy0ddEOxeZRYZetAfSIpp8rJuWbrQLFyVMeGpB4mAjIeJjJPSSvj2k8Vkwt57iD1yjYyeUm3geZQ9gvy5Tv7QFZ9h2tkOsg3jm6qJV7eCfuSNFLG3S0Uk4vYJReZlHTr2D5IRonasNIuX+oKF6WRs5eOdm7kfpsK+5RWckzpEPoQy85NB/lKvogHxq2fxXqA2NNWH8k91O4rips4Ocml0NxlMkHu5vJcbeW4BlmfyAMV2dZjmsD1JxNmz/0YXtEuNNlGaDifSLT/5MLRD9KuOObdTFOfKObvO7SD7V1DceNlnOswdiuGRJgskNcpL8e6e6A5+lJPc2DiPS5fqpOXPdXdVN4Hc5FRDXLdfqZYPOlp9xP6UknVQJC+OZfLBMe0IvvqJ3qsIKmPNXvv9qBgdycRFmfj47go8rVMNvqJGhm1zSPN3rtOEmoPDGnt6Us99lXmm0BSySRxciF+IW8Uk0tLUIp2T74oh/vIp3I/uEyR1tJNt7snkf0+kf47qXsQ5l75QLeQioL6OMGlMLkU/1U/k5X3bGWxU0N9ZCiv3Mh4DJ16SS0T4Dpek7QSNmgIIhnYpnxp6adGKhaue6rTjOLG4jKpuOnLuL3mVbmZIr/uaY5TtA3Uh5rFkx4XGaZm35DWcg/kSyf1GZ2wmcV5TfVgcvt5ftnRHGfjRK+4jNfqXkpQcq8mtRxzUg8eRGGGfKZIto5uPyG/sdiJsCHSf6nwmUK3YSy3ly5NNQaPtPtS5dDNZKxDokh2DhUCdiIAoJ+JsiHbSH2D7iU4JUTJ3hB8kq5tHBsdv5Nil9/O0cWwlmxlo5rB0e1n5NctrkjG8+n2EmGtKkmk7WdGOiS1GmWlKsi+slWHnaUoG7DTRFJ3lUgIvdGxRkJTXIlPcGBumtNcvt+c3JfZSsJPfGawpRk/35KdxWw7uqNC5IepIr9sI3Mk1zJdd7giGf1++XVLcyqmzaGKws5TTCXeLztJRvbMFUbkkoVGd16SUXORDQ4yRhCgoLvoW4sVE4P3T/fiayPR1HdKJu+tCUWCm6bYMiGpLTpKLAngJgmmkgoKN00xrUNXHd2ppKdmVzV2ltEvUnTnoyohoFuHqTqau1OKlzuaezOyZYuyEu6ie5H9hdSMiaVummLWnXjmYgDMAC7sNMU0Dt30koC6bglFgp1l0rvoAqru8ZNUAEFrRzasPyhJdr3ILaNfTnVW2C6lxLdXtYQyw00ydCf3pXIBO8tIlvKB7BY5ydWOkCaE/BVPoPUiM57n8loPupF+RHc0l2CbSYaOaapq14zVFsEoASWdxU8ydCWvC1oJO5cYAQhlil5V+L0JetviJ7mAqLitECsqxrqN3n5YBjrUaQy/955QCrPK0Bup4rGsNoTFbPQq+vlUfI3JbVeyVIYYAVsD26fl9cEMNTZB/paYyNJpCcFxXgDYtoIh+GZIMx2OeZhv2yh/7XvxLradsJDWorQWgPbKQyklnZOJkefFMJ1g7RjCo/JcOieVkoqQgT0Vn5ckyGYpgxdyDNwZmMdvZwyHAJ7IGN7WjcSkLefiuWlC296yrrEfE61GMIlzUZr7T8Abg/R3eM6rj1+n4/K7Qgr7+8Dyd+dj+vF74fg/+KucPFzyycMz3l4ds5c3PLk+wDnN3f0NT947QU8sRwdbLp4ecPhgSd1mvH50TesSnpwf8sadK97+0gPCzPKR186p+pSz8z1QUE5blILdqkCZgFIBdZ6jH1QioTUeaw3BK3ydUB7UNGdTmPUkueVkf8vLy/+XvTfplSxL0IS+M93Jpje6P5/CIyMycq7KKqqobolBSAgh1iwRvWBo6A29AiRaQC6QaKkFYtFs+AEs2NI7hJAQEhJdSKWurKzKIWYPH56/yaY7nYnFd861F1lZRSs7S5kBZZLL3d8zu8O518zOd75phfOTLa7+9Az62R4hCNhtCVF6SBUhP6uAd1vYbUFGpSZjYN4Y4F2uzCgd4J2E9xLRJ0mMpj/Tj9Ra6dKhLB32LxaQJyPEiwpu5dGc7zEOZpIMX/7ZOZ585w2utrPJQxqcBEaJxcMddp8vIUcB9ZT7Xs563NzNmGy7L1B9XMJ+swNeVvAzD6iI2XmLfWJnzGKEdxJhbyA7ieqthPs+GUF7WaN42MJ+MZtCTmLlITcaoQ6YPdij/WIOLFgHEosAuWAarf/pHO7JAPWmRHzaUZJsIoqbQ28iVISsHMKogEECOqI86hE+nMOeOehbPS0yFBuB4RsdglUwl+yRtOcOokwrc61Gca3gywh/zD5R2UqElYNca+DBgDCkVee3Br6OCGWAXiuYrUD3HmtphAPcIgJnA+Jtgeq1Sp5E8Jw/UXBzMJCl8dA3Gm7l6QWVEbHxMFcGwUTIgamj3aMAs5UYzj3qFwrdEw85CISzETGQxcpeOLcIEKsR5uMKiALDQ3ZSqlbCrTwgI+Y/NRhOKQ/un1iYaw3XRMSlRfVpCQSgf+SYsiqTh3DuYe4UFp8C668D4XxE9bMSw0lAfcnFCbuMk4R1XHH7UTP4p7hRqK4ENt8bUX9SYDxmGJAIwP59i/KlIRu1pzQ2+6N0KzCceKhOonktsH3PQ+8pwx6OI+ypYziSitCthLA8jtVPJDYfMPF28THZiOGUSa56L9A9swxZ2iUZ4x0n8N3DCD/3WPxMo31Mdql5w8WK4YQM1uwl+y2FYw1Nf85rBXByXl+mjsU+pa2mhNTtex5HP5JwM8Fu1DeU6/Zn2b9E+bCrmerpKy62jEdMsI0qws05Rs1LgfYRmUi9B7J5u39A32xOPu3P2JNpdkDUOeRFYPdOgNkKFGuey+qnwO13GD40HHN/1TUXqnwK2smBKaqj7647E7CrCLNOnX8D5bTN64jN+0BOgDZbgf48YPkhw1qqm4jdO5Q02yXHoT/jYk/9lsBUePZhFsnHa7ZA94B/646/m38GbL8GVG95DtVbyq3LW3pFg8oLh+xVLO8idu8ILD4hU1NfBXTnksfzNDEyaUFRd1zs6h4I1G+TXzjJVl3FxUXdRtg5e15FBGYv4pS+OhxLgqyWzFh9GafFre1z/p++Q2D1cUD7QE6yfN3Rd2c6LjyZ1Em5eX5YGDA7+iddzUXA5pL3aAZwxYbM0HCSFrV2ebEzTJJWX/IcchJufRXQH0sMJwJ6x0XB5tLh+juGwUsd35fjkmMnIu/3/SOygfMvRty9X8AuxLR/OYKLMHM5LVLYmcDicwa47C7Y+5gTanWHabHM1SBwvWVoUPsgecp18gD2h0XZvFB4v9MyeyiLbYAaI9pzLlb6UkyLkLoPGJYKumPdy7jUB7m+i2jPFJorj/aci2jsm5TT51NeAOECABcvxgUBeXXj0R8rFPuA4s6hP2Viambc7FxyIbP18LWE2XjsnhYo1x5679GfGdiGi2lyzIwp38eIXOSMArBLDdVzgSZKYPaix3jMfsfuvEB9OaI/SxU2d6x3KTYeZmu50Fpy0SovMOjOw5eKcvOSixR50Ua3DNgZTgzqtwyn8aVEsR4xropJ/jocay6OCIHydsRwUqB602H3vMH88w7DaQmztqyEWRjovUt9uRzb4m6cFhPGI4PijscqR4J31qkE6PUAX5uJQYxaQo4ew0mJ8nZkGI8QU82M8BGqI2jKCbBcuNCTJ1OOfgJAsncE5pueoLkjUI2lgRgsK1l6e/BiGgXRWVaNlFyRjFpC7HvEWUUQXRYExaU5LBTk542OXZfbjpUlw8i/E5CdpLWDPXgsEwsZC0MgHQJiWRBIZ7AW4yE59j4Izr/P9SJaI44jROqfnDyWmW3se4imJkDNIDTLW1N/JZAktcn3+SXckeWy92Sv8RcASiHF4efZ0/lLPH4jgOW7T+Oj//I/+pVv99N/5z/9tZ8bgF9LKuyv7OGsgigDbtYz/PDtIxjJipA/ePIZpIyotEWxGoAIXF0tJlAJAD95+RCv7paQMuDNdo7y8R7wAlJElIohM3FQ0CrgfLGDSmCjrC38nP/2nsNX1yOUCoCI6PcFmkc7CBnhncLoNMrKYteXwMUAYzy8VYAXiE7C9wr21MH2mpLBKjDgxgmE5x28k7CdwbAvEKJg/cbOoKgsZs0ApT3iwKqQ4BWG3gASkDLALTx9kAJwg8L2zRztUEA8GPDyeoVxMFAqQKr8QSMwjpp1GE3g8a9L3G4aKBUwdgboFYYzj+DJGhanPfSdRrsrIfYKogiwvUbYE6iZrUD3yMMOGnbQEMcjYqScL2gATrCGI1V17N82KdAnAjJCNA5hZzAOmuAiLW6FUUEEAXPcw5epRiOF0ggVAUd/XrEaMGzKKRDGV4HVCQuP7plF6DSECgg6wp54XpdOQb2kTDJosq+iVYAO9FAWnj5TGaGvDeBYbSEc6D392h5uzv2F5x0rIJ7sWcky8wyf+eYeIaXhjkeR5+CB8qjnZGiUCGWAaiXUHb8swtkICGA447b7C4eoIvrzANkzFCiOCmKnISLglgGqF9BbibgpoLrkLd0qxMbDHRFgwgns3/HQe4H+IRNlfU2wglQHIzxgbhXsioXkfsZjh+REGyIidqwlCVVILD5L4aMiqKyfUusXygjUHuOZx3jE+y4qAIFyRjvjRNDNyErlaxcKHpPeMQwoqphkhYLhVxVBdA63AZB6ZlNC4xmlg3Jgx+ZwEqc6lGAiRC/pvy0jhsd2CkaBpL83p2YGA6y/wSTOqCIEDlUZvoqwM3B1f+S5Dmc+FauzpmM8Ase4TKFEKqdgMrwnB+1kb924iph/TmWAr+iZZUVEqnlI3//t43gIwpEEdK4GhCXIGZPdXKaJ6HCcEkUXBDOsGEggakeQobpDl+14TAlbUJQMj0e4V0OSpJ0loLfZ28w/rAlhrywraXi85TXldLrH5L/MacY55EQNBB3tI4Jq6eiTzF5Gs8/nlDzTc97jUTO5NvtfxyNuL8v27PxQUxEMU02DAdoL+uX67GPFIfQnCoKyYkPG2c0EghGwM5EYUExqilwRFJM/M/s27fKQeipikliWQHGX7p8U0rR5zu8WNdCf7Cpe++xnDErAlWK6j4TDlDYbCsp/bZMkpwLoT8UUZhU0jx9IoUilmCpB1EBlQGYreSHzOBDs9cdqUuLk5NYM5gCCd9URTG+fFocwquz91Ac/LZUIXKzYXxh2n9bcRhS8p0wbMCw5ZnLktkXABNYYhoNpNpO9tNJR7ukqvscyGM+f6QzH4d9qOPS+dicKQQHlrUN/opOXVnxJITQs6JeXlqCQKgSOVVBku6MU02t9SYAJpGug6J+MUiS1kJje90EDw7FmquuM9RAiAG6mUnJ0XmQT8CkAKCgx+ZrJACcwna4plEjvXXpJg04gyfI10sZJbuorBV/I9F5MwFExGCh7Pn0hoVoHX8kpmCcHUGXWuj9nqqqrydzr7udYuCQDNbuAkJhwiJTCqoBcdwJQjRZKyonl4NL7ViIaifG4TJ/LEkFLuEWJWMiJHYxKwNcaxdZOoUB5//crQ2LaN0Jk6iqoiJiClUoFX2uCvRARmoKsrpaIBdPfQ2UOgA8ElRNLWRYIlZ4YbZQFWfayIAgWAkhBRTwWzd/n45NkPGNdTkA2pU5Oz+FBp+cpeWADhSAgLQzZVSUnJjf/fmJWpfhygFCMZFGtRRzuMYXpOaIwB/by/vUVAjCGYFKmZFkhDqAyJ70qdTgHIQEhIZT6c3+mn//cfr6qj3vBvL+yP78pj680Y1k+fxaf/P2/AwAIXrDrcK2B84GhOlcNiksN925PqdkuzWhSkoKomKg6u9ij+2SB0IQUFiKgWjGlfAKcMEYFYGUhbgxj+TcS9pjJneWNwHCSJIezZKpfMyimP2dAypTgIAiQQtp2/ULBV2QAdu9bVK8MhlOPxelvDW8AACAASURBVM8Udl8jw1JeKYwnKQrf8kt2/jmw/iAimoj6pUL7zEG1EuW1pEdnFVBdSfQPwpRkaTZkBGLjodZ8M0srYI88yrc8Dl/SsxRMmqytAvROwh6RobIrD70l6zb7XGL32z3KjyuMRwQCahDwRUSYebLF/2uFm+/RK+XOLcxrdm7lbr7MrDD5lSCruOMHXg4jsXOOmW4FRIrLd/OA5qWETSmUdnUY9/5BgN4L1G8E9o8ZtKB6Ml+qJavilh7mNkm+LFBdCbRPYpqARhQbif6hQ3GtoHpeR4iI6pJytu7Co7hjcI9dRrhZgOp43CZNrs2WoTDtc4v6MzMxb75i2En2JZU3BLF2kXoY9xLxWzuoH86nPjeIOO1LtwTRDI1g6IpIdQo5vdFsgd27ATgfUP64Rvd8RPNhMcl+JwAT+dqchhg0wUyx5j7688BE2QcOix8b9KcMtsmTYLMRKQmRlRXNS0nPGggy1MhqhvYxg3iiJoPVnYkpPAfA1KdY3vC6d4/IOLkm+dp2ZPz6U/rkQgEwfZMTyO5MoFgDwykOnrGR29FbweTWlA46LhmuVGw5GZeWk3w7yxJDepjGY9atyJHeKt0SUJY3rGAo7zjWQQGb71rMPjSHROMUgORJFCefHY/Bl7w+IqYqh2XA7HN6/kKRehzTo1iT2VIjx7i5jAnMYApVEiGBhFpAdWThdIsJZJd3BAXjgjUquefQl7wP7vcx1m/IroaCTGv7UKC6IoM6rggAm0tKiMcVx8ZXOCTpJha0uqM0tj/j2JodjyEHx+iOY2xSDQcnpcDsNdm6DKrsgh2KrjkEa6mekm0GKPG88rXI9gWRrKI+ScDVmJg4JZKHi4Axp9DaGV9frg8ppq5CAgWUKXsj4BrK4W3z5aTR4ZhMZpasM5AIKf0TqG7C1EHpmsOY2xmZtt0zgaOfprAyxWNmwjDHKBQ8tt1jOdkQ1BhR7AOGBWstWEtCGX37QKHY8HvLJRCFCFRrnsdwRMa2PyEwKO9iCrdBklPzs7B56+ErgWFxkOWbVGkyJDBZrpNPb5YSQyOl7uUmwFWUzweD6R6Okqmp3SlZ2Sqxpjl9VFrKcwEyqdUtZfjlnUd3qmG6gP5IJtn7IXXUVWKS+AedElY9pmPKQNSXBOa6J7DK/q37UuthSWbRNgLjXGDxBdnGcu1hdh52odAdK1R3ZJh0y5qPcS5R3h1AlExdib6QX0qDDZoewOFYo7yjjL66dmRWE7hylUSx9fw8S3Jh1bNKw5cSxcZiXBroPWX8vpIwWwfpI8alRrFxcFX6jkuyZdUH7B8b1G8dzM5hOC4YlAOCV5Uk19JHyoBjZKhOBFRPRtTNCBLU4CFchPCBUm4fuLClBKTlcboUxCMHT7AqCX4ZCuTgFsUk0c7+QtU7+FpD2AA1sGYkM4e+Mdy2FFDtCN8UUJ2F7Czcqpo+S9RwYLWiEhADvZJZkuorDTV4DKcVzNYCIULtB4KfIjGA95Jc5egmBhRSwjdm8i3CR4b7SAk/K6C2/QG0xYhQGshtS0ltZSAGxxTX1EUZjZ48j5w4aIhtS7DmA8FkAofwgTLVPHdPEuAvpcBmvyRwOI57/kn2WHpKao2ZwF6W3AKg7DZGSlljZEDPvUdsO4iqInvo/cR4HqpDsofMI8Z46MG8f2zAP10a7M+xqV95Key7T+PFf/GrZyw/+3d/MxjLrzSwXHzzIj7/B38bAsDZfI/9WKBQHvvR4G49w4PTDV5/fgLIiKPzHfZtiWfnt5AiYjuUuNk0qCoL5xS66xrHjzbY7GoUhcPQGZS1RVVYLKsBt20N6xS6TQUhI0xtWU+yqbA42WN7NUOxGKG1R/tmBnU0whiPuhzhg0TXG7irGuJoxGzeY3fXIEagmo/or2tUpx0K47Dd1DClw6wecPtyheK4h1IB3ku4USFGMp3VYph+7r1ACBJ+bygD7TSWD3bYvF5Azi2ik2RBAZTViHZdwzQjTld77PqSYTCDgZR8jt0VTIHVEdVph2FfoF70aC9ngIooXxmEb+wRXtYI5yNiq3H+7Ba36xnC6wr6SYvxtprqPKIC4smIorIYbmpARAgroU972DVllqgYIgMBVmycsLIETgCVh1AR+mVJmeOevYzYGH5RiQi5sAhWQez5JYq5QxwkUITEuqWk2JGTwVAFVK81hvcGRCdSki4QawbS6I2Cn7EWxJeAP7GU+d5ouMcDQ1Iqj9iTyRR7DbMVGM886s81fBUxPrIoXhuMpwTibslaEmHp3fQV5ZJRAn5O6WuYO5QvDYIG6kuB7QdMiYs6orrUGE78xM6arUD/fIS6TenBTqYvQC4KCC9Q3kjYWaQk+jON/kEgOzwwmdYtabgt32jYZQLdxx7FlcJ4yn2Vlwq+IYAMhgsWkGTr9JZy1N3zgFBwAcQuAqq3ZH7sKiKCx59rVIQV8CsHc6NRvxLYfMfC3OppUcLOIvzKofnEYDgNMGvJTsuS8mU7j/BVnJi0oDl5rt9IdA/DxLwBiWndEsA2rwU233IQg0TzipLb4Th9oTqCZHrKUgVLvjZVhJsHVJeK12zMq/0E4H4e0LwgywEQUHYPAhdHZPYip9CPlKJr5xwP10TUl2KqvshF8W5O6elwHFCsRbqWAASw/RoXcMwuMy58rt7zPKVNXacjptTSKDKDQ2Cu+gRUS55HZphz3YpdRBR3lOdOf6/FlFY6HnH7eQyqq5TomtjpYiMOBff9AUCLAOQqiu4BJa++wrQgoFOvpvSJJZQMackVC3bObWZwmutkzJaMrGkxgcz2gmPL8J2Y2F6CXN0yXVf1SL2GyY99F2GXqS6oTL2aR8mzH/k6NzvcW/RcprTY7BksxVTl0ryO00KPLw51CK5JCb/HBHfd2UG66ksCel/z/ab3capsmTzyIS+myInd9VV6XcV95eoVep0TU5lYToALFjkQR7qI/pjvCaaLYpKGuoZMsByQ+h8j2of0qufAn+FEoFjHyX/dnUs0bxj40p3IafEqy3DVwO3X1wydsTOCd91Tepu9/dIeVCqqP7DZiJju31zlw4qN5OE/lVNtS7E9pPQGzfsmn6cvxMR4+yJdw3S9hCdbysoOAvzsI67uDsAWArB18hUi13xgqjLx5uB7z52e+T7I9SWZNSTwTdUlisnB3YkmsJ7Tt8yQnZAWZwnmpY+YpLDJ12u26TWeclfXqHSdFZo3FsMxg2fqtxZ2qaB6VoawKktBDSH1R3LbcgzwtUpy6UMFCmug6Pk2e1adBCNQXY0YjospadbOVKo7idCdh11omK3DcGxQ3Yywcw3d0i/vZgzTISOvpnoQAClFl88NRh7qSUzygLcWQcupw1J40jnDWYHyil5XqnBC+hwPE0gLpT6AFy0nFlymsJ6oUp9moaDXlMLeH4MpIXaSwTKxVXY2VZE4xNR9mdnO7ItFCAS091hLSLC+JNd43OuDFKNlkA5AVtF5xOQXFclbGusSou0Tw1lMstg/9/hSGmv6d5ao5n0mYBlHgroJhOb01pDGKKXIxvvbydvIvtBf9Mjg8S97/H8JWP7nfwXA8t/7a2D5z/y4+O5J/Lf/x38VV8McNkp8uD7Deb3H1pY4r3f4k8sLnM33eNhsIRHx0foUSgb0VkPJCCUDYhRYlAOsV7hpaxjl0Q0FjPYotEc3GrS7ErpwOFm2uNk0OFm2GKzGZlujrCy8l9Da48Fyh88vTzCb9RhGjbEzU/9lUVnM6gEhSKzXDR6cbbAfCjjHDs4QBJzjh0KwEogCqvAIQUDKCO8kytpi7DVmix7b6xkAQJgAkcx0SgeEKFBVFu2mQr3oMQxkaSfQuC8gk6xXyMiuTCehKwchA+y2xPHDDba7Gm6gHklVDr7VKJcDAMA5BT8oNMse7dsZnrx7hS8+PUVxNBD8BgFdOgSvUDcDht7AtgYigdtoJeYnLXZv5pBz6vXDzkAvR7hNATm39EhGEBQq1p3IkwHhugSWFkKmL4IuAbydhpg5IAoCSgCi9qxk0RFQEbAC5qSH6w0X9XRAvCsQq8DfpwkFBglz3MNuSui5hdsbqI2CbwLOnt3h+pNjbjMAIibm0ARAp0mQiIAX7AANQKwCkzo7fmj6YweRqkeiSccWADGyvuXo/Rvc3c0Qo4C4LhAa+ljlThOUzegvDAVBWmi4SiwHOZ1DLCLEIIAHA+K6gBgFzJM9xjcNhKcs0dUHkJRluVFFqI5SaL1RBBkPLVTt4HsN0SnKlzvWpYQkCxe9hNpLuKWH3imERz09tjMLXJXQO05oxxP+Xg6U2/qFh77VCAUZXF8x2VI4sHLE0pMZSlbGhIKT7FwLc/jiB8YnFvrKHCaGjn+7RYBM46U7epLsitJrvUvSzJjkdEmKqfcC/VlALMO0UJAlnFFhSvvUrUieJt6PxZ2cah7ISrK/0824iJClgFluOrH0FVlVnyoScp9priGxi7QgYAXKm/Q5oSKk54Q/FOneDRzXYMh6y5G/i5r3d9RkkaNO419F+loTq+Or1P0oE2u/TkqM+zU8uSPUxFTdkTyfYwI9XkyT/Kz4IJspJoly1PcAbZZdC8DOyEaPS4JN16Tr0RFUqoES5uKWjGpOGLUzHmtmhTOgz0D94JFLMst0HkyMzcwgn8teQv7bF+nYRwLm3FF7P2UUgh5TN+MYAphqTdyMLDgrWJDSQxOATqA+L0iMxzwW3abKE3NPXrxjT6gcDmMiPMco3zM8UR6PT7JtvTs8N59f9nyOK3pxfYVJvit88gIPODDhQ0z34aG30uzT56/Cl4DfeMRJ++SR9JgSV4djgs/Mwup9RPeACxy6JfAqthHSpqqRlDSL5NHO4XW5viUHkflSTOdlawG7ZEiedIeqkHys03s8ya7zokuUBIJZImpnnNgre0j1LTYM/DPtAezunkqU1/Hws6QiCVqg2IeD7DIe9pMTWIEEcD0BuStFCjY7bGPq67yvLgk8l3HBhNXcwelNqvsZw9T1mRc7XEUwev9v04b0nhdTT2z2hE71KOm504JDrulQ96W7fI3qGTzF8KgDCwyk653CtISPGFcaZuenDk5fSqjOTzUpuetSuDh1dOZqmFAcWFs1eLKpAQSYWsAbOZ238JTTCsdaEl8zIEv4ANeQSfUNwaSIgMqprZ7hRaHUBJI6AdisLA1xCmIS9gDURJa5pq5J4djRCSHIaqbfi84e6kmyBFaJKTBJ3O+rzEFBmZkUgr7K/MjjbPQhiEeRbRTWUf4qxSEwKMbkI82M4gFYT9sDfo7pDNM2p2vrD9uL1gIyeSn7/h47mcBn9lr+ZXjjFwHeX/CIMTIs6Jd8/DWw/Kt/fKV7LG83M/yjj76L9qqB2mhIB9zuOOH96JyR9y/1Cp8u6VWD5IRROEwx9r6KuMFhsmStQHHHL+92GVHeCSw6frnfzeYQOuJaz1DeCFQSiLpG2fKL541aoQAwqhp6ALRg9LybAcJX2FzMYTYCpQb2f1zRk6EBMQLFCBh5mPDYRVrpT6usxQYYjipUAeiXFZaXlLz5WiSPDmVlOfF14YHxqISJfG0w/DKqE+PgyxRskMrFfVVA9cDMAu3FCcwgMNsgpQ8WjPYuCx5XFdFcC/QPCixfCbzZPMTJh8BwbKDMYaIGANvvKBz/oeGXQBfRn6fI/UWBozW9JGrkJBBCQ/U0+ZvUu1ddRbiGHqPxuEH9OqJ7UGH+ghK92auI7sxw9brQlJXeplX7yqC444p7fgQznyYK7eOA4x8L+EIxOn+XV70j2kdzLF5z20XM5ySxvjvF6U/SZF2QAaDET3FiuOa2fYkp3j0oOU1qEYH+rIDZYAIxriHjkidgd7tTLF8djll3lK3lLr5gNKrriHElofcRdq4pu0zeI07Q2d23f1QDApi/iHhb1jj5YfJu9UAujUYEhiN+KaiRASDdQ43mNScS3ZsCdl4kOV+c1NxmB7QXBq5mwEp1G7G/0AxEuavTvagxexlSPyFg5wyrqK8od2wfGqw+DFN6bU7BzenEh95Fng9DIsQ0tjk0JGhg2xZYfRQSC5EYgUJgOFKYvQrYPWNnoBqA/pShFvMvAvoTAr76ism/ZKoC5CgRpcTRh/zC648EynVEfyyge7Is9XVIHjYG7JS3vEbVHSV0TMUF7Fxy0uyAYcVrW2wjzD5g95Sz8/kLj/5EIgp2JkYVkcvU23OJYDTcDFh96A/F8WlSmj1jOcikeyChujiB1CwDtTOB+sahO2Zaa3+c/FUpIdc2cpq8tilFd/OuxuyNR1BggvEusrvUcIKpxgBX8h6tr0JiZxI7mvyEiEBzRZkfZcECxSZgOGKKrukYeNKeKcwuLbZPNeZfUILpCzExM+Wa8slyEzDOBJorj2El0Z9IzL/w6I8lFp9bjCuNYkN5YZ50B8O0XfYsUqbZH0ksPx0xrhjQYhuJcntI1u2Pubii7IF90gPfF7bhhDYzXfGGTKs3As0lJ3jdmZ7SvZu3frondRfRnSnUVz7dGx67xxqzVx66ZbotP3Ppo6tuHLpzzQl2+n7SLbD4fISv5CS3dTUn1Wbn0Z6T6cqVDzl5ub4c4SuF7kyhuvUYF3wvFNuQwrEUlI0TCJ+9HlM6sYGr6U9u3lgELeAaxZTYO0dv4aDgC4H5ixF2Sd+ZbgNM69CvC5R3Fr5UGFYK1bWFGg18IbD6eIBrFHwpUb/pp9TYoAXsQjMMxwgUtyPcTCMqATtXaF72iIZhLGrwsHMN1yiUt/bQ05j6EMeVofxyDHAzDV8xCTeHzaiBwChKYPmRxfZ5heomJQsXZOmUlWheHSa10lWoL0em1SrOMeyM18nsmBjMVNmD/9BsRoRSYzgxqK5G2IVBcdNjOK9QXg9AiLBHJYRjnUZ51SMUagohUvsBblmh2Gim7c40zM4ilAp2RhaQoEhBb4e0rxLVVQ9fa+jtCLsqobc87vG0htmMGE64CkMpOTssi5sewShuazcy/dUFuFWFqCRUZ+Fr+grVboBb1Sk9NyUjF7lPU0JveoKsACDOUFzuEeYF5G6EX5Z8/bKivLazEMc11G6EbAdEoxEahuK4ZQV918MvSuh1B1Ubpti2A+Ksgqw0ZGfJ9PnAoJuUBCvnNWQ7AFJA2gpyP0D1yeMYItQdaziilBDDCJXAldIKYt+RcUsgTxrNKo57oE9k2epoCSx9AKxjoq1nx2ZMr5ukr1phSpTtBnoWnWdgjpQHMHcvPRae6cNQ6gD0CsP9hsDnWkf2MIO5gRUq0JpeSSn4fHVvH8AB/GX/Zg7lyT7I+wCyLIDRTn2VwmhKX4WYmFWEkJjNvwBU/nxQz/9feiuj+H9/zlf08ZVmLKuvP44P/97fhao8fvvZC3y+OcZJ3eLF3QrDYLCcd2j7AsOmxOOnN3j5yRnOnt6hGw2acoTzEpttg6K06F/NEGcezaqDlBF9V8D1GqYZEYOkBDUC0QvItwXCmUUcJYrVwETYKOA3BWYP9thfkk1szvfwXmJYV1ic77B9uYBaWUgZ2KHXMChFvi2AiwHBJYZq5iFGSTar9BCSrCeigDQe8aZE82yLcdCUwG4NRO0QB4ViNcC+bhAXDvWHBfoPBsTE+qnKoaos9neUwtp9llEAYk9QllktBAH9sIUbNUzJcKE4KohOYvZCoXtA36g9dTC3GuaDDbpXc7JWp5ZM0iBRv9SoLyNuft9BzSz8poCcWZiPaoxHgSvhRw5irxGLgPnFDsOfrWDPHOROQdok10QKjzm3MJcG9tEI/ZbHT2YgAIFMnHx3j/G6gt4q+DogFhHFW0o7i1t+KPoCaF4JbL/uIIJAcc2fjyce4mhEaDVEr6A6TgrHBw5iTKzcipLWqCO9p8ces481fE1mqbhlmfj+uUdxLSkxDIA9dShfaRR3IgWIAMNxwOJjehJdfWBLgk6A8phsqtpLFOsk0bSUcJbXArv3HarXGnae/L0pvMbccJIjnMB4wsRccyshLUNY5CCgO4H+wkH2Ekc/EmgfMZhk+y6lrG5GSZcagPJWoL2gxNHVMW2H43n004C7bzCxcfVT4O5bEYtPyJ6OR/QcZw+yWZPdCgW9lc0r4Pa3uAjkS3b1RQnsnzvMPuN5lTdM2cxeQLMHtu/7lCoL7J/QZzz/hH4+uwiTHBSBrKKbRyw+YuiOCMDqJ0D3kKyk2UiCzfPAQKgtfYO6pVTv9lsC41HA0Y8kds+B5mXy0Clg/5Sy48VHwP4Jx0/1EfsnZOh0n1ifjkxQfUX2pz9h+ubmfWD5Ia+7nfFn0nNbCGQay1t2EjaveY8MxwIheTSDBsZjem+P/yzg7usEUkwxxeQNlY4LHaEA2gtBf2cAqz+Sr2n/lP7k+jJi/UG6lt8G5p8yiVa3yau5oKSU7BP/rfdkj+xcoD9hCqsI9AEScEZsnwmoxNDplt7X2WdZnsrxMTugu4g4+WHE7hnri+ZfxOlzavuOQHmXWMyWi0vlmq/fP+JC0/Y5a5baCybhBn2Qq2YGs7rmsaqBwT2rj8KUtrl7pGD2HKfyhkwGw6WAYk+Zpdlzku/qxG4meSaAyW8pPK+ZHrgY0x+LyYt493XFxbFHHO/dUybu0v8rsPyEIDlLmc3+EC4TFT2EUQK7x2pajJQjfa2+oKey2DHpVg2YFpCYqsp/z94EDCs5eTW7UwlXE/QWW4Ja0wZ0J6xZqt9GmC6ivHO4/aCAbuPkteyPyDaalozj5h2N5WduqqnpjykftTUnU/OXDsMRK6F0HzDOJGavLVytsHui6BtPMtdim5QuSjDEJ6W91lcO7QM99WVGzfuq2KeqsBEot2FanAW4cGC6iPLGIhQS/YmCGiJclQB5G1BsHLZPS1S3BPk5QCdXlfQnCrPXFvsLg/kXI/pTWhemReB9mEKRghEob5ksy6qtAF9I6NZDDQHjSkOOEXYu0bwe0T0soNsAOQYEI1HeDtg+b+jr3Fp0D8vJl19fjbAzpr+6RqVFppBCbATM3qE7M5AemH28gz2uoHrHdNi1xXBWIEqB+lWP7qJCsXYwW9bGAIDqLMajMkmjA3ylYBcawQjMvugn/6ROoHY41qjfjKxYqTVU5+Aag+zvtAvaHXTnJyBbfbFD984C1dse46qYqlv6hw3K6x6+0nAz1trovUOoFNTe0t+4Zfqr3o1wq3JKetVvtwjLmoA3yVPl6GGXBVRLsA8lpk5JaVk1Ax/gTmdkNEcHPysO4DAklk9KiMHCz0vo2xZ+VRPEuoBQaMhdh7CsIduRVSkpFEh4T89mRSCGwiDXhGC0iFVJ8JurTxIABcCfKwmx6xCbij7LSWqrD17Ntj94MTsufsTSQPQj4BziYka2ct8dwCnAY8k1KuIe2LGW3khjEIeRgFcIxC6xE5oMaRwGpsI6d5CqJk9lHNOxJgnsfQ/nzyfAfin19S96ZND5z4BbflMYy0d/7+/+yrf76d/+T37t5wYA6gc/+MGv+xh+6cff/4f/4AdP/83vUgIKiW+dXOKyW+Cd1S2EBjpr8G998If42XCG9baBqh1GqzF+PkcnFYbBILQa7zy6wYOHa1zfLvD7736GVdXj9y4+x2A0xqAQfjqHdwpRA//8B5/gSlVQxuN7z19iM1YYB4NvP3mDHQzauxrP332L9aahfFFFuCF1wy0HPD5Z4+blCnKQqM9bbtcphCiwPGnRBxZjVq81ive2sD3ltAICDx+usd9XmD/cYXfToFkMGPYF9K1B83gPu65g5haoAppFj34ZEXuF5rSF3ZZQpYcxHuOmRIgCYmug9uyoYwmzxAgFKMCcdbCdAQYF3xqY+QhdecQyYqgFmmc7uHnAk8e32L6ZYxwN5CAgn7WIW9J55k5huHDw3+7gvYS4KSEHgdNna2x8ieJOovrmBuO+4D6PBvS3NcLKwcxGeKfw4JtX2HUV4AWqd3YIUkCdUVIcDHDxjbfY3s2gW640+6XH00e3GP9kxWCl0wEhCpRvNM5/9xJrZVA+bjFKCfX1PcJ1Cf2wwziL8IsAfacRIIEk25UjgaPeaIQmUMI3kKkobhXsUYCASKv3lIYeffsWa5QQg4B4twVuC4QiAnMPN2e33HAcYY89ZbJWobwTcAsg3AuFEs9boA4ITkL1EnYZYS8siiuF+L0dhvMAeZdK5ANTYZmoCzSvJPrHAaoTqK4kxgsHv/LASLBsTzz+jX/l/8bPPnpCgPx7a8QXNbqLABgm5EYJ4MEApwXKawk3p4R09oVE+40B5kZDejJ343FEmAeIUWE8T4mI81Q3ooDhkUPzzhZ9X0B66tBCGbmS3VKKmSV+4zHZSLtKPsOzLDFmbYsIAm6Z/KlGwJ5Q1mp2Au1zBwSBYiMxPBshW67o2qOA9n2H6qWGGjnZ7h9Qbmv2BNFRA9IRIJffv4O9qbH+RoTZEpAjCvRPPISXyUdHxYPuBboHQHUr0J9F2CVB5fidDqNU8N/eI2xLKAfsn2WWmwmhTCkVCKXA/mv02bqGybQiMkW1e8gaEF8L7J8R5LMSAEBEkoQK3P6Bhd4oqgWSZHFc0bfia/q4uoe8JrMvgPU3I6KU0B3QPxRpcUJg/4T+TbMn+BOeYHU4jxhOmCY7ftBDbAx8zUl9DqNxMzFdu/GIxxUNgZZvWIVi5xGhFChvKVPsz5NvUfJzqHvsoTp+3uYE1+6BwLgQ030QDVi1kAATAMSCiwntBce4XPN5GYzrFrArLkzsnwGQBJ7tIyBKiXElcPU3PerXMsmd6SkVgQsBoRLYP+HP+lOmtEIK2JXA7hlg9pRhjkuCREQxMYn9mWQAzolA90Ci2AG+TvJiQ59jPjc7FxiXEr7JAXKC41dy+1EzbbR7ICegaHbszbWNwHAksfsaz0k69vXun6WwKA+0j8Wh31MD41KiP2Vth5tx3ILhvmwjk/+Xk/FhJaAsF3/sgmnMdkY5a/tYYjjmtspNnBJDXS2wfY+LFuUuQjlgXDLhMxiB9lwx1ElxjFwtoZKij4oOejwZxsOFtWIX0J0piq187QAAIABJREFU7N7hIolKHcLFLmD/SLHX1wMCAv0JP6/tgvUm3amEjBKuTjU/C0nwn5jJYGRKuBXs1kzH68s0Zkbg8g8UZq9iqsPgz8YlwTXrLWJiQSN8LTGsFKQHhqVC90BiXKX0SwgIUKpq55qKjQD0p5qujNMCug/YPDcwHe95X3KBxtX8rlZjRH+qIQPIIJcphGhBZld3AaHSaB8auLlhkFJJQC1d8lNaXq/xuJiULO2TGuXGYTgpEEqFkJhx6cEk1MQaCwj0pwbljYOvFdrHJUwbMK648NudG4SSY19ddvAzg+G0YNjUWUXJcq2hXISbG4SKYHk8LaHGCF8RNCN1uApwIcwtiulY7NIk73FEWFRUJUsGCLllAUgBvR3hFgXlriZ1UqZuylBqCCnhliX/X2no7QC17RFLpqyHQpE5dpS0uuMa+rZDmJeAkpCDQ5iV9FMaRdAo2QMKSanq9ptHMH2Sw7qAqBTiok7soOC+ujExlBqQ6Q7xAXFRE8AuG8ieCa9iR4ZVdAP9lP0I0fWIsxqoiukYRGJgURZkQQX4YaMUYjdQquoDGVRrAWshypJPKwsCwmEAnINISa/sx5SU80J8KW1WRF6vaB1kUTAkKKXCCiXTn7SNyEU6acwBdP5Fj5/vA/0lHh/hR69+8IMf/A+/9AZ+BY//6r/7b3+w+Jf/5q98u+v/+X/5tZ8b8BVnLBffvIjf/4d/C1fbGf6ldz7E9TDDabnH//H5e5hVZCQB4PbtAn/jWx/hH3/8HB88voQNCu/Mb/En1xfoRoPd1QyQEe89v8THP3qE4qLFcFuhPO4hBPCvfe3P8KDY4n/66HfR7iv4rcHjd6/w6ifnECcjfuvZS/yTP/oavvc7n+BtO8PrT07x7W+9wJ/+5AkQBRYXW3gvYUfNYJzC4/hkh11bwRiPVdPhbl8jRoHSOPSjwb/+3p/iH/1vv4/ya1uUxuHubgZsDFbP1tj9+BjH373C1c9OUT3ZoSosNtsGeFUSPOgAOAnRSchBwi8dmrMW3794if/zx+9jdtTBOYm6tPja8TU+vj3Fo+UGb3Zz7LsSUkaMn86hn+2xnPW4ulqgng/oXs8hW7Jx46lnsM+rBvJswOPTNXqnsf6jM9gnI6KVOLlYY9+VMP94gd37Dr/7nY/xar/E649PoY9GFKVF+OMV+mcjHj+5wdUfPsTjv/ESn/7pBeLcQxqPsDco3irEr7dwNxWZu1uB8bdauFaj+qzAcBpQPt7Djhp+UFClRzPryeb+0Qr9U4tvf/AF/uxHz+iFDAIwAfrKoPnmHbZ3DeqflBhXEfFZh8Wsx+bjI0RDn6IaBdyjAWVj4T6aY/mda9y8WWJ+2mK/rlE0I8a3DaIgGDj6ISdT+9/tgLclnn3vNT799Jy9pW9KhCqgfKswvDcAa8MJ12/fYfN6AciI4lIjvNdBaQ/7YoZ4OiL2CvULg2IN7N4J8CcO1WcF+gsHFAGiU1g9XWP70yP6EEPythQBcmkhX1Tw84CoWZ0BAEd/KnD7LwwQKuLof69w83tukmcU1wr2yYj5Pymx/a2BlTAamL2Q2H2T6bLdYzKh8w8V2icB4lGP4o8bdE899EaieSmw+cAj6ojmczLi7WN6UWMRoDZkpO05g5FEp6agHRyPKH9WYTymD0g6AV8HlFesu8kBRdFEqJ1CqAP0RkLv6b0rv3+L7kdpLDyrSY5+LHDzLw6IvcLqhwahYJCNvbAoXxiMK060zEZO/ZnSAu0ThkhUb9TEIA0PyEaPRwGqpyfUrQLqF4pg7p0O4U2F8z8Etu+QtXazALOTiTkiIJ1/GnHzW9zX6Z9EvP0dymxddfBsFWsmAXO1X6B7b4DYGKiB4TOhoKR9+WECYRcRdsGeU9WJKQV48nlpej7rNxL9KX2W1aWk17Flsu76Wx71KwW7ZEqzGgTKK94b0mLqOfRFSgbeEjDWl6wGMRsxSb19QUtBZqAQuXhgj5jKXKaEX9fQ8zv7IqI/ExhOI2Riy3NQye5ZOpeQA49SUJAiM17c0T/KoBgeV64c8QXQP2AY0vIjStZ3T8mAhlQLcvRTj827Cu3FQUYck4WiWFOWPJwIStlBQD1/GdCdSfjk+ZQu4va7AfNP1eSTVB39V+U6oj+S2D+NmH+OKSV3XPEeEIFy+vqG23Q1j33xGXsTRYjQfZKAvw3YP5aTb3XzLYfjP0rhKh07DnO4UPPG4fYbZI9yHydTR3lNq1syygUbgSafX3lLxjkUPIfqjmygiLQMZC9t+1CgeUPG0pUC19+POPmhmAKQsn93WBGYzL8IaM8kRIyor9nNSTluRHdOQEa/sEB1k7oi52JaaACYTCwCmWA7JzjPMuPhKHXGDhHz1w77hxrlJmBYykkCnetnlp879McKrkoBP0XyU84JFLOk3XQR2ycEZEcfjrj9RkEmPL2u2EcMC0qlEYHqhvJmnSpcXMl022ElUd16hELAlXLyiEpL6S+fH+BKierWYX9hyNoKyqhziFNOv70vz84hQO2ZwtFHPdoHBHemDVPdiXARuydMYs1hTP2xgmk5zqoPGFdqen6x9QRojUIUAv0Jw3+iFihuRmzeq7H4bMD+SYny1kF4gsHm8y22X1+yUqYnO2sbgeatm+5ZOYbEdA5oH6XXR4Jm3TJ5V/VkYaVNfyeJdHlLGbHqPTs3Gy5WVG97+EbDG4moJfTOwjcadqZQv+4JQiOg9w5RCtozhoDiag8/J7D0lYYvJS7/uQIX/9cA6QLMVYtQG/iaPh9z08KeNECMTK4tFNR+nDySUYgvh/cUOrGOJdNmEzso1y39kUjeVcN6DrhA6XFOXA1hCu0JixrCeoR5Rb9mqaBudnDnS17Xq83UdZm9kGHZQG5aTP2W1lFyu2h4LGPyQaYkWtENk+8xDgPEkp1acZsiyzN7mABidI6sZWYyY4RQEtG6L/s187/z/zNzei8t9n6K7JcSa5O38y8MAPqnePxGMJbP/4oYy//gN4Ox/EoDy/k3LuLv/Pd/C7uhQIwCLkjUhcW2raBUgHOUsEoZURiH3a5CGBR7CJ3E/KgjqARg5iPsmt2Fi6cbbN7MGRpQO4RWM6G0DIBNHwaNQ7wtgKVDdGQzRMkQGdSeiyqd4mt6CbWy8G3qqpSRXYsbBo3EmQcGBrdAAPBgn96xgzABcX+wwgorEOvAn0dAbAwneVVADhSAZFehtGQ+Qn1vFcgEiL3iNsaUmjaKKRiEoSSCMlydkg08E1WhALWXlJc2HrCS6bFFgOy5Cpa7x+LDAfG2gNnQkxMKMlcicmILTylmBKYuQah796KgdHM88VB7xdTTgavqahAYzx3ZxSL5TlKXYVRANJyghiIilOnY7j3y5JY9e5FdkSmcI9YeslUIJcdH7zlpcDMemxoE3Iz1KwwVYYpp1GTXAEyphQR4ACLHNSfSqj4F0/gkbe3TZbNkdYIC/ILpeNUbjeHUE1CdBMxeSHbVmTgFdshBpAAYINe3iEggo7cyBZqIFMzDe2xcZf8eIHyqlukpGZUpsZF1GQLjMsLsyciKwH3o7lDFI+0hWEGOiVV0BIMAQVT9hoX107VSBE6qE9O9kbeV+x7VkIBQAkb3xy+Ye/K+WTovyWMTIYEURylwMOk4fQJiiudm52ml1Iqp6iR3T9LXTPAyLhIjIg8AJNeZsFqDk+YcHJMDcHLiqmvSvZGtK/dCX3xJMBJSD97Uv5lAoE6ppYzpp681g1ufqlOC5u8zyAz6UGui+tRfmfaZjz0zPDk5Nr93WVtyeK2rMQG1UKT01mMeY74OZnd4fvYe531l2WUOUIkSU6F70MlHHgjQROT1kx5ThUm+36ZOxTQOEAyyycE12Tc99RQqsKJD877IYyUcX5N/HyV7E82OAMHsKBcGkNIvU4dgI6YakykcKtBPmYNC7CKF3vAtz3RQQ1CC9NnEf9w7B8vjjnkSlcY613NkWTzvX46LnRE8Zc9svt9Y08Bt2IbnkmXR7Kg8BBfl+0B3vMZqTPuJmGpBckgPeyAPn5spKy5JO2NKTb0XlJMkt3w/H+7lfL6+oow5+4S5DUxVMNP32PR5fQBmWVYpAn2vrhSHQBudP9fFwY+YmU95GPecKCx9ToI9BAIJTzmgK+n75XmKdA9zm0z3PRxbzH1+EUCMye/OahDbMMQnJ8NKGzEsFUwXJg9oBpTSs49SukiJuiJA8UXyfxoxhQvpjt/3wTB4J59frvOYJJghLVDl74UUcuNLqqhyvQtlxIcE3Px5robACpIUrONqAbNjxUvUYhob3TOIR/jIcQIBbCiSZFOJNDcg2yccQ3Xk4OErfTjONKZB3QOSluE+unUIpWI4EHietHuQOWZt2KGeJD/y9WGFSU5gIxAXqe/xvieUSbJh6paUvU3nw9TWHKATJTs14SMVIoWGsJ6MdTse2DVgSn+N97opp+AeAFMViCSwFDb5E2M8hOvcD9DJf5IfE0ry7wzavOeYJiAXpYAYkp9SSob/5P1n+e3PY4FcfZK2N9WQpI7P6NyX/Z5AktQmT2UGkeHe8QP4pepG8rYBbv+vgeUvfPw1sPwVPMrnz+Lz/+bfx7AtkdM8xZhKzpcOeq3hlh5qOcJv0wdKqpLISX92ztV4d2ahrjlbjZLgRJ/1iJ81UL04pDI6ASwt1KsSuhcYl2FKqYz60PuoeoK24u6wGu9WHmovoVp+IMYUfOpWHvVLJk8Kf5gMs9tSTN6z7iGljXmiqsYEeuqUQllFFLcHX45dEizqjhN5X3CyzKoGgSi5HcaDHyZedskPeb3lcbJQ+vCa8YyeODtngie9hpTbySFJZQTTHseTgMVPFXsWPVkE1QkMZx7ljZrAimrFFElvl+m4VNpemjj2ZweAYzYEQXrHiZXZpclpBlv+MOn1ZZqMp8lxTAB2XEU0LymhDalkPIO18YhgqL049Cf6EhhOAuafkUmQyfOUJ7Wu5vHZGRmh5UcS/ckBgGRQMBylBMviMOHKBefFFujPON4Mvki1EMVhkmYXBA45ZXCqgugPwGQ4wpQoGVRE/VZg/W2H5nMNs8UEJAiO6TPLSZi6YypidcNr058eUj7z78d0DG7G61VeM4mxP2M4lms4llExwIogNfncKjIC41KgexDJtC0FVAf4GmBsPr18usXkKYoSk5ctXyc7J7CjHEwczk0ReCCFBvkq+Z/WSAEbfJ7ZHoCabuM0CWcZPfdTX/L90J8JmE1McjSOtdlzMmgXSLUXTFc0Wx5PdZ0YH8PxdI2Ygq3YnRkniWdmXHzF0Kooed/4kqmG44oT8fptnO5lOfIYmYwqUN1we8MpUF6n98A+T2R4H+kEGLPEU/f0S5rkH9R9nIKpmrce7UMFvecYuFpMdQp5Qp8noeORQHEXp3oHX6QJa0fQUW44mfAmVWrcMDwo969mtkv6QygTQJCUAaPueA+pkedQbLgv15DZGo4kqlRhYdoAV6dJkMCU/ptfW9+w2qK8Y3ehGshO6SGmQKE4+SKnHkgwWCpoTKDGl5g6Q/OihdkHSB/RHSu4mZiCuXTHYxKeHrzqLmCcSxTbgN1jhflLTghziFKUBFvlxmNYKbiK+7ezzNqx3zEX3QMERwz5Eii2hz7HYclra1omTw4ryfTW1A8qHb9HeA0ImIIC6hs/AZBhyQl8fU2WKEs8M8OWQXK5YehNPrfccVlscjUEj3VYsuOy2LLaAjFOqbVqOCS+HsAB7wnXyJQiGyYQmQNnhMsrOOmz1WPanxo4HmRzxdQrKUdWbfiKk3A1BgyrJBdNgE/3DLcq1wRR0gb4msynGtJEOUT4OjHLe4KxKJH6IDGloEZFya/uGP6lxsDeyTVzF+xcJ5ALmA0/7FgB4icw5moF3XuC0QRW7Eyj2DK4SLjMYAaMRwZm5+CNhO48xpWB2djJe2g2I4bTaqr1yD2Y1eWAqCXHNUSoLgEwQ3ZQpDTTKAT0boRd0sTMdNUMbCnfVq07JLvWGmpvyfK1Fm5RMggogUw5OLglA4ZyHYevDWTv4JcF9HqAnxWQLdk64QJE7xBrQzA6sG4jgz7hAuAjYm0gOn7R+bTPLHFVg+e+AMrI9z3CrGJ4UKkhBkc2LwMzcwCZHHx3AHs5zVWl595n6e5Vg0wVIcnfKHKq632vY9pO1IrgMAXkIEQyhPmRAnsQAo8BOPg587/LgttLwToxB/P8ReE94j67mJlFxddkdjJGxNH+OYbxz/VY5p7Ne4+/FH/8JX7Lr3zdyPNn8dF/9lcALP/D//jXfm7AVxxY1l9/HJ/+138H75zd4sPPH0CoAGU8lIrwXkCpiOFNg1gEzM5a7G9qBrzMHIITKBuL/q6CrBzEZYlwNkLcFgRKG/qupE2Mmo7QG3YR6rWCO3L8wMo1EpEsX471x8pCvSzhjjyES4xVy8lC7hjMq6x6L+BmBEvjqWeFBQC9lvBNSuE8HaA/qWCPKPkDADUKDCmYRfYC4cxCXRbwFWVwwQA+MZnCC4QyoLhS9O/lugZBIBslWFvhAb1j8EuOZ2e5OMFxqAOqVxp2EaAG1jW0zxzUXiFqShNlLxDqiCgiUJEhVXsJP+ex5/26OYFd95AMoK8zA8brq3oBX/DLNf9O71ISY8Vx1q1ItQacnIbEHPYPPHQruSgwZ0BLZjs47pT9jUd8vrQETP0ZJ+1qIAA3OzFF3vuKPY8hsR/jKkkbgWlFOFc0MHk0TqvEXCUXqQaCskHdCuRIf1/HSXJnF5GLBnkc0vmx3oLgy5eYJk253y0fZ0hVDrkbMIPN8oagNIPsmA49poAeTtyRSt0Jcl2qIhhXMQEnMXXB+ZLbmL2kPNDOeW55jM0OaB+nRZAeX+qQy8AmA6NgMAUFqZHAVaZjVCNgG0rq7ILHPjGDqVJCDTzW+jKie0gQoHpg6v6LBE/9iZjSkNVI8JoBaGZN8kJL85rPzwBKJkDCWPp0DzeHsUMCL6pLP0/7YZpt9v0dqgNc6taz8wNQ5iQTUxhIVAfAmVfnoxITG5pZrdy/Z5cCekc2JwNSO0+LDrlnseJ9l1OGM6CwM4YWiRCnxSGX+gWDAso1FwOE4z1YXROUAIfXuFpMzJAaIkwLtA+46JBBstkdALxuM2N3GJPyLsInFhWCgN/OxVQfksOG2DGZGB57+F7I4+YLMYFkyMP7IINtV7I6YlymBQwNVOuDTDI/gibzJTxBiG0okcwP3ROs2Hka1zb1JqaFKtMF2FqmonmCyWEpJ8at2BPAThUemhJH28gv12JIoLrzGOdctDQdx9+ljk+ZPGbSEYyXmzCxa2qM0/2Vqy4yqDL7cGD8BOsgguZih9kHJuum/efwmem4EqAJhmMtApOdeU8cFgLus7xRAOXaw87Zlfj/sPcmsZaleX7Q7xvOeOc3xouIFxmZWVmZ5ayuqi4b2nK3ACPMFskLIyFv8A4bDEgGGyRgA2KNkNgAArFASEhI7IzaCIGxjWj3VNVdQ+cc03vx5jud6ZtY/L5z7suq6q5sV5boavlIoYh47w7nnHvuvd/v/5uCioygDTGxNQas3a/PkDtW1GsCStW4IUyG31G7ihGeWzlcl/wcZXVG/7nZd2TqyjFZPEo/k42LkvG4UBZi990Rz3F/HGaimCyrxcAsDiA3kQM49AkfQ9UEIjwGg2Y/ZXWGFFAN2S5KK3leZMs6jD6Ju2cU2SnJ61RvDGs6bIBsHbp5ysoO0zOuEtLS2y8CQTIZUBlfN1Z2DOBYIrJxDnacQG8MmVkhENKeEQsDoITAEKijKr4Rg5LRs6oYCFTqWLcBnhfLgYDsHLq9HOltC1dqiM6z3iPTkF1kM/u+ydrCF5qgUUp4Ha/jxsLnmkxo66LnD7CTDDLKSSFE9EfGQQR4rffnVZgYrpMlZAxbQ/9jIiE7MnJBSsjODnUiwjgEpQAlIBpD8Bk7M4e0V+sG8CmcZ3Jw9CZS7urYR6nV53/vd0nCwjp2UDZMxxVNh5CnlLLe67X8sa3fh89VhNwDkcYMwPDHqj7u3SeEsAORw4fiPdzQA87YVxl+UldmCLvk2Psdlz9p+wIBPr/4jOVpOPkPfg7A8t/4p8DyZ96yp4/Dg//w36bUMRCI2Cnlf/2iue9a83lAciMZ+qB2iw9dceFECaAY7qPrXXk2gNgNt2MgezatD8nwKRePyTrKhiyGibNqWTgeBCWErBPZPV9fESGjPK2ffN+Xf/mE7IodkQHqF9U23/2+lza6lIvw3WKhZx2Z1lg9ILMmO4KKXqJ3vwNN1RjSBttFQH4Te9U6Jn2md0ybzG4ptetlcQAXOM1BGKpdikuGc/TnsvexyG7H+slut8DuJXq9ZIkJl7vUyH5RGkRktuLCuT8XuHdJ95K/vpKlXZAR6UvGfZQy9kAaMgKVePy91BHg/7vJjnlSNeVRNr8njQoYzlO/wDDj2NVWRylVvltI9fULZkwmSW8Dlu+G6I/jax707ryla/qRyMaG6HPrWd2dbI+SxoBuHhNO6930X7VhYHb7x+6DV3rWrmfxVMProZuKIWm09231TBuvzTDI6oTH4PMyE14D6TpAtWF4HGl4fj7/nHExuwkD89P3CIpA5qYPKunlij2oUx39Wz1wM2NWgdhMDFLP3i9HMLdjkfqkUBWHoASzPL7eYyUcBrD1o1LDfnMpkFR8HWxB0KI6grd2xs+THkC6NILVvu4n2zFRPTPsEvq2guSxBUkf2cAWOnAx5XgNFDes17Cxb1B1iBUiDHvpmTjdBnSjHjiEYRDA8vnYYycx+LV6+Z204R5Lz+spCLIlZrSrw1HtTtam2jCwgSEGjfQVHcP7NLLnyuzklEHw+YLiYEc30evWhSHZ0yVxfy2frwdU6doPjJvNeR32nxd90f0gixykmNzhZOMGFq4H6z1TaQqJ/M7F88t96SWi0oSBHUUIw3H2IKsHN/q+vLIHZoLH0Y3I3iVbhoT1skjKAH2U1XoCsigHpLRXxO5H+s96KaXPBGSU4XJwshsmUvIaJZOxWsVrPl83Vkg3rAygZFlAb/0Azm1JhlQ4svey4zGohpLFnTSU+8QeTAIlEcEF1RKs3+jrLXqmUFj65ZKthct48lXjGFYDDK8tgQmBqDTxOaQY2MT++u39gT2QHMCw3oU/uYxMorQeLlNRYuoHECRdGKSdsvOQrYUbJTAj1qKoltaFwVsXWVndOA6DJEOBpPMQse6kB3fS7ACdMJ7sWUPZp7BhAFZeywGM+VQN51l2BKRBip3P0u4Y1KB2AMyN0x1Qi+eoB5794/VSUGE8QqYgawNfMDm0/7zrAVlQURJaW0BT4igbgq++WoRyU/4bWu5sE63lficKsrEDeCNQswSuWhIYAj8OnuLt6UEE01HzhJJTpeJziSgr5eveS1IhBHyR7LyPJnZstnZIWkUICGUGUbVkCe+xgaLpeF61AozdgcCmQxgVZB7jdSDcPVbzHggMiYbo2cPh956saP9vYAdE+xTZ+9JX7xG8J1uo9U6WqvWOzewrSXovY1zzD0E5PTDstx7YSRHv5/G5upF+cw4DoxlZyeHnwE66e/93P/oc/fP86M/+qO2fAss/dPuTAix/oXssk9SSCZs6TA432NyVKCYNKjVCcqdgT1qImxRu5HF4eotrs48wp55QZxaJ9mifjck0Zh76VsMedXh0covz6xmSDwq49zYMs7kugNQDgp5GM3dMYNQBOGzhOwXRKNgDh/yzFPUjC6gAOaIXU+23yL5Ton6/hlABbptArdSwwPETpmvkz1MmVEqWwoeRo0dz3qF7nTMl8k0DUSlgbBEMQ3oggWQpUR9YyFqhPQoYPVMMSwHgSwfoAJNbpN8tsX1iIScGvtaQlYKMDF3vMUzvJJo3WpTTBt1dge4RgFZCbRSyG4H1LzcYfTfH6qsO+WuF7ddb6LMUqhaoHzrIlhURx7/BDr71W/xwL14oVO830C8yLlJ6P14XZWVv1hDPC9iFpddxZiAqjSACFt9VuP2mQ36u4d7dQv7BCNsnHrqSaACCnDQMDLGqBewkJqUCqB4FZNcKLn4plmcBN/+MgbpNkF8LtPNA5nBiIe+YBFe+ZK/h9rFnfUQtcPsti/xVgvXbFslKwWsyxDICE58SEK6/YlG+oMTZtQLrN+mzLM8IJOyIA4r8guCnLQI2pwHjZxLbU4/yTKDZE6je7jD+gxTtIqDdstKiPg6QjkEpestBRe8tDKnH6BOF9ZsB5SuB6iGvp9ELMl3VCRfL+ZXA9hGPa/xMoF0A9SE9pMU590F2AtkNnxOBwwQAaPcofc5uWHpenfB+048EKzQ+DgOY2zwOaBsR5byUdtupQ/lco7gKuP62w/hjjfqY8mNbsEZl8bsK9ZEYwF/vCwuSATXcN7KiwgOL7wdc/opHcku9bPWIrG92J9DsB0w+Y4WEGQfMf0hgtHo7Brw4YHVMX6eq+TrLjguo9VtkuEfPJeqTgOKcoBIC2J4Q7I5eslPWTDn4afcETC1gRmTlkg1BtbSsMKmOJbLbgNVbAtNPyLhVRxLlhYfNBaoHZGG2IrLaDtj7IfsrN4/lwDIHSYZbWAFpJdZPOFQrrgi8NicqXmdk4aUBVm8DRbzm0k0/qGBNSLoGhOM1c/DbAddfF5h+wnAWgLUc7UxQ1hgrKbwSA5Bq9phamS4xeBd771xzKAZmX9dAdRwwfsEAmaQKWB5LssQHAqNXAXdfjZURMUAmvwlYvq2QXwdUE3a4bh8JjJ+TvVu9ITH91GH5tqJPdw5kd7xeywsCbDPi0CVbUjab3Xlcv68wfhaiKkCiPpAorj3u3laYf8TXw5R8HBEkg2ZUTACNQ69uvFsw6xa4+Rr7L8cvffRDBmxOFPI7Aqn1qUJ54bF9wL/XpxLjFxwaXP2Sxvwjj+1xD8oAUyoOCcaKQ43Ibl5/LUFx2Q8kAkavyTA2jyUmLyzWT/Qw8CmuPexEoD7kuStuPOq5QnHjUR3owfdJkC93QTcLCfdAIbvzSNdHaq7QAAAgAElEQVQe6dLi5r0MqqH3Mb912DxSyJYExcna4vr9HOWFQ7py2DxMACgkFZlZ1QLllUMzVygvDIQLqI4TFFcWLpWoHitIC6xPFaRhYFCQQDdRfP0coNuA4tJgfZpBBJ6jds7BS37rsHw7/Zw6AFEJ4FImxwbBDsvqOGHYDoDqWKO4cpA2YPNQY/pJFxkuXr/dXoL8xmD5dj4EAE2edbBjhXaeDj7HbOWGwWF9kA4y1SCY0No+UMhvLWRr0RyUSLYOzZ7G6KzF9nEB1XpozWssWRus3mLdCAdiEjYTyJYe6V2Hdj9FurKwpQI8kN22CBDYPipQvG7R7nMCN/pkheZkjOy6weaNEYqLDttjSl/zyxbbRwXKsyb68iKzmMe+0OMR9NrAThPYMQN8iosWAYDLFGTnYOcZuplGdmvgU1aEZBdbuEkOM9ZIb1tUT0oka0pkixdrVE+mSO86dIsU+WVg/cjrCgEC9ekE+XkFn2q4ec5+01db2FkGvWzRPBghf7GGmyRQNfickzGSuxaqs/DjNFaAKKCgvLbdy1C82sLsj+ByhfS2gS9TAk/rIKBg90ZkJXMNn5EpVlpSZjulcVg0FvbBDPp6Cz9mF2YA4MsUcqt3ibBxs9Mc+mJF4J5kBF5ZClE18NOSbGeiIbc1wnQUFxF+ALwACF4Db9d7IoVSTIKtWzKYqy3CfEJ2dLUFkgRhXPDfIQBFzr8bTi5FmvCx5G4g9rmtaSlhzTJWhfSAuIsLnSz2dTpHsO08uzfThIE90Q8aQoDQ+sfrRvqtB6hfsL/yp1aS/KJsf0oO4ydtX4ixFEL8KoDfCSFshRB/FcC3AfwXIYTPft47+Edti/cOw1/7n/4iPlgfIdcG/+LeD3DWzfF3X30Nb8+u8aS4wfdWJ5DCQ4qA62aESdriMNtAS4fvXD/E1/fO8Q9fPEXXJljMtvil/TN8sDzErx19hF9/+R5Op7d4WKzQeo0PVwe4XI/RNgmeHN2gMgm+sf8KL6s5Xi5n+HMPnuO2K/DdVw/xF974BMYrfPfiBEVq8NbsGmfVFIfFBhfVBOs2xa88eIaPVgc4X0+wui3xxqNrtE5hv6iQSosPrg9hjMLXT87wmx88xcHRCu8sLvGd1w/xL73xQ7yo5jjbTjFOOtzUJW7uRvjVtz/C3//OuxgdVuhajaLoUGYdVlUO02l86/QFLqoJLlZjvHVwjVRadF5DioDv/v4T6HmH+bQCwAHSo+kK55sJUuWwbjKsLsb41a9/gP/nH72H9/7sZ9h0GV5dzzAeNShSg9fXMySpRZm3OJms8ai8w9/7B99E+ZQxin/+4Wf4vZsHcF4iVQ4vX+whn7X4pZNX+I3vvYVk2sJZhfG4wep6BJU7KO0wKlrsjypUJsHl7QQhCBws1jh/ucD8cIO3967ww6sjhCBQZh0aw5nJdp1jNGmQaYerVzPsnSxxczGFUAEHhys0RqOqMijlobWD/f4UD3/lFS7XY2yvSvzK+x/hH3/6Bv7M4zOcbyaouwTTosHNegTnBFPBMwPvJbbLHGKrkR5VeO/4At8/P0Z3l+Hg0RLLTY4ss9hcjPDgyQ2qNsWirHFTFWjqFLNJjasXcxye3mK1zdFdlhg9XGNzNoaqJZInW6SJhZIeQgQs1yXCRY7s8YZVLiLgrcNrnK2m2NYp8txgkre4WY+QJBabl1N89Wsv8Go1RV1lyHKD6uUYcr+FtxLHh0u0RmOzzSGVR7vMUSzY6eq9QFOl0KmF7TTeOz3Hx5f78DF51qwzInUJCOWR5BbdJsV4r0L94YxBRDJAGAmMLMppg+rFGJgZlJMW27sCQgZABEgd4FYxdW+p4J/UkCLAthr5uEXXJnCNgtAeoVMoFjXqy5LDhDGrRhCAbNqyj3Vq2OMKQBUOUnpW1RiJtOzQ3hQ4eHyHq/MpkssEZs9C5A6zeYVtlcEsM8iRYY9tpfk8hYNQAUlh0N3mgAfUvEOSOLSvRjh59wKvXuwBQSB/kaB50kHlDsED8/kWUgDXny4QMg+Rsqd2Pt/i7sM9ZKcbNK9HDMTKLbyVSM5TQABm2k/uMSTdCisQUg94QT/1WsLMHcTI4uBgjcvnC5SHW1SXIwZtFQ5yoyCOW8hnOYdkpy2ECPC1hqgVQuE4FCgs1IsctvRQtYRPyKS3xxZqQ2m7rOJiQweEsUNykcClfRCUH0LC1FZG9gIwM0e/dukgNgrJUkJ9bY36uoBaMT3YZyGGTUnYqcPkA431OxayYqCWywE3dkivFLoDB1XJOIizGP0wg5mEgXXfhZoAdmYhRxbiPKNdQHNAY8uAPpTKpwH2SQNcZcN3jR87JFd6YD2l4fknW89uWJ8EFOcK3ZxAR1iB9thCrxQl8nek3ofuSxDsN48M0guNbo91M9mVhJmx87WX3fehR2a6k9D3bH3P3gfNYwgaGL0Q2D7mcK1XZ/TDp3QlYHMOoXTNXto+3CnZkO1u9+jrN5Mob48+ewQxBFf1oVnNicPomRrUCmbEpN4hHCoFXBqQX3OY0U1Za6O3PUsemeMNvc+9X143O4UAQAZfdTt23RZA8ZoDHV2RjW/3MciakzWi/z7EOhMMagnV8bmC2ilWhOV10s7BPtqWLFR9BBSXHG65FMMgafuQAzczRvTPUhETBFUtsuMgK11GOXyvfkoFsjuP6oGkmicHPZ+SPbG9YiRbBmwfcFBUvuZgRAQ+Ty91toWEGVEuLzwTqLO7nQc72XD/XC5QXLC3dHzu0Mzk4HPenHBItHoTGL3CIHvuA3fGZ7x9tiJrHiL77VKgvIqy7jaGX9VULZiSj91OYtLxlhJRW8jBtzpI1SMgd2nPdGOQYwsP6CjRbScKxbWFGashhCnZetT7CuWlY3/qGxqTZxY2qiSEJXPfDxK6uR7UMKphX2hQggz52lKBFNntIDkk01uGASWVhVexciXaCvTWwGsJnykqeW4b2EkWZc8WdpQgWbYw0wyycwwW0hIhlVCVBaJHtZfqysrATVPI1g2+UShBJjd6RYX1A5PrxtnQr0lWeMf+y8ZANAZ+Vg5MMjoDtz+Gut4AUrL7clKSldTRA9mH9BhLwLrepdYCZFoRAsR6C5QFUNU7BrFnS30A2haQnweSQsl7TOnOg8kPQzX8+8eSYO9vfypSYU/Dyd/5OTCWf/3LZSyFEL8G4J0Qwn8nhDgEMA4hfPJT7/cFgeV3AHwTwDcA/PcA/hsAfyWE8M//THv9M27FVx6Gh//p38DJ/hI32xJZYnA83qDUHX7zg6c4PF7i8nyGp08u8enzQ5SzGkfTDa42I6Ta4uZ8BpE6qMTDv87x5OtnOB3f4v/+g6+wO1IGeCuxWGxwdzeCVAEuLlKFCihHLTa3JSZ7WygRcHc2RbZXI3wwhn3SYD7fQsmAq+sJF363CXzhh4Wq0p6L84sM4rhFuMgoE1tYnD66xsvLOXCVIcwNVOpglylGx1vUn03gZ5ZT0cxhb77F1Ys5uxetgLAyLn499vc3uLkbwTcKcqPhUw8RBMSsg6815kdrbKsM6vsj2HFAeNTANXEiFpnM8KCFrzRGBxWqszFC6iFqdiu6ow5YaYSRQ1Ia2Jsc40crbF5MIRcdXKOQvUxgv1LDrVJkrzVZwcc13DKFqiTc2EHfargHHeR1MqSU+olFcpUge3eJ9gczpEuB7VMu/IXyCF6g+EGObhZgDw2ECgi1gtoouCmZW2F2CaSfS4rtWMVgx6z+6BYMRjLHBrACaqPox22Bbs8hiYFPds+g/ChF9YaFXiq4sUd6LSG8QHNskZ9r9gGeboFPR1AVF2fZDRdn7cLDTxzSCz18Odq5BZIAlTmE1xkQvZbZtcD6HQbu1CcOspHwxy3EdYrsRnIxLABzYCBahdGnigFDBx44aJH9sED92CC51jALLlp94ZDMWogPRsivBDZveugNOwW3px75a4luQeZw85aDMIKps0lcGANoj+PPG0rBXemRLCX9tk8Nyk+TGCbFxahPAPuoxeS3cjKDX3VI7iSk4zmTjcTohRyqK8zUI3/NazAoIL8E2j0uZHv5cl81ITuyi8LG/stAKXt+KbmgzSPIGfE1aA7oDfZv1tA/LJHdkIXtU2SDAvSGzNzlv9JAflJwsR7lzbJjaNT0I1ZVqAaonhrkZwn0FqiPAkavmICbXYsh0bN+5JBdqgEMsF8y7lvJ6o8u9nb6JEAaMSTKCieGxXCyDtg+7CXaYvApAkB2E9Du0zPay8p1zWvMjPG5YKN2L6B8yWvSjO8F0CCeZxfl2oY/7xb3go4sMPvE4faraidBB3ZBNlEu3deStIudd7f3nUoDQNI7GxSQX0fPayngM4KI/CbKyNvo+40BVtLwOHoQcj/lua+3UG1AfUAPdXEhhroO+vz4nVcfsceyD77pj605oB9ZxWTOdsEwpeqE5xzA4MFM1wSRQ6jRvWqXnl3rU1j76/Z+IqpP6P3tU477JORenu3TKH+Ot2336FdFAFM1uzBIxfsFv25Ya1LvRankvZCx3m9pone2m1JxEKIv22vKjhHIJJsx2eSddSSek7nA6Nxj80giWTGwanTuhuPim5fnxeYC+a2HMgHVgeS+HVD2q2t6cMsLJqaaUgyBQrrZWQXS6FPuQYjNRfQs+yEEqQ89yu5ilUcu0cT+UiBemw2Prxvz8V2sZOlZS4BghbeXg2ez91+7lIBO+Oh1TinVTjcEKC4RQxKyLQTGLw22JwmKKzd4VUevWlQP0gFY8TWh/1M3Ht1EIdn6KDXf+VpdKpCuXJSeqxi8trufGSlkdxY+YTBQkALJ2hFkNQGqcfCZxNXXExz+Tsf9zSmzFp4gTFcGzUGOdG0gOo/mOIPeOMqNu4B02cGOkyh5xgCwVE0ZdLKxQ+2HTwWSFd8ofbJr0DIqGuwgg/bRSwtPGbct6L3VlYEtk0E+7DNF6X/nYScJ9NrATBPoraUU2Xr4MqGsNVHwKc+BXneQrUX9aIxkaxkipATMOIGu4jqqs2iOS+SvK/o7jadkNgT4kj2esnWUE8cQIbmu4A4mtAN0lj7NCPr62hLROnY6AvRNNoZy3br7PFjTcfDm/U7q6jz9l2W+82r2iappQhaz6QaZbihziKohS9jLcRGBYL8JwT7LECjF3dYIP+qHjMmuoijInMbHEVk/waIns++mDIayX3QGfXfl55Jf74f5/AQv6OC9VIr3/UO24BxElM3+qQCWf/vnACz/xpcHLIUQ/wmAPwfg3RDCV4UQDwH8zyGEX/2p9/2CwPK3QgjfFkL8xwBehhD+2/5nP/Pe/wxb+c7D8O3/6q/CB4GXZwvIxCPcpcjPFPw31zCvRhAWyN9aw1oJ7yTwWQk79oATCCNenHKlUb65wnadI/8gj6b8ADdxSC/VENySvrFBfVECuYe+TBjg8sAgfcWFdPvQAEYOYCm91EjWAu0iQABQb2/gPhrDjjxUxeoMnwLhUQPxMueiKaagtvt+eIy+Mw7RD2KnHmorOVEeUb45einZH7cUqB9ZzL+ncfetDqJSSG9VLFwHspvoI9VcQPZdfd2C027VCNSPLYQRBBKBRenwvI1PAHvYofgogysC8kuB1Tc6zH4nxfqpR3YrobdcrHcLh5AGpJcK0gnkF+zHm39P4vZXOiRnKdIV026zK8r70iU9m7oPe4kLSTPG8HzVQ4KEdhFw8LsByzdlDA7hl395FrB5wn/nl/zSqg/jwsrFiP0RQV55JqOPlFPqfnFVPfTIriWqJxbCCeRnXC3Vjy2mP9SDL0439DKaiUB95JHdSJgpF3v5lYBLdn7S8jUXdKuvWeSv9JDy2s53wTjpkh7Y/JryzfJcwMbFt0u5AG8OgPwKQ9Lo+pQeTjPZBeV0M07ZN48pV5t+BFz+BYviRQJhORnvKzEQuGDtOwfLc4F2TjmlroGrbzFMKFlSllo9CBg/I5vSHAZ0xwajD1ICkDnTSLdP+LmSrOJCGFzwdxMmqBavOd3fPvF48A+Bm69J5Ne7pE/VANuHYrgOvKYk0kwEbMHzIDzTbEdnXCCuvuKx/7uxvH0kMH1usXxTo5sRnK7e8Rh/SnlldcJzld3xeku29Le6LIbZeGDzhO+H8XN65rYnEumKUk14gsPigr7O6pjApLggGMpumeI6Oveo9wU2p2SKEOhZLi4JmNJlwPU3eB3sfRdYvcUp/fh5GNiV3h949y4ZpfJ1L63j9ZwvyRr0nlCv6SG1BQOI+lTSIHjuxy88AcM2oDpi1Y40AfltgCnEAA43jwX2v2dx8zWN/DJAt2RQ0hVllNUhve3pOqBdSF4zF+wy7CaUEnczMQD2yTOHZk8OSbjZKqbAxoCh4tqj3peYf9Dh5r0M5aVj8fxYUNZ5qsnMBHopbRm7CzMmF4+fe9SHEnvf77B8M8HowsMUIiZwYqj+aPYIpsorj/Ujhf3vd6iO9MDauVQMacmbxxLZDZ9TmughNTvgBQCuELGuhN9LPXhJtnz86iGlx/mNH6wPuuG5z295LpJtQLOQ0FXA6NzC5QQStuhZI4vNQz3Iwc2YycPlFcGcGfE8u4Sy5GzpsHpDo7giy5RsPFZPNJJtwOjcoJsq9inesU9SOnD/BMN9+uoOl5OxSlYO7Z6GzQg25x92lOWOCDDSDY+tWbC7s7imVDQooLjyyO4Mmv0EesuORFMSDG6PJKQFRucO9QHlsqOXDdoFF7Eu673d9HPqLUFOH+wzelGj3aMcVjUezZ6OXmQHEYBuqpBsYxKt5vsg2VpURylEAPJrLprZl0iAHKRAsrK4eyflZ1EXE3xvHLYPNGYft0xJdQHLpynypUd+xcexhYLP2E9ZXnSoD1OCpK0bgmv01sInEs2+ZkquEtC1QztPkN2a+P0s4HIOQpKNZVhPTsCmageXSbicXlszVijOGrhCo5trJCs7eP6TFdmydk+juOxgC41k1aF+kCNZWaiG4UDSeDR7CaQL0T8LIAQkK4OQyqETMruqIRoLc1jClgr5eYX2sESyNlDbDt1+QRYv4boJLkS/qGZth6RkvNvLkV1W6BY5io+v0by5D701cIWGsEyFNdMUqnFIbmv4XMPnCRACzDRFfraBK1NIw4AfVRPc+ETB5wp6YwZAIyylz3LbwE+L4ec+1ZAdgWKI9STJdbXzQAoOTmVj4SYZ9MsbhDyFHxf0dcZNWL9jFBumtg5ezh4cZinlrpuKALDpELKEIUF9wI6UDOjJKZcVPsRuSfopB7DoPMFkDw6tQ1ByF/4D0LsZAtB2TIIF8GMBPgDBYZry/71vM26hM7GHknJXkfMDbmAcTQdoTXBZ1dGTKSmHVYrS1zRBiNJbSElw+KNezT+khgTAj7OVAH3DfxpSYf/kA8vfAfDLAH4rhPDL8WffCSF846fe9wsCy/8TwN8F8K8D+OcAXAD43RDCL/0sO/6zbuU7J+Hw7/y7OH50CyU9Tid30MLjw7sDaOlhvURrNJbLEuNpjc1nM8jDBlJ65LlBph3qLoFzEs1tDllYJJnFo70lrjYjNN+dY/yta6zWNNjYTQLIAH2VoHjvDptVgcVig9W6hF2lkGMDIQNcrZFNWlijMZtucXc3wny+xe3tGDpxMJsUybhDUXRYrwqI65Q+za2GyDxk4oGzDL4IkPMOUnokqcWkaHFxMUNoFEQrMXt6h84q1JsMSW5hXhcoHm3QPJsAhy3U85xplPv8spIJJW7BSsjLFP6opQQRgG/0zv90owEJqKcbjIoWNxdTTPa3WJ9NgMRDrjV84ZHcKvgnBMX2sANqBWEkOy6dQL5fw/9wDDPzwMxQbtcplPMazbMJ/MRCtIqMYgDkOE7Y7lKIRYdwnUHstwhOQCUeeF5APt0ifDxC+u4K1YsxDt6+we13D+BOWoRaAyqgWNRotilCrSHySB9sEsAD408UtqdcBEECfmYglwl7K0sLbBKkhxXcJ2PYhYVacTLsJxaiVsjPFbo/UwOvcvYvFg7JpIN7XSC/kGgOPPzYITtL0L3ZQFyn7AQNwOzJEndXY6SvEnQnBuwXE0iudPQOAv6NBtnvFagfOoTCQV8lSN9Zof14yotekE2yBwbpeYLugcH4Bymqhx7iqEVedNheliieJ2i+0kJFGaXdJ9PazQLcxAMTw85VCSD1UJmDfJbDjjzEXgf9WQ7zqEMIwPj3s11lzojJwHqlEB7X0B+UKC6Bu69b6GmH5Hslmq+0SJ+lA/tny+jZeaeCWadQKwWfB6itRPkqAu0zzcfWMTV34Xk9BgHEROWgA3zuyeC+10BcpcgvI/jvJIrXEvWJI+Odeug1vVp6K9Du+wjEBNzEYf5djeYAaE47pK8TeMXan767FG9WGP2D0eCvFR5IbyXafY/8ggvxdhGGWhMRYopxxiFU3wcL0LNZvAaao/gSWg4x2j0CeUo7A0ICJMtYtzELw+JeWg6EGDzD58W9wa+ZOei1QrIkmyo8hnTePqmYoIk/695skH2cQxiCoGafksj6seEgy3F/en9ueSbQzcG02Zae1j7hN1mJoaoovQPsGGiijPJ+r63qeLw+Ye2RaoFu32H2fQ2fcEBhpg77v6Vw917A/PsCN99kmrNqONzqpvRQjz+jF7gPXvEJpYmrtwneu1iXpGr6VzenAmZKj/HmMYd8xWsxhFCtnwLpksMC3UQZoACao4DxMwLNdsHBkC3pm82vQqysQOyc5HnQW97Gjji4G39KH6stRGQdeV7qo4Dx81j3UlH6OTrjgMKO6OPtZhwAuZz3TbZkIhk2wzTXu3dU7Fzleeh7LG3B18PlfAyb7wBxfch90NueJaWHVTUEcLpGrO4JsZKLVTuqBbLrgGwdsH4s0e4FzH9I5reXkaar+BxH9MSOzh02JwrNAVUA1ZGMDDVZw9EFq0wu/qzE4W8zCbc+2NX0JBsONHrpJSW4ZGrTTUA73bH57VwMFSUuj7U8DgPbyMoTDg3KCybm9kMEaSLbXXG4AgFMnhv4VKKZ7xbCqgtYn9IfbcYCiw8MNica7UKguNq9L4prpvf2DGfPEOvG4/arGvMPuYC3OZ+rOpIYv3RDBUzPmkobsDrVKC89sjt6Qk0hkC/9wFb6ZNexqlrWtvQg2ZRkhvMrg26uoaud/LNdMMVUN5SoJls3VLDsUpol8mvD+hPP23VjifKCww/VerKVhUQ7UyiuLEOzCknp6Uwj2bohkMmnDBjKbgyagxT5ZYvqJOMwYT+Dav0QyKS3Dj7l8bHHmHUw6R1ZU9ntAof6VF7hA/KLGnaSxhAlsoHNgn5eGYOTACA/28DOc0pSQ4CsOtgF13r98Q/hbLFnMyiGOUEK+FRCbQ2/21J6TeH9LiTIergioQe4sRCtgR9lEMaheTBC8fENw4HqjrUmLUOShI1hP5HhE62Bz1PIjlJW4fzOW1lEn2Oid3Ui3jNo6F59CHs51Q5st2QX0Rkgz3aAU8kd6NxWEFlGwHcfdN5jC0OIktckZQiQJ0ANbUcZ6/2uS+/p++w9mrG3c5DCfgGfZYhs6o/5Qf8Y258IYPnkNJz87X/nS3/cz/7Nv/VlAsv/N4Twz94jFkcA/tGXCSwfAPjXAPxGCOHvCyGeAPgXQgj/w8+89z/DVr5zEn7tv/5X8WrFRXddZXC3GfReg7zosHk5xezxEnujCh9/dgTYuNJJPESjEHIHdBJiZHF0sMLtuoRpNbBKIGsJ+bhCeFYi6AC3sEjKDs4o7C22uDqbQdQSIYbFqK2E3bOQGwU/NwPISlZcJGcXCupbSzQfTRmyMnPQ0w7uMkf2cAvz6Rg+j4BHAXLRwm0T6GtKR9OlQPs4dm0+qiGeFUg2DCKxC4v0UsM87qAuUnZ3biT8UQcsEyRL+ilYAcEFpFor6Fid4ePxia1CcaZQPTWQlYJqY3CIBXwGAkAvgIcN0u8XaN+roT/JId7dQP32BPUjBzHvIF/Rd2b3LB6c3uD15QzJs4y+mAnZWlf6HZDdUNaZX0m4NMDssRalPTFIbjSrIRoumlUl0B47hgCdWhSvNOpTA32nKXMdOySXGqpj92jxWqKbBnRHFvqGvaYiRJnfooP6lODbZ7F2pJc79v6hJx1gJYrnGt2eh3xUIfvHY7R7DH1pDzzySy6y/EkD+TJH+s4K1SrH+HsZQQC42FC1QLcIsGMHVUvAE0SYfQt9G7/sLgW277fIPs3QPm0h7hKMP5XYPiKoCkmAMALJikxBshKo32uQvMgoC92SIe3mIT4fWePihYb89hLth1NWtNQYFuYQBDJBEdhAkDHXW0pkq5MAV/jIpgsCoZRVLM2hhz/qkHyWobgQWL5vMfkDjc03WmCrodcSei0QErKG3SwCgRdk/tJfvYb7P/ZRPSAYqU4dRCtw+JvAxZ9nANVQJaMC+lqQ8TOBbgo0hx4Hvy3Q7FPKW5zLIRimfC3QzhgQNXohsH1IhhECWP9yg+RZhvxSoN0DzMxj+qGEzRk6o7cMX4IAJh9opMuA5bvA+NP+fLD/dvQCCFJg80Z8nWOacX4VF6yWCclmGkFSJWAnAbagV27v9wKuvylg5xbT30/Q7gFBU6bqCiYZbx8LtHueEnQvMP0Bg2FcjqFGxSeIEjOyun3/qi2Y3JxsyO6aacD0Y2Bzys+CbkalgjQCk2dk82zBRfnqLeDoNz1uv6pQXhAc1Yfc73RJEIlAdrOdUYJdnNM31i76jl1+VvsEmH3kYUtKGF0KTJ473LyvkN3wNukyoHogUFwELN8B5n/Ac1k92HV/Tj8J2DziIr464j53U57/vd8jy1q8ZnpttvJYvaGgalbQJGsCWi6QmRZbnQCHv+2xfItdnfUxn9/lBDebx32qMs+3S8mcc4HM947N+bi9LDBIgdFrB5cIbE+oJGHIFa+tIAmsNqcCs498BBUed19leFF56XDzrkZ+FcORUp6bdiGGTt5+2/u+G/orbS6GEJ/yioAu2VIyzKoWJviOzskEr08ViguP9ROyhumSQMyOxNBx6zLWCTqA3pUAACAASURBVJUXFtWRhhmR9Z59QvlkdUC54fjcwWYC9YEEBDD72GJ7rMjan5EJdYlAtmYQT3VAX9/mlKz3/COH6oDTkulzi2aukN+SmevGrM5SHZOllQmo9yj7n7zYMSyy82gXGi4RKG5s9AOKmDQcBt+crhyqowQuYzAQEOdXDR/Xa2D02uHuHc2Qrji8KC881o8V9n7QwRUSsg1YPdUYnfshfdYnEt1UDnU76ZodqdmdhS0UbMnuyj69efLCoNnTyG8sNicJxmcGtpRopyom/QrMPmlRH6UQLiBdO+iNQbdI0c4omW1nCvmNixJYpiInG4t2kSC/7OAyifowQXbnYEuJ/MZg8yjF+GUH2XmsnuaYftpg+zAbknR1zccdnXXwWqCbaejaI7tpIazH5umIKeUrSlr5/jWoj8ge9/JZqoT4vsjPq6EWpDopUJ7V/PtlhfYwR3bdot3LEKRA8bpG9ahAsnZIli0QApqjEunKoN1LUZxVaI4LZJcN7CSFqizUtoWdFbClgq4c9LpleE/fgbpq4KY5vYYAezA7VqrYXFHR9boClIDPNNSyYUemC/C5hqwN4AJC/Led5lBVRxkryGyGHszp3nsuIasOfpRxCLVuyFTW3a6O5F7XpagahDzF0GkpBMGjsYBWEJuavw/0UvY/h3UIkxJiWwPO0zdpLMSmQphNACUh6pYS1j711kdJq7U/nsgaPEGlEAhtB0gx9FQO3ZVNQ7YTBKFDaqxzlMR2ZgjtGZjKnjX9Sdt9qeyPynN/9KZN80f+/o/a/qQAy4f//pcPLD/9t75UYPm3ALwD4C8B+M8B/DUA/2MI4b/8qff9acBSCKEA/L0Qwl/8Evb1S92y0xjZKwh69FINVSH95HAIjBgHFBcSZhIGz5RLd+EFfSCCrsiwqC56JTyGKpG+KkBXBB7pkjJO1dCH1hfduyL6ojqgD40wk4D8SgwSVBay9xMlTpkZ5U4GQ8UFanMYkN5FRkPwPvpetUffHdiHOPS+Jmkx9Ki1exiCH9JbSqZ62Whfo2Bz0GfYCKRLMZTKs74kno/oCczuuFjOL8iiVA/pdUs2PF4RayK6WcDsA8q2zHi3f+zQ498sQ8eQ/GkmAXqzqxzo5bq970t2nMDXR2Ko4kjWfZw7n7+L5F5fE+Jj6EMQfb3CruDexYVvH/TQ7HNB3gOuZLOb6EuDQWbaJy2qFkNIhYzTcYLIADMV8XohE6Hr6Fmr+Z6zOT1u+XWUL8bEymTDlFVd8Ry1+2QwpKNE1Bac0PayvtknHpuHXByaEfex3xcEDAEgxaVHN2GQgc9isIYGILlYZn/ljmnqJWgi+oVYl8PJra7ISqiaC/h+UZvfBlZrdCH2PsaKkC33tU9TtaWA3pIR6WZiAGTFtYdwwOaxjPUcYagvUe3O62VKAgNpgG7O8I6+Psfmuwl+X8thC4Hiisff+8HaiaSkcEMg0uyLoeoEAMbnDqaUaBZc/OqGjEe63u1H76HMVgHNnKyOdJQ39v5GaZgU2cwVdLvzidJLJ2KYBdCNBIpbT6Yj4WvYh3ToOgweOp/EbtX4XvOax53dBdT7EumGC3AR6FUc6j0EGZl2uqs0AeJ5v3HYHlG+lt9QxlheO1QHimmakZXUTRg8ZEHu/HvSAsnWMV00ykj7a7Cvzej7DPu+xT7dVMbFpxlJJNu+ViMMXY89gNM1f1Zck9lJNx7tVEE3nPDbTKC8tNic6OGzOr/lbXUTBnZIdYjdlwHtjMAoXXvKrZcO3SReAIK3CzLWtzQED91Usb6nrxkKPRsmBwaoWbBXsE867X+fbAmsNg81Ji8sugnftww+8ZBtQLtQSCoPHwEjEP2eEkjWDi6PC1kb0Cx09OCRwe2rP/jepa/tczUY/aH5HQPE/kN6w8yYbBd7GAWSTQR3E55nacJQgYIQOyYF+Djxd6qyqI8z6MojqSxsrgapqU9YKZIsDcwsgXBAdlmhOS6hGi4mu5mGrhxlrBXrR/TWkmmKxyAckN406BZZ/IyPSclNgK4tzJj9lrqKDJLAIPskix4gnYdL5CBR9YlAujLxPLHOhNe5GN4nycagm2fQtWVNR+sQEgkX+x37Gpg+fCUkCkGz0kM4VoXYklUj0jj6/LYWIZWs94g1KkOdivHsdzQE8q7QrAyqd9URwlICKUIY/IVBCsjOwUwzqNZBNib2M3J/e7AVFHsafSKhVi1Crof9Fq2BG2esQ7GsHvGaktjkroErEkAKqA2Bn50VUJsOsmrhy4zpqkLwsUKAG6WxxsRxP4oEctPAzkuobYugFD2GLgy/C1myq6npLEIaexddGGSjwjn4nABHNl2U21LG2vsWQ8oPajJ1UXIZ9w0ARN1RUlrcY+9CQEgTyl5trATpaztkZPbShCxi/H3QBE+i9wre90Vat5OwasXbSLkDh32AjrG8XQ/C+v837fB8kDLe11ByGjsm76/nhRCfZwgBAr2m2QXsADumMXgM9SH3t77qpP998Du28X41iY+dlz07eQ8kDvvV3/4+M9k/5xdMhQXwi++x/AUAlgAghPhLAP5lUF/3v4UQfv2L3O+n1o2EEJwQwgshZiGE5c+4n1/qlpUdZqdL+CCglcONmEFNDLptgnTa4mC2wdmHh5idLjFPLJanBbq7HIcP73BzN4bvJMJDA79JYRL6GcweH7vLHcoPU1SnDFaRW4XH75/h0w+O0Z14oJPo9gBZS3T7DrKWCAsDcZNw8dIC3d6u6sLPLaojT3lmYYFlAggJMw6UjnogvdLoFi6WnNPo74461KUGAqV49QOGj9i5pdxyz6ObCYTMY+/RHW4/WSAkAWqjhu40WzAtUVQK9dcMcJsO7AccoBomL44/0ti8aWEWwOhTjfa0g75MMH//Grff30d40GD8WwVW71qkVwrrt1k1YiceoXRwlwnslOfi9Ovn+OzjI9x+XWL/d4D1VzyyKwU7DrClRzO1UBcpVBOTDtdkw7p9Hn+y5IdNt+fQLfj/bt9BLyVcztshKCQbYPUWwzb6MBsAMAuPcKVY9ZGH6L2UuHvfQ3SCQFkA4bSG+qSANPRsBtlLHAmmmwP6V82Uj9P7LtNrBTNnrUdzQICjag4E6uOA0UveV3ascegeGJQfcxpQzVi5oVrKJutD7nZfG+JPGsqYYwKfcCKG1wDZNWV2wgGqE2j3A14fcaGkKn6ZNocewgqUr1nLYUuP0XOF5dtkE2TH5002At0kDlVmUe5YAX0wiq6Bdh9A4LDFjR29vdcSYsH0RzkWMBMJl/P261Mutq2Pfa+SzydclHjOyfT4hOCyHx64wkMageoR603MCMgNUB0QDMgWCLGqQG8JpKqTyJzlUQKoCA5dvhsk+ZgEaScBXkt0C9725l2CKIa10GsZBNAuyPoJDyyf0nfXzgFIDnVUw9t0U/o8TXwtggS2j4HiXMBnAt2E7Fovz1s/1bHPk5JLMyEQXD8Fph8K1Ee9Z5UVRLtzxNTS9amkFPE2DH7YvjtWWD6ecPzblry+XRwecDAQX894fM0egUj5mlUXPlHopvTzpmuBdk8AkvUOLqaS0rMZGedzevjaOYcvXNRLdFPKMvs0T9UwIEaE3f5QakdP0mbBwcj6VCHZELBwAEEQ24xZAp9so6+xBLZaUeY35h/hFUZnHrYEbr+aIL/2lCw+UTAjfg4mFYFluuUAINkCLsoOq2MGvUgD5DcWzZ4cAH95Qa9TM1PAFEgq1o00e3z9ujnBcrql17u4CnA1hxMcECS7Gg8JtFP6GqUFNif0aquO56hKJYpLDmlUJ9CN5QBc21kP6gXWj1lREgQHJt3kfkomBxs+1lWYQnwO/AcRA3tWAWEiYQp+ZqVbj25EcLI9Uiiv+ooPAm9p2UmJnAOwfr9dpjA6p+xTN9H/OlbRGymxeUhfVnHrsXmoYtcp+xnNSCFdOVSnI6iGlSMq9qaaUkM6oN7XKC4t2oMULuX7WYQ4LMkLBMFwoL4T2CsqCbqxigMygezOoHqQonwNNAsNZQJ05WEyNQygqkPNoY/T8KlgFcqlRTchY5XdWtixIjjNBVQj0OxpSKuhGo+gBYQNaBcJko2DGemdRDXwuvYK0DVBs094/duRGs6ZtAG6cjCFGuSg2XUHM9bIX9dojjkF1ZXjYHFr0B4UyF5XMPMMqrLwZQJbcj/9LGWH50hD3KsFAXYDLQ6JErJysTOUPaABWgmoyqDbKxigYzy6OX2PZp5zyDJLUGw72EnG53Ee0BJ2lEA1KtYhJdCNg0skpJXw0xR6YxCUhNclunmKFKB/tHWwoyT6IvNBitrNU6RX1QAEm8Mcui453GgovxXWwyX55/o1WRcTh0NKIoxTqMhYdns50suarGQE/kEI+EJDbbuhUzO4OI2RgGgdkFFCDOvgJhlko4C6o4eyamCOZ0heLz8fmqMV/Y49WJQSfjZinYiSOxCmJJCnBHKRIRQAWUbvCaTjz0OiIbQCNhWQZ/x/z3JqBTQdH2/CChPhPKW6synQdgSkSTJ0X4YQIKQcwnyEEPd6KA2g43PHYwjG8Fz0ya3OQVhLKWz/XIkmcwkQYA64dScv7z2XwViI++fsD2EtQwifk+P+wm7/5Gren/v2I4TiFwKTn7v/F5TC/q+gifPXAWz7n4cQ/uYf9wm/zK34ysPw6D/76xAy4HC2wbLO0TYJzDrl4qxwCBUvVDkyCDcZ5GEDt0qB6OuDFSiOKnSdhlumg8/Pdwoyc+ynXGuE0iGfN2g2KdCyMwo6QNQKWHQIbT/5khBW0Kc0ZRdjUGTbQhKYsqqAIANCSp8ZfQAERbKWBDXxdnZK0KlqBgLJWjHaP/XDY8s2xsEnAUGzj9OVXKgLQz+eaBRCQkmSWmq4ESWhwogYTR5YAaACZCPhJuzpdLlHyD1EI4d+yOyaKap6S2Bg9uhrE0YweTUep11Y5C8TMkwzdiLaCasGZCVjbHwY2FppxOAX8/F8+MxDb1ScPBKE5FcS9YmF3igk690U3hZM9uz2uT86ykURFyKQGBI3veJzs68rQK8pUzRTz8cT4LlxDNVxZRgWNUAM+Ul2j61qyj1dHuhdqylTJiNMWW56F0MxBGWqrmDwiGoxpIOqhsdjpgwPsiP6GtNbObBfvNB6qRrBq8/I9PYhTcIRkHlF756wGFhY1XAx7DMypcKx9mDweWYBfQM2mRoxpFYKE9l1y7979tmM+djpLRlWVYshqbL3q3RzsvxD7H+sDrBjvibC8lwgxGFD7xPE7ph7htiW3BdpoyQ0ViC4Ij6f4GOq6Jlz2Y617mP4h7TUyJ53s8j+pzHIqSM7zUUsWe1uyoApSt128sh0xdsmGwyKgKB43xAZXlv2UkzeB+B+ZXcYqghUzQWyLSNwTONxR/YzaAw1Gr0iwidgMuUynhe/Y0p75s6nGNQbLkMEcdEXWEdWPspXi4sQQTt9ZHpLdk52gIoBNn3RfFBieF/ILqa4xtdVV7t+RURAIy2GdNShCgS8bTfjbbspZai23IUOSReQbMnISkvGM9kC7UwMPkEzIgPag+ceTLlsF67Th5rI+H7gtUQ225Rid+7uqzrifgrH47fZvfeD423716Af5rnI/PddlwjxuCN50E2ZlNqNY+hOyvum64B6T5KhjmmywkVQKcVwztk1SbDcv291G2KAz64btAfT7VQMaa99yJMIGO6f1DvVxAD+VQ+MsEtpVQLpxqPeY+hTz0734JmKC9Za6KgwMCVZ9t7niEAmuZvuPF8MRQKSjUOzR290rxxQ3W6NQsAQr6+adRu9BLkbSSR1GBJWEXiOhA+w2e52fZ2FNJFl7l/jqNRgai7rKlT08VGVwvdmnxZL3yv3of98kobPoWvPsCAb4FI5fEdJEwZ2V28dXCEHJrkHerJlUI/wvbqBLK9wvG+QDPDxmSTLG2KCraH3T5p4kQcMctRkbWALFWs+JCAI4nwiIWwY3s9DlYWLtRVaQBj2QQ7Hk6shgMgngv9OmcKarLoIbARE5yGth5mkrN6YJExbjQm0LlNQtYXP1cDI9onNQcTPGE/AK2wYmNX+8dU2rtWSXdqzMA4+1UMSK4N14hdQZNWYQOsZKNQ5Hm/rILxnIFBnB6bTZ+yzFm3Mf8gSyIqJusI4dlfWMSgogq6gJRnVWN0RMg3R0sc5MJI949l7F++lpQ7gMIIs0ZldOM8gY42MoHWAMUASJWz9el4IylETzX33kQ0Vgs9/bxuksPcZSSF350srBOsGlrsHogNIvs9eBj+Ays/JWiPLG+6FHg3bH6eXMrKafyoYy3/v58BY/s0vVQr7vwP4y/8khOJP4L1/4va/APiPAPxfAH7z3p//XzcpAnwQSBKHZZ3De4FvP3mOZNoinbfIyw7ZXg1hBMaTBpgZeCsBGSASj2TaQowsmm06PKZvFb7y8BJio6ETh8XBmv14IqC5zQEjgcRDzTuI3NGnuU4gEg/IgFB4hIMObmEAHZA+3iJkHuqkQsg8wnELXzpAAcmdgis9fEHvXMhdBFqBQSkLC6QEg+GIOfTCARgzIlscNQPAUMc1F2meQDLoEKssHIQOCFmU6JQWkAHIPIKI4QYZfaI+8wgjR7ntSsHuG4Q0AFbwbwGEcYwPP2ogOwGzcMheawTNLzg7ddz/uUPxjB927ZOWnsuxQ3KrAIfYARcIblL698y+hbACeiPgJ47HE29n546LFxnQLjxZZAPUb7WwRUA38wQWKSCsQHIjYSce7kELVxBk2Am75rzuE3LjF75DBDMewglkN5IVG7lHfk0ZslcAJEGmWTgIS2BIYEM2E55dg64IAxNnnzZDr19z5NDNfey9C3EAseuO80mUAldc4NQP47luCZBMPEZdxdTiPC4+dfRTzqLXLu/BC32L+UVM+qwEVE1Q2c0DulmUGUruK88vWTIXr4me7VUN/9hJlLhO6DPsgXZQAd3cw4wjQKt5Tl0e0B542ElcyJZ8PbuFo4+1BeB5TD3QdhnQzXm9srIEQ5dd9aCX1GAI3fBpiFJpAu0gOcgxI3YGCkfw2gNrsss9AIsLmXgOfErQXB+GgYV18WeU2obYY8j7t3thCNXpq0t6CbIZBbRzPqauKPvtayqkJYhwRYCuApJVQPWAAM6WiFUpGOS2tsQArHvwzX5AxMRTpgW7jMy27i0ogT83I57DoPh7aeK+BCC786gPeY0EwTRe2ZEl5P24yDMTApFuCmxOCfxcBL5ATNaNKb5mxMcRIcp3VWSqVc9g7xKJee7JNNtCDEDsfn1Gs0dmSgTuv8uZuGxHBON9BYmMwM2MKc22OUHY9mS3Ly7jQr5dUN5uRgSoLhNDArE0PD7h+NrTw4qBzXTZrh+yf00p548hPVlcqLvoC+w4QNs+FPG6YnhNDygo22QwTFARwGc7H6cZ8XFdGqsv4rF0E/qL7YheQq/4GtVHcbCjgWZB+beMATtmItDsyeH1TzdMz21nUe5vwk56WxP8ujQOZWLlh0+ixaCNwC0XQ++gy8Xw/z4gx8VrlWykgBnRYyhjthppGYbD2Ly3XYgI5BgGIzyY8it3VTDdSKAbSbRTOcjS2ylRXL+v1X5kLxUfrxtJ2FwOQwYInlebxWNwrBiRluypqh2ExQAgg/z/2HuzGNuyPL3rt9ba45ljvnGHzLyZWVnz0O6mjWcLJARCxuYBCdkC84AsEEJgN288AY/wApYQD0ayJQubNwQyoy1hWnQ3arurq7qqq7KycrjzvTGcOOOe11o8/NfecbNcTWU32ZLd8pZCEXHinH3WWXufE/tb3wTlgQTz9GxuXz3iTQ/0laTmTkQG2o5Cz2A4blIHwiCZ7jscq4WhmYegnJHG1Fbk/SNNlxtsomlHkgiLlzRbmwr72TN+7Tiiy01YgAhAvLV0k7D47SHeNHRZAIutJMuqLjBVnXhFXZD3trNkuK0+jGXhTjMwfxglMtzO0c4SScY1GjuKqA9TtHW0M2FFVevosjAOF0JllEJ3ThJhrShLXCYgyGYRNtYibU4MNg/pstbjUrlNtxaXGtpJjMtidKj9cLHBpZFIg03omowNNjXYSTKASl22Q1UIgdnEKLpp6IjsHN4Y7FQaA4aEWiXg0mstlSR5LKC26VC2vz29le+C/NyDxkjGQxILwHQCQv0okzFERu6TxLfSWISp7EGnn47wswl+ksvYR5n4K7W69TxqFeS4/ap4YEitFd/mKEeNc9R4hBplqMlYvmuNMlr20wPHfh9Kifw2ANjheYwRVvMnvZLGCIP5k4mww/jUP3r7T9t+mlT3n26/X9sO+C2l1H+rlPqv+q/P8sDPxFgCKKVy4A3v/fv/Pwb6uW4HXzr1X/2r/xb3Jyu+MLngph3xtFgwTyqe7+ecjTb8cHlGZzVvH1zz4fKYP3L3E3ZdQqwcWnlelDMO0z2/9ugt3j69JjGWy2JMFnW01mC94uFsSeMMz3ZztPJc3ky5e7Rm3ySMk4bUdNxUOdYpzqdbfvjsDmdH6/56izTq0Mrz6OKQo8WOWDvWZcbxZE9rDc+vFjy8c8XVbkwcWaxTGO25Xk44Pd4wilvqLiLSDqU8z67nHM33FHXCvfmaXZOy3I8w2nE82fNsOed4tuf5xYJs1HAwKajaiMNRifMK6zSxscTaopVnVeWUbYRWME4aXq2nWKuYTypMeE6tJE1xuRtRvRiT3imodglx3qIUvHF8w7LIubmZcHy0ZV8lTPOaV6/mqH3E3XcvGccNmyalqBM2V2MOTrfs9hlJ2qG1oyxS3jhd8ujikCTpQkWMYTop0dqxvJoynlfsl7k8tkhxTmOLiHxRobWnLBJm05LIOJY3IgGZTCt2uwwTWdqbDIwnmrTYm5TJvQ1GedY3Y3yjGR0VjNKW1WaE0g7bGlwVMT3ZUVUx7TpleiY/T0Y1u31GljdsX07BeHTWwaX0kcZv7qmfj4nOCozxRJFlv87xpRF2OLfo2OKXKX7aobQnm9RUL8fCLmtQiSVKO2xn4DLFjS04BcajSk10UmFf5nBc460mn1ZUZYLbR2BFGqvHLSa2+EdjusOOZFbTbBNUZcju7NHa031vRn0iV8jxoqZbZviRaOdUGRYDph1qJ52lam/QRzXuJiV/ZijebEkWNfbpCDu1mHUkiyQhEVc3StKJawPao0cdbhuTXBnir27YX47ASwgWgDqv6DYJRA5VGNkPoJrw90PpVtWlwU07cEqY+tzBtJP/47VBdUqqhTJZ6Ki/XOKdIvk4w+ae7qhFFUbmdN7inbxeXSrinR5AsBpZzMuEbmGJV0aY2xi6icOnjuxZTHUm3aa6g+a8JbqK6VNdTSUBPFGh0O2tZ9qG6iFgqMMBYd11pXGLjuRZLImxoXS+OrP42Il6QSHnUWEYPdcUd8Un3e9PwNkt062syMRlseJWlmwqRXWvxWwM2aVm/6Z0slZ3OpKlkQWPvRp84T0D4yIB9fFOfOZ25GkXEr6lvEiYu0nwo5+IxBzk+dpDy+hRJL7yUjpG06Ww9PFGgqF0JzU3Oshxy3NLemVwIU23ncs8Jiu5f/5KU506khstv78UX316owK7HBaVrtXgIa8Pxf/eZeJ3ro4FIDUzeV09425qmcfqNPjl41tm2GsBTvE+jPNMFuJGz9Xg/e2ZYq8V+wee8RNJ3U1uRDafLUUm3SykR7ZXAUjaJ6JcCayZ3CYBRv3cqLBA07PffXhSspXHulB91DPGpr5l7HUrTDGEfcSfZp3bmdw/3njiAnb3BBTqWpjWeiGMeVSIZ3Z3TzpCs5VUyNgs9KHOZL7SlSyC6Vb2W55IIJPXkpRrKlnIMBVDpUzPHPY1T8lGwLkkH98yzfFePM69ikGqSgj5B5IoG1WyyOGj27mR2g9ZCOhGISW3Ds8R5hMljO/4paU4M4NX26a3Kpa4YGB6u1QN8l4b30qyo0rAe5fLvm0inZztWAdmWRgh3XqKE012I/LpZqrD8fOY8J42jaTp9ox636HZe5KVh/yyG/osm5kw4iJplhqV3rOsGwmmcpH4U5u5ISrd0DdpU5FPj64ktEeYTRmHJK8KM9dODMnW3vZdNl4YaiQZN9pbmoVUoOzvpORXLfVBRLy1ci4uIum6zPTgUTalpMSa2tHMItJVi02F2fWqB9YOXduhmuX1hNt2GpEtmyCz9kNtC0C079Cdo5vEt4xqZnrhDrpxQ51HX3Giazt4Y/vgJh3YXdXKB7lqxZs6sJowAF3dhBTeqsFliYDR2AysMUFlpcsWn0SofYVPE3RRCbDU6taX2vdIJlLJMrCSoeMS7yWBtmpRbTcwoK/7Pj/FQsJtLUnvsez33deHvF5ZEsfCIvYey7D11SRKKWErQwfm65v/SdbzZ23e/YFgLO/9R58/Y/nxf/C5MpZ/8afd7r3/Gz/zsZ9RCvtngP8CSLz3D5VS3wL+U+/9v/K7HeznuX3pG6n/S//9n+T7u3ss4oJNl/Fzk8d8WJ0SK8v/c/0W56MNzivOszWPikM6Z3i0OuCdwys+Xh3iveKN+Yo/cvgRf/ujn+cvf/HvUriUX1m/w3m24eP9ERfFlKqLeGu+5OdmT/j25gF3sg3fX53zYj3j/mLFHz76hMflIZ9sD4m0o7WGOmjzT0Z7VlVObCzfOnjKi2pO4wyJtrwoZtwbr/lkc8ifvf8dfrg75+l+wUFW8Mn6kMO84O3JNb/y4i3++fs/4lcvHvLvPvz7/Cff/pd5+/Sa62JMGnXM0ooXmxmLUcnL1Yyfv/eExhm+/+oOv3jvMdtWvBWdMzzZzHkwW7NuMlLTUduIo2zPuskp25hJUvPjlyfMpwVfPLwU4FyPaJzhONuxrMe8NV7y5fFzfrC/y7ev7zGKWzSeP3XyAf/Ts6+xLTPO5xu+MLukdYZYW763POfJkyP+hW98n+t6zG98+Ca/8O4nfPf5Xc4PNjx6ccTR4Y6vHb+gthHn2ZpJVPM3f+sXeeNsydloy8v9jElSc12OmCU1N1XON4+fc1lNqGzEYQ1DiAAAIABJREFUKGr47pP72HXMe198TtnFXKwnnB9sOEgLHo6vuWwm/PIPv8Cf/cZ3+PXLN3hnfsVH62Penl/xm6/u8dbBDZtavEGPnhxzfLZhvcs4WeyYpxUfXR7RtRHjccUfu/cxf/fD9/jS+QWjqOFVMeXeeE3RxTxaH3A+3fL9j++iI8edkzWjuGVVyiLEyXhPbSOqLuLV40Oyo5LpqGKzz3jz6IaX2ykHo5JNldI5zVsHNyySgl979BZ3DrYcZgXf+fED/s1f+FX+h4+/weG44NnVgjRr2a9z/tWvf5u/8+FXOZrtibXjT519wP/85KsUtTDJPYC2jWFxuMN7RdXEnMx2OK/46uELnuwPeHxzwDSvuLieYSJLd5nzha8840fv32V6d8soaalbWUG1XpFGlnHScFPkaOU5nuz56uIFv3l9n+v9CK08m8sJ8aThF998xK9++BC3jxmd7Lm/WHOS7/iHzx4MCxcnix2rfc6d+ZblfsQ8r3j8ozOyO3tOZzsePTpBZx3eK/7Qw8f85uMHoDwHs4KmM1in6TpNFDms1SwmBVnU8cmHZwAc3F1jtGe5GqO0LKI4p/nGg6cUXYL1mo9fHTEa1ey3GSdHW9b7nK41OKtwreGLb73gRz+4z3tffkrRJjz58ITR2Z62NXzt7gs+uD6hrmLePF3y5HrBe2eX/OBXH9IdS/etyoW6UTcJ/qDBd5p43NBd5bz3lae82MzYrnMWB3tuLqco4wUERw6cwneK0UFJ+WKCPqwxRl7rfFqwWo3xTpGNG8ZZw9XTBXrc4vaxqC5azTffe8x3f/MhLBree/CKp6uFLFBcpfhxR7SMmX35mvVmjNZOFqjWiYx7H6HmDaNxzf7ZVNjGkcXknczZThZfissxyiqSk4J6nWE2hrtfe8Uobnj//XuS1h07fKdZHO/YfLTAzTp0YhlNavabDHYx0VFJd5UTHVXYV7kwisetpBCvNd3ciYLCKcZne/bXUiGgt4bkwZ728VgA98TKQtBVijqtUcrz9tkVH/z2PVnIcKIqaU/bQNeByizqJoHjGreNidfB33ivgueZKDHysFgQWFd9p6JbJZhCo+5W5KOa8ocL7J2G6EVCe9SRXAjAdlNLtIwwD3fUyxwdrBUuAGKOatxeaq/MpMUtU7JXJigxBCh1M0u0MrdM+tsbyn2KeZEOLKvX0J50xFfi81RvFLTrdOgcbuYOTmr8MsGUeqjE6SYOU97KR7uRqCVMJWBXFgNg9ExTngWrQRQk/pohGK28Z4k2IbjonYrsu7kA/yxUrRwE+beR+2YvjCxAhJA908Du3ZbsWUyyFZDYj6k8lcTueCfAuZ1LKm8fiBVvJHDO1JJu7WMYP1GUp0F+Hph0m8nzxbugDBiJ53/01NAcyOKHVwyJyTb0mHa5yObjffCLTuR35UQq7mJZPIj2iuxaXqPqYPvQM30k+9y+LYssqhN5/fYdx+ipFq8xwnr6SID+7g1FdsUgve/tGT249Sr4rit5/uxa2PM+zK5eSH+qS4TtzkLv8P5cMXnq2N8T//HopRtY3OJMixfdy9ybVhY9ykNJQR4/kyC23T1DdeI5+i0BePl1R3UYic+5EEl4fiWMeVSL1LrLYXwhYWJRGdj7WHz3kxd2YLvjvScuJGFXgtw8+zuG7EZ8ytnKDqy3KaViRXl5X3cjOYfjbYfuPPu7iYSYXXZSxbKQ6pR2LMA1XbW04+iWoQ7EWbJuQydpTH7VCFBshEFVzrN9kMo4t90APJVHUmzLFpcYumkiYUiTBNUKSNSNhc6hvKebp0SragggsqkRGXAkLGcPQAFhEkNvZ7/1wV26qFF1gx/nQX6rbuW0vUy3l7F6f1sTohSqam6BX/Qa6xiAprduYB1VkuCLQryZ1g2gUsWR1JLU9WsP9yLlhdvn+4ntH8EofxB6LP8JAJYASqkEeC/8+r73vv3/uv/wuM8ILP8h8M8B/+drRZnf895/7fc43s9lO/3Kkf+X/vqfZd1k3B+tyE3LL44/4rvlAz7cn/CHZo95Vi94Uh7w7ccP+Lk3njCOGvZdwpujJb+xfMC98YrvvLrHwagkNR0/f/iY82TNX/vgj7Ld5Pzb3/q/+d72Lqsmp3WGZzdzZqOK+9MVWnkOkoIf3NzhW0dP+V9//GX+zBe+x9/58KvcWWw5zne8N7ng/3j2JfK45emLQyaLgmlWczbasm9Tbqqcuo04GJU8fn7E/GDPF48v+GB5zJ+++2Ne1TO+f3mHNxc3fHB1zIODFR88P8XtI/7EN97nh8szEmOJjeWTp8c8uLtEK09tDa8u5zy8e8WrzZQkkhWeuo0oNhlx3mI7w2RSkScty/WYO4cbFlnJBxcnOKeYjSuqNuJ4sudiM8EYR13H4TPH0zYR755f8KOnZ5yfrrhaT+haw8nhlkg73pje8GsfPiR6mnL0cxd01rBcjclHDU1j0NpTb1NO7qxZria8e34hF+UfnzC9s6WuY9o64vx0xeXNlHad8vUvP+bpeo7Rnm2R4pzCO83p4YZ1kVMWCfdOVmyrFOcVo7Th8mbKeFRTlgnOK7R2tJsUjCcetbjnOZxXTMcVi1HJPKn4zo/eIJ1XNEXC6emaoonZXkxQjUYtGvJRw/4mZ3JYkEQdN9dTTGJxFxkuc5hZE9RdClsaYQ8VjI8LmsZgOyPAQHmU8SRJR1PHuE6hbhKSe3uqmwzVat587yWP3r9Dfr6juBqJPNoq6DSkFrWJpe/TKaazkt1OrnBMJB/S3UWOOmgY/VZO+S3R2dpGY5YxduyEad0b3Fx6WE3scK8y4nt7lILqOpfnDFLrxemW3fsHpO9u2F+Mpf8T8J1GlQY1a3C7GGIJuSK3YBXp85h24qWztTEiNy8VvLWnuwrxvIGBA8TbPO7QmwgUuLGVi+mbVJ6jiCASqTYueHJq6VHVWyNyasCszeALdbmHsYRn+VSCnED8vXZqUa2WOpe1plk4YQG/WuEbjdpH4m2upUKlPQx+D+Ml7OrwlpGTcB1Fe9ISXce4uPeheamaAezIDd7r9NJQnbcDC4nuNcbyzRTSnRntFc3CYzMnoVtTO0hR82cR5VsN8asYl8gDk7UwBvWJVBBJL6HHjRxmr8kupCpHOkI98UpARHXqMIWAiYHZTIU1rY9kXspTkffqVnzVppBjUB85knVYcVfyfDr4uHWrhtfUTRy6klCibiTy5cljxfZtqWaxuciaTSUVN+1MZNTZpbCQ0svpifeSEL17IKyf1yLHNrWiC+Dg9QTseuFDuJH4k72+TQefPIHt22CK3lcXWC2FAM0dlHckqbs/ztGewHBy69UzDExJfhlSshcCRuS1M1wQmkr+nl1J16nI4wUUmYpbVrkJDGTw+Mrc3p4jNpPzSzeKdCl+Y5sGD20pNTRwKxGNdyEQ7CYAo1TYmWQljF28E/aqOgrs5V7k281UwJJuxVtsM0JitICLeO8pTm9Z+CGltpA0b1OFwKIp5FciYW9m4vusjtTgLe4l08lW2M12FBJhPbQjAXQiOZcwomahRBbdQR1Yz97jLN5GYXeVDSoKG7ydlciO0TJPceGpDjXZMnRdhnCw3r/sjCJbOeqZYva44+YL8eBfFXYLtJV9xNvbDAHxb8qxkmRd9amE6GQvidrKCShzkbBH7ViTrSzNOAQp2eB13bsgDRZQZmMBeM1EDQqF3hOtW+kDxSOJz5dSPeLM7Viy6/D/IoCuqLAhoCjYIRovgVKegcHsAazuQopzEypWJnpIVM6uGwkzShT11GBa6em0qSK7aqmOYqJCEorjvaQyKwfZsqE8ToYEZ1M72qkhvRGQFxXCekY78ZsCQ7Jwz2B2IzOAOhM8n10mgUzaerrMEO86TNHQzrPXvJniL9WNwyXBh4q8X20srKnUFhmiohveg7rpJC0X0J2TsB/Ax8IQqk5AmxslwmpqDb0EOaTXqtYOSbh4P4QI6W2FT6VvvK888UpJam0PEF8Hi72v8ieSW9FqSLlVdSOA7idxQM96vub9vJXDvsY4OofvLKoP7Ok6SZx9Hfj1vtnXWcafxlr+rO21RNk/EIzlL/3lz32/H/+Hv/R5MpZ/GvgbwCfIf8AHwF/03v9fP/OxnxFY/pr3/p9VSn37NWD53c9SlPn7uZ1/9cD/pb/9J3l/d8ayFiZkEtfc1CPySOSj78yu+N7ynG2VksYdRju2VUqkHZvtiDfPrrkpcpTyVE1MGnfsipSvnL/iqhyzrxPWqxFR2pFnLWWZMJ2UFFXKm0dLnq3nVGXC22dXbJqU5WbMG8c3fPjshNGkRmvHdjkmHjXkWUtVxzTbhNFBSV3HJEmHtRqtPUp5uk6TZy2b5ZjF0Y7V9YQo63ABmESxpS1jfKvJD0qqfYJvNSfnay4vZijtmc5LrNOoYIAr9imu1SJD9KAnLa4xctFvwsp3p/GlkUCiyKEjj90H83ji8DZIBA9ltcluY0bHBW0T0ZYxyji81ZjUYhvNeF5RPJsQnZQAIkFVBIOHR8dyQSxyfEe7Szi5u+Lqw0MJC0qCZ7U2IofcC8jAKQlVKiQpNz8qaZuIrpTfVWVITgvap2P8YYvvu0tDoJI+aEjSlnKdiSRzLf8E3KwjGTe0r3K5EArSS6IgSS0kQMinwoSQOOKLeEjEpdGYrcHOO7BKmJxNNAQfmVKHUBwkqAlQrYQh9cFNeAEQ7qTBV3Jlqis9ACJdKlwWwEu/IJk41DYcJ8/ALPTAAqcYP9EU9wUodHOH2Wpc4vHHDeZ5Chrc3QouUkkULrX4FJ3C5pJs3F/QuUwAiR07zE6jrLApLnMkSyM+qiQAiVhem9kH6WIZgEqhBj+nrhm8f9LNGLynQUbZ+0X7gBMf2A+bCUjRNbQLCYYCbmt+/G1dTr+5RJ5bWyXdrblchPcSNhdBvJe/NYd2CI2S+iJPH1/vDdjA1PQXFF4zyC37+3bB26sbBtmfCuPKriTcySUh+ElDN3YSprO/TS3WjQQZmUoFL7Q8n25v/Xe9t6w5sKRLI3JPJ3NhahlbN/Ghh1TGn6ylMkh1ISE17S/AufWWBgZGhXH3fX7SJSpAOdqr8JpkPvr993VIquM2VEmHCqAw5r5SqAdJ/YV5O/YkGzX4MLWFaCe1SfFOQEfv9zOtfI9KYU5sfusf7S+ue49uP3fK9ZU9kKwEXHnFEARlShlPX9nUy35tFvbdCaCLSgbmx9T9+Stjzi8DWzW9lUj2nkZvgrwzU2F/PoCvW3arP+/7io7eY4xn8JMKKBBA24dR9edoN+5DwcLDTD8nInkVoCD31VZ80y4R0OQSQrWKeFXR/Zz6cA6owYPZy621lcCiPpirDy2SUJ7XwE15G1zVh3/1dUIuEqloX6GTbiQoqe9C7cOC+gTo1wOTejmyVLfIfvqtD4Lp66G6XKS4NhGQ1gctyftTHtdMldwnFUmpcv39ZWw2kdt7cNiHKplaHv+p1+OCJLbxtwsEqbBnfS2Qi9QQdKbDceo/400tz9MHNg1sVHjPgBwTZf3gTY33jm6kB1Bmk5DWnUm1jg7MmM3Cwlrt6UKQUX9c+/3rtp8HN5ynt0mr8liXCkD3g9dTqm5MLaFDzUyAoO483ciEz4IACD1DmJIN3tdo39GNJLXXR5+WSCqHVMX0oT7W0eURppJrCq8UpuqweTTMj1SxSEJutO+GihndOnTd4dIgB1UKl+ihLqaXsvreSxiAma7aW1AYEmh13Q2dljgg1AWp1g639xJY+VnYRZQa9oP3Q+IsRuO1HoDjADZB/Ip9oFB/PLJEuipBui6tkwqVIJtVVSNpsVrfBgD1EtrXt9dBX6gx4fUk1v62HtAGcCn3c/i6EW/ma/JW3/wOAPZ3et6fsnnvhwRa/xrr+bvd/imw/GxbIBT/fG9/VEq9B/wt7/3P/6zHflYn7PeVUn8eMEqpLyil/irwK7/nEX9O27rJ+PWbN9m0GbsmZRQ1dE7TOc3T9Zw06vjfP/gSL67nVE3McjPCecXRuADgcLHDOk3dRmx++4jT2Y6bp3OUglfFhOevFlRNzDv3L8mylrqJ0MaxfLLAe3gVpIoe+NGjO1xcz5hPSj55dcTd0xV1FTPLanRsMcazfTaj2aScnK9FqlZGNE1Es0qJIkt5neOswQOjeUndRujE0lURrja4XSxeu3UMxpOnDfmkJhp1XF1PRRIXUnDLfcL+8QxrNa4KZvnMEs0bkX7FjvSoZHKyF/+kln9kUd7hK4NtJKQomdewjVABzLjLDHeZoTJLcTmmqyNMJh84JrWYjzOwirqKJZDo0ZiTxU4A7ExWsZJnMVyl5KMGnotMLp2LBNRPO/Te4EuDfpWiGk2UWtSiIXmeQK3hKsWMxf9XVzH2OsVkFlUbdK2obzKRcVW9EQxhzxwkP8wp1xnKeOJXCW5s0ZVCbQUgm+NaAlpaMVdEVzG0iuiowsee0ScxZt5AK4wKHlQk4EtbSC4iJh9FqGWMzx2jj2MJPPLCuLk0VNA4CQcyhSZZGmENa42dOEY/yIivI0aPI0wp0imz08RbLWDOKZJrQ/ZCVi9nH+pB8tQupGqEEIAiKb3yffJY9tPNLW7kyN7P6GZWAqRqg+4UPnF4LbUqOIi2Gje2ZFeaZKmJV0aYraUZgEX+UhNtDC6GZKNAe+KNJtppspcRupXQo3gtYCEqBSThIL+U0CY7ciL72irSG0U3sXQT8WjGGzUAt2QlvkcIF7WxLAK42BOvtbAFrbplCYwk3uoW0isdwnA8k8dIynEIIsmuQjBM5tGdQrWa5EY6XU1NSNYVj2FUKEbP9AA6dCcSP1OHNFkrc56sROoXlbIf3SrincxDshZQZypFs3DS2dkKMycXzXJB2s6FHUtXMH6qyC+klkdAu9yvnw/dqKFzNlkJYOxZs37udSvsYDuG/JVI6MRLCH04ULyXi/++wkc5YQB7cBMVMH4q4zUhxCi9ViRbgjfOE+17QOkDWCbU3DCAq/09R7qSdFcVAHt26YMPVRg+l8gCg2n8AAqySx8WAW7/F+QXbqhegQD+M8iWMu5k7Ydj1eUCxNKlMEMQgodKWHzghv5RZ7jtQ3UC+Ewt58v4uTBzvVSxZzaza092Lcm03Vhkl/mFJ1164p1n+sSRXUkNjSTzElhkYd3Slbzmwx+0Q5puvAsBTzsvHsDKk66cVIaE5FibQnMA0d4P4Cze+sGPl107THW7vx609p206dbJ63GeZCMLQ7qT+Yk3PoTvhLqf0gcAosiv3VBnorwnu5FxFWdyDLOV7Fe3nnTdz0tgDh1kN47sRtQQ8U5AngtexPJI4xKYPrNEJSG9OoRVGam8eX3xwCbCZPYAyiZKvIYhOVU3YXEqSPZ091qwkhY/ZR8WlV850rUl3bjQjyySyvSmGzySysH4RTsECfUALi5EMppsHTaW4CITZKlxIcdk9nEpwL4PZFGSMAtyrmfXcvyzq1ZknVetAPBYwo+GFFcvybXxzhJVjmRjSbaWLldDQm1fx2IaR355CyJsqsS/WEtQULR3xBuRhkoQVgDZM01USLpqO5LwqmhvB++ksO7CrCbrlnjX4SJFei0X//GmCWmyWoKJOk/+osQlmvRSVj5MLcwgHqKyoz6KiXfdAJSVlw5Rua8VQNkKeI1Wlfzvm0ToxhHtmk8DSufDgonBNG7oMo12TUi2NZh1RbTuV2HA7BtJus0jXBz8mlUrLKR1EtITfJQulZ5Rl4dAn86hrABTXTTia7QeXbYCSJ0balOwXjo0vUdVASjGBh9HuDQWuWtgH/WmgM4KeAzprj5LII7wqTy3T2K5TalboNaDuygAuyHd1QmojCP50iEwKInla3jsawE8fWAPSN0IDIzlsE/n8G03sJDe2luZa7+v17+Hr+H+ofrkJ7+GnszfTYrsP66b/336+ny3+PVMHe/9j4D4szzwZ/ZYhu3fB/5joAb+FvC/Af/Z73KQn/vmnOa3P7mLSSz3jld8/+U5TRWR5i1da3ihvTBxlYEJOCceMus0TRexKwz7OJHwkrnlk0cnHD5YsVqPefVqgW81tYKPXhzjrWK+KNjucjCepkho64jtPsNbhck6/MuMclzTlRHPHh1hJh0vr+cih7TCmpF7rq6mAg5jjx55XK0pywSswlaG7X5yy0gFwKdiJ0rEVkv/YKpZXszEh1VEqMiTTBqaImZzNSYadTgL1S4NK48KnVi8V7jcobWnKWPqrfxdlQbdKKKDCldm6HWEO69olhl60eCdEt9SBPmTmErFjF4YigegXiak7+yoLnPc3KEiT7eV889UitXfvwPvNsKaKmgPHbpS7Jc5SQXbl1OitaGbWsy8xewVzcjjUocuNF0qQTSqUxB7qMGuY6KdQa0kTr31ibxOp1CZxVVRqHLRuIkleRFLX2LihWkcSXdmu1B0py3Jixh9ndIcOEiFlXNdSG3cGlqXghfmx64SVKtxE/GedYkhfymdmd1ULhYnjzTb9zzt3BNdxtiJQ5VawMMuyMScyC3jnSZaRtjcC/DLGSo++gTIPiDB1ArThPvI+ogwT0ER6XRg9baaZC0MWjMX6aSPQoR9p4j2mm7ar37C/DsJ7QTSZUxx7hj69eaeeCngsJ3KnPnXktl85ImXivoAbO5IVgaz16HLUNiGqBBWs5t40itNcxDO6SBPpFMkSwHMURH8UiEdmCCxU07hUvBa+imLc7lYTJcK3RhcJBf7+/tqSN7tA01MBc0Cxs881cltlU+yElmpqSS4xabCqppKwHpfiVIfyZjSrTBi8UYuEHUrCal25Jk8EpmfbiG7kG7MqARr1ZCo28wFbEV7SSTNLzy7t2D6cajFQcCgiyS4RDfAOMxzkPMBUi+kZK4kJVSovulHmvKOJwq9kkONAnIs491r7EMhYMRUkF+EJEsjfxu/8Gzfkg8gUwpgFG+RMD2SSKvIL+SCvpkJsT+k7jZqCJLJLnt2T2SVuBBisxfGoMsFwKQ3kvzaA6VehokTH5iykF35IWlWgKCnym/nQXk5B3b3FaOXns1DkQXibue9G8m48mtHdailXqMVBlGYGpnj/CL0hWphpnw4X+PCsz/XAwPY++ii0mM9Q6Jt769MN+I5NC1UCwE6yd5TtyFcJSTT9j2hugU3EzniwEYlQRLrCTUXwoxFpYSkZFcCbJKNMILKe7JlmPNI6l/64+eULJLEex/knMJqKSsARD6XBNx0qSKqgcDUxjsBnMnKsT8TX1t1oEXCupDnjwtP4iRFM9m5gY3swV5UyoJStrLUM7lgN60j3bjwnvLsz0LISynnWv/418NuTCPHvFckWIkQIN5DurWUh2Y4XhCqWHaeeq5J14547/AREnrjw1yF+pBo76iODFElAEo5jTMCor0RUJlddxRnsTCCQSoqPk+Z8x4kojzx1tKNdVAgeJLW0xwkZFct7cyEc0UYQRcHqfTYYBNF4oUZdJEiXXU0s2gAlfHWYkOtig9ptsoDVryHpnTUh9IHml3UtHPpoOwmBrN32FzLQsK6wUcpUWmJdg3aJrhYE+1aulFEduWGhFgw8r4E0mVLNzIkNzV2FNPlkiSrnBcJaWAUu0lMvO2GoBwfUkDlvHOBoWzpxobsQkBiHGnwnqi0IfX6tk9VWUl81ZUlbgWkxUWHMzowfjL3ugkS084Ff2lMtGlRuVSIuES6PXuW2gfGMNq3Ij8FTNHisl6eGv5v1xafRwKGshizbwQcdQrVvCYfdSGER2tUKfdRNoyxcwIYIzOks+KCVJYw7jowiqH3cgi+cU5kqn3oTttJGmtR4NNEwGrdCBDLElRRyc+5vEnUZn+7v9fDeuBToM07N0hlfd2gUpEa+KoG78LtNXQd3kcCKq39VPKrD8FBA1sZQKe8aX+CofTuU9cW/W3DpvRP9WD+E7v944+P/4FS6q8BfzP8/heAf/BZHviZU2GHB0hx5th7v/ldPfD3Ycveued/4b/5C7xczujqCN9qVGLRVwn+tMbfJPhMLkBVZvFlhGoUuta4LEgakX/YfXpln0rZV2f0XYntWDxIXoEfWaKbCBeLAV83Uj3Rpxv2rAEIoyP+IWgf1OiLNEi71FCT0F+I9TK4vrohvTK49LYHrZem9AEK2bUKfXmhg1L716LAFc2sX5kWyaUL7E03Em+UKfoIfnBp6BJsbysleone65K1buKHsIFu7sieG6ozS3ZhRNqXeZKlphtJEmTz9YL4B6OhqL2Z3aY5mgA4TK2GSP94B+2sZzc8eajK6HKo7lgmHxnqIwljaCciZWsOPi2pzC7lArKdyEVk7/GRXq8gYVTi/cqWtzULUSX30S3UBwwSQZtKOqSLGHx3LmaQ/fWdjvWhzL+pRP4YBe+U8kFeF06tdiyP7SWSfWJhD4K271rSSzMUmqc3Aoz63/suQG8gXQbfUs3g7+m7DeONeJZc6skuFeuvdkw/iERKVAvQSdbgUpGPeSMX1DaTBM/xMxlvN1ICyEo1sES9/K+Z90BA2JL6UJGsRbaogrQwvZE51a2MJyqCV2suwGz8tPdHifdMdeFC7zVpXd8b6WIZa3pDkJuK7LAdKapTmH4iwL4d33rEulzAaTOT+Y2K4OWqBdDUByrUz4i3rO+jK4+lm3T8rK8yYZAHuoiB1TGV+Ne6TObh9X5G0/hQT8HQedgnWCZbH3o1hdXIljInXgnQcbGcv8pLyma/5Vf+tjPRy377fkoVehvbKYFBDV2HTt7z3UgRb/3gF2snimQlxzzZyO02VWRLAV494EtX8lkiQCgwNI0fKl96L1qyllTQvjOy774DOQYgF9+vMzx9YqhpPMWZZnThKI81+aULckc1dEqmG8/2DU12KSAj2clc1IeK8QtHtdDkSxfSWoVF0a0ACtMK82cTNaSB2lQxe9JJXUXPlu0EHGgryZkQQl2C1zKqblMogcEj18tyvYZ8Kf936rmmmcocZDeeZiyf9aMry/a+IVvKeEzjKY+lO3F0aWkmGtMSKjDfjpTEAAAgAElEQVQEoNYzPby3XXjuyQthp3oJIwRAu3KUx4Z040K/oySH6s4PEs6eXZNE0cBSdVLpISyenDdx6TGVSCvbUJORboLHLJOU0qgSz1o9M3RZGFcmVSGm7kNWDMnG4mJFPTdkN5JG2kzEq9hOpHoj3lm6XBPtLd3YUM8EpEoHrxtqO+q5JlvaAejI4wzNTJPe2FspZ/Ak+kgNTNcA3LZO3oOpnM821cIClo7dvZh8acN5qolKR3VoGL1sBwBYnMaMX7WoLngOPUMSqrKS2Kq6W4moN4p429HOImHb9tLNGO866gMBYNr2ctGg1OjTTzthKHVlsXkk/sGdSEajQhJTnVEBKHfYWBPtO3yiaccR8b6T75s21HQI+9ZNYkzR0Y0jAfpVkLwqYRtdLpUfprLC2HWOdp4OSak2F6bQlC3tPBV2rha/n80MppKxRZt6AF3VcUZ2UdDNUumyjKRaxY6E64hWFe1hjm4cumjBKGweD4ms0b7FZhFm14h/0Xn0vqZbhKCuRsDLIB0NUlOXx+hdAxp8GqOaDh9qSQDMthLmT2thJCORu/rUoLeB0YyjgXVUdXcrWe1BVBwJExgZSW2tG3ndfWdlZFBljU9iSWjVSiSr/e9Gozo7yGFRCunDlL/3vw8eyn4LslScl8d5L/fr79N7IJNYxvfatb//HcDl4J10FvRPVIV03e3z9GDRmFv/Ywj+wVqRsfaA8zXJ608L5vmZ2x8Ej+WDB/7eX/l9kML+lc9VCpsC/x7wx8NNvwz81977n6lD/qwey/8O+HcAC/w6MAP+S+/9f/57HfTnsZ195dD/i3/9z/HB6kTCdPDkpuXDzTEAd8YbIuVIdccH6xP+8Mkn/MrFQ07yPZG2/ParO9w7WPNiPeOP3f+Iy2rC890cAKU8Xzl4xXU95uObQ6ZZTR63bOuU+9MVj9aHpFFH2UZksZzkR3lB2cVc7cZ0TnNvvuaDZ6fM5wWrmzHfeOsZP74+xmhHnrRkUcfVbswfufcJv/b8TR4sVrzaTUgiS1En3JluuSpG7MuUto4wsUVrz9Fsj/OK1W5EU0eYyDLOG6omRinPJK+p2ohZVuOB54+PyA4qmipiNKnZrcS0ko4bmirGRA6l5c3eljFff/sZP748pi5jlAZnFVFsiZOOcpvx8P4lHz054fB4S9NFwrYCUWRpLkckJwVNkaATWV06nO9Z73KaXZ+DD4cnmzBeKDYZ+bSiuB6hao05qul2MfGsHhYMousYe9bgW827b7/kw9++S3qnoH45YvHmis0uB69wNsiBjUfnnShCrlKmb64pq5jFtGS5HmPLiHjU0hYxqjLExyXWaniRYRdhad8qTh7csNrm2M7g9hFm0uEvU9RJjbtJJGim0aA90bSl28UQeWaHe7ZPZuKxTC1+F6FrSXicf+mazQ+OsHdquElI7+6xnRYp7nWMnXWYnREZ7NslXZ++uY5JzvfUmxRVGkmZdPCVP/oR33n/DRmz8hJgMw4re7VG9R7To1rY+W2M3pvgM/GYSouXctHgV5ICadMgd0u8BOU0BrOKJOwn+Pd0o7Bjx+Rsx/7xTPo5D610oJ620AbJVqvxqSV9Hsvqcy7+TG8genOH/9GE5m6L2ht86licbal+41AWXoIUU3UKO3FEG007D/5HQD0oiL43plkIa+dyCcKxwY/Y+y/jraa616J3RgJwDi3R1ghQnImPdvREFi2ivSRB7h6KTFq3EvTiMke0FYbB3mlQyziAUDUAPN3J4kVyowMIFfBlH1RwlYo/VoNPHdEyIioU7cyJr7KUyoz0RgfAKQE1LvWfkryaSqoFohCuE+0U7VQWqdKlSLRtH5SjJHDHJR68LD6NXil2DwJrHMApiAy5G/nhPdqNhWFu5158bcGrmNwIKK8PbkF0N/F0E0tyY0huQhdnL9Ft1VAhIv4oAZ7JVvpC+4qV8TNhenUn8zZ6qagXssAUb9UQRGOzsLA1vpXN2kykqs0cRi9CdUZYeEg2t/5LE8Btz3B1E8/8A9jfC7UgmsEHm14Lc9z3lnaZ/C0qZBGpX3hziYwrKiSQJqoIkmxPeXZ7jJQXgFofeEYv1BDgY8OChPRChgWYw1vvq2kYGNPyTObRZjK/+StFuhYA3IfoKA+TZ479uVzEu1gWktqJ7GPy1IfFOjlOPSDtPZkukWPqjTxf/kqRLaUTtsvVsJjTL4Z6rYaFz56BzpYSJlOeKZKNLDhUhwFk7gOjX9x6/+K9LPZ4LWOvZ4qoup1zF/egUOajPpB9qdDNCZDsZIFp8Ak6eXxUBvlz8PdGlac4DhUW4RJJWwHcxYkEXJnmNoAo3vnbRUIH+dJSHhiSfd/fKUDbK+gyTVR7dvc02bX45WwqCaYuhnqmmT3ppDbECIC3iSLZSZJpL8cdwL3pe0Y1Xa7Il9KpWR0Y4kJAdu/fVJ4hwEcWID1RJSFAzUQPLPHQzxmkrsIGO6qFkUWTXunR+0UzOb4iRxfPZHUYYZrgi0ylR3TyoqWeSzhPf070IT+mduGcUgKiDyWwpzo0TJ7VFGcp+WVDNzK3Ka5RkPM6j6kszUFCvBNwHJVWFnciHeqGGnyi8UrRTuTvfW2ICv2LygqDKKyyFnBtHc7o8NmmSVbBo9h7BBWIDFpLb6fzso9OFlZcbIR1fY19U9ZL5Yjn1l8ZOjmjbS2LakWDnWZSORIbVNlCr5gIwMsbg2o73ChFbwth/YyWgJ+ixmexMKGRuQV2wxic9F9aO4zN5yk0rfyu1CAJV3V7y16+7nPUWhJbjfgyewmtD3UjKo6FyYyMeCi1EQD6enWIDwyl1uKP7AIw7rdPAVrHp8KC+tt+YvsD4bF88MDf/8ufP7D86Jc+V2A5BirvvQ2/GyD13hc/67Gf1WP5lcBQ/jngfwEeAv/G73G8n9vmUfzyh+8CoJGagMf7A+6MNzxfzqi6mB+vjvn69Cmt09Qu4kuLCwAaF5HGHfOk5O2jaxoXcZCU3BlvsU5zNtqxtwmRtux2GRerCasyZ55WfLw6xDpF3UXsioyjvODN6Q2f3BwQa8tmm+O9IjGW2Uy8kuxifvjqlFHasLka01rNcb6jqSPeX50yyWrKLmZXZKTGsrkeo5RneTljlDWMpxXeaWbjigfTFdfrMV88vUBpz52DLWUdczTdk6cN+yrh7YMl96cr/ujpx+AU1TbF2yDlKCJM4hjnNXHakY8EwOkfjVH7COcVXzl7idIQJx1x2nFysKXcp4znJc+Wc6JXCV84vOKLxxfY2hDHliTp0Ac1B9MCXxlcYzg52HJ1NaVrJSxIFYaTszU311OqMuEbZ885Ot6SJa2sko4sk3HF+LigLSSlY3xYyoXLNiJbVHxhdgnzlqPpHh97TsZ7sqxFa4eJHKSSQurKCLuT0J/dLkMpScWdjitMZmnXKfGoJbnWeKfgRSbeQgUqlavduo0EVBZCS5jIoloBsKpVmMwO4KnbxujCYG4i8qTFh7RVpTzplcEfNtixZZbV4tVLOoi8KES8yKntRJ43vdKkS0mNHT0WPWxUKI5me0wunlMfiV9HKwfGE20MZmtg3oL2mJsINbKoRuHHHZNJRZK2kvDay4YPmhBqo/FeZLo2cwJqxxY/6jCxJbqUYmxdSWDP+P42rMor9k+noHwIcNHiUS0MqtPEVxFmp4OHNKQVWmH4o0LRvBjjI9ArSVzFKdY3Y1Qn0f4qJL7qFsxWLiJUK2O0uaNdp9iRx53WJDcapi3t1GFqSW31qSSv2kwkeL1EMV6ZIdgGkLlMhJ1uJ552Aj7umS8EVO5E7mwaSD9JUVYJGFt0uFjAgm4lYMk0Ur3QTcTjt1jsA/hUpFcGszZ0M0s3kovHwQyJAKm+m68biXrAHrZkl0beI0Yu6m0ibJ1LA5udW9JleD1pvw8JctK1JIW6JIS9bBXZlYBZEIDZjSQ0p7nfSLBR6NvUdbiotuIJ7YN4BNDJWKKdAtNfiIo82EfiL7W5H2ouXCIX+TaTC+3pI5Fm6lbRTkU2XJ66ART46Na/KgsNr/0DCKyKiwQw2ozBVytVGAI+RXrrhyAbCenx2JGApvJEFBNeS1CSC/7cdsqgYDC17LedeJpF2M/wmvygDtH2lkHuxgKoo70E4vTBLb3M3ffsY+KHsJ5k/emQmJ6xb2fh+ed2AKh9p6gwyDKmHsDszzU6MPwuCsFDVsBVdaSGapJ2JqCpT77tOzDR8nwukflzkbC7ycaTbOS+7VjRjuW1oUI6bJg7m0C2duSX8vhmqoYOT5FLy9NEpRzTvlvTGwbAI/7ncDHbs6mhrsNUAgptLsxrVAmo1F2Yj8AY2lzGqBwU51I50oyF6e63Hgj1ADbei3qp97S6SNQIvY+znhq6kaIJbHa6ccH7qMN72VMdynnUs+D9Vp2I/NbUMkYJoArhPJohxEcYYFECtCMBPv24ulyCf/q5iUtJr3Um1Io0DhfdSptdLF7oLteDNDnZueGzz6Zyv+pIAmLasSbeO1wiz9Mz3S4wvt6oIf0Y5DVs3xK5rmklOdYmativqSTxFc0Q3iNjldqQZiYTZBMtsvJd8C4GeW8PzHTjBjmsTfQAEqPK0o2jXkFKvOuETe1kEa+dhJqOUUwXGFGXmvB/22HKDrQiCsypDyyc7hy67IY0V5tHAgKdv5XFWocdib8RraWvcgCGagBsLhEpqEtCCKAxUi3Sy0NTAW8+0ngjtR/Ke9worIApJT+33W2KbEiNxQXwHKSqynn8KJNxxBE+T8V3GSSpPolFdjukrP7E9+GNoWWfPdvaA/QkQcXBT5omAioDiJQ6kujToBJ43UP5k5tvu+F+/c/Dl3W3X10nX+3vna38p9vvavt7QP7a7znwdz/LAz8rsIyVUjECLP/Hz9pl8vu9Oa/41htPeHt+BcAb+VI6F6sRf+jBUzqvmacVv7F5g1cfHjMxNZG2fLg8YlXl7IuUysZc7CcAfLA+4V+78w+oO8M7k0teFVMuiin/zMNHfOPecxZ5ybrO+NN3f4xSniTqGOc1mWlZJCVpZKlthNYe5xQaTxp3HIxLfGb50tkFNqSalnXCrk05PdhKBYbybKoU2xmeX8/55rtPOM23KO3xXnE4Kjk93DDPKr5/cYev3X3Bqs4ZjWremN7gnOJqM2ZXZNxdbPjx9TGd0/zyy3c4uLtGJ5bxvKT4eMbZ21fYbYz3iruHawCU8bz9Jx6RnhX88NkdLoopdhsTRRalPLsqxdeGh4dLZuOK7qTluy/u8qPrE2g1WjvuzjbcPVqz2o1ILg1fees5B1nJeFaRZi0Yz+TBhl2Z8q9/89fJRw3ffn6fq4sZnTWk8woUjNOGJOp4981X+LDv/Esr1EGD7TS//PRt3jy/pnMaXWgeXR+gtePO4Yb7Jzdkk4bkoArGL4Z/gM0+YbvOaa0hzRruvnWFfZUTf33N4XwP5xXFGx2Tg0IAavC1Hi72qMxC4njn9EpknZHDZY7ppITIkZ4VjE8KfOSZfeGGcSJym9nDFXHa0Rw58IrR2Z6jbI/++hpjPMlJwcG0YDoOIQazFpU6irdbbAbtJqG8b8nGDTzcU7URB/M9fpPw8FvPsFPLbz29RzJp6E4a7Mjh9xGHx1vG767JpxVubMEpOqsxxpGM5O2rLLgywo0s/kGJUp74vQ3qoKGbW/H31obpuMK/UUptQwf6sCE2FrdoGZ3v+IWf+zHMW7yG2RduRFZ90MCkpZs54o3i5HRDs/C0Jx0+dfjcUj9oMCcV7cwOn0R60uKdoj52xAcVdmqxRy3NsaU7aaX3UXn8xOLHluyopDmy+CJi/sdf4XcRLoLqfiNVIp0k27ZHHbOzHW5i6e7XLL55RX3eyYV76kB76qNwvtyt6HLPN7/0GDdyUpGSOuxpg6kV9f0Wm3mSh9tBHpm+saM495TnHT6zVMfCniYrYYPXW0FE44drqjca7NwSL+qQ4Ouoz8JYjhuaOwKOXerE9/huSf5RCt/cwKzFjh3dSMCIHTtcHIBJakWaDDBpsbkTFjeC5tDRjUXy3s485d2O4o6jut/Snsr8ukQunGcHhbAvx+2t1zfx2FQk+u3UUZx7mq8Usu8DJ8d83Mm4Uk87F9BcvNEN7GJ9KCC3mcvCRjPz7O5LGm47c9QHAkKiQlGfWpq5XEgX547yzNPOnSyClLK/5rST+pAjR3XiaRae6txSnirKM5Grdoct3UjCcpq5p7zjKE97Ob9i87UGr6E8t9jc0x52lO/UVKcWm0L5pYryxFOeeqr77S0bXCt27zXC1E4dysLuDU879uzvu+DZBZyMr506moU8dzuThYv6QC60XCQy7HYKuwcC3KsTN6Ty2iSk+J6EFN+NLAJ0E1k8uPmyHCObQHHeAzIo7knoTh9G1E4ltAkEoNULWWCoD2SubMKt7SAAo/74V0fCVO4eiJ+6mcvt9aEA2S4sNnglMnivYf3QDEywyDtFglwvpMKkOlTy87HI+SsRGrE/M7RTJawzwk66RJ5TB0m6SxSrL0J+JRfINhYJeT1XQfod5t+H1F8TPJA+AOqRVJxs3lZs35APn5svSADZ6j1NvdBDWisamoWA1uooSLdLTzdWtCNYv2VoR7fs7+ahYfI0LEgFcNnLlcfPpa5lf25EVm5FemwTRXUosuBmrCiPNcWppl4I8+diRXWgAsiTearninqq2N010ncY/KddKjJqFNRzg4sU1SKEGWUS/lPP5PX1+/Fakd4Iy6ysMKK7c5EhR6WjOJXX2MuY25GW+hMj4z54X/yh7UhLuBD8v+y9WaxtW37e9Rtj9nO1e+327HPOPbd3leMqx3ZMORCjCCmEiIQHhBR4hxfABEVAyANBClIQIuIFkFAeAEuAACHxEimQPGDhkMTYplzlsm/dW7c77e73Xu3sxxg8/Mec+1S5Uo1VFRyLKW2dvddZc812rTW+8XWyv2ONCxXR9r7KpJkGAygGAYJOgck0JtEUxxE21vJaYwmJK49jv56my2W7NpL9rvbF66o6S72IaOYh7TTExppmEkno0CyWPsdA0Y49kE2FtTSZyJLLgxgXCajscpEF2zzCpKGwn9bJa/rXamcJNgnRrUiDAbpZhksiVu+NcIGmOcgFVHnmU1mHGce4LBJvp1IS5OPBp9MixbajVMBt00kqbBRKaqyS+9GO+shuBVoNYBgtXlSsvQ/g6RnLSBhPEPayO5gIuOwZxF522y99UE4PCKNQ2FHn7n+MQWWZgErlWc3+ZwChEgykAo2Ko29/faVQUSjPgeF3Fejf80MQoML75/5jv7gfw8+Pdkmdc9thd+X3/Hs8f1h+UCnsLwF/Cfg68M8DbwD/nXPuF39fu/sjWuZfOHInf+3fYJQ2HI623JY5l9dTwrgjigyHkx2rMqWoJGhnNKkYpzW7OiYKDEWVoJQAN2sV1mjm0wJjFcvlyF8sPyscCYhY3owJ0g7basazkmKX4hzSa7jMpCLkckJ4G6LeKFDa0exiqeOoA1RiCGNDe5dAZggSIzJDn/5KK+EzOrJSCaIdWMVoVrI9GxPMWswmGp5j6oAw7TCdJowMXRsQJR1KOerbjGAizye2st83wrQEx+UgA3WJyDlVLmDCbUVKSigDu/nhluXZFF1o3KIlOI+xqUgDQVis8EFBsxX5q7OKMDZ0N6lI716EFG+1YKQ+w+YGVQa4zBBNGtpSklfRSEhR6affA0e4DDFHDc4ooouI9sR3H9QButKSPBt4NipwcjzTFr2MRD45MhA5CccZWVxsUa0GC+lFQPlmg96EA/shSaP3lR7hOsBkFhc5wk0gLGSraKdGfK2RnIPwTryLNnFkF5pqX6o2JADGDw58b55yIidspzLwCxoZKCsD0VpTLyR8phtJKmq4lnqQaKsltdQIE4UTiWNYiIQxudN0mcPkMnAPt9rXPSjqhR3+H15LV0yklzG7kgGmMFIO1TJE+ve1BN3IDcmlkvZ5X3zeh/30f/fSuGYu7KEy955dKVoXuWi8ht3jvjNR6j6cErDRJ+IKY3QPcMKthO7oTjzJ9UKkrOmFHiScPdMTVAz1Jb3Htk8x7X2xyrNMkhbqWVUnbM7opWJ3KtexD0Pq+/F6f3K9J8+TuoY+5ZMhsr/34OqG+8Ai5RkxK5JQ1fUMJQMboIy8dlj44JwbQIsE1YYivezPvfXgo52K7LGXb/Y1G+J1FS9lL+nr6z368JheYoqS/UjuRFoa1HJcIMfWB5X07JQNGVJljWfm+kFrvHLDoLrL8ddMttXLE62v3+iy+0CbwB9D63sWxc+rBuamHUkoTT0T6V965agOpX6mr45oZnKM0Q7xjO5pwsp5H6QP5qqdT82U9eKVJJrGazf4Q4d7JxOvaT1X3vPrBkZKAn7U8J6RvkRZLyzlfmwnUk5vAwF2fRhNcievGW/cwK7FGzf4nvtKjr6MvveCB00vgfVMkhZ5qOvrMAr5DNMdQzWJC+Q4op2EzrS5Giox6qkaGKHQV6447ffVp6AGtX/uzAdNRfJYO1LDOtFOznWyEoap99H2nvI+UKdLhGmUGg5hp7rUB1kpf88p78eeyj43YzUcf7y1tCPZZ+vPe7Lq/ZwaEymiUgbAwjqK1LRnBHsPY1ha6plUJSVrJ39PRdLaV7QoJ/vWJ8g6rTyDew+YAA9woJ7L3/HW+ntHD1UioWcY5VzKNRKpL4OvPCwk8TTamWE/e28peGa3snR5IP+OtL9G997YvsNTAtCM+HK1oss18VKSW7txQFjYQWocVoYuE7DkAgEHNtaEhRk8ovSvrWXCwCnpi9RdH3Ll7lNekRRXG2i6USCBOj45NWilMsRpRbgVH2W/2EDYSGEs/RislgCd+6oTeTxc1dg0xIUiV+3l4DbUON9BqWvpl1TW+f5IkaYGVSey1s4f72sSUWCoG+nDe3Tjpa3GoetWkmDBh+7IvzYV9lRZK2E+CEMJ8tggOX0t5KcHcC6OxJPZA7zX/ZG9XLVPWbVWPJdaw3frsvyObbgolOd7KewgV+2638NWOudQfcBPn+jqWVnXiSRYhYHUjLQNRPF9HYmRfXA+zIfeW+lTYb+vr/L1wJ7vsjjrRHL7+1z+wEhh/+0fgxT23/neUlilVAr8n0CChLf+L865//Af8tz/C/gl59z/4//+Y8B/7pz7499vP35QYPn6hh3CLwTOuf/g+678Y1ymP3Hs/uwv/ws82+wRBQZjNc/OFszmBe8srvno5pA/fvo5gXL8rd/4Mn/8y9/iG1cnlEVCGBmaKuQnHl3w4YtjfubN53zj7AF7k4Lzl3v8a//Er/I/fvqzWKtpapGHpmlLU4c8Przj6cU+Wlt+8vScz+4WfPHwgt8+f8A0r3g4XvHVzx8znlS0XUBVxEymJe/tX/FiM+fyakqUdnRNwHRa8s7imk9uDzBO0XUBTR1hqoCfe/9zfuv5I0mdjQxp2lIWCd06htgy39+y3aVDrk9Xi9z08NGSqg1Z5CWvbqeYsxybG3Teoc+kYiK+CWgn8s3jRp2kwu7XmHVMOG1I0pbdKpUakk2EXtTYu4T4sODB3pqnLyUp9/jBkovPF6hRh9uFqKGoXqSUBA69DXGhI9xo2rlBT1q++Oic3/nwEaPDgt1tBp0mP9zRfDxFvVGIlDhpKS5HIt00Cp112E5zeLTm6mwGWryaq/UI02iCWIC20o7oZUxz0AmIBJGfZJbkLKQ+Fdnt/GDL6vkMXSnsYcN4VrL7VDy2fW8j+7X4Us/GhNsA97DC3saoeUPwIqU9bKXnswzAKeKrgPRLS8oqwj0bwaMSe54KyMx8p+V+jblKpZfTKAHw2qE2Icp4EJpLIq5ySDdmqwkK6Y8kksEKjUY3ApJd5MTvaRThKqSbGvoC9nAZDF2DNjdoD9xtZohuQwllOmoIbiPsQQPL+F4i6T/jbSwVIs1CPJY2l3J7gMP3r7l4NSe6jGi9x9KkFhc7omVAdqHYvGWIl5pu5IbeRQKHapQAT8trfYEOXSm6uSHYBJI8uxbvoI0lmbY5bQmW4QBgnQKXyHpBKbUeveSy9yIG726xH409KFPUB4ag0MOAGysAVjfCYLlQJkX6GovqSMC5STzD86Am/jwVIG5lsqAbOV8d4BNnS0WXCbDIzxXbJwaXWPLPI5qZI70RJi3cKXr5q8kc6aX3KXop4uSpYvNEJjz6nst25AgrCdwymaOdyOQCVryDfR1Kfi4hSX2/Yy+7rPcE9Atjdr/9bmSJNnK+kxvxb+rWh3758+MU1AeW9FIPZfY29j497+er9yzxUkuY0773WDoPPmMBmD2IbqbCrKVXjupA3QMqJ+xceqUoHoh0dfLCsHwvoF44Rs8lxEsAgk+c3Qq7F22UJMGOGPyA9Z6TvtFYvJfliSM7U5QPHMm1TPYEpfdD1iLHza6luF7eED6EyEJ1ICBv98gw+VRuXglTg2Tp72XPAvYD8XglQVy9ZzUs5Vwkd15yGgmIaEd8W8hVULqBJZx+Lv5JmXBxw/u0mamh01N1MgGRXt9PLlQHMokReS+ibj2TmApLKhUxbgiTCiu5LvFKroVIY+Xa55dyTvrEWqnxYfCoJnfCZJpEwHgPEnuQXB76NNxA/JrVvkw+2UjW/fZeUzX0Oqa33jM4EUCdLN0g6exl0Lq9nxTo04J162jHfcWLGmpnwkImPKKtgNewFNCcrNzAwEalJOS6QCY1hP3sQ3IsXaqHZFEJ+RLpbH51P5nhlCIqJUgJ5DWbsSK9E9mqDdUQ4tTLYbURABVvLbqRSpBe5ttPPJlYyessBRyaSBHv7AAo+4Taak8CkGwgEx26tQMY7D2N7cR/ADtHWDmqPQlY0q38Lh7LftJOPi/6oCUTSxVLdRBJYm1hacbiA43vGtppJN2WqbCSyngAPw/JLhtM6qWikUwOBqUEN0VbCQzqstB3ZgpgDap78GESTVRIWJELFCbVsl5tJJioNhK+tLuXULpQD1JS3VlMGmLSgGjdiOx1FBFuW0nMHfkAACAASURBVFTdDuE+urX++0kAabOXEN9W4Bwmiwi3jbCC1tLOUqJ1LRUn6xqXvsayeWCptzWEgQBREMDrAWmfSqusHUCfTWP0psClCfQdmD4kiD7Ux18/+X4NUE07gOWh67KQfXZ5itqW9/v1OhbwPkYVRQIMwwDq5v7/XuuUVErh6kYYz9eTXntpLOC6DhWGAir7ZNfv8E/2r/Xdlu/EKX8oPJZ/4ccALP/d7wssFRK+uvUq1L8L/AXn3D/4Ls/9Y8D/BLzyDz0A/rxz7je/3378oJzy9rXfU+DPAB/8gOv+2JZHyR1/+fRvsbQJ/+nzP4NF8V//if+W/2P7RVZdhlaOeVRyEG34l77y68zCkixouarHpEHL882cx6M7Dt7dclbM+PLpKz64OmbvaMPTasEiL3lresOvfO0LTI631HVEFHdkYctPPjqjMQEfXh4RRR0/N3vKq+2MuyLjhZsTxobH8yWd1WzGCcZqVk1G0wV8+c2X7CUFv/LVL3L88JKvv3jIlx+95OV2RhcGLMYF1+sRX395ylvHN4zChlAbPrw+4p2TK95975oPVsdkYcunbcjjvSWrOqXpAlrjo9q15Z3pNV85/Jy/O38b6xQ3yzHtrOMXv/whzzYLrjYjlII8abhdjbBNwMGjJU0X8LMnL/haeEpnNZPjJVfLMW7W8JMn51ineevRFfvpjlWdMfmJM+ouxDjFcb5h1WTM4pKvfvyEJ4+ueXE9Z5zX7HYpqtVkecPntwumx1vaLkAnBhdZ3ju45utnY947uqGxAbsmpkhTTh/ccXU3YX++ZbnNiQLD3tGGug1ZbXL+6BvPOdtNATi/mmE3Ee3cMj2WntKu01KrYhRHXzkH4MXFHnUb4nKDa0KSvKUqY8KHBaYLyHP54NpejCllGgX3sCJJG4ogIooNzcyQTGuai5zjd264WY4JD1s2L6c8eveSq8AxG5VcVCF6HeJaRXa6pdwmpA+3VNuEeFrz/vEVn90uaPKQILDsjQtevVxg55Yw7dAXKaO3VkzSmqYLicOOsokYJw1nNwKEuzqAMiC5DmjfrljMdtzdTEhGDcdvb3h2vuD4cMXF5Qw17nAOHh8ueR4vCK9ilHbohwXBxyOaBy3J45KmDomTjt11zt7JmvU2w1YhTx5ds64Sls/m6EXNapeRzSu6q4h33zvjkxeH6MiSZQ27NIMvFARFQj0J0JGVMeNtLB7G1MmEhGeK9aJhf77l6vkeutCYiSGa1TSLYFAP8KBFv8pxRzV2G8lkhgFGHXbsMGUoXpZHJd0uwp522DIkANpHNWwj/tQ/8zX+1td+CldJBUw9FpBVTwwuNyQvYt78xad8utjHdAHh5ylu3NFVEfG7a+rPJrg6EBlubCF0dOMANWtQGrrLBHVc4YBuFTM63mHeBnc1kgHfz24IHBSPQtwypp3C/tt3lE1EFhjW4RwXeinttCP/szcsXy2g0TQj8e7a3GJqPQD0sFTU+yIrdrFFpx3qImH9noHMoO8i7LQjuo6EWe3Z5tAR/8Id1VcXtBOLOq7pupTwnS2NGQ/hM/FSUe870p9eUWwT1GVC8VZLfBVKz+V+Q/gyEalWAMooqoct9YHG5YZ2Iqx+tNbYUKSE9aGcv2AjzEX5UxVuFeOUI1r5ECajWP10A50mLEJe/mlLeKfpFi2FiYRdPmrJPo+p9y261R7UwO6JkZTqDNIb8YjaAKrTDvdTJd1lTnXomf5HDjvpiK4iorWi2pdO05uf74gvQvGy1sLgtRMrSgGlSG4DqkM33J7tfkdYRJhEwDcOTG5JL0P407fUH+0x+xYsf9KRXmjKE4uJNSaDZmbJLjTFk47sRUi98HLQn2qJX0WYFC6/4ojWMtCKNmrwakqgkngWswsBas1MmHdJVL2XtZbHEmoVL4W5b0cCaPrBqUkddicTMlUoSc89m97lsPPgpx0Lo5ude1Z6JOvGSwGPOFi/a5l9FEiQkmewy2M3qANuvwSLbzgq3/0ZlsLuJneONlWsf75i+hup+AbzQKS1nWN36plOC/HSCaHSCkB0yoPEPeUDnBT1gteS1OVaOSWA08SvJVAHXh7a+0VTmeQJSwkdipduAHbbhwHTp5Y2U5K4vLsHyef/pGL0wicN76DNtByXr2sR76r27KNMDAjQFeAb1FAcKcYv9ZBOnKwsTqmhhiraSVpv0+nBY7ydBkQ7qebZ+0gShpuxwobiaVTOkd0IM2wSAenxxjPLCYDCbYTR7jJNfez3p7lXFZT7wt7W0+g+5CcL7r2VSuS/LtA04xQbKtI7Q9BYikMNDhItIUX2QewnYyzNVP4vbQQo13sh0VaO0cSKSMmEQb0IfSeyhPy045BwZ6hmIeltSzsOUbHG+BTjoJb6k9eBbDvSpDctzsl91eWaoAro5uJVtIFPzM4i0FDtxUTrTjybmiGZ2aYR3aivMNG+CqrFZBEmk/Tdej8l3EoPpo0CSbPNYgGtkUaXHWaUoFsjCbmbWpJjkwibx+iyxWYhymWeOdZo00IWS5hPE/iUWgFtLgnQVYdNowGk6qKGzmD2JhIG1BkJ9OllrUrdhxA5B7knLV5LfEUpSae1XgartYBav55KPHhs7t1yr/dTKqUG9vY7gbD6PdUnrwUivQY4f9gmiz+Ii3L3qp5/lIuTk9fjucj//MP25C3gZxCF6r8IfOV7PPfblh+6bgToY2j/d+fcn/yhV/4RLqP3H7g/98t/jnWb8vntAudgf1xwtR5zON1yu8v5px59ylk547e/+Zhg3GHWEapVTN5YU35zjnlYYSsJhWlejMjfXGOMJgoN200qia6fz2HeEr5K6CZWpKMa9DpEHfoUMQvuLkZ1ivHbK9bXI5JpTduE2CJEZx3OaOK8ob7JiBcVxmhMEaIiC6sItWgYjSu26wzXaUnyjO/fXGHS0W5jYe7KEJUYgrNk6H/kVYp7UJF8kFE+aYlnNe2VeG9Vp7CpJbkMhHUyPk1xZIURbDXpfkl9keO08wllIvUMNoEkpdaavUcrCRV6OWd0tKPYJIQvE7qplcL34xL7MiPaaOp9Q3xSUF9nxLcB7cwSFJrgrS31MiU5i3Bf2KJ/d0y9b3CxI74JWPzsJRefHuAiKymxfh2tHc3TMTbywSKZJd6vaKsQ1pEkaI47aDT585DqSLxPNhFvkotFMusSK3UxsSNeCcNTnXSkh3L8qlHYeUf8MqKdinwV5fsOR7JOvW9IzwPKd2uiixjz0Kd+anCpYXy0o/nGjPZJzezXUspjR/2oRa+EZXOJwym/X5EjexXSTtwgLUzeXtPUEe4sHVJJlQF3XGPLkOg6FG9Xp0hutIRwbEWeVh36TrY3t5iPxyLFmlji4wL7yVjqXsYOm7hhYNocGrnesSW+DIfQl7DQZBeK3SMvTY2FmSvebEnOI/HsHcmgF7xctlaY3HoJsPcRTQ2qlW5N6z/KbATqYUny9VwYwJShmicslVznTlEdd4SrAOsBjgwUlJfLOenlW0u9Dj5UJrmVihTtGbZuIt64/FzTTJ0PqhF5WrSVvs6gkoFdM3c0+xZVC/PZzBz5K035wJJci1S5rxOKlyLzqx43TH4nZvumgQCylwHlQ4Oq/fWZSLVNM/H7uxJGM72RQJGeWe0HvGEhstZwy8AmtRMBHsmtJtowDML7Spt64chfCTsJIlstPXMnTKGXMnuWpk+dRUlwUXamqI4czb5h/HFIOxHmrd5zxEsZLFdHwjibkST0Jrf3A3L8ucQJKycSXdl+dWDJLjWhTz/tJda7U1/lE98zhcriAy7kWlaHcr2irSJ/pQYPYTuF7Fx8U9UBpDcM91Z5LGFKPRPdh/4oJyCkXsjxdCOpNNo9lGMs3pBKo3YE+bmwhPHqvm9S+jmF1WvHEhrkPBsXlF5+C+QXItUuTtSQ6Ko7YQDbqWX2kQcia0e9rxi/EHDSzIRVW/6k9JKGpQzme0lyuBMWs/SsbriDdGnF1+ag3pf7PdrKMabXcgy9eiFaS7BUM5Fe2PxCWLWemQxqqc7pQ3CqhaS69nUkfadkemvpErEvtCM1AEa8DD67sVRzNVTivN776QIoDzXRxrOIu/ue0F4GHa+FNcPB5g3N6JVPTa0c8U6AeHGgiXa+s9HLafvngB/4K+kyTVbiExxdGOppDzbuwXJ6Z9k+CEhvLbuHmtknUuPSJfdJxiaW+zEqha1NlnYIqOm7KJONYXccDEE7Ir1VXsVgacci0e2De0YXRoKTgGouDKELJMV1dN6yPY0YnXfc/kTE6MISb4wPtWGQ51Z7WiS8hcWkimjbs7q+Ysdvt1cH9JJZ3Vi0cazeShi/bO8rWFYGFyqquciDRxeGZqKJN1aCgaJ7uWgfqOMCRbxpqefRUONSz3y9TCKBPM6zvMr0qbSWeh4Nct+gEpYSIFpL/UlQSc+kai31IiG5rWmnMfGyHnoy20lEuOtQ1tF4ZtRpqUpxocZFGhtook0zdFdaX1HSp8W2s1QUBcuaIcF1VQqIqltcHNLME+K7Chf1oC0kuCtwWYwZxQSbyleCGKlGqTrMOCYomnvQ5/s5MQ5d1LgkxAUB2teeqLJPtbpnCF2aSA1JGqHXhYDBtpPakqIapKouiQUEdkbCeno5bA8atwWkiciJewmtUlDcs5YDHrBu8DXKBQlxG8EjKn4tCKiXv/ZAspcRW+clsxYV+NRYgCi6B4zDtr679PU7sUnPkMI//oxl+ujHw1h+8u99/1RYn+76m8C7wH/pnPtL/5Dnfd0592Wl1J8A/iPgrwN/xTn3le+3H79fYLkH/Lpz7t0feuUf4TL7wrH7yn/1r/D25IYXxZzOai62Yx5O14xDCer5jZdv0DYh//TbH/Orn7/N6WJNHjWkQcssrvjm3RFaOc4+OeT47Wt2dYwCZlnFi7MFAPsHG5ouIIk66jZk83xKsF8znxZM04pPnx8K4xM69mY7luscB0SRwVpFW0YcHK65W43Yn2+JA8PLyzlx2jHOau6+uYAHtXgns44o7QgCS3mdE81qujZgb2/L7fmMfFFQ3OYEoxZ7nRCfFJJ2upPRZTauKW5zVGQJz2N4c0dbxASJQQcGa0XfMxpVrO9yMB68OuTxWcluneJ2IaOTHbtlRjatKC9zwnkjXtRGJLcEjmAdoE9LdGCpNwn5vKTcpBwfLzk/n7O3v6X++/vUXyrQWlI8g3EnVR2pgVYT7VW0dykklnRSU91kHghadBEM4D1JG4p1KvtciUQSGHyow6IdhA5VBmBALRrUWYo5aFAb+RZ3mUUVAfqwwqxjlPd4Ou0Ipq3UllSSHDe8vvHbCJz8HjhUpWHaoq9izMwQrAMZEDcCfMOd9n40kaHGV+G3pQSa3JJcBdTHBu3lp/phiXue040lFVW33g/mJZ26k9oInMKMDelFKLLGSio5XGQJlzKj2+5JRKWqxY8a3wY0e1K1Ea0lJMdFbpBGyyDfyyGXAlh7Oaw+rjC3CeFWo2upycBBvBIGqh88ushPaynoZoboTio++rTTXqZlEpF1Nocin+0rO+K1ojySfWnHwtrV+9anawoAa8eybryR8I/e95ddKsoTkYLptk96dIRbLYzDvgz4o7UARoX3CnlvXu+P632xYeWDTBKp8uj7K4Naov0l8VTAWvGGIbrTBKWi2RMg1UwkabWXf+bnMnDvgVM79RMGCvHsBvc+1/48jZ4rqkORbvbdreXxfQVHn4Iar2H3UOSnkpzJ0O2qOkivZdvC/DiSpSK5FW+i+FrvWaR6T+Sn5bFMrGgfBtJMRUoaVAioyO+3n9xKD2k7YkhglX0S8DaAO38/mUSA0bAoH3gSMfjDTCLVFSBhKyhZp/drdhlDCivcyyH75/RVJn11xSDvXDJ4LKt9qdOwofR3Fkd6SCbt/YlDZUQI7VTOm6Styj71vbDRjqE39f5ecT5RVEBjVEgXZ3+svW+zrzJxSkJpxMvpGa/NfW1FdSCv03eWWt+R2vs+cd5P2XsVQ/FbOn8cwpC5wesbViL11J2wZiYROWr/mn2liLJSFeICxJvpE2hlG94vGwoAk1oJPwgM8B7M+7FGX/NhYgFfIN7LZiTMW1DLvaUspEsv1/Xe3f5zJll7ADVSwz70Xam91zVZOsJa6jr6c4HDV29IyI1uHau3AqKt89LUez9m3x0M/phLOb/JytCOPRPk+1JFJqqGkCATK5K1QdeOZhp4b6sd/LJ9JYiN1MADaCMVHiD7GlQWk4lXPmgszTQgqMQrndxJ92fv9e1rQkyixMeZ3EtndefQtcWkwSAXlnvL98n2lSWdlyG/9p7su0N1K0AwaCyBrzEBBrDYzEJ064g2ne8DlaTX3nspabLim9SdB95FS3mSEq86nBI21SmRsw6+Tr9uWJihi9QkevAIhrsOk4fo2gznVRmHTYQtdFqYOJvId4xu78FWD5CDopEAHC3MblCJtFRqn7w/0jiCXS1yV639vinp9ixbCDWqbmkPxoR3pfzdGvFeJ4FIX33nJSAAtekElEWhbK+XuHr2zmktnss4GvyTqqxxYSBS1747E8RD2Utf40h+94/3wT6Df7PvuOxB3WsAdAB8oZ/xcA7XtKgkHmS8gDwWBlI/EkfisVRKjqf3WPY+Tyuprj3z+F17M3vP53db+nAgZ/9QeCzTR4/do3/rL/7IX/eTv/QXnwLXrz30N5xzf+O7PVcpNQf+V8RH+Y3v8v9fdc79jFLqPwZ+2zn3P/SPfb/9+EE9lr/NPQUaAIfAX3XO/Rffd+Uf45I8eeRO/9q/jq1CoquQ9qBDFwE2k5suPYvoMifsR+DoJr7kfX3v7+hGPtVuYkmuA+pDQ7ARZqUbe8aje00ic2TAQXSnQUsISDfCa+ORwddMvpTjlcws1gvH9FO4/ZIjO9cy+xlBO5PBIchsd5fjmUKoFyK1Gj8TSZGN5cM5aBT1viW+lS67oIVmIoM9p2X9LpMBWvlQWLW+zF4G7Y7iVAbCr8/k94OceAU2EYlTeu0HGUuZIc+uHav3pFogLKE8EQbHBTKALE7cIEnLL2S2u/jJijce3HLzd0699EYGd+1Y2BMbyXaqfRmMpreynvQBygBQd9IPp31uTzMVJqUvvO9fS7fiLcvOZZDWjmUd3dz7nOqFADPdSEri9BPpsJN4eYYeORmYCBvVTBzj5zLoqvcci9+5LzOv9xT5uciOXNjvizBu0ZqhF7FnP6oDH/4zlmvW3x/ZuQ+GaXxAh28K6nKYPjXcve89XJ7VTG4Um/c79r4WsH3MwM5k15bOR+z3gRlhIYmM5bGAhaAB3ci+J3cCylbviXyuTxzs/UfKCvukWxi/EHZDGdg9ckw/84mNPrAmqKE8km3UC7k2QY0wSA/Ey9azb/HaUR6Kp23vA98/52D7hoRYTD+3rN/SJDfC2Dl1H0bTpTB9aqnnmnomvzdjRbWviAqR2w1VDUr54BxH8UAxfiHvo+JUkZ07okLSC7tcWCYbSWplWIj/zsQw/1AGvcWxJr+497e5QBggNCy/4Nj/mmLzREDj5LmjOLoPhXl9/yXMQ85Zfi4MiQ0gvZFzEjSSEooWdmrzWA8gqssdo5dybfpjDBoBB8v3NPOPJNCkOBb5YjvxXr21SCSbqQTTgNwb2yeQXimaOcw/srgArv+o4ug3HJtHmtnnhnJfmCGnYf1EC9MzknAf3dx3ARYniuxKmKZyX9i2ZuYljRPF6ExqGWwk+z06M6zfCDGpvO+VEVAa7eS9ll3LxEV5oClOHcmtkvN1quX9NFWMzwzNSNNMBRTWs3sQnF8bdkdSK1MvBAwKIypAymQSoJLeijwvqGH9Lkw/ZgijqucCWuK1vF+chuzSp6t6htbE8rkbbb0/LoDsxgxJn31KaXZjWT+Razn7zLA9CZi86igXAZNnDcVxRJcp8mvD5lFIcucHcD5oyCRenreV7ZtYMX1uJATHB8HUk2AIbul9iiZR/rtDQC1KgnPquSK/lKAaG/v7ZCeAoe+DxEkdRrI0NNOANpP3aXpnxJvt/XX5ZUczFYarSxXzT2qK45gu9R5B75HUtcOkmnKhye4kxGb1lmbvW2Y4T8lKWLmwFFAgwTqKdGmGigRlYPMoZPyqEzYVAV5dFlAchcw+qeirJspFSHbdiffOs2Odl0hKbY8iv2wwsaaeS6r06KylywMv65SKkPyiZf1mxPzjhvIwIrnrKI4jsqsO3VrpOqwtzTy8D8QxvoamsZIeOhXw1WWaLtPkFw3VIiJZdlSLkHgjA/xmJkmkTkF+VrN7lJIsDUFtCHYtJo+o9iPijaGeBSR33X2YTWVQjaUbhcSrBhtpmnksoMo4gtLQjkOSuxrVGopHI/JXJd04GoCgMo56LyK97rslNdGuI7qRL6by0YR41Uri6jgkrAzRTUF9NKKZh2TnNe00GrYZblvCZYmLRLLZLFLi24rqMCN7vqE9zNG1GcBftKxo9jOCUs7t4FvcNNhQE64r2kVOuK4wubCCGIeZpeJVBHTtg3O8n1KvC8zeaAjaAcDI56wZS6prsGtQTSfy06q7B3dthx0ngy9TdVaA4XKLy1N5/rrEZTH6eoU52hvCfXTRCBCNAgnzUUoYR+dQTSuMZN3gwgDVmfswnqaV3+MI2g43ymC5Ro3y+/+Po/tQnh4U9j2S/Xb6Dsq6ETlqmgxhQX2yq7ypFO51+anW8ncfwONDe/pk2eExwJXCnCpfSzIwmX5xxqC8lHbYHt/OkH5PYAm8Hurzna//wyx/yIHlD3VsSqm/AhTOub/+Xf7vbwIvgT8F/CxQAv+3c+6nv+/r/oDA8slrf3bAhXPu//MymfkXjty//N//aWojXZChtpyOVqzblHFU89vnDyhuckYHBQfjHa3VrIqMJGrZFinvHV9xtpmQRR1XqzF7k4K7TU6zi5nvb6V/EjiebVhXCUWVCMO4GnG4WHP+ag+dGEbjirYNOZxuOb+b0JYRJydLru4mmCLk3bcuKNoIYzVKOb6wd8nv3JywXOeSGltE6MgSJx37kx0vXy2YLnasb0YcHK9RynG3GjGdFKw3OQ8Pltzscuo6ZJzXJFHH+Wf7zB+u6aym+miGOW6Y7e1Yvppy/ORWPJa3Y+bzHXFouF2POJxtccByl2GtJo46OqNJog5jNWUZk2YNedxyuxoRhJY0bnk4W/FsOScKDJtdyttHN3z49ITRrGKSVdwsx0RxxzSvqNuQMLAYq9hsM+Kko9zFBJGhXaYEkxZ7F5McF7TPRrgATr5wycXNDGuUVHXsYrJpxeO9JR9965RoVtNWIZO9gs35hGhWEycdxvd08o0J1UkHsSUaNbTrhHDcYm4SXC4sqeoUp+9ecbvNqXbxvSdwncImwqWG0aJkt8xQgZVBUeCwnSKILN0qJpw1KG0xZznRwx1KQXWbMjosUMpRbBPitKPaxhwcbri5ngDgjCKflxijaaqI+XzH3fkUrGLvdEXThexuckHRgSO4jQgeFczGJZsipW0D7F2CSw2qCIiPC+plOoQbYYR9VdoRxIb9+ZbLz/Z57wsv+fjV4dBnGj2PaY46VKdh3Ir8WgGdIprWWKuxdwnJVcD8KxecP1uAU4TThm4dSyrvoiHNG6qXYylGf2tD8XI8MPo3T/dQ0wZnNGHa0pYRKnCwjmDScnS45uLVnGAZYsZW5NhFJNcg92FEoxbTifeGdUR4WKK+NaI5NETzCvWJT3B+b0ezjaHWBDsfWJRbCUmqA1RqcK2G0EoA1PM9gp0WmbdVJGchzdziQoeuNMkbW9omlBm1sxS7L4nI5kRSinVscMsYrCI8Lmh3MaO9kt1NTnQVEm08expbgp3GZg4XWlRmCEKLuU4Id1pYZQDliG5C2j2DroWlsIl0jJrc71ftQx1ahVm0wrpXGhc4oo2kB/dy2m52T7dEy4B20YF2ZJ/HlI/Ew9eOHd2BBFBFzxKaQ1lHFxo7Now/itg9Fva42TeSDB058pcBxalQrUEtDLpNJQk5ugtkwJ46zFSuoWoU6UVAO5UAItXJes3cQuCI7jQuEhY7vRSPYXwVyGTNniFZlNS3mUj596yEOcXiMWzmFju+VwOMnym2j9wAMHuprnL3TLoyEuAk6cTiOcSz3P1rRmuZQGgnzicMK7qxMMIAXXYfBNSNfZ/nTiaprK+fC3cyAaeMIn+pKE4FQPehTdFGvJzptUwE9V7J8oEhexUMbKRJZELMRjLZ4UJhtuOVGiYkcDLxYCNY/A7sToWdbv1Ep+pkwiteCrNcL3xolIXyzZbsWUR2KexVdeBZxI3vn1SKdipezLCUbSkr5yaoZKJRNzIJJEm4UBwLW1/vO6bfEpluH9BTHViyczmPzdwxeiGsVHHqmH1L2MFmJsC9HcvkZh8EVM9E7lo8UMy+JRJZkYAK+O9ZbmWk0sT5ntN47YbJtvJY9jm7spSHmnAn8mbAe1QlTMkkajimfjIo2ggzbFI5D4Mk1nmJ9likwcZLhUGe0wf+9CFH7URCmUR2LBPIex931NNAfJp+Qjsq3JB0qjr5u0/aDUtHPdWMLmWCocvk3JjIM+A+BbmZKNI7mcDrmWMbStBPuZCAIUlI7sOF5Lw6pQYGMyztkAxcz+S7omf0k5VMSrlA+RoS7//0r5fddKJSifXAoPZ9rX1wkDL37G3QypvVxIpk2WEjqYDJz0WyWy4CsltDuDPsTmPGLxqR7y7CAWxnl+3AYIJs0wUKE/dpT+K9NLFnsyNFelnTTaKB9QxqK9+lncN4BrRfTBYQ3whYbmYxYWEGtk85UK2hPkgHAK5bg942dPOUoDYCOAGbhgJU4xDVdJg8RjedANumFYmrT5N1SklwT2uwiXg/Bay5IUCqZ35V3bOXPhyoqEVSmyWoyktTjZEwIGtFftuzjEqhOp/CuyuFvXy9dgSgrCCJcUUloM9Y/zx7nxAL0jvppbMq0LjX/JdyAuy3hQF9z6Xvx/zDACx/6ccALP/97w0slVKHQOucIfPQzgAAIABJREFUWyqlMuBvA/+Jc+5vfpfn5sA/h7CV31JKPQC+5Jz7299vP35fUtg/KMvovQfu3f/sX6WsI94/vkLjiIOOy2LC5XrMw70Vb4zuaJ3m7/3dP8I/+ye/ynU9YlVnlF3Ei/M93nl0xW2RkUYdx/mGXZtwXeSM4pZxXDOPS752fsrhZMflekxTR7x/esHL1Yy39m753bNj9mc7Ho5XfHB5zHsH13xwfozWjif7t5ytp2jl2Hw8Z/TOis2LKW7UMd0ryJOGq9spv/jOxzzb7nG9HVHVEV88uWDXxbQm4GozQmsBKbYNODhas9pknCzWJGHHs+s90qRlu01Js4ZimfETb51Rm5BJXDOPC371a19Aj1psHTA/3JJEHRfnc4K0I0larNWYTmOdwixj0sOS070Vn//2KXbWEdyFcFJjCh+jnRpYRey/fceDyZqPLg5pqoijgzWrf3DE8Z94xYuvn2AWHUFieHx0y+efHYkEaVFRX+TERwXvHl3z8eUBTRnx4HjJqxcLAUVGEe9XOKtQn+R0TyrijzMZQE46Dh+suHq+x+hoR/lsQnBcsj/fcrseDcCCzKACJ2Cp0hw+uePq+R7JoqSpfET4dYzNxKeo5g1cJSJhHcmgsssd2gewmG2EqjVMOtQyIn+yZvdqQrBXY24lGMhNW2g18VU4hMRgYPR4Q/npVGRluSE/KKieTbDzFr2KmL615Hiy4dPLfcyrHN16lmSpKN+vJeH2QYva3YNIVQYyON4pki8t2VyMwSlU3hG8SlBPCoJvjqjerFG7kKDQdFNDdlBQLlP0OhxkqLqQAJvusCW8jOhmIk3VHbQzK7JbBel5KMmsjyvsdSJS1blFzRrsNkKXGjtvCW4ieFBj72KpSonATDv2fz1k/baoBAgcwVbTzby3c9QRnceiHDgusB+OafcEkCkrg/p24lP5PIhop5bouCT9e2OKhw4bCiMVbjU2cp69lzRS3Smq05boLpRexbE8DjIYdxrSKy+5LRTJndSMmLEhPQsxuaPLJS0V5LyoTsn9knoQ0infxyaAQTc+OAVoDjvCpaT16lJYX7Pfkn+UUD40BDvxK5Yn4k0Nd0rk07FXS/hrYENAi7TXKYaAHxf47fpKlHYq/mLdKcKNugcBjSLcwvbtjnAbeCAqyozkRg/SWV1D8XZL8irCaYc2yqdQCuOe3jD0DqpOBlLFmy3pqwhdMwSf9N5IkwrIsqH41bpUEkq71+qX0xvpMwS5buFODYqM3tPa1+QEtTCFkY8h6JNP24mAAm2ETTc+tdQkXuLssXav1EDD6IVj+1gRbRhk6u1ErmG9EIDaAz+T3Msth0RiLT/JUo5PGdmfoBLZqm4YUm6jnfheJ09l/3Qnkk9RSggQMLGAqvTGV7UYhuTdzduW2bc07ViOPz8Xxtik9/5Ep2HyzLA9DUQhUAgAKQ9ke9mlACwbCFM/JOBmAr7yS8vyvYBwC/W+93HeuKFXUqpo3KDo6H2nXS4gVRtfS2IELCVLJ0m043u/aTuWfe2l3MmteE1tAOOX0vWY3Yh3VPlrKRUwcp4Gb6ZXEulW/Js9Yx20AoabqYS9AL4eRKSmzcTbDrxnUneQ3rSc/ULC9HNLujQs34kGyWi0lc+WLhNViPEgtK8eCWsBf20uCa3lgR7Atw1hdGloc009U4zOLWFh2D6MBhlyVDi0EUDYp8MGjQC4dCldnb0HU3fyu43UAN6kGkXul3hrh77LaCOS3XakSdbiPTQ+8AgYvKnR1lAtAuKNpctEQtun0ppIEbQO3TjilTC5zSwkqK1IyRcBXarIrwy6kbTZnhUPK5HG2kQTlJZqPxy2Ge4s9TxgdNbQzEN046tSnCMoRQJsQ1EVRKuGZi++rx+xDF5KtCK+qzF56IONFGg1sMjhth3YPNVZbHKfWekCYZVdoLxkvBsqRUwWojyQdKEeQKULlchufRAPimEd1RrxWI4TAchlOzCgJg0Jl+LRVK2RLktfRzKwqK9JR/HBOC6LBRR2RiSwfRdk24nk1TOPqqyxo0xqTooKF0fyuGczXZ5C3Qx1LT14/LYQndflsOY1gNjLYJ0TdtI6VBwJSNTq/nnfyTwaIyDQ91W65v5a3DOsv9dj+T2BpnW4tvm9j/+Ayx8UYPn43/zRA8uP//L3BZZfBn4ZUZ5q4H92zv3VH/V+6B/1C/6jXKxTbHcpUWSIdUccdMTa8PJ6TtdpNnXCy2LGr37wPtm7K/63X/tpii5m20rq1xsPbrnejuhMwNVvHXNRTHy6asg4rrkuRnz11SOOplsOsq34C1vNN58+QCvH8/WM2biiqGPu6pw0btm0Ce11Rhq3LKuMk4n4M0/+yCWbl1PUrCEdN7RdwOX1lCyv+ZXf+iLLUhK42jrkd1+dcL6e8PSTI5xTjJIGFTiivKHpAtzznEVasKpSpiMxMy3mO8pdQpS3vFzNeHU744Nfe4u//9nbTE42BKHl+HSJ1pZQW6aLHdNxSRJ1wlQ2AWYXkR6WdJ+OeXqxT/Cg4OTBHfk7K9KsQSXCUEQvEtRew90H+3x4fkT3dCwe0NUIkznOlxN4UMsH62cpnz89JJo0HD5aYq3CpQbz6Zhtk9Bc5QSR5W6bo1PD2+9cEBSaMDQ4p+ieVBzub2jeLWVgNmq5/WCf6cmG3VWO22uwL3KqJqJrxfv55nsXRFmLazQ0mni/4ur5HmhH92wkbFiriVcaPW6ZP1niVjHKKLq9jvCwpJ1a7LSD5xnRhzn5foEbd0QvY8IHBdubHCYt1irSk50wAS9ignVAUCjipwlq1hAW8hYzE0PwQAxU9usz4jstKZ2ppf71BR89O6GrQ9yiJbkTZqI8NbgyoHujgkYTlJr2xYjoOsSNO5Hb/pE1zdfn4gGNLUo7ukVHW0SSXHkZoxpFN+uY/U5IdTaCwGEnUggfTFrcQSOD+FtJtgumrXgQ96VyIyg0+TMJO8KCqQPpCdw3hCtN/rUM1SjUcUX6NMEetCTfyEgvAgnZ0ZJiun5XJOcEDjfqcKFj8nFIeFCiNiHtzJC9DLAfjTFvl0R3Uq+iWvFJOiWhMdFdSHPiZz0/HrH+ciOpm9cCttqJpT3sPMsjA+f60DD7RkS719EcGDn/raI+7og2mnCnKN5tJIxo6qTa4kqRPwtpZ5Z20ZG/CuhyCRCa/W5AfKsxqcxox6vXBqorRfmwo14IOAkqxeRDkYWlZyHppZYAnvOY8pEcc7SWMJ/sQgugOTI+zEgAYlArxp9rZh9qlBEPpsyky7aamWX8uWfXAkd+Jvd3cq1oZ9IbGpbS87h7bIlvAqKNIr3QpJcByY2meNwNYTfNnmP0cUR91JHeKNqxFTn0816q7wHFTphJHEx/NyKofDdoJ2AouRVgNX7mqPcd3UhknLqF9fudZ3/ktdZv+XTMqZOKlIUVj+1GkV45kluo59KxaCOR39f7ApLilfNJo7B7KAxNcWIJGgFd0RZfcC+MXnbpJes72LwJ2aWwUbPPZJCT3KrBc9qHHykDo5fOB4N46Xwlj/X+wsinaZaHApKzS8f0qWX00g0y++mnsH1DwFQ7FoDdTgVINBNFO4HZp8KC9X7ToBKp7fwD8cRNP7PMv2Wp9xS7hxL6o1vZfrR1bE8DkpUTKfxOBmfpjSOohK3rcunNRAtIHJ0bH2YjqZ7JrTBYex/K+di8ed8hOvvU0I7VUA2ia5FYqw5GF5ZkaSm9b3f+sbDx20daGM1U0Y0U41fyuvmFZf6JkRqVzwzTp5ZqTw0SaBMDyleQIOCwnXiw4T2eJhYAWR5oqc5ohcVrxwKcwhJhUQvp22ymcuyJl7F3qVR2rN+IGZ050lvD7ihk/Mow+7Rl/MqQLg3RzpJdW7angdRxLDTjVw3KCaDtMt+1GwnIStZyTaWTVGMDkW7XM00zDRi/ksF4dmckYOiuk3AkJ3Jg3TlGZx31RCai0jtDl2rKReil1QZlIb/yskoL2bW5708tLOVBSJdpxi8a+f7cGLSB7Fo+P9tckV02VHsBUeFIblvS20482yvphIzXhuSuRTlHeRyzO4mIV90wqZJdd4xftpSLgLAyxCuRJ4/OG8LSp4UaqPZDzxgq8lcVNlFMP9lRLyKy8wobSwBQuDO0k4Bo3aEbYYabhTCCykmwDwiYUw7CTUuzl0gliVYk57uhYsT5vkoba4oHKWYU0Y5DSWutO69esER3JenLNS7QdJNYGEStsEmAiwRU9ompqnNYDw5F5SDyZRf4MByl0HUn/knnsEmE3tVEdyVoJKgnCSQUSGtU2fh+UCspsHGIi0P5N08EhObCKgrIrLwXU0Ik8fUkzjOJTusBVPbdmD14vJeS+w7MtvM9qT2te38MgDCQYcjQowmoMESFEhikAg11DbqPWbYCHo0RUPl6/2XPVPZ9lj7cxxn/45z823X3gNS/zus/7jUp7f+//HCLc+7rzrmfcc592Tn3Uz8OUAn/mDOWyRuP3S/8N3+epx+cEK310AVnQxlg5WcyWOtGjuRaC0uxUQSlSHaiLdRzwIlcKF7KF9r0qaVaaLaPHONnMoPWZTI7i4Pq2JK/EK8kIAOxGT5xUGby02uGtLlqrtHGsX4LJk9llT5aXII0xNvYR7mHhXiYoPeLyUxrdXCfiJhdOMLaSal1oIb0wGjn2D6SAc36bQkzCWoGeVx2LT4kZXyghZf52FhmAaOdY/22IrmTbYe17wKzDL6e8kT65IoHjslnsPyi4/A31RAXn13I7LWNYfuGSJ6yS0e8c2xPNaNzy91PaCZPJZQBfNn3WGZz129q0iv5e3wmaYnaOLaPNJNnls1jzeS5pTjWTJ8aNo8C4vX9jHp6I7K2cl+TX4lnqFqIn6g4FG+sbmH5Ux0PfkUPAxNJ9pNzVJzoIT3SxDB+IdKozVtw9JuW4kDkR+VCEgCrhZIo+pX3QoUw/8RST8QfpTsZCLQjzfaRHtIZnYLySJIXdefIrywXP6/Jz5QvtPe+2CPtr5kMPtNb7++8tGxPtaQolk66yiJFcaBJl47V2xJaM35puf5pxeiF3PeRLwFPl/JluTv2Eij/f5tHAfmlJb0Vv9f2DUju1NC3F5Yijdo9CChOHZNPYf5xw+XPJcw/Ntx+ISBo5N4anckgObs27B6I5230qmH3IGb1ruL4NzqKw4D0Tmb6u1Sx+KBm80Ys93Xqu+g6YUF2p3LdTaQojjR7H7eYWHP7xYC9Dw3aOGwgs/7NWFMtNKMLw/pJQH5hibeWu3cFRB/9ZsXy3QRlYHzW0Y60lNYvLddfko7P/d81BLVl+zBk/LJj8zgcfGvT5+L72h0FMjBaiVQx2bjB11bPtJeiyWCl2hN5XnrtmH1ecflHM3QL2a14RV0AkxedDJIrRzPRchyH8h6dPjU+pVKOMVp3Q3x+suzoRgHbE+mdS1aWqLC0owATyQx9ftlRHIXCfOyJfzIsHdlVS70XymDQweqdiIPfKli+n5FfGmysqOZyz4SFdOP1lQfVXkAzlfOWXZshkbJcSLiIiRWji25gj5ppQHbVsDtJBJAVjvS6YfMkYfSq5e79mP0PKtpRSLUn7LmNIL/s2J5GpHeGak+8ifUiZHccsPdhzfZRzOyTimYekVzXlCep995JGmd6ZzCpos21Z1s0B7+1Zfskl3qOsSbxyZvpTUt5GPu+SUsz1t6nKIyQDYWpaj1L1HshAfJXFU4rigcJxZFci9GZJICaWJNetxQnEaNXDdVBRP6qYv12xuisJVo3bN4ayQDdQTMJSG9bdiexJIJa2e9o59j7ZiGpl6GmHQd0qSbaWdLris2bOflFQzOVa9pM5Vpk5yXtJGb3ICK7EaYqLJ10Ayqo/DULWmHQ8vOGoGxpFildrqlmAbNPxcPYTCJfFdGiOkdxkmAjxeTTHeVpRpdqsquW6LaiOs2Jly1Yx+5xRnrdsnsQ4QLF9GlFtYjlM/D5lnYvJVzX2CSk3k+Il8KUJdclNgnpxhHVImT6/7L3ZrH2bPl912etVeOez3zO//yHO/a93e1ud9vt2E4swIJYlggoGCyEiJAigQRBct6B5AGB8oDEExIPvCGIhE0ewFKCIhsriWPaeGi323379nDH/3zGPde41uLht6r2ud0eAo5Fd+SS/jr/s8/eVbWratde3/WdvrPGZhG6db3MsDpIGDzdCuuURehachFcZtCVRVct5fEAbxTpdYXLxJeqK0s7iLG5Jr0sWb06lLAd330Ht9TTiOGHa2weY4qGxVsT8it5fy6XrsPOq2gq14fHmELkkHYQk1xuaA4HVLOY7LKiHcWkL9YUD8bkTzf42FDtpxLsEynyJytcFkmXYtWilwVuOqA8zskuS5pJQjwXr6GLtfgSyxY7jIlvtrg8pjwdkD/d0MwyonVNM0lJLjYo7ynPx2RPljT7g97jaEJthVmUuEEsvY7LSgJorKU9mggrWFtsajCbBr3aYg/HwtpVVkJuAOWEJYwul5JWCtSnY5JnS5qjEfHLJe3RmOh6Q3M0kh7M57e0x1N01aI2Mnnu9kbobU07zcSvmUSoosYNUnTdotZb3HiInWaYRdkH2XCX0ZuN0StJDGtPZpj5FjfO8EbjI010sZS00zhCL8Q/qepGfs5XApCyBFXW+FEuPZCREX9kWcv/46hn4ZTzUJT4yUj2oapF2rpcQ5ZCWcnPqpbE1m2BiuT1vglALCSzqiTB17WksnbexqaRfsi2RY2G+PVGtjMaioy2aVB5LqFBm0KAYBTt1l1VwgB6v/NH3gGQGL3zVuo7YBNkX/NM/l+Uwkp2CbBai68yinqQ2PdY/n9d7vgv/4VgLP/TPwPG8j/7f+ex/LNafqCB5fBTZ/7ef/03OJquibQjixpuigGJsSzLlFf2bmmdZlFlXC1GJElLbCyNNSRRS2wcq0LMMGnc0ljD3qDg3mjBO5cnfTiadZqmMTRFTDKosa1BKc94VJBEluvFkDRt2SwzBuOK6aDg5c2EvcmWg8GGbz87ZjCs2CxypnsbFrdDlHHiTdsmItm0CmU8tozIxhVVETOaFGxWGd4psmFNVYgEwtWGbFKRxC2ruWi60mFNuUqJ8wbnFVp5mmVCPKmFCQsprjq1+FZLdckiBeXFe1ZEkFpwCrU1MBaqx7catHSEKeNwTTczhSS6zkrck4F4qVoFsQctDIZZRNjcyWOh1kC1WuSjV+J3HOwVbG9z8TtkVuS2QWLWpbCqQYsvDaoIKa6VgUq+AEena2EPay2pmplFZ8H75oFpgy8MqjSogwq3DL4E7YluI6lb8UgXoRcvmI8CG1QrEQyApHVuNC6TzkA3sOiNJOz51AljF/bXpQ4ij1mJ1FCXYvCyU4sqhXm0mUgevRY5qx1IvUf3mMslOlBVGlPq0BEnXi1dKtmPRkKd4rXUkehGPGDeeHQtQDHaBCnZoSW5MX2Ko029VMlsDGjfS/W6QCVdy2SCyCt135Fnh5bsRUR14EDL32wu++KNyOBc5InXGmcCy5DLT1OKJFV8YB5TqT7FUzXyvGgrx9AmMtEiM7y7iREb0lmrfUnBNWXw6QGmUPiY0NEIduD759hMQKkpVF910YxEamoKOedei5zSxYjf84Uh2goDB6Br1UtoXSLyUZf6/hz6SJhVXYX9iHbHNV4Lc9dJLaOt1GgAPSNWj3f+NZfQ162oUHNh6iCF5U5yajguePpgINXKpNEufZcQICJPbYc7314SJLs220lMu3RN3UgYU3ojk0SdjFGCsUKNiduln2bXnu1JqDHKZN+yS5nE03XYblBMdeEyuiHIh+VvupFJuuwmJL+GyQkXEkGBPjjJlL6XVHZyUUm3FU9dVxHRyUPlGiSAO/k9WckETZfG2h3bqAzHNGynDaE1PoSrSYE9fQKoD+E9Xd+gbnf7nSw9PpL+yHgNuJ1U2GthMV2i+rTPLsVXJKoC7mRySiR7XaKrvA8VJnqEEZUEV8Rrtw1pyVGoCVoKw9cM5W/xdnfuXKhM6So1OqZfJj6D76324R6xu6ZMLUF0co8Mr7M7r2gXIFQeihe0S+btji3IpEK5t5todUl4/yEQL6o8dTjP7SAcTy37UxzIvck0vp9oUTbIbQeKZCO+Qt34vmdSAoVk4qcNAU7S4SnnorumxZ+o+qTWeqxl4i5SNLkmKh1RIYmwLgqfsdb3YX9dcBFKOizvXvdy7CTMKt4GOWshfY3Kyt9UGySLml1yrBIJs7a+n3SKNzJxYmrfJ7zq2oe+Q4KfWFhAXfu+/sPFut+XLuhM2SCBVYAS9tqUDt16mpEJ56Tz7IrUVFtJae1ClHQt9+N2ZDCFC37K8P494D0m1JQoJ3+LStv/7rWS99jukmuj0srnxnSBSOFviQBpbySlVdeBIbsDhFysPxHYI3Jaja5aXBpJAmzrZIIm1JH0KdNVI12QsQmVKO1OMhq+n1AKVbZSJeIc3pg+3VbXu1AlqToz6KLBR53PU1JgvVGoSipEaFoBgnfWo5yTx8Nreh9k0+7Ce+7WixjTs5CqYxutMKKqDV7QDqi1u2TZbvHeS0el9zsgGepPuuOqVEh+9Ts2UxhF9z0gFYAOzH7X8kfJXf+45c+B5R++/Dmw/OewpK/c96/+t/8RxXVOchWh2lBpgIR2jN83VLPdl7zNQsDC9u5NGspjGaTGaxn05i8U9V64IW92/pNmIgyS155oI9KeLsa+OnJkFyLFa8cSCCFG9xBDH2RbyVxYVN2o3pAfhYj6bt+TRQh2WIsnxqaEyG153z6SsINqT/UD16gU9jW9lQFrtFHiqaq6m4v4o4bPhC0VNiVE8Bf0HXXxKtQRDIXx7QayyspAavma9Oi5WHxm+QvN9twy/NhgUyjPLPkTYaWaiex381pB9k5OVISBVCvsZ7KE8kC233eKWZHCRdsQs7+ml9yUB7JPzYh+EJXOQ8dcLQMtr0Tm1g5V3+8nybDikan2AkNaQrUnMrbiRB6LtrKdZCGvbwa79NnsWgZ/2zNJhS335Iu8OJTE1/JAzkU7kHXrhj48xJSeqKRPHO0GPt0A3CYhjTeR97M+VyQrGD21LF4TcFNPAsO+H+SWK9ieeabfoR/MxhsfAAN9p1znZROJqKyjY9JsqnovVD1RRCWYQkB1PZFjaioZkJUHkoyrPD2DmF8Ju9zmIiWMtnIc0rlneyqDaJfswEK0keORBIleO9zJ6bpqiS5AI14JWxyvZPvdwN1m0tM4euJpc4JXxyNpzKJGiLc+JC/K9WtzWc/mXDF47vu+vHTuJe1ztovq7wak46fCuoJcT8LYh/CQ0A9Y7cl6vRbWefKhJMaaWkJB2lzRjLpBXvjsahkwlfuKwcuQhDrenQfdSu9evO1+lwTiLnymHYivjjDo7/oRlfdUM40J3X8CsnbXRedl63xf/b0pMPXKCVveDDTlgSSyFkea0VNRDEiNgYAMU/oeMCgn4KzNRRUQr+Vat4lsd3ui+8F8upDz3Hm3olIG/Zszxf67VgrRpyJT3B5phi+dpGePVA+Gsrmn2NekCycs8FyAV8cId0CgY6zvJqJ29xFT0zPD9UQx+chSjUWBUhwpBi9dn47sFVRTxeDKsTk2JOvgZQsAMJt76qHqQYVsQ2omugCV7aFcE+lSjm+bieeunBrSlTCh+aWkqkaVDIC3B4YkSFidIcjV6I+7N3L+82vXM1vaSl1H58/rwFIz2A36460n2kpfYNUBpTgcowCqqrHuQaxuhc2OtpZ6GmFjAVwdmOm8htltS5trqolc6OMnFZvTRNZphdEs92Pi0GdYHERkc2G2y33N9L2m99SlNw0uCb2L3lMcRmQ30odoCofNNKZ0lPsR+WUjAElLCAwKtscxg4tGvvNzTZtpkkWLzaX3MAqpqFIL5WlGmvS2DZ8vCZExpaMdmj5ops0F+LlYES+tAJrKsr6XMH5SgYNmHAUAJ8mvqnW938w0AhTrSUy8bmkmkaxrZbGpJlk0VPsJybLBK0W1F/cTDvmLsq/kUNYLS5hF1LOI7FJSW021qxcxRQtaCXtZtsJmT2PiRYPNjOxL7dCtC39LeibYxboHbjYVkNWOhJVObmvMtgHnaA4G6NrhEk1X4aG3DS6PsFlEvKr710vvYyNMZutweUw7SYlWNdVhTnpV0ExT4puSdpqia0s0L2j3BuhyB0ZsHmM2jUhTa6nx0FUjzKiXz6VLIgFjnVS1DeE7ZSOy0EEqvsbW4qYD9LbGJxE+NpI2O99K2I33AvScE4axq/yAHtT52EivpNGy3m2Fz1L0eisJrk0rQTdVLayjEc8kSYxabnow5vNU+ijTROSenbeyqlFJLGCztbvtg/we9qNfjJbHYfeaUG3yiVTY/rEAhFsr4DD0YXaLiqJPAscO7HWpsEqDs7KvwSup4vCaptkBXRDGMkn6oJ5+CWE8/fruJL8CeOdRAcj6u2DzB71u5PyBf/hnACy//Z9/fwDL6E9+yvfxoj1xZClzS/PAwipGTWpcJV9uy88GBi61RM9S1GsbquuMClBekVxrqQ+JHG2qaSegRw2ly8IXtxeQtpWePju2qFrjRy0QY4/Eq1QfyEVeHocZeUVIuAvde6kMjpuZRTdR6OCTYI167GnG9JUgLhFg2AV3dLPhugLlurRCxepVcIkwWC4Rxoh7Jf79LMzGe9qJw5UKnDAy8UpRHEvHXrSSigHlA6MQGKd6JiEoKNieChNTvFaTPU56tqE8kcTCdiiSYylEFwmgLgQUo9ixRI8zmqmA6XomoHf7qKW9Mv0MdQf047X4x3TbzTYKyDWl1DA0Y1mvj+R9J6tdFH87lJCPzQMVugtVv25vBFR0+ybAxlMcS9l8edTNDMpAvx0IEC+OBf13klivoJxpqpmcD5DBaTPesTkoqA4lSKWZ+J4xy192g1nZz44da6ZOwEWpUE48c+pxxPKRoZ55koWwMC6VgJF2QAgHUawfQL3nSK81yqvQJ6aoJ7K9euZDYiIUp5Z2oQW0p7K8Lc2jAAAgAElEQVT9rlrDxVDHkHgVZtshChJpm4BLYXNf2DWbEaohdPABSp1EOvfYXGFLOY5tLq/TFXAHzJT7u5AR6XEMISKtpFV67Xsmq/MtSf2M6pmMaiqya+k7VAHwdpUnCrwKKZCy/4zUDkAXAujLfXmvdScND8mQupXBdXXoGDzTNEHOTPeeDGzDZIRJ5PW6he2J7lmf4kDTpQm2uVwfnUwaJe9pcyYSaJcIC4HeMWtdKXsRytzjlaeZCHAuD+RnV6OCQqSnVfjcJaECSUG07lggAYQuCsBqLNdKV5UUr+V8VqHmox4JE9YMVO9zU06YMZxIfdfnGlMI26QD8yzHInQ8pnJcuxqJZhjYQy+dhC5SIZQFAYuhq3GbBa9aJGC+8/Z51TGFATBqRZPL5zUq5LH1fcX+Nxwqpgf2yVJ8j9VUQG6T77yLEgijejYrvRV2DCBd+n4CrhkIMwNyL+nuK3WYHBHmfff/NtXBtqD6sBpldeiapE+ibDMBwO1AKlNY7RIrleuOXQCsJkxCrT1NupugqkOfotdaQok2d18beiqD5aDNQbcC4LyW68brHTvnzK5XUlcChL3x2FzYC2805UwJe7cSiXtUyv53rKbXUvMRF47toVgFTB6qOzACVktHM5RKmu5z53VnY4nudC3qYJOQMBqbiW+uHZpQCaWl59GF+iqjSOfCxrlYAll06/t1GqCeRr1s2WvxZLpU03XLtpkW1s0TqkkCm5xoTO0CuBfgFZWe8iAh2sgYoANaUidiSJbyfBcYO2HcBGQqrWhzE6w2AmKxnnYiMnUfIUBcS1ppM457ea1MRjlsbvr16ka+F5tpIqAvVtwd4ikPaEUziNCpVIC4VF7v0gibhn0ZRpjSiow31pjSBrAmrKDL0x54dsDbRwqXReE1DpcYXGp6htIbjQvppD4cIx/JcfaRRrdO0k675+bBI9ixZh0YCWE9LpNzSBSYyrUwWD41OwASG3wWCcDUWkJ08hjTOkg05UlO9mwHVpQVttHrcO+2nk6y5PMEb0zwR8aoqglAMhFg66DrlXSjgbyXVvXg1+2lO9a0tRBHn6gX8Xkqr4fAEhqUD95IrSEJIK1jIo1c83S+xS4Mpwv9ueuT7KpH4li8kWUlz9H6k+mqzu/AYhecoxW0Yd2R4RNLeB6xVON0oFQphTcC9oljAZz8EYtW/TGWKpM723AeFase0KpeneO+G3/+YC4/uJzen7j8QDOWX/rhzN/77/5DAH7+9LeZ2wEP4mv+6fpTfPX2nJ87+wqvxJf86uqzPN7u8crgmi8MP+KmHfFxdcB5estFM+FXnr/Fv3n+NX7l4m3+/fPf5Meyj/jFxZf4tRef4mfP3uGvTL7Kl4vX+L31Q/7x49f5hU//Gs/rGQ7Fs3JGqlv++uE/4Rfnf4GpKfjHV29gvebfPvtdjqIVv3T5Jc6zOb9x8Sp/8fgDHhd7fGp0AcBv3zzkR/YeM28HDE3FP3r+Bl88eoJRnh8dfcj/9OTHeTC65ccmH/E/fvDj/OTpBzzZzviR2WM+n3/M3734CSLl+JHJx/zDi0+jledvP/pl/t78S3xtfo8f3nvKB5sDFlVOHjXcH8zZ2ITDZM03VyfMEumlcl5zUw2YJCUfLvZZlymfOX7BvXzBcbzi29tjvjU/4sXVlNlUKkseTW55Y3jJ//zln+Tf+/Ev887yjEFUk5uGcVzy1ZtzPnxySDaqefXwmsNszdPNjJ8+/hZ/74Mv8Gh2yyiuuKkGDKKabZvwCw9+lf/qO/86f/nsXa6bIfMm5yRd8fc/+AzWav76p7/M/3nxFp/fe8qvPH6Lzx0/49XBNVub8Hu393ltfMUkKvm/Ll5FK8+n917wvJjycHDLNxYn/Ny9r/DLLz7Ps+WE8p0Zj378CSf5it+/OGOWlzy9nPFXP/1V/unL1xinFfMi5/pqzJsPXvJ8NeZHT59wWw0orXxJNs6wl265rQZ8+PwA7xQ68kzGW5LIopTnwXjOb73zGucPr0mjlg+eHXKwv2a1TTmYbHj2fI9PPXwJwDQtuNiO2TYxl8+nzI7WKOVZbzJ++vVv8a3FMZerEV84fcrTzRSAF/MJs9EWrTyJsRxkG67LIQ9Gt/z6t98gzRupwvnKKfnbczwwTGuu5yOO91ZcLYdUi4yje3Oub0a8ce+Sbz895ocePUPjuSqGPP34AGLH3sGabZlwOluRmpaniymxsbRO93Uve6Mtz1/O8K1GrSN+6AsfsqpTXv6jc9ofWpNnDcfjNd/58AQazec+/THffHHMv/rat/gnT15ju045P57z+IMjhscb2lbTNhEPjm+YJiW//9VXiI4LhnnNap0zHJYiSX9/j9c/+4z3nx1yerTAOk3dGm4vx2STiurZEGY1+ahinFe8fD4D7RlMyvD97ajriGFesdpkDPOazdf32P/CJVnU8tGTQ4azgu0yE1n4JsZMapGdL0RSr0eNSNcHNcXlAGJHPAw1K5Fjb2/N4t0D3EmFLw2H9xas/+9DqkMLszBIcAqTWM4OFjy9nHF6uOAg3/K1dx4S3xrcoxJba2g0ZtiijYRqtVeZgMmDkuY2FRlxoVHHFWlWUzwb4WMPkUNnFmMc5t0h1bElO9lQVzH+MmXy6pzFYgCLmORki/vOiGZmyY4K9O+OKd6qRD69jSBxmFvxouLBzVoIMmyVWfRl0nezuvMSLlNc5qRCZdaVpnYz6B69NajjEjtPJNl5GeMTh6q11H3stWBVSAtWtAcN2Ueyjc2jlvQiojpuSS8N1XnD+fkNL/7guO+ubYdSdeIjjzuq8dsIM6mxcwnvihca8/ZKzl3i0Mso1Lwoxm/OWX4wE9b1sEJ/lH1Cvh5vFDbxfU+nciKpjhYBFL1R4C4yTCGTZy75Lim6Aze0mJXpZc3xUqjqZuTJLzTbB9KzvPeu9IZuz0R2Hy3upCePBIgOXii2p76XpnaBRPVxS/Yklt7fwa4DefNAJkI7VU+bywSY/uKC8r0JyVz1bHQz9f37SG/CpFsAzaYUtYpqRd6uG0UzlO00E5lYjLbSe5wsZJ0uqD0EiEBxLnU4/WSdl5/rT1cM3017RllUDkH1MxRlULKSMB3ChGx1IOdIaiyCmiQkF2/OpXdXt0GZciOA9uaLluPfMGyPZV/ThaMOeQYuFnVGcaxIr2U9XYWI8vQqhq5uRLX0DPKOARW/erWnGD11bI816cKzOQtdqVdhEiN8NFysehWAbmR72dyzfKSlX3g/dDWGPmrd0mcrOCOTEfW4SyEWdr8Zql5S3S1dwFTH0upG1DnpUtQANqWX9Oc3ToKFgnS4v2Y7pj0KaikvHuVqIrUm1UyTXzucoZ+YiEqZ8EmXTqpMvLDrbSZSWBtDHhQJ8Vr81dnc0gwN0TaA/XCcsouKek+sMPW4q1KRiYxkJYC7zTXx2oqfc9Vg80hUM5npfeAuCrJbD9GmpThOiEqR8OraES8b6r2EeBMCcJQiXlbYYRzYWtdXgKjaiZe3cSKnVcIoeyPAWbdOfKnpDlh1tSHtICbaNuiyxSXiB3aJwWzq3nfrY5HYuiwCpaTKRKkQ8hNYQ61RZSXMZwDtd//OdzOJIP7SwFB24UUEKW3PtEZGWFml8JstapCH4B1h7H1ZCVtprYDVEN6zu+gCeA8+zO+W5X7P4hyuLP/45/wxy/cNY/k3/gwYy//i+4Ox/IEGloM37/njv/UL4r9rlfSj5ZY4b6hvM1SliU622MagXqb4kwouU+ldU8K+lGeWaK3Rr63RXxtTvlHirSYe1KjvDLGvlvA8JV5r8aUpaKe299vFCy2DlaMKd5WKX+tBifkwox16XCqsZt8td68i+SilOrQoJwOZ+kDqHdr9hugmxg4cow8M6x8uMc/T3gNjamEWy1MZXLRTS/40QlnYvl4zeC+h3hP/mstkxjVeSDKni0SG2ydV1rvIfZvJwKkdin+smQoroWW8GmaK6Quz20H4ogbsfsPka4msdyn9Z6ZSgQ2QKP3iQcvwgyiEYMDysw16bchfaDZvNOQfxtjc04w9w8ea4mRX8G2HjvTCyMBhIDURLpLtxyuF/8KK8iYjfxz3X3AuFbau2oN6JhJlm4mEuD5sUZUkYXZVCMoKa1uetgw+CrOq3SRa4lFOkd4Ehm0k++UiSG/kdTYRP1/+UuSzXWVFtFEU55bRB+IHvesxs7kwdbqW45ldB9lmYANdSu/76vxcnVw6mcu5cokkkDZTT3odBi2NDLTiJWweerILkRaW+0HWrCU0ptwXZnT6DXm/q9cd2UvdV2XIcZR97eTgpvKsHsHkA1g9EjY7WYo8t3ueMHL0ktyokP32kQwwpUsvPKZ3dRHdwE58Y550vpPBdhLeNpPPQLKU/b87uBQfo9QxDJ9o4pX03HXJne1AWPXZN3cVF20efFXVbjCmW4nCLw/k/Hbevs4L2knHi2NPfqF27zWRwWC5L4+ZQtj1rrbC1CG59FgRbYI8NZGB0OBC6gtWDxXxJgxMg38u2nrWDxXZlQz4Ok9hVPjA2Mjgx6by2c6uVO9vM5V4/tLb0HMIfaJqfulYvK5Jb+klqVGQ0Vb7kgqqrKR4JiuPjbvAMTk+nRzd1MGDOJDt1GN5fj2W/r9k7kmXjtVDgymlz68dyL4kS8/mNDBSY0UyF59geagk8CmW92JTFcCDl8FmvvPmmjLIXAmf/TB4bwfCVLb5zm+qre8DsFb35Z48fOGEXVbCLld7qh/U6xrirUOHUJ96IqxQduNYPjKk8+DHi+W52ULWFRWSWNrmiiZXZAtHHdj9ru9w9Nz2Pj2bqnA+A5MbK8oDCYKSwCfYHhvpQoxlH5SnBwbxRqSLnUzdGZEOd7JdqS+RY6qcAIWuO1G5EAIXwpY6z2RU+b7WomMiTSMS484e0X1eTB22MRKQkS5tkKJryj05Hp0fNd7KIL8eywA5WTlspqgmIUAn+GflPLneYwcdg4QEVOUmSN0DMCk64GBpcx08xa7vP4y3Xra/drRDYfdVK0ysVxJAZTORrTdDkfiOntZsTxKy2xZdO8rDmPSmxeaa9b2I4Qsb6k5U38GYzFuRJE+jcE+ytHlXSeFoRqZndXW4bjt5s24cNjOhPgKiIvgOrTCuIF7JzkKjK5GpRuuGehZSU10nYXa0mch6k0WNzSLaoazDlLbfH68QaWlpcanpGV5dye+iaogwle19jZKKqkMIUEJUWKq9hOyioJmIedlUjnYYkV6X1NNE/MKlRdcWl5j+PdTTmGRe443GbCX0pR3GmK10UNI63CAWGW1QHumika7H4E2UfQphOY3DDiVYqbteXBaJHHld4QYJelsLIwqo1qEXG9xo0IM61Vh8EqFXpYCgJAbrsNNc2ObKiqcxgMJuUVUjzGPd7Ng963bJrkrh8xi92H5CwuqzWKS6VphMtS2D1DbqAaCqG/FS1g1+mIuUNomlS1Lr8Np452M0RmS4Ssn+KCXS2kEmoLEDdnGEX4Uv4S7oCOQ5SSzJryEZ1q/Xu3WDhAmVlXgwq0q2d0cWe9fz6a2VYJ8/JNFVhWPhAxD9I5c7VSY/8B7L8wf+4X/yZwAs/9afA8s/9XLvszP/N3/pJ3ha7THQNalu+eb6hFcG11g08ybnuhpynMkHYj/esGwzUt3y3vqQvaTgMF2zajPeuT3lM3sv+IObM55dTznaW/Ez997l7z/+LAeDDZ+dPqfxhspFtM4Qactvv3zAyWjNfrrlg+U+r05ueG9xwPV8xF969X2+fn3K9e2Iv/T6e7y3OOSnTt7n11++xkG+5XI7ZJJUxMZStDGNNdTW8On9l/zui/ucTxc8ns/QyvNo75Z3n53wE698wLweECnLos55Pp/wcP+Ww2zD71+c8ZP3PuQrl/fxXuE8fP7oOd+4OeG16TUfrfbIopabzYAfOXlCYWP2ky0bm/B8O+HBcM6vf/wap7MleSQBQDfFAO8V5+MFqyalaiNqaxjEDc9uJrx9dsG7z054cHRL1Ua8Obvk9y/PSEJVyL3RglWT4bziJF9xUw14e/qS//3rn+dT91+ilec7Lw9J05bTyYr3nh6xt7cmiSzbKqFpDVFksVYzSBtGacXji32mkw2NNdyfLvjGN+8zO1uSJQ1pCG1qrCFPGqxT3N6OODxY0VpNUSUcTdbcbnMmecleVvDxfMb96YKbYsBym1FXMXvTDa3VWK84G6/44OKA/emGcVpxs80p6xjvFa8dXvN4PqMoY2bjQl7jNJttytHeik2V4JzmfLogjVq+8fSU0bDkfLrgYjMKjGhGbByN1dRtxMFwy4v5mPpywOh8ibWaYp1ydjLn5c2ENG14tH/Ld14cMRqWbMuERwe3PJlP0dpTlTFtbTg+WnK7GqCUJ4ocZSG66sGgwijPcpnjWs3h0Yrb5QC8wpaG4V5B22qcUzRFjI4c/8qb3+YrF+dUTURVxQwHFXUTUa4TZvu7MCpvNbP9NcvVgCRtUMpTlQm21ijjGYwq6tpgW4MrDTqz7M823MyH5IOaskiIk5amjrDbiMOzBbfLAXFsqauIOGmpbnLMuMEuYvS4IRvUwiCuI/zA9pLqZFRTL1IZQOUtSnvsMpGamk3GdLJhuRrQlhFR1mJbjS8MetjiaoPaGLKzDW1jaK8zfOyIpxXtdQ6jBt9qdGpx6xgix/7JktubEb6ISPZK6tsMYkfyPKZ5UKFjRxRZnFPY1pBmDcWNhFbFoxoTOcrLnOyowDlFPU9RjcbnFr2IcAPH/vmc29uRSNsLA4mDVqFakfGpWY1+nkkg1ahBhfAk7xTqJsFHHjWtMU8zmv1Wwqf2a5FZFkZk/kEGD8K+xu/l1K+UUtETSZiUjx261PjYS+iVBlVrDh7dcvXxTI55IXIyv1fjK1m3LhV2JBNeGKmg8dGdwbUDH3viW0Ozb4nnMgBt91r02uCGMtOlSx0mXyRQqz1qoFGyb6kjvo566Wgn6zdl8ECPLKoROXi0VrRjkarXE4epJQTLlKrv23SRx4WJty4MSVmI19Jzqxx96FNz2JI9k5qfbhLFxTIRqWpFeiVhXm0Ilep85M1YpOzliSXaSLp5O/B9Em5nr5AS98C2hXCoZKHFWrFVPQOpQueoi6V/VFdyiOupgM/0Wpi08lDqR+qZky7UYEVwEdT7DpysM1mqMKEYekk3ingpk2DN2Pd9uiDHvBnL5FcHEF0invAuVKmbpNSthEApJ92q7UgmCjppvovoC+5d0rFpMrkTb2RCK1nIddMOILuRiZ0ueKqbHOvWF69koqXrPG1Df6hNZRIUQujUncAwl+wmDifviQdXKnJ8P+HRefjl/YSgpVqY00623nlNXQz4nUy8C8rpXpvedtJ21W8/XgZgHgX/fhv2KRdPcxcohRJrQ3bjewVBH8AUEuNdJOxluRf6PqtuAvCTEzYiS1bkNzLppYIvOt7K711SvLbSwykTD3Ku6rH6RCerNwLctaVfT5uJBNlr1bOZMkkj2wD6iZd440JAkqQ6iyza0+TCQNYTTToX9rEZmTBZ0zG4vr+3mFoet6n4ZbuMgWpq0Nb3kwIu1j2IEa+2+FF9YAC7EDObaeJVCxqc0WjrgvTVYvOQRmwUKsh9lfOoxkp6cEjdVZXtQ39cEmGKRlhJLwAX5GcnI1Z3ex+VEhayG8N7v/vdOflOqBupG+lDhpqdR7K1u27MuzigY1nvdFECfZ1JH/5TNwIqncNX1fd6MrslgM3vCfS5y1r+s+KQ4NH0VfXP9vw/ZPlzYPlnv/xAA8vBm2f+p/6Hf5dtk3C1GqK1p9jKjFk3QB0MK5xTpHHLcjXAVgalPb5VqMjL9WwVybimrQ1+meAzi4od2njcTYI3HmKPzmTQqROLKyNUrWWQWUToYYO3MjhNDwqqmxy0l6RU40FDNGxoN7Hc5MYNdpGA9phRiy2MJLZ6hbpK5GaVOtSghbXMrnnj0YV4XvLzNdX7E1zm0JWSxNGt3JDcqEWvI9zAoioZ4InPUAYdXks3oTBcwvTiFXYs5e0+8fjISZKq2zGVdiil8D63MphcB6Y4Dh5PugGDoh04TCEBR+51kYB1iw6DPB+YVbMyeONxA/m/HVuihcGlni6pFA/tQUN8FdMOHLrSvdRM1apnofuU0TsSn64g3Y4cqlFh4BjSPAuRjulmJ+UxhaIdy3HV9U4upGtFO3SYKsiS1sKO+Ehmw00lf+/Y3I519XrH/JpK9Yyv8uCMsFO66QY0wkbaoUMXMgCuD4RVjzZhsGHEg9oFznTyJ6B/zJvuy0HR1ah0RffiYVSUh503MwRHBdlZJ3HT1Y7Z9kZ+twNh4LoETZvJwDR/ocV3hhzD7txJ0mLwd2WhS7Em+I66lFcJB4rXqpdN9mmnCpHklXK8Oh+UgIUwAEplEC39bTLwk/NFH1ojITmeZKl6NrXbz2QuvtEubKlnQDf0nYt3U0c7lqpboiIM+oJvtGP4YZeyapOQhklI3rTyXrqE2rtJoHcHoKaU96cbqKb0n0fd7j6XfQqrDoNCJQPlbh3aBlAQBuSqBQlokv2LtvTl5MLsB29msrumOpkf0HdUdmFDeGHy03lIpOxSRgnMSqgSMnX3eZTBngQCqT6Z1BTCXMZ3Q2vMzg8pEr3d412xuW5DEFUZXheYTlPugF8XCqVr+azXMxnwt0NCWmnwhiaq962aMhwDt1MMRIWwdE0IlJLjE0rdy13KKOwUHzbbXQcqgApT+p3UL9pde20AC+1Ajsnd60033Web/vh0kke5voQ51rXsYz0JrG2XNBuul7jYAY+uyD7ZCAiQkDuRerpo9zrd7H7HQzZ3ARx1AUmqv1ZQYV8C8Eo6aWisSDbCQtZDCahqBvIT2LHm3b0zXD9tpog3rmfnO7ApT9ht11TC4je5hBJ14EW5cKzsTgHSXT8Sjre7lkztd/e2EPyULl34DAS22HrpWVxbmpFIJ5WT+12XmtrmAn5043eMa7v7jum8txLGE15/h8XqQI98JzjqWRTSgMXPKKBR7tndfbJnOc3u/vzdnxMXOhmlN1F+N4G57D4r2oak1k2Lj7V4L1sJ6tG1+x7wJRejMNEuBOuAMKEq1K9I1yNo69BFix3IY+0w6r2epmixmYTvqE6OeqdD0qUiJb3Lbtos2kk5lbC+LtkBJV3bXTqqkzTYDrAJMFN4Ix7Pu/uO9T1gk3RdLUFCHaunlKyrdZLq2vc+ipcTpXopqzdKGMvOi6nZJcB2ATxtl2br+vX3jJ8KnZUEcGcDIOzSYGH3mrvj+S6M5+556rbVP0eLjJVuNXcY2K7GJIT9+A4Idl7K8HeUlmTZLmQneDhV59Ps3hN8ksn8rsV7v3vNH7f8C8JYPvqP//kDy2/97e8PYKn//96BP82iFFxuhjy/nnI2W7Jdpfxrn3oXE1maOuL4YEkWtzzav2VTpJweLNg7XHF8vODBg2tU5BjvbUknFVqHma1Rw/7xkkdn12hjefuHHjM6W5NMZIbEzCNMbMn3Cnx4TX645Y17l6TDmnivpFpkkDjGx2sevXEBieP8wTVtGZHvFQIky4j8SYQetoxHBef3b8iHNX4dwVElg/9Rw3S2JT9dk51I9xKHFT53lNuEN370Y6KjAn9Skc1KouMCl1v2jldEZ+KdHD9Y4hOH3q/IDgtG95cMTze4w4b4/gb1aIOdtbhcWAhzWggLAeijEn1SYu5vSV5ZQ2bxsUevIkxq8Ynn6O0rdK04eOtaQGKpaM8q9GElX/alYvjlAd5IcbspFfZeBeeFyHUB7kknGtrjT0uUVez90BUucbhxiz8vhZVwEoCEV0zfvha2w4LLHM3MYkcOO/C0U4tLPXbgsCcV9cMamzt0KVUdzVSem11q7Csl8UoRrzRe+zAIFR+WHTgBQlaAZnXUEq81/vUNdmgpH9W4VOS13tCzPeIpUdTnNfWepdmT/alPWspjS7wUcNnmUlyva0XzWkkz9pTnjcyAl0oAT2BlJHVQysjjtaI8bUnnUB1YykNHveeoDiz1nqU6tjSjLmnV00wcLpZ9azNJCC0eNMQrRXViKY8t2aVUZJhCgpVcIn608kgKxrvZc90oilMn/+5ZbObJrjTlsaMZy/ks77VkV5KyLAyGPJ7dKJqJE7n2F1e0A8/wiWL9hsh3instzUiSl5vPbEUyO9glG7vY00wEGDQTx/ZcvuCakcMbGDyXSpbtfUd55Nieeap9kVduX2nIrhXFiaM89GRXHt9VMwwFXG3uO8oDeb6PoB5LKnC159meO2F39pz4qNYhVXno2d5zfYE7elfd4WJZRwcehTkQcOeMPLc8dqQ3nuzSU0/pE3SbsWx3+aajnkiCcbLqUpA91cxTnHjqCaxedRTHfgfypvJ4Mw7b2Be2KtrK4LY8komQaEvfu1keKNYPZd2mhO09GZCvH7pQdQPVTI6bi0OacSbgtB3uenhdLPvqIyhOZaBvc5Hv1uNwbIbCeMzfDgNIL8dle08A4fxtYXqKE0V5SF+bEW+EMbKpJAPXU/l7PVZ9IrENQC3eeFavCFBrByKrNqUEWzWTHXiLCli+DtWBDObTW5HqNkPF8nXoQqfqsQreLCiOpOu0Bz/O9wE9LgQdbU8VxaFMhmTh2rCZCnJMOS4SAibX9vqBfFZMLdLndCFpt93EVDeZYAPQjddeJMdTkc02Y8X6vhZAmMDmXPpry33dV5t0k2fre1rSixuoJroHuOWBpOt2kwKuA6uZBFN1kxam9iwfGVb3NduTEGCzL/67rvJifa6JgyS4nKkeZG4PJSAqWzhh1xYugMadD7GeKOqQBFzuyf60mcLGimqqqSa6B2bVRLE5kXVuTjVtqsjmlvU9kXVWE00zUDRDTbEn0lybqn6Cpgs+6tJt66HIoqup+AvjQs6dKcWTqFvP+lzCSlYPoh6UNQPN9tiwPTTCbAW2a3sYYRONTTTtQLpUXaSIto70pmF7JGxZcSizoMWhoTg0PZDXrWdzLyG9bXGJolp9r48AACAASURBVB1qin2DzZQwtUPpy7SZ7Lf4IDXbkxiXaOqJoZ5FRJtWEm9bR7UXEc8rysOYOqT4NpMIm2oJQ0p0SFg1EoIUSVCQrh2bs4TtSYKpQl9wAM0u0RQniUzMDaRnVDmPHURERZAuDww2NaxeHaKsp9qTkKHiNEM1jvIwk6qP2oUwI7n+m1Ekz6kdzThGVy2bhyMBkBqUFYDajmLaYdwnybrEUO9n2EGMSwzFmfR02lFCM8toJik+CiC1dZjOH5lF+NRQ7+c0syx4ETV2mEpXaBZjh4l4IpMIl8f4LMKOU2EeB4kATY0wjSFIR0KECCBTmD6XJfhI43OZxfPDDD/KRaKaxbhRLt2YmSSqEhlIYkmdVQo3GeCTGDcegta46VD+H+SqbjwUiaox+CQW9tJo+RdqWfxoAIMcBjnqzr/e8xhFeOdQUdT/wxgBgMYIQxmF92ktKkk+CSq7wTp8D6j03vf/ut//yMV5PpEi++fL9+3yA81YDt888z/63/81BnHNbZkTa8cgrrkpBmRRi/WK5y9nDMYVWksylVaSJAvQtAatHd4rbp9OOXp4y+1iSLtMMJOaOLZYq5iOSrZVTLHKpP9xnRCPapxXorKwcgPU2hPFlrqMGI7LXn6oFDRVJF2V2hOlLcZ4osiyWQizmWQN1TplMC1oGkMcW4oXI/SsJopbAauRIx9XFKuUbFRTLlN0KiEczSrBDFtcq/CNRkVeAkZaTZRabG3wrTCz3eSWUp5hXrFYDdDKS99lo1GpxVsF4X2hPThFOq6oXwxgJt2Q+UFBcZuj8xa3jYgnIhX0uUUZh44deIVrVT/FHKUt7WWGTzx62OC2IluLJxXNrUSRmmEjYRqNBiMJuRjR9ehVqJUZW1RmYRlJj2Rmd5K7SuEOGvzWoGuNyxwqb2HRSULCBeTBxy7I+wLDZgI4TC1+E6HHDX6RyHsqDXSMbUitVeEY+SiwwcqTXUSUpy045P9njawXiG5i2omV5wYG2uXCpKpG4QZhZrcU5lg1AobbicVsREpH5EmuREbY5gK8bCqsuI9Cv2UqjLHNHcrvui+FvZRuzexSU57IwCgqJKwDHfyzThCyrhXNvjDZbmgxQZYZrfSONQgz9Xf7KtuhAwPRsjP+y7Gyme9lZj4S4KucTA50LLUpJdW3Yz+6Y9v3Vk4kZMTHQe7XMdpjRzzXwXvo+15Lm3l0qXpGWbWqn+HXIWAkWgtz2+YCXIHeswv0U3Adm9szw+HY2SSkOC534SnJfCe7q/eEXdCtvPdoE86ZEvDsI096bWT73SR0YDGlp5Tep9q9ru9B7BjykBLqQgdol8rsYtleMxK2J15KFVE6l+NW7UlIi1cCProaIhuurY7ZUoFVU1ZYvvJQ9Uxax+Z2LJ8phUlrhvTnVDyb9MfXZr6XUNp0FzbSMXAdI4gWz3A72IGsbpG04MBeBxCWrEKoS7QD9H21Uim+73j9SUUByPPitSQbNyNhvIVVkueZYpfInN4GmV+0Y6JsKvsJd4Jo2K0/KhCWOniXdR2Om6ZnbVW7u0Y6pYFu6atlOua879is6MNgekkk4l+tJyrULcl10OZyrONNNzmg+p5QU/veK9r5WvsexyZMECTCLnZsWteX6CJhZvruzpBCjKdPKjalnPsu/Rh31yd859rt+i/ZnetOMtpJNbtjBbJ+Fwv4NI2cW21l3TaRazoqd/7Vrju131Yt9582VUTV7v124TVdDVKnSiAA63grYTPSlSoA0FS+T2f+BHPZ7Dym3bUbbx3lzBBVPrCFgTEN3tb+2leSMCz+1vCeA+gEObfd81RgXzvPKypcG6nc26Kt6//epQR/Yr/C850JYVTbkAqshTHuvJ+dz9dmmmgjCbySEqvJbmqpMBlLh6WockI/ZSSg2pQueFIFFPeJubo7P0EeHFJufdivzifdJf5GW9szp3erfjp/qHwXWWF4ofeH+igkLVsPdzomJa1XGE9TSseljwTA4/2OzQTQgYXsgm0C49wlz6rqTm2Hczt2s7HydxtS6BuLChUjPbsciXdTNXbHMBotHs6QSOv7pNeoD8mRVNXg++x8lF2ybrcvTStptNYJ82gDWO0GhXCHXQw/29CR+V1/93WDiowwmR3g68N5/I6BtPaTTOr3bOcPZy//yOXPGcs/cvl+YSx/oOtGJknJXz3/PSoX81vzR/y10y/zzfIM6zW/evEWn5++5F8++Q6fyl/w33z9L/Nffu6X+Y3VG1Qu5rcvHzBMa96aXfC7L+/zN/+lf8hptODvfONn+Zm3v8rTYsZZtmDeDHh79JzKxbyzOqP1mp+YfcCH5QGxsvzO9UPemr3keTHlJFuxalO+dX3Ev/PK7/Hb84cs6pzP7T3j1x6/yTQvWZUpbx5c8urwmq8vzpjnJf/Boy/zvzz9Egf3Nowj+VYdRhV7b2y5rkc8KybczIZcrYfsD7ccn7xgHFfkpuEPbs54c3rJ82LCKK549+qYzx8/5+lmys+evsNvzR9xVYxwXnG1GrI/2koy6dNDfvqtb3FdDTmfLBlFFa3XfPv6iJPxih/ee8r/+vUv8trZFS9XI87GK662A9QZ/FtvfJXfuXnIXrZleL/mw/U+j69m/IWHH/Ob7hVmky3bKuaHT5/xZD3j5+5/hb/74Y+RRS1p1PK+PeTkYMmPHX3Ml1++wiBu+ItH7/NL3/gRlHbc21+yGGZo7Xg0veWjxR6312NOTubiOR3f8u71MQ+ncx4vpxwNNxznK749P2K+HlCuE+6d3mKUx3nFs5czfv7zv8v/8dGn+dLpY55sZlysR7ROk0Qtt7cjTo8WeK9Yl6n4FMuIw4dzbuZDDl65ZVMmRAeW9TJnMikw2lHWMed7Cz54ecCXHn3M7z+/R/l8SHnW8G/86O/xKx++xdmnlhzla755dcz8xZgf/6lvcJIu+frijPPBgl//6DX+yht/wONij6+/PCWJLMsPZrhJy+xwzWab0jaGLG9487OXbJqESDluy5yijmnXKa/eu+Tx7YzJoOTlyyneKV55eMmmTthWCWWRYFcxo5M1bx5c8gdP7xEnLeVRgjaetoj4zBces2pSYm359tNjtPGcH855fLHHZFTStAb7zTHNec3h0ZKb+Qi7jYhHNW+cXvLux6eg4N7xnBfXUw5na8ZpxfsfH3N6dotWnhfvHIvceVLjrcJVhlceXfLheyecv3LFi5sJo0HF3qDgo4+OyPcKilXKZG/L8nKEOmpRkeV8tuLp10/QpyU/8+a7/IPf+xzptCQB3KHCPhvAUUVdGdJJRbNKcYlmcG9F9WKMyixfeO1jvvLOq5AGidGxQ12kuMTjZjX7extUSNl9fjWFy5TJa3PmzycMjjakcUtkHJcXE1qvGEwLqg/GVG+VjMYlyxdjeFDTbmJGhxteny5478URnz5/zovNmMvrMZ99+Jx3fvsVjt6+4uJqQjVpSQY1edowfzFGFwaXOdLPrdg8HxNNa+qblOR4S/18iDqocI0mukiwI0d+uqZ4OsJnMlmi9ureX9lOHNP7C1brnOpZhj8t2YwTfOIYHW+oyhjz7pDyyDF4sKL+eEx8b4P71oj6pBX/ZZjcMaOW+nFGe1JBaTBrkbKbs4LmZY5qpHLI5Q41lAmd+LjAvzfEDgTwSxqpZvvpCnUT4w9qoqcp7ciRXRr0FxcUz0eoSuEmLWXk8Y0meRH1QVwA0Vr17KlqhclObw2b16WWaftaS3wd0Q4d0VpT7QtQX3+pwDzJRIp9rSleq1GFkUmXSpM+WNO+M6bec/jEES0iuTYyCRPb3hPJu3LilSzvSyVTfV8k3TaT97l90KJLmdyKVjIx5I0nf24oH0qwWHFuGTw20od84Pr01+TKUB21jD6MWL/aktwYmvsV5nnaS2KTuSIqJKjMJV4mHYDh0xCC1tKHetlM9s0F31s9laA31U2ShUmH8WPH4lRRo0S+vlWktwLg66moJUbvR30vsDfyuupA5PLKEzpSPc3UkdxqksWuTkQ5CdLCQ34h4Vz5S8/iTdnH6Tc11d5OOu4ikSOaMkzKbKX/WBJY78ocA6BtJDzKa+kdNpXGxSFEq4T2ALYPLOmFEf+oCh28A7k+TCnhVc1EUY/pO5O7WrB0DvNPKQbPEDCjRP7tp13qqsbmBMAOi0859r8qg/tmJP7Dck9RzxTxUkCOqVQvXa6nwqjGa5m8GT5zbE8MxYln9i7UIUCu63btFBGmDlU5gbmU3lcZtC8fadKbkGgburfTpQ+qCgncajM5RzYROfr2yBAVnnqmJFzOQLpyRFvH+jyizRSDS0VxoInXnvzGsrqfUk0FCOsmgLsw+RCVPlRZ6d7DXE0iJh81rO/Fu5CrdFf102aKwVWLU4p6In2d3aRgeRDLRIWSUCYQ0OtN1MuWmUbSb6pBVw6XmyDJ3fW8qtbTDoQljgobKmDiHWsc5L82VJu0uSG9FQCtPNQTkfWKlDghuS5p9rMe1KpagF89iUNPKT2AVK1IfKNFJXUtdduDVbIYvW2whznRvMBOh2AUfpztbAl1ix8k8jOATqCX3eqqwaUxRFo6O42Wns6ygSwRcNfJaQOb2Hkr/TCBpkUVIQE2PM9HErSlIrOTvK42Io31XtjLtpX5+yiCbSEBPVrhrQvJr3fAZLe9uzLc7x7sf5cf808DLL9vlh9cTu9PXH6gGcu9t4/83/nfPsPvrB7xrcUxn5294Fkx4aoY8TNn3+AX3/8iP3X+Ad9cHLOpE7KopXWaFzcTkqRlf7SltoZtlfDGwRXfvDjmZLrCecVhvub10RX/4MPPUJYxg0GFAl7fv5JB8maC9YrWGm4XQ37k0cd88+oYox2fOrhkWWd8fLtHbCzjrKJsI3748BnX1YBvXx9RFAlRZDmdrbhcDWlbw739BYsiY7XOmYy3zPKSi9WILG7ZVjH3ZwsuN0Pe2L/ia8/vMcorFqucOJEPfLFJSbIG2xryQcUwralbg3OabZmwN95ycTnh9fuXfHS5J5NhtSHNG+o6IoosbRPhWsXh4YrLJzPiaUUzz0j3C+qLAfffvGA/2/JsPeH+eM6iznn/8RHUGjNu+KHz53z1/fuY1OKuU37+p36TX3v2Jtc3I7yXGgYAazVtbXh4esOHHx+RTSqU8hTzjPP7N2yqhG2ZCNO7jcgOCt46vuDrT8945eSa9792Tnxvw/F0zc1mgLUSOBPHlu0qJfkwg0+vZFutwd6mjM+XLG8HDCYlxSrj7PSW5+8dMb2/wHmFVp7FR1N87DGTBhsYypMHt1xcTtCxYzrecvvRHn5gUauI7HxNscyIspbZZItSnv18y/V2yNXFhHv3bnj2dB+1McQnBVnasLwcMT5as1lnaO1xVkkITCJfmvGgxhhPljTMb4boRYzfr8kGNVFkWS9yhpOSPGm4+nAfPauxy5h4r6SZZ3LD8hDvlzTLVEBB7Dh9eMOLjw5kBjZxDPcLto/H+Mzy4JUrHj89IBtX1GXEYFSxfj6SQdfK0B42qLURdrjSDO6vqb81oTluODmdc3E5wReRBNzUhjhrd8e+NJjMir/ZOHxtMEuDO6oZjCqadya0g8C47tfCxlSGfF8Y8XhU08xTyCz5uKLcJDv2edLAbYJuYfD6gu17037w2kuttZiy0v2C+nKA116O1TqBRqLtSRzxVYzNBDw8ePWSVZmyWuf4i5TkVlOeN+IB3m+hEjadVhEvDbPPXXHzjQP0/S1NEZM+Tqge1FAY0kP57OhS4Y7CF+IqFk/URtMOnRzXWgvbPLREt1JzES+1sN8K1KCVMKaPM2FSBr5nqKK1pjprSF9E1DOHz2V9PpcnmBtRBtixxSwlpdLFHv//sPdmvbJlCXrQt6Y9xHTiTHe+mVlZmZVdVd3VjY3b4PYgLBoJnrBkCQn8ZAy88B8AI3j2O0K8wC9AQi35gZahMeq2e3R11pjDzZv33jPHuKc18fCtteNk0pPAJWWWOqSjM0TEjrWHiLO+9U1lYIjO6YDqxxWf+6hH8aMa3TOL+lOT5LoS/bkft3f+aI2bH5+OHi9pWQVSvtFJXhwRlhai0VBbKgLKG5mYMk4wc7rxcEZPNQCy+FbALQKKGwk5CPSnAWFpgV6hvGCdSHGtYI8DqjcKbhZhTxyqzw3sIjBp+zigvpAMrAnA8MiieG1Gb6ibEfDkGo7ujGxueN7B/LAmMOzI7OrmHru88Ji8YCeim6b7Kk7Qi7sEcmumMYtANjgzjOWKMvIogPlHEtv3PaafKtgZUF0TuLQPmWo8LA8sb679cHWk9PxKws1i6saVGBYEjDL1/IYiQjXsKy5vJbqzgGIj4Ys4ArGoOf76NdnZ4ThAtQLFJqfqErzrnTiw2BNwW3cS9RX9sK4ig1rdUBbtS4LJox8oNE/ISGd2kceAwKl9mFK050D3wGH5fc3e1AVB5n0g0i/Focok+3R7Sp3LW8qcQ0F5s50x0Xn2IjFYijLv8o7ASbfctq8IFs2O0u36MjGfqTbDbLktsycAtnOGMbWPImafUs6t9weWkX7hDKgZoBM0xtcUiRW2NVlul/php6/iyNi2DwSqG87HhiOGLvkKqK8imodMK84+Y19Qlq0GglWzO0jKTYOR9VQDFxb2D+UowZ698qm+hEE57WkCkAmUZs/r9jmrTKKkYsLsyQyaJqA90WNycL+gp9XsAnZPNPqlwPwzDzuh797sYkqJ91BDwHCk0c8l6hvWhsxeDeiX5Dhyr2ux8dg+Myi3EWbnEVLqcHXH7Zp9wLBQqG4dhjmTdmXattl7Xj9tSr1VScrrAuxEj6As+z5FjOiXhv2qDQPgfCWhujAG8CAziUisqA2wC436sy3CpMCwLKF3NqX1OvjaQFqPYVnCbC09olqO21K9JwgMgJ+XUK1FNIoAsdCAC5C9RZSSVSKDg58WUNsOYVKQ6bQesTSHypDMTt4DYaKziBVzPUQ/8DGFIQsa6F0dw3u+BCxHP2ROcs11I/nxKbwnti1ZTaUAqYCQH+8RrWMabLpfKMmAn7x9gPeFCGE0Yga5f9ot3nvO/8fbV4ax/C9+Bozlf/WXjOX/75sUES/6U/xkc45fO/8Iv796hhAFWmvwcXOGRdXDBoVSOVwNU8xLsoHfenwJLQP+6KfP8N13P8dK1Widwd94/jF+vD7Hq3/1EPY7EhNtqSpwEvOqx3eOL3DRzbHQZB7rgsmXv/D0DX7nR9/Au29d4s16jp0t8eLuGKWxuPv0GNP3rnBzO8Nn9RI/ef0AQkSUpcXxtEVjDZ4sN3i1WiBGgaO6g5IR+67A4DT6XiMEgeZmAne0RdMV+N0Xz1HXA25X0xFUxigwnXfoOwM/KPhSQoqIbjBoricwiwEXnx8DOmDdVZz8TgeY0qE0DkfTFjfrKYSIePZohc9enmJ63qBtCoiJQ11a9DriajPDXVOj//AI5q94vLk6wmzZQoiIrjP441ePIHWA3xkcv73Cb3z6bbRtgcWixWbDnod2VeHR0zvs+wKfvDinvEgF7K8mkHuFy9UM7rqGOunZlacDursKf7B7BrEx+ASc0AYvcfE7j1D+4gr7qwmlufsCRW3RP1LAroTcaIQiYPlsjdWrBYQVKE48WhnR9Oy6Wn2+AKoAMxkQ68DJ8psSmAY8fOsWIWk+pQy4vVyw561ywJVBe1fzH9GgcLtigNSuLeGchNhqvHp9DASBOHewbyZwAVi+u8LmoyXiyQC3M5g/2mLnJTswNxpOBcQ3JTZFRPl0D39VoF602L1cQJ70mB212L1YoLUC8lGP7z1/id/76VsIXqG4VvRHnlkmmKbQnQCJy5sFE0t3BUSn0LycUQo8SLz84QOgiBg+n0L2Avu6YD/eVsGdWggVAKmgNhp+7rG/mUAWEbAClx+fIk4cyguNwQvojcTwQBK0aQI2u5BQe5WCZALMRsLGAg0AZQDxqIP8tAaMx/PzO7z+35+hlRFyqyAuJ9Am5WB9uADeps9WOCAGgfJG0p/6gyVgYkrbjJi80GgfUY4lOwFczhDPPPRWwfcT6IFBRqGOiJEpoVEAxa3Gxc0jDKceaiehLAEGosDshcTeGfgnPcRtAbPhpOv2B6dQvYD6/gzujJ4x84b3u1u+rj91kHcFZdcLC2wMJaqLCNEq1BcSzVsO5lYzxVQISp8jUNwqDCl0zM4DzFaiWEkETbYjmAgxSJgdZ9N9EaF2Ek5FVG80dEN5qbUCkwt6FH0dYSf0TVc/qVLqpgBeVAxfkvSbQkaEImL6qUL7kMe7+/455CMGLw3LA8DNFTrSCujXJYYjTnAjElvlKd11kzh+lRdkA8s7+iGDidBbMiztg4hiJYF1mWpoALPTCBqwR/Rd+grQK826m5qpoNU1J7WlI0sTL8yY9llsBIqVSiEuEfVlhJuyUqgJFXQD2AWBT/Ynlit+BlTXmgBuRrY0poCk4u7A1MlBjIFa5Upg/5TnS1qgumA9VVRA9VqN4VsAF0NmL+iT9KVAdU3wVnYMPhKOIE2n+hy9J6iSA//enZC1rVYC9U1A0zHgrLqSKO9YvwMAi08CulMGg4kAuHnE9HOZqo+Qaq14LoflQTasWqC+lKzJKViHolqRgrJ4LYRSoLxTKHYR5YcRm3fI/kWdpMoKye+agnCCgN4zmCafg1xjQvk8E2WZ7koGlownk2nL25iAQpaiCsw/jaPsOWhg+jnB22QbxgRT3RIs6pY6b9OkUJ7ZoaplWCQGryJwY4ck61PKdUCunSKYizB7yo11Q5CmLCWzOaQnp7TWl5SkNucS9a3D/gFBmt4L1DcerpJwFeXF0zc8DvOX6fkh1b4YiWrFY9ofH8Bt9mWqPiYZK38u15GgbEqrwDCTKPZMeFV9RLHzyA30+X06/ywwBdYwvEZ3EWbPwCLdH+S6kwsHN2EYYLUK0ClordzwGImQzxtDj6qrAVEUED6mcKXsofYACBJFZE9msXYEOi0QtIFueHzMxvK4pn0tVgOiZtorwJoXOZCphI8MB9rbMWRHDim5NYEc1RFQqc5TEhs0Q44ExsAg2XuG8UjKvM3aws+Z0qZbdwCHLoyyWuEC9KqFW9ITqe9axNIALiAqBRE8ZOdG6avo7CGMCJTRwgMIAbJ3EG0PoRQlrUZDtAOEFAR9OfwnS01z6E+bZK5a8W/dMAYaIQQyqllG6+8F7dwPBBKC94d4CPuJEbAWwhhEkZPMvihrFVWJmOpORPJ04kuVI0IpQKWxmC9BkvtMZQK+MUYC2a/zLR6sEj+Pt681Y1m/9yR+8E/+IbrBwGiPCOCt5QpaBPzBh2/j2x+8xI9ePcT5yQYXPzrH5PkW3kuczPcYnMb1yyX0IjFog8JbT25wuZkBALpXU8SaoTtX6xmcVWMNQVQRYuKxPNlhdTdlrH/kJFeZALcpAB2wPN8RjFwtILYpqjmFzUDHxCxRwy/nFmFvoNcK5fsbNLsSZW3R7QqgV4CI0GsNd2ZhLg3ssUOx7FmpkGRhcqO5bRMgTEBsNJaPNyz1NhHlpUL39gAM8pD+qtIEcqMggoCvA1kJAZgrM/7z8HUgk7c/pJNGAbhnLHrPUrnJZxrNNweolYY/Yjqt7FlIHgXGNFRfR4SpR/nKoH86cOwLh+KNQbERaJ4E6C19ce0zh+q1poToQRglBFFHzD9SaM8j3IIprnpHBijU9NWF4tA1qjr+Q8jevvpCwi4iJp8nyVTH1NMIoFiTTYmS3j29lUlOw226RUD1WtFXGAHp6LMTnuXiw3FM6YypHPxWjB2NbkowIAf2FrYPA8LZAHldoLhjl6RqBKavItYfsKsy+xezr1F1AuWafYq+zBM/Tsql4+OnLwV2z5iEmhmY7pxVCJmJyAxSsaF8sboSsAuMq+WhYIn5sGRPqbTA7psO9ecads6gF9nn4BFOstghh3HiJSL3e/Ka/qruFGMfZ17ph0g9mNXBZ2V2KW009SOONQFTjAnHAINj7PTgX8oVA0jbNvuDL43l5ny93IuZPVu+wKFDr02JsB5jNUF1kwJxIsaU1Ch4LdcXhwTNKIBhyYk4wG0PR9zmfWlif5oSdhUnz75KE/uW12iW/rGwnmADIDMkQvIzJhkcIsb04lyLESUwueD1kWVz0iGF02BMz1XdoTqBwIjsjZvw+JUrntfqmpN/4Xne7ewQYAOQmRgWSV4WkJJmye7kscqBMjRWY6QQnZosgG5iCj4SqVaBTFR1y2L07J+LkpK3PNHO6bRRcEy6jSg3WQ6ZfIfpMyMzb9KSWSpXcayzEJ4AYZizDqFIUsHciZn9jnZ66JJkF2scPYnDPAXDJJ8kwSUBC4Nv0rFxZK+Cwdj7afZIXsZUMi+S9y8lpUbFY2D22TfKv+cgG5u8ptKT0RsW/N+iu+R/TNvLFRTFNqBfEsiqIabtCxRbnlQRKOuUqQtTWnZe5k5OMljpPafpa6Q0lF6/6s6jOdcj6FE2oltKFLsIxDhWS7iSzHEU9DkiklVU6TrMXZwmdahKxzGUGzJvw1SiWnmOMZMlKYwnaJ5TX/CcZgAZU42ELyS9h5WA2QV0xwQ2Zudh50x8DZr35Z7LqA7XHo8ZazHoNQXMxqE/MXAVuzC7E41i4zEcaUgXUd1QRulLOXZuqiFAdSHJMCP03o+gzE40ivWA7ryE3nu4qSLojhGyD3BTBdUlgJR+z+mjsqe/T+9d8isKdKcakzcDw3pK+iTz9SUHBuHIIaC4adA+m8Gs7dhpKWKEN3JMjs1eSWkp99R7B+ED7MyME+fiao/+wRTKkjFUfYBMDJ7eWwbuJE+iHDzDbKSA2lu4JQ3J5q6Drw1CpaC3A0Kh2Ps50VCbAWFi+FyBFMhj4afFYZsClJgKQDUEmKFmCJPoPWTTIyzq0e8qrB+rO6Jm+qze9ojJryish9y3CEdTRC0hWzsyh1HL8fgLHyHaHmFWcXEhV3UoOYJQOE/Q58OBRczhOrn6I3kio9HpupOUpubnWgeUBVnEzErGSNlqiajZ4AAAIABJREFUTpwVXCAQ3T0ZaQgEfvkWAyAkvyt1+Lv3ECa1E9z3VApJf2f2awKIw3BgPzNbmYJ9vnzLPsz7P/+pmOTnqcfyyfP4zs+Asfzhf/3VYCy/1sDy+BfO47/zP/59bIcKLkrU2qKQDrfdFEZ5rNoahXa4vF1guWgwOIUIYBg0lAootMfgFLyXvO6dxNGc7JsPAne3MxwtG3SDgXcSpnDp/SEhRIRSAX2vIQRQFA7OKXqXnYI2qbPOS0gZYEyqEklhPm1TQCm+EW1jIAsPqSK84/K+TFUoSpOdKUqHYaCsNXoBpRk6FAEEKyFNgN8UlCL2GpARceCHoJkNCJ5jK0qHvjEQKhIoqywtkICMMJWDTfdLk3wLVo4BPpARsaHk0e4Nu/z2BnJmEXoFMx1gWwOREnNjrwAvICYOMckOVe0ZMrTXo/wTVkJUDP2JqwI4YgosREQcFEThoUw49A1OHMLejHJEFIEMmYjpb2nMJgC9ghgEYslZv/AJYKb9E2UA9noE2RCA6NiJJzr5ha49YQXi1HOBIbFJ9GlwEhcLLkWprWJnX+khdlxUiJrblw29c8IKhhi1cgxZiUWA2rFyRe3ShGvmILea9TNOQDjBBYQyQK5ZxB0VECsPBO6falJwT/JPZYkhkJiDIu3nIMZqEtXJ0QPGICMgCgb9QCQAaeIYLJJvuqFML1QBeqPg5wFqJ0ejRNB8TizSeIJIZdcRxR3linovEMrk94rs/tNretLGrsDkK5MDFyoQUrVFcdg3gvtDHxsnYAR/XDVn1YxushfpAEh9ddhODhO5nxKsOo4RgcA+FAcZWw7PCCVBeZbh5ZoZ3aSFCkE2L8oktTw6yBNzDU2UcawlkAmIjvukDwsMY8efScEvPh3DFLKTFzx4TgGV9jmH5oyhKTHVzYBjy4sQukngOoXK5NoPX3M/VJ+AZeA2csjMeEzy2yYrqxJ4ux8yk6/7MUylO4BqXx0e5yYcc17oEhHjey+HvbiaNSvD0eH85aqJLKX0ZVosuBcC4+91HeZwInGvXiTX2+RKhvHaFwdAm6te8rWQ71fdYQwA4Ou06ILDNZ33hYFLSOfh3uvlW3pMHiOPy2E/c2XRuPh0L8Rl3C93CBZSycMoQhyBWAa9/ExhLYqvxLgoIGJMoUyH+h/p4xjKkxcTMujmvieWKBfWp4CcHKTjKvoMc0BOvk7yeQ4qsY/pGOZFLF8c3h/5es+BWzn0Z6xqcYfrJe+nCEzeHQOS7gUHZeCcF4pyQFFQh47TXHGSQ2PuV3aUiR0NRqSAIaQFkPiFVNpDLUi6Fk3qdxSH6yvvt+7j2Ck5Xtt5X+MXPxPyPuaxisCxyxQao4YAVxEg5kAevWcNiQhxTGQVIdfMiC9Wp6SOyFDkCpbkb0zvz/vXEADIIYw/j8FjAmO3JKtQGOzjC5lCp8IYUhSMpGdRCejOw1UqJfKK8dxHCUAKqNaPIFpEMqJffBy3qXpWkZChJFjLv+dwnCwLlYNHSB5L6RLwizEF8gT6IzPTmFlDH1m5koAqAILJDFpz7ckhWwcj25jeK2MnZQKGX2DwQiTQTOE/6AcCSoAAM4f23Je3AgSe91lJ9Sf3WI4gOAPc/Nr5fiEIMjNgvB/6k7sv89+8v7dvXwSYMYcN5QCiP+v28yKFffI8vvOf/wyA5X/z1QCW8s9/yFf31nuNT9cn+Hx9hFVbIUSBZ5MVFmWH3VDg8XyDXVfCbQr0TqHrDSrjcDxvMK0GJsUKsFj+YgJTONyuKJkdnIapGNAhRMRs2qHdlxgGhe6mhtYes6rHfNoheIndqsZy1kDrgKoeUBgHYzyKwiEGifP5HlJG2EGjawuEXsENCtNJz3L3KBCCQBwkitpCG4foWaYuVcB+VUMIoCwtxCUDXcrKoiy52lQUDrKRcFaRQRQR5tpAVh4x8DSHRkPKgLce36KeDICIkDoSBAJAx45P0SnEXsI39FvGTlHOOUjK9zYKdldAbjhupMmW0AH4aErVRemY0tpLmDtuV6V+UL829OOddFBrDVl4IADToxZhZ1C/osEeawN5VZANbTXczhBUbxXTZCMgBgm9TY+XEWqjx6talD4l04K9l51CcaMwdnp2CsJJxMS2Ci8gWv4XL2/or5QDJ0wQAMoA1Un2jRYJYDQSonZMHe04oyquNMGrDhCNph/TM+hDWAmzkYCmDEz0EuWNhD/KM6nD9Z1leBgSIN9LlNdqTDVVN4YA1ZPZFbUHKk/ZJzi2XMQuvED9Wo69o1HH8TjEIiJWlACbDe+XvYTaEhwWa0nAJROgmrsRZAoPVNeJrbOcEMTao1gLAqZIMFSs+NrCijEhF0GgvCNgUp2A3ouDNFBTGgkJFCvuB0AAz2RMbkM3YgR9iIDZckGBSbLclmr5XLOT8CUZ3epajPuPyO0AZLJzj6nwQHknoJJk0Ozz5DBNbjPQTAwkGQKmxIaCY8zHoFgjMaEEjmYnRpDCbWHcr9xRyn3i+TM73q9aTmBDmSfuHEOWQ0ZwHFED0fB1gwGEJegq1hiBpm6Q2FAeVwIkMvj5Pe0mBOQQTFsVHiMo1XuMyabCs/syT8wz25qL3cvbtFiR5ySe2zY7HPoZc+VTOKS8ikDGevJGpAnyYfKcewnt7ABEhiN6EvWe2y3vmMgZEziJJgHGPcFJdR1HcK0GoFhFmF0ceyVzcEsGn5mh0+3hnBVrPkd1Saaaqmby+IsNt2ea9JiWv2cgaHZxPIY+petW13yMSmxjsYnQbZL9rSnLLNcB0gLNY753s1ogJ7xmBk23EWYTx3RXETIo499Un9jH1NdqUrOVtBHFnq+Za1SAe9tP9SgZVKqe2/IV5aSqz77I+IUFgJA6XTPw0y3BqwipM1KI8fjk99uohMgAJgEvSqczOOV5dokJzsDWNLz2GNiS2NXUcQqkc6X4OBH5GKTjqNtABnjIzHhKf97z/0ref/ZksroHAFQbRqBlmnA413v+zGMVx8f4xMQS5EaOzwNm61mDs/JJOkw2NoNplcCeTPJReS/xVQ0ESEELyD6OIFLaOI4vKIHy1t5bjOB+hfQadiJRruwIJKMg6y0HMr9coBEwm4GMcaqbYeqvh6tlCtwR46JIDsQBgOKmJajdMelUhAjTuCSPTamw+tAHGRXDcISPY38sgTaBCcGpg+w9fC2ZcCv4vBz0k1No6SskeJatZU1JkrqybkUiFBJRC/hKQ1o/vu7IbN4DQlEJIIFLMTheS4WC6Bzg4yhzFS4c0miBUfpKuTSBHAExZak5kAcuAUgfACkhup5hPkodFgSMYihQiCMYFIkJjVIkGW8Ci84Dgx27PPMXZPq6B97iYJOHUh4qQqxNzOU9MPgnSF1HYPylhNr7NSN/oVsGlX8e+Py63OLP4OsrcvtaM5ZHv/Aw/if/y6/j92+e4s3tAufH2zHZ82jS4vXlEr/yzmf48c05dncTTJctmm1JoCQBWAHUHugVjp+ssdnVCE4i7jTE1OH4ZEdPXadgzlrYXkPIiEdna7y+XHIQa4M4c1BFQLBk0/TRQOYxCMirAuFhD6wMHn/rCq8+O+XzIljT4SRM5eAua4jTHtELxF5h/mCH3WpCJq3yZOMUP7DNYiBr+roCZISfJdlMqs3QGwU395g+3GN/W5MN9JyAm7UEvr3FcDmBGFhvISpPNlEHiI0hUzdxkBflOIsLdYC5kxieWpw92OD2RyeYvrvG9s0copdcbZ17HD/aYPViSZYyAqfv32C1mSB+NoGfBIKsVh1YsSKg/qRA+5aFWlHu6p71EDcF4ukAeVFyclcA8WkH+aKCfWhR/7RA+9xBeAF90sFuWU8SFb1lxVqifeKgt0x69DP2dOrHDYa7Cmov4U8szOsC7ihwUqsiigsC05BYCj8NiCrC3FH2Go8tyo9KDCcpJOTMo7xSKWAlAiZCGg9lPMRPpgjvtsBnNSeqkd4tiEhQZgk2cdoDV2VKeZQY3umgX5ewxw7wgqmQ7zpARhRHPYa7iuByYWE+KyG/tYP/6QxBAZPXAv1xhJ9E+CkZxJgmdfKDHfrXE5g1AaCdZ8AHtG9biFZBtQSp/QNH2XPDkAl3xP20c8p47cJj+kKjfRgQTwbo1yXKW4Hd+xbV5wbdI8cxejEyw3qjRiYjy2ur99fAby3ZN9mxVzNOHCY/KjGkTsVhSY/acOy5PzqivpSws4jhOGD6QsFXQPuUPs8soZVDCopJNSDdAwaPiACIf2MN9+GC9SEV+zEnr2QKLklF648HiJ2G3knUFwK7dwLq1+zs9NMAs1IEnR5ovzGgeG0wPHBQO8XFgiq9dwzlz9IBCIltjgCiwPwjifYR0zmLO4bOSCtQ3tC7Wd1E7N4hExvmHqKVMGuJ6obhHZl5yoEr5QrolxirRUJi/6QlyxsMMH3JgJOok/8xyb2X35fojwXaZx5HHyqsP/A4/V2J7TsC9QW335+l5NGYFxyQ5GdA98hh+qmG2XIMEATqxZYy6/rNgRWzc2D+IuDyr4HnJALlDbB/RhDZnwDVJT8qm6fsitV3GvNPgN3bvM67M0qQg2Hf5vKHAs1jgcnrCDtlEMrmXYHqGmgfUtKdJdfdWWLLphFHPwa275Ch3b3jMPuYQTLlLdCfcAxjXUbB8JbmcQq1ScCTIUBMEvUGmL/gpHPzDQbtyJ7JqvtnPL/LHwDbtwVmn0XsnwocfRSwe8I00cVHAde/IrD4KT2jiAS2+Xxm1tfXHMt9r057zgn84hOP7XMFs+exiDqDWHocXU2PYXUd0Z2ThVM9gbidJuAik39wB0wuAoa5wHDE+6avCZB2z/h+mrxhL6WvGOhz/COP/UOF4YgLEsWGr1nfBNhaonnEQJ3ujK+9+DigP+L+Ty4CXC0wuSQw2T6T0M1BHlzsDtuYvQwjM1euA4YFexerW586TdnNmWtZMsPcHfNayLJi07DOw5cExdM3Hv1SHpjuJE/eP5U4+WOH7kSh2AU05xLFlkAwLyS0ZxJmT59lkUJ1CGTpe9R9RHvKCfbslUN3rFCuA/YPFCbJY9kdU37dH0sc/6DH7bdLlKuI+trBNA7daTHKk4eZJHj1GH/WbYCdKpQrgoP9I4Niyy7O+ppeS7Pz7Ml8bDC5dOiP2B0pPQF1e6JRbj1kHzEsFEwTUKwsRIjYPa8IuocIN6G3sVhb7J9WcKVAfePQnmoUu5B6ZQPM3o0grz/WmLzusX27wvyTFu3DCuWdxXCkEZVA/aZD+6iC3nuYrUVU7MmsrwbYqYbZO9iZhllb9GcFqsse0gb0p1ViKJGCePg5KHyA7BzcvITsXUpHlfQXCmBYFlzYuGoQCk3J7c5S5tunILpJwRqSQkHthzFMJyoFPzWUyhYasuF9wgeElOwKJRBKDbXrKZ1thlEKG0oD2fSIlYHYd2QdfYAYLBACwtEUoukRpxXDeIzmfUoSHBpNKWwCb2MgjxAQTYdYl/x5sIBj4I/IXkop+dz7zGTGBFme69yY9MoPQ4X7wT4xRqDtyDgqdWAd832Z1UxS2v8X8/mlXss/9/bzIoX9z34GjOU//mowll9rYDl9/3H8tf/hP8JtO8HN7YzSTh2gS0cWckXgBYDSSID9gKmzMOp4kDBWAXBklKQjW+CrJOEz9OtFCcoRM0sX7knu7kmkwoQyR72XY5pg7oiTVsDNCFZkz9TDLFnUrcBw5iAblVYkyWrlDsLsVZQ9HxsVC++TwuOwP0lqEsoIs5WwU8obpRMJKBzYHjuL40qiPXOQW4JuldIOc4eebgWGJX2MwRxWvosV0yLNiuyWXTCYxU0TK+YFIBnGAeRVasFagJQSmdlCNwt83TqivJWjx1G1Am6WxrznRCuY5HFK8jNp07gEDl7J9I9euiTva7iyHgyP4XBCH6duyDC5SYQ7caheGso3DQM63JQs1tgvF8h8yIHnFZIT/9zPJoIY5W6hINNh50nKp7mNLIXMCZpmc/Ap9sdMfNQNmTg3IfOVUwxdndMHD12EWeKXZXmZXcjySmkPUsUsvQvmEIyRn3f/NaLh5NqXqR4APLbCcUKWmSIWoycmxKT7shdRI3lNvyhRk44eR67oH2RtecVftxiB0xj6YpG6JtOEMLESXIUnW1VdJwYtsWwQ7LbTmUFL58VXfF6xiaOsL3vgyBZGuAnPT9RkpYYjAZ32O7/H1IDxMyB3XsrhsN++5Gv74rAPvk7jk4DZcGKv7o1vlOTdk3jl7WfWB7h37btDiia9ejxXWYKoW/rt3CT5PJNcM/dk8hrlc3zB6oHMApmGx0d1aaU+h9msOUnNXric5phlhVnuyvd3khsOZA8pfzwkWqqery3tvWOgD/t4X37HPsokF5SH4wV88ZjZOcNlfMnfs5e33FBOmLcfTJK7ynQddxg/r9Vw+Bn8OIa0qTNSIE1Y+Rjp4sgk5XGM0sQE3vP+ccfSdTbEcTIPkLnSKYQmM1sHOSBQ7CJceUgEzSmhwZBJyp8huuN4+D1ddwXPkfQ8X94ImDaO76/sjR3fU46TzbFnUfJ9Mky5XXmPmBi7BuXhvZ79mWRrA4aZHI9hlt7mW95W3k4eC2IGfgGuph8yFIJyyRDH3+/Lk7PkMwfZ8Nq592KRrKZ0B1aP7y+R/Jb0HfqK0sf7kmT6GmOShvKxuXsRSV4btRhZP9nz/0IeC+XIAaGUI9OYU0t9SXYvXzfBCMg+1VjozNKKMc3UlzJJW5FkqRJyCOP3qEUKoMmS3OQDDVn66hCNhC8VVO/paZzxA1iECGmT53OUX+djmTopVZb0RvoWTQKm1o8AksxZ/iy+dy0pVm1kGXHeLllHeiRFiAilZqhNzJ8jIvmqCVh86oQUPrN+BHPCJ1ZTJBlroSBs7uJMn4+lPkiIbUAsFYN0wNcSvSf4Swxc3v+oKWPNKbH573D0QEIConf4QqdklnmGQMCWwVcez70eyeyxFO5w0Y6/3wvTianb8gsdlhkw5sqS/DpKfgF83vc8jp7J8cUCRu/k/b/ntNcv91DmJFjvASHpscxvR6XYY/kn+Cu/fPsLY5GfA2BZP3ke3/lH//qB5Q/+268GsPxaS2FDFPBRwgd6DqtFj2eP7uAGDecU5XSLHpPjlgzc1CIUYfTbyY4MV6wDymUHFAyu8TMPexQw/eYaYeGgN1y5DDNPhnNuudpfBsSUnhhmnvcDoxTPLjxkKr6XDpDPGkrxegERk7dNgCxeTN6n5JlTHf13whIoqkaQ8ROAfWAxLCKGI65SIvmeggH/iZUEXf7YoXvgDr4sGWGnhwRHXx3CFxABc6WhG4JXl0ro5UCQFFT+MI4IRw5yIPNiZ9zX3E0nkkdNNQTEsQyQJ8Mooxx9I1MPX0a4WYCbMHVS+HuAOZfQJ19ZeZsBONCfeYbizAjOKPs6TMCjRgLAafJseYzcJGJYBsrNdPIEthyTnfOfdfXSUKaXwn7cjCBaWr5Gfxo4cYwEjFEniZylzzAqbqt97iAC4KYhyejEGAQzHAVKGM1h0hnKdD5ymMwgIPsEKjdpH4t4SNlM4FEEYPd2GENbxjCNIn3dmzTuvmVHn1/2wAVD+WZ/yoTODMZ8HaF39NjZWRpbL8aS8GGRwN80onvsEqDn8fAVQW8oCJBDAQzzg/xv9OCBTKlq07WbgOhwFKG6A9jIkjkCeD4GgcfSVwkspfePm2CcyGTJX54UtudxlNraeZLzGYEcdKMT8xc05XzdWcRwHHme0wQzmC+ChQyW++ODfNJXZIV8xQnzsODxjZp1CsMi1UKYw1cOLfIlEniLEI6SxzxZH5YR7YM4Ts6zhFS3/L09S5ULU7I0bsbr3yXGKvsFM/CUPYOQ5JACc3puz9V8z4+eQWCcfNspx9gfSfhSjJ4+X7LWgSwKmR9fCtg5ZYLZR4mQFzHI5ohAoMh6AI7d1axbMG0cF3NCSiKlXDh9ft3zBdo5QU+/TJPdlDiZF1LclJNSV4oxzCiDCYI37mtm5Oh5i6OsNRhgmAk0D+mH9jXDYAjAgX6RVBsVATyTNgPsBGjPeD58kY8dQ2t8mfaj5Pm1df58JHtG8HD4HAsa2D+SZBtnAt3pIYgHOIT0ZE+pN/yyEwFXc79Hia7ndeZK7sew4DkYr2sjMMwl2tP0eZ48kf2C15IaklRR8vzlxY1+IdGcy1FqysUHpCoPAd2FUc4pPfejX3Ky6wuONYcSqfYAPFwl03Um02dXCuRJckfKzMMIGlUXR1DuU7VGDr9iiFre1xxEJ1CsWF3Bz1aB7lgxYEmL1I8IDFPJyoyFhLRkIL3JC8wigScBX8oxgdQXfCzHJhkGU8qUtprAV7jnX4yAnSpup1aQg2fqagKz0oUUriXT/qSQKkdwyo0gAT+eJzehXDKUcty/7CGMksCqP614ngp6De1Mj97HYDhmOXhI68dFq6i5r3aZmEAt4KYGsZDwE41QKIRSEdglsCdChK+5wmLnhoB2So+iL9UIHt28OCzqDA6hpI3F17zgQ2Xoa5QCyN5IRekqAPZM+pCAYIDsBvja0L+YQnkyOPJTg1DymOfwHRESEA1grUfyWHLH40HWCl4/yNJcS4AZs6Q0BfXEyiDWBUavZKrtiGX6m9EQLvk+7aHPcgzgkUmumoJ5DnLX9F3KA0hNslXEOKbBQkqymzqtbuVwnj9JWhoDhBAQKgFMgABSKj5P5W0qAtEEKnku5Ch5zWAx3pfB5vvTV5bE5hCfcUx/2teXkmf/8vbVu32t60ZCEDDS49cef4z/89W7+PXnP8TrboH/8Okf4IfNQ7xd3eJ/+o2/i/d+9RPMHvWYqgEX3Rzvza/wzz7/ZqojkSiUR6kcqocOL9ZL/IN3fwe/u3kLD8sNfqd+G//ev/kh/u/bb+CP/+gt/OL3PsP3P3uMx9+4xsRYzE2HP3z5FHU9oG1KHD1e43vnrzHXHV42SxwVHX5w9wCDU9hsJ/irv/pjXLUzPJ2u8ZPVGW5WM7xzfodPPz/D0fEe3z1/g4flBv/bR9/FLz18g8tmjn1fQMqA9XaCo3mD752/xm998g38vQ/+EL/x6bfx/ukV/vDlUxzNW1xfLrA83eHxYoNNX+HRdIPf/+wZ5tMO69UED843uF1PUdUDrNWoCovearTbEpNFB2sVkLoj4/t7qNKh35WYzju4bYXJrMc/eO938L9+/kswyuO9xTX+xZvn2C4qhE2B995/jc9ulghOwfcKv/zuS1w0M9wZB2s1xOcViu+ugFWNx3/9NVyQmBU9pIj44cuH6HSJX/u3/hj/4vO34L3Edx+/xg+vHqDZlYhO4sHfuUCIAm9uFzAq4G/9zZ+g9Qa/9+YZ+s7g5GiP2z84J6iZefzKdz7Gv/z+uzh5usLt9RxybfBX/u0f4Xc/fQvLWYvqbYvtbz6EeXuPv/32T/BPP/w2YqsBE9CfcmKzXO5xdzMDooBcaey/x9SLzgvU8x7bdYWnT28BAJ9/dIYn717jW8sr/Kb4BRTLHt1dBTEIPHjvBoNT6D46xi//+g/w49sz3F4uMFm2aN/MEGsP+ywCO404dcBVAXdqcfyhwdVfD3jyzSsMXqFQHq9+eg44gfjNBt98cIuffv8J/aACCKek9v7G+x/h//qXH8CdecjKY1IPaLsZhsceDx+v0P3GA6x/0UJtFP7W3/w+/vmLd+CDQLcvYKYD9jc19EmHo386wc3ftnC2gPzGHuGPZ9C/skLz4yP4KuCd9y7wqX+E4gcK9oMGgxcIe4OT79zh6sUxpt/e4GTa4PVvPoOdRbhzC9EpqOOe135xluTpEX/3ex/i5X6JH508Hvs3yzcG/bNUYF9TFl5daaw+cHj4zi02v/UA/XngotHbqcsTwPzRFs2mBu4K5KXp5jFgjzyElRjOHPpGIZQBkMDR9zV2b3Pie/6dK7TrKexVDfvcQtwUOP/gGrd/eA577KDmFuJFDbdkyBKedHCf13j4SxcYnMbm904xexnx9D/+GP/qkycQtwUXVgIB8LBMCa2VxPCtFsWPa8z+2jV2v32GYRmgTgTcJAAKUMc9wpsK808kmicR3RmBsbCUaqtWcmHomxs0f8ROUBG5uHP6777Ciw8fYf6xxO6dwGTgMyC81WL+z2uYHbD6lQEnDzfo/tkZ2ocB1ZXE7nkCqw+A8o7gp/lux0qjf3UE6QT6U0/VRVowKNbA5hctZj806M4i3JlNgVqAnFrMfrfmQsWCYL15IjA8GiC3XNCycxanbz6gjP/2wwUBqYuYfvcO++8fswJCEQACBJvdWUSxIoDbPwsQXmL3VkTTUDGiO/ZQCgfMPuPjogGsE5h+HnH9KyLJxhkiYucC7XOLyccG1Q3DVprHybf5qyvsf3uJ/jRQRu4PbHdUrOdYfByxek+Oi2jCU1I7LEVSIgB3C4X2qcP0Yw3hgeahwOQNJ2FXv0xPcfa/RoWx17J5y2H2kUZ7TsbQJ79YZtYzo217Avy7X4oIE4fiUqd9lyNznQHp0ccen/8HHke/V0ANwP4pqJBIi0h2TvVNMEw7BoD2jJNS6VLn41SOwA0Ado8V+mMCnu5EQHcAArB7pqkKUMBeCxz/yGP9DQU5UIbdPAsYPtaQDuiXCvV1xPInAy7/agmdUnPp9RPj4ll1RRDbPNJjunJQIi3UCPRLqiKq64jmjB8F3Ql7IXPvZBRAfcOFAy5mSUwvQhobMLkMuPh7Axb/R4WgC7QPBYIqML2w8KXE7okelQbSUQbrJgLVjYSdCpiGwJgJw2Qam3PJzsZtRFAaw0Jg/jnQnvI509cBxc5j/7TCMBNj8nCxDaNSYPO2GZUQxT5AtQG7x3r0Ivst60W8EQhaE2gXGvWtR/OI8s/uWKKYKyhLNtwXAvUtpcyqxxgAVN153H53gsWn3OdguFBT3bGP0s6m8EagPxKoVmQVh7nhgoeNUL2GGgLaM43Ziw7Nk4rvnccTXj91PS4K9ifTtHgW4epyZPTNhsE86/d5P/srYwLjclQ/xKVBubLojyYhCv6oAAAgAElEQVSHvspQIkoBP9HwlYTsA8JRgeqq5eKTkQjHVVJzCKi+gNoNGB7MCLSnEpNXHULJRFo5eMh1g+6tZWKjI0G3lqw2MQTceu/gphpm1ZNZlRKxpsxCaAk/LSBbhvqEE7YS5HRZYT3CpIC63TMFVkj45YR+0ABgltK+AgAlIGxBQKsEZbPzelxEkJsGiBHheA6x7yiNLcwBOAIHz2iMo8xW9MOB4YwRqCt+tw7QCrHrDxLYGCAmNcODvOe2hSBzOQwQRXFIjeUknnUjAKL3B3B5z4OabwdGU33960aAg8ri5/D2tZbCnn/nNL73T/4Rns5YcD8EDYmIZ5MVfv/mKb59fIHf/On7+OXnL7GzJWamx10/wXm9Q+cMXu0WY7flm/UcUkY8WmxxsZ3hZNLi05dnePrkFouywyc3Jxh6AyFYaD+ddwhBom0KzGYdSuOwa0v2MSYApmTAvOoxeIW79RRSBZSlRduUqCc9hkHDGI9h0Hjr7A4X2xkEgEk54OLqCHGQOHm4QYwCs3LA51dLTGcdhkGjKBy6zqCqaOwOQULJgBAFpIjQymPflgzXAaBUgB00TOGwmHS4uDjCo0cr+CCxbSp4JyFkRAx8vN0bmKmFKSgrjlFAyYB9U8K3DNyJXkKogNArnD3a4G49hR8kJosOfW84xn2B6CSqRY8QBIZ1iZPHa2x3NUzh0O5LfuiriOgkVOFRlBZ9bxAGBVl4qtuiQLgtIE8G+LXB7PEOu6sp1MwytRbcRrCkUMpZTym0CTC1hbcKoWVJPABARsi1hnrSjEAEyfdZn7Ro1/kDW9CLW5BqkXuFuCRw06WD3RcMHwIQdgai9hArA9UIxHda+OsSceYBL6AmDn6QEI2GWAwIVjGYRwdARYithn7Qwt5VEF6gvFawc/r5ICPkTo0y61iwf3JMrAUYwDLxZLzvDAGTAKXfOqB6UaB/4PmYFAgEGYEiQF8UcAuGDMUqQDQEenHmoG7N2EFHto6BPygC5EajvJFonzqgCAw/CgwOiiqOITe+iggLB/QSZqNgTxzkXqG8luiecMxRUjYmBrL1xQ33P+oIkRJ/ZfLzugXl5Hqd0nedoG/zKPLYTBywNinsh+x6eaPQPaYp0Nwyst/NUhefJCNM2Tt9Y2ZLWbabsBrH3OqRuRWRTLivyH7rRoyMsS8BPwsoLxXchLUq97seizVZrWAi7NJj9pHGcHxIgjQb1tWEXCM0ZFkkwUb7iBvSe/mFhFbVMsgny8cBjBL4kPyU0pIFzxUlOYiHk7mQKmUoVTc7if44YPZCon3AfdRNkjnOmaCbU3uFI+PpakrIVcuxqu4gvx6WXP0H2CXZLyOqawHPelv4IkINrOcpNhxnlDwPI0A7jdA78QXmOOiI+ooAIjNxqge684Dymp8NZpeYbmCsnOmXBM3NE25TRPov7RxjBUmWVmf1hIiJrW/ym46Tet3E1CdIBnhYJIn7VqC+orQ1M4+6oRzabCO6UzF6Oc2Om+yP+TMrRZCCjihbN3sye3nfhyMxVvvcT5Idw4Aq+nSzjNqXfI5qE6Of2HX6TVM3bGLhVeoqdHVKQrUMGooqgTHNceqG+2G2vK7IjNL/JwcCU5Xko0z+jaPUff7Soz+S6I4JCkwT0Z3IkW0UPjIRNqs78qS/i1BJUj2qEwxZb+nY7ajbfM0f5OVZLaFbbj9LikXIEnvKhKMCqlvKb8l+Y5QUDwuBch3RnohRLRFSijMTYSO8OUjCs7pklI5GHiez9eiXamTOq5XHMOXnm+7DqCppHiiUm8AgGkmwXOwCVJ9qbvYB3YmC7iLKlYObKLLIKVhHhAizC7BTAlpXSph9gJ3JsXaGVSNJ6pqqYHQTYGesGdFtCgIyrFcxzb0AnRBHeXK5cmR4vyQDVl2qRWkp0VV9wDBX7NucSVS3bpQCi1RlolvPczRNgXorC2/49/ZBiXJlx3TYnPwqAgOY7ktxyUJyfMXajiFADHRKPY6gBDXUmr/Lg/Q4alafhIIsnbSJVXRhTHkVNgBJGhsVJaQMzQGiIdMLgP2VRhH4uRRUNTj+Ld0nB4exuqSgFJjSXHeQtI7sn+DvShHIpYTX3HMZCwO5bwEhECYVRNeP7CeAkR3lB+M9yS5A8Og9/ZUxku0EEBv2aAljEPsBQquDnDbLZe0B+I0YI4URfeGWvZr4ErD8M24xRsS+/3Mf96fdvhJS2MfP4zf+03/9UtgP/7uvhhT2aw0sp996HP/+//zv43WzgBQRPkicVnv80eVjvH18hx9fnOOts7uxduRmM8V80uNuPcXRYg8pgKZP3TxRQKUE2ElhsWkqvHd+jU/vjuFSHUmMwKS0KLVDBHB5u8DpcodNU0HKCCEi+l6jqiyM8mi6EkJESBlxPt/hdj8Zx971BjECs0mP3b5iQuyig/cS7abC7LjB7maCh09W2PcFdtdTvPuNC3x6cYrptEOf5b7JkFFVFl1nIEWE3RSQE4fomXg6f7BD1xkczToMTmG7rlFNB9hBwzsJbThj8k7BFA79tiRYEoA2Hlp7hMDEWdcrTBYdTmcNXt8cQarAUCMRxzqV4CRU6eF7xVCj6xL6YQNEwbqQIDjuvYY4Ggj4Iti/qSKTX0uPkMZmOw1VMG23azmr8o2GLFnR4jYF1NwywTYInqtGoTpv0e/5+NhqVqK0mr7PCMwfb7G9maKYpYoUHSBkhG81e0C9gNimVFcdCaKTjxdbgzjxBNhpn6TxCJ2GmfcIQcJvCtaN7DUwT7M+GRH3KTV2lgJ/tgaylfBTD1QBcqUhH3VYLhpcvzoi0L2cQEw8q1Z6iXBkIfY6pZ8KRB0gao5FtIrgqvAcZ+UBx/RabMwovwbAflUdD0A1hRixbiSy87STTJ8NPG7Ci4OcroiIdapfqQJyJQ28YL+qT/UuToz9qAiAXmt2o1aB4U+dSGxEknwnmbjoJRN0B0GgbNI2B5FqOwieVCfhE+jOnrrsZxMBCFUEPHgOND3H4zaS1/B+N2WYhDF8KLN/uZIDSCFLxyFJa5O/N41Fpu3m/lP7wKJ4ZZhIOg2j3FivCQz1nkyzm0RUlwwHyn2vuQ+NE9QkF2/IwPmKQUoHTxxGlmkMdImHiW0GdfcrK3wdx8RdBAJt4ZJ03xLU2nmEbg+Ab1im2puI1GEakT2beifG+hc3paQzS2xFYCLuWHPiU7VKktxGw+1l/7AaCFiDIVhl0iQ94tITLGdmb1jwWOQgo3G/krw6+5TtnN9zd2t3ElFfCrJcyf+cE3hzv6qveNyzd1b1SUY84TGXjj7eLLN2M94vPYH1sMBYO5K9yzn1Nne/Zl+zrwh2cwfm6OVMnlxvDuc2L2RkqXQOsckJrDnJNlfGZDmy2R2AWj4+udJC9RjlxsEwSVgO7OEEKJ0224O3VSbQm497HndegNANQUeWNGafafZl+4r7ZnY838PiHsBJ1+kwJ5DzpYBq4yj7t3MudOiWYLDYRbgKY9fl/XoW1qIcjqkvD55HV1NCnnty82sPC4Hqlu9rX+EL1Sjjcb/nXc8JyARrmVFO4M6TDVKWcuL7VTnZU5q7ZqM6eIBzYnROi71/HoMSMG1Av5Ao14HyXnU4xmNVTZIHZ6BtGoJEXwjo/gBY8013yYOb2EjKctN1GGK6znhC83FWXfK+lnJM3dVNwLBQo89VDocU1VBwESP7cnnQkRJ/BXQb4CYE9XrvEYxEKAX0ziOUEsIlb2pHnyvfewFuolLdiDx4McFxByO/8Hv2DTPVmIBKdR7RyAT4fAKAafy9QygPIr9QkJmktNeMoUDSHryQblZA9h5ycPz8TeATjtLcWJrx9/G+8QWyT1Qg5zcAOPhE78tLv1wXkv2a+TVCHAFk1IpsJUAAmKS1X3g+QL+mkgepbfZJZoax6yDKErHrDs/NE2XvgXAv9TXVjYyey3uvk38fOyz/ApUjX/e6kb8Ell/h2/Rbj+Ojf/xfoqwszuc7XG5mmNc9Ll4toSYOR/MGt9dzoFM4frrG3asjmCWZxroeoGXAejWBMgHhTYUwCZBTS7lXbzAkNkrKCNfq1NmoOQGfOdZ0xARmVDj8vFOAAjCzMJXDsC0wPWnRvJxBHA8QEvA7DdkqhLmDvjGUjQFkfCIAz/qF+E4L1+rE6AkmvDoBfcYkVKgIsVfAwiF2CnJqEe8KRBNRvWaJfXjQI3qCPVM49BcTiOWAYMmeAYktioBaDhAvSSG4UwtZelT1gOZqSuDXKpg7Cf9uB/lplRgLAfG8gb+ogQCEOkBMHcRNgfKGnpjmmwPgJNROwh876GszdhGGOkC2ks9bDsBlyb/1BDSQEaIIwEYjlgHFpcbwbGCi6zQHDPGfMyLgFx4Igq814/OLKw07D6wHSQEVZi/ItJkAc8XAHj8NBEQBEE6iuFaQDmjfoWSvuJXonlroOz3KIP0soLhJDFgNFCsBZYH9W0xSDSYmryZDjMoVkyNDSTasfkN5Wigj7JwSu+GE2wxFhH1oUb3IZc/86k85cR6WAdOXEs0jsg72xEMMAiZJoPROJD9hxPwTCTujFHGcVCr+XF/KMdLfzg7H1NeR5zDQ+ygHQA38pzUsmYxbXQPtI/o/M7s1eZ38ZxOQTemB/ozA02zJiOm9xOxlxN13mYhqZ5zkCw9s33OY/1SPTNn9MBTdAO1DeoDLW4HtNz2KW4VizYmvnabJA8jICU/gMXshsH9Ohm/6ShxkmTuCme5BJCBInujqhiv3zSOep9ln3LfyRowSxe4MKRWTHkfd8fx05zyGIYWpmC0DdAg20iS5j1h9J+D8t3n+23OmdAYj0J2nzxbN14uKUj4A6M7pL869jm7G67W+zJJNJpBCAsKRSYuKIAGJcTN7gge9AwNAJLB/RgBtdmTQjn4acfcLTGAdFmk/LwgyfMVzmn2vubqjO+VxNTvep1pOAM0+YvuO4HPAsTSPI2afYUy37E7FmGo7ecNQIwaExTFgp30ooHc5ATeieShQbPnz7qnA5CJi/5h/G44I0oICynUCR2nBodgShEgLtA8EZp+TVZxcejQPFMyOx61YxzHUinLDiPaEqZ8ZiKqe33NwEhldvkcmbyJ0FwAh0J4ImIZSzuahRH0V0J5LVLfcj+qawUBuAiw+9WjPUoJxwCh3NC1DcJi2KbB/KJHrblRPT65L3tb6OiTPYgJ5CYANRwQw5Zr7XG4iVB/QLxQ7Lh330xuBcuPRnCmGIaXalHLrsX2qoQbWrIgIdEsCOzUQLOwfShSbiOrWo3moGdbUYwRv1ZpjK1ecPHcnCsWeLN3+kYLqMI7FpJoVVxEMiZiqW/qAbplXjxKgbcl6tsc8dsU+0itb8r3nDVmaasXJ7DCX4+KLLwXqGw85ROwfa0wuPWWUAISPsHON8s5i+1aJcu3RHylMLix8LWFrOXZb6i6MzFxm0YaFSioIDzsny1beWLQPChRbj2Gh6PE80lB9hG4YHqQbj+ZhgeqOE2k7Uylgy8PsHOxEs9ZjwnmD2TEtlaA1oF/SJ1ldD3BTDeEifC1h1hb2yCAYgeqyR3deorxl3YbwEW6qofcOoVIIKVwoGAk7p1ezuhoQFYGk2VL22Z0UqG4HvkapYDY9fG3IRNqA/thAtYHnb9VjOC5RXTRons9QXbTwUwPVOnpqJxpmMyAUCr5ihUZ53TJwxwe4qWHFSamg9hY+hQ5J61PoDn2Nh75GwC4MzIaMJRUnlGoKFyA61oO4RQXpAiWohUpstBxDfaAlRGIYxeAQSwPZDEBk1Qd8RKyYDJtBViw0pac+MKHVefocU9iP6IcxgTXWBUSXQJ+m51P0AyWpKeFVdMMIHEfGUalRmsrJG18jasXHh0AA6Ty+UP+RbzmZVcrx55i+C2MQrR3ZxBHUCYkxtMcUlKfG8CcCyvy8ETzev/1ZQPJP8VP+XADLf/gzAJb//VcDWH6tPZa1sjg73mLXlbBewTmFXzp9DS0DVk2NwWk8eXyH1z89R2811NxCKUpZC+3QDQblxEKpgOZEAb3E+Sm3JyX7HRfzFts902WVCoiVZ0vGxEJrj3ZfopoOOJ3vcbmiPt7GAnCCQErxjeG9RJx6erm1h9cKYeoByZATVSbWsJdQCwu/NbAlgK0hcyTSKlQZIGYe4VWN4lkD22lAKhSTAb2tICQQFwS9/RmlkUXl4L1EcJIeysTYwQvEykOoyPEGsrbDkQOCIJAe1OjjQQQB6b5CDIBdkmGDFVBBIhSUdMqpO/yzriLkBzvITgOVR2xKCBXhTi3lmnUgW9bIMUXW7AWGY49gUtJHAD14muBxOCUD52asL3FRI0w92a3MtAUBv/AQReACWgFEE+GfsMpEDgL9Cb0XCPSzxf+HvTdpki1LzIS+M93JxxjfmC+HysqsqpZabQgzzDDrFcaCPcaOBnYsMAN2bFhDA2tWLPkdLLRC6kbdNJJaXcrMysyXL98QLyJ8vNOZWHznXI+sLknQaplVycrNnr2IcPfr5w7ufr7zTXWAPCoE5SFbgv5QSvrZOtaW2AUn/KGKgA4IZZJaaiAWgJ959I2A2UjE2qN/HKEOKskgCbZsSrjNCaiuAcazMK3chgKQiYnxFVlD1yQWsaKU0pdxkpMen/HLOpQ4XSchAdU5H99fM4gkp7sy2ZfMkoiY0oF9lY5TCcqYFg52TKuqgkAsDGllPySp6JzsVCwoHfN1wLiiN8kuwySjxcoCew0ryEqKIJKkLkB33LavcgqvnIJxGG4Up3DBDIpzGAoCWTJfCowrvt/MXsIuQyq5J1AcLjCFAAWd2Kck00UaE4Jk8mMvCKBy4ElxAhLBELTk7sWo6dmSPrFDBZk2X1IaGnqm+4YiqYUM75OjgOok7Iw75mYRoxWndFTO82AXnJQPZ3we95/gLiqygkISzPqKclKXQnaKbQrdmZN1y5P7KAm4xlWqfllxWwyc4ukel1ypyR2Gbh4xdjlBmV8e+Zp1FcN7XJ0e2+CBFI/jDSnRmuMh6B8XSRa646p8DorKzKCrKNH0ZWK6DBAXeQFETEDFNkDuMIzmxFbnvkVX85pxc/C90SMBR17z/Rm3dXySw1swVXwMyc+ZJaa+oldOOiQZalqokalXMcki83FwtZySPccl30i5UB4CP5BNhhROdXyskmw2TkyU6iOTSiXQK0mm8UE4VZQMAPJlujYSMxcVtzGlFxuGDFnH8aoRsLWaEnT5e1o8mkue/5ieC8CVMrG5YmICcyCPKwFlT2E2w1pNxz4zjUjhOb6gby2DchHlKXwss4cCKcX3FCyEiNTVKCdWWaQaEUSgX538rb4AQZ+LsDmJNjFjiKfFKogEvGsJlRZZQyFgZ5rBRel962a8PsY5L/zuSk/pwTl4KGgBUaRwKwHKaBOL6usUQmQYzBO0wLDWDLgq5SktV6gUMJSY4rQPuZvT1RJB83M5WDEl49qFToxn+o7Pn9sLQ5bWcnxRy2nbvuI00JfydC0pPianwuZMTzlGhBJwjYI+OviVhogch3QR3kgIzW1BlFOSa4iYAtmiFHDzAlEJuFWZ3pcF+yqDmtKHcydjMHJiBaMSUD6kfZSgxNOkAKMkfU09nsHIqTNS2MAEYCUAJRB9nBhCASCWBIW8NiRkCCnEh9JaESP8jGFCKgQEo6Bc+m4TYvIEZmksg37UScZqNELNv8nUyQmRpt+FwcMwH2jFfsl8v1b0TRpNf2Z+fPJOIkQC1Mi0XUhJx0++PwfwxJj8lPKUIAsktvYBW5pYUJFrQoT4oUQ1+SmFVoj2AfCbtvOAgczb8x5Ca/7/q8Dlw9tfE87zd6bH8u/w7TeesfyP//f/CP/y7hHe3y8QI3BxdkCMAt1o0H2zwPzjLZQM2O6aSa5a1SP272eo1z1m1Yjbuzliq3H+bIPxDy5R7CKO/8EB9vsZWabLHh9c3ePrP3nKL5iVg9ABsVfsgOwUqvMeUga0uwpSB0o0N6lbcWkpOVw6FN+UGB45VBcdhtcNZZZHjeLJEeO7Bpg7xIFfjKL0EHcF02EHJrS6RyPqr0p0zxz9ZY8HRCfpmRNA/VKj+vdusf3iDMUHR9hRA6/LSeY3+06i/g/fMeDGsN9R+CRnUxFilBDLkSueL+kzZLBGnGo63NpDbxTcykMdFNSLI+JXM9gLh+YXBt3jwC7Min7DMHcoXxUYno84v95h/INLVmp80GP+xzX2n3pWqlyxt3KSIyZf3LgMmL2S2P/IoflOY1xyldwtA8q3CsOlp+cvAMWtYvrmykM4gfq14oT40mH+pcG4iinUgYySPpxYHzWc5I5mJzBcBMRrdkxO1SBpn/3KwdzqKXE1qJMcLfvWouJk3X7SQ76qJhYw159kxmk4DxDnA8LRUBZqecwRKRWkPy2geqsQyojhyrNj8kgwPntJFss19AvqjiBj/o3A9if8kJ99o6BbYPcpA1zMQST/E99LxS5N6tNxyZO46j2w/1GgRHArUeyA7vdbNP+kwXAWp7RKbhM4fGpRvtUYLunXrN4oxFRrUt5FHJ+nSXWu5kj+J6bQElyoBFy7RxHlPdM0s48qJ7qq7iQfLW8J6tonEbOXIkk8kXrgyADJ4QQMzA44fBSgW4HqXZoQJDmmaxKb3SawooHqFinIhkxgf0FgzmoWyi85WT+xi/01x6IGTla6S4Jw9mZi6tpcfB2x+Wliw+4FhnPK86r33D85UvrUPk71P6l2ptidJIi6A05l7UwszfJHc+DYXSNg53y82UcM55x4SguMZxHVO3Fi8hMLtf2MnZeu4mdH85YMGwDMv/M4POckWx+5/cwQsisyTkmauQ7HHOIkHR2X9OQBJzCWq02ad+xMdBWrT7orgXJDxvdhgIuvBfSR1+CwFpi9JgNX3bEKJddxuJqLBLpLnrQu1V0IYFgBy28CuovkbWt5XsclezC7RzxOzRteS+UmwjasYcmyx2DI5k11Hfla9RH9maTc07OyBsAkz4yCoGxYUerZnwlU9wwk6dd8HwOY/JJ50u9S2rDwp4TfqBjCwj7HiOXX7D7UHa+LcuNx/2MDc4yob9kZ+bASJm8H4DUwrCWq+zDVnug+Vdp0AYenCqtvHMa5TAmsvEZZvo7JU8uFH56P2WuL/kIniTrPbZRZbsoFvu5KotzEyWtoUgVOdyExe+P5Wgm4+TKx2XsGxyAiyTblVL3DOi5eBwBgDp4ADASbmaEcluyWLHYedq7gUkhOlEC/JhMZFNC89xjnlJ2W9yPswuD4mIxgsc/MJo9Fd05GN8tH67dkDH0pUG4s7IwBMiFJVs2Rzw+aXaDlnYOvJYalQn3r2NF4ZlBsXQKGYvJBBiWgRkpCXS1h9vQAZi+jHAOGc7KFoRDQR49xpVFsHZCkoKZ1cLWGr8iyqiGg2Ixon1aobi1co5I31TG5VSQpaOfhK4Vibyf5qGot3KqEbTSKzQi97dC9WKG87TGcM0BH+gg1eAQloXoHX2sIf6pMYbpqgGs01BCgege7KJIMFAzOGRxTXhMrLHtL4CWAUOoJ1Jr7HqExENZDjqxCESkVNqfWqmNSjNkERHsHaAlfG6iOLKfsLKKU8IsSetMizMofhOwwDTalpYZAEDlYiLZHWM3IBveWfZXWk8kcHZnG5F8UbQ8oRWZzGOnTTCE6Uy3LYBGXMz42hMR62lRj8hDondhHSMnHOQ9YMq1iYE/m1DcJYPJZAhBSkrXMoNBQMhuH9IZy9HyKqkI8ttPLhraFKAqCUaMRjy1TX4sCsPb0OZNeU+ikmnOOP/91mCQEhIfy2/+ft18bxvK/+FtgLP+H3zKWf+ObGxX+6NULtNsa6JnQdzOsKA1dW8gA7Lc1QdLrEi4xPN3QQKmIcTvHCK6wujOH+6/OIZ4GdNcC8bsZQsMSdG9r/OJdzdVUAXrxbgtK3Ur6A8duRtlO6qnEUULlXsm+4KpxYnjMvYLbzaEdEAZ668IXc2gBxGPBCf9ZgNwyHS+OrOFwTYS6NRhXAeYuGdp/UcHO6cnyTQQkcPjTcxS9gI0zBo34DHI4wXv35QV0pE/MJ2YMSJK2QSBsq8k/JTzgcgejTJP2Jw7yPb/MpQPwxQzmIICo0V8HlHcSvhIQXmG89Gi+KhAV0HxR4PjqArIkkCj/vMbxWUD1mqvz2FeQI9A98ShvFZyXkCNYNl+AAS7pNSlxlEkaKSHvTwXw1Z1Aea/pZ1K5+5KrwsIJCMmeTLsMMHsxeRd8QXmp6gheijuJcKgR1WlCHyqJ2XcCdmtSWAkn9uOaLF95LwiaZwGzb1gQXv6relrZ56wTUwUKJMGJvK0ZmDECxw98WoVmJ2ixBYRTk2+sfKdZ2F4TyByfM7GyvCUj5esk8XsUYbZyWuXe/K5D862ermPdnsDwmHonzZ4TSdek+31E873EcJYqbEqg+ucN9/VWYDwDxicjqj8vUN5FDHca5Z1AFIqeOnPyY5EhTIB1w+qN9pnH8ucKvufqPutQBKpbTOA3y0nZ/0l5b7FNUjl/SpwEMNUeREmmQziCSjUQyJUbTJ2dqkvdj4mRKvYE/Znt6q8pM/ZlStsMSIEkXOQBkrxUAH3D68zOeUxn3xIkB0NZZNRAdSMe1HskH1rBvwN8jeY1t6E7srdqiBjXYupXlI7dotLFFJgTp1CSKFJlis4dhwnARx6H8i7JX3tg9KnDtOAiiq+AxUsWvrsZAVux4aS/u5QwB4aqZN/asJapI47HSo08tzklNDOXmXmVjq+bgZduWQfSPpJQA5L879SvOJwJzL4nUNYd/WdyBModQaAaMaWbZq9j9kVm0KWGiPaaYEX1nJDqNoUoJaBrZzyGIgFzOyerY44cX7E5HUfpyJKWG8pQkY63Q5zY9ZBYv3JLFYEaCLqnbksFeAVU9xHHJxLz71l/VKRwFV/yOhRBJIaK505ZAlr2ESZQWwuUO0pjsww2qyCGczMBQlcJhrukYBsRKBcdlgLFnq9r53wd3ROI5xCb/DdzDHCVxLCUE7gjuOYknQ2NfgIAACAASURBVP69mOo0gPrGo19T5aEsCKIMa1ZixNShqTvWuMxfs3LLNiK9j+nbizrJiw2/f9gTmiTh51zoKrch/Z01HEzTpS9P9THV1ETWZVhKPUWFaT+UTQsVZfLlRV4D40KiOMQpkTTXgajeU26KJMkdGaSTvW12LiepbWYNfa0IfiyllfQiCpiDT0FJHv2FQXnnACiMKzJQykaozsMuNcceIor7EeNZCZFAYdQCMdUy6Y4fhNIGxCCghtQpKUDJrFQTIFc2QLiIcW2S9SNAOAU9eggfMVyUKO8cPYiB1w18hNmNsMsCuuXjMHI70nvYZQG7MDAHR5ax0YjFDPpgEQUoRU37r1qHsCwIECsN1TtEaRC0QLEZYVclVM9+zVgqHo+jZcBOiPANnxuNgmwthPXwq+Qn9AGq8wgpKEcdk1w1gzkQfMrWQo5+YkPN+xYRinLWzkIKQf+jIHsqXKBfUiQJ7egYppdkt6K3U5elHCxCaSCtm4J6APD6GywBYwrFEUN6nkgs9Wj5GACxKih3lawdiVUBZHAHpDAecWIo/QP2c7RAVXK8KcQHQkB0A18ryVOn24OE2BjCxDpGHwCXVrpyTQkACEmP5YOeygwqo3OYfKDeE1Q+ALv/mrT2l36ebr9lKH/jbr/RjOXi88fxg3/8X+J81k6VFf/w4gv84f1H+PLuEj86f48//YNPMT6xWFwcMQwGZ4sWx6FAZRwezfd4uVnj88t3+L/+6MdY//gOtXGYFwO+encBuy/x6Sdv4ILEy3fnAABtHMbOwFQO58sjbu6W09+d1WhmPdovVoiPBywXHa7nB3z9/pyy2X0FU1sY43F830DPLaQKEH8+x+zfeY/NdgbclIgXI/2bNyXKDw6QMmIcFYrCYxw0qnrE8VBhPu8RATin0B9KxFGiWA0Y9wV04xBfV1DP2ymJ1g0KP/3oNX7++hpXSUJslMd21yAcDCAjZpct2pcLBpxUHrOzDjECQ18gBiDuKPNdf3yPzbdrhtIIQOmAGAX8UaM6S3UctwRUeqcw/+k9Nm8WEE5CrEaINyX8PKC64PbHtw2aZwd0xwLxroS8HOB3BrKXUI87hCig/6LB+HEP+a7A7NMtdt8vgCpAHBTE2YjQa8AJpno+6uE3BaqrDmOvgZuSDOiNpudxyc5S01jYI2fl8qgQ5o7BPaOCmlmor2qMl57yWie4ULCy0N+XsI9GqDsDP/dQe7KJWDjIm4KR9T86ovyjOQ6fJC/mkwFxUJh9aXD8fCCLrQCz4co1JCA/PMK+q6EPEvNvge3nEWFlIe/NFNrRfC8xnEf4kpLB8r1E/8hDDgLNp1vsb+Yo3miGxaTKiljQmwoA/uMO/mBQvtEYrjxQMSVWbzWqG4HDzwagUwwMKD3Kb8vkEw1TEikCEM8t9OsC9VuBwwumt+qDhL1wmH9B5lofyKrKUWD8tON18m0Dt/LQW4X5S4HN7w8ovi9SoAg9p4efjShfFhgvPfSOUmDVJ2YdmJj22TcKx+ce0USoo0y1CzGNRSGkcKNQ5lRYfnHNk3+THhqG04zXDnpmEd5VTDVNLFFUwPjIYv7zAscPPPReYvklcHjBxR6AzF73hEmuxUZiuAyo3kouktwp1G/jVONA1jWxsp8OmP9ZSXB6GQk+GzLUDFGJqN4quFnE8hepv/A8wq7pF4agF1q/NyjvEuMZCXKjIjvbPaLEtbwn+9h/NKL+qsD8JXs/fcXFkOHao/5OQw1Adx2x/Aq4/z2P9b/Q6K/IZpf3BL5TUE3Dc0sfKRNW3Syifk1m05ec8DZvInY/5uINADRvJI7PAs7/H4H9h0DzVuDwAftq2xcezUuF4TxC9fSOhrQw1l/x9SYGT5Apnr0E9h9T9uyqnBIbKRuveRxEpJSvfSJQv6U0evF1xN3vAPOXqX+yS2D7ELH9LOLi/xY4PCOIVD0XJ4C0QJFCn3QbYZdk1stN8rQKeopXPyeotLMkzyuA+l3A/oWcroP5dxHHpwLVDaW64zLi/E8jDs/l5K1UY5z6P+mfI1DurlPX7VHA7NNY5ilwqs++VTK1xZaLKP0FFzXMnos7i28ChlUCbkmCmkNWqk3EOBPoLyn9LXYR9fuA979L/1n1Pk6A2+xODLSdA/VNRHPjsf9AIwoC5v6c12V9QxC8/Mbi/vMCrgZWvwiUZy8JDqdOy3gKI8pdpdVdnNhWpsVy4c+0BMb00MdpsSWok++12JIVtTN2SebtcgEkJ4oyLVW3DJFhgmySBy8F5t97HJ4ozN54evDPucjpai48mC6iXxGEm5bH3xcC5hgwLiTq955KhEudklq5SDCsJOq7QGA7l6hvHe4/LbD6xkK4iO5Kw1UC1X1AzhTQfUjjk6jfWdiFmhY+ogBsIzF/NRKcjhHjUmL+bY/dxzWkj6jfWXTXBtWtm9RCAKB7D9tohIIhOVEKbluRHfZlUlUkqXB3oTF/NbKjs5KobgYMlwVrQfqAcaVhDmSFi63DcKbRvOqx+6Tm+Ob0i4oQ4So1sai5r7N6N8AuDcq7AYcXDep3IxlMH+HmZgrr0fsRodZMjB14bKSPBO73bpLW5hoSs+nTe1Mh1Jog2gf4WsNseth1BdU6yN4lH6iFXVXQrWW1SGcJLAHIdoSfl5Cd4/dnAopy9ASKkj2m0SjIbYtYFgzSSXLXWFPmKjv6IEJVQLYDYqkhDz3CvILcd2Qf2x7hYgnRDohNCdGNiFWqMRkchHUI8wqitwSkpYHYt2RHM5M5jIiLxKaGU8ovAKDtyFJKgdgPEFUq3k0+zWgthNaIbQfR1BOTGZ3DlPb6gEHNjGQcT6wl78ipRPJXymBjiBBSnCSwMfz1rOZfcft1YSw/+c//7TOWf/Y//nowlr/RwHL+2eN4+d//11itWwgRMToNJQNGq9FvS5w/2uHu1RooAtYXB7R9AdtrLFYdrFMpxdXA7w0lq41nqMzzDupLAgq1tGhmPQ4vlynxMRVkz9KqyygQZx71Lwr0jzjB1RsFv+CHljwfEKxkNyIA2REo+dc1VJ/YTQBYOoiNoe/o3DOBEwCcYPLjsx7yZcWAmZVndUWnIPrkmZQRceaBQUIO8uRPmQWoIxMz1ZFBMzlwRi1H+L3h8/OtDFAbDX0UBB3py0um5M5QRZQ3TMQsb+m/8WWEPaP0tLiXGK75c/WaH7bdMwc5SgYV3TCVFOCEsNiKVHrPCSNX+gQg4uRLs7MIv/A4/2cKhw84HjeLKLYEHLkUPKfyiQAmcu4k3CqQaeqTbyACZivg5pw4b3+MqWrA1XFi1XK6otlxkjFcMAmzuhXorikntdcWzZcF7DKmagWGqlQ33I/jcw/zEBRJMjtktuiFE6kAflyRFR2uPervKTvKniuZ0xM9UOxP7ImrOWbdiuk1QjqWCIB0AvU7sjLjWUTzisxU9lmqTkxjLzb0x+VJKuW4QHfFceZ6AjvnBMxXZEyLe/bvHZ+y6iIH1pgjxzuc8VybA9BfxKkyY7hgX2KxiTi8ICAAEjMYk8xzSMxi8tpJl0JnUil9lDwe3SOyd+WGf8t1EVlKKC23WewYkqJsksZGegynxM7cOpPCTszhxGLZBWXBdnbqFiy2lMfmMQqfmTOCCzPJNgGRisZzMiMlbzyOzes4+QPHJVBmNtZxX6objisqoLpPoMRhqobIyavSkq1GYPAQwOMxrtP57E7MWpa92plAeR9Pjxu5z5y88rzqlsepuiHDFApKe3XL7fiSrxc1pYcxVVOwtJ1sm+rpwRSe25c2hd0cmJ4p3QNG1BOE2BlZouousD4hAZjsNSu3ifHsGUQzLEUqeE9hSSvuW2ZJIRPwmpFtc5VA855BNLlGo7oPaK8ZPGNnPG6hyGDrJEf1BaYKCnPMSZmY0jLzOFWPSRLpchJpQJJrkg2ev/bYP9VobjyOj5miWd9yfGTr49RbqEbKeJVloE17xb7IYsdgLdPS59yfC8zepICcbSAwVZSAulpM1R7jXEJZBgzlAJqgAD0kUNaxyoIJpHEaz7CSia0jqBiWrKDQPa/z7lyiufFJDSEnYDysJVRPYMI0VnrfxqWekk+HFaszbMPjVd15tFca9a3HuJQTw1zdjjg+TpNdgamexRwYhiMtvWy6Iyure548WzP8SHgyrOXGwc4JxmRKG+VnFusucnWK8AwCsnMFOdIvCZBZzWwoQLbUNWpSPkgbU3Ipk1L52Uu2alwXlLGWEsX9CLsyEC6ivO1hVyXkGDCuyADmgB5fS+ijhxoC7FzDHB18waoOOXiEUk3VHawRCVCdxXBRodiOGM5LmL1LAFChvCVgM3uL7A0OhUpMHtlBc9dhvGymKg3dUVbqG42gWKcRCgXdWkQlJ3+jbzSKmw5+XsDNNKQNUJ2DHP2UmBoqBdlz3HLwkC6gv6xQ3vYnL6gSUIcRodAM0pH0M2ZghwD4eQG9SxLJVDfCz02ep1CoCQRGIbgdLZnQOjrAeYRlDXlkwphb1QwDciFth8x39lCKY484rwEXIEaLWFMuIB6kwk6prC55FR1lyigMopQQIfAxRnMbOXwnRqAsTqxmqhD5gS8SSOyoJGvZ9RM7ia7n8/PPAOJixtfoBwhjyCJKOY31YfUHkACicwzv8QGiSA0Kx2P6Ui8AO7KOJI/ZewJHpU7bS1La8MsVIQkoCiVPabAPGNNfeUvPiXb8qx/3V9x+bYDlf/a3ACz/8W+B5d/4Vn74QXz+X/23CGWEagnAdMsJjUvNHuaAaaKai7x9lSftSeKpMXl3snQlGE5ac0CISnI+RE5ufXGSl+ZSauEYfFFsYjKog0ERqSeu2HBF+vCJw9k/V2m1nZ1cdiamWPk8gc9JiCKQzSjvxRRZr/qTNFB4oLyP6C/5ptQtX0+OEW4mTo9L7RbmwAmxHE/HJKrThLzYYuprA9JEb8fHdNcRi2+4apwnTTaFa+iWK+Bmn05Qei3XkA0odjwuvsyT+ZgmBKnzbqD3ixPx03kOhuCheUNJjk7HyxxZej4Vc6fJRbEjGCh2Jy9ZseM4dBtPXrKWCY+QKZSgEFP6Y05ILe8prx3WXPHPPW3SMvxDWk5ifSkmn1Gx52q8awgcq01IPXb8YsrXj3QEM7kLLV9TusNU/nx8pKbJW0x+LtVHSjgXPIf1fcDxkeTxn2cAyzFVtwH9BRdDVJ8kcOskF+s4uVdj5PGpU6dhwNRlFzSB7RQmIzFF3uebyQEj4uTFyrcc1Q9g8szx/cj9KPcR7aVEsef+MFkyoruUk9+N5yOBkCR1zeeQ8jmyEsUhwDYS3nBi7GrxgzEzgCMf4xRWlAJUcqm57vm44sht6e4kvcwywqDILmR/Xe7bY89bApJJHtivKafTKXVTDSmlsqIMt19LlMlrGFS6DszperYNARQEUOxSpL9Jr5MCX7wRk5RzXPAYm2MKUjKYJsx64NjzYpEaY9pvOfXdiZgqFTxBSn5vyRyOksaV6xNi6uYzew9fczt5P/LEXY2clLNKIAGtDDhSV2J1TzBAqXs6HknSrIcIW4vp2ioOnmEmidXRQ2avBMzRY5zzs5V1EDwv1b1Df6YTSOP55+ci/X6mJbOh2wA3k/Su5cCbFOYjB8oiXQIUeXXfF2ICiw+DXnJ/oC9PdoRcz+BqdgoGnQJVjJhkjOOcZfVyjNMiHMDjLYcAX6vEEkW4Rp2e3568dQCgjwng6LTokcNLItgzmJik3PGnelY5PKzrUV1IgUFiqmrQnYdd6ATcMNUxAJkVDvAVpZ/6YDGeFUDA5AM81d7EqVhexEhZ5srwGrABw5mm9zD9TnVB/o5LE+4c4gLA1UxbNZsBdl2m7+Q4AdZQsKYidx3yGpOQLsAbCTdTKDcP2JS0v8gBNErAHCxCCnqJWkLaANkzlTQKka5zSQmq5tj00RKg2TCxQZkd4hhDAjv4QQ0G70wAQtBXGGomuyJGqCPZMTzYn9OXJifpWX4pe4doJIQNCI05+RFVeowA1HFELMhmibR/vJ4k5MC6jGgkhA8cs+R3pxg8gV6ZQmWSxSNXZUQjoY4j3LKCakcCKusJ6nLlhJRTjYZwAX5RQh7GFEgjKGHtHUR48Lo2AaMkBZ1krjnMJqet5uOh5CktNYE6ZEDzEFTln5WEOHaIVQkxjIhNRYlq2j68n87ndL4ebEP4MIHCSaIKcBwxpnGfjsF0H8DHJq/mxPw9AGoM5VGnbcdIRjB3Qma5aVEgjuMJtHl/eo54ENgTw4kx/OXx/vJjHtaOeM/+SaUIRLOsNgHVyUeZpbG/gpX8NwrjyX2Z/wa33wLLv/3bb7THUhgPd2WxvjhAiIhuKFA3PXwQGPYNyspiGBXcqKF0gOs1TG1RVvzycE6hOxaQJqDbG5TnHZxT0NrDWY12UDC1hTYe7ajhW8Z4i4NGXDhWjAhAqgh/0EAQUEuL484AlYcuPcrKot1WkCagvymZBKsj7v8BAJ3TGCTkzCIMCrAScpH6Fp0ATP6ACBjPNVB7diuWAWph4VsNUQTk7IXoJESrEGvPGpOrkR2DZQCCYJdhFExZfXCTvUSYcyVw3GnWUMwdMCigCGitYJ9gq7D/SMKXAaoTGC88zFbBXrFXUQ0C3XMHMUjEIrKIPkaM5x7HVOth1571IlXa/9pDHOhVjXOyxr4mQxjKkL5EBI5PJX2v9xLjuYPsJSWQLRlZSuQE9p8FRBmhdwqhCIhFhNpL+MajuFdMkxUMMtp/5lDcqMQIRkTDyZIcyMD1l5wMRx3QXwoUG4HhKiSmkSxr+ChC70HpCyKGls91M+7D4UNOLO3awWzVVF4fipjCTwTGc3YwUo6VEkQ1fZZT2XxK1BSebJc+CozriEPPZEU50sMbTJyknN0V2V9WS3ARxRecr5bvWWvBACCJ4Syxow0XXOyCLEMuqB/XZBw5+PS4g0jJqvSl0rsamfKJlIArwPTZNPm3C8AuAsxBYNywwkMNTHbtB+q38qIPKwQ45nHJYAsuYBBMlHcC3SNO1suNYtXDMnUx2jSpGul9K2+Bcc1JWnUn0oJBWnwSAuOSHkvVAS2SF2kr0T4lwC7vBNxMMOhophLTnBahtmkCqPMCk5gm0PYgOGEPgE7dihDAfpHZ6pOv1ewTIztLwN6k+x2lga6mtHMCbwn8AynddcmEWt0x4TEvqFGOhVQbw2OiOyQGRkx9ftkjOZwJlPcS7RMy0uOKx2UClSm0KqeMqp4g0dViWnjJtR7tNes5XJP83MlvPC6AIl0n/YVM5ypiXEtUt5FprBIobyMgCXjdjEnCGfxmP7E+chFN2tTPl7yzmY22jSGbnybbXAzhgmF3mcKsLGAKgf5MQjo5+RnVmBaC0iKFnXOBKvciCo8psGcwlAWPC07Wyw2ZPD8T08IFWbkkiZ4J1LdkLqtbhu+4Jv3tTP6gl1CNaTEuJqCv+PycbBqUhEkZGrYBir1keu1I1jhohi3lMZtjhH2sUe7YgegvFNne46mnMayZThoKLhzqVGjfnyVvbFqcoPeSdSgiKLSXlHR2l7w4XSl+wLaWG95XHAm2bCNR7MlM92dcaMohPeU2oF9S6pjTUcO1QX3nMC4YtsP9B4oj/XvduZo8rg/Beb6pgQAuM3u24QJIf1FMi0R5EYCJvJRyhkLCzsjyBiVQ3ju0j5l+Ki1fSw0BdqmmsCPXVBPzqQZ23wpHKaZd6gm8FzsypxBAsXWpsmNEf1Um6bNmn2NKgC03iqm1PQN01BggbYAvFV+r83AzLmZpJeAbDTXQH2oOYpK5lncWdqGhag05eMSUNitixLg00EcuWISSCfFRCxT3rAIBTgtMvpIwh+RfzImrIUKOAfa8gZ1ryAU9kMWWPk2zywyqJaPYWqAWcDMDHVjh4SsFOQZILZnG2ln4SnNBoNLQhxEhJcjKwTGAR0v4FVc0RQJIrjEwu4EgUD5YBDiOkwcyzAt6J1MQT5wzTBCx4uKKNRPYlYcB/qymdLV3XGRLiwTC+gnUT+Ay90I6f0qClfJUE5JrSKzj36U4AWeA4NZoCOvIbGamMiXECiknIC2cT2mwKSE2Rr4mgHhkuM4Pgn4yyBYPGFMjEJ2fGMUJtOYAHsExCqUIwpVCrhvhJiLBe77FmHyVcrofAIT4K4DlX1Y38qv//Jtzi5hIrL+Lt99wxvJ5fPLf/TfJeE7/Vf1GoftJD/V9ifCsh3hbwp87FK8M7AcjxMYgmgjZMYAjR6CH6wH6dQl7aSH3GmHp0HxZoL9MiZiHLKtIrN6GqaHFlv17vmFq57gOCAacYKfuwmIjp8nBcBkw+1ZiXAEQDGkYV/RTmQNXi+2CATjtRxbla5MqAChl7C8jir1I6ZUxTWg4YcueLFZTpLTKEqjeialUmVJB9gW6GlNcvW7JnilLMCBT0bzqMXmBfJVTLSmXFAEYziLqt9yWGhL4SPUOlPcB9VtMsiBfURJJppTl5+aYPFQ6sWgrMtBT2Is6JYW6GaY3ZJ6MB0M20y5ODG6WSTLgBVNNgzkkMJDL1eMpxISl0ydwoHumRgKUJwaVJJopOKTYkKHrL8RUYG52ycN1c5LTTvUD4ITQV5yoVrdIE8nT2KQHuiuyxs07StkoN+QxlmkSm49vZsR1jyTXopfI7OIEWkSgdyymFMtcym1nvEZ86ph72MtHKWg8JVKm7Wama1yK1J2X6jB6ygxzEXdmPIDkTZtxQil8qgCY55CNyJX9JLHLcqMoCA5yGEm+vrMXqj8/XfNqJAs7pWjq0/E5FZ5zsqi7DHS5DTXwWoyZmZVJRZDkmqxRYBCQGuPEkg4rMuY5mKh5H3B8pFDs+Z50NVAcmApa3Qf05xL1+zDVDigL9hoeT8d9Kno/5s6+zGqTyY0iXU9JOZGDVqTFgwJzTCxavubYSYhUiE72TzpeG8uU7pmVBOU2QNkTi+lLoL4NGGdy+vxrbuhPEwEYk9ICEai2Ht2Zgkn7IQIwe2MxLtXE6gl/knVmPxdTUBVMYuz6lURxPHWt5seqIWBYqRRek7xAQkwMJAQBTJTsPnTJ11YcA1QOpTmEqfLBVQQt7ZWCaeMkg3TJO5blm/PvB/QXxQQygsoLPPydvYJh8gVWG89qilLCVQLz1xbDSsE2EuUuwOwd7EJPDK7uyS4jAvV7i/3zYgJsWVoqQjpu6Xd+/sX03eR/ICUlIy4nbxkVMRJFSgsd1knGmnxz0obJu+krCVcKVHcOwkeMKwJDvh9YID+cm4kN9oVA827EcGYmVjQH4UAA40Kh3Lh0npI6wBJUBSOZMmpkqskAio2Drwiu5MhxTd2MSNe4C0wkTWwrmWaG3xRbB+kjK7JSwAqZO+6b6gPBXYwIOiXIHh3c3EwBQHnsZmvhZnqSmNpVwRAafWKsdWsxnJVQQ2AFR+uhBg8300BiZPmZlBeeJPSRC7iqpd/PNUyW1YcRblEQGAGUeAIIlYEcHJNOtZiYT+EC3LyA2fQYL+pJYhrMDxeNRYxQ+wFuWbEDcl5A9S5dU3JiokVMDGPBYB1hPXxjoA4D2UgAbmaYyDr6KRE1SsHOx8S2ynZENApuWcLctohKkelM7/eoJT2IWk7SWmEDmc3DgFCalArLqo5QGUQlmcQ6r+hjnKXgnkIzXEci9VFauPMZ9LabjvfECiYGMCeyAnzO5BksC3ofc0+kTyyh0Q/YWXdiGHM1iBAQ+5Ydk6NFmDdJturZIVmY9P4SBLlJghqT11GEyL7J4dR9OY05M5SZVX3YR5lqSU5j0SdGMktR64pBP3l7WvH3fDzwACj+MhaQ6lcjuB/0X6bXHQa+z6z7AYuaWU0+NjAESIoTgxoiRGJpo3M/DAb6y24h/uZLYR//LTGW/9OvB2P5Gw0sZz9+Ev/h//afAADu+xo+SAxWoylH/M75G/wff/EZfvr8Da6qA/7p6w8wK0dU2uG+rdGUI6SIuKqP+PLuAn1X4Hefv8KfvXmM4Vjgkw9obPrudo0YBD6+vsXr/QJGecQoUBcW3393js8/eY3eGdy1NUKQaMoRg9WYVwNuUq+lMR6lcdAqoFAex6HArBxRagcpIrZ9hc2uQVE6fHrxHn/x/hI/vnyPP3n1BGVpsagHvPvqAtef3GLXVnBWwY0KUkdARHz25B2+urlAUTj0vYG9rzB7dMS8GrA91ui3JUQRcHF+wObPLhCfd0AUkMojBInlnB/Ch2MFNyqsz444azp89c01TEOGN0aBri1QlA7Pzzf44i+e4OqDexy6EjEKDMcC19dbvH29ZiDOoUC56vGTR+/wJ//kY4RLi9mqQwgC7V2Dat1jOBaYr1vsb+Zozlu0tw3gBKqrDnbUMIXD8LYh2yoi1lcHbN4t8OGHN3j5Z49x/fkN7nYzLGc9Lpojbo4z3N/NEY8a0BH1eQf/rxZwL3qEkZMVoZiWJwSwOjtit68RrESz7GGtgntfA0sLXTjYQ4GrJ1vsjhXGjt7Qy4s9br45g1qPQBR4dLHFm5sVTOngnUJ4VyHUHj/97BX+/F+84NhT4IyeWbi9weLRAd3P16h+vEXXlpCKH7z2UABOklXeFHjy+Tu8frtG7DSqiw7+izmqn26gZcB22yAkBl02jpUzMrICp5dYvNhhv6uxWrUIUeDw1Qof/f3v8d3tGuO+gGgVg3bOWV0SqohYe0jjEfaGixWrEeJNheLjPbpdxfuOBnJmEbcFZCfhL0cUjYW9qemxfXbE8KbB7Pkew6BhdyVkqyCHVG+yYveouTGwlw7zyyO6L1acdKjUFTl3qL8u4P7eAe5djVilftGlox/YBGBQZMUbD3kguxivB8jvKx6TgRUyUQLQEXonYc8dhCNzKM5G7kMv4NcOolco3iv4Mk4Eh/50j/7VHHIUMHsB+3kH+U0Fe+Gg9gryeQvx5Qx2GSAvBvhtQsGVh3lTwF46VC8NhisPsR4R7wvEOsAsBtgtA3vmXxgcfzawsuiNxnAeEJcW1TclfJXY517ArtMX/NxCvS7pUbzyMPcKZidgV5H7IDElywAAIABJREFUlQC9fWQhjgryYoD8toZI4Nk1Ec1rieNHDsV7xeqWqx7+pkLzvUR/Fejh3imEC4v1Hxa4/wcO5VtNBlsCkBH1a4XhLE6ePBGB8tMd4j9dwc3IGNtlQKhSnZGMmH+tJ4vCcEkvcffYQw5UHpS3MjExwOEFWe9gGGTUfC/R/qxH9UWFLGkeHjnMfqExriLcswHF1xVCGbH8Eth/RKZ9WiQTXEwarxyq1wauiajfCOw/c7j8Q4X7nyWJd8FxuBkrWLonPAdmR996SMxluQH6cySAntJWHRcK2sccr265H34eoPcSZsdFgP6Kfmp6uFl9M/9aontCBcDsVcThA7KbPtkohEfqD6SveVxzUWP2ivdHKdA+jRNLPn/JYzD7XmA446LV/hMPs5E4+/OI4UyifcKwJDvjMS42EuU9F5raJ2HqeF18LaE7Vt5QRkt/um0ExvWpZieUDB5CZFBUPlaUSnMhK3uMj8+pAGif0qtf7Ak6hjNg/RenmhOf+lHNntd1/Z6fEeOK4158y0WTUADVXUR3kXpoE7uc7Qe652KPHJF6WLlQ17xOTPfIa9kbLtrOXwXsPpInT3fNc+4aYPZ98txJstCm5T4XRyb09udMf15+FSbPcf2ekui8X1zgFZi9DhhnlNYfnissv/Ho1ykFPXA/V7/wGBYEu80NQ3NyV2d9G7D7SGH2fYCrBbprgdmrMMnITUdrweZHCsuvPYa1xOKlQ3utU5JwwLjk+T08VRNDLwI9z8WWEnpfUC5e3tOb2V3wu2f2xqK9ptqouvM4PtaI+oHvWHEBDGClimskVQkLSfn7QqK8dxjXGvoYMJwxvViNAd25RnEIqN8NCEbi+KTA7M2I9lGB8p7sbrFPYUAbB9U79Ncl5BARSgF9YDKw9JRZZ+CcQ35CQd+raxRcRcVF/foItywpe94R6KvWJYaTEmC7KqGPXFCo3vesX9ESej9M8uJgVAKMAno7wJ5VkL2Hak+SYNmlRYoEwkJlKEdO0l/ZjicAKQRCU0z+z0my3A6IVQnZ9j8I6gnzikB7e0A4WxIUdwmMSTmxnlNS7C97LEMgYFTyVIeSmNOcOBvbHqIqEds2yXMTywpM24x98ol5DxhDSeyDBFkAUxKsUOpfu49j+SWMEsOvTo/9/3j7dQGWP/pH//aB5Z/+z78Fln/jW/nJs/h7/+s/QjsatG2JqrJodxVU4RG8QIwCZW1hR40YmFwqZIQbuSoSDgbmrIc9Uhtoagt7X0LvFOqfbHDcVwhHA6iI5rzF0BcoqxHtpiYb4AVQBEgTEKzkZLd2wMHAXHZM9zymCW2nEAtKI6OJlMGKtI0U1COsQPGoxbAr2UuZJnHwgnLPOgBVgLpNUpUAuLWjxDWIaUzRpNcpA8N9FLchLweIlxV8w3RPu07hPAF8TGTqaaw9hIqQNwWla5EhQNMEvwgM4ykC5aglJzRRRRR3agq7cM8HxIOepKxRR6AMKL43sGcB0QToew1fs/fS3Gm4BY9LTgKFjFAda0fCJx3wsp46IO25Z+2KTEyjidB7ykH92kHuNYp7SRZ3zjFSZppWkSvWybj6lKTpZhGhCtAbghW79tB7hZjSRVWXZK41k0R9EVNCpkj+VE4Yc1po/jlKstihiLDPRpS/KDGuAiWgGhN7LUdOiMc1J5m+irBLSkWrd5SuupVnX6d8IEnM7E4CcG7G8bpURaOPZKm6xwF6L6BbTgR9GVPoDDs23TygeqemxEU5At1TD32QlMGGFNqzFakvDAjq1M1p9vw/FJQTR81z5ZqYujlPjLCvY/JxCYyriGJHmetwzuf4KiYlAllPgBP54SxO+zlcBpitPHkpsywxSePy+4RpsSdGOz8+bzPXVQSDxIDx93GNqSLFNfRXIyafdmJE83sEoBd5WJ8AXn78cIapFkTaE9tVbIH9J5xcmz2lnWSDME2sTfImD+eJWU2dnNKdxho0pqoTfUyps2l/mrdk63KFDMOvMB1Xu4iYfUcpbGasXZNY6yMDi1jrw5+rO4bSUC0gHkyCqU7wFVn9KE6sNeWy6XxEjtXsTteum/F3NfJAukZMPY+uJnsexcmrnXsMMzsrPdL7O05srExSTV8xqXNYySnx1RuywMOa3t3c2zr5zecM/hnOki87cvtq5LYzUEJikbMvmf5nSlGlpdIghw1Jy7qUYc000moTYFPHZrHje6a+DROIYMhTSjV9qHp4qMA4csI+LFlVAvB6UDZOFS0i0i+7f0ZG3bSZVT+duyIdk+w9p/f65PHNLCmQVCjvw5SSGlSW9mOqxcle5JiABftE5XR+8nsz+58z8ys8xz5JvOODa6ISk486hx9xG8lXnuSpOXyK/cIC5c7DVRJ6yIBLTPJx0waym8DkVZUuQg0B41xN6gCAIC1/ZjCsLUC6iO6SXtDsZQYAX0pUtxZ2rmBnEuYYJuWJ6h+AHM8QINWRdaNMPBKADXFiWUXEFCKESMASJTspKRFWkDbAzRR0R9ZVHz18xWM+LpjEmisz7FLD7NLvCWj5BLSiEpO0d/J9Rkz+VF+S5YxKTJ53CFabMHwrSYxtoKez4KIuPzcp7w1lvgDAcddqqnVBJHuKAOS0Upm3EyJiISFGzhNCoci4xgjpCPx8bZD9t7JLSbdJCZPZthzAE0o9BRXJgcx6lII/+wi/LOnvtSfJafaIRsO5B4Qg4EuJr5nJhY8Q/UCGMUbExMZlL2k0CqJ/yDiqiT3NvZbwgUxmCtmJSk7ezWg0a0Py8wtDEJgqTPhi4ocMY/Ztpm0jJJD2K3yOIgPILIPNftiHnZcAWc2HfszUiymU5PnyfgKWcRwpmc23h6mxf21/5en+vwuM5Y/+078FYPm//HoAS/nXP+TX9yZkxNuXZ9jfzXC5PsAoj3/3069hCq5mXF7sYa2CeFkh9BrupoKUEdFL6II1C94pNOsO5p2Bu6nw9JP3OPvd91hWA+KmwOMPbzG/PKLdVfB7g+71HOqevka9sBAyIliJ2aonqAQAJ2Df1Sgft3j2+98DKqJ8eoTZKKiLASgCfZV7DVk71N9piMZBXQyw380mIIlANkrOLcLVCNF4ftiKCPGsw+Knd5CNgwgC5rKD2UiYxy1B5tKh+doAIqJ+ckBsPPxBY/GzO2DpYB+PELWHnFuIxkOMEsIKVI+PqL4tIO4NwtUI9fEB6pMDsLIQPZNdF19ohJlH851GPLeov1Nonh0ge1ZRuDNHJuxNicWXmiBoxuOtbg3EZwforYRs6YOJRYTaabgnI+TZiPqVAj5qCXZXFuFJT1D5bY34vEf1XqL86RbFrYK9tvQFLh2/uOqI5Wf3EAODQIZPe4yXHsU9ay8mSaUHmm/ZbYmVhRzJbkQTgZLpvvbco3mpoVsB3wT4OjB453nPRNU1Uw/92gEJVA7nEcN5YC3Di4GptQMBaf9ihF0FLP+4hJ1HmL1kVcIrAXNgwI59MVDCnLyeqhfAeoTqGWCkW4HFzzWlzQ5wH/VkCzTg5gH9Y4fucYC0ZIzK9xL1O4HuqUN5x25ON4toP7IYzgJCETFeelZuHATKW4Xuw5Ey2nVA95Me1VuF+i0nqZAEDP1lQPuBg/DA+uc87vbKQh+B4YMRq59z0qsPAq6O8FVE95Me3VMP10S0HzqEArj6ZwHdxyOq9/R0+oJdjuPvtKjfsZtVdwTKdhFZyXHLxOJxHbH4SqJ/7DCcR5QbMi3dE49hHdE/9vAN2Yv+OqB+x4qW8TygvOeX1HiWJqcK2P3OiP4qYjiP6B6nio5bAt3jjyxmryLaJ5StL78KqG4IzMdlRLEl8BqXqXdxSUl6f8lOx/N/GcAwK6B+F2EOrGrY//sdHv2fEc3riOOzCDnwMYcXAe3jiGHNmpJQkl2ZvWSgkV2yN9XXwPbvWXSPU9LqMmI4p7xbWmD2OuLwAcO/zIELBccXAdV79l+qHlh9wUTY3U94ToYz7vP8u4jd5x71O6aBuhpYfBswrAU2P2HVRfZYigBc/zElfsN5qjFJVSNqiKhvIg4vIsY1cHweYXbA9mc+vWe5z9vPA+xC4O7vB5hDxHARMZwxQCv7It08ycJniVW5oBS3eRfQPuUx2HwOIPJ1XENma1jKBBxSGm0bcXxCUHn/M77XQiFSiBllzrsfkXkJhYBtBHzyjzKYC4lJjGjehQlQAAz66i8FDh8SSC5eOdR3BLiHpyot/BBIDRdM3+0esZZk+5HC/qNcScMJfkjha8oS3EgbMX/l0dwEHJ4LvP89if6CQLBMEvjtxwy/Gs742XB4qrB4RTbg/jOJw3PKcgHKuHOIkp3Tx1vsyXStvxjRn0lsPuf5DBpYfOtw/5nC8TFlkDnALQc+6Y4gXvcRs7cOrhLY/ih9Hq8EDk9lAogCzVsL3Qf0a7Jm1b3H8bHEsKDX0jYC41ymRQ+BwxNFf+cYJ1nvuOT52T9TsI2AnQn0FzLJr5kyGzTQnSt05xLDSqDceJRbj8NjhWEp4A3QXuUaioj7Tw1mr3rMX42YvbFo3tr0/g3YvaB/cvuJmby13YVCd0lvp6skmrcDxqVm2NSOFSOuohS7P+NjzNFhXCqoLpBB7AOOjzXGpWJ6rKTX8viEsmk5Bgxrhf0HBr6Q0K2HXSheI4VICa/jFC7lKzKL/bnG/KtDClyiV3T27RF2oWHnEvpg0Z9rmL2F2VsUmxFqCEyJRbJYOErN28cFjk8MlKWk2C4UdO+hjw7HJ1RshIKpsPARrtHwlQKrSgh2ukdk74a1gblr0V2XKLYjXK2gdwNU72AbnToqJfrrEocXDZB6MfV9B7sqUqekh+os3NzAzQzcgsmwcqTHMdQadlHAVxr9VQW3KDBc1hiumomdK26O0Dd7yH0PN6ccNmoJd1FDduzYBAC3rGDPavq9z2oGCSkJ2VmEpkBoCsQELrPMNKwahFkKWOoHgtGR/ZZTPYgUiAl8hkWF2DDZNRqF2JSIVYkwq/nhUlAWnKWzcTmjVHc1JwhsKv4N4DaqpKLJAC6DtyldVkI0NcS8gZg9+H9WT5JWURQMA8o9mXn+nYOGnCOzqR4E+yTZa8zJs0oB1k6sZLSO/3yY/iED0L/sXwynf7+9/VrffqPDexpjsXx0wI8vblAph95rzPSIz65ucNc3+Gh5iz/xTzD82OKT9RavtitcLQ6QZxE+SJSXDtuhwqrs8fXHCo3xMDLg2XyLV4cVmmcHzIsRSkQIERGCpFzSKpytjig0Za1ZGgsARnn0jYUQEbNqxKIYMDvrsKh7vH1qsGgG2MKhLke0fcnKk8+B9apFPxrgBYOI+qaAiAKrZUvm1Tjc76kjc5XH2bKF8xL1bMRoPOZNj/6nHkoFiOuIurTYfCKhGodCe4iLFjEC83LEoXSo6hHeS4TAkAqnA6SMUCrg8NQCRcBs2UPJACn4d8wGjIPB4ROF5dUB+34JVXi0HzqcaYfuaoC3Cs2qQwgSfVngWBjI6x5GBhjj0XqBRWmxuXAwyxFu0FCFhzcaRW3RVCM2H2k0hYNbWjTzATECdiSbWRiH7onDUkSMlw7NWYfWScjGISh++Q1WQ50NcL1GUTrEwqFHBdE42JkkQ6wDhmigXxwRvIRdptXEhvJkv3aQhUf/iAAflUf0At0jAW08xrVneJLSULWHS91hfu0ghiQTVBG+Dgg1ABOgagcPoH0i4RceoRGIOuD4HAhlJKMdBI7PI/yZg28kRBTQpYNdFogmYtSc4LqVg+wU08SvI/yM7LOcWwQnMUaDqCOGi4DOAGLm0D7WGM8CQk15bqw8m2ZkhJuRIRYBCRxQxigl2V5fJebVJBZRR8AQ7B2eC7iZh24cuicGCAL7j8hg+wIMYAIQrQQMwZ2oPNxM4O5zBcCjvyZwR1RwcyB4gfZJhGtOsi4IbmuwKVG5CegvEmNdUUYVC76Wn7NX01cRwwUQi4DuWiOUDHbqrxiU4puQJvDcb1+R7feKDMFwRqkfvED7VCAWAeOKk7YoMNW1jOsTUxtasr9qFLBNgLQK+xep/xMC/YVIvkoBvzM4PFf0O5dh8rHm4B4ISjn7y5ToKpFeJ06MoLASQRMARRMRIhk/CKB9lM5VTNUxiSHvrkUKj0rXUwUIT3AvBx7vnDI9nLOeRw0M0PIVz4VwBNJREvgenir0F4mBrJGYh5PsUHp2XCKxm/og4eZJFjjLHZSAHGRKUSabPqyTF14LBMWwoWAIMH0V4ZxAd5G6D+ciydsSC1Yl6eMMEIH7FQt6foNJxymmEJzkm/QFIKrcpcjvmshsNoSZSGw1wR+MQHfOa4WdiicvPqKY+iEfglFXY6rM8Yb77stTErWI/JurACW4n1Gd2Gk3o3dUxMRaHnhd5OOSvYyuSQtOFf8fFnJKcxaBXlbXCAwLmVhjjp3jS75EU0zBX0ywjugvFBn+NE4GbzFE1c4oZ/Ylw7aiUhPoHFcpMdjzcaEAuitmCGDaz8TWJnVBTtt22aOY2NGcCk6WM3ImI9M5R1JWlCc2k0wNXycKoD9T0MlfLX1KQo7AOJcwkqC6vy4nmTeTfAUABlHZhoyenfN7IzPS+dafF9N+MylZIOQuU5OY/pLeYztPx79WUyq0SwnLCCeftSjUlBIfFeAaPl9WDCySLk5MoC9EUmIkBnVZJFaYUs3+qk7XlMC4Yi0F60Ae1CGVikFBUk4KCxGyt/s0dQxapjoV/IDNdDOdGD7+PbPefA9SsuwXFfdlRh+ibwqmGguQTUznW41MqQ1aTM8JpSYzm1g5X0iY1sHPClaeFCf/aGaqATKkBOOKHtdVBdnpKVk3SjH1UoZap+8XTTbSk2kkayz5Ogl4RgEILRGj4JuhVEAg0BZGE4hJIJYFk4YrQ9bSBb7Rwwm4xcxQCsHnpdeKQjAAR/2/7L1JrORbfuf1OcN/iDnizjlnvnmocpWr7LYtD9AYFqbVahaMzQLUSC0hudmCxIpd75BZMEjdajVCApagFniBG1u0Gw9ll8uvXMMb6r2XmS8zb94x5v90zmHxO/9/3Fe4bIO6pCrLIYVu3Hsj/nNEnO/5TlqI9CD+TGFEoxS1fdyCwJgge5PtxEYPZpss297jMmUw2wYMKaknibcugbh9bZLufJNtWi5mtx/AzfRb2f4dQJVVBuAv4K+EP5Vd/XG8/VmZRT/utx9rxnJbJyyu+nzjyV0eL2c8Xsw4K4YUznK9zXmymnF1NqKf1Txfjrg7vWZdpWSmITGObZPQS2rON32qeUYvrVlXKV9/fI9NmdI0Gh8Uw7QUwFgk1KX4CRLjOX05YZAJJV/UFq09k15BcZVzb3aND/DBy0P6WcU4LbFZwzAvMcZT1gnDnjyeTtfiwbTi3wxBMRgVhMJQNZZBVnF+NeJ4umQ8KEizhsvrAUoFJv0tg17F5csxxTalKBKqIsEaj84bkrShqBKc0xSblDenL9HGM8pLDkdrHh1ciAc0r3FXGVVl6e1voTToCCqvL4aM+wW3pwvCs5zQd5SVJQwct/bnoAO1M9jUkT5L0DpQrFLSfo0pFMm3+xxOV5RlQjKoWSx6mKGMTEKtO09btUpZbzJUqbHGEyrN+nTAdp2JfDkgx98GysqiBw2bSxn59YclyUC+2ctPRniv0Imj3iY0RUIyLQiNJnmZkJxZ1NLic3ln14WcUzyEjSXNxXPnN5awVxH2akzmIALF2WgD1tMfFCJVJoKosegwVSVpuVlekVwb7LhCZwL6UVAf1uhBjZlFv4SFEBNyDw4X2C2kzxOSuYFGiXcz9+DA9zxNX2azk7kmvMxwJxVoGN9a8u795+AFTKhGekeZ1ITK4JOA7zvGJ0tsvyF9abHDGhWTZ0Mi8tneuJDjkXp8bWgGvgsisksNByWqVqDleDTDQMg9SnvqsQMXB+0DqcjRlcJuNNQaTMCPHKGJg9hhQGcOn4oUWICeRxkBhSH3XUWI7zt0paO0WEBueeiw10aCuDSQ+U66ZtZaJNxG9sVlAXVUyvHQwvCqWkfGFqg05qAku9SYQvxjIlEWUOLyIOcjFyl0ceTF1xoHtM0gEIg+UgN2qUgWWiTGPQnF2txvqGaB8tB3YUObW0F6TYt43IYx/CgRf17bw+kykRA2g4AbeJqxl/3LfOdR1XXLXgc2dxuaYaCeOpEtJ/FaSzzlvpeO1sgwVns+Du6j7HDkJG3XiYcwKCj3XAcos/M4+EslVRhgfTuCgVz8fwIKRcrdSp6bgaeZeOpRoD6oaeto6lEQcJxJMFrr3Wt6wsyaUlJk22Pj08gqx4TkarILIzPb2D+ZhSi/ixMcCWyPPNsj36WtulSk5620tAX11RiKoyYy9xI4Fowwv/VI2N62bqoeyrGoBztZddOXVGifwOZIUUdvna6EifRWQqJcLnLcpieAWRhgmSS4OfDQtbz/tkeKVootNU8C1NsQsDbBtXcmctamL6BfgtUUm2NJNXYp1GMB7NVYGNZqHKXElfgat4eqSywGkS/7RAKSJGROtqEaS3CatwI+i305F3VfdfJauxZ2t+lDuQ+rOwJwqpGwjdVY9qEaiYTZG9k2dGTEw24SoBoLyNsc63gORE5stlIv1fRj1VeQ66KVr6JkX+02ThqNZaBf92NdTynvsbon6ozNocZl0slZ7AnjuN3XsRbJS79tX3eyW+VhfVv2pw2GWt3RsRpo119KgHTpKKdG1jcQGXM50aKgUQJwfaJo+pr1rSj1VrtJgrqnJXTLKMqJ6STh5cwStKLY010VUjFTrE+ymOzaBuUp5g+NbLuWEK5yaqnHhmoirGlbM1MNDXXcnqDk/Dd9TdMX37EAXB2vPY3PNMXMROCtqUamswv4WI3kEi2vyeV73VtFsWck1CoysD6VChj57FPUQ0nldZmhikwtWlHupZ2kuNiLILlvohRbxQ5bSRRXTaCc2hiWFWW/WknP5iSLYU/gM0M5S3C59HS6nqTPthUy9SShGWeEVIPWuH5KPU47wNkCQTeIvd0afJZIemxuY8KsQ1VNrHtRnXy2GeeECFbV9zGNqg3G0bLe6miIH+TUe30whvpoSH0w7KSzMjaJtTFZGkHzDZlsVeMnA/x0iJ+Ndo8ng50k14ed1DaxAiiVIlS1gMRWSusjs2itAE0dZbc360mM6ba9vQfnhdFVEZz+WfdYb/JXtx/t2481sLTa8ze+8E1+4ZWPqJzhlekF77845P337nFrtOSTj4557eEp47xgkFV89OKQEBTfO9vno6eHXG9zzpZDGmcYHa24XvTZVglZXvNodkE5z3lyNuPjs30S4xkPt7x+5yVv3j3l4nqISTyXiwF50uCcZvPBlIt1n/HRisob5ss+dWUpa8v7H96iXma8+M4RzmlCgHvjK9aLHK3g4nrIepXz03c/pakNdydz8V8CyyLDrS3zbc6mTHCNpj8oub4eMN/0qBojgRr9inqVYqznaj7gldvnlNuEpjYUi4x37z/n//j6u6Rpw/PTKZ9+ts/H5/uURUKe1jx68znVdcb2vM/th+csX4xYriUs42rZ56PPDnFDz4O75ygFSb/i6fM90knJ6uWANG2obtWUpSV5npIkjnriSL56xbPH+7jC8ouPPiJcp7iNZTracufeBf5FzmsPT6HSDAcFD956weJiwPhgTe9wQ3CK4DSMaoz10CjKVYYxnlcenXJw91oYYxUwvQafB+4cXouX/CLBpI4kcVBomlsVzcQTBo7Qc1RFwhv3T5neXoANkHicU9i9AjQ8un2OPk+w1kFhGN5acfp4D0ygLBNhIZcJZlSDV5h+I4ygDmxOB2TnimaR4itDvU0wmUPZgK8N3itO7l8yfnSNXhpUozl7MhPm4o0V6pU1IfEYK3GoIfNMby0Y3V+g14byVo1yMPlaBiaweDbivQ/vohcWvVeS3lnDuOburUv0ylAdCuhbfjaWSYQDR1Ma1EUKjzYEG0hfW1CcDgRMJR4qzezhVQcg6oMGvxY2NH+SCqB7dY3aGAnqGctxGLxzhb212dUiTB35cwtOoRJPepoI+3avwK8t6fFGPGxbja4U+pMe/VfnqEqT31lJ1U2t8ZOasFejt1rCe6ywdz4NFLcbKDTJtUEXMgDTG02zX2MvEvyDAv0kJ6wtxYNKKm086K0mmCDb96RHceLE5zrxlMeN+GPPDKN3L2RQNBSpcRg24rstFMWxw00a3NDjpg3JUvy69cwL+3erIWQBO6mkmqeB1ZsV2ZnBPlqx9+Uz7EYG/c3Qi3RbRXCcIecqAlNTKtJzGXAlc03+Wez+KyWdOFlHz9naMP3iOcmlwVRw5yvPqV7bYjYaP2nYfGlL0LD/pZcEJfvYet7ylxZ/f8vgscFulEwQ1IpqFNjeaboEWkBqbrJA/sXrzq+rvDBB1aEjWSjWDx2mgPRSQpJ8Fhh8kLJ+o4ppzyINb/2jBAizmt6Zovez5xT7AdcTKbjLxA9t14rsGgnlmYpHt+mJJ3F7FMjONfUoxM7N6FddarIrzfKhgPHVK47sWgB3PQpcv+s7/2x+alnf9Z0/2KXIOZqLvFbYQGiGsHwowFzXgWok4TX9Z5p6GGXRuTCBQYtcu61qya4Um/sN+UX0Va9FCr65HbraGZ/C9iR03tVmIICuOITemWL6PUeyEo9qPRJwlC5DlzTdAtHlKyLdTxdBKnOG4t+tplG2HMOfqomiOJDjs3zoqYcCVIOBzYmimir2vttgN/K64iAw+CxQHNAF/KRL2NwWIGlK8XX6RKpr8rPIilpYPYDlQ/FWVyNFOYvgOoPsMpCsBHATWjlwILtuO0hBxw7e7Fq8rXYlSdPDJ7L96Tx0PaIoWaaLjPX2UJhfn8SwGScTKPVQEqmVFwl1NVbCmmt53eCFpCTnF9JrWo1l+enSM3guqoV6oGIPrgCxJMp8XaoiiDQUM/HZKic+VgnIkdTmJpd9vn7NMP3Isbol7GSyDvTPvATSBLqE5Lbbd30ktSP5lafuaZKtJ7+Syh+XQDExZNeOxX3L+FMXu08zAAAgAElEQVRH79yxuiWVM+VEsZ2Zjglf3rMxMVr2oRposrmjd+EoJppyFD2ZkVlONj52yxrSlUhus6tG0qhXddebqnzA5YZ04VjdliCecmbJ5o6mr6mmlmTpWJ8k6Cpg1450IT25ybJhfSshP6/xaZtmrUivCtLrBrtxuMywvpVgC5lQM4UAkd5phV0U5Jc16XzXuViPE8q9hLpvJXW8FMtR70WBdlLhYhclbpBIeq3VJPNaPKm1ly7UsiG9KCS0x4VOPmuXpXhIxzkhEZms+EJ912OpQpBkW6XQZU1yFWccXUAvt8KU1rIPQSkBV5GlTV+u0JuS5HKDqmrs1Zbk5RJV1R1wVHWDahwqdnCq2JEZrCHkKXq+Rl8t0RcLeXy5RF+tdgmyTUMY9iWBtm4ITSM1JFkqoNBaupoRpQVoAhjT+SmlS/MGyLxxU0bvKk3i4w6Y3rgrI5Jaldj/1zJ+LG/hh3D/Ebn9WIf35K/eCSf/xa8ynUqK6ZOzGd5r3FKSK4ejgsX5ALW2JCcb/KcD6XXUgd6wJE9rri5GJL0a/a0hxYNK/I2DmvAyF5YogB7VhBiOE7ZWBnsTCc0xgwa3sqhGE3QAHcQ76MAf1DLRVGnyvYL6kyE+86i9Cr9KpDuy70gurQT8FJowcOiFJdhA/zOD+plrVucDutLhSpOdGqo3tpinOfVM2LUwalA6ELzCnsnAv3eqZYb4USHbParRxtGc9rFHW+p5ht7I3ILve7CeZFCjP+iLj+ukAq+YHS65/mQaEzgt/Wea1RsVgw/TyGQEwkmBepl1Mkk3E8/j+JspuoLrnyqFEVrJoLL/ROSiKGjGDjs3uL5neH/B5oMpwQgb4ftx9Dqq4TohpIHhR5bVKw3ZmYQrZBciFVQeTKUoXykItSY5SwhAE0FNm3yoa7quxc3dBjJPcppIMMy9Spg1AK/oP7bYDSzearBzQ++lYvl6Q/7CUpw0HSuWXhjsSgJxBk/ET7V8JOsjwPZ+jWo06YUmu5QZ7paBGX1PZuPrkbBOZqvY3nXoQryZ4bU19r0hppaBk8/ktbqWYJ3D39NcvSOStPLQSZVOrbqwnGoqA9m991oWIvZnbna+0um3hbVQjQxOpcolHtNC/Fvre478hcjg7FY8bOmVDBaLfUl4HH+oWd8N9J+rKNeC1lu4fFU8n1JdIkE/vfPA9Vsiv0TD+EMJDzn92cDgiaEaC5MH3KiniMmac8XgWeDiq570UpOfxW7KvRCDZ1RXIeJTmL7vuXpLwOjwU5H+bW5JXY5qJJ0zvZZzb9fQfykDzfkbwpoMHys2J4H8TNF/KQOxcibbNngWWN8WNsQUgfXdCH7i73Yr0tLsSlg0n8q+LF4L7H9DmKV6qDqmqThow3sCw8cyCE+X8n7Z7gtgsW1H41BYrtl3ApfvCvDpvxAw08pDfSL7ZMpAcaiEQQqybUHLwPbqLekmHHwWWD5QzN73nH9R0W/rhIoYLjOJxzVWNblcalsGp571saacKRl0R6a1HiryS8/ikbwXko30Tq7vQv+ZAFkClPuK9FpSOntnntVdea+m14FkKwBxdUfTP/UU+/Jz8UiTrCC7FnZo8rFnHv9WjSG/EGCRXctxlUoEARIQAcBISapmrhg9qVg8SknW8vx0KaE2bQBOsvJcv5IwOHWxEkZASNDSxZnOBQBWI2GZxp+6GLgCmyNDupQ6l4u3LXvfdSzuGfGHHmnyKzlnPlGMnjZsDuTzzUT2V0CVoxobkpWU2C8e7gZZyUp+aifgJL+WgJ1kI9Uryu8Ce7J5wJZegMbSk13WbI5Tipki2cRQm0RqRzbHCcVMkc0FINqNVOvYMpCuRKpeTDTZUv6HguVd2a/sWkDD5lDvArgcjB6XbE5S7NbTOy1Y3+2RXUn9xvKeFUA5UF1XZrpyND1NNRCgk648+UVDuSc+xrb2JtkE0rlje2CpRorRZ430q/ZFxlj35fX5lQxwq5HBlgIU64Fi/FhqOLYHluzaYTdSeSFeYE16WbG636N/WrE9FB9mNU2pRoZk7buwJruVqpS27qScyAC7d16zPUgwVSA/K9ke5+SXFeVUqlKagTB9upLjml6XbO70sWsnFVEjYdvShZPU0r7FxO5K1QRMIdUpykmYzfYgjesqpPuxdPhcntv0Da6n6b0o2J7k9F4Usr0uUO/lJJcFzTRD+YBZ1zTDFJdLGNHgyYZmJP69ZC7dkJu7fXovCsy6wg0zzLLADTJh+zY1m9s9kkUj/sxFRTPJSJ/NWb+xT//xAjfM0JXUZdSTnPRyK57HWGmSXm67WhOfGEysHbHzgmaSCzAralTpCLnFZ1ZYwRBZ7XFKeingT9cOqcAReai53oD3+ElffIVVE8N5dOysjGExEeT53KKLBjfKMPMtOE9IpSbF9xL0shBAFEFcWz0S+jldZUhVQ5aiFuudFzJNBMy1ATfWoDYFoZ+j5ivCZIi6uIY8E59l46AoIc8EqKoIxCJo7R47J9UjdUMoqy6Yp0tvDcIItgCvuzmHyjJCWaJslPzWNzTfPgioTFIBlCH6JduE2ZvhPFVNlxr7p8lZgxdg+qfd2v9Ff+WPfSrs8b3w2r//Lz6855v/5Y9GeM+PNfQ/6i355Te/y9uD5zgUq0MpxF01Gd9dHvPV2WMu7gz55tUtvjB7ztdG95jmWx4ML/FB0zMV5bHls82Un3jjj/gnn77LOC/51Uf/J//w6S8A8Pr4jFWT8nwz4St7T/jDy3s8Pp/xiw8+Zlln/Et7H/Ab529xf3DJ//3iEf/mg69zXg95Xkx4upry14/f59efvc2/c/8P+K/dL/HOrZc8HFxwXg75dDljsc35qZ94AsDT9ZRNnfDqO+f8sz98m7/9t3+DjzaHfCs/5tH4EoBvvLjNo3cv+dYnt/l7f/N/47evX+XrT+/ySw8/4puXJxgVePOdl2Sm4Tc/fY2TyZLaGQ56a5qg2cvW/P53vsDem0uGRxecb/rkScN800NrzzuHp/yRuUO9Tvn5Nz+iCZoPLg/4qa98yNl2yPOrMathxt/40nv874uf5F/5+ff4rY9f5dHhJe+vboMJDA42fOH4Oe+9uEWySvjif/RNHq9mjJKSP/nsFrf3FnxmDjh5eEHjDF84eM7vfXafr95+wnkx5NsHfR7cueDZxYS/dv8xzzdjKme4/vYJP/k3v8WH9w+YGcfT/oz/+Ku/xX/79V/CJI6mstRbw8+8+glf++QB2Zsbfvn++/zuywecjca8efuU7/zJPXq3VjTLnPpI8ZOvfcrXP7pPvef46S98xJPllMvFgOoyZ3p7wWIzJbu/4iCvuOgPCWc5X333e3xz7xaJV4Sx5u1bL/nON+/hraaZNbg3K8KTPre/eMq2Tjg/GzGYFPSziuaeZv3eHtWtmqRX06xSVg8tLvWY29JuPvinAzavO/wg4AK8dnjF9/b7qOMSf5GiGoW5vaF5PODW62ecrY+pjysGe2ten17z0fk+xeMR+dvXuN+b0X+u4Rev6P3miLOfC2ADhydzQlBUjaGsLJsHge3LPuPbS3oqsPxwir6zIc0a9G9NqaaQvzC4n1hRnPUIiWd0vML93ozFazD44gX3ewWnl3c4/Mop579/TD0O6Eph7q/ZLDKGB2tW2RAzqXjnzgve++gukHL4zktePN5DZZ65TmkmjoevvOST/Ajda6jbWpCBl5CrFxn20YrtRY/yQHPw6JLVSUb4v8YM/rVTNpdj1NOc4nUZyeYf5JSPSl6cyGyynlUMfi/h+c8b7n/lMz79o9voWnHr7Zc8ezHDPk9591/9gE//0etSHXB/i1+KF9LdKtFNzuqRgPzw6lomaJwlXcLiVS81F6NAtec4uH/N+Ysx+7+bdCmq9UACiCTtVHH1roDb4qfWhK8PaHpQ7ksSr7tfcHFsOfhdi0tFdrh66GWCqtaE1JNMS+rCcjq1mIOC9VmOXUtXbjWNiY95wI8bsqcp9otz6t+fiG+yiOm6M0Vxu0YVhuxKURw5qs80zaOCsuyxvStf5OULi2pge1fqSnQtbKVdK9KlYvlA/K7BaLZ3a/qfJnFyQyTMLg9UMRnbp55kKf2L1SxQH1Sc/FPL5c9V2N9J2dxz4CDvm5iACev7juFnMiGga8XmniM7NYyeeooDRbpQbO47Qqz4mX2gOP1pAbtt56q30H8u4HzvO47LrwTqocWuQTfijXOJYvUwsPeedH9evaVJrwxo8cNuT2Qiox6CdpISS4DtsYDj9QORDefnOvaWBqnIGArTXBx6ti81mzuBZKNY3/Mcf63g2S/0Wb9ekawSlg+RvuG+TB4MnkHdk5qHumfIrz2LV4XpRUkacX4Z8EYzf8cx+2MdJzhkMiePbN38zUA614w+VqzuKdRjzeJBLn5CH32UM5E2oizFnmJ7FEROmEHPK4pDkSvvvSfP25xI1cXoCfTOHZtboJwGZVkfGYoj6L0IbE9E6qrrlPUtzfhT8flevGuYvi+BPeU+5OfCfiZr2P+Thsu3EtJF6LywxaGh/6Li5VcknCwYYYKHjyWcZ3sobPbigcVbOqa2OIiTbSMBkbb0bGeGJnoml3ct2dxTjmVyZHMik6e2DCzuGfa+E5i/opm/kpOspc6l2DNsjkT6aNcwe7+k3EtockXdjynLvdjjGxKW9w3Tjxqavo3HOWN1R7P37cD8YUKyCmRLGUBnL9cs71kGL2RAvt3XbA8U0w+BkLC6k5BfSThQ76JBO/EhFkeW3llDMdWYWiZfl/czJh9u2B4k9M5rNkdJZK0TtjNDOreYdU1x0heWbZKxOUpRPpCsLfVAszmUOpBsmNL0DdVIMywd5UFK3VeYaYruW6qpJU801cRit57VvR4uVTS5gOrR1Zb17RTVjKRyZJixuieAXYXA9kj6MpthQjmRWhHXT9ie5PSfbalmKTY31EORla7u5axva46/toWhMHsu04B0f7pMak6aUUo1tigHvc/WVAc5+csNbedicSzWmuyypDjMyc5LfGZwuZXE2b7FLmuaUUJ6sSUYTbM3gBB2faTzEqzB9xJJ4J3mpGdr/CCnGeckF2uq4yHJ5VbAazoBq9HzjQQAxY7O6qCPbjxmLf5GXcZgoYNZNwb2ue0SbvWmIqS2e4wPhDxBVY2kxWoN/VyAZFVLomyaSHBQ6w1vAWkIcLVADQcCmBMry4AOYALCntY1KkvF99k0KBoCoNqUWtj5MSEG+nzeYyk782cIKH8Q4Pyr24/k7YfKWCqlpsA/AL6AELV/B/gu8D8DD4FPgH87hHCllFLArwH/OrAB/sMQwh/+WcvPHt0NJ//5f9L1+OnEiwe50YRKo3JJJnVZgGkFc2HyMIG2ykOtDZhYbzHyIj81AbvSkekQP1fbtRZ0IL2WZM2uJFuJHEw7kXTpUkWZXZTgQFdm34ZymELCM1pmqcmlP6qNjA9GBgz1SMJEfNrWW8gMtsuF2XK5MHtteXewsn2uJ14ZJ00q4jPx4jtK55p6ELptkzJqYfvqYayfiANG3dClPkqBNLuahkp1vqy26qGNkSf6lXQtcq/iQAZerSSp9ZOpIFI25doZdWE6TCXHTmQ24r+TGoTYS9duepwQ8zFUpvUlNX05Pmhw8dgFJexIuy/KiYwru1KdjECCSISp0rW8nujp0rUMwn0S9yPuY3v8vJXX2o1sn7dyvCT2XlgqXclxausN2kqFtvNLBTlnpiQmQwo72Pq/2mW0kkV/4zNe6g0gRHaq7ftzedzvhtjVJj9NGasknLBfLtstu2Wj2n2D+P/2eo+9iKqJ9Q091SXVttUM7XXgMnl+y5q5GOwiB5zOn9SeF3ej/y7YuK64LaohXnORyYnX5+e2s7djNttzqpywc/V453UL0b/VBW/oeM6KnbFe1yJ1k2v2xvlyoWMDpRpEEkiF4RXGOlmIjM9u5XcQps6ndNI1CXlp3wfyWvEQqu76SFbCuiXr0HkSJSVSagfaW7rYsXLi25Hj0u5He6zFE6qwawkpKSciafOpLDNdBcqxpHo2ffGuhXidhbi8lkVrvYlt2nIbuqKr3XutGsX+Qiv72Fa9dB7CGKqiG1m+TyVh1+XxM6TZvTfaipf2+U17bd8IF2m3x9vdMtvPJlsKm2cLkRAmG+keTDZSUZFsPE2uu+NmCwFjQUn6qYvJsTpWYrR1Kt21TPt+3/nuJMF0dw1LjYewvC2jWA1ELtn6zWzhxSN3I+TC1AHdtIE1LTOpY8/g7rMwKGHu2m1tP0db+XKTRZATQ1ZMJceiPbfeynEKWnXVPe37yBa+C9/RTeheg4pVG6H9rJDl6lpYwpYpbiszOhl1W1cBJMtGPHaJwlS+O4Ym9iC2YTOtcsEWPoIHoiSz7SikC6AxZXytBbP1EjITv/eUv7G9PQEesg9yXHZ1NjfeH2WsnonP0ZXHZzpep6HzUIrcM1aFuBg809aFALrywm4ZhU+0SDDr9gTuriNdOvEhgjCT8bkg/wtWx3Oz62cMcRvaig/53NyN83yiJcAmhgqZ0svfKtdtq3IiWfWRCW3XIQ9Cdw5UAF25rstRl072K/Y4dsmitQBDFQKqljRXnxh01eD6KboWeaeKzFkwGl01HWOIv7FOH6QipGzwqRX20QWwwpC1+yA+Rt9Vu4S4j50P8sZNla0PUHUACI8s88YYWTUSfhNiFyQ2fgA0rmMoQ5rspKg3uiFV3cgxbNNVI5unGifVIc535/Fzt5ZZbBNW6yjjNTe27cax5k8b03eBPaGTqoYbbGK3jDaEB+L+uB1T2LKoN9nC6HvEGAGVSkX2My673Wbvf3CdSCuPvcFI/qBbaLfN///3Wf4oMJb9HxJj+d6PCGP5w54G+DXg10MIbwFfAr4N/GfAb4QQXgd+I/4O8CvA6/H+d4H/5s9buNKBr779MXmvIjRKZN4bS6g0ZljDMqE5rAg9R1hbwtBhNhqzsOilwV4kHTioD6QyQjnxLRGkI1LVCrvWInEtpRuyvCvR4yEJ4AUYagf1WEJQmln0Xm0U2bkGr0jmGj+TMBtTxi+tJs6eTjy2UDFgQuRuvicJiu2y07nuZl6bnjBByUpqKuxWYbcCvJLY42g2URYZPwMErEF6JWEipkB6Awt5nQRgeJKV9DS6PJCsZd/yM0WyUJ3PqPON3W5EOrnnyC8kFMDngWQVj0kF7pUt6y9vsSvZXtcP0b8mo+kWlAcr4R395wLmZJsEZOaXcryKQ086l7TIdN6CSbpaDgFlsm/JMgL3CrJrFcGpMDguk8Hr+lHD8EkMQxhJcEiwAoS1iwEgNkjlxVq2odz3JKs4YIxdh8lSglnqiQzwqnGgmsh21EMBSsFIx6HyMdBkJB+QEhQSOkAu/YDiTWpL0HXsP2z7QVtQVU0DdiP/03FQny5ltj/EEIi2izJZQXHsZXC9jWEeE8guZRC/viMBPT4VkFkPpZ4inQtQKw/C5yYLgtkN/pcPZSCXLIP0JzZQHEntRTUhJiPK3+uh/J6fy/5vT6T6wqURkKWyL62PqgXLrQcMJEglu5Jro9wTQGUK8aHpJkip9wZMlE8KUJfaDrsWoFYcyLakS9kvl4sfy67lGLgUygksXpHl905l25JNoB5KiEqxpzBb8XQVB/H9a+gCT7qfY/lf0BKgUuwL8+MTOWbVWNZlSqmHqCYC8EwZ/WRWnrt8pFg9iAB1GyKYFh9aspL6DFPJYNnlO2DW+tOaGCpiYvVZM1AsXhEQ5VORqwIUe5r82lPsS/9jNZX3silj4IuRgJR6qHYTHOlOFpysBVh4K6yTdkHCayr5XwtKs7nITNuJq3b/80tPORMgbQthqaqJrNNuJKAFJLE1XYZYfyNgsR62QAnyK9kfHZNlm54MpFWAciqfp5sjQ+/KSUqshdVdI2mVmQDC7Z6EvPgEqqGwh9LJKPvvUiVhJj0Vw2vk93TtyeYel8H6RLM9EEDQ9BSbQ+k1FKltrN24cJQTWU62cGwPdAcwvBUQWY5jBcdAsb6lWd0yJFvfTZwoL8EzbXJqNRQg4zL5vcmkViO/cvFzKMr0928E0EQgUvc1LmUn7dyIH3m7Z1jela7EFiTVAy3gxEgAzfpYS3F9X7E+kn5CU0cP4UrOSTmRocfmILJ9GlZ3UgHaKwEZyVqqS6qR7o5vspb16iaw3bOYUmovCJBey5ddMTPYtZOeyxsTK66nI6gVaWw1MgKuUtX9vR5qirjczaGlmFmanpIAGgflTPoiXU9A4+qOBKLoSkCBbkIXPCOTHfIYwJQ+Ts46mr5hc5RKMnSuMaWjnNoOqNcxPKc4zDCFkyCbnngAzVbkusVRJhLPkSSaequox1ZksTElVYAUFAcpPtXUY4spHdVU/IJ23VBNbrCD+xlN31CPU8pZFpk/Qz1KCFb6HU3R0AwsPpNU1XJflGJ2WeJzg+sJSPKJlvCbVEJs7LLEbBp02VAc9tC1o9rvYTYVTT9BOU8zSHADCblpxlnHFKpa/gfQjFKR2/YSdO3wmRWJaiVAs57mAtCcl0qQ6KU01xvpwgRh+WJdRkgMzTTHTfI4sFS7jsq2czJ2V7agVHkvVSGxN7INxAl5gnJOQJzRIm31HozGD3P5vf1bmsiESJrswGUTtysEWXZVy+MI3micvF7fAJCt/LX9u1ISstPKXa2R/7UdmWkiXkWt5R6PT1cfAh2jqFr/pNK72g9rQbdSXQtZJpLZ6IHswHQIqFQkvuKfNCildj/bu7VyN7p73AX9fN9dJfYvj8fyL/HthwYslVIT4JeAfwgQQqhCCNfA3wL+cXzaPwb+jfj4bwH/fZDb7wBTpdStP289qzpj1JNEUN9oVKUl6RIIfZmuvvfgHJKAzWO4xl6NHzmavZqQeBjV6F4jADSNoQn9AI1807pMBrnNSJJBiWmbqla4cUM98TGdL9YWGBncNUNJPwxJoNp3UBqaQaAZS5efz6VHMCRBGMtSEdJYx1AKMKpmsmzXCwL4elKjIN4mqAeBaixf1tXtmnocwMeKhAgsvBHWT8CJLKvtBXRZiIBCPqSagVRtNCNHPZTXrB/I9ha3YmjHuIoR8bJNybTo4v2DkmNXHjiqWzW+1vT/uCfJoYa4TxK2opwi6Bi+EoFQPYLqqImslQzAi702MdJTTdrzEz2AQfZV+ViHYWLi4WhXsl2NdoxLe2xcJsC22BcgBeBNoJp4ir1Aueekm3Hm8Ums2+gFfE+WVU92ANVlcSDf8yRrAcZNPCdtkX257zvW0uWyv/UoVnn0hcVq2ajmqCYYSQpt4jnwifzuMzlHTQ/MRhIxpXA+xIQ+kcSVUzmv5b5n+1op2zpq8HFbmzyeq2EEvXPp1GzTQ30ioLBlpFwe2N5u5Joby7VSjSJbbUPHsraMt8skFTNEdjFZy3EQ0CHHoU0VrUcCLuqhimwesU9QjpNLBTS7TF6jaxnUN7H2QpjCHXtS7Mly2roFlxNrC+hqKuphiKEq6nOgtb2GAZqRSEWrcVzeUJgmCR6R60A7ARm6gvLAg96xrhC3Oyde87tJBp/Qhaa4XN6T9SgCgF67DAFUsGPJdQSFkgIqg+Kg5HzmF7EYnt05lC7JyGCG3XKqGPKSLKW3UnlwiawvnQuIaYFpO6nR9HbMc3btYxJjW7MRz9lARWY7At9aZLYtkHapMOYujyAs1nlUox271uRSadKCNp9EOWFkau2mDW9RIi0eybUQdLu/dExNW+mRrEL3bReM7Iup22OpOh+ujX5EYVbbOooQwR1RBaK6dXhLB7DbVFeXxaqOTCS17fmoe+LlNUWgyXXHqjW5ou5J9Ueylv/5mKbaJs82ERyqthrD0zGEbaWCSyX50iUI0Eihq3tIdxUtEI9VEhUn6e49UPciUM7bbZP9DEaOozDCIi0MRlFHz2PQu/qTNnW1ZRclyTRuR646BlcqN+TcKi8dncKyxvM6kBqNjoFO5O+EeA5aJkoJM9nWWeSXLqaH6q56ZVd1oSITTJc2HbTUe7g8dp1a1bGUkvYrEwt1X/4f4oQOyLXRpqEKAJF9lX2L77lUQIBLtYDOGyx0x1AFAaF263YTBXFCLlgBse1nU5vyaQrfsaNBxwmf7tpRuJ4mJDJxJGy+HK+mZ2TbIiBpk1pDPE5yHKKXLbLG3b6lwkwGJfUoPtVdXYbPrLCOkXEMWqGdADtJpLX4RAvL6EL0Onp5XWTU28/ekLRv1iDA0nv5GYj9lImsMzGRoY3Pi/2MIZGqj2C1sJAm7i8CKvHgBtIJ2Va0tOqPtkZElY3sm2f3+hbMNU7WkVipDHFOWNXa4Saxc1IpaByqcfh+umM7W8ayuQlubQcKVeOkQqStDIkpsKHtiozMZEgTwqYQlrQo5f9ltXvcPs+abj2kyW4/tCY0jlBWUJbys6qhLKEsO8D5/TelY8BO3ewkrTekrS1w/Fzn5Q2Glu9nStv3wl9UPfmXpG7kL3N4zw+TsXwEnAH/SCn1daXUP1BKDYDjEMLz+JwXwHF8fAd4cuP1T+PfPndTSv1dpdTXlFJfaxZr3v/WXc6/dUDek1Ce+2+cEgB3kZEPK45PrnnxByeoQuMuM/SwhlqTTQrUxmCGDTZ1JB/1UGvD7dfOUA5+9qe+i2oUh++e4ccN40fX0HOo1EvIjg30HyxQtebuGy9JH65gWuFOZOTnR04Cco5q/KRmdHeBnRvy+0tQItM0a406Khl+z9BMHby9BCAk8iXic6kScLNGQM3Aobxi8MTgbpcU9yrM/TXKKdKfvJI00KOC/ELh7hX0nlpcz+P2a6lx2Gvwr2ylV1CDu1vQ3C6xbywJiUhuh+9ckpwlmI2mOaqoXt9CAPvOguTawMMN/W/0qGee6TcS6kcF/X8+xHxpHiWBwmBmZ4bsWcLoD2UW0KeB7SulMJP3CuxaUY+9sKbrKCMeBoojR+/ThOK1ku39mnLfU099TJc0XZXB9thjt4rtkSeZq8hECkjN/tpl9HNJ9UO959GlgInemer6/9mRg3AAACAASURBVOy1xeWB4ssb7Fa+XO1WUR04xh9IqXbvqSG91FT7jvqwYfShYfvlDd5AcexoBoH1KzXZlWLyXfmCak4qJn9iWL3SUL22FUYy96xer6lHgeFjTTqXNFFvYfiJxLZv7zWsHzaM30shwMnvyKTA+p4kVQrI8aTXiuLESbXCz11RHDmaHqzeqqgmAkDzc9mfwRNN/4OM4tBz8NsJ9dBTjQPb247yuGH1as3yDbm+kpUMjFf3vXgAh4Hlaw3ZJYw+0sy+YcgvZLCzfthExlRx+595yr3A5U8ERh8rrr/YcPy7gYOvQ+8MUDB/3aP/+iWbk0Bx6Ln6uYrNbc/ee5r5Ow35uaRd9p8LW3751Ub8ZQNJpEwWUBx4TBHIz2H5Vk01E4ZxcyJs5OHXPddve8oDef9sjwL1GIZPA6sHntm3A5vbgfXtwMEfyTFavO6E9bVw9W6gOJQJg+JQnjN8HGgGcPllT/+ZJGgOPgv0TxXZFWwPpHc0Pw+MvqdpegLwiiNhNuuhgLc7vyUqB7uB4RNJsBw8DawewP43A0d/ICC8/yIwe9+zviOv3dwObA8kfGb0aaB/Kvu6PZRzVU7h8osShFPOpHezHquOFVQBticSDJVfBabfcywfKsafCJsWDPRPA3YduH5LgMH6tmJzS9F/Frj4ogQObW7J4LD30pMuAotHUsWQnwdW92Wg3XspEz3bA9XVUOQXgdFjz/hTz+qeTAis7wlQXzxSpAt5jW7g8l0BTBc/AbPvNmxOBDRm14FyBoMXns2xgPimJ+zQ6o5s3+hJYHlPM3juuX5DvtbOv2iwW7qJC+UCl+/K98j4U8fmSMJ5rl6X963dCmARGbji+jXTyb19qjrFSAv4XAR44mtUHYusfGD+qmH+SDN87tj7dsXoie8mQdK1MLOjp471LU26ChT7uktevfiiiuBTxaoKovdRmOrhM8foidznrxiKtg6jB/0zT7oObPYNw2eu6/cs94SpVg6u3rCsb2mmH1Vs9zS9c0/vwnfAvHfpGbx0mCoweOkoJ5qzL4mXTdeB6YcV80eaYir1GNnCCyu7r+m/dORzx2bfMP60ZPCiwRvF6pb4ZMuxZnXbMP6kwluYvV8xelKxui3SyeHTisU9KwmmM0051jF8Ss53NdRs9w2mFjZ7dcewOTTMH1mW9xI2++KjXt221L14HY40dU+uie2BoZhJiFJ+4SgmRthkHyhHmiZKi8uZYfLRhsHTgtGTit65I1l7Rp9suHg3w6eK+Sspkw83uEx8lpdvpWwOragCtj4y9UY+2w+ixLcIlHsJwcL4/aXUXRjxP+bnNcv7OfVY0ljrgSa/bFjez4Qh3XrqsWV1L8PlGrsV5jPEeg3lAunSYUpPOUtIFo7rV3OqsWX04ZJqbMmuKuqxZfjxiuIgpThMGX5vwfYoI1k1DD9ekl1UKBfIX2xwuSZ7OhfA4wPVLOXi3QF23dA73Up35EI8B9vjHLupCUqY06CBG9LcapqyPckp9zOylxvKgwyzrin3MvLnK7bHPbKXa+x1STXLsIsS3XiK4z6b+yMAXG5J5gXVJMWsKsxW1lfu5xS3hjSjlORqKwBvkEjXZW5xvYTNazN8z1LcHlEdDbBL2e42STV9ckUzyiJQ9NRHI/E8Jga9LqkOBzSzPmhNczgShjSmv/p+ipv18QMJ8wmpFaC/P8BPBujrNUSAqddbQj+T4J0I9EI/EwlwnuLHffFojvuEoYDU0M/l+VpHT2QEpMf7hCyBg5kA0tkYjvbipIjC74+Fua0b8VPmaQcqUQqVZ6jZBLU3Q03H8nh/hppNJSBHKVSeo9IElSQoawlR1qr6Pdnmouhksm2oTggBlVhJg23rUSLr6Kv6c3dJmW0IXeJss2NHv+8e6uav6kZ+DG4/NI+lUuqngN8Bfj6E8LtKqV8DFsDfCyFMbzzvKoQwU0r9E+DvhxD+Wfz7bwD/aQjhaz9oHeM3j8O/9T/8Cn9wepej4YonV1O0DmxWGZPJhsWqx/HegtoZNqV0PPpao3TApo76vAejmuAUvVHJ9qrH4e1rzj6bMjxcU9eGpra4hQRQ5Idb6tqggGadyExWEtCJE6VAYSRRVMeZ5MwTCtMxmMmowjVamNWlJaQBrIegUBsjQHSRglPC5tkgy9KgVwYOSvwqId0rqEtL2FhUqcEGKYbXAWUC+izFDRz0HPo6kfoLE2jNGaowmI3G9T2qVpJ+awL2SoBWGDWyfVlcZuoJpUh6SbxUalxmhEEj++sV6IBZGNy0QS9EsuP6nt7hhuqToQyQItvBpMbmNeHTgZzIIBLa7EwLQJxJiEi7/3olPY56VMNZRkhFluyzQHphqCcyqxgM0unXCGj3iZTUtzLYoAWsq0ahCw13toQXUrasK0mgtUtN0/eENC576lFVW6wur7Ur3flLXN72DwbcxGGvLOO3L7h6PBPf7sCTXRiqqe88mtWBIz03wuIq2ebkSkI90FLNoSol56i3W34wMsjUtYBju5FKDD9u0CuLLkQS3fTkPLdSbTduSC4s2dtzig8mXVpn0w+kC0kzbr29LRjx0SuHEia9GQR0lBvrKk4KzDX1UFh3u9KYqgXtoUs9RUvtQ3Es4L714aVXAlyrw4b8WYJLhen0qTC/veemK3MPdtcVaUqFj32NLQOeLNpZdZmcAMDLcfJJQDvZLp8FlLTSxMRbRXYpYTooSOct4yTMVD0OXSm9+PRE+uoyduxMlCI3A3mP27XsVzoXNs5uoJzF6ocoUxfWHHqn0reXLKUrsu1fJPrxXBbBTh6Zqr5sT3a98x/LSaOTUiexAqIe7I6R3dCVvLtMZMgiLY3nt2j9ZMJ4eyPvm2BEVl2NRL7qkyh/jp2Qra+1lep6K/tiKmEEW8WEKWQfRJ4s8mW7iczpTB63PtcdywrZdWR0ozxaNbvtwEcp97XsdzWBND4/XUQ5YlRryHsm7psV1rz1LFczCZVpQWI1Esl6sMJeSj9ilKG7nWRX2MEd033z/xC9tJEtq8ay3abcMdbJUtZpt9FnuZJk5XQuQLsayd98ZBVNGbrOwa6TVAuoFcl+ZJ1TAQB2SycLrgfR5xsln3Yrz2/fX/VQSdJzZIJdLqyw9ADG4xmZ7mJfYQuRmbdsp25C571vesJS9M8lZbft9mxvNsrTm56AjVaiPXjhqAc7r2IwUVZqYsjVRmTT6VqkiU0uNSC9Cx/91KFjZm0hADTZhuhzDzGIiF2foiWyfLJfrXwa4jJK8Ssna2GYTJT9uuSGB64laxo676uuZdtMJf5lFQJNFj2mAXzcLxc91NncS2epD5QjQ++ywWWauq86f7At2s/+yJLW4XN+zXpgRA4MElITpcKt/1fsEwq79QKco39X/NyBcmKk8mNP0mRtZEJdrskua+nKnNoohxbA4rK4fgWm8LhUYyrxvLbrMNEDG4z0R9qti97VQNMTIBy0whaOJjfd/10mtSLVJCG7EvDnE03TM2RXJeVeht06mczZSAiSdgG7rKj2crkeSo9d13irO9mojnJau67xqcFlRoCwkWVrF8Qj2nhCYjCrUpJlnYSRoZWoA3xkMaP8Vr5rjTxWiqA1aGFGfT+V7kqjaHMHlHO75bSvjX5nVTY7n+PNn07ktKqoCL1MmEVjxMt5g8FUZSWsaJbu2NX2a6IN8AF5vTVdYmzwHhXlqy0mUErtpK03l9X2YzYifw3Oi++yZS1jt2Vo19fWjLTdmt+POW4mx/5F8IgPhLr685/3A24/Kh7L1/+9f/Eeyz/+tR8Nj+UPE1ieAL8TQngYf/9FxE/5GvAvhxCeR6nrb4YQ3lRK/Xfx8f8Yn//d9nk/aB3912+Ft3/t73A8lOTTSbblq5PHPK8m/PbzV3h1di5JptdjvFfcmi1YFBnjvMRqjw+KopFvk2dP9vmVL7/Hr3/rHY4OF5w+nzLaX3N/es26TtEq8L2PjqU2pDbMxhumvS2Xmx7LVQ8fFMEr9qZrruYDZpM1vaRmWyektuFq1cc5TXWVc/LggqJKWK5zrHW4xnAwW/LyYsx4tCWxDqM9L57PePPhc773cp/RoODqakjWqykuc8bHK4oyka7I2mCto64sTWXI+rUA2KDI85rNKsMmjnqbcO/OBasyZVum1LVBawG2rtGcHM5ZFhmr6z46dQKgCwu1wgwbtPbUyxS8Yni8Yv1khN6v8E6hrccYT/hogH1jyXaRo3QgeZoy+clzlpscYzzriz56adCNov/GNctFD6WCdE5qLx2f87RLdVSVgnEjAP2mVgYEDA9q/Iuc5O6a8qInnYRZwMxKmsKiU4ffWuygxtUarlN0A27k6D1J2D6sMFcWN3aoWqNnJf4iEz/tSYl+mgubbAJqq8nPDNtXys9VkiinCCqGKI0aRt/I2NwKNHs1VBqSALWCnoNaY+a2q1FpAbAfiBdGlZr8paHci7LKqUx8qLXBbkSumiwU9ThgVyqGNKkomQqEvQp1lYr3c6Gpx1FunXl6TxKKA08Y11BKn2C150BDemZwfQGB5YH4LbuQiEokfMFGgDP0ZGeG8tCRLHQX4lTuOzluBgGR8XQ1BzXmWtKAbARmzcSRXJkYgBMIfUd6amNFiUyUmLWMom8GBakA9dRjYriWrmVgV0092aURiWoHQgXgtGFKyVpk1aYS0Kka8akmCwnBcn2RdmdnMSyjFGBSjwU06hp8Bnalun7O4sBLz+OFwvVkX9JrRbnvyS405Z4wo0GLtDe7jgXlRsBkspZ1Z+dKKlSWrbQ0oJvoJ76WZbcATPzJ8lg10AZMpUtFuSfgObtWHYjpQpEyWX8LdLMrAVP1KL6tlIDtZLHztG7uOPKXpjuerTQ6P6cLYfKWeEzleNcj2WbtIlBMBCA3PXB9qWsRX7OATh1lpXYjoFOCxCRYy2yV+GePA72XuwmEFjDXAzmvsNuOdttdJkA22HgdbNvwJNXJq9uJAblmd0FULpNjks2FMU5WUlFSzlQHoFWUB+pGQo6yKwFx4tWTayRZhi4kS/x0dAFgdisAP7sMVFNF72WgnKr4niN6AdXnnteC0VYS23qDm0Ec7EWgNHrq2O6JH1NCdKIcNo37mApbW02ENRZQKEzt+ElDMTNd2JBMJMT30Cp0TLeuBPRIhYkchzTWhNQDqeSphqqbMOrSUdvtyeT4J1tPMREgWMz0Dsx+X/iSBAHtwGDv0rPdk0F8k4s3N9yQ5FUjkci2YN9UO7msN3Kt2BjuU/fEF1uOJLQqv2zYHCUC5lS7bi1djY2sL9l6dCm+TJ9Ej29Pka4kAGq7rwX4xrCnJhc2lBBIVj76PLWcTx/kPI+0sNOFeFrtxlGPDE2mpbsyhi+1HZatHLiVRSerhnKWfA54yr7LsTGll4qVjacZiFQ4v2yoxobeaSWe00THqpOK7XFO0NA7LammkhDrMo3d+hhAFtcRJb/pokHVnnqUdOvWrVS7DTmqxRcr4UqeemRF1quI4Tzy3Py8oDiSHu10XuMykd6m15LUatY15X5OshBNezOw2HUjMvDM0PTET2rWNT6zNH1Ddr4VT2YMFKqnGXZZY5YlIbcxuCjE5SUSkGR2+6kr10llW5+nBBK5CCrFh4oLhMx0sttOG9hiqJvS3LLuAFcwBuU9N8N92r+p5QY/GaI3xeeksy1QU8sNYdTfPc5T2c+tqOjCqA91g1ptuuqRLnwIBLzeTGt1XhJftRZQ2LKkG0mvV0ki8lkbtyUynDQNwXmRy/ogDKQxUNd8fwVJe/v/JIMN/se+bqR/fC+8/u/+EIDlf/WjASx/aC7YEMILpdQTpdSbIYTvAr8MfCve/wPg78ef/0t8yf8K/KpS6n8CfgaY/1mgUtahOH85ZlMm9LOa58sRF8WAF/MR23VG1RgW5wNwitHxik8+PqK3t6WsE7KkZlNkFKuUbFChCs2v/8m75MOS1AjVvvpszMdO472mmGeozFPMM8zcclFrLvUAXxpsv8EtUpRTXCnwFynnW4POHL1BxfnZCJs5eNyD/YaXF2P8KgHrcdpiLi2nlSFUmvn1JLKoGnue8L4+IQS4ro0Uu5cGnGJxNiQZVmyWGaHR1NYTao3OHdWTAX5Wk5ymrA8seEXthGI7vR5RnwlTq0ygLhMBR1vDs9UBelijLhMCiTCHOqD6DldpXG1RtSY9N6zUgOxaUyYim/WHFW6eYrJAuU2g1OAUulLM/+iA6rhGpR57ZWkmDfrKsnw8RpI4A6rUeCVsaTLX1FMJWvJ5IFRiwOg9Tdg+qEjOEupDASuu0minKC976EICMrwKNBuLXllJ7Es9POmhMmE6lQfm0k/ZDtDtlZUvxyqHNA7cr1JMqdDnwlwS4qBmI12mzf/D3ps8y7bl912f1ewuu9Pe/t1333tVr0pVkspCCMtgMwUizMARBBAw84Aggr+iJhBBGIIBBgxDM2PCgBH/ggkjLFBjqVTN6+69p892d6th8Ftr53nlkuUQKlyl0I64cc85mbn7zFzf9e1OpccyVJHywaCcnOdxDsVG4Zaa+p3FzVOCKGAO0mMZ9iLrDTZS3WnGtF+hSszXQQbRfiyFDW6PgU9ZQmt6NZXGD6uUEnyf4vETc1lsJazJBfnCKB80biySj0ehEkC27XFgSpDApnEVcQtP+WAliGguYAe0BCP1Ct2rSZ5JhOpWWOf6VhgJNwd1U0hScp0nBuR660HRXMH624HmJwW+FrAsTKxUNYwpwCpUJEARqd4bxlVIr5cyedNqyjV0l4lF7lN3Y5sG8lWq1ki+peYdU1oummkbdpv8a6OAtqIXb6WvI/WNpr8QGWsuXo9GS2BUYu7KhwQoduJtsm1iuTpQCSSbTgbr9SgS7tlXwqiWayVsmRZmKKfihlJeM3sfJGQme9MS8+wT+2r3TN1/MjhNzFSaECi3ck2kekFAvvIxpU4nhqs7Ap75V5FxJUXw3bkAhPo6+4lTSq6G7KU0KUnU12rqprQHoGZKxrW7o68OldKadQqjWkmYVH8K1X3ENkcwU60FbPenKnn4ks9tLq8v15HDc9l/V0O1lvXZAxCguToybFGLXHlYicx2/1JAm8gtZXxV3csx65EprCtqBASeyHtT93I/2hYgJnY2TkxgDiWyfZRwt/kRSA8rNQFjM0xvC0kuXijp6jzTE2AvNnIvlbswsXLVJrB7YR6lMUfKO/GI7l4aqocwsczBimTVFwo3FwZJhZzAnUDcoIgm0p4Le4NOycM6+1yFhasfAtvXVtQ0kNhKeV8oL12nvpRJo8Vbz+HSTKygL+V6lLvA6DXVxk/nvXrw2C5yeJISZ7Vcq2L/iH1LzLr2wuBpD3ioegG05TZOScTlNk6gJs9HiiJDALAeowQPDfL+DUZRbQPaRbpzS/3g0UOYQn9IgULduVRu9CtDc+umCbhgSInfkuo7v4oTyxi1AGrbhikoqXpwqGin5FkzeElDD2D3XtJi02PFXvbL1woVZV3F1h29nTaxaQqKvXR+6iEmNhSK9ci4LNBDQHuR0cprwLQe3Ygn1XQe4yKq0NOkiR4E6NjW4xP7anov94s+MswStCOJrSqC3Y342iSVS0hsqnizzWHEn9fo3UgoCsq1xzUGu5cQxWgLolEUG4dvDL42EvzjxdMZGwGBtvNoFwhWHycBkhzX+lEmN5O0VcXkNXzk/bPbEd2lGo3OERfJHO8COiXFKh8ltVYn4NiJBFYNDm2Tx9QFYTATKFQxQkrIndhM58UjGSJRW1Q7iKzVeQnb8YGpliMnvCIMp+zsI4bSe9QYv+alzL5NqU6RZN9ozRF8jk48qEpNoUNTDUneZnzEKGYAqDU8lp9mxjImNrXvpwAfeV2653149Lckn/1Z6bB/nlTYv1p+oZefd93IbyB1IyXwQ+DvInM3/wvwIfATpG7kLtWN/H3g30HqRv7uP08GC1B98ip+9Pf+U4bOEoOCTSFJqyqxNzZSXRmGM7lZi42GIAzENIuZaj+KjbBBdn+cFS/XMoOuAiK9OhWJW07DzAXyOlVj+EbemAIAZP25niFa0sAB+otUPN8e60jcTAbj4xKRcKa01u5StulrmSWf4vWjDGzGZZLypJ8hS3uYQg8IMjg2LSJdaWFcMdUzZAlcHiyNqyNLonsZ2Fb3MlCbZpBHAQ3zLyO3v+WZ/cRO8kA9ir9NO5EGNlfHYJIsic01C6ZHAnTao2zN16SCdzVJ2Hwj10NCDVJtRGZJZqnGoj4et5ul1NYk/wuFDCYlVOJ43MMJrH4kyYzSMyfnOQ/cs9xQZo/TuUyBI/Yg1zXPoOfI/+ohTkEZ1YPM8MtAUvyhuTMtFPm+SSxNLwOc9TcF6BQ7WW8+ZypI2qmbyX1khmOJe3+msXthD0JKQRWmR8BEsZV3nk7VAraL5PTQqGXfXCNfiMVeBmZyjCJr7c5VOodyTmbXQbxJLk61J8WeaQCaUzAFXETqtdQ6ZBYid+ahhMVwjcj6XKOo1gKgJARITTIhe5B0yf5EBhH53o6JzPaForkLEkAyFybGF+Kxq+9yYAhpYJuAwyQ3O4bTmEGOyZdqClzKFRc5pTQYOYdZ5mf6SPtEU91LwES/UsyuA/2Jpr4XAGB6GZSPM03RSim7GQTgBKModykVN9dnaPlX7kTCZnoZOLk6g7HjZ7cvFLNrT5sG8drHienJrKJrErAdRVYo+y9MTLmT90DUivnbgf2LckryzINHM6SqiiEyuxrYPy+n/XCNgISxkV7D4hBSlYeZZH15kG87AT++EkapSGXyUR0/s3K4S7CyvXLjGZYmgQmp2cjHkWtDojmel7we0wkzZA+B9sKy/Lxn/6KcmCrTy4DX9HKOm/cD/Xkhg+CUhiqBYJpi71Eu0p/bxIRFxrkgnmLnJY20l/eVfGik74AUCCPvSZ+SPI0klZaKau0liXSIAuQM6X/xe2b5pHhcew7Pq3QeJRjFdJ4hJYrmUJ9y46a+vgwOQqknCXBed07Vtm2AEHFzg6v15NXztZ5YlmgV5WZkWBYTC5f3K1hFuR1F0th5uvNS3qNOJKDKpVqOvASm2o7jZ2GUMJggn3W5bsS2XlJBF4UkspYyaVPddOKhS+sNKfDFDJJIKgmsIol0s0LGBUmWqWJSIQwePQbaZw3KR4qDwxcaMwpYycDOdlKpYXcinZSqiwR4RrkOevDEQqPGgG9sYvZk30NpZH86f6zomFl5bm0lLEeDGiSARg+eUBrMfsCt6ql6w83lHOjBT0E+6nEdxugJZUootVpSU61Gu4BuhbkDeZ12AV8ZYc98xLTCKPnKyHk5jISZJI0qJ4AkV3/o3uFOa8x+lIRVrQm1xWx7VNcTFw2qH4m5osJq/Cz1MbZOwJLW4ALutMZuewHIRUpt9VGAV51maXyUuo6mPN5DSbKqO+lvFHZukOekwJ+oNcpL0E4+hgyiQl0Ko6hBHXqoSgFm1vwzbGHUGr1vJ1YSpY7ALYiHMae7htUMfbsRsJh6HOO8QXW9rL9KSakpuZYhMXmPpaQxTvLXaZ+GEaz4FlVxZITjOEqSqtZJkqon5lDOvT2udxR2NDoPwTOlu8bwNTAr5zwcw4JyGqzSTFUfo5u8lRgt+wdHNjEcgXJM4UZyAzzaxk9XnPyLLH9ZGMv/8OfAWP63vxiM5c8VWP68l9W3n8VP/uv/hO9evuftYYVWkZ9cn7GY9ZRWbv52KHCJdWyqgRgV231N0wzUhePm3Yrl5Z55NbA51Dw/2RKi4rN358wWvchIS0/wmgisFi27Q4UbLPVsoDuU1LOB9qEWb2TjiYkF0pXHGPkGdYN4Is1yxPcG1RrseYcbDHFIb06vZBZYgZ2PuFZYNDQ8e/7A1fUKXQhzpo1nPJTgFHYx4m8r1OmAsQF3XYs/sQ7Qa+zJgDtYYVbvK/TpIK9/qL92Ps9ertnta9xdDctUqXKw2JMBf1/BYmR50rL9bEWcexg0utOTh1MVgbOLLXfvVyL3vBywlYMfz3HPB+JB/KaqCujSw9sav/SSvquFjYtlQFUettI5qpx4TeuLlqGzhINFt4awFH+n3Rh8EyRs6MSjGievbTx6Iz7WqCPF2jBeSnCTGhUY5P+IyFCDorgzjC8Gka/aeDy+KqA7kWjFmaf6qhCJaSMe1XwOYu1RvUF34v/UncKvPPbO4ufy3FiILFX1Wli8RUC3emIhtRePZV5UL49FLfLd8lYYCt3D+Hqg/kGFryNulsJ3tNTegDCSRIhVBC/7Xr4rUvqlYlyEyctbbIUdjMlflyttAGKqdfGLQPFgMAfxXAJTJYzpE2BPbKubxVSJEig2WhIqSWxdSnfNHYUE0F5eI/2HkhbsZikp2QirXWxTP2svzIr0sylcOg638ix+aDk8D5hBZKTFVnygpk+go5e/h+TXBCbZNVH2Y1gF6muRBBdbjavjFKASCnle9SCJmWZICcRJ7hcR9lc5SQ4utgqfGPB8XuxetjucZsZO/JD53EQt7/kMUKRXViYBQCYhHvdfZqCe+0ftXl6fQ6ryYg8y+ZQ9eVl6ln2LOYky946qNPh3TWZcE8PWJzCSqllybyWPtjfOI+UmSW3nsr5c5wRp/ZGphsW2TInO5ZajNy+myZXqCILyevJxZD9v3uecZlvsYPcmMHurJc01MaW5T9O2MlGn0uSHsMEkb6lcs+5cUeyZvHk5WTj7ZHNPra+O10uudZocSAAsJq+g6Zgkij4BfZmMS0AyyU9zn2qekNSjyF2LxMg9lri6WaqnSccoYS7J0zrGyU+Y5ZLZq52vfZb9Ss1Q6n5Mk0LSqcoxBbdIz5vJhFa+H/I1CcVxcu/xfeRLEsBkmoCU9x6TLFd5eZ8NS3WcUHPH90A+t0f2U7yQPvl9fSG/6zGmpFc5Fl8l3+VS5LLZ35kngJq7mNLDVfKTPuqbTBM8uX/V1YpqI5Mwtk8+1DyBi4D9fmWEWa5SH6lR06SOAHEYFjKZ87gTN1e75CUqUScUe5lUKfZhkhwD2taw8wAAIABJREFUk4c3s7D5MywnxkaTeyHTCtNniptpTBdFHqtIrKdIVEMhrGW0evJRKp/S10ud9jVOkmPtZXKl2MqFCqV4Ku1eJLw5Pbc4OMaZnSpXopWKFV/qr3lGpaNSthOsHGuecDC9yEplHx2x0FOViN2PE9CW7s0g66+EMZ3SYvWx5zeW+tjvGVM35hhE4jor0L2b0lQld0DAuACtKJMIEWE3BycJtEqlzlKDOQyQGfMiyWJTT6byAm6jThMD6WdAAGhZCLDLdSGZ7cwdliB1JfuWaI0E+0w3Tt5mAtvDKMAty17z+nUCrePI42XyWJalgNjMaObHE6hT1hIfv3byg8q+xHTOp25L73nc6zmB9/y87Nf8s5a/ApZ/6vJXwPIvYLn4zpP42//jf0RtRmrj+GJ3SmE8J2XL2/2K2jpOypa7bs5hLOhHy9msBaAyjrt2htEBpSKbQ82sGlnVHV/dn/Dm4o732yVaB0LQ4gNUkUNX4UZDWY1UhcMHzaLu8UGzbSusCYzOEIJiOevxQbFvK8rS0R4q6mZgGAzGRIrCEYJmGAxNPeLTz/NmYLNpKKr0YZ2AsTYerSPDocTW8obWOuKcYT7rabuCqpLtlNVIu5VkXDcalA4Eb6jqQaS9uxJdBJSOaBVxo0HbgNZh+tzyXhNGTVE73GiYzXvatsQ/lGRfl16OxKAoasewLyVp97zHD0b8jfcVsQzo2mFsYDwUmDp98AQIo8HUDt9ZlA0CjLcFFBGlI9EpdO1FOlx5mcwbtAQKBRkgTEDQRFRr0Gc9RIU/WAHrlYfOQBHk9yJ9cB6MBBRFJlkXCtnvwUBQEnqUpLhEJeCwCrL9Ln1Ql0FArI3TRABOTetWPgGhCKaV0KRibXCzMAHqMPfgJXTIL4TmNjt5LkUEp6bXqKCm/k+/9AK+tchBQ532wYtk1q28gGIbsXuNWwaiSs/Nkl8tQUVuHqdBn+lEZhuNAEwJGXq07jZV7nhh3l0GpUm+rJOky80DppWwo1CHSRmQw0aKrWK4COhOpcGNhC3pMfWIJhloHvCTZKrjeUAftMhmT8X3ZNpUqdCkbRqR7iovwMt2CldHFEhVy0wApsiOpcJFJ3Bl98fCeD9LnbLbrBxIoUBO1A6+RDpaK5Ht5m7SPJA2fQ6QUlPICaRBdXMEmr4UIG0G8VGKd1a2k315Uo8g103CeeS+jFZkxMOpePp0mjQhCvPuZnlfkjzZpa7StO1iK+oI7RTl/dHrmHtnx7mAYFFopP1I4NandH2TJKoC+NPfMmgNRyloXnJPKTAx2SCArb5NSoVUNWI7uYZZghtKJqDzuFs1VwxNzH7a12IjEtMMlnNwTFYMZHVDruzJ+/UYDOcQnXGhpu1lBtAMR7Ceq06yDFX6WdO9szsG5BRbkfYWu+M6i4Mw2Nnfq+KRwVWOr4XATJMH9tH+Bhnsu1pk3yD7LL7eR0xBTKE2deqpzN7nBMaE6WQK9ZL+0zi9P9xMTWqFHMZjxvi1yYncj5z92VnNorzIg12VGMlO6m3kOss5Mf3xGtlOgMckZ01gt9glhcYjL2auLcmKC0kqj5NPLgPBok3gPgXnZFCaWf4cxJNrMPI6cq9mnighZvl5nPbNJIY6KvGP+lJPvmoJkopHpr/Wk5S22HtcradrNnkzExCM6Rr6InssxZdoOpH0ap8muKwASuWlIsX24gXNPj9XS7hO1FJ1U6395D3V/ZFZ1n3ALQzF1k8MMAgwzX2kvpJJDbG0qNRvKkyofI8Eos2hbipJguU5ObDHzQx6CAkAHkGzGUOSYev0eAK06f0l1+0ogSWC7v0ELGPxCDiCSH+TBJwgfZTygEh7Y6ovUSFK/ycCJh8vKtWaACgX8PMS3Tt0OxILSejNDDAaCe6xmUEUuSxOvvenrkcXxNuZ1pu9j9GKWkBlL+QwCgua+ygLi+qGI5B8DDC1/hqzmh/HaNShO4LTGI9dmXBkS+EY3JPlufk1zomsNQPGyWebgPDjGpKQwoCyrPZnJLr+eWpGfunDe56+jt/6OQDLf/L3/wpY/n9eTr79LP6d//lv88X+lKvtIn/e4IOmMJ71esbJyQGlIqM3xKjoukKe4zT1bMA5jesKimYkeI2/rWDp5H1kItErimYUxtErcJr6tGPoE3s2c4RW/CbFfGDcVqjKE/vHVIGwjM2qo72aUV22jIMl7C1oMDOH743MLDoNOqIbR3Bafk/ME5mtrB2hN/JYWpRPbF+n4WSE+1JSWx8/xykBMSsnYCnPSjtFbAJmPhJuK2LjUTbC1kqyKKDbI3OnnvUC2nREbyw86VHvKvzKY7ZGwmLmHkYl6a6rEbWV8JY485i1wWe2zEgfaDQCDPXeEGYCGmIps61Z3hyqgBp1Aj8JFBXpOVGBBxXE9+ebMM3Q6kHh5xL44pceszOEQl5v96mT60QCeiZwNw/TdlUOpBkU/tRh7y2hkFlm5RMrc+qFhXQCyoanbvJt+iYkRiozIsdgCoKS+pcotSuhlAAZgiLWnuLeSmJqtjsMAojMQVhRu5NBzrjM8s0kiz4J07GHMlLeG9wiCPv5SHJoEis5LsPEDtt9Yld6AWv9kzCBvqgFyBR7AWkA7lRSbu1e0T0LFGstoUAR7EEGVeVGJYmysJKmU8QCxkWg2GlhSF0aSB5SD2CTOjFrCUbKabbCmqSk13R/2oOie+Gp30m8v0veTOksFDY3B+pELVUx5qApH1IVRQIVUUPMqaOnEsxjOmET+/M4+VojTNdaeWEniRLsk18vkjfZF2GHhL2MNknwbUr8LeWeMJ34UHNqbtRMntDcd2n3amJP87XOYEonOTYwsU2ZTcqJqMon0DU/MmoyABQp8ziH3L06zkV+Pi4EOKHkuDK4c408L3sefSnP00O6TqlPtlyn2ozheJ7zOXd1kuumCQOf61ouVAqwSQE9B9leuZZ91yNTAEtOqS230J9IqqtvxCfp6mN/qbx/5Gc3k3UNK9neuFBkr2gGF1mCPbG0Wv7X7hHQSe+lnLqaU49tK/fDuFCTxNoeOAbyHMT/Zvdx6q/MwT31nci07SENmBH2Kg+sM3smxyz7btsUwJP8rSZ5Dotd/JrkHKSD1M2UsI4JcNsujQMSc4Y+Mqs5IMunPlNfKur7OH2O+EquT7CZgVYUB5moASbA6JPfD4QdnBJx93G6n3MoU7YXiOz5yA5mX2WWqReHR+Avpa0GIyB3AqLxCNIzux+snEfbxel9FpWswzUCVnL3pvJy/syQJqK6I5DpTiV0SN4Hkraagd9jZs8MX2fMQFjLHC5k+gQghzCF8UQDw1xTrQVUDQtDcQjiY6zScfapTzf5KfP6Mwto0nXtTw3lVjyS9hAY5+LbJMapyiRa6fM06fgysA2FQvfi4zStTyqGHOiVGOG9n1jN6VgfTwQkqXOWw2dQ6SuRGY8zS7FzjCvxaebE2ahEPgwIYExJsqYTKXK+Dib5JEMGcRqUi2myIb2mF1mwSmAolAIip45HDWY/yjV7JP/Nx6I7AY9onSTHybuZeoLVmLot9REgT5LcIqvSongxxxQKlJ+XA3syS2m0/JyBXWb6HvkvVfJTToDykcxXZMtagoGyzzJvIz8HZBveH4Hjz0h//enHJszgPSjptHzMXMZhnECnMpo4PGY1HzGWP/37n9Jv+dNLjJHY9/9Cz/1Zyy8MsPwPfg7A8r/7xQCW+s9+yi/u0jvL716/4N1myW5XE6Pib7z8MR+cPrBez7g43zGvBqrCsbubcbnYU1Ujy0XLYtXinObiRCIFx+uGp+cbopGwmHI2olRkcdqiFNhCpJ+q13QPtbB0q56idOAVs9MW1xWUq564t5SrnmLZY+cpGbTXVIVDpaCZMBj5tArgNwJs9cZiHwxm7iYW0DxYdCueErO2mJkjPpTgNHY1SDCNFrZIHaRGZLboQUXK5cDzN7eYvUg6iwctrNFsEIDohL2Lpby2+KOZ1HAAqEh9ZajfWpovrDAgRliLMGqKGwu9Jp4PhFHL40Vg/oUm1kGYw718mOrUkakd4BTNlczE6dMBuxbWUPdyjMVOUb+zAvq2mpN/aggLz/KHGhWUDFJXjmIr26nfWcxBCxj2CnNQlA/qCDi9SjUXSoJ49gY1CsCu34mss7qXx8sry/KPLKZTNJ9bqmtDLAOhDhO4UJ10ToaZp1hrqYgYoP5KPKbFVgsD5UUOWWwUce6pr/QUTBMSIDMHqbvQvaZ8scelBFeCYvaV5vwfizdi9UON8opiJ0FBelCYQVFsNMVOAJgemSSi4vmVXs2oob4ymE6Cddws4hthoWZvE1AMsi+hiJT3sq3qXgJ8QsnEJpb3ArL8qUONAu5klllNDGM0yXfcSy1LsVYTw3f+B0GON4XF6D5NGiAD/iJ1moYiUm6FNTStYvZWej+l7zQBK0eSzcLsrYTe6FYCbLSH5kr+RoTqThgQnaSv5QZJnE0/205kneVWBs52JzUWepDjqm8T2NvI+qs7NSXNhjIB270kwxZ7qb2wnfiETQf1tfjZTA863Re2FaBpW7l+JoUnicw57e8o52j1o0B9wwQoZUJDthGNgLzqXga9xe4oWxVvdJzCg+ob8UrXd5FQiF+3uieFzxxlqPYgYCvLSrNMNYOyYivdouU2bcvJfpJAY5Y4ayfnKiooHyLFPmIPkn5p20h9G1l8GaeKjeYmiLzWZCAmj5VbYRyzf7DYxik51vSRciO+4GInLEG5jcy/DJJa2so2TSvrsompKnYChJqrxPCmBNfqIU7y1MyuZqAGArKDEcCWPZFZJlqto/go+0ixk21FI9dJmFIJuCn2kflVkDTQmPYbCSAq1wJI67tIuY0Uu8R6DnGqnrFtpDjEKRlU/jFJRJWPzK795MnVHppbmejJAS/KSa9pNFDfy76YDqptZPHOY1o5Bkn+ZGIri106pwl8qJhAeJD9sN1ROltu0ufcEKnWnmqTgluGNAnTxlR5kb7QE1AtdwE9RgnPGSWkSDthGbXPHug41YlIEi8UWz/JKYtdEKYzhdpMrG+IidUM4t3dBcq97MDysw5fiV+3StvOrGkGgNpBsQ8Uu0D14ASYKmEbkTk3qrUM1F2Vtx0ptl6krVZRPjjKBzdNRghDHCnXLlXqBIqto9h4OeYxUOwctgvYvccmoKW9PFY+OEwXsIcg/ZlpQkCyF/x0zHpMgUULTX07pkklhd3JOoNVk6dYhUj54GSSYOuFZRyCeER9Ov9bn8CXyFO1C5SbUeo8Co2biwy1vD2k4xtwtZHnjgKW7cEnICnANktvtQ8ToEQpzH4UGaoisYtaWMWI+FODMH96kH3Sj2SwkkLrU0WYJLgqH8RTqgSkmnYU0BfT++puLz/34iENjT36NAE1enSXPIVGJQ8s6G4QljL5UVXXSyqrz9fEC2MZgvRPjinUJwE3lUFY/hvCWCofpiqRDCizf/NrQDEvqXf0sZ8xloWwlo8SXicf5mNv5fSCBCR9IIYgHktrpYbk8eOTzDWxl4mVVEqlv4VpG18jsvLfQzjKYlPdyc/858PXA4H+avmFXcz3v//9f9n78Ode/qv/4b/8/r/xH79m70s8iifLPf/4Bx9xcbLn8mTHF3/0jOfPH3g+3/Ll1TnffHHFQzej7Ura+xlF4/BRGKt/69d/nz+4eo6LmuK6wFwMfOvFFdebJcOhYLHsGB5q5q+2fOeDd/TKsL9aQBl58eKe+/sF85OO9rbh9MWWphrZ72rsT2qef+ea7bbh9HxPrzV25ojAb37rJ7zfL/mVT97y/stz6hd7zl+v6fqC2aJnNe/Y7RvK5wfCtuT1d98zovnGh1fcdY1IoZxmcXEgvG2oP9oSV47hiwXlywO//vItP3z3lDj3hKAJRr6An7xcs4sFeu6Zn7TY2cj5sy3+cqTvC5588MCvvXzLj/0Kd+aIUWPf7PHOEE48n75+z931ir/11/+A634OOhJ2Jc3zPeHDHkwkbgqaN1vOn2/YHyqRiS08v/3dH/KTck592jM81OinPfaLiubba8IXM8yoKH7jnt5bwjzQv3aYe0v7xrF4vqM1lvl5i3uoOP/knm1peP7JLYexoDzrUecD41CiBw1Pe5ERz4O8Nlpmr/b0ShOLyLPvXbN5mNM/8TTPDnA+0pYWPwu4k4BbBn79Vz5nHUtGNOFy5OLlhv2uwZyOjMtIOB+Jo+Hkr90ylJFhHqm/ueVvffNP+NHbpyy+d0c/FOg3LZ2y8Lrj+at7XB2ZvdpzOFV849O3XF2dAgpzKsx5sIo3//ZPeO9mRGfRn+4wzzs+eHPL89d3vDc19lWL35QUv7qhi5b4qhN5rlboVwfst/d0Y4H6oMN5y+x794wlhNYyPhsZXo94I97R8LonDgbzzT19sIQ3Pd0qfYF80EHjGWbClhYnvQRidQb/omfxZE8bSqJWPP+1K/ZXcy6+d0Pz/MBuM2N4Ksk3298YGBuFfzpiNobulWf18Zr+riY0UeSf39yhngyodxXLf/OKTdegvrujrTTu+QidYTz12L1muAwsv3NPfzNjXIJ71TPMZPDmazi8kkj7/oWX8vhvrRnWNf0ZvP7Nr7g7zAhW0X7oGF447NriFpH+eZD011cH+hrGxtBcKdzf3OB2JcN3WhYfbWhrhb0tGBeR6nsPHM6gWyr0d3bwrmL3JuJm4L63ZywVwxPP8hsPxIuRw4lCvepQtyXjJx3x2Qg7i/+gI5w5OBS4hbA+618JDB8PnH28ZjvTOC2+2/1HjhgVsVD4maJ97aSHdRnpXnjcmaP71kjfiGby8JFn+HCEztJfRIbTSPdmhNctw1mgvLHsPgx0H44ELPVv3dH8o5r73xop7g3dk4ivof1kpF9K+u9wJuFUfg5nv33FZjfH9Ir+XNjK7tOBEAyHjx12a+jPFP0FHN6IL3r97Uh/IV2fw0rRvgzM3ik234ioqPGNYvvdge4Th38xEtuC/SthsPcfBqp7zfZj6L/Twb4Qv6tTbD8RQLb/QFjD3aceV2raZzIQ3H86UjwYNp9KHczmE+gvI+2/0qE2Bd1TYWp2n3iGVQpYKqB/ErCdpn2qGM4gVIrDC/Ht9WcSSDScCnPkS9nH7qVnPI3oTnN4AbvvDuhtwe5DSa+9/Zsj8x8b7n5d2BM9wvDvPtBvZriFor9QtM+F5eueig9u92HEzcH0OgEWcHNF+xSGc9CDZvchgKI7V+xfaA4vhR1obiLtM83mU2GTb//VKPdqLUnN/anm4btyzP2FsOvaQftEsX8Nw6kkGA8niu1H0J2L53jzDU33hFRhEVl/Kj69YSWIsb0UlUx3odl/ECFo1r8mfkVQdJeK/lKk5v2ZhECFQrF9o+lP5dwKS6roTzTrb6UJnY0EjGEUuw/M1P2JlmNpnyV1DsJkuVqxf6XpLqUzsX2iKTeSBNtdaHYvNaZXHF5o+jNF+wJ2H4oqYv2pQjmpEhlWhsNzWbcvtTB+M832jTBE2sH2jUkgQ2OcJN6Oc0t3btl8QzN/F9i9tBivuP+WpdxJKu/2TcG4NFz/a1BsDYdnlu7UgBJGbpzLfkej2Xwsfrlhabj5DZ1k+XK+zSg/r79h8IWmuzDMrj3336pQQWSwt9+rKPaK/QtD+8QwzhVRKe6/Lbr2cWE4PDO42mCcTCZuPirwjQalaS8MvpLAp82bmt1r+S5oLzUYzXBeEQstyoQEdHavSswI62+UzN6PrL9Zo6Ni/9SCEeZ/+2EFWjymflGwfV1R7AOHFxXaQ39ZYrrI3a/WzG48oTYcXtb4maE/KyjXI7HQ+NriG0N5tcMvK9xJJSFGWoNRtM9quqcVoTbYFCTWvVwkAJUnPj2hsoR5IUFM1nD4cEF11xMry3jeiPXjrAGl8MsSULiLGUob3FlDaAoUCn9SCeA9DERrCUv5PVZWmMx5TawKAZPOE5YNRHBPV5hNSziZS3DQoiYWBWHVoNAin21KAYBGi1TWB6kdyR2XSkl4UE6OVQrKUl5blShjhOG0Fgq5r1QIsFqI9zMEKAqUlteqwoqSok9eTK1QVSXbR02MMiGg8vqVYgoYeuzf1BqFANKf+U+n9aGYAoT+HMsP+f233//+9/+nP/cK/gKW//zv/Tffv/zuv06+w/6i/r37P/73f+nHBvxyS2GffPci/p1/+LdpfUFIBoBn1QaP5qpbMgTDb558zj/8w7/O6aLF6kAEtl1FCJpl01HogFaR/VDSFCOF8Xy0uOOP10/46uaUj5/f8Ha9wnvNrO75xtktP7i7BOCj0zt+cHeJUZG2F63VvOmJUbHezHhyvsXowLarOJ+1XG/nLJue8+bAH/zxKygCL1/c89WPpeajbgbafSVdlbuSb334nt1YcrNeUFcj+0MlN5AOzJsBrQOHrqJvCy4vtjxsZrx+cs8P/+QZ2IjqDOasZzHvGL2hPZRcnO0ojOfd9QlhlOqSIvkwjQ18+PSOH372lPlpS1OOrLcNLy/WfHFziu8si7MD+y+XxLlD3xeYlwfGTUV91tFtKpYXe3abhrOzHfe3S6r5wPDlHC56TOFxXcHy9MDm3RJ0BKfQy5HghCX2zhCvKrjsCYMRefG+xDSO2UzkD7tNg7YBvy3QMwc3FWEuPZQEuPj4nrt7qYKpT3q6h1pqTp4MExM9HJJmcG9Rq0HOxaCpL1q62wZ0xC5G1I8bxguHchp70RI/m+OeDKhNQVwI6PWLJCM1Mqi1lx3+XQOXPa+f3fOTHzxFzR3sCliM6KtKvJGDglORLYunMYCBaEXyWz/f024r6IxIfE8H9Ge1ePheDigdaRY9h6u5+C5twGzstG7lFKEOUAbUwVC/3NN/vpDP5aXj/OmG+x+eC+M98+itpdiI5zFWkpaHiqitxXQadyLsvPLyAR+LgG4Nix9pdh+JXNbuFcMHA82fVAynQcKJFiNcV9iXB+KfzBmfjejU5Vm/s/QXHrMXH6hKvll36ihubfJYJR9l8n1qJ0yhcsKGjqtIqIMEG1lwT0bMbYFfOfTBTH7DLGnLASZRQ5gFins9Fc5L2nEEJanS5Z3BLUSu3VwrDi8Ddqsmeee4EildsVN0l9I9V66FGdZv9lLto6D+StKpqzs9BdxEA91zR3VlKHaK3acj9tZKgNMiUj5IVQzJX2c69bXglewpHFdy/1V3iv0nI9W7AjcXqXmxSf5XDbP3Ejg0LiPNe0X7PCaGJAUaJd+rPYgkV/o4U5fkALmPrbsQxrPcHENxpCNSZKrHWpocGCRA0CWmPPdHuhn0l57mrcE1yTtowS0Dix8ZSXOuoLyH9nmcGHQ3kwCnYicgJ1fKuBlTmm5md4dVkurao2TX9FJpooIwtt2FsLlufrwHzCPmLadI65EkEU2SYS+/2wMcXkTKe5nQyMFGOUjJHqT/UgVJyz75AfSnx4Ad2x2lyrkvVCel13Ai7HlOnJaUWEmejkoqctqncq3KjXgmc/px+SDg2DciYy32TJ5SXwqj7qvkPbQCGLoLWP44TkFVto30JwKamysJENKjSHVnV0c5bE5Rzsxu9k4enmmRkZ8JC276YwBOf6JpbiRdur4LE2PZneuvdTDmfcsdmjl1Pfs/gYmRLLfCpI5zlVhReZ7yTJJxX4v8Nqcu52TxqCSRWju5Ps1tqvgo0kRBAra+yn5CYW7z30MpLHB3qphfedZvLPN3IUmEJWzJJEJqbLLcNf2vkhe6Vonxl2tX7sLkjxVPpppShsutTGjkKh85PsXupWb1mZ/kvkViMocTm7yiSfHQShJyuXZ0F5b6zuFrPSUY557KHBbkK+ndzLJf5SPdmaFaB4q9YzixU5JvkcKLJvmzAntI5zOl+ioXKR8GhvNSpLhaEoDHnDqc/J5HOa4kG5f3g0jUTwpMJ/sxLkRKG7U6+iMrw+TH9MJojquCYu/S9TRTz2X2uQ4npch1E7tvUpKuGr0E+gxHFlV3I6G0cm1SNQm52/YwgpUgI9V7Ym1RncNdNBRXO2JViHw2Jc5m+WxUCuX9JMOfpLI8ksOn1021IZlB1EqSeLU61pcokfXiU3LtrIZ+EN9mTn0F+Tkd1ySZtclb6cOR2YRjymxK9o3DyNdqQnyQ31N9SMyS3okB/TNA4Z+WEPt4nb/s4T1PX8dv//t/8VLY/+u//8WQwv7ceiz//1g6V3DbzwlRcdMuWJQ9D0ND7y2jN5zXe/7Xz/4aF6s93WjZ9yWllRty2XTs+xKrA005cvcwx5jA+cmeP3x4Sjdaqnrg3UY6PNxo2LqG/7t7SbcrmZ90/N7bF9TVyG5fc7o6cPcwpzOBvispSsfdes6rywcOe4lzPNzOiOeK9b6hPusYOsv1wwJsYDbv2W9q6rmAPILiR1cXaBPQOrJ9mGFKjzYB7wwPtwuW53u8VygVWe8aQtB8cXuKWcq3lw/yWEje0qJ03K/n+FFjS0/VjGgd6bqC6DTOKz6/OQWvOOwqWlWiTORqsyCMmnI+0B4qWI0UpSNWnhhkxrbflygb2F7L8bR9SRw0XV9Tv9rjRkPw0tW5uZnLINVGzImbPpO0jrgA6llHGDVqZ3EmoktPWTq2V4sprVWtBjCRsnZ0TSHgqAzgFXcPc0JnxSfX2eRh1SgFw30tYT4+GThsIO4LKAJmNTCOBlV7bD0yHkpMGVGD+FbHhxpjIvQikcUpCdexgYjIpGMV8G8bwiygouLL//MFahnEcztz0Bnxk2qRqcS9xTgIKdwlzJ2Arrmn21XY6xJ36mTd7ysBVF5RvCvxdeTQilRZPIMC1lQCf2HlUHtLeW3pnzncHy9RRSQ0cg7vvjwV365C0nId9K9Eum0e5KMhpO5JqfaQDjXxxILykoDbvpAvgtCk4+wFjMWcvHtVySB/W6KWgeJagI8+aIIVGXdIPlKVZLluJR7SaKBfBAHhcAxSCWryCIaTEb0uxF+oIipJsFVnJsAfdQr3SdLV8kHTX8g1zGFF40kgkEKDGqiu7dH7WEaGJWSPZ/b+9hkjAAAgAElEQVQPZt+ir6SPVHxuIi/2n88ko8HJsRQ7nVJUxSvpG6a04f4sYtYG02fpoYDnmMKFin0COKNIZ3MgTkT2SQVJmS3uJUSqutWSjnpUtrN/mXysRgCZadUUiqMRMGl2agJnMoAG6yVMKGomOXcOEQrJV+iatD8KYgEqeRIlvEVNIToh1alI/Ulk/pkRP++QwOxagUoJzFa24eaK+RdSxTQuJG3WV0w1MMNKukD9ubw3gKn6qdyIj7PcHANlxln2OyYvpJKE1gxEMjDzpfgw9SB1TbmeY1gen5fPf3kvEw3FnglkiAc4gcWUlLr4TI4DBbP3yVeZJZT3x2TaItXQlGumgbl2YPYyyM4hYcGIhFxFUpVGGm/OSSBQwK0uBOAGm6TYPklIk2zZHgQg1dfHOiWVngOk9OPs3xSw25+ITNzNFd4yTTyEFE7kZtLHOSzUlBDsKzkfOdXWVwKM2kuNPYjkfEr8TUnF1V6SX7NX1ZcK60T+PC6YpPG+Fh/pYz+vL5mCjcZFGqhHpjAe8VuSqrfiNPWfZdBjo4+hUEbkuKFUk4xZOwF7kCYcGkW1FjA6fyeyUu0h+qOP09XiaRV/nwCKyYM4SrCRHqQSJRRKwFV6rfJHkCqTAmkflEL7SNEHTn8QppomE+QYlJPnqRiJUVJjXSP1T+NSJKNuZibZ7DiXgJ8hPVau3eTdlM8nqfEp9/L94RpDMIrmZsA1dqqwMZ3clygIKdFWZNNBXlMZolZEK0qTodSYLhxrZ1yWM4MZJeAnWk0otQBNq7CDgMvs1xRPdQog8hISpCNQSK1QTneV6peUBjuIZNh0UhljDimMJ9UgYbX0WiqmsB2R46bk2BDE0xgiKqpjbUoClVkKa/YjoS6FlR2TZNR7KbnWJNZPi/QUOVY1Skqv3nWEWS2gMAfi5GAdkM/NDAazJzKDTqXEi9n2RzksMCW5PpbCTiE9KYlWKaZakfx4jMQQUmLsEfAR4yPQl2TDSsnnZDom9XhdHIHmFAb0uI4kLyHKNtJx/qVYfnk5vT9z+aUGlsNg+cPrZzin8T8SJsa0wm4UO8Vnq0DzlWFvoP1wpP6iQO2gexoZU1CG3St2s8j8WgYrbTdnnMmXX7mVQQQKml0aUBZwege+rjAzee/VAwzjjFkBtmvQTRpEWbjp5lQljEXDYg8qFuzeeGZfGU4eJLhhtYl0F6fMPURTUwVY3UXcvKR9FqluFPGUVF6eZrdLUL9bUjQwX6cZ15Q8t7iNSfYa8XXBOJ9TBfHHtE8js7tjP5/dwdLnL3uItqLZQVTFsd7AwarLX9xSYn54Ebn8f8RHcnguA8fyIdKfS9rhcFKxCLD8LLD+ZEntmaL23UwGJP2JlRj4VBY+rCrmvQw+yo08t7usUnBKzWIjA6XZu8Dug1kqKy9Z3URUkJlWCcywjEvxT7VPimnfxkUlvZKlwTePghnSG9w3pZS030MoKmoPsysJ3tk/rzFDZPHWc/+ppblOBev3ErceSkW5jriZSfUEmsOzGruP2FZzeGGEvejEu3R4VlBs45ToWD9IcfY4L4SxmxWc/mDk8BTapwXVbQJEuyj7+BDoV5rhxHLxewPbDwqqbaRf6XTvRrrzkvou0J1Cc2VZvBXf0P6ZSQNndSy1b0iDwIJqIyyFbSVVcHbt6U8VepTXFTsZTIVCwjOyVygaTbnz7J4bqk0K0rBqqhy4+/WC5Y/Ee9WfSAl7cxtE/oR4z2wXqe9Hbr9bMn8bGFbCvLqZVAWYMTLM9ZQoWm09D59UlGvxXOUuRgmtkdl9ifHPnxpy79cPnsMTTSgVpz8YGRea7kRjxuzZUiw/Gzk8tQLotLAcYyP3erUVH82wlC5KX6TZ5xSoIUmRIvtbfi4epXGmqe6dpC5GGWjtnhtmNyK/6pdKfGVOzo/pYwIrgXLt8JXGzUQimMNEolYMi2NgS/tEU2wjy8973NwwzuRLOIfXlNtAd2qY3YySGJl8aPbg6S4sxEh9J+xDfTuw/qikufOpX1TCNqJKdQn7MLEk2kWadx3bj0S2ZTvxeg0rPYG3YS6yKjNE2Y9zI92QiUkeFpJsGQzUd2PaHyhaj6s05384MqwsxdbRn9nUpVmBgup+ZPFWBsb9Sv43Q2D2bqC7LI/szMbhZhI8ohyUDwN6aJi/G9L9IimetvW4uRF2pBD5YrF1AkZKPdU0SNWKhJIMK5N8bYFQKsaZZvF5j+kcw1mFcnHqNuwuCpqbgfZJSfN+YDgtpvXvXpY0N/KzBD1FhqUkcTZfbmlfL9F9wPSB7ol03AQLs7c9blFgOk/7pKC+dYxLMx1z+SCF9eNCBnZ2L2mfpg+oELH7kfZZg68Us3e9eN5aCUnpnlbYvQwA7cEzLsUHX932hEpkkihFsZVthEJSLk3ncYtiSs/VCURVtz2+sRT3HYRA/3SO3UuaZPusorob6c8Lio0TL+EgHjx8xNcGuxdfXH/ZSGerVrjGUOwcysvncNQK23l05/GNxfRewEypKdejdFFW0v/p5sI8lfcDbm5FeXCzT8ySvHfcsqT67J7u43Ppkuz9VNHh66SuOLgEXD3jqpI+SBcY03UxncM3UmWhB894VmNaRygMdtvjZ4WAGqvRhwF8YHyyENA5ONwydZiue3Q34Jc1ZttRLWpUO6L3Lf5cnh+tnrZlth1hUaP3PX5RoRNw8osSu+nw8xJ7vYWykPvOGPSupT6ZSeKpUmA1xbKCGCmutoR5jZ8XFHcHiFH25X5PAVJnMUhwTJhJX6Rb1diHDhUCat/iL1eYux3m2Qlm8yipdHSEkxnmZkOsS7kGRqHXB5F0tr2AucISjUENI2ZXg1Hoh71IPTPrNowURfJHZhYwV4YoNT1H7WTdejGDGFHdIEDs0BETmMuATO1b+Vv2OqYk1thUwvL5IOvLvZJlASESux69nEM/HPfxcVgPHNm6EKbQnbhv0VVJ7Ae099APROemLss4jsefQxB5awaWVSnPTymukk7riUE8kaoqBQh+zfuYwN1PeTRzWM/EBCh9DNFROjGYCsbxuD1jplTYMIwiZ/3pJQHF4N0/Azqn5TEj+pdkUX+JgeUvtRT21a+exn/wv73mx8MTfnf3AZV2/NP1M76xumFpO/7J/St6b/nWyRXv2xUfzB74ndtXHPqSZd3zjZMbfrS54LLZUWrPw9BgdeAn92f0g+U3X33BH98/YVV3bLqam69O+OST95xVB97tV2y6ihgVdeHoneHlasOPb86Z1QOLauB6swDg5dmatw8rLpd7xqCJUZhEoyJ3uxlniwN32zll4ThpOt4/LLk82bFpa5wz1OVIiIpF3bNpa77z5D1f7k64WS9QKrKad1x/dcrick+MiqoYeXiY8+sffsXvffmCp+cbtl2F9xrvNUXhpWLEa0rrOXQls3rgpOn4/OqMJ+fS5blrKwrrabsC7wx1M/DxxR3vd0uMDjyd7/j9L5/z8bNbfvCD51y+WnPeHPjR9TnjoaRe9vyN1z/GR8U/+vwNQ1ewWLW0h4rV8sDd1UrkmPcLlquW3a4m7Apef3zNV7cnAASniV7x8sU9D/uGw31DfdITI5SlY7+rMdZP+3exOPDFHz2FpUPZVEkCzBY97b4UOW5v0UXgex98ye/83sfoxUjVjAyDZTbr2V4tOH+xZrOdobTQMN7JdTs723H3xSlo8RoCuJuGszf3FCawbSuGvpDju1liyoCxHqWkFgbAWuk37buSMGqaZcfhei6sae2xhSMEhR8NobOoTmMuemzh6dYVqgxwX3Ly0QPbPz4lXI7EXnx0gHxilVIlY4owve7lB3d89cNLSb9VCFvZSrKeXoxH6fF9TX0pkmCz1/jzkWo+4EaLbw2qNxKW1Br06UBROrrrRqSpz1rclzOKD/YEr3G9hZ2waHoxYgvPcFdjDhq/8hTLnvDlTGSAZYDaw6gpbi3+dUfYFKi5wxRBEpsBc9Gjf9QwngTsRYe7Euly83JH99lSGCArUlY9aMyzFv+uIV4MqLskgb7sCb2hfFcwXDooIuXbAjeL8KQn3lZwOlDPB/rPFuhB4U79dD7oDZSB4r0Ur4fXHcErqcIpAvq6FPZlTJUwo4RIhUr2iyKgekNxp8WDqsA+WNxCQqj0oI6Mr474uTDjAHojYVp+IfJg08l1H08C1bWR1FrFVH8TSqma8U0kNJ7yyuKWIicNrzq4rqaqGIUkHFfvLcOZp3lnOHzgKO9FZhwKYQVVzHLkzAQj7L2G8lYznMik3Xjm0a0EQtmDhEWl5p6vJwNvRB7XvXCs/qll+7Gnfm+IhUhf/TxIONNB0T/1VFcZHCmGk4hbBso7g6+E9ewvAvWNpruQROBxKcxKfS1Mbu4T9c1RwuqamJKWJSRKjZL2K2BRUnx9Had9dY0kDwtLF1ONiZrYLjNAfyqTR9EcWU2RESvcHIoNjCsJfOqeiFRYj3B4KQnGRJED60Fk17nOo3/iKe4N9a1MkGUJ7bhg6hXNKba+YeqpFNm2TDR0lyKhdXOpuqnuZd/dPIVIjUgNSwqLcgsBsMHKxF+umwmlPE/Oify9eoDuUoKeopbJpGF5rGkZl3LMvk4TiWs5ToD6WljQkEC1m6upyqbYxamCqH2maN7HqfdVJLeKcQGLz2Nad5KD7o4JtNEIeynVLCmYaGBKBg5GJK3tpZpqZEIhzK+vobk59ncenilm73PNR5IEpwlZ0yfGNKkETHf8PZSybyZVnJQ7OT/lNia5bbIENIr6PjOQ6fodZAIrs7wqHFN3QSZks6y82MvE2pjqYGS/IuNMUxwktGn/3FCmAKkhJQzL94gw+uNcS+KtVRQpHGhYpO+NMU16HCTMx80ly8EmNjEYYVK1SyxswgfdmaHaSD9nfefoT6XjMk925QlBkIm1mJJ8TRdwcyMhQ1pY2HFuhFGNEV8ZqdAZI0XyWIZCQ2IjfS2fG7lqRLmAr03qkhQprh4C40L2Jz9H3tuPGEDAV4biviMm1lVYQak0iSlVVqXOSl8da0x8bSXF1icGOUleozkmyyovIUPChCawpUEdeuKsEtY0SWijSR2Zj9Jos2w2s45Ra1Q/iOfSmgl4Z8Cck3NViF8HmY/WwaMqE/nCGUUW6/0xFdYYAdM5xMcYka6OTl6XU2AfJ8D+rECff87ylyUV9lf+vb94Kezv/INfDCnsLzWwrD76IL78L/4zjAmEL5skAQQ9Kvqnjvq9pXvuKG8MwxNH9a6YmItQSP+cFJTLQEPSFdUkdXNNpLnKlQUykMpfHno89mrpUb60hxMZKPRnMuCw6Ut5nMv6QimDllwiDXytBw3EE5S/tHV/7FTrz2XQkaPzTU9iyNRU4OxmKaUxyBf6cCr/j4skC6qhvpKBjHipRGria4nEn9IpFVP8e30jzy82cnzZd5QL4Zc/iWzfiGQrGjg8jyw+V3QXchz9uaSUFjumHqzsh8rVATm1MnfSubmkTo4L8YTZVs5/+9Jz+nuacaGmL8zsmSu2Ed+oSfo0DbZSHUGOtg9lkkml/c/R9pIYJ9up75InKvlychfa44j6HEVvhpjYi0f1A2kQIQySsLPj8lhuPqyONQA+yahy151tI4fncp3788jsS5E95S8fPZCK1eX8VPe5kkMYmOIQ6c6yDEj2U5gt8Q3VN3HqFuxPVUoIBUK6R62iuQkcnmlZdzonw0pNLOHsnQxsYvIyhUI8X74+XsvuUqF78VY1NzFJ2B6dTy8SRV8r6lthJvN7SWRPkf5UBk5uJo+V6yj3ciB5pOTvLhXVDyeKxVeeYaGPSZOkwVIXE+smLFt3KkxaTrR0lQwibR/pl0li5kU2qR3C4i5T7YE9Vj5kOdw4Pw5UMzPjKoUZ5cs+10DYPrOe6Xp3qeoivS+y/9L0GahIgmV3IqEa2kXqexkU+EJ8WXqUdY3pPNk+Tj7MLCfMA9WooV4HfKGmovicsgnCUKOgWodp4Lt/piXxdIy051o8jbk7juP6c8G99jJw1aO8ZkySu+Ig28kl8tXGM85lUJMHo/3SMLsa6U8txSEkYKCmYvjiEKZ1y+enDIbGmaZ+8NM9MSw01Ub6+VTInraUmhqF9S4OwnCXO2GMM1CSGp6A6QL9mT3WLySFg/gcj94xUUrEaR2mj9LnlxjKzGpn+SWAaYPUPCRJYrGXCggUlA+O/ryYBvOTJ1crqruR9omwE2aI2IOA+SwXFHY+effGiN17hhOL6QK+1tjD0SOWmWY30yndNia2KqXgRqmYkO9UYeaCVYRS9kOeq6bkVekBlPNi955xmVLBh5i+fxTV/Zj6Ew1mCPSnFttKCqqbCUOjQkyDc0ALOLGdP6aJ5hqPeDwv0aqp69BXBnNw+JmdvG9RqQkguHkSaoU4MYzDaSms7eG4nXFl02dzViLI/WX2I+NpRXnf0z5rqO6H6ftDH0b8qhSGT4HuhMVxjTkCFR9wi0Kkqp2XlNIQcYtySlQlSZtDZShvDgwXMwEjiQ2NOsktoySqBqtRPkgFB1K/ESqL7t3EFupR6jZMOxLqArMWUOQXJWYvCam+ElloruDQLkzHNo13ulESTTl+b+ICuh/xqxrdOXCBsCjlZxCwNBzBlLCXB/5f9t4kxtI1PRN6vumfzhhzREbkzcx76w7lKpdx27RlusUCxAaJwY1gxQIJiQ2bBu/Y9hYhEGwwk2DDCqmRegGyBAJkuW23y5bpmm7dMTMjMiIzhjP+4zeweL7/P3ndVWW3uqz2bXw2OZzpn8453/M+kx9nEC23VW0a+CLhNdB0QwqrqDuCI6MAFxCMglyX8JOCXsNA64GIyaQhTwaZ6VeASmcRspRJrT5wH4RASKjUgIuprFoB1hF0GQ1hHcGYEBBNO9R7hDTZJaD21R+JIeP5p4FUvPVyWTiHkJidrNX5ge38ynPeXp8bTU9ln8KapQibLcGbcxBpOgAukXJKE6yFiJ7I0Has/+hf01pAKqBrf7KvMU3xlZCcPoXW6H/08c6RpdQaoaq+ss8AIrAkC/qVbQB2r9WzpD/rFpnLr73H8ugvCFj+1385gOXXWgorVMDB3gZvbmbA2ENkDqFSyA4riMqgfocXof7mFu6LCU5//QrPf3wyhKzACdhD1nzguEFbaXR7EuZBopt5BBPQ7SNO3OmVsmNPtmajETLHgJxKoHy/AzqB5kAiJB5dy4Q90Ymh3qE9tOgm/aRMDAEXdr+DWmqGrZiAdj/QGxWDXkQAPVt5QHvgITrWapSnDEPwpvdvUd9fX7RQSw23Z4FGwiw4XTRLgfIsDCXmchoXniHAa7Ia3gD1eQdhJZI7ieaAP+D1Mafa0gL1I4vkVqM7b1GfKMhWQnZMiZQOqGJ6ZNBAe2SRPzdo9jgdrw/JINRnjsciBpO4lAv4/Jol7904hli0GPrQ9JZJfvWxQ3Gp0BwEmKVAc+hZU6EJmpIlF0ftjD84yUpg+SHfvxt7Fs43Au2BQ/FSsRz+uGd2KO3tgUNz4BGSgOyK8rluygCWbhIYdKNZN1Kd0fDWh7Ns3nXIXyruy7T37Ai0U/YjCsfAF70FqhNuk1kBTeD+pw8MXnEZhxj1AQcEZi3ItlT0DqpGYPWeR/FKsvB+xfdn6TuDbWQjKOuOx5XMERcEzR4BUTeNrMQowBuJ+qgHs5Q9D96aWmDxIeWpZkvmoTl06K4ViuuA1btAfsPzHCT9X9URF5vJgttlM7ImdhT7N1smV+oKaM84vMluJQvbCwIoDmgoL+1GXNB0E4H6KGD6KUEl31MN/qw+DIWMCO+nN4fnO0hgdAmmJ7b0uNX7XNz7BKj3A9KFGPx41bFAdgeUj8iAJQtJD5wVaGf8TKkYxJI+cGHpMqDZj/7BB4FuiljNwmOS3gPbczIy0pK1ko1AfrOTGm8f0bPUxk5IO1JD9QeAIYik3hfI3wRUB5IsVMpj3g+KggAgAWn5Oeo9qvBkrYpXZNdcKuCVxPZCYPSSQxbVAttHXPC6/vwVHOSYTc8CSdZslGKogPBGkrGL28BAEBkZKcEi+xiikt2RAZJOY/VEYnzJ1ydjxPCU/A1TRsmksE/RZoJ+QitR70uMr1xkBgU2Z4pdjyOCH13R8vD2oEdaiXqPIKkH2t1YIVnyuLrjvhuQz0uW/H4bwHuy6wtkn6jgZ7sJWD6jPNYbXl8EosDoGlhfKBQ3PAZeC1RHEqoKcMagPJHI3/QDPg6aXAqsL1Ik674TMiC753edVzEZNjJT2b3H6h2NZK3i4CWGRu3JYShnc1aktHGw0de3eAO4AzF4YXUdYEqBLudnozpkN20vwzYVq0ZcIngucoHxtaBUf0IJe7rysKmEO0k55JoKJFFSX88lslyi3qO0ePrcojrVkaniazZBxWFPGJi7ZibjkIXDISWAbqzYsykJcptZBLeO3kZT8ru3HfN4mFKj75UUVqA6VgMo79NR6QsERNCwqcDoRsKlEs1+inYi4ZMUydKy2/Ew2Q1yALg04QCr43ePzaMcXgnUxxrZnYTLUpi1RX1gkC5V9CbLYRiAUPC4n2bQpYfedLAjDZdLyIZrBdkRbHcjGStnDAcRD4CbpagODbK7DnasYFYKLpcQYw1dOjRzA6MlXMb9AiKYlgLJskM7M5T+2sDKD4DS7shE2oKg2SwFumkCMTIcFhkJMTKQrYdLmVYbhEBIJNqJgQgBzV6K7LpEN0/hc01mzwYoLRESJh7rrY5JrDs20I345df7KfW2Q4AZHicr+xUWTlhKRH1hIFKFYBRBdqwhCUrCpwp6SWAWphlZwzZWh/Q9lalGSPRbvkemwoYIpIYuy2430JBlzWRXIQiipYQsawJTgGmsbRfv7xDyAujsADqD0TtmMQQ+vgd5+/OBTQwhAEWkyyNAFVpRjipFlKZK+iKtY+qrtUDGHJGvALsevEqz66GMjLjQmu/VdoPMF85BKMpkRZYS4DsHoRTBrZIDAO7TX/mF6QH1ljw2/v2nE17qzwaff3X7p377WjOW2Xvn4Vv/5b+HRFu8vp/COwFfa6hRhySxqBYZ4AXgBY4eP+D2s30+UQLBeEAxOVVMW/haQ24VfNYDNwl32kDcJwP7ERTgx+yW9GkYQkwAgoRuTNDUHjgIK5DcE6i5PMAnHrKW8Dk7G2UMI+nLuBHZiW7uoNeKwEGA5fAeQ8qjqgXsyMOs2aHIxMW4gM65oWbJBUDPQHZz7hOi/Axg2IhsuHgItDIx3TRK9kQAugOL9BWDVlQV92tsoS9TBn1UDGUozzx0xW2rnzQw1wmn3vE5QQf2VcpdgEHQTL20RQwFqQRcSmkdjf89OPRRUsdtM0vJ8BsQ6Pf3JwumLLo83tf1i2bAFQHprWSJPRgO41PK2lTsaJQdgalPgGRBeZ2MnXyqiRIpEwY5o2wIALNbsqAAgXc35QK0l9x1U4/sjUR1HM9ZlMm5jEEftiAzrZodgEZkmtp9x67NWAbP47KTXNUHu5RNsqp8us352n3SqbAEcQj0DQMEd90sIL0XQxqobMm2J0uBZj9gdCnQzOOiLAeveQcUl7sAEmAnESMI5vv1oKedE9DLjvUf2Ru5KymP7HGf4DkwQpEZ9QkGls0nZM05CMEQ4iMcwa2uuE+6pi/abLgQN1uqAIQjOz2EoDT0yAaBgcEdpINxyJEsAsozglkRf8dtEYNgsq8mhwbF9+/L5Ps0wv5c6JILb1OGr4SI9AwWFRIxLCYGgqiGMsBkSeDmjYhdkzGpUmFgNXk//262YWA/bS6Gx/TpmNJGhjzj4CK9J6DkdkX2OTL/LiOw6dk2l311X1ysdqDEcpcaanMOXHqmuE++7AOa+qAbl/LY9oE53pDNf5tJ5nGPKoWEj+m9yl1BYOqMiMwrBgZYNWE3ZIjHKllxe3pfZ7LxKGPdhM14rFQbdhLImJA5pG8m/LwM127cTgQCl6AJ1nqGtAclXkfVSGRe++3rJY5ex23udu9B5jRKHOPnYPTaojzU6EZMVqWfmI9JNj3DCzRzieK13wUISUEGVxF89NswBC7Jns1FZO93YVoiMMlT+F3voksIYPtQmyDF4BlSrR9YZrP10TvL1022Pn7meqVFPL6ZGFjdZq6QLh3aMdlss2FQDHsbY0JoR1azmygmqOrorW4JYM2WPlfVENTINnYlSlaCqCYM12TPTvbhNEHuGNhB7qujpDN2cHZjyVTWQg4gDKCUsldzkOWM6amHJiapxmFEyn5Il8no84+qgYAY6rM75/y8CJiNRTuNIM/FcxIAvSWoVV1kcyI767IIxiPLa0earK4j6JOtAxSHSOx39AhGDn/KliE6svWQjYXLNHyqIBumlkIAPlX0iGq584zFhb9PFZlTozhA7nzsnPRMWu0f1z/Xk82Epz+074zsfX6Ue/ZeRD8wmt4oCOcBKSA3Lew8g6q6QU7qMz08v5eZAvG7VwgCxNoOjxUNjwtciINvyk5DqiAaRxAYGdSBGY0M5cCWdpZ1H1UDP8khGktQZTRBXazmQO/T7D2RUhI0xrAd+Jjm2oNJG32MbYcwyiHqdvfcPtEV2L3+W+djYEJt3Na2g9AKwTq87Zvs/y60ZqJrDzh79vLt9+sBaP88pRCqt/yy/bZE3+U/MWMJ4J+JVNijx+Gjv/UXwFj+1l8xlv/Et+AE7u7H7HLcK7F5KJBdGsjWoHyvhbk1TKibOCz+30OkNcGD8GQ4uJgWwDZDthBkjmqJk98F7n6R4Cd9vSuVTx4kbG0gHYFDu8egCYgIMOKPUM9w2lFAcSlRHQPJg2YiZamGFMrm3QbZJym6KaP0m70AvVHQpUB9ZpFdaRSv6BkpXilszwmEkqXC9v0We39gIBx//NJ7oD6kv2r8kuyFS8lwmbUaYtW7SWQGToHsjgDBh4CQAnojkTywK80sJPTGIBhg9EJGSS4nh6oWmH4KLL4ZPUEHrDpoJwH5J0wAzW8DNhdASDwmPzID0Ns88ShekrGQLZC0EXScBUy+5JfR9jygeBVQH+ziBTYAACAASURBVAgcfcL0wTJIZLf0oDRzLra3F8De98l2ybdkq31KomxZ/l0dSdSHTI4UPZCWZFXS5W5B7lIRQS7Q7AOjL8EOv9eMP89eR8B1RcA1/RSojhn2AwFURzze6YNAfRCw//0Q5YYBqiEDN37hsX4isfcjj/JYIivJRCZLPnb8MmD5PsHs9McK9QEw/dJj9UyieBWweUymsLjxPH8jDPJQ1XCRlywpW85fB7QzATvm/1UnAb7mMShuAtRzHmPtKSWuDzkoKF6T2fEG2P+hw/odLlBcR0YMIcBsOT32MU4+WQXoa0pmx1cOi/cVhAVmnxDEeAOMXgFBhYEpqQ4lytPIOM5FDGMiUJ987rB4T0HXfO0ueq72f2ixfqyGdMr1Y4HJC34Ow57A6BWPizM735HXAukiwGwJJnsWXMWy+s05weP4kgFK6YLfL15Tkm22lGpWRwJ7P2bojy0IgLIHgqt2Fj+Ht1ww20ywpL4LsCkZsd5rxCRDvsf42mLxHhnJwz+x2DxSMNvAfR4LJAsWxNf7Mhapc2GqqzAs7E3pmfwYgPTBop0qNDE0p/cbjS8dmjn9T8Ubi+0pqwe6Cbd98sLBbHdyWrPl+Rlf8fzrNd+n2hcobj3yNy3Wj1MUbxyqfV4f7YTgK7/3qGcS2dKj2pOsgOgltzFopZ73YJOMF2WIHttTjfFVNyRUmi2Zr2TFhez4mlJnUxKMjF47NFOJ7khgdENglb10cIagphurwW6w+AbBlohyV7N2qI40jv5oi25s0Oyb4belPJSYf9bh/iOD0TXPARMfJXRkyboRmbse9Ob3fhj69DLVel+h3iMbr9qAyfMW20eGAN8BMqa8SuvRTiQmL1sgAKsnKcavOkppxxLjqwbrizQGL1ksRwZeC4xecQHtcjVUUQgHnPzOEsuPJkiXDragBLY65Dmffxz3d08ju7PYPjJIFwza8amE1zHR0nMwkazDAIJ0zUFgdm9hVh2agxQhFdCx9sHmEcQ1AbricR9dd/TnRXAKT2CW3VkGFd0xlCsoAasFipsOLpFIl26QSZuNG4CLWXboJgYuk8hv2jjwUDDLFosPC0xetNDrFs1RRq/dio8he+VhNhbdWA+DR4DyYyoMJIQNyK82WH40Q3bXoT4wMKUfAoSy1xXSO4aVBZkgKIYRycaim5Ex8olEsurgFQFceh8HMAJIl9ye8lGO0fMtulkKs2pRH2XILzcIWsJOUgLT2g3SVdlYmHWfLKoHRq4+TJC9YaBNOzfIbiru1FuLe5srJHcV3DhhsNBYQ5cdROVgD3Mkyw7dNOE+9uDyLUDXzvm7nr5YwM2LKO81SO5KsnVaQm3jl1oEfQAgNy1Epgl0c4NukiC9XAKS4UR2mjEIaFZAxHAivW4QOklwW0eAWDXscVxW2H5jD8XnK4YKaQkkCuphyzCf5C0GessprmzaQb4pGsvORs19hBDwRlFCnCioZQXRtPDjIrKHCWWxRkAsy50s1nvIGPDTH2dhHbCtyeLFdNZgNOTdCmE2JkhcbRHGBURZk2nsmb884983JX2LWkG0niDNMLAI1iLUDQN3nIdoOoLENAGabne+e7CnFEJdD8E+UNzu4KP3U0mEtt0tqIUgeIzHL9gaQwJrYvhZEWKoHgkuBgUBABzTXXumNEl24K8HlQAgxU9nI/80qPzTgT1vBwf91e0v9e1rzVimjx+H9/7d/xjVmYef2njhA+mkgf9sjG5K9i+oAFfErsHcw0waVkdMW7hlAuiAyQ8Myl+poD7L4DJg8qXA4p9roe8M7MxBrdk1V59amAcFO/UDywgJ5FdMzGwOHfRGwo7j/QBCyn5B2VGSVz52UBsJe9RB3xnorUBzxOd5BUDGH+839Ca5nHLBbkI/Tc9a9tH6/cRfn5cQPxyTIdWUdbZHDmahuJkd0F60EKUGxh1CrSCshKxYT6G3ZEBVzfj/bhK9cWkYpJOj5wSZvXfLp2HwFtoiYO8HAne/7JDcszohvwm4/2XGdEN76DuD7A0XYotvsUMwMAgVuhLs+rsjMO+74KpjgsL8zmP5nhwkh92IFR2qJjOmK4IGrxlukd4qspAx/KKXcuqScsrRVcDmiRiCN7I3BNoujUELGffRbBnCIUIvY6RfsJt55NdkJvvXb2fsqdueUbI6+wS4/06AWbOTUG+j9zYybYPct+jlmwHZnRiAxeo9snwuDRi/IACTkcHUJVAfRZltPCblKcG+yzjskJYgs50F7P/DWHL+DmWc6R2weeqRPEjWBhR83/qAgMYWlG66LGD+Q6A6jOmqAoMkUlVMzq0P5NDt1zMduozezj3WPrQzsn224POTFUFWsx/Z2G1kbgsyjuUp90FXO59lz7QBEUTH4+RSYPKCQ4Se3TXbHYuoa4K19TsRpAQCgHZKYNwzE80BOx6FJVPT10psz+gThiDo6uWn1WHvRcKwqOdnVsBEMGQLMbC4ug7QJWW1fQekanfhItKStXd5lChPCG7bGYG3dGGQDA7l04LP7722QVHmGSSGEJT0IX4ZiT5ghV7aek7JYH7rB2/u+Nphc6ag2jD0F1bH3J90xdel742Av++xbOZiYJ6zO26rzShlHN04lMeS+xjPh1e7lOt0EXbSWIEoX+VnQ7WIfkUCX9WEwVvaTngtDJJOQRZsfOmwviBIR/zuG6Rx8TfPK7J83YhBJkHuJJCmDGgmEunaY3OmkD1EFs5gCHxRXc+0IQal+CHR12Y87tlDfN3IQqcrekTbqcDo2mF9rlC84bHvtzVIMqk2lwNDaHMmBjdTxdCVkve3Y75Wdu9ZhfFWqKJqA+q5QrL1MBuPdqrQFQL5LRknn1AqqtqA7LZDO48pvBuHdqq+4s8OUgyBPsnawSe999EPycOD37iQsW6jQ4hMYl+xUe9x+5OFhR1F2Wkboi+dDJnNOCj0KgL0il5Ms3Go99TABKcLi/XjBOmKrGh+a1lpET3pNnY7mg0Tfs2Gnl4IwKws02sjG6xqj+y2RnWSI1nTB+pSgoN2pqFaD7N2g89QdR7OkBXqqy7MqkN9nCJ70w6ptEGTtZQd6zWogPFk+VJFgBdZTr3p4CK7pssOtjBQlUU7TyLryO8kW+xeBwFQlYXLdfwNIKtkczVUZwQhIF1ktyuLbmx4PeZkErPrckibNQ8VfG7gDdlGV1Cemryp0O1n9K7mmttjPUJCgOZybndys4UfJ2Q2K8vux8ZBOD/0QMqKQTt2nEA1DLORHX2dQQB608LOUqjSxvoiRV9nBOlBCMimQ0g0XG6g1g2Ec0yl3bYQ2wp+NoIrEqgyAicX4CYp9LKCT01kRj19liFA1JZMpNFk9mIyrawtQbL3wxBCdBa+SCE3FUKRAi5Q2polTIRtOwJOrQZGsfesvs309QAYHcHuV9jGeBueBwDLDcQoB+qGMlbvCfgiUxmqGiLP+H5lRUBpbeyRdMD+nCm32y1EnpOpBCiXBXaJsm/dQtMS6Aa/822uN3E7BV8bZDfhenAZGdkofR0ksd1PYRn/McBiGNhX97Mf+DNufxkYy9HR4/DRb/z8Gcvv/jc/m7EUQjwG8D8BOAFXD78VQvgvft7b8bVmLJOiQ/1LJfKswzcObnFTjmGdQmMV3PsrnI5LvLqbwb/JMH20xup2hPOLe2ybBNlsCwB40AU7Eu0EWBlMf+ke1kvMfqnC5nYOMW5xMi1RtgabRUFf5khC7TfwMXFUaQfxyKEtE6QpEzKReCSTFmnaoWkMxo+WuL+cI/3mBrJKYI0BvIB+tkH3fIT8bAPvJboNuyNDrVBJQBw0SFILrR18baC0R71KkU13qVhtzdPYPmQovr1Ec1sg3avhPxlzAf3uBl2rkY0aJAD0vkfnFFxcEHSthm0lvPEYjWtsP5/BhgB5WsNk9Ftaq+C2Butf7pB8maJ9VgMrg5A76FuDw++8xvXzfdz9WoBMHey0Q7dIuCB+rWHfLylP9gLh/RLL5xOEzKE95OIbIsDOBYqjLcqigBp3KGsNOAGROODDCjc3Y4i8pRT3wxL+KofbbyGXBj7zsPsEDHrcISwTNPsemHVQiYO7zOHmFvpOozqNgReJQvNOA5NZ2OsczREBsJk1aJYpIAPMnWZK4qMavtJIXmvU7zcIpYbILTaFhpq1wCV1rG7isNAKyQNgn9a4G6cIOqA5cFAHDerbFPkrhXaPX5D1qUfxXKHZY58jjhqUJsPqQwe9pD+rurCQtcT6mQAEWev62EJVEm5mkX+RoD7mtofcwe4JJLca6w8opW2nAW6vQ32QoJ0HBBPQHDqyg2OH5riB+H6ObhLQ7APtowbNTYJgPNzIY/S5xuYx0Ox5SoejRNtnHuZBQXiJ9bsefmKRf56gfuQw/lShPohS7RBgx0B70qFq1DBUUQ8a8x8C9fsNzJcptk88zFJC1QKbf6GE+UGB9XtMYoXg8RGOQVHVOZNW/WuJ8rFloqmnz7ebMSEPATBLeqDXxx7THypsn1lsRcD0Bwb1GPTnbjhYaY4tRCsHGW9+Q3CwftezX/RTMyRpSkuAVB3zeGav6WuUHUFkc0jw7hIOaoobgjQW2MfFuga2zxz2/4hetHYGmC0XuuU5B2E+5aIVguqKIAWq0wCbc1gRdEB7aKEfNLI7Mup9sFMP8tppQDunhzdIJo5mtwLNTA4BY8t3JeoTh+RBwY4VylOP6acS28cOsuM1ZAuC0KCpMEjvuRC0Y/pH0yWHEu00BmWlYvB41nOJzTtxYJNSgr195jD7nkKzR6C7fhKQ3wDNYUC4EqhO6IXP32AopN9ecPBCMCtQ7wfIKPPfvAMUV3wdlyqUZxz8+ITA2qVUDQBM8qSqQ2L1LjC6VAia8uf6gP2e1XFA9oZSy805j5VqAsrDGFamZZRSkjnenml2RRp6onm9SqRrJms2MwJn1XJIEwQ/I9JKlGcCxTXVGe0sYPJcoRszACvM42BCK4L5CNwggPKsT5qVrN9ZBnQF0E0F0jsOJFwqGVZVU0p8+x0NvWXyaR9OVR+awTupGnqOg2Bw1OqRRjtlqJnsANVKPHxAgGc2lPdXR1QF9Omizb4AhEF+57A9VUPiqc0IZgtNf2Z+T5a5OpRIl5Tobi4UzJrXsK4CzJahQc1E07soWbfkEgObcd+dEWQWNwG6JNtuyhD7OTW8EmhHcpC5y45ezHqPQ0wRJJbPJsjvPLpxinpPYPqcUzTVEsBVhwRk2zOFyUuL9YXG/JMO7UShmUmEMx09sSnSBRlemwmkLmB7kkL4gGTjUR8YqNYgXVjYnJLHZqqQ3UsGllUBdkRgb7TA5lxT/dAE1PsKzgikaw4x2qlifsOMbLQRGAAzk2M1pAPGL2o0hwbaSHRjhfymQXdEH6Y8yFHvaWR3Hbp9/o7ZXNHHmQno0qM9yAApsHmSwxmB8WVLhWOhB9BbnhgENY7SagEt6AdHoVixs2+gawV/lCK9a9EcJsheNyhPE4xe1qgPE6T3Hew4QVfEHmUt0U0UvErZkTlOoCqL6mKEZNEHSGXsw1QCdpzALDV8ZhC0QJfm6D247cSQaR2RvTSbCFYbCyiBkBh0hwWBcAhwmYbIDaAY2ORTbpOsO7jcAP3nv7Fws1GUDzu4vYKSWe/hRykgBOSmpsTX6Nidqcl4GjWwqLAOvkhjCBF/g0KqIMuWya8A/5Ts0BzqRNoOyBLeHz2WIspqw2wMsa0A16fISojZFBACIqaMo1covC1vBYCmpV8yMaxI6QFvBJhCRVmttQSfUrBOJLLDPWklEnqFesja91YON/8WeP4zQKaQYgcuv+63fzq7YQH8Zgjhu0KICYA/FEL8dgjh+z/PN/laM5bzj47D3/m738Zvv/kFfP6wj0nWoOo06tbgaLLF5e0cAcD+bIuqNZjlNR62OeqKF7o2DsY4OCehYyVEVaZwtyny8w2kDNguc0AEJHkHKQOkDDiZrvHla/o1fcdFXz4i0Ks27FXTxsG2ClgkkEc13CrBB+9f4ccvTmDyDl1lABkgtYdvFYIXkImDbxXQSuyfL3B/OwFaCZE7hK2G2avhHWs1ylUGbDUwYq0GXqfwRfRSTixCrTA5XaMsU1Z2NArCeIiFAQ4a+JqmH6EDQidZZeAERK0g91pI5RCej2CnZFd9Fj2lex32DtZYrkYoRvVQzfHw+R70MSU43SKDaCQw63B8vMSb+wl8paEfNMTjEv4yh35ni+aex1ZuFXzBiaBeKfjzGr7UkIWFuszgEi7ek5MS9sUI/qCDeDAI+y3M8xTteUdvivGAF8g+TVEfO4TMQ64VhBNwUwu5VQhJ3I8k4NF7b3Dzxyewxx2EDMBWQ7QCPvWADpClgh85QHuItQYUcPj0Hos/OYQtIps79kwdPu2gcge3NhC5hU4c/BU7/VzhoVcqemUDxH4D3ymgjmzXtIPfGIiGckZ1UsG+zjH5QmLzmME+ds/y/OQWQgWIVxn8SYPk8wzNieV5DwQfqt75fMPYsvLk8wzigw3aGwZB6LUcvumFBbrHDbA29Nh2fD99p8nqHTiYeQP/smBS6h4BWHqnUD9uUcwr1M8nyG8kth/SY9udtkAroTaUTnczD1VK2BnPQ/ZGojn0EGc19A8L2BEBc18Lkd5LNAceshVwGY1gIsRAokOL9JWGT4Bu36L43MAboPuohPo8h94Kvl7FP21BP2/7tIa8TbiPz7ZoHzKMvtCoTrgwyl4TmBL4SDQfVQhWIv0shbRAdeqQv1IEspWEGztkrygRqs87iFYOvuv0nqXkwoOS+SjB91kcIOgAfaejCoILifyVQnPg4dOA/JUaPKLdKFZuzHkNJLcK0u6Ydl3y3LczstU+BZp9/nibtQQ8oi81hgjFEK1kwXCn3rM8eiHRjRlYlb2RKJ9YzP9EY3tB363LqJ4QLoJcFXa1IZrHILnhPnnDBXz5mMMNl/A1ei8rgIHF7sYMSVINg79GlwLrZwSFwpNFZoUMkN4SPLNSJGD0goFY9anD+AuF+jAgv2Yva/pAkNgnQCMy0yJ6UmXDAcD8+wLVSTyGMTir9952Y35mzZLVKEHyfh8rMwafaAyUUU1Mtb4mO9rsAfUxPerFjYhAiHaE+gjIbwKqE4HRi4DNE8r4kzW9vek90Pchm00/mODnxBa8nsYvMDBzfXq5tMDoymP1VCJZ73ys3Zj7lL8maGv2gfSO6eH05kYWeCoGf7NwZJ91DdR79LEDQHEd4FJaSgCy6t7QgmHHAfMfMYhJuPietx7NTAwVHNUJmfBmHoH0F2TX4YHJpUO9J9lFC2Bzzse6VCBbeHTRN7w9E5h9RiZZBKoIqn3aLPI3XKD2fto+5VxFtryeM2So99wmaw44XcKQsD4ZW5f8jHUjpnvXhwKjKz9I2ssTifzWD77TIGO4VBl7dWOYkin5GetyhjrZjBLv4tajHXO/NqcKxRtKfrfHHCK0E4HJJaXdQfY9yEzcrecS+Z1HdShRvKZE3CUEnX3oU7IiwN2eSIyuyewmGz94ZHXpUR5p5PcWzWzHUgtPT2x+x/7cZiqRrtinCwDlqWHqcM3u5GzhhhTlnsnut0G6AFWHIVXXa0EZ9m2H+tBgdNmgOkkGVl12AemCEmRdebLLOftXkwU7dnVFr67ZWLRzg2TVQVYW9XE+eCh1FRk5R/sGJeMEwgixD7Vit6nLGc6X3TRAZKtVZXdoCICsLBng6Om0hWEPK8Bk3YpyVAZ6MV3XJwp6VcNnZvCAukkKvapjIJClL7Pp4LMEsmwQMkOfZttRZRHZzFCkBKGRxRR1O6TzcgNjCq0Pu3qQEIY+TLTdkGbbs6qibneS3B5Uvh3goySBa//6wCC5DU0LkRhKXpuGUlUlKYcNfpDKDixmL8n9R4DlTwaTP0syG7r2J9/357j9pWEs/82/AMbyv/3H81gKIf5XAP9VCOG3f57b8bUGlum75+GD//zfx+p+BPWgAS8QdCAwEIBZcFHbzj38yEHfa/qI+pCd2AsGRNIsLpi84UTdpWC3VvTu2VH88UmZfKjqGKBhuVjpxlzYNQeBgSx1TEZMAZbehyHNkWmEfL++J6zvv9LRdye7fsLPH2c7Yn1IdRK9je1ukdbf+mAMFYNYVL1Lf0yWXKSqjtuqy/gFEcMbzJaPa+YByVrs6iH6ihbFx9kxX9frMKRtFq+YLNkH2tgYTtPsB3QTj9FLFdPyKG+D5J/tlIErsoupp5udfLFPjO0rPPo6h+H++P/diF4syna5WOvGYuhF6wM8VB/ssuVirZuGoV6mD0qQHSWc9YGIibtRihflr+1elJg2GCS2suPjpeNxFT4Gl1RcvPoon+vPlXBAfUBvpjfcZgamxAV6lGPqmiyRbHltDPJjvfNR9t10MsoFAcSFFh8XZAyQ2bC3bvzyrcTCZBf0sj2nt1B2IS6wxVdSeb3hddlX3fTH0+U7GasuWZWSrKO0VzEsqA+C8QmGPrhkxUVlN+YitQ/l6ZkNXWGQOfZhKX2Ijh0B2W0YQpP6c+MTXrPpw64vzaUMc0kiA2LWDGapjrmIzd/EEB9NnynlyFwYNgeUPhbXlF62EzEs8HvPsK7ISJZnlGT33weqiXUysYaD/qrdOe2l2C6NgCCe5/5G5ovnxxYCzZwSbpdy/6TD0InXTz57SarN8ZUFkVe7epMQQUp/3Lox4vVFqXmXC3RjgeINgUD2ENCO4nZnbwFDsbsOkvVOkhrE7nuEnwsJvWU4ULIOsZqD03FTBpRHBI+64ndxMyfz2YcDBcm6mD7MJl0GlCcS2Z3/ymOa+Ve3FcCwzUA8vh3lqvRS8/tddkze7UOC2glZtCFMygPtiMevlzSna49mKodz3B/rIZTKCGQPjsmrGZk51YJe2T1Jb3kMtUlXfpCJVnsxnXjl0E76JPIwpML24UfthJJ04eh7JagU0BVlqZTs70JuuoL7q2t6ylTLupluREDQRcltsmVHYXmsmPQro180ApBuTI8usOsWDFIMAAYSaCcq+ggpp+2rd8zaMT209fCG7GG6pAzZ5gL5nRuCjMzWwSW7gKFmrpCs3CApZshXQL2vkS74A860WLLSzVyhuOmGAJ0+gKZf9MvOw8f0U2FDrH2J4CNwH3TlB5ls/51qttx2vSVYk61HeWowuuoGWXrQYggS4nd9gCmjJ02IKP31cLmCN2Kw7/QApwdDNqNkV7gAXTvYfFedIjoPKAFnGGjkUgW9JQCUjqFGwnrYkYbZdPCafsVeNi9rB1doyMZBtg7dNIGu3FAnozrP4BqADNzI8Duk8xCdhyo7dHsZw336EKD4WDtOmMAaa0W8lgzH6SWwnsyZz1mF0lei9B7Svi6Ff9dMobXcX58oyMYOLGNINFNbY+0KU1vNwLiJlvaoYBSE9wznyc0QAhS0hKgtmUdFuaratqyKUQxAQheB3FuhQdzAeP1U9LYO4To9oPMMsoFWO6mr84P3cagZ6R/n2S8pmg4heyuoBxiAJCni+B4+7PyV/e3t8B4h+HznBrlsqGpKXbUeJLJCfXUB+RPxQA8ErSV4jPUlsHYHSt+SuQ7eyrhvbwftCPGWx/Lt0J7+3/LtH66fjk3+WQCW3/w3/qOf++v+4X/3m18CuH3rv34rhPBbP+mxQoinAP5vAN8OIax+ntvxtZbCGu1wON5i+8lsYMDyS436/RbwAupa49/6jf8H//M//FXkP8jx6//an+Dldo6PPzuDyxTya4npv3iD65f7+Hd+9Q/wdz/+Dppa4+zvGfzN/+T38N37x3j+9y/g0oAPfuU5Huoci02BD4/f4HvPz9AGASwNivMN/u33/gh/8PAEzxdznBcVnn98AtcIzL8vcPc3Ophrg4u/doUvrw4oJZ3F4I3zDX714gW+e3WBi70lXt7PUTYawQkUkwbVp1P4kwb6MkW3b2E/6CA/y9GNAmQCdO9QlgkBCCuQ3irM/vprtFbhWwdv8Ht/8g0AgOgE6guPyQ8Mjv7Vl7j8nQuUZx5hQlRs8g7VNoFcauRP1xDSw/5wzuTYDti8Z5FdazRPG4QAhJsEF79yhUWZw69zrE8lQishS0Wf30Zg85RSVACoGolgAs6/eYPrPzrFs3/+BV48zNF9MYYrBLrDDullApcA9QkHAd96/yV+9PtPByD35G99ho//r2cwv7iE//05yg8aZJ+n6B63sAsDdVIhPC/Q7DHMqHy3w/jHnJJVv1RBPs+AZ1tYAN0qxdNnr/H6/zhH9c0a4j5BkGSKmn1g+lnA4kPAnTUwqYX57hjVCb28QRmo9zawn41hJ2QGhSMLahYS3bsV1PMMNgee/Y3n+Oz1AWyrMfluhvoo4ORXr1H/7hlW70c/7nkDNauw3WRQzzP4NKDZF9gWXFialUD9rQrm0xzt3EMe1ZB5i3qbwnyW4Tv/0sf4o9/5ANkHS/g/mCNZAuunDMVZfhiQ3jGJFRLIfuMGd79/wvPzzQbJZcKFzYYDC4BgIQhKDnUZUB8C9qTF+AcpXAps3rUwS4X8tcDmqcPoYo3lKkPyZYrmtIN/adAcO8hGoDkE0juJ9QcMowoKqI/8kMwLAJtkx8T4NKD+axXkZzmCCoPvN7sV2F6QjWr2uci3RUDyjRXy/22K7XlkJbIAaQnc+G8C+O050Bxb6GWs6kh5XMtTMiZBA0n0ywFAaximZUceqlPQG2B7ETB+zuHMu//K5/jep+coPkkiU0igl6xiCq3n0Gn8MuDho364EeAvaoSHBLIRCFJifBmweJ/bqOqAh2+zQiZ7TR9yeSqxPffIbwS6f3mJ7X2OdSeRX+qhqzW7JcDffIPXe1DA9il/zIsvNYprMi3dmEyZjWmw6/ct5KSD+TSHqqO09ICDsW4sUf9Chf3/M8PiQ/q9g+Y+zn9MMLR+Fj3jZwKiA6pzi8knGtIGlCcCwVB+27NLXouBEbMjgdmn7C91CaBrXmvtjMBy8QEw/5jSURsZW1WxckR4oD5g8NPokj706iwgiMjCR9C7PZPD1W9P+wAAIABJREFU9QwQiL35tYCjvx/rVixlzie/Bzz8AmW05SNKyLbn7DCuTnidmDVZUHqv2aXrMtYFbZ46jF4qqIrnIVkErB5rVqKMBTbvcPDV3ZEVac+ByReSAVKtwPJdib2PPcpTEb3OCg/f8dj/rozsCQca1aFAdeox+xjYPKYUdvqpjN2qwP2HmtfxMmB85bF8xsoUlxKAr58oJCsgvyUbFRSwOaUcd/QqYPUOq1nGVxYP75uhJiVZB+icC756X6I+BGafEnQtPgRmPwbqPb3zYfaA1wW8/hWJ/e/xvG9PJIooa272KBNtZ2QNyyOFh18I2PsewWN1oJA9OHSFRHkshwCk/I6g22uB7aMIuCVQ70lMLi39ngGojgw9pjMJUwWECX20LpHQpUB5pGAL+lybmUR+7+CMRHVISbHOWP/iMmD6hR+6Z1ePNbIHyoqbmYJNBeoDdq5uTjWSCMi7QmB8ZVEeawSpYcqAZO2wPdGYXHZYXxim/F5TKjy64Tbld1GeHb3gPhGYvLRQdcDmUYJs4aC3QLNvUB5JTJ5b2JFEO6XUdntM1hNCIH/donyUMxTrRCO/d6jnCuOrFvWBgezI/HnDkKD6kADSZmRVZReQ3XcxIMggWTnkV1vUpyM0c4XRZYOgBLYXOVTjkd/UqA/5YdOVRztRrFZJJFxiMPlkheZkhCCAzSOD2ecN7j9KcfTHW7RzAwQ+1ycCxZcW9XEKVTNsyaWKLOd9h25EOamqPFQlUT7KkL9uh37QdqKQ37YI42QA0gCGECQkGrImiBVSwKca7Yy9oqkkwOz7PH2qoLcdwXLn4NOEns+yg52n0BsCO68lZK0QjIRaVrBHU0jr4QoNfVdBtB2qZ3tIX1ccqq8D3LyAWtfwRQK5rslUejKMQUrA+J0X0zn4aQFRNpHh3DGdsG7HNsq3WMGeqQyB4NXoQX0pHBnUr6TSAoP/fACuUT4bvIfIsh0YDgFQEdRqDV8v6O90HqLvuXwr5TVYC3V8BL9Ysnu0v7313vjToFbhJ9+8R+h+yn1/dbv984BmIcQYwP8C4G//vEEl8DVnLOcfHYen/+l/gKpJkKctWqvRfW8K/14FuzGYHG/QNAZtaVBMa1SvxhhfrNB1Gl2nMCoarF6PAeORjlo0qxSPLu5xdbmPycEWm2UOfZXCfLDCo/kKV4spyocc6bRBc5/DzGvY2xxqv4GQHkoF1IsMQ6hP4iEfDOWU8Zbt12gvRwgyQOy1CE4ilJRryoMG4TpDSAPy0w3K2wJ6qWEPOgjtESod5UkCGFvkH6co328GT6ZaariJAxIPIQPUdQI7c4DxEJUa/JDiSYmupMcTAKACYAWlmFbEL2MBN/IIYwuxMgiph14q2H0LtYyfeAHIhq/R7TvISiK52KK5Gg2MkZ9ZpqpKwE65LaboEJ4XcGcNt3ul4LMAtZEQT0rY24zy4rMNypsRROBk16wk2gO+D7eTTJBPAvRaottzEHn8EXnNmhgdOzPJSAaoin2fIWcHKQJQnG/QfDwdpJb1kxbmxsTuOPZ7upwSyTBi3UzQQNBhOA5qwxoU2Qp0Zy3S5ym8ib2AGWWgwZANNGvJGpkY6JIsBcF05pFda9SPW6gHjeKVZBdnYBcqAmAnjlU4EwfRSISc/h+9oL9HlZSdpnfxPaL80WumALfTMOynHcV90gHJg0LQ9LS5PACeTK3wYlgwky0kyNs8c9ArObA6faiUS/j6Zs2Fe9+j2c4DZc92F2TT9+QFGSte9igZTNZiqD3pmXVvMNyf3VL+GGSAdFzgekMWv5sG6I1AdeFgHiRUQxbEFgyAKs8CggFEF7c3BmM1Rw56LaErgWbfx/TlyNAJAlG9FQNr1OwH5G94XdpxfEzC6yFZCFRnHvmrWK0S2d5utKsTqU54HFS7C10avwCDk5ZAfchjKht2fTJwhJ9Nr3nNJcsdOyk7gilWIfCy7IOS+veXLeWk7Swy0aMImrZkErsxGebtBWWyffWK3u5UEy4jW7x6j52zrATi6yWrXeBSN+Z2Fa+47azH2b2mbDEwcj4G1DTzKFuMVSBMOo6hPlMmTfcBQbpkomw3iUz6W9LNdg5MvqSH0ie8v0+DBviaZsPO0t6PqhoCqGbWV6pwG7yJTLZmkJXe8mvHK+zqOFzsE04QGTG+l9c8XyomqfbdmekqYP1Yxv5PEetbGHJV3MTk4jjkFzG0qHjj0Uw5+OiZ6l4F4GL9Sl/hIS33TVd95Q6Z4fEVgVp1IjB+4YdaEwAEp7FOxhYC+Ws/AAy+Fpn5nkVspwSE1ZFEccMBgWoYtmMzfiZ1HQbGvp0y/EoEnrdsEVnZ+DvRFXEQc2VR7xGw6YbvZcodQOwKMdTBMFSIzw+Sx0F4MspeATaTSLZ99QkDkbqcUlCAMs12LJlIK7hP2QMZYpdSvuuNYEdlQRazG0sEyaqQZo8qHF37+ByehxCvdxGYjsvALDnUjwRJf/Ps8xo21+jG3AldxmoUG9COZTynHkGIIZSIXms/hO90I4nsrkOzb6BahjqxtoNBPu3c8Hi/rtAcZlCNHxKX+4RpSAzMqaq5z8IF+JRpwsnrLarH01jHwhAiaQPDdYxkX2Xss6SKgwm6NlcM8CokzMrBJ+yN9KZnkMnMCh+QLFt0YwNdWrK9ShCsWQ+v+R56E6WhRsJHpk3VdhdQ1LCKo++xhCRz3NfH2ImBqiKQUruuS5eTLRWONSgDs9t5AkzrCURjFUl/DQvvY7psDMPK9BD2E5SIFgRWp8jFFmGUkS11nnUlWlLN0ocG9axfXxXSgzPnv8J4Cud3clQpCC6NJuMZWc6gFcOIej9mHxhUt0Ni7QDqurcYxreZ2bdDhKxlQE9/68ElsOu0BHbsphQM6+nluf1+9QE/w+uEgTUdqk3+HLfQNH/2g37K7S8FY3n4F8RY/ve/+WfumxDCAPh7AP73EMJ/9nPfCHzNgWX65HE4+zv/IYIV0PecsKpGxHoQBoHoLReZyR1L7ZMlJ5pehxg7L2BzdhqadZ+gGKeoZ7vFHAIXkLbgojtZ7RaZvVyyX7zUB1wUq3on3ex7CvvOPm+4sEsWsWS85AKkOo4l8FGuazZ9iiOGCXIQQLpA9AwIdAWQLoHNBRe7IWKu8pSS3r7HTnj6ZboR/TV9Yf1QlB64H+0sLqx8lLnNxFD8Tl+Nx+ilxPbCY/qZxPpJ4NT6gBP10QsyB7YA6lOyGL08sD7gPrVzSgzbqYiJjVwc6DJg/RSYfNkXWu9+6PuFZjOPRfBHAsV1wPaCHpx2wseNX0ZvleeiqhuJ2E8ZZYxxUVaeBhTXXNh1vdwtj3LHfW5ndUywVdxwkVSeBRz+MRdUvWcrWfO82RxDf2BQXNz2/9fOYrF6zkUyuwR5Lts55bsuYwDGw0dA8Wo3/R9d0e/TS/dcysV9fRhDSA54fsyGSZX1XMLl/Hd9wEVeuqCHK4/7q1oM3X7SIpa08/3SBd/PbChJrQ8obQ6SzBp9H1EuOufiPr9h9cbyPYn5jx0W31CU0kZPEysyovQ3AbJFQDMV2F4A+99joqeMUkhpgckLi4cPNLLbtxJhmzAkueZvPLpCcBF855nc+khi/NLH2gV+R3jNIJH0ntUa6SJ8JYxldO2weofbmj34YXFttkyQlS2lp6YK2J5KjK88NmdyAD6jaw9nWNbuk51EO33wQ11DEAQN6ZIAg14sHvfJpcXymUYQlOV2Yy4WijdkSaSLC/6UEksEIFv4CKZZ/p0uY0LoiF6toMjg9AsknocwMJXFa496LpFs+JnpJeDZwg91I7oJ2JwpzD/psLnQMSyFC+JkHZA9OLRTSih1xc9N7y3LHlidka482pEc2OPi1kcpLGspxpctlk8TsiOWaaPVvkb24LA9UxhfWXgtUM85yPIJkN1Hn9iWICFdOQbjTAXG1w7VniL7lEqYDRmiZOMZ7JFScmkLyXRXy3NRvGFFizM8X33ib/bgUO+rKDslCPQ6Amyz++z0bFbPniGwjgNCYHuqhyTbZBMG2azZeDRziezeoZkrZPcO5bFGfmfJ7p0apCsOxGxO2W073rGT7ThWnNxaqNajG3HY1RUEy+mDRbOn2QE5igs3wc+iqlju3nvr3pavImAATH2qb3bvkCxaVotIRDBjCWAPGGiTPnTwSqDZp/cuWVo0+1ykqyYgWXYDKyQ7j3bObevDaEY3lhJlJZiqOtJMNVUC3VhBbwlIVMNFqjcMhsnu+oqL+EdCEJLc14OEM2gCGpsr6JIgoJtqhCgL7j15vR3CjhSy1w2aQ3opXCyG16WHyyVGX27gxglE61GeZ0gfLFRlhy5PVpn0dWasdujl3y5V0OuOaauZRLLo0I01kocG9UmO/OUWvjBwWbzmjUR6U8JNEwIV56E2DUJq0M4SqNoNXsEgBVzO15edQzdNWQeSaLR7KfS6g88UQd9II7mrIKxH9WiM7HUJN07gNb83VGW5DwFACEx5bT30ogI6i+54Eiu93NDFKzctfGEoVe0oi9XLBlAR1PT9kIIhO3pRwx7k0IsabpxCbVu4UQKEAL2oYGc5VNkOXZG+SCCrDnaeQy3r6HV0CImGLFvAOoQ8gc8Neyn7Co4oERV1Cz/KIeoGUApulkNtW/hMEwwKAX23oWw01ZCbGsHoXf8kQKmq0fQoZgl9jkZz/yJzGFJDUBjBl+gswrhgbUndUDZbNTs/Yv+n0V8JyQltS+nqUBciKWc1Ue4rWRcitCbwy1KEuiFYKzKCvRD4fCkQypog7q2AntB13NbgMVSL9AE6JjKSvZS1//8eYA4pumEHGCN4hBCD7BchRJAZWdEeFP8kSez/T3osR4ePwy/86z9/YPkP/oefDSwFD/7/COA+hPC3f+4bEG9faynspKjw7WeXWNQ5jj9cI5EOP7g9wdl0hZfLGSbvN2itxi/tvcE/ePEYv/7Oc5TW4L4ewSiHL24OcLK/QtkaTLMGme4wNg3+8PvP4A8qaBk4NJIBRjkYAHtFBeclXrzew+PjBzy/3sdHj6/xo8sT5EWL/VGJ+y0TDqT0WF1PkOzVaL8YQb+7wfY+pnalHn/zwx/jdz9/Fyf7K9zcT6G0Q7NJ8Y0nNzDS4bYc4e7jA+hHJdr7DCJ3mO1tUdYJKhHQ1gZCBMznW9zfTKELi/KxQAhAlre4mGzReYlXPz5CSDxEI7F9BwiZh0gcGgFkRQulPKQI2DyfAgAuPnyNRZVh82KKTeEAK6AnHdxdb1rkomr63gLL/RF01uF+nmLvZIXy1RTdX9+iuxxh9t4Dnk3W+FHzDn2vMmB0VGJ1W0CPO1S/aNE2BiEAxwcrXH95ALVlUNDDtwBMLZIvU3QzD70WyO4FHn7ZQo0smqsM2XtLbP54hvDtNR5qA51YdKsU1RNA5ha+0siuDJo9j+Rsi+r5COJRhQDANQpSe6wmCfRWotvvYO75Jbn5TgvxYNDOBM5/9Qpv1mM8XI0Rcod8VuPOT+CeVghvUviphdiy0gaZhygVzEJCfLDBIid15C5qqBcZqhPAJx7F0xXa78/QfauBeEigTios35fwjUJybeDOGnSbDNICqw86yM5g82sVxMsM/ryGSSwerkbwiUN9JHH67RtcfnEI0QkkD2Qq7VELc5OgO2aITjtXcKcNtu94+PsUyR0ZynbPQW8UgvRwcwv1YLB9RDna5qmH3kokHyxRfznhvuQC7ZGFSD3knYHXHIA8/JKjDP3UwRuF+sxClmQA+3TT0Ys+zMQhu1FwRYAdBdx/i6BcV0BzZCEKC68zlOcezZ4YvJ22ALJbsoHbC6arunmH9vMEzaEH4NHOYvhQFgZGJ2h6geqDgO05Bh+18GDP4EUHUUvYF0znJSMmUb7fQN0buDRW7KRheB038dBrCZdKJsNqprgmS3q6t+dkiosrSnnbJzXKBwYH9cYtN3EISmP1iy1EpZCsJHtTbwxBWwaIwMEXACAEuJHHMpC9Z2iOgN7sBjkuk2j2BMpvtFB3RD/pfWRuPX3XtlBopxzCVecOshJI73iNCAc0ewSC9ZFDUAbNPn13+W1AfURGq51otDNuljcchpXfbJB+lqLZJ0hr9hS6McG9CGAwRxyONfOAZpZGdpXez/Seybr1oYbNwdTPUT9IY6BNO1Vk92qFdgKkD5rhMx5wuUZ5GtBdUoZa7Wum1G4YnNWNgepID37FAXQ7hfIoJmRPAGk57KvWPNcQlALbPH799Z7tt/zHZku5vvA8D+2EKo3NUw6QvALSBcNguomAqjkoqvd5bdX7EvUB4BIdz5OAzRS6iYhe+wA74ucnvYt+9pmACBr1XFL2eUC5KCtTEjQzAeEVdBkAySRaXQHZvRoY0vrAkElfcRgR9G4f9JYBQ82tRLLSA5itDwTaseEANHprqz2mKfe+RLNHxm17JpHdhkG+6hIM4UnNVMZrCehGhqqHJiDIBOWhRH7HOhUIoH1XI30gqNZ1QDPnYK8dZ2QIHYdlzYzDjepgxGtuLGJlkI4+bU2/saFcXnYKSax50bVHeUSQ7XWG6kDGACeyhTmAzbmCV2MOJVKB6kCiOpDI73QcApB1L49lHHhxiZUuyRZ3Y4HxS8Hzsa8hHLB6RyMfKyYpm3GsSxIDK+6yUfTKRuZ4bOLnQEDXGs1UwmzZz1rvSeR3aqhF8ck4Mv4SSSoHH2szlcj+P/berFe27EAT+ta0pxjPcO+5Y6bTdro8V5XqgZbqGQk1zTNCTH8BoX7q34FAvDUI9S9A8I5EQzdN43KXy21nOjPvfM98Ytjjmnj41t5xrl2Ni24XOMEhXZ174kTseUesb31TKVOwlUD/oMLuqUZxx4mbYqPgCskQHsf9UkNE1Vj0zxYYVhrCU97bHWtO0JxniFpgmLNSJiqgPzJQHQGJTD2nZuc5sVBp9GuNQkm4SkHOdWI1w6GKpeF6ZO9hVxnkkMPNFDIpYJc6MawS2U2SsFY8D1JLYJ4RyCZALkLJapMhJwgv2GdJ6Wtiz0/mZEkzhVBlcDNWvgAg8F0UZBpdYDrsaoWgJYN7APo8jTr4Q0OA3LZwqd9UbxQ9oUZPDCcA9nHGyFTVzBBoVgViIFspeoswLyCFIJAd/ZRFxt8Hi6gU3xtjktJqgtosAdOqYKJqAsFRCjKY4+N+Iux9iapWBIijHzN5SKNSB8+kExAqKdlG4On9xFyKzLByxPuDB3Za7z1prPotPZdS4IMU2T88/u8+/hzAfwzgXwghfpKe+wcxxv/hd7mSrzVjWTx7Hj/99/9zzvKP6kxLOZedcTbZlfx9WFI6FiWmyHGfJ4lYCuIYJWPtGf1e+W0KgUlyPZZccz2uYHCKT8E8pk7l7qlUvT86SP5GueAYrGGX9IqNsraRjRzW9PVEwUGQbg5diq4iWyRthF2KqeB9jIUfQzymYCCb2MYli+zt/JDUOIag2OU9mZgC+hMgu6X0iMETDKup3o6SMjJUs7dky7IN2SSzI3uo29QzmPNLCOB2tA8OvYxmz4HVffZQ9Ymtu2H4S3FDydPI7Jqasrb8LqbSaDKKRZLGZVtuh27JbkrH4AqV+g+lJZvAL+I4JRMuX1rcfidjGM42TtfQxERrMmsA0DyQE+vQnpC16U5E6tmLEzNnZxx8SQfsH8uJOTM191f1ZPbGYnUGcCTGLg1Y5+8c2mM9XTs+576NzJBuuR7dHo61T8E1IRMpgY8Dotl7j24tp5CRcRmUhpGxYtcbnwe439k+TsyfsvTbmDqwRy51zI0dftmeg5H+SGD5wqE+U8g3ZE+HefKSpWMuksS2X/E45ZuAfsXtsxUZt2lgdhWmxMuJKcq5D2M4RpSYEj+ra3qI1MD9GjsoTZIaisAgG58B1RVTDLffMKlDEVNS4shCVZfs6xvmjJXP6jClI7KigD2B0lEaZZowDSwnGeTAY0c2nAXxquP1OZ5bU6eBl4vwqUR+mMvpmpWW12Bx4+FLOd0bY/AYfYdx6l6UngmZInCd3lBWKAKZk2FOWSGAqfdQOg4kRkmgbiOKG4/2RMO0YWImReSgvDuSyLcxSVVTmItkHUkwXI7PuF7dBEpDVQovcQxaaY8V8h3ZV+kjbEmAJF2EHOLEFuouUNa45f6P5123Af1KId94iLF6oaacMJgU5HQXphCbkIl0LMLUU+pzmdJcYzo3lAryM54suuoOZfEylaofJJj0p7kZ0yzZIepQP86Y4Llz/Dz2ZBXHMRRZyDDdr8NSIb9z07abrUPIOeBVbUhy4sMMv+rJwo7dkwD3QXWUHLpSIttSwik8tzuKwwCWYT8pJMdGMn1aQNrAeg0g+RE97IL+yfF51Ti4OQewuiGT6IskEa0pL+zXBmbnk5xQTgE6wUiMYTVBCZi9pfKm0JMSYlgYmNphWGrIgeE3IckGg07qBRug9gPcIkdUgvdNBIQL0DW7H0d2U0Sy/uMxG9lnOXj4Uk9so680hIvQDQNvmKTMvkYACZSkgJoYJ3mmtPT20QsXJ5ZP9g6+yrhfgkoP4QJCqaH2AxCAUDFMxlcG5raFWxcQNjAQJ1Id5WesvBA2XYcJAKjaMo20GQhkBncANoUh67csKOWse/h5DtlYMn2R0k2ATKNf5qzdCIDsBoQim9hFxMjKi4zMHgAG4yABqYwM7SGVNH2mDA6hTD2R1iNUGWWj6V/MEgARgiE8mSbLGCL8UQW5T4OttB1icJP0dHqktFMoRc9hO0D0A+IYanPveMVM06OYZ8nn2CJmhuArSUpjmRN0pUAe7qPja1I6qwgRMTcQ+5YSVCk+SG8VbT9VeCBEvqZuWN0xhuoYzZ9j2A/wYRjPfY+i5D7GECDuyVPJIOTTcmLdQMxIasS6oXRVKcSOybUiy8hQjiwp8KG8FfgQ1DmX2Ef5gUQ1JtmryAwZUuBDyeu4bQCBZJK7jp2Xh9f82u/A36xKJB5kuP86j98bxvLf+1tgLP/hb5fC/j/x+FozlqLwOP67b3CxneM7p1e4amc4LhvctBUu7ub49tkVfvHmDD/4+BV+8uoZfvD8NTpvsO0LDF6htxoP53tshxx1n+F01uCuKfHN5RZfXp0gZhZ//OA9hqDw1eYYRnmcX6/w6GSDZjC4u52hmA2Ylz3qLoOUEbPM4vL9CkcPdvBBYpEPqHt2DG1fL/Hk25cYvILzEvu6gFQBs7KHkhFDXWDzsYEyAXlhMa9aFNrhfLtAvy1gUx1JuehQZRbrssPFbo5h0BgaAyEjTMHOyxgFtp3BelVj33C2LAaBHz59i5988RFWRzV+eHqB7VDgxc0RMu0Re4NBAF4FWKsw1BnmRw1238gxm3VonUJ7XaL+o4jlcY1Nm+FkvcfdvoJ3EqIcsPtsDfmshu00q1KshGwl6h/3ECIiKxy6bQ6hIiAPEqRiNmBXZxAyorYK1apF+34OOAH5gJrRZmMAEyELh+Wyxc22ZC3JzHECzwvWqnQS+rSDbQ2rXbYZ5g9r7K8rIAqITgJLi+tGA2ZgTcldBqhIX+dtCVU5+L0BkudUzAZELyAEUC07bG7otR3OK8TS06faK4hBsCJj7WBmFuKLCvh2DTcohNoAOkDmPD/RSVaO6AiRe2BjUDypcfN2xgHVVsJVEe4hGa2oA2RDX6jZCYgg0D12rHY56hGdhMr9gUltFe7+1APwUBuN8GBA7BSgOcOYXWoMjy0wSMhGIswd90NGoOUXo94r+IcW2GmolgNje+IgdwphxvoL2UnEVPWy+0SSHR8EYgaoHRCKiOxGcvLiiWXty5qssBwkxEcNbKehbgyalsdv+LTF9iaHiHGSagtPb6g9iklqlOpUbjV8FXDjBGLB/ZYDEE1EyAK3T1HWbtf8MtvfaPQPHWQXEUpWxvgyIuSso5EDcPt9xZqbu+TB3Ei4OQMjEFk1oxKbWr0VqD8C8kv6cYMZvaEiJelyG+wyILtR0ySPnUfoWkENIjEerNaJkv5ld+SQXSVvddApECkc6j4kEmAU0A17LSmRFYhgFYld8vUQnOwCgGgEdE1PbNTcPp9FqIH7PqwB1ek08UX/cMiA2Us9+S133wS94VZA9twnSErl8yuZWEpAd3qq4/B5hIgCquF+5DdqkiD6IkL1rGmxizglUutac/LvI4n+OCV5R8DUvEY3il7WYR1hdipJEDlxhiBTJQsn56QFgiIzp2sAApMUeExn7te0UwBgL2lUk+ydHkZM3lmm4GoMCzJxnLTTGI6QkprVtO8MWuIkmqsA4elJCwrJ65vB7DmpWFyRzeM2ceLFFWQqETlJKTytDNImr20Eyks9+VCj1BiTtTkRQptDdkfLhrTsgVQp2ZrM3D3vcwoCipKTVMA4gZnBF9w/1RNIuZLvFYGy8uaRQLbVk8xU2qR2mI6Tga4jpMs4OaQ5KSYCoFrWQoxp0rrj9T/6NUdPrhzoxY8prTwo3mumyThJ1HGf8jteuzZN3ukmTinDwQC6zVLSLNLEIENfxklNOztI64E0AXbL1GTdkpkeLS/BCBQ3nIjJt6wCiZJpxkhMLSKQbzTaUw1vwEmeCMiHWQqySoyO5+SMnVEq3J7oKWFaOsAVJXQXoTszpfWSheekTVYnZi4DdFel+yJiWJJNHuYMNJK2IBvax4nJH1NwEYHmgcLsvUPIBfolvaYm+UFFODCpPuN7hI8YVgrlpYUr1TSBA6QJoYbL9SXl6lGmiaiSkygisMdT1xl8IQ+s5I6yYWm5PF17nn8BhJw+T2nL6TiMybSuUFOyrnAVQs5l4nTG75cQJzAcx9TqlKqr+iSVloDaDvAzwwmJykDPC/hcJXm3h1tk0Nse8XRBz6f1kK0luF7NeC/2HjFXEI7puIewnINvMuSaPkpwO4RLktqmY0/lCKZjRFhUBNpaAqsZE22FABYzIAFsERdp/Bcmv6Zoe4z+ylgmJdp9sJYkvFMSLTBtn5gT8MauhzxaI/bDQUo7gtFxjJ7niH1PADvWjkyP30yklfr/2mMZUx3Kv0kq7B8ef/vhowLmAAAgAElEQVSPrzVjWX77Sfyz/+o/wq7LcTqv8aDY4/O7U/ggEKPAUdXi28tL/OzmMd7fLPHoeItcO9RDhhAF2sFM3ZWnRzs8X9zh7X6F3mksiw5n5Q4/efcUZT7g49UtNkMJ6xWezDf4xdVDzPKBEtI+Q6Y96j7DUdXiel/haNbCR4G6z6BERIgCSgbsmxy2ziB0gFAR61WNZdHjcjeHEBF9ryEEcLRoEKLAvs2htUffG2SZg5YBm1crVE/2cE5CqYDcOLggsd+UiFbi6OEO3WDQNRmqeY9mxw+O+apFvSuwWjXY7ir4XkHICF1Y2H0GWAm1GuAbDTMfEIKE7zRgBarTBm2dAxuDOPMQKgB7g/nTLYafrhE+bdjbuTeIlSOoLDxiz3AePO65Ph0Q9xpi7nB0tMd2V0Ebj6HXBMbGY3g1Ax72UNrDdQYq83CbDDABqvQIThAgZYFgLnV3QjDEaP6wxv5yBqgI0SWZqozI1x38F3P4xz107mDvCgYbmYDYK4KlMgBZgL4y8LP0bbO07PqUkWFMVQByD7HTiGWALBzERQ4/Y4+osAIx55dVLD105eDuOEsqCo9oJeSeUlRIQNUS/lEPISPE+5ydi+sAv6TnCQIES2nABx0hj3rEiwKhCFALi3CdIc49RKPYX6kj9FYi5CyQd2VEcSXRPnXpi1MQmJkA6AhzrSkPdQJu7WHuFBNWU4hT8V6xgzMSDEES2MlBMEjICkQToe8U3NKjeK8RNaWuCAKhDCjfKrgywq7IwBDA8YvELT1kR8mmPfKcOR8I4MxWJqAQ4asAVcvp9QCmapX8ViKoFJhUMmzH5xFmL9Efe+Q3CsOa50h1AGTyMm/JyIecgUpmJ7mvnuFPISdw40CUagZ6rgkgo6RUEgBC6jQclqw9Kt8p2NVhAgXgQB+Sfu1x8iDbiOk1dhETeEnHJiWiQqRzOUsDUQuIwNof4emHEqP6Ip33KGMClQQ9UTJ4SLX8mxivBWACIT4nwPxw3Zhqb8ZaldEbPawism3yH3aYJKMhZ5CPLw7ALahDCFBUVEjwPCRZak85qvAENsIdam6k57J1k3znSQkiPSZ1yOhDH4NrpnApMXqdWdk0emFHkDyqFnzOgTjkQd2S3aVTo3l+ogZUiwl8jJ72D6qhRj+9TwFEiqmyYy2QXYjp3pbD4eeYYFudB3THcgJwuiUgtXP2UHIgflAYjHVVo2rElQzDkQ6snwgEViIAo9/e7CmvRcSUyBsMFTcAEjMeUxUDl/tBcNG9c+Rm3JZsQyAWFUH6sBTIN6xZYcop94/VT2Sv1XA4d8BBUaS6VImUFBqj6gGC2yDd4f9jCNKoUHDlwQ87TlyMXZ9jLQyQwOVYZeRjql051NzkWyoORjXDMOdn0uibnL2z2H2UTQqdoMWUtDplF+CwDTJVkwUFmDZMDKw3AqYN6FNFi7JpX+pD52QcvWkKqf5jrO9J4N0jsaOc0Mq2fgoGUn1M3a2sV5EpT0BEoLhi+I/wVApEfWDkR7Y+KjF5uoNmdU3QYlouq1SQvKVJlZH6RcfqGwAH5n8I6bMngegYYWcapnYT0By9m2L83PQx+Ww92e+M6a1mO8AXOgF8AkWRWGSy4gRGdq5h9g5Ri+k1IwstEysdcjVtU5SULMNTzaH2A2KuyGanIKNg0ndQYtqE9QSPBX2XEILPtQPCrCBT2pF5nR73XjeCzXHfx/2ewnbGtxg1MenT89ZNUljR2w/CfBAToyolRNORMR1rQkZ29d62IMYDkzr+PkpjnfuQ2fSHdUz/FxJjl+VvhALdf8Q4hfl8UEUyPv46BvP/Ix7LH/y93z1j+b/9N78fjOXXGlg++cE6/of/6N/Gq+YIn90+wNl8hx+v3uByWOAvrp7go+Ut3tVLnJQNrtsKhXZ4fb1GjEBZWGhF5qhuczw7ucMXrx7gyeNbfGt1hX/8xbfgW4U//c4L/PT1U7JXAETuIU2AAGAyh3aXA07iG9+4wMv3x9CZR/ASRTmg3hX46NENXnz1AHACak0/oz0vcfbpFa5uF/C3OWLhkS969JsCCEyOtS9n+ORP3uDzlw+BXkEvB7gtDSqPPrrBxb98gONPb3D1cg0AyE46+FcVwqnFbNVifzXD7KSBlAG7C3r9RJ8+BD3Bj18k4KIihJWIuYfcasRjixgE9JWBWzuonUIwvE7MWQvvJdm7VKzuG43svcHw0EFkAbFVBGy1hjzt8Xc++RL/yz/5LkIRoHcKbuUgBkqiikc12psSiMDR4y32f3WM8LyDbzREK+lFKkJiRQTcPEAcDQgNyfbF2R5Nk8PvTNqHALVRnMGuCLriZY5Q+en8RSch9hpx5mAuDRmyTzoIEYHrfPKDypZelVh4yFpB1wLDmYO51qloXmJY8QvSzw4SIBEEWUwreeyqgPKNQnvG5SIC4miAOM9ZjbE+JN2GPECe9kAUKP+PEr7gwF14wK091I5fAqYW6B5bFG9MAj70DroZByTZRqB95iCsQPlWoT8N0LU4SKV7gf4kIL9hSq74dI/h3Qxx4aCvDAvie0ypqb6MUI2AWwWovYSvyIIJK+BOLeRWE8gVYOpuTk+vHNIASxOQ2aOA7FqSjfBjRYeYBmB2HqaOWPbIRviM2zB1yQJT2mvQACQQdEQ8GSAvMwJRS+AX0+A/v6GfsHyjCCKzOIFK3fC46Ya+yGFFabO0BKb5jZxqTHwepx5HgADF7ATaMwZa9WsyCtmWHq7hiH7SUebuygRCWkyAazxn4yA3v2F/pi/ZD9udpH00lFDrJgVeRQI/4fj7cMSwLrsgqAYINOyMx2Csd9E1wURIxzW/YwqtcJTbd6esMLELDoZZsYEpQTIKMq9mK1MCMDtSdU2vJEICHGMNWzcyZAfbgfAHoNI+OJTYM/CIyxzZqLGH1c54TQ5r9s+OzOXI4ARDhtTsCCBnb2IKFCIDq1ugvAioHzNoyZURixf0GtJrhsTu8PwEk7bZEdQNCfDqBHjGczACbLJBlM9mG/aQ1k8EyosD0ByBUHV+sEvoLk69sM0jMU0ABAMUNwS7475LhyRp5bVv9nEa1Js6ASqDqW+Wn/cjU8Tj3R2LCdzL4cDSqi7CNJRCIxx6QEN+2J+xM1Y6Svmr88DALctz4QskSSztC7ohqM32EbY89OuyqoWhWCOrNyYCV1dMwB07SqMUibE8MMYiHOwNUbAjdJSh7z6SWLwKky/RlVx/dyRR3CaglgKjJsCZfo7BWlFymWNqrAisaBntBMNcIN+NHaEEs7YSyHaBnaTH7KHUXZwmlFTqFB3Phxp4bhETM98H9AuFbB/IlIXxOhCUTifwOia62kqm74KQ2F72hVLezXtRNyFJ0xMwjJw8y3YWrtTT8sd1jQ/hkUKh1ARwx89o2RPU0aLAPlQRgfJymOTuSNckIpBtDkzjJCHXBwZTDvckynoEwmNwVVLOdD51jiqohsFYcWS3IpimayNU5yZwOEqQQ87kWb23k9w5akqLIQS7K42CT2BQ+AjpKB2XA4OuopYTKBYuHIDkcGDzYqYnme6YCiv7xFiKw3sBAnFoCdEmaXJ63xhGJLpEkd9PcB2BXmYA6xggVOaHupFRVnsvsGhKk1WSMuMYJ/nvBPh+LaX18P/09xFUjmDuvhw3BRRFawGpCPr6YXqvSJ2b8T5b+dd1goaAGOMU7PPXPe6H/XzteyxPn8cf/Lt/C8Dyv/39AJZfayns1hb4n8+/idZqxChwvl/gJ/EZNj2ln//i7RNIGXG9nSFGAXtRQh4PyHKL3V0FmXlEL5CXFl989ghybnG9nSFEAd8pQEX85ZsnBJUmQGUBwQv4RiNf9mjvCoKyzOOrrx5C5B72qoRYDrBWIXqB882CYMsAvjbwKgKVx+2ughCRssG9xtApoPRQM4dunwGzgF/9/Akwc1ALC/GyhHjSAzuD92+OIE4GXF/POaOW84b1J5YM7BdLYO7RvJsTxCwtohfAIBErj/lxjd35HLJyrDuJHPijI9iT1wb+yMGXAbJ0EHOLmKStw20BUTlgYxCKBKZUnFgU1JpySgBYD/CtwlfbY4QyQM4tXKrokIOAP7Zor0tKI08H3L1aIx47oKP0TDiB8KRD3BuESGYm5gH6dYFwzA+5elcQZJoI0QogSPi1AyzDHcJdhrhI8shOwJ+kD/Y8AFbCnjj2RQ2SYDiQSYMkA4cA5O8NhiOP4aGH3KsJbNlF4GsS46P3ZOui4vJGFgkeaJ45iMCBibmTGEoNrDy8P3yQSkcpjt9mEE6g/lEHbAyBquGEQMi5/X0moPYK3Sc95J3hsdIRUUWIwAAZAEAQaB97yF6iPyXLJwcBe0KQ66ok0XIKuhGIvSFz6YHhhIE0MTFZIYuAB3yZmNm07fmbDCEjGxGKwG1wApCp7mROMGqPAmQnEkMg4GYBvgpwXk1yxJBHeCcSAxMn4AAJuAXPh7QHNmBcTnYn4fscvoxwc9ao6IaMaJQRdhWRX1GuGTT32+xFCsghGPRFujZTXQQs74ux6sUXB8mUcLg38ANCGZjAi8N2+yrC7CQrVyzTouUoCc0wyWF1I9A+8kAUyG+ZohpV6uNcEMRJSxAiEjsmIthlazmA85qS3XGAPrI+7YNUPdAcJh1EAkTjttfPeL6VY2+j8EwuHo/RCPjG+pP2wT0/skTa3sS28RJnArBOEwGCg+coKBccQaarxNQBOtbSyFRRowaCa5UmCKaBao8JcAaTJK8Lng/pE1PdilTFwm23i8O2No/klOaa7cTEfI4BQ1FzO+ycADIqYEhJuSMwHmWRI3A2O3q+KQVMPZ0lr3OzA4GD5/umYKQTMfl0GTiFezJRAlnhR1bvAMbHJHDdRoQZB/GshImTX3/atnSdjIBvrG1SPZ/3hgC6O07AQnHiBJHMu5MJYDYRPifoCgt698lY08tsZwLVRYCd3Rvoj8A7EES2BQNw1MB1m3t9j8wfiJNMuztKfvPE2gR1j9FM0uWoDjkDMklGR7lstiFDHCUmVs/lYyqzSMBYwJnDvZjt4nQ/R3W4t5UlIFb3kstHL7orCFBdIWBLgWzPdbXHEsXGo1upyf/OShJKRvuVnAD69BmiASvHehQgGDUxUeNrvKGfPsqU4O3oR/YJ8FI9MCbxBrhMTVUbIwPmjUSxs2jOcphdSls1gom3CSyqgWxkd0zprRoO7KPwlJ2OYBUAsq1PsncNs/cY5mrySUMCbeqjDObAPqreU6LaeAb3uAjpI/y9EakrFT3NQJLHapidhc95kftcwjQO3kioIVWczA1U4+ArDWkDwWPExCyOLKRwTLols0kaXXX08kYtEQDIwcFXJo1XPMLMkM2UArAB0UiybFoSRA2sGhGD470GIMxywAVEIyA7BuyIGAEtmWQLAPZQYTIylxNIjBHwgUm0ozdyfB5g+M7oIVUZGcoYk49UQrQ9w3vGlFohDqDSJ6/imB4L8HM2AT6RZQcWUusPXjM9L8TBOxm5TCEEvZ0xHhjLEVSqw3X9r2Qxgd9kLv/w+Fo95G9/ye/vI0LgzbsjPF9u8GS5xePFFuushQCwawr80aMLdLscIQh879E5sLI4WtV4uNzj46dX+OTsGtW8x9lqBzFIPDjZIcscTsqGwMRJzKoe+ZoePykDVqsGcqfhrMb6wR7LowZZaSH3CuImQ/loDwDQOqBc9Hi03kLogGrZQd9qCBVQrHoMnUFR0t+nGwHMmC7qtxmyygJZoFRNAH6Q0N/eUbpYeHridKCfcjmwxzIIZJVFXlkOxg29YvqkAwK9gbHyyGYD6i9WmD1oIBVHHipn8qtwAvpJw0HWXrFnMwoo7SEzT0BpAvSbHFFFlC8MWVAnEY8HIIEsteQHZvQS+srg/C/OICoHk9GThwCYjeR2dQp40CPWGphx+erGQLYKqhfARQ5ROcQ8YPGFhCg9B08SEFYiNBpqr4DExkrLD/TZWQ05SMTEEKqGslB9nkFfGshGYvG5BgS4vq2GeZUjVAFxTu/GCCJ8mRg4K6EGykyjivArD9kLhCLA3CiojqC0fG2Q3ShEGZFf0/umakpfZUewYy419I2GbCSK95pprolRMrdMls2+4gQJAxUEsmsJs5MwW4n8ViK/4jEsLuiBUwMQigi38JxlvlEwu4Pns3pNVtEtPZm2LSWlIYsQr8rEavD1+Y1k92ZOcFlekFE1Owld8zi4hYepBda/DEmCGDF7paBqheqVQvWa61eNhNkL6A23efSU6Vpi8StN6WVLRrp8q5BtOMAd60oWX1IKm91yv81WwM0J7svLNPhfBx7rWsAXYZKWRslyetUKVO/Y2SkisPxMsl4oI5BSLUGmLwCz5zqqdwKLL9TE2I7VPcWVwPJXQPVeQPXc1tXPeGzNXmD+UgKSktooWB1TXgqYWmD2Cpi9EigvUk9mCgabvVLI7iT0PoV0Od4nwyqieidRnguc/Cxi+QVSSiMI+sZJYJvAVEXmML/hcrINz6fqgdkriflLCekE8lvuX3FFf+jsNScOdM0KnyiBxVdcv0yhSflthGq5P9VbieIKKK45oC0vGOrFzkuChPIyYvaKgKC4jodBewoQCxrpOLAGR/Vch24EZm8TYyOA5efA7HVEcUFQOntNQDl7y8mMbAPM3/A+r96IyTfoKqRuRSC/4X6oltubXxPsz995CA+svgjIbyOOfp6ef8Of2Yb9jdkmojqPyG9H1jeyumYgsNFNRLbh72afZL+G3rriNqC8IMsxexOx+oKD7PI6wC4YQjb2dOa3ZB1XXzqYfQoFayOK64h8E1FeRxQ3/Ln+IrEcjuBAOh6//I73TnkVIByDnMpLbp/qx8kTboe0EfM3AdVFQHXJQbhuIsqLgGzDcK0xjEq3EYtXAYvXjtUlDSt6imsGOpma2zZ/x9Cr+TuPxSuH/I6ApLxJAWERqC49dBtR3gTM35Cdmr13qK58Yssi1Q+a4WCmjsgToB0nT9a/skmGyf23czGth2CN4DKrI3QXJ5ZJOi6zuOU2VJdhknDmu4DZOwfdRph9QHVukW8D8q2H7gKqCwfdRZS3ZCbzjYdpUodoYlGLDYOJpKMMVLdhArrSRWS7gPLKYfbeMhBuFyAdUF3wu9s0AdV5j3zjMX/VAmCqbHXB71VXiFTx4ujFrwN0G6C6iOJ6oH8wAqb2lGy6iOptC9VyucNKo7i2gGSK9Ox1i2AE8juH8n0HXXtkG4flr+p0HDyyjU2TFXHq9tSNh+oDsruBx1YAurao3vdQXUC2tdCNR35rGRRVSIJXF6G3PRnW25aBV9uBTPdmQH7dwRUK+W0P3bjpOzC/ahki1Tro/YD8tkcUAvlVS1lrrpDd9tB3Hcxdn5jdCNV7qMbBLjNKcBOTqfesI1Gdg7ltIWtS/LJ30NuOtSmp6zJqCTEEyN5BthbRSKhNSzZSSsh9R9DW0ZcoLAGe3LaQg4PsLETTQ/b8KQZHACgEZM1jIEYvo/MEk5mZQJyIkb2UzlPKeh/kpYdo+4ldFD1DjAh4LUTdImpFOeyQAnyc47qkPLCcAIQkYIa1fI2U/H96TewHxMESOFoL9D2BZ2I0o/cH9nEEjcYgekpYY6omidbxn+M/eP/B//9V/z7owfy6PuJhgvh3+e/35fG1Zix9Akw/f3+G09Ued3WJ50d3eH+9gt9rfC5OUSx62K/m+Gn/FOo8h1/XeH+3wMmyxsufPYZ61OCrmwdYvJS4OFoiOol/+dka+mkLkzn0VkOpACEj3HWB/csKWSPQZxl2IsLfZdDrgT6+KNDuc6j3OeqFgVxYXKkZwt6g2RuUO4EocvSVQVQRu3oBFD7NxEZ6F1sF78m2ZbcKrpOIZUB/O4eMgJ97gpbjgKHOIJoULHEaEL+sMJwNKHYSfk25o93kBH+CoRlDAIQC2q8WDHjREUoFeAFAAMM+g+4FfXEuRywC+iQ71RuFkFGe17cp1ruX0LcC+F6L/rrE4guJ7VLBXBqojrUOdhEhLzPIjweoFK4xskPle4nO50AVoM7JfOU3LCOPAsg2El1FABYMoF/nBOLQCcykwf1RAFqJ7FaiPY6oL2YQAsgvNIZTevb0Lsk+Hd+TbegtHP1qPgP0nYIvx5oMsoQiAG4pIDuB8p3A9rsc0Orrw+1jdiJ5ICWKlArsCznJu4o7gU5KzF5LDAtg9RlQP00hKwLoTwNUSxlbfpt6Ox1YJQEOUqt3QHsmODiOZEoaJDbEkUGsXqsEkAgSVM+B7vZbjOt3FZlVsxEp9l8hKGD2mmwVg0EILnwhACiYvUh9jinJtIxJmqxhdpSBqZZeQb0HTClgdpiShsv3DB3JE6OW3wrKUGuCj/o5j5HZEwg0jyNUTyDYnaYk3etUSVKTTZF9koN6MkyQZMkoqZJQbWKEHFOho+JAfv+RhGrYD2n2BEiTRFOzMoJhJkwwJnsgKKVsAASGgQAMGZFDYmCHiPIqefsyUL4aCD6zDVks1SafYIYp0Vd37CJFBO+X7OBT0x3PK99H+Z0Iaf8HsjUisYRjKnS2SQEoKalYOABbMjnSpnOV0qSHJQcSpmbicPtQch/TF5RPqcAyeYxYZUEpYcjIKGVbslZIMsLF64Ctlim5l9dpeU7ZoTgC8qvEWCaPl+ri5DEMqS9QdUiAKsk9Gw7gu7UChKRUVCQ/Y5uYMc/X02fHpOp+LabjUNzS5xcVATfTYxVsKWES20cQyoF4UClEpsAk5RSevrg2qom1lC4i33pEwYoFCIHWpPux577obly+QLYPsJUkULz16I4pcdQ1/9aeyATCBUzN+zjbxUm+KQJ/HyW1uo6TpNWVAmUT4CSBBxl9phmbnUf7QDOcKXkui1sPW2kCj9c99s8LqAETCDN7doQGLYCS54GfRZTSKkuWSXe8BmTDBN/RV2p2DGZRfUSWrindUP4qbUwsIF8XNHsddZveP3BZPnnzGOTEz4wxFTvKkSVE8ozyesvvHGxl2J8r2Sk61nNISzZTTrJSpguPqbtR8HNRdwcWRbfs4fQ5E5f7NXs17VxBdQGylFCWYF26yBTfUsI0YKpuYvDoo2Xy8JhQm986Pm9ZqSEtv8/VfkAUOaKSKe03BdhsHYIxHJiGSAav0gBESv5leq8vCOBiJvm8ZApwFNx3XVuIoOEzfr/o2jOwp2PfoCsVQqaSL9NP6wpGcn8GgtdRvqp6vl/0HrFigq/wAcICqrVwy3xi4uWvpYPKIQCB+y98SH2rPi2D15jsHYJRUJ1Lnl92ZQrHfRaBab5IbKCwAcKkbXDj55ki6xcipA8Q1kMBBzmrdZAJEMI6CKsRKsO0XB8BI6cgHQLFDmKes5/TB4iWnZbjz3FimkE4iS3sCKDF4A6MXYwQvZskr8I6RCUhRhlsCPRC+sDgvTFYJ0QCQ60OEth7IUCUwfoDM5lA5gRKx47JlPL6G12Wiu+foOtYJzKMMtRRZpvWM/68nwib9m+sHrkvdY0SyVeZnvubJMICEDL8dYGyf3j8Hj2+1h7L5R+dxb/zX/8HuOtKtIPB4+UWv3jxCEJGnJzscfVmhT//0Wf4i/Mn2J/PUT2o0dyWQBBQGw35vIbdZxC1xt/78/8d/+Mvf8Bk0bsMcebxR994h1/+/Bli4TE7amGUh/UKZ8sdXl0ewe4zZMsetjWIQWB+1GD/fg45tyiqAc1VRTbtUQ+/Mfje917j5794BgAQvUT5bIf2zRyffP8dXv6zp3APLWWVQcCsO8yrHpvPjxBKskarjzbY3M4wXzfYXc4BHaCuMvi1g6wV1KOGgTS5h9ga6Ict1C9m6L/ZQdxkCDmDT/yRI1M4SotURH7awg4a6mUBe+pQHrfo3s0muWMoKGeUViD/7gbt5ytkn+zQvZ8BAsjPFbqPB5jZAHw5g5slryCAJycbvP7sIWQnkH+yQ/dygfknG2zfU/tlrjXwSQP9lzO4RYR7xMTQUHmYVQ/x2Qz2KGD+bIv2l2smbu4FhmcDxE5jKu2SkUE7rYbeUJYqnIDeSdjHwyFQqNUEYY/3sD9fYjjllztUhLql7ySmIBtxRr+n3Gqmdc4d8ldZYoYkhmNPaeszftia8wyLH1wDAOp/foryT26webGihPdWwy8CTj6+xdXFkoB2p+k9vSNoN3uB/swBQeDJJ1d49/kD6D3ZTHnWwd/mEMsBYcfet/K1RvedDvI8hz+x0BeGMskTh2zVI7yYJfkoYJ8PiK1Cca7hCsqUzJZenO75AHNuYB9xGX7OZNfiUqL59gC5IwAPcz95TPVOwK4CnnznEhc/OaOHckEvZ/98oDd4Q6+RnwWYOwZfhAzIbgXsKmL2vVv4/+kY+08I/mMZIFpJX+L3W8hXBeyxg77TiJLBPGYrYHb0EvZPLcovM4gANB9bFO8MdA3UzwKqdwS7ZDgE3KcNyn9ewc0B+aMNulcLJsUuOOgb/Zb9UYRbeUBGZJca0gpUbyNufxhQvVXsNnzaAu8KFBdMJG3+rIH6sqR88Shg+ZlC/SRi9kZg+2c9sNf0pj7iOTAb+pZnryX2zwNCGTB7odE88YgmYvaCqZjV+4jtNwkcZz++wd3LNYpzBbM9hNwQgAD1c4/lZwrDapTsJYbGE4jHJAEuzyXsgsDDzRIwBycX+iOg+3jA8i8z7D7xePDPJK5/zPfQe0r2VvZMTs3uDv6s7mFAeS45WfCYy7SrgPKdhF1wX1zJ621Yssaoe8DJgv6YQT+uIhO5/RZQXBCU7L6BFKwksHjB16iOktzqnN7H/jhi/lKge0Ava78G1p8HbL4pU8ptTMA9bfs8YvVL4Pb7wNHPgeYsyVFXvAcZ5gPUT+nnDFliiucR5TkngEaf8LA8eFrHY1FeEtDWTwS6BwHZlmx0MNyW2RuB7jRi9Tmw/UTg+K88br7PgdviRcTNDyMWX8lJxioHBhyNlVhM341Y/gpoH7KqKUr2ShLYkbHNNocgH5/zWslv6ZdszjiBtPk0TYx5QO8JLLtjMUmWzT6yKlrrRGgAACAASURBVKugB1REYPWrAFuSJYwKmL+hd3Vcz/rzgP0TiWEFzF8ynVUNkbVTMzHVQLVngutM1VpRA7M3YaptEpEMnZ3Tr4mYqpBKgf5IoHofpnopU7OHUvVAvmNvJL2Iad9agjnKtxODmKS5oxdymAu4mcDsfUDzQE4S5CiBxWuPzTcVFq8CujWTYYc5/aHZngAIYJ1Pcechh4jmoZ6AvGkIlF1Bj7kvBRavXKrIAvaPFWbnfkqC5bkXWH1pMSwV5by7gGzDGpn2WB1CfhpOBuwfK5TXEab2kzc1KqA+UyivWd+0eNFh+0mJ8tpBtx53386xeGnRPDST3FXZiPqhwvpXQ6o4Ym3R/J2FcBHNmYEayIDbuUS288juLPbPi+R59RgWagLFIgK6dpMktT/WKM8HNI9zrH52i+1318hvHfojqoiKK4vmLEO29cjuBkAKtA9z5LcW/ZFB9aZB86zi72uD/NZCuAC7Mvy+8xGq9YcQHAFkr+8wPF1zH+9Vt/hcTnVB2d3AFGwtoGpLMGT9BNYItiXUngmxqrZMva3MQbbbWvhlzt7LeQ616VgBU5mpRmYM6pGDY59lQ5+n3HcIVUG5bJKrxjJjTcqsOIDXJEPFKDUV4lBhEiMlsyFC7BvE5YygdlsDWjEptk8SWkfA9wGou48JlAT6gYxjS/ZclCUBaT9AZIb+yLo59G8qyZ8JbE4M5AhMx8fIZv5GYM9vR40hVaj86zx+LzyWJ8/jD//u795j+U//uz94LP+NH9YrfGd5gf+1/Qa21zO0bYbj0x3aPsO+zWFWPf7x59/Ew9MtmrmFVoG1DVmAP7II1yXkckDsFf77v/oRAed5Brfy0FcGv1RniCZAbjWalgE4MQ94FwTsLoOw9ByqnQJExN7PYG4VrAlofQHRS2Q3EgNyCBPx8vaIQC0AcWnRXM6gBoEvXjxE5gBzYWCPHcQg4N9VuJMVoCjjlL3A5uUKwgrsZYnyhaGvzQC404gqwp1XMK1gclor4E8l3KmHepdTprJTiAbQbwzTDBU4qyaAwVcpfVPAmgD7+QIm+cgQBULLgaTqgVXZoTFLCBGh9hLubIAaFORWw7UKeSfgFoDfZjDrDq9enkLvyQLaz5bIOoHdixWyncBw7Bmc8qLCcBSQ3UrYU4HiRiJsJeIFB/blG4VdMUPRCHSPPPSFht1p6IYVEOZOMiGzzlFccvAZpUqBFwK2VciuFexKQllKH5t5jswDy58b7D4JBH81ByPDOiDbSPSygPSUNYoI9DGxWTuGgkQpWenQaPoOHbD/6Qnit2pkLdD+5BgZ+DpTC1grcY0jMshBwGwk+ixAdwKiTtJGL5DdKty8fwSjKeV0FRDaEtIAweUobiVckaoX7gzMXiC/yaZ7Q3UG7lZPtRe6ERisRHatIHvAJB+jdIDcAf03A0IWkb0zKC4E9h8TVEoHBjMdkSlHo6H3DNeRPQAp8ebLU8zuKOm7+y4lmsM+9evFJD17o6Bb+siyDQNDRBDY1wVW2wizkQwcekg2ubiOiL8oycQFTVl05OC/P44oL3hd+tzA1Imx2mgyRA0wey0ZGiPENBj3L0uyT7dA/eUCygoUl6MPE5i/iuiPgPJcoB/I/Np5xOyVoBRwL6FrQM0B/6acmDKzj8DbAnovUmIoB6TFNb161c9zgouNQGhy+Iosnd5LBth4AXOrCGbuOPAiu8ztmr0lg13/xTGMYErp2JnIOo60vB2XJ0u+npU0BzYzGDH18wZN5tVVEi6xcvmGA3T5eQazi6jeMn0ju2Oip7Rkbs2GbKrZkSnONmQH81uyv1Ew/AcARJSTP04OgI6UbMuedQy+oK/L7ATKSzLUAO+xfBPhisTCxvE8pjRLx4Aks4sABKQXyHYxgQIyiCIQ4NHLmFj3Db2pdqCaYvU5GVs1JOAiBfLr1JN7G1Be8ZoNmUhJogyd8fkB6FA+Pkp9CZjKm4AogdYq5Dc8pwziAdTASoooySaVFzz+xSUZ03wTsPhSkfUVKUQrp2Q4ZJiChsyO3YzLL+nzG1NqpY2orgiMTDMm/x4Sa3XHrADpRAJySSlxR+a6O+axjIJpr2bPbWpPFOavqcYwTYB0XD4DiyLKq4BhTu+d8JFS3pQuPH9LoEeGl9892TZOSbrFLQGnnfO6kE5Cd2ECBvk2wOUisakEg1EAuh9VFgG69SgMe2flEGEQ4HL2ztIjCMjUuzr6tMsbx4AakdjWIKAsPXumFlPirrIR/VKiOiew80Yj3/gEIgMZzuRTNYK9q1JHXuOZQJb8jN4IlDcOQad+XJU8po1HtpPINqkfMhhke8/Qni5AVhLGk0VWLVm7fCeY2jp2dApg/g5TvUwwQHE5wOcKeqnSNihAkR2XA6XS2Za9qMWtZ22MZxrs/D1ZXJuqR8qa0lfVWuS5RMjJkuZ3qR5p8Mg2fpKgVo2fwnxEYlhV6yBdqirxZNLdqoDuAv+2UInVdSgvyW6OLGJ+J6E6h/wWvE9vLeQQkN+xS1R2lGsGIyEHn7zlCSgqCWQEf7IhGHTzbGI9eW14qG2PMM+APk6hPlMHZ2+hW8vO0N4CSkB2A2JOVlN2SQqqBPSmI3u87wkCtYSqh3s+R0/2sR8glCAjmYCXsARi4j4LCUD0dpLLisEe/jYyhEoSMPrAWpKRMbzXWzkB0fEnB9KUrqagnpFsIruo+Td7r29nAoThsH6ZQK4UTI1NyxfAJF8VYxjQ/ZTZ+6DybwAov85E2P+fHl9rYJlrh3968TF8EPjBt94AAH728+eAAJ5/cgkfBf705A2+2J/gYnOCbF2jerqHUR5VPuDydoG8sGglsFrWyLTH3ayCfzvD9/6tLyER8dMXTxFzj0+fXeCrq2N4p1BkFq0scPTRLWIUrBC5KVAdN9APAvy+ACLw4NvXOM+PkK17hF5jVbWIj4B2n6P8vID7wR6zxwO0CrhyKyzP9rDvFyif7nEyb3C9r9C+nQOPBjgrITOPPLdo6xzxj3fob+jBK09aModzB/OsQ/diAfu8B5yAedDCuQohC8A6Qm0VVj+8wdXlgiE5EZBLizx36FcGsWHdCT5qMewMqy22GqHykK2C+7jD24s1Fi8k9icZwspD3RjU3x7I+kWgexQhWwYB/cmzN/jiH34Hm08ZQpJ/Y4dmW8AUDuJxAG4Lfph91EH9qoRdRsBK9McBYW2Rv8pglwHhIatd+mMySVEDsQgQO3Yo2hUQZ/wgtaeAujUM1ukEuocOwgkMDxhe48sAv4jQ5zl8Aew/Dgz42RP8tI8YQBNVxOLjDbreYGgNK0cEZ2jdA8u6kbnD0EtWXzzo4SITaA1YG2Gf95CXGRCYaKt6JsaacwM3Dwxm6SW6jwaIWiG/ZI2KrjXsj/ewdwXKC43meWKbT9g72T1i9+KwjsDKok/Ms95SvuQXHnqjDj2GTkHNLOxCTaEnMZG9vozQb3K4JU1tzeNUK5Bj6vNTtcJw6qHvFLoHgSmvrZwSaduHAUElRlLRQzv2QprUYTnG/7dnAW7GpFV/naM9ZZ9id8qJkqgjuhPKll1FaS4EQZ7qmZraPCZg8GWgbMsACPTy7Z8zAEanrk8Ak6SxOz0E3ogkuw15ZMjRmZjqMJiiGjB/oRK7wMGsm3FQalceZkc2tn4KyEGgfeZRvFcQDujXPK6zVwL9MRlTu4qIIPuW30g0jwLUQHld1ED1LuLu+0zf7dfcxmEhpoRbV5F164/BJNiUaukLTlbpFqkfkRJKETFNHBXXAuVVRHci+HpB9ilKHg9IoHnICQxIoDtl0q1qJdw8wjZJ9poSQIdlko72wO5jAsn+hJ7ioLmNPo/IdmkCI4X1CM9tDDmwfyqn2gtfCHTHArYigyn7Q3ei2fPcDcsUmJJSZl3JRNdhdfCWDsuIbk35YHdEVmusF+Hrktx24La3ZwROo1eVHZoEWrYS2HyHTGgwQHZHEMvKitRHWRIY6UZMybRMFpVT5cnYIdmvU0qqo/9vWALSslKkPlPoT8iaCy9Zc7IFK0A0UF4GtA8I0ss7grtRWr1/KpHfxWldUQrYSk6hQGPCa3ciqUizZMzyO8pq908Jun0GJnyGNMGQwLKdCfhMQXqgT8tsjxV0F9GcsXe0W8mURpsCk1K3YVQEZe0JJx2CFkmSyuPUnXASyeepg3If0R2pBOIlfM7Jlf0TieqCcnDTBNiKLLI3iY1dSaheol8JrL+waB4k//xwYJFtxq5GgODbzgVqowlaK1ZXBJVAe03WMN9Qajl6H/s12a18E9CvNatNCgGIlJKqyFiKpFqEJugdwevo3XeFZHrwnUN/rNnhaNnfGJOU2WcSw0xC9WqqMAlGwC6zCXDauYI3gO45oVGfqWkiAAD6owwiRpiW4TwMaErhN4UiKB0iXKn43pbqDeQS7ZFElBrzL/dons0QMgllA/qTgoA/AtlmQH+SIwpu17BUtBnsPJoTsprZxsGXEq5U9CoGVp7EHe0icSdS4qtOUugIX2h0p4ZJs7lG1AKuYthOd2JQngf0a4PisocrFbKOXsGQ0cepeg9hPXyVpZ7KiFBl/L6bmcmPxioVyr2FlwiVgXABvjQEowCkUfC5ggKA3iIUadgsBJepKL0NhYZwgb7LVGcCJKCYG0QtIduUqpUescopr805KTyuE5lBTGmpyAwwWKbFtj0ZQqMJFq0DjGbiq/NMfHV+CtaBVgR4ziNWBYRLgFVJ/m2wgNYpwC75ORPAi85Pyf8QgkwlQC9ljICQDPq5z3YmMCtnFWLTEggKwSCge3/nCj6UzAqlfrt/8m8ol/19fwjg98oT+bt+fK2lsPk3n8Y//i//E1zdzZncKoDTxxtcvVpTFplS8IQTBCE9A0hUJ+AWgeAHlD1OwSqOnX8QEfnjBv07MnlTjH7ymAkPRh8FMjT9EwvRKOiaDJqwh5l33Qj0p+nmSb/7MiK/YmLk+GUTMrJLUQGqZYqjCOlvqSZCxDRg20iWqzecvR/W7JKLgvIz1fCLN2qG+ExdUI5fmsOaVQtjtLrqMXnz5DCmBCYvnUXymgF2wQ+dUAYyqYntkZYesZF1dHMOMocVE2OzxGaYmpIuu4wwmySLWwWU7yXaxwHFpYRNlRIiYpLiAkjLoYyN55ED1KDI6kWBKTY+2xA0RJ1kXqlEfBx0mzolUm7JvriSQFA3AmZPYCAdl9OdJN/Xva680eM4+tRYEg4EzTTPYCLlmnP6AykZ4oB+7NyTFhjLtaPg8c82HAAHQ29af0Sf5FjDMPa72TnIQjeURJo9rxP6V/k6Wx18Z8MqsVepvy8opGOTvJDHXJZueK7tjAPbsSS8ORPT76o/9An6nKBLNzwedi6m64nXCyYWy9QpdKbG1Nc3DtjHrryouN1qYJl8f5TO8z2mBjhUKUw9gv6e97BPXW754Zof3z/WQoxpmtkuTr18Y4H6ONs/xeunFNWxi26cQBmPtdnTt2VqMl0yefVGaSI9sXHq/qN3le9jP19Eto/YfCJRXsRpWwEmkw4LMkDcmcOkwJiSGSiYYGfgHQe6uo3Jn8hz6TPWN0hLwGTqlMJoBEwb0TwgEEDk59p4HOyc6wOQ6ji4vb4kc0c2h+cs23Hdyo73whhgcjhuY48jB3RI1QxklRAJYsbSczKWArqP6JfJ4+lZ2TAlpbpDbYMaUvVGStnUXZwGwOO5VH2cugPHcJ+xE3GswlBDTOeaaZ5jCucIckzD8ybtYXljJUW/OPgPxzTOyScYI1zO42PqgH5JoMMaBgKmYSGnz5eQQJnPkrdWk4GXPk7VF7w243Q9jp8lY2pr0GI6NjqBBlZkiEkaampKJEcW0FaUctqK4Acg2yQCgRzPH9clPJcZNIGNzyWvuzpMIHN8z9h/yGv2nnQ4AXXgsBwAk8TVZ5I+zFTNwfuYfxvPpfRxSs21cwVTc8PHKpYxRXq8r4IRMDsyf77gJOjYvShtmJ7j9yW7FfsjDbMnQ2MXCtkdpZ1q4HeiXSjoJg3KBT74WzDcL35WEUDq2sMXCrr1077Tk5rqPBy9mJCAK/UktYyC+xtM8jKGdL0qAV3b1Ok4VoscEmHphySjnF3UcOuCQC9Gfnf4gzyUy+R58AWBhxwCfMG6D3YtSrKDvZ8koyHXqSPSE3Cl+g43M/SQDm56DUJENGraJ9XRRwkByNYBYypwes1YASJ8mMAcr6EAP8/ZF5n+JjuLmBvAjeDOIxqFaOTEbI6dkWJgr+SY2BqVINjzMd1bESE3XH4K3BEjU2dTVHeShE7y0uRpnJjCsSokpboK5xmkEyOQGaa3qsTmAUx71ZTditEjGSOlpVJM6a88QIkxHKWx43piPPwNIIiUTLCdklp/jTX8jX5KpeiDvM9YyjESPK3rfqekkIijB9P7yUd6v6vy1x9TquyvS3I/eNGHbObXvcdyfvI8/vDf+c9+58v9J//o7/+/vm/A1xxYLr7zKP7ov/hPkWuHVdaitjku6xmaLkOZW0gZcFo1eL9bHBjK3MIOGkJGGOMxDApaB9Z07HNkpUWZWxjtUXcZhp4j4Lyw8F7COwmpApzVCF7A5I6fL62BKRxsa6YaE2kCwsBRn87SF10UcL3iTNA4VpQRwSrIzPP1TlCyu3BQhYPvFWuKMg+3MxB5wMOHG5y/ZYclrIRcsHtS6gDfaMBSGooA+g6thK4cQhCIQSCmHkkkFk4U4wgCwNYQkFYO4v9k7016ZNvSLKG1m9NZ6+bN7e9rIjJeZERkZGYVKokpFAgxYsKgJAQSkxqVYMSUCXNGSIgBA0aF6H4AI0CqopCKQQKVWRmREfG6e69fv95Ye7rdMVh77+MviSATiBLxnsok1/XrZnZ6O/atb61vLRkQrETw/FKHAESrEBoHcYqWfDIAS4vQKkAHiFEyviQQSIeYgYmKJkhCe/h9wagWG51bVQCcgFwY+EEBVgIyMKqlV4xU8YLZkEcJfzVCPJTwc0aApMzBUHmUm56xKIMEUgTHyOYCXDTMSZEYTgCbEeKuJMhuXC6uRcsbaHFg/mIQZOncnE0JX3nIXjJKpaPbZhCTwZJZRDfZuG4xsrEhe867QQC+DtA7Gdm3mGfXCdgLS4nXVjNLdEe5c2o8MFogIFQBxQNnCVMQPEBA7hqPYidzFEzKgkS02PczDzFEsOt5rnwZAZCfGgoIkcUrWewyozEgFGw8qI6NElcGVA8Sw4WHPooov4xFuxMYLxxkP2VUJqDrmpCLvuIgIQwwnnvIYSp2M5gLyMwU2QiBceWhRsFmShUyC6sGMjHCicxopTzIZCw1rjk/6Ev+XRoR5zID5m9otPT4PXYeGyKxgZEAcjKkUpFddVWg7PwYAVQbXxulwSzG2cxobsjUAbFYt4BdgMVeZB1lBMf5nhHBRxDT33OzCwnAReZVT6A6rVf1bEZJS0m36qYojAQEqy1lkTSlIshl1t/UhJEp3P4RYwpEsB9iwyk1GWLDIBnl+HJ63taTVJNsWmxKJGOf2IAwKwL3oCkBNguRozzsjPsxrpGbMSk3M2f0pWu5ICvqi9jQ2U/FOaMc+L7E7LmabLbuQ2xgTY2QlC2ZJb8mNjI8otSZq7ZzEWNRuN3lnoBYuAi0o2uoq2LwvJyus2TWQllsvIaAHGuiBh4PgMd92AgU+5DnJHMjS8XmQE/zHFciOq8SwDILMjBqJTYYbMNmSHEMNDjrpkaNGtlUSY0Y3XFbXcmGQ9r+YZ3ALeAKoN56mDlPStF6jPF33ZPdLNopNzOtA5jAobQBygSMc5lBa3pO9zyGlLemvMtkoiRga4JpEcgusiGBaErE68A2zKRMICuI1HwIGJZk5dLyXTllNuph+nvO4BTMwEzGUOMijhgc6JSajKyKo4eJuZn58xxBezL+cTlL08ft5HyjnUnIIeQYoXSvNHMZZaYedq4omZUiMnQhN8cp4WbxzmtQxs/PBGZdreALrr/Y0+THlQTP6fMgB5+bcOm7yJUybxfjS+jM6op47iLDGGQ81jLKWWNUSPoc6M7F9QjGf5SKADBmWorEEibwEhsujAMB5yoj0KNc99H/jYMcHeyyghxsbsR7LSGjYQ8ErylhCPYTaBLGxUiRKGXNgI/s6+OcSl/r6Cr7KE4m7kMGplFSGgoaOQljo8R0YvOE8wSmiY3UamoG5PgRmV+bgW46LolpTBLW9LcEOh8/UkRIYjKTNPdx1Ih/BFLNN0FfSO99tIz8d0xgMyQp7V/1+A7kWH7XgaX8q1/yu/sIADqjsSwG/Nn7Z1iWPayXKAqHU1diu5vjqjnih5c3uLlbEUeNGqYrMO4r9F0JpQK62xn6tkTwAs4qbO8WuJydYK3CxdkRAcA4Kvh/uoB732A17+GNhJAB475CXRvAC5iHCjACflC4vDygqgx0bSBUgB00/NuGdWGvsFx1+MNP3iC0GvNlzwgRGbA8P6FYjgRd2jPTUAWEGKchRolyNkKIgM2TA/MXVUBZWQgRoLUDRgm1HlFcdsDKQDyUlLPexPiKXYHmi2ker1iNCIYzIcWXFeAEls8ONBkpLfS7EtViILAbJMSaRjih8dBPW8zeKIh7MsaiI+isr3VmNH/0+1+jvFXc1nji9F5Bzi0++sF7qFZi+eQIqAB/LFAsRn4x9hJhlIxUOWgayJQebuUQRgVfevyb/8I/RvHyhPCyJ8AFMD7U+fcgAkRjgRUNjhBAZrqT0Ed+4Yn7En5tERaWIDU67cpRQFwMZE3nFtWtykx0KCg79UtKc/3aYvaOz5W3Cu7TDmoQ0FsFKKC418DKwDcObumjfMsiyAD7YozZegLQNA/RD5pgUgaIIYLYUWD1x3cwTw1Z9pKAWUZGWhqCJXhmWsqLIXbduV06RmrAk3EW6xGhIADTr09wC49wOZJlPxIIujpAgHNu1b2A2xi4MsAt+aVfRBfSACDogPGMgNCsPcYLh/ouMlCS8mg391D9FBJO11M2QIIAXBnQfWQweythzhzB1xOTmXE7Jwj1JQu4ce3hZx52SSbErD2Ko0D5IHksgFiUCAxXDsUBGSAiPJJp1mSvZXIcnXlmA8boDH3ibG9iel0dMJ6FHH1jVhH0F3T4lUYgAOifOqgeGDYsNIdLj+GcYKP7bCDLv+L7XRMZDU8gmhoIrgnon3ioDixS5yz8zXxiN4MOsMsks4wOuQJoXzoefw2cPrEYVwHjxsM1lKYjAMfvWc4kx2ZQ+8pBt0D7nMtRMc8xFe7dU0+muo4NgajkOHzqM5BIMl3bANXWE2gM/OwXR4JGFZctbED7yud8SWZ70k0aALondD7trwhk7QxxZpAMrY9FvTQEXc0Nz01iWoUlsCDAD1BdQPuMMuu0Pl8IuIYS3sS083qbmit2Fp10Bf+emwNzRNdjsoJFG1AcaGwybASqA+dlXbzl9udRxhrZ9f6CQFL3AcWR23h6gZz7p4aAw/ci05eyD1Vy7Q2Yv/MxEJ6AaFhH1jauT1pKgIcN7wNqIEDpzymLDkpgXAqMa4Fq61HtA4Y1We/uiq/Rp8Suc78Jiqlm6C+i+cwrsuv8XFP6K22IDYWQ42OkBboLMps+XlNcRsDpOb8jeA3zeFJazPUGQTdbX4gMZkRgfEhi6VPEhoqArTsna2xmlOqPS843moYNPa9EdB7mPCQEXYR1x2uLMt/I/PZk+rsLfnekdbhHoFQZynltLWHmAsMy5UMGDGsVQSvnJ3ldUYqZnGLNTGYjme6CLJorBEFi5yPzOZlmCUfZLCQwruhU60uB4SzmQw4e/YWGmUnYRmawyAPN9wgX0F3SmdcsyYrZmUR/UaC7LOBLGR3YI0hdKAxrlZUUh1caciAw8IWEayS6ywKqsyiONvvr+Rg3kvIgh00BOXqYOVlMs9CcyxTgPloP6TzK7cBlREZRGn4eXK0wrrmM8KgJYBZkbQFQRlsTeAnrM3iVx5HX40imFSFw5jI2mIRx0NsW8jRAtpxVhRR0nbUeXkuIweQ8T+E9fENZas62bDlr6UudgWIoNVnbIc5kSsllJGCY2McEKhXZVEg5zV0CEbgyliSD5AhQg4rrjw6zCYQCmHItlZycXBPgSwAwsqOQEnAO7tkGwXsE76c4kqKILKOb3Gd9ep+YjHsSO+o9wWQC5EJEV9jwTaY0stm/8ec78vgux418qxnLs99/Ev7dv/8v4U1/hu3Q4KuHM7T7GtV8xMvzHT5/d4FnVzs4L2GcxGg1nJMotIPzEn1foKoMxqHAbDbAeYnTtoH+UEB8fMLZssPtX1wAAELt8Mn3bnAaS2wPDeyoaV6jPUIQ8E4QADqB9brFw/WKLqNbjfBkgB8VNlcHPLxdY/7khHZfQyjeHJV20/IKB9sXbGZZGSuLAHQK9VWHsddQ2sM8VBBOIJSez1uJ+qJDf9ewyq8dqvmIYV+RvWwcghUQJ00WrXYIg4QYJB1QGwecCGSWzw84bmd8vlfA2pCNLAIB19wi3JeoX5zQv5tj/uqA4/tFlB8T/IaFhYzHBocCWBpgWyDUPjrYdrD3NULlyBoGgj0EQDwdgLc13NJRvhzZGTzvEa5rhHOD6lcV+o9H1F+W6F8wR9SvLNSW8w/CiQgOCDCCYOEsnIBfW7q8bkaI+5Js59wjaA9hJcQgIpsngTNWp+K+JENYe1TXGsMTh/ID5w7VUcFXHqHxkEd2Lf3SonpbZLmtWZM9BADz8YD6ZzX6J47rjtsjArMau+8PEK0GHBBmDmqr4VYOxa2GmwXgYoB8WyMxjWbtIAcaPBVHSrnNxqHYqVj8B6hWwl4YiBONcPSJrxNABnrjiixk9UHCxqKewMfDL3guZB/lcYFgs7/0cEsHeaJJSZI4d58YiE4CCqg+KNiawfVyBMw6oP5A99Xx3KF5pzGce+Zqzj3cwmP2hUb3ko67QQMiMq36OLlsBgH4ii6fqudy9WmS4vL5CBpPAsMFJeJq5D7pLrKnJTv9iW1MDFT72kHvZYweAdrnHqtfSJxeEnDSfIcA7ow3kQAAIABJREFUYDz3KO9lZtyKE8FkuaXk2px7FA+8vl3s7+iWhiztC89syVsBsyJYrG9ELqR9jBQZznm/qO7kxFxGEOrLCXABPBaqFXCzgOZaZCm7L4D5u4DuMs6oRQOo+pZGOMePGHPhNbd/9Qvg8ClQbgkqhk1AfS+gOhrlBEVwAEEGtLkWWXbIsYHIrmue88QcmgUB4HAuMhAuIgBTI6Xo9S0ZSh+ZsHHF95DRAMYzSsfNnPsxf8OZxOLA45EiT+yM14edc1uLI58vjgRFyy85v5jYueLIdZR7mhJBJMY3MoYRsPmS225WQHU/McN2RtOgIAlY2xcB5Y6xMwjAcC5yXqXqA3wpsoycLr88Lo/jbmSUxCKyUa7iMZ+/IcAjQBFwDXKeZlrPuOYc4/Elr7fZB4KafsN97J6K2HCJTF88n2k9s/eUW9qZyA6z83ee18iKM8JJmj4uY9brnYdtCFjLfcgmP7pl9Ex3KVEeKLGXFlh+6TCsOG9abUNmu9iciNFPgecUoHurLwlwpUOOpbH11LRK11piFQHk13rNc1dvPfq1zMA3gfPi5NE+VVDdVB+VJ0qQq72P7KxHd67JCJoQz5HMQLOI5ktAkn9zvdWeMuNhLbF4Y2HnErr1OD3TWH0xwjYKw1pSFmsCyn0EoY+YSldRxql6Os9WO7J540Ki2rvcfEBsTJyeaszfW5i5RHHyGFcK5c5G0CVRbS3GFSWvKYPTNhLVlut2FRnVcmfgS0aOAJRbJ8a9OFqM6wK+ENCti3Jukc18EhBWvYNZF9AnBztTKA58X7kzGM8KZn+epr/JyPYN5xXKhxHjeYnyYYSd6ezES0MjzlgGSTaWjOeUXyoGB99oxocAkZ108LXKLG5111MWKwRUZBaDACDFZM6TgGWtGTUCEGx2BtAyP48Q4GYl9P0JfllTWusoyZXtSOmtsXkOMpQaoo0fpMcGNyDA9PNmcnONs5fi1NEldjAIdUlTHx8oAwY4e1lMVipBPsrLBDiXaRldIpSM7GI8XnpycQ3eE0w+ks4GOxkOZTmsENPvcTkhZVAWxQRgf4Npz18Lj3xHGMuf/mu/fcbyH/393w3G8lsNLOc/eB7+9f/i38Db4xpvb84QRgnVOLhRQpUeflsSeMkA1Vi4bQk5ynijCHzOxS/rKMXUMQZADQLmjAV7kKmKI1MF7ckwjjIXchA0Eim2CmZDlqD8oMhuVCG/V3Y0JYEKUCd2Ml3D+U4AcAvGeviSToXu0b8ismh25cmeRfv7VMh5RQbALunkOFw6yiTTazSLZ2H5u0hzSnEbfB1QbCmrlIOAwDRflSSHqeDwM0ag1O8Vuu8NqL+o4OoAs3aobjRcjDqwS4di00P+bB7BHcGKWRFIuCpEmQpy95nuj2TffHy9LwmWmq+47PzFmUYKoqTTNizizDLkeUAEvl8NAramhBKIzq8PivO1hkYmdhY455mAVcfrIxUrrua6Vcf50CRtNHPuR2KcVMcCK82s+mLK5fNliF+wgseyooQyMUJmRSao3Ip8zORIQJJmIKWjmU31IKIsFnmGL3X3haNkURqCAjPnNqQZPduwMCdDEzKzWRzpvKpPlEm6Bugv6ZIbRMyFFPF4VKlIj9dgEWWW67TflJCm+S8z5zoBvm9cMb7BzmPx7qb9cE28NmPhPM108j2umuSTwnJOtNxNwBBpG+uJzSr3yOwTBOWVdh6/vN0EDNKMYfeMUlXVR9ml5ecssUnSUII4bESWeQqHDL7zXFfAJEWN8lk5kp0aLlIHGnneV7fxHFoW4uOK+wtwH0VIMsok8WNBnOZB03xdklC6SuSZwsxAyni+RsrsOL8dQ+V3EeQNERB001wq5cVcri/F5NQaUuHIv8Mjz70mthCI56KeZkzTPYWfIzqm9ueSIMRx9tU2EUQYZNfSLO8NnGUtd5Ok2sxFduRM+8nQ+PQZRJ6lTTLlPFMb5bdqJFOU5gnTMtK8Y3JVTrOZ6ZpLs4zCEUjYaPyUmhBBAEVHkFK0nBnVfYjB91zvGCW+6d6eZOW6Y2ZkiLOIxYnMqK1lBgXpoUy8duYiSzTTDKYIyLOqQDz+jq+3VZyxE/Fz4HkeXATSrhSot4zOEH6aMQ1yYmXVEGDmZN5EiMY3K5rQSBdNmEYCPjKGPs+SPnaDBXgMdesniXC87wYZcykTaBx5/s1coto5uGoSZBHcTJ9JWxNMqjjzmYBakCLOgxN8kWGdzqcIATJGbFQ7h/ZKo76PYy6RdcxOqBFQ5TzjVLB7NguAKL/tPKShVDWb16RtF3SMtbMoi4xsrPABw4bgLP3fq0csZgJxMVbDNiozompwsI2GPvHkm5WGjjOp6XX5Mzk62LmO5zHkv7loYJPAuxo8wValJkY0Hs8kVZV9nLMLAXZGNtPXzMRMs5qhjKCunUCdHF2cqY3zmFEKm2fypIjXgge8h5tFt1cXOGMZZatpzjLlVub5Tp0ko49e78HX1xqij/OeaRY1SjpDMUlxE8sXSs3lJ0bwcfxGYiKB6e/J5TXNXKZHkon6MElU/zIYy7OSUQ5cFQSSAMFkdmyNTGbX83et+J4IDBOw/MszjiJKaTNGSIBRR6DqHHLuZXpOCuZiJrAYtzFY+2vnKx8/QgiTLPb/Dpd8F4Dl+T8jYPlf/m4Ay2+1FFaIgFI6jE5BiABReLhRQqiA5aID5habp3tsrg7wtxWefXqHFEQOL1A8aKDiB+CTT27I3r3u+KW7cUDlEWRAda8YITBzkAuD2aYj4CxCNpfxlyNNghQgGofiXsEuWAhJK1DsJTYfP8Cd8QNYPDB+w1UBfhblF5osCEBZIz5uoTtB0BXXFRQoYZSUzuljssEH3NzRsEdGm/rzAWbjM7uheoHmRmRwp0aRi37VMe8xSCCUXHay0vcli63qTkI966Ced6ivNbA0NBg6aBZjATh/tUUKnFcdsxhNV0AaQaZjoKGNfNnmQkv1IoIfwSy7MwuvCPJ0R9MO4QFRsxM7vhy5XZrzfnbOgshVAXbBTnrzXtBNNPD9+iS4zDOH4dLxOJ6PqG/5nK0DZXt3kXWquF1uFtA/cxk4jZcOiy/xDYOaYh9B0hMLaYHix3t0r/m7+sk+B76nWadx47PbaohsF2e9KIWzz8csyQyaMRiuBsZzh3ETYBeUaI1PLYIE2h/32cBmdh1ioYTsWAnwueaP71nkes6XATyvxYH7niSmCZCWO8TGCMjoCuTron3poQbui//sBNswf9AsQwaDbhZgViws+ysPVxDYueqRWdRHfbS75/b0TwKGixDnpKJUMhluJGnfIHLxbheBDI2NplqxyBdhAroJtPRPHGfcbMDpE8tzEIEeQXfIjIhwwP6n4yPGgdtdHLldKR+z3HEZ/VMfAU6AXdCd1sf5tXFFgOoLglZbE3gHjalIjqCku0pAmEWM7ujQGiTQvXLon1DKpTsyW2Qsuf/DRYBuuV3tU0ps7XwyC5pdc99mtx5qiMcvusgOG8Z7CAv0VyHmM1ISmLMTQ5SIav7bX6Q8R96X+nOyeeUhTOuuIvt5FtnVdHw90Nw5DOciGz7pLmS56XgGzN+77LTZX4XskJtmJ4cN710AI0HUyJnQovWRXWR8SQJ4PjqIApHdqwXNlQxZPQTKOc2Cc3g6NhOU4TaYhUB/STBmG4HhnNLy0ws6iPoS6K4EukuJautRHjyGjUD7QqC7EjBzOvGaFbexfZZmyoBq69g8qHntdk8EkllUYifHtcDxFTP2hjVfo0bkWUXdE6iz8cTfXUHQ3Z9J2FpkUBniPUKakLeLjq28JoLiMeguyVZJE+LxlRg2IkZ/eHSXMt4LeV7NXKC7kJhdj3QpfSmzwY4rgOLIc2OaJE8VOL0kYBzOBPqNyE2RFKFhK4H+XGWDL2kIrton0bzGESCq0UezqAB9ctAtQZqteSzJZvK9BJeUeyajHltJ2EpgWEnG8FRkXIMSMA2loO2FQtEyLgSRxU2A0tYCciQgNXOyer7gsn0psrGQ6h36M4VxSWdXX/E6HqIk1VUS3YXGuKLcVJ0MzFzxdS6g2A1Z5ukaSldtTcfVYaNhFyo2fSQVRqOHbWi0Y2cSrpDxODpI42DmKoJFRSnsiu8fznVk/mg2lMCfag1cFecilaATayWhdx3GM432eYUgBU4va5hlAdcoeCWg2hGqHeG1xLjWkMahvyigtx0dXx3lpS7KVsezIn5HUarqZhrSes5WBmA8q7iflQKEgOgMfKVh51G1ZBybvRFMyu2RRkQRDJp1BTgydHZeYDivckXsk2HRoYeIxjNuUbF5U0ZgJSUZPyEQSk3w5gKCUvDLGkFK+FkFOA8/q+DnVX6dGA1dX0NAmNcIhWa+5COgKQYDGEvpqqOrK6K7K6T8pqQVIHsYwWTQKoPYoBVdZKUk4FTxuaqc5i6BXw/8EiOZjH6EgNCa0SRKQShJpvORmY7QGiKypEIIgnohJmmt/KthR5LJ/trHX2cO81vwSAqJ3+bP78rj281YfvY8/OF/8u/g6ewAGxR+sLjBr04XKJXDh26Bq+aI69MKtTY4jBXWVQ8fBErlMFgNG9ttzktcb5f4g+fv8Oe3T1Bqi/NZh0YbdLZAow1OpsQvf/YMi+dHOCcxr9kxqbRFOxbYHxs0zYhV0+PmfoUfvbzGfTdDoRyMUxisxv12jsWih/MSSnpUhYWxClJ6eM9tqUsDJQK2bYNxVKhrg2U94NBXcX0O3VhgPeswKwxuj3No5bE71lCKM5Y2RqIMRqMqLE5tBYgAKQMuVicYp3BoayxnDJkdjIaxCj7OmIYgoAsLrT2qwuD+donzywPavsLY6/i8gw8C5+sTPrw5Q72Jy3o7R/XihH5XQRSeBkZWQhcOZWVhjIKzCu6k8ez1Pe73c86Wznrc3S+QJFC6cBi3FUTjoAvHedZdBb0wsK1GtRpgRo31qsX2foFm2WMYCpofyQBdWti+gFAeQgJSelijgIeSDYLaIuxKYGUo1Z1bhFFCVLH77PilnJjsYm5gRwVxV0K/bDG2BXTlYAcFVTl4KxFaDRQexYcC5omBqi38bYXqeYv+UGF+1qE7lfCdhig9TYxaDXiRWXB9NkL+smEUx4UBBsWGQk2zpFAm9jxAP2jGnrQKQQWgpjlS6OOXgfaQRw3fOM7stnTcw9JCaM/9X9CyNc3wAgBKzyiVMkmIJA2MVsx583XqRgaog0KxlxieOITGQd1rRp1sNdzMZ/MmObCh4ev4t5JmRsVOYryM29fHDmkRAMHZUrLZIro2R7Y/VcaKJjsicHnFrc7S31ASAAsryMTGhpI5cxBWoLpXsE3gzGcbVQmBEmjZCbgzi/qrMs89Zhl1SXZZJuOTOjZgTjQwkgMdpxFjTthQEJkpSYyXGgSGCwcV429SQwkCEGaSy/pyMh0q9gQNw4XPRizSCLjaQ7eSnlMGk2FLyZlVCKDcMd5FWM5WqpaNraDYVJCOjQ194syrWUb2akZ3Y19ShpzmCn1BJQVZSb7PF4hxHWwIZeMZwQaG15P0VjqBYeMxfyPhFRsE3GE+B895WTkKlDuRAb5dTNmISU1hlgHVlsDtsXmOLziv7DUy45SAZWIbVQcMF4w4oTMocryMKye2HSDbOa65jckcyFXRtKiJf4ssdnc1NZPqe84spjxJKiuQ3ZVVNB4qt7w+7DzKdaPxUpJXqwFZNkw1xuQ0HeS07xB8fTJvUt00V5kMk9JyXB2ZVJ2upwmgqiFEdlbkBoTqydj4gkxsfcfXBs1tSIZEdkbHad3G+dI4j5kY6uIQJcBHApf+QqI4sIHGWb7JxIgz0CIzfomtlSMNkaTh9sADkNO+ypF/V32YDJwE6KxrwzeuTbJ7kT2OLs4+RnGYKM/+xjmLLHwVTYiS0ZAraYSUzHHMYnL1Tg1er4HmwcFWMrK0ZG7rHZ2CpQkouhDBKiWzyUjIxUgXGgEhG0ol8A9wX2wts6NxkEDRMdNTR0ZVOpoICUc22cwpr1dxGUkO6wvuR3kgY2wb3qeKIyNaihPnXV1B0FEcHEJs4ujW51gUV0dWMEQn6YHvT6+RIyNI1OAZo7JUPG8Cmf1NzKYap1zStC+IQFqOdNL1hZwMuwDOfmpKdF0dTfn2I+yigGoJilRnYJdVBp6uorlQMuN5bAyUQHYGuiaaCblAs5+UV2kn9l24kOcxQyEhB0tnWueiLNZRSquisVKqzV36Hh4nUBZZ0NDEbmsIEL3h/0OA6KJk9rE5T6HJkIbwDafZUBZ5GY8fmf0MgSxnZi7dxIYCBJVCAiH+LbGewU/mPgBy1MhjcyXgG46xfyUeSfeA7wBj+Yf/6m+fsfyf/6vfDcbyWw0sq09fhT/6T/9t7Nsa3UNDUOAFZqsep12Np093eP/5OTavdjh1FUHTTYXq9RHdtoasHfxJQx0V3MZCtArVjcL4WQf5toYvA/zSQtUOblAorgvOtK04Tzf7VYHupQMcUN8odK8sZl9ojOsA/6pHuKt485s5zjQeNYv/g0aoHYp7zS+BDV0/1QiMVxZqr1A+SPRPPLA2wK5AqDwZjKOCXTgIJ8iWxtOXpJy+DtB7+WiwH1M8xwiMGwe5MpBvatgLAxgJfVBwNQ1HUvFc7BkdIhxdTN3CI5Qe1bsCw3MD2SoWMgNZz5RnSPfIKTuyfFBZKpqKuqBYlIYktXIiF5wAYC4tijuN+oPA8VMHdZKw57z5l+81dCvQPXcot3Q8lQOLz8QkFScywXJkzIscRZYBV/csmlNB5Wowe68E+it2OFUs8tXAG2B1D7QvyIbNvxLY/YgSY1cDs2uB4TzAnHlU71XueLo6fRmxyE3nmKyNwPh7HcS7Orq8ErQUJ+D0MZedTDqa98zpkwPn5qo7gf6KsStqnExQ0hyeKwOkowlN99xh/pXKMtNiD0CSMTPrKL+VyCY3uhUYzjm/6GquQ3gW3d1TRJlyBBEamVGW41T8pogPGV0/U0HV3ATsv8dit7oX6C8jMAEmSXcERfM3Au0zSpd9lJKrQeSolSStdHWUdAqgvsfEuhZ8T3Gk7BuS0t7ZGwGzmKRids7jXt0jO2u6mtdwf+mx/rnIWYXpWkkskp1P8T46OpEycoczd8IAVZz/Uz3ybFwqZAHOGQbF42vnyOYmakA2ywEiKwqaodR3jyJYYm5hknwnsGAWQP2BxSYjhCL4qSe2z1UEk1VcnvA89j5G0KTCOy0jyQh1y6iRcTkZDCUpcrWLM3Mji107E1mqzvUSTKgYct9fCDQfyIQdPpL5OlJDjJJQEzBNTKidcb7TFyLPUI5rgeqejLCPcSi25r/Fiftq5nx9eWTOpQgB1ZbOooeXsdBsKS21NY1sdBugW6A6OOxf68xWS0t2lXEX3Fc1BBxfKKghYHbr0W9klsVPc3zTOU33Zt4PkQ1vvEKWICf5LARifmNAc+vQnycHSL42GyaN0/Ey8wn8JgOcovMwTVSlxM9NmktM7rZFGzAsOR/Ie2AEkXF2MW27MpTrNnd+et4D1Y7zg0k2m+YSlZncdH3B9VCa7KFbh+PLEkl+yflFArryxBlDMyfzmWfq+sQ6yhzt4irJTMc+5KiTJIF1pUD9YDGuVI40kW6SvgKIkTwe/UbRzGdrMZzpHEETBFDuHYYzneWxuvM0BYpglZ/fANU7+ELCLBjjw+uejKqJ7qzSBvTnGvW9hZ1JqM5nZ1RpuQyzLGDmcjL7QbyXH/l9OGwKNDcDxlWB4mg5C9vQPCXtF4SAPhqCqIEMG5sblK0mQ51yO2LclNllV46e2YvGcxaxUuiuCpQ7h3JvmPsZgafuHJfnQn6tsDESJYEBJSAHylrlYGFXFfR+4KyjYZSJtB5i9HAznYGf7Lk8fRzhSwXVMkvSF/y+5dygj8cmNV6RjXp8qeBqjeIwkmEtyI7KziJUKs9TCuPgZ2UGXGQkPSNSSv0Nt1d4IDQF5GlAii3xDSNJgpSQ/TjJdAtFo54Y3wElp/gS5yYZanJ9LSL7mdxXB8M5ytFM2+UcWUjBmckso81ZlHEfHjnEisEQkApBgGntJGv9NTLbMBoIrThfqTUlrsA0i2ktIMlahmGcJL/O8XnnIMoCIbKwv1EO+yiSRCTW9Tc8El4Jw/AbX/NXPX5ngOW/8s8AWP7XvxvA8lsthS0Ki90/eoJuX0M28eL1Au3XC2BQuNsuoDqJ0/9+DtNr+DsasHT7GrASr588QPQMlBdHBTiB/jljHuzaotxKiFahbsb8hTeec34STqB75iL7QjdMiID+KQf7/agAFeBXlknJPjInIkVQSJgrQ2DVS0AGjBcOsLzpd68typ2goU7hcfZ8T7ZIBwgjGYvhye6EgiyNj7OcbhYYfdALVA/cvqACxisLNA64rWjk0kY2TDHH061tBpVyFAiNQzgfYc8cGZvo6lq/LeBrT+bGxk7+hSOwu7AY1x7qFIu1EwGbW3oEHQgMnvXZHdCsPR0sVUB1L+A/7iEGznkev++grzqyVAGQJ0UWwyEyUoKMEaLrpg5wz8Y46+kfAVkAHrBzz/m5keY1zQ1lt+1Pes40PigUWwm39NC9iFLJaA4RJaLDBggyzr2uLNoXdLyUrcyMhCs5AwiwWNedQHGvowssi0X1VZ3dPJsbEcPQJ/YrqID5VwLdVcC45o3W6ygTPQr0TzzaZ5RLpkJxXHuMTyyGKxdzDkV2PDXLgOaWskZfcVmUIgdAgteJBMF6w2tgOA/oLwPMEqjvCGDho5w1AWc75V22ry0lkyDjMn8TMLtmAXj43iQFtg2PJ0AQk0Ds+MIwuqSMklvP15GFC5Q6z6d5N18SlAUdYBZkS8wyZLBlFtHYxPMnzXvZBT8nq19ym8ySYKy/DDF/kZLo4gian5zxuCcGSHc8XmU0GUnxGQlUpmOQoiYIpsMjEOyjjJuf0+bWo9zymBFocT39lY+uowKnV2S+GN0yxXrYOh7Tevp7AmhJ8ufqEGNiIkCOoJI5idy+oIFxyetv9xOL4Sya9ZzHY6ymedbkwFqcQp5nTY6YjuKI7A5aHGncUj942DmZrPFMZHku2RWayqTcQFdGF9LFxDLaxQSeWcwSNA3nEZwaguDFW4fhTOTIjfZZdDWNQKR9yntYc8fZ0dNTxXPWc57Sa7qn6hPlsa4Guo2KEQxky2wjsqxUGrImtqGple4BVwiMKwEbXVRp2OIxnFEyKzybBPmzoAkQ072KbKvI852M5uG1ZBu+rr53lCsvJ0Y1zYUm0Lr6YszmN+NS5JnIIUbIlBFEl0cy4ADlqnrguTRxbvb0XMLWqYkQUO8cn/NAfyazMQ3vQxFExmurPHDsYFxyu808gV2fWUjXsDAvTh5FzIE0Mx57Wwm6lvLrFd1FdHNNBaaMUtkzRclrBHvCI4PdYSWzYU+QMfakp7tq2kfpKLWGAKqd59+jDLTaOeiTzwAW4GfLNARk1UN0PQ3IuaHCA0XL2AoR82KT+ysjYgSKPeXCwkZXVxGluY2EbRS6J1WWSpuFive1qPzQZMPoVkuAlZ1QZZxbPRj4IgJy66dZ3ZLsZNAgIB4px4Uk+C0fBhR7E6OdmPUZCkppyz0ZyXFVQLeGsuMuymgbRbBZqhwLkpxbi4cOtlFwM41xTQlmkAJuVnK+1ZGFlB3Xx/gYAkY309w3Hbe7UBjPKqjjCEQgq49k6IIS0Pt+YhZLgmc1OAznFeSRXT0705ADjW6yKU/OkIz/T/s9KyFPPeyy4tynUvAzAkO3qiE8WcmcfWkcJa9KwtdlBJbfZBCFc5TDZlfWQBlsWeD/EstRl3F75DSzmMBnYhujkQ8KzZ+2Q3aH7Xo+5z1C1yO0HbLra3p9oclKFprL1Iqg0hjKXr2PjKRH6HtgGCDqGjAjMy7jfoVxnMyClCKofBwrImV2gQ3Oxx+XDX5CBKXTc9/8+a45w35XH99qxvLpj8/D3/rP/i38jbOvsLMN1rrDf//29wEAf+vqS9wOCzTKwASJ/dhg9ApnZYevj2dotMFVc8Sf3z/ByyXdRA6mwmA1Xi23+MPlG3w9bPA/fvl9zCqDs6aDEh6LYsBtt8BhKLGsRvggIEXArqtRaHacdscGdWVQFRbPFge8O6zQFAaXzRE2KOyHGl/9H88Qzg1+/MlbfPGwQV1YLKoBX7y9wGLdQQDohwKLWY9SOxTSo7eUoT5dHHDbztGNBUaj8ep8i9vjHN8/v8Vtt8DXHzYoK4NuW2Nx0WLV9Dj2FX765B3+wT/5Af7GDz/H2+MaSnp82C4gJWWym0ULAeDNry7xvR9c4/Y4x6wacTU74ec3VxgHjR+9usYXDxs4J2GNgh00wihx9vSA46nGk/M93t+vMJsNOHxY4Cc/+Bq/+B8+hf2M1baQHi/O97Be4v44w/mixb6vYIxGv62xvDqiUA5tX0Frh3XT4/pujT949Ra33Ry7rsZx2+Ds/IT9vsHZ2Qnb7Ry+0yjXA4QImDcDdocZ/KggC4+r8z1u/+QJVj+5w8PDAlVjMKsH7PZz1A2jWwBgHDVMr1HUFutFh+1+BrOrgMpBVQ5NM+L4MCP4lwHPXj7gfj+HEAHWKhSFQ3+osNi0aE8VlHa4WJ9w/fU5isUIqTykDOjuGqAIULVFWVr0b+dkpo8FJa2a1NUfffYl/uQXr7m+ADAeg7/L8wFladE9RBdgTalr0AFybuCtRDUfMV7PEOYOoleon54wfr6gedDcUQ47Kqj4euwLlE9ajF0BoQLU1zXMFVlt2UuIJz234U0N/7yHP8Us0kEi1A5wAvKk4DcGxfsSdu4xf3XA6csVpAGWP9hi+2FBp+GlYSF8XZGB9wJyM8LtCxpovYzLDwCiTFW2En5lUb0pMTyjhLe4V2h+vMX+egmogObzAv1nPS4ujnjYz+girMDcVRMLk8pDdJLH8mqAuK4gLLND5UFxnvlshD8WqG407NxPJldnFnKnUe4JUGwdyOarAMgAeVKZ/RHPe+Drhmz1ViPlTArPmJac9ccIAAAgAElEQVT2IwuUHs0vo+Q2sd0NAavs6WyMQHmqrwOEERmgA5hmS11sEihAxtlizgrHL3Uv0LwnU2xnAfUtGwjJoAWIDHBk2XwVUN1SEZBMvlKWZpCAeN0ifDXLua31tSazPHKW2TU8LuUHBWnJHNvlpEoQliqHJO+FBMzS50xU1U4ZpsVJ5BlUOw/ZyReI7PTSQyfH5XMHvVO5yRMkoEaBckvQnfJThSera+cC7Qs6+ooIjKWlQdK4BEJBxcK4YhPEzgjKaVA1HS8IGlHpDtEVOLqTCppdpVlT6dh4GJdxlnaMktRogpWAdH3H16Qc2CS3LQ7x/ESZrLTA/vct5p9rynnB9+8/s2jeaqieDQBXT+qW4ZzyZjkmFULcBx9jVAIBMhBZUEeQXZwY/TJ/Q2CsRjY0imMEeDG+JWVY6naS69pooMVmikB959FfSEb6xBxW6dg8OL4UqO9DZmPLPWNPigObQ8ktlQ7Lk2Oua9gIqrZ0dGW0TYxhmSHO8TJTs77nbGlivl1NxUu5jXFGmqyxMowyGVa8vpJkOCkXqn2Kt4hg8kjZc7UL2SgpSKDfMJtTmpBnipdfO253/ElmRtIG6CGC0YZNkcRmQ7CRkVnaUqC5czg91TRAiqBWRuMm24hHYF1mUJ8erhRZSuuioVDREiidnkjMbzzKLRui45nOM7qqD0jmYq7hrGsCg+NCYH5tCYZ9iE0Ezqg2NwbDhm6uNFsSeZ9VNJgSEaAPa4nZe8NlrjXqW96oaLQE9BuF5o7GM0FLqGgOlHIxvU7O4ZIS2EpRGTB6SBdg62hUBDaMiu0Asypjo0TQXKiK7rdjzNlMhKihw6wYPSAFZGvInio5SWJHSylsoQg4o7GQakeCZA9AkxUVgTLZUEio48DnhZjMhjwQKspls2EQAAyPJKERAKYIkswgJqAa2UBhbAThjCNJLGMGm4+Nd4TgcqyL7rWPQG+KLzEG2cQneC5Pa4JOF9nOFDvyeB0pGzS50hY6s5u/9hF8ltw+nun8f/r4XWEs/+hv//YZy3/43/xuMJbfamBZ/96L8Ow//PcAGRCMBKygrPXMolyMGI8lxFFTQveqxbitkGazYATOX2/x8OUGajPA9hrFdQH7YkSwAtVqgPgnS5hFYJREGYDKAaPMcRrl1yXGZ5ZM3sIgWAm51zlPUB9FLnzCzAJWQpSOM3BBQIycPyvvVQ6x93OH6h0BhuoFuk9HAgsVoGYW4aaGXxvom5ImQxcWcALlrSLjmaSxvUR1zziE408GoFMoNgPcuwZ+5nkMQLZUn2ScCwMgyb4FTRkwJCALh/BQAsmFVRDY4G1Nl9rIqMlWwj8ZIO5K+EWUJX+hMVx4+CdjjuzA0gIHjWLHKIlix+50kAHjlSOQkIHOuyvDdS/jjWRXQJ8EzJWF2sUifmCxbc48Qu2hdgopv9FXMULjvcK48dBHAeEF7MJDvuhgHirMnp7Qf7GEtGSE/cZAvy/hlnToLfaCmYV6ymIrdxL965HZmoGsmRwkiqPA8P0ezZ/VaF9bFDs627nnA/dfRrOgdZTTqID6RjNOQwDdRwb6XnPUNHCurv9xB/Wmhp0xssRXBBFBB5QPMucymkVAWBDEFHuJ/rlF8aBg1g7FXqG+YfFk1iFHbMDH2S8PsoZrGiDJQbKYnU8zbckUR47Ix8rVEfS5KFntozQvUA4qTQI4gPv9E5yT0J/XnMkSdL7tn3rU7yW6Fw7VnUK5BY6fcBZTOMAt+DvnNAOqO5mNd3RLM6j6jpJVswjQvchOzDrKolMepO5EZmRTrEJ28C3J3roSqO9ihMkcOR8yORELK1Btgf6CsuBkGGRnIYPH6p7XJAFEwPwtGTRfAuUD2dHZNaMommuBYROltFFSTJkvJcAJIIUiyuAOBDrJhTixuoyGoStyKvq5PwR8KU4DIkB4ylSrhzhPWPBc61ZkR+fqjvtmaxb9w3lA844FvJ1RSp2MvYJCjvQQLs6gRTAo3CQzZjEf5zaLlDnJbesvyPTTTZnbpGJuZZLd2oYmRONKILmzAojzoARkyShI+AgG58gOvKeXXFf1QBZwOIvGQT1ypAUANLce+48lmlsW70lyTgdtgtJhIyidbRip8fCTgNXPyVQmJ9rZuyi7j2yrqwggxrVAfcv3mgXlzum57kpkFUC5jzEklvvY3HocXkuUe35+x5XILr26C3m+djgTaD74bHg0XAjKo2tEdiUwVD7O0bmKMR/Cc5kJtNb38bNTIctL63tPk55Lgs00i/l4hlAawKwEVl84HJ9RIplk3oyUCdmFWVqez2obnYnj3GF3KVEco7KhDeg3Es0dDYNm7ymPldE12TRkx5LJFkBATTddrk8NAcNKQg8R7DRxvveRUVO1i8ZCNTC/ZmSKcFGG/kCJagJc3TllrtWe8llguuZn7y3U6HF8WaK5pQSX86IBx5cKugVmN4amOAoZWALIbKu0yCxstSfDOi45g0knaD5HkyMRZeB+coQGQViad3xsfORjrmdxYuZlEALFwcLVirLWGUFYfTvC1YpusdGYiOc7QHUeriFA4nOUCCdWV7eUx9qZRHMzcgbSethG05m2VtC9y060eabREXAmR17hA/qLAvWtic6vAeOmhBo89GGEXZZRtiuirNXCNRq6NfA6Ar0Q6EorQHnt6KFaMm3JoVY4HxstEinrUp1GgsHDgFBrMqAj5ymHiwrN2xPkoYef1wSJj+pp0VsynIWEbA1S1EjKs1S7U44FmZxdabIjjCOYkyJHiUwzl4/YzLKA6Aca8aTokih1Fcn4J4I4YR0wjAjzhrLYNH+ZWNK80NSBGgkCo6yVTwUymECOIAnD+A3QB6UQ+iFLdh9HlADIM5X/r5jHEMHstx1Ybl6HP/rb//5vfbn/8L/9D/5/3zfgWw4sq1evw+v/+O+i+JMF7JzRC/wCRO6Mu4pfts21wPFjj9lbDtoHCZz9hcfu+xLNTcDu94AmFt7LLz2GtUT7gnNaab7KLAPqW4HjRx6zN1H6dAo4vUoFLFBtWTwVRxYgZ3/hsPuU0SDd04D5G5Hnp2QsuI+fOKx+rmAW3GYVC4U0JyMNv+QOryV0NFpI4douOt3ZOQsMaVgw+OjcWD0gO+7Vt3ydbvnFlORq6QtKtyyi2pcBy1+yG6y7OIMT3StT9lv1EGCW7IKnrndxmAwRggS6Sxbg9QNnjqotCx3dAQ8/AhZfithNjyc0TAYP0lI+lb7MEZALrDHtywxYfm2x/0gzN/CMxUmScw0rCd1zX48vJRZfe4wrytGqbcDuM2D5S3adTUPHR91NLpsIXKarkaMMcoE7i4Vb7FSn+S5hWRyZucDsvcfphUQVu+/rzy2GteS2vOE1VpwCTs8pBZQuYHbjcXzOL+/l1wa77xXQLZeX5Ibjmsc/MVz3f9Nj87/JGB4erejXMrMHAOff0r43tx4pKLxo2cF2pYizXbxebMMZtzqGhh9fSczfRuv6Os7CRWOLFDWx/qXB4bWOhafMMr5kRCItJYfjQqCKRhXjWmD5leP1sfNonygcPw54+r/wS54GEyzejs8VVl9aHJ9rqJHFU/dU4OznLrMMqQMfBOfAkiNkyhR0VZzVi7EazT2vTWkYvO4qgXJnASmw/X6B4oTsMDqsJZZfW7RXGqeXArN3ZBZyZl4gU+MK5LgR3bOA7c8ETT0U5aBmyfzEauvQPtEoTh4QgGli4RudTe1Mot9IzK8dZ/iik3J55GyiV2QYxiUZGj0wBqK7kHlecX5t0J9TOqVbumf2Gw1fUCaou4B6R2bi9LzA8usB7ZMSw1ri7BcDDq8qVHuXi71yZ6PUlIWkjdEI/Yaf1+aWrpFkLySKzqNfK1R7n9mScaVQHB2OzzXqLQv4xZsR/XlBmeFMYvFVj/Z5lUGiNJw7czUbSsNKobm1cI3M2zWcF5h/3cEu6JiYZH+U5/J+IM2UL2gWEsvPWxw+naPcuwieApf9wUyywehIqmM0RH8ec+ICpZjz6xFmoWFr5v2lWUCvKeUE6PzqKhqblFsLX8kMAnTrYOcKKh2fM83XFCLLKW1Dt9H6weL4vIAeAmbvyVh4JTBsOMNd7iz0yaJ/WkHF+IqgBOxcQY4BzXWL4aLOs3XjWnHbagE5sLkibMjSzPqOkkP+X2FcaxQHh+Jo4RoFFYGI7snqUM7PGbzjRzVm7w3K9ye0n6xQHC3k6GBWZZyBI1ukWguzKrL5StAC6mTgGw0fXS+l81AtZafDVQ3bSMzeDflcqpOBPaugjwZmWZJlKmSe0StuW9izGqq36J800K2DHFye5/Mxg9DOFMrdSOlkBDpycLERRzMXr3lfClrCLDXq922USUZms4yzhXH/5Ohzca1vj3BnM84LDg7jpkL5MMDOCxTbHq4pMG5K3vduO0BKyHaEuZxBHUfIztBhNMovx7MK5W6MJmOUmErrIXtL5ssFuEWJ4r6Fr3kfcLWG6m3MfPQQgyNrVijYZQXVW4iRP6FQsOsGet9T5qklpZ4AIEQ2eRG9gb1YMOrDekZ3aJlZN2EcfFVAHjuEeR0NbzTkoYVfziCHGDGiJeS+I+CxjgBJSfhFA9mPXF78fAXFmA25PQGaoM3PSqhdl+tE0Y+cUYxAS0R5prtc8X2FpkOrEhAnUvVhXkOceoS6gjy2fP+xBZoaQSvOOzpP2WcIeQYyaM5vciGBTq/HluvQCqIfv8kAjgZoaoJCKSYw94gNFFpnQBe8h5g1QAKT/TdnDXM9n3Ipk7GOjC6rWpNhzHOdNr8+u60mwOdipEtiI9PjEcgEEOcvJ+ZU5AiTCDLjOoRSBJV/eV2/7vfH///LAPS7wFj+c2D5u/uY/+B5ePYf/T18/9kHvNuv0JQGSnosihHvDktczFt8dXMOvKtQfnJECAKzesDz5QE3pwWU9Hg4cFjJfjXHZ3/zS3y1PUN7rCgNNBL1psePnl7j8+05Ht6toBYW7qiBIuD1yzt89cUl6k2P4WaGoAKayxbdfQNROyxXHarCwnmB+w8rsp1GUGZ3KFBfdTCjhr+toJ+2MG0JWTg08xEfbR7ws//1I7gzC9U4PLvY4e2fP8HZpw84/uk5Lv7oBtfXZygag6Y2OGzj0JUMCCcNuTQI9yXC0gJGZgawOO/x+nKLL27O4Y4FyvWA4AXMroJejfBOALcV5zJnFuJEoyEAgBNQe53ld/ogYM7oaOk/6uGNRBgl5pctTu/nXKcXzO5cW4jCI/QKYpSQvYB63WJ8qMnczh2lnI2Dvi3gqwDfkDF0FwboFdRJws096neaM63JAMMLhGc9XU49IJzA7JM9ul+sKPusOdMqB5HZN3PmsPqZxv4PRoiOHcT6g8KwITtd3SoMFy4zuL6mkVEyNYIEgvYoP2i6bPYEJ+O5w+KXNHzonnlUdxL9Cwu9V5zb/aBpVtSEzKYFBdi1g+xYxDJfkb/3zy2aNxr9E4f6g2K+po9zcYrnobyP2aNxbs7NuK3Ntcx5nr4MaN5LjGcBZuWgjwrNtUD70sPNPdZ/qtG+CKhvBIZNgLQiR2oURwLO/orn2tVJZhVQ7mQGjt0Tj+JAp0HdgmxZRdCV9jdlgSbjn+a9wOlVZCfDJMkLms0aswhTpMwYMwVNjPvQwOwtYzXMkmxbcWTsRpBAcZjiJeBp8HN6FZ0+4za6Evn4JJdPNYCxNzOC4tOrAF8E1De06c/5hTHiw1U0WUqNId0C3XOP5a8khg1nTYcNGdVqm1hGRmgIS8Y0yQBNzJoc12RcAbKnZgFUdwS57Qs2l/RpmuMVlkH27XNmjybnzyz9jNLQNFMaFPdNnxjJkaJkyq34BuszXHDWlzOsZAVV/8hgp5yySRlLwBlZNXLZuqdJT3HkMQBis2zgvHJxSkYsBCSu4XqLI5k71ZO1o6QvOpDGz31iD2lmwqZYitqpbwNOL9j0KbOEMkk1RW6QJTMiFc1qykNA+1ROjqoDYv5pyFmydsb3pTzg1HhM+ZcAm2Cu5mxtMsixDUFruQ8Y4myiWXBWd9iwWeSikY6OUTIpg5VMOWIeZZRCemCI7CKZZZFNccoDP/fjkiwcYrMLiE2ekSzYuJSo9qnRJDNTqfuQpZl9NHcSFrkJMKwkI0biHK1ZsLFZHTifOC4FmvuQ5yi9YpbmsOa9pDyw+VTtafbjtcizkd2l5DUSG4pqDNnldFhJKIOcBYrABmJixvUQciaomYvM7D7OGZWGDRgTzXCS1DXNljLr0WcTG1eKvL0Aj3W5tegvC1RbspFBRhfUmcwsW7dRqHc+u6KOc4nySLMingeZGxl8v4NreCyqBwvbKOjWYVxrqD6xafyc+JINsJSRmVxw1UBH1eQQm1xwi5Nl9AmA4Uyj3NkYE8J1mYXKmZg0y5IodyNOrxrM3vbw1TQ3aRvJBsvovwGyh02B+nbMBkKJ/ZSW5j/DeZXZ8uJgCKw7i3FTotwZ2LmGPlD66mqCezfTsI3mumK2qjoZuHnBGJazCvpgoDoD1xQ8v51lQ6KiFFcdR/hSYzwvUd/0sUlEV9YEsEUfjXwKRZBtHNySN8GU/ekqBRXnLyEE5LGH3cwgjSe4TnEjhYqZn/wRg50Ak7EEjprxOfAgcI1RItm5VasINiOoS8ylkhDdgNBU3yyGQ+DfZ3X+HYUm4G0TWG4Ifrseoq4I2P4yU/mYXZQyMo8TKAWA0HUErlU5mfYoBQwDIGQ26Um5lME57ocxj1b1G3DHX4fF/I4Ayz/+l3/7wPIf/Hf/HFj+f35Un7wK/+J//nfw1c05/EOJIAPmX2h0Tz3NZVoJ8aKHVB7meoZQeBRbFWVblNWRaRMYftgBNxXqDzJangeYK4vmiyIGyCcpCYAXPdQvGsjoZgqwsDy9JCN6/Nhni/z5G6B9TjlS9/0B9a+qOPMRGbBOoH9psPynBcYVGdZyT7meXQZUH3hjKVqgfcrnhgvPTMoUEh9ZKTcLORJA9cDxY4/6g8zOgulLtb+gDM/OKO9yNQtIOQg074HD9zzUIFDdiVxIJ1MOBOD0PYPlzwp0TwJm7+hSOv8K6K9ELODJ0PUXlFWWDwr1HbexfUGTnmFDOZxZsph1DU1CpAVOLwIWXzE+AGBBSwMYj/lXEuOKmYmnl0C5435UD3GGKjJjqTAsd7E4ugwxNxSAiEHaK4+zP5UwcxrVSBOlXT2lis2NwOkFj3kqMPuLgOUXMbA9hshLC7qY9gLVAwvm/pnD5T+WOD2fGM3mhoX38bVHdU+QogZgPOMx0C3Z0t1niDJDusBKEwvuwHPRPQ2Yf839TrNXaQ6Lkjrm6SWXRV8A83cB9z/1WHyh8hxRKmaFB/bfj3EWBbD8gkV5tQ2odgH7j2UuoAl6yVYXR+YGDhceq78QqLcBu08pH9x/iphPyrknABn4QvAa6S4kdj9xePY/CTz8UKK+jSYtHbB46/DwmWJUw0bkOS45RsnaA4uM/oLbCTCLb/4156LGhUB54LEwC77m8JHA6ld87faHlNE2H6Kz6FJg83ML20i0TyTMHBmULr8MKFqPhx8ozN9xn82Cx3t2TQZj+xmbDuWezHB9TyZ69blH+1RiXBNg2hmv6+LE6/XsLzwefkAX4PM/8zi+ZKE4vyZTrDsW/0ECh08pLV58EVAdPIalRNEFFEcPs5A4PZNYvPUYVnQ1ZXwDGeogCSxcKbD82uH0TOaoCgBYfE329PhcI2jKHY+vFM7/dMTtT0vM39E0q7sgoEhOpAyN5z1oPBOo7gJmdw7DSmXjoeLIY7L6Kna6BZno+bXB/pMC1S6gO5dYvHPoLiQWby12nxaYv6e0//RURWdNKhS6c81oiEaguXdorxTMUuDs5xa7TzVWX1r0G4XV5wOOL0vogcsXbgJMxxcKm5+TYV//YsT+0xLV/8nem/xKmt3ZYedO3xDTize/HCsrq7IGVnEyRZGt7iZF2d1oWLIML2xDm9bWhgEvvfTCBvwX2CvvDO8N2QYMwwYMyDKkBjU11WQ1i1WsqszK+U0xfsOdvDj3fpHFZg9Ad0MkwQAeXuZ78SK+KSLu+Z1pEbE5pdzSlQKzhz0W9wqUK6ogguZrub506CeK7GMK21EtGXJfiHQ8e4gALF4vYKd8jyivAnTLVNr6ikmxk6cOmzON2acdLt8tUV9E1C96XD8oUV3SMwfw9d0nUJ29rgCw93GPfq5ZrVEIXhPJU9fuS0wfOYSSYS7tARUS46c9gpFY3zSozz2Wd8kaZ3llP9lVjfQzgfEzj2Lp0M80uj2Ffipw9IMGwUj0c7K09TkTVH1BBnfv0x7dXKM5lJg9dEnGLFG/7BGURHegoZuA9Q2NchlRXlpsbhRQPdNYQyGhN57ezQmTOzNjrbcO3UEJO5GoX1gGGzkys/28oK9v6+HG7CyULg4JqKr1sDMDXxHAmZVPoJRKHwSgPVCoL/wgb5UuScMXTGatXnZoT0rUz1qs79RQNsIsPVRHNqY94aLfrJmAaseaia0+wo13MlA3kqjP+yS5tGhu1KifNgiFotSz8ejnGtOPVuhORvCFRHXeQqZaDDfWUH2Aq9QuJfbQQK89GdhaQyaJ5/b2GOUFZa3FRYvN6xNMfrri++btCUafr9GejREM2VXpIrp9jdGTFqzA0RARMNcthAuwBzWCkdAbBzvR0K2HeblBd2PKlNhVBz8pyHynVFK17lk30ll0N2YozjewByMUTxbob+6heLFGdzaFtAF62aI/HEGve6jrLaAk7OEYetnCHoxQfnqO/s4hAWWlyaY2PcKshpuW0Ot+COHJXZbqYgW/P4VMrGSY1RC9g5+U3L7WQW66wdsor1aIkxFE0yGMKkpOASCxkGE+IcOZpKSZGR18iQkYim2LOEkTM0H5qbBux3IaDeE8Gc3VBkPfZIxJDkuJbRxVDOEpDJlOrRDXG4iiQOx7iOkEcb3h40/GgBSImy1EXXN7Msvp/BckqnHbDCAQwAAERVVh8Ezm5wQw9GY6gmRRFAiLJUQKEhoqRnIqbNt94fH/xO1nU2HzLaXT/ryU2NC2P/+x/gK3XwPLv/6b/re9AX+Zm1ARf+PwIZZtiQWA8bjFUk0wOtrC9hr6xONktkaIAo+uSxzfvsZL7ANlgFcR8twgnPTwY4MHN17iE32A7lTB9xL7h2uMCotn4xmM8RBWcZ0eBEZ1j/VRATm1TIV2Ast9hXK/xbocob65hj1VsNcVolGwcwfhBVTp0T9IVSY3GU7iK4mz25c4vzhheEjlsD3UEJ2EPmrQVBXURiIasnWdZqAHgYYagFJm8tpTevdEAPbvX2LZHpIZ8UCxlOiOPNRWop/Sayd8As1lYIVIlMDNFv3GwJcKesMaADuLEK9t4J+O8PaDJ/hxvAXRC6weAFFEmJUegkKa48R6VJGM6dkW248mkFbA7nkACuqNNdrPx4gm1RLoCF9JFNcCkAQTuRvRPjdQjQAOOzRtBXdkodoC3YlDf8jFAKV3O3+PiNwvO6YksL9h0adkANGw93H/9gKb5QEQI+yBB0yAeFRwsZhK3d3NHvZAoThXA8hevsEJvxul6H4Z4SYR/ZmF8AX6dxsoEbE9GzEpdaXgT3uoriSbc3OLtqyBABSXCvbIwk0U9JJMid6wM9IfWJhViX6PXrvmpoO5VoBMiZU1J4zbexbVE4N+b1cin+XIdkr/oR0zHXj9msckXTfdAatZ+hlgT3tEZRButmg3Nc93FGgP+Dy+BAoI9Hv0wlBeKhBKoH5jie5yjuZEoDvl+fV1gD2x0BcGqmd9SHlFptBOIuxEcrhw1GBzg1J21xKwtwcRqicD3B5k5jR7bjiUyIioOQsQXoKVIgyd0BsCvOaE7FkoKOvuTjzsy1TYLiIZ2EYOrOX2JPmdKiCUBLibNy1UZ6C39EFvz7i4b29biE4iaAXditTfCHp2KqCFACSwuktPrJuQtdjeiHAHDupDg1AQANsZXzeL+zyu0jOJU/YM3HBjgcnn7J/zRU4rlbBTgd6KNKgRWL3hIbwauhTbQzH4z+yUzHB5wVJ6O+bgRzf0821uSLRzOYTT9FbC1cD6loGvgW5Pwo2Zvjz9jJL15lgO/s/uMKJ6SYlv30m0h2J4LbqRQD8H2pUcfJ7BpKCPGeWz3SFgtmRQ+iltAf0mMSwHAC4px98es6Kpn5EZ6icMJPEF0M3JXPUTifZAQNqC6ayOMu88XHMjkRJG+fG3PTX0vyoG8uhGoDsQKJca/ZzVG90+exbbAwERFLq91F2YJM8+pQHbKSth2iPDoJaUNBo0BxxuRDavXPIzrNtTsCOBbt8MfY3bM243wo6JiyMMIDMEbod0gN4aTD9rsb5LP4GdsrBeN7xO2q2irF2TucIIWN0pE4gENjfoR28OKbfOCb85IId/K9DNDX3EFa/z5rQYElfJMuqhYkT1QL9HUGmnAs0RBwPdVAIogAg0BxLlIiXaxojtqUF7REuACHlxqYYuS18nmXvj4UYa/UzCGwE7S3URHWX6diyHYYmrJVylhoGW9ApmLdFPFdp9MrlZ4hzLJN0vUmprqlDxFVBdUdK+OTUo1gH9XoHNqYJZmXQMaHXJfkMR+Nzsqw1oDhUtKznzxIjhXNqJJsAV/L0fGYQiMbhp5m/nFV/7lYCdGMhawxdMaUWqDvG1IqvnKYft90ueQ0VJbtACdmpo1wgVfcP71VD/4fZKblfBD45QpCqVWTEkwwYlICcF60AKSU9jStr1pYIal/ClhG48uuNRYl572Kmh5NYx4VVvNF+HezVCIRELQ+A/ryF9QHdodlUhtQbkGKJLEtzSICoBe3OfctbaoN8zHC6MOG1Z3yow/TxCAZRwK4FQashxDWhJPyR43WVQKUKEmxguiFMHZRxVrBkRAn6vgkp9lBACKAvKZ6uSyinrEesSoTaUIBsFaAnRWoTDGUTvWEfiI+XFqmDa7JZSYaSaEkxGBJmFQSw15KoBXgVkWiHWJYTzCHtjyMRyCikJUMuS25/ZSoByVCxUdkwAACAASURBVCkoRS1MYh/VjrEsDO+XPJIi8n1RaPYgRRH4+Il1zdJcVp8oQGuCSp3AsHUM4knPDWMgrAWMQex7yHLHtr5KbOXU2AFIZpD5Ktj8JSbC/sTtV2hXfvb2S81Yjh7cjLf+u/8cMQic7K+w7Q3eOnyJTxcHuLieYD7bYt2UaC/5RlLMO8zGLTZtAa09vJcQAuh7gsBiv4XWHvNxg6fP5xAq4s0bL7HoKiw2NdplSR/g2CJGAVM4NMsKquJkzK8NRodbNE8mwMyinnSY1h0Wmxrd1iA2GjABsvAIawPoCFk7hK3G+GiLrjXwFyXEfg+lAlyvoAuPEAWiT9OkIIAoYGrL9NMlJbBCAOGqYNpmFJAmIFwU0MctbKuBIAAvMDtZwzqFdl1CmqTjD0CwCtJ4CBWBJxX8hABV7Vn4lYGcWISlYbrkE4Xtgw7mWQF/u4X6vKIU1gnoxyX83Rbxkm/yeiOZqlkHiNIjbvmmpdYS4aTn/yUlSDARQgWUn1To7reIW00J6T5TNYpnBv2ZhboyCMc9sNSsMGkkcKOFvy6gGgJw6AD0EigD4AVEKxGLCLXim1QYBVRPFNozglfRqUHyGlUkGJwEmGvJUJyaVTDluUJ32w5dpLJRCFWAXvJx3dSj/lwjFEB3q4e+NHAzD32d9wMon2nYCTsUIckU27nnts88xh8RJOotZbe+DtBpu0UE1JbSUIDgvbyU6Oesc4ljSokRs1yTzKcvI0ZPJNqjSKmsE1AdQWOoAlM9CwLY9oZDcaEYelNGqI1kZ2N6f+8OWVzvRwHmSqI6ZwiN2/MoLlgJUz9RQ8qlLxgWE0yqwem5v+ZKYfIZcPVVD3OdKgUcF/3tqUNxruArSnFZZRAH2Sc7KAGzpAw4VJQEh4IqhCwpFgHQawE3jaheEuBCUBbsK8p5VZtqCQ75elCtgN6KQfJKT2RA/SyB4XWqh0nDGYCKBZfWK9JTyhqzPK9JEt05f6+3BLJRA91+wOShTLUQGNhtX8ch/VX2Isk9uU/dPCbAwe7aHLpTLCkdFoFAN4fo5IqHPHDp5lQL5DAi2ZMRt+OI8lokr1bqLz2jKsGOeSzMivvhKx6HPMAwKwJ4acn2FwsxpI3KnnLX9pBgG+D92+OI0RMy78KzqsWseOyqc4H2EEkKm9hqR694XqDnUChp6e9ujgXKBbevvAa6PbLKlAPy3znV02xSeX3zSkVMpGTWV5R1tkcpHTUBdZkIEFeRhfcl90e1O6ZSWfqOQ0kwU12wG9OXYuiLVB3ZRQ6nGJqT1QPtAR9n9IygNCgMfZ3SJt/7NiZ5KIFqDjcSkbJU9rlS3munAsUiDom3wYjh9VFe8b5mQ393c0RWt7oMsKMkI95EtPtiYJ5zh+f2VA4SZBGQfPFx8BF3+wLVJQdwQe9CnuyY10y5YN1Idc0eyOyLz0E50mI4zzmkSXXpNZJl3W53PvM5lhZDx2bOKsi9m7mLUwTKaHNwj3T8fZSANwLFJqSkUkBvc7hNBowczFTXAe2c7Hq3p3ZVHpoprLqN9JkLPjZAH6xumZBarNkn2U/Zw+mNQLlIfutthFkznMhsAppDjfrCIRQEd65KYUT9Ti4/hPYl5jpLi3P3ZXnt6I1de7iRRLFwaA/5OVGf92gPC3rL0/umr+QQ8DTIdAuJkNJjixWBXxSU9KqePm+zTl5sKaBa+oaLhUvXgBxUNDkgqHrRoj2qYFYWdmZglhZRyySjdXC1HrIEmOxKia4bm6ErVNqAYOTQAaq29GqGIkmXLYGinWgU1z18rclqr3tKYTcWIoFJt1dCbWz6LC+gGvo7ZUupq6908u4qyNSnKftXJKs54VWJwdssLKW1HAIkltCowa8quiR9dR6xMkOCLITg71v2T4qmIyOamDzRW4TJiJ5RrSDWW8QxmVGx2jIYSCkyrT4MUthBahvjF8N7fhYLdD1/pjUZzlelsAABX5akZgltiGlBmTywOfhHiOTZlLvwnp+9/UXDfH5VpLDf+2tgLP+XXwzGUv75d/nFvdW6x3/79X+E33/vD+CjgAsS3//kNVwuxviH7/8zXHy6j9sH13jv7c9RHza4d3SJq+UIzXWF7baEc1wpe6fwjfd+yvqGZxOc/4tTqOclTg6X+Ojf3MbFvzlGuy5w/7UX+PKDz/HWjRcYjTq0z8YQOmA2aSAe1TDTHs3nU+zfuwIEYD+cof0/TuCdBFYGX333M8iFhjIeZt7h1p0LhF7h+NY1ms+mEJ/WuP3WC8RFgeP9FaKTsNcl/MYgrA1mswZFbXH7xiXiJ2O03z8EXpbQxsP8uMbtBy8QGw35soB4XOH22y+Az0b0WHoBNbEI/2QfzYsRICP8yiA4gbAxuHnzErNZg3BV4rWvPcHkdA2UAfF5CTFyiNepS2ns8O2//wPIa4P/7D/4P2F+WuO7f+cHUJ9W2D9Y4/63HxI071k8eO8x/sG//49ZpaEi5PMSZ69dQK0lvvatj2AeseaheK4hRw7FUwPzqMTb3/sY5mGZGKGI6rGB0BE3v/UE5eMC5t4a1U9KHN6/wvRjzVqHpxXkzELe2qI4VwSKM4vqYQE4geJsC71QkHc2kLdZffLv/Uffh+wE1EIjjhz8gYU+blhLcadBlBE3vvkU09cW7BotA278xhNMf1gAKsJcaYSxR/VUo3qwwOzdCy78323w5nc/Qf1ZAX/Ss+Py3ob7YSV+43f/CGHmMH3rCuG4xzu/+UkCCUDxQmP0nZewZxbv/96PEU866KVCvNPAHVvYI4vj33zK3s8HS4yeSHRvtAhjj1gEjH5SQDhg8s4V3N0W02+co3xnAb0V+N1/8M/gb3bQGybJZn/f6DON7tjDHjuEL60x/lTD32vhRwGznyjc/RuPMfvt5zDfuEJ74tmbKiOqZwr+fov9v/uEyaefaJh3l5h+pPD1v/cj4L0VmvsdZXtil5gaTMThP2da7pd+/wPMfqxh52RL3JsN+jcaHP5LhXe/81MuAr9+he7Uwe5xIm/vdhg9lUxffm+NYiEw+0hi8jfPUZ2zu1U3AsUiJTq+09D/9b1zqI7DjsPfeYLmtqU38p0t2hses48k6udySP69+e8+wvbLDXwVMf+RgPzNK4w/F+jea7D5zQ3a9xrUzwWKa4HR7z7H9nWL9jQgfGuB6iVl8sVKYHu/R/V7L9DesvDvbuC/vUT/5S3su1sc/qHA5Peeofw755AeEN9coPrGJepngtLsVeqTPPE4/buP0H9rBd0IpuAmn+rBH5G13J6RkVQNZffNTY/V+x0ThCUZ6vZrW+x9BDQ3PBnYN3tUf/sc/Txg/wPKspsbHgiA/Y0V7vzfFup7F4Ov9frrPZrbDuMnQL8f0ZwEREF59v2//zH6PSbi2info9fvdli+32PxWy3K67Twb4Dl+z0mD4HrLzu0RxGbOwHFgqD36AcBi/ctqguCweuv91i9HrD+7obVHDXBwvJdi2IR0RxHLL7bolgBq3sB088i1rcjDn7s0c8jfEGwu7nNPtJ+HnH5dQ+zirj+qsXRH1pszyK2NyOuvubhavpqx48j2tTl2h5F9FPWbURF2W9zFtHtAZdfCWhOgev3HbanApu7AdXLiOnDgOUbQPefXOP8W45JtzVw+XVe6+u7EeMnARff8Ax5e8BjdvDHDud/k8cVkhJvOxFYvM3tWr0OPP+Ox9X7EdVlQHkdE9iLuHwP2NykR3N9lxLw7RmVAFfvBbgaOPpBB7OKePkbHiIAL75r8fIbyVcbybJffiVi+Qbw7G/xnO1/6NDtC6zvCjz7TsD0kYdZR2xuCyzeAuqLAF8JXHxZ4OpdgtbFfYnrt8mWlwsOBcpFgN5GXLxPduzx7wS8+LaHsmTNL74qMH4ehkFIFMD2lExoe8jH0VuqXBZvEuCNzj3KRcT8o54Wh5HA+EUYukuzd1u6lK7qIla3Ja7fkonBTn5kw+N0/mWFcuFx9UDi5Vc1nv6GxJPvUlp8/hWB+jKgn0iMzj2ef9NwQNFF1Bce5SpgcV9hdYtS8OaI4VtBC+guYHuk6Ec/1HjyWwrFipLocuHx5Dsa00dczC9eN1jfUnjy2xr1OeXS2yMO32aftpAeWL6mEZTA1Vtc8NuxxLNvG/QTCekjVnfopVRdxMPfKRAFGEi29HjytyroTcDoaYvH36lh1h6LewXOv1zi6kGB5lDj+TfZMbk9Urh8u2TI03mPyaMG2xON9U2DqAW2Jwp2ojB+tEVzpLG+qREKgebEIBiB5qRAKCSKhWVP5sLi6kEJs7A4/8oE5UWLxZs1VBtw8X6N9e0CunG4fqNCMAKjz5YornpszwrorcfV22PojcXmZomgBdZ3KqithVm06OcazY0azUkJ2XpKccEwpfEfPUUsJPTWQq86tKc1RAQW70xx/o05Fu/MoK9bBKPQH1QwVy3sxEBtegI5F+jvHBmoVYvNvQlk7xAKDT/S9FlqCV+bofPSTwoI69HemsEejBAKje7GFFEyKEpYjzAqAevg92qIph9ClMSmgbxc0Z9pHfrbBwSGZcGOzL0x5HpLULlpEOYToLf8996ETOaCct5wMB2kvHFvgqgVYlkQ7OVuy65nauymQWxaPk9ZMOwnBfTEvocYjyivdQ6irimZTZUhKAxQlmRJYySTqRRiT8ZSaA1ZlsOXKAoIpfhVGAij/+SXkvxK//4Cg/lLehNI70t/xV+/KLdfasayeuNWfP+//4e4XIzhFwWZLy+AIkAUAeKygL6xxf50i5c/PkJxewP76QSqw5Bymjvgbr7/HE//8IyskoyQS40w8pBbheJKIhgyIW7fodxv4R6NGUyTJqJ+FCA7AbOU6B80UI+qQcaUS++79xuE8xLVC4aohILMQX9GAGRf7XkLgD3hz/OHox0n6eKbDczHNbvmLgX6aUQ09GeWlxLdQaB/8X4L8bQi+5Kmv8WSiyyzSgxoYhx8xelnTr2VTqA658Q8Sgz70p4E1K+tYH80g7/fAE/YE7j3Y4HNLaA/9hg9ZAWHCID7xgohCJT/YgLZU2rm6wi7z5Cb9Vs9xj8pyFoEeicvvmVRPSrgSy6c9YbPv70ZUJ5LbO9bTD8waM4iZbDTgNEjBVczPGf6U0XG4Zj7rFOgi+oEumMPYQX0SsDfbzH9pzVcTUklgsDoKSfVzQm9le1BBCQw/YR+o807HU7+nwLrW/RE2gn9gosHMVVhKNi3t/CNxt6/LuhzPed0Py9i+hkTQPWGrGF74lE945ul2QLLL1mopeL1tKa0bntKP2qewustpYx7HwHX75KNzJUKbixgR/TcImYGDui/vYL8wRTlFdmlfo91CMIDF990KF5oiCBQXSSf6+ciefPIIpXX3IbJZ8DqPjB+RAZQvL2G+ldTTD6PuPgat2X9OiXXsgOKFa9RgqG0wLuI6A4FNu+1OPm/CmxPyNq1x2Tf9v844PI9gdFToDlJlQqJzUMEJo+5UO32gemn2bvK10N5TdYkqh0zqDcEMSf/Hxd2V+9F6LXE5BGZNF8D+x8wNbg55t+2R/QaT3/K87J4Exg9BdojMfQpTh4StF29FzF5SIl5KCMOfhixvpmlnHxtTT8BmtPd605a7sf2jJLd8pLsmfAM4un2BOoL+i55bAKiBvZ/yLRmJv9GlNdkY67fAQ5/QDbJ1SmcqGLwDCKllG7M1OvmmFJON2LwTtDA7FP6FZuziP0PIpb3JA7+2GN5T1Heuf9FX6+IZJHMkpUhdkKGce8TJlgzcTp1Qs6B2aeB0twRWbP5Rx1efqViQFbPKo3NDUpOt2cC+x8yZXT1WpLodsDeT/nYo+dkKKePPPqJRHMqsPexx/WbCns/DWgP6CW9vq9RXwRsziSKFdnDKATWd9ht6Wvu9/WbCsWCYGr2Md83RucBzaEcgpD6PXq09z7m8SMLmoJrHJlHSliB/Y/6tOg3rCrpd5Uidgwc/sjh+g2N+ccO6xsK+x/2uHivhFkzoffyHYXRszikKIuY2LIUlNTvkak7/JGHHTEJGZGSVukoZ+0nlBk3h7wOVc/HywE/61uKfZIHlDmPnkYm+O6neXOkfHf0IqB+YdEeGmzO6Lfe/9ClgBzed/zMoTlUaA8ZAHT8rzusbxdoDwXGTwJ0S69wfU7f5Oq2RnXNa6rfS17kY0oS5x9bdHON8pKpvO1B9joCxcpDdgG+UticKdQXDMdhzUaAG2tsTzRmn7QIBRNe+wl9p66WqM4tQinRT1SS7VL2Wy4I9ru5Qj8TmDymx9LVTGBmHyp9rvOfNFi+XmPypMfVgwLTx36oy9Abhu24SgzpzjltV3UBzbFBsfTo9xS8EagvmcxqVg7rWwUmT3q4WqGfKYYVTRXmP15j/doI0gPlpYVqHdzIoDvQQ4iW3nIQ4St2Nuqth51qlOctIAXWd2qYNetBqpc9ugOD+nkL2Tos3plh9tEa29v0cZqNh2oDmiOD8eMWbqyHBOHqvIVoLLavz1AsLbyRcGMC3vHH19je26OX9nmH7qiA7CITkkNE+WwDESP8uEBzs8bo8y2Wb0yw98E1mttTVM+36I6pwKqerrG9O0VxbQdvZn9Yw1y26A8rlC953+Kqpady1dF7Oq/hRhp666CWHaWvFb226vk13K0DqEWDWOjBixmNQr9XAFKgfLEFQmCy7MV65/+rC0QlhqRdse3g98fQ5yvEqvgCs4gQEUblADpl2yOMCgw1IiFANASr8H5IoIXREOvmi2xijGQaWwbziOUGcW/CQB4h6JvUGnCOftD1ln8zGTHsqLdAYZhWvFxDGEP/Zm933s22/WJwT05j1TqF7wQmzKZaEVFRmhOTxxIA4mpNEJl7Jn3gfqYk2OgcRJbK/hzPJIA/24eJL8pmY9f9qff7826/CIzldH47fu1v/9Uzlv/kH/1X/9b3DfglB5Y33tuP/+H/9PcgEfHp5gDrvsTVtsb+qMG8bPBsM8VyW+Gdk+c4q1f4ly9vo9IORlFeem9yie8/u4NSe7RWoy4sXj7fw3jeoDQWi+UYvlE4vrHA5WKMqrLwXsJ7gXHdY3E1hjQBpnCQMmBad7hcjGEbA2k8glVQlYPWHmXhsLwYo5x2kDJif7LFi8sZitLC9hrTSYPL8ykOjla4upqwE3OhMXl9AesUpIzYXIwwP1lh2xbo1wVgJfSsR1FaNJsSk1mDtjWoKovV1YggWwCq8ihKi7rscflyhsn+FlJEdL2Gsxq+VdC1Q0wexPGow2ZTYTxusdlUeOfWM/zww9s4vX2F54/26WG1CrcOF/jsgzNUNzfoO4MYgHrco9kU+PLdJ/jJ+RG6tsD4+zXcby/QrCrEXkKNHXwvIdYa5dkWUgY0qwqQEafHCygR8fjhIRAEqsMGIQg4qzAad9isKsRGoZh36BclYAIm8wbr5xOoqYVvFNBL6HkP12iglygPG/SNQewVZOUQgxi6ROujLdrnY8SCsq1y3kLKiL1xQ/nztgAWBnHiKK01kb2iEZAjh7AxUFPL98uWzz35RGPzmkesPOrPCrRvpDfBjQZoQYRsWXgfRw5oFNMVrxXssUW916K5rjA/XmPxcA9iv0dcFBCdoIx05rkdIMMprUB35KFXSQp6aIG1RlQRcmIhVYQ/Z4eralO3Z/ZuJV/s5GOF9esesQrQFxp+Qt+tWijgRodwWeDGg5d48ukRhBWoXih6GnXkgCWl1EZNSWp3kiYkOkBsNCYPJZrjJJdNflWAwVnNDT/0xMprA9WxIzNqPp7c7xGuC+iVhBsFiEAJKCIDc9zrLfCiRKgDRg81uoMIP3P0KW8lcG+D8PkIqqN3VfaU5UpLP6YbEczmGiL2twlsb3mYhUxMqRiYzKi43aHAEGAje4H2psXoU4N+Pw2NFnKYIravdVCXZgD6UacgI0mA6urUWZlkf0yypFxabwS6+x3Kn5aUA45j8lYTbAFANKxUak7jK8/BfRSe/lMRKF8uLhS7L3uGda3vpc7dkj+rzoHVPR4TEYGgkpS5SPcrGETUz8mSVi/lMBTLnapmxaGbtAwd09v0+JZDDlcRgOeuX2AnfTQbejmFI+urG4H9Hwdcv0G/apYlZqmwr1IvqcbQmelGIDCacthQLDh8MGv6kH2d5LuGILm84lBj8og1SarndZBly4gAJMHh5iZZZL1lR2aWynb7BH1Zuge5A+5RcrurixQWNRdDTUZ8ZXinG55T6ZPs95VwJL1lndDqDqXY0hKoSktPsE/XY35tsfM0DhLBXF0SFAFVN2dYGY8ZWUD6V7nNqkvVVUk67EYMgiLYTj2YPqcp76qpdMPH9iUHJHbMMKtuTrksJa3pb9Oxy8FBmTXN+xBlSmm+DljfVEOdE9NisQuW8ymsbZVkx0lWrlpKbF2VPNo+XT+RCcxmi9QPyX2ffk4J6vaEg4hcVSM8ty8H+Zgt/b1BJxmtECiXDPdBxAD2o/xiaFmUIlUlpeMrBYFXxO6+WqC6sAMoFiEDYHzh8TIjW15YbG+Ug1ycB45+0XLpETSTXIuVT3JVJvvmyqb8niM9/dhmy0GYTFLbbk9i/MTCVxJuJKG3KWNgLGFWPtWb0TMt+wjV0++qul14kXQR3Z5Cdel2abgpgdbVkkOP1HspIiuBhq7VVLVTXvXwJX+fE3PtRKNYpLoTHyBcQCg1/y5EBCUZGOTi4P/MwMmNNfSG3Z3SBV5bPiBoeldVY2H3SoYZdQ6yS72UqaLF1/R2ShsgNx29kjGiPxqjeLEBtIRcbuH3x4AQEK1jiu6o2KXICkplszw2agnZWMRCQzQ9YklQzNcC31jktqXX1CjIdTtsTw4BQuC/Ebk/UacXk5S7AKIMIhN4hfdfrDQRAhjVZDHzfQe5L2Wo0XkCzpz+6ljjApt+n3+e5bAhJBYzJY/lSpWfrSP5824hItr+z7/fn3L7NbD867/9UgPL8rU78ey/+S+gigC/1hC1R7QSr905x8NnB5AqovigRvdOg3hVAFOH2CrMzlZYXo4Z4lJ7yMJDf1Zh9tULXP/RIcxKwE2Sd2QSEUcesALFlYKdB5QvFLoTD9ExEAURXPSOU7R5EciadgpqLeGOyMAFFSG9gKsicLeBtxLyvICwya+2klBvr9CcjyAbCb0R6A89ZCshey4w2zOH+Y80Fg8C6ucSzS2PqCKKC8XKiS6FRZQRoQxQazXsT3El0O8xPKR6YmAnAX6U0nNd7mkDupsWcq2htmTUguFC1s8dyicG/YFH/UShPQn0AZ5amHODYCLKc8lQlK2AfHuN/vMxwUxBACIiF+UIQHvqMX6kCAQqFq5LK9DP6WdjyTZ7LqsXAv2cC1MAgAD6vYDJZ3JYTItAFlAEYHPPoXqqh2Ly9ozPFRQX5uykzAycQHPCfbdTPpZZC9hxRH/iYC4I3kTcVVTkUJy8EPIVty3oiOlDBhgJz4Xk8g0uZsyKAKw7jOwjLCgb3NwhuzcsjgMXulliWCwFujk7ULc3WNVRvZSDNyv75KRNnjPPdNz6ZcTiLYKT4jqV0J9kb6L4AvNRXZC5MysMNQf9XmLE5wlYKPrtlm8G7H0o0B7yWOd6i5wuPPkMWN/hgqy8SnUIgRLIXM4NcFtzz2VzwhTebp9S2elnuxCoV/1T1WXE+g5rKAAu3EdPk6/I0HuVvZ26SQX0U6C45uKsPaCXzY14PKRNjJNnNURzkrpMU79qPyOICCaxb0uCiVxfIN2ua9ZOKAHsZwz92PuYbFB1FXD5Jfo29WbnCXMjYPQkYnWP16Ju0vHYcN8y8NDb5PtLnYe5kkM4cCCRqiVkz/8Lz5RbgNs9es4F3PaEgMmsmVhbXhEwLO8LjJ7tFs45sbef89yINDQolrxGilXE/octFq9XlAG2BA8ZiOQ+2mLFBezqjkwVEzwfOvnbchVC9iVOngb0Y54LmxQMGXCZdToXbUzBYOn1vt0xt8U6JcVe0b/nSrELbkmdrjnhlT4wHkezjejHvH6qa0pnV3ck5h8zHdZX3PYMLvoZAWqUYpeOm6oQREi9omOZOit53OyY+523JxiBYs3Ki/weonpg9NKhn0q4it47VxIADH+ffIW+4M9FAKYP2ffZHOTajoj9P97i6p0RlI0pJTsFHXmgPnewEwmX+j+lY5Ky6gJDa0p+hpgNfZfVFSXo3Uxi/NxhdUujvgoIA/hL/tIERIRnPQiA4bVfLuiP6ycS9QUrOrj9MYFCevcgCLiki8nnyN9LS7ZN9gQIPh0XehFZryE83weyv82X3D+zYbquWSdPnqP/MWiBYkVPIF9nfkiLLa8t7EQzPKuWvJYrAjXVhgQ4I5Nxpwq6CTBLC7tnUjcoF/bVJdN0q8se3b4haDy3rOc4YMBTKARM6lA1S4tQJj+9CwysUQRJvqCXMQqGd6k2JPZSD/3R0gX4vI8JdEVFf6NZ9uj3isR+SxTLVEESInTjE+PJGgxpI4rzDbZ3Z9CNh/CRVRshIhSKnsWphlkyldfXVCkVlw0QAvqjMdNdVx1rQABAskZEr3pEJRELCdmxYzQKAbW1CK8AKb1o4GcVPZ6LFm5WETyGiFBq6Kst7OGYdSFNTiwV8KOCclJBhlH2Dn5cQHbs0kQCYbHSfA0bBbVoGbbTO3obhYDoeoRJDSgCK9F5iBAQk78RRvN9pNCQq5ZJr1VBIJY8kuh5fMI0yVx7Sx9kZu2sG8J6hnqSpiPj2HbIKbJD92X+LhlkhMUaoioRmwaiqihb9Z4hPlKSeXTJ45g7N5uW4C4EAsq2xc/rsRTjEfspQyR7qQi4h5/ljsnEmCImoJjYSiCBxVc6LmOMu+TXn2Euv5BM+2fhkl8RYPn17/7VA8v/93/9NbD8S9/m75zE//h//j380yf3MK9b3Jtd4PufvwZjHKZVh8vVGO+ePsPGlvjw0zN88+1P8JOLY2yaAogC3knMZg2un85w743neHw+x960wdUHh1B3N3j77AV+8uIYUgaUxuH6coLJfIuvnDzFH3x6D/WoQ9saw8JNuAAAIABJREFUTMctFqsRisJByoi2Nbh5uMDFeoTu0ymq11fYXNe4efMST5/tU3WwNHjry4/w0dMTvHXzOT744Daq4yYNhiLGVQ8lA55/dgAz7xA+H6F+c4EYBaxVHAxtOPkRKiJaCVk7vqddM8Tn9ukVHr+Yk6HbUuaIiYWQEaHVgBOQUwttPMoysZxWotjrELyEu6aXUG4VcNwhbDTUUqO8v0T3yRTV/RXapkC4KAAFiFmPsNWAihAbjfLmhszfH5ygvUnDzL03nuPTn5yiPtmi3RSInYLcKKgbW4THI4LsCT+448RBLgxlq6MIP/WAjBBFgHlUwr3W0vs5tYhB8Dh0CkjgU84sOzlVRHG6Rf90DH3SwL2oITxw9qUXePajE4Q6cCjhBSXQEw+1VBBO4PDLL8lCpwAkWXqElQESwwmXV6cBQgdgaSAPeuzNNlj+6BDhVovQaAgToJ8VsPsO4+MtNi/GDERqFcykh3tRIxoCOEQAAcDMARsNvZCwxw6yJtsqdYC/LgATUTzT0F9aonk6gTzo4K8LmIWCPbVcrF6RfSwuFHwV4fcd5DVT+cLYQy0YI+8OLcRGIY495DKZ9BNICakXEy696dceCAJ6oeBHgdvtBcyVHIC4mwbEkUfxxJB5fCVkwp32qD8q0R0HhD0L87RgAE0j4CYBcr9H+cMazSmlk1EQePWHDAfydYTaJsbw7hb6h2PKq+cB5jItaEsCrZBYT9VyiCB70hXhuAdWBsWFQnfMienokYab8P0waMCP2dspLGA2ApvXOKzoD7jdYmlQXEtEGZmcu++gXxr4mpL03Au5veXJkmoA856L70t2rlbnEtu7DqIXGD1WaG6wSojMIFlZX3FgZU8s9LlBlBFmRXk+kMJ9Yu4VJdhpzgLDpTYKqhMDKyo9UFxzWODLmPo7BdSWsmtXk10EgO19i4N/rrF8HRg9owxTtxiCpXw6xr7idWLfbmB+XKNYpiReQzm2WRKcZUm7rziIGT0W2NxmWnB7lGuHIupngsE5aWDR3Ejn2zAwyFdJDjoDqnOgPeEworimVLd+HmFnghL4/dQPOsUgR8/bXJ1Tojt+vGPY3Ii+WJf6I/t9DoGEE8PwpDrncCHbCKJOgx23k2vrxIQ1pxzIAEB5LVI6LAc0/Zz70R5zG1Z3KRsvLyOu34mYfCaHUJj8uN1hRP2cSc4isMIod0iKENEdcDsnnwesXpPDdiACm9scmI0fR9gJWUPV8vjp1EVbXRDINSe7iqVimbtGWVEkAu8X9I55La+YQpuZw+nDgO0J5dZmHWE2HHbohsxxSNVd/V7qPN0SSIoAVFcBzZHE+FmgT/FQDkBX+Px3At08yXe3HExUi4D1DYXqKr03C1afqCRlziFKIpChDDqBc8eBQ0j+zKg4bOjmEqqNqX+U27W+JSlR7vicUQBuTLlwFEiJstwf3Uamx24ilE2BS6UYek3thFUum1OFcsnhxhAqVHMI1M0F6pcBuovox5T0FgsHO9W73taCQ4oogeVdhdGLkMKmdl2kzaFElSTz9TmBPSJQLD26uYJpApp9xedPXaau5OOaTUB7QPBeXTr4QqI5pgy3WJOllS5Crym/9SVDirzJg8cUipbSZKUne1leOfRzjfoZZbPCc4AVlEB1adEeGAL2leNQYk/zb/Y0iqWDG5G1XN0pMP9wi1Ao2CmDeXJAENUEPDc5dIesKEF4FCL1ZqYwqmUP4QLctIB0AfCRNTIxfqGaRHiysrInKxsKNTwP+0YlZO8RjIJetgi1YW/nuhsksVGIwU+ZgajYpGoSFyBCYIVJVewYwyRtHdi+MoH2GCG2HeKoHP4NKSG6VypQ0v34/G4AvrEud1JYkPkFsGMrvd/VlAD8f4w7iWyMQ4clYuB3JXfMpiWTKQqzA52v4o4/RRqbbz8Po/wqSGF/DSx/QW/la7fj2X/9XybJZ4TcKMQiQq9Y4o4IhH0LOAm5UilNjh+8bhJRPZcsIgcXkcWlQn9I+Zu720I8q8iwLXaVCzIr/NZMYMyLDSBP0Pnz6lygO6TUKwdfBJMm/2nxxxj6mDxGfA4uwrhA7PfIgkUFCEu5m/D8kAxlYvjS4CZXUkQVB5leSEBF9tgtULJkaEkJWE5WzIsX9o3FgaFjjDqZ0GiY8AkA9YsU0JGkTJQE7diB3J/oRjwWOSEyGAaEREUWMnsrZRo4kqVMpeMln196Lty6wzh0LWYmUvWU1Zk1hqqEQX6WGKX83L7kQjAYPj47TPm8OcWxWPD5gwHMkgtP3aS+zzJ71vgcZp3kZ3G3yNRbJmnKjoxQewRUF7vET7Pm4o+JjDsZYK6Z0Ft6x7KMa1gIl9zX/KGfUwB9nWRyTZJ9NVw05XMZCh5D1TPYpH4uUSzjsCBSLRekvuJiLydL5ph/NxLD8+epuFlzIa43SNcZtzGnV5I1AP2vNRehdiSG/XMjMtZRZxkimSAkqVeUKW3S7xbuMcnIpI+JHaTsitdUKhkX6TES6zEwnZ6Lr1DQD4gkZw2JXVMdr0eVfJyuSvLE9MEvAhc7MsvwAtM5VR8he+5D7ohl3H9iDWfZ75m6NBdkw7JfK2gM8kdfpmtaUbrnzY4N8yVQLsiE2hHZMmVfvW64YM1sp6sxMFdRUjLHxV7yinVMK2znPCe5gzFoLibze1n+GlIZ2zhI/HwphmqKYMRwXbJigfcDEosFpm+KAFQLJo4iMpmTLBoXv6aJQ4CK2aSi+zqdI5sHHbnwPqVjdmkQYFItRbpuc1pm7n0UgccoKqZlBiOG91KymUzz7GZyOE7lkiEt5dLDjjitF56eMVvvQF+WE6o+ARcfUS65WLJjOTB7AM9tUAxyyddfTi3txxLFOgxVF/n8AVmFwOtcRJ4nEcnGcT93So58PJFqOPK/eT64mM4Ma5QibUccfIhkT8kmtnOFYpOljUw0zX777CtUHWt+KIfk78yGH5TNoSboSa+nYulSKBETTmP6G2lj6lBM5y7u1A2squDnpHD0Kro6sXohpgJ7Hk8RAdnTgylCHGouoqb8Um/ISmbJaD4H+T1cJqZT5FRLwXRTZQN8mcHDjjH0pUwsK5NJdePgDYNZpA2QnYev8ram6zQBrOwLjQJASlKVnUdUEiGxeVACog8IlRqkmhDpvpb3jTLJUDu/u1/a96gF9NqSZYwR0ezSXtU21YhodjjaWQlpueiXjpJQESJk5+BHBaT1iRVOiaYqXQsuEDQ1FqHQgKS8MxQKqnXwlebrxiXgFSI9iZ6Mr2wcYqkgtz2iSjUd6e9l+4oXUEuIziOMDEGi2p3bocaj0Ay+6SwQ8AXgJlxgHci6A7RiomtL0BOL5P/z7CUemMMqAarC8HWTQZJL+5+2TTg/gKWY+hzhPT2IKQ2VCasKouWiLdecRCUHYCd6S4ZyqARJoCvEITkWWu28k6v1AOhEVQ1+SVHXuzedvMYPKQG26wfZ6+Cx/DmMJUxqI3wVlP7sLb9OnPuTbOMrstbo/U4W+6c8xl/49iuQCjud345f/85fA7D8334NLP/St8lbZ/G3/sf/FCPd48OLY5TaY7UtoVRAVVj0TmN9OcL0cIMYBbrWwDaGzBKA6azBalGjqC26dQkz6uG6tIpdaYh5j7AxELVjVQgANbUIViK6xF61GqKXiCKy2sJKslkAAW8EYLI5C2TcVETsU/R2LyGsSB+aAmHiIFqFOPIQa4VYpr8tPcRGI5rA+5oA9bxgtYYAZCPg5y5VaygC3j3Pmo2xpz8wAEieODFywLUBFMhEKYbPCE9wKhx9bLKVCKMAvVBwJz3QKabEdmq3DTc6qKcl3NTDHLRwz0epwkNifHuFzeMpFyu5CiRgqLtw4zDsf/bNxZmFflEM5zmYiFhGxMpDXRpEE3c+uZ7MjuoIjPzYozxX7GG09JnlhUN+LL2W9D9NA9SGzI9wBFn2IECvJOyRhVqkPief33ABd8CQmwxQpQdkR6YtVBHwgOwkco1IXrBk/1r2arlRgFmwLyyqiGLB2gwAsGc95JWh7Dexdb4kK6ZaAWEB6SgZLi/l8Piq4VAie8JyXUmoA/2bSZLtK25L1GmbwOCnDLizZ9OsmLwaZaQkvIzQS8lOUbnbrwgyegis38iS6yG8Csnzp3iOpRWDD64/8Bg9VrBjysSBnW/Q1xyM+OQJ1U2a0FeU6HqTjmvyJranAaMncgDxGXz380BmUSMdOzJ6ei1QrDgIAsjm+ILHQnW7Y15cy90AJnC7VCcGea60ZLDsNHDf0naadfobpGslVSi8CjTysCVqXoOZoZUZ9FiCxexTBDCEj0WJBIzTY/ndAEBEAmTVp4FLnybpKcjIjXeDi5AeK0t+81Aj38fVBL8M+cIgaxaRj62axBDb3X0zaxcMBmYzvwZFBqB99t3l4xjRHAnU5wTReptAUi0G4C08WcjyivePgvLxHGoTilT/kTx/lO5m+SlQn5Ph88VuQCNtTN2LTEreVSIkYAuCnCyfzSBIeAwewzyAyeAnD8rcWAzprjIdX7Pha8OOBVk0w4FJlgmrLrF77c5XJzwHNqqjLJbXAQcb0mcJdfIKCoFyGZI8eVcFIpNvNA8Msrw2D1pUl85xOpe54iTLU10lhmtZdRgqT2KqRBGBjB8k/9/NJMwmDvf3BQcEInDwUKxTZUolMDoPQzKraWICzjFd02TOMpjO+25HEqYJr3TcxqGLsbry6KdyeD0BGAAsXxtiGOZl2e3Q4amS/LjesZ15AAABVFeeUuU+ojlQqBZf9IXxfPK77ghE85CLgVcBdrwDxjw3kf7FNCgIhUjVN/Q02rGEbncAPwrA12TZslfxVS+tbgLciB5IX8ph8BJMGmYoQLVMyLVTRUCfn7eLw3YN3kkXd9vfeLixHq4lJE8kkK4Zn1+XSZKcQHSW6hKYK/obSzKYUYoBzLKiJbGJiRXkweIAJBQK0pNNzD8XPgwMYdTJi5iA8KvVH/nxglGIhYTowwCoWJGSWDYlILrMIsgBEA+1ITFSEpslry4MktlYGkpzlUidlYr/zl5JJb5QMcIfRgI5t7uWMojNfknEOAw8CLQVAW3envz/9Dcih/8ouXs8H/i8mTnMjOQrjOXgscxhO8l/OWxXBqQ5vCf/fQLSse12fsncS/lqaM+rXZXAzwebP+/2CgD9VZDC/ju//VcPLP/x//6LASzln3+XX9yblgFbW+BffXYHRhFU2k7DOUVQeT5GvdfCOYVmyxVeMe4RrcRo0mH90z3EXsEYj2rawW4KPLj9Arp0+N43f4gYBSana4iLghUWlcfBfA2hAtRCYz7fACJC7PVAxReKHFvcv/uCQE5Gyk2vDcRGofqswPxgQ7lmEDi9cU0JZsEPhurOCggC6iDFQk881FoBAdDPCy4+ewmoiOKzkiDNC4LKqUf1qIAcOahGQN/dACoCexb6hUlJqArCSlRPNGIvGUjSC8TaE7CaANky0CbqCGiCCXPFTkdVesixhdQBopNQhYc7toit4gdKK2GvK8QioHpiUFwqbJYV1FqS8RSAXkpgzyIe9BAW0EctQaOJiEVEeSEhtvSM+Bsd3Jxv9GolUTwzCHVAmDkukooI8doGJ++8HECl7Fl4r9eC/ZJrQRAlAb/nEMepVkNHAlyfWKG5Q3/Lku0uI+BYyVHfXyLeagnKARR7HZm2WSDrqQA7C0lySdAfTjvgrIPqBYorVkEU12QCiqWA3SOY8hUBRSgSs7RhGI26NAOICibCjvmhXlxKuBsdgWMCnEGzf3GXfkrZY3kt0B9QElY9I0AuZ93AsotAUCctgAjYAw/pmOqrOqC4YsBOd+wR396gvJKsTfEpFXQUobd8PrLEfNPPQCuU9M1GjV1wiwOkJzPPfsGYEpeTx3BN4NkeB+iWnlRh6dPMQLFY8Ht7yIWscFzI2lmEXglsbnEBUz+P7ABN2yAdAaNZc/tmP5GQjkCCCyQCY6Z7CtQv4rCIzSmoeR/ZZ8dBTAacxYLguXq565y0U7LzAEFnMNxuNyV4MEkayqk/uy6l489zT5/wwPJNXlf1eYTd43O+eqzJEgOr18OgimgPeZ6b4wjNtymmR6d3/Polqzfy/uUQFZXUDf2cQCAPTwhs0nHecAEubYRqAEgqJlzF4x4TMMvAXjgm4wYDditOKGPN3Z12lABoQRDoyyTt9KnL0QDzjzzBoI1D52Q3Fwlc7RYcZsW/eRUYDQx4JJhSzU5pkqWVAO+3ej0kH2cajCSQnoNJZM9zZydIQw6mmoaCjC4CYGeUMWb1RGawg2H4Tnv0il9SYQiqyeEzmfXOYEfma8HF4ZjSY05QPX5qeY2WOwWCHQmyypIDAjcSyfubXq81hrRxXxAQmi3llHay80Wv7okhsCpKpCAb/k636TWfwLirBdrj3b4NSoA0+FA2AfEUUCNTt2TUQD8hG59DakSgxDUqbp8rBVwthsftp/Sm9mMucqUn0AzMZCOo3YZhaCLdLh2XbF86f4WAHcnU+RqHAKl2T8KOObzKnZFkRslodntkrPPz5us9JOa2nfP587nKiqB+IgeFhSsFRk/a9Pw7hjGzsBAY9if3pJLlE7ATBdVGmLWDnarBd9ruEyS6Wib/roTsyShHhUEhwMEIWUS99vTgTtXgc41KoD00CIWEnTCllgA4dUama1avLfTWo9vnZ4yvFfqZhq+Sd3PD0J+YACoCoFqXBrMSdkZvphspsoU+Dgm0oZAwVy1iYoAzUA2FHO47AEcXBtbeTlNwTecHySlcgLCeIDWF2qiNhXQBcpt6K0dkdUWfwmSsh+wsfxaRzo8EXCALqsQA6oT3ENsOwkeCzHyTkl7OQiNU/BrAXGIfY13we2Yps9RUK4LCzGgWhiCvLoeAHDgmZ8P5IbgHztPvmTyRwrodqCzM7m+1AhTDemKM/B4CEChzHcCjEEx09Z7P7zxlsPlnGZQKufu/FPRSDrLZBDIB3lcpfuWfAX9mIuzAqMo/4z6/vv3C3H6pGcv6zZvxK//D76OzGtu2QHddYXqyxurpFGLsEJ2EujDwew5ndy7x7Ok+PXRlAMrA8J6CDBpMhFxoyE4MstjuzEKUAfLCQPYC7lYHcZkAnhVpUZsnSEgphxJ236N8rtHvM5QlAwR1c4v4cEw/21qjOpdwdYSbBYheQDoG9IQUCBQMJbpmkaS9gQuH/iwlXi752Dm0JZT0IkWd2QNum+p2nXeujtj7icT6dmJpNiLVApBJcuNIv9KCYUR8bDI0iPhCTUv9TA7BOLIlU5PDZwZ55IYLw1BE6E2qgCgi7CyieiGHVMfiWsDO4sBshVfY08wOEdDwWFKuxbAZOwXPYZ5MpwVQMGlh06RJnCI7OADNEf01xYILKV/HQSrdHSWWK0m7kKWlJia5aJL/JfmkL9NisIyJ7eDxsnv0ROUwkszmQOwe01c7diczRVGyJL055uMoywWW2STp74b3t+PdeVd9OjeJJZOe/3djspn9fkT1Qgzbazb8rpud/JMLWKDfB6qXu9RLO6U8m4t7DImoedGpevq0CJZ3kui8QM3n5VVWLCZAmKXSsscQkiPCbh+F30mOdfJJ7eRyGNiAUCRWJDFbr8qgg07HucGQJprPX2ZkhpL7tMDsE0CB5LnKICO/NoslfWWZIdt5+OLQnZf9bwAXy+WCC1w74fm1UwG9iYl14zkMyeOVGYHMWGWmSreUU3Kxy9+5UQrFSYCEEtsdW8/wkp0UNgf1dPPEKiXwbUfs6etmArp9BVwmdi+/tmICxEF9UXrKJMowBOdkX50veZwyi035J69f6dK2Innc0nWiux1rZMepNgUYJJd8viQTnoq0H2lBnvYn+9l8AeSAHL7essySz50X2tmTluW+qnuFtbQYfmZHBDki8jnLZQrsSUyoTEymtBGbM4ViSZ+dq3hNiSybE2J4ziy7HqTSandMiw09a1Fh+J6P2atJsFkanFmm7FvMr8fMWjPNmAtCltxjSMJtDiVGLxlWI/wO4GX2cEjl9bwOzSbsGLYspfaU3fYTDh2LVUA/kxwEJHCZ94Xvizs5LfAKi6h28mPZM+hGuN15yGyrtLtwm/wY+XHooyNDR3CCxGiHIZBpkMemIULQO3n3q4+pEpuXGWs3klBNSNJiMSyyKcvloEFlMJak3NIR0PlSUWJvE0tZUl6bk1QhgGJpEZSE9Azlybf8PiR9QD8zKJaUfg5UQQCDtMCBRFTp+rIBoVQIRkJ1/gupqcKFBGgJ1DIL6MfsoaSVJrFmiT0KlYJeW6a1+v+fvTfplSxLzMO+M90pphdvypdDZdfYxe4mmwRnE4YFwRIEaGdAC2+8sRfe2Ibhja1/oZ139so7r7wwLA+QRdCAQIgU2RS7u7qqqyrnfJlviPGOZ/DiO+fGq+4WREpNsItgAA+ZGRlxh3NvxDvf+abAQJ4mVnJEmerIhCXWL9oW0meAQUuUzab3eaMY1DOQCUwsZHoEISATAHQhAs8I0AUOUlkXxmtC4CYh654gL9NQNetG4MLhe965w1jGsQhGjYBR3AnmCXlGEJdrspkApbixeiQoFbcXDmAvMo8h1YoASNUj9IeS3QuZIeuYGD8VpbFSEhwmkJWks+Fwrl9JcfVxG0odqkbSNoUgmEyPu7LUGMYjtGbVSGQ+kTyYKbxHST4X4j4GSyCZjvluImysL/nK464E998GHJP09uvOWC4ehV//D/+bn/t2f/9//+//2s8N+JozllIEfHt5iceLFRdBpgN2mxKidCiqHvCALzzyoxbdoBlaUzlKVq2AmPeUc075G9cfDXCVh95H1koC6k0GV3rYiUfoJfzMIlQsiXcTR/kjAAQRi+PJWnYXA9NWQXAoPGBbwy+uvUKYWbSnZBiSlE92QHth4fKA/pjl1Wp/WIUNitI80TPFlV7KQK9kS4ZOOMBV/FNctJBDDP/wiIBRYP8wwMfzGyZhDD7pl56T+8qPoJKyRaa10mcaYO/3CCZg/66FcMDR4xXMhrUMw9LF7ssAOw1ov9Mg/PKWK+dVQHvfkb1UAf08UHbryOwkENFekHVNUleXc2LT3PMHP2cn0M9jCMWE52InAd2ph26Afk4/m4ySxu6Y73UZgayrAoYjhqr0cx6rcGIMbwEQwyo8GUPD8evjsfUzHlMCgcMioD31sFUczxnHnLJQHltiCtozlrrbisCuX3rYiq8ZpkDzbo9+wS6+oGNgyjzATljS3h3T79sd8xrpHf2vAEaWxVUhdnl6DLPkfQxwZZRGDkAfS+xdBnSnBPvDDICkl9aWGINO+hOP9oQAKKWScqWZY+4iU7Z/5KEaSk274wCfHyazKVEVILMEAMMRF3Zs3KaLry9uAppz3hP9jIsHXhNA1fcPrE57Gr8LbMD+IZlOEdI4cOLZnKW0ZoyyuP1DjguBA9kvU/OYuiOyI7YCth85elS7gG7JSVq3ZBfs7hFX4INgEEtzHjBMgGEmoiQR/A7KgO6IjE59LpnEqxmQomsW2W/fi0zhnGyY1zEIJE7ou2OBfpmYxCj/nN1hcVpKPO9WE7jI8riMHrX6nkBzLuCNQH1foL4n0R0zfTcBAp8B9Rmlv7tHBLrDTIyhJu0J0ztVfzgOb2KYyrHAMCcAthMxSmeHiRjrUUZ/qzl4Pb3CCIDshN2L3TGPWw2s/7CliN2bAu0pJ/3ecKGgn/G5IHkMQabFjoBuFkN5SoFhFsMyBCsUbDy25C8dpgL1mYKtxNgHub+n0JxI2JzPDRXfkzpAbcHttws5LuikBRqyquyI7JY8Hx99xUHE1FobRqlqN5ejP7g9SuCGYUVDJdHF54YZWdJueeis5K8ggXYpMVSUabYLgoGhJNhOAFt1ZHqaU55XtxCjhxjgRJ3PRW+y4aJAP5UYSu5DDel3CtCcqnGBqZ9KdHMueO7u65E1pVQzLgpIgf2FgtcC3ZHkcVQSzYmKrKUfF3lGsBoI4oTFuHjZz+X4PRSEgNlTototFMzOxe8TOfo0fSbjc6zfaJdk1lwZfYqSz3cLAjxbSLKwS4V+JmMiMD2MACB7dm/6XIzgCQDluAXHTSQ2Ln43+wgO+5lBd6ThMwE7VVFGLCkHVUmaCbTHGaTzsIWCz+ToJxU+oFtq+jkV2TQ70XC5Gpk7WyqIgWC6OzbwWqJfZHAZk2zhCcC6I9ZmDHODfpnBTjSGuYGdZhiOCtiJhi25fxlZv2GRwc4MU3EXGaWeEUgHzfH0paYHNJ43wZ6kFLbSCEJgmPK4fKbgY5/ksMjZd1kZuIkBYrqrNwrwAcOUgMtHP6Wb0DYjm4Hs7sSMz1H+Kgn2oucvRLAIyURXNzFw0wx2xv1CMClWWAK2EI8PSkD4CJatQ6hy/tvcqQQxiu/JWAciYqVHKAyZRgBhUhBUJg9jkokqdQB7qToksYsqXtcENBPIS1JY78ftj2E7WhGoZQYoi/F1MJoy1zGdNYxsJIIHvAO8g9CaoNJ5vl5I/mQmBggFpsXq5MWMioA7/kehDhAjMZJCyZ/4UWQ3I8splIIQ4mf/xPf87eMX+/G1vkLWSfzhq8d4uj6C9wJFMSAM8Yu7jymexz2GXmP15AgQASJ3MFUPyADfaOhiAESAuja4uCBAOvphXIDXlDsKJyCOe/ohtScwVIFdghMLf9oTyA0C9oh9kCL3oycuTCyEFwidRPmWABSthDztCBYnnr2BGtCLHm7uIOcDwabF2DkHCbiTAeULBZ97MjQTDzd1CDIgW7b8pdvHBEwR4Moo0e0FxEBZ6HBs4XMPV3i4+x2GY4JlX3i4KPezpwOG2WFlaZh6+hAbdkBOvlBA7uAzYN/kcFVAmDhkbymbFV7AzyzwJoe1TCTtTxxmDzfINoDeSbiLfmQxh/PhwIQNrF5QPVmTEP2Xei9j2l1c0Y9AGoHVIy5nuE93HOCmDgJkvYIG/DQC9R6QsYcyf6OQJKWJkQIAnwPZSsJlAWFmEXRAP6ccMfkrhyMPlwV4TbAmLJDfsnMBZJeBAAAgAElEQVSsekXfZmLLhpnHcOTGUnk7dchvyA4DQNCsEEkLANlLg+kTifbco1/wHIeFR3EVPZSlH0OgXBl9WIKgzRvuo1+6KG8UI/Navpbojj36Bas01ECAZCsGVMnuMDkqrsXoC+xOPMwtJ4vCcqxVvC6qFcjf8rx9RqZWdWRIhWU4UmJgbMnaDwIOMjdqKwlcj/zIQtqKgTfdqYtdjxj9Yfkq+h8Lgi+fERR0R4Jy3Z7HmPyJXvMYZewdTJP/bM06n2Eu2I0oyZalcKYgee9UzxXGkJMsjN2MEIgr2oCd0mdodqxw8foAkLNtYKKm4UKALXEIhlpGZvSUK+T0PkYWRnB8+iOCArMhm5KtgO4Eo8d3lMvF1fbE4uqG19VWGDvqUi0NAKQ6FmHZGalaLsQAkb3NY9VPFsc5ehx1rHmZvBoO7Gcmoi+QMlkXvYD5hos5SXKZfIXCE4i2x4cgLpf6KSPrm204VjYy4CbWv9gCqF77KE1DXNwRX+mblJbnOZQsqb8b5tTPItsZFRApbIgsBxlOYQkyGfbD/UvHLsUk69QNxlA0byiFBciySgvWl6RgmCgdRQDyDYHSMOH2XCZgKzF64BhQxc+HafxXxiddu/LKM8ApMpbpfnE5GJwlgeK65zWXQJaqXypKSotrSv9SrY0tBfJtTM4cAmbPOQE3DcfcK4Ixl0UJtA1k/xJhMn5+YjpoDHnKN2RmUkdlSi61uYh+Ti5S5KtDz6TLxRgQlB4qVowknyn7LiNTGqWb3YIAZqgkihsXwYyI+0pME2tQ0iLGyK5G5lnv3Rh6VbyuKQ0WlOl6LUZAHiTIUoaA4saP8lfdWLgYfpRCkdqlIrBV/Hx6w/PM1gOTaneOr9dc4Ejsouo9a03idw9/UWCUZbPPEWPAnlcCsvfoZwQgLicI9bmKzChGf17yQ7pSwU65IukNpaaqcbCVjNJvx9qU5Lc0BHiuYvKqLcl6ShfgjITPFfS6gRwY5mMLRZ+jjxLXXGOY6uidjYxnRsaUIW08Bkp35cjkAqBXMTHRUrBXMnkyNb2MPtaa6N0QFSw6BuT4A3tpPUJBqazsbGSSyfDp/QDRDQhSQsbtBXPHEJ9An1L0PYJAMsT9hijtFL0d2U4MFqLrx9eOIT+pW/Ingn+QQoHEncUKKWMIlSOgjImuIQK8kBsgAdHMEFRGljK0HVKtCFI67F3GMAQIrSCMGYGkMAbCmMNrvONr4g8SYy0FxiAdFWWwifGUMjKsdxhQpeinlPIrElh4fwj3cY4/IfzsH+d/mvH8uj7CX8HPL8jjay2Fzd99FB7/l/8dhoc9VOZgMgutHbaXU8i9wvyDFfbfX/LL+XGD8KqAPxkQLH8ZCRVYWj+xKH6c06uWecwebLF9O4V5q2Ef8EtBXWajFy6/kRjmHm7iIRsJJrHKkZXL1pyg2pMBMncI1zlC5VA8M/SsqQAdvZO+CKheSNQPfZxcxvTKLScI7cMBcq8YoLNR4wTFPugQWnomzUqiP6Ef0y0HlE8yDJOA6jL2Hk7ZhSkCYB7v4X48JUApGF4Df5BR9ucWk88NvU8LjgcME3cTeJ68FNi+yy7D+j69YbsPB2RXOkouPdgFKTF5xuj/7bsESNlKojt1mH2u0JzHcu2SEmJXBtiZQ/VMY5gHmLWI14QS1eKtRPPIYfJEYffhgMnnBu2pp6S4OoTjdKcePvcoX2q09yhHLt4ScJpdnFRkwOxpwOqXOHFKPYL9/FDBAM+aBWmB7TdYO1C9Flh/7FC+UmjPPMo3Et2SYTJmC3THQL4C4IHNR+zZTGDCVgG6FSjeMGafDE1A9YqTBVuQ6Zs+YYdgfkNg0FwwlCbJRVV7CFGp7wccfUJ2STcELwB9ibaIibQVn59/xglrc072W1oBl5HN07uD3M9Oue9hwkCc6RM5etdSsAdluOzyXP5owO3HBs05j2X9IXD85wxg8VpgmHKyX19wnKRj5YLZCCw/s3j9u4phPBlQXAWYGrj5ZWDxKdmqlMIre0TPZsD+AYHc9LnHza8QxM5/DNT3KftOsmxdc7yGKTB9EbB7GIvQn1I+Wd8PmLzkvd+cixEESMvKBm9EZPoClt8Htu8Bsy+ByaVFN1fYfoO/IOdferTLJOtjZ6R0BELlG06uu4WEqQOloZG9uvpthwf/jJLy3UOJySt+TtYfyDHwpHpDsFNdceK7faTgSiBbM2G3nxG457dkJc2eHkpnMDJ+wvP10gH7+wKTV2FMmw2Sn4H1BxLFNf2IzbnA8Q8sLn9L4+TPPbaPmCZc3LKbk548MhQuJphOXlvsHmimHt+EuIgTE1/rgPX7Mk7igckrj803JKbPwwh863OmJTfnAssfOWzeVZA9MLmkbEu3Aev3NPJVwFABsxcOq/c1TB0wezrg5lsZqkuP7WM59rKWb6ksMLswprgmqastBCavLa5+1eDkzy26hURx67A/1yjWHvt7EpNLP4KeJHHtJ3JkCb0WKFYO7VJhTDLV0XdpgNlTh/KqxzDRaE41VB+QrR12jzQmrx02jzVmLyzW72rMnjv0U4l+LrD8pMf+vuFn4pbhLcXKkTmcSZRvB/hMYvW+Gb1nk9cMY7EFmdf5U4vmVCFfM922uCVoqM8kzC5g9rzH7kGGyauei1FzjX7Kc8vXCWQ51PcM2qVEee2h+oDydYubb1dQPTsxvRHoZ5S5Zluyabv7BtWVhVkPqB8U6KcidkESNM6edeiODCbPa7hKoznNkK8shAtYfZCjWHt0MwkdtwkQvHZzFSXlHtl2wPZRPiYk91NB8N96eg0dFw1Uz7Rbsp8SzgCT1+ySbE/NKIP3WqC4tdC7AZv3Sxx9so/eRNZLdMc5issa62/OkG15TabPW/QLg2FC9higrFfvBtipGZmx1LWpa4dhppBtLMxti/rRFPltj/Y0Q/V0j+bRBHrvYLb9yPLtH0+QrSzk4DHM2a2Zrx2y6wZ2nkOvOgzLgr2R6wb92QSpZmOYawgbUH65Qv9gDr0b0J4WKF/u0J1VcIXC5Is16sdzFG9qBtYoATfJoNcthiWlJHrXw85y9AsDCKB83SAoMo7ZdQtfGXTLDOWLPRfASwO9o+Q0KKa8dqcFzN4CHjBXO7QP5yie3KJ5/xjFZQ07zWBuGwQt0Z9UyC938KWBnWUIUiB/vYObZFDrBvZ4AlX38KWB2rTsuxQCqu7H2g03ydiNmfGesbOMCbmaYFZGGayse4imA0KAOz+CGOiX9LOCstlJDtEyuRYekPuW/7eu4WcVZNsTkJb5IbX1Tnqsn5SQ+4bS2dhPGaqCrzGalSKZoVS2zIGup1Q2VomIlr2foukQJiVE3R4mwUaP20HbEVgKwaoQpRCKDGLfMJCnLAhgYyosQJnrGKRzV0YL8DXOQRiDYN3IFIYIlJkw2xE8Zgaw7DkNd32XCV8IwYqQu2Az/MUBYvjJ5FjvfvYL/wKPXxQp7G/83s9fCvvP/49fDCns1xpYlh8+CH/3f/pH+MGzC3z08A2u6wk2+wJaOzgnoZTHg6MNXq7msD+cwz5uMZ832DcZlAroXlXILmoMvcY757d4u53gW+eX+OMvHiM4gW+9+wqvNnNs9wUmVQclPTa7Eo9OV3j6+hiLRY19kyHLHIxyWK8rFFWPIhtwezXD0ckO3aDR1hmwMghTB1gBtdZwM4ds2aKvDWbLGtubCbJJj/66QHleo2sNPn54iR989hCQ9H+q+w1TbWsFzAYUkx7NVcXQndxBXDLQR04GlJMebWsgRYC9Yh9KUARMIYtS4EECOrB/stWAo2806ADIALVRI2sV7nVQzwvYuYdey0Nv3jt79M8m8POYZispSy1eabQPBojSQdxkcbU5IFSOybk5vxjkrQEkEI4GBgo1ikzSxQChPeRlDl95qK2E2Qk077APU28U7Iy9hv2xiwxSgLASkwdbtJ8sYOcOsqNs2BdkHNVOwp4NPP/CwTzN0Z86BtPEVZ+gAbPml2h3zwK5g7w1DBkSAWajMCwtqicGzQOuOJuNRL8kgA+n7EjM3yq0DweojYJqxcgIpeRalwEp5RSSAUbVE41hwQl7cS2w/cAiu1awE4YbwQlAB6g9J+nDmYUsLMJtRkb1hkmu7aknaxgwdhgKDwxLB7NSZK5nnl7R6D9LARM2gj47IVgXFtHTRfDZHzEBV23jfVDx+Is3ZKZTbYmwACRBa3vuMfuc3sLuhOmpvCcxss7ZhqCwO3coX2jYmL6arXhNmrNAT7DgMcoOsDMuzPRHAaMH0PFYyaQwTTV50II6sEfp/IKOjNUgRk9xCjXR0bua2LBUczMsCF6ZuhkTW6MMOSV52gnHsV9wQSNIjgf9iQR01StKpelJo+yX/jeMgTq7x1xY6BfRM9zSa52Cg0YPqwATUA1luWbDa5ut+f/J65hkpMOMLHnxVo6hLKn/D2AAUHEdg1zcYUGBdS8Eg2ZH+W4KOGKqMNnzdFy65sKAsFwUGOZiHMuvpMrGpNH6gos0usZXr1sck5SGayexqsaQAUye3CD5fJAx4OfOOKWKITmwT3GouODQz2OK7PEhmdfsA3aPgeJt7HQsEPsqY92KOnxm5MDxHaYidj9G4BlDU5KvWTjeA8Jye2YXRjYsvW6sGImAyWugvAmozyT6Ob253ZL3avKXZtuAdinGrscQE1YBsnHSHhi65HVNjKutGGgUotRc1+HAjiW1qeQ1JuPM/zd19DTHeWb63MshjGmr+3OFyRuHdiHHOqrkYbYx7Td5X4Pi/QFEFnXtRz9tArbOMGSoXRKgmj2PJaWwBomxixHA2MeoY6qttAydoiSX8mDdhdGv2i1in2Ht0c8kilXymvK5+lSR1Y6eymzjsLuvkW/5OhVDp3z0jqYalFRFw88hmUWAPZL9TI31MynhNqldso0/pOMW4ivprWTagOaYxylTAmvvoaJn1eUE1CnkRkZpLO+vMO5PdR7DXEE1nn7TPsDnArr2UI1Dcy+D3vuxjiX5Le8yy0xH5g1jdkyjBYD8pkc/NzA7i/7IcEwiKxsEYHYDfZ8qdj/mCrJzGOYGunYxCddhmOoxkdlsBrhSQzUWPsp/2VMp4TIJs2Ewj52YkfmUPStMUk9j8jvKIfp7jeIcYT/A5xqyswe2ONbW+ChvHYOBIvuZuil5cTzcJIeMYFTd1vBVzs/94EbZLA8ikOkEKNVNqbBSHkJ7gPF50Q8Ik5KgtRsOnkvnKJ/9CelsqqcZ95Vek5jDrsdXakGSDzIBz7vM6R2sEEL4Khs59JTLRnAJSS9lYiHHc/I/XRUy1pTEXsxx+/+mhw9/IzyWfwssf0Ef5YcPwsf/5D8HANxezgERILcaIWM1hdgryk4DIKYW4jqDSiXsnYS/10G+yQ4TPhNQPVPojhk00y8ZnGMrJgEGFQNeBjHWKPiMbKXZxgqDOkocvYjBOIHeLnD7ZivRnTgCA2AMLEmTX5eH0UeSZGSyOwT6SCuiPFIw9ETz2Nmbl6R6kT074oR3TOy09DYiiJiQeQgc6RdxsrA/SDSZTMlfPGTjuL1hzmMEgMlLMnjTLxT7HxVf0514mC3DiVzlUbxRY/iN3scvqujLSWEoquPf73Zf2kmcgJYc62zFCTy7QeON4ON7BcZOx34RfU7+MEEKEvD5IYTGZxF0xF/k0hIM5NfsIFVtDBJpMQZepDL0lPKYahhSSqTqDpNI1Rwms66gjJHpifH0/Ve3k0JmuuMwMr/F2yhBja/L1nFMGnod9T766+ZivG+CxBiykyohsjVBS3HNDrPk9cpX4cAqOk6oixvWPpg9Jx/dUfQKldxmtsFYP5CkgmbD7do4WW5PxBhgIwdO9LujFKJx6LMcptxft+TkXQ5MDhUBYxVFCpDJb+M+KiC/CaNnK4GSPham+wxjOE0Q9LdVlzym4pr3eX3BiTmZrMOEO6gYuBMLxocJJ+26TsdzmCCy6oEJt+0yHtdtkjOSBUk9jwnwJi8kpZ30rg0z+tlSf2iSZ0LEseoC9vfkeO8lljHJI6U7yGHTNnUXxgCdFHyU/G7FjY9+OzGCDtUhShExJm/aksxee0yGS/V8LqWtpqTY9PlMCawJfAvP+z7JQVM4jI4MkW7I+iTGSA309s2eOzTHauwNHaKPNCgCiW5Bdj6dYwpmKm5iiE/N0J/yxo/BOzad+4rPDRV7RW3BeguEMLJaZKwZGNNFAMB/Y+w6TDJXl4lYlwBedxsDbRr66vuJHENgdHdI5U1M7t1x7ieUlBerQ1WGu6NIAyjVrc8lvy9isE7qCE3XR9yZt5o9r3XqciUgpJR+KNnZOUy4/xRCZcsDQEpAUThWXySmvbjhhF8EpolmGwIJF6W4qYM0gcgEtlR3+Nyqjix+tiPL6Er671Iv5lizYu74P1NHoxbjdzuQFpTid5shIHK5jNUvnpJLTzmoy+V4HfSeUs5+rsfz9IbgbJioUT6agpgAQHUOtlAwO4v6IkN+yyAV1TiydXd8jqmnMn3HpPMYfYcDZd2yi4CqJzufgnBsqWG2Q0xiFWP4Dc85LhzEbfB7nSemGvZH6nqg/7LSUIOHVxJ616Nf5shuOwQl4UpNUKbZn6kadzjegWmrcnBwpYYYPFQzwFXZIY019mCOvZQ9fX88b7K9PqcPUVoP0Vu4SQbZ2VES6XIFve3oc/yJgJ+xazL6Hl1poNcN3LwYw3mE5bjBBfjKcExid2UKEBJND1/lBwCY/Je5GqtFxODiwo8/sG4Ag4HauNKT6j0iyzj2WmpFtlIIiLpl/UfPHs4QE12TxFVYAkE4h1QLkgDiCOSSJ/JOKBSiZ3OU01r3VdbyLpsI8N9J1joM3EYCoo5sOn4WwwgQKPpwYBaTBHX0d8a6ESH49/gIbXfwi94N74nv/1keyb8UDvmbAiz/g//6577df/5P/4e/9nMD8PX2WHovULcZ6jaDmfaACjj55jXUsgOcgDzp2SmoAhaLGsLSoxYKD/GwgVQBPklCa4LA7jjAzh3rBGRAdzHAHQ8MfMm5cmi2TI71RYCfOPRHHm0MGmEhPTsl+7lHd9/CmwB7NkDVgl49QQ9jQEqxJBBLvXk+A5NMg0B/auGqOFnOyTrqOiYuLgO6E/7iqh/ZOyAmoF+QnbRTbtNV7CLsLizrDALQHfsY2BJXsC3Qnjm4nF2EPiMraScezb2A4cQSBO0FhlOL/FagWwKqlmjOQwQl9P0JJzDMPWZPgPxKoTvx6E4c9J4y4ZR4qTqCWoJBVk1IK9AdAxCs8hgmDPlRkYUSjgE49JkF6PbgJRMe6E4C7JRVEz5nKI0rGH6TrcTo5dJ7gf6IE0bdEFS6IlA+XIbY98jjsNMQGQYG4njDRQZbRllr4CQ/HZ+uo/euPzBk7Sm3oVuOdVrQMLsITieRhbkRGBYBk+ecANWPWAWioydK7yLIXIVxnGwRJa9LBgBl63gvRIDZngDVJcFMt+C+VE+w1C15DGNQT6xL6BdAe5xAV/TYNTxHrwVZqQBMXnAVvDkj6GlPBUwdQZbgdelnAsPssFhS3xexXoHALz3kwN7C+t6BKdQNx8uVYqwrqC/oTVNdYDBORdDcLQW6Je9dbwRcnCS3xwKLLxy6pUB/JDB9HqIkl5M11VGC2S9iamRBL1u2IQDefiN2280IcsyWvqN+QdYr2wZkW4z+xu6I2/GKbBArGCgPTems+a0nyNwxLdZWDK7JV5yU21IwCKcQmFx6TF94FLee91yUGduK4Sv9nAsuKYQmSAbfpO65BCp1HdBP46TRcb+y57g3ZwT++dqjW5IF2T2UlF/OGbSSKk5YgcH3es3zzNecOLQn/H5SQxiTWqWNoUgV0M0JTup7lPbbgtdrqLi4sHugkO3Zw9geUQ6Zbz1MzXHNN9H7qXjddccxS1LIbsHzqM/kWGmR/GjDREbJY/Qp7zyaYzGqFbKdx1hDUUqUVxZqiKFsEWz2EzmCLtZieKj+EKKTgnjaBbsHi7VHtvOU3Lee4TxLiXzl0B5J5CuP5pgAS3cB9ZlCfuvG45aOP7qllLK48ci2Ph67jD2WvFcSME6JssMkBqmoAzBqlwq2YB2GLXjdUn+iyyI7t3dxAY+AsD7TsLlAtnYobh2GqYKtJGTnka0duiOCsGznYGrPAJwujNtNlRz9lOOfryihLW4c8tXALsUuwOxd9AlirNCQNiBbDXHhRMJWrNcwtUU/kwzWiRUc9DAyZGeoZKy64P8jsCtyPMetg88kbBl91DEYR0efoRzou1Sth9lZLmyse9hSjWxgvqYPkfuPHsKYsKo6x95GHf2ORo4AXQQgW/VwBdNUXaUhew9XHnoYhQ0oXm7H/UnLdNn2OIMzErJ1Y5BQ0AJwAWbVQrVuBIv9IoPPFcymB1yArge4iYHZ8s9hkcGsW/ojtx3Mpoeqe8jBI3u7Z9iQBODp//S5QndaUnIamMaqdz3E4GCnBmrfUz5qJNS2JSgEIAd2X7pYvaE2HVxpIFd7uEpDtfQ0qs5Rvqol9KomkKwMgWlvEYyCagb43EBd7zimrYUvNFxF1Y7sLIT38KWBzw5eyOFiRlZQMZyH4IxMo9y3kLvm4KlM6b5VxuvhgdHXKPmnaDuEXDP99W5Ca3rEEJ5Q5oD3ZBhTB2TqpoxsZcgp9z14N/m+kMcQolgTkv4fUh4AaAKTXY/Rv9hHEFzkBJTDQO9lkR8kqeNx3tlWYiiFIKi0lnLZEAg0k+Q1AcmxGzOG+6RHYinvJs6GGJj0M7yTfG34i/38JSS0f/v463l8rYElINDXGfrWwL8sod9kWH3vFCdHO8ALuFqjuJQQrcLqehr72OIH6FkJ9UXBwBITMJxaqL2EnbtxJXTyTEE2CvqtYd9eEQvQe7KIeitQvDIoriRc6SlRiyX1xXNDRrCRyFYS1WcZ5XAKKC4puUhsDgKirO5OlPvA0Be1pa5feAIO2bGncPKM1SS6oZRPbxTMVkDX3H+/9Ehl9GYnIXuB/FYge6shB6C4Esg27GrUe/YtZiuB6ZeKclgJmI1AthbQe6ZE6rVmbUoHmCvGoSc2VPUEHvtHHrLnBBFeoLnHPsX8SqJ4o1C+ZVBNCmpxhixpmrCa6PWzFT1gbsKJXuoyDPoQMiI8KOmcRKlelMqpViC7kazgaATyKwGzY4ruMAXac49hEQF1moQFSjazG8lxWnNsVM/jWvzoUMthNmQMyui/NDvQH9tE4HkSoPdkuWxBUJatyUqLmAw7xC5DGQEcpaaRWVwA+TUladkWKF6rMZ5/mDEJ1RWUQ9mKgEB13M/sC4JaevmifG/g/+0fCdiSEjYTZYbZmuAuMeTFNcfWxxRc1gocGOUEyofpga1tzhmcoWtK2Mb03ci+yYELAXrP89UN+yQTe6viuNE/xgRT1ZO51HeqDmQfgaAjawoAkMD0eXx9y0UFYfleG3sKk9Ru+0hB17w2KfE2X9+RmQ30IXLxBGiPJZqzGEK0iwsh0d7iyhi40vDfw1TECguOga55bP0RF5u6hYiSvUBJYMZ7J9tw/yl1tZ/yfpaR3SxuPbolE0n7GZNcVQvIPrDzMIajkNmNwUk9AZgzPK5sS+CclAE+MlcpfAeg1NDsA7INQQ9C7ALc8V7icYYxfIfbEhimkZVNaaOeoNsWBJtJ2dEd8zjSgkoC5/30IIeU9tCN2C75GVMDx6ebk/nTDcbEUFuIUeXgNZnHtKjD8C/AFjKGRzFMCDjso58StAcpmMiqgPZIjWmo8VdMXBAgC+oMP0PNyaFaxGVy/N5mzQSQ7bgPSo/JkHotUJ8Q/MiBAM/UXOzRLUGX2XMRid2EB2msVzxfW6mYThvvvx4orgbWdcRkUyAm18Y08eS51I2H2fuRBe2ncgTH3VKR3RVAc8b+Q5sL9FOeW7b3I+PXHjEF1iugPWWiqG5iHUhUCpTXDEvrjtRY1ZEkoUkmKXxSJhh+L07ivnaOoK6h9BIBGKY6/p7xYxhMtzQwOx99q5aA9XaISckR8Heen21LNi0IgWxtRxmntCmQJ4UCEYz2E7KJdqrGOgyXSdgJj8OWrEJxmRwDeHwuILsYUDRleI2qyWjp2kGEEDsWmRRrZyaG4MS003kEFVKgO8mZxFpQOuoyycAem7YFgtHBjUE5Pid75I2E3g8IRo4JsrLne3zOvkSCwIES00URX89tuGkG+ECPYmsBR3+i2tH7Z3YWPtMQPX83Qwi4KoOwAcOyhJ/kfG5WkMGtDHyukK066G3HsZzlPJ5piZQkKyKbGZSAaoZDDUhiDQ1rUhjUo+HnJRnfgeysagjcXJUBjuBeBIYGBQGoXQ9fGIimh3AOvjD0V7pAcBhBnC8YiCNaguSgKH/18xJjBUlkFkU7UMqaZxjrPrw/BPBkBiHTMYn1wGiOzGACmeBnawy1CeHgpXQ8R2S8X5BA6WARuo69kl2HMK34Z9syBTaxqFWJMJuMAFSUJYGukhBZRhmrtQSjd35EZiDy7CB7jYBSFAVEUQAmA8oCIs/4nshEiixO0JJMF5GRjGmvP/kQd57/6cTYn/75Cij+Gj/G8K2f488vyuNrLYW9+PZx+LX/8T9D5zT+o7PP8LQ5RqkGvG5neLWf4+9efIpX7QK///mH0Mbh4/M3eDy5wXm2xZ+sHyGTDsdZjb3L8Lad4qzY4V++ege/9/AL/P6TD/GrD17gotigcQZn2Q6f7c/whz9+F/fO1vil5RuUasAn63PUg8E863BW7nAv3+CfPvkWhkHhg/MrfHP+BuuhxHbI8cO39xAC8A/f+z7+t09/BcNViW9/5ykyafHR7C3+9PYhPn9zgt985xkmusfeZtDSYdVXqHSPT29OMckG3OwrvH9yje8uXuDz/Sle1XPc7Cto5ZBph++evAQA/D+ffYzvPnqBPhpOXm1n+O7ZK2jhIYWHDxJ7m2HVl5CCKbIn+R5/8PkHWC72MMphanq82U0xyXu8eHaCbN5B/8kU2e/eYPVijm996zmerxf47oKiNe8AACAASURBVPkr3HQVfvhn7+D3fvMT9F7hX/6rD6G3Ehe/8RrNYPDO/BY2KPzZ5w8BJ/CPfv2P8L/+4W+hOtvjaNJgcArbugD+bIbHf+cpvrw6Rr/JMT/b4Z2jFT75F+8i/+YGbZNhPqtxezWDLizsJsM3P3qJN7spVldTPHh4gzc3c5RVh18+e41//fYCIQgMg4KzCrZTODreo/1Xx8B3tiiyAasXc6B0ODreY/VqDllLnP3SFW7/6AzhwxrTCb/ktz84xse/8yVebWe4P9vi+0/vo5p22F9VEL3E+fvXuP7eGXQt8NF//Dlebec4Lms8Xy0wDArGOPSfzhEeN/DX9F3AAb/0K8/wbHUEAOh+sODk9Ntr7G4rSglVYCLxZQ45CPjHDdQXJabfvcbNiyPoeY+P7r9BPWR48vwU2aSHe16N3s5w0aL4fon22w3ycoDWDm1rUBQDlAjYf38JO2HVzsWvv8bzyyVCrYHcIXuZYZhTfh10wPn717h8eozZxRa7F3PklwriuxtIGVA/nyK7qIEfTmFLwM0cpvd2qL+Yw88chPGQmcO3H77Gn/3gMRbf15j+w9d49XYBvzVMYB4Efut3foQ//OOPgPkAcZ2xyqeMXtpeAsZDVRau0ZAbDXHewXxSon04AJmHUAHBEiBlLw36M4f8jYL7sIFtNYqnGWwRIN7dY9jmEK3EvQ+v8PZmjvAmh59bmDeGvtQj9szm1xLNOwP0rYY9schfaXSPBujCovyjCvuHnHybnUT7uIdcax537nHyhxrXv+GQXZEdQQxu8pVH+UJDOKC5cMDcIjgBmTnIlwXku3v0twVEL6BaieqVwP43G/hBMgHbCmTHLfpNjvKpQfN+B1Fr5JcK/RHDpfbvDZANfbV6J9B+o0f2mpP5fukxeaqw+yB2wvUCxaVCe84FonDRAtc5ggmQtURxHWt+juhfVq2AnXAbDKMCmofRFxx9vfk1F6bqCyYam1sJ6QTCt7cIn0whLX20248cRKw3UlFOLwIw/0xi/5Bqg+aCC3hec6GtfmyhNwrFlcD+EaXj3anH5KnE7gOHyZcK7UmgMmBNBYc39JX2C4ZJ9UdxcawVmD0J2D/gwlr9jkV+peiR3XJhLkhaDiC4f5dzYcZWQPKnugxo3hmgNhrzzzH2KLqc42N2B89pvwgo39JLPf+RxuabFsIKzD5XTP2OXruU4JqUBjqC/fWHtFZkKwmfAbPPCeJdTvtCf0TpPxdj6F/1hpaLxadM5+WCScDuHY7B8Q8ddg8UggaOPrW4/aZmQNkqsd7A7p0wBp5RJs5xyFdkd9cfAovPIoM+YR1PcQ10S+67eilgp1xcKG4Cdo+SNy8uXgQuyknH12y/we2xhobnNrn02L4jI1Dj+OiWC1XDFOPiVepNBfg8j1GMiy0ypgCrlmqL8k2ULEcPZ6rzUB0XQvqFQHnFsaxecdGhPRZjv+70Bdnp+pw2huKans+gDzJ03QU4I9AdcQyHSqC89tjfpyqgvPIxWddjf19h8eMB9T3Nap85Jf1JKpwW3phIHMbFgHzjMVQSwwRY/qjH7lGG8sqiOdHINw71Ge0ps6cM3Zq+tDHEiYsrxa1Dt1BQkbm1lSJDbsGFgygfd0agfDtg+06GfOtHm4RuAppThfLKkcHuCRJtIZBvKKU2O7LfxXWP7aMc5ZWF3lvU9/PopVVjCJhqPbojhcmrHv1CR6tOgG4suqVhqNGtg9kSVNtCwRWsjgHIduuaDLPZO+jawWUSunVQmx5QAu1ZycXpzmHzOMfseQ+XcZFBtY4MswdcKVG8ruFKQ4BsJOxEQdcOqrExPZcrOLJ3ZFUrQ/mvEPAFZaZBSah9R5DaW/hJTpAcFwaSR1JYD7FrGBbUkbGVuwbDvfm4SKJva9ijCpAC+mbPmpMqZ0ptrDwR+3ZkTUXbA1rBz+OHJ0mwFWXmcrU9VKI0LVBFD0/yaDr3VfluCPRP9kOUwnpASIS+Z3BQPzDx9Sc9lom1Hey/tUokgdy/CVLY3/zdn78U9v/9P38xpLBfa2B58q3T8A/+5/8EvVd4uVug7g12+wJ/78NP8Lw+wvef38ff+eBTrIcCn1yd43y2g5EOx3mNP339AN5LPD6+xfPVAkp5/MbFc/x/X74H9YMpvvP3foTPb09weznH8t4G07zHqikQgkDbZDCZhVIe+22Bxxc3OMobvNzNAQDLosGnL84xmzfoB42z+Q7PPzmHudeg32U4Pt9gXnTY9xma3sAohwfzDXZ9jsr02HQFpAjoncKbNwvoYoD3Em6d4fSdFa6eHeHjj1/gR8/uIa8GVEWH7a5EWfbYvJni7OEKSkbDugh4+fkp5HxAuM0gW4n/9O//Af6XP/odqNJiMavR9gbNLse98zVePz/GvYe3mGQ9vnh+xkCdLJrdvYCpekyrDtsfHuPj3/4SX94cwzmJYVCQ0mOoM8yP99iuS4ReIV+0+PD8Cn/+2UOo0kFph36TQ00GXBxv8GY1hXtd4eKX3uDV5RGwNXjw4Vu8+PIU5WmN5k0FORsgXxUQ79RwlyWyh3v0zycQ91q4RuP9b7zBy9sFpPSoryvAC0AGiEFCDKx3mTzYon4yhzjrMJs22GxLBt54agVD6SGMZ2KwlRCdRJgPWJ7scHs9hag14AB50kN/VmL525d4+6/P4WYOopPAYkBo1Rj042cOcqvIJp83sG9KQAF6K2Eveqg3GbzmxHv+zVsoGXCzmiDcZCge7NE9nUI4AfnOHuZ7U+S/e43mj09gfu0WIQjUT+bwmec5HncItxl0LaG3lNHiUQPxpMRw0QOdgtpLuKmDWXQY6gz6ykBYwC481E5i8lxg/S0L4QTMmgm0duZQvtRo3u2BQUL0EsF4QAWIViFb028bjntgZZiU+50G2acl7Mc17NZANrHjrnLIX2kIH3tJBf2uPvdQjYRbWKhbA3dkkS9ayD+doXmvh742sDMH1Uj4PKB4TVm1asnWhsqheG7Yr7rwkDVDnrxm6q0tA9zUI3+jRqbcbATaj1uYZzlUTen2MPeYPFMYZpwElm8FNt/pYd6acfLZPRhQ/ThDe+5hHu5hn00gB+4jlA7ZW00GPAe8Dph9wXRfBKD/do3wpmBysae/188tpj/M0Nzn5yu7kWi+MSB7q3lcBeXsPgsYlo6guxMIeWD9CzACHFsc5NU+i6m7W4Kz+WcSrmT4kXTA5JnA5gOPkAfotYR7p4V6XqB8zR5KlzM0yX57D/O9yViBohoxgrHp84DVN+MqadTb9ice2ZViim+Jsd6ETDuBYaor6ecEGpsPPIFkIzB5GbB/yGqY5NUMmuArVRFVLxh2pBuy4OVrymuHacD8C2DzHn3fCHx9e0xglXzXuiYrvX2Px+3zgNkXwP6BiPtgoFN37HH0CVOvhQOKt+zWNNtoI1gcPMl2Qt94fhtiwAqZVpdTZu4NrQFy4HhQ7h79qU1Mkr6hGiFb81xvfs3j6PuUUCJwvFQfpb+xWsXlwOQ5f38LDzT3yKbqGlh+6nDzsWLFThE98kuO2/ypI+s5Ecg2lI/nKwKdxCLaSiBbB+zeIftevqWaQNqA5p7A5BnZ5+aeoCx8IOufwnnMnse3/QYweUGg1C3oc8w3HrcfKZRXlFgHQZn+7h2C3OItmf8RTE/5GUoS7GzP/sggefzJfpCAlO7IHFEGfACWIkS2OjL3zZlEtiLbnm0JEPsp2eDpa4duJinrjsFMyWoye34IdWlOmAZsmhCZRcoI9xcEvOUVQ3nyrYPNJepziZPvd2hPDHYPJeZPHIZKoLhlsnD1hsCPYVqUVM+fDWhONFwmUL21UK3H/oEZ5fWJXZaO/t5s7+N7FeXVtcP+gim9zQmTiV0mUL2xsJWk1P5lj/pehtQ3qnp6bptThXzjx8qebNXDTg36GaXUCBi9tdmGflVbMgCpPlNYfNlhmGgIF72uMUxomDAZd/cgw/RFj26pUV71aI8zqN6Psm2A46C3PeqHFRnpCOCED+jnBqpxyG5bArZcw2cM8NG1G2tTyDLGOVEzIGQawyyD6hzlvoK9n/lVC+EDvFFj+I6P3k5fKOhtD58pqE0HP40Mp1FQu34EfAAQjCRobAbKhI9KAumbGm6Wj0m2ctdHqTGoYmsHiMHCT0r6Mz3Body1oydzZDWNhrhZI8ynEJsdWcp+ILNYsitTxNCfoCRlrINlAFBmDqzpavMVP2l6XizmQN0w0CclyiJ6KAF2XPYDWc2mGa9XaFqI5PGMwBIhQGQGIUl07z7Sfu+e2896RFltCIFy3n/Hxy8EsJz/FQHL/+tvgeW/96P88EE4/8f/LWZnOxyVLY6LGpf1FPsug5Ye212Jf/DRD/D7L96HcxLzirOa220FaxWqqsPZdI9VU6BucwLAyyWyYsBvv/MEf/DJR6jmLfaXE7z74SWcl3g0XeEHV/fQW4UiG5Bph2nW48nbJZxVuH+6Rt0bZNrhZj3BsMswOanhvYBSHs5JaO2wW5e4f2+FXZtjvy8gRIDJLOZVi9tthX6foZh1KLIB+zqH7TQmiwa7VQmx1TD3GthBoZp02K9LzI5qdL2GEEB7XULNexIjXpC5AbBY7rG6mkKVFm5vCKR6CTOLPUtBoKo6bDclvnH/Gk9enpAVkUA+7TD0lPGEdSoklggXLcImgzzqETygnhWwjzpgZYDFAP0y56S4signHU6mNZ49PYWoFcxFTUXHLhvZJ6E95JscbuIgphahURBWQq8l8F4NOygUVY/25QQ4GlBUPZrrEsg8EAT0W4PwqEVe9KhvS4hOQcQEUr8YACtZ35KzasUXAcXFHu0uh3xLZswvBmQvM/SnlkDspoAI9ODKTsJXDrK0ECrANVzR09cGqsUYfKQagWHpIRY9/M5AFA7i1sBXnI1mVwr9mQUUA6eEB9zCQnTsCG3P6PWFAMxRiyExUh90TNWVgQDYceIJD4SZhZAB6nUOV3BCPcwCgg5QjYTZCjTv9pDbeB1PegQryaxpTqhThU4KfQqa4C1/QTYxSayzWwI1aYHyjcD2PYcw4eu6+xb6VkepMtCdMYl2WDqmBd+3vHeyQMA3CSMQzK+ZICusgJ3xfZTTCgzTMIY09SdMBPaGk3vIgPJSojlnh2p+I9DFnshsJdAvU/IrWZTiiixS88sNsh+X8IqsVlCA3hJwJKCVEibtlMdZvpGUeprofTYBxRUZI68YiuVy9nBmt2qUIrengYFdnRjvFTkwjGvyQo4T5KC4T11H8PRGjJ2N/ZzHqWoBbwKko1SXnZYEd8UrjqmwPObiLT//7M0jgFEd0NwjKwkBTJ8QsA1zpskypAZfSahMfYr8siBIARg2la0ZeJXfCPSzgGxDVlG33IZjMPWYMDvMAqbPBNqTmDY65RiZPVm52TOPzXsSes/jhQBkdwiwCjoyU7MoT5aI9Tr0vSZQa3YETqlGpj0LmDwXZNg2UQLegVI3E3tMV4cQJVvFeyL60lP/XlBRahqZSG9izRAO148dyDiksmaH8CeXsXKlPSGjpOs0xkyrFQ5RHhqlxRHopT/TAkIKxPEx2Cn5kBOwo4yWoGP/QCBfAXofRjauOyIAAjCmvS6+tFh9aOhPLaPEO6N/0ytKmkWg7NUbSmKli9VAa0qn6Y2OYULu4H9XPVlY3YUoPSWT2i1ir+0Qg89i1U46tlQdJAJBURBAsfaoT2X06QsUN0ykznYEmClROCXcFiuHoaSkPN2XpglojwSyGE41VARMXkXWNGeAjM0FiughdiYGBDU+JglTLivuhAm5jD2bLouBZdHHnHyzBG6sQUm9mtnGwU7kKDMXMSRKRYmxqaPMtpRj4qy0ESjH69MeKZiG94fZOrQnGuWVxTBVIzhUvUc/j/VhPqB81aA9L6NkNnpCYxiSKyT0nsE+rhAwm9htCabbulxA7x36I41s48ZQK6ZfC6jWEQg2BKe2UshXw+g3ZYhTSv8V0C2tSMNUUwbtA9TgIQZPqS8ie1xTMhs0Pbu6tnClht4PUVacMVCptjG8j8BSDp4S4s5BdI41Lc5D9J7dlZmGcJ6BQ/HvAFnHxDKmLtEgRezbdJANAZPPNdS6QSgMgpZjpYlwDB9K9SZBS8p8AfZsao4HvGcgT8YOTVHfAVBJXht7L0OesSMzhvyItkcoODdLfxc+EGw6R7BZZBCbPZnI1IuZwoDuPpKk1zqCxDvBQKGPktqiQKgbbvsuQ2nJPIqigN/uxu0RiPY/3UP571I78jWvG5nPH4Xf/J3/6ue+3X/2f//jv/ZzA77mwDJ//2H46J/8F2O1SGEsrq5mmC4atC1nnPamQFAB3/zoJX70xQUmxw3uzbd4dnUEBIEQBNwgkX9RQP7KGt3nc/jCs3JDgBP3iYXOLWynIZRHWGUIuQdkgMw5+a++V2L3EWssIAMByS5D+SQbGR+IAFkr4LSDHxQmRw3qTQH5NosBDAE46+B3Bmo2QH9Wsksy9zArBTvxnIxfauS/eovue0foHvALTbSKdSJOwKzYswkBBBOgVwpu4iEGAb8cgE4BxpOFcoIslPHIn+boTxxC5Qg2ssB6lI6TfASgeKNhC/YxesPk1PH8tEfxLEN7n35VdzxAbAzya4nu3EEddxAiQH0ygVcB/aMe1Y9yNA8cggnIrsgYlZcyesHCKBnMbyVDfAxTeeUADEuP6rlCc+Ep1Yt+Ua+B4XyA3GoUbyQDjjQ7NIeph9lJdEvPyfCpgxgk/aq1iAyJRxbrRvoPyMD1S14HvVWsMWklfO4JsMoA3YgxvTe/EWMqrdkB2w8dRCdgtvTHJXlad+xHz29KqOU/6Mf0huFFZifRH3lUzyUB59mA/LmBagSa+w66Tr4MBkEJy0qQ/JqJxv2ZQ34Z/TsqxEk5/WnNhYfeCggfg0I6HneSbrkijCBODmSTulN6ZlNK8uxzTuwo9Qsjm6g6MTJWZs+JXNBMKy6uCPyki4FaINhzORAME3SHCbiocUu2JSgyO90Sh1RSwzH2GUONEkMYFA51DQajL7O5RyYvvcbrg5SsuKYUTjjK5VzG62j2BC3BAMXVIdU3bTs9+qOA6oWAK2MK8JrjCQDD/JB2PEwPnlXZA+05Q3CKKwIe3QBj8nHJbs9uSeZomGJM/Rx9xZGRSQm6wnOMzJYBX9mKzI4teH+ZPfsu0376BUF08ryWcX+ciEbQ15FNa084Kdd1iICRLFmSLpod4meNE7B+EdnUAWOyNsKBRbMx+CgBGJentNboBfVprAhuhOUxI07OvebrmDYcRmBOsELfKIO9eH97xW3JAaN/0TRhrKpQsYIhVXgMk69KD1WPWDaPEXjKHlEOx3Ppp2K8funeSwm/agjoJ5RNljeeQUYRFPQzgekLPwYJecPQKxlVXwmMJXba7MOYmOo1vbmsoaGftD1S8XNAH+7+PrsUq7d+rNpI2zM1ZYVBcp/CxfspAr8UsqQik2Rqyub6qRylnXdTcYMksziU9FKO6cjxKy75XJtTieLG8zsnnostxFiFkXyQzsR9xJCdFEomXbwPYzfpUMkxwTel8WYbsoGTVz26pRkXDYI4nHNQAnlM402JpgC3380p/8y2ByYtySJ99OIKR59ntzTxe4WAMHWbJvAlBzKbwgYMcwXZk1k1MeRItUyzDXdqahhYRoliCo1ymYRuyNp1RwbZxiIIUOa5c18ZOzl42FJBtX7sPG1PDMyWDGN6uEzC7C09w3f2JXsPW/H9wvM40ucpHVvy0arBR0WGhKrt6EnlOAu4QiK/agHJBFvVOdgJuzHHBF4b4ApWjogApFoLXyjo3UCpaGvhItCEEtCrFiHTcLmC6tz4f6q1EINj+E9K001+TuvhC41hniG7aQ81IDFlNhgFO81G76bo7SFlFkCQkqAwVo2IcJCRCufovZSAaHqESUGQ6D2ZQ+cJ6qwjg1jmo5cy3PERin6g9DQxgN4f3jsm1ip6MNPzwOHvsc8SAEKZk8EcLH4yyTWd91ceSo0M411v41fYwrvviYE9qWaE/x0ohVVq9FwG928AhbGm5Gc+D/D/YnjPT8pp/zKPvwWWf/UP/dd9AP8+DyGAR4s1Xm9n2Gwr1C8pId1tDDAfgLVB+Vqh/XaDpzdLoJdov5jhCz1DyMhsBBUgrGBi6adzCADTL1g5wh46CbeVMJscw7sD5JqgMn/FDzoTUGMf20qPvh/1/SlEFqAaoHiSsftvzuRW+aJgcNelgU7GWytg1gJuW7DmQtFDIByjQ4WnnEGtGOoj/2iJ8hoQPqPXYIh1IA0n/6pR6M488isVf7nRD+W3OfJbBsyoNsb4SzIc+QqQvULz2KN6IWErThS7k4DqqWYNRgu09xxUr+A1UN0C7QcBxRcM8xlm7GLsjwLMZYZh4bD4HFhDofhBheYcQCAoPP4XGZpTIL9WKC8ZQjN9ItBcBJSXIvZVAcPsEHpTvpQYZkB+Daheobhid1dxBXQn3PbkRUC7pazH7ICUUaVrBu/w7xK79y3KZxrZlhPyoLl4Vr6WqB86TL9U6K9yuJJdiUHTrzb9jJMH1chY9RDlePNAD9spmajyDScWx38iv7qan3Pcq5cypssC2UvBwwzA7jca4MsC8y+A9oSyItnxuKtXAsMu43kXwOwLNfYSZmuMfy8vJfJVwOYDQLYS+S0B7uzHKnaHcTKQrcRYC5G6AM0uYPdIoLjmfeUuNfaPyNYVO4HJU0qHOMEX2D0maJs99di8LzF9FrB971B7MHvKXwwup99q8lLEyaREfT9g/mN6jHTD1FW1YjIpQhrfyBjFYJgkD1RdGJkoOQDbdymntKVgeFJHv1Z7LEZfW35DANOeUAJ79EnA9rFEto3SOxHGaozuiH9m64Bszfswv2WQRojM2+RFiHUzBAu6DTA1r0O2jayOBqwVmD7l5CnVPpSXsZeyY2KubkLskaWErp/Se8jQGckaFMsew8Qc0n8XK1BGtiSG2ewD8jUBl8uA8pr+ruKWk6HUzWj+f/bepFe2LL0OW7s7XTQ3bvf6zJdZWR2rimSxiqRkCzQhaGLD1swe2Ib/gDSR4IFh2SMP/BMM2IAn/hEekbLNIk2ZlClWshpWZf/6d7voT7sbD9beJ+5LlUQapMBKogJ4eLeJOHHinBNx9/pWtwPyFQdRKZ00CAlbCUxeOeweKh53yefKVwGm8azSMEB5FRNdo8y1uEw9deEN5k43GGtYwhYolp5+MMVaDN0QDOqGTM70hWOdiCH7w2qKyPo43re89hgmTKKdRrllvuZQcPrKwRkRwTdrXqo1gZ3qGaLUTwTytcMwUWOSqnAB1WWArj3MXtBbeKtHMx13l9FX2M8Epi8dgVyUXk5fWgwTslH7+wS4JnYoFqvE1hFcjqyUUqwpscAwZRVItqXPrHpt0ZxrmD3PXaoEKZYuyhEFu2EFgaSwPPbVpcMwkUwAXvkYksbPpX4qka+Zws0hGKtY3JlEcc1gKGlFDOJhrQgEE15lH/sVA1De8GuzdajvGogQUF44vobo+8uXA+o7GbK9h+o92mPNRNWphC0kyhsLETg4mCwH2FLFv3l8vskr+uqyLeWNtpT01F3aGGKUgplip+EQYqASw25Ky8oRs3fIV3wPe0MgZSsGKnktUF302D3IkW1jsm2PMfRJdXyP6L2D7BykDdi8k2PyykK1jkzx1hIUqTCG+kjnEQQrUIQP0Fv2NrJ+yEP2Hrq2qO8XKC4aaC3RL3I2UHQeavBjt2LQAmLw0IFAS7b8W5xAnNlT+ioHh2FqoGuCiDDR0LsedppBOg+zpRdRuIB+kaG4ajHMMvoJbYDaWQQtoDY9mbpKQfaOfkEf4EozSjjl4KAASj7nOfSugxwoHzWbGGQD3s9sBWA93Dyjj1AzaMhnCrIexuCg7KpmeFA9MNAnMYmDQ5gYgtIYLOQLjSAl5KYB5iVTZmNSrXCsLZEdgUgwCupmD2iF4aSCXrUwINMYJgLmqub9tITc1DDdMHoV5bYh8zlIgqQiZ6VJ3zKYph/Ibt85gto6BvyE6JMcHMGf9+ySvMU0AiDYkxFoJtYyMXvWjuFCYh8f1/VkNWuC9OADxLRC2NVA8BCT6tCJmUJ16vYQptP37JgcyPBShpoG2/FDzmhAkokMnT38Pv2saSmRTfvoA8Gnt/xT5BxgTHwePl7cTolNz/WzwOTt2+3fSwH4L3jmaLr95YnaL9ztC81YFu89DPf/h3+MrOCHRpENrB4xDl1rEILA8dEe2zpHt81hJj28U4fgC+1HeffiZIfl6zlm5zsY5bCrCzjLlYZrGK8njEfwArqw8F5ACMBtDZB5FLMO7ToHVKBvb5CUP2qP4ASwMwiFg8wcQ0ombOEOg6QMtFVQhYPrFJnNpQHOOwgZn9/HyWgvIU8Yz6m0w3BVIogAORsQvEBwEjJz/HqQgBOjjxAA5MTC7wz0WsGeWMgdZaEpmEX0TAr0lYNoFAG4EwiZ53Y8KAvNHUStIM86yCclbBmngXnyMUgIy+fN392ifj3h1H6n4I7JJgYTgNkAeZVRgllLhp2YQI9kLxFKB9EqehNnHsF4ej63mvt10gNbM8pUg2I1id4qShXnFmKvOX2fsqMKg4Bs+TqDAHxF6SU8oPdyrDEhyxAlh1YcmIpAD51wAm7qIRuCAdVy0m3nDmpHBtnPHOReQbaC9TS5h6pvfTBGZkLXXBSG+JwuZ1WKuJVk7Eoy1map4MoQ04Al7NRD13IsKEdMcrUVvV+pVzMFWKg+prZOHcxaxX249RojYyICGcvsJi4MLOtMdPQlkh1mnY20Ai760yDB8xGPaZCAL8g083WGA9MXIuN2i/mzE4/sRsEVhx5WykuZKGy2aWIc4HMeOxcZ3/4ojAAnGDAhNqYt2ynZu1QVk63ImmYrPkfqkrWTEFNu+TrJbsVeNkkmNmhuOwEO9tmygiVIykltxeNHv2AM6ZgFVhtpIIX4ZJsI7KMvUjgOG1JXbL84/Dw9b1p0RzbZnwAAIABJREFU+1t9t6mLFYjsoMTouUvSyQTEEXgMZB+lihnBe9qOauNrCxhrdFxM8A2K15ZqxcgchziiTHLj1Keqa+5TH6tmUqesjMMwW2EEdCll2BWH349y08hwpq5XaTECaekO76PRZ2r4s9T9acsooY2MZXcS9z0er3Qe02uExFilonqyLYmZVR1GFlZ4nkdvEP1+scReYJQF8vEEfy4mFWfbBJAxgkFp6W306RxEYkAOBx8kOyfJ6qRew7E/NL6vU9jMyC6pyCJvqABJA5Akr07MfwK42SaMKcWJ1U3Pka4//jHCG52lQwTswpEF7qd8r6WuZgCx6urQscr+TjGyxl4fzmdKrE2pySpeDzr2zaYeUd0e/I+qZ++p6sN4rQ0Tsr6JJU3y1PEYyEM6si0kTEOmVdpwODeOstjUQ9kt5OjL9Dp1Nh48nIfP2oB+xuoVEfc3saHjPt763hsmDwOspkl9n/1Mxlos1uQEIcYqGNVT+ptYT9xep0fG1OUynl/6ItPxTccxneck9U6srfABtjpIWbmffgzFcZmMfycok3WFJAMfFQbpvSBcGBN4AYwMpysUsvUAlx+Od5JuByluvccJzik5jim+JiULhzE9Vjgf2VoxMpCUmVIG67Xk/W/dRhbSh5HNhIyBOTF0RwxkN0XqzHQBqe8yyMM2gxKQsQ5FJAZPKTKYhomwoo+pxNHXGDTZ0BFkxcCeBKiCUgcgmvorBzuC2dQV+gZQlfR4MpDH4/P+SdgIJIXgtt6oBUnAMlaLpAqTyBKGeF8hJYJ1nManTssESkf28nPsZJTYvvmjJG/9S2KRvy2M5W/8O2As//kvGMu/8i14AT8otIOCaBRaSWatfuQQKgux1Vi+KOBmHDUP6xzZlYY7dTA3CqqLnjETsBzm0Ncau24OtZNwZUBxIemZy5kkiMwBVsItc+itBDxgeoF+4ZG9n6F/x6O4kGjuUmaJoFG9DFh/NaB6KVHfF1CdgcgCvDMImUf+WqO7B1SfGPYoPuxhXmWwM4/8g5LBDxmQrUVMWRRwdQm7sJi9X6K5Q+Yqe6bRnnmYWkBaBsP0Zw7zn2i0J5Q/6p2AK/X4h0dfawgAeqfQ37HIX2voWqB+yyF/aeCKALXSDBHZSLiKx6R54DD7cYb6XkD1/1ZozgPya0k/n6eXz+wE9g/5AVJfV8iuebzb+w76Om57L6GuC+QrYPtewPxDieYe+zbLC4l+EWABlM/JBMheQnZRIuLJEObvF6gfeuRLTtNtGWD2AnZCb5n5NBsrRsqnhqmemyjp0GR4By1QXKpRrtfPAJ8z5GX/9Q7TH5Me6o65oNEbfiDbih7BfAns3vbsHgUgrYJqCKKGKdm9/tijekH2uTlPHXhcdPULD7OJgD6+I4trgf2jWIGyBvpjQLUKw8zDTikBZqhJXITs+Uc4X3IR7nIw1VMF9Ccek88U9t9qYZ7kTLfUAC41nAHyNXsepeXxKy/pfROeSZr1l3tUH2WsAlixo7L3AvkySvUsGcGgCNTK5wK7dwJEy9dhC8A3gnUjA+9nduyNbN4ecPQDg/o+wVh/FJBfKVQv+XuzJ+gB+HrmL8nula9D7KQMmH526DT1WZTUxoV/Cj7RNZ832/Bn+4c819ULyl1VF9nOCnB7Mp7dCUFdfs1F5O4RMH1KmWk/peTZbDkR379NPxylztzfbANgzWPdHYfYaZkkmTFp9IZhKM0Dh6MfKTR3ua9k4mNdi41yac3zU71igqXXInYHUq43zA9sZntCINvc8SiuJfSez+MzYP6Bx/6BhNwR+PcLdsRWF7GmxBPk9XNg9im7LKvXBBvdMVm6+acB/TTWrDQEVP0CyJascEmAtj0TQAreeR0BVibgc6C89Lj4TWDxI4HuhB657kigek3voa5ZI7N9HOt0OrKvCQwFHdnajNLV4jo+b4uxJ7c74sLc7KMX0BJ8lBcEZMMEKG8CdvclWfycEtj2TKB65dHGFEwgjF5O1XHxa3ZhBJVyoFQ0yUbZaRnQnEl0xyL6RwMAghpEpjnbBWzfkiivAraPJY4+IiO3fSSRbQ4Aqrjh/po9j5GIi/5sEzB71qE5z2JFiohJmWQfE0OeanrkAJQ3ZNvW72hMXjns73O4VF4z1bifiLF+qD/iccg3iZFlT+z8iQUCUN9hhU+xcmidGtNTTe3hjIpVKh5qS8CXb2Kyc5AwDa9b4QFTewynCrolkyx8uAV2+H5SQzj0dGpWoUxe2RFo0GcrUd54qIZSVQCQQ/Qwbh29kz6gP9JRCkvQoxtPqXXwaE7lKBdujyREwc/GYulgS4niagCEQbZxaM5YGaM6dl0OlYatkirGA5JsfLaxsIWC8GJkjF0hkd8M6Bc6+iENJs9bDFON9lSPQ4tsPcAVGZnExkHvLVxFnzxVQgpm52Fqi/YkQ74cIAeP/sjA7MnquSyD2fP1Z5sBF9+pcPZnLVRrsXurxORFh35hRhCJEOBKheIq+uo0vZR6P0TfdcHqkdrGag9KUIPmsRbWx9Ajh9WXSyw+qOOiDZDWw04MVD0g6AJ62cDdm0B29HJKALJjF6eOFSesNNHQ2w7DcQHVxH7Mjumr2bJGEAJuno9SVb3mZCYohZAryOUOomIVCrfH3k27IOsmeg+16egfLg3kvoOfFhCDh+yY0irbgeBwIFAU1nMV7QJlq1GyKpzj74QAfLxvZFABRGAoGaYzRECoFLsuvY/+yQHoenocraM/sm5ZbdIPCBXDeoQQBHJ5RsBoLZlCTwZ0BKnp63SzHtCK0tYQRiZyDO8xJgLHgf5Ia3E75Cf4AHgHkWWUzIow+i0TWBRKjZ5MSPkG8OUfNc+eTCB6M/+CKhFP9c5fBVj+vNw+P+D423T7wjOWd/+bfwIAkNOBQTZPZ/y+Y9JkcdxieDIZF+xQAaKPbwzDUBPhoidvo+FLD2gPMxkwrHOInqmiak8Gzs5dlK1K9PcHiL1GyDzUTtFnNHP849EyCEV07JDkcwmyS1sJ1VN+q2oJN/HQO4nhbEDxJMNw5KFqgf6eRfmZieEmcZIL0Hf4qEPYa5i1gt4zAKS4ooTTxOh52SP2y0XWSwDdqYPZyJGpCHGxqjpAWIaWyJ5eQVtyKlc9p7dP1+Iw4Y9T7PacqZP1w8hgRaZAdYcpaPIhuSIgGMBlBH12GsaJpCsD9I6Mo97HgvuMv5c9WcPiWqB+EMNFosSuueOh95SByYGAsD+iB5Cx4AGyix2Bxx5ZStMEF6BeY/QDAnE/I1uU2ARvuO+qEZFlCVA9OwylZShLkpQGjdHrFDTPVTBkW1LQSPIvecMgixSL7zMeL7M7BIKkBbKwGNkOs2OSpOwjKzSySNw+ZZhMwaxeUF7qCow9lInp6k7pwwyKxzqxm4lpGmZASkSdf8okSRun/90xf54v6Q/sTsk2VK/SopyMhW5jmIrka9J1rFqIxyBbHX4PxK7QyAzpPYNMVAQKielSPaP+zSYOGKZ8LYufCDRnYgz4SYty3fD/5DcjwOQCfZjF450d2D5vCBKnT3j9DVNeb9ma+8q+Si4QXRZ7C9cBwzSeE02QwWqDyAbWkdmI57NYerTHcvydsAc2z+wD2lMZhykBw5yJo7rhNhJYSp8HiZFJniqA18H4visPi1nhKemdPmdVQ7YOI9M1TMiKpeNv9gH1HYnqgtUHqscYxJLOsfAhMlWU8rpMjP7cIMmSqT4CzxnB7hjOkiMmW0YwE0NO+pnA/KnF7h6HPSYmctKHSPYneWgTS6abEJMqA/Z3OQhI13pi1dLnloq+X/YzBrTHrHpwhvuourhGiv2axcof2EUApvGoTwkoCN54zc6eOfrxIrsFcDvZ1sNrSk8Tu+Yi+2Vz+ha9Fsh2lAYDPI/SMigmMZ4p+MfUYfT6tYs0aKP0O12TtpDj49LAKnmOh4pAqbyyZJg0z4PP2Ldp6gAVgRb9ptynfOPiZ2TAMKVfUw58nFd8TWbPcBhbSug6MluNx/4+6yxsKePnE++jeg/hwPM6UH7cLaIvb4hsVMDI3A0TSYBobjF59ta10XMbtuLXwoex7iX5BoMUhwqKMtZRBMDHHtgk+83WNno4RfQCMt20n2sy2UNAcdGhPc9vMbthZOcYbsSqivQ/P19kTD5lKE2qxsiXHdrzgomnYw8jMMS+S723GKaaPZWNg9n06E4L6IZ+Q9n76M0UkHH7rmBwjFm16M5K6MaNQTY+4zBW7wb0R1n83Ijn2PoxnCYxeMkvCR/GfswEQnyu+H0KpEme0IBDUI+Rb8hpfa4pW+7seD9VD2NfpRwiwC1ih2lrx5Ab0dH36DM1sr7BKEpeXaDkMgQydzplEBzYWG8U9PWOdRx1R9mn93BHJdS2i8eRYJBPLkZmUwyO0ty4D2OnZJ6RpUwMYWLnFD2QochGiW3IDG73PJLJ5BuUx9zRdxnBoOiHA7uYgndiJ2ZSJyR5LAAG9mh1+DoEQGuC2fR9fN43klhvsYnBe8C5N4Af3zTxPlqzA/PWLQzxPglE3n4e5/Az8cZtpvIvGeTzhWcsZ4/Cb/z6P/5r3+4//z//2d/4awO+4MCy/PKD8Jv/83+BWdbhz1/ewdfvX+DPX9zFb737Ef58dQdKBFzvKrx1vMInl6d49/wa1/UEp9UeH746R1n20NKj6QyqoseXT67wxx+8A5072L3BvUc3eH15hMVij3nR4enFMaRkeisAdI1BUfVwTqK/rKCOO3jHQKDFYo/dvoDtNJNlLyasz9jmkNpDaY+8GFDvckxnLTYXU8ALqHkPpTzmkxbtoLF/MQNmA5NdnUB+1MJaBa0d7KDheonHD6/x5Kd3oU9bDLsMIncIVmJ6XEOKgM3VBHACes0p7fxrN1jdTCG0h1J+DDAKvYJoJMLMsmbj1ZxyVBkAHUGgCpie1uh/cIThccc6Eh0grw3kgwa2ix9CrYKcDJjOWuy2BfzeAMajmHewn0xhjy0Wd7ZYvZ5BZB7lrEX9esJwnKNY4SEoN/UzC9Ey3dXnHqJywEZDnnXw1znUKY+7t/JQrTGhhFcMTHPF3AJrgxAlyKqycFsDYQWyG4nurR6i1lAR3AaSi1B3Gwy1gdjpUearNwp2YQErIKxgXcbdHqFn4qxwwPk3L3H5o3O4iYea93CbDGqr4OYWajZAioChNpAbDXm3hZAe9rpk+JMKmHyq0H13D/eqxPwDidWvpA/sgNmdHeoPFvAlJbziTgs8LyEd00LtJMA/aA/pvZ2A7AWrR1pFObUFsiU7/1QjMBxxYBJMgF6rUeoqPNDfsRBdrG7JA4LxkLUaU2QxHaCf58g2As03G5hPCnRnDigd1BWTdlOHIgAOTtoY2nOng/m4HIcpw90BolWYfKaw+5KFWakIKuk/TD7UfEm2cji2KJ8Z2DJgOHEoXmkESXCk4t88VwQOgk48zIZDiOadHnKjoXcchgQTFQpRvgoA9k4PdZXB7Mi27h87FK9UDHLi/mQrCZcHDKcWasOaADtl2BGBnoAtAuyJhVprgo7YFzrc7TH9YY7dNzugUTBrxdffKOQXahwMeR2DcI45gEIAZbdVGNlir8k8FlcMNmGibwTTHZAlaW4ZUFwK1PeZRNsdUyEACUw/46LfTgCzAfZvecw/kBzmbDmYas+YJGu2lAkDkRGyDArSu4OflYCLwyQEYPoE7O1rOXzJNkB7yv/7oxToxOHE9jHThlPqKn2sTMj1BmMQT7YFmnNeH+WlwO5tj6MPyNCZbUB9nyz1UCGC36hIMGT3KeckO55kvgCHS8UF0J4RTKsBZHePAoobMcp3U5BQ8gKLgNHHKC2weysqBzSPadq+2ZEdLq7p9y2uycDLgcFRzR2BbMUUWNVTeps6Mk3N71XHoKX2OIJPdTsoiQMJ1RIIpjCYIMgou4yhRNkmDp4yngfdUKLrSg48hinvk20DujkXscOMjC/AIUWQZFSHVIenBVTD6hX69MkYp+GQy6Ict0n1LBxuDFPuY3Ed3pDkmyZg90COqa+6CxhKXgf5+sBgAwfQnO04DEiy1xR0JS2Bt8vjYGFHqa3LmLrazTloq2KPZD8jUAN4PGzBKpIEwLtZTGgdAnTL4YTL6dd0GcZ6HbMn6HOGoUbOCHRziXzjY8Ist51tPYZJDDwaAppjheqSQLqfSeRrB9V6DDPFMKX4GZfA7DDhZ7vqKI8tLwf4XKKb07/rNQcaQ0kvtbAB3ULR7zpTYxKuabiP0pJ9DoJeZ9V6mO2A7jSj93nLlFjVeei9wzDTo/S1n0pke88BQy5hdnaUzvqYAGtLhWzVo7lbwOwdJbFxaJKGjWZLpmyYmdHnqlqH9iwj0xxZWADwmRyltEkuno6P2vWwi5xsuA0ISY5r6EEWPkA2lpLVONAQsTfRl5rMZe8OPZM5E17tvIDqbkk+rUfIyWamwB97VEJaTxY0Nwz9MYrVJzHwJ9WNwOgo+3UEkAlIpmRWKQkQfTiwm203MpkI4VAt0nQI8wlGXyYwAt0EiP81mWz62mggBIS6hcizw+9v+zQ1AXlou4MX0jnQwxXZyz7uc5LF3mZNP/+8n/dgvnG3g2T2i95j+Qtg+XN8O/76eTj6Z/81posa59M9lnWJedHhs8/OMT3bw1oFZyWGVY7Z/S2anyxgzwaowiEE4Pxki8ubGbRx6Fc51GyAtxJZMWA+aXH1yQlC4SB0QDlrsZg0ePHZKRAE9LxHVXXoB412m0MXFmfHW7y+PBorMjAbILWHHySmRw36XsMOGlJ6OCvhGw0YD7k2CMc9snJAt4xm6CCgVwricY2h1VA535T+KqcXsnIIVkAWjimy836svoDkOdWvCSzs2QBh+IEsRIDfmdELqjdMjA2F4+OCgLrRZGfPB5gJ38DDNme3o2ZRejjroZ/nY5egngxw1zlC4QEnIKcD/M6geKXR3rH0gG4yiE4AdzqE6xwhj95G4AAapxbiKqM3sbQcdLUK6OVh//YSbmFhLg2G84Fy3QgkgwpA7iD2GnACYWaBXo7+Tb2Vo3cIANxb/MsvXhZwE4fsRqE/cwSPZwOyl4aL7SMHWUsChWPWaXgd4BcW6CTMikBrmHn4wqN6qtHcZ/dgSnvtT7jdfCkwTCjbDDogv6Lk2pWRien4OrK1RPPAAoWDeZVxUTZhnUTzbg99ZeAqymjthOXzet7DrjPkl4rgqiOY8BqYPJdoT+jdDCqMVRMA2WrhCFia++yNtAuC8PlPNJo7YfTiyZ6L5f6EibK6ZkVIkAwNas88spU8MNYTPl8CY2bD1GKzpTS0Pw6QHTBESTAQk4AvJfpjhouIgdJn4YH8WqK9Q7Y/2yRfJWWzbQyw6k7IxqbkYlclxQBDiPIbhtfsH5HxDhKwU3Zqmg37H6dPJFxBMCW8QLYUYx1H8ismr6HquVAd5gfvZhU7FoOktLW5I97wGaqWgVvzjyk5bE8J+nxOBsybxDLy3JSXZKyGKc+pasUoAQ4SyFciBmwJ5NeRcYuMOz2AfN7uhEMD2WMM8HEZAXVKNxUWqF4L1Pdjkm3Fc59qLOCjlDsmCEtH9rq+K8ZFPP1wBLbCU4o7egpjJ6PZI6oScCssDCguIys/HCTNTI7kAr8/ogfMZxEggsy12bFjMu2n2REoIwY8DXO+5nzJDkUAIwBIoVauONS75DcB/eKQkEvmRRxqSnyUw+q0gOXxTCBw+oxMNqXFIoZsMagshYaZHcFdfkOQGRTB5W1WPAHVlHqb/IL7e3IEkrcBnW6i1HeK0WdZLAkou2PKeHVDgJrtyLymEKB87WFzMTKQ/ZyfE8ISeJo6jM97uxcyeX1TmqxqEUvuJetArg5Ms3SUVpfXMTTqNIYuBbLQwkU2uuPQQzeUkqf3HZ/7wPAGCSTPp/B8PUzOZSWGVwL5hveXLkSpMgFOqgBxhkE9rIM5MMM6Mpr9XCFbOzRnmoykobRX9gH1uR5TaxPQdEYg27kIMg8hVO2xguoDihsGHKk2oD1WfJ6USBs/l/OlZTWI5+u1BQOhTONHZtZlZECDFMjWFv2Rho7VJMOU0taRzYzJvapz6OdkDLMNA4xkz+fXjR39jcnHO0w0VE8Q55OPGHxfZqsedqIxzPTYQemVgKkpDQ5aIFsN6E4MhxxaHECmYwJsftMhGBnPiYwJzX70UsIHJtjGp7UVAWnyVfqcMmwAY9ARwCGH7D1czgGl3jNEJkTwCwAq1X+ArCcZbvZfUrmgoBoyqYldu50wS0+lJluaQFNkI11pIHvLTkwhACUAFwhKty1CZgjyAIK4MiO4lOLAhib201G6KpqOtSKDpX8ySXA/Dwyjz3JMjE3ANEqBx1sCc+5NqWzwMd1V61seSlaQAIBQ8iCPlYr3Cf4NBnL0WN5ib39mlyWfkID033aLjOYXn7F8GH7ju/8OgOX/9d/9jb824AsOLPN3HoXv/i//JbT0ePLqBEIGvH33Bk8vThiC4zgJUrPh8L0AJa+LgWDFCehFD7s3KI8bdM+mkC3Bw8mDNZafHJM1m1tgkCP7xB1w3EYQBDW5g9xpFtdXDmJloPecXLojMlx6wz8S9shDxoWhHPhBxxLvFPEdZZeTmFLZxiL2jOBI13xsf8SFd5AB7shB3+jx/u0DC1lL+IILZsod5SFgI8qk2AMI1p0AlAPLgOK5gd6TiZCDQPduC3GdwU8cQ34cGZvwjS3k+zO0dxyDVaL0yt3rEbwgK9eLMQSkvBDYP3bIr8jK1I8HlM8MvapLgiSz5SIiBaZkG4HuOyxsH44C9Fagvecw/0CNC+e0yEdgjYbsmXoaFF8fgPjHkq+zu8NeRV0z+ba4FGjO6dEcJmEMkbAVpbuu4II+v+ZEP9UlJKlmex4BlePX1QsyYCJuo3rBcJf6oUP5QsHnGDsIgyTDVL3kYp4STp5zszv0CqaY/LTwzdZciAsXg1VikIbL6UO8+TUHsySY3T8iU2g2keUy3KbLuEB3BX2DTBhmmEuSBjf3WFczeU4ANHkO2AlBDn1AGGXPZkvwIgeBfh6QL8XY5ZhkgypeC81dj8WPBdozMUpWhxlBVLc4SDyDIqBwhUBzh+CNLAf3U/XA5j2P0+8L1Hf5+uQQYlrrAejQp0cfJDsuyQrpJsSpPC+TBLBS72FaPPsYusOwEgKXtNAd5jwHqWbC54gF7GIEO0Fxn7M1H19dMLm3uLglt52xWoTeRS78EYD6HhnFfMlFrc8IbvKNj9JbMsyqI/vhMzJLQcZkZY9xQWirA0smPGXHlFMLFNdcXDMohscQngCoucvHVRce/UyOXYN2IkbPGqsiotTcRCCzPST4qp4MYrGM9RZzPra68tjfUShvPOo7EuWVHxN0ExBRXZRKe7wRLDTMBMpLsmbFioEw+TaG0ET5cZI7JxBi9pTx5usAZ6I8Nfr7honA5MKhPo0+vSjBtgWvEbMneEkVKrbg6yWjmEAgy+JtSQDDBFbcqmxI+xW7KA0TaslsRXZ/Et87PaXlaf8pSQ+orjxU69EtVOxcJCuXrz2aU0UGrGI6az8lWNKth80lAVSO6GUNY9BMeWnRLfQYODR5OcAVEi6XbzBRquW1nUJt6NEW8dzH9N8kX4/XpKnZvThUB8lwAlzJU5hAEQTG0JnyklUhZkdWi+eKLJ0tyXLpvYPPJGwlofcJdFFinJjiVGOSqmiyjYUtFWwlUV4yeVb2AcVli+XXpzC1H0OEEtundw7DXKN81aJ+UBBcrS0rNjqyc4dgrgH7h8X4e+EpM7aVIntaMOjG7CyGuYZZU/LoSoKYoATBkBJk1mLATZAC3UKjetkyUXVg6i3lq+m9GCBjgE17kqG86DBMCbjoK6UnkvUYh55ISlY9VD2gP87HxF3hA1Rt4bUEpEB3nEHX3IbeDWNYjnCeqbGgfBUAaztaPlb2Dq4ykJEBDIrZD3rTwVVcW+lth2AUQV1rITcN/Lxkx6aK9R5RRgwQ4Ol1+0ZFiJtmTIe9FeITlIRas3vRns9ZESLYVSpcgOgtQWI7QNQd/FEF2Vom1N7sCIyMJkOoFEKuCW7r6FNUihLUJFVNDGLyDiZWb7BkGLsDOA4FGUeREmJTGE9iK0MgEI2PCUYDyzW37RywmAPrHX2PR/ODFDVJXVNtSQJ2zjEVNiW6xjqPEAKEEBBVSZYweikT6BRa8+dNC+Q5f5eSZY1GaNoDQI2S2OSfDElaewt3JPAphPjXOy5v3xLw/NsQ3vMLYPnze5t85X747f/1P8OXZ5fwQeDY1FhGPY4NCqu+xA8u7+E37j2Fh8DLeo4vz67wrF5gO+Q4L3e4bKb49vEz/HhzDwBwktd4WK5w0c2ghcfH21PcrzZ4tlug0HxDXu4neHdxgx9f3MXXzi+wH3JMTIeLegYpAqQIUNIjVxbzrMWqK3GS1/jh5T08Pl7ipqnwaLbCs+0CE9NjN2TIlEMzGJxVezxZMf2j6wx++cELtM7gyWqBR0dr7PocvVN4e75E6ww6q6Gkx1U9waPZCuu+xNVugpNJjet9hRAEvnZ2gWXH43Je7vA67ufMdOi9wlU9wWZfoCp6TPMer5YzvHt+g85pWC/xaLbC63qG47zG080x7s820MLhw5sznE/32LQFhAhwXsA6hVnRwQWBbtBY3UyhiwFvna2wagp4L6GVRztoeC8wK9ltWXcZtHKQAijNABcEbrYTPD69weAVWquxa3MclS2udxW+cnaFT5YnuDfb4uV2BqMcQhDo4/HwQcBahaNJg8FJbLYVTGZhBwWpPPIoZ54VHdZNcbim8h6rXYn7xxv0TuFySc9ullsqRwYFYxz6zmA6aVG32UFO7ASMcZAysK9TBKxuJjDlgPPFDk1v4LzEbltgMmvhnERuLHZ1DtsaQAacnuxw9eII2VGH7z56in/xwZdQTDt0jYHJLeyg4Qeyt6FTUJMBrtFMFO4lqjm3y0GjwzCo8TO+yAZsNiXycoDRDpubCbIJpdwpKVkWDkXZo20yCBkgANhaA5Yurv2gAAAgAElEQVRDlZDSkFsFoflHwBQW/TaDriyTinsN3ylUiwb1dQWROyAITBc19tsCIQgIEZmeTqGYd2h3sdhZBYRaIztu4Z2E7RRU7uB7hdBL6OkAuzcQWUyd6xTUxPKYAJCZgxQBzkqEIEapti4GDHV8Du2ZmCwDRDyOYpDAlMnKjJAVh4RnFcYJjC4s7M5A32jYOwMlTCpArg38hPUCCKB02Ar4heXz1ZqMfxknuDHlOTtuYZ9XCHmAPmkhPqowHHumLrcKoXLAIOjhHiKzUgb4gl5w9JL7pwPkNg61goBs6JNG9JnCCYR5HKaZAPQS5iYGXs39OGyBB1n7cwu91hxG7QSGWQCOBsgrAx+Tgrs7DqKntNhnAdkyBnipANXKOLAiW56kir4kQ6U3sQ+24SJcdgKuokTY5VGKHqXwwh3kpN5wf1xF7zQERtn2MGPiss8imxA99D6GdCXW00f/p9kyCXuY0T/uU7KzJEvmDcOzCHDow3Z5wJgWrem1ZphJDPZquL/9EY9DfiMPzxvnkQiUCGebGMgWkmRZjL5w1VAqno6Nzw9Ds1H6GUAmvhZjgnKIKbopYRogsBum9Ki7lL58KxRHDgc/qXQHCWIQlADLnkOQlBJstpEJlQffJsDHsoOXgxuvgeKG0uMkGR79cPKwbQBjKrGd8HFmx+dkAEw8ZlHOnIK5QpTFpwRkV2JM8h2VAeLAoqZE4TFpNKYXp6CplJasBioDVMPrIMjIbE7oGbalGJN08yWHPz47hFgJF8OiNMbuXuF5bESgh9jEHtjUb5r6TtOgLyUOpxqslHSdenfT6xmZ15hArLsAhAAbvbMpPVo3YWRBneHAKFUUSZtqj+KBEWngHcauU1P7UeKrWw4hhpLJvqzKAWwpkW8cq1xicrNu/C1ftBiBd7Zx6I4Usq2L0l+M0ufxehqinD+CMdmHCADpyU3yYno9cVDfuIOHGZ6spbA8l6PkNPZqpvog2fvDda/4WOEDxOAZlORDPE7i4D+VgiFCKcxGHK65z/dvys6R0ZRyTLOVdY+Qxzdk8qoaBdnbA1sZDh2mY9ps8n2m1wO8AZARwiGp9jbjeUtKKgay0iKmy+Lz9wXe2Ifb+5i2kVjM0PUjKOXv3EH2mpjdxFj+DD9nAokJeP7M2+cSY7/4UtiH4Te/89cPLH/3934BLP/Kt/zdR+HBP/mnCCbg7/7aT/HDy3v4zr1n+MMn78BaBaU83j2/xsf/4m3Yhx2yTwuob61Rr0s8fHCDFz89R36/hrMS1R9OsP66xcMvXeHiT+/iP/+Pfg//2x/+Pdx7fI26yyBFwOrlHFAB1ccGzdf5SZ9XA8q8x3ZXwmQWQ69h1xm9cHd38F5ivypxcr7B8uMT5A/2aJcFF64moDqpEf7kCPZbe8ymDW5eHEEUDuplDp+HWDkSEC4KLN67wfJmiupHBdS/t0QfwdnwqsKdr1xh/Ud3oH91BfunCwxfbTD9f0p0J0D7dg90EnI64OioxnZXwr8uEE56mMJCyoBunwF7jdnDDeqfLBh48ZUaeTGgbTLMZzVuni+AANz9fYnlNwSKC4HNtzvM389x/x9+hg//5C2UryT27zjorcRwzFRaYYH1t3vMTvfYfXaE6q0t6qczBOMx/VijfuThS4IPUVnkHxUYvtqwZiUu7mXuEG4yQAKzDxTaf3+H8MkEw9xh+rHG7j2mFJqVwm//g/fxu//yWxADvYWuJBM6TAPmH0usf8lBtgKqJvvgH7bIPirR3rGABBb3N6jfPx4XeHIQPIZe4ORfauAfXmP52TFkQzZ48mgL+68WYzl8c9dj8SOB69/kB6basqh8+u4a3fePYbZcDNUPAnwWcPxDsmzDUYB71GLxewVuvuMw+VSjWwTo93boXleYfaQYDNTzOWYfSeT/yQU2//cdtOce6ryFu8qhWjnKMcsLLhL271qc/ZHC+itgbYsM0DsFO3WQswHV98sYWhPQ37VYfN9g9S0e0/K5Rr4kkygcKzNsCbi5xfQjg7t/3OLT/ziDO7G49zsar//DHue/k2OYRE9dAzR3A+z9HvpVRlbyyEFvFe7+kcfFf9pC/2gyJtL2M8B/c4fqe1PsHwRMngv0c7Kg5QUDZtZ/v4F4UmLyVGD9DQe1lTj5IXD56/w8kzGgS7XA7Alw8yse935f4NV/EBBEwNv/O3DzDY3dOxbTj1lHs/mahV4rmD2ltGd/IpDtPS6+KzE87HHyvQy7x8DskwMrXD8guz19StYpRDZh/XVez0c/YUDP0ScWy6/osRczyRlf/92Ae3/Afd2+zTAdsw+4/hUR+2wDsjXDdoYJwc/uMddR+ZLXb/OIXtTqOT2KkGQy0/WY/GSzJ2TMrn5VYPFTYPcW2TATOyVf/bZD8dIgWwPb9xzufU/g5d/3OP6+wu4xMHkiMHnNgJr2nH6++acBy6+TaZ++cBgqidXXgOKSMuB8GeICM+Dq14DqpcT+oUf5WqL+Zovj389hS8p/d+9anP6Jws23PR78H8DFd7jgPPrwEPy0eVfg+Cce28cHZYRugeqVx/KXBE5+FHD5bYHpM4Hdo4Cz7wfs3pLIr7mQXn/DYfahwuyZQ33GRU5zT2DyNMR00oD2hCxaPyfjS8kz02KTDHTy0qE5kwiCEksG5ggm9B4JtOc8x/f/wLJgfqJQn6sx4Gj3WOD8Ty2uf0lj9sRjf1+ivOT7qrkrsPjAo5uLsXbFNLETtCPjGCTllcuvKgxzqiFcAUxecLG2vy8xfcGE02wbRg+fz+hDBICTnwxYv6ORbwLytUN9rtEtmMpLtQZ7NHf3Na5/3eP4fQlh2dF59SsGsgdmTykXXL2nkC8Dr4GJxP6exNmfdbATBVtKrN6TmD1lNysEcPqDFpt3C1QXFno34PXfqbD40MLsHC6+kzP8KwZlyQGj9JRy3QMr3M1iRVApRmA0ee2x/JpCcc3XXSx5XUpLyW2QwPFPO9hKYfM201ezHRndKrLkzYnE2ft1TJDNxooNXTu8+HsF7vyrAet3De7+wRrX356PQwNhua3ieoB0TJ+VXUB3rBh2djNg9yhDsXQwWxdTWB02jzNMX1js72qUNxxQBQ1Un+1x+RtzzJ5ZmM2A+n6O9kSiuPGoLnrYUo/HJUlcGfAjYDYW17+cY/rMYfbxDt15Bd1Qnlpctdg9ruCMwNEHO+weT5AvLcymgzcK/SKDbrh/xUWD7rSA6jyaOxnWX5J48L0aPpPo5xrFZQcIgc27BY4+rOFKDVsplM92aO9NyPi6QF9oBJXlyz22X55h/mfX2HzrFPMf3aB+d4HyxR5BCDSPJihfNfCZQneSRan/gCAF9LZHd1Yiv27H+pDuJOc1ogSqT1fw0wJ2YsbAHZdL9phu7ejjzC72cPMcet0yfdU69I9PmQ7bDHCVYShR7BENRrIHc9fBTXOoXQc7y8muZvQaCst/PtOQvcVwVkFtesi2h2g6uLM55LqGO55ALfeUvtYd/KyKybP0ZgYhEAoye+pmh1AVEPuGbOktVi/kBqJu6bPsB7KYt8J+xs5M6xCmFSAFRNNRGpuSYlPIj/MH7yaAsNvTW5mYVUMQHNrYg5RkqV3P+wFkQ5WKDKWM/ZUHf6XQGr6uf+Z6PvgA8Rd1Wsb7wf8bAOhf4vZzAyx/7R/9tW/3d7/33/+NvzbgCw4si/cehl/9n/4rhvRsJ/BeoN9lMNWAouzR1DlsrSEyDyEDfJdSsiS9jYWD32vo2UBWxokxpEY07CFUjRw9aTAeQgeEOibxDZIezEFyGhnDT0LmIRtGuKfJe4h+KWZpB4hWQg6c0puVGrv6hpmHamWcgLOrUNUSbsqE2TRlhmTFxDDlxFs1AnZGL4hq5NixF3SAL+LkzgRkVwxmcRWTbCHJCsADduEAFUb/YDDxuaOPzxUB/nSAvDbsPRwEzFaivWehtjT++/st5PMCrvJ8fROP7JLPicCFsnBRfhd7AkNcCA/zEJNdA8xGjlN8lwfYBdkRvZOxT+3QwZeORT8Po38yyeS8QfROMdQmsQ5BAq4i4EwTYgigP7fIrjTshIyIrskqqP5NpiOl1rJTMEqRizjxF4jJtuxrNFsyPrphum3y+pkNf67ag8RPWsqbk/fU7A+hG2Nibg7oHdDc9zBrSa9UlSo1oscrvq2TvFkOZDWKa8rfklfrNqsBkEnRDb2E+TV/6TXPDeXaYmQZgkTs2uR5SH69JO8VnkBV7ylb7U7INiWfnSvC+DpVf9hn1ZGtSCwMgNhXyd+n4JR0rJPvsz8KqF6y9kIOGCXfdsJt3p6Gp04++BjMEg5Mh4sMhssOU/R0nSXGSNdc4CZfWRB8HspKD6xSSkBN7IiJfraUniw89/92J2Zim5KkNvn/kmdS14fjm1gbnmOMKbc+O7A6SQoqhwNLwYANjN2PwvL18RxHX58L8XWJ8fgNs3geEnMWpdnScmFrKzGyAqohK5CYJPCjkOxBZLB8xvOr93HfooSPibgYPXX9nPLgdA6EZ5J16l+8XUuRwn3MLoxMHEDmwmzDGAYkB4yMlGojoxg/s1MybArJYTckDp5SG9+v8fwLy/RdeMS+PoyVQreDaCA4UBgmYjz/Y/q04rV4+/Podj/iMBFjx6Q3ZPLG9FpPtop+zNsMGBfbQfL3QYJeL5HeayF+RjIAKF2bug7xfRk7A8dUXTEG4JiGTNXtAKN0PQbJ86KGdPCj5HcIYx9wSrlN73/ho49Sp4AeJsWmpGOvxJgwrPoQ7QEhskd8vsTu6NixmLyHMibvinCrH/HWGtbFZNts72MlCcbKE685NFAxWZmdlPzalm/KU8dE2sjMqdZTEl2pMQGXEmM5dkqmhFteR2SofCYPn4f9YVuulJBdTJfNokrDhluf42GUwbpcjqAuVbIIF0ZJsxo8/YzRH6y3A+vIkD5f+XsZuyGDEKNnMQE2nzG9VXaOKbOIz+c8XKlHKe7hQKd+0Td7JhFu/Xzc13jurCeT19l4X/BnmYJwfkycFdaPfZPSejJiqScyAb6Cyb4Yv/bRJ2mBuB+3Oyp5zcv42cXH8eCAIDCG9gjPPktIjGmyIoXjKMmf9QOBmxCHZNXbYTW32cJbnZTCecp/Y1hPul/QMYQnPU7FfkuA0tnkmbzt+wRuMYt+BHzh82wkAKHk4XcpkAc4yHmtxeitBJCkqmNYz+1qEecoif2LUmH/TbfPpcX+bZDC/gJY/pzeZl+9F+7/j/8I9xZbWC9hlMOTH9zH/EsrFNkA5yUy5VCaAc+XR1DKY5L3UNLjcjXFW2crvN7MkJsBpbFY1SUWVYOzco/eK/z0xV2EAJQVH+M831i5GbDdF1AqMJ3VKvSdRrASp2dbbOscWnu0rcGk6tC0Bt4puFrj/MEK1zdT6Mzh0ekKL1dzHE0aXC1nyPIBubEIUUYKAEIEpkwPCt5JuF5hctSg3uVQmpLPPB/QbAtMFzX6XiMEgUnZYXk9hco8009Li6NFjd2+oDKuU0y/bTSE9jCFhTEO+1cTiMpROrjLUB03aLYFVOaYHGslhPH8X3no3GLY5PwjrT3gBWWZew14gfykoTRya6DmA4OMFjV2z+Yw5w0ena7w8cd3UZ3WaF5Ooc8aIAg4S4ncGN7jqS8x0x7eS7i9AbTHl966xKevTimFDAKqsHA7w2Tcmgl1cjLAd4pJsJ2iV3Y6wC9zqOMOWT6grTMm13pwuACQVc78OGSAAETpEFquFPVsgPcCwQuElnJK0ccU2z1TZPVKwZ5YwAk+1gpgkBBeABOLMEgm74oA2cbwgsrz67MOuMyZ/uoYWmTPmBoatIeI+yEs01rhARwNwIZpt76IgUbGQ+4Ut1vL6Dm9NewAABNllQAZr1qOQEO1ZM/6hUfIPYpXGv384O0NSfJVOg4l8gCfe8ocJUZ/bVBkS5NcMUhKJl3lx0WerAmUXQHYuYPayViyTtmSPRmglhE5pJsAfBGg1wcZpDc8j7IVgOeAoF/QA5tSWSECujsOaqvgc49sqUZJo5t4VE8V2nPeN6gQk1bj0EPG4xQHFdJRbqi6Q9CLyznMYSovxqGG6pOPlcMFOwlweUD1Qo4VMynIyZuYjJwHFK8lulMOeYKOEtMAnneB0b+qazEOVuq3LPJLFV8/rxGfUwXgszAej2zJc0WpXRyueXEAOJJAn15ZDnHMVoz7KC2HRkkGJpwYZbRjeX1L0D7e4n6nYwXE6py4HymgiHUY8ecZHyejr9VrjFU6SeaY6kRuS3Bvg0jZcwiQxSqYdBxlBHXJCxo0wd4oa1QH8J/++cgGJ9Ape+6bLTECrmxNUGMnDOVJVTRjLU58vcIzXbVbiMOxCocBgrR4A/im7Y8DS1Au6DWHK3p/C9yawzAphfDYMkoM7aE2ZQTncdiRALFuCWjTgIIDGDGmawrLaiARvaghglmz4/uxn6fhUxwapEyPzw0QRsAXj4twQL5JYUHhIIEFkG1j1U28fhJwHd8P8TyOsuAklfQYZbEiSjlT2E+SxSaPJ7eRPlOiF1aLg2fTBlYqTeUIgFMNTnntokf0cOy9TgA2jAMUW8RBaxzoJKmrbmJCbKru8DgAsIQVoi+3XbAD9La/VTdMnE1yUR8lnqyLkdGzH+I1JUbWl0FQQD+hl5bX+uEYyD7AFel80C9qS/GG3FZFVj1EnMprKYzX1+h7Tv7cIQH+VGEjY2VNrF6JAJmydDWGD5mdGwdNQVGaOtaTxF7NlPrK9weDfKQ7VL3Ag4+NHtBRzupD9NbKUf76hl9TMuAHUkDWPXxhIIY3AZCsO/hpTiAegefYZWn9GOYjolQ0jF5MjDJXIIHa9GEWZadJuprA5u1019s/B8bvxecYxBEA/tvAnfeHZNrP4wXv6dNMz+k83vBCfr5+5GdIXf9/YZDEpn7RpbDTh+HvfPuvH1j+zh/8fABL+Rff5ef35oLAbz3+GABwtZngYjOFLz1Wr2ZwXuLyxQJSBKzbYgRcFxdHePHiBMOqwCc/uo+mznBzOYeWHtZKPP/kDN//4WN8+PoMs2mDsMrQ/2SO3a6AlB5CBMyLDkNj0N4U2K1KtOscvtaQmcPNhyfIMov9TQm3ydD8eIGhMcDLHHcfLnH55BjhJsfQanz86R00qwLbpoC7LNA2GX2Jz+dobko0NyXqTcFtPa0IYhqF/YY+Nf+iBF4UqC8nCLXC9tUM9skE/SrH6tMFqnmLcJkTpKwyrC6nkB+VcI2GvswQXhYwrw3EdYZ+m6F+OUV2rRC8gPcS5sKg/2gO9SqDu8ohL3KIWiEvB4hawRQW9rIEZIC+0UBHACg/K/h9AEIQUJpeKvVpgTBIND9ZIFQO/U2BT16cwVxp1DcVZCMhfzqBDwLidQ6/M3Abg+yVQXahMT/bA59O4LYGxTMD9BJP//gh/M5Av84gdgr+soBeaugXOWQnkV0pZB+WDFW6yoEor3WbDPq0gf5phfpyAt9qyEbCbBSySw1ZK2RLvh5WkiioWiJ0EtVnGmqnkP1ZBVzlyD8uCHzjQkbuNELpUD1hWq1e8f75xznMpYGcDpCNANYGaq0IyDZMlc1v5NidGtYZ5XlLNW5b7BXUcQe1VUACOTkDndReYvJ+geqZIrjZSUp+t4rbLfkhX1xI5JcKwgqULzSqpxroJVTDf9UT1mborcDkGUFMf+JgdoI1GiDQK1/FSff9GmYrUH1qEDTBCowfF+E6esDMho/XO4HZR5Is+L0Oix9IwAruU5TaVi9FPB8ydipysV08zRBUQHElkS0JkosLiclnKiZuSlQvJcpXEtUThexGsnLBAmFiuY0rie6uRRBA8UIjW7GXdvKcUltdC5ilRHvHw5cBumZVhvDA7CN1AH6CYKm4EeiPHcrXlCCLAOQ3PDbVs9iTu/BjoE9/5NGeckGXbQ7Mn7RkeIc5g6SKK4Gjnwpka1bi1G9ZuCmLx/MbgXwlUFwKzD5jIJYtgcnz2BWZEXimFGI5kEUXDph/wG3qWmCYU1kgAjD/xI8sf/VSwE48ps8CXBGQrfmzFG6kazEG8WRrEVNdA/KVwOQ5gWC2oiJBRNBt9gH5EsivCSamz9OiFkBgzciYiJoD1UuCP0gy8kGxl1X1DEmyJYEYO2IDJi9iavANlQTzTzzMnpUk3oALtchkJlanPQtYfORGBnOYcz99RllpAn2pl1fXrMNILGYC3qyywFiPMnnpMX3mx8qR/ihWjBwJdCd8rRCU8LoMOPrYR2ArMH3m4UrE7lKM6b0E94jHE+gWlDBn2wCz5YLcTrhQnzwPoweQjGyI/boBk1cOug0xKTiGZNnUKRpGQAXwGGfbgHzlx+PSHVMSnS/9CHaLZYDe87HDVGD6nGmp7Sm3XV14qC6guvSorvwYfDRMBLoTgeImSr0zAkmzDyivPPJNQHNCplbXQL72KK89sh07VilP9ihWHkefDMjXlMhWVw7ZPsA0BEomptkWSx+BV5JRE1Tma/489ezmW49+LjBMCKZczhAgWwoUS4KZ6oI+Na8EipXj9b3yt9QEh/5YU3vkawdpA8qlgzNAcyKhG0+57rVFu2CdiKk9uplAP5XoJwLFjR2VB9nWI1/akUnOVw7NqWaYUn/oyjU7N9aDZGsbgSeZT904hhqtLLIVpaHFTT8qGjhQIautdw5BMRE3KIH8ZoDZDrfAP0Gr2XtUzykpYUiahwgBqg0x3MzDrAfoxsJserhCwmwHeC1QvG5YR7Lp4TPKVrMV5Zaq98jWPcymh50oyJYpu3rTQbVkSgFAdg5m1cIVTKFNNSGi99B7C9k5ZK93VC8tW+h1x/PTWIYHFQrDPIPqHNlGH6A2HZnd1rLOpbEEkgB/FwBZDxCDgy8MZD0ASkSPJFlMd1yxUkRKMqIgKGUHJiB6BjNhsJEB7SHaDqKzkHULWbcQ/QCx2TMoKLGQic3cNxDWQdQthw51y6/j/UQ/RODoKYG9nQYbtxPajv/6nv83DULT8HHWcluDRWhbhK7j984hWMvkWGOY9BpCZGQJKuEDhImThSF2fir15j8pIZTiPyEOX6fvP/9PyZFJ/cXt5/f2hWYsp1+9F07/238KDBJnj1borcIvnb/GD17fR70pcHS8xy/feYHv/enXIbyAPmlhMou2yXA0r7F8dgRz0mJY5/TyATi/v8bl6yM8fHCDl1dHuHe2xqvLIxRVj/pyQl9d5QATcHZng6unC6ijHo/OVnh2eYzZtMH62RGC9pje2WMYFLptDshApip3kJeMO/WFhzzt4NYZ8tMGSnl0H8+BBy3cXkNPB7hljlA6Bphc5RCnHdSzAsOphZl1DCTpJOtU/nwB96BDsBKm6uGfV+yWvCkoQbUCoXCQuSNwywNCGb9/VSDogHDaQ75mpUkKI9FrDXs2QF8a2BOL6mOD/ihOj79cY/p7FTa/1QAvCsghdhFuNQFGDOCwkwC7YJefm1vIWsEfWaibCL62CvaUHYbVE4X9VziRMlcGtkrdeRLdqaP0VAfojYSde+iNxHDiIFsJ1QqId/cIn0zYkxirV1IqrrCsNZGdxOS5QH0vjIoor7gY7mKia3vC12hnadpJX+n+HUfpWc2AEntsMfnIIChWU6hOIL/h/ULhIGqF8qVCc48VGfmKzxgEgYZqmRQbVIDZKOTXZDdSnUWSG2YbMkXla4HN1yyKC43+yP9/7L3Zr21bft/1Gc1sVrv7059z2/KtW64q24otF4FYESEIRYHgBzA8EIGETMIDiD8gBMQLDzwhISCISERCRAacFwSSnUBQrNiOm7Krv9e3O/eeZp/dr3Z2o+HhN+Zc515X2RGpInbhJR2dvfdaa/ZrrvEd304kzleadk9krFGJ/w5IICRy83nYfweu32YI2ehDPtzUM3vPyjpdSrR9rnET6Kbycw84fCGSU5I0t7zQZGtYvSJ+0dlH4qE7+KbMgNfHCDAYyQDLVsLadYee4syw93tw9UUoL2UQp1vEazoK7H9Ts72rKG5ke5tDQAn4W74RBACuxOPoS/HibW8rYd5MmuUOitGLyOYBTD4Rf6JpxFPnxuKFK67ED1jfkp7L4kr8atlaBt6rh+J/3XsH1o8U4+eSlJptItu7clzmH8g2dnPxhW3vJslwFPADUimim11IRjSK6rbUjegusr0j19PoIrJ6pAYw0cse7VaAweo10E6RL0Sq2e7HdCwSK5KqH/p118cS2lJeRrItLN5QzD+MbO9Id+PoUlJY6xPxs/bpvsW1JNbuvQvNofQq9lLQ+ih1Li4i1bFUvdiN7Fdz0BejC2DymVxnq4d6qDwRUKeYPRbA0x+fbAnVbTj8lvgOoxHw5cYysVDdkuPbJ192U/FBlovA8pH4/JoDRb6Qc1BexiGURbtUYdFKL+P6gWb8PHVd3sQhMKWbyGBZ9lk8cz4TBlN3DEyfqeMuSMWLD7I6FGDf7KtBjju6FI9fO1eJ/duxcdWJYv7Ys75vmJwGtidaEp0vBPz07FrvofO5GnoNQ6bY3krVDE3yiG6EmfK5yGrbmWJ0sUtFjQq2t8VvuP9+x/qeZXzuUR7W9wy6g/ImpO5LkWxuT/QAtE0rLM7iVUu2EZCfrwLrewa7jYyuZODf7Gmmzxz1gay4730MVhjoyamjOraMzwU0rR5Y8nXv6TTYVFXSjSTRtu9WHJjBKLLV6sAMrBhBgGK2jVSHmmwbPyVB7mXFpoVsI99tzZ75lFw3GkVx49jesoxPO0kbrRwEcBNLNFAdWYqlsITzjxrW93NMB91IYZtIvvREraRT0kfsJtDumSHFVnfiFy0vWpqDXJKiJ5rZ45rFGyNsFSgvXfIEplTZpUd34vN0pRyDfOlo55Z84WgOMuzWY9qAKw2hEODYHGboLjJ6sqG+O8ZW0js5+XDF9tEMXyimH22o7o6xG0+2bHDTnKgV2aKmujth9HyDH+cCwAot739aS5Js6nz0452vkxgJuUng0aE7jxtnyX4gn+ls0eFLI32bU0t5tqU5GeRwibsAACAASURBVFOebgilJWQGk1jAdp4JeF11uLGluKjoDkrsqsWXdkiZ7RlLXTmU97gDkUeoVkBuu1+QLVpikhDbRUPIDWZVD0yYO5pIzYgLuL0C3cjPfUUIMDDI5mpDmI+GMJ1oNbp24mfM7I51NAq92BCmY1G5rCriuBRJepLcRq3FC5mnmbskoUVrYQq1QjUdsQdVSqGcJ4xLeV9mUYv10FmplhtJaM3sADrjdAzeo+p2l1xr9C6x9jN1JXFTyXqzTABmkUOIxOSRVHm+81saLWyl90OqLCTJ6stgNj3/qUfPlmr1XRnNzz5+KDyW0/vxp3/sr37fl/t3/+Ff+6e+b/DHHFiefOEo/jP/7b9J5TKeLecA5NZhdKS0jtOrOaNRy2ZbUBTdICn1XuMbw+xgy3ZTYqxnOq5ZbUp8Cv3pVjlDIqQXSWQ5kw9RljnqOsOYSPNiTMwC2V6Dd+nG08s3myR9GDmi15hS0itNFsTTCbL8Tu/+z2QmKp+2dFUm6ZUmMj/csHoyh6nD5B79wYguSRPREVUZmHfEymCmTtYfGcJvVFouNxlxFDCTTjox815vD4y87KuJ4DTYgFpmxEKkmVFH4jS9RkdmRxtWpzOpc1lkYCNvvnHKe49vM9mv2D6bMrm/Yn0zgkoYMhTEPIBXsszDjrg1mErjJ2E43sorYh5Q9c5rog5b4k0Osw51JYZ+TBTwtrHycxHILizBiqQ0ZmkZCunK3Eu1MRv5gotZhJQ6qbzCzT3ZjYTtuD1HdmnpDr34RMcRP/ey/EzAlcj0FD6PxCISUwVNtjDiSZ2I1NJUcnPtDgJmJZ7Z7EYPHZOmkZChbKFxr9QC9DORXuaXBjcR9gxAV0o8tynoqL7rsDdGfG21sC6mUqlr0kvapFe4PWFGVaosiNlLPlUENOZLhRtHugORofZJlO2RRzkJMuqDcZSTVMr+YSrxb2YpoVM3CjcNcr3dWPGCJl+a7mtWjDBpKAHRIv1EkkbTQL5nAHpGjwDluRbfp5bJgL5uxm7VAHp7iWPIQDcit+tTO+tjCZCRICHxfZkW2ccUmFLdCkn6pwbw2e9zsAxSUPFJySBWeizF46w7qbHpfaK+EPmoCkgo01Zha2HzdJcSR9P2Zut0jJPcsn7YUjzNUD7JipNUb5DrJUbbjQWc+5KdzzjV9ph0DABUJ8CjZ0LaeWT8QgJ53DSSX6uhZ7KbCOvnRrv6I92Ib9ZNxO+LZqge6c+tHM9dPUgvUbVbSQrN1tDOIWZQXvTbkVJHZ8Lm9gzp9l4Q9rkRb3EvGybK8txEWEz4tHS0OUie1FyeN60M6n2Z/K8VQ8KmT+HQumPwn5p6VynUTXeTMm4siackmakMEFPIUPWybC95mkPfZSlVM6aNbO4pRmeyPf36IMlUx2qYHPisn7b3U4MsR96rBhliMBJuo7zcn7pZYqwrYXLdWO3uXds4gGlJKI2DfLZPRvWloryWShvtpCpGtlOWbZo41AL1+9ofL4Co1MB2qsHLveuobA4U009SANJIJivcSKW00Z00FBhSbW0jPZEmSSh7v6XuRHabr9LkjRJQa+udhDSrIs2sV0KIXNQVaVlq51UNRpgv7YRJ1k6kt+WVbKsK0OxJaFXffdm/VwVZr2njkIhKFGCdryQpFeR68lliTksln8kkA+4VH/laXm+rdL9bicS2mwjANCnhtZcY94muKCQgaGQIefKfajVIUpVL96yJxrRRZKKjBLq2YahXaWcmrUfkoNmqoznKsRuPL80gpzVtoN2zmCagm4AfmUH2qXwUj2mIdGOLH2nKS6mQMZVINoduSC0VKyHXqU9yl/7qc42tvVSepOX2fk/lQppISLLaBNzwUWpNkny1/2yGXNJa5ftCD3LYaPTAxKougToXhq7KPqVWRXZy1l5q2u9v6yTBtU9r7TyxMLtKlk62R/n0fO+/1Fr+3jpU3RKtEbCZwOrwul4Ka80uiOczclbVJiCaqktU0zL0WYYg4LVnFPvHdwN9qauyxwrKpvf0qa5dX18SduAw1ZYMPsuXE2df9pXyGSnsHyTJfcln+cfeY/knwPKP7mP05r345f/qL7PclnStHXx5YZWhZx1hnXF4/4bFcoLNHO02F4+bSwCurxyoNdntiva6lCCbCMXtrYQBXZbYtcGPE4jzSgDXNlUBRCXAZSZ+OTrpxfTn5QDIVKcE5IwdNAaz2YElPxYwpTfC4NlzKb23a013nEJxkl+p/1IOGcRCwnxMLV/27ZGX5QaFn/nBIxdKCc8JhQy2dScBMtlCJI52m6oE2A1c/Dh9wW7VbqCYsyupj0gHZwoH2iVw7rYzZilQZ1+kdjaF0ITkDfMj8aP1vZKmVjtgkwJ9YOcrspVim9Iv3SxQnpphO6ORgXjIIiqk6P21ACQJsBDvj5vs9rPvg7PrtF25fNnk1xI24nPxwrXzOAQNgYCX/EaOWXElgGbweOXClPlSBlb5SlEfy7aIPDEF08xlGfJ6+YLypTC75SVsHkRUJ+sMafDbh8D0Xqy+6zNqAQL98X0ZdPRMpwSmSB+pqWRbooHmIFJeqEG+B7ugmW4iA/s+MMZnu2ARUzPsoy92ITt2K6xOH5GvW5Eo5kvxFfWhMv0+BCsDIbsWIAE9Syvr7gNiepYMdumg3XTnfevrGnpp6kBB96AyyRW7mQzqdfKA9Qxtfy0IA5VA6igdixRs0/tISZ6wdr5bfx/s1PdERiODl26aznvRA+gdmB48cjAMRiQcY8doBiODbz9SKQBEmKji+iW5Yvrf57I/fXVB1Luahr4+wW53nysQ1tG0wjRqlxj0xC4KON2FoaiQegpL8VK1e0rel0KJ+roD04lMtJ2lz/MiDgEzPbgR5oOhO9SNklfsRuSQ/XXQe8X6MC6ZxEmDaJXuSSFdZ1MBP+1UDcd2CGwJO3Db75t485IcdCXgTruYOhllfbaONPPkHUO2se8J7cN3JCxLAFi+kdcrD8VKmL12ogepczACWqS+QTyLvbdMe1m3fenc97JUn6lB5tiHLfWfh75P09Zx8JjqBCj7aghXpmCeBJ76ABkJ8xE2u99nlwBKVqXgpgTw+v5NV6phvcO1lK5jl+ooUIhcFKgODeVCgFhUUCyFKfSlvLln+eTY66E+Q7y3It18OaxLJhI8bmKGbsk+UEj5mFjdVLXADshG20/ASPdm/5nuj4X0Rfphu6JRKQgppo7NFHrjonSqTg3Z2tNNzCD59IUsA63opsLG9ee3/24NhRrAnsg1RSYq17caAmyAAWiZSvo5Xw4rCpkWD2EC1STAFFKvo0kS0agELLlplvyFWnoxE5ALmR7CdtqDHN2EIbgoGiWMXYj40gogDVHCbYz4avv1BatErjrL5Tw0XsJ7Wo92gW6ayXoBFSN+ZDGbDjfNseuWUFhhKAtLH87jxxazdYPfMRRWZKe5wa6k81L58KmwH5Qa2M4+qXW4x4ZAKDPMphWQ+RI47QOEhAkMEAVU9rUdIbdDIE9UCuW9BPL0IOnl/sUe5L3cWZlZAY1GoeoO5bz0VvasYQrmUYn1Qylinsnf0nMxs/I+o1EuSWLrBqwV8Nf/H6MkuPYs4suBQDYF/LzcI9k0nwZ+w34YAXM9o9mDvpiWqc2OOeyc7K/Skgir1A5Yvlw10oPR74U7/jGDfH4ogOWXfwDA8lf/BFj+Ez9mb92Jf/5v/ixaRT5Z7TPNW0JUvLV3xkfrQ843Uy7O5lItcrbPwcEaoyOX11Me3rrmfDXhZLahdpaz94/Yf+WGg3HFB+/dgVzCTMYnG5SCps4oyg7nNF1jMTYwnUjlyHI1ZjypWb2Ycv+VS569f4I+aIheUYw6qlUBXlHMG5pNzmSvZnM9Ynq4RalI9e4+2esrmk/EI5rt1xztbTi7mBO9ls/8xsK8oxy3NFVG8c6I5jjAcUM5EpkuEdTYEbcWPe0ITjPdr9gsRsP9wlzkZGtF/WoDTqOT7y6sJeDH5n7oQQwXSSI8dbC0sNcRXRpNRASYRyhPKurLEXrSEZfCKBIU5bRhOmrY/oMTqi9WhE0K09kawl6Hvs6Ixy3GBngyovzcgvWLKXZhJACmDIyeWOo7cvMqLgz1g1bY10Um21NZYR9vNcSo0Oe5gMSZjJTMaYE7dJiFIYwiqpMgGjcJjO+v2byYkF8YukeNBOnYiM4CvtVQyw31+OENlx8doFoFJw36uaTeAqj9ViYyak08alGXOaZRdLc61NpISExMDEgObuYFnFR6kOTGXCowdKOH3jFT7QYj7qRDrS3FtSTw6lpj14r2Tkf+XKbwu31hO9vbTvytSpbh3qyIL4ohUMWPArEM5C+sTCpMo0iilTBzfhok8CYyDBj9zA/1OPZSZNHlk5zmSKS90YoM1q6SNHklsmM/CahOUVyIL7GbB6nQSJMWbtwnAb7k05sKONedJOeSpIA9eFROlhOzSHZj0u+SqOwe1Yy/PqI+Sp1/SvbLl7L8vluwZyF9GSkuhfnMljIJgmJgxjaPPNlSC8DPEcY+SU3HzxSbhykMaCyTJO2eyKDtOk2oOJWAq4Qc9WxjcxTIVtLXmC8U2weebKGHkJ5gI/lC5Md+xCBxdSWSmLySfdGtBOb06zE11Lckzbe6I1LhfCFy1/o4MYBWwD4RJs+ELepTUX0pYNlUScqbwmh6cC4S2Yj2itGpSE6HcI7kjS0u1cA2qpD6Do1MHoSMIUQmFHLOlBMpb8/+5jcJOKaxTA9Ii0Vk+bpMVtmKgaEuLtXAjKkokuvyUqpCyisBy72st0ugszmQ/ZR+TAFvUe2AeTSQrcSvaGqRDNvtzufYzcXr2RyqYeIrW8vvdg31iXgou6kAwT7xN1/uwmfa+S69tbiR66oPkDGNvK7ZU0PoULYSMC8J15F2X64lu02+ysTE9mxmto5s7ivKi0ixlM9BdaKTGkD2uZvK+rJ1HCa4em+dG8ukgC8V47OQJgqUvDZLbGYC/tHKPmabOPgk+4oQu03X1VrOqysFxPpCDUE+biTnu6/U+VSabZow6gOIyutAfaiHsJjiZtcd2Xv+fMr26gNzsq1c7/WBItsyhMF0I0WxEsmxK9XADrpSk68DzZ7IaX0Gzb6muBHfZbGQepZuYqTCJaX2Dmx4lj5TbSSrAj7rmV4B0TI5EWinhmLhWbxqh5oTFSBby33IjQTsqAD5wuFLQx/YJMdSMbpwtHsSYtNNzY7RTF2SthIg3U41owtHyAX86y4O3zUvfwaUl2CePrHWlxq78YRM0l/z64b2oCBkcg7KK5Hj2q0fALH2aaImgeps0eJmWZoE3q3DVkFCeGaZJMu6iBul9Fz30iSwFzY1ZCmFNgHkPixpSM+tZSzjxlZ8pKVJ5zpKuNRIwL7yAd0FqZHppNpDN55QGuyqHdJt+6RYXxh06zGLijAtxAvp+4AqhU6BPXrTSLXJpKCvBDFr8TWG0qIr8SHGIgHGPmE2gcvh740TANhIuqrqHHGc5BTJKxmLXBhIGABbzDNhNGOU550fAGmfGqu6TwOyuFwNDGQvYX0ZEwzPOYcqix2DCcSqQpUlsa4F1LadyG/rBpXAaN9L2YPNzz5i8sLKfqhPg9fv8fhhkcJ+5Ut/5fu+3F/+tf/4D9w3pdTfBP4icBZj/OL3fQP69fxxBpbzt27H+//5X6Xe5kxnNaO8I0RFjIrFusTagDFyga6vx8wON2xW5XBzVhqUiugPR7T3WuxZjr/TMJnXrK/H0GjMXsvJwYoX757s0sKOGqkuabSUjSuR9Jgriz/qUBsrEtAXU0kkTQmmGEl4jbWBLFDMGrqnE+av33BzligbrzDzFn+T7xJJbRSPZqKf7jy64vJ3bxEe1sSzglAEKANqk9JLe0mrAlUZdK0II0nI9AeO7CwTiWIRJD210ygTYWPRWwE8AuI69NgJIMwi+sYyPtXEryzYPpuSnVS0m5zRBzn1LY9NCZ6hCMSZk4CgNBiPqb4kuzaER7VIZ4HZvRWr05l0d76Q4w+gLsV7EkuPWVqiiUwfa5ZfblFb+YI1a82X/vR7fP3JfdzWYq8y/CjswOMsyLErPfm4o3sxIk48qpJQl6ihe9hQvlvSHgQOPn/FzTePcDPxa0oHphz7aCLZQiSt9T0naaWpLiOMRU6jGgnLiRbCXge1IbsWD5CpFO2xJ+aB/MwKwzxv4bl8YZhKYRpFdd9RnBnpiZx4CVNKA7cu1ZBka0V7KFJT3SjcXEBl1JK6SoDi0uA/vyH72gTtoD6Wrrt2X9jZbi9QpBoY5aE9dpilxR90mBtLttICtBAmujwXlmn7Wsf8mxnrV4WJLs9lYO+m4oPtAalpRA5bXOy8nypIcI2APlmuzyNuuvORhqnU95iNHuR/IRPg4ZLsNr+WyQ03Ewleuy8AvbhStAcCEqt7Drs2gzcVGMrT28NAcamHhNGQx2FwgobyQrH8EUlTLS8U27sxSSUlLAkE1JSXwraZWtjgXj7qxsLS6k4ktzb1peY3UuNi1yrJ63YD4JeZ374ovTkQwNpNhAUsrhMwPWCXpKrBlclbayLFtfSXSv8qNMfys1w/Ed3Ie+xGzlW2geokgVOf2NcEnvNFSk9NktTNvcjsQ/HNDlUwkaGuxec75stU4ok1qU+zl60Gu2NUVRCZb3kO1S0BI+VlZHNP5NT9sXXjSL6ScyjBJPL/y3Uvk+fiyfS5PO+SXFN3O5Aj/jkBJj1YmT31XHzZiEw2sd0949/NYP8dkUpu7qmhzmX0ItVtrKPITJ2AsL4uqJdB+lImk8qLHQCTipWdmsC0CGCZiU91e0tkozHVbmQrGfRXJ4rxC2Gt80XENpKU2k902I2ATF8Ksxq1AL12piivYmK1Zb+aAzWk0yrfgw8BmuMXCdyFnYy6m6gh3bWvUekBYV9LIqx1AuZ2Jx0dFBQkz+dUpJh2Kyzy9LnIOtupsKvbEz1UkeQr+VyOLhzre5mAwalKgFf2t9nTlDcpZCmB4vLGU++ZoftzqDCpdx7PfB2G5FqZHIhD6qsvFN1IMXvqqA/lHlIsAtWRGe5JKsjxHF0LcOyX2c40xUIkkb7Q1Ac7CWvPRndjzeRZSzezEphTCkjMtrtzUixSoFQbqI+zoUZExeSn9RFbedb3cuYf1WzvFtitgEEVoZ3pAaD27Ku8N2AqOV7tnhUQaNM9J7G+Pdu/uZMxPnM0e4bJaUOznw2sbzRqAINSQ2SGn91IS99qlRjgBKTdxCSQqsjWjm6WfJdapUAhPWyvqXy6Bj2hMPhcky87OQY+4maZyC0T05itOtr9nGzlBiZWt36QxPYgMFs2dHsF0Siym4aYGezVBnc4QdciHe3Z0T5519ROpK3jfEiGjSb5KY3aeSy1RvUewZ75DIEwKdBV9ynWs++5BAYGFOchz0QK+5JvsweOqklBOn39ifOE+Ri1bYjjQnycc7nx6cVGGM6qEZCqlNSTbGvxSnovbKZSwpzCp1hLFSJsqx1b2UtuYxS2METpqeyZwz6wJ4FLOjeAShnXvvRzv8y0zv51nwW1n3r0DPAPUSrsPyVg+TPAGvhbfwIsv8dj8rm78eSv/YegI1nhyHPH+maEtgGby8Wa507A5Cojjh1Kp9m5rUW1CnPc4Gqp3IhbK8mZUaFMIHTJM7myAnBmHaE16NzjayuBOp0WCWzyHaKjSFtvMsLMCePXiL/QHtXi3dQRe5nh9iU9DBVRjaSD0uqBLbXznaRWtYo49bt+zbUeQGzUUQBuBEKS0YwcLMQfqZzIcYdezly8e5KSqETiG0C3wj6ZTaqAiIAWUAiQXRncVHo2RUIbCGUkv5SUTVWbXVUHSLjORDx2qhJGBkSeO0TkRzBrqXtQvvfwibyz7/HsaziypawHEODhFW7PoZyW1NQkY46HHdknOW4irKepNGYrDFjMJNBHt+KDU166NokQs0h5anGpVqGXBQ6Sr5c6P/uqjv455RXZRliw7iBJmJ3C7UtIjQoKV0qnqKnUTsKpdhKv4bil9M7mMEl5k2xL/IBetr1Lx7KQEaqpJdDI1GoI/Ql5ZPzUiC/tdhxYux7IhFIAqe4UzS1PcW5wZUw+uAQEDwQImm0CSrfF69aceEmibRLrdCjybRUTELhWu+5SG4fOyn47e7aNuOu17Ks+IMlX0zd773nr9uSaLRKA6wd4cvwFUFS35RhINYUc62wtAMnnSX6tduvoZrtt6atVtBfPoQAGtevkHDGcj74Cw+cCjoKVUKF2LufKbiTkyDQi7e4DgV6Wr/b1GdmKxFCw85O2PbMj3kxTK2LyigJDNUQ/oO8ZQ5vO0yA3LuQ92ieZrGXwG/bgy7Q7UDg8n3orTfsSY5SYKFfKvmUr8YSF5JvsPZnQM4oyIVLcCJPVB6n0nZCmTT2RaYzgc/Fy1odIYFOapOjBUy/L7ztKfbkDP9lKmJnep6iT7UclGb9ImnesZQ+OerlsSBJvN0k1Hf3g+aWuxX5gL3UNwpz2D+36AbqAzR6IwY6l6WXWUmEir5d0zyRht+J/tNskAfYM0tD+WndjSUjtjxdAfSwBVf0+oYTtLG5k2cOEaAKF/TXW14m4kp1SIoHrftKj32eiHNN2T4Cd7Be75Odqd46lzD4O10GwMkmQbRDZohJJrivUsI7e62uSV1Ou8R0jtb2jGZ8KyO+PWc9O5hthXPvj3Psvh/5L2zNpadLEKrJtkufq9PtnOjl9IUmxPYvsM5EH++Rl7bs4bd2fC5W8qanWw+8kx/21oZ2A1/57o68U8YUEMfXXmPgud8fG9tUdJr0+14lxjEOIUw8MXSn3L7uRqg0BQjLmCbms21Z9V2fyImowdcCNzKcAbLNnKK8lEdWNDKYWma/I0KUKpE+ZdSOTPJpGEnSXXhjnvvcyCkjWnawnWpHyutJgKz/I7fvzLgoCI2FAGoIR0CmeSJG9tnu5gM1echqjAL02DNcQSg1+Sd3sqkh23s9ehqzRrduF4yQQY9YNYVKgOgkI6tnJvsqjl8vGTKO3HTEz6Lod2MeYmQEkymfGoJtuB0ITSJQqjrADcN4nL2QCVD6ItNX5HdBsuySv1YMMmBh3ALFf7svrAD5VMaKVLPMzvkcBfAFlzY4R/cxrlFLEvl7kM/7K/vhF7/msb3MAnC8zlf3v/c/f6/HDBCy/+O9935f7y7/+1//QfVNKvQr8bz9IYGn/8Jf8v38opf4j4N9FhghfB/4d4C7wt4Ej4LeAfyvG2CqlCuBvAX8KuAR+Lsb40R+0/JHtODhasT+q+dH951Q+55U3LvmdxQPOtjPe2j/j6XaPx+GAn3jzA752dpejyZYYFcejNVpFPrw5ojiUDks/6vjS3We8PTvl7z1/i8vVhB+795RlW3KxnRCjwnnNaj3i/r0rGmdRKpIZz7ou0CoyLlpOL/Y4/NwV1niORls+udnn4f4N7zy7zZtvnHK1HbGelUxzh1aR1c2Y0cGaMu+4uZlQjlu61qJ0ZP76Fc4bZmXDpsnZ1jnWevK7nuPpRpa1Lbl7sOT59ZxHx9c8Pj8gzz3bRYYeO6azmgiUmeN4vOE7H99hMq+ZFC3LbYnWEec0zhmO9jacn83BK07uLpjkLbWTy2R5UmKCpns8wf7YgvbxnPxkS+sm3H54zcW1sK5Z7qRn+MBICI1X5He2jMuW67MZt+/d8OLZvkhaFdijiq6xKBM52l/z4vEh0ztrNosR072KrjNMRg1XT/aZ3RGG8/jtC148PWBytGXzYoI5aPFbiy49t4+WXI/HxCpjkgKXNmcTsv2abpODV4RRhz3Lyd5YYYKiuS4hi9SvNIznNdWmQCmp/CBCeXcjlTXPS8o3VyKhPaxptxnltKU+HxEzTZg7mRyICvP6GrcuaO6LN3d6smG7KYjPSvzECwi3EftJLiB9FNCzjviklO5Lp8BIX6Y/L8FGVKexG0V3vxU58H4Hiwx36MEG9Ej6W3lWog5amqrEjwL6qKV9XOKLSBgHVCmdmt1ehzYRLgqagyDgeb+luygktbd0dDc5urHUB8IAtweBmAfCqw3+Rcn0saa+Gwn315hvTfF3GuKqRDlFtxcGSay61eAqC05h9lvCVcH4iWHzVoM9y3FHHfZKukT9mxXq8Yhu32PWZpgQiHlEBZEus8zIbrTImCuDepzR7Xs6s2P4XaXxpaQGj55Y6ruSvjt/x1KdRLoTh1kaVKfwdxvarUU3wtqPnlp8DtUDD7OO8p2S6tWO8mk2+F6r2zIwyW809XEcvMjbezIx4luZLIlWgK5IGYX9NS3U9zuy72SELEl6FwLoq3teAqxMHNjb4kqAYnMYBSQnn7Pbd6hW6leaI/FIyeSETILUt4Rdzhby+vquo3xuB4AfrCx3+1pHfi6f9eYwcvAtxdWPecafWNoDYW3L8xR8Mw2YVoPuPdGKfCm1Gu1BGDy/+UJRHyU27m4cZKF2q9i84pg8tkOVSLsfUEHT7QXyhaa6J5M3xaUMiFWAbZog8aMkI74Xh4qXzX1JTN7el65PN5EJjt7vC+KfdlM5lt1EAOz6kaThdjPI0j7YShKDy7OdNLQHtfXxrjsxJpauTXLOfvJsc1++9coLRXktcstmRvKZQnVbUZ7D9q6AwvUr8lpXQnUC8/cj1S2FCmoIrjG1sHTdVICr8rB5KOurbsk25SvxbrYHUc7HvsJuXpIDj4VxNrWww+1c7aS449SvWe96LvNVpD5UdHPIr2X92TZy86bGtDA6j4nNFvl4toroAMvXNKOzKIBzLhUrIkuW63tyGnFjGF3IgHj1QDN9Kum16/uabB1xY5Hu5gupU+kmSt7vBXznm0h9oCWMiCS5XYmstd2TFGFbf3qyyo2EIdJepJrVsWx3M1cpNEt8wu1MUSwlmVY8rrtk29UDIzLgUmpV6gM9TBQqLxLd3ufqCyVe/WmSuW4C9ZEm20iq6+ZOJvUi+5rJqWdzx5Bt4rDeYuFZPcjI1yK1rQ6tBKRtI/ky0I01+SrQTQV4wCVDYQAAIABJREFUZeuAG2uafUu+9tQHBtNGJs8btpOcfOlpZ0YqSI5Fnjo679jcyShv/NDd6EuR6bqxTt2kwji2U/EJjs+cyG1zhQ+GkCmavYxi6dMkgyFfOrqUDqu7SH2QJ79uJL/pqE9yRqc1m/sl4zMJ8imuBSg2BxnFVYcfGdxYE5WivGrp5hn5TUs3yxILarEbRzvPxGLSBLLKpVRa8ynvqj+UwKHQp8KuO6kJqT26dajG090uBQj7QBhZ/DgTOa0VAOcKI12ZkwK7bvGTXDyyIeLnhTCs4z4hTS48Pykwq0aYyrEhdp4wzdHbdgCesczRtSOMssHjGbNEbNROnt82EuQTAmiTAK0Vv2ehRTqb5wIqk4wWa6DtUD4Qy0LYyL7LUiUAWnz3VFjV91amDsshtKcP6zGSCEwbZLu8l78Zdp5LL69TShHbdvgZXpqw/25S2GE7fr8kNv7j+DD///s4Vkr95ku//40Y49/4/3ojfmCMpVLqPvArwBdijJVS6heA/x34C8Avxhj/tlLqvwF+N8b4Xyul/n3gyzHGv6KU+jeAn40x/twftI6jt4/j0X/2H5BlnnvzJQflFk1k2ZW4oHm62CNGxauHV3zzw3v86z/+W/zKi9e52UgMdQgareXC/Uuvf51fOXuDe9MF716esK1zusbyo4+e8/VvPWJye8PPPHyfJ9t9buoRL25mWOvZXo94+41nvPv8FlnmOZptePrRMZNbGwn/+XCGfrCFD8dMfvSazbbANZbYaX76C+/zW48fkRcd26sxeIVdGtxhx70HV3zl5CP+zq/9FLH0ZOOObpthR45bh0uenx6IF3CV8ebnnvPB795H3Ra/5eZsAllg/1jA6ov3joljj17awdsWph476VA64DvproxBoXTkcw/OUCryzvv35ObjFXriiFc5k4cr2taS/8YU99Mr6psSXTrUi4I7XzxjWRfUVU78ZMyDn3jGyHY8/qVX2T506EaT3d/QrAreeu0577xzXwbxZUBPO+J1LoPjWxLwQB7IzjIJpdCR/FrTHgROvnDO6UdH3H/tguYXblP/ywvWl2OySYf3mniVCwsLZNcWArh7Lawt5Z0NXWtxlUUXnniVi+zzliM/taigaB60qLXcRH/yT/0erbd889dfx520vHL/ksu/e4/NIwlLcnNPeWpp9wPjV5dsPp6z9+oNe6Oap1+9S/bGiup6hL20ZBtF9WrLT3/+A/7R73wOe1jTLXMevnrBs4t9fG0onuS0r9VMvlFSH0fcUcf4vZztmy3KRIpJS9cZ4mlJmHrGH2b8cz/7VX7pt78EJlI+zWj3wyDFnd1dsbqckJ1l/IV/8Tf4vz75HMvTGeOPLfkCFm+Jj7BnMGMeyC4tbi4/2yvLv/Lnf51f/K0/hVka6c384opmWWAWFnW3lk7Vi4LJU8367YbRBwXd29tBnhyVsN6jFxLEUb3akZ9a3CyS3dtgvjoTdhPg82u6xjL+2oj6J7boD+Wz6nNhie1G7fxgFrJX15R/f8b2TqR72GKf5QMrqryiPQiEPJAtDeFhjf64FKnigwZ9Vgx+Y7WxlC8M3USYyvn7cPKXH/Per70iAU8Wmlue0RPD9qET5r+MTB4b6qOIO3TYazuk0e5/W7F+VcKhumnk+MfOOD3dl4nktR2CwObftlRf2dCtc/a+nuH+7ILN2YTRJ3L9aQ/bh+nzMO+ItSE/s0OSrnQRyoC5nUfmH6YqkJ+6YfN0RtRRKm6utNTOfCkw/46h3Rdmrr4VRVWQRebfsegWFj/qya403UHg1q8qzr4SmXwi/q3tQ4+upYd0e0+YEFOlQfmXb/C/uS+M776AZDcV2XW0kb1v2dQLCM2xZ+87hs2jHStVXqiBCa5OItMnIrdsDgUAVw8ckw/twJJt73umjyXBuT7u/ZaRvffkGIzPAmc/BdOP9S5QqpMU3j6Qy5dw8O3A6qFIo+vjQLYWViZfKTaPfOp8FfmvH0u1j6TgxgRABchGLeFhoYjMf0+u0cufkH03ayMAODGD4xeRm7ci04811a3I7CNYvAmTZwIat7eFaW/3klz2Urye7UGgvNB0Uzlu8w8YQpfaPTUEfO195Fm8mgamiSWuD0W+PT6VqpdmH4prYfndNDD90AwhTi5VwGwfOubvWCangepQgkwkuElA4fXnNaOzHtAmRnMZRSUwgtVrMHkilSftTLav3VMpHEx6Lqt7joOvGwlE0nDwrmNzy2ArqA/VsP12uwuw66bCMh+8K1JNkXHufJREkZ5ub+kk6474Qry62SawuWMG6azINoXpXN+TYza6CDRzkVC6kTD0h9+OXPy44tZv9LUnsi8hg/lH0g0ZDVTHfYBOktfWkdFlSFUlmtGFJOxWx5qD9zoWr2WMX3g2dw3jM5Gp+lwN/aLTp562B46bkOpQMgnSagQcz554XKlYP9DMHod0z1SMzzp8qdkeG0bXgS4xsPWBYvIikK0cm3s55bWAUFsJyJ0+c6zvWiannmiRHswqMjltIEB1O5ce4EvH6mGeenBrVq8Uko5byQTI6EICjEztMV2gG0tHcn1gGJ13VCcZ+99ecvP2nPLa01fc2I2nnRvyVZBQoGmGGwlADbkiX4i3s7jpqA9zRmcNhEh7kA8hTqb24o/00ndprre0d+dpgAquNOSLlm6eJw+7ZvK0EoZ2Iuxk1ApTdcJ8KpGm+tJKemxiQgf5qAuYdUPMLfXtMcVVQ7Aas2lojydoH7A3NTG34sfMLbhAmObCaiq1+3uMQ8hPGJfoTYW7vYd9sRC5q4+8XGeiugQCP8OAqs5JCBAI46mUgD6AXhqb3jMEIMEOZPpAbFuU1iKhBWE4X0p3jc5D14octl9PjIP8NbZy/H4fc5rW8/uksH9IMuwPhcdycj9+5Ud/AIzlb/zRYCz1H/6Sf6KHBUZKKQuMgefAPw/8L+n5/wH4V9PPfyn9Tnr+z6nv5vZ96dEFw5955QNeObhm0ZT85uNH/OrXPsfFf/cK2y4n/to+uXVsuhydBX7ht3+SF5d7tO/P2Z5NqFeF9Ew2GV9b3Ofp+T6/8Y9+hM3vHhKj4nP3z/j2r77G9H3LZlHye8sTzrcT7kyWtKtcei1Lz7ffu0/0ivq65NmLfQ7uLdhcjmkfT0FFukWBH0WsCbjLkhgU2YXl17/1Bq4xOGdQtcbMOtzco1eWq9WEX/z1n5TAGB3hozGzow1ukYvKY2kJFwVmZXjvvTuYWpHljvbdOapT6JWlzDtOT/dlkL22vP7lp5TnSQrolbBgH07gomA0baAx2OcF7753l8eXh4w/yMSjuTFwVjB9bNgsS/amFZtHnkeH16ha8/D2NaZRnF7NWS9HxE/G6Aae/PY97o8XVHcC+ZUhv9bcPViSnWW8f3oioHIqnsKwzhg9Ez9LeVQNMjhTq6Fsnh9fYmrFxfWMvW9bnr3YZ3tbsb4ZoRqD6wzRacoXBrO0YIXNiAb0eY6uNPWLCfFZSXaWMZ3WjJ4b4utbkQI7kbdl5xn6uCFbK7768UO+9tF9CX3ZWh7NrqSLcCOeRjXyQ31HVQmgdUHz8bMjsrWie39G+TTDJvmrPc/4jQ9fER/l2QgUPHl+iHpSotZWZtkzz+aRF5nv2tLNo/h5Fxn1dYlfyeysWcmg+u//8o+n4yVMVnGpMXnAbDWr8ynTwy3FtWLjClafzCleiH9k/UqkuDLCAB10Ipvu9CB/zs8s5ZXi7/zDnxIvrpUBnns8Ra8s0480bplz52gh+12CWlt8GSXYSolENF8qsqUAm+YooDeGyVPIFpos8+L3GiGy2G/NsI9LbJJUmq2wnroD3QhwcPuebKXIl4rmbCwes60iRhid952KErhi1wq7kkF9OWrF3xkgK52cUy+fBbPRFNep/zPCzdvwncd36W51ZGvYey8QlQyYTaWHJGTdwOiFks/TKGC2wjL3QTLFNbgjx9l3TjCXKVVkKmy0WQmb4C5LMJHiOhK+ugc2MLqIg1dx9EQ8xdpE8j25Lvu0WbuFbCnslEhZZfs3T2ZDzYxpVKreUKhOvGwxeQi7IwdBMXpimTwPAhajHIf81pZ8nSTqSRqbX0oS9eaBhO+YSiVJHxyMK0yTGK8Extt7LeNPDPbGYDeynbYWWXuxFJm16gRUShVGSn4di5+wuBG2T3eQ3RiyTe/vRKTVVZKJPtpIKrFJ4SG5eOvGz/SQQh0NbO+GJE8WsNH9yJbyUmbglYeYC0AtLxXFZRwYeu0EaBYXmtF5JL+Ra067NHkwEy9tcS2AE6T/cvTMMDrZyvZv5Di5BAp1p4RF3A8D4BTJaHreRbLNp+XCg50g+SXzVRDWtJVroZe7+lxY8fGZ9ISOzlJIlxYWbnQu+1Rcy/nObvSQgIyWgKl2L1KcG/JlHKpBlm8KQDt8p6WbJJByImmvIVfUtwLL10XO2O4pJp+oIUE2GihWnuI6DGnP9V0JM+u7XnUDxWUngU1Lz/g8UB/J34MRxtDWkXwZBzCQr8XKkK88tk5S40oATXETGJ0FOU461bpYlTyGvSS591mma0BBvgysXkv3oEbOtWnFo6wCFKuAbSLtHOaP/SDj7I+hCjB/7MjWsr2mDuRLn67R5PkzQIDReSDbhJSG7Xd9q3UUdjlJO20VRApbaklMLsWL6fMk7YykhF9hPG0dhuuouiWhPW6smD3eppCogN04kfmuHNs7is1dkyYJNM2hwrQSbtNLo826RTdOmEv9UkjRJqA76UMNmZy7yXMB2yjwI6n2yFYSLtPsKUwTqA8EhIgfN8gkQRsprhpWD036DujIbho519cty4cW1YlH1i4a2UYrADCm6g83FiZQv5zWamV5dtNh191Qf0KMrO9aYdCSPzNqha67T403dd1hth2m8ejGEQqD2bQigdVKPJ2ZEf/mWpJXQ2FQjcOkLlTx8mrwAb3cojqHWTXoxVbAZNVIbUrjhDH0PqUHh8HbqepOanBWFXpbo7aNdFNqLexkDxb74XOS96rVFnWzEglvld7TpH9VLb/XDVQ1NC2xknDKgWGsavmbSr7VNq0zeGLX+z/V4N0ERCoLEt5jtCyrl+t+l+oROdDfY9ifUmfVyx7NP6YPRQqV+j7/+6Py+IFJYWOMT5VS/wXwMVABv4RIX29ijH001RPgfvr5PvBJeq9TSi0QuezFy8tVSv088PMAxa05+9kWrQLvPr/FeNzQZZ7tv+YY6cD2CzWhzWg6ibAu5w31VYnu/YiNxumMfNzy0dUhRAj7He3IwHXJ82KOm3tqZdCLjPfev0N5ULPcykAQAmEp3XLBZ6Aj+qxgmUlQT1QCVkC+aJabMoE8Q3enFV9mZWi9QjtFuMkxBw2hK2hOx2L3SKCw2/d012N0o7lYTtJgLw1uN0akdOcjdCkDAFUrVlUpPZRjDy7y/rfvoR8E8WNsZXDc9wZWz6ZQBvGZREW9LLAHkvTZe7nWr3qy0nF5NSXOHM9XMnh9fj2XL7WbnP37S9bPCxnYZZFvXN3hz33l6/yfv/ol7AaefPUeGHCVJV8r2olUd+itprrrKS4N9TrHLg2h1tKH6IQFKDJH5RRuk7G5F2GdUZ8E1NqSLTVhVYhfEhnAxZWlPhYZYJ9EGU0kjAAFnTNgwZ+OUHmk3RPmZvKJYTPOIQf90Qjm4iv0o8iv/ObbjMbJi9dCq7JUYxAlcEnD+lRiKkMWcTOP3UrvpXj5IGwt9R0nEsfK4vY9+ULhkh/MnY3EV7ixA7MS0pdiwCRZpKZ5q8I8H9HNoTg36M4OPjrzoYRURW3wj/cJE/h733lLPJWO9GUfKC/MUF1gFla8iguFm0pfpc8EBLSFGTyko3NFu5dYhKXh2XsnlBdGrjsnfsyb53PxBafBbDcLLD4n7MPoqU4hI5H1iyn7JqVu1gzVNyGD+Ez2QSfPZH4j3spuX9icbiIBTm6UVD0bS5cki71fzlTCXgYD1dmE6RZUUGyfjMnWWnxfH+TCRhUysOzDbfTTHPeoFvnpgRbfaEkKxlFEJfthWlCdYvKxsJdmoyXcZSWS2PJpRujZ1mWJL1N9g0u1GyuNutEs35D02Px5Nnj4tBMgNv7YUFcF7ThQ1vL3wc+Y+iO7vYBbSMn9+KlJ/rGUZqpE7mrXinaeKmc02Gsrky9a2KHiWuFLYZ3c12Zsj4V5BDk2bhKxa42OyWfmhfnspnD2a3exBnTy6ZkKik/ywU/mxmqoXVFOBhjZUlhJENlilwB7ttBkS5lwKG4UzZGEMulOvLrRwPhUglFCprDfnGIaqc/pJjtvoKnT95ERtmz8TJ5QXs7l6HfGLF9NFUppwqj3V2sP83fskKTaTRQkj15MHtZ+X/LVzts6+0hja2Gl8iW0vzUnjwJiqmOFXSnsNlCea1QMzN435OvA/jtqqGPqfY2mipSNojoR79zoRTpWW5kgaad6YCmLG0nqNTXUB5rihiS9lvqXvXeSxLWU5FK7lfdNnsn5t5sk4Wxh9r7e1V00MfkvJUANxH9nWph9JLJRV4h8dfo4MYQbT7NNVQ3JptX7GVWU45ZtIgdf05IubAL5Sr4zt3cL8k1MyZ8wfRJTAE6qVckUWSUpoYNX1qUE0k1ge8swOZN0VxXEr2jrSLEI4u0s5RhNn0hqq2l7VnQXRqRCZO9dgIhJ8mVXKGYfCZBa3TfMHzvyG53ST+U92SZQJv9lOzeMLj3dWNPNDHYbUt2LfOcWN+Ammr43dPJMFADZylPaBNi2Wvojk/eyTyG1PpCtNcEq9j/wIi9dB46/IYE6ugn4XMKBYoT544BpAuWVopvnjM89tpJgnGwr6z36RifBO8nXu/eB+OWChfG5gFo3LzCVY3TpJaBoZMiqkPIKJC3XNrLNbixy2r7rF0C3ntwFpqcCKGfPHH5apL5PT74yQ9XJ4bc7dBfwI+mDHF10oBWH324wjSdbOvwow259ug8aiiuRyeY3EsRjli1hnPWDUhl89+xiJcPQbNWx96GMCVVEvJWVQ3Ueu6iIxoBOaa4uYDYtBHm/ah3kFlN14sOcFZhFBQFUjGTXFbHMJOhHI2xlKymwIStRTUs0Fh0iqvHEIk9sY2L7spS2agy66uTYZAq1qYZeSwX0tR6q7eirSwbWcag0SYAvJL9m50TC2rSSCAvio0xeR5XnO7+m1jsJbF/hkiVZrrWoLMlgrYWmITa79FllLdEn8Pmy/xNA611lS/o5xu/htfxMgM+fPP7oPn6QUtgD4H8Ffg64Af5nhIn8T2KMb6bXPAT+jxjjF5VS3wD+pRjjk/Tc+8BPxxgvvusKkLqRL/yX/zYHZcXTxR51k+E6g808McJ03HB9KqDnldfOefzBLbCBbNoSvOGV25d88OFt8fotrfi3WoU+bgjnZfrSihLws5TQHVQCchMPnRqCdnppXO+Li3lA1Zo4lYTWcr+meTGWEKCxI1wVUjkxkgCVbj9I8f1YmKao2dVWbAz2pKZbFFJrYSSExi6NMJy1hO7EkR+YEJ2CUoKJtHecgFiTbp6Nlt7NrUG3GqIA2Ggiauwxp7kENdztoNOosUNd5uLNa7RI3+7XZB+MMI14uHq5YXerw1xZ9P0KdzaiuNIEG2mP5UZplwY/CpJaWuxmsk0tM/mhFIbErmRQ0O2lm9heS7zO0Z0SP9uJJ782tPshfbknqVopCZduLJUO3V6QtMwzi5uIZE1AlSQWNseSQpotk3ds36FXvTcDRi/k+FT3PKbSFJdSD5FfGdojkcm1+8KC6CYF03QyaKtuBWzyxzW3pFtUO/GpVbfiANSKKzXUIrQHIUXrywDetMKG5NfiGes9bhKKIQO++YewfpBkZvOUvrvUhEwY23ZffFAH35Kag+ZQlt+nzYpULEl9MiCxJ31HYXEFzT5DPUh5ITPYfYBLtpZajG4qv4eUzBksA+izG9g8EkYvW8tkgrDRsL0fsSuFS3UVeaqWyFYKP+59gBJMI5+LFAzjhelYvinyxN4D1+7J/UFm5Xe9mqMXIgGUOoldfUeevIfdJIWopHHI+LkEcWzvyjkd2KNOrlcZPMp7TJsK4aOkqTZHkN/I3+xWZH/dNA1ck3QRYHtHMXkmYS3dTBikkO0qNARAiTfONJFsC82+AOy+usLnMugsriObe4p8AeMXQYCclnPeexSHEJ6GoQIDJed8e0fAS7aS14xfBJavCPjug4vy5Uu1Fn0vZwKvo/PA6pF8XgbmaC11GL3sEeS1phF/X3mZ7gFKGNVsneS1jWz3y32WvbyuB0eji8D6gXz288Xu+NbHAhz6mo6+LqWX2bqRbF/Usuz1fS2JqyORQFZHmmIpgKkHO91EgKPp5Fz0fZx9cJLpIvWhxlQx9cyKZHvyIgygpxvvfHbbE02xEOamvBYfY/+7SBgD3VQNaoi++3Ry2rF6lFEsBKBUxxpfCOCWbRWGqJ3LMqSXM4GxFFjT7MnEgq3Sca3TPcfKun0ptR4+V5SXIrvse3S1EzBWHcg9MtvI/bmdSjWHbSQYptmTz+Po3FEfmUE26DO5xvK1LD/bBEzlqY8yso2wr/WeETZwJufT1nHove3Db7JtQLeRZt8MVSTBCIA0neyXzxX5KkhvYqbI1oFuZvAZFMtd/2HU0E00rlRMnzlJhR1rRued9B0mXZcvNeVly/Z2Qbb2uJHGbsOQwqpTYqryYGrp2tQuir/wUEBecS31IADlRUt9kpPfOAGeG08o9PB+Xxps5Wj2M0wtSbNuaohKpKC6C7T7GcVlQzcTL2C27mj38iGh1Sdwnd80dNOdLzG/aejm+ZDQ2s4zRs+30lcZhGnTtVRwqBBRXcCNLW5q0I2ksKIUbmLJVh269TRHJdlKAJZUcgRh7FIgUDfPsJWwu2bZ4vYKsqst1cMZ5YsKN80FdAF+ZMkWjYTtKIUvNPl1LWmqtSOMbBq3aEwtnspoFWaT6jaM+L+V77uutTCMtZPPCEhYT2aGkB6UGvyRqumIKcU19oAn/a+chPJEJQnHqhXAFItM/rcavaoZ+il7n2PnBDiC/K0HaHUrADKEXWAPJIZOyfNFJjUjmRVm0ehPS0u13rGALwf3mN166GWm6XXRuZ1v8jPyVJQihiAMo1Iid+0fPbiLLy1Xm10NSftSsM5L2xjbTpJke8zx3cJ7/nEePwQ9lnuTe/Erb//89325v/Rb/+kflgr7PwF/FjgGXgB/Pcb433+/t+MHGd7zLwAfxhjPAZRSvwj8s8C+Usom1vIB8DS9/inwEHiSpLN7SIjP93yMTMfVcsLp6T5vv/qc09WMDQWH8w3TvGVkO65fzBl9kqFejxzcWzAvG5Z1QZsCaeyk42Bvw3mzT3Zp+fGfeZdFM+IrP/oh/+Mv/xnMg4rXb13yzuYedtJhM09bihdPTwLxrETfqrn7xoLz5ZRmkxMbTXZl8Q9qisLRNhZjAtlJxZfuPeMbz+/Sehn8zN+6Yf3NQ6b3l1gduH66B7GvsNCgI8XdLfcOFnywuYW+1eKWOdPba6p5zmzcUFeiU+muS+xeS368pesM4fcmuFlk/9aKTZUzGbXUbcaDwxtWTcG6LgDY3IwEEAMP7l7x4pM72Epx/OAKk8KJPin2iWcjjt64Yvm7R8z3N6yKkubEM/3I8rm/+D5f/cZr/PRbH/Dr774GUaUqAPFKbT9f0WwzWBi+8OWP+eDiiPB4KlUPGrmpF4FX3jjj49ND7v3IFS5oWffZIV988Izf3T7C3G6Iyyl/+iff4Tf/7tvEqcOfOJplDoVnPK9RCtwnM3wZiRPHaNYQT+f4Q0fILOpIAn3cIqc8qfipB4/5B7/zefSsQ13nlK+s2C5GmNLhliNhp8YedVKxHY84eOWa1eqIB58742l3m8lrCzarEndVgIqU9zaY/3vO6I0l23VBqC2YyPj+8v9h701iZU3TM6Hnm/4phhNnPvfevHlzzqqyXd1GNsaLBtTqFhI7kEAg9YpuVi1YsQPUYlzCjh2tllgjthiBRAskaEy7TZVdLmdlVWXeIe9wpjgx/cM3sXi+749z0y53WzhxleWQUnlunDjxD/FHxPt8z4TNqmYP5oMeeFXCfLiG/8dzDAsCp/kHSyxfzCECOwjtLMIfWQzRwB85qJ7smrsY8PDiFtfrCe7kFKEK6BsPdacRz3pgXQMfb+G/NyW4LgJ2F4a+oMOAMHMQO4VYBajGYYgVmtcC6/c9+0F3GuZWQXy4xdoqVD+omQIJAjDx3RXcmwZuAZg/1Ng98inFl4sL9T8yGOZkmtVOYP2pxcMn11h3JTZfHkCftegHhYMfFzD/yg02/9cJwrsttpMSeKrw7q89x6v/5R0MJw76TsMtHGwnIZzA/CcSmw9dko8a4EGLdmow/bHB+tsWiMD0dIsYBdqfzJiCedSjuqrRvj9AFh7y92u4WRwXNyAA/35L1rkKaD4vcPsrAfPPFIZzh9NHS2z+j1NWm1iB2U/prxrmMQEYepfzAoGvAIBVIaEg6O9OIxAFps8idg+Ao9+PEP/qNWb/5wHu3jepX5LAYPeeZZDSTmH4dg+8rFC/IhvUnkbYeQqxKYH4uIVfFegPaT5ta6A7TQtLOsIeechWYPqlRH/IkJyDHwls3wGqa4Kmu295xJnD5Icldg8ihgcW5a3B7onD4e8qxCkXVNRnCr4GNp8OKF6ZVB1C/7PZ0G82LOIoqRRf8b5+IbB5n5VE9tBh/gOD9ScW/ksD17AKZ/PtAfUXBdonA6qnBVOcBdC8SizmFtg8CahfS9gJMMwlhkVMUkVg/WHAwQ8Fdt+hR3nz6QCxZf9rdYkkcSbzWf/I4/ZTsvDDr23gf28K4YDdAwVfRYSnEsvvOhz9DpUC3XFi9TqRgDz/r3cE1mYFrN8LaF5K9Asu7vgiQlqJYkM2vD0h0zl9JulxPGQIkRoEuiN66oKhDFX1Au05FwmC3qfuSq9TSi3DddbvAa7hOQlapL5IeqeLlcTqfcp6u2Pg4HOy1bd/xaN+pjF5mRaQnIA7Azt+AAAgAElEQVSdsKfSV8DmA4+jfyIxHAgIr7H6kAtgk+dM2NTPI1Yfkf12U4nz3/ZozwXiNeAGlSp3BBnQQ4XuSKYkWrKsrgGO/yBid6pwuPTYPKLHbZgrzJ5abL9DD7FrMMq4dct0Xl8SRC5+FLH8UKNYEzRyUY51OACP02wjdMcOR0SCR8+vPExeebSnBqv3JCXzWx5TtZToZxKX//KAj/5BQD9RKJcW7UmRnqfA6j2JxecRN99ROPudAFuzViR3c9qpwPHvE6yvHmssPh/QHktUtwS2m4dUiZS3lKcOBxq7M4nqVmJ3xr7MoBnSM3ktsHmgUKwp+d2dqsSoG9RvBvRzCWmLsQolGAk7lSkFOmB7oWG2EdUrh+5JA90xWVdvJNpjnZj6gN2JQnmtYA8MzMrBTjSGiwLTZx3c1CBOydpvLhSZ/C8jiusWdqYpHRXA5oFG1UjYRqC69ZB9gJsy8Gf2rEe/UCjuLO4+rDF7KtAfGUA06BYKZmOwOy8w+3KH4aCAa2iL0a3H9kGB619WePQP+TmtdgNWH9SYvLSQLsA3GkFzH4aLBtVVh2AYLOPnmj7J3qM9MaiuBexMIWiB6tpCZyCqCYCGwxJm2cMf1PClhOrZmVl9tUb37hTCUT6ceznNykJbD7eoCcBdgJ8Y+GkBfdvCHjVMKr5jhHSYVRAd+zHFziFMKqCi0kl2Dm5eUVI8MI3Xz0oo5xFLg6gUAW5BAAsf4A9qqO2A0BRQb5bwR3NACajLO8TCsIZkuQEAxElNmWw/AFUJ1CWJ5OzBdJ6ANwFQsesIFusKuFtBzLjaGTeUyohJw59lkqYWhpLaugIGS++kZ+hPlBKiMBBKIfb9Hw8iY3i7iuT+7X5ibPimHXx/cW8xxn/7/4/tfJOM5W8A+PsAfh2Uwv4DAP83gH8RwH9/L7znezHG/0YI8XcB/Mq98J5/Pcb4b/5J25h/eh7/w//hV/E/vvoldE5DAHh1fYAPLy7x104/x3/7j/4a/sZ3f4AvNkfYWQMleKyldli2NUIEJoXFzbZB1xn82pOn+P6rB+i+nKF4vMX5wRpfPjuBqh0mTQ8lA44nO7ggselLXL2e4713L/FqSUO4tQoniw2UDOisxmZXwVmFjx++wWdPL/DJu6/w+atT/HPvPsPvPn+E909voGTAs+UC1iq8d3KD3ms8vzzEX//oD/EPv/gIdtA4mO9w95NDfPQrz3G1a1Abh91gsFxOUNaWgUEvD6GKAN8rwAmUiw7zSQejPL766gimYW+AXZb4je9+jh9cnqPrDLQOKI2DlAF36wZhUJgudqiMw+WrAxRTrj4FL+EvKxQPtjic7fD6xyd48ukr3Gwb9IPG0Bo0sx67VQVdWdhVCTVxeHiyxLOfnmJ2sQYALOoOzz87QywDJic7eC8xvJhg8t4dNi/miI2DqRxCkNDGwb6YsJ7EBJTzHv2qxOxki/73F5j8yg1uX85x+s4Sh1WLu77C5c0M8bZgCM1Bj/h0AnfoICuHGAQDjzpqzepZj+HLKar316gLi01borusoRYDH+MFLt65wbqt0LUFTOFwPNvi5R+cASc9IIC6GbC9bFAedpjUPXpr0HUGj09v8cXTU4itgjga2AWqIxAEGetlAXEwQAhgMumw3VbwG/aFmtoCXzY4+OVr3K1r2NsSDz+4wlc/PcHswRrzqseLL06SHBso5j2GuxKi9MDaQPYCxZMNQ4p2GvWiQ/dqgocfXeLFl8cw8wH2rmQFTRkgd4qJqzoCZQCspB+kpMz48IMb3F7PEHvJ6pqDAWFtUL3UGD5psTjY4ub5AnqlIN7dwV1WOPvoGuu2xG5ZMyVXk80XQUDMB1Q/qLF74lCf7NC9miDqCL1UcKcW5awH/mAG890lts9mDLpREaIIwMpAHvXAixrRRITaw1xrhDIinAxQr0tKk7MKaBogDtP2PhyAICA6ieP3b3H1Zg59aeAOPFAEND8q0J0HIKWqHv3aG7z66TH0mpUt9oMO5R/WaN+10Nca7pDdnyIA1Sd37L4dJGACzKWBTcFO3XsDpPEILqURlh5N02PzxQGZ8o84dBTfb7B7wg6I4oqJtMUdw2Hi3KKcDIhRwD+dcLgsI/3UlkCHHYmUHoZHHcLWwCw6+FcNzJ2EayL8wmH+ewar71iIgcoFWXj4VYHZZxrdKYNc1FZi/kvXCL91guUvOzRP6fUNml49tSXzTwaUJ/v0O5e4+9/PyTKnOpzuCb3biMD0pxrDjN4wiIjiTmL49g5+a6CvNXQr0J94TL9Q2LznUb1Wqb4nwNxIDI8smh8VDAbaMPRl8qVGv4hQH2ygfneG3TsO059wKFeWgTWqF7BzeiCHE4fyNY+leiPRftrh9H8ucfsdspr9uUP9lPJ2aYH2QYBfOFTPisR8MAAHEeyEVVxYYIUIFwu373pMnnLQ7w8j7OMe6nWJ6g1TadtH3Md+QbXC7mHA5LnE5gkB5cGPgNX7BB52RtmuHJAk2WEMbnJNxMFnEt0xWfJggPaCEsnp04jlt6gusFMqF9afOIhe4uH/xmTT7iRi+hS4+zjLeqk+6E654KIGgQigfkOQ1p6K1EkLLD4jQ9sf0ks8+wKwM3ao+iZi/pnk+TFUPJR3ZCClpXd2827A9JnE6lMHuZM4+V2mB9splQKuIUiOAti+I0ZVgNlwISg/x+Hvk9W2U4HZC0/QZshUC0/gnBnwDCqFA3xNJUL1RrBL9SoxfGWuGokJlPM+13C/1u8Bi8/IePYzifZcwKyoJCDwIyCXPZNbXU0frdkFgsjEgtuJwOaxwPHveezOJOrriPU7ErNnHv1csh/Vsgt2/pOA9lSiXDKRV+8Clh9qmB09pKv3FSYvWI3SHwrUl0yQdZUY/brrJxKzpwHdQmD+zGF7pim1Tey/6iO25woikvn2BcF58zoxshOBYhtRpzCgzSMN3UYU64B+rmB2AaqP6BcS/QEDioDkFe253yZ1WEobsXlgMHnt0C8UJi97rN8pUWwCukPKwlXH/alvAspbC1crrB9rLH48YHthMHveo01s77DQaF50iEqgvShh1mSLi6VLVSwRqifTGEqCyvt9pK5R8BU/rybPW6a/puoVCDBF2IYx2CZXlrhGoXrTIhoFO9UobnuEQkH1DA0SIcLXGsXVDvawJsBcdXCLCvq2Tb7YiFAaCO8RKgO13BFsSkBuesB7xEkF0VmEphyrSiAB0TvEyiDUhjLZziJWBJ2id4CSEOsdYsOubGgF9MO+ZiazodVe8ir815jLEAhEtR49qmP4j/d7r2VOgc3AUNBrGZ0bZb1Ca0AKSmLz8wsxso9CKYLRfxpz+ReFsfzWv/tn/rz/0+/8p3/uxwZ8g8ASAIQQ/wkohXUA/glYPfIIrBs5Svf9rRhjL4SoAPx3AH4VwA2AfyvG+JM/6fknnzyIH/9XfxtaBVy+OqAkdaMY6z+3ECuGprg6IjYe+pordL5KXX4HHupOIScbuinZlWJFpgGgl87OOci4OnKFuWMghp3yS5xeyizVift+whStz54yMfZB2oOA8o1KckwxyrNEVghohk/orRj7zbqTVOpuuA9mJaG79EVYMCClP04ynDZ5CgsOQsKllMzAwYeeN67cR819hcToAwyaaYeuiaNHNEsH7cG+fH1YBEyfSqzf475BAN1HParPS7gpPWXtBz2qL8qxly8HVMjh3pd9GlBFimi384jmFb1SqiP7Y6cxDRaUmEmL5OlLIQxZetlENC/FWCKP5DOTLvnnkHxnQ5I+3rsfoHTRbIEhdQIC3OfcLegmlDjmgSUYjF16vtpLDCkbAxCRqg74t3K4N+ykY/MF/VE+lWOv3w9ovmJc//YhC877xdv7YtYcOool5Wl2IsZC8e6YcjPZcxh1dXr9GtYDMBhif8wxpazaaQqq2NKrxX46HkN/hHFAMysOfvmazxJLvUv7sYroj9L+CoyF71Hw9TabmKSjKdBoFxH1vQ7CnhLFYc4BOoe65L7H3FuYgyGylNPO6VOyDdIXJ59vmCNJHbMklMdntpR29gci+SpjGtgFiuRXy4X3uTi9uKNsUfX0o+VOxeGAjFbuuWSYSBxTMrMclVJSjH5Odopy+6rj76KgpM/V3I7qgc1jykYBjJ8J2QuYw0KkS6/LwNczKD6OfY3J51jzNQ0pPCX3Z8oB47Blp8ljmboHXc33i0w1DCq/l9Nrwc7RdEzJ15jlpj51wQqf9jGFjCB9PnqTpMk7erlypYOtc7cgJZR2wu1yeOX++YJ9iEGRaSuXcUzK9AXP3TBL9Q+OPsPmip43Jqkmae7Awb+fc5j0qSuRslfJwXSgnFQEJICJez8nyZ1M0riwl+96k3pQcyelIPsWdPJB3iUpcsQo2c1yW9Vz8PKGf+tqslbdQVqgSGmmuWdTpLL7/Lt8nm0tUK7COGRHxce4iueGIVsidSRSxlquGIgjPc9H9rKKGNmjmCSevsjyZl6brubAarYew1TtGbJtQChFCjnhZ53qub/DVGLy2iIYSh6zzBRA6hGVUG0Y9zt3ENIT6Ue5Z64Vob8NY6+oTHLIYLjfwgVELQlyBob82DnlumazBz/5OOkLTduU3KYc6PEcDjTq1x18pUevp6sZ4CIivY7SR/hSjYO0WVu4iSbDvHXwRjIxdaJh1mlgloA3OV02IBQSsvcELl2SHBqJoHl9Uu4rk98fUK2Da1jFASngK4bnRCMTsxYp89QSdkafYg4VEj7ysZr9iFEK9jR2nu/bwcPXCcBE8DEJlIRCjcxd1AyQkR39jrlvUUTAJ1mqr80IZmTvEIrcM8mgGpmYO0pWuegXag3Zu31HZE5G7RygBB/rI6WuQgBKMEVVsypFtuybHLskheC+SgnR2vH887njKFuNRkG0w15qmjsvs7w0f59WZv+4EFLVR8HKDusoZ3V+3z+ZvY+5X1JKgrHcXZkDb1I4D0+Q2N+f/5/8kgD4c05j1Yr/HyzlqGUBOEdf5f0uyvEA0nMKSemqexsIUl4bgeDfksDm8J6Y60ys3VeShMDt3PdO/mm8kl+rHfmFB5bNw/gvfPoNAMvf/fkAlt9oj2WM8e8B+Htfu/snAP75P+axHYB/40/z/BIRf+uD38b/ev0JNgcl+s4whXlucXa0xmu/gG0EinnPxNDHDq5VQGIOhAkIpwykiUEAXsL2CtIWsEcBzYMNhh/OEUxE+46DnlkoGejdvJwBg4Q/8BC9RKw8hI5QbwqERx38RkNuFcpbifYdC2Elpu+syMrpgO6dADHQL9nLCLlVBH9HA8SrEuHEAm8K+KMI6AjZCQyftIh3BbflGBrUnfs0LCvYIwdzrdE+dFA7CfV4B/+sgZ9zpToaskK+YUhNVIol9UHwMU6yQuPIwvYKei3hUwhR9qOFuUOsHXyvUE4G7IYpLr7zBq+/fw4/9ZjMO/TTAlFHtO9bzA93aCuH/qZC+UahO3cobhS6T3vo5yVCEVMQSsIIHvDnA7aygJ95mNvUBVVHVJ+uYL+3wHDsUdxIDMcBzVMFN+XQYWcRsQwwGw3fRPQnyWtYkmlhDUUauFXEg++8weU/PsdwzNdQ2sy+KNgDfpkHk/ysa8kFi4se4g9rejkHxv+XlxLdhUeceIgt5aVwTNp0MwJxOwsM3DmKsEfsHVQtB1w/8XANF0TMRgAnPXYogSgxHFJKORxkgzsQGo/iUmM4c0A07Bh0ZFDkQDDUntFj2l04eoO/MugecqARnufCTbi4ANAnq1oBvaMXrH3kUL3WHExnEe7DDvF1Sd/qlOevuBXYPvGIU4fieYHqRmD1icP0pxqbD1zylAqEW/bqqY7A398xgbU9jbBPeky+V8HOeZ7chINudSmwfRygNzzvEAmY9wLdA4/pTxVrABYBkxfsWts99EAKN4pp0SFLM6MUGI7osZVWYPeug1orzL4UaM/5BVeDUjs7Td7KhwFh4dD8qICRTEKdPhXYPSJ48EXE5DnDN7bvpK7OCCAycMZXBJu7c3ogEdk36KsIXwfUrxSKFfsXczhRVPSuTl7Qqxo0ayGioowzyojZTxVZyVR0n32P3TGlkyJwsSUDXb1DSgol0CxvgfaMCz9Ztqo6Mh3dsUheWoHN44jDHwCbd8la2Wliv1ZcmGDyKhcbQvLcqpYsj50CvmV3pVnzPJS3YEk7+HfVdUR3SuBULpmC2Z4JyBfA6mNg/mMuIvVHZFtVL1C/oaw2e1Kbl+xq3D0MOPgsSSA3BB3FHRcJhrkYAfZwoEdArjru3+EPgduPNMHnQqC6AnqzX1zwpRglr5Tn8bW0My522DkX83y9r16pL+l/9gWwfZzO7xsyOFHTEzrMBXQH7M4FZs8Ctg8lqqsIbwW2j9JCQsTYW9mdCOw8F0LpBxaYPYujdzEYAZfSRLEEuiMG6vQLgkX2eUb2apYCuwuB+g1fc5kW7ShJFegXClHTR1suKetsT+To250+D7CNQL/gd2l1zZqM7phhWoef0f8pbVqAUvR2FusIUQhsH0hU12RBKcfVo6eWQUkyLTZG7M4V6isBVwmUdwEu+RnX70gcfJG84ZJ+z36hMEwFpi+5gDDOCol9ckJAOol+TnlutRQYJhK6J9M/TAxcA0xee6ze1TDrPWAu7wJ8KdBcAn6hobuA7YWEdGVSC3DxqFtImDZCdwzQiTIvqIgEnimrdZVAtSTALZce68ca8y+4yLZ+RzN0SwLTr1w6H5rAOEa4SqE91aivHdpjjeaNg01JrTqBcDtJgW+lxPZMY/KKjynWAXZCAK/6gPZEY/oVga2dSqg+QgQNX0o0r3sMM4NhJlHdenY/FgrdaQFp94sXeushB4/hkB7CDFSDEsDCQKYE2qglgpLoTgpUl0B/ZFBeD0mCy/tVH1De9OiPS+hWQ68HhELBzioUywH9UYliTVCt1xH2sIBuPaSPsEcVFyK6AL3LYTOUUcoEZMmmRbgFg4jcxKSkYIHqMqZFB7UP7skLGtYjNiUBtwtkHq9bwAChKSA3A2LNQJtYsBIkVBr6rkWo2c8sB8fHrtpx3+KkJuirSoiup8/SKwbx5HRVrYDCEAje92fe8z7GqmAQTowEsgm4xsLQb2odRFXSr2lTQms/JN+j2Ke/JgAopASU4rpXrg8RcvRlxs5DaC7IoO8JIpVMYUKcLzMLGWOEKMweNAP8ovrTTPv3SbD/D8DyL2/f/O0bZSy/6Vv5waN48R//+5S3tWTMoopQrYSfekr8Di2w0iiW7EfUS/pq7JSsnE/Apn88oHhhMJwTnIUiIpgIcyf3IScpfdFPKBUbe/U0WOTeSugN00MZt5+izC2Lzn3JAVOvyVAAZAldE2HWBJkAhxM757BZ3nDAHBJrGlKirRr4hcrQmgSIIoeD/piBNv1hQP2GYEbvMmNCNgFAGo73bKFrIiV1IbEbDbc5HBCABEOQETWP21cEbmbL5wkG6E896q8oR1IDpWChZAWG3gK5bLx912L2QzOGghDUcdjrjzig+iqm4Wb/c3nNYVj2HOJzuEVmPzNTWtyRqcqDPg+YbCgiYFYEotV1SmpVBK9mwyHLNRwileXPxV16TMWB2E7EGPCT2U9Gr1OOpjcC1aXA5gklX67aM0x5H6XlPnnD7WTPVg7ncQ33T7UJIB3w9cv75yYshO9OOOBGtU+GzIEoamDVwO6CFQvSIg3eSImbcexqY8cft+ELjMXsrk6s3cAFhvqKA2GxSq/lGRlWkfxTxTJ5DNOCRLFiOmpOtLzPLtjEokIQMGR5kt6l66FP6a+p7sI13OccyR8lX2M5JGYtBYxExet9DN6o+VqVqdYgM5blbRz9cyKSjfVlGrAXfN3z+4whNfvfM3GTAMJOBSYvA9oTngezSYzjQADB93JEf5ykq0NiXkRivA0ZtJjDYBKbo3qynuUNX8PijoxWDlgSMYUZ1QStsy/INGY2ea8SIFMUDNk+gOcPYC1Gf8B9FQ7YPhKYPudnl+6AYUoG1U7EGFQjBzLjxZp/42qyhwCPfXe2Pw+8Dvj8Od0zaMC0ZBTz+1P1e9ZumAk0l6wrcA0BZt7XnPTJahKkVE4yREFzOzlspz8g409JG59DJmYv9z/yGtsfIzvwMMoEiw0TXm2T2MBuL6vUOz6GbCzvlwNQbJm+bSf8O5k+p/J+NFeUUhZbht0Ua4/+QBE8rXgd6R1Gxg+R51gEMp624bWSqzykTaqDmvvvi3S+O7KuwfDcqWGflsqFBz5WRKTwq5gYZp6jzIbqLl0zNVkt3Qb2lU4kq0ZyCX1igYu1h2skbCPHgJ9cdcJrT6Wk0whXkp22DV8z3YWU4hvgC5mCxPJ+pZ9jRH+oUaz9GAwkXJLWScBsHVytx0oSgjwJvfP0BKbvdJ/CjeiNlFBdHN+DwkfYqUK5dARIIHg0a/ozyzseo+oDZGIVdevhKsVrTInxWvMVgVyUZJeGuWLQUEtwF7XAMNcwGw9fkoUMBdnF4mbAcFQAkZ5Dmeo1fMkuTjXkEJX9bCQtHyNcTHOGHNnmfc8juxiHRYFiOZBVTHVFwsfUJ+kQi1QtIgSTUH1AqFn5QbZPQAyBwG1ewBv+nT0oYO4GslQxQg4ewnq4RUUAvB4wLEoU1x38xED1fG18rWFuOoTGMMwnRnomQR9iNInpk3JMf4YLECEglJrhOW3uapT8XSRDG2ozehyjURB96qgsNNnMzQBIICoF2SXWMTGHoTRj4I/Y9WQCMzuXAnogJYR1CPMGoh0QSw3hAv2L5l5QTv67SEZU9GR1gXtgMYfsFIYMpybYjEaPjK7o9swotOJ2QnjbNykZZIR+eNvDKCWZS+fJCOYeypgSZfW9IKEYeB6APVOb9zGxofdDeLK/Mm+PdSNf80/mHsyEQcYey591u8dwRjv87Mf9U24/L4zlb37yd/7Mn/e3/p//7M/92IAR3vyC3qLAX/+lH6I52iEWTGLVK4V41qM8TB08ih4r10TIiYWfBNgJwVGUDLboTx1U6WEP2eOnN4Jg9WgYB8jh2CMUEe7Uon64QTSJ6StBGd1KJdlrhD/v4RoOZvUlGYDiTkA8Ygx11GRaupMAO43w01SKXALDUSCrdeARSvqRXB1RvxEYFoGPOXNkm7Z8rlAQyI7Mn+fgFk8G+o+S7DGk2oXujM8Tk4wwFBHdeUhVCgSH/SGZCeGAOoVn5Fj00HgILxDPe0AC62/ZlITI1wAiDaQHESfffYN4NIxSue1jyllELzHM+RiABedZFmtnHCJ8RY+Q3hEw2yN+WfgyoroGhoMANaQwmgLwTRzTTH3J/VEdBz47i2l4p3+oP4oov31H2ecioj8JEIEAK6rUldcSNA2LNAzNCIijFKyySOmkZC8iuoeOniibKlQ8WYVhzuNSQz6+QO/VlACuOwsj2IcA2gufVoF5fUkb0Z3ymu1OIlbf8mPSbCj4fBnIiZQOmisLutOI21/isNt/0I9dlDKxefnWnUR0x3FcROzOYmK7uPCxexjQL/icOWREDhHDPMIfePiKIKI9Z7Lj+iOH3UWEm3Hw1TsOja7hgNm85jG37w+JDWE9RygJ/qubgGGO8X3hav5O74D1E/AaCtzP8oayvO3jMEoukQbnKJnQatYR/VEY5YzdeUB7mq6zktfa9EXg4sghEnMYcPdtB91GeqoWBDLDnOfLptdVDmQAlx+TrWlPKUntFwRUriFgHxYESN1RxPYdAtr6OmL9AbB5Qrljv2ACr3BIPYa8htszgfahx/Kv2hEgRZXBYySAS9ePiMDqQ4Ln3TkHRb1LCwgTPn73QMBN2WW6ecTrrL4OgOTCg+4iNo/Zk9gf8noxm4jdAwKpfsHXk6Ev/BxafchBPYN04YHtQzKdwwEBUU739BXBB1lQsrvKclFEdwH9MRJYJGjvTsjGiQCs35UpcZgskp0KLD9JizpnYlxMaF67BNCB3RkDavgZTfYsSqA/Emiu3Phe6E7yIgWBWXvG8BRXcx9dzc9YBD5mmAnYGRf/XJWkxDMCLxEi+gPJ1+6MUl5Xk0ENisBceGB3Tsnl7kKmeplAEB+4iJH3e5gJtCcC2wuJ7TsCuzOBYhWSNJl9n3ZK4FdsWHQflRh7QNtjgfZQobhjPcX2nCeqO5YYpgKuFOjnEpsL+tm6hUS3IIOlU6LsMCfTqds4LozsThUXXFLi7OaRhBwChqlEd0Qpit6RJUMAohBoj/h6bx4qbB8S7Lg6JeemLknh2P9oG5kYPj5vlICbKOzO+HwyLbQQNArszlKYi+S28udB/g6TLsJVEu2JGjs/gxEjuNyeK8ghYP2Ohm0ENg8MNo80QimwveBwrLsA3Tq0RxK+yAM5ry07U+gPmFLK1FmBYsnvLtZ5sEO0PeL2tw/L8XwhMnRnd2bgC4H2WI9S5SwFVtsB3kgmzfqI9tQkm4FEd6zTQkRkl6Pjdbi7MAhGjuB3d1FQltw72KkaE2bbEwM70xgWBt2JQSwkXKXQL1j5wa7KgGFm4BoNXyp0RwV8oyFixDA38EkKHCUwLAq4WjG1tnOIhYZaDegXfMMxBdfDTfnv4aAYQfBwYOAbTVlr52DnBtASdlFC+IjhoIDwAXZWMAyno8TVzguESjPBdUjyWCEgdl1ahAgQveUCg1JwsxJ2XsBN+PwIlBnDBwLOUo+g8n7yqp9XBKVGjxLeWBnEukRM92f5aphUlOgCTJtNEtsRRKZEWHFf6mod4APvs24PXoWgrNZl8CYTS7qvVSF7mICc0dyX/O+y2PslY9wDyRgob00AUkhKYdM/uB2VGEul+DitGc7jXHpMel6ZpMWjFDbuJcT5lhJjRWI27/8shPij/6m0Pz+r5/Ivbz83t19oxnL+6Xn8+L/+OxicwmZTIbQaovRQOiAEAaUC7M5AtJR8ypkFLkvE4wFxy7hqMXHAnQFS15JwYux3NGvK56KOrAZpPAMvwMeFAwd9ZUYWMZaR6ao6QiXpJABEQxbVnVjoNwb+YoC8Mim1kbIH7KIAACAASURBVL8PBR8TVWRtSSD7qteU9tkzi+IrA1/HcfuZoRWODKhvAvRKwi489FqNMlMEIFRkX4OJY0dbBOglSN7GUIbkBQWK29Qhabkd2bNjz9fcnkxFyHor0B/x74Qnm6u3klLRTiAUEcXjLfznUz4mEhjaAw+9UWR2kzQVSKC43DOWuYg+lPy93gj4Mnk/IxJY2TOtvtqzqWp4m5nOPtmoyPgi7I8fCRgTTIlUcJ+Z1H0YSZb+BLM/t9JTPikt9ykYLiS4BrBzVoUAGP0vQRMEmzuR5HcxHUvyrqYaj/kXAcuPJYoVWbsMHLNktj/keeJ52w/z9+sJMsDMrFtm3TPbm/1do9fVJ2YyefEyc9kfkj1lUiOH6AziMnOb6zhGqWGfmMu4Z4NHlj0xPlGl7ZRIcrK93y/vbwYjjOFHArEE+gAXQzJzT9lZ+n/yvgrPY8sDNtlOglTp9n83Hk8aQvO+IzOWEiiWEcOCMszqcn9+hwOguowM3Uj+w2JFttGskw9OEIxzX/geQromIMiI2xnPR+4jBdLxCLKuMi1OCJevBYweR6aG7v/91i1vO12HvuBnRgapZD5Tx2HN4/SlGL2a2TdoJwwEyd5IAKncfM+Mmk0cwzDoy437Y7H7a1Q6AiE1xL1KInXMZk8zPxu4mFHexlF2mz2J+ToRnuxXZirvh3PkDkBf7N8Tuk8sbZcWkgp6O/M1yMUQjB7D/L6Vae4bPbJpoTAz1Hs/axxDQnzKzshKA91iZFVVh1GGCXAhQsR7XuSkZMi1GJmdBfI5vOc1TSysTn5HaRO7LgjWIRJYHeXIZNTyNZHPVb4e8gDtysRmB27D1YlZ7Alm6aHeX3OuEmM3YndAmWn2nFJFkRhkyT7PYSb/CCMe0+/lyLjxGDI7mbclPOhND0Du3cznKPtN2aXJ57p/XYyf50gLtJrbd3WqQekj+gM1MppR8T3EzkskRlqiWHsEReZO2ji+L1y196Le94fqHRneUMjx85fVGfR8Zn9oMKwVya+9r/K++PFzydWcD4QL6fM/wlcMkMm1Gvl142d1HFNc9c4jFDKxqQ7CMtwmpuske9R5/iXUzhEcSCAoCdU6+EZznz2vr2Dk6IENRqbZwafnI8iG5Ode9mRK+zX/XLq+pSXoyhU6/N6Ko084358rSuI9wCHu+/d8JMtpA72Yjox49mgKl5lXSlyzXzJKSZ9lYj3DtIDckSEUjt7IqBh4FPPz5l7G5HuMWpLdTDJR+DC+r6LRENkTeX8OF+ItietbXkrvR0A2/t39ipH7VR5SJFaS3sf7VSNv+Tmde2v7fywmuH8+s99R0V8pjEG0dnxMdG4v1VVq74f8GkOZn1ckJjcOw77+5Gfc8t/Gvv8TH/cn3X5uGMuP//af+fP+1vf+8z/3YwO+YY/lN30bnEbvFDZ3NczzMg2thr2CHYf+ItDf4ssIXCum5N1UyZMWoa7UKIkya4HyDhhmSFUPgGrVyPCojjH39KoA4bZI0h2RegU5uHTHqd8vyWTJOgFuW7DjbiggXCqFH5BWpzGunlLudD9MBRhWBcMrXjM6vrrkh6drJPQuDXKCx6J6jeoS6E6yzwgQt5T+ISaZpAZkCreJgv41O5Wj9FY4oFjJke2QjqEloU3DUgmU1/vQmfnnTNob5or9fXORAnu4yjt5mUC0BOrXgPD7JDaA++Um9B65hkOknaZBqaWcUPUZJOy/wPuFQP2GMjb2donRV8TQFcG+vSJLfCgFZL8fZaKuYfiHCNyf5nUaUId9sAywD6gISYIY1F6WmIF1MAmYBnrCyiXlVW7CY8pyTfrhIq+fltH8UfMYihX31RuB6dMIX/OxHHg4zOmOw1LzmmzaMOUQw7ASDk55W5mZrq89hplEFLxe2zOB6TMyEbsLerpCIVDeBnRHTCHMQxi7OZlQWdyl1MFGjDLIfP7lhEDKTjNTkYrLXwK7Cw7r1Q0TEvP1UGwY1pJ9iNJR6uoq7mcGQcWKQ0V/KFFfBxaMV2K/OACBYsNrVXj+Lgcb1dcB23OFyRsOgcOUf6c6QN7s5YCUv/G1nz0P2DxUKO7Y3TdM5ehpZK8lj7m682iPOE2qqzgCWzXE1KUXoS/pSTNtRD+TqG7CCLr6OQeFYIDyjoNxXiTIUjqAMkQIBt1ksA3sh2SbPofKZUS5dOgXmr2Cmv60fi5Tfx/I9E04YFZ3YZROF4rvLbMN6A/SCnsC60xXJIM1TIFyHeHNPv1St3EMfvGGYGdIni5fJuYoMiiHFQABw1wlgBbTZ7pEdePH91yU7EjULT10IuxltcXaY5ip1DcM6DZAt2TpXCnQvHYEUpVMckiged2jXxjK3iyv4xwsNYKd65ACUQggshQ1pw0LH1lsn/4NcNt+RzCju5AWBrlIhzWHY9UFVkE4slLlbfrsS0BauAA7pwfaNgRbqg9QhYArWVxf3Dn0h3rcf2njmPJoNg79odnLRl1EseY5D4WE3jnuX5I7Zi9a3r7q6IvzFT/35RDG8BxpU8CWEjC7HHKTJI9TDd1lcEVfp7QRw0Jj+pJDpeo8kzYz6FApaTMCxV3yfgcBvfVjiB3lsH6skpA+vgUYhGMoTmbrc/AMEzqHMUgmFJLbSsAiKJkYygC1s2PHYVSC9RWGrB7lnwzPyeE9akvJn50aFMseUdaUiwqwU3HH30cloSoF1RKMhYJsKxJgE17CrAb4htei3lrKX41kMI2S6fojYAmF4j4lRYLsLEJlUAyBdRNFkor2DrIpoHYDmbHBMUVUCYRSU446OPhJAbXpEUsDveb7T25Zs5GBodwN8LOSoDV1N8IF+gRrw0AcANKFMfRGGgWZZKKh0nvQlo4DPjLApi6AECBzhYZRkK1FqHQCnT6luBrIzDwKQeAWI5k7o8eEVAAQUkIuN/CnB5Ct3ctCpUQsNeS6Has6RAZjWZ46WHoKMwBLHY5ZeooYoRIIFC4F/ihJuWyMBNZajX5J0XP/sN1BVCUZx8ze3QOJos8+0HvgOgPI/Pj790kJBMuOSa3HZNbMUsYkSc39k3xjJCBr07YSIxj7gZJZ5/ZA9D6AzP/OgD2nueZ/+8DU2syuplschr1fMwHYEUzm47q/nXu//2eVwv7l7ef79gvNWJYfvBMv/qN/D6JTkIcDwrKAuZGwi4BYe5hLw+j8VsLXAbKTZB8HMa7GZW8hUhdb9mGKSJaw+rIYZYPDQRjZGYKBFDaSnicq+iNdzf6sKMlcdSd8TC6YN3f80rCzwMRWRY+laxKwuGMEuhxAJmLCgT6H1kQFCLcP0Mir5L6MiGbvSQyKz+Wm6U0rMHp9fMk4+5xwmGW8+aZbyjn1ToysjK8oI63eyFFWarZJPqo5bA8H9Hdm3xc9r9wHaQlQgkll5G/2cfqjXDczaiEl167FeA6yPDTvT2Y5GJqx9/UREN5nHAkg7BTIyaZRYiyKz/UFOfBFuiSjzcm4ljK2KOiPLK+T/E/t9zWzlpnhyx5K12D0ImYmxRdcOKCki/uWPaJy4H5mpuM+w1Os997PvL3yRnAhY8N9zKA3J/1m9jCDOLPF2Is3zJM8F/sYf66kJ8ax5XO4av+eo0czgUrP/VHpOgV4Tsw6jnH5oz+1oJ8yH7fZ7sGn7O+xchX3r7jj65qZq8xw5pTWKDGu2o/sQ+B+Z49Ufm6mIzP11adk2Mx2ZyYqpN7qvHhRXzFMBdgvwASNlNwqkFlDZcnyjezdkAbg9JkhXV4s2vs+IfgaT58H9Idy7B7M11KW32dfp6vFeDz5ODLgH8+F4GuXQWnu1PNlui95Le8ntLqai2f1mzgufuXH588Ula5HBi9x/82Or51KiwkMxBLjceeFJ9Xzd4j33itxz/yJ5EmMWqB5Q+DtiwRUK4FyFTFMCUhzsmrenur3DKQvuKil2ojqjgsI1S0XTELaflZYIDGv2YOag3G4UMQkarNFYgATs2yR/GlcQMmJsvR2czEnB+Lc93Hm63WYksGL6Vq1k5TGCoz2AhGQPIYEu8OUTF9eIDG7AFenhFqL8byoIY6LMSolmWYJeN5+9nDvfdc8XlcmyeyBHL8XpON+6DYtZO04COf3Z/acZgA9zBQXUBKj6CoJZeMofeXxpeTeQoyLClRURLRHCsWWADYzmK6WI1OZP1PGMvpM5ERgmNHfyQVFgt9g7rGiEW95De8n4mZ2KxTJM7rj4kdWFNgJfYXFZv/8ZuXgJgp6S1CN9HmpWwLGzDT6SkJv/diBqDoPOzMjM0c7AcN/VEt5vhoCXE3wSGuFHhdL6IOVKFYWvlL7z9p0XMGIPUPuI5QNI+gLJWW9vBh43KM3MoO9BEyEj/skWJPDcQaEkl2V0geIIRA4a8HkWCUhB54P4QLcrOCiautGz6CvFMxdz237kB5XQnVuZCxDoXC/6kL0fkx/Zcc1wZ1sHe9vHRcUXWDdxmag5FVLhEJDdo6Jr0mVlKXQ2T8pMpCLETHLWCUTbEXvWO0x3AuIyeBMMAQHWo0eTdFbhuVYRzB5vw8y+yit2wNAKSlvzWBKq5HxfQtg3mcWs8R0sHtvZ2G4rXxzjr7IGPmzMUxmzbf0u9j3EGYvmx0xwB8H3ILn3zm3B6qZgdR6n1J777Z/vjCC4eg95a73QWa+jWE+El9Pf/1Zt1/8VNgH8Tc/+gYYy+//F3/uxwb8gjOWohMoXhqoQUD/pB5DPPxSwU4Ups8jomAnlKsVQxzALzjdshJBb+iJUR0/8JuXagyRGG4KJi5qDpKTF5QB5doIBp8kdiL5oJpX9M71C2D6nMmRUZC1CKmEl6vAgpUhbeTqZipWzuEovhSor3KIA1m2XafQvGI6YHXFAWTP6u1lZUGlLzkjUF2zA0u4/cDNwJm4j8HvCQSCEalDDAAimlcZGHB44hcggEiJX3XN4+qOmQTp6ojqSkIO9Ce6hoNqdwxMv8wv2n6oozcssSeNGGVqOeSout6zijlNsVzyCzCzvHZCpi0P2L7ikNoeUdKsU4UDIlMpx1qGWrAKYskEzGIZx8G6WOfaBwY5FGsyMKEA4g2Pf/Iyoj0lG6n6VNGwBNyEw3d9xcCP6Vdk41SfQJ5OpfRTDpHFHQDJnreggX4u+PoXqZA9MVp5GM7XqRwYklPeBVQ3/MLK5eHVDYFBrmjIDGeWy3pDYFxdA+WK++dLnls7BcQ2XyM8V1GQ0c3spxoiytXbrGCUHMpFJGgFuL/FHYFClEB7KuELYPaMvh/VcfDMMrkRKKbhuU/VFhmkjAsCBqhuQ2I0+RqbbaBnKQ3bOZRDuvQ8lgOuSGw1h+o4sl057ISJwfwCPfzM4vYjg+lLDxEjuoXaS93Sd6Oyke/TI763dc8h27SJ1S3JbBbrgGEuR8mt2ebHJL+skOyo6wNcKfcslk8+u2Yf9lRswjioMyWSr5OtOdTz/PE16We8T/XsHZQ9X6Mc7KLb3DOXFz14YGZLcFPfOOheQfgI096r1UgALTNJ3pCtk47HDfD5hmkKd0kVDq6WKRQr7mWOel9towageW3RHRsIH9G8oVRSDfvt9QcynQO+N+1UwlreJ3zE9KVPCzEBcmApvTfsJsxgkPsXIZ0ieEBi8KxAeetSOIoclQFqiCOgyox4VIDsI5rXHLaHmYLZ+FGOGAxrN8w2AcIuwM4Vv69aMoVMQFUMw3EMeCFI5zWm+gCzJahSQ0SxchgONMpVko0OYQxokTZCawK3ESQByFbqzGzmsBiA13OxDvu/33mYieK5SAypHFhT4RoF6Xm9FHdkKqsretWKO4bjFM6PNR5xrlB/1cHODIrWwc40VMsaC35nEXwFldKUEWHWFroVaZGIoMU3muDDkOVzE0OWLGhWf/R8TGZFfa1GaWfUEtGSHVWtHestQpk8XTvWdJgVw2tClWpHVpY1HK0fr2W9c+OCgbKssnCNgl5bgrQQIY2E7rhP6nYHcTSBiBHlbc/3jgBCqVFfDdw/KSAiq0RMqkIRMf8/yV+HANV5IETKUUGZqJsV0Btumymckuelp9cQQrCaRAgymj5CANB3HdxBRcY7gbvM5noloDY9/KyCsjyHKjou/g17EJPZSNHzPpUYRbPskFNX1dAzSKeTEL2HDIFASAioJBmVPkJ4z/3OktIsT5UEmCJGMpYAfxfCKGcV7UAvoHWABYQUo5xV9G7PkKkE8DrHug3rxkCcDD5lTwmn8AGyE3vQJgRE25OpKwuymF5BoOD9zqck1ECQuOsw1oQAlLy2ffJCRsSSybG5H1I4vwduSu0BnE3BQpmdDIE/O8fk1r6HKMu32c17clORwNdYK9IP3I6SZC/vs6F/3M8ZaOZgnq8xm7G/x0y6xMgGdkzm+wGMQT4jnPw6gL0fGpR/Hv847AHnnwJ4/tzfIvbn+S/g7RcaWFbzHr/0L32O17sZZkWPzhkoGVBri6np8XuvH+BvPPlDfLY6w49enuH0ZIntUGB5O4HSAaZwcDKibwv8+pMv8dtfPsFvvPcFamXx/esHCFbDRYHtroTWHo+Pl+idxlmzxsvtHDfrCQrjIEXE+0dX+N6Lh0DpcFB32CynWP5VwBiPYVAIxuPiYI0QBZ5dHqIoHCbVgG1X4GS2xVdXC9RNDwlguSsxn+0QtMd21UCpiJ1VmE463LUFZpMOV1czHB5vsLydwFQOReEwDBrOKkymHUKQkDLgzaaC0lxunjQ91tsK0QsIFRFuSsTaQ5aei2VOYPvLPLfZpxqsRNFYBM83fARwMNsB1mAx28AHiTdfHWGtIoSM+ODBFX784hRCRMQgcH52h/X1HHcfAKFlEMH0ZIvKONx8foSzTy/x6uUhNxqAYjbAe4mqsjDKY7eusUkF6+89ucSLqwWOFxvcvjmANAFKBYTCwVoF22tUzYDlbQ250hAXLWIQCD0j+k1tUdUDus7Arkoszte4XdUoa4vltoA0AX5VADoAqYYFVsIc9HBXFWIZMDvbYPNsjlgFiCFCH3WwqwKiDNCVhb0r8ejJNS7vprCXNSaP1tg+nwFBIFYeZs7ji14gBgFVBPheAYMETAAGCdE4mMpByoh+ZxAHBTFI6CPSunZdwMwGzKYt3vzwGOrdLYZlCZjI5xGRX8omQKiAYBVEq3D47i1uLueQK41QBMCw5kZYwM/SAOUFYsnqHIiI2CZ6WID+4SpAXxm4uYdeKbgTi8XxBsuXc/595RE7BTmxCFtW4kQdoKYOeMV6mc3HAXKrEKYO5aKDfT4hu3ngARkhSg95WcAfD5B3BmHmuM+Nh55Z+NsScicRqoBoAsxSMZiiDGTIdwxg8NMA4VlWHqoIcTBAPavgmojYOIhWQm8lQ6GigOwk/dVFYkYuHMJSYvVphFmq0bvsJ1QahNpDLzUgBNzCwdxo2BMHWInqpUZ/zDAhnPYIWwNzI2AfDBA7jVg6mCszMokcOAG9VVQmXKvUmSthD5JOPgKxDKiem8SqJLa/5z67SUjp0gJuFvh6ycCqnBRaFEp29brsCQbTr2MdUD0zcE2EOx9QPC+SrNagP4wobyW2n/aQdwbRRMguqQ90hEl+7lxfozoJO49M3B4AERV8GVHeqFF9EKVA80pg/ZGHGBg8lj24179cwNcR5TVlsv1RhJ94yE6ivJbwTcS2UxgWAeWVobT8JELv1Lh4ExMrv3sU6C0vI/RGwk25j8OcqpJhEVBeGtgDsq320KN8UwAijh74zJj6KilKUkJ2XlC0KaDKbFgFEFRMiz3AMFdwDZUflNVyYaK6VuhOIibPqWipXyl0J2RTymWqg9kI6FaNHcdUzUhUVzL5WSOKO0UW3/H6YZcyPchZGdGe0W/OjmMgd4jaBijv+Pz5GIqlGZUGXEziYk1xR8tDVi8Ua0OZ8kyzX3hNub+r+NzVlUJ/KHDzrQblLRCloZphQ0DXnQiozozhdNPnZNBdbVDdpM7YFRfLXCNGpYDwBT27BuiOBKrUTZ27ggEu7pmNGrta+0OR9r8alTG+wj45WgFmU8AXPDdRiLETVngzJjRLr2GnrOXpD2gNgAT0kUJOOs7+bbMFgMn4/PcXifJCVk7xBrg40x1J7mfBYxCJlJm88didSJgdF0XYaZoXlgqyxX1mhbm4lRehhOeia3coUWy4AFRsqlH1o7cBu3M9VqkwCbqE8EB7JDF96dAdMX1Wd1ywUi0rSqqlT8dBMGI2AdsLMq31bYCtxbhwV6wD9JaLCr4SsDXrS/oDherWwU4U1JBk+7VEfWnRHWmy35u9FzT3mWb2WdqQJMw1F5GmepTy6s4zBTdS1p0lxTJ3c0pKnwHATvToaRWewUdmZUcGF3GKUKr9dgcPX2vodTFKqfUmXYRNgVBxAcPXGrLzEHG6T331TNDNIDBUBnLHxQBxn7FM/aSh0Kw6MQqyt/ClIRsLALlrU4o9KMyy3MyUGk2mdduSPcw9mvKel/NrvkcOgfItkBmHJAVObOfY7dr3ZCPT44WeEIzGgGgdpFKUxxaUBUX/9jbzv4XQ3Of7wDMDzfv//4sCLv8C336hpbDNxw/i+d/9DxCPLMrJgOPZFq+uDyBEhHcSi8UWqx8dws89ZqcbrF/NqN9vOMAeHm5w8/IAcALCSqhOQH+wgRARw6Ahn9bw73QoKwv7+Yyy1vOA8kqiPwkIZVrp7SXC1AGevi25VggTD1EGNPMO29cToAwQa43m8Rrb6wZipxDLALXmMOSbiFAFmFu1/4I6TMP73ENOLXBFH2ksA8qjFv22AFrF8J4oUL2RaD8YIO80YsHuxSgZqAMv2LH2qIP8qoJvwuhNgAdrQ9ZiDJLRO4H2XYvioMdwUwEyQu6Sj6ynnLVYSrSPHBbf11h9HKDXIslROfDqHWXB5VKgP0qMmQP0d1ZwP5gzNKPhAMUEVwGXh/aph7nioC53CqEKOPodhZtfdzCXmmFGLw2GBxb62jCspxVwhw7CSpRXXIkcjuh30rlL8ir1hQZ+cXdnHnonWfEyZfru9okHvEBxx+MpbpnIqbdkfocjj/qlRn9ELy+DlwSE5yrdcOJRveTgm6XEahAYPmqhnleYPBXYPIkjQ1su+XeuArrHFvUXhpLbGw4K0XBw6s5YY1LeCLQX7JHsjxkOJCIraULJ62j+mcb2UcD0qcTuIYf88oYVBv0hQ5hUK96SR+odA4Gi5mPthInFky/I9veH/L3eCuyeOMhWon5FL+bugYCdRNSXlObOf5L8r5dM1rSzOEoX8zblQDl2exYxeSawfUyZulmzsmXyXKI7Jqvm6jyAkVXfXRAomTWBhwgC9auI1cf7QKOoWe2TZdzlrUhJvHyd3SSFsKQqob2sNMLOIqZP6V/ePuRrPHku0J0CesPUYwQmzqouSXC/Jj82mxRyFDmYs2oAY2G7tMD2scD8J6xvyXUzxYrbtPOIYpk1+8DsKZnz3P2YlRVZMl7dcIjsF2KsZ4mKwKK+5CA8zPmaF3dUOzSvU2rqhUB3EjD9UjIUKQKzpxHLT4CDH2P8u/pNQHcs6fO+wtjtKB0w/cpj+aFiIE5SHjQvI0IKeGlPCfjtDKiuqJKormOSIgv0h/Rtt+fA4kcBm3ckzAajjE317L00aw7MuqNX2awjik3E7lzCrHid7gNw9hJaEWNi85hK3J5IzJ96XP0VhdlPU69kS0Cht1SG1JcRZhfRHstR2aGTrDXL40dVQBruAb4GGcBNXvmUbMr7yiXPoU7e1OomYHch0zXCc10u+btyGRk01WFkeKNMPtg2YPmBTsnX7KcMqfrGVQLVbcCQwqRsIzB7QSCwek+ivOE5C5r1Jut3FMplYrYPKUNmPQjTYzl0R+ie3tbVY/o86XWN6FMnZ+5Q7Of0ejeXDuvHmn7xVwRI0hFouEqivnLwlUR7JNFcke20tRytCqrb14+IAAyTPXvsSznKh7OyIwcnZb80K1jkHwkpMlsOs+2xhtlxAag9kZi89Ey1fqhx8v0WvmSSKl9Tpt8yMZhVNrNnjrUqtRirfXRLdjoUAraRqK8dhrka1R7DTKK8CzAbss+qDeykvKQv2uwCVB/SNe+xfVBCtwHFyqE74qKSWXuUNz22jxuYlYOv1ciK+5I1JcJHJrRKYPLTFbbvz8c6E73z3HYfmQib9kN6hgMJS1DElFrKWn1NJjtKkMlOMmPZe9h5AV9ImG2qZpH0x7qKDJxZWwxzA9VTQdB81aI/rlAs+9Gv2p3VqC47Xj+HJYo7BuXYmWHX583Az5rBozurUb9u4TPzHOLIQutlj1BpekEL1o9AYmSmQ6nRnRSYfLFhmqxlDQoAuIMKatMjVIYMsCXDKqyHmxbp3HgC1M7uPaSGoUdy1yNM3/alhkJBbQegHxCbkum1TQmxaZnQOliEgwZyueW/AeyDdfwowx09mdkX6j09nUlCKzYt4pTx8mLTkpk0mpUkMfLvQhi9kZCJVU0e0vGWAV/HgBwh5Vu/jznsJ0tk71WSROsgjCaj6vceVwiBONj0u5yE+zOwxz8DcPyFl8LWD+JvfvDv/Jk/72/94L/8cz824BccWE4+eRD/5t//1/Dl3SGsV1AiYrOtUFYW27sKh8cbzKse06LHHzy7wHTWYb2qIWQkW2k137OvSrz/qy/w4xenQBA4P1/i8genOPjkBiFItL1Bf1dBmEAW7mKJ188PoSaO6oOBjJNaK4SzngOFDghOop702F1OuMOSoE109HCGqYPYahx+cIObFwuCsTsJ93AAthoffusrfP7ZA6gd01xjESE7ifrdNXbPp4hTD3VtAMHKB32r4euQqlAAFAFmOiB+MUn1KkCYeFQvzejnFBFkNyKgV/RO+qmH2qjRq6d39HciAPUrhd0TC3hB0Pv/svdmv5Jk+X3f55wTJ5bcbt6t6lbX0l09Pd0zw+EMOZRpSYYASzJkwNA/YMCG/Wz43a8DwwIkWa82YP8DfpZfKBqyAJs2ZRmkZI1mOEuv1bXX3XOL7Sx++J2IrOYCSvIYnCEYwMXdMmM5EZkZ3/Pd7vRUDwyY8gAAIABJREFUn+WEPPmyOmFDoo7YjaY98pRvhNHwuTAmQ4hONJEw86g2MUVlRNeJgUl9dpAkZ510WK6/7ok2UD2zUnUx95iNHhNnha2Q5FpfBWGieghVhFmPvsgleTYTAGXXeiyKD9lwQyL7VlxouoOInwf0TgswJqkgk1fT1AKas1pqT/JrTbeUJOHiwryV0JrA42TvtZKUU4VP5wIlvwMjkMpqhSslkbU+CxSXEqwxpKfGbC9f1E6NCbbiX9uX0w9+v0GKGTPxDBeXeqws0b2EXOUrPfpXh/TDbEtKuRWA1i5lrPIbAYamke1LOmtMycryXO2EOZEER+jmkuqrO/GsZhu5ZvLbYYxS8fyMMSF279kdvID7hE/U3hObNeIpG7oGo0nyYU8Kh9ozWs2xALfxOm8EIAC0x5HqlfoKG+ImjGm3rpTHD4Fdkua77wgVdkn6VoeU4+JKjt8Xwmh0c5EtN8dyU5rtGGXbQ6LukIIraZUylqbdg+DRu5g8hLpn7/d8i7UYGJIhFEiYQGGw7EY8dK6SyRZhNkRWPlSamI70+pYZ+24hIVduosZryhfJL9wLwHFTAWGjHzKdy5DGp1iJpD1fRzb3xWuq3P55w7keQZzZpwdLsFHyXqZ0Y+1IIV9yvBJQJCC7PVCpNkZk/f1EGLGsiWNX4/C6ApLvVapQysuQpOsiLy9uw+i97BbJawoUq0BzOFSGyPthu9RkW/m/L0iWiCTjT9e36eWa1cMEh4N8G2kTKCQBkpCJ3HzwKUOyMaRjFK+3gNtuIdJMuxGgpb34Mof30/E8veWPHTooVST1jSbfbmK9RMqsUiDV4A3ch0u5UsZeQpCSP7dSlLceVwi4E5Ztf00OXs1o1Jj8rL1I1ctrn1KGhSHLtzLu+cqzuyOMWXNgsLuA3YrfU/soYWmJLXvbtzwwe1G/5Qn2AoaHGpCh41P3kXapx6RgnTya/Uxky0MaLFEAp92GMcXWrh3dMnvrdanG0CEJaZI+0SGddfAAmjbQHgrYy9c9IdPJhqNE5jsRWWzIkwc1xK94Pk03gEogyjYkEXbwOkr6q7d6TEAVGX0gZFKbMkx6Df7QbJtu4JWiX2RpktaP0l+5rkWGmW2dyIcbn7yfCYReNzR3J9ityId154X9cxLGNAA1SWwdpPYSzKR7AWhDQqyfWtmH6xp3XMn6kkxaxUh229IdV9irJvkpUyDPsISYPKRGfJxlNoLH4ViyTbdP1B36Lyc5elUTS7sHcr0nlLk8pu6IpRVmMY1t1Bq92REn5dhjKV2ZDbGwY6LsCL66nqGfEiBW+Qh4B4Cpejf2Wo5gb/Bhvi1jffvvsAeLdpDeijx1BK1/DA6IXiT+KqXQxrf9nDDKalFaftYCfmMvnZUxRpTREiiUgHH0QVJgwx8Fj4Nkd/j5T1xC/OXvsfxzDiz1n/6QX9zFe82P/vcPuHq2pGksq8+XAirfTMmf5bigefVP3+HTf/KYsLaszmdkuSM0BueMSBDfFJx9+w2fvTjBflnw7v0LTidbvvfvfsz2XxxzNN3RXlQQILYac5Vx83un6I3BNwb7WQmd5v57F5Lgmnvmv19ifzaBoDg7WAugtIHDf55RHDbEQti9xb/KiXlA/c/HTO9s0YctdqPg1jL50vDZD+5TntSoew2PvvEa1QngcU4TbeS7X3uKul8LO3KRyQ3WosfeGMyyY/pJTl9b3GmPP3QUjzYsz9Y0jzrc0pF/sKI/7dG1ojg39Gc9+eM11fOMofvLvrfBTSUMKdto+I1bjn4/45vffEZ5qTi5sxIf4nduiEbqNdxRL/7RHRz+SFNcSeciCqrXmun7txTXEn6kakP1wlC+0eidFFf3B4GTX30j/YxFxM0C6psb1h/IB/nsY4v6tVvsRpGfG8pz8aH6IuInAbuSupPpU0P1YE3MYP4zQ9xmLD5W2LWA18UnBjeN7B73ZBvF/AudAnrkWNuTwNk/C0yemvGGPb9WRCtyun4hzJ/dKuxKUb0w+BKyrebRbwHf2GCatL0G2iPpQl3+VCRx1bkAweqVkh7SVtE87MQHF+CD/+kW3QpQ9jlj9crseRzBjs8jy48TY2Yi3bGnOwhMn0sqre6F6ewO0+RBLb/3MwF3u4eO5r2WyUthrI9/oGgfN7iJyNDq+453fqcGBDQd/yDSz4QZHQDr4Y8F/DRnjuXHnvbEs/w4cPxDT7YTYLC7F7n9bidsMDD5Sxf088jDf9yy+6Bj8iLJEzcwfR7hr10zfxLHG/1uuQ+TAbnOmtNAvhqOXZ5X3w3U7/W4UvpA3USAzfZRwK5g825g/aFj/mVg9qVMChTXkXwFu7NIfSYdo/mNYvGlXG/b+5Gb7/TMvwy0x/KBePxjT34r+7e9H1l+0pHtBCzY5D8tLxTNmaM8V9z/nTqVksPZP2tRHmbPIjffDJz+S8fxDx26ixz+1DF9GdjeF7/r5mEcqy3O/u9WmL4ZbB6IF7o7gNWvdjQnMjbbh3GcdJCQHAGuvpR+U7uOrD6S48pvJfl2ch7YnSm29yO+kr7E24+EZbz9dk91HmmOUspwSkqu7/sxNMlNhIU7/oMeN5M+y26Z2NlO9r28jmzvKzYPFKuvQ3kTef1XIpNzz/WHmoPPPOt3Rfa4+kA8kuvHsHmgmFx6mhPF7EXP7kz6LNtDNTKtbiIhM6v3FMVNYPVYgMvldyTFtL6jmL0M5JvI9kzjc8X0VWB3T4Dz5a/tr2US8NUerr+5D0rqJyLTLa8D9bEemcV8LYFB/UKxOzVjguzN1zUXvwbLjzsOf7pjcu4xrTxn9ryT9OgadvdE1rd+X9i94iZw+dfb5DVPaeOpDkW7SHklUsj5lx2LJx3be5KY2x7Jscyet9idnJjpq0B9R0Dm5p5m+tpJP+kDOQ/5rQC3g09rmbDoJHdgkFsGqzj4dEfUsHpXp+RpOP7BmqtfMaze0+xOBGDVR+LJLG495aUTJtFFjv5gw+5EOiclmVixO9Ep1Vhx/HsXwiLnivLSM/+yoTkU0Hf1rYzmUHP7viXfBJoDOXfXH1mqC5/CesTH2y4Nt1+TwB1fKlaPZXalm2naA423is0DTX0sE4TlZU950dNPFe1SVEK7kyylF0eaQ0N17pg+2TB9uqW4aOkWhuLGcf1hhukiN1+zFFct05cd/VRz8V1DfWyo7+QUl9LtHHJFthMgNkzytEuR7pcvN/RzQ8gU9Yl0G7YHInOuT+SNL9s6bj6wIs1cO7bv5KwfZHsAiXxWuIkmWM3kyQp720k3ZReoT3PqU0v5yRtJJt50rN7Lmfz0De1hRruUXsnmKKO8aMjWPeXrHbr1VF/e0s8tGFFila92NMeG7b0cu+qwVzWuNCkBPbC9X1Ke16g+pP3txPu9LCmuWrq5JWrF9kFFtunZneXozrN+PCVbtzRHOdlNjb3c4iqDvW1QPlDfK1k/nqJckBTgqx2rby3JLmvy8630ehYaXxj8NKd4disVIUDIDb4SeezuXkU0GrcocFM7sprm5RXmYoXetrhpJv5UDf1RiWp7+pOJfD+d4RYlqvd0p1NiYUfgGascNy+EmStsktB6/OGcMC3EA5pSZQXsavTtJp3APbDzB1PiVPoxVesIZS5g2HnUNqXaai3rcJ6wnEGMhIMphEBYTMa/0fVsv3VXfnbiw1Rt6sQcui6LnDifytfBnDibjF9jumyWJSmrRWVGGMUYJO02RuJ2yz5NN8r+pcqRUIvnduyptBmhrgldT2hb+ep6QtcTu+4rP8feyffh5+H3/w+g8hdpUSmd+Of59Yuy/FIzlrMPz+Jf+R/+Y1qfURhHlfW82s757slzupDxup5zvp2RZw4FWOMpjOOo2PG6nlMYR2l6nq2XAByWNa3PaL1Bq8jLj0+59/Vzvn30ki5kfLE+4mZX4YLmZLbl2fkhD06v6bzhejOhzHsOJzVfvDrmo/uv+fjlHZaLHcuq5qauuDif8+79S1ZNwaJsUSlX/fXtnKroWG9Lvv3OS356foezgzVX2wkHVUPjMnqvaXtLnjlurmaoLJAXjrPlisI4nlwe0e4s00VDaR3Oa7a7AtcZ8qrHWs+iaji/mVGVPdtdwd2jFbnx1L0l04HOG3pnUCpyezvh4GCHDxofNLOyxQfN5dUMkwVM5ulqSzntqFcls8MdSkV2uwJrPSEoTg82vLmZEZ5OqT645XBSE6LizfWcEDR50VOvC5SJ2MLhOsNyueXqYk4x7QheY3NHCBprHU0jekPXG6bzhq4TSYj3msWspveGvje4PuNoueFmNaHfWrKJYzZtWG8q/NqCidhZR19bJgc1IWia20IkRpOe2bTh+vUCM3GjJ1MB2gS6m4LDeytuVxOODzdc307Ji562tTJR0RrUxKGucszZDndRMXlnk4LaTOoh1mgTcL0ZvZa2dPRby9n9a978+JRYROb3V6yfHIh8+qYafZhZ5fCdgU2GPuyIrwvCYc9k0dC1FrfLUI0hTjym8ASniK34LDnsxP/qNerKwmlLlnvCF1P8LFCdbeRYbnKKk5r2qsIuG9ybijjxqJ0hu1MTv5zi77bE1ogEHJidbdhcJHY+Aiaidka8qiCezUoArsoD0SmyK4t6uKNf5/KcoDA7jT90qCwQayPeUadkPVkAp+V7UOLJ3BkwoGthB9/uRI1ZlIkdBdlNhjvtpIrtPCdUKeggD6heE9PrUfVyA616uaHy03QH52WdQ+eou9OhtnINZitNf+owN5l4Pyee7Fx8ZUP9iC+FyZa+WtBNYq6OJFXJXmZjynS0IfldA+Y2E+9oJ9v21b6aBpDpQc+YLG1qYcGVE59pdisf6n4msnAixDxithq70jRnDt2IR9WuDDE9VtdyEz6w4L4KZIm5z68VzZ1BAiVfCmHwm7uO/NKI3DxIerGKe/Y1Gtn3UIlCwa413UEgGpi8lBvmbhHFB9oqhqTsfiE+0vK1kTTpIOdhYGG7g0D1RgvbOhk6FkW2r1yShM4E9NskkXaT+JVO4OJKknKHTltAOnwzkeqL1FgmhXxK5DY9iaEVhYB2Is8W5lmNrPvQLRuVeELf9mpGE1Ogjfwtv5VJA+Xk+hm8uNkuJUynIK92CeWlyH8HLyfsGfGh11SY4HQ8qQt1WAYlwbD+odf2bWY85LLf5ZUwvoPUWHeMnsxh0kH3A2AWpYN+S7U2yKwHxUZWp4CtA7U/zpROO6R6241cXP1cJMrCxspjhkTlkKVqlyR1HWS0A5gbfKgDG+5z9mmx4zEKi+0q2ceRJU+drUMadjSMwXrFTRoPH8cE8kFpk6/TvvrEOBup4HHFEConzxu6S0d/sAZbJ1m1GiTdMgbeCuge5NBvn0dI7H4vjO+gHOAt2fbQJzn4H00nTKyrNDYxwkO4YFbH8XdIioBUazXU9Nitx1WarJZ6Igkmk8CtIQFYwohIvmA9Jh8Hq0TGW4qMtzsw5CtPSEy23YTx8UM9jAB1YVN1t2cqx/egiICdwYeZznHMJNE1v+noljm6D2Rb2XZIEl4Zv8SYaRn7YHUKfnLj36Pd8zHKxfT+O4xRkHUMCbhDhUwKPlK9J1RWmNYmJdv2fmQv48AuVxaz6fZBRanqRZJuozCnPo4dm2M/Z3q+6v3Iko5s5tu1IKn2ZGBAifvaov37QtiDUqWEnYR9qNEQ2JOYSABC8ny+FcTzh3ss/8TQmj+GxfzDy+DB/PPAWP7V9/7zn/t6/9FP/u6f+bHBLzmwLB49jGf/9X9JVjqqScv6fCb+uyMv4SF10qKbiCk98bwglAFVOWKvoRdZoV70ZE9KurMetTXk1wY3C4TjnuxFjpsFYpIbKqcwWy2+ukbRHwgTN/s0Y/OBw16J7DPbiW+wuNA0d0OK4d9XhLhZIBYBvRHPn13Ji7B/0JK9LAhZZPZEs/5aICqoXmmaU0kSrF5p3G+sKX9nzvZ+TDeriv5RCytL9UICI1DCmuzu7c+xOxBJq/KKbKvHN+T+0GNv5AbUTQOmkYoWe6vp7vXYN1b8pj+pJAhjJ/LRyXPF6iOPriU0prjUdEu5ufczkZDOnmp2ZzLz72cSTDLUBuS3iu5AAkTKc017JB/Qdi2MhJsP0llhqco34uFysxT68ERuBKOW8ArlkHEykfK1obiG7Tty/HYjPrLiSuSqgyeSKDeg+S1sH+xrRuxGGMTFD3OaU/mQthuFr+J4M5ZfqxSysl/X2B2aGJv6LOIeNZQ/riiuI9sHMHkhabmDZmColhmqX4gQv72Gn8wgClM9eZE8nEuRiuo+Up/JNTUEdQxhI+2h+By3D8SDV74RWa/dqpFF0l4CPAAOPgtsHmpMneTRw81VK2OS34ivcvaZSLyKK/FOQkrqDVCfyr7YjfxvqISpLkQqvLszVDeIpNJNpapGQIFIbE3DWO/QLVKFSoq11K1I/96uDEHJl6njmBDbHipJDF3JTWVzLFLB4nYI3SAVosPQRyo36qmrLyMlDUc2D/QoUVUhbSvJJ5WXc+ArhV1FVl+Dw58IKzVIFqsLSTxuj/a9qC4VwQ+SVRAQMH0hN66Dj0ukwnDwqac+0WOxvXb77sVsl3xkSbLZzUUOPIxrsHKDqLzsFyAM5R1h5wdv6fxLz+a+dPYuP3GsHmVjJUZ7KF2wNrF3po/Mn7bcPpYQkIFdy5r4VtdjSgM2sDuVdNU/vEjVkWL62tMsBcz7QnpOu5kezw3A5EI6WIcE2UFW6kqVqm3k5lWkjip52VLStBFpansgY6NTYm515djck+ASkLEqrj3dwoyS1+ZIY7dyXFkdMG1kdycbE2L7KnUQB+lXLS8DppeuUp3qaLzd35wPgSXdXDF/2tMcZ5IanWS1iy+cVMyUkiTbzXVi6mUs7TawO02BOpuY0pUj/VSPr5MhzdkVCluH8aa+n2qCUUxf9+maE4a1myrmz3p8pekrGeMhSdiVInt1VQo7SR2bQwJvNPJaG2Sg0ocp/aRDh6rsizCWxTqMXsCoFbs7GQdfNPSTjFCIrLibS1r6IFHOdpIHEKxCt7JNXwjA0H0gFFqkrG/tb7CKfqqxm0Bx2dId5ZhaAElMtRwhHwKNvCTWNrKdYUKhS4nKrtSpI1W25wtDtnW4iTCPIO9X+VVHf5ALsMmEqVSpB3ToBzXtPvF1GBvtZMInZops3eFLCSSSZGIzAptQmNGnPQCnAbBEq1F9GKWo+L3ncJCsmtpJmEwn7OLktXjogtHpuHSqDIq4aUZxUeMrO8pQYwqI8YX0c7ppRlb7Mbk3GE1+VYsf8aCUzsze0x1PpGolRqLV2KtaejABs2roj6fo3qNaT0wBOSHb+xsH2enQy+nnJXrXS72IThNLdT/2YYYiE1AUQHdO3iuHipGhWiXJW4mR/qDE3jboXSdsW+qvBAiLSratNXpTS9WHHyYk7Qjo1LaWhFmtRcK6bYQhbFowRtjNuh0fQwgC+nyQFNlcPJ3EiGo6YiX9Wsp5aFritBKvpNEpGCcS51O4WaMyQ3QeVRYSsOM9qhTWM3adhO0M1SUwVpCMQTl9x1f6KgfZ9nSy7758O601DMFBbp+464XFBIi92x+j1nuvJfypUth/reUvgOWfuPwFsPw5LPOPzuL3/vv/hFc3C7LM07YZ06pjsy3JrKe5LcgmDn9ZEIuAygM694TrgmhTMuWyJys8bmsxK4N5sKNb50wOa3bXFcoGuJU3wcHjGA874i6DIkha5cwJA+MVapMJaNsouhMvrEdQkEXwsg7lFEPEdZw67OsEXisvwTuZpEHGlEzqp35kX5SXfkwSE6M34nuIRcSsNe7YoTphX5SXbcfSo3qNSt67UA7MjBrDSiTlMkAWMbcp+bLTYBJw3YqP0d3rsM9y+oV8COs2Fd1PnQQSTT30GtVqbArlGTyFMogQSgHI9lokJ34A6SeO/DxDOUV36MEgntOZMD261eKlahXuyJG/yaRbM3WUSuG9BACZrbBCQ8JncSn/r+87slszBrrk15r21GN2WsY1quTdGgJW9r6jCCldUhFTWMvQMfk2W2NvjbBFk4jqZX1DoIwvhHkaWJR+Id5GYRXUyJTYTWKnStmXYAVs+4mMVVTQnvXY62zfnVq/5RUrpHM024nE1pfDjKoa/VntqaN8k1jfUkJ5lGOclBj+rnxiCJIUN9g9gEAh/s+7kWy9H5ehl3MYL4D2RPy/5SsBMMFEFp/D6rEEGLnUQxq1vDbym8RwhD0AEr+TsEO6l8TXfrFnk+JbQH0ItRkCZsYi7YEZSf7IkFIb+3li+8rI5KWiuSP7OFzHdq3GPk27FZ+l6aQSx64TW+H26/MFI1NnGnkcb10zWQP1aeQgSZmbU5V6KuW5WS3bCIUcnxl8ozqxWIn16Wd7RkuAb5LBemGgfC4ANb9OrMFUKny6uQQYFVep4xXZpi/26xn6OoeuTKn4SZ7WVkD68NoOiZHzFckzJtsqrsUH2C/knAab2MsqsUhVYsIO1HgcdivrGDpqTfL3Dem5OrFJpk6/t3KMeznyfpJGu2HiIZ3/JFHsp4wVNnYroMSXso6slnCjgTmz60i/kJqnaCScRzzScl3p1Gc6VEAVN2nyIVXeDN5Z6QkVoPv2xIZJ0mXxXwow0V0cx3YYf528rjHbn/N+JkzbwJQFK5LioZt28CRndWLgcgFpptn3U5oueR7tnjHMUk/pkBY6+GztVsKMska2OXSqDnLi4e++SOOROm/lmo+jXBhku65U4zaLtaddiLxyeIzUy6T3rz6O57CfSGXPsH2f7/21UYvndei11F7qeLIm+TS3Anq7qbwp5FupPdFOnt9PRII9LD5N0AQrlWPSTyvnK1/L59lQYyaTLCFNKIhf0aVtD/5aAcIheadVqgka6pvS5EeG+B57kZqSwKaM1aDKSOc8U2Olj4pxrLExXZB1pUmcwRuZbX3ytQ4TgEkpMTJ9OlV+qPG52c6nHs2UReCi9Id2YfSpx0yn/k0/jmfIZZJE+ZjYNz0el6Twpmuh8SNYHNhI3Yn/MmTiM5VjjuN+mTb5M0Eem+n0PpWOZ2APey++zVQ3ozuPLzNMLQE8Y5flwPwNjOMAdltJZR1+ljdLnXosHbHIRrA6JLgOtSlRa5ET+zj6aZVPjKLWAlbbTnopE6s4pLai1N53mZk9e5hqUb7Sdfl2D+YQzAPyt8FL6TxD7QlGCzgc/p+WAROMoT1DjcjbYDAk/6RzI9Aen991exA7VKH0/biffwRz/EkhPn94eSvU55c+vKe8F//qu//Zz329/+hnf+/P/Njgl9xjGaLiLx1/yeOTS/reQFTcXM5QOjCfNKg84FY59k6NmfXkk07qJABMhNMWnKYoe6rDGs5avDNkF5b2i7nUcFzlqKDAyHOqR2uywsnfUsgLThO3GXglkqs7Df1hQLWa4lzSX1WnUEedgDkTiVYAm76xuIn0JpkrS5h6kdktfEpLFEA1sGvRRKl8uM6onqT4ZgPFa4M78Oi1EYnbjUEtByNeArIIsFC9EmDdpg++engzVgIqD90oK7HXEkpkdsLMxQjh/Vqkh0ctplHM3r2lfCqhOMuTDdmNQXeK7tRRPVqPTE/5RhOLSLZK6W1bAUSmTYXZa4PuFf37NfZWo9L+5TcaU2vC1I/VAeVTSz8P4s1sNeW5Tqymp3phUigB2JWk03aH4te0V2YETQfvX4MaQoviPuG1EeBnOoWfBPxU0l9VhLDsyVeSsGs3cpNQXkoYUPOoQ/Uatwh0p17SWhFA1p6EBIoU7ZF4HAdw6RaSHutLYQb9w0bSeRHZY3mRJHinwjZ3ByFJ1iQhtjsImFbYGbuRsSbIuLWnnvqBEynj+80YtGJaKF9nwgIOEfxFFC/ejaJfBkytqF5pSdQ9crSnbgwcUkNqbgGrb/aYnfRv9vNAeQH13SAAdbgxzsDsxMsbjSSKaq+4+Pd6yis1BtP4IrHXb4TpJQi4FTmmsJDtcaC8EKDVHwgItGuoH/TYjRxb1sh5URGaU58khBFTC3BoTwL9VH73pUgkp89IfZdyg90tA32SaIpEMmLXsp7mOEk1ezmWfhFpj8KYuKtd6k1NN6DN6V5O5kth4l0Ji8/g5huweTeFAKXeznwlN1z5mjEZtltG+oNIvt4HEhElebW4kXNfXkiq7FCR4SZyPeQ3CVRWMHklrG7WyP71c3mtV+eJ2V8LGOyW6fsBIxtZn0oa6+JTYeaHdNuoBQS2R5I4i2Jk+upTuU6qVzJe2Vb2afo8jPJQojC2wUooUHsgjKfUWLBPza1jArBpXbUwlfVZpLgU3+kA/qqLkK53Aa19YnNDAlB2I/s9fZXk2QkcCSOqKK4HeSqp2zBK2ispjCrKDWyw8niUBB5lW2GS85Wwa7u7inaZQmuisFsykSGs9MC0u4mwpHa797OOYNSltNep9O/uzlIP7yayeOIZOmr7uQCP+TM/hh71U2H43ERAbHkj+9YvBOC5iaI+0dg6yTmT9LOfKrq5+Fenb/wI9OoTTbEK2G1ME1+S3GrrOD5PACVJJgrlbQIFQUBRtxAw1S7FA2t3svL6yFDcegFXyfeZNdK1KUnAcT/uedruLmC6SHnlx1qW6as+1XzEEYiZToB0cRvoppp+oinWgXwTRnAzVIHMXjjqI0NfaZoDQzfXNIeG5lBTXEv/ZnXuxomsrJb90El+6nOdgBxjom3Uauz4VCHSLuSzKBqFqQPN0lBeOrKdp10KcxwV2LUbk14BqtcNykW6mQAuYbXTtnORfA7dptlWAEq7FJ9hTMCmPZJZlmwnDD1aEaymnxn6eTaCyaG6I9jU67rppYZDCwM7SD2HXlIVIrqT11OwOk0USnLsGBbUemImHZu+lICgQcLZTzKpCBkqQazG7HoBqrlMOLi5Hc8XIFLaPrFiWuFmlpDCeXQnPawqRFTjJAgoyVpDbpLnssRPMrrDQtbzNrDOtEzcl5kA9l2Sp4Z0zQSIRSZMpTVSaVJKB++QFqt6J1LWBFoFZAZn1GmNAAAgAElEQVTI5H8jOBz6MpUiGunmVE2q+HBe2M3EVkoAUbYHcErJ70nyGgdWMEZiZoRd9YG3+zXHAJ8BhKYOTZW+0FrY1q6X4J4kw1VaC0va99Kvad4CkVrt92no33x70Xpc/347CfwPQUt/0pfS/JGey79YfiGXX2rGcvL1e/Gdv/NfsJzXOK+p8p7bncQtax3YrkuWyy1tL2+iu1VJVvZoHelqy/HxhsuLOVnh8E6PqbEmC1STlt2uwK9y1MSR5Q7vDEfLLVc3YlZROmKMdCnWlxWqkg/fatZijGw/eo3OPVoHXCcywph6GRd3N6xvJsQ2yWnS82OTfGURketamRHV1hOcRq0sLIU1VROX7u4iJgu4OoPE7H3FgxWAMqBqw+Tehu2bKWhhREH+r6ZOALICPesJu/2bgiqCPH5tiRMnfXYKxqRbr0Rm3GlhU51CHbfE65xoIipKn+HQtah6Ray8bN8pyN/yzmnQN8LcxplH1QZdK/GIAboWNjfmEWxAr7N9H2AWiYUwr/pG2GM/9QJMi4jZaPxU+g7tjcEtRJJMALMSCZKfBvHrWYhzh7m0Ui0TBZDbW+kYVMnfFXJJQR2YwmExdbpxS/st7I9KjJkS756CbJ2CBqyAbJ/WEYpIzAMmdb/ZW+k0VF7hZl78cIkBG6aIfJFqXYJMJgxjojs1+rwGBjnYOCYnDsm1kcSGOalnybYCMstXBjeLKdVQ2POQalBMo76STjvIPN/2BOpUUZMlRnBYR9QC/vu5zNDrVgDnIGm26z0LO1zLZic1K3ajZALjYOgJTOOXR6JFmGX91hi0apzhHxY3l17HIYF0SFbtF8LgDp64mO2Z42ynRrZ2YEvcRB4Pe7YyGpH3Dqyj7uTcZrUwfdK9KLJrFeR6GXxdUe/Z1+F8abeX7wIjizRKgtN5HJ5jOgGNOqXWDsztEO7jqzgmIlevFL7iK8mcIPswJPIO6a4DAMt2jEmtA/M1sqle1mU3e09g1Ht2YmCLh/M6dP+Jp1NCc7qlTBh0CwHabpJk0n4/NiSWTLnEzJWJyU2/hxwGlntIAx7Y9HyVklAVox/LF4xBXdkuJoYxpt67tMk+jv18Q5opirGGBAYmUtYt/jq5ObdbuYZclX7OhFUZWMpsK68DEBAYs2H76bz06dxEGbN9QMwAANL5Gc5Fkp5HIwyfBPPsr7EhTdi0jCyc6SSlFp1YxyRjHq5105J8fnKMQz2Wq9K+JTXO0MM4sLh2G8VbNoCVBLxIYNDu4shEDuyrinG8duSaF9A6vu5Ktb/WdjKegxx6YJAHn+GQdOuLxGwWwo6FTCZBhv0aUkmH9Uazf10P4E5kxvLZ1x6IxNXuhJGMiakTcCHHua8yYqykGaTC0suZPJPpXGm3Tw8mputll/x7Kj2ml8cMqbbKM06aDsyuLwW0Dl7EqMWr6CZapP9apMty7Qt4HZ9biOrHrgQIukpLXUmZvNeZwnQBQkzrZ59a2wepH9k5XGkkZbY0I6hWMRLM/j5FJY9fMAlAD4C0Sz2VqdoEHwm5BAP5wowSYN35MU1Vu4Avs3QNRpFOZnpMoJUHJfVNpkdfpu78yFSiFKpzxMLyRzyBcWBezQjS9E4qPaLWAjrzLLGKe+ZUxUgohdkcKkyG9UVjGLopRyns0EkJjAE/A6g0bwHRFOSzB3iy/7GwEtaTtoHNiJlBrVOtSZD1xb7/KmuYmEFVFFIRYrRUi6QU2ZEtVHoviwX5P4jcNQHG6MNXGdFBHjv8DHtJ7hAA9Cctg28zhj8HjOVZ/KuP/n9gLD/++3/mxwaQ/ekP+cVdTosNxwdbjIr89Xc+5pvVC7qY8b9cfIvnmwO+c+cljc+4bKacVhu+LA/5a2ef8rtvHvPw/g0Pq2t+232Db5y8Idee3/34fX7zgy/4YHqOVZ4fb85YdSWrtmRqZeaodpay6vjo9A2/evCCT7envK7nHNx9xbuTK35wc59p1vF8fcDkeM007zjfTPmNs2e82B7QBcO2y7m8mjEpOvITz8X5nOXRlt4bms/nvPvdFzQuI0TFq6dH2ElPUfY8Prriqp6Q3Q88fXPId771hNpZPnl5ymzW4IPm7tEKazxaRT774g7zky0PDm55tZ6TmcCmLvBe89e++xN+fHlG6wzWeKZ5z7opmNztOL+eM5/VbIuCadVStzkxKpSKhLKnbzOOPrii6eXyeWex4sVqwbTouFpNeP/OJY2zPHl+DHmgOGyYVi1NZ3mwvOViN+HubMPz2wNWr2fYwxZU5OxwzdMXR1Abpu/f4r2mqXMWxxuazhJeTTh5/4r1rkTrQNda7p/c8GY1o9kU3Du7HvH0q/MD7n3rindmt3xxe8TV7ZTCetRZxO9yudladuA0J0cbVtsSe7pj+2bKRx8+52dP75IVjiwL3Hlwwa637NqcD44v+OHze9w5XHN5M6Moe5SKrC+nTA9r8sxRWMe6LmmbVCQcFGFtwQbyRUu7KsjnHb7N0FlA3VbEr23JdKQqejbbUoKNNgUmi4TSE7cZhx9dsW1ympdTTt+9Zr0rcZ/O6O912KqnKnseH17xBy/O6HeWozsrrj8/JJaB8tGO+C8OqB93qCxgbOD9u5d8/vqY0Gvu3rnl1dMjslnPZNpwezNBXefc/c1XPHt5RD1NATNVL+EuzpAXjmZdQKc5e/eSV0+OyRYdfZOBU/jkH85OG9pVjp44sB7vDNFp2GQUF4b6Gw3lJyX1ex30Gj3riUERtxn9QyeycwX2oKWvLflLS/FoQ31VySQGoHtLf69D7TKiDZipw2ce93IiDNSBw0573GUpILPWhGUPjUE9buRxpy3BK1hbsqOG9qrEbHQC0SJjUgcd/cYyvbul+Xw+TiSoXuHut3BjyXaSSpxtFPVpZPntS3a/e0J7GLHvbdhdldirDF9EYfWfSf1Pe9ZDENWDdOKmG81sHzBjbzW+AH/Yj124aDA3Gf64l0kbp1CdPH76TLP+moREVM8zXBXp76TH9Zp+HqleadYfeAkxakUBEPLUHzqVlOX+IBBtILvJZMJj2ZO9zAVYloFsq8cQGt2liYZKJkHK15p8Fbn+1UB+aUaZu9lqJi8Um/ekG3j9gSdb6VQyLwoJEHlqH2H7rsNszAhcB5+zSrLgkIGfBCb3N/Q/WpCv96B+8Ef3s4hbevILQ7uE6hzqu1J1s7snx4qKlOeKzSMBR+1duWGy14axQiJ1sso+yPOlegS2jx3FGyMe2cm+8sdPIi6lSWcbAQuDpBWVulr1HtzpTlG9kU7T8hLChLFzVTuRSfsS2ocdsx/nMumxFQ92v9hXE2kvoHz6XJjq7UFiYrcKuxXpcFYzTlAU15HNI2FT7Rbc0b4KxBcpzfhDNQYFhSKx11rWZZr95MP0hfiUfQnVG2FdlVdjOJCbimx58zgw/0yP/uJREo9MMvWz1B1LCkRK8s7Ne1C9TAzpPPmGE1C2m0iohKkWyam8nnwu3Z0+V9Sn6RisrNdV6bxo8VkPgLafSU9syCSFeEhw9YMcfq5oe5UmGKC6DLTL5Cu9lRoaooDf+iR5La8lDbdZ6rG2yE1Fwpw1yT88E+A5e+FoluKJDUb8td2xgNtmacjaSDdVo69X9yLpFfCtx0kAAO00tg40Ryb1tia1zyTJcL2MUzfT5NuAmwpjO9TPCHiGfBVoDrPR+ytVLQJ4tJNz0S4L7E5SYlGgSoUr9Tg5E5LfdfSNJ3ZZJPJqDD4zbRw9vUToDibCQrcBP83Idp6QCxA2TWITtSJM9xOTdt3j5/lYq6L7xIoaAbBuZiEl3LqJSRLktF6txFc6yeS5raefW+yqRzdOAnm0MI3R7CushrAk8XwmgGvFK2t2vYDTBEAjiRm1MjsYSotOjxkTP11IExQKva6Jk1JkuI0wmaMHdACSNttLa20m7OnBfAR7qukk3RX2jGYCnDiHyq1IdJ0XcBnjmBcVfdh3WQ7MZIgSSmkzvtJjOSwp7Ic/JKv9115iIP6bPP4XefklJvX+tOWXmlfOleO/+uC3eW9xxV+ZfcI/uf4Gf/+f/4fcKTd86/A1VnseTG6osp551vJgfsNVN+UbyzdkKvDbX36D79x5yc6JpDSveqam47eefpN/dvUemkjtLGfTFcfllr988jmLouFvvvszupDxj19+xBerI37t8Bn/4slDnuyOuNpVbF2ODwLEvvjhOzw4uGXdF7xaz7k7WfPewRX3Tm/51uFr3j244j/41k/Y1gJa/va//3tsu5z7s1ucN1RHNfdPbgRg2YZdJ2DlaLnlR8/vSSJu2ZMZz6ToWBQNT88POS63fPj4FbtdwaotOZzU5MbzN979mOaiovGWInMcTWruzdf8zbOf0nvD3cmav/XBT7h6ecCv3X/O0aTmYFrz0Z033D+8ZVJ2hI1l11oOJzV9b7jYTfibj34GwAd3L7htS17dzHnn3jXzuxvai4rKOh4sb7HGczSp6YNhuy3RE8d00lIUjqdfnvArj19AGQhBczzbsTzYUreWR0fXnLx/RYyK3DqUgsx6njw7oSoEWDV9Rt1n+KCxhaPuM37/80e0fcbZ0YoQFA8Pb5jMWopJz3KxY76o+drhBd2qIEY4fXjNy9WC2UFN8IZZ1fLyeoHzmruLNT99c4fvPXrKpin44OycadExyXsmy5p6l9P2GZe3U3brgghMJi2hM5w8vAETcc5wdHdFtyooJx1V1eHebehvSrFABI3bWtomJyscNnfMD2qKk5rLyxmFdTz48A2busCYwL1ff8XDd644WuworONffvKQvsnQuZcJARspDxu0jjQfNphKGO4YFZsu5+hgi7GB15+coPKAUpHVqiIGxcH711ysp0wPamzVi/1jm+MuKpSC5qYkKx35ueHyZsbkdItb2zElNlSBOHf0tUVVnnLS4doMrSOx16hlR3OvZ3m4pX6/FT+zDdh8mBGFGNQ4C2+ygLnK6O711BcTMBE76aDT9CeO5e8VFHd2qDwQrnO0joSpFylz5YlfTjBHraT2OgW1QTlFlnnUqQRZxE0mXuinE2Gt32nwB4448eIjfl2gnGK3KYSpaRRx6sQ7rSIY6A8d4aCnf9Ti54GLVwvpXF04musSoiL/aEV5IeCuPQy4mcjbVZKtxzzIvp2mPrZSfu9OHX7hUK1BeYWuDSRWXmXyGN0qwsIRbWT3GztUrzFbTXMq5e/mNiNbS2qwvtuwfewwO43ZGGIpbD4HPSGLZGuNe9RIgq1LvXQ3Gvs8ZyibJzGRbimS68FPaG+lFqk7iFx9V8CZL8UCEI14gLe/uRvBWdQRX6bwrJTe2hzL4+s7gfJ1lnznic1Te2ZSWOXI8kea3fMZplNs7wd8GYVNboTldgtP9SzDbhSmUezOktd5mgLD8ogKKfAssfr2xoiEHpFE64FtV6IQkMRfkt8b8guDq4QRthtFf5DY+KFr1zBK3aORLteQC3ue30g90cB8NidDeq18dUsBY81x8kdmkdlPcpFXV3H0n+pe7etgKvnbICcurhTVGzUygYP/uj2M9HOpizFtStYt92yhpJvKerJaiZQ4E+m0S2y3XQsg052w1c2JgO1sJ88b2Da72Ssbsjqy+FjTT9N1VO3Z7agGNYUcw+D3daUEYhUXIiP2yR/rJgKMfZWkwxNhgbWL5OvEQHvxiA5geagw2t1VYyposBJa5StFv1CjEmDoGo1aCRuuJMxrCKeKSiTK64cmMZHQLvbBSr5QlNcB3cvfBzZZArfU2Hm6vadpD4ZeR9icZeKBNXsGOmuFcXcTYVC120uh+4nG9AIWvU0qmRjJGrkoNvcM3opEupvqEcQByWOqx7qibq5pF3uWNyQffbAy4ZAnkC7yXj16he02oIJsSx4r+1Lc7Fku3ccx5XbwyMr4D9kEgWwb8EViVJ3IfU06jqjlGHylZRtWZL/BymNNG0aW1M1SVkaSGPtcWExXadzE7L2bRmTBMVO0h3nypPaE3Ajo7QWEZlvpbgxVhpvlUm0ykdqYASz5MhOv5MDIRSBEdOtGBhIl110sjMhwCytMqVbi4xzCk4wSYGiMhBgdTAmTVHEyKYhFTswzYpULsANhOm0mXZqDRBYYU2DLPD3PEqtCvheWWOZQlQJUh0TYwaeZZZBlqCJHZRmqLEXWaq08riqTJFdLUE9alBEZLIPMNn0pm8mXUvuf/7ivPEflOTq3/MXyi738UkthZx+exb/zD7/Nb59/i1w71n3Jp89PibuMD7/+gs/Pj/hbX/spbcj4X3/6Ed97/CWvtgtu6xJrPNfncz587xUv13MeLW+YZB2//8Ujpv+84p2//YRvHLzmH/7+r6PywLfee0HrM3a95RvLN/xvn31AVXVjDcemLriz2HC5nfDu4TWfXRwD4H42x3x9Q7Mq+OjxS3762T104YlRYQuHMYHvvfOM/+vzxxRlR555tnXOcl7TO8Pt9VRuGF8V6Ps1eeHYXlWc3rvl4pNjOOzIrKe/LihPa5pLqYfoa8t333/GJ5cnbC8nMtNeetRljr7b4Haii1O5x9jAZNKyupxCr3nnvQusDjz58gS9SX6Lyo+S1fsPrjj/vbs8+MvP+ezLO5zcWXHx/AC76OhvU5qZVzDreXB2zdPPT0cW5Zvfe8LH/+d7TL59Tdtn1JcVmMj8ZMvmy4WwPQ8a1OuCcNphXhYSztMrysdrup8tsF9fEX5wQPyVNfpfzWm+1kr1xlErdSIaVK2JVaB8asUL9+6W/tVEKjMKeaM+PNyw+cExbhoJlUcltke3CnfgydaGe999xcV6KuyYAjvrCM8r9P2a8GyCn3vp3ywi6qCDi4Iw8cxOt/Q/WNI+6ESmnAeyT0uigoPvXbD6/RPaOx7daMzdGu9lRjv7oqQ7c2RXGeF+Q1hbpk8y6l+p4VLWXSwb3NMpqpd6jeLhhvrNBKx4T9EQvrlB/8GM5t0Otc4o32j49RV9lxGfVxJQZJB9a+SG089lXMoXGd1S0nPtRnoSlYmwzSQgaukxG01xpdk9dBw9uOH6syMmLzTbx47Jlxm7r7coHWFlya8MvohMXyi6hXgkq9eSAJw93sAP57TvN9inhdycLx3Tj3O2H/Tkb2RGXCWZanGt2D30FBeGqCPug5rq/5nQT6G965g8yfaBKL3UPoQsYjdSq2GvDHYj/Z3Kq5FB061m/kVKGFYCFLpj8bROn0kS8errnvknhuY00j/oiLVh8lRYwGjlxr56JanI2U58t5NXiu2DSH84JCdDWDjMVYafBA5+krH+d2rCxrL4acb6A49yiskLuZt35V7G2ny9IW4yspUhv1WjpHUIvmlOIrMv5UazviMz27pT5CthnwaQsPhCOh99GemPHXpnmD7V2E1kdyaAxG5h9e2O439qufr1QPXckNVQ3xE2b/oycPUtWfeQQFu/1zP53JKvRL6qHXRzYT77RWT6LAWp9JHtfZi8VrTLfbXG5GVk+0AxfxK5/QBmX4qUrz1iTD4uLwd5osh8y3N5bnMaWHyi2d2LHP5EElpNK8ExpPPZJ9lzloKUpi8i68dw7//wXP5Khl1Dv5BAoOZOZPEpXH1b0qrtljGcJr+N6WZW9sVNRVKskyvBNMKwFLeRzUPN7p5IystLAQ71HUV1LoFG+W1k8y7MnsD2oWL6TADQ7fua2TNJNAaR7danIlsuz+Vcm06x/GlKp50KkEJDtoH5C8fVNzLKc/FuZjvZTrZVHP/I0RwZuplINjcPYZJYP0k7luRiU0N7DNNnkayV/W0P5Fye/EC2uz3TI8spXk4BPtUbAQO7u4pZen6zVBQrYeLaQ/Gwrt6H4lIxOQ+s3pUE3MUTTzfT2CRF9nYvYR4ChbqFAIrBR+sKxfSN4/ZdS3UlCa79VNNXe3YSGMF2eyDPn74KoyzY7iLbuxrTwux5z8V3c4xU+KJ7OZ/dQlKMo0pjd086jqsrP4KjaKBZaqpLSYRWianUToDX7FmHmxrW9zMOvuhTP6Zne5Yxe9bRHWT0U50erzj4vCcaRTc32G3AbhzNsaWfSjpvO9eUN+KzXT3KmL72IxtoN55gFZt7hsVTRzts617G5I1H94HmOKO8dGzu2zH8aADKxUqOtZ8KCJu86ghW0x1kBKvIV55+KtdAdd7RLWxi9SNuqslX8pnSzwzFVU93kJGvHPVpzuRly/pRweGPVmwez8lXjnaZiV9166hPcvK1J7+WSb/6rCK/6dk8KJh/2dAtLHbn6BaW/LpL4TwZ/cxgNx7T+FTLIvugNx39iSTU6s7Tz6V2pF8I+FMuUlw2o99T1CIaU/e4WY69riFG/KxA106CfFwQwJnSeYcAnWgTUA0BfbsjHM4IucGsG6I16HVDWFToXZc6L6P8fbXby2CVQu0aYm5RXU+clJIsW4hfcvj7KIXNrSTGwshcqrolzsQeNoYFabWXyLZvBe1o9dUgnsSAxqb9qldyqGRpGlRRyPE2bfJBKtmH3I51JLHrJam2KIhtO4YBfWU7f9ry1uNjjLKef8vlF0YK++A//bmv9x99+g/+zI8NwHz/+9//s96Hf+vl7/13/+33Z//Rb/Jss6T1VuSaWYA8EpRi1xS8aBa8qWf4qHmznlN3FucMu22J0pFNLzLPq+2Ui90UFNRzhSoDP3z5DspGQmO46Up2vaV1lpfbBW0tz9vWBXWbE7ym7i3NLmfjcto6BwW9ESldrDM6q+j7DFt4fJsJg9Ubvnx1QnSavHKsr6YEr2ldhvMaX1uxIUUNladdF6gs4pXCKQVBE3qZTnU7C1mQmhUFb9ZzvDfi4dRAJ9JEVQRik94ovCY4TbuT/cUrNm3BtrfEm5w4l5tukQ8amDnW25IYDDc+JzpNGwyxyQhBSyiSV5Kea2DnLarwxLRPF12FM4qmzsVz6jWq0bRejErKK4JWY8IlXowp0US6XkBu31q5oXNZqhkRDVfsZJpbdSnsyMvNn+4VzmVj4BKdgc5QNzl6Z/ALMYgpxLMXKmEtUHBbV7htjmo1KqRewRuL80bAQw5ZrQlZlPG1EfJIt84haJTTqMaQnVu5sc9guy6TP1FuBL034pvdZagoYFF5Rfkkx01EfuW1APM4CfitRTdpW4BrLLrTkgacPiNca4WJ6LV0IqJorYKrQoB3ENZlSL11hw6zNZKOa0mJuuAn6Rx4TX6VZti1GrUOUSvqTSkAuZEZ7GCR12Kdke3SDHYRcTNZp/LC+oQc+i6T2ptOgptCLteZCkr8xex9YDGT8zJuP0JwkoobrWiPQi4eTV+kmWAjx6q97FtWi/dDvJKDJyfNJCOpu37wrmXyP/G9yv9VUPgJqI3BNFr8oFEYNruR9Uqvovw9ZBAtmJ1GRTlusxG6LdtoQqHQtxkmXV942Z4Kwk5kKfU2WKQzMwpbJMckr5FBOqiiGtkzDGNS7kD56oHBmsr4Ka9QXiePbDonVgCY7hUxatnfel/GjpIxru/KOI+dhBHp/+zV6O9TUZJapTNy/5ruZ0r8t8gY93NhtEIhqcJuomQ9yaMYjMgtVXjLN+rkb6ZllE1KCJjCl3sQEfMhQEXeB3Rg9NkOHjsJaxJZaj8XdlMoQzUGtw2yW5Eagq/UPnE2eYmzJAH1pfzcT3SS9qkxXMsnmaX0gg7e0/S/XG7MYiaAWpipQXgmY6+9TBKYbqg5SfLSNDba7499XF8OaBlT08l6+ql4M03/1vXhSYEpcjzRDGFWjBLMaAdGVa4ZeaoaE2eHc2a3qc4DNT5fQtxUqvoRH6V2aqyJ0j5V4wTSusWX1s9EHi1S3SHZNO1/J/Jh7REZoiaBuzQ+MSWfsh9LnTzo8hpX9NNh/+WcByvrUkGuS5N6SCXxWaXPIyVBOSbtg1JpvFM6rLg7ML1MDGVdSrBTgNH4QsBYlnotlUr7qWVfXSGv7/H9JQFNUqDNILc0TupQTDek8IJxoFM9zZBCrcMw/vIc45LkNMprwCScob2A4KwRn6N2cmyhUGStsOQh17KtXvZnCMoDuSZC6r7UqTYkGmEYFXt/sHgGZSxEAZD8miR/aoSoxasZrSZmGp8YQx3l+BSMibQKAbOhNGS1T6/duH8NKYUiEnMzBikphYT5dGGU/2qfmDyj0S6ge0+wRiphUkCPTsympM2mIB9rUH1g8GsPADPkmYz74PEMcq+ikG1L6mryeSreYjbj6JXESJjR6KFMaa267dN2Bv9i3IO04fvANPrkbR3SZAegNpyHgT39YxJb1fC4wSs5eD3Tfux9lGk9Cvk9CLAevJeEsE+IfXt76Sv+ofX/sfs0pEW/7e38N1w+4w9efv/73/8f/61X8HNY/u5/8w++/3Dx3Z/7ej+5/t0/82ODX3IpbNNY/unv/AqvXy+53Ey42lV8cHSBtY7zp4ecLDesL6fc/uRYAnt2ljwXGaXOAjEo2k3BYtLQbnOaXc7hfMff+M6P2bUW/7riaLmhOqoJURGCZrcu2DxdoFSkry1+laN1QOlAc1OiTGT3eirBN1FxeLbCNxlm3rP7eEkMiv6iQumIelYRGoO+zdCFZ3tborYGtckI5yX9VYmZ9igdyU5q4qsSWk32UrZ5eLyhmLUM/rMhOVZPHQQlbF9j0FOHyj2qchzev8V3GgLYRYcqPThN/tpKCM7UkT8p6FcFHMi247LHzHvCRMZs+qOSuOw5+N0SZQP2ZxPyo0b27dxCKcm2qjYc/ZbMlsXSE6uAeV7KB+r1oMmHWEltCzqi7++onliyZSfVLXca9EkLGooXljgX6eXxh5eYtUkJsZJIa68zSZY96ojHnRTRzxx++v+y92a9lmWJmdC3pj2d8c4xZGRGZmVWucrubty2BFiYFxBSizck6G7gB/BG8wa/ASGmJwQS/4EHqHekFsZ4qLJryIrMyMiYb9zpjHtcAw/fWvtEll0yRoXKaXGk0L1xzzl7Pvusb32TR3bLOhHZ08MWCof5X2QYLgYgY4l8+UqxQmEvkd8o+MJj9nOD6ZcESPBA9aMSw/yuxgMAACAASURBVLGFuZMYlh75tYJXQPlWYfaEwKx4nuHozzTcaU85XekZTV94+MKjekkQZbYSwmLshmQIjMfJnygmygbKDoMki6h6geqLDNMvDVmlrYSf0pemazF2adop+1OLK7KKk+ca3gRUv8ihGoJnVzFJ1i4c7MyjfGmQ30hUb+mry284WLEzJtYufqZQvhVkZnoxBtFkK4mjH0uYrUD9ycCO1aXD8Y8klj8TyG8I+nzpGd5Tc3DVfY9UwPGfS3QXjqAsDygvJcor9odWb7hP2UrA7AXKdxIuZ4k902OB+S/IEA7zgOXPBOzSoT317AKs+CVU3Aj0S4/Jc97uggw4+2Ng8jyllVIa2DxwYxCM2QnMv5TcnyqgeWQx/wJoLzxmXwHnf8pe0X4eMEwDjv5SYPaMfajVq1SvwOTY4lpg+pzMzOSlwOIJ+2XzO6YVT16SsRpmAbPn3J5+wQF2v6BUc/4VcPqjgMUvxOhhBLE0hkVAtwxjTYmwQHnJ1N7qMgahxAFmtmVib7ZmIM70a2D6goPn/QdMnJ28Jng8/fOA/Yce86/8KOUrrwKyDSA7BuuYPd+bbYCLP3aQA8GkjEEzy58TZFVvGawyTBCv64D2lGm2xfUhUKe4DuiOAiZvAoYpWcrqbYDZAEefewyT6H3TwOILj35BNu/8Tz3shKxnv2AHY/JIAnzP9FWgpHIAZi84sDr63GPzKf+Wr8hSCsf97BfA5E1A9ZosYrYlW0mPHX8vbgPmzzzMjp4+1QZMX3o0pwL9Api99Lj3f3Q4/txxMOzYGes1sHjqMEwEJq/JTC6+8shXZDSPf2aRStrzNYN9yiuPySsyZyc/GXDyE4t+IUYQp3pg8tahvPHYP5BYfB29tVc8brOXHsWtx+4DgqDlUwuXCRw9sZi+dmNgTnnlsXxqMbn0mD8jE5Y6QWcvPI5/1qFf0r+abQNmrxzBdAMc/WLA6Y87dEuJyaXD9JVjX26sbnEFAejRkwFqCDj7sxonfzlAWmD+3GL+9YDmXEbvHydNpm8csk2IzCPlrqYJWDwdKNuMqbf7C4lsy3Ta5AOk1DEF6DAkx2UC+dZh8VUP3QWUt2Gs79FtwOJpD28Ejn/aYPayx/zrDvNnPVQfcPRFC1sI2JwdpounHe/bObdBBLKXkzc9634a9pqmoKJ842BLgWzrcPIXW3QzheLWYSgFlp/vWL0igOqdRb52WH6+Y+jTziPbOlbdzPmdkK8shAUWT1tIG1DcWCz/YoX8lieyuGZ6a9DA8k/eQdqA2Vd7DBOF4l1HJnIqMX22gysEpi8azL7YoXpVI9sMmP1iDdl56MZh/mQLvXcjm1q9blG86yBcQL4aoBqmyxZXLco3LaQLUA07ROXgWVWSRxVGpVB9tYHPJKZfbjBMNYp3LVwmUT7fIrtt4Y1E9WwD3TgME41holG+reG1QPXkFj6TyK5rAEB2uQccuzfNpkd2uYPZ9gSkAiPD1p2WEDEEKBgJ2TEsUW9a5F9do/ziCiH2kuoVmcWQ6ktcgJ2YsUbEZwrqZseEVwCyGcbUV7FrIHoL2VrodYNQaMjVHvJux8mhbU2wmwCp9WQt6w5+UsRxUUDINUKVk500GmLXUOKaKkIAJss6gl5Rt0hJrqLpKJstcv7eRKtHboCazCuGlK4lyXCmn6lrU8Y6EaW+EZYT+p6VIkohdH1kJmUExNFLGpnT4BhUBCEIMIeBjOP7/5yP4DQc/jn31/8Lnsv8tj+iJPrX/u/vyONbLYUtPn0Q/p3/+d/D2/0Ml2+WQC+ZGJoFyKMebm2gdwruvIdQHmGTQW1jGEfOyg8R2Qph4wxn7BQMBmNxOgKZjn7pY4dYgFmRBfSa8iL2/YEzlnF2nQmGsXdSstJAdUwDVXuCAR/TX9n9xAEzB45hTGBkqECIgzWB7sSheMsQC1vGmdGGX7hMpAzINgLdMX1L2VrCliFGfdMbM8wwpicCZITGnrrZoXsQMnpb4ox9H1NJ02x5fiuw/8ihfMUaie7Ex8E4ABHQHzNRFIGDUFdgrFExGzF6jbINgzVSt2N+GyPM20OXHwvhGeqhGs78q57HQNdkw7zhQLVf8LylDrmUlJiWBwD9EWVlQGR5ED1VO4H6nkf1Vh56A3sOlrujWP8R/UEuS/6Y944bJ7WhmngNpDTWlAqo0/nmteJNrJeIyaS7Dxj4MXvBAWdKCE0skMs4AG9PGb7BkBAOsBkLf9jfYQZ4Hdg1eR4w+zoxQvFDFD/+qceQ1Rpcdn7HgZkrKFlUbQoV4bFWLd/HsnqwjuEoJZ5iXEe+CiNbYOMAVLWxAmLK88WBaUxCdJwpH2YM7hCBxzfVN/QzyidZORC/2PuA9kygekvfkYqx/0ELdAugehfQLchMCM9lewWU1zGZU6aeOvq3zJ774jVBiQgMlMk23G6zZciI6kLso4wMVpdAVRjZpCxK6HQToGvK+HzG4yU80B3zHJstT0bQDPBw0S8nQqxt8DzP1dvDvSFdM2Nqpkn3gChtNJFRweF5EwFoSg32WoxSvW8kUVYES/2c5xTAuB/pNayaYIpo8q+lbtAUapKSNLNdGLvtEtOTOgxVZEP6Kbeln4nxevbR3yY8g0mGKqbnCv7fGQa0lNce/YQVHP2MoSnexM+PoGS3uvbop5FJTKmabRhBRzqOqo+DyfLwWUnpr6naw2ViPAamjv2JgqxHSny1pYSNAUNZrF9h+qVAPxXj3+jXFMjXDl5HNjcyhy4TMA0rMnSU93rN455tffxcHaSs/M7i/qRrQHc81vS08XrpJ6wN8ZrryHYeLpeozySKWz8ec916yMiMJUYu2yQJaUpUjtf7RMIZgWLl0M1ZYQIAZmPRHenxWmTCKOWqXgPVFSs1bMFeyBEYOqo8dH1IMvWRKfVGwGwsU0htAm8EeGbvR18k0jHP6LsTMcWUXk/2OdqKJz4tX/gwSipTTYgI/I7XezeCjfbUwGxd7J0UY7cje0J5DKU9pAUHLaEbThrYSkF2BBWqdbCVgq5Zy+EqPSaKytjVmDodVWMRMslUVfCaTL2PKV0VLsDn8bWxgoOewPi51wKydYAUY1orU4AV5ODpZTQSuh6YsCq4HbzXcjuCkYAL4+dbDtxu2TlKOwVG5o/pq2Fk6VxpoFpLNnCInZKthSsN5MD3ey0he1aE+FwTQAFjqA20BGwCWHKsHElS1TF1VUUGUAiyoD3rTUKmmfyqZXxO0PvoAkFiO+CX6zNGFjH93bpx2ekRCsP3AqwTAb6R0BqMprRVSdaISIYDja9PUtFf7p6UkmBQSfxydyS9j+oghTX60HmZJKrp/yEcEl7HtNb3cEDqixQRLPqA9zskx8qPBCjT8nyI0tdUc+e4LyEQDCY29lc9/hZ9lt/6VNj8XviDh//xr325P/zqv/qN7xvwLQeWJ98/C3/4P/77yJTDk9tTZNphkbfQ0mPVlti0OfpeY1Z18AHYbCtUVYd+0NDawSiH3moyjgD2+wKL+R7fObrBn7/4ALZXWCxruCBQ74txvUIELOc1rq9mKKY9nJUoyx7bVQWhPYKVOD3b4OrNgl7I6xmycsDwcoL8wx36zuDB6QrNYHBzM4XOLYKXcIPEYllj32QY9hlk7pAXvFF0jUFeDmg2BY7PNtj89ATZpxu0dQaTk4Vt1zlk4SBlGLdzaDWqeYu2yaCUh+00Qq2AzCOb9ZAyYOg1hAgwmUWzjvsZq0jKRQvvBbyndMQOCkIA1aTlMREBfpAIjWZFSR+TE7VHOe3QvJnig8/e4c3NAlk+oL6cQEwtQsNv/clZjabOELyAVAxAESLAewlXaxzfW+PudgqpA/xdBn3SYthmmJzyfY/O7/DyegkpPYZtTqBkBVB4yMzBdwqqcPBRVimN56RarSEKB2k83M5ATSwW8z3Wm8k3vif8TewKzeIkhAfC1EGuNfzCcsLCSZhygL0uRg2AmPesmrk2BMU6QJRxfVsDUVrozMUEVQm5VxCDgFta6DsNezpAFxbieQk83sO9ruArj9QLKXzsUAX7KLtTzr6GwgMD5Yv54y3q6wrmTmNYOtarTChZSTJa0UtABuizBu51BZx18LWGrBV/T/UzNc+rnzjInQIkGbkkkwYAv7BAK1kdEwSgAsROUUo4CFa2qECm2YQoxQTc1ENtKdMOJsDEiZCQB8AyHEd0PD5+blk/08YBbSvg7vUIrWLS68JCblg/4yYesntPlOEpDx2OeCzo1Yty4MDXq0bClRxkqa2Eq+LgsCejmmKH8xuF7sxBNpQY2olHfqswTD1c5WFWCsNxHIg1cqx78Rn3W+85iSQCJ4JSumx2q8b6EYAAZpjTUyqcGKtbggxwFStvILkfSWIZBJCteXz6Bf19PgfcvY4VRU5Ar8mI6x3Z0bRfuhWwMcxFNxzAizjR1i89sjuJfuEP4TACMGtOYhG0CwwTTmyxF5UTZeyNjBNwEZQmkBYUxkCQ6i19uGmCDOCkUvFOYpjH7doRmKeJMTthKmt3FMZ6mv6I7HiaBJFpQiTKvFn9Q3DWngSkKqAg4/scJ0GKG4bluOxwGaVaHBFltao/TJjpOoJ2RCm1i5Ne+gD+U/2K3iPKQiOAa4B+dpjM7Bc8jtJyQitbi7ESR6QAI80JLLJziMCPk5j5nRgnC22U54o4GSlT0mrD9w8zxMAdTtbY6jChmK4NFyfwEohsj5mUastDEBBL68nquhwo37FqBIj7JeKEbfR4m5oTQMOUkympqiMqkQ8BUYHsJetDCPxS2AwnojgpkapNVBuiPDQC8z0nyAiY+Lo0YZAAbKpPAvi+fBMwlIeEz5Tgm3pLdcNJpeT5HStS4vbKGLxjS742VfI0p5L1JvGYpWMcxv3lc5RkcmIlTRSljlXdpn7Ow0RFkn5yXw6THD6llEb7gIlhONxYvJfMirHGBeJQ+2JLgeLOxW0kwA5CjLUtqR4lCE5c2DIy+luylN6w4iR5Pl1BgO9KCdkF+DxOIMbJnTSxyGTXeK8IBK6pw9Rl793XxSHsKr3GxcoT9prKOJHLPktnYipsAuOWYDF1Z/o81qJYPwIh2Q7wuY4S6SjdlAIqgm2vJeQQ60qifDfVlvicFSNi8IBKVg4Bte2YCgsQICsx1pXwwgwRvHoC524gGE6AOQHFlLyq1QHoJmDpPX9P1SQhENim1wEH2eoIIuPf08RFem70b0a1mbUHVjOBSOfIsFrLoB6A4HUYMPZlvv8IAd/wW37jqV+NTb71Hsv8IvzBg//o177cHz77r3/j+wZ8y+tGWqtx3UyxLBp8dnyNq2aKy+0MubHIlBsTOwttYZSDkgGzvEOpB+z6HMu8wXUzwTxv8fMX9/Dxg2s8f3eMP91WUMrjsw/e4YvXZzg+2qNY7mCdhJKBMlkvAC8wLTvMixZPn15ATweYzMJaCSU9zLTH9fUMR8c7LMsWX7UGQgQs5nu8vl4iL3rMFg36QcPBQyjBvstOI5916K5LDDKMKZkhCGSTHnc3M5z/gytcXi4wO6ohAOy2Bf7RZy/w1d0xNu+mELmDUAGmIOg0mYUxDpOqgzzxKIyFkR71YNBo+lPrTYHl6Q67fYGi7DEvW1ytpljOGuzbDJl22LkCIQhs384gJwP8LoNe9JBFi+AFfFAo5h1CAJp9jqADXnx1BrPoUGQDZo9vcfnqCAgCat5jf1PBTHtAAfamwOThFvt1idBJyEbh9s0CZt5Ba49mIjHsWduxXxeQ2uPrr88AAA5AGgGIqUVeDmjvCohWYnKyx+YdKTlnBeROU157lUPer4HpALfOsHp3TLBYK4QZKy/UaYfwtoA+b4AgYK8L9olOBc4uYoDSfIC9JagMlYW8M/A7A1GxQFuedDhd7nD5ZgnXGPp7bjIMCwvIAFEr+IrAECrAZwHFswzF7+2wOjJQLyv4pYW+NQiKTAhkgN4q2NMB3ZljV2gn6SFVAf60x/BkDjH1GJYOohewM8/E0yxQdlwriPOWns1fTOBmAbjJiY0DIF8WkJKARN4S7Iqa3k7VsFLDFx7la0UfYEmfoLqhbtJnEbB4oLiUqD/ykHsFnxFAogfkQF+um7kR8MDzVPqCAF7embGLU70zsYsxxMEICCpbiWwl4fcG/YmDhBz7P1MKafVGoX5A2bOu2Z3pswCnCKJkz8AdvWP8vuwZmw9BWa2qJbyhAkD24LaCqZr5tUJ34pGtos9QAeZW0b+oCMJTbyRDYMLoG5W9gGwj04TIZBeAagS6hz0mT7KxVsOsY2Kq5Wg/dS7CC2TbCDyH2O2XMeDFTgPyWwk5EPHIgRMkwhGUQQDlO4H2jEmtxXWU4lqQxahiP+CWcsj8Rh56IRWZ+myd6kGicsNTXWB2lAsX14e0WBW9yanXVNcEOUFHsFPz/HMfAbHjADf1nUoHBEuWvFvymHlzAC4IQPVaoLkIUfrNYzpUGI+xdGSMg6RnUEWgmLowfUzqHGaxVsQeAAC9uhyEJ5UE/GFQnm3I+Kf1Jv9jkOznNHugXwjKXi/EqEKgvJGsIysnxAg0zZbHgAN5/q5rAjgAY3KqcDz/cmBgT4g9iapFlIEyaCfI2DFa8FyUl0ym5fVBNQmP0wHMiPoAgL0h25/ADZDYTQKUYc7ndRtGNUxIqZiI6gNBQFld+RH0en0IG7IVAUO58ujnAtk2dueCqbPZzqNbUNnSHpHpdgUDeYYy+p0dqz+cOTDaACdBEsAVkc3NYrBQtvPA7sDk9/MYaLVjCFE/lShvGcwDIPZqsk9yqFilkfyJ/VRG9logeALy8trDNBxM9xPJYxRTVIEIHGsPnwtYeZDXIoBy0o4qp7Te/JZssG49+qlCVtMXqGIli/CBr41ex4N3mWA027rRD6wbT++oEMhX/F23HvlqQHsSL5AAmL2DtBFkeUS/KcGbtPRgukxCNZw5SrJuW5KVlRHEqS7KOB2Pt+wDA3lmhsAwpsZ6I6EGTqy6XEK3Di5XkRkNUDZ1ejqCzAB2oHYOwh1CdphKLCA6P27TMGMIkNr0cJMMsnNkTycmTkZagkMfIKyFcIlBPShx9KaFnRdRlorYqykg1zVETG6F9cAACJU+swSForOsDLEeAmSw3weVYrAIOqNPsh1GFtRXRawhISuIwRJchjCCSjFYJh1HQIkQGPiTGEzvGbSTypETu5seyScZxHuVIrxOg3UQGgScqXYEIKiMoDQkZtUYwDk+F2tL0iO9Rih5eP2veoS/4flv0+NbTOr9TY9vNWM5+e798Af/wz/D7yxf44+uHmNZNLjczVCaAd9ZXAMAtkOOn727QL0u8fDBLV59fQIxSOQXNfpOQ8iA2aSFEAG7Oscn5zd48uocJreYlB3ubqfjLH/QAWeP7nB1uQCcwOxsh92mhNQe3koEKyH2CuffucHlyyPAeKjcwfcKYZCYn+2w2xbIvizRPkz6KgAmAB0DZ8r7O9TrEvrKwC4cWZ9eYvpwg+3thO9pJeRsgN8aDnbzgDC1ECpAag+8LjjIuddCPS1hH7cIK34piF7AFx5ikMjWLGd3FWsKQhbIKG0U3MxBTSzEiwLuXg95YxAU8PD7l3j59ghYZRAnHcTbnExKwXTR7/9rX+Gn/9djhrB0Aovv3aLtDZq3U2Q3HAQMH3WQ2sN8XqF91EPUDDMJpUN2qRE+2yM8nxBEFQGiF8gf7RCCgHsyhZ15MjUC8CXZIbPhYNieDpC5g3xZMAQhMk7ZSqI7daM0GhsNc9EATyZAAIaF55eODsjfafTHBDqyE7BLC3ND5i6/40y8LWPa5TQWhRfxhicBTCywNlCNgPl0C/Enc9QfMYVU7SVcZA3VLt58TUD1SqK5FyA7YDj2TOEVAeZNBrNlZYHL6Z1kAA5gf6uGfFrCTgI760BJ7e4jD9WIyEzS53b7DwhGdCMgO/ow+yOH6oXmsmPv4nDkUb1Q6I4DsjuChXRs+xMek+VPJNbfDcjv6Hn0GUGQK1mXkGaPIYH2zGH2pcL2Ew952kE9K5CtCDZ8LKMv3jGNlVUUiHKzyEDtyaCpPjJescPPx0AWXQPNfQ/Z8RqcPZNozsI3JNN2EqKvNKB4J8Z19HMAgmBvmDIYKFvH2oWFx+S5Yg1EGRiaEidJ0+x/EICpCQr6RcD8KbD5hMEq2QpoTznAzO4EbJT8AodKhPoez1Hq/RumPGbFNbct7Z8c6Hu7+y0COtXGwbyPbNxAsLP7jsXsCx09opTsJgZLWi4rMVHZluCnnwemr+YC3QmQrYDmXsDsK25Td8RjrhuCq/acHlevuaziJsrxPVBdBtQXlAnvHhPcZRt6BhMzmZg+AKP0WHUHqW8CatmK53yYkAVpTyjhVx2ltn2sP0iMi24oRfU5rw1XRJ/knNszzHhRlu9YocH0Ux6n5oKVGElObbYRxO8JNGavLOpTBd0G1PeYniscwaHqDsDSG7J62Yqg730QaisCKzshU2n2YayOSDJg3XhsP+DgzRU8Bsnu4HIxdlGaJmD7gYx+0QNjmK89mlMJXfPclpd8P32JAtU7z0Te+wp6T5Dk1aGWxNR+rMDINx6yZ3BOPyFoSPLjMVgtUN6cOhIT85fYPpvH93WUoUvL684WIq6bjOUwEbCFGBlgn9FX6iIzq97rZAQIns3eo5+pkRVObJwIgDMC1ZVFc6q4/JJgsZsrZHuP+lSiuvYRqBHQDZWkNHnHdRV3TEztF5ry4Vi7kW3p365PFKZvLeXBwMgs5isHO2EicbtUMa01YPWpweQNGSYf5ef51sW6HA9Xxq7E1o+TcUniKjzQnBlUlz36pYbsA1TLrsVhIlHcWUqLtw6yd+iPMmSrAf3SjEyftIF1HZER1TtKMW2l4Ao5AjpbJN052ThvJPTewuUKriTTCCGgd/QUDpVml2QhYTaHv5ntgPpBgfKqH1lOHigCPq/pcfQ5lTDSecCF6IsUUK0d2S1KkSnLDYL3NNlYuHkWJ8jIEMIDcnAY5hmym2aUsLqCnym97dAfl1CNJesaJzpUM4xy2CS/hRBkBl0ANMOD7DSD3g8Ee85R3toMYBBa7OHMNERr2UfZ2vh9wf5KuIBQGibBRqZSxHXBewQV/zZEBlDKg+xVK8p4dzXTYAcLP68gNzVGma9WEG0PBjxFljKxmgDQtEARE2dDOEhPf1lWms5Vep+U3L4ocRXGjM+HxHaGKN2VChh6wEf5q5QQRYGw3x+Wn5Yb15M8k0Kpg2T2b3iEof8bX/OrHn9nGMv7/+Gvfbk//Pq/+Y3vG/AtB5b5xx+EB//iP0MwAcsPV1g/PULQgYP30iMYsjpBBbjSY/pUw8YC5/qBi/I0ATt30CuFxROg/idbtK8J4KZfK+w+cTB3EtlGoLnwjKuvDx1WLuMspCsZZT8mBDYcoLqCFQTNPcrI2gcOk2dq9OsEEQM14mBYtxwYzZ973Px2LCqvBVxGxqN6E31/78l3yncB+w84o9/c98ivYvDLmgOP7ScYB9h26nDyJwrtaWQDkl8o5zY3FwQH1ZuAux8gJvrFSooIBM7+PODmdwQWX4AeQMH3zZ4B9f0D+HJFjL5/6bH+WI6F4pt/2CN7bXD804D2RKJbHqoG+oXH5JVEe5pKxznw75cB1RuBYQ5Ur1m2vviZgiuAyVs/+qy6I6D5cED11Iwz68OUpd7NmUB+G2fCJeVyZst1LJ5w9rw9Jbj0miyOsBwYDVMxys7a84Dp1xj9hN0RMH0RPYEeWH/m8fH/MuDd7+VwBpi9CNg9JNDJbwSKuxDDMLhNyy8c1BCw/UCjn5FJ2D32OP4xZ93vfhBw8mNG9E/eeJidx+axBjzrAJZPPLYfyigX4zlafkGP0zCNg21NX6Yckq+T4SL5HUHM9BXvA/2Mf08z+t5wgI7ATrpsHZDtAuoLiX4RMHkFLJ/0ePePcwzzgLM/97j5bYXFFxwgDROB8sajXUhsHxO4mT1Bj2qB6SuH299SmLwmCKiuPPI7h3f/2GD2nAEeqj14iYMEqncetz9QKK4DJm89Vp8qZOuA6tpj/VixdH3Lz5fZkXnbfyBw/FOH/QWvmfM/a3Hz/QI+J3MlLbD9KBbRR0nc8ecddg8y2JLX+dmPBtz8wGD+teNgdKrI+CigunbYfKhR3HmoPmB/ocgAtCFWKHRojw1Uz261YaJQXrZ4+69PcO9f7rH9uEQ/E5i95Jf8+iMz+qGKlY/gI6BdEOB0MbTF5bwu5MDPQTc79NFlOx8rC3i/KW8oTds9kDj6fMD6YwOzD5hcWsjB4/W/kWPyKmBy6bB5pHHxf27w5g/nmD3ntSQ8MH3VY5hrtEs5SvOSl23+ZYPV9yoC2YED/vLaspOuD1h9ZlDcejQnEtU7lsYvng1wuYTLBHYPJZZfWtz8QOPeH3XYfphBDgHZ3sMrgfJdj5vfKTB9Q4bF7D26hWJgy95h/bHB8U8avPv9CuW1hy0FJm8cmlOFfONhizjZIoHJm2H0x939lsHpjzv0cw3VevRzhWzjUF9o5BuP8k2L/QfFKO/b3Tco1m7swaOE2KM50TxPlmBvmAgsn/SR8VBozjKo3iO/GXD7/QKzlxb1mcb8eYftoxyqDyivBtx9N8PySY/9/QzltUW/UKjesoaiuO7RLwwg6PVbf5xDdzwP2caN4GDzkcHiaY9hpqF6lteXlx18rnD7vRyTdw7ZymKYaZi9G72BzQl9kOU1z0tx1aI9L7D5UGP+3AIeyO863H5/At0F5HcO2WbA7lGBfMWKBzl4tGcZsrWFaiz6RYbdQ4NsyyAbBGD6izu0jxYwmx5BSeweFZi86WCua6x/sISpI/tnAbO1UC0L6ZlhILnszqFfGgKR3qM51cg2Hqr36JYEPNnajiCYnYc8Z8WbHQBg/8kceucQtER7pDB92cEbif0Dg/nTBmrfw5dm9AIKF7D5ZILiZkB3pFG97ce6CluyXkM3FmrXQ3SOAMN6NA8mkDbAbHo0FwX03qF4tUH3YA5VW3QnOYrrFu1pLzKLDQAAIABJREFUgeK6RRCUZGYv71B/9xTFmxpyW6P59BTeCJQv9xDOYTiuoNcdfGUAHwiSjCI7JoHmwYTrenKJ/pMzmOsadllCDI7hNAJQHT2Oes1ANWFjqExvx0G+n5J1648LDBOF6ZM1IIHhpEL2ag0/KTAcFzC3LWTdwVc55L5FKLPRH+lzTX+mUVD7DnZRIntxg+7jU+RPr2DvH0G/XSHkGfysgNx3CErBzXMEI5F9fQN3OodoLdwsh77dj/7B/nwKvesJ5pSAaHokf6EvsjGUJ9WCpBCe5MtMMtHhwTH0ze4gK43Mm2j72PGoIdd7hEkJ9APCpIBc7Q7AbrDwswlE14/SWmSGtR+7GuHiBHK1RZiUEDtWi4i2R5hVrBbJM4iO2x4y9kKKuw3CfAqxbwg4k08yejH9bg85nYyy1LErEjisJwQyhVoBXX9gBhMDmIDme52boR9iT2V+qBMB3kt5/ebrIaIs9v0kWOAQ/pOWO3wTyIrxvQOEUviVDyHH7f3Weyyzi/AH9/75r325P3zx3/7G9w34lgPLk++fhkf/5X+Cs8ke//bFz/CjzSPMTYvbvgIA3Cs2uO0rXNZzKOlRqAFSBGz7AifFHk/uzvC943fovcKL7RLdoPFvPvwS26HA1ub4/Pocv3/vBZ7tjnG7r1DlPY4iKypEwHFZ4/VmjuOqweV6hmnZQUkP5yXW2xLTSYsPFmvcNBXaQWNwCo+P7vDzN+fQ2kPKgGXV4Go9xelih8EpOC9glMfV7QxZbrGYNMiUQ2s1jHJY7UvkxmJedNh2GQanEILAyaTGXV3ifLbDly/PcHq6xbYu0O4yLI/3cJFGmuQ9nJeouwxV3mOwClXeQ4mA19dLHC93uFtP4O5ynD6+BQAclzV2fY5tmyMA2N5VULlD8AJF2aN5O8X971zhek25aZZZWKvQbnI8/vAKz764wORiD6Mc6jbD9+9d4vN353BO0FtqJbRx8F7CO0G/Z/SNChFgrURRDNi+maE4bTD0Glk+oK0z5OXAbsYAaEN/6cVii9c3Czgr6T9XXLbW7A/V2qHvKP8NAcgLelcRAJk5mNyi2+WcAS24n0IElJMe+3cTVGd7tHWGyazF9q5CNW/R7OnvDE5ifrxH/YslxIf70Ts7WzSwTqJrM7itoYzYSvZDeoHQKkB7fPL4HZ4+uQc5G7CY17h7M4esLLyNX3BbzaqChQWcwNH9DdbrivtlHKXCKrCaIn1n5J6pvr1CiDUsUAFm0mPYZZCFg99RYgwB6MLCbjmIyxYdhusSaidhTwZgoMRUPGwglYd9UyG/kuh/0KCatNg9nwPLAepNDjujL8eso7dzPiBYSQ+sAGDFgUHea4ZcqXDwyG40faReADKGbXX0K7oFB4xqo+CLGFJ1o9CfWYjSIdQayDz9mLWEOx6Qv8zQ3Ru479cGXrMzU9SKHtpYnwFHX2f2xtDXJwNC7pFdariKkywpPTYowM4c8iuNYeah+uiJmjgg95Bbnge9j1JQyUkenxPw9vcHTH6RoT+KYUMqMqjHlH0FE6B2VDOkdNdhSmompcP6ykHu+YWcvJyJaU5BWbqOk2glQbTeCbRnTCmefK1hJwHdhWWlUFxv+VqhucdU4KD4niS97Y+oGhCeUlkVZa9B0ouZjmVxI8ZQsX5OSWZ/6lBcavot7eHYtPcsineaP19rSmsF5b+p2qM7CqMPTg5UkQhHlrE956RYd8x97449srWMsr84ORblyLoGumNOrLRnB4kyfao8zq6grFQ1GJlGSIzAGeCEjak58ZFkoUGAntYsIF+RGUUgE+dyThylZbiSkyDDjNeTrSLLvRZjQJMInOgIOv5MpFL0SY6VEjay1xnTd7M1xjTfxHJ7DXQnPD664aSSbuN9QnAfknRTWk6K9HNO6OW3h3V3S04+yR5j0JWuyfqKQIbUbKhAsAWZ2nG/HdlYO6E8tV3GuhdFJj9NlIzS6Z4SV9nz/2RPY1iP5jrpjSMjrvpwkM7G64Y+XrLfDB/i5KEr6C9UPRnocZsrgeImjMcl1VHYSjBYah3QLwXKdyFOoGHstmSNSjxH0SdpU1VNn0KUyBAnKWU/40Sb13xNqovJ9pyIydeBYVKFhCvI6EobA6PidctalDBWlSRfqm4C8pVFc6ZHKXVieNXABOUhsqyJYbYFGebkEwV4jQ0TGVNt2RXqCslgpVLGmhUPG9Nfs51DP4v3JRvGsCxBxTyZ4ZsB3bFBedWjPTHIby1cSV8m5ef0/HrNJN0gEMPcKDlm6JAYzy3ltH5kZoOiJBcxwEsO9D0GCejajaywjymxdqIpp5ViDGMKcZmuoPRWdpyIED7EuiIygJTKenijYtBTIFsaCPgBAlrZDHBVNnoyZTswrTV5KaOHMghBkNzbEQjLXcueywR6E8gE/qqEVQq+rx0O0tpUFeL8N1//PkuZPI8pqCiFAr0fYvT+TwDBewghCPgSSE2g1b8XEgTEoJ/3JK0yyWbdN4HlL3svI3MKAL5t8f/28f8Dy//vH99qYJl/8kG495//p5CTAco4ZJnD4+Nb/OTnj/j8cYNuVSC70vAfNwivC7iTAerGjAEbduYRMo/qmUF/FGBPBsjMITiJ4Dk4yt5QUtKfOMheQtX8QrPHA9RKk+GIX4R6z+f0TqB9zEjH7FWG/pR1H9gY6J2E2fCLtHloMf+5Rn0/wM4dFj/XaE85aEyM5hDDMmyU9okAZN/bYH9bwlwZqE6g/aBHHtcz/VKhO6GUsX7o4XMPs1EQAwd1qia7JWMYjTdMd83vAlY/8Fj8XKF+wOvCFfShqYZVFLtHHrNnEt0RZZfNWUB5KbD5wQBzq8cUvwBg9jXoIRro5ZI2+saOKWcDGKZipwFmLdDcdyjfKGQbYPuYjCW/8DlAyDZkXbOVwP6Rw+SlItO6PwzMpAX2H1uULzRnVGc8ZosvA7aPYk0D4hdxyy+pfkkZKBN4OVjzmgPR6m3AMBcx/ZQMpyvJxELw9d0RB6i6Dth8B7H6A+iOOXA2e4FsFdDcE9A7+ov6JV/TL4DiKsCVHEi6AqMkLI/F98mDZXaIYSFkPF3BFN78Jp1PRPkgxgCJ4oYDnd1HPNfFLdAtuO3DnANM4YD23KN6JVFeB2w/TLJDYP+IxfbwHFz1Cw7w6vtMz1UtcPTE4ub7GtIysfT2t4F7f+TRzyT293kObQE0Dx3yd4psaDwPsxce+3scOKVACeEC9g+Ak58E7B7KsXw9BbYUNwHrT8nQL75yuPpX2IVWXPF4pnMse57j4i5g9wHBxzDlfuWrMLKzxTXlSs1Z7PkLiPJieqyCBOp7ZDw3jxXy24DylgxYfU+OSbfVJVk43fA4NRdktl1OJraLg90xOAPAze95PPrfOHhafcKy9XzNbdENxmTI4paDZXigX/KcZWtKSZNMNEmkXR6BhGP9xPoTgsXTvxzQLRS2jySOP7c8P/ckjj8fsD/XaE/EWAKvWtY/3PxAY/GVx/5CorxOsjyJzace02cS+YpAwGwCihUTVwmyDmmw+YaD19VnCmbD7SyvPJpzierSo1skySzrd+r7ARd/TP+cy4HZc4v2OKZgF2JUe9BvxfVU1x77C0WFwgSYP/PoZwKmxig5lQP3LcmJ+zllruvHGpO3lEb2cxHPAfe5uqY8VFom30rHJNaUZiocZaPtUjKJWAu0x7yX+oz+PhG3O1VFFCtPpqinvLK6cmiOFUzD693sAqavWWafgmaYksnvhGzrxwCTdimx+j5YYXPnoVpeX+tPFOZfezjDao5+KlHcOtZy3FeQQ8DsOdlQ3XqYrUO3ZJUEAlDeWIKGPf/enEpUV/QoLp+2uPusgG7IJjtDhkR4MqftkUS2C5i8Yon9MFNojyjFFZ4gZvKmRz/XyDYWzVkGrykjze56rD+rxrRbb1jtIUJAvuZxlzbAbB2GmRoltAloqtjfmDyPIoa02ImCLRja0k8kll80cLlCc07krfoQZaMECO2Rwux5x8G/EJQySsArifqegW6ZtpvtXARwsa+xI1gsLzs0FznMhmxre5ZDtR66ZX2G8AFmbykbrS3aYzK8/UKTZUViZnt0JzlU46BrCzs1GCYKZmtHsDsmigZKXLuTglJXIzFEoJRd7TGcVASehUL51R36Bwuy0pc1hiX9emrTA1rCG14HwnnIzsLOCwJSJWBLhfy2g+wdutMSZkvg4goNs+3RH+UwG87+2ImJvY7xntwSlMmOvkW9btCdT5DdtAy8GSgz9VVGINgzKTYYCbVnSmuSkvrCQO17iMFhOCqhImAzb9eAEBjOZ1D7Ab4yXF+mKJNddSMTK9uBYK3IgMHCHU14r9h3CJk+hOpoSm0hAdnasR4EWjGspjAM+3GRrcsM4Dz8NIese8C6g3x124zr+4bcFCDz5zzEdo8wrSglTQxmBJIieiND3UJMSq4zpbO+n7yaQKcPMcFVHQJ54jK/kVT4vu8xxPdoDaE1QttS5jq+wJNB9I7M5phQLA+psCmYx5ixZkQodWAs4zL+ihT2Vz3SOsMvgeK/5ePvDLC8+Ge/9uX+8OV/9xvfN+BbDiwX37sIf/g//Qf4i59+CLXoIUXAx+c3uGtLrDYVfBD49N4VXm/m2D9b4Hd/7wv8yRcfAZ2C6Fj3MdzrgVbh3/39H+F//Ze/C3Ovxv2jDTZtjrY3eHi0xtMfPYSvPFmmXuG7j9/iy7dn7IPsmLD68MEt7vYlUzjnPcNcZgOZk/gwiw5SeXR3BUTuIW8M3MRjdm+L3as5qvs71C+nCIbsy7/6b/0E//tPvgv0ZCxC5iFrherxBs3TOT76R6/x7McPIK2Au+ggbjIEHaDPWgyNgcpjYtld/PtKwS4cZMdSdFvF1ExJk39KHRVV/KDfGvjKQ28U7Nwhu1IYPuyY0GoFpPE4O9ri+kfnDGoB4CeOCZ+KNSkIgHvUQlyyQ1F4oLuwULMB8usSw5GDbGjed0cW1ZcZmgcO8riD2xpAB5hLfvkP5wPEXkOetUxW7SQZnflAX6mO6auDwOSlxO4jj/zBHvLPZmjux4TO43jTe5vDnwwonuSwk4Bh6aCWPcSLcmQj7CTKcE8d8ksNO4mJn2tKiMw2+vcGQbanjsEtjxqOey8LzD9ZYfN0yfCNTqB5RK9lKFxMLxX0s+7kGFYCkGUqrpiEOc7SPuA55o7Q15ldK8jv7eCeTOEqei2DoHzZfaeBelpyOyugO7eQDbstu2MPP3EoXhuC439Iv6YrefxcSZbLa67b6+hfPCfQGpYOxTtNduqsQ7jMMf1aoj1jd6Ps4sSIZDAPBKDWXJjw6YuMDF1+IzlRUwbYhUfQHnqtka3IZHkTeI1HhivV/9gJfZ/Vc806HS/Q3RtQfW0g+4PH0Wzor+zPLKZPDMFHShjdi9GrqDou2+ecUBmWDuZWQTcxvbYII+vnSg+94+eI/siA6XOB+gEZj/kXwOY7ZEp4zXBCqnlokd2qUSKfmLB8HbB9jHHCR3YYAdD6d3vkLzLYqWeAUSOQbTBW4SQWqXncY/JFxlCimBKq94IsW85rGcAYwNId89jmtwL7DzwWn0c5+UnsCs3IpqVtbO5FFjQHumPHCbItmSqfR+Bl+Z5+mSTXgR2hF1ymbqKsOTJs3TGPaxaZvbRP3gRMXxLoARgTX/MV/Za64WRFtiEYcJGlcbEGJ/lYm3OBbEtmzmwTQ0h/a6oCUi3DZnTNjs/yjRwTQFOoj604ASV7gekLTiCpmsDV5Tj4Nc94PMgwYaysITMSmU3N/szVZxKTl5TQF7ce/VSMHZn9HMhXZI8S8wSQlS0veV8dpqyOSXVICcgHLcbJheSFJRvI64tVGvSHqo5JnP38UAMEgZGpFC5eLzVZvfaU65Z9DBKKHlJpKc8OiizP9K1DfaLQXJD5U230jW5Zl+LyaBlpOWmUrwPqc07sFDcew4TnGCKC/T4cqr9a7md9IZFtGEyTfNfSHTo3izsC9mEqRk+rqfleLp/MnMsFpm8dvAL6KW0kw1Qgv/OwFUNsgmJvJfsuA5pTTrSkh24C6nOJ8iaMbHRQGNlJNQRka4fuiFL2fhqlmTaMKai2ZGiOdGGs3QHICA9V6t+kLN7lgoytPEyaiMBjkABDkqpLG8Y6lVTtk3olVUevprCpUknGz4THMFMobgbU57E3eU95eXFrYUsF1Tr0S84iSxdQXPXYP8jpH105+CxePzFFNlsPENZjWGQxVZc+2cmLGs39ktUuAvA5JzRczkkE4QNspcbJTb23o+9SNRau0vG+FmLKa4DsY2VJzsAe2TrodYP+fDpO1nkjoXc9ZDvAzQpWrfQuVtMw8EevWnoshYAv30uFjZ81vW7ov5wXfG8A5K6Fn+QQvYWbFaxGAeALDabBtpQiKwXhPYazCcx1TS+q9/Rp1h1BYOyzlNtmZCrt6RR61QDWjYA2SMlQHiW53C6Oc4we2Uaxj8sARu9lSIAQOIDWBDhTyE/yQsYkVmEM//Z+YE/dEEwnAPue3DXEahMkNnIY8Nc9Qghjkuyven78/dueCptdhD84/6e/9uX+8NV//xvfN+BbDizLTx+E8//iX6A8anA8rVH3Bh8uVvjF1RmGXuP8eIPLmwVcpzA9qrG7rSAzB2UcymJAP2gMg2JVxW1B6VrucP90jVcvTiA6iaOP7rDdlRh2cUDvBNROwp9GaUMAhAzAKkOYWEr5GtYxyMnAipBVAVVZ+KsCWDBtlHfx+OWyV/Bzy/8HAIYSuiBBELWjvDEF/ARNMBUCmD7aC4jjHn5noOc9wpviwMhWlPEliWF1vkf79YzrU4HAGOCAf5DAbABWGWVjF9GsrT18E2/eNYNyhmMPc0vgoxrBxNBGQe0l7JKhQ3KrkK0Z8DIcURqobgzc8QBzyQF+MEzsTPuVALCrPMxWMlQn8HX5pUZ/4qA3EsOphb7V9Mdu1KHbUgFuaQErWD8RwbKq5Vj1oLpYybASqB9Z6J061B5kMeZchShhlBxkHTHgqLiSqD8iO2vnHnotYaceuiZwsFMO4OVAhk5veI69CXCTQ9CQnYQxBENvBYIhCOuXHvm1GhNL9Z5prtkt6zhUzUFT8go3F36sYuCXMj/P2UqOsj9vAoYp00wJnOj9VQ1ZwNR16mMFCTsqCeRcwfoPs6OELiWxCk8/bH4jUb0N2D8Q6E48Ji8kmguCCa8xAsZsI9CeUlYJUPJnthL5HbB7RBAbdEzxHLiu4jrKKG1kIZMscOAgX7UMyanvcVBUvhVoT8MhIRSICZkc/JdXQHvMY15ehhFE6S1ZrH4RopyQ11HxDvQpnTPwpXolGF5zI2LMP9CeRNYqelXlECcBqMbnBETsGnUlPbrvh/+koBxXxMF9TTDTnmG8JrM1X5+tec10y0NQSkqsFV4guyNzj0DfdpqQ6GcccKsOsS+RzyeZYJKbNWdhZNfbE/aorj4DyiuBYULJpIkdkUFilNulwW0eWdV+Htl1H0FfwfqH7ugg0VMtmf5sHaWOseLC7KLn9yVDdXgth1E+1y2jD93E3tSTqAKIYDExqAlcZitK2UKSi7oQUze5Xfmdx/6hRH4TYo8kQU3qIM3vQpQTE0ABsYbBhbHKxOwO6awAj6srCZzMNowBN2PtRh/7TTeBlR13BKTFDQGf1/QRdxFUJzWM2ROgDKWAaSiF3D1QnGyJDKiuI1gqCIhsFStXNJlLl4kIKgOybRgDa4KkNJFSzJg4KjB2i9pKMPCnDVADPcSqI0vtjBi7RXVHyeP+QqK8DWMy6lAd0mQR6EkeJvTJShvQnGgG40igPWYAEdlogrbEeCWGUg4B+dqhOSWwEYESUN0ysGiIgUaqPwCu9LnzmmE+KZwoATX2HB+AZ3F3SGJlWA63tzlR4zVbrOj3TVJMht74UcqagKMrRAz78pFxdFCdR7+kr3eYSOR3ZCxVF2B27PXUe4vmXg5dk311pcRQSmQ7D7O1sBN1kGt2Pl7fapS6+rgN2W2PfplBN27srUyBQdlmQHdkYDb0xyIcOjD7RTaCOVsq2AmBWn4zIOgE7ggCXaEIHn2gpDSG/nB4QXAoHD2dqrFjyE57XiG/6+BKzd5NIWBnBtldB1foMXRI73r4UkPth7Hv0ms5ekSDEpCdI6No5EGuqmT8TufzXvM7WXVMbhUD3wPv4RYl01kHB1dlEI6VJQzX4aSD6Mm2ql0HXxLYJV/q+9UgSXoaDEEq+oEeyn4gIIw+V9EOBICD5c8UiKNTFYkDpKAXUyuCP2AEhsLFbesHhvYADACSkn8fLFnNLALCBPpSRUgCj4m9TJjAuYM3M3k7AYQEDCNjyZuU4u/OfTPZ9X25bPR//rWY4/9phyWAvxc9ln/PgeW3um5EyoCLh3cotEWhB0xMj+tmAiECqqqDkR5F2aNXCpm20KWFNg5FNqDKe7hswKYu2NFYOUyXNXbrEjfbCdRkQH48oG5zFGUPqTz6OoOeWgyZgVQeWWFRZAO2+wKucEwjVQGITGGWDxgGBV1ZmMyiO+4QvIAuHGyjIUuLEAQ/UzJAJD+fJpsEGfh9KgNUaUd5Chp6DJttDlFaBKmA6MHzXsBPOAK3M09QmT6zucfQa/iZg8g8QsdENsQewlDSo+hLBxdvvsEKBJviCwOgJIYFwaAruG12HoGrxNiPCCfgpw49KA2F8YATcHOCPlcE+MpD9AKh8PT+AYD2cBMxJs0iACHjzak/cgjG0xfTEjgJK8YON1+k7qvABG6Fcd+DAkLuYa517CELUT7LiglkoLdMRS+foydtWFD+HEoH1KzVgGNvoYj9fEFzvf0yyrd0DNFpxNjFxW3kPg6zWIouQVZPc3uCokeO6agEW3YSAP9e52ER4CIDkXxsLLnnMQiKx8SWcUAVtyX19vk8xv/792TBKgLfyLwgBjbBc3sAslZeAcomhiyBUA6KQ6zsSAFPKWDK6xCj5AnSE/MnLXs4hykHgC4/9BsmObAt6VMzvYBTARICiINsrwAVXyPA7SbrJMbjKzxZ08S62gKjJ8+VBDI853FCtRXjxALDspLPJ3qzyni9GIxVBqmHLoVVyTg5mwKv0jkA4uDW8HyzH47rbE8iII1A0hUYwQJ8/D36vhI7OfYWIvoq3eF6l+6wzz4CQNanxH1VBF2pUgMg+JCDGK9JaQ9JrbaM948ItoTHAdjiIL31Gf/JWFMSJOhjjtJEDtAwvj8N/IMgq6pbrlt1PAZAvGZ1TJUMcVLDBkAf0n29SawWE0HhCaBVXF56+BwQLSKjT3AyTMQI9pNnymvKOnXD56XjNgYRYqVIGCc7vKFMfrzPyLivkemDiDJYiTEhlpUHGPsuXRb3U3GbEMTBr6m4HhXBISXGEWA5MnlBCaRCeltwhxMbl+4PuiPrFeI6hMPYm2jz5A0M4/sReMxdBCjJr+3y6E9r+Bnzmv42RH+mMxgTXL3itSUcJz5YGyGgBozewH4qke19BABcXkpEDQKQcTIpMfBjP6SK1RtIx53LZcm9h7QHkCx8oFfN8dgKQfbSl2JMuU3AM+2HHIChSvJmQORkY70WI0OaqjZ4/ONnXwakpGJewIdrMF1bqmewjivi93nghEHQkaELAT6Th05FGw7f4fG1aXlBCo7iAhC0hFOxC9K+N1APgKt0vBdFT6YW47q9kvF6FnBSxs+nGNNUheX2eCMPE7CGYBqIywKPB0G4G8Hu+x7G9PkAAK9lnNDSkIMn2AuAN2oENi7nBD0nEiJw84ArzXvsvIR30TMYS6R93O7UbykGBxhFy58PkNaP8mGvJYRInx9N1tAFJrAOblwGoseR12bsvDSK4DFVdYQ4BlICSKmvNvB97wXqkIkEWUT33nlKoM46psBGBhMhMFE2yV3Tvik5MoyUx4YD6EzXfGJX3/dQJr9l6p0cr5NvgruQZLS/7HUcX/fe30MEw4KJc2OHZfJFysNn9a8Fl++HAP1tQOa39hH+yvH++/SQf/NL/u4+rJW4fHGE17dzPLs+xtvtDC4ISBmwuZ7g+dtj1Lsc4skEt28WkF+W8F6gtwq32wk2dYHmjqOm7HkG7yWwytC/mkAAqDcF+lajbTJo7SFuDYa7HOWXGXyrYQeF7a4EgkB2rRE2GZyVsFuDvBjQtRmGVQG7N+haA/WigLzO4K5zoJeoph1CL1G8NhAbA+w00EuY3MLMO5h3BnhZAoOEv8sRGgUEgfKVRvjLOZlRAcALYKcppxTA9KkGTIDeSohOMsBkpyBXGsM24zVda4hWkqUtUtIA4GsN0UuGpqwN5E6j/CoDnICoNWQrUFwpFG818jsB1UhAB4gtw1eyOwlklGiK3CG/paxM3xgUyxYi9zB3GsUVL73yrQIGAdlKbv9eo3qp6Hm0AnonIfcK5lZD1RKyVigvJbI7Nd7QdS0YYNILmI1gYIrxgAzcjoGBL9k7DbOJvXWNwOwZYt9bLEZvuD61l6heKIKqOLAStYJqJWs8JIBA5hOCQCVbC3YF9lyeCJQ0FjdkT7I7CdmKyMgx6CSFciQJJjyZRt2QGU2SNLPlQK18S6CZyuTzW67D7OIAKhBk6D1rRUY/7podjdkmDk7joLt8GwfhhgM/3QgWsg8C5aUYQZPZgaEYLSs/+iOH/I7rUY3A7IWnJA+IhfCRvWi4buFTB98BPIgBkB19j0Ew5CW/JcjUzaEzUbVM8tWxjzDVbAAYX2vWYgy34T9uW/JYmh1gNvRVpl5D2ZEt03tuqxwIuqWldLK4ZkCHtIivjxLAuE7h+B7VHoBiSj3W+3geIsNVvQ1kG3VkDSXZvSQbtRNex2Yfu/JqDvzNNjKY0f+l92Gs60iSYDnwp+opV6Rskf9PD+EOlRfZJoxS1GxLRm+UakaPstkhTmJwe82O57W443aqLqC84nbmqzACTID7ldJ4i5vAMBfHwbCuI8hNwCFKOKUN0PsQr8uA6jIgX4dxgC8t3+tNDBRx3AbhowQ0HIC0iTUf2Zbe4jIGsIx51NduAAAgAElEQVRsXsXBbr6mPJPXZGTiejKa+cbHCpcoS5zxZ7bzBO1Zur4Dz1cTfbvrgGJFCSIQgdHAdas+INtw2fSucRuTnxog+Mu29KNymw/nWHg+X6wcTO1h9uxCpOTTR98yt0tFj6jqgeTJTMdFDjw3BK5hDJFJID/bBZg9X5tYN9WTpUzLKlbuvUmBgwRa2v+bvfeItW3b07t+Y4wZV9x57xNuvu++WOVb9VyuJMDlJHCDZLCAFlhCogF0oEufpkWDICO6FkZCAgSyMZSNyraebJdfvC/VDSefs/NecaYRaPzHnOvc57ILi0JV13hJR+fsfdaaaY011/jGlwKmkzRi+XxKaIxp+/2EoRqnD41xeWSzZ8IcJpUfWMp8FQafZH8NkzouNkYPpYC8fmLPsIjRM7j9+DdtkLAcLden79zMVj7KLaG86IYxmG482dJhKh/BhUzIBQTJvrULcTFCdm9qeW9dpjGtH6SoygXStR/OmUAEvwETmUMQ8NSPB7O1JLWLQTtyHzetH8ZG0NGDuRT2SLeRaUx1DLSRsdwDW+k39fEz4zFbO4BV00SmEki2sSLDyvOro4T0tsYWBtN6slXcXwx2STYWZQOmE7lwsrGYTRdVAgqzlYRcU1tM5QX0+kCylhuU2bbCLIZ+QcZHRlO6JFXn5b2sRf4q1R5yrsr6oS9SvQaedKz56MGsigyasLqWYMTjqW/XskgUAliPbjoJ2Uk0uumGz3D/elT/3R8imPPCUPZeQrfbv2peY9Re75WsG2EPNcIc9r2Sfadk26eCBWEaozez9zCjtTwnhM9XifS7Sszud97v2EmlBjA9AMU+aEfrHfDrAbDaAb3Qg9X+ea+DZPg8A9r/6R/9z133OZnr5ySvr7/GOQGp/5g/4f8XwPOL/fhCS2Hvf30v/Ed/5df47vIhHkVpOn559ik/3N6nNB3Pqz1K0/G3n76DMZ6jyYZ1k5MnltQ4jss1R/mGv/HoS8zHFfcnC35yecJvvPkx/9u3PoSp5S/84t/mop3y9y7f5HY1omsTvvzgnEnasGwLbqoRV1dTDg/XvDG7ZZI2fPvVQ5QKJNrzK/ce82h9wLrNKZOOdZfxlb0LfnJ3wqIqGOctB+WWTDuuqjGLquBkuubeaElpOnJt+d7NfR5O7vjo8ozEOGZFw9PLfX7l7Ud8dHmG9RqlAtuP58y/fMM7eze83Mx4+Wqfw6MVq23ObFxTtSm/9uAz/sYnH2ASj3OKouhItMfowF5Z8fx2jjGeaptTjhrKrOPmbkKSxqTUVpPeGsqv3gHwxt4dP3p2RlG2ZImj6RLSxPHewRXf/ulbvPXmFcs6Z1sLON8uC8ppzfZ8jJpY3jy74XI1JgTF/mRL5wzL3z5i/5cuOL+cExqDGVnKUYO1hiRxFKllsSrptinHZwsuL2b8+pc/4VufvTP4u90q5fiNW07Gaz769AF7h2t8UKzXBWGRDfJbpp1MSFqDLi3jSc3qxRQ0jI43hO/M8T+3otlkwgpbxS98/TO+/dO3MKXcDNPM0mxTQm3QI4u6yMErxl+6wzpNYjybT+aM3ltQVxn+RQn3avxVjvJKanEOauxNwfj+iu3jGX5qhaW+zdl7447bF3PSeYO9KUQKfNpJeiriXXR7lvfffcVdVXJ1PkPFlNBQepLrROTJWthPdCDfq2luC/TGEI5a8W4eNfhNSnm0xVpZpGhvC/ILQ3O/QzWGkHhU4dAXOfrhFntToDqFn1hO799x/YNjeFhJqmxkwk1p4aU8T7+zob0thiCrdKFpH7SDJFu1slhRfLBg83xKGFuUDpKaGxSzsxXWaapXE0IqE7n0JsHejzrPVYJuBMy6whMmTiTafaptZPaThezP5wE/lUmWSgKh0ahOYw4a3GURr6EjuU3IbxTbBw5TyzZcHghTi75LhDVEvKdmo3ETJ9daCeM5emKoT8Qj6QdGnKHzUxJCA27iIRF2X1kNTpje8FaFPy+kr7MMoKOf2Ev/q14klK80m3csyirMViqXypeG6oEjZJ7kVmSDbuzRtSJbaNqZhDDV9+Lr1hp7YEmvJIypORZ/bHZpsBNZOKofdqhWDwC/uDADYwOwve9J1mpg5AR0qzgZDkPKbHNsyS8T+V2taPekjzRdx37dXBiA8qVme9+TruR19l6LucgGBiu/Fvay2QtMnilsGYOzvISF+RTat2uKnxb4PGBHIb4PcSFkLa93RawR2ouvOZDqpnZPjll38pzeP56uhPksrqS/M4vhUNKLK4sYpt6xNKaRcCndyn63bziKl0ZktzORWG9+rmb0w2JgtAfQWQvD2c52IWj5nfRV+rgw1I1FPl4fBfJbNUilu4lUNblC2NFegm8q+Xd+J8ffnDpGTw3FpYSLEWD2xHH3vhkWAdJ1oDmQ/3MFUhM0letlWgbJd69c0G08h7hI0OwxMOtJvetWTdbCeFUnivFz2f8guw+RFW926bt9z6huoZupnc80wOaBeHZTaRSJTHEY/LrFjRyzCnLuSeXZnsZOzAi8fcKQ+jqoE4yEeCVbWaBp9mTxa3ThqPf1wFTmCwGfLhNvbLaMfaK5HEdQinzpWL4lHZnpRtQUysk+s+VO0pxWgWYmXl6ILHAP+nMGOXa+CIP0tt4TkJ5Wu/7NXhmQNGGQ6vbdrCpAM5eAsa7sA5CEudMWtsea8kaCoTb3DNkyDFJ45QLNXFPeOGyhaWaK0ZWcf7pyLN5L2ftYQsNEBiwsXTPrtymA1LSe+iAh3fh4b1UkW8/6XkJ5I4w2Cuo9w+xxQ7OfDp2fpvY0+0m81wSypcUbken25wHCEKcbh41AvrhoBskuevf8YISlbOcJ2VI8l6ZxQ+9m0ApbGPJbqe9RNtDOU1kAW3WoENB1DB3KJG3WrBr8KBvAr08N2u4Acl8P44sUN0qHvky5ccROzX6u7uPvNjXdgz05rtSQLCrcJJdFl2UtElulCHlknusO1bSEyUjqVWLPpJ/HfvQecPaLKKtqqFDhde9lElnltoMk2YX19A/nBLS+Vi+i+soRYz7nsXzdV/n/CIvEtNkvvBQ2PQm/dvRv/r5v96+++i//wM8NvuDAcv8rJ+Hf/cu/wffuHvDze88B+GR9DECiHWfFii5oFl3JJ7dHfHj8nMYb9tKKV/WMH16c8ZXjc26bEY1N+Bfv/5Dfunyfx9f7fOPeSw7zDT+4ucdxueEw37BxGcu2wAfF9XbMe/tXPF7uk2rPusn44PCS2qa03nBTjTgabXh6t8eD+YKPXx3z6+98ym+/eijyV68IQXFvb0mZdNzWJfO85tVqyv3Zkq/PX/LXn36ZzhnmZc35zQylAm8c3/Lo5SG//v4nfOvx28wnNWXacbGYiKRXQZo65qOKIrHUNuH8ek45aqiqjKLo+OX7j/no5oxtmzItGowK+KB4cbGHNoGDvTVVm1JXmRBpraGc1vKz1Zye3vHq8SH3376ic4bOGpQKGB24uR0zmdZstjm+NfzGV37Cb/7oy+jEozScHi548XKf45Mlb85u+QefvYlSgbfvXfPpoxOwGjWyUueyv+Hq6R5qZAVgTjrcJiWdNtjrkux4S/dsTPHWCq1lHLdtgnMa32lM6rFL8caevnnDxdWMJLMURYdRgdWmEAmuguCk8kNZLeCpkYnC6MEarT3bTSGy6tZg24T5fMvdxRSVO9lGUOwdrllvCpzVaBNwG2FOZ/tbVncjAS2tJjvdCkgHqDXFccXhdMPlYkK7ySimDc2LMey1JJkj+WiM/caabpOJ3FoHXGtgmcC842tvveSjz+5LWNPLQlJ8Dxv0pyXtWYfOHeEm5/SDS25XI9oqJWwSdB09qCagtgZ91BAucnwf3lQLYPAjD5kX3+rGwHEjtSHLhDByJCOLrRPMTYI/adGXmciSjazOq07BzFL+JJfk5X071IQUpxvql2P0YQPPS9xRSzFpaV6N5PhKL/7YRKTT2bWhPZM0ZjfymHlL/v0RzZHHjT1mZUAH3NySxHAsAugeDO5ZkttEKk5WwoLbfQGW+XlCNxPPpc/A36/xnSG5SqNMK5Dfato9j3m4pTsvY6hWQDUi/TYLqUjxY0d6k5CsxXvq9zuoDaoT6XdfuZJdGUmM9or83FC/2aJqg9nqweMWlAQAhUzGZci9PCey58JiikdVgpoCzZkVgLwWiVm6itUkaZ+q7MVHO/HorVzn4pWAx15y2t4TL7QrYsfuWlGfeopzjWn71GNFN/Oxa1eSm/HgRiHKXtUgD0y2Akh8xsCWtAeyPZFSi182i1Uh+ZXe9fXmAp5M7AdOaoUt5LkSxuQpXwkALS40xY34RO0ohlClIsuWiagaQIpPA+MXAhhM3UtkoTqT7TWHIqvsq1ZcLmCoPwefRTAVg4Rctvt+6hOtXc5Q8eIigMiWEi7Vg2UQr6xpFOkyhg9Fv2ifANzsK7p5DFBKBWCVF1FqrQW0mRj6NH7hWbynhz7abCne1aDFD+xTAWFB7cKZepl0L6fu/bhJJdckaAGN7VQSu/O7MAAtCXwRr64dyTVKNoHNA+mGFXmhGkKR7ETApCvkGpYXAm6TrTCJPlHiXS2JSaQC5nrVoctFglxcCwgMRpGuxDdt6jBUq/T1Kb2/tQ9raqcSQpVswhC0ozupHAEJENrc16SRVU5iEFI7k4oU0wS6kR6ul+nked0ojvFY85JU8RiqQDsWMFtei7+07UObppp0G/2uKwGyfeiPCj07H7ClJqk8SR2o92TQJLUkUaebCF6jLFlHuWyfVNzLl3v/sEsFZGq789I2Mz2E7fTjV/lAspWQHdMKCLelFqa5DZjKY8d6WMQIEVwLEwi+ZxOR8J9+Oz5VpGuHHUldyTA2nCgT+goTkW3L+bczQ7oWhld+xyD5TSoJ62n3Yspv5USOnYrnFcBnO2a4lyX310nqbhTpxoKT8B4QMCq1LnoIH/Ix5Ed3IvH1uZE0Wy8sbzDC9oVEE1JNsmgIqcHlhmTVCENrPW6So7fdcL2DigAy0YP8VTknYT5VK4Cus+KVBAgi2R2AJyI9HeSzSn2uixOtd+xiXzni/Y6B/Nn6kf5nGxnFstixr0DopOcSraXnEuh9liQxNChNCJGFVUZ/3n/ZP35Watvv/x/18P6LXzeSnoRfO/w3ft+3+1fP/6s/8HODLziwHH9wL5z8h/8pe+9J32IICus1RgXWEQisb0aojeFLX3/O7/zoAemx6NPS1PFgvuCnn95jfLhl82oMpaOcNhRZh/Wa+sd7dCcdo3lF1wmg0CZgPi2w79T4ThI4pNMRgtNSVfKqkIlkUPKzV4ynNdvHM7IHG0KAZpWjEk/wivRlhn3QCCujkUln7iXg5q0KbzUnR0vOn+5DGjA3wlCpTUJ2byNsmlPglfQOnpeE0lM8TbGjgH9YE7xiNGlo6hS7Sgc/JyApua2CeUeSOexFKbLKkxrvFNoEbGMopw3VsqD8LKP5oKL4UUlz6IWZeVDh7jJIgzAuVmNGluxHYoKr36/RSSBc5Jh7W8KjsSTJaggmyMRfB0bvLdh8Oo8eU0gnrXQzaqDVkDuyFxnu7Rpe5bixp3iZ0ByIyb4PANLjDvWqwOeeUHjSm4Ru35LeJigbw1zOFdUvbglOYZ4X6FYmunq/xV/lhMKRXSWYraI+iRKttaI9i+DktCFsZDWwuEiG8BzlYPRCcfd1i9kIOHMjTyg9ZmkoLqWLzBtJUJ080bRz6KYenwaKC0N97CWMKJWuxexFKpUzE6mRqR46zEqDhuxW0e7FAI7Y7zj6NJWAIC0sGsD4SUI3DtixMGZJBc2RXOe97yXUJ0CA5sBLQNNcVoSLczmHbh5Q3U5y2xzJ88bPYHtP0bzTMP4opz4OFBcSstLsxVAcB92+pXiZYouAnTuShWF0rli97xg9MVT3JJFTOVi/bxk9TujmkjArwUsyuctvFdWZXJ/iSrG9J3LE/FrSR92oDxkSSawwAZ7JE832fsBlgclTjakDy3dle8FAfSJJp7pVpBtJFbWFYvWegNPJE836TallSSITtT2T+2d5rmj35T0dnSvWbwbKVzL5ze9kUlcfCIvSsye6he19YZN8KmCiZ0g2D/tJlshyXSbVLLZU1IeKdi6gxCe7MTf7BG6/RpQpQ+8v7f2uppbJ4va+1PtoJyFGzYFsp4o9kKYV8Lb/I7j5OZg8UjQHO0mt3GxFXmhLNbBJ41ee1Zsal+3kwqMLz/ZE0js393c+y6Ri6Eb0qbBn7R5kdwI2yqvA6i0VWTIBgfldBCpX8pz8Vp4zeingoDpV7H3sufmaJl3JORSXAiDya5Ei98E+5ZXUmZgmsH5DMftMUk0nL6VmJNkKkCuv5P1wqaSIaicgNFv0E1u5HD6TSbS2MhbXD+XeOnneSzAV1aGcT74K3L2vmT6W4KDiMrC9pyiuRI67uS81R+1UDUmo2sUxvnBsjwzFnUgxb7+UMDoXUGI68YXaPHZCLoRV023sZbwVKejmTJOtRG7cM0zNTMceVQaQ0o2kr7DZ03QjAXqja4epA7dfSiVUhxicFN/jbC2ga/lWQrYMFLeOzT2DywX49fLqfOlpx5riTvx46/sJ0+cWnyhWDw3ZKtDsCdOWL4Rl68bSlejTKLW+dWyPk0HKbQs5R9MJIxY0ZJF9BJEHN3MtLJ1HwnMm4mHsxlIbND536DawPUkYn9voqRXgYUtNunLUhyYCQc3owmJLTTvRA7hUXupQ+o5KFEPAT7aM466VOpbt/QJTx9qZi45mX1jM3leZVI7qOEM5kRU3+4n4SRupiHGFJl12NIeZsJ/LDjtO8KkmXVvqo1Sk6y+3NEelVJ8cpmQLCf55vRqlfFlDlKP6RGOqjvp0RLYQibDPDK7Q2FJTXrVImrYkswajqU4yiusOs+3oZhmmE3lou5dRXFRU90t0lPLm1zXtfk5+XbN9OKJ8UcnPVxUhNbR7OcWrDXaeY0vpgsxvGuwkJblrqO+NKF9s6PYKkk0nAC8zmG0nKa+THJ8ZkfhG5qzZzyhfbvB5IvUly3bwYyaLCpTCTXORoTrxBbrcCLjftBIYZP0Q3qPrDjfO0bVFt3YAjD5Ldj93DjcrMFcrSZsdFajOSnhP3Uni67qW3zctYZSjqpYQwaPq5axpIuE9RYZabQWwpRKqo7rIBLbd5wN6jCYUOWpTCQgtc/GGbitUlu28j6/3Ur7+szGEpkVlqQTvJHKtQgR1Kk0JTStAMs+h2zGXwcVey7bdpcFGuevnMMfrfZexSuT3lLr+01A38s+A5R/ex/TLZ+Gf+0t/nkxbzrdTvnn0lP/1t77J8Zev+JXTR/zo7ox5XnHbjPj0xRFffeMVB/mGp+t9LlcTnNMY49Hac2+64sVyxoP5glHSclOP+fff+i3+s2/9K2Rlx3TUoFTgg/1L3h5d878//wpFYrndlmgVWD+aEw5aCQcqW6w1pImjzDqKxPL4+SF/5us/5NuXD7FO0/2tQ7o/uuLDB8+5rsd8/PiUB/dveHk5Z29vw4fHL7hsJnz/J29w/OCOm7sJf+ztR/zo6pR/572/x1/+9I+Sp5ZXr/b4U1/7Mb/50w+4d7zgajmmrVJOTxbcLMc8PLzj2fUeXZ2wt79h+dN9/uQ//12+e/WARHuulmNGRUsIamDb1E3G/N1b7p7sMX6wQqtAYhz13znC/PIt60VJ8jxHv7/GWoP5rIT3N0xGNeOs42YzYnM9IrtI+E/+9f+Jv/jRn6BtExTgrHhI3z66oUw6PIof/OAtvvGNx/zgu2+Jr/LBFq0DXZuglLBz771xwaffe0D51oq9UcWLT44pz9YY4wlB0dQp41FDYjyNNbTf26N7V26Axwcr7v7+MenPLaiqjA/feIb1hvPthFfPDnj3nXMS5Vl3GRffORUQGFkhksD4/or1Xcm7b1zy9GoPHo2xZy3ltMYYYTOnk4pR3nK1mPDw8I55VvHtn7zNV99/zk+en8JVTnJvy5tHt3x2fsh41DAtGu62JXWViZz2QqJEQ+EoHme89yc+4ycvTpn8rRH1b6ywnWFvJpqoxbpgXLZs64w/8uA5v/3oTVmkeJxjp4GD9284naz45DffwWdStXH88I6bHx3iJh5VWuZ7W+6uJuAVf/znfszf/M5XSecNqIB3BvNZgX27RumAv8rRjUa/sSH5aMIf/5f/AX/9N38Be9KiU4/SAfNJSfr1JdtlIezvyHJ0uOLq40NJQ66lmgcv8l3OGvlOuc4lRCPznL19zd16RH1ZcvTmHVfP9oQ9BsLUMppXNJ/ORFZ4Gn1CTkX5aWQ0jyzlYUWSONZPZwMwTY5rumUGQUnH7U0piyAKVOLJHhXYiceVngfvXvFHj5/wP3/rm+QX0i/ZfmOLelLS7Tmya/HgBhNoThx/5he/z//5f32IelCRFx3uO3Pe+Y1H/Ph7b/LhL3zCt3/4DnqrCXkYApaKsw3towm+DLKI8SqF9zcY42kfTdCteGrLhyu21yMO7i2omgz7U5Fq+1SqblwmTKr6cIH7wZxu5lGnNeazkvt/7AWPnh4z+jgjvwncflMWRWSiGEjPtnSXJWHkGP8koz7xpG9usJ9OsIcdsx9krN5z5NdG6mAOO/CKP/9Lf4+/8nd/SWqbTiqSH0wof/mK9XcPpVf3xHH/S5dcr8Z0jyb4TOpVfMaQNpxs1SAt7cNwdCthWuq0If+oHIBze2LBQ/EipZsIyGruWcrHKeoXFmyvRhQvUua/fMHNd44HuWh1z5EuNT4Vv2+7FyLzGBg/jbUiHpZf7Sgfp/g/ssJ/PKG4jmD7nh8CrIIOhNOGMh6XHcvkXwJKRKKc3Sqab1SU35bFuXYuizh2JExjc+jxpWf244TlNzomv5NiSxi9DKzekWszfqYHtjCkDEC8OZAKnPGjhHYuC0QHP1Ds/1vPuPhf3hh6TIOG8KsLmt+Zka6EQVM9Uxw9uCgB3pOnsPggELJAeifJ1q7cBXmlGyUMp4kS12+uUN+dkt8ySI61Fb92dSqLhPatmvnfKUQOGtnabiqAsriUjtee8bVjkVvvf0/TTWK4UJTPClMktRvNn12g/6+9CAhlzrJ4XxZRRi9lm33Sb1IJW67jtWinapCCmlaY7GwRqI+FXVdB5Mw+UTSH8h6MXkSWrxZgWp0qJk+kZmbyROTS6UrYvWwVYpiS7LM6EhDes3g9A9svFAWzC8aaP7JU+yKH3t4T6a0t5PoIWwvzTzzNntSNTF56eS8yYb+Dgm4q52PavtM2UNx4XKaGlNvqUBQ4ysLo2lHPDcVCOjrrfc343LJ4Ox182srD5Z9uePg/iny+mZmB4dRtYHssixsA6/uG8tqTLRx3X0pZftnx4P+QfW7vKWafycLK6MKSn2/ZvDOhmco+N6cJ06ct1XFKUnuamaG4cxDg4hcTjr7vyBYCkG2umD2pqY4zylcNz//4iLO/29JNDPmtpNFuTyVBV3fSGWrLBFco0qUjWbcCbruAbryk2K46bJnQTbTIXp2E+7TTlHRtMbXFlQm6cbu/RyKfbecpo8dLuqMRtjBki1a6PG8r6rMx+U1DfVxQnG+xsxxT2UE6q+sO1Qg76YsEXXU732Ivj0WY+JAl6FUtNSMXS/x8JEm2iw0hS3F7I8zddtel2VlCmYFS6JsVfn8i49w5VNNJ5UjbQWJQ28j8+SDMpvsZ0JYmQyrt8OjZzaaFNCXUDaGuUUUhoM8YAZXb6vNSV+dQWfa7S2dBmM9/XI/la48vfN1IehJ+7eDP/b5v969e/Nd/4OcGX3BgOfrSvfC1/+Lfo0gsL67n/PzD5/zk8gTnNHlmKbMO5zWbOiNJHKurMbOjDU2b4N0ut8g2CffPbjm/mXF6sOTh9I7vvnhAvcxJRx1J6piUDZcXM7JRR5o6mjpFqUBRttR1SpJ4qruCdNxxMN/gvGa1zSnzjruXM0lvrRLSvRr/fET+9or3jq75/u88pJg3NC9HhKklH7cSFKQCVZXhW0M+bmURqklQJuCWmXQeTizBKvJZg9aB9rMp/lgm+klqaRaF6PEzj75LyR5u8D+e0B468b85hSqdpNCuU1RpSV7kwnLsWVABlXpYpWQnW5qbEl1pijdXbG9G8vqRJfusoD0QH1ff4alahTqtCU7hG0Nyk0olifHko47m5YjkuCbNLNXzCeqgJTjpxnS1fJkVs4ZJ2XD1fI6ymmSpGX/tlsWTOWHswCrefueCp5f7sp9VCqVjNKvl+ACzNOJ3AznnVsKFAGF5TRz/EaDoyqBOavxNjllr3FlL+jTDTgQMuP0OvUwGf5sv47UMkF6kklZ4rxFG56XIXUMq3sZwr8ZvEtmvlveFAGqdSKJqIR62HqyowpG8yqRP8U66D3vvpdsmqE2COmwwT0X+6tOAbjQ+9yivmL1zx/KTPQEfpXSW+kw8dd1EfHqmks9B+u6K+sU4pggr0ltDt++EyQ4qdi4KQE1vEukU3RrShaabxW0XIlu1c0dyF6VaMdnWTh3Fq4Ru7vG5BEv5NODmjuw8ASUyQu0kUKebBvp0yL7H0icBP7cUTzO6iUd3vbQyDKDE7lmyiwQ7iWEwlYQSyWRZukGbA49pZKLXxXoYOxGQlqwV7f6u41OkgrH6ZhZltQPQiGFCkRk0jZyDnTuSlZGkWwfZUuOyGDbTRQnsAtZveDmeQ49PoTzXkoQMg8Q0v9FU9x3prSbdKNpZGPphe+9aiKy/pIQSU1/lvOoT8Sb2KcumEfY22cj7PnqhpJs0Ed+gqWF7z1Nc6SE8pwciIJNdOwp0c6nF6X1tdiTjqq9RGXyFnUxm272dhBN20kM7EtZy86YnXWpGL6VmJF1DdSxewfowDEDP5wyBT71sVfkIXibxPcjleLJlH1YhoDXZSsVJz9z2nZlArEEQAKTjMfssSiermNQb2KWHhhjcZHZ/v14B099Tmn1hrAG6aGXSTkBdfq3i+yHn1Hv8lI8S0WR37Yf3ogqD/FKSjgVQiTQ2etstGQ0AACAASURBVK7b2FNZi/w0qSW5Od0IKGv21HDMffiTnch5mlZAWu+PbGcxLClKOnupZrIVcGXa6AVcB7qZwlSvyVCN/D4YqI4V4xdhAEPpRlh7E+e19bGAKtj5QNM1O3ll+tp1iL7CpNr5FnUH9f7Oi9h3myZVZH5jmnQv8/1conEj27Zl76+UcVrcerqRSCH7ceLynSRaOShvHdW+9Db3wUI9oEsrj43Jrzqm9gajaObyvmcrCfhppxpvFGnl6Uo9+AmLa0c3NZjGU++ZoepFt2EYYz5VknAbGfVdKFOIfkFh29ON1KWI6sBT7xuylQQjtXNDUsm5JnUYzteWkhBbXAmT2gN6kAArgHQp/sRubGK6riKppcolJGqoNUm2Hm9UvJ+rXWdllKeaxkvoUeOlZqV22MLs6lG6nVxSUpkDrjTCLnoGZjloMLUbpKxDCm0qgUrKxdRdo0kX9ZAwG1KN2Vp8ogmZHsCfG2U7kJdpVOslvbex8T4QYlJz+Bwg9HkiwUOALxIJBYKh3sQXKbrupHOyc4OcNcS0XEmnFV9j6IN1NPJc5wlZit5UQzJsSMwAAiWxVmpLlHU79nNTCdNZZOKx7CWwfWBQ74GEHXsZK1NIDGEjchWVZzuprXME74c+S6kl0Z9nJ/3PgNXXH7+bDPZ3ebyOU5RS/3RIYf8ZsPzD+dj/ykn4j/+HXwXgb158iWnW8Hw5Qyu4N12y6TKeXe/xp9/9Cd+7uc8obTlfTZmXNcs652SyprZRk689ZdLx4+dnaONIEs/JbM3L2xlH87WkxLYpzmumRUPVJSxWIw7mG6o2paoy9mdbFuuCLHPcmy95dH5ICIrpRD6QmypD64AxnrP5ipd3M9om5a3Ta17czqlXOe++ecGz6z2UEsZuNq2wXrNZlKRlF3tp5cNYr3PSsqPIO5ZXYx48FMZTmYCrE95984LH54eDNzB4TYDI8kFRdDinaZqU6bim6RLqTUYxbjHG09SpJOWqgLda2KfrKb/y3md8/+IeqXHcXk94cO+WF58cU5xu6NoE2xjSsmMyami6hBCU/L5OOD5dcPXZAclRhfcabyV4KCss9SoHqzh744ZXzw4AxE/ZJpjEk+Ud28sxpB5lAmnZ0VUpSW7p6gTds08vCwmWaQ2hSlAjy2RWsboayxeyU6jcsX+wlo7SRY4eS7+oTj1ukZEfVrRVSlpY2mWOLiy+M9JZqiRMRo8tfitAMTuosa0RcKthfLJhczkSn+JBw/HhivPHB2ACxb7cFLvO4BojQHhRQqtlAcIraDV7ZysWj/bABIIOpPsN3W2O2WtFbl16dC1f/m4k544OEqwy6wiVwUw7tPb4ZyP8SUvoNDQiodVbTchkgYH9dnh+uCjwY4dqxaNn5wJyVeLxqxRVOvSVgGhfesYnG6onU5kwnm3xj8eEB7Jo0D6X2bTZCmvoCwHoyoIvAtnJlu7FmHDQwl0qXZYbIxPuB62A570Wv0rREQT7uRXvXylAU60S8UmdNahX+TABNRGsh4MOFikh33kww0mDrxL02kgqbqdIV9K56vMwgEYfg42yhYTWJDcJdk++RM20g+dlrMcIuH2LuUlxM4tZJkMFR0ikw1S10Y9ZWvRFjptZ8lcpzRstOnNwmYtseuJJbyQwJaQSNKO8SKXNVkvQYwfd3GMqLb7KiVQoJBUiz68R+bZDAKeF9n4LMYW6OfDkt8KMtUcOvdWDP7NPNa7uWSaPEqoTP4Tg9I/JE8X2gYCMdCULAT2DmmwjE+QEUGongKy4UtJ7u1WxK1UAYH4nHZrZUg19o+s3JAlYWZFmlxeabioVLpJkLNfANNAcyaQ9Xckixvi5YnsmwLqv1BG/nfR0SgCMgNVuItesnTGkzIJUAuV3inYqgNlnMWgmAkwd60H6h9RfyPZNJQBY2VjrEv2MvTw5aIY6Gd2J1zJbqGFxQls5HtPyuTobSTeN20xir2UNxY1ncxY/G/E9yu8EQCZbIru922+2FC9kfSDb6H2h2SoCskwJwG4FkGcxQbc5iONwDKNzkUFLIFPcZqyA6Rc3moModU5EttwDVp8I+5ls5LXpRuYg/faK6xD7dXcMbFJJt2S6DXK8SkBMtgoD+BQwpHYsYi59nr2stF8Q6DtFTQv5nScokZr36cI+UZTXnupID2NHd7JP04YdKLeyj57hSxoB2X2FSb/QFBI5xx749N4+W8Q+3Cgr70pFceeHyhfTBeq5JtsI2OtG4jns025tqQbva9/X2Y6lvsUnSqTBN+Ln7MYia/apIt1GoFoFkkZYwqQOuJQhYbVnhnvgmlZhqEoxlaebGLqxprhxwzXuE3CBz4FYH2tH5IOy83DKeBU/ZTeW1FlbSEKsqWKYTQwBkooUSbztpka8lx4IATs2ZAsbq73UAIjTdUySDXE/G5GnDvU8pl+YjOmx1pNsI6vow5Dy2gNPXcWQn0SjW4cvE3S1k8CGWJkSjEK3TsZbH8qTJWjr0dsWN80FgLogUleQOpFYaRL6qpLOfQ5cqihvxXmRucb99qAtZMkQ8tNvqwehaMSnGYLIahtJlh08mD1QhJhaq3bAs25EYtunw6YiTQ7bGlXk8u8m3jiDF6Da14b0Hsv+PEPYgddhELz2f6+xtv/Ixz8VHsvj8Gt7/x8Ay6v/5g/83OALDixHX7oX/tR/9+d4tZmyrnOOp2uqLuX6bkKSOgm72b9j0RScv9pjsr+lrjLsNhEmDmSi3WreeO+Sl9dzbJXw7lsXfPrxGd/8+qd858kbuGWKnnb4xqBSz3RWsVqWJKkDFei2mfglo8+xmDXCdo5bXGfwVSKSyr2KzY3Uk+iNER+hh3y/HibX5lUuE9cksH+04u7pHkHF9MeRg1aT7dd4pzGJwz6e4KZOujNrM3QlEoCJJRu1NNclqtXREySsG1ZqNZRVuJGXfr5SJr+6VYTTBvO8wI68pKey8xemkxZ7WRCmMsHWd6n0B+55ggoStpIFko3G3tuFpKDAThzJ2uBKAUJh5AToRFBUPEtpjp2kZjoJ2ghaJu7S/xZgakmfZ7T7LoaneHRl6Hu5UFBcaqpTTygk0RMFdibJqMpqCVspPPml9HvV9zrxK76Sn7uZePtU2CV9KqewM8focUJzKAyLHckKvR3HlE4rxzt/+47qOwc0D1uSq5SkkvRGKbNn6JBTPkr/grw3+a2iOpFr42dW0j7PNe1cZHDpcnee/aS+PfAkG5lw98xet+8Jqad8looXJg809yzpZWRHM5HSZXcyAajebzBXWQy3UNiJj+wpA7OXRiYx2Qiblq60+CVPW/InOcUVbB6GnRwvDcNELtmqgSVs5wIagpJk0NnvJLRTuR7dRLpbszvZVy831a10eYaUWNnSy8CEwfNpwM49o6dGJnZ9R6fesTLVmfgsfQLtvtz3dMsQMJOuZRJlx4H2SD7b5ZN0AAXNYWStS3mtaZR4Tj00h8I+djNhWstLxfqNQH4jcsxkrckWck6uDORXmuYgUF4o7DiyXkqSYQlSDQMCBOojAUHVqZyrqSJbU/THL+dZH0oqaDfdMWc9uyMTRQFG6VrASQ9gsoUwdeMXwiK5XCb39YEaAEq2ir2iuTDE3VTY374qRwXoRhJyki1j0Mx4B6h8IvvpQz16z119uOuhTdeRxWrlOMsrAcuuEFDrMhhH2WN/XroRkOJTAS+b+4ryXH43euVZP9Tkd0HkiVsJmHGZeEbL8/i6haSR6gbafchud8fTHOxYYPHxwfRJBB1qd7/xmcg8mwMBKeMXUo2xfrj7DPV+TFfC5Jln/UAzOhd2Mb/12JESZu+5bCddhgEoqsiEtXORzXZTuSeXV57VG5ryIt6Hyh04sKVIJF2hYk+k9Bsm9Q5oppvoR0wYQm9e78YNRknC6FaARXWsSDaBfBnZxVTCevqkVRf9neWlSDG7iaSkuky2Y1phEruRfIa7idwT8oUfJKBF9DZnay/bL9TgEXWpolg4upEeQF3SxICXIEDEG6k8cblMmG2h0U62WdyJt9MnuxRjCV0RVrEbyWJLvvI0U3ldDyp9Igzx5KVIOMtrh8vl+aYLkS0Tv6by8ne29rFGI557Fr9TRgpvFPlK2D3lA+1UklN92veYBrpSvJf1oagn8kXP9KmhK9Nlcq1knwKK0nVfiyIn2c4TdCsLCunaDaE3KgS6SYKpHN0sGUJxdBdo9iQsR3deOi4zSVXtZaBBC9Bsp0bqb5YtzWEuHszLliYG6WQLG1lLN3Ra2pEhW3S0eyn5ZU17UJAuW7pZJimsW0u7n6FbT7qWhNFunpGsO9woIb/YUp+OBCz2FSsh4HMTVSUip0XL94xuHWbT0h6PSTbiGZVeTJG+uiKJQWgOvHRG6s4N4M+NM8y2kz7b3KC3LSGVv32RCePYgzwXCHkEiVHG6ke5qJT8a4Ax+hl9kaHrVn5u2p3PMfZMhiJHb2tCngroS8yuvsQ6Qp6JPzMxUkESgjCWSgkz2Yf99MC0D/XRaldzArLfn5Wivgb0+uAdFcOAQg9A+xRY2LGVfQos/MPAcUi3jXPw+LNS6ncP93n9Ef2YX/xU2H8GLP/QPor374ev/MW/gPOaSdGwqnO22xxXJ4zmFXWVsTffsK1z2jrBt8JWEBQ6dfg6QRfCEIUqwUw7XGXQq4Sw3wkTtsgFWI0tKvr58EoAqQ5DAA+VQY1F7krmoZMPpOo0oXBD2inLVFISNTsZpge8kj7I3Am7EgNt/MwKCFxr3CyySBuNeWeN+3SyA4UTJ+meffKkgpB6keRNRMLXH4fc3OIqnRNGJeR++F0/YVKxziKkAeLzdF8zEIGDWesdWxaIEkdAx7S+wqE6LceUBELmMWsz9M71skJXCPviSkFdQ19kvD4hiSE2L1OZKAeRSJqtVEv4bCcRszOHbmSf/QQ8ROAKIkf0SawNqBR27kkWOsrphHXxmbAjfdG9ywEV2TZeW03vUxT7yYeJ0rHIyvhMALErZDtmuwvgsZPXPnuRheknsd1U5Ky6FeBvtpr8Rg3sQj9ZdWV/fLIf5aLfKyZmdpPX2LfosdKdTObsKAysiKnUwNSZOjI9QY6nm8o1CYlcb4hgxkQ5YivjQYCqXKv+uvWSxH5xQHc7UOhT4gKEot0PsV80sjNhJ1Pr99GzIT2b0V+rHjz2HXA+ZZA+9p2fwjB8nn3oKxN64NkzGsRjK64E+PQpoMOCgNuxUP216NmrZCv77msRRucChPqFhIGxcrtj6LeXxpTMYftxf+laJoTtVPbZh8YM/47jcCiRV7vjGtix1z7XupPz6SZy/dOVorwUMGiL2DXabyeCKZEox+31t614TV2xG8Y6jieQ/xvOVUepc7cbv8P74HcMUC811DHNUoUds6BiL9/Qx/bax6dPG+0lh0Cc+MpkHoRFJAJGFyWkzb6iuJFxH/TueFwpwLed71JGe8+dSyX4qH+PtZXJfT8pH65FtxvHPUNpWnkdyHHYQu5zPSupYw9lL7ntgeWwQBBlnL1H8fVx1DN3vdTTlhJe00szm5kwZGn00fkkevVyAajdSACXacPuGiPXtAdx/bjsuy4hJpH29yS/+4yZTl6XVsLG9emoPdMUtADM/nc2l6TSvhezH7P9NZbPeZSlvj5BVQqXQrYRKanpekmtMIPp+vPJpS52KQoTFD8Tdje2pF4iRPmpHz5LydYPEtD+XtuPrT60p/9333UpgW67a9knLg+fkW43iINWQydnnxQMoKz0U74OnpULw7gZKoucfGakx1IYIxf7J/vfh0RjGjeAyd47p5wkn+rYd9n3VvafMZHbq8G3qOxuH/35aBuloH1oUZwneLNbhAmJEnBYJnI8r9WBKB/Ps5eeRkauT1t1uVR02HFKumzxyecXd/pE1v49DFoNEtrXWcoeAPbMoE9N7MSUvsgdU+lxo0x+D+KJ7M/NBfl3HIfDNnv2MspXVS8FjeO0l7p+Liinr/Iweidl7fsovZefjZa/Y7iS7MsI89g/XvdIGj0AU/ownD4dtmceQbbTV4P8zKNPce2B5PCS10HdIIn1n/v/z1WIeA+vAemfffwTYZB4nOG1oKB/0scfCmCZHIdf3fvXft+3+9eu/9If+LkBJH/QB/D/5lEkln/73b/Px9sTfuvJe7x9eMOHbz3jsp3ysppR2ZSnl/v82Q8+4vu39/lw/xnfunybB5MF3395n7OzGzZtxtlkRaYth/mGv/Hpl9i7t+Dy5Zxf/NJTHi/3cV4zL2o+fnxKPmn42tkrWp/w0U8eMj2saNqED969pLIpnzw/5vBgzeFow11dclBuqWzK8+s5755c8/HLE0bjmofzBT/65D7JyPLw+Ja7rcw2XFCsgdOjJZlxLKqCUd5yvRiT6ECWyYf6w9PnnB/OeHY358vHF3z/2QNOD5a8+OkxX/3wMT98co9ffu8Rt/WIdZdxvRzTbjOChX/pl77Ht16+RZ5aRmnH06s9ulUu8skImH/hvSd8+wfvkB7Ucs8z4iHNjiu+evqKR3cHVE3G/J2KdZ2zeTUeWN8kcThrKPOO7V3Jn/zwB/zm73wgTFRmcXcTOG44PlpycTGHTUJxuhGf6E9nHPz8JUoFzs/3KKf1IKOllmoGksDx/Ts6a1g8m5Mdb8kST9skEiLzbISbW+klLDvsbUEya3E3OclRhVIwLVo6Z9hcjkBB+uW1dEzWBlaJdBseO2Gpr1LcxGHmLSZxdM/H+MKjSgebhPROM/q5WxZ3I0zq6VrDO2++4qfPTlEq8ODkjovFhGaVo6cN9qbk7W++5JNnx4ROc3C6pOlS6jqVasCg4CrHjx2+UIyeJDRfrzAf1GxeTZmdrVg9m0mPY+ahsLjzUlbOG0W35zh974rltqC9LVGZh7sUfdjiKkke1pmDqxy77zBrQ/q1JfXTqdRMnHWoJKBuU3SjsCcdJneopwX22DN5e0FnDfWLMclxzYdvPuW3//aXyW4V03/hnFfPDlCZwyWBcJnTvrsR2fN1TijdMCFFwenZHVcfHeNLjxshibw3gp7DyKE2RnyfaRBmOwCJMInbtyzkjuRCnq/e2mCbBH2V4cq4qFJIMA5BSXfmXYZpFO9+8yk/fXw2dFC6MgyS0nDQwipl/9cvuXu1j75KMZWiPXboSmPOKuxVAVNL2Bp0rcnfXNN9OmX7fgc6kJxnKGD1DnSHvYdFvI4hlbETrEbfJVLnArLwoUCXFnUu/tzsTrP5+QZuM9KzLfb5CD/ypLcGO/OYtRYwFxmo4lKxvS9VIjZKZ+3MYVYGHytLiueiCiheGrqZsI/NL1YkH41xZaB66Bk9MWy+3FJ+mslCRx6oD2R8NX1oUhJQlSa7k5RjlwfCyFE+iYnECgl3uksJaaB8YahOxWtpOmFL7UiYflNLgu/2LUvxIqG+7xg9ScSTduDxhUdXmvxaPKnJRgCZqcV36ktP+cKwfb9j8sMMOxEw103Ev9juxUWPMgwBLcqDHUkH3OZtS3prhoUa1C6NuPc+N8eOkAbGjyRdGS2Kgd6PGbTaLXKEXW+lHYX4Xiqq+zKGxs8U1YkiW0iysMiE5aOfbEUx0fdrqqiI0J1IfZUF0wrLni3U4C3VXVwEsAysb3Us17o6ZvCpFtcS8jIsZuSwfEdT3Ciyu8DyXYVpFbaURR/TiFey2VfDcZTn0rnY+1zzW5FNukzOubyUBF4BJorJs4AdK/lTCmNuYvVs0Ir8TvyoyinGL8UTahpRP1Qn0R9aB4LSqBBItrB6Rzy5w+IFEgDU7Gv2Pna4VLE9FnYs3cRFlwgimz0t4LeNi0jXwnpWJxplBTj7FOoDI4sIBZjW4FMYnUe5apAOzWQr8th+QWy3KCDARbeB4taDUnSlQsfuR4Kw18WVSF3tSAmDHCtPTC3BPKMLTxuvXXEtvsxubGinu3qSdBOGcwtT2a8tFeW1ozqKaZ7K4FORc6SVpNr2izlJvQP6EkIU0NYMvZnaClubbg3JxtMcyYpH0NCVitGVG+o/mn0zVJsk9c7vmWw9uvWs3ipJtx5tJWE2v7FDeI8tNOlWvJCu0FHtIiyuLWLAToD6pIiVNA47MiQbhyuEmc3vOrqxsK8hyo515zG1o9nPMLV4GUU1IayzdgFvMtK1le3VTiSvjcONMnymd32YuZbuS6VIFhV2VuBGyQBik1WLG6eo1uPKRLyckwzVenRtsbOUZN0OYNftj4RhdQHddNj9XDycsULEj3JJkFUKPy6GBXe92OIPZ0P67M9WhgwVJIl4JdWmFrazZyz7ypFRXBmMctthUSmm6VJ9PihH9bUjnZXt+UBYrSDNIHhU0PK7ECRRtm4gTQUkZ9kOSEaQqXoP6e/ltwwBjDwvdP/4p34hHr9X+u0X+PGFZiwnH5yFv/Df/wafro94vpxxeztBm8DD41uu1mNGectyU/DG4R2dMxyVa35ydcI4b/FBDf2LISiqJkXrQLXK0YsUddwwGtesrseY0pIksT/QafK8Y70opeD9pCIE8M7gWk1fP5LlHdXFiGRpsHuO4rCibZKhoiQbt7Q3BWSecl7TVCnqVQH3a9yqT7hgkHoCoAM6d/hNShrDUey9+O28Tpg8XLJ5NCfst7BMSY5qbGcIbfzgmoC+S2WSNu0IVgvb2mhhOytDmFqR7N6WmMLBy3xgSouzDdVtST6vaZYy8U2uUqkzuc0IuUdvzcCA9hNkXwRCEumXzJO9THFv17hFKkmjpzU8KbEHdseqRO+bqbR0EVqRAqeXCXYi8lw/kZ/dSLoHu1iwbhpFe2LRa0N+o6nP3MDAqsgk97UcqtYkGwlN0Y3C7ltUrWXfG40/6lA36dD55guPbkX+FEaO7JXsP2jwuSdZGpKtoj6zmK1suz1ypEcV/FT8hu29DnObCpDwMol1YzmnZKlF8rowgz+um8r/ZXdS4bF+x5FfmiGV0ucyGU6XMfQmMh3ZnaJ6w6K3Oha8y8Q6AGns5atOPUklDLHPRKqb3Wp8LlUNroD1l1th8VNJjuy7AF0usk6fBbIbswtxWahYEwJ2GiguBPzYsTBkulNDSI3LRBY7sBy1GlgdW0aJmo2prmthitqphPmATCi7mUyAXSHP6YM9ZPsM++qf00ty6xPpSlSR+XGFjL2kih61uXgC05VcB59K6f3rvYPZUibT4rOMISyv+cOUF2no+g255n1likgPhZmoDyXldOgnjIEwKkTWaSHbqo93bKV01u1AhMt3DG/PdPYl5sDAyCUVURr9eSCS34q8Mmhh0obePyvhMn3/n09g9CrWUYwZntd3BiofGTPLkNLZjSHdxv2VApz62olkK8oGn8ZJViEMYn0gE3Y7VuBhdOmp9zR2AvmNHKuO17xnUfvrPQSzFJCuBPDkC5Gv9qx1thRg6TLZtwQqSbBHM5NeQFvu2Dk7UsN72k3kGLsxBCWAcPbIs3xbk2xl7PT1H8O5Rc/a6NLRTjTNTDO+cGyPpBPUJyJvHL9yVAdSB/L6te0Zy35cBi3HnFSBbB2kyiTKWPOl/NyPoXQjE/NupGIoThi6Ifv3LFvtWMxmT+pWRHK5+z+XClBoZ4psGQNeCmEc26kawI1PZNs9O206qQZZnyWYTsZtFkN92qmcc71n6D2MfcBO7/nLF04k9DMJl2knUtdhmj5kRklATRWGfkbpU5Rrny8lSTXoney6Z5eTWiSz7VgATTA7T6YtVGTQhQnM1n4AMqYW71y9L4nRSR27EVNhKruxJr+1bE/TQdIrdSRhqNwwjac+StFW6lSUfS0IhsiyeZGNZwuLLQ0+Sor7cB3TBvKblm6SDN9R6dpSneZkd1akurFXUjc+AlgzjHlliQE5GlP7GFYm3z3ZqsMnGpfL//UATO6lFjuOgFVH1s4FTOcx6xY7y+VeuhHpqk/0cE6mcajOSx1I4/C5EX9uBFpJ5SRsp5Yak17imiwbfB4lnpEh86lIf4MCjMIsW+w8x2w7fBkDdAJSoZKZgck0jcPlkbHMDaaysZpE6kLMqiGUqYyHdYufZOhtlKkSwVpk5VTToeoOPysHj6RqLWiNXm53EtaeUYzBOr7I0NtmYI3RGtpOpK0906gVal0RZmNU1QiT2QOwWC2iqkZ8lD17GEN1VM+EKrVjPBMj8tifZTf7x+tgx9qBZQzeDyxk/wivh/30EtoQht5K2LGe9F7R12pL/iEf5e8BLPuE2RDCFz8VNjkOvzr7V3/ft/vXbv/bP/Bzgy84sMzfeRje+M//A5zTjMYN87LmxSfHMLYEpxjtVVhrCF5hLwvMUUM5aqirDGc1vhavZWg16bRF6SBBOC+n6GknSoSbjDB2mNyRpJauEUmtXiakDzZ4p7GthLqE24zJm0vWdyNCq9EjK4xXZcR7l0hIiLIyQe6OJbofJ6ufZqvRjaI7tKjSMZlV1D/cEwCBSBz9cUv505zqg9h7GXVpZr1LQfNlvJl6hd5v4GUhctlUgjm6uchndRcllEkgWYpUqJtGuc1ICt576ZUdhQg+ZB/5jcgcuwOPWYmPyBfiy+t9eHYS8Jln+nFCfRR7Fu835J8WtHOPn0rCZ3Pg5Tw0pAs9yEWDkdRK6SmUePftfU8wkMWi+tELTTsT750t4/ntiyfSVEpknTFYQzxFshLvRgFdK9KNGoI9sqX4hbpJ2AGv+5J2m9zKF7edio9v+8BRvBL2z44k4bE5EqmoHYlPMFnJjbjdFz9mX8RdH0uJu4oTfG8C3Z68J+NnmupYwEwfJpFuJEVSd8KOuJj4qWMBu4rgtJexgryPo5eK+kiuSS8HtROZUBWXAohcKa/XDTtfonotkVFHtudWJHNJTA1NNnqQjorEVYCsj1JdZRk8JSamS45fyHu6fJehBkGeHMfUXZTHpv12ZUJd3Oz6EnvJanMQAV8u5e/Nvsyqiithqfoky25C9E/FIA0txyPBMjs5rbI7uR1RtupyyK9FomXHKnZjynXy8U8S9zE6F5Ylv5Hr287l/HqQ5CP4/L/ZclkEdQAAIABJREFUe5Mey7LEPOw7053eFC+GjByqsrqqurpJNwTCEiBSC3ttwIa99VKAN1oa8B9oayV4YwHaGTAM2IAB+x+IS+9ogBZFqskmWUNX1pCRGeOb7nQmL75z7ousZpMC3UKzDF6gUBkR7935vXu+800A96O8Z91BeUtfqtkxNXP/HtmZLLWLitejusssDo6AJ8kcH3cd8gIC/TlBbLHJ6YzHInhfHgGsCIAcMHnfdEvZ57ggYDJ7rjIogjE7p+8SILDUXe6GjNOgPKQ0SjXQz2ebFESTZOM5PZLplWmXs0T50XXPcm/pmaDZnbNrsXqIBEkyn08O2qM4BpqUm8RyJIAjHX1tIvB8qP4oW9VdnAblUQLjUkxBOe0lQaRM8lSVwmeyXDkk8EPJZQpvUcdt5qoLgK9t3oYpGVRa3gu6P96buuNxmzaBh5IAoHnj0D7RrCa5oeQzewuHlZxCcWQKTJGOfZXmkNNKCZgJpuLUV+kLAVcLVA+edRsLCZP6KUXyHIpASd/kv0znnfJasl45wCaDogzEulOF5q3DcKJQbj3GuZpkkxAC4ywH2PB9U5BPJaaqD93zGE0bJ8l5TnzN0lDpUpqrJpDOgIwTRQRFrua1kDamiST6HPVAqajqObGhek4w+FJCdQG+4gHr1mNYaxRbD90RYErLzsvq3hOsIcm4jWCyZ+T2aOfg/piDgy8kfCVRbBzsgt2VwtF/CBwl3Vn+qlqHUCkEIydZaAYSwST5KwBfSegupZ8m6acvmLJKACZgF5TBTimqNiAoMrvl/YhxZaCGMFVrlG9bDE8aJgzvLWLaxygFVJ9ktQOBdQ6tIUDV0L2fwJ5wAW6Wejp9IOgTAnJwCEYhlASa0zMhgUqmwAZEo9LkgIB56BEqAlbpwhEQSZ532dN7GY0iGxjCJEVFOH7e83tk79hpGWLyZ8b0Wj7HQmWg7w6IStE/mfosY6ERpYTsR4bntANiWfA1Azsm2U9ZMYBntEdAVRgG3cQIlAVgHcRojzLY/Pq/ypdYGIb5ZND4WN6al+yBTMvku+zH47nqhyNjCLwD9iYPZeqaFCax1Hlbzh2Df0KkHzRGwNp3mckEKvO/fwlzPLp2fy2LF4/A83vvsdQX8Z8s/qtf+3r/9cP//Bs/NmAi1b+niwCerHd4cfGA/X2D680cUUUUzYhyMaB9O8O4KfFkvYM8GyFkwO52Bu8l4rYAdCA4MwHaeBSFQ/sXJxBWQH5LeYDZSmCU8C3TTYUEg3/OB4TP5rCtQXQS6psKsQ44fLFCtBLNWQv1VYXqqwKx8ilQJPn2igj50R6y9JAHhcXTHWRHUKk+2UMe+PQ8fL6ipKxkEb1feGCv0b7voK4KFLcKwkmgYGpjvBzoTQRgNgrV5QF4XSFUkV7MNCMtHB9wbuGTB1LAPrHsYdsI4GyAbBx8kQY3DesI/Ec9Z95fHjCcB7gPe+ithPnhDsWDQJw7esQcwVX9cgc5Sgy/u4ddU0YWHcvLQ+MhBoXxhJ2RsYgwG3p8uhcOcmS65O4TJsIFE7H/wEMlkOTqiLi2BIIngXUOadAz+0IjlJF9fp7Ax15aiAD0TzwZRhXhTjz6C58AY0R/HlMojcC4ChjOAuWYNoHAlB5qZ2Qz3SxiTJK1ww88e/5mBMH+Rc/kyuRL6p87SM/tm6ct+uce7UeW4PxlqicZBIZ18j0aQP/jewalzHjd7DLAriLsixGujhhXrGNw8zB9HrLfxFcR/VnEeOYxnvDLerj08HVIs/+YAB3ZxAhXR/SXnp2NK6C75GDSvjegfR4omTw91g2YXaqeOOX1Gs4iDh9bCAfs/6MR+48c7ILrCYZga1iTUdYHDqDbT0YmTp4FDCvg8CKgP+cgvXtpEQrg/ie8NsEwZGb/oUd1Q3A0rkOqCSAbmesisoQrSk5G2DkZSoDXY/dhQFARckyTDqnPMPsHowDGZcTmJx7DmgzN7iPWgrSXPA/dcz9J+h5+lNietcDuozQZMyNIscuI9pMB/VnE4UVA+9Jh9yEwrBOD8N6A7scD9u+TUd6/ZJF9FJi8qP0Za0H68zTJZIH2qUBIdQuhYA9eUJTMicBreniPANnVAje/63H3OzyG7jKBwzmw/SSgfcbBeX8u0F5Sgjmcsg+vveT5aZ8KdE8jth+yGsE1AofnnODozwV2HwbYWZpcSOd+977A/gNg8yPeZ/0Z97G/4LlunwmMS4HDC17P4VRg9sZj9wMCHjcT2PyQXXm7l1xvf8L3ZPA+rrhv9PGSDWufHhkSu2DoTCgJNnyRJKIjmTYA2L2UaC8l2qecYBtOxTQJMS7FFEzTn3LbOe2VLBXPb/s09WL6NEkjKPk8vBBTCFC/ltj+gAPWYAgkD89EYnO5b8U+oL2QGFYCwwl9r3e/bejV1QKbHyi0lwK79wmWmree4UYlj6u9UBiWkiyrAIaVQHcmMaw5EWNaJqHe/0jBNgLtU4Ht+xr9mgyYKwUOTyXaS4X2qZgYx34t0uslyq3ntdZAf8pz7WqB/lRinFNmeP+Jhl0ItJcaeojozvSUirp/T8GVAv0ZpbzCE/jvn6sJIOeEVjsjq8uJgXTOtUB/QuVI9iWaPZ8V/ZoBLlEAw1KiO9cElYnJdJXE9qVGv5LQPUF6XnwhMC6ZvtqtJXYvNfbPFPbPFfoznbyiArv3SyAA25dmYiqZTMrqDVtTsutKicOlnkZavhTozg18LdGdKgxrQwAMoL000L1H1FyHaxQOzwuGJj2rcHhapG5Qldajp3USYAgGFoUIVysMp4agSwC790vYRmNYG+jWY/+imADz4WlK0tZA+6yEnUmMK43uaQlfSGx+a0lg3nAyUe8pNW2fGPiSYNeXTEqNRsLONezSEEwEYLhskl+a4Fx1qdZDSbKeQqC/pAKqvywxrtnB6EvJqpGRDKevyDiGgoDRVxqhVLDzVBmiJYJR8LWGPa3IbiaPdqgMRO+SN1UAKcU1aonxpMR4VsNXGnJw8LVhXYiPCA3D74QnMEXab8QIv6rptdQSoeGMaljWECEkDyrBXJzVCLWhp1NJhJMZYplmYKU8Jq4aPdWGxFzpgcy0S8TCkJlUnFSY1qEIRGNZ8P0g8IylAYaR/24qgtwc/qMSwH4MKuWRhQUAMasnMCmMQczMZWEg0naE1nxN7q8sDERVHgN5ctCPc9PxyLKc/hNFAWE0/9MaQslf/Z/WPNbvpsr+/fJ3bvleM5bLH1/Gn/yrf4qL+oCrwwJCRNxuZvBWYbnsIETEvi1RVRaXiz1+8eYMQkSUlYW1CmfLA242c3jP2gtdONj7CtV5B2McdvcNYCXMckBZOgyDhhuPttTYK5jlQBlsp2BmFrbXgJOQtUMYFYRiebwuPIKXiFGw/mPQfG0UEIpf/gC4D5sS0BFyq1G8d8DQGggVKaO1EqIIKGcj+usaqD2EiohWst+w4/6JXiGHAsW5A2RENU/vqTjAVpWDPxgGDZUBzUmH9rZh4FARABWhK4cYBfzOQFQecZAwK8pv7aak1Hd1lLAKm2b5lo779KDhl45BRY8Ta70AqgBdOfibEnHmIVqFOPOolz2Gr+YEn7VHbPVUwyEOGrKT8KcWYq8hzwem9vaszogidTkuHdQDAWbUcZp5lC0fjqGKEKNAOLGAldw3E6Z9FCYgdgpipK+rf99CbUgfhTJJcVMlB8BJA/2gpiqInBKr95Jy0TlfK9v0RS5YvQFPoBUFwUKWbebXq1bCnVqojSZjrAHfBOj9MSSpvKHvTHWUA4vEgIfGw9xreu32cvI0TXLLBCiFZfKqCGDVxb16JxAnB8NEzSqO8oayNVfHKUAoVzsIexxU58GdiGRSh3N/rPWo48R+iACGNiEx1v2xyD4UEaGOkB2DjATDWjlBkliS4dyjuFMpeARAFBjXHmYjIb2YJh3yvRcKghw5srOyvJFT6icDmMjolPdkn8ZVPPYaJgYfElMIlBxTeJTEJOsNhizk4+Ak4TGxtDkBV3omy0ZD5jCfb4ZlpPXsxXQeRaB8dZKM56TRMrGnC0q6iy3X4WZkYynVjlNYVjDcH7MnMCsfkmy4xsRe6xYTSxyK4/1AWSSOYUSJ4VUjWUw1cps5eEb3+f7BVIkS5aPrYI+hRZkVAxLbnM6d2WHab9UfpbSu4TrNITGIqbLDzbju/Huzf9TfKHn9zD5O4Chvt9gRbB5DklLaakPGmhUMiQEvxXT/CsegH3MgY+WSV40sd5KZFmKS/o5LMqGuEdPnQDoyShM4nSS+x05Gs0/vSef+2AtK4JTDwzKrmq9L7obM5zZ7MLPqYApSAtlC6eNRdZA/OvJ4z2XGNjOlWR6ez6OrubJyG1gB4h99Zv3x3lED+x77lZoYNF+lCpg5WUs1JD9sYqOpKIjQHRnizFrma2FaeiBzaJEaKZ/VPYGlLxKLLI7nOWjK3at7vjd3o/oi3c+BgLt68LCzNLCX7GzM1R66j5jCpSK3m6s2cuAOk3vzcbG7UYRc5RHxWMUhHNn1rKIIWrwj45U2wtUSZufg5oqqBJdkwObon8z3VQ78yexploVmFYEc+Xc5BPhaHcNukjzV1fzeR8C7QUHp/uM9GJiyaiibZTqxhu4c+tOCbPoYWNdRqHQPk4UVPk5eRkphPcO7hJjkqwgp+RWYXid9gBgDJa09PYKhVBA2HJnJFLojcrDRo+CnKBnoI5NPcQoUivHok01+ReGoOEAAoBNb5yP9jFoiCnFMiA0BYnCIlXm3FiStM/+cvZDckCTI1Oq4jzkYx/upKiSmOhDhuR38VazjdGz+uO4sWX3MEuZk2O+mtuZ/M2Rjem8MZIgnmSt/OTGXj8N7ACDmbSa28ruS2uNuH//21+KSEL//4T3qPP6T+X/5a1/vv97+L7/xYwO+5+E9g9W4283wcKgxDAZSRNh9geXFHtttzT7HTsPuyskriSjY3Xgo8LozBCspKXbeDLjvDPqbGmMvIQqCD3kSYS1BhRARYUhfsCpOIBBRwD6UMOsedlsygbanvwGlh31TIxZMUT2cSYg9K0ggAFTU98VWw/YKMBHL0wN2/RL9QwUkn6W61/ALD/N1gWGpoQaJ0HggCIi95uTZjUF40SMEAdFJBoUcFGIR0duKSbNFAKyE7wug9oCltLdTFdm5QQCOCbrxLgHVMgJWMzTifga39NBbxUTYVEcisqw3Bb/EmiwpO5QoG4UHinsFNw8IjUe4qiAkgIOC6iW8APzrBaQGQi0g36REhH3BZNIkj8M9Qbl8VUEldkYMAmqkj2qsWaWibWJnZwwymfxMWRZcaIiBwSHSawxnHnonp249s2dgRfFG0xfXCsqoDT2pOjHEcnwE2LxAc8Vi90lutFGpOiMNqqKgnNQTPLl5RBBAeU9gLq7JfrhZRPVVAV9mHyEgPY9NjQAgWN/RJn/cVk6Ap/jGwC4jyms+yPSe1RUQrP+YBusKmH0tYZeAvJaw8wg9iilIxJeANIAcxJQEq1tuO/cVSisQkFJfO0wDUNfwZb6MqK5UGpAjSTBFShWNEHv6QPOAVQ2g57QTELeY/IJRpIHngWAsCmD+Cw07T2x7WsxWks1tgCLJgzOLqHruPwePOTiA+6hGgWIvpsE4Imt2RABkOl8IRyCU62XqN2IqtS+3wHCWfItzMfkgMyjKx2AOrIPozxnikn10OclWHdjdh0iPYlTctsgT1cMRSElLP2GUAmaPSeal26Nk1s4JOMcVX5+lx7rlcaqeA3rhkCTaPEbdAf6RnLi8i1AD0J8ek0Lp50uy7ATKdEuw5CsBnQfnTS5yFwiW3w3SAfr+UdG5B9yMFSdR0Fdo50xp9WUKdYmpOgMJ8CU/YwZmzbcRdkmJKkfaqWvRCKiOEl2XEoF1qtLQPcFK9ubxO41AxOwfScSUAFyE6vMAjNelvubkQrmNaM0RWDExFpN3T3dHwC48g2jsLHkiQ4Rr0jURR2BT7LieYMQvJQIz+RZTwrHuIpRNHkGTttFhAidMVhWs9aj5s7IEcsID9S0nOr0Rx7TUBJhyVyMnCHjeyl3EOOPfsv8PQqLYBYxzifLhCD5sk+SoCdDpIU5S1smfG/iaMgEo0zKxttxyPbaWlF36kGTEx3oONfkXI3JqLtdBIGcbiebGwTZJ3trzda5hX6Q3yUu58+hPNXSXU3IB41NwSxfY95n8lznsBkjn6OCP8tmekmw1xCmltkppsQAm8KeGHHKDKQQKSjBMplIEco+ktmTzRPJ6pnvCR6jOk9GTEbpzQFSUjh7cdM7GhWEiqxSpp9S9E0gTCtZ3lHcDxnWZAB2g+gDpw+SX1JaJrUHRD+nmXK/wATJJUukDdkAAygeLoI4eUjkeU1ZVT8+jTvspfDz6JCP9oQAIHltLH6MnyFMHm96TEmW1hDpYAlcAaqBizDcGKofcJDAWNIGp7LhOekUBBEpv82RLlAKyt5SnJsAtDwOZSi0niaoIAaKnNzEqAagUrDNaxKaEaIdJ9j11TSZJK2I8AsQQjoAyEuSKfuQ59R6Q+pgSm2+mkDyUIb0/pCCdx/2QQhxDd3xgz2aS3b4DUB/5HWMIlLcm9nJKZvUBj+WplLymifacApv3P4FT8Z06k+8CyF8pk33nReGXf/d9XL7HpN7ftHyvgSUAzOsBD9uGvsfS4ccff4tPry4QNwXUecfkaElmSABwg4JrNcNqbumfFJ1CWDrcv14CXuDkxRabL1eIpUfzrEO7K4GHgl7EjUS4cGT5Kg/RJwN77Qk0HyrIjmlmce4gdIC8KRjrf8dUOXlvEFYO+tqQkQHovQQlrOOZx+HTFeKCX+hikJA7hsIAmOSx8qKH+KoBAuDOWMXh64hgFeROQXgBXzvuTxAoHiSGHwyoPi8JZmYRcscvT2kB7AuEEogpnVLvJEGF4yA/mIjiTsI3abY3ALGIKG7VxLJU1wL9RYSvyHyV9wKd1BPboYb0IOkERErzdKcW5trAnjoU1xrlPQvOm1caruGDt3jgzHR/EaAcH4h6Bw560gDYbCX0gR2F5oF9hm4eUF4rRJHkVAuCMLPJQEaiuqEXMegIs6U3c0iMmbSYwkBUKyaWxOwlVI8EWjlLXN6yk3D2dcT24zTIO6Rutmsed5Z9cYY4+foUPY7FQwKHB0o+heP7xiXZMztnkAsj1TFJm8vEkgGYZuHZt0jwVm7IJEESUOqex+ArsJhd0W+YqynMPs2yK0xVEvqQ2COTfJp7MQ2S45jAhE/nJyVxuhoT01c+sJdvWLMCRgRWi5QbMkm+JDjJiwjs8+ue5JRCritLXXkPJVauTpMaaVFjTs3kPmdvYVSA2WYgzHVlUKW7iPHkyKy4GhM7mnv/ig33Xyam7zG7YvY5aIjsVnknmHIZACRGLoP0qI4BNv2pQPHAwWt/ShA6dTRaXm9fY/IEktEUU8KlGtKxgWBM9ceAnKiB2etAv+T8yBLJMe2v4zkt7ymXzEE1ZHgEjKUkVHcEjMLFdF2PhfLCA/UD5drmwPNoGw6UsxxW9dn/xg5EghyGwJS3DMnx1dEz50t6Qqe+y32c6jDUyNcjJG9i4HGUm4DuTELGCJlqZsw2+dP6OAU66TuPcSH5eQBB1LDktpl+yckFMiUEJK5MDF1BEJp9oGZP4FfsCLb0wFoNXxDki8TclZtUK5JAjvQRs6vAIvcugeKBElgG5wDVxmNYqqkeZJwxHMc2SVLYRfYuJpCVJe2m434PiyPrlj+3/C6LE5uo0udI95GJnKWe2DE1xKncPhhOKOkhMqFTpRcBmF1ZuJnixEdiWKWNU2UJfctiYm0zkMuBOBmE6YgJwKpERpiWYCpoSq+jFFNno6sTmEpj1/rGw86P4T4iYgrzyZLyYcWeSDujIqPY+hQaxf3RrYcs5OSlVIkBFB4wO4v+ooDeediFhm0k9EAAbmcS5hBSHVKE7j2BeSUBy30wQ4BKEwmIcQoNCkpAH0Lqe3UYzgzkGFHe2aNKoVTQe4v+opz8osJF6EMGM3HyVIpHA3FfKuiWQTgTQBKs4Mj9jz6QLQz6WIXB9QE5kGc4MShvR0gfyD4eKKl0c3Os89CJYUzVIKFQ0B1vMGE9xnWVkmZjCslJgTybAZiZNImZQ18YNCN7O/kqhaUcNi4L/j4CcAEigbMwK6ZAH+ECf1ekLkkBCB+geno5pfVHBZP1DOVJlSGhKageSl7PqT5ICQb0LKq0/gDRDZCPex5T1UgGb8JlEEiJqRgcZa8JSIosPc2/k3IK1RGDpScyg8oc/iMEoiGohPOUpD5iGd/xWkp5BC85xVXJI6j0/l3Q+d0eyxxOJAS9pd/1cWaAJyQQjqByAqWpz3I6P2ndQh0DiHIYz69cHnlEJ5CZt/P3y9/Z5XsPLAenoLSHVAHLWY/Pri5wsd7hrRdYzXvcbFhMW5926A8F6sWAZydbfHu/glgMTIJtC5Sf1Vj8oxvcfXqKw5+tYT48wL1ucAgNXry8RfnC4fMvLmHPAtRGIZyzVsA8beGsgnxTov5oi/1DjWgFyqcthtYAGwM8ZdCOAzgAPx1RFg6jCRAiwnxdwS35QfMfdUCnoc86hDcNYkWfXwwKdhERTy3qPy8h//EW/Z+vEN/vEbwA9pqMSZ3SXRU3Jnr2TOZ00dgrjD/qEA4aovYIMiL61DF5zYoLUXrETsEqykuDo1QSAvCHNHC6T8lxNwrjRZLCmoB4V8LNPUQQiB906HRNn9Kph16NgAqIn8344HqvQ/MHDfaFhr2wkHsNuw4QnrNuwzom+WWA84qSxsSO+UD/YPOVhpszVTWUEb6WsCsPtbSIb0ump9YRbu0grzVZNwnYUw/cKkbt/9BDtwSUdhEwPrMovzbwBdB+PKL8usBw4QAJFLcK7sLCDfxyU61EKFK/ZSRwswuR5E0cGLhzi/1cwjwoSCswngSogQFFqhWABNyKA4DQBIS3nBQQ4ljJMJ5EVDeSPtALB3OjUTwItB+PqH9RwM6ZzqpSKq5dBJiNhFtGDBf0gTZXAt2TgPGUwL5sBdpnIYW8SKa8VpSt6jb3VwLjGaWmDIxhuJTZcBLDV0D9lsAwGmCsYxps8bhUz+ePL4FwykHe2DDV1S7ZzzichsT8iakCYDglqHQz1lMMa87mVzcC/Wny06qUBNuKqY8zypT0mYBqUJh6BKvbiN0PeI1irgRIrKTLhe0z0AvdpXAVw+eYncVp/3yZGMH0fLNzAqBxGVG/FcnvGDD7WqLYpBqGSLbY7LJHGNPkQPucEtViQ+8jmVYOvHNwj5sdqx3GFRJgwiPmg39zTQLNc75m/4IyX4J5Hqtuge6SAF8OQH9O9nJckKmTjkmu2bPcnfMz01wRvEobUW4i2gsJIekhzNLX7PUrNmShXC2OoLqmHNTNMEl6o0zyUAPYxF5UtwS2dkbg3J9JmJSSGhJoc0kmm0FSBguZVVVjYjnbOIW2DCuB9omCGnIyKtCvxTsTD5RcJqnqbUR3JuEasqZkZwVcQ8+tq/mzHwEIoL2QBJG1mIDP1DdbIAXrEPjtXirMvg30UGbf3olCdUsQPM4ZVJSPTyTprbIEl5RAH3ttIclQsnoCqO8DunWSeA9IvmoxJeJGyQklX9D3WW4p6xwXErZhqI7uj1LOHIIUBSs95Mj52sNTM01C5TTWcaFgZyn5VZJRFElua+skM45kRLsLjSqBfVeS/czyXnorI4QXiGkCDpHnRoQIm+oyomQtiO4C2nOF4kBJ8LBIALMlEK1vGCI0BYCd5EFvmqCqNJSNGBcmMfgML/KlwJA6LUPJn5s3I4YTA1cfPbasAZFwtZk+k64++tZczfsRkf7BmO53X9PT6WYaMgFyO6f3kIAPkFZTrltI5D7IkNioqCXkEBAq+jJzYnAA4E8Mcg0JQ38kQiHRLcqJcfelQiiTP9gFhEIiFALSFbBzheqGqbNklSOGMwOzY9gTQHBcbC2G0wLFg03rlJCW20KtE/Ofql92Dvrgjj21RkLYADs3UDYAPsKuCpTuka8SGqgB+Ag/Lwk2Nc+hW5Rp4iLSJzkSaEYjEYSA6umbhBIMJKopTZUuIJQadlWiuGl5L4QAGEUmVUr4OZNlhQtkHF1ALFKq66KeZK1i8IhKHVm6LHf1moE8dUFgmbyUBIhMjxX9iFglT6UjEA3zipMQw0iQWZpjS5d1TI4VOR1a4HHXJR6B3SklVogUOpRkr496Mt9ZMhAMYZK/RufJimdPp3MEgy4CUlESGxLozGBTSspalUK0lmxlBpQ5gGjyYSZfqPfTv6d9eGffxDvv+74v8W+qV/keL99rj2X13vvx+X/338IvPMQgGbSy1wykGSUgwZoLHVEmWWCoPdlLK6EGls8jAPogYX60hfvTJSCzb4QDTN9E+LWjFHXlILc040eZdepAeaMQZcR4EqDbY31FlGBNw5YPdd0eu8jcnNJBgOsyO4nhzPNBWjAghn9M3pXA91Rv6VGLkv423wQUdwp2GVC9JZCi7ImD5PKWM5bjSUB9JdE+DygeCIjywCR7Y8gOiSlhNZepu4ZBJ7rl34o02EdiudgJF1OnHQHOcBoQVcTsKzWVsUeZgEgB9JdpPxSloOWNZOqsTomfiRV5XCORE2oJ4DgIJ7NGsCEHgWiO79Et5Y8++cvMHuie8LVmywF8+SAmxsWmQa/ZYyoyn4q/CzJIQXPgrnoCGJGYKs6C8z39OVm77NGTo5gSRX0RUd1SwpqTdoMm2FIDUDxw//Yv+f7FFwzlKDaUWAbFgKDMTsk0gJWOLF4wBOX1W8GqCI8pTdSnahLpuP85GVXndeXAuARSih3BrU7rVV2Skya/5uNC+Oz5UinlMjNaGWDX1/y89GfHwXJet2vIJkIyRbS6jdi/BJqrIxuTn6x2kbYvCU7KB0whOroloMiSypzQmfczM4FkRuixy38rtjEBTLKoImbJL5kIOycIyumn41Ic5ahJxhhlksQW/Fzkdbo6gbskATQtK0jWfxZx+zvYZc45AAAgAElEQVQC9Ruu25eU71a3PFfDmt2CWSbKICtxPDZHKWX2ZAI8D8MJ99HOBZorplxm35kayM5Wt/w87t9jd+B3fXO5386X2WeF5FOMmL0J6E7l9F1ZbCPaS8k+w+SdjKkmY0rIvQ9wJZm9YSnpCUyMX7+WqG9TsNRcTN6wDIZFfCQL3RN4RkkGzM5EkjbzeOvbMEkdH3sNRSToMwcyi+WOvjnThhRcg4nxGU4kZleeSaqerKJ0BEYAUBxCGuiLKZm13JB5ku7oQWWqL7/7laVUM3vyZJLHukpMPsIMzvK9Ps4lGboU8OIaMqDZF6rGgJAGXN2ZhE73YbkhSJw6BDsmtJouoth5uIrPBF8cJb8ZYDVvLboLw1TYGY8LAtO1G2eSKbKJTaS3mcycSkyzK0UK+FGJZRMw+0BW8RDga5lSdhNz2dFvSPAV0vdGlkkGuMRWZw+iq8kys8ols/lZakxZJUEdoLoweQoBfn6C4j1W3ntIT6+iHOOUqlredGhfNFN6LNULWWrOY9eth+w97KpI31FH9s41MiUFOwxrg/LewhcEWmoIkDbAzjXM3k0+R19KmJbSVz6TI3whUWwtXE1Jqz44nt+Z5qRM5+CNTM8+DzdPoNaGqdoDPk4JrUFJZF+j2dmJzZRjONZvGDG9NkuM5egniSolngKuUig2I0KZWUV6J/3M0B+qUipu56bqj2jkVKsiB0dWMcmIRYwIWr7DugKA7Bzk6OAX5cRayZEAEYJ/5zgkTOxdBnw5qTYmP6TwkVUgANnQCQBGyPx7LembHC3CrILwBI1ydO9IWqEJJO26hn7oIYbxyB7qvN5AYDmrE2gU78o7Uy1IDt0RgcxiBpoTUHzsjYyRLGcIiIcOoiwQhxFiViN2Pdc5nyF3VSIzn9M2sx8yEPTlhNXcW5n/XVf8f/CAVL+873YEtE7rC4g+/DIDGcI7LOjjypDja/I4+jvr/+6SQGvM+/S3XP6ueCx/r/7Pf+3r/f3D//obPzbgew4sy49exE/+5X+Di/kBX745Q1Fa9PsSsvCIQTDsZlBA6VEvBnTXDbI3UgzsZexfz4AFTSvRShRXBv6jDvGqOtYprBJwLQJgArsg5x7FGzJsCKB/ToCS0SogVgHmVsOXEWHmARmh75L0tSErKK8LiEA/XKgCzIOCfWJRfmtYD1JEuB/0iA8FYs2yeHYRBgbrbDTZSJdqKOaBsllLqaadBxQbie49C+Ekog4QIzsaJyA+SPY1mojQBIiR3j+3CPRnjjxueVAIMw+1V/CzADmzwHUJfRCw6wC95SDTLj2qtxr9MwtIoPncoHsakg+V100OAtW1xLiKcLMweSflKOAuRqjSQ76qoVqB/sMBcsPuTQBQO8V+xhcOZkuvZg58yZMBUTNYxmw5yIfkrLcvE2OUZLvSMWlVdXICG/bCwdxqgnMdUX9Npm5ccefLO4HuaUB5JydZrS8j0xb3abCb/HD9eZiksO7JCPOGXsniXmJcBwapJBlylsF2TwOqa0nQ/rxH8fMa/SWrTYazgPmXEuMqAwwOdudfkoECeAxukbokF3EqOqfsVhwB2SxOYRwApm7H8YRg3804OVHciynoJZejs2/yWOVBiSABe7ERacCaKg2SR7W8FTi8DChvZQKtnHjJYNQ19O3pA711h+cMCjIbif6pQ3HP6+4rTDLiHDrSPWNwkXSUFD/uggTYc5mPTw2YuhtFYJVK801iHZp0T8wJeJpvKCH1DY8rs6HSURps59xOsUnbTOEkubYi17eI9NzOAUIy152sE5NngFBSblpdEzhnSSBTLzH1L6oh+eLcMSgnA8piwwAZ4RPrtxRTXUmUwHBO8Kj3lEMDZO76M064uAqTHNYlaapNvrtQZEn8sWYkM6Nml6TCW26fnwVMaZ3jIoGokcc2VZjk7tKUzhoVzyvAY8pS5ZBAmj4kv2ZiIeubgP5MIncz5gAVXwqMaTtqTHJKRTCfpdmUK3P/uovUy5jumaiP50z3fN+wyr2QiQFNn6OQQrDGBe9lNUaYA4FyPr/Kxgn8+ZTiOy7FJKHNoTrFPmJY8nW652t9CvwZ1gz7qW89di80qnv614cTgeqOMuByk7ovE0AkUOb5dBXXKV3q2EwTIjkJOcv982RF/jmfT2+STL87fm9kBi57kaMiKFRDxP4ZVTPVPcF7Bn6ZTcv3hhri5E8UHhhWEq4SKLcRw4LdlLqnBHdYqWnCVHdJvlunxG7FtNdiT08jg1f4OckVJTmwJ4P0HAzEyT4JWws0N26qCtFdmCpJeG5EAnvJfz5ESnlrCVcxYTb7VzOgBJJXOaRjtQHtE4Ny61mRYgR0y+NQ/REYCc/zwtRVgmB9eLf6JO9TcT9iPCmgO59kqBLByEmCzO+gmPyhfJ9rFNTAehMm1guUNwPcTE/3TA4VsguN5psWw3mFUAiYnYccPOzSTEyoHFkloneW16hUDL6yfmIibaOhOw/hAtTgYRcFdGvhmiSplZh8mVEImN0I32jARyB5KUOpIfv0fxvgKz0F6kQjobcDIAR8bSCtTwysQkw+0PyM5P3GzkoAkL2jbDZ3V/ZM+I2GLKBwCdRICTk6JsoWmiBViKm/ki9K4DUF68RZBdFbgqwiVXZoyd9JcfRSCnEEV7lnUklKWNP+Rq0mABvLgpUjj5fvBu88XjJDFuIR8D1mKB+vI0t0cx2KEJMUVihJYJg9lZmlzPUk1jEF9rF387tS2Ef79FeCzL9uieH7XzeizuLvVf8BgGX7v/3Gjw34nkthjfb4T198jqt+gatqgffXD9jPS5xUHT59e45y7rB9M8eLZ/e4ul3hg4/fYjeQoru7WUCpgObFHnVh0VuND0/v8LP9D1AWDnEjUf+jW9zfzlHORszqAYM1cE7ig5dvsB1LXIVTLC/3GEaNsWPtCBKL+eMPrvAXxSWikzg532N0CmPjEO9KnFzusN3V8PMAlB5FY+GdggsCunEYVxrPf/wG3745QTxoyNUIqSLMcsDYa+BgyLq+PGBRWUQA+2+WKE87hE/nWPyDW9y9XuGjj95g21eogsDDw4whQFFANA6XH2zwsG+wmnW4vlsg9ApCR+gTC/e2RvX0gO6hItt6r+Hf6yEjIBYWwko8vdjg2+EU4zri5PSAh7cLLC/28LsKfRWAXmJ2cYD/ZoWzj++wPVQYdwUW5wfg/1pj99sWZjEg3lVQlx3Djd7rgV4jXlWw5xauU5APBrgYYLSHu6nh5x59LTB7esChrqCvC4xnHig9MEoIK1F/o+CeDxgXlA3BSkAHqI0mI+MwSW9jFeB08pCNgoB/1FBSwC0c3EzCrgKi4XV1C8qGew3EEwu3MQSXS8dZzac9xJ83GD7uETsFJxXCwgGjhByB+rc22C5mWD/Z4f6bFfdPRYxPI7IXWH1doHgQcLZCf87QoXEVERYOux8qFE9a2G9niOsR+nWJ/cvcTUpWWLUS3W/1kNdFkvhROjh/JbD70CNWAfCCQU5LB7nRsGvP+hov0F+mB4Bjl+F4zhnh+V8ajIsI++MO+KYGJCXICIDeKc6k/+SA8HWT0jwjsBMYnziMawE5sHM01Jy0UHs1MYtkaAhC3YxAy584RKkhe7Lgw5qfe3vmUL02sHPKn4VNFRcxwp6n5NMIiJh6WiPQP3XQD2rqAw0Fga0agfZFhLDA+N4IsdOQg4Q5COx+5Dngv+Pgq3sSYHYS3ROP6rVC9wMLuVes9/Bk88eTRxMGIyXkYc6/Dadh+n7IQVZRKAzPLYq3GuPaUyL3QQf1aQ3pBbpLrq94EBjX7BPtLwLMXmJchaQq4PYffkwALBzBUveUQN6m7lK79DBbhe65h+okFp8Du5c8191lRHXNup/hFKhugd1HgNlg6iYNJg3Mkzw5T34BlNVuPwIAfpaYKizQvgjQe6ogKMENEF5O0kokOakaeA8MZ0B1E3F4xvfnBNhhzWNVQ5KflmCCZOCEQK6isHOR5NxIzLGY/MqhyJ21lKW2T1OiqALa56nLVdJ7GLI01pPVLx5EUicQJGfwZg6UJmdw6CsBOyNDTfBG6Wxmvykx5nF0SSLqGgLqXotpcsOXmO4Tn8DncCLQn2uYXUR/SkbMzgR8Qe++iJgmAuSYAqYCGPZSiMmLLB1l5pwgIaOu+ghfM0zKG4Hx7MicZ4/zsKLMXx9SgnQJDGt+NjIw94YTjDkx15WUxVf3rNAZ1o9AWgpSM/s49YEy+ImgMiqCwPZConyQE6jzJZmow1OVpLJpAsaTtRWBAUreCAxLpPTWOO0TwPOUfdF2fpQ6759pqAEotx77Z3pSqoxLgeYqYJyx59OXvHbjzNBPKgDbSDK4CihLAdvIKSCqOORjlClsSWBYKlYltTLJ8OU0kUT5MF9nDpR59meKE6ASgFDoTiXKbcS4rGEOAf15AnpDhJtlABpgZ7kui/aEYaVQ3nt054b9r4mVDJWCr9XEYuueLLKdCXTPa4yzFKAUgf5JOQVBicA+zigF3CLVr+jUZdsrqDHAVSpJ6wXcyiB0nCQYVwWEjxhPzDFZeCSwdnMD19An6gtJkK0EjBZASoQNRgIpbMhVClFWUyqtnRN8qt5TMjszU/Ks6unBDAWfQUoJQBUTmxoMa1RURwATyiSxNYoVJAkMqZGhQKGsksxWo7zaw62bo/d1dIil5vhgsAizkqE9ScY6pcYmaWpoSgjv+UjMIM9oxNHy/yEgViVk2yPMqwm85QRaptS6d8Fi6sTMnZexNJTFWodYl9PLJrY4ezK1PvohVZzAogiBzWaCY0p6ScVxe0q9m/AqBOtCHi8prEjk/ZQSEX8NE/n/J49lxL8fiP6eLuqnP/3pb3of/tbLP/+X/+NPL/+L38G/+eIl7LbE7e0S/RcLXO0XqFcDxj9eo/jggIdNA/N5hfvCoLuaY7hpgFFB/3mNVmvYL2cYa+DNzQliEaD/ssF4EtDvU0rq6xIHX8AFCW81ggZuPzuDCALjQwUXJcybgpHeo4SoPW6uTqDuDRafaWxFgeLnNUZNdnDYVECrcfLBA4bbBqIMKP60gV0FBHAArpYjul2F4togdhrVLwr0QkPsNVQrIUeJ4ssS7rpGZwuG0bwqAQUMdw2EE7j3JYYvF4ifNRCtguwU6tcK6kGju5pDvyqxv2+AViMW9GYWn1dwzy3spkT9ykC3EtWdgPcacqchHgyiU3A/W8A9cdDXBbq2RPVao/UFxF7D3CssP5XYrhTqj7do/+0p1Dcl9E6hVRp6r+AuLdQvauiDhHpdwldA+aqAuVPQvWDi7DyivlIoXhvI6wJuFVG/1kxK/aMawwpY/VyifxIh9xoy1XOYViBaBQSJkz9RKDaK0rd7CbsOkI7rdxcWi58VlHZ2EsWDhLqnzNnsBapvFexPOgSr0LzSDMRZO8w/NQgGKL/VCBXlvxH0qOjXBXwTEYTA+t9qSsDuNIo7hfJBwN3UiC8G4A9XHPDsJPwiwFwbFLcai7+UE+skooAMmGL/q9cabhXhhUD1lUEIEs1rie6Zh9nRbzJ/JVFfA2OloA+UTEsrsPiFxOY/HiBbhdkrjfpKQjqB+lsygcLLyWe2+IL3VzBAfSXgPhog7guGHh0EQqdR3QnMvmFCY6gjqjeK/ktvMP9KoP3YQh0URBRovmLf6snPkzRpkDj9Ewk7l/DzgJM/k4iC9xkkmdbzfwPYWqF+w5RbsxdAFCh2AlFI1G8F+0bXEed/JMmcL4D6WqL5FqhvAEQBNXKds1cS/fOAxRe8DgxfEVh8Sdmy7gVO/kShuqNHdfkl2TDhBOZfpYCbU4GTv0TyKUUIK7H8jPs2nhA8lQ8MTWpec2A++xaAIOAsNpLgJAioXqK6ljj7M4/9c9aiXP4BwY68M1i8ovqhuUqSwMQyD5cei88VyvuIYiNRbIDFVzFJTyWaNxHmQBl1sZGwq4jlF/TNmj3P5enPeAx2xX2EiqjfSJx85hEF+w+rW4LNp3/gsf2Ix606BjPVbyVm3/D+NIejhLd76bH+dxLVHY+52FNaWOwom1/+gtJmJgELrP/SY/+BQLElWJh/RVB68jm9cIsvI0wH2AWlwlEJLL7kYLtM7Ojq8wDdAv0FsPwFAeXqC6YULb/yGE4kwVFgOFpUQPOG3l56biNe/n6HcWHIwp9EVDdANAKrLwIO75NFr2+oeij2EeU9Jo9jTsWVKRCKQUECJ194NG8C2sukbOgJEIotMJwJLF4FjEuB5SuPwzOJ079w2L+UmH8TsPqFRXepUN2RYZUOaK4Tu5kSk4czwe187lHfBjJ8KzkloC6+8RjWErM39BdW9xGH5wKmA85/1kNa9k02NxHdOQe4DG5KFS4lq1P6M4HmOqK+84iSssn+XODkU8tQK00Fw+yth6tl8pcDy1eevZ8XAssvPapbh/FEY/bGs9JlLtHcsD/V1wKLrwmAdB+x+NoBUmDxteXkUEEJLgS3U24C7ILbWv/5CD0AZoho3jAJVKVkXD1SBVHsI6oNB6zVvUf14OFKOV0300aUGw+TgJQvgOrOY/9C8zoLTjDMrjz6c4n5tw4iCpQbj/6UgKl5a2EOAXrk9XeVxOzKQnoB5YDqxtIDWEvUNxZRS/RnEmc/7zGuNOZfD+jODWZvXPL5SkhPZn31+QA3Y5BTfW1Rv+3haw0BgerWon2i0/Uh2C72DF5Cku0WB4/9ewazK4dxoTD7ZsDuByXm31pUbzuMJwXqqwGH5xUneUYCgeFEob6xUCPQnWnMXltUbzomy54Y6C6iuh3hGvr1yusW7TOuwxyYvioDZbdqpLzY1wbldYfuRYX6TYfxrEDzxQZ2XaO6GQDB51h1M6A/L1HcW5iHDrp1sKsCxcOI4axA9foAvyihDxa+0Sjue5iHAaE2cI2GHjyK+x5qDFCdhRwDzNstIBV0ayE7S/Fa6xAqDV9r+FKiuGkhO4doFIrbA/ysgGotZO8guxHCRQgpIA8jkIKDpIuQLkA9dFA20hNZaMie2RNy1yE2JRNiHTtk1bYlSHMeQkiIfmSVSNtTcisERD9C7FsIISC6AXHeQG72QFlADiOfpe0A+AC579mW1Y8Q3UCwJgWwO0Bk8DbyHoRMNSW5QqQf+DfrEPuRLOVoyUxay0RYKckSpqqRifUM/K6NXZcUORGQijbPiInZjM6nvsu/Akw9ChgSQvzKOpLJLxrx/0kK+zn+9PVPf/rT/+lvvYJfw/Iv/vv/4afv6U9+7ev9zP3xb/zYgO+5FPa9n6ziP/s//hPc2hm2rsZpcZj+FqLAm2GBpR4QIPAw1iiVw5tugRAFCuVRKQspIkIUmOkRr/ZrXNY7vN/c44vDGa4OS3yyusbGVui9wcFy6n5dtmi0xRfbU6yrDrddg99av8V1PwcA7McSJ2WHh6HGy8Ud9rZE7w0qZbEqetigsLcl3rRzLIsBRnloEVAoh81QY28LzM0IISJOyg57W2I/ljiv9/hyc4rfPrvCz+8u8WS2R+cMWmtgZMD1do7TxQGrkttwQaKzBufNAdYrGOVhvcJ9X0OKiJOqw+A1OmsgRMS8GLHpK/ggYFTA6BSeLnb4ZrPCohogk5avdxo/ObvC/3P1Hj4+vcFf3DyBUR6lcTgMBQrt0I8GPvV2/u7LX+CzzTnPTV+iLixGp+CDxPn8gK9uTlBXFoX2ECLCeYkn8z12YwklIm52M6znLbrRQMmIEIFZYXHf1vjx+Vv82dtLPFnusekqaBWw2deIEVjMetTG4f5QQwjgcrnDzX6GrjfQOkCpgHHUeHl+j7u2Rm0cdj1n73Q6V7vbGcxshJQxJWYHGOWxP1RYrw5QMuBuO8OsHjA6DaM8pAzY7WucnezRjwaD1XBOYjXvoWTA9e0CT8632BxqxCjgrOKsn4g4XR1w/dUa9XkLISLabYUPXtzi6+s1itIybOpQwJQOxni0+xLRS8jCo65HzMqRDHSncfp0g/u7OXTBeFMhANsZ6MrCGA+tAg5tCQGgaQbsNjVWJy20Cti1JUIQkDJi7OhliXsNtbLH54AXECqiqkdYqzBvBuz2NepmQN8VCEEieIGishACaKoB/WjQdwWUCtPnFFHAd2TMY6qraE46dHtei2o2Yhw1BACpAoKXcL2GkBFCBcQgED1nMZtVh25XIXoBWXrK4R0Z63I2YtiXUKWnxaTTkJVDOBhAB4giIHpBNrf0CJ2GnlumSI8SUBFy5rjOXgElO2pjp6CWFn5nIGcWwSrACfaiApAmAG9L9qw2nonSEux4XXgyxyXl7bJVlH2nAjYxCsSFo6RfREBFHg/wiP2MZOUFAE0trugk+1tF+rsA92dQfE1WVwQy13E9AlvDkK4qQLQpalaQpYMAwsxTEl8FqOUIv6GUf+pBdUmGmVjkqCNi5WFuzNR5mpl/1TEsCjrSy36QCGVi7PvE9GafomGIl2wp44cA1QUhMXAjJs+56hjoFKrACbqerDVltgJuESB7OcmS834Hk7sm6XUXTsDNw7Q+RJEYl2MX6CQ3dSJN4GDyR2aZqUznQbjEqC5CksuLSYbOa8j1uCZO8tSYw6/E8e9RJpAc6cuOmpMGURx90MHESQIt07WJmhLoqJFkqzElcaYws2VOP80sKn2cWdqeK3ByTROAiRnObG3uDIxJ4uzqiGLHypNcFZTPWw7deuwdzp2sj73ZuUsyByypniFQmf2VHpADz0++X+2cDLBKsnhfcVKEUktMvalyxOQRzv2gWQKcE2qlTfLzFJgUDIBHkwmImPzEU4qr53tyr+s73Z1I0nCLdypyRIyJqWYicjCUoucMhFCk5OKYzkvyasskhZ+CuYZ3uzpDSsXN51GnGhYm7KYEbUu5su6zB5me2CjEsQvTPuoCBhCUmGpYgibwj5KBTAAgPZOBbZ2k6inBlucw2YaSNFoPya9sGE6VPxe8L46VN/kekwm8ky2Nkxf32O2bZNBJ+pz7P2mdoFSY30FH7yW3JyaWMRi+ZvKqjiFVGTEdmz73mLo2MYUMRSGgOjcFM+X6Fr0bKZ9VyesZwXRbxd/lnszco4lAf+XUJZp7MFONiXCU54YURJTlu6HSyZqV9iHJdKfF+aNM9nG4T14e4QFh3fE1j9Nlw6P3JnAZH/tHY8A73ZvAL/suHy/5b/8eYTZ5PXEY/sbX/qrl74QUVp7F3yv+s1/7en9/+N9/48cGfM+BZf3D5/En/+qf4m4zg/i6ZhAPAHPRYTwUEJ2CyIXgVQCKAH1j4GcB5Y2CXQQyD6nsXh4U6zTqVHpexslrk316UQB+5SCshHmQ7DVsOHAYLh2aLzWGUw6EqmsGP7TPA8p7if65hb7X9BqtA7dVALkGwlURbklJYjSYUgWnAV7ycg5nHtVbymLcjIMI4Y4DGDejZG84ZVjPuE7eqgMHEeOaPj67iDj2DaYZ+Q3ZBek5SAP4gM4F26GMDMpJ4SHVG4nuaUD9hpI7e+rJrNVx8jCuP7xH+4fnadBFBs3OOZssnJj8RAAHDMNpTIPLNDh5oAeuf+Yw/0JjXKTQozS+Dialq4Lno3nNig7f8Pj0gQmsZsdU1DyYcsuA+ltFqWAEv5CtmPr9crBMFPQnjcujbFMNmAZMcuQ1Nlt6VaNJg7jEMmVvWnUj0D3h9bInHuU1K2T6M+4bt8EHdLERU5pn0BwI5QAiN4+T7LL5WmI4o4fRlwwPyv4+XwC5XoIJhekcpw4/u+C58iUm2STAwUZ/FlHeiel94zqieks2Loc1qTHJACWlkWQzGBpkF+lDGjCFC+ValtnXSUa2ICCYfQ2MJyktteHgsdimDkzQ36p6MdWM+BqYfRUxnJJxq9/yO2xYc2CYA3PygC4npbK2hf/ev4yobij9czXXU96zkHzM4UCa0s35lxK6PXoRxyWPRbeUd+qOQTzVTQruMUDxwHOf00Dz4DkvvqI3Mfef5loXXyJ1EabB6UApoV1w8K939KDKkQO+XA2BCLTPBKpbrienruoMTMSjYKrDcX/o+eM1rm7JBvYXwOLLiO5CYPY6oH0iYQ4pdKbm8bjqWHWSz3ceEJrdcZDORFj+XD7E6fiA5DVcpGRDYApEWr4K2L6UUy+knbO6pdikkKCnXFeUZNl8yUTU8oEdkObA4yl2cUohHVcCZe7KFMd7i/tBmaca48TW5uMDjomfvuIxqhxoFBhoJJJ8c5xTJsteUZ6v3G0aJb2Qh+cSqgNmbzz6tUT1wOCg+oY+yXw+uwsJs48TGLEzDrBVT1lmDpMpdvGd50NOjM3SUOFToFRJ5i5LeKMkwDFtCtpKn5XiEKY0VURMnZO6YyelL9Jn7jbCp/RWdhXmShmer9mVx+EyhdB4MoX9WqHYEUTYhrUh3GfJAKVTiWJL72T2YorAZFYCswjTklVUY0B7rlHfeuRaGzny89ueKzQ3rL7wJeWpwjO0yBcyPVdYrWEOyd84JBAmj95RSnIVTMcO0WIXJkDhKlaMtE8U5t+6KVSLIFROoEaEBHIseyqDEUBgb2YGTEGRWXQNg44QI1yj6OctJcqNn3yWZh+mkKDhVMPs2SWJkNJhLX2TauCxmr1D1AI2eSejBMzOwS40dOsnGWrxQC/deFIQTCWQpQ8OvtHpGUhWVO8thrOKfyspRzWtg2odhrMK0h4B0ePJE3PfE9wrCbukJ9TVCrp1ZBlT6BKrzkb2UCoxMYhRElDJkT/7Sk+MpRjDBKxiIVkXEiKEDUBKj1W7AX5RTqAs14P4KcWXNSUx9XQKFyawQwktOzVzNYmfFVAPLaAkpaXWI9QGctMyMAcgyw+CtGg0JaRSQuQakiRvFW3PehEAufZDDJYhQNnjWBiIYeR6RjvJUidg/Oj3U0JsBqNaQRw6Ar2mpszVPZK6ftdjCZDRfAwaUz1J7rOMyVMppEQ8HI8Z1tFPGSOE1ojO/fWJr+kc/Y2psI+W0Pe/8m9/0/L3wPI//PL99lgqj//6gz/E//nqH+JORMwqi8O2wmreYX1+h04Rr1QAACAASURBVE//+D3MP9zQg7it8eR8i8NpgcOmxmgFZ8Ibj3Ix4CdPX+Pz+zM8nMwRnUB90uPJ4oCvPr8gaCsDLl484Mlsj0I6/LtvniPsG4zvOcwuD+h7A+EkuhcC9bM92rsGbR2gNwr6skXczPD85S2+Deco1j0waIwLQOqIorTo3jZYv7fBoSsxokL9pEUIAuNNTUB8Z+Df7xEGhafP73FVnAHgzL2+6OG/buBPLWAlhBXo5w71eYtuXSCOEggC9klE9aqAft6ibVJnQunJdgR6L7u5gVoP+IcfvML//UefIGaWxAuYBwXzyRbPFwd8+ekT/IOfvMKffPoequWATjSIC4cPXtziS3kBtWMf5/Jij/OmxV88dWRfqgBhJfRZB/9qBvFyD/vVDOFihH5dMEH1RYf+roI+7SFlxPB1A18HnL//gNuwhj7rMLytoc4HxP+3vXcPsjU7y/t+73ff19597z73uWokgSRkGwkndmHkSGCrgD9IFYaKSYzjxBUTTDlJQagC56+QOGUHVzlOKOzIcVzYDiE2oRxjIC6oIkYICdB1ZjSa0cy5n773vn3XtfLHu76ve0ZSSfKcOUdnZj1VXd370nuvb+1v773e9Tzv89zssfHOPe7entD2K86DBEQjPIKlW0j0NZ5ktDOlLCOaOuTC2in3ptvUuyW9Ua4s2zKi6UdUEzVcavfFosOIer2mt7ok/OiYxUXjiryG3u0IdgvM5YZymjBYW7I2WHDzM9uYlRqZRa7vSSg3GrL1JfW9PsXFisLCZHPG8Z0R4SwkKIVqpSGoIordCmpRx99LJXYR0ttasDFYcu9zm0yeOGR+us7gHUecvjQhOdIFWLEKxeMFUVZRTVNkEZIeBFz99pd57tmLxCeh9suhsragUoOqAjA9S7IfUq2qE140F5L3HrE46lOYhHrgTAxiy/jZiPxqwfb2CUe/v0lyJDTbJUWZUD+WK5OYhyoPXymxTYBdhkwfDxjcCFheaEi2FyyLEfVIx1Ot1RBYws8lzJ4pSW7FVBNDBYTzgGgp5JdKwjyhnFh9nSTCxJZytyQ4VkmT9hSe9Rb2bkWYp+dUnxlgA5i884DDF9aAgHxdo1qCWnsMlxdU0smHD5gfDclnGVFfmD9RkV2PyXdropOQ5U5D73bIchvk8Tm5HegGwKQmPYyZXbW6iJw0hKOK5sStxAXiSUF+mBEf64aMSlWFamxohobsVkRQaeF6+kwNsSEeltSA/dxQ5bfjtrjTwqHYrpEmQiwsL9aEM5Up926EmEQ3JuoVw+j5kGJNNwSqNdOxYP27ActNKB7PkSZjebVicCtkuWXhnlCsWd2MAn0dktYUS3sty2sF2XMZYoR8Xce2eLzUjNqNCvucSshbBqx/B47e1UAA/ZcjwlIo1i1TGzC/2pCcBhRrQr5hMJk62Q5fCSjWNfg73zSMXgoox7C4UjP+XMRy92yTRQycPgHxidt02tXNmvQIptfUmXn+ZMnq78Ust9XoqnlmhvmDoWZYOgdpk1h6d3VsUgv929qH2vSU4av7tts1aHvgBre06J5f0A3H6DjEJDrW8mqDmIDZFQieDzh6uyX+mDC7ok7V0QLmlyy9O2oeFS20z0/7RY0ydOjm1NofhM6kRjcFqqEWcq1jdDzXjRKTwjSE5FRluPlaQL4Jclvdl6XRrNxiNWS5oxsYNtKNh9HLkAYB+Wqg0Ti1dDmu84taPExeUElvuaKbJb1D7cesRur2HC+0rzYshToVZpf1fFvs6DHt/I7G2iw3hOHN806zAfm69qRKA9mxbkAtNiNmlyFysU8aZ3HWg9k64lb9oJuHYiXQuA80ZqYaBtoeMRQGd5RJrgZnmcD5mvaulsuQcgX6d4V8Qxhe12xY0LaFPA+7PksbCqdXQ6KFdbEuyh6mU0NkDdNLEf09Qz4JKFaFtedqjdOZNcx2Q8bXLVU/ZLmp7qnzi7DzUWF2IXQ5uSp7n11KnCO4ZbkWkcyUdSzHwuCuypLLYUBfdL5Or+rzFqOAfiDkayHDm8bNRUg0b8g3YjVMWmgf6GIzYHQjoJgErk81IjuosKGw3AhJUmXXqr5QroQaZ7URUoxjxtdrirE6Ctf9gGhhCKpEi7VAWG7F9O7B7GLC5Pma08cSVp/PyTe0DzKoDbMLKb099XSoBxHFJKJ/uyBf6zvDokidi/sRSV1pT2o/ou6FBL2I+LTExmHHGFI31P2YMK8JrKWcpMSzinKS0GQBYi3Z3UI3KuKA6DjHpjEmCjSaJA4cS+dMiSapOs7GIU0aEh8tNe5kGVOvZOecdA3NKNWsz4WOKTycY8MQiULqlR5R3WBGmWZqAlgtVluTnmA6x4x7BCcWM8gIZmhBuiywWaJS2tDtXDonWBujt8cRbQYmvTaYOtT/qxvIX81Mds6tvVSltdZqbEjj2M5IXWbb+BDrDHgEcLIu5LzDq5PcShTpYwTBmWkQQOXyTs/lW35Zwqt1rX0T9CZaUHfbNykeacYyvXrJ7vz1HyHs1ZiDFFxIuo0NMqjhNCY9CCmuFRBYNcIpnOxMIBhU2JMEG6uDa7XWkN4Lu1iLthjIN40GycdQDwzpgbKdYa7sUcty6g69k0oFEE2lkxg1mS5cAOqRIZwF55wetfDp3QtY7OrjItC7rU6arTlHUCrTBzB6WXeO8zV1z0yOA/IrJdn1hMblGTY9NSWxkWMbE40qqYZnMrPWLbTJtJ9HZR1nC6Z4KiyeLOl/PmG525AehOTXCuLbCSbVx5tfqxl8MaIa6464GCg2G+2TzCyrnxVOnmyz9JxzZ4pzibRuF1U659FqbOjfVFfBVoYWlMpABoWONXGup638BYvKC3MhPdCFWVjIqxjQoKKLw1CjC2U7+neEfENZUKmUlW0X+wivYsuqoUatgLJZyXHgMuY00zHb19uqkUaJ9O9pD1O5aonmOp42fqNlFFu2oJV3lavKAM+eUAa8lSCJOZOemTZGwx17Kw8rVs8kUkGtDGIb8bLcsvTvSMcapcfqdhk0uF10XcAmp7oojRa6sGxdVotNZVnjmTqQtkYp5VgX3qBOp727uiBqnX7jKY6tcuYpkZqGmFgo1ujiMvQgXewIauQSz3ROsBp9Ug1xZiFnr2XddyzrhmX0koszkTNGLV+zDK8ra9VKolqGPDmBaqxjCHNlrKKFnlfLLX29WilZ3XMMYKDSuXLFxVw4a/1844wxbue3dTtt2VIT6RyHOZ3srhrTyR6TU2XtlInTz5z0SB8nWtruuMqRdLEe8VxZ1+RU3UODykWHOLOayDl5al4iDO4Y5rsBg1uGsLTMLoRd3EWTCsNbDbOLKoWNZ45lrHX+wqUu+EfXK6aXYqqRmqBUw/a94uIt5tbJNXXzIj7VcTbZ2XEHNd1xqWyUjlmbXdTokibVxfrKixUnj6k5Seto2jJ1dU861826L4xuNCzXg+4+UU7HSEnjnG5rSE9Nx1y1hkC9fY0qaSV5UW4oxsqiJFNDsRJ0DrhtfqPm8enY47maxrSup03q3m+lOqrGC0MTi0ZhtKd8owWRSfT/I5cvWA6cvJkzqWO8MJ1BTVDpYwYu21LsWRxKdqwMY5TrZ32Uux5NEZK5MkpLZwYTFfo5kB4b59AbdNEhJhKyk4agtOSrYSfXTKamez+35j7RwrBcDzuWtUmhd2iUTT5qKMehy7NU06HW9RggmalRVr4a0tuvaTJlmZQpM84xNlSGsR+44kN/rOjmRljqXNR9jfRoEi0WgsJ20sTWkMkk2ruZ7VeIhXw9JiwM8UyZPkRZRX19lB2u+gHR0pKc1B0bWo5DssO6iwUpxzFiLPUgJFq0TqKOOY6k+35sZbMqtzXOhCokPSioR7HKNEOhyULCoulY/VbOWffDMymmsc6ExpkTucW3DdWdVjc0IrL9knIlJjmpNDOzMt1j2kBlnsbJPqU5+241kWisSBJ2RVXTi2l6IfFx0TF8bR4lIsqaLtS9Fejkml2mZABSapFpA10vteY3YixBoUY50hiColYJKWjMSD8+ey7rmMbyrDiySYSJA4Ky6eSa0mgEiRQVGCAUmkFyFkXi3ofBsRZw4h7TJi5zUoSgqJSNSxONIEmirmdS8gKCAKlqmvURwfFcC6yywiaxMo9tH2M/U6ayff87eaqNlT0UY8/cX8HJX/WxbRIjixySWK/L20JUi0nqRpnFLD1jJbMzcx7yMwnpq5xd257FIDz7uzXbqWtlf12OpW3lsEVx5hzrzIW0L7M+G1P7XOfiUiQ8++zT21rznnNFY+s0+xXwqLvCjoN1+/7oQ/f9cX+t+oWHfmzwiBeWa2/ftM/87F9gnOa8Z3KDjx9eYbt/yqxK2V8OuTQ85rnDTWaLjA88/jy/ffMxPnjlWb4w2+CZ0V0+cXSZflQyrTJeeH6X7auHfPP6bf7NzWv86avP8Zs3nmSll/PUyh5HZY9ZlfLivXWaOmRn44Sl65+7MDllEJUcFz0uDE742G8/Q3B1zsbKjMYERIHh0uiYT9/dZXM0I68j8jKmakKe3rjHJ29c5ANPPsenD3fJq4iqCVkfLHj52R0uPLXHldERt+Yr9OOSKDA8d2cLEUscN3zLzg1uL1a4Ox0yn2Xsbpxw53DMkzt7PPfiLvGwZOB677b6UwKx3J6PmeUpu+NTosBwnPfYOxkSBJbN8YzjZcbs5pjRxVNW+0sARCxRYHjpzgZyI2P9m/e4+/IakwunHN8b8V3v+RS/v3+Ruy9s8Njbb3NvOiSLa/ZvTBi8FHHtu15ifzHg3r0V+mOVMYSB0R47YHbcozcqWBsuuHl9nbWdEwZJxfUb62xsn1LUIYtFShw35LOk6zs0tcBxwtZT+yRhw407q4SxIeuVLBcpTREyXF2wcP164c2Maq3Wfro85OK1fU7zlNlpD9sI/XHOzsqUL95eJwgtzUGK7TeM1+fMZhnmNObC4/vsHQ/pZRXGCsOsYO8zm+ryGxuyVxJMZOm/64jFp1bhqTnlIma0umB6WzWi8bGygpI1yFFMuLOkqQM216fce3EdmZQENzLq7ZKNzSmn84zwk0OWVyvtwxs0hKchdqvAzGLS9SV1HbK2MufweIg5SghyNRsK0oYkrWk+P6S+ULKxecre9VWi05D+08dEYUP96xucvlOl2sNnjpg9u4q9vMSUIelLKfl2DbElOoqoNypkFhJsFJj9lOxuSPmOJWuTGUef2qDeqMhuJMoirjbKuK4YbGqQrMEWobJvhxn96yGTb7/D7We3MKk64DYDw2T3lMWnV6mvatxOUAjNqNHNI7EMt+YslwncymjWaqhFx7bSIP0aWwVIHmLFEi4DzGZJ9nzG1p+8xaKKOf34hm4UjTVix4YWWS1VhvVS5gx5tLiotiuycYF5dki1ZoiPA6KZUKwbmnFDOskJf39EsaGbQPGJkL9zSXg9U+npabsppZsdoC6n2b2A5dWKeD/qNo/CUjcc8h2V24ebOdHzfZrMSZFTWDxZko4Kyj3VcwaTkmYaM/lUxPE31V2/YjQPCApXcIwMQSFES2F5pULywDHc0knbq8sFnMT0bodU754RfnZIfq0kuRlTbjRE05DkSLo8zmINECg2GuITNQbCQH6xoncj7vJum0ylwbNruhC0kcbI2CcWpJ8YkG849vBSw+B6SLGmmzD5VoNNLIOXIrJ9LZrnV5UlLld04WtD/f4a3AiYPtkweClkua0bT/nTOf3PZhQTy+iLWhgvdzUCaPwFZ5RUQzHRcVqB0SuG02uaBzm7ooxo3dNNjGihhfbpk5bRF/X6Lq6nVGl3u7Ewu2YwPcPaJ5RRbI2xEpdBOX0chl+Ek6etGp09qZJrk+pGyPhFdfYVo0VbPLVdAYtBN9EKy3xXug2geqjS9nbT0ER0GxNhoVLxNu/Uim7U5GswuKny/MUF3ThITpzM2m2g1D3deIsWQv+2xmAcvU3U6Xeu/YynTxtWnlVprYlgdkXo37JkR4bZxbDrnyxHri/V9ScO7mpG52JHmVQbwnIzIMytGgE59jWZWvI13fTJN3SjsWUxm0w3q6RRmX60tCwuCOmhHmvvnqEa6MZHOVKpc1i5DYBGc3O1kBSyfb2tWNFzo7dvXF6nFlhWVHK+9rmaw7dHrD6vWaety3E1EuJTbX9I5lrctxmfTaYbEto/aQgazSltZdTpictkzfVzx8RCb7/h6G0Rky/USKOFd9u/Gy+sc1wNXP+r0DtQeepyLWBwV1lQBEY36q44nV4MmbiNGiuw+nzJ9HJCf68mKA0mDaj6AdlBzXIzJp7rHCSnNcUkYrEZML6uxXSYmy6vcrYTqbGRsZQrEYMbS5bbGTaA9KhmuRXrhmgN6XFNvhYxennJ8dN9Vl5YMruS0dvXIqYehqQHFeVKjHHxLm3vZ3pccXo1Y/yiGg+FuRYtxm0IJEcl9TDWDQI019REwunVmNXnCzUbsnTRKyqvVebRRtLJgFsXWrEQ5g3RrNR4E6s5ovFpSTWMieY1NlGGOZwV1KOUoG57MtUlOb4z7WStpp+obHlRqlPsosAMVaLb9JOu37LrnZwVmH5KMM+xWUwwyzH9TAvNXoIsS3WWXRTYQaZs4qJQ9nGQIfNcj2dtSHC67CJKbBio0ZDLzHQLPZXQArS5nFGoBfVAe2lkOu+YTX1TV0iWYkt3/8Z0LrFdkRmcKyars6Iavkxh+dWKyjdBjuVY1uz7ow/e98f9tfqfPPRjg0e8sFx/+4Zd+ekfAyt84Kln+cS9y6z1Fmz3T3n+aIuVNGdeJZRNyN71VZ586ja3jscMspIkbLhzOCbLKkQs64MF906HfNPObU6KHl+4s0mzDHn7E7dYVAl70wGboznX766S9UvG/Zy9oxHNIuLpx+7w/HMXePptt3hpbw1zfYCNLL2rU+YHfTZ2TziZ9pDAkiQ1cdgwX6asjeecLjKSSM1gev2CPI+xJiCK1cgmCCxNE1AWEYNhzuy4z2iyYDbN6A1KijwmCA1NEzAeLhlnBa88u026uyA/6BHOAux2wXi85PjmmLc9c5MX725QzRI2LxxT1SHTecbKaEEcGuZFwvpgwSvPbjO4PCWLa7aHU146WAOgLGLqaUywDEkvzSiWMaYOyIZqNrScpmTDkmIZc2XnkJevb0AZsHnliMOTAc1JwvbVQ0Qse5/dpJnUKt/NGqI7CdW2bmNLoLmiQVZjlhHxuKCpQkweIbEh7lXUVUj0xYxySwvFIDZEca1mM1ZlmPFKQXWSMt6ZcnpnRH9zznKWYhcRZI0zX4kgMkT9GuPMY8w8JlnNKY9TNWw5b/SSGMgDgnHV3TccVWpAM49gUmLrAIkNUdxQLWLiQUl1nEFgyVZz8uOMeFRQHWdE45K6CAkPY9gp1BxmGnX5qlFaU+cxtgrAQHSi0s+Wsdl82z57hyNdtFvBLkOkCGBSqQzaYXP3hIOjIaYMoQxI9kLKLY0SSe5ElJsurzXVAiDbWJIfZVrMpQ0UoRqouEiTZrvAGiF0+aw2sgS5Fhj1uFEDl16D5CE0QrS1pDpKO9VANA1p+pr5KosQO6jpTXKW05TgKO4MOYzLW7WDhmg/dhEXej2xJb4bq6nIPKDYrLUXOjHER67Xt7XoBzXQcYYt8YmTSe+WWoguVJ4TLYVorgVYdOiK1V5NcC+FQOe9GWgRHJ8EFBuNM5HRBZBJbGfOEpZCsWq0D68SmtWK6CCm2S6JbiU0mSoV6p4uoPNNZcHLVS0E67E+djgPaHpGnX4zZdfPsx42VjZf2RhnZFMqAy9GOtOVNnu1zQ1t+4ul1oW3idzcjusuT7fYaIgWAU1iyfaDTk568nRDNDszwmnz+lp2LSyFamCVnah18R8Urj91rmoOAn0tkpOAOrOdGU411p7fYk2LPhMr8xYupeupa3qWaKa3JVOhmKjKoC04qpEqHuKZdP3qJlb2UKxjfsfak1msufiZQhnl2v1f26/ZpHQ5qCah61MMc1yB2zJTdMWmSZRxr/t6/IFTJRSTM6OdtsddGmXjW/VCNXLFbvDqAqpJXMblawxxopyuN7jNlT3f04vQKUXSQ53LckU69r5VPkQLLeLqnnTKCnGmNW1fvUm077l3T588XCpTHM/UVCVcag5stNTiPZpDa2jUMu6AKxaUSSXQ86ftGbai/dyteU6TnGfElcU2IQxvG8qBY+qcxDoslCVPD7RAjpbKfLdqgyjXAg6r/bvRUrNHi7Ey1eVYC9jOWyDSy22PYmsmpIY+Kk3t+u4TtwnhFAliXO6sW0ufqU/0mIKKjhms+5r7aUI6Sa/mhEo3B1Hh3p+hEDSaj2piXA80r2K4257O1ninjZ1R4x7H8DZnxj1RrkVqey607G6TKtPfKibqXtAx8tgzoyBw74tQz53k1FANA0wI/T0tePV8aShXImd0pKyzifQYbaBsMtapDCLpej5tJISFso91L+x6OZssJDk51yNaW5eVadxYpGNvbaBmOtJoH2t8WmHjQPMzoYsXqUdxxxRLbQnzGhMFXV8mgZONhgFBUasbcdFAbZQN7cWEs5J6JSXM6zMTnqKBUJyja6XZl6Eged3lYLb9l63pjo1DqI1KUutGi8hlqZLYvDzH8p3JWV9lzNNKWONIGc6q1r7QJNaezJaBdMZCNMo6E4gyoC1ilcC+KtvSGJXNtsVhXSuLWRSddLW7zRgkDF/NNp7ro2wluK9FW6NotImO1Vbll9zva8U3SmH5vuDfu++P++vmn37VYxOR7wR+FgiBn7fW/sz9Hkfw1e/yjYvahPT7BePRgtJEfOjS5xglOfeWI7YGM962cpdeXHFwOGS0M+X2yZi3b93l0uiY2gR869WXubJ6xNXVI165tU4cNQyiks3ejA8/8ylGG3MCsWz0ZnzrxVdYy+b0hwUisJot2ZjM+La3f4EwMPS31ZH28a0DmlHDd/zxTzHMCq5e3WOc5bz3ynXqKuTq6hH9pKKflaykOZdXj5kvUuKkpp9UDPsF/X7BZLhgbbign5Y8sblPmlU8vb7H6voUY4WtjVOe3rjHU7v3uLJxRBgaTk77XBoeY4cN/axAioD46pxve+IlAMYXplzsn7AyWvL047e5tnLIe7Zv8vTuPTYHcw5PBjy1vscrd9a48PQeG8M57926zt3ZiAuTU+o6JEkrXeSvlSz3+wxGOSxD1kZzhr0CjKgz7cqCg7nGnEQrJVlU857LN3j3O16mrEPu3Fxl5533oAgY7UwZTRaET8y4fOGQ5GbMhZ0jwkHF+tqMyfaU916+gckjdi4dYo3w1PYevX5J/5uPwAgXdo5IsxIReM9j11lZndPbWJD1StYuHjObZqRr6jRqq4DhzgyMkPYr/sg7XyQeloRRQ9Yr2Vyfap9faJCeOotevbrHzuYJwTLg2972BWRY8y1Xr5NkNReu7WNqQYoQO6zpjwoktLzr6k3qImJt65TJaEm2vuTClQPyacp4c0Yvqxhuz2iOUoLIwG7Ok7v3SF9MwcLW1gnRzZTJSB1CokGFZA3NdoGZVJhBw9Yze9y9vooElicv7PGOK7cJBjXBeslovIQyILuRIIlh8VubGMfgbl4+orxYsbp7yuaFY/rvOiJYBvQuTbn6+D2CZUCxjBltzYhXCqK9RN1LRw2jS6fUGxVhrJqu7F4AKxWPP3UHMbDxzfdI9kJsYgiPYnUnjSyPb++zceWY6MKC/vYcubIguxdy7do9bGKQyLI86NF/PmX3Hfcww5pwZ0mwDLHDhiBtkCsLdRtdK5HUqFvrY3Oa1VozPCclMimVGb2yhJVKnTMvLAgLQVZKxrtTonlAuVUz+qaDrmAfXj1B1guqrYr8qZz+S+qS2hqAIRa7VWAj6F/X4r7YbpTpBJrM0AwbbGKox4b6Qkm+U0NkGbwS0KxXyDzChpbk5RRpYOXJIxYXG4qtmuWlBtM3VGND7+qUetIQTUo15bKoy+lSMD1DPW5oRg31Ss3o6SPNVF2vqC8VOg+F9urWQ8vgvfvUY0O+02ASS/P2me6691AJWik0PYNcXujifNQ4ibZQbDVqCNazWGdmlm82HL+70jxUcPmbGqkBynyaFKrBWc5p685aXqywGyXV2DK6ekJyFJCcaLHcf+YYE6vhV7YfsLiir2k90PYCMdo7jOh9orlQrhrqkZP+XluCwPxaTbFhKbZrkmPt0SwnajpWbNYsr1bOOROyfZhfaYimyoK1svp4LuRPFix2LNVY5fkmtZSr2meqsv4z6bjUai4WuGig5dWKYqvGRJDtaYGpMRJalCy3jRrxrFjqnpp/tQXlcscwuKWbAK1c/7zxVlBBb8+SnOh4Ftcq5heNRmucWuIZzC8ZZWPXXYGU6rjqHkyvwezqmXGUFm9OIhproVAN3LGdWKohzC8bygkuQkdfj+WWFqj5plAPHANcaR7m/ALMd7UfsUlhfsF28v/jd2uV1fScMVJlWey6YnShzHTT06K/Gug4y7H+b7GqvahtK8DpVe1zXG4L8wvC6ROWxbZQZ9D0tMezXNFolmqgvZ2nV4Pudbah3p6vqcS5mAi9PUsxEbJjQ3ZsCQuVorfGR6dP6CZJvmldG4voGLaExa7mlqanhig3TK+obLkc63sjnhuV6uMMj2I1wlpsK2M5u6A9n1HuHjvV4wKVoC83AhZbQde7WQ20UG6dYU2om0vlUIuXcqQ9n9LAYkNlyjaEZKamXMUoIJkb5rsh5VDNg0yk/ah1Jk5OrdeVo4DZhZDlmo5HJd3intsy3w46N+PWqdUGuHzZkKqvfZ/LzZjkpKYcBGR7hZon5dp/ChBPGzWKSoVqEFCNQhbbkZNQu948gWoUQqCv7WI3ZXo1042EeY2JA8qVmGoYUY0imiwg30goJzGHz6Q6hpbZTEPCvCY+yqnGMcV6SriomV1I3XmqRkE20cdsehHL7ZRq5CS9FgiF5U6PepIqY2gsNg277EupGo0fcS60phfp36FAo/cVa7FxQLEzxGQR9aSHfZVhKQAAG5ZJREFU6SdQG8xQ4+xMXx/f9hKCec7yygpm3CO/vIIZZJQXxuRXJphhT5nGKMQOe9h+inWSWrM+1qLSSXDr7Qn17irN5or+7K7RbE/09Rv2taCMo7NCtZfpTxgivUwjSQJXiIIWnuL6KM2rI0S+nDusPcd+2sZ8yU8bb2JbJvSrMJoeXxkiEgJ/B/gu4B3AnxORd9z353mUGcvh0zv2ff/LD7A/G3DqGMH3XL7B5w82WeYxaVqzXCY0eUQQNzx+YZ+X760hgaUuQyS0DPoF80WKbYSVlQWzRUq1jJFFRLhadFEKq6MFR9M+pgl4fHuflw/WKOZJFzWwNplxfNqnyiONK8gawtBQnaQkqznV3R7vevcX+dTHH4PNgiSryfd7iBHSrQV1FdKUIRIazDxm9cIJx8cDAF34CkhkwAjWiMYsLCOCQUWUNJQz1b8nt2OCt80o7vbp7c7Ilwm2EX2MWhfQ2eaS/KAHoSXs1xqT0EYOzGL90h3WhHcSZZQiZR3CWUAzaojHJXUVIvsJpq/sljRCuFbQHKb6v/cSmp2C9AsZH/ru3+X//uw3w16KjTXzMH8qJ85qmjokuJ45ExYXt9BXQ49sklPc6WNDS3Yv6sLn63GjcrBeQ3xXWR87qbB5CKEluxGTX6yQUiMXkoOQ6mKpY+qrWQmhRbKGYD/BbJYkWUVxlBGeRhBo32y+o6xbNHdxAs5xN5rqIsS6qIT0IKTYaHsINC6hXq276AjJGphFRLOAuqfPbVND4Mx66g3XvB4Z5DCh7Rfd/jdw9/04GajRCIvIwqAmeTmlvFoQ3k0IH5vRvDTUPsCBGm+UOzXRUaRFhnP3TQ+F2WO1MmCzgMEtYXbFEM8C8q2aaOqyKAe6CIynumve9Cz1esX4kwmLXYuNz6IIyolh44lDTj++QbQQ5k+XhAcxNtbnbd1rg4sLeHHgHJTpmoyiK3PMi8OOeTGpdS6zmgtqIp13adQhuGXd+rfUDdfElv5t3RVfXNad0MHLUcdQVEPXl9vXRXo0V+ajvlgQv5J2zGK5akmOhGpF+5LHL8L++7RPOCjPGLL0QIugVlKI6AK3XG+YfDri5G0NyZEWgNmBSvKi+VkR14a555cq0jvRmcuvc2wWo69h7Hq6Q8fwSS3kOzXRLHTMmWMnY+v6oXWuooV0PeI2OGOc2j7roFAmaX5RXX/rYcuAaK9x3VP2bnBDyLdUUtrGKxjnHtrKEPN1J+21OOdV7fcevhR0rqvFmuvrTdpecmUI25iGYlXnMJ4qO5OvK7tYD7T/VRqYX7bUw4b+9UgdjXtnURLiXJmbzLm7rmkhZ2K3wE00asLEygyen5NyrK/p8BWVpvZv6UJde4DpIjCqoY4v33T9ynt6stY9XB+oEE9VuqkSO+f+KefkqnnL1urrkO0rK6eSO3W0Vaa0jSXRMXQ9wa43tVjXfmaTKPPZu4tzqtT5bnrKrmWHytbMd9RptZjoeKSG9ERZsWooXb9z3Ve2NjsylENdrDcp7rNd+4XrnhZRYp2kN6ZjKnGSXdDir3/XUPWEaqw99SsvV0wvRl3/Zd3XIrTtF46WltqxiP17hqqvPZgqCXXnnQuvb2LtC12uB/TvmY5dHd7SXMkmhcFtdaFtxxgtrBrcZFq0VX1X6C0sxVjoHap5z2IjcNmXrsCaqFFRkzhm2Wqvbcua28AZDbVxG6l0ShITtlLms77ZJtY4j2qge/rlWEimLuIiOutrLSb6/o7n2pdbZ663d2ZIjmuWWwlVX91rW3ZQI2UaTBqoQ60rOPV/1VEWUfltvNDjCxrnrWAs0aLBJEHH8IWFvlgmko5FzPYKqlFMPQhJjmuqscpFw8Iqg2j1uBHNyg0LZRuze0uafkyTau9sUBq3uRWSHpVdb2or1w0LZejqLHRRHxrpkUy1R7IeRESzStnBNCTZW9IME0wSEJRnPahiIJqVnatsULlCz2ikiNSmk382vYj4RHtGpWgw/VjZyDgkqE3nPtvmQOprFhAuK+phogynQHiypNocEhaNMpmzQh1j+wnBVOWsUhtMEhHkOrYuTsRYdXctyrMCte29bOWoLSNZ1dgs1f7ONFFn2V6qxzNfam9mICp9DZx5D2iupHNwldcayJyvB8pKi7+W/QwCOpdYQJIYWxRIHGPz4lU9krYs1YwnDM+iQZxhjy3LM/YROifYL3GFhS9hNLu/H/G4kbGs2ffJB+774361YxORbwP+urX2Q+7yTwBYa//b+zmOR7qw7D+1a7/3H36YL5ysU9YhJ9M+Ta76nWxYauFXBSTjgs0VzTksy4i61L6DOKkppinJsOTC2glffGUTiQ0b61P2X1hnePWE6VEfmUZII6SXZ4ShYbW/5PorG8rchRacdDA8jmhWa3rjnLKMsPcybKALcWKj7qpWCI4jzIr2hWGFbN0VetD13JnUqDxtXGNr0aIwVslgvLXEfkEX6e39pBZMvyGchZiNEruMNGtuHmumXc8w3JgzO+iT3oopLlTIMlT5nFFXR5yMrVlpCGZhJ+0KC9HCL7SaYzeuWd2ccvzKRHPdxhXspfplmrqcuNq57gb2TL5XaIRLUAn1qCE6CbFXljQnCfFR2B0PgUVK3VUOKiEonPxN0MIwsSrxdZ9BNrLEx2qgZKOzCBTafLxYi5PGFYVWoOkZ7LBRxguQIugKpeQopNxo9DFdBl641DlqVhrtiStdken62aRR5obQEixDoqkWBtmhSvR0AacOiBoBIm7hq4vC5FjHGc/VLMqkWhDbyFKNDOlhSJNoEVGu2M6zKJ5pcdCaO7VSxsDl5pVjSzNoSPe0MM/2gm4uq7EaRbUh7yo/coVXX4sbNZXQjQXNymvllrpgr8bKKIULt3B150vd18LSpCpPbKWA+Zo+VnKsLJhJ0YJupMemi2r9n6Znu9dejHu+EdRDQ3wSdItUabS/rM0KVMmcLvzFOOOsQiWTbc9hNFfTnKA4k3xJu4ittf+sHLmYlbaQcOxJK4Fseu2ctbI/2+UAtkZKxhXYbY9eKwksVyz9uzovyx29TzyH5FjjNKIlnbyvSVTCGJ/qmNNDLTQ6Aydzdgzt72qot5lEiyIxWghGyzN5pXEmtUHlzJ4qPcbRdcN8R81v2miaLpPP6hiLNWF4QyNJWtMdE50ZSklNFzOj2YoaZ6KxHk6SuLBd/9p586CgUmMklcXiYnc0aqOYSFecimMH4cxsKsyVMWuL9NaYpzVMCaqzxatYVxTObJdRWPf0crTUyI/AxW3Y4Ex2CrwqX6/NzGsLjbBw2X+NdXE3Z1LKdhw2pHMflXNywuRUzYWyQ2WUQBf28dx2r5cu/PVYgxp6hw3zLe1jDEvrZJXSvTfaz/F6oGMDx1a6DQigy8rEqow0O7Bd4RLPrBr8jILuPR4vtDgyIV0xdJZ5iCtotIgFZdUWmwHR8mwOw8qqjDXSYtLEahzTJIGLS3ESWHsmXw0LNdEBuvd/VLg8SNd3Wo6CzhCqhZgzA5/WkClaGpXJuhiZ1lk3nhvKUUi+Kgzu6YPUqXSFcyspDZxplxpKafF39oSQHNcUq5EaQQGtSY5Y28lkm8QVfXPtA9TvBks1CNznSBtxoqY9TS/oxlr3AqL5WY9jmzsZ5g3VKCKe1k66qusdjBZsNpKu/zCojB5PFnSFextZYgPtVyzWU2eSpMcVn7qIj8a6TSXrWD+NTAkLQ1AbgmWNSSOVnhp339oVq0lI0BhMGHRjt5F0USrRvKJcSYjmNQTOxGhZY+MAE2qcCkbff+3cmjQkXFQ0/djJZFsJpmZCNsOEcF45s6CzXkrTSlBdPmNrFqTfB0F3OZyXGjnSaJEojVXToMa9vm0mpohe74yDbKrS1daZ1YqcGfNEIZ18tW4dUwP92xhlCts+RVcA2yhUthHUObZuIAqRvNR+SWP18UX0vm00iXN37YyDWofWVuLaoi0267r7n1fLV+2rWcPX/m/TnBWBbWFqTFeYvqrf0jnK6uHJuf/7MnXJ+cL1EZfCisi/BDbegIfOgPNZLD9nrf25c8/7fcB3Wmv/orv8HwDvs9b+lfs5iEe6sFx5Zts++Tf/IttDNaUZRgWzOiWShv3lkDhsWM/mLOqEZ2/u8EevvUwvrGiscHuxwqKKWUlzlrV+Y3/bxkv8ysvvJA4b4tCw1ltwlPeIA8Ol4TF7+ZCijggDQyBWf7CsZ3M+cfMSq6MFq9mSw2WfLNI34sG8z8Zwzt50iLWQLxJWVhakcc3V8REvn67Sjyv2ZgNCsYyygjSquTQ45nOH20yXKWlck5cxIpbJYMmd/RX6g4JeUtEYYa2/5MbhhEtrx9w6HmNMwKifd+PcOx4ShpamEZo6ZLIyp2pC4rAhiRoaExCIJYtq9mcDhlnBzmDKp2/uEojSEWlaa0xHEzDoF5zeGTHamepz9XIOTwdcWj/mped22X1ij9t3VtndOeLuwQqDQU5Vh2RJRT+puPnyOv2NBauDJXcOVgjChtEg5+hwqEH3rnEsSDSkPh6WVCcpwaBCQku/XzDdG7J98Yi711fJ1nKypKJqQpbzBFMHbGxOmS1T8lkCVhitzZnuD8hWCsoiIowa6oMejCpsEdJfW7A47GvxLxaZRgTrJePRgqKKyBfKCEdJQ3mYMdyZsZinhFFDtUh0rEUIVUCwDFh76pD9WytaNE4KZbJPYhjXXNo95PpLm/Q35yyOegzWlvTTkqOTAfWJRqWE0xC5qKuw+jQhXimoDzPijSVJUjM77CPzUJnBlRJTB2ds9jJk/eoR+zcmBMMKM1cWndgy3JirqdE8IjyJMFuF9phaCEYVZqZ9dUEuNJMaKULijSXlNEGKEAykOwvK6wOVRva0T5W5BjPLpETupZj1ClsEyhr3Gqi1p9MkqPnPMuw2QyQPtM+y1I0a6TVEtxLqCyVy7FbTFmxikUozZyV39x3UhHfTjokOl+5LyzlEm54hyIWg1s2RYOEMa1ZqosMIG1maUYOUAeEi0NiRQhfH5Vatr4N7rHq9IjyKlA1LdIMgOgmpJ8pOt4Vtk2kB1xaVTWa7XmJiVR3IUs2Pwpcy6ms5cjclPg3Id2vto50qU1qNz9yJm74ei7rhSlcwR3P3HD2dn7DUMdgQfSzHyrah7kHl+ianuuFjBg3xQeQ2PuwZc7pe0ftiQr5p1D10HuiGQXmWyRqUuuEiNZSbDcmeW6RZXKau7Xr94qkbc37W69mktmMS46m4HkBY7Dr5a4MzIlJJa3oQdBLUlo1vGeRooc+XHkjX89j0nNuzY22bTP+n7msPYzmxDF4RljvOzRt0rjPNzy1WLU3fkroNmfOZkV3frtsACEq3keFcl8NSN0K6DQ2jstnWCbllGcuJsuUtqxrmUE6cI7KLG2mzJquRGh21jGw8OytQyzGdW2x6pMV/UDlWt9bolKAQRte1j1D7B13f5uKMHY1nluW2dD2o6bH2UVYjd34njk02ep2JnOtzpn2PYeHmw7m/6mM7xi4V5yzbFrs6Jy0rq+eJMypqGSxpN7LcAtS4Inl5du60xXc1UAawSemckKOl7fKg26K3yVrHWUvb5xrPLYstNdvJDjWSpUm08A0q9/zuf9qNhWrQMmzOldixp21fpLJyQu9A3YBb12MxkE/aDQMt4KueOvZWfR2DCZXR7B1oUVesKLNqIo1sUUmrGv8kMy2uq74+ZrvBFDkmNV8J6B2pURLQGQi1DGOUGy3Icy2a2w2QJhHihek2Y7KDCpMGlKNQDaBm+n9iNOu0e70icf2sRl15XZZoNQhelUVa9QOSqT5GPNPCOaiUqdP8UTXOCfOGYj0hWhqaNCCe1jRZ2L0Hk+MS48x2gM7pFatFP8KrHWvzuru/dQWhNJZ4VnWMZutWi0GL99qxlFlIkDeOtVZ5aFDUumwJhSCvqYeJutXWBslrlbIGosVpGqlZUNhu/ljtzYyCrpfSRgE0VlnKLD7L1zRoFmb7t3OdlaI8K9iM65EUZSttGzXiGMvuOvc3zbki0c2ZLnbCM5fZJD4r9s7nXrYS1rYIBV4bC2JLHZv2Yn4Z050vI4/9cjhfdD7qheXDgi8svwaIyBR47mGP4y2GDWD/YQ/iLQY/5w8efs4fPPycP3j4OX/w8HP+4OHn/MHjG3XOr1prNx/2IB4GHpQUNrqfD/YQ8NxbdefhYUFEfs/P+YOFn/MHDz/nDx5+zh88/Jw/ePg5f/Dwc/7g4ef8GxIfA54SkceAm8D3Az9wv5/kUS8sPTw8PDw8PDw8PDw8PL4CrLW1iPwV4FfRuJG/b639zP1+Hl9Yenh4eHh4eHh4eHh4vIlhrf0XwL94I5/jkc6xBH7uq9/F4z7Dz/mDh5/zBw8/5w8efs4fPPycP3j4OX/w8HP+4OHn/C2KR9q8x8PDw8PDw8PDw8PDw+Ph41FnLD08PDw8PDw8PDw8PDweMnxh6eHh4eHh4eHh4eHh4fG68MgWliLynSLynIi8ICI//rDH82aBiFwWkX8tIp8Vkc+IyI+669dE5NdE5PPu96q7XkTkb7vX4ZMi8t6HewSPJkQkFJHfF5FfcZcfE5GPunn9JyKSuOtTd/kFd/u1hznuRxUiMhGRXxSRZ0XkcyLybf4cf2MhIj/mPlM+LSK/ICKZP8/vL0Tk74vIPRH59Lnrvu7zWkR+yN3/8yLyQw/jWB4VfIU5/xvus+WTIvJ/icjk3G0/4eb8ORH50Lnr/Zrma8SXm/Nzt/01EbEisuEu+/P8PuArzbmI/Ig71z8jIv/9uev9ef4WxSNZWIpICPwd4LuAdwB/TkTe8XBH9aZBDfw1a+07gPcD/5mb2x8HfsNa+xTwG+4y6GvwlPv5S8DfffBDflPgR4HPnbv83wF/y1r7JHAE/LC7/oeBI3f933L38/j68bPAv7TWPgO8G517f46/QRCRi8B/DvxRa+03oVbn348/z+83PgJ852uu+7rOaxFZA34aeB/wrcBPt8Wox5fFR/jSOf814Juste8Cngd+AsB9l34/8E73P/+T21T0a5qvDx/hS+ccEbkMfBB45dzV/jy/P/gIr5lzEflTwPcA77bWvhP4H9z1/jx/C+ORLCzRD4EXrLUvWmtL4B+jJ7fH64S19ra19hPu7ym64L6Izu8/cHf7B8D3ur+/B/jfrOJ3gImI7D7gYT/SEJFLwJ8Fft5dFuA7gF90d3ntfLevwy8CH3D39/gaISIrwJ8E/h6Atba01h7jz/E3GhHQE5EI6AO38ef5fYW19reAw9dc/fWe1x8Cfs1ae2itPUKLpC9ZxHsovtycW2v/lbW2dhd/B7jk/v4e4B9bawtr7UvAC+h6xq9pvg58hfMcdBPqvwLOu1L68/w+4CvM+V8GfsZaW7j73HPX+/P8LYxHtbC8CFw/d/mGu87jPsLJz74F+Ciwba297W66A2y7v/1r8frxP6JfhsZdXgeOzy1Mzs9pN9/u9hN3f4+vHY8Be8D/Kio//nkRGeDP8TcM1tqb6G72K2hBeQJ8HH+ePwh8vee1P9/vL/4C8P+4v/2cv0EQke8Bblpr//A1N/k5f+PwNPAnXLvCb4rIH3PX+zl/C+NRLSw93mCIyBD4P4G/aq09PX+b1Ywan1NzHyAiHwbuWWs//rDH8hZCBLwX+LvW2m8B5pzJAwF/jt9vOInZ96BF/QVggGcHHjj8ef1gISI/ibaX/KOHPZY3M0SkD/zXwE897LG8xRABa2jb1H8J/FOvLPF4VAvLm8Dlc5cvues87gNEJEaLyn9krf0ld/XdVv7nfreSB/9avD78O8B3i8gXUVnId6D9fxMnGYRXz2k33+72FeDgQQ74TYAbwA1r7Ufd5V9EC01/jr9x+NPAS9baPWttBfwSeu778/yNx9d7Xvvz/T5ARP5D4MPAD9qzwHA/528MnkA3rf7QfZdeAj4hIjv4OX8jcQP4JScz/l1UdbWBn/O3NB7VwvJjwFOijoIJ2iT8yw95TG8KuN2mvwd8zlr7N8/d9MtA65r2Q8A/P3f9n3fOa+8HTs7Jrjy+Cqy1P2GtvWStvYaex/+vtfYHgX8NfJ+722vnu30dvs/d3zMQXwestXeA6yLyNnfVB4DP4s/xNxKvAO8Xkb77jGnn3J/nbzy+3vP6V4EPisiqY5o/6K7z+BohIt+Jtjd8t7V2ce6mXwa+X9T1+DHUUOZ38Wua1wVr7aestVvW2mvuu/QG8F73We/P8zcO/wz4UwAi8jSQAPv48/ytDWvtI/kD/BnUbe0LwE8+7PG8WX6AfxeVSn0S+AP382fQ/qbfAD4P/Dqw5u4vqMvXF4BPoa6PD/04HsUf4NuBX3F/P45+EL8A/B9A6q7P3OUX3O2PP+xxP4o/wHuA33Pn+T8DVv05/obP+X8DPAt8GviHQOrP8/s+x7+A9rBW6OL6h/9tzmu0L/AF9/MfPezj+kb++Qpz/gLaS9Z+h/7P5+7/k27OnwO+69z1fk3zOub8Nbd/Edhwf/vz/A2ac7SQ/N/dZ/ongO84d39/nr9Ff8S90B4eHh4eHh4eHh4eHh4e/1Z4VKWwHh4eHh4eHh4eHh4eHt8g8IWlh4eHh4eHh4eHh4eHx+uCLyw9PDw8PDw8PDw8PDw8Xhd8Yenh4eHh4eHh4eHh4eHxuuALSw8PDw8PDw8PDw8PD4/XBV9Yenh4eHh8w0JE/j/3+5qI/MDDHo+Hh4eHh4fHl4cvLD08PDw8vmFhrf3j7s9rgC8sPTw8PDw8vkHhC0sPDw8Pj29YiMjM/fkzwJ8QkT8QkR8TkVBE/oaIfExEPiki/4m7/7eLyG+KyD8XkRdF5GdE5AdF5HdF5FMi8oS7378vIp8WkT8Ukd96WMfn4eHh4eHxZkH0sAfg4eHh4eHxNeDHgf/CWvthABH5S8CJtfaPiUgK/LaI/Ct333cDbwcOgReBn7fWfquI/CjwI8BfBX4K+JC19qaITB70wXh4eHh4eLzZ4BlLDw8PD49HER8E/ryI/AHwUWAdeMrd9jFr7W1rbQF8AWgLzk+hklqA3wY+IiL/MRA+sFF7eHh4eHi8SeEZSw8PDw+PRxEC/Ii19ldfdaXItwPFuavMucsG971nrf1PReR9wJ8FPi4if8Rae/CGj9rDw8PDw+NNCs9Yenh4eHg8CpgCo3OXfxX4yyISA4jI0yIy+FofTESesNZ+1Fr7U8AecPm+jtbDw8PDw+MtBs9Yenh4eHg8Cvgk0IjIHwIfAX4WlbV+QkQELQ6/9+t4vL8hIk+hzOdvAH94X0fr4eHh4eHxFoNYax/2GDw8PDw8PDw8PDw8PDweYXgprIeHh4eHh4eHh4eHh8frgi8sPTw8PDw8PDw8PDw8PF4XfGHp4eHh4eHh4eHh4eHh8brgC0sPDw8PDw8PDw8PDw+P1wVfWHp4eHh4eHh4eHh4eHi8LvjC0sPDw8PDw8PDw8PDw+N1wReWHh4eHh4eHh4eHh4eHq8L/z8JPtB5VvjafAAAAABJRU5ErkJggg==\",\n      \"text/plain\": [\n       \"<Figure size 1080x1080 with 2 Axes>\"\n      ]\n     },\n     \"metadata\": {}\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"We can observe that some movies tend to be widely recommended or not recommended, whilst some other have more variance in their predicted score\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Neural Network (non-linear) Matrix Factorization\\n\",\n    \"\\n\",\n    \"We don't have to limit ourselves to the weights of the linear embedding layer for our user or item embeddings. We can have a more complex pipeline combining fully connected layers and non-linear activations.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 15,\n   \"source\": [\n    \"class MLPMatrixFactorization(gluon.HybridBlock):\\n\",\n    \"    \\n\",\n    \"    def __init__(self, k, hidden, max_user=max_user, max_item=max_item):\\n\",\n    \"        super(MLPMatrixFactorization, self).__init__()\\n\",\n    \"        \\n\",\n    \"        # user feature lookup\\n\",\n    \"        self.user_embedding = gluon.nn.Embedding(input_dim=max_user, output_dim = k) \\n\",\n    \"        self.user_mlp = gluon.nn.Dense(hidden)\\n\",\n    \"\\n\",\n    \"        # item feature lookup\\n\",\n    \"        self.item_embedding = gluon.nn.Embedding(input_dim=max_item, output_dim = k) \\n\",\n    \"        self.item_mlp = gluon.nn.Dense(hidden)\\n\",\n    \"    \\n\",\n    \"    def forward(self, user, item):\\n\",\n    \"        user_embeddings = self.user_embedding(user)\\n\",\n    \"        user_embeddings_relu = npx.relu(user_embeddings)\\n\",\n    \"        user_transformed = self.user_mlp(user_embeddings_relu)\\n\",\n    \"        \\n\",\n    \"        items_embeddings = self.item_embedding(item)\\n\",\n    \"        items_embeddings_relu = npx.relu(items_embeddings)\\n\",\n    \"        items_transformed = self.item_mlp(items_embeddings_relu)\\n\",\n    \"        \\n\",\n    \"        # predict by the inner product, which is elementwise product and then sum\\n\",\n    \"        pred = (user_transformed * items_transformed).sum(axis=1)\\n\",\n    \"        \\n\",\n    \"        return pred.flatten()\\n\",\n    \"\\n\",\n    \"net2 = MLPMatrixFactorization(64, 64)\\n\",\n    \"net2.initialize(mx.init.Xavier(), ctx=ctx)\\n\",\n    \"mx.viz.plot_network(net2(mx.sym.var('user'), mx.sym.var('item')), node_attrs={\\\"fixedsize\\\":\\\"false\\\"})\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"execute_result\",\n     \"data\": {\n      \"image/svg+xml\": \"<?xml version=\\\"1.0\\\" encoding=\\\"UTF-8\\\" standalone=\\\"no\\\"?>\\n<!DOCTYPE svg PUBLIC \\\"-//W3C//DTD SVG 1.1//EN\\\"\\n \\\"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\\\">\\n<!-- Generated by graphviz version 2.38.0 (20140413.2041)\\n -->\\n<!-- Title: plot Pages: 1 -->\\n<svg width=\\\"348pt\\\" height=\\\"630pt\\\"\\n viewBox=\\\"0.00 0.00 348.00 630.00\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\" xmlns:xlink=\\\"http://www.w3.org/1999/xlink\\\">\\n<g id=\\\"graph0\\\" class=\\\"graph\\\" transform=\\\"scale(1 1) rotate(0) translate(4 626)\\\">\\n<title>plot</title>\\n<polygon fill=\\\"white\\\" stroke=\\\"none\\\" points=\\\"-4,4 -4,-626 344,-626 344,4 -4,4\\\"/>\\n<!-- user -->\\n<g id=\\\"node1\\\" class=\\\"node\\\"><title>user</title>\\n<ellipse fill=\\\"#8dd3c7\\\" stroke=\\\"black\\\" cx=\\\"80\\\" cy=\\\"-29\\\" rx=\\\"47\\\" ry=\\\"29\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"80\\\" y=\\\"-25.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">user</text>\\n</g>\\n<!-- MLP_MF_emb_user_fwd -->\\n<g id=\\\"node2\\\" class=\\\"node\\\"><title>MLP_MF_emb_user_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"160,-152 0,-152 0,-94 160,-94 160,-152\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"80\\\" y=\\\"-119.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">MLP_MF_emb_user_fwd</text>\\n</g>\\n<!-- MLP_MF_emb_user_fwd&#45;&gt;user -->\\n<g id=\\\"edge1\\\" class=\\\"edge\\\"><title>MLP_MF_emb_user_fwd&#45;&gt;user</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M80,-83.7443C80,-75.2043 80,-66.2977 80,-58.2479\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"80,-93.8971 75.5001,-83.897 80,-88.8971 80.0001,-83.8971 80.0001,-83.8971 80.0001,-83.8971 80,-88.8971 84.5001,-83.8971 80,-93.8971 80,-93.8971\\\"/>\\n</g>\\n<!-- MLP_MF_relu0 -->\\n<g id=\\\"node3\\\" class=\\\"node\\\"><title>MLP_MF_relu0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"146.5,-246 39.5,-246 39.5,-188 146.5,-188 146.5,-246\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"93\\\" y=\\\"-213.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">MLP_MF_relu0</text>\\n</g>\\n<!-- MLP_MF_relu0&#45;&gt;MLP_MF_emb_user_fwd -->\\n<g id=\\\"edge2\\\" class=\\\"edge\\\"><title>MLP_MF_relu0&#45;&gt;MLP_MF_emb_user_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M87.5943,-177.744C86.3876,-169.204 85.129,-160.298 83.9915,-152.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"89.0289,-187.897 83.174,-178.625 88.3293,-182.946 87.6297,-177.995 87.6297,-177.995 87.6297,-177.995 88.3293,-182.946 92.0854,-177.366 89.0289,-187.897 89.0289,-187.897\\\"/>\\n</g>\\n<!-- MLP_MF_dense_user_fwd -->\\n<g id=\\\"node4\\\" class=\\\"node\\\"><title>MLP_MF_dense_user_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"145.5,-340 42.5,-340 42.5,-282 145.5,-282 145.5,-340\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"94\\\" y=\\\"-314.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">FullyConnected</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"94\\\" y=\\\"-299.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64</text>\\n</g>\\n<!-- MLP_MF_dense_user_fwd&#45;&gt;MLP_MF_relu0 -->\\n<g id=\\\"edge3\\\" class=\\\"edge\\\"><title>MLP_MF_dense_user_fwd&#45;&gt;MLP_MF_relu0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M93.5842,-271.744C93.4914,-263.204 93.3945,-254.298 93.307,-246.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"93.6945,-281.897 89.086,-271.947 93.6401,-276.897 93.5857,-271.898 93.5857,-271.898 93.5857,-271.898 93.6401,-276.897 98.0855,-271.849 93.6945,-281.897 93.6945,-281.897\\\"/>\\n</g>\\n<!-- item -->\\n<g id=\\\"node5\\\" class=\\\"node\\\"><title>item</title>\\n<ellipse fill=\\\"#8dd3c7\\\" stroke=\\\"black\\\" cx=\\\"259\\\" cy=\\\"-29\\\" rx=\\\"47\\\" ry=\\\"29\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"259\\\" y=\\\"-25.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">item</text>\\n</g>\\n<!-- MLP_MF_emb_item_fwd -->\\n<g id=\\\"node6\\\" class=\\\"node\\\"><title>MLP_MF_emb_item_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"340,-152 178,-152 178,-94 340,-94 340,-152\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"259\\\" y=\\\"-119.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">MLP_MF_emb_item_fwd</text>\\n</g>\\n<!-- MLP_MF_emb_item_fwd&#45;&gt;item -->\\n<g id=\\\"edge4\\\" class=\\\"edge\\\"><title>MLP_MF_emb_item_fwd&#45;&gt;item</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M259,-83.7443C259,-75.2043 259,-66.2977 259,-58.2479\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"259,-93.8971 254.5,-83.897 259,-88.8971 259,-83.8971 259,-83.8971 259,-83.8971 259,-88.8971 263.5,-83.8971 259,-93.8971 259,-93.8971\\\"/>\\n</g>\\n<!-- MLP_MF_relu1 -->\\n<g id=\\\"node7\\\" class=\\\"node\\\"><title>MLP_MF_relu1</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"298.5,-246 191.5,-246 191.5,-188 298.5,-188 298.5,-246\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"245\\\" y=\\\"-213.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">MLP_MF_relu1</text>\\n</g>\\n<!-- MLP_MF_relu1&#45;&gt;MLP_MF_emb_item_fwd -->\\n<g id=\\\"edge5\\\" class=\\\"edge\\\"><title>MLP_MF_relu1&#45;&gt;MLP_MF_emb_item_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M250.822,-177.744C252.121,-169.204 253.476,-160.298 254.701,-152.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"249.277,-187.897 246.332,-177.334 250.029,-182.954 250.781,-178.011 250.781,-178.011 250.781,-178.011 250.029,-182.954 255.23,-178.688 249.277,-187.897 249.277,-187.897\\\"/>\\n</g>\\n<!-- MLP_MF_dense_item_fwd -->\\n<g id=\\\"node8\\\" class=\\\"node\\\"><title>MLP_MF_dense_item_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"268.5,-340 165.5,-340 165.5,-282 268.5,-282 268.5,-340\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"217\\\" y=\\\"-314.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">FullyConnected</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"217\\\" y=\\\"-299.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64</text>\\n</g>\\n<!-- MLP_MF_dense_item_fwd&#45;&gt;MLP_MF_relu1 -->\\n<g id=\\\"edge6\\\" class=\\\"edge\\\"><title>MLP_MF_dense_item_fwd&#45;&gt;MLP_MF_relu1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M228.472,-272.307C231.123,-263.596 233.899,-254.475 236.403,-246.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"225.553,-281.897 224.16,-271.02 227.009,-277.114 228.465,-272.33 228.465,-272.33 228.465,-272.33 227.009,-277.114 232.77,-273.641 225.553,-281.897 225.553,-281.897\\\"/>\\n</g>\\n<!-- MLP_MF__mul0 -->\\n<g id=\\\"node9\\\" class=\\\"node\\\"><title>MLP_MF__mul0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"211,-434 97,-434 97,-376 211,-376 211,-434\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"154\\\" y=\\\"-401.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">MLP_MF__mul0</text>\\n</g>\\n<!-- MLP_MF__mul0&#45;&gt;MLP_MF_dense_user_fwd -->\\n<g id=\\\"edge7\\\" class=\\\"edge\\\"><title>MLP_MF__mul0&#45;&gt;MLP_MF_dense_user_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M130.149,-367.428C124.249,-358.383 118.019,-348.828 112.423,-340.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"135.672,-375.897 126.44,-369.979 132.941,-371.709 130.209,-367.521 130.209,-367.521 130.209,-367.521 132.941,-371.709 133.979,-365.063 135.672,-375.897 135.672,-375.897\\\"/>\\n</g>\\n<!-- MLP_MF__mul0&#45;&gt;MLP_MF_dense_item_fwd -->\\n<g id=\\\"edge8\\\" class=\\\"edge\\\"><title>MLP_MF__mul0&#45;&gt;MLP_MF_dense_item_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M179.044,-367.428C185.238,-358.383 191.78,-348.828 197.656,-340.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"173.244,-375.897 175.182,-365.104 176.069,-371.772 178.894,-367.646 178.894,-367.646 178.894,-367.646 176.069,-371.772 182.607,-370.189 173.244,-375.897 173.244,-375.897\\\"/>\\n</g>\\n<!-- MLP_MF_sum0 -->\\n<g id=\\\"node10\\\" class=\\\"node\\\"><title>MLP_MF_sum0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"208.5,-528 99.5,-528 99.5,-470 208.5,-470 208.5,-528\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"154\\\" y=\\\"-495.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">MLP_MF_sum0</text>\\n</g>\\n<!-- MLP_MF_sum0&#45;&gt;MLP_MF__mul0 -->\\n<g id=\\\"edge9\\\" class=\\\"edge\\\"><title>MLP_MF_sum0&#45;&gt;MLP_MF__mul0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M154,-459.744C154,-451.204 154,-442.298 154,-434.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"154,-469.897 149.5,-459.897 154,-464.897 154,-459.897 154,-459.897 154,-459.897 154,-464.897 158.5,-459.897 154,-469.897 154,-469.897\\\"/>\\n</g>\\n<!-- MLP_MF_flatten0 -->\\n<g id=\\\"node11\\\" class=\\\"node\\\"><title>MLP_MF_flatten0</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"214,-622 94,-622 94,-564 214,-564 214,-622\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"154\\\" y=\\\"-589.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">MLP_MF_flatten0</text>\\n</g>\\n<!-- MLP_MF_flatten0&#45;&gt;MLP_MF_sum0 -->\\n<g id=\\\"edge10\\\" class=\\\"edge\\\"><title>MLP_MF_flatten0&#45;&gt;MLP_MF_sum0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M154,-553.744C154,-545.204 154,-536.298 154,-528.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"154,-563.897 149.5,-553.897 154,-558.897 154,-553.897 154,-553.897 154,-553.897 154,-558.897 158.5,-553.897 154,-563.897 154,-563.897\\\"/>\\n</g>\\n</g>\\n</svg>\\n\",\n      \"text/plain\": [\n       \"<graphviz.dot.Digraph at 0x7f1d5006e978>\"\n      ]\n     },\n     \"metadata\": {},\n     \"execution_count\": 15\n    }\n   ],\n   \"metadata\": {\n    \"collapsed\": false,\n    \"scrolled\": false\n   }\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 16,\n   \"source\": [\n    \"net2.summary(user.to_device(ctx[0]), item.to_device(ctx[0]))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"--------------------------------------------------------------------------------\\n\",\n      \"        Layer (type)                                Output Shape         Param #\\n\",\n      \"================================================================================\\n\",\n      \"               Input                              (128,), (128,)               0\\n\",\n      \"         Embedding-1                                   (128, 64)           60416\\n\",\n      \"             Dense-2                                   (128, 64)            4160\\n\",\n      \"         Embedding-3                                   (128, 64)          107712\\n\",\n      \"             Dense-4                                   (128, 64)            4160\\n\",\n      \"MLPMatrixFactorization-5                                    (128, 1)               0\\n\",\n      \"================================================================================\\n\",\n      \"Parameters in forward computation graph, duplicate included\\n\",\n      \"   Total params: 176448\\n\",\n      \"   Trainable params: 176448\\n\",\n      \"   Non-trainable params: 0\\n\",\n      \"Shared params in forward computation graph: 0\\n\",\n      \"Unique parameters in model: 176448\\n\",\n      \"--------------------------------------------------------------------------------\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 17,\n   \"source\": [\n    \"losses_2 = train(net2, train_data, test_data, epochs=15, ctx=ctx)\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Epoch [0], Training RMSE 1.3127, Test RMSE 0.6534\\n\",\n      \"Epoch [1], Training RMSE 0.6074, Test RMSE 0.6405\\n\",\n      \"Epoch [2], Training RMSE 0.5929, Test RMSE 0.6255\\n\",\n      \"Epoch [3], Training RMSE 0.5789, Test RMSE 0.6122\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stderr\",\n     \"text\": [\n      \"INFO:root:Update[3126]: Change learning rate to 2.00000e-03\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Epoch [4], Training RMSE 0.5650, Test RMSE 0.6006\\n\",\n      \"Epoch [5], Training RMSE 0.5560, Test RMSE 0.5965\\n\",\n      \"Epoch [6], Training RMSE 0.5532, Test RMSE 0.5929\\n\",\n      \"Epoch [7], Training RMSE 0.5504, Test RMSE 0.5914\\n\",\n      \"Epoch [8], Training RMSE 0.5476, Test RMSE 0.5893\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stderr\",\n     \"text\": [\n      \"INFO:root:Update[6251]: Change learning rate to 4.00000e-04\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Epoch [9], Training RMSE 0.5448, Test RMSE 0.5856\\n\",\n      \"Epoch [10], Training RMSE 0.5431, Test RMSE 0.5855\\n\",\n      \"Epoch [11], Training RMSE 0.5425, Test RMSE 0.5856\\n\",\n      \"Epoch [12], Training RMSE 0.5420, Test RMSE 0.5846\\n\",\n      \"Epoch [13], Training RMSE 0.5414, Test RMSE 0.5833\\n\",\n      \"Epoch [14], Training RMSE 0.5409, Test RMSE 0.5822\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {\n    \"collapsed\": false,\n    \"scrolled\": false\n   }\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"We can try training with the Adam optimizer instead\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 18,\n   \"source\": [\n    \"net2 = MLPMatrixFactorization(64, 64)\\n\",\n    \"net2.initialize(mx.init.Xavier(), ctx=ctx)\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 19,\n   \"source\": [\n    \"losses_2_adam  = train(net2, train_data, test_data, epochs=15, optimizer='adam', learning_rate=0.01, ctx=ctx)\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Epoch [0], Training RMSE 0.6292, Test RMSE 0.4896\\n\",\n      \"Epoch [1], Training RMSE 0.4623, Test RMSE 0.4818\\n\",\n      \"Epoch [2], Training RMSE 0.4539, Test RMSE 0.4811\\n\",\n      \"Epoch [3], Training RMSE 0.4486, Test RMSE 0.5017\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stderr\",\n     \"text\": [\n      \"INFO:root:Update[3126]: Change learning rate to 2.00000e-03\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Epoch [4], Training RMSE 0.4462, Test RMSE 0.4950\\n\",\n      \"Epoch [5], Training RMSE 0.4144, Test RMSE 0.4506\\n\",\n      \"Epoch [6], Training RMSE 0.4054, Test RMSE 0.4489\\n\",\n      \"Epoch [7], Training RMSE 0.4026, Test RMSE 0.4497\\n\",\n      \"Epoch [8], Training RMSE 0.4013, Test RMSE 0.4503\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stderr\",\n     \"text\": [\n      \"INFO:root:Update[6251]: Change learning rate to 4.00000e-04\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Epoch [9], Training RMSE 0.3997, Test RMSE 0.4504\\n\",\n      \"Epoch [10], Training RMSE 0.3912, Test RMSE 0.4476\\n\",\n      \"Epoch [11], Training RMSE 0.3898, Test RMSE 0.4466\\n\",\n      \"Epoch [12], Training RMSE 0.3892, Test RMSE 0.4470\\n\",\n      \"Epoch [13], Training RMSE 0.3889, Test RMSE 0.4472\\n\",\n      \"Epoch [14], Training RMSE 0.3885, Test RMSE 0.4458\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Deep Neural Network (Residual Network / ResNet)\\n\",\n    \"Borrowing ideas from [Deep Residual Learning for Image Recognition (He, et al.)](https://arxiv.org/abs/1512.03385) to build a complex deep network that is aggressively regularized, thanks to the dropout layers, to avoid over-fitting, but still achieves good performance. \"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 20,\n   \"source\": [\n    \"def get_residual_block(hidden=64):\\n\",\n    \"    block = gluon.nn.HybridSequential()\\n\",\n    \"    block.add(\\n\",\n    \"        gluon.nn.Dense(hidden, activation='relu'),\\n\",\n    \"        gluon.nn.Dropout(0.5),\\n\",\n    \"        gluon.nn.Dense(hidden)\\n\",\n    \"    )\\n\",\n    \"    return block\\n\",\n    \"    \\n\",\n    \"class ResNetMatrixFactorization(gluon.HybridBlock):\\n\",\n    \"    \\n\",\n    \"    def __init__(self, k, hidden, max_user=max_user, max_item=max_item):\\n\",\n    \"        super(ResNetMatrixFactorization, self).__init__()\\n\",\n    \"        \\n\",\n    \"        # user feature lookup\\n\",\n    \"        self.user_embedding = gluon.nn.Embedding(input_dim=max_user, output_dim = k)\\n\",\n    \"        self.user_block1 = get_residual_block(hidden)\\n\",\n    \"        self.user_dropout = gluon.nn.Dropout(0.5)\\n\",\n    \"        self.user_block2 = get_residual_block(hidden)           \\n\",\n    \"        \\n\",\n    \"        # item feature lookup\\n\",\n    \"        self.item_embedding = gluon.nn.Embedding(input_dim=max_item, output_dim = k)\\n\",\n    \"        self.item_block1 = get_residual_block(hidden)\\n\",\n    \"        self.item_dropout = gluon.nn.Dropout(0.5)\\n\",\n    \"        self.item_block2 = get_residual_block(hidden)           \\n\",\n    \"            \\n\",\n    \"    \\n\",\n    \"    def forward(self, user, item):\\n\",\n    \"        user_embeddings = self.user_embedding(user)\\n\",\n    \"        user_block1 = self.user_block1(user_embeddings)\\n\",\n    \"        user1 = npx.relu(user_embeddings + user_block1)\\n\",\n    \"        \\n\",\n    \"        user2 = self.user_dropout(user1)\\n\",\n    \"        user_block2 = self.user_block2(user2)\\n\",\n    \"        user_transformed = npx.relu(user2 + user_block2)\\n\",\n    \"        \\n\",\n    \"        item_embeddings = self.item_embedding(item)\\n\",\n    \"        item_block1 = self.item_block1(item_embeddings)\\n\",\n    \"        item1 = npx.relu(item_embeddings + item_block1)\\n\",\n    \"        \\n\",\n    \"        item2 = self.item_dropout(item1)\\n\",\n    \"        item_block2 = self.item_block2(item2)\\n\",\n    \"        item_transformed = npx.relu(item2 + item_block2)\\n\",\n    \"        \\n\",\n    \"        # predict by the inner product, which is elementwise product and then sum\\n\",\n    \"        pred = (user_transformed * item_transformed).sum(axis=1)\\n\",\n    \"        \\n\",\n    \"        return pred.flatten()\\n\",\n    \"\\n\",\n    \"net3 = ResNetMatrixFactorization(128, 128)\\n\",\n    \"net3.initialize(mx.init.Xavier(), ctx=ctx)\\n\",\n    \"mx.viz.plot_network(net3(mx.sym.var('user'), mx.sym.var('item')), node_attrs={\\\"fixedsize\\\":\\\"false\\\"})\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"execute_result\",\n     \"data\": {\n      \"image/svg+xml\": \"<?xml version=\\\"1.0\\\" encoding=\\\"UTF-8\\\" standalone=\\\"no\\\"?>\\n<!DOCTYPE svg PUBLIC \\\"-//W3C//DTD SVG 1.1//EN\\\"\\n \\\"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\\\">\\n<!-- Generated by graphviz version 2.38.0 (20140413.2041)\\n -->\\n<!-- Title: plot Pages: 1 -->\\n<svg width=\\\"597pt\\\" height=\\\"1664pt\\\"\\n viewBox=\\\"0.00 0.00 596.50 1664.00\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\" xmlns:xlink=\\\"http://www.w3.org/1999/xlink\\\">\\n<g id=\\\"graph0\\\" class=\\\"graph\\\" transform=\\\"scale(1 1) rotate(0) translate(4 1660)\\\">\\n<title>plot</title>\\n<polygon fill=\\\"white\\\" stroke=\\\"none\\\" points=\\\"-4,4 -4,-1660 592.5,-1660 592.5,4 -4,4\\\"/>\\n<!-- user -->\\n<g id=\\\"node1\\\" class=\\\"node\\\"><title>user</title>\\n<ellipse fill=\\\"#8dd3c7\\\" stroke=\\\"black\\\" cx=\\\"105.5\\\" cy=\\\"-29\\\" rx=\\\"47\\\" ry=\\\"29\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"105.5\\\" y=\\\"-25.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">user</text>\\n</g>\\n<!-- ResNet_MF_emb_user_fwd -->\\n<g id=\\\"node2\\\" class=\\\"node\\\"><title>ResNet_MF_emb_user_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"191,-152 20,-152 20,-94 191,-94 191,-152\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"105.5\\\" y=\\\"-119.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF_emb_user_fwd</text>\\n</g>\\n<!-- ResNet_MF_emb_user_fwd&#45;&gt;user -->\\n<g id=\\\"edge1\\\" class=\\\"edge\\\"><title>ResNet_MF_emb_user_fwd&#45;&gt;user</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M105.5,-83.7443C105.5,-75.2043 105.5,-66.2977 105.5,-58.2479\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"105.5,-93.8971 101,-83.897 105.5,-88.8971 105.5,-83.8971 105.5,-83.8971 105.5,-83.8971 105.5,-88.8971 110,-83.8971 105.5,-93.8971 105.5,-93.8971\\\"/>\\n</g>\\n<!-- ResNet_MF_u_block1_d1_fwd -->\\n<g id=\\\"node3\\\" class=\\\"node\\\"><title>ResNet_MF_u_block1_d1_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"219,-246 116,-246 116,-188 219,-188 219,-246\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"167.5\\\" y=\\\"-220.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">FullyConnected</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"167.5\\\" y=\\\"-205.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- ResNet_MF_u_block1_d1_fwd&#45;&gt;ResNet_MF_emb_user_fwd -->\\n<g id=\\\"edge2\\\" class=\\\"edge\\\"><title>ResNet_MF_u_block1_d1_fwd&#45;&gt;ResNet_MF_emb_user_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M142.854,-179.428C136.758,-170.383 130.319,-160.828 124.537,-152.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"148.561,-187.897 139.241,-182.119 145.767,-183.751 142.972,-179.604 142.972,-179.604 142.972,-179.604 145.767,-183.751 146.704,-177.09 148.561,-187.897 148.561,-187.897\\\"/>\\n</g>\\n<!-- ResNet_MF_u_block1_d1_relu_fwd -->\\n<g id=\\\"node4\\\" class=\\\"node\\\"><title>ResNet_MF_u_block1_d1_relu_fwd</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"227.5,-340 133.5,-340 133.5,-282 227.5,-282 227.5,-340\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"180.5\\\" y=\\\"-314.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"180.5\\\" y=\\\"-299.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- ResNet_MF_u_block1_d1_relu_fwd&#45;&gt;ResNet_MF_u_block1_d1_fwd -->\\n<g id=\\\"edge3\\\" class=\\\"edge\\\"><title>ResNet_MF_u_block1_d1_relu_fwd&#45;&gt;ResNet_MF_u_block1_d1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M175.094,-271.744C173.888,-263.204 172.629,-254.298 171.492,-246.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"176.529,-281.897 170.674,-272.625 175.829,-276.946 175.13,-271.995 175.13,-271.995 175.13,-271.995 175.829,-276.946 179.585,-271.366 176.529,-281.897 176.529,-281.897\\\"/>\\n</g>\\n<!-- ResNet_MF_u_block1_dropout_fwd -->\\n<g id=\\\"node5\\\" class=\\\"node\\\"><title>ResNet_MF_u_block1_dropout_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"292,-434 75,-434 75,-376 292,-376 292,-434\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"183.5\\\" y=\\\"-401.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF_u_block1_dropout_fwd</text>\\n</g>\\n<!-- ResNet_MF_u_block1_dropout_fwd&#45;&gt;ResNet_MF_u_block1_d1_relu_fwd -->\\n<g id=\\\"edge4\\\" class=\\\"edge\\\"><title>ResNet_MF_u_block1_dropout_fwd&#45;&gt;ResNet_MF_u_block1_d1_relu_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M182.253,-365.744C181.974,-357.204 181.684,-348.298 181.421,-340.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"182.584,-375.897 177.76,-366.049 182.421,-370.9 182.258,-365.902 182.258,-365.902 182.258,-365.902 182.421,-370.9 186.755,-365.756 182.584,-375.897 182.584,-375.897\\\"/>\\n</g>\\n<!-- ResNet_MF_u_block1_d2_fwd -->\\n<g id=\\\"node6\\\" class=\\\"node\\\"><title>ResNet_MF_u_block1_d2_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"207,-528 104,-528 104,-470 207,-470 207,-528\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"155.5\\\" y=\\\"-502.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">FullyConnected</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"155.5\\\" y=\\\"-487.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- ResNet_MF_u_block1_d2_fwd&#45;&gt;ResNet_MF_u_block1_dropout_fwd -->\\n<g id=\\\"edge5\\\" class=\\\"edge\\\"><title>ResNet_MF_u_block1_d2_fwd&#45;&gt;ResNet_MF_u_block1_dropout_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M166.972,-460.307C169.623,-451.596 172.399,-442.475 174.903,-434.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"164.053,-469.897 162.66,-459.02 165.509,-465.114 166.965,-460.33 166.965,-460.33 166.965,-460.33 165.509,-465.114 171.27,-461.641 164.053,-469.897 164.053,-469.897\\\"/>\\n</g>\\n<!-- ResNet_MF__plus0 -->\\n<g id=\\\"node7\\\" class=\\\"node\\\"><title>ResNet_MF__plus0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"146.5,-622 20.5,-622 20.5,-564 146.5,-564 146.5,-622\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"83.5\\\" y=\\\"-589.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF__plus0</text>\\n</g>\\n<!-- ResNet_MF__plus0&#45;&gt;ResNet_MF_emb_user_fwd -->\\n<g id=\\\"edge6\\\" class=\\\"edge\\\"><title>ResNet_MF__plus0&#45;&gt;ResNet_MF_emb_user_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M71.412,-554.061C60.6178,-516.778 46.5,-458.064 46.5,-406 46.5,-406 46.5,-406 46.5,-310 46.5,-251.731 73.1335,-187.798 90.6118,-152.115\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"74.3539,-563.985 67.1973,-555.676 72.9328,-559.191 71.5117,-554.397 71.5117,-554.397 71.5117,-554.397 72.9328,-559.191 75.8261,-553.118 74.3539,-563.985 74.3539,-563.985\\\"/>\\n</g>\\n<!-- ResNet_MF__plus0&#45;&gt;ResNet_MF_u_block1_d2_fwd -->\\n<g id=\\\"edge7\\\" class=\\\"edge\\\"><title>ResNet_MF__plus0&#45;&gt;ResNet_MF_u_block1_d2_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M111.684,-555.987C118.892,-546.777 126.539,-537.005 133.393,-528.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"105.494,-563.897 108.113,-553.249 108.575,-559.96 111.657,-556.022 111.657,-556.022 111.657,-556.022 108.575,-559.96 115.201,-558.795 105.494,-563.897 105.494,-563.897\\\"/>\\n</g>\\n<!-- ResNet_MF_relu0 -->\\n<g id=\\\"node8\\\" class=\\\"node\\\"><title>ResNet_MF_relu0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"142.5,-716 24.5,-716 24.5,-658 142.5,-658 142.5,-716\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"83.5\\\" y=\\\"-683.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF_relu0</text>\\n</g>\\n<!-- ResNet_MF_relu0&#45;&gt;ResNet_MF__plus0 -->\\n<g id=\\\"edge8\\\" class=\\\"edge\\\"><title>ResNet_MF_relu0&#45;&gt;ResNet_MF__plus0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M83.5,-647.744C83.5,-639.204 83.5,-630.298 83.5,-622.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"83.5,-657.897 79.0001,-647.897 83.5,-652.897 83.5001,-647.897 83.5001,-647.897 83.5001,-647.897 83.5,-652.897 88.0001,-647.897 83.5,-657.897 83.5,-657.897\\\"/>\\n</g>\\n<!-- ResNet_MF_dropout0_fwd -->\\n<g id=\\\"node9\\\" class=\\\"node\\\"><title>ResNet_MF_dropout0_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"167,-810 1.42109e-14,-810 1.42109e-14,-752 167,-752 167,-810\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"83.5\\\" y=\\\"-777.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF_dropout0_fwd</text>\\n</g>\\n<!-- ResNet_MF_dropout0_fwd&#45;&gt;ResNet_MF_relu0 -->\\n<g id=\\\"edge9\\\" class=\\\"edge\\\"><title>ResNet_MF_dropout0_fwd&#45;&gt;ResNet_MF_relu0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M83.5,-741.744C83.5,-733.204 83.5,-724.298 83.5,-716.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"83.5,-751.897 79.0001,-741.897 83.5,-746.897 83.5001,-741.897 83.5001,-741.897 83.5001,-741.897 83.5,-746.897 88.0001,-741.897 83.5,-751.897 83.5,-751.897\\\"/>\\n</g>\\n<!-- ResNet_MF_u_block2_d1_fwd -->\\n<g id=\\\"node10\\\" class=\\\"node\\\"><title>ResNet_MF_u_block2_d1_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"198,-904 95,-904 95,-846 198,-846 198,-904\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"146.5\\\" y=\\\"-878.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">FullyConnected</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"146.5\\\" y=\\\"-863.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- ResNet_MF_u_block2_d1_fwd&#45;&gt;ResNet_MF_dropout0_fwd -->\\n<g id=\\\"edge10\\\" class=\\\"edge\\\"><title>ResNet_MF_u_block2_d1_fwd&#45;&gt;ResNet_MF_dropout0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M121.456,-837.428C115.262,-828.383 108.72,-818.828 102.844,-810.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"127.256,-845.897 117.893,-840.189 124.431,-841.772 121.606,-837.646 121.606,-837.646 121.606,-837.646 124.431,-841.772 125.318,-835.104 127.256,-845.897 127.256,-845.897\\\"/>\\n</g>\\n<!-- ResNet_MF_u_block2_d1_relu_fwd -->\\n<g id=\\\"node11\\\" class=\\\"node\\\"><title>ResNet_MF_u_block2_d1_relu_fwd</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"221.5,-998 127.5,-998 127.5,-940 221.5,-940 221.5,-998\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"174.5\\\" y=\\\"-972.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"174.5\\\" y=\\\"-957.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- ResNet_MF_u_block2_d1_relu_fwd&#45;&gt;ResNet_MF_u_block2_d1_fwd -->\\n<g id=\\\"edge11\\\" class=\\\"edge\\\"><title>ResNet_MF_u_block2_d1_relu_fwd&#45;&gt;ResNet_MF_u_block2_d1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M163.028,-930.307C160.377,-921.596 157.601,-912.475 155.097,-904.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"165.947,-939.897 158.73,-931.641 164.491,-935.114 163.035,-930.33 163.035,-930.33 163.035,-930.33 164.491,-935.114 167.34,-929.02 165.947,-939.897 165.947,-939.897\\\"/>\\n</g>\\n<!-- ResNet_MF_u_block2_dropout_fwd -->\\n<g id=\\\"node12\\\" class=\\\"node\\\"><title>ResNet_MF_u_block2_dropout_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"297,-1092 80,-1092 80,-1034 297,-1034 297,-1092\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"188.5\\\" y=\\\"-1059.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF_u_block2_dropout_fwd</text>\\n</g>\\n<!-- ResNet_MF_u_block2_dropout_fwd&#45;&gt;ResNet_MF_u_block2_d1_relu_fwd -->\\n<g id=\\\"edge12\\\" class=\\\"edge\\\"><title>ResNet_MF_u_block2_dropout_fwd&#45;&gt;ResNet_MF_u_block2_d1_relu_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M182.678,-1023.74C181.379,-1015.2 180.024,-1006.3 178.799,-998.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"184.223,-1033.9 178.27,-1024.69 183.471,-1028.95 182.719,-1024.01 182.719,-1024.01 182.719,-1024.01 183.471,-1028.95 187.168,-1023.33 184.223,-1033.9 184.223,-1033.9\\\"/>\\n</g>\\n<!-- ResNet_MF_u_block2_d2_fwd -->\\n<g id=\\\"node13\\\" class=\\\"node\\\"><title>ResNet_MF_u_block2_d2_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"238,-1186 135,-1186 135,-1128 238,-1128 238,-1186\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"186.5\\\" y=\\\"-1160.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">FullyConnected</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"186.5\\\" y=\\\"-1145.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- ResNet_MF_u_block2_d2_fwd&#45;&gt;ResNet_MF_u_block2_dropout_fwd -->\\n<g id=\\\"edge13\\\" class=\\\"edge\\\"><title>ResNet_MF_u_block2_d2_fwd&#45;&gt;ResNet_MF_u_block2_dropout_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M187.332,-1117.74C187.517,-1109.2 187.711,-1100.3 187.886,-1092.25\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"187.111,-1127.9 182.829,-1117.8 187.22,-1122.9 187.328,-1117.9 187.328,-1117.9 187.328,-1117.9 187.22,-1122.9 191.827,-1118 187.111,-1127.9 187.111,-1127.9\\\"/>\\n</g>\\n<!-- ResNet_MF__plus1 -->\\n<g id=\\\"node14\\\" class=\\\"node\\\"><title>ResNet_MF__plus1</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"248.5,-1280 122.5,-1280 122.5,-1222 248.5,-1222 248.5,-1280\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"185.5\\\" y=\\\"-1247.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF__plus1</text>\\n</g>\\n<!-- ResNet_MF__plus1&#45;&gt;ResNet_MF_dropout0_fwd -->\\n<g id=\\\"edge14\\\" class=\\\"edge\\\"><title>ResNet_MF__plus1&#45;&gt;ResNet_MF_dropout0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M139.717,-1215.09C100.905,-1181.18 51.5,-1125.86 51.5,-1064 51.5,-1064 51.5,-1064 51.5,-968 51.5,-911.315 66.1138,-846.301 75.5899,-810.102\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"147.817,-1221.99 137.288,-1218.93 144.012,-1218.75 140.207,-1215.5 140.207,-1215.5 140.207,-1215.5 144.012,-1218.75 143.127,-1212.08 147.817,-1221.99 147.817,-1221.99\\\"/>\\n</g>\\n<!-- ResNet_MF__plus1&#45;&gt;ResNet_MF_u_block2_d2_fwd -->\\n<g id=\\\"edge15\\\" class=\\\"edge\\\"><title>ResNet_MF__plus1&#45;&gt;ResNet_MF_u_block2_d2_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M185.916,-1211.74C186.009,-1203.2 186.105,-1194.3 186.193,-1186.25\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"185.805,-1221.9 181.415,-1211.85 185.86,-1216.9 185.914,-1211.9 185.914,-1211.9 185.914,-1211.9 185.86,-1216.9 190.414,-1211.95 185.805,-1221.9 185.805,-1221.9\\\"/>\\n</g>\\n<!-- ResNet_MF_relu1 -->\\n<g id=\\\"node15\\\" class=\\\"node\\\"><title>ResNet_MF_relu1</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"246.5,-1374 128.5,-1374 128.5,-1316 246.5,-1316 246.5,-1374\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"187.5\\\" y=\\\"-1341.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF_relu1</text>\\n</g>\\n<!-- ResNet_MF_relu1&#45;&gt;ResNet_MF__plus1 -->\\n<g id=\\\"edge16\\\" class=\\\"edge\\\"><title>ResNet_MF_relu1&#45;&gt;ResNet_MF__plus1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M186.668,-1305.74C186.483,-1297.2 186.289,-1288.3 186.114,-1280.25\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"186.889,-1315.9 182.173,-1306 186.78,-1310.9 186.672,-1305.9 186.672,-1305.9 186.672,-1305.9 186.78,-1310.9 191.171,-1305.8 186.889,-1315.9 186.889,-1315.9\\\"/>\\n</g>\\n<!-- item -->\\n<g id=\\\"node16\\\" class=\\\"node\\\"><title>item</title>\\n<ellipse fill=\\\"#8dd3c7\\\" stroke=\\\"black\\\" cx=\\\"353.5\\\" cy=\\\"-29\\\" rx=\\\"47\\\" ry=\\\"29\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"353.5\\\" y=\\\"-25.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">item</text>\\n</g>\\n<!-- ResNet_MF_emb_item_fwd -->\\n<g id=\\\"node17\\\" class=\\\"node\\\"><title>ResNet_MF_emb_item_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"440,-152 267,-152 267,-94 440,-94 440,-152\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"353.5\\\" y=\\\"-119.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF_emb_item_fwd</text>\\n</g>\\n<!-- ResNet_MF_emb_item_fwd&#45;&gt;item -->\\n<g id=\\\"edge17\\\" class=\\\"edge\\\"><title>ResNet_MF_emb_item_fwd&#45;&gt;item</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M353.5,-83.7443C353.5,-75.2043 353.5,-66.2977 353.5,-58.2479\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"353.5,-93.8971 349,-83.897 353.5,-88.8971 353.5,-83.8971 353.5,-83.8971 353.5,-83.8971 353.5,-88.8971 358,-83.8971 353.5,-93.8971 353.5,-93.8971\\\"/>\\n</g>\\n<!-- ResNet_MF_i_block1_d1_fwd -->\\n<g id=\\\"node18\\\" class=\\\"node\\\"><title>ResNet_MF_i_block1_d1_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"492,-246 389,-246 389,-188 492,-188 492,-246\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"440.5\\\" y=\\\"-220.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">FullyConnected</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"440.5\\\" y=\\\"-205.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- ResNet_MF_i_block1_d1_fwd&#45;&gt;ResNet_MF_emb_item_fwd -->\\n<g id=\\\"edge18\\\" class=\\\"edge\\\"><title>ResNet_MF_i_block1_d1_fwd&#45;&gt;ResNet_MF_emb_item_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M406.972,-180.545C398.108,-171.172 388.662,-161.182 380.213,-152.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"413.924,-187.897 403.784,-183.723 410.489,-184.264 407.053,-180.631 407.053,-180.631 407.053,-180.631 410.489,-184.264 410.323,-177.539 413.924,-187.897 413.924,-187.897\\\"/>\\n</g>\\n<!-- ResNet_MF_i_block1_d1_relu_fwd -->\\n<g id=\\\"node19\\\" class=\\\"node\\\"><title>ResNet_MF_i_block1_d1_relu_fwd</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"514.5,-340 420.5,-340 420.5,-282 514.5,-282 514.5,-340\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"467.5\\\" y=\\\"-314.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"467.5\\\" y=\\\"-299.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- ResNet_MF_i_block1_d1_relu_fwd&#45;&gt;ResNet_MF_i_block1_d1_fwd -->\\n<g id=\\\"edge19\\\" class=\\\"edge\\\"><title>ResNet_MF_i_block1_d1_relu_fwd&#45;&gt;ResNet_MF_i_block1_d1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M456.355,-272.026C453.824,-263.4 451.179,-254.386 448.79,-246.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"459.252,-281.897 452.118,-273.569 457.844,-277.099 456.436,-272.302 456.436,-272.302 456.436,-272.302 457.844,-277.099 460.754,-271.035 459.252,-281.897 459.252,-281.897\\\"/>\\n</g>\\n<!-- ResNet_MF_i_block1_dropout_fwd -->\\n<g id=\\\"node20\\\" class=\\\"node\\\"><title>ResNet_MF_i_block1_dropout_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"588.5,-434 374.5,-434 374.5,-376 588.5,-376 588.5,-434\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"481.5\\\" y=\\\"-401.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF_i_block1_dropout_fwd</text>\\n</g>\\n<!-- ResNet_MF_i_block1_dropout_fwd&#45;&gt;ResNet_MF_i_block1_d1_relu_fwd -->\\n<g id=\\\"edge20\\\" class=\\\"edge\\\"><title>ResNet_MF_i_block1_dropout_fwd&#45;&gt;ResNet_MF_i_block1_d1_relu_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M475.678,-365.744C474.379,-357.204 473.024,-348.298 471.799,-340.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"477.223,-375.897 471.27,-366.688 476.471,-370.954 475.719,-366.011 475.719,-366.011 475.719,-366.011 476.471,-370.954 480.168,-365.334 477.223,-375.897 477.223,-375.897\\\"/>\\n</g>\\n<!-- ResNet_MF_i_block1_d2_fwd -->\\n<g id=\\\"node21\\\" class=\\\"node\\\"><title>ResNet_MF_i_block1_d2_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"505,-528 402,-528 402,-470 505,-470 505,-528\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"453.5\\\" y=\\\"-502.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">FullyConnected</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"453.5\\\" y=\\\"-487.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- ResNet_MF_i_block1_d2_fwd&#45;&gt;ResNet_MF_i_block1_dropout_fwd -->\\n<g id=\\\"edge21\\\" class=\\\"edge\\\"><title>ResNet_MF_i_block1_d2_fwd&#45;&gt;ResNet_MF_i_block1_dropout_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M464.972,-460.307C467.623,-451.596 470.399,-442.475 472.903,-434.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"462.053,-469.897 460.66,-459.02 463.509,-465.114 464.965,-460.33 464.965,-460.33 464.965,-460.33 463.509,-465.114 469.27,-461.641 462.053,-469.897 462.053,-469.897\\\"/>\\n</g>\\n<!-- ResNet_MF__plus2 -->\\n<g id=\\\"node22\\\" class=\\\"node\\\"><title>ResNet_MF__plus2</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"429.5,-622 303.5,-622 303.5,-564 429.5,-564 429.5,-622\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"366.5\\\" y=\\\"-589.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF__plus2</text>\\n</g>\\n<!-- ResNet_MF__plus2&#45;&gt;ResNet_MF_emb_item_fwd -->\\n<g id=\\\"edge22\\\" class=\\\"edge\\\"><title>ResNet_MF__plus2&#45;&gt;ResNet_MF_emb_item_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M359.966,-553.751C354.131,-516.238 346.5,-457.358 346.5,-406 346.5,-406 346.5,-406 346.5,-310 346.5,-253.934 349.715,-188.43 351.788,-152.033\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"361.556,-563.741 355.54,-554.573 360.77,-558.803 359.984,-553.865 359.984,-553.865 359.984,-553.865 360.77,-558.803 364.428,-553.158 361.556,-563.741 361.556,-563.741\\\"/>\\n</g>\\n<!-- ResNet_MF__plus2&#45;&gt;ResNet_MF_i_block1_d2_fwd -->\\n<g id=\\\"edge23\\\" class=\\\"edge\\\"><title>ResNet_MF__plus2&#45;&gt;ResNet_MF_i_block1_d2_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M400.028,-556.545C408.892,-547.172 418.338,-537.182 426.787,-528.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"393.076,-563.897 396.677,-553.539 396.511,-560.264 399.947,-556.631 399.947,-556.631 399.947,-556.631 396.511,-560.264 403.216,-559.723 393.076,-563.897 393.076,-563.897\\\"/>\\n</g>\\n<!-- ResNet_MF_relu2 -->\\n<g id=\\\"node23\\\" class=\\\"node\\\"><title>ResNet_MF_relu2</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"425.5,-716 307.5,-716 307.5,-658 425.5,-658 425.5,-716\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"366.5\\\" y=\\\"-683.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF_relu2</text>\\n</g>\\n<!-- ResNet_MF_relu2&#45;&gt;ResNet_MF__plus2 -->\\n<g id=\\\"edge24\\\" class=\\\"edge\\\"><title>ResNet_MF_relu2&#45;&gt;ResNet_MF__plus2</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M366.5,-647.744C366.5,-639.204 366.5,-630.298 366.5,-622.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"366.5,-657.897 362,-647.897 366.5,-652.897 366.5,-647.897 366.5,-647.897 366.5,-647.897 366.5,-652.897 371,-647.897 366.5,-657.897 366.5,-657.897\\\"/>\\n</g>\\n<!-- ResNet_MF_dropout1_fwd -->\\n<g id=\\\"node24\\\" class=\\\"node\\\"><title>ResNet_MF_dropout1_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"450,-810 283,-810 283,-752 450,-752 450,-810\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"366.5\\\" y=\\\"-777.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF_dropout1_fwd</text>\\n</g>\\n<!-- ResNet_MF_dropout1_fwd&#45;&gt;ResNet_MF_relu2 -->\\n<g id=\\\"edge25\\\" class=\\\"edge\\\"><title>ResNet_MF_dropout1_fwd&#45;&gt;ResNet_MF_relu2</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M366.5,-741.744C366.5,-733.204 366.5,-724.298 366.5,-716.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"366.5,-751.897 362,-741.897 366.5,-746.897 366.5,-741.897 366.5,-741.897 366.5,-741.897 366.5,-746.897 371,-741.897 366.5,-751.897 366.5,-751.897\\\"/>\\n</g>\\n<!-- ResNet_MF_i_block2_d1_fwd -->\\n<g id=\\\"node25\\\" class=\\\"node\\\"><title>ResNet_MF_i_block2_d1_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"472,-904 369,-904 369,-846 472,-846 472,-904\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"420.5\\\" y=\\\"-878.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">FullyConnected</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"420.5\\\" y=\\\"-863.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- ResNet_MF_i_block2_d1_fwd&#45;&gt;ResNet_MF_dropout1_fwd -->\\n<g id=\\\"edge26\\\" class=\\\"edge\\\"><title>ResNet_MF_i_block2_d1_fwd&#45;&gt;ResNet_MF_dropout1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M398.87,-837.148C393.609,-828.186 388.065,-818.74 383.08,-810.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"404.005,-845.897 395.062,-839.551 401.474,-841.585 398.943,-837.273 398.943,-837.273 398.943,-837.273 401.474,-841.585 402.824,-834.995 404.005,-845.897 404.005,-845.897\\\"/>\\n</g>\\n<!-- ResNet_MF_i_block2_d1_relu_fwd -->\\n<g id=\\\"node26\\\" class=\\\"node\\\"><title>ResNet_MF_i_block2_d1_relu_fwd</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"494.5,-998 400.5,-998 400.5,-940 494.5,-940 494.5,-998\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"447.5\\\" y=\\\"-972.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"447.5\\\" y=\\\"-957.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- ResNet_MF_i_block2_d1_relu_fwd&#45;&gt;ResNet_MF_i_block2_d1_fwd -->\\n<g id=\\\"edge27\\\" class=\\\"edge\\\"><title>ResNet_MF_i_block2_d1_relu_fwd&#45;&gt;ResNet_MF_i_block2_d1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M436.355,-930.026C433.824,-921.4 431.179,-912.386 428.79,-904.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"439.252,-939.897 432.118,-931.569 437.844,-935.099 436.436,-930.302 436.436,-930.302 436.436,-930.302 437.844,-935.099 440.754,-929.035 439.252,-939.897 439.252,-939.897\\\"/>\\n</g>\\n<!-- ResNet_MF_i_block2_dropout_fwd -->\\n<g id=\\\"node27\\\" class=\\\"node\\\"><title>ResNet_MF_i_block2_dropout_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"568.5,-1092 354.5,-1092 354.5,-1034 568.5,-1034 568.5,-1092\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"461.5\\\" y=\\\"-1059.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF_i_block2_dropout_fwd</text>\\n</g>\\n<!-- ResNet_MF_i_block2_dropout_fwd&#45;&gt;ResNet_MF_i_block2_d1_relu_fwd -->\\n<g id=\\\"edge28\\\" class=\\\"edge\\\"><title>ResNet_MF_i_block2_dropout_fwd&#45;&gt;ResNet_MF_i_block2_d1_relu_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M455.678,-1023.74C454.379,-1015.2 453.024,-1006.3 451.799,-998.248\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"457.223,-1033.9 451.27,-1024.69 456.471,-1028.95 455.719,-1024.01 455.719,-1024.01 455.719,-1024.01 456.471,-1028.95 460.168,-1023.33 457.223,-1033.9 457.223,-1033.9\\\"/>\\n</g>\\n<!-- ResNet_MF_i_block2_d2_fwd -->\\n<g id=\\\"node28\\\" class=\\\"node\\\"><title>ResNet_MF_i_block2_d2_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"485,-1186 382,-1186 382,-1128 485,-1128 485,-1186\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"433.5\\\" y=\\\"-1160.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">FullyConnected</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"433.5\\\" y=\\\"-1145.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- ResNet_MF_i_block2_d2_fwd&#45;&gt;ResNet_MF_i_block2_dropout_fwd -->\\n<g id=\\\"edge29\\\" class=\\\"edge\\\"><title>ResNet_MF_i_block2_d2_fwd&#45;&gt;ResNet_MF_i_block2_dropout_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M444.972,-1118.31C447.623,-1109.6 450.399,-1100.47 452.903,-1092.25\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"442.053,-1127.9 440.66,-1117.02 443.509,-1123.11 444.965,-1118.33 444.965,-1118.33 444.965,-1118.33 443.509,-1123.11 449.27,-1119.64 442.053,-1127.9 442.053,-1127.9\\\"/>\\n</g>\\n<!-- ResNet_MF__plus3 -->\\n<g id=\\\"node29\\\" class=\\\"node\\\"><title>ResNet_MF__plus3</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"392.5,-1280 266.5,-1280 266.5,-1222 392.5,-1222 392.5,-1280\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"329.5\\\" y=\\\"-1247.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF__plus3</text>\\n</g>\\n<!-- ResNet_MF__plus3&#45;&gt;ResNet_MF_dropout1_fwd -->\\n<g id=\\\"edge30\\\" class=\\\"edge\\\"><title>ResNet_MF__plus3&#45;&gt;ResNet_MF_dropout1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M328.527,-1211.95C327.651,-1174.36 326.5,-1115.22 326.5,-1064 326.5,-1064 326.5,-1064 326.5,-968 326.5,-911.023 344.662,-846.379 356.509,-810.271\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"328.766,-1221.96 324.029,-1212.07 328.647,-1216.96 328.528,-1211.96 328.528,-1211.96 328.528,-1211.96 328.647,-1216.96 333.026,-1211.85 328.766,-1221.96 328.766,-1221.96\\\"/>\\n</g>\\n<!-- ResNet_MF__plus3&#45;&gt;ResNet_MF_i_block2_d2_fwd -->\\n<g id=\\\"edge31\\\" class=\\\"edge\\\"><title>ResNet_MF__plus3&#45;&gt;ResNet_MF_i_block2_d2_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M368.951,-1215.1C379.727,-1205.57 391.268,-1195.36 401.568,-1186.25\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"361.269,-1221.9 365.777,-1211.9 365.014,-1218.58 368.759,-1215.27 368.759,-1215.27 368.759,-1215.27 365.014,-1218.58 371.74,-1218.64 361.269,-1221.9 361.269,-1221.9\\\"/>\\n</g>\\n<!-- ResNet_MF_relu3 -->\\n<g id=\\\"node30\\\" class=\\\"node\\\"><title>ResNet_MF_relu3</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"386.5,-1374 268.5,-1374 268.5,-1316 386.5,-1316 386.5,-1374\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"327.5\\\" y=\\\"-1341.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF_relu3</text>\\n</g>\\n<!-- ResNet_MF_relu3&#45;&gt;ResNet_MF__plus3 -->\\n<g id=\\\"edge32\\\" class=\\\"edge\\\"><title>ResNet_MF_relu3&#45;&gt;ResNet_MF__plus3</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M328.332,-1305.74C328.517,-1297.2 328.711,-1288.3 328.886,-1280.25\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"328.111,-1315.9 323.829,-1305.8 328.22,-1310.9 328.328,-1305.9 328.328,-1305.9 328.328,-1305.9 328.22,-1310.9 332.827,-1306 328.111,-1315.9 328.111,-1315.9\\\"/>\\n</g>\\n<!-- ResNet_MF__mul0 -->\\n<g id=\\\"node31\\\" class=\\\"node\\\"><title>ResNet_MF__mul0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"318,-1468 193,-1468 193,-1410 318,-1410 318,-1468\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"255.5\\\" y=\\\"-1435.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF__mul0</text>\\n</g>\\n<!-- ResNet_MF__mul0&#45;&gt;ResNet_MF_relu1 -->\\n<g id=\\\"edge33\\\" class=\\\"edge\\\"><title>ResNet_MF__mul0&#45;&gt;ResNet_MF_relu1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M228.675,-1401.71C221.928,-1392.58 214.786,-1382.92 208.379,-1374.25\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"234.728,-1409.9 225.166,-1404.53 231.756,-1405.88 228.784,-1401.86 228.784,-1401.86 228.784,-1401.86 231.756,-1405.88 232.403,-1399.18 234.728,-1409.9 234.728,-1409.9\\\"/>\\n</g>\\n<!-- ResNet_MF__mul0&#45;&gt;ResNet_MF_relu3 -->\\n<g id=\\\"edge34\\\" class=\\\"edge\\\"><title>ResNet_MF__mul0&#45;&gt;ResNet_MF_relu3</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M283.684,-1401.99C290.892,-1392.78 298.539,-1383.01 305.393,-1374.25\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"277.494,-1409.9 280.113,-1399.25 280.575,-1405.96 283.657,-1402.02 283.657,-1402.02 283.657,-1402.02 280.575,-1405.96 287.201,-1404.8 277.494,-1409.9 277.494,-1409.9\\\"/>\\n</g>\\n<!-- ResNet_MF_sum0 -->\\n<g id=\\\"node32\\\" class=\\\"node\\\"><title>ResNet_MF_sum0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"315.5,-1562 195.5,-1562 195.5,-1504 315.5,-1504 315.5,-1562\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"255.5\\\" y=\\\"-1529.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF_sum0</text>\\n</g>\\n<!-- ResNet_MF_sum0&#45;&gt;ResNet_MF__mul0 -->\\n<g id=\\\"edge35\\\" class=\\\"edge\\\"><title>ResNet_MF_sum0&#45;&gt;ResNet_MF__mul0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M255.5,-1493.74C255.5,-1485.2 255.5,-1476.3 255.5,-1468.25\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"255.5,-1503.9 251,-1493.9 255.5,-1498.9 255.5,-1493.9 255.5,-1493.9 255.5,-1493.9 255.5,-1498.9 260,-1493.9 255.5,-1503.9 255.5,-1503.9\\\"/>\\n</g>\\n<!-- ResNet_MF_flatten0 -->\\n<g id=\\\"node33\\\" class=\\\"node\\\"><title>ResNet_MF_flatten0</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"321,-1656 190,-1656 190,-1598 321,-1598 321,-1656\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"255.5\\\" y=\\\"-1623.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">ResNet_MF_flatten0</text>\\n</g>\\n<!-- ResNet_MF_flatten0&#45;&gt;ResNet_MF_sum0 -->\\n<g id=\\\"edge36\\\" class=\\\"edge\\\"><title>ResNet_MF_flatten0&#45;&gt;ResNet_MF_sum0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M255.5,-1587.74C255.5,-1579.2 255.5,-1570.3 255.5,-1562.25\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"255.5,-1597.9 251,-1587.9 255.5,-1592.9 255.5,-1587.9 255.5,-1587.9 255.5,-1587.9 255.5,-1592.9 260,-1587.9 255.5,-1597.9 255.5,-1597.9\\\"/>\\n</g>\\n</g>\\n</svg>\\n\",\n      \"text/plain\": [\n       \"<graphviz.dot.Digraph at 0x7f1e1dd1f1d0>\"\n      ]\n     },\n     \"metadata\": {},\n     \"execution_count\": 20\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 21,\n   \"source\": [\n    \"net3.summary(user.as_in_context(ctx[0]), item.as_in_context(ctx[0]))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"--------------------------------------------------------------------------------\\n\",\n      \"        Layer (type)                                Output Shape         Param #\\n\",\n      \"================================================================================\\n\",\n      \"               Input                              (128,), (128,)               0\\n\",\n      \"         Embedding-1                                  (128, 128)          120832\\n\",\n      \"        Activation-2     <Symbol ResNet_MF_u_block1_d1_relu_fwd>               0\\n\",\n      \"        Activation-3                                  (128, 128)               0\\n\",\n      \"             Dense-4                                  (128, 128)           16512\\n\",\n      \"           Dropout-5                                  (128, 128)               0\\n\",\n      \"             Dense-6                                  (128, 128)           16512\\n\",\n      \"           Dropout-7                                  (128, 128)               0\\n\",\n      \"        Activation-8     <Symbol ResNet_MF_u_block2_d1_relu_fwd>               0\\n\",\n      \"        Activation-9                                  (128, 128)               0\\n\",\n      \"            Dense-10                                  (128, 128)           16512\\n\",\n      \"          Dropout-11                                  (128, 128)               0\\n\",\n      \"            Dense-12                                  (128, 128)           16512\\n\",\n      \"        Embedding-13                                  (128, 128)          215424\\n\",\n      \"       Activation-14     <Symbol ResNet_MF_i_block1_d1_relu_fwd>               0\\n\",\n      \"       Activation-15                                  (128, 128)               0\\n\",\n      \"            Dense-16                                  (128, 128)           16512\\n\",\n      \"          Dropout-17                                  (128, 128)               0\\n\",\n      \"            Dense-18                                  (128, 128)           16512\\n\",\n      \"          Dropout-19                                  (128, 128)               0\\n\",\n      \"       Activation-20     <Symbol ResNet_MF_i_block2_d1_relu_fwd>               0\\n\",\n      \"       Activation-21                                  (128, 128)               0\\n\",\n      \"            Dense-22                                  (128, 128)           16512\\n\",\n      \"          Dropout-23                                  (128, 128)               0\\n\",\n      \"            Dense-24                                  (128, 128)           16512\\n\",\n      \"ResNetMatrixFactorization-25                                    (128, 1)               0\\n\",\n      \"================================================================================\\n\",\n      \"Parameters in forward computation graph, duplicate included\\n\",\n      \"   Total params: 468352\\n\",\n      \"   Trainable params: 468352\\n\",\n      \"   Non-trainable params: 0\\n\",\n      \"Shared params in forward computation graph: 0\\n\",\n      \"Unique parameters in model: 468352\\n\",\n      \"--------------------------------------------------------------------------------\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 22,\n   \"source\": [\n    \"losses_3  = train(net3, train_data, test_data, epochs=15, optimizer='adam', learning_rate=0.001, ctx=ctx, num_epoch_lr=10)\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Epoch [0], Training RMSE 0.7046, Test RMSE 0.6775\\n\",\n      \"Epoch [1], Training RMSE 0.4861, Test RMSE 0.5299\\n\",\n      \"Epoch [2], Training RMSE 0.4662, Test RMSE 0.4835\\n\",\n      \"Epoch [3], Training RMSE 0.4567, Test RMSE 0.4819\\n\",\n      \"Epoch [4], Training RMSE 0.4505, Test RMSE 0.4653\\n\",\n      \"Epoch [5], Training RMSE 0.4491, Test RMSE 0.4582\\n\",\n      \"Epoch [6], Training RMSE 0.4427, Test RMSE 0.4555\\n\",\n      \"Epoch [7], Training RMSE 0.4405, Test RMSE 0.4524\\n\",\n      \"Epoch [8], Training RMSE 0.4360, Test RMSE 0.4507\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stderr\",\n     \"text\": [\n      \"INFO:root:Update[6251]: Change learning rate to 2.00000e-04\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Epoch [9], Training RMSE 0.4328, Test RMSE 0.4504\\n\",\n      \"Epoch [10], Training RMSE 0.4172, Test RMSE 0.4442\\n\",\n      \"Epoch [11], Training RMSE 0.4141, Test RMSE 0.4426\\n\",\n      \"Epoch [12], Training RMSE 0.4114, Test RMSE 0.4416\\n\",\n      \"Epoch [13], Training RMSE 0.4106, Test RMSE 0.4413\\n\",\n      \"Epoch [14], Training RMSE 0.4095, Test RMSE 0.4390\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"### Visualizing embeddings\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"Contrary to the linear model where we can use directly the embedding weights, here we compute each combination of user / items and store predicted rating.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 23,\n   \"source\": [\n    \"%%time\\n\",\n    \"\\n\",\n    \"users = []\\n\",\n    \"items = []\\n\",\n    \"for i in range(max_user):\\n\",\n    \"    for j in range(max_item):\\n\",\n    \"        users.append(i+1)\\n\",\n    \"        items.append(j+1)\\n\",\n    \"dataset = gluon.data.ArrayDataset(onp.array(users).astype('float32'), onp.array(items).astype('float32'))\\n\",\n    \"dataloader = gluon.data.DataLoader(dataset, batch_size=batch_size, shuffle=False)\\n\",\n    \"ratings = onp.zeros((max_user+1, max_item+1))\\n\",\n    \"for users, items in dataloader:\\n\",\n    \"    users = users.to_device(ctx[0])\\n\",\n    \"    items = items.to_device(ctx[0])\\n\",\n    \"    scores = net3(users, items).asnumpy()\\n\",\n    \"    ratings[users.asnumpy().astype('int32'), items.asnumpy().astype('int32')] = scores.reshape(-1)\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"CPU times: user 26.6 s, sys: 5.26 s, total: 31.9 s\\n\",\n      \"Wall time: 26 s\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 24,\n   \"source\": [\n    \"evaluate_embeddings(ratings)\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"Top 5 movies:\\n\",\n      \"Schindler's List (1993), average rating 4.43\\n\",\n      \"Casablanca (1942), average rating 4.39\\n\",\n      \"Wrong Trousers, The (1993), average rating 4.39\\n\",\n      \"Shawshank Redemption, The (1994), average rating 4.36\\n\",\n      \"Close Shave, A (1995), average rating 4.34\\n\",\n      \"\\n\",\n      \"Worst 5 movies:\\n\",\n      \"Scream of Stone (Schrei aus Stein) (1991), average rating 0.00\\n\",\n      \"Mortal Kombat: Annihilation (1997), average rating 2.25\\n\",\n      \"Home Alone 3 (1997), average rating 2.25\\n\",\n      \"Crow: City of Angels, The (1996), average rating 2.25\\n\",\n      \"Children of the Corn: The Gathering (1996), average rating 2.25\\n\",\n      \"\\n\",\n      \"5 most controversial movies:\\n\",\n      \"First Knight (1995), average rating 2.95\\n\",\n      \"Half Baked (1998), average rating 2.85\\n\",\n      \"Just Cause (1995), average rating 2.96\\n\",\n      \"Murder at 1600 (1997), average rating 3.07\\n\",\n      \"Twister (1996), average rating 3.08\\n\"\n     ]\n    },\n    {\n     \"output_type\": \"display_data\",\n     \"data\": {\n      \"image/png\": \"iVBORw0KGgoAAAANSUhEUgAAA5YAAAH0CAYAAABGqLIGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsvXm0JclZH/j7IiIz773vvdq6utVSa0MIBAYZAwPGHpiRZ8ASMos3DFh4jG0GNDMeDwaMDcZjwWHRMD4YMAZsz9gsMvsYzrAdgQ8IMIsRjM0iEEaIbnWr96qut9x7MzOWb/74IiLj5rv31avugq4S+Tun6uXNjIz4YsnM+HZiZkyYMGHChAkTJkyYMGHChAnPFur5JmDChAkTJkyYMGHChAkTJtzdmBjLCRMmTJgwYcKECRMmTJjwnDAxlhMmTJgwYcKECRMmTJgw4TlhYiwnTJgwYcKECRMmTJgwYcJzwsRYTpgwYcKECRMmTJgwYcKE54SJsZwwYcKECRMmTJgwYcKECc8JE2M5YcKEPzQQ0Y8T0V9/vunYBiJ6GxF9zh9g/R9HRL/zB1X/swER/QUiepiITojow7dcPyGiVzwftN0NIKI3EdFbdlx7DRE98odN092GO/G5mDBhwoQJzw4TYzlhwh0EImIieuXo3M7N652MbXQz8ycy87f/AbT18jh25nbX/Wwxnktm/jlmftXzSdMW/BMAf5uZ95n5P40vxvPvBgAi+jYi+so/dAr/AEFEDxLRxz/fdPxRwl3yXEyYMGHChGeBibGcMOGPIO4kBuxuxPvQ+L0MwDuebyL+sPE+NH93FP6ojev7Sn/fV/oxYcKE5x8TYzlhwl0EIrpKRD9CRDeI6DoR/RwRqXjtRUT0/xDRU0T0+0T0d4r73kREP0BEbyGiIwCfTUQfTUS/QkRHRPQEEX3dGe1+EhH959juLxDRHy+u/X0iei8RHRPR7xDRf09ErwPwpQA+PZpT/losm81NieiziejnieifxnrfTUR/Op5/mIieLM1miejPEdF/ivQ+TERvKkj82fj3RmzvT8V7/iYR/TYRPUNEbyWilxX1fQIRvZOIDonomwDQGf3fNX6/GGl/jIi+iYjqWD7R82uRnk8fm0ZGbdkXEdGvRxq+l4hmxfUvjvU+SkSfU2p6iOj1RPRbcczfS0RftINuRURfRkQPxfH8DiK6SEQNEZ0A0JHG39txPxPRK4nocwG8AcAXx/78cLx+szX3/XHMjonoN4joA4noSyItDxPRny3Kf3ZcA8exrjfsoGnnuiWiTyGid8Q5eRsRffBovP8+Ef06gCURfTeAlwL44dinL47lPiau8RtE9GtE9Jqijvcjop+JNP4kgKvbaBzR+6VE9HRs/w3x3EdF2nVR7i9SfE621PFtRPTNJKbkJyTPzf1E9PVxbb+TClPmm8zLznUbrzMRvZGIfjeW+edEtPXZoLv3uXh/IvopIroW5+bfEtGl4vpLiOjfxfG7RvJ+GL+zrgF4E+14xmL5WRyba3E83k5ELyjqOs9637AU2DJep96/8bwion9ARL8X2/8+IroSryULj79FRO8B8FNn0TphwoQJ5wYzT/+mf9O/O+QfAAbwytG5NwF4Szz+GgDfCqCK/z4OwhApAL8K4H8HUAN4BYB3A3htUYcF8Odj2TmAXwTw1+L1fQAfs4OmDwfwJIA/CWFE/jqABwE0AF4F4GEAL4plXw7g/cd0F3W9DcDnxOPPBuAA/I1Y71cCeA+Afx7r/rMAjgHsx/KvAfDqSP8fB/AEgD9ftMsATNHWpwJ4F4APBmAAfBmAX4jXrsa6/3Icx78bafmcHWOwbfw+EsDHxLpfDuC3AXz+rrmM9D9S/H4QwC8DeBGAK/H+N8ZrrwPwOIAPAbAA8JayPgCPAfi4eHwZwEfsoPtvxjF4RZzjfwfgO89ab7vWI4BvA/CVxbXzrLkWwGvjGH0HgN8H8A/jmP+PAH4/lt0DcATgVfH3CwF8yA6atq5bAB8IYAngE2L9Xxz7Xhfj/Z8BvATAvDj38UXdDwC4BuD1sX+fEH/fW7T9dZD1+d9A1tBbdtD5GsiaSuX/20hf6uNvAfjEovwPAvjCHXV9G4CnIWtuBuCn4lj+DxienZ8+57ycZ93+CIBLEMb7KQCvex97Ll4Z57YBcC9EMPX18ZoG8GsA/ilkXc4AfOzonfW/xv7NccYzBuDzAPxwpFXHsbmAW1vv34bN5y6PF85+//5vAH4JwItjP/8FgO8evS+/I9Iy30Xrzb5Z07/p3/Rv+lf+mzSWEybcXbCQTcjLmNmy+CcxgI+CbH6/gpl7Fr+4fwXgM4p7f5GZf4iZAzOvY12vJKKrzHzCzL+0o83PBfAvmPk/MrNn8ZHsIJtHD9m0/DEiqpj5QWbeqv3agd9n5n/DzB7A90I2/V/BzB0z/wSAHrIJBDO/jZl/I9L/6wC+G7JZ34U3AvgaZv5tZnYAvhrAnyDRWr4ewDuY+QeY2QL4esiG9SxsjB8z/yoz/xIzO2Z+ELJxO4uebfhGZn6Uma9DNnV/Ip7/KwD+DTO/g5lXkA18CQsZ8wvM/Awz/3876n8DgK9j5ncz8wmALwHwGXR7TN/Os+Z+jpnfGsf/+yGb+DfHMf8eAC8vNEUBwIcS0ZyZH2PmXSa6u9btpwP4UWb+yVj/P4FsmP90ce83MvPDcf1vw2cB+DFm/rE4zz8J4FcAvJ6IXhr7/I/i+vxZyJzdDKn8zwD4UcjcAsC3x/YQNUmvBfBdZ9Tzg3HNtRAmtGXm7yienaSxPHNezrlu38zMN5j5PQB+GsO63Ia77rlg5nfFddIx81MQ5j/R+NEQpvbvMfOSmVtm/g/F7Y8y8z+L/Vvj7GfMArgHwvj6ODZHsZ7zrvezcNb7940A/iEzP8LMXRyrvzx69t8U+5i+B7tonTBhwoRzYWIsJ0y4s+Ah2pYSFeSjDwD/J0Q6/hPRjOofxPMvA/CiaMJ0g4huQExRS1Omh0f1/i2Ilued0ezpk3bQ9DIAXziq+yUQKfm7AHw+ZNPyJBF9DxG96Bb6+0RxvAYAZh6f2wcAIvqTRPTT0TztELJxOssU8WUAvqGg+TpEu/sAZOOYxyMy5+PxGWPjOolZ548Q0ePRDPCrb0LPNpTM7Aqxr2P6ttD2lyDM8UMkppl/akf9LwLwUPH7IYim5XaYuJ1nzY3n8unICKXfgGiklxDG8I0AHiOiHyWiD9rR7q51u9FXZg6QcXuguPdmc/wyAJ826tPHQoQ5LwLwTKQ14aFtlRTYVj49H28B8MlEtAdhmH6OmR87o67xWG59TnCTeTnnut21LrfhrnsuiOgF8V313kjjWwoaXwLgoSgM2YZxm2c9Y98J4K0AvofEdPdrIwN4K+t9J27y/n0ZgB8s1sBvQ74vu74JW2m9VZomTJjwRxsTYzlhwp2F90DMlEq8H+LGhZmPmfkLmfkVAD4FwBdEn5qHIdq/S8W/A2Z+fVEPl5Uy8+8y82cCuA/A/wHgB+Imd4yHAXzVqO4FM393rOe7mPljIRsZjnWdau824LsA/L8AXsLMFyEmwcn3a1tbDwP4vBHdc2b+BYjJ3EtSQSKi8vcOjNv4FgDvBPABzHwBsnnf6ad5i3gMYsKWsEEbM7+dmT8VMnc/BOD7dtTzKGReEl4KMeV7YnvxMzHu/3nW3PkrF83mJ0CYuHdCtGzbyu1atxt9Leb0vWf0YVufvnPUpz1mfjNkTi6PnpGX3qRb28o/GvvxXohp7V8E8NcgG/vbgZvNy+1et3fjc/HVELpfHWn8rILGhwG89Ayt/ri/O58xFquSL2fmPwbRnH8SxHz53OsdYj69KH7fv0HM7vfvwxBT63IdzOK6O9WXs2idMGHChPNiYiwnTLiz8L0AvoyIXhyDL3w8gE8G8ANADqLzyrhpPoRIoAPEJ+k4BnKYE5Emog8loo/a1RARfRYR3Rs1Ozfi6bCl6L8C8MaoMSQi2iMJpHNARK8iov+OiBqIP926qOMJiKnj7XrPHAC4zswtEX00gL9aXHsqtlvmXPxWAF9CRB8S+3uRiD4tXvtRAB9CEjDFAPg7GG3YzknPEYCTqG34n0bXnxjRcyv4PgB/g4g+mIgWAP5RukBENRG9gYgusph8HmH7vAFiLvx3SYLO7EM21N97hjbmLIz7c8trbheiBulTIxPWATjBjj6dsW6/D8CfIwkeVQH4wljXL9xCn5IW8bWxPzOSYCkvZuaHIGaxXx7n4GMhz+bNkMp/HGSz/v3Fte+A+IK+GuKbdztws3m52bp9rrgbnosDyBo7JKIHAPy94tovQxjYN8d33YyI/uszaNr5jBHRnyGiV5MEaTqCWJ6EW1nvEL/g1xPRFSK6H6KhTH0+6/37rQC+imLAMiK6l4g+dVcndtF6Rr8nTJgw4RQmxnLChDsLXwHZCP8HAM8A+FoAb2Dm34zXPwDAv4dsRH4RwDcz809H88JPgvgi/T4k0Mf/BeDiGW29DsA7SKKDfgOAz+AtvmfM/CuQQCvfFGl6FySIBSD+PW+O7T0O0RR8SbyWNtDXiGiXD+Ct4H8G8BVEdAwJTJK1EdHf6qsA/Hw0/foYZv5BiPT+e0jM3X4TwCfG8k8D+LRI+zXIuP78LdLzRRDm9hjCfH/v6PqbAHx7pOev4BbAzD8O4Bsh/m3vggThAGQTCoiG68HYrzdC/Ly24V9DNGE/C1kXLSTwyLPB/w3x5bpBRD/0LNfcLigAXwDR/lyH+LvtYni2rltm/h2I5umfRVo+GcAnM3N/RrtfAxHk3CCiL2LmhyFBn74UIqx4GMJ0pG/lX4UEsboO4B9DGMOz8DjkmXkUwL+FBKF5Z3H9BxFNFuMafs44x7zcbN0+V9wNz8WXA/gIiHDuR1Ew9XH8Phni2/0eAI9AzFZ34axn7H6IUPAIYor6M7Hsraz374QEE3oQwE9gczzPev9+A8TC4yfiO/OXIGt3F3bROmHChAnnBolr0YQJEyZMuFNBkjbjNwE0z1LbOOEOBUmql89j5n//fNNyt2F6LiZMmDDhzsKksZwwYcKEOxBE9BdI8k1ehmhef3jaPL9vgYj+EsTP7aeeb1ruFkzPxYQJEybcubijGEsieh1Jgt930RDtcsKECRP+KOLzIPlDfw/iS3u7feEmPI8gordBAt38L9FfdML5MD0XEyZMmHCH4o4xhY0O4/8FkrT4EQBvB/CZzPxbzythEyZMmDBhwoQJEyZMmDDhTNxJGsuPBvAuliTDPSR59s4IZhMmTJgwYcKECRMmTJgw4c7AncRYPoDNZL2PYDOx9YQJEyZMmDBhwoQJEyZMuAOxKwHwHQsi+lwAnwsA9Vx/5NX324chjwCCYw0AqMhDIaANNYgYgQmGAnrWqMmDAfTBQFOAIoZjhZo8LGtoCghM8KzgWEERY6F6tKGCpgAiIDCBwLCswQw02sMFBSKGj3X1rEEADHl4VmAM2ZcDCAoMQwFrX4FIjj0TGPLPB0KtPVTMX+xYQRNDUQDzkGvasQKDEAKh1i6e0zAU0Hrpo1EBvdeoVAARwwW5BwAa5XI/XVColLj6BKZ8LkD6CwBGBfigclblVEYrzuPCsbyKY1+pAMcqX9fE8EzQFOCChlYy5kYF2KBRKY/AlI9t0HmubNBglr56VhvtMQjMkL7FATcUgGLOKI6ZIgYY0ErqXdkKYEJTOZn/IP1PfSzhgshjNDE4Xk9z4uO4pTkkSBvMJOsjqNhfBUUy1gQhl0HwXoEZ0DpeI0CB8/oKQUHrIP2MbYGQfwOA0QEhEAITdJ5PBe8USDF0vF6uSYp9THRSpIdZaJDxBUKkgwEgtqdUQAgD3d4raM2ZJiKWe5iEZsUIgXKbabyAYRx5VLdSDO/l3nRPYAIRAMi4SEekJiLAe6FVqQCktcEEDgSVx1fqAct1pTiPSwiJBmkzhKH91K/yfHBCgzKDu1x5T7qPg6xP0uX5OM7FWmJPUCYgOAXSvDHu7EmIjP1HoM36Iu2kw3BMQztsFWACSDE4KMlWl19QBGh513GgWGb4C+I47kOfZNCEDihIfSr+zouMh/oZgOa8hvI1lvrz31wWQxs+3qPjsUr309CPVIclKafi/aG4Nw1XegA9AYY3j128P5VLbecXIA/3gwb6Eg2JJjXqK0VaUh9L5HuL+aDiWjkOxXkKAKd+pvbyYpY+k5N7ONczkMRU1BP/5r6mbut4PrZF8uKS+9VQliNpZT25fBjVVfaLi3I0/E71UlnGp7EahmSjrRDb8EUdiOefy84nYBDJc1EvS3tpDaZxzP3DafqAYtx9MXZc1E9FOWw+Kir1DcVYj+so703PR/nyL56DjTb47LK57+l5xzAuZd9PrS01qgubdG+0gaFsetWluvK4j/tQ3lq+VrD92saF8u8YGzfvwLhOFP2/TShfjxv9Ks7nNVPck8YG2DIGGw3sOH+7MSbmrDm4jTStrj/yNDPfe3tqm7ANdxJj+V4ALyl+vzie2wAz/0sA/xIALn7QC/jV3/wZAAAbdGQIVN5MWy+MZkgbtVgHAXmjLnUOm/7ERPigEOINlRZmqtz0SZm0YRtWvGyCQ97ojttgpsy0JSYv0ZS/JXEDnvoxpr08V27EVWIiEpMbN9ZEmxvcUGxkNje9sQ0ajhMtiXYVmaTUrjA7tLlpjX9TnUolpmKoP2/GwyajsbFh98IIbdQXIj0qnKJVCtDGHKVy5VeOi7kkJRvoi1bWijBtp5mucgw41pc2+hvf3OKLvnVDHwBSu8cqMx1pXiiOl4/tJQaCsTFWaa6kz3yqHmYCOwXSkZlgGja0aaObOzHu1Ob4pi9YZjJSe6mviQkp5jv1DzxiVIr+5zLFfJXMjDAto91S6v9ok53PKd7cZAORcSrrKf6eesCKNsPoeMwM+bgzMqfX3AbNIf7dRkPZvqeCeeLT11LdhNNjk+Y2M3e8+ZLxRfvjXU+5cU5tJObmZpurciyLzfxG3wJATOCS3vL+on6K67gsSzyco0CnrwFgYjl2aQ3EutMrVRfH5XqJzxZ5AhthxFhvjuupDXu5cyQuaB426PkYOLXhVm40BWrEGIzpzGOzScfGpj3OVd5sJibLFeui3MDxsEbIUx7bzDyQ/EvnKQAhMYZpaaVXQ9F+YjrS9UQTgMyElfeWY5U3y2PGMv0MA13pJPl4uIWxLMc7VHjWIB9pL+cg0a5H8zZ6J23Q56VDA2N5+n1BgQrmdPS+AeL6HK5TfA/uZCz9MCZ57EZMYUbJhMY+lMzwxnyN1+j43lh/HrvyPQHsZIiHDgxDMH5dZSY4nxj9Hp3bWT+N/o7Kbb2vvLegc3zPtk/BzroSveNzZ9G6rc9n4Nx0PMvyN8WtlgdAzzUeTLz9177lCx96bhVNuBnuJMby7QA+gIjeD8JQfgYkyfJOKIhWrFYOtfJYuhpKe8yNhSGPZ7qFaMG8xsxYHPcN9qsefdBgiMZJR23ezDi0zqDWHj4o9ACcNTA64KDpcNjOUGkPTQwbRHvYOQMfCAdNjy4ysZ0l7NUWy74CM2FeW1ivC2aHRRtpPLQKOG4bEACjPUJQ8LFcCITaiIZRK0ZnDbQK0KPNWGtNZu5MYqiZUGmPrjOoKo9Ke6xsDWNs1MTozAw2VQ8XaXdewWjR4IlGUdpNIEraLgXvS02UgtY+M7LprZg0Y5XxcF5lZtsYD+d0/ivaOZV/V5WD9wrOaTTGwfYGSgcoFdB3Mq6VCXBOnWJKS4ZY6OU4nshMtk3HAIwJUIph1xU4EOoDB+8J7HVmSrUJGwyw62W8lJGvnIpaIYIIGyjulIIVxpiqkL+YzApEQTaokVkYNJ4kWi9PoNkQPV9pj+CMfHidAum4O0kMUvkh5KjlAhB6DVXJrpUdgF6BK4AaLxqHvKHEoO2KjFQez4JOMMCeQFqYx6wt0pHxCATUHnAKaALgZb5ZCfMJpoFJ8gSOuwQyPjOmSJrHzAAw4BSoCeBeARUiPQy2BDLpqcLAGKSvoFOAiX0jFi2fo4EJI6mLPW1qwdISdnE3mjZCJcMVIOcJEpfSANQrGaM6XiehHTVvbFTIiYaQS61ZYv4SI4i4aawCqFXgmje0XmQpMyBQ8pubgfa0SWXNkRba0BDqtYLfC3EuIsMRN6hqTfD7UXDjCVxFBqv4W4J8ZKQMC11axiSVz0yJHjbD5IXevBFO8+GFkUuMHRzJJriO608zqI9zVcdxqDA8A3F9kSIgAGZN8LXQBiYoK5t1Pw8D3XFc9Jrg96QPqif4BUO1hDAbGEjdpXUa9+SGwZoyg8Va7s1jzwTlAV/zwDhEDarqpa+pfAJrqTdt1pWlzHxsMEg2TcDwO9RyPUTmgSD3KAv4GcOshGlkQ5nxQkgbfmFSVA+wkbECkzC+WubI13Je9wQf54+83B/qgWER+gmhGsYsVNInZWXc/YyhIqPLKtYVZKyUo8ywhGK9pfKhYuhWysg/hrJCK1gYR+UA3wC6A4KJtCn5bffwrKF66avQg/xsUwBCA1AcL9VHpjjOGwDoXuaKdRIoSB9kPaSydIrhkjFKBzLprACzlj4KAy5zw3qTkUlCgERvMLHtSBv5YcxKBiVpQ7MmNa6j9M4RAVFkVOPnKmmCc98R15YZxiSNHcLQP3KxjUKLXWo2swAjrtnMHPPQx3RPyfQmpHpLDfoGMzsWQoTN+2WcGKwoMzhMp4+zpnbUThr/U8KTMSM9YiwTbSPZ1YYgKQlfNmgfMd/pu7C1b1vKjRn5U4xlqm/LWJfYJtwYn78ZDXLD9ja2yFruCrz2z+zxtev+ttf7q7/evZWZX3fbK75F3DGMJTM7IvrbAN4K2Zr8a2Z+x1n3XDBrvPbe34JljVWoUUWz044NbNBoLjp0wWRzSkWMLpislXSFhjKdd0FnE8W5tghMOHIzVBdCNmFVtGnKuPY1GuVgWaEiKZfgi6cpsEKlPDwTXHzTNNrBM6EPBnNtI50BVTSTTaa6CpzrTUyfZ0IVaTDKY+0rBFZolNsom851wSAw5TYBoPUV6mh6qojRpzcghOExNJjGAmJGamgYi/J3GlMTTVvTPanuWvmN+l3QMMrnv30wqJXLc1Brj9ZVeZwda9TKwaiA1p0WOQeQmEXHuQmFOXPZh5L+1su81/d4GOVxYhsYFcR0OmxqvBNMNJ/tvayppCVPWm1FnJl8F9SgMS+0nql8wKDBBoBKeTH59XpjTGvlN8a8NM8t70/X0/wljbgiRqMdbNCwftP8uNT0J1pLzf+GBrjQ3hoVcv0JY+1/Ok71VtrDxnErx7YUZqTfYpIuptQ2mmm7aIadTKx9XGeaeOOeVA+P2gCQn53AgzZeRUGDK7TZqf++sD4oNfXlcRqvNHdle35kvVCOy1nlKu3RWoPa+Cz8SVYUYwGTtDvMhY5aaR8IOgoKnFdF3QG909kKQdFm3c4PtJRWGKU1Rpq/ZOodinVd/k3jlOZBF8+GHq2DNGdpH6Fp07KDizWb1n05j7mO+LuKgsJyDaQ5Kq1JADG7FuHYcJxMuweT7mHMswVK2Gy7RKpb76CvrKe8Vlo0ZCuKM86neigKtqigKZmfi1AoCTni9fJeRGuWOPiJJI4CNQBRg8ainE9yjFQubPYhzWXqdZrXUMz5lv27rNtyR1sIDBHb4yi3Skla8vxEa5C88Q2nx11pRnDFRN4iBosQ2rASIGKwV1kbvFEOQle2IEkCu6JvY+ubDUsP4PTOPAnI8kRhYCi2bc6HCcApC4Z0XJpdj7Gtzm1l0riUdY/NvbfVm47zPamrUQgZjwFsaoPH94/pLPvMp+nYMA/eRV+qhxMd2LyBeXPcbsYUbaHjFF0F01T+3kkj4eZzdEYdOyl6FnVt1Huzcuep5zxr7y7Ctesev/zWl972evULf/fqba/0WeCOYSwBgJl/DMCPnbe8Y42WDR5uryCwwqVqBcsaN+wcx3aG+2bHWPsKc21xaOe4Ui3xVL8PALi3PsGRm8OxwlxbPNnu43K9xrFrYChgz3SYa4u1r/HY+iLuaZbovEEfdN7kX6lX8Ex4qt3HwvQIIByYDk+0B7i3OQEAPNPPMdMubx6XroZRHitXY+VqPLA4BACsfSXMsHIAFJRyOHZNZEwU9qsOfdBYuXpg9kDYN11kZIQpcqyxbzqsXI0XL27ghp3jyM7wgtkxWm/Q+goX0GYm47CbY24s+qCxX3U46meiEYx+o/tVlzdyAHDUzzDTDrN4Tx9Mbm+mLVw0SQaEaZ1pi2e6BebGImgLxxpLW2Ov6rG0NS7Wa6xdhUornPQN9usOR90Mc2NR6xUO+xnumS1x3M/QB417Zg6GAo5tk/ugKcCzgqYAaGS/VEUBy9ifRrvs53lQd+icaHqP+wbWa3zIPY/DM+Hx5QU0xmGv6nPdJ32TmShFjEuzNRTE3xMAOmfQGAcfFPbrDmtXwQbG5WaFHgbHtkEVmeq9qsdRN8vlEvOXNrwHVYtaezzlZJ36oNB5jfnMwnmDvarH9XYBrQIa7dB5A+s1jAqooiZz2dcwKuBC04rvKICZcbjSrHCjn2NlK2gVUCuf11FgQogMpw8KNm7kK+1hoyadAOxVvcyXCjLeCjjpaywqi0p5HPcNLs/WeKadD4wWUxY41MqjjePVOWHq17aC0cI8NUaeFa0CiAltrHtlK+xXPWyohT6voCrO1gBVZYV2r+GDggdwZb7CytZoo9a9Nh6N9pgZixvtPJu7VzpkWnuvoSOzMosMUGsNtGLMtMeyr6AIqI1D7zUCA7PKoQ+Eq4s1FDHee3hRGBMQZpXDuq8yM6kVY6/pYCjg+nqRGUyjxYKhi1YSALCoLE7aBlcWa1xfzeEiQ07EuDhbo3MiKHJBYV5ZdHaeGb951cvz2jaojT3FfF1dLPHo0QV4JjRahDh9ZLYuz9Z4/PgAzGJZsXQGF5oeJ22DmfFo+4FRC0yoFUMrj3VfYV5btNZgVjmctA2aysHFcfKROW4qh0Z7HK5nqIwXl4K4xhrj0TsNoz16Z1BVDjPjxLIjMoSzWlR1J22DRdNj1dVRKSvCDmbKjON9+yc47GZYdTUUMfaaHkYFXFsuUBu3UfbSvMW1Ezl/Ydbh6ZM9XJq3uLGeZR/i/VkHQJhjTYxlX4mgJs6L9Rp7TS/ww8uEAAAgAElEQVQ+4k76O6tlHivthWlignPSD+s1Lszbjflp+wq909mFYNH08NEFIAllAhP2mj7PgfMK+02P47aBUgGdrfL7wAWFeW1xvJrh3gsnsEGh7SsYHaKFBWdBQ2cN9mY91n2FvaYHEaPRHl18rk7WDWa1xYVZh2dWczSVQ6U9KhVw3NWojTDyrTXZ4mUe6Wz7CloHLGoLIsbhUu4HxG2lqSwUAauuQm1EAFVpj7aXd1gIhKqSPvW9xt68hwti2ZKuzWuLwNLWrLZYrhvszTu0/VDnxcUaTx/un2eLsRWzxmLdik9+VbvMAGoVsF7XqGon1jaNhY+WOj5qq+czeRat1agqjxDErz54hflc1o1zg9DDGLEEAgAX11PpKrK/12LdSSwJZzWamYW1esP1RSm5BgB149B3BqaSejkQTBxT2xsRksT7tA4IXolFDgPBa5jKiX9+UCAl1j5iZeTBLFZezNKOj64B2gR4Jz7+pvawncn9SIIAbTzAMhaJ4Q6eMkOqtJgIJLcQtmqwsskWMQDHPg2uJJF5Lyxlsh9odMXI/upnuR4EEhcHVwhStzHtidEvrVs0i6VMKlMwzVtVbWNGfMxhlu4NpVAhtb2DgQaQzdy3YhtzSoMwSUiJQhLi7JKQab5VBvJmDPD4+vsQY8kAQl4E73u4Y/JYPhvc88FX+ZO+/VOw9hV6X2jCWG9IwEvNjQ0aCoOp31jjVt5nKGTtz/hauUkrfTPH2hG35VquA9iQ2JfPaLpeaqN2tV9qW0qMfTrHKDczpcQ8MLIGQ9GmRH18rfxb0jWWwKtCc1GWKbVbic7ExJTn0viUGozEAKSx2vYqDbwpuR9rFkLBQCXNXTLxTdfGfoCn+zdcK/1Jx76l5XiUwWxK+lLQnqQh4NGaGWuMxrSle5N25ZSkf8vv8fltGsqyrXGgmfJ36W+70U4R9IVHH8VS45J9YMe+sRg0ALu0hmM/WFUcA8P6oGLdYHRt63dsLDIeaVJSf7f5MJ/yAcYWTcRZYMjma/QNypqZsmjYHKt0brxe0/Xkh4vRM57Ng8cagW0f+3JjVG5qtm1+cuO8/d5U5WidpXOpHGmxN9sIKFTWXa4ZHUSLVFwnwun+AcOGr9z8lYFJUhmMypcotTXjjWBJX/nCGr+byw1beX859ozTAWQSreVcoThOpvMMbN3kjWlMdI03utswnueSzvE4l+bk2x68U1UXm+dxn9JxuWnftuHfGE+I+fqzxZkb4hGjUK5FhmzqmU6ZNp4L28pt23xv2a9uecxOa8TG923r503WwjYtWx6Sre8P7FwDG62M77lJ+6ew7f5d/d64b3tfT7VzRj+2ayzPaHMHtr02b4rnWO6mWtLbiHP3KbV91nN4Bn77zV/wq8z8X936nbcPH/lhM/6Pb33xba+3euHvPe99A+4wjeWtIjFlD8xuYB1qXO8XAID762PMVY/3tpdQK4cT2+Ce+RJPrA/w4r0bsEHj6XYPc2OxMD2udwu8eHGIa90eLlQtWm/wTLfAYTdDrT3e/+LTeGx1IWoeQ4y66vHk6gCKGC/aP8SNbh6ji9a4b3GMp9YiEb06X2Lp6o2NrA8Ki6rHwvR45PgSiBiLSqTXNig4r9E5jSuLNaoYGfbE1pgZh0a7gckD4aQXDU5nDa7uLxGYsOxrzCuLx29cwGLW4aDpcW25wP6sQ6M9jtomM5z3HZxgZasscT6IUuokLT5qGzhnsvnaftNjbSuR4BPD6IBlW2PRWPRONF5JszKrLTprcGmxxrKv4aJf5qK2WPUVFk2Po/UM+7MOvdM4qC2Oo4ZmZSscr2a4tL/C4XKOedOjNh6Hyzmc1bh4sM6aDRc1dj6o7MuZmKu9eQdFQOekP0SMVdugqmQcD+YtKhXwyGNXZL7uPUJrDbquQl2LdDb1rY6+osu1OIk0jdTRVE5MNJnQtRVMJe0sj2cgxZjN+yyh7roKs5lF11aYzXt4rzak0qujGdgRqn2R8mvNMMZjva6hVEB7PMfsoIP3BO90bst7ygznPLbXtwbNXLQ7tjdwhzVo7jHb79D3ZohiqhkqajeC12IaqQO8GyTfifm1nYaqJFIpxyizZmZhWwN2CtXCoj+uYfasSJZZNv/aRB/itYGZO7jWQNcBHABTewQvziLeigaSvQYRYBoHu6pQ7fXoT2qoxoOURHr1Vku0VACuM8JomJCjstrDBmg8TCN9805JNFSroPYsSEWTR6fgrQYYUI0Xk0HF8K28HvXCgQMQegPVeIDlmCrxgfa9hqo8/NMNwAR1tcvBlnyroecizecQJedrLZLj/cGPNknSyQRpnwAsDehCj3C9Bu87KDNEeOUblfgNEiS6a6uBfZeZsbAy4sO05xDWRuqupG4A0NcquCtWzq0M4KJPIwP6WMNdsVK3VaCZB44NeOGBVgF1KJgiAL2WzWwdQK0RX8uewAsP6kz2n0xBjag1IEsICw+ySnwRk4+lVUCiswoyVpbAi4Ac2fSwBgjguQetDHg+OCFl31MjfszV0xXcHiPMAigQ9Er8YN1FD+qi4KiSa+ZIwV7xUK2CXhPsRY/qhoY7kP6yZpijqDWK/mF+xtl/jwIQapY2AIRGfK50S3B74kvJBLBhsJFyoWaYZZzT5FNYSz0kShyYtfg+MuReCuKbaZaU/d2CBsyK4BYcff3E5zDVq1vAHjCaawqsEf0jkf3KlBN/yFBFv9RG/DHB0Xcz+t/ZPYZpCWYF2H1pg5z44/noXwiSY+VkfPSK4jnxndQdABZ6dBtpNIDqZBrdPPl5Sn1+ztmfTPUUyzCq4zguBgha6jKtLEs3E19KtweYFeDrwa+wOgG6K3jWMGvALWRuUl8o+qm6vUi7ljFnE+UE0Z+xWspchSr6WypE/1WpV9YHMrOc/FvByAGHkulm0EB9KLQQi++kWUu5HJ0W0a8y+jXqNs5NP7Sr+/gKqgffVCDOu4m+vGmNtch+ranfoYrjAKkbAMxK/HgBgBwjVBTXFsPN43k/9EV3DFZAqGjD3zIxUeSQfRiDJoQ60sUABYZKYQc0xA8ySroTsxIMiY+klr8UaU9zQ0HKaFs8z4xBokXIvrMZibEphDpjv01Ano2QdtqcBAxyvBH4KFUb1zoQ13z0JS0Z4exTqYd60lopowuP68w+rLyDkUuCxsDZbzT7km5hnrPP6Za6tsnLNq6NGPwzBZg0zNeu+nfiNjPAtwcMP5YWvw/hrmYsL+kVPvPqL+FGWCAUmsWWK7ShwkfsP4RjP8sar9kli2VoYFljcVnMTnwUnx66BT7ioEUbvc8r8piRbMqvu328el8C1Pr4FNhgcHBPC8sax36GxcUONhhUyuHEz/Axl9bwIKx8A00h+38ms03LGpY1PurSQ0JzTGeSzDhnyuLYzwCI/6f4cA4+f6lPyZ+yIYdVqKPvmfiULl7QSztBo3mB+Fh20XRV+qBz3R4KGgFdMPkYQPYVTOiCgUbIpptdMFgoaWfs09dFv9FjN8Nc9XH8VJHKRKNRFutQw5DPtHTBoKKQr9X3id9lAKG+z0Ej4MQ32b80YPCrSz6WJvqNrn0d6Rn8YisKsHG99MHAM+Gjrj4ED4VjK2OezGwVBfTBiOlrTPky17Iu1r7K55NvraHBbNjEr115f6M8uqDRRJ/YMfZMBxfnpfQTTf63c22xdHWuP/uRguOcDPQa5dH6CiamatkzHfpgcrum+KL10Qc5rb8+CLNe+nSmtD3pdzpO5xWJz+pMu8F3VXm0fnjNJD/b0ry690Nf05il6703qLVD702eU2Dwt+3jF7tWLpqyDm3NXmbRezHXVnFtpLrbOAapvjrupvrCBzuVT22UFgzjtR6YsHi5vFNWrs7nkolpgiLGTFsEVnmMyjrKZ3theixtg/lLLVpvNrSOlfa5XBICdd5kX+wqjmMy52am7KMLAM0rxRS5DxpNNNUf1m1AV4xj8vFNbgBpzss5TWsmPTfJRzm5DZQ+12ntlOsgWU/oGGwt+xxHwVrydy59YpOvsB9ZapTzOv9ACxdULpMsE5L5ePLJpXitdQZVNCtf2QqzaPIMiECw1v5UOynFUCqT3pnJj10nC4uC1tIftdZ+Y4+V/LLTnIz9VEu/7tJqJtVXXk+pk5Kpa/NBLgsV03gCyPU4n1JHiSVKQhYsORE8acWoy+jihGxWy9HyA5ANRulvThjSLymmDYud/CzE+Uhjogtas19qoBxITu4RgQpSaqfof0heSWCgYrPq4/lnC1YMeOkLJ+uWZC0R22MmhCINU4ItrFgCDXyLlI9jsWHxMNwcxppCJvgYIC/9tkU07k0rgDjPKmxERS/bSWmXcsTZdF+yqEmB6cbjwRj8RZNf5NiigpGFjKWlxFmmjrxLMzrW6G9cw6ZKb6xx3NbelvE6E2dpT3dpeRmnzUjP28b4/BnM19b7Nujdou4c0TJomfPAnEHPzbm2czGA59VAnneOzsL3PPcqnisYQLgzOd7bgruasVyGBj978kEAABvNX7swdEmC0cjvtOFKaUnS9ewrFBlGv2XhpmAfAHLQnXR/uZEfB7Yp6wU2GcLydzoen0u/y0BBiSGT4yGojSGf83iWgX7KenZhl7ltudndhm1mr+dBGp/ybzIZ3mW2W26UxrSON0dj89Jt/R3370G65+Z0b6mrFOKlOretoXT/eG7HtJZBXcr8l+MAKGMz4VvBWfMViqEvN4uluXOJdH48B7fSZnl9bNJaIgkPtvX3vONQbuB3fyu3p3EpUZoyl79v1oftdYzP3/x7vWuN72rzrD6cLrtJw3no2dbWzfo/Tkezq/3Ni+df68kMenzHFsH7qU30We2dOS7jDdI5NkvnWjdndaTcGO/axZ1jnk/hrEAuqf3znk+0lXSMf5fnxtqMbfXubP82bD5vBbs0L2MT6lMamILx28ZsnGct3YyxuNkYlXNQrp20JsePw456TzW7pV1K9W1p7nQFp5mdjXIb6357Wzsxft8GnOJFz6TtZuex+zHcvP9863Rg9M5TJ87JoO1u61bK3/Tazep9lvVNuLNxVzOWN+wcb3v8A7DsarigMKscfCD0zsA5haaWgCqV9uisQVO5HAigrlw2m9Q6oO81TJHCoqq8mFcyYb1qokO+xLJOTt9V5UEEdG2V8yFKXQZ1HQNDWEmnMd54Bq/gvcJs3oOZ4JzK9wPyt2urvDnSMaVFCGrjY6C00GCMmFmCSdJTeI35QkwevdOoGwvbi9ZDaZ/9/2xnsomd1mIWWEoWtQkbUk5ntZhOFvksU3u5j4nxcwrKBPhOQ5kA0pzNAZVhBKugap/TcgSnxMyy1yAdYGoP1xno2gtdnqAbD60D7LrKZoOlBI9UjNyYaOh1zP0Y/bIY0HWQqIAESWERCM2lFsyE/qQGaY7mnrGMVcNGQQGqljH3vcr+WqlvqvbgaGKqawnOwE4iBeaE970WE8SyXgKgGLqWtCq2NYM/oidQLSk5dB3gW50DFnBK8xH7DmJwp8VksIopOgBAM6q5hbcaIV0v/PBO5T0sUnKUUl0yRXTJWJatkpQqisG9gmo8QmuGfI5+qE9o0nItzZ0dIinm3IopNYYlMb3sY8oRq6K4j4Y6mMRsklJbkdZZnAs7jAE0i99dGwVEHPuZ2k3pMRhASnOQxrAM3pCCPBTHtHDybJ9Uw85Gs/SBMOysanlXcFskvUttlpv5xosp6MJL2cInjeZisguOfa4Y6FRBezQ77TS4CkO5SL7aswjLKqb1CEXQCoAaL+axcT6oV+AmSD9M7Gu5KU5rwUU6UhCMVD5FFE0pUKrYXp/mXeaMmIb0KAqS2sSwjHuv8macU996Ba6DmM/G5z8FpkhpTLBv5XlI82qEVlrrbPqbU57MPGgVz9dBjhdezIzjM8DJDDgFwLBqCG6RaK6KdR/bpE5tthcg6Vg8ydhy3Iyn94kb1gGndEWM3FY5DjlYSBOATiOnn4mpiCRljJhL874TupzMNSc/UpJ52kgrU8f645wjQPpRM5Dqi88UFA/PcYjpZIjF1LmJ82WVtJnmL81BnC8x7SYxo05joxiqV0N6hVTeUjZvRkxTwoZzKhzqCVwzVKukXB/zknoCzwLUSWnTeGsITYBqoxl1MkdHXLMdDebKFYsvJ1POX5lozms7IOYFLdIFlfkso8lk6rMcJEKAMA9QXRwfD3AdU/wUvsLpmqwlSaeTxoICiZkmRbPmIrhLTr9hpC3ylE208xZEARTzghILjcSDqXVZTzK3zul10vPKgxm+pJmJc1iGDI5pRJJJJEmMQ6k/vVvjmJTMaz4Mw725/JgBCwOtZU7PnBM1pawZMz5lPcnENZ0fjf+mOStvTSOSjlPnWMVcpyPGd4PZ5CH3bNbQbxHi5L7RtrawkTN2GACcxmhfcDNw2c5A8tmM7JiG8Xzd5XhfDt5zVzOWV+olXv+id+AF1SFarvCe7h5oBLywPsRFvcRvrF6Cfd3hGbfAC+tDPNxewUtm17EKNR5tL+FStUKjHJ7q9/GK+dN4b3cJl6sVTlyDG3aBp7p97JsOH3bhEbxrdR8umDUa5aIZpsdj7UU4VvjA/Sdxwy4QQLjRz/GKvafx7qVE/X1gfgPX+71sJulYYe0rXKlXuGRW+PXDB6CIcU+zxNpXWDlhko+6GV7+gkdzypNn+jn2qw77ph9yXbLCtW4BxxpH3Qzv/8DTCKDsK/pfnrkXD1w8xNXZEg8eXcH9e0eolce1dg+dl6iorzi4hmPXoFYeT7d7uDpbQlHIaUiudwusXZVN2u6dneDIzrC0Enl0biyurRe4PFtjaetsClcpn035XrA4wo1+gaWtUSmPC3WLZ7oF7pkt8cTqABfrFmtX4d75CR5bXcALF0c4sjM8dnwBL3vxdbz35CIuNi0WpsdjywtYdTVedf+TOO6bjfZs0OicgYtRQ7ViXF0sYVQQekmif15fLzCvLHxQuG9xjJl2ePvDLwUz8OpXPoKlrfHMao4Lsw42KFyerbNG1QeFx48PAACX75XzKVKqVgGH6xkWtQUDuHa0h8YEXFys0cdIqEdtg0vzFsdtg4vzFjaawlmv0XuNo+UMtje4eu9xjli6V/d48ngftXE4Ppnj6guPYb1CZys0lUUdI2lK9E3CPXsrtM7gaDXD5ftWACSC5tH1PVSLHvddPcKyq2FjtFStA2ojTFHvDCrtURsRxqRorimC5HLdoGksrDXwTiIG7s07rNoG3insXVzh6HiOS/cdo7MmByRqKoveGbTrGvuXVliuazSN+LAezDt01ogPacz/mgQte/MOJ8sZ9u9pcXS8wOxCKxFtjceyrXOE0OW6AQeJ0piiqh7fWKDa67G/6MT/1RrxLe019u5dQakgZp/WwFoNDgrNTCI5KhXQtWJyPL+8QgiU/WOZgb6rUNUOSjH6zqBuHJZP7oEBHLzwOPvMdl2FvUUnmvEYAbI7bsCe0FxZ52BH3sv6ShElAaA/arD/whOcPLmH+nKbo0gyE9rrM6Bi8SmdM/xaY3Z1nX2Lu2UNMGF+/wnWywakpG4fI076J+Yw97UwlUe3rsCtFsY9EOh6DXVfK+amrUFzuUV7Y4bqcge7qqD3LZQKWchmWwP2Cmbfwp1UoLkDtxr6ooVfa1Atuz1difGPP6mATkEdWIReA0r8YkkBvtVR2KShLjj4ZQXqtDCIgUQwcSyMO13swScVsOdEWMA0MKC1R3AK1WMN3EEAz70wICstm+UrFmi1bLAWktdVX68Q7u2BVkNfr+Cv9tBP1/CXXBayqMMY0dILQ+AXYWAQIlOplrHeWs7rIz34dFJkfmqGWmmEJsBcMxsbXT9jYcYSX3dD/FaZImMQFGAY1TNCCxMQKkb1tIY9kA19mDFUp3IeyOq6gb3sMXu4RjCcc0iGvOkX5sDPGPU1JTkvr4uDoO6S3x7DXmBU1xTMiYG9IEyC6hH9TcXvjVX0i3RyrnpGhBRuLr5wZimMen+Jo48i5byUyg2+l6GW+t2Cs1+aOpFvn58B80eN0GVY/P8OFfRaNqtuAeg1hN7HDPxcfDhDBdSHGu19z36Hap40sAeyHvShyr6C5KU93Srxd1wNfpHJ17J6VIGN+DOatfjesZLr1YkcJ6MrYuS8lADgI/OWGQwDNO9WsPtyLhjx4fQNNn3pYo5RcPQPnYuvZKikveQn6uaReYqpWHQ/+E8mv1CzouxvqTyEua3FdxMk88IE8X+NfU75VUMNmCXBxRyikv9UBCi6BVhR9PGkDd9TxHEQv2ZGMOJjSW4Yd+U4+n7SwAAC2dcyVLThUyxCmsRAi99lqAjKxhyvnjcY2+SLGsodc3xe09pMYy0M80BDeraEnqE+ME4HB0NiOuMPxQMzmxjBoh8pj2XQHH0sOdMA8EZgJslfi7xeNxDr3/DfVMXvMY3R9zJdo20q6jEjWeQATeMGbPK/UvdppjMJlrZabey4d5sifsIfDu7qqLCvfPWCv/aHXoUbfoE2VFioDpYNWq5w4me4qFdouUJFHqtQY6F6nES/xYXqsQo1PKvsF5nOKTA0Bexr8aF8xu7FNCCb/o4H8fqhm2OhJT3FQvc4dHPsR2/2xIQOOfTExzL5O95TLRFAaOObZ6Zsbv/Yz7KpZmoz5dtMZpAL1cNDYaF6POMWuWyq+8Q36ILB5WqFQzeHCxpz3aMiH1OzLHLf5rrHkZtFOsWks1EulwUkZ2elfPY/c6wxV734QqrBF1KB81gtvTB1pd9XFX0M59qii/5zS9fEFC8VGuUw1xZH8ZwLGpYV9k2PWjlc7xfZV7L8m8yTE5aukTGJDKj006KLx8nX7kVzSfvyZCsBmRIdickuTVj3YrSD5KtYmkAPeTilDsdKfN9ITKVr5TLTnnzsShPZPdNLOhUnURCSn+F+1WVfy2PbZP/FPujM9CafyBPXoFYOMy2BqwDxx7ynWWLpGhzZGWbabvQp5QtN9CTz6tL0GoCkmYn+aUlQkvwfxceywr7pMg2AmI8nv8yZtjk1TTIhb73J6VySD1vyle2DxsL0WLkaC9Oj9VUOlJRS4gBArWVN9sHkcxfqFq2v0LnIQMd0Jyk1T/I7rJTP/nalAKExQv/aiS+tViFHlS7zkCahxsV6DQB4plvkcayUz/MPiMnu3Mgznnxl03uBmbLfYVoLN7o5LjQtlrbOEYsB4KDusu9goiGlr0nBwQITWledingMAJebFZ7pFtmiQ/LryrgdVCL4ST6PrTPYj8KTxjhYrzciUBslApvOGcxjELJKe3RRSJFMvtM9jZGUQUkYU15LeU5NzF1aa4869g0Qc+iZcbFvJvtA5hyZxMK8xr7eM19haWtJpRLbVsQ47hrUkbaUu3S/7nHYzlBpj0VlcdjOcHHW4rhrch/mlc3zSCTpacr8tdbrTF8SAFZxDMu8qD5QTs0xM0UQJwA2qBgITSxAGiN5h8e+kWl9MiRdR0rLk3KaasV5PdTGYdXVWZiV/FOTFU3ybXVeoakceqcxi6lACIOLSNtXqIzHzDgs+yqn6iFAUvXEOvsowLBeozHRBzcGd6vj71U/pERJOVIVYSMoW6IrRekux7COKZ6S0EbrIeVSWkNdygMbBWSeJQXQsh2evVuF0QE25oDVOmTrHUUcfVAlOraJAcuSMAgAqsrF9CMStI0j7WUqFe8HH0hV+EyW0cDlN2Uhn6wJwJgQU5wM9xIh0oBslaU153M6Wmt4T9kyCUC2iMlpOcKQeiRF7pb0H0VKkiCqRW18juCtYtqSNEfeDxYGZURtBjb8OEOyyEAqGxnBQJv+nsSFBo2GqOPpcrQCyOlGssYych+JYUllktXImEtJ9yeUW+dUrqAj05CYx7G97Xlwluau/JssD8aavWL8ksWDaCuL8RnTsq1uKq4V9d2Kyf+tmKjflBncpVU9Bx0Pfv4XPe+RUz/8w2r+mR+//7bXe/GBh5/3vgF3ucbyabuPb3nPa9DEzcXKRg2DkZx2R91sY4Oy7Ov8ES8/ir0z2G86nHQNGiMRPnun0TvRolyatzjpahgd8oeaiHN+vL2mR5+0DM5gr+mxiia3s8rBus30J4EJtXHQinGylvxsKdl1SsbtPWE2s2J1RSwfZR1ynraElOTcOYXZLEYAtSZHEtU6oKo82rZC01gQIWpn5CU3ixoaADkflUQZVdmstwzrX9Uu590CccyRZWAq0YaoaLILYmjN8E6hbsSMON2TPm7GBNg+5tRyCqbycFbnHFu2M6hnYsKrdQDFfFzslJh1Rs1SGYwgxMAAHE3HTCPzHYq0L97FHFjxA6h1wNtvvB/AEo01MImZrmFwALSJvp2xLd/rbFKbPopJ8xSsErNbQMxBo/knGJvmvlbMhJPZMTh+MDstH7bGZ5NeZTibv6KPUTqZwI4Gk9b0QWUx1WUvEVApRkRlq8T0rAqDiWgKolCagiZzyGxmi8FULpmmZtO4+KGMpqpgkuMuRg5NGwPNw8fVSdRPWDW0GSN45o87Y6jbsJRN9Zoo6lQYaAEGc1fF2bwymx8mk0FHg7lfU5g0pr4kWtOmx0rkY1QstPvYXqIvmQpH00+1jMG19v0ghU5mqukjHOkiFrO4jY83UvvyfKtWIcw91Fo0W2UydLWO0VQBQCObBmbae5I10QQxoSORZqeP7iPHGn4/iIS7jyZw6ZloSa4VpoyqVQh1ELPEirNPWCoj88iDKaATU0SyFCNicjYvI0tRi5LMHZFNS5OJYDKNpJ6gnEQpJZY+qFbeI0mjJRFL45xHU69kFnb9RLQbIc6B6iUwjJ+JRg1Ajqx4rSX4hdCsrGjdbqwJoeFsPqc7ynMpmoKoPWBkUzplkbUWYt4n2hqK05PoS5qZGNcsQ6KcDlqCsr5yn6v7Ya/GGjjuSPoVhvtlzKQNPwPak6hxMryhjSBPCEY0GutONIirLq61GE2VVYzy2hGWNkaBLbR1oSo0NtF8MlQMF+sJlZTtYjRPP2e4InLtEFkXsG6gLUn5vU8AACAASURBVBQxznw0wwwVg1saIpQqgC0QonYnVECI4277qHGLG+S2A8IenjV8D3CMfsoW8IlXCHKeLQANcIy8SnHOATkHBSgNIEY/JQVoAtDLbxXXo4prKjFQlOR7iV9RAK0BE7WRaZ7JbPIwVEQlVRbQMdpuGjeJuDpoMNO6y1qrQguXNNJJu5Xazc+SGa1NQtawpWdjQ3uX+uKkPBfrPrUjDFHRnsKmRjZqMpkKprFkrAintW+MrHUD4nGMJpujyqbxS3xlHI+Ebaacpaltlm/Hb2guXzBFp86l48LMd1fU2I0oqTT8LrV1+Z50Ln6Hb2qCOur7qfJpzauC/i04Syu5E2PmGFt+78BWzekID968mgnPEXc1YznXFh966VGRTAaDpWugKGCuLTQCbtjFhqbo2M6wZ4RxSFq0FHxnT/dYLkTrliI2Jo3KhXqdtVZllMzWVQgg7JsuR0rsg8kaFgCimfFmI6BPoseogKO5aAhL6Tmz5Necm0GrVGqJysA8SRvTB53Ld96gUh5Hc5G+N9phZWs0xkGTRHxM0QMXVZ+1MCmqI4AcwbZ11YZGLWmJ0v1JKp00TmWUx6R9SFqMdD1pNxrj0DqDWvusOREJt2hFVrbCorJonWi0jApY2SonA7cxl2V6D5VSfR8IipCl+inwC0OYfxOTldcxuub1KD2+OG/hmXKiemaCiVLwpIVKfrpVvCdJ1ylKrI0OCCztKOLcllac05akv2UeUJH0S5LqpnI5IqRWjG5PNC7W6XwtzUuZ55Oj0CJE4UjSDngmtG0FY0JOj5KiCRJBTBsxaA4I2AgulKM9epUZ6SQVN8bDxsTYVeXR9wZVFA6ke9M9IqlPUvM4vlGynwQaQofQlsqmepO/cjYfjVJzF32DVUxFAoi5qtIhawaSwCYEhboetERJo5Ck90nK75OvrEnSeMrXk5Q+lVM6wM7F9Hc2t4OWwat8f2rDOzEFrCqfv9pJOJIZepLUK3XtYecmJ2JP9bqFyT6yKe9krg8D7aYS/2fEdZKEUq4x0HPR3nmvcooVAHC9hp7JDp699NkvpB/BKSjNGzsD9kkzwVmwwl5t+ECnZOiACF9CIKjKx77K9exfmDQlsb6QTGCjcMbHZO9kAryVdtIrMaRULbGOvtHiD2jivDklG6uZmMpmzQSA0Gk57wneKqDxCHM9+O8CCL3a3LkloUwhpEljn/2GQyGMSQw2xXIqMu+pLCDXCyFA9g8tN2fEQiMh+3e66Jso1zEIK5LvZx3gm8gF5zQFwoDnza+OzF4UVgADgwECQh3grIrmjclPT/qY/OSAYePPpmAeo+8hOQBMCE0QU8i4kaaCKVQbjOWwu01mmmzE1BeKB0bHST8JkXF2gzltFlZEk0Y3P8cudQeUFcZbxobyxjoxvNk31NHA+KclkYQEZVqIOEfKFpt/IDNNZR0Uz6f7/Ez6mMcwmc6WzEEUeKT2UzqNzCzGV2Ew2DDNTEKalPYmrSMQR4aHCjPMKDyIgj5V+JUnxjJo8eNMfrJg2mBiNxjAtN4LxjKPxxbGcrB7HDFC4/VYMnKbD5T8zxTLEMaMTWl6Gsk/fVz4tuZx5IIhHS+7EcO0jREbM6MbZqs7GMutdQCDye625c+jY9r8W772TvV7S/mzcFON5JiuczKWt6IVfb4xRYW9Q+Fi2o6Hl5cRmHCxarH2DR5bX8TS1rh3foKVq7FvOjyxPsCleo33nFyGVgEX6hY3+jlcUFiYHo8vr+JC0+Ipuw9DAXtVh0v1Cq2v8O5DuWaD3mCeLtZrKADvOb6cTaTmxuKR40u4Ml8hMOHx5QXMjc00BxA0BZz0e2idwf17xwCAY9ugiuaERntcNj2eWB3kCKFzYzdMugB5zvarHloF3FO3eHJ1AM+ERWVx1M1w/94RTmyDk77BlfkKR90MqyDM2l7VwwWFp1d7wtjF+66vJRdoFZm9eWUx1y6H+08mZIn5s15jr+rFvzKeS8zOsq+xqCyuLRdoKodGe9igcNzV2KstbqxnOGh6nHQNjPbin9j0ODzeR208Ls3XuLZc4OK8xdpWWPUVLs5bzI3FEydSposmZilqrFGiVU4mU4frmZhNGo8+Mit7tTCrihjPrObwXuEVV68BAN5z4xIq7bHf9Nl8bdVtmk0dzLrYvwqKgFVXw8QARoumR2sNFAH3XzhG5wyWfZWZyf1Zh5O2waLpseokLYVl2QxpFXB1f4mZsbi2EpG6D4R1X+Fg3qJ3GpcXaxyuZ9AqYG48rFeZsddK+n3SNtAq4NLeGkfrKLgwDi9+wSEOuxmO2wa1caj0YFaWNO5NZbOvpiKGMQ7Oa3m3E2N/3gnzXAeYuEbWfYX9eQejPZZdjXsvnshczsQcPCTTOx1wMO+w7GocLDp0TkMTo+2rbCJV/f/svUuPLcmWJvTZyx97R5w4JzPvo7qquEgNg2rUVY2E+Ac8BBItJvyQZsSMETMYNWMahBgitdRQA34DkxZILSRA/eC+MvM8ImLv7Q97MFhrmS332HFO5MlblTdL16RzYm93c3u5uW/7bH3rWwyEBXRPS8DtYcZ5Dri7uWBaPRILX4VA/oilAGNPfn9L9BWcfvXmAdMS6kaA9wljn9CHiPvzUMGm95kBfGZfTwLBhxvyzbzMHawtGPsV0xJgbUHfLZUOR3FKPX7+kw9wpuA3729hGaQN44JpCgzeCfTe3ZCf6P15qPQua5vYl2dK3OvbC94/jPjpV/f4cBqRkqnl/vSr+8ZWYFrg43kg2lk2ePWKaLmnS4eRfUzX1dXn4o++/ICv72+QosV4WOqGkTEFt8OMtw9H5GzQjZH8Yl9dME0Bh1cz5jk01cdiEA4rrC2YZ4/jccKyenQhYpoDumNsLAwGXOPdjOASTpe++o1SHsAPuW5epEjiZp1POE8dbTxEh+PtCRIzduB2CTB3Y67g3diMn/z8He4vA5aFfMplntw/kP9tbZcpuP3qAR8eDggh4Wac8f7+gFdfPeDhcawiauMbMi96DvUxLYEFzTKzNxz6Ya3jbUxB30VcLh18SE82N2K0OI5L3RgyAGYWXJO5IX6/cq2IevXDWimiKVkchhmnSw9rC7E8fKobH/2w4nzq8fqPPiAmh4V9mmVDRzaH4uowjAvm2VfGTB8i5tVT+KZLh66LuBln3J8GeE/0U+8yznPgTTWDefZwjtrRDzRmstHT91Tu6dzDSxsTP4cuY7p05Gu8OjifscwiZEabPDAFcfYIh5XGMjmUDFif0TFDZZ4C+mHF5dyhGxcsc6jMmONxwocPB3xu8kPEfKZ3iuVYtpW9cwnkS5wMbE8snlIM0ToBZtoYZI5BTLROi5wM3EDIMkUC/wYg9gv/bslGSKWKRoPuZkacOsAQGyb0ETk62mCx8m7hTQgQgycunjaJErFcHDM60sqxiZkJYXym755M7TlaOI7zKywmY1FF+FAMEjNX/BBrny0zl8pq4YaEdBGUi4pWLIvTSWzkUqBYMahMFGNATB3elKFySmN6iHibAlzUOUZAmuYqYKXIBophH0XT2CCSCraCbXJMqhGKKW8IiYhObV8yleFBINVsgZiuypQmjsYbBrq+jUiP9Cejtl0LPj2hwoo4k65Tvcs3tF/e4CTBsG0bK0NENq/25e3zVzB/Jb++5hkQubfCXvu+78J3Aq9/zakASH8Alr+fKReDU+yr4I1YzS4xYIoe59hV/6hLDBjYn8/kUj8LEJqiR+88gQkGJp2jOICX1aP3HnOkmIeBgeXZEjAQqyEA8i2MvvoECYCpVE225M3JEd2Ww5/M0SM7QyI5xcAbt/FlsqZgzbaCFilvtpkCBJvm7xNsrn5Hc6J202eqM7Av3pIdZqbzpkwWvcgLGPGT2ftASX6ArGBkYbRYeBwXttxZLmu2RClucdIcC9UQCJgZcOVCMdRicrRgBypwXbnd8rmzie+b+O0YBpYG8ATQcmpliPVLLJqdT1h54RajQ0qtfyKcsuZUY7ZpvxlrS12ArwzyY7LVWpgy9UH8rVIxiBwHb00OXU41T5QFtFgBeZxTsYgMjlIhmjPldzUGn7UZLhfE5CqQKoXiz62rAwL7cvH9FKtmyhYxOqZYg+9Ji/MnVG+hQtPcsFUUJrnmk0bjbhCjQ2Z/p5QsEls2s7W132ShbG1KjsGELeTbU5rFkdpEf1Mif7R1dUidqaAyJwtrbQUkzlnY0gRyIPdDAQdj7KZ/AFkUyQJKCzwqj+613DdazBtEZ+tnuW9Sd1ILDjpOY+ccKU8bACUX2rkvtCMuFl1+m5E/lvKvogWpqz5k5L9U6rsvZZp3MTqyUnO9Jdsm8JUcUuI4lqpd0i/xAcu2WaMNn0Np/l9EM6e+lKxCEBUZb24/X1N4bNu1W/9Asd5K+ZmfoZwLj0HiGIAc4zLTgqldq9vD85PXmlKmgW2bcLw4ysXQuovbJGNZY8yx31kuzWpe6xSKnlrRNGu3Qc7YuDwUZRXJqs8l22rlLcpvTocakTqeLD2KQUGLm7itS9rWLNNiKZe+WQO2XGPj36Y3K6UeekfQb6Fltge4XtmgEEaCY+ZAzafSPpyM42uoLPDnxm6o10s+NT56tUjA2G3O6XYJc0HeKQIARdzrc1NtkwASLl/el3Se3hG59sfUNpdikbhdqYD71Hwik4BKBqu1tRUsFWkIvMuYTbt/UqcRACpjaNp5Yg9kmOKYBSCmNbupw1imvAuDgO9vtoava2Mq990YoKCVT8VmGMuWQNUWqQM8B4sBUipNwV2DQMt59Q9XBW5o7iQCFOWcBpYCuGo5dA0pJBdWSC71OW/jDAJZwkzQ94OBUEFpbYUCb6aB3lJa3ytwFeCoASC4PNDvRC2rtPwFpf01gEHrX3V54N8ZQIHR6tqCp0nal0sD/abVVdtb+15QYLYgXafdS0UAagWF10yraug3h3dg8QkWVt+xy/OH9NefftTiPV/92VflP/xHf7+Kg+wDjEtcx71/47U4lvLjq2Mr6msk7eNQXkvPxZi7ds0+LuO1eHz7DaBrd+y5+I7XAiE/F1vuSb3yw3nlGh1Aex/7cH9MjktMxH35+vvnxMW82nb+q99t1959ut6XtEHnvd6HtpB6aRzAl/RXt+ulY6TbotvzXdLH6vne5V8r+8kvytP27Mf/2XiMUuSV42Z3/kVN3/yyXalP3W/5/tFYh5+s73Mb+rI5Vet4Sdl6RbDfCv6Oz+u1+JUvil35ibnR8u8+X3t5PkFS2PZR6pPF7cfq2I/dS+6RvuZ3NbYfm1cfewECV+dB9aHVYFm+f/SFqs5dy/OxB/NT6Zm5UIdt34/n6rf4/PRcf3Zp/6owBVfHuX7Pn7j/V+b/c5abJ+3b/xA+J8iyv27X1CfT61P3eNPYp8efvW9X6tlbpdq568Dk2fQd799H871gzn4ny9lf0XL82bF7Qd3PzingBb8bL2rex+v6WPqM8fo//+t/8IML3Py9v+jK//aXP/mdl/vTP/7lD9434EdusfQ242f9A1lxiqtAEiBVyZMoa4JCfczZVQVRUbr0NiFmV/+KT6ZYj8SvccmN4ikgZkke1uRqKQWAXCw6R8qfFqRQmsv2V0zAq7cJ59htytXAV/tUilrlHnhKflK3ZX86tijNrLbpDKlZekO0nVWpfAab6uJc8ug2yljoMdeKkMDzi3tR9NOqlNp6a9h6pJVRxa8wsdUlsPiS476v7N8pPqmbdqBZBgBUy+n+3aeBt1BnL2tALqg+iXWjgduj+7YmW63G0k+x+OmxEL9JZ8vGl1JA9v549QPkdkkfAFQrqJwTS47s0Ov2ik+n0INlToiFVKjcekwc15PYUiHjWX0sZXzZB1MUCrX/pVb9c26bp449W31F7AiQXfztBo8ADLEMSrniX0kUQLMBcdU/lI9JTFqxBGhLmezcS516zui2GnWf9/XtNxByUn6NIuaUmxVW5yuFlRP3C0X1HJVMiooiiqXP5Y21k+cpKzOC75MBmpgWsFmFpmgrrbCwZVB+1SWmrZQrz6VR80EnPXZZCX1pS0etn8tHQY39qs/Ld/GnFMueWIXq9cCmnnZy0zTklWLi1knNFkmJ3UsF0XXiFwpRnXR5Qw3UdW/ElupJHkOtPClgQlslAEU1A8U61I03peUvO3CrAejeeiLWiIxGDdRAwpUWh1VR3TZtVwCncNurNYavKwXksykxR4EtNY/FiaRuofWJEqURgTWnqHzS/f1YST55TuquBEg1h8e58D1pdfHwOxaSUq6x5Cv3+St4k82G8qh/YHQ9mzAScivEX9HgqY8cDfbGj48uUn3er7zZ2tbGxDRrkDyXQD1WrUYyPzcd24FgObYDoRuQKf2SZ0g9UjXJHJRxUvd3D1jlvkF/V3mNGucKNq8A7n0/Nns3ul11gK78fUnaj+HuuqubC58qT9rwTJl/nem5/a4XpRde8+I6Pvce7er6Q/qrTz9qYLlmh7fLEb+6vELKFnf9BbkYfDsdcVo6/PT4iEsMOIYF76cRX42P+O2Zwkl8MZzxsPYs077g2/MRr4YJp6WDsxm33VzDFfzL96/x+nDBIlRMR+xo8W/89ftbjB3Fm7zpFvyLD6/x5kA+TvfTUCXuAV74swjNvHr80at7pGxxv/ToXUJwCRYUkuDryxEL02zHsBIVdgmV4gOQv6A1BTfdjN883iIXOnZaAn5x9w7v5gPeTiN+dvOIt5cDlujwapirT9X/9+EOQ4hI2eDQrfhm6lHQAOSxX6qkPwDyl/SpSuBLeaKoOythnMsSMHYrvnk4YujWKmO/RI+xW3GeO9yOE05zV33tDgPFRPQ+4c3hgm8ej7g7XPAw9Viiw+vjBXf9hF/ev6p9kNAozhQEnyp1CwA+nEeScvct7MHNMOO8BHib8e5yQE4Wf/vnX8Oagv/3my/hfaI87Dv5cD7URbO1Ba8OEwDgNJPfl6jw5mxw6FdMq4cB8OZwwZwcHqcewSVcVo9Dv9T4i6epU1RY8tG6GyeMYcVvH27qfFkWj1fHCXN0eD1O+PbxAGsLhhAxR4fIisFCW3s4DQgh4TgseDgPKAXouohffPEOby8HfDiNCCHWMAAxWcwr0ZW9y0SNZTAn38UXa+wXzGtA5yNRipPF6dLjMCzwXcbDecDrmzPePx7QcbiClC35DvqEoVtxnnqM/VL9FadLt1EdBsTH0uAyBdwcZjyee9weJ1zmrioMd/2KZXFAIX8z7yJisnQMwBevzjjNHZVvCnxI6ELE2K1493CotEQfUg0bMM+hAt+hJ9/oy4V8mPo+Yp4omFzXpVp36CKWOeCnX9wj2Ix/9fWb+oyGLlI8TAbn1mXc3lwQXMa7h+bnJQJG6+oqQLy7veDt+yO+fPOI9w8jkqKUf/n6EZeFhLVidLjpF9w/jhUEHg7k33p6HDY+f/Jc/OnP3uGX394hRYeuj+hYvdragi+OZ/z67SuUQuEMpnOHw82My7kn/7spbDYAWjzPgMNxxrJ49IcV53OP8bBUYaXENPPxsJBP6GmAH5rKdOGxjNGRaNPq0I/03ng8DTBMcT/cTvR+OXc43k6YLl0F8o791lIiX60//em3eHceeY5ljAPFfX37/ohupPsbee6//vIRb98dEQ4rXh0nfPvuBl9+eY9394cKho93M80lvr+Xuas+lkKtP0jc1NmzP27E6Uz+pOL/mZNBN0Ssq8Pd7bnSiAHysVwXX0MwjOOy9VM15Fd55Huc+d6+Ok748DjC+4R56uADCT7F1WE8zHj8MOKrP/qAmCwuc1c3ayyHAJE4r4fDTH61x4me+RBxWT1icjidBgzjgteHC759OCKEiCFEBJdwfxmqcNh5ovKXOWDkdk6XDs4nHEf6/uHhgK6L/A4loSpnM87nHv1Az3YIiZ9fEmbyLGK1Th6H25ko8skiRwvfRRzHBcYUPJ57jMOKx8cBNzcTLuK3GR2+eHXCb7959cJVxtM0HBZcHmnT2nUJljdhvM84P/ZwHC92PCxYFoqPW/2Lb2ZSPF88un5t1P5oMd7MyNlgXXylp4oImsxTAJUunKPF69cnPJ4GwJCoVX9YsC6+zn8D0HPDglfduGKeAjyPRUkGviMl+GUKvJFCGzHO56rWnrNFjga+j5XKbRzRjdPiEfpIc3siF6DusJB/Kb/b4uqRZodwWLCeOwaYhdXZAd/xZujimk/n6qpl1bjCPpbkS4rZVbVvOQeA1M4lbAjQANpqmo+k0DxlE0aORUMiWwLW9f55RlM/l6TBrIAd2VApqg2uNNVyvXmjr9NJkJbeaNmfz2a7OSUbR3JcXy+gPtOGEFzb5NkWa6p/Z7FK9VvGR9oL1PNIqnx1/iow1lV+DDirza2Wx7Sxle/783rs9GbDHt3/HqQCIP2I2aKfSj9qKuxP/s6X5T/6R3+/xkwUgRnP8fjOHNeNLJG5xuYT1VeJ/bckh4EFaja+aKz+2ruIOXk4sw03srJ/ZGdTpc7GbGt8wVIofuLe6lfY0iSWMgB1oRLZbyqztU4rrwpw0NYpsTwltq4C7V0iwjYSUsWzAmpiq58xpQr36H7pz0msGUD1pRFrmKRcAGdLVWIVGqxY5iRem+SV4wJatOVOjpdi2MKWazwzY0r1M5QFzJ4eqtVRZVylH9SmsrHKObVABICBNwh0zD8d77EA9YdeLFGb8WKrIABlYSs8flDWvG2cMXkMJQaZ923zwNqCGC20yIVYCKUumlfcJ7YWSl6ALGeRfUa9z9UPTdq1p243X692nnzC2ku7WjMZFJJlKVfL4ZM8ZWvxFJVSqyxt4nNWfw95EW1d2oSMEUtbsya2H265TsLKSD0FoDZlEpTQ9xTSVz2eomgqi5ZrFktTWMmURCwAVCELugZVDVWuK+y/KAsosgqY5n8kY5EoNE1aKDSN/vHOPB+oQmpDDV9jCgrPUSMWy90Pc1kthcExqj0imhHlnLI8JqX2asvmB7zWxfOgDqpe+PC9oZtlms9SvdHyAJitL1TatV0vyuS8thrqcwW0oNOLRPGFlRA3QGufLCxZ5ZRC3XD5Zle+Ulx9kvS5oixc+2vYwmii2dzbIu2Vqb6z+EmqAiDSh4Snaq9yTsQ9dFgeGU9ZCPNiW6yRVbFVLU7luOGg7xu1zn2oCF4US6D44sBWRf7OIUnqVMot3yZ0ghpjXb4olOp7o+syCS30jbTNgEOjfP66R4u7iPVVjLpZ15PVmlluhbRfr4u5CLObN5K/Lh32Vm+DpgIr35WCby1PjaEovV5TL92ExEDLczWEhiS+QOZadhQWaDM3pX6jwgsBjRqs26Defxp81NAk8r606vryFDvscdPGYinXfSLty9jci+fyS3+g6tDHPpWugayyazuu4yqZd1fr02P6TN+vWvP0O+J3Ud5zZejvz6Q9ZvxcK+o//Yc/PBX2L/6iK3/5v371Oy/3j//kVz9434AfucUSQA3STsHd2y9QLE29VeiqQlUDgMHFCga9JbqpppkaUxD4V4oEH9oslsWsBBU3pjQ/cj5Xw27oOkqrowZQZ7An1jQBh/u69HFZ5GvAY5SVTsrvfKzPrHep0h0FpOmg7XJcUv2sqJaaAqfzCk1JVEaNAhRefoDrmMl3Hhe+xkJEIVBppLXvLlXAKtZQANUqtGkXjw1ME7jQYhMArb32KUgwcFPo99s2L1u7f4NJP9HehbV8tlzUsbnymdrdwKmAJwBsOZN2t+uaMETrU/vcQGVtoqM6dDkSgJtEIbDxcWvAUs2vHc3WmNLobKYF7raWYn+JwI8xuZ430AIcRoULyarORjktJtd2UD2ANYkAdClb2mO9/5RHJ2MIPIq4hNB5UQyKMU2wAjxf+ZqN/6ZrYyN0zApkeXPCmNKk/DkkhjWluTBJHDugUcb4W+1LBZnbX0tjadVtfd7QMVEMbHj6njCkSsN18NhU5cWybUjYlmlMC/IuYhECdKnPmdub6zGZK3JM2iFtJKCi2q1BkIQs2a8QBGTKcSPgvM1PyWOMLHjVM7CnOJayBZ4CqCyegiuPBiLlr0cT+9B5tQCGtFfO78BmBYr6uGl5i1wvZemxwm4xq+fnHtTuRDk2dE8+VzrhrmK7kJP5YdDEQq6AZulLcdx2XkwXLrMAbXxkKijgg6LeVVZdC3Vc7qX6i2t5du0zheJw1vN+2596qVNt+oxU1EZF3ZiX6WxbmzWIrG1UAEr3BcCWsl4r0xeX7TH+DdjQfOU+68dKjWENRfKcj6Vul5qKewzQ2ifPRnuWisFGZKFObwOiv1/p0ieBSnk6LATkzHNDc62I2uSrJ1RbjZqXH6Ve7o5de5Q/mV6a7xPpGvC6CkA3zz2u9u2T4O0F9+sl7f1O6WP34YXps8DoH9J3Tj9qYNnbiD87/gq/WV8R/cstmHLAJQVcUocvuhMuKSCYhMfU45Wf8H4l+tkrf8Fj6hGzw+hWvF9HvA4XvF9H9DaidxGjXTDngG+WI279jDk7ZAaxAMXRdKbgm/mIwa3IxWJ0Kz6sA+4C0SUfYo/Oxk27nSm4pIAlObzpiDI7Z/LXdKbAm4zeRrxdDsggK9zgVizZV/VVAHw8wtuE3ia8W0YABJqn5PGmu+CSQv38YR2wJIqzKVbY+3WosTsHv+IcO1alJcB+8MsGcE8pwJuEjmmoMTsMfsUUAzoXq98pACzZw5uMSwwILm1ihA4uMk15xjl26FzCJYYaViXYhJsw42EZMPoVSybF22OY4W3Gh3mscS91uBHxQ5WNgIeVKEuyCQBQ6JaZaYVTDIjZ4k9u3wMAfn0iiu0+j+ONAQA4BhoTUf6VNuh6LAqOYcaSSSFYxlNiisqYaT9TYwpuuxneJHzgeykqsMew1Nimj0tP1mabKOQO1y99P68BzpQ6lgCFj/lyOOFx7fGwUMgYbSXXFnHxb3Xqu/R9DGu1hEv9EotU1JVvwoLHtUNQ1uKV+zio2KUr06ujso53HDNUNmIWjoN6WQOO3YLzGqrFvGMlR0s0OQAAIABJREFU2lxaPFLxwQWA237GHH1VbfYuIVgKj3BaOki80+ByHQt5vqRMAHUeBJcqNV1il4pf7sI0ZWsKvj0dKsDqfMK0hLoZYA2NoYTjqe8E7q9Y8GWsH6Yet8OM0xKqhR4AbvqFFG+5/8ElnJmaXQrFQTWm4Dx36EOsDAC5/m6c8OEy8DimTSzWY7fUMD3eZUwrUdfn1dfQE5IKGitgTQ5DIDqydxS6hdgKzXcYAHqmq1+WsPEHlrKE3i5hVJwtlV4ubAVrgGn16DjEkYyNqFPKdwpV5GtM2c6nOi7a/xhApal7R7FeT1OH40BhgcRHd+jWuukiqtTCVhDGgoz3yuMZHNFMJW6rKPyGQDRgKVM2cFZWehbfY6HxA83PV2LdyjsiZYs+rJhXmmtCj5drgk+Y5oCbw1TD/+hwI1otWcL4SOgfmRsFwDwHhJDQhxZ6x1umka/tPRmr6rZF11E560pUa9nEk7i60kbHcWGFDl5VuNem/Kopx13HoWxYcdhxzFqpy/uMZfE1rItQ3Puewr98bhJqJylC57papbZyGBduT93MZkaEYzp0Tg7WEbNBVI+9nMtNzdgo5scTobjM4UNWj2Iaa4HUlLFBgRL6Q8KMGAbA5MdNqC3HbTxb40qlvBZmGRjxTRbwbErzTS7EpAB4k037UifTYtuuakeAaZvat7mimowtSpP+ZPOU7iqIQdgQewQh5ehx2W0ISSiOTaiQej2qpVeLWW0srkaVrcG8UewBfd2+PZIEPAlwrhuQu/MaZEm+SoXd9Zv7avSGwj7pMvXf5zYhytNh/mh6SV6NiD+S9ln2+5m/76mg/I0ON/KjpsJ+9WdflX//v/tPYU2uoAsALTJRcI704xGLRWdJUEcvxvUPqvgHykJaL9hHT6FJBLAIRTbyInb0a6W7SlmXlWi4Eg8SaKIx4mfp2NcSAILNdeElVNXeN0AqdFVntuI9EkJjTRaD8mkzptBixhamwjp49sNb2V/H8EJLgI0WexGrp8Q3lAWr0FazWkwJVXUv4iMLF+8yUm5iMs6WDc3VC81x932Jjv343IbSW3gxL5RW/R7UAi1As4hW6yVQRWw0Jfk80eJxYH+0VPM0yqtYFluYCba28oIFwIYKK/RVff1TKuxWkTZFWxclco1zmWPikWiLiL7UWGKm0VZRDJxvsfwcL7RyNhzbr1R/0MKLeU0v3fNt9pbqSj8GmgS9bYsZawtyosVK4TJknsgCqor3yKLCbMdXKJh6bCWuoSyCDI+XtLUuYsRCB6KmGvHj4Tx6gSRUVpQWZ+4JFRZo1/MiS8oSi54czzM9K7ZP9UdUL7qqX2K01NaguHLKQlevjRQfLs+u0lbpBGhxJuszUyhOWmhWz5K4Dk/01Y1VDyBfoY7GoEi8OOl7NEDgVUPmheVqiT4a7VNro9BLLdqCTyiqepEnViyhWDpVBsCLGLWwkHKENit9FQEaV65QYbGpyyyWArKrxaopBiVoKizPw8jHi6FFps8wq20B3UGLT7kHKCBKpFpwFltaHrlfQskUSqtYhGUhu2q1FlTxm7qAFWO8LFzlhadj6pld/D2hO/J11J8CsxhoKmEd/qzqZeqo7Ifq2HfZ83Gm90pwesNU27ra48VtsXS9kbEqbQyLL0/vgVB2RWRGXSfzQ46Z1VT6JgAYTbO1aPTctF2v2gjkz8eVVJ5YPNVjWftb2rmNlR6AiWjjry2aBpXGu1n4qzLKbpoAgF0Nsi8V+GhxIGmT4XZR23X8w8IUShrTLFZ6edQ4T41jyHPsSfv0XFOUW31filXtU/56dQ5qXz0NXjhfBTHSF7M99jnpKgBRi4kn5z8GyvT1+zr0+O8A0FUgJG34SJmfskw+16+a77m8ejG178fH0ne8By8CfzrPlTZ9bjv+6X/7w1Nh//wvQvlf/gqosP/an/z6B+8b8CO3WK7FYUoevzqJeM+EWCwe5h7nucNXNyec14DbfsYvz6/w5nDBr+7Jaf+LIwl7LNHh0K34V+/vavB2bzOO/YJX/YRLDPh/vv4St4e57k6LyuktB4D/F2/fVKGSQ7/g67evcXckS+S3jwf0IVYfRID8ET+wUMLPXj8gFYMPl6FaDrzNuO1nfPN4rFadsVsxR4eFRR0kjf0CZwteDTO+frhBzoZ32QP+5PUHvJtGPFx6/OT2hHfnEWt0OA4LurAiZYu398capLwPEQ/nngCXyzV4d/CpxlW8Pw3oulh9HJdIgjTnuavWAwEql0uPvot4d08iDYEFItbVYRhWnB5HjOOCx/NAQg+LR9+vOJ0GWJdwdzPhw8OIGxFtWR1uby4YuxVfv79B16W6Cy473t6nuoNeisHjuWfrQKqxBsdhxcKCHZfTgBwtfvHH3wIACa+4hOO4VKGky7ln0ENg6HiYKxgFGoAr2XBwcdrJ/+LVGfPqcZkD7547jAOJmgzjUgVFcqK2iLDLGCK+fThS2clgnUk8ZV0d3tyd8P7+AGML+mEl8QpWDXWuwNqEy6mH8xmHw4zTaQBAAit//OVbvD+PeHgcEboI3y9cB4lJGDTBBgH6zieKR8ggfhhJkKILqYL+6dKhHxZ4l3E697i7O+P+cUTHQjwxWhojlzHczFXUZZ4CjEuIs+ddcRLgAIDQ8+bC5DEeF5wf+41QS04kJJFW8mXuhrUGhheQePf6jMscsFwCYKhvYSAxmPuHkXbsI/kxhj6xAI2vvoLDkcZnOpO4UBgi1plFjligo2RTg45/9bN7OJvxm9/cVQDqhxXrJRCItUR/vfniBGcKHh7H+hxbVmdOq62CFjdfnvDw9ojXP3nE/f1IfpX87L/+ySPm1ZMlkoPan+6H6gc53JLIzOV+QBjJbzgvjnw1Afz05+/x229eIa8Ofow1IL2xBW9uz/j621uyWI4r1lOH/m7Ccu4QXs1YZ9+UQ7OBO5A4WZw9+tsZ6+JJtOgc4PvYYl+uFsYC3e0M7zMup44sKMWQpSMb2DEi8wZCmh3Cqxldl3B+7GFsQV4t+lfM8jh16O+WJjpiCwVaT6aqwf7sT9/i3cOB2mxo/nqXcf/uANfzpsviAFtwd3fG+29v4IYVt8cJ798ecffzB3z4cKjWlP6W3vlidbuc+7rhYW1GXByG40KbfYsHTME4Ljg9DGyRorlL474iRYvj7dSsTACWOSDOJGJSisFwWOpmVlod+fMmg8PNXC1c8m68fxxpI2rycIEZA9FiOCw4vx/x5hf39ZmVZ926jBASlsVjnTyGmxnz1OFwnGD4N+08d0jZ4vzQIxxWvL4549v3NwhdRB9IyOv+PKAPkWJJX7rajp6fo/kSYF3BzZHYPB8+HBB6sqzHlcRsvE84PQz0rK30Xl3OXZtrfM/SxWN4NSOurlraXJ+qUJC8Y08fRhzvLricexKsWR3evH7Eb397d3VN8ZLUHxdMDz1QADekSqv3IeFyP8ANkebpgd7PMvcB4PBqQowOcfbohhVZxHtWi/GWxHvi6uvmmVe/W3HxdTMLoI2z11+c6D1iCvLiMNzMWOaAHNsmmHMZcaalXndYsF7ouUzRoUQLP6xkKb4E2kDijSPXZeTVbp6pMJLgkGymSdm+Z1bE5IECdDcs3lOoD3FxyLODP0TEU+CXXqkWUN/z+29u4j1ltW3jIeQax7KsljbGZMPHlWbxjGYr0CMATfqV1AaeiNzwMRMtbSZFS5sZ+w2vkJsID7ABwHXzRQvnyMalL3SdBozVf/WKddSgAvXqn70H3BrQ640j2VyqYLxtZplsaNNQ+4zrVHZl1w0sPBHvqXVnsx1nXc6+7GsmRqjjCiHXzQyVvcjYqqw/2lRI1PpvavpRWyz/zb87lv/mH/8beEgjElpQ8LV4JFg4ZKyFgM6cQw39sRaH3q71mEPBVDyCSVhLo7vK9zWTcI9OUsaaffWjTMUimISZy7IoWHfOHM5kpGKRYWp+AJu4mULjJHpsuz8bSyU/6Y7fQNa0uhxy7X8CU+VMwpyZxqesklJmgq393bSXy9+Pp26P9hvVx2UcY3E1Nqj2VaUwMKmeX3dOM8FkzNkj2FRDpEhIFR0ypY6bhEpRdOUl+02dur0Aap09lxt3bUjF1PLkrw7N4UwL36Lrz4WEmwBUazdZqlMNdZOvvN0l3EtkOq1QoffzQH/XoWj23yU0jDWlhq4Ruuf+fkk+fc2188/FW6V2O3Q2ImZX768cB7CZB/uYsVKu3C/d932d+/mmab3ad1iH8JE81pQN5XXf1/3xa/FdtWVePgemzq7J1T7tY+LKsSo2tcun+6XDBAkjovpwq7BAclyHAZJ8exaCJG9z3bTSFmm5divOdf0eXEvSDqG0CqVZ1y9zT8IIPVdeQQuDI6JcmZ85+fxce+TeaLqrMFTEn10Lfkm79GcZOz0WWixsXxe1E9jH8t3H8N3Pm/34l7JlCmgfbJ32bdBzcj8m10IVberU5aIxPOS7nNdMlSfUTNWG6npYrvf7uXPXklYg1lZ3+bwXGbt2TStrW8ZnJ70Y1oevCMrVc9ga4TZGGMmvy5bydRl7DPKSvnzkXlzLd72M/b2+fi9qXt1u+aw7va9rb5GSa6+1aQ9Snm3zy46Zj9V17fiT7y9DOC+20n3MOvcd5+2zdX6He/3Z5bzg/Iux4ec+r1eu+7//i//8B7fq/d0/D+Uf/xVYLP/2n/7BYvm9030a8Zfv/gIAAYRLCtVHMReDSwo1rzUFS3aVCrsk1wR2CvlT6cVmLk0VtrOxAhS9EBQVWikTwJPF1H4xDLQXvGVQsl9obhZgapEN0GJT8kl7Iiu3iiVVFtEz+xWJ/5peZGnxIK0AqxfY2r9Onk9ZCOrNMwFYQteV69sCuVxdGOp8+79Sr7RB+hF5MaxFfKDKkvGVRZ2zT2N/inotnad7tUZSmxX/KL0Q1PEnhQYMYLPwk3r3Y6gXE7K40+FR9m2v6rUiTIMt2BPLLH2+vsirAjO786IKKxZv8deqFFalEquTXrhsF488j22pSrli6W6xG82mDTqO5Z4KLPl1nfs4lltaclNpzVnHpwT7MbEVSwm/ZFZItS4/WWjJ9Zs2lOZzpGMmVl8goC6kMs9NoirzqWybUJB+DxRTrerXQKqACesSWaTt9j5v5j4v3PbjuO+PXrjmaGCZIlp9puTazOd4DCSepLGoMe1qrElDx0TcSCjCVU32yrwUldqqWLsDBJvr2VJnFB2yUpRt2dwTPcYyJtoCQ4NPfTVeKdjKuch032LIiuHZQqGosNXiodVn92mvNpuhaMHqnsn3tJtLWixoU56qy6jjtV5s6cX7hb0pW9ryfqFvsLG+VL8wTcd0yvIhFMxrip5KFbZSeeWY0FlFgXffH/EVk/7qbioLiUnKqmGoXRsqKNdf6ZdF+ocm8vMZSaiwhcvi6kHU6G09zc+OL06qjzXeKd/vyO/j3aMtqajs8t1qaimw9fWrQA+NGq6mgpwy/KGOG3+vzSjb4/q70Fw3asDY9n1Dw8zY0kJVG01pZe2xjZF3cW1nqd+/M+iQaXXtuv0zsWvDM/sJT8r+Pmm3l/A8Vfba3+9Qx9X0mf36XY7HiwD49yj/D+mvPv2ogSUAjG7BiUV4bvyCNTvEYjEnj97FuoCfk68COLkY3IQZEwNPAZWdS5giWTE7F3FgC885diw8Y5A5ZMmUAm7CjFwMC9pQXglb0ikLmOzQa7AVGTiNfq3frVJ2lToEaATb4jDqFGx6YhkTAZnXA8XeTMXiGBbMyTN4Wur1IkwDAN7HClLFZ1L7eQJkjRElWwE7I1sn+ivAUPLtLWpBrmERoGAzVvXXWxJUmZPDIazkz1oMjh3FDp1YUEW/U2Uya2AnPqgCugFg8LmKyYgYy+vDBdYUPMw9vEvofdn4vDoBogYYePyE9ttCk5Qaf1HmVSqmiqKUYjCwEEZvc/V1BciCZA3QeY75tvoKZEsxOPYLhZRhIRBjChzHH9UAyRpU/1XLFGYBdnfjhCW5ekyXX4qhRZgAQL53e6uO+L8aU6q/bUwOfeB5nBzGw4olNss2gDqWWpxlv+Gi752eK0KvGzqaBwLsvfiaFoMuNKucjGk3LDRvVHiYai1LIgjSgLS1mUPEtO8GqDRwDb7lswaG/ciUPyVuY22LQwfeQBJRlDU2q/t+M4CePaLSDTcTVhYLoby673SN90QLl+TFz5b7I/69Uk9/jJgXotLaUDZCLt4TLZJEagqFrelow8B1pVLzah8DgXkJb1NFV6KFc6VZlWTsXK4bBnv/YAl5QoDVwHSpblYABNTdwHRIpoBrtW8ROxHqduC4gVnmvBKIsSovQBTdZSb6u3UZcfXwxwUxurrSs6zUWzc05Bk2ze9XNoWyukb8m6kPW2AsftB14yHbFpS+YOOvrBU7NyFzuF55NnIS/2sC/sYRjVjiBepNpRo2h+m+1rGftG8WftkMEYql+FLDoPZLKMwlo24YFFVO9fVVlEb5LD6+xpRG+eZxlXA2BJC4z8kAIhDDmwXFgmIwcl3GkVCM8bQxooViio5H+F2TK9XPtyrogsawUi4FzIkAjdwqAePiG8zzHhltA0NvGGiUoUE4j0fyarOgoNE9NXrUoF3TQfcoSTaY9psOO4BYP9ey0aiS4mssfVN5hNZpdu+PTZ/y7vO+jdwXEqHh52kPwjTY0CBRgK3ZnquCNtq/e49uZMMnm+tZ9Njsvgug3rRJPd9Xx7Q8+bhJ+0t1lfvjVzeQrqVnQNpHgd5LgN0L8nwnMAk83Xn40SWD9HJ77Y8u/aiB5WhX/OvDN1iLq7TWc+orvXQtW0qaUEtzMUiwiJlUXOW4ACA5r1PMDt6mDc10LQ6X1KG3D5UqqemdUi/RXpulCmgUrzm1W+BtqjRKALDm3OpnaqRQKaWda7G13MDgVoNpq+J1aTqn1OPHbXk6WVOeUDIFQIvFtlE885PradxI6XMvXCTfNXVRzmursY4/SudpjG/D/IRqqdsp7d9bfKUO3UeLgonvw0+Pj1fz6HY9F+NSNgE+Rv3KMJVWrS3Pco+lnGGcNu3TY31gEHeNxrn/vgHZ2eIQVphOtUfl2Vutr5WpabIyz6xZFPBfkdSGxP53LBWDg9lSGYGnv3X6d0aP/SgL+F3brrEBlujR+XVjyZN7JEqcck7Av+2e9n3snqfD6s8C8m/GeXPMdts5IED1Zpyvxn6Vz6WYqrIa2J+7tk2xDBwD78OwbNqVBIwziJY+y30c+7XeEwOgOHpKUrbou9jGNdB4izppt4sBWLitIWznvYBfSRo8lwJ0XbwSi/UKhbCgKosCqYJ7P6wc83UbakaXJRb+FtbH1Lolj8DxlCxCFysQDl1ksJw2ZdJ5MBDOlf1QI7poEFpIJMt5pWbCFtSSyb9RYrfqcbJ+Ow8NsAG0dZzrgrsJXwGg6+X9w1ZMG8in8prFWZ5Aw4BR/Fmh5qOAyJINbS7wArkoIJN3PmjGlqoUKpsFFQxaZTUGgMh+pnK8GK5Ur7LbGJao6qptcRWAluRonKVN3OYS3XVL80tTVMpHGiQVHpPIx65NSw1o9xbn9cpCU8p8btE/WfY9Aw1VtChGgfV9fr7/NBD16OYP0q6+fd0aqOh21uIK7Lqd03LaRrPFis+BM3Wcbr+5Pgb8bHwS/KhyzZU+NUBm6t+n/Tab/K1tH6n3uWMaAX5iKj7bvz26/QjY0vsI8v1F7X3muFx/DYN/sryXnn+mzu+cvsej/of0eelHDSzfrgf8z7/8t3EbZizZ4X4moZJjWDD6FV9fjnCG/KmO3YL3lwG3LFjy4TKwqA7J3r85XPD+MmAU8YElYJoDvE/4+d0D3p4OCCyTv0QHZwtOU4dSDL64OeO8kPVzXj1eHy94fyJhjttxxmUJ1WdFFjt9INGDbz8cYQyqdLrIp6doMR6Xak1cWK5e/LgAWnAsq0dKpPh5ONLic1k8Qkg43w9wXULXr7icenTDihASCaAwte14nLCwRPoyB/TDWq0bImaiFywi3Z55IW4sO+93SSmp0ttG6IvDsGJZXA1w73zGupBYw3wJCD3J2/f9inkONf967tDfzFjOHWyX4H3CfOqAaBFuZ6TV0e660OayIVlzWbgYIIy8eNZqrbOrcuj9uJJAz69vAAD+i4ksHLMj0YJs4LtUF3cFQDrTY2NZcMD6zAHrS1PwLAbl7EhYoM9EJfQZZXYwQ6K/faIdUKYElmxgzp5UJI+p7h7bLiGfPJU1OZRjpP6xUqfxhUQLeMfZHkgABZODOURqy2LhPnjkMQM3kXbstXolWxXqTroFLZBEkZN/mUhlk4QSzGoAB5Q+AYslatqQYU8O+ZjaQsly+cnAzBZlyPS34+1jUei0pYojSND40mVaPI0J5uxQukL5XNkIOJjF0sLSl0oldA8OuS8oPfctUpvtYpBu+JgrdDzS2JWu1B14O1kyNvD1ZjF0Xj4Hqses9Nm/d2S0+CLW+Uf9ZVBRAJMM7MXCJCDdKlVYtiKU0JQX3aNFvEvw7x3STd5QB/2941iCFEfTztwnXqzYCy3M002Cndi60pW62O3eOZzeJBRfYCcLK/3JBv5scH5Nljm7GKRDhn90SGOGu1iknsdY5udM8yT3Ge5skTu6Lg8FdiYrRbGoSppuIspiGgvsSrRLia9nVhGhAHIA3MXARsqLTOfchcYnDfQ5d3S9yTRvqC5SLO3fWsRDQeoBkwA3k7Vjuc2wfM+zp2vDo8Fyl2FXGoPlVUH3wWC94dWo4fYUsAUGSH1BCVS2yUAJgDtT+3KgMfKzQTxQX2GordkBfjLIoSCcTMVQKEDuS20TAHSTqUqYmdVliyvwZ95MMUDxgL8AceR2dVwfaNzdBKy3Bf07Kit32FBEbaRxKAFwFyD1VB4A2JXOwQLxQGW5C322kf6ZRNfYSGWmjmiaqaP8ptB5k+k7CrDeAG6lz8VRPSZRH+yKqkybes5jOT+o3HCicSyexsCugJ8Y7/WAm6mN/sz9jVRmeCyY3zy/CP9U8hdgPfJcnvk+FervegTcQuPlLzxu4L8G8CdqQw50bXGoz4Y/FxRjkEMDXTZSjGCArhHKqNzXcF8QDzSncwD8pSAHs6GnmkTHqL0FqTOwa0FxlM8tvBHVGRRP96DW7QxMLDz3DNxcNqqsNgEpGCrDUBk0Ri2+r0nU/xwM3FSQBlPbJdZCu/D85mdJ+ifARZ4vmSs50NwQi+PmGtV3eb9Kv+o5vg6mHcuuPQc27cbatPGQpEFVbSfvwhRr6DPXQ30x9XyzZu6OyU+t3Z7fGJiZTmxTafmszBU8tcqqa3T/r6Vqjd1bVNsSoOaTevU4bdI1h2B1bg92n7Rhh5k/CoavbEj8PqcCtWH3NzD9qIElAKaieqzZ1Tm1ZAdEtk6wdUxi2c3JNfEVtVM8RwJQEkIEQI2rdV4JGEamPZHVQfy7CuboN2VNigq3sO8egM0ufMqWKY146mNlmP5U8MRSItROnYwBjN1S3cgvKZPqaLaVNlXzGMM+i7bS6bTfWta7nWwBKrpu9ZSL+lztCx+X90pKtr5BBFiDwStMG5fI1MB1pV8UwzQ3oTfR54JSaLe/vjT1cPBYyLm62y/jb1pgelFMzJbDDECNt9rRzsU0apfa/axjoe+vTAPD4EdAmRyvNKnSfvS0dcbnpv7BynYlc7kGLfSBwVbdzQL0q9soh1Whjj/ngcGJ9MGqsjbloF0vfeG5XsMrFAFxUhZ4p5zA0aYsVQ+Ffmh/qSDpfBs7rSFVxPrioH7RVJuBRklTfcmhtPALpdVfWUmmjUXNp9rdxprr12MjgcZV/tyrvsg5Xa6hdpZAgdyLmgNG/uM80n4ABJz0vZJjMka2IHc0/gYGxRQUL6uC1rdiCgzPLQKH3E9H5cFSe9OACs5LXRiXBpZ2Yy0AFwbUN8d/rfQTm/mWA2AsAxyZDzzXIfl53EsAPZ/yLNgGJOpnp9oBDsBuqfdxLMgBVVkxUxbqs9waS+XGkReAmUBrcQVplPL5+g5PFveF6zL8Ofelts8ASAImuYzC1yW+h2komwWWjFkdh6E958UBhU2jaZD7Sv2NIJANBrhZhfwAaAzTgDruokhpClA8bwA4KjCHxq7Jod1DAssEhHNPbc28CC8eSFn1Icscpk6XAGpbncNo7xJLC3uT6biALQG7up8o1KY4tHxQ80E2C2heA5HHzQQuw5jvFW4kMgDap03bZbz4GZVQIfHYjsvcgeH5zD9kFZAVICfTwAP7b9YFuQFw0/pC5ZkGrgrnz6aCtOIIuAr4pGdc3WfTrpO6LfuyFkfXS3tNoftdGDTKfaf+tnbb1O5HsUY9Q6a+/2xoz/QGtED60Nql54aMuwZ0Gyugek43YKkApsjzY2AKAficCDxmAYVyveFxuMagVr9FNaSQ4f4BW6pwXT+1r9WXWbWtmUQ5U2nnBXjuAaR8122WtJkzql/P9UWXdRXU7X/nrpS3iff5pD0GzybV13bBc3W0e6i//yH9sOlHDSxv/Yx/72f/DI+pV+qv5Eeo1SkBUljtbcQlBfazEhVJorieYl/Pe5sQTK5UMDkm9FE53rsIi4KH2FeVRm9TjZdpTa7+lwA2tNxYCMQOX3JIhtIohhalig1pMZ5cTO2P0Fm9JX9Hb3KN2yl02cGTT5r0USt1Sh8WRQ8WkSLtGzmw/6a0XUSPtOJmZ1OlakqScwCp5TrT/EfX7KrSa+8i5uQ3CphLpjqCTZiTr/TbXAyCpXAilxiqWuZGHXSn3rvU8Wp0Ri22tCYa45uffAtvMx6XHkAT9ZE8mr4rVuN9fFKpR4717LMq9FIRdtK+pKQqy+JEQI1vKhshNH4GgeuX64H2TteiSQZEczSmbKjX1hR0jsKD6PNyx8Q3UQs2iZ+t7qOIPWmlUf09JrcJO6N9bskvM9e4pzJ+WijJ7ShqKRsER/6o8hfAE8qksyJSpXwsfap+rOKDZg1qPi3C9FRN0sC7vBFrkjir1jSfRP0+6HyCMQXQ12D6AAAgAElEQVTT6uvvoL4H4O9Sn1BUddK/qaLcKuOpaa57dc9PqcIWYOO/KWUKFVX3P7hUfY+tzYjRwfvGSNCbTuKHSZ+b76n4UOpzMrZSn44Ve028qAk/YeOP6VzzH6WYsE9Xe7K+oDBDtm6YVVEsFkQC2iaa9wng8EXOZZTVUYgQ8V0tpvpo6nq0iJQwEIwq19gCk9qK2fBmlxGRGkeruDqfRdxIvnN/sdtULOJPyOeyopZuNtQAwGaU5BD53aXVS2Xcqx8lH8/SXtvousIO0e9TI9dLmUBtx0ZcSfZGZCMm7c7JYjtv73/dKJMVrkFjOMi48LhD3QdU0Skl8iQbluvTZ++lybjcaLjqfVVjwko9ldKrboSIKwnY0Ofku160XwFJdSyApz6Tz5mB5GdRRKRqOdiCg2toZG+q2pe/LwPqu87DfaxxTE2L1VrUs/Hkmtoe9f05f8hiPi3Ks0smFwZ4PLeuZdLXXxnf/bBc+/6kvJek58Z2d73B0y5ehW4fKeO5MXoWhOryXpJekPdZIPv8Fbv78RHA+nuW/uBj+XuarMkY7FpDdwBAMIl9Dx2FzzD0A9LbdQMoqyqqybAmw4VSfRipnKx8Ett1udhN/QBw9EtVorW8YBeAI9bRva+fLwbZWvQ2EtgrW5XQpyEy2C/Spt3x0saChYC8ycgMOEWwxJtchVNEWCjDVDETAT20v8vjY1uIBup782mUsiSvVUC85betHtUfbzOPNf213G5r6JjLubZZ8muADaAKFjnkDYCRdmqQu5fzt6bUnS0tFiObBXKNFtbRyak5pJP2QZV81hZEHmPHgNxwP7WfpgYMADaUZ88g71q/SjGwZesnWdV4bYbd3KNSy/WqPFmE1kW36q/d+c5aUzbng/SpDk6ieSz1gH+vuH7DeaTPGnRq/8hWXwOtjsdM+4ZqYEXXNrBlTEFwW4VmOQ5YOI4duZ0fDWAR+G7POZWf1GcCnuIXKffIq3Y6pa6pBZYAEkLSdcnnOpS2oBQCwwIkt4CrLcyMKZuXeS1P91vumxp748rmvSO/6AJGDZi5of4aZYXdt92aQvRYUwCXVXvb2NZxsAKy6VzrH+pfu3v2JDyWZT6qBovbfG2DwJgCa2VcCIg51TY5R6C1PYskPKR9QLc+jlJeUyrmxTLXofNulGu5b+3zdg4SANqOleSt34vZ1iHH6++NHowGVFu7W/lPFIb1X6CCyoJWX+EyIUBSjfd2fJ6uEJsIlQKV6u/TNmGbYdc+AE3sB2pFrwCpMajHN4rOn5OKubp+3YTbkHr2wGgP6vZ/96+ia2BSn8/qXOFKPta1DdDdIZYn5Zunx+sxPAUh19qPls8Ug6L4f0ZUkQtw3afxmfIYPBaIKuyVcbmWnr4mmlV3fy9wvdyPAcT9uSs/K59MHwWD+vOV8szu70frVo/5Bsh9rM0f6cOLHqdrGx7PlPWdH8/v8Tj/EKngD8Dy9zblYvE2HnGwCxKAb9cjrCm4cTPeuDN+Ob9Gb1c8ph6v/IS3yxGvwxkJFm+XA0a3orcRp9jjNkx4WIcq5vPNOmKKAZ2L+NPxHX413eHol2pFA4B3CzlafNmfcIo9WShjwJvujK+nG1hTcBcmXFLYWCSX5HDwK0a34l+eXsPbjIMnAZQle8RssWaHN/25Ao3zOsCbhM5tgeX9MhDVN3m86Uns5z4OOPgFvz69whhWHP2CX08HvOoneJPxsAzVkveT8RHn2IEUUQcc/bKxtH6Y+401cmQrqKjJepNxWjvcdHM9Jgv53kVMMeB1f6l0ZTl+iQHHsOD9POLAirXHsOBh6XHbzZijxzfzEXf9hHfTiENYEWzC1+cjlujwxeGCOXoEtsKJBSwVs1GkHcP6JPTKaenQuYQC4CaQH+s/f/sGAPDzuwfM0WOOHp0nJeDBx82i/tvpQGPRESAOrF5rTcFlCVUZ9jx3cDZj7NZqcZtXjz5EzKvH0K3NosTtPs8BMTrcHKYKTHof8fY8kqV27qroiyj0OksWuJQJvh/7BUt0mNeAQ9+USh8fB3R9xM044yG6jdpp4PAtssEg1jkZx2rhXj1CaP7AxpAozLx6pGTR9yumS4dxXKrqqZQfs8WyePT9Wv2Axd9YlGIXvkbKDiGS322/YpoDQkgVFK1sSQNQVUydyxU0nE8DfIjoOg4hk2z1D+6HpYKSGF0NQB9CE5xZOSB515PITIy2nhfFU2vJXzh0Ee8eBpRscHx9qdaquDp0fROEydlgmUgNpxubmI6oljpWVQVAweqPC96/7dEdVrbO0bn51ME4onUbS8HRw7hWsBEXsnB2w0r9YCuc1LPed/C3K5xPWBePvLgaCL1MDu6WNuLi6hD6WIOqx4X8k4UaL5bHkgxcyEiLqxYhN0TkhYRSDINYAEizA6KFHSNZfrgPxqCqjpbEYjOc1/Tkc2xcQb5QAHk7JOSLh+nTxscaACwHXseHQD69HX9nX1xz5LoLyEe5gPybb1aUxcFMFriNwKNHGRNbjwpwcYBhq0sB0b7Z5xjZkM/z7MgCw9Ypsxry0xXBG/ERnslH2Fxcs8TVMkv1gTaLaVR4FXbFzAo++gJ7MchjYSpsaX7DrsDOFvmQ4O49UadDo8KimOpbBl9gJ6LUim+uZd9EotOT36ydmt+o+L+JX2UBUDryFc5dgZt4jnfkO2tX8gmMR/LFBaiNZjXsi1ng1qYgmrvWZ/EbzR3514o/LQzIf3qloUwd+VymgX0dlY+l+M9+bnJzoyHbaGg+McjLfaun+lAaVAqln2jMycfSILNrQXGovsPZlYbvo2lUe/EnZ5picUB4MERd5zF0E/lJZqc2tFKj1bvZVB9o7ZsKoFLGq28m123Yb7b2Sfnqmcw+iaoMGM5neJ6yf5/4lSam7opvpPjOFoONb7H25ZS6xF9RaNOmqHIMaCyv+Vi61g7tY7lpo0MNJSN+pgAa9TRy+eYpIKufhW5rUOeF+G9KWbVvpdW9KUNAO7btlVTrlv5I+7QPqbleph63ZwGn2jAwpVSq8JN8APmSlnId6O4vMWU7bnrjYV/3ftNC+ntl7K+ma2X8If21JVN+xJzkn/6dL8t/9j/+B3iIwya8SMwOU/IYXKxUSR20W2iiUwzIIAqoAKmJQ3Z4mzG4FbkYPCxDtfRomqiAvEsMFOIjW/Q+4rR21fKl41TqlJgKewwLWVhVPlnIS3gQbalbWTRHUmdTVRqdk9/QGQWwpWwr5RTApi9T9NViEVzCxBQ4eSYDW1+Ezik+qtqKKOBur54qz/bCAEjKFepmUvdEqIGew3E4mzn8i6/0y5QtOh8RbMZ5DZsYlwJmxSoo7ViiZ2uSWLqIIikqoOI3e8uqm49zVy1dQtWU8B01HIPLXLZ7srvuXa5Kn57bLeE9crbwLm/KA5oqpzUUisKasglZEbnfcr2E8pAy9PXGlHovnLIqO1MqAFyi31hPBYhr+mzOtobb2FssCxqlU+6LlBezrVRKuT6xCqgxaPdMh0ZQu/hi/ZEYjylRKI0YHQKHTxFqqwCtSnNVIUAAojbmTGBSQKS0YeHxbbEvr1MuAWyuF8Arn6XNOZuqNDpNoVp2JFyHjhfqPT1TEh5ErGJCqxQBrK6LmGePvo8MnNv7o+taGBNRRl0VZdOxGFOS0A3AxlLT9xHzFKifPkH8qo0pCCFhnsiJTHy3vU81vEdRY1zQLIZN5ZRVRVP7Lu8bubfOEWjV1s4CbN8hmeJ4yn2RvllHq8is5pHML7l38j10ESnyJogtdQ7E9Wn4D+cT0upgLCh+6OrgAv2t94nHta6FitncFwG/AAjkGlRl1E19nI/CjeStoUAEyGQOsb89eHNC6tGxRlEoX2IRsZJsBfREEyXFVtfz2KWdpd6gip4ZT1RPI+FGhP5cgLw6GJ9JNC22/hmDWmal83JfSMwMLdwIW4Xz4tpYCVXVlLrZUOmkorJaTPPvlTijYoETaqxji7EAsmhIHCxapoHyRsL3DjciNwdthWu4PgvU8CG8AVGtNTpOag03wueFCaAta5ouu1d5LWh9q/nL03ySt+BpPFXt/5fRrIhAjWUqvuAiGiXWx5onqeO5bWZomqsphoTDHLZqubLQkDYIrdeghSXhhYQMoSnYCE9tTHW5AY9ahQZrO7Ci780GeO6HsGALWKVd+6Tr2LfryqlN+3f1PVvutbQHZc+072rfrpXxkuP4BLjblPFcpdfT9yEUfApU/l//5T/430sp/873qOF7p3/rz7vyP/2Tn/3Oy/17v/hXP3jfgB+5xbIzET/v7vFlOGHOHgmW6a2phhORsCF6cSyhQNDhSRiQGuKCrXSxWNyFiX0Tn4bUWIvFlz3NYCnrdXfe5NEhPvZpX54Oi3ET5hpC41peOXaNkil/B7fWxdoxzCoPteWuu2zadxt2YQ2uvIWutQPhaZ6PUVQ1QN8DUkmlUJzPvd9VhsEb1+JrCtVW/5UkoTmetD00+q0erzcHkkK8Rs/ch7QYuYw9/UvXE2yGDbtwFeHpWOhUQFbH/THJ2+1ii+7TiPXqcQLmCX1ooRYAwJhW3rVwD9f6JXm9i7vv9Mt76JdNWdsy0+bvtfpq+I1An3ufkEujj+q2br777XfYjG73livFYNy1T1LYMQL0MfGjlM/S11ou/x2HdXOsC0/zCHD8WCrFYBhW5GzRdXHT1lJMDefhPR3v++29AFDB8T6lZKslVUClFzDKYTf29bh+3Yxx2c0FI5Z9lxmYR74HlF97tZUC+HA9TIjj81a1fR/yA6bAMaXf7dwDahiNAg6vUVqsSOBp7MgKbmwF5PJZC58BSphMCjNP6bF1oaoEO3TMyQqmGAzmPRAw2NKN1S00YtlUx6vlWGIrqntAY4UNuJQ6yjUAIsCO/5ayxTmG/RaTCrlRZEB4zKX8emwTgoSVu6UNArwg7eE2iR+e8k0lNRuFLlRfANC5ZOr8qK+Z2W2ulf59dlpb36mO3ThKvWvrW82ix0JUs83uu0omNnAGNEBX0+S2gCJeuad6oa3BmtStX0MVbDVgZxRwtbvyxZPX7PpiVD6jxyeqOvDMbbj221O2YUWKIev4pi8KVH3s9m5K3+Uzu1fSs+V81+OcvlOUG5nHz/wWa8ud/H2+LHwUcD53/Dv3/zvk+c6P4PcBm39If63pRw0spxzwfzz8LdyGCTE7vFsoxMddN6GzEb+5vIK3CefY4VU34d10wF1/QcwWH5YRvYvobMLD2uPL4YR38wFHvyAWi9Pa4bR0CC7hj28+4NvpWIVjRGzmYemRi8FPDiecVqKTPi4dvhzPeDdxuBGmiIrFlKyTFE+wdxG/ebyBNahCLyJOskSPu3FSVNiAziX0TM8EaAFxXsnqMK0ebw4XEhtaPcYQ8e3DEUO34tAvePd4wHFY4F3Cee4qcHpzuJDV0mZMS6NOCs3yPHeValoAHLoVS3TVcur5uqFbqyVM2jeEiDk6HPsF80oUXwOgDxGXJWDsVpznDn2gWHtjt+I0d5XKebr0uDlMOF16dF1EcAmnS48UHW5vLphXzwIrploEhdKY2UJxGFp/xLIyM6UyZwIYwWX89rd3gCm4vbsgZYtl9iTcUQy6EDfAd7p0BBiYYinCJsYULHOoi9blEmBsQegjcrI1zIoPFF5Fx/ETS9t6CSjJIBx4EW8zW5DIkhonj3BYULKlIOauVHGNUgxKBvpxRYwWcfHoBionrg75MQB9QhhXpOjqAsvYUoOwl8QB0Hk8S+YFsoDgxVWLhghV+D4hLY7okGNEegxwx0gWDV7kCr0xLw5uSEgzlYMCuJDrYjAzPVEWlK7LSJOjck+h0h4lzp5YbdJiq7XF8O55PgUgZLiBweBqyZIRLcwYAYNqeam0yC5XYYgyExQyA/H7yuIqJbMsHFbGFArCHjLwgaUN3yytjMXBDrHG5SvJADOHZpGQMgAt+GxpwdwNYM4OuF2JznlMNO5Cb3v0lQIIVygMzCG1BcRM/cGY2sI6lLqwdO890utYw7bUMB/ZwJ4t0l1sC9U+U6iXIRHNM+RqUZIFvsmmhYYJhcKx9Blm5Th7FnVVZWaqLx8yhwcp1fJjVsvhRgyKzzDctjzmajGxF94sHDPsxSL3uVp2hL5X2DrVvXNIA1E4kQE7UaiX+CrBLhRORixH/mSx8nF3MYivMvy9RTyyucFSCBjI45CJklk8IOFHigcsUz8Lhxuxi2mhVQyq+qrjcCOew5NUY6QHU1Xpu525TwDR9ApQbKOYEkWWyosD0c1yKHCVCgvYmain3XsLWKKbauocjTeFOXGTorAWojmKUm0cC9xs4CYKAWMi0WgrFZZpk1lTYS/UztQX2GRgee9SqLCmNDplDUPDVE0TmXbK89oyWMt9gT+x2qmj8bCLgeP9uNQDdgHSSOFGdNv8BVjunllUvCC5iRSEAaqj3qfIIVgWGi83Nwqk7GX7C6qiqV1RaaXFtlApomxrCioNFcAmDAlAx8MjFBWW6+R7VfF4RFWxdRPRde3KbTOKxtrtKKZM6a3t9DSPYNS8yVS2XVoZxbR+6vqLp/qlvSaj7lrYlcqVUB+F6xAQahPYGsmqrRw+RJ5Dm0SJ2TSLMVApnMWZZp3lcjQVlp4TAxsLsuMQH5sdlTYeVC42FtQK7HZUWABVFZcaIs8vtoAfrSyxjlJm1LAlFTQry6qUQ+FG0Gi86n2i21jv28cAo8F12ukVcKrVh/dpT5+t6rtCrd1vuujx2I2LbtNmrF6afs9AacHfbB/LHzUV9qs/+6r8x//9f4I5+U0Aea1+KtZJEVbRKqFafEYrlUpesYasqmyhpWoRnEsMCC5V8CFATH8GsNnh1zRSAE8C0ztWhNTCI0Iv1cmgiXOI1VWeSVEezcVUBdBrip5a1CQxQGniI3ljKRFAKm2S8vbWPKGdWkPKnqL2mQuq72DKplJDBRhqyqmzpdJGZffds//cqmioe3VDnQTM6iQUUcNjUIpBH1ZYQ6FihIosFNO9RVHTgPdpL/qjw8NIWzXdUT9+WohEaKLimydUSa3KKVaZveWu0ljNlmYq1FABwRsaoho/TSXUSeqXvJqGqBU2pY1CTRTQLHRTnUfK1WOg7yMFuC+VplnHrCjVRWAjjiLnrVhtlJWjtokptbquaxwqovgxmLD0y6bnnP4N3N87ABWg61/Ba5bq+juqtp6tKc8qn24stfJX3WtNu6yCK4oGJTFm60LCtr7LPUOhTQXZwKjXl+27bFOXKHWa7fdNfra6ifon3TOprzQaqLTNlK11TT7ulEHlnmlhG+MzW8X4vLQ1qXGR2+/ISmdsASx/dpk2OXZ9rWNfsKUWllbHpk26T7ofhevVx/eiL5oOuV98yeei8sl4bMw7pdEvATy70JIx1X3S5cji3FzJo/u4V+002Fob99/37dVpP2ZAawf31xSzDcnDc1VTOTXtstIsPyMV0/wX9Y9LPX5tcSx/9T3Sc0bfP52uldEqbPNfvn9XC9e1dtR7igaUgKd9utJOfQuffJbrr1nKdQPV16t1lY/ij4+DiCvnjB7f5/r3zPUvATet/7v5/pH0Wda85wZln+8jdV61fH6qLS8ZN3wGEHxBmZ+T/tl/9cNTYf/sz/vyP/yTP/qdl/vv/uKf/+B9A37kFsvORnzVPaK3EWtxeIi0FXZ0M3ob8fVyg2AyTqnD0S24jz1e+RlrsXhYB/QuwpuMSwp4FSbcV/EehykFnGOHzkb8fHzAN/MRg1vhTKm+ihLe428dP+CSKNblkjxuDjMeVwpbcXOYcY60XViBb3EY3IrOJnwzkQDQ0MUagqQUEuN5PVyqCuuUArzN6GzcgE2pZ4oBrweicc7Ro/cRX5+POIQVo1/xfhqrUM2cPFkkbcbrwyOmJDE+PUa/VkAcXMIlhmaxLAaDX7FmslgCQOgWTNHjyOELtN+j+HUOfq2+nnL8EgNGv+Jx7XDbz9UPVI4v2eG0dPjieMZp6eBtRu8jHuYea3R4fbiQ76byIyxA9RcUP8ebfuKNgwYw5+gxdlSniPv88v0rGAN8cTxjzRRj9MCWShH6kffceSbr9NitdXNA/DHn1Vfa3nkOsLY88Y8c+oglOvTdugEp/z9779IjybasCX3r5e4RmVm1q85jX92+LdHi1ULAoHWFkEBITBBDJJgzZ8hPYMIIGNMIiQFCYtQCJghmSMxA4tHSRbo0Otz32ffsR1VmRLj7ehgDM1u+3MMjM6v27nvOPpwlVWWE+3rYerjHsmWffZaLwTgH5Gwx9LH6QHY+4zR1cJYwR49DPzc+p3k5EAArNQ/HESlzHw6NBfp87hFCxqGfEbNb+Qhay+dn7P+YV76WCgUmADH6qpQkCdnQdwlz9MjZVN+9fohIyYlFt1QldJ4duo79ARco58LOmlbkPVT9C7uO6/UhV5/IlNgHDmA/SFVCq2V67GBdRicQ0ZwtcrIcEqVnaKmGxciifDhfqlKSIsfGDWKZThp+giAWaPH/SxbeF0ynDkTA4WFayHuSQycQUi2XogOKgR9i3WAoM68LbN2GIaTJoztEzOcOvk+wDXlPvITqq2YsW2P9EDmMhC1IswcICAMT7hgLhnZKe/FjVwl6crbIk6uENzR6uPvEZC3RwncZafKwXWaSn1BglCkXAjctQrYzS6zZaDl/tDWubI2lO9tKyFP97sRq2SpzxheU2YGSEUuxhIoYmbzHdAU0ieVYFdjGjw/FgL7tmDgnyHH36NhiccyL756GgHj0TOoTLcxsgWMCHgMwKHkPQBfHG1vdHHuqvnsKqzXjM+Q9AFbkPY6Ak+N3kyq6nhaLMMAwQ21/j7zHEODYUpoPha2PgRjCSFzGzg5lKHAfHVufgliFd8h7zMTl7SS/M0lIcoS8x8wGbjLIA1sslbynBrc3i8VVSWpAYsUsqIQ9tbzIaBKT+uRerLs75D2aXy2qagHme5YtX8SWMxvZcqlWPCPKqLsYxPvP37G6WeLAythU61gRS6lYGe2MauFSq6ObTI3vqpbA1uJM7RwDNYZkXReNgkaWrewcP1es+bMVK25DwpNNJf5xM1vJq2Xf0kKIFGgV8N4ktghrf7RPLYEN1EovrzKNY2nnpc9qWSPP10tD3tNa9ciYlR/jFXmP/C0ap7MZ970ywKLMqCVWLV98EcvhTEPes1g11/Ne/TrrhWWs6rhtFXG5VmMOkxTcKu9tta3e2cqriZbrK4tlg0Boy1cZzWbcbijtV4cZOwpetQRbI/XvVHbrkGQj07bfewoumc3Y7tX/TPpkRf136XulH7ViOZWAP378OX42PGEqDr8a71HI4H1/xkMY8f8+va/Q1Xf9GV+dH/B+OKGQxV9f7nAfZgSX8d14wO/ff8BX5we86UfM2eFx7vE49ggSGkEZVp0wvwaX8c2Z8TDljcE3lyO8LXicOnx5/4Rfne9AZPD+cMYpdpWgBgDm5PDQzziGGX/+4W1VUubEMQZ1A35+CNWiyZDRhEEgs6qknmdW/ObZY3wQhXcOGELCNx/u0PcRxz7iw+MBd8cJwWc8XXqkxDHk0huL8xzgLOE8BdwNonxGj85nPI1LXgA49DPm5BEl3pv3GeOlQz9EURZKVQqCz5ijx9u7C8spSknnEy5Th+Mw4fE04O7AUNm7YcbjucebuxFj9Dg9DYgPDk+nAaFL6EPC49MBJbG1UFlCmaGziOLAygDHLwOmO2ZXjdFVf7Np7OADw1DvjxP3/VfM9GptwTx7xMmz0kIG/RCrBa4Ug/HUAQYIPUMcvchgLWEW5aeQQX4MQCD4IaFkVlrS5Jldc/K4iGJhha2zlskG6YGVMuvYP2w+syJBZ4/44Dn4uDB5GicQ0sLWuXRvkaJHvjjMRz7UyNHCfNNhPBSkNw5J2DYBAI5gBfarpB1W/LE0PpxRC+vFAZ0QRiQ+MZ8OGTQ6lvsuAR8Dyz/zJhye6y/JAqNDOibQxWPuM5AN7JBrvLkyyy+w+DHNh8x9PibQk0fsGepqXKmw3GXDbxAdsaIBAB8Ccl+Qjol/fGcHJAM7WYwPwlYqShAib8JjV6plxwgDaDqyIojZIvUM30W0TMRhAcwWsStwXweYAlyA5VR+shiPbgk9EC3sheGY8WHZoZlkQY6QQql+Uu7JYXpn4b4OiG8sKzFSr//W80ZQYJV2NFIfWIk789jPDxbm4tgdLJRavvuVw1wMsieYi4WbbN34d48Gk1he7GgR7xPsR498dAyTPZRmY0QMbS0GuS9wstF1o0W+s3BnyzA8S8i6uT0zvDXf26o0kRflYDLMKJoMitRno0G6E7htIPhHCxgg3xX4J4t8WJQ7F01VnJANhq8s0j3nMZmVCpMN5i8s/LRsqEFA98Fi+gngL5aZQ98ZdN9axIclmLx/MrLBZYUiH2gJvi6KVDgJ82fH19zI8le4n+P+ugsHqw9PMgaymco9LbBHAP5kKpROFTZyLAu/tPi6Pxmko611VJhpILizQXxjMPyKlYXcYwOFZdlLx/WUniGkgCgDgTeR8Z6hu/4MpDtWSG0EQ1YP/BcGyL2pipY/AyAgD6yE+pG/xwfDUFICiufPNgPpwOyuxXP9eTB14+/mpa7wCIGVmgoD9Rcexzww7DLeL3BRhaiGR8L4k+ctRs8l7buOjRUGUZO5PTdxO+6yKFrF6/oByBkelxGVbbV4IDwRyBqUYCoTqE18DQBKZxZFRZTR7gMhHXWeDfyZkDvDbJ0VosvXAMCPhNwb2JlA3oCMgZvlwKdfIKYA4GbiOYgsA3nAX0gUO4aL2gzkzsBN/PzmjteyP9MVFLZ0Bu5CyAftB9WDCDvLsxGWNaRjxuOwKJDFC/w2LeNe4bMbpVOVD5J+tYyvqqho/iJKtMJxW4WOjMyF21k3otgZAkwmqdPwZzwPhdVxbFNVGMH3bV4OVtr7z0FhqwK8VSz1XbVRvK/60ijhV6RFAPT0dSF2aipSKBW2SvJyfdv2rqV0Z5yXwpvvTZ2vqus3IO1ylcJ9GzoAACAASURBVPyWpB81FPbd3/05/ev/+b+NMYfFaqUWAKyhigp5bFNL2rKFk+6lLXFJC3t8joylTbdISm7l3ZKTALiCgG5hZgxDXZP67EFqt1JevWNutLcn3x4kddvXllSD8yzX9/p9q669dlt5n0u3lvueLM8R2bR92f1bzP6b8zUvE0NXAcxrNbieN71ei+9ca+9RK08r3wuphce2c/hJr5AbJ6BLG/trau85eFU7+lnT3onnM/JwPv1Fvz1/e2PzXFqN294vYdvm9tqqIrqGYj63AG6ldixeKrt7nLz5Rf/UH81b7e+tl5fmq21/i8f7Pj/mr1kvzz10O+UMmRUxSxs0/lk5XrOOX5LzNfe3bdTvt2XcWwqvso48l27lbTbOry7/2jY/Je2MR7vcPhkG+Jp7tzbXO5e3U2leaucFeV6c2225rVDPtH2t7Jhn7r3Q7qvqf135772GPzF9Fhz2h6j3hxjTT6n7hfS9lcRN+X/4n/xmQGH/i//u93/wev/lf+IXv/a+AT9yi+XRzfh7b/8UH/IBqVj0liFvH9MBU/G48xPm4uFNxlQ8Di7iMTJc9kHiS85y/WMc0NuEqXhYU9DbjDs/IRWHr6Z7DI493FNx8DajkMXBRVhT8PV0h6OfMRePBz/hQ+R4kABwSh06myvTLMBxHE+JrZjvugsiWZGzwJrC4TxMwWNimOucHQaXUGAwplADtgPA0WssRSYhakOp/Gx4wmPqMWeH+8Dw3EIWg4+w8rR9Mx1r3Rp2pfUjPfqFnbSQqZBcCw4Qn8jVcp1NSORq3XNxHHsxhQo7TmQrzFahtxquRUPGTBLypcYW9RFT8szQ213gbcHX4x1fz762B6xZPa0hPM48hp3NSGSFHXeu5abMpEK/f/8BAPDV+YFjT4pcAK4gvoPAhS8pVD9TVXy0HQC4CzPm7KqFu/ZboMqXGKovp4Z8OYYZRz/j2+lYDwgycSxNDSHzcRo4NInU2caaNIZwiTxHhxBxjkHWR8GbfsQpdjjHgGDLKjTLLNBm/Y3UOls/XWs4HImuDx2TKTv0jiG5Y/K472Y8zR2CQFI1tqgxhF5CyHQuI8p4zslXBdJLPE0vsl1iwCFEXGLAXTdXsqpCWIWE8W6B7ybpy30/YUy+xsYM4p/bu4zTzHislBmirDFLNQxNLgaDsOde5lAhyVqX+gir32/KFm+GCcYQvjsf6nrwEruUyxRhE06whnCaF7pYa/iQqTRr6dhFfBx7vBkmnKZudWD1MEzMel0sCjFR1tPY1/nspY3zHND7jFxMjRWq7McfLkMND9P5XInD7rqID5cBJM/TFJloa4wc2zVmt/IbDjL3MTt0PjHU2GeM0df4qPocAUAfIoIrOE1dJSfTg0EvvuXO8pj2IcG7UuegSOgdADUm7Jz84lOpB2vix/v2eMFlDnWNdT7BWcLTpa+QdQ0hc+hivT6EhNPY4W6YcZ5CPcAbBP6u7MRz4vitCs/O2SJIvUnqdbYgCmwcYNSDsvBqHFd9v6obQm78OpU8TPtVx6Epp/WMM8PvFcXB7VmEkDCOAffHqYZAUj/nFlqeMzMQp+Rq/bruCRxKJ4SMoYu4TF2NGavrRP3hK6Q9LQzDinLRWLTjpVtYeAlVDkaXLLJpWJpCpvqO5+QQhPxM/aWtywgCjVf0zDyFGnJGoeTDEHE+Dfjc5ENCnOWZdgtmUuHz6hPtQqoIGoWw+5BBhcMoOZ9XpGuKkLkK70RLvFhtB2Df526ISBo6Seoo2XHoFyXLssIYTEyUxmPBPtNUTCVAK9msfI8rJF181WsoGglLU8PZCGwdBCFfMxwTtw2rI2RvJhQO9aJd1LAyEjpGCeF48Zi1Zm7A/7KpZGd8T+sBajzZVhsxWBAk7YFJNePJV/HHNUUOe7ba+IoUaFOfKtpWDoaadjQky1qeptz2MK1VVPcOeFQO/av92/oSA1eHotVKe+OM8mabN9KWUKdp6rru7eHOtv5nD7puHIRvG/u+h4d/Q4nw203e86NWLDuT8Afd1xjSG0RyONoZGQb3bsKHfMBbd+HrbsK38Q734ntZyOCdP+PJ9piKx72f0NmEezfhKfdwKDi4iKOdMZFHJIsvwgVT8Y1iafDGj5UYpxOlVpXb992J/fMsK6waAkXDoRwcK0Vf9h8xFY+pBHib4VDgbUEwGcHeMXFQKCK3xcUt/praHivCqSpYBxdxcQFf9h/R2yMuOeB9d0IvSs/BxeqraQ1x3aJYnkSZ1fAjBxcrqZE1BafUI9jMSmRxiGRx72c8pa7Wr3JMxbPCa3sMLnGsTzJVmT+lDg9+wqPrMbiIp9jj6CPOiRWjd91ZFMwJk/cYs8fbbsRB4ot2NiP5hXyIlV1WeIMpiGThDR8CdC5Vwp2j+HB6U/AxDihk8NP+BAuSeKYZ92HC2bEv5Tl1HGpGXgT3fuLrMhd6KAAARz9zfptZ7uxxdvx9Lh5HP0s/Z/RSv0KbrSG8CSNCEz5hLqz4vulG9t8NU83b2VTvq7LvbcajHeBswdHPcCJX5zK+HB7x6Ht8a47ofaprIJHDLLEv9UChjU3ZEhUNLmEuDp3N9Tk4xV58cwvOjhmYLWjFYKxMykc/4xR79D5hks3+JHDzQqbGf+2EDCu4jDuBrD+ECb3EplWfXD0sOMgBSy4Wc2HF511/xugDzpGVyN4lOIlPq8qbxp7VGLJ6qKEHEIVMVZB7lzC5JRasKvXaly/6C8+zxHpV5f/sQv0J8bbgvpvkEOlQ51kJsGJ2lRzqTT8iE8PpAawOIN4fzpizQ5aDmkH6r8p1G2bnEKL4bbul/HCqBwa9y+hlXvV5U1RD5zJOlvDQT3C2wyFEbldjkBL7IHtbMCZf7w8+wdmuQvcJqPFVB5/QO/ZxDdLXGm9VDiucLZilvuByJRzLwqhdiAnA7roZY8r14EL9nWO2cJbwbrjAGcLo+CBk8AlelB9VDvUw4e0wIouCdt/NKAS8GUbpJyvR9/2M3KyJyS9xd43hWLy9TyCgKrPHEPE4dei8xuNl3+U+JMRs8dDPq2dszk4UUZbz2MVKAqbjngtfB1Dbuu8Z1q+uDK2Ce5C8bw8ju05sDsu8LbXdYz9X1m6ja927SmZ26GINtdT5XOMaBxnjQgZT9PWwQQ9oJses4Xrooc+DKspeyp8dH8RpXOPJ6aGCqTGE1X+8FAkrJrFWta3RefTifz50sT5XqVg+lElt8JtPS31IFXGgSrKRtXuxnZCksaxFXAxUKRxkLjUurx4K5GzRd7xu9KBDnwddFymtY1iXYnAcZkxygKB1xFSqUg7wYUv2rMCGsPi3Z1HI9YAlJSds6lyfcxyDV5V8jbnKZGwQsjlWWvWAoPqlC+s6kdTjDUp2rJRbvyJpI7CCzj7oSx+1TWDxz4YRdvFkK/u3+nADWJReHSdRVkhY4lskkRKWaRskylolKWuVoAKG22sYHGCtwOi0WAlJpIoepE4Nq3MLkdEmbU/bUCV3q4CpQqnKlJJurWKZrs3Eq75tEt2S6VnF0lzVRTvlaVvPS/Vf1bEZg2fN39/XxPk3kQzyTvjB35b0o4bC/t4/957+nf/y30QiV9lcAYbBxrKwwbYxCBMtisges2eqrLHKnLpMvioWFhzfUpWJtFkgt2Co2/tbeCqAK/inxmZclW/kaL+v+2FX1rPn2n0JWriF/O6ltk5gzVDbXtcNaxs79Dn5WtZZlUE3RK1styCr29PfLXvtc5+VzVYtZEs9C4R5D665B03e+74nb6vw7M3V1fiQuZr9vd8vANVatf19ehW0tK2fbkNTPxWWvMdsu61/CzXeg0U/B5e+1YdWhj252u/Awoi7y1NAC3Pvc/1vLWvqt7uF0W7hxXtw4xXsVjcf25NbYHVtb0xusSm3a6yOFZ6H2T37S3JLtmdOmG/Nw9V47J2Ev3AKvgfzv7pe1szDi/ybtlX+TV+q/FrXzjxqf9r82zk1zbp6lXmg7X8r894E7m0i9zax9b7ZXVsVkq2b2luLZSvPa+p97l1yq5/bNdHmayxPn5taCLN+f5V8n5PntVu0ds3sDJte+xzL1Cr/jXxXr9vn6n9pz9Fa9Dblt8vzqs29dEN2s1P/S/Xt/qzcWr830jM/Ta+S4fvmfbb9V47Dq+t7ZR0/RHqNHH/0H/76obB/918c6D/7b//gB6/3X/s7/+jX3jfgR26xPOUO/8c3fwuTnKArc6meCitcrQ1lkeTUXD+rkqPkOu3GQjfiChvahmHQDW17kmgMaogAAAKTWocbaeFaXlgoWwichmXQmIzAOsxEW5eeSlpb6ilshRXKqWQbsqENDQEwpEjDKGw3UUQcR7HdECnr5JJpvTnchmLQcAt14yZlrhTCzUau3Xy3Y99Cztq8K7/NVulRCErzo2Js2x8DFAPfM/NnThKKQ08uNY8mg1V4hdopqV9JbrgdWmTYbnIM6qliuwG9rttUAp3KipnsasNa+619llNMDmauoTKY8IaKWZgzW+VkO5cqY/sdQBu/sI57hUWhxsGsfdbx0fo07EFLrb9zwryi5bdYQiVsT3LbTX3dVMo1zd+GBjC01NfuTrT9NlyAba5p37dKRjunAimrAdS17tX6oSVcwxYe1SoU2PR7Gx5BmSPrvGCdp4WTNQyjq/LtKXo7ls29bLDArtq+bjfq2MnXfgfWsmr+W4qgAUwxKGqNqAyQBkWsFy10TctUH0V9voVJU1lcSeozieeZpG0CKjMpDFDkc/G0CvbOjJiNQlHMWvYCQElLylJGSUqM9lUtDCT5aRkKo8+UJkcLlKxd923okPY5MQrnw5LXcj80tuatcCCGsMABG0tQfXdmITISeF9tw6B+r7LKGFRGTH2fSixOI4Qrq3eiwUI+0tSxDLjWJayxm3HZMpEaXf8NVJCJTl7e/N9Kdc0B6zE2VMdnV4kDVnH/VjEAaRmXK8vO9vlpb7XwTABkzGotabHVWU57Qwvq97222/vmOk9LImPaOdbP7ZponuWrzmz6aNp3easx0ALpbC5dj12b6jvl+lV4pWjfqmNH9vU8m/Wcb2XbyvM3kF5U9rf3XlPHK8p877yfIsOPMBGAUn8If/vSj1qxHFzEP/32rzEVhyKQuK2FTpPGttTrW6tlG9eyTa2FKAvMc6uAtiZtC0KBWTG3tnEP2/ra9Jzl67lye33dk327b2vLvUQ69JxFcFuOyFxZGLfWrT1yFrUA6mFAaxW8VbZNr7EEPkcspNfzZg1sf1drXqzHtDxjDdqT7zlyI1WouRx2rRxLvev7n0Ko1JI/PSf7S+k5Ip/n5H/Osruu47oPe5a9deXNr/mNzcmnpNeOBwFoEK71mrap0K8rGW9W2CgAbdpce2k8mFAK15uK7UZju7kyWMenxMvvg1elZx+mZVyWTXBzrW13I399VnVNKtxt+y7p9tfHyurdcYXU7bQln6mV+9kN2I15lHZXqT0Iec1mt71Hqz+rOmo/9uTSr/X3YrPD37lOYfPOkIOV1dVbz77B4suGZU73yIt217RA45ZDsbU2pXA8ZRuu9yW/MhR/VmoVjDo2Ut9mTPYeb9L/TPO5vbfN/Ir5N/J41mX2XJur9bJ5B7Vjv1mvt5SUNpRG/WywCs2x6qe7rdus6tVCV0uRrhT21ccXptZs+k/N9RdfyZstYj2g3oh/lV6z3P4xKGlt3z63zr3Lr67vpffiK+v87DH9Xfq1pB+1YgkwaU0hg0kgrl7e+AWLMsbKHfsRlObIrhA7kSupjPp1abKGxO/Oi/JVFiUMaOCua4ukBcdqJDIoRqCsZqkPZmGtVd89hdfaxhqoyqkR5c+sNpK8udM+uqZ97aezBVb63/rTtEmVaWvo6oCyjoMtDFu98eR7LM+4jp8za4tkK+vVxtQQHBol1+rcLlBYYK0E58bC3Fo9t9fWLNgkfQWKxmESeUKNpZiv5NtTYtvrxhDsnmLv8iqwfTVgqOK9M54GAnE167m+1a5+X/pdmvulWurVAq+xKZfy7TigsWKuLZp7yuceZFPJRV5SZqXmq/48B01cz+3air+FTC7lGyXD0Kp/1/Wv5V+NL7Z9v4bFVnKWvDZZ1OfW8A7Q6NhtrZ+btPIJ2jw3ijJQ+dSqfwXrBaQ9Qmlu1kMnlXGbV9etjtmqP3KrmpRknKoCet2v7TrRa7vKsFpbRSAeq83WTcbPmh0FlwCj4ltZAO0atFuLOjXXxZqvn7dW6p1N+KvYrknL0lJWxrSFutYDgG1qX8x1t7hzf7vh3iqnLRJAFa6rdvZ22PJZLfqG1gpw22b7mcximdU6zXKrrm0tYGnd/1vK+HY8tj9cbZ9v/f0eybRKkykVGrv1Obta29r2nhx749imW2VIFTce35sHdLuKE20+0qaN5T69Yuzas5pbiuhGV1w9C3t17SmPu5Dya5Gv07ZtLO+zehjxXD83116lYOF6D3Ervba+T13HVP/7jLZ3ru8sj0+vd6/O7/lc/pjS78h7fkNTKhZ/cX6LziXM2VcWTyUSeYwDLAhzcehdwpgCBh+RyeISQ41RGYVwYUp+IX8oFnNisoGHfsLjHCohhyp8s5B3HEPEJIyHSt6gLJudy5UVs1WsnGWSnm+FGdK7Ukkd1A+O5eONXhIyitYaaAwhJpYhF4s+RFjDcTK9yxjnAOe4HY1LqYQK+qLT+JlO2BiVTKEl0GhTcFlibS5wVYX+5goZ5rx6rfNLmfa6cwzfdcLWp9+DkFzE6FZxKo1htr+SLfoh1niZK/gtCQOf/FgpS6DKBqDGnCQy9f7Hj2xmGo5zjYdpBc6sfdINoxIUKOGAwoyXdgqo6W9LoqCKXikW1uXVj6pCcVGYVU/vWVuQIwe5LxJ4niGyssGp5QEQmGCBmOBASRVKMSgjB5O3QSCxDYSwheAahbWSfNfNIGG12a55fWHWPwKsBLW3Xa4w3LpRFxiu8aXGrQSWtgHUjX2F+EqcSdtljlvpqLIRlmQrcQNlw1A+R4uiMzuGEQpjYSV2yAamW6JEk64XkvZUHGElNIHnGdnAeLnfbrCTxLScOHakOeRFacxNeWkDUU9OynoDrocqalmMlmOGTpbztrDSLbNiMSyDptS0obA/tygDZbSgoXDZZBfYKgAzG74H6acjIBqG+iq0sP1N1HXUwJxNZqjiKgi57jCyYWuV39wH1myHWk/GAuG04NiXkGvRVEsZgDXklQCrcTQVEisB7WkoC1Ojjqv02yQDEw1oINjRoHRLf7VtjQVHHhVqbAs4SH0USJzCAyVGpMJh2ZqzBKnXOjVpbNIaSy5p55q9t0ENbK8KWoW6Kky09k9gvR3BjbxuNWB8jWMp+3pyBJtMjemn8pPjNjXGn52B3KHGFtT4gtpH8jyH1NSj14TfDrnjuni+UOML1piDGquv+QnSGItFYlqSRY0xahJgBcJdvPSj43iexRNsZnivnQ3HPv3MZKPh2KcqD5l6ZlCCwmF5HcA2CrSU5XHmcSCDmkfjnF7BgPXd21gGuTLATsv4kOU6dExW595NG3V8ZS3qmNYYkNq2/gS18R4XXjnOQ8s8qQyA9M029UjdNvHcteU5j6kytBDa1iKLsrTXttnGZdRnZKuUruDJy6u/ttPe20KUtd5VfMjtGJimDBa5Vb46Fk2b2zGoSZVG7NTb3m/yVSv6S4cVG/9iQwQyBmb3gGn9d+9g4qa94RMVzmfsFs/K9Enp/0dK6687/agVy/Pc4f/8E44FQ40/Yk1bi0A9YaW1L5VsWBZ/JKBCsgD8sj1xbZPm29bXLvq9B6B9IWzytf5By8tlsXpsoUO1DgLOzYt4r+6Lfi7L9bNZ14FbdQDrp//GKWOb9PZl59rui1zS1MiT5Ec2NrIbAi5uuH7Z7rQ/2uZ+0y/9fdQ9t/pRja5f9r9Nnr3D+yz1lc0cF4A3G5pdFT850VYjVGk32zKOrp2Hpjkr11xzr8q0+U6yFNu8lha3PLQ/nGjquFHfNlXRpP7tcnYEkAlX8rVldn8ct/dbuUyoLmWa323zbOqq/jSbZ60u2+0zuO3nZhPQLveWtn21iSGAnFtf22uHAJgdZsrts2edbBg2efdkNq62t/Td7T8bxQDGNhuX7bO97KD4OW0f2u0mZGeA5Hv19du2TxDlz1xf3za1t0aA9Wbq1vu0XQPN5oidqLdjApC1zedlnpdxuO7L1Tt+u+638mN9XWVchm/9W7IN1L4/NvogtjvNRkC99+Lm6lrQWxtAf97p7/adIvdXz4589uf1vVvWrqs+Y6n36l1S70tjZ+3zMi6GADfd/t16KfHaMMvnleyiZG5fjHURPVPv9ePzctobg9ekW++lnWvbubtZ3yvLP6uM3OpDfXfzhytl6LX9xgvjuveufE7mF8rW6y/81ryUnrUEvyY9K8N1hZ86Rs+lz5J3296nrO0fQSL67WaF/VErlt4XfPHFCYPEMpuEvKb3Gd5lPF4GtvaI1WycA0JIMADGia15zhXE6HDoIy5TqDTZKTkksZAcjxOmieODaX3WEuaJrY3DYcY8O2h8rX6YMY0d1GKWE1sh63NNBs5nOFcwnjrAsKUHZFDkxJWShTsstOxZ6LVtA4EDhOAnm5ofAIrEqcpnDxMKjC8oo4PtM4wF8myrIuyHxG0atoY5seSoNaxEu4Ls2S7XmFSwYoma2RJGLVGCASBxsNoyZMDyJAsbMsrEZUuW7zPLWaIFJgdzSKDJAV7IZ0bHlqlDBsmJ/+qkTk/pdYz6zPupaBfNKoqFiQxMl2EsoXzDYVboTWRttj1kCLQmGpnkhaCWEiU7MWDLDmtWMDPLR6EsBDfJgkKBiXZ1XQ8uzGTZwnEQa5YSZUyWrQmzRRn4nkliFZIx0EMJ6rheMxtQxzKaZOBOFqUjlANbZdBYNJgIw4iVgaq1qLVkAWBLjli+bALIAaXj/pgMUE+wF4MyLKQnrYx2Nig9LfWQ4fFQIpa4bOwBgDqCmQzXO4qlwGLJq0QuUfruUKGU7mJRAlVfMJMMTOa/eZAdrRwomcxfS1iUITvzr3jpWE6b+L5+5lN4sRQ5wJ+4v+l+eUZtRC2vp+5uZlnzQMuGSRVVh3rw4yaDdCT4E8vbnua7i6mWGrJs+ck91R9hO/H7Jg9soQHUmsLlw5NBPALk+b7JC/GJH4F4BNQKVjqCmyDWH7ZMLaYGwETR8TzfV2tM7gAbqVpLqjUjgtd4D5hMqJY4g2rdMIUtGzYudam1wkpo3dLxZwpSnrC2rhDgH3ntlMD3rI79gWDUQiTkOf5skI983Y1s1dKxB7hON5qqDIHYEkmOxGrHljFVWornfEbWgFooyZL0g9eNG+X1ZXQNoj4bMAQ7mao8k0MNi2Cm5bkkx2NROpaLPKqlU+/lgRCeeG0Xv1aaVZEuHjzXgS2C/NxgsVj2gJn5Xhp4bqrFUqyyZKR9sWLqfJHMgUliKB8Wi2W1nBHYF1TXQZb51Wx5GXc3olrCqsVSouyUIM9eL2uksYK6EUj3+Oxksj4DsqbysvbyAECsdS6iWhlJngE/LeOs1kWd17qulfyJsLKS6TqtyQL+Qsj98u7T56+1uuk8AICbCSUYXhtqxUwkY2aW56hp22Twb6iV59kAZA1MIVlrBjaSWGJ5rbqJ2Cqu7TvJJ+1zh8B1mMVaW5p3nI6N9qFaJi2/a/SdXZ/7TRltQ8duxTSrzzCW565ayG3TFhbF0m4slnuHIe1B1mJRJRRnal2gZX52FcVGCdw7zKpWxkI8DyR/My3zUsuLAi51ktX3F63bom0bzbiYRe7dAzxcl699bbu1re9W2pPphmL5WYcxvyFpL5rDb0v6USuWRz/jD7/8U1xyQJIYbErek4qFffiu+hAWsrCmIMmb3huOKwhgBW9NjR+i/o3ZwbxZ+y4UmBojMEpMxJYcyH/B99rwJnqvJQry70qVV5PGKdRre2QZrb9hG6KiTa2P6a3QFW3ID4MlTEibp32mXyIJ0kSbdvfGoA0lwvKbFXkPQ3KXawCT+zB82VaCny3RzzYciI7R3meVs3//BICDrluzJmja9s01LL8KW9YcLRHSlhRJ8z1HxrRHpMQ+dOVqHbTkO/odwAqm3PpO6lhv73Mbazm0LfU7bOvXtA0V8hw50sJiXCpL8F7bt0J/bP3Ytv6723vcFlX5lvo5r90y8t6QG1hc/oxZPi8CLr+WCun1G588swmNA1k/duMn3MouGWCLBb6Uv2h+2634dcmBVbsGq2jNNQL44EDfFfqLLAp9262k/ZJyIKAIlLmYnbWCZW70PsnfvM0IOYMxWJMJtRuVZh7JUEUG6LXSjFXRMd0OgLb1B7RAvqV+Yxr/SVr6mhp/yiifk/pjypikdgHIGK79Ixufyb2d556sezuwV2yqnrX0b/MbAMVgav1Xd+7vpnb39trNZZVjMwZaX/v9VlvbdLWgl3Yqi+0mb0sQtLSBCpn+nLRiHr710LV/t3O5I389RGzz76znjSD7Y9u+73Y38tfjxJ93Gtmb780XVnS0jRs7+/beM5v/K4v0bj24+c571mp4q809COleecmzb23dDPJK29l8x2cqQLVM/cGSa2Z9vR0V2nxfCfUKBe8T7726X9v1/7v0W5V+1IolkcFcPDPCgplY10ywtt4DgEKuKlqpMcPotUXB4Xvtpmhrtub8rioCqtDWINfVIeJaSWjrjfk638aNge/tMNZulRpg/ZzGhpgnNuU12PZzKct47Mmi/bgiz7lRT/uX5W4cl4utwcdVkSQyKNBNm0FqwiikvPiXZim3RaeljXL8UiKwQqljdyu2Z/VrbeasrQNYSIX0862UbmxsCvZZWl8X0FuVJZ2TRRYNwn0ld6NYtdfa+rb1axsAGoXIPrsONOXspL39eJCvTmRA2t8bJ3+5XMu7W9UL17XkHgpytWHYhUkuG716UJFcHfNbBFGaDwByWc89AWtHJb3Y7LZuo8REiWrnq1HaVnW11YtFntr7O23c0mu2m+DXnxUZZAAAIABJREFUTn2p60xlXM/31dy3ChwAavwTVbC612oVTkAw783nVUEp3j5Dm0187XsbxmU1r7cUJrOS70WFYq/8DZmu0jZsTdvOrgKxrR+yuTbVj/XZGI7tGFRFy9wcinrvlZv8Npm0/761dB0G4hU/CzfTHlHb1cHIXh9emMtP2pTXtrbr32KrbK5W4K6i+Yq2bn2XZDdZdhW+TxiTXSveph5zKy/w7ATfrPuqjhfKXeV74fl+xZh/ksK5PbD4nkrapyqZr15Ln9unV6Qfk4WyTQQg/y7cyG9mmorHnzy9w2nukItF5xOTvmTLJDohVdIbJbRRpSC4jCTkO94WzMmh8xmzbPqCy9W6MkVfY0IuDIxM3mIAzNFXNleG1nJ+gBUC28SC1PIaX7LvIyubQkSjG6gg5DtbIhy1NmlSeK61hHn2q7xdl5CzRUoWXZcRo1uVKcUyxNYuLJu5iYVZiAl1lH3TGKpxL2tsyWLgfFnqaUIFKFFNTo7jTwqpjVrgcrZwPqNkjgNaclsX4HxGio7zFCttZVhLiLOvMgCLgtTGAgUYQkxkYJWUhQDrluD0JXEf+jvGIc1jgDF0lafdMFRCHCUcaJgArRDTAAuJTt2UF5ajJHPNOmn4l9J6Zh5OUTFUUk4Id6wvDBMGlvGmpbwxhBJdJdWpeR3BB4lrGhlWXZVCqYMIS3gGZfg06zY0RqVRGDQxDNv4Utu2IbMMriz1KxTYEecXmDQPJJbxteB2VJHQuJiJCXcoiW+cksoUsGxKXEOmKngMz7YryG+FbyuBjo6dtFshzYTFiVM3rG08SZ03Wq6bPvM8Xxo+fUeNkiL98/I+UAIeQiWAaSHXphPod58Zvt2G4ejLYonTsZgbfFWQQWgh4G35IaNcPM+1jItC2E0ooFFwciq/EvdsY2q246QEQhqnNG/mSDcBQsCE1Kx/rav1u9TvsmbqWlEIeiuXbjhVNFXyh8LwdH1GNY7lzNDyVd6wQM7hiPN0XL7CzLVtlbNtG2ALWVuvtGmSrRY1jX3JEFoD8mVtLcxYWdQ0FudKiWtlISz1yNwYmQP16ybtz5DXz4jOv2F5lQBoFfOyIX0y0QAONeYnCYx+1V+S9k0zRzpfpoGmz3aBAMizT04g7g3XQSUhIgjUUaD0Ip9R9xGHutZNZlInM0t/8hLztHQFdvz8TR15WoiTlEgGPKxW4a9CXKSQSoUzloAFZqrQVrlfSYraA6pmWZi8uVaEnCgt30kgoqsdv0GdP4aQLu4MRtYhGXBsT3V9oHW/AJVxEzfWcn2VRLohVzLSNyWSMjJHLQFQbUPaNM1yUD4FzYdGjvZ5p7a7+m8DLbn2Y277JWubrvPU8sSojZWle0+p0fd5807YvaZTs2lLq9w7BNkDP9R8Ol97KJCdcdjL07a928Ym7YVa+b7pUw8+drP9kAL9Ln1W+lErlkc34++9/1P8cnpAIYs3YUQsDh/igKfY48vDIy454OAivp2OeNef8e10BAC86894jD3m4nH0M74dj3joRjzFHp3LOPoZD37CJQf8+ekt3vVnzMUjZlchhV/0FxQy+OvLPe7CjFQs1zUd8ZPhhEIGH+YDepdWsSw7m3FJAZcU8Ht3H5GKxTl16FxGZ9mvsrMZ30xHxMLK78FHpGIxprCyZh18hLcFg4v4eryreS8p4G/dfcDHOOBx7vGT4YTHecCUPe67CZ1NSOTw1+c7DKKQH3zEKXYgAMEWxGJxH+YaqgQATrFD7xN6lxCLQyoWd2Hm63JNrbiXGND7hMeJxzQ4DuUxJo9DiLjEgId+wtPcIdiCSww4hIjTHBBcwfvDGd9cjnjoJ5xjQMwOb4cRvUv46nSPwSfEYmsYDWeoMv1qCJnHqQeRqYcOhUytzxrCeeqQisXfef8NvMn4xXfv0fmMQ4gYk4czhPMcKhzXGMKbYar9A1APJXKxSzlb8Kabqgydy5izwzFEPE4d7jrupzVAEiZZZwkP/YRgM74bmaW2kMEUPd4eRszZ4aGf8M35AGcJg08Yk0fKFt6xQupswePYI7iMuy7icWTf0T4k/N7dIz7MA747HxB8Rucyn5wVKwcqVPuZMh/EBFcQs63wVWYR9nxPwtA8jT2O/QxnCKepw9vDiA+XAX1IwljM9TlbcOgizlOHoYuYoocxhCmGCvUNjpmLg+Pn5TIH3A8TnsYebw4jzlPHLMvZog+pMhwfuggn7McKIX93vOA0B4xzqIdF3hX0PuHjyOuiFAPvCrwwMI8ChS5SJwCcpw7GEDqfMEV+/rwtlelYD6R+cn+GswW//PBQD4n0gEjDpVhLuB8mOEP47nyoz7KXsVSWZCKDh8OIbx+P+MmbE747HeocAMC7+zNidkjZ8nuni/h4Hirz8P1hQiHgdOnRdRz6SA+5AODnb57w1cd7pOTQ9xG99MEawsMw4eunIzQMz2UMuDvMuEwBhz6uDryUWdnZgmn2OA4z5uTR+YTL1KGTNdBazPsuofMZp7GDl0OjLORr3i+M0TGybMFlXGQOUnI4DjMKGYxTwHFguUrhAzsdO2V+/umbE05TV+fg0M+wBvjwNFRG6CRr/83diO8+HhG6hLthxnePB3zxcMHH01CZpg8HPoDyjsuOc6hzpvINfQQBiLK++5BwHrvqv1+KQckWoUtIyeH+OPLhorhhTNEjyeEQAAx9rO/flGyFpw89r0+SPtwNM05jJ4eMbtW/vo84n3u8e3NGLvxOWQ4MeZ3OySFGh2GImGePvkvMXu5KXWfjpUPXJzwcRnw4HeA9v9eDKzhNXf2dm+TgL84eg4zZNHk4R+gC/8Y9nQaELtVxC4EPc8cxCBu4g/cZ0xR4b04GXpnNJ49Dw+BdCq/DXsZkHAP6PuFy6XA8zJgmX9fW/XHEh493n7rdqKnvI6ZLYHlCFhZtfranS4ALmbkW+lQPNvUQcjjEyuHQhYycmQmeskEv97KsR4APJ7OULblxvgQfut3dTxgvXWVn7/qEFHk8dP1YQ8w2DsB3GWl2fOhZLEpZDkpTdAtbtxxy5mT5sLRwH1wocqAMVvot1+184d+S2YEICEPifhEfWJdsUKKDGxLSxYtSSPUg0wpzth56grAcIgJ80CNlKNnlwEwOBSv8PC+HmKuUbMOFQMshkx56GlrKNQfFdbz1kG17qLb9rOcVLTeBsmU3yrDJpkKqn4VqE5aDH1oOgionQTHrwyM5NGjLa53Q8am+HRvZqcnX/m0OJFeJzErZ3Q5X+/1qvNrx3VZtdq7tybaXd6/sb2T69ZH3GGN+AeARDI5MRPSHP3gb9L3waL/e9Lf/+bf07/3X/wqCyRhLwEQemSycKTjaGb+K93WT2NuEp9zjaGdEcrjkDt5mOBRcSocHP+KUehzcjKl4TMVjLh4WhPfdCd/FI3qbqp+mKnwFBndSpv3+MfGG/uAiUnErR91MBsEUeJvxIR5qvin76pM5Z4ejjwiWw5XMxcObUr+rz+gs7bJSyz+qsyh3T7FHZxM6l1lxtazgztlx3wzh6Oe6EU/FYZA6UrHwtmBMYeU7OriIIvJoXM65OHQ2Yy6ubrjVT1TLjDlU30JvCubi6vXOZo4jahPGHDC4iEQOp9jh4Hlcgs0cNiX5qhzr9dZnkcggka3XepdqSBmNpXlJoW6AVOn/+nJEIYM33VTHEODNTO/Sav7OolB2jRKrYz4lX+uOMg5BxsSLsq5KpjPLD7+O75g8cjEYAh9DG/Cm7zR3rKREX++pj2r9zRDFefC8mZ+zQycb4FQsTlOHzicMgZVs3aw6CTGjPqPOluoDW+OSiuLXWvYVctuHhDk5lGIRfMYUfUULALhSWDufqzLNdS+QaoVC63dFG/QhYYy+hsNxdgnBA7ByrrLrpmyKAc4WdD6jEFvQNY6nKrCqiCms3NslDmgS63rnRTGTOWz9llfhh6IHEXAcZuRi+ZBaEBGtT7MiC1Th03UGoCrYAIfW6buEcQroZAOu46JkYlomJVfrM4YqOiGEBWmh4XEAYBo79MPMhGPZImezhOOJvm74c7bwntEO3hcJ/bNYuzVPKabeV1/DFsnAcmr9vHacz1dIA1UQK1og8zhWf/IGVeF8FjREqRtLUn9WITmLo4cNgrAAb8YpW/g+reK5EoA8OfhelL9o4fqMPLlKaAZDjACgxq/SNf7YxPIVGQO1ummInWqVtOtwOdUaq5tHtejqpi3ZxvrBuzYu1+zOLNgiGEpjKV7aQ7IwXWYLOLDcby3lFktoGUfVUl8tXABbpJMVi2CpVntDTXgZlae2bVbXTBILY7eUr2RhhFUYmpUFGFjVb2o4D1osm7rx91StpUZD0sim2s5MLvbZKTHhmI6NmszMVvZkFpIWtaQK+qWG7lCrm0G1PK7CjTRithZOHQM7LeF2mFzHVAtkLdsoWlXxECI9mCUsDTnakP6YhTxGZdQwJVjkW+VpLZeaT9YWGawstQZNP5uQJ9vQJdWSSajkOi3BjqFFESPb9H1Xu1nmaqucrCx+zf1tH68SXZdpFZxb4UZuWue0PjTtviK9ZHG9NRw3+4IXZMQ6725dL8hbs77GSvmKR/Y1Fs9/+B//+//rPw5l6lPSP/UvHOk/+m/+mR+83n/rn/zfX+ybKJZ/SES/+sEFkPSjtlg6ZLzzJ0wlABZwVBDJIZiMTBbv/BkFBpEcMlm89RdksgjICP7CCgc5vHcnTMXjIDR4wfAprPUjovhl3vkJTn7VgsnwhtsBgKmEWhZgoqB7P8PL/bB5M7RKyvvuXP0PDy4uvohhyZ+KwyFcUOTt5A1QiDf/27oBoLesCHU2cexNEA6OFcZYHLqQIEE9UIitXTygqdbhxWJ0H6aVn2KBgQVhaPIOeg2y4fbrp7qtRxXATsq/cSN/l3xHLxYBZLzrmc9elVlrGM55FyYZhwIr/VdFWz9vk29i/HVdrv3QufjZ4bSal0D5po9m3/Rd79+5XBXVbd9bAiPN37tFqQBQN6cHH6ts7Zi/Gy4o4Jipmlr5t/JYQzhI3kIGPYC7br6Zt/1e6xcr78pCLnUeQlwpor1b8E1VxrCqTurne7Zbj21LdrQd90OIsmbWc9I1bXb++hBUFfAtedRWJmuWfG37W1/XWyQ7vZQ9dLGZ57JL5AQwa/XWh3ab1ArLiu21p7PGuK396OKKzKrzzTrcecv3IS0yBFRlnm/GVT/18AAASBgdV/60jXx9WK+j7TwDkPfPy+nWQTR1Tfluvy6VjRXk7T2srun30jVjcpA6tvV3S937JFXgBfGa1K9lveWjfIvc6opEqdiqUK92kob4niFmYN7M7VUfBlQL4e6OVBRrXX/qYlBlazbG9TuwbOA3752VHISViwM2Va3q2pKKqcKvirfO62ac8p5F61NSMy61Fj0HaMaAZByoGY/KsL4ddzl0urLIvJDK0OyijRCtbd8tt7SNVR5gV2nSe9rehgxJLW5XfrZ78u/1izZ/d9uXeVbFrV1n8ncJQ4Il761KaVkve2RHN/twY26eGdVXz+Nr87+ofH1Km8/ku1Ykb/fyWZk+pf8/RN+/rwy/Sz9I+lErlhkWT3nAh3xALA5BuMs/pAOm4nHvJpxLh2AyIjkc7Yyn3KOQxcHNiIUtd51NeIwD7vyES+4QbIYFobdsbfwmHXHvOb9a1ubiq7L2lDr0dlEsPsYBb8KIqXhccqjKXZsiWaTicOfnankEWFkCgGAznlLH5ENk4ImJiJQkSJUKVa56m/EoVlKFgaoldS4Ob8OID3EAAHSW64/F4Sn28DJuas0ElCnXoROLXyUlyh7WlJVF0tuCMfsrBSUVB2+zWCWXDZqOefvXG65DrXneFhz9jHPqan9U8fS24OM8iHW0WcJk6/hZU1DI4pJYu2nZZ1ur6pTZ4vuT4cRrZz7AmVLl0DytcqjKo/rrpqZdJhUSyJ/mKw4WhEgWweZqaW2tompN7V1Cb0uVu5BBzI4VTjK1PM9jFqipreUBYEoewWV0krcQQ0zvwow5O5xjV2Gz2q/cWG/UUqlz3H4P4qesn1Ox1TKqFluFOWsbWSyFRsZeLbla5yzW5CTW3JaZORaL3mXEYiv0F2DrbHC5yu3Vyip9KWRw182I2XH7sgbUgjwJhFatvr6x0ipxVRDFVee5hYTXfhWD4ApyMTiKQvZx7GENIxOcZRjhWjlPdZ62rLDKiAyw4nyaOtz1My7RrzbgqsSqPIPPuAjcMwu8V2GVfEiEFZT2vp/xNLFvuheIsG50O59wmUP1hY7ZVaisWpu1XQLqOLXWaO/Yb9027LV6aBZchrOEKJC/ll25hbVnebcYQ0jZsS6xmRddj5rad5UxhGMfkbKtiAUtq/BrYCFGU3i2c7we1PI+xeUdo3712pbCodXqqtBqaup1tiCKnzk/73zPuSKW3uXdaCDW94YxmOHCpr4r9K/68VdIt4+V5IvXSqkWZRcih9U6zCuf/i37s1qoFY68ZZdWy7VClW0DA1WotdYPoFquAfZ3hyF48ceOs6/WbIVMAmylVv9xYwklL2ut/sYUC9cnhmgS+4RbW2DFKq/+/SU7WMdwayv9cD4jzp+/9XGuVGipaaypCjk1Fit/dR4HSNsLlFTzqbKtIceoOSuuedAopU2yErrLGBkv17Agt9lVCVffdoF8Vjm1/kYz22U5bpmUjbBJy/UKRQVgPK2U6Ao5VTjpVj6td2tFp8VkV3Wbvf61aUeRqJBRnY9WOVboq1okNyG2rqxzK+VzI8TeQUxzCNJefjZtT9W21/byG6zHZXcc8Py47Sr/t086fjBF9yWlEvtD8mNOebt2fpj0U2PM/9J8//tE9Pc3eQjA/2B4M/Cf7tz/3ulHrVhG8vh/Lj/DT7onZFj88vIGAPC+O+FdOONPLu/R2YRfpXu87074v08/w8/7J0Sy+LPHn+M+sD/bV+M9fv/wAX81vsHbMOJjHPjfNKB3Cf/s21/iz85f4OgjrCn4erpDZzP+4vQWBQZ/cPcdvhbfzVPs8eXxI/7448/gTMEX/QXn1FX4nMI578KE+zDhj779Et6WCu2Mmf0Wp+Txs7sneNn8PM49epdw9DNaptvHuUcuFqc54Mt7Dplxjh2OYcYfffgSx37G237EH3/9M3xxvKCzGR/nHnNycJbw87snPE49vCn4OPd42488tqKo//XlDrP4DBYyeDuMOMdQr3Uu43Hs8TBMuMQAL5t9IlP9Dd8fzvg43df7g0/4ar7HQz/hLy8PuOti9R/8q6cj3g4jHscevzi/x0/vT/j2fEAfEo4h4s8+vEWMHj95ODHsVEiYtN0p+urTZS3hzYH7o/IaQ3gae4GDGrwZJgSX8b/9yd8GEfB7P/2AOXmcpyAQSoNjP6829X9+fguAN6Lqh6awz3EO6ENCJoPzuYe1BcPAG77gM8YpYOgj5ugxdLFunhVqeTn3yLPF8c1Y4YvBZfzyuwc4VzBdAo73E5MyRQcfMrxnfyT137o/Toh5YP+iI1t3pylg/tDDHhKO9xPm2dfNn1qCAdQYrc4xrLFkC+sWsqZ58gsJkED5+iFiGgP7CR0jxsce/f3Em1xRTtRXKM0e3RAxjwG+S6Bi0fWxtqswSYVQdn3CdA7oDhHzuYPvE4zIN88LSVacfSU3cmKB/+uPX8AOCaFP4udkkaMFzQ7+LsLYAue4zZyYHMd1uc51mgROOnD5PDu+D/YFsoGfwhwtXCj45k+/gCkG/qcXVgwApMkhHCKgylM2+ObEhwb2Lta2dCxtWMifylOAfzvju794A3sfhYCKN4PffgxCggOGJ44O9iEum/GLBwrg7hPyxTFsLmTeVBrCh6/fgd5H2FD4frSVCMeeLcq7yBvK0cHdJzw+Bo4pe/EcG7bx06HJMdFGX5i4yBPDKQ8ZGJ2QJoFJgQjA6BgmeMgLgY+TDVg0C4yxK8DFcczXQ+afQ08wJyZHomOGOTnQUBZopxJtCZHN9LVHuiNQX9gadmEYZ36basxUhVpOHxzSuwQzWbizRfoiYf7OIz3kutF0Tws8FASUQchokkAhA8GdLO+LO45v6SaDdFc4bqUQ1FAguDPHWfUnhroSeKOWe6lTNqNZYpaSgZCnAOSAfJaNiWWylHLmuKemAKWnGrOzBAJdDNKbgvire5Aj5B5LnEJi0pkSOK85G75/EQvizPdggXJHyCNgzgZ0T6BogAhQBjCAWXgNgF7glj0BKufARDNGYoHijcToFBntxPFk6Y4qxJPXCVU4oZ0FxjgQ3KPh+JSeKizUTdyfNHA8znRHHJ+0W4h1whOA959vxnAXA3Pk9epEZl4T3B7LvsTZBFDjXvoTXyuB5SOn8wq4C+fR+KcghscqAEfPUFV3IQt0H4F0FN3NcQzaElDjQRpiuXLHZf0FyP0yFmQlXilxDM4KB8WSR+NtFukTWiiqxC9VwFbupJ/nRV6O5cv5/AVIB27PFFTIrJ2l72G5ViGvBJgaL5VQvOE4pUnqyYDNEkfTmjVpkmjlxcvaauK7GjFla+xHchILUv5WCylk7TVxSbkCbQML+ZAS9SnUGQY2L2OxzV/jcGuVetBQoeSosVtb8qI25qbm19jKdfxa/VYhxLoubsFrpZ0t2dHWWmwKx9JsodPbZDaQCo2puYqtqePYtL2r4N6S6Yb8z1n+/9GNrv+WpF+9Aub7rxLRnxtjfg7gfzTG/F9E9D/9kEL8qBVLXTQf0gFJrD8FBqfcYyq++iYWMjilHtYQTrkTghn1g2Oo63fxIPm6ahUcfETnMr6Z75DIYcyLlW4uTEZiahm2ShlDGHOAE6ueWgAXKx6fLs7F4ykCvU+LnxaoKj++sY4AqKetY17jC1XxvBMLxuKrZnHoohCs2AplKzAItsAGJuqPwoxbDFsJYlnkjYWtBJ1P9RlNQpbTQu2UpMWL1Urj9AFsnVALm94nseSw1YnnQcN4OLGQGAC9QiA9k4MksVI5OSmtBAeNX5qSauiJf2u9APhkWP3rnGWLWCoWw2FmxaxabfLK2lLI8G9AYylorVD6OQjM0YFJHoxhK4zx3PdOCDFU+XFNHdYQQpfgvPq7LVZIL2y4oWe4npKF6Ck/s/ey0qFroOuWefM+I9/HWo6JMhY2YFctVqVaIZQRuPWn8yFX2ZXtWK+T52t+SLVOzkdiHbJAx/d8l+AcgWyGMVgsH1LGtHX3Ul+Xq5KrfdY8zmdQaZVgwB7SxupS4AJQHOev4+4WH1O1eBgDuI5/vdjyQLAd6n0EVIuXDSLvUdeFWFYMwXVszSIQky06gA6JFW5HIPBzaCzvAq2RwOIA6Ji43kOuvnzVenTMi9+U+BVZSwvzaM+KmDEE24tcjuqOojxktpAYYvZZtxBpFEdyD8AgUch7loH6vNzT1LHjlXGFYZSWQI5JOGjIlXTDWO5r6Q3IG1Y0G1IO3gyJhcSK9WUozOopSqmxBDrIvPjCn4VlF47LwagMhPS2sD+cMKtWw0Eoy8ZLfT/fAPAEQkHyBHhipTJQlS/fS5nWl9AS4Pm5gyWke1GSRX+ijv0RsxelV+s6Fs4veStUtPpYym+GBJ1XX0KeTHAweixyxMCMp0bYQSkY2XQSomd21PmLImVptfmqcSAdED3fL51ZNsuyxqgjkDcoPSvAJgvjpzCGKtEHb2AJxQNFfQA9kAsrMCBWfktHdWOce1GQAsH00teChZ0WfE/rUuVKZTMdkA+SzwP5yG2XsGyoyXD7+Xv4WJZu6VMJMn6yJkoATMeKbh5Q50vXWgkqM8tKhj8TuP/AolQYwmJFa67XZAAKhsdEx3Awi6+kugaXRSHS+8qSC4Mazmu3nAGs+GOSJeRB1+JaCUqNjyQM56vGz0apyT0qM64ql2TwvI8l+D6XMXX8KmssYRkk/Ujtl+01Hd+mjNTLzwH/vVJcnrP2aZ0N62pV3mgzp42CdpU2ypReaw2hVbFq10b7fauwNfXeVASvxuZ5xXIR5lq+2s/dTrYvsJ3bz6VX5F8prr/BiWB+beFGiOjP5e9Xxph/AOBfAvA7xVLTWDz+anzAh/mAVGz1W7ukgDH5CoU7hIinucMxRJxmZlA7hoivkq+EHL+Y3+PQRYzRIzi2xPU+4XH2+MW373DXz0jZVZgZkalK4V9+fFMJQjqf8Zcf3+DY8xHeeXqLrvGpqtYQgWepRW2MHs4u8ERnCL86H5FEMWXWUVOJdjSpr1XvE/7y4xsQ2IdrSg7vjxecY8B3lwEPw4QPl6EqZwzfs/jm6ViZGYPL+PbEv8xKoNH5XFkBAeA7YUBsrWydT8ycahYYWyFTIXHTzIx8FT6YLUJI+FaYBzVcizIDfj0zxOo4zPjqwz36LuE0dhyeRRglv/54V1n+NHRKq3A5Ua6/fTxWy5/CybouVZbJ72a+//4LtvZ+/d09nC8IIWEc2eL4dBpWb9helLvThXcCatVTZsKpIWcBgIsQrSjM7GnqEULG6dJVuJFCj/qeWX7PY1fXS04OXR8xz8w4ebl0jdXO1fKsnAGPTx2sLQhdwunMMjpX8PbhjCkGXM59Ddui5C0a3sRYQkpgCJmSoojCbcA+ayk6WFfq+E9TgPcZzhHmKaAfIsYxVMUyJ4tYfFVA1dKYIsMTL9PiIFmVRs+Ww2kMCF3CNIbKeJgELmYdIc1c0IUMYwtKtkgSCqAfIlJ0mC7CymoB6zK6QJgnniPK7BNsFXIZOdwQEarymWYliymIE/fDOEIW66r1BWn2ODyMMAY4P/ZLf3xBvGiMAVb0wsAK73juFl8wC4AMUmTLZSFTrbT93Yx59PwukPdId+SDEBIoZjhEbkcgbl6stEnIZ6gwOY0yLt795IzzU48yO9guC0uk+FM+jLg89VC2xjx6uCFVIpsS7eLfRwY28K5PCW9Ksqv8VHiNl8jyu54PbdLsahgdFIEHajgbCTFiQ4YdCHl2NUSPOwp51eTgjgl5tjWkj1E4oXwfvjxhngKT7hiCO/CzG88BphMoqVhxwxcT4imIH1y6AAAgAElEQVTA9Bm+T4hPHcK7icdVNlb2PtbnxMjaJiUoshzGxop8JTKJD4+hg/FNe9nAHBMoWbiHaYEUGkKZHRPzCDzQ3IkVnRZZUQzsQ15gksnCdhk0i4V4toAqT9nAvMmgs4f7PWYy13w61hrSh6LlvLODHVhehZKiGLZYv8kwXQJdAocZchxKKs++hnUqstZotijvpN+zZXbPjjWCcgpL6J1suC5LKKNnoqFkax08oGYVRofe8fhVJlBfQHKIUkaP0mfQxaEccm0bxcAfEvLHDp+bci+WeIAPLNR6bwmY3AL37PMim55o9Jk/R8sW+QKBhhr+DixkR3JYUg8xksVK2yGDdEwLIVOROlLTHrAOedSVhVG1DZ1kwGF19IBByykRFJmFsInMomQYySOHCpWASYmZtP5s2PrcFZhJ5tNiOUwRRV1D2BgyzFlJGxkNOBxPMhVFQIKIUCjtlcJFqKRBRsmHmoOcRbldlO7WB1eVttbaWW9IqoqVPrNlUdy17poR2LC+Xk3rosgRFuIjau43Vst6AOOa6215s+Qx8n1P7VKdtG51mu9X8Fq9uVVibyjGTdevhu9Zy+Nz31s5Vp34zVYqNe1xgfzjTsaYOwCWiB7l878B4D/4odv5USuW3pTqc6dQ00IGd2FCIgdvcmUvPQjbaWj8CXuXqoXqECIsqBKPOMOKU2czfnJ3XrWhSX3Q3h0vK8WxP14qdLQlNWlT8YsfF5GBk3AJaqW0hsQKeU1O0T42RRQ5a6gqs85Qtdxp+A0lF2n98Ywh3B+mVX+CX6CAalkEUMt4tZSJ4qbfAVZw2wWlMrhBywCFMsgbOEsNw6fEQPQaamKxCB7Eb811pfrDFUK9rqf2hUrtQ5UXHNpgO3bOEIxYcFX5iYktt/3A9RoAQfLYIa7qqNZJgY8iLL5ObEXMG1+oxbqo1kJjlvIteUcW/6rWUqjKshdorsJW9dqWBCg0JCRVRrBfGRGYIt/Qql31FVICDmsX0hP1Tak+WeoLVXjjrGOoFr/FP2vJ76xakk21jqp1Un2wql8PLWOiyp0eDBhhwYXlX1onCpmW48MFlk+hvrV+7gxyXhRYNOPM/W7GQcfHUf1Ns01dfJ3vWV+q35b1tPq1tKFpX+Tiwwj9xUXNbxXuBvZZM5592YwFWvxSir7uAPRwgJUqAJ4WvzOFn5pGdoMaogBSpsZkJWaMXVhWWVmryl6RMavjKe899cXSerL6xzVrU+a7ZFNDJ1Cz4TK60dPNmRxq5HaTbDgObPvZGN0s8ngQoW4Mx1O3+HqJom1ko1WVtHrP1+tpDIABX2s2MWV2K1nU34vIVN+xMrllswcgZ8d54nojod/TRSZdN0+yQa5WusmtzQGyM8zn9c93SmbJR2CYqtZx4X7EU1gUgyoH7+F5o2gktAqQT+LPbPyyiSQDujjE0QHFoEwORS2pxSA3pguKIse56V/kAwEWCkByVRSaGUKMAlDiNnkTvfS9sv4WAzp5tHE9kRzKbPldJf03xTCLbTO2abasmHxuyl78/7D4CwJ1zgWiIKZfFVzaa+YE0dUxNQTQ3MyfpmY8rzbgBNDULToQATTZ5y1So12vMyyfl/pNFXllsQIALPGVd5UJlV8gzm0lhgCazCoO5qquVT2tJIC+cYzeWykajRZEqyG7lv9KCWnaIoi1a/nbpkXhu147y2vE3BzHVY9eq/t8Dx3ppsKmYtyqe28+NnWu+7RT/w+cdtv6Xfqc9CWAf2B4c+cB/FdE9N//0I38qBXL9/6Ef/fn/zP+Kn2BsQQ8uAvG8v+x9y49tixZmtBnL3/sR8Q55z6yMquyKlutpkqiWt1CDBBDBL+AYUuICRJjxAghkJghhkwYMaNntBDiFyAYAIIhNCWgaJLKm3nvuefEY+/tD3sxWLaWm3vsiBM3b1Zl3iJNCsXe7ub2ctvutmx96/scxtzgFDu8syfM2cIg4TF1eGMuuIsUC3nUIx5Thyk5HMyI9/6IW3vBJbZwOsCpiDfmgpg1/nL6Arf2Ap8skQCZCT5Z7AwZZe/9EZ0m44NlTm7tAKci7mMPp6KQ9zBr7SU18Nngc/eIlDXG5EQmJUKhUwEfwp4gvdA4mpEkVdJCJONURFtIcY5mxK88xZh22uMSG/xh+xH3YYdTbPFl84D3/oiQNHrjRXblF9MteuNhkESSJWWNVnv4bNAbL+RHBgkPoUerPVodhDG31QFTsitSnJQVpuTQao9TbKUOn40QK51iixs74hxbWBWL3IvHUOC+n7cnvJ8OeOMuGBJBmG/sgN54/Gq6EQIiakuB6JbxMEiI0Hj0nTDkMhHS3k44h1aIlkLW+NPDr6CR8X8Pn8GqVIicqB3nQorErLNvCkMvn/dZw6kkUjN8/F1DbMOPoRMCIu4fE0ZplYX8R6uEN25AqwM+zHuJoz2HBm8b0lE92hEf5r0QNoXSDmYABoBH36LREUc34r5AvDvj8ePuAXe+x8d5h854IZxKUNJmJlNKWQkhUcIir8GyOFZHtDpiSgaX0Aib75wsbt0gdfCYsxzL3s549K0QM2mVhaAJgLAFMzHRJTQ4uAkn3+Jte8E5NJijRQL1qf7Msi8s7/NFe8I5Njh5gsE3OqDREa0JuJ87hDJ/WYYHWMipQiY5HADSzs741WeW0OF6f9w/wOqIf3Z6JxtRfA3NH9pMOdgJVkfczbuFUKr0f4xW2nLjRnwzHvDj3T0+THshoAGAz7szxkLMxG19mDvZ8Dk2hIS4n3shfpqihSn1/Hj3gK8uN8SSrUm3d4wOViXcNAO+Ho6IWWNvZ9zPHW6bEefQiEZuvcHGkj5DcLhtBpx9S5q4oZHNu5gW4rG9m9HogPu5F7mgWOYJa+G6ovXbW4/OeNzPPaxKIjUEAKe5xaGZMAQnG2Z8PX//8e4Bd3MvBFy99bAq4sO4FxkhHtfbZsT7YY/eeRzdiK8vR3zen3E39fJc47r5Hg3BCYSfyat2jmKyec731uNh6iSPjwYxE8PzFCxuSlw7jymXyc/RQzPJ+IieZdLSFq5r72ac5hZGJ4zBwmlCiVBM/4z7qcOX+xPmaMpvOEnbnY4YgxPt4rNvsHczQeoLkVnOCvdTh30z4+gm3E19CU0gBMsQnDyTuf6Ld8JGzbrAzCx9P3ZC+sUkXUplDN6hKYRdrG/MKB8O6Zi8xa4hbWcm0Wosxe8rlXEuCKWHscWhnanMwnJ9bCd8e97h101943EaW2QQ87SEK2jSEG5dgA8Gu3bGWEi3+L7tWpIi8kVCidvOoSsA6SIDZZNMM7kVBL3E9YWkcduPuMxOvnelbkYO8fOCCa66ItvExGupII6oXiuoBQCCVGKGeCamqkNudLkvjY2iw5uzQlfuDbc3Jo0QNFxBrHDisAVraQOMSKXoXGRPbhmLmtyJY/L5HFsemSWyeIO51BMLwRbrbwJYoP+xbJrK5ph+suEoxEgrHctqwyOvyxTmaZWlTMmjICijp67IvC6bjbyttcwkRzXxkMZTo3BrSHL+bT3YXFv/ryHA2/K2Fvxr0mpT4IW6X7rmNQby73DKwG8FCptz/r8A/IO/7np+0IblN/6I//T//VdlITgVxkj20p0L+yXDXZktE4C8+AAI4+RWgzCUBcfOeUyV7mAsi+y51NdaImvhF0hjg5yryWw4pQyBvY7lpWALsyTH81FsYfGYKYjuXy0pACz6fbF6QYgG5exWDIeuxPmRqDxd37pAcZ9YdBGB5XdMbccCDzRJhI8BiLdEPErspVIL0yBDbfl8fU19LRHGGGEl5BdRLUDuvUFOJC6+1bwjNsDFOwEs3irOC0CIYfiFplTG/zT8CQCgaYMImDN1vy4eNu6bkN6U88KaB6zYB1OB47GGnlZ5xfbIDIoSNwcgFRIZ8XKVl3csWnY5lnN5LVK9eGUgJC8pKvFS5aiQZoKnEfwQFWvf4iUShtLywOaXLacVo2A5x0yDOZMXL3sN5QgKJayDXF7xrBHMrYxfxUjK8Clum+S1icplUewyFsJoGMtOfIH3AiCPkKZYvFWeqARSx/NGPESmetmzviDDv1J1nl/kgHz+3+Y/ojLaykMZ1XI9747X5crAlkYzOyJKPpfwf/gfU97qJfq/e728aHmBwfVwvVwHQ+j0cv1fTH9EhDYKT1galSfIGnuhBErH0LhqPkieMvZ0Hov2Yb0o4f8JEgco5695LxSVA47fyzTOqkCdhdyFiX+weLdYPPzn448pBo2184qnKjVpgacx8cZcyGYK1C63GV9NX1KMX2k/1y2eEUN1iUdGQeLFpNxUCFn4dqvSvgLn+4Vfb/9TnFo1BkyIU00TKEDzOCgaexWUjNMKDlfGMbmMD9MXNA1Nta5l75umdn0diABFB74WKwKZbyOdY5IZJnHJVR9ZqzBr4L6ARlKpk8lIkluPiZRTCFzE2VU5iU8FzpcMcPYQmCC3U0W6D8lmPEQaj8FT3DDDEx9m9b1iLC88zgAmifNb6p1K/OJYdCypf5R/rnQsx42Opa91LAGZO3Lvq+N83cd5uQ9ZAyEsYxIq44KvvRSCl8T3VAFTFd+Yq2eML3WHKuaRYynB79vy2TOctNgtviKQyQWeaRSQI2DNUgdPe1UeN6p6RtUhxDJHeZ7yTyavx6m2bWovl5Dd1NdhnfeTeRI+aTutPGtVH1f3rozZpzyCV8vkMl4yAF8y3MQgfK79mxjFuo5NO19s47bea21/Jj31kj/9/mx6RZ7/5xXF/D59v/SDNixDIjmJb857hKTxph8Rs8L9fY9pdHhzc8HoLQ7dhPtzj7eHC97fFebYwwUfLz1C1Ogaj68/3GC3mzCMDsYQRPSmMKD+/FdvsT+O8N4iBk1EJVnh0E9QKuOXH24Ecti3M3714QbHPekz3j3s0LReWDXZOHs8O0Rv8Nm7E3JWOA0tnAtwJsIqgua+f9wjBHqKdv2McWwJFleNQdfTrvKxH/Ht/R65ENGczh3+8PM73F16nC8tvnj7iA+PewRvsNtNErN5f7+DcVHYOU8nkiRhg63rZ9m5VAq4nDtYF+AcUeDPs8G+n3EZGzRNgPdGoI7TZNE0Aae7HUwT4cr56A2aLuDy2KLbzxguDbTOGCYD1wWMjy20i7i5GXB/v8P+OGIcGkSvsb8ZsW9nfPPtEbaJmCe7MtSMTfS/GMCXU0uGWhMp9i4D3W7GPDkonTDcd0BQ+MM/+RYA8FdfvYV2Cf1uxjQ6aJMwXxqg2tHsDxOUIjghAKRJi7HW9h7zTN6RN28umIOVmMbgDTGoDg5NFyj2T2fkQPFO2mQcbwbs2hnf3h3A2oJ+tOhvRvjZ4nhzwf39DspkNB0J16dAsEqm258eW+gmot9PGB7pfpom4qc/fY9vzzucHzrYJgqkVlhRQTBZiRszCbYhI5+NuHY/w88Wpg1oWw/vLaYzxQEak3A5tbj57IzHhx6u97QLHYwYh/1hwHAm1lg/kdxAHI3Q5JuW4saco/9+cGgPE6bHFv3tiGlwZMR6BdNHxFlL3JTSCTEYiqcDcPzihHFo4C+0eaNcgm0C2jbgfN8T1DMoqCbBtOQtCZMFDBnFzS0x885nuv92Fwm2qDNMX2INk6I4v8Hi7U/uYU3CN794Q0YYALP3iCcnMULKJezeEVT+8W4nb0ttCWaaJkNENUlh/+6M8/sdjn/wiNPdTgxlALj58SOmmTwM0Rv0+xnnD72woTZvyZs137cwRy9loyyIf/TTD/jlL98AXkPvPVwbEEqs87vbM77++hY5KpiDR3x0cG9Hijk8zhRzGMubPinovaeY04uFu50QZgvXBsynBrYPSJE2IRj66fYergm4PHQSk5mDBhKg24jkDbSLSKOFvpkpJvqxpd/KbODejjQ3Tg3cu5nuTzF8VVOYb71GNgk/+rsf8P7+IPGn7W6GsxEP7/dQJYYwTfT7O/7RBQ9fH6BvA26OA+6+PuL4hyc8fNxJnFzzJXkXXYGTX84d0qyhS6xhmgzaw4ycgXl00Caj62ec7zsiUcqK5o3XNDdmg/2by2rjbRqb5TeRge4wI8YST8ubYkGjvRnB0hreG9wcL7h/2EObCH9pYDoi74peoz9MuHy7w7u/d485GFwuLayjZ5KxkWLDJ4swOOxuRoyXBv1xJJRBO+M8NQhJ43TXozvMeHc84+sPN2g6j77xaGzAx9NuYcQu74np3GB3Q2M2nFsYm3A4UOjIxw8HIeYK3qJpPZyNeHzo0fYefrZwTcB4aiQO0/TUp3B26N8O8LMlpuegYfqA436EUhkPjzvs9yMe73Y4vLngcu7QtBRz/e7tI/7qq7e/9rqjP04Y7spztY/QHHbiIsa7HqYPSLNBfxwxTa7EF9N7fP9mQAga8+jQ7WaEoOWZxUzgfnJC1OWaQM/frJb4bl2MOq/x+ReP+Piwo/s8G7p3I8UUq0KMpU0s0G56d42nFk3vEbxFCgquIy/vfGkoFrVA+U0bkQoTdkq0MdnsZ2EE14UEzQ9OYr794ICk0N1MxPAN4hqYJ4c4GDTHGfNDYSkyWeD7rgsUHz8tcdfZl/hZBagmygZlmg0wLe1UthB5JUXPEd4EAxYSrFkvm2y80ckEVhxz6hURdXm1MG5zSiDW61kvxsvWg1iXyR7F8szPU7GGuW7e1LpmBPJGIaoNutqS4/N1nGwCtZ37Xm0019corwDeDNsk3nhRqdoU1JC416vwaWHBrbRMS59Y0kWO1xEh283E54zl2miuNh2eGMp1ua8wXn/bKUP9dcmN/E4klX8gga7X0t/7+33+j/+rP0PMGj4bzNkgQUMjIUEjZo0ILTBOdj2nrBGhngTPakVQTc4DADFrgY9xikXug7UkGeaaoCQ/axnWnzmP1Ics9XEdvsCnuM3Urqci4wC50p8Tf09ZSRnb6yO0wFKdijIu9RjV5aSsBQZajw2P2XYc6zGpj8WsBDK6jKVaeYLruvhzKGMi7Snlb3Uz67rqxPdp+7mum5lrW0MLI9bg5L7VbeL66nHn+1C3ZVtnXfe1+1YzBzPUbnvuuftd56nHZaUXmSysWmIuQ5l7NJZJxjtt5ms9v5+r79r8fKl9L81bPr9t//aa+vu2foZfAVh50XOZPxxDvap7M28ZAmw3c/a5PjH0i1EPL41DrV35XNoK2Nf5fdF6fWlsuF1Xnx3V3GcNye26AVje0Zz/2ob7tbo4Pvta3XV/tn3b9pnL35ZdX7+CrG3qq5EeACRe0+iMtBl+1j7M1f2pURx0vZZx4XHYjvF2jHJp/7Ob+vW1V+bFNoZaJGp48cnrtoLKqBEmfJ7PsX5lPcbXyqfjkM3QJfY6F9KoBckhGofb+SwuxzKejIbh36Veu4AYtaHrPvLYVd9lbK+4NVgXUha+mmJ6GZEBhScojO+camOgrr6auwytfOJBEpRIZSjw3K/u56uXZVwHl1N/37Rv1XaGYvJnvr7+YW8MBaqP69qUvenLq2GRXP61B0uVVFZrA0Rtrnv2x3VlvPF0qj4xWKpjanvupXtzpd9XbvUL13/P87juIPzOZW3OXW33q8bj+VOv8kCuynr9nHqp7L/4D/+d//kVkhx/renv/P1D/o/+yZ//xsv9N/65/+G33jfgB+6xnLPFL/0txuwwJY53o8UyG3g+G4lPAfDEiOIX15TsKpaQjS/OA6wXxEEkS9ZGplVRpEeAp4txTrURVct8UD1JDJjtYvGagfRcXdsdkWtGEieO87u2QH/JWNp+5nau6133g+OVtuUv/2lctuNcj81r07ZdWuUnBh63AwAefLfKe21h/Jwx81yft/meM2iuGTxb44bzXSt/295r7a8NoOfyXFvsvzZtx+27jNWnjCGt8ou7fNu13tb4uWZ4AYsRdO341vh5aSzEcH3FPXjOuHzOiLhW77T5vjWU6j5sPz9XF39OzxjlL43BcxsBz93LJ/MOEGbp7Xpc1nZX+rNtz7ZfqEIRruevvxlskw9Pj12r79oYsZyPr+D6r0mvHfNtCk+53qpC2RBTT49f6ccqS/kfKy66VI/LlQX8k74m8/z3apEaF0uJ6gSXd+25r64YT9VvmGGeadOf9LrxfC49dx9zwSBfbevqGnX9c/6E7fDiPLgyFs/WX9Ur9+5T5W/KumpsfKINqDxYV9uEp+euGbBS1pX812t9xmB94dqX6npVncv550b11bPwuxhhVd5n633WEP/ux180EH9DxuAn2/Ha9H2v/w2m9FuSG/mbSD9owzJmjffhKB5LJpMBlgV0hH5inABEuAJAjDqtEgaQcZryQjQRsxKSEleRmfA5LoMNRar7qSG0alfW4nGb0xKQvzLgshKvVe292aZrniw+Rh4qIoWpP3P/tMovGsIaWcbhJUPhU160mlH3mjcNeLqQ4vvLbagNQvboPef12y7sr41tfR5YyA0kxnS7OC2pjoG95lWs28Tz49oO4rWdxIzFm1V7SRSASylrFQf7jKHAniLOy8d91NBqOb+9/jkjs17Ic3xvbcQYnSU+mD9vPUJ8zbX7vK2v/lzHoXJ86rW21WzKfIxjh+sxYO+cuTJvtvckcxu4/9X51XqmtENYaM16Dj2ZP2VRq/XTe7f1SLGXqWYJrsvga1LSBYK+eKjq8rafWfqmLq/2fHEfrsUxq83E3XrGrhlE2zHYlnNtE0O8cmIQcQGLBywnLZ4wPlenGLVIgwDlvlXzqr6n4vEDGR66sNoSIy8ZTHn9kwfUUw9TTpvfJZOC1HNeLfmeGDlqY6ReM4IUFk8T15Mh5CFbN4m0ocRqrwKZ2Bjg4zUhCIAV4YcuxxnqVxsS9U4Of65gfSsvU1YLbFA6UI7Xccb8ndM1T1uduM3cTv5fPXRVXGIkf61Ux1dXBpbKBTLI37few2vtA657AznVzdw+fLgv1fiopFYMycBmKpR4xzp2sy5v9VPb/ldYYmirfLk6vvLMrjJU96w6LtPwWt/rMagMtLzNmq8c3N7e6sInr53tHFTrY0sbr/TpGePzyTg+165r6dqL6Era/oRfNvBeV+Y1T+XVsq/95n+NdHW8nkvf0yh8leH6N5RyxgrJ+Lct/aANy0tq8H9evsA34wE+Gty2A0IiJtCLd3jbDcIseJpb3LQj7kbSaXzTDTj7hnQurcfHscexnXDxpMe4dzP2bsIYHX51OuCmmzBHAx8NGhORAeyKRMn7yw6dI/bDnfO4Gzq86SnG8jS1aGwQbccMCFueDwafH84AgNPcwOkkxENOR3wceyH+6Z3HHA3mYFaLlr6w0vXW48OwQ8p0bPAWPzqc8GHa4Tw7vNsNeD+1mIPBrvFwhVTobujQlnjL1gVcpgYZEIbXXePls9EJp7FFY6Mw7Pmopb7GRiE8Aoi5r3WkQdnYCGej6Hd2LmCYnTDnGZUxeYuu8RimBtZG3PQj7i89Dt2EYXYIUePYT+hswFf3N2hsEPZENha0TjB6gcmexpZ0/mwU46NvPKZAsarTbJGixk+/+AgA+MXHW1gb0bmAKRBh0zi7lVbmvqO41vuBvJtMLpQz0DUec6BYmGM3YQ4Wo7fCnsf9a13A5O3KYNI649BNaEzExwvN05SIaW/XzQhJY9/OuD/30JpkWXw0QoCkixTHMDpYm9A1Hg9naqNzEZ8fzjhNLe5OPZyLYhiHoi0KLFC3pUz6zoZF13iMlZZpTBqni0PbehiV8XhpcNxNeLy0EnccKxa/xgVMsy3xuNR/P1uC1aVFSoUNm2lo0HUe49Cg72dMkxP9Ro4HBVAkVoApkJQHABz2IyZvcb60ZGDaCGsTGhtwupBOY8pkRNXkUGwotEUL8lKkOVyJFVIqw9hEsahZSazazfECozM+3B3IICjjTnHAAMr87DtP5GKXVt52HIMdA+mHAkDfzzidOtwcB5wvrRhJAHDcj/DRyPxomoDh0opB0naFifPSwBVCKiqbrn97e8bH+z1SVLBNLCRZFEd9sx9x97ArfQsYLw26EnPctAF+toshlYkgS+kMPxvRGuWxso7JsIyQWbkmwLmEcXRCgBUTEYoZGxHL/PKeYjWtC5jGEuccFZouIGfATxZNFzBPVow0bWjcORbxze0Zl7ElKRiV0ZZ49/NjR3MtLyRbu/2E80MH4xL6/YTzY4f9ccTl3ArxU9N7mZ9KAfNk5b7osrnQdF7upVIZrqe4ao7NTyUWzLYBMVD8Yw2/5bhBvtdN78XAZtbMnIBm7+UZkSLF1g9DIazzBqb8llIkTdTp3ODw7oIYtfzmmKTMmCyx1k2Jo26LzBI/t1PSmC4OdhfQdx7nSwtbflNGJ4yTk/h6P9NvO3oN15Eb1U+kc9kUqajh1MI4+t3FoGEcaevOI40VPzPibMQ4ZlKzNBm440yxqUEhR9LxdM1Sl20i5otDs/OkmWroWdbvZpzv+qtritck2wWKtc6AapLcJ60zwmChm6Ll2sYlvrjMH9vTMyXOdJ7OlfnQB5q7hZhLqSyx5+DjWDYPclRwhxl+LDI5QUN3AdlrIkljYjOdJb5Zd7HEcVMsco4KqtyDNBuACYcUxS7moOh/Bsm2tJGYUcvGgiplq6JbmT31U/eBiOgA0aSFV1BdJC1UYNmoKOOYMyjekQ3lUMUz6pJXZWoHk3YBZNzUsY1XCMM4v8QE1nWXYwiK4jY5TrE2xAqB2EqmZmWQqlV/6hhLIeqq2iMxlBlC/iUHigGuisGcq40tzrOKhyyGea4Iqtho5zbyZoKqSJueJN54KZsPq+9V8+rurjZ6rhnz9ThtjPCXYNNbm/1VxvPv0+9M+kHHWP7kn3+T/9E//tcA0OKYYbCL3AU9wOpYPY6bC1mvYv4o3nGJwWTYqUgubL4DwJxM8QIunkeGLM4lBoqv5cSeVJYemKNZ2le0N7cexRpuy/kArGCRXHZdH9fDTLCq1MneOaYL51RrXHLyyTwLe2UvZxDvbF7BNrkd/L1uTw2H5TZv4XN1nBN7olaxcptx3UJqX/LwsiG69FMXgz4JVMpbShwAACAASURBVJTZf58rYxvbVrc5F48ha3nGZ/q7LTcmJfFgfI9qT+FC/879rq9fH6u9i1oRjb0uxk0dX1V/v5a2XsvtOLD+ar0hwV58o9Ze0Csbwqs8195N7B2uIZJ1W9g4r72bAFZwTmHeLd6y2ltIecs91YuXrWYx3qb6HMei1WU8Fz94zRP+nBe3bgcb2ny8lgVYyoN4AmOkjYDnvMOJPXk8LtVuPHs/6yQeL14QVUzI12GQVzzS5XwdS7fN+wRKWl++GZdPrjayIp1LUzEFpzLnTdV+zl60OpGxyAtsvY8b7UtZBNc7+GndJonpu7JSehKfWD5Le/n85rqr8XTM2Fs2MVaeQvayFVZS8QpyeznVHsYr3lk5Lx7LTfuutXN7v1JVVn0tL4SrRe11GAHlUVGRgVAtYMVIKNdmzYL3WO5RVEJi9WulZzyWANYe0mve7bh4FMW7iPI9bu7ntq46yb3DepzqeXjFUFB5MTJWZZVrVgZEPT/K/Kzb/MTIyEsZihlv67bW8zBX56qynoXzpCVfru6vqsup69nelytjtHpsbObq1UfKS8euGVbYjONz6TVT8RV5XmN8PTn/yp/BJw2631Q5XNYz4/l96ub0v/4nv/0Yy5/9+TH/+//kH/7Gy/23/vS/+633DfiBeywB0rL8+fgOCQpv7AU+G3yY97j3HX7S3+McWhzdiPfTAT9qH/DVeAsA+HF3jzu/wxAdjm7EV8MtvmhPuPedaAB+5s44xRZ/8fAlfrx7wDk0uESHzgTMyeCzlryNf3V5g4ObkLJCbzy+Gm7wo/4RAPDttMfBTrKIClmLLt/HcYefHb9Fyhr3vsOu6ENaHdEbj18Mt5gLqcyNGzFGKzp6AMFWD24i7Tk34heXW4SsceNGnEKLP7v9Fd5PB9zNPf7uzXv8cjhijA6fdWfSI0wWX11usHcz5uLxvZ960mUsemS3zYBGR9Fk+zjtRFsuJI05WXzRnPDgO9H2Y4KYs2+xdxPeDwccmgl7O2NOpujdjbifO7xtL6K1x7p09xPpm/1kf49fnG/x5e4R93MPHw0+70/Y2Rk/P73FznrRvPPJoNMRjYmwKorR92EkxuC9o7oNIJp8RiXcTx3mYPAPv/gFAOCf3n2JzgYc3YRzaOB0xP3UrWK/frQ7AQDui27gGCx5sTNpzp19A6sTPuvOGIPD/dyhNQFjcOI1PzQTHqaODP1iiJlyzc7O+OpC8zQkjcE7fNFfMEWLz7ozfnm+gdFJNAV9NMQmrBOMSvj2skdrA971F3xz2QMg7/ofHz7i22mPby577JyH08XozVSHVosXlDXkxCta+n5oJozBoTVBzj3MLQ7NLGP1RX/GN8MeXZG/8dHIRgvf35syTlplnGeH1pBxyhp3rSUEwOPU4m034MOww492ZzzO7cpTzvp6+2YQ/T7eOPlyf8JpbvE4EXsv69zt3Yz3lz1SVqKNx5p/57kRQ5n19x7GFkZndDbgPDtoRe2bgkVMCrvG4zI7/PHtHbRK+MuPny1eRxfwOLayMWJNwtuO2vrN+SBGCyMIWBYoZ4XPdmf84uEGf3R7j2/OB8zBCAz6D46Pot/ok8bOeXw478SzftsTE+eHS499S7qKU2F9BYC/8+YD/vLuHeZArNmdDaIz+Hl/xs8fbpESedhPY4vbfsTj2OLYTbjMTrz/KSv0jYfRCZepwbGbMAWL3nk8ji127VzkjZSgBNgrfz90JHcUjcgmNcU7Zk0SRu/OBtxdeuiyKcZokIexxU03CWOpUlnQF4xk+JO3H/Fh2OE8NdA64djOMDrh68cDOtZELO368njCV/c36BuPN/2AXz4c8eXxhPcnmispKbzdD8iA6C8+Ti0mb2FNhNEZo7e46SZkAMNM3u1943E/dGgK9HgOpLO5a2dM3uLd/oKYlpj68+wwBytIjGM3yRjyHGAWdPr9KszB4k0/4MOlh9EZw0y6jbyhdOgmfDzt8NO3d5iixXluYA0hTIxOaEzEGCyG2eGmH+Ve0rNjxhgcQtK4KwiSd/0Fvzod0LuA1gb57bdFH/E8NbAmYZgdjqWd54meizel3G/PhPRRitAqfUHHPIwtIUYKCuYyOYFZs47lMDnc7Ef4YAS10bggbX4cWxy6CXfnHjc76g/Prc8PZ/zVx9sX1xUvpX034/HSEjKgIDX4t/1w7tA2pCW572ZMwSAVREjOhAYIUQs6J0QjGo/H3YSUIYgXBZIr4w0yRrjwpkMIGp/dnPEwdAVtYbErKJmFpAnQOol25K6bMUwNGhcQCpKkLV7ecXJIUUOXTSXniHWcGeBD0GgZ/VC8ycZQ2W1LXvppdFAgtEUobLbORszeCiv6OBaWbqAQQAFNYZ4N3ggpVIxaNrK0zgR5V7QplmYjMlJKZ0E+pKQXsiZggfd7LbJYvBkkElvlWPa1FFbGdnNIOzp3dUOtbPKogrxBrtpgEpI3kpcZjleGNlAZxvXmjloYbrlelnViVlhV5asN93rTivPUm0/bVG9W1Js7sSpja0i/sCH9bPnXDO3tua2xWG841flfU1/9/ffpbyT9oD2Wt3/6o/wv/mf/CL0lmOhpJhrrvaPFw8NEC/85mrIobGTheiovGqUy5mCwbzzOs0NTFnWTt+Th0RlvdwNO5UVJ3j+KVZuKGPC+nUXU2EeDvvG4TASf61zAHMxqTqekyRAwCaeB2uwc0YqzRmSKGm3nl3ixCn5Yx4PNnl4YMWg0Lb2kOe9waWBsRNNEjKODc5EWZ0ULEiC5klDgd/wSqVkE59kscUxZCUQp8YtLZ4KuWYYulbgnlaENyT80rUcI5gnsypYXl7URKWoYmwRGF4JGmC2azsNPdoHbTZYgQL1HitTulZczFq9QGXDbREBlpAqiG70meYdM8hpaZ4x3HZABdzMRbK+8ZJAh8Dp+3sWpeIwdnRfhZAWheUcGcpExYJgQv7x0s0gq1J4RuSYqoEghKAUom5BGQw/WWROcKGPRDGTvUSlDtwXmNBN0CQC9LAdDcJ4uIgcl+aGxvLz4pcIvtwrSg4yFmp3jrBSAJhGde6bPajTIbVyo0DUWT0dQpCEZ9AJfqvUMhT69/Hel7CZVFPOlPG6LygtsiiFTANSoKZaKNUGLpIUKCrkrx3SW4wBWOpaqQLky3+egkVl7kr0gannJ67Ohr8cou/wqaLme49H0RG1N3eI2kFip6uWpR420i9Bng9Sn1UtSX/Si72cylFdUXpmoatZ0C7sENdN9ymahhDePGvGYyKPjVYGKUXf0qBAPpb5AmpZ61MhNJq1Ht1mYhBIrZcv5ooGYGipbtAbL/VZeka5im1Z09TIOBbKWbYaeS96m/KZMhhk1Ob6aDDPROVk/Ff3DbMgrY08KqQGSy9S3IjkUu1z0FEt9GTCjQthlaK+gPRD6DDsoxDaLJ8ZMxaNRdPWSXepSGUgGMHNpA4czBiyajWo5pwP9N9PGcWcXDxMAaA/xxGSz/EZ1aQvPGz0ppLaMk6V6AbpGF+1GdyKoY7JrOB1D6ZKlPiaXl74W7UMoGjc9K+gZSC2dU4n+koNobnJ/k8tLOx21Xfvys+mqNpbxQAZSU44X7192dDzrMhag8s1Yxkkv5zRrZlr6HBu6H6wjmhVgR8Af8GsnPVMbeWyAZe7FFgI3NDPkN8rhVGZCGf/SF73MBzNV9xjU51pDNC+2iZRpL1Qn/TaojGSXsQNKv23VdrfMvazWY8ZzGSi/4UrPNJn1XESGaLTy8QISo77zczguZZuZ7gl78vie8PVcP3sl5XedOH9GNorawkRSCdAxl3FUG69kLmOn5B2jCrSnrofml6Jy9FKfJFX9DjZp5Snk95gYXqWe6p5yec8ZUbWTnz3Oq3TFCMuaxqPW4qzr4TJX+rbbtn+ijmt5n/WQXju26avcm2L9q0/ZIp8yPj+RuJ3/43/x7/7WvXp/8ufH/O/9l//Cb7zcf/vP/tvfet+AH7jH8q0741//8f8CgNheT7HEk6kIrRIusa2Idggq22mPMblniWSYNXZKVqCzBzPhIXToDb3VmDxmiA4pa7Tay3V8TWBorI6F1OWpRInVEUOk3buaRCgkI7DbLfkPy2Fw4np81mjLk5YZXtn72uqAx9Cik/YvT8feePisV/XTmCzSEwz55TYxLJjHrYb9Ak8hsTXpDsF7E+ZkVp7QkDQaEzGXGNaUFcbosLO0W25L3+ZkkbIqHtNlrGuGXr6nAFZjSG1L5FUt7eX7cP+uR8oKRzcJlJlhu8JgCyIS8pHuj9MVRWJJNexzihZWpydyI1uZGrmXXHZWaG2Qeo1KmCJ5MKZgxZt3jc00Z4XWBCnLVVDcMVB72hKbuhqnDRERQ5A5bT2XGQuREM2dEutYvIaNiStJC1TX1DG7KSvxctR5uA0M4+a5wf3hPtV5eO7VMb7WkNeVxysmLeRCpngp6Vgh3qkkMliawpm0Gu8tZDkmJR62VDxRuSrDmpo8aCGKqvvCcbY1DDQkjdZGjN7CmbiCZE5+0bpTZSwaG5Y4vaoOHtsacjzNFrsmCNPuEies4L1B33op15lInrKiDas3c4WhucYkidXNWZX8VF8NPQ6hxFOaBbr8nGQGe0d4w483zpQib0kIBq7Ex9ZQY/7uZ/LSuuKFiWWTryleGIZFc0ykc+yRoQ2vGIzoFCpF13O8HwCJrcxZkb1X/274eFQUk7u5huMl57husyqxa7w69HFN+sMLqxCWeqCAHBW0TVIGMpYxi7RRNvBml4KQ/aw8LCrDF4/NzHFh1eaSbKJxPBpvcGW1bDjxJgJft4LfqmXzyKVq86ls8ACLJ6b2wMgPBUueuv8A9aGUJ5sTJi/xeLzp4hVy8x1XpVUSbUFgMR7KplGuNskE2pqrjRNfVvYGAMNFVdUuGadSV03Gs11M8zX8Olcllk8vC36GpYpRWj6L6lRluGS9GCY1lLfeEFFb71Wme1yPO/VTPfHEMSxZ8lTxkNUtpHZXfSxdW4y0eqMzbw26ZZNpPVZZDLCt0bjUXW22cN9QjWMdu8lpayhuvnM78tOp+rJBtjWgtsbUtbmQ1uXWm1XP1vvKn8GzhucnynnZCN0Opnpdm76nYfm7krbSfn+b0g/asPxmPuI//8t/WRgp67i759gta+0vFpfexo/VMVh1PFN97Ussh6vP6ZknSA2jKN+Z6ZB/gbla2MvCaVMMG3FPfmelv5nrWm2DcaFXGAevPdheOl+NwTamahWXpZY+oF7Q1OVs28mf61ieJ2+gpYFbpsnnNsBWLI788iiLqhwXggS5d5UHVPq+HY/6XN7ke+6hfG1u8EKDF1Dbsbh2H7epvm91W3gcr7EpvnTdtu9P+qOexpq99IZ7bT9ekz41R+vv147x9c/lvbJYuJaEERLAJS3ukKwq4Wg+xovMa/ehemlmk3Gq467q35t+Kkh9uTYXt9AoThqY0yZvNZY+9+sX+JZd88kArNt+NdVzuZ7/n5oHKsNn9WQ68aXhhQVa1igMfFU7AczVNZHza8CnVuZDKGMX6/iuzVogZ9os4DbVn3khKkZc1baE8qzKFP8sPxdewFbp6vKjMh63z1BVjSl/VmXRLPqNm0Uzp+1CsE6rxfqnnnPcl/zMdFCAyuaqE+LJbHju/l6b13VbqsX2k7ofrzXqdelFr821Dj1pn7r+W5Hv134P6smr4mndm/jfa2lzwz9p6KwqW3Vi04iX2s51FaKZzXPr6ntn8/mJt+xTbX22DZtmb+8P8OR5/SS9pu7tRH72vfGJ+fSaer5DelW5zz6/f8P1/Jplf++6fp/+2tMP2rDsTMDPbj/gaCcM0eFu7qFVxo0b0ZqAry43aEzEFCz2bsLHaYfbZkSCwt1IcX3ORJzmFu+6Mz5OO+zdjClaDN7hUqCxX+5PuBt7tDaI98jqhMeJ4r0+21EsHQBho/04Ulv2zUyxWHnxkPoCze2txy8fjzA6rWKN2INws7uIJ2nwFs4k8eYB9FscSrxTiBo3O4plGWeH1gV8fNyhaQL6xuPx0qFvZ4l74RiMm91YSF2AscS5MNzXmYTz1AhDaM4KraO4EF9gwswG2Ba4K3splIJ4OvYdxRKFEpfhTJQYk8vYoms8fDRoncc4O+xaYm0dLi0O+xHnoUHTRDgThR1zfxjgvZX6ajbTnJQY210/S/wJE5LMBVqbM9C3HlonfPjmBoDC4e0FIZgFopsUmiaK0ZozMA6Eg3INeYhsgQEDgJ8tjCU8nh8tlM6wDd0zrROiN7BNFCgwC40z66MfHBAUzJ7wScRAGjEPDtpmxMHA7imehYlJ2JPExCRN5xGDQZiMMDLGoJEeHdAm2IMnRlP2lOgsBCdCdqLJM8GbA6os0IlRkGJNBALaBKQC4dW7gHSx0HtPcSUZUDaKpyNPBqoPBNPl/jNUVEGYD9mToduINFpiGbxYghUXjw7H2SiVqf7iNeHYm3yyQJMIGpyJeRFB0V9PkG+wBEOkxZ5q0rKhMBlahPYFesyQXIBguDZTPq+RXYR6cLSQfeuXMmYNMBy57NCr0RDEbxcri4T6D5slpkUNBvkQoB4c8j4sXiAA6mQBk8lINSAIKpenMtREY5/3kSDECoBdPET23iLcRDo2E8siDD1UzEXTOVWgvG2EuhjkLkFNBIldkb8E8hTltsCgXYKe9QLD1cWYZoN60tAeiLtEng2TyYuhsGJvzDZT3lkh9eTFzCbDXAoUtk8wF43UZYF0kncs07VJwd1pxC4XKK2CGWmBHg4VRLh43uxJI9zQcTMohGOCfdSIO4bZZZhz2Xgq0NfYoLAx0kZBchlmpHKTpX4YX6C3xQOYTSZ46kR9NBe1bG5kgiqS54v6YsbFK8VQ2GwAe2ZvFpAtYAaF2FOdyaGC+hK8NewzmjtNUMwmL56sBOigkGxGchD4rxvKXKugsGGfYQYFMwGhJ5gkw2EZwkrlU/2pITgxAKSW2qZnGjuGHZOnD1CeyokdQTYZzptamjdZAWamsmKbYR+pb9lQnXomqCWwQGBDT9DXWME/7QDMN9fXFK9JZqI2QlGd7JlSsYyJxxqWimVDwhb47goKW/7MiDJvFoOnJsFJbqkLIGhq80h18hiaUa0hrRlrKOxE0HBuo8CLM0Gba4/eEyisXaC8vGnFEOgaogwFmAFrL6ld4MuxLcfTYvibOSMrRfc8Lf3j3wTDpAUK69bHeJxoLMuzPC+7GtkqqEjXqphX48h9ToagsPyfr+UNJR4zTrVhKjY9bwqVjTiVueylLL6WDiz9XRm6ZbNJzl8zstNSb9YQ2HTtWd1ucqUKtk8ZsEpSVnpaR72RpBLdr8UzuxSUlboKa31y/OqmytP0nYzvaxs2v4MpQz1xfv1tSj9ow/JoRvwr7/4pfLbw2WBMDkYlxKzhdMDP+m9XDKY+GzgVSfPyxsCgxExm+nzZN2h1ED1CnwkO6VSEP1AeqxOmAscEIEyuxAxLw2lVRLgxy+cNMJ8hghoZf3z4eFWDcguFZXjrVo+TYancxwQl5Q+3DlYlOB0xvHFoOJgCBCllGCinUGCpXC57Q2ttS66nZoINSaPR8ckxhjzWOpZ8npk+Q+WV5TY0hu7B/Nai0aFocNIYhzcME44rFt0aWlrDdJlJdwuTreG6APDFnoiY2HD3cSFK2bK3bjUvV+eSgVFkhM/JCBRW7heUnOfvNdNuKLBMhklyHxgyXOtYXmPFzVnJtVOw8jllBf+FWRH01H1jb7PfwJq392wVz4pir5T7mAGBxRr2lmO9GZ3K3Kw3Wq5t7F5j3uV78hxTbw2D1CpjfmdhdFohEmpI6/Z4PR40FgyPXfItupkEha1ZV/07gmg2RdqG4MQEu62hsVxfYyNqrU9qF+l/1qzC4a0psN1FG9R/ZgTaymNUz8eapbfW+BT4+ucGB7vMjVT3LWrsSjw5w9frehh+u50jKzbXvKA8GOFRw1UZOrtlzl2gsFwOnmh0LtI4T6G5W0hx+MzA2QUqy6iTroLpMsohBo3WpiJno9DahPi5htULvDpVUN9c2rDcQJrbjDQRKZUydrmeYyWfAlahDTwWuppXNbRW8gDwlcdbKfKsKn6cqjXCxidAmYzxc9pkqL2dHN/Nx6dCfhKkr0sdyqRCyoIF+XFlgcSID6UzZmGnZYuo1MfxzDJ+5Y89kar6fyXPzLBMHhAutyYyYVht7XGvCVF+nVTDc+sFd1YrGOtVpEbdn+0DMm2+r4yJtUYm5xsE+lo2Zio5jSeeQa6jXnirql69Obf9X8q/1q2twfSEdZY3Tq6NnWIDOj9vMNTyHPXvIK/LWTe4bnvVmWrKrNtY5owgpNZNeBEK+6S+5TOP0aoZL6VrVlTdjmrMt/9flZ7Ld+X4s2VKh/i6aq6h+v60xJfLfa49PwBj8fdpST9ow5Lnmc+GjEOVxLg8xU4W9BRfSS/3KVlEaBgk+Ew6cFYnXFIDpyjmkReXrgQiTMmKAcoLcjZq2PDRmRbLrQ4YIhmoWiWRPNkKsvvEC/tUyUOwHAc9lUMySHmJtwQgxmstlQKw4VzYRcvCfW9mMTR74xEzxXo6Te1OWa/iLWtDko1Hq0qMoBjStDCxKokh2ei4xF9ye6rFaC0tUn9/+p/aMkcLrRI64zFHuxpjrRKtEbIu7aJYUBTD0ZZtO5aVISNziSfkmEnuz5xMidkMq/jLOv5tK43iKq8xlyvnSkytUqSFWhvUOS9GJS9063EGKIY2VxoiCQohaxljXbWrlnWp+yXxuCUWk8f90Exi8G4NNDaE6vhNVeYSf9YgY5oNs9qYtXpZvHdF43Qrr0Ljk2RT4RpsXZW28u8lZSXXNKXv9bs7Vn0X6HdJvfMUD1nKNzrJA69ezKuqvjqxNu1K3qIkU2LZaoOuE2//QhTlTFrFjFqztDUmDaNqg7XMCUV1NdYTU27jK+NwaRuAlWEWK0ODGUgpxjNKXt402HdRWBttNacACLMqb1Lw/a7/c0pVmamUVRtE1q43QZQiBkdl82rTie+rLURZzIiqAGgbZSMjZwVX9d2asJo/xgQxXpXK6HpyYZHBuY7FXMId6FrbxWLYQhAQpl0M0JyV6CTWaRtOoWwAS+AAGbYyKuv6lImrGM91mQoKuSAWklxT5zN2I0uCTZ4N0UjOgKkIo1bGoRxUJXZSkTe7XhVX/avr2UrrrHbi65XvJm1DF7bjQ+UB+en+HSVX9XdrgEphABo8XaQ+Fw7wmlT6wzGq9ao+l42UZ8d2a4HV7eSyriyoc52vvmzTt+zWRT4x3D+1QH/xXCHuupL9Ofvq6oFn6lDVc+DpNZuLtmEk9fP8OxhOtefydfmfGnmvSt/JkHp+NL/zrP2uBhzX88kb+t3qeP04fbcevrrc38H0+xjL39F053f4r3/1D4R2nyn7fdICN2XvCRN/MG17Uxa/7AEiGvdCcqCykH7EpHGZHRkT1QKRFjX0xpuCkYWp1glzsEKkwZp+nNgIiJmosVvnJZ/C4gUzOgtVOe9gM+V9/cJieKct9TIpRiwyCqFQnduywOWFjHioigQBL7TYG8DeBhYC5/qkPYVgo76u9iAwXTgRbOhSJ5fB1yhYmwSWy1TnMRgonWAtXct5ciJWWqXyIvC9eZFzvySGtiyeJXYVWKCnAFIkIhFegE4T0aVrpg1HZfiUhQGPuXgvsLyvmRUXILH3lOg+r7w+ZYxE7JwXWGXRrTURlHCdOUGYabUu46NQLZKX6wGIOLzWmcTfQZ4M6wJS1IhBk2ejWuTw4o0hr6Lft1mIKE0LPbk+EzmJLuysKWhom6QNPEAcT6ttEuZcrkO0AbPCNSp4oYC3ieC73GcmEgEKlHchIgFA+VOBwKq8kKIoLLTxeSEtgQJpGZZ7wkQo9fVC2sK/ibzEEeumeAALhBbcRiYvAU0UzcL1vlr5194ctmNtIshwk0R4XMptCnw4F7iyKXn4fhVoc2bmXGChjQdIKH00VB9Devn34eIKWixELTXFff1Cr8tn8hKNRWic8/Ji3uSFrKXuNwBhD1ZYPE5cVplLK9IYaY+Se8jjAlUYkL1eaPlZ+HzWS1ww183sxjpTPq+JYMZXCwARZa/6XG885KrPvPjV1dhzXu4b5+e+cXu245uq/lf3eOUd47Hg8rnZpS7lNXKblrbV5ZW2CbkKtw9YvH25sAfbcv9EM7KMaRUPzBqTDGnmY1BYEbykisiFSVmYIVklhaQ3ovTsvA3FyMlL26DzUjZ7KWXuQjyh2WZiZv41U7aFSRlYxOtB46gLlFsnJYL14lUr1yJjGeesBEIp41QZvfU6+xo7aHbV+GSsILDLhcu1TNxTQ15ryOrK+OV5xm2QPlX52Egvz2tuu5Al8TO9Yh6W9qqljhWUtPo5Sb/qcQD3YXOwjPO1fQzxNuYqPyeOha7Ib55cn6sxk/FfG7OrNtbf1XKfn+xR1uNbp7z5rj5xrH4uqOrY9pJrXtdrdavN/+fyvSJdNVK3bfhUXX/LUgZWyMO/bekHbVi+a874N//wv8c34QZTctjpSdhhL6nBrRlwSQ12esZ97HFrBrz3B2iVcWsvmJLDmBx2esa3fo8bO+JSeMR3esbRjBiTwy+mN7ixAwDgksgbGZLGjaV4zY9+JxDag53wzXzEO0fQylNsBUILLIblEB2G2OBH7QNSVuIVZS/pzsx47w/CeNrqgClZTMmuDNVGBxgk9Mbjg9/LMdKXfMQptDjFFl80J3zwe/hksLeTlHfve8l/tCMeAzHrskdyb+YV/PYcGzQ6yPUpa/Rmxjm0wjDLaU7kbbz3PXrjhVF2iA57O2OIDjduxIPv4HTEOTTY2xmPvkVnAt64Ae/nPW7dgHNoEbLGrRvQ6oCvpyNpcRa2VPKcRDjFhjaN48d5J/1hb2p93ckTc/DP9qQn+ouB4nI5DwCMcf0zuW1GaGQ8BgoWmaMV+C5rnALA2+aCkAweQyue2Z2dcSr9u4QGVkeB+tI1A7TKuPd0H1gr9MaNmJPBzS9HegAAIABJREFUscQKk0c3YIxWvOe2eBEfZtLHPNgJD6Wcznh81p4xRIf34wE7O4unj+tgaPScrMCbtUoI2YhxvbMz5mhh9aIVOkaHzngZzzfNgLu5l+tT1hjLeO/sjEtoyBtd5vwQHJyOsjlEzMEEdeb46Eff4bYZcAkNxRRnTQzJibzSrQnCNszH3rYXjNHh7EtMbNE57YzH/dQLU3GjI9rihRmCE2hybylwiPU2jUqCNGhNwBRtYbWlz1/0J2iV8cvzjWxwtCbgUq7nDaKjm6BVwn2JCa8ZlGsI9tFNeD/s8eXuER+n3coL/KYdxNses0ZvPe7GXlhud442Sk5zK55b1nYEgC93j/j6coRPGp0NaE0gVmhkHJsR74cDeYtNxMU7HJoZg3fYuxlDcOJBBCCe5Dka7FyJlzYBZ9+gLRt4teeY63uYOvGIMpTaFVi1Vln0OZ2OIiUVs8KuSEZdvMPOeYzBisedIdj8/YvdGWff4OIdjCIUAQDcFw1Z/o0plXFoZtyPpKG7dzM+jj1uuxGPUytMwDfdtIKbD97J9byBuePxZkmnInXF1/hoEJNC54idmfVS2es3RSMbnXx9LN5xngO1zirXtXMeF08uK9anJEZYg9YGnKcGnx/O8NGQZmkl1aTKOPioi0SWFWZq3pBNWQnvwL6Z8Ti1cCaKBu8YLJxOoqsJkIYmsyTPhc23Lx7n89TAFc96KMy/SmWMsysboQQNZymvVI37HCx27SyapSzhJTqXhWdgmB36xovWaEoa+3bG3aXHr5uYC4Dj6/l9bE3CZXKyidvYIKEesWxwdgV9wFqtGbRZm9JyboVwKOPJZdBxyPddN2Eq9zwl0oyMpTzeeGT2ZIBkzWpugpSUoApCKDJiZd7xBjVD1mNURdNyKZ82QTWcK79jTxv3beulvVwOsS6TxBhQDETepC+szyxJhrLxvoJhl3GOiWTFZONSQTzHOS2bmDyGABatyrJxWG9CCYEhG8h8fW0RpbUsGBVaTYra0OZz/Jk3Ya55rK8ZWWqzkVRDh3ngtoYY53uOrI3zVmPz6vSCsUesxesMRNKUV9+flFO34TXG5P/PDM8fevpBG5Yf/Q7/+Kt/CY++lcVUTBpn32AuL2yWXBi8Q+88zjMt8vpqQdKYiEt5EfELvTFRFkUfLj12mxcCC6NnAKexFQhYY6msXUOLn2F2Ip7NUDWt6EXqo8HtbkBMWl66plDRW03EObyAa4u4My9ieBHCItjWRJynRhY9Phrc9iMGb0UE/VLKc6U9KQPD1MiLg8l22AvJLyperAKAD0ZeprzQaYp8RR2DBZCsgLUR0+RgbRTIF7+IWFiZvaaso8neyL71uIyNEAPFqNG2hXDp0hVyHS3eVV28dMAS+zRNrnhxl5ijpgnwnsiH/GyRksJXb26QMvB46qF0XuUJ5UXI8DrWC2XRafa05kyi0vwS71qaM94baaMtWp2s2anYy6toq7VtaTE3Tq7USV5V1wTESERC80TsDsaQ15d1Q9lL6WcS0nZNoDZmBWMjdh3prU6TgzFJxorLAABVPK4iA8Hfy29ukWBIC2FS0THVOsN7g6YJmCcnsN3MCxGdYW1E8Ba6LPIUQF7V8tJQhha5vLiIQcO4KKRH7LnOqUgrFN1SwyQ+hcQIAJouIHiDWDx52hSPsEnwky1towUDj0XdFsMLrnL/2dvKZbE3WNuENBv8ck8G5XhqF8+gzYizXi1mHM+fccGt8SIpBy27/64LmC8O3+wO8KMVmB0AfNV78sYDyEXOgsmiclIwhTAojpY8qXnxZCMrfH04YDw3yFFDu7jSYm3agOniZIzTbGC6iDiZosGqVwsjVTRmk6fzJHtB/dau8jLz86+huRImQ/ew3E9kJQLjqpAqKZegbRLtWCQF3Zb+zEY0YcXrYIqma+nLh8MewRukmWRCTEPwznixUJYXnZTXdAHx7KBcgmkiwtnhw94jDla8L3cdIVF08YwlX0iwWMIjKPIm871UGdoVHVr2IkcFRMqXg8K3XVwtmFLQK2+ucvRbEK1aBSABHwopFffBNDROSufFU61oUatcQh4NHo8dclQ0FxQWci6TpF7VkJdXlzkkxFuJyLfgEkwbEUcLZdJC/OWXuZu9pgWuV3hoCuqleINlfAaz6NJGBZR5lCez9pDPFaKBvahB4dz2lIc96KYqmz39o8GpjZX+rsJDF5DOFWb0O6azS0SIBSwedYD6Pi31nFlOJSuJYxzaBNHzZc97IfUamuIyq2NH2UgAFm+0eBCBsWtpfMr3wZVxY88UX1KuHV1akARJQSWFqcwTFfTi2QUwFxKtmfVno4J3uZDSFONBk/fYsyc20L24NM0qdlMViZrQJChuL49ZBoItMaFx0+bS98SeYUXHmYiJy5cw3gRU4CQpizUedSWXInZeQRasNHRrZ5Iq3nObYWrveW3obca7NhhZZmV775bygatGYP2xzqOWn4MqU0g8quU4KhtXmpMhZDxXU228Vf9rZuXaLq498Ve9pNtjW6Pw2jXPpGse4VU7cP37725SiN8d2PyDST9ow1KrjEPZ/SevT0JQGQc1FQKfiEZH2fkzKgENwFqK2mXZ2T60ZNA5gaKWGDKVcewm2Ulv7Tp+UKuMXTvLTrlRGapdYrZ27bzEbJrFY2m0hrM0taxOUCV2oY5La0uMF3s7lMqwm4ATgc6qjM4tOnYcb0b6bbSLy3FX7DlRWaF1QaCrRDwCyUMGZiokJXFVZx1zR+QkhQylIpdRtrSvaOLJuDWkfYgm0H8Xyu5lyd8EgZs2DZ1jQ4jH1bli8CpqqzFLuzmpkq+OYyLYaJZdWteQbh0TVTDUlnXylKJjAFZji9ImpTLBtRTHIpLxBSwQWs6nlJaYKWZ7BQCtF528XF7Y6/ivWMopC80Kyqx1BjZ6mtpEKYvHUSnyTPI95bKVKvFkSEu8k070klcFvszfeVwrwhKOmePydDVv6vI0aCe6PifkPRUidEXMo4itlvJkuafMCqtVRtIUi1a3h2PLohh+WcoDSvxaRXIiO9/ACuIrcWPFaFBVO/i4fLap1EfG0fJizgLT5Q6kuBDMiEGtyv2oro2RIMPiRajeQwzxljKTXo0VG6HKpGWRImWUzRJFxzLDtDKNJRvS3D821hieXMOuc7WAEkIYgODSTxZM5ZqkkLjPNbS4zpyVjIPAnctisCabEW9DuZ9CJlMWecEbuR6ZmJT5Zj4J2yrGXE6qbEbkhaW4tJ8NMom7rLwdYmvXxoRSiyHOtyKv87HRu4LJ8hgA5G3ZHEOGsC5z/REAYoG1Z5ABoUvbArUhTgX+zJ6GTN6fHJUYOAwdTmzM66pNSQFB01xPCjmYRVew2pQRGHBSwFw2pnK5JquqrKpP0VCbahKeuIyjQCsB8pTMBR6ei6EDII+8AQHkZMhImMr/wvSZolvDa79jyllXGpVKiHIY+stxkjlrgobWC+u8tE/6yX/MdyCWxPLviQeKx7M2xpIiXZ2MRf8SEMOIxljLOTEMNI8pVgYRG4n8n41L+cwPxQRid+YywHOb6y/5E81lMbLq/vn1/S3N3hguy9xng2pVTq6ybhP3i+fS9hwIqgvu6yoLHWMG4/V119PKEFLqKYxZyuAbgbX1lq/24kn5L/1/Wheu9O3T/XnRUPsuRtx3yPu9jcPve/3v0/dKP2jDUoFgim+bAVOy+DDtoFXGu/aCvZ3x8/MbtDZgChZHN+LjtMPb9oKQjciN9I3HaW7xeX/Cx2mHm2aCjwYn3+Du0sPZiJ8cHvBh3KF3XuRGnMp4nFrkrPDF/oTTTHDH89zgs90ZHwaCYN52IwbvXpQb0Sqja/xKbmSeLd4cBhJGBwSC1Ng1WcWlyI3M3uLN4ULHpgZ94/HNxyPazqNzAR8e9tj3U4HrNBLX+PZwwTA7GJ0xzE4MYV88s8PsKvF0iBQIi5Rbk3AeSG7Ez+SZ9LNbyYrsuxmjtyRrojMaGzBMDfp2xnlo0be+QKY8zmND8iTB4HTuSG7k0sI1AY2NOJ07pKiwP46YZgvnYonvJO9ZKAuolGhh2Pdz8aRZMWiGSwtbBNJ3HUmwvP8l8c8fPz/De4tpbCgmMSm0bVgJt19OBMtzbSAIVvFSKpUxXJpFbuTioGwiuZGkoU3ENLRwbUCYLWyzlhvJSWN8aAGvoQ/k8WYv33RuoGxCuljYg6f8Je6Q+57LgrHZzQjBYD5buJ7KCcFg/tUOuUuwe4/gNS02QYaDSHQELV4I8gQpMeSQAV88OiLdYTJJgpQFq+oi/H0LtQsSF6gslZ+9RR4NyXeMRqQ9TBuRCrSJYVIsfq67AH9qofuA+aED2ijyKH62IjcSRwfxWjBhzKlFbhIUy314gxgUxZrtgxhecbS0sAEozo69QkWmQxVSF5EOyeWzS7TjzjIk37S0ePrMLwbHWdP1sghUyAMt7tK+8lTxQtxlEZTXZ4N865G+7ZAOcYmpy0D6tqGYLAXaZZ+pT9L2kWBkeR+RRW4kL16Pr1rkNyQ3oiaL7BVgMxmEF03nFPUt9xHqrkXuIzA6itOrDGVcNHIEcpOhBgc0GZgVcp+gRrfENXIM2ahJRmKXSDCeY+MU6D7YMgZNhhostFfIu0SLQpuhL4XAbJegLxapzUs8WygL6hLr6X7eIPYZsUsk/TGRoRKOCbpIVyRH19qTgr+l4+ZCn929QziUcVaAvZQFdCz3oYFIg6gSJ2dGakO2lMdMRVojKLEtsskwE0lDOJENofFJDUiqpMwhM1b2CIfvKoiMR9YkX8JyI0hUBssxsBSJPya0/8wiWyA2GTpSjKNK5AFKrsiNXEhuhPuqPUjCQpPciB0UzAiEHZ1Tkf5iC/EksdxJakmeBABSS8fMRO3yB4j0CMtvaJbsmCHxgLGDeFp0kRNJbSa5la3cyETnY1vkRnaAvZD8iPY0Fu4ETG/xayczALGne2DmxSBTqZIb0UWawxb7r+xP2AEix8LSHdnQfbUX+j2J3AgAFSCbJixdwh6kbIDmISP0SsbQjDT2dTyginQMKNIrLUTOJatyDzMdZ/kZlSEyM9wfllBheRSWC0muui9NGaORpD0AQIeM5FRpX0bsFiOUDXQ9ZyqXZUEUbYrk4rHVIcs4ZwMkp0jmJpPUSB1DuoqFzDx2JCFSx5ZyvCEbYiytInGqG68iy69sE3sLkYtnVFH+1fjbJS9KfrouV0ZnXoxm3lgu5dS2pkiApLqcMs5FKkWkQKqiobDIjTxj6KpM16qUxVjPunzf9BlY+v2sIbu66IVzm7KveiKvxYduNi9+KCnj9zGWv7PpD9w9/oOf/je4S52wg/psELPGmB26zzzGQpPms8VeTzinFj4b7PQk5xoVcRd32OkJj7GHUQlORXTKw2eDh9SjU+SV9NlAI8Fni057kTlxKiJCo1Mz7uMeR0MxmWMilllTPaUitDDZ7n4yC0NthIZTUfJy+7hvKesn7nNTfk1OBYzZEeuriohQi7RKNtjpGVNy8Nmg1R4GGWO28MmKRMvOTDhFislj8KMr7LFSf2FabbXHVN5WTkWRePHlCUpstNSWKVP8nikMs76SXzHlnjkVcYkNnI7wycDpKMc4npPj3DSysO0Caw8y99mUJ+clNjL+zAbMDL8ApJzbnw7QyHgI3Upug/M4tTDBtkW2hds0JYtWB+kH5+Ny6rY6FaU8boMw24LiP0PSInWTskLIBr2e5VgdD8nnWVaF43eNWuIUmUG4NzOmZHEOLVoTVveZY2OZUZjbRPIhRu6/01HkbACsmIZjVgjJSMwhIQDofi5jEhGSkf88V7gspxa2WpL80Wh1xJQM2tJfn/WqTk6cbxlrilvkWNm6z7VkELeT+8J1M0NzEAbnhTW5ZkWWmMzCvDzGBWa3ylf6xQzAYVNXLecCENvyJTjsrJd4Sk7MElxfU0uD8NzjeFiuR643EXM0SIVBmROz77IuL7ez0UFieWv5pDo+tP7MY8L5t9I4VkWJsa3ZgbdswdzPWpaI87wkZcReMWZwZmQK549VSAHPP6cjaRQX9thYmKf5PtUyQHXiced7sO0Dx0TWIQUAlhi4zfOmjqWs58M1CZ4VG22pZzsXOP1/7L1dqHVNth70jKqac6619n5/+uuv06dPTkJAQ9QgUSMqeOHfjRFyG2/URMTc6JW5SPBeSBCFQFQM5ELRJIoJRMQbCah4oZAchKAhaPTEpE9y+nR//f7svdaac1bV8GKMUTXm3Gu/3/u93X36fE0XbNZec9Ws/zVXPfWM8QxGV2Xel2XJvtv7cEC2Z7Nx29/n2+DD91j//JzY+wPTk/bLvUKi+VfLY+O2V+L148FMiLaeIM80ey8FMNKXMEIfSmRj4sbIiKDo+tF2yEzdApK65Uwj0rV//r0PbeT7vh+rWV0p7H2xul3yisKrG7Nbyfs0+ve3GDRfxibUELZrpv3vx8SXfbO+XRv3hT5TRjvQ2zT0xnty/1s9+zpu3ePz/yjpORD0oTYAnRl/tg28fb3V9y9tEz9zfd+WLyvnK9T9sWX/iOUCAP78V8z/8/SV09caWH4vv8B/9L1/rgm7mDiMCZqMobN79v5ahrZh8Ru+WcUMTMBiiKVtkh7XUTeefVNgoh2VZeNq5SVjNGORDWxJm42D/RBZvMJjWtvGyTYkgXrsQh/SwEQu9qa4gPxYzyqYYHEFjyqk4WNKFiZMbnNkIg7F5QH6D8UeZC27WIJ2n21E/MbDhEjMb7UztgFDrFhVsMEr8iaNkyZmuUVFF1Q8oFITeLDrPsSCicT5jY+FVPCbEH+fmYfeTQuIGI/zKDFG3Q+2Kfa2zUMz/eyAyFqRYkHWkDSj+sX6zVqkbYxPv0myjS/p2FkdtVITekihbmJQMrDZZAFoSrkx1GbiSySiE8aKe2VgH9/PTDK72e/2vbGjwZmBGmMcQm1+tVnZaWvPrfubMq+bQx/z0Mr2IhIm6OD9gAG0/vi+GJst9aP54AbHjHaVZD92XfTCRCvMZLiZ11rdrP6ALPUBwDIPrf0UuKkHi+mrmGEzi/9o86OzcXZzGVR0YxgKShYBHBupmJx/nbLeTUkYaOx8KWEDTOx+73/rfWnNVNvGJxBvxt6+FzZ21kfSsdvnM5P85iOo+a3cJrrhNrQtVEMFKBiQ6Js/UjXNWrZ+tt602MYmqdhIUz0238i8BSYyphVlDY0Rr6v4iDZTVnKm2bZezOSvIwcxP0bf5IoycPc/tP6S+bmlutkcmc9pN4/mmxvqZpatZVKsTWGZnQiLmTFzDuIDy9Sd0iwRN4sHUV8OrR9NDZlJFI4jtz49UVQOvRwbj+YXaYyDPTts/dMun/qstnEtfe6bP6P6ZDaTY6au5gt0P8IcJJ9TrG2Kv5+avPqu3wFbW3VNNyEYaztc33cga6Pi+1xibNYaVQKn2uqg2hV9nxNQMbXdBk4YjR3bmO0Ct8Vg/P9tzbu221z4NUty3dq3MYW123TMbEj0UbltT+s3miXNE9NOA347ZLIx/7Ws5OrQNn6Q+Xpa7AdQ9I127e/7kmR1MT1978dp044vq+NWO2/Ue7MOX7lv2I0277PebM+HxvpWu79K/q9B+rmP5W/SZGzL59MDlprwbj0ghYJfGIV9+mI5NZXL+zDjnEd8Nj2icsDDOjVlzGse8K3jAx7WCS/HK3INoiaZR0Sq7bNTWDYntY+rmER+8/CIaxkwkZjdfnZ4xHtV43w9XZ6oihYOuBsWJKp4M4s63RQzMgvYXJmwlIhX07VtcpcqIkQnjY1oaS4iQDSvA16OszA0quj39iqqi2MseJgn3I0LBhLlQgOAL6YZc04YhxVLiTiq4mILw1LiRk3yNKwtnIv4ilZc1tSUBJOGaGESAaRcA15My+ae45BVXGnFZR1wGEQk6X4S5cl7zX9dBhyaol/FNBRcF1FhvJsWbWN/0rACSa9YeRhyM+0VX9HaFAKZRYAphYofvBdF3fvj3JQGowLlSQPZ2+m5qfAZe+FD0ZiiIQN4f54Q1JTVQPa8JgwGmNXHMgbJz0x4OE8A93h5RKI2uKgp7/UyYpwymMWk2oCShTUBRPmvlIDrPGDUcnKO+OHDPdJQME4ZOccGuIh62JqcQ/cdrT0Qur1flyDAY+lAZxgzllkYwGEoOD8ekIbcfPW60FDAsgSkJMJNZl5tAFSAEAHcy45O/Gm+js1/1ESVgrZ1VSGlEGrbUJ8fJoRUdZxFZGit4k+WRjH1JAJqCa2vMXYF5LxI+2MSU+W8drC5ZmpgNqvI0/nNCDAw3K390EHHq7E4NeByGdBEaLTftSjAM59KAtbLgHTIuL6bEA/S3qYc+n6C+UgiMPg8IB5KA2jrPAAsZsZZffjICVjldyfQKTfRHS6pbYjXJYCOGdD5CEMVs+6holzl1RIRN789ShXrdQACUEpSk2lqFI8BlXqNMs9j7cHr21z0DTlFRr2Kvx8GBV+RgUddV2NFOSf5TEFY29jrJiS/GcAj941uJtl0H4q0DWib4vJ2AB+LAKaVwIeK+lZNfy2fCo+YH6OE3sB2472qMJSJs2iYD9tQN59EDU1BJtRlG7HEzSxZJtOB0ij1EAG02I5TAcNC0lcmESbS8BEUJTwGTxXhB4OYzqqZbvN/KxAzwsQIs5jL0iobfQtPAQLKQUyFw0qoo4AEMxvkhGY+yKkLnpjJMQ8yr2aiW6ceeoKD1EOspsnNPxQ9rAih50+MsETZ2yruDwWgVesfIObWAxDWKP3RfsQrIZ8+fYca1m7y6c0woWahpECLVnTfX9P6mSWUipj+yoSbaWk00+zI3RTWCclYCI+2rw9AvETUUcctMuJCarbcTaktzIe0XT5v4jeEbtY97O5TIR8y807rE+FJuBI14mkmn0HzWf1VTZbDqmOk35dmLmrrxosV0Rb8NZPQZiJLTZDGi+fIPVtAb+atvt1PQJb11Zt3ujzts13aALB9mdzb7NuzOaneLUV/3uBNalvy9+mrCBP1/u0xXyPPQx/Hm2nfppttJDWZ9eU/U+D+YOAjAOItQHvjrOBp+V+jJPuBn5vC/qZMK0f8+vUe98OMaxnwbjlIyIO44pRWfDGfkELFJYtM/pvrEa8PF+Qa8G4+YEq5SdlXEN5cjzgNC+aScM2pgYQxFry5HlsoBAMJ765Ta8vDKkzX4zKgnghvLgIY76cZl3XYmLsxk9QdC744H0EQoZ5SCVll3XOODdx5CXYf7NyuF5cfAK5rwpQK3r4/YpwypmHFw/mAJUekWDCvA7KpAt4RZgMtCuRMtdbk071JyjIqC2oqtrFinhMWBSueqUhJGKN1FOBrbJAxkesQm+qrxd28LkMDdtfLKKaFlxFpKEipyDX1v1uWtFGGNRbIAqCDGGWSfvZYmoxlHlqMycNhRSQW30agsW6mdFpLQJ7WTR3LVaX0laGyeJtEAkZMMTJfEigy4lgam5PXiDyqj6VKtIdYG6uVLwnIhHKnLFeQMTbFz3pNnbVz/pCsgAlMKAdCLRFljq3csgbgYcA6yvqqS+yMSmBkY1lKQI4VFICaLUZiZwN5CciJm7IlAsuYLwGohHoI4HNCPVLz4URgFGVBeAmoUwDPEUXVLk1JlAKrkAman2GZAvgawUdCPSeUMbR4lHWJ3Td0ESGTkroYD58jSmLUScGA+oVSDliPoYEtXkOLk1hV9VD8/dRUdSpyQr4GVIsfmWXTbH6BdWCE97IuVh93cA1yvzErlUCzjpU/uTcxFecHSdeADCC8TwLEXGy48Bhlsxig4CQIC28b8Fn8W0uFqmrq5lLbld5G0U1R/0wylcoKxHNQuS/ZyNepgi4BPAYFL/0HkUnVJCvAQ5B6k4CcetS+Bmunruk5IKxAOYlPpLWNScAMJ1VmHBjhKiCvHgkW6y+epT+1EMKVUEdq4O6Jj+W7gDJB+lAlPwHITnDEQFA8k4z3QggzITOQHlSoRsFbOpvDm4Eg9Ycs5ucmvpNA95ULq3z/Nn1VIFFT90G0PtREDYSB1GczdgBmm+B4RbvGQfxHy4HV70180ORzEn/DEjC8J3AwUIjmUxd0PfNAiFdZz/Gq61D9zgT8BcQrISxAOQhINHBVFcxxAHiQ+aqjADkBkgKUzK9SfDSpgRPKWs5EoBXNx7JOsuY5CHACiz9gOt/wsVzRPo8LkA/qVzh0/7d0wdOd91dIcRa/T0ABlIIHqlJf87FcOtBqPrpXmQ/zKUWgDizVP7SmHvvSgxlOtAGWNYq/aDlAx5AQZ6mTYxeMoSL3MkkddejAkkPvQx3dfdx9Clve53wsrS+QskFbv2DKACf5LM7i7wr0+5k6EK0JTYuuRjQQEhT8UhXfTQPHkhHqVyhj64GLAR6O1JnVKtcbAGvgTPwTaxSfwg2go/49kHL7EtoAIQdw7ZBFgL2W44HmVwGWltfKcz6WnwIs7dqTtGvTTday5eOngHqX2vjvBJOeBaI32iA33K7jOVb35+mnm77WwPLx4YC/9r/+/SgTi+O/O/2sg/5g26mnOrb/PXVsj1f5cQXJD9LfO3bH96BiBGEmXCPw1++/Kc7p+oMOfVCYuMH/cy+bCRNu+FsnqZsYeDe5U11b/AV4N4rgQnqUB+GiJ6rQh22swMP0Qh3YGWElXHUT2RL3H/ZQgPcHqYNWOTSfzvKgPg/AMAPrIPWEGRiKPLjfHO9BmZqz/nnUh1QGrvpj7R8Jl1HymYVsCcCoP0BRNypRH2xZH2LnsQsEAMCcpIw5AcMCZBV6uAyS76L5DwuQjydMs5SfI3C4SJnr/RHDsq0vMJDMylfrKgcZ0+Qetgc9NU0VWCfRC3n1hbRvfjViyMCU+6bA1gR0Cu/mft3W26jzO6koArj/uNYRbRN3XGTTc1AhCUB/vLX8eJG85Ti0HwxOwOHaNx/5NMiPsm34Ut/cgYFymGSOVqBO8iseViA9MuoYkU+DzKs7JWZjQtyJcMh9rVsKbsNnfaoDnADEgPTIyKeoQeMLAAAgAElEQVTUrpnwB7GsvXIY2gbHTsrt+xFWNPYBkPLitb/WUfLVuN24hdx/aK296cwoI6EOQ2s7VSCsjHyMaCfvZXvibj+mcWFtQ+obLW1zsPVB/f/hvTANy8up/TiGBah6v22kwiKblnJwIQ+0/7axkg0wY72PGB4Y+Tj0IOKQueRom1JCXCWPrfs46+HGMbZ+1NQ3m+N7xnIvzFScDWTIxjVdgeU+AUHGqkwR6cIoEyHOMqb+9JlyBzNxZhHKWGXc4sJg2zwrOIqL1JcPCSFzmwcmQsjcAYWWF7IcMBgAifpdqCPpmiAn8oFWF1XpZ54IdQjyXNXneD6FttYsr4y3XI+z/D88MvJBASER0tW+J6xrl2RcG7gipKsIYHRgKWNHqt7JQTf5K1Cj5PcP2WJlVnnixIVb/Z55iQv3Z0QgpLkiT8Im1CRjKc8XKSMfCeP70oDNxuywcAMkca4CAhYGmNuGlQMhH+TzuMi4mKhKA7OFdR0rmExAXKo8G0bZsNt6XI+ybsGylq2sMqqAi14rI0meAMnDAhTTtbrvgIxzXPtaD5mRDwHpWlEGav1Il4rlxQ0Vlo9McZb59M8IUtCTp6BrmBCXiuoPBIiQLlWemYkQVm5AqEbIurHvia6xBv65f38aCxeAdK6oE7XPw8IqykNtjQr4I30eiZBOWHu+sDKIGWUM/RnD6N/FrOs2yhg3YRhto/UFkDVgY2SA2OrnBISZ5SAI9jzk/r0ndPYf2IDEBvRYnycq8rMtR8b4FsjYAzHpI7d6qPKT1z1jZvN6K/nvkfTDt4/bOvigOWsrDO158MRcV8trbTXARs9ch42FXNsIG7nrT16deM9t8LtdB/t0Czju6/nS5A4CNsrUuPH+a5i8a93PWvpaA8vXLx7x+/75v4LKASsH5BqbcISIqnRxFABYamy+kF5wwUQnLIi798sEIMIVWpaJjgSqTaTjlJb2v/lvLi1/91v0rKXVvZa48Qk0MQlrU/M1a+IqdSNiYNct0LvdC2ATbN0LT3hxCBO38Nd66AzeiE0UFmGP4vrj+2cpEG/8DlvfIM8hu8eCoFuoFtI6vHqu+ZZaf1nbeVJT1tviD7RpixcW8PNgjDAAzKvEfrx3/pwWezQZm2fmjbXXK+a1PUaptaPW0EPLuPE0v9Wspra+XzbGRdlimW9u8xuIkd34WMxQ+y22R7VtmbzwBwO45Nh8Ib2YBoAWcqayhiNh6hEO3DivzvfL7g+BReWexTz0UsTHblVfO+ljv4d03n3IFBPoyG78rF1WzuJ8Q5mp5YVrp40nAFxbWA+4PGg+nl4wYi+WAaD5qtr1rgzcTYjN/48IOGtcSTMrBtDiBFo9ck3yWezSnU1R201wlfioj2oJ4HclG/9FvebDBpkqcjv4dmuMAdQcENKunZrZfBet3NbH0MOY7B1xzB+SnZ9Sew+51v1IdZ5ceJLWUOsiO389Rg+LQtz8CClwC3zeO+LKYzTfxo04iPkR7vyKWxB0hsRzHKqw2al2YRDXv3aatU/7S+Y/6K8TGiO58f9jNNPglnwYD/+orXA7Wlee6/+mrshqCXBj3bmNa/u/AuaHJ58xOFa5XsS/r/nlsbbZ8pKBXu6hOaxtdvjnVI7bvYCMadHX6vKR9VnHoez6W6mZccoGWk2Nc+iAhQAqEfVH2PlQJQea/BiS1mPAKWyYJwAgtSCx9tlnTP2zPveETdgQ/32xHDkJ+GtjHVp5rShGB4x+k07oYwr3HnDzsp0a2q8TxnaTb10o2zYYM9jAnd7fuurbcwuotB8jfTb6qd/5C2/A2v754q/dBEQ7MLcpeNf3G+mDLOC+PR+BrZ6rYw9MbwLU59rg23Gj7Gfvf668D9T7lRnEr5D/k9nJ//kT7/t5+uj0tQaW5zLir7/9Bby5HFEq4X5aUJjwcJ2wrAn3xxlLjjiOKx7nEfeHGe8vYsPy4njFdRmwlohpWPF4mTCNufmyHcYVxyEj14Av3t7hcFyaX5ptHM1s9O3DocdFTAWXy4jjUfS3L9cB41i6KSGjiYfkHPHqxRlV/QlTKk28ZRoy3j4eJSA8A5OampoZKACAuJU9poyH8wHMwDgWLEvEN1894s3lgOt1wIu7K87Xe5QcME65AarHx4MIgVRCGkoT9DDwMWpMyaI/emZ+ar5mOQdMU8Y8pxZ2w8JymJDLch0QU22mpaUEpEF8v8ZDlrARxChrQBoL8ixhJI6nBZfHEdNxxbok1EKYjiumIeOLN3dNKMSerESMoPEuzSR30cDyYegbyXHKWNcoJnFzBArhs2+/AwB8/9dfIgwVwyh5AnHztbON5HgQux9pN1AzNYGNYcrIClIPxwXLmrAusqZKlv6tc0JSc9i2UVbzzumwYkgFj+pryQzUJWI4Lagl4nBc8P79QUxkU0HJEqePIjcRlXyV8RsPK5azMpZDxauXj3i8TFjOB4ShIthBQw0Sw4+BkBhV49pRZIQo723TmkYRkqHITVRmvSakqSDEiutjwnRacXmYEIYKIKLm0MpLY0a+DohjaUIpdY49YLf67xmwKdcJ6ZCRLxOG04rloia8LcC8tnsqbS5Y1+p0P8u6uWrMxsgIY2nhW8DUgseTmv+2APOVEA9CY5ZLAggIY8E6a1mp101DBS8Bx9fiE33+walthi1Ie9tYR8ZwXJEi4/p+ar+OTcTGArkzMNwtWN5MGF/PWB7GDQCZXl2Rs4wF54A0ZawPI8w/MZ4ywIT8kEAH2eXx0su+++yCxx8eNayHBLyvOh/HuxnnN0eZj6mAzwPC/YryMCAcs4yH35yN4tdaLxHhmJvoTTknud/WjwkfHTMosnxu30sdSwxV/o8MXgl0KFLW4wAEBmdCOMm81HNCOGXUa+q+lRaUXgHl3ednXM5jWwPxUOSw4t2oZgaQcSFgeDFjfSMhbez/4fUV6/tJfSoJ4W6VtWD+tXPqc0YM5IBwzIB+bxEY8ZSl/QbWzST7UMBrQHqxynDa4dIcxYxCH/XhLjczdzbfw0KIL9YGvGsOGI4r1vMoIj7XKKFrtE3xxYrybsDwjSu4BuQ5IqQqIkaBm0gRz1HGdI6IR4khPIziL11rQHlICC8yxkPG9WEEDRVpKAiBsVxTE1YqVzVTv0bQC3le1msCIiPp9yq/H0AauoYVyIdUUR4G0IsCXoJ8f86OXj3oGM4B9HJtMTdRCBgLwkF+D/M5IRwL6vsB9NmCeo2y1krAdD9j/uJ4a0vxcemYwY/qRzvWDqaitlXXMJ1ycxGwmJVkczlHmf9K8h2sJH7N9gzQwwITUgLQQyIZkKuE+GqRtaXzTMcsbgEGugnyvbDQUgcJ9YSxtnrNRxmzmGs3qygLfaRm5ZQD+FA0tqh+3yL3cEuVgLX7MDeBoyR5KMt1uvT4qGxgeXB12LPTYkoCLbSSWEiQmORbfOLYy6FC6pfKGyBl/TLfXbZx0XuaqataU7GZ4VqqkHBKLpblhn1UYNwsfwxAkwL+TFuAbbFH94DPl4d+OOLBrmdfrV4q1Ezy2Xx0GR3d6+9Csz7Y1Wn9sbo5bN8/B4zbocOtz7G99hyu9bc+MyTSb3LXb2b6+iQ54/kad+BL0tcaWL5MV/wz3/q/8MP1tLl+qeKbZ2EqLGSCyckbq2kpEOMxjzjGtYWGmEJubM/7V1NTnLX8PhzB9fXQFGSN+RxN4t8tnhZygWoLH+BDCXgF1gDGci8PYGNIpQxlcqgic8Qhru3e88tRyy/ts3wXWpiA+jK00AFW1vVlagyiie14ZjWF2vrmP7N7PGu5ZwEbswlq4T+MkbUwFJFEidfPhammDqFgfpmehDOIoeL16XJT1t+HOyDiFjbBMzaD9tPaWJhwGlYEiLCQiRIZg+fHA+hhRHyYB0tDLE21dUoiSrQPp2As7bpT+BWlYdkYvTjMbYVa3FNTlX1xvG7WYVNcNVZRmd5IjHwvYW9iEF/hw5BR9JqNi42DZ0J9mfv33vcXgLCv+n45RVHDPcZNmAETP4qhYr3rqsCBsFH2NTGmVva9KgXrq+Wttb9nPQjxZTEThlRQjoSs3yNrYyBgGvv3G+iMLd93ZrmNzUnm1yv+mpIyo4doaSJP33rsqrAA6gva9MkOj4Yht/rtB9Uz20MsWEaJXTuOedOmFCqqCjNZG9apl2dMcTmEjTqvMXdjysA3LqI4rAqzrPHlYqg4vb5om4FyXMVf+rDKIZPGRu11afl31A6hQmDUw7pR7m3qqybi0wSqOkPdxkNZUhGSAspgvuUd1JUpS/lTH5s9mw4A02FtdTU15Fe8GRdpFwOvr13M6vVVDgxfzJ2Fsvt1bCis4MnRBewYagWSFBh0tz5RoSUSs8DgGW4G6MCA+vAxlNm2sy01I2zX1cY/ROm7gTbZiMtBE0+iPxgU4BExogpHRT1cIsjhE+sBVTC/YBJxKvNZD6cMChDBqqErD9cCkJvCMCoteShtA2jxZJsC9aG0scJQ9ZCNWj7Sazj0uW/M86HKxjaygIikPtpZd+6q6otRVIMRICCOGMt1EDD1iYkrAVPtbF37wIE0O3gyAOMPjqCfm6KwgiA7qGpl8e7afgce2B3ySN/M13xD57jnKy8OpNp3xESsCOLba8kOsrzolLWf0QEmINfbV0DzGSCxfgLN39vKaAqx6jsrYIV6Hdb13PtgCKMxlRVo9lEGYvbKs+zK0ClqvuyODaUs80SFsUdfpO5NT8r1eYprQ7+6zUvY9G2fCLsyN6z47kNXTjTrgbzru5Vq7arbIlqO59p0o8++D5u2fqBf/f6vBqg+mZn81Pt+QxL93BT2N2vKHPD99R5LTQ2cZDdZj3na5H+v7/fx4uz/Hy5HLSM24OljsgHYXLdyDGhktzk/5/Fmm32MMct3K6aUB0cGxsKNb8oldxNcK7+y1P1uPjSQ581DHzFuNjez7gF87Lr9Pfu4Zfv4aHH3uV0Hbh9mnbGta1/v3vx1H7PM2mR59p/Zex/qpcVWW8bNfQDwxZqaWauZwhoI2cdT2/Rx1zagh5p4nHs9HjDs++rX4Rm3D+Oua9qc5u1P/D7mGWphIJ6LYXYrPe2f/wwbc1BA3l/msbHWzcxwV97FlXXLnNV/dpm34VQszWsvz4MmG+fzddyERbE2b8uXVxOW2vfdt+/Kw7NjAQDX69DMV31dz8aE2/epHUtr/1iYiPk6gELd/CBfdN3ciiVH+l0kd22fLlXnCGKF4Ntg4Sl8m9Zl55Tn+7EZsz7n+7m3cVi86d2TArBZ0Cu2axDYzouAhef7uZyH3pcP9A9MYuYdawutQYGxOvPSZg7r2xhujMX+y+lP/PfXgU0s1o1pp5ZV9utF29wOZCyvmTnuHxLUP6tF2S3/+b5+Nyab/vhsyox4MNDYJ8tr7Wjmqfq5ey8YSIWrdGw5qjKprh8OEhqDXFuIAItWT26sWugNbT8FBjnFY5Ca5iY/SV8x+VAXthatuNDHrTE9/oHu1/5uXJvPux9nt3ZuLgNj6n3bbs2vreFK6CFHnpbn5731ab+O9sDGPxI03xOWy742vLvuyiHGVhXWpQ7+0JEX9/v2/ezluk7u1vVNwNIGn56044PA61af/PvnPt+lL/lJeP7+j90EfGS9X1bf7lH00fd99OcfqPMrpx9hXH6ePi39xIAlEf02AP85gG9DpvZPM/OfJKLPAPxXAH4HgF8B8AeY+YdERAD+JIB/CcAZwB9i5l/+UB2FQwvrMdfUmD9jCy3MhwV7X2rasG9AB0yedZN4j51pOqZV4lE61g1AK2eKWYPIi8/kFHOLQenZMUu2UUou9qQPkM4QkDG4eJHeJxToIMl8GY3ls2uRGJc1YYhVY0lKiA1ro/VhTD3W5y0/s6JxHi0ZC2WbGos/GdVPzgMw81EcorA7BtZ8zMpSaQPe7HplYcKMJTK2KZcIhjA2nmUzf9D2G6NtTjtgbfNLeppvPpjXRYDuNAgz6GNDevaJgWYW3MNk9FiUxtYwSz5jP8w/r5Sw8YWT+9FYFjN9jrEzIyFwU50tOTQ1WQ/smv8co/kBdl9CAV45a4gYDZ/hgc/eV9WPofWRmVCLACeuLii5mo6KX1yPY9hiMrpfIfPhrH7uzJeOxK8QcPsHV14tcetH6MBTZeobR21zLdTiIFqZzWcvbud06/enbTETKWMbKpqp34Y9Uf/CuurcjX1sJcakjadeM7O0DVPl69drJYCSmCgGFd+wz2oOnYhQgNXiGgLSdkDjPJpNpSt7ieCxtI0/+w2qmiXKGIjyMKv5s/cZ7W1H90/c+ANyZzRs0woog0FPY90ZuNiAEm2f97MzBsXiCQYVzLi1gViDABp7BPt72dqpr5mamimKmtpZPETontPHVAS2jJUl72/GCsLUnHBzj4K9ZvKn+Z+Y8bkxNZDFBpBaJm2b9kvM4ciNGTRER9iYAfYvgO4Qgyj1Nt9IYOMXZ4q9TezJXH8ZKhwiZXr1WtKDSw7UxqOV1cCUqZGqKZ8DZSZm0wALQ0VltP+2Hiv1uqK2MWm+YGXI9Zo8OvpqycoFd/VWY6NqJGeeuMUpdi/sUeX8ChkOPDkQ4vH9kxiSBBEHs76Qlb8DjTo3rf5AW4Bo9QSdQ1vb1g870yJbi/15TgzUyAie1YRbs1aOmr368Clt7LCt4xYY8+NspqwebPYm3QaNIhjTXz8GbGyAtR+HG/kawLK5sfuwu/Zlad+u3XPtOUC3WU+36vNj+kzfnwWIt+751PI+lL4k/+a78Cnl/yZKDDwhYn6W0k+SscwA/ggz/zIRvQDwV4nofwDwhwD8ZWb+40T0xwD8MQB/FMDvA/A79e+fBPCf6OuzaakRf/Pd5y1EyKMyUcdhxRQzvricxJwsJxyHFe+vE06T+D4+XCekWBADY80RLw4z3l8nTEPGWgKWnFqsvc/uz3h3nZBCbaDIwAgz4dXdBedlkLAVy4D744zHq7CCp8Pc4h5aqlXM9EYNCWK+kqWItD3X0PwJDRisa0RU/0GgA4B1Fb/PkiOmw+ryMubzIP6CQ8F8GTBM4i+5LEnDOwDTYWmhQywYuwGkltexQWnIzU9S4glW5CUiDlU28rF/+0MsqCU2Hx0zp4qxtnAecq/kS4P4W46T5M/XhOG0IF8HhFFBsfpTpVNGyQEh8kY4hBWY2QY1HsRPqBpIIPWjUzO1NIo53fyDI8DA8noRn8A1aOBy9c/kDgDrRb42NMrOgcysiQCeY5egtVALGq+PEne/IXtlD1zkHloJ66nL0FJSn6kA0BKwHov8WKsvGjQotvny0EH92tYAmElbJsT3EfnAcv9KjSFAwDaIeYAKaFA/ndcfEFoDsgYcp0KoAeJntKrQxFhBl4hyLN0kx8ovBFoJZaygNaBYnD/ziwto99jGhIcqoTQOEvKijLqLsLzaNlLTq+aLAyCcA+rIYDN70zbTSihHtQfSftom1YcbITUbq2r2Rhq7Dyyb1arhPkjDjaR3EahAfl36hmkJKBYHsaKFvAAT6rH2H1MF1jX1OHLhElBeFKR3EeVUO5MDID0KYOIg4xtWQj3V1nYLeVGOFWFWQQ8HbsZ3AeuLKKFBZkJ0vlXxQlhfyDjHlVAnCbNRDhVxDqqmbR2Uui3cRJo1dqHeFxZq7TS13jjL5r4czOfJKUhmCzcCVZHUskcDTBq+goAyihp3Hbj5BFGRTbWBk+FdQDlwUx4WNWRCvqs9JqKKn6QzYb2vCJlkDO4Zw4PGPFQwEK9aj/k2qrq3qXlbmwG0OJZhlTAgFoaixy0EagLShTab6jqIYqdtZOOV2ma6AeKAFg5ErkvbygFtvIL65NUo45RPjPGd1FXHXr6MG1ThmTXcCFpYkLCiKTuXg9RjoTxMQZ2KqF0HVRAtOt51EJVhQMJMUFFlXgD5RC1MhVehriO1MBuhaHgKA6lZ2lRGIJ2hoUao3W8hO3q4EUK6aH/Vdy6dgeUlPjlJuBH9jq4d4FAB8hFNOTvOfc1zlLFOF6hqvYwvbD1EUQQHoSn2wualgXpsQJSEG2FpC/c6a8IGhNq6ZHLhRlYbO+zCjeBLwo10lWdTY62DqNGC0FVhr9z6HrIoHXOipqhr42UKpnGVw6Ga6GbokFC490VVdamwfhexibW5UT5VYGTK6zfjWKrPqqlR95AkvQwOpuLsFsIeBHMHdx74hsIN/D+J13kLOHKfc3+Y0pntrYKt5TfVWlNm9sqwt1Rhbyqz7kD0BjTfyLcB1DfShg2/cf1m8uOyf/2S9HUGmz9LifhjZH9/HBUR/SUAf0r//llm/rtE9B0A/yMz/y4i+k/1/z+n+f+G5XuuzBe/6xf4H/uP/xVcc2qqp6UGrCUgF/H1MnVPY8LmLKIt5iPV/N7KLgajMmSlBqwqQiNsEjVGKkYxkDXBH8DYpdiCzBsgtLRRDa0SUB5wfif6zYixYl1TYznMV2lv7mXA0wSB7Fop1AV/CjVhne6jJPfnNTYzOwrc4h42s7pQ4Z5PErzdmZeKf4/GYdyZmzTfQWMIyQSMHGsVugAPF2piMRQEmBYdSx/AngJru7tKZWOQHGMl7bWnMfpmX5kXAE0UIx1WEKHFi/Tsi+W1zX9QM6pWtnvwNVaHuIHsWqiZ7Vm5TdHSH8NZ2aT+QTbExkYxRFzHiTt4/xm7Jr493MQ5ZD7F76kJ6Rio0vY/Ucg09s+/B5qpYFMGBRqb1Riu5l/ET8wHpd87dmuv7gjXNgPkWV/NX4dpy4jF3pdmDteEYbQeBaQUuPs6MTpw9m3xjJpXtbT6rG5GA80YVT30Gnt/ImvcHdc/M8NbqffX1qc3s0sshwMK3De/zqOCSGOlnEDHZjyKK8/fPxVgjv1eEwSx9pkvVuAu4NHKou2PvB8bi5UZdvltrK3MNtbYfuY3EvtDDgigYz24aeaYFd2csm2+9P1YZVzskRUlDxmTqXkByCHG0hlOyiTX3FrxBxcA2mGBbLK4iW0A6IcjJhri+6qHINIf3m4uK4kqqg2bsY9wc2RtYTRmkqNTYLUNMlxda5BDIO5tbmySth1mJqoMp8wX2hhbnFHfJ2NYTRilf2e4s7U6Hkxoa5NWN1a1C66Y+Eoz27Sx494nYR37prsxbm7sDWxWC+VljHCSA49PTS0EBzuhG6CNgR1qeJBm34E2Z8oCdyaONp+1ufeb+sb+y3eKWA4CSJV+29qrtPlZkfG1tncz2D2IobLbv9szwZh3y2tt9s1xeQzkeoaeqrzfh7vYz6tf3xvA4sfDDnH8s9PKYTdefvw8Smp193GUzK6c/fLwnzGepGeKbu2j7ZbpNru4//+rpFttv9XOW2b5X1bWV23Lvs6Puf9HrOs5JvdW+j//xL/zV5n5H/+0mn486Tu/+xv8B//cv/BjL/dP/J6/8FPvG/Ab5GNJRL8DwD8K4H8D8G0HFv8exFQWAH4rgL/tbvs7eu1ZYDmFjG+f3uMXDu9wKQN+7fISKRR8Y7zg9XDG33j/bZzSgnfLAZ8fHvDdx9f4zukdMgf82vkF7oYFp7TgB9c7/Pb7H+JXH1/hG4czrnnAF9cT3l8nHMcVv+fb38V3H1/jmFYkqrjkAUMs+P75DpUJv+vz7+GH8wkBjLfzAb/0+Rt89+EVAOBbnz3i/TIJcNUVv9aAl+OMu2HG3/zic8RQcT8tWErEkiNyDZjnAZ+/esCo5qvvrgdMKeM4rBsK/d31gFxEVfY733iHyoT31wkvDjP+7g9e4XBc8NmLK77/9h4v766YUsbby0EUTwPjt/+WL/C4jIih4t3lgFfHK1KouOaEMRa8uRwaUGYmvHr1gPM8YlH11yEWPJwn3J1mzKsoxhqQPAwZl2XAZ3dnvL9OTVTmOK54f5nw8nTFm4cTXh0vuK4Jr45XvDkf8dndGY/LgLfvT/jWZ+/xxbsTTndXHIaMN+9OWK8J3/jsAefrhHHIYoqr4HNdRSnT2vDy5aWxyymJiMX5PGE6Ligl4PX9BWMs+Nt/63OAgc9+8S2uy4D5OmI8ZJRCuD/NyCq4U5jw+F7Mrw+nBbWKcu6qSrDzVZhhZmB9N4HGiuG4ouSIYVqxXAZMpxXzZcB0tzTxlFLkoGF9Nwm7+HoBFIAPY8blvSi5lvcDhlezKDQuAXGsjRk29dbjyyvWJSFfEsZ7oQfWOYF/7QA+FQyv5q5mCYAiI6pCpil6CqscwEXCUlAQgFnPojLKS5DAooER7zLKWZjkcL+C344ILxfURQ4yKKnqZCbwJSHcrVLOJMBvuJe5oMCiiMnURB7iXUZ5SIj3GeXdADoWqTNV5GsSBU4oi1wBDAyaVIX5hyNwqIh3Mh91jlLuHECvJMgmRRb1TgVuje2NFXROsrE9ZWWTVcmRIaBM1VB5DqCpIvz6KGzKL8zNFBXXCLrLHbzngPA2CcPzqvQfVNs8T6UpmsY3CfXzBeHXR5RXBTSUVm78wQAeZIPPiREeIuqr3A4/6DHJxvNlBp2jbNoNZAEY/86I5VsFmCroHBFmYWOpENJ7ks+IQdcAvisI7xLqXUF4jMK0OuaMrgFUAuqhIr6PwlReA8p9RXyIwgKHDlTCY0RYCPllQbiG1gcQEK4BdawCOkZGOAfERRhGKCOe3ohKb74vSG+3bC4tAm5YN9yH747ILxj5KCxoOgsDunyjIp41HNUoQGD63oD584L0EJEeCPPnBYfvDlheV5h55/iDoBtbAipQTsoy577RT49ysFBG2eTECyHfs4ubKWMR3wXUARjeGzCV/OXAjd0DSaxjUanszCtHIL23Qw6rl5DvZBzKgRs7W0b5bHnNOP5tiV1aJjSWUmK7CgtYJ0Z6CCgHYHgEwGpuOUg9y0tGegwYHoH1XoBbWAVoFWPrQmczyyQMIVg+pyysHRhYXgmDZsxmWKS89W7LrOVT73Oce1njW8lTY/8sqS5ZPsr/ywtgfC/vrczxHePy+acDy+FRygWEfQ8q2BKyXBcWXfLVQedcGcPxnbBtZVL2UtteEzA8AExyr1FhZikAACAASURBVIGqsPbYiVWZWwOGnIDpC2C9F2uROgLpURjfaqAW6LFASeaiHHq8WWMhqQL5SA2Ek9ZdE2ksWmEj01mYyBqFWRSmmlrc3DJpPY87xnKUdqUzI59I+8FtjUfVx6pDX/fGqMo4dEBZk9a5crMUMKa8xi1gb+zuwMpYc+ufgFNuYLemfhAR8hbYWtzYjQl1A8CdOfRsa9A4lsLY6i07MO9BZ7MeUADezY+3688rwlpfa+yMavAstz0i9JqfX8mwK1vbRuVpXM8n+bSfpHNpaQPu3MHAzeu7sfRl33p9Usf+EOBHBMG/UYlBP9OmsD9xxpKI7gH8TwD+PWb+i0T0hplfu89/yMzfIKL/DsAfZ+b/Ra//ZQB/lJn/yq68PwzgDwPA4dsvfu8/9Wf/zeY3uCoDOSijthSLY0nqZxgxKHs4O5aRmTCm0q5ZrMTiANKSOzC0lNX/cEwSloS0LlPOBNDVJt0i8sqGOXeW0RhJmxJjOk15z5jJjVmmqZuq7x2A5lu3LqmF38irmJ5KWdSeOGkojS318f2sDVVZ2ubvp/6CXDtzWUtEiKX5x/W5EobS++QRsPWbU1+7vX8emFByQExdZdLayIUQh9pMa/3Th+3Bbea4Lh6fNIqbxD6AHqJDAU08CIhorCILi+rH/ImPHKGxlI2FBDbx9prfnvqoNV81Y97sBywHAQcuxiBFZdiUlSI1zW3+aP54lNFj9KnPVWMNlyDlGqvkT3Jv+O/cfG8slBcxsfIqOsuW6rZ9hJ6n+cZpmXa/ZwTtq2aMX6rd9FfnceO/15hp7qffFrLBm/mya6fda9ftfkuenbQ1tLftsXYQmq+csEKuH561A4RVqgJ+NuycH2eCgKuB26tPBqAAdN+l1MtrfmzOfJap++eFOYiJL/X2NAYiC6izMWvsVIQwYfvY8qztj+jmycZe+rAHNgSF1DzPzTu5smwMlDVsJs7GohtTpu0y0OXZBxvuOKtcvwsDgEoNTDa2hNWceJTNashiYhtWMe2VgUEzn91sED0j7HwTWeedimu/3azj3hg5n0K/V9aLGzu/PjyriQ6+Wp9qv8c+C4u1eTvWfjNL1cZeb7flTAJmqKh57+g36WIa3sY/9FcVIe9zUBzbVnrZ7bqFTTDW0rGC3pQwZDMhls18KN7HEo3VDBY+Qtd4WAXwfmpq8wn97vhHkmNHScGO9Q9w14KbP83TxsmeX7zbTLvrgNRhQN4+3/hwuk285Wmgw234/Zje2vxvjGrcmvIs5sbEE7s1a6y4jgknVxb3/9s4ebDg1ueGmQ7ba18GYJ6Ez8D2nvadco/uPZP5Y/Ox3L/313h3/cY1M2v1pqyb9x8ov+Xb13OrL/v7b+Tz779S+ir5nxubj0i32vXLf+aP/NRZvV/43Z/xv/Znf/yM5b//j/w3P/W+AT9hxpKIBgB/AcB/ycx/US//GhF9x5nCfk+vfxfAb3O3/5Je2yRm/tMA/jQAvP4HfgvfDQsuWdQa74cFS41ivloDjsPamMLChLtxbWDzblqaqI6J20yDhIcAMaZhaQzVvNpnPewCM+FO/TXnNWGIIqgyBm6xMysL8CM1q93IzGuYi/vjDGZCLkHj/kkaYsVl6b6Zowtd4f0sxyR+lXToojzTIEJER42RmWvAUftba8Do6sklYBiqMm91Z+5LGIZ103Zh2IAQ1maeGqasoDRv1DkNqOYcEIc+BlZuKQHDsG6Eb4ahoBQBoxZv8jAuzXR41HiF6xo34jdWV1978n92pqD23B2OpamAlixiQKeXEmZgvg4IqSCMThDHNuY6LkFDF5j5L1cVOGFCnJZmThzTqiCcWj4zGw4uNmhrM4CooRxK7rt3ZkIYVnk9ybiR9rEdWhC3a0XXUtD/zWw13Uu/a+lj5+vYvkc3d3WHHbYu/BPbQDLpnMfTKrEuA2Bgum0gmikwJDYgoZvE2rwR+hoqJPHo9JWrtpXRRX+03DZX1tYTN79buSCHORSgYQnQwUpw99tmxq558ZudKSy7dtB9bT68tntpJs9ul2j+vbBg6VYX97UCAHRXwWsE3ZV+4GDTdNIyqqplEqvJrVTVDibWACRuBxFsn79YgVV8QvmwPUyiVCVsgfUzB/CxdHEYLxpjByME2UCeeGcKi74hcGbLbO11wiuwftj6qgQ+8tbMl9GBhpnC2uaeHCDTtVVeVYn9aGPXBIq6iIgBsGqHIoHFf3YNqC+rgGXubW/ttHZ7cGyAWse7HXTsRE82Bw7ejNvK9JvDW6bMjO19vJub/QY3MCgH5G84E2qfyI1bwLZ9rt0tRqB9F+wZYe22aSzUTYOdiWubI0I/IHgm3+bQpfUDbe7l8Mr1P1T3Ofq4mw+6lmlmwZ+c7PCArS+2JqDCRwaAZPw2zJe6R3hBJGJsDk88eNs8aj2o0/Fgx0y2g4X6/Ma/MV0OCLV22Lpw/XmigLtfi1aRXjff49ZPHYd23cBvaxi28+pNYdHHbQ/MnoA8NyY3QY5fS/ul78bjJqiyIgjtkK7dyzfy7K8FfnLfR4Gkfd/be3s+uN9VFtC4T/6R09chPS3zY9rxTNmfev+PMz3p59cg1ZvKbz8b6SepCksA/gyAv87M/6H76L8F8AcB/HF9/Uvu+r9NRH8eItrz9kP+lYCowr6bD6JSyoTLOnSlSmI8zBZeBE1x1DbUlzq0fKUOiKFizkN7zl3X1Ji8FGsDbT5cw7zK8Ik/pIb9qBJ77fGqgemVYdwyltTY0nfLoatLunx7X0gDQm1jr2nRuJuiPqlsbE0gAq6XUTfSjEsZEdRfcqmpsYch1uZDtyhrKAXKN3Rd0pZEMDBj86z9aeDzxpOWiJFdmXKPKBTkdRvGIK/m60lYrsIW5iU2sLHOErcrJG7AROrpY+vfkxOesZRntwHSDeH5rYSaCUMB57T5QaXdmPM19bLtR1XXSp5jm7P1vN14EQG5ig9nbgoN2Gzq89lUFlyDiVFKfLpxtYmx/tkt+rwqhTabz7KOW7Ef2+D6zeF+Dncb0OoXg/3vGcwgYLDVgV0bqxsv26Q49oTbJlLrDRpWwV4du8XuPfsfbm1/2W98bd3a5tnyuvXsUzsQ9mvM/ndlsLX9UX2iHbvIvi5rRkltg9uutba7schJQOFjbL6BLbOtB7tHmeDma3g2pRwGrgY+3f3vEmjgfi+j+1Jnks8sbwBwCZ0FI1eW5WkbN7dDa+DDbVQhefzGWz7brT/3ejNgN9wmNTwtpg3n2ySbXPtc+8oJ2/XJ2LC+9j+9j1vGzHw9rbn++WLPt8YQ9zpto+1vfuJD6Mb7YxhLi8fXliljo8a6YSz1M3obtt9H/10Oeqn2+w30NGZNmcDnfBw3zyDGFkjY+Yo+ezZKueSuOwXgDQtLvU1Srh7stXEO27VhbHsJneEibfvOAuCrJCph4xsqFWp/47Ye9vMGbBnF2v8XUNrH2ZeJXRn+x9cEiZ6wam5hNJBpbQrbuvdso1XU/QCp12eHU5aTzV9zv+Zpm0e/o6GIueYTVnQ3b5v+Am2d+L74/m0e4Lvf+1vXbjJyt74TvogboPRJurGsbIx8vR8Esn7Ob5W5b+ut68+lJ+PwTOYbZfnDio8Ccp/4FftKwPDTv8Y/Tz/B9JNkLP9pAP8qgL9GRP+7Xvt3IYDyvyaifwPA3wLwB/Sz/x4SauT/hoQb+de/rILKhLUGnOcRhamFoChVVEvF30+AXlYBnnURdjOp6at9fp2HJnpjISIMFJ7PE6KWBU4KNDSgNoBlHlpYhRAYyzI2M9ZlCQ1EAmjfxLyKomwasrJIoYFAQMreBKYmloCqDnhamAi7LyvQFbBMGEZRcDUz2FpDY5cMIK5z6mahBJSll2FmoJ6xzGpG2gEvhAGyU2Z3am3mpLXExhYxC/NCsaJmCbLNq/zScBG11rrI+zBUlEXYNy4aPiNVUATKEjYmpRtBGUIDg1UFjfxT0VRciSDiNkyIJ2Ehy1VBbHR5TMjEfryVDeKiOzG3MfTCQDSIOaiFcmivFsLATObc05kSA6FuxWVKELBSARqLsEnGhFTgiTmhmYCm2gNam6+gqcUG3gLUHRN3EzQSnBmrq9+AZNC6TWzGAyq7RxVlNyzCJmyC/lJ48Rmr0+71INKzP1aX/frp+HczWc2XuDNgxbXTM5LWhltjA2xNh3Uc+KiB3a9RwJ3ma6aRBoLHKmX5YOH6I95EZxjgSYRj+CDj6SX8+eCYJ4aou66hb/5042xiIo1N0E1ZfZFBc2ymrh408rHIZwxld6j5LPLAG/M/MIAofaOiqsCVwKF28Rtro03RWOUxmDWfMWU25tXWmfSrBm2DsjF1cn2bIPn0O9Q22fqdqC9LV/0ldCXfOfQDAMt7quLzmfr/5UUBzaGTcmOFsWyAzm0DzwIqa2Ok9XqEtF9/B7qirIxVPdquWtdBtpAVUkc9sNuAUwMD7MF/JV0D7nPHRJlgTbnj7ffRPzsYTpWXmt+rZ5zCQqgn3yddO9rudhinQj6UIf6ZgFMXlXJFMVjvbeaSFUHnuvmAZltr1L7nVAh1qB3YVgLH2tRXwwox211F7KkBKoYq3346W1BTbYq7H2Qs2/OVWjvN/H0vtONN4zcHE/53q+4awujqvzqGG8EgSw6s1b3yKeDAIDZgZQN+oOXbI0vzMZ7mkXHpjzbL08O/uLZRzw/giQmtZ1p9v0PpYK2JcO2+Iz55v8R9/zy49Yzuk+QPW1xbtqc7u+tWj587W++0zbc9rUfr05O2+Pt2+W6Bv1t93Rb26eknxVreAt1fJzbyyxLzLn7xz1j6iQFL9ZV8buSeGBezOHv+W1+ljjFkfOv4iMOLN1hKxDkLS3iIGYEqHtdJlFlrxBAKrnnAIa0oHHBZBw0fImzkMa24ZGEuWQHrkkWQ5Rdfv8N5FTbT4l0C3YfzblxwzQmBuIU2uSiDaXEim88lif9ligWRGGc1d7WYjlXNXRnA6TAjKghcNCTIEOvG6Ter6WquAePp2pjZECou84hxXDDEIuI1cW39NZPVSU1RAfEZHaJsjksNT/ICPY6lXTM/0Ri3ITkAAeelBIxJYnmar2QIcj3FimVNGO5zC29irybEM97NTWjIlG9rCbh7dRUT27j112zxGXWMhjvRn/dxKeU+bm0kYpwfhN0+vb6IkI6KydgBgo9bmFcxn40H+YU0/1gALU4lmES5NlWkqW4Uck1sp8Vk1PzMpII5hHhQ8RSgmbcSAXUNiHeiAmJhVrpJLAAG4knnKAfEadX+E+o1goYqAjuFOstHaH6kzayV0MxOjdUFK5g2wMwAJYigTZZDi3DKqEtEOOVuQqoHH8wkCrKj5KdJyjGf0BbwnrtJKE0MzgHhIOXSQcFbUF9ZM4HNoQHMZsI6aziNg4jksJktFmrlNMCvG2cLASMTrUBtUsEeVX6ViVZAStzB71UPMe66iA5yAE/m+KU7sVXNWjWMSfsMAMfaNx2LmqBeAzDWLSM0hw6IFXjzofRfXwXO7MK+8FgbMKZzFHBqZqbOBLJ9Bsg8n4rEg5wE2PNQt092M3MbKyiHpk7KGmansXoGxrQ+1sMSDtz9NiuAAQ2wWniY5o+qYEH6w02ldGOKR1DlWIiY0AA5TGAgLLJO6lR7HEj9LD4GlKMA4vgYUA+M+BBQJ5ZhJ62bgWCbfxMLcSaALdxIlLEhFb/pcRfR/VaD1OuWQAf6OtfhGtrYsZ4JcQDiue+2OTDCJYh/orJ+Tfk0MMKj9CO9CwoW0dEAdxVRjox4Vobvqt8JxwrXQQSQwqLhQyo1P0vrr4w/GhAZNNyIXQtqWlwn7j6RBC2HmoBL2+RrmA0QxLeY5drwXgGEsWEzSdmAhLzJ4jMb3oeuwEuQth8/fYeazqExnhIyQ+ehoglgcWCEtX9HDfyEBx1/XdvGvnEAwoOBVblHACfhCfhyYCu+kfEydtjCiGxAh7G+QGM4WxgTHXcrb2Nau2c3gx4OuO9+Z49dG8nlo16OhY+p7qy3tbH0OvZMbme54dapv9bNhzd+wi5tQKUDYz3DjTy7JbJnLG8BHc8Ye1/OJ7E79+DWtUMy3rjm3+/BJQEWusV8MDftsms2nh/JVH4yq9rq3a4Xf/3ZtAfruPH+Q/f+PP3U02+IKuxPKp3iin/iG7+CH6x3KAgYqGDliEsZsNSEXzq9wVwTppBxKSOOccH7LIqeL9IVj3lCBWEMGT9cTviluze4qHzXMa54MVxxKQN+MN/hl+7eYK0RmQMSVWQOuIsLAjF+MJ9wOGVUEI5xxdvlgPuXAmiuZUAi8XczQBiIcS0JuUZ84/UZhWmTz+r/4XJs94xRwNlSEwIYFYQARgoFKVSMIePdctS8Gdcy4JvTIy5lwDmPeD2ecc4jrmXA/aBgiwlv5yMmFR86xBXXMrQQLLkGnNKieUMD60MsSPrUXGrEKS24lgFjkHqTE09KQVR0p5jbGMwl4ZjW9nrJAyLV9v68ikrti2HG+3WC+dHmGvBinJGo4O1ybPPgX6OFfdG6LlmY4UHHD0CrOxDjsg4oTPid3/x1AMD3L/eIoWKKGWuNLY/56UZiHJOAtbnI1yfXgKSA28YSAO6GBWuJmEtq4W2GWHBZRVV41YMJKzcQ4zTImnpYptaHa064GxeUGnBMKx7WEZG4lenvJ2I8LiOGWDDGgmuWNg6h4uV0xSUPeFzGpjZsIXXs9Iy0PcwieGV+xrYOp1iwqkKu1X/NCYeUEUPFZR1wNy54XMa2DooerARijLFgKbG9AsBagsTsZjRxrRQqGBDf55RxWRPuxrX1J+uBxargdYg9PFCphEASz9aUlgE5vImhYggVj4uZrod2nfQAx9oyJvGbtkMdC1tkZdmhjv3/4jAjEOOH52ObuzGVFuIIkIOZacibQyW5zqgMZFVOZiacpgXvLge8PF6bVYaVc5pkPYjQmNRzXrop/5Rkp3d1/t/ZhLIAvNS4vbYmh1Ra346jxPy1eZjXhMO4YsmprVtvGh5DbQJqJmSW1G/dfMNZx1rmqiDo59YedmvORNNqlbFNGotY5kv6CqD5vi85tgMQO+Sxw6a7w4I1R51DqdtUos2qxHydD+OK83VEShVjyrjMIw7j2ixbAGBU/2rSZ8xaYj9M0rIshJSVm1LB6sTi7ADM1KCnMbcxsIM6E2sDgEEF1tgd6DGj1WPXUypNDM4O6HybljnheFxQmZDzNryWWOuETWgqK9/yMQPrkhBTxTBkLEtqB34h1NZHZhLLFoKoStt4uDBIgFjHNAE0tWIhgvgVx9rM5c1n3dwqxDdeDqhMXM5cR+DnNNaNtU4TiRsK8vXTtz4xSbmAHMjZLjmQWBlZbOUQ+4GjHeLFJM+1FnLJDiSqxEu2A0NbZP6gdmPur2MWxyJjoYdyIYkbyZNNuR0oWSgoPYhpoay0Td5v1kTfLMQUNO+TcFTFlWHiVhbqiS20lpbtrWh8d/Z+yWbF4Ok3y19p685gn1veGyarzey8scG8+Ux++NAtN/aENrvPNkCQtnU31O/QmBt/39wvTXuQ6d/vwVdrn40T37iHtybQHwlYN+4Mvp9MH9ePfX++LM9HgMhbuNPf/nVIP1eF/U2aPv8HP+ff/5/9fqwcUDkg67GQn7DKhFwj6m657fMA+sPglE292M7GjxBowM6X68v0QHL/mU/7Om4lu9cA362ybl0rypIBnYG0vF7h1ve/chcoauU4tvW5e271R+6VOKL7flaWzbSBAPODtVe714+9le/f++u+Dbfue27MDDTuD+D8s9Vf93U8l/brZX/9uTbJ9e2GYrO52NX7XBkmELXPaxsz2+T7tjy3Bj1b6x8XPqbrvi378f9QW78s+fHwY7HN5H7gn/kR9O24NXf7ufnQd/LZ3z+779nOUO9T2LbraXu2/d58tt88PbNQLUzMvsGbMbANpPXX2oZPmLOPOWXez4ufi/08Kmjw/b81/5t14cv42M0ZYbsh3U+w/b/fSG4a4a77+fBl2f/P1bu/197f2gl+WRuea+ettO+nv7YpX663jfq+vx/a/N6q60NteO7e/VwCrZ+bff0N80e7j/fA4SukW+XdTLfW6Q3csXn/oY31jesbs09i3DQD9UvCj8mtdPPH7nb9T1bWs3ODvubr7n/XRuD2Mvfs4X5Jk303nuvO7qfAX9v078ba/SBL9xXm/Ln2fDQo+9h8XyE9W/cz17+0rR/Txq/Yj68EWr9C+f/Hf/DTj2P5W/6hb/K//F/8iz/2cv/U7/2zP/W+AV9zxvIQV/x9p19HBWHliLXGxlxWlmv2amwmIOybAcNAFZljA4nG3BgYNVZwqamxdAbM5hpROWBSnfCVAwaqDWwak7S3pc7KhAWqG6Yyc2jtsFP/RAIkM8uJqwezz4FVa5+ULcyUtL8oeI4IVFE5IIXyDFAVhtLEgSwZuNyD5n0oFgOcHgxbnwzA26sHyz5/M8etEZGM/RJT4fTML6MfG2uTZ3gBtLG0PAGMS1amOq3IHFrdHnxae419uwUQfb2rhqDxde1f9/NmbKExdnb/qnU+d0CwP9QA8KQPS46Igdv91g9Lt8D4c4cOe8bKmENj9VLsIle+PDtosFc7YOj1YTOmPm8uW39l31YPlK191j8vuGXsWbS1YSbUrp+W7Hsb3Ri3sXUHETZPRZmoacib9Wz3WMnGWJkZtu+vnwNjnXKOjRVq/VBGy4P7PfjaA/+NSJT6nJuisqkqe/bL2mqm6/b5LUVhY4waQ7Or2+qXMrf5zaTez6m9VmU9LMyRsSjMPYxRY078JtPmKcfmU27zAe7iab6+WggxyfeFi/iXV2VjGi40ZdPaTcSfHAI5xmnDCvlh82DehSgCOqhv4+tMyj3oarFS9R4uPYzSE9XiosrVphy8A+9988+dtdorzBJ3f2I1/2amDhD85txMeQP3cuyasTd7ETLz5fbhh5i6/zTcvVZuGwLN40O9mB+jmWPbHFVq4UI+KZkaMaA+d32cLLyP+RzvmS/2IUY8yIKbT9fWDSLy1yH3UXYgmXgrDmTPUl+/+bLuDlHIwPb+YIE6CGX3P3SqNmJs1ka73psl9wen/gs08/E2FtC23Zoaa6Orm2DfeW717AGkH8a25G/4qlqfbocboQ+C9ufKegJc4fpmz4xbhwW7fjypx9+3f30uuTZs2vGhvrj7PoV/+lHB6P7jrww0P7Ken6cff/paA8tzGfHLb38bDjFjqd7HckUKFW/mo5hylYQpZlzygGNaUZmauWVSE8yTmluOoaBCFGYXNUt7NV1bfh+jclaTsBfTjDmnBp68j+Uh5RZf05IBhxQq3s8jAqGZgFXdEOYacBhyAwEGCoa4BYJmBpZrwGmUvtnG/vE6IsWKMRVclgFjyoiBMefY2KzjuIp6JoC1xGZCZ5v5OW/N3oZYNjE+A8mGf+/rCHR/zDHlZjIWY22b8CGKgNKQSgOUS46YBjGnW5aEccxY16RiSmJuVQphmnLb7PZNMVq4FNtoD0Npm3TLYxt1QMzUAjEeHsRE+nhaUKuYctlGOqlJpG1ATVQpJjMV6xtkMYUSP72yRlBAi/EZnA+pve435HkRRdowFtlzkAkzyQa5rgFplPHyQky2YWYGYpL6fGzTUkLzsYyDxSLtYIZUtdL8NkEuTAg5UJC7CZeJM4Uk7WIG4lhR5ogwlr5RIjTzsJoDQqob/0jz7wT65sraFgYRcwpjER/LKP59FMT30kzpTISJArfNDc+iphpG8bGsFrKjEmgs/YfWxoIhwky63r1ZFxRs2OcmxORNwfgqgjfXU+5ma4X6/bapNlOw0ftY6qttqgHJNxXx3TTBH/suzg4gBKkHLozQpg4T93Bl0yVgOZqPJW18uWgh+Qyu3CWo6BFtFWotjwECE20pIpryRBwJUHGabdxNqdiBGgUle39MBDQfyzw4H0tru4a5MBATHwPqyCjqe4gsPoGr+lICCgaYEK7UroeVUA6McBFfQAMA5uMXFBDVoYMYYqAG7sIuKiRkvn7ejwxOkCiaT6ZufmsSESLbSIk4i65pE4YJQFz6nHFQn0INWcHJxccM8n+dGOks81GdeqkwkGjCOiGTBFrP+tnOxzJkFfExH8kqIMHiUoLQfSyj+P0BcD6W0tkyMUKROsxPj9j8UdHmcONjqflr6jFKbUwpo4+9ChDVUXxemyBRAOIM5LtP322GReOdQtts31f1sZTxY8SFmm9j87GcSX1sHSjU9ocFfd0AXfAmuOvsgZr0per42Fh/rI9lGzcX+9OzmRsfS60veCBrZQfv46jPbZfPRJg4EEIGaqLeP33cbuKPuvp8PU0AKfQ+GMt7yy/T6mhlOQDW2GH0a15l2g48OsNIPQboPu3LBTbAfmNa6+vE7tq+PNe2DVPs4lg2X0rCxseyFUXU7pFxIFB1hX0JuPWvNxnbDwDajwaW+zI/lP9jgfQz6ZPB6U8glaec/89M+loDy0NY8Q+//FX8MJ+Qa8R0XNXHckTmgM9fiI+hMY5jyLiUAZUDfuvpLTKHJuzzmEd8fnhoDN14yphCwVwjHtYJn03nDZNYmTAGeRqe84BvTP3aQ57wndM7VBCuWerfs1RLScgc8It3b1FBzR8RQGPRriU1djVR2Zj7Nh8uZUvHWPCwql+U+hyeXknsymsZcHi5YqlJfSlzY2rn3P3/ppQFIIMQSdiRIZbG1AFo7KFnFe0+z45Z3iEUzCUJU6lPgqVGjKFgqbH5MgJbwaAYKqaXGVf1z1yqxuiMArYN6Ft9zcfK/pSlNMDvWVEDx0A/HPjOL74DALxfJtAujzFXNn+jChztfSQZ4stoYzWpf5kxlwxhv8xHsezaDogPIxE3/0M5KIhSFhMm9Zsk4laGZ1BNQIqIN36MMVQ55Ch9zp8zd66OWfQHKQCQdFysPmbCWkLzcVxyxGHIuK5pc7+VZ+ymN4POG/atIhAc6yh+dmuOzQ/QmFTzRaxMSLEgEFqsWQCYhoy1BGQ3ZAe5FAAAIABJREFUBuYraaGCbE666XZns5ObZ9L7W+xbN39Rvx/Hb0ms0YsqTwPio2jrxNjUQds6qz8c67pidIaQ9UDmugw4fC7+jV5Ey0S3TOgrEjc/QmbxAQWAVZnJvX+e+A4OjTn1Ju1Wr7GOprDdhbX6GDPTxpfP5/OHOUBnlf0hkfXVyvN+f61tgZvvoIhpSX3GutphEjnmsq2B76zqO9jrJgKWJTZRLhsXUQ1PCLFqCCnxM1zX2AB9TP0wyA6s/Lz4sE92WBSj+gTaWGh/KVRw7Yc/jfVUH1F7kraQUIDEzLXvSqztPmJhW020aW/NQKECOSJqzGEqWzN5RcXSTxUTg6nYhs4ccwngwGCNx0uAlE1ozKufz2oHDEBnexVk1dJjifoDrapjdYuZhT0bSkBNFd5HkQJv/AVN4CuYOFjofohluYUQPi5RlEMtGxvvS8dunn2cXe/jaOJkfj2AsY3Fa3V51n/vY8ly6GVtaYc7ZZeP3L1eCdiYas8yen8672/I+rlnmX3Ze6ZyH26qurrcgeWmjcDGLLwxmja22i4yH8tb9+9AWB8r6ddzirEbM91926xcHY9nmb19O+yZQLjBkn4AVDjK0rOXW9DK7dXqk0OnXePaQ0Xb5N/7dvvqXfOfDMUeXO4zfAC47cdtA7736Va5nwgmn5T78/QTT19rYLlyxK9cvokxZOQa8b35HgAwhoIpZPzq5SVSEHPTU1rwa5cXTYzm+9e7Zmp6LQn3w4zvXV40kZy1ygY8EOPz4wO+mE9PzA6vaj55P854v4qozuM64vXhgl99fAUAOKQV83LYmLqtChSGUPD/vvsMgbgBkqybxVyEgTThDQMjk4ID21j/IJ/a5v44iECOgdT/7/oaUyoYU8Z3r69wHFe9565t5l8eZsxLRCTGF5cTxqRhNxRAX85pA3zGVJBLbMxqDLKhHXXTb0ImtoEvahqYS2ymjEOseLMeMaSCL9YTxlQagDBRlrIO+LXrC9wfZ3z/4e7/Z+9Nem1ZsjShzzpvdnO627wmmoyMILMUWVUCBAk5YcAAgSikqhohKDEoqX4AEgMGMEGM4AfAAApVhQoY8RMYMQCknNAoUxGRFX289+679557zu68sY7BsmVu7mefe997EcnLG0qXjs7e7ubWu29btr7vW6i0gxARr+wazkms2wE2tTPDA6PIIhN8rqlou9wVsNQxq9iK7MX95589QwRwselo/K3O4Wqq5FFlLqa1FK9QpwUTG+YAKc7qtPgcBwMhA4yZFtvWKlSVxzjSopUX77xYt6NG8AKmnhRnlQq437fkuRwVTO0SdI+8oNP99FxUlYdzMgtwkCCIhD1VkJVHVbusbMtHhgYGCSFTnuWCPS2SSi8xL46U9vBOIQYBbTzeDBo6eZQBzMLbeCuhTIB3Mi3SRfKwpoVokNPiUsQslKGNhxsVpI6TgEeC/gkkYZC0OONzd50mDy0LZiQva7QSikODCII7ZqGJpCYKAQRLC22VVGGjS6Fv0mf2nvLnw2kNRAG5sfkHLDgJxUqzaV7GFOYjK9OmugmBFOaG+uAwkLru6eUaonWzeKrH0zp5DiP94LqkdJt+reOQjNnG59A1Qk+CGt1xg7j2VJ6VZJBwWJJBIq5T3ZyAqAL6XpH3lD2X7D0VcVKVNZGu8+LWhCm0jcC0+LQpfZUMIfYwlh7LACrHSgrhwN5dFSGGFM7FRNhRkuGyhFkmw8TvVwhNnOJTJoMiNh52AdF0vURoSS1W9gJ+HeBPEqGhcYYAPIepSAvMYCIp2oZUto4QKU4ulxls8u45VjGl9ooxeVs7mTwt1KfRxCnGowDiIGZeq7zYZE+njHR9FPBVTF5PILJaqwTiKBDbAH9oyGOmqa8ZnsixK6OOwEjtAKvKOkAk21rU5GkVg4BoYo5pKb1ArGL2VHGMS6EnT1wwdE6O5N11bZwpygqbjOo6zjxrmY0hkrcT7LFE9l5B0P3SUt6+TuFVmgjdUb/IFHpHdQJ2+9VXmGqgfMkbDbCIifCAb2IO8UMe1VRfViHvaW4Ek/pFchsiVC8mxdP0vshhYyKy5zYbGArQJ1LnnXks2aNXeh5TH0qbxqbwWJZ9es5jKdNrJUpAcf7p0RcBk3c75QExtS3no1I6ixx+hu8HkGN4ckghqmxhfCSPJc9nbkPOg5+VAjVOnTTVS5Ye2KVxnAxHrms2IIs04hGP5QOvHufJbQsRUbGVuSjzjLFE+YmcbuZhxHTfbG5IAenjo+FS8j5C8li+06CL8/sepOc5KBftPpNmqsQ8/9JoPlfXssxsnJ8z+N/DI2JOX/pdO95r8Z7tH34Y/+X/5j9C73T2mrB3hENnsAfAeZXVBUtvT6nEx7vsQtBintMwFJMX/2xkTbEqdd7B5pAYzFHihTgfJXcoBDFTECzjUnI+007+3HjgJ1KmHXghYqFUR7vJVe3gHIXO0IaMDYYLzmChadEvChirSHWVys84RKyux09/SF6mciedD4aoPuApFeeXynmZ15TqWBogwVMcSynYA4CJK8IvWy6eNwHY4CgW5QzLBEA78EHAtMkAHTQgYoZ7MuSyPES5A89jkfojK+gBkDqF8ChifFL8ToKBcpzMkp8lNI1nSJL8DJ/knW6hA4KluZIXogXPDCJmHlVWAEwGF8NTQ4LVZnhdMjbyjjJ/LsrgH+6sZJiMTUSR43IKAUQnpl30Mq+0q027/WIyPgTmXCl+Vvg7h/JwyVDxi7qWO+TAfDdcB/rOcUbzoj5OUNFYnC/rkowYREwegNIbwLv5/NkLoE7PCocdWd7D85Khm7Y8n8os5hLYA8UGWnlwjE6keqg4xeYs+yPHFiz6BSAjMRm4kEVfJoMNKSxH5rKp4n/pSeD2l/XgNnB6LjsW6UuO3LlFnsDExyp4eiU/TiSuW46rKTD3aghQqBcn5xBZQTE/Z1wvgGJujhQuBSrS54piieaFFHPzeN3nxRzqFjAZsZxvEVok91NENkhzKBVuN+eJqczM4+PzcaoLxwGNCRoMWYy7mOokx2QkB0z8v7wgj1NMzgR5jgytzp6aFPpFxRzflAwjKocNKhFEjuGZ42HyeAlQ/gAZ1jweeayRy8gCQQxJj5hBOoOZGx1RFX2f4nFKmyC63P+R+kn2X31Rx/ExqW/itDhmmHba1CiNO+Q5Fqf6FotyhhKzMUmNnC+wc//x5UjQW64LbShQn5f557EV01zM4xQLY8wv7JyFx5KNuuWzPIuZWkBaZ+/HYozEtJ+Wn6PslSvL4+ek6AtO88B4WsBXy9co3SOmeVauEWKhbFp6Z5fTI7X9QYzQeVbnv5dwYzFd53zPIiLfdR2YD1Ysvovi3CLtWePsXNli8f83PB41ZN9W9l9SmX/2X3394j3P/uhp/Ps/+Du/9Xz/uz/+wdfeNuA991iuzYi/df0pnlQH9MHg1bCBFBE31RErOeJn3RO0ymJnG1yZDq/GNZ7Xe7io8Hm/QaMchQexDZ7XB9yOK6z1iCEo7MYW92ODVlt8d/Man3QXWGmLWjocfQWJiLuxhQ8S31zf4d4SR28/Nvh4fY9PTxdUl/qEk6PwJxMMVmFjBqz0iF/sbyBExNoQbHXwOodw+NbVXYbb7m2NWjms9AiXoKNSBNynOvRO4+nqCAA4jDU21YBf3l3h8qLDthrw8rjG1VUPIz32Yw3ryaD8cL3H3tYw0uMw1rioKegYw1h3Q4PRq8ztvKgGdM5kiGWjHXZ9jU1NsTwZQipExMpYnKzBddPhMNaZs7oyFoexwqYacde12NQDeqdxUQ24HxpcNR16Z/Dm1OJmfcLdqcWqHlEpj9vjCs5LfHhzQmdN5m9yuaNTiasqskeWQ4boBOvd9zUa4+CDwEUzoFYOf/HZMwDANz+6Re80TkOV4YYc2oE9k7tT4mPWY/J6+hwy4zSYHA7hcGygjUdb21zHfjRo6zH/5w0P5sl2pxreSbQbCgmjVEBjHHbHBkoF9F2F7fUJ3k8eSRZXYW/tdjVgcAp9V6G96GjOWY3+voZqPbY3R1iribeZDMRyI6SEH3ovoNQkHDMOBqYijydtGATUW4thMIheoNlY9IcazcUwxftUIYdCcKNCvXYYBw29Ik9l3di8ocOwQ95oqWqLoTOoLyz6YwXd2uzF5dAHAIVBYIl/5r4O+xqy9qgvbebGeicRrYS5GCAE8uaFdyl0yNrlzRPeZNAbCsrmRjV9tgpKh7yhoysHe9cgRqC66QkmKJKHuXEZ8ua9RDgRj1Zt7ZxbKghqydBKf9AwlwPsfQ25sbS5xd75XZU9lqKOiIOk/BKk0HcKCAL6coTrNCADVBUoFAEAcVshXo+QJsB3mgzXZIjLnUa8srRoHhTUxsIfNMTGIXYKYuUmw0pEhFERl7T1iJ3OnEyx8mRkcz2ToRI7TUYbx9hUmLyvVk5czpb4pWKQFEszpGD3pwSLXXmITpE3k6GTyQMaU4xU/crArwJ9jwKyl+RZuvDZsI+JA6t2Gv7KQYwScqfgLx30nYbb+rzJIo/JE+wnz1g0IRtl0QSKSymQYxqqo4RfB+JnpsV9NBR3MpoIfZespbSoDVWcOKUC0HuVxWJi5tNFmF3qB0EGlbqT8Csq09cpnmYkA0sfJdw2oH6hEVVEqIrFeUTivlHZei/ha0Dfp9+ZUWQuqV9RXuok4NbkoZRWZG+dtGQs+po5joC+T5SChjyRqidD3W4j1Aggymz4Cg+4VYQaZYpFKbN3MApAndKeRQ0094K8bIo4qeogIOm1iVADMnEp61eSvo/URnMEhuuvvnLVqe3UNzIbDcJTeWqQ5FHtkb2pMXl8zZH4q8HMPa7kfRSTBzZVr+QqBj0Zopyn2Qm4VTIqFaB7HitkI0t4kb2EahDwFY0ZcyppDKhPy/iQMnmN2QsaNKB75JiUbOBl7yuAUFE79QnZS8qbAMEAugNcO20IsDdUjcgc3gecz0h55PI0MtcXkfIQPvEN5bztc49l4g4nni5Znyg2XzDzlM9hpgLKxhyDkxo2lcNllt5dWcwL9rpznUqu9AOYbLFRlXmfhcFYKvtmQ1/iIX/2jKE7j/95ZnLz5laYG+flvMt1EMDbPJbL/M/mx/1YlH3WuBR4EKfzbCzOvwSj+K+PL3+83x7Lv/Fh/Ff/23+A3umZAqbzKkMrS0VJhm2WSpHMqWKVR1646cTHIpjpFPMLwIwPBkzeRr62VFhkj2XpmSo9lhP/KebrDPVbco+Ww8V5M8yyzJ/jny25UXwPQyRnapLsBeHdPBlnnkj24uX0EdnrtvzPsErmC4nE3Vkqo5Z8rbIsbjOPFXOuICKCV1PaBXai3JBjb2jpGeSxAZAX2toQrsaNmtqcoJ+lymPp8aS+mM5nJUte4AIzQZycLufLnj9MniKBXHawcnoxxmkXnD2duaH5BzLmc8wv4ryZ7yM1eWpnKpRFGu6n5TjP4pYV/ZI9nqxGKQr1yVRG7vfU1sw7Krd0yx/W/EPJKwIx95qV3rKl1xDcH6m+LMST+ytOkMvSu5f6voQv0WonjWUJr2QvW/6lxuTtYw9P6V0svG25v1n8xi3Oc1/wuLKBxf/LcWbhm7IODCvlugNJPKfoSz5KLyj3C5et4vxaKPp+uTW/HCtOV3qTy7RYpBeLa+WigBdYosgrLaJyX5VzoOhHVuaESf1fzhNBokCRn7mFZzF7ztwk+pI9lgvhotJbynWaeSwFJq9bOUcDyKCOyB6/mceyfM/zMwHM+mbpyczeW+67sm8VsheSPZxnPRx5DNM9AKKY4v+xt4u89Jiem/LZAfJ4iVQvHhNgai/1depHnlbs5eQFd5FvXsADE7wzPe95kb2AX87+s8eyEBT6KgcJ70zvhNlPED9vcWE0JM9Y0AmGGKZ0fC2q6Xs+itW5OPM8ZVEiMfX1jJuIeR9mwyL/dhR9tjBwcruKbMpXUD5fvBLytcWjLYqySo9lLqP0iuZnGPPCi3azB5LzmHXXGYtqsUSYlf2o16/sQ0zn3maQnfvOY3M2/fL5K8r5IsfSO3uunQ/K/Q2O2fi+La/H+hDzMS7z/FLHuX77gsef/dd/NTyWf/cH/95vPd9//Mf/9GtvG/CeeywFIom/eIUgSKxkChpOHiJAZZ5irV3mBjbGpaDbgQxQq1Fpnw1Po/3Et0xcvnLjKIqYFVS7ZNRGIHulTBL+GJNQSimOIUWEExIxqsxp5GDhrAKrVUA3mmwYaRVofRTmxpfWpC2l1RREm71Dq3pEn0RKpkDfEsa4/MxP6qRiZpwS9DapphaGmLUqe7RI+ELm8krhDQDwPhm8o4RK+YQgAIbmMncuGbfeUzqG51a1w9CbnEaC7tPao+9EhvyWb1OCJGOu4ApkuC5ARiQH646RuIFNOya4rUrcPp/z9pFEURApb20SD7UQFIGgBUO+D4CpXFKYVXn8lQrwTkEq4iUKgWmBmzyHSgUMMBmOHDzVOUSRA51nNdggEeMEgxWgkCxCkVfPWSpbqIimsRhHBRcnSHGMJP7CHrPSEBQyZsM6iyMl6HKpRhsEcn7OArpycKOeGeBZwTYJmWRlWMF8yvRMJ3i54F16S7zGYCUpwyZoL4dwYCNb6kCL+CDSojZCVR7BCwQQaYnFPaSMhKjlRR0rzYoi9EMUGfIcxnS/iohCTp8TLJcNaV37xKE202JNBUTI6UdQRqiKNkf8MG1/Z1hxXpgSt9N3CqrxlDYHvwZU4/O4MFQ69PxyihBV8g4OpAQcI8jwTEadWVnYk8lcyAzNlhHKBDhoWkRp8oaKiriYBHMW08ogIkF7I0FOK0+cUx0QR1UESscEZ00exsh8TB4zzov7wAnAROrDNAYzA36USfVWTi9m7sfUV2rlSKglbdQIk8rukI3vPPcbj3jSgA7U3pOGWJGXNre34vmZ7nViUr4Vqe6muCYA6DC1lY063vjwAqIOs82b6OR8M6IK0z3Z2BJARb853LeiSuUITH3In00ABgmxTorFzH1l456hziU31qTngjeKIih/HWmc2RudeM1xVJNRyu/Xc/2R8o2DzHWMbGjJSJzfEnKfNgFmMGjLvFsBVqWNKmaYLasXM5w5qxkHAdQB4VS6nr7cEXUk73P6nJ9tgcybzXVNEGJaeACxov8ZIpzqjlBcYwOcN3yS8SPKDUWA7qkTVFsA8CA4NG+k5HcOXct1Z5RAwAR9Tn2WVVGBvFHBBq/wYvKks2UhirbEZHBzOzldmlvSAcEUqskC2fiISc2YvW1AMkD5uSsgxyIge+foRDEGIameFkYdEKd2lf1SGIox5Zs3A4q9QTaMH3jnCkMwe/9LY6eow8ywLNI/apQvjd3iyJsIi82X2aZBWb+yrRx658zxqJFaGPGzykbx4HSZT/n9bH1w5vxjx1sM93Pt/Ovj6z3ea8PySXXEP/j4/8QLewkPga3sYaPG3jc4+BrX5oghGNTS4t6tcKlPeGPXAIBL3eEUKvTBoJEWb+wK1+aEe9dCIWCjB2xUDxs0ft7f4MqQqIuHhEKAh8SlJpjhy3GLVo7wkNioAS/HDZ5VB4QocOdWaBknAsCD4l4eXY3OG3zU3MNGldUmtQwwwsMIj9d2jRAlbJRolYULCkPCVMj0pmqVhUJALV1O36oRna/wcXOHe9fi6GrcVEfc2RYhSqz1gFpSSI8XwwVq6RAgsNU97mwLYIrd1yoLLTxceosfXAUjAtZ6QOcrhCiw1gP2rkElHULBbu+8yVDjRtnchqOvsNUD7m2DS9Nj72pU0uHkKqz0iIOtUSmPp9URn/VbXFcdjo6Ufi9Nj1o6fD5ssNYjqf4mFVwtAkzCn3D8z3vbIESRQ9IAwNYM6LyBRMTB1XBB4rub15Ai4GfHJ2iURaMsem8gRcTJVVmVFwAuqg5KROxtncSSdI4HyuVoGbBWIwIE9ramUCpeYaUtDramdroaWvgMk5aIuK5PqKTD64HmqQsKvde4qrsMob4bV5CIaLTF6FWOsSqTWu9ubKBlwMYMONgaIQqs9IjnzQE72+B2WOWQPBxrlOtQSZfVg1nJ18UpXM7GDDk+Kt9/clXO72BrXFQ97oY2C2WNQWelXx7flR4pHxlwtBWM9LBB5XBADAE/ugprPeLoKlzXJ+ySENaY0g6enodaudzHHOv0uj6h9wZHW83SNMriTRLjYogyzxsbVG7z2owIUeBoK0gRc8giEtui+vogs2rxs/YAAPjseJGN7lbbHCOVN47WZoQWHrf9OqMsWBGaFXsB4LLq8apb42l7xN3Q5tivAHDTnNB7EtZykeqwG2jcXZDYVIQJPIw16rR5xWJkAPB8tcerboPBK6yMzc+QFBFb0+NVt0FIm3HHscK6Iqj7KoVS4t/uEEVWMu6dxqYaMTiNWjucrMHKWAxOp76l56fRjtSzbQXD6tJpHtbKY/AqK+022lHbxjqfW1c0Lp01aI3NiBUpYo7/WkL978cGnTVZGVmLgDd9m9Wdx5R2Ww+4PbVojcPajLjtVrhqO9z3NOdCBLY1lc0bh1w2KzLb1J8sxCYECbMxFJ9RMqWo2aYeZtz03ukcXgoAWmNz32Wl6iCyWBuHuVpXI45jBSFI9VgntV8Wdtv3NW7WJ6IMpOeRn0sjA3qns2hcZzXatIGWQ0xFkSgCDq2xOAw1tPIwMkCl+1kdu7caUlA4rDYJqA1JBbhJ+e77ehbeSivK5zRUWdBNyZg3R2PR76NTaCqiKvhIqBytAuqUd2816hRmq60sBpc2h4PEphlwd2zxVY/akGoyMG3sAiTi1g0VjCHhskrTxjSjbWIUaCo7qy/HBvZeUnvS/ECRZ4a/sxhaeoZDkFg1Q66L9xJV5eBSiDBG+pToJ5M2aTk8WBmz1jlFEP70/lEqZipEjMhaDYy2ylSClAZA1nkw1STexmlCSH2SxlOISXdBJ+VjpiFwn2X0D+/NiSQiZ6dwVUTnQEJJiQlZAUyGa5jQQnQPsgWUETQl77gwLIFUSd6IKQ2XcoON8+Tz/FnioVLvWwzHB8gZNgQLQ/aBR5DTLTzhU57FptaybWWdzub9SHpgQky862CDvMy/vDbL9My5x+r2nh4x4kF8+9+l472Gwn78N6/iv//P/m1cmxNsVPhsuIASEVf6hI0a8NPuaeZEXpoOr4YNntaHrCC71iNaZXE3tnje7PFq2GCtB4xBY29r7McGtXb47uYVPuku0SiLWnoySkTA3bgCAHzU3uM+GWQHW+Ojdodfny6hZcBVdcLJVTNvZe8NNmbAWo342eEmL+Z4AT4Ghd5pPFsdczgRNraIYzm9GQ62ho8Sh7HCs9WRlGldhVZb/Hp3gW09YlMN+Py4wXXTwSjiUvLi7flqT4aTDJmbWSrLlmkB4KLuM8dSgEJq7PoG62rMyrXcVo7nedV06JwhDqYMqJXDwVbYVgPu+hbbekBnDanrjjUu6x6D17g9rohj2TVYJ4Xcu67BYA2ebQ85Pxtk/s8waF5obpthtuiRIuKQFkc+SFw2PZQI+OnLJ4gR+Phmh8ErHIcq8zDXaTHAYk67LnEs04KJw3pIEXEcKlSaVFsPpxpaB1pMpB/V02Cwqm1e8ISIrJ4bgsSxqxC8wmpNXFeVvOeHrs4cy826z6JSWvtsTPDiZdMOGKxG1xtsVmRgDFbjtGuga4/Nusdg9WyhwiJSHMKB45OGkGKPph/7cdDZgxwSRLxuLIbeIASJuhnRnyo0q3Hm/aYFjYIdNepmxDCYBANHhoMzx5IVbwGgbiz63qBpLLpTBVO5zLEcBp3rPQ4mK8yyqFZ/qCErj6axOTapdwrBStTrMS2MQuKSyqxqy3BuN6ZFUlLhZQ87gOx1ljLCWQ2lPca7GogCzZMOPi0kvFWoEseSlHQlXOJY6o2dkI2ZY+kz5N4fDerLHsObBmpLHEteGNldldVZReJC6o2dxMh68jialc1cUWnCFPLhTQVxNZIHeVTZE4UI4jBeWoJkjwq6cXBHA9k6hF5B1lMMUCFAHkEvIGtP/NEqcSzbwqslI4V9iILOeQE0PntRReJYRivJS5rif8ZOkceq9eTd0ZE8iEDidCqgDtPCqfASIgioW4OwCiTiEwREL0koZesmuG/yyKi9gr9ywCihThL+0kHda/hNcvdIZH5nVsGs4wSXDeQxkidJXK/k3ZGDSBzL5MXT5IkSA/W5Ok5CQogUIzFWIXuoZM8iN0jeI/ICqZPkW4hj2Qn4NhIPLYm6iEheItVJuK1HdatIodNM3igRROaBRUP5hIpUSoHEsUx1ditSO1UnAbeiOJQPOZYRPnE4gyGuoYjE4RMhKaNGwG0iQVIjEJN6rPCpDSkeo3R0H8MoGcLqK+YrRkSd1tUWuc4hcUzdOkIfSWmWeYX6BAxXX33doztqOwSNbeasecCvkmAQq7xmjmVM3MPEozTEAWUeYVTEP6SxRF6IM2cPQOYs8qI6KsDsE2cxcQQ5ruVM3TVMHD81At6QumtgjmUaA18BZzmWieeZY22WHEtfcCxFUqhFastCaCkY4mi6ZvL8sYeS72fuZBSYxePMMVWTZ45jnYqIGVczw6EL+DSASQk3XROcV/Iych9JhzkPkw8BSBsRWN0VU91EjBS/M4KUWQVm6qusnMvlAIXNtziXPaKiqFspdoS5t3LG1SxVb8X5PDl+7czzWh5xqlP2Wp7jgfIc/P+VY3kmj0fq/zYD9E9/8J987XDRp99/Gv/OP/27v/V8f/Cv/w9fe9uA99yw/PCPbuJ/8D/+W+hChcHrvHtro0SIElKE7GVyUebYhiEK8mhBZC8A/ycPjsyxJwFkD03pjXNRZg8de6iYM8jeHgDZCCxDlUzliAy3LeOOcXre7QSQF/YBIreD40Jy3EnPMS6Tx8VInz0anLcLEpX0COlpLtvN/2ecQTHFIlyqvvLBO98P4L5F3mUflEYaG4SiOF++b8pzIX3mPJb1Lsvgo9z95aNsB7e30gQPHpJqcJnnlBaQYtpFXnKRPCfUAAAgAElEQVRCz/ULx4SUYrrfh3n/lmPMMRw5ZAsbGbz7zHzhMgZdqVRc1leIKQalFDFzjv1iTnI/5PuL/i+/87lyw7HkJ5dzn701yzpxv87GgDmbxVHGNyw5zdwfJZeX2orMb+byGOI98WzjLB3nz1zpsi/LPi37c8mzns6LHH6God0c75DL56OM+5j7lRcCRWdzDMll7EjOY+ImI/Oq+TvzqEu+d5m31mGmEj3nNscMFRcCOazNctc/99ciVE2GdBbfy7rP0p/p49wuJCdA8SxiMS6C3xd8nVcduQ/ncTIZLj6LPcmOBcmK1JQu+jncmuqO2cFhZOb9kerHvHkxfc71iJj4ysyH5QUle0VEka70jOS6FH1X5Mf9NPOeyKk9y3zK1dqMK7+EzBXenQfeHyyeycc4sWmxPPtetH32Oe+6nHm/RmTY88wbssyj5OEWi3ksF8pf5ijrdG71nNPN5+JZ79S5a+deuMt0fO4cx7icI5G4xiL1U5QFL/fci/6xl/1ymRgfScN5F1oDlP48z/GtbZ+1Y3FvOJfRvJzy3nN5TnU+35SzQ/VYP7zt4Oe4eBzPGkWc3yMG0VuPL3jfo+Vyxb70Pe+o15dM99ayftM6pOOH/+XXz7H8XTcs32sobKtG/I3VZ7hSR9io8cJS7Mineoe1HPGT8RmM8Dj5Gpf6hFd2ixt9hI0Kr+wGKzVmmOwHZodXboON6hGixK1b4862qKXDd9uX+HS8wlb1MMJj75sMVXVR4fea17h35LHcuRYf1vf4bLiEFAHX5oSTr2Cjgkq/AEMwWOsBW9XjR8fnMCJkj+XgNWyU6L3Bh80OJm1X7VyLVpGHlRdZIQrc2RUCBI6uwofNDlJE3NsWW93jJ4enuDA9bqoTPukucF11qJXD3dhmWOg3V3c4OlKFZfVcqqOGkT6nZUN5qwlG2idDvlEWd+MKF6sevdeopM/pW2VxdBWe1kfsbJOvMzz2uurweljhsurReYOtHvBmbHFddei8we2wwtPmgFf9hjy8KRZp5wy+s74vIJU6ew0ZxunTRsJ1c4JERO91bsP90BLkEgJX1Qm19Ph/Xn+EEAX+4Poleq+xt6QI7ILExgwzA/Z1TzDVbTUk2KbLUNLd0GCVIJR3CW63aQfYoMhTm7zCJ1thUw3ZEGMI1Ju+xWA1btYnAASNbrXFbbeCUR73XYOnmyNcgrMxtG8M5OlzQeKmPeFoKxzHClcNwTNPtsLL3QabdsCT9QGdnZR9lQwZzsh8YqM8BqezYa/SAv04Vqi1w+gVrFPQKuAyQfCcl7hedXhzanG96tBZQ3zVlP/gFU5DhetVh8NQoa1ILXddjdnD3CfY5OiI67quLPZ9jW0z4K5rsEkeaCMDTtagSkq/h6EiqFflM1zuzWGFprLY1CN5eL3CYDWsU9i0Q4ZOcltCFGiTd1mIiNNAsMJN05PystXYtjTmQ+JkM+ywrSxe320QATy9Okyxbq3GOqkHMwTy2NUIXmK76bLBwIZ4neLBAsDxVOPq4oQ392ts1j1MggcCwP1+lbnOWgcMvcF202URs2NXI0Zgu+7RDRWEIJggb3Ttb9fYXJ9glEc3VEnlljai+n2NzfWJ2jZQvodjg6Yd0XcV6sZOQmMAhlEjJJ5y31XQxsNahbq2GAcDmSCOmvPvDYKTaFYjKRPLCM3zz6oM16uMxzgYuFGham023oeOoH9V4zB0BqZ2mfPtnaI4rMkYt7cNxNpBVwRV950GvEB1OWTYHqvtur2BuRzgrUI4VlmRV21tNkjdKbnH2NisPXlik8dbGo9w1OSRNGkjZJQQK4cwKNp51wFSR/hOkQf23swFfKpAHNUUjigc9MSX5PiWKgIH6ocoI33vVPLsihRzVCUOXYDoNcTWQryqEHVErAr+XACFXzGB+JCdBKoI0RGvUlri0EWFpMSroVKcT2EFeSxDih2ZPJaBVWHrkD2roY6AF9lj6bdJKTcihwYRTsCvQvKSJg8Sq8LKCDmkvJoAfZcUbjXIIz4K8iDG5D0957GU5DUcr766Zal6AdeSpUCqsMmwD4Bv2TNKHkv2FLLokzoJQE5ePvYEzjyWBpNXjcWSYvLSAdlIjhIwe/LwstdXdRS2ZBbH0pEHGjGpwtbkdWYvtExj4Js4805xO7jfok5e2KXHsqK+B+hzFIDuZPayC0d1C4bu9016dyRPeRSTJ/qdHstIZZPHUmTPV+lVLQ0svs79MVPLZU9hagt7KufKsWnQxeTNXB6lV2/pBQXwdlXYpZHJZTLPlfNZGtmlh5Lb53DWY0l9neZMGaMT83xz3qLIo2jb7Cg3NpYbEjmv+ckHiq7nNjZQnDtTtymvR4zQr2KYfg1HTA6u39XjvfZY/t7f2sb//H/5l/CL8QlsUHhqDrBR4fNxizu7wrfbWxx8nXiPW3xc3+GT4QoA8HF9h9d2DRsVNmrAr/orfFDvcTuuoaXHE3PEtTni4Bv8+f5DfNzeo/MGnTfY6BE2KDyr9gCAn52eYGsIunihe/yyu8a32jcIUeDluMFGjzPvohYee9dgb2t8b/MKAHA7rtGqESZx1y5Vh5/3N+i8SfmSQbd3deLTkZFzVXXQwuNC9/hFdw0XFG6qE3auxh+uP8fn4xa34xrfbm/xYrjA0Vd4Wh3RqhFD0PjF8QYbM2AMCtfVCS/7DQBkT+p11aGWLnM734zEnWO+ZOcNbqoT7myLVlkMYYLN7m2NrRnwotviouqxVsSJPLgaV9UJd+MKz+oDXg9rrPSI22GFm/qE1/0arbb45uoOPzve4INmj3vboPcGH7U7rPWAnxyeklHpDCrlMHqNSjk0ykKJmDmWL/ptNg7HoBGiwE19zNDil90GPkj8K09+ASki/u8330CrLa7rE+7HBpXyeNOvZlyX5+0eSkS8HtbQwuNo6yQMJXBh+syd/LDdo/MGb4YVKkUc0pv6iFf9BldVh9thBV3wnLQM+KDZo1UjfnWieeqiwtFW+KAlyPKz5pBh1tz+zhnU2mXe4+enLRpt8bQ54kW3BQCs9YjvbV7i5bjBr49XuKw64lAKgj2fXJU4lh6919kQ5o0CGxSUCLiqOxxsnXioxC18M6xwWXWopMfrfo2P1/f41fEKaz1CioDeGwyeNiouqj6nPyTu6W5oMo9rZcbM9QxR4n5s8KQ54lW3wYfrHd70K7goMTiNTTXk8DwXdZ95ur0jg/Yb63vcjS12Q4Iua4taO2w0zUkfJGyQWBmLWhFfbT/W2eN6WdEzfdsTH3NTDdgNDaSIWJkRJ1tlPuPJVvjOxWsYEfBnbz7I3MFNgnsrSRgBozyetQdoEfDp6SK/yyrp4SKFDaoSF+/D9Q4/u7/Bdy5v8dnxguDcaXPkW9u7DLEfvMZl1eHT40VGAlw3tEH06rTGtiY49Mma/Gz+weVL/Pj+GcZk2K/NiM4ZKBHw0WqHf37/FD4KbBLX8Nn6gDd9i4tqwMFW2esdosC6GhNsvsJ10+FkK6zMiPuhwcaMedODNw0u6x61drjtVqi1y+MQokCTNi144+Cy6dFqi9fdCirN1auGDPI3fYub9jQLn9RolzddlAz4g6uX+LzbYj/WEABu2hO08PjV/gorQ6vZPqEUPlrv8PP7a2zrETfNEb/cXeNbF2/w6fEii8I9WR2zEBwA7Mea+JPp+e2dxlXTwUeJzhIf96Lu8eq0RpPeETaFRbpoiALwfH2Y0xs4zzQPrpoONtDGB1MQeAMJIFRG5wyetge86ja0sTBWaDTN6cErXFQDPj9s8L2bV+i9wWGsYZTP7x7mDx/HCjftCfdDg+umo3mfuNouSLw+rXHR9HjWHvDJ4RKttmi1RaNtfm+PXmE/1jAyYD9UuFnRXNwP9Gxd1D2kiHhx2KI1NlMVNtUIJQJuu1Xm9HI4K4D4d02iHxyHCk/WJ9pMTNzQprK4rHvEKHA/NNjWA14fV3iyPmE/1Gi0gw0SH6z2+OmbJ19ghXH+uGx73J5axCiwrsf8TFbK4/VxhVVl8/gyB9f5xPtekVZDZzXWlc30jdGpfK1EzVTKZ37tUHATAcB5hQ+3e9x29PvUjQaXbY/OGjhPauqMsOkSD3PTDDj0NdpURxeIUwsAp9HkeN4Acigv3uxyQaKtbP6sZYCSEYNTaAzRBbrR0IZWO2BMXM/aOAxWY7Aa62bEoatzOxh5URuC8Q8pZniMJHqXxQu1z2gT51QWEQSQqRExEgoksEp52VeFuCAjdUQhQCiAKa52EXebD6ZZeC+ztVYunRnNkUXYAjLyQcqQhf4m1fUpX7H07rJBVyARynMZ6VAqzHs5bUapErmAqa1B5PjSpap72Z6zCv9L8Z7y8xKFsGxHea5UFU9Ijgf35A5dpM3c0aJdX9Eu+8U/+k+/dq/ek+8/i//uP/nteyz/2Z/846+9bcB7blh+9Dev4z/8n//N7LXjwwaVoaouKAQIWignA8klMRwWcJEiYgwqL6JZPERLn8VT2NtVwl2rFNtx9DqL6VBeGk3yLHL5ugDKhygzJDeni9NWmESElh59Mio5X4bPljBNrpdEzF7IEpLI9/BimeCU0/ZTCRUVImbhE375yuKpFyKSuMkCQlkKf5RHFh1IXpTyXAlj5eu8yGHvVXmOvXFswHGac1DYsv4uyAcoEV58KxHzjzYvMk9pMVimLyGxQsQZHLc8X5YfI8GcWZSEPVPcPoK78o/N1G/sbeOFcikMUvZXmRdDbKcFB4XWof5j2Cuyp5G9XqqA2PklVK8Y8yU8dIJII6ssKxkR0o+0ViF73ZZhfaQMcMWYzvo3jdNyfmVF3dTecqMzLMYkRPFgnFmpmMUi+Pkox46vlbuIpUhG2ZZzEOiQxFgA8rZOeUwLkDx/0oKIwwcRHDfmz3kuyZhilZIgR3nw4qu8n2OBlm3icD2heM75fhL0QFJSXkJ75Qxiy3DcHEpo0U/n0k0w3Kn9nH4JEea2zEIfxTmkl06SGnSM4iw0lxc9rABNnFWRYbe84Az+DAyXVa2Lz7O+i3NBFCDNl8ViawbPFQX0lhedvNjjxdsCcjprT0SG385+qovzfK3si1m4IO4nLyBTzM649EAUZc8guouXZ/S8MJ3CJeV+ZIXZiGmxyvUQBXSW61WGPYrz+j5YVPLBH0sRklKldAm7LcVMuG7nxFS+zMF14rzKurIAzPJH55yHZ3ZdPBRpWR7nPBxc9tn0eLh4F5iXzZ8FMqeXobPlfypqDqslHmDMYXUATLBceaZMLuuB96sYo/K98lg/hMeb/U4vFt9UzrvlPcuxeSwvfDHbJg/D2/J9RznvOsop+JXzfOT62/N8dw+89f4vWIevnO7M8aP/4uuHwj75/rP47/yTv/dbz/d/+pP//mtvG/CeQ2FdlPjJ8Sm+0d5hCBqfdJeQIuJ5fcAH5oQ/232IlbbY2QbPavL0fLTawcDjF4drrM2IjR7wstvg25s3+PXpEjc1qS1+errAfddgVVl8//oFfrJ/Ql4Z5bEbG1TS45f7LUIU+P2LW9wOJOSzH2t8a3uHn++uszjOfmzn4j1O47LucVH1+OGr5xAi4rLtMXqFPoUE6QaD55eHDNncDzVaY7E248wAu+1WGJ1CPxp8cEke1MNQY12N+PWrK7SrAVdtj5e7DS7XVP+7roFNO4MfXe6wH8h7d981uFmfSAk1wQzvTm0uL0SBi7ZHbzWGpLJYaY/dscFm1aMfTVbCEyJiXY849DWebo7Y9Q16q2GUR1tZ7LqGdn4PBKPtRoPrVYdXh3VKX2O3X+HJ1QFv7rZYNSOayuKXr67grcLN9RHdaPKuKu++W0u7nBw25GJ7ghRAV4SAOZ5qVDWFArnenFApj5/85AMAwNOP79Fbjb6rUNUW3kus2wHOqxTSRWK3I9hzsxrzjqxNRiJBAZMK530DYQKq1QjvSPhl6Aya1YihN2jaMYvl0A6thN3XgBVQV6SoKmVAXTscdw0Fs98bVFcD7cyOCrIisZrgJYIno2e97XHsNMZjhWZL3qqx18CrGmHtUV/1GHtDu5cAhAqQhrZmOBQI77Jy6AheFPuOxFvCqEj8REXotYXrNGAl1NbC3xuoS5tCZADCUP7BScSThtxYhJOm8A5ewDQuG5C2T6+kFA5BrRz8roK6GOF3FUTrcigV2+spdMeJII6oqCwAiLcVYhug17Qb7gdFsMRRZHEaoUigJo6SxF1al+NyZpGYtQOCoLAbDWG34iAJspiEc0TtMb7cEPzvGz31rYiIHbWXeXPRSsi9JtjclZt2cZnTWIcclkHdaYSnI+zLNfyVo3ASKcSJem0I3qZIDEaeJMKVy/E+xSGFyLi0wFHT4rAJQJL6rz7XGJ87asNRQw4JLhcEzE5gfObJgOgl4tZB3hmEjYc8KoTWTwtHGSGSGE9sPeRBIdYRshPwmwB5kiRWo4CYOKjyqCCtgNt6yJGEa6KhHWwxUHgICidB7VKjgNsEKqMK0PeKhGS2Hnqn4NdhEqEZWRyFxG+qTxXCRYRf0Xd9FCQQ88RDdtTnBBMUqF9LuOd0Xh8E3FMP81LBXwUSLFER6l6S/ZLCKmAVKTahF1k0Rx2oTcHQ0KpOwG4j1IAEKUywwKNAqABzT+PNsDbXUp04VqI5JkiliAgVQeCCIhgk3xdMRLUXGLcR0gu4JqbA9wSNNAeB4TqifSEQNeAbkECOJpibtAQv9E2EPgj4BjBHGmM1JFEWCYwXBC01B8Bu6T5pCSZo15Q2qkmoxzeAJjQ+fAsIR+I5IgLDFYm5MMxTDdQ2zicYytutkEVb1EDp3Qqo7whiGPR0TZ/S2mBF0NJxC1RJ4IbzrO4iuudf3bA0B2C8ACCoDBFAAjKOznM55kj/o0AWGKp2EVEJ+BpQXTqvqA1mT8+ArwqIp03CMABCEjHKwek10LyOsBuai6EC9DEiVGIm0iJchK/TRnQX4RsBNUQETc+KGmKad2ImiiPHiGAE1BjTHBPQp5gFhxgK62sB1afN1obmpDnEDP+UlurmawF9jHBrqq/wMUMuGWI7g8ImESLqB2SoadCUHws/SQ9IRxDLc8JFEJN4D8c0FVHkNNxXQZOQkU91yHkkcR5pqc/ykYzEmXiPiwmSKyB9zPVjHwGPSUjRkx7ATON0HkCGvJaGdAnX5bqzyBKfz/aemMqNcuqHB9BWTs4Gaoh556sUIopFfvTuoTTvNOiL/jrHOZ2VfcZIpjo9zONh/SeobYbcLo4fnW/6Xx+/xeO9Niw7Z/DT3Q0+PV1g9CrBMAR+WV2hToYSe0g+1Rc4DQYvj8SP64aKYgaKiNEpvOlbHPsKn+qLrLhpR42DDOisRjdUeCG3sx3scaDu60eDwRJ8xY4a+75G35E82t2pJQXNUiwiCLwxK2jtcTw0EAKZd8US28FJfOYvsrAGQzlY+pt3u52jBaQfJX5t6W3lncK9auFOGgcn0XcVbGdgR5J7dzYZDDLiF5YUMQUAbyW6rqIFDIsaWTUjyvddRbEB0/2sHHk3KgQn0RWB1E+6QbASQ2/gncy73TsVEVJZvtcYB4NgJY6nGmFUUxmdxsuwRegVbK/pvsSRusUawUmcFrAQpIU374q+cQTtZb4SAMRRkgETBYbeUDsSd+f17YaMglHCGVLvHDuTICfp/p4MjlNSDe1VJGNAAhglvEoxAAeJaAUG2wBewOkAWIkuxdU79TR2I+/WBgExSOIZ7Wj+eBFhdQUMEl5SnuN9TWm9gO8VfCGDLoLAwUqKSWYlek8wUDgB5UgVc7hrIKyclNmkgmdRjwB4SeVSTDCBUEBxhBUIiVfEMbGcryCsAIKAjwayl/DC0DkAUFRHEQRxp6KBHCQpgEbAJgPUS+R7OOi3t5LyQ0VGijMIMiKktD5xl4SlusZR5h9lOZD33SUjQDiqs3ACYWfoh1bSAl6mRXx0ZlrsjFSH6E1aNAiKscefB0kcJUvlsrER76s8/6QViL5K4wvI1AciEncu/3jyD/UoIVkN1FI9pROIB+KTcXpa+AhAJGVPKxCTwRrlpI7phSZemkhxI/OCS0AeFWKvoAbiOk1cMUFKpYLq4KOBHAQARRw2r/KCDykNIhA85RVSzDocJbU/cbQgp3YJD6gTzXUIgTiADGsnKP5lAKKNqTxKiwDEUVHeAlAHRYvLo8xGGatMhlFl/pawgD5N44YIqKNMyq4ic7VEcV4E+ixi6ksBRCEmgzIt8FnJlOMR8mJbBOYR0Tk1pDkjgOjTnLPToixiWuypUeRFNE1KXmwLoloGEFSPn1+eZ2kuikhcOlauZQ+T6pE9ellRs1xkBmSuHC/uBRI/LCEAee5m5VKRDCNeEHLIT+4ji2lx66Y0mTdHQzBx0wqOmMjzi+cs8oKU+YnUPuQ4jaFY0WSDIZXHoCBfi0cX1l/kiMzB4+8CFO6i5DWGef1ioN/soEXuj9xXnJ6NljTFqG9Fzqcsjz6QAZ+fxUj5l0ZFFADUlC9fD5xvOscL8LJfYrovqKnOZLjx95j5hFmxltOZOR+RPwczpSODmd7fwfC5NIbpcxQCAnFS15XTOHK/hPRs8vXc7iLNg/8RExKAeZmiKFuV48d9Oh+LbARBFM51kfskpAVDlJNBOuN1sgFVjm9pqHIZYjKYOL8oRf7P6aOYzpdjwfmWfc7P4PKYjDruQJGMX3E2Ha0L5l7fd3pOyzrlDM/X53zd3lYG1xtIeF/gryAqM2KOjvpdO95rw/JJdcR/+O0/RR81KN4jxbBjwZtToMV59oYkAR0pYuYMlp5Evu4hM4QVAFo5YghmpjIbIDIUkiG2BK1UM06iFCGfX5YnRcgcSiMCbPHWCknVlQ8uyyQ12wnKqSBFgEsQVf7MUFqOj8iCOiz0wn1SxoDk/9w+rm/5AJTqtksYKtebD4YSc4xHvs79eO6/iyrDhik+osswW24Hq+kuH8zyewkhXUJmy885fuAHUzzEUvwHQIYJZpXXBBfmvmIIcdnPDNctYcdcr5KvyX3K361XiACMDLNyGRpbKg4v4b9lfqUQD6dl3plJsGieV1LEHCaGNwPLo3wtL8c8xglmzfH92INcQnb5Xm4//39M0ZbzVwV01geZeWdlX/JnPpaQ4DJ/n9RVOUZgVlvmBXrR+BIyzJDfEh7LarIMtWWo+Tbx70rV2RI+y+k2RVu4X0tIqvUKW2MxpviTJUR4qbobFnMtFue4L8v0zimsUww+X8BeAeIptSm2Hdffe5nhqwxPncrm/Kff8HOKx5w/x8Gr5KT2uzz4flbONQVsm8swgvLSspw/9F+lulgW80n35/HXU9/zzBmTAm+MwJjG17KyLei5YAXrUmVWFu0E0ru6nNNp46s8LwDY9N35aeGeIa6FeyArz4qpj4UAxlKtVsQZ/HXGyeJrKmK0pFj8AB7KdROgTbgSYszwTgCCQ944mUO60KCKuRJoCQct4ZEMW43IYV5yfbk9zKXiPighoqV7g/stt2Fxne8tobVIdVDT+HzpoxzfvABPf6WRUIiocBsz3DRttFEWcXZtBl8tX8jlef7uBKDCvK/LPsE8j0mwaeqnWT2KMRRBPGgq/eSVVgTYusr50b1YpEGCzaa2l3UrynqIcY1TtmV/FGMuyqQPb33Yd4uhX7bvQQgNfvbCfFqW905lLuYfSsN0UdY5A4l3FB60ozjBOz7n/mORtmjj1K1n4L+5nMfPvbXdZdpy7v0Gx29srP4VP8q18u/a8V4blrVw+KPmV3jpLuAhYQQJfhxDjT4a/J56hWOoUQmHY6ixlgPuPEFWt7JHHw0ZcMLh1m1wqU/ZGG2Ew0oO6KPBK7vFpb6FjSobQEMwuE4Ks2/cGis5wkeJlRrwxq1xqUiw4BQqSESo4k0rRcDJ17BR4TJhd4ZgcjopAozwuHerZERKNNLCRoU+mGw4hyhgpIcRPqvbhihQS5fzPvkaQ9DYqAGnUOXPSgTYqHDvSPnWhYl3Sn3pc17lcQoVjKAyPWRO03mTy6U2kvFuhMfR16iky3kOwaCWFkMwaNWIzlfQKT4o18GIgLUecGdbbNSALo1LK0ncZeeabGRLRNgoYUTIRizX7+hqqqNys40AFxV8FBQ7NCh81NxDiohP+wvU0qe20IPP9ckbDcqSaI+rUzsVTBpfNqYBYK0HhChT3NOYub0HVyXF3HpmVAMk0iRFwN6Rp9FH4umu9Zh5w3xfLT2GoLLxrpLxfW8bNMqhVRZ7V2fO7pXp0PkKe0fjoYo39eB5I4TqyYazlh5j4QZg3jKX56NA7w0alYRQkrpvKTLlosyhZ5jHzP9ZPIjms0SVRHQ4pM7oFVba4mBrbMyA3uvMSSaBH5Hv400T7n9Oz/WvpMt1ODl69nkzpVKJH5n40mXIoUnYyOW8tPBwUeV+GoPCTU3P8u2wysY3iyFlzrKIWGmCOXO+JSeY48eGKLAxA+6HFpdJMMkXxiQrFXMbKulwtHU25mrtckzbOrXNJng4AFxUPXZjAxsUWm1nmwUrPeJ+ILi3kgGdM1ibMQswLTfJeAOM82Lo/OA1PXdh4pRzep3y5c0Zl561kq/tgsyiSjw/+VkGgN4Z1ElAquQwlxzsTTWQGrAnREmtksiIIyi/TBswALAyYxa1YQVnFjWiDSWBlbGzDSnrVZ7bUhBnuyl44ADxpjnmLnO+6d0d4KPI6fN7M6kx85jUaRON68DPHSs5+2JTpxT30TIggjYOtAwkeFUPiFFMBjLma27rVRZu4zbqot6j09DKo05tYlEqfo51ahO33XmK30t9JSEFoBMlYbAamkPvBAGjwiwdc8eZr808a6T21UklmXnXHPOXYwLLtCFnlJ/xuo32WczmqxxGeYxJiIrrw5tCY+oTfqYjpg2mGAGtQt70KTnby2t8cBoAOeRQyQ02hYr0kqvNRxkmibnV/J5Y8phLZBVvJE2bafPQRWWYmXy+2LjhDScK40ObaTKVn+sGdi5NHPdsMxZhd2b2QsTE5y3aiHyPeGCVnCVHuvUAACAASURBVBOiyRth5YbD8qEoypwb62J+rWxQmR5A5uQu83vbUeb/wNj+isdyo+LcdbAhPOfZLtPMNnPK7+8q/wvX9e3tfaeR+9fH13q814blJ8Ml/rMf/v0cMsCll5dWAVp5nAYyRiLox5Ff+lLErD4G0A49K5fpFKDce5ljua3bEf1osugEv5hdgp5WFcnjU7B0BVM5Cg4fKeA6i2SULzKZQgWM47SgL39kYhBQOuTdcoamTrHLkM/HIKb0aZdfygg3KEgdMo9MGroevMiwVJWClkNEhMSvE0A+F72YiUlIHREDsvCCkNN9nCfvEPOut9RU5ux8UjELdrpX6JA5fTEIxEFBNo5gbSnIeubC1T7DUx/ET+MfCxEhmG9XQlkZtgoQf1BEhCTdL1aOoLCJq4YICglQvrzGdHMKKA8Vp91zx/eJKVg778zLSDv9nF7lN/m0i+8SpFPH6Y2pInEOBQgqWKUdav5x5T7Pkz3Vm0MIgNLKXqZQA8RXo13iOO0A8/1iUafyN85NQdWRAk5HnfhgMX0eE0+OeYUyZm+FcALRxCmfOMnpz3btGTKkE8zTUL7QMe2sT3XherHHgKFAspeIJk5p0hwUHohVLHbRxdyrwm32DPubyuDP5MWJk6dFReLsAQhtmH6kfXFP+iGWCTYcasb/oZCDn54fOUiEJkD2kniAhTdB9TKHqGAeTijWyizPX3L1Sqgey/5HmWC1foJQSVuEBEj5qoG4erM+QBozB1q8aQo7EPUUyoIhzRBxgii7xEc0U1vzHGPPV5hCHTD/SUTkcwAQdIR0IgUfp/k6cX9oXFVP9Z/BLyOIq8hzLC2K1Ji4h57gl74h+CgHfc+wTH404xSGIwuKSEoTBQrYaeo7rpuY+i0qgr6WR9BxBpXj9uY+TPOcYavcXmmpHER+Rqf2yRRy4pjCXWQoXGo7e2qiojFkfiNf43qHivpQWuoX5muJOEFPARA0MtD/IeUTDF3nvUpfTZ+jBPrEJct8uASLZZgltYP7CAgcriP1p/eAS2EiOFSJr4A40nef2uFHIDT4yod1xbOWyvNIdTVUjygBWGTYJr9j45jqqwA4GkeR5kpkyC/bXZH6MBt6S/iiAEQPyKqYe0XYjRmUOsE9pQVE6huumyhhxDwfijnB8PIo07POvxc859WUB+cpLSYoKcOR07MRDPK7kH/iZmWEaUxzU8N0D0NVy3PECUSGxE5GUlovyem3TDA8ZWkUianNZd2m36apTeUxM3JKL/Xi/pkxlo3OIt2yTstzi/Ie/A9T/z1mGGeb+xFj7Fzej9XjbP2LfN55LNtavI++qNfzHIfyXcfPvvQdfwlH/N0ON/JeG5Yf1Hv8x9/7X9FHgzGSJ8ODdzcT1A+0q8//2dtohIcv3hI2KhjhyStZPJXsOfFRzryO/J1370vPQxmzkvM+p5h6Dg7J/z3kLA/2DnLaZV5vg6QyxJTqLVI4Dp+9duy5mtpK+fDOOMNslwd7e84pspZemBBF9iwuz5X9bKPMEGAfp/Hi9pSfy/w5P/JWzvvJFZDWnHbx1uV2L+u87NPl8bY0b4Op8nUXVa4bjxHXpQxPU35+rB7Lsh7Oh4dz57G0b8u7rP+kDjsp1S7hj3zEYrzPta1s37n2nhsb9hQ9Vu/H1GUf66dzsNxzSrnL/JbntQzv7M/l8bYxWNj2+dwyv8fG9lx9H4MgP3Ys+3NZ9tvmwvL/Fz3YY1X+/6L15TSP1ftcX5X9zJ9DFKgKDxF7f8Qj8+5d7V16esrPfL9+x7icVW/G3KNUwnP5swT14bJM7gNuE7//XV4YixyaQYh0vlh1xsLD9q6xzkVymqL+EfN8sfy+yG9cDsHi+6P1iHigxPtljrNz71x259r/tmK/ykJzaUQsJzHnu5yvcTK0zhoz5f2Pnfsq9T1XP2Da5OTvZ8bsbJ3KNp5NwPnFeTe8w2A6+3h/gTF+pyftSxhNj5X92O1vHY1H+v2xMnKeXyLtFzME351myu+LzK9HYL1/fXytx3ttWJ5Chf+3+yYOvsYYdIaqMcSRYXtsYGgRMnSwVi7HBmO4H0PaGIrHhsrJmczpKxdolfQIEOidyRDMSvkcWxHALBQJMBlIDKNb6ZGMjGygkkeVYqKZBxCx5eIzw5XEFJ6E69coizHoHBqFeXAcvzBEioGnRJi4fFwPUJ+VPE8AOdwI3xMgoETIHEM2FgJEhlMtDQDuQx9k5uSJxXcppnAjzNcLcYofNyTv8zLsyHLBxzC3coFd1skmvtu6IojtYaxmnEMAM1gaQPxHPs958qGKxRJD38r7VTKISt5kaQToAurFeZMX3s/4lgAytGsZksQHkcONOK8QIvEEeQ7YxD0sy2Y+YRlCpAwpwke50OdFG5cnRIR1KsOzyvt5Ma5kzDHWQtHGKf9kaHM/JDXeUpWXuaC88cH9znMo939Kz3xEkfKXRR/yIpzL9ouxLDeAyv6aoGQit6XiuWl1XryTITLlKQQe5YWWEDOGxpX9WRqDWoU8N/j+EmLG8DbvH3JBAcBoD5vixEkZZ2FBlApwKQadlCErFzMSgutZtonbSeFGZP5eps+Gjoz5emnkMJSP4XeUX5y1LUZAKW4bXfd+3r8lvM1UjhSUQ1k2qR+zscRplSZhM+Zkeqcodl0K9RKLeVkKsTF8T/B7SJLL5VwYE84nG1ZRQCo/W0SFKGYIiwzzw1TXEsLI50mYLQk1LfqWkSUqKfOWfFWOacfQQyFBYjPs6QcIcRJJmIlF20j5uLjfpy8RiIWyikjIoBLlAgDBJ2lMgclTDYCUuWI2iLLhWloGiTOaxy8tnJchVoKThDgpeJZCBYThTKT7L3qoSMgTYEIt8MLdT+XMQmjE4t6FYZ6vy6ltZ60EXkZweVEQz5WRMRHkJfeL/AHMlYSKOi7LLQ3Vsl1cfiEUNJvQ7OVjvmbJtxWYoVdEWd+i3pxmqjPOW2lRnPfK5XuK7yl/wfPj3J7F0gA/t5nDdS8hrY8YNGLZPwIofASPGolldz5qUpXtWFqYyz5dHuHR5k31fqzML3P+XXkux/QLGJDltP1dOCIedxT8LhzvtWFJC/WAz/oLuCDxzdUdAODT7hK33Qp/cPUSe1vjuurwotviO5tb/OJwDSUDvrt5jc/dBgdb42lzwK/2V/jm9g4vuw1q5fC83eOjZoc72+JHb57h2xdvcLA1emewMmQMfrTaAQB+ub/CtqKwDit9xIvTBr9/cQsA+Kxf4arpoEXypoF2pI9jhf1Q428/+RQBAi+6LVZ6xEqPUCLixhzxw/0HOFrCYT1pjuitxn6siUuXFjFP2wO0DHhaH/Hj3TNYr3DTHPFmWOFvX32Cn59u8OK0xR9efI4f7Z7jZCt8uN5hqwd03uAvjk+xrYd03wlvegr83GiH3mk8Xx+w0iN6R0brfqyxMSM2ZsAYNI62wrPmhJf9Bmsz4OSqbBx1tsXajHh9vMBFM2BTDeicQWcNblq654PNHocUzHvf17hZn3DfNWgri3/h6hV+ePscT9sDbvs1OktBwJ81B/xfr76BbYrLyAaBkgGttqikR5VEeH61v4IPIgejjlHgpj3hrm9hitAr/9rznwMA/rdffw+resxlmhT0mj0VWgV8tKZxf3HaQoAMiVVlKWj56ojbbgUtA37/4jX2tsZnxwuszYjdWOOmPeHz4waXKWg6K/5W2kPJgG9u7/C0OuLP7z9AiALWK+z7OvVTjW9vb/Hju2cwkgKN74YGfdrQqLWDkR6f7C6gpMMHqwN+ubtEjAIrM+KPn/4cPz0+wU/vbrCuRqwNGdNHW+E4VhAi5nEfU504YD3DyJ+ujrjraVzXZkDvDV4cNniyOmKlR3xyuMR3L1/jL+6e4rLpAVBs0G40qI3Ds9URnx22eLo64k3fQomIXV9nHtS6Jv7h2owIEHh1WOOj7R6/vLvCty7u8eK0wWA1rFdYNQNOQ4UQBS7bHkZ5nKzJEPjvXb/Gy25D4wegrSy29YDr+oSf3j2BD9S/tXF5Y+Gua6BkhA8Cz9ZHxCjw2X4LJQOFxzlSMPKLZsCuJ97jZdvjvmvwLz7/BFoE/O+//k4ObbNtBrw+rMh4ExGV9vjWxRtUyuPHt09zLNFa03w9jSZzx757/Rp//vkH+P6zF/jx7VP0o8nw/e8/e4G7oYWPEidr8KQ94WdvrvOmyUdbCj30q7tLXK8p8PpxqFBp4hj+yUc/x//x6e9hsBpX6w6XaS4Z5fGHl5/jT198C84rXLY9Xu3X+Phqh892WzzdHEnpmjcFgsDFqoeRAXddg2ebI/ZDjeumw2f7LW5SEHubVLsB4OnmiMuqxy93l1hXFqNXGCxtCK5roh3UxuVQRZtqwCe7C2gV0FuNj7Z7BAh8ttvi44sdPj9sMq9zVY9wqSyjPP6Nb/x/7L1JrC1beib0rS6a3Z1zz23yvedMO03ZZUOVpepAqgLKBWIEJWaFGIDEBJiboikxQ2LGACFREgZmCBWIQYkBglnJjJDKEr3KOJ1OZ/O625xzdhfN6hj861+xIs4+99738j2nX8pLuvfsHbFitRGx17++//++P8Dv3X8Lr84rKBnx0WaPRlv8P59/gF1L9+exr6FkwK8//Rz/52cf4XrV4Re3t/i/Pv8Qv/HiE/yj1y9oUy5I/OKTW/ggsa16SBHx6WmH41ChNg5GBpxGg29tjvBB4n5ooGXAi9UBf3T/BNt6hAsSg9PorcbT9RmHocKfefIaLsjssfCqW+PQ1zkW8cPtAYOnDcLTWFHMptX4zu4eADAGheNY47u71/jB/ilq7fD6tMKuGfLz99Fmj++9foa/+gs/wMHV+Oy8RastuhSnujED7scWb85tfs6+s72DFgEvmgNeDRv0XuP7t0/xwfaAP7v7HP/Hm1/A0+aEK9NjrQf88HSDXdWh9wafnnZotcWr8yq389PTFq2x+M7mFkpE/N+vPsRNe4YQEcexxrP2hEZb/OD+BjftGcexxqYa0vNH7O1XbQ8lA14d1/jlmzc4jDVOY4XBajxZdfj2htYAPzw8wUebe/z+m2f41ZtX+Ph4heumw8lW+I0nH+N3fvJnvvS649tX9/jB7RPEKPBk1eXNzrUe8Ye3N3iy6nAaK3y43WM/NHBBorMaMdIzfXYV3nQrvFgf0TmT34/fvX4DFxXu+jaH7GxTnDgA3PUtSs3ezmr8hecf4w/2z2gMhxrf2d3iTb/OcmFSRNTa4a6jaz/YHPDpcZvHd/QKNy1JjL08bbKElxAR23rEaaywSeuD82jwfH3C4DV6p1Epj1Zb3PYtbtozfJR4fVohRoFfuLrHyVaIUWBX97jrW9x3DT7YHvDJYQuANsUrTZvGN+0ZLkrcdQ1aQ3wI56HKm6aNcdCK4vrPo8G5r1BVNO619qhSnHI/mvwu4E3EGAW6oYIxLsusSRkoHjhIWKegVMAwaNS1wzBoGONnJGXeS6ybMSkPIL37Js8M3nQzxuVQKqUozKbSDl36TeJNNedUvo7jjPNmbgq3ipHer5rDdRZxslr7XI5zEnVNIVhahxyPy5t0fI2zGlL5vBnI9dLfKR5WJj1i3mCTuY3I/QgpllmANs+mTa+0Acqb7MsN+DjpLHP+0h4WRRmXvnMZEPFR2/qbkH6eXWFF/BNIxfu+6Tt/fhf/7b/317FVPc6hwhu3hkLAM3OEER4/Hp/ACI+jr7FRA/auwU73sFHh4BpU0qGWDgfX4MacsHcN6kQIsndtJjn5sNnj1bDBWg8wwmNIARZ7R0jpB80+s7sebIOb6ow720KKgJ0ecEpBOiWTa6ssaunwcbfLhB6MLoYocHYVnjUnag8ETq5CJT1aZfMNGSBwsERO03uDZ80RIcpMdPLJmX7cd6bH636NtRlQKY+zq3I9L9oDzqnso6ux0cMMbTzaeoY4cjuZcKVSHidbZdKOEoFc6REnW+NJc8bZVRgc6Vg2yuIwNthWPe7HFmtNRkQlXT4+Bo27vsV102E/NFiZEZX0uBtaDF7haXue6WvKNK7WExLMY72phkz+wa7Mx5E0QV2Q2FYDtPD4wd0NAODF5ggbVG6rDzJriTJxyXFMBE9pQVEiWaexygQfx4GMNf6hNDKgswatseidzvn4+hAFDn0N5yV2qz4jlJXyyfgKOA0Vdm0PH2Q2/jJSF4nddFuP6J1GNxpsmoGMX6dxf2xQ1w6bZsDoFFyBfPKPGzOpMhrs+YcpGUCsRerSj6eUEW1l0VtNTKPNiGNXY9UME7IqIrQK8EFgsBpNZbPx4IJEY1z+cRsTOkSIJ1AbyruqLU59BZMMcCUjBkdkIUJEDNakH2EiholR4NRVMMajqUjH0nqS1nFOom1s7jv3JQQJk9oiBMU/CwFUxiECGEeNuqLPNsVjCxFhrYYxDqdDgxgEtlddvh+sVfka/qEf+goxAHVr87uM0TtGBoWIGPoKq3WP06FB3drEWErldqeKJCcASBXgRoVmNWZSkGEgAb2mHUkWSUSYFO8NAMO+RrUdobUnuR9PCBAiaZVW2zHFkWvUzYi+qyh2fNTQxudFgRDUxxgontwOGjLFTuvawVuVWU6ZcdWNGtEL6NonBIwWOSJpp/KiRWkPbxXCqKBaR/HaKXYcAHTt4QYFVYW86AhWZn3SGATifYXYeIq1jkAcSXNTbS2CpcWbTHHm8aQht5Y0WgdJn/cGYuUzshY7NSE9EYCJEztqSAhSr2iFk44LKxEbT8gSAOjUvl4BJkCcKT8jItEEipNmtt1xiqfNW/eKNEYpAyg2MsXkIlAdSPIxUCSnFFcecq8pns9EwCPHw3LsbDQxxfRSbK+IoBjnFPcZWjouBwHfRorPTTGzvo5ZUiVUdC5UVB6Q4n0DSEIGgFuFLFUTVSrHA6FJcdUpRjfHIosivraKJImT2gVJ7STtTpLhoJjZCNUnfcYUI6fPAnb35fVG5JhikCNy+wWofJ/bTvVn+Ys0VapHltSQFikmHJNEkIg5nhgRFBcuyI2TJV4y0CMBfaK2ZB3GQVCMbil94qcYXzlSrK1M4xsFZmNaMqJS/HIacwkEFaEGkcebJHaoXSwrw7GnqscshjNqbh/F6VKFmKRlONaWdRnFNG4iUhmEiCJrweZjfD7lnyFbxVjNUK841R0FshZkqXW5tFg4fpXTEnQWcRrzHOsYp7LL9jBYxXXnY+9CO/EQdCY0FbO45Fn5RZnL2NtleoAMPoaA8vcSwX5Q1vzgA13JS1DtW9Klfj9IDyzUh1n+9//i3/ndGONfeXttX2968usv4t/4r//WV17u3/9n/u7PvG/AN9yw/ODP3cR/47/957B3LTGQpti8MWh0iZ2SGTtLxtEAgVZZ2gWOEo2y2I8tIXNpd7BRDrVyGLzG0dVolE3GmJy5vUpE7G2T2Q1Z5oMZKHtvZhIe7Kg3BpV3vkNqM7NwsrsmG4AAshvjMn6Jd0u1IIMxRmIHHLzG1pCBZj2xNfZez1ggXZDonMkuvUoQkyMbNOzuW8apsKuwkeSaye6y7CJbxi+5IFFJMjxN2j1ltkQ2QGvtMCSGPTa+Bq9gZECtHTprUGtHRkFCUo30OI71xZjOkqXPB4mhMG5yfGmSEVAionfU36crQqdu025xncab+wFMLryNdtnFFZhkL2K6J8ZkWDeaGDFtMsyZnXFMBjgzC3ISImZjc2JKFXCJXMonl+bB6WTsTcYfu6YyMZVMu8GDTSymKmBdjVnvlY1HrmPpCssGSukay+VwXqOofucljPa5bibCkskAdF5ml2V2wdQqZHdZVzBUMtrNbrSDU6i1z3+dl9kIZwbKmD6zZi2/0Tg/b3Tw/cFGaenKqop7lu/3kvmzdDXm54OfRZbjWDcjGbR9lQ0vdivl71LGbAyX85/j27K7KfL8NRVJjrDBCQC1cdklmGVFrJvce9noHUcNrUPa/Z6ub+sR3VDlHXHugxBY7LBT+40hgjKtA5ybVldcF7dd65B36zl/uXsOkPEsZYS16oGrLO+ss3FJ/UCuk9sLIJfvvcyujiU7JURE21iMVs/qFoI0iGUy1DmvMWRkSxVgjMfQG1S1hR113lXn63kcnZOIhSRJDBLaeBrv5PrJu/5lHnaBpc0MP4939POQBzbImfCBnxVGr6lMMbnyinkcaOR71CpUjSOkoWAGBSbEwXtJjJ5ekYsuMHMv9k5CqpjZPQUAxfOf+hiDzBsGwQtIdr9NBGe8QcJlUfuRNxj4eOSFP0t0JPdXLkuaJH0SRHb3lYloLaaNkmAl5XMyu6ZKE+D7L++sJY1HsIqMBjVZeqzpzJsaQgdqO29EABA6uUn7hStvQNr8mJPlZXdtAJmYTkzfRe1JW5fP8ybHEsbhMpk4rtA+ziRyBcEedRRz99cgCldeTHkKortZPSUhWsDk7lzKqpRGAIrPEXPXWIEcg5nJ1kqitdIou2hwJISLZU3K4WFD7F2ssGxMXzJs2HBbGHLAZPRNFRZtXFqnwFRB+bE4Vh5/29+HY/BI38rzF9Jj2d8bGuR7f9HGXP77lLNsd2mUfkET5nv/4c/esLz+9RfxN/+rf+UrL/d//Gf/859534BvuCusDQr/6PAtPK3PGILCJ90OAHBddbgyHX50eoJKenTO4Kru8P3DUzxtTnBB4ePjFWrt0GqLz89bvFgd8Ml5h201YHAan523OPY1Ku3xS1dv8PHxCq2xRNhjSUrhx4cWPgj8wm6PNz25yO37Gt/aHPGj/RMIEXHV9LizJhshMi0o19WIRlv83qsX0DKgqSycVxgdIW7jqPFke8507qfkblWp6Q0VosDnxw3FSqb8SHlXlcUPPnuKurFY1SP+8HCDzWqAVgGf7bd5Ufd0e8J910CrgPNQEcIlIsahglEBL/sNnJN5MbBpB/RWwyaDpTIOXVehacfsZsJxXZX2GJ3Ctk1ug2nh2lYWr48rrGqLz+62WLcD+tFg0w64Pa6waQechgrnU4PtpsPnt1tUtYNRHp+92SFYie1Vh2HUacGrUmyXgLOqcAEB2tWQUSUpA4QA+t7AGKJDX7c0Jr/3Bx8BEdi9OGK0GmOvoStinq0bm91CQhD4/LSDEJHOR6CqPGxa0NlB58WSOxlAB5jG5QWbGzR07eBGDV05xCAzNXsIEi+PBvACcktolpC04L19vSEk6KyhtpZiu0YFWfncLmbwrdcjnNXwvYJZUTluVHh1R+hNRmt4QSIBWXkgYmIGTotCZunNMYCDgqjSYiYt1uTKIXSaWFRXDvGoITaOGHwjLXaY+Tf2CqJ1hNZUtJBSGbmihRk93LQIla3D4WQg1xaHgwFanxGp0CtajAlQeWlhJRKj7vHeINYBckUoZBypzWIUiBsuJ1CdPBZVmBYYA8WQofFUtpWpzaDPqW6MEqg9hh9sgSAQno95EY1BQrR+WkA6AXlWhG7s/FQX5zdTzFR/VAhPLMY3G/itTygWNXO810m4nlhvZS8Rtj4vuERHYx83DmOnCPEqUDD3Zgt74wEdMHYqo1IiCIxnAfvEE5IwCsSVhztohDbAdpJYbzmJCJuYmmMdYM/ExCtHAb8KsL0kplsBQskAuLOEdHSekSkoWjjKka5HYj/2ZwlhBfwm0BjpiHikefHrgHiS8E2YYrwSuhLTWI2vNnCriNDQ99gTKuauQmZVzWylBwF/HRAHCXGmz/5uhbAJE/J0TG5bIS1w65jQP5HRm0AetogmrXl7AawisYAC9KurIkQvIAyAY4qH5IVTFYGEsEEAskti6AKQCbWCBORZ5Oc3qAh5FpCrpBFYAcIis6zqTiBuI9QbASkJ0ROJ1RmJqTVUgDAR6iwQ6gjVUflqpDGKAhBrOq47wK0BkRhipQdcM6FwzBgbaiApb8HXVI8aqA92C8gRmVFWWWqLT+UwcuSbos9DKqsBzJHylIyjqqe8vqF67Bowp1TmSPNjDhH9zaPL5Xcm3QGOVMsIIU3GhnA0Jtx21dPYA8jMtvqU2lxNqB4jmOY0fZ5QQ2SUjNlUeWEeFVDda7gVGWBR01gzC/IMMdN0jeppTuQ41U1zEOEbYlhmWgVpaWwz06wCVB8RlcjIl/ARwQioke7PYOjdpbuIwAyxiUU3GAHVUz3AdH0UdD0E4I2A9PSeCEpM96dPLMf8jBkB4WJGKzNimfqUUbkwjZ30Rf94XAqjNHJfmeWZEd+IzPbM81kaR6XBVJY5IcYRQU99BrBAUIt7MU71AciI5qy+EBGlmP5G+itdRFA0frF8n6RyIWhMqbx40SDLbSqM7DzXiz5zP0oUfZaWxxbtWdrTD9pQXE9tisiB4/nEF7QqAXzvC1/xp+mLpm+0Yfktc4//4Dv/E/po0AeTtBk1JALOsYa5cbBRz9hfPQRs1GhE0oWMBhJ0XSMtTqFGI8ZZPYfQonlm0/XE1mqjSjqXFWxUuTwpYtZpBJI+ZXqbKITieg0PgeYDO2OyLfOyziaApMsoZuVxUoiQSZeSU4gSq18a0UcN0ur06JOfCmtlhijzuLGGpo0qM96WTLglO26AmLHq8nXE7KryNSVzbkkCxN/5L2tzMpsu94O1KJkdl8eJ3JH1oyy00xgk7bdCl5M/c11cZv2Rg0LAOVDMHmtyKoQ85jNktGhnOW+EBKtZ/qUQLsXZ+hlTLydyxZY5sJvZcVs1Jl3QkPoeMgrL5Zesufx9DDoTUXHZJ1/NyKkAZPZcJrriOef2MtpbEjRx/2yUiWk4ZA8BPsZsv5y4/exdwO1dspjy/cnkWqydCSBrSS7ZZfN4p7Fj8q7cn9RGKcKD4yUzMCctQnYLlwViWRJAzdtM5bDXAx8LRT+lCJk8i1Hpsgy+d9gzYgwaza9ajH7+quYYYpo7lYnLuB9lTDePVUkW0CiL3pM+Y+lRweOWzyWEnZluL7ENl4RTJSPukiG3nOP5uNDf0lOCvy+ZZtmzgonDlv1aziPraLI2Jl87hqlP3C+dPDaUDPReSvGHpYZm6REBzAmYyjzM8db0aQAAIABJREFUSMt5+R5avq9CJA+TWIwFI9GXTJ8lELVk/bVFPVx3SZbGc5VBsOK9uUTjLzHfsh4k3xd8Lbvj87Xsls/yX3TtROAVo4AOc01GxYRTYd7z5VgCU1hJbvdivGTKqwq9SD4f0/EvnUSE5OtFnCHDSJJbMQgEOSHCjJi7Qi4sCmSkPkYBJGS6ZJ0tUeWL7NSJGIrX1xldXnxmrwApy/KmfPn7EkoqvpfkYQ9YiXnjEVTGA8bfOKHKGZFdTBrpTGIyaKK4jEbFOXnY/Pql4bG8trwgfY+L9jxmJC3z/zTpXfbQY+hhfExjMj5yfFFWjA9fLG8bI+Ct/X0n2vi+aOQ7Et2Klxhg32Multf89+/Rpj+G9PMcY/mNNixPocb/dv6VbFSdfQ0AMNJBIeKcnPlZSuTgG2xVjz6YbJiwkVJLlxbuZHTYqPJC7dqccfT1zMAJUWS5jp3uMaTF7hB0drlVYi6qDtDCh+uTIuDgSFCLF8G8iA9RpjxpwZkW/qUxIJOLLy/Aa+Vy+UZ6HGyNSpLY9/3YYKVtXlSPaTtxa4aZ4Dkv3nkRzPGUvCCppM/stLxwZBbc5cKTF9+V9BgzA++0qC4ZdJm5tncGjSa3496bvABmN+Gzo4D+dSIP4sU/Gxncthz0n9h5eWEJzEXoK+mhpcerbgMA2NU9Macmo7dcOHMaPEnbsLFcLt4Gr3M9zFy7HAf+zuWWi2HrFUav0Jop/k4XLKaDV2gSAQuXwT/qeTGdXHV5jAFaAHeWiBmq5IYcMS0sy0VaaaD7tCDjPnKdLIIuRczuv+wePjiNKrkBA8iou48CzitU2mWhdWDOpJsNiNQXdhmutEvxnSEbGxxjCyDHi+b6REQ/moyc8/iweLpWIedj5lxeBPOiiRfCfMwVRAvMbMsC7lqFTOywqi2YSZfrKr8TEYxAXcRzToypE1suEQtRjCnH0uZ7MDHP8rEQBCrt8/xxrGqlE8tymgMuYRgMqspN7umFC661CnXtZuQSY4op5XxTjCUTTYiZ2y+7rLL7Lv1Lc5xYWsuY0ZK5lseA3XO5LB4rdovVyfWTmU55HLiuGIGxN1A6ZPIJdps1FbmEIk5SGt4paOPJFdUL6MrDjal8NmjSuPLCVybWWl5QZ3ZUgNhVI+ZumxHZ5TMklDo6iXL7XqiQrwWQy6NCkcc9ODlt4IuYvQu4jMgswTIiWglZe/IsKGJQs/HASLcK5I3AzKcRmDG2mkDHvZjHlkYkV9PUoKV2b3mMYz+Tnm7uV+maWbat0LnN+VWc2FBT/0mjViT0Lj7UzE1arXJMsahfMs20XENCnCKmepN0hij0kvO82eSKmbWP00JfYNKbLcZxFi9Xjm/qM+nGpvMCcx1LNhxT3CAABI4hZCRUYK5BWbwkGH0rUTaOoWW3UkL64nxeANJjLtGt1OesO8vHi7q4D5OpNLWH82Zbt4jty3qW+V6Yyp257HKeAoksE/V1uu8eGDHFXMwvnLczlv3m62SRt2zjJUN22Zf3ibFM8/TT6li+rY5Zu5b9uHD9W9MXNTgfM/i/aL1/QlJcbGL/vKVvtGHZCItfqz9GgExalmqGHhFaSQilEgFj1A+MQ85nhMvoFSFXda7HCI8brWZIIaFZER4CHNXl01OnEnLHyAOAGbpXfu6NmSF85S48o4jlcU4TojXX1+RjRnjsDbESGuHxop60NRlZdUFmZO4S6seo3KVd9nKsPOa7v9koSEY6I49Lw/MSmhOqaf5cVNDJ8KfyuP1yhorx7nWJ/F1C8ZaIHp9Xgtj3AGT0xpV50rgzcsb1lue5TYwmllqbUwxqmPq5QDF53Eo9zbJe3pwoDdKL18c5SlSiEaPX0NIvxl3M0KhL0jDlXJXjyp95XFyUtEGS5u1iHxf3d4mu8fmyXpbZ0SLANXPU55KO5QyBaabx4OQSmlIiTy6U98M0XiyhU+7Sl8bdctfeNTJvViz7W37n+rhdy7kq26plgK3HzHKY31mFUc1GtlGTYfkuXdGxUVk2h/o6IUAc61y2oUTclm0tfyDfthYo1ySXPAwupbIfy2OMDC77Wb5nXDNmKRyA5syH+QbCVC6hZhxjrFW654r7p5TIudR+UYz9Y+86/s6bNmHRB1lstHCbsfjOxvcc0ZrQp0vyNVJGuLRhwOeXCFQZn8koW5lvYoOcS8ksV6Fc5wxF4rp4TgsmybJvHKeZ2ygnq2PZzhkzZG4LIJNhLgUQQ/oepvawZM2XSbm8wniJi+dCCEzu8MWCvByLLAlTbA7xtTQWi+dHFHWm757rSGV6PxmzOWtpwFwyZkpLaGaUifn1AAI/CmU+3hTgMsprF8kvDywNwEcNFTFvf9mfsr3lM7kwwsu4xQdtyOMS89+L+d5l4Cz6XQ7Ho6+7L2pkPZIWt9rD68Xi+xeo5+JYlJV+gbIulnfx+gvW8zvSN8Ww/HlP32jDcogGvzd8hCt1wjnUeOWIxvpKdVjJAZ/Y6xlS+fm4xVNzgo0Ke9egVZYMMNfgqTnh3rdopEUfDA6uQZeQsl9uX+G1XWdGVjak3oxrhCjwrXqf0dGTq/G8OuDlSG25Nmd0vsrugEoQqtUqi7Ue8KPzEwDA1vQZfRyDwug1ntYn1Alx21tirG2VzYtvG1S6hlhknzdHAMDB1VirET85X2FjBmzNgNfDGjvTJxbcGqPXCBD4sL3HydUZ4WS5k8HrjHSWhsKu6nB21czF8mhr7Ko+IYs+0+Y32uLsKjytTzi5Kp9faYu7scXO9NjbhphmvcZKjzi7ChszwEWJl90Gz9sjMdrqEZVyuB1W6KzBB+t9Jkli5NJFmd3WePF+VfWTS2gy1g62Rp0Q1o0ZUEmP3799DiEivr29w9lV2Q0uQGTWWoAWgfuBUOYlW6xExNFOrLCHgWQMVsZmwqLjWGFTjThbg1Vipi1R4H1fw3mFbZIK0DKgVg77JDNzGipctYSqsh5mqZcZo8BV3WPwGoehypIDvdO4P7VoKotdM6B3GtbzAj3CKNoW4TKVmFw1hYgwifBosDoT77hEgNRUFoMlRuNVZXHs68w8yyigTgy7g9VoK5vlR5yna1jrs0+xu9YrCCDnbSuLc5KQYOIhbosUMUtZlAy3+1ODqnJoTEKtE5GPtQrrlsbFqJDbTt99Xkgz8VHJKssyImNiDRbJqNMy4HhqEILA9dXpAerI7m4uSAyDRggSbTu53HNMMDPuChHRdxU26x77Y4u2HTNSLACcTk2mjVcqYBw1VqshI33jqBE81TGMaWPGTIyz532D1a7P13KccgwCttdotxybrFBVE5HNOBiYyj1ghQ1ewlQukd+Q9qOpHBG6CNIuZITSjhrBC5iaYo9lOk/ELURtHzyR4NhRE8NsY3M88thR8KKpHWyKhWYXO++JFVYm48TtK4jGQ6Z46TAqIAB65SiGOBK5TAwC4aShtxbeUiyw2lr4g4FcOUJ2BOC7BLcwKUsVZiQtQgXExForqnTcSojGIzLCxojhoCYWWWBa+KU44YwoDmpaMSXmXshIMcB8HTO/prhlmDCRsahIzLIrB7E3ZNRUCcFLJCvCJxSOyzH0V0TMWGFj4yFGCdlLuBQjKxwSe2vMSFg0idE1scsCQKyoLpnKdWu6nlEvaQlxdG3IsbLCE9ss91OkuNhYRcgzxe8GBTIwnYBK530VoRJ7K7HCJsZaCZgOsLsvvwKVPTHiAoAcxQxN83ViXJURyoopVi7FGate5HhFaZHRxShTvCboXEbjOA4WFO8ogLxgn7HCxhSnOqS54jjBVA4jlnJMY8GssBITK6yJs/hEjjck3UkqU450D2dNx2LuogDVLTCxxwKZFTaqmFlpAUwkPCKNhcAUG5oerxw77Yq+SMxZYcPimgvGDverjBdcGkdRUvxlUJfzCDeN47zwqYwl06yIyHGh1OepPZeQzrKPuV5GZC/Ul/+mZyX3U0zXc72z8eVH6h3G86xdy3w8ruU4XciT02JeSobYB4yxswK4svLiy3U+2t4/gelnqWMphFAA/iGAn8QY/+ZXXv43mRX2oz93Hf/Nv/ebeDluESCwTS6pJ1fj5Co8q484OTIijq7CTg+4sy1CFLiuOhxdBYpfs3g9rHBV9TjYGloGMvzUiM4bvBw22Jl+5s7pgswo1+thjUaR62IlPfa2wXV1Bkt/VMrN2i1FRO8MxqDwLBmDZ1flOB+Kh/N4PayykVYph9HrzMrKqZLk2lhJh7txhRBFdh+9qU9kJHmDnelxdCQdstJjNmZuhxVq5TJz7MmRgcxI6VqP5D6b3qidMzDSz4yhRpEBycy1OfYuKBr7keaA46f4eOcM1mZE58iA75yhNtgKtXbYmgF3Q5v1L61X2NU9KumyxuQydondUxkN2g9NjmNiJGFlxuyyerYGPkh8e0v6Z5+dt8RIq1we697p7MopRcyapWdbJfdXQn9CFGiNxeCo7JUZyQXVGWLtTC6urDHWp3wcmyQAbKoBWgbc9W3u1+A0dk2PwWlsqgGHsYYAcp84ror1TQ9DhUp7rIzFaayyi+qz9oSDrXEYalTKz2Lj2JhRCcHjWLHyOzPicn85noo1zbQMOFuDbT3gMNTZWGUXWGbKLSVX2G2T0Ql2+WT32N5qtMbhPBpskowKu7OaxPoaI7mVKhmIPTXtnrMBXTLjauVhZMAxsZ76IOlYMkbLtrCrKl/PbLcAMrttSG0encKTVQchYtbNBMgVtbfE4ivSs7+qRzIOB+bdR44DY2MdAFb1iPtzi+tVh0NPskJs0G2TnAsjcJX2OCV5mxgFGkPG37nQrmSXWAC4bnvcdQ18Yhxm7U4hItaVxV3XgBmm+9GgrUcMyZWa6wXot56Nezai2QDn/D4QZstSJ5X22XVYyUAIZooTIxIumdloa+NyXna7ZUOfGYh5AwMoGEeTAf1kc8Z5qNK8knyNFMCxq2dstgCwbkYczjVJ1BiHw7nGuh3QDVVG5uqkncdosfUKzqlMDOa9zPI07BZcaZ81SJmdNwSRDf22Hmexms7LVGZiB65cHp8STWQdPyYrq41L7t+0IcA6et7LzHK7WfeZyVmIyXWYSdfYDZo2FFx+lzLSP/S0sVAbYg7WiQ2an2OePyaHc1bljQg+ZgyN+5DclAFk12ghIuyooXSA94QsWlvEIqdx906hqh28p3dBTNfrVDbL4thRpw0OmTcs6sbifJo8kr5oMpWDHRJ7eiI2AwAhI9yoiNE2Tqy/xPRK1zIhHDMn84ZTDIAyHogiI7HAHOmOpdswgBgEqoZYi4UAuW+neypy/GraEAnJzVaaQARtKrliR5BskQARugVyDQeQ3akFtzMx/MYosquykDETvsVIZG8AaCMnu4RHIm7zAsIEIlHjfrBhVbDlZqOhZIVluR2k4+xGzOWwm3DAZeSSDabSLZVTMgZ5o4Vdpssk4nTuMurL7cQDZDWKyQh/4Ep6CWLkdnLbuS1LlFBg7jLOTL9vQY3LjYoH6ZLBWtZ7Ib23ATcbqwuW3zvQyeWh8vtyqt/Vpu//uz97VtirX/9W/Gu//a9+5eX+z7/5n71X34QQvwXgrwDYfR2G5TcasQyQ2LsGNkoivUCNAIEhxSp2vkKAwNFVcEFh72r0yaDovEGftCfZxa/zhlz6EvpkkyEZIpE9ZEKLEJK2JN3BLkiM0NmNMUSBo6UfrjG5CHIsIICE6pEEwzm1jV3m+uQaeBYxt4/JJ9goAzBzrxuDRiVJezHH6gWFs6twdhVsUNijIY1HELp5xuRCx32Am1wFg6RyOmdm7lZsZPggc10+EMrqo5wRnQxO57g8QEPFkA0hgGIdYyRj14pEPOE1IpCNMxskjmOd3fWOYw0lTZYR4fpYPsSKSZsRAGzqT8Tkcna21UwqxAeB+7EFM/aG5FLIBpst4vc8kKVOSkmQMsZxzG6KVTbakGIQI5C1NtnYIu3IuTsml82Gw9maLKXBMhXTXJDcCIDsIjamvYyxkKC4G1rYtKiMUWAs5pWNMTYwYhTwUuRrSzdD5yVGMZFwkJwIuREONs2bU7nMUBimPogc0+gS+QfPRYxT3GZpaHFfT6NJ107jnd0Lk/Hmw+TifhqrjFJyGdpLjEXMagg0J66YSzEbl2IRnvoqRCJVSot9NiIOQ0Uxl4W8CMcvcqLjVUZEJ+Rvao8XU8ys9xKnoYItjCcAOKa/IRkVLshcb4wCiZg0xSNqiiH0EkJQW46ygi1kOErXRTIM6JoYBZyT6IWBcyqXUxqWpSHIdXrF2qAmG2Wla6X1ZDD4dN+HIFAaXVJKeC8QIxlkbFzEQH1juQoyIkVeZfBYs5zGvmvgnKI8mAwpZ+kYLcgTipv6yK6ezikSXE+I74MNeEGuiOS2ScZATO0v20D1y5w3AsnYSEQ9sZrNbUwbJz6tnjgWNJOj8PXFHMRA480xoCwpwuMdvIR3EqeungxUFOvfZMDEIDCk+46vZzSYyhEIkcbJO5XlTbjvOd6b40fTHNJ4kNHiHHkjeEdt4oZ4SchscNQXRMA7ZGMlRiCo9NkLjFGnuFVqd1TTsxYdzQMZS5riTZPcSAhyYp/+EmkMIjNqz4iGUixrSEZOUCK3j40Ny+31AlHJyY040LyzfEp2PWW5ETaagJmBNcZpfBAErJPZuGJ5Dj4HpOfDCXg2xCImIyrVzddFSWVFOcXaBmbPTjdOxOJ4vven9kY2gCIItV8aWUAhmYIJpSwMy9LQE1HQD3H2+8ZkCGX0bGFUJYtjJjeSnq+SAVU4ard4wD4KCFnMARYGDI8HGzZF/zLpzEXD8hFLKDVSREzoZWlJzQwvMSO3uUxyk9JjcaJFH5Yvuseyz/rxPmmW91KpeWK+GsTxqyjj60pxHkLyx5mEEN8G8C8B+I8B/NbXUcc32rCshcWvtp/hldsmltYRPkqcTYWjr/HMHHHwDYzw6LzBTve4dYTqPTFnHF0ND4mVHPFy3OLanHHyNSQiWjVipUbYoPDj/ho31TnH15lkJK7VgBAFXqoNNnqEjwIbNeDluMFNlaQ/kpspgFlsWeeJQOhb9R4eMucrGTzfjOsc68fkPhzDByCjrQCw0x1ejpt8bAgKz6sjjr7GydW4Nh32roYLCms9ZvfNW9Vm9HGrBxxcPYu7a5TN/aX+VNCCED2bjOa1GnHyFarEOsppDCojuI2yuR52/2S317MzRAKUyHrOjsq6qvrswnt2Bi4q7EwPIz1uh1VGcUsWzSUhztHWuR8uxYuSXqnJ2p82KPzi5hYAIbWV8mjUxMTJLJ8cm7dJSPU5obtlfBu782oZsNHk0ntMKDijxcfkcnyyNe3QJ6IgKSI2ZoAWAXtL7rYukHvvVUWI+cYMuB9aQkISaQ7H2XLfGSFutcUhbXDUyuGmPuPsDG7VKmuZ0n00bVgwUs2xdTnGN70EW036ryaNNWuh1spBy4CTrXBV9bgfm6yB6uOkJdlqm/NznYPT+X5jV1NijZUYnM7XMHLNxiMjtiGSnqsWIevDAkTE1DmDPhnijKqyDipvVhgZss5qZ03uFxMona3JCDEbpPzZR4FaeQxe4WlLz/wruc4GY608ztaQu2facFkZC4mIU0X3TwS5yPMmDG+KbMyI277FTXvG/dBkoxsAtvWQdW25/+x67YPEKrX9NBL6zxsaHJd4057xRq9gvURjXHanBgg1v+vbPMadNlhVhMTX2qUNId5oIAQSoE2M1rh873B+Lpddr7m+01jlvlI/Jm1U3nCpU97O8iabQFtR39hFurc6I3mss8rfn67POAx1rrtNbtGnhJoCE0nTqrI4KCJ7ao3FvWqwbQacEilSBLCux+wqrdLm2ehU1kflNk+bEIRYnocKOrlZ85xV2sF5hU0z5M0SHkebNl1iGq+82ZPuAR9FdvHmzSdGLEv3bIA2QWrtcR4Mrtdddkkv4zS57S7IjErXyXug1NztRkKh2+Tyri+4pvsgMLppg6lO7eR3QKUdpADOg8lzwKECvOGi031Q6rOWGqaErFoywDPBk88EVtYpGO0xjBpV5WaSVKvaYn+i9+uXSVXlMAyk36HNRKolZcTQG+ik+VpVPhu6vPlR1UQaRZqvPm+kxDCdC4WxyghwLgOTPRW8RNOOGHnjI8wRS16d8+YJAGqbndzNIyY91AnpBCAm5mYpOJZW5A0/fgYYEVea0FafjFxlfG63lAHBKwRPCK9nI1JMmxYyIZa8+YAIQjkZlCwQy+hF0uLE1MdkKMcLRit1TmS23hnSGZGOYdLjZJ3PMkVM5/KxC4bBJWiNDetLaWlsPjA88dBwlpgMcEZbw+TWPndlTZtPIjHFhqK8S21ZtqM8fin/JcN4eawwkpfZLh1/tA0XjN4/TV8q/acA/j0A26+rgm+0YXn0NX7n9s8SKhimHcgAkZg/nz9wlWSkjWUwgDmBB78wS3dT6xV+JJ/MUClgIo94UEcU+D4mggZeNALz5yNEgR+IGwCTmyUnkY5dIqcAHpJHXCKN+AP1LLutPUbiUe64SkELxTI/jQVm+ZcEJsu28fGyXUuyjyUZUDk2jBoxIU9+p6b8PF9lG5fEKtyGZZ+X+fhH/OVxTWNeuKSVxBdlOTPSisUYMLlFyYLJ35dtDfmemxYLJTsmnwOAT8UOTMCxRJmWafrRLwkhIv5IPckLmSVpx7K/nEpx9JKU423nPhZXc8KO5Q/tI/cKt3NJKlKSeDDxx7Lbl34Hacyme4CJPnKeOFHgTxcv2grMSTb4muXYR+Clovf0A9KO5Q4wb/qXHEzLBUU6FoPAG7VBDPNxennBPaqk8ec+5UUT5xVTW0u2UVHIEJDrWtp+T+5Vd6Wb1YV7ho/f8nkRJ7c6fr6L+6YkL7l4vwHz+6p4T90xihMEbgvk5dI43r3ZTItNYO4ytxiXW4EJkZAAvMBerWeLycOlBecSSeD3S3l8OQ+8QIzAXq7n9x2fz3UW38vjb10czttxSPN3fp0EGJdzeeG5PJT94XMROAng9kLVs/u9WNh2l1z5gMyeyvmWn/PfYu5zVHIErGynuiJJ33acNQB9WniPi3Hqynn+EskWhoK7sFrmwBfHiFiR5VwMty3X37Foe7Gotvy9/FvkO8tm1hefxn85dVzPmD77ojxflCeKzCH1JxTzX37Ot3HxauBfz5lHZmqySh/UW8ZeF/nfZtBcetTEcpweuXaZpusuPVCP5H1Lee9b7xeq523lvuW35ou04611fsH2f+Hyv4Z6v1R9f0wp4mtDLJ8JIf5h8f23Y4y/zV+EEH8TwOcxxt8VQvyNr6MBwDfcsFypEf/U9R9iCIY0KQudRoDIbUr2V2AyaiYNQpFZS4kEZtJ/Yxe9OjFdAnM9uMkwfcjyuWQAzCyc6Vp2ZX1MkzAzg17QJiy/c95Sy4+lSUqJjUtafOX3ss7y88zdcNa/OctqOQYck8ljeXF88FCDbtmGpcHO53l3eznOM3bK0hAs+sjnlkYox1wyusyMoCWrZ9nWpWG/HCNuJyMyNC4xozHLNpSuxhHIrJycXIFUPCYNwm1g9G+Zl41xk0hgyrT8TeLvy02AUt8ux7UllIzHu9Szm1gSJ/fOMq50mfImQ5w2eDjvcowZQZ5cY5G18ICHunlUflk28rGyzrJ83qVfng+LcSjdcPneBJDbXhqFZTzrst/L55GR0UvPKo8rt4fjFcu5eYz90zmVYwx5k6TcCGEUY9oEmYzASxrVy02Vt21msQuxXMYxFfnL65d1lgbqchNnudHCZED52oT8ydy/ot+e4vDYRTXLnizmSSzGN7cTmJ2bbUSUUhFpgyOyAX6BPfRBf9O5mTbhcqxTPBwVQNfkc0EQYRBLmywXo9y2wkqYNiomayXrEHoxEQxxXFtCmqicdCkwN9YjJrfIOjx0lwQQZUgumSnuToapreWGFY9bYUw+MJS5P2UKAMxPseqMAsTMgnnZEfPVbKT/IqY2Ck/GMhWRjLZ0/kEcXi4DOc+D5DGt4kT6Lmb7AMmdFdkSzGN9aQNEFHOY+pk3AESKMSz7yGWnKeJ5F34+VVFwH4Fopn5lN9Q891T/cnZKt1iIYl8kYpb5seGaDeui8Fj8t9xrmV38kMT4QVmXDd0LY/2WxPtQl8qdteeLGJVvu6/edexS2cVcPFbvhT2Xd9f3oJD3yPNI3V+P3fbVpa/JsHz1jhjLfxrAvyyE+BcBNAB2Qoj/Jsb4r32VjfhGG5a34wr/3Q//ciYY4Xiy0jUHoAlk8pRstCQ9NmBaoPIxJkNgAgh2pVkuJDmmqGRbdI6IEjgeSeuQ4odKw4YWVVLGFHgfZ3Es9I6j4H8A0469mNAyLgdpsR6LxSAvfnwK0heCAvWFogVNKH7AlAo5P1PIA8Vvy+LHLreTUS8BCspXcbbQoXYnoyJRs8+uSWgGa6/lMoqyopOQxuf4GCEo/gaBSACyvlqx+z4TeY4gUoA0nhmBSsyRMQIi0c7HxMwoap8XTszAmPPyoo63ePlHtBR9LpEQZoFkXTYZL7vblC9n1o5TxTkZp7ICtwtTmXlxmMpg5kiuK7VLOGJRfODSAzz8RVwumvhNPVuMYhoH/i4x6byVPz78sSRIKPXpOOUFR1rMSFwmVSjHLP3oi5DO871qBaAwXRORmQijLvItYnkyUpLGKKrJjaj8PG3Pp34zw2VV/GJ6AGU4V2LgRMSkhcf95kUyd9EJYu8cxay9iMismXnKuE+pnFkdF9BjOQoMVbFYLJ9xVzI3pj47QYyCifyh/E3MLINLZsKUnxeV0+Ka8mf2SxTnykWwBISjDtK4p74W8yKTRmF+vBflyVEgSKR7TWTNPq8nJIxj4qSfjksPBD0xY2a7y09DiUhtZbtLprmQ5SsnzXlmT+RbLeWLAij2Ieg6WVyLYgyLfkGksUmfo5jaymXMtAE9EA0gBxooZg3l+02kvkAUc8iul8Wc83MoHRDMRErC881lcn+jLJ4jtmlTf4Oe2pif4YgHbYtqunXnmorTZ74FB6B8AAAgAElEQVR3siZj6n+ew4KFU3gg7UF/qSQdlZvHBsX9oKe2i2Tk8RzkY9zf4l6Ngu6t5VpzpmN5YR06m3Mug++fhWFUns/MoTymfC8XLJ+zMRfTffTAu2IxL3lcxKJ+fj8Uz322q/O9KuY2GN+f4UJZxbFL4zSz8UXRn/I9gSnvu/LMfqvK38FZObHoB3ekfHcXv19luY+l8vm80K9Z/0o32OW9svwtfqTOS2W/s43vc/596/kK6/ip6/s5TDHGvwPg7wBAQiz/9ldtVALfcMPyw/oe//6v/C+QCLBR4xSY0ZTuIhsVxqhho0IjLA6hwUqOGeGspcUQDCGGiOijhgKhhEMw6ANpTG5Uj7OvYdIvGaNwQzDwUaKRNtfH321Uk4ZjgdYpQdqQjIKdQwWFkGPKQhQZZTXC5+OkmxlQS5dQzsS8WmhStsrOdCdProaWHmamBekzWgsg61iyZqUR0ypHiogh6IzgAlOsJ2s5lhqOpb4jy5X4KFCn2EvOR2NFnztvsqQKgBxv6KNIpESTnIhM+andPpMi0ZykRWJCmhkh5bjVEiEt6+D23Ftij31SdZkMipMudQnTHJH+4qT/KEXIWo48Nxz/yQiilgGjV2gS42yprziRRGmMXuU4Tp4HLovHBJh0PrldZXtDFDlmlc+fXIVauaw1Weo3ahkuIrSMlLKcS4ki8wZCrV0mjsqsvspldLNE23js+W+IxFjL/ed5XOq+1sph8KRHmxHCOGm4xihyTC9vbgxeJzkWim3j/jDizfIzPGdlrCwnzrtEoJfIrUqMuD5IbOvhATKb+5XIq2IUaPR03y89CphtuNEOp7FCo90MsRz9RNTDbLgcE0rPEdVRKZ9JqMp10HmosKrH3KayndYpNCmO0QeZY3kZhV4izUTGI2FUgPUyu9SX38vwglKGhhOPJZNA8V8mmtIqIERy1+cNRK08nFfQys/c+MtYsG6oYIzL7yWXXME5LpRd7QHkuLwIQnSN9hithjHFPBVkWMxiOyG1yBuOJSLrPTEYs3s7s5/yZqVzcoZQPtDd9JPYvCw2uTieDQCR4uS4vQk95o0xZoZlIiK5QNGz+7wM8OneCmwQ8sZekoxxieFT6kgu2hybp0Pe5GPXaqkiMZJGOh+DyG7W0vi8IcfsopyvRE2lnp5x3iQRKiAkQieKv0ubezwmvHmmA2AloOOEnDoJ1A8UFd8/OUnlAhl9zZtuXI9E8nVN1/DmFrev3BBlAyqzvhZWRJ6goozCwhBWzjao8uZbYWCUm3h5k44RQlF8lrw7AuRNuGTwT23E3GhJG53CJVIcZpRNG6F500vSOZE2grgf2bBko5rjBlHUyYYjj7OIc5mLKB4YlpeM6uXmRGmolPZYtsuWRuMFxPIBQhmLE4s2LOucVbwoc7Zx98DAXVjA6S8TE126fS4iqbMyL3xftO2txtn7GJ4X+nopzYz7d5X5DU0Rlz0Bf17SN9qwfGW3+O0f/3U8a47ovcHrfg0pIq6qDlemxw+ON1nW4qrq8elpi5v2DCkiPjtu0RqLVlvcDw2+vb3DJ6dd1gC86xqc+xrGOPzKzSt8fLxCpTzR7zsNJSLuOiIA+PbVPd50KygZsO8afLTb45PDFgLAzfqM41DnWEYipSDChpWx+NHra0gZ0dZjpq73XsJZhd22g1Yk63Dsa9TGotKTzl6MAseeWP6GvsLV7gQA6IYKbT3i9s0GpnFYNSP2+xbtekSlHY7nmvTlZMTN1QndSOQiXW+wXfeQAhgcLbJPHeelt+p6NZDun6VtZGM8+nOFqnFwVkFpn7Xr6sZiGDSuNj2OXQ3vacFTVQ59b9A2FqdTg3Y1YBg0NqsBx3ON7brHYA3O+wbrqw6nQwNTO1SVw2nfIFqJ9U2HvjeEDqd6QxDwlhat7PLFenzjYKA06f4NZwNlyG1wtRpglMftD58AEWg+OMGOGm5QUDVRptetzfTw3kuMhwoQgGodEAXRzyfk2fUaMlHex32FqAPkyhFKqgNCp6HWFv6soVYuI81EmU/XwAnEHS3shYpEm3+s6Af8pIArS65igwRMhDSeFmVJt85sRzirEM8acpMMy15BvzHwqwBxPSL0alrkiAhU6dfWSapHB2Lqc4IWS+zK10vKa+W0SGg90CtatKw85F4jbB0t6HixpQO5z50V4tpDdCq7wonGTygyI8cuLZBaD3HSwMYB9wax9bRA0SmvSS5pg5y0+NKiT91r+DZQ+6KgPE5ADgJ+5+k6HYBRQqS2UpvoV012itaLK0/tHCRiQz5fYiStP8ikEVgHmJf0On35wmY3PdFLxLWfFqBOQB8UxVhd+YzMCicIFWL9QQHoewX31MG81rA7nxat9B4xtwohacNFHaHOEq+ufV7UqhMh+37nIM8kgBfqkNGj+nOF++ce0ZAeoBwF9ScImIPA/XMaHzkI+HWA3iv4dYA6SRrTtACNMkL2NPahDVBHiVBHqEHArQLUWU4aiMn9UJ0lpAPcmnTtoiIELcoIOUiEKkJaUDknCWkprwiENJkjLaDcOsIcSceP0S85ikJHEWheSgzrCN8ScqY7Qi33TwLUkDajEvJm7gWOT+m4OQicngZUbyS6XUwoSYQ5yKxPJwPgmwhb0cJYeMBX1D5I0lIUAdCdgN2SrmIUwGiov/os4CvqA930NKa+AbyJebFsTiKjUtzWoCOqNA4QhALVZ8Cu06PWROiB7rlggKYDxquI9ecCQQG+niMc0gKhon/1mc7rM51TI5URJWA3dF6fAbul66QFhAN8O2kz+iYdqwFzTq+WhhAz1dH4jTtADchtVAONoVvT52CoPJd4dqJMeSLgWqDaE3IYNZKGI6C7tIHQCuguwm4F9JHao0bKZ44Rw9Mvv/TRJ+o7AKh+Mnqko/rkQG3S54hg0j2m01weI4ISCBWg+oioaG6jAvSJ7rNg2FilMnnPkMvieQsKqO8i7JrQsaCpz8EAUYlsoEkXESq6VnURvhFQI+l/RiWghpjuO0EIr+d7gspUltrA5dOziowQ+0rQPALwdXp3nTFpZzrAGyBUAvoMuFXqR3pmIGhuoqA5z6izngwM4ZD1KoOm8ZMW2eiULo2dEjM0k42TkHQv2YtCRECEiCjEQ3Q755mMwyhpHIr95rmhxM9R+qkp9TKFp3HM7Yl4qFOJuXE6O79AuUUIiEpA+IAo09wrQLr0PaSdJxR7EOkYj8NkeE95y+80NgsjrzCQsy5maeAv0tIwfGAsl2OHovxLRqiY5mtexxe0Lr/BxuhXnWKM/wDAP/g6yv5GG5Y35oR/69u/g5/YJwhRYqs6jFHj1q1xDhX+0u6PcA4VVnLEK7vFs+cHfG53kIi4eX7CvW/ho8RW9fhkvMJfuPpxZpHdqh5X6oxzqPHD4QZ/6fpHOPgmI4k2KjzR9Iv56bjD5nqAB5X16XCFv/bs+wCA13aNVlkY4WcxcZ036EKF33z++5CIuHWrjFAa4VFLi0+Hq4xebtSAIWgMCx+etR6gELBRAz4ervOxk6vxne++wb1rsXctnn/3gFu3QucNrk0HlX51Pht2maFzpzvsXQtgihtd6yG33UPiYBu0ymZUzEWFa33O/XRB5djLk6vRKos34wrr52NCR4kBd2d67G2Dm+qEO9uiVRb3tsXVBx3ubYtKOjz/pSM+6a/w7DtHHFyDwWvcfPuMWlp8NuyovhQLyQgoo4uMRL4ZV4SkKWIzBYCt6dF5AyUi7scGLir883/192GEx/93fIFGuTyGRnocbJ1RYS0DrkwPKQIOtoEUEb3XGX3c6gGnpEl6U50wBI2DbaClxxg0tnrAvW3yX0YjdWKFfVafoIXHm3Gdxldi9Bq7qsuMvrdjCy1DRnNHr1CpiVH4blyhkqQDejsQWcdKj/iwucedXZHuqraZxZfQTULBK+nRe53RUEarGKVfaYuzM2iUy/fN0da5vLtxhef1ES+HDRpl87gRCy8x6u5tQ8y8zkBLP2PNbZSDTMy8IRKr767qsR8b3NRn7G2TUUdmHA1RZISXx0OKmFlwT8yMqx208FhpizdpXFhTlRFxZvR1QWJX9SRzkth7m7RJxYy8S5T2F3/jFhIRPzje5Oe8Vg6npHcKAFoEXNUdtAh43a9nqCMjuozCX9UdPj9v8cE/scfrfp1RTQB43h4xegUXVUa4eWMtRIGrigRH7oYWKzMiJASX0fNf+Mt3+MnpGtYrrMyYNWgB4KY+45PzDiGKrEN73XQ4jjVWSXeWY2pDJO1WJQLOtspMvK22OI50X7AnBjPqcn2HsUYlPXkRJBkZRr9NGt+1GVFrh/3QgNlOd3VP98ZYY1MNWT5IgNh6YySpIAHg29s77McmyRQFrM0IKSJedys02iXPApK/eNae8Nl5g0Y7bKsBL89rPF+d8KZbIYLQ26uaxlVJ0u482WoWA2uDxLYaCC11tGFXK4ejJZ1fHyR8Qr9bYzF6lfMDtPF4tlVGiEMU2FRjloHiOfJRYFcNk7yUV9hUA/ZDQwzbSVuW7nFiCb7rGnxrc3ygrasTK/Lg6NlZVyPO1uTy2VsgRIHDUGNlbGYONoo0jQmxrzK6f7Yms+Yykt6neWWkft83qNNnl5Bx1hbm+1HLgPM4/eaxZmo3GmyagVixA0kY1cbN2JBbY3Hoa6zqEaPT0Gn8r5oenx82+LJp1Qw49DUh6drluG4lA1zSEB6dQl2PGZ1ntLqqRzivMDoFndh+WY7JVJa8NoqwG9a6BJBlWlgaJkSBqu1hhwoQEd4rVJUlGZgyNlhMklnSWDinIdL4hiCBxMZrnc4hPUJM2qalNmxIrLuMijPyzkg/yxS1tc33LIcZOUdMvsNgMqLP4SXG+Bx+xOE+k5QPIeA5ntzLnA8ARNKQZdkdjl0GJvskh97whp9AtpqYiTbyXyYdK1JG3zlsAJhZStk+4zjjsg1JxzMnkb4/As3NyMoiLiLVD6BJzsfhMVE8MMzoJSYeMt7mTqQyyzL4+zJxeeFh+y+W+9OmbGx+BZbh3//pi/gq0pLw8ucpifhVTNTPKP3Sn9/Gv/0//JOwUeHs65mbqY1qRraydDMtU3ZTg4RCgC8UZBVCdpelvDIbVZx4Ee4KF9dcNsSMKKUkvSmPPZYetDX1qSS+KV0Xl6Qsy3pnBDsFK+6MBGfhDrmsj7+7ot1lGZdIRi6RAj3Wx2X+su08djzWpYvrpTaXBEaXiIHmY/SwreU5Njje1hcAM0OhPAYgu3BeIgQq63qMVOlSeozE57E5uUTmwm2LF+p8rJ+PtYXrWx57Vyo2RN957FKb3tbGx8iSgHm/l6kkHfoidVwqrzzHbp3LFIppZvfOx+pil9G3tf/SdWW/LqV3EVO9tfzis7hwrGzHY6zO7zM3j7X9ImNv2aYL5y4xWr9tHst6+POy/Pe5F8pY+WVZJVHSnH14sbh8S3qfuQYuAwRlXTM23/dq41T/JabutzcalxGMZbsu9H8WX79cfL/HeP1Uadnu8til+XtXOe+7LHtsbC4d+6L9v/jgzr8u2Xwf5H1bXy7cM/Pz83YvUbN83YVr32rrXBqz4rpLQ3jRVfSR9NZR/iLL7XeM36Vb/J1lva0Nb6nnYd7He/ne7fmi6Qtc87a6f+8/+q3ffQfBzdeetr/2QfyLf/df/8rL/V//hf/kZ9434BuOWJ5Dhe93z/Gj7glckHhan+GixMt+g+NY46P1fdZKfNWv8eFqj49PVxAi4kV7wN24wuA0ruoOHx+v8Kw94X6kHd8n9RlPqjNOrsb37p7hxfqIzpmsq+eixNPmBCkifnS4xsrYhBR0+OS0w4frPUKUeNWtsTbjzNDQMuA41jhbg+9evUGIAvdjm+PftAxY6xE/TogC7VoPGLzGaaxmsXm7uocUEddVhx8eniBEgV3dYz80+LXrz/Fy2OBNt8Ivbm/xk9MTjF7hSdPlOLuf7HdYV5Y0OM2I+yGhskm77LrtKB4x7bje9i1WxqLRFjaQ3tp102E/NGiSxiG37zDUWFcj3pxWWNUjmoRGdFZj1wy47xo8XZ9x17XkotvX2DYD9n0Nozw+3B7w8X6H5+sT9mONwWo8X5+wrXp8st9lTT2O/TLKZ61CjsV7eVoTOlDZrJV33fZZQ48F1P/iRz9GgMD/+/JbqLXHddvhMBCStu/rzNqrRMTNmtyp7zrSkxysTvppwK4ZcBoJoXqxPmJwGrd9i0r5Wb839YhDX0MWqIGSEc/XR6z0iJ8cr2jXNkh0o8GzzQm903i2OuHj/Q5aBayMxdkaWKegVSBNORFxdyYk4WZ9xutTQiwri3/s6jU+67Z4dVqhrWxGDgavMtEV77YzWlBpD+slXBrnTT2iswa1dqiT3t1dR3p/lfK4P63wwfaAz4+bHKc3OoUxoRWbesR9yn8eDZSMOA8mx4TxNU3SAjx0Da5XHW5PLZ5vT7jvEmLpJZrKYnS0Q75uRkAGErRPu/MfXu1x3zc4doQ4VtqjqSxWxmbEwjmFyjhUmlCX01Bl43HTEAp66GpyV68sDl0NIYDaUN3c5m6o8N2nbyBFxPc+f5aJtJrK4nhu8u6+1h43qw5GeXy63+ZFPyNt/Wigk7v78+0RH7+5wref3uGz/RbOTbv0H17vCRWSAoPV2DYDXu43GWW4WncQIuLNfo11O8AHidHqhD4Av/rsFb73+hmsVVg1I5rKYrAaSkZ8a3PAD97cTH3rKlxvO+xPDbarAae+yjI2MQrUtSVkqa+wW/forcaqsrg/tVg1A1x6h9kUE7ddDWiNxZvTirQc03zGKFAZl2MFh8Fgs+rRGIfb4wpKBVirsFsTanh/bHG16ZKbPY2NMQ4hEDoiZcSvvniJT49bnPoKUkbs2h5GBnxyu0NTp/vTkhv7i90RH7/ZoW0sbtZnfPzmCh/d3OOz+23WF7zZnRGiyDGv911Duouanr3BalytCVE9D/SO2TYD3hxXM31LaxU2qwH9aPBid6QxSAb9oa8xjhoqxaLtVn1GvlgL0nuJJxvymLHp+X22OeHVcQ2tPM59jcpQG0ensFv1eH23wS+/eI3OGRz6Gkb7/O6otcN5NOiGCtebM459jSerLv/GHMcaNki83q+xW/d4sT7iJ/dX+XmqlcObboXWWFivsO/rfE9cb7r8HGkV8GRF9+andzu09UihCk6jrSwq5XF7arGqSZ+0MQ6HM3kchCDQNEnD9FzjydUJg9WwTsE5iaax2DUDpIi4Pbe4anu82q/xdHfCvmvQGIfRKXy42+MPXz79cosOALt1j/tDS/d+YzN6bbTH3X6FurGwVmG7GpLG6nTvX287OC/RDVV+Nvh+uN528EFgsCajhrX2cAnt6hNyyyzV1ip8dLPHqyN5Poyjxm7doxtNflcw8jgMdO1m1ePU1Whret69l2hq8iw5dRWCV1S+iKgqj3FUqCpCE+2o0a4G+IQYKhWgVMhhLSEK9B3xXGzWfZI6Q9ZXHQeD1brH6Ui+zRyzGwONI9chGSF1iiSZRIpLLhBSPyiKvRWJCDG911gvEwkNzaikJcbk4CSRGIo4kQo6iiGOVkIYCgERKiLHvwJAEJCVRxgnX9hSaovlhIRKTMaMeooIqSP8mOKHOR6YEcyAiWwP1F4hMElBMVkfW2sRc5JDZkb2RGgYXcHWXBrdHLuaQl0uWl4lGWKJWLJmKMryUj5GM0tGaE4PNloWxii34X02PC5tKHyN+0R/mn669I1GLG/+8efxr/+Xfyu767FgfaMcKuWykDy7u51thUZbxChwSm47TNqxMuPMlafPwtcRT5oOR1tBiThjl2Xh9XU1YkyL2dFprKsRp5Ha0hqbiTQAZNITrTyMDDj0/z97b9JryZJuCS3r3H03p4kTEffeyLzZvJ6iCkgQNWVGFWKCGDGrQkj8AWpS4xqUEAgk+hk1Ysi/gJIKxEPwyKrKIsn3sr19RJxzduPu1jH4vs/c3I+fiLg38+nmfUqTjs7e7ubmZua297b1NWvRpleEpQO3I2BILL4hahidi8i2lMHbQiyyqTbysrmwlvNCR4fWBWidMHhXaPq3rS+yDGOwaGyAaCzKsdoq3VgivaGQGbKGe28KM66QSwAom9zGxkKaoRSK6HVNjhGjhjXT8ZQVht6h7TzGkTZuWmcMA4mhd53n+7EkAP8nBl6UEBrXEHiqQ2dEIDpnBecitE443JEm2mbPP5zBlB8rayfCmCwb5KxgbGQCj0lbkq6jRIswGiiTC/OuVpOYdGTBbgDlRyNnheg1ctQwQi6hMrROiN5AaSCOfC5TCI+w/uasitahbWhznbyGcexpjwrpZIEmwbSxsOvKPQo7rsg3aFB75QcSQEb5YS46h8CU4yk/vr2F7sJExsGbAmH6LT/eLIotpB6q+sEtIUuc66mbiDQYKEs/ikL2UfoddPmRLhqOvQFsJgbhDGqbf1xVx/PLzMSFMMJUP6gs5K0aWgeFtCOriXFX5XJcnSz94O3DFMoUFOWBAnRdAuWnJoVcE4gIUYaIXAPAqIEucT5qnIg1ADpm+LmoTIy/vC7qvks+7DQ2HvbRIG0jHQt6Iv3IoDzHbSwbFljOI3WJyULSPBdGwsMMk4kImYfLU66sQtkIqEDzlznnU9h/ATxgDVZeUf5mk8g7wjmtAJCbVPpV8o943QhBiTlQzmZ2PLaR+po2mZ5NNS+610jbBAQFMyhEzhGNXSp915yXWRhlLeWczZhxPYNuS3mSwrIr+WTCvqk9kA1gBmYC5zFkmyd2UQC6IoGpmZS13Ad0X+2Z7ZTbKMzAJkN7hdhm2BO1lWqmVaDknyXDuYgW0CwaqXltZI2S26ZHak9FOo9E+Z86Ut5nZhKb7LidrDg/lPudQXmvAYUJVgUiYUnNRPKiIl+Xac6kfnKA6TmflklnVFAl7076GdtM9Th3L2vqT9h+9X2PGWjsAKADb/YzPa/UTgzKesTEoCycPQM/f8P5fHxO+kXPGFObVTcL514FFkyPwnBb1tQKu2uq8h0LW3PFAqwy56vW+Yn8VVezwuqAsoYLm6xBYSiWvEjJmSztcB5pvUZlzqCmuSjM03wPKSUvME/zVxh509RnGdPMa8xzVb6vFvmXxeunp3mR/Mgy9zxPVTBbdbK6p1wj30ecp5m1eF35t4lZY2fsscv2pN4ibGXVGb/o+2OhP0s23WV5zNG/CLKrLliMuz72hvJGb+Zj5R3u8S5e3P/jH/29r92rt//jD/IP/vu/8xtv93/9t//zr31swDfcY6mQ8aQ549W4wZhsyYnqo8O9b7FzBPh2bsQQLC7bHkdPgO+6O2OIFiHpGagU0HjZDGi35BG7H9uSHyPepZQVLlqyjh7GBk7yYdoBfbDYt+Tt6IOFY4um5EVYnTBEgz5Y3OxOJc+nsxlakQW30REH3xQgum9HeM4tk6JVxrYdC+A9e1c8LT4afOvJbcmbef/qHmdmrbxmSzQAnEYHZyNiouvEMi5Abd8Ns1CqMRh0NsK05CXMfL8QTTlWg+HWxkIEZByF7sWksetGDMHgYttj8BatI+/Xph0xeAetE64vT+i9xdX+TFbppHG172FNxHGg5xWzKmC3sbFYeiXMsPeWLMtNKB5jIUpSICbIGC3ef34LALg7dXAuYttRXpPCJE1DeSUJl3vaAYhnLCUFa+m8XAcAV2yd9mwFFq3BEAkYCyskgXhil2z2lLNzZut0zgSKu4seKWk0F2T9rXNgBFjXQN/aALuNJeelaRO2T47kPfQWuvUzuZ06t6bWGazfA4DdTXk3NdC225Hkc7xBc9GTx2XLoJYBv2YLtfcGdheLcSIGZm3MCrqlHUoxAgQNt4vw3qDbjoVxMycN145IDETNxpdnIaCuuRgQAhkJAEB3ofRhHGwZu2pCyfeJovOXFcyew669KXk+cj9tElKkgGazJSNHd9VDqYzzqS2/aFonJrMCAUAA1nFO0zh9/Qo5lhh0MoDmOpBx5ZLmM6eJHbS56tnLB5o3k4hAigG8vaA1GkYDs/UQCSW5vnt2Qn9ukKKC2YYyHjHGDH1T1kAMGvZiQPQG5oKefx3uqC0FdKeoYOxk6BG5o2JUYyu8sYnYRz0ZXlAZRbRID2kgBQV9Sd+dCJo3hQrmioi3ktfQV4kNGKBNsljOGajb9zyCN4iBGYot7RhT79iwAZZyAvTzgHh2NB6bgMFCPQ3ASIaknAE8XbCJMmFW5s9ejhNjaGaQpW1CHs0ke8NrNDH7aWoqnQlFBpMcJ91QuFTJGdFuKScFiMGDx5BdQvI0TzmwpqXiftiENBjED8LEzFoZByTHLEeNzIYizQaRiWEWSKOBsgnaJfLeMFus0pm8QpwfV/Qy2ZMCgI+RsUipjDSYSg5JFS8RGZ3YeKXzRDLGxgta2Aq+SZPXRIxKzJBaPDcDk42Jpyaxoehcs7B8yeIyGX2AufdHgYw4YhyyeTK6yPNlI9fMSJX4vEho1cBA/suarksGj62gkMelpOTaWuoqT/MGgJ6VGLjARgxhuM1kTCmGDQE/ajIGFcNOBhOB8f3ZW1aMTb7qn5CA1TJOcjpW0iOVzJSKagLGfCzzeIqBZzkFC5As01WDd5GrEnbVB8iSvXQzsFKDSmD+vOQ+Ok8EPFInZZScwfoaOV+Gmx/iJwFOVdXaSLQy/Kn9R0+ujEVu9yYQ+GUA4peouwYM3xpe+7vyW1O+0R7LZ3/tWf5b/+jfLyQnKasZeUxg6Y1a6kHy5NYExH2qdC65jlJ5NbcOQMm9qiUIAMy8mmv5hlJE8L4uddtG04atrrPMtatJNKQfMjbPXk7yyupCMiA5XtTXqb21/JdaUH4uWj4Jx9di8XVby9ypKLIHapJhiEk/yOdagvB67nPV9lqRvkmpBczXxinnRUZAiApqL+1SEF68k2t9qNuWekJGQCCtFpJfXju/ZuojiodZrn8sD22ZryV1qW2ex2UIypu+vRdrYrYGgEImUb/PoHW61se13676DuU490Mt7mCt1jwAACAASURBVPloX+t+igeYNxG1OL1sxFVleS5fgXy8fl93sCZ1KJqsdf8YdBaReqyv1RIGVZMoLDcmwFzbVbx+9bmy+XxIOFHLOOS1tcYb72lOVJk3eoDVvL7NDFxfMyOdwPomWTbSAgIf2+hIOxKaJaUOv1qeW36HiaC7eCxkrMsNsrRbj+Gx3Y30qx7Tss9yTu6hV66R9VkBBhKinzf34PzK8eKJqdh6y/n63FIuou4jMH9msoYqj1XZXCcsJB9Wxri26ZbPX8ZcxxTV8cq7VKQqVuqo5VjyNCcFIFT/S65exFxD9kuW0h4wtcnjKvqdAmLKRXItindawAGq5YZp6PNSAYfq0Nw7Jc95eXHVYHluafH1L/Neg5q178FUvV48Vxk3lmPnz1BWFXjL1Tlg5nFc/Wmv142a33PWx7Xr1uYDb7jPCriS+suvl98E6HonsPSuddZ+XN/w/k2evTfd9zfa5y9Z3trnt5Q/+6++/hzL/R9/kP+1/+7v/sbb/cd/6z/72scGfMM9llYnvOhuMSRisQSIsIW0Fwmg1KyhhsEjAZZYGDGBiUSlJukQgpWa9XOpRyisnULoo1Vm3cVJX7AmlqlBr/S3vj+Acm0NjJegVt7XbKXCkCn6eZKDWJeYdcmZNDrBp6kfQiyzLDVxT5lXvmfKrOcINQOFUqfW8it9EK/MAnzGBXivWQsBlHvWr9eImJZAdfl9K/mS9Xy2W/IinTm/pa4zAzZAyZddI+gQsFzXk3HJmMQgMY1bzZgFZY7qdmv2UC9eOi3EMhMRTB01I+Quch+pL/dbGg3q8hj5SK1/VxsZauOBNZNmX/0c6vmp18nSuKIwkeTk6nnJ/zVSEq1yec4yt0azZ6v28rOxJvO4pO/LsYjhJC/uVxsEliQmEtocKwZAMQpM7/NM37A+/nCuc/EOpzQ3mEiu5Lxv83FiMfbaWCLebhrn3PhiTEII07kYVQn3Lt6r+nktAXZlRFky2cucPDaXs8K7B115igEGy3k6j0fWBIAiQ5R5HhQbVhJHIwAVptIJKYp3OpV8s3oOVbXOlnM+A/gLQ4eA+zUjxgMGSsmPUvN63FQ1D9XcSb0S5rZi+EisPZkxhZ4vP+e5MkZUXqICyIUFkgFmASfcfgnnlfEujTHyWgwxlWFIaZrbnKbnOntANUiQ3LNc9U0MCMAcHNcGjAyqE+Zr+EsVMT5IW/VOV/LRKpA/81guDRo1wKoZRYF1g8ZjfZGyhn6AaT6WgD/Pj9f6ltScevD6UZIeuQ8wNzJIHZmvZVjlcuxyv9pItfiwrmlKyvAf1F/Oc6lI72fgdDk/s8Yf9v0NGP6Nx94KqL4kaHqb7e+d2n1k/G9s87F1udK/L1X+sut/7eV3Opa/tSUkjWNs8flAVPx7NyAkyrU8hgYXrkfIBp3xuBu7IluQs8JF0+MUmgLA7scWG+txDg5WJ7QmYGtHjNEWAh6hxZc8zNFSwvrt0BVq99ZQW0IhL7mcAvwAAsQ+GgxMpCNSACJEPyqSj7hjIh2h4ffRFLFzAIVEQsDW3dAhA2hYa/O5JTmIk3e4avtCy99ZAoNDtLgfiChHhND7YMumXOZGAJyE7DomiRlFIsBEDNE8AAAhGjQ24Dy6Qo5C4cSUdzl4Imw4M4nLGIgynkhEUiG4oRBdixA1tu0IxxT0Qiyj1SSsbnTiUFiao0NPxA812CHSFwOtKFQ2Ro0X13cAgONA5EgUmkv1x2AL0FEqY8u5rEJ4IwA6A4VsgeoRMJHxxKTROpmPgDHYsjkVILBrR87f5VzbrApRTYgG2zaW/uSciri9ACbDBDZaZ7TOF8KHxkZctERaNHhbyH6UyoVcpX7uAkJqoAwArhqv5PsO3sLZyIL19Gx7JgUBCGwKKNEqlzxaCTH23pRNt5UwwkJQYaFcwDhabFpfQn8p/DgWcOZYOzQmVY41Hc3xOFL7QopjdMKZySxSItAkoFBIL1JScKy76DkU1tpUPNvWxpmXOwRdiIf6c1N+Sa1NReNUQJI1TKoixwEoRlgx6LL5dtZj6B26xuM4NiVvGAAaF/i5EPh0LmEcSG82J42mnQg+miYQlb/o0QLYbTzujx1HG8SZF7x1HuOwYWAZkaKBtR4xGn5fcV9nxfqw1I+mCQiB5jQGA+MiIufjpkRAwtpIwDY4KAHIicOKGfBq/lwYrhuCKcecDciZQpStCwhBlw2OzI+A1rYJGEaLwGmS1tFYzyfKoQbo3lCsyesNlEF57bopLDgnBddKuD3dK3iUcWmVkZSaGRjknuNgy3ViJNBqyvOugXUMpoQNAyg523U4c04EmgGUaIS29Rj6hr5TIkr+MSJgmwh/dmjaESmpaT4zybRIKHQMmp5ZMCU/XeY/ZyAMFsYluIY+k1plCm1WmULGGfRLmHMaDUwTZ+ctf1Z9b6E5Z1q881plhNFQqG3Q0DYVwpScUYhX0kg6w0lCkROgbILh74/I+eWhtzBdoLBpS3IRrg0Yjw2+aqE8ctbbFY1fECBPvaEc8kh54TlwuLpEorSx5Jprx6+TAqKic1nNohGUmcKgZ5IVAJAAsw2IAxuPo6IwX8kZZ8ComJgGAFQT56HGSVF+uAKy18gRUwSA5NIzGM5RAS6T8aIG8qJ1nPl1VkAbp5BcTX1DVBTuK2HEChNqqXPXxdAh4F3akJJRdH8BTDncYmCpw3kFCEt+pISgFjCdp8gBziUuob11yaqEEa8CLcGptRefpwKadYplzFX9VRBcAViVMXns5ZrHwJ94oiV0eWGXUHka42pZIlP+X0KDMcPjPE7pwEofF3PzwHO6PP+GssT5yynDI+9/V76e8o0OhX3/X77J/8H/9Lc5FNYUnUIBFUMyxUsJACFr8q5lVWQo1kJlJaRW5DQEYGo15UpSe5O3T7ySYzJodMRYhdUC6yGemsFZvaEHULx/Tk+B+TkrJCiYKsZEjhW22eINpXbEM2h1whBsYa+bAQXWs5LjAsrkdWQvqABjAaECIAHSSXM6FS05AMXTtOZttTrBJ10AqAJKe/X3pYD4Zb/EK7bmbROAJKX2lEqp80AlNFdAohVGusqrV4cQ0/XzTWwtC1H3SXI0xUtZhwAHHn/tLZC1If2SUnv1cjWfy9+kXM2vePYE8ObMm0mdqnDfaZ6W4ZrLXMt6/LWcgni1JFQXQAHKtRepPJ+KaGkCVfnBsyyeyKqubKprT2I9fw9Df6X+5JETcifJqaQ+PfTeyHHJX1xrv/a2KYUCuCx7yR54bGSNMIh6NCyZX6ekodkg8kAaJ03eHiXrolpnYkQRD1upJ9fzpl2unclG5AqU8A6i1oGTvMi6lOslbLJ47qqbl8rzeSllxUuQs5qFu9Yhx3UobFk7i8GWvL6KrRGo3tfPUUKPMyZyJvlfJp7vUe8sa89U2eCquRdG+rq225N8x3oe6nWxFk4MTBt3xZvl2tNXz720YzJt/Nd2XmsbXFkLefJalXy7Oq+uDrktG+w8ebjq8FQZTz1vcu0yzFL+rxCv0I+GmsYOzPLfssI87LNyjqoIJPfV9z20OUfZdNcb50LkJH2txwJMIZ8K8xDj6tyDTbpcsHymAobK2piDAEyHZ/d4QPAia1cvANPyf3XuwXJZfo1V7c/CmZcXYl6/npdp7FPdeizyXryOy3GX83iImR6AmXq9La6d1XlTeeT8bP7ftY0vCbzeVt6pDyvH31p35Tv7bW0+Vt46N79m+1J++F/8NoTCvsh/47/5D3/j7f6Tf+c//drHBnzDPZaNjnjP3cNnQ3+WPC+OQ0mHRMPz2cCpiIHp00TvUYBRzApOJfgs+YoEpiT806qEYCmktgaWPmukrNHqMNOQlLZq/cU1XUCtUhGOl7DdpXZhDRYlPLcOO5XwVwCzcF2AxOI1MqyOCNYUYCzHExSsiqWfazqc9b0f0zOscyCX2ol1+OpajqlcuwyjTZnE4p2ORfdRQnVD0mhNKMC6LsvQ4TWtyvq8hPEK+ZIIdi/DdeuyzAtdtr98XdetPbpmMZeZ25aQ2dl1VX+WGplLjco6VLeuKyzHJcy0Gtua1udyPNQPNQPSWqGE89L9MpahtnUer5yLqb43SjhvDdCBOegUgLkM8ZPXy2dVA3upU4dG1/m+y7mswbw8p7W9t7wGptBWa9IE4CpvdOnDSh7tMiSXrpVQ1Ek6oMxjnOc1C8Avz22W34sHJRgzI2Cqw1LJizvNmdap9EHqLQF4bWCoDQ31M6nbr0H6chxSxJggYZKlTqK+ScgmVK5A8RygR2YJnj0rHpN46+px6Cock0Jw1QSy+Xj9X+n63iigWvovYyp9leMCvgUo198nCrN1MMv5rIDKLM8WE1gv4bW1wSARO3PWFSivd9uyoJUAYBQQNwuTZ0KXbBioJoXMzJdLoJhzBtgYQffkYzKemgF5CcgfA8nSl2UIKD/dnBgIy7pnb1muPgcqqV8rxzKnPOWycqMSHppLiPAix7VcW88PZufrNcTDqYf2EFgCUAYLYMnvKySxBJa09jB5AStgWW67vDf3s86dLPUeHYvCA0RTI9dcGQJm6wYPS7X0q9ty3Tzr4xo4fABG18ao5v8fAssV5Looa9uCXLX51lX3lgrvBL7qNVT3AXjj/C7vMzMy1HXeBibfoazO05sOvOM9vxI4/V35jZdvNLAck8GrsMXLcQefNTbGIySDc3Q4hQZP2hPO0aHRAafQ4MINuB1JQ2nvBvTRYYwWnaVQ2a0dcQoUCtkZj854jMnii36HC9dj5NxN8Wpu7YiUFT477+E4FLbRAfe+w4XrkbLGMTRoOfRtxgobLMZk8KQ9IUCX+xLITGhMxOthU3IeW0OhsEtPqNMUZtvogC/6Xak7RIsn7Qmn0OB27HDV9Lj3HXw06CzphYWk8XLYFg291gScg5uNsbUBRqXSD/F8NjoiZPIedtYXVt0aKIq25GvWcRTPna/CbrfOow8WRpF8S2cDzt7Bmoh9M+J1v8HGeYzREJtsM6LRU5t+AUydgFien/uhRc4KjQ0FtG2dx4nDSXtvESJpmwHAZ8cdnKH5L7Iy3s7Az4Y1S3tuQ4iRMs/XyNdtHGln9jxnIRq0PL7GBhxHV8J4ySNLbTsTcT+0iKBNXeBQWJrr6ToKPTUFrIlX95415DaNx2mgkC9rEm62Z5y8w6FvS2gyQOBxlLBOnTFGXTyrdZ4mQJpkI4e5ipddWH21SjiPDvtuwKFvSVeTrxcPreMQ6KYKGT57W7xtYoSwJiFlCkPuOHx4245FXiclxRI9HAprSDamDuvddwMGbzF4WzydVidYQ/2UkF9j0uR957w6z2HcmedmGRas+XUdknuxIQbl14dtARXWEjMvQGBB64yu8dAKOA2ugBIBWeJVBihk+3husduQ3mEQdlkA225A4siBlDQaO7EFU3hlgALQD66ECUuYb84KTy5OuDuRhqtzkZmh6XlsGo/7UwcJTR5Hg7al0MemCfDeFI+o1FGKwiWbJnBIbMQ4WjgXC/CMDISco7U3sIyQhDYDVSgs53Y6F4sOnwDNhpmDx8Giaak/AjCJlVaV99dXR9I29YZ1Likssz83sBxKmhjY7bYepxNJNDVNwPnUYnfRoz83ZLgA0LKOogB97w2x6VZ5hG4T+FnqEoI9Dg6moeM50TOzbUCKGu2un0UByPyKkc9xKLNcC/5ecNtQXseoaM4HYgYWKSbqk6Zw3N5i++TMobBzQ4HMd4oSXkzPUsYqOp5+tLAuwnF4rzbEZi0h7WLIkFBbCa2leeaQcg799YMtckninVeKQpwpr5ijZpjRV3JEASB5nr+kSzgshcJySDzLOYXRwDaxMBTL+hl+zVBYCT8VLUWAwHwciDUXkVh1k7CkyvqehcKmIumUk4Jhlt9cebBrr3/i9SQlJwXTBUQOc81RQzURCPphPqsw67pUGIMzszsrST/wek6GVXvss5q0EpOackjVog6HfKomFYOCMhmZJYZmjLpiKJF7ZUw5qhlTdAAwGT1oQKsyRnSODQcLhFHCY6sQ4RImK8eELbeeg6pdGFBI7QrALICdDTY1ngaH15am1FR/1ZCwhqDV4rzMfyEAw4z0as3Dq2RO63lfjrF0sBr/GwD1OwO52miyAnzXgP3sPotDa+/rZn7bS8ZDo/1fpfKNBpbn6PDD2xc4jC3nDhJ4OHmHEDU+a3YlnFI0Gs+8+SKxZIvIG9TeW7QsDi9ixxKGeehbvHSbsmmt8+mUykUIG6DNPmlGkjC95LOtefJCMLjrKAdQ5C/EiyKbXwlTs6yFGBd5FtZGKL7vWUAE53/ddR3GYOCDwW3TFUF3uSYDGAZXQiOXpB1St86/8vyjL4QiKWk4F+BZfL32VAjxiB8tDLeTkkKKGsaSHMFdE4pMQmRhdNmUyDnLGpkpUn1jEobezTwWNdGHeFRyVgjekDGWc88A4N7FIitBIsyqfEmdjh2UykT6wRvOFKscHwXcSU5ZkOPTF7HhXCVg2hBGkRXhORatS5GhKOylCjg0rDU6iDgZ6VUeHeX1Whs5Z28KEy3hfGrKUVIm42AjIgtzK0MSJuNoSv5T7aWUfJYiGs0bE+mfbHplc6bY85mzQgwaR8ted29w7h3CSJtGADM9TG1oo6gti1WrPNswCaOqEJMkr3FmGYW+cUVagjwwmT0mKNIItOmhxobekX6nJ9E+pUmvU+uMOJpiVRedTQCT9ygrnFmOQq6XPst8ipanbNJGBj/+6Ir1XxnS4Sy/djqjbygHs+RGAbNwTPEmDW2DeDYI3lLdKkTMj7Y8lxwVbWJ7U35xh4blagYD79IU3snXx6DhTw5ICqNNE/usAvqmQTiTJudoM/KoEVqLPBp452iTKD+KGfCcJ4eg6byElHqNUXKn5P4ARsf3G3XxepVNpLzXtGH1NtMcjqZsYn3DZqNRwzeWJB54R+HFE8dzdRsNiZPzMxhdIi/auZK64HkJo0U+WUSbMDYO+WRx8Jr0UHm854aNBLJeZMMsX8tRwfPcS9jpaBMwGIQirUHrLhoLRIUg60DmNKiZNy44O72vyHkC90XmNjSJ5pQ3yVE2x0khuAQMGqdIoAOh2kQrTGGySSE6A3g9jVVASALgNUZraU0N/Pw0pjw+AUJBI/LzSs7O5iOIruugkaqQ66gtXe81UuVRVUEX71hk7VblFfxAsZtKdDRtRrJ8L6+QWH/VO0P94I13Pxio01eXG4nOQI30PJKtNukKUCPfNypEN2mJiqcvDobeB4XIeYkqEctubPh99TlNFbAs4IQ/4yopBK+gJH8yKqRGkx5ovcNW07WiRQuTWZqDQaEC6YvWwEvPGXVVArIlDd4CWDh/MMtYGFjmRk/91aRvKiHIepzGUb5G+ONdZERkzmpUJONJaqYpKf2QOZkRFckUpAkvFUAi8yjvEzhsBvSFXO/7p+U4QzQPnLCan/cMWKoiN7L0HL81DOYxYLkEYBkzoDlrox5rWoyrLrWHWU3v35xjWbVfzdVqKO8aElwef6TM5nPxfomBf+e1/PrLNxpYWp1w2fTFeyYb5a0bMUSL1oSJ/dRRWGVnAxPOBAzsqXMmojGOdCwtAVEJQ02gDXXL3jEBfwJkS8gqM1CSpmRGw200NhYvTP0ZDEkjNeQ9o7BPqi/tk7doYhAVkFuHPCquB4C1Mql9oxOijdg1I6wxGI3BxgX28Cg42bwrYj8Vb5HRGdHOmTFtFS6nVIY3k0dF+mENeYKMzkzIInmKlAtp+HwJu8sUohkchckaw7Iq/KysTdA6wbEFu7GB5os9MxJaKQC/no/pu5zO+GqsUs/ohGjYGs4ejpYt3YH1HcU7pVRGCHk2B+LpiGZi3ixzyOAamEIiA1v0JcQwaNag5E2VhOABZBTQKiOxZ0XCFMWzI/deA5YlpJABNoU68nzoTJ5p3vMakydtTWDmMalz7qTfy/eTxqWC0pPXQkhQ5J4yPvnGN+yh0CYhaepf1FPepJDL1GGSxkZEZQjss4EgYxF2XXlnZE1YFxG1KiBV2D4l3JEuUAWQQ2Ukk8vvtda0G5HrtU6TTInKgJlCOpPJTGIDxDaWH06tM20OgWI8oHYy0Dz8pcwG07UmIjUK2kTkBjOJE20istLIoH4olYkwg9e4tuKNxsQEaqbrlc5ELhI16QrqjKw5zFWnoj0oAErbjIRIYJ43T2X+dObrM9XTigwQKnM+JhtfDA1M7pcyiu7hWhhk1kzoYujeAJA16ytm3gfaRLhVQmIlBLSEyFIfMljnVbwzLk/gUNaHzrTx5vBZeZ1d5g/IZPiAkr7ryXMD0CZSwoh55yttlfBT2QDaxPVlt5un64rOHRsx6mvpIc9zd2Vzbaqdsnh0UqbXTDSjgGmdyYZdZfLIVO0U9l0Za92WyjNQCcXHpZ+i71jv9LTcE9P7ajx0Pab/HHJaiFQyiidp+pzkaizcluI1qvhaheoZ8e/nkpzlyxT+7JcxLI9r0OdS5QkkqeozU/rDw9IZJSibAd3UJuavVX28AgHA5GVbAoz6OUs9VbWx6EeuriteQdDnUcaZUT3XZR+52SUaKCBSL+ovgMG8rekeZSkp+jGa+pln16t6nqruQGEeslv1NQOTN3jl+lmpr6vqPeqAWltqa3XfdM+1NpbH3rSkv+py/zJetfzI67fVXSlLIFkfX3v9pmO/lSVXa/GvYPlGA8utGfHX9h9jawYMyeE2bJCywhN3glMRH41XcCriHBtc2jO+8DtcuzOGZHHvO2yMh9URx9Di2p1wFzq0OiBkg0NocPAtGhPxgye/wCfDBfZ2hEbGOTpolXEMDcZk8MH1PY6RGFfHZHHTnPD5sINWGRd2wDm6mbdyTMRUuzEevzxdw6qEznqEpEu47RAtvnPxuuRYnkKDzvpCPiTlENpS/4PdXam7tSM+Pl7iqu2x3w/4ot/hZnOCVRGn0BRCo+9dvqIwXBVx9C12bmCPqoHVEQffwidTQPbGUliqaH46HXHyDbZbAvNWpQLyJST3whED7xAtGk2e4HNw2LmxsPH6ZLCxHkdPIcvn4HA3tnh/f8Dt0KGzAa0JuB3I8/rtq1v0gY0B0ZRcQh/NTJLi2f5IEjRxIi+6H1p0DFb3zQCrE3766gkA4NtPbuEjMek2JiIDMwNCygp3fQutgLYbkEGgXkKAJYw3Z1UYZne7M0I0sIY84pebHr236LbUrpAW+WhwHh2G0WK3GQCgGCnu+xa28TgPDS53PYWXRoOm9SXHMbHh4cn+hDFY9N7iycUJAHnO748d2tbjat9jDBRabMwE4oVwqDYSSC6ogNAxGDSO2Wg5ZHHbcYhq0thtRpz6BrvNCB/JK+hcLmHS42ix2YwYvUXXeaSksd/2JezNM5kVgXNq+zw47Hc9Tn2Dtg3FICIhuQDggyHg7nwxVBw5rHHT9QCAEClKIASDjkMaxYAQAoVjUp/YiDJaQGVstgOFAXqDthuprTDlKIZg0LQR50OLnBT21+cSuhiCxpZDI0s44WCRk0K79eXHJUUN6ATbToaJsXfY7AacDy2ajS/hgQAwHBsWmifAHL1Gs/GTAWgkj2O79SUiwDWheOHH2xbuYoRuA4K3Rfg+RYV4bGH3voQhNtsR49nBtgFhtMQCqicXgHj9bRMRBgtlE+KoibVzNMUjLGA3DORBMpuIxF4sYgdlJlAOITQthRzGwUBvAoX+tRHxTJ5Z3RI7p2oSjIQV8vh0QyF78YsWuUvEhpkU8pG8f2oXJqZMS3XzbQN14cnrd2yg9h75tgE2kQCZAjJ7chUbW+ASe1nZa2oz3QP0GkkBwQBtIq8pMAHAnr2mR7b2yNe64/PC3XY2FEmg8hQmqAAcaawKQLYZ+mwRN6nkEKqRvfsmQx800jZCf9rQxt5V7JfsAco2Uzv3DrnJ0LfkqdWBgFzWGWmToQ4KZlCIHXmrVAR0BGLL7JeKPFMqAckB5pZ+a1JDx/RIICtuM5Rn75fhthK1oz31TUWF1OYyZuVVacveEolONgTgtFfQI3+eGkCPQOwAMwDJUh+zAczJwl989V2dGamPAKCFgRWAinQ/HQg86ZEBMCYwZQb6zCRH85C1gGfA9ASWCilSxkQUxM+4hBSCrnEHeg5klMowg0KymGlFqkheVOq7Iq+hJyNWVtRfZJrTbFC8aypoZMPjUXQ/PQKF5CehPGM9gp87L/FxWs8qAckA2dLx1ADikRRPog58vZAfyZypeV2VuS3up+LPYskHNtU11VyV5y/ERdW5yZJIbWYDIE39kDoqTM9z6TEroLf2lsq6SNN1xZu5ErYqbUHOy33TvK5cP5NcUSje3pr8aOkhpfl9RGKlTAYmg4V87iods6wUvVf8WnKsV8q7AMQHZWVeHrxftPNN9FQu+UH+KpVvNCvs/o8/yP/Kf/t3AUykJzXRgLB31p6YmpiDDLBUpyZLqBkx6T24nfn9J8INzEJAl3VqZss1Rkm5x7KNZXuPkVzUifJzrbS1bz7MNNzqdoXMYsbOuZjz8vmWOZX71J/wtaIearktdd0efM6WXzCPfLlM8/DQTFk8IqsXLNoUj0k1n1Nd9fi35KwP1bjkOS5DdqXUz6WYc6vG6lvVoTMaj8/D2jyWNlbGVPelHufy/Np96lL9CM003N50fWX5LW0s214+n8d+Pdbmdda/R+b+beVN81nXqf+/a32pu/bMMyAkJKs5Q3muLweg5AwV4pC3WcDr+V6+f9Nnce3cY/XWrnvsea99vpffYZi/XXusclnZ6NUeEhnrkpF0efyR754vtXl57LO8nJO0MoDlul8pD/pSzXE9bXKubDqX7a/1W+Z22fV6HtY+r3J8+X2Alfrv8N3+po/tm9hD55vilTbmGSVfriz6+djPhKq/b1cW62Nr99F7rS2DtTlYey6Ley6dyXLd6ldsPadroZbVmniw7upm1CPP7B3KAyC0uPbRz+XKeniTZ+3BvOCR695232Uf3rJ3Wa3/SB/re791/bzpuKZ10gAAIABJREFUHm8ob12Lv2b76/f8Chd9hUv+r//h733tzKm7P3qR/6X/+j/6jbf7p//uP/zaxwZ8wz2WTke82N2hj27GTOqTKZ4XOe6ZOMVznqR4KoSoRuQvhAzGqIk9s5bGqEGqYe1AIWuRkNUxWFgm8wnsTZNSh7LGpNC6UMJGlxIYlNMJPjZnzCxhmWaSGAic2yfAmAg51EzioTDScuhhqECmyGAA0/egqb5hhBkTeMiUKRqP9bWJQ1tlbBOwrqUqJi+MhIoK4+WaQLx4iZa5pjXIn45hLrtQ6kyvRZ+tZUIQ0VRcsnPWobSih5gqw4WQVtQGAsldlfFM7KbrQvPApOMo4yt5gNy+4RzFmRFh8Wsu8hQCaHNWhWRDclyJmXLa9CyNGWV/uDAy6Oq9eK0eMB6LIUWeA99DQk6LkUXOV/MguaKyjiSfU1g6xTBDRgMUlsWlWDwAaMP1F3WgJqbKwkqqMOW7ls7M+zdj9pTX1aZB8jyTn9YmSXTMySokd1WIPep71eBfmQwEzWQgar4pNdVuOfN9WFqDHhTdK0f1IDwUoL7mmuSj2lEpnTmXdS4xkqtnUUrGJNGR1QMjyqy+jLXk7s7bKfNQA5KaIKSMjasLYYXkYqlcgA9kDbs05YTy/ENhyjEsc46S1zl7LQQm0sdlCGW9BqSOPP6l96Lqm8wDkpq3mRd16npLVFCHTEo7aXGtFJnD4vVaB6urhoB6B7tkE63ryDxmFGOISpMmYC07AqDkEhYSFWBuFKnXQX2/em1In2Q+VHVMYcrXrcdlUEhmvlLRExlLrsevUOQ/CLxyKH0GxIJaPInLNVOvm3qJ1YBkbRMt6x8TYJ552qTIV33t0Vr77qnLYr2tAcclXn4gmVLfX8Y4RZNPQ5J1g2pqHgMNGQ8kZJbjXLvmsaLSfBBvsks+AJzvUuo1vDz1hjl9p7L8vNb/H6v/WDe/wjVfprx9TCu/B3+p9/v6Cn0l/BrfQb/l5RsNLJ+4E/7OB/8YPxpeIGWNF+4VfLb4yF/j0/ECf7L9GLdhixt7wC/GG3zYvMTPhqcAgO93n+Oj8RpDsnjijvjz83N8r/sCH41XaHXAM3ePb7nXeB23+NO77+H3tp/jPnY4R0fyIlnh2+1rRCj8+PQert0ZIWm819zjx6f38IfbT+GzwcfDFXZ2QKc9IsuJtDrgC7/Da7/BDy5+gQiFT8dL7A3Vcyriypzwz88vMCSLlDWeNgccQouXfoeNoTC+mBWeuiNaHXBjj/jR6QMkKNy4I177Lf7V/c/x0XiNT8ZL/PXdL/GT83PchQ2+1b3GVo/ok8OPDu/jpiH23JvmiM/HPWJW2DAj7o07Ym8H9InCeT8ZLnDtztibAT4bnGOD5809PhsvcGnPOCUiEHIq4uVIbLq/Ol/hyvXY2QFjsrjzHW6aIz4dLvCdzSt81F/h0vX4tN/jve6Aj88X2FqPP9p9in92+ADf277Ep8MFztHhe9uXeObu8X/efhc7O+IcHTbGY0gGG+PR6gCnprzWn5+eIGRdQpIB4EV3i9d+C60yPukvMASLv/3+PwUA/C8v/wBb6/F+e4eXfodGB3xyvkRjQjFWfGtzC4OET4ZLOB1x71t0JiBkjQ+6O7wctzAq4/ubL3CILT4ZLrAxHve+w7P2gI/7S7zXHvDpsIdVCX20zOwb8f3tF7gwPf7F8X0kkOTKne/w/d0XuA8dXrS3+MnpGRodcGkH3IWWwqRNQKspXPhnxyfYuwEfdHf42fEJrZ/2iB9c/Bx/0T/Fnx+f4qrpsTG+hHbfhxZWpRK+LKHXG+Nxrgw3z9ojXo8b7OyIjRkRssHH5ws8bU/YmBG/Ol/hD/af4/+9f44nzRkAcIwN+uDQmID32gN+db7E0/aEW9/BqoTX46Zov142PTQydnZEyBqf9Xt8d/cKf3G4we/vv8DH/QVCMuijxWXTF6PSs+4ApxJufYeen/OfXHyCT4ZLfN5TWPreDdjZEVfujJ8dn2BMFiFpbO2IzgRYHfF63MKqiJANnrcHJCh8cr6gfG7X4+VA6+bS9bjzHULSuG7PeNVv8YObX8CpiH/yxfeLBu1l0+OzMz1npSis+bvbV2i1x4/u3y8h5p2hUPijb4vkze/vP8efvfoW/vr1R/jx/fMSag4Af3L1CY6hhc/EKP1ed8D/d/esGHK+tbsFAPzF/Q2edkekrHDvO7SGvrv+5s1P8b+//B7OweFpd8SV68sa+P72C/zft99GSBqXTY+Pj5f47sUrfHK+wPPugJfDtoTSp6xw1fSwOuJ23OD9zT3ufIfr5oRPTpd40p0wRoMxWZwDPZfnmwMu7IBfnq5IIzhZDMEiQdFnOji0NuB+bGd1rU44B4cX2zukrPDR6RLf3t3is/O+sGXv3IiQNM7BweqEv3nzU/zsfFOYu19s7qBVwr+4fQ9XDYVI33sa9x9cfo5/+uoDXLdnfLh9jR++/gB/fPUp/uL+KcZEIdof7l8jZeonAHwxbHH0LeXp64Cjpz6nrHDnO2iV8bQ94hfH68IU3keLk2/wbHPA0bf47v4VGTCzhlMJnw173A38rKDw/uYefaQUCWEPH6LFh7vXxAqcNQ6+xYfb1/jZ6QkaHfBq2OLCDdAq4ehbvL+9w09un+HffP4zHEOLL4YtOhPKd8+FHXDrO9yOG7zY3uLzfo8XG0qtuGmOuAsdxmTxs8MTPN8c8L3tS/zo7n1cNmfs7YidGfDL83VhT/+s36MzHq+GLd7f3NNc9Tu0NuD97h5aJfz47jmumjOsTjj4FtfNGa0J+NXxClftuaR0fHbe0+8dpy5olfGq3+A7F69x8C3OwWEIFpdtX+71yfkCz7oDfnZ/gw/3r/Fy2BYm+N/ff44//fw7X3nf8cHuDr88XNHab3s0bETe2hG/uL/GZduX53sKDWLSOHr6Xfzw4jX64HA7dnjSnmjtR4uzd/jw4jVC0rgfiUBOvrPk++x+JKI/+W07e4c/uf4Uvzhew+iEw9jive097scOQ7TEFYFczqWs8N72Hp+f97hsewzRlnnTKuPleYuBSQ4BYO9GHH2DnRsxJoM+WDzpzhiiLaSITkccxhaXbY+UFW6HDjkrPNseMfCalZSXw9Di6faIL07EXi/fhzErXDYDQtY4jk1JUxmYXBEgNnIx9vfB4jy6wsRvTSqvx2A5RYLz/Hnv3ntbjP6SEiPGdWF/98HA2Vj+19v+lBU6F9D7acsshuE1Y3LNQ+BMxOBdyccHJrmopRwVQOkvYlgWY7vcA5jzHMh9ifgwzQzxcn3NxxC8gWbj/LIsDd/lfc16zZeJUVsIEUtqRF1qpIw5C6rCQ6P1DBcv0XVWM2P0akSZ3POBtWPej9+Vv/zyjQaWDhFPzQF/1H6CPjlcmxNexy0+bF7imb3HjT1gpwc4FfHd9gvs9IAXzS20Svz6NYbksNUDsAFuzBFORWiVihZmpzx+b/s5rswZV4aAk1MRPhu02sMg4TvdK2z1iAiFrR7xYfcKV+YEny10m+F0gAHpVvpsYJBhVMKl7eFURKsSnrkDjUlFtNqjzw5P7In1MTW2ZuBzAU7HiYpeRRjebD5v7kmTUUVcuxOG5Oi1pdd7OxD4UiQNYVTCi+4WrQ7Y2QEXhjZaAn59Ntga2kB1mkAIWrqn0xFIgLbUr40Z6YfQDOX53DRHaJXxXnso51sdoJGxMR7PGprvm4bA8XsdsNEjnncH7M2ABIVnzREGCRe2x86McCriFGkTstEEblodMCSS/qDnR3NjkHDdnBF5PFbH4oHdGI8EhaftEbFRuI8dEhSe8OZG6mhkPO8OZV60yjBsftzZASlrXLme5GB4s73nTecpNUgM0o3KZTNa/xedUcfP8BwdrS1uT2sCMSFTnuspNaVf9PwTbSCRS4j33g1odEBIBlvL8ggq46Pxip6ppespT5Z+ZDvuo7SztSOsniR0LLv7POfepqyK0aMx9P4YWjQ64j506HhTLNfDooBYqS/6rVs7MgHWtK5DpvzOLRsPtnaEzwTsRQNW+p00ba4HPt+BCLFe+y1tCAytd+n/MbTl2qBMpbOqqa8AGhWKDm6j6TshZI2OjToJCo0O0IqiI1ob8HKc5H4EMApRmOZNkUbGy3Fb6mg2vcuPrmwGAeDT/gKtCbgPXclZlrX92m/Qc+52ygqvx00hHQOAY2hKX1JWCJnIy+T8L85PiFxMR4RkcB9ajNEiqIRf9NflMzxGi9YGnIKDYSMIgAJwoehZIdHzPXjawB48bYL74BBYe7bOF5d+n0Izk08KHKMoRhwxHEh0hVZ8D9Bn8c535bVWGSPn6Mrn9SenZzj4ltqNwCf9BSx/t8tYUlYI0Pj4fEGfweDwq/MlAOCj8xXGCkSLYUFAo8hGJdAGdYgWL4ctRIdXq4xP0gV8NLjNGwBT2obIP/3ydFXatzrh6BuOHCHDx2fYl3mR6I+YNH51vCqbszEZhHyDo2+gVUNAnechJo2PTleIWeHH989p086AUvLTb9WGAAPXPXtH/UfGR+dL+ESRPifv8Em+wOthgz64AsqVojz2z/SevhuCBbBBSBq/SjS+IRpo33AfKdd9YMmmMRocxrYAB5kDqzcYgi155CNLlgze4uf315TrnXSRNBIANUaDu6HDaXT483SDMVi81PRcCOR8dbmRn6drnIYGGRSRJJt0oxNOQ1MA0cm7AowCM4v/JD6FSBgdhhYpo8gGyblYeQ6N3hc94CJ1xPcLSeOH6QOcR1e++4+jQ4jmwUZd0oROo6O1eO7KsdtzN63LpHBSDGRNixg1bnVXwNJxaB4QxsWocXvuSnROzsBpdAUcCQAKgTgEvDez63NWuLPT9RL9k6Iu4OYAQKJ0UtQlkgWYQIp8hiTlBnwNgCLhUhOglYiUqEpkSS/RARUbL1VSOAk7shxanH8QRcTvKUKmirBaePffGCJen5f3NTlTqacwrPV9CbjqKIo1b+/yuHj9f91Se+nXxvSWUlI8/sqAQ/U7uZHf1vIq7vA/fvpvlTDUIdGPfKMJ+MjmSrQqxWtSW36tihiTRWc8TqEpDLP1j+9Vc8bBt2XzIpsiARLCPgvQxqg1ASdPEiROx+KpBFCkRqwmYCc/psIAK6GogTUL5Usw8qZKSFCkeP4RiUkXJloJ3T2ODo2NcDrhODp0rG0X+MdYqYzOBviiSTfpCGagaEvWoauiBynjNToVEpVpQzdZClMGOkchyDEpaDURphidigaksNX6qOFYn3AIpkjAWBOhFTAEg8i6jnWIbz1PdZiBWF/rOmKhBFDa/d9+/j0AwLYbicyFLX/CGiz5uQARxQAoIbE1Oy1ZIvnZsK6bWBBrjT55X1sHRV9OdOfKhsUkjKOl0OCgWQdOPRCyF2ZX6ZfIt0hdP5IkzEzrDwBUnjG4imQLhRKr2QagDocsYdUsoZK5r8EbWBeL7Ipi9lfR79MmlnBdOl+FEUsIMM+n1gkpEiOs6NstQ36hSBJGhOHLxssbaJOhTYSE7IpmnLZp9nteh7hKyZHDry2x+0pIqJwr4bcJUBr4s/HbQCZtOflBzwnM+jm9lzBT3cQpRE2+H0wu4bjJG5g24p/136K6VYfTaEoopITB6iZOVmaWRSmyLhImyPf7Yf8hVEdtZtYBLO17BdWmsiFSlmQsVNHAyzPLcNHDE7062YxICK9sZGRDJ4QnEpYp54EpxFH+B0V16rqeY+BspvMSSlpvgniT9efDCyI9qXXyMoggZxG2p7xGbqjPKipihR01MbpyPZF2KM/NVPPKm74pTJKfT1DEQpuq41wvaybZqa3vTJQj41W+ejY16UcdzqmJCCc3eQp5FdIPTX1ITcJn5+eFiKU27st8Z8PtmInUpcgNqEwEPEFBe1UIeoQ8JdtqjMIGbIhUBwDJcCTqiwLJa0wSGpM0RxZiIQ45zUw8kxXNmcrU1v2oCpGMXF9kHbj/yQGHUIVNKuDWA6nFVy6HAGTeOZ2nryMiqbF0LKuJLKb0HUDvqQ/ZAH3k17z+h8DPxKCsr1l+6CJUNmsgDBMxDzTgPc1bIYcBJskQgLRKDcl/yLzFOD2zmhQm8TNIAsp0RvDqAalONkAUoilWPApMxgMAkdeW1kAKgLbT2Mpa5q83I8Q54D1EdZ8agGQzP7bmlFo6tGbrnedx5uCqxq4yZvd78CweKbMc0qpvy36t9XPphJsdWwJQhXm+a/2+BqiorgHmOeR1v98EMuUZVP1+U59n93tTWQLYt1zzpXIw10Ayl5++eyt/qeUbTG/z1vKNBpZGJWyMx7U7wWeDl+OuvG91wC/7azQ6lBDET3GBm+ZUAKGED74aN3ivO+DluMXGeMSscDdu4BN5jp62R/b+kBenjw5WJbwcSKvyWUchLwBZ4591B3wOCt+5bPpitZ/6TeF3W+sxRFtCuIRtNTHA3LqxAL2jb7CxHo0OSFlDq4SUNe7ZQ9BnVcKEDmOLznocRwejMrZuJGBpydspITUAcNn2FDamKMxs68ib5hN5OKQvkicq4TECehyDw8ZEjBweU4PWMRrs3YhBWwzRwKhMHhDvsHUe90OLrfMYosG+GXE/tLhoBwzBIkSNfTvgddygc4HYV2OHpIBd49EHW1hh5T9Am/TAG+zGkpdyrHJdY9JobETKwEU7wpmI23vyJuzbEX2wSNkVQLlpPEmncA6qgGrHAKl1AT5qaAWck57Cckbqj9EJSrFkTJzE7J2JiEoXo0LUpLtZA0atE1rW9jQmwXtTpEdy1iXnVETmU6KwocBMpzPWVE/1RStULOMk4p4Y3BFYl/xW6sMklxLYixoZoGU9aXemqGGaRMLnOiMnCachEB6j5lAd2qwoQ/d0Lk5MrBmgfEBaQ9YmDJXYOVQuUil+JNCnVMaYGKyJnEgmfUeYzGNTpK2XWZy8CSjC8EEjKcrFFL1TpXL5zGrDDKOg5yKvRb4kJtYlHS0By10qQFWYd8WKnaBp458UVMu/fwrIEVP+r2KLe1DQm4QYLNCiGDoy6BwsUJKREoHe8iPPuX+qzUCge2tmfQVAYGYToS23X4ERNWqAQScygd1Y51eKPElmgwMDEMpL1ATaooJqKm1DNYH2nDCJpYtupxgz4pQERhqghtgpXeQ8wsSAsLre5EoPEzMpB33WSJtUgJryCioppCYAYqATFs5BIbe0e9K9QmxA/221e2KNQBV52lsGVAKKVC4ah3A8fyNrFgZqRxmFDAJwsMQUmtW0801NLqytUMIeCgYkU96i6VUBJtkSq2hUDHA0CuNqsjSO7BTMmUBjqlhhVVQFIKRmYg41PY1JsyYkNF1rRgVzUghbAk/CDJtaMCtsRmzUxBg68PS1VM8MMlcTeM1GQY/U3whmD3UK2gMRvJbr+krB9gSSsgGgFZSnOQCA2FH/8y7DHolZVgUCovYEDL+G3Ig9K4QtXa9HVQCOigC2pNOYLN9fpEYN5f/aEzPZMpNq1vxnAA4Yomt486/jtJlPbg54iBUWiK0q780AJKMKOFVZAC/VMSMQnSrMtVCqzGls1IwVluqoApCzVhMrrHxM+RmbgdZnbGSOMGOdTZbq2R4I3dQvAUOFVdZiYoWtGV4jJhZZw3MRJqOIzFPWeJB3CnC7zPg6sclO10i7OqAw0z5khc3IIjOzBvQy5mypTIqhI90fqMa2YIWdgd48GSSgFgBc6gqDbV3vnVlh5+B6NUczTfdezdvleqsgvAJ0SxBdqrzp47cGCh8BiW8FuL8rX0v5RgPLjR7xr+9/CqMSxmzxomngs8GFphDTK8tSC8lhb3o8cSfsTQ+fLN5vG3SawtpedA4XpseNo9DMBIW+cziGFlZHvOfuce3OJTx2SBZORbzX3Rd5kxN/ow7J4tL2uHaUX7bh3KmIiTgnZVXCWjfGw+kIqyKF8/E30JAMLu1QAKTP5IVs9Fz64sz3PUeHS0f5aefOoTUBl01fZE2u2zN2HI46bCjnSSPjuqGcCQDwWaPl8EcKy4o4R4eR+yQhrEOyJCnCYZN9tCXHUCMXuRGRHtkZCmcMWZfwR5Fc6TtXvMSNDnjSOs43Mzh0LfZ2wEVDoZ2Njrhse4zR4Lo9Y4x1GKPiP41QeYj3LJ8yVl7KU9fQPTIZCzQy0nO6/ml3LDl8DXuHt3Ys7aWscW5pB9CyDEnDEjUAMHAOR8oKp64peShC9iT6qvJfwt8k3O/YUQjYRTuUOTc64ZIlM/qtxd5R/mFkECtzLn1sNK3hc0eSLgAwJoPjZkBjIjbOFxKrEt5bgW7JQUlZFU+5lCEatDwvYnBoTEQfLGLS2DqPY9dg14wU+qZIDsHynPTBFoNDrcEqXnzxngt4b0xEv7Gl3cYGtmaTZ95xuwL4bRUOeuoaOBuLbEzKJNES2ONdj0tkTkQqBgDGYIpxgnJxdNGAFS+9VpPsyZH1Oy+3/RTBEMmIIWs0Z4V+a5GSwrb1hZBLPOqiNQsAfeew60Ycm8D9nb77Tq0rOTyK79M1fnpOGwpna52H3xryBugpquDoWmw2I0UNbMwsN8fvLLrNSI6ExIaQ1lH+0YYMG/VeIXDImrURfmOLZ14MGJJDJMaScUPjb5rIHvxJNkU85GJc8RvDOq5iaMkYN/RZcy7Cbw2srTRyFx72oXEwLsFUXvycFLo2FE+NGCLC1qDrAmLQCDuDtvMYWwfXUNSDUkDYclisXGvzjB1bqYy4JQ+x1mSVTkHDsNwJgOJVjzsNrYGw06X9nMnLrCsvdxynHCdhuVY6I+wm7QOlM/xoSFYlMyiPGikDymSEvYZuI0ZraeNt0xQ2l1QB6Mom+NEALBlDHVCFREm1Cd5r+MsK2GfMSHoodCGX/zEw2xIbLnxUFNrG3mEZQyFCqj3RSU3eamDySJuMtNFFrkPIiZQQNdkMH+geYa8m8K8APyjE3SNsKu9QwsieYYDalI0/j0kFmgs96kkHkoFE2KnJMxwmw4B4nGlRocxj7SUTL/PMY7llQ0Sm65SnZyqSGcAEmpABHQj06jB5NjV7vsl4UF3HRh8hJCp9VJiBnGQIPAH0GgB85bFUEUUixPs5cJbvEcX1BfiJF3fpsZSx1LIhqiasUnnhlWTQzaRJWaF41Ve9g2nedl2UfA6WpQaZJQqIGlwCxSUQWgVEVXvSp3cpy/bexUv64L4rbT0Aj2rl9dcMLL+J5XfkPb+lJWeFTnt8Fi4KoASAl2GPU2pwZU/wDAJfhR22esQrT3lQV0w045NBqwM+Gq9wZc+4Cx2sTtjqEc829zjFFr8YnnCeouUcyYRTbrA3A4xK+Gy8wI7JbC5tj8/GPZ66IwDgLmwov5BjPSg3M+AcG9yFDZ4394ggQNlyzqBWGVs94gu/K4Blw2CL8sOmbxrJHbxpjnjtN4hQuHA9ztHhj/ef4i50OMcG39m8wjG2GKLFpR3Q6sPMyytA9i6QN9OpVIhxJGcQAA6hKWDYJ4OQNW6aI46hxY7ztQDa+A/JYKcD7kOLznjsNAGic3SFeEdIQ4jcpsXWjjgwGc6LzS1ejVs8bw8FmD5vD9Aq49Z32FpfwKr8F9IUKfe+pbBiBr4pKzzrDhT+jFB0QP/k6lPErApRz94NRAhiM/rgCsAHiHxEq1S81GOyJW9r3w2FSOTb21sMyRaCjDEa3LQnHHyLy4a0PQUUNgwQ39/ewahcPNFC9vF8c8AYDb69G/B63GKDjMYEyovLGhvlS97h/UjEH0+7I+5Gyp954gZ8/+JlyQG76PoqrFsX75xViXLGGPDK3ApovemI1InyC3MJK79q+jIn723vcTd2uGynnF3PuUg33Ql9tLjpQsldE6KJlBW2biwh5wDlu910J9z7Ft++uMU5OCQQkNzw8weAvY4FoEtEwvPNEX20xXDS6EhAmMPkBTgbnXDB3n6JIEhZ4WYTSh9Es1XaumiHYgy4aAf4aPCtPRHDfH7eo2WvdduFklMGELh7vjuUyIKaYbruT86qaLj+3tOXuBu6IpEEAO/v7wlM87OxKpX7pKxwsyXD1mFscNmRkWLkiAEA+M71a7w8bxGzwtWmL+RJAHDRDHh53iKDgP3ZO1xtejIK7MgoULNbSwj+EA263alEEPTBouXnGvOUI3azPVM+mnfFMCCM21YnRDZ2+GjwZHuGMxEn78qxZ3sC0Cfv0O3n/RFjheS+fffmFc7BYeRoBkkvOAwtGjYMSd2LdsDrc4fGRnQ24K5vSaN4bEqe2/ZqylmWfECJ0KD+aXRu3i7NoYWRcEQOCZdIh42jdSZ7pJGjNYRcpHVhZqiQ1IHW0WdQUhNaG0o+oq80XmNSaGzEaXS4ft7DJ136JpsbayJCNBiDwabxGLxFy/2Sz1XKqpCgbFzA2duS2mB04nOppBIAlHYh6Qg+koFD2j2PRLCk1JQeIlq2wsIuerVibLBspPHeomsouihGjZTIkCERM2J0k3H4aEraxKbxOJy/eiyscwHjaIsxRVIftKaUBdGblSgNAIXVvGkCUlKFbEWOk6ElcNSJGBry7BlJWyWfMCm0bShM5jGqWZtShIAFAIxNiBwaL1EUmteJ4nks7O4SvaETFBsh15jAkTTA6QYll9DGQuwCbiezhzyGahwCHNmQkmtvW82ELXUBIHE0ihhbFAqCmTF110XarZixC1iRNIA6DH95fcYMdNOxRxDT8rgYb5bt1R5COVb3C285PwNw69JUD3IS3zXHctmHB+NEMaa8S5kBwLXnswYm184/1p/fld+q8o3Wsfz+37jIf/9//jc4FNRhSG4GunwipkGfTSHcAYCYdTkGYPaa2PkMfJ42K3sz4JSawgabsioeSACF0IWu0bA6zsBGypoIJvgTIa+1SjjHhsmCEpOhqFKnkHsAFIKocnkvICdlNSNJEZAgRClCmHOMDRodyn2kUA4o/YDLproGUXX7uvpEj2nyFtbSLrUHUTa5QtoxljHk2TWlHoMv8RRK7usYbQFNKatyvE5+pjZN8ZZKH+yKuU9AFi58AAAgAElEQVTqTAQyRBISGawkqOINJY9knF0jOa5yHzkOTOQaAEqOrsybUhkxaxiVEPkZ1etC5kiARgEjKpUQ6fq+sx93fi8bquUzyFkVD5/kuMqzq3N85Tp5X9+z9nDGau5FekbWUa6O1c+nrrOsK6Ap8/3rtuu1olbmXDbXKasCUGvN2loqpgYgS33bem7lWUrdZaJ9kSnBZHmMieZfJISkTPegvCGpJ8yFdallhUgiiXKMJXRbipCBSP/qz149VgEMy7HV9eVZlvmvQt/l97wAXhlT1WeZGwnJrqV16nmaNsS6MCTW59fmVzbJIiUkecQiPSQh1PW5es5D0CWUu85DlrDiusSgp/xj7l+Ketp48+Z9JpVS3XvqPPeZN+H161JULiHXJby5mtsHWsJ54YqoNrCSW1zn/T6mmZCCnm3o1zwDQnAiYyySNJg8pllkcKo69e5R5qTMVT1PWRUwMbuuzo+VvksfZTz1AlzkyCJjntO73CjX4OLXCIV9sMktG3C1TpwCzMdYXzv7IPH/Kj/ywcKo+8C5s5I/SeGQ1P6M6ITv80DKZfn8l4BjDdxU674eX700ATzMNVSV9205d3Ic6x634i1bzqd81utnIONYPt6Vca1ConpOFm086h17Eyh6h2X2Tl63dwBwMiVvam92/k33fdexP1L/rcdXyjvNw1dsuy7//B/+J1+71uPmD7+V//C//I9/4+3+P//eP/jaxwZ8wz2WEap4In02uA2b4u3bmgGfxwt02uMQKKTyld/i0hIt9n3oGGhFHGOLG0eU6sIwOiSLe99hw6Gkx9AiGQphHRKFmb32G2iVcNOccIgth7Ia3DQnZg1MuLQDjrEpG2mA8hdFGuPWEzvbleuLdytlhT463LSnAvz6SCGjrQ7MFEvtieeF6h+RssYpOHQm4LN+j431NPZ+i4uGPBOn0BSQd92ci0dN6N3B5EWNjjiEdraxF/pzAVfi/dlYX8JApTQ6PpCFAFCIkvZuIO8a04tv7cgskSxLMnRAC9wNHTrr0ZmA25GYFK879nTpKadTvDeSa5nylHcqDIdKZZx8UwDIRTNAA/j4nhgh39sfELPG2bsJJFv//7P3Js2yJFma0KejDe53eENMmVlZI1VF1wZEWgRhzaYXiCACLGDBmgU7WLYIsGLR/QuaFSv+DIIAIjTdVFd1ZWVmZGREvHjvDu5upqYTi6NHTc2v34gXkVmVGUWqyHvX3UxNJ1Mz10/POd+3AYSHxVLspfFkUVCr9W0OGqaAwkdnoeRKkNQVy8tofHUJ5bFl0HWYO6LVL1YmIoBKeHQWRiWcFoOr3iEmWV0zGSgSuAJ2FvBJYl4Mxo5cGpeo8HDs0dmAfe8we73ZGW+tG0oS0GQdVbZKtBYLH1UFIJ0JcJ7mrdUBk7PY9QuWsMaYspXFFUvDEnTtN8ewAmSRYYBMbqgBzlM/jrOF1ZHiKAW5o6pi5XCenkkp0xqXPHUwJqIr9ykkivH0XqHvfXUP5eM5CxhN80KIDO/J+mM0udKGoKrFhGNemaZe64jTsaeF836uYMt7haGjTZAMsjo4Z4As0A9LBTpMmtRSxTtngNHheOjRD0u1hADAdLLVFVaqhOA1un6pIGxxBhnAMCxwTm/iXIUA5scO/ZUja1GQiF4RSVEWCLOG3S30TAUFYwMWZ6BNQPAaSseNbmvwRNykdYQvJFMpCigTEYMi3U6gArSwKOQoobtA7WHCJUEkTLLMCymJFCp5CdWRRUTKEmubBZSNiIuCtHFdC3JcYgFA8dEAHcWS5iyQF4rFVWMgvVGRi7UESCeNtKfjeVaQe490MBBDWK0orgB6dr8zaQVIDJYcW214h0ACrdunJJfT7BTlmeV2Ua/zSnwkAbBLqsire6jMwCKrpQIyQziJ1BWwZxKRJHF9i0QeIuSjJhfIQibEVg5EEGmQJsKipDNk6Yf05EqaJZC6BLEIKCeRenL7FIkATbK5ujFmUwhiTIZ0BXyWOmss5ZjIDbOJb0MSSF2iOos7Z7a5WpPYXTSbDDlJcr+VQJYZ0ouVKMjQ99hlqEJwQy6qFMsY9t8dWEonkPoCtANo/Gj6ItriZqryGhsLVDdh5USNqZSe/magxkeyO2h1FW2AWCrEOK0rrD5RH0UuZTqKj91YsyKQm3jfZLltuYJTkWnMWhdaGahMjpvNqo33JRRD8Zt5M+5tP2v9mvIpR+0FAMluxILGEWUcnouxrK6wEsh6G9va4txLMYHVxbY5x2Pb7tNUV9i8LSOLdTzA9+AclHP7yrPc1sMuwm2M5QYINuWJjC3pU3o+78ZlNuKpxbVuCJS+no/NBQB5EZw+AzQ5VvQiKHwO3J9vbLT5v2bDSyTiH1jb8C2e4V9hH+nvKv2OFfa3NAmQ9AVH/EiRq/SGS6bKQlDsFrmgAmSh1CJWK+VQgCmfMyIiCYlBeWgZKaayuFhGyGo9G5SvboJkoSEX0iVpkrMoE0eLBC0aC1N5aVSrpCyWK5FIpiGT9a7KiohV8oAtNQxSbSHYAciKKQXFvGkZMWhf5RF2xtFCHgK2yC+0lkIqK9SxlbX8AAlV66L+xNoHKUj7sJWU4L5xW9prSF5jPa4KMy8kalu4P32RyuhUaNrI+lTkmsjyEwAgc95Y8bi8c8tiV0iM2AIXoDA28WlKEDjpivsssPZLIlc3R7ZGJqxj2KlYwfVgAlnkShuUTOh1qHGJnE+ITFIfBWSlLNDrNZZWiFwZfbtCRqTK9aL0SxRAxfWlLDbaXxnA0Pmqz8VMvWy547YouY4fuwrWZwuocYgMFKVAw5oLsqyl0kcmLCh9F0ICCCVOMVS3OT7PeQECmKQBlpBzqABWMZgSGVav+dlKKwuIBQBrad6oCoJo7KTMa35Bs5UtrG2MJdgVkI/pta/QZE0VItNnmWA7T7GBMkHWMtb7y7Fv6PzGKshzledULD+e1tL9MzYQm2+5zwBgbKjgSZXxZCucEBnaNM+JiWAts3pNHyrxE407KljM3eriJwTFRmoTSpwjX4fVIlXAsVIJqYBOIUWJfUS5L43l2NBCW2laMTELsRCZDE6CwBPHVQqZK7uxEBnKEEhXBfBzP3ImciQiVaK+xI4IpditNJV8UsUaGynZathRPhgg5kJK1UVI094n+svWPKkTLXrKGLSswmy9ygKQJiJLXrECUmWknCBUQjpzvROKgCLHKWbDZZX5U1Z+vDajB7fEeVoClkJlZJlWsAtAmITUl1WqLmCYCZkaEqScCdhW45QWK/utSciQSCIh21QW+AJIubLCMuttZeMtPxZZFzdKyeAwERlKyZJ0KcdkpAIWRW5iN4HVOqfo95YWy9S2JFdQls0KhAFJbTEoK2BZYyS/S0oCSDaVtgFgMpRSb1R0vyrhkljnAg0okbkwuQ0v+HkScH9FxsZ1syXkobGgecqssFkWUN8yFQNr/GuZi1nn0qYylRRt/CS9vS4rAoeilJfLPamWRGQC/nJtcwVebb4mLhIiVyKb3DAzi0pytAXTtYzIY7wC4nXcxQqWynzgOEpOq5toA84uAJ9nYyxF04dzMNmA2g1Y5b41QPFrLalo8ovLx74WWDJIb50jztp4DiwvAsgLbXoWWIrt96+75rzc9ypjU177ksTTQeI6nwOnv0t/b+l7DSxjlnjj9xgVAcNDpLiJUZK24UMYKgHNtZ7xdtkVkWqyWB5B2nAPhTX2MZCFkrTubLXkqe6IB0/WTI4dBFBj7HSJRyTLoa7i7+xGyuc4sXvloHzVQ2OB9KW4w/qkNu5qZLEMWHg3vzyFrBvnosaL7lStl73yuJsHDFpj0B73S4+dWWBlwCmQYL2SJKXC5Dwu6qqxyDGLx2CruyQADI0UCxP0TMHAa1XdY9kyOWiPk7dAh6r3Rtp0FPc2aNIdi1puZFpGQ/0/LBbX1uHgLfpMun8Hb7EU7TOO5Wq13WIBi+ekM63EyFysZRnEcqtkwsNcYkuLNY4YeskKOBhfXQFjkjguZB3jWCpTSGekyJi9rmQtR2fJzbPESCmZGoudgtVyE1OXssBU9L9icQFk8efD3NXrgdWdUskErVIlUElZYOwWhKjIKlkAsw8Kp9mS1cr6Yo2kOSlE3rg1E/hKGz206gYcJbRSNNaF8CVEiSWQBTOYgNmZeg+YhKXGP3qFaAMWrytYMGoltuK4RRaQDjpi8RTH5hYNrWnxL4AqCQOQBTHnlRVWiIx5stAm1vg9ZqVNUSL3a9855oliplbtNL9oWghZct0MQSHqtX0MaNiSOZ/K+0CvVscQJLJdXTBZ9gV5BWQ1BkpkxOZa7+heL7PZuH7yMVkWc1GmGre0aTvPfV8sx6phqz1pLI21M0VRpU6SU1iEAWvUpSTgnUYyRfJFNwLbIiN6JrSRiF4W2RGJZCOSX6VOqji4l8g8XyOz61JZ7M6ZokBKiayHQSJ3q9RNWsjMkzuBtChijC2LiBxXK2ROApgUSSwYWu2RxbIQ5ZYxSwWs5VkhyMJk6yWCzIBT5Cosiiunk+uCJgNJyzWGKpcF9lIskExgE4itGaEASwnEki8rCeEkWvfFrNNqmRQAGrmRLMuqtVgo2dKZFX3PbTtYRqJY+rIA5EnS4p/lSNjCUdxDsyrWzQTIYkklNtUC1BIgFwnhaSOpWixTkR8peSsLpyHrGDKQjKLul++Rr88FUBTrX4oZIhCwERF078pcq9Y1m6EmIsfJSoCtXtVyFgspTZbFYsnspgLSYWX3/A5JLQXFAtXiWpqHlFDcU4nRtgIKSeOvXLkfgcanAk9BFkygaVsueWS5pw3pDZUJqElAWJ4bxaKozwBEWsuUXhADdXhqsRRme1218MWykSHL+Lbgrcw1UQiAWBqmtdYyG2vWNBeELZskGQ1REPd9C85aSyCDvVzYiVvyng0YE+IJoMhSrIB1fXXRObFe/3UWy2oRPAOW7edvtFi2wLCpe63oDIi2x8RZm88AeP3exlFyai2W7Rx6BoC19W0A7dn5FrheTE1fnxz/pnQG7C9dI/KvYMX8DafvUVO/dfpeA0sjIv5i/BR/NX8EAPjz4TPM2eCNv8LnyzV+v/8Kj7HHJ/YeXyxX+PP9Z/jZ/BIA8I/2v8A7v4NLGh/tHvDT6SV+2N/hy2WPQS34Uf8OL/UR93HAvz58jE/6e0zRFpBKbop/OH6FmCU+nW9xayYkCLw0R/x0eoE/3X8BnxXeuD0+7h+gkGpcZicD7vyIx9Dh37v9OQDgzo/opCeBd+mxVzP+dnpd4yF/b3iHQ+wwRVOtlTEL/MkVlX2tZ/x0eokEgR+bd7j3A/6DFz/BF8sVvlz2+Pdvf4afzy/gksKPd++wVw4uafzt6SVu7AEuarzeH/CFu0LKAteKSFp+MN5Di4iQVRV3H5THTruiG6pxYybc+wE7tcAlXWM0H0OH39u9w2fTDV71R+w0kfccg8WVdni3DPjh7T3ufY9eebx1O/z46i3ezHu86E74i9vP8JPDK/zx9RvcLQNOweIPrr7CtZ7x14fXGLUn9tZCPMLyMWzNZDKeBIFxWCoBxavuhMfQQSLjq3mHKRj8x7//LwEA/+fdj/CqP+LGzDhGItdh8M9A7M9uv4AUCV+5HZGw+I7IfqLGi1simpEi4wcf3mOKBl+5XXULfmEnfOV2uLYT3rpdJcthTdWPh0cMcsGn822N5zz6Dn988wanYPFx/4ifnl5Ai4i9cbRJUDYdmGzn8+kKw3DCB/0Bn51I6P3azviT3Zf4crnCz463uLYzWdKLfM5cNj/6ct851rVXvroxs+v0wZOczaB8JQO67aYqwfPD8R6fnm6wN/SczNHAFWmYWzvhzbzDbUflaJGqyHrIEjtDLL2sN3u/DHjVH/HVvMNHwyPeuhE+KfiosLcOrriO39iJgH009diPP/453rkR94XAaNC+yPws+Ox0U8FvX44DwKPvq8szb9R8Ne+gZcJOL7hfaC4M2mMKFJe7tw5Hb/EnP/oJpEj4V+8+rtbInVlwNw81tljJhI+GR0iR8dnpum52sEv3FExlG/5kfMBPHl7iD37wc3x2usYcdLVA//jqbXFpp3t1bWf88nhdNz9e9idIkfD56Qo3HW2msbYuAPzpH3+Bv7z/EHPQuLYOuzKXtEz4ZHjAXz28rn17Nw/4YDzi7TTitp82REIxC1xbus9Hb/FyOOHobe333roaHzwHepfddMRW/WbaVwImZinudKgbRidvcNPNGPWCN9OeNHOjwquh3JdpxKvhhAfX142dvujssrTRn/3pF/h8vsKDo/v2sj9CioxPDzcYDa3k56AhAPxgf4+f3L/EVefwqj/ipw8v8OPrd/j0sM6VD3cHAKt3x6PvKwlR2+Z2vG/sjC+nXfVC4I2rq85hDhofjY9rbLzIeFy66oofs8CLfqpj6MqmmE+yjgNv9H0wHPDltIeWCYfFom9Ic667GZ8f9vh3/tEbzMFsnjktEjpNJFMnb/Ci3OOXwwkSGXvjcPAdlqTwdhpx1Tl80B/wi+MNRrOgVxSm8NW8q/fzwfUwKuLRdXjRTxAi4971MDLhtp8gkfHL4xUG4ysJEm/yvZuHTbjAw9zTBk6U1bPk6Cxe7k60YVmYngfr69g/Lh32ZsGb04jX44k2Jwu50ce7B/zN3cvvvO64HWa8PY51E48JqDod8PY4YuwWzF7jqiNm7JwFlrLx9bKQWx0Xg6tugU80931QuB0nYo0uz4kQuTJwZwBz2VRkjwwfFD7cH/BuHog8aSGSrZM3leSJvUHmshm66xacFoPOhLoZyAzZh7nbbNZxfDeRRwksQWPXLfCF5Ie9Q2avMRZPm2mhTbCrYa595lAJ5zV2/VKJk4TI1cuCNmkB502N5Q5B1Y00qdYNQ+912eBiLxXaIEPdPBR1E4sxB2kaFx1pQUio6keXTUzeLGSt6Tb2O2VRWa5raowFTDokZfEkaOKSFes8Z6ySSxwLfAF4UVwyl7tqJ1dALFG1k9t8UufNBiGAJzHnuWoaX0A17AnBbW+/M4DlNojyuWXB/bp0Bkyf5P8mS+M5Aj9H2bXcjIuWzN+lv9f0vSbvef3vvs7/5H/5T6qVj/9xqgQ0WM+xVQTAho2Rv7cpoS2LPvOPcXusBR2XSD3aMjjvOflISwhz6TzHaHGJuSmPy2YylPP+sDVPXnqZYH3pfJ3P97YsgVb6IJ0VywQk53l4bFqCkpbk4+vqfK4vbdsvjVM7jhf79Q195R9QABvGPE4bko2zetqXcHvsCdnHWb3Plc95zufXpfG7dIzrbtu25n/6o3FOwPJceU/buC33Un108HxL9D2OAc/+eJwTwTxH531pE/Wbftc27eDPX9OW59JKaLJtxLlr6TcX1Hy+0OB2blw6/uxO8vm59/15+KYf/rNxaufitn3r568di/Mxf+4+vHf7uZyzay71q73m68bwuXzvM9menYQX2vGrpG9qQ0uQ0+QVeWWf3JDE4Cz/c3VeKPO9XNjedz7g6Wvk7yR91/ZeSBfbef4svu+8eOZ5+FZj8uwi//lLnsFK73f9+W/S19V7oW3v16ez38n37duF4+89hr+OPN+23l+xjveq7z3L/Nbt/Tvu3//zz387yHv+4J//17/2cv/1f/o//sb7BnzPLZYc59gV1k4XdY1bBFYWUra2VAZSKaqMR2uNYyBZASoTiJRdQ65TZSY62dJ/A+villk/19ifAhKx6jxK5Er60pWddiEo1o4ZLhlA8bnW1ZTrY1B5zhbKu/Yru2h+ltGTY3bOf8PiZjFYYpzkdtEu8soiWsstf88ZRjlmkMEhu3Oeg2kAxe0yVUIZGgfKp1Wq4DXlXOWlRN5SrUuZyHulAR2pGT8GjUuh6a9MmcyWyrFYWUBrYvtsWTQ39/9sDHl+cHzXFiDjAmhbWTB5x5Wv551bPscL7pVEZc1/HvdG16Hstq67xOfMpXSdqOUCa/uojefg55yRE1AqI0YBpXJl7OQ20dhT+9OGaXL1OeJ+lYixknf9W8GIWMljANRdZyov1/7z7nQtv31mW+DTbOLwvT1nik1nn+uzIkqMUnENYxfVXMquGwF8b4qLYo3Ha+YERIYoQXs50m51CpLKbBaL7djSuJWdar4/xbWs7mrzerOMa4qyxgcio7Bzrp9liW/M5R7nKEkbseTbzF2OAxOoMWG0O8+DUBL3N4mnq9s67mcbC1msMYPct5Yop5EduLj5wHqI3JZWKzHzTSn/gijHBd1gnSupTU08Vbke1ik8X/ReAtZnzKkoz/xlptAm33NWgZaEo7gscizduZsen6suixy/KFaXxuriGASNF5MWc/sF1ti2KGj1UNxgyUWx6TJbPIoOYntsJWcp31uww+U0boucD01bSX8RNY6Rx4M1DbMq9aq8dYEUgAiiksx8l8TuksDaZh6n3NRX9R9rRjRjgeoKW2PyeLybubDB2k/mCKq7Kl9X3UixShNtfpSKpWv7AkN1R675m+s2hDNtG7nvXB63ofSlPoqFJKqdh229nIfqaJ7hJlW31PZ5zU0ZTR83r5bmGXjyXJy9fp7saX6HKfIsUBVn37/h+nYILpZ5/s45/36e2jnwTDu+dmMjX877jZsU73PuPcemrfs7bRh990f+d+lbpu81sHRJ4yu3wy+P1xuXofu5x8lZvN4fMQeNq87h7WnAy3HCV8cRAPBqd8Kj67AEIm65Ow3Y9w6nhVxNr3qH627GFAx+8uYlrnczfCAXPFsAxlVh7nzzuENX4u32vcO7w4gX+xNiknicuko6wlY7rRKmxcB7hY9vSY/u6GzJRyQdg/b44rCv+n99cV1ZvK6C4jkDY+ehZMLOLvjicV+F16fF4Ee3d7ibBzxOHT6+ecTb44glKFwNDlZFLFHhy4cRXUf6Wb31OE5dAVHk9rEbXI3hEyLjNHcwJqDTESGSG8++dzjMpAvn4ypDMjmL3nq8O+5gOw9bXGtCUOg7j2my2I0Oj6cOSiU4Z9D3HvNkIVXEy+sT3t7vcLWfcJwtgle4uZqwsw6/eHsNa2OVE6ixfIUllcf78dgjJ0mxduVHfTcsmBcDKRPmaUAOEn/4oy8BAH/7xUsolbAbXMmTMc+G3q1lIb3fzRAi4zh1mxi7lASG3sMVZsyb3YQlKJxm6t+yaIz9guOpQz8smE4dhCSmTyLgybjZT+h1wJtH0luNUSIsCrurGd5r3OwmvH0YIWWGtaSnFgO5BaniLnQ6dFA60dgeeyAL2M7jR6/u8O404OEwwNoAU8hpYpQ1nlDrhBAkUiJ2Tv6eE83DrvdYFg1jSDMuJIlpsuh7T7Ggxx631yfcPYywHbmthaAQg4JUEX3vMZ06KsdpCJlrvGBOgLYROQPW0ni62WDcOZwOHXZXM+bJIiWJFAVMFxA8xdvZPkDKBL+QmxQA3N4ccZo7uIniBZVOMDagtx4PjyOBzyigTII2tIGwOFNBE7O2zicLIYkwx02GyGRKvCEytdk7jdevHmFUxGe/fEEkK6Bzy4muoX/A/mqGVhH3j2N9l8miAxcW0k7NSeD6esLd2x1evDrg/mFE9qqCqBcvDnC+xOMGiX6/4HA/FCAqMOzp3XR66GEGj5zW+Edk4IefvMNnb26QvITpA2zVw8t4dX3EL9/cICca4+VoMVzPmI8W3eixzLrq1aUsoLsIIROC0+h2C7xXsDZgPlnYwZMbGECxkQDsuMDagONjD2UiyY9EiqFUNiIGCWkiolOwo4e1AadDBymB5CWG6xk5C7ijxXAzYz7ZEpuZoQzF7caF5DB+8IM7vD2MdN9UxjAs0DLh/t0OuisbRSUO9OaDE959tYcZPfa7GXdv97j94BH392OJCQXGF6QPyizK02SpvYqIlaJX6Edi510WigXvh6X2FUBlujWDR/QK19cTxYaXzcN5spt71Y9LjXVN5fcgRYGx3GOO2725mnD3MELpiGU2NKdlRvAK487h8G7E69+7h/MaszMUHx0klMqwJsAtGstssLuaMU0WVzvSoR27BdNC7vCHhwH9uODF/oQ393tYG9CbAKsD7o5DkcUROJ06aBPhJlPnIr/X9yOxEd/d7WC6ACFQ54zREYdDX98z1gbMR0tgKQqojp5TPxl0NxP8ohEDxewqGzGOxAJ+OPYYR4fH+wG7mwmnUwdrA0KQeHVzxOdf3HzndUe3WzA9kHu9Ku8dZl0+PfTQHc3hYeeInTkLYiAGMF7PCEHCOw07eHrXRpoPw9Vc540Abbhpnap+ZVyIFpbdFJOXuH15wMOBXO3DojDsqc7YbEbJ8mwCQL9bMJ8sTBdoHgYB05O2qz9Z5LIRIwQgS4y0MrSxyXM2JVk3u6SKCE7D9AE5CfhZA1mg2zuk4gprbEBYNOKsYHYL/KEEhUpUyRrdU7nRqZXB2cu6qSJMok0KkZG8ojjmsvkFlemaVHQ0eTMJWEEL5w9y3QxSmQAxsy/7srHE19dNgZKniytDM7BFfwzAmKQpNxteOm+ZnXkDDljjnNvEyCljJV46P8/M0LzRF0tsNh/PTfkMxEv8LxSqt0ELGtuNJtogKazR7LbbpCpn02yOPBmLtvxzoN5uDrTXXPrLY30u5XMJWX4TwP4tSRnbjf1/aOl77Qr7w7+4zf/V//ofYVQL5mTw1u+gRcQLc4IREb90N9Ay4hg6XOsJd37ElZkRksKdH9DJQAQ6ocNLe8KjJ2F5FzWO0eIUSPvxB8M93rj9htgGAB59h5AUPugPG/KeF3bCu2UAALywEx4DxRS0rri9CuhUqDFwo15qnA3HzLzoThWkcVuYTZWtKadCrjMHU+OHKG/El9MOo/HY6QVvph2uuxlakpA6W9NeD4caWzUFs+mjFLnmZevIzixwUVfAa2XEwVuMxleWUc7fqYA5mArQXdCVkZVIehY8uL7GyjHZD3+/n3vc9HONF7Iy4t71WILCq90JkzeVvIfrZcIddtfdd0X4vsQmCZFxXGxlBR2NhxIJP7+7RYsz7UQAACAASURBVM7Ah9cH+KgwFRKemCRG46vlNSaJgyO5kd6SjEUr6n4qGwQpC5ychZQJfYlRMWVDYewWnJytMhuqsZIeZ4sYJcZ+KdbhDF3Ie7RMmJzBbnBElFPInVj4my3tu46kPubFYNfT/XRB4XTsYWzA2Dv4qGq8SMuS2upasoSJlCtrKktrhKBqLEpvPebFIEZRNwb6YYEvMUGyEAzFQt7TdbRo1DohZ1qk8++ID6qCdSEIPDtn6kaEsbQQVTJVAiAhMpwzVJZO1do7TxZKR3RdkRsJROQSg0I/LLXvfBxZQJuVfZQXeJav96qeZ81DIXI9Pj12QBYYb6ZqFQ9e1esreU9ZfJnBrxZaZsfVqW6ABKfRjR7zwcIMHkqtlmp3tMTmKIiohgEix8d4RwRBdvDwC8mNSLWKlocHC329FPIeWtgykMmzgroiOZYYJLQNCLN5Iu8BgLwPysJe2lgXhmlRUH0kgMfkPYUVNDoFRAHZB9JVlLnGEbE+Yo4UMxQXItIRfSyWy4Q0l3nVB6RZQ9i40Vdk8h4kATxo5C4RW2oWtMBLgNgF5EXR4oQtZycFXHlkp4jAZhcgjhp5KGYamYFZ1QUfgJX5tFjKUMhvAKzsnp4YZ1u5ETDJj8oQs1oXnBnElsntByAWURefuVkwy1kWAo1MjNssg5GoDBGKdU9nyFkijRHqQRO5jmkWkLGQ5GgAOkPORZJiltSsYhXLEkhDgnQkGRL7QqYTyaKUzGqFSyZXCRI1Uz9YjkR6alccM4Rn8p71c+xykRtZy+AXBJPzxC5Dn1i6o1jDPBHmUBmAXIDYk7xIMiDyHgWok4C/+u7rHuWAWORGmLxHZAAJSN06XnJBZV5lq6KeabGeDDaSH1mSRAdQ2FmLJU7ERrKktbKD+mIOdB/YWkpERVgtp6AymDlWLaJKsTArrQxUZrJ5S95T5FlkKH1Q2z5xn7NeiYeSLvdpwWpVLeQ1WdPxotZWCXlqGwTlay2k1fW65BW5lNW0U0ZsyGlaK+tqicdTxtfWYpax9vWM6ZfziFZupEkXrZwFyHE/W2uwyFjZZc+A0BOLpWjaK7bXV8t5+S6LlXhDfsTFpPUZ5nG8mLieMys/1bsS5YiUSx2CCHNaMMnlfE36VtbGC+P0q5T7f/zP/91v3F20/5Mf5t//Z79+V9i//M/+h99434DvucVSiYQbPcEU6RAm3ehEgJEB13oqMgkJg/KImIm0Rugqt2EEyXh0MgBmhkIqRCjE2ipB7rZXZoYRCZ30cMlUl9uQZSGMIcKaThFYTZaeMi0jrgrZDyefJToZYWTEy+4EgCQ1mKAhQWCJCqMmrUtfCBaYlIYTgxkAmFWoZCmyAL45aox6Qa8CXg5EzELnQmWCHfVS2V0lMgFcrG6rWkQsSdd6euWp32p151UywcpYj/G1DIT7Mo4sD6KLPw0TplgZSSJFxHqdFBmpExiK5Ahfe4MZTmt0KlCZqpVbEVXLktvQqVD7V10HbZFpgajnr4a53gdb6mIXXduMOZrPXHcrkSKwSlKw2y8TMJgiEWKLi7Mt+RjEcRkxCYzWI4NclZVMkENZpIpcZUxiwxrM3wGg16vMxlAIStgd2eqAwRSpDx2RMqpuJdC4ADdtar8zC21UEaFsJLC2ZEoSXbGC9iZUIE/101+vFFmVxerObNQ6fpo3Tgpg5LysJcn9WiVMKGZXCXLx1s0xcs1N6Eo/q/6mibWd7P7MzLE8ngAqgQPfT2a3BYCgZSXkkWX84khAkaw2lE8VVl+en6yRmbOAtau8TyoECUomkl0AIARgTUAcBLoubNxPU5SQRfZDiIwgJawNT+JMWWqE25KLj2zaiSpZImVGkGT1TkkgqlzbJqSEtUVeR6dqRWkTgboIpVa9yqRy0btc/SNVBUW5gPhArKqikRuRRLKRlSBCDqGQdSJpklJXEKhAGYL6xXUkLau1JyeJsBMQJlVQm4o7rzLURi4ToM13bRKSzEhaQXWRNqi6WO9HBGgRXxaRQufVRbjsqle3v3I8a2oD5xEqEbutLHnOd/xVrrIiOYNkShqZDV7FJgajACAzopakX5kEoEkKhNuUBCC6hLgvDLrsRsxEH2zpUBlRUX1RNVYJtnKYjKgEkpbIXSJrRiyWFZOrBaNKj6iMyPIiBRglT4A1dQlCM/oAYBlIlrYrAJHkR9jlMuu1rJjlKj0hAGhUvcZkCMgkmwFJYCsVVs8sQLIr3zFlJUiuJGPD1CoSg+cCeGVjbSrgMDDQVLnIV+QKfKo8SWNxEjGvmwpnrvOQGR7cBmL0zXoFsitLKNUlyiZDVgVElnFPDZBrWUNFyZMMKlDh8lvAl9UKFll7skpzoAE1qsmbV2AKAeQgSrvzFlzx9IgriyyPm0iFITifgVGggpBallrr2oDO9r7KAozZ3fscpNjzmfA0ifXWNW6+Tbu43ksg7BJ4asDkk+ubcjLfxxYUtmWeAexNvRfaX+vbgDrR5BO1XkA870Z7oS9Pzouz78+kto73csF9TzD6u/TrT99rYDkng0/dLd64PRIErrSDSwqnQNbGWztVxs1TsNhrh79cPoQUGddmLqyKCr3yePQ9dnrBVGQ4euUxao8lKfz1w2tiN4wKCaICWJIukfgb9wq99ohZYtAef+Ne4bojoPK4dLAybghXlExwUcMFjdfDEQkCX877qp/I8iOfT1fwkba6ek0yKC6sIA8gpkshMnoV8NPHl0hZVMZKtka+mfa47Sd8cbpCSMS8yaDpL999iF4H+CTR64DPT3sA60KdQQovir8MOxhJLIKhMNrtzII3hQWQXWFJMkXCyITjco2+aCrGTKyIvQ748rjHzi54U5jgnNcYLLnxapVw1Tl8+nCNq27Bo+vgo8RVt8CoiM8PV+R6O6sKZtmNmBf8PkncTX11P2atxcEEHBcLKTK+WPaISeKHN/cAgM8fr6AV9e/kDaTIuPPDJi507BYIAHdTDykoFpTjKDsTMM1dtZb6qPDoLJTMlY3v7jTQX0e/VDGLqom471b5EykIZPqoKhvfvlvwOJMLrtURp8VUwMcWzjeHHZRMGLsFdxO5bBkV8fH1I07e4KFYPwm0UR3OF6kMFSurKsWyrt+FyOh0hA/kJssW16OzsDqgMwFHZ6trdLWCRgXnZbXeTouB1QEuUAzwg7M1VlQpikZmoHucyZ36OFuMnYcLCksg2RBjQpUJ6QqQ9VHVONX94OC8xqG4LDPI0zbhOJexjxJapzp+S1DVzZxlYyZnCYDqWOVedBkHdhtfvMaLqxOEyHj7sKtWU60jTrOtz7+UGbvBQQrUNvDxnFHdqHMW2I8zHo89rnYzTnO3YTu82k9VDiZGid2w1HpyFuiKG/I8G5jigrksql7/4esHvHscEbwiK/YY6sbE/vqEu8NAbmwmws2muCYqWBvhvVpJdYqVl9gaVbFU01/nNGwXVle+Iu/RdR5aJczOrBbgKJCTJLe5QKA5BnKP1DqSO7qg2NBhJBflxZlaH1t8WcKG2R9ff/SAkzPFzRcYRtpIOp46mAKeY7GOjy+POB17aBvRX804njpcvTpiOnX13T1c0eYdxy4vi64xptw+u3PVwg2RYS2NoVRkoc9FUsfuFqQo0ZUyOS6Y3SOFIEu+2S9lTqP2M2UBW6+j8eu6AOfIOh3LMyoksV2aa3JL3X98IKu5VzUemrVKuV7bzfCLrq7szJSZksTiNLQhLwDyCEhV4oe8EMi12Xt6L5PFOxbLfoljL/PFTaZqmaZIrvJSklQOSetIcrVk6ZwMyGJpjIuEunVE1hbJBVLoVN2Ns1eQOiI5DdUHZK9qmabziIcO3zVJGxEc/S4LnShmEbQxkp2q7p0wZKXOpY0AaJMiCwJSOld3/Fw2MFiCh8sDy+YA1RrPKSey+rOLbI4CsAkprNI8VBCqtI40Ecmr6hWALKgPosj48AYDUGOYOb4ZQdIGScbqgikzcqCNE+TivpoB0cXqLi9UovoDtW91C8Xqtlk2flDkapDFauEHmjhqAFEU8L7dWKkuqBeAFeuritLuzJsy5RyVizVe9Nw9NWON221BCyOcBuxvjjM45r6U+9dqarZA9ByUsjWyPcZWye3mAaoEyxNJkaYPl6yZnCqobMqoFtAnmZv+v0d6bwvl1wHC9vg5OH+urEvX/jak8q7/h5q+18DypT7gP3/5v+EujvBZQ4mEJSukLLFkBSsiYnlD+KwgkTBnWsz1YsGcbV1An5LFKBe4ZCDLk9QLD58VTrcdTLGy+awgRYLPCldyhs8ap2TRScprRMQpdhiVgwKVK0WCQkaEgCqzey5Wz1EutVwANa8UCafYVdIfIyJ8VogN061EhiptNSJSXeVYzBJ9aZPPCp0IcFkjZrmx7p6ShRGx6DFGzGXrkeVR2rwAxbUa0VhNSx7uu298RSqJ0Nk1Lml0MlRrn0saCqmW4ZKBlmSFnqLFUGRMuJ9SZEzXFlqu1kpO58y3bJltSZrYusxtAYBbQ5bju/0IIyMkMnwmTUfW7OTUFakBX8oIadXIbMvuZEACaXKu7UtYkt6wC7dpULSYm6KphFI+KQyKaPytDFVyhstgMiGWoXFRl/FLtX9SkNU+FbkXLWMF+nSvZM23JF3GicYqNO2nMV/zsvt2S5jFkiWyzHW2gANkKQ6F2IrndiXVOrMY8jmWk+FrW4t0tXQyc29e6+pVwJJUbb+WsVqv2+P8vJObuqr90pLdg9d856RfKa+eC2z9fzWcalu47ev8yNViv+z1xsLfjgWNVcBpZzHqBfPebPKxFb31Llhie59SdQFvrdEMcEe94Kab60ZXS25mZcRN2RgTIsMnVeR7JJRIGzKyatEu74j2L1/H+VtmbS63JRzj8WlDBrQkT4qwl7UPnSrPVZElYe1Tbm9LFNapgOtebsjPAMAN84ZUDKDNF9eT63ynA6beodMBrndkuckCnVrfY6psZLD0CkCbRIbHm8uVCW5QT1i7+fkzzZwnrwuxIfVizd104TiAek6JjNCtmr4tUZqSCWFQ6PTq7dAmzlc9KPrVRb8d09hTiIFWEZ1eN0ylyLC6eEIkidiJer+qF0Ln6+ZXykBnwmbDlUnCgg3VQ0KKjNj7mqfOl5E+53KcSM1WHvdkaQMhWvKmaL0DtEwQv4orrEoIdg0j2IyhXWP+ebOoJUpTZXOB2cb5HG+qXSJVq6zjZxYz9qKQlaCtEKPppwvWbAFk8gJIhkUjtyhG6XSxbv6bu9UFfrP+NyUPQHF+ZVxyAZtCZALRhTgoN/WsxHdUWNYNmV0LXBpUlXVZlF9EZfkigKghB5fK45MazwOQUt/Tn+xa8JkV8uz7WZwk13nJgMefV6tgIdq6kDdvLmj6eaELBGZwGZA1w8Zju/lb0hOA+J6hdO8bcfetAOjv0m9t+l4Dy1Pq8H9Nv4/H2G+kQXxWdVHDi3oChLkuPJldNUI+ATXtQjNlUYBRWbiWemIWMCLVhT/XZSRZeDpVwEezKG0X5CGpDYMtL7a5Xl7gt6llaAUIDEhRrHTIGxAB0MKF3Ws5djRBVA22lGVd9PIimBfu5wteTrz4vCSX0lo2t+1c8/H3S7IqIasNAGSNRSsDUpYVTFfwc+Et1IIZZt09z7epsyywPldXAIAlqto/7mu7IF/vz1a2BsBmkQSgWoX5e2vtbtmB23HmjYK6gdAsxtv8bT/Ox7eyCzdtZ0DFmoHqDMA9tzBv28B9OpeuOc/bjt2lNp7ff16Yt795fL5dwPBC/JLMjxTkhnnOeNwuyOtCueQ7X0TxOLQLKh7P5wAQl8sszikLhAbg8Xi1qQV656mV6dGKyK6UHBHT9plSMj9pPxNsUZu47Zflj5Qk2aAt66+o50Jc29ber3Mwduk+PPd884wRJV8LyNpz7QKHx50BBPcVeLo4b8+t7Mbb8/ydXY8pb+lHse4x03HLYFzbU8FWGZtmDDnPpTZQudv62DLO4KQCiLydhy0L9HPH2/K4XW19zKarSlxzW9emXe2CvfSjLZctphB5/Qys34sL8rrgXctBc4/ypXNn+WoR51aKs8lSQYhAIbZpxqll3OXLZK4WxO+UBBqm3rw5ziRS7Ti21n1mrH7iy3d+rpbZgKe2/+d/z9Olc1k0hC/5yTjX69p+PkE7F+q7cO/SRdD3tI76kctt8KG41D48Y4271P5Lcy9fOP3c/Dqr8/yWfStw83V9ee8yvkXeZ645f3yevUaUZwrN3Hsm/Z2AwW+R91uN4W9T+r62+z3S9xpYxixxHwfch6ECQwKCBj5LDI3lxGcJIxKmSBY5trywFegYugoOGawNitxPj8FuYiBX6wrt/BH4ibWsU7AY84KQVLW0ACsoZVARksRYYghZdoSBppYRp0BblG0cH1tU6LjcWGE4v5YUFznqBSFJ0vkscZshKwS5gpY5mgqy2boCoOljqmAPWIEYxyi213EbGVhUi2TUNTa0XYz7pCrg5TFhYK5lQqcCpmCQlICLNFU7FaClwBTME7DFlpN2ccfXtQtcvlcAqjA7uy7PoVhsm7hJBgDVCtIIj7fHGVxwfgOajz5u3XXZXdgn+eRFz2W7sD6aIUnYAlpMYfMFtkCLgRVbrtgt2Je2KJEhC8HSEhVEktXCyeCLx4n73IJTTiqLeo7rjyV+lMeEGYfZYpKba4yKWIKGbsaJiZY43rNth48SRiX4KGF1rKCNJWhapmVuK2ukcn4GXEqufVpaoWvgCRBrwRmDLGZHBrAhTNKK3H25zW3ZSor6zPAPuVYEHrmsFrC1YEuIXOYYuZhv4yfjE0DlG0CbS4wnbyIwMFtBc6huw6rEsa51r5JGQhDY4r5noLq2cmJgxGWlJJELmOLvQqwAjfMz+RNbbKrFpbkHWeYKzKgOKpPbhbP21OsKYGIgyXnqfCx1c7vZpZmPr2Aw1mupzKdAkvrVAj7UcssIPQWyDZA4B4CpxGLSAptc+GpZZ8e5DTkVa1SUFeSBLZrlc8qCPPVKfr4WDVDMKN6A+WxTMTf5eRxxZqQBAH5XcH4Kk137jfUismCJi59b4JRr3+m7EI38DYPKCuSwybs53rxs89lGzbdJLQAUm0ehoI/cgHlu2xMw1rSPL70EWp4McElMFsVutwx2W1dMnN0/BlHngEE0ZZ6ndtzaMp+grPdIl4DbRaD6zOf22HlZ7ecNiF0B5Tm2O29+O32+MV1s14XNkWfG6FuBsfOp+tyYibP8z9Tx9YDywrV/36Dy/yfpd66wv6XJZ4Vfuhvc+QGhAElfwNwcTQVWUuT6ncFXrzyWqFdg6Tt0OmApi36OcwxJ4n4ZMGj63Lp7MfHL49KhU6ECppO36LWvDKwcY8kuYQyiYpKYTWHtjLq6iEmRn7C3nrt8te5bWlCM2GHp6jEfFYKVlcHVaV1j5UzjznXyplp2TAE7LcuqlmnDvDp5U8cHWF2/fAEqrfWJr2EgwSkUyZalsMS6cn4JCrbEsWmV0OuA42LQG1oExyQwlFi6R2er29V5/CqPYcoCS4mDYxACoMqeSEEgoHXBvJ96YkEtFh3O0y6yFqOKZeqpxZKvA8jVKyZZgU1KtEBfgoJWK0hqF3GLobqmxWysfV4Hig/SEbPXNQ6xBSMtsGGiGQYkqhDULEHBlZjW9vrzxfK5JevcCitFrtdTf1IFbMHQvWXXtJRXC5GSqeiTMuBDtRKxaxewBQShxDOmFOr9bEEMsMa8MUgAgMDjz/FFciXaofJErbO1gp23ha/nWLP2MwEJXZlcAZKg4PgmVVhX+RdYFpKVyjp7Zn1qAUgsMW2pSIFUvUgAyUqwSx270fllfZ3HErsYvKqucimSRQlZQAwZzmnkRPGMLaDjuMqcRY11jCYUyRgCL+06geUWKF41VmKhUOZhjSss+SswLLGN/AxwWSy/wxZJLkuU+6v0KtuhdESKqi7kq0teGUchco0dhMg1TtUvut6jTd5FQagMpUj6BQDColYLX2pIwATFBjIQohhL0gDltrZzoOqbJhStUJINYfdBXtelVGLyyjyQBQhy+XXO1DatwCUG2VjO1vNSFwmWUk8tv4AMjqfLUZKrYhTIah0XBuvJy9I/ieTlSlQk85ZZmN+NSUCUTY5cGIDB87GMNYplsd4PBo3818u1rbI0OTZ9Z/IhlSkmMa/fsy9xgazNSegX8E+9Bd43ZZakyNiyhAoBBIEs+VxG1U1lvMH/RUHkRAz4CuHSExDa3NuNlZQXpVKs8Xt8rL6LzoAOt6O14p4BS5acoLKxxhvytbLtD5VDshR0TDCwLhqnPC6i3KdWS3WDcFpw2wDZ2vzGCsxltXGUFdM1Y13LzajhjheBWmrKFXgehMqt5utF8FV3ibgeBv1n5eFpMZduycXdm/OLN6D6uXat4wB8A8h+ruzz9BxwbcbvIvA835R4D4B10WL8u/Rbm77XwNLKgB90d7g1J/isqkurzxQ/xXGPbeyUMxqpMLm2bqyTnVcLGAiYMCPrqH2NcWOiFZ9UjbXba1ctoFSWqe6m12a+4IpJrpwpE1Mr0MZx5WoxJQkSeui4bc/FFGqRKusrWwx7FapltFcBc6S+W8WusAI7o6tFsrVM8nlio03V7XZnyAW1dTllltk2bo76JJ+4ybbf279sbVIywVtyhe10oH8qVFBvioXWqLixTp67wjEoi1ZugBvHkkYraxtzFhgLwAcI/LQxkAxeKysqM4OeucLyta1llGPBVitY2lgwuU3cZgb9tgHiDN7ZssWsss+5wvoyngz4Afp9GIyHVwTeldi6M7cMw+dtamO7ePOALaQA0Jk1HstoslhavSWsasFoO56tu+7GXVWslk7ehNAqEjtsFtW6yZ+NWt1L2w2EmCSiYSv5yiYbtKrWTtqI4H5jYz3NWSBqBtXrpkELuM8Zcdt7IUtd7XHWQOR72JbXzlUiRSFWW7Y68pjzRk27CcBAGEAlg/J63RRp30JGRaAvFsCyUVRj/2QCiPOJ7pOOxARciFouxYHxxgmf5/a0bp6ty6kQGamQhrTnyMqz9o1BKPeNQTgAJM2AeK3zfCOE5WhSkWZZY+LCxmJJeYk1V8q8qbdto27iw6RMiHLrNp2UqOy3Gwttqbv2NROQkorHAxuLZVbrClfW2DtAyvV4BWJyHVcaxIwstxtuQiagy2s8nhBPVoAZErm8W5Js3geixHhlAWFpPIRMgFnnNE2qtK5Hq7UU1aqXZWEo5vlt43ou5U0+AqvlHWiZ8hPrAyCJXXcFOgUcMABhwMv6h2ad/YLj/75jErLILjT3oI6hkJv7UwERDyUDX4mtiSznxtrYVtZ8b+ti66jAGr9XwM8TZCTyCmyY8OYc0ZSxpRi8vB4XbR5e2TfAFNhI4GSUPoi8+vQDK9GOLMC87WIuRDwZ5BFQ5hsyu0xzO8p8FuVBOVtX5dLGS92vOS+NbTPPs2iGtsU+PD7tPXgPQFSrOtvHaG97dem+cP6iefESID//y1/P8j6JsbwENJ9DvDgb8nMkfKGsJ+U+KeTC9wuHuJxvHPJvcU9+0+l7rPT4jel7DSyNiPiBfYc5GcyZfuU4jpHDzH2JO2RLYYLYEOAAqPGJFYyUuEv+fq2fgsPz+EIpcgW30dD1nJhAiIHvpZikS2mnls33dOEtw0CXPwOti6TcgkLN7qlbUpJ2fM7H5enYbetY6962dTtWW7KX51JL6sJ1j3rZAFY+HvTlcs7H89LYXsrD95fB+Qr6nra9dUV+0tem7fWYFlvQbbZtuxTLytZwYEvoAqBKsLRxnpdAIVDxQQUxSqQa/wtgs/i81I9Lc+5rU5E3eZ/53bb3PNb0vB9Gpg1oP99Q4L48cS9p5Gjaspkw5dLYX2oXp/Pv7W8qH933q7yQAJAbwNnWp3kRf6Eersvq4ureAMnzGFA+bhFxntiSfOkeKBmeHFvPNXGXmst9Wn4LxKHi+hnYeEZcSlmmy/fsQtIyXbi369+vm2NKpaqVysnaLbDgsq1dZV2YTVcWAqIm99oufV4O/SU33PU4f36urxuvC5Vr39b7vdb9dWPGMZStLNLahvWYqGQizaZcszEh21UPgzagWrxx9tzkLDZrW7Z6icaiJ86aLBpCk5ovN+6ljHnR1lMbuJ5rQQeDOH4oGay1i90kNnV/65RRAOWFBXED7oGyUVLb2LT/CfpBA/LPij5/wdTjDWCsZTafa3sL0C737Xlr29l1Tzp31uZn8zV9+bp+VCBSQCXPw/M6LrX3vH8XmvqNqckj8oW/52XkX09dz557j3Kaqf6r1fcNt/li379Dee83Nu+3xnivsi6W/x2v+136zul7DSyRiV31Ss0wKeJNIAKWG3XCKB0+8y/QS4/7OOCFPuKNv8KNPiFliTd+j1EtMCLiMfZ4bR5xH0bSucwa92GocZd/OHyJX7obDMpX9lUAmKJFgsAH9hGnSOyqx9jhI/uAX7gbGJGw0w5IIO3L8ss6RYO9chiUx6fzLQACkS5puFRYbZPC6+5YGSSPoYOWEXvlyCJbLIyHMECJjGOweN0dAQCHYHGtHf7t8QWu7YwbM+Hz6RovuhOsDLj3Q42B+7A/YCoEOfd+wJV2G2KZY7CYo64xfju9wEWNh8IsylIu12bGwXewKtRFXq8CTsHgVXfCFA0OnmQueuVxChZXxuF+6atldacX3C89buyMKRq8m0d8MBzw1o1Vj/OtGzEFg4/GRxx8h175xhKqELKscZUAcGOnEne4ape+cyMxS2aBazvDyoC/fPchhMj4/et3OAWLo7c1LpQsx7LGrt67cWPlbJk/D0tXpWGOi4WSCXuzYCnxpIelw946nLzF3rrVShup7cfFYgkKNwMxdhoV0amAd/MAU1yAb4cZMUm4qNAVbcyYZLHqClxbV8u66oiS3wWNd8cBvfW47kkGJRS3Z6tDBVouqgriOI9uLFqT15vYRSkTdtZj8gYhSozdgoepx/UwY/YazCDJGxHzYjB2S5WUyVlgPqzXkwAAIABJREFUZxecgiZWy9ImtgwO1uMwd7jqHd6dBgzWV8upCxq2xKTOXiNEhc6EavV8OPWwOmLsluoWHaKCDwr7ochGyAQXdI2LJPdlcoGei7TIYD1yFnBBoSsgi92ZAVR20sfDAAC4vT7V9i9B1+tTJgvhcbJAFtiNrs5dH0kewjQu0tNscLOfcf84YBgWaLUCnMfDUF1CtU5YnMZudHWzZ3Lkyjr2C+bFQMpUdEPp+uNDj901bZj5oOAXkpFIScDPGmORsghBorMB02yqnIUxsVrH2KU3JVFlNZSOJBXSeSzOFL3N1eq4OIMUBWwfqtwFWzFDkFUGQ2tyIWXJCnL5zVhcYYi2Ad5pKEMamikJijEEqianv+8g+ghlCFhGR5IKZrdUF1pm4wxHA3PlEL1CmjT03iMcDdQYKrAKk6bVXXGFFF3aSjeohMRSFKxduUiIIVa5CCEzhMpIRZoiT+V9xSDI5FWyQmTkWaHKH6gCJlQGXIPaVIZwErmP5Appi7tnBqAztWEMwJ0lF1HWu2QsU+QvoBPErJBNglgkkASkBzFxqozc03F5koj7SJ8DIKJA7BPkIkhn0GYg0l85lY3conspHY1V3EWIommZdYZYBGlBDvSZ5R1SV9CTALUJQLYJ6ihruyAA4QW0ow7FLkM6gTgm6JNEtBnSU5lqEvA3391iqSaJOBSvAy+AoqcoEhD7DOFpLJUrepZiterpSVS9ROlB1maJ2q6qJVnqEh7V3TYrgDFqLt/No0AsGsdZAspR2VVyAoCIdEwAkE4gdRlyoeuzKH0ASNuy0TkUgfJITxbFrAE1cxuLjmQCksmQy1oGBKgvqilHUz41C8S+AMlyPQSoPYLyCdZCVavlUITSrlyOc74y7q2OZbU2tthTUX7+C6zjw/mzLHnK2LUyG1kAMqz34glAFmtbsqDv3F6RaPyBbTsZp9d6uIzSltq2dAaquL7GTZnbjqYPbZnVIt30/xLg2rSp3QjI2Owr1LGT67lNugTm2g2C8/PNOZHJI+DS3yep3RR5rt7fwpSBf9AxliJ/j+2xV3/2cf4P/8V/gcelq9qIFEem4QJp/3Hc3rwY9NbjVLQDx26B87osrCNOjjQGOTbMqFhj+x5PPawJNVaMFyK8qD3NXV00GbXWBQDzYmCKxaK6UJXFHOnP0QKO62ULoNERJ2drTBdr9p0TZ/CuulERk7PImY55r3A1OtL9W3RdYKZECzaO2aSFYCrHY43TEpLiXnRZRHL8lfequooBQAgKxsSyIEwbQo4YaffdLxpSbRd/Ssf1Wq8qGQcvSqXMsJ2HcwbWBgSvkJKE7TxpA566TTxWdYUrC1TeHfeLBjMvZgDIJMzO8V3B00Lz6vaEDOD42JcYq1QXqawrxnWYjuJpY6C4rar1VeK/OO7P2oCURNVvo5g16p9SaY2HYuZOQYtlpRJc0TjMGchBQlkCFcYGLLOGkA2BTGEirCLzi4JQJCrPcWJSZwyjg3MawenaR3Yb5Nit6pZT4pu4ffwDI8siWkhAKuprWhSkJctaWBR0FxCchiwuTymKSiyhdEL0kuLLAi1sqz5bJk01ZGovx3QpmxCdguoDxXWlspjTqcZySUPucxwnBgC6D0hRkD4byEIiivtkcGXVUhbpslieUpDVLVEaaksqOnFCr/FeHAOGvB43Ow8hM5aHbrWo6IS8lBjLsuBQXdH1m9cNkEoIEkVdLKg+Ih4LwJl0BRoAoMZAfc0UjydNQpr0Oo5dsXBOanUl9LK2y14tWB4t1VeATC4gT/ce/kDgFzoBTkEMgTT6bKpadVQBAFNWFl6Sfl2Qa785biyVvgmQO6VKSLMmgJSxxpJx/Joqf7lts6K2RwFRgEZ2EqJLyAUAQeRSnih1ZZjrBcFpyiMAYSOBuoOhtgE1Jk3uQj0uu4h0NJA7j3RiVXhA9M2qVIDK5Xsm8grqAKDE5gobqf1sJePnoegcir5oG2bQ3POyth9ZAF1cx4djI1M5Xt5pNC5xHacgt/V1EZgU5JVHToLujSrtlbmOLYIsoLSMs8hr/GUSwEzzSdpY71+rg1hdPRdJYxLEk/knyvc8lTaI0kadVsCs09oHv1qyGKAJL5B7Gj9kArZZ57WuRZJr7qwIbHtZ+yuGABwMvmvKNkHMBeA2LraQBdwbAsQVvJf2AUDuEsUyBopl5WdDREHn2nsssLK4oimDraFRII8RcGVuB4FsU9V55M2ILNe4xmxW4FvjHksfCNgXciiRyWoeUWJBqV3ZpDXGUa5lZ1PKCwSyeRMBQI2rpPZlCMfopAFYpoDNIKr7sIhnAEYU0BMByfn4HAMcjretqBrrWMlyXyroKvVwjHRCBdYMDtebjq2OJdbicymK29eC/9rHuMVVnP+SJbqWx19k8xln152DzDYm9sI1DD43xvJLoPCs70/SBSD9TaDuvJ4sLtT93MmK/n91MPZv/vv/9n/POf/jX7mgXyF1f/TD/KP/6b/5tZf7b//Lf/ob7xvwPbdY3ugJ/+Sjf4mfzK+RssCH9hE+K3y5XOFuGfB74zscQ4eddnjj9vioe8Bn7gYA8El3j7swwkWNnXb4xXSDj/tHvF3ImnVjJrzQJxxih3/18DE+GR7gCikQkfoovO4OAICfHl/i2k5IWeLWTPj56RY/Gu+QIPDFvMeVcRu3Ui0SHkOHg+/wR/s3AIC3y44sojLCyoBRLvjp9LKytF5pR1a/0EGL1YXupsSGXusJP5teICSJGzvjfunx51ef48vlCl+5EX+4+wqfzreYg8Gr7ohBebik8bPjLfbGYUkat/aEt26HlFf9vVs7VUZcALhbBvTKF2IkIkp6YSd85UbsjcMcTY3BPIQOo17w5bTHziwY9YIlKhx9h5tuwjs34lV/xJ0b0KtQrZV3bkCvPX443uOnxxf4oD/gwfdwUeOj4RHXZsZfP7zGoH1lo10KGym7smpB1pkv5z1CktiZhVgwIfCqP+Jh6SFFxoPrsUSFf/zBzwAA//e7T9Brj9tuwsPSQ8uEB9dvYgM/GA6QIuPOkYVqDqZq693YCY8L5f9keMAUDd66EVZFuKBx0034at7hxlL/dbGYmeIW+dHwiJ12+NnxBQCKGTx4i092Dzj4Dh8Oj/j0eAstEnbG4RQsXNSwMtb4zy9PO/Q64FV/xBcnsuLvrcMf7b/Cm2WHXxxusLeushUvSW1Yd5nwia2lLura9xs74xgsBu3RK485GtzNA277CVpEvJ13+Hj3gF8er6tF1yeFyRsYFev9ve5mPC5dIWKivzELjIbuH1uU712PV8MJX552+Gg84M4NCMU6OxiPhVl9rSNyqWDqsd+7usM7N+LREalVrwN67bE3Dr88XiMDdUOq0wESGUdvwVIYV5Y2fe7mAUomjMbj0XUQImM0HnOgcdnZBcfF4o9uvoIUGf/v2w+r1XRnF9xNPWQB/UZFvBrIc+Cz43X9PbaFIGsucyFlgY/GA356f4s/uH2Lz47XRXqE3iM/urrDKVjELOGCxnU34/PDVZ2nt8MEiYwvjztc965Ym021ev7piy/wb+4+gPMa+27BaBbMgUidPhkf8Fd3r2vf3p0GvN4fcTf1uO4djoutFtmcBUbrK6HWy3HCYbHY2wV304B956rWI1uFr3uHQXu8KfOUNwNTBvqyuWd1wMlZ3Awzeu3xdhrJuuw1Xu1OSFng3WnAi3HCw9xX63lfCJucp426P3v5JT6frvAw0zN520/QMuEXD9cYePOvWNY/uXrA3757gX3v8Go44Wd3t/jhzT0+e7yq8b6v90ekLKqr+oPrqxVfiYzJa9wOcx1vITKurcOb04jBUF9DVFiCwlXvMHmNj/YHig0u7ucPS4e5WPRTBm6HuRlDXbwJJF6OEwBylV+iqs+JEhmnxdT+LUHjup/xxeMef/zyK8xR43HpKnuzKQzcR29xWgxejBMeXYcX/QQhMvbG4eA7xCzx5rDDVe/w4fiIz47X6HXAoD2sDHjnRgzFW+PRdURk5yxuSzsf5g5WrxqpXxz25IEAknkajYdREW9PA3aWnq9eByJUK94M3KeTs3i5O2GJCs7reu6mp7LvpgHX/YwvDzu82p1wcETOF5LEh7sDfvLuxbddbtR03TvcnQbkDPTW1/huqyLeHkcM1sMFhevewQWNmERlbH4xTvBJ4uQs9sVzxAfyDHkxTohZYFpMjfu2OlQCv2khMLwyZit8cvOAr44jeQ4EjethxrQYhEKmx4Rys9fIWeBqmHGYO2qjLzwPRU/05Oz/x96bx+yWpPdBv1rO8i7f+y137Xt7enq62z0ej+3xzNhjbGcSEyUREGMECThyiAgCLGQp/JEEmSBQEEnEIiRAQkoUyxIQRZgQGSWAiKIEr5Ez8diDx56tp9377dt3+dZ3O0st/FH11KlT73m/5U63p3uYku793nNO1VNPLadOPfUsv3BADDgf9dYH2tOGo1UCo6KB0l10fOkP0sdFC22cRYq1DDvjygVIswxFplC1EnWdYTKqsVy79ZisGKxlKPPWBStrpQsAZhiU6gKWCWFDkLC2FVCtO5wl3+VgdaB4DxKHDgnDYa52AaQYEMy66Z5uOXjWHXyGxLpDYx1HE7fdYTPB2nB/sELBqCgytVbc+zp7YTg+IOJ96SrQA/pm2yQr8y7YFdVrfdCwEPAqiiJNbYBlXRCrIYmOhLcQXTi6JnkuFW5JiE+fU574HtEhwTVWGW/wgr6wSgK2TZ5/O70v0wdaY/n0d+/aP/d3fginahzgRlzgHo7aZJjIOmA7UjCfhXaLWsEVlBUBj3KpcydsxbAWTKM2EktVeHB5FmBLCM6EAOcLoXq0yD/S+W66hToGpCezzR1ZhyBCcT7JDJYqDz5uFEwm9beT3rwz4xpLghvxeXPuNqgOtN6BxTdaopRtMHclE1FlRRAUAAT/ujzxw4rB5o1loVyjJSTXPTD5xrhIt2RKSzS3Be8hYYYgEkrR9qL1kmkoZx3wO33gyKSUEvVRgBuJVrwYX7Ixji7BjZCwE+NNxlifNDeADqM09pGLoUxGsg0mrpQvDVYUlweAUraBL8B9uJThKPzHPfMCahwIJy7PmIcb8UIMwZYQfIuyXniJ+s6Na+eraWyHH8mjOQugFwGYUuxr2xqOQmjUngfAf388jdg3l4IApXAsFBjIAj1T4VzoEMGW7vcCzvj+iiMma29aS31D9BvfL+QjSAJXHEiI7rWaB8EwDhxEOJD0mza9tAF0fHX4ksQDbUTbKKpwGpUWcBu7tRcQGiV6eJpxECM3bl30YuN5d3WIEGyKytNmjzaWmdAh+A5jLrhQ3UpYP170nsbjFpvxxIGx4nw0z8NeILLYIEEhjURMwX/oHkUWpnExhgV/yRjOJPZtdP3o+jSXzjIjjtAKOEuLNHiP8NFnOTcuym4re5GAyWw2tJvbEBmYxja2aOkF+dkCrULWFLHfbEyT+IqxNONIwmFOhijJLAr0Y3u/tRLIchXyp6kHgxJFaSWLFYruy/wGP45SnF4b4yPQRnPb+o0qT+a6ha83yUcb2xRuBICHVyHYj64exhHu0YabrEkI45IJF8H2SZPDwaSISd06SDAogfcI+iPMT4qaG/iJBQjTyxv6NbxAiZrJdG3rCQTp2JJGC+i0Wj31mf8bRzylOuI6081+ep9+x21HVOYqWicaQ3hNdfS9YZb1o8IOlUufbetDy/rawW2at4FnW7VtcX5fZWxW28830CfRdfwolsPOrTvlYwtfg8+xVZG6XSN5SV4uzfMVaF45L4CX/5O/8E3X6hXP3bV3/9q7r7F89Se/rbH8xpMFBCxO2jFay3GnOIWxDEdqgkfVFM9PH2GtM8xkhcftFHdHNeaVC2eyNz7G47rAWmc4yFc4rCe4Vc5x3Djt2UTWuJnPMdclXptfw9OTEyx1jko52BIDhoN8BWUE7jW72MvXMGCY5ivcX8+wO3GntI/qqY8Ma0JAnYIrLHSBZVvgzugUygg81hNMuAoC7UxWeEVd99oXjv1ihUYJ78fYBSS5US7AYbGXrXBcj51GrlhiUU/wsZ138NZ6HwtV4MPjI6fR9ALxRDRoLcfLZzewm6+DUHTWlAFKpdbOhHYiGqx1Bsk1Hqxm2MmroLFcKIHdrMIjPQ1wLhQBdqU4yqzFqhojF2vkQqHSmaMrV5g3E9waz3FYTcKJ+XW5xEk7QiEV7o5PcVyPcSNb4MiMsdYSN0YL7GVrfOX0Fka8xVo5/zENJzzlXIW/xnLcW+5CG45x1kB7wfxWucJJ7eo4WxWoW4lPX3sDnFn8+sOPIOcaB8UKJ83Iwb6s854gvFd4bVA1BYd1PqqyRWMF9ssVjiunXbk7PsG8LV37pMPk3C9WeLSeYjdf46iauAilXkiWXlu0Iyu8snBa+NYI1FoELevd8SleWxxAMoOd3Pm1VsppA522W+Od5Q7yrMWN0QJvzfdg4QTWT+zfw5urfbwx38c0r0PU4UpnQVNH2KGxxrI1wkHCcIODcol5W2Iknda6UhkeryfYK9coRYt3ljM8PT3BW4s9TL3Gb62yoJUlLeqBH4NMaJz6gwQYjrEP/jPJHG+H6zFujJZ4Z7mD25MzHFaToJGc5A1WrTshjzWWlRcaP7xzjMNqguNqBM4sSqkwzWvMsgpvzPeDxnKctSiEO7WfN0UQnq+NnGbs8cqN005e42g9Ri4Mpr5u0uqdVSVe3HuEnCt8/sGHgvC4U9Q4XI5DMJxcaNyZnkIyg9fn++HjTQcHyyZHLlXQuH798Aae2z3Eq2cHAYYHAF7Ye4yztoQyHJXKcFAu8ebZfjhsuDFx1hTvzHew631yl02G3Aukn7rxFr7w+C6qJsOsrDErKizbHJIZPD05wZePb0FpgVlZ4fFigls7czxaTnAwXuO0KoPG0hiOSdGgkAon6xI3JkssmgKzosLj1QS7pdO2tYYHn9Xr4xUmWY37yxkKb5LfeDihWFO1qHPsj9fYyWq8vZg5c/8mw+2dOYxleLiY4uZ0gaP1GEpzCO4iSWvLULcSglv8wK038Mr8WtA6XxutMJYNXjq80dPkCm7w7O4Rvvb4JnbKGk9PT/DVw5t48dojvHJyDdowGMPx1OwMxjKMZQPJDR6udrBqM+TC+5u3Ga6PnVZz3hQQzOK6fw8neQNrGSolUbcSe+M11m2GZ2bHTgvvHbhOqhGWTYbM+8TenCwCVBTBQzVa4M70DICzOFi1OZ6enuCN+b4fixEmeQPGLNZthpuTBV4/3scnbt3DSuVhza21RCEUxrLBaTPCvC5wezrH4XqMpyZn4MxiP1/huBmj0RL35ru4Nl7iw9MjvHR6E7v+WzCRDe6vZ85qRWU4qsaBj1vTOQDgcD1GITRujufgzOKVk2vYLatgKTArKuRc48FqilleY+n90B8tJ+DMYcxOC3dge7IucWd2hkplWLUZWq8Bvu6tSd5ZznBzPMfrp/v40OwUj9cTzIoKqzbH87PH+MLDu0+87bg5XeD+mbN42ClrlP59LYTCg/kOJkWDdStxY7LEqs3Reg06ANyZnWGtsqARrrVE7bWut3fmwUKFDuVGsg3R4JeNW6NJAK9biRcOHuPthbPCIu33aV2GKNoUHfyscpYW18crHK3HmJUVlk2OVnPMyhqcWRwtx2HdBxB84ce5g1pbe212rZ2vuhQahdA4q4qwxpyuS1gAN6ZLVMrN2XHWYtnkWPr3+Wg5BoDgcmQMx07p/MMXXqttrNO0ExRVLrWLks2AWglU6xxZ7g54pTDRIaAIh0Mxjm3TOE1o2woIHyGaDmLokEm1AtK75gipe8GmjGEoihZNI6ODL/fMwQB5rGTvVkS4soy5yNS19wsPh1kR3ixFUCaJkXPrIJ3osE1EJu+ehonchuhQUHgcXuE1ltZ2Ya9CBHnSxg5IeXTYYUgbSgc7uguoFdrsD5/o0IfFBwmUEo1lDzvWH9Q6mqwvuA/9DVpUonkV6frb6fc7faA1lk99fN/+5N/+owFeJI5aCqB3L9WUxBFRgc2IqzGMAwlK2yKMkranX55HebqIospyZ6bp64sjb6b0t0WLHEq9SJV08u01btS+NOor1T/U/pTuUPTMy/AVp/OclYciYm7r80Bv4F6w0Ij6bojvjeielmAnOniN81JKP+Xz4vK9aOzhOoazGKKXRgO9KKpm/Hxbvm3RWElbufVgmLQE2HLIm/Aa30t5Se/FWqChSKhp+VRjddG9y6SUh/PybKtvuEynsTqPL8pD2rdt9VL589rZ03wA2z/M6fG4vzc0ljEvPc0jzrdS2vbeps9J4xX4uYTGI+6HYJLGbG+TE0zSbFp2E7qITM02mBvqJyTPBnjaxnOaztVU0fVFm7GhjjZbxp3qift5QKuZBgMJmqiNfP3588Rpm4Yl1sJFY8sC1qO7piAzPRrkh/Zu8GSTv7G53pBWLe7/9B20w1AZ0ePNa+8z1+MN2DQjZP3fwTcOfXbouldXr8+TTP439Xvwj+ytNa5s0NylvF2QWJxvoH2D03lIyNlS51b5JNEaXqqeof5PWUmWskvJR9vqTt+Nc+pNebh0PU9K6wnoXVlWfAJ+v/xfvz98LO/+1fdAY/mn/3+isWSMCQCfB3DPWvtjjLGPAPh5ANcA/CaAP2OtbRhjBYD/GcCnARwC+Alr7Wvn0S5Zi2eKIxzIBSqT4UG7C8EMrss5St7i9fo6Mq6w0CX25RL3mz3cys5QmQzHPgKsYAanaoSb+RmO20mIunqsxjhtR5iIBs+NHuGN+gC70mmqFrpAxjQOW+fPcis/w8rk7lTP5LiRz/FOPYNgFgfZEmeq7AmareUhKuxrq2vgzGDmzXaV5WiNwEpluFEukHkbiqXOkXMVsDMpzZXzPay0xI1yAcEsztoy+OlNsxq7WYUH1Q728hUKrnHalmi8ieid8SnO2jKY0u5mziTUmdKqcFItowipjZFovM9ILnSIztoYiZyrgBnqosLmuFYssdQ5Gm+Wl3NXZpavcdY4f0plOMaywVkzCvkPqwlulAscVhNMshqlUHhUTdFqgZvjOVbK9UmMwUkms+R7uZs7zXGls4DJOfftNZZhr1hDMoOvHN4EAHxk7wiVzkJ0V2sZJlkNZUXA6zyrS38S607PCSOUosISHMjReoxcKudH5s1Y122GSdZg2eZOKwcW8DiV5ThZl2i1wMF4HUxDC6FwUo1cX3u/M+21PHkUFVZ7QXy3qFCpLESQBYBKSRzNJxgVDXZHFSolQ/RR0qRxrwmhKLDanwBnQofv1bqhIFfu1Fpwg3HeYlnnrk+KBvN1iZ1RFXzqBHf06dR7XDRY1TlG3q+mkCp8E8l0V3lTs9LT3ilrnK1LlJkKJ/GV920DnM+VhddaSze2FHRr7LUcjRJolISKosJKodEo4fxvDUeRtUHAqJoMjLmosHSCTgG7Gq+BJd+mMlM4PhsDlmF/dwltOhqTsglms9pwrNc5rGUoig6WhEwlKagWA7BaFZiOVzidjzEaNcgEmb8yLH3wKsacqWTbSIzGddePjYTRHNNJheW6cLQzFQJLLU9LjGcVhDBoGtcnzoQTLirsrAJjQNMIlKMG61WBvFBoahkCepHA1LbOFyrLFeraWxAogbxQPthXxycQR4Vtw4k7ne5r7+PlTuRdu4xikIWG9afyzToDGJAVCm0lIXMdgh8Z3dGzhqGdF8BIg+cuQA5FhZXTxgXPAoIPk5lLiJ0WumWwtYCYtlArCT5W/oQd0Gt/IOmjwKLQwbfJUkApH0wFFNm15UCpXVAeHxE2BC+SxgXEiTf+mXFBjGjX2VCYSrhAKoa5oDmVcCaCDP6au4A2FBiIzD2lAasF7FiBz6WL8Jn7QCYUkETDR4VFCEDDKxekibcMRlqAA2bkIsGKNYceG7DWB1lRDKa0YMpNQOPpm9yAV9wHdLGAZhA1Ayygp648jKub++AxujS+TheJ0xT+EIADjKKPFgZyIWCkdZE6uSvPa3rurvXIRVs1uQ9aIwC5+gajwlYMykdi5a0XZi1CVFiKPst9VFgKcgMGiJXDKLUhKizzUWMBuYaLaioR6MURPimya9AAcSBbOF4o8qisGExme9E6mQZM1vGuCwvRMBjhxjSO6GoFgsAe2uED5VjKyxzfFI3VZDENN20p+i0YwJTLY6UNUWkB12/MvZbgrW9jhhC11AoEQYmrrj2W+3wKIfor93SozlTopP4godrV3z1jBjDC0aG/sTkumJ+LfX1EqINobESFBUJU3sAP5R8SLKnOJCpsLDSGciaiE0e0jQ40guzv79H4bhwaxG1JDxXSw6S4HEe/n3q0+jc3IrvGjy8piHe0Linofzt9U9J7rrFkjP15AN8PYOYFy78D4BestT/PGPsbAH7bWvvXGWM/DeB7rbX/PmPsTwH4V621P3Ee7Q9998z+9P/6BwA42BHCosyYA6avTAYBE+6TH6bzleyAw41lyJhGbbLgb0haTcl1zzeSkvHmKQB6wW2CRjLKn2oDSXNK+JmELakiXz7ykezq29QqEfwFXZNPJdVHkCIEt5F738BYE0gbfsojozqBTaxLybRre6JxJZ/NVBtLAqqxPECYxO0h89L0WhnC3Ox8EskfD0AIcLJN+0g8xf6UqbaLhDrOLFZtDsas95XlG3yl/ojWC31AH9sw9tGL/f+CFtn7BpK/W6oNa327Bet8kWKtuLYs+EcOHVgCzi/OWNbzfbRA8F1NtbKxpjZtT6otJP5jf0RqD/klNkoEMz5KcZ1DOIzx2IX5ZjtfRvpLWl7HS3cd+xpSIt9GCnhD2mBtePBBjOsd8vGjuuJyNuk7GsvWCzXk00jjRWMQa//j+ROPXfx9NfFc4f1VRHlBNOaTR/OM/A7JB5F4pbxkMkX3tA9+E/vrkdaMMYTnxvMyNGYxfWNibWs3ZvTM8bv5nK5DhFSLnj/iUPlY80vlQ7u84ErmZtY4Gw0yF3MVu52LofH3eTiVKYNxAAAgAElEQVTr/E47/jp/S1g4E7FE5ZNqrq1B5/sX36diiWaQgn7EzwfflaRc7BcZAnsAQdPEhIFpXYTinn+fdbwz7nat1gczsZpeLhYEW4oQ2zNNs8xvhjuaPT9D8t2jCMBEV9ouAirQbVCF7Qf6SAOJUB4VCdzEZ7wxp7JJ0BASqJ84bVt443qpHho36haKlMptT7tqmQ3PtvrADWzqmRf+Qt2a9fuExof8Sw0LdfeEDpBQ1s3lWIAJ9W+BxujRAAJkCIAwf0LE1QgKJQiApPHsT+mu6TSu1PeRsBML2+f2XVpnIrAMalQpPwlyaf50GqXzv/9zo+wwjXPaET8f+rslxTzEfGzQ3ZaSZ+f2wWXoPUG6sjaTUlLuK//l+0Fj+bS981fefY3la//mf/xNbxvwHmssGWNPA/jjAP4agD/PGGMA/jCAn/RZ/icA/xmAvw7gX/G/AeDvAvgfGGPMXiD5VibDR4qHqGyOr61uo+AKt/JTHMgFfmPxHHakw4e8mZ/hpeVtvDh5B7XJ8OrqOvayFWaywlvVPp6b3sPLq1u4li2x0AUe1Dt4uN7BXrHGZ3Zfw5cWd3CQL1FwhTNVouAKr68OoAzHp/bexNu1w6M8XI/xPbtv40tnT0Fyg+fGj/GgnqH1K65gFpXOcLNYYC9b4XOHzyLjGtfLBZTlWLUuyue8LvDC3mMUXDnfxvUMs3yNmawDLQA4a0ZojMBZXeI79h4BAB6sd3BrNMdLhzewP17jqfEZ3jjbx53pKUqh8KDaQaUkGICPH9zHYT0BYVbeGs2D9nIiG7y13Av+cS5S5RyL1mntMqExki0eLqe4Pl4GbV2sLZzXBW5MF3hUTbFWTmu4X67wYLWDW+M53prv4eZ4jrkqcWs0x73lLp7dOcJhPcFbp7t4du8IDxb7OBitUGY17s93ULcZnrt2iJPKRRdd+6istZZYe783F0HT4OZ0gVxonNUlCuEwDh/4SJmtFrgzPcVENvjVt18AAHzX0/cxbyROViPslHXwtau1xEi2qJXE48UEAEIUv2nWuLZxg6PVCGMfxOXxyRRZprE7WaNuJcZFg5PlCPuTNU7XJfbHa4dzmLkgP40WODqdwDQCu/sOk1QK46IbzifIpcbZfIRr+wu0SgRYm0zqEFnQGIabswXWdYHTRYlru47OfF1i+XACsdPi+v4cyzr3/iIOG7D0+JDkx5dLjVo5fxUpnf+YNhyrVYGibNE0wm3aucVsusZiWUIrjulOhbPjMXb2V6jrDNY6nMXSRyGsVjkmOxVWyxJF2UBrjp1xHaJ6rioXXKttBRiA6aTC2XyE3dkKJycTlOMGQhjkUmG5LpDnCoJZLJalE2wzFSB4Fo8nEJMWs+naRQptJNpawlQSo/110BBWTeZgaQxQjNqgPayXOcAtxtMaxjDUVYai9MGVatmDhinKBuv7U8AA02fOoLy2tlrlmM7WMMZBBWnF0Z4WgAGy/boT2BUH4w5uRrUCjBu0JyXGN5ZYvzOFPKiQ5SoEdmkfjxycgbAOL3Elke9XMJqDC+MgTywwOlhjPS9AMCKGtNT3SjR3HM16UTjIAg9VIE8k2qcqMGGhVxLFrEZ9NIKYNdDzDHyiAnwPZxbtKnOaqImCPsuAUoOtJNisgVl6WA/uYFasBcwiA6s5sNs67Z5vA+MWZi0dbEYjwEcO/oOvOcyuCpAaOHEaS8xa4DSDnWinKTTMQVV4SAurGYq3crS7Bmbi8B3FgoMphuZmC7b2Gr/MuvE4lGhvt2ArAXnG0d5qkb+TobnmcQ+EhTjOwKzTaDADqImFyY2DUzAOc1GeOtxJXTrIBdKQcY8BaHLrcA7nAqY0yE487JDfeOuRhSmtg+pgFtmcO00J85pAr53KT71w4DU42RlDO3PP1chhBsK6MtmCob6uMX5TwmSuDqYA6zU/onaaL1NYZGccamSRzZ1wIWpA507b0ewZZGcc2RJodp0GSrRO49ROELAZdek0PGpikZ26Ha2aOG2eXAGwQH1gIdcMME7TJSpXpt1xvJvM0526Nlnh8BGZBdqJRXHstJomR8CBzNxyh3YCZCug2QXyU0CNunYUxxbr21skmEukbO7abpnTfhLOIG+Bes9pHk0GZAtXH1inscrP3G9dAHIFh+/o/+Vn8POj01LxCMdSF35z7YU4kwGjRxbNjLt5UzjedOnKxDS0C2AOuQDUuOPR9al7psZe++i1g6J216JB0BLKpYWVLGj1mLLQJYOo/OHbiPk+sqHNvAV0wWEKIFtYtFMvcGkEDbdcO22pzhm4cn1rJQuaRtFYDyXi+kyVzGuLLbhyPLsyTqsYsB9t11dcea2vf3eDdtEHNKK2Bm0oaRytoylrC51FwmKkFQ4aS227d1m7yrmyMBkLgiuzFkawnqbREfX0jIUVdADhysfCZozHSdimVjDw1nY4lQw9GBAS6I1k4NoGvjeS54k0wL2Dg5gH33cx5ueg4Br5AWwI01ujVPWvHQ8s9G1M60Ih/P2cPki8XjG9pxpLxtjfBfBfANgB8BcB/FkA/9Ra+4J//iEA/7e19rsZY78L4F+w1r7ln/0egB+01j7eRn/0wh374n/777iIieS07E/faYPlwkSb4JCtlNuwkuMzbY5oM0kn3pybcOqrWhHw+1zD3J8A6u1xGAGAccAo1mH4tQ7TDYB7aW0Xdc9qBpGbQV8gxi2M4sGngUJI2+iE31oE/D3GXV3WIpwqi0LDKB5w7qzyz3n0gkbYdoxbhylIbTRwm774dNvjnoX33rIO0y9eNSzrfFkIl45Oj70pl8OpM93Js2Ydjh1DZ84l3cYPxj1nwsD6DeRgCOv4RFOx7sTWj0HvpJxw0Mbuy2QrjzmY5omTsB2/QH9FJQBz+Hos+thzMXbckA+T34SjZR1dPw5uJ2MCPl7o52RxJcyu0Mfw4+Dx6Xq4e3EdlM/Cn2jb4WsaV18+aACoT4Td0ETQSbkDcO5M8MJJPeWlLqU5SXkJK41wxCw6XoAO347mHeCw0TxGHNEOPLRUETpTNV9f8M8imoS3KTo8uLhu0hRYj6/I6uhdIN5D+5xQEZv1xe0O2iHrMON4wx0mXMP77m9eaIHfaDpTQt69fn79Icw6GgNKpjRgNQ8bjjBvGXpmkL2+V5HpJCXrNzd+HK20QZtF+YPPFY2LdO9ooDekaWLdxskKZ8ZIWgOTOZq9jSK93gQi7+vSIwPW8m6zxRHMJoN2hTbqhXXCn0Bntle6v8H8kCAtSYPX9v3imN/4urmEYOZIAhflYcaZl5IpoZ8aoVzsi0dtinl188P27hvZ1cNizZVxeXnNXH+QCSJpkKLNNVeeLxpL/87QGPHGm1rGbfLvD287QYDqp7kD+PHm3dwMY8BsWB8s70wwg3bNv3MM3WaZzAsDCL3fAId5rzozTiNpLvp+yr3Q/YTJCjiBCOibXnJ3P5gnktkhEM19BFPEoLnz/0iA3DBlTMeehAXrBFfyjiFzzo2NPuvKBoEkEhZCvbpfjvor8Onndixk2Og9ZRZBSApjg64umlO87QSNMMej7wzdC+sCurqonWn7ehrM6H2kNKiJDGtu10fx9qVX3kZ9Fif69kZ89u77dvbKpXyck4Z4OS/fkHY08JDOiy28pLTO1RResh1XzTukYb2Ql0umL/037xON5X/+Hmgs/8y3uMaSMfZjAB5aa3+TMfaj7yLdnwLwUwCQ3ZiF6F7uGQC462Am5mei8L5Y5O/jIl8ZtxFl1j23DDxyhKfIWoIEQ9F/ay3gzLT8aTkJYCRUWgsPHu1+Oz8dv2iGUPEI12FzBXiwcm8+EpnTBEwjdGsYAAdCz5zAB+v6wigeftvecxJGbI9eTwADnD+P74fQvzxqm2+XNcmKFvqI6NhoV9DRCP1JbRfRfaATwOJF2wDWdsJwx2/yJbbUHmI27qxIkKH2NPR19XmD0Ig+D3Rvm11JnE9FPNH92LxsKCmOzVU1qk/FE5TGsV8vCUkb+FMN7+ffqKP7F0y4og8iswzW2PC316nGEWGWwWqXd/ODxRzgNhCEk43AGjZ6zvq/e+DUSZt7wg7da5N7tHEh/5mob1JhKdyzUR6Vtrf/u1dfJLD3PuSWhU3pNlMpCy+IeR85EirpOQAnvCK617KwqQL6vLJ4LvtEgiOADlQ8zm/7ZZlim30S8sMvBlEfaN+GqPHhNYl5U0mgj9hC2W9U0zp5dChAvlkdPSrr8og17811Rv9Z1tvsMQvwqtvMEl983eePNUS+D1PQ26vRxtuvkSD/MN3lD+2wgIjv9+a151mj1/4wbaM5ywCwtivOejn998QCYsX7T5L3AAAE0aUDPvTfAxHDfwRi/TFyfdPNhV5qovK9sqxrQHiX+nWF75vt90t4h5r4nfJj2PTrJ1/OJ07n7PYvNDlsBgoln7BzN+DJePEGvXnDo/4B+usKXV8oKCRray/VybOBvIz4wObahpol+aO9QjLvY3obPG5JW9t2XruSOfRuCFLb+v9KfF0mXbHcpQSzK7bx3aL3DZdJUurj+b5KFhcGePwgp/fSFPZHAPw4Y+xfAlACmAH47wHsMcaktVYBeBrAPZ//HoAPAXiLMSYB7MIF8ekla+3fBPA3AeDOx/fsv/zh38VCFcFn0vndcdRGIuM64EMqIxyUgs4C3iThRxZcBTgNCrLDmQl4lY2RG76EnDkfSc4sVioLvoqSm14AFAqwEvs1ks+csiJAPqT+aFQ25if2bSSfP8eDr4v8NQOOpfNtbIwI+Irkt0hpCA8SQPBRin3yYh9H8g+LfQZ7vni+rm4MbOjDmMf4eYpjKZkJZrXGshAAh8Oi1nLQVy/2SzS2A6aO2xD7WrYe53CncNADBKtBvnPkywh06z+B2Qc/xOgZ+TcCXTRhwn0cik4cLy6MudDwgPO1JB6pv8n3UiVjtIFjqQQEd76PKhqvUqoQKl4kfnLxuMf19vzmrPN5jLEcXVkW6iP/SsJ+pOdEi3AdyWcy9lsD+n6CnDlfQilMz8+S2kzzLp6PHa/MYRhqHvyjCTCcMQulRXcwxGxvPlOie9Tfsa91kMNt5zNKYfobJXvvT9q3MY5jPPbkuxnTo3fBWNbzR+RJewn/sOtHPzYBP7GfVwoDpbnz4xREC8E/kWhxbqEUh5Sm52dJiTASiZfYD1MnfqD0nSdLDxOtGb2YDlEfdHlZVAe1jUGIDmMx9q2kwy7im55TXxjvRwigA2H3WI8Ox9LDD0gdQv8DCBiOgVfDNt4VZ3li+76aNvEFpzbSWNquffHfQJfmXLweReue8fSMp21IqKVx5RZGe+xHDG9qYr/R2C+UDkxT4HdLByoUaCTSCvUA2nsHXbaDMtgWITVRl/RosY4WCziWrFuEUxpknREv0txuCrtXSeTLCPQtYcg6J/YJDe2O6o7vUVmLzreU7rGIZ0R/KcWHsnGbz9uwps8TQbV3PfR8UJDsfEnpQNFydwDp2LLdQSND/1BuqI64fQnbjHhM+YxonStEDwqW8WHEFoFpqO3nCbjJ460jclXZ54qC6Nb2n0dj6zhvyz9winAZXs5L34hMGMp+6wpu7/f0ngmW1tq/BOAvAYDXWP5Fa+2fZoz9bwD+JFxk2H8LwN/zRf6+v/51//z/uci/8vBsB3/7H/5BmNzZ2dMpvRUWVroIaGEtE95nQALgAK8Bm/m1XrlobuTXQdHYRO3MZ9qJhfAng8F8yPt7AN53pPH3vK+K9P4turDexKRbsZjuzInkqosKRyYe8H9N4b+ZZGbkzcJcI90frnxUPu3zR3lF5dptpTvZDD4VdOrInC8O9/3mfCGAYNbGXbl4wTU5gv+BZejMfyKb/xDJzvsPUBni2YrO34G3kf+D96mh/KIBVOn9HoQrJyvXP2rc+Z+EQAEWPb8IwLfHRjwBEK2fB749YMD62JVpZi5vMLOz6EWqA/wpMToTtthEhrcIUf1E5fgzedc3vOnaRyZz8QdTVPDRBRE2KzR3LXd9Qc8oSl1v7ljXZq79eOY0T4BmaWFy5vquQc9UKZgQ+jlO5mZkXhXa3nb1ad+nNH7MuPrM2vk0kTbJCgRzKu3bb8iXxXZmU84Mrqtbw/t21a5NtvJz2L8v8LxYBkB1NGic9dq1l1Y54/NwZYHSbewYA6ABQ+PrTdXAAUvaBe/bZBUAzzNTcBp97v17BMCW1p3W73TaLN4C1o8BjRH3/kKijDZO5LPiTTstA3hlwScMcmGhRt6awtPNltaNPWf+HbRgRWeWKWqXkXtfJKBvnpYtHG0jXd7grwM3B9XErUtcWYicQVYWumAQtYXOo40y8+uBtTDSPTfS+UmZnIE31o8RC+ZyvHX16ZKBKQtnbuk3d9r5FzHt/JKIN5O7+qxw98CcP5aorZsTnPl3wvo54CIQ5nMLVXTruvBjqkadCSH5ZMnK+wkq1wfkp6dK107L/Prjx4Dmbuy3ZaV/V1m3ftC62jPHZX6t875bbh1gYZzIZBKse/fpXaJ5Lhrrv0ksXOucde8URdfkgGgtVMmQL5z/msmY8+XykRqdaan/bvgxJr6Ytq5/mfNtE7V19AoWfE2Ztp1/HGdhrtEYAghjwFt3rUoO4X9b/x7BEp1ubde555U7+swCOnP80ftOawdXvi4/B2nOkm+Z5QyiNmgnkeXHFZNoXNsB9y2BtcHMUJWedz8etHZS/wnPs5EMovW+hH6+ytqE9yR836nvaZ5GOyLLGbKlhi5414ete0di/z/33nh6LXp9Qe849Vlsnsu18wWkMaU2hffLdP6+4V2S6LWT5o/be7i+14W3pLKdUBzz0EWFtd13XXf5LWdhnXTf/Ei4Zt2774gQrf58D8IpEA4egh+fNwvv9TXrj0Wa6Bvu6gDInItZAJHPZKrJTO9RfnDWk9difLJ4jbSCBfpcRfcTNum7YAXr6A2kDeH6IkF26EAg0OrftNGJ3BNpE2NeriBcp+mlJyv27qdvRHh+n6f3HG5kIP0MgJ9njP1VAF8A8HP+/s8B+FuMsZcBHAH4UxcR2pmu8Yc++zsAEKA6lHHRVEmTSJq8nCtUOutp8WJ8ypxrVFr2tGO1dkFrJrLBUuUhgihp52oP2UGRRAEXebMQCo3/osR1pKfQkhvUHkSYtBOAm29O29dpC2MtWJyUPxVvtQhAzaQ5XbcSmTBek+QAk0njRfwQoHisMYyjd1K0T0qkgaF1Lo7OSVolWrNIu0Ual5a0AQNaKxdR1IT7xnrweqGhtEDutYhKCwdrIdXG6X6sXSRtSOH7K9UWS9+vhdcyVU0GaxnKvHWwHaaLYJt5jQEi2rGmZigaptOGdhE2Ldy3pvVaH4r0CYRvEABg3QpYwyF9ABrSsNUexLlW7hlF72TMBt/gMEbCOJB4r2ly84Rh2UgIYSCkQRP7h3oaaRRU0iT1gKK1A3QmbQwDetecG6y1ABcOHiLQ81+sNMqu8znuO7eEqKvMOg2Yp7f2UVeJnvGwHHEUVMZt+Na5gDidqbzxmixrWOebHLUTQN+fmA4pvMm2NclvTn3i6OnaSbkUpMYBZzstketPAJZM1Bl41p1WdBqxbpNkWgGea5hGuLzRx9S0IkBskG80y7q1gcwkAxSG9bxThMZauEA4DF2UT+o4xcByt7OymjufZsWD73bPfD60y/uOKxY2Zj0NDEO3W/Em+sF3mqG/QWFRecXdb/JXZuhMNGN/ZaozOT1nDYcVpotISn5bVDe6ulnLYX0AI6qTNdwFSaJE2ndPJ2x+I97jyJ9u4xv5DDIb2ku+0OQ/2u2yu7JARy/WYMW+h64ST08aBO1Z3D7NYDMN7k2oraB3zv11Pmi286kVpvNt1M71wB18ON9e3nIPKeL6hBnyryX61h9SeT6Z7QJ9aDf+oTzxaLwpfTRP4a/dT/LHtn2fTjqwNazre+H6wwk9JDAxf5As3IHjEyauWRAYmQYC0m/kg2g9Pz0PDV8WgD+M9MIDzYdWhIOHsLGP/S1JUIsEEdFkDgoGnXBOB749TWfk+xr823nXb6687c0bOgzv+Oz84ePD0DgATRBgleheRX8Q4g6aRcgTC3dhHsTaZcTCVr++nvD3hBv0ngZtQGjZ0LBFY+EIDNRt+896PKd1IhHi0vLpvYjvRKkfXuXEFn6jfT2t71BKhLatWty0ji30UuH8UlrLqwiOTzL2v/gEZd6TNHxI8a2Qfl8ES2vtLwH4Jf/7FQCfGchTAfjXr0K3MQJvLPcdBh0Jb/Df0UToANAT7kgwig9uUlM0So8xCeUDbAI236v0XbgMwHqcbygoVhqaPg2eFdNdYPNruYpo90zF/PWS5RumaFfnf5Ov4YxseKU5h/babsKEWAAV8j7pC6o+j76j7a7bpnsltrVnsK0DHyZKiRuYp73ZtwGQHYAmDLpQoOs71QgMpt4Rp7tWUf9aC1jFoeqoSDTHtl1v1pMsG9HXR8MJVxpbeIR3o7ORNdo5g6fD3wF6cbnBj7zbyJu0rGXQTeKretWU1OXa7X8rmTwbLmvazeV3Yy+jJGAYzIa/qBdS4r/pnAFg6q7tPdoWsGvZu44FJFtF5Sx3m5KmyzqUNu63OH+MSEA8Z/zJnA5q4CHRHxq+aFPNLOv5dcIiBO1Ky7CKb+SJ/Vm7vCR0bV/DWMRY8FWN71kAjHUbQkq6ny/drPbopBvZdJ5EeZjhXYCS0B+Jj6thXvMS9Q+zYZPM2u6+s0KJhF4T0SIf27D5Z72NNhD5yqZ80tiQ3y6VpfZb1vlaaqo86gd0m2xCz6J6GYOzHNl4Ka+QbFQ+3fRTfWmbtgggvU0+234PwEbgGGbhsCij52EapuamJipk3d+UJ2axuVjZKPCYZX0ePX8segfDpydeXqMARzZ+32306sSvWNxm6guadkPti78jlxVI0H9lWfpjYHlgA8vAubIB21wWetd24N4QvYFt0+DfWGBP+yFeTthmPSydb8m+c/ATme55tmUZ4P1S6Qk2dZcSXL+d3vP0zdBYvmtpJFp8fO8+GiPRGgEVrWZtjL9oWc83MfZJ6+FNBqGz82tUVoBHMzxGkyN/yzQN4QEOPY/TNiGO/DrTutM2bKtriJ8h3MfUX28IN3MbbZ68zcFf0/a2R+nedZDH81Kq8U3vDfGWpqG2X4beFsuRDf5iOhfRHLof4yJSG7ZhTALn76u3fVviexu+XFcYhw2hODqwGC6HDe3qVZMdmLeXzU98Dt0/n8bl+Lps/52X77IWQr3DjaEDmkj7O9TmlIe0H20yx0I+bL67W+9vOziKVS5b0jbMx+HM5zMyhJ26dZzSRaq3EWabO5fLtMFHAU/HjPzSggy1IXywS/XVIP9X2WBdlHeo7nRTuUVwupBu2r4LDho36r+orktP1qsldpXxuMr9q9C5xPwY0m71aLKIeCxc2IFKBwTRrbTTOnr1blkXsF0A38rDUH3npYHnoc5LjtW5n5yhcbhsWy9K70H+S4/ZN0LvijTei/S+FDjfjzy9S+kDLVjuijU+u/MSSt6gtRKHagrBDMa8Rs40DtUUAFDZDCVrMTcldniF1grMTYmMaWRMozYZxrzGyhTImIYGw0oXWJkcGdM4kEvMdYmSt+DMoPVHQyvtggYdyCVaK6AtR20lpqLCqRpDMIOpqLDQJQAEAbG1AhnTKHmLh80OAGAqa1QmC4JvbTLM5BoZ146ukSi4QsZ0qB9wJsCtFVBGYCJrCJiQ91Gzg4IrTGSNk3aEkWiRMY21zgMW5l62Rm2kL5dh5J2QiMelLnrBZkaihTICreUwPvjOWmeYCIcXSkGGAKDgCrWRmMg6CP+cGRRcYakKTGSNpSpQCIXWCIxEE+7XRmKpCsyyCmetww0thMKJx+3cz9eojUDhgxJRYCA6YCCBbCRaSGZQGxECNq21w9ME3OFExjXeWO4DAG6UC2jLsFI5cq5hwIL5NPdlFq3TDJdiU42yUjlK2cJ4GpxZlKKNAjtJ5EKh0TLko2Qsw6It0BqBncypFTkzkNxgpXJIZrBUOXayCsqKQDMOigQAY9mg0RKVlhhLN56NcdijI9kGrNHWuEMTMssmM2EKLkNm1fF1awQEM9De7Jwzi5FssVYOP7SULVZtjnHWBPNwDhtMqGvf7lrL0L9ZpD6gAyEKbFMIhUplgW4hVeC5NSKMI5mlU9AnA4ZFkyPjBiPZhv6plYS2DKVU4MyGAFEUjKnw5uQA0HjTdDIxbyJz89YH6GKsMyE/qwoYw7E3XgfBpTUcuTe3tr4/162EtSzgnQII5uhx8Kd1K7FTNDirCozyFllkVr+sc0hv4ktBk8bejFswi7XXho4yFdpBAcYYs5ivS0zLGoIbtFoEM3VrGepWYlI2zo1ACYzyFqs6R5Ep1K1EJnXvMKn1QYAyqT0OqjPrL3zwJAAhYJC1zMFDGY5cqnCIQmbhSgtIb/5OdLTmKDIVAoU1XiOcSxVwV4XnndpH/bhaFcjyFlI68/G2FTCGYVS23sy/M2mv1jlG4xpKCahWoChb1FWGLFdBwG28tQCZeQupw2GJtc7sXbUeK5QCBRnuggD5AxXOnem58ubMDq6qW0M4Nw4qy88DrTrnutDvzEJ700mqy7QcPHNqKy50h1nKHE5qVig0q8yZiwsbAvFYDwHFhAETFqYRYNLAkGWEQjA3FoV22n5veh3MqA1z8FBkwkowM8IETTqTvi4PpcRK7cpbdAF1NAt4qgGSiczWGZyptWWu7pojwB5R4BzlzWml6Wg1vDPLZgBrGOwoVeldISkGm/s1Szt+YOFMRol3ZsEU72Ii0Dh5s2ebdSbQAQKpcdfBHNQCBN3jJpuvn+RAbiDWziQZsIDwbRO28xsEOjgo4j2YwsLz6ZgjGKRgCuvzBN/gBDYHQIg30aMBp40OZwXeF98KZ75sCH7GdAJXbEobw6p0fpqRDEz3KZ+NeGToCUWxBjVAp0Tlwth4v/r4bydw+vGkPotSLNdvaJojk+JYm9iTMxMBLqXXo0TwWdkAACAASURBVEt500OduD6GTSiWuM54fK8iVCZpsN2p7HwOjfOF8wEetvH0LSycfZDTe4pj+V6nF75nbP/yL3xvEOo0eE/oApyAVJsMmV+5NBhaI2HAUPA2PKNytBkmTSEJWK11G+o4VSZDydsQAXVIC6rBIWBC1FpKcX4SBI1lrh4kkVijNgkY6MhuxFjmhWEOZQSk36Q74VQiYwacmfA7Tqn2lYTNOPplGpWWIuyGcsxpbWmDT79JS0wReIOvaKSBjXmII3eGvosEWqrbhMi21M5+X6Ta1os0oSRwhCi7iUZwm+Y21SjS75QHGpP4PmnB4z410bwjWiQgDWmn43GjKMVpO9I2qEQTmrZliH7ar0Pa9PO07OdpiKitsUaehOS0/tBeZjfaPMRrWi5O27TU52nttqV0rtA3Me3DlL9Umw9sfpvj+c+Z7e0ldDR2NC5pdOchbW7sO03CGPkuU3nn76wHLRgussaI64j7MI0u7Gj0YlL0nsW0qDxdD2mqU61kEMr8X+nbGkf1TfueIgTbiB5FuE3rJiEx1c5vcykYsmQIbU6iBsf9FbtJxBF200i5cbmYL+KHfL+NxxuOfapDZFVG0V77GiUTzVXyhybcaGPdbj5EkSXf42hXHLCX/b04mmzPxSO+H32q4nzxxr2DwfJtsf6eZZ3/MfkFJ1jMAVv6SZLt90/cf72UQikxG0WTRd8HlgSDlE5MI9V8Met8kMl/OBJmAn1GNGz3exuGMtGkNg6l2Gw7Gouenya665CPwfnIDsFLxWOb9ktcj2XbBTm7pYxvUozZyuIyyVjGaWP5TE3W03ptX/5hcZ60zZcQ3jbqGOBvQwAdqm+Az4t42aR5/rdwq6D4LrX1XN6uSPdrf+V9gGP5kaftU3/5z73rdF//t/+jb3rbgA+4xvLeeg//1Vf+GIzh0JoPfmyD/19yP3y4iVi6wESzNpgxpQseug/iNrO8reZX9EHlW57Ryxi/QUMvd/IBDzxZdGHiTfeh7bV1iGbahIGFejANnS7Z6COW1peuikMLH/0mGvThYQMfxrT+be1J6yB+4jDwMX/fSEpppItsfB02D9Gzi3jfWm0UJMQimG2FYCL+efy3x88Qr9vqT/kfKjf00U/7Ia0DA3mH+KE5tG1MLfpjEG+wenUOvA9+/IaWhfQVCnlYF+Bio/2+bYHWNj+h+F60KdqYJ2nfpHtlhi5whk2mYxxwgyV0WP9ZDJK+saFLeO8FHhnaUKVtPW8+p2sF/eSbHT+0vAAIgWB6z9MxomfUz+fN09TzwQ7kG9h4b9uAMuIp6oOhPeHG8NJYDOTbuPC/DY0jvABPRHlEc2BNImzEeA4Jf49HDG9s3If48PktzbG0scTTwLzovYfGBz5Kvmkbz2MhOf37pIkEJGySOXezT+2j6/PmyVC64HtAAhSN0yCJi8YnfWfT+i/g8bz18cpCBra0I13L0rxDbTi3X6O1Je2fgbovzWdEfuP5FfrgKnycly4UxC4YnycV5N6T/FF6X+NVbkvpvuRbLH2gBcvr5QJ/9oXPYWVypzUE6zSOlkGD907a4xN20gKSNnKb5ohSrFEjDRqADa3TthP9bdqYlG7IH9GM+Yp/Uzu3+nKC9fJdJqXaobTfUm1czGPMe5e/w9/kyeqR+o9exa+T/sok6uu2vENal/SQwYBBMNMT8oc0rNu0enEfXcTTkKZpm59eX9uzvT1xSjWU2/rlvLTtO8PQmW5ehd5gHQNtHbpPdQxpVy+rOUt/X+QXGWvTzvOxTelcNljXZTSk22hd1HZgu1x3Hl/b2nGR1u0i/8XzgkSdty8Y8j00kTb2Iv/JoXqtZQHP8qI2bOUzlWDje4F5t1Pd1ndsS7mhPWh8ryfrJjt4CuSU7ry3BVjbGogs4tVGWqbQjhSHsVf+YiFuyBf7UvvDlFa6SF2WxpV2yReUH3oHt5G/hIBzNV6w/fBmSNAaqD/Gmxy6f67g+yTtDHk6xrcJkOnvQeFvaP6dk84V8tL3Y4jAEwieW+s+L53Xtwybc/+Cei8Wmge+P086vpc4fLh0ujD/Fd6/b6ffl/SBFizXOsfvLO5iqfK+qaH3s5PMBJM52mST7xaZi8UCX2xiByAIGbWWASw9roeEmtbDIMRmZpzZ8JvqS4UCYx3MSAzKHgPdtx7EHRheP4xlG6ajcdtyoaEtC2ZvsfBC9TSqaxuAHpj7VYOjxNAjjj+3MY8hReK/xnBIYaAjbXK8meXcBF8r433gRAQ7QqZ88bikKTY5pKexHxv1SZG1oT8AZzqnLUs2qY6/zc1nDOhuwsaXzOpiQHnSqjOGABcS9zOZu8Wad6LryiMyw+vM5Vg0b6i+GMyeMQe7YQzrAdenY9jdG95QxxtRek5mdvRbCBMsCOI6rEUAtO8JLTHG68BmM46QvE3rvqGRB8K9FPZk0IKBzB392FnLwj2CNQk8JX3FPB3ufSmN7nzi6FmX2W4ePCRt6fra9uBr4jEJczAqbyMtbLCE0KwPYO8T9+NAm5O4b53JJFWEni9e8MmL23+OwN4TjGnecBvobnRqb0PMAG474YboxJroILxtCpCAh1vxEDOMdWXDNbr5x7iDV4l/99prESwbemXTjTuZXVI7UtPDuK30NxbALBDDbQRTx7gcmTTa6JrgV9JdJP02rINZSYU+FvEYbVot+rxZ0iaSTyRL2k7JDNCPf6fjT4n4GHrhon4mCJdOk+qF6tg/kSFAvDDLOggXDjA1FOLzcslpQsn5L7rPvCY89tWLeQc2+NvQhqfjnPZBj5FoDtD1FkE+nQoxXToH5ulcpnpZ8jsVvoiGRdBMxpYLoc54rvYKR+s/QZsgKjPQlm3yFBvik2GrFjzUk5bbVsF55TEsvA0KdJc8REhfmwtTOl6B0EB/ntMfFwuhl3h2Af34+krpqvnfp+mDqGi9bPpAC5aVlnjp5AYYXKAN5xPUaRlIYCHBSkUbRMIRjE/n45NwY1i4Htoouzz9Dfy2jeC2k3HGAE1h6ClfLKxws7G5j31jAPdx7mlAmA2bY2N42MgSf4yEqkh4G9RKhAoijL9oA9Rrm0XPv2bonuMV3SY2xVwa+NvbyDKEDSbhEPY2rFQ2PaXnm29vvKmkr9SyHblL6ey5zhV2UnPZeF7EmyLa5MUbyNjvZYC3AIMQYwVSYAoq7wNjhHanm9IYsy828aU2pRsY4o3KD20s4k3B0Jch3aDy5B4AWDYIvTKo3Ug/TvHfbSk26wM6rLa4bbQJpX6hPk37AUnepFzAJES3qTU6d/NWdh0bNrzRtSGzVtGva6O92uXR2reLRUOmonlHZeIQ/5GvVfybCBjFOj4t+lp6DdjMdu8TCRG09iTroLXe1DTkQzcWxr/v6Zyym+9Vnyh6JsBxXjKO6AXkSPqR4CgchmEM1u7XOjEAt6AZjHSg7w4TkjAdu51s8B2Lxsv6uR6mceoyEQsZVJzyxfnRPUtNHEOV0fzYMKWO5n+6vyTeuI7oR30WXj1vIh1jE/b49sFVGM0fS2PUN00lQaNnck3Ch4nGgMYybHw3TVxttE7GwgrXHb4i4OkmPoYOzxK9uc80YDI8cWIaIRhLMFH3bFiBsMZvBFFBvy96QWpY1N/pu4KB+/6aKXQBZYgGw8ZnKSzFQ1iH8VyOyw18noaExbgtQUjV6C3/NOeCab6n32lFKXPCG/FhN2nFguyFaeB92vqNQfI8afu5aUufPbFweA7dreXSOXPONbMWlrFNc9IhWjGZaB7E11fi7YJ0kcB+YX3fTt+09IEWLK/nC/zM8/8A/2TxIpTh+MTkDaxMgS+v7uCN5QF+6OAVPG6neKY4whcXT+MHZq/iN84+goxrfHr6Gl6qbmOuSny4PMTnjp/FJ3bv4WuLW9jL13hh9BDPFQ/xUM3wC/c/iU/tv4kTNcZRM8b1fInWcnxsch8A8CuH34HbozkA4MPlIX796Dn80MEr0Jbji/O7uF2eheA8xrrIqveqPTxY7+BfvPklaDC8vLqFvWyFA7lExjTuZMf41fmLOPUCz93yBCftGG+vZyiFClrPZ8eHKLjCh4vH+NWTF1EbiWfHh7hX7eEnrv8z/NbqWXx9dRN/ZP/L+Nz8ORw1E3zn9B3czM5wrCb4J0fP41Y5x1LleG7yGF+d34IyArN8jbNmhI/OHmBXrLHwiNIvL2/gerHEneIEx+0YZ2qEFyfv4OXVTdwuznDcjkNb71e7uDs6wReP7+LO5BTX8wVO2jEeVVM8MznCK4vr+NjsHby6vIbdrMJbyz08Mz3Gy2fXsV+s8AcOfg+/dPgiPrn3Jl5dXcNJM8Yn997E88UD/P1H34eb5QJHzRgj0WKtM+zIGjO5RsEVpqJGbSV+++RpNEbgernAymu2v3PnAd6q9jASLV4+u4Flk+M/fOEfAgB+9q3PYq9Y46PTB3hrve/yzK8jFxq1kiikwif33kTGNL48fwqSazyupjgoVmiMwAuTR3hjfYCCK/zg7it43O7gq8tb2MvWeFjv4IXJI3x1fgvPTx/j6/MbKH2k27FskHONz+y9ijvZMf7xyXfBWI61zvBgvYMfvPYaHtQzfHrnNfzS8UcxEi1uFnM8rqc4bkaYZjWmssGIN/jiyV3s5mt8z+xt/ObJMzCW4e74BP/GtX+Gz6+ew68evoC74xNcy5YAEOa1e6eWeNxMsFI5prLGTlZh3paotITkBi9OH+KN9T5u5Atcy5Y41SN85ew2PrrzAFNR40vzp/DD+7+HXzt6AU+PT6CMwJkqcNKMMcsqPDs+DO1/fXWAkWjx1nIPE9mg0hJPjc8AAAfZEhocL53dxPftvYXfPHoGP3z9FXxlcRsrlWOtMtwazXHSjKAMx4enRxiJFo/rKU6aETiz+NFrL+Hl9U383tl1MGZxUKxwvVjgQ+URfv3oOSgrsFYZrpVLHOQrcFi8vZ6F6L3fsfMQxjJ85ew2cq7x9PgkzIVb5RyPqikaI3B3fIp7q138idu/hQmv8bNvfhaFf0efGp3h66c3kAmNjGuMZYMf2HsdY97gl4++I7zHu1mF2kg8Wk+xk1dQRuBHrr2M/+vt78GfuPsF/OPH34nTpkThIxH/8Vu/i/vNLlYmx1EzxndOHuBXHr+AUihUWuJ79t4GAHz+8Bk8t3OI2kg8riYYywbGcvx7d38ZP/f2Z3Fcj/Gh6TGeLk9CFOl/fvcr+F8efAYrleNDk2N87eQWPn3tDXzp9Cm8OHuI15bXUHvNvrIcT43PMJE1Xl8c4OO793G/2sWHxsf40ulTeG76GHNVYq0zHFdjMGbxnbsPcDOb4/Mnz+BascJaZzhrSyjDcb1c4KwZYZavcX+1i4/OHuJGPsdvn95FKRSO6jE+sX8PxjL8zskdfGL/Hr52dgtrlSETGgfFEo2ROK7GKKTCT9/9Rfzi/GP4vcV1lELhu6b3sStX+D/e+V7cHZ8CAB7VU0im8ceufxn/+/1P4pnpMX5g9ip+4f4n8a899QX8g0cfd1GttcA/d/1VaMtxPZtDwOIrq6fwsJpiN6tQCIWH1RTft/sWVjoPa8xHxw/wT08+gtvlGZQVOGlGOGlGeGHnER7WO/jR/a9hZYoQKO7l9U28tdrzY8XwvbN7mOvSjWE9xUi0mKsCn9l9DRougvjDZgc/vPMyfuXso9iTK7y0uInb5dytT/UUn5i9iX/04GP4qQ/9Cg71FF9e3cG+XOFYjbEr17iZneGN+hpeXx3gM7uv4SvLp/DpndeQMY3vKN7BK81NLE2BXz56Ed87u4c/OP0q/t7xp/BseYg72TFuyDN8bvU87mQnWJoCv7u8i/1sha8vbuL7d18HZwa/M38ae9kKPzB9FQDwfx5+As+OD1HyFu/Uu/jw6DF2xRqfO30Oz48f4Z1mhtv5Gb44vwvJDFYqwzOTY2RM42vzW/jsta/jYTPD42aKeVvgw+MjfGr6OgDgNxYfwfdN3sA/Ovou/JGDL+P/XT6DZ4ojPG6n+PHdL+C/u/9Hn3jf8Znd1/BrR8/DWI5np4eYihqCGezLJX7t6AV8ZHKI+9Uuvn/3Ndyr98O4GTB8dv/rOFVjvLq+jo9N7uNUj3DYTHHUjPEj+y+jNhnu1XuQ3CBjGreyM5xqtwd4s9qHYNZHKWc4aUf4k9c/j1+dfxQZ07hf7+IHZ6/g1foGjpoJRqJBxjRmssJr62sAgE/tvIEvLp7GC+OHuN/sYt6WeH78CIIZfHV5G8f1GGPZQHKNp8ozPK6neKo8xUIXeFjt4Lt33saxGmOpC0xEjV25xqur63h+/AgGDF9d3IaxDJ/ZexUPmxk0OJ7Oj/FGfYDXVwf4vtlb+K3TDwFwUdVnWYXWCLwwfoi5LnGv2sNBtkRtMhw1Y1RagjOL/XyNkWhQcIUH9Q4erGbYLdbgzIbvfmsFTtoRFm0R9kgFV2gtx1E9wTSrcdaUvn0GI9GiNQKLtkAuFI7rMfaLFY7rMSayQe7XWs4sKpXh1ugMh/UkuAApK0KQPfo+zrIKjRGodObqYQY7WYUH61mIMC+5waItQjmKLk9Wc7lQWCmH011riZ2schG6jfT91rrvc1b7aPEay7bAfrnCWVNilldotAiR47mfM42ROKlGmGRNsFKLLem04ciERq2lixyvBXKhsWxzZFz38kqvqKn9+GRcd9Zj6IL6AQhuRsQLWZfFAR9jXobceIi3mNfY0m5b8MXB9LNXeNnfy/QtLPx+oKPC3viua/bH/scfx0TWWOscp20JgpjIucJxMw6LQilbLNoCUw/jcNaUyLkGZwaNkZhmtV+QWjRGolbSvVBC48ZogdN6hEx0EVc5s1i27uXfzSusVQbm65oVFc7qEow5KIbWiGCCC7gXrRAKmdA4XLsNfelD75Ppaqs5doomvJwEbxC/XABQ+Q1e5aEJ6F4pFY5XIxSZQikV5lWBcdGAAaiVhPFait1RhUpJCGaD4BRHrayVDJpfxizKTDkIAH9PcIuqle5+BFnAmEUmjIMuKJpAhzGL3MMS0N8iU2g1Ry41qibDuGjQKIlVnWFnVGNRFSiyFoJbrJsMWnNMygaNEr3IlgQZYS0L5pZFpoJZsWAO1qBus2AaXGYKghscnU5gLcNsZwWlBZpWBogCgkUgyIymlbAWyDI3H+LFrG0FpHQmp20jwbhBljlTXiEMmkYgyzSUEv4+Cyaq1jI0jYTVDFmhfJ8DUmrUlYMJ0EogyxWsBYw31SN/MePNBfNCQWuOthXIc0dHtQLtOgPPNYqyhVIOIgK+77iHrnA0TQehYHjQnFsL6FZASKehNz7aopAO2sAaQEgD1QjI3LWZolkybmANh1YcMtNQrQD32mEhddAQE0+kZRPSQCsOkWmoWoJnznOYcQPjTdAd394k2PcHAKhKgkkDmekwJ6zisJpBjLpNgzEcxsMY8MwEDb1pORhDuGc1A5c2/GZeq2wVd/AMS+nas9MGGlbxHk1rGGwtnCZvpMNJrtVOi86E6cx0awExUdALCVZqV59/b+1KBhgGcAAtc/R8P9rW9SMvVYCNYLKjzZYSdqKcRUDLe9ElWc1hpx5KR3GwXMNWooNuyGz/+Fhxp6XJbAftoJmDn2i502QydFr4Jsofm1YyOE1sKO/oMe1hHLxGjjXcaYEz66ArZKdRDGZ2XsssFgKmsEE7yxoGZhgM9ZXXqgEAX3OYsQZTHKxhMCPj7pUUrtT1DdBpqmxmveato0WwEWSqyVsGU9jOgoE7LSdvOKy04FWkrYHTelnZ9Q1vWNDShCBK3ILXpIZ1GlneuHqYAUzmtK2kReMNgx4ZyAV3xg6SVJjefNN09fKawWYWvGFe6wsY6dqmc3dfNIAunIaMNLwms0EjZrw1uJGAR7CCke4ea512UY8smJ9mlrt6mAF07jVeXhNmJLq+pfwSEBU8jIVvinLtBACTO2gLXVqIymmiubdgEBVDO33yfY9oGHTh55NybWEWgAF06eq1wvUTaTbJckJULPAcoDtoPtQ0bxA2nTzSSFrqB2KdA2INmNzPRQHwxvdHFIiKad+H8HXKSNMZaUqt7GsfmYo0vsC5cCNcuTGgsRANIiuBjjZvvbbYduVDO3154runmTTROycivuJnSRmgW6aM6LSljKBEEo0laekDD9EyF7T9QxbUkTZuSKPLjIUVrMtLa842LV6ssSRLhCgFLaOxsNxpGy1nYNr/NXZY48nQPad7SdowwtrCY2qZsFWbGPfx0P1tZS7SvCZ8XCX91s/9hW965NTi2aft7f/0P3jX6b7x7/7MN71twAdcsJx99Jb9wb/xk1i0efApNNZh1TVKYFw0wUcvFmQAoMhUhKemUbUSmcdNc0KRRi4dntqqztxvw4O/m7XOP5Ixi3WdQ/iNueQmCEvGMrRKQAizERRDaxfJdlw2TqBQApzb4LcphcG6yYI/HQkfpiegdsIN1Wutx1JTHJNRjUZJtK1AkTvBzxgGKU2HWVdlENJ4/zeDljDYmNsEy0z3eFfK+ecJj7OmtcNoI7898h8E4AQC6WjyRFgRshOuHJ6bExS40NDK4bvluULTSGSZhvaCUJYrCGFQVZlrQ+wT502BY/9B5YVAzp2/Iwkyxps2q1YAhmE8q8CYxWpRgHGHT6e1M5023peITHOFjHzpgJ4JsBsnt/sj7DoSgIL/oeLgwmHLdeDp7mMgM4eL1zYyjLHVDCJz4y8zDdU4/LquPx1fZKpsGgFw64QywpAT1gmUrYBqBJiwnR+epwGgh2sHbrtr2uhIGwRKMssm/DzGLHTLIXMd6gj0qUxm/j/23jXmliw9D3rWrar25buc0+dMT0+PexLb47ExTkwEgcgIiEUQEiIogIQESIhYgT+QIBKCEBFB/kVEkECAgJAoMkSKFCEifoEUEBAicQmY4NhYjolnpscz3X1On8t32bsu6/Ly413vqlX17e/0Oe0eexq8pE/f3lWr1q1W1X6f9/K8JU8eybqGmXJU5edIQFTKwCxNBqaN7MKZx8dt5PnZ/IxFXeZiuoAU9JxHzxCgidd+FIkPZZ7rsegMSMmzr6Uyic/ntkjc2A2BgoLZBsZGB1d+8ZSl0r+4ResmQmkgDuJPh/kXXfL3AdBtRDpa6G1AGs0c0wZAb0LOAZhdw11EGuy8ji7HjE6aAR4pBm3ZbdPuPMLRcpuW5rVUxLkKD47Xxmaw2EYGiS7NQFKKADuv5hyEhrh+HdMn1zjitRzN0m2bMOcb1MT9OM6tSFP2d5XchACPq0klLyLf9Ko9AHrv+XnIOQPhEu/p3s5AV/IOtgnUG14Pl0CDgdoErit7RfrOzxtNenYPV7lfl7iu3C9Zw4Vbej6eFK+tuLYDfJ9ql/qsgCnzkr0i/cgcmgSMZuG6zONWgE1QowFtA7cR1OyyLKBf8kjaSjFQ73XKa24JquH1gSH+UzQrEWQfq5zLMbdT+rSJn6fBzPkV83MoSpJa8JYcibWbufIK1PC9LsDYEo8dgPIaZBMrIZo055RMANoEdVimJXuTQg0VBQMMza66OiseLAEx15N1ExfslhUkKjB4L+67KStPZD4yZV2xdteussT3ldpU8mGqoJCalNejQgcKswt4zp9J+R6r1ZrWrqol36XkJI0oCokC5DRVbtEKOgP/JHOXdoIqLsgC/uswkeJanAGvynt9BjDzXJDUwtVW8nGWcdXATJYsvxZUdY5qMJgBdu1ev7zpWIBuabM+X4AtVm3UbuV1/eq65WBRAKii1Vhq0FU9vzX790k3enlFpaX7+HJMal6rKpTlvqqnwPkry/r8K66p3Yfr758GSK7LL/3090C6kd/yZfriH/tDn3m77/+BP/obPjfgc+4Ku7MTvnr+FN/XPcdN7PDBcAGtCF/qXuLC9Pjrt+9iZyZchxaPm1t88/gQv3XHbmHvHx5i70Zcuh7f6c/x1bOn+NbxAR61t7gJHZ6NOzzvt9hvevzku38Dv3zzGBfNUJLUa0X4qD9DIoWfeOfr+HA4h1aEp/0eX7t4gl++fgwNwpd3L9l9ojL3j8HiQXfEhevxc8++BKcTztsBPhocfINInAD9vQcv0OgIqyOeDTvs3FTco6R83O8RksZhbPCDjz8GADzvt7jsevzKx2/hfDvg4eUR33p5ibcvbrCxHh8fdyWR+VfffYqraQOnI67GDm9tjtCK0AeHjfV4mus2lkH7w80Rt1NbXOE2zuPFcYMHuwOOk8Om8ZiChVKEs3bEYWrwhd0tnvdbTJlw57wZcTV2uOx6PLnd48GDHkfv8KDr8fSwxztn17geO3x0dYavfOE5ntzscbk/Yus8Pro+wzRafOmtK9yODTaNny2m0WDwFilphMDWpi++dQWrE27GBo2NMIrw4rBBtx2Qksaj/QEb6/HzX38XlBS+8uWPcZga3PYtdpseIWlcbobiFjJFg+fXWygFnJ8dQaSwbSf0k4PRhOtDh+1mAJHC4WoD3URsdwO8t2hcQN83uLg44vbY4uJiQEganQsIUfN9vNqAJo3uwQCAgeq2nfDiagfXBIy3LXaXPVskJ4umDbAZpMfIioe3Ht2gnxz6Y4P9RQ8FoB8chg93oF3A2YMjhr5BFJCkgXbLCo7gDUwTYLNCIAYD24ai2Jh6B9sGhMlm8JjQ7iaMvQMFg3Y/YnzZobkYESaOezY2wW5YOeCPDs1+wnR0sC0Dt27fIwS26I4DewHEie9ft58w3LTozkYMVy3sNhQr8NA3aDbsgTD1DikyCLUZ+PuXHdQmoD3jOn6ySKNBPGjYC/YGMDbBT5aTvie2ZIo1Mt46QBPczjPLdG+LpTMOFqZjpUsYLOw2ID3pWNnzzrEogGJv0ZyPoJTzpXoNumq4jwfTbGEMLJSbXUDKQJaeN7CPRsSnHdSDCWaTiqIJH7csiGZBnV420JdTUTKkWwaG5nJCvHEs3EhCehDUNzfA2x5mE/j8wQANC976owbxsWdBfzTQew962QD7ANxa0CZCuVlyod6y0m8rIgAAIABJREFUgLOJUDeWhd2DBe0D1MGw0KlQQJm6NdDeIp5FBhc6AwJNs0XU6wIA9KgQ96lYOfVLfvekswj90iJtUwakLNiX9pJC88sb+D0h7iNUUDDPLVQC/MMIfTCM15oEFTXcU4vpER+3N/zZvd/CX/I6kyG4JxycJ5a6sMnWsQAgKaQ2wT1zIA3EloVr2wP+bLZIJUcgR7DXBrEhNB9aHkdOjxE7YstjFp7drSkWp+RQBHx3Y4twnxzgrhX8GVsNY8cWM7Ea2qPF9CBh+0stksnnw2zJUx5ILSE2gLtla6K9zURmA1snSQH+gmBvFdwtMF2whUwHtkSFDX+GAkLHn8OW4G4ZNYQN17M9AAKmS4LpM/iw3JaKQNgBZuRxSxsSa2kGBg9hR2iuFJLla1O2YGbvfvgdYI8G/hxw1wZxk9t0QPOS0L+9luZfv7hbYDrneZoexWKmAuDPeRzJAfaQrXNqthg2NxrJ8nrangFVMnlc1xqk2QIpAElPKJbK2C6FebJA+0zBnzF4TS7fu1YtrG46ALHja+yBEDcKeuTryQBmYNQVN4r3VwaHZsr3YAKg85yOhGRmpYSOhNgomJEHFTsGLs0NlThW7RViA8RWwR0Ifsf7UsXZymZ7KnMX62Wys4JB+/mZS47bMpMoFjiWWtZ5EZudAUlyud3K0ikWSLFUJre06C7WWgNmIh6TFPlYgTkd2ZpImj8Def2dgDQeM5nK4riiHF+A+2yJrIGWjrxn5L8iIBkF46kcL5bbVZvJgttbWUFlIgXAJRQ0Xlt5Z+BMQJ7nSYtlNlgtAOKKha++Zg0aT35fx5SfAJmfBfD89Sqfp7G+aflcWywf/cgj+od+5vcikir+53V6jJD4B61OOj8zxy6Txa/TUNSpOtbpJoRFdn1dnZT8FHV9zRhbl5qhdO0jvmaQXIxz1YZRmYW0moeMoWaqrb9LKSk3ajXRqo36WH3NHWtsqscv4zvdVt3efW2uWUtlne72u1Qzyvt+vaZrJSGdmOerUn6siZ7qdTqdqgF3GCtfZy1OlU9K67Du+3Qbp645fbzuc53GQa471f6r2rvT76sYJEvFE8dOjO2T1gbIyt4Ta/QmytdXjrWM7e76fKrX7Smt9koz/YlNfMK+eu14j09SG5N65ZrwWPj/a63Nqf4+aS6l3iuOr18Ar2zn1CavPst9qCwNC0bQU9edunf3zbVW17/u3F9V6rG+ag8V3cGSgApAlUfyFWtzat+uj6/X475r1m2X76v3NM2Hv2uC26nxr4+vz+Vyysr1qvr3j+ET3pmKinXq5Dq8yctudX6Npxb1PmldXgcMnNqXNNc9OacT+6Cud+ea+z5/0rhOlTd4F7/2nvwuiuX3juEVfd67hz7F/vmk8kbP7adYp1/4d743LJbv/OufvcXym//cb1osf83FqIT3Ns/xNw+PAQBf7K7RR4dn4w5X0wbv7V7gJrS4cAM+Gs7w3u4F3j88gFaE7z97ho9HJil52B7x7cMF3t7eFNKHy+aIx80trsMGv/jibXxxd40hOvTBobXs5vqouwUAfOP6LZy3AxIpnDcDvnN7gS/vXyJB4Wm/x8ZNi0DlznhcTRscvcMPXT6FT0zo0BlfLJTndsQ3Dg/RB8cB6m7AMTS4nVo4MwdSP2iPJbj9/dsHmJLBg/aIq2mDH738AB8O53g27PD9Z8/wrcMlhuDwaHOLrZ0wRIdv3Vxi30zw0eC8HfC837LHV7bOPeh6dMZjiDyO5/0WOzdh50YM0WGMFm91B7wYt9hYj7FKX3I7tdg3I54e9jjvRj4fLY7eFavlo+0BL4YNGhNxPbS43Ax4ftiidQHv7q/w/vUDfHF/g5fDBkOw+NL+GudNj19++Rhb5zEEC6cTfHaFbg0Hx1uVkKDw5LBHTBob5+GzK/PDzRHXYwerE676Dj4a/K53v4FECv/nk3exaTwedD1upxZGJ7zsO2iFwjj89p6Jml4MmxKHKnGYl12Pm6mFUYQv7a9wDA2e97ynjt7hoh3wctjgrB1xNXQwOmEKBs4kGJ3wzu4aWzvh/ZuHbCUjhePY4O2zGxx9gy/urvH+9QNYnbBzEw6+wRRNJofhNj6+3aF1AY+2B3x0uweRwr6d8LXLJ/hoOMO3r8+xazw2NqdYSQa9Z/VyZwOGYOGjhjOp7ANRmjzoetz6Bp0NaE2ATwbPDls82PZoTcDTwx7vnl3h2zcX2LfZmhgN+ondyS/yHjvvBtyMvL6HseH0Lkljm+OEOxtApHDVd3i0P+DJzR7vnF/jeb9FiBo+Guy7EcPkkEjhrBthdcIQLEZvkUjhKw9e4Hm/xVXPsdetC9g6j30z4oObM7ZsZ4uxxBbfjg2MZuXIecfjf3HcwOiEbeNxPfC93bYTjiN7F+zaiT0GHn6MxkT8/EfvlGdg2064PnYcZ6rYxf7x7gCrE75zfV5AsXgE9BOT0BApvHN+jW88e4jvf/QM37664Jji7IL+lQcv2LshaUzR4Kwd8eH1WSFVeLBlYounNzucbUbEpEvbiRR+9PGH+MWP38boLfabsXgXGJ3w7v4Kv/zsMULS2LUTrg4bPD6/xfPDFhfbHjdDWyyyKSnsugnWRNz0HR7ujjhMDruGPRnONwOmHF8tqXwutj22zuPJzR6tC4hJwUeDlDQ2jccYDFobcRwdzrcDts7j6e2uuPu/tT8ikcKLwwZv7Y942XeZtRtonUdKuoQW/OgXPsR3bi9w1fOzJvv0m88fYNvy/h88e1h8+eIK33j2ELtuwtv7G3z92UN85eEL/OrVBcfhJoUvnN8yAVt+dl4MmxJGIfHmD7fs6XDMXgyXmx4f3ezROTYFjd4iRI2zzYjj5PCl82vOw5yF4+uhyx4QCQTg4bbHmFnPR2+hdYKPBo/3bJ6LSePoHb64v8EHN+ewJuJ2aLFpPHufTA4Pd0d8+PIcX/vCEwyRSTzk2bY6YeM8bqcGh7HB4/0BL44bPN4doBThvBlwPXXwyeDJzR4XmwFfPnuJb1w9xK6ZsHMTOuPx5HiGrZv496zfoLEB1/n5BYCXxw2cjXi0PUArwvsvL7FrpzLG826EMxFPbvY460b03mLjAl4eNyV8ZNNO0Aq4Obb4wsUthmAxeIsQDDbthEfbIwDg4+MWD7c9Prg6x9vnN3jZd9i4gCkavHf+Ar/45O1PLXe8tT/i6fUeRMC2m9BkD4nGRDy93mPbjRi9w+W2R589aKbASu7HZweM0eB2aHGRPWF8MJiCKfv6ODFfg1ZAa0PhZzhO/I6WFGNTMPi+By/x5HYPpQij5/t8Ozbw0UArgtEJ1iQcx6Y8e9d9h132solJY9+NUIpwfewKRwAAdI3H6C26xhfegbPtAB+YM8KWtl15x9z2LYiAB2fHEprTNR7HscEwOpzvBlzdsvlUayp8BLtuQkwK/dgUbgPvTQlrkBARpQjeG/jRwlg2oxkzh52EYApPgPRRPHGE3V8TK5xNylwFpoRxaJc4tMNSQTgqv+dcE0poTWG2zxUk1ENlt2FKKKEgxiQOuSEwQ74ipKALy72EfUhRGnO6p5TDQzKA47RWHI4i/yUERLuEFDIPQMrKRAHkOaSFPMfAF4Z/oKDuwsIfdYn1X6RgqpTF5bO4bdcs8LUiQRC+KI3KJLEEpWtlV51OScZ4KrXO57Ws1+P/Y+VzbbH8gR/b0b/1F38YN6nDRHZhlTqmFq32SKThySBCwYD4M2k4HeCThVEJWiXcxg6tCvBkCjtfpz08GQzJcb18vVEJkTQ67TEkV+p7Mui0xzE2cGoGfxEaBksWLp+DCpyKiJk9QixxBgkRemF9XV+XSMPqWOo6FTFmq23Jw5nPhWRgdY4/TRZtDoTgtdEwiuCTKQxqdTGKioUWYJYyq+eAgUQaTkf4ZOB0LLk0AZR+p2QLsJ7Zu1I5Xohv8likjVZH9NHB5f7kuFGEMc5z1blt+cGNpOAysJR6UtbW4ilbtTvjYRThEJoy1rpOfV2ZS14T+ZGt/ydSaDLZU03cVFtP5/+63J/GxHL9nO90Ps+MaSa3JcoKU6zqsq7MuhaLJV8rQmfYpXOI7s4c6/ytsu/WLGuSu3U5tnl/ylxFYJXrT81f1gk4bXWUekSqkCYJ4VIUt3LMeWeFha72NnCac8TK/avr+WQWYzrlhSDKiTr37ZoVrx67MLbKnpO8q2trodEJWjGhlJTaY0D6aW1A71mRFTMIliJEYvU163XWiuCTXjwXqrp+imaRC1f6XZ8L1drXz+cpK7Hkf60Z/tKJ+kZR2SNS0onrZK2EmKveazWhVp1mituSec75e2XtY9KIScFoaSffU5MQcuy10QQfTDlW1rViIpQ213mL1/l1pc+1t4nIUVYvn0WZa8lNW52jqt/6+VwbhU55XKSKWTFV96L2DJF49DpWvq4jpGjrOnz9fG0qz/Ys3Ncps+a2UD4LT8GpHLjSVp2Wq+Sureaqs+JFvCCEfKy28ClNSHH5O/cmpU7ZVec1hqICUu6k5anWloBlHtVK+Ee1TtzXypJfWRopMbgQgHPKs2RxnUIBM/dZzNfeBIt8xnXcJt3zuSbPKmm5sLTkS8x1XrM7bUmpH4v76lRzK5/vmFLnuZ60sEuc8RrknLj+VP+r23yv08EbWajXIGxVd71s9/V9b5v1RfeUO338Gi229/d0or3XqfMKK/iryt/8174HyHu+8l2yWP7zr7ZYKqU6AH8ZQAs2LP4XRPTHP+txfK4tls/9Dv/5B7+rUD/3wUGDikXv4JsiwLmcLkIEMmFCFYFMiHtEaJmigY8aRlOxjBnNIqsITqKBF2sVgEwGFMo5EchqoSpEDZstVEPWQkouzNqttrEBRhMSobhe2vzDKYJTiIbdX3N9YAY6ov0TTb/NMYZsHZhZUyW/p4yrZoWNWVMvP55CTiOxXlqzJlm0gXVeTDnnXCjXiFAiZD/1NUw6JFYZZlgVBlWtmdGVtZgaTeuLlUKKuJjWwobJ2tf6x1nYVIkAm0lfhiNrdJsuZMKdWSARYUV+tGMWSnQtsID7TFV+VCH9EWFCNJ8ifMlxccskgOPriNlJpWidMsERLVhGa9IeyR0qMY1EirWXmbiFEpg0x2QCHSGVyKUm8hEiGxBmIaQs3iyg1UnoU+BxK5tAk2ZyDyG3qdsTbWj+L4Q9pR9hRBUyoUyYo2wCeQ0lmuRKSwxwu7yYNAtCmaFUyR6IuhBlFGIS0cjKckuOUKAQbhTylMxKyuuAWesqx6c8hjbNP3xCZAPMQo8kZxdWV1JzfRHIFBXyGiFMWWhscxwm3wCayVZkHeuxCyFR5Z6pRj3HaMZqbgQm42iqsRmaiX8kT2UtwdQCZG6nkH7IOqH6n8BuehXzaX2uzoOpomIiFMmrqVDIXMhWRCTSRZ4/KXbT1L1CaqjcNxV4rSnHk9bj0qNC6niuOnDMoR6Z7VTqFKIUYai0S8FR2E0B3GHFrHMsCukJdG6zTIAKU2i5V5KzVC3wEceVQfpakqiUHJHE942JXQjTqOaxVQJhMSwYyrk/UQhHhNREYth0BJQHyMm9zHOvGT1zPkfSAFWsokjVdzf3QTrna81rippIxVZjjXnIBoDPoEfWJfJ+UYQSO5oyu23NCqu9Ato3lEarooMqMcYcrzezwqpMWlP60ViQpWiv8jrPsYQcJ8vxffK53Pvq+anzP3JjKLGScr6wtq6BUr0Xc5yh9F2zvhbiHMLMolrnxqyfmQrkrvNYyjUyh2TmvhassrSsv8iJKe8FwjJmUi/r1blE62ekzH313JTrVrhYztVrULexHns9h5PgsbqH6znXeO4UPpdyp90FwK36knfv6r4s6gLze+XE9l/ksqz7OwEa1xj/JKBdH3sN8CltfSJIvq+Pz01Ry03w61dGAD9JRLdKKQfgryil/msi+l8+y04+18CyNQE/sH9arHJ9dtcUC1cfXbYyzlY5sa6JVhxgi1SjI1s/shWE4zA5X1FnwsJqVcdwJihYJWlIdPkesmWx/lxfK1aSobUnLTsxWwJrSxAwa7cFPIrVI0HB5DdmzPMYu5FdYVQqLrwyBhlHmXdlKZA6MqZaAypAWdKf1HXXFora0kQ0x7WaPG6x+txntZA5CpBXQEnHIgqCdRGAL2OoU8TIuvtMIiTtKADHnJakyW44dbymzLlYoJNY0rJwUVlsxBIiygAAixQs8l8UBeuYQLFWiDukXB/kPtNsNZSytoTY3F8ktbBWCUAXFzvKCg+9spjUZR0zOo9//ixKBW5rZr+VOuLCJCk/ZmuHrF/d37LfteJC1kwpFOWGrGdJbVIpAGR8AEodUWLU9+RU3B+PHzAZGBZlAM2f6+MxsJLHubhQcki9YvWJbF8+te61hSJFU9iJdZXyReay/t0vig5U+z0zJ9f3EABip4tSRawv8oueIisnVB6z3O/CYKxXv+Z5ngzSVRmQ1mBriggfSuY139taOCruZdXEhAFY0uEAKEoHXRiVq2cwA0tR1sSOSZFkzGKpUjZVgJjnHTtm06WkEKOCsoTYqdl1rBr7gglWZfkmL0skFEDH2iI1s9WWG53vpQKDdrU8t1AiVDHr9dqkNZlF7T5WC5ByzhBSq2blSF2knsox8iKoAguwDg3ExACObMVqWvepUIiISM/soKXfDMZgaTXG6ngFTBbjrcA5KsAt66QyYQqDJ04zIwCeNymgArGy4VMWBo75M6GYxxnQoqR7UJEq4pL8L3BFEqVJpTDQ4S5jpxJFW32c5utUN6c0gZrnugZXM9MqZsXNChTUQG5x/+dm5jalHlXtAUsAuwJFdToPabAA5zVgXcvcNK8FFFVtqyUIPAWm1u2ugNVJgHkKWM6v10X5JGD5pmUNPOsxLsp6ne4BgcuGcHLu88mqrXVfr2r2def6BmvyRm2+Jmj9niq/AeMldlG9zV9d/vvMR/K5BpY3vsVf/uAHixAfRVAWYXrljlO72KzTdoi1TATH2p3GGEIU60sWCJRCSVlRUjKo2fWmCHPV59IfsjuXopKyYm0FAqH44wNYCFCLXZCtUUSV9Uo0flkgEB988dUXS1I5JwJ2JTTWxxZvORFcKlWVjO0+spK1i4+qBMLix0+48x1ZuCvCqqLZErMWjmj9X+rR3ZdO7dIj7YjVQISd2lJXW5GA5RioPn+in3rNauFvRQd/R4hcCJpV+7XV7J4f4KUkUDUTWNC7o7WsBQq653vdPsAuRSshkq0Ws8Ze3I5q4o8ibFbruz5fr4cIptKuzJ0UVYIG7lifeC3z9/V8k4KXfQGcnnc5rhBK2oaq/RpYyJrkfTHKHuKBLn8hK0EoVHu4CBOVVKEi4LNgHMzdc6TmLpCAeEIAD2JFkrWRbZ8ZQWXtamFGZ9BAec4LTb4IV5WgpiQE6JQwW1tcpP3KknZnj9K67ZwSoQKzWoRtTdAr+nxVvZcUzWkJFoI1sLCulb7FskAo+fkWwqSq1rJ6du48cmsht7IGnxIE63tZrlP3nF8dL2OQ7+X5WAnMeW61lWxRqv1f0i4I8Knf9cIQWaehoNN9lvtQgyPMa1jyf9ZjoNVa0XzvqFr/2qJaL/56ry/2YLVu6dcg+dQWqDsApB77Gvgjj1nVpDrqzrosNW11G6o6psp+nPeOuheUVLfwrjWovGvvznNtAautiXfu+/pBqNuXPbf+/Sq/J6fbLeeqeqTm34jF+s/LcPdYXRZtLT+fWpfS7KvaXF+z/h1Zr0u1H19HrJfclfL5TrdKzcdfRy64t597TnzCGF8bBL5me/de84br9pvlblFKGQD/B4AfBPAfEtH/+ln38VqvV6XUTwD4a0R0UEr90wB+B4B/j4i++VkP6E1KZwJ+4MHHsCphSgZTtMViaHXCrW8XlrQx2tmaGRysTjAqYYwWG+vLMQDw0cBnq8DOTRijXcR+AewuCzDRiFgOI6kSY5ZIoc0xc/X7RCxJViccvVtY8QAUV1chdJHxGJ3uML+GHNMRoi4kAuLCO3hbAuwlvYjRXFdA96bx5bO4wgqzq1ZYxBclUrDZkhaq2C1xJZZj4kYrLrXOxAL8FdgCWFtcdZ6TXGvyGvtg4GxEyG6zdf7PpgnF6re2PNWWNckhmiqrjbjhikJBgdNxgBTaziMlVdVRxTVXwHPIRAxiCauVFpw2I1u1xGW2so6lqKGzNU/qSbtiZaOkSq5MUSaEwOuUkoY2LGkWhtiilOA9Yar8mdIOJYUwGWhD+XwlgWK2qhYZQNaz2reU2ymKgix4mGyBBIEJBbyGyZYfEFuuFtawinygWPTyr7pYoyS3p9YoFrTkNeffzEBCFA6s5OG+VAUYk2eXZ22puA6DkIkO8vpqIFUuhUpXMWfZhVRn1+J7gWVWFqScG1N3YTmPhSssuyiD5lyT/GxV/WelS/Iaqomch9Gmsu4AkOq8iOLOaitJVlLJiNszgAXBwqg5/Ye4t9aKlKDmfI0pt5tJHxBUzls4d7XIQSnWIRlPUQphlj6kv9dwhYW4bMtcZb1lPpKvsWzQ6lxSSFN2D61dYYHsMoq5TQBqUsVFlt2BiY+5ue/itipjyHkMF0oOATwCMuq55jp3XGHVLNiSqeaj5jEDmPPLyfWY26ytQfWYBHSSI+iB980dV1gBSxqcS9CgpJ0QCxwpFAtgsQYmLK1V4gqr57lIOwIkBZwkSwtXWJXdXJOkhMjtVQ4/C7fbNUhWhJJKgmwGkJJ+Qdw/s7tobPCpi6R5KOPBDHCKy7PK6yev2bzeZcxivSvPBorL8xJQzWhnDYKgAC37O58XN+aFEqoGULIWcp/y3gDyOtfX1dZEGaPUVdWca/fVe1xhxX1VUn5wZcx7vN4HK2C5XueFYisfWyi8MJ8r15xw/V6UU6BlDSxXiqiTZdXvou11X+trToGl+r1W5qbu1lW4wwK8WD85JoqzU6BsvWZq+f9k3VNjPtHefeWTAPMry+cdWH53xv9IKfW/V9//FBH9qUW3RBHAjyulLgH8RaXU30pEP3+qMaXU3w3gq0T0Z5VSjwHsiejrnzSI19Xb/UcAfrtS6rcD+MMA/jSA/wzA3/ua139Xys6M+LsufwW3scOQHJyKSFAYkkNIGu2OyXi0okJa00eHRBqt9ojQmJJFk487lTAmC60YwDU5WObGd2hNQCQmUtEqIZHGxkwYk0UfHTbGL9pqdYRWCYfQFrIcACVdiSed2/CZlEcXohwhpJGxhlxPCEdqN0irYwF4fczxmnnHbozHmGy5foz8ucljS6RxrMC0VlTAsgDoZuVKKq6rtYvufa6Z4kocyNxxF250wBBd+b8mv9F5/QvBDxjMdsbD6oTrqSsuynX/Mi4hYxnDXYVA3YcQrZw9GGF1xNW0KS7FAtziSpXrMpFOcW2sfiWEMAZAAds+GViVEEgv/gtxUJ1CpDVhMS52vZ1JNxody3UyD3E3lvnXjICirDA64awZcfBNOVa7CcuYa1KetYuy9FmnewFQ3JqVmuOZp2gWMcziziwMvvJf5T0kQFbcjp0oVCplRWsifHYVljWRcdfxz0Jo01l+Zus10GomjqnJgdZpeMQdOZEqyhtTuSTXJDDWJMSksGv4WRZWR65HRTkjTI82x4BLHLacE1dqubaxEaO3aF3AFMzCJZ3ZVGc3daWYbGZ9PlQxzPX1XeMxTA6RVLl3UoS4BmCFw5RZj0M0sCaWuO5SP++FUyRWwvYLzL+jRlFhNl3LgSrvNVmP2vVfFESy12vyHlHQ1AomYYkNFauxtCesscDsUm1Ngq/cxUWx5au1lxyp4kIuLuC1J4ywWq7j0NcpiGrFVV24zWp9Taqum5Vn9VikHYn/XrtqCzOla8JibFLEtVxcv2NWAMnekn5iYMWYUkCMcx/rORbFCqnC0Elp9vDhm8pJ+FSWXsselLkiv49qU3xWUFBUSMLAmZVG2swSc1HOZMsqJUBpXnfSVHL4fppChooyau0WTlEXS3xRUlUKDyX3UjxX8kQX7KBrRLBSus2d5Zh2URzR7K6+KIqWypjaK4QwWxDFk0DK2rOmuFqvxlcfPwU45HPtrbMuatUuqvFJG5i/Lzxb1iDtPtB0DwgqgOw+4LiexxpwVt9PgbDFsVeM485Fr1FOAcCTQLC0+6o+7+njTeq/xvzqx5w/vwbC+ixA2OcdiL5e+fh1iYmI6KVS6r8H8A8CuAMslVJ/HMDfDuBrAP4s2G32zwH4iU9q+3WBZSAiUkr9IwD+AyL6M0qpn3rNa79rZUgOv3D7Ls5tjz41uPYdAmlcuB6tDvj28bII8J3xeDltce44LcjLaYOtnYpl87LpcetbNJnZ8WrcoA9M+f7u7grfPl6gM2HBunkztQCAB+0RHw97AMwIedH0+Pa0AcDWzimaBZPllAxaE9CZgG9cPwQATocRTbFuTsHivBsKqJI0Hq3JggE4plLG4KPBrmHq9iFYdDbg6e0O28ajswHXQ4tt4+FMRO8dfP6xfrjtMQaez9E7bJ0vAEHSN4gFFGDrrBAbacUCdj85bNuJhbXqLSQkRrtmKu0IhbpQyffeorERUzDl+67xmKLBYWyw70YcxgatjSU1RQgGF7ue0y9oKnGNMakSEyqMg5vGF6Fb6wSjCcexYVBCCtvGw+iEbz55CAXg8vyIEDVGb+FyCogmp76QPobJgUjB5fQBtfA8TjbH2CkMg4PWCW0bEIKBtbHQuXtv0DTSLoOplBTG0SEFjXbD6Qx0tjj3g4MxBD9ZNK0vQp51sQjcIhh2nUcIBn6yaDsPIULyhwami2hajxA0KOkiIBdhMQuOIiiKhVLiImNex5Q0U51rgmsC/GQBAoyLCIOD2zC5kgiMOhP0BG9gm4Do2XpKQBHEgTn+UKyRxkZEb2CbCD9YaMfgUOWYRq3ZjzMGU4RLOeYPDZRLsE2OLY4GKShQ0LAbPqZNQgymCIrazTTrMVsgTcumhZoMKQUWBJVmC612Cc+v2QxizqciGCSvYZo4e0VFhdTqCMbQAAAgAElEQVRbjhvc+RnsSbyoTcVam44WZh8QbxzUNhQLLwCkW5cth8T/Jw29DUWwpYnXXneBLZ4AW0hzP1c3FjgLc92g2cKYFNSgQWeBLeleQ7URt0cL1UVQtnQWbz1FoImp9OESMJqZMKhJTDxUj5MUH0vg82LRFKuuWETFyjfpmUyIuJ7K94XaCDUakEuzYCvum9naNVxbpE3iOqSgRs1kIrtYWUO5a3PQiGcRymvoQSHuEqZbjbhLs/Vl1AvLWGqy5S63RTZbBRUyOZGC9mBSIAEA2cqpJw0yBNNn4J3nkJq8Vhlv6mmO6SMjFhhC6nWxOkET4qgQOypJ5FWcrTVpUojbBHWt50TplXuzuP4mR8DIbtJq5DmJ2zRpIHZ83gwKtCFQALvFRjAhjpA8Sf8O0EMWInM0j87tpi0x0REBMGxJVEkBHTE5kFglWxSBVeonBx6DJraAaUB5wGRyotQweU7YEOyR11QFjke0PeDPPr2UqfM6A4D2swVWRV4fHRj0mkHNxDqSeqLne0kO0BPPkRT/NwPPkQwKgKnBTrIrEKMBewBim++zqdrMREnAfG95vLx2YjmtyaaSRbHsAtnSK5ZVndc/kwPV+yZZPg5w2wBgxvkasZImC5iJrcVqNT895XYr131ZG+lHrONUj5Oqc1iOjQ/kf3aeR00EBFSALFvpyxhWdXQgkDkBs/Izpmhec7YgUln/ZJZWOXFdXbiwVu3JPSeloBItrc51WIJ4BujV/NbDzG3W8z9VCjBduY/fqS/f12Emq7YWl6yB71opsJ5/fX0e0xpv3wugPw/lN2Ds2eroM6jcAPg9AP7EPdV/H4C/DcDPAgARfUcpdfZa/bxOuhGl1P8I4L8B8M8C+HsAPAHwfxHRj71OJ9+t8vBHHtNP/pl/DEN0C+KbRGphJRKryakUAUIHL0BKrCg1oY7sc7G6SB/iliqgQixj4rYKYGFNqi0Ga2tJbQmRtuSYfK/ZYBfU/Rn0xUoTKEAoEVauokuK9FAx4Qpdfl3EMlVIP2hOWSCurrX7a+k/j6G2FNaa77ULKzBr1oXtVUhDBNQAuEOEst6+dXjKmmSmJoQR8hgBZC4DtBDMwnpTWz/WdPmnxlDTtK9p9uX83PdybkrNbce4pKOvY1jv5JJal1plWX6NOGeYgEU5Vi5Jpyn+1/T19X2r69aus3LPF0rmPKaTv1XVGq/HdVIdi9NrMbsE52O1yy640+JGK+6b+XhZz+oHVSwc97n9LNYocf4wALPrKeGuBUHJuHCXcbcsVq5qmAW4sOZWa6AqK1ZZ0EU/eY/Wrsv1GgvRzmrtxL23PlfintMrLCL5XpX46Px9sQ/rdVzf03vWWMZaAChhaWE5VV/6A2bSnNW9XhP3CGhduNbK51St86nY7nX/i41fjVH+06rvdaxpXaeuty4nnvFXWjruE+TWdeS/vue4yn+1ACvfy9BWe3P1+U6M9LreqTHdNxdZr7x3l+M98f6oCYU+RVm4PmN1q9YC+Xr89X6tP8t6n9qrqzYW06lBFJaP3HLQJ/pcX0CnYzRP9Xun7fv2/Il6p8Z3Z8q0+r8eyydZF+85duqRWU94AeJO/BTdCwhe556vxvFG5VVr+6prXqP+K8fzacb6um1/Bn296Vr+33/iX/6NTzfy3vfRO//qv/SZt/vNf+GPfFK6kd8G4GcAZHUc/gIR/fQ9df83IvqdSqmfJaLfoZTaAfifiei3fdI4Xtdi+U8A+CcB/BQRfaiUeg/Av/2a137XymN7g9//xb+CCAVPFofUwiDl3JAKx8TWvIksOjVhoAaNCpjIYkwOWqWSfxJAyUeZSOc2+bvkv3RZlRfz22PM6jk5Lu1IfkGfbMmFGUmX/JfiDmtUwjE1pQ2xaiaoklfy1DWlH8x5NcUV2KhU2G9vQ8txpEjFJdgglXyXERpbPZU8mnVJpBa5MaVolfvMOTIBZsdt8zyljhyTHJ71OfnuVESfmuz+m13eyBQXZHEjFjdmdg9uFq7B69yZkuNSwLLE1PrslivjlSKuxM+nLQDgzI753ppFzKcGweqIkExZE6djcbmVMUjuTgAlX+R9rLfSXq2sCKQxRYutnco953XR2Rrt0Fl/0l1V2hCmX1GmSH/HnKNTxsduxjO7rtw3yZm5vu+SU1PcqGVMwnwcki6uuo2OCxdhWafadXrttlyvT82CLO0O0Zb5SN11/dpNe4y2pB+S+jHpBYNynS+zPg6gPI8127K4Otf3Rq47en6WN9Yv5lPnDE1QxTNh4/zJ+Za8rtFgYz1ufYPOhoWSRrwM5vumC5OwKIjEnbd2VRZX36N36Ozsoi8uxwT2lmhzf5HYLXkIFk12RRYXZxm7MDXLvZJ+xJVYxiTj9ZFdS23FfCznxWW1nse67pSVP84k+KjhTFpcJ+0RKQyeU0zVTM2RFFobiyJO6k7BonUeKfGz09iA0buyf8STo1YoSYw2AUWRVisaxXvC5rjmmrlYYsRDtrzXLq214qZWdklsuFJAkDhahRLfLcnta0WipGlyjj0mAHZ3LWzGFcgSN1h2dc3Mw5VSyNjIFv40p0uiSjEm6yJKNVGgzWPnOGtuS+aMwigsCpqFgkcs3apShuRYbuS5izJEFDAlpZFNS+UMwAqDpkJjb1hIrPvArCCS92WtWKnIxor0K6l/RBEknxXY2pvbWChbpNTHRTkTZtfbEpNcA2ygclWtPtdKLQFLdWyvXFcrEBTYOl3tTZUqYjWgkGwtgLuMTYHjj9cxlpB2cS+qU5WyDsDCslkscGugeQrcVcqduStaWAFf6fJ6H5itr1mBy1NTWus7TrZ3ah7r/tb/q7Hep5ta9HePFuHkmN5kjHW/p08tr3sdsPwaAPLeNf3NAgAgop8DWyFfp/wFpdR/AuBSKfUHAPx+AP/p61z4icAyMwj9eSL63dXg3gfHWP6Glu9Ml/h33//78XLYwEeNjQsgAMexwRQMtq1HiBpd43E7tNi1Ew5jA6UI28ajn1xxdTwOLRoXSoxN6zw2jt0+X95sinshCwn8Rmssp6i4PXYl3sWahGF0aFsPBWAYXbGGiZAhDLQxaJzte6TErpcm55DUiuBsxG3fFotb04RyTf3UNA233diAQ67fNCxAXOx79JPDOFrstyOOQ1OscxLXczx0MJbTI1ib4CfLvwPZ3bFpwiJ+Zpo4H+acCHvub5GfkTC7f44WxiYYmxADk+9YFxEmA9cGBM9xkOLaGXK+xW4zYTg2aLqA4A1SVGg3ntfmtqtyZ85CjM5CmViz/GhZuLIzmYz0qXVCGC0oKlw+Ygbmq5dbKMPunSFwDFiUnIH5R9m1fN/9ZNkCGeb4GONY8FKa0OZcm36yvDZRsavoxOsRM7mMuC8qTWjaAOcCjocOADj/ZNCwLRPytJ3HcGwW7qmSz1FYhONgoUyCbQPC4FjQcwn7/YB+cAiDY5dSSReT9OwKajl+SGJ+SgqJ/LY2ju+htqmksgijgWk5bjCMFm7j4XtXcnGmmMdoiOc9GWghpVEE8rpo6yXnpBDupMnAdAFxsLBdQJxMTkEBTg0RNF/XpGI5pCxwu92E6A2T6igWRpVNMDbCH5tZ0MrEOOzWasqvk2n5uUhDvt5xjk4oFIEVJMcNmvMRWhOGF12Jn1IugfL14vJpu7x/jq78OBdLn5DkALCbgHDj4M4nritWOwDubET0DHIoKpgmIR5sERp1dvVNRwvVprntLJx2lwOeXrXFZVU3sZAjNZ3Hx9fn7Bbb8PjVNoB6w+6wk5mtU8RCutIEGjXUJoK85vZ6y9fn+yWuoKqLDAx6W9xvi1uqo9kd1mugjZwntc9zCxoqz416C7UJoNHMQnp2PxUBu3s44HDd8ZgVQbfc9811w+CCUNJ9mL3H4ckOcAlmE3B7vYc9n3C42RYBTG0CxOqsFBAHM1s2NQCvoLpMrpXXW7cR/WHDY0Neh6g432lQ0NvAAEWA2GRynlIq61XAi6QmSWDX53ycgoLZBvQvG147cUnOgEVvA/zzDvZyQkoK/sDvWMpj1y4hTQZ+0mVN9SaAVCYDiwYxKoTrBqqLsG2APzgol8q7JAyuuHIH2fNe83oACCPvbd1FBsbXDbtPC6hyCcoQ4o0D2shrYAm41WXOyEzGatSgXQCC5ndLVCCXeG8p3i+qi6CD5T05GN5bUcHuPMLLXwN7T5eAa1Yqsxt2Pq4JuLU8p6i4ns9AV4DXNvC5Sc+u4CmPP69Tuff5fVEs5l6YcfK/qIEzDzqyj6yaNGgTeQ96zSBP5XEFBUUK1EbuOz9nKvG6yZrWeXeFqEmIrpRXoJbHLICSTOK+xNXc83NMXX6uKd+zoHh8bYIe9Awi83s/dan0UZjZo6rcYmkGp4HdyynnxkwWxTNBRbXMNylrJS6uJV/mTGAleWZlruwyvQLnlF2RA+Z3QQ0SEyD5aQtozetPmttWcisVzUB6BQAX7eV+Fyy6wNINtiJYKgRV+SdsAY5zaEAhUqomdgejCjiT75ViYQ0EX0WadApgr+d56ni5Zg1cTwHZNSC9B2h/L5bvdfBLRH9SKfV7AFyD4yz/DSL6S69z7eu6wv53AP5RIrr6NY30My7nX3ub/o7/+J8qVr3eOxCArfMwKuFq7GCyNr8xEYfJLcCny3F7o7fYNh7HyRXSi9HbwkZ6vhlwHJsSoxcEZAWLlBR23VTILnw02GTQKoBPyC5mDS5r861JuO3ZqmpzvjrR4Kao0bShCP/rHIRSvLeL+lyX8/6NQwNtIpyLGIcGrglFey2uj20GzKLxdpktMyV9py4ABn45Pk+IGII3hWm0zpunTUKKGq4Jhc1VZ/AXA18TvM2CSwaV3sC6yELMZBfxeEonhImBoNt4pEyqUtaD5tQrImjaZp6PPMnJ6wJcTAbJ41UHEGD3zApbtNwZlKLqI055fmIdEJfJ3LaATBoNA0aJ2zMMopRL5T8IM8lDUhzDlhSQhVOlCcokBkcaHHMmgmYWAop7Yv4hUU0mdMjxccjgQx0NyBFfH9T8w68xx7iJYCG/grVWHZiBT8Ks0XaJjxN/VqNhAUYAqaFZS54FyIWmXQQJETAFxAMzI2mTmMlUUmRomscimvuU5yECw6hZKKoF+pQFiDbNP5ZxXrsSB6gIKgtyRWiKCmTzL2kWJKCoWCZ0z/F78SzOMZNZ4BUtOQjQo14IUwDmXIGytgrQg0baJOij5rqVY4E+6pld0RCUV3N7KvcB7kNNuW07WxbMrUbcJ7Y2eFXizxSB4wv3eW8HhdQk6FEvWVLrH/AsNJHN53NsXGq47ZIkXsC257VMbZrz7lWWjpKyxnJyexVz3CHx+pS5NQQzKiRHJdZImDbFomFvFVLDsYOKOCYOxLFwhSHW8DkzKIQtx+ZpD47P6xViKzeJ4x35fqEIm9KXIjALaY43o4yHVMgxj6s4MIlzM3NILp/PbZYYS49y78txzXFpgKwtx/6llvcPOcxsrJrnHVuCO6gcY1nlmUyqCJzJcWygrC0wC+VQOYZwzHGjLa+3ipjjOkOen5VjXJ+fIx57icdrV2OU+MkGM6NqngtI5pGvdRyTKEy2LLwDRs4LK2zD8X4lplTx97DDpy564nZJzeNRWciODQqwMTneUeYn1woYkvtKerkPamBQx5VJWwXQaMAeOcaSnw1uI4mTm2zbDJzK2CXGUq/W3VVgBfM86pQ92mMGS2nut47ThJpjLEmhsOhSfjYkDrOO9dMhr4tZPic1gCn19VxPETJgyiBOqdNxf3kvFYBF82+OxDGSyWzHeo5rlEJqXrNSTgGtNdjM87zDhCtjPAWAahAlCpW6W5rb/9QxlvXxU22v/98zhnreawvputyxJJ4ChK8odUzqydjUTyq5+l/9c692F/31KO1730df+qOfvSvsN/7Fz2Zu2aD439YGxTcpr+sKewvgryul/hKAgxwkoj/4aTr9rIpW7KJ42faYsnsZwMyal02Po28yUYzDvhnRe4ed41/jwTPBjcsshw83R4S0x76Z0HuHEHWxNj7o+hI36XTCpBjQjN7BmISLbsDLflPSaOzbEUO2wm1cQI8lsKFskdw6j+tDB61ntsKQGQhj0NCbNLuwRY7VlJQiUgSwyXmjE2Js0NiIPihok1kFE1shm0wgwwnPUYhpiitVtpr6mNOCBLNkacypLEICQBwjKgx8dZFUH5QUk+CkHHOnONYvRQaxwbMF1Ccmw4lBo3EBEyx8ZOtqmCyUSvx5cEBO6ZEig1dJIE+JmXZByBYyFFKa5FVJMp8mA6V4bE3DyoVp0iDFqTgoGCQCz4lmchl5j8WYLVgZUGozWymRMlMfstbYJqDhtZI+S85QRVzPECDU8zEL+e2szlOitVZZUN/QDL4MZtCWgaK27Kq6yAnqNQMVnXJ9PQNL0OyelL8Wa0YCn5MXv3xPaqnZTcjab2TNNpbAVfaFz3VCBiikAJ0qd61sBRQNv0PR7i76E0ZF+S/9y7kMApIGSAuAY5CiJ8XCmFyf11yAS6krCcubPPeI8saUdBRA7tckmIHHXLzABHTK9bmunnhdU4s5Fk2EAcPHCCyQpy3BTLwfqJIyzTQTqhC4zdTlaxXNmvUOc9qJMjfADgpxx32qoKC9AhHvIzsoxC3fN9Hwa68QHYOuKOBdljDwvSbDQDC1XI+FZ1WsDSXlglcspNlsyUkE2HyPc56/onkPioXhQjajZgE8Axkl+zPx/kJOm6ASYI8KQVHW5jPpDAMOmoVZMDA2PZO9qAiYIwMxc1RFUCeFco9Lmo2OFVsq8ncNwGaClpRTlwjJjPb8DJNhwVhAsekz8M+CV2wzEM7CnBnUDFAsPzNM+pPvq0YhpQlgkBjVDAyT5XOpYbIXskBsxLLDc9chrzEp2CGnxOp5PAJGGIjweXsEJjBQ0J7BQ+hmsBQ6ASwKNrcTOl432/NcJs0ARGWAbiZeR48MBh23F7MwLmCt7OEjzyXZ+ZzpkQEwA0+/nwlutOf67oaQ3Fryfv3iDsUQCTvkfVcpNQTIyvhIzf/dLQOY1ABmyJ8z6HIHyqBfFWCpAyEZFqLLmAW8GJ7LTBwF2J4QnVoQ0OhIiI3Kzz0htgp6IiTLIMyM3GBs8zObQaL2XEf7/J6xCqbn/TvnM53BOykB3AruSPzcqBlMxkbB9oSw4fnUpDsFcDsslEPyu6PDDC6TZQBfH5tTptASPMnPg50Brq5If8R6ye3yOyEZKnWADGh0Xofqd7IGNwWMVha8Mrf8bNUA7VRu1gVIE0WSmgF0sfBlQCnkStIej31WBNTgsiiOqvmfKvOYsgBES6B9x6VXc517rY6Lxpfn1mD0zhhWwJ7vF80N1H2o5eHPRfkeHisRRaVUUkpdfBqD4utaLP+Zezr/mTft8LMsj37kEf0Df/b3AUCOTTMlBkwrwpRj4eoYrRLnkww0cm7EHP8ohD8JqsRjCUX9OuasxGXRnJYAmMl6amKgNSGOxCZpRfASjwOUOL263SLP52vW36XtmDRsjgWSOKopmAooMrOrXCNpDZxJ1dg5d2Ud4yTWWRlXTQgkReJ4xGJZ54yU/2viIKrWa00etE5lUufFDJlsp7FxQRhE1Vjq2J51DFSd0xJAsQhPU46bLBZbVdxpa1dbyTUp85D4JjlWrLakEAJbvNfxRkuX4dW+SjnOqEpBUFP7i6W3HuM6jYHET9XtyLiFUZVJaU6oGOs3+vo8zSkrKJ9nq3VaMMyWfSAuQyf2QPmPO783i77Xayb0/irPqR6rxGqVtQy6uLhKe2LR1mKhVaeJfLhuHoOef8ROEfoIgE/ZwqnFQpnHKGQ98r2OE1tPfJFHM7K1PHkDZdJybOLWCswEO9WeqcdeFD/VL3mxmCPPp465yjkoF8Q9NZGNtFXfK7mRApBFoVALTdK+WLsroHuvK5WMTe6hgHVRIEgcm8xt3Z5Xy9gxqbPuW/oRJY5Yv+v2gaIsEYWAsH3WlogShyZCWLbCrudWLIZipZcl0NV4MSsGFvMC5nyZsraxWtP12kcFWFZMFeWLeCNQFW+WiW2oIriphWWxJhcrmFiNaBaWeQ71euTzme20KI3sDJ6L4JhBcx3rVlhSsRTMtSh+ZK2TWuRk5DHSnNsxybpny9qnLHW+0NrqpmTeVT/lFarma4sSoXIxpGpu9bOwsBytXCJZKYSZKbhqcyGY1y/Zymonfdc5JBeWI5rnIedLrKP8ZldWb66Tj5d6c5+1la00ceK+LkBHHs8CKCrcYTe9A0JWc7hz7J5yCjzV67AGQuufyoW18NT6nyqvEsFl/qfGVTdR76cTv2V33nWnujp1XMa+Pvdp23tVua+vV5Q37gPAz/373xvkPV/6V74LFss/+NlZY5VS/xU4HvONDYqv9Xolop/J1LTvEdEvfdqBftZlZ0b8nRdfR6s9xuTwPOzQaY9We3TK4yN/gU57XIUNtmbCTexwYXp4MngRttjqqeS43JoJx9iUXJcRGje+w8Z4PG5u8MzvMoFMzsuXc2BGUji3QyF06WODS3fEdeAYub0ZcZvNI0ZRyUXZ6oCNmfDReIaQDC6bnvslzv84JYMHTV/IaercmHWRXJfH4PCgYVWwkN18NJxhayfs7ISnwx6Xub0b35Yck2+1RxxCg9YE3PgWZ24sY7A6FsZdAddCDDPlPIuNCTiGBls7YYp2QfzSmYAhWmytxxBtAcOcxsRhaydc+w5bOxV35SE47N2IY3A4hgbnzYCrcYON9bA64ta3GKPFg/aIQKwcEBIZIXPxFTnPzrKFesokOValkkYmkSopZ751cwmtCG9tjkik0AdXlAytCZlshy3SfXCISWNj/R3SEbGSA8Dt1MAoKqlknIkYgy2EKq0NCxKbkDQOU4MQNfYtj1srKilitCK2vrcjYtIlH6QoLxIxK+vW+dLWrsnzjwY3QwtnIs7aCWPO7Sd9CBsxgALiORepLjkiCUzsIrkMRTnROU4rI/HMx7HBtp0QIlt+jaaSc3L0Fl3jMeY0MzFptDaUfiTnpCg0Ghsx5HQ2t0OL1oXCVDwFA5fBlOR5FHd1ACWtTJvTwvioEaIp45R0OXKcgAUBjhC1dC4gEco9JJpJiOS+ORNxe+xABJzthgIOfTTFKwBgRcrk2YW+a/2Mx3KfQhgFAOPosNuMOPQt2tYv0tpI+hmAlSMhGHStL+1ME8cWb7sJY/aeMCYVwpjjbYvNbirXsjt7Ym+EyaDbToUgpmkChsGVWGpr48JDIQRdvBn8ZIuru2s4NlplshrJx+jz2FzDHgoSLwyg5E8sxDJVTLYoeYLnd491nOZG3NmZGGZ20aekOA6wSbCO30vJsyVfXOkBFAVHHGxJk5NGwzGuvYXpYpFg0mQgeRNBgMpxgeW7zjGSCgz0ieMfi3s6kN3bWRHBsakSg5ulR5sWOQ2pjrmTGC6d08SUB5bYBJhj4ErsqoBMiZ28dSz8W5qBaMrzycdJUsSIK7ikENEAtTmGdmD3bBWym30CqCFOO6IENKriag3i80iVm/Ymze7IGWCrBKQmH89uv+SogJPiWeAIGHIcoUG20qviqizWztQRMMyWMHbTVEiblW/fG5Ticow8nsrlMbU5Rk+zx0RRPNQWb5WtYz4D3hzvp0dVwHsB+wIUUSsx8sOnCSZyShMiPi9tLlxaKbvHYrbaiqsxKcykSyYrDmQvpxVY1WDCsgrYSboXFXhs5GSNZgWJpF8hm71IiqcHCsBlaz5bC9nzhBbAUsdqTGaupyi3EecxQmFmBq8VHWm5LkUZJD9uOq+3AO8TwHANLO/gmqqeAOc7rrBS7wSQuqPTFUBbj2GtHMj1qK6vqnOYj9WW7Lq/O3Nbje0kLlbLea4Lred4Cuiu2pN/C0WAfD/h4vumuPLTANH/H5f/Mv+9cXldi+U/DOBPAmiI6LcqpX4cwE8T0e/9NJ1+VuXyh79Af9+f/sdx4zv4aLB1E2IW/Kec13HMDIdH77BvJtyMLZP3OI8h2MJm2HuLzgUM3sJoQmNiAQQvjhtsGg+fYyBFcG1sgAJwyPGXkvx8mBy6hoMvjiO7pdbpQpzhuE4fDS62DAalXxGaWxNxPbTFetnYiBB1YV2U50tcY1sbcDO0RTj20eByM6D3DmMw2LVTSYguLIkAcNu3xdXT6oSpJEanYhmsrZBTMDAmwWTgIf1NwS6siwCDA2sSxsnCZrKXlHSJ5ZRcjl6IdKpcj8YQusajHx3ahkmVYtToWo/GRtwc25I+o7aAsYVwBkLj6Aox0ZoISSmOD01J48EFK2Subtk12bm4iD0FUN52QpjkhTwlWwY5tyWPXynCpmWAJ8RGMepFLsuQiWJKDKsitC27Zx8HJpdggiReLyZKChgHB3EpLulD8tyVQiYLSnBNwDRy3LExCbvNiNFbjCO7cAs75Z0UJDSzrSpNJbekAko8rNICVFS5b1ozoGlbX/oAWGgVQGOd3N9ULKhMSMU3TCyJvJ5ADKbE3gqhEok1tiJkMjneNVUWQSF9ikHABoMwYxOmkfNuikWxXguxVBvLPkUh309taG4ruwpLDG7yGt1uglKE/mYm79FmtmQir6dxEcYQxt7NVv1sCa1je20T4HuHdjdhGlwhaQKAZuMR87OakuJ45cEVS5Ru2NshjGYmUcpABgC63YTh0IAiE+3MKXiApvUYDhxEpizH95ouIk4apuG51ilFtIssFPp8Puf7jLlvuUeUc+dKf3EyZa61FVfmSYHHpgznFFUKTFTUZVKl0UC3TDokltQC5rJVszsbMY2W565QiGb80RWiKMrxvraNCEcLZQmm4c92GxAGW4RQ3fJcxQKcvOZ5CZgNmutQtpgrlDVUOT5X4qMl9lpIouQhS14viJp0M5P3FMtxVNxPfr6knbJOQoAElP2QjhbmzPPzmGPBC/FXTm0jeUvJ69K+PKuUFGg0UE2EcQlhsFA2cfy7opmMTMBwjoNWMs6JY861xL33NsdM83yUJYdSMOIAACAASURBVAZZQjwkwHeq3r8S7x40E/xk8huxNqtm7ks1kQmeushg2WZg20XOA/tpi8R7A3P8OFAAfLEGC4lPVhCAwMA/Yc4bS2q24AuZVHXvF+Q96xQpBI7Fl/WRPjOp2Aw6aFYy1ORYYpmXGPSQXdNluSXWuSLGESKf0r7CXAcV8G/SPG4JN4gc/qCmCkHk974oD0r/eR0K0CoeJnxcS9xvXlqJ11cypwpwAZjjD08R+8TlPMRivwZCc+7M+cQdS3FN3pPfEdI2qksX1kU193GvtbMGoCdAabFoC+hcg71cV0iGTpY1CJT/p6yw0t4aCFdrUn9fXCvX48Tx+8opYLoeUzWuV5X/54/94e8Ji+W7f+Szt1h+/Q99tvGjSqkGwA/lr79ERP51rntdh5B/E8DvBPA/AAAR/TWl1Pe/4Rg/8/KWO+CnvvQ/4VvTWxjI4bG9wZAcnscdPvZneK99hmNqcGF6PPHneMe9xNfHx3Aq4p3mJV6EHa7CBo/cLT6YLvDI3Rar5wN7wGN7g+dhj188voMvty9wG1scU1NSdDxyNzAgfGN4C3szwpPBI3eLbw5v4SvdM3gy+NjvcWH7kppA0mxchw7XYYMf2n0IAPjYn2GrJzgV0WmPC3PE3xi+iDFZGCTs7YghORxCC60SAhlYFXFuBxiVcGF6vD8+REgGbzW3eOG3+Fu238EH/hLPpj2+tv0Q748PcYgtLu0RezNiJIuvH9/CuR3hSePS9XgynkGD0JoAnwwu3bGk/HAq4ul0hp0d0VYpQaS/jfFlvABwHTY4tz0+GC5w5gbszYg+Nbj2HR42Bzyfdni7vcZzv8PGeDyftnjUHPBk3KMzHu9tXuAbx7fwTneFl36LQ2jw7uYltmbC+/1DbIzHbWiwMb5YWMWqLGlkngxnSFDYmQljstAq4WFzxI3v0JqA7xwvMCWD3/34l5BI46++/Aq2dip1tCJcebY+C9j60uYKiRSeTztolTBEh854TMniUXPAdWhhFOHd7iXGZPHBcIGdHXHjOzxsjvh42uHC9bjyGwCzy69VCe9tnqPVAd/o3+L9kgxufYvH3S3GZPBOd41vHh+i0QF7O+HltEECp7SQ9CdiqX7UHvCd4wW0Sti7ET929m18e3yA9w8PcNEM2GSmiz46HHIqEq2ouJU3JmJjPPpstdaKcOGGYuGWlC/Pxi0eND1aE/DxuMOXNlf41eMlztxY2h+Cg9URX+hu8WF/hgdNj5vQwqqE5+MWnQkIpLHNFmaxND8bd3i7u8EH/Tne273Ax+MOU7QYosV5M+AYGiRSuMz9H0KDY2BQ/lt2z/HSb/DxsINWhM54bK3Hpevxq8dLJCiMwWLnRnSG982170r6lMftLRIUPurPYDU/gy+nDbQibO2EY2hYOdT2uJ46/PDFR2h1wM8+/z64bLk/bwY8y/0bldCYiLe7Gzgd8f7hQbEMd3kP98EVS/a725f4lZtH+Or5U7x/eIA+uBJX/tXzp3jpNyWtzJkb8Z3DBVtgk8HbmxsAwAfHczxoj5iSxcE3xfr+Y5ffwS9cvYM+ODxoj7hoBlxNHaxO+L7NC/zi9RcRk8ZF2+PJ8Qxf3F3jab/HW90BVyP3K8/EWTPCqoiraYMH7RFDdDh3A54Oe1w0PaZki6VfK8Jb3QGd8XjS8z6dEivZAmns3MRrYAIOvsFFM+C86fHB8QJOR/TB4e3tDRIpPO33eHt7g2fDrngCbCwrc4bslfDjD34V3x4u8WLYwuiER90tnEr/L3tvGmrbuqYHPV8zmjnnavZe++x7untOXRKryqTKaDS2CHYRAgYUIQopMOIvIRq1QkAJpRDFthQCwQb9I8QEG/JDEFEiQgQRUqVFRau07q2qc2/VPf3Ze6+1ZjOar/HH+77feMdYc6699r7n1j23uB/sveYczdePMb/3e5/3efAbt09KvL3U6/3Nc3zz+iketQe8vbrGr11/Db/n/At8sL0qnuK3NjcAgNoGOJPxRb8p/VozGuRJu0PKBtuxgTUZj+oDPjmcY1MNSNmgjx6HscLVao9DqPD1zQuMidpvkfFiWGE/1gVF8HS1RRcqJBCSQsI2ZIyH5NDFqjwntY24Hlqc1/T87cYaT9odvru9xE9ffYRdaPBiWKG2AUPyqG3A2o+4GVvcDg3eXN/iebfGm2tq64XvcRMaDMnj490Frtod3lrd4ju7x1j7AZdVh8pGfNqdMfrE4Ytug8YH3PQtrlretBtWaFzAG+0WzmR8cHuFTUWooUOocF51qF3EJ/tzPG722I0NGh/woluVsJGzijZvXnQrvLW5xSFU6KPHmCzW1YinLT2zMlc/3F7inbNrPO/XWPkRffB4/+w5fuX5m6+/7ljt8enujOZ+08Mb4kFoXcDHu3OcN/1sfMfoMDKi5M31FkNyuO7bwt3QR48+Ory53iJki91Yl1CZxoWC4tiOdZH6AYAueHzj8hk+2Z8XCaEnqz12Y40hOlQ8f5xN2I9kSF82HW6HBuuK+mJMFhc8T677toTPACib7y0jSvrgcdF26IMvMlyVi9gNNc4ZRbMbaqQMvLGmtqdssKkGbMcah6HCRdvj+kC/p4JIGaPFRUv3H8YJTTQGV9Ab3qYS1tOPHj1zM+hzORv0vAEtm5pie4y8IS7SPsaglCPM8oLGCHytlvxJibkfxmnJnGVj0kybcoLKEMSFIDVkE3qpU110hlXSEkA52cJ0r2V7ZuEhBkjRlk1fyxu+cr9sWudkqVyXZzZaaQ/31zJMRYegzPC4edKI1gSGRxP3lSSRCNLHZyEyOjSD79dhIiehuA8wLH+UHpaMMf8gSPPyA1DPvmeM+RM557/20nsf6LH8P3LOf48x5v/KOf9BPvbLDxHK/H6m93/6Iv+p/+bvRQTp6y01IUXjcR/rmd6fkPwAKLqMkopOXPJwfI9nPUdnSLMvsa6kvFwBzLQUU7ZznT6Til6h5K8/i2HQJ4oRrWwsGo2SYp40BCVpaK0uTz4X3T8FF70vLeum4aVJveBDssWwFe1D0TaU+FYx9LQuYsoCvUzls9YDnPqCFvbH6iXjp+NeZ32p/up2HGunJGtI+9KbVH5sshobqV/xOuO4hqRFnmk4am3F+XwgmKVR36VOArMUyKUu976267HS80DGS+51NpU5m/J8Dh/bUFy+vyWGWMcNa/1CgQXLNc7O45N1jKW0TY4vx0i3Q7zwus8Ffqzv199TRokZ1tfI8aksNYbquOgc6rhi+azbM9Nc5P5Ytld/X87nU0nGRsNu9bl8ZA7IYkD6Rv9Y6yRe5mUbtCSSTsuY4PmiS45Niy0AauGT75zX8dY6LRcfepF2qjwNy13+nBGpVip1z4xacD7enXO8KMuZFpKOIb3WTe9hYbzW/aDnM8DPjFqcZek7Pr6MZxZvbfGgiKeDr9HeTN3WFBfxynHydk+xtTz3k5ngtzp/ebClXgYl9rJInIiXw0wQYwRbPK7g2MjC8gxMUD2JU81miv+U9rIHcVYPHecKTHUqjVZesKL7KOek7pi8dFb9lTELhjxqr5vSYgwlaS8Ke2HveM44nja7KbZW4nFLrO3SW8T3Z5sptleKtsxcreNqBbasF9g6npn7QutGzuIj1YtfyNBmcZsR8/HgcoVwp3gxA2Z5FfiseAwlX5mGug7aeOE+NvIMZMz7SC5TP1Qz55Z+1BbHlhBU/Wguz1Md58Ot753uuTs3jnnw5LI7ZUq5y3KW9TKn/56qJxWOmafxTjpVl2VfzF9hL8/ne0gn++lYesA1v/rvfjViLN/90//Kl57vb/7LX5431hjziwD+uIQ/GmN+AiQ9+Xe87N6Heiz/H2PMHwfgjDE/DuBPAfjfX7fCX1YK2eKT8QIOiWIiQwuLjJWjnegdxzb20WPlRtyEBmd+oB3C5IuHp0+evW2uxDiF5Iphdu577GJdvEI9kwB10SMkh7Oqh4jbk/chYB9oh1A8MceMjNpGbEODkCx5LLKbGXLkReEYsuSLuL0Wqh+SL3GFK08eKFmI3vTklatcxG6sUdtIcX4cHym7iQMv2LUxK0bJIcwhQxTzObXH21S8BdrAkbbGZNF68mzJYre2sRhgh1CVez3H2Mn3/ViVXVOJJewCeTc29VCE35eGlV50106M38mwHvi+DJR8n+3WAIDzlmJMJZZQIMIZKHOjG32BPesxBYDA7MGy42pMLrBjMj5MEXZ3vNjVxmzPsYJalN3ZXPISEfdjZEgaNp0yxUPWHO+ptVIl3jFjMnz0BokBimEovyvSxqjIicourVybScd1GD2qKsxInMr9DAfW+qOVi2VjJpWFO8032WWufMSgdqkFtjrtAKuyOF+BIGtW38S7tkLSJMaJiNfLjrPkL3XQBtfS2JEd8mEg6r+mHUo7tJEmdYg8xp5jP+U6KmsyxkJgUfvBw1eh1BUguLOxtPMspFGiR2sMCkzWV4GMK20UmIzQVfBNKDvZMpYAEAfSTS1kVMx6rHVjl7v5Ev+YIrFNJyYeEkiqkEwBIOgzwzP1eTGyZPdbSINSsIUQyVgUaLFoiWpioyVpUe4ckdZ4ul+0R2Nt58ZLNsgi5RPJcEoN6R/Gyk5xaCLTA9AiRsnbzBb3wGTMJIqpm2Ie8xTDaDNBF/XqU85LF2vtYinLUDkZKMaJCRZJ4LZs5BWCK4Eh9rYYNjP5hTQxLhfyHmmHGFIGlEdgxmRvmQ2V2jWLDRSj0an+EEilxFVK7GSejB3wvaUOScEvDSaj1DGk0mIykITdGZj0CCsmLFKEQCTNc2rl/fJEzNY8Hoq8CJl1EEWmQ2IoZUz53plhqepvZTzVfJqt6Y+Q99gRxBoLFMkV2Ekj0fD1mScwMaOaKc7QasIjM1vAz+QrJH9NGKXmvCZNAsRInlsfwvI8YxIvZYmRreCvZrK+xAiVx6TECmJuoBYDeLlvsDTAjhhKcl8hoTpxzfKYaiLKA6kN3yVhkbr+QYblfcbeAw3LUveHGJYLo/1kHTG/9mhe96Tl3sxLDUd9/r5yT5Tzo/TKqdKcOjnnXzPGPCiG4KGG5b8I4M8C6AH8ZQD/E4B/81Vr+WWnMTt82p+TURg9rkeCcm0cQZueD6sZUcw2NOgiwTP2oUbLUMAheQyeIIetG5EywbZ2I5F/YAVc96uy2BeimduBDFeB1FFeDptqwG4k72njAiIbnGIACXFLZSOu+xYGwMERZCaDjLohOJw3QzE6BNqiSVZSNuWeIXiMNb3VBb7yfL9CUwWqa9dg3QxwJhOUhRevkrcBSjyqwI6kruK5AcCxnq7ET5LsikdThRmDLIBiQI3JEtRNEbIIecuBtUNDtGiqUOJTh+AoJrS1OAxViQvtRl/0PiU/7SErbL288K8cxYiKwQfuK+nXtiKx+t2+KeQvY3QYlXFU+zArQ+AwYrBor5zAaFIyvPhPHB9pCySmYi1QLzqYJpf4z2HwSMGiWY1lse5cQt/7QlxCcZdmFutIfQIgG8SGdEPH0dECGmT8DLua4rCakWJLZUcVKB4cgeYU6A3/qhQDLLrC+Cqsq74iwyMninUMgyPq+jgJrxU5mGgQPMU9WscGIBtEABuW2RTSCvIaWQTvEAaHsaK5aix7k7iuYeTYYDcZPaEnEfhYTX2QI2mUJhYjt1bFqWYUfVOSqGFjsyYvFrG05vJZWF1zNIg+I27JsOwxGYA5WMSatUQzSpwaEpBW03OVGRoVWaIHAHLvkNcGaeeRWjPpo2ZDouii4clGSmpNWRjknpEMDce7AUW8HgDMzmPMoNi6wZLxIgv/waKYvMEi1WSgpTpRORIXB0zxfInkTyDEL9Eg1mmKOWPyF3oALZCAGMyc4VUMB/EycbtMNEjCtKti7nJlgZEMnJnkDibDyW0dUpPpGgBmMDDJIK3iZKRwvezeImXABAvTGzKO9xZpZabFD8fWlYV0lWeEJyS5Yqc6ZDIwUpO5rdQPoseZfYY9KHKeDNL503qmvZlIScQGtZOep5RFxCi0ME8VG1pMHGIHg7hK8LeWpSMmcp3CpmpRdCczS8uQIUhGBwwQGyrHDSRPYRKdN8lwmZyP43p4kqKAQZHRsAO/p1a56FjCGsonksyL5TJNBMlsyKJZ+t0b0rFk6QsYul/qnGppMxHmkNwEGauuMxi/B4elU3JF0naTQXO6RZHYscrwFcPCMUFPqpjcx1J/FfIegNh25RFT8W13CGAsyaskljOicZ76RAwdzYKryXuK55C5qYqOpcQcCuOsMhpFe1VSmV9hMvhKOQz/IBZdo3QsTWmDSGPo+7WHlF4wUx2RMemkKtkQfc+SuEgfKzIp2nA0U75F5ihhZpxmg5m8xywpY+yYoSkambOxWxqEx/KT6xZz9ZieY7ZmpsF51ONpMJ3XdZnlvTAkT9TxlQ3VY8dPpWNlPtCQ/KExIvX8++qmXzDG/BcA/iJ//xkAv/CQGx8EhZ3dQMKZm5zzzSvd+H1Ib/3UVf6Z/+oP4xDr4oGUWJUxuRIbWGIEbWQWVWI3BVBihYbo0LqAgaGlEvMmRugSzgkQpNNbirGzyEgwqG1AFyvUNiBlWyCocl6SMG6u/TC7TqC2wsgqBql45sTrV8hq+J+wnQKTB2nlRwzJUV+oWA1t/PXslQUmKCe1bTLYvHrD6v4BJimXQUF9i6QLG3PiWZR6C9RTDOw+kAc4ZoPKpiLBUrtYYv3EgK5dhOeYkUrBPAEUr6LUHwDGZAtBk7RaPI9ihMZE2qMAsBvqwlwb1UaA9kqKDIyUrT2mIjUjfZSBAhvVEjHeRQzBwZo5NFMIoWSjAiBIpsSQ6Ps0IRS1mf6Kwe1dQhCvm83FQB7YgyXXa/kY7f3U38uct0nNPbo3RAfvIqwhcqfaUx3FkBfPp2MDU4z8kCz9birvq12Mn4aDVi6WPl9CauX6jKktcr0Q8gipk3iwl3BGAIVkSDYZTOlP1nXlzzrmxRjyOAr7rLCw5myKx3WCg+YigSNjI3nomJ3CssobHKJvK+eqKkxe6jRdS30DjhEixlZhg9WwUYkXKpI87J00hjy8Ir9jbS6bIOKZjaresvkhfSCbKiKR41xW8Fd+/lye9R/1u5n1g8QfWSaZEqIr5ElCp8QT8UaTxBJBzd+qDjNyKrlXILL6WsfxVdZkOJ9mMVeymrY2lY0WqcNsrcjzRuYSeE7PdIwXsVJ3YceKHImvK+u8NJdBKnOe8ynkUwkwQsKSaAMhBWHXndAGZU6YPCfFYs+xtDllWrWnQBsc4p0GuJyFt1k80SKZA2AiOZINGSZNokbThoIxuWxYTVBh7juBtAKF/KiQIbH3VSR3Cix4tDBeyeVkIqQqjLqvk1ymjRhgqo98Fi90NtPmgLQPIPhvntorns6yaaK/87iUlX5Sx2Xghd1XvmvvudxnVPmyAaPdb9oa0pNZ8pTPmvhFX6eOF5KapZyP8ooXKRKVCsxXE8UcM/65LdqzODN0pe1LQ+QeY+WOEaWhzuV+M/WdLmvZjhO3HpOSmZV5Xz3vSUv7VP+9W7n7Tp4u815j7aHmw0Pb8+D8zHzevkL65r/+FYDCvvdefvdnvw9Q2J/9UqGwDYA/CeDv50P/G4D/OOfcv+zeB3ksjTF/CcA/DyJ1/usALowxfz7n/B+8XpW/nHQztPirv/WTGKNTMUDkmaAf7AlWJoHTRH+f+IeRFyTMVqqfZ1kcIE+MlnrxQx84pqzA8DD9wDIsKMVJMy8vXnQ5q3OLF4+x9ANbdmplEb3cORKIjUDEDEPeop08FIk07krMjDyQ/MM/W8zpHyTxVqm4B4HSFWgYe7hm90uSH8+l1pyCYM1os8VLsYwTkWvk5S7wMR3To38kzaIOyx9jHcPD+X7e8NYpQ5V0TE75gZN+0TE9Oi0XE061V87rmJ9kcCcJ+53+8dX1P3WfTqcWACXeCbjz4zJbHasy9XdgPl76nO4TfQ3nbfKk7SfaZxNUSK8M+BYxxJKCxC1ijPQDW86pthRPkn62DOU9W9xIW6T+nO/sfqPatew3vv7A7IpmnIyI2aIOoOPieYi4m/TzaQFEg57HTS+ieq2fyM+NCROUbeDnx0RDn7OZ/Wj3DBUEr4VnC1iHSSKCd/EDwxNH5Q2Q8qOdFk+jWuyZpKaxmkNBawDeM79M4rwV1M5koDjCI50vUD2o1wDn0SnPHUDXy72yJyXnRk/tTjYjch+MFXvVOONk87SOMVO9pHxkup8upv5Mi8Wq4bHINsMmkmIoGfC12l7RcWdWTausLpJ8LMfOWbXwNjyOLpJHEpmul9dR5t8ty3M9u+l+asY0fxw/39lSWZNnafpeysRUtoxdlspL36nXkZxjYt/pFaXnhxrfom8p5ckzyuNbtCyVt0rmtD327D0wLSGhSw/ezAMmdS/vJHVMLziyOqffnbOCMXsHGL6nyJ1kzLx15TYzvWOL5uQd1xqK1316B89fC3fapJ7tZVuPtpN/J2bPjJnqfcfzdWTcdT/oPCSdNCzvSfp9NVlmiwGQY3l+3zKdNCyX1z6wbnfWLtB1XORzos3LMXwdw3LZhqNtWuZxagyWdXhoX9ytWanfj9L3JXkAfz7n/B8BEKdi89AbH5J+f875xhjzMwD+RwD/KoBfBPADNSxrF/DeoxfMqGhxzYyNGz+gdgFfdBt4kzCwx+52aHBWDUgw2A51gZYO0eG87rEdiIUuJov9WFFMmk14utnhuieYrbcJfaRYzMNIemyXqw4de5j60eOsGbDtaxiTsaoCBtbDkxQTxcHVLuL5fgVjCLJJun+0gx6CxfrsAMe7uP1IMX+Vm5PLSEzeODpsLnquQ4XaB2x3LXxLOn67Q422HQkKy54KY4B125Nn0UyQVsMeHWdIKzAw5BAgT0eIkxfIOZJuEM+AZioTaY1VQ9BWOS+QUK2PF6NFVQX0fYW2GRGiRd/VaFcD+q6CryJBQrsKKRo05/2sPPkrHgTZ2a8bjjsNtnhWSGuPDMm6ofZun1GMZXvVIUZTYspSNkXqQ8oYe9HRi7w5MEEMw+hohz4bxI4kBiSWTMgzbJWQgoXz5NG0ZvISxoMjOvXNWEg4nEsYO0+wxd7Bno1ABu3sO5LBSBKflAx8E5GiQRpIJgLgTYedR64T7CpMEg0AeQoUrNPYKR5Pdv7LYmlwk1aeiK5X5BlANDAbpvhfR4JIArTYc4nKGyywYop82b2vUll4SwxcqVsdidp/HYHOIVeR6mtz8UYAAAaLnDMZz0J9v/fIPsE07MUIhkhHokFes2wEx/Eh8AqxneqCwDp5AsMMls7LZ8/9MtJns/PkWLgIkwEaLLVBftSjgRmo7/MqqoUlt9dPEE3TW+RNgN16pFUssEUAsHtXYpVynWAGi7SOlK/NBbKZNhFmsMgmlRg4APA3DuGC+3KwHDNH4+F2FuGc+2ckzT9zcPR3MKQNqBdzAj9sJ3inGQ1Sk2AHO9VTpGR6S4Zdm4v+HEEG2ditAEQgNQQrtYNB4n7PDrAdG3lthu1YF48NUiElEc3E6sYiNkCqM+0J9WQghU2ieoMggCYD7mAQNgl2NHB7g7DJ8DuCbAKkNegOhmwXWUB7THDSjAkCaYDkqD42EIS0kLVYkI7hQPBAtzfF2ALovlxNBCuuU5BJtRD3eyFgoX7xO4vYUjuzJ3ihSZSfPwBhBVTXtkBe9SaQjQRxzh7U5praDoBi/9hpG1cZbm9ge4PYkpFgIhgKy5BGi6IbmVQ+qaY22YEfjTXBbcVAMiPXt8EMhplqmrfZMjwXVP9qx5qNDP10I0EtTQZiTZ9jQ1BcgsLydR0wnuG1E8GA6bMdUZ5hk+i4HalObkDRjxTYqevAUOGpr8q5nvpYQ2EL2Q2PKTCVly1Q7Qh+K8+GtHUGB1VQWDcY7psJXm3D9BwkgYHmaQx0Pe2IYkyLEZw9t5nrng3gu6ntNnKdGKrLHIsLHUtMz4xsFDmU3x2aYyjGaaqoflIHgX9ma+aGLKa8ipdTNqnkniWc1NHfpSGlJU7uGHxSTwXJ1f0/u4/bcNIA1nkb3IXCKkjvBI3HbBPl6MaxUecfakSeqGPZ4JB2nzLuThmy9xiDR6G+/Pel6YfIyPwhMIj/FwB/GMCWv68A/M8A/r6X3fhQw7LioM1/AsBfyDmPZrmj8wNIZ77H3/X4A1hk7BPJDoicR4TFj62fzYhkdrHByg7oU4Ux25kYeWUSxkKhbwv1u8Bb05m9w+aoWVyBCbYnhEACzZVEkNZYGGKtSXjnjMh3vEmFHVbIfjSjqyTNlAlgRkqzJM4ZLhnOZhLSo4lRVqC2Q/KwyIXJVb4LZFc+6zKknpq8Z8lUeozEB5igsfpcPJ/YRHW7hH3VmVQ+W2SkS1P6C8AMXiz11fXQbJpSr2WfAcC42Rf20SVrrFwnMGSB9kpawkVlXARKK5qfM/ZXTO9bOZ5xHBZqMLEOCzxZ+mjJujrLL8/ZQEN0R/saQIHwCpxXs77qFJMpMFzNmho5RlTgt84ef2tKbK+um0BqJR1jBi1EMkcYV7OaM3KPjJPh/ijEMQtCIt0PGShjJd91zy6ZajXbrDUZ4THDjhXVfVqMgcAQgQnKuGyPfBa4Y3w0EeZoYqEZI2ieM6cK/HTJzFr66olBZXLp1yXioLLTOUFr6E2jZSpxqUfG9+611DZvT7MDa9huzoBTn4v3zhDxkLeprIaEel9gruGK+s6rccsAvGzmQDnb+f2dQfPU2Yz4xBQkiskGSXs1JKlFYM5AzGpucl5Gw1llDPnvWEiEjnWWWlfqFZ/BLPYdJmOMk07p0iMVuA7dlZ02VFRf6nvGRIvzUWJCBWHBC8kxYw6nXC6WpS16Ubo8r2GtUMcMl6Pzl+ukvlzP4ch7+ujCfbHQLrGlr5tOoTF0u4E5gsKoexfX3UFL6LacSnxdp9hdAd5Y0QiE+ce5V07XQ+qojaVlPZSxpI+TUaPGDpjDSZWLtaBV+PjMgNH1WaZ8t2pGf8mLC0/1nW5DLOzPVAAAIABJREFUmZ+ZP8//3nl9qXYcg18+yFB42TUvOb94rB+eXta/i7LvGJivUMeHpPvrbo7/Pfa8v1b+P0oPTG3OWYxK5Jy3xpj1Q258qGH5nwL4TQC/DOCvGWN+DMD1K1fzy04ZWNsBHw2X6JPHmSN9xRfjGjehwdN6i0OqsbIDng0bPKr2eDZuELPBm80tbkKLIXls3IBP+zM8Zu1CbyMufI8Lf0CfPH5z9wSPa/o8JIeNH1jj8QAA+PBwgbOqR0gOq6rDJ4cLPG23sCZhF5iN1cYiGbJyI25Dgy62eH/9HGO22IWGNBiRceECVnbAR91lMTIv665o9EkcpjUJa2aCfVQd8HF3Xthku+jxjbMv8GxYYzs2eHN1g+fDGkP0OKuonxIMPtxdYlP1GJIvOn2Sx8CMt3UVC0Pui36Fs6rH2rPsSHK4avbYjg3W1TCLedyHGms/4Hm/xqYacNkc0MWqaAfeji0eN3tc9yt4JkM6r3vcDg1qG/G03eKTwzmetDtsxwZ99HjS7rDxA3579wgrPxKrrJ1YZWvW1xRD93m/RsoGjQuQJaRopFUu4qZvMSaLn776GCkbfPP6KRofcF51RQ9xP9aAsM8CeGO1gzUJt6xvKaRHIVmc133RtXvS7tBFj9uxLdpzm2rATd9iXQ1Fq64P1LfOJjxu9qhtxBfdBgBr1AVPmmfJ4bzq8KzbwNmExgXScGMmXTG2r/sWrQ84q3u86FbIIE2y9zfP8Vl/hmeHNRofuE9IK1NiOrWsh3j0x+jKponE7dY2FgN0O9Z43BLR1HXf4p2zG3x+2ExEUNEVL/8ZIwNErxAgsqnaJWYQpjqJluNuqHHZdLjuW1yuSaMtZoMQHVY1MRqnbLCuxtKXQTTjzq6xH2vshrqgAmrWOnzerZjJluJWG0csqKKjFpPFpibXyk3XwlnSoDyMFRFQcfxvTAariuKE33v0At4kfOf6UTGsGx+wY/QC1SHhvOnhTMKzw7oYqrJBQMgEGsfH7QGfbs/w7uU1vthvSuysMRlPL3foo6eYWWZJFvRDzgZnDdX9+tDict2X2FqRS3n34gbfvblAiA7rdYfGxTJGj9oDPro9R84UN7rtGlydddj1NdbNgMNQlY2HDKDxEd5F7Psa522Pw+ixqkK5XgjAAmNY182AxkXcdE2JHQ7RFkZjIQbrg8NGXWtMRogO522PDOC2a3De9tj3ddksEiZnmcPvXl7jebdCN9C4nTEh2he7NRpmGO4DbUBcbfb47PYMTTXiou3x+XaDJ5s9nu9XZaPmcn0oBqi3CbuhRh+mGPIxUp1jshhZM29VBdx0zUT+Fokhe8UkZZerrhi8AHAYKgzBl02UTTMwmgUFXRKSxcWqA4AythdtX/ppYKZuXafr3QpvXt5iiA6HoSpzQRiu+9FjCB5nbY/9UOG8JQRM6wO6QBugN/sW62bEo9UBn283hRzOcV/UHNdM+Sf0o8eaUSNybN2QduWL3arEJY/Roa0CnE3Ydg2aaixtOPDzk5JBXXMcc19hs+rpNyhQKExVRazqqay2CtgeGqzbHv1YlZjry1WHz282960s7k2rZsS+q5ETIV5knJzJ2Hd1IWcTpE7OpoTLSJ2HwaOpCR2VEiGUNqthxiYOEMu2RjHpTZ0YDc43XemfcXRYtSPpP5Z3BYX1UIw20DQBfe9RVYQSStGi4j4dR4ekZLCIlI2QNSlRG2oVs2xsLuFFVR0KcgqZmLEDY94pztwiBgdfh4L4KbG5mRiyCXHkyoZMjqboGQqKBnw8ja7E09I5lNjemTYkG1KZN11KrC3fh2wIyWJBqBWJWZWNFEkJFKsb1EFt6Ij3X0I89MaJxOQW4z1PpGV600GSbIZIGXpjBbgbRqSv07G7uo5iKeowmWXSG0H672IjRferbJ5ko0JRdJb6uN4EMHi5kXtsA+93k8H41W/Lzhjzt+ec/08AMMb8IQCHh9z4UB3Lf0N9lf1Al3P+udeo7JeW3v6px/mP/cU/gsd+jz55fDacAwAeVXucuR7fOVyhcQHbUONRdcDn/Rmu6j0SDD7vN6htLEbe15pbfNqf47I64BArbMcGtyPJdby/eY5PuzO0LqCxAYdIC+IbNiyetlvs2AghI+4WnxxItPiq2eFmIEFxIfQZksPaD1j7Ab+1JZH086pHFz3GRAvlw1jhjfWuGEq3Y4vKRib7meRGbse2SHM8XZMQtRg2H9+e46wZcFb3+Hy/wWVLQtbboSEmWJPxZLUn+RSQSPVZ3RdjQBteQoqzqYYi6Gx5gb0bahJcZuZa8a7JguSy6bAf63K+8QHbocZF3eO6b7GpBzY2B9wMDR63B2yHBrddg0frA266BrWPaD0t0PqxwtVmzwZJLB5TkglxSGnyLl/y4kva60zGrq9RMQvqRdvBm4Tf/OwKAPDWY1p47foabRXIcKjHmcfxlgWe6TjQVgHd6OFsxmEgGHLOBrtDA+cSmioQMZGLhfVW/qZkefFAC85DVyFFh9WaFnXWEIvu9tDA+4juUGO9JnkbIhdJZcFEbLEW67ZHiA5dX2HdkoExBIf9TQvfRJxtOvSjRwiErXM+FYZbghenQjpT5EjYABh6D88LJ1kENO2IofdIyaJpB3T7Gu16KIsM51NZXITRo25GDIMvZdZ1KJ6wkdldY6SY57oZ0Xc1/yVItMCph97D+QRrE/q+Kv3lPC3yu20DW0fUvOiJgSDcabSo1yNEmiSMrnjtfBUnSY7eAwao25EWVqODY5kSgUobvt/5iPG2ARLQPO4Ko24MFlUjRDsguZO9B5KBOxvLD39hwfWR2HRNRtxVqM57jDcN3CbAulh0FMNtRYsgA4qfHhzcZiyLqnjgjYL1yO3IsFUqmon5RQ3zaCDW3N4xnJd36vcOuBwpzrt3cKuAuKsIQn3wME3kRRnVOQ0OyICtI9LBE3RYpDsGVxZpRkhcOoJ7m1UkCLWGYgdTyFaMT8RuGwxDlHlx1zEKpCF4NOo0BQuKFqAnuLZ74ZFWmXQLE2B6S7DN88DBpSikKXbrkC4DMFjYg0U6j3A3DnGTpsXowZVYJYJt5iKPQXBFZnkFChzXDgZxPUFv4UDxuz31ud1LMCTlm+pMUFjxbHcK8iqyJS7D7W3xVmVPMN24ygU+KuVln+E6i3AWUT93yA6FNVbiN20guGKqM/yBoLCuYxbXkRh/swXimo4LVNgGEEwxEPOtCZRnqrh/OB9khrgmlO9hk5lVFcTyORIsM6zyjL00NdQvAsU0maCufk+eR4HC2oFZV0HQY9crOHOTma0V8HtgePT6qzrPbQeoTIlNNJHgvW6geFfXmwKBTPyseobvppqhr1z37DLcgeou0GzJs0Ck2Q1QvFYOqG6B0LJj2U9QWG10SP9KfVPF0FWGtzI5PmKNGQusDRNkV2Cqus42UhmpIthvNvQZWEBhA8NsuX6FUTdNHlQ7cBl+arPArw33g7DAZoc7rLCaVXYWZ6r6SkNIj8FWC1su94H2ptLcU+yuwFHPrzCuFvbVPEGBJS/D+QEgiKfAdYw6L8+7MbBxPleXUNhSPx1PrJ19Ks/CunvMIyv9pY3JYwYe1Bw8Fhur85rdND93xKF9N64X0zXmJXVa1k/nq9Mv/JdfHsHN66bmvffy1/+lL5+85zf+zJdK3vOHAPzXAD7kQ28D+Kdzzr/40nsfaFj+afW1BfBHAfxqzvmfe/XqfnnpnZ96lP/Zv/wPo0+kS5myRYIphg1AsE8N+Tt2nv6ach4gWKX+rvPQkMbZvdnMIKEAGPY6QVCtSTNmWSljyRor3wUKuyxTl60ZWpf1W0Iw9fkllPBY+5fHdbnH4K4aZqr7RkMI9WfRrlzC+vR1p8o8dk6XeSzJOykeyX/5rtLlnSq35JuP989D67eEQd5tHwqLqz72KkmgpEuoqS5zCWEUOOR9Sc+hY7DcY8cl6Q3b+5pz6nfkGGxWf15CLeWe5Xw7du+pdOqVeaoux44dg4qeOqZhofflr+u1JNJafj/27J9q9xKWukzC4Pog/NE97Z+XeXosXmWeFO+FXqksVzKlzEUhi4XQsToeG6+7F6qKLfM0mMMGjzVoSf50ZNxlDE4SqZWLT6zilnWT+p1aHGoCsmMP8X0LwMz1WEJh9fFln5VF/pFxXOaxLF+8NMpQOAo7fmh6wNwAcNSDcy8k8WWPz7F7lnPn2GpaP+NHFu0n++MBxsJJI2CZv66/nisvq5vO49i7Z9mGO/MFx+f1qTKPzdk8v1Y+H033/G4cLe+h6SHXP8Dg0vV4pXJeYsS9NL1qex+YXrkfAfzqv/PVYIX9ITAs/xhIWvJ9AP8kgL8bwM+JB/O+9CAobM75P1wU+PNc4A803Ywt/tePf7xIVHRMplN7lqRg+JPIIQi0BkCBnBVdyWpEzzC4DNIjFOjJuh1I4kHFOQEolPxNMxaoSwhESiPnBHKif+RJHoCYafuOJUJ8LCy1OZOOn69jWbhozUJNFR+DK7ARX7M3hb1OoatgxFvUezjOT/T8jM3wNZVrTC6wF7AxIYy3WcVoifZginRM2GiJkEYWNupcMHAV1bkQw2gSm8HRX6all+MCd7F1RBqIBMfYTN6RaGDbMC2kGC6DZIoeYfkxqFnmJNqykBEvCQBYn2AsEG9pHMw6FIF0eCJyMULeAgCZdQEzyMMDTHUAyGMi8TsDxTMVchqb6XyViX1WC3+D68y6fbmW7WZM2n2GRNBzzfWRRYWQuhTCm0TlRENlA0CwsJ0lLcIqT+ywGSiMvdw+6pjFopF/EU0U7TswuyBm37Mn4XLx1pjMXhH5QWaSmCKULeWXh4OvExZDJoLRQud3GGiByRtkUWJ4bGeLzmDJM4G9K6lcb6IpDK3ZT9AdMzIRTJXoWFTno/Qb6LMD7MEA2SCtVSBSMNP93N92oM+pPrL4U6zCtrdI60Raim2azRPb0RzOhtsQyHsm42UHNvTrSeh+Rt6zJ89Qtix4r+BOpP/HfcZMoranvpQdfR32IsQa2YMJaTJrTwphDY1/YQcNpE8YmcwFJk/n9G48C6pbYTNNSh8QKFqL4g2SZ8IAxRNX7cmTJqQgDACZyHQwzSHXsw5hJA9KbAHbsxfGSL9OYyDajYV9VerMHqC88NgUTwzPG9ESFG1DmQLZkQeyzINxWokWJlQ1xhJTZ0fDrK/8jEUaoOzoXGyYSMfO8xHvjZAKuYF1H7mt4gnJBkyoxIRENfcn3588JuIVrn9W+YgXU+6JDYpnSMhHTEYhZpF5IB68bKbxS37ykmk9RjtOZdmRPIN2xJy8pycio9dNNgAMWiKvXUYhkUk1ComKHYG7OpYohENFE9JO9RKvkvzeFO3EPM2n8ltkH6BjCRQCJwBzHUt+jkVLNC9Jf+I0LjJvS531vPEqD87TDup5j5y3kPeIxDrfP6vDHR1LdV1G0YSckxPlad5ZzIle5I8jz6D2EOqfPD0Hj3n15D13FEaqf8MX3kfpf82MLPnJdaXu8pjn6fpyjypj9o5c5HNq36XsH/F99xqWuu6nDGG+Ttf5VdKDiHhO1On+fNV4vo4B/zuUTP7q1k2ln8s5/7fGmEcA/iEAPw/gPwEZmPemh8ZYLtMawNdf894vLRkzPc8AOK5pIuqQv84mOJtLHAxAuP9C4CI6jhxTFlnMfkYwcoSMRJN0yI6+GJ9yv+SrPRWi42ZMLrpqxoA02ZJhMgBTYiQAFNF6qpO0jaRSDIAEq8hQ6D7DxquxBIMjYy/D5DwRUphMRBcAjJviGCzPfGOnOug+NRalHsQammEc1RMJJX/rUYxMaWcpC4BxqZw36rhokpEhmkqMhXGJ+oZjKmTBZ3hxWjxwbHyXWAs7zQu4XHbFJc4DzORqDBkumQ3Zknd5q+dJSF7nzedyJqbWLIanyVO8hskkNSF/RQZCG8fFWJiMsVKmych5qruUXWgq9ctXjFszfU9NKmXMdoCNui6rui5f5oYJSCx1fFbtL98tL4rlmPw6c16Z251V+2ceCq5IIcNQ+dE9qm5QC+4K8x94YDIqZcGuCEj0AkFfAwvqY/1i4WMGZvZZ2i+fU8UMdkrDzXjwDzDPmTwt/ovsiOr3bDKEGC3VGdkQLFJYU2kYyNAu/W9oMaUJSwr0yqKwvdJ5uidVKEa6PEeyGE2St8q39CUbKrN+ln5hCKQ8h5kCJkpfa3mMYtzIZo9aIImhIPcno6413B7QsZTzjD1S9wlyRmpoUS1tTSwZkC3KxpC0JVXUNtpP4c+VKeVRv5oy7TLyZDRwntkq55jky/XPeo6ZaQrHRvWLLKituh9Tf8t9GUBu8gI2x89KlnchjQ61m/opNXmCxOlnmzdpsgUiVJ8CFLfGz13yXI5jeKwzxaiaGQVWzxWez07IW6S/M6yl2NJs53Bi48wEUXR5cm4xbCPbjAhTnuds6F1eIIYCMa0wQSTddG0xbl4jZYsCLU3KE1qMYKmHmeojfRkxHS/XqDboa5EX9VTHpR7Ur9P5rPqg3JanvIvRWd5L9NlklY/cJ0ZfUnVU80Lqki1g5F5lSM+c46o/7jCkguvDbdKbXNpwMTIJjLpOLsqYPQt3PIdmuhd5Lr00a4eCW965Zun5Nkc+HzlnEjHOUl/I++4etlP1O1bOL+uC+bgeMyyla+Ra+X7SI3wk3WtYSjvvuX++ftDfzd1r7ynzwUbiQ675UXpoEmGgfwzAf55z/h+MMf/WQ258qI7l38A0ZA7AUwB/7lVr+WWnykZ8/fwFulAhMORVi59vKtoq1XBWLVJ/F8I3qs84CYO0hhn9gBmsdnYdxxHcB6lLmeL3jjGiArgD35vfO8/rqNj9Ot+5dwktW0LO7oPOHkt6La+rdKwN5d2b78LwcOR4geC1qt6tOq6ueRk88RjkUV9TPwpH6zC/jtuxKFufO3X/MdZLyeulm3fLLbh75tTJLB5wz/I3UY+rWfx9eWYnfuFO1m/6/KA+OVFhEYeHPqU2iPTxqew5DPkORPTEfbJuL583wMwm5T5Yfof+XjKjcV32kgFIGmVZ51n9cScd7UM1f+KCuyQv/i5PZu7jl60fTv3VN+q1xbE1FfLEsinfT6ZTLyBO6b5rl5P51OR+xeftXonEUw/RQx6wBz6AM5ZS/i4pnB8n2bg3b31crlsSepycnNoywXz1m5cF6jrnE4vJrP6ceukfP/xlpZcu0l9WjwfOp4cspMf7+n5Zlwe+xJfv/6N5vcLx1+qjxbm7ho2ZnZuNxTHDb5nvkXfGSePpnnot0/G2HvkFyfr4y+bDq71/Xslb99J3yevd99Bn8LW8dr9bjMfXWMf9DqfvGmP+MwD/KIB/zxjT4LjP/k56qMfyj6rPAcAnOefwanX88lNIFjdDi5u+RUgWq4q0//ZjhTE4tPWImCyRpowebRWwF33JmljnQrSofCykKyJRIGx3MRs8Z/Y6MVgte24qF2ENsO+r4r30zIQnbHea5RGgZ8IZgt8SE1yPkG1hMRQpCe8SDkNVpAkIAmsL25skz8yGjY/YdTVyNqiqgBAczlY9Mf2NHuu2RzdUSMmirqahO3QVPLO+CWELAAgLn2corSZXcW46Jkxx4+jhXFJsdBmBCU6EqEXOp2jhq4hxIP3LMDoiHIkW3keEkRgRm3ZEdyDilnH0pSzvIw77psCH9ZvUMlOdpGEgYS0hPslgeHKg+2JwyNHg/PKADGC3bUmeYHGNGCzGAr4KMAalr3Qcl/MRkY/XDZHzjCOxQyb2hAfuQ9ECFeIWY4BK2ndgsa9skIKBb4glta4i+q5iRr6EFF2pFwxJiYTBwThqgzDwWZ+wWhGhztBVsEx6A95MyCJubkkTUzQ0xZsqMhLWp0k/0ybkZBFHC1clGJsQBo+qIeY/K54lxdTnfCpanykQvDuNtpRjfSLjjL1jabRwdUToHaqW5goyiOXPT2Q01ieQ1IQhyGQGqjYgBoscWGKEvdACDddjly21HULfn00hm0kDuXdsRVBtgOqXoynX5WBRbwYYk9HfNBNE12dkgUQDBF9rIgyA0Pnyy1o2KISRMBu4lkhz3GZE7PzEVgjArwITBAGIFraORArEiynDZaTOk85oBhsDVFB9NmDY1gUubX0iuLghePy4rygfrr9pI3LvYGom3CkuEJCWqWVd0Sayvihpm5qK8wUKE6KpYyEGgsBzE/U5XJ60LYMB6gTrI11rKA/TkAZtHixMzRqqYuiIt5jLqs4HhN5PJEFVIij+zk9efb7WrgLStgJ8gm0j0raCPRuR9h7iMjOtMJuwHAtD8wvCIBoYhrHnYSozd27ykPIcRUWQdcmzzMXREhRf3vN1mmDuwiaZMOmzJlPCA1LnkW0iYiJBOUTqR3QO9nyk53Fk16q8e1wuOq9oSGfWNIREsT4jBQ4z6B3Nl4aJmhwjShyNp/Gs6Ttw/jyGAKhONk/907lpDBIKGRVE+5S9lRJSoGVCTDDIbSx9Wc5JWYOl/u0t6e5Kf0QDsw7A7eu7LFOdiHgpM0Rbfn8sacISwRK3R+D3Au1vEsPxLTKjZEoYgQ5xAM9ntZFohHjL8OZA4k2nnvsnEuwe0UxhCozuKHB4CSlg8iaCK/M1A/ejlOtQYP7yTOUqlZACQSmYYArZVNGGbdJUX8dw+0AhEqaf2ld0GIU4TN6/wIxARzyeMldIWzNP59iTSNqYyqqS96Ho+0bpl6lvZ57ZJcRU8hCPfJwfBneNYWNV9kqKTWumvHHk+qNeUKjvxyx8fZ+uu5u+z/LQ/XAKzqvLWv49lqfkpzZZgLvf9bHjBxc3nzC2lqf099fybP4oPST9UwD+CICfzzm/MMa8DeDPPOTGh8ZYfvt7qNz3La3ciJ+4+BS3Y4s+eazciJAtDrHCEEkqQ7QZhYl1O5IrUSQ5RNOxixVqGxAysbLWNqB1ASFbXNcrrPx4R9NRGFt3Y1MMwtpG7EKNlR9hkbELNZxJM1IfiesM2WJTEcur1oW0JqO2AbdDWzQsK0sMmgPjVcSL2LjA10fctg3XPZa8+0ASKSs/FnmCliVKYrbYtxUqoUtnuQYxfkXXUeqesikajiKPIIa76Ctqr+uYLCqbChus1FmOj8kWmQPpk9pF9MGj9gGVTdjXI1ofMCaLmGyRlbhlwz+kuZ6fsNfKGHWB4m491xHA7L6e43LfONuV9jqbS5sAYIzzN3HLhvkQPMWmsrZjymC5BCIjknoH1u8co0Xtida/9hFDmGOPjMlYcd5tFYr3OSZLzLKJ4oe7KlBb7aQfWaDVhuQTnMmofETf0CNe+4hVNaILHn0dCjxcPN8iBSFQcPlN8Y7as/wu98dkCoOwMRlj69D4iL6hOhhGB4ieY6VinWXOi4ENYEbdT31PeY8txUGPcaLv9z6W2GbnUtmwkSTXa9p72bzpa1/mqcDXJf5YvH3OEdZhVPfr8uSzEBytmqHMRcskKt4TG64ka/O06VRPr9+CGBAIt8nwPqJ3CW0zYqjiTI+yqWmjIfH4O5cwMKycNpdoBTRUEZ4NZF3fdTvMNo+snbQxmyrgwFB+2QjxVUSo7dQe9SvvJP9AG0Yx2sICLJtWFDduyvXWJozew3I5oqloXSrx5LIBJdfCkG6l85E2dmpXNnJkflpHhk1i72/TjrAu8ebQxBg88MaMbKwY0KbOYGgO+ipiMLQ5NAisM6OUXWLffSpanzIPnI/IGUg1MRsbm5D8BMWXDQHruP+rabWaM5AqWzZ6ZK6J5qZA/GV+ygorJQtfBbJHTUaqzSTnwBqkkTfMsqcyDM9/2ZBKlUWKieQlfILlkA0KmeBn1ZKhaV0EVrQZU8ImuF8yGPZqKLbdcpx38gTFt7JhA3AYhdqcAxsrPI7GZiRhDE682cMbYRL7npnDwLCRCxD01zCE1tYRSW2SGZuR2nv9yvcm4xND200x3gHqiyybG4k3ZXisZe6bitqVXZyg6XJe4uHV2PMgUfsVI2lm+IBxDJU3zFjq81E4v0CT4dMEb5c4eunfRoVkAGxcmmkVz6EdM6itfLcEn8/W8O55VjDNTKEKljcKGlUvznsKq4AKg5gkeJZe2WRVm8zU1pwEYroYs5TZkM13+ragh5RhOes76QK7OJ71eKg+42dVG7d5yb6mDbTjttRpY9Ms/nEdi8EohurSwFrcc7S8Y4alvmdhWBYAgfTRPc24A08p1ne+e7Wug/xZfl/mf+L7rMivktH5VarLkZRz3gP4K+r7RwA+esi9D2KF/aqmd37qUf5n/tI/gjG7wgzrTC46j2NhfKWFrWZftciz88LWSkylEQMHKQk7qhh43qQ70FptNIr0hTZAJ4ZWMmKFKZZgu9NCWNcvZTMrS/JZsqtq9ldgDt/VLLHa8DvGCgrMX5Q6SX2sybP6LZlglwywSwitGGrLcqVsbSBGVfdTjKrHmFdnefJft6iH3gQUI3o8Ut5J+O/i+BJSLXUao4XjhZc2uMV40+2l/iEDNTD5kmaBfQgD7DFGVB3fK3NA1zeqeXqKOXWqw93+0JsQ+tix3yf9m5RU/oIAOFauvuY+Vl59r+6Lu9dwHZa/8+r4Etas89d10p+1UXqK7VSu0+Rby/jlOUx9Xh99rTYyj8HOBemwbGfJmxfYUp/ZVu+xbd+ycuAM9flsTvaVbtvy3H3su8s5c6cuUocjn2cQ9cUicZb58rDu0yUx2KmKLc9J/Zb1scfrKueB02NdGHfzkb+SH4A5e+yinfJ52R/luCne8Dvi9noQNKmXXsAemzOljEU9T/TDg9OpyZFxB+abTS7eu5LEE/qa6Y52n85K93s6cvNyYa7vS3f7ZdllUx1QUMZHx/NYecA0/qrsWRlH+nU2rMdYaGVOqItNujuuUl9NLCRlnTJKSh2OtUXl8SCDYTlvjtwzy+fE3LxT1vI5O1XuqfNH8l4+6kfzOzbvTqVT8+5I2SfvfcX00jF5xXy/J6NQ3fsr//4PnhW2/fp7+b1/4We/9Hy/9a/94NsGvD5+ETrYAAAgAElEQVR5z1ci3Ywt/uqHP1k8Q91A8JamCmh9wPP9qhhYbRWw62o0FQka7w5N0QDshgpnq76cDwxfHAcPazMuzvdFk1B29q3N6JjRdbPu0fO14+CxXvfY72lbrmlHjKOb0fHnRDvx3kfsbluGCEbS0mPYYI4Wvg2w7IUQeKhjqCBAC7QYHEEXg0W1Yk9ksLAuI2wrmCbCVQlh72HbWGCAArmr2kByEoZgXcIcK7vwcbTzXfKaGFxzpGOweYLIRasWYxNs0jcEK5V7jKPjtkpIPTO/BgvXRMTeTWUcHOyatfPqRPcdCBJoNoGgdgJJlIWbsJ3KwnoVaMdQMcFisAWWZlmTL31G2pR4NCAHy7p+BEEzTZxYXwHSzgNoh5l3gQt0TSBXAMyB4G9ZWGEdQ6XqRKywFf+ayg9zMjC9JdZBYRY1AHyC6RyxO3YWacVkS8KSyqLOoqdW4Fa9RW7lWgt3a5HajLSKMAIfBKadZGBifeWFUfkulwrUK5oCq0lNghksMYi2CW5vWbfP0o+jI+IXkw1sZ5BaZo6tMi3yqjxBlRhKJZClVCdqc5vgDpaYVAWCNU7EKnawBa4k/e92dH0W9tVAbJZmJF1BDeXSUDFZPNmenwtmW7WjofLlM5MKCbSsuiHPyfgolb61A7VX5qTW8YuraTUlC0TNrOkOBuEsobq1xODqaY6bTHp4UchImPUzSH4GsB154cYV6+kZECkK511dW4yXiZgaezMxQAJwe4PxgjxJZqT6uwO13Q5MPiSGm6E2GjAZC2v3FdZQzYzpcukTG4h11USU3fZsUFgwbQKSZwZSZjQ1oHOuo2tTQ59TTccJUsh9wo9jdW0QWxA7buK2ZmDckFYi9R8A7tPxnI77A32ubkmzUBbd7jCR1ZhMZQspDZG3sB6hwaQd2NNYG9bYzC4XNtjkAX+YL+pTJWNF4+Y6TOQnosfnKF9J2U0agaQfiRk7rRtIY7G+Zh1FZmiVtbPoWGZP5UXWWQRoDIX0JbZ03PVAWNM50RiMDYrmYaxRmGIlH2FMtQO/KjcoupTZ8VxJzMbLZUq+ssjWrK/+MNVLWFjdQH0YG27ziq4TBttsAX/IGC5ew6Dl5Dogrqg+rkcxbEyi8uww9aOQaAlhjetoriePaf4xoY3riMxFrjUZMGEifpH5VLw2Fqi2GbE1ag5kJG8m4y0DJtExAHADEVLZMSM5mo820Hsj1YZ0K9mZawJ5QW3IXEcD11N9CrtoAmJl4QZ6J8SKN+C7PLWDdRyTp/tjMxHSyPtI+mLGLCwewowJFptR5q9mJC73KO1LbUyU58ZM1850LDlfzQpr1G5u5n6a6ViWk9P7puhYmnkb7zD6lvfnxFKr4aVLncvZPtIRz6p+d2qyJV3H8l5M877R6c6GxTEjVl23hMRKXR5m6OvdDTP/fl/dThncs7zxYGP+R+nLTz/UhuU7zQv8uZ/879HlCjFbdLnCmB0sJjhdl/iY8nhFTJ5KALAmoU8VKhMx8hsgwqA1AWN2iDBwPDsjz9YxeVQMhRWPpxyTvACCmx5L4omM2cIpAL6+/pi+pZzXXrWKfwnG5O714EXYO55Gan8uf5fHlmXpvJbJIc2OHyMAEi+tVW2eeYy5rUsd0ZhNgUfGRbucyeUYeR/n2qFU/lw/VLcpwcD+nvnbR/oeAMZsUUl+qn7Hx8fM2nQs3dExVe0Rj7buM12Xpdbp8tix9um8ZIyPeYIfqrt5n+d82U5vI3v2784XPSYPaZfu62W7Tn3W9TuFJDiVTmmw6r8vJf454ZV72TxatvWYpuvUj6e9ucf6RqdTWp73pYeO/33335f0mmB5/CFl6rakTAiBpef9vvekzkNQBNO1KLB3YP75WB46r+Xn5XXHPNDSBp3uey4rQ95/j/l6KmeDWm0YOhlzPid9bvm4xdSuqOoj7Y3cjpDnWrGjgnIXFAcw8yKLN93ajIX2O1ISGPnxeak9uuOsgVNfl0PcnoOgQzBfK79uWuZzat1a4K9GH+MP+sYTC+As3lAxKI49n8c8iMsVt1qNa83TYx7k2fp+WZbJc5iulK9ZynWZs++42/na+MtH6rNMxzpa5sPL7tH9faeu99Qby+7Msz/l850xnC6YGYV32rAcq5edf/nMfbBn757+vJPnqevufQBOF011XOZ6zxjeV84Dyyzpv3vANb8T6XexwftDbVjuc4Nv9m/h17unSNnga/UtYrb4sL/EF/0Gv/fsc9yGFue+w+f9Gd5ur/Gdw2M4k/Fu+wLPxg22ocZVvcd3do/x1uoWn/cb1DbizeYGT6odruMKf+PFO3hv8xy3Y4shObSO4pvebq8RYfEb2zdwUXUAgKt6hw92T/D++jkSDD4+nOOy7gjWykaCNwnXY4ubocVPXX6EMTs8GzZYuRGNDfA24rHf49d3T9FFGqLLqsMhVtiGBha5LMqfNDtYk3FV7/Dr2zcQksOTZofnwwp/6+V38WF/ic+6M/z4+Wf4rf1jbEODN9tbrNyAPnl8e3uFtR8wJI+rZofPuzOkbEoM6hvtDis3ok8UM/fJ4RwXdYeVGxGzwXZs8LTd4ot+g/OqxyFWxTC4GVtcVB0+2l/gsu6w9gO66LEbG1w2B3zRbfD2+gafdxus/YBn3QZX7Q6fH86w8iPeP3uOD7ZXeHt9jRfDGruxxrubF7jwPb558xTndYft2JSYydpG1C7Am4TGBYzJ4aP9BVImhuAxOiQYPG23uBlb1Dbg0/05hujwD7z9LaRs8Nc//zGsqwFXzR43YwtvEp5160mGxia8u7kGAHzeEbVmFyq0fkTMFk/aHV70K3ib8P7mObahxqeHc6z9gN3Y4HG7x2eHMzxu9vii28DbhJFjFisb8e76Go+aAz7YPSkxrbuxxlubG+xDjXfX1/hgewVnEs6qHtuxQR89GkcxpxYZn+zPsK5GPF1t8fHuAtZknNU9ft/Fx/iou8SH+0tcNB1aNyJlgyF5HAJ53ysb0UdP2q4+oHGhfHc24XGzx+3YYuOHMp6f7s/xxmqL2kV8sj/He2fP8Vvbx9hUA0K06KPHYazQ+IAn7Q6f7s/xqDngeljRs9C3qF3EmCzO6gEWmWKas8GLboWvrW/x0e4C752/wBfdBn3wCMliUw/ogkfKBo/bA6zJ6KPHfqxgALx//hzP+jWeHdYl5nVTDbioOvz29hG1PTqsqxGtJ3bmm76lmM7ocLXaI2WDLw5rOO7D5x2hIM6qAbuxRkgWl02H677F77v6BN4k/NJn78KzXu5ZPeDZfgVrKH61dhFvbW7gTcJ3bh9PsdI+ICaL/Vih5nu/fv4C33z2Bn7y6jN8+/Zxiem1JuMbl8+wD1T+IVR40u7wnZvHpe5vrOm98NHtOR6tOsRksRtq1J7eXX/gyYf45S/eQT96XK46XDQddmMNbyPeP3uOX3n+JiL38bPdGm+e3+KL/QZXqz2u+7ZA1WMyOGsGVC7ipmvxZL3Ddmhw0XT4fL/B4/aAPtJ49YHeZVerPVZ+xGeHDaFNosMQHSITsA0ca70bKjxadTiveny8O4e3CYfR4+lmBwD4bLfB080OL7oVhuDgbMaqovfSYajgXcIffPrb+Pb2Ci86eibfWO1Qu4Bff/4E5w2xhu+GGs4mvH/xHN969gbOmgHvnF3jm8/ewO99/AW+c/MYMVnEZPDW+W15P3ob8UW3oX7lWPQu+DJvtkND7+Z2j49359jUA2Ky6KNDN1R4tD7gMFZ45+waKVuETJseL7oVdkOFiuMF31jvSh8OTC4XksXXNlsAwBAd+ujx5voWH+8uULmI667Fph6Qs0EfHZ6s9vitF4/wt731XexDhef9ujzbtY1Y+RG3Y4PbvsHXNls871Z4utrBmoTH9QHPhxVCdvjo9gJXqz3e2Vzjg9srnNc91n7Ayo34hN9zITk869ZofMCLwwpPuZ7PuxUaF/HGagtrMj64vsJ505PxN1a4aDrUNuLj3TkueT6uqhEvDqsy19bMon59aPH2xQ32Y40ueIzBYdMM+Nr6FilbfH6gufrdmwu8fX6LF90KZ3WP/VjjGxdf4P/+7O3XXnc83ezw8e05MoDzti8cBY0L+HR7hk0z4DB6vLHeYzfWiNmUWP43z28xJoebrsWj1QF98BiTRT96fO1si5gttkMNx8/5yo+FV2HbNxxbTxslQ3D4xqNn+GR/DgOUMrdjjSH4EvZQ2YRtT2Rwj9cHvDi0OG/o3TlGi3OODb8+tOU+azLaekQ/eqxqilWXd8UQHcZoUblU9MLPmoHWA12DDODJZl/4DVbViN1QY9/XeLQ+4MV+VWKEPf+unrc9vcuGCt4xn0TwZROi8rG0Zwgefe/hOV7aOUKepUQkiKLjDaBsUgyDK/HhThAtioBQyAa9j4V0cBlaUNeR0GdLw9tMut8SM56iLWgz5xLGkVypEkcs2uF6k2vS/6b4cilXYqkzv3MNEwEKeZ4gzCz3geHjkmeJoc5EhmfcXEJO0rTpgCLnJoR4RVpNDHlL10mowEOY8MsmiGwizDYYTmyQ8F9NkKjr+qP01Uw/1DGWZz/xVv4Df+FPYD9UiImIUYRMJDLbqxgDIVp4lwoRh5yTHdJxnF4qlpkjJSax76pCSpHz9LISRtZx8OXBcy5hHHwhZAgjEUwA853oFC0xtDYjk5FMO73GgNlUXXlBaFIKnawlNkxpgxxLyaJpRoTAfVFFhNGVuDuprzCICsFDYiY3eXFYl2cvqhTNpDWJ6cUncFr9QkiRWD7T4FiLcnoZCdOgrZgd1GSCxyq2UFclxGGCyiLR9dYmqreOgeJdLcMvLHnpxEFextNLqbCJGjCjpEF1TurU476i9jHTJ2ymv5jKsDy2qRyf2ix5i44oMkp7CpNpsEROERh7KLATztu4jNi7KW8mqsjJUF8wQ2Zpv+zoyYuZ4bjCzEmTAnBtQAoWmdk4hSCDcIUo1xUosRAdpKmNxjN7pOXyMwgGzKysebQEix4WjI+Sn8t0vZPd7zxnwBTHprB7BkOQYYEOM5Nhie2RHyUmBhEWRmRD7JbRUP4AXS91GBQBjcko+phRPV9e9afUSc7ruCL53BCxSz646UfTZmqDfDcgCLUB1UGStF/Hgy1ZLXU8VZPmsG+fCnsmuaz4ORjm7JPF+7GKyAc3sWlK22yme5n1Ei7DjJbYIIXJMprJQ5BR4MgmGjrPY22CnWDUCRNTpejI8jwiuBgvGKyCRUeGOluug+FzUl6gz8JuSVCviTETBsirCIx2gqHyPDH9VLfCFFknmI6h3j7T53ZiAAUwwaplYRQM3S9DydBugu9N7TXj/BokZtiMDAlXyUQzMeNmlZ8ew6TqklH6StqJYJhYBkCkPGxnkdaEPTbBzPoalvoTkdhJzWim/AtUn6HwFcH7Sx/ys1PGO3FeFrN8zEhlSXttr+cH1TfbPIPbg6HqAAq0FmDobp0LvFzghpJ3YT7tmYlUtTe1CW53HEn0kJQqgmgDmOt9WjouUMoC68wo8EuCZJsCDzUZFFKQGZ6cMXtOdTyiCdNx6bPUpln4ALG9AjpmNhtMDK3CCusy9xvpkVKfqvll5s8iMvVzZqbb4oUz6jnNuBtSANAYRoaKe4LtU4U5j4yiq1pgnBlzOKv+CU6YsZtmg1m871EdzMTXaI+xma7PBtN7Z+kF5oykz7TXUC95oE7p/skGsOp3hcqayp8ZWPqzPmbUMX2fvk7at8xD3XOKmGje1hNlHEuvYD482JOq63CsTt9j+n//7R98HGL77nv5/T/55cdYfvPP/uDbBvyQeyxrF/HO2TW6WBXY21ILcgmd0/C1Y8QkiWFBArtM2SCvupNzOmUDrKfvxzQyl9e/DIon9VvV8/NLGBaVN53Lzd12NVWYDFreoZ+V1U7tPgYvexnMLav6HmsHAIA2nO8n61BG9wzO1IwTEYcqoqrDjADl2P5IzmbGQHeqXNksAAB3oeCrRYv07n3HylxC3oQZNC/G8VS+Ok+7kqCRu/3l1unOsVk91Dww9Zz90FYR2aejY7H83Vp+LqlefG9UGTUHzfmA5UbDdM2y/vFOuAXAfSHjtyxzWWH9XSdP9btDLrMg7zgGZy19VB3p71M/cJnlDHSqj/16A2iVkXLqGWMNS7gjLJZi2IPnzpFrsmKtlPoBoA2FJs2bofukmdqcfWSjjP7C5/l9xQCSG/J0PTAZZYvrUavx0HXmembV71kkRrAYw6zGkhcfs83SQAvsXOs+ICNylo8cl3YnNojSXcMPXD9ZzBVGS4CkGZb5AsUYBsAL4jwdXy722NBbdkw2ecpfL5L5HhPNZFj7eZ1MEgOLMzaToV/kIwDaSFBGYblYtymB4seV0SfXGJbgkU2abEFjADEYDGyvvisjxQQDUXIt+SropRhs1F6+RsbcUH/YzkwOEdkkFXkLLstt7XzR+orJjlOddLxaqbso0gzKyuAGu25qu8SeyqMv8Zp36nbPe87t5m0xnfoi7dd1j7S+sWqzyw1mXq4YaHK/MopKX5bL6X+rxxg0DlP9VVl6ky2ruo2qDfJ39lIw5Vh5VSZ1zcuMnyN9d9TQKZnLpNMnVdtPvv+XZX0PE22RXskw0/X5XvJ+1eMPzfdUesXrX6tPvirph7nuL0k/1Ibluevwjz/9JXyrexN98vhG+zm2scUn4wU+7c/xN28+xrOwwZXf4bv9I7zXPsO3D28AAP6m9Sf4aHiE29DizfoGv75/A2+3N/i8P8PKjXhSb/G16gbXYY1fuvk6fmz9DLvY4BArnPsOffJ4r32OmC3+v+2buKr3GLPFk2qHD/ZP8I31FwCA7xyucFXvVMyXxcoNeDZscD22+FsuPkSCwYfdI1z4A1ZuRGUiLv0e39q/iV0kKM3TeoubsMLzYYXWTQvXq3qHxga8UW3xazvqhzeaLZ4Na/ydF9/Gd/orfD6c4feffYhf272FXajx7uoFzlyPbWzwzduv4UmzQ588HlUHfNydI2WDtR+xDxXeWt1iZQccUg2LjE/7M2z8gHPfIWSHQ6z+f/bepNe2JEsT+qzbzenuva/1PprKJCsbSlklVEUNGDCA+gdMYFSiKSZMGDCiExIjQOI/IITgBxTTYlAiEVKBsiozoyLD8YjwiHD319zmNLuxjsGyZdv2vue+99wjMiM9FSZd3XN2Y/3ex5atb30fntV7fDVssdU99q7JMizX4wpX1Qk/PV7hsuoSpFZn+OyX/RYfrW7wVb9Boxxe9ms8aY54kaCwf2PzAj88PMMnq2u8HNc4uQrfXb/CY3PEP7v5GLuqw8lVOXaukp7gxMpBCw8XFT4/XcIFiV3Vo09wz+ftHV4lyPOX3RbWK/zb7/8pJCL+yYvfxtoMeNoccGsJqvmyX6NSHmOSRfnu5jUA4ItuCykiTq5CJT0CBJ43e7waVtAy4OP2Gl2oMrya2/2Lbpdhx5X06L0mKJ3w+GhFY/Pnx6eQImL0Cnvb4JP1NfauxsftNX54eIZKOlyYHre2yVI5lEfAL0471Mrh4/U1fnx4hBgFLuoOf2v3M3wxXOCHd09xUXfY6gE2SrigMsS6Ug69NxicRqtt/s6xiY/qI+7GFhszoFUWnTd40W3wfHWHVln89HiF729e4dPDY1xWBE892BqdIyjs05ra/6Q54PWwRiUdrocVKukxBoWLiiDlGzMgRIEX/QYfrW/wk8MVPtlc49VAUNjB6wzfDFHgcXNEJT3ubJNhvb9z8RW+6jd42W0gRMTajNiYARemw2f7xwggdMPKjKikh5Ye+7HJsOer5gSJiF+cdtAyYGt6XA8rSBGxNQOOCYp6UfW4HRv8q1c/hxQR//eLT2CSkXdR9XjRrckekwTR/mh9g1o5/OD2WWZ+brWFCxKHsabPUeL721f4k+vn+L2rL/Hp/jF6pzPs7ncuv8TekhzRyVW4rDr8ZD9BYT/YEFz7p/tLXDUdQhS4GxrUmp7Nv/f4M/zRq+/iZA0etyfsTI+Dq6GFx/fWr/D/XH8EHyR2dY+vjht8tL3BF8cdnq32uB5WWdbFR4IhaxnwulvhvfUdbscWj5sjvjju8Kg5ovcGNih0lsblSXvEruoyXNomKKdL5Z1shUZb3A4Nnq6O2JkePzteQIqI3mm8v75DiAJfnrZ4f32Hl90Gg1cZruwjQYqViPh7Tz/Dj0+P8KLbQMuA91e3qKXHn1w/x0XdZ8iqUR6/tXuJP371Pi6bDh+vb3Lf//D2KVyQ8FHg4+1Nej+OOTTgZCvUysEoj8NY4731XX5ulQh41u7xk/0jbCqa0wzXftwSVPK729cY/PRTfDO2uB2aPNbPVnuMXpOUljPQImDwGh8kSP4YFE6uys97oxxe9ytsKoKZHm2F56s9Pr15jH/t2U9xcBVeD2s0yuLkKjTKYmMG3I0tXvcrfLC5TWEKlP+l6XB09Nv3k/0Vnq32+N76FX6wf46NGbDVA1o14ot+h7UeYYPCi26DWrs8J6SI+b3+vN0DAH50+wS7msJEjq7CRdWhkh6/OO1wVZ9wtHWGTKv0Lryo6f3wulvh4901Tq7CyVb53NOWYLcvug0eN0f8eH+F72yv8aLfYGt6nFyF729f4Z+9/PAbrzveX9/h8/0lYhTYNT2M9AQdVRaf7y9x0fTorMGT9oijq9L8puflw7QRfjc0uGy6DGPmee0ihT4wA/sMCjvSLhtDYQev8NuXL/Cz4yW9Z8cKz1cH3I4NxiTxpWSAFgEHfk+2J1z3LbbVgN4ZDKnfpIi47tusuy0Fwco7a7Cu6BntrMGj9kShEUFCiYhaOxzGCruK9Ljv+gYhElx48ASFXZkRh7HGYajweH3CywOFkEgZEspMYlsPGa5fayJQHJ3KkPvauCwV1VuNbqhg0sZVpX0Bn1VZE1yKmDcHxwRvLWGuzGbuEhnjOOqkya0yUSOnEARq4zKkmeGvAGZM31qTLBRDVYUgvXO+TwjS2XaOGMQZMVcmKScpK0KEsWxXMuJlzPdxud6TbBRLPTGEeLlh7x1pamf23nKDPLUrJjgtinaWcN0yv3vx4UU7lrZ8SYAoBGYbLkto6z2HQUSG5+ZMf3U2+2/Srzh9q6GwH/3BRfxH/8u/kQlzTp7cGrV0MMLj2q1ghMcQNGrpcONW2GpaTNy5BkqQXmTnDXa6R+erTJ7SeYMhaEgRslEnRYARIR8/OnrRP6qO+fMQNHamx51tIEXIi29gInRxQVFMnHR4Nazzj9IYdPa6uqDQaAstAqQI6D0tKIgMZYLx9F7TgssbrDTFp/Ve0+JiWKFRDpVyuBsbrPRIcUDJwwvQAp4Je8agUUmXCV6W1wKk3RmixBiml/fgdP4xKAmAGuWyfujJVVnzkg2mRrms+cnyKlz3AIF9WmQPXkMLiqM4Wvqh3tU9bFBQIkxEG1HCBZkJL6SIM81OCXpB985QTEacdEC/Om4Qo6D4KAgMKdYkRpGNBCZU4RgYbgtrcIrULv7B4jwq5TP5EGt4jl6hSfFunPge5yWaiuotBcXInKyhH9XRYFWT9inDvFXqc24zxyuOTqExjlCXXuE0VDDKo6lsjhkj+ZOYdVg5D5Z84R8ONrYYUu4Daa8qkXQPR4MI0t/sRoPGOLj048jSKSEixW56DI70UAHk/FhqRRRziK/lfHWKs1EyYnQqxzI6r3LdORanGyooFVBp8tr7pIXKiwQAuZ2MZOB5QXlSHSrt8wJEK/Ly5YUDJhmXrqsAEdE2Nv+Y+nQPgHwfx+lUhTeZ5UMYZg8A46jQNBZ9V6Gq3czLPA5z+L2zClXtZtB+LoMXMQyRFyJi6AyqxuX4H44JihHwTsFULi+AtJ7g/QztLxcBPnmpsl5kKkcnjUnWQuQFlHPEkq20zzFCTBzC9eD/3hObNLFhp+c8ed2UDpkBm+H63D+SdXaPBrJxEIpcHcFNjNsz2H8EsWIzXNxKqMbD9wTF50VMGBNmL0HQhV5AymUk+LnADCrOrNnczgyxF5FgzuUiScXpXuA+VDzS/bAye01zPsxUvYRtW0nIgk7R9QxvZTj5EqqeoMoiCoLlMjI/MVoLKxHrkGG7gj277FFkVluGARfHKF8QjJM9XZIhzRNEGBIZxstJJPg3wVxTvzB7qEeGWQZNnsOYYKv8HZI8ib59M+rjTWkG72WvKZBZgUXy1EqLSfOR0fhjggOrdK+Imf2U65494FHMYZ0cHpBdsoAcRIbQRhkhHZXNl1BnILOSSocZQyrE5GElht2YYekzFlWe3g4z6ClDkBmmm5lgE8sr5x9Vus4is9tmb6+Y98WM1VRM5WTWc4kZLDVDoRf3UKOK/l/AYQu7ZmpHway69Cjdk3bh9IBHtLyv5K5bAlTued2K/PI4LjytJRBIRMz67bwnlvLMbXugrkvIaVnGvevwlnYvjt3L5x3yeLBu5Ri/JS3r8cf/468fLtp8+HH85D/+C4DC/ue//rYB33KP5d41+Kevv4+jrWaLQiaDKBf0/N2mxSVLlLCx01sNo0jEXgoSgleCgFd/PHyQF7/AtFtTpR2zbnw/L8yXC14WeAeQd3ikDHkh3BiHEJFF53nnx6iAweoZ26kvdog48UJbJOMmFHqVlXa5HBak5/MAGS2jU3mRyDtvJQxQqfkPMEutTDtWmO2clTtNvPjmuFXWVOS4TA6WZ/0/3nXjhXBVOYyjTruAEt6T8LtSAV/dbKBUzAvyHPgupt0zIeIsaJ4XaryzB9BCNgJYrQiX9OmXTyBkgFIxx71yIH3uc5ME0P0UIJ/L5LjHdF0IFMjPO4G8WFZqqkOOXwQJsEsZcTzVma2RFu0+y9Tsj00ev7xrKIq41xQHK2XArScDQ8iIurY4dhXu9m0eQyBBsnlepV8S1v9bQn7LsebF+50XWUz9eGigVMDpUNNiHrRTycH3UkYcU/1ynCvD3SKSAUCLfQA4pDjcg20hTchyPIhitqAXSXQ9x5xGAVV5DEHg6BIWW9KCXciI/lhNO4V6HqUAACAASURBVKUpllTIiBMvqsOU58kmfUrF8cBFP6X7o5dQjYOIAnev19OOcDqXk4iQhnayj7cmzdOy36cVg6w89q8qyNrjeNPOfohV7REdtXcMFMt8PLVTP6ZYQnuoUryqmBZWEVCtQ7+vyfjQFHPtBnoXSBMw7GmjTKgIm+Jm3UlD6AjX6TkRA8NtnaRYYK8hVMRw0nNIJ+9Yp/JsX81io2YGkUz/FQnd215PBl2CeYZOAzoi9NPinuPKvNe08F45MgaTBEhu695M0N80/0Tt4e+ov4QJ8HsD0TqEk56eD5MeDI5fsxJxkNOC1otk3GE6riNip/M4xLKtQUyyQ5ycyDHt3F+5/LFYSeoCLu9FisVN7RxlNj6ydFKngIbqlmMwebHMkkhWZvkiVAkqzfUMgvLXEbFxwEixt6iTHEsRMws2KlPMJvdzVDHnKziWMs2PaCIgA7WfjdQKU9xowBQXO4pJVimIbPz6irwxwkmEmoxavyYDluMb3cpD9L9EjGVL8koAGct500FwPDLV3a9iNoZ47ruNz8Z6rNiopzH0a+6nNCaIU3+m/qPG03wQEbBXfoqxDAKumfokO4jEVA/XxCwhlY30ZjKSRUSOuWRDi59h4QHbzA3CLNHRYh4v2k4GDF8jvECsS/g08nsytjHFQk7WR+47sGHGx1HE28a5MTkNB/gOUX4ojcPintytNWaGdFnNswbn0sgRi+N8rFxGsSFbvEKLrpgbYA9aimwQLy3MRaZlsdzW+6fut6X8fy5uk39nztX5obQ8/zWMw69rTP4m/XrTt9qwrKXDZdXhDy8/x8HX+KLfAQCe1Xs8Nkf8i/37aJTNEMQfHx7h4zVBmX5yvMKzqssQzL/9+Gf4yekKl9UJvTd41a9xfWqxqiz+/gef4dP9Y2zMQCxoroIWxIAJAH/nvc/xZYJFXvctfufRV/js7hGkiHhv/Qq3Qzvz8HXW4HK7x870+LNXz6BkxJPNEYPTGL2C8xK3xxbPL/YwykOJgJu+xboaM2SOYzmZofHYV3j/8g4AcNc32DU9fvriCuvVgKtVhy9vt7jcnNBoh9fHFWwy9j6+usF+rGFkwG3X4NH2lOtYp2u5vAjgyfaI42iyh64xDnfHBut2wGBN1gEFgN2qx2mo8N7FHnd9k5gbAzb1iNuuwePNCa8OKzzeHTNj3OvjCu9f3mE/1LjZt3h6ecCruzVWzYi2snhxs0HfVXj2+A53pwarxmJ0GlVil7RWw3tJBrCIuNh25L0ayUspRMT+2KCuyYv16JL6+NNPnwMAnn90jdNo0J1q1I2F9xK79Qk+kGfPeYm7fQshgKYdESMyREapgNOxganIG9ZdtxCVR72ycJa8QENn0KxGDH2Fph0TsZLLdR7uasBK6IsREQStqSqHw10LpQP61w2qywHBS7hRQVceKnmGQjLgNrsOw2AwHiu0O4KODb1B/5Mtwtqjueox9hrB0uMvVIBK8WwhGYmqIu9TsDITJoUosh6qHxS8pcWyXjkyNpyA3lrY6xrqwsL3mn58DOmvBi/g7iqorYU/GMjWIViJamUzk58bqE4hxXDplYPbG+ithburIFbkfVLawfUGsiIBIJf0TWFCMtwi/MsaceWhV47Wb6NC6BR5Oi7ttIEwKsQh6bWuHJEvyYiwN2RDrC0RZx0NxU9GJO3WQGRTnYZsHfCzluK8Puwno/mkobY2k1ZFJ4EXNS0er1JeAOBSzFrts8cLX9WQTweIL2rgykGkxX+MAH7e0MJURjJMrg1wZZNRBuDOkHF8OQIHQ16QOuR4N/WjFv6Zg6gDcNSkL5piCvWtgH/uKJ+jgtg5iNcV4sZDXGvElZ+MVRkhToqMhtZDvjIITYTsBPwmQJ7SQluCjJAIyGsDOQq4C08ELkzoIgBxUok8hjxi8lZDjQJuG/JiXL8go9FtPfS1hl+H5MEQ5E0SoDycQPNZC7uLcCvygumThnAC42MPuae+YIOj+txgeO4h9wpmLzA89ah/3mC8DDmW0rwwOX5PBMCtI3mpXCInqSLMXiPKSFqOEdAngXEXoQZkzcJQRegjeZvMnZgW6RHwTbo3GRjmoGkRLWLWnwwKMHtmsiFPmdmT9qbwAr6NUyyjjtBHjeFRwOpHlJdvAFF4r6Ql7cdQJ+3OFjAH0CbEANIKVcC4izAHAXMAxh0gR7pXuqRrOdJ1rFHqG8AcyI3lWjpmjpTvcAXonj6z3qV0gN2kMg3l7VZ0DWk9pj2MFVDd0H3BTOf0ibrErUi/ctwB1R2VzXnWNxKn5w+trt+eqj3lGwXVn40paYHxgo4FA+gj/QfSf0FjHRVpq+oT6SKSxiNQ7clADxUZVDQukx6kr9iAnPJsXgnYDV0fDGCOpBOZ41EjIB0dA6hM3wjIMWlbStK+RARcK2b6kFnzcoxJO1KkOk9GlvA0Nzhe1DUizdmYiZaUJR1RXwuYI2DXVF/pJ++u7ijPUGwkBD2VI10yTgO1kzVy2ZiVLqZ7xMwrx/9ZxzRozPQvgekZYG3RoAHpMelLRiBKAWWnNi29lFGS3iRrSbIXVHD/l2OX5jJ7X1k3tPROZh1LSXmW5QlP80Z40hQVEQgqUv2UgPSF9mnei4iAmDz39z2QdJ7/ixAnT27p5UyJbddJv/O+Ib7sp6Xtew83+8B3EWPu31xd1gldpncwPH/w9kv+UtJbDfFvcfpWQ2Hf+/1H8d/5n/4BAMAFBbvQyhvTW8AGBSM9bJiYFVyCRvJnjvHiWEiCFiYonJpgnmWa9AYZRihTXg5j0DlOotSnK3XlJCLGoLLXlGGcpAemMkyzJCXSaYHPyQfSfmRYZ4gia2MOCSbLMTnsOXXFbnglPVyhjakXkEiXKLy5XPYKc+yDFBO8cwnr5NhHo3y+nu/JxnHyKHN55XeG43qGZ6a6szYd57XU6luWVXo0uc/4WqZW70ZaATDssfQMq3t9PkFImVCJYyDKckovNB9nKCJ7rkvvcIwie6V5EwJAhoYy5JXHgt/ZD2oUYr7RuKRR57pwGeWxsk4z0pvcjukcezGZsIg9sucIeeaxIfHe+Rgx6/tl3zGNfFkX/sx553iSBKtlCCZ72LmeUz44G+9S6u2V97G3WjLENTJEU+XNgDKPUpePjP8UmyMXv9YLz3oIk2dbijj7JQppY6fsi7Jvls+EKPoSALyTUOzlBWZ07jEge5tR1Ju98bP4mHTN5LkV0262RPZ8z8Y4pD5jGCama6Z6FNlz2fysLD3WomjbYkc+w1JVQPYIRxBjMreFr3UT+zK8mBiQVaHvV3jEqYMXv58CE9tx9oiICXLKn4EJfloSmnCe5UKr1A4sr8serMJtwOUw5I+PJdbVe2XlFWLKm73CyVMh+BwwkQoFTF4vPh9AcFke71T/DHEFCm9mKltx/2DyjJTXIf1XZ65Z9nPqz3LhLjwyU242tAQyI+s3TQzt5O7LnrLCYMgQ0MW4ZdipRGINpvpHgamfyvEsUzmH+L2d2sb3ZSbi6ZL5M7GAg4IhlFzHct5xO2LxtXxey8/l2GFq+8xQKgyVyWgo6iWKvBZNn+CbyWCSycOZPLcPeuKW9c3P5CL/wuA5a/jgzPcz977L9b9sOlfeWz2G58btTL5n7zt3zzfN76G6vSGfc/m+k4f0gfTP/4dfP1y0+fDj+J1/9KuHwv7L/+LX3zbgW+6x9JFiEV+PKwSITBpzsDV6b7Cr+gyBvenbrPsXosDGDDh5It9olMWrfp3j+ZSguLiNHrJO36YaYL2CixJGevgosdbEssq6iwECRnq87DbYVLSFdzc22evokwHHRh9r5QEgIxBx0pBK8YcuwXU5iP9kJ6MpRNKIkuAg+jobf52tM3lA50zW8LIJBqzTm/p2aLIxSeQTVTKaQtaVMykwXnGsn4j5Hs5vcDoTh7BR1Dsipbnra1R6ghbz8eNo0BqHPsUinkaT4/WUjFhXI/ZDhXVl0afYw1Izr9Iu624xE69RHlL5/J5ig1EX0NPGTPcdhwohSLx/Qd7er/YbaOVRaQ+XCBeoPiEbpBz/OFjSFeS4wBgpdm9McWXbZsgaZkqS4cntq1LsIBuLNC8i1vUIIwP2fV0QD5CWmA8Cm2bAaSByB61IRoeNcTYIB2sgZchxiTy2j3YdTtbkWMW6ssQmnIx4vs7HiXyADXs2SGpDmma18blNo9Oojctlb1d98hCHbBQzhLqt7cyzTXBlnY0kgl7HbDxbp7JHuK1HjE5nmDXDqGMUMCmWlGMoAWCz6jE6DWsJ4kmEDBSr2yVdNzb8pIwwhuDZeSxrep+Mo87EDAzTNpVDCCQ/ZAwd321PkAK4PTTZQK0qj3GkeUR1iKiageZ7X2XDhjcZyCgn47JtBnRdhfVqwDBOmm5CANtNl6DtNFbGOPS9yQZQw3UfiJCCiBsmY/hyd8LdsUFIUkRswIo0v49dDdKaIzh5044UZ9k4OCdnUE1liKHZWYWqJe+z1h521FDVRCTB0PGqpfjNcdBZ9y3yxoTxs/hKralu42AgpEfwClVDz5+1ClXjsoySECFruzE8fHtxQj8aOEvjqg2hFoa+gkwkaMFT3GSzs+hOFXTlYSqHvqvQbkcMg8kGt0lIB97AcFbNNmNCkNCJidt7meHtzuo8xqRzJxJUXsJULm8mcD+SdhyNpTY+9w+3KwaR7hP5eeDQASlijj3lOa60hxs06kuLGGmTiTcdpCB4eI5nNYSA4L7i8IQQBNyooXSAqRzGwWTpKpbZ4s0SirWNCFZmNIRnGakUo2t7DanJIosM3xcRblQkR5XQE8FyyAGSBl9M8bCe4mSDoPsNIQgAwFuCZYdeQzIKIG0m6NrBH8/RTL9bEpUnyaaISToIAEREGBRtTjgJmeJqY0TeBJCJoTo6kp5CQpkgACJBhmPqJ7phMiZjNjxTeQFQrZ+kqVIe0ckJTs55pLhkqpuYIOdBTKzXSXorG+t8TYZCg2DSvMEiMG2M8HFmHWZoNZfPZekwyTeVhiDHj5abKLyxgbSpUR5n2PXiXGY4XhqZfKxsHy8SyphSOf2fjXnEBB/mVBo2/LmE9qcyopg2FKZ6zg3ZXMZib4HzKQ2pmQHM7SqMsyxRU9YvnSs3HpbpnHGdyyjymH1+BwOzrEKZHtoLmJ089/836a98+lYblo/1Af/e43+Km7DCGBUq4TFGhQCJY6ixlR36aLLn0QiPY6hgo0YjSUibP9/4FdZywDHUMMKjEg4ybcPd+jVqafOPsI0KCiF7lG59i63s0UeDlRxx61tcKGJi7KOBQoSHyP8B8m7aqHCljwhR4BTqXEcpAhph8doTe5oNVEcbVTZOOTWSFlkrOeC12wAgw5XzslHBRoVaWgzBwEeJRlpIEWCDxilUWMmR2iQC+kBkPezhXakx5yMRcfA1jPQwgoxrGxUaadEHk68ryy3Jk6jd6Qc29WMjLbFRSiJFWskx18lIj1vX5vrZqLBRAxpp8dJuUMvJk8xjY4SHxzTeB5+M7eJ4LR2GtGU9Bg0bFJ5We0gRcXfREAEMAmzaCh6ChmGvNATWCffDZE9j0NkzTeRGybBRA06hyudtlKilQ+crtGrEEPSMiMlIj0o6KAQc/KST4oJCq+zsfr6eN1fImCciny4RPTFRFABo6bFRA7pQofMGEjHfk8XZ01s7QGRjlevNMb5GBNgoYRKRlAuK+kcSI23nTSas4mMuStigMvlV5w1q5TKCwBV9wORQZVtaRRs+zCpM46bQKIs+EWNVCU/EqAMAuR48R6hPIoz0OLoqe9UzGiDVtZxPjCrgMsbcn2FGmMRMygBwt22gBZFsVcphLBg/Zdq0AohQidOUn8wIiEY5HFyNlR7RO4MAkedZo2xGMnB7uS+kiHkeEmPw1Dfc3o0ZcNjUaWPN5XnAc7jfTXnx5hx7y7ndecyK9jBhl2aCKuXgosxIApE2pTjfpfQSQ/zzM1HEqjNBl0ntOYdE4f7h/Dh0wKZ5YaTPzKzM5MzzQ4oIuyGUgZEe/ZbmKTO28qbdcp5wfZkIionMeCNKFG0q0Rr8uS5IvHiTp1x0lX1Urq9YEovP8dgA8/UeP8suyDxWHLtfrtUYqVFu1DFShK/hDTktA9xq2lATIsK3E6qHCLGQkR0lq6VK3mff0jkqG5nga4lYYPbNcq771F4mantoHRracfY8c79V1UIS6GskISLCatK0LlNYTVwDjFQoURgl+qFEW5QIFj7GZS3l0UqEhVIBwUxt4eNn5aTSffk/G348j5oFQqW0dFL/87dyfs7W/A9YLBkpIEHSQWeMhHtSZMV5UZzL/bmoH587d/90rLie708GUlzeVpbJ/5fSQ+9o6JyVLEvz9lxW8Uze+Zg4V1fM+vRstc5lCtzrw6VH8KHbzlZ8kWZe1a9RpbfV8W3pm3oz/1LTt6GO3zB9qw3LW7/CPzn+TfzZ8T3YoPBhe4POG3w1bHHdr/Dd7SscXY1H1RGfny7x0eoGPzk+ghQBn6yv8XLY4GBrPK6P+Px4ieeruywB8bQ54Hl9hxu7wj+/fh/vre9wclWWYQgQeFofIEXEp/vH2Bqi237aHPDZ/hG+u30NGxS+7La4qLrZgqxRDjdji7uhwe8/+gVClHjRb7DSY2aLvTQdfnD3PLO+XlYdDq7G3UAeUF5AXdUnaBHwpD7gz/dPs0zCdb/CHz76HD/rLvGi2+B3L7/AZ4fH2Nsaz9s9tqbH0dX47I5o8AevcVWf8LpfkydUW3TO4Nlqj0Y59Glx9bLbYFf3WOmRmEZdhWftHl91W+yqPklvkHGwtzW2ZsAXxy229UBl2hpHW+FRe8KrboXnq332+F73LR0/rVFrh+9fvMSPbp/gw0R/f7IGH29vcGF6/ODmGVZmJKr/tJCrpEetSWqkSpDdnx0u4IPEuhoxOGrDs9U+SwF8ddjAeoV/8+MfIkSJP/ryO1hXIx41R9yOJDdy3bcz6OyHScqBZSyOY4WVoQXsk/aIm6GFkgHf377CrW0yzf5hrPGoOeFFt8ZV0+FVt4KRgWQklIeRAR9tbrAzPT7dkyyO9Qr7ocKHuzscbYUP1zf49PYJjPLYVgP2Y52p5SkPjy/2W7SVxfPVHp/vLwEA23rAH159jtfjCp/dPsK2HtBq8nZ3zuBoE6Ny8j6zMdEai8ERvbwUEVdNh8NYo9EkUdB7g5enNZ6sjmiUxS+OO3y8vcGP765wUfeIcZJXqJTHk/aIL08bXDUd7pK3/K6vMznWOnmDV2ZEjAKvuxWeb/b4xX6Lj3e3eNmtMXoF6xTW9YhT8sherTqSLbAVuhQ7+jeuXuVYaQBoK4uNGXFRd/js9hGYhn5Vj1gZCykibvtJbuTJ6ggfJF4c19AqYFsPuD612Zt+HCs4L3HR9rjra/zeky9RS4fPrq+yQbStR7w6rjJbbaVJe1fLgM/3l3mhzERjp4JG/5PdNf6/14/wu0+/xM/2F9nDDQC//fgFDraGjxKD07hsOvzs9iJ7sZ9tDhAi4me3F7hadfBR4JCQAwDw/tM7/Oj6CXqrsWt7XDUdbvqWJGE2B/z49hF5yOuR4p4v7vDVfoMnmyOuT202lEIkLzrHaD/ZHHEcK1zUPV4c17hoCTVivcKQxuXJ5oi1GfHiuEZrJmmBGAVW9YguIReOQ4XLtseu7vHytIJRAd1o8GxLkhJf7jd4f7vHqxQHzgzFIQr0VkPLgN967wU+3T/BXRrXJ6sjNmbAD14+w6ahDaLTUEGIiN969BJ/+uI5LtoeT7cH/MndDv/K4xd4efMIzhPM+YPdHWIU2JoBWnp8cdyhsxqVpuf3OBqS6ZEet4nJ+Wl7xE/vLrCu6B0xeoVuNHi8PmFwmuRJEiM4AFz3bR6rEIHnmwM6ZyCjyOzQo1P4cEcoi9ErnGyFD9a3+PxwCZXGYlOTUTVag8frAz57/Qh/5/2f4uQqvOw22Wg2ymOtR+xtjZuuwZPVEa+6VX7PPa6PuB5X6L3B57cXeNT2+GRzjR/cPMNl02Fneqz1gJ8erzLL+FenLWrtcNM1eLqm38mXp3XWngaAH10/wa4hqYuTNbio+yzZ8ag94TDWeZ4IERE8PfNSRFyfWjzbHHCyFTprMDqFbTPgvTX1yS+OOzxtj/jJ7SU+ubjBy26NTZLY+J3LL/F//eI733jd8Xy7x8/vdohRYNf22VivlcPP73bYNQNOo8GzzQGHsYaPAn16T314cYvBa7w+tXiyOmWpkcFqvL+7gw8SB1tBgDabVmbMGxt3fQMhJgbvfjT4nSdf4af7SygRcRwNnm8OuBuaPCepXoQaEiLiyeqEl6cVds2A3mmMTmHXDLlPS7kRfhZXad52o8Gj9SmhhxQq7WDS+/ui7YnzoWsQATzfHtBZAx8FdtWA26HBoa/xZHPEi/06k78xGmrXDLBB4tDXqI3LSBhGUdTG5Q2K3mr0vYEx5PlWMszuKUM+MuonkQBaq6CTV5tJAVlexI4apnKEtEgkepy8l2hqi2GclswlgiR4kujQOiQUySQTorXHMJi82SAEee9LQj9gMvhprhebKUm2JGtcyzhjzwYjJRLXgkqSJyiQRsyanVm0z1he5QYGEwOWYRB00XzDI/hEmijvG89L/fG53Eic7UyUZZ/9X4RD8D3n3J3nNlR+k/7y07c6xnL12+/H7//3/wF2zYDRKxwTtKytLNbViC9vtzDGwSYY3f7QZvbP46nOELChN7jcnXC7b9G2I5yXGAYDf9IQJuDy6ojDsYFM2kbOEVxtSHCazeUJfZIacL3B+qLD8Y6YO5v1iHHUOZ4nRiB6guNUlcfx1QqQkWA9LpF7JPY9ubUQkh4WNyhIE2YsrTEKIh7xAhgl1C5Ja/SKIDc3FZFYNB7YG2DtIHRAOOpM8qF2I0GNBBB7BbkiWFtwBE0KTIqSIDmZZdEm9kwdITpFu5AusQ8ynMUQ7EWtHXyvMmugMAFxkBCtRzxq+j9KInPpdC5DHBSwc8BRUztMAPYawgqES2IlhA5ULgt3O1lAaiLiJmFQ+FqA6lsRVEesHaSKkD+l8bJPLWAlsQ5WlCeaMMXzRAF5UIAAAjMsmpAhQLKXCFWAiAJ6LxFMhG+JSCSaCNlJhFWg/3x/pm4XUAdJBBbbhJVREdEEyKNCVEQE4nZEZCJHIluIhvJnQhG/8RBWQnYSPrVf9hLVtYRfRdgdkaaU9PahoM+PCjSuVkxU8eldrToieZFOZPIP3waoXhJZRxuhDwJuQzT/SIQmUROMSHUCfhWheoGQ2DBDPTEVMuGIdETC4tsAfZJwqwB9IKbHKIGoY6bah4hUfgCCnsguzJ2EryN8S22To4B0gBoE7CZkIhUiH6Ft4FBNQvOqp3nsWqK2l6OAb6bPoaK6qFHAVxH1KwkRgf5pyMQrcqD2IjEZCiegO+pXu435B5LZF4Oe4FbmIDBcBdSvJeyW+pBhW9WtQNRp7DSgOyKIYWiUPtF1dhNyO5j4BQDqlwLDYyKe0ScBaWk8RBDQB2B4HHP93SoRuqzoWt8mYgekKTrQ3PUNEdL4mvrOrSJ0JxIJR8wyA6oTRMqS5giPZxTUl0yGEyoqTziaVyLQOX1I47Kmz76NmZRCWjH1YwCalwJuNRHH6I76YLiM9/qlugP6R3TcHIlcpr4Bxi31aZQToQ0zU7oWOZ5PxETacqJ+YRkI1QN2TSQmUSLPO9VTrJ85YpJKQCLR0dNY6S59j5iT9xzT74Ci7/oE+Jbq52tAjfQ5mERkswXal0Ta4uskHZGcw9LSdaEi0hkml0GkfIKha+1GQJ8A3UXYtYC0RC4iHN3DchS+FhN5z5GeP9cISA+onvK1O5E/ByOghgjhAbemz8FQ/q4V+T0pU5t8Q2Q3QQFRCwRFJDSauMrgGpqXbi2gj5Haa6mvqkNE/+ibs8LqUyTCHFBbRAQQiITFblLddRo3Ju9Jc98caaxCRePPRDh5HshprBHnY8R5Za+UAurbCLtiNlcal2CoP3j+SEfkNRB03teCiHkS0Y0aY5ozc3IXJrJRlshTggZ0n96/iUxFBMAbyo/mbnp3nSaiG+kAb2iMdTeNJz1D9J5R4/TM8ByivhGpH2K+PmhBc8PFnE8m71EL8p449T//lmXymrT25bYETXkyMU62vSKNAZH3nDFs0nkRkd/7JYRUOhoTvpbJaPJ994hxIqLg32YiOQKQCWvKdmQSICUy0RNLplBm6Z/nuUXkPg/GRTIcmJmJC09oWS9R9B2fu5fKQ0tbb3l56cEuva/L/+euLw8vvK3n0v/xj/+zX3scYvPBx/G7/9GvPsbyB//Vb2Isf+m0NQP+wSd/hpfjBoPXuKw6DF7j6CucXIVPPnqNMRCE7nZscPGMBM4B4PF7R9zaBmPQWOkRN0OL71y8JsZXGdAomwXkf3a8xPevXmWheIaF7T7o4YLE9bBCfUG7sI2yuB1bXD35CiEK7G1zj4Snkh6dMxi8xt98+mWGrDGEjWMsX/brDOFqtcXoFXpnZmQyrba5vtcD7dwzzPD59/ZZMP5xc8zstGszZjjXdd+i0Y4gXMqRdAsAkzysrbEZbiZBYttG+RmMbG1GHG2VoYIMt2N9y+NYoVI+e4JGr7LHZ/vhgMNYZc/dylgcxwpaeeyqAdd9i4uPSXB69AoXH/eolcu73yzUzAQ/LAqdYcp9A5KGcdnL0hqK2VQi4jTSmH7/7/8cAPDTuwsYFdAa6m8BoLMaUiAzw66rEULELPhuvczwLvaeAuQldIFEnzn+tNEOx6Tz2Bf5EuwM2DUkGH47kKHrkzdnmzZPtvWAm66BkqRXySzCvMssRMRxoDm8qiyOyRujVcCzNe1m75OHsNbElsplsD6k8zJrM7IHjHcOa0MSNizHw162trJQMuDQ17hadbjtmux580HAJdhjU9l8/ZDizjgmtNQM1Sn2bnAqt2PXEsswx4AyEzDFttqsg8n9f9H26KxGP5q8s11pj0o73HXUvyEIaBUyydLoExHBRAAAIABJREFUdH5WK+0gBcXYcvwuxdUij2eMVFfrFJ79QfKk3W5ze2rjUvk0H5UKWNcjeZVObd5hVZIkGFiaBwA2zYCbQ4uL3z1h3zV59x4ArjYn9El0m+CUHrGrs1ZlW1P8d+yrHGPpkjh2jAJXf/uIV4dVYiX2uS8FgG0z4PVhlWNXQ29QNRYxCYiPVk+70JHi95QKiGnHX3hJ3rbBwFQu6aEix6+ayuU4V6Wm2NIYSZsyukT+5BRUca0QpP2ok06o7Q10Y+FHlj8BZPJCMInQ+u8ecOhruJF0P7Vx0CqgPzQQJslAJdKlat2j27cQlYOuLLpDC73p0J8a2hAMElXamMwwVKeS6HiKuUz1QxTwlo4r7eF6kze2vJfkaagoXrVekTg8r4W81TkeMUZAJn1S7ifqd0ClWFMe26oZ0HU1bX5ahaBDrpOoHcZTjWZ7QggyzzNmY+Y57KwCGktxxTXlL5TPslWkqWphmhH9sYHWpBMrRMQwGmhF7/5x1FlPta4thECOqWUY6ulUQ6d4S+8o1lQIYrDWxmeCKTvqDKdU6Z3iRo26tfBe5PFWOqAqYotN5TD0FepmxDiY7MlpG4vDoXnj2uJNyVQOY2/I6DYhx5VKGTH2hhiwQ4qhTXJW0dP4cZ05hjWmGOngBT1jcYrNBUAxyGGKy4WY4KwxCLTrAUNPm9rBSZjaEZt3ik9lUq2QNlx1xVq0xNIdA8k5CYA2qQvvlNAhx74iAsFJqCpkhmsI5BhYWaW2WKJEVY3LbZY6ZG1YWXuEnpmPkDasKbY6x54ygRnHZQJJ0ghkMXgxaa0i5ZGJocQ8vlQUx0WcYkP5vnyODbVkuJUxnUh5qEhs3oXRIqJAFEnORaT74nSc4yxzjCUKo1NgIlNCka8Q0+eAxA7I58X8vjIfJabjsfjDdK0IgozDc5YXeyPjtJFcEkItL+UNlYcDJef9Vxp9S6fi8ty96pWG/EPfgYcN0TL943eo72/SL5W+1R7LT/5gF/+T//Xv5/g7jonjZIOCh7zH6BqigIeEQshxd3NGUzrGsWT5eJrJDEP1UeRrOUapjGuj43GWfxknxfFrwLRQWebJi/eQ4pS09Lkcvm7JCMp5cxzQuXSuT87FOy3juMrry0TxQirHh5X9WLLmLusoRYBL8Ztl/wLI9efYLi53Wfey3nx9Wf+H6p5ZTQWx8Kqi7vGB8Vr2bRnTVZa3ZMgt6/OmxOWyEV7m8S73L8exjCH0Udz/rVzkuXwnL9/T5+qxZObl+KzluYfihspYo4facu74rN5FPueOsfFmPc+Nc303HWdmc477mto6xYGVzLmZfCdMOrEPJTYi+Jl/CL6zZEouUykBxO1b9tWb3uzM3JvLKsZsyY47y7MYw/IYw7imY9MCmFPJDAxM8WZvKuPcnOGYu3eBPXl/vy1l++JsDk11jEHMoGb5mtRP5XxfppKFl68rnBv3Y9OWqy0xj2t8KL7oXt8VmS/7HhEQEmScoOhPOcW/kdESs9FAC9PF2KaFc8mom+8v4XJlvxTtog7i70W9k4Fxb8EcMS28kY7zvcsYQb4fD+RRMos+ML/fKT20qC3b/dC8XL5Y+TMwbxvbRWygLO/hSwt23pmBI+IUl1i+wM8tukvjo7w+4F49hRdnwxSXzZ554Ti/Ms9F2aV25NklC8NOF0YJe3bv9eUbhnc2Mg89Q7n/z5x7U/7L8XngkXgwveV99m55vCkDnB/7t5XzpjzfUqd3qvM75vWrSH/23/76vXrNBx/H7/6HfwEey//619824FvusTz4Gv/nzffggoSLarbQzCQGhYEA4J6htFz0nlswlPIcnCImr95y4V3ef66snEexOD2XyrqVXsqHFnfn80Cqz7QgLu89t4h702LtTQbUQ2n5OxwWRsC5fngXQ+Hc+fn3+3W5h/tP+SwXvbyAfZvxc7a9/KO0XNzdu+5+u99UzrnF+tsrs1h88bFz24Zn6ras52ye4OHNyod+g8+sjc6sFBbnztR1SUSxPAa8Q18tFyNlxd9hEfHQs/TG8s4tSN9hMfTmfN8+J/OlZ/rmnAEYC6HysyvJdyqM78H9tr1Lm99lIZev/Zr1PZf3uUXX2Qk7Xe9L4+ZrjMMbH543Xbd8mM4df1tesbCF7l1zplJl24o5kz+fK+ehej30rJXnl4bamTrd82w81J9/UQvVNyzIz71yH7z/TfNryjXd8+7z62zZ5+b8m/rtgbo9WIu3PS/n8nqXPnrX+97QP280lN7VuHrL9ffOvaEvvpax9bYyf4n01nr8MvV/1zp/g7Z97f77q5j+OrThgfStNixjJAPuaGti69Qjeq/RO4PBK+yqAWNQqKTPsMw+EbisEimJjwKV8jhZg0ZP0he18qgTo+FdX+cAdvZIEHMqwQ8PQwWddv8r5XEcKqxrErjnYHhVMNgpGeC8wugUBb1HkWU6lCQT2CiPw1AXTHpzSCFABlpbETtkaywOA0mFGE2QvatVl4kB2kR04oNEW9lsIBz6Omto1tqjTwQb7HliiCEbqC5BBXUKukdMcD9LJBCs3cgQPaM8+tFAqUm6wnsJo32Wkhi9hJQElTOGKPOVooD8U1+jqUjiwnuC+FXa4+5EMCyCB05GhZQhe3dCFBm2x5C7EAQq45JECGAtSSc8viQY4+u7FZSKMNrDAfmaiW0PqGuWO0hskVmrEKgql+UzauNobBPkk2GH46hhDBEJ8DzmoP26clAyoB9Nrm/wEnVDMg6VcegHglayRMSk30h52JEgZ8Z4jEMaTxmxbgcM1hAkTUaoNIZcDstLeD9tHdMCMmZPjU7SGkrFDLt0lmBdJAuhUdUuw964fwIzsxqSoWCoG0TM0gQoNmFYhsI7RTIfVqOqLUH80rzj8glC6TOJAkBtqWoL5yS8U7kPpApZvgICpNkoY9aAZOkJRILdxZikE9LcojonAoWkmScVERy0qwFCAKdDPXlsVYRnLcUErzIVyW0MhTwIw88CSw1EkrYYO4N6ZTH2evIkARlSx9IaSnvYXmcPlK4YMkix2TFJMhA8TmC1GXA61AR10wSFDIHGo2ksToea2qYj/Cihao8wKsjKE6wu8CYUCEonMEHiUjneSkidvJIRiOwtNgFSkfwFw+xm2pReQqiAYKk8Kac+jF5A1QliPSiSM7Ey6WXGKb/kaWy3A8ZRU1y4iFS2iLCdIbkHIMs4mNbCHisIE2iedoaO9RrMHilT2ezx9FZO8EGBLDNR1kGaAN9rKo/bmuQxopfQDUHS2QsY7ATlBZBhhpwnj7Gq/axvVeXhUzujk9QXIiJ6qlM4aZjtSNBLjpFPWpqsQcr1577neR8C9THLaSgT4EcFoQJJgACZGCQGIFh6pqOVEKnP4igpTjLl608a0ImYJM1DIZNkh54kSOLIuhIgWQuA8mW5kULKIkt2jCQ3EnsF0XjEUREUOQiCYx44YPEbpNoDLPGh4hySOahJpqNKsfkRk3e19vTZSorNj0jXiEn2o5QVKTVdWcqj2EARjUccJLIH1wTiMiiNLJb7ACY+AB77iAlmmuL0Y24PZlBN4QWiTrH/S91JRe9MkdobTcjPDJfFPANinOY2wzZjkiuhGP/kbfWY2vEmuZGin0ThfedzIiLHGLLMR1l2lv5gCG0pScIpIscvntvwyEOSfsdERO5H1iw9u+lwxrCPAoVu59T/9zYjZpswi/FaGuViaivkfft3Vpfl/zfAXbmeb9vzyJ7lMt8ynSv73Pk3fefGvG2D4zfpLzx9q6GwH/z+Zfx3/+d/CwokVeEhMQad9SzHoLMcA8tCaJH08ZKEAqcSlsqwU4Y7VtJlyYgylTIEDIfkGEpmUWUIZymLASDTzbNEAMsTAJP8AkMtgQmSyWnpgT1XFmtyllBPLUPWzGTdTYYu+ihncFAAWd5kCQctUwk9XSbuD5YcKGn2l55PNmbZcLde5Vg2nWLgrJ/kHs55S7NXN30vjcySqp7fO3y+dxSv1hqSoSnrsYTU+sTuy5DHclxZYoA/M9Q2S9WkeMzlfbkvU3wjM0LKZCyyVmYIMsUxoYDyTt7oGMWD50dHTJklbJPrcc4TzBsh5biWfcfXMlw7LsZw+ePF93C7Zwx0i2u4fF7UZh3RAma69HCXBjqAHJNYwh6Zqp6PZXa7BTyTNxLKYw95zFnz07kUc6nndVx6531a5ClVHkdRVppLXmQNQd44yONQxCKV8gacyrqX7IX5/hRbN8VryXl7FCu50yKK25ivW3jAs5xD8fzlceY5UORfyjEw9JPvm61puO5F29joYmNIFICSpRc2pkU0G+vZgE1GXpmCk5A6ST4EMnIDG2hF2dnDG9mgnK8Y2aDM14VkIBXezZlXuIxh4gVimWcJkSw7pyAUy/Fjy9iyKCbDQkUyOjgfFGXGIv9S868cFGBaeBfGBC/oY2kEyeK+MD/Gi38io5ralo+rmGLBEsSzGHvBGxqy0BQs+46PcV0kZuRzQDKQlrIRXyeVcXElvLbsn6J+Mwc6k6KUC9/CYMrfua6l/VK2M30nAyvmctg4etCbW9QtCiqg7NN7HmPM7Nip/pxKg6aoY76HP4vpcyyHjdvJBt+5fuHXTXE8ty8W54H7hlvRFPFAu+YXLupQHJsZpVyVZR5l3c/1//L8GQPonJP+rR7oMu9fIpWvk3tlvaGO38i7+1D6Gvd8E8/lv/jvfv1w0faDj+N3//1fPRT2z/6bX3/bgG+5x1KKiEf6iD6YvIgp9RKNIj1FpQZ4SLRJ+81FhRrzWEkA0PDp3jn0NUSBWg/3jIBWJc9VisVkg8tHgY3087hJOY+L5MQac2VZs+/vgJe6B0tVdIyp35epSfUurz/Xbk6lxt3b6vBQO2rhZsdqNa93/iynfFpN41WrSaer1fO6vxVO/A51pvGd5k19xuCbtUn5d4cBL65piifu7P1nzkcATdK6W25O3INXF5+Xv2WtIc/IQ/fz93PG0DkoLIB7398Wj7uMuXwIkj2Pr3PpmDs7zufiIWMUaIybxUeeS2+DOZ/Xg1v2Q6L1N+5sX9wbZ3M/9vpsIn4QGO3v/dZWyeGS1y73+ux+m8q2lvGh9N090DYk49Xn/3zsofQ2mPpD4/im/GbJzD+/qS6iPn/+bPmVnxn4IQgos2DdqM6svO7lxX00n9MP1nGxWbGcE2fWnvMV9LnP5aVxMrRFMx27f+GZAhcvkTc9LyK1pZQVeBNbh3ioTwr7dGnoxKKdsbj+wVQu3tP/eLZDv0YSuB/fmcsrnpszdTtb7LvW5Vx/PTDm5+/HfYMHeGfj5E3jNW1k4N6cOXv92eOL+bQ07h64/21G0P0y5heIZX+cva+4/g2X3bv+ob54xzGfGfXvcs8Dxuq7lPuwcfiGd9e71uld0td8Jr+JUfmb9JeTvtWG5e3Y4n//4vfQWQPrJUxipGNmyFr77OlhWOboCKaokweECU4GS/BL3iHP0M0gMaZzpdciRpHJL6ydmByZCc8YWqSUmkol0QdDGKu0mCEPwuQ54nz4x5sXOnNCBiKYYLgSQyv5WmOIIdE7mSGMvMvOXhjnVPY0yAStBKZ3k5Rh5vUIgRjrhKTtO74v+PvxiByjyBBSkTxTMUE3J+a6lLefGOSETFBPhldFgRgIWihkgLeKtJNKggGBidEudREz35UaSAzXYuhaDAJmRSyabtCAKK4BsgeCX7DShJw3fZgWT0KHvLCSeoIgZiiKjBPUz81hvAAmONg4QTJjgs0hUP4hQXN5dzzfzws+9tIogsQBVC7DGGOCoJbaVFznzMjHsEtud7qUPS+lpyamXXIhEwQvMXtCFp6Z1N9UJ5HkYVIZ3I/AHFYWBZ1jSZkEZZtBuPhehnMF5HEiqKGYe2lkpD9bwOvYQ8R14fmU4Fn5fvZGlA8I3+8JlgYAsZ/GblZH/iVkb4kt2p0nrJhfN0qC1Fkx/4GvwtTWkOpqi40h9rJxX5djCkxwvogJZsaLEp3K5Tp7QcdKqBgnbj8w1eMcAyOf57rJol85H84reYSYmAQy5mtFSMdQnE9shxCYWBn5e5JBIg9NKldGiFFmDw2zHkYd6LiKgErX1HQsG1Y63mvTjDBlUT+uUwndo7EAbegFkPesWIgKL8g2TeVkaCD3obh/nwiCJH1ccV0xLkFHyFHCJ4kjUc5zpDoyo6aOgE9txTQmIgoIS+VERXJEkJj1Y163J28qwSeL8eL5BZCkE/cVezwFSGqIvZaSpI1yyrBMZHkcqhuSjMt8bkgr5lITqd9k/47G2JkUNSAc6N3H7500d4XluiNLQYjinZQhn2GqD3uWy3PUiIVBsYjnFREkj2RTX/PcC3PYJY8DRJpLCe6aPcRFn85skgW0kvt46eFmiYrSs8htL59pvj8zpJavEW5aAbvMfVP0BR+feQ/53R6Lei6GVxS/0dPBqR+jKMoun51cQdyXBinWYjlrMT++9LqWe0A5j3NT8YE88veIeVnlu7PMb3HP0rM7q8tD6W3n3zF9LUOwbN+59v51SH+d2rJI32rDslYOv7V7iaOrMmyVWUNdVKikywytLkpoEdD7FCOYiHd8AbMrd5ZZsiJEgc6ZDOcEJm+DEkkSwesMi9Qi5O9SRAxOZwkF9gapwqjlcwxLZOgk31vCXJfexwgiEGJY5phgogxNrBTBXG2QqJM0B3tR2Js6ejUZyjJkqCm3vewHrmfJeplhj8mgPAcFZZkOhkmGCChJx42amDqpDrGIK6XPWgX4MME8AcAm6YSSzbL0krD3ynGfpDoCmN3nUoxaW5EntG80xOJ6v/iVKj3Rs3d72qTg61n8uSSSWsJ+OXHdGfLL9WL4JsMGS7bYabMCk2Eq4gwCyjIFUpJcho8iSw2U870kLeJyidTo/PeyrxkmKcTENsp1YPhuGQPLmyi84RAKQ4U3PHIMW7q27APeYOHv5fX8HcBMrJraVubLq3YykktG1BJaGs8Z3bFg0wQAQTGoOnm3bKUm+CfPtWKcOfbUJ/mNMpXQUakCfEP9ybIFeQ6mja5ypVLGhzJkmPuP+ybfrz18RXkIOcG1BZdbq5xvjnULi3YDabMkfQy4d13+L5A3aoSc8l0umsrNIt6AYrkE9obJZDzw+ZLVNMNxU36qiC8VAmmsgVBPrKZsCEsVEes0b9NnqQNiJSeI7UJcnKG1QFr3RGTobFkmf0a6ppxDQpWr1XmevAGU+7wYwxk8OEzxqfeM+ghARtLXTbFvGYYbyzkspgVqumcqgDyEsRJ5cyZWfO80/jPZB85TTuOVN6sARD0Z2zEbKmleLObBPYxepHxnzwCwKCvCG74u1Sfd59U3NyzvwXvLlPQiZ/0IpEV9MvxKw4znVRT3pSqWxc5+h2Iy4CJE1kiM0yRcGix5Aj9wHbfpjOF1z212xrM4M5i4Xgyx5SxKxt97jZuaVeZ7dvFdGoMPGWvLvEPR5nlmUze9i/GyOPcuxlJufzkmX8eYeweDatkf39T7+OZ733DsHev4jdNfN8My/pL98Vc8fasNy53u8K/vfoSt6hCixAu3g40Kz80t1nLAD4fnaITDKVTYqB5f2R2emTvYqPCL8RJb1WOjery0W7xf3eCl3WKlBtigce1WuHMNaunw2+1X+Hy8woXqYKTDwROe6NquEKLEd9qXuHUrhChw8DXeq+7wxbiDFBFPzAG3riVILshI63yFtR5woTr88PQMEhFb02MMGoPXCBDY2xofXNxmaO+NXWGtB6zkCFsw4N7Y1XR9ewspIm5ti63u8ef7p3hUn3BpOnx+usTj+ohWWbweVzk+9Dur1zj4GkYE3LkaO006bUPQqKXDjW3Re53jSLd6QOcNxkB1aJTDnW2wMz16r3PsoxQRazVi72o8qY+4s03OZ2t6vBrWuKo6XI8ttobyvDAdXg1rPGsOOLgKX3VbvL+6w5fdFhszYK1HfNlt0TmDjx9d425s0SR9T9Z0HIOmDYMoIRHxuDlCioiTMxRfKCKu+xXWZoCLCo/rIyrp8P++/BAhCvytZ7/AyRncji3WZoQLEhsz5DkXosDLbgMhIrYVHdfCZ8mU27HB2ox03WmNSjlsqgE2KNTK4TDW2FQDTrbCphpmUirWK1z3LQar8Xh9AkBQ6Vo7vO5WMMrjtmvwbHuAS3qgjXbQImAMKm9WPGpP6JzBfqhx2XQAgJOt8OJug7Ye8fzqkHVBATKiG+2yEWyUJy++07BBwsiQY107a1Brh9GrbNxv6xH7oYLzCo/WJ1yfWjzdHXLcqlYetfIYvMJpqHC17nAaDZEbBYltPWTyKibXYl3NdWVx2zW4aHvcdA3W9QjWKz1ZgyrBkpm4Squkqwfg1X6NtraZSMt6hcFqjFbjcttR/LTymeAqAlhVNm+InJJ24qYhncFuNHkDYrAalSZt1lPSJX19u0YMAs8e3+UNjcEpKj+KjKY4nGrEILHddNnYc6nMEmVxODZ4dHHE69s1tuseRvu86XK7XxESQgZoHTD0BrvtKW/0HLsaMQKXF8esAVmbpFUpIvav19hcnbJGpLVEkhSCwHissL7sIETEMGhsNgP2hxarTU86hiub43SliOgHgxgE6pVF11UwlYOzGk07wo4awtBmCcee9r1BcBJ1S2RM3AYhCKWha4orNcZjGAzCqFCtRgRPxnt/qgAQgdHQGZjazRAYSgRCUQQB96oF1g66Jii57zTgBcxugE8ID1V7OndXwVz2cKNGOBiYiwH2tobcWDJwAbijmTz6AFB7MiRDQkbogNgp8uRUyWPeKYiVQ2SPskrogY76BgczeVMDgGQAsjEZT3oy8tgjrCJwpGclykge1rsKsU0sJVWYvM46QBw0sHWQLysSrq/DRJTiJ2KVaALEUSHWAfIkyZFv0zkJhJWHPGqoXsCtA4QVkFZABMDX5F2MAggN5R/qAHVSQBQILR2TPW1guG2AGOneoAE5CggP+FWAHCQRvTggNGnFKwExUJ18G6BvNaKKiJr6QI4CKnkifRNTHSP0SVDdrECUgD4KjFfnrJt3S6qjfAFADYK8cSBPnFtFyIG8j6oXiGmFFRQAEaFPkvrRAHIEGdCKhkyfyLgMejJEpJtiTIPB3PMlAXMn4Fo21KluwcSZN1R6AV9FiAioPvXFmDzPMvV7pPGLybuLCEgrydNtiXQpqAjVS/JGsucxADG1JQogVGnKdanNAKSn8Q0GUD3gm8nQ5JhNaek/e5fJ009/IgLCpf+B8i3rKTgfTH3JnkVevAf2Vqdz2dAtjGr2RDNJT2moRkl1nCnaiWI82GNb1J3rxO3nvMrrl8ZSNszlVEb23C7Ky4REkdonXdH2xT4UH+N+eNAIL+skABEiohQQy5gS3muQAoI3KBZpaTjdM3xLo3RZlzNGZJnfgwbwX2Nj7duUvtXkPZ/8wS7+p//b38XeN1nDkmPqbKCFKht0+XwkDUsjfD5G5D8GWnq4oDJJDX8viX5YU84GhVo62ChR6kpKEWYEQg/FLYYoZoQ8S3IePsbeLy7zXDIpTnNI3lhOOTay8JRN+pExE+pwfZYkO5THPKav1JBcxgKV8ZLsYWV9S64P90nZX1mPE3OtSO5LneJXl9It52IEy3PAnFjoXGwhe//Yc8wL7zIxqdGb4tfKxHOQ7ynjPEsNTK5n2afLtnL9S7KiZTvOxURyPcvPpZf+XBzkclzfJQ5wOQb8fCzHhedx8Zs11XdRh/J3phxnJWI+do5IZ+kFZvKlkoSovG7ZztLjfq4/yrLO9TN78rkOTL601MGceYrjdHz52y0Fsrc/y1qk9FAMbOmVj2f6KeedPKklCVV5ryuQC+WY5jlX1qUYL/HA/2Xf8fFzsaH3YveKYxxKULZt6bUvf9JKrzblRf/JMz3vFyljJjWia2TuJ86X8yvvPSd5M40NJhKfRXszeVX2KCN78JekTFP7p+Nv1A0942mKQSa2ZTzouThXv6XEUDl+557jXOa5ehVjQIzM6Vycf36obmVhObyB2yPi/QXpcgUaxX0499dNYvIgi0U297yrizpniP/sJs4XD7Z9dl3+fqZtyzLfVkY5iGcX6m/I8031LMvJD/uZ8oGZh1Ys++tcvcrxPlOvswbHMs8zx97ZUFm+n77OEvqBPnhjuW+4/mvP4q/bN/gGBtxb2vi1+gv4enPvHcr4l//lr5/gpn3/4/i9f/irJ+/5078CGp3At9xjefIV/uj2e5AioPcGY/L2aUEel5Or8uK8kh6dMzCKdqcHr7NBZoNCq21mUfWRYjTZi7KpBnTO3GM0tV5lOGvpeai1w5A8LybBUWeLU5ChqGRAZ01e7AIT4yh7etigZY+GSVDLiWVUpfsEKj3BaqWIGJLMhVEBo1PQKuRFNXtFKu3zQpgXseWCwvm5YaxkRIiYQVDZ61YuyoFpUcr9w+dLKCof88kYLaHJPnnPrCfPmOD+CSSpwuXMF3nzOFSOgy0XrgwFjREZWtsl6YeqSp47J7NEyVJg3aW4RZUkGsrY3El6RGQ5DaXmTKMMFeX8y8WxdxRXq/REGiJlzHmVi8MYJETB7FnGvAIEg9TaZzinT1IFzO7Jc1FwH4qYZUG4rPJ8Nly4vxPEsZQjkGoeO1sytUb8/+y9S68lS5Ym9NnT3fc+J05E3FdmZWWLLlUJ0dUSiAkTBggxoGAAEo9JM2gx6AlCjBAtJkgIoZ4gVCOE1BKIhxDiDzDhNUIMQEigpotWo6Iqu25m3nsj4jz2dnd7MlhrmZvvsyPuoyo78yZlUsTZ293cHsvNfa9l31rfwt6tslecuZ/miioxmexGKO3KMUhbnRtmraq5OgKgWFRTntWRWFVaD91xoLGHyvwkbhYVW2zpxXqqmeJJa6RYPD1s7wNhF23fKyj+tILQKsh9wxY3K/0nTpnAqRN2MXGMIjRlNStK0SCKPLv6NjbSyp9FKZPYTVX3Ma4AxbN6GRu3n/QWLyoxmVLEhU535wVV6xV4eY669BC7WF6WV8+s2VLm4AMvAAAgAElEQVQxyLUK+7jafjy9IsrfVdRbrKCMVa69VBCTojQJRbXYQEHr2g6+pI9ossHehVHiKYFdjGPvGkjxYlsye0h9UZR13eorCBfQXuadHFRF67e6TnZyr/UW61hX1dAuQUZURUNvGtKiAeZz2tau6hCt1he2FA+ScoLH02SQFQ2H3UCbDHmsWsYvsYEtDhDtu0y5Ma2KnJXIEy32js7TOBsKJfNU9L38KbKNtJhCln2/Hnb9ZGxxdyLCdCFneS108r6aLkKO98cUKIbUbLJWgnB2z4PicRG6x3Gr72GF7W51q/teJl6Zex9DKR70aZtbQ+BEJlKnB4171+I+TrDrp8la5NfFYPZ7KH25dlxdylb6UNihmpd1BCFsx68Yzxd7Ofu67+nzGjJ3dbxS+uvUe46/r1yef0/dfm/k6+peHePXlW9T/5vMq42joipGUf+8/NLK99qwzFXjnBxSNc2tDwDm6gg1YxdBSb0BAPNKvyhiAIlRJS54UlRX5+0yteO9QinnQ/A7BCisQ4vRW5ks6LKEqlAq4AwZUal7k0hsWkiS53AzPsLFjj6Ngwy+Ndq2ox+LaTv7K+dVjMnskBsAmMP2C0txnXtDsn8fKgAxd6hDfY6siQEsKJFSFUtwLaYvFQ0IosNGRGLjQurFLvYscA7BUrZ8jVrXloeS3h/q2b2RkthQu0RFJAZwDXTfvSc3QCFAknhBpYCU1E7mffwfGU9bDOUuxrHLnUnzpWvFMJW/vZaimeipdIRBDa2qimL2uvhFMUr69ZCzbpsgksMRoLg6iq/aYjTlF6xUBVU5ZlTQoO7e7wwp6UtctPgXSBmWgellTfNqBE6dUSlFDMb2O1zRGYNkPEodOdYUjI7oQnC0ZmQ2g1Da5bVrN8RBYrsUox21dEgNbz60WEDVKdmiQBTdlHgxFFuMnKpkJPSbM6puBmVndO3i/eSYEHF0bpHtnOSJkyL5IkV+bEQ2BIXn1grnFmwGINDGrGzu4ke7Os142Obf6oghY7u/BRdxevyXyXFae/25ZpDw2rTcWU8IZDeDfHf9hVIJBdSRtfvWvsyp60v+eB6TrmSMZoU6dMYvQHn8OlltmjggT8azVBb8CNYLWTSD1/T1tzGJQQUDXFWqJJZTrpTxo3bKGNexZCDkQ7dhsO+xoUXFEVlQ8Vsl1a376rp26eTFw9sPCs/Iifo57q5F3dabv1jfXLUI8U+VcXb9dI/dTl7yEyfjGnFdnt+m7OazfZclKp+flWsGbf3AuV29ywUO5OFCTr5e73e7yfw4Xch9V0c+1915OlyfPWZX+/qQjN9z/KoBJX+vyPl629dHd9VQkvbas/LhcX0bQ+v5uL6+yrc2zr7tGL5r/1/Txzce95+xvK4X1ju+A8r59738Gd67X7XyvTYsP3GP+Nd+9N/jD8MnyNB4aU4oVeNdPuBNusFv+q/wWCY4lXGfJ3xiH/GzeAcA+Mzd4zFPWKrFQQd8Hl7iM/eAN+kIpzLu7Bm3esZjmfCHy8f41D8gFotz8Rh1RK4ar+0JsRr8LL7ArVkAgNu6w4+GtyhV48t4g4MJu/yPTmU85hFPecBfGN4AAB7ziFFHaFXgVIZTGV/GW6zsnH8wAUtxHA+5pbs4mIBBJRzMip8HijG9MWtr+z5PuE8Tfujv8WW8wbl4vLJnHMyKcx7wRbjFoCNiNXjlzvgq3KBANeKjl+7cUFyJIR10wsEEnLNHqQo3dsVTGjCZ2MarVcUpDTjaFV+uNzjalV2HDebscGNWPKQJr/0J7+KEycQWGyrfP/FP+Hy5w2t/wikPWLPFa3/CwQT8ZH6Fo12xFgunSosJHTTFwInr6JfhhtK/2IDIcaEv3RlPeYBRFe/ChFQ1fvf2cwDA3zl9ikEn3LoFc3YwquIhjs2NFwBe+zMMCt7FAwBgzo5S2UDhaAJO2UOj4rPhAefi8RjHlkdV5vfCrngXJzidmwuzVgWfDY/QquJNOAIA1mKwZIcfjA84pQEv3Rk/W1/A6YzJRMzZYc0WVmc4zln61XrAaBJe+zO+DEeUqnC0AT8a3+Ehjfjp8gJHG2B5TqnqtvEymIQ1W6RKKL+MT9x3JcZ20Km5ij+mAbd25fjeEZ8OT/j5eoOjJabdNVuEQrlIJe721q44ZQ+rCp7i0FyiD1ZiKIlo6CkOuPML3q4HfDSc8JiGRs51sAFLIm3sYAMGkzBnhyU5FCj8cLrHYxz5/lWMJsKz3L5aKT46Vd2OA2Avh20sAPCUBmhUjDbinCiIaDQRS3YtBvecPH50eAejKv6fx49a3OxoIp7i0J4JrzNeDRQ/+9V6bO8FrykGcMmuXfvRcMJPTi/xm8d3+GK5Qcymua1+PD4hFYNUNUK2uHErvlyOzd35hV9oLSxH3LoFpepGXAYAPz6+xR+fXiEWg6MNGG3Ekhy0KvhoOOPvne+Qi8ZkIx7jgDs/4zGOuHULTnHY5dGdbKTY8eRw52eck8fBBjyGEZONzb1eXPmPNsCbhPt1ah4kuVA6o8GkNs81W9y6tdVVisIBbjnm+T6MuPMLkbcx0u40ocWyofij4zs8hAmnRM/kLcvli/mmpTGS5++j8YSfnl7g4AJe+AU/P9/i4+kJXy3HtlH2cpw5bICuPcWhrW2jCpbk8GJYkIpu7R5twLt1wmASClTzhjm6gFAM7vyyc5efk8McXfP0OLjQNkBlXrlo3HCMt3jg3PkZ92GCVQVzci2FUswGo424X0f88PiAVA154PS/STojFoqhfjEsOEePgwu8ZlOLXX8MAw4u4s7P+Go5YjAJVhd4nXCKA5zJSEXjHD2MLliSxY2n5+gcHZwuGHm9vF0mDCbTfc0Gk4utnmdPlcEmnIJv8/QcsjBHixcjxWaLe7+3CQdHKaqWZOFNxtM64MW44BxdI7C7GxZ8cbrBdy1HH/CwDKhVYfSxI/OjWO/BJYRkMLmExB5I4vlzMwTkqhCShbepeebkonbnZHNUPJ7oPu43fUvRuJsWnIJvG8ejjwjJNDdugFzqU6ZxTD5ijbYR4knOZIA2cSW+vA+dsLxBnVjGpejmHi/3znOMfkgWtQKjj82d3vK9TMlgcAlLcG1+vedQxcaiX9jDRjbyxPNINk5zothsaaN3cW+baRey+lBeXaiOpOwiNy6A5lnTE82hbpuVQkzV8uXKJiYfK0nv+mxEVr13gRQxcsWT5YJE65qrtdRrXjXdceqXPzMx1tXSG9j93/eRLfEYv1G5NOAvL/sa4/47uWT/ipfvtJHwPSnf6xjLz/7S6/ov/Rf/NNZiWyzenB2czjgl32IGJa7NqrIphjYSAQrHQ/bssVrVFmNZqsIpDhg4j6DED4pykappSk1LXdIxwfbnLhlBRZGSdoEt3ktcd/t4PTHu+noSlylKl5wrlZU0NqZEcZBYSoBiAddkW31ncnPh7Ul4etdY+ZG7FutoLtxhhTVVfoSkVEjMKBHDxEIxrCGb5vpqdIFjplur6UcnMkmPZYVFfvgu48161lr5ke5ddHvG18g/XgdWEE7B7RhsxR2478PbzbW39D8iwC4ezgmDbda7eLmUdfthB/ZMu+Ku3KPV4iYs5wXRpbGZXYyg1Jd7mJiRVKvayHKWaHdsxOKKfLEpfDX05trvgpw3akvrE7txVaDFqZluXbWYUUaiezfiPo5O1pX8lbQ//Q99z3Ir54mMRnfMuFtqng1t3o7LWKT0yPQlQ24fe9diLPl+py6NjMTtNZSf+yck/Pkv/G4tmYKUiFRH0hPJOMXFuWe8zXlzLzZGxr7JtH/VO5cpnRG7UxsjqHJtbaFyCqJkYGwmpJyVl0uviZ2bNis3VJ9ZSPv6EtPZPbs9Ii7KXwV2btlSr18bWmKPLxB1QcVl3OLCLW7gpXdrLpvympOG0rxGONWRpBWqldIhNWRX1ssFoiKxgoJ+C+LeK50QRbB2LLeCjEibIq+G8u8ZeVXn8nip4DaEV6H1VZKmVEkVzxDw1k5RDSmX6/t+K6+BPl1SQ347N2ZJQYTcocq98g08ZwXuXaV3iq3aoMC+LxmnKK3MNtzchHuX6t7TwNTN/fi7FHbvpc/9jZKxc3+9O3Kr041FXYzrWvxlXy5VNZlzvfL9yku7uWoXimlUvA4FSVdVPXdDLRdtXvtR6Ppo8+2Npd4IUdL/XrG+iiZ+gF+pEdRc+7G6IsP3Ipb9XD5g3Fy1hz6kOveVrxhnl7bh18rife1/E/W9q3vNNvv6vq6vyW80xg+N5+vqfZc+v0H5g3/3lx+HOP3wx/W3/uqffYzl3/obv/y5Ad9zxNKogslEfOofsRTXWFxf2BnDlPCz8AJWZczZYzIBpzTg0/ERqRi8i8T6aXXGnB1euhnv4oQbG7Bmi1P2eIoDvE74nRdf4E04YDJkfKxswJ2Th0XGJ9MT5kzIyZItPjqecB+IOfb1sGLJbhdjmYrB6BdMJuJn8y0A4MatCMU2l17ZjbcdknKwy47EhfpziNlgSQ6vx1NDJo424WfnGxxcxNEFvJkPeDEszS1YDMqPpycs2bXjr6czj5F2Kk/R71KxHFxArrrFdg424RwdDi42g1qMgInRrpd+bX0aXTCYhHP0uBsWZkkNiNngxbDgYR3x8eFE9yB43A4rnoLHYDIOLuIpeJyXAa8OM9ZkMdrUxkqoCI1NjLijD+RWnA08v5Xm6GjntSq8PswwquAn714CAD65fULIBnPHWnozhJ0xdFp5c8JHMmA6A3cJDo53gJ+WAVoXjC41Y3KNtu0aD47G4K2gNgqnxSMlg+O0NsV5MAWn1cPqgsfziMO4ohSNwPIWI0fic49DQEgGIVkMjALEbPDl21s4n3AYV4Rkm3FlTGnIUUtHw3NqqURA7/8QLSwzl0oqC+8SQrRYi8IwRDydRmIEZTdkrUuLR11WB+8T1mBbOpCBd/bFkAKAyLHH3icsq8MwEOOoHxIZDCYjcn5ZpSpCoJ1ya0szMs+nEcbmFjebs0bOCiV7+CFCqdr6FEPKui0+cl04Tpqvj8HCeUaBgoVht8gUDazLON2PQFWYXizN4IvBwg+xGUKlKKzzABQFN8VmTMiOuu7idcM8ENPq4wg7xmb8AcD5fqKcp4pSY5RgYMfYduvXmd5HbkjEzKoAbXIz0M5fDrC3EdoUpGgQZ73lXV0MzC2tm7Q4WJ+wnjzskJFmC+3zTtFKkfJhalcQzo5y0UYDM2Sk2ZEhomtDAdLZU9zdmFA5RYoYSTUZMloy5UONwQKJc4RyvGpkVlg9JsTZEeusGM+Sh5eNsfLGo/rSXCtz0BQPdkwoC/uocmxgPXvgJqEsGiVo1ENCvbfkTstjzydLChobNNVVNnAAcc1VzMZaLcdQR3Kprb0xYjgmV1eUVVzhcdEma5grxdNWhRbPCQ2oRbc4ORhALQp1rJAclyrxGGVMU4b60pOLra27GDrFzJXKUo7H4iv0wpuIzJRZdUUZifVUrwqYqA+Vifkye/pbAcqvWIQJlDeQ+JgONK4s18sY48ZOKgyuqtB1bc5xy99oZrXlrlQgxtnAz5MDdCQGUrPwd47vM2eFePvdNVSzEussyUa1+EIUoAxoMZ07FlEKq4ZdmPnVkRyqrhynywyzYHdfNgIk1pTWkxiq/N0A7kkhD/K9wqyUt7NqdomtwrpbeeyK+pZ7qugzWKY7RtWkUIVtVFF9HWid9jGOUkfkDmz1AOyYXHXY6vT5KDXHZAprKYAWl0nPGxqbaemYXxWf62M0r8VIthymfWzmhaHc5sr5ZVVvoCs0xtjLsjMOe0NX5Fi668S4+7assH1/3Tyknap5fXdzv7afsWPFvVZkTL3h3oxSil+k9ir3sf++a+eZoK7MVap/yJi+ZkS/bwPgQxsIv2rl+zDG71i+14blQxzxP33+202xF5THaCbGCW5HDCMuIBuxDe1gi4vGyopqU0KZMOXvjh8jcJxij5II4uB9bihFThrOkzIHANZlVuC7gTMSoDUpxADaTnpDXLKC9cR5LfF+WhQwYKeQSp62P/F37ZhSFWk10LY0xfOnTPufk247uX/i7porSEmkXCqg7fAXJhqRF7A2lWLYmLBEddfJMVGIBN3Qlnf95bipRExiC0rcFFo5/rkouEHjiyGjBEO76LqiBhr7u/FIbeiOsKSi28HmF56jt6jstANA7ZKVi3JenujX7vw40Dy6xPJKYvLkbdVo/PmHXkhEFLrrVEvq/iiKp6lA0niwhchQBP0QmVW6XhWFt461BQU6H3VTnhY/svamtvMtDhN4kvi2rHCSOK6ioBeN1XksbmRSlG0X+1kwrbyhZceai0oKixCL8I/PYjclNrgRatV49MOOaKMRPCSFwErvyonfZ4m3U9jIIfjHcWUCleBGqKgQu6TtPVGF9J+ETAWAXjWiqQhdgnZUkaEnBUl1c6kKqR8LoxqLxIhlhSzoS1ZIIrdM15mZ47XPpv1oqKwwW9H8+D5E6iv6LahKVeJoSZ1BoVeNdXTQi0byjvrjdu2iWwyeKBXJuQ0sYWUxekdyApBEaVOAmxXy2SBrqms5obyqRAiSz+z9UIBkPexKaQwkeX0DWtSmVFYDWE4Qb5NCcRY2bSQhoixZVsarMxuhjaAm/F6SZO9WDAljWSGrcEkUcAOXVEvloFiJBhtgqig2KnRTFjWnKMgnAy0pClg50wEoZ8/GD5CfPCvCm+HHnvabgmj2imhPwlINp+vI1EZLRaBkrbNBEVSTJV1XN+IaRfdD3j1NIdV8vG7z1ZHuESrfS17TVdN9y2cFe1aAUlsC+05xFyNNR8XpP3jaosAq1Qw2lclgVHlTVE23vqrhe2xZZhUoTrXUEKoCWa6voLQFfD+sVVsqhczXicLI9YtVLUVFVarJU+5pMWzsLoCJ2KWbMCsbhN+x6ATkWTXZNEOMDeneWGpGIf81geZaDa8THrsYolR3G9uO4MeonTJaNRnNOmwylD6rlvc72PBQbf0Wuxkh/fNbrNop7Lq7B82YFFIeqcdrsU/3AWBH3iOGpRBCVbtdu3tfqW59A+1ZacZj19+lgbgzdLo5tON6e1Za/U6vERmplu+yM07k9+B52O+z0tZ/bxDymNtndPK7ZiD14+oM1NZHZzxd/dujzhdt7ozufjyX87jSNrVRL+ZRP9jGbgjf1OC7Jpf3GpLbff9zwp5fnfK9NiydzviNm3sAQCh2h7gAQBy2FCOX7qh9WgIpZdyeRqHUV9hcCqXUrg6AnVtfc0ma+IdH9akFtvYbC+yor9STOt3Y+JjRe/IeccWUNmQMAFCP2/HLFBPi+ti7hV5LsXD5qF59D8qc3+cyoTYXQnEpLB0K+r5rm8F46I4dVEPZegr/y/H0fT9v9/n55DK7KYoM+/FjZ7zWw17e19oF8CzW473pAHbXkHPzLh2I6lwB+dz7ZN2f73+fAKDcqC3dhYyh6+N6g2o7z5seWu3l03/XCqhHQOvt2G6kLMdnf/sJdIVYW9mds6i9XLq265Vnqx5ZmerlJc+LGDKX/XYLvF7eo/77hVygKvKBn+XOPa5czK9WevZQsbG0Pp82ACDfkPtkPmzjlZKO/b3DpjjItb1yduVzOXKbCsgXikveEfrQdelwqWV0pZNF6eQMdd2brXQK4k7u3XVyvFzeF4WNzIo3Dd+rwVSFdIPnMUXSzrOBdWOS8V1zDZSvldMkXLwUnxFHXD6IAHpW2HTNra9XPHsh7h7oi2N9vNalPPlcPmyGfi8n+bpzVSwXbWBTTBsK06Efl4+EjO0SQdkxt/a37n1K97X3g8xJOuQOVHneRrxYUw0x+q7lUr79325NX9O5Vfecqn4u0sb75osrxxX2TKXYGzatP1zI/nKtdWjfrv1LBb9fE61e917oOpQ1JIfe9/nZGr/2s1Y7WV2TT33fhfKMXh7j+ld/8p63c7XvS9lc1P3GCNw1herrrnlPuWoMXjZzefwD8/jaOeBrxvRN5nXR57cqz+779TXwK1nauv31LN9rwzJki//3/lWLW4td7jXDCGVvFEiOMolvktijUlSLOdoQS9Xc04wt5CoH7OKVCsevESpJ5yX9RGaXLM0xRn3cECq5B0JtrJ2C7gFoMTZakBa1sWpePn3iXlXZdUypLZ6nRImHYYTQFHqfZtVikIQ5s7UhdSSmh9HWphxLnBCzlori38Z/aUgVZufsgurbcVN4XHX7zmOoRQGSbkEQRlUJkiog1zZB7PqXY4fcoYIo7YG9gijxN8DWrrjFSbu9kikolhSJjZPjDaXt2q7yuTt/+VdkJWNnhFAVhdynSFC1taVSd07mdKGYZKH9ZxRKzqlAqReEzn+3joTZVBBMuUaUZxmKKAwVTUaF0SFUoJgKFTWKK1sfGltMTwaxXMrfCnbbUpuy3b90uW6x1G6bjygjMv8uJUNTXBg9a4oXIziqYGPkVNtxGut2r5Ww1QpyzSha+9yhbFXXhjz1bJp6dw8otYIk9q62+ymUKp2yLWkRTOAE9f2SidsaFiOgIQEKG2om6SG4belHB2b9VDKerXGdOMm6tGvAbnG8bnTdKWuCjLW0FoI46s1IgRhgItcqcr2ikPbXZ0EsN2VH3C0lkbm4iap+3cjtDGhJ0+m+UT1BlkR+qGiJzFXdZK8jyVCaFUSsv189wtEnRt8MPh5rp3T2RoEgRv253TrIm5h6I7elNVGVUR9Fz0nt5ttkxojz2q/p7SY2xFJvqLS03xsr1W7ur9XKved/nSHeENdLNKvu3SZ3spJ7I2tWb7LrxylyFxdNQf3QuUs2F8Eucbw8GzoB+U+bbkSQuW53oq1TlsGl0Qfw2hX5dMbU+1C7XbL7fiOGr+vdbQXRbc/DBWLWUFTTjU1h935o607+qb3ML9c2rRvV1VGbXLr131BGSTci87tc472h0n+v3bgu5Ve3dpocLpT2q+jbZel/Uyp2CFhVCjp3vyW9zoH+mitjkLq4qI8rxy7H1b9bulfl1T3PS/fVnQDQ1tl7539tXjKM99X/mus+VL7unn2wj2/Z15+Xv7/le21Y3rgV/+SP/g4+X+5QqsKtWxpL5VMc8PFI8YNeZzzFAS/8jIcwoUDhhVsQimkMhm/XA+78zAyVFQcbcOsoNvBn8y1u3YpQDGIxLf+lsEZ+tRzhNbHbbW0t0KrgPkyYbGxjlnjAOVFs5KcHivk8JU8Me6pAq4LRJHy1HBujouTZFGIbQWCEYe9gQ2MvPLiAc/T4wfEBD2HEOXp8PD3h7XpAzAY3foXXGaEY3K8jPMdFTjbiKXom+yFSnRsfGnEQsDH7DSYhcazlwRE7pzCKCkFLyAajTXhYRgw2NVKXVDQmF3EKHkcfMEdHDILR4ugjTsHBmYK7YcH9OuLoA87RIReNow+YbMSb+QBvMkI2O+RzYFIaIcN5WEaUil2+zoOLWDje87wSo+SPX76D1QU/ub+DNYViRpmAZmYWu5QNrMl4MVL84zm6RrTjbUbMGkdPbWtVcTsQa+EpeFiTkZjl8BwcJh9bu4LeGl0xuYjJxpbihlLWGNwyA+LRB9zPI4yu8DYhJIvEGybiAi7xmAcf8biQG6a3GZ/dPOLdMuFxGeBtbnGmkteU5EbkQ7JO5bsg3sJ4aE1p8l+ixcHHFmN6HAIelwGD45ygWbf2Bpda/OoabdsAkg0bIRVyTE4zB4fDEHBaPG6nFXOgeOVSFKwpLeert6ndI4kJvhlXrNE2FkJrCqzJ8DbjiVkdSyGSHyFHCsm2mNXJR5QKvp4YGsXdnu63aTlVYzJ4fUPxyV8+HBvy7WzGyjGO4kp/GAKMLnicx/ZeEEIayrFKn2+mFQ+nES9vZjzOQ0t/AwB3xxmBY1hjNhhcxHkZ2ibUYQy0Rhe/y80q43p9c8abpwNS0hiGtIsTvh1XvH06NAKkdXEYp4CVY11jtC0coFYiAhK3/nGIiMm0eTuXG3mSbLZ5T++CefWNAVLqCFGRMQUxGnifua5rpELDQPd6WRyGIVF8LW/UNNIidpN//eKE0+oR+B54l2BNxuPTBOuYYI3DGI7TisenCcZlHMYVD48HHG9mPJ0HSIokP9E7X9L5rNE2uUoqIee3dmW+y+xhOH5XNi0db0gOYyTdmd/pMRqUtFmifowtHroRNBUFP9LvCslPY/QR88opm4LleGGgZg07RpxPA+5enClsJNgW97yLNY4GbowIwWDk9uW9XYrGslCM9OgjzouHsQVGl/YcS5shUK7gFC0GXospWGhTYDlOeT0PMJZ+N0vWMLZAm4K4OJJN0hwDTGpKLdjimoOBnyJKpjjwkhWMLRTPDCCsDs4nrLPDMEWEQLHhORkcDivOj9uz922L9RlxsUBV0D5vRHqmIMwO2pUWypIzh6rwhpmMOSdN60HIxbKCGxMkl7FsIvc5h4VESpTqWhTGY8DKOZhLUrBDolzIpUsLpoESyWK0PiMF08ZYMxM6qYq8mm2DVNUW3iJ5eGvS0J5TEYnLOm9gCylU4XRdZsg03kob5CUp1Kihh4zCset9/llptyYOp6mqhWtQg9vGFLKiMJN+c1g2BGVs/aatXCNkUi1+kftpm3Hb5lnbDJRS0UJe2iZod+220bTfIJPxtdytCnTtJUFVX8TAlX4v3WhlA0d39Yqi0BzZkLk0HsVA7WJ2dzLoPj7722/KXxrBl4byBwzna8b+tePtmss2fs2MyW+N0H6PyvfasJRytCu7wupG6f/CU/oPrzO0KvBMLe+ZhVXKaCKsKjja0OjMNWojsxHmVq0KrCZXPKvKjh5+srGRx5SqG419qRpO5517qSgkTmdYVZCKQQGxtpaqkKChoXBO9IPvVG5GnVWloZhSpO1QbFOOJX/kOfnG1ioGszBrLjwWpynmFLrwOKiN5iJaSGlqjKqK2Uh5TBTDSp/lWF9XaOBlnMLwKkaEsH46XVCY9MaZ0tIN9EyzYIX7HDf6ecPEMpnH0ufTlPkakVNHmy7sk9YUaDb0NRt3WhFjrTCsGtOcQ0AAACAASURBVI4l1bxdG7LZrSFpSyvs86lG1xhaRWmU9jZ0nX9oVEWpQCwahedHKQ6YQZcJieboeO5kBJGrJSvTRQHQbWUKQZPUf7tMZDhxX3L9bnyFmGJV63/vyi3GTC4KM6eWUdjYdXNVjVk4djk0lapMpW9Q+VwF98VGl1yvumuFRl9rUuJF7krtf2N6V3WAlO0lkPwbrXxVSKwg05ylrkYHGjWPhJXnKnX7PmI27frMa0WMePKSoOtSZwxSu9tGRc8WW6vu+iIj68wkUafV78asFHAOrh0rRWGNDn1O157SXww6qQsAD8vQPgvjrBgZD/PYDL2NTInIkcSo7F2hU5LNHSJ3kmtLoeubESqERtE0wqRShP0YAPdX2dBEpbyyOeuW17UWTWy2lfKxhmDpHOj6XLd7rbLCu6cDzY1lPWcHrWkdpbiFSgDAefE052hwqrTxcJoHlLyNcV06qIs3hVAVCj9ItSisHYMt2GjKbADRHGjxRtBcz+ctLKDd/4K2yOfZNy1sc6Xn4zxvMkh023woeWMMRgXm84CaFR6eJqpfgNgpg0qT4VaLwlw8alE4iYHT7g8ZqWtWCKtDyQqxN15yF9Yg75WiMKehfVbKkFGmgJp0M3hQFVLYZFiC2cIPunmI4YICrMVvhkEFUtTIfB2KQlosUICZOQWyIuj5YTWb18l3KCGaxiqbkyKCWFF8s0KOtCbCajbtmce48jlUbAYWnwvBbN+5tLYvjtN3hTmYzghRiGIcSvUeQqtAXGhMed7azEvXb2cYFEVtV7WdL6vejaMCUFWhdB4vqEDmeqoqZLXFP9fVbCh3ZzjUmd/3/RzL3uCRoir2HkjoPl8aVN01z4ykVp+e056ttu9vd/172n+vgSXHLq65tCXfW3o5XRp1z8qVVi/77ZYDfX/O0PtBI+7aeK60L33s232PUL9BuYbSfqfyq2TM/SqN5c+4fK8NS8M5H388vkWsBl+FG84F+IBRR/zJ+rKxwr72Z9zHCT+e3mIuHk/J485FDDrhlD0+Hp7wmEZMJiIygvgQR4wm4rdvv8CX6w0mMzcWWQB4igNSNfjR4R4zM78u2eKHhwe8XQ/QquDT6bHlABMlPxWNm2HFYBI+P79oiGPPCkt5yRaMlvJynZOHt7F9B9Dy3kl+McmPd+Lcf1/MR4w2NVbYG7/i6AMe40DKuqp4PZ5a7ryZc7Bp1JabbU6uGV1KVbwYloacArSb3bPC9miqoKwfTWecokfkdCKjjXtWWEdMvK/GmdDVwwkxG7yZD3g5zXhcB0yOcg3eryPm4PDxzQkr5yqLjIzmohG7vGDCCguQISBopuRJy1Xh5TRDo+In74j46NMXT4jZ4Nyxwkp+NSliQIyejk+N1ZSQMLnucR5gTMFgc0P5zpznTFhhAWBwuY35/jQhZ71nhXWpoZDv5gHHiVlhGQVsrLBVYY0at9OCkAxOi8dx5FySyeDd/RF+SLiZVizRdu7adWOFFXZSNgQTp2sx/H1ePe388/XCCjuvHjlrDEPEu4cDpsOKJdBzYnijIBeNJ0aZzqtraFZjhdWlGZRidHmfcDoPGKeAp9MIP0RoRmaFoVapinUlw8qYykyxwMPjBGMzxjGS8ZhMU779kKB1aShTyRvZVmFjd10dFAA/0PUx6MYam6JhtAUIq4GxBY/vDqhF4XA3N6MvrK6xwpZChlJaCWHzh7gZ7R0rrNyX9TRiOAY8vZvgpo0Vtlbg8c0RypTm6p5XAzfF9usbmBXWT7EhGtrk5r7/+GaCeRGgdcG6OCLIYrf4OluYF4EM/EXDDgnLkyeW18U8Z4VlxV37jHi2ULY0dCLO+hkrbJxdY4UtaTsPBZRlIwJTtqCcDKEE08YKm55GQAF6TEhPA9SQNxSHEQ9JmZG/HFCHsrm4R01owTEhJzJewa765TwAtwk1aOR1QL1JxCo7cfsKqGLMifugr3tWWCEYAxqLZ2FW2EbQpUHXREWfe3bXqlBd2QjAAKhVN3SnNvd7NNbWqgBilyVW2JoVMeF2Lsd60aiHAvUzg6orqt+QGVUUkEDagKtQs6Lzq25kO9XQ/POhQC2KGExHdlXOVEd5NBfI4kj+1RHLLEAu4iorGCYFSodKDLHgMbKLbR4ru7EzAY/f3HuFsGjPCkuy0z0rrAf0qpCnSgROHo2t1Z4V4ovvrtXpldhmAXItFUNJZYU81uZuqlfVGGvFXVXGXBy5zlddmwuqWRTJzm6Mrj0BUHFivPHaM4B9UigD9V8sscISoVRtxFhEgETrSQeWTeR2NaCENMhxKANfJ32rhObKqsN2nbgeV4ud631VRJAk49YZe1ZYfoTk+ubS27vi8rqW70LaJK7RwvKKirZGxUX2a1lh+dndufgWtbkIm64/bqO5Xu/QPmyGbOdqv7W5HfumrLDAhb2l8NwQ79FKcfmWefUo5qUBrfbjulaE+bUnMbrqjvueWN5df88b3x2na3gz6UNG44cM3Wt1/7z80sv3Oo/lR//Qx/X3/tN/Dg+B3AYlYXso5A4nyrJmxMcZyg1p1aboSn5LQcfE1VQSXteqcIoeg6XE3T3BjbQfsoERBEwXrJ0Rs2bTUMCe7EcMoIOkg5A8f13bS7LN1U/cS/fEPYSwVr5uYaTI6ILEbpNrNkicwJhQFgVrtsCQkAy0IkTLmdKQJ9l5FhZdGbvkdHTshpiybm6g0k4bH6eqiCwfcfGTdCSS81ASKUsuzcAGqiRWljoVaO6ZazJX3e1FfjL+xGiS7gxeq0tDEgVJOIwrtALO7HIn8hZkqS+W8xWKy6XEv5Jhs+U3NIzOigEmeRrF1W9DejZUzVpS2lOXHJrcNUtz24wXDMX99UpVdqcU97bNeBQ3VkE1JIF2nw/xss3+O4DmuijXi4uj5HDMWVM6kpZrE80YEnKknNUudcYupx5f01DGTO5w4ionaUFqVVCdm5hiI5ESZDPCz66Gcj+EVZmYkRmF4uPyqya5FSm/YGmx1CSbsmuroacsQ0pLgpbe4zKpdu/eRgzSm7tj01EklhpkCG45JA2jWLzObW5ykPVNLNbUkDKUZ1FyMdLcto6sz0jRtPjoPqm2NhmZUUFtyI1NmJ37vI4iQDEapV4teJa7sb/PytRdLLi49dF93JJ872K+04Zs9cckLrtfP9KXUuxixy5/UJ0BGzuWaFn7rpCBrTlPYyQjunRIkrKl3YMWg9673sn4ahdTruvm4gdsBiYbv8p2wZ7c5g6R6YmbOvRO2Y38Sfqtsgb6OHLpK2ooXxqit3uBqrq5EQrLtanteRREubFe60qon7gzqrq5GVZsLNmZDXd0x+Sa2F0vSjKwi3Ojc9ucd2hJt2bb9b2MVTcPmS+wGfXftYhrI7AxavPYJMa4GRsFm8sk0DYbmqtk3c63+O3uN765XMqc+1KpvRbLXjuDsq8r8gXwzI2yqp3MdohWDxNVJnjS3Weus81F7Q2rrq1mzPRxmuiaF8Os67KhicDe6pB+1PN2mqF3eXvrJor+kOo/fIMYxd6wv9b+bn1ednat/rXSKTQ7Y66XZy8n7I/v++ku7u/bZZcfGs+HDL5vUL4RyvhtzZA/hdnyB//eLz/X4/SDH9ff/lf+7PNY/p//wS9/bsD3HLFcs8UfPb7G/TwiV4WRY7rm4BCjwWEMLf5oCY7jQigOaRpCy+VnbebYkdxiQ7xPLf/gw9OEYYjkpsXKIkCxMwrA+TxAs2LtHMUk+YEMxpWRGQA7EhxBTpabBbUqhGCbMUCGW2Y3rAslOZvdk+ocITbOZpzPAxlfrNjnG3IHDMHicFixLK7F9oiBsJwpTqaCE7IHIRMiRdS4vDMCUiT59Ep3T3zUkqKDFGdjCtJqod2GrEk8TA4GdkhI0bRYDeMLciClz08RYXZwI9WpScNNEc5lnB+Hpuw244wV5N5QSAshWZLSBNiUaqUrykqIiP2EkKjz/QRlSqujGQ1qipeusAPJXGTVFOCqYDzFBilF8SwSTyPKpfYZeTVtngC2eBUF2CHB+YT5NLRzNSmkKbcYmsi5AVvsiqRSkZiX2QK2wAwZeWZ3VVugbgjZyzOhSpoV0xbbIkpwUU0hU4YQEFHMtM+klBsitSpFoSwGesiEnC0WZUxIs6VULwBdz+0ZX5AXg+wzubWp2lKpoAJwJEdRtutqoIaMuhjkQ6L7VRQpAa42lzTlKU6oJt0U2HoEudSt1H42tc07n/nVV5jsyPDcw8YSowZiiamLIZIiV7a2LPddeRxRQd1WKF0odY0Yxq5u6WlUpZ3/KRHb67l7/QrqlXQjhaoHoDw5qFvOn9gZ4OoFPS+otD7qAJST28Y+8jtnNsiekEikTtvzGeVsgUwImbK1JbXXY6FzVSG7AqwaecrAym0FvVPGqmNDI2o6nxQZMIvZIWeSVL6OtFbqbEjRlvUGVq5FOU8KdaC1UJftGawjSDtaNeqgaDyiFFo2Jhid0GNEXDzVURUYCj2rs6Fx90aQLVCzIXlMBVg04HKbryoKdaL7SIy+FDdG19O9RVaoA2tvgQ0nX2iNC9pYmFhLZCNrvQIKbLwJYVkFMJTNCJC+igKswKZUX7lKRGeG1yajjPTsZUIgD4nGGPUmK0MbA7WCrrMFyBrKE1JrLG9aFDbw5TmOGzqsbUFebEe8pp4bjEw41d4Lq+YxsmWheB3Oht4DSQOmQEV5T/JaU5SXsx4KIcdZbaRR8h6YLeAL1MmijgUIMi8FPSXUVWCzb1+qz4BsNmgev+JnOBrKKVoUqs1A0qhlQwEFHW8GfFHbeQ5zqV2qDpiKyru1ijcJm0HKSH6dDRm1UQGuoiYQ8ZImw1Tyf9LYSyNxg6DRTHCnIt3/ynMioiY6rwroPeMrXSOEPKYCAYTcV3rXqQrK28pGdeX1qKJCGQr0zO97RXOplZ4FVRQghGugOSiee7GbnFVCI2YD0HKsom7pb/o8kCIribGU2McNDSV3UA1CVnsCJCmqEiKsOXUTNdAvCrSxCLLa2tC1I9ri+qUbntq30aOOmwHfne+MccWv9vZd6lfpqrb/FWpD7ds4+nY747n95Y2AZ8ao9HHpWvyhcmm4931flmuG+Yfq/3n5lSrfa8NyshF/+dXn+PnhhtxL3YpUDJ7SgFP0+GR6wjl5jCYyec+Ctwvlrng1nvEUB8RsMNmId+uEF8OCxzDAaoq5vHErzsnhZ/4Wd8OCObmGhALA0QVoVHwxHik2U1UcXcBX8wGvxhmpaDyFYedKWaqCVeRiGrLBp8cnFEZFvc4wHPN4sAFfzDfN5XRyEanoRnQjaMloEwaT4E3GVyPN7eAiztHhs8MjHuOIc3R4Nc54CgPWbHBwEYNJiMXg3TjBsxvpaBPO0TVUUxBVQXK1qs2NVOI512xw4wOegsdgMiK72mlVMUcHb1Nz/xSkcI0Wk484rx7HIWCOFs6URtRyXj2MLnh1mPH2POFmIIKflDVuxxWDTfjKHmBNQcp699foAqNqIyM5rRRn6m1uBDXHITS33aeFSFF+/PIdSlX4E0arJx8R2A10EfIIRvGOQ4BRFWd29RRimsyEL4HRuhfjipAN5uBgDbl59uQ1S7Ske3G7htueXMSX9tgM8ZgNDkNAyho3Q8C7cYThzYSYDMVtMhKmFXAayV31OAQ8juSz5UzGpzdPeAgDnqYB3iYY/oEVRuXe5VVQSPku8XdjI+/JcIbmfF5p08boinlyeDEtuD/TupK4yZQMrM0YbMZ5pPoi3xi315Btcbb0dwkO0xBwXobmwlsKjc+53GIAvUvNM0EQ5rvjjDNvrAjC6m2GsxlPw568x5jSYjuFUGlkQqLz4om8x6XWlmze9IzSr27PsLrg54x2AttGkyA/WlccxhVGVzy6jUBENnpS0jCchuQ4rXjQB7y6O+HBjS3OEABe3s5Yo22I+OASntzQ0CUhXpn90FxxZTMFAD65e8IXIETY+dRIc5QC7g4zvtQ3jZhnXR2mKfCGWUJY7RYPCCHvKYjBYhhju9frQAQqpehGSgIAw0jvlGVwe/KeohmdZdKWZOBchnMJi/e0aZI1Bp7bsjiMY0RY7UZCY/MWLwrg9YszHl0mFJk3DK0ueDS1rTUZ1+3NjHtFG4bTEPGo6dgTK7q1KExTaB4NALCsrj0r4i0w8KZi3+fs6ZmsleI9S1bwTLQyHdaGhCtFJEg9mj2wK3YjduH3wjjG5lGRs8ZhDDg530hzLG+0iov6bAa8uJ3pHby6xl6udWVPCNPIdsJKhE0AmrdLzhqL9fBDwjQEPJkBzuVGinV2AyzLX+aeooFnsqWwOnLt5/t3NkTeQ+7opj2Hq/OwLrVjcRXyHgXLRmkeDIZDaPGrNSsYVzDwWg+2EnmPqZgONB9ZW4fDisf8TbXh58WPCcGwm78vzbtCm4JoHYwrKJk8GArHYlchcpoiaqFn0TnyeBJEXc41ki61J+/J4n0iqHzWGI4BwbJ8koYbEslDXMIB8jJg49z6jBxMG2PNCsazTFfTDEYA0C5vOap581EPeWOV15XaDoTso6LFxpppI+8xlvovUZP7u2E6bNmMqRt5T5ENmEJj21yR6maMJoXS5aKGrttmHHu00MPUWSGCpotx099+PlaSaqzHl+Q9YiDXft10hp4YgpVzM2+GJRv27zEsr6Kjqj+v9p4HwHPUWeG5p0Hl+peI5RWjuXV7afh1m4dXxyiGJa63t6su4+mv78s1o3Z3Xn0Ynr2Y57eJ3/yllV9jI/l77Qr7w999Vf+F//z38Kl/xFIcPl/uoFXFJ/4Rd3bG3z79AEcT8JAGfOKf8MfzK/zGdI9QLH463+JgI166GT9dbvEXj1/hj+dXeO3PmLPD2/WAN8sBBxfwu3ef4+8+fYIXfsagMx7TAK8Tfj7fIleN37r9El+tR2hVcb9O+As3b/FHT69gdMFn0yO+Wo8A9kbl3TDjzi34v95+BqsLxS4mS/GLReO8evzwxQNGQz/Cb9cDji7gYMOOROQ+TMw86vCjFw8AgHcLGcl/+OY1bqcFr8YZf+/+Dh/fnDCYhLfLhDVaWFPwm7fv8BhHOI5fvBuWRlw0mIR3y9SMMAB4yXGQEgvqTca7ecTLacEcXTNItaL4xlPw+PT4hHfLhCVZOJNx4wLu1xEvxxlfnI54OS04R4e7YcGb+YCPDyeco8cXj0f84O4RXzwdcTuuGG3CF09HrKvDZy8fcQoOk0sI7C4bs2kEL6IwvTzMsLrgFDwxhwJ4d54w+YhcFD46njGYhL/1Rz8EAPzmZ2+xJIunecDoI3IhY1aMt1wV3jwcycDkOEhxMQWA88LXVYXT4whjC8YpkMLpEs6Lx3FacV48DmNgcqOMXIjd8/w0oKwG00sinzKGjNx3DwdYl7E8DTjezchZIwYLP8SN0ZHd8m5vZoRkMZ8HHI90P8+LR3gzQh0Sbu5mQi4ZgTembAyZ0cAyoh2jaUp+Y3tcbENzSySlYzwELLNHiRrjTcDyMGC4XZGY8MXYDGvJLTfOjpSh2RGiWxVGZr3UuiCsjkAdRn39mLCePMabFfP9CHeIUKwIr6vjGEsgLBS3qF1pSv76doSaMsZDaEQyeTXAamBfBChdYG0hAphIsnNjakZKmkkBcoeAWjTSapi5USGzMqV1QVocIdVfDUBR8D88kSswgLQajqUEMUBmRShkVlAvw5ZiiNk+e8S7PDq4VyvSlyPUy8AIMT375Y1v6A1shVoM1F1oCl95YsX3RUBmJFOPuSm45mce+bMA7QryyVIqF0tosb03SJ+SAVUXA3Mbke8d1DGhni0w5i1XpyJUFIXjIE+WEL+gUQ+ZEEB2nRQ3THWyUEmhHPOGbNlCitKqN4RvyFBnC7UqlBuKsYQr0I+W4q+OCfrJokx5Q99E4WR0avipQ7wtKAdKf6PPGjoD8VWi2EXFiE0B/FuD8EmCmg3ck0L4KMN/aRBflaYg2nv26OD4sDRVQnQSuecVX2FPlIu2DIT0mEUh3nD8m64Uh+YqzEmj+Ar3wClZWFHMI1CG0pgj7ZNqKU+qxC6aCvfEa8cQ0uKeFOJtbXGKJpBiXhyNKbzKmD43KI7O60gpSFSmlDZ5qCie+stThX0i1zkdgOKon3hbYU8K7gSEF3ROJ4qRS4ctVi6PdDxNFe6RxpkO1KedAVQgvKI4SZIbxS7qDMQjKC6S072kY23oiVloTulQ4e8VigWqJTTJztw2gDQBdqYxukcaj1mpTX9fMX/63ZVO90TtAjROWQsqA/GW+zGAO7PclCBugH+gz3mg8Um8YLHUblUcD8quo306keKxGQygeQ9vKuINp+RxgD1RvZayBXQfMjvA2DPLZqE61dAcUIE8URsSL2kC1TEcV1kcYM8V1agtn2muyIOCWcjwywOtWfdUUYT8NdLx4gF7qi0Hr0obOmiWyn0oKE7CW23XT6pNziQ/BRNoXejMY2Y5t2tEVmqTRx8/KSlXJD6xTy8kyGeTtd7k0UqHNEqffToZ3eRfkR3Pmce0i/ks2BVVKqpWDamk8VLsY39e54piFFSlvybSX51ru7YqOi99FSuo7nW9v9llnZHZy7PZbxJj2bkwPyuXxy4Mx53bdd9397c1pXjMl0nF3zf+D5g1//N/82/+0t1Fp89+XH/7r/wCXGH/w18NV9hfqGGplHoJ4G8C+MugJfWvAvgDAP81gH8AwB8C+JdrrW+VUgrA7wP4ZwCcAfzVWuv/9qH2P/lLH9V//j/7ZxEKGWQAUEBxdRJ/KEaYIHyhECumRm2pPCT1Rs8qCqBdLzGBudtVzlVDg0hEYjHQqChQrV3Lb4tUN9KevhCDrGpssv0YexZZQQbE3bOvK6ynck2fx1PiFzPLQhA8AO2zxHpKvFgfNyblcuwiC6kvcYZSJB50J6uOuKif4+W1QriTCzGs9vGHjSGVFVqps8lTxrs/dik/YEupIciUIJpKEfOoVls8qMwZ2N6JIvN6cS/eJzs53jYAuzldlv4aqdPHiPbpSXqZX34HtpjALb6x7OIpezZWeQ3I+1ra7L9Tm7V97/O1Sl0xEN83xn78PeOqFHUhkj5Osc8hSzLZXLRlXPv5V3bX3uIWt3GqZ31dxsvuGDGx/y3bxZ3xr5k2Eju5j/mr3bMgsZsAGlPpfr7boASZ6lELWUNK1Wdtl6raeHS3djbm2a1tLTHbbIhSBb5Wl8aEKvPvYx/RrZf+nvW/91d//0V2ei/X/pyMs++PmEH35wBsuXar2gRz0Z4SJKOrI7GRLUWBvCMkXlORklyzaiRA7b7r/f3qYynlextfv1Yu6vQxgqpbu61up4C91yWsjzHsj+PiuvbiUYRmyLF+KnKtICT9ePsisYqXiICqO1ftXZ+XcY8K++8ik8vPl+/Ha3O6nG9//rJekxue5fH9NkXyrJLSvZeRuoxdvTa+iudzwZXv6I5f3q/Lc8D+3uHCEBDR6i63LC5EfmHgPEOO3jeu7vOHjIXnMrlYPxfHVC+7vr96XcRS5YNuk1fW/K6f91137fp+jN+kXArlA+VbtQtcX+fvq/dt+/wW8vjWbf8p2vyu5W//+7984+vX3bD8RbvC/j6A/7bW+i8qpTyAA4B/G8B/V2v9G0qpvw7grwP4twD8HoDf4X//GID/iP++tziV8Yl/wq1ZcC4eb+MBTmXc2BVOZfw83MKpjLl4eJ1wSgNe2BlrcXhIAyYTYRRR0N+5GY9xxGASoU1pwDk5HHzAp8Mj3oQjjnYFwMYpKh7TgFIVXvm5McUu2eLOLXgbJniTMZnYGGNbapBsMJqEyUT8bLkFQDk5l+SaYbxmi4/GUzM8l+xgVcZoEgpUa09SiqzZ4gfHh41BVme8WymH5mQj7sOIF8MCrzNOyTfioY8nyvU5mohTHFrOTTGOT8k3V89SFY6O3EjFEHc64xw9JhsRitmMdHbRnZPDrV8bGjuYBMNss0cX8LCOmGxEqpxHMwy48ZQ/9HEdcDfOeIq+5c58CANSNnh9mBvRUm/4J2ZHzUXD6T0rbHNfSxaDJfe2G0c5Bf/ePbHCfnxDjLSErlJqmqEjOypV4cQusBOzwgqSqQDM0cKzy9s5EKvo5CMikxyt0eLQscL2hnMuurG1ToPkywO5ma2UC3ONFochttySpss1KRsPh3FFSAYhWRwGziGXNU7nAY5z0Em+R+nfMconbUpOSPmdErbfwHGzwrKq1EYKlJLG4BOW1WEcIrdPBp5sZpB7Y0JkRldJOyOIsJAticE6+IQ12PbXMHqqQG6rw0AodGB3WmNKi4FeZt9YYQEyeoUZ1g+xGdvi0lorubWKTGIkan4/MCkPu68B9Nnazf3RuoTlTGkappu1GXEpGspTyMaNuAmiAm7Y7r+4uQpJk1IVYXGEBp893JCa2ygAhMU1A0aLy+G4scwKGu2HxGRCFdalFrMdTw7uGKEcbQ5I3CyKQjx7uOMmM+vIHdG6zGy4m4ylTi0cN83tlKi3eFwmONHsUloCkQYJsU5jjFWVXO8MEQCJC19Jilzw2AgTMh3tM8pqoFzZDEjZ6ODce+XBEdOpY9KawGv+kBp6K3IsT247HigesT5ZYCxNM6rzPkUEbGUXNN2MvRoMfRaXtMQorBitmuphNVTnRGk9qhgF0qYo0UkTYgps7n+qQkXTFMlqKvSqUXzZ4tq6GDcVFOpYoN9ZcoVjlLYRzDBDbLUVeqX4U8WIp2bXwKqAMhbooIlxdSS0UycFMFqrE6GyxYEQIVuhA/neFU/IieZ280QMqg3B6VBfJXkKC1/HurhmNK24CjNrZoWl9aWSgo4kj2KpbvGV2FktGU5VEaKYjt9de9WBkOaqABN1M3SwGzsamyqAhiDpFRybiF28W1WMDCpscXrALr6tWuyNNwOYM8lHZKijbvJomytZNdIgHTWK2+5TVYBml9Pi6g5B00mhMFsvFOhzlI0FHgsjiL3cSUbb3AUprAZQEaiSsad08+Q+WtygQnPZFHSxNzAFeZRjPVLYdPa9wAAAIABJREFUl2t7Fa3Pgr2tJ32LDC4MTkE4L8sz4xmdfPiYXEd9q6vjaaXbdLhq8F8zIvU2vn5el2PcCKWez2M313JdZrt6Ms/++7U6bRD7Y4p30xoS+3Ug1zc1ni+7/QUbqt+l9Htgv47lF4ZYKqXuAPzvAH6rdp0opf4AwD9Ra/1cKfVDAP9jrfUfVEr9x/z5v7qs974+fvC7r+tf+S//KczZI1YNpwpi1UjFYC0Wg067406VZgAKg6wYUJJeI3AsoeTDFEPN8udSdbvWmwSrCp7SAM2r3OqMJTt4/gUMxcLrhFJ1y30pqKbEhZaqGpIq56WdSxRTYuGkSKyjVQVLtjvEcLCpIa7iKkqyrQ1hi2wgApS+RZBfOS/suC1XZceK2ep08hMjtK8nRp3UF+NFGF8FEcyV8mjGouHYuOnr5KowWk6uneyuHykyLilrh+I2lJcNQQAN0T16TijPMawyDqmzu17Iibq+5T7JWCVesWJjj21suJ1BTMc39NWaDKMqVo4/lJyRlhVla3Jj8u2R3B6pTYwaGl1bXKlSlV1uNeeFpPMbKqtavR7p6r8D2CG5jbmVUTWtsIt57ZEykY/n1Ct9flLJKwpsvxeyrvv11N/v3iCXsbQNbR6vxA7LfdK6tPhfMWD750HuaUMWhc35AnGUz5dMusKgvMbt3vWortSVcci9EfRVyLBExqbFGSYmDttusnPpWW7LfT+EnlLM5oZUC4LofWo5KWVMjTXZZoRg23gl7rOU5/ORjQNh/xVZ9PF7YlTLr4AxtcUGbmtoL0tB7noUWFBMLQnb+XPpmKxFE5K5eCFd61hnxRC/RHINx81K2heJ8ZM4XsiakLWqCHXe6ZL9uune3ZLDUuoICoyuvhRBkvs1U7n//udajHs5p3Vp8XnCACyflSGj3Upc3yUyr0i2JNParunnBVAcmhKiK3bVbQhtVo0ESFDXWlRrRxBK+f6MKVcx+VjaNjkg6LIUxQg2o8nNa4CN5J6RlwjTFBECCTpamT03XrEQvmkxdcuD2SOWChsbb6/8NssQOwKnLXZO7c9dInn99X2p2FzA5XtH/LKvq7bxdrJoffTj6OfTGTl9eotdne64kBS1tDhdHWGPbay16qId6aP/LOMSpJvl05PGVF6/rd3LtkU22E+7dX3ZzzVLqjt31VC57PObGIffoPT2U2/DXd4a9MO+HMe1fj9goL3XmLx2/df1d9Hudy6XN+07FOn///53fvmo3uEXhFj+H/8/QCz/IoAvAPwnSql/GMD/CuDfAPBZZyz+FMBn/PlHAP64u/4nfOy9huWkA/6R4x/hi3SLWA3uDKGRj3nEUx7wqX/AOQ9wOuEpj7gxC95Gind85U4456EZoG/TAXd2JiNRVRxMwJ2ZcS4ef7K+xGt3wlos1mIxGSLSeeXOWIrDm3jEpAMyNG7MinfxgJfujFIVHtKEQcedYaJVxSkNmLPDD8d7Ml6Lh1MZThEJkFMZb9MBgbcAJx2wFoeVvztNKM9kIgwKJhPxJh4Ri8HRrpizww+GB9ynCac04LU/4TGNWLPFrVsw6IS1WLwJBwxsNE8m4iENKFU3o/zGBliVUdiwPGUPjYqjXZGqwZotjnbFiRHgObumpM/ZYdAJj2nAaCKcKihQfE3AYxpwNIHqmYTHOODWrbgPI0aT8Nqf8fP1BnduwVoMluzwyp8x6ISfLi8wmohQLKzaXHOdzs0wT8XgPo7NLVoUvaMNzYB+SgNS0fjt2y+gVcUfPn0EbxJGExklLjgnYhAUV+ePhhMKFE58PGQLb8iIP9iIhdt+5WekqvEYh7ZpcbBEJHWwoW1I9K7Tr4YzrCp4yyl0ZGPj9XAmNNwv+Go9NvQ6sFysKrA6tzhfZzJe+hlfLse2SfKbh3d4SAN+Pt/iYEMz3mRjAwCsyi3vqje5uXiLQUeEVuQBIK7iMh9vMh7CiJd+xpv1gNEkaFWwZNeM6hu34iGMONiwy58qmxeDoY0D2Yw5JY87P+M+TLjzM87JIxaDVHTLkwpsRFqhGMRimizPyeMUPbeZMVi6t18t9B6I2WCwCRPnh5V0RAUKRxuQqsYpemhVGwIPoJFfiSu9eAwAwE8eX8IZQtgGk9r1silx61ZYnfHlfNPcyr3OSFW3TZhaFV6PJ/zsfIvPDo94ux5aWiOAYp1p7dF8Jxvxbpma8X3jybvifh1bSqM1bZtXPzg+4KenF4hM0EXvE93k9tPTC0LrbcIpcM7Z6HFwEXN0yHVz/5bNniVZ3HjyaBiYCEzyy9ImFrUv/T2GAY7nn3nzwTHBl1EVIRtMTDT2GAYyxotuXgin4HH0Aefo2iaF5RQ/Qjb2Gzf3eLsesPDcb1yAUhVv5gM8eyLIZuLr6YwvTjcYbMKNX/HV+YiPDie8mQ/NsDp6ul48ONZEuYeFNCwxCRqwbWqNLEPxEslFI2XdCMJejEsLe1CKSM8kfRNAxG2lqjYvgAy9yW2/K3Ifz5HWZ0gGrkuLNLmI+3nEp7dPjQRO1opStW2krdE2srTBEiGWeKPEbHBaPQ5DaHMyujTiunN07X7O0VJaKCZqU6piYRIzmc8j5/wF0NJWGU25fh2TrVlTGnlaKRpeZBstjmNo5GWFN0+lvTk4TD7iaRlwGAJW9iQRArg3JyK6+y5lcBHz6mnziknDANrMmXnsuWgMLrYUX7IZI3H7Ml8ZeykK0xCaJ4lsergunGcLO0DbhLqZVszBNc8J2XASrwepL2mnZEOpJ80Sz4sQ7JYiR6FtDsnGVE5bHl/ZRJINIiJtIgb0WhX8ENt4ew4AIToTy6VtJLL3Ss5d2EzRzQW+3+QowmLOhnmfLqkW3tS4sEPEM6KFRXSGaiNFE+IbIfoBNiurgtip+zCH3lDqDXfZHJCNA2ajfebuLW1cWoyqbm7lslkgn/vNg2vkPeWivd4ov2bQXyuXlusHmF/bRoKstbr/3h+7imBeO96P4dqYfl3Kr9NcLsov0rC0AP5RAP96rfV/UUr9PsjttZVaa1WXQUZfU5RSfw3AXwOA8bNb/M0/+sfx0XjCOXncryMUSMm89Qv+h6ffaT+YBxfx9jzhbqL0Hm+ZwMWbjKfV45PjCV+eDzj62Mhz5oVShfzg7hFvzxOsKZRfkllMH+cBpWh8dHsit0dVMa8er/4/9t4kVpctSw/6dhPN35zuNu/e+5qbWS+7yjJYNsIuoWKABUhYCGHLEjAAISNmMEAWYsCIERITJCwzSYQESEgIAUNEJzGwkSirSNJVlZWVldiVzWtve5q/iWY3DNZeO1bEif/cc+97WZm3Krd0dP6I2LH32k1ErPZb6x3OtwvEqHC8bLDrilGsk3MGi6pDaT3+7vnXABVRJTQ35zRi0PC9xvKozdaNtrMoCo/SDpaKEBXajqwOrrNYHRHgS9talKXH9qKGqTwhAm4qVEsCemmaIicmX68bdL2lROktoSxqHbKrYrMvKaVHSthd1R363hA6oQK08XAtAbpwzryQtMXGhIx62KZUJ0oRWh8njm92JcqEJFlVPZp9SSh+nUG/K1EftWg2JWzlYaxHu6mATqM8a+A6k3IbDnkio9NDjFIEufSpCJ9ShwCAayx0+kBWC5qTP/z+B0AE7P09fG8QGgtdO0SvYes+o0xGAD4Bo6iakPBMkVwAdURobHa7UxtLLmW1J815QWkT1ILSZ6h6SPrO1gK1tQTLvk4IAzq5/G0KSguw14hrTy97ThmQgEr446XXPbkLNhpYENBKbAy+f/Eh/CIgrh2lQmBNt0JO85E/rpyygDXrHA/W6iF3WkptECsCa4EH4iJAXxmi36kEux/zB1u1GnHhoRpNEPUBKZ1C+gB2BKiikstVrAONeRGgNwZhkSxGNhLQi6WPpmo1EDD0BcBcWoQ6ULqBiAR5r6E7BX+U3JtTTjvlaO5iAk2JmpLKQ5H7HyLI1TD9Vp2mRPYa5GZYRnzy8hFUALq7Pn8I9V4jLNlPC1BOwewo8bw7CrmecuQeFcuQEQQ/2Wr0px5PXz6AOwpDioCo8NmFJkAXHXMydn8UMhOg96mPYw+z06ntwbrx9PkjdHcCJbDfq+T2RlP70UbRNZ3cK5cBz680/DLA7DRCHYfYMkV9q0DgL8+Ta57uyFWSAGswuEdGAmBRbgBzke6MulPZVS+UEaahpO5uSRaJYCNe7mgMbkm/fYWUvy+5bSrkNCab5+/ALSN8TVaDFwkspjsJ5JIJkIthAC6vFLqzANMqvNwqdKcRu/N76I+TxVADmwRow8ndfY0MgKMC0W53KgGREM3seqk72tPB0prt91R/txtzbaGie9lis9ur7DIZ2KXTROx3zMTRGDZ7AmBBAEKV3BYDEAtg3xDwzpNnZwg2gbQktz9yd6VzoQD6HYG9tAlk5zIBmkRNwDquAXY7+q0doFwCiKkFeE9F7YcC2DW07q4GnAd6BthZA7GjPqIFfIcEiASEBJTiU7vM3/cMsF4C7RbkYmlpbXwHtPQJRKzpt1oD/QZABbie2txtItSdN7eAhB2gV+m92YCEnwgED5gVgA7QFgh7+h8VoDj1yzZCawVbAqqNMFpBJzdR7CKMAlShsgulchGWjaNFeoekRy8YAJdrFAsCaDFWwTYJKEZYLrWPKAp6LmwTYUoCviGgmwEEx1YEhqQ974kBFIaBX0xD36TsLprW2JCuB76kcdr9AoEzsoi9ZRpaTx6H8kjP/Xh/A2lt1VBPgvcwyBBfI8AalVKPSCtkkhmL5C4r3FwlAI0KNJ/aIc+BBLiJSkG7mIGUpiWD5HjRJoP3+MFFmOlh19iRNVOJ6yq1o5BBgPJcMK3smhtprrTDdVdY/sSncwwadEioydbKGaCckZE7zx1fmz5P6rAgKedh5jGcs5jmeZrW5zqvIXT+5HbVfu7lC1lwf8nLz1Ow/AjARzHG307H/wNIsPxcKfVIuMI+Sdc/BvCBuP/9dG5UYozfAfAdALj37XvxmydP8LJbotQev3b8Al0wJGR2C7y3vsiuqJu+wodnz3HV1QhQ+Mbdp9j0FXzUOK33OG8WuL/aovWEXHp/uUVteuxciZfpGlsUThekYb633AIAXjYksALA/dUWF22ND87OEaPCpi9xf73N9LN76b4vyFL28GmOkWQLklGkBb5sa3LZwxD713kDk7SyISqcLfcwKf7woiXr3N3VDo2z+PDuc1x1FRpn8cHZOTZdBR8V7q232VJymTS6PmjcXe+wTxrvZYoLPLnTwCbhXCG5oC6bbDlxSRveOJtdTFnj2CQt8a4tcXq8y+64vTM4XjZoe4uzd3bYdwWMjuicwd2zDZreYr10WJ9d4aqpcOfRFp2zcF7j7vEWhQ642Nco1j67dZIL8OCmyuPbtSVpRJdtdslcnG6SO2bEvqN0AV/51mcAgGebFeyiQ3XX5bQhnBaDBePVKaWIaVmTHhXsirT/9dk2uapGrB4Rgi+Nj65Xd0kxUZ653C4LxkpFLB/0OQ2KTvPZ9xbLky1cSjuySblYeT7ZfVKlce/bEnbVoi4cdm1y/T4JOPkqIffu2iKnZgFwzV2UNd4muXZ6oQEvrc9WNWtIS83xogx+tHq3w7YpYRikJuis+S6tQ9MV2SKhFMUrstKF7+H/XWdR3U9xlvcpr2ym7ySMYiNlvCQALB9RrtquS0qQhCZbGI9dk6zQQcGYIX8s0YLsLso5ZgEM2nYM2vwYKK1A31ms321gdMT55TLncrX3Qo5xBABtAsrSUbqaZsill11hU07SGIH6gx5hV2H1qMF2XyIKS8bq/TZZQ5AtAc2+hEpAP5xHNzYl7F16X3CqCkSF5Yd7+M0CIWjY+y6j9modyaqyrcn1uvDoWoviQY/YDXGW+ZsYFbShlBHRGZjCU15YGxA6C50sETEogPdYQfGiMaWkYOsLACgdoELKhesMYD1M4eFbUtypoKFLl+IlLVTpqB5bKfTgJqsUUHxzD9cWiL0lC0npiNZdCYxAlAC96BC3JaINUJVD2JZQX+8Q9iVZQqKCqcnapjVxRqEnSwznz/VeQxVDGhOlIlRByje2/FDKEQUUlOLFpNhY5uH63mTFH208n2NjY+K4oleIpc/xuSEqlGVKhwNyHTaWx6dgCg/XFLC/3uRY45EXTVICeqfhS0qDEkraN7w3QlBwrYUvPWLl0OxLaEN5jjndjE6WMNprFA9rEp2c7sam3MvdvoC2MY9HG8oDy3mP2d3Vt8IVOc1t6AzMwuVUHTFQvlOJbm1sgGsNbCWUnp6Uic1VhTctlB84hZUUIe1hlfMiK0sgUBxDjKhymgpTUxqO6DQpN5NCEV7BLDxiQELEBthtOLsVOz1mrANglg6+Te+sFIscnB6eByC7BDO9sddD3tqghryinJOV953Ic4oA+l0mJSArHBVIwciKyZ7Gq2o/pAth12Gf6uW8vhikiIKVfwJUixXEwNiCGFSOYwVIaSXdZ5UYey4JfXaUbiRrimksnGYkgxuNJBsM1kxBymDVi4NAKc5HBRzMY6nE/dPCejshhN7kxkrKprGy6JpgFpFzds5KNVwppvlL/2k+1DBmUZX6F/M9J0zGG67NnZ/cNh3yFy7/05fRyK/KTeXnjQr7dwH82zHGHyql/iMAq3TpuQDvuRNj/A+UUv8igH8XhAr7mwD+dozxL9/U/nt/7jT+W//dP4NCeTTJTXTvSxzbPSrt8LQ7gtXk2rcybQbgCVFj60tYFVBoPwLvWZg+tVNkgJp75Rbn/QKVJrcXl570q56Y/5OiybGbLmoc2RYXPeWoY7dLdqNk4B2ryH31absGANTJvdYllVjrLFZFm921GldkV0eJHMpumhyvCSC7Rp63C9S2R53cTFcFubXuXJnd3o6KZtRnZV12z9QqYu+KTC+AQZhJj7hRAY0jV9YpAq5RIbvpMSCRjNkstMc+3cvHDPDTB4NdX2BddNi7AoXxBCbUl9kdTroMsQDEgjgzTLUd8hvSnowEzJMYfx7v0+0KMSqcLvfwQWf3tjnwHha+K+sQgVE8pgQJ4nycVeGyKyi7qDGYD58HAOcNWkeC4qLsMwouu4dRfKXNqVJCIFcxjtNkppX7a51BLVzNdk2JovCoCxLQOM6Uc2AyDRIQiF0MOWaQY16dEBaLRFeMFEPZdJSnkl24ON40RIXemSxUcowYx6XK2Ed+LXH+T87ZyQKgUXE01yxwah2yUNokjwNryD2SBVzvWRClTti1i+MamcFn9zFrQxbg5G/uh3+3TYkYgcWyy3GFLPRJZF0pqEokW6XiCPmWcwo2+xIFC0SpXpvAexRIGAveoEgCFwsn3EfOfyfAf/p9gWLRQ+tIQoTwSvCdRlGTOSB4Q7GHHQmNPq2BZEw4Vx95KOgcz2cs5cpjFFjeY95pup5y4nGcngKyt0NOm5KYZF0M68I5+XQR4DsNXYSsDGCk2Ozmti0obQmD+/SECMLeCNRQutYY6KVD6M0A3rOz5FkAkACUhRwMDDHHrUUQ85qS0WcEVqepHjOlmqw+YICaZKXP3BN7DPAU90KgYKZb9pPuVb0iUB5ug93pVCQAoMpDbS2gCaSHGMYh7x6lhUnt2EheA1GRJSRZj2IVoHpFluUqMcyBGNZQCotNshjDDiBAnCpFJ6tjqAeGO+pxyhbyGiCrTx6Ton7IwplAeTQGkBcPstKmvpQjGnVD88KeCKZVcMs353t0nwBzQFazHHcXkcF7okkWahZ+0noy+A1bx7JlSg8AOFEsq3RDlAAwETRusx9oiQo5XYa0+LDQAYDAcywysBDPG9EoBCNgAIMRVjiifwD5YS8CndYx2JjnSApRBN4Ts0cC0TXEK3K6kGgwEv7yOMJgaYsGGayG2+f5Z8u+FMzy3MXJfRNBLVsz0/FU9roGiqOut5EteQrZEpmFPa7Pz28WSidtRNHPHC1SqOS9ocYW2bn7o8LYmovrY7zmcjo9ntz3RulGpm1MpMU5i+WIlkPtv6qk+r/7n//i4xCXDz6I3/jXvvwYy9/92zePTSn1AYD/BhSCGAF8J8b4n33ZdPy8Bcu/AEo3UgL4RwD+JuiR+u8BPAZZpf+VGOOLlG7k7wD4F0DpRv5mjPF3bmr/7rfvxb/6X/3LOSZsSMWhs/DHAgaDf7CAQfTFLGBNU0ZI4Y0FGJkigwFxJBgNM8cMPgMgp/XgktN0REIRPQRAAyDHHclz3Aafk3RPU2tMwSm4DjPjsi0uEgyG/09TgnDsD3uKsBAif2uFUUqPaeoQCZDCAoHzZrQG1vh8jufDTH7flOqD45Km49MqDu9NMZdG7A9Jt0xXIedX/pbpKab1pqkZ2Po3pNuQ8TADEIqkb7gWr4HJcN9yvWWaDu53mm5kmr7jUAqT6bpJuuYAbab9AhjF7lx75Yw0pfQ/p2CYuXaob0l7RBJkxDOkuJ4a0nRIUuS3Wn7DJO8wume65/RgkSI6kIWkYQ2Rv5hxMkd8j6wbAzIgiiRqlAolpjoSIELM+1xKEwZoGaUbEfdKmm+clOnEjJiSCXeQOeZ4bV7G8Ixi/hSy4HYw7YsQoKbt5RQimdZUR1o1+LYMbILByjOxUowYwggcZG4UBsFuCpgy7W86/1znpjmHOM/XRKqJWevGCDDmwHWeo1nODsM6yb7mypQxnf5mhvrQ3gGgYhJ8ea0OtTV5Nq7RMX2wNciC9aZlCpAj+5XrPOGcyfokpQbk81xtlkHnMnftJiFgsm8mr9FXCwOT9RjdLImX7+2ZsYysW/K5ObSWE/LnaJtDPmVyDglD19pJda4LMDPjlO3fQNetXTJvU+dQf69q9zZjf0WfBwU7vKL9V7T7xuWL3Duh4Q/+kz/TguUjAI9ijN9VSh2BsG/+WozxD75MOn6u6UZijN8DMDfIf3ambgTw77xO+yxwLEwPrwfrXWn6BLpCQDKNLzLoyML2CFDoPKGYWpBgt7IOjbco0ofXJwub0QHHVZNBOwrbw0edATvIItVlARWgFBYMKrIo+hEC5RTwok8pHVhIihgESqMiKhZe0/GcEAWQpalKbfokPHIsKP8urc/Wu4yamMA1OL2ETda5GBVsOucFA1GYkMA4kAVMRgAlBFM6Ry76MVvQeoFOysKnNYGE0CRckgWPXIWldazntUoKAgZxkPkxpSAZk7BP9CbXqTRedkll2tj9eNeUgxtgVHDJMsNWLFl8so4x4IAUKIZ8i0Df2xHi5uD2OgZFYCtZjCoDGzAaJoDs4kkolOTixe6DLIhKd0KiV2V3MFpPoG2LZNGL8F5l9zoA2Z1LChpjoVglV02T3S35G8b5Fvl331kY64c6SZhDVKM8l3le9MAl8D1wpDrXiqxm2pC1jF2GeU15Z8p8nYwQ65Kr5cgFMZBbmk7uX0qMM0YkpEmV2hyEMD7OORhnmOK+SSlPypDnLafPSAwP9Z8sbjZkITjPshCWg1fQNsA3KaWGivkD6zuThW+lKS2HskPMJru+aRtGrnU8z74xUCWp+keucxFkSStDFtoYRVOZJGzpYc8xQmcWbJIFcgCTEAIcM9Xs/sdWOSXmUuaX1GkcAeSWx4wqC3omufOxZRBq6I/Hui/GFsCg6T+7+ak4MJjOUNxsIBAQ2AjVGrI2IbXZsZBAY45sfRQM7TVAC69TugeVySQLI6URUVMXR80CSKrux/fl4jFwgApkKSuGuRtZQryi8eyTQoVd/gRKJ1n/knXT0H8VkYWlqJHjq5VLVs1A11XEgAQaVY555TQiuc+IbHWkGNSBRmmN5PQnCANIS1RiLti9UCG79qmgBusbx81ZEXvGU+MpRu9NC6fPAGiO85IEpLQmaqA1Xczy+CStBtNK6ydoj0ObcxZLbtN0KlsJaQ5TX8Ltc+QeKddcDelGeA6zZQ/IdaRrqPJTd1yVUn/wAKn/bI2F6F+LuUvPy2ic6f5rgjHPs3jGRpay4fNyfa3EXE3PzSlXhrkarNC5zi1i/GaVBFHcN9PnnDUw0zsn/MaZ+29ob3QPXz8gqL1SqDxQDgqNo4/bK2ic0vqKPqnfW1T6ZS2/ANITcOqn6feVUuoHIJDUt0ew/HkXHzUaX+RcjmyZ9FFn9012z2Rkxl1CiCSXyCEf5KYbYtoAEvTYgvXcLfM+lxYyo8g6uQ1lZgiNDhktDyCkxJDOS+tT44bUIADQTwRPrSJab9AC2SWyiypbCbPrHL+8VcS2LbNVK0YS3HpPMXDWBLS9zYIQF47lYxrbJJzzeI1oDwB23kCBBLQuQe0bFUepN/L6JOGha+yoTxYqOjcIGRTLaEbpBrQOaLoCWlOcI8f9cSyf1jHH6bEVcGo15DjI61ZG6nPfE7BSmeKJ9m2R++Y8jc6NHxMWbjgej10mqW3AJUZKukqmq+m6mdw/3GsMvfVDGIQd72xGtNMmwHuV64egRl8CpSi+CCCBg90goSKKwlOcFMc0iq9gEIJ/DANCJe+1/D1UMcf65W8lC5WKrKmc5zJfj4oshIqE7JDaZmE/Ci+CYRwp52G6L3iTLazSGsrPAwmUvL/Ss2RJYB3mINVL8VtAerfzM6SRXTMzo6HiYAlUyLF2PE9yzkyZYtpEPI1ScThO51iojVJgyHM5HGgbhnyPQWUrK1TM4FMqDUIXfrDMJQEZEUOsHisNmP9buCRQputprqAiUBBt2eLqdY714v+DCy+yYIM4pHZQRRDCIy9M+m+iSAGBTDNfGyxo6ZjdKnldkivdSOBk2icpG+LSDW6qKrWXBS3m7BOzX/shpqwgwTOfg+hbDR4PIxfYVCdaQR8w5IyUzFJUiKz0sXFybaCJrofRfbmwEMntlRMzjgQaSfMUl8OYeRRRcjgRWTiN5QznE9L1KeM3oV9FhZCEklAlJU4SUKIOg5Aj39dRtssPdhyEDqaNL1e81jxfYbyfFICgEGreP7QXvRExb29QsmCG68KMZLBHlrpUQj2hT655jWE8c+TNCCB+EUdT9UpLZFZODe9+r4Y9MRbQGLAlivYFX6HNAAAgAElEQVTjzS6KiX6/kH2OxzVH3+yQJ3Mnx32tz+lcHhBi5udGifHdICRx23OE33TPTLmxj2lft2n/hjHP9XvjON9wTLL9G8vrjEu0O0/zmz/Hf0rLPaWU9PT8TsKluVaUUl8F8BcB/Pbc9S9S3mrB0gWNl+0SrbPZtZTP+2TVkvDsHOMGkKAGDDn/ZG5Buk5xZiGmuDlhEeN+ZG5JFpxk7FeEtCrFTAf/D0GhsH5UD6Jt58cCi7RScRucr44FIeleypawgdkf5ycEBiGH7hsEGv6thXUCQD7HRd43dZekcwbBa2gzWM5i0Dn/HP9XworFQomsE0NSAqTxsuVLujSycEMTRm+iIJhxfjNJq28GT0i0MdptFkSFAMGd8PiDEAKyBUe4+HH+uZwzjt0a0//gZ3IxaqI1Cz7JMpKTuKf4Na7P11mIAUAChha55kDXQxJURlDtsg1Z5Ftcck/SrS99GLKVUwHRU1Jthnfn+3N8nB6uDfMiBS+mC9kVNtdN+yC7Ckp3PNFXZrTSeCEsB9na5sWYlbg/TMYa1VibLmQRZmg5sX1IcYAQiobIVjzuB0gIqHFUb85FMyZk3mjp/2h5skAlaHBC+GIrW4pV47qZ9ypUBtrI4Bf8ECWk3DzOkKxHfhjrqLB1jduKQFRq2Ce5oaF+zGONuMad50FiWJvJMzj6nRlNNTkGgAQaItdapTVRok0A0CrH4VGCe40Y4mCNQVpPqb0KyC6bQPrNzxa/V/QQ38eWTtq/9DtbRPk5DoLeCLIIxQmt6f4cIxmRrGAH5inFdMYyXGt/ZMkIGPa5nrQBDIJ3Gnu+V4EsqLzV0/yqgCzgZkseW5t4b0JsAyXiK7n9GbfTHDeY5lLxnsuWOeQ2olb5Pz0DKlsH36ikfYKJIjNbKdMzoEbCznWrrnw2VMSAtDw1rwnBZ1TScz5a80kMHd2I6+8wSZsEZhHbW24hJU6o6UX5W/Ytn7lkcZzG+I1olGPkZ2Suqpy/me/UVLi/1q7sU/SjeHzx+hJwW7Pt8LF6xfGBsUxpfKVwK9uf+3+LMtf+aDrn2rlF268llL8Gvdeep7e9xFus8ZuVZ7dx81VKrQH8jwD+vRjj5ZdNxFstWK5ti9+8+2NsfDWyJLbBwgWTQWC0CnAJiKdNGNhkbRxUuvJ+AsgZ3n6Ut8/nuE2ub3WARkSTQGmGGM90TcUMujPXT4gapXHp95i5oryHA+gPgAz8I+kEAA2il3MRsuW21D7nIJzGckqhWFpbWbCW55hugKzEGvGaQMTnmV62FMvf3A7Pz1zSe7YSazWgnk4FfnbnlYLsFGSGhayMdiqESdkeuxlzrr/GEaqinONprKtK69OLNuQ1+T2lORtbuflYWnm5L3bNlWvlg4ZNeRHl/KhJG8Oa6gz6Q4ivhJjLeSdlzHAWsMVacpwtzVUcuULL2NuB9uE8o/NyfC2XcRymcINV5G6d5y8rOQb31Wk8rpyvQzHIEeQGHcIAUqREH55dbsVaSY8FuX5sYVVirYf41oGuwiS0S2HhljTLcwBG5+didaXLtPfjNqwJo7hqak/OVcznMuKsUEpZ67MiagAN4n4pJ56kQSqawkSwZEWMvC49D2j9r9fnscl44usx2MM4uJ3p2OYUPMytDW7jatR3zl2Hoa7SMSu5yO18UHwBiW9PbtvshixdwnMd8a5TGPY8+P0gOck4eJBIpaNA+R8UeWKt+Xye1qjyGlyPQUVWZJlkmc7rwcx0qsfoptIlXirLWCkkFWNUCTmeV7rQB9EOK+qy4kTEdOZrfI/icAqMFU95vBgUP8wIK4wVRKzUkcIUqI73M23etihgFI8qz7Ngyf1PlXZSWSWs3Fn4x4RWWeYUNYzYyuduin2Vc8bHTLekY05gweScnA6pWJHKFN7PanC3vhafmhsR/Wd6x4JifmwCkK3b4xuEkmFKoxjDdMx8ihUfE3l1VOYEylcVMWfTKT1Y5gTgV9W/hYD5SsFv7t4b7rm1cHRIefhF2wXeXmHzF0S3UqoACZX/bYzx54KR+1YLlltf4XdePAYA9CmBMxcG0eHCCbeZoeZUD8zgM1CMZKyZ8WMUSwAjxoeBZaQ7pxR0IkgAkcx/FPQAGCFnSkZzFC+mIqYWzSgYiRy/mT/gyC6aMomxMYO1bwr4AgyM2ijOa4aJnIK9SEZQtivpmrYjGadD/6VlU4KuMPM2BbbJjJnoRwsap2sgrz/JyJnCysgf4clbjq2ADKoyAiIRX42MeHmA+VKCyWKisjujuKYUuSPmL55knrge3z+xHvJvROQ2smtkHIQYblueuwYkwww708FjVYKZYWZuhEqJYQ7nGL3pB3G0QBhbUeQ98lgyenyOLZVTa2QU57hMmP1cj/uS980xhrJ92faUi+A55L5k/3ItmUnNcYrjdR7FLr5ByTFski6+FibXgOtrIOmcoBqy1Y2tafm/tOTF68wnTcnkvqmrKUR/c/RM1sy7wQKZr/MzJOtGMe6gcq7IENgKS2swcpVOdE1fEX46rUw/dzlh3sKE0Y5pPLmKtJID2bIyBWsa3lnUn3jt0Pg0cr5UNeG+uU2l4ojBzow695EsTorfE6IPudWj3AtxfC4fS9keQ3/5PL+3xJTneZ4+/zzGyV6k+L8xgUrmFXyDkuPwgLH1Tawz95OnmNcuWTqneyamZ+ZaX3LuphKTiuO0GwrXc/1N32Fz71JeY7ZQR3GvGsY4K2zNvQPksSRbibmLw7VrdM69d3F9vvI5cf2QQDmZtnGfc+OJc+sj6JvQNtff7YWuW9aZvDdep9yKrhvanl134OA83Lbdm8obxU++YV9/VkoCSf0vAfwgxvif/rz6easFyzO7w7/08HfRR4M+Guw85aaqNQHs9JHSYkyRX/toRhbCwYI4nOtTdH6IGpXu83Gh/OiaViH/b8OQy5HqhlGaERm/yWA8ZA0buCY3sfxJ7feI1nS9F/dOEVP3Ke0IWzM5Zch07NO+p2WaRoTniq240noqLUlTq+RgDY6j81N025wH0xcotcvzw3PdBYMyWYGnFl05PtnfaDyp79F6J6UEg/1ISymnHeF7yMqpYVQYWzYn1t4gvvCSRrbiWrE/eI4Z6EkJuvk892EE7YfAnACMkItjVLkNGe/L57gviWI8tQhOnxH+1kmrrlFx9hs4PZbrjMl5WaSll0GXps8EgFklA1tm5XgPWTine2+Kssx9TF3aqS4yCjIwuNmz1Xdq3WUF0tRSNUVvZTAttvBKGiSCshHPmVRsyfmbzqlM1QJM1iYMKXAy3xjHaMqY1OfxHEKAlkWCXHEZu8/LuoMrPxfvh/CCOSunLENd5PYBZEup7Mt7ymnKFk6dLJjaDPOaLadhUHZNkY5lvtG8tpothQpSSabm1gfANUXZ5HlRYq6yMi4gp4zJc85tRCSL7Lgvds3PlngR1pCVgYKL5+sUdyusnxOONQM/CaVbRnpmpciU25XSAlv8pnWnCiyeDL7GggYLT4zqK0Fhghria9+kcHtcpq66+aGR0g+GsebfgnY+njsn256U7LqdrkvgKOk+LZU67Jp9TShgoXAiWF5D75Ulrd0o5lTNt59Tl0wVcvH6Vpg0n+dyEFKH/SKFy2tlKgSK8SlZB7g+dj4dDwjtk3oH2xLldgLezCRM2pCPjPx/sByi64DAeq3NVwlsr7h+axpHtMzzNW9abi3o/wmVXxA9vwXg3wDwe0qp76Vz/2GM8X/+Mjt5qwXLbSjxvavH2f1VClxTt1UAyR02wEUDq3zO3ziuo68xvIeYXWDsGjp1Bw2TB2NOmJ1j2OeYwSnjdEiYmyvT2FDg5u/XlF4pLE/L1IVS/j4kwMq2p2ObYw6He8aumJJxD5MBTNuYtntTX3Pnp8LHTelNDn2D5/qd6+sQU35oXm4qIwuwaE+eO0THofk5ZK2e63toa/78TW3fVOdQGTHk048l81m3faHfcr4P0XPTXj7Ylrwfr/6eX2PKb9PHHMOjbpiXA8zH65ab9tqrbz50XnArB/bgnEdEF+fH3E/bmb4oJS2veoHedt7m5uI11xTArAXvEFN9iIaY+r1pK/CPqPR1IWRSL4qDPG2yL1FGrpM3jf3QXM/QMVs/qozu+4XKTXtPCFujqocY/Lm1mas7qUfLpWYY9+n41GGGXdI+I9CM2p77wCk1oV2Nrx8oIypuIRSpV/Vxi/fTtU/LjND2OjS9sv3XuHfS0o33qJn/ry2ovMb6vLLt28z9bdp5zTa/lHv+DJUY49/Dly2xz5S3WrBsvcWPN3cyeA8Xn+LIpGVn0PRzrF7MMWHSDZVBgKQ23yUwmSjaAQYLIcf7AchWpWmMGBfJiIegcsqKqRspu69i0mecjEcKCn7irsta+SHOamDs8lz5sYvtNC5KJpGXcVRzx9PCuQsziqh0GU3npaurFNqUGEd2s4Ww8ght//S7K+dgGs+Ur0kaI3JaDp+g/68JQvJjrsV5YMS05ZyDEIJMGF8PfqzJlyXHI0m3TTGwDMgzJ73yLRMX2KxV5iTx7FYrr09KdollRpPHJGN01ECrpHt2bEyDGvbFtX4l0yItFHMxS1MGcspQRwxupHKOZBwTXnGerRzSSjIXv8QWFgbUkVahOYZxglw6YvrluKZuxXHSBlIb0r1N9guIecPAHUY1zs8o9wKPW16LYg3eRODhcoAHl9fGFh/MWjeuxaLNMBM57oufxzQFzF+zqy7XBTCy5GTAHSPAeyJSyoyJADRlRqf0KWDW6iPHOp2L6QtNXpu7T54n4sZCjxjPiIZR2wOIDFu25DUAYzdQuRXUdRfa0XzIvc1bS7pzi/HooOZlHTnm6W/ueHpdxiXmBgHMoJjfuihpoTskmBwQDKb0zc3ToTIzDLnMwgt3fM/0Azk9N6o//+4SnwI6nvl94zpF3NyvHJfcw4euHxrXtM9DbQPze3X6TN1EnyyH7p37Tn8JZc4COD2+Vm477695323H9oUsdF/y/P1SlD+NY0rlrRYslQIK7aFtRJeAeHpvUFs3AMKkunMucAYQ7nIpJ+FksbUClBpyO1K/ceTyxaAdUxfGAOT0CtnKxf2rCK2RBSiwkCTpyy5YAdIKc8i6J93FYpIEKI9izELiyGUrDoItj0upsRvX8JuvI9elPmO+JteFx69UBPQwf9KtLCpKHaITeq1O7etEJMXwxOF/JjzlPsTAw6k0p6O4ISF4A+ndz3PHdRi5lefAzADwTL5v+ducv+wYMXHTlCdqRCRGoBizhedz7uMYx/dFzCguZHycoI3jO69t8pm+lGAAgUkaF8mwqjiKReU5mc7ByE0uu27FkRA+azHjtsU9maM6VJh2Xhs+N60jyxwjz+fkfE3jJ+X5MKmTn/cpBzBHL9eT/avELc6MQwqlk/01KpJe8e6hDXyApjC5Boj41Bvm/VXlplvVzO853v/aw4nZNcvWsKkCYXpdNi0EHRYc1ERJIK1Q16xqUyYaM7+n9eX1OSZ07pkYxXOLvcbHcWat+B7hUnit8BZMSozR2AXjrAJGqKuSePkqzMWL33K8N5RrbfCzFCY0yfpTSzMTdE3JhZv34qupm/Q7XyU/ZrLLm9bW4/Ylr9X49LVXz03337YfUaSAOXq1XLvv1RN8fY1x8Js3W39Ko1zXG8Y3R//rMPm3ogOJlsn6qNuuz6G2ZVuHjqeb7hZz8bp75YvS/mXW/0IC6y+wvK1036a81YLlnWKLv/HwuynG0sJDoY8GtXI57hKgeMlCeTShQKE82mhzjFwfDIok+DShgEG4Focp4yr5NyHMpoTo6Z42WIphQ4AHxRQWyuffAEbxghQ7Z3L8ZcExkOKVwbF5fdQwKl6L12S0W4oB9LmPEUpsamMu3lPGgXIfAEYxoFzCjHvxNFaNi4whtCqM3JS1iqNYUlmkG7OM0+S+pqiyHB/I53lOebxW+ZHrqhwDrwNA8ZwAUJs+0W/G9QStc/GNUxq0IrCoGFVGsg1R5bjH3hsUCaVVFs7BWiRFhwSYkvMq13O6tiYpImQsqos69ynXS67fnKvvFEH2kOuzRO3tvbkWw8pxjjK+dS6ukemYrrlcb0m/rDPlFzkm04h7YhqT9Ejgtnitpm7afE7e5ye0KBVzKqPSDms3rUftUF+l9Te62UsvCzMB1WLU3RgpXy3X4cJ9WBNmEXpd2od8LF3sQ1KWSRqm45m65Udcjx0fMdSTdYpi/ZUalEbTWE727DBCIeVT7to5D4qp50TfG1g77Hke6zRmNUYF73VWtDmnURQefW9SbtlEX1CYunjL+VGKvR6k0D6gyMp7mHaf9k1uZ7I3o3hHqJn9jzTHeS4wgJtFcU2iC4+AyaLKXhVTILKYBVbk+3KKJmkJB4a2xO+RlT/3NVZIDZ4fauhjMsfDXIg60nMj0TsLIpZTgLBwDfIueNPC7cqxs4ApzYaHPAiAsQJg7viWZRpjOYsoKwSMHGN5QAgfWfHlOA4JbFJ44f2kEl1CYIkzwx4x1nHSFo8nXct15TWxb+b6OVimguhEQJ5VAswdT8orBc6byi2EwBv7AEbP823LrWl+hXB6Y3lDJcZN/b22YP66tPyqfKHyVguWl26B//3Fb8CqABc1Nn0FrSJq00OriMuuTmk7DErjsXcFFraHCxp7V4yY7so4tN5mYah1Fn0gQWtdtmhckQUEZnDZBXdZ9PAJnMZHhdo67PoCWlGKh96bsUEgKhSprW1XQilKrcEuvADl16zLPjOlDLbB4CjMgHFqA+c1qsKluhqFCdi2JawOKKxH01tUliyrvTOZ8VuU/ahPa0JiIClthKwLIIOJSKZdCkkji2tiMIuEuMvvThY+WMBiRtKK8z5o9M6gsB69MzCG5qvrLUJQqKs+Cz1T92LpVsxMJbv4KkWpFNhV2VoPBWC7I+CnxaJDCAou9ckuxTK3Z9+nlDWJ4ZQpJbwfQElcT8jDxvoRE2msH+X2lPGOrjeIQcEWiRlWEcZE9D3Nk3capiC4wewynIBBQlRAVLCFQ/CakIDz+DV8a6BtgLEpVQUzaWpgWEdWaiC/wfmbF5wm5NyAnODe2EDMdAR0EeBbA1OGIR2BYFijV9A2IjiVXXMZcARK5O9MtGkbEXoNXQSEPgGGKKKZXYoBpLyUxEAyMmboDFmgORF96h9eQZVp7hODmueCGUYFyvcJ0P0RBFgif5uhXaUjYmOIyVm6/AGLTkMVYTgOCuhTbFopOM80lyOmutNQtUfcW6DyY6tuawaVe8p3iTIMX9w+CSNFoHyZKo5c7dTeINZ+cHuV+fd6TddUumYj0GkCPHF6DHyiMOTYNHHoi12DveAAWD7yamDup0i60t02CQfKK8RiGJtK6xJtoFyTnBc0qrH7KQC90+jLmPMHKkf7JFYhW+Q496ZuNbpFgHIKqlfo6wDdaLgyZoZbd5rICDSX0Yp5TYKF6mktGU1WOyCUMTPbvBbKKTgdoTlnKG8Fg4xOC0XXM6/I6LIa0N0wt0EDulcIBSWw9wbg3IGB+6oC7I7oj+KZQ1TQyQIZTerPROi0rsojI4aGgq5rB4QC2XqJAERLv6Oi+grpf8oZSfk403EEQinmUSfHlgiE1A5b5IJFfiZzfQPoHmme0zWPPJehIMTUUEaYViHYIZeoaRXc8s05TN1TuwCNRSWhSkXAlxHapfH0g+sx6zNpzWielVOZ/qgjTIr7zPk55dzz2ouNEjVgmrQOaU5Un+4XQkrO+Qmas2CRaeR5435HiK0JaCej2/JaKsHYy7UD7V2A2ud62qc9rVO/rLMVj22ur4c9JPvhdednLFvL0zneL3zfSBiFaFdY2WU/47ynom1Rcv18Yr5Oplv2MzPmuTamArI8Nwwm3cehAvKY410PKReyEgCzRcVIMbshjqXs6TgndM4KedNztxCcua1bC45vq6B4wxr8aShvtWDZeoOPrk6x7y2cN6gKBx8Uem/gHB2zhaRzBqX1aHsaclU49N7kNBykmR6QAq31lCokKjx5eYSidCnvG7KGvChIUHt+vqbcXirCmICuLVBWPWJU6DubmXsASVtOAkIIGlXdAQCcS6lQUjtaRzy/WGVtNQsjUwsHC07GBGy2Nb13k6CzWHTYtwWutjXK0mG/q8iykIRHANhc1Zmx1zoJCEBGC9QmZsEFSDGZWrj7Bg1tPBpfZrff/O7yCtpEbHpNDL7QdmsbsE0CA2uexwJEhC08mk0FU3q0npAITemhTcDVxQLKjLXrHFcorQjttiQhU6S/0MUg9Oy7GggK5XFL8/FySeMrAvrGDgKG+BqYKikkGp2EIZ0FAl0EuFRfF7Sf/K4EkgCjbYBrKigbEHfF+OWd7rGFR9/YLEj0XkEVycpXBPS7khhskwSxYEmgSm20uxrQEbr06DclraeNKGoH7zT6bUGJtRm9NKjrcYRMkzwGoIoA35BQw3PqtgUJTzrC7yx05eH3BjAREantZOFQNsDvLGADYq8HwYTnga1DaU/6VgFFgN9aqDIgtGagxwyCKGz6EPY6p2FQtUd0GrEphjZNhCpCEgLpeYSO9KdANKX9hIIEjriztP4mDr81EHsSJGEDYmugFo5QM7eFeEAD4tYOH3oVgTIMgigXFhikBbtM9y480NK7J1uOap+EaZCgVkSgMcM8FmkS9mYQ4Do1MKfHPbCzA3NiQ1ojAEc9sLd5jtXeIJYhCZdxEIwBqJiYZ52EzyIOwmivRsKjYsa5CEM7bEli4ZQFXQ0ScE0kIbdLsc9OEy0x/S4C4BRU0JlhR8AQK3mnR2x1Tj0Ta9qnqjEkFMahbjh2UDs6H2sPtTcIRw6qNdk9NCySJZfn0akhHlFRW2FJnDgJkgF+EaE6jVAlZUYSrEMSbv2aOXshTDuVnwW/GhQTSrzv/PGQz0J5BXfkqR9NQktk4ScohEWAajX6e71QJCAJW2EkxLslCde+SMoFtvoFBdUpxGWg/dVpWq/0bICfZ4D615HW8CiNu6c+oyXuWjW0N6Mioc8Zeq5UpxCTIJgF9TTmmCyNqldwR0k5EIb0MLFI69ApxCJCNxr9iU/zCSAAXRmgd2OPlNcp7iRAN0mAXWKw3GpAtwpulYSJItCcRmTrZTzlvasQk5JKBQXlgf4s5P04KBJS21FlATDPcQDa+wG6S99snxQYTo2FsyTQAYNiIFiqr5JCICoSOjlOFirmVC3RJCEpqOGZCUPbNBY6z8oIUqSkMRsSSJUnxYdm4CQ1jCUrfrwaYnj98NqMoi4CCatcbyRMpuvD+3Zoi1Fp87PL93D99C6UipRcWHBPyg8uUyGIBfPMLrDgKh5xWjt1swTFz3XEYJGemn0FT8L18rtInOdb8z6UcynGlyclAmM3gevkTYX3WwmCryNoToXjAwLur8ovZ3mrBcvTco9//fFv4/e376OPGh8unqEJBX66v4On7RrfPv4M5/0C98sNfry7i2+snuCHmwfQKuLX15/hZ80Z9r7A/XKDH149wIfrZ/hkf4LaOLy3OMej8hwv3Qp/7+nX8K2TJzjvFti5EsflHi4YfLB8CQD4vfN3ca/ewAWDR/UFvn/xCH/u5FP00eCn2zs4q3ajVB+VcXjSrHHRLfCX7vwEHhof709xXDRY6A6VdrhXbPC9q/excyQY3K22uOprnHeLUaqNB/UVCu3xoLzE968eofMWDxaXeNqs8Vfu/hF+tH8Hn+xP8BdOPsIfbh7isq/x/vIcp3aHXSjx/fNHOKt2aHyBB/UVfrY9RYwKC9tj7wq8u7rAynTYe7LAfrw7wXHZ4KTYowsWV32F95fn+GR/grNyh6u+zi65L9oV7tcb/MOLe7i32GBdtNi5ElddjfuLDT7dHePx+iU+3R1jaTs826/xcHWJjzcnWBcdvn36GX7v5bv4+vFTfLo/wbYv8fXjp3hYXeL/fvZrOKn22PQVSu3RBYOl7bC0HQoVUBmHPhj8eHMHvTc4rprsmvru6gLP2xVK7fDp9hhNb/E3vvIPAAD/y6e/gXXZ4r3lBZ42a9S2x2fb42xlNTrg68fPoFXAz7ZnsDrgsq2xLDr4oPHuiu6zKuAbR0+w9RV+uj3D0na47Gq8t7zAT7dneLC4wuf7IxgV0LgClXEwOuCbx09wZnf4BxfvAQC6YHHeLPCN06d42S7xtfVT/P75u6isw2m5w3m3xLYvsbA9atOj1B5/fHkHx2WLx+uX+NHFfQDAnXqLf/rOP8T/t38HPzh/gLNqh3VBwjSviVIR66LFpq/QeovKOCxth50r0QcDq0Ie37pocWRbtMHgZ5szPFpeYmF6/HR7hm+ffIYfXDzEWbXL7W+6CrXt8Wh5iY+2p3iwuMKzZgWrA57vl6itQ+cNzuo9AOC4aOCixmfbYzw+eokfX97Bt06f4OPdCRpXoE11N10FHxXuL7aobY/zdoFdT8/Mn7/zCT7ZH+PT7TG0ilgXHU6qPe5VW/zw4h34oNF6g+OyxapooVXE80STCxqPlpcIUPjo6hRWB9ypt3iyO4LVAcdVg8u2Rh807i52eLFf4p96549RaYf/45NvwSZ3xjuLHT7bHOU0L7V1+PDoOSrt8P3zh9kddFV06L3BZVdlD4hvnX6O7z75AL/54Cf4/ZePsOuL7KL6F+9+jPN+gcZbbPoK7y0v8AcvH6AyHq03+ODoHFpF/KPzu3hntcltV+n+v/Lgj/B/fv5NbLsS91cb3Kl26d3i8BvHn+H/evohOm9wd7HDx5fH+NrZc/zs6hSPVpd4sjvK3gk+aJzVe1TW4eluhQ+OzvGyXeJuvcVHV6d4Z3mFnSvho8ZVW0GpiEerS5yUDf748i7WZYvW2+whclI12HQVVkWHl80C764vcVbt8MeXd2F0wK4v8JXjlwhR4aeXZ/jqyQt8sjlBl57NddGhCwb7voDVAf/8oz/EDzYP8dn2GIXxeLx6ico4/L/P3sNJ1QAArroKRkX8+Tuf4O8/eYx7yy2+fvQUf//JV/CX3/kJvvf8fWI+mD0AACAASURBVPhICsuvnz5DgMJJsYdRER/tTnHeLLAsOpTG47Kt8f76HC5qPG9WKLTHe8sL/OjiPo6rBiEqbPsS+77Aw9UVztsFfuPsM/TB5PCAz5sjvGwWKA255b6/PkfjLVwwuOorFNqjcQW+dvwMANAGi/NugW8dfY4fXj1AbXo82R3huGpgVcBVX+Hx+iW+//wh/rl3f4hzt8Qnu5P8bNe2x0mxx/N2hWf7Nb52/AwfbU/xteNn0Ii4X17heb/C3pf44fk7eH99jn/s6BP8zvljnJV73Cm3OLF7/NH2HdwvN9iHMr/znuyO8GvHzwEAH21PsbIdvrp+DoOA7774APcXG1gVcN4t8GBxhYXp8aPL+3iwuMJ5t8Bx0eCjzSk4L/VZvYdWEU+2a3zz7AkuuwWu+gqNs7i72OFx+ib/eHsHj1cv8QcvH+Jbp5/j490pTss9dq7AP3H6M/xvn377jfmOrx4/x4/O7yNGhTuLHZaWlMNHRYs/Or+Pu4sdLtsaj49e4LxbovMGu75EiArfOn2CrS/xdL/Go+Uldq7AzpXY9SW+cfI0r6VWEVZ5rIs2h2m8bJfQiNk1f9eX+Ev3foIfXj2AVQEXXY2vHr3A83aFbV+i0KQcL7XDy3aZ99Knu2PcrbfY9BX2rsD9xQZaRXy2PcauL1AaCh85Ts8ifzs3XYWHq0s0vsA+fbP4u/3O8gouaDzZHSEC+MrRS2xchRAVTss9njUrnDcLvH90jp9eno3ehz5o3Fts8rduYXu4qLHri5xbfFn0MDqg0B5XXYWrpkJduOwVxvc0zqJzFkaH7FEVAezaElXhsO8KlMlzi1NxNV0BawL2XYFF2aPpLQrjc2gBe3kd1S22XSHCEYZwDfZiY+MGp4rSCqisw7alb5LWAVoBnTOjtGvA4IWmdcgeXt7rTC/3Z2UoTfL46r1BVfTonEVpHULyoONQA/Yea9sC1vpRqicuOcVSYOBHAqEkDy+qzx4zbFxxjsISbkqdlUMmwjg9kvSSkvmzOUxAyrYDTSrfw6EHc/m2b0Q5/2Upv+z0fYGi4i/97B8uJ7/+IP7Wd/5V7F0xQoHtvYELGrV1aL1BoUN2a+1EvkJ2PbXGo3MGRseci64wIecx5JcRxy2xGyXHPbW9zTFk/FKg+CnkB38uJi1EhbpwObedBJsxmtrl1ck5Lyc50tg9V+uAztl8jl1pnTc5ro9zJGodsvKHXxpsiZXIsiFQzJEEK+Lr/KKRsTv8UuL22BrMfbAlNUbk89JKzHnkvB+sxs5RjJT3GiGQlVjriLa1MEbkWktFi48BxU0pjGKcoiLXzdRH8DQn9aKDUhHNvsyAQjJnHTAozbJ7qUDtzf2bkK2oxlIbUb4M9fDCDt6MtY6pX63J9ZXXiC2/ETRv3tF9OR5PqAsV2EWUXvYypqosHVxvRjFqPK7IL3IVM1IuW38lKi5bibNVGMk91tKe8k7TuBMNNG5k4CBlYnanZatxRqlNdBLIE62XdJ3VNuXhYw2+CXQvMHJ3zZbpkqx6IX38OK5LabKOM22DpVvQktpHGp9SyFbnTOco3kuTJVtF+MaOx+MGSw5bpYGBBlr7NFciHkyXHqE10JVH6MzoQ6TLBLqV9peSFuCIDOAUnYKyae+LmDRd+cH6y94EzDQUyTKcLJboNVSZ2rdkhRn5arGVOVnW8xicFmgiGO7h/txAz7XYMxUzbbktbofj49gi6of9OnX1UgtPdI8s25FciZlu9uwtw+BizO6/FVmLR2NVcaA7WczyOaZZxaFP6SKMNA8Rg7WWXYt53MmtPPfBViuuI+eBzwc1uCrz9WzuSXPea8QqWTnZRZnbY8+EgAGFmOdHeC1k12N2v07PRXab5jbZldWrwcroExJvapct0gDGVl+X0HgTcq9E5ZXu1LFMFss4zHtuj9voyaotLZaxiFDtOLb9tYqN2YrKNFOnyJZaxDR/gdaM3SJjQedUUIOVLs3tYA1Mbad3Fh9koJ78zAAxWSjzMVt6o6irMLhFJ0sle0hklF+VzgsU3ZFraJpjQkVG3l+R15jPh4QszCEFop1rKMtMX66vButh6kNu4TzHEdctjxgqXnNZhXitCGumdOcFxFgP3S/caIcFwkxFNTqmftW1aofKlK5rk8Dzo8Q5+bxPz4tz8hV0kA7ZlmxjltgbrnHXt6jzJ2mR/MP/+G/9PzHGf/Ln39Phsrr/Qfz1v/63vvR2v/tf/OLHBrzlFstH5Tn+/Q/+V5yHJXzUKJRDHy22oUQfLWrdZzCJK19jqVvsQgUPjZVu0QTSBGoVsQslatVjFypoFVAoj1r16KPBVahRK5fBgQwiPBSWukOIGle+RpF8TQwimlBgacga1KY+CMRHZWCdPlgEKFS6z/cYFeCjRq37EfgQjW0AEJKAN4Xy0IgotMPG1zQeEPjQkWnQR0P06A5ttOiDQaUdTHpDXvk6t13rHjtf5jmh8ZCWsGfU3QRcxPeEqFBphzbYDFTE4EWsjdt7Ak2SQDx8fmH6fL0NBaz2Od/owvTYugpVmo8QNSrdw+qAK1dn4BqZm1SCKxkEbHw1Aj0CMMph2npah3slaW1fdGRlkGBGPHZ2Q14kgB8Gb2LgJBcMrPYZIKjSjuJ1vc1CIMfyEtiTGQGeAECpHYyK2PvBnbIPBgvTZ+CeNjCQFMUD92m++NzWl7Aq5HVh+o5ti60vKZZY+yFdThiAqjIIEeaBlkrtMmAU99kFmy3yLmqU2qHxxQhkiNvneGirAjoBPDXMY1KUpP44PtoFjdo4dInWYa7pmSqNyxpgPre0HVzU6LxNe85nYKXGFSN6eW/y8xmizpZ3BsEqtcu/eQ8NgFEaxyVZwDZ9ldduCkTE7WgVsyViNO9xALVZ2g6bvsLSdmh8MXKDX1p697gE9sSx5AApMCrrspKNNfMyJnlVtNj2FQJUBi/jYnVA62y+1nqLQvsMeMYAU7m+WHt+V0zr8/qGqFBqD6NDfg7kNV4H+Z9B1gCyElSGnisZp82FlSWs3V8WHfpg8jFbfFs39M3WhsJ4tM5Sn2kOKuvQukHBVyaFpFQW9ikdFZBkOKYhUliA1QGtN6P4eAkKVUzAerxYK6T2+D4uMc0V98nt9BzrLWhkpSspPN1ofqbATEwXK1zlMUCWFmsCCuOzZYjb4T5JYTuk8bImXFOesuKU75fps1gZC5BybLS+6f8U9IsVpmayplLhnMGuUmjMmxZWpHJYy5DyK44UtdJqwjH4HK8vY9mlspWPmVZWYvMczaG6y7RkEqCJaaI2qY1CKG5Z0Qq2/IQxOBu3yUBQcv6nehyjhvReEYPSERjCAEOKRQ8zgiWn6RqNb4rmmy8MAFDXfDBZaTNp/6DgwvWT8D0S2jC5jxVHb1KmQqE8rybXRf2cNin9JpJjPi9BmK7lfz0kNHJ5FS1DwweHdWuhcXp8W8FV9DNd6luVPwFB9VflenmrBcsn/TH+66e/hU+2J3BRZ5fOy7bGtivwYL3Bri+xLDpctjXuLHZ4sl1DqYi7ix22fUkuBNbh5W6Bk0WDTVvBGp/d43auxCeXxzhdNGichUuuCQBwUjUIUHhytcaiJGFjUfR4vl3i7mqHGBUumwqLss/PKzMA+77Avivw3skFAhQumhq1ddkCeVLu8cnmBH36SCwKchVs+zEztqy6DDD0ZLMmprHqsOsKPD45x4tmiaumwjvrDV7sl+iSS0dlKcb0xXaJMqVnWZY9tm0JTqHSe4Nl1aG2LjMs27ZEZT3K5MLS9hZHdYtNcjVxfkA43HcF6sJhs69QFo7iWh2NYVV32DYl1osWu7YkRq6zWFQ9tvsSZelxZ7XDs6sVjpcNtm0J5wxOVnusyg6fXx6htC4zN8xo2OTCwhbpy12d4mFdZgZWdYfWEaO33VXwXuPDR+RW9pOnZyhLj2XVYd+RO92uKUcM2PGKBIhdcm/pe4OiIICeRdWh7QsoFXG23JMLUVMlRsygLnvsmgrLusV2X2UrMVt0T1d71NbhydWa1jiBBR2tGnTO4mTR4MVmCa1jHj/dHyh2FsBuV8EWHutFi4tNDa0jisLjvZMLnDcLvLxaoqr6zJi6oNEnoCGmhy3RzKgwjYuqQ9MVKKxP662x3VdY1h20DtjuK5wd7aiPkoQb7zWc07A2pPGXqEqHrrfQOqBti8y8FCUxvoX1iACafYnVssVmW2O9arBvi5SblVA7XQJSquo+M3YcJ3x6vMOuLdE0RbY0F4VHXThcbGogeQAY67N7UNsOsa2LukcEsN+RC2dZObQNra1NfYeoUJYOXWdx/+yKXMqenWTLeVE6tHuKpVVpjY9WDYyOOL9a5HcZW9+9M9CGrEpH6z3Oz1e4e2eD88slvNPZCnx2skXryLW77w0WVY+rzSLvU7bAb69qVIueAKn6waXp0d0LfPr8BN5plHWfx2BMwNlqj89fHCNGhbLq0exKLNct9rsS9aJD2xaDFT4oFJWD1hFda7FYduhai7JyaPYlyqqn/ZT2FADUiw7WBOx2VQa2CkEjBpViyWkOfG9Q1g5F4bDbVRm8arEi18P9tkS97NA2ZQZyMtYjBg2frOIP717g5XaR99hi0cEaj/PzFUxBgEiuN1CK5vT5izWKyuF41eDFyxXunG3x8mJFYFVRYXXU5Pej1gGbXU1rpilu1nuNuu4RI9C1BZQOWC06XG0WKEoH72msodcoFz2co2dbKhWatoBLwFOICotlm99drjfk9eANluk9RM+XwfGqweW2zmtRlC6j3S6XLTYXC7xz/xKdM9juqyyUaB1hrUffW/SdxWLZotmXWC5bGB2wLEkQc0Fju6lRLzqcrva43Naoqh6V9aiswya97ztnsG8KGEN0LFctYlRomgLGBKwWLXwAttsaRenoue0NysqhMB7bXYWq6vN7tdmVZB0LCrZ0UAro9wWWxw26ziB4g+AUbOWwWtDeaJuCMAa2JdZHDdqmyDgJJycbXFycvj7Dkcpi1WJ/VSNGoKhdFsyKwqPZlrAlzWtV9wTGFhVCAghbHrUIQaPvbH42vNcITmO5pnWm/UhWfWPIkwcA+s5mzwuAPClWixZX7QJKB/jewC49YlAZC4HeLXQNAKzt4XuLonLwziIGwJYJ1K+3BDaWBFxTEOaCLehdHDqDYkE0x4SMq3WA7yxM5YAA+C69j1cdjR0gzIDO0v3LDv2+TFJphLbk4WOK5L7ZGfK+CISpwN4EqgjZsyR0egATS9KttilvuFM5nj8XBbKu20gW/VEcfxws7ww4FhK3NonFpLhwsRGk2S+Br42QktmabSLRy3U1xgIqS+j5OGaBNwo64iBtEo6AjkOO7ORpwOezwDzRAKheAQZZWJWCZBZWw8RjQMas5qEn4dbP0D8VHPncVKidXp8K8EKKZCyEEYL0a0uZv2TlLSf/pvJWC5ZWBbxTXWHvC7ig8e7iIqX8OAEA3Ks32JgKZyXFbT2or9AmQeTR4hLP9RKNL3Ba7tF7g7v1ljTX2uNevcW9aoOrvsaL/RL3FhtsU+wZx9PdX2wAALu+wFFJH8+zeofOm3wtRIV12Y786K0KuDIVjA65nlEBdYpZsNpjnawU+2RZOSkb7F2BjSnJApesGqf1HlZ5nJV79MGg94bO6RqPVy+ylebxmmJP9q7AvcUGS9ujCwaNs1iVXb6PGdPSeHTe4M5ih9r02VIDAOuyw8L2JOgWFvcXGzzVaxyVLVpns7bzylQ4Kls4r3FUt1gWHVpvse3KHEt3Z7HLcRJXqsLpco8QFRZlj4erS+z7Au+sNnihl9j3Fu+sNjgtd7hqK6zKDo2zWVNfGZ9jFeV8O09CM1t07i23uGhrFAmcqe8tvrp+Aa0Cnm1WqAqHe8stzs0iWzjYxdkaj3tL2ifP9RI6CdAUW6FxZ7HLbb+7vsDOkdBcGo99X+C03uO5Ak4XNP7ChOyGbXTAo9Uljoo2a+n7oLFtS9xbkiLk3fUFumSpWRcdNn2JzllY41EZsgQ9ieRifX+1ye7bR3WLD4+e4yNzis4ZLMseqyIxYd5i16d0K9ahcTYj8nLsY+81dKJ7U5RYFj0Wtkfryf36zmqHyjh8DuDRipjXo4rbN1kYPav3eKYi7iz3ON8vYI3HRscMlLUsSUCsLTHFLwHcW2/ho8KDoyu8sEv0jvb5um6x7wpEAMc1PWO7vsgAXQ/XV3hhljgHKTrqwmFVdjiuGrgEhOWDRlX0WKR4nStdZZft0yWt0XOQRn5ZdbhKbuTrmhQiLmis6xYbU+G99QVK7XG+XWTrw6rqcJH612mcD9YblJrQjvlZIVd7drunsT9Yb7BrKjxcX6HtbUZHjlHh0dFlVow1zuKs3ucYYB807q232XX9eNnAB41dS7FEAMWJXexrdL3F0bLBUdVi2xGK9OOjl6TISWMLgdb3eQTOVntcCqtICBrrRQtrPK5MjTurHbZFgaOqw3MVcbJoCD07qLwuZ6s9VkWHz0Co1L2nVDgh6BzjVFqHfVviZLXHuuzwOZLlr7e4t94iRIXnEbh/tMUL4W6/KHs4TwovYwI+PHmGn+o7eGEWsDrgbtqnTVdgVXeIUWHfFdA64NHRJTb7CkfLBg/XV9i1BR4dXaJ1RJv3NK8xKizTs1OYgF1XoLL07O26AvfXW1JqFRWsCTir93lvh6jyWp6s9ti2Jd47uaDxJy+Bl8UiK9sigPvrLRpnR/FgnTN4eHSFAEods+8LvLu+wEfqlGI9iyo/S01X4N56i763eHz8EjtX4nmxzM96oQOWRZdj1x4cbfDCLvBgTV4cd6ptjhX8OCrcXe3w+OgFQlQ4LlsclQ2WtsPH9hRHRYsuGDwrVqitw/m+xoMj+sY9t0tUhcP95ZbSSnlSclodsO1KHNcNKkOKpZNFg21XYlV2eJ6sgt5rrOoOWkVc6AXuH23QOIumK+h9s2jzu/kze4Q7ix0+jgqPji/x3K6wrlo0zuKrxy9wsa9v5C1uKvfWW3yerHPrRZutwLV1+CwCq7pD01vcX29x1VZp79P79dHJJVpncdVUOFk06ING01u0fYEHx1cIUWHTVjlGcFn0aBPyu4zTA4DOWTw+OcfH6dvBe++qrdD2Nsf4WePzvXdXO7ywSxzVLfa9RZ/WwCjad5Seh9pn5ap8pk5Xe3TOJsu1R2k9NmksPqqsyL13RHs2RoVF0WPbFdg1FU7XO7wUAi+/z9Z1m95RZVZaMwI8QEK7SZ4ETVegawvYhIJvbciW794ZOKdHiO0SRJER4ZUa8oS73kCbANcZmMITcrsJWXDhkJmq7tG1NiuY2YNAYbDGmhQKw6CGHNLTtfSd0ioCKuYwmMihQ2lvKSBb+gEKa+EwkwiAQRZzOE1gAEEO8aHwIw6RkTGRIagMjEh0pE6TMJdpSaE3YKu7RF9PY9YMxJis36OQBy4T66u0QEsFCdM5jY+cphlSM4LloRjLX5VfbHmrYyzX33wY//G/82+iSaiw7LrjEhNgzeACIwORAeTYPpkbjR8+vp5ROVMuM85hxhue3WBIixzzOdKWJw1akA8l3cduJyG5snA9flBkO9I1hd1MZGHrJWvL5YuH007kl1B62chYTo6341izjLIpXFRyfKgacoflmDBxH7uzcMkvA74nafNioFQT0VH6hpyPLGka+WWlbMgaVIr7UzndROT4ralLDJPP88UxceKFIxFiGVlT1yluMsVYcQwdv1hHmj2O5/NZlThc1wMt/z9779JrybKkCX3m7hGx1t658zxuVXW3WiW1kEAqATMYITFC0BISDUj8D5rmZyAhMeoJ0GIAP4EhUwrUoqSGMV1SIaq67j03T+7ca8XD3Y2BmblbxF575+Mc+p4sXZcy91qx/GH+iAj/3Mw+s3aa2Y6dpNrfQ70glS3wnp3UTj9V9p1v2q0HudVrbI6AyDPUHu7C+wZZG5pvZx50bMOFQdi15/2sIu/8+XYnp9ZvR2t/cw593Vaf81nayWqfbSztmuWvThYvp58795JryU6fm/8fdnP17HPUujdfB7pvmeU9yuDrOIw15SDskYX2Zk/JOQJpH7zv0tFfrfluWZGhgjbRCHC0U3S0dUPK/MqBd75RdpK9kzOg+cK1k+5wyG9yAt0Pzq+D41gSGkuj1dXqsHAMVT8XPLv/bU3xwI2FVcZUZW2MtGjjyIMwbHJgIMrpPg98k5XUr8/jMjQWx8awaWPh8kBDEBD38eh+UbTz5WqhQYCdD1ov1/3a2jh59kdty8KegNGYcNumktBCh3DsZUxgm0PKaCyipNoKVmZYX6f5qdka8LI3n75Nn/fadrvefCxN/htxCtWfsP3GaOFSSPt3XIPGmllTD/HyJcnYRQF0rZbKFoyol7F7xjX5/Xp3n+n4m6UbdfQG0RhbGbb2sPfPA/q9ZPPs31cmB6C+n+jPiIO/YVsbfHj02tzpZ6DLZeNiax5u7HYWlt7ns9V7mHfb65gvsI2N75d/1/ghqC6Pa5t8/lumsG6sj2aitxRmR0Xarp1jff8Sku8vvTA2H5PpVcWgf26/UsfN3z6ljJPBr5efoqz8v/6r370f4v0f/jH/yT/4hz97vf/0v/1Hv/O+AV+5xvLb4Yr/6O/+MzyWU/c/VJOijePeL1F9pnKN2Dg0/zTzNzOfray+PKa53GrEUiMGfUqa311hwqQ+XtcyNBZS80ky/yzvj2Y+UYlq82PzbLGWLP+q/nFA903zLGKW15I3p8o1YFTfKh9YvjGP6YnbWtMzX0Xv55X8DgfiRxVu3NXe5+WWLMmArct7DHxv/jBbjY39zq55H6nK4ldo/lv2N2BPkmTyHsfJ/PdMnlJD00Bc87Abn2N4lz4/3XfO+8t4n0Hr2zHIvP21GJ7ev9FOXq2/gPhcDVqX9ylr9WP/rjCfzaDrtOr3Fh/U/K9cGT7Mi+/rS99tTM0fzNjprB0Arb9Wt/f5Km78LPlxtLr9OjB/qspA8C8aN7+mpTUtoBFyGdmWrbXKfe6srmovYqbGCmjlj2vLzz0ztRiwW977/Pm5NKItADtfMj9OlqKbcx8HloHuX+fyFz+O2pb5+fFBViPzskMne6YBPZ6spVqlb95Hs/e/E2Yd13cn8qKWV/qK9rs/oLLfPDuhJwlr86J+artTezfexzi2ZkYKdG2PHahZ28Zs6InGjFys+7D12LV+bHzbXmZ/KLgjGTvsNo/jV/UQrdXrTurZzQF530w2/7X9HPm8VQ9frR6fjFQM3H0/A/p9tWN2DLw7F7J8pj3xmhI5eDTZAX8ouYtD6x5i1fwzbT21gwnqmhE7nNODgXbgS9w0O6YRaoeEdugXGNkf6nxmsoNh+bLXmhTbwTf5aA927JoHRsAzoCl1u98UYFubO9IldyCxOxiEq8fLcARJ1s7Rf/C4m/+U9JLsrv1mYsn7v9Ik7eu6BSB0zl+S9SboOIJHd92fae7y3urTrbZfatOK043fPwUYfQZ4Ok7ri7P2OcDP6v6CMp+T54tA4ueuy9+nf6npqwaWlzriT3/79/BmWJBrxLtVfJbeDItQrl8fMCohwzlt+HE94WFcUJmE1nrYMISCD+uE708XvFvOjezhug14WkYMqYhJ3fUOQyxt00XEeFrFvOT78wWXbUTQa9+fL/jtLLI8qPmNJ2PIJeI8bJjUl85oqrcS2gZyUxM187k0Ztqk5lGWrqsw4uYS8HBe2rVpyPjxwxnjmHEaMt4/nXB3Et+meUstbufbu1mpr9HMXoRAITQKbtPmMgOncWtxQi1u57IM6hOTkNRnCgBSFHbau2nDkjsb6ZjEFM372xmL7XUZcZ7ENPd6mXB3t+A6DxiGIvLMA0qOuH8zY1NzN/MxFHM12ZTZBv3kfO/MBGaZB/HFYsJp2pBiwZ//xR8AAO6/vaKU0MaHGRjHstvsLhoXMQ0FzGKKU3RD1+KWMmG7DqBYkcbSTGPKpuY2+hdMjYGWa0C+JqAQwp2Y+ZD6P63zIBvHOSLeZTFDUY2vyWVsrsNJ/EnLEsUHCOJHhscBPBWku4yyha5xpc5UamYvxt7a4k/qBqb5wJTQNJVhVNbSQgjnjPo0gO6yaJUZEsNUtc68RInBuETRoDIQx9o2mY0p1WKBTgV1TginjHpJLQYkRW6yAACvsimGtgVAYkAODNK4o40dNBNwFjMlihV1i3KNIcygptmz+HCnoppekt9ZPw+6K9kCMFTQ+0FOxr/b+sZzDVLeNiSFhJGyEPg+9xekMWpaHEgAdA3ghwx6P4DvivoEQcb0KTUNKUfW+IKlz5fGyORzjwHZNLgA0o8J+W0BUgUtsfsgMRCfAvI3wnBLW5CYjpcIniQWIo/cdwOErt1MwrZpmr96qhK7UrWOphmhJSBkoJxrjzfp2TyTxmFU9s6wAfXMjfUyXoLszU8V8RpQJxZtFOvcEjd2zPouoJwkLxVCWEQLkt+obNY2E9ITIb+V63GWOIn8GFDuuGlM+KJEPToPnICaWLV9og3DIvVW1UiGDagnZe8kGQuOQFi0r1ddbwpQeEDTxoEYcSbRzAIylgwpP6PNgXwn8EnXiLF/KmAJK1DvGem3ARzR4gyadjJkAg0MTkC8EjAx0lXXoQa15wCUs8gTVqCcRAtq8RDrIJ+ZgDqixVWMs9xbdZI5Cave4neMuFIbR9qknnISeS1uYJ0UrAbRNBIDZWKkJxK5VLawAXGVPpcRiBuQT0BchA01aD/SFVgf8MUpLkBR9+ig7RGL7PkkcnDs7QJoGr00y+c6aNkg3zkAUX8zTXYbewPmqbfFJNeHDzJeNoZxlro9gykVuWay10HHIvRxA8uceU2jtd3yap+snGm4W19o307TYmaVKenYTb0f9jwKm5ZPXevbYkmyXBNmWwZHF4dT6wiFtQx16wtIfgDg6MC81uNjfYrGmxAKo0YCVd4zwFK/D47J4+8dc2/t11o5mz+nbT0Crx2eJ+Bwtu8Aeq+HA1pszx2rLbk2yc0v97Zu4h5T5AAAIABJREFU1X1s4yUZdppQN1Y3waS7TgdLSSZ6du1Z2rXBN9t4dkjwS00vHZr8DUlfNbCcc8L//dtfYUil+QAAAmjGVPDh2v2lUipYloRBCRsMXBBJiITfns5YloSkISJKjihqj35ZRiHtUGBip5/bIv6Ej6dTc7gvOeDxMmFbRJZ3w7kRU/jQDSFKaIn1Mshp+SCb62Y6WQjrLIHVbcNtG2oA7clRs+bPAfNJnOLNjp6vEesw4ilJQHgDOgYCEBjzZexEHDngMkz9lJh0M+4cqa/DtAMVZvq3DqNs6ryppJowXsdJQIZunp+Ufn8eJvAasAwVyAHXQUIazOMo+deA93MC1oAtVXkprAFUCI9L78POxNQ29PogfBo1OL03zcyErCff6zDKBvoxgRh4qucWQNxw12bB4zWRbtRXnYutmX7KRnyzYNGL0OlvKYEqoUQxq9uG/nd3vMhAmIO8rBXU1MAoQTbjHCW4dFlD20RzYGRPMc/Aek2ygdsImwIMyoT4FFBXQl4DKBOCAlEzIRO1pZiOsb0Y7QWiIoYsAaxDRaN85yHKxrQCdQ6IS0DdSIJls238ZH7ipnk2Qk0RocrGvL0c1UTNXo48RMSFtF4Cx9jkC7mb2cXNtyWypquAkzqEJjtVGYu6hLbJD6UHH+cU26m2yVLnoOZ1+jujm5iSXo8R8aqghQa3MSSwlrcXfNDNdN3649fMJpsJIIC4EHIZkC6EssVu5gYBJaZmlSDlQN36RDXZFxJAY8DPtLdXAhDBMUjeilZfnAmg2OahzoSwypjZptlkYdJNn4GdDagxIhQxK7dNqW1CZB5kbdEWu3mkbczcxsA2vVR73zgoeIAccoQVqCt1S9gCgKgBpjhrsPhVwG+wMeIAyrao/ZgI6JUg7gHpQs3kEyTA1OSU0BUKICu1DWRogFXnPMs66KajHSQhEMLSnwFMBqrMDFCAWAMXZvqoYFGEcWNVHFDLJrfKUAlxtfWyD+1g65KT1pPlXpU1rH0hqSNkAQg2lwYwyJ4JuiZs3oLVo2MUGgGK1AUAvOm6MJCmbQpoNXNjamuNMgl4ib3/lOHaku8JQFiAkPrzKqwC8L40hdWt/a1vrG3DHrKsibii9y/0svbMCJu7N3Rds1833GUGANbx6YIAYWPIxkJ+D2sv0wCizgvQ7yfKUp5VFmLVHB4AaTP7bWuM2xqWPuthQeZm+gwCwsq9zwoAOZOUt5tdy4P6ONkhRXsm2Po0l0c13afS+2RrT8aZdwDG/u5Mq92z2OZG7ik9ICq8A19tuAtQG2DdA5hnAMvdV1S7xcIx/4vaTPc8OII6Ym5AjFvwR5GbA7UxlfL2PtV50+fDiyDOA8pPlLF9v5FeUno/B36voCwPdj+Sdu+PXzjA/CXL9lPTVw0szTxn3tLO3GndEtYtCeGXatbWVbq6bZ211MchXGah81/XTh1vWpxZNVTszMNQBRwCwtbGugkAyXe7EcoW22dPM14LCfuhspkZBbf4EBKQsGNeBMnfWxTcFBgYXEw/gvienOQtz4WAppFS3yIzMWtxz7SNHLo/mD0RnPlX03J5vyCLp2Y+ZmauZNoI1Wi0O0lBGGcFhNp3icPHYGWWazHqzJ+sSN3sfCRN7vbd2taN5c68x7rhzdk2eXvxSZjvdrHnLB3i9jU/K/+7yuJ9sOqpvxEMAPHg/h4fmFbm+BBlgAe5zmctb9ot+92z4FX5ndWfytrPb11YiQjUVPemR37uAHj/odaO1slRNtVtbQ/cfIvqud7uY2RU3TTVUfthbVg9On6++3Xi3V/fp5bH9dVSudtf4Kj1j25962bK/MRE66nlT97h5yCrzaNtavnQni0JmyPdJDFE6/NsXF1MQjvdzlpfdvU2Bef5sIEasTPzKhPv5qaVt/v+bW+v2tzZuI3cbtWqITn7HOzElmuhXzDtiUZ4gX9ktvpTL8yHtX7YD6J4e2eILLmT6aKe3eaF+9gBAAKwfbMfa1t3gMtrfdUxKVHHD8D2sF9X9a5v2naC6gfSNvy+68ZtLu3ovJRTfzza48wnG9Pjc6G8xj/DAE7us72Ppr0gx5h9xL1ee1ocN2gFwPbmlba9DC+lGxvFFzfrLhX7AXoPvGAStyt3t/9t/akbzjs02V+bg3yH/SIAgPtX8n+BTNsben3j7Reiv/YsfcI4Wj4+fL9Z/nZ9dr3X+0K+G2vjY3k+FXh8SlsfBSWfMldt7I9j9pnyPc91+GvyvDQXz6/RC/fNs/RTx2DX5hcs8J9ynz5r/+er6/fp9fRVA8sxFfydt++bzx3QfdO8P58l74N09JWyvMfrR18uf93X1X17+udb9ZuPmvd59OnV+5i7L99Lvl4v9e8Zy5btd6l/Ptb3Oe0f23iJqevW759S32t17gsddibH5I7QPtafW+lWky+901/zA3j1GfvJm4EX0g2B2PxuDCc125gbbd1q09d53Im+9P2l48pX0jH+26ssb7fa8Z9f6ttPHd9jdsMeXoZXC3/GS53wurwv9fHG4cSzay8u3GNbnzeHP2v6mHy3xuiVMj4WnP/+ar23rn8s3+fUuctz2NH64/cb6bj0b1XxoizAfuw+Y6P+6ub7Y/186T75hHUmRY4L+4W2PmdufoZ0U/rjOB9/u3lI8Rltfu483/oNr+R5rY2PvWs/R75XfqMb0y0/6PePrFMT71Wzz1fSy/1/vZyV/az78VPSJz7rXpLnS2T4eeT/dID92e1/SvqlActfmjw/Y/qqgeUUMv7emx+wlLQLZu+JcWYXnD4QYy1Jf1MWUA2Inh35D9DJQwBgLXFHPuN/zzU0Ahm7bsQyABqpzDFVfTJEUsrrA8mMDwp+BKQmhw+ybYQkHlxaXgtc7ft2DCTuiXQsMXqw79am/mYjYUQxx2e0gfRbpDb2+Uja4olajuQttwhjLK8nDPHJk7R4cpYYjCilA0ojdjHil9eIWo5g1OczenMvj++Pn09PrOLH1PrVsIOr6+b+4MahwvGgg4gbQctLhwY2D55U5UiGInk74DtSg3fCFiuPpqn39fnxPZK+vNa3o6z+s7VvbRpxjLXvg5mXGyQnL1GXH9s4ktfcGqtdHYeYXC8drhw3Z0Y6cnPzduParXE8frb79EiXf0y7/agHNy/JcmNOntXp1gQAcEUndjmsl1a1y9v6c4va/tD/Vt7G8JDoY5szwt7E/kZfP5ZePPhqAuD2rtPX7x+stw6M3LXdZxzyWn+IX3iIvNKn42GN1edlPMp6rPcLDpn28rl2XwKR6HmMIOal9Mkam1tNkfrzHoGhl4PxnIxHZfuoZvHW3Pj77yOyv6qV+xiQ/cg47+u+gfheBGHPnx2fBVbs+fCxfC9c++hBiP39CNB8Dk4/bR19MjD6XLDxGfm/GBz+FMD5CXX81PR7TeQvL33VwDJRwUOa8ZAE4C11ACIQSAhvnvKE+7Q0xte1Jnw3XrDW1AARAGVznZVBswOttQqg/NUk8fqODJlrjUDEDnTmGvB2nFvMRIsH6ZOB2RQq5iImuPdhaYysVs/9sO4YTI8MrXbdwOA5ba2sgSoQENOGax5wl7YOtvQRfT+su/wnM0XUTVrhvQbY2Gn3/XmdTdTGx7OHHsEwMyEOG0oNGEaRaSsRp5R3TLhblfiD95rnlPJOO318xoyxtN9Mrq12Zt9B671uAyTmVgYDe7lC3QHpokyvSeOXtbHWclGZa0sljSOW+3hWwhD6X5+YJeZmrgFTKk0rHkMHvMYS+hrYiYFRWUiiTEZmwpoTQqitvI0XAY0tEwAqSZ8bSyl1ht7KpPVTY/eM1BlyxUe5h/oBBODZ+7ra+Cijp41fO5A4MI+GIHljECIoY9A0FkoyVmUdnxhrA5RG6uS13pZvUOKlRnzEvTx2sgDRmDQdcDEA7UMV5RzANWAY+5gzS6BxP8cW8DyEZ7uUHbCqJSCmirxFpEGcy/xvAhL6Zs0IqQgdTEVHpuUBf8mx+Zhb6KNgfdOYaCAGK+mUD1kUfLvWT7uvdf52jLDtcKnPASCuBMYy2ubIgCN3hlSu1NwOCNzcBkLkRvpEBO032ndm8cOkWDv7qLkLaDgoQMEnCxlWGEojxgpDQd1iC3EE1vLkwGbYz9kuPJHtZiuUeKmDkV0+c0ew/D4UEKETOx2BjGc2DSw+7s1c2wEQ4k7OZEROwdXH6LtuD0DNL6yE7rtmc6MkS0KEQt33zQ6LrP4ADYFC7ZqFRPE+vxaY3QiaWmgcRiNmspA2Vj/t+guRxREkhSp+6T18Calv9t4k+nNTKKGb4ZceEgOMXTukhDR+zpovt/qPsvu9hTDxc9wOltT/0R8kkfgMN1kIPayNY4u1UCvSxt6Psn3WubKQLExw4V6cjA4/kvX5OWbchS1p5taBOoGOVcL7/Efw176zk9nGzD2G+jrC/qDL5AqMYP3yQBLuM+3/PgeRuJ1eAaQm+4tWQb7eW3V8BOge27o1fk0OvACuX5D7xbG6lT5Dxk9OX3CLfk0g82uS9XPTVw0sCwc85hO+TRcsSHifJwxUcZ8W3IUVP9Q7DCThQB5CweM24buxShDq9Q6ntGEMGZc84FfThqd8wn1acS0DPmwTnjYJGP6vPPwav10kfwALoATwuJ5QQfjD0wfMRfwwn/KIP4wf8LidEMBI4drAo9e2ndOGQAU/XO+QQsUpBWxVAr8XJmwl4hsXMPppGzGGgukApC7bKIGzNUg6EeOyjTinDX/5+IC7acWbccWP1xPeTCumlPFhHbHmiBgY358vDTQ/rSMepqX1MYWKD+uIrIHXAeA8bFhLxJoTYhBw/LSMuJtWrFmCMmdlzbUA3N+cZsw5tQDuUyy4bAPuxxWPy4j7Ueo8Dxsu24AHZd798XrCN+cZj/OEacgYY8HjPGHbEr5/eNJA6gVrju2vseraBvfNSVgmrL+BGE8ahJmZMEwrYqh49/4ODOAPv3vEmiNmZdbNNeB+WnchIz5cxdFs0gDNQ+rB7ud1QEqyQ7hcJsRYMU1bi6u6rAnTmNtf0xRbaJPrdQQXQj5vTYs2DRs+XE7CwHsdcL5fUSsh54CUKpIGfjaAdHcSVt1lHnC+k/4vy4D1w4gwFpzvV2xbbD7GFGoLip2zgKSUKnIOqEVishoAy5uw3pYSUHNAiIxhzNjWhFoI03nD8jRivFslZiwTQiyIkVEKIa8Jw5RbMGoDYqZpzOoDXXXTPYwZ2yJl1nlATBUUKmJkDeYtY72tSQBIqg2wbk/S32HKDWjWLYBzwHAnhzAhSlgJLgos1eeZiJGXKO7Ok8xT3QLiIKzMdQsIgxw4lBwQUkV5HMS3+bseAqNuAWnKYNVAcyHUawIqIdw3FpMGWMLQ47rWDwP47Yr8fkS93zoQI0hbgQWwBAEM4T63TUy9JoCBeJ9RZvHzNtAEAHg3In+7CrvuEoEtoChJFV0j8jfCSsJrVKbfhHoq4DmiTtXt6CDEOJVQRyEJQxIyrjoVQOPCIjDIDq3mCCqEfCodDBlw8QRgqQJLBG2k/q4Q1l8lpCqnApoj6lj7RlrZhK2O9C6h3FXUQXa3tARhyXwojQnYWGHjh4DylkBbQLwSygMhPkaU+9qAYrjEvumshDrVxkALFlAUr8Jay4PIEFZCPVcBOCR94MgIcwAPjPgUdpvqOnL3uWUhAWs+8Uk36lHYcQ241CQsruXEOhfciFCMgba8qRiMFVYZXFv8QGWIrYOwuDY2Vwh4qUnmp5yFQCxdCPmelTBHiInKiYVQi6AMsNJOWKRfZRR5rN58z42RVIifSFlhuZFEUdFyMCAly64OQLoIqKpJxiBsaKy/5cTS5ztGvBDqiM4KewHWg+/t56R4peb3HFdScC2y5nsZH446jsYKG2X9RGUArqOMi7GtwlhhCTfIewyIYwfGOAHDI1BOtvaU9TVKmQ66gJokT1w7Y25V4iMjTaoDGntqbxudpTV28qHXWGGZhHnXfJiNFbYOQpr0jBWWXPkjK6yuz0b4xejrN/c69mUUwMIBqmessA7wko1RJ14ykqiWCKAs5Dg3LQgMjBpJDimzrLZjY70Dx3h+zYCo9+dvwN8D9dDnh1hlzu6ggPbt2LWq5F87wHjsjztM+KmssLcMXPz1m+kWUH8NuOMFwPxLT1+bvJ+RiL/EofYXkv7gT/6A//4/+QctPqRpTQzArcogYbEjLQaj1/L5+InexNTHObSypjH0Grujv6RpuCwOnGl+buW3794M9Xh4FVVeu9eOJqXmN2p9sDoBNEBIxA1YmTxbCe27N5U9xrv05rU7jQ3vzUstBUIzMTVNm2nQfOox20IzH7XkzThNFq9Bs3E5jqnXWLIbI2D/rDqaltr4SZw604D1OW55nbaqx9M7mnViZxZ6rONorvjMp9e0BO5yCLWFatnHycMur5ltmlzHvAZKbrX7zITwRp7do8Ln9eaGquWicIMVhOlZ/4/pllx+fulw3b6zynTLz5hMI2pvn5eOdU1Ou3yYi1vmwP4zV2kzxK4ZBAMUsPt+1DbvO2ydkfYpcgsBsxsnp9F58e+hvl35ElpYltbWsW9urDyrNdGhLrcOez8NJVEDZb0A7TZKUviwQ/Bz4jZKbdPj++brOfb72L4n+TrudmzjaRslC1twq22vCmgPlhuyf0q6NWe38vh+tTYP+V5bA4Sm2WnVkjPrfKl+P8Z+XL3Wsx76foyvCLTfvRZp9xy5sQSOw7nLc2Oze/y9M37uN9MHo5vPSkdlsNXZtM23xsTeIao1O/alfX9pPQM3N9tH8qVb9z1ZG4ybt8yL6873jQ7XDulFjaXvk90/L/TvOOe7x6NbN8+Svw8+IqeX9VkdH7n2avs/Md1cUy+l4/2NG99v1A/cGNdjvZ9y7afU90r6bGD4heP+f/7X/8U/ZeZ/68tK/zzp/g/+mP/1//Af/uz1/u//wz/6nfcN+Mo1lkSMgSqeamrmnLkG5BqROWAMGZnjDlSuJYqmqpkrCiAzv8itStDyAAkov9XYtHeZww7UJaqoRFhyauaFMVQsOTXNogd8zPsg8ZVply85UzMDpx5IVaYGJAnd9xMQk0/7zcw878e1mY6OqZvzmaYRWgfQwVsunTXXA24zfzOgGcmD8j1QimpyZ/6MBi6lHcCDTgm8Tn18VAYLJC9mlWKKV5lagG8fHN1Apw82D3T/UT9+AHbjnNWsdUxZAbjcElHjhXowaoAgxfrsmdZl7QcUg2oBi5PVQLR8fw6uhqGbdhqQFLNbifk4DD1gu42Z9VnmoLcTQtfchsAYR/lN6q678reAnvlmeh9L01x6sF0riSZPwWxqGkgX2F3fQFH7HagfJng/Qm/qauOQNM6n/ZVC1M132fxIFdjYPWrxLCtJnlicz57rd6h4tjMCWpxNCyYfYu2fzXwSZnpMSKPkryU2INjNQ22hMGLoa/hZapvTHt/U4qB6UGdmr34/5f1Lm+mnu+YXdBpFi27XwtB3qHaQAa2nFkIcK7gCIaABaEtBwRXbQYbWyZVkDOv+EEUOHdDAcfuN5Te2XbCBIXIHF0xi0gnc9oHU7zZWlIpS7LsdL0HAejvs0XENLLFhgzzruATQVCX0UVsT+50wu3tRVBOkZrZ9LVHgbvbqN8EGYg34ejDub0Wfz2/Yg6vLvjfzRbfjszVVCDzVDgzdoVvL5x96dpjhgfPRlNFkJ+zDP7GrU+fb0E1jsK6GdqTvbPVU2nXL973FRiyE6uO6MnVzXRzG6fi3med+WarBmeS6apjqzoS3mS/bWgb2gPu4Ho6g6Pj91kb6OOfH+wEiRwO0DsEYkG1j6mXx8nnQcgSL2Ofphwb7unxb5A9nrIBfY4cDiDaW6N+JD8NxANLP0rEfx98OTXxS+Vfy7w5ECDcOQQ4I+sYh7GvppfPRTwKln/nbR8flU9Kn5vuUNl9r4yMA+xeT+Av7+JWkrxpYmmbRwORJfQzt2hhUe0dofnrFHVV64ppcAxD2GkRiQgA33za/mSqqmbD85bBJNO1lYULQPAZmgv5e9FrzZQOeaTNZ/9r9ciTFsTIGmoHnWtKifnkGZok74Y6BPm5yaJ2Qz9HVV/X3AAF4RowzBMZWqZWxgC0yP+rvVuE2coTKBkpL609R2fz9VpkQgN3GNBCwMQHanwDsfGah+VMsDSiaNYkBHBtLAyoGfI9kJ1BgIN/RNTn2++4vdt9DqA3se3DW/3atrYE4ItY9FB3qY1eP1c9NS9n7jXYthL2c1MbRwtI8f7K9RNZzlOMlkp2uEQ3wYO+o6brJ9urejN7nrrf5vL22SbrxHj5qvq1+0XbeVlnsNKo25ri9Wbn1Xmjza/fAzVaey9v2Va4sBbWhcmPdxsw2XyYf7aUkn59ULt9e4BZPYifLcU24MW7+m7vd9K7Z/fpu17jntWvxZZKfXreAflIw2zavDRDpXB1nx68L06br5pQUfHDpaw263kPovr4CBtG13cfxIZfPLhHa5p3s/iXbQWgBD7xsDqOCetXytyVPrk2Crt+9LE0bzgZozTxWyxDkNE+BpZf3OWDgPThRE+FWppoMLL/l0E2YbadPro+EHrMk6nqz34Hels7mLrzRUfPnxhyAgkhb3ABb7L7m26lyGthlV0fgRgb1RcnGtckBJys6QPZgxw6yzM/UgLB+BqPFEt77UXa/1WcPEy1Dfm0F+wG7A4m2imXzIf6UNp/+EKq1CzRAan61jN1ny7fDQfqi9Rpi1nEhbxEA7J9rZpLNGg+TuR/O0D5vO+Bw495eHdyfebcS3RhK0v/4eBhxGOsjeL6pefXvTPfaeZbY1o37uxOU25+bmNP9bVW66zdBi70vXhqbF15WH2Ww/4T0uSDqJxlS/pSyv08/S/qqgeVaI/7i6Vtct6ERsZh/Yi5BfOTUHHQrAUOsWLNqLFNBqdS0O5tq/3IVWJRiaYDrug4CzA6grmn9ctpdW3NEUm1JLqGZV3rtkmmO5qFrLD2wiKFi0QDqZt7oSUYsWd0hqK+Y5i0lYBk38dksAUn94kxbZg/YdY2I6rtl5UxGAyce1JkmyjQ2vtxRm2X+eSVHUOikKna9loAllebrV1W2qj6aUX0Hk+ap3MlZti3uzCNlL9Gf+o28owaYRosBiCZINECk/WFdBwCwzgkU0IhKADhtCQBCG5Mj4PRMpACwOobYZk6o2gszcWyaHZU7RAYFGTNrk4v481n9Jeta0U1lK29i6oYvREbNfYNdxoCSI2pWwpO2KXV9bBsEt+lrL3hGDr0fNvDerNJkrVvYbSCbBinWHku1uvG1TbaVsc1ODqAkWqOsPoCyydUx1E06RW5j3DRvQwGXIOOsdTZgYVoo3ex4DZ/1mTR+qi9vsVk9QCiqNaijmJ3XOfU3aYD48tm4BhZNC7HEbfVjbuNu908K4DWAJ5LYrn6zMoS+8aoqg7XDEJ84QOK0mr9eobax25jEH9J8GZ3mow7qK8loAKIMFciEEnlPGmMb5KD5dI6ahsxvJG2+zS9U/SubNtXqcuuBQ2h1t2tRx63oZ/Vv3Gn+9E9hSNlGGKOIYgvdFNOWaSFgDeL3luRzrZLX8nDybWHXNut64khuLhVouLEHC9Aw8hsLWu7XwI7V9BbZCYsfXyvGEP8vXZ9UOqkLVSWv2Wi3Znab6Lb5p+Yzyk0r18fJiHAQGZQdqc+hj2T3SQE4AcShtdnk2kgBINp9yEHkRHQyWoxnxp6EZuAGds3vtNWnvrq0aT6nTeWhit/qFyZOKiMACt20FSo7qY9fIyGyOWegDipvARCpgaHmBwnstVthn2cHMqpgRMq9HCebR+yATfPjM9KgRtpD3ReyUDcbhpZnNZFvwNJ9Jtkn2Ppq8w6A7ZACPY/ElVb/ReyXvLFDo/bH4V5j6Z6hjJ1/ocjRx+QmMPTrHT1/GzdS8H9LFaiDaMRGrZzPYs+AW0WDWweuD77cs+TqoEMZ0n6Q7w/3OfXP3GeArmo53+Yxz1Gm4+8ufTJg/NR8/z+mX5yG8Jcmz8+YvmpgeZ9W/Nvf/zkudcRaExIVVA5YauqmsEq0s3HAQBVXJdmZYkZhQq4RU8x4ykKOM5eEFCoSVQSqqBxwyQPGWJDNHFNBxxgKKghzHpD0aRmIMZcBY8jtc6LSQJAPb1KZcJdWrTfswqBYWStn5rzHuJemrfSkQlZmDKX5nnp2WgPEAHDNQ/MdNZNg355noiXinamwD5liYVXM5BjoWuLNkf8A+1Ah/q/lW0vEEGq7ZtpWYUuVcV3yPoyM/T1q0qzPftTsAMEYYksN+OY0KxmSjLmZE5P2w4+BMcluDVh2YGjlAGAwNlyn8fH9NN9Wz5ZrDLTWprHLDmpia+bKlr8cZAP0MCNwO1CRueW2hrcSZP9lmh/uoVfMH9b6FEi02t7M2b7LWuu/B5K2jczITFX3Wta9SbBp9W3cTOvszZlTLG0d1BraAY9nkyW37u1acky2VmfUNWIWCL2f3Mbb+npkM96Zc+rBgmeWNXPz5dwPmqKSA3XTYbRxMb/Zo19uG9vAzw6F2ho2U1juoN2HULEDp5xD87n15YchI9/F5ofrD4VCYORz7AcZejhmeU3L78fC+//6cre06p5Nd2cd0Ma1m+9GBSG9zW4u3Q6o3GGCaddY5ywNOnZa3syj7bALQDNvtoOsQNxInfxhWzO/9poW9Ytvez/N4w+UzBz6pobU+UG3OoGd9shYcq3/fhy7HOTmEMKO6zakIaCz/fI+JI6MO7qP8Aty2QFUsMOZai4S+pu7d9pBAe8PbYisP/27P3Qjl2/nr2t9N7BUsWPaZTvM8JtwM0G+4Stbh0PfPifttODo80rYmzxbPjd+BrxZP/tDlXaY9nzYd+NwU5aPJf+gNLntlexBHbsDDQVbPubrToPa6vbj3ss++6zt1lvaYgeSrK+3tW6a/ehbatW8NK2HtdQye5BsWu+mRbxRwWFyPmHke/lPWXI2N5+4PD0OvgVqn9X9sbZfaON2/k/r/ScBus8ZznoYAAAgAElEQVS5Hf8GADL/2PibmL5qYDmXhH9++RX+6vqArUR8f7pgrRHvlxOelhF/9OaDhNkYVvy4nPDd6YpfX+4RSNhQP6wT5pxwP654dznj4bTgaR0QA+N+XPHNOGMuCX/x7lt8e3fFklPbODMT3ioY+fWHe5zHDczi1/ibpzt8d3dFZWqMpgaszCz1ug5YtoS//c0jSg14WoWp1ADVFDP++ukNsm5q7qYV85awbINoHCH31/20IhDjbtjwm6e7xmJ6WUb88bfv8G4+43Ge8LcfHvHryx3WnPD2PGNQcPXDhzsMSfxNz+OGp3mUTbVuZu+mDWPKTSv7NI+Yhty0wWuOeDgteJwnnMetkQQBwHUdGqPpOGaMKYs2OUecxg2XecT9ecG8DoixYp4HnE8brvOAlCq+e3PBb97f4+39jOs6YF0Tvnu44Dxs+Ov3bzAOuYWgMM1pCrWBqsqEx8sJtUp4Cdsg3p2WNo7Xy4RaCA9/Z0Egxm/evUEaCu5PK67rgBgqrvMAC8tAgfH2fgYAPM0jADR21lqFkXVeB4TA+ObuijUnXBbp37YlnKcVl3nEeZL+ywa8b+7f3s04DxveXc4AZEO9rQkPb65YtoS3dzMelW12TAXLltrGP0bRmF0uE1IquDut+PB0AiDsqt99+yN+uN7h8emEcSwY1F8114B1lfk9ssG277rZO00b1i1hGDKGKEy81+uI03nFEAsu84hv3lzxdB0xTVnHJwpzaqw4TxueriPOpw2zjssyDwpcAtKQQSTAp9aAZR5wd7fgcpnw9s0V12UQrWslDGNuzLPTaUUIjHVNbZ6/e3vBsgUsOk8xFQxDwTRsuF6mNr5pKG3s1mUQP78acL5bwEyYr6McKIwZ6zIAxBiGgryJHOMkrLgP3z0ihor378/tBH4YM5br0DTKITAe3lwRA+PdZepAUMOClBybSebbN1c8PZ7w3XcfcLlMqOb/B+D+W3keMMv4nqYN16exacLv3ojs6zxgPG1S9xZFs8uEXz084S9/eIu6RqTThmnKCviBt+cr/uqHt2Ad4+06ID3M2JYB03lD3lIL+QEW1twQGHmNON2tWJcBcdqQt4jptCmwCyiqJT7drUipSL4kTLXm0xmHIqFQYkVeI8J5wzhmXJ5OAkZzwOluBQCsc8R5zNjWsREcxVQbgy8Fxh9994jfPp2xzDKH45gRifHjPCCNsjs1BuKH04p3P7xBOG24Py/47Q9v8PbNFT++v5PNeyWc3q5gFj/rQIyn64iaI4KGNMlbbOFm1kXm/XRe8fTh1Pyka4ngHJDOG3KNeHOvoa70wGWeB+ScGgnUScew6jgFBeXn89qfEVvEw/2Mx6cTYmSsc2rhZEqOON8t+PDujO9/9YRlS5iXobFJx1gxpIJlTdiWhLv7BfN1xBt9zp3GDcsmPAZPjydM5w3f3F/xw/v79lwfU8Hj9YRRw0PN1xExVaxzwvle5JyvI0IsuD+vIGL8+P4Owyj3u62VECouTyeM04ZtTRjGjPkic8fqbwwA25xwfliQs1hhcA6IY8H5bgFBGLnPdws+vD/j/mHGfB31eRHw/dsL/sVfv/3CXQdwul9xfZTnR5xKswCKsWL+MCGOpa3TbU1ycKdaxfObBaUEbEvCeMoohVCLWJGc7lcwy1wSsDsYATpjth001Ez45tsLHp9OotjVNtc1tWeFHWTlTSwQprsNy3XAMOVmvZImmYNtTuAtNPbmMBZhtR7k3Va3gOEkz4laxBUoBLlPjfk6L/I8Hu9Xvafl2Za3iLpEDHcbtqcBpiUntXCIYxGjiiUiKDN33ULXeg56oEMQa5M1yMECsWimVWNazTqhWb7IMwqb5rffCN1suoX9IbHuMGZqj5sqAWMVC5BbgMoOLrwVhLUTWdq3RNxNxW/5LNtnoGuD3TV/cGH1NAuFY31WRg9XWogewu7AwPK3cDP+bwufdEMGfzhxBEqHvjwjszr+7tMBJXuT7H0bh2vH779Pv5P01bPC/vv//X+CKWQsNeGSRRs5xoJEFR+2CYEq1po0rMiIU8yoIFzzgCGIJnErEWeN9WgAcCmpacXeTnML92Fau0QVV23vblixKAOthRJ52kYEYkwKprwPpcUiTKHicZmaFsxiJALAVgLuxx6OYFPtlvcjtOulBpRKOGn4C9OIXZYRYypIsbTwGUJQE5uG4KzmsjIOYi5sxD12zch0AAmtkUtomiRATIHH1M2OPcmNmSSvObY6jaDHQoRYiJIxFSw5YtI2li3hNG4ttEkMFWsW4HAat6ZR9Id8pkUyGYZU2pibmbEHo2OSOf3wdAIz4Xy3oNawA4umHTIQYCCsmyF3v8SupSHkLbbQGEftj9XvtZ0GLAzsAOgapE0IYUqOSEPemRSblgeqJUlDaZtQC8dRS0BeEsJQBcR4DZjOCdA1Pc3s0MkHYGfybAPftDosprwlhxa2wzQWprXhChd/EDp+nUzJ5s9eNr6+sgVQ5KbZsBiIgG4ogF1MxLJK/iZLlcMBLgFxKk020z4YC2s7nDaz8KRkMyX0z65tMwWuszwDwjl3whpXRsaXxASWCTSW9gJt2htnCspLBGmID/IhNQCwhfEA5KVeXH2AtEEs5Dk2NtFpN5YATAKGOJNsrkwDtBFw0h1DUQKeNby86TJTzsg9fEghQM1n26bE5DXTUjUF3f3mTUbNBLaim/OayS/wXB5G98fTsQrXCB64m19aiJGx9g2l3QZLAJ+K9HkL4KkizEHCq0Dy0UZNw0QMcOK9iRw5+Zx5oJhjorXn4zDSut+4iakp2tjQRl1TZ5uroGEtbB4Cutmn5usxH9FiN8aLxGDk2LU+ZOMW9Pom5rBB5SJbvkE0fZQJYYOGNFHNkjehBFpYA4sjSXaNAcqax0KemPxqKslpL5uVY+r5JTyJlGObm9rr5oQW5iJs+l3NJ+NCKOcv3/dQRgsjYiE3pFGAtU9MaOEfbF0wSXgSDqzhIvQ+J+lH2Gy+7CannUlrjaxmsT1fXEjGUUGBxLU87Mu5y2EhV2wsYGPKMkZ+LTdzU123HHR83Vq1cDZ+Xph0bjxwIM2XpR3pUJfRh/7YgRf0OohdXf3V1c1f0fvp++5NVNu4eNB13ERY3w51vKYl3dV7BEy3rh365uXYYaPDNWf0cNvc+ZXxs/Eh3/dDP3b5j+OCw+944bdW3yfcY7fG5oV0C0N+Sfqzf/y7Z05986s/5n/j7//nP3u9f/o//pe/874BX7nGcqsRPyx3uGwjNjX3ZCYsJWLZEs7j1thWl5xE0/V0DwA4Dxk/lhO2HDENGX/5+NCATgyiDTKzzH/+w/dNG+fvAwMlPzzdNUCaYsEPT3eYBtFg/ubDXTORAzrJTK4BpQTcn1ZUBj7kqZu1hoohVvz2cm6bc2MbPZqPDVp3igXvns5N25hzxJvzgmVLeJpHnMYNj9ep/W4sqb/+8U0zlUup4OlqrLBomj4iRlazvesyNO1aLgG1BgxDxmUWLZrFc2QmXGYBUU+XaRcL0QK/X68j0lCaRvBymRBTweUyyRgMBY8fzhgn0W7VIjEPUyp4ukwCNsz3SmXeE8gQLpdJ9qOhM3qmoWBbpdx8HcGV8ObtVfKrZiSmgnURXznTUtnmPw1dI2HzYDKkVLAp8LT4jNsWmw9sjBXrMiDEgqVpQmVRUWCkVDDGimUZ1CpHAFFMAt5Ea5YaWCs5dlNKBc7LdVDtTcE6i9aEIuPu7Yxti8hrbH6kxixassbni4yaBXSS+iRWNSUk4l1Q+0AdQIckGpy8RaQxI6+icbFA8LnKCboBxZBq8+HaVulPZWqB64MCgbJGhLGgrLFpArhQD8WR1Rw1iXMOq88sM5CmglpITtEh40uREYeMsiobspngRQNZ3XEnDMKEWlfZ2YakvqNaV80KEFNF3SKGN6KZ2Z6GtuGlVFE1jiQAIDDinayfYr6Y7EJ9mG9qDQh3GXWOEotyiTtfzfhma7E3uRDCqUg8SjuZnpQFeomd3db5vQ7fLtgugwDGoYLGqlq/inhfkTUOpoFKGqtoM8YqQNT7HSfd+WRlUTUwuul3O4QwX7lTERCzRNECqDYQDMD84QykjlXWxBq7j+tJn6drkM9b99/DYPXpPfVHs4yLzeuJpZ6rtg10gPntClyTXH+TgUsCf79KXgBgAt+XNv9M6idr5oKk8p2595cgfp1rACbdGZlv21nB7ZvcfR8B6XclAZcA+K50v9YGYghFZWl+hmPpBw45gM39oBLKQwXmgPK3ROOKts4EULbDgELAQwW2gPqN7PzFL1wXtK6X5odr/pZBYp7CtCY23oVkTq1fNkeAHG7YoUCTg7t2SQ9YyPyhFcgDAn7LW9npU6Hm0whjN96kXVoCsh0iaH+3sYIuOrhfkHhgOQyAtulAAK3UfCvNh5aYGonRNuphQ+4+ouZ/yKP2TceJibsZL9CZbG0zXwnrOYPW7i/Oox5YeNNfdxjDg8RTRYTIxNRYlmnTcnZgFrlrwhg7v9bGMqsHJH5eAO2LyauawpDlYCKsvR/mD2hj1gBvA4yaN3AHk4V6DFZgR6D0zJfRkj0P2OEXa0evNXIhr/Vrkw7nl+rmwKbeA1sPDu1f3QOjI2jdNWUg0X73Gki4tr0Mls9k59tlzI/0VYB2tKl9RQP4Org7duy1vEcZ8By0vgJif59+Wemr1lh+/yd/yP/ef/efAhBfOvGzrKgQNte5dD88ACgcEDXMgcWi9D6M5qMIyIbefAQNpBzjPHpfOvOJMxIh0zC2dwB3X0DztQPwLN5lZ4kFBtWImQbxmM/XZ6E9APF5M383It75xZlPnNce7pgwHWgVQLl/AVv7pi0EXj7cau812vu+ef/F6to81mHaPdOSmV+W+ZAdtWkGwu0z0P2QXoovafVuW9z5v3nG1+MYGYmSD41xJDcSnzfagVzTLBrANs2mL99iJ4bO3hgU3IF63E/Ju9cuNnljD5NhfmSwQwkHCP07jW5cO85rG5emzUT3i3LfGzmR64vJ4GM+0vHlrfPUGgcaI+iOGRTYA3L93NaDYR4jMmnto/l/+biQnTXWxt2BJrhr/nfbCLt8LayJ01Deekm3cTE2UJ/8XFYF2krGtPO1O/pVMbo81i5jr6X0Oxsz+yLeg0RCB3Z0uOaByLFtdr/74/VbOxgDV9H97n/z+ethrHWTtmtv12/0fLr5ls0dtzEB0E3WfPIbypc2mbbJt3kgN054/ltj/Ay8L6P5mJypmY01ubJtfE1u99lfJ/Sx4EM+y2vg0cb2hTky0LDb/NpHxYs7AOCm7rjBNk2P/Q647+FQ1l932pcdSYttuK2/h3vLylnepnFz8kC1bF+aGkGK64u0j92470CMbexVBkb/bEVvbtKPL9NbslDP84zg5zg37MbXydUL3GjbyeDn8nndHbQcSW5Y70Efx9PL91ENGHf5vQaylT324VYdxzaP/fxIehFEHTc+N8odXwMvpmOeW/UeX8wfaf+WPJ/c/qeUeaXcF+f7lDa/sP5/9t/87uNYvvnVH/O/+R/8/BrL//V/+r3G8ienp8sJ/9v/8a/KaVemxo7GgcWUZw7dWT6q6VCQl0pY9SSOAMqEOlXEOYipTwVQqZnalJOyvdkGQe29zXSlnLjVFzZCmRhxlbdHVbMe/5IhFhnMlEVO/tBNQ/REsI7cH6L6cuTDg99Mcaj0/KTmLnGRWF8cxQSnDqx9R+tDGfvLIqjpUNu8aV3+wWjmS34TELK0EzS2WDsV0xdYVTOk9iIyU5wo5j9VWQjrwCKD5o8rYdGxLZGbGRMqsJ4ZITtfAOpjZ2MM6AkxgFC6jb6ZA4GBbZAxGx5lYPMdtxNR23xkY/fTFFbth9491b2oo5kaMTBuMj41AUHrGzYZw6TfCTrO+uIMm9SlHFMwE6JhtbHWOeK++eLo1g4DddTxLl1GKsBplrJ17PMK6H7bmcg0Eyl/Ootez3FzaWZ1Ntdh7TLYGrEXn81vKN3ErZnNBTTGQNsgmcmbmbSZCZqNhTfnIpXd7pG4SP/tvMhkDBkok9voMnamXNbnoJboZex5qioZ7R6ztjkC8Sq/5Ts3XjpfNrY2x8Ii2+e4MWa6TXVYgHKWeutB3jijmZ2ZDHW8LXsw80AzrwSQLipn6PIYgLJ2mXSeBrnW5sDWJrAb+6pzxbGv06D3QFsDOiZUb6wRm3cdA5v71je9FleRrQ7yuabDmrTnKQPpwqgjtbUWNln0ZaJd28RAXBj5LNftc5oZZaT2HI5rf5BLn6mtf7DJx7p2Sde8yECFwcpuKeuXwYGkTvT55djLAiqz/lYjQUIyEOLG7X7loGMxUMsXCvd7ZZM+pwvrPWvkKFaxvZNI69F14Z8zJPdNXPUesueM3ftm2krOfDX2tViT1BdtbU7opqt679uzLzjmUrvnba2JhkueMxz63IQiY+r7X0YZ35q6NiyuMrdfmuweYJJxaPd2lfZM9rBxYwi2+zSuMv41kshKAJPsS+LWZTczwgbgIX315oUcdH0O/XvcWJ53Hshyf5e152/uz8WgYyr3kbbB+2cytzXb14KvO2SIBt/q3KrWJeNi90nYame/dUAxZL1njAXXjZmNrX8/cdRDBGZto6+jW4ciu8MJ9yy23/whiM/TEkHu3/D6urH5YSIhA0J/lnl5XgOE/uCi9989I23vYKb11r9yqNcnvbaPI3qQ5dAPJtr9vVkf9fXy2cm37+r8KdrKLwKjv08/e/qqgeV3b57wn/07f4qnPDXW10XfQkuNGKhi44BI4hc5hIKlJGVfFdNSYz5da9yxhQJo2s85DxhjbrEoLZnpqjG/Vg5IoTRWWADIHBHcHWHaVGN4PcWtMcK2esHNN9Tk6ayyzx9sxmI7q59n08I6VktjIa0gROpaOev3UWvr6/AMsLfksfI++Xib1r61aeymRmTUfDxraNpeAtpv9pch2mEiidnpNZ9+Tnwy5lbL67XIANQ/NeB+XFGZsDrG1abp1j6bltgTMVlf7XfvYxrd+PvyxjLatJwM2PsqqTm3Z3u1WKNWp2d1LR7w6sejphqQvMYsa4ytgdDilvJh/o7aa0te+2qfj76z5jPr58LmybPhNrZWp7U9zl97R+o68wy7xszqw/RUtx6i5icdS8tnbdpOvjO2ctMCs2mTiIGizJ+qIbP9uI8XygxwrKIks1Ax6NrmXRs2d6XPcfez7XmhoXrgQg1Zqu5+agy6jmW2Mac6zbjfG7CSMjVWXKepNbNtk0uIsTpr7LEu08oftfD+u8j6PP8z9k+/+9O/3adXnyleY+9M3Nv4cte8h1jVNFrnWmNF+nVizYfDejLrgF1YobCPmsnATXZT6HXoutkxnR42VF6jb+34EERNW+/bZdpp3f1mjA/rullFmA+xk+1Z8lrPG5vTXaghrzH035l2IYSeaaOpm8jvtMmWz7TMxw2ml8nLuTshQr9G7PK5zphJ9ZcmQjdJphvXj7K/pkI8/nbj/f5iurURf2kDfgNE3NKgwzO8Htt4qW1fb7uZgP0Dx+b4eV3SbQc2jzLbA3cneyu9XxfP5NyP803gcVxfr4CTZ7PzqUDmE/Mdb4VX6/tEsPWpMrzY5gtr8pNkfOn6cY19YvpZgOP//DPU8VOTv1f+BqavGlgGVNyFFUtIQAXOcUUKRUOKSBiRUCOmkHHFgLMelRYmfDtc8W47IzBjChmZA85xwxUDAhjnuOEcNyw1Yc4DTlEY/NYaWxiP+7QKSFT/zlxZw4dE3CVp68MWcEqdhMcA3FwS1prwZlhQOTSz3VFVNlPI+HE7YSWZImk/CIh1QO4uifrMQp8YWJ3LgL91esRjnnDJI94OMz7kCbkGnOLW6vhxPWNS0HyKG7BNqCAMQUJ8TCk3YBnAjeDIykt9+SYwN1BoxEcGKjIHTDHjmgdMUQC4hDVJKovEDb0fVjyuE+4HmeNcA+6HFae44dfXNxg0fIYHwQaa7dqchwb4ko7ZFDOWIn6QK2QD9v3pCQDwV5cHDKHilDbM2tc5p917+zxsbSwMEJvPqpE1GbMvM7Ux20rElDKWnDCl3EKbtFApAN6MC1KoeDef229ribgfV2wl4n5Y8X6dEIkb4DbAbMROMwnZ0SllPK1CIpVCxbenK562EU8YMcayOyAwQG0A3A4DDAibjGMsrb+2BuacWn1zFkblp3XclTcwPcbS+r8q+FydX+6oDMXGWDtvCech46p/1xJb/FkjkgK6v3PW34kYD9OKOacWDzao7/QQKj4soi4sNbSyt2QBgGVTxlz1IQY6MVXVfGuO+OY8I4aK3zzdtffmmApmLU+6Lk/jhkCMy2IqSzRz6+zW0t204vF6wjd3V3yYp53Z+MNpkblX0DQNGR/mzjJrRF7XdcCYcqvbyn9/d8UPl7MQPMWKacjNP/rNtOLd9YRaA8aUMa9DI9EyZmcPLmOsiDp205CbC8Gak4SIUSBnJuhjEibNZZP7goFG2mTswCEIi/Jp3FpeQIDypLF/zT9+zbEBQguLYt+/ub/isoxtnRiB2WWeGimX+a2fpxVP1wkpVUzDhqfrhPvzgss8tUOEadraPJpcYqovzKA5B0xj3vnDp1ixrKmRfdUaUAphHAtyDjhP/b0UiRvTs83VOOZ2CGGHB7WKLABaH6YhKyO1HAz0eLviKz9fRzy8uaLUoIdL3GS3sDg5hyaX+df7sD3LPGAYhRX8uowy96EixYLrMkrM4RrUr3zPkmsEZKN+F+bY0twGorKMbmtSn3K5ZiypYuYv92TJsfmwCymYsoIr6VneorDSLgnjlMUPXA8ZptOGy9OEL01pKMjqoy1+5HYAIMzI5ocedSwEaIv8acwwt4SoB33mF26/2UEGaO+2wN7HUr+P5019+oFaaBdepx1QUS9rMYZDEgI1VEJUn9e6hd3BiMUcJjucqep3XqihH/M1NyZXzuKLG8ayP1gpAZyNCCy2/jVcmLQNT95l/pIMuWYHB+YL7M3j7fDA+2VasgMGD/x9FrPOKi5WJXXALVVQ8zltyWTz9YbDda1nV+6lAwj/+/GwDYfvh7871tQbQLpVc6v/rx0aEECVb2JLunFI9uy7v3ZrXm5dx8tnLj8LsPyFJKofz/O1pq8aWL7fTvhf/upfw+MyotbQNhzLlrDliPO0NnCzbBL6Yl5lg2IbpVoJKVbMy4BxzO2FOKSCUTeuT5cJ45Qb5btpA4bBvyBllaQkpCyjhlpY14iUnJ+mvgBzjhLOQ0MaWLu2GUih4roMbTOWlLzFtB+NeVVf/ha2wVhB8xbx/9x/I2OxJpzOSkNeAtJQ2kZ2XYbGyhlVQ+LZMmMqO61DMW2XaVFqaO3Zi9ueAo0BdY0IiVs7rPHU8hblRZgDKAA1y4urrhEUK9JYsC0JacyNkj1NGSlVzJexE8C4h6CNn8mQNwkCby8uGzPbqNU1AhX48duzbDjfnxqTqGlUyrqPNxhH29yoJqN0NtEwVKF6J0YahX21KPU7l96/MFSp14gx9CWdxoIQK9ZZ7Zv0ZfubqbTNR57FttJCS3gtAgVGnSMosbRhxDWp4oe7OwnHMSdQqjvNim08GkuqnvTvvhMQhiL9i0LEY+Q2YSygAJQ54odzbm2YlsP89sJQUY1Qxgg+jEiEqVHI23g2wpglgqYiZdRUHalrHkg3R6ybJQB4d85CtmNU75Gl35FRr2bHjE5AQnDkOZ1lldcgayxVqYvQqesrhJxkI3y4Pwlz7/uxa1MGFsITQPoaAJqKzNPVPX4tfw6NTfTxVMCXhOv9iHpJO83M091JNmuMztx67WRAH5S8B3PEkxGalP5mvjxMKI9DH8dUG9nM+3OW35jwpMQz15P8fRrqnjqfsSPvuYy1+2+uQeq1DaLOy8UYbpfY+rpjc7VnSCHMw0nqWGK7djWW1jXgOtY9eY/fYBKwzIOQJ+m9ehlk/vmibaO3PZ8n4ClhSRWXqQJPCev9CL4qsRUT1kkPA2wItn5vmHzL6MYbEEIZI6phGQsqhE2JhpZp2m9GtyCbXJ2rRQlTmqy6MV3HsV+rhOtU2jhRDj1MQQWWsYKuEe/WKPfIM/IemT8qQm6DTFiGKs+UpPdqJdASsI0V1/EWeY+7j5WAhjIhj/Isoy2AA2O1eq8ROXUZt6j3xxKwKfHNFtAIYcBAsVAYa8ByGsQFQO/5nFjGlABaA7ZUQUvAPCXQFqStCmynAfT45VufVdmCAZGnsbgGIMwBZWAgE7apNvccY8bdTnIITpmwDdyIe6gQtim1vLYxL9G5YWTq4IoBqoRljqA1yKbbtSnrB/2Zo3KUsSKsAdWIhdQFReanm/ECEBejRt6jvw36bFTTTCZ1SRnkPo46V3XU9y6jMQOHAmEmXvrz0ExUa5KlQ45JtzEOA42gh0lcb7wvKQe056eNZwM29sg0E1cz67bf9NElwBLPGHOlAfnnTd5VfPnZ1+v8jrkBy+7iwU6eZ2arx8/210yRGmjTDpDriOWr+teDPtfX5hLkZTmC2Fug9Yj99HfPTPvR5PtEJtQreW18Dn34ffrlp68aWH4/XvAf/90/w4/5DhtHDCRau61GfCgT3sQFmxq3b6yayzKgcsB9WtpnALiWAQ/DjGsZRBtDBQMVFAT8sN7hTVol1hj3TdUUJHTJ++2Ec9yQOWDUNiz/qqa53hw2EGOpEblGfDtesdXYTHft9ylkvM9Ta28IBYUJWU1VzTQ1UUWgiikUPOURFYQxZKw14UH7mDngPq54KmPTMAaqqBzwYZuame8Yyk1zWtPQAsCcB6RQnmks55J2GstAjLWkXbgXCfsRm4xzGZp21b6PoWCtosk6xU1DxGzIHJFraBpakbs07Yhv14/1XNKOLKkytXKBWEyjmfCH5w+iwVWA6evcHKmTzYWFqbG5MK2uaXoB4Jw2rDViKQlDKM0c28LVrId6AzHOqt228DVmSmuaH9P0em2jaQPt2nUbWjgbC5lDxPhmnHHNA7tbDQAAACAASURBVJYiGs1Etc2rmSOb6bI3+/VmuaYlNW0kACyqmbJ6zsOG6yaa9dYHpp15s40v42BKrHXa+tpqwBQLlhIxqbYUQGN7NtIsa4vRTZTPg4TSWV3fTNO6qNbNNLMxyB1lWr0jwZaNhc15M8/WfJUJpyRr/PFh3Jk2r840NgbR2tq49etmNk2t7JQyLuuAu3HDklPTxALAachtnmztWAzZoodlQbVfwiiNpglklpi11zdDCxskh2gi55gyrm/0kCpWLFkO5ywkkGlqvew2J/Z7irWFEhLtZic7s1izxsDdSMv0NzOVLk5zvZXYTJ39NbNaqO4eYHST4vO4IZcg5XXuiRjL26GRYFneFCrWB43rGAvmh0G00Q8yLsacDfQtkWlivXmvEYv566aBNG2jEZDVSi2OsJnfMneZrE07YLSxBNBksbYs9rB994RnKVWsa8Q05Z2W1pK1bfJLfbWTbbG0k3NAjMJevd11rahoHbvMRlwmDOA95BGIW/ilcpZwTC/ls41tVdcIb/ZdK2GIndXaDtfanBY5ADWNvJk+M0QrnsfO1P65KQZGvVcCN4ulqGui3GmsWYYevMkmv7lHaN9ZzcvNNFoOcftvNifkNtOdhKt/D6mCT31t7Ey/d2OodbZQT9zyyaGk5vFtkMqvhwXM2H9ucqAdtBQ7TNEDJSmjeVjqKhbKyAMpR5DW0tFcW/tTgT14dGNiLLxe2wigaSGfARWry8tzC9RY/oNMRw3aLa0ak7qcvAQcb+ErB15f0ha+pNF7UbPHQOEbQh/a3AuH5/6VLdsrKO/GGN7C0cDt7n9KfZ+dfmmg9Jcmz8+YvmpgyRDA+BBnzHXARdkw7uKKN3HBj+WMgQqWmhqonIKAqKc8KSDbsNQB3w4XXOsoALFGLDXhsZ6QqOL78YKnPGEIBQPl5rd5LQMqCG+Huce2LGJy+yGPiMQN5BkQLPrwHKhiSgXv1jMAIAWR0/w+BURtCAo2r0VMdJMefVUODbTVGvFhm8T8lqE+nxX/7/UtxpAxxoK/mh9wl1akIPE9DQyd0tbA2SWPGNU01UDlWhIueWz+pmPIyDViVsaARBXv1jNSqJg3A4ahAVUDlZUDVgMBVJuMBhAlnwDbUUGUmek+bVMDMo/rCUtJuBvWBhiyticAZr/pHZwJLiAv66dN+rMpEAzE+PP33yEQ42FcUDhgXoe22TczYwOPl21s5rUydx0kLhpzk5nwfpUQMoMCvKRmtUMQX2ADQ5aWEvG0jthKwHnIsgnSzfIHNWl9P58wpdwAlAEiA2isAGfVumwsCxPeXb7DkIpsmnPajYn3gbXNW64BVYGppad1aODFgMqYCq7z1MxTn5YRQyq4qgmq+YcyU/vtaRkb4ImBjY0fq764m2ljYMzrgBQlLmuKpfmXWh2AmKvanBj78a8/3Iv/cazYdBNlIGRMufU9a1kDNrYJNNA6KDiy+LPMhOvawf28yRg9Xk5gBs7Thqx1zFvaAeytAI+Xk9Sr8UhlUz/s/FYB4PEyYRozfv3+vpklWvpwOblNvYCHYShYs9R3VTPblEoz5SUCWEHu+w9nMeskYN0SPtTuK5rzHcYxaz0DUqp4nOXvvAzNpNPGb9Eg8DFWXOeh+ZXGVFsoIaD7fc5FLB2iAiYDXwAwO1/EEBhzGZs1hYGca5G+xVhxLWMzQ21MyNRVBk8/nkFJQh0x0EK0SKzXvv4ZYgoYhx6cPY4Fl/cnxLG2zdj8NMKbFlpcVQMPLWQN0Ez1uCgAMMBEaPkoVlzX0253RaZB1Omey9TBi8s3W/xQ1Yw0TTGjWTOYHItqtbcfJzSNu/eDNHAQWbSwkbFYPxxwwFCxWdzTpHXYZt5CjZg82nbOqllVzVdWudk06dqHXLEL0yFmeCThWuyZbksvQMLeEJoGhiqhuhinNZO0sQapU80nWcn6vjTxFqReiPmpLCzVriXu7NWZOgmLrQWLSRoAdnFNJcyK+v1aGcZeI+SuW7mwiMaRGBpnUteggQsDpbYkczfpNI2mhejgtC9n5pXGWsyEThCjL53Wbt2344llbP1YeU9k47WFrfwNxNHinWp+TzbXyWhE+B2wOlxjXVPAC6Cwuv4cfn9RO+fRks/zAji89f2ldJzHF9Nnga7b/ZcGb9f1LO8REL/U9hHIf4KMr4Lj1+r/ffrFpK8aWP6LywP+8Z/9u83M0AKaN9v/Le5XqZlNqtnPzvnfzNy8aZa9eO2ldCQ2sPpS7fVV6mZyjOfEAYB7MXA3lwpoL+huesH9IX58YbtDJXIbA/8igNGs2+eIbu+vfeHoaO19nurqAPpDwZgldw/Tw1GZ/bUywcpQm5+dmYWPl2VjXJXFL7kXW1D2X0Zj59y/RPq42LUdG9vx4c/997jIj3+pJj27h7/Jb0PhXqL+xSovzr5GmtyuX9ZPciYp/oFLum7emzmSvfwtdEZFY1xsL9njJkI3cDsmOUZj7KwuoHVr2q8rt3E4vkDILf9b7Rs7nQ8l0EyPcHhxWx9dc7u5hBtbK0O9PvKyeVlt01PEAtGHkCCWv6tjmtzFGPNzofO8KFMssdbHXSabEyZlTGTCk8Xt0zqc5dduI2T17tJhLczKeL3as8CyZTmVt8cXcbe49bKvx42afh42QlZGZCq0e0wNBai2yayELTBiEWbmUAjVkcZYnsBA1XwcgKjjY9aYTPJcAZNc07VirlPWtpVrltFFwzla3/Uak8xr9GZrN9bAoGu+PXpsA5vksd36oOulDlCWbXnGjMb+a6ljzNaH3fOB3PPB2rT1+8IG8/9j7116ZVmyNKHPXu4eEXufc+/Nysx6dKl5CISQgBa0kBBSD5gyYcS/QEgIJP4C/wCJYfcYCYkJI8SAEfSUFkKIrmoqi6x83HvO2Tsi3N3MFoO1ltly3xH7PDKrKk+qTLr37HA3t5ebu9tna63vU9ZNOwdfvNfQy7cPjC3zFpvkpi7f+/9igWyf5QbqdvW7/hw2huS6ex/Wfm9aOw0Y1OeS34vh9rjZ9ylJPlPWpi9u239n5hvn85v3kT6nNZmH5TMTgyPf6rRjtGEfteDKzL/9ewam/dr2VpcBKrcsV8xw7tr48Li5F/cfmzliXUmdeXdvr+N+7CRcbJ+03zf7IgzI+rfX75f83eqgXg5MWfs6gNvfPDnPf9/SZ6Q2dn298Eqddsx27+bPAjovxnFbxmcDy30d+zbv234v7dr1WXXYYqjfX/39Wekzs2/rvtPmryx97e1/LX3VwHJKGf/aH/8VZnF3BNAsV3rP9kycaqWxGpB7dk1Nypq5Z6Dcs+6p6xZ2dWo7bnwP7m/wGIvRnqnT5tG010jkY2i7+5z/dl3qlnWvbNvHvd7ivbQlgrutj2ndq+wOvD2+b2cTEJdy/GYldqtzfO5We20/+NvTmSYVZN8sdfd1cHqtlmvqbWXv2kg6Rraf9mOk98zGYJgsDRDvm6b/3pgbmqqxrNxMNz6Kn50+8b5sLrlRl50HrFPpNtqeL+qxb+n9BsZu0bO57taGzX5xc2ssbi32LXHDrTbaMbDtfO3YvgzbLtu/OwvJjWD25lpsNks2fd6f228Y3bq3tzaWPvKu+Gja34Nb9+XeXNVj+3vy2j11kHgieb7JkHns2wPA6lS+mnQ879X9sf7YY7fu8cee2/192R8D2t+v3b79I/bqYtQeu5UcXmgu3qrn1fI347J75l9LztzTL0ivLmo/Nif31d56j9xJn9ziT6n73prgxmP8yXXs66LbE+FVELKbfDfr/8j8+uiC/d7H81PK+dLv4qeU/bH0Bdd8cj0fu58vS96V/crs/Mx2/0aA67d0f/7aU1sI/n6mrxpYKlj5k9M7XEvE9/MRAcC34xmPacY///AdBl9wyQk/Oj3hr54f8JPTE3L1+NX5hENacYgr3s0T/vjNe/z8/IA3w4ylBry/jvhwHpFSwb/83a/x/z094jTMCL42qQtlTvyTt+/wbp7gHOHpOuIP33zAzz88AGD2xedl2ADXXD1O44JjWvEXv36LENidMFePUvi/vAa8ebwgBnbtO88DUihIsWxi4Z6vQ2Phe/NwAQBc5gGHccEP358Qx4zDtOLpw4TpuGCIBc+XASUzQ993b5+ZtRLAdU44HWY4Ry0O7nwdhGGQt8MPhwXLGoXhryLGivmSMExMfBRjkbIrhqFgWQIejjMu89BIbKZpxfUy4HBY8Pw8CbFQwOmw4Pk84uF0xbxGnN9POL294vw0Io0Zacg4fxhBq8fhmyuWJSLGgpw5LoqqQxEJBSWGOTzOAIB1ifDiwrdeEsLALo2H44whZnz/Z98CAA5/9IR5TsjXiDBlUHEYD+smfmh5Ele8A7tTBu2zI+RrhBdmQno/gGKFP2ZmzksV9RIRjhnlEhAOBVTRWASpOtCHxC5Ub1Ze6CmJ0YcBiAR3DsCbld3cFg8MFT4yYZBa1dPjgrwE1HOCf+CYzXoNCL9OKKcK92ZBnTuhCRwBlnBESG5o9d29Tqfv1TMhTXFwq1iZDwW4BrbCHgr8h4T6mDupSiQmMMke7uJBpwJ3CaCxcjmHzOyDnpgQxFEnF5kq8ByAUwbeJWAq0j7JOwgC0vqViAZA+CGhHCu3r/J4udXBzw7lsTDoEGIYlxkg0FibBd5fAuO2Y2FLyeL5PMnfidgKLcfTzyNAwPqTtXkzuEuQ6wWAFIfwHOELsL4tDci4zOY7SrVdG99F5O8y0q8S1jelux4CSD8E1ACQ6tReHfLb0gBMeGbfs/Imwz8FJtoYamMnHH8RMP+ogAaCP3vW4RVijvTkMP9BARy72pVTRXwXUU4V4TmgHGoDbORZL9gVh3qoCE8BdWR93nysCBd2QyQPvjcAwtnDZyCfVFuYQFHKWhwogvU/R0J49vAr51Wt3vSBraLlSIjPDmWiZj1rWsWJ4Aow/tIjHwnlwNfHi2OL9Te1eSlUIVFJ7xzmH1XEq+cx+K5i+D5gfaTmDZI+eL7/YqnLBwKptm9hbcf0zG6GdeQ2hKvD+sDaxuR4HGoA4oX7E58UFPGYlolJTtQ6FJ9ds5pqW2vkvut1NbI2aTlIGaqlDDl3AZa3hMNfedSIpl+s1kHVHa2DlDPyvyA0XcsagHzi4/EMrI/sBaFao2VC0y0toxwbuW5HQJ7YOqt6r8sb1poFcflh5mvyif9WLc08ofVTdX7zAUgfwB4YCaJxzHUBQD44hCuQH4D4xG1TzdPhPXD9gy8HlvEMrA/8uowXNLdJV7g+1XyNZ3St48jtT0+s6VhHIFzZekeBz6cn0bhMrpXJWqdSxsDvEF141wiMPxDWI2sJ1sQ6pTWhWzEJ8IXPcXsJZVRtT7D25cwFlon1XpVoxov+Z1ilXdEhXkj0TqXfFU1PFuD7rWOkngJh5flWE49Xljlqrd1dm1bnPbXrHaFp36quaR2w0VlVTdAa3NZN1oyVJedxhGZVtXqpqnHNeXQTBluNUE1249ezbqXV/FQLuis8js0qT9joJVtLdbce9zr2etKusD6q6mo64mdTdXFdJeiOfbfm97lmNcVfJNndVybYzQaDbUM1/bQbd3Z4dsde7KPuN/taxu3Y9nHYWkm5nN9fcPY1p68aWCZf8JPDBxzCilFVwAEmpwGTlQwhY4wZU1jxzXRhSY0A0ME1KQ2AYxy/nS6IviIRg7dDYjKO6AreTtcWjzcGBi/+KAslX/F2vKLCYQwFwVV8c7hyW+L6wmqnBCcehG8fz9KX2mQeOI/HSWQJtI7kayNxIXIIaW3xfEsOeBhFeiQycUt+e8EYC8sNADiNC/fHEGtMMfdYwVAwpQwHIAmNfPCELJT0WvYQC/LgmxZijAVjLFgSl9nIMEJF8LFR/KvGYfAsvTCmDJyuLAERCrP6HlyTf3BvgcOwAg8c5+Z9BT1w/N1hXBrIrkNucZRFyC+U5GIaGFjFWBoxzBJLI9hIErMXvmNAPQgLbk6lxXXFUNv1xVhhUypdIkFi4NZYEIVRdg3UwLeSYqyxIKWCHEOTO7Aahqsw3aahx9RFoXL3oSIPTLNP5FBH1vELFpiSQxoy543UqP5LqOzpPRbElFECdWutIyZ9IDRtQOcAGtBi1pTsoY6+W8cFb4ZUUBIDrhgrciAkYVEGgRlkxYJfRyYSKakykzI5IZSpcL6iJm0HW/19IJRYecwCE3/AcXklMMOnA1CieCIIky0g+EzqIQA0uLb5ECYhYXGEauLfvLrCyyIH4DGjCtAkZB0AaPS8UeEAGphuf/0ReCE3lb7gGDRuj89RdSiJUAhwYyeCUZ3DxqwMIHuCP2Ss3wJuKmKx5XMr0P1MAyEfmcVWLbslChvwUFDFjdaFbvGdAdAps9dCrBwrJmCxHjyDYUcok4OLhOx5cyBHzwDXUNvrOCEQM28GQj04zj9IXocWZ5bF3RyRUA/YnCuj666lgZCTuI3L/IIHliSLi0RYBt4Igec1UVXhe7HOXx1AIzWG1HLkzReaCuqkefmfOnjUQwGNHuXAQHkOQFWyEceLY8AsAKOEE+jGgUcrl2QuuexAAyFbl1AHlINjMpORF526gKuJuqUVDJKaJ4PnxRR5vjfaLvJAPjFQ1Xbl3PuXjw51qrg438CtDe1w4sJMkZAPHLeXT64BJgrS5oHPh0cGa650gKEAm5wszqmDUUBAloBYV+V6XWg7ATSVQYPTMI7awRmA7socGMRQ4HwEvn5VVtJE8KtDGaU/A7uUUyCsJ4f88OUL0lXH2XHfFBi4wvUpOMknibF0HRzmoxOQxu3T2EPy/AzDGdABnjttfsbtgpwCz4EygK3rAVgfXAOxDVhV18ZwXRwDdon/JA9uB2RuG7dujseU+Ssu34vMqX1cpc8QYMfHwgPaZojOnxoEZKYt8CPZMNC+N0u2eSZ0jjkBZTYOkgFnb5dlZu1jtd1I4fMdfDkBcw18auiOAT/6jOxTN7C67hq9GX8zFgY43rQK78AWGeC5yaNAS92KPTrQvAPAyPV3631CHnnXGIvxa94JG9B5I93ycnjVEm7K/iRr/leMK38jy+zveHL0FSP+b/6Nn9A/+u/+08b8CQBZyFuUSVOBmhJvKKvj3lXVskEqe6SOzJIjoilLk1oOlY1Rr1VWRGVqjKETglTqLra1+maRVIumMkJqOdbtdu+OWsk11sXG6kfdDVcZGlUfzfZXAantt7oM2/pUZ27vPrlnX9Rk26iurSqsbtkFW52+C6/X3T3QY8pkqUyFwJaBUIGOJQHRtHcPBrARjFfhdZZgISxL4AW+cbvcC8I3Zj/9MOk5AX96XMuwQvAbt07D9Khj7nzXnoO5z7rB4IQY5Z67tB1HYPudUtBixeHbfZP2713l3K6ONv/0HHp/9Dp7P7pLMlr7N2Lxer3W2+a/nkP/SHs0ofj9s8CELZyvHdu5/m503XbujS/Gk9CB3K7tN5NYl+GAuvru4u36Ne2YjtWNsjZu4jpWSsRi7o8d73atKa9JyVi9N42bBppGXXshmC9+k8AhaatYsTdacDaZGNYXsXU2xnO3aNq4520WV7sViDNjpW3S/ijAtWWTKVvj4/W8Xlvcy7o1Nl4Wta3fVoPO24ZqXTqoukqm28f315k50vrobvTnVqI75+6NvS6MW9ArXi7WZLFNlkcA2Ixp08tz2LgN29/NPXjfjjbXpFzrHmzT/rh5v7b5X10D7taFV8dSyWkYMHTJBAYm1OQ3viRZUPWi/TaE4VPmvg4f4YVUhZ02NxehhGaBau8+27ZN3ttzcT819zGdL0DPfs643fH2nGF7z8lk3T+r+3HR3wrYTX2tTffmDr3s076Mfcfs3Lib7jwv99KmDe7OPbHl3vr7U5Jt1/7fO+36pLI+I30SWNTyNxd+el2fXMcnpP/jv/kv/ikR/cPfvKQvTw/f/in9g//oP/+tl/u//vf/1d9634Cv3GI5+IK/d/wBc424lLSR5Mjkm6SDlY5YRMZB2VWXGhFdQaYAD2ryC9FVRF+Qa2jyFxq/qVIIe7kHtYAptT0AFPII5q3SYjoFACfPwLDCtXyax8pc2Po1D4BmwYyuYi4RSrlvJSqUPVTlCaxwvYrd34oL1X9t/KjKRmibFLRqOTZlU69aRYHOZqqyFRYoWwbXFEoD9SoNwayghEXYV4sBUbadej/WojG1HdRbQKrgfRpWEDnMITbrrI7zHsQoE+mtuFxlTFWpBo3R3dx7My7A9nuirKdW0sECy6jzy4x/67OUpWDcybUM4MXaXVl6Qc9r/fs+3oup3VvfAWy0XdUyy8Lr/Rhfiw3oVGBpgbDdcLBls2RCl1OwefXvvqHBx1Q2YV+/lqebA037FNtNAG3DXirCbpJo+2plwXsiYI1hU4aNY9ZNC+cIORsAqvfCPEKWXbUosNRzob8rWj0y19W6DYA3ITybAOw9Vf3YNrfMSkg1ZnVcdT5VmUfVbrCZuWDfG/ou2awddvfKsje/WPgKIHfi3bDZnFHQbDdopHLeuOnFhlRQS2jHnFiQqrjlt3Ica6eqBq3+7UWeoQGA0G+QitLbzQm7ydI2FHRjoC2+ZdNNwLpuSDTringftP4qgDLj1MbBlNnI48z5jUh99ixcT6xbu4lzd1v5ic1mkZTbpDA8tQ0yyLMAR83qT0AH6Ar+gQ6cdQyV8M6ec8ALMrh9jKsDj2cDlvKObWy6rklg6MYKCakeiWs+rTdMT5+avLmfu+O4dVzbTthomW5fwOgbNPeA2w6cbJh49bdaoO2GkyNj0TPEfPt6dy6XN0HjrZjlTZ6uXbmflw3k240aua65VO5ike8CCZ0j+3Eio31pp9GGs4B219C2K3YM9mnfnj0YuwWWbhz7LBD2KQBsv2i7k/91YGkG7JOv+Ui7vjDvbwVI/ibX/k2k3/X2/QbpqwaWk1/x9w+/BACsNWKmCA9CcBWFPOYaRTMyNlkKlQiYa0Suocl56EJIU6bQQJNKbGiq6JpsuQZEXwRchaZlqXqNqhWoUhWa1AV3JY9K/qZOpWpUAmjyIsnVdo13teVXIK1/D77gLHqHUeJCVfYgm4WhylHo+ChoVgCtUh46RtyWrZ6nHRftp4JdK1FyK28m37Qn9Tr9fS0Rcbq0dvHGQEAhz67HBnC1coklR7S84DqIvgf2vSNcMuv2fXc4i35i30zQfqgGpY6P1q357BxTQK9W7bZZUALGmBuo3hM/5coW5kNa27EUStM8LORwGta2ARDMPdGk962QQ1KNQWkPQsVpXBiAWqCyK0d/W41M26eNpIvxBAi+ssvzuLzom5ZnNxvsZoK9TxYwluqQJtlEGF5a7jUPW9jNWBaOa/XO1s99GM34Wku5NxsdJPdZ3ant+W3dvt1754CH4/xis8RuKjSdzam3Yb/OdDJWaVyx5IBB5A30XC4dlCpYH4YeDqB6iGFcN7IamtY1YJx6vxQwA0DO9hwaoO7eBWUDSjpw79Zy1Rm0IHRP4BXs+tKcs8d0AyBJfCbrJYp3QuR6fOz9qtXBma9akfhw1zQCZdNqzN3CK9fXEhCHDBBL6cQho5aAECtUwqTKPCf1zDU6huqxoOPhY9cp1DbymJEAMQeXtsAVYCCp1zoogJNzMpe8M4AaaGBHwaAP1ECigjU/lMaUvvGgIACVV/ZqyVamdZ4sHQA60Wak7AzglHYwNXBrj2X45otltSixn80K3ia8HPfyt8fGyt6Am15bTLkKQI37L8Q9E+pOKvIkTuJ+vzgZ91RrkXTFd6DnwVZRxcS6cbb4Bo5fEIdJzOBdS+ceMDjAX3y3QjvAr75bbrliWP8Uv25jEFmipJfXQNiub23+ra3YLYAzbrFcj3nuBfCSB8Lit6BT27VzIW1/a5K9F20L/9M3HG4RQekw0G7s3H5sTbkNVN4CtfsKWv9elrMpfw9Ub/1+Jd0FV/fKfa08u0lws6471tsveFw+CRS+Bsi/pLzX6vgdSvYR/X1MXzWw/H454n/4f/8dEbHX+LquZbfk0BaBXha8utBXcKXzOYoFzbLFqqUiiUupWjzaolFj2lIWghteCKnFRuPv7EIUQLMweE9YjWXAulxS5Riu/YLLumhqG6BAR2PjZDeviuXU+doEoyGLP317BBPTZcFSczGlvnvO9dftMV0g+d1CBRBXRPeib3ZHnIpri5X9b9Z/Yz0yXSBR4bp9qs3SsAEa8sFqbq/iOmsXZ3a3U8utM1Pau4FXjLbPVlMOgBADobGUbtw77W57kb91oWDdCk1M2+ZD1GRqgLaD3xZH1PXjCH3h1j4Wcq2VzNFFhy66PG0XdPoQ3Pqg6wdo84HfLRZ1fPT33nUS2L5B7fVahI6XLUfr1QWjN2Nmr2k3BX3cNIvEVO3709zodgsXbn8fh7bLr7Fyxv2uuQE6dL030fsgmUPcTCMrQ/wbBdx2u7jVLN7I/4gOn1s5rrG5GQJMnKTlOhlXnRdAX3SrJuBuweoXh0XixFC20jQ+A6tIkegcclb/zs45yWOtEeqOt6hbnkNf7JLbkHBszgMby4LGPDkCiiW7MHF2Lsu5/YJJfvvFNZIjR2hyPjnRC3dGlzn+1VX+uybicY5ihXNoLpTtlvZXaWuzkp/AmcWtjfuS5037GncSDuT4XuljY9tp144b9zoZF43P28v9uMIxblFj2fYxXmrhE6CxIflQC7Lrx31hAqEW30bosaYAajDaikUJRsw9AOT6/o7Yl9MMllqWPGt6zKuOpZk7m7lR0IigNnIjK1BHfHFyGT3us/Z7D5L6BOTs5UbISSyibZ++ex0awNvcO/uKsMfRy9sTveyBVG+4aRttx03L3wMwa+0jh0ZO04q0edDbYvuuEic9htG96N8LAhvs6jFusS+A4j1L667vm4dnd94+C/Z53uS5535r0+790465O+dvtPWjUh6bl8Cd4+3ij/xuddzox6ttlGw3PvEvyvlI3Tev+dS8u/T7DNa+pvRVA8tjXPBv/+hnzZX1KQ+o5HGKM4IjvF+ZTk6tlH4MHwAAIABJREFUiE/riGNka89THjH43NxjH9LcLHzeEc55wFwiE/GMF5zzAA+CdxVLjRh8xvM6osLhMV2RKYgAesApzXhe+cs1xtysfg2QivVr8AXvlkkIgtiyaWNCleAHQHPpHQOXp2XN4to7l4CHgcl75hyRQsH764QxZqRQ8LwMOArZzzXHZhl6GJZmbVqrxxD4y6wWpaWEjRUr+dpAvFqy1hKa1cq6tqolaooZq7hgBl8RHFseB7HEDTGzm6tY2qIQGV2XhOO44LIkIQKqmNfYWHWzlKftU6sVUbcoTym3c2q5UuKiWn0jFnr3fACRw+PxilJd24So5DCm3K6vxORLRK5ZgC34V4slAVhFmF5Jerxj19sglhyNX7Xv0HlOoOowjLmV7R1hWdlFd10DxjG3jQ/va7MktfhgKX9dQ7NileKxzAk+FAxDaRsf6mIYjGuluouqGym7b3KZJftmlVFin5gKcvag6hFTwbpEpCGz+ybQ3D+J2IKkLLohFtTC1+icUWuUunXGVFo/liW2/jqH1haA8+vGiFpt1jkixNrGX+NLa/aN1Eg3AyrxysmH0vqlVr8YeTWlLqE6nqHFw3L+fGVW2HRcN5tPceO26lAWRiNhyu3LrGsHjdUGgLJ4xLEwQ/HIZE4NN16jbIowyKPsmWSIuE9FQG4YCzMGQ6xYskAr5wh3KKL367t1C0CZPZ9z1NmMlwCf2OLlxILXNnSKRyWw++jqxY3UwSVhK1Zrl7o3C4OxG7p1r52zG0yxoq6cF0r444ktaAAzDS+ez9nNHAN86RyY7VaIopg5GYAwEvPACOBdPGgqQHZwK//troE3CiAPqrAHt02HaOqW9rmVN57UesXkQztQLvkoENy8RQwUabvBsSpCRdt4IEfwupHhwOB/YZKgBs7MJpVbmbwnPHsGwyrx7Lf5KHQw7Rd+Jlyhdk5JcWhlIh9XmCzEVXRGT+1D1XK47Q2IZu5OEebb1g45XhN1MKjl6oJf8tdIjUlUCXJccaIly5sWXnRJ/WI2MTyAq0M+fvkK1AmDriMDakjb2sGb6gbzHJNx0TYHAwqlD34RAG83gIzesSX1afPxwoRBCuLVIsngXLJVxwRekPOxbwDB9XhTZW+2ZEQbvVAPkN2YUOKlYECxEAf52YBMIa+hIKA+yXjphoaCcJhydcPA9bFtZXls22nObxyptA7wpordtNoAR31XNODb+2YBlC+78k09eyvnBpxSv/+fAiz5YpNvF3JzCwRvtF9vAWCpaxODewdgbiyWd9uI9k563QJq3mvONdD8uuTMjTo/ERB/NbiS43D+tlvx15a+amB58Av+wcOfs4sePFYKWCkgyVtqFgq/lQJGv+JdPmLyK8c3wjXQNteI0WesRgn7Uvha7wjHsOApj4i+IqCiwDcAwWUz2Cvy9kiuYK6x/a1t0PIUCCVXcKlDc7kFYGIWA7PQgjaut0ncMm2+Sq65nALdbfbDYcIYMqKrLQZVAewqbsCjLy2utJBr5QPsrruSF5dh+TC5ikq+ufd6R+1vAJtr9Xd0DP6su651hVVXU/09SJsUfKobr7rCVvIYBGDv3VH3broaS2vBscbV6jhFX/HtdEElh2NcWv9aGbsxWWTMFVhad09tq7qNAmjg15v7al1ztYxSPfKDb/GxOoYab2vdlbVPtm3a/6HFD4eNa7OOp3V7bUDS0au/NekmgsbrAtj8jrLxkExcs41LtrG89t7Ze9TiiIHGxJt8xVxCY09uDL3oSZ8RJd6yGxlkytVNk/046FhrUldYjQ+2rr2bmGTJox4SY8qbcu39r+RaXG+KHVBrsh4RRUDpegqNgbnNs2N44WJ7i8jLsjTbtJx4Q0f7aTdEqhmfQt0lXsd8H0tdpJ97F+kWZyn59cyeIExjLrUtNlkSML2maNy0bH5oLCzQY2A1raeAEGhDAFYrMzrvXYTzGhBTaXlCIJRT37zQzYytuae7lbY263hLPvUmaXGu4lmi8a/qXgszTlu34ZdmAWfHBrpBopsHMibmOioOUfoDoBFbOV1UazyrbLQ4Z2J2Tbylk02lsnreGNDY34oePyjl63VZXEedJ1RCA/Tqytz6Zt1c1aNBy203WECwxMA2rw49V8FtUK8Q+68suFf1+vjSpAAV6ItqHWvr3WLnst47Sw61c4VVSy6ZMjpzqemnASdudUBAy+Ny/71pn4JVbbut246ZBQr7fgh4V9CnwGnPEMt9Mf1SbwjXgZu2rVt3b4zfrr/Noq2AWOeO2fehzR/3j23G0SZz7IX16x5g25X/4voNUjO/7b+763aHblviFIDtf39q3l16tY5bwBKv1NfK2w+Y/N538LeUbvbh79LfePqqgeVCEX+xfItzGRpwC6iYa8JKngGfkPLMNSL50ix8SsyjgEbPK2HOKNbMSg5/lr/DFNYWJ6hg6hBWtqyV1Bb4Shg0+NLA1D7OzIKTKeRNbKGeB9DycHs13nG7ZeYlhjD6iu/no7SBLbjHuODdMiFTaBZb7a/Gbv6ypM3iX62r9pgFAjZWE0DLo8ct2FPwtNaA6GpbYGuMo/3XgkwlSxpDxrvl0MY2VwadzhHeLdMLEiW9NzqGFa5ZY+1ik9s5tjZWcngYWO/yr86PvIFgyHUsQAIYSAHAWSwnt8CS5qvkUHLa5DtTaot0exxg2RnnCOc1tXe2WnNJzj8tA1vmAFzxMi7yeRkaoLqsqYGjSSzDz0J8ZEHSPh7UtsmCheAr8pra36V6zFKec4R5GRDFQm6JpC7E1jj9bce3GJdmjZHU36Uyu7FarXMJzTJt4yH1OiVLUoC35LBxgXeOEDzhuvR7om0HXsZbAsC8Dpv+6ljbONc5B4yxwLmKs5QNOXcVy7WCr+ArvK+4rv3129bGZORsQsF5HjDEjCXHDchPAiIbeHWEeTX1tranBmqKicsc04p5TY1syZLpDDFjlrZ5GasQKmbRi10MgZLeK+dYIkkJk7wnrBIeoOBBx1bJiyyzM5FvY76/B94TsujE1upFfgfIuYcdWFIla72fphU5hw0YDaG28jhmUzYEB9bi9Z6QxFKuxwAA5Fgih3/0GFADvKg6hCibWeIWH0MVECm71AJwlVSJ5YN0LNlqT2YzIMQisZoCMnTOSz16zieuB4EamCWdo0JilMbcYk33RFz63AQB3UE2HhQkKjAPqcJPLCfkInr8qXlPUhXipuoQDhqOwP1zozxreSvF4wZwOdnDqbyMQ4/1BBrIpOLgp9zCH6iyhbsxOVcHN7HF3I3U3emJ413Vov8lyQUmQtKx6SfkuFrAfe0bDmqZH+VYa4/rIG6ifg7Sd4sybsUhHkwsbHVNZ/cFmLGhCi9iXzXP7jqtp4FNB/K9/MYIvPsbEHC8j63VsnYu3BvguxlOtwOZWoeWYdq5AcQvQYvFcZrab7WaVmwsfx+7fgOo7hYuF2+sjg1xSrl099JXweMe1H0CALyHA2/CvNdA5Z2C7DR4AfL2v18BpZ/cls9Iv2ug83etPb/N9FUDS+8IR8/un9eamjXRuwVRLJfebCWr9aqQw+jzix34KBZCdoet7fwUViRXmYxCgFkSEBkdGvmOTaqraYl2AGz/hsMYciPt0Xo1j7VERVeZodBSHsoYWC1OPRbl7T2EAl97nzywAYsDlU2ddvefy+p5AbTdULX4VcdAFoHBnLr9Vbjm4qvuxQ2soor1TY4T/+vk9wA04JN86YDVk+hOUutnA1VQgNHBpiPWHbSkQgCY0Ef6mXeAcJTFlHcEL8DQWvAANItg3gGvIGO7B3rKTBC0XGXJNQQ+mt8y9jaLj6+tXOeoEfK087YuAAgFTtoCoIGfaEhzFJi2f4PRVJSxU0tfsH10hGKuU1CmvxGYyIrMcTNtGuCxLMGuxZNQA4AAZNHLfVZXaLbMdSkca/3Se86PfAeRHNQIKCOw18U+8XpGj0NmLrs8EzqDcAeeMfTnvMU7Sr1atrXoeV/7S9bZNvFY7i2J3lgHgqc2vnZeaZ1ahgKwoIs6Mx/I1z4/wu7d4SucM5sheg/RY8MtEFQA6j3XsmfkBZQ9t19nyY7U3VoBu5bbY6SpuV2zFZDac6rHmvXT6fVobdJ2bGLVWx9g2trr5rLRY7KlTP1b/yWzUtIY+QbKfG8XORjrqhls024CdSsjtrJGunJSsOZ21ylg0ThyJ/VZ+SNAFop6P82c2qf9/LOL4X5fbH5Zu0tsv+aD/LdZ5zopT1flmuS6fdntHLBdQPbXQz+0+041eSJTz976raCjbj/Vn59eWxDuV+i7NtgNghfn6u323q2fhP12U+b98m+267W+tLLo08q3QLC6l8cNSFJ3bpD8Tf3vbf3mbwssLHIxnAmgLj2zB4PuBXDd9RXoLrA3xsJi27v9N2W3tym9DrLuPYL7fJux2J+78dvdOnerb/u0q3M/jp9U96159TGg+bnXb859tFd/l/4G01cNLH/x/Ij/9n//R/CRiVxokcCRQPCxol5ifxkF6ux2njg2R10/iuOYm9l3sovi4IXBjA5FmCjQd9w8ODYGAB0Kx8t4iU8Zaz+XxGXDulFUtDgaf7HxOK6/2IpDHStfIzEyFEhIKMxLXog3XO75NUYmnD0oEWoEwsyizhQIfnaNcKRO1FyQ/Iom/KzxQH51m93eKiLYTohpWlyHxm00NyZqBAUsyIz2YmiEClHINZKQgyhZhsTqqMC1n12L/wgLj2WZSASce+ySjrP9ANRBFmYmVkXFoUG9v+mdhyNgfaxw2W2JMDQ+SrH1ImVHOW+0xJqwN3HfKBD3V8gzfO7xP9QNIe0FHmbuQx37QpUCEGb+1y8OZaJGVMGi5lKvfDg1zsevrpXjMhDPLI5dDgS3YkOc0dyTLImHiJ9bI7nPHK/jNVbNc7yTl9izmgjhym1U4W0KPW4oLA5loDZnQK7HZnm0eB8dz5oIYeZ5EGYW/yYvz84qsUOO29Xbzn0OV86vovE+67PC4uxwJDEpbkv6IR9SLwyILZ6qcHtAromgk+v3NF54vNaHTj7SnilZ8KDyMVdZ4F1TiysKaDv6YQHyAShnyWtir+qF55O6pfkieWQBVnSOjuhMjho7BABnIB91Tu3u81VE7R2PVR0AzPIcr9gIsGseLR8SV0YSh4ZdHBnAeVwFKHFfyfV+6DwgkvyrHBukPt/74xLgFvlX2qNxUO1ZfQLcYJ8JzkcHbIlUKhCuAE58PMyAOwLhAvixtzHM/TlzxGOh1yshkex1tmfKr3xvGvGJjIXG4IVr7z/Az3OLCQRajCK5Pgf4Xdj/5t+EMrjWrh6/yHXlAzB84Gexxu0C3BVCjU7eNYSaHPyi78s+18voEBaOmyyjgy/UxOtr6n2skWMva3IIM8lzxMf0uSqTg8uW5Eeei8HBZ/v90PcU4LK8FyOXq30nx9fYZ9av/JyHa/9uwfPv9QFfnMLM95OcQ1ipzwd5pnW8/GLe8b5fq+9sv5q56l/OGzufAX7e7YKbPJDOhDwqoRHPCQrYkkXRdi7WiBfvL0Dmsu/XaN3tXRyAsAqxTPtmEWrgsecy+F5xPrltEmNZA49XkfvpCHCV2r3TvrvNO0HWJoVaf8hzWa5qGegPj9v13YxVe1/tYwwVO3vus8ZjtutlB8cV+Y41SmU5h37M4Ns2jtr/F3Vid0xPUe9OH2dsx9PEorb6TAzpTZddZ667Ue+2DT0WssVE2nL3/bhT1s3YV3P8s5Ktw+HV9t9Ljgj/1xdU/deSvqD9X0v6qoHlMGT8K3/6C5zXxBIH4h62Fs+/vynNvW7JAUMsmHNolpZNXJiwpmrcTRB3SCKHeY0bDT0lItEd+HWNm134LC5jQHfX0rSJgaoO6SdlEzekO+rOcTl2J92Sqehute6qh1Cbyxaz0zqMYxY3MIcYO1Ot3ZXPa4AXZljnCc60Q+OBuswA4FTTT9lpSSj2RcPMma3CWj2ToeQAiBukkqh4cWFzocIJ1T2UEVbZP30FJA9VzwtucXfCymUq2yvpS0+sbO19V7o2W1uUKwutAygLqcsf8oqkzkyK4jx4N9hh64oFMGkIOdQiO/oaD0ToDLeOGvO+zVelf9Ub3TdloHWEEqgRqvAmBlsmirCR+qjnqL9crUXCASQEKk1vDTxXylC47jXweOiYGAtPixdrVgVsdtmdzhVpL4+xb0QyJG5sJPFPjSlXP+qBz220/ZQ9F+bGqUXCxEm5KNeY9nEDZONDx0PdzoQ5uEsQMCB13ujYaQyX1mtZfZvenrRTmWm1LO2XxCzNg1jarmYFGIj1+hpapR7fpaQs2m9tu96XSMDqMQ+V89pd2aF2lzJl/7XafCqDsdcK1DQWQJiQdVzagxMrn9N+ZsdtsWy/+4+8o94O62q3Z/rVMt2uPbtFnt3A24w7ocfymXixZgFpiy/5PVQeA110Sd26Eah5ATAD7yLHg5DrpAqXfWvfC5mKiqYLqHqBJOy8llXYZbetTzYXW37tmwNv2plFIln21LbYRWM2brFnkbrEhW5wmbxu9XgSV0mNlWsLTmc2QGVzo8lYmDHWucBkO/yebuzIuw2+dsyyWIuFt22A2rmg90bHSt9B9t63GD4mEtK+u0pCEEOtf23cFayom2PizcovTTXqppMCDTnh0EiH9PfGEgYwuY2Alg0AsvPGTDH7yL9YjJMDJTSwDeobrVtwQtu5aO8T0Oel0WTdtL+9G8GMrhtXVn0O3aaN1eZzts9989yY7RuAdHqcsBmLLUsqZH71YxYQ31qwt00UY93s4K+3/QVb8na4X1g9+0kzTvvf+i67kTaA0XzOOqjdlXuv3lv/3qrvhpvvvi33LL436/6CtHdg2Nf9RWn//Xgt/Y9fWMdvOX1xX7+C9FUDy1Nc8O9+9y9wKQNW8hsym5V803wEurvqXOILQhnVfFSiGI1ZVPfJa06IEjPZ3FSFQAYAznnA4DMqeURfcC2pEekwUYyJSTTurRxjuXYyHJlpln1W81lCGEvw0Vz7fG1ah+oOOoSCXJmoZwysnWi1HSs5zCW2spXYZx9jaZPVAt23Tfu1j7FU90sAG0IUG2tniWnWyo6tyjSrrrCEHoO4lIDgunuqvk/38ayrcTPV9qmLs7qcVnI4iq7hNceN66uy3t6KsVQ3TDtee1ZcG79YTZ3WjbNS+y43V8ushB7oJC7q/rkW3fzocYjcZrlHyhbr+FqdJylwnXreJjuf1HXxFqGKjWvscZC+uWzqBo+Nayxy7/T+2Puv42djEO39Uld1yxZsJYT2pDubtrpORqN5nGMHJa27b7TI/Lihadl1GvFio0c3mmp1zB4L1oHs49VjCbUNuhmkm0m2/ZZohzepPFIqbZNJ11Lsqrq9bxpDqX3SOvRv63YYY2UmX9qS3+iGmW6i8SaY27h+ap12LLjt23z2t82vG2d2k4wzuBbPpysNG7uo99dr3yuT36g2pY6DxhU6oDEPt3FS4islK6Luvullww/6/AsLcN3NlVaXtMfel5bHbNZ4JbJpg9zHAYTmpmvvgW5O2Xza/z4/LQLhcjRSYn+vvOONLCu/9OI8uc3GxsaVV6spneRHgaLTwdA2m75bF+K2gNf50ohstqtKu1EH1/vUBh1o7dzEMG7mMZdHLZ/8ls2dMnw5sIRs3Lb22AX93mpk5rZeu9kY2yDH7Th9NN0DM3eAj0oftU0XoG/IyN83LVG3ytwv5A345H7urrnXZ7wEVy8AnC1fnx3rAXarXZtjFpTS5jDXo+N+p6+2rR+r77Xrb/TjRfk3ANZHwd0nAMq77d3Vffua2/Pxk4HRp+T7TJD1+wzKfh/SVw0sc/X4F5dvcQoLVvL4YTnCgzDFFaew4OfXRww+45wHPKQZP7884jFdUcnjh/mAY1rgHeFXOeHb8YxfXB5wSgvmNeKSEy5rQgwFf/LwDr+8PCCFIgQ/A4Kr+PmZ/Wm+nS749fWE4CvOa8Lb8YpfXd5wDGha8T7HDYOlSomMIePP3n0L77osySqLoLUEvDlc22Jc2zJK3J8mJmcBlhzx5nAFAFzXiBQq3j0fMKSMw7DiZ5c3OI4sN3JZUgN9bw9XPM0jt31JOA7rBiCc52EDnA7DijWHxrjpHWFeI8aUm9VXFzUKCI/Divk6NlCqluMxFpzn1K49DCt+OB9wHPkeXOeE02HGZR4QY0EKBb+6Dsg54PF0FdmQugHbSuSh3woVoc+ywHaOMM8JMTJBxWFcEDzhz//yOwDAw9sLSvFYl9gYIochbwDHfGWSFCbdcM2aDQDrEoVsw2G9RrjAchxqpc05IMaCvEbElFvsmVqj8zUxg+NBiTMqQiAsYkktc0A6rKDqUQvHkOrimq3BwDBllOyRVybrAFjmo3xIwFiQDitKDt0S64AQmRxILde6sG51SN9ZToJF1BUAhLGgzOxXFcaCco4Ix8yWVYCt0rLorUuAHwvqwrIVIHRNUi9Mj4RmzQ1DRbkGhKmgXALcUMWiTFxGYN/nuogOaSA+BqA+J2CoTYaDsmdh9+zhDqxt4EUiQy2hTqzRcAQSq52bClsUREIDAFuFRWqCViEb+ZC4jG+Wbo1eApOMSEwaVQfMnhd3h9IXBGK1bVZZB7hLAB5WLH91ZHf70BfS7jl2LU4v1rVD6VY8ccXHJJZJR+Lmz/c8vw8obwpb/xbPlh0JAwgXj/JYuOzsgLHCXVh2wy1sxeu78dQtgomlMyiKS/vA1j4rsQEATuqrUxUXc4LqtrpsRObFyqZSGWqx8lcWgdeQAxpqW1Q1K5pKfbwLKCOBRv6tYQDlobT5qfqf7pn77RcPmh3qQ4X74FGPtQEId5E5rVaXRN0KJdY+twgIjuwa51dxX19lXnk+52Ws/MW18kHiapzIuER3N3wKHXA3q5u4U4ZZXLzFKueaK7q4VR8JwzuPGqi7rYp1Ut25KQJe3J7DzJYvt/Jx8hyCEGYex3Jkl3YOjWCXadVmVGtiHdg1HsTu/a4AfnHNZTwsci72slqYg7hh1pHvj4ZdOGKpknh2zUqp59Sd9JYrrJd+hAuwPuKLU7hyOAEACRXpQKhM3cVV/3V6TwHEK8QVuZ+Hw8YluhqSYGdcYfdyI+wK213gNWSiubQ2q3GvPyziFpxl3nhs3Ic3Mh7q0itW2BaS4Xu+5v4tMiTqJh+uaO6fGlaibuJV2qvXk+vX21AAMvq06ibtiMfHhn/40q/RPjVrp4xXk58J6DqzFgSLu2qTGykGhBJAXtyz7SaQBXLaTvVycmYcS+/LxjWUdscMoLZW742FV/skbd5Yy0u/L2SOA/2YlVK5lRqw3WyQUBtPuwGgbba/N+kjoPtWGXbv4QW4vrdp8wnpVXmTv41EYIvC72n6qoHlKcz4D7/5v3GliLVG1NOWcfSPpnctb5MagFjk3nSLm4Kmv3/8NVbaWqf4Wo8/Prxrf3tXkSkAj9iQwmgq5IAHbJg/9VqbvKv409MPm7r2ycpc7OvRuqxUia3Lf1t7nd9srYn7OpsFEraMPk72uj376f68tc5tx/Fl/TZPJQ//WG+OVSvjbSfi2Y/JrfLvtXVPsPPj09PN9u7HZ5/2lgEAzdKgbLcv+rC7j6/d/1v92jPQ3hrLfducI9CPX47Vfnz2fflY227m+dFHL7n7TXjte6HnbrXpZt//gNrfQL8vL0g97qTX+n7TovsJ/f7cbwmRg/tuy94JAPSjl/PuVvvuWZ7rj+/Pn3vP7816Pq87m3Z86v34+HlswgM2a4ifAvHGGCS8nOtEbvNBjADwE/73U9clG08SU3bc5HnZ3o01fFOgGSvqFkFbnx6znDTe9M2he8bmn/Z7ZteX1ihYpN5s2stt4H9XXdzKA3nrftqx1XJ6o9EWvNke0/7eWlnuxqO145Xzt+q7+ftz0725YBe8DQSYirRf+4WxM9d/rI5duuzH62Pvtn0b93XdegFvwMCNb6Cjl9bOW2W8Vva+3F1fNhbMFw0weW6du9VPU+cngSLQzfpfHe1PfTl+JN9nW+c+Jf8rwPJ2/vs9/Wj7fkvj8Mn13b7qSy76u/SF6asGlu/zhP/5V/+6yF1EXEvcuLE+56HJWKgb7OALKlyTSFBXx0Nc8bwOza1zrRynGXzFaVjYRRLYSGqwpqLDFDNW1bU02ovAVuOvUbGTazGcl0WlG1jjS/XnSK7duzsG42qmx4lEikMYTdXaqDIB0Vdcl4QUC8tFVN8scEPMrZ/qdmrdV9cSmssat1P0CDXm0rELnlrdbrkLMgNnv8Z7Y7mTeFSVPdDjpXjkHMQN0AsjJbCuAbV4DOO6cVVrFjV1mZN/Y1JKfjNmpbuRhcAxpOfnEc5BynVNHoBdA82Cntj6B8Do23W3PtWmA4CyBsB19zmnC3bP2nW33FGLitnHfs77irIGOA/UzLICbP0CnFh6rPtZEGtszb7JI9TqUGe2EqqFcBP7qG5k+tvhpQsaoVkW9XrnqdHvE3G76xLgh8JWO4AtajL2ZOJonRJWieUSQLtGraFatosSmymxnGrddCo/kDX4irr75BLY6iXlU3VssSsOroneU2e4JGzc6azeHhF6jKGe07Gv0qerWE0PpTM8VnO9jrdq/6Xe77Yw1J1ogC2BQ2XisFi37mU2nlLjGM2cgY5HqtsYUXXrnT1LE3jivGYB5jJbG7n9Uu7qt5qANhkSKB0Xjq2r5hz6iqC4TSxiOw+8WPhqvGHL67l9IHSyr0jGYiDPv7gc+itbBZWW2K18bR1ri8/TcXVLt6IyCRvBzUwm1sZGrIBtB1/j+qrEtWnsIdDjAYXIyMac2RhFtWRqUuuQHnK5j4+1dPjVtWkLJY5L3O+qhGJyjc/Gegg061dbbomlm2Mx2RK4sew4/k+Jurzpk1pALIlZIz7z23Y4QiN72oyJ5iM0a1SzvlhdRmO1alYujdlUUjmgxxKKJbQGaljFkpp9SVLrPmmd+kwRj4/G6vnVdWujEsdJXLXGf+q4KlEeOcCugW08495iCScVWIFmAAAgAElEQVQWwNh/N0ukxZvSHgAbi1yzwhvisg2eV4unWsCcuQ69LRtCGHm1NaIq9Pp5Hpr7Sb2NltjKvou0Pc3KJs9rq5N6OZrfAlhn2ri3EL4AwmasXlj1XB+PTdqVYdvRxvEWuLXl7KbiBlPvyn1Rtx0na9G7V8ce3NONYwBukvfs675V3q4fX5Q+47ovruN3IX3Nbf9I+qqBZfIVf3x4z0CxRKzkkSWeEACL3aNb2KweYU0vLVdjzBh8bvqWWWJrprDilPhtaC12Wp6CTQBN3D6La6ICMQAvdt5VW1AlQ7StFnzesirY3xorBqDFBqpr6JRYUsU7wmFYW3ziWkJ7nyjw1TL3MYoKcvXcLf1F64pqxeSt9EcrSwHesPtXjpfkW5yiir7nEnrfBo9KaPGCt6w21iqkcYc2jvGWbqJ/oAbmAdyMtdNk40T3FiEb81mHwjIJ6O9vHQ/CevO9X6tHKR5J3GT1vtahbOLW7Hzax73ZuDbbtjz4puOn8YFtnEy8ko0Xuxd7qSBeYwYV0HtPqGPeHNMNCBuPZnUHtS67KdFiv3zvs90QsGVofhsDR+RAA9Ni2lgxG5tn6+Y/dgt8K3Qvv/fxfTBjUsVN1m4Y7J8pHcN9vJ5de9i83leUwcRJSr469rFt42fL07E38X6b+zqqizNAVDYxfTYWDwTe0BAX5JZvN3nthoNOeI0t5IaYuShA3hJq3VywOUDJq7QfG0IqT6DR4VYsoB6qiTqRETnQJGWHl+8NGh0QzaZLrKDkO5B2EDdlaQsBljRqE9enjbCL4d1N3sQH2hWplLnXQITrdQE3LOrqBq15zfkqba0aW2jJmsj87dDIXfT5tQtqCuziWqopw16/vwEOaKbUTX4nJEz7drgeh2jHow1aP1YNUzeckgZRr6sqs6wZN8e/6TdZ+ahOUWsPaRO2AEbrhQFZBf0+NjB24/fu3t4zRroJrS0KdPU+tvbB/Jb75nRPazf1XtS5K+rFQn4PTLSfdZunAVsLcslcdw+oWGBlv8GGfGhTzj3AtvkIm/6ZR2vTn/312vYbYwSYT8cr1/8myRqlW9o/c/t7gY//vlkuIAeBDcK9Vf6uje3Sj/X9M8bmbht/S+X/baWvGhR/JH3VwPJSEv7Zu5/iwzyiVIfTsKKQw3kesOSAx8Pc4vCua8RpXPB0ZY7/h2nGdUnN0ne+jhhSxirC2dOwYpLYvx8+HHCYVuTCi/4YeWF+GBd4B7x/nhpxxxALztcBx2lBJYfrnJBS2SyAva9Y14i8Brx5PKNWj0XExJUYJ8WCp8vYPu4pqRVPiUj4ezZKDN0QM57OE4iAYWBx728fz3g/D1iWiNNhxvdLQs4e45gbscn5PDYikBgr1iXyOkgsgcOQoSy2zlGLIWRw4hv77LLEjQWSyCGvASEWrHNEiBUhVpTiUHJATIVjAIeMLKy6ee3HfSBMhwXXywnDuCKvASV7jIcVMRa8e39inUkBnbq4901rj48tV57iytYKcGxkFnbgsnCs4Td/8ATnCN//+gEuUMvjHSGrQLoQQQwSt7kuXDYVBx8JVFmIvGRmXZ2mFcsSkdfYQGGIBVniN/MqbZfFuPOEYcyIseB8Zo0DqkDNHmniOM9xWvD8YYITgFiKb9Y/55lII18jXKyIQ0aWeFAXK948XnCZE67PE3wUIhu5tzX3NjTLnsQytt+OEFJFyQw8QqgoOWCeA+JY4HzFckkYDiuuzwO8sF3W0i2VMRXkOcIPpTHfksoA1R7D6CN/xZc5IE4Zy1NEPGQss8R2itWRxNrmxwLngDy7ZvUcTgvyGlCvbDl2Yr0MsWA9D3JP0eSJ4MBxm9LnMGVQ9ShyvU8F+cLXMfNtZAvKUEGLx/C4wHvC9fupgQOXarteLZJxyvCh9jYAHBdKjsdCFtzpuGL9MCC9mbE+DxKHyfnTw4JafZsfYSwoz6l9hb3E6JYPCW4SC6qOM4DDN1dcfpi4zEGs2MIgO54WXN+P7X7U5wB3zKjnCDcV1GvsH+7K1ztPqJfAdUnMab1GuKG0+UMqJTMV+IFjcVtMqVoPlX3WE7d3LPBJpaMIyNwWAKBLYOvwNXRwppZN6cv07RXzOTUpKj8VjlV+P7A116Gx88aHFVmOh9OK8n5AeLOgPKVu6T1q7DOD8jIHvj6StNlxTC50XoPn+iX2tml/B7Ymh8cVVNE3KJbQ7xUB7tBjdNvYFAd/WuX9w++AcMgol8jn59D7l7lN9BQRv1nYe0Hnub47QuX45sUDjxmYA9yBy4/yTqvFAecInDLitGJ9HuBShY+V39/XxPeqOI5PFnZh9yD368oeBH7iZ7V8SNxGsZq7VLkd59jZfCUGuD2riVGGmwPqw8p51KqdKtwoG3DnyHPjmdtLVxmP7BFPK8q7/ux9dpoK3JmfAUq1bzwEgjtHPqYSZmJdb6zGB47XxireCMW19rdzaskEuvwZwNcYIOGKAx5X0EW+T8UBU+X5WE0ZnjoztsqqJer1JgHGi9+wCW8kxIAmpYbq+maDl7ho8b5wq8TIT7X3OXD9TYrtaoJI1bIdTR2yeQFhtwV4Q6OB+exEqoffsSo/BYCtwGaDpVk8xVLaYlYVfFM/5rNrFvkWF66J0OJ0+7GeYdNOQpNzU+t983SQjJbl2QJtku9E86YgbCzQgIDq4raSI9XcL78D2+h5nHom4GWy2Hvz7x7AmvN9M+RGgbsLLJC6uYnhXgO66P3aNODv0u9icvS7FtT6GekP/83v6D/5x/8xvolnzDXiF8sDkqt4TFc8hBn//PwjHMKK5zzgFBd8vxzwo/EZS434fj7iGBeMIeP7+YifHt7j++WIU1xwKQlP64inZcQYMv6lx1/hLy9v8RBneEe4FgYUH5YJFQ5/dHyH98sB3lV8WCf8eHrCr64nOEf4bjzjh+WwaXepHoe44hgX/PmHb+Ed4WGYsdaAOTMT7GWN+PHpmd16HeGDtGUKeROj936eJH/Cj09PqOTwtIx4GGb87P0bnMYFb4YZf/X0gG+OF0RfBYjz2+ynDx/wtIxIoeCyJpwSsx8sNWDwBU/rgMUwpZ6GBXOOjYF2jBnP84DTyMfVwqjW2GuOeDte8bQOjfBnihnPy4DHccYPlwmnYcVSAk7DgvfXCd8cLphzxPfnA749XvDDZcJxWDGEgh8uE9Y14rvHZyy7+ooAdEJnw3yY5J6tEcETUij4cB0xxoJSHR7GBWPM+H9+/iOAHH76o3e4rhHXJWGIpREWAWguwh/OEwDgMC4gckixtA2JyzxgSHkD2sdxZbfeWDAvEeOQMS8Rh5E3QqKRvrlcBtTVYzotzaI3xIznC5c1XxIOJ14c5sybHAowlQDodFiwloDrNeF4nAEA85ywvB/hDxnH04xliY1wyDvauMwGAedapjcgNq+8saCAFg4YpxXLnFCLwzCtWM4DhuPS2FG9EBCV4pCXiGFaebMhFVD1GKcV68obEnll93ItOw0ZyzVhPKyYzwlxKA1U6yYHAKxz5E0bGQ8AWD4M8GNBGnMndlo9aPWIx9w2IrIQGVF1CKlbofMc4BwQ9frVIw7CDr16+MRPoh5ffxgZEH53bUyidQmIE5MtEXgTop4jg7/T2i3AQvbjU20WufqcEN8syO8H+NMKb6xs5X3qMimygPcPa/vg1gsvfMMpd2A8lC4D8/0A+maFV+C7irttcfCXgPp2ZeAxB/hjRn1KDHCuoQFJfigAWnhx78bawAOy68RBnkFXcwm+Bl4ATaXLoWhfFt/dd4fKeVcHOhZeqEaCO3N/6FAaqVCTOdHFrLjhpl9H5GNtC2J/9UwQ84a1hwGwyy45xPcB+ZsMt3qEZ4/8NiP+EJHflLaI8WfWu1XN4TIJmC2AWuFYm5hQB14g+tmhnCqT+jheAFKg5qYbz75Zj0BAHYldWsU6Gs5eyD86URAF1qVVQ4KSv5SDWBRH0XQk0TS+OOTHivFXATVSJ9pR99SVF86UOG8dgHAx5D2JF7T5yO608eywPrD7phet0TJ1opcyEnwGygBEIe8pE5P3hAsP5vrI2rQAL8iVvCcfVbOW9XKL0fRl7U4+lp6caH5S0zgOV/nGCmnPemKSnzqg6YbGZ2D+9svXPfHskB/4ej93DVxXe9spCmmQas3Kv/HM41MTu7E2Hc7AOrhwaPrIgAAZAR4lbQEDBWB4z3q0IDQt3TpsF+cb8h7R4FS9yyqEPIBocxoXaiU7anqboZPyWN1R7Qsc1w0wQVLTzhS36Zr4eNHlUO0usEoKpJrXZMYMxMesy7USEDkBhmodpuC2rrkyBi/IewhNQ5NdYFnHlfVTWbNSr1VLtM4fC4DaeQWw1m1Y2usKjyO5bR/adRacGTAJbK/RtJHOkeeXAprUzV7fUq9p87DuAJpJ2lftx8Y1eJ/PgvN7j5Me34PAVx6/W+635FgD93PKuZf+t3/yX/5TIvqHn3/lby89vv179O/9B//Zb73c/+V/+q//1vsGfOUWy+QL/nj8AW/DBec6YPQZBR6P4YqjXzBPEaPPeIojHsKM5Au+S8+YRQJk9BmjX3EIK76JZwDAQ5gxh4RTWHCMC6aw4pt0QSWP0Wd4VzHXiOAIgy/wjvBNuuAQVuQacIwrvhueAfCC/U26ilQJk/4ALEFyCgsOYcVy4oX0Ma7I5CVu0+M6RPzB9NylTXzG4AvGkI0rqheZE4dzGvDNcAEATCFjiivWh9AAbIXDY7oi+ooxZMwlwoPbN4UV0Vc8hRHHyMBS5R2Cr809tZLDQ5pxjWnDCjvGjENcMZeI5EtzUR1CwRQDHtKMFEo7P4UVgy84xJWvFymUU5oRXcXjcMXZMxvtm5FXCoe4IgUe72uOeBxmrDFzfUKSk8mznEbtlDmPA381rcuxAwPiUj0eBqnzdG35h1CQAgPgXH0D23vSnWNaUclhjBlz5vjeUdyPAVnzh4JDym08h1jY5Vr+JbBLd66+SaCsOeDhwO0OvmIMpcXXek94PFybbEgMtYFqjbd9GBes4or8MHE5Yyz44AjDkHEaFywSW0tgkqkYVC7EI3hqsiYax6puxNe1NFdvja0dmwXbYUzsBnsYGdySAGe1gM+xYhpWLFF0YqVtQ2SglyOvRrJIZ4xpRQh8DQAMKTdpkxhLk8OZg7qhUjtGxJb+MeU2p9c1omSP6bDwhrsjlCELiGYX5Gadl7YMwv6bU+iSIilIfC57H8RYUB55NXCY1vYMrDFgHHJzA6/VYRHr+XRYGlDUGF0dR4Ax1nRYcK4O47Ruzl0r2NIEjvXNqWKcOlBdBPhN04rFs7VPtWwBYHnjMRy4zOxr24ghciiBkA4ry52kipQKFrDlKgdCiGUjK5KjMALHgtWz9bdmz5auWFscsLoIZ4mxjWNhjVePZj1Xt1/17MgSY6vMvt4TssZHjwXFM3OwrnRUM1Y9FFaxHHlhZq6R42D9Mbe4XK8eFuDjdfXIgQF1JrFAirti9aL9qa6wyYBaAhAIReYNosjVjAySaRArkmOQXQMD6lV1LHVhGam7iQLI0Vhr1E3UAzkKMBZLeB0kblbbMUqbPIEiswYvb9kiRGmnZyignQKhJgajdRBvgJVBHjxQp4qaPMoocapFWWGdAGluZtXyE7vfOgKfL66Bj3KoPDbSH5c7KK6DuLFW1qts7q8DDxIlArxvjLDkufwycXk18d9lJFCS2EeJaa0JyKcvB5aU0MCuGwyAIEjbGSiXwcRYqtNL7CCtsa06yP3jOdB46wSUqAsxKeCUzSMSIFUHaufrIPGfvrfJAss6Oh4btcp5oExyfxLtWGF5rqgFjzxYpsVa0MRS1uJ9ZS7bvmtcJ0VCSRLfSm4DmMrUweSGOdVYFRVUtxhkc6xryfZrgA54rIvyC9dax3+TB3xxHJ9c93qdfSPmBViy5SqAM4DRxgnfYoXdu6ZaK6YFqC+Ap435dOa3ndoGWG7Gdzc+Ldl6XL/uZj50YHsX4O3bcqPP+8Svcrepk4Gl6e++7I+k32e309/F9FUDy19cHvCP/9m/L6QvvDAEOfggbnFraCQkLnR5AueBsrA7n2p3+VQ3x6i4Jmzup9LISRqLn2cpARDYzUiIRWj17P619MVFW3S0FwPvbLNFQN4gdnEii5b/Uxcc4vqkrictqUsKyb9KYiHuUm4V6n5naPwDsbsWCc2/obW3ItXNtcy2XduixwB54XZhZvLCEAd0zSwbSwP0l6LvHy/N11xvKp/7i2SEuD2xVAEBPx9rO74Z27ZTyW34WarcHhsTpO5k6B/C8Mxz593xLSCEDEq4oILh7eMhbkW/3AmIA9of+aCvXM/3se/8ebEMNApyHUZ90a/8Av1Vemjja6nr/epwGR7lY8vl7+nVf5DyXXF4kjnhCu+uXwLwPJAQZRgh7R15T184bL8qrjhc1MpRgcUBZ120AZgjwS8O71IXNicBFiAHX4D3kTbjdDbxc43QQsbzKq5Js/x71oWY624/3K62Pmhz1a/AEthoZvO4AlzicbMA0C7m3UcVBFzMoiAr6YkY1TRf9kCauQHnaeplFOBsySoAhJXrW9PUJ4DkzzrcDogLcB0npBnIaVKPTQBAWnpfyQMpA3mY2jMmYeZY0qFZILJZFE1XhzomfsUVIJU+34fFoUwJ1QGhACUCaeEFaRKXMvt6CLLAowAMsghNhcFIEqubLp4BzsOWDkKw553wGUnZ5IFRXNuYvIfLnmQgaiSk1RCpyAJT5wfINamJ9kxmHoM6pG08GvF8KWPkZ2UF6hiZHCXF9iy61TwO0ue2WJMF5IZURuaKWhq1HLtQFcnjbTiTIWrR+wf0hRw5NHIdvY4lQ3zPZ+pzBagpNEmLVr5ZaCvIUekRdfvjBR2Pa00ePsv4DKER7rDshGsWqEbeE3o5Cox8uwehE8foc1iBmtyWvCf2tlqiGb/ygtOS97SxDE76rG6Trr3D/QoM7+wK9fOSywBF18aq3SdtewXLU6icCNDmPrcZHViaZ7iNU9iW2axPBnDq7zATKLo2hmpx24ABA2ysRY68k3GjNmb76yCWombZM3Gjrc/BwUsZNbhWjwU1NaBJdtQo1ijqdb0AXMDmO3SXvEfa2QCkeZ43yQIx8620dewtdHvXTW+kX+4lG4e5eU/43TFT7l1wZ3/bOm4Q69wj2mm/ta9eLH/79KKO3bh8Zhs/lj4O9G5UaNd5t+7xV5Z+n8HuVw0sj8OCf+uPf4ZKDkvtXVECHAC8Y2yegHtyEpZ45Z4sx61nyc7vPZnNvbSXprB1A3hR9wupgd31pTp4h85bYP7+FGmFe5IEtg23pAduHfvUZ/4W8cy+PR9r+15kfS81sB+3W8caIc2Pb/f/Y2lPitLb1n/v+xNeuV6vC3jZH03t+jttfe3b5xwhmHkFYCNN0PLtrrv1XaE75xu2t18mPSe/2zd3d/5FPcRehs4BZXePSf/nbs+3/ffz3py8e3w/Jh+Zk4Ww/QrfexD0+P6Lbcs241LwMtl5Y499alr3q4bPmfev1fNJD/+nV7UNLnqljfcWHfuX9q1jt9pjV4ef+U6427Z7dX2sbZ+4iLJi9/ob4GfslkzER9vzsQdm367PmEv7bPYW3/x0fmwc9JG7s3DflHtrTvym6bU5tEu3Fut3+61l3+s7aYE2D93Nz4dJum9fpLgxeDd+3Gwjtct5rt083du5L+ROv1+W40w9u/t5L73y7N2dZx9p26uA4HOe9S9IXLdW8LF/tR37efhyXr469+6lT3onfTzP55T3G5X/u5T2C6jfs/RVA8vJr/hXH36Jx3DFSgG/Wh7gXcWbeMXkV/xs/gaDz3jOI97EC369nvA2XVDJ4dfLCYewYvQZP6wHfDc840OecAgr1hrwIY845wGDz/jD6QN+vRxxCCuiL7iUhOAI75YJlTx+PD3hUhK7pOaBYyznIwDg7XDFcx42ciPsMsuusH9xfgsAeEgzlhqxiPvgXCK+GS8YxPSgbZlCRqYuX/J+YavHNSf8weGpteEYF/zl8xs8DAsOccUP1wOOacEQCp7XocVYfjc941oSoq94Xgec0gIPajGWH9Zxw/B6SgvmEpvrZwoF5zXhmFbMOTaWWeeoudw+DjMumd1nnSMc4orzOuCYFryfJ5zSgqWGdvxhmDGXiHfXCd9OF7ybJ0wxYwwZ7+YJ8xrx3emMa44YQmkyKepOqm6h3gGnYUHwleM/xYXyaR4xRHZPfBgWJF/wZ7/+FgDw07cfsJaAyxoxRO6LurwqE/DzPHD/hLlV63aOcJ4HHAZ2hXy+DuzGmTJKZdmVeY0YU27/EjlEGTMih+frgFI8jhO733pxn326joi+4jInnA4zavVYS2D3YHEzLQKST+OCJQfMa8Q0rPAOmHPA+XlCTBnHacZaQo+xFNdXoLP5Wlka72tjA1a3z5xDY2kdh4xljSjFYZpWXM4jpsPCcZjkJMaS27gsoZE9pVRQK8eo6vc4yzWl8HimVDDPEdO04noZEFNpMaDaFkBkaKpDjLUxqF7OA0IsGE2MZcnswjtOa4thZW8H3rqO0iaVtnHgOE8lo0rDuiGmcg4o4vZ5+cCES8e3PcYyr6FJ4wDgONM5gqpDOnQmiFo9WzNjQZV4y3xNTKTzYUQ6Lv8/e+8Xcsu27Af9aozR3XPO71trr73PPuf+OTckuYhgIoIxmgdFJCCKBPRFEgJBRAn4B8Hgi6Loiw+KIEJQiSgakFyMj74o8Q/4IIKCGAkxRBBzL8Hcc87ee63vm3N29xijfKiq0dU9e35rrZ19PHcdzoC1vjm7x58aNUb3rBpV9SvEuCB1TueuWZlDZJQpSH8qceVJYiy7Q26eG0GfYQKQFZjGUtm01C0M8CUhvhLTXM0Bqc+YLx3iUCSVTCf8tzUrOYjXR19QrkmAjeaAMAhAE6kXRFALfxmjxHIesoxLEPAignh+xAoukmKmzgGYgrijMgm4yyWJ9WQQ4B4aShOkDYDIUtngm05iOXv1HJmCuM09ZAHXAZqZlJ4j8CqDpwCaCPyYQe+SxHeSTvYqbUi9RbjnxYsDkLGnIHMxIA8DPfFeJwGgKSyxlk7W476qy6vUt9hM3ycCEK7iqsoBEtt5JdQDhC7zRKkARQKNhHqqSG+jIL52zjrDaOAm3Gk/PSOMYXUPJO6rYSLEkTRm0lxhoXGbQqu5wtZe+iMARV1hg1p981G8EJqlVxPd14GXVB1VXFythEkYVXuJBTW3SJD0ZRbl2kvcn8Va1h4SKxqBdCHMr769VBdG6RdYx/5RlThFS6kRRloAZvQ0sNFs7qNhceWNV+Vd4uaq6a3q1UB0TMcLQHomVMEjRI1LfKdHTW0WfwhNLd40yD4is5R20oe1s5jClsIkalwooVkAzaJsKWQsltBiJgGNbUxoVubaoQnWZpFtcZUuRtBbH5sFUfeKd+mkiiXNjXOfFYL0T1z6bd5FG7dVD+6za7HM2LdY6vPY5gOn+PLSt6fnJcvb6tCFlnk2fdC8r5wXVkvrs3WR9TR6/t5V5nkdz+ho9HT5PegtzzdjrjpfX9ueF773oMXz68WDlvfU+UX5qZdPGrzn8W/9Zf4D//6fwHnuMOeIocsNwCWXgEFj2zpVPjrN7WiAKJajMUaJkzKAEh+/lWvAqMiuBmZiqKcpSezbOHarvIjzLPkXmUU4TclivvTEjRQYRQVcQIRbE/6sn3HsmhXCwE94Yw010JUYK6ZRzglCrKhF4sgMfbYfBCBFXIWXVAvzHJvFijS/ohCpyJgqPBsdLf+iWVVraHwLobY0DKyKEhE3FNFA3JAsQxS4+OjivriIS7K5LMe0IMUacmnqiyBqjkncm7PjB6Gltmg5K+fQBFJmAhgLQApBBNdKGF6NgiJ77kCBV3UsFsusRKEXIbeq+yfXJSejoSICEpNmQC/yg0ZNWLY4NHHVlreo9R2Ikc2VmhVRVVH3YlcbqIwJtJbqQaRANJTVkCrqrO7akZEOM2qJKNco9Jr7q6VXABYUWIZzj6X2gqZuySdJUVMzzIboyKij5LCUNeTGn+b6nETpsFyWouXQIkibMB4V6EXRIm2MFYKo5VUEFh9KRhPgaVCwmpZXkJ0LuvFX6Frlp7SioC7NB9VAaciN7eigoyJSGjorrI2TRgK3HJo8ra+DaXHTZogyNCrS6mg+wLoOhvRq8+3q4vNrtBM0B6b2V137UwYrsiUCL2isxrdzWugyIJ05LPW84GbAOxqn1wSevKxnc9cHxP3eUF9NWrH+TEkzAbC578szREVzbLL2r/SICyw7AU++00l5V926BQZd44J2WbTtUAUMKDHQ6edjWVAsGYJ+aQIvCV3ebV7yc9ZlL+mYLSyBnVJqcYg2HxOGMrWYMTCWHJ600EqVhBY43qZlf1qIQWvTCRonP2Q05FEvhBoqc5a4SPmr/duaVxKFu9P9Mmsfwa2n7mNSl2iag8yPoPk60fqlMQARMh8LgQiQMRThsu0jmBKjfJ6XnKMGpGSxo3afO1GOa18FqEn3JvcV4ez9Rj6ucC/5TQFVGFzcK02Liz/b3mS0vc8aD2m5MC3/aTukqGiKHZPrG1jnR2VVZI8VwaHm2p4il0ey5QiFKPwSM4sWo2iANGHGai83Rau5euu6OEXHFBpz3w1KY+2WeFtTTqmIO70dDrTnCFieDY9qWhf9aKtokXdLVeXQDLV7ilU7RCmuvrXR+m0edxQzD2wErBUj0t/wXUcHx/9232IJ9jwyXONdBwB7/i3UyLpy3z19jUbe8G1b9pQy5fc9J4QV8NBLhTefd/j701YIjR9/+d/8Uz9zgJvXr3+N/+Af+ue/837/+7/wL//M5wZ84hbLIWb82uPXmGrEVFIDugGArH+9eysA4AGLwuOesG1uSl/KaT9fYnNjPV5XbqH3XGl9sRF8XshV3wDKMN2226EBEHfGeYg3tKOfFxoO0v6l3Jp77vdWvIutb+vTfey5tX5oaa7EJznhDsqb9uOC5c8NITkAACAASURBVH3U9/mD+nzf+Eu+RHGY7jr5BdjmhpRri3Lpr98rVt/n6gQAHO7TZutpBwZ7xZBQ79Ehuf1UOOiX+YTAqIH1sIBXNOzN654rLhxSPxHAg/ustMW0DXyxBgx0y97b7pl7rstbHm5/f+xwxLcmoCF+trHbdVdz7xfY/+APC30v8YZU2QgPc9uocnDgpRHH51hX19vE3DW2NDJ2MOJ4RKbvGI/izmKZgnlDKwNH9wwRVr8GdMwLD3r9m/iOpOM7tlyXEEvGzh5t10K5v8eA9Ro4QKbGK1NoIy/TM2XMuqgkdG8YwGblBFY5DXlY9i0POma/eRY9DxKDmZe+2gbBQjvpPtwI0SLgOmLdvFZr6dbPLB8t/tvoCdSUGwBNwZL5Cb081OWQYxP7Z9ZRey6ERiXIUiXwcr8d1jCWlBn2WWkmPShqljMV5snTwACBVICl5QDKKdYtHt4JxE3xZJk/K7pYS+vAYr1kwqKQWVz5GN4vCL9UTCm34pRDBLfssxvElBjHS/I8BMQyjXUTf9DVXlPudRIvninaB2/qt9gEEvRW+2v92lmJtbNDRm/Zs/7npf9GI9Biba2E0dE9L3Wjn6N//fl5bnz/id1y+bHLco029feK4Qp4urf1X7LmId/eW4/1wqba7ZM2f7Ue7fPYxmN9gYZbjfODxv/YrX+Pn9+5K+wH9vk3Vf8X5f+X8kkrloVDS+Uxl4gKkryUimowVfkVLqoseqVxUrdMU+yi3jehzbtTWkoLS7nhlbJSqblMSjtxecxq+TRlbCtE271JUzLEUFusJWvf5rrpYzC9wmqKF6CWQxW+mpKsyJoWixiC5S30ltBFYLU6Njdr5wV/QWzEEpuodczVcU8xMDRLr8RZXRvT92Xf7bO4JmreuEIwq+tenCYzVoKvTxrfaFILn1emShZtKcTS+NmUdK9ouLVoPwtOSWvrrOM0GrR9sy7u+XqwoFqCIXkxjT5arHstZ6bJyv5XwsgNqtDaWHrPUlo0l0cn8DUe8v73VhyYjykQzXLK0Nx4oeVlXBZChdtKzdLpgbC28VlbC2pzj3TruVo/VtrMckSQvIkGHKT8bZY7BbVaXfeLCiyCjwFKGC8I61Ntu26CYueUBUdTq+vzu23mvdprRehky+dnfAcasJjRzDYno8WPsZ0HlDemmFU/aSyWQuNNsyLe37er+dGiAK4UQavCKryaVRqujnPvMj4T00qRahadls/N3eNNnKHlBfX9M1r+vDa2rYtZXI2fDujL85V0P6wUQOXNyuIErFxO23VvdbB94+TM1VnInlWA1gL41nK0cl+ze0kVA5PUXb32fgpooGBN2Hd8MoC11TrBbeHtetqzgqX+ig87Am97JIw8R6tve4sSuSCNCr8t9YRZdfRvodVhwscWW0/jTdtuvF7nG9ROYLEkKv3s7t+4UdoXAGYN256BCWIrWp2XLFZC+yb/oaeTtmNjvX5bmuFeCUa7gdT4s5jte9MD9Lj9s537ah4vKBAvKi977/XNmDf1/b507fcsdzd9+PfdHbq2Z3MfpHxt6brpdEPzhs6bd90dul4c/4XywQrkh9b72LqNjlsAo9+x5b7t4JMvn7Ri+Vm64B/6/l9CAWGsHWaOKBwQ9a1WOKCCMFex5M0OhtMD/HhlrWCxdHrr39bymDkikbgtTlVSdxhIUABj5tDSTniLZOWgKTJEmZ2rpBtJqgznunbRWVlCN28L68PGA24toJbmxCy5/vMWMMiP4+ez5ZPQKQp8BWEqi6V0azHds95ur/mx/fXqjqg93Z7G1VhY978HPLRnSd5aq/esz3YfwA3YkaX78JZbWR/lORbX3MwBieouoJSNuwVF2u7FPf7eA3raFn/A4RXyqvNgvqXr3nh7z4f/TdsDd1rRsvm+95v3nt/oF/sHcDPP1u/OtXvlJQArG8P3+9I9OSRaW/+3ngAvzWNvXi/Nxe69b673Dq78vRfpw8tyz719+SFAXS8BfS11lkMGO4iSNsv3rcX3fl+3NG3BkvbAwvbmeuvBse5n38vDf9lqU04q3UrgO+O/j66tHGzyv5fl27pamx2NZUXy6ssNGZtOb8vuutzdWPbe+/D1edFK/qFlexC0/dzqbT6/74Xm1/jO+t70d7evnfttcT/s3bfb54fc24xrBz4rEKmX6L733Z7hDYtebLvDI/qQPbrDo7vK6IeUD6z3wSvzHStpd8WGj5jz3gHBbduP33sfrLg2t5Bvub9/Ub6T8kkrlj+eHvDnf/MP4JvLAZUJhy6jMjDOHeY54jAIiEoXSwMyOV97EEly+3FOLUZwGpPkadO4t77PAvBSA949H9D3GTkHcA0I6sLWdeJSeLn04kpJQEoF49g1V81pSg1sZBVjmSW+83iawAzMU0KIS3L3FCsu166BfqRUWhv/A5o0R1/XFVwukhk5at2HhyumOWGeEg7HCeO1E0VI8w4yE8ZLh6DugiFUlKyxkWopSl1Z4u5IQEFCkvx9YBL+6XghlhaDySzWtxAl0XxIVeIqK61iKWNfUBTgo84+Pq8iDRn52iENGSUH1BzQHTJiqhife4nXszgmhrgMqoXKXCPLFMXipWlfmCX/ncVm1lFyJ5y+J3lMz98cJcayd3WmuDptjYOse9E4SC7U8uZJ37JAaZD9Y/OpOSD2FWWMCF2R+EfASXSMNBSkrmA8d3KrEpADwiGDS0AaBEiluUeWIDwI3EBS+BqBpHPQ+DBKjMPjiGnsUC4RlFh4wjKGxZG22EelyebVoPp7oZtiRUgSJ8tTAPVV1vAaEY+ljSHW1tAsP6EvqGNscZPN0mc//hZDpXuSp7CAtByLtKkk/fVV2jKAXp4/dm554UFyEmLUoKfIbd7VYgjNMmUWQaOJAeqrCKBjXOIQJzVJWDwbk8Q3zgHxlQAC5W/6RTroqrbR74ERjhkgoD6n1fUWY2lK0bGAnxPC4yz0mnsggPAwo2aNu8wkPLq417m5b17DYkEtS/v0ekJ+2y+xeZ26SQZGPGaUt+rvnCpojAKAM4WlnrnJscYUWizcUMV10njV1SUm1ixgfQUlln1qFlWzmjorIWWNz0sVuLpnUF1U6RrFpdXiHImbi6LFIsY3E8o5LS6gXZW99dStYyEB0EMG3nUSj3gowLsOeDWDn5MIxAzhg9EJyBxtXYiXfQksY/ZV1sEs1AyJseur1D+Wdh2AzCc7S2zveOjjTwczE8lYdMyyBzappqC5POk5Ap/N8jxafKRZZDVmknJoPIXO1eLBUUniTocqwEvPneyPxIDFLZvVXOMtaQrNpdjAiqB5R+k5iWst8eo5pGuUtVFrP1kMIaO5+IYxoJ6KxDDqP+6q9r3sDbpE8EH3pAE6nTLobYdvW3ioCJqDiDt2gEpKeyepsAywqeUJBcDHInzUGFCw7FUUkr3FugZBX8DeKuwt27oX+DGDLrG9R/lQxErvU2ppjK/RbnGylEn60LjhMIYlBo8g7sYtFlSfqd49rwZCNZPEseq8wFjiX1n6QSbELDksw3WZR0tLY7ywfQloaiBqc7BTD8po6apkM7iUW0ans+KCqQHbNKsjYQE42sRYeh7AdVMtZrZtBLuJ1gfbK9afMbgYyxsr8fawgVx/Rr6zjq/W38VIN4+ETa7NpugF443rb3vI4ft0XiNbC72nwceWeuVv7/zrrrn3jgJ/c0azWQ+7tj3nMp59uDL6symfhFX1W5ZPGrznl37fF/zH//N/EAUBuUZcivxYDDEjgPFcRDgSi2VFrrFZBq+lE0sUGJnF+japZTOAcS0JmSMCGKc04VrSjTXLUpwc4oypyGfpK7d73jrlrWApVCQqOKsLpgEFARClDIRIFUnjIa2tJX+3fkoNKxdgUitoFwqepgFdLOhDwSV3GJLwZSypWSQOaV4sugpw5GktNaysF2ad89fmKtbZrfXMgyYVRS3tlP65BnQ6Z2uTQl31lWtofImKTDqViFJDQ3X11jNzR7Zk9IC4MW9TwBQ3ZhcLCLhBep1LQAySyqWLdWXRW9yXVQFwhwalEmJgVAayWnLtfggVucTWj79uVqViyrryKZAklp8UhdcOSmxP+D1pT7Lxz+oCaIiuIVQknY+MCXjQKHNR9pauFRhSCSt3aADN1dkDOdk1bzlixg3Yk7+3AonStbQctUsbbn0aYq19tj6M3nmOCIGbezcARaoFUlrWYO2mrcIYo6GzxrjM1+bu3cbtc86CgpF0D3nXbivMhKL7x9yu5YYqN2FxD61FkGc9Aq3dKzk2t2BSGuywSMZVa7mLp/ZpZYoe6lhdrlhcqPXgB5BDh+aOHuTQwQ5vGumq9FAA2ASsuhxOtT1gGCMqcJpLdpsE0L63w60i7tPBrRcXOwSpDc22PYOmoCroCY9yyGK8MvAn6svibm18NaAoVQSoqwKwZHGFxIuyyFgOQkwQZBW2TYE36chci015JKzrecAoo90OGuDuE6+EXX9QIIyFi53khRemXCZV9FQh8K6OzYU4oLkXN0XGlAinbBjIjykZBp7TXDEdem1zEXcALU2Z8G6+1k9yOZErLe7GnleRFzAgm2ZdBPimIKj7r9CGlgu4Dt9e7pE8lgvPyfYCL+A2TDpPsxCroGt5ej1qK+teWYHzQPoke0RM4dDr1qchtxqvvastHNsajxzqKYJuAZ1DVeRfG5Pc/jC6KGNRoHTeHml0yZe5VixYlUhDzG20Ka9CWeefbXzTj2QHKzrPxj9e+NR4iWVcKzeG4O3y21zd30UhWvPzpbKr8Hjl8c5Y9zvE2q14r/3e9W2xe3b/zpgfagneKn4fZO38QB5+G35/TPmL/97vAPCeV7/Gf/cf/Oe+837/u//hX/mZzw34xC2WiSo+786YOWKuEcc4NQUFAFIQAdzcVsea0KmiN4SCQLW5owaqeHB9DzFhVrfUY5wxaNoP73Z6YFFuhphbXKcJ+z0XBPDatdO9Beye3fcKpBcEt+5o2+/ZlGbnmmsKdCBGChUBkk7CPptiAohSnPXXoAtlmQes/7USaUqMv+aVNq8UA0AfSlMwgUUJ66LUMddlExoHp3BZypPMoSn8ptgOKTfFcwvQxI6+bdypp9cDGR16QRg4JFnnGGJzZbZ2vg05Xnhl2tyTGcAcuCnE9q4saVF0jTZ7h5oyyCwpSIwnMg9W5T+gi+s5kVOq/Zzss9UlyMFA0vbZBHTtX5SONXiRuW02xTIu62MKusXMAmrJDQEplqbcrBRTnXcJoSnYplgb/UJv1XWoiFo3x8W93Obc1pS1L/fMmJLp94gpXzHU1e/X1k0SgCJEr5VufwiwKLgWvywKigFA7bleMi9xylHRmU1ptTjipjwGRkri7eDr2txMYTWU6ZSWPeNTyViKkhAWxRIQ1GJRLOvKE4Ljghxtc7PUMjWsFWWbfxtLlXEwrTwdjGapp89ncEFLW4lM/3IIqz1Coba0C4HEQ8H2l+1zYJG3CkSB9YolMzUvjTY2E2oUJGU2r4rEqMQN3RjAGjRHFckVkjIBiBvlti6o0e06tJ4pyk5SawjPOgbHhc6Vu6dXSE3pctazpsiqEktpQa1saKP6AHCT4CFAOIHBtKbLrP5MSlObE8uBQgAajJK1C/oCwUJTUxQ9QBAgm1sVSVaFnSsvilZrA42dhijD1kUloYMhaLNFldRAQpvOtwagbgGZPqJQVEVV+dvO5plX4yC7/WLbw4Ev+ZhKELd7N4cFtLnuFIpKG8XSUmJ4xdLYTE6x9BZNU3DdPid283AHFBQWZbgpaO691BT9sOgFTbFV3u9Z1IqzSC4vuYWtYF6PFzbX3qfAbPixmoTb3yuU1W2fFft9+2r+9dD+w3rt7tG4d39Tb1dxte97VtB7/b6n7CqYvt9Nf3cV0nvlQ+q9b013yrdRNH9RvvvySSuWQ5jxe4ffxtflhJkjDjSjIGCsHc61xy91E661Q6SKmSOGMONcBlQQTmEShZQjOip4Vw44hQmjWhqHkNGRxBB+NUsOy20M5ilOKBzwVAapy4QhZDyVAceoaUQ2KLM23lgTKge8Pl4wc1SltzTBOKLiqQw6VsAQZswcV3GGwKI8S58dKgiJCioHHKPMZ6wJr9MVb/OhXbfynAekUJBrbHknpY/aYj993OZUExJVBP1V9O2ElkURHWtERxWX0mE4LAiUhQkdVYw1YQhZ5q75QY9xxqV0SFTxkEZ8NZ3wKo2SJxQkuUSp4l0eGo1ewTO6TAF5NlAeJ3iasgoIiFPlgF9/9QwA+GY+iEJOtSm019w1HgWqOOghg4FDeRoOMbfrpzQh19is3ZUJfSyr/oB1vOghzc1ibmufOcgBQI3oYxZrOxgpFEx1sT7LYUVt1vg+SF1AFPzHbsS1JDzNA/pY2pgV68MMsxZvLfTWj/HFyqR5RBNVXEvCIWZci+QNDcQtNysRo4+l5Tu1McaSmmU/Ul2NO7v9dUySK9XotTaVCV0wy+yy/4Yohw9LHHNtXgCXLHwxRTfZfnbxx1sQMO9VYPSZVXguEa+HKyoTnqZhZRG3nK/WbogZRIzz3N8cWvi9dExzyy17yd0KIfmhm1o+W1OUrzm1399eraFTiatYb2t/7GZc5g65BhxSXh1OHNOMp7lXvlZMJUq+WPUymN0hBoBVDlRfz/YFY+1F4L0P/CGI8cEODMwzweraupgVfsqppY0yoDXrz+exHUts4GvW9jqn5nHgQdpGvd6njMvUtZyzNt9OUY/tQCfr2P7wIOnhRVGBPMWKOcfVoUTVw6NcJE2W52ep8vyZ54GAw63jZK19+17DCjTOt2fdu1NOmmP3NubW+G7PiR3aAXAHRwKSF0Jte94O2PyYhakdypglHUDzOLCDmpzjykPAPlvKL/MQaOmoNnXsIEW8JaTudqy19wSat8Jk6Zy+RYmRW8iIhcU0b5gcBViuyj3Lj9s8IqK411taLlNq2kEL5F7rk7ilGLuJdWaS8JMalu92iLY9sLGDDzsMMGWQ0TwJ2iGIHoQ2zwPb2w4MbhX33MDwqCmhrDwA0A5e7ICFffoldwhlYywmTtwqFzaXPYVb57CvsG2UyNU93CqOd5TThnCMnftGy55S6y3zvu0LytqK7r15bOezc/1GEX1J6fTf36PY3m37Utnj/Z3yXgVxj2+fVNmeXPx8lU9asXybj/ivf/L7m/Cd3Q+jCMeL4A5gZeHaCojeldLamODbhdKsl1sAG3/NBCQvHG0BUgwExn60t3RY75WpCYTA8hz5evbM37g96g+Rp2HKEZ3GeppVTCxV9YY+/1mEjYXn5h7qi7fkAIuQvFj41m1iEMRaE1g8Sqy3BHnhaA3qQysrlC9yuEk3Qu/q4M3RFvRH9K/k7wPASlgx68Dizrmg1ZrQ468LL7wla42Sa26Hfr7bYkJSjOt2JmzZPW/lWs3f0VXK0k+thFpEmDHLVUPQpcUScgOCUZf7vr4XfprgwGpRsv3g56g/ds1V0yk5W1Aav14rq/CGZ6vfFidINflD17AJSoDSgBskWm8JXOYuA+zxxt+zz7+pFqSG2rslUr+v0IJfKIauy0Xj1fzcvZXL/norlLeOeZdIa5+DxBpioacVs7C5Oa9cWzd0s+/fYgDZjd0WeiPkendMx5+VUONdTO/NjXhpsxHEfqRxo8um0Dp7brhm8fNjOovrauxt2xVDNve2SJheIPJCJ9x9J9C1+NHNeA211Y1jljTLdWlWPUOgPVs8854wbtdNwLZ96uVXXQeLvSPPDzcv9ttzK1Q718r1i9ld90K+r+d4WHxOQgCV5Z3HJESbG2015GDdG8XceL9l4bJYXYvuORPa2XAE3HvBl+p44a185Oe2KVv9aSVPVyBs3JrDZv/4tWkIxVtnAV0z3/lq/fza3VGEyPoAVm7A7TeDsM75as8rljHu6VG2R9te3NB51zLombh53vzrs421N7/3lA9Sgj6gj+3cG03b9nv9+ed2+071bfy+2KHhpryn7i5972l7U+db8PvFcX9R7hYi+k8A/BEAf4OZ//af1jiftGKZOeCce3w1njCXiMd+RKkBz3OPy5zw5njFmBOGlHGeO7zuR/z28yOIGJ8drjjPXTtlf3cdcBomXKYOMTAe+gmDWp9+65vP8HgYMeXYToYrA68Gcb398eWAQU+yh5TxzeWA18crChOergOGLi8KoZ40P18OmEvEFw8CGvOs41rM36mb8fXlgKwn9UOXkUuQU2L3RNm4x27GN5cDWK9d54QfvHrCu3HAderw+jDieezFQtHldtr9zfOxKVNyqi3WJYtpG7rc5gsAz9ceKdbmqjmXiGM/4zJ1LZbSlMBxTkix4vnSoetKsyacrxHHYcbzZcBxmHEZe4TAmKaIvi+YRgEyenUa8e484HQQoKWcIx6OI05dxlfvTui6oqfeiyttjGLxikrfZexQKyGl2lz2hj5jzsLH69ijVsIPvvcWgRh/4yevEVNZ1ZnGrilTRIzjcQJDgJkAicWz9CfDMGOaEkJgnA4jcomYNLYx54i+z5imhL7PGK8CwmMKHxFwOkzoUsG7syS7rJVQSsThMKOUgMfTiKfzgBBYAJ1KaOOb2+Z47RBTQd8XjFexzMVU8IMv3uLpOuB8HhBTQdfJ6XmtAUVPkUOsqCW0E3SL+auq2BrAVdfnZgkwsKoYK66XHg+nEefzgNSV1rYWAXfq+4xJ689TAoWKMscW39fAq5LQlueEfpgxXjscjhPmKbWYwJhqs2Ykcz+t1K6dHkZMU8I8JlGAIyN1GSlVXK+dxhMGxFQb/0qO7ZerVyv7dBWQHQOpAoDYKZhWIaEjBzx+dgEBeKcAUML3ijympvxQYBweJsRYcX4emsIegii/dQ4t9vDwOOLybsDDZ1dcnvumYALA8dVV40UlZrPrZwF8UsEhHWYQAdO5Qxwk8Er6ln365ssnvH17bIBSqcvyLOn+fnp3EPCuviJfE7rThHzt0B1n5Ck2gZAZCH0R69AUkU6yT2MqyNcO8SjxplyhYFhAPGbEVDBdOwQ7JNE0QgbIRVFAs+JJ9tV8TfKs5IDuJN4g8zWhO83IY2rWEEqlgVFRYDx++Q6X84CioEzdkBEC4/puQNAcr3UWuo5fXHH++ohwLBgOMy5vDzi8umJ87pura/dKvD1IFf15TCvF3wC2qvE7MFJfMF86hK6CKxrgVhgKOAd0x3l1aFPmKABZ+uylQ25WqWoxeoWQDhnQAxcusl9tr/IUQfpMwEC/nnocfnBGKQF5ig2oi4K8N0oJqHNAGgRQLQ0ZROLaPc8CgjZfOsRDxuEw4/LcI6Yq/6I8UykVVO0fUQB94jGDAZRRxky99Ds99whdAQVZA9lHVcboq1zrKooCkFlsLADp9yGjZmrgYNQXpF5+P+drknmcE9Ipo4xR+syEw8OEy9eHFySLl0s4FAHTYggQlR5chMgoz0lAv7Kur8Z0W2xuPGVRdHNA6HSvKv3hmAGGgLrpYV4DqAMWsDNTDgohPc7IV32/KMhbneMKvIciS1tAQL4MPE1Bg0jdgrkBUemPfccSH2speHIABgEfav1HFqCmTjVVBVqiYxbLJENcsDNJvaECo2mfWLSEvi5jOECvFodqQFQk86aZJGUQoDHD2g9jnYLIlN6s9RXEp93Dck0AgVwKI6+cVWqgTLvWQFOMlQ5RsrVChACsNaVYU/bcUQJXhzF2sGKf4Q4GwjIuFaAaeE9cK+MrPui67Ro6N302WuyQgHb6q8tBzvsskk1s1TlvjeprJmz4Yt9/jpTJn5Fi/J8C+NMA/uxPc5BPGrzni7/t+/z3/0f/OL43nHEpHX50fUAgxhfDGcc44689v2mud6/6K350ecSb4QIA+Mn1hCFlDDHj7XjAD07v8KPLIx77EbkGvJsGPF0HHPsZv/zwDr99ecAhZXShYCwJfSj46npEqYRfefUOb0f5obrmhC9Pz/jJ5YRSA94cL02BJZK8mWNOTXH96+9eIRLj0M+ihGRx25umhM8eL80KeR57DF1Gr/GRgcTl6DJJ39Oc8NmDzM1cuL765gHDYcZpmPDN07EpLc/XvrkqffHquSnTl6nDaZgQVcHsYhVlVJU3ADj0M+YSGxBMjLUpFjnHFWjL0GVRak9XnKeujdmn3FB6n84HnA4TphxxGmY8XQZ89nDBeepwPg949XDF83lAP2R0seDp+YBaCI+vBPHWg8WUEpqLklmwDscJBuRiSss4iqJbK+F0mJBixY/++mcAA69+8CTKyCQowVwJgyp1Fq92PQvQT1IhJyVRcE3QjAo0kp87UFcR+9JcwvIU0R0y5jGhG/Iqp2atAeWpAyohPMxNEUmpYHzuBZ3xnBBfzeBKqJOgywY7lVe3q/40Ic8J5RqRjuLmmKcIvO3Ah4r0MAtarlk8gsSWASL8h1RF2CvUhPwG5jNGAT7JoSUKD8cs6LOFEE4Z9V0HeswiqACAxa6VoOiuWf4OItzEQ2mn+3VUacGQOg8FfE6CgvrcNUGOUhV0WU08z2OEIZySCSHvOnBfEY7CZ56DALTMBH4o8gNvQpeCujQBh7CguapihiksiJ+GeEoQxMm+InzVidHuy0n4QwDGIOiujCaQ0SWCKlAfyzKWCRpdbXMP54j62YzwVSd1TcADEL9JqB03IIswBdSH0n6ASROn80MRpEpiERS17+6riPlNEZTUS1yhVKbngPlzjbWdCHwqCE9JkB5HRfl0x8akAqmgZQZwxwgjoZwqwlWRQAMkbg5AuAjqaT1VGTewxMSRopkaqmbHCJeAMBPKQ20ANOGsStapIpwD6qE2qxZNsie5Y6AAw08i8pFRDhLTFq8CEDO/ri2Je+1EmEpPhPmNXE9nwvQZo3tLyA+sQiOQntU6pSwovYxluR25Y8SLrGVNQlMcCfnECAp8UqMIyvEq+RTTWYVnfRxrz6jWJ4RmQ8WsUQXA6Nopb9Oz0lplTkETunOSPuZHxvATQdOsPTf0TGKAZqD2QO0YhSoKKAAAIABJREFU6UwoAyNdRFAPWedCQH6Q+cUrkE9yj4r8LYOCswThCxWgDsoPSP9UgDjKPPMDEKaFxjDL3MoAiRWM1gea1TdoFEc5yHw5SD2Oci+Oui6DfM8nIJ31uwLddM/A+AbfuqSL9MsExAkNuIaK8kTHiVc0kB/LIpbOcq92QitHtDmks/RZu0UxILXK2j5tFjrImvRvGfkoe7Imoa12WFBSda9ykr7TVdbGaOQgc4DyzJBFAalTdV1sP8cRqxhOqjYXib2tndxLF0bV2OCQGbWTe/HKKAd9t5dFiQmT7u9EID3F5rjEO1KR55eq8LL2JOiwzNKPWa/j2pplllm258YBDa2spFXn6vZd8xhgNKAkjZS6CT22z2GTp9Tm2dpZXO3G4cQXj8JqtK3G81bn4OjLwvNQZC32FDN7f6ysvH7sbR7IOwrdynLMt3NobVeN1vfu6aJ7fLmxJm/H+Ail83/+jX/pZw5w8/rVD/nv+Tv/2e+83//2f/xX3zs3Ivo9AP6rX1gs75REFd8bzug07uuLQax/gSrGGvG6v7ZYs0CMN8OlxSS+OVxajNmbw0ViJrtJ6seCV/2IUze3fI6fafxUIMYxyYn554dLc7M9dZNYrBSt9KGXviJVnLq50Vy1jln1vjhdGkBLiQUPvbp7HpdYInHHra2eWT67WJY4msMoOS1JwEq6WBA/V+sdMcKrc4ttiqfFwkcAjv0MwoKgSgAOXQYBeDyMKytkDFVifjQuKBBjSKXFC0XnohqIkdT99tTPzdVYAFnEDff1w1VinxTU5nQQqWNIBfHhii4VPGqdQMDjg6zD4KyunNS1NRG4p8ZnBhoPOxePlNSqCSzALA/fOzeglzjMYoEjH7upoDARSK9Lc6P1fQJoYCsAUPq8AjohYvSDKHoplRsQFADIamWIGosjNDLCK+FLHXJTkOuwRnAFIJZnSyczLCimMVWUviBEcYVtFj778dy8sBdU2IUGZmpzEiuUvP1jZBRFLSViVLUAVl2LJY6KwEMWRbovTaH2oDIc17+iFJb+ONX2a0ME1MASpwQAXWmuuFZqYnXV1RP5Tp94XuKZQAx06xjTRosJQDomDy7eSA0e5K7X7yt6cKoC4Q+Ae2rWSOjY9VB0TZb19+tg7qd1KIiJUb83IcQ1cIsHHwnEGtPFy2nwoEpBZFHUlJcS2kGYe0m7QUHuN/AWAPODoKMCAI66Poaserh1hW3pP4hRe7HSlQcZj48L8nBzER9EmqPAzRPC5CDmsuGBekt4V8aDzA2RUQZSIBe5z0e4hQGugyr/US3Cj9QOIMoRrU8GMJ1EcS0HQnklbaYhLKlFiDHrnlhJlbYuWq08rCXOrJ+L8Vj/1pPUy6/W7FxZcgDk15t7Wlo7vTZ/5iw13p0UQFa30MtBGX1HQOPAyK+lzmxbzLsbB2B+7a4Ba3RTI92tF72m1jeYFuE6rpFRV0K3c+tr7pVOGOYAzI+2LvqvLn34ttOb5TsIuHqk2W9RprpPU0Ow9d+Nz7qe05udOdr8Tdl1a0PWnd9jbi7TZ1gBSm2rbb+Me4qCF879Pa9YOaWm1fXtqjZorrBo7tvSnpuyxYFXfTcGbAdzxYCSVi7bvLR70frDiwK2mu/2sxv+pu4HKi7W7qaPzTtiafD+fu/ObbXINij2tbV74930bfzc9PFtabxXPqb+3p59T/kZWQN/UbR80orlpXT4P7/+AR77EXONeDcK2M2xm3FMM350fhA0yRKbq+jjYURlwvPYo4sFXaw4Tx2+OF3w1fmI0yDWsyknjNcOIVZ8+foZb6+DKB1B4hVjqLiMPZiBz1+dJV0FgOvY4bPHS3NlPB1GjHO3xCqxWKb6LqNPBV+/PYFCRVK3vlpCc4kbTnPLgTmOCSnV5rZqZRwTuAaUOeDwMDXXza7PuLwbmpvbdO6RDjNiZExjElexWHF8mFpahnkSt0MibtbHaexQyyJM9kNGniOKupqZC5zlfbScZ6RACiUHDEdxDzW3q5gK8hSbi1g6zKglInViyesPMkY+J3QPM+ZrQuwld+Z87oBMSK9mlDm0XIohcnO3A1PLB5hO4oZVcxDBG0C5iisQKjWejP/vCWAgfm9EmYO6kolyF4eyikOz/IPmQkSpqoVKc7lp33SJYqWxfHWpAmMUK9w1NksYGdABE+gc1Zpjx8YsrlVnzU93DWJtqyQWw8hLTkWLiToUcCbJP3jUfmZCfJuQD1XaT6FZRNhcmgAs+clY3JJMqNcXO01hsdDo+HNf1WoFsVo9R8wPmk+NIdYjjVczixdNLieaWdIii/UPWGD5+wq6RuRjQXiOYp1SRaHlxVNLl1m0TGiM7yLKUMXCxpIiAYUQJsL8UJpCQmp9JYbMReOAgrpslUMVmWkKKMPymTsWdMdJrGzdNxFgYP6iNMtimAKytjeXrHgNoArkh7oIWGo9qmatI7Ucvi7ovonIj4KGaoJ4ehfa6TwnRhzVqqc/wlEtluWhIlw15kvXjQAMXwdMbyo4SV65kElOsxlqratyCj4R6rEiPgWUo1jZas+rPHFhEpe12jPSlVATi5VxYMRJrUqBxXrDQBrF2lCOvKxzFPesOFNLN1E7lrozUA5oqSii5sErB7GGlQEt9i8o76yP/iuxFtZe5havoljOr1gsMUBD1eyeCdNrRpyAeCbMrxndO7EC2ul8OouAa5agOnBzQaOqNF9ILU+CWinWW+EJ21yDWDJrJ5a3ZjnhxXLYrNNXagJ7TYviZpZAsyali/AJLBbJoDkFOQFhBLKzWJpF0RScoFaV2mk/vVoWWa1bSermo1i94hWYH+SeWS3LQeuGxTpZeqlvFjGxGgvZWduDZQ2axfLgrGVZ+rB52pqVXiyPNWFleTNraBnk8/wg9bzFMj0zxs/XIHgfU9JZaJfnG80CFgqQT4QwOeuhWnrN+pbOAMLCX3k2lK4zgwO154QYoCzXgLUl0/Zt945RjrLONQHpyqiJmoVOLHNyDaQWw4EQJ7UoBiDMrHuGZH/ZfGY5rA0zK40klsmw7BuqQOmkP1kXfT9duaUeoSy0146QLoxsB3Z1QcYNM7d9TIWxtT6KdRKLxdJZ5G3/Mi38bBZf+1nzbqJ7FkuWfpvF0h9SQOqEzAtC80ZZbJZ/o50Wy2twFstmqQvWbqlnxevWHMQCubq/sVja+ME9z8uBhmtD6/n7sW7+uoMdb5VcDhhEoF1Zxrdl6w25AjDglRK+N/+Pslh+aoXX++s7LF8S0f/ivv8ZZv4zP5WRXiiftGLZhYJfe/wab+cDAhi/+vgNpprwPPd4Ox7w5ekZY0nohiuuucOvvH4ruR1Dxa++fotr7sS6+DjhPHf4/HTBNSccuow3xyu6zwS05+31gDfHK+YaUGrAaRB/nEd1G30aBzzotc9PF7wbe3z/9RMqk7jhHq+NZkNwnHJCLgE//PJrVCZcFSmToBbKWPBuHBrozaPG62WNx7O+Hl6LpW2IBe/GHsyE16crxjnhV37XW5znDtepw5evn3GZJT7ts4dLs0g9jz0ejmKVfHW64qoIiH0S1M7PXj838J1AwJgjjsPUEBWnHHF8M7d4SkMlFMAgmdM4J5xOI5KmCik1NPfXhy9GjRudMc+pubgOhxlvXp9xmTo8PlwF+bEEPH7/ij4VvLsMiKcFBMdQAX2KEUDiPGsNiIe5Ad4Mj5cWqzpNCSUH/ODXfwwiltjNPmN4k1udrHF1djjQP8p6znq9VkI4aBqVzy4N7XD4wYxaA8Y5NYTC7nOJsew+k7glQ04kEiV9+J7EtJ7Hrrn0lkLoH0fUKpbay9hJrJq6/y7pLjTua0oIp7pCP0yp4uGHE65zkkOK1xU+vYXN0fo0F11z/zXrpMW1xlib2/M8JXS9WLDnOWL4QcY4pl3goK4rmKek8ZEqMCmfGYsF2drmOaL73kXiUr+4IM+x8SW8UkREJs3zyC0+FACG78/IOS55I4NYPu3AhAGwoVYqL0oJ7ffUDnFaXKWLsTQAJosLLSWg/+VnAVp6GtoPYEwFPEmMJ0j2T1Je1Wl5/dr8W55IAOGXMujSIf3gjHIVF2mz+qVfnts6VZ1/0ZhdrgT6Ulx9MSbQ5yJVsAPUSL97xPQ0yEHC5wzEAhiQ2DADl17GSxWYIvBFabFUFjslDwWJJTmoS3EvcWWIFTxFVMsLyWiAQ7WTQ6I6pSXGUutUi/tT9MhWd17i7GCxkVME+oI6634lCBqlzpUI4F8fUaaEqvFprNbvcu6EbqDFftJhRjn3qKmC+4x87kG/W2JLW0qGYUnHQ6TooDoviwGtfV3iRgkIqbb4QpsDF0LWOLc6lGZJl+dB3bb1HRuMh7Y/SPftQZIF2lillwM7ChKf2sCZisQv1ksC//q0iv+057y5vs8BYRCexl6eixAqaolyKHqNoL5KrPWYQLGCotTJbj1b/1MEKc9sTlEP5MolgtRqLzGydUU7F+1jNq0aS97QKeB6KC3naHOD7+S9Vu1g8Crj82yxe4R4zCjvOnzr0ssBIQDps1mJWa4nPUjs63I4ZweTgzsUdHlAUWlxuTeAJesTkD48WBPLfqFjlsNM3VfoqxzOeck8uLZdlec46mEeY8nTmmkdn6hKQ/MWqLjNx0ruUJBpyVHa12XOQeqYezvNrr25fOqzQVnzlzJJ7CO7PvRwB0UPHq1t4IXWSjeWYECUyps8n6YwqdJFZYmxbHy3oocf5JU8Z2Vt1mmjwymeLb2LFdqCXq21pbZ09i/s3PduvQQ51I1YK4Rwf51yCNrXx5oeaYodNn2sKtNC392ypnutfNKN8riepCdo8/dvtvyX30Efv3PLj37Wbr7AJ65Yjjnhr379JcY5NVRUgzovJeDrdHR52EJLtg5YUvfQkDNLDg24xNz7TPnKc8RbtSp5JE6z4pkwCkCElhzxLipgxg6SZXNnLYTnSy/1WCwJvm8THNtYTGukOUYTVgIpLDujJSp/Og+oNaDmgKcksYnSl7YnbgIXEeOJsMTdEYBKOMe6erBZURabW10lPMXDAlPuUCC9gEhmBasLKieXgIuBdejYY+raGOc4gHPAOdUmjF1T34A9VnnynOukPwpj+/G0Fz6Aa+TFAqlCyW/rvqhXsQze1HHlqgJbi4tzNFwNeICAcxoWwUfrXfUHfdQf261L0SVpvkIDCdCX96w/3JekgoH9qBvCZvuxZBFmAi9jQJpc+77FRs6+vgkNTGtUTHN/UyEGUDrsh9cEjkKYQ9+snHMSgWp+of2se0GYvuxpY/Vs7lKVkIMoVZfYLfNloISuWXVKcIKi3j8nVcaKoGQWm3PglrSdWGmxJXY/zMUJS0yMErp1X7y0p0p4Pqslewytvzlws0DqlsQ1iumBHOx+cxszAYWBMXWgmXC5pCWGT/u5nFNbN2LCqAnjre0clZaZUCzPnWv/dE4tNrJGRmnJ7Rk5dnKPIW0zIXdiSS8GgLEQ3qyNKECOsiY5yPfqnjtb75pYls3f1z3C9o4IarVMLM0yNYGsJvnZoqyfCy3yt+7JoOt4uQjdNnaJsnZBLakAWt3cpXY9xw5hIuRLBE2KYMxATXEtL1VanbyHipZn0wA6amAEJwiDpZ4JgqVLK4GJCq0ErgbGgUVwFAuLO5goQO4WC3BU64XNr6aEOBHyNayE763rXigApyTnDBYfqO8I4sWinFVIB8l8q9Jt+y/qPVlDPYzRgw2zZCWLr7V3qIKKJAVRsWs2J7PM2Hw5yaFSs8CERUGxPgS9Nq7iSWtKSNP6nf4xhaPwyfjmaZMY07hYpZzHvbSNizJj1h5Y+7iui6Vvf315p0Lm1lyWl7FXArhTqDhofaWt0Um6nl4p28YBVkgMpvGbTCFze9RoCe7wyd6VtuezyRhOLjJe7LhYtzrWrDr+WXE82fOmvbHE+WLPQoVaECExitti7VdE2eBWh5br+plv2n343tuby2o+L/y9KXd481Kfy1zcZ5vCnTGWDvfrbGm4q1y+VPZoeomu34nlE8a3eV/5pBXLN/0F/9jv+t/x4/kBM0c8xhFjTbjUHpfS4bPugkvpWs7EY5zwLouL6oPmeMwckKjiOfcth2IgxhAyjnHGWBO+mo44pQmVA6YaW/68hzihgvDNdGh54/qQ8TQPeOxGVA5LPj/d7ZYj8loSpprwuruucg9afsKOKr6ZFRmUJf9hrgFTTa0vQPJYBmIc4oy30xEVhD5k5Brxur/gWjpcS4fX3RVP84DMAac0tbQnltNwKhGnNOHs8j5aXkbLDWnztzyPlpLlEOeW39LyAFr+QssZ2IXSUoTMNbYULpajMFLFWBKGmHHNHYaYcUwzvpkOeOgmjCUh14BjmtGHIjx3ORUtFjIqrTbWeZb5+NjPIeaWPmbMCQzg+0fJY/mjywO6WNCH0uZqeQi3Mbaj5pq09DG5Bhw7ATcCJO621ICxiMWq1NAQig8pt7yDhZfUMg+duDObNd1S4xw7iVG1HIMEtFxy9o61ONDzLAi9fSy4zF0DjXpzuOCSOzxPfcsxafug6NyMnoayS9y+A2h5CiNx4/E1J8mLSYyrAlM9Tz36KJYYS4dj8ctWP+uhy5QjAgGVgU7jR229cokYUsZlTjh2ueXELFXyBVpOvl4tlmIRF1qPneSxnMziqHG9KVRc5rRYPkNt484lIJDkEhw6QTQdZ1k/y0coaXrEii+5FgtyCXil8cFfXw4tPjfpGsGt0aGX/XOdunZ4ZcjGlmIHEKCsp+uAx8OI8yh5Je3g6WGYWtog48V1XvJYGlq0oTXbPrX0Oq8OI95d5eCpi2WVLujYz3h3HVZI0UOXV3kjl5jeJb7YH+5Z3kT7bml6ADRU6XHuVrkdzfprh3GWq9D6AsT6bbkk5xzRKe9bnkB/2EeMx+OIcU5t3Sxn5GXsVzkPSdflfJXr4hnQ49DPuE5d83bo1dps8em5hEavpRPyKYsAsXxPU9K46SXvoiFV931ehTfYwWizTjtEa/vLTOhcnHStAV2XMetezTmsPAaGVDCNHR5Pox6ortM7Ge1FEZ/nOWJw8d6GQDxPCSkJuvM4doLCrd4SHqHbDnBrDW29So4t7ANA4wkAVEUStnpRD3IpcENq5rqkUqqFELsC1meAKwRxNhXlr6Q8KnNE7MSzwd7hQ58xXr69xTKmijyLghgSL4oQgKwhF1XpadZ6PSSMGs9cCgmgmd2v1O5Vp4SQO+BcHfoCYn3tBaXbEInJ+nQKGmhp20JVVHFnxpJaSFFiW5yks2gb+BipRbrFz2rf7bqhO5vngutHrLR6MGrFDictNtUOLf1hJ7Accuq8Vyl8CIu2wlgrHcbHuk7BA8hhXovd1IOxVWwyub704GJ7qNb6N2Vpe1Jj67VRkneVNtftS6iwuxY8q7cxPGzLrmLdBr7Xt9M2NxP+KKXwp1VXiyf9kyg/A72SiP4cgH8A4jL7mwD+dWb+j7/zcT5lVNgf/v43/E/8uT8MAJhZ0FTHmjCErJ87vRfQUcXckp+HplBWpqbsjXWdTLxyWClV/rP81Ze1tjGlyhQsn8TdhDpr55PATyWh12Tuvvj69t0UgW3JHFcKZwWtEtebAmZgRNaH5dMEFsHTj+dzRfrk7Va2ibYBEbpmp4BH0h9MEALUhYxDo2Xb93be27yVJsD6+x7UaEuL0elBbvx9ZsJcxeq8zS+6VdpM2fMvMM8Tf71s9ofRbkrTqp2bi7lLV/1rCdUDMeYS0MX9HJ7Wj086763l5qZsSpwV72lTefled14NvKEXWNxCjZ82R8/zD12HhY7bnKr+2rZecW3J3ZP5uZNxiLAbQ21rVXfohNaz+W1ztG4FclOEDDxp77mwYkL9Xh7Tli/U6FQvCxPAt7R5mvxfU0A8MJLnsc9xakpT41Glln7F3L9LoZu+Wn1VCkOw1DkbgRTLQb71vwVaWm7eHr/frMvm/URYP+crPpUlJ6f1b0rISmhlkljtVJfPirbswYpWOT91A93sbatjkg6bsLw0bcK29rnkRxWXUC/YNa8Vq6Nrwl55AJqFz+qaK655Noj3iF1b6F+5tHnh1Lvr2XzM08DcIk3ZsTl690dPl10zfmy/bwVXX+7d20qRrFYuRgMGMst3c68EgIKGUPxtyjoXI90K9+3F4hvpX5+6YaP83eSbZbqxIt3QYsoQY3GN9OMZXbrON3ksnRJj361NG9vHIjp3+l1XUFOyd54T68vHQK7qvFDI6uytue9rO2/fwZ4yZrc3fCa/T12d3bLtdzP+S1v7xX53+tod9310rAbDwrN747yvj71uX+rvu+jnO+jbyl/6t//Uzx4V9vGH/If+jn/mO+/3L/xP/9rPfG7AJ26xfC49/o9vfhWX3K0sVwAwl7gSKE1INwuPWWIqU/sMoAnxMSzIoVNOiBofuCcMWvyhCeIi3KsCtXGf9QKg5LSsrQ+zXDSad4RHm4tXeLzAXVTQsjn6uqzjbxWdPSHRhJit8LwV3JpA6YQjuy7zWYRc337bp+VR831tBWazaNipv69r37e0bXm+pcNOX1Nn8XSh5ahrqKibOVvOynYmc/OLpPVs7jvCstHVaNU12grI7PvHWomlnftkbXd+/IPbd+aO7OtwRcsnacUrOtvv3j3b3KtbH/dOTjc/8HaSf3PCvPO8eGH85gdwOxZTy9noXZV9vkHpX/e659d2LR0PV0rASjBaFIJrHlofK5qN1Kgxj97ty/jqFeTg4g3NBbj1ofzxwqgfx/NjT1AIjNm7eG9P/zcu8U1pCJtxbGzjk1egthKxXd/su9W9reBnr8AtmAQBxV/bPHvte4slc8oMsBbi3foWyw9BC4rrS4cEgK4b3X6+665lLLJ7tLnNAXvPDK9443ht1zduxQQ0qwzb/lDLkClZzWrjafR83hZvzZmWz0zq2k1YW5qMTkD8nokbnVTCEs/nYDubQtj+btwqjYdeQXH7AkBLO0PV/aVlqHB9eU1fLAQEOxTYPgqb+L62JVfTdO+8e3wCVu90+ereDdafAzwLjMa3m+L4R5r2hjzfXJ9+nv7a4tVO6y3OolQGz5DtM6ButyAF3bHb7hm4UXi2n93+39J6TxlsU/HrsK2y6dMMptv221fMxyg2u2i6rt+79L6vbOf1HovlizTaHOvOXD+wv49Skvd+lz607c9BoU/YqPe+8kkrlq/SiL/3e38VT+WAWd0WzXK5tQjNHNXaF9VCWZA12COAV9ZMAAhUm8KZte+V9U6tnACapbMwNbfbToMwhK66Ul4jcXPF3LOCSkqU0KypRqNZ/LZWPrvv3VD9/LdWx+QsZmaZqxyQQkG2hFtaAq2tClvLqNVp9G9otDE9XVsaFxrU+qQWJbtnbqbWDlhb/oBFAdkWs4wabcL/xQJlPFuvF6/o3lrMtntha4Xc8nlPeTfL3rYY8FBx1uO9Ndsemmy/G69szxHEdbao26v/LdvT+3zxc92zHK7GU17szRnAzWHBXv/bsbf72fNwa0HzSkBs/Df6lnplI3xFtcZ5K625u2731upAwNFhh0S57B8I+Wvbfvbma3y0v74E4tUeiu4ZAd6v23gXZ3Pr9L/3W68Es0rfo33ParpdI2+9NKvqVkn3BxeLVfj2mv9shyV7vPWutv7+vbENSMsfVi3up1gdXlkdX8xS6/uVL05Sc4rsrgS33V9hCY71fN9eX/EaO2cLnnfYKXvS9Obe3jot1lYsh0qe/s2atO9O2fME+xR62zkv/erBC7vDP2J4y++9oLG9dfuo4g9xtt1sXQ23e2BjuW7Xrd89LWOjVK2u+xjme/U+ZIwPKXuK3s64xLRKKWLXdvvZ62tbb/uu4Z02bfwd/m7bbsudce62udfHPYVuq0zeJ+/947xUl/DyHsB7FL979z6Wn6u275/tByvR76HlF+V3TvmpKpZE9C8C+Kch2+EvAvgnAfwKgN8A8D0A/yuAP8HMExENAP4sgL8LwI8B/FFm/r9f6n+qEb81fo7nPDQFwQRpUwazKhbZKW5eMM6q+JliBCzKkV33Cpv1Yd9zDWsFbyP431OqvEC4FRr9D7FXLLeKo6fJj2Wf91wJXxKShR/hRjna9u3Llg6LF/SuoXsujG2urs49y+KeYOIF7nt9Y9Ov9eFdRU2R6Cyty5xW1mZr42nZWqe3tN4KsfuKpVesvDIB3Fcs/X5pa4p9WcZ4a+MZKq9fMy+87fF56/q65zL6Eg+WOrcKhQnxe0qB5d/0Y5kwu+eWapZ+r0RslQprY26r3tVzO991/1iNv57XMgeLNSsbxdLm63OC3hNufVoiU2piZEHQdcJxCDu0rSyWTolRYdLPzYDKGPrseoGTGOxcm7fKPeuea0MZ7biVbVYyjuvfFJ0bZWuHF95S7NfL8+reQ3Bj7XWKx81+d9bllbXY8d6UmZVy5BlhVl1TdoB9K6+fo7nq2p+thZvieq7Kr62Lrnc9vlFkVRlaxcjdEOPm4flpe8PzmbCvRLWXDrVrbSi/1MRLPN+9fvf6c3Pinb2zd+1mmnR/PT6obOfuad2h3StVpnCZFdlbi1futbuE73z/YMXSaKVbvhIve3c71fVr4b4la7PGZOPs1NtLs7Ba3sbLFxS9j1SCtlt6t817FKibub+k5Nz7Ud70+VFl7+DnPYrb+xW/b3nvfX1/yL17e/XbjvmplV9YLD++ENEPAfwLAH4fM1+I6L8A8McA/CMA/l1m/g0i+g8B/FMA/gP9+xUz/y1E9McA/FsA/uhLY1xrh7/8zS9hrnEVf7eyJm0ULC8IbpWlfQvC/Xvvb7sI8HuK2fZ9TFjivfzn9429R8NWGN5TBu4J0ts+txaCbfsPfT62tGz7M2F6r88boX7vF+4D+LSnQK3u43ZN7PPNWNtfG3/tRRpeEIj3CNkl4KX+FxfMey7Au/vqnmDpBZBV/T0pAx+4DjrUjrL2csOP5/fL/eFl/n+MALNX3wQkz6etpLb9vCnzHRI+mNZt3Q29Zbfems9EUX6AAAAgAElEQVRF6St7dH7gPL5V+Zi5bdvh5batih2e7NzjTV/m6np3z34kveTbNHp26pliuzvm4oHwEufb6+rOGEt/eP8c7tTZPpp323pifV9713b7WM90d6x7/f60ys6zhc2lLWkf28du/ffM7a5C2NrYs/sCLXeuvdg3cHvzQ9bt3vju+/25vKfc2a+r+x/z/H4H67PX54s8bX3Tt5MRPqDex/L3g2n+wPG/03au/Fwppr+Dy0/bFTYBOBLRDOAE4K8D+MMA/rje/88A/BsQxfIf1c+AZJr500RE/BK6EIur4zHNyDXgPAty4rGbMcSMH19O6ELFXAOGWPA89XjoJ62b0KeCGCouU4fXhxHvxh6HTlAP5xwFrTIwPj9d8DT2SLE2AJVAaDkfHw+CPAhIPObjYcTTdQAAnIYJ45xWz4QhMXap4O3zAUSCHFh5QdMrOeJwnEQIIMasSHkexAYQZMRaqdUHJJdgShWXc4+YCrpOEAG7PksOvym23IGH49TQ/OY5outKS88SAmOaAqpzD01dWVALIZaPPMd2vbnMYbGK9IPlE5RchkGR/pLmNExdRi2KRqg5EUsJmK8J3SEjTxEhMkKsmMcEzgHdaWrogS1WEwAXtcwYIuMgmZVtvgCkvyQn90nne/3qADDQfTai5Ig6B0HQ4yX+0n5vylXWOvSSRy0mAfkAQdKgaBwTX6PE92g9ipKTLfRF8tp1FWAsVpVK4FHSAWCQY10KDEoVdZS+MAWQz98WebG0MIBKCIcMzkH6OihyZA7AcwT3DDoU1ByWOLqABZXPLC6WqsTisoSa2xxohAXpjwH0FXSNku/PLD2W662SxAP1Wj+xuM1ZPjfCksPNgEdSBaYg/Y5RcyYqjXOQ+8CSL47Q6KVLAHcs9DG11DI0B/ChoFlILA8ejA8qxE8SM2Y52ahozkamlvMMJLFlnCrCs0D5l9el7T/KtOR0YxknTAGoQD3W9mNJVcdy6WrCJaCeKuJzQDnWFTpgPIeWQBxBUo20/ggIo/CjHqvEm2nfFn+VngLyY5VrU1iBgMSRkB8qQJJ7rg4V4RzBA4NG5amPycxB4nI6RhhDS2xfe17SetjcAIRJ+FcGbikrpA4vaSEqwIkRRqlbB+ELJ0a8CK/KwIhXQu3ZJSg3ngh93TtC6YUWqkCYJZ1APjGCpmfhKMm644WQH+R6GIH8wEjPhHIQHjMB6UItDgkMcIeWVoIgn8Ok73nJ+IIwAWXAMlelj7LUT1eXO1DbWfoGBhAnbUOS+N4sVVHjBG0OcRSegiFpITSWjaOu6ZHRvSMgALXjVSyj5cHjJDwtvfwF6VpGob0cZJwwAeUgY1j6h2qpKADUTq91QBzX14KelJQBLfG9pKGQz6WXOpI3EKj98vohbcsdEK/CP/sX5iV+z/ZgGWT8miCJ6iOQLsD8iG9d4iQ0gpb5A0JrGWQcpmUO0H0BCC0cZH3CjPZs1Aikq62lDsRoSe9tTb0gwQFI54U/HGWvcVzaEC9rA8j92jnaaM13b0EOeVkXKF1htr2o/VsKEttrbpwlNQyB00JfW0/dNyAgzAzW/d3AgpwjFxWAqu7t6OqxXG8ARAFgoiV+zf5EAlWWdCKai3IFSsTQfJNY3iXepZuwvCO3xSuPm1jKxn9d0z2D+ksW4Pau2fR5c6ZKjvZNnO9qrm5frMbaHGj4OMumMO4pyXeU5r34wdW63OuD8F6a1uPcDPNpFAZW4F4/Z+Wnplgy828R0b8D4P8BcAHw30BcX79mZgvf/k0AP9TPPwTw17RtJqJvIO6yP7o3xkMa8fd9+X/hqQyYOSKioiAg14iZA37Pw0+aK6nFNFYmzBwwhNI+W8xj97qsYh87RTW9lA7p1dol1GIsK1NDmLW/15LQf1YQwJi0P1MIfZxc5YBff/PjVRykd6mdSlzFfFYOuzGMiSTVw6S/HnY/fVmwQsCFoMH2Gk/okWPN3XIqsY3/Egqn54WvWzZxWKS8NURaG9do7ELBWFLjiU/zkahiqrFdMz4aX7eW4D2XWLNgG0opsHY/tri04xdfozK11CLbGFA/lkeO3b4bt/Uq0w1iqVnR96zYnWuzdZm+x+PWt14zN1ofgxeIG92jglHtWe63LtOehi0ddYfGKceWBsRiGr3LscUtendi7ypsf+2+0WHpA2zOe27HVg9QN+4oBwJZDzrMzZiIkcs2lnjZ03uAWLZ2fr39Z3t+iLilF/H89GvUZJCdtfO8CMTIJSD9ctXUImhuoIZqu3IPd+69Pu3GnltxirWl6TAXXu9CnDVFyx7IlgcV87T7+/5vc/l2SLVG79ZDwcc02nff1rtJB6ab+gvNypdfY0QGSF1prW3cILHKYIxo1mWSOgjcgFoqE9j2oA3GIpmuPC+c9ChnNWvvAP+WYkCswG4t2dra/tuOp9IeE6+u+3EaOJfythLAFZjiQtu2mAdF0TjJokBeqxjXSsjKn63gt/KQaIIr4Qb9tT0AhJWkbfe2cau+L6uva9NcwE0Y3Urue8KqHZp922L9bOdDWFBfeTM3m4Mhw3yokO75fKOAON7a95W2sUf7C/e3ygOwT+dNl8te3QWF2va1J1Bv57mzD4x8gG/dsP0YG0JpdY136kOe4Y3C9kFWu49QbPw2fG+5twf26t1Txrbjv4/Wl8b46P42mu32+sfQ9QG0fGqFwLvK989L+Wm6wn4OsUL+XgBfA/jzAP7h76DfPwngTwLAL/8w4o+8/t/w2+UVCggHmjFzwsQR7+oRb+IznuuACMaVOzyEEW+L5IY8hBnX2qEi4EAzflwe8SpccK5Du38KI661w7t6xIFmFISmwF65wxfxCVfuca59u3+gGd+UEx7jFQBwrj06unU2G2uHAsIpTIiomPWoL1BFREVHBd+UEwCggNCRKIkzbwXiigNJrslv8qnRPnPEKUy4csJcE17FK861b9cDVYy1a/RJHtArnpQ/pvid9Pi9qII7suTRtDEKB3RUMHLCQFnAk1RxnDmio9LGsByT19o1/p+C5BMNxLiUDkPIrd0QMt7mA45xbqBMQ8iIVPGUBwwht7bCC1Y+LUrBU5H1tDlG1FW7S+lRQfh+/w4A8NV8av0YryVHZ2lK/jFOjV4DhDKQI2tXOeAhjaKsqsJfOWAIM57K0HKmGohKy2Opx/vPZWigUrlGPKQRmSP6kPGcB8kLqgBUHiAqgPFceiRa5lmZMMSMV+mKS+nwLh/Qh9zGBNCU+0DVAVzV9h3AKkdqCqXRbjlMZV9HPKYJT7nHYAcYeoAjvJt1nUuLf174I3T6A4ixpNbmIU0Ya1zRZwBXfcjoqOJSlvx0Vv+q1w5x1jWSehUav0oLArTR5MG57FBjdcDhUwopcM6rTtbu3Ty0+ViOWNufgbgd7EzOhd/H0xotpzTh7XTEYzfinLtVHPghLQ6yBnB1zd3qfmXCtXToNf2SX6fHbmx5bfuQVwrwIWY8ZZmDHeJYztg+5HaAtXfQYjlt+5B17NL47Ov3IeOc+5VC73lp9FhdW0PLBQvIAcn/x967/eiyZPlBv7hl5nerql37cs7p27Q9zYxtxvBiYAZLPPnJb1gCISFxByHzgASCRyzwgJEQbwgkS7aQRkKAwG9IHv8Fg4QNjIAZzzA9PX05t7137V31XfISNx5WrMjIrPxq1z7dPX1OMyGdU/vLjIxYEZGX9Yu11m+ZlA+VSbryRkZSYDZ6gIsyg/1ap1RUKXduTBuDUkTUyuFoK1TSwyiPNuXT7b3Omy21GvNYAvReZHll2lSolZuMuUqbZ+WGVrnZxfW5WK/ymoYoUOsxFVW5yWNS7lgmCDNFzlS+vvx37xXWxub6S5spPors5WOKjRjulzciTbqvVbFOnN828GZO2sCp0pxzn7zBY72apFPi467wLmEPIS6cColzt8YoJozqTMLlAzGzD06h0h6+AL1ahexh9EVKmZuW36G8sWW9mrzTOQY+Fvd++ZzzG5i9mMqcwdzuPM6dS5n3lX8vb7TQxkyMItUvNjniNG0Sx73zdeXGwrkwknIjK6S1kkU/+fogJ7HdpZSiuBczbg/lPJQbDPfnYiLXLPSAN2pY1nttlpsQ/HeOf0rgVvY9xwZL4K4E03OQPt+AAN69OfBQeddGwIPA84ENh/c9viTDe4zp/49g8+ep/DRdYf8SgD+MMb4EACHE3wHwFwFcCSF0slp+A8CPUv0fAfgmgB8KITSASxCJz6TEGP8mgL8JAB/8uev43938Km7sBr3XWSk9+gqdM9iaHkPKp9h5jUaRMhMgcGE6dF5nJejkKjTKZmWnURZN+uC/6db5484fcACokt/N7bAi5SAKNMrh6CqskuLXO52JU1xK5qxkgA0K1itc1N1E0alSfkYtAo6uyh+hWjnYoPIHnl+ktaLk2rV22A91VlSGoPC0OWI/NOi9xrbq0TpDiplyWck52CorEEoGdCkROVlxJRrtJlaywSsYGfL1IQoY5bOSV+YwZEUj59NMykdIykvnNBpNSe9VSoJeaZfTu6yMxXGosDYWvaexr42FUR63XZOV9pI9swRLUkQcBwMpRksbAFRJXiVJwfBR4Pn2CAB4c1pByYg6ySUFKSflB36VEtxz0nbnJbQKCJFIgHiNVsbCR4EuEQI5L1Fpn616nLSdFS4pAyrtUSuPfV+l+52UtMY4OC+xqixOfQUhYm6TlTImFuqspvlWAb0bLb0XTY+TNfm8YqtWEPfIpFjxKQmCAFJmXZpztkz21qA2Nve9qmzug9qn50ayQloAAgDJFTtmRYnvP4CUz0o79FajqSwGp0kRDVMlSamQLYesRK3qgdzaU9J4KWOWgZLej8oWW0PZ6heCyIndh3Q95ZRUua1SWXNOYt0MUDJif6pHy6IiF3NmGgWAytDz16Z1LJ9nnguu13YGm9WAtjcT6+SqJvd/lkFrj743+T6tKnovDAO5xccoJrkot+sOh1OTXLkDtPY5v2alPdreIAQBrQPsoFHVNrupW6smSgLnvPROZpd4nhOtQyYzYkWR3c+HQY0WS7ZmJsVTpJyYpnKksA8aTCpk0tjYdd87mRVSZkrl36t1D2s1XFqD7PrembzmIT1/pnLou4rukfTvuhnQ9yZb/qp6Ciydk9n1XyQrLd/DIc2nSnOY+4sC0dO8hzQeetZpPn0Kb8j3kPY59+dowScXfb4uBpp7a+mdla3Bad618bCdRrMZcigDEzEJEYEELKIXUMbnUAW+hylHK+AGTfdL0ZeQ9GzYQVMqpiDG58gJKEPW9ZBSOan0XLleQ6Z8kpxbVADwVkKm8AKpIoKVyAROKoEgqyArTyDDSzqnA6Sm94B3ElJF+F5B1R7BiUy8pCsHe2J/zPcv0niEIVn02c0/0r9Dr3JogzSeZIvI1jcOn4hOQugi7VAAhUaAxsOaNVud6TgDDyCZwiFXLsvCbUQvp0CGLbRR0HknxnCGIABN6w8rR1DG12UyKpIROh1j8CBSOzpZTlMoD0wY6036SmEQKORDqhPSPPGxUFhDC6uw8ILM+MUee+A1KGUrS7JwihJ0clcMgAMmrqQlDhIR01QucxDKvxfIqKIAJrYFgXuuptwHMLOWnjNxxqmsE9KwJVCX6vAYl8qSxTYmWe9XLsZZ/n6gnMPqi2V+cj7PPw/l59hiKR4KYfyxGhbinwLwtwH8EyBX2P8WwP8G4J8B8D8X5D2/HWP8r4UQ/w6APx9j/LcTec9fiTH+8w/18eLPPY1/5Tf+MiTIelSmDCktL7xrfHJmtkOe6kWVQSW7jJY77LxbPy/lLjv/LneOA8QktQWACcurkgG91/fa5MKKPYDJrnhp5eCdUwYCJQCa7KQnkMeunbyjWu6gcptlez5OXSVVIc+caXWeYuJc+okSqM53Z0s2VAZiHNNaMrXyzvT9NRl3M3l3mo9zKa9jBtjO6om75IRBU07X0CcX2HK3upS7rMe7wQAy8OF6cxZVdg9ksDKZl5mbI7sPlvk+AWTABYxukDSe0c1yidGV+1liXC3rl/lC5/lGy/QMpftlmaJhmn80Fn2jaLN0y4x5TM7JXO9d5E9AUuxTTG92t+QxZWvMbNd6tlUag8xgJV83m5fSXbcExyjWO8sIlgFZ8S+/l/xvAYyKdWL8LOvd281PivlkZz4pu0u5N7NiC9z7WOeUGaUwfN3sfuBjiBhjfeeKDg8sX4D7GsySMlTM1TxHLmJBUFWyo7LOyOvKsbqlCyIru3PLQ8SoGHJssU/jKOvx3Gdlt2h7zgArQEptyd6JWb05udPSfKGYs2JupnM1u25+ruzzjOKd5WUlfN4MK9cPKrCYxuDOrTX8ez4nSw/BkoKZ6ggvUrzfqJGLtI45TrdUviNfB8wcf96vzMdeypribUvwUJYy/m1xbPM1TuOii2drL8aY5Nx+XLhkLm+xxvkYMInhLv/mx39BtMUXGDBxd83gSWAKbOIMQE06u9/PwvDzjyVQNJFvLvNSmT+C8zbOyHWuPNrq9th6P0ZZBK3c9wOA7ewY3iXzI8f06DniNhe+Se9b/u//8t/7+zHGv/DFrv7JlMvN1+Kv/tl/6yfe7t/7+//xz3xswE83xvJ/FUL8TwD+AQAH4H8HWRr/FwD/vRDi19Oxv5Uu+VsAfkMI8f8CuAExyD5YrvUB/8LT38Jbv4aNZPFjl80uGhjh0AVyN+wC/bZRJ5fVAV0ki6ARHl002SWV3VEZAOxDgyZFp/v0lmK3RhtVdt3kv30wqBNDAbtTqvQUeAio5Cbqo8wupVzYpVIhJlfdEYhO2ivGqgS5znbBTM5lt8zkSsvn2S2VZeCiRMgxpiSrvOfGy/2XrqKlm6mHnIAnvobnk+ZufNOXrm9cj3OO0lxqGOFzuyzP3CWYNwnmfZS5SgGOjR3dWksXTQDZlZLjazmPKecWLV1O+VpObVOWMnfpPEfpPBUOx/kCyO6nQ9DZJZPzo44uxjLLECDuucKSm2+Y5Esl11naRGAXXo7b5U2Y0hWWxnB/vZZib3kcPCZ2E+V5540aHh+7HubrC1BebqbM3SPLfKbzWN9yc2ce4+pyfK7PY+AYZ74fxg0cNZkXvofmuVAna404aacr3HGXYml1bnd5s4qvqaTPnhbsNptJtGbj5TGV/QJjjtX5OrLVHkCOpZ1b9PP6FDHaSzlqy3Uw0me31PK6sn+Ot+b7gI/P76t5Pltg3MQq44BjFPeu5TZN8gAp444BsoTPN9VKd07ejGO3UG6Tn9N5/O25+QDGnKFlKWXn9SnbKDeqyvjw8v6YHy9dbZc2PnwQMOr+vTDZ+IxjfLMsRJ6zo3OM7jmmcV6TiHEDbr55WebTPYdTyz7nYyrTCc032fh3uclVunb+OHksy1hu3qibyziPAX8XC3YZO3zeFXZ+0ZkUM8W8lu1HIFup83HM5n1h8yKmNuepfqZzMm5m0YH7/84pceYbL6UQYRzD2dQ5wH1gXPY3R/Mozi0dX6p/rixthCyUMu6Ufz9apnmdcw/GUr13ga4HxvreIHLh+INA8b1A5OPX5L3A6Zep/BxbLH+qrLAxxr8G4K/NDn8XwD+5ULcD8M+9bx8bMeDTeIkQJa7UETZqfOouceO2+E79KTpQbOXeN/jQvMV3+w8gRcCH+oS3HjjFGhvZ47P+Et+oXuMz18DA40qf8Fzd4W1Y4/vHp/h28wqnUKEPBlvVoY8SV+oEHwX+sH+BS32CER7X6oDfsx/hA3MLAPjhcI1L1WYlNUQJIxxu3BYHX+Mb1WsESNy4LWpp0QgLIxyu1AnfHV7glGIEL/UJB9/gFCooEbIXCPf7VB3w3f4FbFS4VC1u/Qq/3HyCT90lXtkdvlN/hj8anmHvG1zrI3aqxTHU+F73DJe6xclXuNYtPgsX4FhGH8f2Q5TwEHhjN1irAWs5wEaFLhg8M3u8sjvsVDcBt7duhUvd4gfdNbamp3jKqHFwNZ6YE17ZLZ6qI97ENdZqwMthh+fVHi+HHTayx7fqG/x++wIvqj1u3AatN/iwvsVOdvjd9iNsVZ/jMvtAsXiNtJCIMNIhRIkfdNcIENjoPscbvqj2uHUrGOHxWX+BPmj82sUfwEeJ37r7RWx0j2fmgL1voBDw2m5SvAwpSV+r38IIj0+GSygEHEAxkyEKPDEn3LoVFAK+2dygDwafDJdYywEHX+OpOeLlsMPT6oBP+wsYESCDzjGT32xusJYDvts+B0AAeu8afNTc4uBrfL1+gz9sn0MLjwvd4eBrHF2NlRpghIeWAT9qr7BSFl+rb/H99hoAcGVO+OX1p/jcXuAPTs9xadocP2ujwjHdZxxDx67l/JvB61NzxJ1bYaUGrNWALhi86re4ro4wwuPlsMMvrF7jD0/PsDMdjPA4+hptivV7ao74rL/AdXXEnWtgRMDNsM6xbFdVC4mIWlLM8s2wxkfNHX5weoJvr1/j5bBDHxSGoHFpWrTewAWF6+qEWlrcuVXeHPjFzUu8HrZ4NWwgEbHRA1ZqwBNzwvfba9igMASFjR6wURR3/NauYESAjRLX1QkA8Hm3gxQRF6bD22EFKSI2us/5cy9Ni1u7wq/sPoZExD+4/WYGfpemxctuO4mv/PrqLRQCvt9eZxC7UhYuShxdlWMwv7W6we/uP8Cf2X2G752eovOje/F3Ni9x8DVcVGi9waVp8cPTVQbfHzYUM/zD0xWe1kcECOxtndv+ld3H+O3919E5gyf1CTvd4ehqSBHxp1av8Nv7r8MFiUvT4bNuh6+v3+Jlt8XT+oRb24xuzFHhqjpBiYjbocHz5oC9bXBVtfi82+JJRWsUokhhBgFP6xNWasCn3QXFwXqdgTPHfnKc5/P6gI3u8XF7meNInzcHAMBn3Q4fNHu8GVZ5I2adYipPrkIlHf7xyx/hh90TvBlW0DLgRb2HFBHfPTzDVlNMLMv1rc0b/N7dC1xVLT5qbvEP9x/gO7uX+N7haSY++2h1R/GYkuLab4YNyasttAg42BrPmwNtSKb5vK5O+Ph0ia3p4aJE5ww6r3Fdn3ByFb6xfgsbaaPDSI+bYY2DrTMof14f8ruL74HOG3zQ7PNG0NFX+LC5w8ftJSrpcWcbrDU9350zeFof8cPjFX7l6mO0ocLNsMZKWfReQ0uPlbLY2wZ3tslzyvN8oTscfQ0bFH50usR1fcLXV2/xR6drbNSAje5RS4fP+h0u0nv2db9GpTzuhgbPmgOUiHjdb1BJhxep3e8drnFRdZCIOLkKF1ULIwJe9ltsdY+Tq7DWA276dQL69KxKEfCmX+Oj9R06r9F5g95r7Eyf7/XX/SaP+Rubt3jdb3Kb39rc4HduP3xfVSOXp80Rn5125FJe9TkkplEOn5wusEthJ0/S+nKoS4giy3y0NXZVhyFoDF6h9xofrPfEWJ/CcTgkZ0heTUdX5ThigOLiv727wWftDgDQOoOnzREnV6H39E0BaMPkaMn190l9wpt+nWV0QWJjiDPgrm/Qp1AXAFhpuj8abWGDQu80Luouh/EYRRvUJ1thWxGfwGGo4aPA09Upe2PVyqF1BsehwlXT4qYlLgglA4wM8FFga4b0bGiY9A4bvMpEb1r5vOHTOY3eamjlc5iLTmRmg9NwfiR6482R3tFGEm8oAaMnEpO7cUiLS678JbQJUeQwHS7zDQwOlyi9d3j+rVeTzQ8OO5iT0nHIBG98lB5IpRcPhxuM7YgcdjDfcOFrOHuATDm7y3HQ33GDYPQqQvJCmZo8+XgOQSg2fLjMj8Vi4wMiLm9oLP1l2eTstyiuXyiL8bRflhKxTGL1c1J+aq6wfxzl2Z99Fv/Z3/jL+Ky9QIDAVvcIELgbmvxi5xjKvW1wWbV4kz5SO9Ph5CrYoLDSFm+7FS7qDoehhpIBGzNgrQcMXuOz0xYXVY/ea7jk6hqjwNrQh/tNt0KtPMUnSY/bvsFlTeQ9d0ONRrssM+9Q906jdxrPN6SEnGyVyRiUCGiUw023zjvdVXKvZRdUflgbTTFIK23xplshRkq30lqDF5sD9kON1hpcNS32Qw2f4iZ5J/9t2yRyAzreWkMAKrF3royb7Oy3KU0Ls5e6IHN/rNTyDnxnNVbGYd/VqLRHpcdULk2KFdw2PdrBQMmI3ik0xqEdDIzy2NYDbtsG26ZHZzWcV9g2PRrtcHNcw2ifd835BW0UxxrSfb3vKO6U1yxEkVPACBHRDQbeS3zzKbHCfna3g5Ih11Eyoh3MZBd6Uw8QIuLUV9kdmYkXGkPxgFJE7Joeg1doB5MJH2rt0VmdxyllyC6rIl1jZMBtSyRKPgpYq7Fd9Rgcjf+ubSBEzPGO5YeQY/eUCikek2LzjPa43pxw6GucegOtQo7LdF5NYixDFFkmZvfkD2ed0vFwDKfzEr3VqI2DVgGn3mDTDDh2VZbJB5k/hJV26AaD2rgUu4ocgwgAWo/xvgDQDxp15dAPGuuG1iQECe8Fxdcl1+EqxeJx/BjHEfbWYBioH6UoltAoj2ObCHaCgFKkEAgRKW4sfVTHOMUUJ61DbktrD+dIWdCa3HSvdi29D2432dWWU/1wHJuUEXVFMZbHth7dw5l0JMWhAcC6GXA4NthtWxxONbnlpvrbTZeUFVorYxzaUw0hA2KQqBuywHedgTEeMVLsHst1tWvxdr9C8ArauKyUCBGxaQbcHpoctzf0Gs2KYg2rlBIofzWigDYur2Nd2xz7yCmOOD6PrTxVSnvUdyaTfER2L88xlmEiW9+Z/JxXdYpx7g2q2mIYdHb3ZaWKlaLLiyNOXZ1jLE3lIGXE6VjnWL/gKbfHajXgdKihjEddO5wONdbbHu2pztaYqnFpvSiulFM3SRkAQWmUqtohxhQvK2KaQwOpKA9FCBLBU6xf8JQmqrSg2UEjeAGRWFlN5eh8upYVSY735LVtVgO6toKQAd5SiiaeM20chlOFzWUL7yXNR+FKTCmgJLyTMLWDszrP83ivC/Stga4cmsaiPdEcKjUqeiAAACAASURBVBVoPXuT456dpZhSZxVMktMNCkIix812pyrHRIYgoDTFv9qe4jhzrG6KIaS4xbRBOyiYlaW5dCKf47hQNygo4+E6A93YPB8hCNSNRbuv7ysUjyy69nCdJvdMEyBVzIqv7zTFYDqZYjsp7U9Mcca6oXdKsBLSBMSAHCOqGkdrzCmXgNw2AIo1RaFQewGztuP8+NSnlRkMCAFKlZOulTXFhwodKGYzUNylEBQfmmMcRYTQEdEJ+hsE4EVOncX1hEx1UkqnmNJOidrnMQsVKN2VExB1oDRcoD6yOzSnhHJidFsvU1qJOLpqe0qDxOmL8nmArLW+SN+TrHd8THiRXX5jGZcpQH2X7u8lsoygtEQsUz6e5jkkt+ySAZllkKltHjOQ6xfYqXAZnsZy5rQtU2w3NS6ndEHZ/bu0Hk/mAV8sxnJuNWWraJyO615ganFsblmcXzo9iXvz/5OKsfzuf/jv/8zdRS/XX4u/+sv/5k+83b/3f/wnP/OxAV9xYLn9pQ/jP/Zf/ctok7LJOz/WKyJRkGMaDWagc+llx2xqY/yWyko+K6GsVFur8gezdMdhMJBJC5I7i3cSKinIRCIw3XFixYAVNwDwfoxX47/OSXA8F5Mi5N2fNAes4Iik1ESM7nekeItMKMFsbDIpuSSfyjtBLBfJiBy3VbqrxaTw8PUxk6gsuPVxjFgiThCJxj6yC48XmaRBiIjoJX2EvCSa/3Qt1wErFoIIHSYxXfwdmX1QmDyh3O0SaS5pPER0oNbJ1bnTVJfjkZJcEHH8mGomWeCX//gW5rb5g55dgNIOH39khYrF9cgfTaECEZdYNZ6LyDFh+SMNjDIW19Mgkjz8kUzzIauk6Dia33txZ8W8LbofASPJQh6fGD/I/HHmvJayuJ7jrphIQo7zmeXl/nlsPBYeh46jshGLNri+APXD8powkkYAoyIj4kgyEYu+BCauWJPcnnz93EWn/OCZQP8eimdBTteA0H/qr1Agcyk/4ipSnXLcXJgso/w6lykUyhxyalSgsnuWiYAVdIzXjsei0rmYlDEnEE3MylyOseMpSHNf9kV5Gou+43gN90cxcnE8x8pbjtOkdliG/ExznlInEHWh7PH8xnEsMeVIFWluct92VDB55zialLtURerTSsQqUN2Y4rj0qHBGkfouCU84NhMonkNe60L5DUmWKMa1LO6BMr4xzylmCmlOHYKxnaTAlrFsXFdYgVgV9/RcIQ6CFM80p/m+KOaU1pTqC5fi+3j9XVLoo8g5O6k9vhfT6ya92so8j1leAcrvWSrJPHeFsi68QNCUfxRBZHKV3F5SoKVL9fidE0keOcwfvMeXqNL9iJmSLoo8rEmeCdkL3z+8/iUAiNO5oH/cBw98HHy76fHZoD6pXvk481+g+FTxseKVJiKmcaEcL1oS0pT/TrKIBGqQruc5LuXNx1OuyDwOlovnonj/5XlIQmc5l0ho+PtbPPuTEmd/i/4zkc4sFnd+fY7XnbdZFoHFPuZ1F8HUUpks3qyPpb7e0e7SJ+ydMj3Q5hzsPljm5x8h7+Ta96n/QPl//osvQYzl+mvx137p3/iJt/ub/+df/5mPDfgpu8L+tMt1dcK/9Au/hR8NT+CjxLU+ogsGt36FvWvwvNrj5Cus1YA3do0n5oSXww4SEU+rAw6uRh8MNrrH5/0O18nNz0iPjepxqVvsfYMftVe4ro7ow8giS+53R7Jy9ReUDiIo7EyHz7sdXjR7sgjaFVbKTuJsjAg4+gpHV+GbqzfwkNjbBlp6GBFQS5tciy4mMYBD0BQfVzxdK2VRK4eVHPBZT26sGz2g9QYfNbe4cw32tsGLZo+bYZ1dCNnd72W3zTFcl6bDrW3I7SOxdu5MP4mZux0aNMqRG1XQE1fAlbKTGM2TM1hri7fDKrPsspvaVvc4uBoXpsPB1qiUw8HW2Joet/0KtXZ4Xh/wSXuB6/qUUyNcVS1WyuKT9iKnNuC0G5V0Oc8lWyzfDquUBsHl9BAXVYfOGUgRcDus4IPEL199BiUi/uDuGYzyWOsBnTfQwmNvmxy3qkTAVU3umneWrIrsNhqiyNdJEXFVneCCwp1tck5OHt/G9DhasliVqRKeNUcY6fGyowzeLkj0Xmfr+4Xp8LrbQMmARll03sAGBSXIHUiLgLf9Ckb5bKEHyK3po9Udbm2Dl+0WK22zW6SL1IcEWUHZMl9JnxmMObZrpW1mAOZYwb2tU3sOt8Mqu1txaobSdWqjB9wODTZmQOsMsR/bKlu7V4asJDyfh6HGRd3hrm9w1bQ42grWK7IOa5djATeGrMi915nV9tnqiNYZHJIbWKU8auWyNwCnV6gVWTElItpE8BWSR0KMAgdbQYBYfk/WQIDSVvSOGIXX6fhHG3KT/OH+Km9qrY3Fvq9zfJ8SERd1By0DXrfrrDOwVwAzKEcAT5oWnx12+GC7x5tuBevHWN4XmwN6p3MqjbUZ8LZbZcv6piL3trftCptqIDfKxIQMAF/f3uJHh0s4r7AyFnXyJgCA6+aIT44X2Yvh0Fe4aHochwqbakDndJE2A2gMpa5prcGmGtB7hUa7zOjMTNCcO5T7Y2ZnH0XK0zm6m2nl0VuNdUV1eQ6tl9jVNJ5SLk4nwTGEzOT80W6Pu75Ba01mmpYi4s1phdoki3Sqe71u8eqwQW0cdnWPV4cNnm2PuDmRJ4gPEruG3GeZGby1BtZLKEnvnMEprKspI/jaWNwlrw0mJXNBYl1ZDE7houknsbLs6scxlJt6SDHBYuKhsanJYyaCNlM31YBDX2e2ayYuc15iXQ+4Pa3wwcUe1qt8L3AsJ+cw7q3OXiQ8Dr43fBTYdzXWlcWmGnDXNdCKvFeM8jgOVb532ctjcMQSLUVMninkSSEA3HV18tKh9aoTW/Kxr1AnbxStAtqBXNtjpNhOKcjTZLvq81x6L2G0x6pKaXaSV8ixq7BpBnRWZw+Py1WHV/vNA5rFw2VVDzh1ZMU2ZozPZm8RZliujctM8MyMvKoH+CBhnYJJ90NILLrrhs65tLldssnTRvOYW5at8rtVj24wEIIYiuuK2MMn+XAlbVLHKFAnVmf2UAhBoEqb29aq7C0AAEpFeE8uliGIzNLM18lk6XZ2ZBC2A72Pq9pmzxG2eHuniFV64J2GmGM+lfZ5nmQxvnHDmOoLkHU7WNqc5flg4jLegJ4QbgFpQzxtXmdLYtpo4I3jchNzvk8eMd0MTsfGfxcbnLzRw5tNMo4bmYy+yg3xEjiKVKfckChJlfh3KP6WG1pz6+LcHLoEmudlviNRtjmfEzY4FKjybHzpErBcOl7KsCTTn5QvfflKWyx/8c9v4l//O/8oPCS6YCZkMh4SfYr3IzIYIoVphEMXNcKCP4BPZB78b26PYwMlYia9yfUjxSMyiQoT3oQoJv8GiAynLEb4gkhlzGPHMpuCVISJbeaFZQlRZnISVlDm+Q5LApIS5JbRBGWdMo8iy2Gkz0ADwISwZU7aUxKyhChym3OSFyYEYeIZnk+O/ylJc3qvJzkM56Q5wJQUhfuby8vkIQyuOE8gxyUNacxzQhkAWcnjWI1J3wXpBt9jE5KcqCbkQeV5lt16cs8uiZuY6GTwCpUa17mcZz5W5q8r5eaUNgxiytQ1S/O4RNrD4HpO+sTPAgNozl9YrgOT+MwJgJaIcZaIe9jrIK9jMb65rAAyECtTzbA7dNnOUm7B+fzy7zlLcznnfXJNXVqfclx8/3A9YHkzmpmbhwTKy+/wUIBEZiUuSWBK1+Y50y+A7L7M15ZkIy4p6SWrM4Mnzg9YFgJdIp8fSWnG3zQPVJ+9RuRMNm6rdDsP/Kwpn5Vlju/Mrvez+eU5D1HAOvI2kYIYeNlLxWhPelc5bqdgEtj0XkJrD2t1dpPOyi44pgqT+Ki8FgUwWGIrZsIXdjdn0MGFvVbyWvqR1XfO0FzGPAUvk+cMJrFRfE4lBV8U98Kc/VmkeWMFHkDyuCAlOafZSd4mlOqEFFYxG2PpmcJuk6UXCbmCFvXTeLJHCVvmZCFrUSckZZ2BArtr0o1PAIHdOSeAgb0qvmhhAAKMyjor8YV1+R6jsMDopVAq7KKoD0zZlEvzUvYAmcnCtw8DEgYnpULPzcwBicDUml+WGTtwttCzzHOgw/8WoHQgXBhkydmcFMCHLYHZcjm7NFvMeVhi/H0PQJUXc5mZ6iafu/LlW1oti2nL9ZbAzRwcFoXlm7uz3ivn2i7neXZ+NqT77rFL3ZTno7h3/mwb534/APYekmOxzXeVR9R/zBz87n/6JbFY/iP/+k+83d/87V//mY8N+IpbLI1weK7vCAQqRYyvSfEFgC6QVYqZYW1UkAioCzZYYARxDNKYoZTAXCR22WigEDNAVYjoop6wps7BKquCChG+eEVloIQIrwVsGBltWRZWsidyzZ6sEgQzMCuPryVZLZhdleM3S6siszgugQgaw6h087m5Ysf/Lnfd53+lINDI7LDzOef18JCZXdYpYmNlqyQArBTtnNeJnKckReLCVl5T+K3MGS0ZWDOTqpFksWJwPmdGLfth4LoEpst1YFBeK5eZXcuUOEyuwPKEtA5OE+AlwOLzcSkinB5BfAmwy3XnYwzeuXRSZ6umK9osxzkHo+WGx5zhlBlUSyDNrLflJse0HTkB0jy3S8f5HFujyYIcJvWWmEqzfGpMt5PvgwTstJwyCJ8DgQwa749jyv5JFknaJCrXdSk9R8nUO2HkLO6d8rdRKlviuQwpHpv75jRH+T6ZPZdz2TnJPZeShdTrEfSy/JWaMveWZb4JIMU0Qfx8Xr1O4H7WDl83B9cxtc3HXAGaOaUPyzZnDuV8q5mJNEhwvtnyXgYAKwMBzijgFaUqEgAqXc4T9cFycpxyCc684g2l1OfCXJT1yvQ0zHZaSubVaH2ag24uHMbAhCFz5lIGzWV+1TK1zwRYFnH8FP7BfYzXjSBW5BANTsUDMBika4JM85FAb0gARhXhHVkOpA2HQobcrog5FleKOAnJEGnugi/6VykUQ6aUNEnbCUJCVV8cWHKIB4Ac2kE/UKT3WWA/BRBlIV8YNwxyyAUw1YoZKJbHSxDl5AhmBUb6Xgky8ZZgidsrwxEKmXI4QSzqCiCGmMFkLOY8y1IiHD4nC7lZDmAMmUBxjvsCyJU6TC1eJXiPPE5R/AamoCPLE2eumnEUt7xOzP6WU13WCQtjnb1D5oAm9yNmv8+Vc8By2sxiVZ6bSR9z4Mfn88Bm3f2xAsv73+xz7bwLLOYmy3X7Upf4iJvhq1u+0sByiBrfG55jI3sMUeET+wRGeDzTd2ikxQ/sNYzwOPgGl6rFK7fFC3OHg2+w9w0aaWGEx943eGb2eOM2me30jV3j4GvU0uFPr17ik+EKl4rIOU6hygyYAQJfq9/i5MndrmT+BIDn1R53rsky26hALJADtqrH99qnibGyh4sKNqjMDvmiPuS0JczEWQJZdqENELm+FAF712ClLL5/fIIL0+GqavHx6RJP6yO0JNdOtsh9bXWb3XCPyTUVQLYW7m0zpo4QETvd4+gqDEk5bpTDnW2w1T2GoCbK5UYP2NsaT+tj7lPLgI0acGsbXJoOb4YVtqbHEHQ+zkySr7oNXqz2eNVtsTU9GmXxebtD7zU+Wt/h5Co02mJIlimX5o6tcUJEXNW0Zp0z0JKU8dt+hZW22bXWSI/fufkAIQp8+/IGg9c4umri3gogu4vdDiuEKLAx5JJX5jk92oqsjVHgbUdMlNuqh00A8zDU2FY9TrbCOl1fWhBv2jUGp3C9OeU535gBN+0aRnncdTWu121mzauUz3PuU37Sy7pD7zX2fY0nTZvlenNcoaksrtctWmswJNdEZudTaXz8b+tVTrnAintrTSZhYre8TUXkTYNT2DU97roaF4m4KEayNtXKo/cK3WCwqQecEoGPDxKbasiEQCdrskuhEBEr43DsK+yaHrdtg1VloSRZrzunMwA6DSNJkVFERnV3atBUFqvk0meTq591CttVP3EBtE5l8iW2vrE733rmXhejQO+IiEmImI+/vVsjBIlnT/bZYthZjU1y3WTL3rElN7rNasjPMj8zbJkDgGNbEcnO3RqbdQ+tQs7jejg2mXRFqYCh19huugwQ20TatF13aPuKwKH2GZQd3q6wvWrJ0mo17KChjUcIAsPJYHPZkSvjoNHUFm1n0DQWXWdQ124CcKzV8F6grh26toI2fiTyGShmmcmPAKDvDIIXqBqXyYoYXDknMxkSEwB5J1E1NjMf9i15olSNw9Bp6Mpn4OXTfcOkQPZtA7F2NLYo4DsNBKDaDRSTHpGZHN3BwFz08FYhtBrmooe9raG2DuyG59r0yUxxnaIOmQglBgGpA0KrydJjCEihVxBrR8QmAhRHrSJCp6jOSU81RhOIDIWV7U6NSn+OYY1AJkGh46KViKsUV1x7Ah0BFHvZS/QbB/GmQlQxx55yzBzH0UIHiFYhVgGyTZtrA8UpRgUMKw/RS6iTxLCl+FPh6HrfRIqPlKBYTi+onY4IXVwTIJyASr+HnYccErmNphhQ6QTcmtrleLxYJwAkItUHYOsAdVCIOpKlTQJiENB92jisI/RAbemTRChihOujxHD1xYGlagX8Onk8WAHJ90IA3DpCDtSP7kWOG42KQI05UYxq0IAa0lxJOq9aAmOBb4dA42eLZNBprZIcUQDmIOBWBKCCAnQnEEyKNU1DFB6Ihm4b3Qn4epQxShqDiICvYo5tpbGBfqcY16gA2Ysx7jkIIADBRKgUs+orElC1qZ4ApAOCSfVauk8AsoRyLK4c6G8wgHA8Zxgtk25q/Ys6zU2a99LieS8WEuOx8tzESprOSU/zKEJhDQXJQbG/5YM6biiw1ZbjNKMo+injjIs+GbDmY2I6xtwvW4MxtlHKzP1zXPEkTpbnL4GzvL7xDFArZCr3DPJc5A0K5I0GETHdvCjbKsscJBfAKgox/l4C+2cA7HTjYGE8f1J+JuUr7Qp79WdexL/0t/8KPj5cZopr6xVu+waHrsbz3QGd09hWA25OKzxbn/D5YQspA55vjnjbrYhpsx5wc1zjct3irm0yI+l1c8LRVfjB6ytcbVtSwLxClVheL1cdYhT4/G6b3crWlcXNYY3r7QkRwNvjCuvaZkXMB4FKe5wGg743+Nr1LWIUuOtqaBXQaAcpIrZVjx/dXmZFcFMPaAcDmxQnpk/eNAOxmBqLl4cNvJfYNFT3F568wet2jX3b4KPLO7w6bEj5XxGzqg0Sr263mf2yNjazZbJyt1n1MIpc4aQADm2NyrgECgQGRzE5x77KjKiZFXYwaCqL/WGFqraJCVTDWkXKalthu+lw6ipoHdB1BqvVkNkGry+OeH27wdWuxbGrYAeNq4sTNtWAj28uiaEyEysRQRFZKHx2w9sfid2yZKfcbbrMnHo61ohO4pe+9SliFPiDz55Ba4/dusepN5Ayomtp04AV6YvdCVIA+1Od2TC1JqV8u+7R9hWkDHi6PaG1Boe2hjEOfW+wWfU4nGqsGhq/kOwSFyBlxNW2xa7u8fHbCwCA9xJ20NhuO1in8GR7wsu3WygV0VSWWG0TWRQTTh0PDZT22G063O4pxrKqHL51/QavTxu8vVujqm22xFivMCRmQZNAAc9nZoRMu7WrxqIfNIzxtN5O4XSqsV73MMpjf2xwfXHCze0ms1Y6J+GSS+K6GXA81ZllVAhigeSdfm0IJBCTqUDXVthsOhwODS4vTjic6sQEyeyVJHfdWGKUtBTLAwBPLo84dhUBkWRtqWqHVWXx5m4NjsnRxud++76IsVz3iFHgdKR1rmqLvjUQEtCGmDNjAEztYHuND5/fQomIH376JMf/mIrYOJHAlVQRu20LrQJubjf5W8zWHDdoSE1f9YvdCW9vtnj6bI83txtihUztPL0+oB0MgSersGos9ncrstB4gfU2pdK4a1CtiXXUDTqzTH7rgxv84PNrBCuha5fXQ6mAFxcH/PDVFbFO1g79scLmssPpUKNZD+g7MyGu0rWDVAFDZ7Da9Bh6Q8ybx4rAYyIQY1bLZjOgNg77w4piq1KMGa8/38+u16jWA+rK4XBoEqmVxHrXI0agPdRY73q0p4oIrWTMADIMCkIFfOvDG3x+t0XfEbher3tU2uP16y1M4/KcCxnx9MkBr15eQDcWV7sWr17t8OzZHq/fbDMT5uaSNmpMYgE/nBq49A4SKd5svaH7pu8MhAzYrAbc7Vcw6T3rnUKwEtV6gLMalxdHYjpOls1jW03WarPt4JyiNUxMs95LXOzabC0eBo0nuxNu7tYZfDPgdlZjt23x9maDr330Bp3VOLZ1fta1DqiNRdtX6DuD7bZD21bYbVtIAWxrimN1XuL2do31tsfTzQmfvt1hVVus6wFGBtycVqi1z5snWnt0bYXtljYrT+m9frHpIEXEqzc7VDU9t8OgUNcOlXa43a+xWg3oe42q8miPFSmxnu41IYDhZLC9ajEMmubTCZjGYbvuIAVwe2iw23R4+3aDy0t6b9Q1fS8+vNrj+59ef2G9Y7PrcLxdUYxl4zJRn9Yex7sGpnFwg8J622MYNIIntl0A2F20cF6iayusNz2cU3BWwTuJ7UVb3DcxvwfZVXroDcq4RO8kXjy7w83dmmIsB4XNrkPXGUorkdpQOmDoCNmstz1Oxxp1Y3O/9cpmlt5g6bmBAHTl4K2CrjyNwUrUawvvZCZL5LbrFcVU2l4DQWC162CtAiKxF/edgesV6s2A/kA6hpCRNmWCSNeP7ygA8IPMmyvMvgsREQaF2EsiIAMg9MjMG50cXZ9TH4gCcZAjuRWTTTHjLpPZWTkSz6k4dWMOgphu+yJ/NoMtdvEVkfpIgJuts0KH8bqSlI6toKVrsMCU8I1jJ9nyDIxu3fyX29GBxqLDSOrHCJPJuVxByjUv2TKLkaSp/M11gBGwlpbmJSBZXJNJ2diyyt4GM8tlFAWBW55/TIndZpbgxViSEggX5Y/+6n/wM3cXvVx9FH/tO//aT7zd3/y//rOf+diArziw3P3yh/Ev/Df/Ik7WZFcwHySsl3BepTQa5NLEwfK9pRcsWwb4Y87xW2zp0jLkGKBuMAk4yOzeA4wpEYZErQ5woDwBDQCZbbbMoyMEcgB8Xdsc01PG1ihJMnOMDoOnvKmTHkbuh5XqGEXOb1TXloLmPe3+cwB/6XqVaecjEqPtlCBAqjBJ3MzB/RwzFIPI1oEyATWnF5ApbQDT8ccUz8WMu7Kw0MTkysVU+0p7Utp47sNISe9SupBJbqQkd/nXe9oRZ+p+nqscR+doe7Fap9jK9FEv8ziFrESDdldnrLClK5dUEZF3fBXFdXGMEc1VYsllIgD+KIiY25YiZsCEiGwJiTFZRJh2nllx43i9EBHBpRivxLAbI31EdVJSmFF3EpuU7rP8IQ7JfUxM+2C5+foYRWK5TWy2TiYZ1fT69AEVKub62fWr3F1mbzCWzae5SpT2Ma3nhKwAmMY88U4xx3CVLLosd5rDSfyPKD7o3GYU9+K2clv88eZ4rtrTOrdq/KAxs2s5Pp3eB4Mc6/E9XO78mgAMkij6WcnibdlqpjyoSGy03E66RyeucsX1og6knEWR5yWzHpvi3JyZVs3mKGK0ppVrUsZTsYz8qWGljRW68nypCHDbHBfHSo5ODbECWbLUskLEv5sAWDG6FLJSOcgxpozvocQKG1UkC+BArLDMkMuWtbxegsaY2XYFRpbdsl2JMQVCTIpUoLZEEIi6NDEA91hh9Xifl0QYWZY49puZdlPsHStpkcfTEDVrychbyo6SFZYZcLO5A2RJ1LQmmQWWn9lyjPxuYzZnXi8ZR2beQRb3DkZLmC3uC4EcA5gtOWJkKkZiCEUka0wGFMwYPCRGY043EQQCW1G/YIk6QlpWqJHv2SjpOFuNgp5Zh2KyyEWQdWmJSRXI7Ll874yKOWZWI4FQB+ozzRPdU5i+K/jZBMaUGbxOxf1KDMfI63nPy7VU7jM4GI9nNmCM9zbPi/DM3IsJK2zGAGw5L/AJWw2jKMaMVK+w5LKsi3NUtFVaK3ObsZimMN4jcy9Nkj1OnstFf8sS6PB9IcZ+H7y2bKN4zSx5jN5zD+W5XTg+aXNhbt4l05IlcGLFnMn80yjv4w77rvI7f+NLEGP5cw4sv9KusI2y+Pb2JjOU3toGUkSsFDFUvhnW0CLkRMsHV2OtyS3tLrGbaulxchWuqhZvhxXlrgyUY7J1xGL3netXuBuIwVAiZpfO/UA5Eq/qFl1KBtw5g6u6xe1A7q/MflkWHyRq5VBrh5enDaSIxI4YJGwCWb3VuL485TgdZuIs3S6pP0q8PDiFZ9tjPlYrjzenFbbJOrnvalyuWwgAQ+HGuLsYYBMBRmsN6mQx9VHAyJAZF9kCuK7s5HolR1dAVxB8MEMjsyTyNUpGymWY8mFyTkPrJRrjcOorNJWFS/kfL7dtzmvJDIHOKVxeHOG8glZjnBrHPpUAfFfZzDrI8VDdQHkcI4Bae2jl8eoNuS4/eXKA8wqD1dkKyeyRDB7bntbTrMYE1DlhfLLMxUg7zFIGmNpnC+AwKKzWDsOgYdY+p7vheKahN/BeoEp5CKWMeedfqgA7aDSbYRLnxCCYyTya9ZDT5NSrPsklYVsDaQKa7UC7zmnelA4ZyHGbZUoc3kjgBMvkMinTBkCE2QxwlqyaZmXhOpNzzAGAECHnV/ROwqxo11sasmhyrBVEiguLgsC5AHTj4QYNs7VwvYZqHOm6MsCnuQYorQ9vhPCmgms1hAnQzZDjyIIjIKxWHiXLIBOQyCqMYT+WvtRq4ygWzEnIJv07AV4hqJ6oPcJRA1FAXQw5vipYCbVzGbjHIBB7Ral8Vj5/jDkuK+eXE0DsFOTGIRwMxMol5kO6z+NJp53sSCBykBBrNyp8aYd86oLpx7ivvQa2bgTZDBqDoHMbT9qQillP7gAAIABJREFUo3Zjp8i9cpAEavMGDuj6ADqe0nXAi2naGRlH8D/InMojg89kJZmARRUBKxM4CDS2KkL0isAQA0FO85LBM5LlAFA3GqGJKc0DIDpJ4GLli02C9GevEDYewkmIk0RYB6g7Bb8OOS9eBiRpCoKJgMKY8kQBsk0bZalPaQVCHSF7Oab2kKDfKkLf6qToxnRdASYFoA4yu8ZFNSq5+i7lLxSk2OtBwlcRMgoEndwLQa6X6ijh1wHmtQYkKE1FkElZllnxDiaS66iOkPsUApHSf0QB+IbOywHwFd0i0pEQwcTs2sigKmpA7tOzYAgYS3ZXbQrAoUDujj65alqR3R2DQQEyqY1gALWn+WOXSeEAOUjaH6rJbdY3EeotucJKR/VVp2G3X1xDVb0c3TkdyBUWJJtbJdlVRNWLnEKE40qrNyLNP7masjthVDHPC68xt5n3EtgVlt8ZCtAvFVIUDrmqDtQngQyRgeuY4oXuDQb+MW16iEgyTVwlOWVLkSZGWjECytR2UOMchLQWchCTDYegU1s2rSfG66Mo5oJdeJNsXIQvgGaaP7oXCjCJAugXYD6PcwYu5+lbIj/H7NZagjMG7gv7ESxT3iQoADcAiBDHONYSiPHfmRwUCyoWj4k4brhMwauA9HHZFRbj2KMUEGFE70uALcte9hHHunxdrrv0KJ0D3fFMnUcA01LWd4LMco6+rOWLe+N/6ctXGlhKRAJo0kKKgD4oKBGxUQNqabEXFDvYaJvSdaicEkPLgEo5IjKRlIagkgTcmPyCUypU0qNSHpV0EzKXWhHgqFJahQCRfzPbaKPshOzDBwmpImrt0CiLOsWI1crBijGtg1chp3sIUcBLInSptYMKctJeiAJRidynlyS7SrFyRnkCOOnfTIihJI1BCJXBJANLtuBq5SGETOyayCQafD3nBq0SwCM31ARYZIBWYkISwiDaplg+l9xXAcAkoo1GOwyC4uwq5dELAqFKBvSCrMNVItkwarQy+0CWBZfkY3BLTIfIbJaq6LPSlCqBXRE5XYFPBC9BkMWbx2tT0m4mFAGIbEIDGYwpJhoREUqN8WNKBmidyIwksZVKiZyKRgiy5Po49iElgR8GkGxNZfBEjJU0t0IAIRAlvwMgpciWdqUiLJDj2WIEwAmsixg3Gk9MVnaJIGj+qE8gBpJBxAipQMBLjflfZQIRTOxB7QcoFREEWbZlsr7LRMrB9PxCAGQ2jQggUMdjVirCZ7lSPzFklzECpOl8srCzdZJlCV5CKBoDW9Bp7iS94yOtJbNiZjAh6IPtZRz/nSy6UqTYIxkRJIDUdoDMbYxkJEhpNiPAhCJlEZgci4LbJUtvzjcLmotsfWPFqbDysOVIqgjvU25ctg7FsS/BFqTAimBqk3PvspVZYvzLffPHW47XRCGy1ZKs24WcbIVOc5iBZmkxloUcqZ/IbmVJxpzXMeWbnCQzL2VCApn8m8lIBEZrNDCORWK0mgkCvFlRZdBXPGsRRf1kfuFrCUCCLJO8HrmtNAYnxrUqlPUo49imwOy/iIhiHCx/KU+gY6ycZusMg9N0r/HxDFyK36WCXt6fuZ/CSyIKQM5jrLiujEmZLdrLmuF4LOvIpZWt/A8MgpJ8vCaFzGX3EzkwbedBq81jS3H9RMmdtRvnc5L/TUKUc54BihinKV9Xgoh52+cU6BlYWJbjjIIuFuo+VFi+AnhMbp34CHmxsI6PBQezNZ2MKS60e6bNudxngdBC/6UFT5T15695Bo7z9s7KFGd/pwBrydNDoDhXth8LWXP796/leSj/8rmJrGL2+x1jwUK/LOc9cLvQppj9PVcmcv9J+WMvX2lX2G/+ymX8q//DX5ykGilZRkvWP4+CWTOxWTrexuM6qX7JNgpgkm5jzsA5Zxot81UuFWbMnLNfLrGpLjFdlvVzuo7ZdfP2zl0//X2erZPHO2esXGpzPk9zmca27zPHzlM4zOswuON/cynTKJTynWMLPSc3p/WYz9FDY+KylDbhnGzz9h8jX/nteJdeNNcF+fe5e3Mux5xhs/wLTMfI7uNzttBzczeRc9ZuWZb642vGPgiH8KaBDyL/Hscy/b0kw1KfDxVOWTIf7xJb6rln8KFnc7HP2e+l7/l8vebHlurl9s7MMW+WlK78j5Z5JsdSOTfnj1mX0u1+rDuee19ZyjYnbZT1J9rQ0vXL74H3LQ9OdUa38QE5UhWxPE+P7fShZ/RB5fJRz9E7q0w1zIyAi34fq9AutftFy7l5fwjcvQsg/TjyPEaOM8fZFbr8u3jdI8Dr4+WY9vEoi9cDQOi+Jez8XL6zr6U5epds73v+nBw/Rns/kX7PvaPf4376InXeG/z9OJAlAv/w178crrD/9J/6V3/i7f7d3/kbP/OxAV9xi6WLEp8Ml3hmDrBR4bP+AlIEPK8O2OkOv3f8ACtlcedqXFcnvOq3eNHsAQAft5e4NB1q5fCq3+Brq1u86re4MB1cEHjdb3E7rFArhz+9fY1Pux0uTZfZU430+KzdIUSBX9je4NYSU+jB1vhodYdP2gtIEfG8OeB2aCZ5JIegcFF12Okev3/7HEJEXFYdhqDQOooXba3BB9t9tpLeDsRkutZDdrscIjGUWq9wsgbf2L1FiBK3Q4Nd1eOP3jzBrulx1bT4ZL/D9bpFrdwk2frXtsQKWyuHt+0GV01LqUGS6+3bbkW54hKgu6w7nGyFwROJUKMdbtsGF02PzlE6CwbYK2NxGCo8XZ1wGOrM4rk2A950K1zWHV4et7hoOnRO47LucNOu8WxNie1fH9d4vj3i5rjGph5Qa4dPb3ewTuHFxSH3x+kjXJAYnMpAR4iIqxURRXBdANh3dSYf4jn5vU9eAFHgG8/foHMah67GqrJwnpKil5sGbw5rIgKpLUKkdASDU5ACOPUmu87ujw209ljVFi5IVNqh7Sus6gFtT4m7OQm4D8QKezzVmaAEICtnpT3ujg2MISKLza7Lrq7GEAuq9Son2r7ctuisJuKbNbXTW432ZgW5dthuO/SDzjntpAqJLAc5PlipkONzVRFn2/cUb+ydyrGwdUPEL9ETCUN3qNFsezhLFniV2rdWwQ0K9cqi7wxM5RC8RLMaUixyzCRCxO5JbXenCs16QHuooWuXrcBDr3Ni7qE3iIHceplMo7utIRuPOrkV20GTy+wgYbZDTp/AJBYxCujKZfdf21LicbOymeTF1OQK6wYitYCI8FZBGQ/7poEIAtXzE0KyBrtBoVrZHI8cgoQ/kMus3Nr8LmP3V46DFQLwewNz1cG+aaAuLKTyOd2Cf1uRu6eMlKuvU9kFV8gIfyQaSL21xGQqI1QVxnQMryrEpwOkCfCtHgkrIiCPCvF6IMtJr6A2Fv6ugtxYhJMmJtRkBRaS2E0RBETjyUXXBKCXEGtPLrTJZVek3IHxpCkGbuMp/lEA0Ml6yqQcTkBUAbFTEL1E3LrsXisORJsZ1x7ipBDrMMaRJvdlJtEwLzXcNlCdKCBPktwtn9jRpTbV1bcK7toR6+lRwj1x0Dca7tLnbXx1ULRjnlxf/Spd70GxWSZCniSiiAgVxWTJTsBvA0RizowmJhZQctHUB0mWRlA3vo6JVZXuDX2S4AxRzOYaFaCPSRFMrq36KOHWFGMXGmL+FJGuUScBexlQf05Mqr5GjvMTIbkoVuQKq04CoR7bl0NyX5SA3QaoVkCfBNyW3D6FJXdJ31DdqJKbqyO3Vn1KG68Nucryb3tJ7p8ipH57itFzGzoedWpjldZWgupHwDURZk/ussxYqzoB3SXdYAXoFhh2EdVewDXEwho0YPZA9+yLa6fmKMiVVlCfJTvpKDugOmQXYo6f1Ec6FgygemS3VVpPJHfk0ULEzKwAyQ6MinhQQH0L2A3SHAL6lNaxYDYVHtldVre0TqpHdk9VPdK9PL2O153Za4Om67OrbRjbVvSZga/pJtbHccyCWWErks+tx/li12nVk2U7mOR6LYp+0jxQ7HFE1AK+YhdsakM6Wo+gxCTOmq18QYvxfvd0XGRvApHblTamunEaFynSM1KswWR/R46yZMttMf/Z/ZdjPFlGURzjPRRuD8X5AkCza3NmgY20bsqyW/J4fZYxeQyULsRUoRxjnDLOpr6XWHbZ6prvgyWG1/nmnpwyv57bnyktsmOb4ximjd6//lz50lkvv8JGvXeVr7zF8t/9H38Ve99MLJWU0mOaV43PD0HnfIWU648An40UZ2gDX+tRSwcXFXqvxzx/KTegixKVdHBB3bOK2aAozyJEBiRlH2XOQi19zm9YWkqViOj9iPvnuRJLKxuXMi2ICxKNcnBREqhJsZllXkkAOeUEwOk0dGoj5LyEPK/0d2rNLS2dc4snWw05JrW0ILJ1sBxXCRDLVBBVys9X5h/kOnMLbVlKy2a5616SNHEOz1oRgG8dsYJyXtHSisl9cG7IJWtxmSuwkpSPtJS1lDnnvSsstSqNzybgHotx83neFOBr52vK/fE8xXQ/qURGxTG15b0yL/wdYxdpLmJ2bm5NnVvtz1l7y+N+Nj+8XgC5JLPcpXU0y5nmcJInkXM4prjl0trCcbZLx+eW8OyGHlh25PhhtoSWFh1m2R0SAVZJUsVj4mtJzvNWXOo/wiXG4FCsc0h98fXlfcSFY49dsdZljkijPJxXee14bvlam94LAsveA2WZW0fL68rnrnwWl6yofG6sj+z2nQm+0jEmNWNL3DkPAXYFD7yGyR2XmZhjsea0mSJzn7ypwqyc5fVcKM/j1Bpakp3xNWG21qVltGT55jZZsxJiHNPkUx3FxN2cNxQmVtbS0pfmkOPGl9hJyhyR5ZhY0+O5yvku0/3EY5jcF1kuUL5JjOMbCcIwIVWjdpDjq/nTNreWco7IOfkYxIyQTGAksSnmg0nGvmjJpGkY1zAPm9ehXI8oRgWY7x9mAy2tsPz7IQtRaZ2LYiTL4rGWZFhZ4JjJe0qXcJID2UX6XsxX2Rf3N0dUS8fLD8KS/EuxZfO+gClRzryd+Rzx9VEsApZ8C4Sla5ABXLx/uyyPYf6cLSGkeed4R9vzfh5T5vK8ozwg5cP33VJbj+nzoTl8qM+Fvh41b++UR+D3/6MvicXy2//KT7zdv/u7//nPfGzAV9xiGWPKC4kRXHiM4Io/dH0Yh6kTJdmQjvk4vmR95ETf5Kp6dHUCOD7Xz4AIEa03qBNAzW0hgbIEKl1QUMqhD0Syg5RAnkuIEn0clWUXx364cH/jNeN4ITABFqULcOd1PjckwAyMie1LGbT0GVRSnyqBQg0tfHYbDlHk7bDsipuALjJYLqyIcQRNc4BWgsVyTDEKBCDnp+ydHglZCuWZwSHNvYSPuOf+msFK8eIPIW0mFOvQOpPBG8ki7wEulnmIKitY3D4r4qFQbHuv8/iYlMljBHMyKXsluCvjccMMIAKADTIr7eU71ifAWAI2W8gfoshWZgaLc/fieTnnxlfO+3xuOBdmbiMBnrxOZ9rlZ4fkIhCXgXWS27vx/mT3VloHwGOq4APIDNBAAVRnrMdje1OgGROgnNQpzsV0fQmYT4OZfLiliHDFXGRlu6izqBPk55TOu+L+53nqZmMLaY64DK58lsXkfgWAjmNauU76HSNgk/V4nIvR9X3Ztbyce3FvHuf3Siw2n86dy8+7G4m4WF7v8UCZ9u9c2Rfg3FgzBDUBNd7pBMASaZWIOS8mj8VjCqLmsyFEhPXT94Z3U221XHMBZFdjMGCag5XZ+Pl3BrysqLmZQMXYeB2GGZEcyxwLsBGWlOdcOQIxMU1ngDgDsVyPSwlqSuDEfZRKZsRocVoqCTdBYCSiKt7vsWxjgYCE5wLyXAfvLrGQ+Z67tChiYBc27B6MaYgLx/g4pmPg32KQU1fWQERdsZh/PjbpMyy0V/YxWf/ynJhWzsBKLmLOyTAY2M3X+x1FzOUoGjwLNB5qdw5g31XnAfAm+GZ8sC8x/X1ufd9R3g1GHwFyH9vevbn+Mdp6ZBvvXe+x/X4ZS8TDMTpf8fKVBpYBAnvfoA86WykZELECxBbIMm6RrSrANK6SS+vNpD63N/8LEEBdihUswdJgKW8TJ36n/kaFtFTqQ7Y63rcClkpdKcOSJaEEKMvxXCpbg0IUQLLK+sK6wspIH6e3Cc9Bf+b2YQsvA7el2Muy2AL05vEVMj8Y5zNrs7zuXXFs83ODV/eUvvI74ItxlPXOWXL4mPdqcQxL83IufvNcDNz7lDImsWy3VITfNddjuQ8eYhSwUA9c85Bs0zV08zmY9XpWn4hT69FE4gfG9VCs3zzec6meK+ZUFpajc7Ft74qDfJe88+tLC9i5ci7Wjkifpu2FsDyHS+W+9XFsd7ENBlisfJfawT0Qk0CrHKkOH4yRxOweWdJ0WQGb9TmPrxSC2Ynvy5/X7J7CKc7IJ/K1S4plLOpkXXQGWnI3vE4zEFqu72KcaGlBK4BujPL+xDH4uDeMBAwTkBnlmyvP4r5CntPjpONLynpM/ZbXnlHqp2wwxfVlO4XL4RQofbF36KSPpd8FKcvZeEXufnFMZ86dA9zFPSCEyCBelHXL+53fY2eAXx7GHFgt/Rt8G41tPgTSxCJoHa8obsn71qkz4FLMz5/rX8x+F30uHV/sq5jC84D2/gy8C3Dfb6O8eEGuB+ud+b49SvZHHsMjxvFTwEzvBSJ/fjHbV6J8pYFlFwx+7/ACg9fZ5bMs/IFn11Euc/AEzKyARWHFnwFoeZaBxruISfjvQy5lcyBVyseAoCwMDs6Rk8yVm8eQV7yLTGTezeL379HA5P51949NFaUHFad3yLBEQlP2O1fM30V4MbqEjXXeBT4e0y5VEve/srN+z8m01H92pcPy93UO1sTCvydrXX4ll/79wJgfWrd3lgcUhLOXFHNS/n50fwDuKdzvoVTkcmY9z3ZdKGOlW12ev9JKNJfhMX0sKdvzS84puEtNv9dHf3bPnAOWS/09Rp6Jsnim/fdcj3v9xUcO+dxDVhQGIJO8kgvn71+HyRpNYFY6PlG8y6l4l8I6Pz/TjDnn30NWozz1j3nRLPX9jmd9sd+ltmf3QJb9J1UeWp/HyDc/d+78Uh/F/6dTNgPb+cWBcdNgqY+FF/6DwOihd+IjFAZRrlF5YL7JNn9PPRIELpbF+/t+tS9kDX1H+cIWtsfcCwuv0y/c7hcFlO9q94u09Z7tfjVK/IJK0FejfKWBpfUKnx4v6N8pZopBmyisg+TuF3PuxXncmo/kvsmxaxEgko1Un/IsktuUSqkShIhEfiLKdAsxWyzYTWkex8MsixzH42auVvQX9MFOqSUA3IsR4noxijHeo4gvEXJ04xIpRkuKCI614TcQp3zg41yHX+zk4joqYJyOYeL6U7BGlqApp0dgYFMcz8pyIoAp421y3JSTlNePrSkCiJ7Gq6qQY3H4fIxiqkwAEDqkZ7ioU8SicG7AMCRSjirV5xieONbN8z6Prynma6LwcwJ4bkfEMb6l1ALKL7dLY9BFhzJSWyKOCccjxnFwe9wGE5lwbkJWJDinYJlUXhR9xOIYy8qF/5n9QQtFgN3OYuqvzEXIdUoZ50p9mbqijInia3mcPHdL269xNh5gTNguSGlHFODk87GcX7BsmFg4ODE7p7eYJxPntllJE5aeCyKKSTKUSmxMICFZZKK+z8LI4EIAY7J6K2nN07tJRAHB7pUCowylsZjdRTkvXLmGEZCDQKjS2jHxTDovHegc0nqolJA+JVfnOSjnLicXT7neRMQkoXgs1o2JbyZJ1Od4L9C8CzfmOeR6MrmzlgQW4/2FcU4iMgkMy4SQcvaZeI80QziRjwtP/5ZDuldYPlco6KD7ZX4blgng85hKC5Io6smiPsY2cj5BYOI+GIt3RyYKSW0Kh5zrMKZHpaybcyemuZtYbsI4j5kUhPMZlo+YFjl3I+XCpBN5vdJ9FGXM487PUTrGvynv5ThJ+V5I9w3/DSoW98VYR3K6FjGOoSRHySQnDok4ZLx/QjV97t6nMAkMDWIKenjdoqR+87qlPW8mpuF1j3yvirH+BGMV77TyVcFzK93YNj83VAGT9UUxR/NchxMsV1z3/7H3Zr2yJVma0Gfj3tvdz3jPjYgbQ2ZUZWah7ipRNDSCTiFe+AO8ISHBPwDxG3jop24JiRZPPIPU/QfqhSckECAVArqoLKoyKzMjMjLujRvnnsHd92ATD8uWbdt+/NwhKksZkUqTjo77HmxYZnu7fWv41iFQOVZ/meOalAYHazq/V8tzLef65/ckt1/9hlfvq4WCIrfDx44qL+q2xXIcOPxfP1Px4Fo8PHdY3gi8+XelbvOxcmwcB/fUOS0f+/9o9fX51/TlKEjltXi4ro7U81ag8cg8v3P5Jvd8W8rvgeW3s3zcvsK/+Df+R/xkeh8xSXxoXmGIBl/4Czx3Z/iT7jN85U9xqbb4zD3BD+wL/GR8BoWEHzVf4oU/wT42uNRb/GX/Ef6geYFfuwus5IT3zQ0+0Le4Dhv8r9sf4o/aL3EfW+xDgxM1wCWF79uXcEnhL4cPcaW3CBB4qu/xF/uP8MerXwEA/nZ8igu9g0IsJEKtcHjpT/DSbfDvrH8OAPjSn2ElJ6zliFY4nMgefzV+iPvQAgAu9Rb72ODar2FEKOQyV+YeChEfmlf41/0ncEnhfXOLl/4E/2T91/j59BSfT5f409Uv8VfDM9yHFt9rvsap7LGLDf7f/Ye4MHsM0eCZvcEvxyeISWClpnKsFQ5DoticF9MpNmrEmd5jjAb7aPGxvcbn0yUu9A7b0EKKCIWEl26DK7PF3/Tv4dLsyj2v3Arv2Xv8ajzHp+3X+PV0ho0a8Xw6xfv2Dl+MZzjVA37UPce/3n2EP+y+wovpFNvQ4A+7r3Cptvjz3ae40HtsQ4NGUgzrSk1YyQlGBBgRECDws/4pfFI413uMUSNA4qPmFV66ExgR8OvhDH0w+I+f/jkA4H+6+YfYqBEfNTd46TZopMcX41khWDIi4vvdSxgR8NlwCS0jblyHUz1gjBof2Dt87dZopMen7UuM0eCX4yVWasKt7/BR8wq/6K/wQXOLL8ZzWOlzrC6RB/2oe44T2eMn/YcAKD74xq3wg9VXuA0dvtd8jb/af4BGelzoPV75FXahQScnNNKjkR4/3V/hVI/4pL3Gz/orxCTx1N7j31r/Ap9PT/CXu2e4tDusKIM6xqhx5ztIEbFRI+58hzFqdGpCpxz6YDBGDSMiPmhu8cqtsNEjVpLWyBfjGT5o7tAIj8/HC/zR6kv85e4ZnpgdAGAfLXae5umD5hafDRf4oLnD19MGjXR4Pp6iUw5jVLi0eyhEdDn/66/HM3yvu8bf7p/gj9Yv8OVI89UHg0u7xy5YhCTw1G7RKYcbt8IuUyD+w80X+HI8w/ORlE+nZsCJHnBl7vGT7TMA5PZ+YgZs1AgpEq6nNXSOg75qtgCAXw+nUCLh0uzwfDyFkQEnesC9b+HyddfTCv/u6S8gRcT//OpHheDpqtni8/05tIzQIqKRHn+wovXzk90HJU55rSa4JHHvWho7BH6w+gp/fvMJ/vH5L/HX+/dw75qSq/ZPTz/HtV/DR4ltaPDUbvE3u6ew0mOKGh+1N5Ai4afbK3zQ3SMkgVvXwWZU9uOzn+J/uf0Btq7BB919GY+VHj/qnuN/u/0DTFHjyu7w+f4cn26+xhf9GT7sbvFiOCnx0j5KPGn2MDLg5bjGJ90rXLs1Ls0Ovx7OcNVs0QcLnyR2nublg/YOp3rAz/dPcGoGWl/Z6+REj+iDQaccXk0dPuzucKp7/LK/hBQRe2/x/dU1AiQ+35/j49UNvhxOMQUiCDvRI1yS2LoGVgX8++c/w8+HK7wYTqBlwLP2jmR//z7ODFGI3udY+n+w+RL/580neNpu8XH7Cv/X7Uf407Nf4S+3HxDjdNT40eYFXFJlvTwfT3HvG3TKQYuIW9fik+4VXFK4cR2MiLhqtvjF/hJnpscUNcagsfeWGMNdix+uv1qkw3o5bvBq6tAqIoD7oL1DH0yO+7doFL0zPuleASCegXvX4nvdNX7ZX6JRHi+GTRnfLlh80N7hb+6f4seXP8M2NPhqOkGnJvTBopEenZpw41al/8/HU3zU3UAh4sLs8Mqt4ZIq6+mH3Qv8xfZDnJt9eRf8ajwv78Hn4wkaGfD1uMLHqxv67RhP0CmHZ80tAOD/276HC9tDioidb3Bpd2ikx2f9BS7tHltvsdETvuxPAFDYyXm+/uWwwaeba/TB4N41GILBue3xUXdD743hDO83d/jp9ik+3XxdGN+33uKP1i/wv7/69JvvO1Y3+Pn2EjEJXDZ7NDmP9VpN+Nn2CS6bPbauwfvdHe5dC58k9nntf7qmPr+aOjxp9hijwhAM9t7ie+tX5TllToUTM2CKGiER4zyAEt6y9xb/4PRLfNZfQCJh6xs8627xalphCho65+TWIuLO0T7i/fYeX40bnNs99t4WuUkRcT2u0XsDk+/bmBFb12BjRvgosXMNnrQ7TLnPVnpYFXA3tTi3PSIEXg0rRAg8W91i7y1iEtiYEXdTh9upxfurezzf03xqGWEyR8WpHRCTwP3UotEePsrCkA+A8neLCC0jem9wPzZotV+cAygMZwqUz5wNDCmJwkg/ZLZ7kdtn7gElI/qJGN1HR3mza8+gECVWxmHw+oFnGZ0nxTWRqkmEKEpObKsCemfAabGIkyGHN0Viha+9ztgIwu1qFQpxHJ8PkfJrs7EkBAWrAzwTjsVlH5kAbvKKcl8fWddFt5vfQ/w9cM7pqr7iTXeExIqNA4dEbIdeVI8RrzFZ22H4D4/hdaEpb+Ut99+//vTvy9+9fKdZYb/3J6fpv/pX/x7GaBCSLLGA/FJ2SZXYSyPCA+bYRvpyjI9zYbIarqvO41g/li4T4TCj7GO5IJkUqCbu4bpCEgW4ADMjrJZhwbB5yMjKbTK5zyFz7ZTzbx50QHwxAAAgAElEQVQys9bssNTHtBj/IdvqMUZPHkcdC3qscMxrLRfOIXrIElv3vZZz3T67O+sjhDPH3JkfY4qtX2IAkf+QTGemxcPcnaUOLPMYHvvMbKdMlvMYI+yhSzT/kDITKkAvS2ZOdVHCPEK2U495QQKU//M6UmJmc62/AwsF+VFlYF3nsZc795X7X4/x2A9IXQ5/EOoYYFnV+5hLOeexPHQhfxjb+tC1/NjxmvUVoE1A/QN7WC8z1ho1q+uPxcfW88AyBR6u1dqTgr0ruPjKnZ/rrTcDsZL1YeENgapZRavx8Ln6OHtZ8P/D+ubPj4+b70vpuCzrungjUfeHmWG5jkP36rr9IkOvIFWs5pDMF1ItN0oAEIPMTKtAivQ5Bko9QxeLRf8giIjnQd/ztSLPx+JzLpWRZJEndOEFUdqs7qvHGivZCWTvjzQ3ADxkKS0WwqoDhx3LHgKJvSQq00Uh7Ali6eXAVhmuiz0M+H99nr0SqrQQC4+G2kOEv5dJyvcKkEcG91nkdg7ZT4unQzWObIX/xiUe6SNYBmn5/eA9I0J2dZbVZzGfY++KhSdDsbotjydBFv1iDROYXVz5PlTngDnuNFRyrGVWl8Px4RGLXJk7HGWYLRY9tvpV1rsHVsRHLGDF5brIorJUvs7ydlB9feyBK+zBM3F0rMfKwT2PWi8fK28CQnjL+o49zw8qwcNxv01bb9P265p9l8ft7xmS/OSffgtYYdsP0o8//s9+4/X+2U//2W99bMB33GL59bTGv/rs38Z+tCVPYIwSLlD+PWN8oYz3XsHogHHSkDJB6wDvVdm0uElD6YgQ6Eeec/mlJND3FsYEhEBvNI530po2LNNoyuZDqQTnKBcggJyfb7kpFII2MTEK2Ia0biH/iPPmi/L0mfl3Wkaigz94CUlF/jRK0xj42hglmnaCcwrBKxjr4T0l1ZJy9t/xk4ZUOX6romHnzYjUsTCUQtB5qVLZKKUgstzkvIHJXYxeQOqEMElIHSEkKI9eFOTi6iWkiYjZvSwFCWkColPkBmwDwqhK/r0UyAVWygg/asqlF5abn8ONVcx5EWvqd2lCuS85CUQBc0LWO7c3gEyQeq47MbNkfnFLS6kcFsfzBoZddwFAGnKJ5PGkKKjPXkKohJRdueofVWkDhATCqOa6c/4+7nscFd1Xb9oE/6Xi8ip0Hh8AqATVekQvkUY1u+gCDzdkvPkrrp5VGzrmTQlorEGS6yvnIZwkRBOoDVXXD9p06DjnTMz1lA1i7icSlu68Oo/JxtlVmDec3O+cuxL5GQUANIFciCt3O6j8V8tX5k2uSLPLcRJUZxLz/dxnros31dyPNhBA2Kt598KuwbwmBWguBYCpignn+mrCERMp12IbgElmN8Ms0zbOfeH1MVb1ZVdf4eRD990EyjG5V7N7L7sui0Ry7hXF+6kE4QSSjRBeFnfYhVtadrcWXiCZWNYOX8+bv+LGmPNvgjfFEbRhTpjdbdmtlnN1+sx+GShXJI2NPgs/9yepvPnObqJpnWXHbpOaZC8GWdyhi6tmEyEGCSggGfqc2ggxyrLmUrPcMQsv5828ALkv2yzjPL6kEuWwZJ1iyOs3579M7HacnzHhxbzxB2iMLLuSWmJuJ2XZJp1lKjC7o+d7kkk0nhX9Xgg/yxgS8+dAcyi8ALhfMgGRZCBGSfLXEWJSNL+SrhGjnNdXHjutnWq+ZMrzJyDHeW0y2EsyQU6UaxMZvAonyiPCBO/SAbGla9gFNinMa6OXSDZB7PN/fg6SRGoi1P03IxkDkF2kc/obhRnESzrO7rC1qy+7hkabIKOcXZOjKC7HnOtQVoC5dvmU/L4sHZGI7ZwfVQRyb5ce87MMLN2RTYIYRXFbRppd2+W4XMu1S3Jxd845XwuA4ueUwwVyX6Kt3Jyz27KIQNQJaqjGl3//kso4N87HCnAEwHleAbpGehT32tr1HbmO2n2Y6z3mfoyqzdr9GiIddT1m1+/F8VnU5f4ZTFM9PKcFdB4yFh+WVP0/dKOt7zu8rv79OHIPu88fju1B2+Lg+2Gp5L34/kipZfS6Y4u6DvvwhjZ+X7495TsNLAHSzLNVpDb7cyFNcJUaosqLRub4fE/WYM9adUo2P2vOucJU6iVN86FmHou+iEU/QC9ypBz7OMdO0s0CkZXjac4jl/K5uv36HurrXA+37ZzKtPkEZA/jJgWQrRRzXfVYRH47RtaaF2099WzOu0Zv/ZQBCPWXOp/iPAfF30Ok8oJIi7xSqVgCAJAGPtfB18cgCo17oZvP/RMQy/pKW0utf4rz9QzQQp3XLAmkIBcU/AuR1+kEKtmVPuX7EqdKqGUTKlnx/eUtT8C9sOfVU53vj57nMS0BYcpjrX5RU6jqCILAKoNE/s99T8t2ZstC1ZcEAl4c25rkfG8QZZ6Sk+XYYowx5fvr4wfCrUEm38cpIvzBeOtnpwaEXLLSYG5D0KZ+MfbcRsjnU/Wf1wRvAsIjMuPPk8w/lvSM83iKlSGBgKGr6i2FN88CKWYSF5/l6GTeBGYAJoA0yeUY+BreqHF8F8d/ZtBaiGIGNdfJz0PKcZZjdS7LggGUgJj7zY8zP0cxX8dzE2bAuLg+AxhElHjTci5V44wUz0d+YNyXfCy3IdxyHYg8n2UpDJLaY5etnHpBRAFUgAWg+RMhyyLm+xiUchlrFltR5LQwjYzcFq+bDLQquYnI70SUNVs2qeV9x/UckWH9DkGWYxBl4ygiinVSpPwuDYAYVI41Bf32cX1CzP2KcnH/It4yz5uYVN6Qi7JJp027KLGsHDfJ72Sy0AnA5c22z9/5OglACLrOs1zEHNeXgOIMEAHZz+dEXvOJ5zQAcBkMseylKO8nWc/pOxbh53aTr09we1kkLsdKp+W93F/hMJ9Pc+zwYrNelyPxjSLIss5EzACm6I3nuOkCfHLfyzwl6vdi3fE7X2a51cQ/RbHH/ROljgIGQXM7j0MUQCbcQ5BVOlg/t8Bi3g9BRg3oHoCkw+trmda/N5Ucy/hFNZ7D+8XB/UfLYcPH19nbWzOrKup18Zb9qRwfHpdN3da7HD9W/+vK2475kXbeqo3vSkmY98O/g+U7DSxPzYD/4P2fYaUm7IPFnac4gk45NNLj5bgpOSg75XDnWqz1iJgkbl0LKwOaHON2YoYS3+RyTMTeW2gZS9wCxyuwu+adayGRcNHsS/zQEAxOzYCbqaM4BT1iigpT1At3z1Y5WBnw9bAm33wV4KMkdzoIuKCwMlNxX+29gZIRrXLL1CjeLK6XgtxojQx4NXRotUejPO6nBivjYGTA3tmSR/G87UueyDFoNMoXt0MhEgZvFm6VnXaYoipER0pGDF6j1b7ENgAoLoEuKGzsiDHoco9RAaPX6IxD70y5rjMOe2ewMRPGoHE/Wpy2I3aTRas9lIzYjg18kDhpxxIbEZMoLsOc+D5EImyy2meXYRqvANA7DaPovpVxUDLi1zenECLhyXt7TEFhdBpGk2XSar9wu9yPFglAa2gnoGQsuRadV9A5xmF0ZB232sMHVRLPNzpg9PSf4y44Z+IwGXgvsVmNxf1VK4r/kDJimjS6xtFYszWex84kUY1x8EHBeYXWukJW1e8baOPRNa5Y9QFSxLB1nl0diztttkCXdCLZGk/kVnStMQHOkfXf2oBp1LCNLzkEZSahSomUHXy91tSmMb64OXKfWFHC9TWtwzgYqOwlIGWC97K4bDqXrfEqFuXR0FsoHWAtJYWPUSBkLwX2FJCSrO2cu1DlOWdiLQAwJiAlcqvU1Wd2sQxeQemAcWeBJGDP90W5Q54TYUG05Sfqq25dyZdY3GGrtBp+1LCrCePOQrd+YXV3g4aQKf8B0Unoxs99d0RGpW3I3hDIc5D7tdWQF47G7yWBWCZJGRTkBbG8RC+gTITPngNhIi+D2mUzZmWB0BFxUsUqL5uA6LKiRCQiykL2IgiCLNtZeQRW+GVrfqlvzNduQrb4J7KGA8UyLkwsO48CtlQm/dppJBsBkwoARwLS2hfrdrG2DArx3BFIdxLxzEP0CvEk7/gFMrCcwV60cU5SnwTJMOcWLCQlTiCuw6yoEYmsCtm6KCZR9udIQGxSttiLcn9tRaJNP8iSmucAmuphK2bUqVhpY7akhU2E2pHyI+nlzpwJaVJDFq1oE2QGyHIU2TKXEDuykKlRIDSJwGq2GkaTCmBjEqTQJKhxeUw6GmxoMzlTJCIg6QhYBEuf2XodbTXmDBxTS5avlK2tUVK9asoyNICcgNAAagJiJvGBBPRWwq2/+aZOTTQuAIVcikF5tCjkR3JCsWgy+FETfU4mf5b0B0lyBq8bBvy15bEiDALoPrPN85DovOpJblyHTCjyBeh8NAnKidlSzR5Der5PpLyespWPrac5LH+hbGCCJIh5vKrHvP7ZmqwANQhE2ipBBsyKr0qJUbjO8ppn2bKck6S6GIhXDlgk24pMqDiW5GP8/wHIyvXyc3CMuEqGeUyLUoE8BvkFqPL52urI34+Bw0Olgpjnm8uCgKl22KqtoHUdVR/reTtWjtbF/x8DufX3Us9bPF9v+Qi+E6h8awXAb7skPLSC/O6U7zSwBIAz3eOL8RwuKpybPVxSuJ5W2LoGH3Z32AWLtZpwPa1wafd4MW4AAFfNDltvC1HEr/bnuGx2+HpcwyqPMzPg49UN+mDwi+0lLps9hqCxTxZWegzR4GmzRYTAr/enWGl6457bPZ7vT/H+6g4+Kty5Fq1yaFXeqCWBlZyw9Q1upw4fr28QksDd1OUg+gQjAzrl8GV/QnkyIXDW9JiCxr1roUQsG/8TO0DLiLWa8Ov+FD5KXDR73E0t/tHVr/ByXONuavFH51/h63GN0WtcdVu0ysMnief7E3TawUWFi2aP26kDAGgRSaZND6s8pkBL5d416LTDRbNHTBJD0Hh/dY+bscOZHTAETWBYJOycxXnb42bo0BmHtZngo8TeWZw2A7ZTg8tuj/upwdpM2DqLjZlwNzVoVMAfnF/jxf4EH2zusXMWo9d4dnKHVjl8uTvFWTNgDBpKRoQo0alQiAe0iIgQuBk6hCTQag9mDX52QvVpGXE7tPBB4t989gW0iPibmytYFQhw53ycu8mWNpSM+OiMyCe2U5OBPIHimAQu2h69pzyol+fXGILBdmpg1IDRazxd73A7tjjvemwnCyUoblRpAogfn92iVQ4v9icEVIOicZ/dwQWF84seX/VrKJHQaI/Ra7gooUSClhFKRtz0LTbtiI2Z8GogBceJCvjj977Ezdjhul/hpB3RZMAcE5EbAICRES4rOJSMaFTAGFQBuevTqSgSWMlyPza42uyKPL938Qov92usNiSTkAQmT/O0PptwPzalHikS9qNFYx1CFGg6UmxYRSQIu8ng6ckWt32Lq6sdekeEDj5InHQDJk85Qs/XPZRImILClNmQPz6/xdZZ7EcLKMDogFZ7dNrhul+RATUorNsJSpL8e6cL0F9b6v9utBAiobMO+/zZ6lDabk/2GCaDT6+uoWXEL2/Oi4KlMb7cLwBoFXDajtAy4nrflfeYUbS+Rj/HU55dDfjqfoNPvvcCL7frogwCgE+evsLoNRlKvcbaTrjed0XJcHIxAgBu9x1WpxMpLZwudX/00S0+uzmn5+bElbWkZMTVaodf3Z7Rc2M8tn2DJ+db7EaL9cWE/WTmOOQo0Gw8lEwYnMbmyYhhMmitw260aI1HiKKEKADA6mJCqz1u+zaTXZDiI2VlFDN2j06jPXPojMfd0JCCKChsrmhs26HB5mrEbrRFAWIUkYH4ICFlwkffv8Xt2NIcANi0I4yM+Op+XQg6prz2rzY7vLjboDEeZ92AF3cbXH20w/VuVeJCz9Y9YhKw2XS2Gy1cVnApGTF5jVVD8p48KdJWzYTbfQeb30E+Sniv0DUTXFA464aiGAKAfjJwXhXvmVUzFcWTywqsGCU27UhKhKCKsu1+aCBlxDAZ2ByO4YKiPmw7PPvDO7gosR9tIf+QMsKoiMmTQm3dTugng3VDz0WrPQZPisFt36BrJpy2I272HYwOsCpAy4j70cJmxcx+MlAiYZgMVi3N1zAZKBWxbui38nZH97PcrPbQKmI32MLErvNYeK0ZQ3M2Dgab9QAfJZzTCIGUWqtc926wtAb7BierEfuR5OGjxFk34KvbzbtuNUpZtRN2eyLSMdaXedIyYrdvSthJ2zj47PXESraum+h9OGk01hPRSyAlXdM6Mgoym7tA+d0BSHnGxwFSvq03A7Y9zblzCrZ1mJxGDOydlZVnkwZEgm48xlFDmkAKtSBhrIcQwDRqhPzcAIA2AcGRMi1GUmRaS+FFHLMrJSmxjKW1zeE4TTsh5rFrHeBzSI5tPKYhbztZ2QVS3rHir8RWh1lRJDMrtgB5LUVHCiwApFzLihgOtSkohI2tXhaFF7JcFsookchDhcMdOAyES0IJ31mCrXwvK43q8JL8uYS98LUCcxjIIVjjfrGCrI4HZrBUM6TzsSjmMJU6dpkLX8vnX1fqPh1+rwFlVlrUccIP2qyO1R41BL6zfBam64M2j/3/ffnWl+80sFzLEf949bf4qX4PMUmcqR0iJK6bDW5Dh2fmBvexJSDXtjhTezy3ZzAi4FJvcRtWcElhJSdc2jNcmW1hXT3Te5ypHmMk4HmhCbQSO2ZAADGyxiRxZnps1IiYBM50j3PT4wN7B5cU7nyLRnoYORN6SCTc+RZ9tPh++zVcUti3lkBlZny90Duc6CeFpIMsqQo730BnJjUpErGgykCMspp+wE/1gDvf4tP2a1yaNe58iydmV1gJzzWxOIbMRrnWI3xSONEDrqc1AMDIUMB6TZBy41boFDGQAkAfLE51jzvboVMTxjgvqTFqNNLj2q7RKYe1HjFFXdj+7lyLq2ZbLMXX0wrntsf1tEKnHK4sAWC+Zowal3aHjRrRKlcYSxsZCoFRozx0DmiJSWKjR0SIwrrpo8K57dEHAy0iXtkOPir8cP0VgJk86NLucOM6NDLg1rULsphLu4dEwo0jJtUhGLTKYYq6sFpKEfHUbjFGg+tphUZ5jEFjrSec2hYnesR90xRCIgbD57bHWo3QIhZr+xQULmyPMWqcGwL6WhBz6i7YshHXglj21maNlZ5obg2xV7bK41l7ixM9YqUnrLQrazIkUVgLbZblFDSs8rAykMU9aGKNNTSHVvpCOLW1xBwIAGe2x5NmRxZ5xfMgMAQDLQIxBJoOKz1hygRQ+4ZAvo9kyZciwcqACGJCPDUDOu1w1W5xN3UlZy0rR5h5UGYLOx97v7vHvWuwtbQJtCpgpYnplgFIiBKN9rCZlXfvbZnrjaFn+s62hSXx1nRQMha5hCjRaYeh0Xi2uoNEwniiCwlUpx3ubVNItJSMuGz2kHmuuLDc2duAGScB4P3uHiorehhYvt/dYwi6yPbUDjAqFHKuEzNCigirAk4sjaP3phBjvd/ew50oTFGh066sXykSnjZbjLnutZlwoz3O2x73psGJHbGzdkFC1Wqas94bnNkBvTXotMOtbtFpV4ibxqyc2tiR1o8KaBRtrn0iZUarHcagYWVA702+NsBk8DJ6jfN2BnenzYDOuAV7pM8gVmdZ66wgUTJibSZoQQCjM6TsG7OC47LdwUWJVnucmBFuI3HVbcsaTgDOGmJaZU+SzrgC9rh/K5OBZV7fnXYQuW+85lxmmXRB4bQZFmRcvTGYgir7so0d4aIq9/I74zT3JUSJKSqc2BEqK/UG48vzx14jAHDe9JiigpERTX5+pEhQImIMGoPROLUjrArYWH53OBhpC0HU2k44MWNRBBgZYGXIiiiaT6tIwWd0wElDbe81kdGtzVTWDl/noisAVYqEVvtCVKazZ4APEp2lOetVwKYhkOaMhw8SjfFY5TmVIpX5XVsCyFYFuCixsSP2ncE3LSs7K4k760r4CCuTTFaWtCVNmSgsn+vcZy0jOps9R6KADwqblp7TSalCOmaygi0mAa9nRMAKhUZ7xJa8VidFdWoV4LOyAyCPmElFpNx38oLJXidpJhtj7w32UNEywutQiPJC5rHg/jA7rdf0fNaEW43xCIp8tLQkL5KgY/YCQgHHfL3OColQEW2xJwlA3BX18aAyWWEG2oWcS4niBUKC4uP0jCY9myE5ZKkoyVSCVAmxJsHikjJArI4v+C4y8FmkPcvfhQQiD5hvUbO3wFF/0vJ+nWVVvtdkXJHDhfJ3tcytXbwe8rUl/dghlqstkof/4+N9LOSfx0DfwbHHiELTMbR4DGyyvH5XyneYOPVN5TvPCvtf/st/ggCBMUe+hyQz0yq9hGMSCJg/AyhMsIdMsfRfPmCAZfbSmiHUJVl+SJgNFkBhSyXwt2y3Zm09rJMZUGvGyHDwEMUDP4xDplVmny3XH/S5ZmCtGVcPWWDrvtaMp7XF5Bjz6iE7at3vmjV2Zi2VR2XMm53DfnJha+Rc/8MxHGMNrftX3wvgjQymdT8fsIxW/S9jzN+PMeoe62ddjrHH1p/fRK+96PORearn8Zi86vuO9e8xLxhes4+x5B5bG28qxxhDD8d3OJbH+v26Ng7rPfb9GGvsu9T/WL2PFbZMPXYuVbJmNl4uzMr7JnkcU04/NoZDlte6L4fHDvt57J7HZFD2KmL+Xn+u63pbWdbXH/blMP8ut33IcPu6dg5Zat/4s8p1HWwk55jwx+X9aJX4Bkr9Y2viETk8aKzq21H5vOvW4sjG+nAzeVS+Rzb4Rz9z+U1YPt72/neVybvUW8vmbTbdjz3sh8cf6QvFYD/MvfvY9W88d3jsyFgWP9cPrsfDa96m7cP38SP1vqmet/4Ve5uxv6EcYtB3bu8drjvaxmvW1m+0T+947Vu1fVD+6r/+FrDCNu+nHz/7T3/j9f7ZL/6b3/rYgO+4xfL5/gT/3f/9H5JSJYoFlbqQqcT3FC1PZqMUAjMjpwDF8JhYzpd4Ga7Pxpm4o9bk8HkTZ4bDIMglgV0mCiviwYMpsuaISQSY0QsoRBslibvI547tMWN1PbMzZveEwkrIcRSK3A8KmYVMiyTlnJC9xBDl+hfj5sTaFZFAYYVjzRa/Aet7mAkRmF05akY51qYVGntklsmcnD2zdnJMCzPuFda8Sh4sXgCFRbBmiqsp2Vlmak+xU9HS/C9o0WuWNWCOSeGYNI53yLKo+8+skIfjLQydwDImxBPhA/PisLxEjokRkeKFAMyxL4Xhj/rJ8VN1Im9m0ksqxxuFeWmleow81+LgO489VYc4nqRiLuSk3VE93CNyP8p4ZXU+V7qIS8lykIHqk9V4Es+xmD+XOePXgMvX8zxybE0EEr/5RHU8t1f6kmXEyeQ5Vgh1W9WY6hixMt56DrKMRMhrWM/XPZA7iAAj2AQ15RipSo7SicI6yEIsDJtgYhReGwdjA6hOkwpj5OKxDfR8JX6XZFbHsm6rdpKoyDhk9UzyPIeDec7yIoZILGKS+BzLgO8vaz6hMDuWeclruo5xSpUcZbUGeGwAkMz8/CCPW3qKzRORZVDdz/2ryVqAsv6RqldfTbKCaiyPjLX0o5Jp/V4rYzps+qCdeq0dYjReu9LlY1V4Zjkv5/6kSs41ZkmaZCBDNX889mp9pXq+/PIYG+rL/VUfRcrvjjqOjOsSc3v8XihzzfJgmeT+Rz2/9/h9JT2QU91+o1LkzHNXAZyo5zUvfPUbUs0Ry/+QvVRW5+o6C4Y8+B2CAKRLSEosZMhrrPwupuoYP4/1fiICIlE9i/uOxOyJrE0j8p+U50LMx7PmTYa0WOcUSyroeB1viPmaIqdyYzWONP8/fObK71Ql51L4eC27A9k+kHPVZl0eu77I/rAfR9p5Uzn6m3nQj0cti/X/RxtYnhcpzXP52jaW5w/H9M4A712vxyNz9F0tCb8n7/m2lsZ4fPL0FZ52W2xdU+LJNmbCWdPjl3cXsCpgCgprO+Hldo2LVQ8hEr7erdBmd6H7ocGz0zs8vz/BSTti8Br70WIYDLQO+OTJDV7cb2B1gFYUW6Vkwt2uRYwCH17e4bZvIWXEdt/i/fN7vLjdQAjgbN0vYoCYFKRrJqysw5fXpxAAbOMQAsXepCgQR4XVeV+S2Q6jgTFhkSMPAHa9RYoSYVJYnw6FtMQ2DvvrFWTn0bYO/V0Lu56gdcDQ20KycXrWY3Ka4nJ6i9Vqoti+TK4y9BbRixzDALSrCW7S8BOl0FA6wvUGuvUIjtOHUEoSpQP8pNCtJ4yDyS4mFJcyDRpN5zBsG5jOwTuFtpsw7C1WmxHjYODuLZrzAeN9A9UGKB0w3TWAFzAXA/xgIG2g9nREyrEXxDZJbz1OI+KdglT0SxR2BrL1SEGiyTLZ/+IUSID6YA8/asRBQbSUlsR05M5H71eBcE/WcdFRXIm0gQhJZELca1JEREDeaySbkLpADJScxmEVkHoF0WVCkpyiJHkBudUQTiCcZmpBmSDbgHhrkXSE3CnEMyIeEU5S+gSTZsbUCMgThzgqiF5RugWRgFHBXCuEJtH9Y8V2KUFpIoAZ2OcUHwVIMFgeVE49ISAngaiB1AWIzL4Zuwh9p+BPA8SUiTmaWJQGai8R1hGyl4hNJODShhLfwoQkTFgSVwFyqxA3AepOIawiyUXn9Ak2x9lk9s5kUlHImBuF0CbELszXOAE1CfiTSMBeUioCZu2MbSxKGLmn3X1YUz/FKOh8os/J5vQII43Ffk2su+4ylBx1ak+kKQR6BOAF9I42eO40jz9vdpMEka94+tU0dxLuIkC8VHAnidJA5HVtryWiQSFiUSPVxwoMvSPFlj+NZRx1CgD9QsBdJCRLfZTTDKrMPZ2DBNQA+E2EvpMIqwS1J9KVGiyqge4LbYLeEYGJHEHX97RGIGcgrXsB6QT8KhFwkwz+Eq2pTAITbKJrR7qWwbjZEgGNX0eYrYTvUlGucH28wW9fCnhDfREBUHsimvErklkSBDIRAf0LqbcAACAASURBVHsn4Nd0XO+ozeaVgN+kouixd6JsaEUEfJtJazIrcrQJekcyjgZAAvQAuI7GBkF9S4pkkwygtyibZZFIftHOCj+9nwFjzIA4aTrO4C0qIoPxq7mOQrSSAaVfJ9g7mo/QZuCTQZx0RHQTNWBGGpcaqC41JkRDCoOpofN6n+A2JG/hAekTfCegJgIooaF+hhZQjurxHbWleloH8ZQYfhmQqZGAS1pRvdHMYwIymMzAOFpA7kkOJE86p3PdvhPQfcJ0Qv99KwoIs/cJ/dVb7vSPFD3Q2JMAlEcBZtID04koCgo9JASTQZ+m9aP3iZR7VkAN+bOi943eExiLZgYpsgKnwc5gjuoUsHcJbj3LUPd0P4NzYAbSQtD50GRiIU11qzHX3yRELSACPWvSZWDOzyjXr5DTahAYDk2CGmgNM6kRjTODTJ8QLD3Xuqd1wkogVprokcc+MxtTOyh9ETFlhZRAsDQuqiPN788MYFlODD6CIVAbFf3n9vn5IOWVgPQkAxlmhRwrMpSjc3zsgeIoASIkzCy7qcg/5nWwAOApLUA5l1ppyWC8bu+QvKdW6KasNF4oNFjxI1DGtrT2srJgWTcrEo71cQb4PJdpCeqPYaYD8LtQeB4ZV+XktiCReh2AroFyym4ub0Uk9PvyGy3faWBpZcCz1R2aHAvGcS8rPaFRHu+tt9AilBg4fRLRaUeulptUGFAb7bHSE642OzTKY2MF9sZhaInsotMOF6sejfLkcmbJldWqgASg1Q5iRW6UrSFykKvTHQCgMw5tjvsBUJLct5r6fHG6L6ywIVGsBQCMXuGkHUvy+sb4QqZSu1JyPIcPdH0CYLXPxBREZtFaBymJRELJSIQZGSht2hFTJlBojC+EFt4QiymTaXDy9dZ4eOPhOxqPlhGj9YVsga/jc76RWFlXmEoBwOqA0Xg0OVeoUQG+kWh0gJQkQy0jehWxaqeSd7TEaniJdTfBN764+LHbILN/sltW01B8U2jmhOxTlaO0MR5SRuyvRgLOrUMwAa5Zspamqo0p10MMoTl5uyW1vDchJ1kX8E0ggJ2TriuV4BpiRfV2Zhhlts4QBHzO2Wk6V9RyWkdMWSkR2gDTeCARmYJWpOZPMTs9JwFjPYIJCI2CsTmOsglwykA0AcZ6xFYihexyJxNUtp5GJmWo3TBFKu/x2AYoTXEoIceiaBMRGpK9agK80VBtIPIF0Pg4xiO0kvKStqQMQAIB/jzW1OaNWM4hqnRCMBGqCRRXk/N8QiTERkJl61mYaPxC5VywCXAyASZC2Zz0vhOIXpKipA05RgekkIikHhe5T7RJys9sQwqA2EpiIE2gz5rYUWNDxydB4FGuHSkMAHpXNKHE3aQo4BpibEUbZpZW/iFXqRBWTDZBdB7TEwBtJLKKROkwRqlQcm+qhOAk0IYiR2cyEmkCgiFrPFQqRHSD0IgbD6gEb+SC1CG0EnFDConYSsBGOEWyjI0kpUMxpwE+ywkqYWoESvJ3ExFWBKZod0iXTW3Oo2gTjZvHITIbcCZPgkpwjVzkhoRMxJqa6Fi0BI5551QnoEcCBqEQm4RoSSHgu2xVXkWEheeBQDQSYRMQnIDvJMI6ADIrM/JYx7xB5Fx+0cygFonqCg2t3aSpD94Rc6f0KVuRaRyhY5A3p4VAApJJ2ZpHMvHd7LFRLFwSCJ3gW5AU4DcZQGQALqvxyUkgdNmKxMC70vyL7BWQdO6XAfwaGdzMTKOhAcIKVF+bsvWSlFqxyrfJDLBRU/+BDJYD4E5IVqFLBVgSsygpokLzCCuswGz9NEBo5n6xx4AseRRRxuw7YiKlOqn9vwsrrF/R2AHAZ4UUew3FJnvYKLouEQF0Wfs+Pw/EWisKI2ySgNvwfGEGRVU3Sbkg5mMS8K0kltU0g2uWR23xZi8X5WalC2/oawsybejzhpzZbfP/JDKwl/O6Ye8PmX+uGECrUVT1ixn8TzMrbLEwC5obBpMLyz6DqYDZK6dSHFEfxNLjRCB3Zl7jZKUV83/M7dCNDMpEkcGhRUwEAllHwU0BSxnQVMCSQSt9noHlUWsh97ko7gjULaylB+APmEHhgz5V9Rbl1SIF18EY05G6joG4wzZeU1+5RRwcr8H5Yd2HbT4KJI+3+50ov8OA9zsNLKfMunpzT8yfq0zi8MX2FKPTOGlHjF6j0R5f+Q3WdsJXOyKn2TQT7oYWIQpYHfD87gSddYU5sTG+EAj8zVdX6JoJ16FDzGyZCRQIr2TEZ6/OoXNwfGs8PtueY91OiAl4tesKOx9AYFDJiPu+hfcS5xsiorgbmszqSfFSp+2Iu6Ep6ResDuijwU1mjOPS5JQXnXV4tesKWLrdWVxu9tiNFre7DptuRD8ZSn9QBdt/dbMpAErrgF1PRCdSJmKqy+CNW7zddtAZAALE9Gd0QD9amEyiwPFQu2xl/fpmA6UDtCZygP2+gW0cbvYNmtZhu28gBLDfN9A6oN83kCqgbR3utx3absI4GvRBou0mrNoR99sOSkcCN6T8IxY9RakzGKz1+wYJdDwGsjTaxsE5Yl28u+uQosDFJZF03NyuCexan1n4gN22XQTKNy2Rc3hHv9ZuIotvTALWejouEtYnA0KQdF5FYs8rqTYCMeiJuV4hElabEVZ7bHeUOidFiWGvYRqPFAXWpwP6nhgujfUIXhWyAikSICPGvYHUCbadGfiUjrh4dov9aDDsLaXtsDGDZYGYQX8NcoVKxUob8w+NaYkVUNsApehePynohtgR3aixOu8x9BbaZPKeKIsVu1lPcKOGbh2CV0RsMLHfrIDUlFJD53QgflJk0R4MzHpCcIoAa2DrOO1AVBPIqzXMLvHtxQDnFMJI8yhUhGo9Wb4HQ1biKCB1pBQaAIKTZa7NmlK1hNw/1XnqKzLYzG1xWo3u6R5SJuxedWBGQNkGSpmRwZVQCfp8gFIJ4z6btAAIm7LVWhYAqc8HuJ1Fc9Vj2ltiJMzPln0yEOsigOgl9GaC29uyM1AnZCYKew2ZLevF1R9A97177G5bsnx3AUqTQgMCaJ9O2N+1QCKQHAcNdeoQBkXW8EnNP/JRQHSUCiVOCvLUU6oRE+n6zhc5c+iAPKW14gdNQJ7PJ1AKEWZwdBI4cZA6IvSaZOolxEUmfxkUpSHh/KwiAm2izxmgmh/sMfYGmBRZly8CkZTcN0C2ZCO7DcsnPcJdg9QGyKceYWshPtkj7Q0492k68zOjJIA0KSR2Exagz5cTKR8ceXUkE5F6hZjXNrLlOp4GJCcRn/r8rJOiJowKcHImCrkKc05JXgMREO8FlDy5QUI2HiGnoUmTQsyKj+QF0AXEe4P0wUCkLaMiJUZWDklNoSPJSaSW1rlqaecudKDUO0kg7DRiVm65vYE0+X2rIvyoi2IkZo+WNEmENmRZSbKCtx4CgN8aCJsVY54UNVLTmhU2lGNpyM9PFBR2AgCThH9Gaw0c2mEi9RlA6BU9lzsDsfJIEyl/kpcw6wnupsU3LaIljxMA1J8MgISKSHtNx4KAaOLMBso5RTvylCkeLLxWIym7UgKlxOH3hU5ziI9js9j87MmNQ+yzlsBLqsNlmfA+QaU5J6+NwCRnBtEk6DNA9/scopEVUQgihwcRcJ7DgsSsLPKClE0JEDlHb2rCHCqU2xdOIDWxSpMDsnwmUkIhijlEJBHIq0E090t4QXVxKIGc60EGoDXJDoAC9heu+ZKVUXn+ONUIg+4DYJY4hQ/XWYOjHAZEgDAtXMvJmjgrR2prJh07RLCYwW+qCIMYmVWhRIt8sXWoQqLrZ8BN1yxCBQ5LGQtm12H+zv06BNI8pjdUWb4cjvPY8Qc3Vt9/l7DY7zCw/E6T93z/T07Sf/Evf4xGOozR4D5QKo5GOiikwvDK7KT7aLGSxFw6ZgbE+jwfi0lUDLARaz1i5xticsyEP1Ik9BmodMoV0h6XZM6NafP1MZMJ0VOiRJUvEgl3nmj0OXUDQAQ+PqrMcBpzfyRMTjMCoJB2jFEhJmJVtJJzNioYGXDvCKxa6bF1DdpsoZ2iKlbIlZ7gM9vkFBWsDIv662sBFIY4ZgklhkIFq3y5jol5+Bjn6KyZ5Jh1lNlCuW4+PgWNIWis9ETXyAgtAvae2AnXmZ2U5Ta3KxbkO432C0KgmERJDwCgjPdmpNQPp02msI+qEB9xrkweO6cTYYZNJq4QWfYsK2abrFMocNtcHxfutwuqMFZyncw2yalN2oqZj+tgKzanVglRljY5bQnn77TMiJrvIVbIVJgv6/d5yHPGx3xWrISckuQwB6vVHoPThRGx1J/HzPk8l6yFsayNQq0fJKRAcT23OmB0GlrFQlbDdVDKCJXrSkX+/WSKhR5A7nNmgq2s0Jz3lO/nwukitIqIiQBy/Zkt4PyZ0yKsGkqdwn3U2X2dGQhdoFyarBRiOQvMpDvcfmN8Ud7U5zhHKq8/Vhjx/LmsgLI6FNZSVhClJDCOuiiNfJBlDJwawdpQlA5KRTinKW2AV+SeX22G5nyolFtUyvm+On0B/3dOkfJAz+MReU75es6nWqdE4GOsuFGKvBconyj9TnPIAdfhRk2KA56rkNs2Yc55mccSnKK8n4Gs+SqnW5A1I2XeoHO/2b2eLclCkHIDADFLMugzcU5bJuhcykA+8aafS7bwl42or81PKPKK1X3MISBMLPk+EcXML+AluewPGaDLygTCG2SJBRdBif/n2HmBzCcgK8CBYh1a1MnnalDDx9hiYtIMPniTHEEgpY7Xr0FCrOqv8nvyOOqcjCVGn62fOd2BcOTS/k2LcFRvcW9n+SVQexkccR5RGl91r6DxknUdBaCVHJ2zo8hik38YFwtBbvjcl+IOWax2+foqTvUYeBKZ/4BCHqr7SqztDJo4drvIIqIAGgAl/rq+jvtbx8svxsefMePV2jDL7ZT2spW3xMYnLAFTDUAOgNACEHGpOsqgtHBMVPcvXEzLvcuvD8ZRPVvHrn8Alur+Vsdq74JS/7taLIEHLqaPlQeWwCPjfGCBPFYWk/iwnjfec6TNQ1m8Vamu/Yt//i0g77Hvpx+/95/8xuv9s1/9t7/1sQHfcYvlEA1+NZ7j2q3hoqK0GVGhDwZ7b7ExY0kfsXUNTm2PrWsoDYUZMQRdQNHd1C5AjJW+pDP4q7v3yIW22sz7KEvuytupQ6N8ocC/n4iWHwB2zhaXWwYPWhK1uwsKl+0OPqmSQ46BlxYB1+OKWG6TgJFkYeRUClwaRZvTVnl85TbUh5wK4aLZ42bsMAaNMzvgeX8CFyi9AG/mf707hcmbLqMCemfIwieI4dOqUFIhAMDgNUzO2Vana7geVjAqYPRzrjymix/y2EyWW0gCjQolH+KYXWg5l9mQAdDaTvj8/pxo+XMKgZWhVBGf358XQMNAR2SAxDIEUPIVGhWL3Pg+ISjPWkiU21KIhM9vz6Az+JpKvxTqsrKUb4zz34UooGQq7XBfuN/3o4VRES5IAkhDU4ASF95Yt9lSvh3JchxTzkNnHVyQ6IzHzb4jwKkieqcXAEvJhLue0mN01uFuoHq0jDjrBvTO4LZvoTKNfw1iACzWKQPCmm3U6rAAbC5I3E4trKZchtuhQWcd7oemxANz6gqmz98ODaz2GCdDloswk2ypnJtMy4iQgD7Xfbdv0WaPAnZ5JqBjiyxormRxhV63E3yQuN2RvFS+xuqA3UD+WDHKbOGmdseK8EvnNcPXah0w5s9KRQSni6XfTwYnqwEpCdzt2jkXnA7Y7tuSf06IRO7XJmI/NAWwkKUYGCddrMZdM2HXN1h3lBtyCKZc3zWu5D5kWQyjKUDZWnLb3/UWxgSEDGb5/rPNgPt9gxgktAnQOhRQd7IesOub4ubdZw+DcdTkxu3nNABIgqzfImEaNWzj4b0k9+1pdiePUcDnjbM25Io+jhpK0bhDIP86IRO8JwDpJg1tAoxxmCayOruJcuYB5ClgsocAx0DL7HbuHKXrOD/fYZhMAaNtRzHkfW/LWosZGK82I/p9A6UDbOfR9xTvPfS2uIXb1i+eV+9VlhvtcmOU5ZqQwb3uHKZJQxna/cZEFlplKOVDsxkr5lzqT4yzddqspiLvFGdg26zntB0pCpjNWLwggleQmj0SJHQ3YhoMVpd7xCjhnYKQcfZ2UBExUE5F3TrEoKA3Pq/hWFI/+ElBtR7GeoyjgZQRSqWSR7EA/EDyjl5Cr12Wh4SQgOL8oYMpwDwFQQBeRvhJl5h5odLs0ZBEyV0YvYRaZ6+BbPEWKpW16J2C0hF+VFCngeL/NbmCaxsw7b95uhF5QtZ4Ek4kMJ6o/2FQZAGMmRCQczFm8C3OArkqOwVhwmytjwLiLCugfH4HiTmEoFZAJN5ZRwBXI8XTC5C1vCHOgUW+QjEDeCYpLJbQKGgMAqRQqN1DdVoqEAK5OxdlQgbEtcUS2UIbMs8A+cXmeoJAMJEsr6j6xlZTtuwWJIoZENSALmawW9xFZ0BcgP6h+awmLeJz/D+3Sda+OFtNa+RSjaMcPgSEB0qO2W19Bt6l3RrcHQKuGvxWCoW6iDSDRAaV7P68AF0HwJpj+h/1UQXeHjQCS3m+rqQH2H7Z5sG5B1UeHqy/fxPw+lsvCb+3WH5Ly4d/fJ7+8//hPyrWxTr1hk+qWGQ4DUfZNOcVGbJlrU7bcSx9wbF0EsdSKtQWO+rH4ykPjh2v+19/f9v2lyk+Xt+Hx1JdAHiUPl6IVNK4ADiatuSx8b1t2o/DdB+PlcfSWBwb07H26nMMpIHZcgTM76fX3f+6sR5N1VHJ4PCex+o8lnLksLwpLQbnRaMUFPP54pFz0AaXd03ncFje5v63TQtS1gbXjTf/hhxLi1Eff0yeh9e9LuUJv0IZEB6mqajrYoCZ0vG23zWNxrFxcp+EOLwGC7fxuu+vS5HxNuk2XtfOu/zEHMvBdtiHui+P9etw3/G6zczyxnzXwe7qWAqOo2N7i7Ee6/ujKVuObajetvAO8x3m7cH9j16Mxx/A152r+/W2bR3W+Vj97/C8/L2Uw53xMQDymvLaNB7HNtGv68frjj86b9W8PPY+f5f1/ppzS4vhY/dX7+7H1tlr6nh0NbxhDI9axN7UhyPlraxqb7luv6mF7ptc90Z5f4P23qn/b1nnNyk/+affBovle+nHT/8eLJZf/Ivf+tiA77jF0iWF5+MpAAJfO2+LxU6KhCEn5PY5z6RPClpQfskpuywyIGQrZHFjzFYWKYjkp3afZODDeQrZJZK/c53A7DrKhe9l6+To9cIKyiUkUSyDKYnSN+4DJ8p2+R5OSs33KpHQO7IssSul5mTUOWlzTRrEboj1GOtrARR3OrJUigJWfCaR4WMMVqRAiWHle9jVkD/X9xx+ZzfCmLXqrH13QZUE99xfAOC8fQnzBo1jSWtwWOcH1cVSpUt8KrlsVi58B29ETnatVFy4EAJkeWFiIJ81z0zeIwQWLn50vyjtxEiEPEz0U1uzOCY0BFGsQFznIcip29c6lrq9V8XCUCwgFSji6xYJp6vvbBlhLTqvIakiQrZ4ChmLFTCGnKi6AlNFXkeSWFOHlsBbyogYFKQK9F/Gcn1dR0yi7AY48TWzEPP8pFx/iijxo7xuCsCq3O44tkkobm9OnM1xsfQZFCvq8tw1oYyDz5U6IyjGCijWF+5DaT93IeZYxTgpIgqq3CM5lRIJKbs6ZjKp2k2SLD9VbBbvGycF0TBrR7aa8I4qp18q8b8yIYUc9xgoBrXefbFFpnbvLG6Q2dqymOdsoSmpmOrzfD0fZ9dJTnEiMLto1umJuI7abTOB4snYxQ+Y00blmK5SJ0CulWyRCfmzE3NS8ep+we+ZKvXKos9VvRz/tDguaC2wC2Sqrie24nmuapfKOnasxHvl+oTPcWcJyxREgvqdTIQcyRJap0Aq5CeCZMpxdlw/p1BKAiX9E7dV0o2UMeY1wSlDFBbpRgRbm4BC8MNWmToNT0nbxPXmcdbpOqRbkhqJIOb6mOlTp0LaMxPFEAnRNy0Ld04eP+Z2uZ1jqXakRyGTEVW6lCToHH9GHtYyNVZuK833yama87dyhWW34FmORe4q0Y8235ctdzK77HLaoRInmOq5q/pYj5Pbl3SupH7J8qrdbjkesU49UuoIy/YOXWEX5XVKjVouh8D0QCnwAAg9Zp2r3mGHqYEOrY6lv49hSW7zSF8Ocf83doU9JjMsj9Xt1XP9mHX1sLwdoH5zPY9e/0g5ZIX91jLCJoCIK343y3caWGoR8XH3Cl9PG7gkcdIOkCLh3rUYo8Z77RZj0NAyYOcbrPUWO0+/KJ1y6IPJoI6YY60JBXCe2x6N9IgQeDWusGnHB4nlGVBufYNWuXJs6xus9ISYRIkhrK1qWkYMwcBHiScbYo8dvIHO8X5SJFgZsHVNcUG10sMnVdwzGeBxu63y2PqGYhqzK+zHmxvsvcUUFVbrCXsG3ioUgH3vmuLGyy66wGyNNM1QQC/F5UmoKtYzREkusEHDyFDAKQNoIRIGb7IrbICLqrTlAsWRjkGDE7wbFQqItzLXW8XwrcwEqwJuhg5GheKmySCzdoOVSNg5S/vI3KeYBDrjMGY3VgbmH5zcQ4qEr/sVlCCmYI4RdAeW6FbTTmnK52vQamSEixICFN9J8ajklsfKAhdlcQWuixAJTWbi3U62gP0QJS7X+8Im3Dty42I31UOAza6xrFDga08aIrMacrxmrUTwVRwe1TnHK/J3KYhxeMpKCh7L5BXaHC84OI3O+BLPyTLzGWSyC7DVFDvJygIu7JLKCgjnFYwOcF6hta4oAHjM/JnjFX0geSUA62bC5BXFGObnSitS1vTT7ApX4odVLHGVMYkSG8nrgNcmMLuKA5hd462DEAn3fVsUISorZ3h+ASLaYllx4fpqRYXVFF+5vpjQT2ZhzVo15OabEsXKGhUwTKYAeI4rnXJMJEBKDwajm3bEdmhKLGStWGuMx360RUnhnIY1Hs6rEmfJhd1l2S3UGorvlTLC5+t5fjkWU+us4Mp9IyZnjtOclTUhkGKE3Cx1ifk0mRTKVWRYRcmRrcbs9nuyItKyWsmjZMQwmgO5JDTWox8MlEqwhj63rcM40lpJkRiXUyJlEI+Z22LFkTGhKImESMUtuLZopyihNF1nbVhYCznlFO/qdK4PmBU93Bc6JgvR2jjqzJytIPP4UiR3ZTdpdKuxUjKl0vc6nlWbgOBlId/iOU9JwE0aSgcYE8qYpEwPXZs9udoGr6BMZhrPc6gyY/k0EtGQyM+blEw2pAoZmhQUR4s85qLk8YIIfpIo7qYip7gCgBQUhAqIk4Y0ATErClMSEMYj9t/cFRZZ2QOQckhIUoQygZVQMZOCkettSqIoMZKl2N4UqvNZsVPYq1k5g6WiiRVSdUyvbsjNl8/LTHpUK8HI1Tg/XyaQG252NUbMTNhcP7uiCixJYpIgIKdZkQQCzzLRXDAxVVZosRswNZqKK2xR1nBhBQormWoX3oX7aKVYqkl++BiXQ9BUgGkVO1nAWCb84es5nvUYgEyY83GXY/NFM9inD4vv3OeEpdKA630MrNbnDwHjIchLR/p+BLiJx8Bx3dbh/3jkhmqcLIo3gsl3wXjvAGIfb+jhxY8N/bdSvq2g9zdQvtPAch8s/o/r78NlghkAZaPJVjsXZdm0qcqSOLsF0gaVN9a1JY43mD7IYk2r76Vr5k1zyj+ONTEJA63aBbJ26dPypGwOefPLbfAGGZifqWPuqXyeY/t4s/4reZYVI3P/6vQWvFGrXbti9RJZbtboWG095O+1xa5272NrHLcx1zuTa/B/vl6pVJhe2UJXb8hoQ4cHbLCH7oh1Hw5VfXPcypym41qtSebMulv9CpQcllnOh26ptUW6fqfL6rr6GMVPHagED+azllntuljPWS3Pw+90bf6cACGBl2IDtthBLOV2uNCOuXECWFjN+L5yraBN0U2VMqNYU3kjwhauxY/iwQaiXuKVVeW2/hXL15Z2DpjzUgK2qqP+lb6grINC3FKtjUPXxtkiObdB8nw4bwCw5Y18TcbCm6rq0i33tf7B5o/1D3++dy+7ZUwOgJ1q5zp4U1FtxPZ1GzxnVX93qputdwcMiou6BPVhqK2P9eVlU0h1DYekMIebH66fx8oby8MXHai/o0gPxjZWm7XxyDzUZbxpl2MXc18PN2BjXmdOAEPun1PNPE8JcKI5agmov095I8ipT0aBHNuU1xkrW/LG1h15pkQla8eMlwcydbX1MooH19V7YC9oPLsbe7zfVf9Drt8fijXPUxQZGxxYZpGAQkfF74s04wiR7w35i4iihOEB9Ozxo8V9D1VfF3vrVC2JNGOf+lhA9fjx8s1zeLjk36UkAcj6eaweb5kbEfOppYtrnvPFeVQ/B5gfx6Ob9cNnSszGfPpuls/Tg2IW4Yr179rxwQKHz8k3uWYBQN7GUFO/hx/I4CFIe1RWb6j7eP14PYh5zRgfAMfHyt9h/b1LeWM/gDf25Xgd1fvo71D327X1m23j9+Xvv3yngaUWAZ9urvHz7SUA4P3unghp9qfYDQ0+Ob3Fzluc2R5f7k7x8ckNPrs/hwDw6ek1nvcn6J3BZbfH57dneLre49XQodUel+0eT9stdt7i/3nxDE/Xd9g6i8lrdMYhJoGrbgctA356fVUIXZ50e3x2c44PT+8Qk8CX9yc4bcdidUtJoFEet2OL3Wjxw8uXmILGy36Nzjh02kGLgAvb469vn2IKZOE7awfsncF+tJAyFjfTp+sdhEi4anf46e0T+KBw3vW4HVr8o6ef47PdBV7u1/jh+Uv88v4CvdO4Wu0Lq+ovXl1g3UwIUWLTjLjZU+qWRv//7L1Jry1Llib0WefNbk53+/devHgZLyIjSSqLpIoCMpGYMEEqIZWUIyZMQYyQgBH8AIaMEEhIMGICDEBCoiSkQiABlSmCysqsiKyM5kXE6257mt15Zw2DZcvcmn78rQAAIABJREFU3M8+9937IiojbipMOjp7u5tbs8zd91q21vo+h84qXCwaVHrAEGN/LvcLLMoeC0NgRs1gcH+xx6tmgVXR4zAY4vcMAtuuwLrs8XyzwqrusDADDoNBG6lgbg417q/2uDrUKI3Frilxsmhxs69hjMUHpzf49PoMj0+2uG5qNL3B+6c3OK8O+P6LR1hED5ZWHjaiYlbawihHaLoQeLpZw3uBqiAeTR8E7q/22EQAnavtAnbQ+MNvfgIA+IeffRNGOzxc73Dd1NDK4WZfT8JXH52Qd/PlbpnQQEtDCLgXiwabtoSSAe+vb9BYgxf7JepiwK4tcb5ocHmocVa3uDrUhIQbvQdGOTxeb3FRHvCj6/tpU+PQFnh8tsW+L/D++gafXF1AK4dlMWDfG/SWvLpFRA59tVmiKCwerPZ4erMGAKzrDr9370v8ZHsPzzbrtIYA0DmFfUcUJqWx6AadwpvrKOMhhqGe1S12XYFFMWBpeuyHAlf7GufLhmh9dsu0bquKAKy6QScv5dmiwcvdEqd1i21bQkmPfVMm8Ji6pDEtSvL43+xr3F/v8WKzwuOzDa4PNXqrYK3CourRDRreC5wsW/JE9SaBIn107xIvD0vc7Gj9jLFYlj1Oig6f35wmr1JRWOJOVQ6bQ5XQTM+WDd3zW1rnZdUnYJ667NH2Bt5LLKoO+6bE7zx6jkJZ/KNPP4CKXgCmxkHcoNHa48kpce/+9PIiGe/sfW8jRU8IwHtnG/zs+QW+/fgFfnZ5nsKhAeDbD1/iuqVnte0NLpYHfPbqLHmX7p/uIAA8uzzByZoAWw5tAa3p2fxb732G/+/L99F3ButVg1XZY98bKBnwrbNX+PNnT2CtxHrR4XqzwP3zLS43S5yvD9gcquQBDF6grntoRSBFFyd77LsC66rD5XaJ02WDbtA0zpY8qufrA5ZFj6c36/RcDtHzV5cD2t6Qx7ApcLpucFK1+PL6BEp59L3G/VOiBnp5s8L90x2udouERluX9F5i79nf+eBn+PHNfVzvayjl8SByFf/4+X0sawLNaXsCofnmxRV+9PQBVssW751s8MNnD/CdRy/wyauLBF7z+HxL62oIOOfVYYlDZ1BoB608Dp3Bg/Ue1kts2xJaOVwsGnxxfYJFvLeb3qDvFc7WDQ5dgQ/Pr1IkBwBcNxUObZlC6B+e7NBaTffHoJMX/P3TG3gQyvC+L/DN00t8cn2Bylhc72ssSvKgN73B45Mtfvb8Ar//jc+wG0pcNgtU2uIwGFTaYlV0uGprbJsKT043eLlb4htn1wCAe+UeV90CrdP49OoMD9Z7fHzyEt+/eoSTosO6aLE2HX6+O8fadOi9wov9CrUZcLlfpPfl8+0KdTHg8XILKTx++OoBTuoWSgTse4OzukWtB3x6fYbzRYNtV2BZDHi1W8TNRIlV3UGIgJtdjfcubtAMBofeYBg01osWj1dbSAR8vj3Fk/UGP3l1Dx+eX+HFfoVV2aG1Gr9z/hx//PmHX1vveHiyw5dXJwhBYLVoU7TGwgz47PIM60WbZH7TVrBOpnfSB2f0m3DTVLhYHtA7hSa+s947I51h01YJjG1V9Cnq5CaCso1AYwa/+/ApfnpzASkC9l2B9042uGrrCIDnUpTJpqGNqIfrHV7sljir2/TbcVpTVNKr/QJdp6EjANKyokiJZdljcBJtb3C+bNBZnZC6S21x01Q4XxBtGlOePTrdohkoImxZ9Ni0JfZNiQcne7y4WQGg6AET02RO6hbOywTsxs+li5tyRWEjjzW949umSB57bUb6tG7QsFamVBLeFO87AzVDteZUFmspvN/2hApth1gnQ932keqsbU22eTlunvJmsY5RCJwOIkSAMQ5dSxEnInrlnaOoJnZGTHKtxRjFExxFG/AmOEAAZayLOEdRJi5FGigo7VKqSkKvjpEgrif+6JSekVnDnLbBqSLs9fZRPvnGa0qHsZIQjmeb0LzRTAcweu25v3hNcpTkxnzcNEwbogAYuOvYhvCkvG5D4Net/DX2WL7T4D0f/I3T8O/8d/8GPigu0QaDn7b3UUqLR2aDR+YGf7L7LaxUh+thgcflDX58eICPFy8wBIUf7x/gzDQ4Mwf89HAPv7f+HD9r7+FBscXOlviyPcWzZo216fAHFz/BX+wfY61b1GrA9VCjlBafHc5gg8LfOPkCz7oTSBHwol3hd0++xPc3T6ClwzcXl3jWnaSQVgnK/bxXHnBmDvje5Teghce9ao+DLdA6jcErXDcVPj5/hUoNULHdlelwappEbeIh8KJdoXMaN22F3z5/AQB40axwr9rjBy8f4bRu8XCxxY+v7uHJeouV6fB0f5IMwO+cvcBVt0ChLF40KzxabKGFx8EaVMriy8MJGRXxIX642GLbV9gPFKq5MMQNen9xwG4oUMRwQS09VkWHTVfhw/UVXjQr7IcCtRmwNi1eNis8XGzx+e4UDxZ7bPsSjxZbfLE7xXurG2z6Cp/fnOKb51f4fHOCk6rDuujw2c0p2t7go3uXuGprLGJYK4euNoOB8yKN4eFqByU9Nl1F1CPRIFyWPaHBLjcolMOffPJNAMB333+GXV/iuqmwLHtYp3BvscfgVaI2YWPtbNnAeYll0WPfF9DS4yoqdD4QJ6Y2DifLFp0lZNebfY2zFSnnJwv6Ma20Re8UBidxfbOEHyROzg8UwhsNyBebFfGMbiucn+9gnULbGZTlkDYBXER1vH+yx6E32O5qnJ9SqPWuKdG8XECuBlyc7bFvC0KQBKE0loVFAND3hPxZaItuMClPk8ObGSHUDgo2UjEsli0OhxLeSizXLXZXCyzPGvRdRE01LoVStvsCi3WHZl+gqIhCYll3ab0OEXXVRoTJxbLFflthtW6xvV6gWPTQ2sMoh0NbJPTTw6FMFBEcKnl4tYBaWiyWhNbatYQOGlqF8qyNhp5D1xnYnsIPi3qg3FAZ0O8LCBlQrSh8sG8MippoYIZWQ5cWSgX0rUZRWXRPF0AA6vd3sNHz3R8KLE7aZMQ6q+A2BsILyPMu/UD6yJ9pSptCBu11gfrhAe3TJdRFR4pD5JCzryrifpOgsLODgjnvxtzoXQE4geK8Rb8rABmgS5eoMOQXFdyTDrqwGPYF0MlE/6BvFNwTUuB9o2FOOgzXFXHm7QxEbRPfqJDEOwgnIBcWfmuA0kM0ClgPxOunyKUkyxi6uNeQnYQ/sZQDKQNgIrpmo4HSAUPk5NtryEbCn1rAEjcgbgwpWmsLsTEISzvmc6b2aC7lFwbDiYdfEAKn2ktIKzDcHyAismeIfILFK4X+0QDRKOitxPDAoniu0V+45ErS15o8io48L3YZEApy7wkHhDJAbyO/aBUAD+iDwHDqIVtSPn0REEyA2in40sPcRC0vhrK5OsBXY66j2cqU0+eLiF5pAswmKoQK8BowG4HhhPLnXB0gW/KMeROgdxL9PYf6c4WgAbsIkAMS0bzqBVwV4MoAs5FwiwC9Jc+Q6gBXAEECw2mA3gmYPdCfALIH1EB5i8OCPnsFuIry6exiHKddUI6ePpDsuosAdYj5fQWgWhrLsA5QjYA31K5dkoIZFKAb8jbbVUB5KeAMEAzlB+qG2qa+AN0A/SlQ3AC2AlQPeAMU1wHN47u93F9VzJbmDkHjlA7kNR2A/gxQTVyPA/UXBH2HoLF4Qzme+gAEna3fluq4SEeLEMfMa18ieSNFoHaqVwH9mnJLfQHoHdXjdRUBEDbmlApA7wFXk6y9oTVVHSB8gF2QzIWl62SUl+riGA2g9wFBiTFf1ga4UkC3lIPrKjIEzTbQnME5rQK+BMwuYFhGI8oh5VaqliITXCEgB6I2CVqknEs5hEiLQjJzJdWjnN0xtzQokpdkl7fn54bqeB1zWcPoOeW8WG/Ge0TY8bwIUU59gDOjJ37iMY3jvJUrGgBpA7wRyVsrQoBXo9c1zymlcZGMAdziwAQwIsBG6hh6NqLcVDYGkV0TqVK8FpAu3O01jnORbrTd8rnMc0lfl7M5t/vmgUmT0CrE73chowlBcsls06PezbmBeqTO//0//Me/coCbU/Mw/OHFH/3S2/1fn/+Xv/K5Ae+4Yfnx7y3Df/I//j4kPNpQYAhj7o+Cx8FTPmXrDRaqw85VKOUAHyQOvoARDkY4tN6gkkM6dnCk3DJf5Vq1OLgCRjpIhNTPEGiXuZSUS+dA+W4L1ePG1lDwKKVN9RLwDSRMzLvc2xIeAkvVYQgK1ivahfYKterT9QxAxHybAIVV9l4n3sul7ggBF4Qiu3cFJAh8qHEm8Vy6INDHNpaqj/ydHtarlOfpg4SRDo0zKYzYBYFSuoSUO4Ybq4Rky9fS+KjNWg0YMjRZIzwaZ1J+JeVF+sTxyfVbZ1CpAQdbTACZrFeUw3okgMcHEfNBJbR0CZwnz49lsCYGegKA676mndOipbxIr1Meqo59cxi1jevJObZaumTsMxeoh0j5sNwX8XeqlM/GYEs8bh9E8lwwvU2eyyqirDkn9i4kV/aOz+vuh4I8m3F8eft5DjCAZETzuPk857/mFCR5rqfJaGTmdZiehutzm3muJ5+f5zWabMMCQAoVz9c0YET3lSJMaG54XgPzbaoxn3lEjxbpegCJzkNLT+k/GbgVjwnZcc6DZG8rn2OgKV6nIdJJTHgsmS4nk01vFepiwKErEo8ln+ui8cnrb52acHO6uGtutEuh+gyKJQVw6AyqYkiRDzamDPDcCk0GOwNosex5rvluM4fsc950ylGVPslWZPeYZbqYmIMrstDBtGGdvS9zgCvm3QTI42GzHNL8OeDx9b1KdBgAIjjWmAeZ17VWRmqWkTuTcgrH+bJhnzwBajzHofrzcHofKA81AV5xLmH0CDirJrGCQhDtBn8/Fmovopci6WHRs6AisJcQkbOU0w2i58MODLJFyip7KHgsQoJArwTl5c1Dw4X2lMPnYm5eENNcukCGn1DjxkNwcjzmBTj+lTk3Uzg2f06gTBhBoEIGDsV9Mb8nC8WLMRaW29Ce6uXgSV4QPcbXLdxunOsk1JrnIfh/dp0IY1i8QEZrEU8zF2T2DsrDaEPuweHzVlDuX7xOZO3TwfjHIefH8gh5vHmMbMAkDYHHn/gds3o54E4CqcnisJMRJkajKPXBdTgmej5HNg6CmADIJMOCDe1Zk3PNYGJvvMYgSUabz44fa+SOcqvt3PrMvx+57pixNDn2uv7z48fkd8f3o7KYt3Hs+7z7N23nFyh39vEmJbv2B//ZrwEqrHkQ/vDsn4Fh+fK/+pXPDXjHQ2H3vsSf7T+Ag0yG0pxiJKcWYSOIiw0SEhEZ9gjdyJw+JC+5EcZGX27UcTlGFTIvc9TYuZJ/jMIkGQAzNFkuxygC5mPhz6+jCpn3Nx8Pf8+NCFb6c0AZ9xpZ3kVFcqzPse5ynOutFsdyF+3HXeW6rW/J7a6xzD8DU8U2P3YXLYI/MvgQBK5QHzUaj9X9KrqMKU1DMckvfh1FzKQd3P1jnX9HPLZFeWt8d83jLlqO183leDvTjc7XbH5iz7/zX9HXXeM+Rn/B/xlg6C7KDx5D25ujm7Z5H+zBbbrjYCP52No76hzrHyDwm7tK38WfBhGAUIwn8q362Tje5H7NS4fXK4B0YKZt8fc36IPl7+ztc3ZQRxU1l8nE2dF7fqxQTnb2E5o/ALNbyjN3XxCYwgTz/MT4HQBbC6+jbBHi9vvDumysse30bs3mcVS3tuPJcOTBDgIIOXDTMPvdYUMviGQwBr5eYDQK+fo+uz5HuOV6bJzM+Rsn5Y77gJF0eYzzhNFbCaRvWQZ5VIjCyWgYCtymD+E1Rrp2wm/Ixg0brDNr6ZZRx00mz5qYfKdrsxdLmMpVjHsX1PYxLxbnYefjz8ZEc7j9ef5KFZM54/YzwmPL5jhpI//sZ/b6sc8zY4rbOvZTOTHcgNE7d6wcm9dXlTd8X712bMDrjdp5vWPHj3w/Zswe7StZ5cfH8Np2XtP/25Q3/Fmhbuf9/IKP+y+9BCCE36DC/lqWxhn8+fV7KV9w01EOQqUHrEyHL/cniei+0habtsKqpLyvm6ZCaSyM9CnPkkMrBy/R9AZNRyTaT043uDzUMBE9sY9em23MW7hYHrDvDaQgpMfTuk0k9ssy5oHNjMfKWBTa4tVmCSEAE70NjHDorMJi2SYPTdsbaD3mEnBcPyuxw6CwWHQQIJL1orDYbWrowqIsLQ67EmU9JPQ+F/P6Vss25fh1nUZZ2rQbLmVA33Nd2gkvqwHDoFJsv1QeQ6ehCyZYH6kkeIe8XnToOhNDDD0hCnaGSNfbAkU5YOg1ympA1xpUdQ9rFbp9gXLZo2sMkaprj+5gEAaJYt2n/hhAyHtJNAsBiQ7C1JRnZO0Yzms7Teh5XqAoLZTy2D8nQ7U4bylcsVOpji5senMGL2Gb6DGOdA2c6yAE4DoVd/JBYX0yUEifE5Daw7casrLwnaLQwBw90EmERtFO9oLaFjJAGk/hhipQiOHSkuIW6RGYBoKpGdTSEvVFqyAWpC2GXkJuNEIZgNWA0KspQEsRd+CtoH4Uof0lgmze8e5lIs5m6HoUHuglfa8cxF7R+ONaQIUUqig6SXU6RaGcAXQ9eyhY2eR3bukhWkXXHBSFLcrYZi/JcyAAdBIixPFENAsZQw1Rxn6sJKqEXsAvx/BGWBo7AuKY4uZAT4paKClUUliRzidqBxEgBolgPPQNhUnaCzuSbveSrg8AIyuqRgJewK1cAnVhT0PQIc1d7RXciYW+0bArN/Eo6BuFoGPImAqQnRzbkwGykeQgWVIIZhA0N1ZOzbXEcOoRTIBsJWQvEv2Daih0E4Jk4GsPtZfwFYVX+jKMP9SxjvCAKwJUI+FNgOwFXB3pLSSFaDFthGwlhaXVAchC2CDidSaGrZkA1UoK5atDpI+I4ZAAXBWgWgFXjPQWREEREGIoYHEtYWsKPxVeQHakRA9rD9lTO0TXIGD2Av2Jh+oFyeDEw2wl7CIkhEd9EMi9J74I8NHhKDyF4emWxuA1HZedgFsEiIHkFRTNWfZA0BQqC2S6tQkUjsf3QUvPRsjvD0nrxN6boCi80VVxHEWkBQkUJqo6AbsKKK4ovNAXSOFxIgDC0li8iWGoJaKsxhBCCApnVY2A6gBbU8icsPTflfQZEnBx/BxKKUIM8XQUbghQuKocSI5B0WfhY9hqDKnlNtK9Zse29GGUZZAxJLeLMixJvramkFhfxDkqCgftT/G1i26pXWAcM4dH85yCpvXgIKqgaJ14zCnENN77QcVzgq6li8YQTwDpfuLiFVBsA2zFNC1jiGvySoLGx23KLoaFDtP7EKDjKZQScd0jRQgEfVZ9QJCjAStCoPDKIcR7l87pJqQQXulobN4IqJZCZ3lcFN4ooIYQQ4YpVDMIjP0EjOGbcS5eCwgX6N51mZGsMiMn349i+pssrDNRg8Q+JnQofmyTBjvKY15EGNvN26TrA4QfZcHj4udu4h1Ng8W4B5FRuaT+PK1B+h/ifzceH/epIuVG4DUUKRw39XXHXPI9vIk8xUx2yL5n7czLLePzdUbzEQM2H8udhuyx8g5HZb6r5Z0Ohb345x6Ef/O/+Xu47muC+9c9fJBorEFjDVZFh8ErCum0BrUecBiIcmNVdOgcgZKU2mLblajNkKghajMk/srLZoFl0RPlRSDKCBcEFhHEZ9uVKdyt0ha7rsSq7AhsIQKr5JuPSlBIZG9VSnpvLYEysCFZKotNX1KYWSDqARepHZiKAaDEfAEKGdx1JXxAonI4q1u0VqMdNJbFgNYSKEtpKCQ2ANhFgAnqwyUaBg67K42deAub3kArBxXRV50XCfAl57IEkBL8m94QoE4E2bFeJtqKRF8RQWyYjkIpj2XZY9eWBCBjFbyXqIshgRGwtzb3GOWhkgAZ5MDIOcky45C9PoK/MCDI5XYJpXyqI0RIVAfcRxXpIigcERN02zH0ECjNkHg32Vhn6gyt3YRCgbkiq2KAVh77tkhtWytRlpSPWBqLpisS+ACDijDsvxAhUQ4UhU1UCVo7LKse7aDRdbRJweGFHAYIjKi6zokEgOD9GPrHoYJS+lS374n2gTciypIoGhi8gEMLpfKJesEYl/IQOXQwBEHAA0EkkBk7KJjCpo0IpmLg8ELvYhhmpDBwTiSQg7IaYK1M9BjM4amUR9eaNHcpfaI9yMeiM1oLgRE0gdsK8dlkjs2q7infc18mD6RUjrxg/F0GmJgX2nVmEtqYaDJiSGFRDugag2rRo+vMhIagrIbEecrUFUOvY4gloOLYba+hI78l83qGACyWHQ77MlFRyIx3tCgtmkNBCp0KcIOELi1sDAv1brwfEABpKAaCKSoSsISVkCqn2KBrlPZERTGo5LxjWgapQhwnhT1KQ4AdttcJUVgXMRx/kFCG+uFwPhFpC5i7s1r26HtNABQijHJpzUizED1turQYGgOpPZRxGBoDUw+wrUlowKqijRoR18xZSX0xGIUj7lFgDCkVKqOhCALMGyqNQ3ASqnATZ6cbZOILDQFEQxHlk3giIz0F3cMkW20cbE/3WrAyo5Gguq5RMMshyW5Ex0aSe7BEWeEHmdpn6o/gBW2IFY7uizgnoWLUT8y5Dh5R3oi0IHEcA8kpyadVEHFzhnlShQzwnYp0FXGMyduLkVO0l2nDDuxt04HmLGgjTRhPm2ulp02yuLklSwe//wX21AsPdNFS4I0tXjze7GJqDfb4sSeu4E0uQRt0fM4L4lYFsg0/TBGd841AxGOloz5FbIf7DlldkfWfc7/mocex3xy9OBmnKbxVIHAIcHZ8Eq4836ALSH3RxlyAGMRoPPFrJD63YkI3gtvzYIPWjvVCJqfkec3aprFPDaYkX2SGXR76y/1xyY3AzIKaGzn5cFM1GSae6cBGJxtPuXKI7Dhm55Gdn12f+gvT68W8mSOyyed4tLzGRMjn/ZWG3tuYGrN5UGdv2cYd5Uf/6X/4Kw8XPdUPwh+c/L1fert//+q//pXPDXjHPZb7rsT/9ZcfE4m4Eym0RsQfGH/Q44tChwhQMXo4oOPLwwqgckCr6HzkXJK9pN3vpcNLfnkLfvGAPBpBEHhELwlIoxcIlcerNobvRLCFaU4DEEwAZMDN4Ry5p4JJnoWjHX8IEP/SQHkiE4Ls6GHinXquLwbypmz2EsEEeBOwaSR8GaJ3Y/Q2+SqkHyw5YPRGxB+ffS8mDzmDR9CLPcQdz0iWzR4sVn7i7mfaPY8yCApoBvqRaTsR26Sd1m6g/8IBXUeAEl1H7QcF9C21Q+ATkeyZc1WCuJW07yqSl7Qj71U/iLT76wvaHb38wQoIwHDiYa1AnyfGm+zHJgg43uGNTw+PAaCwOx/tCNcKeBUQTFxzCbiBdoYdA2cg7kjH27JpCQSEx8072kPc/fadgKvpR9Szt0cDwQGOd0MLwHug7wV5KAFYC+z3gnal6wDXj2HjvFYA4FhhkIB34y4vr78bBHz0WAb+sS4C/CCI77cIsI0A6oDAxO8qJIPG9wKiDAidgDBRGYmeMhGfRQHqGwIQhurKMiC0CwgDQAYoSfc8K5qsrChJHhoACAeqrwqSl4z3YLCAquK8ZTzO943OlIW4zrqM99YA6h/jznaQdB96DYg90ZuYdYgKGj1TsojzDLTOsie5mmp8l+XPBt+7sqthFoDc1zAV0q47AKimhmBvSARbEAzuIcgbAtA8ZXa/8vVyV6Nc0b3Dnin2jKgWKBd0DwiLBKyiiwjooTF6sAXVoedgBBuRjrxW7Hni80D08rjoRcp2voOIoBFRgfOa2mPwEd5Nzz0sDCwzejbiMxEfV7OtoCskD6DsqV7ylolRrqoFhiUdVx1gl4Del3D1OEbVksGdPA96lGsac8RkCuzJjONnzw972NhjqRpMvNE+a5PXktcmvZNkXONA11J7Ab6g58BHr2/yxPQE2mK2JgGj5O914QKCFuSV6ghsRPX0HLEXMkjAVgKqC1A9fZZ2BFBhryDkOH5vqD4AOEPeFNUjAb0wMAl7i+AxAq3wfVSMY5WW2iLvl0qyDJK8XnKIMmTgn4o8jPybEiSgW41+dUyzfrOiuhEMR8aQavZY24qOBUX1+DeCPc46jtnr6DWU4z2hOvrhyL1bPOZ87bkECZiDJA9g4OdPwisxAVXh+xIA1BA9jJYAZJJMA8k0xOc675s8iCJ6lUV8VskTRm2LcV00yVX1Ysy3jJ58Bpjxhq1JJO8at+uzd1z+uyP8GCXAv4m555GeFTHxos1lNc+fpL5FBlIT6DlIbYe0Dvm7iRueG5R8D0w9logeS5HayusfNcgyY3LiLWRZsIcyenupYaTv7I0MWR6GCHEN2bN6l0MpN+hm/2/ln/IaYXps0lZejhi+xwqPNR/j/Pud5Q3a/9FXt/Kb8guWd9qwfFBv8e/+7f8DAAH0HHwBBZ9Adg6ecmQ6r6Hg0XmDUg5wkGicgRGeAGaCwlJ1aFyRvg9eUR3psFQ9GmegpZsQozfOQImAWg3oHIlyCBK1GtA40kD58xzQplQWEgFbS7loDCLTR/AWHwQqZeM1AX1EJeXCgDGcG2q9RKVsAmXR0mPTk1evUgN2Q4lKDdDSo3VjLlalRpCY3msUMpJuRw9l76dhvIW0sGHkDdXSo4te2RwUh+fUOo2F7lM7DATTO4VKWTTWoNQ2XdtZjVoP6L1C53QC+CkVeU7Zy7wwfQI8SYAlQd7KN9XSQ4JyaKUIkKA2OP/TRICfq5bAe5iCIweRKbVNYD02SHRWUwRnBG9h0BIhwgS8hL2pDB7DQDFGegxewmSk9AAZeoNTcF4kCHvBMo65UzYCq+RAN4kXNd43DCrDHmH+3FmFKtKShHi9D0ge8ByUhz87L6DkyOk6WJW8sslLHD3V3hNFSW8VltrBeREBaeL8sjoMKAMAWo1w68wVyzm53N4qesV53ZQkEBcGZeHcMilGGPPBKRRYzt67AAAgAElEQVTSJ488g8x4L7HQI9ANy42AuMb7l4FllBjzjE0cc36OgW36iLLLCLuIfZk4xgBE7zCNlb2yeeGQbgCwlqhQhujBz3NB+16PgDbS07qYMWZqGOh+1JrDtGkdGLBmH8PlVaTRyfliOydTW86T93iIIDlDDJHPnzsbPadKBXTML+sFZFxniJA88rRW7On1UyJ3AJZ0vQSEY10EkJKksYk4Xp53F++TBH8f5cnjOwySPGoygtlEbyuD3FDluKbRAxqCgLcCyngcBgnJQC8iTHMlAfJG5IAjAiOPacYlOoHKF4jE8jFMmHMTU90w1uf24vkcrGfClyow9RyxNzLEcbgIltOrcWy5EubHTZpE/MheFvbkALQZa8VYn+eUe7BYpkGkvtOmLHu7MI41FVags02LWx6bTM7ME8oemLSBG0AbnQxsw2OLhrqwAr6YxRe+RSHPXUgGz2TseX+WrY1szDG3M0jeEEWqM57LjI25cZR/FxzSGsb2HUZAID6cbRrxeQ7TnALv3L4ueTt5jE5kXjmRDD2Ww9iPmBp40Xhmj2Ty2vHzynVwe7lFJt/kmRPjdwDjfZidm8jviFEzyYENIspFpk5uGXy5cyAvdxgzkzV8032Mu4yxu85n9fLwVWD6aN3pWXwDW23extHyy2qH2/oK4/AXGsP/8pbt/rMq73C06FeVd9qwbLzBn24+AEBAPL3T8BDQghFEyYDKUSVZMU1opNHoMNIlSgnmBmOlv9YDGmsiL1RIiJV8XsX8w6R8xrxONgoYsXL2niUjL4I/5KiZACmgOvaXI4DOgW0sh1l6ARPREXOUSg5bZR7BHDGSjYJcqWcjgNEjXaasAVPFFyAFjhEl5+AvyZBSjjiUAnmj+DjLho31/D9zOJLRItPcnSdKDc5JzUsKucuOM2LkBMTIj0AjfL6NoZG7GHLqM6N1bIOuHxVbVvZCqs+5qSEAPirVrLATF+YYNjtejzRuH5XubTQOWSHntpgfi0MgE8JkAP04Bgon9Z5CQiWP3VNYqdR+DOME0i8N31dzxXz+m5iHaob4Q8vzAZBCGEVUGlkJRuwrcWRFxUPkCqcIKTeW89qFIrTLvfYTPq28Ls0PI4BMbNdz6F1EqEQgOQQv0OQhbPE4jz/dT3GdUwgjK/Rh1IA4/BEypIiJvjCp3aRcI+uLUTI5rCzeWxNZgIygXlNOMYcMJjnGudGNSMr7wBQggq5FAHo9ynoSn9VLNAW77cWYSxs/uxi+CA/0KhBYSfRUT+bDhgiAITci2KDJkTF5/ByqpzLjhs/72fWxrmNDSI79udi+y5X3PIwtAKKX8DqMmCCeFEqbyT5NZRCwJhoMlj6LQcDpsX1W/nNDwssRLTPly7JsYp/JEx0VX8/1ZNYmlxgJkhRgNvRim6yEy1l4HeelCh8jOXI0z2gMqT4aBjPjjY0P8uDTeNkTnRtAnLsqbMzJjRE27DFLtyjfS1kYIEeViJje7HUuN4zt6NHoEj62Fd8PgkGBVDQc2ZDgtc28XSLKPQ+DhATEAPjibuCqryqJ5gEYvUnx8eR822Tkiem9IJLXWowgNuw1Y1s3Nwjy2zS7J7ie7CmCRAReu5hjnHke2WgC6J4Jcoyo4D4AkmlueHCd3BuYxoyx/Twvc4IKm60Le7xziowku3wM+Rxncphs3ojZ9a8xpvj4LQ9hmNYlr9j4/5ahePuVkV17u0/ug9dm3uex8lXzSG3MDa/8+/xHOz/G9e4Yxy0Z5de/Zpx3Got3jfF15dj87ihv5MX8TfkrL++0YXlP7/DvPfkH+IvuPQDA++YSAPBpfw/PhlN8u3qGratwpg74dLjAd8pn+En3EA4C3ype4Kk9xcEXuFB7/GX7GL9VvsCXwxlWqsWF2uN9c4U2GPyfm9/GN6pLHFyJIShUcsAQFD4sXgIA/uzwDdw3OwxB4aHZ4IfNI3ynfgYfBH7e3cO52UPFp2QICgvZ46Vd4WpY4F9afYIhaLy0axjhsFItKtHjnt7hTw/fxMFTTuh9s8PBF7ix9UjTAI+HxQZGONxTO/x58wGGoPDIbPByWOFfXv0En/b38GV/it+pv8An3UPsXIn3yyssZI8hKPzj3Qd4UOzQeY37Zocv+9NEXdI4g8fFhuhWovf38+4M5+aAlWoxeI2dK/Gw2ODlsMapPuDGLlDJAS5IbGyFc3PAJ4f7uCj2WKkOB19gb0vcK3Z42p3gG9UVnnanqFWPy36Ji2KPl/0KtRrw24un+LPtB/h48QLPhzUaZ/Ct+iXu6y3+ePtbONEtGlegjOtRSouV6qCEj3QuHj86PETvNU5Nk6hZ3itv8GpYopQWX7anOFiDP3r0PQDA/3b1u1jrFg+LLV4OKxjh8KxbjzQWCPiwvoQSHk+7U2jpcDPUWKoeQ5B4XG7wql9BS4ePqxfYugpfdGeoVY+NrfBeeYPP2nM8KLZ41p1AS4e9LVGrAVJ4fHvxHKeqwfcPdE9br3DZL/Cd1XNcDQt8VL3CD/ZPUEqLM3PAxlbY2gpL1UNLh1Ja/OxwgbXu8EF1hR8f7sMHifvlDv/q6sf4QfsePjncw5lpcKJbAOR530Q0ilr12NsSnSfal6Xu4ncFIzzul7s43h6nusHWVXjerXG/2GGhenzWnOPjxQv88PAQp6aBgsfOldjbkjhmyw0+a8/wpNrgZUdyuuoXyRt8v9zDBYFVjOX8vD3DN+orfNqc41uLl3jRr7G3BXqvcK/cYztUsEHiYbmDlg7boULjDGyQ+Jsnn+Npd4qn7Zpog3SHtWlx3+zwF7vHsNHDvdA9VrqHFg4vu1Xykt8vd5Ai4IvmFFp4XBQHPO9WkCLg1DSp71PT4mao8LdPf45SDvjfX34XRaR5OS8avGiJELxQFoV0+LC+xEL1+P72SXqWazXABombvkqRB99evsA/vnkf/+LZp/jh7iF2toSOGty/cPo5ruwCvdfY2QKPyi1+vLufNms+WBC5/U929/Go2sJDJL5aAPjD8x/j/7n+FnZDiQfVDqemwd6W6R78k+uPYL3EvfKAzw+n+Gh1ic8OZ3hS3+BFu4KNmrUPAmdFg1JavOqW+GBxjct+gfvlDp8dzvCg2tF6eIVdjM54VG1xZg74ZH8Pp6ZF5zUOlqI6VqZLHLrXfY0n9QZn5oCfHS6gpcduKPHR8hJDkPj8cIYPl1d42qzRe9ooPDEteq+wG0oU0uFfu/cjfNI8wPN2BS09HlcblNLin9w8wVnRAAA2Q4VCWny8eok/vXof96s9Plq8wveuvoG/df4pvr95nCIuvrV6leihAOBZt8amr7HQ9Pxd9wt8tHyFzut0X98vd/j5/gIrQ9ccrEHrDB7VW1x1C/zuyZfp/eqDwGW/xHVfp+iR9+oNGmcwBImDLWLEh8ZHq1fpHXE91Pho8Qo/2d9HrQa86FZY6Q5aOuyGEu/VG/zTzUP8K/d+isYZvIjv2MYZlNJhqTtc9os0/qftCb5RX0FLj7VqcWNrDEHhR9sHeFJv8O3Fc/zZ9n1cFHvUasBatfi0PceJpvV83q1pHO0KHy6vAABPmzUqZfGkuoGDxE9293FWHKBEwN4WODUtatXj5/sLerZtibXu8GVzkjZ0T8sGUgS8apf4cHmFxhnsbInOapyWDR5XG+qrPcHjaoO/3DzEt9cv8bRd49S02LsC3109wz989dHbqhupvL+4wU93F/BB4F61T9FGtRrw8/05zssDNn2F9xc3uBkq9F6jixvIH65ozJfdAo/qLRpn0EZMiA+W1xiCxKavE+I8R/tIBFz39NvPm9+d1fju6TN8djiDFAHbvsJ7yxtcdQu0TqNQFLmlpcO2r+Ah8Kje4mW7xFnZYDeUt2TaDOMG+tp02NsCS93DBon9UOC8PKD3Om3WF9Lipq9xXh7gA71jnJd4vNygdYbwLEyH667Gri/xcLHFs8M6bTAzNsVp0cIGiV1fEkUWBHFSR8ulihzUUgR0VmPbliiNTRFB3A7xQY8OBH6/toOGidEvKkZcGOUmuBWdVYkTmmQwrrnzAnUxJEopRr7nPlJ0UhYdw7RMRvkE4Mjjsl7S5koYkdnZROKNd86517w5HM+r2F+OMeGcRKFHai6+njeKOQLJWpnwGLgkHJ+0+S0jJsX4fY74ncAS4yb9HNU8R0PnY4wlwZufeZvHKCwnaNgchZF9f3P0nln5L77eZb/UEgLlK/01Le80eM9v/14d/vP/6WNcuwX6oFAIhz4oeEjsfYm1bNAGCo9tg8FSdtg6+oFcqRZDUGQoigGXdpUMKCk8KmFRygGdN7i0S5zqAzpv4IKEEj61AQBXdolS0MtwIXvcuDqdO7gSZg7rBaALGoNXODd7+CDRegMlKGyTDaMbV8PFrS4jiHuw9QYqwwXnsN9KDriJxgF7X091g4Mv0HqTuDjZYDDCYQgKG1slrsyFJE5LBwkFDweJUk6x+jmsmENemcfz4IvE58kvy85rGOHQ+AI6coY6yETV0nuNWvbovIEUHp3XKKVFEzlD17rFZb/EUnfJKFypDlIEbGx11NPJ3JPMq9m4InLquRSOTPOV8TwptE+qG/gg8KJfoZQuyZDmrBJVjZEOtYrgPdFDPngyaj0ECmknHKEeAo0zkf9UTpS5xpkUGs1cnivdQyJgE5VwHyRskMlwLaXDPiqXPA/i3PSJC3RvC2jhUasB+8jJWkiLk7hZsLVlUoaYJodDqm/xV2Kkp2HeT6bpYZm2TqOQDlq6CfdoHobM3v1K2VSf++ydTuNghYrvrz6GUh9sgSpye/ogYINKa81y5x94XmdSylSKXOA6hXTJyOEfaFbiWYk7FvnAspnLiX/gV6aDFh6X3SJFGnC4N/+IkgyG1G5OG8Th03ztynS46WucFqQEuih3IUICKuMxFBGgjAuHSDfWJC5Tvh4A1kWLbV+laA2eg4xt30ReVyMdccvqYRKazoXnL0Hh+qWy6R3J9V0cYx4ir6RHY82EU5Xlw/ceh6GbbG4hiKR4dlajjCBbPlOgQibHHKSNFVkAaAYz4bgFgKXpsRsKGOlRKovdUGBhBhwGk+ZaxutzPtM8EmPwEmVUcn30+BeR25UB0zgChXk+qxiuzkobc61y1EARFUUAt47zuFwcG4fM53RPzKXaDhrryLE6zOifUjRIBFbLeVr5/eqDQG8VjPLjnOJ9JgD0rNR6SSHTgsKeTQq9F0nRBpAA34CxHgAMGQo5c6kC0UEcnyMGgONImBDvGQ575+ea0wHyyJgyA6n7OsUoAsfjaJYU/SJCAp7zXkwA4ziig8PfWWlnpdr7MQrH+1Erz+uwocD9eS9QFDaF1nOfuUHBha/XmkDTOKqGjQm+PvhRYedIlDwiJin3mWLPkTE8DwCJtxWgKA+OxJFyRFCnk9GLLUbe1XkUCpeJIeKmIfR5xMKcWmoy3nkERRBjJEGGo3DLaMldnvmpufqce9pE9p/Dfo+5JF9Xjnkf8+Ov+5+PjyMTQgbOxKfmY7mrrSPtvXH46RvO941txVwub2nC/PQ/+I9+5QA3p+p++IPlv/VLb/fvb//bX/ncgHfcY3lpl/jvX/4dbG2ZFBwAONgi5hwOSem2QaKQFofITVYpiz7DjmYFjI2JUtmk7G26KuXZ5YpkGXf/D5F4ns9xniArPzmfY04074LAyvQp9DZXPo10OAw0Vs7L45DavHD+lophtT6MOXwLM2BwKn3Oz3NprR49oDFXMP/xYkRbVohs3A3kHTAO/eVdwvyHgJWb3ipoNeYCctguH+e8upw2REqfEGJpJ24kldfS49CbpGDlhUN5efzdoJNMk2IbcwQBUlBCEPhxfY92fdsSAiOybr57yHIxyqV8SgATeaVdxNgGk8+zDPPdxvH6cQev0DbR1nDbeeivUS6h0fIacD4az5sVBxVz74SgHcq6GNANenJsJH4fdyV5LbgOo8Lm32nXk9aKdzi5b2MoL5DzHxPaqUBEw1Up94/nz/XyvEEeFytDWrtEcM85fT4qEjoiYFIYc6SaiQilzo4hyfzXc65Z/LHn45bRLLM27aBSnRTyKzBRnJi6BgC6drw3GUk26RiCaHiECBN+RFbYmKoHALRx6HtNCLqDmjz7xrioyImECssKJjCGeDNVEK8p/3Izcm+IymiuUCkViOOSFUEroYyDi3mWvG5ccsVUqTGfOIWTx3rMYygV0RQ5O8ZOcj4ggzxxOLSMyMTpXokotgGAtxJS+5Q3yW0Hj5SfWFSEJMyosNye7RWFa/O4BMnbdhpC+YSGq4yHG2RSjBjRNI3biSkqbMyRTHPitbVZ6HK8ZwWjlJox1F4IwFsxhoQH0ByjQjs5znQ9sS9pPHxPMmVEVQ4Zl5rO3VQuURvla04h7JIUduNTOLsQEXgrKvmBc1aNh+tjaHrMzQtZGHRq3xI6KwBCZhUY0Wp7NSKSBkQZgsK8VZjmKqabLda3khBWPckz5XzGtmFloiTaMyBfDMXdRbTYr120T9yaEGEMUZU03xEw0I9h36wIc/g3562y0ROANkeJzeebe2ny4oGOkWfj9xTOnive0bARQVBYO8s1IIVNAxhzVOO5FDIrAxkfMSw559nk0G/LOadxrZwe63E7cBH4zYrxXca/OSzDDJU2AfaA+4offQ6kg8n1KWR2bsfkiK/cLxtbk5DfO64PuE0NMrNrQ/ZhYo/F/N7cDhJz421ecsNQYNrgXQbkvL38ByPN9Y7+jrXB//Nrbhmq8fARW/x2+29oTGN8Nc2/f10n5W/KX215pw1LGySetWssdA/rFa66BQCg1gMWuserdpl2vms94Kqt0+7wZbPAwhDHYTMYrMsO276k3WEvsWkrdJY8VfdXe9y0FYxyiSpEgLgwQxA4rVvse6KA6AaNVdnjxZ54EetiQBNpQKSg3EXvaXe2UA6f35wmg8LGnXeimFBY1l0ylLaWAELynEgfgI3NeCwrojvYRjCdV9craONgtMOrmyXKkoyym0EnQ2JZd2isITlFYBb2/EgRsGvKSd4g7ZAWIyCI8hgipUGiGXAyGR7WKlJiU59kXOwbAhvZHUoUBRkbZTngcKii0lvgZkOclttdDaUdlPLY7mp4J1AvOwyDmhg6rGTnoRVFOYDpM9h42+8rSOWAIGAKCykDnn5xDgSgOu3grMR2UEmB1XEnneW+7wjOk88z7QUbC1LTbq1vSXFiugCmHZCFS/+BbPPPC+zaBQFt1JxjScpwuy0p169XkBUZmUwpkIA6osKqSgdvJYKVUCUZIMFK7PdrhMITj6aVGagGIDjcJiqkDB6SK6gpt4+VE86TLDwd94AoPdq2JioABvnQI7gKBgEUHh3n7Pl4PVNMsMJmx7b7toKoHPq2ItTmqMwOvQI05ZgOvRyVudjE0ESuTJMpmxHROFRTHsu0o8y5dyJgYKCWRBEgbyuuUXmG8XDPCBXWn9hx9zueSwqkB2wca6gcbu2ks2wB2FYhLC3CroKv3STHKhwqUggFKW5ukJP2fMz3DKVPgDMhyyvsNzXc2lE+30D8ngz5H1pJ5xCNnMIjtCVC6RE6SVyi2Y9+iIi8wQSgJSTqMAj40idkbQgkwBP0EsGBEKkZ4CQiQDItRPCxficpL6sM6V4Kkb8RZQBagVCMyn1gcJEox7ApgRJAEeXaxYEvPCEJA5Q36AG0AljScdEJYBkg9gKiCmm+kq+PSprXcZw+Ig7rMPJjxlwyaQFXhjGXUpKCTpybIO7FXImKbXL+oewxooey7SZG9Fm6BwAVUbR5XMIh5f2pTsAuAsy2INRWE0bE0UC5eV6HyL9ISN2qjYZxQkCO3KGdIJTZivpI6LOxzyCQeDK9obxOBEyReUHcpJLvHQ2IIY4345wUFoRUzhtAcc1cEaAPOnFAQtD1knkuDct9RGdlY0S1wLDC1y6qj6iwQOLhBGhurhzRXieIyPFVojok1Fyux2ubowkD0fbJ1ijosS9uUx9IXny/8Vyn6KcYUWF7JMRcH+XG6+eL2XVxDaQdxyV7jOA7AWBeyYQMzXyZ/Thn5rHk632B0SjhPP2I0JxQYcXUcExclQFpzTkQjO7zEGXCDwgLaZQNGcl83cj3yIijEy7IECZGZBBj7nFqV9z+nHJFWT7xPTG5DjhuHCKrI2b1ciM2M7LSf85flZgYj/k4kkwzg/CuMRzr46jResyIxXEDMIhwHDjoNYbwtM0MBRfjmr9V+TUyTMNf41DYd9qwdF5i01XY9SWGyBcYgsBBkaHUx7wG5yle3zqZPEGDUxjiDj+hcdL1OQKktQpOerzcLZNHMfeSsffEBZFCQFxsh8/1Vt8ClAkxpOggCL0RAPHzxXocutF09IvBO/a9GGPZqZ2xvvcSh7ZIQC+DJN7HoSeOQu8Uug7oBVJYCkTAvimTF8bHGHxqPCqoaV502CePEZ3PeQWDF+SFAFFQ8W55fl4I8gCFQHW9E2i9pGsd/ec+vBVomwLeRWNbkKKLQMcRYj+xvwSqEscfAHTe0As1A+wJTsJHcJMUlhOv61udDCoX12GY8fYxqEvytEqZdkKDIyM2ydADro+/xDYQMEpQBDrS5WzOXD+2w8aAAHy8LngygDx729iQTj8+YpQ9g7CAeURE3LGN1/OuNv/gONaA6Ic2RDnz91TYAAvj/zDItGseegKmCf0oEwyxDb5miOeGuE4MyCEwDUcKs/YC6FoRxro5CBEbf/HeS/Nm/gmM64wcUZOP59dDjMAibGD67HwKbaI1xiBHKqAhjjW2EXINKVcSnIhhSGJch+hZSHMYaH2FlZMQpvQ6CUjjFDb2y7IGiKbIZ5/5sgR+ItL5dO+pzAjyAmKQJM4hehDmYDPsWeCoecfjEWl8ISr+SQagdkS8J1JuTjQOU3t8nPkAIy1Bal9g9LTEdREsoBAVahGSkQUWkRtlzueCHMf8OkUtlaiYp/lHWcw35lmpnXgKQvaq0pieAxJIDdjQyF9t3BX/esdbyJswUR7B3l8gGqpZHUHj5/l5E5KB4w09X66gnoQW4zOX+hFpY8OpCHoiAxDpJhLCqBjHycd8IZIsvck0V0PvbK8ChIh9ynjvgPr3mcbiI80MjYv69myExVccGdHxO78GKkwRbN+yeDOuF9NWASRr2kAhuTqM/fA94Qsk8CR+hbCcHBuIuUHF9xhw676CIOOejZYggFCN5yYevXitKzFStHC72fUTz2CsM2lfZu2F7LmZGdB5Pc/gPWLaXjJW8jFkm2e5wSEywB8eRzqmM+HMZcTty2ydkpGXV47vEsWUK9kzna/rvH28gfGVH/uqks/9TqNvpOOY/7+zP54D17vDwHqtZ/ANjLK38ix+hSF5u76YyeVNhfqb8ldd3mnDEmIM3+TEaE6ctoETrAnp1HkxQTTlsFkXcy+G6IXkXBYpAkykfLBZiOYxxNEcJVVHRE4KDxtRGvmp4TwDNgiNcSnUkK8BMKJyhvwcYhtx+gJj2Jy2yUBkJFBT2GR8quR1G8/T2MfQqpQXIcY3uMxRRwF64Uo/vreCGMOZ1DhHEQ10Kem/ACY5ERwiJxX9Z6RRkeV1yEiyLPW4DZhC0fglk8mEvapTedPHSd+pPfrzXkBFDyGTs4tEiDluJNCYidic55kMBQ5FzfM+VJ5bG1/mKvvvZ29gEYDSJ2V1Unjexo95KGqc42TOMexHJDRUQZ49Bitlz16SY7jjF9OPnjRunrnUsv6AOH9+6etsjtn00zW5HOa/Jiy/pAVkxzJ0WWDW/hyyLwChdNmxkCkIsx/WO37Ek6HI5+TYFHLvQfwcIvdoTs/ASJh5XyFb4xQomq1BOqbpWPJEZoXayNrWMznmO+Tz3XIAbsljpfNBjPJxZVZXBPJKlHQ+6HCrLUQ0yXQ+E2/IxsUGs4/3nggCnsPe0tyz5gP160UYDbMA4oVF9vk1SpArxsFQm+N9GvSsehnnKgBU9Nmxp5OL4fWK3/2R5ycL5cv3COYKlzfh6KM3V8qOKmrpecq+VzguCx6vBzyv7fx+zz8X0zFDjDLkcVgRMoM7RxPNFz+3DLKJ5m6dNPixnsjDIW95JcbF9GWYXDoRUj6/u/7/AiUf+VyMt0r+3L3u/LFxzY5NxBiIj/VOJf6r5vi6fu/o73g72Q16h7Kfn/4qb9mt8R3r+y3n/IsaTG89jje9/nXlzrri7v9vdH+LN1vX147haxiPb1K+xnP5VjL9tSkB+Doe13ekvNOGpUBIBqJkz1oQSY+iXB96elRUCl00xtjbpKPyL4FbXsXUT8xlyzkD85y9IMJU91Yjx1v+n2uwcclFSkCI0dgcHUQjelduTN51P8q5EyYIjEn6uOXtzJG8Eip6rqxnRhWDCGDmtYXIKA4CsrpjPuld+0qC25u94YIcDetbhhfiu3P+NuEmJsrWLME/Oz65JjNCpbpdXyBk8s8U5VsaIF7/Umd5I94Td+2aB2RG1O0253O6NWc1Km4Tjr0j1wdk9B3Hxn0MIEHgiCItbteZdJrPbfZ/Xo4pvHl7udE5qRNmyuP8/uExiOm88k2K+bh5nMfmcuf85gI/cp/MFeGj9TKj5di5udVyrIg7PrOMjng1bn0+tl6vk8EbGOpzEIljgBKc2yWOrc/cUMhLfszPrptfm5c8lygBehyZJ28o3VVma3HXEvL3iafjyHyOko8jGz9fM88Ry9eKvdazvZaJkp+3K+bnpxMQbFCzt/zIvKZGRP7jI6bznKyXmLTx2jKTy51Kv599P3bPvE3J5JrWMG/3yDq/sRL/BvOevNLyNX9N/ddtcEzyDifjub0Wd44/tXFkLLm85vdy/gzMy/yePFLextC7a+xvapj/UgypN2njSHkr4+kNIyzZf/Da8lX36xv08dbla8roa/f3qygBgH9XBvv25Z02LJUIWJsW+6GEj8iEAKF12ghYw5DQNiLj9THkb2EIZCWhxzkFJV0iZlfSp7DYzmqUpo9AKWNIJdcjvkgyDHVEjDMRxGKIADUjEWO3wY0AACAASURBVDsSP6SLYDT58TS3CHpzC955prQwpLUUQB/DUBWDC0XwGOcldOSSZKOY58AhvsnLGOVFnyPqHcbfTBe9t3yM0fh8dv3o3cUErprzIJmLkYFZ+Dx99wkSO8/XZFAWrUeQGD6fj39u7DGgSY6CR15lDvGlcNKqpsSfrovAOFnbfuY9ZFCXHHwmJ3VPEN+KAWjE5LyPeYsM7pF7RKUiDzEDziCQvDgnVGkPO0Sgp3g+vychAjyHbWceeikINMbHkGOB0TBmvkog0/XY6I/rjPw+TB5on3JaWfb5+qfrecNE0vecCxNA4qzMB5BzZTJ4iIxgPYjryEYzf058oOy515wnSoYJA5FAEDcmdU59srHNIdMEvBPvGQ7tZP5NHmcYxxqcTN5018tkrENk18djBIoCyn3MNdOA0RsNQCifAGr8LHRXmhF1EV6kuohrlQBSItgKz437kybmXjLYTKZ9igj0kuZmBYQe+UmTDFh+vBkT2wpxfVP9NM54TQyhzAFVklItMXoNvUCQntpgzkIGaQFG7s18PLyREj2conAUus51eK45DyiPTwcKtZWx3iBG7k5+QDIy+nTt3ICbgQIFSW1MHHS8ucFzzkuKkcz6TEaSGNcqyR3jd5edzy0KGSiE2eTyyeqxbAPGNZHZXLkPBneZb2bkwCg8R+AWOEwQIc03fRcYvZRyPM6bD4mPk3+EBAAHCu/lMbEcs42BFNKdc4gGTEO9v06RGO8nlh2yOeXyi2udjO24lgyKk84FPiduiTa1nW+SZO2J7PkKnKOY35Nx/UTAlE81PfZxDWbuVNrYyQ9M+56MJSkE2f2W3+ecg6wyntdJXxg3WLL+8kcm73KyCTSTyTHDJN3q8+uyunfxVE6un4zvDe8hka3zvLDs5t/f1ubIrzsmtGwob9TG6+rg7npf2f6k8lf09ZvyzpZ32rCUwhPEfwaV74NIiKwAkiHGBlUho8LEhh4EJAJKfRt+nWkHjHKzfsMEzj1xHMb/Jno2AcAql7yqORUHQ9ErERJYDlNmcFHFqHHnzyDD1QsREoS9wOh9BYAiUGgwobDKiCAa0rhYBlpNt7fmKKsijo/79GHkd+INFwYlokG7aDj7Uc5KTIyYdDzm47B8naI1kNGoZ5lyiDOHO7MRo+K5fN3yNQLGcOXJnAAiNRcBWo9Q9VIEhMIlbzJfO5cJhzLn3uSce4mPsZGZe5KFAEI0Hn1mWPKc+Hrug86PkPZCBGiDW/Kcjs/G/wEhjB5lmpeEyDSIOZx9WpswXbNkbMlRkREiIIhxnYFoPIow9YrmP96CDMWEqioCgsgQaee/4DJ+lmw8TmVGY55uKARJz5aUHhACXorR4cX19PS+SL/Fko3pEI1vjJ42llfqaFQuhcxoBwq+b6bX5xD7AmTA3b5v8ueTDMAxLHwmRw51jka1zOfEfRk/roXPZTYa3uNahfRPaOqPgJtE+s8G+q0ikMKhhQiTeikKQo7ygohDTFp1bqRka8Fj4zAUmXGmxeclHNP6UthyPM7Xp/XPhRk1WBHoF5GHosM4r7lcUz9hGtKeg4fk+Wjpcxwrh+Rm90QaL4XPTPtJbYZZfYxj5/GEI99ZJkkxPNJO6odf8Gx8ZsdyueYlN/6ykkKs+TOQjIIw75vPzYuY/Y/jSOHRdxoKo8E1OX7EsHmrwvINt+c8EUvmfU23Z2Z8i9wQF5gY0Hcq6GH6WeSbHhiNyjS8MGsvA/RJbczbxazOZHKzZzVMRZH6yWWcfU6yn9/eDMSDma0VxuOYHbvT28kNTO7p2zJIfc9eBa8L532jdbl95d1jO3btfDxvWo5sPNwqX9Xe157jmxcxW7+3Kr+kMfxalHDsZffXo7zThiUAtM5gqXsMXiUC4ZXuUCqLZ806GZ2VGhInnA2KKESURalsIv696etEOdJZTVxnyuHRYofLdoFSW+Kw9MTXtesLhCBwUR9wGAg4pxkMzqoG1y1xSq7LDs1gJu+43ilUmvp+vltBSo8qei4bS6A7vVVY1x0ZUgCaSPBbRPoPfjqbgVBhu0FjXXdgZNrSWLzaLFEWFqWxuNnXqMseUgUcepOoAE7qFu2giSB40KgLQtgYnIRRPtVlQ6cyFtbJRNNhlMOhNyjicebuYlqObjBYlD2hwsZ2mEesNAOarkBVEC1KZSya3qAuBvRWoWkNlnWPQ1vAGDLQd4cS3kssF13iDLOZVzPRUcQf6bIaEuIt02u0nYaOEP5VMUBKj6tXa4QArM8PsFYlpFvvBYwZEVq9FzgcCBKQKR/YyyoEgTGpyOllWwOhPJRxievLDgqmsLCDhjY2Q7QlD+/QGMAJqEU0DgV5GrvGQKgA12iYxUCbFFYSdUPcQGAQHFNaeCfRHzR0RevprILfGaDwMPUAZ2UCIYIMZJRkHkKeKxse7GH1Q+ZBix4jVRD1ALyALB38QUMubEIjFYqQbb0TCL2kOr0iGgIvCCU3gij56EkIVgICdK5TdM1OQxQujgewHSHwQgC+02DvGRsy7lAAxkNGZFxv5YgMW9s0tmBlAisSxo8GYURWFTFXMwwZdcIgEyJtOr7VpDie96NXb5Dpeh8VS9Eq8i4sXLZbFJVkpigQoHpLi7DVVDfz8oqdovzFaPiIQSLUbszv7CR5fBYW4RBf85FGAgDkRsGdOMqPHCTEIBISqWwE/JpQc4MVlPfbKITSQ3SSkHbZSBEgEB4vEEw8rwO1V4QEsBNkGPOQe0Kh9XVEZpWYerXYq6E90Mex1T55v0QTIwVqD9HIMWc1gPqL7QkvYK4lbB0IOdYDspOE3rnKUWHJ+NF7Cbum46oVsCsPvaPreezqIJFC+kIEutEgj5AHggZkG+cc0WIZFVZa8loGFQAJyE7AmwDdRG8m6+0a8Hr0Fskuzgkgj1QgeSruR2BEhS3pOm+oP4DOyR6wi4DiRiJIwBcRFTaiawpH6++PoMLKgUGQAFsTKqxqqT1pBYSluTOaa0Kd9TQ/bseVtK6qo3naZY4KS2i6whPyrBxEGpvjXEqBhArriwB14LUmechBJMRZX9C4XQWohgB3GIxJN8Bwgq9dZEftsmySMefpuByQEFATmm/8r1r67DUhtIaI3xYUnYPIAIECCImU8cLiY5zvQZg9YGs65hWg2whWFJ9PMmaR8olVRyBBPMYgaRwAHc9RYWWGzAvBcxJpPsJT297EuYgR2Eg3SIi4jArrdew/IuoKn/XFqLARWTiXGUBjSHNRSAjELHfhQrxGTGlBWFbRk5vOBdxGhVUCwoZY9ytQYfOSWcJp7GIcwy1U2IDp+/Muwxjx/9wQzubBnt67wMbmbR6TzaTtfBMkH8t8I4B/iuZG+7zkiu+x40fKLTAi4Nb3o+3/dTI63/HyThuW5/qAf//JP8D3mo8wBIXvVl+i9Qb/tH2CL7pT/NHj7+GlXeOJucJfNO/hn198jn9yeB8A8DcXn+LT4QI3doEnxTX+0fZD/Ov3foiftvdxqhs8Mdf4VvEcl26F//nl7+MP3vsJbmyNnStxolv4IPBR9RI+CPy/24/wqNxgCAoflq/wvc038Xcf/TkcBH50eIQHxRYqPs0uSCxUh6fdKZ51J/i33/9jtN7gs/4CF3qPlWr///beNNiS5Lrv+53Mqrp1t/der9Mz0wPMggEwAAVQgEhApmlToAhAEkPQB9qmxAjRFkNb2DQlS3aQZpiyP5k2FZbpCFkKiZIgiQqSEkRLsII2BEC0GCGKg4XEPhxiNszWe7/lvrvUkpn+kFl1696+3dM9vaFn8hfx4t3aMrNOZVVl1jn5TzKpOZEc8LnpYxzWPQyK+7N99us+54ot+rpCiaV2mtO9XXJVcTq7xOcPH2VuMh7KL/NKscMPPv4lvlE8yHPzE3xo+yk+d/goe/WAR/oXOJFM2DMDPrf3CCfzCaVNON3b5dnZCSqn2EoKpibjLf3LbOs5s6Bp/q35MY5mU46nE2amx0Gd82j/As/NT3Aym3C5HpKrCoXjXLnFiWzC1w/u51Q+YSedMTcpF8oRp/M9Xpgd4x2jc7w4P8owKTi72OKh/i7PT48xSgs+sP08/37vMd41OsPLxREOqpzfN36F+9NdPrP7Lo5mM6Z1j74umZuMYVKwncxJxTBQJZXTfHXyILVTHM183gAP9y9xrtwiUzUvHB5jUvX4H77n1wD4xbMf5Eg255H+RV4tdujrkuenx8iUobSaTBneMTpHKobn58dRYtktBxzJZpQ24eH+JV5eHKGnat4zeol90+eZ2Um2kgWXyyFv6V/m2dlx3tLf5YXZMXqqZlL3yHVFTxm+c/wix/Qhv3X4mA/Dtgnn5lu8d+dlLpUjvmP4Mk8ePEpfV9yXHXCu3OKgyhkmBX1dMVAl3zi4n61szjuH5/jSwWkATuUHfGTnq3xt/hC/c/AQp/IDtpM5AId1j8vVEIUPLZ9UOVOTMdRlu1zYhFQZHsz3uFCOOJZO2U7mTEzOM9MTvHVwmbFe8HvTk3zX1rd4cv8R7s/3ATioc/arPn1d8XD/Et+cnuThwSXOLLZJlOHMfJtBUlIazYn8EIBj6ZTKaZ6fHuPdW2f46v4DvG/nJZ6dnWBuUmZ1xn35hP0qp7aKB/oH9HXJ5XLIfpWjxPHBned5YXGMF6bHSMSwnS04mk55sLfHF/bfSu0UC5Oyk83YSecocZxd+HpR2oSH+rsAPHt4nEwbTvYmvDg9SqZrjvemXCyGlCbhgcE+Z+djPnri6+RS8stnvptB4ltrJ/NDnpsc81MFiWWQlLxrfIZtPeff7z3aetvH6YLSJlwqhoySAovw/u1v8dnz7+TDJ5/ic/sPc1DmZEFU6vuOP82ZcofaKg7qPqfzXX5n7yEyXbMwKe8Yn0OJ48u7D/LW0eU27UFSYp3wQye/wK9eeD97RZ+HhnucyCZcqob0VM0Hxs/xf198L6XRPNA/4OmDk7xn5xW+eXiSR0cXeXF6hNL6+W9Lo7l/cEBfV7w82+Hx0XkulCPu60149vA4bxnutvWn+fD36Ogix9NDvjZ5gO103l7P2mqO9GZMqh7jtODcfMyjo0v+GTK5n0wZDqqcJ7bOYp3w1MEp3r19hmcPjzOrMzJlONabMjcpB1VOpmp++NTn+OL0EZ6fHiPXFY8NLjLWC/6/S2/nvnwCwKViQKIsH9x5ns9ceCcPDvZ5z+hlPn3xCT50/Hf5zd3HKE1CaTXfufMyBsXx1B/7/PwEF4sh43D/XSyHvGN0joVNOV+MScTySP8iXzo4zfHeFOOESZVzUOU8PLzM+WLEB3eeZ2FTKqfRYnlpcYRz8y0/jZZTPDE+y37dp7aag7pHKpapyXjP+BUswsKmXK6GfOfoRb44eSsjXfDS/Agne/5e2qv6PD48z5OXH+Y/vf8LXK5HvLA4xlay4KDO6euK4+mEs8U2ryx2+I7xqzw7O8HvG72CEssD6S7nqh1mNuOL+2/hbcMLfPfoOf7N/hOcyg64L91nrOc8NX+Qo4m/b5+Zn2QYyvHE6CwAL8yPMU4WvGNwFo3lN3bfzoP9PVIxXCjHnM53GeiCbxw+sPKceWZ6gkQsszrjgcE+CscL06O8/8iL7FYDLpdDZnXKqf6EJwZnAHhqdj9vH5zlt/Ye5f3b3+KZ2UlO9Q7YrQb8x1tP8yvnv+t1tzueGJ/ld/YewjrhwcEeI1349kg64wu7b+X0YI+L5ZAnRme5UI4pbMJ+lWOd8L7tlzg0PV6cH+Wx4QUO6x571YBJ3eM9W69QOc3ZYotULIkyHElmHIae2LlijBZHEtoS+1XOHz72FF86fAuJGC4WI941PsOrxQ4Hlb+uqTJkqub8YgzA46PzPDs9wf35PnvVgGmd8dBgF43ludlxDsqcQVKixHGyd8jFcsjxbEphE3bLPm8dXObQ9JiblL6uGOmCM4ttHuzvUTnNy7Mdaqt519YZDuo+lVOc6h3w6mKHc4sxj4/O8/TkPgAyVTNIKmqneDDfo7Ap54sR46SgcoqDst/ONb6VLchUTU8Zdss+F+cjRlmBEscgKenrCuOEw6rnnwXaoHAk4Xm5WwzoJxXTKqOfVCF6rcY6xbTOSJXhoMjZ6i04LHv0k2pl/vHKao7lU3YXg3Zd7ZbDTJohRcO0pLaKwiT0dO0dHWnBbjHw85yLnwO8meLNhCFasIyq02IpzHL2gn7q21O180OZerpu0y9MQqIs8ypllBUs6pQ8qaisbqPMgHau8WmZ0UvqNkKrm287v7ZV7bzpibIUYbaEbmRY47ApjW6jyhra4VZh2YV1phNV2BzTLWNDM6RmfV7tRC+Hqplmnu3OOTTXYl0DZX0ffvHa9/edwAHuDTzGUtxGxY57g1PvPur+5C/+AAZFaZP2ZmoeJmX4xNeEmBYmWalk3XDZTBlqp1Zu4IZuZW1uttoqbPtgse1vi5CIoXYahWs7gE1YbZMOgMJRtmW2K2G7LoR9duke2/xuxpA2Ib0iDtM5jyakd/33pnNbDyftlgVYCYtt8tVB2KibDtDayXbOozmmm3e94dgm3+a6dT/gmavYZlN518vc0D2fphxVGHt7xUNog402hYw2/42VlVBhX1a3MrZ203HLsNdlHek+WNsHqlsNl95kt2a5u28Teg2r4UbrHxSbsbJXo+u9Xh9b2Y4htbIy1nQlTHXtfDexHjq7Lg7VXd9Nd7nNrZRvfd2mMq0f23hsX0vwqhtC3JRHNyGqnfPt/m5CoOXqJvD7N2MVQ0jiSllWwugczipErX5i92NPNw8c6taT9WvhNtSBbn3Z+PF503Xtxsg1y23ZVq/t+vZlWTZk2qm0q6G8nW3NoumMpexsF3VlHWzGhOJCumE8aTtelo7dO5/1rxBUW48LbLzIV7NN18PQnGv35uymvckOTVm6Cs7r93BTBiOraax/6W88JMLqmMmGJvR30zVZf6A065pq2XhpXsuL0UmnEXFqtrfqwdJRCu4e383Ldv53ytN4Z183Tbrdc2lYuW5XXm/ZdK275V/nWjceSy9Vu9yGGS+zXTm2vSayWkeaOrSh/N2qulKt1q9X44mDK0OaO8e77vW9WlrruM3F3XiPXIUrvXFXGnRln/U0rzOPK1i3/2uxfs3XjrniPLhyeWOabE7vWmzK63rTu2Z5umlcLzdxy3bL8vW//t980Tn3B15/ajfPlhx1H0w+fMvT/XT9K3f93OAe91hOqh6//srjjHre23G46CHi6GcVw6zk/MGIJIRK5lnFZJoz7JeIOCaHfdKsJkkMi0XKznjO/mGfYb+grBPKIqGap0hiOXb0kP3DHK29oExd+5DP+dR/SRxvzZkvUpRylIuU0dac6cTHyvSHJcUibcVQwDdgksyQZTWHlwegnJ/I3sgyxLBW6HHVirDURYJKbRtm2YxRqyvtwwZLhd4qfWRaodGZxe5muNygcoOdpMigRiUOM0takYd0qwxCMWAXGj2o/XsnNLLMIvGiD6FhpAe1D3tsxC+0g4X24XKVeHGIsL+kFlcpf0yhfRiiOCSzuEKj8ho79eVyhc/bzHW7vxwmsF3hDhPILJJZmKQ+NG6nwhXKj4MKk6rjpA3La15sblQvwxqbcLuF9uF8VlDDyivBvuhDl+uTlZ+vrwwTwVuB3CwbZA5k6m+bdhqI1LZz+/lQQZ9PcqCxqfMhf7XgUodaKGweQgZ7YdxcM/WIhWSiEQP1uIkRApda1FTjtEPPFPWW8SGLIZTOpT7ksBlvY4cGqRRqLpiRT0ctFNmeos4d9Y5BzVU7rsUp2jI3oYiNwIXUq40wvVA+jK7Gh/YpMH2LWvj0zMCSThTV2PrQNuPD6lwSwuDmQj206JnC9sLcernzOvwhPNBJCHkTn7aeKszQhyWang9VInGoELLXnJ8Yfx5NKFl6INS5TwPnw7hU7e1WjWwIqXJIKb6sNOF6hInUmzI4sKBLacPydCnt3H+qEGzP0bvs4xUXJ2xrW10I1cC14VeqFpKZv9bVeBnuKKEq2GwZMplOhPKIJdtV1COHSZfpZPuqzd9pSBZCNbatKEgy9S28amzbUMQmHBCgf1kojjofqjj3599Mf5FOhOKYn0JCFz5kMZ34kNB0LpjcLdtkYR9xPpQunXlbqEIwfR8G6evTMjQumfvwyXrkwx+dcu1cjU05VB3Sm/vwxnrYzMPoy+eUD8VMpqE8ITROVX6bTf11zC8K9cBfQzFCMvf2K7d9SKe3SwibnUBxxJc5mfnfvV2hGvuyebvKauhjb2nXpnzJzNdlG6ah0YWfFqI7CbxLfOijTSA9ZNmZZDVNJ5DOaCeztwltaF0y69y/YbkeLMulKp+/S3w4aLUF+UUf8md6y06JuBDumgabT6HOl+nr0t9TTnlbpFPQc0c1ElQFqvbXq84FVbsQaittOZKZv2dMLoiBZOGXy61lWKxLQJUOZaAaSMhTUJXD9P1zrQ3bdD7kNJ34fVxCCCV1bTip6Xm7VyMhPXStPZyG9NCxOLYppvH6SKb+3H1oq6+X4rw9q5GgC1+uZO7a8FWb+Hdseujrusl8+Zpr57S3q+vUGwBd+WsvLoSZstzmNPT2HdVAWhsmc4dJZSnigy9XE6KqFyHvEmzqnzO6BHGOuifLEFOWIdC68vXQJv7aObVMX4zDZP56AZgwR2k6d8tQ2Nqvb46v8/CubEJhBXQR6k0iKONCeLC0nRpVL5993n7SrluGwko7R2fbEQ226oZCt9fLujaMvLmnVL0Mx5WuR0l8frZbbbpflMJvFe7ZlXyMrw+wLFN3ns/u7DtX3d7pnIoJYbvGXwtxDqube0/aEF+fZmPrznUzbA4rZZlP11bNMZv264YWL+2yXG7CWtfXN9va5F4j9NVJ57yu0uHuluua+7zJEZGPAj+PVx74Befcz97qPO7pjuUwKfn+07/Ht2ZHqa3i1IkJc5NyqRiyt+jz3lOvMqszRmnBxcWQdx8/y5nZNgrHe0++ym7hQ0G2j805PxvzwKlz7Jc5ibIc6c04EkItf2/3BG8/dYHCJF4hNqmxTjh2nw/xeWmyw8nxIdYJ29mcc7Mxbzt6EesUF+dDRjt7K94uPz6zx6xKefc7n8M6YbcY0NM1mapJlGWcFLw4PUJRey/rKCuY1ymzKl3xWG31FiTKMkoKXp1uU1vVhnM8/tgFLhQjLs8HPPzoZc7MtljUCUcenJPritImnJls0U8rKqvYygp2w9jQJnTiyH3z9reIY2/Rp59W9EO4RVEn7ZjSQVqyqFNSbbBOmFUpg7Ti8nTA4GhJntQUtbfhqFewP8859sCM/UVOpg2TRY/t4wv25jn5ds3JRw559WCLk6cP2S9yiirh5OlDRmnBt/aP0k8rFnXSli/VhiyIJekwfcuF6RBjFf3MKwRbBzv9BbPKh6IczHOqSvO+7/1dLMI3LtxHLzHs9OdMih6JskwWvRVv4fFHpgDszfsrY1pro9juL5iWGVpZ7hscMq9Tdhd9Mm1WznvUKzks/Ljc2mivQqwcJ4aHDJKSVw63AaiNZl6mHH94yqJOOD6Y8urBFom2DNKKWZVSG0WiLak2aHFcng5ItWFnMOfS1IfuDLKKt+1c5MJ8xJnJmGGvpBdCcAqjKaokjIu1VEYt7Zl4leOmjKNeybxK6SU1vaSmMpq9ec5WXpBqw96sz8nRIecPRwx6Phy0rBOKWpMoy3Z/weXpgHFeMCtTtHLMirQdm9tLK5RAFu6xw0WPI4M5l6cDjo+mHCx6Xs251vR7JUWV+rGxnfHFZe0fa/dvHzApehzMckQcWWLIs4pRVnL2wIeGGaNIE3+eiTZMi8x/WHHCVn/hr/O0j1L+g9V0kSEC/axiUSVYKwx6FbMi5aH37pEoyzPnj7fCT3lWMZ33WvEkrS1HB/6eOnswbr18WVJjraKoEpLE4JxwcnzIK5e3Of2ePc7sb7VqyQAPHtlnVqWYMB57kFVcmgxbNeOdke8VXD4YMhossFaxKNO2XI8du8gzl45TVZpBXjLole1Y61OjCc9dOoa1Qr9XMpnmbG/N2J/2GfULZkXaUUQW8p4PG5sXGeOBH7M97JXsT/sM85Ky9lEZVeVbZaNBQT+tuHQ48PeNVRijsFaRpbVX0k4MRZEyGCzopzW74RpUlWY8mmOdcDDN2RrNOZjlrXp0Gsaql6Uff/3IiQucnY6ZzHso5djuL0i14czuFr2eH6tchrp/bHvCK5e2SfOK46Mpr1ze5tiRA87tj1tV6vH21F8vbVDi2J/n7TVLlGVRpgwHfqjEvPTPmHFecPlw0M6LXBlNVWn64SPmkdHUj1kPz/TDRY86lB8nDIbzViG8rBOUstS1Zjya4fDRDEWVcHw05fxkRKIN1aJHllXtPTEazrmwO+bEyUvM65TDImvvba0c/bRiWqZURcZgNONgnjMezNt3z2Hp77tLkyGDwYKTQ/9s7mcVg7Sip2suzQfkSY2xiv15jtaGxaLHYDj3H3PnOak2jAc+BP/c/pi0V6IEFlVC3ivJtPH1Ivf1MU0Ms1mvFTvr9SoEWMwz8q0ZdZVQ1Zq6VuR5xai5Z+c5g7xgcjBkuDVlsuiRpzVFlXDf9j7Pnj/+utsd26M584OBr/v9shXn66U1+wcD8ryiLBO2hguqKsEY1aqTHxnPqIxivugx6heY8KytKs2R8ax9Boj4UEKV1u29tijTFYG3qtKMjxwwmQ4QcW2eizKlrnU7B7VWlqL0z8Rhv2C26JEHHQNjFP2eryezRUZdJV6BHMgyQ1VpssxrAVRlQr/v1fGbj+uJtiyKlEFeYpywmHvdifFoTlknOOftUlQJRZEyHBRMp/6ju4hDh3m/87zy2hKlbiM+6lq3EQI6MUEjAepKY0rdqnArZVcUz50Rr/4d8nCADerYrlbtdFwSPkbbOsyfXWo/pr/SiLarAnRW0KnFVB33cDfyookqWYt4AC+SZkvdKqMjLMfgd/ZrozfELfUPLEiy3A60KuSird9PnNcNUDhS2AAAGOVJREFUSK0/vyDK1qptwzLSoPkIv8H73UY1tBEW+P0a9e5wzivpNR74TR7Zdl3Tu5dVF+jKMWs9wE5HemV53cu8KSrGyZXru/za5tV3mrsRCisiGvibwA8ALwOfF5FPOue+cUvzuZdDYY+886T7nr/7n7GVLiitZrfwjehxumCQVJyZbdHTNfM6ZZiW7C76jDM/HmK/yMkT34k7LDNODKZcmg8YpBW1VcyrlGmRkWrDyZHv2KTKx8c3YZOz0n9CPDacMav871mRsTOYc7Do4ZywlRcs6mQlDNRYoZ9VZNpwYTJqBW3q8JJpHqqjwQKtfCinn9LEN4C74XzTImsbUqOBf6kWlRfG2Z8MyLKaPKs4nObtS9C/eHxDbGu4oKj8S6esEvq9su3sJNowL7K20Qa+oVxbRVn6l1+SWMoiIc1qqkqTps30IZCmNWWZMMiDeE9IJ0kMZZnQy2rmi5Rer6auNXlWMVtkDPKSymjms4zBsGA+D+I9iWE+62FrYTAuKEtNEl4oTcPDGEUjQgPQ6/uXZlXp9sVUlYl/oRnV2mTv3BgE8p0Fxgim0v5l5YQs8y933/kRyrm/1knmr0WTFuKoy6QVwjGzxIvb9IwXxdEOU2iSnsFUCp026rJL8R07S1pPajONhk6M954rh1to1LD202iEl6RKvDAOQWwnHZSYSmODFxgnfgqMSYLLLXpYt2I7zUtEgghOk2YratOEBzbe9kIvp2AILxzpGb/eCpIb3EwjA+MFbiyQuuVLsFR+n4WGzPoXZ+MZVq4VzGm92z2LW2ikX+PmiRd0Ufj0GgEggkfaSrsdQKYal1mfhsN72a0EMRjj0wnTadBM1ZHa5Yu0KUvjWa7VUk22VsupK2rlRYL2E++lOVovp30oFeTh+JCGWni72GFnUsEmvK4zZYSaauxWjTpIsAPr824aWhMdBFW8N1gVCjsw7UtVFv4Z5QYGKby3xHWOT/c01Y4XBJJGICf1jYVkJlTbXrxHKsHl3mNug2faZqviPVL6RozLLGruPfaqFGzuvcFO4a9t09AqFKoSzCB48pWjmSqiERHCeDEgtQj7NuI9iUMF8R7b9/nZzAWvPyviPVgh21OY3GF7wetceE9a41EHWvGdZCZUWxZVKC9OM3YkE8EMgodWgZ6veSwzWm+8OLDJUlSn9ViW3sPWRghoXz5VeA9RMmvEe0JDN3OtZxKWnq1GQIUg2pEET7T37vh8Te4bhDb13pfWY7kQ6qEj2/NeC+/RlXaKi8ZT04gJmcz/h+DFCUIvZuDz0YX3aqq68RgFUZhQBRsPke15jzjgxX2sPx8I3tUgfuMSWhGgrnfRp0HrxVFlaEf3vEfVKX9ss60rRKNLL2yTzDuCNQkkUyi3ed3oBRj//XUp3oMva2MTL6ZEK5qz9NYT7O+3W+3rlW080MF2jcetEYMBWu9n12OZTpZCQk6HNJOOJ4krPZZ2TbynK3jU1C9x/no0XjwX6p4OgkRt38D6/FQV0mjymXuvms/fK8Db1Ht4TU9ar1gzxYcug/cwkWW0UfDS45YexMaDZlNZWdd6/TZ5LAWsltZrJ9a1ZXeyzKP1WGpfj9c7SdIV7+l2fDresW4ocitMZJbXrlvOTV616/JYron3tPdGI7bU8YB2022uofdYspnmmW6XAjobhXMaj2Wz7SrpbRjdsLL+WmW4mhf0Wrxm+C3w5C/91bseLrolR90H1A/c8nQ/Y//pNc9NRP4g8D865z4Sln8KwDn3P9/KctzTHsuHe3v88mOf5JwpqRBycVQOFk4zsRlH9YJFeBLMbMq2KrhscwyKY2rOwmmmLmMoJWfNFif0hInN0ThyqRioiplNOW/G7OgZ1ikWLiWVmsol7KgZBuFsvcOOnlK5hLGac7be4URyAMCeGZCJQeG9aJVLSKVmYvssbMqpZB+DULkEg5BLhcaRSs2eHfj1TpGrksolLGyK6siVNWlvqQWXzKhNY+FSTuoJBzZnZnvs6BkHNmdhU3b0jEwMC5dyqR6RKz/wfahKJjbHONWe41AVK/lNTJ9UanJVhbIJW3rBgcnJxFAGe2txVE6jsExs36cT3hjN8sKmjPWCicnJVcXE9Bmogn0zYKAKdvSMS2bEUBVMrQ873tHBE1OPGKiChUvRLO2ahtabFotxin3jPzb0VEUVyjZQJUWIK5rZDOMUb3vCe5+/VR5vxX9mNiMVw6HJV2ywrYPoTXijL2zKQBdUTpNL3eZzNDmkdAn7pk8qhsppxmrB5XrEWC/YN316qqKwKakYjFMcTQ69J8T00bhW1GM7mVHYlG0942I9JhXTHluFMbypqtG4VkBppBfs1/78c1VxPJkwsz0u1iMGqlwZG7wI9shV1Qo1peIFEKr2bUq47j6/RpBqYnIGqiRVNfv1gKPJtC0DeMGqIoi9NHYdqJLCJSgcM5uFa6gZ6LJN2zjFzGbeVnW/FbIweFXinqoxBEED5a99YZO2vEeSGTObcWh6aCw9Vbdl2K29mIJB+euty5VyNmn667w8vggthGacdeU0PeWv+fHUi6WcLbbb+tLXFdPa110lfmqckS7QYtmv+61dUzFBrCklUf73SBdcqoat6EgzVhtgJ51ROe29faggApW3Y7n7obV4aHr0ddUKQaWhXCeySSss0tdle25KHEeSGefKLYyTVihmO50zrXsMk4K5SZdjyp3Q1/46N2kVNqGnauYmo69Laqfb8e+NTXqq4qDukyqDcYJdE6ZIlKG0CUPt6+nc+DpZO8U4WVA7zWGdMUpK5ibFhLSb8/PnYjmRHTK3GXOTonBtWfeCmFRjFyWWnXTO+WJMX1eMkwUXixFHsxl7VR/rhNopttNFey21OA7rDOtUuzw3KUNdYpH2vPq6YlL3WrGQJq0mamScFK0tm7KXVrf1sFE9b86/GSc/TMpwfypqqxkmJdPa26kRGmuOyZThsOpxIj+kdqq1R4MSn2ZpNLmuKa3/D5Aq014/L4xSM0xKJlWPTJkgBGJZmLTVKmjmiy5t0gpZLUKeeeKvwUGZk4frUTtNFu63hUlJxLa6AYugjdAVOilMwjCIG/mpuhSpMuS6onaa0mgybfxH5aRkEUROaqsYJCX75fLeu1EaERiATJl2PHKiLNMqa23g67ZqpxlzTsiTChPKnCmDRXz5raIf7FJ24i2btIAQcbOqYzBMSwqThPH9qo3e6dJoGTggC9ubCBzrpJ0erbKq3dZoEzRTlTW6Ck00UvfjdnfqtdporIM8rdvyNjoMtVGtl7yhidho53zuaBDYjhdfwnkY57UCmo/U3XH+/pi1eZ0DjRhMd9x89xiRzvzSjeeww4pnsl3X+R3K2I7lZ9nvFNmgX3E1b9panv74zrLDz+u83oHqdsau0glzjQdzPesr0lr1+F3T9/Ra5+Gu8ptOOTal/1pex6vlcz0d0F+6viRvO3dnupEHgZc6yy8DH7jVmdzTHcvLpsc/OHiMymkqp7GuecHUFDbFdEa1dxvHTYOyK8jSfVir0CnSoSNkggtEY9vfDbVtRF9WBXyAVsRnHbN2s3RFgDat75a7y7ooje08MZoGx3p6TTmbxsIm1kWCmhf8pnyudny3fM3/TeJDmwSDuuk0Zd50Duv2aravLK/ZpLu+W57fdI9eUd7uuQCtGNG1ytwt+9XOb13AZt2+62VvOgvddNfz6nItu65fm037X4/AznpZbsQmVyvHa9Hd92rvjquJGW26168l6NSsu147XC+vldbm9P1/JVeu66Z7heDOdZZ/U/nWj9tU77rbVtJgc9vlWu2I6ynT+rpuGa92nmtDoF5XPq+V/rrAU0MTRXE1oahuuq+V/sayX6vxdY1G2SaxKrhKm7TJo/m9Mb31FVfmt8kGG+fidXQEq66d7np52nJ07dK1w/rv18smO2y8PjdwfPeY6/TOXPeN9DqeOVfNY8W7Ju1cpLKex6ayrZ/XtTod7fpQV65l3w3bXtMjtiGP7nGr98Dauk35rf1et3jbKrnua7Z5dfdWvKEq/Fr16jXyuyLTDXXquspzo2W+AW7mlr7TTNj91GfcJ15/PP7VyUXkC53lv+Oc+zu3IZ9rck93LAub8vTslJe9twmFSaicoqcMqTLtl+w6fP1tvuLWnS+GzdfARGz7lbi2itotv7D7qRCSVm0WfIeyObb9Eh2+PmbKtF9Im/QausqyShxFnWCR1kMDtJ2XRq7aK6SqVvW1ScdY1SrYrn/xTJRtx2em2jCv/NjH5qttE5qbBgnnJsR3XW3VWEUVOttAK0PdPKeaKQcSZTEbGp/GKvIkjKNyS5XS2ir/9dSqVvlVh6+maUirrBOypKasE5IwpqkymtooemG80vqXU6+qusw/DeMpuo3frnKrDhPbTwv/Bdp/aZUV9VMd5r9sjqvD+AfVsVXTeGrGIjrn64KIQ3c6NMZJO4YvUesfEoQ6zMOptV0qmgY7ijiMUe0YvPWvtV1pceOEOoxVa8pXln4sWKPc2nzF7Y7bWW8oNw3jpnHb2KVRgW3GDjbLXtzK59uVDG+UY5e/l8qxzfFK2TaMulFObcYMNv+bsqx/fbadL8kqfHU2tUZp2y4D2Ma+wYat7TYor9omv+Z4t1Qg9V+37cr4GlNrcKDTjtLBhpdxV0F3nZUPDtbXAxPmK11RCW5CdyU0xI2g9FJB2QblTqU7duo0rG2l/Bgl8Y14Py4p2MIIKvXnZkO6ttnezGsqq52eVsG2Ebmy4sOV1zox4PdtxyB1epwinQ5FZ7yPcyz3hXb8URNe7cct+XO74it/I/DV2DqMbZLEtfO+tq3ASnx4dhPmnVkfytyIVwmtSFc7Xkh17NoaP1zrJk8rbagusDymsVVXqdXhj+sub+pkiWNFtVVYhrbSObbZxeLneA1jxJp5Pp0K6qoOH5LcpKNcKwYmNoTqhnBmMbJUVrUsjw9htdL+9ucqzXyaYV2TbhN67W0S8rO086m2/cLOfSJhf6dcG/ZMY267mlcrPFJLu9yEW9ve62+FdgXNVoRL3Go+q9cn7G+a8rMMc5VQ/mZa247i7EpfoKu4Go6TEHYrwQYSRNBa29HpYzfHdpVym3PA26pbZZp9uiGjzZyWbcFcJ71OGaUT5d/Us6YutHXULR+NK9e1qW9duzX3qlvmQXtsx0iy1unqrG9CRq97HsfXUUWu6Pg1j6a1Ml/P8evrVo5f7yQ2z51m/frzotm3e62ukne7/8aO6GpH/5od3Ney3+3sOL5WR/ou45z76F3K+hXgoc7y6bDulnJPj7EUkQnw9N0ux5uM48DFu12INxnR5neeaPM7T7T5nSfa/M4S7X3niTa/83w72/ytzrkTd7sQdwMRSYDfA74f36H8PPCnnHNfv5X53NMeS+Dpuz0I982GiHwh2vzOEm1+54k2v/NEm995os3vLNHed55o8ztPtPm3J865WkT+K+BT+OlG/v6t7lTCvd+xjEQikUgkEolEIpHINXDO/Rq3edIV9dq7RCKRSCQSiUQikUgkcnXu9Y7lHVc7ikSb3wWize880eZ3nmjzO0+0+Z0l2vvOE21+54k2fxNzT4v3RCKRSCQSiUQikUjk7nOveywjkUgkEolEIpFIJHKXuWc7liLyURF5WkSeEZGfvNvleSMgIg+JyK+LyDdE5Osi8hNh/VER+bSIfDP8PxLWi4j8H+EafEVE3nd3z+DeRUS0iPyOiPyrsPyIiDwZbPsrIpKF9b2w/EzY/vDdLPe9iojsiMgnROR3ReQpEfmDsZ7fXkTkL4fnytdE5JdEJI/1/NYiIn9fRM6LyNc66264XovIj4b9vykiP3o3zuVe4So2/7nwbPmKiPxfIrLT2fZTweZPi8hHOutjm+Y62WTzzra/IiJORI6H5VjPbwFXs7mI/Hio618Xkf+1sz7W8zcp92THUkQ08DeBPwK8C/iTIvKuu1uqNwQ18Fecc+8CPgj8l8GuPwl81jn3OPDZsAze/o+Hvz8H/K07X+Q3DD8BPNVZ/l+Av+GcexuwC/xYWP9jwG5Y/zfCfpEb5+eB/9c5907gvXjbx3p+mxCRB4H/GvgDzrnvwEud/zCxnt9qPg6sT759Q/VaRI4Cfw34APDdwF9rOqORjXycK23+aeA7nHPvwc8b91MA4X36w8C7wzH/Z/ioGNs0N8bHudLmiMhDwIeBFzurYz2/NXycNZuLyB8CPga81zn3buCvh/Wxnr+JuSc7lviHwDPOueeccyXwy/jKHbkJnHNnnHO/HX5P8I3tB/G2/Ydht38I/Inw+2PAP3Ke3wJ2ROT+O1zsex4ROQ38MeAXwrIAHwI+EXZZt3lzLT4BfH/YP3KdiMg28B8Bfw/AOVc65/aI9fx2kwB98ZM0D4AzxHp+S3HO/QZweW31jdbrjwCfds5dds7t4jtJVzTiI55NNnfO/WvnXB0Wfws4HX5/DPhl51zhnHseeAbfnoltmhvgKvUc/Eeo/w7oiofEen4LuIrN/yLws865IuxzPqyP9fxNzL3asXwQeKmz/HJYF7lFhNCz3w88CdznnDsTNp0F7gu/43W4Nfzv+JehDcvHgL1Ow6Rr19bmYft+2D9y/TwCXAD+gfjw418QkSGxnt82nHOv4L9mv4jvUO4DXyTW8zvBjdbrWN9vLX8G+H/C72jz24SIfAx4xTn35bVN0ea3j7cD3xuGK/xbEfmusD7a/E3MvdqxjNxGRGQE/HPgLznnDrrbnJcRjlLCtwgR+UHgvHPui3e7LG8iEuB9wN9yzv1+YMoyPBCI9fxWE0LMPobv1D8ADInegTtOrNd3FhH5afwQk39yt8vyRkZEBsB/D/zM3S7Lm4wEOIofOvXfAv80RpZE7tWO5SvAQ53l02Fd5CYRkRTfqfwnzrlfDavPNaF/4X8T7hCvw83zPcAfF5EX8GEhH8KP/9sJIYOwatfW5mH7NnDpThb4DcDLwMvOuSfD8ifwHc1Yz28ffxh43jl3wTlXAb+Kr/uxnt9+brRex/p+CxCR/xz4QeBH3HJet2jz28Nj+I9WXw7v0tPAb4vIKaLNbycvA78awow/h4+6Ok60+Zuae7Vj+XngcfGKghl+kPAn73KZ7nnCl6a/BzzlnPvfOps+CTSKaT8K/MvO+j8dVNc+COx3Qq4i14Fz7qecc6edcw/j6/G/cc79CPDrwA+F3dZt3lyLHwr7Rw/EDeCcOwu8JCLvCKu+H/gGsZ7fTl4EPigig/CcaWwe6/nt50br9aeAD4vIkeBp/nBYF7lOROSj+OENf9w5N+ts+iTww+JVjx/BC8p8jtimuSmcc191zp10zj0c3qUvA+8Lz/pYz28f/wL4QwAi8nYgAy4S6/mbG+fcPfkH/FG82tqzwE/f7fK8Ef6A/xAfJvUV4Evh74/ixzZ9Fvgm8BngaNhf8ApfzwJfxSs+3vXzuFf/gO8D/lX4/Sj+QfwM8M+AXlifh+VnwvZH73a578U/4DuBL4S6/i+AI7Ge33ab/0/A7wJfA/4x0Iv1/Jbb+JfwY1grfOP6x15PvcaPC3wm/P0Xd/u8vp3/rmLzZ/BjyZr36N/u7P/TweZPA3+ksz62aW7C5mvbXwCOh9+xnt8mm+M7kr8Ynum/DXyos3+s52/SPwkXOhKJRCKRSCQSiUQikdfFvRoKG4lEIpFIJBKJRCKRbxNixzISiUQikUgkEolEIjdF7FhGIpFIJBKJRCKRSOSmiB3LSCQSiUQikUgkEoncFLFjGYlEIpFIJBKJRCKRmyJ2LCORSCTybYuI/Gb4/7CI/Km7XZ5IJBKJRCKbiR3LSCQSiXzb4pz7D8LPh4HYsYxEIpFI5NuU2LGMRCKRyLctInIYfv4s8L0i8iUR+csiokXk50Tk8yLyFRH582H/7xORfysi/1JEnhORnxWRHxGRz4nIV0XksbDffyIiXxORL4vIb9yt84tEIpFI5I1CcrcLEIlEIpHIdfCTwF91zv0ggIj8OWDfOfddItID/p2I/Ouw73uBJ4DLwHPALzjnvltEfgL4ceAvAT8DfMQ594qI7Nzpk4lEIpFI5I1G9FhGIpFI5F7kw8CfFpEvAU8Cx4DHw7bPO+fOOOcK4Fmg6XB+FR9SC/DvgI+LyJ8F9B0rdSQSiUQib1CixzISiUQi9yIC/Lhz7lMrK0W+Dyg6q2xn2RLee865vyAiHwD+GPBFEXm/c+7SbS91JBKJRCJvUKLHMhKJRCL3AhNg3Fn+FPAXRSQFEJG3i8jwehMTkcecc086534GuAA8dEtLG4lEIpHIm4zosYxEIpHIvcBXACMiXwY+Dvw8Pqz1t0VE8J3DP3ED6f2ciDyO93x+FvjyLS1tJBKJRCJvMsQ5d7fLEIlEIpFIJBKJRCKRe5gYChuJRCKRSCQSiUQikZsidiwjkUgkEolEIpFIJHJTxI5lJBKJRCKRSCQSiURuitixjEQikUgkEolEIpHITRE7lpFIJBKJRCKRSCQSuSlixzISiUQikUgkEolEIjdF7FhGIpFIJBKJRCKRSOSmiB3LSCQSiUQikUgkEoncFP8/rlYJ16dR+iIAAAAASUVORK5CYII=\",\n      \"text/plain\": [\n       \"<Figure size 1080x1080 with 2 Axes>\"\n      ]\n     },\n     \"metadata\": {}\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Visualizing training\\n\",\n    \"Now let's draw a single chart that compares the learning curves of the two different models.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 25,\n   \"source\": [\n    \"train_1,  test_1  = list(zip(*losses_1))\\n\",\n    \"train_1a, test_1a = list(zip(*losses_1_adam))\\n\",\n    \"train_2,  test_2  = list(zip(*losses_2))\\n\",\n    \"train_2a, test_2a = list(zip(*losses_2_adam))\\n\",\n    \"train_3a, test_3a = list(zip(*losses_3))\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 26,\n   \"source\": [\n    \"losses_1_adam\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"execute_result\",\n     \"data\": {\n      \"text/plain\": [\n       \"[(1.2344593836784363, 0.713362567743678),\\n\",\n       \" (0.648405224943161, 0.6597177934874395),\\n\",\n       \" (0.5851686426639557, 0.6617723923579902),\\n\",\n       \" (0.5194843295574189, 0.5935856424699164),\\n\",\n       \" (0.45305402550697327, 0.5900366710629433),\\n\",\n       \" (0.2977709783792496, 0.49029243049348237),\\n\",\n       \" (0.27699643795490264, 0.4890738264390617),\\n\",\n       \" (0.2709969203948975, 0.4920012930016609),\\n\",\n       \" (0.2653717215538025, 0.4948571656539941),\\n\",\n       \" (0.26125567021369933, 0.4921688241943432),\\n\",\n       \" (0.23106717193126677, 0.48678314078385665),\\n\",\n       \" (0.22838653769493103, 0.4875897347547446),\\n\",\n       \" (0.22782497510910035, 0.4886196310732775),\\n\",\n       \" (0.22744231758117675, 0.48978534644576394),\\n\",\n       \" (0.227169718003273, 0.48986973580281445)]\"\n      ]\n     },\n     \"metadata\": {},\n     \"execution_count\": 26\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 27,\n   \"source\": [\n    \"plt.figure(figsize=(20,20))\\n\",\n    \"plt.xlabel('epochs')\\n\",\n    \"plt.ylabel('loss')\\n\",\n    \"plt.title('Evolution of training and testing losses')\\n\",\n    \"x = range(15)\\n\",\n    \"h1,  = plt.plot(x, test_1, 'c', label='test loss Linear')\\n\",\n    \"h2,  = plt.plot(x, train_1, 'c--', label='train loss Linear')\\n\",\n    \"h3,  = plt.plot(x, test_1a, 'b', label='test loss Linear Adam')\\n\",\n    \"h4,  = plt.plot(x, train_1a, 'b--', label='train loss Linear Adam')\\n\",\n    \"h5,  = plt.plot(x, test_2, 'r', label='test loss MLP')\\n\",\n    \"h6,  = plt.plot(x, train_2, 'r--', label='train loss MLP')\\n\",\n    \"h7,  = plt.plot(x, test_2a, 'm', label='test loss MLP Adam')\\n\",\n    \"h8,  = plt.plot(x, train_2a, 'm--', label='train loss MLP Adam')\\n\",\n    \"h9,  = plt.plot(x, test_3a, 'g', label='test loss ResNet Adam')\\n\",\n    \"h10, = plt.plot(x, train_3a, 'g--', label='train loss ResNet Adam')\\n\",\n    \"l   = plt.legend(handles=[h1, h2, h3, h4, h5, h6, h7, h8, h9, h10])\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"display_data\",\n     \"data\": {\n      \"image/png\": \"iVBORw0KGgoAAAANSUhEUgAABIkAAAR8CAYAAAAZ7cTyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XtYlVX+///nQhA8BYTON095+pSB7C0mKEYmZOOJooONYx9JrV8Hf2VO2RDamFnfnHR0yuiTOR1My/nszJpsTH95SBBrMgXCs4UHGtQycWInopPA+v2xt3swAdGEbfp6XBfXxX2ve631vm/vrive13ut21hrERERERERERGRi1uAvwMQERERERERERH/U5JIRERERERERESUJBIRERERERERESWJREREREREREQEJYlERERERERERAQliUREREREREREBCWJREREftGMMdYY819n2bevMebLcx1THebtaozJN8YcNsaMq6c5HjfGvHaur/UnY8w8Y8wzfpj3cmNMqTGmUT2M7Zd7EhERkeopSSQiItIAjDGFxpij3j+2T/z8TwPHcFJCyVq71lrbtSFj8HoMyLTWtrDWZvy00RiTZYy55+dMYK39o7W2TmOcybXnK2PMaGPMJ+dorEJjzA0njq21/7TWNrfWVpyL8UVEROT8FejvAERERC4iN1lrV/k7iPNAB+Dts+1sjAm01pafw3hEREREBFUSiYiI+JUxJtgYU2KMia5yrpW36uhX3uN7jTE7jTH/Msb83RjTpoaxTqrAqVpdYozJ9p7e6K1i+q0xJtEYs7fK9ZHeMUqMMVuNMSlV2uYZY14yxiz1LhP73BjTpZb7SvGOUeIdM9J7fjWQBPyPN44rf9JvKtC3Svv/eM9bY8yDxpgCoMB77gVjTJEx5gdjTK4xpm+VcaYYYxZ4f+/o7T/KGPNPY0yxMeYPZ3ltE2PMfGPM98aY7caYx6o+w2qew+lifMcY86b3mW41xsRWae9hjMnzti0EQmqYIxKYA/TxPrMS7/lgY8xM730cMMbMMcY08ba1NMZ86P33+ZcxZq0xJsAY8xZwObDEO9ZjVZ5JoLdvljHm/xpjPvXGtsIY07JKPCONMV8bYw4ZY574aWVSbWp6143H88aY77zPcrPx/jdjjBlijNnmjWWfMeb3Vca70XiWNpYYY/5hjHFWaUv3Xn/YGPOlMaZ/XWIUERG5kClJJCIi4kfW2n8DfwPuqHJ6GLDGWvudMeZ64FnvudbA15xFFY619jrvr929S4cWVm03xgQBS4AVwK+Ah4C/GmOqLkcbDjwFhAM7ganVzeVN/LiAh4FWwDI8SYfG1trrgbXAWG8cX/0kzj/8pH1sleZbgN5AlPd4AxADXAr8L7DIGFNtIsXrWqAr0B+YfCJxdYbXPgl0BDoDvwZSaxmjLjGm4Pn3DAP+DpxIijUGFgNvefsuAoZWN4G1djswBvjM+8zCvE3TgCu98/8X0BaY7G17FNiL59/n/wCPe4aydwL/xFP11txa+6ca7uu/gbvwvCuNgd97444CZgMj8Lyvod55T+s07/oA4Drv/YR6rznkbXsduN9a2wKIBlZ7x+sBzAXuByKAvwB/9ybPugJjgThvv4FAYV3iFBERuZApSSQiItJwFnsrGk783Os9/794EjAn/Lf3HHj+2J5rrc3zJpQm4qkY6XiOY4sHmgPTrLU/WmtXAx9ycvLqfWvteu9Sr7/iST5U57fAUmvtSmvtcWAm0AS45mfG+Ky19l/W2qMA1toF1tpD1tpya+2fgWA8iZ2aPGWtPWqt3QhsBLqfxbXDgD9aa7+31u4FTtlTqao6xPiJtXaZd7+ft6rMEw8EAbOstcette/iSTjViTHGAPcBj3if2WHgj/znPTuOJxHTwTv+Wmutrev4wBvW2q+8/xbv8J934XZgibX2E2vtj3iSUnUdt7Z3/TjQArgKMNba7dbab6rcS5Qx5hLvv0ue9/x9wF+stZ9bayustfOBf+N5thV4/i2ijDFB1tpCa+2uM7h/ERGRC5KSRCIiIg3nFmttWJWfV73nM4Gmxpje3j+IY4D3vW1t8FRUAGCtLcVTQVGn6owz0AYostZWVjn39U/m+bbK72V4kko1jVU15kqgiJ8fc1HVA2PM771LvtzeJVahQMvquwJ1j7+2a9v8JI6TYvqpOsT403lCvMu62gD7fpK4+Zq6awU0BXJPJCWBj7znAWbgqQZbYYzZbYyZcAZjVxd3tc/HWlvGfyp+TqfGd92btPwf4CXgO2PMK8aYS7yXDgWGAF8bY9YYY/p4z3cAHq2amAXaA22stTvxVLpN8Y73tqlhGaeIiMjFREkiERERP/NWkbyDp2rnDuBDb+UHwH48f+wCYIxphmfpzL5qhjqCJzFwwmVnEMZ+oL0xpur/G1xewzx1GatqzAbPH+d1HaumyhPfeePZ2+cxPJU94d4lVm7AnEW8Z+IboF2V4/Y1XfgzY/wGaOt9didcXsv1P31mxcBRoFuVpGSotbY5gLX2sLX2UWttZzxL3sZX2ZPnTCqKqovb93y8eyBF1LFvre+6tTbDWtsTz3LDK4E07/kN1tqb8Sx9W4znvyXwJKum/iQx29Ra6/L2+19r7bXeOS0w/SzvWURE5IKhJJGIiMj54X/xLNMawX+WmoFnb5+7jDExxphgPEuGPrfWFlYzRj5wmzGmqfF86v7/+Un7ATx76VTnczwVIY8ZY4KMMYnATZzdV8jeAZKNMf29ex09imeZzz/q2L+2OE9oAZQDB4FAY8xk4JLau5wT7wATjTHhxpi2ePa1qY8YP/P2Hef997gN6FXL9QeAdt69jE5Ub70KPG/+swF6W2PMQO/vNxpj/subhHLjWX5VWWWs0z3/mrwL3GSMucYbyxTqnrir8V03xsR5K+2C8CRDjwGVxpjGxpgRxphQ79LGH6rcx6vAGG8/Y4xpZoxJNsa0MMZ0NcZc753nGJ6EWuUpEYmIiFxklCQSERFpOCe+GHXi58SSMqy1n+P547cN8P9VOb8KeAJ4D0+VRhdO3r+oqueBH/H8kT8fz75BVU0B5nuX3gyr2uDdP+YmYDCeKpTZwEhr7Y4zvUlr7Zd4NnR+0TvWTXg2Qv6xjkO8ANxuPF8Qq2nPn+V4lk99hWeJ0jFOs/TrHHkaz4bPe4BVeJIi/z7XMXqf1W3AaOBfeBKIf6uly2pgK/CtMabYey4dz5KydcaYH7zxntgP6QrvcSmehNRsa22mt+1ZYJL3PfF9KayOcW/Fs+n523je11LgO2p+RlX71vauX4In6fM9nmd5CM+SOYA7gULvPY7Bk2jFWpsD3Itnmdr33mcx2tsnGM/G3sV4ls79Cs8eSCIiIhc1c2Z7FIqIiIjICcaY/xcYbq3t5+9YzkfGmOZACXCFtXaPv+MRERGR2qmSSERERKSOjDGtjTEJxpgA72fUH+U/m4wLYIy5ybvksRmeL9ttRp+XFxER+UVQkkhERESk7hoDfwEO41ni9QGepXnyHzfj2YR6P55lbcOtStdFRER+EbTcTEREREREREREVEkkIiIiIiIiIiJKEomIiIiIiIiICBDo7wCqatmype3YsaO/wxARERERERERuWDk5uYWW2tbne668ypJ1LFjR3JycvwdhoiIiIiIiIjIBcMY83VdrtNyMxERERERERERUZJIRERERERERESUJBIREREREREREc6zPYlERERERERELmTHjx9n7969HDt2zN+hyAUoJCSEdu3aERQUdFb9lSQSERERERERaSB79+6lRYsWdOzYEWOMv8ORC4i1lkOHDrF37146dep0VmNouZmIiIiIiIhIAzl27BgRERFKEMk5Z4whIiLiZ1WpKUkkIiIiIiIi0oCUIJL68nPfLSWJRERERERERC4SJSUlzJ49+6z7z5o1i7KysmrbEhMTycnJOeuxa5KVlcWNN954yvl77rmHbdu2nfP5LmZKEomIiIiIiIhcJOozSdTQXnvtNaKioupt/PLy8nob+3ylJJGIiIiIiIjIRWLChAns2rWLmJgY0tLSAJgxYwZxcXE4nU6efPJJAI4cOUJycjLdu3cnOjqahQsXkpGRwf79+0lKSiIpKanWeVwuFw6Hg+joaNLT0wGoqKhg9OjRREdH43A4eP755wHIyMggKioKp9PJ8OHD63wvVSuXmjdvzh/+8Ae6d+9OfHw8Bw4cAODgwYMMHTqUuLg44uLi+PTTTwFYv349ffr0oUePHlxzzTV8+eWXAMybN4+UlBSuv/56+vfvX+dYLhT6upmIiIiIiIiIHzxcUEB+aek5HTOmeXNmXXFFje3Tpk1jy5Yt5OfnA7BixQoKCgpYv3491lpSUlLIzs7m4MGDtGnThqVLlwLgdrsJDQ3lueeeIzMzk5YtW9Y4x/79+0lPTyc3N5fw8HAGDBjA4sWLad++Pfv27WPLli2Ap6rpREx79uwhODjYd+5MHTlyhPj4eKZOncpjjz3Gq6++yqRJk/jd737HI488wrXXXss///lPBg4cyPbt27nqqqtYu3YtgYGBrFq1iscff5z33nsPgLy8PDZt2sSll156VrH8kilJJCIiIiIiInKRWrFiBStWrKBHjx4AlJaWUlBQQN++fXn00UdJT0/nxhtvpG/fvnUec8OGDSQmJtKqVSsARowYQXZ2Nk888QS7d+/moYceIjk5mQEDBgDgdDoZMWIEt9xyC7fccstZ3Ufjxo19+xb17NmTlStXArBq1aqT9i364YcfKC0txe12M2rUKAoKCjDGcPz4cd81v/71ry/KBBEoSSQiIiIiIiLiF7VV/DQUay0TJ07k/vvvP6UtLy+PZcuWMWnSJPr378/kyZN/1lzh4eFs3LiR5cuXM2fOHN555x3mzp3L0qVLyc7OZsmSJUydOpXNmzcTGHhm6YqgoCDfl70aNWrk20+osrKSdevWERISctL1Y8eOJSkpiffff5/CwkISExN9bc2aNftZ9/lLpj2JRERERERERC4SLVq04PDhw77jgQMHMnfuXEq9y9727dvHd999x/79+2natCmpqamkpaWRl5dXbf/q9OrVizVr1lBcXExFRQUul4t+/fpRXFxMZWUlQ4cO5ZlnniEvL4/KykqKiopISkpi+vTpuN1uXyznwoABA3jxxRd9xyeW2bndbtq2bQt49iESD1USiYiIiIiIiFwkIiIiSEhIIDo6msGDBzNjxgy2b99Onz59AM8G0AsWLGDnzp2kpaUREBBAUFAQL7/8MgD33XcfgwYNok2bNmRmZlY7R+vWrZk2bRpJSUlYa0lOTubmm29m48aN3HXXXVRWVgLw7LPPUlFRQWpqKm63G2st48aNIyws7JQxP/74Y9q1a+c7XrRoUZ3uNyMjgwcffBCn00l5eTnXXXcdc+bM4bHHHmPUqFE888wzJCcnn9EzvJAZa62/Y/CJjY21J3YmFxEREREREbnQbN++ncjISH+HIRew6t4xY0yutTb2dH213ExERERERERERJQkEhERERERERERJYlERERERERERAQliUREREREREREBCWJREREREREREQEJYlERERERERERAQliUREREREREQuGiUlJcyePfus+g4ZMoSSkpI6Xz9lyhRmzpx5VnOdTvPmzU85N2fOHN588816me9ioSSRiIiIiIiIyEWitiRReXl5rX2XLVtGWFhYfYR1TowZM4aRI0fW2/jWWiorK+tt/POBkkQiIiIiIiIiF4kJEyawa9cuYmJiSEtLIysri759+5KSkkJUVBQAt9xyCz179qRbt2688sorvr4dO3akuLiYwsJCIiMjuffee+nWrRsDBgzg6NGjtc6bn59PfHw8TqeTW2+9le+//x6AjIwMoqKicDqdDB8+HIA1a9YQExNDTEwMPXr04PDhw3W6t6qVS4mJiaSnp9OrVy+uvPJK1q5dC0BFRQVpaWnExcXhdDr5y1/+AkBpaSn9+/fn6quvxuFw8MEHHwBQWFhI165dGTlyJNHR0RQVFdX1Uf8iBfo7ABEREREREZGLVeIXX5xybtivfsUDbdtSVlHBkE2bTmkffdlljG7dmuIff+T2rVtPasvq0aPW+aZNm8aWLVvIz8/3XJ+VRV5eHlu2bKFTp04AzJ07l0svvZSjR48SFxfH0KFDiYiIOGmcgoICXC4Xr776KsOGDeO9994jNTW1xnlHjhzJiy++SL9+/Zg8eTJPPfUUs2bNYtq0aezZs4fg4GDfUraZM2fy0ksvkZCQQGlpKSEhIbXeU03Ky8tZv349y5Yt46mnnmLVqlW8/vrrhIaGsmHDBv7973+TkJDAgAEDaN++Pe+//z6XXHIJxcXFxMfHk5KS4rvX+fPnEx8ff1Zx/JKokkhERERERETkItarVy9fggg81T3du3cnPj6eoqIiCgoKTunTqVMnYmJiAOjZsyeFhYU1ju92uykpKaFfv34AjBo1iuzsbACcTicjRoxgwYIFBAZ66lgSEhIYP348GRkZlJSU+M6fqdtuu+2U+FasWMGbb75JTEwMvXv35tChQxQUFGCt5fHHH8fpdHLDDTewb98+Dhw4AECHDh0uigQRqJJIRERERERExG9qq/xp2qhRre0tGzc+beVQXTRr1uw/8WRlsWrVKj777DOaNm1KYmIix44dO6VPcHCw7/dGjRqddrlZTZYuXUp2djZLlixh6tSpbN68mQkTJpCcnMyyZctISEhg+fLlXHXVVWc89okYGzVq5NtvyVrLiy++yMCBA0+6dt68eRw8eJDc3FyCgoLo2LGj776rPp8LnSqJRERERERERC4SLVq0qHWPH7fbTXh4OE2bNmXHjh2sW7fuZ88ZGhpKeHi4b1+gt956i379+lFZWUlRURFJSUlMnz4dt9tNaWkpu3btwuFwkJ6eTlxcHDt27PjZMZwwcOBAXn75ZY4fPw7AV199xZEjR3C73fzqV78iKCiIzMxMvv7663M25y+JKolERERERERELhIREREkJCQQHR3N4MGDSU5OPql90KBBzJkzh8jISLp27XrOllnNnz+fMWPGUFZWRufOnXnjjTeoqKggNTUVt9uNtZZx48YRFhbGE088QWZmJgEBAXTr1o3BgwefMl5ZWRnt2rXzHY8fP75Ocdxzzz0UFhZy9dVXY62lVatWLF68mBEjRnDTTTfhcDiIjY09q8qlC4Gx1vo7Bp/Y2Fibk5Pj7zBERERERERE6sX27duJjIz0dxhyAavuHTPG5FprY0/XV8vNRERERERERERESSIREREREREREVGSSEREREREREREUJJIRERERERERERQkkhERERERERERFCSSEREREREREREUJJIRERERERE5KJRUlLC7Nmzz7r/rFmzKCsrq7YtMTGRnJycsx67JllZWdx4442nnL/nnnvYtm3bOZ/vdBYvXowxhh07dtR4zejRo3n33XcbMKpzQ0kiERERERERkYtEfSaJGtprr71GVFRUvY1fXl5e7XmXy8W1116Ly+Wqt7n9RUkiERERERERkYvEhAkT2LVrFzExMaSlpQEwY8YM4uLicDqdPPnkkwAcOXKE5ORkunfvTnR0NAsXLiQjI4P9+/eTlJREUlJSrfO4XC4cDgfR0dGkp6cDUFFRwejRo4mOjsbhcPD8888DkJGRQVRUFE6nk+HDh9f5XqpWLjVv3pw//OEPdO/enfj4eA4cOADAwYMHGTp0KHFxccTFxfHpp58CsH79evr06UOPHj245ppr+PLLLwGYN28eKSkpXH/99fTv3/+UOUtLS/nkk094/fXXefvtt33nrbWMHTuWrl27csMNN/Ddd9/52p5++mni4uKIjo7mvvvuw1rri/+RRx4hNjaWyMhINmzYwG233cYVV1zBpEmT6vwczqVAv8wqIiIiIiIicpF7+GHIzz+3Y8bEwKxZNbdPmzaNLVu2kO+deMWKFRQUFLB+/XqstaSkpJCdnc3Bgwdp06YNS5cuBcDtdhMaGspzzz1HZmYmLVu2rHGO/fv3k56eTm5uLuHh4QwYMIDFixfTvn179u3bx5YtWwBPVdOJmPbs2UNwcLDv3Jk6cuQI8fHxTJ06lccee4xXX32VSZMm8bvf/Y5HHnmEa6+9ln/+858MHDiQ7du3c9VVV7F27VoCAwNZtWoVjz/+OO+99x4AeXl5bNq0iUsvvfSUeT744AMGDRrElVdeSUREBLm5ufTs2ZP333+fL7/8km3btnHgwAGioqK4++67ARg7diyTJ08G4M477+TDDz/kpptuAqBx48bk5OTwwgsvcPPNN5Obm8ull15Kly5deOSRR4iIiDir53G2VEkkIiIiIiIicpFasWIFK1asoEePHlx99dXs2LGDgoICHA4HK1euJD09nbVr1xIaGlrnMTds2EBiYiKtWrUiMDCQESNGkJ2dTefOndm9ezcPPfQQH330EZdccgkATqeTESNGsGDBAgIDz66WpXHjxr59i3r27ElhYSEAq1atYuzYscTExJCSksIPP/xAaWkpbreb3/zmN0RHR/PII4+wdetW31i//vWvq00QgadC6kS10/Dhw31LzrKzs7njjjto1KgRbdq04frrr/f1yczMpHfv3jgcDlavXn3SXCkpKQA4HA66detG69atCQ4OpnPnzhQVFZ3Vs/g5VEkkIiIiIiIi4ge1Vfw0FGstEydO5P777z+lLS8vj2XLljFp0iT69+/vq4Y5W+Hh4WzcuJHly5czZ84c3nnnHebOncvSpUvJzs5myZIlTJ06lc2bN59xsigoKAhjDACNGjXy7SdUWVnJunXrCAkJOen6sWPHkpSUxPvvv09hYSGJiYm+tmbNmlU7x7/+9S9Wr17N5s2bMcZQUVGBMYYZM2bUGNexY8d44IEHyMnJoX379kyZMoVjx4752oODgwEICAjw/X7iuKY9keqTKolERERERERELhItWrTg8OHDvuOBAwcyd+5cSktLAdi3bx/fffcd+/fvp2nTpqSmppKWlkZeXl61/avTq1cv1qxZQ3FxMRUVFbhcLvr160dxcTGVlZUMHTqUZ555hry8PCorKykqKiIpKYnp06fjdrt9sZwLAwYM4MUXX/Qdn1hm53a7adu2LeDZh6gu3n33Xe68806+/vprCgsLKSoqolOnTqxdu5brrruOhQsXUlFRwTfffENmZiaALyHUsmVLSktLz/svnqmSSEREREREROQiERERQUJCAtHR0QwePJgZM2awfft2+vTpA3g2gF6wYAE7d+4kLS2NgIAAgoKCePnllwG47777GDRoEG3atPElQn6qdevWTJs2jaSkJKy1JCcnc/PNN7Nx40buuusuKisrAXj22WepqKggNTUVt9uNtZZx48YRFhZ2ypgff/wx7dq18x0vWrSoTvebkZHBgw8+iNPppLy8nOuuu445c+bw2GOPMWrUKJ555hmSk5PrNJbL5fJtwn3C0KFDcblczJ49m9WrVxMVFcXll1/ue55hYWHce++9REdHc9lllxEXF1enufzFnNhV+3wQGxtrT+xMLiIiIiIiInKh2b59O5GRkf4OQy5g1b1jxphca23s6fpquVk9+LGyksrzKPkmIiIiIiIiInI6ShKdY389cIAm2dkU/fvf/g5FRERERERERKTOlCQ6x1o3bkwlsPvoUX+HIiIiIiIiIiJSZ0oSnWOdvZ/V213lk3YiIiIiIiIiIuc7JYnOsXbBwQQao0oiEREREREREflFUZLoHAsMCKBDcLAqiURERERERETkF0VJonrwULt23BgR4e8wRERERERERE5SUlLC7Nmzz6rvkCFDKCkpqfP1U6ZMYebMmWc11+k0b978lHNz5szhzTffrJf5alNcXExQUBBz5syp8Zp58+YxduzYBozq7ChJVA9+164dI/7P//F3GCIiIiIiIiInqS1JVF5eXmvfZcuWERYWVh9hnRNjxoxh5MiR9Ta+tZbKyspTzi9atIj4+HhcLle9zd1QlCSqB+WVlRQePcqP1bw8IiIiIiIiIv4yYcIEdu3aRUxMDGlpaWRlZdG3b19SUlKIiooC4JZbbqFnz55069aNV155xde3Y8eOFBcXU1hYSGRkJPfeey/dunVjwIABHD3Nvrz5+fnEx8fjdDq59dZb+f777wHIyMggKioKp9PJ8OHDAVizZg0xMTHExMTQo0cPDh8+XKd7q1q5lJiYSHp6Or169eLKK69k7dq1AFRUVJCWlkZcXBxOp5O//OUvAJSWltK/f3+uvvpqHA4HH3zwAQCFhYV07dqVkSNHEh0dTVFR0Snzulwu/vznP7Nv3z727t3rO//GG29w5ZVX0qtXLz799FPf+SVLltC7d2969OjBDTfcwIEDB3zxjxo1ir59+9KhQwf+9re/8dhjj+FwOBg0aBDHjx+v03P4OZQkqgcfHDpEp88/Z3tZmb9DERERERERkfNYYuKpPycKfcrKqm+fN8/TXlx8atvpTJs2jS5dupCfn8+MGTMAyMvL44UXXuCrr74CYO7cueTm5pKTk0NGRgaHDh06ZZyCggIefPBBtm7dSlhYGO+9916t844cOZLp06ezadMmHA4HTz31lC+eL774gk2bNvmWa82cOZOXXnqJ/Px81q5dS5MmTU5/Y9UoLy9n/fr1zJo1yzff66+/TmhoKBs2bGDDhg28+uqr7Nmzh5CQEN5//33y8vLIzMzk0UcfxVrru9cHHniArVu30qFDh5PmKCoq4ptvvqFXr14MGzaMhQsXAvDNN9/w5JNP8umnn/LJJ5+wbds2X59rr72WdevW8cUXXzB8+HD+9Kc/+dp27drF6tWr+fvf/05qaipJSUls3ryZJk2asHTp0rN6DmdCSaJ60DkkBIBd+sKZiIiIiIiInOd69epFp06dfMcZGRl0796d+Ph4ioqKKCgoOKVPp06diImJAaBnz54UFhbWOL7b7aakpIR+/foBMGrUKLKzswFwOp2MGDGCBQsWEBgYCEBCQgLjx48nIyODkpIS3/kzddttt50S34oVK3jzzTeJiYmhd+/eHDp0iIKCAqy1PP744zidTm644Qb27dvnq/Dp0KED8fHx1c6xcOFChg0bBsDw4cN9S84+//xzEhMTadWqFY0bN+a3v/2sU5IlAAAgAElEQVStr8/evXsZOHAgDoeDGTNmsHXrVl/b4MGDCQoKwuFwUFFRwaBBgwBwOBy1PuNz5eyetNSqszfLuVtJIhEREREREalFVlbNbU2b1t7esmXt7XXVrFmzKvFksWrVKj777DOaNm1KYmIix6r5endwcLDv90aNGp12uVlNli5dSnZ2NkuWLGHq1Kls3ryZCRMmkJyczLJly0hISGD58uVcddVVZzz2iRgbNWrk22/JWsuLL77IwIEDT7p23rx5HDx4kNzcXIKCgujYsaPvvqs+n59yuVx8++23/PWvfwVg//791SbVqnrooYcYP348KSkpZGVlMWXKlFNiDggIICgoCGOM7/h0e0adC6okqgehgYFcGhjI7mr+QxIRERERERHxlxYtWtS6x4/b7SY8PJymTZuyY8cO1q1b97PnDA0NJTw83Lcv0FtvvUW/fv2orKykqKiIpKQkpk+fjtvtprS0lF27duFwOEhPTycuLo4dO3b87BhOGDhwIC+//LJvf5+vvvqKI0eO4Ha7+dWvfkVQUBCZmZl8/fXXpx3rq6++orS0lH379lFYWEhhYSETJ07E5XLRu3dv1qxZw6FDhzh+/DiLFi3y9XO73bRt2xaA+fPnn7N7OxdUSVRPOjdpokoiEREREREROa9ERESQkJBAdHQ0gwcPJjk5+aT2QYMGMWfOHCIjI+natWuNy6zO1Pz58xkzZgxlZWV07tyZN954g4qKClJTU3G73VhrGTduHGFhYTzxxBNkZmYSEBBAt27dGDx48CnjlZWV0a5dO9/x+PHj6xTHPffcQ2FhIVdffTXWWlq1asXixYsZMWIEN910Ew6Hg9jY2DpVLrlcLm699daTzg0dOpTf/va3TJ48mSlTptCnTx/CwsJ8S/PAs0H1b37zG8LDw7n++uvZs2dPnWJvCObERkzng9jYWJuTk+PvMM6Jvx08SEhAAEMiIvwdioiIiIiIiJwntm/fTmRkpL/DkAtYde+YMSbXWht7ur6qJKont7Vq5e8QRERERERERETqTHsS1ZPD5eWsLSnhhwbYWEpERERERERE5OdSkqierD98mOvy88mrZUMwEREREREREZHzhZJE9aRzSAiAvnAmIiIiIiIiIr8IShLVk/bBwTQCfeFMRERERERERH4RlCSqJ4EBAXQICVElkYiIiIiIiIj8IihJVI86N2miSiIRERERERE5b5SUlDB79uyz7j9r1izKysqqbUtMTCQnJ+esx65JVlYWxhhee+0137n8/HyMMcycOROA0aNH8+67757Ur7CwkCZNmhATE0NUVBRjxoyhsrLynMd3IVGSqB79344defGKK/wdhoiIiIiIiAhQv0mi+hQdHc0777zjO3a5XHTv3v20/bp06UJ+fj6bNm1i27ZtLF68uD7D/MVTkqgexYeGEnfJJf4OQ0RERERERASACRMmsGvXLmJiYkhLSwNgxowZxMXF4XQ6efLJJwE4cuQIycnJdO/enejoaBYuXEhGRgb79+8nKSmJpKSkWudxuVw4HA6io6NJT08HoKKigtGjRxMdHY3D4eD5558HICMjg6ioKJxOJ8OHD692vA4dOnDs2DEOHDiAtZaPPvqIwYMH1/m+AwMDueaaa9i5c2ed+1yMAv0dwIXs0PHjrPjXv0gMC6N1cLC/wxEREREREZHzycMPQ37+uR0zJgZmzaqxedq0aWzZsoV877wrVqygoKCA9evXY60lJSWF7OxsDh48SJs2bVi6dCkAbreb0NBQnnvuOTIzM2nZsmWNc+zfv5/09HRyc3MJDw9nwIABLF68mPbt27Nv3z62bNkCeKqaTsS0Z88egoODfeeqc/vtt7No0SJ69OjB1VdfTfAZ/J1dVlbGxx9/zNNPP13nPhcjVRLVo6+PHeO/t29n3Q8/+DsUERERERERkVOsWLGCFStW+BIvO3bsoKCgAIfDwcqVK0lPT2ft2rWEhobWecwNGzaQmJhIq1atCAwMZMSIEWRnZ9O5c2d2797NQw89xEcffcQl3pU3TqeTESNGsGDBAgIDa65lGTZsGIsWLcLlcnHHHXfUKZYTVVMJCQkkJyefUfXRxUiVRPWoc0gIgL5wJiIiIiIiIqeqpeKnoVhrmThxIvfff/8pbXl5eSxbtoxJkybRv39/Jk+e/LPmCg8PZ+PGjSxfvpw5c+bwzjvvMHfuXJYuXUp2djZLlixh6tSpbN68udpk0WWXXUZQUBArV67khRde4B//+Mdp5zyxJ5HUjSqJ6lFYUBDhgYH6wpmIiIiIiIicF1q0aMHhw4d9xwMHDmTu3LmUlpYCsG/fPr777jv2799P06ZNSU1NJS0tjby8vGr7V6dXr16sWbOG4uJiKioqcLlc9OvXj+LiYiorKxk6dCjPPPMMeXl5VFZWUlRURFJSEtOnT8ftdvtiqc7TTz/N9OnTadSo0Tl4GvJT9VpJZIwJA14DogEL3G2t/aw+5zzfdA4JUSWRiIiIiIiInBciIiJISEggOjqawYMHM2PGDLZv306fPn0AaN68OQsWLGDnzp2kpaUREBBAUFAQL7/8MgD33XcfgwYNok2bNmRmZlY7R+vWrZk2bRpJSUlYa0lOTubmm29m48aN3HXXXb7P0D/77LNUVFSQmpqK2+3GWsu4ceMICwurMf5rrrmmxrb777+fhx9+GID27dvjcrnO6hldzIy1tv4GN2Y+sNZa+5oxpjHQ1Fpb4y5UsbGxNicnp97i8YdhW7eysbSUL3v39ncoIiIiIiIi4mfbt28nMjLS32HIBay6d8wYk2utjT1d33qrJDLGhALXAaMBrLU/Aj/W13znq2mdOxNojL/DEBERERERERGpVX3uSdQJOAi8YYz5whjzmjGmWT3Od17q3KQJl3s3sBYREREREREROV/VZ5IoELgaeNla2wM4Akz46UXGmPuMMTnGmJyDBw/WYzj+8d2PP/Knf/6TL8vK/B2KiIiIiIiIiEiN6jNJtBfYa6393Hv8Lp6k0Umsta9Ya2OttbGtWrWqx3D8o7SigvTdu/nU7fZ3KCIiIiIiIiIiNaq3JJG19lugyBjT1XuqP7CtvuY7X7UPDqYRsPvoUX+HIiIiIiIiIiJSo3rbuNrrIeCv3i+b7Qbuquf5zjtBAQFcHhLC7mPH/B2KiIiIiIiIiEiN6nO5GdbafO9SMqe19hZr7ff1Od/5qnNIiCqJRERERERExO9KSkqYPXv2WfUdMmQIJSUldb5+ypQpzJw586zmOh1jDKmpqb7j8vJyWrVqxY033gjAvHnzGDt27Cn9OnbsiMPhwOl0MmDAAL799tt6ie+Xql6TROLRpUkTClVJJCIiIiIiIn5WW5KovLy81r7Lli0jLCysPsI6Y82aNWPLli0c9RZkrFy5krZt29apb2ZmJps2bSI2NpY//vGP9RnmL46SRA3gT126UNSnj7/DEBERERERkYvchAkT2LVrFzExMaSlpZGVlUXfvn1JSUkhKioKgFtuuYWePXvSrVs3XnnlFV/fjh07UlxcTGFhIZGRkdx7771069aNAQMG+JI1NcnPzyc+Ph6n08mtt97K9997FhplZGQQFRWF0+lk+PDhAKxZs4aYmBhiYmLo0aMHhw8frnbMIUOGsHTpUgBcLhd33HHHGT2L6667jp07d55Rnwtdfe9JJEBooB6ziIiIiIiIVCMx8dRzw4bBAw9AWRkMGXJq++jRnp/iYrj99pPbsrJqnW7atGls2bKF/Px87+VZ5OXlsWXLFjp16gTA3LlzufTSSzl69ChxcXEMHTqUiIiIk8YpKCjA5XLx6quvMmzYMN57772Tln/91MiRI3nxxRfp168fkydP5qmnnmLWrFlMmzaNPXv2EBwc7FvKNnPmTF566SUSEhIoLS0lJCSk2jGHDx/O008/zY033simTZu4++67Wbt2ba33X9WHH36Iw+Go8/UXA1USNYCDP/7IQwUF/MPt9ncoIiIiIiIiIifp1auXL0EEnuqe7t27Ex8fT1FREQUFBaf06dSpEzExMQD07NmTwsLCGsd3u92UlJTQr18/AEaNGkV2djYATqeTESNGsGDBAgK9BRYJCQmMHz+ejIwMSkpKfOd/yul0UlhYiMvlYkh1ybQaJCUlERMTww8//MDEiRPr3O9ioBKXBhBoDP+zbx8dQ0K4JjTU3+GIiIiIiIjI+aK2yp+mTWtvb9nytJVDddGsWbMq4WSxatUqPvvsM5o2bUpiYiLHqtljNzg42Pd7o0aNTrvcrCZLly4lOzubJUuWMHXqVDZv3syECRNITk5m2bJlJCQksHz5cq666qpq+6ekpPD73/+erKwsDh06VKc5MzMzadmy5VnFe6FTkqgBhAcFERYYqC+ciYiIiIiIiF+1aNGixj1+wFP1Ex4eTtOmTdmxYwfr1q372XOGhoYSHh7O2rVr6du3L2+99Rb9+vWjsrKSoqIikpKSuPbaa3n77bcpLS3l0KFDOBwOHA4HGzZsYMeOHTUmie6++27CwsJwOBxknYOE2cVOSaIG0jkkhN36wpmIiIiIiIj4UUREBAkJCURHRzN48GCSk5NPah80aBBz5swhMjKSrl27Eh8ff07mnT9/PmPGjKGsrIzOnTvzxhtvUFFRQWpqKm63G2st48aNIywsjCeeeILMzEwCAgLo1q0bgwcPrnHcdu3aMW7cuGrb5s2bx+LFi33H5yLhdaEz1lp/x+ATGxtrc3Jy/B1GvfjN1q1sKi3ly969/R2KiIiIiIiI+Mn27duJjIz0dxhyAavuHTPG5FprY0/XVxtXN5D/atKEcms5n5JyIiIiIiIiIiInKEnUQP7YqRO74uMxxvg7FBERERERERGRUyhJ1ECUHBIRERERERGR85mSRA3k++PHuWXzZj4oLvZ3KCIiIiIiIiIip1CSqIE0b9SIDw8dIqeWTw2KiIiIiIiIiPiLkkQNJCgggMtDQth99Ki/QxEREREREREROYWSRA2oc0gIu48d83cYIiIiIiIicpEqKSlh9uzZZ91/1qxZlJWVVduWmJhITk7OWY9dk6ysLIwxvPbaa75z+fn5GGOYOXMmAKNHj+bdd989qV9hYSFNmjQhJiaGqKgoxowZQ2VlZbVzLF68GGMMO3bsqDGO6ua40ChJ1IA6N2miSiIRERERERHxm/pMEtWn6Oho3nnnHd+xy+Wie/fup+3XpUsX8vPz2bRpE9u2bWPx4sXVXudyubj22mtxuVznLOZfIiWJGlD35s3p3KQJP9aQuRQRERERERGpTxMmTGDXrl3ExMSQlpYGwIwZM4iLi8PpdPLkk08CcOTIEZKTk+nevTvR0dEsXLiQjIwM9u/fT1JSEklJSbXO43K5cDgcREdHk56eDkBFRQWjR48mOjoah8PB888/D0BGRgZRUVE4nU6GDx9e7XgdOnTg2LFjHDhwAGstH330EYMHD67zfQcGBnLNNdewc+fOU9pKS0v55JNPeP3113n77bd95621jB07lq5du3LDDTfw3Xff+dqefvpp4uLiiI6O5r777sNaC3iqqR555BFiY2OJjIxkw4YN3HbbbVxxxRVMmjSpzvH6S6C/A7iYPNi2LQ+2bevvMEREREREROQ8UPBwAaX5ped0zOYxzbli1hU1tk+bNo0tW7aQn58PwIoVKygoKGD9+vVYa0lJSSE7O5uDBw/Spk0bli5dCoDb7SY0NJTnnnuOzMxMWrZsWeMc+/fvJz09ndzcXMLDwxkwYACLFy+mffv27Nu3jy1btgCeqqYTMe3Zs4fg4GDfuercfvvtLFq0iB49enD11VcTHBxc5+dSVlbGxx9/zNNPP31K2wcffMCgQYO48soriYiIIDc3l549e/L+++/z5Zdfsm3bNg4cOEBUVBR33303AGPHjmXy5MkA3HnnnXz44YfcdNNNADRu3JicnBxeeOEFbr75ZnJzc7n00kvp0qULjzzyCBEREXWOu6GpkkhERERERETkIrVixQpWrFjhS7zs2LGDgoICHA4HK1euJD09nbVr1xIaGlrnMTds2EBiYiKtWrUiMDCQESNGkJ2dTefOndm9ezcPPfQQH330EZdccgkATqeTESNGsGDBAgIDa65lGTZsGIsWLcLlcnHHHXfUKZYTVVMJCQkkJydXW33kcrl8FUzDhw/3LTnLzs7mjjvuoFGjRrRp04brr7/e1yczM5PevXvjcDhYvXo1W7du9bWlpKQA4HA46NatG61btyY4OJjOnTtTVFRUp7j9RZVEDaisooK+X3zBva1bM0YVRSIiIiIiIhe12ip+Goq1lokTJ3L//fef0paXl8eyZcuYNGkS/fv391XOnK3w8HA2btzI8uXLmTNnDu+88w5z585l6dKlZGdns2TJEqZOncrmzZurTRZddtllBAUFsXLlSl544QX+8Y9/nHbOE3sS1eRf//oXq1evZvPmzRhjqKiowBjDjBkzauxz7NgxHnjgAXJycmjfvj1TpkzhWJWPVJ2ocAoICDip2ikgIIDy8vLTxuxPqiRqQE0CAth19ChbjhzxdygiIiIiIiJyEWrRogWHDx/2HQ8cOJC5c+dSWupZ9rZv3z6+++479u/fT9OmTUlNTSUtLY28vLxq+1enV69erFmzhuLiYioqKnC5XPTr14/i4mIqKysZOnQozzzzDHl5eVRWVlJUVERSUhLTp0/H7Xb7YqnO008/zfTp02nUqNE5eBrw7rvvcuedd/L1119TWFhIUVERnTp1Yu3atVx33XUsXLiQiooKvvnmGzIzMwF8CaGWLVtSWlp6QX3xTJVEDcgY4/nCWZUMo4iIiIiIiEhDiYiIICEhgejoaAYPHsyMGTPYvn07ffr0AaB58+YsWLCAnTt3kpaWRkBAAEFBQbz88ssA3HfffQwaNIg2bdr4kiY/1bp1a6ZNm0ZSUhLWWpKTk7n55pvZuHEjd911l+8z9M8++ywVFRWkpqbidrux1jJu3DjCwsJqjP+aa66pse3+++/n4YcfBqB9+/Z1+lKZy+Xybax9wtChQ3G5XMyePZvVq1cTFRXF5Zdf7ntGYWFh3HvvvURHR3PZZZcRFxd32nl+KcyJHbjPB7GxsTYnJ8ffYdSr27dsYcuRI+zo3dvfoYiIiIiIiEgD2759O5GRkf4OQy5g1b1jxphca23s6fpquVkD69KkCXuOHaPyPErOiYiIiIiIiIgoSdTA4i+5hJtbtuRIRYW/QxERERERERER8dGeRA3s1latuLVVK3+HISIiIiIiIiJyElUS+YmWm4mIiIiIiIjI+URJogZWYS2t//EPniws9HcoIiIiIiIiIiI+ShI1sEbGEBIQwO6jR/0dioiIiIiIiIiIj5JEftA5JITdx475OwwRERERERG5yJSUlDB79uyz6jtkyBBKSkrqfP2UKVOYOXPmWc11OsYYUlNTfcfl5eW0atWKG2+8EYB58+YxduzYU/p17NgRh8OB0+lkwIABfPvtt9WOX1xcTFBQEHPmzKkxhprm+CVTksgPOjdpokoiERERERERaXC1JYnKy8tr7bts2TLCwsLqI6wz1qxZM7Zs2cJR79/WK1eupG3btnXqm5mZyaZNm4iNjeWPf/xjtdcsWrSI+Ph4XC7XOYv5l0BJIj/oHBLCd8ePU3qa/wBFREREREREzqUJEyawa9cuYmJiSEtLIysri759+5KSkkJUVBQAt9xyCz179qRbt2688sorvr4dO3akuLiYwsJCIiMjuffee+nWrRsDBgzwJWtqkp+fT3x8PE6nk1tvvZXvv/8egIyMDKKionA6nQwfPhyANWvWEBMTQ0xMDD169ODw4cPVjjlkyBCWLl0KgMvl4o477jijZ3Hdddexc+fOattcLhd//vOf2bdvH3v37vWdf+ONN7jyyivp1asXn376qe/8kiVL6N27Nz169OCGG27gwIEDgKeaatSoUfTt25cOHTrwt7/9jcceewyHw8GgQYM4fvz4GcVc35Qk8oO+oaE80q4dP+oLZyIiIiIiIhe1LxK/OOVn3+x9AFSUVVTb/s28bwD4sfjHU9pOZ9q0aXTp0oX8/HxmzJgBQF5eHi+88AJfffUVAHPnziU3N5ecnBwyMjI4dOjQKeMUFBTw4IMPsnXrVsLCwnjvvfdqnXfkyJFMnz6dTZs24XA4eOqpp3zxfPHFF2zatMm3tGvmzJm89NJL5Ofns3btWpo0aVLtmMOHD+ftt9/m2LFjbNq0id69e5/2/qv68MMPcTgcp5wvKirim2++oVevXgwbNoyFCxcC8M033/Dkk0/y6aef8sknn7Bt2zZfn2uvvZZ169bxxRdfMHz4cP70pz/52nbt2sXq1av5+9//TmpqKklJSWzevJkmTZr4klznCyWJ/ODasDCe+6//4tKgIH+HIiIiIiIiIhe5Xr160alTJ99xRkYG3bt3Jz4+nqKiIgoKCk7p06lTJ2JiYgDo2bMnhbV8wdvtdlNSUkK/fv0AGDVqFNnZ2QA4nU5GjBjBggULCAwMBCAhIYHx48eTkZFBSUmJ7/xPOZ1OCgsLcblcDBkypM73m5SURExMDD/88AMTJ048pX3hwoUMGzYM8CSiTiw5+/zzz0lMTKRVq1Y0btyY3/72t74+e/fuZeDAgTgcDmbMmMHWrVt9bYMHDyYoKAiHw0FFRQWDBg0CwOFw1Prc/KH6Jy317mhFBccqKwlXokhEREREROSi1SOrR41tjZo2qrW9ccvGtbbXVbNmzXy/Z2VlsWrVKj777DOaNm1KYmIix6r58FJwcPB/4mzU6LTLzWqydOlSsrOzWbJkCVOnTmXz5s1MmDCB5ORkli1bRkJCAsuXL+eqq66qtn9KSgq///3vycrKqrbiqTqZmZm0bNmyxnaXy8W3337LX//6VwD2799fbaKsqoceeojx48eTkpJCVlYWU6ZM8bWdeFYBAQEEBQVhjPEdn24fqIamSiI/afPZZ0w+zzKGIiIiIiIicmFr0aJFjXv8gKfqJzw8nKZNm7Jjxw7WrVv3s+cMDQ0lPDyctWvXAvDWW2/Rr18/KisrKSoqIikpienTp+N2uyktLWXXrl04HA7S09OJi4tjx44dNY5999138+STT1a7bOxsfPXVV5SWlrJv3z4KCwspLCxk4sSJuFwuevfuzZo1azh06BDHjx9n0aJFvn5ut9u3cfb8+fPPSSz+oCSRn3QMCdEXzkRERERERKRBRUREkJCQQHR0NGlpaae0Dxo0iPLyciIjI5kwYQLx8fHnZN758+eTlpaG0+kkPz+fyZMnU1FRQWpqKg6Hgx49ejBu3DjCwsKYNWsW0dHROJ1OgoKCGDx4cI3jtmvXjnHjxlXbNm/ePNq1a+f7qboBdU1cLhe33nrrSeeGDh2Ky+WidevWTJkyhT59+pCQkEBkZKTvmilTpvCb3/yGnj171lqldL4z9jzaPDk2Ntbm5OT4O4wGMXTLFraVlbG9Vy9/hyIiIiIiIiINZPv27SclF0TOtereMWNMrrU29nR9VUnkJ12aNGHP0aNUnkdJOhERERERERG5eClJ5CedQ0L4t7Xs//e//R2KiIiIiIiIiIiSRP7SLyyM57t0oUmjRv4ORURERERERESEQH8HcLGKbNaMyCqfGRQRERERERER8SdVEvnRrqNH2aUvnImIiIiIiIjIeUBJIj/qn5/Pk3v2+DsMERERERERERElifypc5Mm7D52zN9hiIiIiIiIyEWipKSE2bNnn3X/WbNmUVZWVm1bYmIiOTk5Zz12TbKysggNDSUmJoarrrqK3//+92c9ljGGRx991Hc8c+b/z97dh9lZ1+eiv9e8ZNYzJASC4R0hw4waSMLwEoRGFMSC1S3V0stDC6W4u7e1rUXdLUU9aO3Z0mJrq8XDFrtP2btWDdVTta1QS90HCsX6AogQhTpkEsAgkGASksxM5u05fzDJBtGA5JmsNWt9Pv+YzMy61y8rubwu7uv7/L4fzgc+8IHnfP+vfvWre/yZN77xjTnttNP2+DPz589/3udsFCVRAx1bFBn2uBkAAAD7yGyWRLPpjDPOyN13351vfetb+dKXvpTbb7/9BeX09PTk85//fDZt2vS8X/NcJdGWLVty5513ZuvWrRkeHn5B52oWSqIG6qvX89jERHZMTTX6KAAAALSBd7/73Vm7dm0GBwdz2WWXJUn+5E/+JCtXrsyKFSvy+7//+0mSHTt25PWvf31OOOGELFu2LH/zN3+Tq6++Oo888kjOOuusnHXWWXt8n9WrV2f58uVZtmxZLr/88iTJ1NRULrnkkixbtizLly/PRz7ykSTJ1VdfneOOOy4rVqzIBRdcsMfcoigyODiYDRs27D7nf/yP/zGnnnpqTjzxxPzd3/1dkuQ73/lOTj311AwODmbFihUZGhpKknR1deWtb33r7vd+uo0bN+b888/PypUrs3Llytx+++1Zv359rr322nzkIx/J4OBgbrvttme97vOf/3ze8IY35IILLsj111+/++vr1q3L6aefnuXLl+eKK67Y/fXt27fn7LPPzkknnZTly5fvPvP69evzspe9LJdcckle8pKX5MILL8xXvvKVrFq1KgMDA/nGN76xx8+mCrabNVBfUSRJ1o2OZtkcGDsDAACgOu/88jtz96N3V5o5eOhgPvraj/7E71911VVZs2ZN7r77qfe96aabMjQ0lG984xspyzLnnXdebr311mzcuDGHH354brjhhiTJ1q1bs3DhwvzZn/1Zbr755rzoRS/6ie/xyCOP5PLLL8+dd96ZAw88MOecc06++MUv5qijjsqGDRuyZs2aJE9N4Ow607p169LT07P7az/J5s2bMzQ0lFe+8pVJkiuvvDKvfvWrc91112XLli059dRT85rXvCbXXntt3vGOd+TCCy/M+Ph4pp42nPFbv/VbWbFiRX7v937vGdnveMc78q53vSuveMUr8tBDD+Xcc8/Nfffdl7e97W2ZP7/KcnEAACAASURBVH/+T3zMbfXq1Xn/+9+fQw45JOeff37e+9737s77jd/4jVx88cW55pprdv98vV7PF77whey///7ZtGlTTjvttJx33nlJkgceeCCf+9znct1112XlypX5zGc+k3/913/N3//93+cP//AP88UvfnGPn8/eMknUQGcsXJjPHXdcjuzpafRRAAAAaEM33XRTbrrpppx44ok56aSTcv/992doaCjLly/PP//zP+fyyy/PbbfdloULFz7vzG9+85s588wzs3jx4nR1deXCCy/Mrbfemr6+vgwPD+e3f/u38+Uvfzn7779/kmTFihW58MIL86lPfSpdXT9+luW2227LCSeckCOOOCLnnntuDj300N3nv+qqqzI4OJgzzzwzY2Njeeihh3L66afnD//wD/OhD30oDz74YIqZIY0k2X///XPxxRfn6quvfsZ7fOUrX8nb3/72DA4O5rzzzsuTTz6Z7du37/HP+thjj2VoaCiveMUr8pKXvCTd3d27S7Dbb789v/RLv5Qk+ZVf+ZXdrynLMu9973uzYsWKvOY1r8mGDRvy2GOPJUmWLFmS5cuXp6OjI8cff3zOPvvs1Gq1LF++POvXr3/efwcvlEmiBjq8pye/ePDBjT4GAAAADbCniZ99pSzLvOc978mv//qvP+t7d911V2688cZcccUVOfvss/P+979/r97rwAMPzLe//e380z/9U6699tp89rOfzXXXXZcbbrght956a/7hH/4hV155Ze69995nlUVnnHFGvvSlL2XdunU57bTT8uY3vzmDg4MpyzJ/+7d/m5e+9KXP+PmlS5fm5S9/eW644Ya87nWvyyc+8Ym8+tWv3v39d77znTnppJPylre8ZffXpqen87WvfS31ev15/5k++9nPZvPmzVmyZEmS5Mknn8zq1atz5ZVXJnnqouwf9elPfzobN27MnXfeme7u7hxzzDEZm1lq1fO0IZKOjo7dv+/o6Mjk5OTzPtcLZZKowf51y5b829atjT4GAAAAbWDBggXZtm3b7t+fe+65ue6663ZPzGzYsCGPP/54HnnkkfT29uaiiy7KZZddlrvuuuvHvv7HOfXUU/Mv//Iv2bRpU6amprJ69eq86lWvyqZNmzI9PZ3zzz8/H/zgB3PXXXdleno6Dz/8cM4666x86EMfytatW/c4vbNkyZK8+93vzoc+9KHd5//Yxz6WsiyTJN/61reSJMPDw+nr68ull16an//5n88999zzjJxFixblzW9+c/7yL/9y99fOOeecfOxjH9v9+12P5O3pz7x69ep8+ctfzvr167N+/frceeedu+8lWrVq1e5ff/rTn979mq1bt+bggw9Od3d3br755jz44IN7/Dz3JSVRg/32Aw/kyib6BwEAAEDrOuigg7Jq1aosW7Ysl112Wc4555z88i//8u4Lln/xF38x27Zty7333rv74uc/+IM/2H3x8lvf+ta89rWv3ePF1YcddliuuuqqnHXWWTnhhBNy8skn5+d//uezYcOGnHnmmRkcHMxFF12UP/qjP8rU1FQuuuiiLF++PCeeeGIuvfTSHHDAAXv8M7ztbW/LrbfemvXr1+d973tfJiYmsmLFihx//PF53/vel+SpCZ9ly5ZlcHAwa9asycUXX/ysnN/5nd95xpazq6++OnfccUdWrFiR4447Ltdee22S5A1veEO+8IUvPOvi6vXr1+fBBx/MaaedtvtrS5YsycKFC/P1r389f/7nf55rrrkmy5cv333RdpJceOGFueOOO7J8+fJ88pOfzMte9rI9/nn3pdqutq0ZnHLKKeUdd9zR6GPsU+evWZP7Rkby3VNPbfRRAAAAmGX33Xdfli5d2uhj0MJ+3L+xWq12Z1mWpzzXa00SNVhfUWR4dDTTTVTWAQAAAO1HSdRgffV6dpZlfjA+3uijAAAAAG1MSdRgfTNr+IZHRxt8EgAAAKCddT33jzCbTtt//3ztpJOybL/9Gn0UAAAAoI0piRpsYVdXXr7//o0+BgAAANDmPG7WBL6wcWM+v3Fjo48BAAAAtDElURO4esOG/OnDDzf6GAAAALS4LVu25L/9t//2gl77ute9Llu2bHneP/+BD3wgH/7wh1/Qez2Xzs7ODA4OZtmyZXnDG97wU53r6c4888yccsr/3gx/xx135Mwzz9zja9avX5/PfOYze/yZj370o6nX69m6dese3/uOO+74qc4725RETaCvXs/w2FijjwEAAECL21NJNDk5ucfX3njjjTnggANm41g/taIocvfdd2fNmjVZtGhRrrnmmhec9fjjj+cf//Efn/fPP5+SaPXq1Vm5cmU+//nPv+BzNYKSqAkcWxR5dHw8I1NTjT4KAAAALezd73531q5dm8HBwVx22WW55ZZbcsYZZ+S8887LcccdlyR54xvfmJNPPjnHH398/uIv/mL3a4855phs2rQp69evz9KlS/Of//N/zvHHH59zzjkno8+xsfvuu+/OaaedlhUrVuRNb3pTNm/enCS5+uqrc9xxx2XFihW54IILkiT/8i//ksHBwQwODubEE0/Mtm3b9ph9+umnZ8OGDbt//yd/8idZuXJlVqxYkd///d9PkuzYsSOvf/3rc8IJJ2TZsmX5m7/5m90/f9lll+XKK698Vu7U1FQuu+yy3Vmf+MQndn+Gt912WwYHB/ORj3zkWa9bu3Zttm/fng9+8INZvXr17q+Pjo7mggsuyNKlS/OmN73pGZ/Zb/zGb+SUU07J8ccfv/vMyVOf+Xve854MDg7mlFNOyV133ZVzzz03xx57bK699to9fi4vhIurm0BfvZ4kWTc2luNtOQMAAGgbZ/7PM5/1tTcf/+b85srfzMjESF736dc96/uXDF6SSwYvyaaRTfnFz/7iM753yyW37PH9rrrqqqxZsyZ33333Uz9/yy256667smbNmixZsiRJct1112XRokUZHR3NypUrc/755+eggw56Rs7Q0FBWr16d//7f/3ve/OY352//9m9z0UUX/cT3vfjii/Oxj30sr3rVq/L+978/f/AHf5CPfvSjueqqq7Ju3br09PTsfmTswx/+cK655pqsWrUq27dvT33mv5l/nKmpqfyv//W/8mu/9mtJkptuuilDQ0P5xje+kbIsc9555+XWW2/Nxo0bc/jhh+eGG25Ikmc8Bnb66afnC1/4Qm6++eYsWLBg99f/8i//MgsXLsw3v/nN7Ny5M6tWrco555yTq666Kh/+8IfzpS996cee6frrr88FF1yQM844I//+7/+exx57LIccckg+/vGPp7e3N/fdd1/uueeenHTSSbtfc+WVV2bRokWZmprK2WefnXvuuScrVqxIkrz4xS/O3XffnXe961255JJLcvvtt2dsbCzLli3L2972tp/42bwQJomaQF9RJEnWPkfzCgAAAFU79dRTdxdEyVPTPSeccEJOO+20PPzwwxkaGnrWa5YsWZLBwcEkycknn5z169f/xPytW7dmy5YtedWrXpUk+dVf/dXceuutSZIVK1bkwgsvzKc+9al0dT01x7Jq1ar8l//yX3L11Vdny5Ytu7/+dKOjoxkcHMyhhx6axx57LD/7sz+b5KmS6KabbsqJJ56Yk046Kffff3+GhoayfPny/PM//3Muv/zy3HbbbVm4cOEz8q644op88IMffMbXbrrppnzyk5/M4OBgXv7yl+eJJ574sZ/Fj1q9enUuuOCCdHR05Pzzz8/nPve5JMmtt966u0hbsWLF7hIoST772c/mpJNOyoknnpjvfOc7+e53v7v7e+edd16SZPny5Xn5y1+eBQsWZPHixc8o1qpikqgJDM6fn0dOPz2HzpvX6KMAAACwD+1p8qe3u3eP339R74uec3Lo+djvaU+03HLLLfnKV76Sf/u3f0tvb2/OPPPMjP2YO3R7enp2/7qzs/M5Hzf7SW644Ybceuut+Yd/+IdceeWVuffee/Pud787r3/963PjjTdm1apV+ad/+qe87GUve8brdt1JNDIyknPPPTfXXHNNLr300pRlmfe85z359V//9We911133ZUbb7wxV1xxRc4+++y8//3v3/29V7/61bniiivyta99bffXyrLMxz72sZx77rnPyLnlllt+4p/n3nvvzdDQ0O7Sanx8PEuWLMnb3/72n/iadevW5cMf/nC++c1v5sADD8wll1zyjM9812fd0dHxjM+9o6PjOe+R+mmZJGoC8zo6clhPT2q1WqOPAgAAQAtbsGDBHu/42bp1aw488MD09vbm/vvvf0Zp8kItXLgwBx54YG677bYkyV//9V/nVa96Vaanp/Pwww/nrLPOyoc+9KFs3bo127dvz9q1a7N8+fJcfvnlWblyZe6///6fmN3b25urr746f/qnf5rJycmce+65ue6667J9+/YkyYYNG/L444/nkUceSW9vby666KJcdtllueuuu56VdcUVV+SP//iPd//+3HPPzcc//vFMTEwkSb73ve9lx44de/wMV69enQ984ANZv3591q9fn0ceeSSPPPJIHnzwwbzyla/cfeH1mjVrcs899yRJnnzyyey3335ZuHBhHnvssZ/qEu2qmSRqEtf94AfZNjWVdxx5ZKOPAgAAQIs66KCDsmrVqixbtiw/93M/l9e//vXP+P5rX/vaXHvttVm6dGle+tKX5rTTTqvkff/qr/4qb3vb2zIyMpK+vr78j//xPzI1NZWLLrooW7duTVmWufTSS3PAAQfkfe97X26++eZ0dHTk+OOPz8/93M/tMfvEE0/MihUrsnr16vzKr/xK7rvvvpx++ulJkvnz5+dTn/pUHnjggVx22WXp6OhId3d3Pv7xjz8r53Wve10WL168+/f/6T/9p6xfvz4nnXRSyrLM4sWL88UvfjErVqxIZ2dnTjjhhFxyySV517vetfs1119/fW688cZn5L7pTW/K9ddfn0svvTRvectbsnTp0ixdujQnn3xykuSEE07IiSeemJe97GU56qijsmrVqhf8Oe+tWlmWDXvzH3XKKaeUd9xxR6OP0RC/sGZN7h8ZyXdPPbXRRwEAAGCW3HfffVm6dGmjj0EL+3H/xmq12p1lWZ7yXK/1uFmT6KvXs25sLNNNVNoBAAAA7UNJ1CT6iiJj09N5dHy80UcBAAAA2pCSqEn01etJkuEXeCM8AAAAwN5QEjWJvqJIV62Wx2ZuTQcAAKA1NdPdwLSWvf23ZbtZk+gvioy98pXprNUafRQAAABmSb1ezxNPPJGDDjooNf/9R4XKsswTTzyR+syTSi+EkqhJdPg/BwAAgJZ35JFH5vvf/342btzY6KPQgur1eo488sgX/HolURP56MMP56GdO/Nn/f2NPgoAAACzoLu7O0uWLGn0MeDHcidRE7lnx45c//jjjT4GAAAA0IaURE3k2KLID8bHMzI11eijAAAAAG1GSdRE+mYul1o3NtbgkwAAAADtRknURPqKIkkyPDra4JMAAAAA7UZJ1ET66vUc3dOTndPTjT4KAAAA0GZsN2sii+fNy/rTT2/0MQAAAIA2ZJIIAAAAACVRs7nywQfzhnvvbfQxAAAAgDajJGoymycm8pXNm1OWZaOPAgAAALQRJVGT6SuKjE1P59Hx8UYfBQAAAGgjSqIm01evJ0mGx8YafBIAAACgnSiJmkxfUSRJhkdHG3wSAAAAoJ0oiZrM0fV6XrFwYeZ3djb6KAAAAEAb6Wr0AXimno6O3HbiiY0+BgAAANBmTBIBAAAAoCRqRu9fty4v/frXG30MAAAAoI0oiZpQT0dHvjc6mtGpqUYfBQAAAGgTSqIm1FevJ0nWjY01+CQAAABAu1ASNaG+okiSDI+ONvgkAAAAQLtQEjWhJTOTRMMmiQAAAIB9REnUhBZ3d+fiQw7JsTMTRQAAAACzravRB+DZarVa/mrp0kYfAwAAAGgjJoma2LbJyUYfAQAAAGgTSqIm9d7h4Rzy1a+mLMtGHwUAAABoA0qiJnVET09Gp6fz2Ph4o48CAAAAtAElUZPqs+EMAAAA2IeURE2qb2az2fDoaINPAgAAALQDJVGTOrqnJ7WYJAIAAAD2ja5GH4Afr97Zmf/rmGOyauHCRh8FAAAAaANKoiZ2xTHHNPoIAAAAQJvwuFkT2zE1le/s2NHoYwAAAABtQEnUxD7y8MNZ9s1vZnRqqtFHAQAAAFqckqiJ7dpwtt7l1QAAAMAsUxI1sb56PYkNZwAAAMDsUxI1sV2TRMOjow0+CQAAANDqlERNbHF3d/br6DBJBAAAAMy6rkYfgJ+sVqvlEy99aV46M1EEAAAAMFuURE3uwkMOafQRAAAAgDbgcbMm9+jOnbnxiSdSlmWjjwIAAAC0MCVRk/t/N27M6++9N4+Njzf6KAAAAEALUxI1ud0bzlxeDQAAAMwiJVGT66vXkyTDo6MNPgkAAADQypRETe6YXSWRSSIAAABgFimJmly9szNHzJtnkggAAACYVV2NPgDPbfVxx+WwefMafQwAAACghSmJ5oAzDjig0UcAAAAAWpzHzeaAtaOj+cQjj2RsaqrRRwEAAABalJJoDvjak0/mbd/7Xta5vBoAAACYJUqiOaDPhjMAAABglimJKvZvW7fmiuHhlGVZWWZfUSSJDWcAAADArFESVeyObdty5UMP5fGJicoyD+7uzn4dHR43AwAAAGaNkqhiAzNTP0MjI5Vl1mq19BWFSSIAAABg1nQ1+gCtpn+mJHpgdDSvqHB1/ReXLcuiLn9dAAAAwOzQOlTs6Ho9nXmqJKrSrnuJAAAAAGaDx80q1t3RkSVFkaGKS6Lv7tiRK4aH88MK7zoCAAAA2EVJNAv6i6LySaL1Y2O58qGH8u8V3nUEAAAAsIuSaBbsKonKsqwss69eT5IM23AGAAAAzAIl0SwYKIo8OTWVjRU+GnbMrpLIhjMAAABgFiiJZsHTN5xVpd7ZmSPmzTNJBAAAAMwKJdEsmI2SKHlqw9mGnTsrzQQAAABIkq5GH6AVHVOvpzOpfMPZjcuXZ7/OzkozAQAAABIl0ayY19GRo+v1yieJ5nf56wIAAABmh8fNZsmuDWdVumf79vzqffdlvcurAQAAgIopiWbJQFFkaGQkZVlWlrl9aiqffOyxfHdkpLJMAAAAgERJNGv6iyJbp6byxMREZZl99XqSZNgkEQAAAFAxJdEsmY0NZ4fMm5fejo4Mj41VlgkAAACQKIlmzUBvb5JqN5zVarX0FYVJIgAAAKBySqJZcky9no5UO0mUJMfNlE8AAAAAVbJTfZb0dHTkxfV65SXR3xx/fKV5AAAAAIlJolk1UBSVPm4GAAAAMFuURLOovygqnyS6Z/v2vPruu3PXtm2V5gIAAADtTUk0i/qLIpsnJ/PDiYnKMrtrtdy8ZUvuGxmpLBMAAABASTSLBooiSbUbzo6p15PEhjMAAACgUkqiWdQ/UxJV+chZ0dmZw+fNy/DYWGWZAAAAAEqiWbSkXk8t1ZZESdJXFCaJAAAAgEp1NfoAraze2ZkX9/RkqOL7g35m//3zyPh4pZkAAABAe1MSzbLZ2HD2oWOPrTQPAAAAwONms2w2SiIAAACAqimJZtlAb2+emJzM5omJyjK/s2NHBr7+9dz0wx9WlgkAAAC0NyXRLJuNDWeLurrywOhohkwoAQAAABVREs2y2SiJDp03L0VHhw1nAAAAQGWURLPs2Ho9taTSqZ9arZa+ej3DY2OVZQIAAADtbVa3m9VqtfVJtiWZSjJZluUps/l+zaje2Zkje3oqv7y6ryiy1iQRAAAAUJFZLYlmnFWW5aZ98D5NazY2nL120aLcPzJSaSYAAADQvvZFSdT2Booin99UbU/2m0ccUWkeAAAA0N5m+06iMslNtVrtzlqt9tZZfq+m1V8U2TQxkS0TE5XmTpdlJqenK80EAAAA2tNsl0SvKMvypCQ/l+S3arXaK3/0B2q12ltrtdodtVrtjo0bN87ycRpjYBY2nD0wMpL9brstn23RzwwAAADYt2a1JCrLcsPM/z6e5AtJTv0xP/MXZVmeUpblKYsXL57N4zRM/yyURIf39GRsejrDLq8GAAAAKjBrJVGtVtuvVqst2PXrJOckWTNb79fM+mahJOrt7Mxh8+ZleGysskwAAACgfc3mxdWHJPlCrVbb9T6fKcvyy7P4fk2rt7MzR/b0ZKjiqZ9ji8IkEQAAAFCJWSuJyrIcTnLCbOXPNf1FUekkUZL01eu5ecuWSjMBAACA9jSbk0Q8TX9R5O83bao0800velGOLYqUZZmZiS0AAACAF0RJtI8MFEUen5jIk5OT2b+rmo/9jYsX540tetk3AAAAsG/N6nYz/rfZ2HBWlmU2jo9n6+RkZZkAAABAe1IS7SOzURI9Oj6eg7/61Xz6sccqywQAAADak5JoHzl2piSqcsPZofPmpd7RYcMZAAAAsNeURPvIfp2dOXzevEoniWq1Wvrq9QyPjVWWCQAAALQnJdE+1F8UlZZESdJXFFlrkggAAADYS0qifWigKDI0MlJp5q5JorIsK80FAAAA2ks1u9h5XvqLIo9NTGTb5GQWdFXz0f8fBx+cFfPnZ6os01WrVZIJAAAAtB+TRPvQrg1nVT4e9jMLF+bXDjssXR3+KgEAAIAXTrOwDw309iapdsPZ5PR07tq2LQ+7vBoAAADYC0qifejYej1JKr28erwsc/Kdd+aTjz1WWSYAAADQfpRE+9D8rq4cOm9epSVRb2dnDp03L8M2nAEAAAB7QUm0jw0URaWPmyVPTSgNe9wMAAAA2AtKon2svygqnSRKkr6iMEkEAAAA7BUl0T7WXxT5wfh4dkxNVZbZV6/n4Z07Mz49XVkmAAAA0F66Gn2AdjNQFEmeurz6hPnzK8n85UMOyaqFC1OrJA0AAABoRyaJ9rH+p5VEVXlJb29+dtGidHf46wQAAABeGK3CPjYbJdFUWebvNm3Kt7dvrywTAAAAaC9Kon1sQVdXDunuztDISGWZtSQXfPe7+etHH60sEwAAAGgvSqIGqHrDWUetliX1eobHxirLBAAAANqLkqgBqi6Jkqc2nA1XnAkAAAC0DyVRAwz09mbD+HhGpqYqy+wrigyPjaUsy8oyAQAAgPahJGqAXZdXr61w8qevXs+2qak8MTFRWSYAAADQPpREDTAbG85+6ZBD8t2VK3NAV1dlmQAAAED70Cg0wK6SaKjCkuiQefNyyLx5leUBAAAA7cUkUQMs7OrK4u7uSieJyrLMNRs25Cs//GFlmQAAAED7UBI1SNUbzmq1Wv7r+vW5/vHHK8sEAAAA2oeSqEEGiqLSx82S5NiiyNqxsUozAQAAgPagJGqQ/qLI93fuzOjUVGWZfUWR4YqLJwAAAKA9KIkaZNfl1cMVTv701et5eOfOjE9PV5YJAAAAtAclUYMM7NpwNjJSWWZfUaRM8pBHzgAAAICfUlejD9Cujp0piaq8vPoXFy/OL7zoRVnQ5a8VAAAA+OloExrkwO7uHNTVVWlJtF9nZ2VZAAAAQHvxuFkDDfT2Vr7h7PfXrctfP/popZkAAABA61MSNVB/UVQ6SZQkn9u4MV/ctKnSTAAAAKD1KYkaqL8o8vDOnRmbmqoss69er3RjGgAAANAelEQNNDCzjazKUqevKDI8OpqyLCvLBAAAAFqfkqiB+mdhw1lfvZ4np6byw8nJyjIBAACA1qckaqBZKYmKIgs7O/ODnTsrywQAAABaX1ejD9DOFnV3Z1FXV6Ubzt5w0EHZcsYZleUBAAAA7cEkUYNVveGsVqtVlgUAAAC0DyVRg1VdEiXJbw8N5aoHH6w0EwAAAGhtSqIGGyiKPDQ2lp3T05Vl3rVtW27avLmyPAAAAKD1KYkarL8oMp1kXYXTREvq9QxXPJ0EAAAAtDYlUYMN9PYmSaWXV/cVRR7euTPjFU4nAQAAAK1NSdRg/UWRJJXeS9RXr2c6yUNjY5VlAgAAAK1NSdRgi7q6ckBXV6Ul0Ut7e7N8v/2ybWqqskwAAACgtXU1+gDtrlarZaAoKn3c7PSFC3PPypWV5QEAAACtzyRRE+gvikoniQAAAAB+WkqiJtBfFHlwbKzSi6Yvvu++vO3f/72yPAAAAKC1KYmawEBRZDrJugovmn5iYiLf2LatsjwAAACgtSmJmsCsbDgriqwdHU1ZlpVlAgAAAK1LSdQEZqUkqtfz5NRUNk9OVpYJAAAAtC4lURN4UXd3FnZ2ZmhkpLLMvpniadiF2AAAAMDzoCRqArVarfINZ0t7e/O6RYvSUatVlgkAAAC0rq5GH4Cn9BdF7qjwoumX9PbmhhUrKssDAAAAWptJoiYx0Nub9WNjmZierjTXxdUAAADA86EkahL9RZGpJOvHxirL/IU1a3LuPfdUlgcAAAC0LiVRk5iNDWe9HR0ZcnE1AAAA8DwoiZrEwExJVGWp01cUeWgWHmEDAAAAWo+SqEks7u7Ogs7OSieJ+ur1TCd5aOfOyjIBAACA1qQkahK1Wi39RVFtSTQznTTskTMAAADgOSiJmshAUVT6uNlLe3vz64cdlsXd3ZVlAgAAAK1JSdRE+osi6yu8Q+iQefNy7UtfmsEFCyrJAwAAAFqXkqiJ9BdFJsuy0juEpsoyT0xMVJYHAAAAtCYlURPZveFsZKSyzDeuWZOf/fa3K8sDAAAAWpOSqIn0z5REVV5efUy9nrWjoynLsrJMAAAAoPUoiZrIIfPmZX5nZ7Ubzur1PDk1lc2Tk5VlAgAAAK1HSdREarVa+ivecNY3M500XGEmAAAA0HqURE2mvygqnyRKkuGxscoyAQAAgNajJGoy/UWRdWNjmZyeriSvryjywSVLcvx++1WSBwAAALSmrkYfgGcaKIpMlGUe2rlz96Nie2O/zs78n0cfXcHJAAAAgFZmkqjJzMaGs8fHx7Nm+/bK8gAAAIDWoyRqMrNREr3rgQfyhjVrKssDAAAAWo+SqMkcNm9eejs6Kt9wBTMQGgAAIABJREFU9tDYWCYquucIAAAAaD1KoiZTq9VmZcPZdJKHd+6sLBMAAABoLUqiJlR5STTzCNtwhZkAAABAa1ESNaGBosjw6GimyrKSvL56PUkyPDZWSR4AAADQepRETai/KDJelnm4olLniJ6e/M+XvSyvOfDASvIAAACA1tPV6APwbE/fcHbMzK/3Rketll899NC9zgEAAABal0miJjTQ25sklW44u3/Hjnz5iScqywMAAABai5KoCR02b16Kjo5KL6/+vzdsyC/dd19leQAAAEBrURI1oY5aLcfOwoazLZOT2TwxUVkmAAAA0DqURE1qoCgqfdzMhjMAAABgT5RETaq/KLJ2dDRTZVlJXt/MBdjDFRZPAAAAQOtQEjWp/qLIeFlmw86dleQtMUkEAAAA7EFXow/AjzcwM/kzNDqaF88UPHtjQVdXbhkczNKZzWkAAAAAT2eSqEn1z5REVV5e/aoDDsjB8+ZVlgcAAAC0DiVRkzqipyf1jo5KS6JvPPlkrt2wobI8AAAAoHUoiZpUR62WY+v1DI2MVJb595s25e1DQ5mYnq4sEwAAAGgNSqIm1l8UlU4S9RVFppI8XNFl2AAAAEDrUBI1sf6iyNqxsUyXZSV5fbs2nFVYPAEAAACtQUnUxAZ6ezM2PZ0NFU3+9M1chj08NlZJHgAAANA6lERNrOoNZ0f09GRerZa1JokAAACAH9HV6APwkw3MlERDo6M568AD9zqvs1bLv596ag7v6dnrLAAAAKC1KIma2JE9Pemp1Sq9vPqYmeIJAAAA4Ok8btbEOmq19FW84ezmzZvzuw88UFkeAAAA0BqURE1uoCgyVGFJdNf27fnT738/mycmKssEAAAA5j4lUZPrL4qsHR3NdFlWktdXryex4QwAAAB4JiVRk+svioxOT+cH4+OV5PXN3Ek0bMMZAAAA8DRKoia3e8PZyEgleUtMEgEAAAA/hpKoyfXPlERVXV69f1dXFnd3Z2NFk0kAAABAa+hq9AHYs6Pq9cyr1SrdcPb900/PvA79IAAAAPC/aQqaXGetlr6KN5wpiAAAAIAfpS2YA/qLotJJon984on8wpo1mZyeriwTAAAAmNuURHPArpKoLMtK8n4wPp4vbNqUh3furCQPAAAAmPuURHPAQFFkZHo6P6josuk+G84AAACAH6EkmgOq3nDWN5M3XOEjbAAAAMDcpiSaA6ouiY7o6Ul3rZa1SiIAAABghpJoDnjxTKlT1YazzlotJy9YkI5arZI8AAAAYO7ravQBeG5dHR1ZUq9XuuHs3046qbIsAAAAYO4zSTRH7NpwBgAAADAblERzxEBRZGhkJGVZVpL395s25ZQ77siWiYlK8gAAAIC5TUk0R/QXRXZMT+ex8fFK8ibLMndu357hsbFK8gAAAIC5TUk0R1S94ayvXk+SDHuEDQAAAIiSaM4Y6O1Nkso2nPXNlE4miQAAAIBESTRnHN3Tk65arbJJov27uvKi7m6TRAAAAEASJdGc0dXRkWPq9Uo3nL1u0aIc0dNTWR4AAAAwd3U1+gA8fwNFUdnjZknyV0uXVpYFAAAAzG0mieaQ/qLIA6OjKcuy0UcBAAAAWoySaA7pL4psm5rKxomJSvL+ftOmHHL77VnvXiIAAABoe0qiOWRgZiNZVY+cLejszOMTEzacAQAAAEqiuaR/piSq6vLqvpk8G84AAAAAJdEccky9ns5UVxId2dOTrlota00SAQAAQNtTEs0h3R0dOaZez9DISCV5nbVajqnXTRIBAAAA6Wr0Afjp7NpwVpULDz44C7v8MwAAAIB2px2YY/qLIl978smUZZlarbbXeR9YsqSCUwEAAABzncfN5piB3t5snZrKpomJyjLHpqYyVZaV5QEAAABzj5Jojql6w9mXNm1K72235Z7t2yvJAwAAAOYmJdEcU3VJdERPT8okwzacAQAAQFtTEs0xS+r1dCQZqqgk6pspnWw4AwAAgPamJJpj5nV05Oh6vbJJooVdXTmoq8skEQAAALQ5JdEc1F8UlZVEyVPTRCaJAAAAoL11NfoA/PQGiiKfefzxlGWZWq2213m/cfjh2fsUAAAAYC5TEs1B/UWRLZOT+eHkZA7q7t7rvLccdlgFpwIAAADmMo+bzUFVbzibnJ7O2tHRjExNVZIHAAAAzD1KojloYKYkGhoZqSTv1q1b0//1r+drTz5ZSR4AAAAw9yiJ5qAlRZFaqpsk6qvXk8Tl1QAAANDGlERzUE9HR17c01NZSXRkT0+6arUMj41VkgcAAADMPUqiOWqgtzdDFZVEXR0dObqnxyQRAAAAtDEl0RzVXxSVTRIlSV9RmCQCAACANtbV6APwwgwURX44OZkfTkxkUXf3Xuf9zlFHZXx6uoKTAQAAAHORkmiO6p/ZcPbA6GhOraAkOnfRor3OAAAAAOYuj5vNUU8viaqwfXIyt2zenE3j45XkAQAAAHOLkmiO6qvXU0squ7x6aHQ0Z33727l169ZK8gAAAIC5RUk0R9U7O3NUT09lk0R9M5NJNpwBAABAe1ISzWFVbjhb2NWVRV1dWWvDGQAAALQlJdEcNlAUGRoZqSyvryhMEgEAAECbUhLNYf1FkScmJ7N5YqKSvL56PcMmiQAAAKAtdTX6ALxwuzacrR0dzSnd3Xud954XvzjjZbnXOQAAAMDcoySawwZ6e5M8tZnslP333+u8wQUL9joDAAAAmJs8bjaH9dXrSVLZ5dVbJiby148+6l4iAAAAaENKojms6OzMkT09lZVEmycnc/H99+fmLVsqyQMAAADmDiXRHDdQFBmqqCQ6qqcnXbWaSSIAAABoQ0qiOa6/KCqbJOrq6MjRPT02nAEAAEAbUhLNcf1FkY0TE9k6OVlJXl9RmCQCAACANjTrJVGtVuus1WrfqtVqX5rt92pHA0WRpLrLq/vqdZNEAAAA0Ib2xSTRO5Lctw/epy31V1wSve+YY/Ktk0+uJAsAAACYO2a1JKrVakcmeX2S/2c236edHVtxSXRET0+OrNcryQIAAADmjtmeJPpokt9LMj3L79O2ejs7c8S8eRkaGakkb8vERP7owQdz17ZtleQBAAAAc8OslUS1Wu0/JHm8LMs7n+Pn3lqr1e6o1Wp3bNy4cbaO09Kq3HCWJO9dty43b9lSWR4AAADQ/GZzkmhVkvNqtdr6JNcneXWtVvvUj/5QWZZ/UZblKWVZnrJ48eJZPE7rqrIkOqC7Owd2dWWtDWcAAADQVmatJCrL8j1lWR5ZluUxSS5I8v+VZXnRbL1fOxvo7c1jExN5cnKykry+ej3DSiIAAABoK/tiuxmzbNeGs6qmf/qKIsNjY5VkAQAAAHPDPimJyrK8pSzL/7Av3qsd9Ve84ayvXs+GnTszXZaV5AEAAADNzyRRC9hVEg1VVBK975hjsvUVr0hHrVZJHgAAAND8uhp9APbefp2dOWzevMomifbr7KwkBwAAAJg7TBK1iCo3nG2bnMxvfu97+acf/rCSPAAAAKD5KYlaxEBRVPa4WdHRkb945JHctmVLJXkAAABA81MStYj+osij4+PZPjm511ldHR05ul634QwAAADaiJKoRey6vHptRcVOX1FkuKLJJAAAAKD5KYlaxMCuDWcjI5Xk9ZkkAgAAgLaiJGoRx86URFVdXj1QFCk6OjI2NVVJHgAAANDclEQtYkFXVw7p7q6sJPrdF784D55+euqdnZXkAQAAAM1NSdRCBnp7K9twBgAAALQXJVEL6S+KyiaJdk5P5w333pu/evTRSvIAAACA5qYkaiH9RZFHxsezo4J7hHo6OnL71q35xpNPVnAyAAAAoNkpiVrIrg1nayuaJrLhDAAAANqHkqiF9Fe84ayvKDLsjiMAAABoC0qiFlJ5SVSvZ/3YWKbKspI8AAAAoHkpiVrI/l1dObi7u7INZyvmz8/KBQvy5ORkJXkAAABA8+pq9AGoVpUbzn75kEPyy4ccUkkWAAAA0NxMErWYKksiAAAAoH0oiVrMQFHk+zt3ZmRqaq+zyrLMKXfckT988MEKTgYAAAA0MyVRi9l1eXUVW8lqtVo2T05mzY4de50FAAAANDclUYupfMNZUVRSOAEAAADNTUnUYnaVRFVtOOur1zM8NlZJFgAAANC8lEQt5oDu7ryou7vSSaKNExPZNjlZSR4AAADQnJRELajKDWenLFiQCw4+OCPT05XkAQAAAM2pq9EHoHoDRZFbtmypJOvsAw/M2QceWEkWAAAA0LxMErWg/qLIwzt3ZnRqqrLMSZNEAAAA0NKURC1oYOby6nUVXTjd/7Wv5Z0PPFBJFgAAANCclEQtqOoNZwu7umw4AwAAgBanJGpBu0qiKjecDVeUBQAAADQnJVELOrC7Owd1dWVoZKSSvL56PevGxjJdlpXkAQAAAM1HSdSi+oui0kmi8bLMIzt3VpIHAAAANB8lUYuqsiQ6bf/987tHHZXOWq2SPAAAAKD5dDX6AMyOgd7efObxxzM2NZV6Z+deZZ0wf35OmD+/opMBAAAAzcgkUYvqL4qUSdZVtJVsx9RUNo6PV5IFAAAANB8lUYuqesPZ8m9+M+984IFKsgAAAIDmoyRqUQMzJdFQVZdX1+sZrmgqCQAAAGg+SqIWtai7Owd2dVW64Wy4oiwAAACg+SiJWliVG8766vU8PjGR7ZOTleQBAAAAzUVJ1MIGiqK6x81mHl+r6iJsAAAAoLkoiVpYf1HkobGx7Jye3uusl++/f/68vz+Lu7srOBkAAADQbJRELay/KDKdZH0F0z9H1+u59Mgjc2hPz94fDAAAAGg6SqIWtnvD2chIJXkPjIzkuzt2VJIFAAAANBclUQvrnymJqrq8+s3f/W5+d+3aSrIAAACA5qIkamEHdXdnYWdnpRvOhivKAgAAAJqLkqiF1Wq1DPT2VrrhbN3YWKbLspI8AAAAoHkoiVpcf1FUOkk0XpZ5ZOfOSvIAAACA5qEkanH9RZH1Y2MZn57e66xjZ+44Gq5gWxoAAADQXJRELW6gKDKdZH0Fxc7JCxbkC8cfn+P322/vDwYAAAA0FSVRi6tyw9mi7u68cfHiHNTdvddZAAAAQHNRErW4KkuiJPnq1q35/zZvriQLAAAAaB5djT4As2txd3f27+ysbMPZ+9aty8j0dP7twAMryQMAAACag0miFler1ardcFYUGa4oCwAAAGgeSqI2UGlJVK/n8YmJbJ+crCQPAAAAaA5KojYwUBRZNzqaienpvc7qm7njaF0F29IAAACA5qEkagP9RZGpJA9WUOz01etJkmElEQAAALQUJVEbqHLD2bL99ssdJ5+c17i4GgAAAFqKkqgNDPT2JkklG86Kzs6cvGBB9uvs3OssAAAAoHkoidrAwd3dmd/ZWdnl1f+waVM++eijlWQBAAAAzUFJ1AZqtVqlG87+6tFH80cPPVRJFgAAANAclERtYqAoKnncLHlqw9m60dFMl2UleQAAAEDjKYnaRH9RZN3YWCanp/c6q69ez86yzA/Gxys4GQAAANAMlERtor8oMlmWeWjnzr3O6pvZlra2oskkAAAAoPGURG1iYKbYqeKRs756PUmybmxsr7MAAACA5tDV6AOwb/TPlEQPjI7m3L3MWlIUefRnfiYHd3fv/cEAAACApqAkahOHzpuX/To6Ktlw1lmr5ZB58yo4FQAAANAsPG7WJmq1WvqLIkMjI5Xk/fWjj+a/rl9fSRYAAADQeEqiNtJfFJVMEiXJv2zZkms2bKgkCwAAAGg8JVEb6S+KDI+NZaos9zqrryjy2MREdkxNVXAyAAAAoNGURG1koLc3E2WZhyrYSrZ7w1lFk0kAAABAYymJ2sjTN5ztrWNnsoYrKJwAAACAxlMStZEqS6K+osi8Wi1PTEzsdRYAAADQeF2NPgD7zuHz5qXo6MhQBSXRoq6ujL7ylemo1So4GQAAANBoSqI2UqvVKttwVqvVoh4CAACA1uFxszZTVUmUJNdu2JC33H9/JVkAAABAYymJ2sxAUWTt6GimynKvs9aOjeX6xx/PdAVZAAAAQGMpidpMf1FkvCzz/Z079zqrr17P2PR0Hh0fr+BkAAAAQCMpidrMQMUbzpJkuKLH1wAAAIDGURK1mf6ZYmdoZGSvs/rq9SRPPXYGAAAAzG1KojZzeE9P6h0dlUwSHV2v59h6PaU7iQAAAGDO62r0Adi3Omq19BdFhiooieZ1dOSB006r4FQAAABAo5kkakP9RVHJJBEAAADQOpREbai/KLJ2dLSS1fUfefjhrLrrrgpOBQAAADSSkqgNDRRFdpZlvr9z515njUxP56tPPpmRqakKTgYAAAA0ipKoDe3acFbFI2e7Npyts+EMAAAA5jQlURuqsiQ6diZr2B1HAAAAMKcpidrQkT096anVKtlwtmuSaNgkEQAAAMxpSqI21FGr5diKNpwd1N2d1y5alMXd3RWcDAAAAGiUrkYfgMbor6gkqtVq+ccVKyo4EQAAANBIJona1MBMSTRdlpXklRXlAAAAAI2hJGpT/UWRsenpPLJz515n/fFDD+VFt99eWeEEAAAA7HtKojZV5Yaz+Z2d+eHkZB4dH9/rLAAAAKAxlERtaqC3N0mq3XBWQRYAAADQGEqiNnVkT0/m1WqVTBL1zUwlDY+N7XUWAAAA0BhKojbVWaulr6INZ0fX66nFJBEAAADMZUqiNjZQFJU8btbT0ZG3H3FEVsyfX8GpAAAAgEboavQBaJz+oshXNm9OWZap1Wp7lXX1wEBFpwIAAAAawSRRG+svioxOT+cHFWwlK8syP5yYqOBUAAAAQCMoidrYwMyF01U8cvbHDz+cg26/PSNTU3udBQAAAOx7SqI21j9TElVxefWLe3qSJOttOAMAAIA5SUnUxo7q6Ul3rVZJSdQ3UzitteEMAAAA5iQlURvr6uhIX72eoZGRvc7qq9eTJMMmiQAAAGBOUhK1uf6iqGSS6EXd3Znf2Zlhk0QAAAAwJ3U1+gA0Vn9R5JYtW1KWZWq12gvOqdVq+a/HHJPj9tuvwtMBAAAA+4qSqM0N9PZmx/R0Hh0fz2Ezl0+/UO886qiKTgUAAADsax43a3NVbjjbMTWVu7dty3RZ7nUWAAAAsG8pidpclSXR/3z00Zx45515bHx8r7MAAACAfUtJ1OaO/v/Zu/Pwuus67/+vz9m/2fPtvmUPZWmB0hZUXBmwGcdbx/13Ozi3My4343qhoMCoCC6ggDqO4jbe48yIzvhD5xYdbd1wAFnaUuhCW5omTbqkTZuc7MnJ2b73HwkIUmiS811OkufjunIBOd/zfr//KFwXr+vz/bzjcUWMUasLIVEjG84AAAAAAJi1CInmuUgopPpEwpWTRA2Tp5LYcAYAAAAAwOxDSAQ1WZYrIVFtIiEjThIBAAAAADAbERJBzZal1rExOQVeOB0PhbQyHuckEQAAAAAAs1Ak6AEQvCbL0nAup5OZjJbEYgXV+mpTk5bF4y5NBgAAAAAA/EJIhGdtOCs0JPrLRYvcGAkAAAAAAPiM182g5smQqHV0tOBa3em0ftbTo/F8vuBaAAAAAADAP4REUG0iobDkyuXVv+3r0+v27OFeIgAAAAAAZhlCIigaCqkukXAlJGpIJCRJbYREAAAAAADMKoREkCQ1l5So1Y2QaPLVtfZUquBaAAAAAADAP4REkDRxefXBsTE5jlNQnUXRqEpDIV43AwAAAABgliEkgqSJkGgwl1NPJlNQHWOMGiyLk0QAAAAAAMwykaAHQHF4esPZ2JgWxWIF1fru6tWqjvBHCwAAAACA2WRKJ4mMMR82xlSYCd81xuwwxrza6+Hgn6bJkMiNy6s3VlSoqaSk4DoAAAAAAMA/U33d7G8dxxmU9GpJ1ZLeIelWz6aC7+oSCYXkTkjUmUrpzmPH1Ffgq2sAAAAAAMA/Uw2JzORfXyPp3xzHeeIZv8McEAuFVJdIuLLhbO/IiN7f2qp9o6MuTAYAAAAAAPww1ZDoUWPMrzQREm0xxpRLyns3FoLw1IazQjVMvrrGhjMAAAAAAGaPqYZE75J0naSNjuOMSopK+hvPpkIgmixLraOjchynoDq18biMxIYzAAAAAABmkamGRC+W9KTjOP3GmCslfULSgHdjIQjNlqWBXE69Bd4llAiHtSIe5yQRAAAAAACzyFRDom9IGjXGXCDpo5LaJP2rZ1MhEG5uOGtIJDhJBAAAAADALDLVkCjrTLyD9HpJX3Mc5+uSyr0bC0Fonlxb70ZI9P1zztHP164tuA4AAAAAAPBHZIrPDRljrpf0DkkvM8aENHEvEeaQukRCIcmVDWerEonCBwIAAAAAAL6Z6kmit0kal/S3juOckLRS0m2eTYVAxEMh1SQSrpwkOjg6quvb23WYV84AAAAAAJgVphQSTQZDd0mqNMa8VlLKcRzuJJqDmi3LlZNEPZmMbj18WLtHRlyYCgAAAAAAeG1KIZEx5q2Stkp6i6S3SnrEGPNmLwdDMJosy52LqycvwWbDGQAAAAAAs8NU7yT6e0kbHcc5KUnGmEWSfiPpbq8GQzCaLEt92aySmYzs6MyvnVoUjao0FGLDGQAAAAAAs8RU7yQKPRUQTeqdxncxizRPngAq9JUzY4waLIuTRAAAAAAAzBJTPUm02RizRdIPJ//5bZJ+4c1ICFLTZEh0cGxMl1RUFFSrIZHQiXTajbEAAAAAAIDHphQSOY5zrTHmTZIunfzVtx3H+U/vxkJQ6hMJGcmVe4n+/dxzFQ9x4AwAAAAAgNlgqieJ5DjOjyX92MNZUAQS4bBq4nG1jo66UgsAAAAAAMwOL3jMwxgzZIwZPM3PkDFm0K8h4S+3Npy1jo7qyr17tXt42IWpAAAAAACAl14wJHIcp9xxnIrT/JQ7jlPYhTUoWm6FRHlJd508qZ2ERAAAAAAAFD0ujMFzNJeUqDebVV8mU1Cd2nhcRlJ7KuXOYAAAAAAAwDOERHiOZ244K0QiHNaKeFztLpxKAgAAAAAA3iIkwnO4FRJJUkMiwUkiAAAAAABmAUIiPEdjIiEjqdWFkGhNaalixhQ+FAAAAAAA8FQk6AFQfBLhsFbG466cJPr6WWe5MBEAAAAAAPAaJ4lwWm5tOAMAAAAAALMDIRFOq9myXHndrG1sTK947DHd29fnwlQAAAAAAMArhEQ4rSbLUk8mo/5MpqA6ZeGw7hsY0BMjIy5NBgAAAAAAvEBIhNN6asNZW4GbyRZHoyoJhdhwBgAAAABAkSMkwmk1T4ZEraOjBdUxxqjBstTO/UYAAAAAABQ1QiKcVsNkSOTG5dUNiQQniQAAAAAAKHKRoAdAcSoJh7UiFnMlJLq0slJRY1yYCgAAAAAAeIWQCM+ruaTElQ1nH6upcWEaAAAAAADgJV43w/NqsixXThIBAAAAAIDiR0iE59VkWTqZyWgwmy2ozpFUSg0PP6x/7+52aTIAAAAAAOA2QiI8r2aXLq9eFI3qUCrlyqtrAAAAAADAG4REeF5NLoVEiclLsNlwBgAAAABA8SIkwvNqdCkkkqQGy1I7J4kAAAAAAChahER4XqXhsJbHYq68JtaQSHCSCAAAAACAIhbxqrAxJiHpPknxyT53O45zo1f94A23NpxdXl2tRCgkx3FkjHFhMgAAAAAA4CYvTxKNS7rMcZwLJF0oqcUY8yIP+8EDboVEVy5dqm+uXk1ABAAAAABAkfIsJHImDE/+Y3Tyx/GqH7zRbFk6kU5rKJstuFbecTSez7swFQAAAAAAcJundxIZY8LGmMclnZT0a8dxHjnNM+81xmw3xmw/deqUl+NgBp7acNZW4GminnRa1n336TtdXW6MBQAAAAAAXOZpSOQ4Ts5xnAslrZR0sTFmzWme+bbjOBscx9mwaNEiL8fBDDS5tOFsQTSqsDFcXg0AAAAAQJHyZbuZ4zj9ku6V1OJHP7jnqZCo0A1nxpiJDWcu3G8EAAAAAADc51lIZIxZZIypmvx7S9IVkvZ71Q/eKItEtDQWc+Xy6gbL4iQRAAAAAABFysuTRMsk3WuM2SVpmybuJPq5h/3gEbc2nD11kshxuL8cAAAAAIBiE/GqsOM4uySt86o+/NNsWdqcTBZc57ULFmhhNKqs4yhqjAuTAQAAAAAAt3gWEmHuaLIsHU+nNZLLqTQcnnGdy21bl9u2i5MBAAAAAAC3+HJxNWa3py6vbivwlTPHcXRifFw96bQbYwEAAAAAABcREuGMml3acJbK57XsoYf0ja4uN8YCAAAAAAAuIiTCGTVOhkSFXl5thcNaHoux4QwAAAAAgCJESIQzqohEtDgadWfDmWWp3YU6AAAAAADAXYREmJJmy1Lr6GjBdRoSCU4SAQAAAABQhAiJMCVNluXaSaJj4+NK5XIuTAUAAAAAANwSCXoAzA5NlqV/6e7WaC6nknB4xnVev2CBauNxOS7OBgAAAAAACkdIhClpLimRJLWNjWltWdmM61xYXq4Ly8vdGgsAAAAAALiE180wJU0ubTjLO462DQ7qoAv3GwEAAAAAAPcQEmFK3AqJjKQrdu7UF44ccWEqAAAAAADgFkIiTEllJKJF0ahaCw2JjNGfVVdrSzIpx+FmIgAAAAAAigUhEabMrQ1nm2xbR8bHtY9XzgAAAAAAKBqERJiyZssq+CSRNBESSdLmZLLgWgAAAAAAwB2ERJiyJsvS0fFxjeVyBdWpTSR0TkmJft3X59JkAAAAAACgUJGgB8Ds8dTl1e2plM4rLS2o1t3nnaeaeNyNsQAAAAAAgAs4SYQpa54MiVpduEvo3NJSlUXIKAEAAAAAKBaERJiyxsmQyI3LqyXp9sOH9bWjR12pBQAAAAAACkNIhCmrjka1IBJxLST6bX+/vnbsmCu1AAAAAABAYQiJMC3NJSWubDiTpBbb1pNjY+pwqR4AAAAAAJg5QiJMS5NluXaSqMW2JUlb2HIGAAAAAEDgCIkwLU2WpSPj40rlcgWC7EnBAAAgAElEQVTXOsuyVJdIaHMy6cJkAAAAAACgEIREmJZmy5IjqT2VKriWMUZ/uXChIsYUPhgAAAAAACgIO8gxLU3P2HB2bmlpwfW+3NRUcA0AAAAAAFA4ThJhWp4ZErkpnc+7Wg8AAAAAAEwPIRGmxY5GZUcirm04k6SrnnxSL9mxw7V6AAAAAABg+giJMG1ubjiTpNpEQo8OD+vE+LhrNQEAAAAAwPQQEmHa3A6JWmxbkvSrvj7XagIAAAAAgOkhJMK0NVuWDqdSGnfpHqELysq0JBrV5mTSlXoAAAAAAGD6CIkwbU2WpbykQy6dJgoZo1fbtn6VTCrnOK7UBAAAAAAA0xMJegDMPs/ccHZ2aakrNd+1bJleVFGhrOMobIwrNQEAAAAAwNQREmHamktKJMnVDWevqKrSK6qqXKsHAAAAAACmh9fNMG12JKKqSMTVy6sl6cT4uH7e0+NqTQAAAAAAMDWERJg2Y4zrG84k6c6uLr1+zx71ZTKu1gUAAAAAAGdGSIQZabYsV183k6RNtq28pN/29blaFwAAAAAAnBkhEWakybLUmUopnc+7VvOS8nJVhsPanEy6VhMAAAAAAEwNIRFmpMmylJfUkUq5VjMSCukK29bmZFKO47hWFwAAAAAAnBkhEWak2bIkubvhTJI2VVfrWDqtAy7XBQAAAAAALywS9ACYnZomQyK3L69+y+LF2mTbWpVIuFoXAAAAAAC8MEIizMjCaFQV4bDrIVFlJKLKCH8sAQAAAADwG6+bYUaMMRMbzkZHXa/90MCA3rxnj0ZyOddrAwAAAACA0yMkwow1WZbrJ4kkaTiX0497evT7/n7XawMAAAAAgNMjJMKMNVmWOlIpZfJ5V+u+rLJSViikLcmkq3UBAAAAAMDzIyTCjDWXlCgnqSOVcrVuIhzWq6qqtJmQCAAAAAAA3xASYca82nAmSZtsW61jY2r3oDYAAAAAAHguQiLMmJchUYtta11ZmU6m067XBgAAAAAAz8WucczY4mhU5eGwWj0Iic4qKdGODRtcrwsAAAAAAE6Pk0SYMWOMZxvOnjKezyvr8sXYAAAAAADguQiJUBAvQ6KHBwZkP/CA7h8Y8KQ+AAAAAAD4I0IiFKTZsnQolfLktM95paVKO462sOUMAAAAAADPERKhIE2WpazjqHN83PXa5ZGIXlpZqc2ERAAAAAAAeI6QCAXxcsOZNLHlbOfIiI57EEIBAAAAAIA/IiRCQZonQ6LW0VFP6rfYtiTpV319ntQHAAAAAAATCIlQkCWxmEpDIc9OEp1fWqrP19frkvJyT+oDAAAAAIAJkaAHwOxmjPF0w5kxRtfX1npSGwAAAAAA/BEniVCw5pIStXoUEknSeD6vX/T2qs3DHgAAAAAAzHeERChYk2XpUCqlbD7vSf2hbFav3b1b3+/u9qQ+AAAAAAAgJIILmixLGcfREY82kC2MxbSxvFxbkklP6gMAAAAAAEIiuODpDWcevg7WYtt6ZHBQyUzGsx4AAAAAAMxnhEQoWNNkSOTV5dXSREiUl/Sbvj7PegAAAAAAMJ8REqFgy2IxlYRCnoZEG8vLVR2J6L7+fs96AAAAAAAwn0WCHgCznzFGTZbl6etmkVBIO9avV00i4VkPAAAAAADmM04SwRVNluXpSSJJqrMshYzxtAcAAAAAAPMVIRFc0WxZah8bU85xPOuRzef1dwcO6P8cP+5ZDwAAAAAA5itCIriiybKUdhwdSaU86xEJhfSHgQH9oLvbsx4AAAAAAMxXhERwhR8bziRpk23r/oEBDWeznvYBAAAAAGC+ISSCK5pLSiTJ08urJanFtpV2HP2eLWcAAAAAALiKkAiuWBaLyQqFPD9J9NLKSpWEQtqcTHraBwAAAACA+YaQCK4IGaNGHzacxUMhvX3JElVHo572AQAAAABgvokEPQDmjmbL0v7RUc/7fGf1as97AAAAAAAw33CSCK5psiy1jY0p5zie93IcRwNcXg0AAAAAgGsIieCaJstS2nF0bHzc814vf/xxXblvn+d9AAAAAACYLwiJ4Jpmy5Lk/YYzSbqgtFS/6+vTeD7veS8AAAAAAOYDQiK4pmkyJPL68mpJ2mTbGs3n9YeBAc97AQAAAAAwHxASwTUr4nElQiFfQqJXVVUpaow2J5Oe9wIAAAAAYD4gJIJrQsaoMZFQqw8bzsoiEb2sspKQCAAAAAAAl0SCHgBzS5Nl+XKSSJKuq6nRWD4vx3FkjPGlJwAAAAAAcxUhEVzVZFna0tenvOMo5HFwc4Vte1ofAAAAAID5hNfN4KrmkhKl8nkdGx/3pd+u4WH9+NQpX3oBAAAAADCXERLBVX5uOJOkrxw9qvc8+aSy+bwv/QAAAAAAmKsIieAqv0OiFttWXzarbUNDvvQDAAAAAGCuIiSCq1bF44obo1afQqLLq6sVkrSFLWcAAAAAABSEkAiuChmjBh83nNnRqC6uqNBmQiIAAAAAAApCSATXNfkYEkkTr5ztHhnRcDbrW08AAAAAAOYaQiK4rnkyJMo7ji/9PrxihU5deqnKIhFf+gEAAAAAMBcREsF1TZalsXxex9NpX/pVRaMqCYd96QUAAAAAwFxFSATX+b3hTJLuPnlSr9m1S45Pp5cAAAAAAJhrCInguubJkKh1dNS3nsO5nH6ZTGrXyIhvPQEAAAAAmEsIieC6VYmEosb4epJok21LElvOAAAAAACYIUIiuC5sjBoSCV9DomXxuC4oLSUkAgAAAABghgiJ4InmkhK1+hgSSROnif4wMKChbNbXvgAAAAAAzAWERPBEk2Xp4NiYrxdJv3bBAm2ybfVmMr71BAAAAABgrogEPQDmpibL0mg+rxPptJbF4770fFlVlV5WVeVLLwAAAAAA5hpOEsETT2848/mVM0nqTqd9PcEEAAAAAMBcQEgETzRNhkR+Xl4tST/s7tbSBx8MJJwCAAAAAGA2IySCJ2ricUWM8T0kuqSiQpK0hS1nAAAAAABMCyERPBEJhdSQSPh+oqfBstRsWdpMSAQAAAAAwLQQEsEzT20489sm29a9/f1K5XK+9wYAAAAAYLYiJIJnngqJ/L5EusW2NZbP64GBAV/7AgAAAAAwmxESwTPNlqXhXE7d6bSvfV9ZVaVvn3WWzi8r87UvAAAAAACzGSERPBPUhrPScFjvWb5ci2MxX/sCAAAAADCbERLBM0GFRJKUzGT0na4uHR8f9703AAAAAACzESERPFOXSChijO8bziSpa3xc7z1wQP/V2+t7bwAAAAAAZiNCIngmEgqpLpEI5CTReaWlWhGLaUtfn++9AQAAAACYjQiJ4KmnNpz5zRijTbatXyeTyubzvvcHAAAAAGC2ISSCp5otS61jY3Icx/feLbatgVxOjwwN+d4bAAAAAIDZhpAInmqyLA3lcjqVyfje+/LqaoUl7SAkAgAAAADgjCJBD4C5rfkZG878XklfHY3q5KWXyo5Gfe0LAAAAAMBsxEkieKppMiQKYsOZJAIiAAAAAACmiJAInqpLJBSWArm8WpJ60mm9bvdu/eTUqUD6AwAAAAAwWxASwVPRUEh1iURgIVF1NKoHBwb0056eQPoDAAAAADBbEBLBc02WpdbR0UB6h43Rq21bW5JJ5QPYsAYAAAAAwGxBSATPNVmWDo6NyQkopGmxbXVnMto5PBxIfwAAAAAAZgNCIniuuaREA7mcejKZQPq/urpakrQlmQykPwAAAAAAswEhETz31IazoO4lWhqP66+XLNHSWCyQ/gAAAAAAzAaRoAfA3PfMkOjFlZWBzPAv55wTSF8AAAAAAGYLThLBc/WJhEKSWgM6SfSUsVxOJ9PpQGcAAAAAAKBYERLBc7FQSLWJRGCvm0lS3nFU+/DDurGjI7AZAAAAAAAoZoRE8MVTG86CEjJGl1ZWanMyGdiWNQAAAAAAihkhEXzRbFlqHRsLNKDZVF2tjlRKBwJ+7Q0AAAAAgGJESARfNFmW+rNZJbPZwGbYZNuSpM3JZGAzAAAAAABQrAiJ4ItnbjgLSr1labVlERIBAAAAAHAakaAHwPzQPBkStY6O6pKKisDm+GpzsxZGo4H1BwAAAACgWBESwRf1liWjYE8SSdKrJ185AwAAAAAAz8brZvBFPBRSTTweeEgkSb/s7dXdJ08GPQYAAAAAAEWFk0TwTXNJiVqLICT6ytGjOjI+rjcvXhz0KAAAAAAAFA1OEsE3TZZVFCeJNtm29o2O6nAqFfQoAAAAAAAUDUIi+KbJspTMZpXMZAKdo2XyXqItbDkDAAAAAOBphETwzVMbzoI+TXROSYlWxePaTEgEAAAAAMDTCIngm6YiCYmMMdpk29o7OirHcQKdBQAAAACAYsHF1fBNQyIho+BDIkn6cmOjSsNhGWOCHgUAAAAAgKJASATfJMJhrYrHi2LDWVmEP/oAAAAAADwTr5vBV8Wy4UySbjt8WK/bvTvoMQAAAAAAKAqERPBVMYVE4/m8ftbbq5PpdNCjAAAAAAAQOEIi+KrZstSTyag/kwl6FLXYtiTp1319AU8CAAAAAEDwCIngq2LZcCZJF5WXa2E0qs3JZNCjAAAAAAAQOEIi+KqYQqKQMXp1dbW2JJPKO07Q4wAAAAAAEChCIviq0bJkJD02PBz0KJKkty5erDcuXKiRXC7oUQAAAAAACBQhEXxlhcN6/cKF+kZXV1FcGP36hQv1zdWrVR6JBD0KAAAAAACBIiSC725taNBoLqebOjqCHkWSlHccPTk6GvQYAAAAAAAEipAIvltdUqKrli/Xt7q6tH9kJOhx9NnOTp23dasGstmgRwEAAAAAIDCERAjEjXV1KgmH9fH29qBH0SurqpST9Lu+vqBHAQAAAAAgMIRECMSiWEw31NTont5e/T7gcObFFRUqD4e1OZkMdA4AAAAAAIJESITAfHjlSq2Kx3VNW1ugK+ijoZAur67W5mRSToBzAAAAAAAQJEIiBMYKh/W5+no9Ojysfz95MtBZWmxbh8fHucAaAAAAADBvERIhUH+1ZInWlZXp+vZ2pXK5wOZ4/cKF+q+1a1WbSAQ2AwAAAAAAQSIkQqBCxuj2xkYdHh/XV48dC2yOJbGYXrNggaxwOLAZAAAAAAAIEiERAndZdbVeu2CBPtfZqZ50OrA52sfG9OlDhzQa4IkmAAAAAACCQkiEovDFhgaN5HK6ubMzsBlax8Z0U2en7uvvD2wGAAAAAACCQkiEonBOaanevWyZvtHVpdaALo9+eWWlEqGQtvT1BdIfAAAAAIAgERKhaHy6rk6JUEjXtbcH0t8Kh/WKykptTiYD6Q8AAAAAQJAIiVA0lsbj+viqVfpJT48eCOiVrxbb1v7RUXWMjQXSHwAAAACAoBASoah8ZNUqLY/F9NG2NjmO43v/FtuWFQrpiYBeeQMAAAAAICiERCgqJeGwPldfr61DQ/rRqVO+919dUqLkpZfqLxYs8L03AAAAAABBIiRC0XnH0qU6v7RU17e3azyf97W3MUaJcNjXngAAAAAAFANCIhSdsDG6vbFRh1Ipff3YMd/77xsZ0frt23VfQPciAQAAAAAQBEIiFKUrbFsttq3PdHYqmcn42nt5PK5dIyNsOQMAAAAAzCuehUTGmFXGmHuNMXuNMU8YYz7sVS/MTbc1NGgwm9VnOzt97VsZiejFFRWERAAAAACAecXLk0RZSR91HOdcSS+S9H5jzLke9sMcs6asTH+7bJm+duyY2nxeSd9i23pseFjd6bSvfQEAAAAACIpnIZHjOMcdx9kx+fdDkvZJWuFVP8xNN9fVKWqMbmhv97Vvi21Lkn7FaSIAAAAAwDzhy51Expg6SeskPXKaz95rjNlujNl+KoCV5yhuy+JxXbtqlX506pQeGhjwre+FZWV6x5IlWhGP+9YTAAAAAIAgGcdxvG1gTJmk/5b0OcdxfvJCz27YsMHZvn27p/Ng9hnOZtW8dasaEgk9sG6djDFBjwQAAAAAwKxhjHnUcZwNZ3rO05NExpiopB9LuutMARHwfMoiEX2mrk4PDg7qJz09vvY+mkpxLxEAAAAAYF7wcruZkfRdSfscx/mSV30wP/zNsmVaU1qqj7e1KZ3P+9Izmcmo5uGH9Z2uLl/6AQAAAAAQJC9PEl0q6R2SLjPGPD758xoP+2EOCxujLzY0qC2V0jd9Cm3saFTry8u1mcurAQAAAADzgJfbzR5wHMc4jnO+4zgXTv78wqt+mPtabFuXV1frpo4O9WcyvvTcVF2thwcHfesHAAAAAEBQfNluBrjBGKPbGhrUl83q84cP+9KzxbaVk/Tb/n5f+gEAAAAAEBRCIswqF5aX638tXap/OHpUHWNjnve7pKJCFeEwr5wBAAAAAOY8QiLMOp+pq1PYGN1w6JDnvaKhkH6yZo1uqqvzvBcAAAAAAEEiJMKsszKR0EdWrtQPT57UtsFBz/v9WXW1lsfjnvcBAAAAACBIhESYlT5eU6PF0aiuaWuT4zie9nIcR3ceO6Z7eno87QMAAAAAQJAIiTArlUciuqmuTvcNDOie3l5Pexlj9PVjx/S1Y8c87QMAAAAAQJAIiTBrvXvZMp1dUqKPtbUpk8972qvFtnVff79GczlP+wAAAAAAEBRCIsxakVBItzU06MDYmL59/LinvVpsW+OOo//u7/e0DwAAAAAAQSEkwqz2FwsW6JVVVfp0R4cGslnP+rysslJWKKTNyaRnPQAAAAAACBIhEWY1Y4xub2xUTyajLxw+7FmfRDisy6qqdDyd9qwHAAAAAABBigQ9AFCo9eXlunLJEn356FFdtXy5ahIJT/r83zVrFAmRqwIAAAAA5ib+jxdzwufq6+U4jj5x6JBnPZ4KiBzH8awHAAAAAABBISTCnFCTSOjqVav0b93d2jE05Fmf9z75pN6+b59n9QEAAAAACAohEeaM62pqtDAa1TVtbZ6d9gkbo5/39iqdz3tSHwAAAACAoBASYc6ojER0Y22t7u3v1y882kLWYtsazuX04MCAJ/UBAAAAAAgKIRHmlP+9fLmaLUvXtrUp68Fpn8uqqhQxRps9CqEAAAAAAAgKIRHmlGgopC82NGjf6Ki+e+KE6/XLIxFdWlFBSAQAAAAAmHMIiTDnvH7hQr2sslKfOnRIQ9ms6/Xfu3y53rxokfJsOQMAAAAAzCGERJhzjDG6vbFRJzMZffHIEdfrv33JEn2irk4hY1yvDQAAAABAUAiJMCddXFGh/2/xYt1x5IiOjY+7Xn84m9WOoSHX6wIAAAAAEBRCIsxZn6+vV85x9MlDh1yv/cGDB3XFzp3K8coZAAAAAGCOICTCnFVvWfrQypX63okT2jk87GrtV1dXK5nN6lFOEwEAAAAA5ghCIsxpN9TUqDoS0TVtbXJcPPVzRXW1jMSWMwAAAADAnEFIhDmtOhrVJ2tr9Zu+Pm1xMdBZGItpQ3k5IREAAAAAYM4gJMKc974VK9SYSOja9nZX7xBqsW09MjiovkzGtZoAAAAAAASFkAhzXiwU0q0NDdozMqLvnTjhWt13L1umHRs2qCoSca0mAAAAAABBISTCvPCmRYv04ooKffLQIQ1ns67UrEkkdEFZmYwxrtQDAAAAACBIhESYF4wxuqOxUcfTad1x9KhrdR8eGNBHDh509VJsAAAAAACCQEiEeePFlZV6y6JF+uLhwzo+Pu5KzSdGR/Xlo0f1xMiIK/UAAAAAAAgKIRHmlVsaGpRxHN3Y0eFKvU3V1ZLEljMAAAAAwKxHSIR5pdGy9P4VK/Td48e1Z3i44HorEwmtKS0lJAIAAAAAzHqERJh3PlFbq4pIRB9rb3el3qbqat0/MKCRXM6VegAAAAAABIGQCPPOgmhUn6it1S+TSf3ahRNALbatpbGY2sfGXJgOAAAAAIBgEBJhXvrAihWqSyR0bVubcgVuJrusulodL3qR1paVuTQdAAAAAAD+IyTCvBQPhXRLfb12jozo+93dBdUKGSNjjJwCwyYAAAAAAIJESIR5622LF+vi8nL9fXu7Rgu8T+jXyaSWPfig2njlDAAAAAAwSxESYd4yxuj2xkYdS6f15aNHC6pVl0ioO5PRFracAQAAAABmKUIizGsvq6rSGxYu1K2HD6s7nZ5xnSbLUkMioc2ERAAAAACAWYqQCPPerQ0NSuXz+nRHx4xrGGPUYtv6XV+f0vm8e8MBAAAAAOATQiLMe2eVlOiq5cv1na4u7RsZmXGdFtvWSD6vPwwMuDgdAAAAAAD+ICQCJH2qtlal4bA+3t4+4xqvqqrSB1as0JJYzMXJAAAAAADwByERIGlRLKYbamv1s95e3dvXN6MaZZGI/rG5WeeWlro8HQAAAAAA3iMkAiZ9aMUK1cTjuqatTXnHmVGNnONo6+CgkpmMy9MBAAAAAOAtQiJgkhUO6/MNDdoxPKwfdHfPqMYTIyO6ZMcO3dPT4/J0AAAAAAB4i5AIeIb/uXixLior098fOqSxXG7a319bWqplsZg2J5MeTAcAAAAAgHcIiYBnCBmj2xsbdXh8XF89dmza3zfGaJNt61d9fcrN8JU1AAAAAACCQEgE/IlXVVfrfyxYoM93dupUOj3t77fYtvqyWW0bHPRgOgAAAAAAvEFIBJzGFxoaNJLL6ebOzml/9/LqahmJV84AAAAAALMKIRFwGueUluq9y5frm11dOjA6Oq3vLohG9Yd16/TxmhqPpgMAAAAAwH2ERMDzuLGuTolQSNe1t0/7uy+urJQVDnswFQAAAAAA3iAkAp7HklhM19XU6D97enR/f/+0vjuczeoT7e36Na+cAQAAAABmCUIi4AVcvXKlVsRi+mhbm/LT2FZmhcO6s6tLPzh50sPpAAAAAABwDyER8AJKwmF9rqFB24aG9KNpBD5hY3RFdbW2JJNyphEuAQAAAAAQFEIi4AyuXLJEF5aV6fpDhzSez0/5ey22rePptHaPjHg4HQAAAAAA7iAkAs4gbIxua2hQRyqlrx07NuXvbbJtSdJm7iUCAAAAAMwChETAFFxu2/pz29ZnOzvVm8lM6TvL43FdWlGh4VzO4+kAAAAAACgcIREwRV9saNBgNqvPdnZO+Tv3r1unm+vrPZwKAAAAAAB3EBIBU7SmrEzvWrZMXz92TAdHR6f0HWOMJCk7jbuMAAAAAAAIAiERMA031dUpZoyuP3RoSs87jqOX7NihDx886PFkAAAAAAAUhpAImIZl8biuranR3adO6aGBgTM+b4zRwmiUy6sBAAAAAEWPkAiYpmtWrdKyWEwfbWuT4zhnfL7FttWeSk35FTUAAAAAAIJASARMU2k4rM/U1+uhwUH9+NSpMz6/ybYlidNEAAAAAICiRkgEzMA7ly7VmtJSfby9XekzXErdaFlqsixCIgAAAABAUSMkAmYgbIxub2xUeyqlO48dO+PzN9TU6H8uWeLDZAAAAAAAzAwhETBDm2xbV1RX6+bOTvVlMi/47N8sW6a/IiQCAAAAABQxQiKgALc1Nqo/m9XnDx8+47MdY2NT2ogGAAAAAEAQCImAAlxQVqZ3Ll2qrx49qkNjYy/47LuefFLvPXDAp8kAAAAAAJgeQiKgQJ+pr1fYGN1w6NALPtdi29ozMqKjqZRPkwEAAAAAMHWERECBVsTjumbVKv37yZN6ZHDweZ9rsW1J0pa+Pr9GAwAAAABgygiJABdcu2qVFkejuqatTY7jnPaZNaWlWh6LaUsy6fN0AAAAAACcGSER4ILySEQ319frgYEB/bSn57TPGGPUYtv6TV+fcs8TJAEAAAAAEBRCIsAl71q6VOeUlOhj7e3K5POnfeZTdXXaf/HFChvj83QAAAAAALwwQiLAJZFQSLc1Nqp1bEzf6uo67TO1iYQWx2I+TwYAAAAAwJkREgEueo1t67KqKn26o0MD2expn/nxqVO6+uBBnycDAAAAAOCFERIBLjLG6PbGRiWzWd3S2XnaZ54YGdE/HD2qnnTa5+kAAAAAAHh+hESAy9aVl+vKJUv0laNHdTiVes7nLbYtR9Jv+vr8Hw4AAAAAgOdBSAR44LP19TLG6O8PHXrOZ+vLy7UgEtHmZDKAyQAAAAAAOD1CIsADNYmErl65Ut/v7tajQ0PP+ixsjK6wbW1OJpV3nIAmBAAAAADg2QiJAI9cV1OjRdGormlrk/MnYdBf2LbqLUu9mUxA0wEAAAAA8GyERIBHKiIRfbquTr/v79fPe3uf9dmVS5fqoYsu0qJYLKDpAAAAAAB4NkIiwEPvWbZMZ1mWPtbermw+/5zPx0/zOwAAAAAAgkBIBHgoGgrpi42N2j86qn86fvxZn32nq0sLHnhAg9lsQNMBAAAAAPBHhESAx163YIFeXlmpGzs6nhUINVuWRvJ53dvfH+B0AAAAAABMICQCPGaM0e2NjTqZyeiLhw8//fuXVFaqLBzW5mQywOkAAAAAAJhASAT4YGNFhd6+eLHuOHpUR1MpSVIsFNJlVVXanEw+Z/sZAAAAAAB+IyQCfPK5+nrlHUef7Oh4+ncttq2OVEqtY2PBDQYAAAAAgAiJAN/UWZY+vHKl/uXECT0+NCRJeu2CBbqlvl4V4XDA0wEAAAAA5jtCIsBHN9TUqDoS0TVtbXIcR6sSCV1XW6ul8XjQowEAAAAA5jlCIsBHVdGobqyr02/7+5++sHoom9V/njqlVC4X8HQAAAAAgPmMkAjw2VXLl6vJsnRtW5uy+bzuHxjQG594QvcPDAQ9GgAAAABgHiMkAnwWC4V0a0ODnhgd1fdOnNArqqoUN+bpk0UAAAAAAASBkAgIwBsXLtRLKir0yY4OOY6jl1dVERIBAAAAAAJFSAQEwBijOxobdSKd1u1HjqjFtrV3dFRHUqmgRwMAAAAAzFOEREBAXlRZqbcuWqTbjhzRurIySdK9/f0BTwUAAAAAmK8IiYAA3dLQoIzj6PsnTuiJjRv1jiVLgh4JAAAAADBPERIBAWqwLH1gxQp9r0+J3L4AACAASURBVLtbOceRMSbokQAAAAAA8xQhERCwT9TWqiIS0YcOHtQ79+3TtsHBoEcCAAAAAMxDkaAHAOY7OxrVJ2tr9dG2NoUkrUoktLGiIuixAAAAAADzDCeJgCLw/hUrVJ9IKBEKaXMyGfQ4AAAAAIB5iJAIKALxUEi3NjRoNJ/X9qEhnUyngx4JAAAAADDPEBIBReItixbpvJISSdLPe3sDngYAAAAAMN8QEgFFwhijO5ubJUn/RUgEAAAAAPAZIRFQRF5eXa03Llyon/X26qaODo3n80GPBAAAAACYJwiJgCLz7dWr9caFC/Xpjg6du3Wr7uvvD3okAAAAAMA8QEgEFJkF0ag+W1+vsnBYnamUXvH443r3/v1KZjJBjwYAAAAAmMMIiYAi1FRSoj+sWyc7GlVJKKR/PnFCZ2/dqru6u+U4TtDjAQAAAADmIEIioEidX1amByaDotJwWIuiUV25b59adu1S29hY0OMBAAAAAOYYQiIv5HJBT4A54qySEt1/4YWqSyT09eZm/WNTkx4aHNSabdt0a2enMlxsDQAAAABwCSGR244elTZulO65J+hJMEfUWZYe27BBr6yu1gdWrtRvLrhAf27buv7QIa1/9FE9PDAQ9IgAAAAAgDmAkMhtS5ZI6bT0gQ9Iw8NBT4M5ImyMJOn/P3lSL3vsMb1jyRL93zVr1JfN6iWPPab3HTiggWw24CkBAAAAALMZIZHbolHpW9+SjhyRbrop6Gkwx1xRXa315eV6yxNPaCib1d6NG/WhFSv0ra4unbN1q+4+eZKLrQEAAAAAM0JI5IVLL5Xe/W7py1+Wdu0KehrMIVXRqH51/vl6eVWV/nr/fv3w5El9pblZj1x0kZbGYnrL3r163Z49OpxKBT0qAAAAAGCWISTyyq23StXV0h13BD0J5piySET/tXat/ty29b8PHNDO4WFtqKjQ1osu0h2NjfpdX5/O3bpVXzpyRFkutgYAAAAATJEppldTNmzY4Gzfvj3oMdyza5d09tlSLBb0JJiD0vm8ftHbq79ctOhZv+8YG9P7W1v1i2RSF5WV6durV2t9eXlAUwIAAAAAgmaMedRxnA1neo6TRF46//yJgGhwUEomg54Gc0wsFHo6IHpwYECfaG+X4ziqsyz9fO1a/ejcc9WVTuviRx/V1QcPapiLrQEAAAAAL4CQyGvj49KFF0of/nDQk2AO+2lPjz53+LA+0NqqvOPIGKO3LF6sfRs36r3Ll+srR4/q3G3b9LOenqBHBQAAAAAUKUIir8Xj0pVXSt//vvTb3wY9DeaoWxsa9LFVq3RnV5f+dv/+p+8iqopG9Y2zztIf1q1TRTis1+3Zozfv2aOu8fGAJwYAAAAAFBvuJPLD2Ji0dq0UCk3cU5RIBD0R5iDHcfTZzk59qqNDb1m0SN8/5xzFQn/MgdP5vO44ckQ3d3YqaoxuaWjQVcuXK2xMgFMDAAAAALzGnUTFxLKkb3xDam2VvvCFoKfBHGWM0Sfr6nRHY6Ok5/7LHQuFdH1trXZv2KBLKir0gdZWXbpjh3YND/s/LAAAAACg6HCSyE9vf7s0NCTdc4/E6Q14yJm8l+j4+LjKw2GVRSLP+fyu7m5d3dam/mxWH125Up+qq1NJOBzQxAAAAAAAr0z1JBEhkQfGxyeWmj0nBxobm3jVjIAIPsjm87ro0UdVFg7rF2vXqioafc4zvZmMrm1r0z+fOKH6RELfOOssbbLtAKYFAAAAAHiF180C0t0tbdggffe7p/nQsiYCosOHpd//3u/RMM9EQiHdVFen7UNDetXOnTqVTj/nmQXRqP7P2Wfr3gsuUNQYtezapbfv3avu0zwLAAAAAJjbCIlctnChtHSp9MEPSrt3P89D73yn9La3SX19fo6GeegNixbpnjVrtH90VK94/PHn3Wr2yupq7dq4UTfW1urHp07pnK1b9U9dXcoX0UlDAAAAAIC3CIlcFg5PbLuvqpLe+lbptHcC33GH1NMjXX+97/Nh/mlZsECbzz9fR8bH9b4DB573uXgopE/X12vnhg1aW1qq9xw4oFc+/rj2jYz4OC0AAAAAICiERB5YskS66y7pySel97//NA+sWyd96EPSt74lPfSQ7/Nh/nlFVZXuveACffOss8747Nmlpbr3wgv1T6tXa8/IiC7Yvl03HjqkVC7nw6QAAAAAgKAQEnnkssukT31q4pWzoaHTPHDzzdLKldJVV0mZjO/zYf7ZUFGhpfG4Mvm83vPkk9p12mNuE0LG6F3Llmn/xRfrLYsW6ebOTl2wfbvu5RVJAAAAAJizCIk89MlPSg8+KJWXn+bD8nLpq1+VLrlkYh0a4JPj6bR+2durVz7+uLYNDr7gs4tjMd117rnacv75yjqOLtu5U3+zf796CTYBAAAAYM4hJPJQODyx8X5gQLrmGml09E8eeMMbpG9/WyorC2Q+zE81iYTuX7dOVZGI/mznTt3f33/G77zatrV740ZdV1Oj73d36+ytW/VvJ07I4WJrAAAAAJgzCIl88Oij0pe+NHEN0Wlt2yZdd52vM2F+q7cs3b9unVbE49q0a5d+nUye8Tsl4bBuaWjQjvXr1WRZ+uv9+3XFzp06+Jz0EwAAAAAwGxES+eCyy6QbbpC++92JC62f4/e/l77wBemnP/V7NMxjK+Jx/feFF2p9ebkqI5Epf29tWZkeWLdOX29u1rahIa3dvl2f7+xUOp/3cFoAAAAAgNdMMb0usmHDBmf79u1Bj+GJbHYiLNqxY+Jk0erVz/gwk5HWr5f6+6W9e3n9DL5yHEfGGEnSEyMjOq+0dMrf7Rof14daW/Xjnh6dV1Kib61erUsrK70aFQAAAAAwA8aYRx3H2XCm5zhJ5JNIRPrBDybuKHr/+//kw2hU+uY3pSNHpBtvDGQ+zF9PBUQ/OnlSa7dt0z91dU35u8vjcd29Zo3uWbNGg7mcXvrYY7rqySfVz8XWAAAAADDrEBL5aOXKiTfK/vVfT/PhS14ivec90j/8g7R7t++zAf9jwQK12Lbec+CAvnLkyPS+u3Ch9m7cqKtXrtR3jh/XOdu26UcnT3KxNQAAAADMIoREPrv0Umn5cimXk/bs+ZMPb71VuuWWP3kXDfCHFQ7rP9es0RsXLtTVbW36XGfntL5fFonoS01N2rp+vZbHYnrb3r167e7d6hgb82hiAAAAAICbCIkCct11E4eHDh58xi9tW7r2WikWkziBgQDEQyH9x7nn6solS/SJQ4e0fXBw2jXWl5frkYsu0pcaG/Xf/f06b9s23X74sLJcbA0AAAAARY2QKCAf/ODEPUVve5s0Pv4nH953n3T++VJ3dyCzYX6LhEL6l7PP1m8vuEAbKipmXOPqVau09+KLdVl1ta5tb9fGHTu0bQahEwAAAADAH4REAampkb73vYltZ9dc8ycfLlkiHTggfeQjQYwGKGSMLquuliT9rq9Pf3fggHIzON1Wk0jonjVrdPd556k7ndaLduzQh1tbNZTNuj0yAAAAAKBAhEQBet3rpKuvlr72NeknP3nGB6tXS9dfP7EO7Te/CWw+QJIeGRzUN7u69Fd79yozg1fGjDF606JF2nfxxbpq+XL947FjOnfbNv20p8eDaQEAAAAAM0VIFLBbb5Xe9CZp6dI/+eC666TmZul975NSqUBmAyTp+tpa3dbQoP84dUpveuIJpXK5GdWpjET09bPO0h/WrVNVJKK/3LNHb9izR0f58w0AAAAARYGQKGCxmHT33ROXWEvPuK86kZDuvFNqbZV++MPA5gMk6ZqaGt3Z3Kyf9fbqtbt3a2yGQZEkvbiyUjvWr9ct9fXanEzq3G3b9I9Hj87odTYAAAAAgHsIiYqE40zcTfSs+4kuv1y6/37pne8MaizgaX+3YoW+d/bZWpVIKB4q7D8d0VBI19XWas/GjXpRRYU+dPCgXrJjh3YOD7s0LQAAAABgugiJioQxE1vOvvQl6Z57nvHBS1868eHx4884ZgQE438tXap/PvtshYzR4VRKPel0QfUaLUtbzj9fd51zjg6lUlq/fbs+1tamkQJOKgEAAAAAZoaQqIjcfrt00UUTB4c6O5/xwa5dE/cT3XVXUKMBz5JzHP3F7t165eOP6/j4eEG1jDF6+5Il2n/xxXrn0qW67cgRrdm2Tb/s7XVpWgAAAADAVBASFZF4XPqP/5Cy/4+9+w5vqzz7OP492vLeI85wEidx9iAhkzATQijQAk0oLdCwRwuFFmiBshqgQKGEt1B22SUQoKwwwkpYCdl7byfx3ra2zvvHI1mSZQcncSI7uT/XpUuybp2jR7IsWz/fz3O8cMEF4PEECoMGqdNNN0FlZUzHKASAUdN4vKCAHU4nE1esYFc7LD6dZjbzXGEh84cNw6ppTF29mgvWrqX4EEMoIYQQQgghhBBtIyFRB1NQAM89BytWwNKlgSsNBnj6aRUQ/eUvMR2fEEEnp6Yyb+hQytxuTli+nM2Nje2y34kpKawcNYq78/N5t7yc/osX88zevfhluqUQQgghhBBCHFYSEnVA06bBli0wZkzYlUOHwg03wDPPwPffx2xsQoQbm5zMV8OG0ej3c+OWLe22X6vBwF35+awaNYqh8fFctWkTE5cvZ11DQ7vdhxBCCCGEEEKISBISdVB5eer89dehqChw5T33QPfu8PXXsRqWEFGGJybyzbBhvFhY2O777hcXx1fDhvFCv36sb2xk2JIl/HX7dpyysLUQQgghhBBCtDtN70BTOEaOHKkvWbIk1sPoMEpK1PSzYcPgq6/AZAJqayEpKdZDE6JFLr+fi9ev5/quXRmfnNyu+y5zu7lp61ZeLSmhwG7nhrw8pqan08tub9f7EUIIIYQQQoijjaZpS3VdH/lTt5NOog4sO1stRfTtt3DXXYErgwHR4sWwa1fMxiZES6o8HlbU1zN55Uo+b+dF1jMtFl7p35/PhgzBomn8fssWei9aROGiRdy0ZQufV1bi8vvb9T6FEEIIIYQQ4lginUSdwOWXwwsvwCefwOTJqG6ibt3gxBPhvfdA02I9RCGalLjdTFq5kk2Njbw1cCBnZWQclvvZ3NjI3MpKPq6o4Ovqaly6TrzBwKmpqUxNT+eMtDS622yH5b6FEEIIIYQQojNpayeRhESdQGMjjB6tpp9t2RJoJnrkEfjTn+Ddd+HnP4/1EIWIUOnxMGXVKpbX1/PGgAGcl5l5WO+vwefjq6oq5lZWMreigp0uFwCD4uOZmpbG1PR0xiUlYTZI86QQQgghhBDi2CMh0VFm/XpYswZ++cvAFR4PjBwJlZWwbh0kJsZ0fEI0V+v1ctH69dyTn8+wI/j61HWd9Y2NfBwIjBbU1ODVdZKMRiYFuoympKXRxWo9YmMSQgghhBBCiFiSkOgoVloKWVnAwoUwbhzceKPqLBKiA/uhpoax7byYdVvUer18EdZltNftBmB4QgJnBLqMRicmYpIuIyGEEEIIIcRRShauPkrNmwc9esCXXwJjxsANNwQSIyE6rjmlpYxbvpwHdu484vedZDLxi8xMnu3Xj6KxY1k5ciQP9OxJgtHIg7t2MWH5crK+/55frVvHK8XFlAZCJCGEEEIIIYQ41kgnUSdTXw+jRkF1NaxYoY6AJkRH5/H7+e2GDbxeWspt3bszs2dPtA6w4HqVx8O8qirmVlTwSWUlJR4PGjAyMbFpLaORiYkYOsBYhRBCCCGEEOJgyXSzo9jq1XD88TBhAnz6KRg0Hf73P7XC9a9/HevhCdEin65zzaZNPLtvHzfk5fHPgoIOERQF+XWd5fX1zK2oYG5lJYtqa9GBTLOZKWlpnJGWxulpaaSZzbEeqhBCCCGEEEIcEAmJjnLPPQdXXAEzZ8Ltt+kwaRIsWQIbNkBOTqyHJ0SLdF3npq1bmVVUxA8jRjA6KSnWQ2pVudvNZ2FdRhVeLwZgTFISU9PTmZqWxrCEhA4VdAkhhBBCCCFES2IeEmma9gLwM6BU1/VBbdlGQqK203W46CLo3RvuuQfYtAkGD4Zzz4X//jfWwxOiVbqus6y+nuM60RH5fLrO4trapsWvl9bXA5BrsTAlLY2paWlMSksj2WSK8UiFEEIIIYQQIlpHCIkmAvXAyxISHR66DhFNDHffrRKjTz+FyZNjNSwh2uzjigr+U1zMy4WF2IzGWA+nzYpdLj4NdBl9WllJjc+HSdMYH9ZlNDA+XrqMhBBCCCGEEB1CzEOiwCDygQ8lJDq8vv4aXnoJnn/CiWHYEPD7Yc0asNliPTQh9uupPXu4ZvNmJqWm8u6gQcR3oqAoyOv380Ogy+jjigpWNjQA0NVqbVr8+tSUFBKky0gIIYQQQggRI20NiWL+qUXTtCuBKwG6d+8e49F0TuvXw4svQmGhjVufeQb27gWrNdbDEuInXZ2Xh81g4LKNG5myahUfDh7c6aZsmQwGTkhJ4YSUFB7o1Ysip5NPKiuZW1nJ66WlPLNvHxZNY2JKClMDC2D3i4uTLiMhhBBCCCFEhyOdREcBXYfp0+Gdd2D+fBg/PqwgH0RFJ/BWaSkXrl/P0Ph4vho2jMROFhS1xu33821NDR8H1jJa19gIQE+branL6KSUFOI6YQeVEEIIIYQQovOQ6WbHmJoaGDEC3G5YsQLS5zwNH30E770nQZHoFD4KrO8zq6DgqO2y2eFwqMCospIvq6po9PuxGQycFOgympqeTm+7PdbDFEIIIYQQQhxlJCQ6Bi1dCuPGwZ13wu0ZT8PVV8PLL6vDoAnRiWxpbMRqMNDtKF5Xy+nzsaCmhrkVFcytrGSzwwFAX7u9afHriSkpWA2GGI9UCCGEEEII0dnFPCTSNO2/wElABlAC3KXr+vP720ZCokO3fDkMHQoG/Gre2datsGEDpKXFemhCtIlf1xm+ZAk1Xi+fDx1KQVxcrId0RGxubOTjyko+rqzkq6oqXLpOnMHAqampai2j9HR6HMWhmRBCCCGEEOLwiXlIdDAkJGo/u3ZB48JVFF44AmbMgGefjfWQhGizZXV1TF65EovBwOdDhzIgPj7WQzqiGn0+vqqubuoy2uF0AjAwLo6p6emckZbG+ORkLNJlJIQQQgghhGgDCYmOYboOI0dCRQVsOOtmbE88Aps2QUFBrIcmRJutbWhg0sqVuP1+Phs6lBGJibEeUkzous6GQJfR3IoKFtTU4NF1Eo1GJqWmNoVGXeSIhkIIIYQQQohWSEh0jFu0CCZMgPOn1PP6rSvRJoz/6Y2E6GC2NDZy6sqVDIyPZ+6QIbEeTodQ5/XyRVUVcwNT04pcLgCGJSRwRloaU9PSGJOUhEm6jIQQQgghhBABEhIJHn0U/vhHmDULrr8e1VqUnh7rYQlxQHY7nSQajaSYzei6ftQe+exg6LrOmoYG5ga6jL6rqcEHpJhMHJ+YSKLRSJzRSJzBgD1wHmc0YjcYmi43r8UZDKoeVrNomjzvQgghhBBCdGISEgl0Hc4+Gz79FDbd9h/yH7keVq+G/PxYD02IA+bw+Thv7Vquy8vjTAk7W1Tt8TAv0GW0pqGBRp+PRr8fR+C80efDdxD7NUCLAVJT4NSGWvNwqqWazWDAIGGUEEIIIYQQ7U5CIgGo5qG//Q1mXrWbhFH94aST4IMPQD6IiU6m0uNh8sqVrGxo4PX+/fllVlash9Qpefz+psCoeYDU6PfjaGOt0eeLvG2zmvsgf7cEg6RD6XxqrRYeThnlPVAIIYQQQhxDJCQSUTwPPor5z3+Et9+Gc8+N9XCEOGA1Xi8/W72a72tquKpLF67IzWX4MbqgdUfn0/WIkMmxn3Bqf7W2BFcHw6JpTYGRvVkYZW8WMB1K3S6BlBBCCCGE6AAkJBIRSkth8ilePq8ZSYZeDuvXg3y4Fp1Qg8/H9Zs381pJCSelpPDJ0KGAmo5mNxpjPDpxpOm6jjMsZNpfd1OLX4cFT45mtwuvO/z+g+6OkkBKCCGEEELEWltDItORGIyIvYwM6NLdxC8+f5qvjadgXLgQJk2K9bCEOGDxRiPPFxbyj969Kfd4AChyOum/eDHnZWRweW4u45OTZaHlY4SmadiNRuxGI+lm82G9r/DuqP0FSgdSr/B42B2oh9/2UAKp9gqh7AYDFoMBq6ZhNRiwGgxYwi5bAzWLrCUlhBBCCHHUkJDoGGEwwEsvwbBhoxlj282XY9KQPiLRmaWazaQGQgEd+HVWFq+XlvJSSQn97HYuz83l8txcUg5zcCCOHUZNI8FkIuEI3NfhCKTKPZ4Wb3uwgVQ4k6Y1hUnNg6WocCkQLFl/IoBqvk1L+24ttArWJCwWQgghhDgwMt3sGLNgAZx8MlwwXefVS+ahnXYqyBQdcZRo8Pl4q7SU5/btY2FtLbvGjqWL1UqJ202G2SxTcYRoQWuBlMPvx+X34/b7cek6ruDXYZddgZo77PKB3K6lfbcnSzA0OsDQqi3bWANH5GvrySShlRBCCCFiSNYkEq2aORN2v/wVT28+Bf71L7juulgPSYh2V+R00tVmA2DSypVsbGzk0pwcZuTm0iNwvRCiY9F1HU94uNRKmBQVQrVjUNXSvr3t8LeSAVoMj+wHGDa19WQ3GiO+lpBcCCGEOLZJSCRa5feDx61jPWsy/PijWsS6S5dYD0uIw2ZOaSnP7tvHvKoqACanpvLHbt2YlJYW45EJIToDfzBcaiF0cvr9+z05fL6fvM1P7sPv5+CO4xdi0rT2CZ8O4LZWgwGzpmEOdGtJUCWEEELEjixcLVplMIDVplH34JPYjh8M19+Eec4bsR6WEIfN+VlZnJ+VxQ6Hg/8UF/NCcTGrGxqYlJZGo8/HLqeTwvj4WA9TCNFBGTQNm9FILHsQvS0ER4caPoWfGn0+KgPrVrVUb49/KWqoKYDmQHgUvGwJC5LMzS63eN1+9tHS/tpjHzJdUAghxLFCOomOYQsWwBcn3ss93AWffAKnnx7rIQlxRPh0HY/fj81o5KXiYn67YQMTkpO5PDeX8zMziZd1uoQQoklwGuDBhE+ewPutO7CP8MvuYD3sctN54Lbhl39qH4fboQZNLQVXBlQIaUAdrTHi6zbWDJrW8tcx2PeB3G9b960FttPC9qe1UAtuL2GeEEK0TDqJxE+aOBHm334r8+/7Au9cF6dKRiSOEUZNwxgIgqakpfFgr148t28fv92wges3b+bC7Gz+2bs3NgmLhBACLRCCWAwGkmI9mFbogbWjWg2fDjKsOpTgqsbn+8l96LqOHzWl0Y86Wmfwsjg0UeERBxA0Nau1tI9goNXWIKs99tHSGJtrqQGgpQi1tVi1xdsewj5bvZ923uf+Gh86Quh5KEFuZ913e9+vhMBHjnQSHeN8Pph0ms6iHzUWL4YBA2I9IiFiQ9d1vqmp4bl9+9jqcPDt8OFomsaXVVUMT0gg1WyO9RCFEEIcQ3RdjwiNwi/7m9Wavg6//X4CqObbd8R9+wI1Pey50MP211LN3+x2etj+D8c+9LDH2Jb9t8c+ml/f0kfmFq9r4cN1ax+323ufB3Q/h7DPlq4Lf84O5vV8IK/1g913x/k03vGFB6tHqjvy/MxMbuvR44g/1sNBOolEmxiN8NrrGscN9fL12Y8z4IMzoH//WA9LiCNO0zQmpqQwMSUFv66jaRr1Xi9nrV6NH/UL4vLcXCYmJ8t/MYQQQhx2WtiHGSHE0at5ANieAdRPhVs+IkPMlraPVQB9KPfbfPtDeS6TTcdeZCKdRAKAJR+XMeJX/TAMGwJffQXyB4kQACyrq+O5fft4raSEWp+PPnY7/+7bl1NTU2M9NCGEEEIIIYRok7Z2EhmOxGBExzfyjEwMDz8I8+ez+76XYz0cITqMEYmJPNm3L/vGjeOlwkKyLRayAlPPVtbXM7eiAl8HCtuFEEIIIYQQ4mBJJ5EI8fvZ0f0EEvZsouqHDfQZkx7rEQnRoV25cSPP7ttHV6uVGTk5XJqTQ77dHuthCSGEEEIIIUQE6SQSB85gwPafp0immjVTb8HhiPWAhOjY/tWnD3MGDmRQfDwzd+6k16JFXLR+fayHJYQQQgghhBAHRUIiESFn0mA2XfUoj1TN4MYbYz0aITo2i8HAeZmZfDxkCDvGjOGu/HyGJSQA6qgsd27fzrqGhhiPUgghhBBCCCHaRqabiRbdcgs8/DC88QZMnx7r0QjR+ayoq+P4Zcvw6DrjkpK4PDeXaVlZxBuNsR6aEEIIIYQQ4hgj083EIbnvHi8f9bmB0QsejvVQhOiUhiUmUjR2LP/o3ZtKr5dLN24k9/vvWVNfH+uhCSGEEEIIIUSLJCQSLTLbTUwdtJv8/9wF27fTgRrOhOg0siwW/titG+tGjeLb4cOZkZND//h4AJ7cs4f/Kyqi0uOJ8SiFEEIIIYQQQpGQSLRu1ix0o5H1p/6OG66XlEiIg6VpGuOTk5nVpw9GTQNgbkUF12/ZQpfvv+c369bxdVUVHWn6rxBCCCGEEOLYIyGRaF23bmj33kv/7XPZ8693eOedWA9IiKPHh0OGsOy447g8N5cPKyo4eeVK/rBlS6yHJYQQQgghhDiGycLVYv+8XvwjR7FnQx1DrRtZusJIz56xHpQQRxeHz8fbZWUUxsUxMimJdQ0N/GXbNq7IzWVKWhomg+T5QgghhBBCiIPX1oWrTUdiMKITM5kwvPIylNvx/8LI9Onw7bdgscR6YEIcPexGI7/JyWn6ervTyaLaWt6vqKCLxcKMnBwuzc2ll90ew1EKIYQQQgghjnby72nx0wYPptvJBTz/nM7OVTUsXRrrAQlxdDszPZ3dY8fy7sCBDE9I4IFduxiyeDGNPl+shyaEEEIIIYQ4ikknkWiz8z67irMKl2M5fiFgjPVwhDiqmQ0Gfp6Zyc8zMylyOllaX0+cUf3cTV65kgFxcVyem8ughIQYj1QIIYQQQghxtJBOItF2p5yCZeUS+Pe/ef112LUr1gMS4tjQ1WbjnIwMQK1flGoy8eTevQxel41kTwAAIABJREFUsoSxy5bx/L591Hu9MR6lEEIIIYQQorOThatF2+k6TJmC//sfKNQ3kDGkC/Png9kc64EJcewpc7t5taSE5/btY11jIy/068eM3Fy8fj9GTUPTtFgPUQghhBBCCNFBtHXhaukkEm2nafDEExg8br4Y/Ad++AHuuCPWgxLi2JRpsXBjt26sGTWK74cPZ1pWFgBP7N3LkCVLeGDnTj6vrKTC44nxSIUQQgghhBCdhYRE4sAUFMAdd9Bt69fcfHEJDz0Ec+fGelBCHLs0TWNscjLxgfWKulut2A0Gbtu+nUmrVpHx3Xf0//FHgl2ja+rrKXI66UhdpEIIIYQQQoiOQaabiQPnckFDA864NMaMgaIi2LIFUlJiPTAhRFC5282K+nqW1ddT6/Uys1cvAMYuW8bC2loyzWaGJyQwPCGBiSkpTE1Pj/GIhRBCCCGEEIdLW6ebydHNxIGzWsFqxeb18v6ti1hkGi8BkRAdTIbFwmlpaZyWlhZx/WMFBSyurWV5IEB6tKiIDY2NTSHRtLVr6WKxMDwxkREJCRTGxWE2SNOpEEIIIYQQxwIJicTBu/deuv/973RfsQIYQHEx5OTEelBCiP0ZnZTE6KSkpq9dfj81gSOjufx+ilwuPqqooHHPHgCsmsZd+fn8pUcPfLrOkro6hsTHYw9MbxNCCCGEEEIcPWS6mTh4ZWVQWAgDB/LFnfP52Vka770HkyfHemBCiEPh03U2NTaqbqO6Ok5JTWVqejrrGhoYuHgxRqAwLq6p2+jnGRn0tNtjPWwhhBBCCCFEK2S6mTj8MjPhoYfg8suZsOVFeveewW9+AytXQm5urAcnhDhYRk2jf3w8/ePjuTA7u+n6rlYr7wwcyPL6epbX1/NlVRWvlpRQYLfT025nYU0NjxQVNa11NCIxkWyLJYaPRAghhBBCCHEgpJOonTmdYDSC2RzrkRwhfj+ceCKsX8/G9zYwYnIGo0fDvHnqeRBCHN1K3W4SjEbijEbeLy/npi1b2Op0NtVzLRbmDxtGn7g4djudeHWdfJsNTdNiOGohhBBCCCGOLdJJFCPPPAM33ghdu0KPHuqUnx953r27Wvv5qGAwwFNPwW9+Q7/UUp54IoMZM+Bvf4O774714IQQh1tWWKfQ2RkZnJ2RQY3Xy4r6epbX1bGsvp5ugTe8WUVFPFJURIrJFNFtdEFWFkYJjYQQQgghhIg56SRqZ4sWwbMfLaFmd1dKt+WwY4c6RLzfH3m73NyWA6TgeVzckR/7IdF1CHzImzEDunWDe++N8ZiEEB3K+oYGFtTUNIVHq+rrSTKZKBk3Dk3TuHfHDordbkYkJDA8MZFB8fFY5chqQgghhBBCHLK2dhJJSNTOdF1n0L8Hsb1qO9eMvIabx99MujWHPXtg50512rEj8nzXLvB4IveTkdF6gNSjByQnH/GH9tNqa2HWLPRbbkWzyjokQoj98waOppYfWPR6xoYNvFNWRq3PB4BJ0zg3I4PZAwcCsLq+nnybjUSTNMEKIYQQQghxICQkiqEtlVuYuWAmr656FYvR0hQW5SS0fHx4nw+Ki0OhUUtBUtgSHwCkpOy/Eyktramx58j5+GOYOhXuuw9uu42vv4YXXoAXX1Sz0oQQ4qf4dZ3tTifL6upYXl9PhtnMTd26oes6ad99R43XSx+7XU1XS0zktNRUjktMjPWwhRBCCCGE6NAkJOoAwsOil3/xMhcOvvCg9qPrUFoaGRo1D5Lq6yO3iY9vPUDKz4esrMMUIp1/Pnz0EaxZw7Nf9ubKK2HmTLj99sNwX0KIY4ZP1/m4oqLpyGrL6urY6XLx5+7deaBXLxp8Pi5ct64pPBqRkEBXq1UWyBZCCCGEEAIJiWKmxlmD1+8lPS696bqtlVvJT8nHaDDy2MLH2F2ze7+dRQdK16GqKrr7KPxyVVXkNjabWkC7tSApN/cgj05WVAT9+8OECegfzeXXv9GYPRu++gomTjzEByqEEGEqPR68uk6WxcKWxkbOXrOGDY2NBH+rpZtMPNOvH+dmZlLn9bLP7abAbscgwZEQQgghhDjGSEgUI7MWzuJP8/7Eab1OY9qAafy88Oek2lOb6jd9ehOzFs3CarT+5DS09lRb2/I0tuDlsrLI25vNavHp1jqR8vLUbVo0axb84Q8wezZ1Z0xjxAhobIQVKyAz8zA+SCHEMa/B52NVWLfRdXl5DE9M5J2yMs5bu5YEo5FhCQkMS0gg3WTimrw8si0W1jc0sLSuDrvRiN1gaDoNT0zEajBQ7/Xi0nXsBgM2g0GCJiGEEEII0alISBQj68rW8fLKl5m9djY7qndgNpiZ3Hsy0wdO5+x+Z5NsS2ZzxWbu++Y+Xln1ClajlVlTZnHFcVfEdNwNDWoB7dY6kfbujby9waCCohY7kfK89PzXTZhuvB4KCli+HMaMgb/+Fe6444g/NCGEYI/LxaeVlU3h0eqGBup8PtaNGkX/+Hj+uXs3N23dGrXdrjFj6GazMXPHDv66Y0fT9bZAiLRt9GhSzGYe272b10pLiQsGTIGw6eXCQkwGA++Vl7Osri6ilmA0cmF2NqCO/Fbt9UbU441G0ltN44UQQgghhGg7CYliTNd1luxdwuy1s3lz7Zvsrt2NxWjhjIIzmDZwGmf1PYvi+mJmfjOTy4dfzgk9TqCsoQyf7jsinUUHyuWC3btb70QqKgK/P3Kb3NxQcGSzwfHHh8Iku11Nkwue/P7Ir4/16+x26NsXCgtVR5cs/C1E+/PrOhqgaRo1Xi8lbjcOvx+Hz6fO/X5OSUnBZjSyuLaWH2pro+oP9+6N1WDg+X37mFNWFlFz+v1sPP54NE3jmk2beKpZ2p5oNFJ7wgkAXLB2LbObtXTmWizsHTcOgHPXrOHLqqqITqd+cXG8M2gQAHds28YWh4O4sHpvu51r8vIAeLusjHqfr6kWZzSSbTYzKCEBgL0uFyZNa6qb5E1HCCGEEOKoIiFRrDgcKlGx2cBqBU3Dr/tZVLSIN9e+yVvr3mJP3R5sJhtT+0xl+sDpnNnnTOIt8fxu7u94YfkLXDPyGm4ZfwvZCdmxfjRt5vHAnj2RAVLl+hLO//xq/mG9nbmlI/F4Yj3Kzsluh379VGAUfurbV9WEEJ2Druu4/H4aAyGS2++nZ+CHeE19PbtdrqaAyeHzYTYYuCRH/dPgub17Wd3QEFHPtlh4om9fAC5av55FzUKs4QkJfDtiBACDfvyRtY2NEeOZlJrKZ0OHAtBz4UJ2hB1G06RpTMvM5LUBAwAYs3QpDr8fW2C6nc1gYEpaGjd26wbA7zZtwqhpTTW7wcCopCROTU3Fr+u8VVYWsa3NYKCb1UoXqxW/rlPm8TRtZ9Y0WXBcCCGEEKKdSUgUK48+Cn/8Y+hri0WFRYHQyG+z8n2en9n59czpUk2x1UOcz8jP6nI4wZnDwoQq/hu/HStGrvUO52bTRLKt6aHQqS3nza8LhFVHXE2NWsS6Sxd83y9iX6mRyZOhpARuuglyctS6RgaDGl746Vi/rq4ONm2CDRsiTzt2qE4jULfr0SM6PCosPIxHrxNCdEqlbjf1wS6nwHmC0cjwxEQA3igpocLrbao1+v0MiIvjokBINWPDBqq9XpyB7Z1+P1PT07kzPx9d1+nyww9N17sCb1J/6NqVfxYU0OjzEf/NN1FjuqNHD/7WsyelbjfZ33/fdL2Gms53f8+e/KFbN3Y5nUxZtappPajg6Xd5eZyRns5up5MHd+2KCqF+lp5OYXw8ZW4386urVQhlNDbVC+x2kk0mXH4/DT4fNoMBq8GAUd48hRBCCHEUkpAoVpYtg/nzwelUHUX7Ofe5nHxjL+XNjBLm5FRSZvOR4NY4aY+JBqOf+V19/HYFPP9+O4zLYjmwYOlgwqiWzj/5BK66Ch55BG68kaXLNMaNA7dbHT0tPx+eeQZOOUWte7RqFfTpo8IPk6kdHvdRxuGAzZujw6ONG9Xi4EEpKS2HR7167WfBcSGEaAd+XccdmH9sMxrx6TqbGhtxBqbgOQPdUL1sNgrj42nw+XipuDii7vT7mZqWxkmpqexxufjDli1R9b907865mZksr6tj8qpVTQFXcObzGwMGMD0riy+rqjh15cqocX4waBA/y8jg/fJyzlmzpul6c6Ajau7gwUxISeHD8nJu2749okvKZjDwaEEBve12FlRX81ZZGdbAdtbA6YrcXFLNZtY2NLCqvj6iZjMYGJWYiMVgoNLjoS4YUmla021kcXQhhBBCtCcJiToZr9/L/B3zmb12Nm+vf5tKRyXx5nhO7zWZGYN+Q6Ypmbc2zOHm/peTrSW2KYSKOj+Ubdzu9nmgVis+sxW3wYZTs9Pot5PW1Y49xca+ajtL19txYMel2TEn2bCn2pl4up2UXDt1Hhtuo53ULnYM8XY118pmU+f2Vr6224+JtMnvV+tCNQ+PNmyAfftCtzOZoKAgOjzq108FS0II0dl5AyGSxWDAEjgy3XanMypkOj4piWyLhW0OBx9WVESFWL/Py6OX3c7XVVXM2rMnavvZAwbQNy6O5/bu5ZZt23D5/bj8fnyBcWwbPZqedjsP7NzJbdu3R42zdNw4Mi0Wbtu2jQd27YqqO044AZvRyB3btvFicXFTeGQ1GIg3GJqmEv5fURHf1NQ0hU9WTSPFZGJmr14AvFdeznaHIyKkSjGZmJqeDsC6hgYafL7Q/jWNeKORTIsFUNMkZfqfEEII0flJSBRDh/oHlcfn4cvtX/Lm2jd5Z8M7VDursZvsOL1OLEYL1466llvH33pk1yzy+1VQdDBhVFERPPwwDB4Mkyerdpjg7RyOppO33omjyoG3zoHe6EBzOjB6nCSaHGiHElIZjT8dJP1U2HSg25jNHWa+V02N6jTauDEyPNq8mYh1orKzW+4+6t5dFs4WQoi28gXWnrIFuoGqPB5K3G5cgeudgTDppJQUzAYDy+rqWFFf33S9KzBl7689emDQNP5bUsLnVVVN17v8fnTgg8GDAbht2zbeLS+P2HeSycT2MWMAOGf1at6vqIgYY0+bjW2B+mkrVvBFdXVEfXB8PKtGjQJg9NKlLKuvj+iUGpeczFsDBwJw4bp17HW5IkKskYmJ3Nq9OwB/27EjIoSyaBoD4uObQqq3y8rQoCmgshgM5Fmt9A6s17XN4cAS6LCyhN1GOq2EEEKIAyMhUQxtvHIj3hovuZfmknpaKprx4P+QcfvczNs6jzfXvcnb696mwdMAgFEzcuHgC3nhnBcwGTpBp8zHH8Po0ZCWdnDb+3xsWOHkxwVOdm9yULTZQfF2B9XFTuZ94MDkdvDUYw4Wfumge5aTbhkOuqQ6yE11MKJ/ZBjVFFLt7+vgAuQHy2A49LApOKWv+dS+n7rOaGzTEL1e2L49uvNo/XqoqgrdLvxIa80Xzo6LO/inSAghxOEXnIYXHkAB9Au8gS+urY0KsZKMRs7PygLg6b172RnoxAruo7fdzl969ADUelXbHI6IEGt8cjLP9usHQMHChex2uXCH/b15QVYW/w0sip6wYAENzQ6PekVuLs/064eu6xjmz496TMH1rhp8Prr/8ENTeBQMkq7r0oWr8/Ko9Hi4cN26iADLomlMz8picloa5W43j+/ZExFQWQ0GTkxOpjA+nmqPh29rapquD96ml81GitmMy++n2uuN2FbWtBJCCNFRSUgUQ9tu28beZ/birfBi7Wol57c55MzIwd7r0A5F5fQ6+WzrZzy77Fk+2fIJXr+XzLhMzi08l9MLTufsfmdjNLQtIIgZrxd8PhVutAO/P9Tl8uGH8OWXqkNm82bYuhXy8tRizwC/+pXqpunTJ3QaNAiOO24/O3e59h8ktSVsOtDbHCqjsW1hUivX6xYrDT4bJTU29lbaKCq3savEyvZ9NnaW2HBgwxk4pXex0rW3jR79bOQX2igYZKPvYCvZuYaO0kglhBCiA9B1HXdgvSoNSAhMBd8YWK/KFXbKtVoZGB+PX9d5taSk6Xp3IIQalZjIaWlpNPp83LJ1a9P1wdtMz8pielYWJW43Z69eHbGty+/nzvx8ruzShXUNDQxcvDhqrM/168dlubksqq1lzLJlUfXgeldfVFVxWrP1royoLq8z0tP5tLKSKzdujAixrAYDT/bpw7DEROZXV/OvPXuwBEOmwPkt3brR1WZjaV0d8yormwKo4O3Ozcgg0WRim8PBFoejqRY8L4yLwxSYaunW9aaaSY4cKIQQxzQJiWLM7/JT/kE5xS8UU/lpJV1v7ErBPwrQ/Tp+hx9j/KGFOQ6Pg7mb5/LWurd4d8O7uH1u4kxxTB80nRnDZjC++3gMWgebI+RwwIQJasrZAw8c9rvzeqGsDHJz1dcPPAALFqgAaccOlVVNmADBg+78+tfqyGHBAKlvX3U6omv16HoomHK5IqfwNZ/Kd7iuC5+DdpDcmPEYbPgsNnSrDWOcFVOCDUuSDYP9wIOrqOuCR+0LPwWPJBj+tcyTE0IIsR+6ruMNBkiB8ySjkQSTiXqvl/WNjVEh1HEJCXS12djtdPJBRUVUCHVJTg594+JYWlfH/xUVRdRcus5jBQUMjI/nvfJybgusZxUM0Ny6zrfDhzMgPp7/Kyri+i1bosa8ffRo8u127t+5k9tbWO+qbNw4MlpY70oDLJpG9YQJ2IxG7ty+nVdKSiI6oeIMBuYPHw7A40VFLKiujpjql2Iy8ffevQGYU1rKFocjIsRKM5s5LzMTgB9ra6nxeiMCsESjkT6BLrbKwN8bwW0lxBJCiMNLQqIOxLXHBQaw5lqp/LySteeuJeuCLHIuzSFpdNIh/0JcVbyK33/8exbsWtB0XXZ8NhcMuoBpA6cxpuuYjhMYzZgBr74KDz6o0pfMTDjrLFVbuVKFFOHTrRISDn6K2n54PGqqldMJQ4ao637xC1ixAnbuDB1m/le/gtdfV5evugq6do3sREpKavehxV6wg+qnwqSwr3WHk5oSJ2VFLir3OqkudlJX5qSh0om3wdXUe2TXnKTanSTbXCRZnMQbndhxYvE7MXjC9u/1ts9jMZn2HyS1Fi4dzttYLB1mvSohhBAdlz8sOAoPorpZrZgNBva4XOxwOlUtEEC5/X7OycjAYjDwQ00NP9bVRe7D7+f+Xr0waBqvFBczr6oqYluAjwJ/GN22bRvvBda7Cm6fbDKxafRooOX1rnrbbGwJrHd16ooVfNlsvauh8fGsCFvv6se6uqaaBpyYksJXw4YBcMqKFWx1ONAATdMwACelpPB8YSEAJy5fTqnHo+qB0xnp6TwcCLFOWL6cBp+vqWbQNM7JyOD2wFTJE5Yvb1pH1BC4zS+zsrguLw+X38/PVq+O2LemaVyYlcVFOTlUezxcvGFDxL414OKcHM7JyKA4cFRGLXC9IbD9b3NyODU1lV1OJ3fv2BGxrQZcmpvL6KQktjQ28s+ioqj6Zbm5DE5IYH1DAy8UF0fsO1jvbbezpr6etwLrfWnNtu9itbKyvp6PKyoi9h0cX7rZzPK6Or6pqYmoacDF2dkkmEwsq6tjaV1d5PbAhdnZWAJrra1vbIx6/qZlZqJpGsvr6tjhdEbs26xpnBFYq2xlfT37XK6IsdsMBk4I/Od2TX09lV5vxL7tBgMjEhMB2NDQQJ3PFzG+OKOxaZrtVocDRwv17jYbALucTry6HvH9tRsMTQv6l7jdTa+d8PElBjokawJ/x4Zvbw4EpQCuQEdl8++vhKTicGtrSNQJFrPp/Kx5oalV1lwrmedlUvJaCfue3UdcYRw5l+aQ97s8jPaD6y4akjOE+TPms6liE3d/fTdvrHmDBk8D/178b2YtmkW3pG78csAvmT5oOqO6jIrtG9DDD6s5YX/8o/q6f/9QSHTddfDdd5G3HzUKfvxRXR45Ui2aE75uz8SJ8Nxzqj5jhlpMJzxkGjkSLrtM1Z94QgUgdjtmm42+drs6JjzqP2bv3r0SLBZcBjs7S2xsKbKR0iUOsFBfD3PnqjW4w911F9x9tzr8/GOPhcKjggKVb3VKwfWU7G2fHqkBKYFTc3V1oUWzlzdbODt8PfKsLCgcodY76t/Hy4DeLvrlu+ia4cToaRZOORxq42C3VfDUlutauk1Dw/63a6+j+wUFA6TDGVAFO6mMxpbPj1RN/uARQoiDYtA0bEYjtlbqeVYrefuZvj82OZmxycmt1i/KyeGinJxW6/f36sX9gaPkteSdQYPwhAVMbl0n/J/P/+rThwqPJ6JLKj5s3cQ/devGXrdbhVSBetewxzMmKYnuVis6NJ0KwxZDHBAfT47Hgx/VEaYDuYEP8QB5FgsNfn9TTQfiwzqMbQYD/rCaP2z8fl2nwedTtbDbNAaCNB+w2+WKqPl1nYpAd5RL11leXx8au67jB6YE/vFZ6/Uyr6qqafvgY5iUmsropCRKPR5ml5ZGjg2YlJbG4IQEdjqdPLFnT8S+dWBSaqoKiRoauHfnzqjv2ZS0NLpYrfxYW8tfWuhCm5qWRrrZzFfV1fxx69ao+tnp6SSYTHxQUcHdwfUcwvwiMxOLwcDrJSU80vyPZuCXJ56Ihlrr7Onww+8CcQYDDRMnAvDQrl28XloaUc82mykePx6A27Zv54P9BJTXbN7M180CymEJCSwfqT4bT1+7lqX19RH1E5KTWRDoopu8ciUbmy0BMTUtrSlAHbFkCXub/W04PTOTNwIL+nf74QfqfL6I+uW5uU1rtdkWLKC58LXWkr75Jirgu71HD+7Mz6fE7ab3woVRAeS9+fn8vmtXtjkcjFm2LCrge7BXLy7KyWFNfT1nBgPQsH08UlDAORkZLK6t5aL16yPCU03TeLyggJNTU/mmuprrt2yJCiif7NOHkUlJzKus5K4WAtCn+valf3w8H1VU8Oju3VEB47P9+tHNZuN/ZWU8t29fVHj2XL9+pJvNzCktZXZZWcTYNNQ04TijkTdKSvi4sjLq+Xu2Xz8MmsbrJSUsqK6OCvgeKSgA4NXiYpbW1zc9viyLhVsCB2I4lkgnUYx467yUvVnGvhf24dzuZMyuMRhMBho2NGDvbcdgPvjOn00VmyiqLWJkl5G8ve5tHvj2AbZXbcere8lPyWfagGlMGziNEbkjYhMYud0qzHE4VMtOz57q+iVLoLQ0tD6P0wmpqXDuuar+2GOwe3fkGj4DBsBf/6rqZ5+t2oDCtz/jDHjpJVVPSVGH+go3Ywa88IK6bDKpOWjhfv97ePxxta+8PPxWGx6jDRc2GnU7dRdcTp9/XM26hbWsGXs5DuxN6/UY4+0MumUqJ905kYrttex88A0yu9nI7mHDkhwIsQYMUAsnOZ2wZ0/0otVH8XQpr1dN+wtfNDsYJpWXh25ns0UvnN2vHyQmhnKJlk7hucX+TprWhhxD19Xrti2BU3uGVz91XXt1XB1OmnZkQ6kDrR3Mi6Yj3v5w7Dv4w7G/U/B73KYfJCGEEEdSeIil6zrGwJQ+r9+PJxhQhYdoRiNGTcPh89HYLGDTgQyzGaOmUev1Uuv1RgRcOtDDZsOgaZS53VQF6uH76B8Xh6ZpFDmdVHi9EfetQVMn0JbGRsqDAWBgHyZNY0wg9FxVX6/qYdvbDaFOo4U1NVHbJ5tMnJyaCsAXVVVUejwRjz/TbOa0QIj3Xnk51V5vxNi7Wq1MDtRfKS6mweeL2H+B3c6UQCfUE3v2NB2JMriPwfHxTfX7d+6MqPlRoejpaWm4/H7+tmNHxPOu6zqnpqYyKS2NWq+Xe8LrgZDwnIwMTk1NpdTt5q4dO6K+d7/JzubElBR2OBzc3cL+r8nLY3xyMusaGiL2H3x+/ty9O8cnJbG4tpa/hY0/+Bz8vVcvhiYk8FVVFffv2hUVgD7Vty+F8fF8UF7Og7t2Rb12Zg8YQL7dzmslJTy6e3fUa3Pe0KFkWyw8uWcP/woEpOH1ZccdR6LJxP07d/LM3r1Rj2/X2LEYNI2bt27l5eLiiJrdaGT32LEAXL5hA2+WlUV8X4Pdj0cDmW7WiXiqPJhTzeg+nR96/IDu0cm+KJvcS3OJHxB/SPuet3UeU16bgtVo5cT8E3F73SzYtQCv30vv1N5MGziN6QOnMyR7yNHf4lhfHxkgOZ1qvliPHioEeP/9yJrDAUOHwimnqK9vvjmy5nTC9Olw0UVQWor/hBPx1DvxNzrRnA6Mbid7r7uPHo//ka+e2sjJ1xRGDWn37U/RbeZVVH2+lNRJLfy8vvYaXHgh/PADXHBBZIBkt8P998PYsbB8uQqzwms2G1xyiXp827apLq3m9WHDVMtTXR3U1h7UUdIOh/LyUGAUftq2TTWDtbeWcowDPcViO6Pmx6y7seLCoquT2e/Cgpu0BDdDBvkZPNBPYpxPPXG+ZuctXXes1Hw+9XPf/PZtOQW3FS1rS5jU1tDpcO+rve6zeaj2U1+3122O1DaHst/9vcm1Vuuo20gYKoQQohOTkKgT0n06FR9XUPyfYirer0D36iSOTqTnzJ6knXbw6/JsqtjEzAUzeW31a1iNVmYMm8HgrMG8s+Edvtz+JT7dR7/0fkwbqDqMBmUNasdHJQAaa71s+b6UnRud7NniYO82JyU7nfzpyV70mZjLfx4u56tb5hKnOclNddA1zUFumosxD51L2omDcS5bh3XWQ2iuZusE/f3vMGYMfPopXHFF9ALU334L48erbqrf/jZ6YMuXq6DoySfVdL9wZjOsXw+9e8PTT8Ojj0aGSHY7/Pe/qtvr3Xfhs8+iQ6gbb1T7WbJEJTzhNbtdTQcEqK5WXTHBWisBlcsFW7bApk1qil9Ln90P5LP+oWx3JO/rYLarqVHnmgYDB6qXwbhx6tS7t3zOOWS6HgqZ2vub15FuH3ycLZ3Cn4cjdbtY3Gdbb9f8+fqprw/XNodrv0I52NS/tVp7vRkfrfvpaDryz0LzQLv5dYfrNp3pPtsS/rf1toe6/ZG8r7Zsv7/n9HBed6Tv72Cuy8tTS6QcBSQk6uQJXPBuAAAgAElEQVTcpW61btHz+yh4pIC009Nw7nTi2OYg5cQUNIN2wPsMhkWbKjbxw2U/oGkaRTVFfLT5I2avnc38nfPx634GZA5g2oBpTB80ncKM6O4X0f527FCNPps3qwBk0yZ1edcuSE6G229XGU1BgZpq1bevOv/1r9UsuRZ5vaE/Quvrobg4ejHqMWNUJ9G6dWoAzTupbrlFhUDvv68CoeadVJ99prqx7r8fZs0K1YPztD0eNcDrrlNBVDirVd0WVMfTyy+HaiYTdOmipg+CmvY3f35kCNW9Ozz7rKrPmqVCqPBOq65d4Te/UfUvvlDPQfj2aWnqCQWorFT3aberc+3Af746mvp6tZzXd9/B99+rZrTgbMusLBUWBYOj445T3w4hhGjVwQZjBxpW7i/E7Izb/NT+2ut7c7TupyP+Pu6IY2oeZje/7nDdpjPdZ1uC/7be9lC3P9j7ErFx5ZXqH+ZHAQmJjhLB74+maWy7bRu7HtiFraeNnBk55FySg617a8sZts7j82A2milvLGfgkwO5eMjF3Dz+Zvy6n3fWv8PstbP5Zuc36OgMyR7StIZRn/Q+7f3wxH6E/230+efwyScqPNq4MZSH1NSo29xwg2rWCQZIffuqwDtmobffrwKg4AKTpaVQVhYZMnk8as0oUCHO+vWRIZXFolYGB3joIZVyhG+fkwP/+5+q//zn8PXXoYWtAUaMgKVL1eWRI0OXgyZOVMETqIWONm5Ulw0GFSSddRa88Ya67tRT1ZMd3kl1yilw002qfvPN6jy8Pny4ug2oTqvwms0GubnqMeh6aN9W62H749PvD2WB33+vzoNrUlos6ikK7zbKyjoswxBCCCGEEAfrcARaLV0+Etcd6fs72Ouys9WRiY4CEhIdhXyNPsrfLWffC/uo/rIaNEg/K51B/xvEwawntLduL7d+fiuvr34dm8nGtSOv5ebxN5MVn8Xeur3MWTeHN9e+yXe71RHHhucMZ/rA6fxy4C/pldqrvR+eOAAeD+zbp5ppQGUoc+eqECl4sIj+/VUoAPDnP6tcJRgg9eunGm0MhtiM/7Dy+1Unk8ejVrcGlYbU1kaGTElJMGGCqr/8cmSI5XSqJ+nyy1V9xozo+uTJqoMKVNdTdbW6Pvieeu216oh6Xq+actfczTerb1x1terWCrJaVWD017+qowCWlMCUKdFT+S67DM48U3WIPfxw9FTA005T4Vd5uTqioNmsThaLOh80iBJ/Jj9+UcfGz3aydJWZZWssNHjMeDCT3juVUeMtjB/rZ9w4GDDIcHS+XoQQQgghhDgGSEh0lHNsd1D8UjH+Rj+9H+oNwM6/7yRtchoJwxMOKDTaWL6Rmd/M5PXVr2M32dl2wzay4kNtBLtrdjNn3Rxmr53Noj2LABjVZVTTGkbdk7u374MTh6S2Vk1Va2hQzTKgsoQFC9QUpKAzz4QPP1SXH30UMjNDAVJKS8eyFz9N11U45XSqBC4hQYVWq1dHT/Xr1QsGD1aLKz3zTPSi6lOnwumnq5Co+XpTTifcequab7hmjVq83OmMPNrZiy+qaXzffRcKw8LNmQPnnafWs5oyJap897hPeXLzZCaWzWEOv8SLEZ9BBU0GixnPB58Qd+IoePttNR8yGEIFg6iXXlKP8cMP1REEm4dU998P6enw1Veqkyy8ZjbDVVepwGvZMtVlFl4zm1UIZjCoOZmVldH3n5enHkhw6qPZ3DGnCAghhBBCCHEESEh0jHHtcbGw90J0l0780HhyL80l+9fZmNNb6GBoxaaKTXyy5ROuH309AP9d/V9O7XVqRGC0o3oHb619izfXvcmSvep7NabrGKYPnM75A86na1LX9n1got3ouuoyCk5Zy8qCX/xC5QqJiaHlgUAFRjfdpDqQ/H61JFHfvmrBY1m7pgPzetV0O6dTdRPFxakQavt2FV4FT263Ws06M1N1In3zTWTN44GzzkLv2o1dn6yj8tk57Nvhpni3h9pyNyY8/NPwJ1KG5XNJ96/4efFTZCR7iDOH3ccLL6gj6736Kjz4YPT+ly1THVj33Qd33qleaOGqqlRaecstqlOqpcdqNMI118BTT0XWbDYVuIFal+q119Rlo1EFSLm5obl2l1wC8+apACm4hlfPnmr6YrC+cGHkIrMDBsCbb6r6xRerlr3w+rBh8O9/h7bftSuyPno0/O1vqj5jhgq5gvevaSrUu/FGVb/8cvX9DB/fxIlqO4Crrw4trBisn3yy+uF2u+G226IXyj35ZDWFsr5eJcThYwvWjz9edbn95z/R9YkTYdAgqKhQUymb18ePVwFhWZkKAcMfm8GgHn9urpqGunRpdH34cLVmWFmZerMKf2yapl678fHq/ouKouu9eqk3qurqyOc2WM/NVa+Fhgb18xF+35qmugw1Tb1Ofb7oevjilkIIIYQQnYSERMcgT5WH0jdKKX6hmLoldWhmjcFzBx/UkdFKG0rp+mhXzEYz1426jj+N+1NEWASwtXIrb659kzfXvcmK4hUATOg+gWkDpnH+gPPJTcxtl8clDj+3W+UIGzeGQqRTToFf/Qp27w5Nawt+fu7bF66/XjWguFzqs1xennxuOhZUV6vMJLi20aJF6rM2qCmM4QtiDx3a8ky7Fvl8kUFWSop6wVVUqFMwXAqexo9X261apQKf8Jquh0KU999XnVzhIZXdDvfeq+pPPAErV4YW3PX7VXj20EOqPnOm6tYK1vx+yM+HRx5R9RtvVD804fVBg1T4AnDRRWoB9vD62LGh+pQpqlss/P6nTIF//EPVhw+HurrI+vTpofHl5kYeiczvV8HR/ferb0xWVuR9+/2q8+uee1RAmNvC+/RDD6npkJs3qx/25v79b3Ufy5apVc+be/VV1eU2fz6cdFJ0/X//g3POUV1mZ50VXf/iC/UG9MYb6k2ouR9/hFGj1ML1V14ZXV+/Xk21fPRRNWWzuaIi9YZ1zz1w993R9ZoaFRT96U+h73O4YHB07bVqDOEhUkKCCr9AjW3OnMh6drZ6zYIKAOfNi6z37KmuA7j0UvVYwwOu/v3h9ddVfcYM2LAhOqB8/HFVv+wy9QYeXj/++NBab1dcocLY8ABs3Dj15g7qe+xyRYZsJ5ygXtOgDijQPIA78UQ4+2z1c3b33ZF1g0HVTzpJBZSPPx4dYE6cqBZIq66GV16JDiAnTFAhbUUFfPBBdH3sWPXzWVamwu/mAeSoUep7UFYGK1ZE14cOVe895eXqUJrNH19hoQrfKyvVf12a13v0UEF0ba16DM3rmZkqoAx2jTY/0lBCQiig9Hqj68EDKwT/dpdfukIIIQ5AW0Oi1o6LJDohc6qZvGvyyLsmj/rV9RS/VEzS6CQAil8ppnFDIzkzcogriPvJfWXFZ7H6mtXM/GYmj/zwCE8sfoJrR17Lnyf8mfS4dAB6p/XmLyf8hb+c8Bc2VWxSgdHaN7n+k+u54ZMbmNhjItMHTue8AedFBUyiY7FY1DSzfv2ia9nZ6nNK8KhrwSApGAwsXao+r8fFRa55dPHF6uBh4Qtwi84vJUVlGMEZal6v+swbviB2sMkmLk59Jg2GRmPHRi6/FMFoVCdbs8X409PVqTVDhqhTa84+W51ac911rdcA7rhj//V//nP/9Vde2X/9k0/2X1++fP/14CJkLYmPD/2gtiQ7W30Dw0MkXQ8dMrFXLxUiNK8nJKj6oEGqS6p5Pbjq+ciRsHZtZICl6+pDPKgXxsKF0fXBg1X9pJNUYBJe9/tDi0dOmgTvvBNd79JF1adMgYyM6HpwPu3UqaF6+G2Cr8HW6sE3tNNPVy/olp674OOzWiMfW/C5A/U4w5//8OcOoFs3FViFjz28Hh+v2kDD6+Fvtg0NKmAMr1dVherbtkUHlOGh4bffRgeUSUmhkOj116MDSLNZ/by5XCroDN8WVNfgSSep/d5+e/Rr8uGH1eumtDQUVoV76ikVEu3YEQqCw736qnp9rV2rptM29957anyLFrUcUH75peqkmzcPLrwwur54sRrfnDlqSmxzGzaoX4DPPqtCxub27FGvzwcfVCFlc7W16nv65z+HguRwwefx6qvVNOUgTVPbBQ9hecklMHt2ZMCUm6uCL4ALLlDvPeH13r3VL3uAc89Vb+jhIdqQIWrxQ1DP3erVkd11o0eHujanTFHfo/D6iSeqUB7Uz255eeT9n3666iwF1ekY3uUXvM9bb1WXTzgh8rGDCs+vu04FcMGDYoTXL7kEfvtbFTD+8pfR9auugmnTVIgc/toK1m+4Qc3T37wZfve76Pqtt6rXzurVoXGG1++8Uz1HP/4Y6iQNd9996jlesCAynA5u/49/qD+q5s0LdaqG3+bxx1X4/cEHarp38+2fflp1aM6ZA2+9FV3/z3/UP1BefRU++ii6/tpr6vLzz6sO0fC6zRY62uy//63e18OlpKij0II6X7EicvvsbHjgAfX1ww+HDiQS1KOHWqcx+DwFj3wbHFvfvqF/CNx9t3pfCzd4sAr1QXXXBn9Ogtsfd1zoe37zzaEDoATrY8eqnxmAP/wh+rk56ST1jw+XS+2/eX3SJPX6rq1V/3hq7swz1c9HeXnoH0Th2//85+q1s2dP9JGCQb1uhw5V/+194YXo7S+8UAXcmzaF/skQXr/kEvW+uWaN+p3avH7ZZep9a/ny0HtAeP2qq9TfaYsWqffQ5vXrrlPvT999p07N69dfr35Xfv21OgpP8/pNN6nLn3+ufr7C62Zz6O+4Tz6JfO1omvo9edll6usPP1TvS+H1lBT1Ty1Q/8DauzeynpkJ55/PsUhCoqNUwuAECv5R0PR1/fJ6imYVsev+XSSfkEzOpTlknp+JKaH1l0C/jH688otXuOOEO5j5zUyeXPIkN45VUyB0XY9Y96hvel/umHgHd0y8g3Vl63hz7ZvMXjuba+dey+8+/h0n55/MtIHTOLf/uWTEZRy+By7ancWi/vk6alTL9fx89TsrGCAtXar+Bpk0Sf0989Zb6v0/eOS14Pkpp0R+XhKdk8mkDiQ3YoRqLADVvPD996HT3/8eOtLzgAGR3UZ9+kiIGDOapoK51hiN+1+gzGJRQUZr4uPVN7w1qanqD9/W5OSoU2vy80OBU0sGDNj//e/vjQ3Um1TwCIUtOeccdWrNJZeoU2tuuKH1GrQcIoT717/2Xw8enbE1X3yx//qaNfuvV1S0XktMDK0HFhR+pJicnNBC/+EhU3A+c+/eqtuneQCZnKzqgwapkKt5Pfh6GTkyukNQ19V+Qb35fPttdH3YMFU/6ST4+OPogLEg8HfVaaepNLx5PRiyhQeI4fXg+M84I7IePAUf/5lnqg8nzY9GFB6Y5OZG1sLbNqdMCR09M3gKHsgBVAiTnR1Zz8wM1SdMiLx/vz/UUgzqDT8tLfK5KywM1QsL1WMNf3zBdeJAtZ3GxUXWg89N8PVjNEZOQw5/fBZL6DUV1NKRFcJfc+H7Cq7fF7598JeU368CquZ1jyd0u9ra6Hrw9e7xqA/7zevB4MHlivwg2rze2KjC99a2r6kJhX3htwnef0WFCiubbx98zCUloW7G8Hrw+dmzR3WJNq8Hbd8eCoGCdbs9VN+wQQVd4cJfWytXhoKE4PY9e4bqP/6ojmYbLviPA1AdgqtWRY5t7NhQSPTZZ6Hp5MH7qK0NhUTvv6+eg+bPbTAkmj1bdTo2f+zBkOjFFyPHDurxn3OO+t4Hw9vwemqqek9oaGj5fbtLFxUSVVXBY49Fb9+rl/pdWVwc6iION2iQCol27gwdUCV8+1GjQiFRS79XTjxR/S5dvTrUaRrujDPUGBcvbvkfZ+edp0Kib7+NDMmCLr5Y/UzPm9fy/V99tXrv+/DDlrt3g9/bt96KDMdB/Z0RDIlefhn++9/Iek5OKCR6+unQYqxBffqEQqLHHgsd9ThoxIhjNiSS6WbHENceF8WvFFP8QjGOzQ5ST09l6CdD27x9RWNFUxfRz17/GQMyB7Q4DS1I13XWlK5pCow2V27GqBk5tdepjOoyirzEPPKS8prOs+KzMGhy+KSjgcul/r4zmVRI8PzzoRCprEzdZts29XfBiy+qUCk8QOrbV/0+kvDg6NDQoP62CHYbff+9mokBqlEjPDQaOTK6mUgIIYQQQrST5p//g9NYmx8GHkIdfX7//7N33uFxVOfbvmdmd7W76pJlWXLvNjbGuHfTq4EQSughgIHQQgIJJQkGwg9DgBCSUAwhQL5QQgsdQq82BONu494lS1aXVrvS7s7M98fR7O5oJWODwTa8t665ZvY855wpO1rtPHrPe5JmaqruTAwSj3esZ2Qkh9E6ZmuqHgwqvbU1XbdtFcEKKkrQMUNT2zv/zAqF0ttrWjKEvbExvb2uJ6PV6+rS23s8O45m3weRnERCp9i2TcMnKtQyb0oe0aooiw9ZTPFZxRSfU0xGyY4zE7fEW5j58kyeWPoEfo+/05xF7fe5uHIx/172b55f+Tzratdh2qarjkf3UJpd6jaPsrvTI6eHy0zye+QJcl+mrk4ZRmPGKCPpwQdVBPqaNclcw5qmtjMyVOTz2rVuAyn1H6LCvodlqX82OsPT5s5V9wSo7xqjR7uNox0FkwiCIAiCIAiC8NWISSTsNM3Lm1n989U0fNQABhQeXUi387pReGwhuq/zyJ5V1au45aNbEmbRa2e8xvQ+03dqn6ZlUtlcSVljGWVNZWxt3JrYLmsqo6xRlTXH0vNpFAQKkuZRu2gkp7wgUOAaDifs/ViWSgWwerWKdnZGaZx3noo2Sv2oOuCA5JD2V15RptLgwSpa1iODaPdJqqpUhLljGn3+eTK6vl8/t2k0bNiOR0kJgiAIgiAIguBGTCJhlwmvDlPxaAUVj1UQLY8yfsN4An0CmBETI9D5E9mq6lXc/end3HXEXWT6Mvmi/At65vb8xsmqbdumsbUxYRqlmkepZtL25u3YuO9jv8fvikrqkd0jzUwqyS7BZ/i+0TEK3w0tLWqIuTNkzbbhuuuUNmJEMo+d16tSRsyYkRy2vWSJGrrWRVJh7VO0tqociakJsZ1clDk5MGFC0jgaP16iywRBEARBEARhR4hJJHxtrLhF0+dN5E5USQSXHr+UaEWUbud1o+tpXfHmdT6ntW3bDLtvGJsaNu3UMLTdQcyMsS20zR2NlBKV5JS3mq2udhoaXTO7uoa2pUYjOds5GTkSlbQXU1OjjKPUZfBglSzZttVQ4ro6lWPTmcHtuOPU5C2ghif7xCvc67FtlS8zdYja0qWqXNeVWZgabdS7t+S0EgRBEARBEAQHMYmE3cbWv21l24PbaF7ajO7X6fLjLnS/tDu5k3I7rL+yeiW3fHgLTy57cqdzFn3b2LZNbaTWZSB1NMStJpI+W0umN9NtHrUb4tYjpwfFmcUYuox/2RG2aWM2m5ihtqUpZftrLGigeTQ0Q1PrDhY8GnUNGs0tGqEWjaZmjcZm6NlbY/8DNUw0nnxaI5ilkZ2nkVugkVuo0bOPRkGR9pX9d7h8C/XREaOyAxoa1Iyrjmn06acqbyGo6LFJk5LG0ciRYgYKgiAIgiAIP1zEJBJ2K7Zt0/RFExWPVFD5eCW9ft2L3r/tjdVq0bqtlUCfQFqbVLPo8R8/zmnDT9sDR75rRGIRypvK04a4JUylpjLKm8qJW3FXO0Mz6JbVbYcJt7tndyfTl7mHzmzXsGLWbjFyUhcrYn31jtvQfBpGltH5EjRAAztuY5u2Wu/K0tbGjNo01Kq1FVNlBjYBn41HU68xv/p4vwsShpHBTplPRtDAV+ojozSjw7Un1/O9M55MU0UXpUYbbdyoNL8fxo1LmkYTJiQnxNC05OK8Fr572s9m3tH2vqLt6vbe2tc37ReSE+N0tN7bte9iP4aRnA3U2W6/dKbtSrmzP0EQBOGHi5hEe4jyOeVsvHkjmldD9+rqgc2rdbjeLfq33bee/o3CjJjYcRtPtoftz2xnxU9WkH9oPt3O60aXH3VJy1+0umY1/fP7Y+gGf/70z5Q3lXNI30PI9GaS5csiP5BPn7w+AFi2ha51nix7b8CyLaqaqzqNRnLKG1sb09rmZuS6zaOUaCRnu0uwy05fA9u2sVp3v6FjR3f+c0EP6Ds2dL7GsqOE6d8mTvLszEw1TG3ZMvjVL23WrLIp2wIGykT612M2xxxps3SRzb8ft+nfx6Zvb5u+vWyKu9holv31DKxvYHi1X8wmk+i2KK3lrZgN6U6XHtA7NZEyuqttX4kPT9a+nQm8vNxtGi1YoGZp3Vk6MpDab3/V671J29V+vktTRdg1OjIfOjMjvun2120Pyfe5s/XeoDnb3/f7sDMTancZUd+G2ZV6H32d9Tdpuzv6+C73/1VlX6fNNynb2/b3XW/vS8exO9jb+tmdff3sZ3DPPbunrz2NmER7iNo3a9n+9HbstqiE9msrZnVYvkM9lnzw+87R2KHBhA1ms0m8Po4ds8EAX5GPwMAAuj/diJpdMpsnc5/E0pJRJcOjw3mm8hk0TePEriey3rueoBUkYAfItDMZHR3NjQ03gga3Zd9GWA8TtIMEbVVnYHwgB8cOBmCedx5ezZtoGyRItpVNUAsmzif13KDdMB6N3VavmWYq9UoqqKBSr6SSSiq0Ciq0CiqppFKrpIoq17UA8Npe8shDszU0S11j19pKrrFQ9dDQbLVf5zU2ifLEa10Zf7qup23rRgdrQ63Ttj1t2x4dTdPUQudr57rsqM6urv0eP0FvkIAnQNAbVNvelO1Oyh3NZ/h2KZImHIY1a1TOo+nTobgYnnkGLrgAGlP8QL9fzcw1fLgyJJYvT+ZCyu14hOa3jtls0rqtlWi5Mo0S6zL3aytspbU1coxOo5Gcta/Eh+HfN4ZbhsMwf756jyKRzr9Efd0vXHta2937MIydNwu+DW1P7vvbME12twEj7H5s+7s1qkwzucTj7tdfVf512uyt+zD3kqjdXSHVWN+Z9ddps6ttv6rs67T5JmV72/6+6+196Th2B3tbP7urr4kT4ZRTvnk/ewNiEn0Pse22qIHdaUC1rb9JWzVsxyRWGaN1cytmk0nmyEwwIVYTQ/fp6tjb6lZ6K6nMrKTF00LYE8Yf9zNu6zgAnh32LJWZlUS8ESKeCBFvhL61fbng8wsA+PmPfk55Tjlhb5gWbwu2ZnPwmoP5w3//AMBRM48ilBFyXbejVxzN9W9dDzb86IIf4bW8BKIB/DE/gViAg1cdzAlLTyCuxbl/2v0uzR/1M7hyMP239yeux1lftJ5AayCh+aN+PLaHxO9R6q+TTVpZ+3qmZlKbWUtVThXV2dVqyammMdgIXtB8GngBD8qUa9vGixp65LxOWVQuHsDAvdZBeUW2ej++w3Xi/t1NfVq2RavZSiQWoTnWjGWnmxtfha7pO28yedqZTCn1/J4AraEgVeVBKrYEKNsY5NdXBinKC3DbLUH+dHsAbBUZVVyszKJXX4WsLFi7Vj0o9O2r/mO6J7FtG7PRdJtInaw7ijTzFHi+2kzq5kP37t2RgoIgCMJ3T2emWSrftsGys30IgiDsq4hJJOwxbMtG0zWsVotPij/BClsUHldIyXkl5B+Zj+7ZPQ+Jtm0TiUcwLZPsDDX/9WdbP6Mp2kRztJlQNERzrJkBBQM4pO8hWLbF5a9dTigWSuihaIhTh53KFeOvoKGlgZ539yQUDSWMDYCbDrqJG6bfwNbGrfS8u2facfzpiD/xy4m/ZH3deo578rjEMDpnuWj0RUzvM53ypnL+ufifZPmyEnUyfZmMLhlNcVYxkViE2kgtmb5Mgt4gXt3L9y1nzLeBbdvErBiRWIRwLEw4FiYST9nupNylxTuvl1refoa8ncWrZeCxg2jxIMSCDOijTKaNq4NUbA2gxYNkB4IUZAXoVhjksOnKiPLYQXKDQYLeHUdFOdteo/OZB3cXtm0Tr41/tZm0rTU9n5MG3iJvYjhbp2ZSkQ/NkHtfEARBEARBEHYXYhIJewWhZSGV7Pr/VRKriuEr8THwvoEU/aiI1vJWGj9tVMPRUnI4ZY3MwpPrIVYfI1oWTehOHU+hB92jq6gq21azQ+1GM8UxnxwjKTsjmy7BLoRjYd5a91bCfApFldl0eP/DmdBjApvqN3HVm1e5tFA0xOxDZ3PSfifx8eaPmfrI1LT9PXfqc/x46I/579r/ctTjR7k0r+7l9TNf59B+h/Lyqpf5+as/x2t48Rk+fIYPr+7lsR89xv7F+/Pftf/lz5/9OVHuM3x4DS+3HnIr3XO68+GmD3l51cvJtm39XDj6QnIyclhcsZgllUsS5U4fh/Q9BK/hZUvDFqrD1a62Xt1LaXYpmqYRM2NomoahGd9bc8u0TFriLbtkRu3IpKpuiFDbFCbUorRWK4ztaVvY9c9mQzNSIpz8eHRPYvEaXtdrj+7Bq3s7r6N13G6HbVLqGBgQAuqBOrBrbaxaC7vaxqq2oArMShO7xsYwDTyWB8My1IJBoEuAYNcggW4Bgt2CBEoCZJZmktE9I2EmeQvFSBUEQRAEQRCEnWFnTaJ9OyOpsNeTNTyLAXcNoN/sftS8WkPFIxVkdM8AoPF/jSw/aXlam5EfjiRvah41L9ew8pyVafroBaPJPjCb8gfLWXPJGgCXkTRmwRgC/QKUzyln8x2bk0m425YRr47AW+il4p8VbP/39qRB1aYPfnAwwYwgza80Y35s0uBtoMnbhObVGOkbSe9rewNQ934dkdURtPUaFd4KAt4ADwQfoMupXQBlkMVr42gNGo2fNzLCM4LqU6qx+9iEoiHqttfR3NLMgJwBxJviDMoZxAPHPEBzvJlwLEzMjBE1o/TOU/vrltWNowYcRdSMEjWjxCyl+z1+AFriLdSEaxLlUTNKzIwRiUcAWFyxmHs/v5eoGcW0kyEepw8/nZyMHF5Y+QI3fnBj2vWuv6aeXCOXez67h7vm3ZWmx38fx9AMLn/9cuZ8MQcNLWEk5erH+LEAACAASURBVGbksvVXWwG46r9X8dra11wGVnFmMS+c9gIAsz+azcKKhUkTTPdRml3KrINmAfDIwkfY0rglYU55dA/dsrrxk+E/AeCV1a9Q31KPR/dgaAYe3UPXzK5M7jUZgHlb5tFqtrr0gkAB/Qv6A7C+bj22bSeMDkNXhktORg4A4VgYj+5J5EL6tswJ5dvbRM0ocx4JM39RhNUbwqzfEqaqLsKYSWFuvEWZTLNuiaD5whR2C5NXFCG7IExWXhjNF0lE2cWsGHEr7lpipiprjbeq1zuok3idUufrDO8DQAOK2pahu9jWAraCvllPmkmWgRevej81ZVh5PV48Hg9erxevz4s3w4vP68NjpBtduqZyaumajobmep1Whr5r9VNe78vtUpfUPtr3tbvqdVRXjEBBEARBEITvDokkEvYY8YY4LRtbVL6jlATdWaOy8OZ5adnUQuNnjcmcSW11ik4twtfFR+P8Rmpfr02UO3X63NAHb6GX6per2f7U9jR92NPD8OR6KLu/jG0Pb0vuu63O+FXj0TN01v1mHWX3lSU0bGVGTY9OB+DLc7+k8rFK1zl58j1MqZ0CwLKTl1H9XLVLz+iVwcRNEwFYfORi6t6sc+nB/YKMW67yMy06dBFN/2tSU54bGpqhkT0umxGvjABgybFLlEnlTItuaOROzmXQfYMAWH7KcqKVUTVsx9Gn5tLnd32wbItl5y+jNdxKQA9gGAZNvibMcSZ5P84jakZZf9d6YnaMAzgAj+FhjWcN24duJ3BggNZYKxWvVBDX4pyRcQaaR+N9831WFq/E6G7QGm2lfnk9hm4wq2gWGPBww8N84f0CK9MiGo0Srg6T7cnmsWGPoRkaV6+5mk+aPyGmxYjGo0SjUXpm9uT9w95HMzSOfOdIPt7+set6jSoaxbxT5qHpGmOfGsvi6sUufXrP6bxz2jugw5AHh7C2bq1LnzFwBi+f8TIA3e7sRmWz+/08ffjpPHHSEwBk3ppJOBZOaIZmcNHoi7j32HuxbIuiO4pcBpRH9zBz1Eyum3odoWiIKf+YkjCfHP28kefx05E/pSZcw09f+KnLoPLoHs7a/yyOHng0FaEKbv7gZsy4gRnzkJdjYGgeljx1EhULxrKqbBuRwY+CbTBurMEZP/GgaTqRpUdx3gkDac3YysurXqZ9IvGjBhxFr9xebKrfxDsb3kkkBgfQUHpxVjEb6jYwd8vcRHsblRfq4D4Hk52RzZqaNSyqWIRpm4mcUaZtMr77eLyGlw11G1hbtxbLtrAsK6EPLxqOpmlsadxCeVO50m0L0zIxbZO+eX2xbIuKUAU1oRpi4RjxcJxoOEo8HCc/kk+0OUptay3NsWZi0RimaWJqJmgQiAWI63HCgTBxfxwrw8L22lheC9uw8Rk+LCzidhyzbWychYWNra5TWw4v0zZVZJemZje0sRP1nB+r7ce21XZqX5ZtueoLu07COGoz63R0V5lj5KUaes4acJeRNKPa96uhpfeN5tJc++vgB0j8rnSopaxdP1rbRANaJ3q7PnbmZ0f766yfne3/q/oEkhMndPB+ppVpGu1/Pb6qrfP71FG9DsvtHey7o7Y78eu60/vupMyjeRJGt/PZ7/wdMXQjEdWZ+rcj8XfGaNs2lO7VvRhGsp5T7kR9GoaRmLQCve28ddRrDde2Uyd126nfUdmu9iHmryAIwnePDDcThN2MbSoTyZnFKd4QxwyZLgMLGzKHZQLQvLyZaEXUlfRbz9ApPKYQgJpXa2jZ1OIysLwFXkovKgVg61+30rKhRQ2rM20wwd/HT69regGw/rr1qr0zBbppk3VAFn1v6gvAijNWEN0WTbS3TZu86Xn0v11FziyYuIBYXQxMEn0UnVzEgD8NAOCTrp9gtViu/ZdeUsrAewZitph8FPgo7Rr1uq4X/W7tR7Qqytyuc9P0vrf2pfd1vYlsiPBZv8/S9AF/HUCPy3oQWhJi/gHpnwUD/jGALmd3oeqTKhYevRDN1shuUfmoqrKr6DWnF7mH51LzYQ1fXvIl/pifHrU9APiy+5eU3l1K5uhMqj+oZsPsDeSGc9mvbD/Q4YOhH1B8UzEZ/TKo+aSG8ifK6V7fnbGbx6LpGk+NeYqiS4vQu+jUz6+n+r1qhlYNZdrGaViaxZ+n/pnCkwuxAzaNKxppXN7IxLKJHLHxCCLeCLOmziJ7ajamYRLaGCJSHmHGphnM2DyDGn8Nv5z8S3yDfZi2SUt1C9FQlHPXncsJZSewOXMz5084H3LAtE1irSqq5/rV1/Oj7T9iac5Szhl5Ttr1Ou6Zu+i+4gTCUz/jn4ecmaa/dPJLHDfsOF5Y+QIn/vvENP3Dcz9kau+p/GvJvzj7P2en6QsuXMCBJQdy/+f3c8lrl6Tpay5fw4CCAfzxkz9yzdvXpOkVV1VQnFXM79/9Pbd8dEua3nx9M0FvkCvfuJJ7PnPPPaprOuYNyti54KULeHjhwy4928hmTf81RMujXFx3Ma95X3PpXRq78MyfngHg2jOv5bOB7vuxV1UvHrv3MQCu+NkVLO29FM3WMCwD3dIZUjaEex5Vx3T12VezqWiTK8Jp2JZhXPOiOucbTr2B2uxadCsZBbXflv346Yc/xdIs/jTjT4R9YXRb6ZqtMaRsCEcsOQJLs3jsoMewNAvd0tFttfSv7M/IjSOxsHh11KuJct1SxkXPmp70qepDzIixsI/6XdHtNlPD1ihqLKIgVEDMiLGlcIvS0cEGwzbIbMnEH/cT1+LUB+sThpmmKZPQF/fhsTzE9biaRAAbW7MTZpphGejoxLU4cSOOranPR1tT9SC57SyWZiXWgOt1R3Wc/XVYvot129f7qjrt+3TO+6teQ+d1EtdkN79O3efX7lPbe74rCt8M53PI+bxJ/VxyyhKvHc1OL0utu6M+2pen1bUNPLYH3dbx4MGwDfc2yTyWjomaaj7q6MqESilLreNsJ+qkttfamZo7MkU1wFZ/exL97aBu+35d7ZzjSD3e9mZrW5ljRjv9oOEylJ3P9ITe9qua2E4xRhP92yTaONc1oduaSuVAO91ON4AhpX8b17GkGqxps++2O9YE7V53+Iz6FW12qo/O2Nmqu7seu3Ccu/JRvKsf2xqJv1OapiVf70JZwvzdW8qc2/gbluUfkk/J+SW7dj33UsQkEgThW8O2beyo7TKg7LiN7tfxZHmwTZuWLS1JA6pt8XX14evqw2wxCS0MJcwnRw8ODuLv5SfeEKfunTqXQWWbNjmTcggOCNJa0Ur1f6rVQ6elHj6xoODYAoIDgkQ2RKh6ripRbts2WND1tK4E+gVoXtFM1TNViXKnn9KLSvH38tP4eSNVz1S5+rctm17X9CKjJIO69+qoetatY0O/2f0SUWxp+7dh8IODMTINKh+vpPqFavf+bZthzw5D9+iU3VdGzSs1rv41j8aI11QU2abZm6j9b22irWVZkAtDXxxK3Iqz9ndrqXm3Bsq86NUaJq2sCsS5JnggTz8NvluX0jCvgdxILn6fH7OrSXxSnIH3DgRg6wNbiTfF6V7QnazCLFryWqgvqic4OKgeHi31cNs3vy9+j5+acA3lTeWJ2edAPZQO7TKUDE8GFaEKtjZudc1QBzCqZBRew8vmhs1sadiS0JzogMk9J2PoBmtq1rClcUta+8P7Hw7Asu3L0tp7dA9HDVA5vj4v+5yyprJEO8u2yIhlcGSPI7Ftm3c2v8PW0FbiZnJ4Xa4vlzMHnIlt2/xr9b/YHNqcGHZnWialgVIuHXIptm0ze8lstjRvUW3tOKZlMiRnCNcOvRbbtrn484vZ1rKNuBXHtE3iVpyJBRO5eejN2LbNkXOPpCZao7S29jO6zuD2wbdj2zYDPxxI2AwTt1X/Fhbnl5zPHf3vIGpG6TavW9rv6BXdruCGHjdQG6tl0OJBafp1xdfxy6JfsqV1C6PXjE7T/9DlD8zMm8mX0S85ePPBafrd3e7mjLwzmN8yn2M3Hpum/73n3zk+93jeD73PqRtPTdOf7Pckh+UcxqsNr3LuhnPTInb+M+g/jM8ez7O1z3LVxqtwHvaceq8Me4WhwaE8XvU4f9j8h8SDpPPQ9MaIN+jp78k/tv2Dv5b9NfFw4/T/5qg3yffmM6dsDo+WP5qIDnJ+3h73Nhl6Bn/b/Df+U/GfZOSRpuHRPLw2XpmO92y4h3eq38HQDAxNPdBme7J5eJQyLe/bcB/z6+cnNEM3KPQVcst+t4AGD214iNWh1UrXdAzNoNhfzOUDLgfgn5v+ybaWbYmhd4ZmUBoo5dRe6po+t+U56mP1CU3XdEoDpRza7VDQ4I3yN4iYEQw92X9JoIQDCw4EYG71XEzbTBybrukU+Yvom932z4b6Fa59a5pGji+HAl8BFhaVLZWJ66rr6j3I0DPI8GRgWSpKT9eT1w6S0T+puL6LpphpO6znlHViYHXWvn0Ey+4+HrRO6nbS3rRM9dkTV7/7MTNG3Iwny51hv2Yc0zQTrx09ZsWwLCsxJNjZtiyLmB1L1m1bO59BzmeN87nibFu2pXSSZaZtJj+fSG47n0euctrqo7Yt21JlJMucbUGAdiZVqokEaSZXh/VSjbEO9M6i/To9nh3UTzXXdqbNrpbv6jHttnLHVCXlH0porvKOtnX0ZHn7dbsy5x9RTv/Qtm2nbDttOuqzszLnWGzdZVa2L0/VE9vt+7U72JelMWHYBGb+YeYuvWd7K2ISCYIgCGo2smaLT942efpNH3fdBZGPa3njHy3UbYxxQN8Yhb4YGaUZ9Lu1H6CizBo/b3TNTpZ/ZD4HvHEAAPP6zCNeH8db6E0s+Yfn0/NXava/bQ9vQw/oeAu9eAo9eAu8eLt68WRJGrzdgTOkz9ANbNumsbUx+eDXtmT5sigIFBC34nxZ9aVLNy2Tnrk96ZXbi0gswnsb33M/eFpxRpWMYnCXwdRF6nj+y+cTBpwz3O6gPgcxpMsQKkIVPLP8GZxhdU6d4wYfx6DCQWyo28C/l/87Yd45dU7f/3QGFAxgRdUKnlr2lKtvy7a4eMzF9Mnrw/zy+Ty59Mm0/q+dci3dc7rz/sb309rbts1th91GUWYRL696mWdWPOPq27ZtHjzuQXIycnhi6RM89+Vzae2fO/U5vIaX+z+/nxdXvehqq2s6b579JqByqb2y5hVMy0wMo8zyZfHBuR8A8IvXf8Hra19PaJZtUZJVwqcXfArAKc+cwjvr31EP1W19DOkyhAUXLQBg0sOTmLd1nuv9n9BjAvPOV2X7378/y7Yvc+mH9TuMt85+C4A+f+7DpoZNLv3EISfy/E+eB6DLH7tQE6lx6ecccA6P/egxADJuySBqRl36pWMv5W/H/I2YGcN3iy/t/rx28rXMPmw2tZFaCv9YmCh3zKRbDrmFa6dcy+aGzex3735pOalmHzqbmaNnsqp6FYf885A0/fbDbufk/U5mUcUiznjujDT9j4f/kcP6Hcb/yv7HlW9cmZb/6vbDbmdc93F8svkTbvnolrQ8WLMPnc3QoqF8sPED7p9/f1oOrVsOvoWeuT15b8N7PLXsqbT93zD9BgqDhby74V3eWPtGmn7N5GvI9GXy3ob3mLd1Hn6Pn4AnQMAbIOAJcNJ+J+HRPWys30htpNalBb3BxGyu+zqpQ5Pbf/44hnrMjLnM/69aO/3uybqp/yj5ruqmHl9H26ntOtr+Ou33tX2m3X90Ur6D5+JdbbMv1bdx/w1s/zdzZ7Y7at/Z9nfZ1+7o98JRFzLnuDkdXr99DTGJBEEQhE655hq45x5obYX+/eHMM9UyqC3oxLZt4g1x4rVxYjUx9AydrBFZAGy+fTOtZa3EamLEamLEa+PkH5pPv9n9sG2bD30fYsfdf1tKLy5l0P2DsOIW80fMx1PQZh4VevEUeCg4uoCCwwqwYhYNHzW4DCYjaHzXl0cQ9gpSH6QdEwkg06eGNTuTFaSaVBlGBiXZKix+Tc0aWuItrj7y/HkMLFRRg59s/iQxmYGTD6w0u5QDuilD+LkVzyXaOXUGFw5mfI/xmJbJQwsecmmmZTK2+1im9Z5GOBbmrrl3Jb5sO8d4eP/DOajPQdRGarn1o1sTucic5eT9TuagPgdR1ljGje/fmMjp5egzR81kWu9prK5Zze/e/V1a+2smX8PkXpP5ovwLrnvnukS5cxx3HH4H47qP490N77r1tmv96I8eZWS3kTz/5fNc/871rr5tbF4/83WGdBnCwwse5nfvpe9/8cWL6ZXbizs+uYNZ789K06t+XUVhsJDr37me2R/PTnvPo7+L4jW8XPrqpdw3/z6X5tW9RH+vTLtLXr2E5758zmUiFWcV8/qZrwNw97y7WVy52KV3y+rG5eNVlNpb696iJlKTMJ8C3gD5/nyGdR0GQF2kDl3TCXgDeHWZSVIQhB8uHUWh7quISSQIgiDskIYGeP55ePxxePddOOooeK0tdU9dHeTn73qftm0Tq24zj2rixGrVdnBgkNzJuZjNJivPXZkod+r0urYXfX7fh5atLXza81NXn7pfp/+d/el+aXdatraw9sq1SYOpzUjKOyiPQL8AVqtFvD6Op8CD7tU7OUpBEIQ9ixM10xJvIRKPEImpWSn3K9oPUMNo19WuS2jhWBjTNrli/BUA/HPxP5m7Za6rbdAb5N8n/xuAi16+iDfWvZHQIrEI/fL7sfry1QAc9OhBfLDpA9cxHdjtwEQU29iHxjK/XH0n1zWdgCfAwX0P5uXT1WQPJz19EtuathHwtplMngDju4/nqklXAXD7x7cTNaMJgyrgDTC4cLBrxlGv4XWZWDkZOQkDVBAEQdj9iEkkCIIg7DTl5co0GjoUtmyBfv3g4INVdNGJJ0JOzre7f9uy0XQNs8Wk8dNGl8EUr41TeFwheVPzaF7RzPJTlicMJidiaei/hlJ8ZjH1H9azaPoiAIxsI2EkDbh7gGq/spntT2zHU+DBCBhoXg3Nq1FwZAG+rj5atrbQvKwZ3asnNM2rkblfJkbQIN4YJ14fT5TrPlVP98tU7YIg7N3ErTgeXQ37LW8qp76l3mUi+T1+pvaeCsBTy56irLHMZUL1yeuTMKlmvjSTjQ0bXe2n9Z7GAzMeAKDkrhIqQhWu/Z814iz+34n/D4DA/wVoibe49ItHX8z9M+7HtExyb8tNRDE5Q/UuGXsJ1065loaWBsY+NDZtKN/l4y5n5uiZVIQqOO7J41zDCHVN5xfjf8Epw05hY/1GLnjpAtcwRF3TuXL8lRze/3BWVa/i+nev77D9+B7jWb59OXfOuzM5a2Lbctm4yxjWdRiLKxbzyKJH0oYy/nzsz+mT14dFFYv4z5f/SRvKeMGoC+ia2ZVFFYt4d8O7Sb2tj7MPOJucjBwWVSxifvn8RJ4wp84pw07B7/GzpHIJK6tXJvOwteVbO37w8Ri6wfLty9nUsMmlG5rBof0OBWBV9Soqmytd+/YaXsaUqufK9XXrqW+pd+kZngwGFapQ5K2NW4nEIq5jy/BkUJqtJmapDlcTM2Ou8/fqXnL9uQCEoqHE0N7U4/MaXuD7FdUhCN81O2sSSYIIQRAEgdJStQD4fHDddSrC6Nxz4eKL4fjj4dZb1dC0bwNNV1/4DL9B/kGdhzBl7pfJuOXjgLakryGTWE0Mb4H68ujv52fg3wamRSo5Q9bCy8NsumVT2qwfB849EF9XH3Vv17HqZ6vS9jtmyRiy9s+i4rEK1l6xNk0fv348gb4BNt+xmY03bVQmk6/NSPLqjP5iNN4CL1v/tpXKxypdmubVGP7icHSvTsU/K6h7t85lUul+PTErYfUr1YRXhF0mlZFlUHxGMQCNnzUS3R51GVhGlkH2gSqPScvWFuxW273/DE3yRQnCDwDHIAIozS5NPLR3xGnDT9thXw8d/9AO9W1XqWT9jokUjoXxe/wJ/ZXTXyEcC7tMqKFdhgJqcoGLRl9EJB4hZsYSQw7756vPQUM3GFM6Jm0oX35A/e3Q0Oia2dU1jNDJ4+b07wzDdIYhWrZFJB4BIBKPsLpmddpQxLqWOgBqI7W8u+HdtP2fOuxUhjGMTQ2beHTRo2lDHU8ceiJ98vqwuGIxN394c9o1mzFoBl0zu/Lx5o+56s2r0vRjBh5DTkYOr695nevfvT5NP3rg0fg9fp5a9lSHQxlbftuCoRs8MP8B/vb531xa6lDG2R/P5rHFj7n0fH8+tdfUAvDrt37N818+79J75fZi05Uq/9l5L57HW+vfcunDioax7BKVP+34J4/fYa61iQ9PTMu1dni/wxO54Pre0zdhcjlG1IlDTuTpU55Wx3J3L+pa6lwm2OnDT+e+Y9XwzZK7Soia0WRya03j/APP57bDbiNqRul5t8qvmKpfPu5yrp96PXWROobfPzxN/82k33D5+MvZ2riVyf+YnKbPmj6Lc0eey+qa1Rz9+NFp+m2H3sZJ+53Ewm0LOe2509L0Px/5Z44ccCRzt8xl5ssz0/QHjn2Ayb0m8/b6txP3Tqr+yAmPMLLbSF5e9TI3fnBj4ro6dZ46+SkGFAzg6eVPc9e8u1xtNTSePfVZSrNL+X+L/x9zvpiT0Jw+Xj79ZXL9ufx9wd95fOnjaft//czX8Rk+7v3fvby46kVXe6/u5ZUzXgHUMNm3N7ztap+TkcMTJz0BqAhF595x2ncNdk3kC7rlw1tYXLk40V7TNHrm9OTOI+4EYNZ7s1hTu8Z1bgMKBnDjQeqa/Pad37K1aaur/bCiYVw96Wp+aMi3QkEQBMFFcTHcfDPcdBN8+qkyi557DjLbRgHMnQvxOEyZAvoeHNGlaRqebA+e7OSfMn8PP90v7d5pm6KTipgem068IY7VYmHHbKyYRUb3DAAKjy3kwLkHYsfshGbHbPy91cNN/qH5DP774ES5HbOxohbeQmVSZY/KpvSi0kS5U0fPUBfKyDLwFnkTmtlsYsfshEkWWRuh/r161/41Q0uYRFX/rqLyX5Wuc/IWeRMm0abZm6h50Z2I2N/Xz4T1EwBYee5K6t+pd+mZIzIZu3gsAAsmLaDpiyZlMHnUFLA5E3MY8aqaWW/h9IVE1kRAbzP2NMg7OI+hjw5N6NHKqPrypgMaFBxVwIA7Byj9oIVYzZaaMUpX72Hh8YX0vq43AIsOXaQMPK2tfx26nNiF7hd3x2q1WH7y8kS/jl50chHFpxUTb4iz+uerE/06x1h0ShGFxxQSrYqy4XcbEu2cPopOLSJvSh6t5a1svXtr8thS+s8emU3L5hYqHq1wnTsaFP24iOCgIJH1bbMqgmsK3aKTi/D38hNeHab29VrXFL0AXU/piq/YR/OXzdS/X582/W7RKUV4872EloVo+l+Ta1peTdMoOrkII9MgtDREaHEobSrgopOK0H06ocUhwqvCae27nNgFTddoWtREy8YW15TGmkej8GiVfLppURPR8qhrimAtQ0uYuqHFIWLVMVf/RtAgZ7wKQwwtDRGvj7vOz8gyErnOmlc0YzabaXpwUBCA8KowVqvl2r+RbeDvpX43I+siajbMVD3LwFeskmu3bG5J3FvOYmQaePPV725rRas67JTzN4IGRqaRyNHWXtczdHSfrkyEFivtvdcMDc1QU4on9u308QPAo3vIzsjuMNm2E7XSEV7Dy11H3tWpnuXLSjw0dkRxVjGvnvFqp3q//H58fN7Hneoju41k6c+XdqpP7T01YYh0xPGDj6f+2vpO9Z+O/Ck/HflTl4FlYydMvAtHX8g5B5zjzodl2xQECgC4bNxlnDXirDQTKt+vfhevnHBlQk9NxOtE4lw96WrOPuBsl57KNZOv4ewRZ7uS9zoGG6jk9OeMOMelpxqA1065lnMOOMeVNDjPn5fQfzP5N1SEKlznX5xVnNCvnng1VeEql94nr09Cv3LCldRF6lz7d4ZpApw78lyao82JtgDjuo9L6KcNO42YFQOSSZ4P7KZmfNQ1nR8P+XFaQmzHwPQaXo4ZcEya3jdfzQjp9/g5pO8haQm3u2er7yUBT4BJPSel6V2CXQCVb25Uyag03YmyyvRmsl/Rfml60BtM6P3y+6XpGYb6jhPwBhLmcGod597ze/yJ+yg18beuqe8wHt2D3+NPO3+H1Ps1df8OrWarihRLaZ9qXje2NrK9eburfXOsOaFvb97OxvqNrvbhWDihb27YzIqqFa6k5pFYJKGvrFnJwm0LXeeWGtH4xbYvWFWzytU+Zsb4ISLDzQRBEISvxLKShtCxx6rcRb16wemnqyFp+++/Z4/vh4IVt7CjttuEMm38PdselNdHiNXGVHlUmUy6TydvmvqCXvdOHa1lrS4Dy5PvodvZ3QAou7+Mlk0tCQ1bRWf1/KX6z+qGWRuIlkfVlytLDRPM2j+LnlcpffWlq4nVxMACbKXnTspN6Mt/shwzZKq2bX0UHFmQ0BcdvAjbtLGttodqC4pOLaLnL3tiRkwWTl6Y6NdZl15USo/LexCtirJw0kLXsWFDr+t60f3i7kQ2RFgwcUGiX0fvf2d/Ss4rIbQ4xIJJC1zHZls2Qx9LH8qYyvAXhtPlhC7UvFbD0mPTHywPePsA8g/NZ/u/t7PitBVp+qjPRpEzLofyv5ezeubqNH3sirFkDs1ky91bWPerdWn6hC0T8Pfws/HmjWyctTFNn1I/BU+uh3W/WceWO7ak6dPj09EMjdU/X035A+UuTQ/oTAtPA2DFWSvY/vh2l+7t6mVypfqv+dITllLzUjuDsr+fCWuVQbnokEXUv+d+cM4amcWYhSrq/YuxX9A0v8ml507J5cCP1MPb/4b+j/DKsEsvOLqAEa8pA3Nuj7lEy9wzsRWdWsSwf6tEzB/lfoTZ6J52vdv53Rjy9yEAvK+/nxZh2OPKHgy4ewBms8lHWR/Rnt6/603fP/QlWhllbre5aXq/2/vR6ze9CK8N87+B/0vTB943kO4/705oaYhF0xapCEFfMsKw/x396XJCF0KLQ6z++Wo0T3IIOHgH8gAAH3dJREFUrO7V6fXbXuROyCW0OMTWP291DZHVvTolF5UQHBCk+ctmal6qSdMLjy/EV+QjsjFC8+LmND3rwCyMoKEmKaiOpemefA+ariUetn4o5pcgCMK+iuQkEgRBEL4VQiF48UUVYfTmm2Cayix6ovN/7ArCPo8TDWJbKSYUqAd6Q1MGXqudjBpp+3qlB3V0j44VtZRBZqf899UGT55Ksm5GTBWtktIWWxkxuldX+bDq4q5jAcjomYHu0RNDLJ12jh4YEEAzNKKVURXp06595vBMNE2jZUuL0tvaOnk/skerKJDIugix6pjr/DSPRs44FSnUvLzZvX9Ay9DInaD+A960oIlYbbJ/bJU3LHeS0us/rleRRim6p8BD3lRlcNa+VYvZaLr27+vmSxigVS9UYYWtpMFog7+3n7zpSq/4VwV21E5efxuCg4OJ9mUPlLn2bds2WQdkkTc1DytmUXZvup4zIYe8KXmYzSZlfytzv/c25B2SR+7EXGK1Mcr+Wua+9jYUzigkZ1wOLVta2PLHLa4IRTtmU3ppKXlT8ggtCbHuqnXYcdtVZ8DdA8ibnkfdO3Ws/NlKV/SjHbMZ8foI8qbnUflEJV+e+WXaPT3q81HkjMmh/MFyVl+UblCOWzmO4OAgW+7awrqr0w3KiVsnktE9g403bWTjjRvTTKTx68bjyfGw+fbNVDxakaaP/HAkmqZRdm8ZdW/XuXQjy2DQ31SOm4rHKmj6oskVJebJ89BnVh8Atv1jG+Ev3VFyvq6+hPlc/lA5LRtbXJFcvu4+ul+sojvKHywnui3qijLz9/HT7SxlnpfPKSdWF3NF+AUGBCj6cVHi3rHCySg3gOCQIIVHFSbaWzHL1T5zWCZ50/OwbZttD21LHLdzfpn7Z5IzLgcrarH9ye0uHSDrgCyyRmRhhk2qX6xOvimOPjKLzCGZxBvj1L5Rm/beZY/JJtAvQKw2Rt27dcnmbUZf9rhs/D39RKuiNHzckNZ/zoQcMrpl0FrRqiIc2+m5k3LxFnppLWsltCiUrk/OxZProWVzC83Lm9P0vKl5GJkGkY0RIqsj6fq0PPQMncj6CC0bWtL03Gm56B6d8NowrVtb087fiYAMrwoTrXCbyxiQN0V9LjSvbCZW5Y4k0X16IkKyeXkzsTq3bgSMxOdmaFkozZxOjaAMLQmpCMoUPDkeMoep0O3Q4lAyStHR8zwEB6vIoaaFTeofOil4C70E+geUvqAp8bfKwVfkw9/bj23bhBa0vTcp/q63qxd/Dz+2aRNaEqI9vm4+MkoysGIW4RXhdL3Uh6/Ih9lidvjeZXTPwFvgxYyYRNZF0tpn9MjAm+fFbDZp2dSSrvfMwJPtId4Ud7+3bf37e/kTOSSj26Lp7XtnYPgN4g1xotvTdX9vP7pPJ1avUhZoXi0Rsfp9QEwiQRAE4Vtn+3Z4+mkoKoKf/ASamuDkk5PL15khTRAEQdh92KaN1WqlmUi+Yh96hk60Okrr5tY0PXdyLkamQfPKZkILQml6yfklGJkGde/XUf9ufZo+4O4B6Bkq11rNKzVu3bQZ+fZIADbevJGqZ6tcuu7XGb9qPAArL1hJ9XPVboOwxJfQl520TA3lhISBFxwUZOwSNYx20SGLqP+wPqFjQ/bobEZ/PhqA+aPmE1rofhjOOyiPke+p4/ts4GdE1rofZguPK2T/l1QI7SfdPiFW6TYKup7Rlf0eV0OgPsz8UJlIKZRcVMLgBwZj2zYf6O5Z5gB6XNWDAXcOIN4Y5+Pc9KFxfW7sQ59ZfWgta2Vej3lpev+7+tPzVz0JrwrzvyHpUWyDHhxE6cxSGuc3smDsgjR96BNDKT69mLr36lh8yOI0ffjLw+kyowvVL1Wz7IRlafrI90cqg/LxSr48K92gHP3FaLJHZVM+p5zVF3dgUK4eR3BgkM13bmb9r9en6RPLJ5JRksGGWRvYdHP60L8pTVPwZHlY+6u1ahhxOw6yDwJg1cxVbPv7NpdmZBtMbVQJ3FecvoLtT7kjKH2lPiaVTQJgyYwl1L7qNuECgwKJe3Ph9IU0fNjg0rNGZzFmvnpG/6b33tySuWkml+vey/pQDfFOoeTCEgbPkXtvZ++91Pfz+4CYRIIgCMJ3ztKlcMopsGqVSoB9zDFqONqMGeD//vwjRhAEQfiekGo+pUbZ6V41xtpqTYlQQ9XDUBMtAMSb4slhqm265k1OCBCtjqZFCOoBHU+OB9u2VbSD7T4OI1vly7ItW0VTpLS3bRtvvhdvoRcrZtGyviV5Hm34inxKb7U6jNbwlfjw5reL5kh5JMzoqaI54qE4LevS+w/0DeDJ9RCrjyX2n9o+MCiAJ9tDrCZGZH16/5nDMjEyDaKVURXl1a7/rAOyMAIGrWWtyWiSlPbZY7PRfTotm1pUvrF2eu7kXDRD6ziSyIb8g9V/sJpXNqdFm2iGlogwDC0LpRmAmk9LRDg2LWpKRGg6GAGD3MkqQrLx88ZEBGhCzzbInaj0hrkNmE3tIonyPYkIzfoP6jHDbt1b5CVnjNLr3qlTudpS8JX4EpNV1Lxeo3K1peDv5SdrRBa2bVPzSk3aMNtA/wCZwzKxYha1r6VHoQUGB8gckokZMTuMUsscnklwYJB4U5y6t9qi1FL2kTUqi0Df9Cg2h5xxOfh7+Yluj6o8fe3InZJLRmkGrWWtiSi31Hsn/5B8fF3VMNrGeY1p7QuOLMBb4CW8JuyOgmuj8LhCPDkempc307SgCU+Ohy4ndEmrt68iJpEgCIKwR7Bt+OILNRztqaegogKWLFF5i+rqICcHDOOr+xEEQRAEQRAEYfewsybRHpyXRhAEQfg+omkwZgzcfTds3QoffphMbP2LX6iE11ddpYykvej/FIIgCIIgCILwg0dMIkEQBOFbwzBg6tTk65NPhrFj4a9/VUbS0KFw33177vgEQRAEQRAEQUgiJpEgCILwnXH88fDCC2oI2pw5UFwMGzcqzTThwQehqmqPHqIgCIIgCIIg/GCRnESCIAjCHsWyQNfhk09gyhQVfXTEESrh9Y9+BJmZe/oIBUEQBEEQBGHfRnISCYIgCPsEettfosmTVYLrq6+GZcvgrLOga1c1Y5ogCIIgCIIgCN8+YhIJgiAIew377w+33aaGoH3wAVx4ocpbBKr8ssvg008l4bUgCIIgCIIgfBuISSQIgiDsdeg6TJumZkjzeFTZtm3w8MMwcSIMHAg33ACrVu3Z4xQEQRAEQRCE7xNiEgmCIAj7BPfcA5WV8Mgj0Lcv/N//wc03J/Xt2/fcsQmCIAiCIAjC9wExiQRBEIR9hpwcOPdceOst2LoV/vAHVb58OZSUwGGHKROpoWGPHqYgCIIgCIIg7JOISSQIgiDsk5SUQL9+ajs/H377W9iwAc47D4qL4ZRTYPPmPXuMgiAIgiAIgrAvISaRIAiCsM9TWqqGnq1dC/PmwcyZ8PnnkJen9LffVomwLWvPHqcgCIIgCIIg7M1o9l40RcyYMWPs+fPn7+nDEARBEL4H2DZomtqeNg0++gh69oTTT1fL4MEQCOzZYxQEQRAEQRCE7wJN076wbXvMV9YTk0gQBEH4vtPcDC++CI8/Dv/9L5gmnHMOPPaY0vv2Ba8XsrKSy4knqogk04RZs9xaVhYMHw5Dhih9wwbIzlblwWDSnBIEQRAEQRCEvYGdNYk838XBCIIgCMKeJDMTzjhDLVVV8Oqr0L270mxbJbwOhZJLVRXU1yu9uRluu02ZQanMmgU33qhmVRs4MFmuaWp/s2fDZZfBli1w5pnpJtMZZ8CECVBdDa+8kq737Qu5uckhcroMEBcEQRAEQRC+ZcQkEgRBEH5QFBWpGdIcNA0eeqjz+jk5EItBa6sykJqa1LqwUOnZ2fDPf7pNpqYmFWkEqq1hKONpw4ZknXHjlEm0ejX87Gfp+336aZV8+5134MgjlfHkGEjZ2XDffar955/D/fenm0ynngrdusG2bbBuXbqemSkRT4IgCIIgCIIbMYkEQRAE4SvQNPD71dKli1vLyoKzz+68bb9+8N57neujR7vNI8eEGjdO6X36wO9/79ZCITWsDZQJ9NZbSd2JeJo8WZlEL78MF12Uvt8vv1TD5R56CP74R7eBlJ0NDzwABQXw0kvw+uvg8Sizy1nffDNkZMCbbyqjKlX3eFQUFcAnn6iE4qm63w8zZih9yRJloDntDEPlijrgAKVv2QKRiHvfGRnK7AMIh9XaMJKLmF+CIAiCIAhfD8lJJAiCIAjfE2wbolFlGOXkqDxL5eWwYkW6yTRzppr97dVX4Ykn3FoopMyd/Hy4/Xb4058gHleLaap1XZ0yc37xC/jLX9zHYRiqDsB558Ejj7j1vDzVHlTE0zPPuPUePZQ5BHD00fDGG2596FB1TgBTpqhjTWX8ePj006S+YoXbZJo2TZ0zwFFHqWuUqh90kBpiCPCTn6hr0153TLCLL1bXxNEMA6ZOhZNPVkMFZ81yG1gej4oAmzZNRaf94x9uzTBg5EgViRYOKwMwVTMMGDRIXaNIBJYtSzfwSkrU+x+NquGM7fsPBNS28xVQTDVBEARB+P4jiasFQRAEQfjWsSy3eeSsnYir6mpobEyWO+bR/vur9cqVKq9TalufDw49VOkffABlZe7+c3KUuQTw5JPKUHI001QmiRM99cc/wtat7v6HDoWrr1b6pZeqaKxUfdIkZe6AMpFqatz7P+64pInUv78ya1L188+Hu+5SJo3fnzRjHK65RrWvqUmPTAO45Rb47W9h40aVm6o999wDV1yhDCLnOqby8MPKnJs3T51Le5yhjG++qYYytjexnn9e5el67TVlJrbXn3gCRo1SubRuuindhHroIXVdXn0V5sxRZbqeXP/lL9C1q9Kfe06Vpeq3364i2l5/Hd59160Zhoqs83jU8S9Y4Na93qSB9/77sGaNu63fr84dlJG4bZu7fWYmTJ+u9CVLVG4yR3N0ZyjpunXQ0uLWg0EoLVV6VZX6/Ujt3+dLRgHG4+o8BEEQBOG7QBJXC4IgCILwreM8+HZGly4dGyEOQ4aopTOcB/bOOP30Heu/+c2O9Xvv3bHePoqpPevWda75fMoksO2kgZVqDOTnK5PC0Ry9oEDpJSXwxRduzTSVAQPQq5caTpiqmaaKpAI11HHOnHQTb8SIpH7DDe62pqmilACKi1UkV/vjy8pSujPsL1WPxZKRSU1NysCzLKVZllpiMaVv3Ahvv52u/+EPSv/0U5V7q73+298q/aWX0t8/ny9pEj3yiMoXlkpBQdIkuuMOZYil0ru3Oi5QRuJbb7n14cNh6VK1fdZZyYg1h4kTYe5ctX3wwbB8uVs/4gg1wyLAgAHKIC0oSC6HH548v7/8RZ1PQYG6VwoKVML9bt0QBEEQhG8NiSQSBEEQBEEQ9jlSh0CmmkiOyVZXp2YnTDWZIGmybdqkIoVS23q9cOCBSl+0SEV7OZppuiON3n1XRcql6l26KGMNVMSWE4Xm6L16wUknKf0vf4HNm9Vx1taqZcoU+L//U3ogoCKVUpk5Ex58UPVXUqJmQHQMpPx8OPFEZYLFYirKztFSF6/323k/BEEQhL0bGW4mCIIgCIIgCPsoLS1uA6muTg1lGzNG5bP65S/T9QsvVNFzFRXKRGrP7Nlw7bUqwmvGDHeUkhNlNXYsNDSohPROeX6+GuYp+asEQRD2XWS4mSAIgiAIgiDso/j9yujpyOzJyFBD8TqjSxc1q6BjHjlG0oQJSrcsle+qtlblbaqtVVFPw4Ypk2j5cjX0LRXDUEnmTzwR5s9XQxVTTaSCAvjxj6FnT7XPykpVnp+/4yGpgiAIwt6FmESCIAiCIAiC8D3C41HD6pyhde3p3RteeCG93BmSN3y4ShqfajLV1SXzh0UiKp/SqlVKq69X5aNGKZPotddUziaHrCxlFr32mur7nXfgqafSI5mOOkrVDYXUsQSDKu+ZpkkUkyAIwneFmESCIAiCIAiCIKDrap2TA9OmdV5v6lQVTeRgmmqImpPUfMoUNQteahRTbW0yX9SmTWp2u9paNXTOYetW1cfdd6tIpfY0NkJ2tpoh8M9/ThpIzrqxUa1/9SuVtDxVz8pSUVOgZgd86SW3XlwMH3+s9MsuUyZZqt6nTzLR+SWXwMKFyVntNA0GD1Yz+4GaNXHNGrc+YoQa7uf0X17u1kePTibav+KK5Lk4+vjxKicVqPOLRt3m2aRJcNppavvKK5PXzNEPOghOOEENY3SSo6dyxBFqtsPGxmTy+NT2M2aoe6KqSs3e2F4/8UQYN07NRpmaUN7RTz0VDjgANmyAf/wjXT/zTHUNV61S+bTac+656j1YuhT+85/09hdcoKLuFixQMyO21y+6CAoL4bPPVD6x9lx2mbq3Pv4YPvkkXb/yShXB99577nvf4de/Vus331QzI6bu3+eDyy9Xr199VZ1jKpmZyRk5X3xRXaNU8vPhpz9V2889p65xKl27Jt/7p55S71HquXfvrt4fgMcfT5q6Dn36wLHHqu3HHoNw2K0PHKhmvAQ1e6Yz+YDD0KHJXG1z5pDGiBEqqX80qvpvz6hR6v5vbu74vR8/Xs3k2dAAzz6brk+erAzs6mr1e92e6dOVYV5R0fFkFIceqsztLVvS742cnOS1+yEhJpEgCIIgCIIgCF8bw0gaQKAilXr37rz+eeepBVRUkmMiFRersiOOUIm7IxE1O6CTHNwZtjZtmjJPnHKnjvNQPHaseiBN1VOHvA0dqh44nXa2rZKAO5SWqgfj1H0XFSX1YFAZCql6aqRTOKxm90vVa2qS+qZNaha91GPPz0/qn36qIrVS9YyMpP7888loq9T3wDEKHn1UrVNTzwYCyiSKxVTy8/YUFiqTqLk5OZQxtX337uq619Upg6693r+/MokqK+HOO9P1/fdXJtGmTXDrren62LHKJFq9Gm66Kf34DjooaRLNmpWuH3OMMok+/xx+97t0/eST1Tl+/DFcf326fu656j19++2O93/JJeo9eOUV+NOf0nXHJHr22aRZ6JCVlTSJ/vUvZeSkUlKSNIn+/ne1j1QGDUqaRH/9qzIwUxk1Kvne33GHMspSmT49aXTcdFPSLHWYMSNpEl17rTJTUjn99KRJdOWV6t5L5cILkybRxReTxq9+pUyi1lZVtz2zZimTqKEhaYSmcued6v6pqFBmYHvmzFEm0aZNcP756foTT6j7c+VK+NnP0vWXXlIm0aJF6j5IZdCgH6ZJJImrBUEQBEEQBEEQhL0W21ZGnG0nzaXUx1gn4ip1JsNU3etVujMjYnt8PqXHYqpOe/x+pbe2puu2nYyia2lJ6qn7z85W63C44/5zctQ6FErXdT2pNzam64aRNDmdGRtT9+3xQF6e2q6tdZ+/Y6A6elWV23wEZY45emWlu29QBqSz/23b0s8tGFS6ZXWsZ2er8zPNjvXcXFUnFks3sEAZrFlZyhjuSC8sVNFaLS0d6127qmMMh9X5peLxKAPp+4LMbiYIgiAIgiAIgiAIgiDstEmkfxcHIwiCIAiCIAiCIAiCIOzdiEkkCIIgCIIgCIIgCIIgiEkkCIIgCIIgCIIgCIIgiEkkCIIgCIIgCIIgCIIgICaRIAiCIAiCIAiCIAiCgJhEgiAIgiAIgiAIgiAIAmISCYIgCIIgCIIgCIIgCIhJJAiCIAiCIAiCIAiCICAmkSAIgiAIgiD8//buP9bXgq4D+PvdvanxY6CJVlwmqIy6OgFjzKRci2pYTKzZ0pTRj61/tNTcCqtVa62x5bK2WMrUoMm0RjKZI4Wo0doyIeKHgCajkksYt1Xkjxkin/44X9tNuQXn3HOf++W8XtvZ+T7Pee7zvL/bZ+f7nPf3+T4XAIiSCAAAAIAoiQAAAACIkggAAACAKIkAAAAAiJIIAAAAgCiJAAAAAIiSCAAAAIAoiQAAAACIkggAAACAKIkAAAAAiJIIAAAAgCiJAAAAAIiSCAAAAIAoiQAAAACIkggAAACAKIkAAAAAiJIIAAAAgCiJAAAAAIiSCAAAAIAoiQAAAACIkggAAACAKIkAAAAAiJIIAAAAgGxzSdT2vLafbHtP24u381gAAAAAbN62lURtdyW5NMnLk+xN8pq2e7freAAAAABs3nZeSXR2kntm5t6ZeTjJ+5NcsI3HAwAAAGCTtrMkOjHJfQcs71ut+1/a/nTbm9vevH///m2MAwAAAMDBLH7j6pm5bGbOmpmzTjjhhKXjAAAAAOxI21kS3Z/kpAOW96zWAQAAAHCE2c6S6KYkp7Y9pe1Tkrw6yTXbeDwAAAAANmn3du14Zh5p+4YkH0myK8l7ZubO7ToeAAAAAJu3bSVRkszMtUmu3c5jAAAAALB1i9+4GgAAAIDlKYkAAAAAUBIBAAAAoCQCAAAAIEoiAAAAAKIkAgAAACBJZ2bpDP+j7f4k/7R0jkPgmUn+dekQrDUzxFaZIbbKDLFVZoitMkNslRliq55MM/ScmTnh/9voiCqJniza3jwzZy2dg/VlhtgqM8RWmSG2ygyxVWaIrTJDbNVOnCEfNwMAAABASQQAAACAkmi7XLZ0ANaeGWKrzBBbZYbYKjPEVpkhtsoMsVU7bobckwgAAAAAVxIBAAAAoCQ65Nqe1/aTbe9pe/HSeVgvbU9q+xdt72p7Z9s3Lp2J9dN2V9u/a/uhpbOwntoe3/aqtp9oe3fb71g6E+ul7ZtXr2Mfb/u+tk9bOhNHtrbvaftg248fsO4Zba9v+6nV96cvmZEj20Fm6LdWr2W3t7267fFLZuTI9lgzdMDP3tJ22j5ziWyHk5LoEGq7K8mlSV6eZG+S17Tdu2wq1swjSd4yM3uTvCTJ680Qm/DGJHcvHYK19rtJPjwz35rk9JgnnoC2Jyb52SRnzcwLk+xK8uplU7EGLk9y3letuzjJDTNzapIbVstwMJfna2fo+iQvnJkXJfn7JG893KFYK5fna2cobU9K8v1JPn24Ay1BSXRonZ3knpm5d2YeTvL+JBcsnIk1MjMPzMwtq8efzcYfZicum4p10nZPkh9M8q6ls7Ce2h6X5GVJ3p0kM/PwzPzHsqlYQ7uTfEPb3UmOSvLPC+fhCDczf5nk375q9QVJrlg9viLJKw9rKNbKY83QzFw3M4+sFj+aZM9hD8baOMjvoSR5e5KfT7IjbuisJDq0Tkxy3wHL++IPfDap7clJzkzyN8smYc38TjZexB5dOghr65Qk+5P8wepji+9qe/TSoVgfM3N/krdl4x3XB5I8NDPXLZuKNfXsmXlg9fgzSZ69ZBjW3k8m+dOlQ7Be2l6Q5P6ZuW3pLIeLkgiOQG2PSfInSd40M/+5dB7WQ9vzkzw4M3+7dBbW2u4kL07y+zNzZpLPx0c8eAJW9425IBuF47ckObrt65ZNxbqbjf+SeUe8i8+h1/aXsnFbhyuXzsL6aHtUkl9M8itLZzmclESH1v1JTjpgec9qHTxubb8+GwXRlTPzgaXzsFbOSfKKtv+YjY+7fk/b9y4biTW0L8m+mfnKVYxXZaM0gsfre5P8w8zsn5kvJflAkpcunIn19C9tvzlJVt8fXDgPa6jtjyc5P8lrV2UjPF7Py8YbHretzq/3JLml7TctmmqbKYkOrZuSnNr2lLZPycZNGq9ZOBNrpG2zcR+Qu2fmt5fOw3qZmbfOzJ6ZOTkbv3/+fGa8e88TMjOfSXJf29NWq85NcteCkVg/n07ykrZHrV7Xzo2bn7M51yS5aPX4oiQfXDALa6jtedn4GP4rZuYLS+dhvczMHTPzrJk5eXV+vS/Ji1fnSk9aSqJDaHVTtDck+Ug2Tob+eGbuXDYVa+acJBdm4wqQW1dfP7B0KGDH+ZkkV7a9PckZSX5z4TyskdVVaFcluSXJHdk437xs0VAc8dq+L8lfJzmt7b62P5XkkiTf1/ZT2bhC7ZIlM3JkO8gM/V6SY5NcvzqvfseiITmiHWSGdpy64g4AAAAAVxIBAAAAoCQCAAAAQEkEAAAAQJREAAAAAERJBAAAAECURAAAh0zb7277oaVzAABshpIIAAAAACURALDztH1d24+1vbXtO9vuavu5tm9ve2fbG9qesNr2jLYfbXt726vbPn21/vlt/6ztbW1vafu81e6PaXtV20+0vbJtV9tf0vau1X7ettBTBwA4KCURALCjtP22JD+a5JyZOSPJl5O8NsnRSW6emRckuTHJr67+yR8m+YWZeVGSOw5Yf2WSS2fm9CQvTfLAav2ZSd6UZG+S5yY5p+03JvmhJC9Y7ec3tvdZAgA8cUoiAGCnOTfJtye5qe2tq+XnJnk0yR+ttnlvku9se1yS42fmxtX6K5K8rO2xSU6cmauTZGa+ODNfWG3zsZnZNzOPJrk1yclJHkryxSTvbvvDSb6yLQDAEUNJBADsNE1yxcycsfo6bWZ+7TG2m03u/78OePzlJLtn5pEkZye5Ksn5ST68yX0DAGwbJREAsNPckORVbZ+VJG2f0fY52TgvetVqmx9L8lcz81CSf2/7Xav1Fya5cWY+m2Rf21eu9vHUtkcd7IBtj0ly3Mxcm+TNSU7fjicGALAVu5cOAABwOM3MXW1/Ocl1bb8uyZeSvD7J55OcvfrZg9m4b1GSXJTkHasS6N4kP7Faf2GSd7b99dU+fuT/OOyxST7Y9mnZuJLp5w7x0wIA2LLObPZKagCAJ4+2n5uZY5bOAQCwFB83AwAAAMCVRAAAAAC4kggAAACAKIkAAAAAiJIIAAAAgCiJAAAAAIiSCAAAAIAoiQAAAABI8t9cJfvKNrGeZQAAAABJRU5ErkJggg==\",\n      \"text/plain\": [\n       \"<Figure size 1440x1440 with 1 Axes>\"\n      ]\n     },\n     \"metadata\": {}\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Acknowledgement\\n\",\n    \"\\n\",\n    \"This tutorial is inspired by some examples from [xlvector/github](https://github.com/xlvector/).\"\n   ],\n   \"metadata\": {}\n  }\n ],\n \"metadata\": {\n  \"anaconda-cloud\": {},\n  \"kernelspec\": {\n   \"display_name\": \"Python 3\",\n   \"language\": \"python\",\n   \"name\": \"python3\"\n  },\n  \"language_info\": {\n   \"codemirror_mode\": {\n    \"name\": \"ipython\",\n    \"version\": 3\n   },\n   \"file_extension\": \".py\",\n   \"mimetype\": \"text/x-python\",\n   \"name\": \"python\",\n   \"nbconvert_exporter\": \"python\",\n   \"pygments_lexer\": \"ipython3\",\n   \"version\": \"3.6.4\"\n  }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}"
  },
  {
    "path": "example/recommenders/demo2-dssm.ipynb",
    "content": "{\n \"cells\": [\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"# Licensed to the Apache Software Foundation (ASF) under one\\n\",\n    \"# or more contributor license agreements.  See the NOTICE file\\n\",\n    \"# distributed with this work for additional information\\n\",\n    \"# regarding copyright ownership.  The ASF licenses this file\\n\",\n    \"# to you under the Apache License, Version 2.0 (the\\n\",\n    \"# \\\"License\\\"); you may not use this file except in compliance\\n\",\n    \"# with the License.  You may obtain a copy of the License at\\n\",\n    \"#\\n\",\n    \"#   http://www.apache.org/licenses/LICENSE-2.0\\n\",\n    \"#\\n\",\n    \"# Unless required by applicable law or agreed to in writing,\\n\",\n    \"# software distributed under the License is distributed on an\\n\",\n    \"# \\\"AS IS\\\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\\n\",\n    \"# KIND, either express or implied.  See the License for the\\n\",\n    \"# specific language governing permissions and limitations\\n\",\n    \"# under the License.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"# Content-based recommender using Deep Structured Semantic Model\\n\",\n    \"\\n\",\n    \"An example of how to build a Deep Structured Semantic Model (DSSM) for incorporating complex content-based features into a recommender system.  See [Learning Deep Structured Semantic Models for Web Search using Clickthrough Data](https://www.microsoft.com/en-us/research/publication/learning-deep-structured-semantic-models-for-web-search-using-clickthrough-data/).  This example does not attempt to provide a datasource or train a model, but merely show how to structure a complex DSSM network.\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 1,\n   \"source\": [\n    \"import warnings\\n\",\n    \"\\n\",\n    \"import mxnet as mx\\n\",\n    \"from mxnet import gluon, np, npx, autograd, sym\\n\",\n    \"import numpy as onp\\n\",\n    \"from sklearn.random_projection import johnson_lindenstrauss_min_dim\\n\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {\n    \"collapsed\": false\n   }\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 2,\n   \"source\": [\n    \"# Define some constants\\n\",\n    \"max_user = int(1e5)\\n\",\n    \"title_vocab_size = int(3e4)\\n\",\n    \"query_vocab_size = int(3e4)\\n\",\n    \"num_samples = int(1e4)\\n\",\n    \"hidden_units = 128\\n\",\n    \"epsilon_proj = 0.25\\n\",\n    \"\\n\",\n    \"ctx = mx.gpu() if mx.device.num_gpus() > 0 else mx.cpu()\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {\n    \"collapsed\": true\n   }\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"## Bag of words random projection\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"A previous version of this example contained a bag of word random projection example, it is kept here for reference but not used in the next example.\\n\",\n    \"Random Projection is a dimension reduction technique that guarantees the disruption of the pair-wise distance between your original data point within a certain bound.\\n\",\n    \"What is even more interesting is that the dimension to project onto to guarantee that bound does not depend on the original number of dimension but solely on the total number of datapoints.\\n\",\n    \"You can see more explanation [in this blog post](http://jasonpunyon.com/blog/2017/12/02/fun-with-random-numbers-random-projection/)\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 3,\n   \"source\": [\n    \"proj_dim = johnson_lindenstrauss_min_dim(num_samples, epsilon_proj)\\n\",\n    \"print(\\\"To keep a distance disruption ~< {}% of our {} samples we need to randomly project to at least {} dimensions\\\".format(epsilon_proj*100, num_samples, proj_dim))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"To keep a distance disruption ~< 25.0% of our 10000 samples we need to randomly project to at least 1414 dimensions\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 4,\n   \"source\": [\n    \"class BagOfWordsRandomProjection(gluon.HybridBlock):\\n\",\n    \"    def __init__(self, vocab_size, output_dim, random_seed=54321, pad_index=0):\\n\",\n    \"        \\\"\\\"\\\"\\n\",\n    \"        :param int vocab_size: number of element in the vocabulary\\n\",\n    \"        :param int output_dim: projection dimension\\n\",\n    \"        :param int ramdon_seed: seed to use to guarantee same projection\\n\",\n    \"        :param int pad_index: index of the vocabulary used for padding sentences\\n\",\n    \"        \\\"\\\"\\\"\\n\",\n    \"        super(BagOfWordsRandomProjection, self).__init__()\\n\",\n    \"        self._vocab_size = vocab_size\\n\",\n    \"        self._output_dim = output_dim\\n\",\n    \"        proj = self._random_unit_vecs(vocab_size=vocab_size, output_dim=output_dim, random_seed=random_seed)\\n\",\n    \"        # we set the projection of the padding word to 0\\n\",\n    \"        proj[pad_index, :] = 0\\n\",\n    \"        self.proj = self.params.get_constant('proj', value=proj)\\n\",\n    \"\\n\",\n    \"    def _random_unit_vecs(self, vocab_size, output_dim, random_seed):\\n\",\n    \"        rs = onp.random.RandomState(seed=random_seed)\\n\",\n    \"        W = rs.normal(size=(vocab_size, output_dim))\\n\",\n    \"        Wlen = np.linalg.norm(W, axis=1)\\n\",\n    \"        W_unit = W / Wlen[:,None]\\n\",\n    \"        return W_unit\\n\",\n    \"\\n\",\n    \"    def forward(self, x, proj):\\n\",\n    \"        \\\"\\\"\\\"\\n\",\n    \"        :param nd or sym F:\\n\",\n    \"        :param nd.NDArray x: index of tokens\\n\",\n    \"        returns the sum of the projected embeddings of each token\\n\",\n    \"        \\\"\\\"\\\"\\n\",\n    \"        embedded = npx.embedding(x, proj, input_dim=self._vocab_size, output_dim=self._output_dim)\\n\",\n    \"        return embedded.sum(axis=1)\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 5,\n   \"source\": [\n    \"bowrp = BagOfWordsRandomProjection(1000, 20)\\n\",\n    \"bowrp.initialize()\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 6,\n   \"source\": [\n    \"bowrp(mx.np.array([[10, 50, 100], [5, 10, 0]]))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"execute_result\",\n     \"data\": {\n      \"text/plain\": [\n       \"\\n\",\n       \"[[ 0.35554492  0.0736109  -0.1220893   0.11155054 -0.20963743  0.21141198\\n\",\n       \"   0.12296599  0.12428369 -0.10999548 -0.16867855 -0.09068598  0.14154953\\n\",\n       \"  -0.24029303  0.11956739  0.02830955 -0.14226514 -0.45963028 -0.5456747\\n\",\n       \"  -0.5663947  -0.10585886]\\n\",\n       \" [-0.31655627 -0.13582113 -0.13815539  0.42596683  0.25674546  0.5024462\\n\",\n       \"  -0.3122709   0.01826438 -0.0277671  -0.14526835  0.44378105  0.09626544\\n\",\n       \"   0.24572927  0.36588538  0.17922089 -0.21583243 -0.30497772  0.19484927\\n\",\n       \"  -0.20705326 -0.13759173]]\\n\",\n       \"<NDArray 2x20 @cpu(0)>\"\n      ]\n     },\n     \"metadata\": {},\n     \"execution_count\": 6\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"With padding:\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 7,\n   \"source\": [\n    \"bowrp(mx.np.array([[10, 50, 100, 0], [5, 10, 0, 0]]))\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"execute_result\",\n     \"data\": {\n      \"text/plain\": [\n       \"\\n\",\n       \"[[ 0.35554492  0.0736109  -0.1220893   0.11155054 -0.20963743  0.21141198\\n\",\n       \"   0.12296599  0.12428369 -0.10999548 -0.16867855 -0.09068598  0.14154953\\n\",\n       \"  -0.24029303  0.11956739  0.02830955 -0.14226514 -0.45963028 -0.5456747\\n\",\n       \"  -0.5663947  -0.10585886]\\n\",\n       \" [-0.31655627 -0.13582113 -0.13815539  0.42596683  0.25674546  0.5024462\\n\",\n       \"  -0.3122709   0.01826438 -0.0277671  -0.14526835  0.44378105  0.09626544\\n\",\n       \"   0.24572927  0.36588538  0.17922089 -0.21583243 -0.30497772  0.19484927\\n\",\n       \"  -0.20705326 -0.13759173]]\\n\",\n       \"<NDArray 2x20 @cpu(0)>\"\n      ]\n     },\n     \"metadata\": {},\n     \"execution_count\": 7\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"# Content-based recommender / ranking system using DSSM\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"For example in the search result ranking problem:\\n\",\n    \"You have users, that have performed text-based searches. They were presented with results, and selected one of them.\\n\",\n    \"Results are composed of a title and an image.\\n\",\n    \"\\n\",\n    \"Your positive examples will be the clicked items in the search results, and the negative examples are sampled from the non-clicked examples.\\n\",\n    \"\\n\",\n    \"The network will jointly learn embeddings for users and query text making up the \\\"Query\\\", title and image making the \\\"Item\\\" and learn how similar they are.\\n\",\n    \"\\n\",\n    \"After training, you can index the embeddings for your items and do a knn search with your query embeddings using the cosine similarity to return ranked items\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 8,\n   \"source\": [\n    \"proj_dim = 128\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 9,\n   \"source\": [\n    \"class DSSMRecommenderNetwork(gluon.HybridBlock):\\n\",\n    \"    def __init__(self, query_vocab_size, proj_dim, max_user, title_vocab_size, hidden_units, random_seed=54321, p=0.5):\\n\",\n    \"        super(DSSMRecommenderNetwork, self).__init__()\\n\",\n    \"            \\n\",\n    \"        # User/Query pipeline\\n\",\n    \"        self.user_embedding = gluon.nn.Embedding(max_user, proj_dim)\\n\",\n    \"        self.user_mlp = gluon.nn.Dense(hidden_units, activation=\\\"relu\\\")\\n\",\n    \"        \\n\",\n    \"        # Instead of bag of words, we use learned embeddings + stacked biLSTM average\\n\",\n    \"        self.query_text_embedding = gluon.nn.Embedding(query_vocab_size, proj_dim)\\n\",\n    \"        self.query_lstm = gluon.rnn.LSTM(hidden_units, 2, bidirectional=True)\\n\",\n    \"        self.query_text_mlp = gluon.nn.Dense(hidden_units, activation=\\\"relu\\\")            \\n\",\n    \"        \\n\",\n    \"        self.query_dropout = gluon.nn.Dropout(p)\\n\",\n    \"        self.query_mlp = gluon.nn.Dense(hidden_units, activation=\\\"relu\\\")\\n\",\n    \"\\n\",\n    \"        # Item pipeline\\n\",\n    \"        # Instead of bag of words, we use learned embeddings + stacked biLSTM average\\n\",\n    \"        self.title_embedding = gluon.nn.Embedding(title_vocab_size, proj_dim)\\n\",\n    \"        self.title_lstm = gluon.rnn.LSTM(hidden_units, 2, bidirectional=True)\\n\",\n    \"        self.title_mlp = gluon.nn.Dense(hidden_units, activation=\\\"relu\\\")\\n\",\n    \"        \\n\",\n    \"        # You could use vgg here for example\\n\",\n    \"        self.image_embedding = gluon.model_zoo.vision.resnet18_v2(pretrained=False).features \\n\",\n    \"        self.image_mlp = gluon.nn.Dense(hidden_units, activation=\\\"relu\\\")\\n\",\n    \"        \\n\",\n    \"        self.item_dropout = gluon.nn.Dropout(p)\\n\",\n    \"        self.item_mlp = gluon.nn.Dense(hidden_units, activation=\\\"relu\\\")\\n\",\n    \"    \\n\",\n    \"    def forward(self, user, query_text, title, image):\\n\",\n    \"        # Query\\n\",\n    \"        user = self.user_embedding(user)\\n\",\n    \"        user = self.user_mlp(user)\\n\",\n    \"\\n\",\n    \"        query_text = self.query_text_embedding(query_text)\\n\",\n    \"        query_text = self.query_lstm(query_text.transpose((1,0,2)))\\n\",\n    \"        # average the states\\n\",\n    \"        query_text = query_text.mean(axis=0)\\n\",\n    \"        query_text = self.query_text_mlp(query_text)\\n\",\n    \"        \\n\",\n    \"        query = np.concatenate([user, query_text])\\n\",\n    \"        query = self.query_dropout(query)\\n\",\n    \"        query = self.query_mlp(query)\\n\",\n    \"        \\n\",\n    \"        # Item\\n\",\n    \"        title_text = self.title_embedding(title)\\n\",\n    \"        title_text = self.title_lstm(title_text.transpose((1,0,2)))\\n\",\n    \"        # average the states\\n\",\n    \"        title_text = title_text.mean(axis=0)\\n\",\n    \"        title_text = self.title_mlp(title_text)\\n\",\n    \"        \\n\",\n    \"        image = self.image_embedding(image)\\n\",\n    \"        image = self.image_mlp(image)\\n\",\n    \"        \\n\",\n    \"        item = np.concatenate([title_text, image])\\n\",\n    \"        item = self.item_dropout(item)\\n\",\n    \"        item = self.item_mlp(item)\\n\",\n    \"        \\n\",\n    \"        # Cosine Similarity\\n\",\n    \"        query = query.expand_dims(axis=2)\\n\",\n    \"        item = item.expand_dims(axis=2)\\n\",\n    \"        sim = npx.batch_dot(query, item, transpose_a=True) / np.expand_dims((np.norm(query, axis=1) * np.norm(item, axis=1) + 1e-9), axis=2)\\n\",\n    \"        \\n\",\n    \"        return sim.squeeze(axis=2)\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 10,\n   \"source\": [\n    \"network = DSSMRecommenderNetwork(\\n\",\n    \"    query_vocab_size,\\n\",\n    \"    proj_dim,\\n\",\n    \"    max_user,\\n\",\n    \"    title_vocab_size,\\n\",\n    \"    hidden_units\\n\",\n    \")\\n\",\n    \"\\n\",\n    \"\\n\",\n    \"network.initialize(mx.init.Xavier(), ctx)\\n\",\n    \"\\n\",\n    \"# Load pre-trained vgg16 weights\\n\",\n    \"with network.name_scope():\\n\",\n    \"    network.image_embedding = gluon.model_zoo.vision.resnet18_v2(pretrained=True, ctx=ctx).features\"\n   ],\n   \"outputs\": [],\n   \"metadata\": {\n    \"collapsed\": false,\n    \"scrolled\": false\n   }\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"It is quite hard to visualize the network since it is relatively complex but you can see the two-pronged structure, and the resnet18 branch\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 11,\n   \"source\": [\n    \"mx.viz.plot_network(network(\\n\",\n    \"                        mx.sym.var('user'), mx.sym.var('query_text'), mx.sym.var('title'), mx.sym.var('image')),\\n\",\n    \"                    shape={'user': (1,1), 'query_text': (1,30), 'title': (1,30), 'image': (1,3,224,224)},\\n\",\n    \"                    node_attrs={\\\"fixedsize\\\":\\\"False\\\"})\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"execute_result\",\n     \"data\": {\n      \"image/svg+xml\": \"<?xml version=\\\"1.0\\\" encoding=\\\"UTF-8\\\" standalone=\\\"no\\\"?>\\n<!DOCTYPE svg PUBLIC \\\"-//W3C//DTD SVG 1.1//EN\\\"\\n \\\"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\\\">\\n<!-- Generated by graphviz version 2.38.0 (20140413.2041)\\n -->\\n<!-- Title: plot Pages: 1 -->\\n<svg width=\\\"10034pt\\\" height=\\\"8697pt\\\"\\n viewBox=\\\"0.00 0.00 10034.00 8697.00\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\" xmlns:xlink=\\\"http://www.w3.org/1999/xlink\\\">\\n<g id=\\\"graph0\\\" class=\\\"graph\\\" transform=\\\"scale(1 1) rotate(0) translate(4 8693)\\\">\\n<title>plot</title>\\n<polygon fill=\\\"white\\\" stroke=\\\"none\\\" points=\\\"-4,4 -4,-8693 10030,-8693 10030,4 -4,4\\\"/>\\n<!-- user -->\\n<g id=\\\"node1\\\" class=\\\"node\\\"><title>user</title>\\n<ellipse fill=\\\"#8dd3c7\\\" stroke=\\\"black\\\" cx=\\\"6280\\\" cy=\\\"-7134\\\" rx=\\\"47\\\" ry=\\\"29\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6280\\\" y=\\\"-7130.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">user</text>\\n</g>\\n<!-- dssmrecommendernetwork0_embedding0_fwd -->\\n<g id=\\\"node2\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_embedding0_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"6417,-7272 6143,-7272 6143,-7214 6417,-7214 6417,-7272\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6280\\\" y=\\\"-7239.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_embedding0_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_embedding0_fwd&#45;&gt;user -->\\n<g id=\\\"edge1\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_embedding0_fwd&#45;&gt;user</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M6280,-7203.58C6280,-7190.28 6280,-7175.63 6280,-7163.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"6280,-7213.89 6275.5,-7203.89 6280,-7208.89 6280,-7203.89 6280,-7203.89 6280,-7203.89 6280,-7208.89 6284.5,-7203.89 6280,-7213.89 6280,-7213.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6283.5\\\" y=\\\"-7184.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense0_fwd -->\\n<g id=\\\"node3\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_dense0_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"6331.5,-7381 6228.5,-7381 6228.5,-7323 6331.5,-7323 6331.5,-7381\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6280\\\" y=\\\"-7355.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">FullyConnected</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"6280\\\" y=\\\"-7340.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense0_fwd&#45;&gt;dssmrecommendernetwork0_embedding0_fwd -->\\n<g id=\\\"edge2\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_dense0_fwd&#45;&gt;dssmrecommendernetwork0_embedding0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M6280,-7312.58C6280,-7299.28 6280,-7284.63 6280,-7272.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"6280,-7322.89 6275.5,-7312.89 6280,-7317.89 6280,-7312.89 6280,-7312.89 6280,-7312.89 6280,-7317.89 6284.5,-7312.89 6280,-7322.89 6280,-7322.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6297\\\" y=\\\"-7293.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1x128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense0_relu_fwd -->\\n<g id=\\\"node4\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_dense0_relu_fwd</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"6327,-7490 6233,-7490 6233,-7432 6327,-7432 6327,-7490\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6280\\\" y=\\\"-7464.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"6280\\\" y=\\\"-7449.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense0_relu_fwd&#45;&gt;dssmrecommendernetwork0_dense0_fwd -->\\n<g id=\\\"edge3\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_dense0_relu_fwd&#45;&gt;dssmrecommendernetwork0_dense0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M6280,-7421.58C6280,-7408.28 6280,-7393.63 6280,-7381.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"6280,-7431.89 6275.5,-7421.89 6280,-7426.89 6280,-7421.89 6280,-7421.89 6280,-7421.89 6280,-7426.89 6284.5,-7421.89 6280,-7431.89 6280,-7431.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6290.5\\\" y=\\\"-7402.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- query_text -->\\n<g id=\\\"node5\\\" class=\\\"node\\\"><title>query_text</title>\\n<ellipse fill=\\\"#8dd3c7\\\" stroke=\\\"black\\\" cx=\\\"5334\\\" cy=\\\"-6807\\\" rx=\\\"47\\\" ry=\\\"29\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5334\\\" y=\\\"-6803.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">query_text</text>\\n</g>\\n<!-- dssmrecommendernetwork0_embedding1_fwd -->\\n<g id=\\\"node6\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_embedding1_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5471,-6945 5197,-6945 5197,-6887 5471,-6887 5471,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5334\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_embedding1_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_embedding1_fwd&#45;&gt;query_text -->\\n<g id=\\\"edge4\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_embedding1_fwd&#45;&gt;query_text</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5334,-6876.58C5334,-6863.28 5334,-6848.63 5334,-6836.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5334,-6886.89 5329.5,-6876.89 5334,-6881.89 5334,-6876.89 5334,-6876.89 5334,-6876.89 5334,-6881.89 5338.5,-6876.89 5334,-6886.89 5334,-6886.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5341\\\" y=\\\"-6857.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">30</text>\\n</g>\\n<!-- dssmrecommendernetwork0_transpose0 -->\\n<g id=\\\"node7\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_transpose0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"6661,-7054 6425,-7054 6425,-6996 6661,-6996 6661,-7054\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6543\\\" y=\\\"-7021.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_transpose0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_transpose0&#45;&gt;dssmrecommendernetwork0_embedding1_fwd -->\\n<g id=\\\"edge5\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_transpose0&#45;&gt;dssmrecommendernetwork0_embedding1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M6415.31,-6996.04C6415.2,-6996.02 6415.1,-6996.01 6415,-6996 6227.61,-6973.73 5752.88,-7004.18 5566,-6978 5517.23,-6971.17 5464.01,-6957.63 5420.87,-6945.04\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"6424.93,-6997.3 6414.43,-7000.46 6419.97,-6996.65 6415.01,-6996 6415.01,-6996 6415.01,-6996 6419.97,-6996.65 6415.6,-6991.54 6424.93,-6997.3 6424.93,-6997.3\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5586.5\\\" y=\\\"-6966.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">30x128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_reshape0 -->\\n<g id=\\\"node8\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_reshape0</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"8052,-6945 7788,-6945 7788,-6887 8052,-6887 8052,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"7920\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_reshape0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_reshape1 -->\\n<g id=\\\"node9\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_reshape1</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"8334,-6945 8070,-6945 8070,-6887 8334,-6887 8334,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"8202\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_reshape1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_reshape2 -->\\n<g id=\\\"node10\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_reshape2</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"8616,-6945 8352,-6945 8352,-6887 8616,-6887 8616,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"8484\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_reshape2</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_reshape3 -->\\n<g id=\\\"node11\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_reshape3</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"8898,-6945 8634,-6945 8634,-6887 8898,-6887 8898,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"8766\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_reshape3</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_reshape4 -->\\n<g id=\\\"node12\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_reshape4</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"9180,-6945 8916,-6945 8916,-6887 9180,-6887 9180,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"9048\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_reshape4</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_reshape5 -->\\n<g id=\\\"node13\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_reshape5</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"9462,-6945 9198,-6945 9198,-6887 9462,-6887 9462,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"9330\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_reshape5</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_reshape6 -->\\n<g id=\\\"node14\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_reshape6</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"9744,-6945 9480,-6945 9480,-6887 9744,-6887 9744,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"9612\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_reshape6</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_reshape7 -->\\n<g id=\\\"node15\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_reshape7</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"10026,-6945 9762,-6945 9762,-6887 10026,-6887 10026,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"9894\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_reshape7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_reshape8 -->\\n<g id=\\\"node16\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_reshape8</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"5753,-6945 5489,-6945 5489,-6887 5753,-6887 5753,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5621\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_reshape8</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_reshape9 -->\\n<g id=\\\"node17\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_reshape9</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"6035,-6945 5771,-6945 5771,-6887 6035,-6887 6035,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5903\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_reshape9</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_reshape10 -->\\n<g id=\\\"node18\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_reshape10</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"6324.5,-6945 6053.5,-6945 6053.5,-6887 6324.5,-6887 6324.5,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6189\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_reshape10</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_reshape11 -->\\n<g id=\\\"node19\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_reshape11</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"6613.5,-6945 6342.5,-6945 6342.5,-6887 6613.5,-6887 6613.5,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6478\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_reshape11</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_reshape12 -->\\n<g id=\\\"node20\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_reshape12</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"6902.5,-6945 6631.5,-6945 6631.5,-6887 6902.5,-6887 6902.5,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6767\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_reshape12</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_reshape13 -->\\n<g id=\\\"node21\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_reshape13</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"7191.5,-6945 6920.5,-6945 6920.5,-6887 7191.5,-6887 7191.5,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"7056\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_reshape13</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_reshape14 -->\\n<g id=\\\"node22\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_reshape14</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"7480.5,-6945 7209.5,-6945 7209.5,-6887 7480.5,-6887 7480.5,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"7345\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_reshape14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_reshape15 -->\\n<g id=\\\"node23\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_reshape15</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"7769.5,-6945 7498.5,-6945 7498.5,-6887 7769.5,-6887 7769.5,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"7634\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_reshape15</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0 -->\\n<g id=\\\"node24\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"7799.5,-7054 7468.5,-7054 7468.5,-6996 7799.5,-6996 7799.5,-7054\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"7634\\\" y=\\\"-7021.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0__rnn_param_concat0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape0 -->\\n<g id=\\\"edge6\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M7718.62,-6992.34C7758.67,-6977.36 7806.1,-6959.61 7844.86,-6945.11\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"7709.14,-6995.89 7716.93,-6988.17 7713.83,-6994.13 7718.51,-6992.38 7718.51,-6992.38 7718.51,-6992.38 7713.83,-6994.13 7720.09,-6996.6 7709.14,-6995.89 7709.14,-6995.89\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape1 -->\\n<g id=\\\"edge7\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M7798.34,-6994.11C7877.74,-6979.54 7974.38,-6961.61 8061,-6945 8063.94,-6944.44 8066.92,-6943.86 8069.92,-6943.28\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"7788.25,-6995.96 7797.28,-6989.73 7793.17,-6995.06 7798.09,-6994.16 7798.09,-6994.16 7798.09,-6994.16 7793.17,-6995.06 7798.9,-6998.59 7788.25,-6995.96 7788.25,-6995.96\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape2 -->\\n<g id=\\\"edge8\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape2</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M7809.74,-6995.94C8047.21,-6966.19 8110.34,-6979.12 8343,-6945 8345.96,-6944.57 8348.96,-6944.11 8351.98,-6943.64\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"7799.76,-6997.2 7809.12,-6991.48 7804.73,-6996.58 7809.69,-6995.95 7809.69,-6995.95 7809.69,-6995.95 7804.73,-6996.58 7810.25,-7000.41 7799.76,-6997.2 7799.76,-6997.2\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape3 -->\\n<g id=\\\"edge9\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape3</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M7809.8,-6995.96C7941.25,-6981.82 7976.1,-6985.95 8104,-6978 8335.57,-6963.6 8394.93,-6975 8625,-6945 8627.97,-6944.61 8630.97,-6944.2 8633.99,-6943.77\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"7799.75,-6997.06 7809.2,-6991.5 7804.72,-6996.52 7809.69,-6995.97 7809.69,-6995.97 7809.69,-6995.97 7804.72,-6996.52 7810.18,-7000.45 7799.75,-6997.06 7799.75,-6997.06\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape4 -->\\n<g id=\\\"edge10\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape4</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M7809.99,-6995.96C8022.75,-6975.52 8078.51,-6987.52 8288,-6978 8563.22,-6965.49 8633.64,-6979.25 8907,-6945 8909.97,-6944.63 8912.97,-6944.23 8916,-6943.81\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"7799.74,-6996.96 7809.25,-6991.51 7804.71,-6996.48 7809.69,-6995.99 7809.69,-6995.99 7809.69,-6995.99 7804.71,-6996.48 7810.13,-7000.47 7799.74,-6996.96 7799.74,-6996.96\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape5 -->\\n<g id=\\\"edge11\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape5</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M7810.11,-6995.96C8125.96,-6967.42 8208.07,-6989.68 8521,-6978 8818.04,-6966.91 8893.98,-6981.38 9189,-6945 9191.85,-6944.65 9194.73,-6944.27 9197.62,-6943.87\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"7799.73,-6996.91 7809.28,-6991.52 7804.71,-6996.45 7809.69,-6996 7809.69,-6996 7809.69,-6996 7804.71,-6996.45 7810.1,-7000.48 7799.73,-6996.91 7799.73,-6996.91\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape6 -->\\n<g id=\\\"edge12\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape6</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M7809.77,-6996.02C8039.25,-6977.18 8565.9,-6985.02 8778,-6978 9086.18,-6967.8 9164.94,-6982.47 9471,-6945 9473.85,-6944.65 9476.73,-6944.28 9479.62,-6943.88\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"7799.73,-6996.88 7809.31,-6991.54 7804.71,-6996.45 7809.69,-6996.03 7809.69,-6996.03 7809.69,-6996.03 7804.71,-6996.45 7810.08,-7000.51 7799.73,-6996.88 7799.73,-6996.88\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape7 -->\\n<g id=\\\"edge13\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape7</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M7809.83,-6996.01C8098.49,-6972.58 8772.87,-6986.28 9044,-6978 9359.31,-6968.37 9439.86,-6983.16 9753,-6945 9755.85,-6944.65 9758.73,-6944.28 9761.62,-6943.89\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"7799.73,-6996.86 7809.32,-6991.54 7804.71,-6996.44 7809.69,-6996.02 7809.69,-6996.02 7809.69,-6996.02 7804.71,-6996.44 7810.07,-7000.51 7799.73,-6996.86 7799.73,-6996.86\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape8 -->\\n<g id=\\\"edge14\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape8</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M7458.48,-7019.26C7132.54,-7010.31 6445.32,-6990.67 6207,-6978 6008.96,-6967.47 5958.52,-6971.7 5762,-6945 5759.03,-6944.6 5756.03,-6944.17 5753.01,-6943.72\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"7468.49,-7019.54 7458.37,-7023.76 7463.49,-7019.4 7458.49,-7019.26 7458.49,-7019.26 7458.49,-7019.26 7463.49,-7019.4 7458.62,-7014.76 7468.49,-7019.54 7468.49,-7019.54\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape9 -->\\n<g id=\\\"edge15\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape9</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M7458.17,-7018.53C7235.85,-7011.22 6841.38,-6996.96 6504,-6978 6299.35,-6966.5 6247.14,-6972.35 6044,-6945 6041.03,-6944.6 6038.03,-6944.18 6035.01,-6943.73\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"7468.3,-7018.86 7458.15,-7023.03 7463.3,-7018.7 7458.3,-7018.53 7458.3,-7018.53 7458.3,-7018.53 7463.3,-7018.7 7458.45,-7014.03 7468.3,-7018.86 7468.3,-7018.86\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape10 -->\\n<g id=\\\"edge16\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape10</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M7458.16,-7021.34C7207.41,-7015.73 6733.15,-6998.38 6333,-6945 6330.22,-6944.63 6327.42,-6944.24 6324.6,-6943.83\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"7468.45,-7021.57 7458.36,-7025.84 7463.46,-7021.46 7458.46,-7021.35 7458.46,-7021.35 7458.46,-7021.35 7463.46,-7021.46 7458.56,-7016.85 7468.45,-7021.57 7468.45,-7021.57\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape11 -->\\n<g id=\\\"edge17\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape11</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M7458.23,-7016.29C7256.05,-7006.03 6914.46,-6984.41 6623,-6945 6619.95,-6944.59 6616.87,-6944.15 6613.76,-6943.7\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"7468.34,-7016.8 7458.13,-7020.79 7463.35,-7016.55 7458.36,-7016.3 7458.36,-7016.3 7458.36,-7016.3 7463.35,-7016.55 7458.58,-7011.8 7468.34,-7016.8 7468.34,-7016.8\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape12 -->\\n<g id=\\\"edge18\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape12</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M7458.27,-7008.82C7311.75,-6995.2 7097.72,-6973.04 6912,-6945 6908.96,-6944.54 6905.88,-6944.06 6902.78,-6943.57\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"7468.26,-7009.75 7457.89,-7013.31 7463.28,-7009.29 7458.3,-7008.83 7458.3,-7008.83 7458.3,-7008.83 7463.28,-7009.29 7458.72,-7004.34 7468.26,-7009.75 7468.26,-7009.75\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape13 -->\\n<g id=\\\"edge19\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape13</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M7467.86,-6994.1C7387.26,-6979.49 7289.05,-6961.53 7201,-6945 7197.98,-6944.43 7194.92,-6943.86 7191.83,-6943.27\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"7478.11,-6995.96 7467.46,-6998.6 7473.19,-6995.07 7468.27,-6994.17 7468.27,-6994.17 7468.27,-6994.17 7473.19,-6995.07 7469.07,-6989.75 7478.11,-6995.96 7478.11,-6995.96\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape14 -->\\n<g id=\\\"edge20\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape14</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M7548.47,-6992.33C7507.92,-6977.32 7459.88,-6959.54 7420.67,-6945.02\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"7558.07,-6995.89 7547.13,-6996.63 7553.38,-6994.15 7548.69,-6992.41 7548.69,-6992.41 7548.69,-6992.41 7553.38,-6994.15 7550.25,-6988.19 7558.07,-6995.89 7558.07,-6995.89\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape15 -->\\n<g id=\\\"edge21\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm0_reshape15</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M7634,-6985.58C7634,-6972.28 7634,-6957.63 7634,-6945.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"7634,-6995.89 7629.5,-6985.89 7634,-6990.89 7634,-6985.89 7634,-6985.89 7634,-6985.89 7634,-6990.89 7638.5,-6985.89 7634,-6995.89 7634,-6995.89\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_dssmrecommendernetwork0_lstm0_h0_0 -->\\n<g id=\\\"node25\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_dssmrecommendernetwork0_lstm0_h0_0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"8260.5,-7054 7817.5,-7054 7817.5,-6996 8260.5,-6996 8260.5,-7054\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"8039\\\" y=\\\"-7021.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_dssmrecommendernetwork0_lstm0_h0_0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_dssmrecommendernetwork0_lstm0_h0_1 -->\\n<g id=\\\"node26\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_dssmrecommendernetwork0_lstm0_h0_1</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"6406.5,-7054 5963.5,-7054 5963.5,-6996 6406.5,-6996 6406.5,-7054\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6185\\\" y=\\\"-7021.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_dssmrecommendernetwork0_lstm0_h0_1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_rnn0 -->\\n<g id=\\\"node27\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm0_rnn0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"6663.5,-7163 6422.5,-7163 6422.5,-7105 6663.5,-7105 6663.5,-7163\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6543\\\" y=\\\"-7130.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm0_rnn0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_rnn0&#45;&gt;dssmrecommendernetwork0_transpose0 -->\\n<g id=\\\"edge22\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0_rnn0&#45;&gt;dssmrecommendernetwork0_transpose0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M6543,-7094.58C6543,-7081.28 6543,-7066.63 6543,-7054.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"6543,-7104.89 6538.5,-7094.89 6543,-7099.89 6543,-7094.89 6543,-7094.89 6543,-7094.89 6543,-7099.89 6547.5,-7094.89 6543,-7104.89 6543,-7104.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6560\\\" y=\\\"-7075.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1x128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_rnn0&#45;&gt;dssmrecommendernetwork0_lstm0__rnn_param_concat0 -->\\n<g id=\\\"edge23\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0_rnn0&#45;&gt;dssmrecommendernetwork0_lstm0__rnn_param_concat0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M6673.87,-7120.16C6872.73,-7100.66 7250.81,-7063.58 7468.48,-7042.23\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"6663.66,-7121.17 6673.18,-7115.71 6668.64,-7120.68 6673.61,-7120.19 6673.61,-7120.19 6673.61,-7120.19 6668.64,-7120.68 6674.05,-7124.67 6663.66,-7121.17 6663.66,-7121.17\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_rnn0&#45;&gt;dssmrecommendernetwork0_lstm0_dssmrecommendernetwork0_lstm0_h0_0 -->\\n<g id=\\\"edge24\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0_rnn0&#45;&gt;dssmrecommendernetwork0_lstm0_dssmrecommendernetwork0_lstm0_h0_0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M6673.7,-7127.38C6903.6,-7116.94 7395.2,-7092.06 7809,-7054 7811.8,-7053.74 7814.63,-7053.48 7817.46,-7053.21\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"6663.63,-7127.84 6673.41,-7122.89 6668.62,-7127.61 6673.62,-7127.39 6673.62,-7127.39 6673.62,-7127.39 6668.62,-7127.61 6673.82,-7131.88 6663.63,-7127.84 6663.63,-7127.84\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"7584\\\" y=\\\"-7075.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1x128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm0_rnn0&#45;&gt;dssmrecommendernetwork0_lstm0_dssmrecommendernetwork0_lstm0_h0_1 -->\\n<g id=\\\"edge25\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm0_rnn0&#45;&gt;dssmrecommendernetwork0_lstm0_dssmrecommendernetwork0_lstm0_h0_1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M6439.27,-7102C6388.57,-7086.84 6328.03,-7068.75 6278.74,-7054.02\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"6448.94,-7104.89 6438.07,-7106.33 6444.15,-7103.46 6439.36,-7102.02 6439.36,-7102.02 6439.36,-7102.02 6444.15,-7103.46 6440.65,-7097.71 6448.94,-7104.89 6448.94,-7104.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6403\\\" y=\\\"-7075.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1x128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_mean0 -->\\n<g id=\\\"node28\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_mean0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"6650.5,-7272 6435.5,-7272 6435.5,-7214 6650.5,-7214 6650.5,-7272\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6543\\\" y=\\\"-7239.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_mean0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_mean0&#45;&gt;dssmrecommendernetwork0_lstm0_rnn0 -->\\n<g id=\\\"edge26\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_mean0&#45;&gt;dssmrecommendernetwork0_lstm0_rnn0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M6543,-7203.58C6543,-7190.28 6543,-7175.63 6543,-7163.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"6543,-7213.89 6538.5,-7203.89 6543,-7208.89 6543,-7203.89 6543,-7203.89 6543,-7203.89 6543,-7208.89 6547.5,-7203.89 6543,-7213.89 6543,-7213.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6560\\\" y=\\\"-7184.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1x256</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense1_fwd -->\\n<g id=\\\"node29\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_dense1_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"6523.5,-7381 6420.5,-7381 6420.5,-7323 6523.5,-7323 6523.5,-7381\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6472\\\" y=\\\"-7355.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">FullyConnected</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"6472\\\" y=\\\"-7340.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense1_fwd&#45;&gt;dssmrecommendernetwork0_mean0 -->\\n<g id=\\\"edge27\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_dense1_fwd&#45;&gt;dssmrecommendernetwork0_mean0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M6496.19,-7314.55C6505.35,-7300.74 6515.63,-7285.24 6524.22,-7272.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"6490.65,-7322.89 6492.43,-7312.07 6493.42,-7318.72 6496.18,-7314.55 6496.18,-7314.55 6496.18,-7314.55 6493.42,-7318.72 6499.93,-7317.04 6490.65,-7322.89 6490.65,-7322.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6521.5\\\" y=\\\"-7293.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense1_relu_fwd -->\\n<g id=\\\"node30\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_dense1_relu_fwd</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"6443,-7490 6349,-7490 6349,-7432 6443,-7432 6443,-7490\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6396\\\" y=\\\"-7464.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"6396\\\" y=\\\"-7449.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense1_relu_fwd&#45;&gt;dssmrecommendernetwork0_dense1_fwd -->\\n<g id=\\\"edge28\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_dense1_relu_fwd&#45;&gt;dssmrecommendernetwork0_dense1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M6421.89,-7423.55C6431.7,-7409.74 6442.71,-7394.24 6451.9,-7381.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"6415.97,-7431.89 6418.09,-7421.13 6418.86,-7427.81 6421.76,-7423.73 6421.76,-7423.73 6421.76,-7423.73 6418.86,-7427.81 6425.43,-7426.34 6415.97,-7431.89 6415.97,-7431.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6449.5\\\" y=\\\"-7402.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_concat0 -->\\n<g id=\\\"node31\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_concat0</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"6390.5,-7599 6169.5,-7599 6169.5,-7541 6390.5,-7541 6390.5,-7599\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6280\\\" y=\\\"-7566.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_concat0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_concat0&#45;&gt;dssmrecommendernetwork0_dense0_relu_fwd -->\\n<g id=\\\"edge29\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_concat0&#45;&gt;dssmrecommendernetwork0_dense0_relu_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M6280,-7530.58C6280,-7517.28 6280,-7502.63 6280,-7490.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"6280,-7540.89 6275.5,-7530.89 6280,-7535.89 6280,-7530.89 6280,-7530.89 6280,-7530.89 6280,-7535.89 6284.5,-7530.89 6280,-7540.89 6280,-7540.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6290.5\\\" y=\\\"-7511.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_concat0&#45;&gt;dssmrecommendernetwork0_dense1_relu_fwd -->\\n<g id=\\\"edge30\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_concat0&#45;&gt;dssmrecommendernetwork0_dense1_relu_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M6317.83,-7534.1C6333.21,-7519.91 6350.76,-7503.73 6365.32,-7490.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"6310.48,-7540.89 6314.78,-7530.8 6314.15,-7537.5 6317.83,-7534.11 6317.83,-7534.11 6317.83,-7534.11 6314.15,-7537.5 6320.88,-7537.41 6310.48,-7540.89 6310.48,-7540.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6355.5\\\" y=\\\"-7511.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dropout0_fwd -->\\n<g id=\\\"node32\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_dropout0_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"6407.5,-7708 6152.5,-7708 6152.5,-7650 6407.5,-7650 6407.5,-7708\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6280\\\" y=\\\"-7675.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_dropout0_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dropout0_fwd&#45;&gt;dssmrecommendernetwork0_concat0 -->\\n<g id=\\\"edge31\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_dropout0_fwd&#45;&gt;dssmrecommendernetwork0_concat0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M6280,-7639.58C6280,-7626.28 6280,-7611.63 6280,-7599.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"6280,-7649.89 6275.5,-7639.89 6280,-7644.89 6280,-7639.89 6280,-7639.89 6280,-7639.89 6280,-7644.89 6284.5,-7639.89 6280,-7649.89 6280,-7649.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6290.5\\\" y=\\\"-7620.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense2_fwd -->\\n<g id=\\\"node33\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_dense2_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"6327.5,-7817 6224.5,-7817 6224.5,-7759 6327.5,-7759 6327.5,-7817\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6276\\\" y=\\\"-7791.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">FullyConnected</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"6276\\\" y=\\\"-7776.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense2_fwd&#45;&gt;dssmrecommendernetwork0_dropout0_fwd -->\\n<g id=\\\"edge32\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_dense2_fwd&#45;&gt;dssmrecommendernetwork0_dropout0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M6277.44,-7748.58C6277.93,-7735.28 6278.48,-7720.63 6278.94,-7708.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"6277.05,-7758.89 6272.93,-7748.73 6277.24,-7753.89 6277.42,-7748.89 6277.42,-7748.89 6277.42,-7748.89 6277.24,-7753.89 6281.92,-7749.06 6277.05,-7758.89 6277.05,-7758.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6289.5\\\" y=\\\"-7729.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense2_relu_fwd -->\\n<g id=\\\"node34\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_dense2_relu_fwd</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"6262,-7926 6168,-7926 6168,-7868 6262,-7868 6262,-7926\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6215\\\" y=\\\"-7900.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"6215\\\" y=\\\"-7885.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense2_relu_fwd&#45;&gt;dssmrecommendernetwork0_dense2_fwd -->\\n<g id=\\\"edge33\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_dense2_relu_fwd&#45;&gt;dssmrecommendernetwork0_dense2_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M6236,-7859.16C6243.82,-7845.45 6252.56,-7830.12 6259.87,-7817.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"6231.03,-7867.89 6232.07,-7856.97 6233.5,-7863.54 6235.98,-7859.2 6235.98,-7859.2 6235.98,-7859.2 6233.5,-7863.54 6239.89,-7861.43 6231.03,-7867.89 6231.03,-7867.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"6260.5\\\" y=\\\"-7838.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_expand_dims0 -->\\n<g id=\\\"node35\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_expand_dims0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5753,-8035 5495,-8035 5495,-7977 5753,-7977 5753,-8035\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5624\\\" y=\\\"-8002.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_expand_dims0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_expand_dims0&#45;&gt;dssmrecommendernetwork0_dense2_relu_fwd -->\\n<g id=\\\"edge34\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_expand_dims0&#45;&gt;dssmrecommendernetwork0_dense2_relu_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5763.4,-7979.76C5893.78,-7956.16 6081.01,-7922.26 6167.67,-7906.57\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5753.33,-7981.58 5762.37,-7975.37 5758.25,-7980.69 5763.17,-7979.8 5763.17,-7979.8 5763.17,-7979.8 5758.25,-7980.69 5763.97,-7984.23 5753.33,-7981.58 5753.33,-7981.58\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5965.5\\\" y=\\\"-7947.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- title -->\\n<g id=\\\"node36\\\" class=\\\"node\\\"><title>title</title>\\n<ellipse fill=\\\"#8dd3c7\\\" stroke=\\\"black\\\" cx=\\\"137\\\" cy=\\\"-6807\\\" rx=\\\"47\\\" ry=\\\"29\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"137\\\" y=\\\"-6803.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">title</text>\\n</g>\\n<!-- dssmrecommendernetwork0_embedding2_fwd -->\\n<g id=\\\"node37\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_embedding2_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"274,-6945 0,-6945 0,-6887 274,-6887 274,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"137\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_embedding2_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_embedding2_fwd&#45;&gt;title -->\\n<g id=\\\"edge35\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_embedding2_fwd&#45;&gt;title</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M137,-6876.58C137,-6863.28 137,-6848.63 137,-6836.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"137,-6886.89 132.5,-6876.89 137,-6881.89 137,-6876.89 137,-6876.89 137,-6876.89 137,-6881.89 141.5,-6876.89 137,-6886.89 137,-6886.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"144\\\" y=\\\"-6857.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">30</text>\\n</g>\\n<!-- dssmrecommendernetwork0_transpose1 -->\\n<g id=\\\"node38\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_transpose1</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"2391,-7054 2155,-7054 2155,-6996 2391,-6996 2391,-7054\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"2273\\\" y=\\\"-7021.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_transpose1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_transpose1&#45;&gt;dssmrecommendernetwork0_embedding2_fwd -->\\n<g id=\\\"edge36\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_transpose1&#45;&gt;dssmrecommendernetwork0_embedding2_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2145.31,-6996.03C2145.2,-6996.02 2145.1,-6996.01 2145,-6996 1948.91,-6973.8 564.584,-7004.29 369,-6978 319.882,-6971.4 266.298,-6957.75 223.013,-6945.04\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2154.94,-6997.25 2144.45,-7000.46 2149.97,-6996.62 2145.01,-6996 2145.01,-6996 2145.01,-6996 2149.97,-6996.62 2145.58,-6991.53 2154.94,-6997.25 2154.94,-6997.25\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"389.5\\\" y=\\\"-6966.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">30x128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_reshape0 -->\\n<g id=\\\"node39\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_reshape0</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"3137,-6945 2873,-6945 2873,-6887 3137,-6887 3137,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"3005\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_reshape0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_reshape1 -->\\n<g id=\\\"node40\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_reshape1</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"3419,-6945 3155,-6945 3155,-6887 3419,-6887 3419,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"3287\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_reshape1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_reshape2 -->\\n<g id=\\\"node41\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_reshape2</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"3701,-6945 3437,-6945 3437,-6887 3701,-6887 3701,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"3569\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_reshape2</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_reshape3 -->\\n<g id=\\\"node42\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_reshape3</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"3983,-6945 3719,-6945 3719,-6887 3983,-6887 3983,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"3851\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_reshape3</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_reshape4 -->\\n<g id=\\\"node43\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_reshape4</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"4265,-6945 4001,-6945 4001,-6887 4265,-6887 4265,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4133\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_reshape4</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_reshape5 -->\\n<g id=\\\"node44\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_reshape5</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"4547,-6945 4283,-6945 4283,-6887 4547,-6887 4547,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4415\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_reshape5</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_reshape6 -->\\n<g id=\\\"node45\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_reshape6</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"4829,-6945 4565,-6945 4565,-6887 4829,-6887 4829,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4697\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_reshape6</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_reshape7 -->\\n<g id=\\\"node46\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_reshape7</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"556,-6945 292,-6945 292,-6887 556,-6887 556,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"424\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_reshape7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_reshape8 -->\\n<g id=\\\"node47\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_reshape8</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"838,-6945 574,-6945 574,-6887 838,-6887 838,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"706\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_reshape8</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_reshape9 -->\\n<g id=\\\"node48\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_reshape9</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"1120,-6945 856,-6945 856,-6887 1120,-6887 1120,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"988\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_reshape9</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_reshape10 -->\\n<g id=\\\"node49\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_reshape10</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"1409.5,-6945 1138.5,-6945 1138.5,-6887 1409.5,-6887 1409.5,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"1274\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_reshape10</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_reshape11 -->\\n<g id=\\\"node50\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_reshape11</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"1698.5,-6945 1427.5,-6945 1427.5,-6887 1698.5,-6887 1698.5,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"1563\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_reshape11</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_reshape12 -->\\n<g id=\\\"node51\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_reshape12</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"1987.5,-6945 1716.5,-6945 1716.5,-6887 1987.5,-6887 1987.5,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"1852\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_reshape12</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_reshape13 -->\\n<g id=\\\"node52\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_reshape13</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"2276.5,-6945 2005.5,-6945 2005.5,-6887 2276.5,-6887 2276.5,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"2141\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_reshape13</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_reshape14 -->\\n<g id=\\\"node53\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_reshape14</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"2565.5,-6945 2294.5,-6945 2294.5,-6887 2565.5,-6887 2565.5,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"2430\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_reshape14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_reshape15 -->\\n<g id=\\\"node54\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_reshape15</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"2854.5,-6945 2583.5,-6945 2583.5,-6887 2854.5,-6887 2854.5,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"2719\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_reshape15</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0 -->\\n<g id=\\\"node55\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"2740.5,-7054 2409.5,-7054 2409.5,-6996 2740.5,-6996 2740.5,-7054\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"2575\\\" y=\\\"-7021.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1__rnn_param_concat0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape0 -->\\n<g id=\\\"edge37\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2697.42,-6993.54C2758.7,-6978.29 2832.38,-6959.95 2892.26,-6945.05\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2687.7,-6995.96 2696.31,-6989.18 2692.55,-6994.75 2697.4,-6993.54 2697.4,-6993.54 2697.4,-6993.54 2692.55,-6994.75 2698.49,-6997.91 2687.7,-6995.96 2687.7,-6995.96\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape1 -->\\n<g id=\\\"edge38\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2750.88,-6995.75C2926.79,-6970.64 2973.69,-6973.14 3146,-6945 3148.96,-6944.52 3151.94,-6944.02 3154.96,-6943.51\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2740.76,-6997.2 2750.02,-6991.33 2745.71,-6996.49 2750.66,-6995.78 2750.66,-6995.78 2750.66,-6995.78 2745.71,-6996.49 2751.3,-7000.23 2740.76,-6997.2 2740.76,-6997.2\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape2 -->\\n<g id=\\\"edge39\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape2</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2750.76,-6995.86C2914.72,-6977.77 3281.15,-6965.02 3428,-6945 3430.97,-6944.6 3433.97,-6944.17 3436.99,-6943.72\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2740.73,-6997 2750.16,-6991.4 2745.7,-6996.43 2750.67,-6995.87 2750.67,-6995.87 2750.67,-6995.87 2745.7,-6996.43 2751.18,-7000.34 2740.73,-6997 2740.73,-6997\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape3 -->\\n<g id=\\\"edge40\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape3</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2751.12,-6995.83C2914.07,-6979.24 2957.39,-6986.39 3118,-6978 3381.16,-6964.25 3448.56,-6978.08 3710,-6945 3712.97,-6944.62 3715.97,-6944.22 3719,-6943.8\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2740.72,-6996.9 2750.21,-6991.4 2745.69,-6996.39 2750.67,-6995.87 2750.67,-6995.87 2750.67,-6995.87 2745.69,-6996.39 2751.13,-7000.35 2740.72,-6996.9 2740.72,-6996.9\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape4 -->\\n<g id=\\\"edge41\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape4</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2750.85,-6995.88C3010.99,-6971.62 3079.07,-6988.44 3337,-6978 3628.24,-6966.21 3702.73,-6980.82 3992,-6945 3994.85,-6944.65 3997.73,-6944.27 4000.62,-6943.87\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2740.71,-6996.83 2750.25,-6991.41 2745.69,-6996.36 2750.67,-6995.89 2750.67,-6995.89 2750.67,-6995.89 2745.69,-6996.36 2751.09,-7000.37 2740.71,-6996.83 2740.71,-6996.83\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape5 -->\\n<g id=\\\"edge42\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape5</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2750.68,-6995.9C3123.03,-6962.72 3219.65,-6990.89 3590,-6978 3894.17,-6967.41 3971.91,-6982.08 4274,-6945 4276.85,-6944.65 4279.73,-6944.28 4282.62,-6943.88\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2740.71,-6996.8 2750.27,-6991.42 2745.69,-6996.35 2750.67,-6995.9 2750.67,-6995.9 2750.67,-6995.9 2745.69,-6996.35 2751.07,-7000.38 2740.71,-6996.8 2740.71,-6996.8\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape6 -->\\n<g id=\\\"edge43\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape6</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2750.9,-6995.91C3009.79,-6974.73 3611.77,-6985.66 3854,-6978 4166.19,-6968.12 4245.96,-6982.86 4556,-6945 4558.85,-6944.65 4561.73,-6944.28 4564.62,-6943.88\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2740.71,-6996.78 2750.29,-6991.45 2745.69,-6996.35 2750.67,-6995.93 2750.67,-6995.93 2750.67,-6995.93 2745.69,-6996.35 2751.05,-7000.41 2740.71,-6996.78 2740.71,-6996.78\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape7 -->\\n<g id=\\\"edge44\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape7</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2398.85,-6995.99C2074.02,-6969.88 1309.84,-6991.52 1003,-6978 807.971,-6969.4 758.426,-6971.39 565,-6945 562.033,-6944.6 559.034,-6944.17 556.013,-6943.72\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2409.27,-6996.85 2398.93,-7000.51 2404.29,-6996.44 2399.31,-6996.03 2399.31,-6996.03 2399.31,-6996.03 2404.29,-6996.44 2399.68,-6991.54 2409.27,-6996.85 2409.27,-6996.85\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape8 -->\\n<g id=\\\"edge45\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape8</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2398.98,-6996C2137.55,-6974.81 1531.63,-6989.08 1288,-6978 1091.65,-6969.07 1041.75,-6971.52 847,-6945 844.033,-6944.6 841.033,-6944.17 838.012,-6943.72\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2409.27,-6996.87 2398.93,-7000.51 2404.29,-6996.45 2399.31,-6996.03 2399.31,-6996.03 2399.31,-6996.03 2404.29,-6996.45 2399.68,-6991.54 2409.27,-6996.87 2409.27,-6996.87\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape9 -->\\n<g id=\\\"edge46\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape9</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2399.22,-6995.99C2034.36,-6963.64 1939.71,-6995.24 1578,-6978 1378.13,-6968.47 1327.28,-6971.87 1129,-6945 1126.03,-6944.6 1123.03,-6944.17 1120.01,-6943.73\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2409.27,-6996.89 2398.91,-7000.48 2404.29,-6996.45 2399.31,-6996 2399.31,-6996 2399.31,-6996 2404.29,-6996.45 2399.71,-6991.52 2409.27,-6996.89 2409.27,-6996.89\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape10 -->\\n<g id=\\\"edge47\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape10</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2399.01,-6995.97C2163.67,-6973.82 2102.02,-6990.41 1870,-6978 1668.86,-6967.25 1617.66,-6971.6 1418,-6945 1415.22,-6944.63 1412.42,-6944.24 1409.59,-6943.83\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2409.26,-6996.94 2398.88,-7000.47 2404.29,-6996.47 2399.31,-6995.99 2399.31,-6995.99 2399.31,-6995.99 2404.29,-6996.47 2399.74,-6991.51 2409.26,-6996.94 2409.26,-6996.94\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape11 -->\\n<g id=\\\"edge48\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape11</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2399.1,-6995.95C2276.58,-6982.54 2243.94,-6986.59 2125,-6978 1939.13,-6964.58 1891.65,-6970.16 1707,-6945 1704.22,-6944.62 1701.42,-6944.22 1698.6,-6943.81\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2409.25,-6997.08 2398.81,-7000.45 2404.28,-6996.53 2399.31,-6995.97 2399.31,-6995.97 2399.31,-6995.97 2404.28,-6996.53 2399.81,-6991.5 2409.25,-6997.08 2409.25,-6997.08\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape12 -->\\n<g id=\\\"edge49\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape12</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2399.15,-6995.9C2219.13,-6970.47 2171.47,-6973.1 1996,-6945 1993.23,-6944.56 1990.44,-6944.1 1987.62,-6943.63\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2409.22,-6997.33 2398.68,-7000.38 2404.27,-6996.63 2399.32,-6995.92 2399.32,-6995.92 2399.32,-6995.92 2404.27,-6996.63 2399.95,-6991.47 2409.22,-6997.33 2409.22,-6997.33\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape13 -->\\n<g id=\\\"edge50\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape13</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2451.44,-6993.54C2389.59,-6978.29 2315.23,-6959.95 2254.79,-6945.05\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2461.26,-6995.96 2450.47,-6997.93 2456.4,-6994.76 2451.55,-6993.56 2451.55,-6993.56 2451.55,-6993.56 2456.4,-6994.76 2452.62,-6989.19 2461.26,-6995.96 2461.26,-6995.96\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape14 -->\\n<g id=\\\"edge51\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape14</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2528.72,-6989.85C2509.17,-6975.42 2486.67,-6958.82 2468.09,-6945.11\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2536.9,-6995.89 2526.18,-6993.57 2532.88,-6992.92 2528.86,-6989.95 2528.86,-6989.95 2528.86,-6989.95 2532.88,-6992.92 2531.53,-6986.33 2536.9,-6995.89 2536.9,-6995.89\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape15 -->\\n<g id=\\\"edge52\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1__rnn_param_concat0&#45;&gt;dssmrecommendernetwork0_lstm1_reshape15</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2620.93,-6989.87C2640.27,-6975.5 2662.51,-6958.98 2680.92,-6945.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2612.83,-6995.89 2618.18,-6986.31 2616.85,-6992.9 2620.86,-6989.92 2620.86,-6989.92 2620.86,-6989.92 2616.85,-6992.9 2623.54,-6993.53 2612.83,-6995.89 2612.83,-6995.89\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_dssmrecommendernetwork0_lstm1_h0_0 -->\\n<g id=\\\"node56\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_dssmrecommendernetwork0_lstm1_h0_0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"3201.5,-7054 2758.5,-7054 2758.5,-6996 3201.5,-6996 3201.5,-7054\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"2980\\\" y=\\\"-7021.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_dssmrecommendernetwork0_lstm1_h0_0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_dssmrecommendernetwork0_lstm1_h0_1 -->\\n<g id=\\\"node57\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_dssmrecommendernetwork0_lstm1_h0_1</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"2136.5,-7054 1693.5,-7054 1693.5,-6996 2136.5,-6996 2136.5,-7054\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"1915\\\" y=\\\"-7021.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_dssmrecommendernetwork0_lstm1_h0_1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_rnn0 -->\\n<g id=\\\"node58\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_lstm1_rnn0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"2695.5,-7163 2454.5,-7163 2454.5,-7105 2695.5,-7105 2695.5,-7163\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"2575\\\" y=\\\"-7130.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_lstm1_rnn0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_rnn0&#45;&gt;dssmrecommendernetwork0_transpose1 -->\\n<g id=\\\"edge53\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1_rnn0&#45;&gt;dssmrecommendernetwork0_transpose1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2486.16,-7101.52C2443.67,-7086.47 2393.23,-7068.6 2352.08,-7054.02\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2495.65,-7104.89 2484.72,-7105.79 2490.94,-7103.22 2486.23,-7101.55 2486.23,-7101.55 2486.23,-7101.55 2490.94,-7103.22 2487.73,-7097.31 2495.65,-7104.89 2495.65,-7104.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"2460\\\" y=\\\"-7075.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1x128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_rnn0&#45;&gt;dssmrecommendernetwork0_lstm1__rnn_param_concat0 -->\\n<g id=\\\"edge54\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1_rnn0&#45;&gt;dssmrecommendernetwork0_lstm1__rnn_param_concat0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2575,-7094.58C2575,-7081.28 2575,-7066.63 2575,-7054.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2575,-7104.89 2570.5,-7094.89 2575,-7099.89 2575,-7094.89 2575,-7094.89 2575,-7094.89 2575,-7099.89 2579.5,-7094.89 2575,-7104.89 2575,-7104.89\\\"/>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_rnn0&#45;&gt;dssmrecommendernetwork0_lstm1_dssmrecommendernetwork0_lstm1_h0_0 -->\\n<g id=\\\"edge55\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1_rnn0&#45;&gt;dssmrecommendernetwork0_lstm1_dssmrecommendernetwork0_lstm1_h0_0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2691.02,-7102.35C2748.59,-7087.14 2817.65,-7068.89 2873.82,-7054.05\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2681.14,-7104.96 2689.66,-7098.05 2685.98,-7103.68 2690.81,-7102.4 2690.81,-7102.4 2690.81,-7102.4 2685.98,-7103.68 2691.96,-7106.75 2681.14,-7104.96 2681.14,-7104.96\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"2819\\\" y=\\\"-7075.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1x128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_lstm1_rnn0&#45;&gt;dssmrecommendernetwork0_lstm1_dssmrecommendernetwork0_lstm1_h0_1 -->\\n<g id=\\\"edge56\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_lstm1_rnn0&#45;&gt;dssmrecommendernetwork0_lstm1_dssmrecommendernetwork0_lstm1_h0_1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M2444.27,-7111.81C2342.22,-7095.26 2199.52,-7072.13 2088,-7054.05\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"2454.33,-7113.44 2443.74,-7116.28 2449.39,-7112.64 2444.46,-7111.84 2444.46,-7111.84 2444.46,-7111.84 2449.39,-7112.64 2445.18,-7107.39 2454.33,-7113.44 2454.33,-7113.44\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"2302\\\" y=\\\"-7075.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1x128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_mean1 -->\\n<g id=\\\"node59\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_mean1</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"4330.5,-7272 4115.5,-7272 4115.5,-7214 4330.5,-7214 4330.5,-7272\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4223\\\" y=\\\"-7239.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_mean1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_mean1&#45;&gt;dssmrecommendernetwork0_lstm1_rnn0 -->\\n<g id=\\\"edge57\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_mean1&#45;&gt;dssmrecommendernetwork0_lstm1_rnn0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4105.07,-7234.34C3802.54,-7214.7 3001.26,-7162.68 2695.7,-7142.84\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4115.19,-7235 4104.92,-7238.84 4110.2,-7234.68 4105.21,-7234.35 4105.21,-7234.35 4105.21,-7234.35 4110.2,-7234.68 4105.5,-7229.86 4115.19,-7235 4115.19,-7235\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"3515\\\" y=\\\"-7184.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1x256</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense3_fwd -->\\n<g id=\\\"node60\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_dense3_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"4859.5,-7381 4756.5,-7381 4756.5,-7323 4859.5,-7323 4859.5,-7381\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4808\\\" y=\\\"-7355.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">FullyConnected</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"4808\\\" y=\\\"-7340.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense3_fwd&#45;&gt;dssmrecommendernetwork0_mean1 -->\\n<g id=\\\"edge58\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_dense3_fwd&#45;&gt;dssmrecommendernetwork0_mean1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4746.22,-7339.7C4647.46,-7321.64 4452.75,-7286.02 4330.71,-7263.7\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4756.34,-7341.55 4745.7,-7344.18 4751.42,-7340.65 4746.5,-7339.75 4746.5,-7339.75 4746.5,-7339.75 4751.42,-7340.65 4747.31,-7335.33 4756.34,-7341.55 4756.34,-7341.55\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4561.5\\\" y=\\\"-7293.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense3_relu_fwd -->\\n<g id=\\\"node61\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_dense3_relu_fwd</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"4944,-7490 4850,-7490 4850,-7432 4944,-7432 4944,-7490\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4897\\\" y=\\\"-7464.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"4897\\\" y=\\\"-7449.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense3_relu_fwd&#45;&gt;dssmrecommendernetwork0_dense3_fwd -->\\n<g id=\\\"edge59\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_dense3_relu_fwd&#45;&gt;dssmrecommendernetwork0_dense3_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4867,-7423.94C4855.44,-7410.03 4842.41,-7394.36 4831.54,-7381.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4873.62,-7431.89 4863.76,-7427.08 4870.42,-7428.04 4867.22,-7424.2 4867.22,-7424.2 4867.22,-7424.2 4870.42,-7428.04 4870.68,-7421.32 4873.62,-7431.89 4873.62,-7431.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4868.5\\\" y=\\\"-7402.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- image -->\\n<g id=\\\"node62\\\" class=\\\"node\\\"><title>image</title>\\n<ellipse fill=\\\"#8dd3c7\\\" stroke=\\\"black\\\" cx=\\\"5166\\\" cy=\\\"-29\\\" rx=\\\"47\\\" ry=\\\"29\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5166\\\" y=\\\"-25.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">image</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_batchnorm0_fwd -->\\n<g id=\\\"node63\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_batchnorm0_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5331.5,-167 5000.5,-167 5000.5,-109 5331.5,-109 5331.5,-167\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5166\\\" y=\\\"-134.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_batchnorm0_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_batchnorm0_fwd&#45;&gt;image -->\\n<g id=\\\"edge60\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_batchnorm0_fwd&#45;&gt;image</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5166,-98.5824C5166,-85.2841 5166,-70.632 5166,-58.2967\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5166,-108.887 5161.5,-98.887 5166,-103.887 5166,-98.887 5166,-98.887 5166,-98.887 5166,-103.887 5170.5,-98.8871 5166,-108.887 5166,-108.887\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5196.5\\\" y=\\\"-79.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x224x224</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_conv0_fwd -->\\n<g id=\\\"node64\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_conv0_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5213,-276 5119,-276 5119,-218 5213,-218 5213,-276\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5166\\\" y=\\\"-250.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5166\\\" y=\\\"-235.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">7x7/2x2, 64</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_conv0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_batchnorm0_fwd -->\\n<g id=\\\"edge61\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_conv0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_batchnorm0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5166,-207.582C5166,-194.284 5166,-179.632 5166,-167.297\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5166,-217.887 5161.5,-207.887 5166,-212.887 5166,-207.887 5166,-207.887 5166,-207.887 5166,-212.887 5170.5,-207.887 5166,-217.887 5166,-217.887\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5196.5\\\" y=\\\"-188.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x224x224</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_batchnorm1_fwd -->\\n<g id=\\\"node65\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_batchnorm1_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5331.5,-385 5000.5,-385 5000.5,-327 5331.5,-327 5331.5,-385\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5166\\\" y=\\\"-352.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_batchnorm1_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_batchnorm1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_conv0_fwd -->\\n<g id=\\\"edge62\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_batchnorm1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_conv0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5166,-316.582C5166,-303.284 5166,-288.632 5166,-276.297\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5166,-326.887 5161.5,-316.887 5166,-321.887 5166,-316.887 5166,-316.887 5166,-316.887 5166,-321.887 5170.5,-316.887 5166,-326.887 5166,-326.887\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5200\\\" y=\\\"-297.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x112x112</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_relu0_fwd -->\\n<g id=\\\"node66\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_relu0_fwd</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"5213,-494 5119,-494 5119,-436 5213,-436 5213,-494\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5166\\\" y=\\\"-468.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5166\\\" y=\\\"-453.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_relu0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_batchnorm1_fwd -->\\n<g id=\\\"edge63\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_relu0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_batchnorm1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5166,-425.582C5166,-412.284 5166,-397.632 5166,-385.297\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5166,-435.887 5161.5,-425.887 5166,-430.887 5166,-425.887 5166,-425.887 5166,-425.887 5166,-430.887 5170.5,-425.887 5166,-435.887 5166,-435.887\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5200\\\" y=\\\"-406.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x112x112</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_pool0_fwd -->\\n<g id=\\\"node67\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_pool0_fwd</title>\\n<polygon fill=\\\"#80b1d3\\\" stroke=\\\"black\\\" points=\\\"5213,-603 5119,-603 5119,-545 5213,-545 5213,-603\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5166\\\" y=\\\"-577.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Pooling</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5166\\\" y=\\\"-562.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">max, 3x3/2x2</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_pool0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_relu0_fwd -->\\n<g id=\\\"edge64\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_pool0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_relu0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5166,-534.582C5166,-521.284 5166,-506.632 5166,-494.297\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5166,-544.887 5161.5,-534.887 5166,-539.887 5166,-534.887 5166,-534.887 5166,-534.887 5166,-539.887 5170.5,-534.887 5166,-544.887 5166,-544.887\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5200\\\" y=\\\"-515.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x112x112</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_batchnorm0_fwd -->\\n<g id=\\\"node68\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_batchnorm0_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5209,-712 4837,-712 4837,-654 5209,-654 5209,-712\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5023\\\" y=\\\"-679.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage1_batchnorm0_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_batchnorm0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_pool0_fwd -->\\n<g id=\\\"edge65\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_batchnorm0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_pool0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5068.61,-647.874C5087.81,-633.503 5109.9,-616.975 5128.18,-603.297\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5060.57,-653.887 5065.88,-644.293 5064.57,-650.891 5068.58,-647.896 5068.58,-647.896 5068.58,-647.896 5064.57,-650.891 5071.27,-651.499 5060.57,-653.887 5060.57,-653.887\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5131.5\\\" y=\\\"-624.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_activation0 -->\\n<g id=\\\"node69\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_activation0</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"5070,-821 4976,-821 4976,-763 5070,-763 5070,-821\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5023\\\" y=\\\"-795.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5023\\\" y=\\\"-780.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_activation0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_batchnorm0_fwd -->\\n<g id=\\\"edge66\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_activation0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_batchnorm0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5023,-752.582C5023,-739.284 5023,-724.632 5023,-712.297\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5023,-762.887 5018.5,-752.887 5023,-757.887 5023,-752.887 5023,-752.887 5023,-752.887 5023,-757.887 5027.5,-752.887 5023,-762.887 5023,-762.887\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5050.5\\\" y=\\\"-733.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_conv0_fwd -->\\n<g id=\\\"node70\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_conv0_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5070,-930 4976,-930 4976,-872 5070,-872 5070,-930\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5023\\\" y=\\\"-904.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5023\\\" y=\\\"-889.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x3/1x1, 64</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_conv0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_activation0 -->\\n<g id=\\\"edge67\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_conv0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_activation0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5023,-861.582C5023,-848.284 5023,-833.632 5023,-821.297\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5023,-871.887 5018.5,-861.887 5023,-866.887 5023,-861.887 5023,-861.887 5023,-861.887 5023,-866.887 5027.5,-861.887 5023,-871.887 5023,-871.887\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5050.5\\\" y=\\\"-842.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_batchnorm1_fwd -->\\n<g id=\\\"node71\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_batchnorm1_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5209,-1039 4837,-1039 4837,-981 5209,-981 5209,-1039\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5023\\\" y=\\\"-1006.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage1_batchnorm1_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_batchnorm1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_conv0_fwd -->\\n<g id=\\\"edge68\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_batchnorm1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_conv0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5023,-970.582C5023,-957.284 5023,-942.632 5023,-930.297\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5023,-980.887 5018.5,-970.887 5023,-975.887 5023,-970.887 5023,-970.887 5023,-970.887 5023,-975.887 5027.5,-970.887 5023,-980.887 5023,-980.887\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5050.5\\\" y=\\\"-951.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_activation1 -->\\n<g id=\\\"node72\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_activation1</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"5140,-1148 5046,-1148 5046,-1090 5140,-1090 5140,-1148\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5093\\\" y=\\\"-1122.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5093\\\" y=\\\"-1107.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_activation1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_batchnorm1_fwd -->\\n<g id=\\\"edge69\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_activation1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_batchnorm1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5068.9,-1081.16C5059.93,-1067.45 5049.9,-1052.12 5041.51,-1039.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5074.61,-1089.89 5065.37,-1083.98 5071.87,-1085.7 5069.13,-1081.52 5069.13,-1081.52 5069.13,-1081.52 5071.87,-1085.7 5072.9,-1079.06 5074.61,-1089.89 5074.61,-1089.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5090.5\\\" y=\\\"-1060.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_conv1_fwd -->\\n<g id=\\\"node73\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_conv1_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5209,-1257 5115,-1257 5115,-1199 5209,-1199 5209,-1257\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5162\\\" y=\\\"-1231.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5162\\\" y=\\\"-1216.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x3/1x1, 64</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_conv1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_activation1 -->\\n<g id=\\\"edge70\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_conv1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_activation1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5138.24,-1190.16C5129.4,-1176.45 5119.52,-1161.12 5111.25,-1148.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5143.87,-1198.89 5134.67,-1192.92 5141.16,-1194.68 5138.45,-1190.48 5138.45,-1190.48 5138.45,-1190.48 5141.16,-1194.68 5142.23,-1188.04 5143.87,-1198.89 5143.87,-1198.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5159.5\\\" y=\\\"-1169.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1__plus0 -->\\n<g id=\\\"node74\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage1__plus0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5357.5,-1366 5042.5,-1366 5042.5,-1308 5357.5,-1308 5357.5,-1366\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5200\\\" y=\\\"-1333.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage1__plus0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1__plus0&#45;&gt;dssmrecommendernetwork0_resnetv21_pool0_fwd -->\\n<g id=\\\"edge72\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage1__plus0&#45;&gt;dssmrecommendernetwork0_resnetv21_pool0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5219.67,-1298.74C5228.52,-1278.65 5237,-1253.1 5237,-1229 5237,-1229 5237,-1229 5237,-682 5237,-651.457 5216.67,-622.961 5197.76,-603.068\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5215.48,-1307.85 5215.57,-1296.88 5217.57,-1303.31 5219.66,-1298.76 5219.66,-1298.76 5219.66,-1298.76 5217.57,-1303.31 5223.75,-1300.65 5215.48,-1307.85 5215.48,-1307.85\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5264.5\\\" y=\\\"-951.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1__plus0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_conv1_fwd -->\\n<g id=\\\"edge71\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage1__plus0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_conv1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5172.05,-1299.09C5170.51,-1296.1 5169.13,-1293.06 5168,-1290 5164.21,-1279.71 5162.5,-1267.83 5161.81,-1257.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5177.11,-1307.8 5168.2,-1301.41 5174.6,-1303.48 5172.09,-1299.16 5172.09,-1299.16 5172.09,-1299.16 5174.6,-1303.48 5175.98,-1296.9 5177.11,-1307.8 5177.11,-1307.8\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5195.5\\\" y=\\\"-1278.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_batchnorm2_fwd -->\\n<g id=\\\"node75\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_batchnorm2_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5243,-1475 4871,-1475 4871,-1417 5243,-1417 5243,-1475\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5057\\\" y=\\\"-1442.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage1_batchnorm2_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_batchnorm2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1__plus0 -->\\n<g id=\\\"edge73\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_batchnorm2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1__plus0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5102.61,-1410.87C5121.81,-1396.5 5143.9,-1379.98 5162.18,-1366.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5094.57,-1416.89 5099.88,-1407.29 5098.57,-1413.89 5102.58,-1410.9 5102.58,-1410.9 5102.58,-1410.9 5098.57,-1413.89 5105.27,-1414.5 5094.57,-1416.89 5094.57,-1416.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5165.5\\\" y=\\\"-1387.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_activation2 -->\\n<g id=\\\"node76\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_activation2</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"5104,-1584 5010,-1584 5010,-1526 5104,-1526 5104,-1584\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5057\\\" y=\\\"-1558.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5057\\\" y=\\\"-1543.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_activation2&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_batchnorm2_fwd -->\\n<g id=\\\"edge74\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_activation2&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_batchnorm2_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5057,-1515.58C5057,-1502.28 5057,-1487.63 5057,-1475.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5057,-1525.89 5052.5,-1515.89 5057,-1520.89 5057,-1515.89 5057,-1515.89 5057,-1515.89 5057,-1520.89 5061.5,-1515.89 5057,-1525.89 5057,-1525.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5084.5\\\" y=\\\"-1496.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_conv2_fwd -->\\n<g id=\\\"node77\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_conv2_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5104,-1693 5010,-1693 5010,-1635 5104,-1635 5104,-1693\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5057\\\" y=\\\"-1667.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5057\\\" y=\\\"-1652.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x3/1x1, 64</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_conv2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_activation2 -->\\n<g id=\\\"edge75\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_conv2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_activation2</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5057,-1624.58C5057,-1611.28 5057,-1596.63 5057,-1584.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5057,-1634.89 5052.5,-1624.89 5057,-1629.89 5057,-1624.89 5057,-1624.89 5057,-1624.89 5057,-1629.89 5061.5,-1624.89 5057,-1634.89 5057,-1634.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5084.5\\\" y=\\\"-1605.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_batchnorm3_fwd -->\\n<g id=\\\"node78\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_batchnorm3_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5243,-1802 4871,-1802 4871,-1744 5243,-1744 5243,-1802\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5057\\\" y=\\\"-1769.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage1_batchnorm3_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_batchnorm3_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_conv2_fwd -->\\n<g id=\\\"edge76\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_batchnorm3_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_conv2_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5057,-1733.58C5057,-1720.28 5057,-1705.63 5057,-1693.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5057,-1743.89 5052.5,-1733.89 5057,-1738.89 5057,-1733.89 5057,-1733.89 5057,-1733.89 5057,-1738.89 5061.5,-1733.89 5057,-1743.89 5057,-1743.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5084.5\\\" y=\\\"-1714.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_activation3 -->\\n<g id=\\\"node79\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_activation3</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"5174,-1911 5080,-1911 5080,-1853 5174,-1853 5174,-1911\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5127\\\" y=\\\"-1885.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5127\\\" y=\\\"-1870.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_activation3&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_batchnorm3_fwd -->\\n<g id=\\\"edge77\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_activation3&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_batchnorm3_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5102.9,-1844.16C5093.93,-1830.45 5083.9,-1815.12 5075.51,-1802.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5108.61,-1852.89 5099.37,-1846.98 5105.87,-1848.7 5103.13,-1844.52 5103.13,-1844.52 5103.13,-1844.52 5105.87,-1848.7 5106.9,-1842.06 5108.61,-1852.89 5108.61,-1852.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5124.5\\\" y=\\\"-1823.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_conv3_fwd -->\\n<g id=\\\"node80\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_conv3_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5243,-2020 5149,-2020 5149,-1962 5243,-1962 5243,-2020\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5196\\\" y=\\\"-1994.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5196\\\" y=\\\"-1979.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x3/1x1, 64</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1_conv3_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_activation3 -->\\n<g id=\\\"edge78\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage1_conv3_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_activation3</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5172.24,-1953.16C5163.4,-1939.45 5153.52,-1924.12 5145.25,-1911.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5177.87,-1961.89 5168.67,-1955.92 5175.16,-1957.68 5172.45,-1953.48 5172.45,-1953.48 5172.45,-1953.48 5175.16,-1957.68 5176.23,-1951.04 5177.87,-1961.89 5177.87,-1961.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5193.5\\\" y=\\\"-1932.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1__plus1 -->\\n<g id=\\\"node81\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage1__plus1</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5392.5,-2129 5077.5,-2129 5077.5,-2071 5392.5,-2071 5392.5,-2129\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5235\\\" y=\\\"-2096.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage1__plus1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1__plus1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1__plus0 -->\\n<g id=\\\"edge80\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage1__plus1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1__plus0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5254.26,-2061.41C5262.82,-2041.35 5271,-2015.92 5271,-1992 5271,-1992 5271,-1992 5271,-1445 5271,-1414.46 5250.67,-1385.96 5231.76,-1366.07\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5250.06,-2070.8 5250.03,-2059.83 5252.1,-2066.24 5254.14,-2061.67 5254.14,-2061.67 5254.14,-2061.67 5252.1,-2066.24 5258.25,-2063.51 5250.06,-2070.8 5250.06,-2070.8\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5298.5\\\" y=\\\"-1714.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage1__plus1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_conv3_fwd -->\\n<g id=\\\"edge79\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage1__plus1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1_conv3_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5206.18,-2062.14C5204.58,-2059.14 5203.16,-2056.08 5202,-2053 5198.1,-2042.64 5196.38,-2030.63 5195.71,-2020.02\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5211.39,-2070.88 5202.4,-2064.6 5208.83,-2066.59 5206.27,-2062.29 5206.27,-2062.29 5206.27,-2062.29 5208.83,-2066.59 5210.13,-2059.99 5211.39,-2070.88 5211.39,-2070.88\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5229.5\\\" y=\\\"-2041.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_batchnorm0_fwd -->\\n<g id=\\\"node82\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_batchnorm0_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5421,-2238 5049,-2238 5049,-2180 5421,-2180 5421,-2238\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5235\\\" y=\\\"-2205.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage2_batchnorm0_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_batchnorm0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1__plus1 -->\\n<g id=\\\"edge81\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_batchnorm0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage1__plus1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5235,-2169.58C5235,-2156.28 5235,-2141.63 5235,-2129.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5235,-2179.89 5230.5,-2169.89 5235,-2174.89 5235,-2169.89 5235,-2169.89 5235,-2169.89 5235,-2174.89 5239.5,-2169.89 5235,-2179.89 5235,-2179.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5262.5\\\" y=\\\"-2150.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_activation0 -->\\n<g id=\\\"node83\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_activation0</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"5282,-2347 5188,-2347 5188,-2289 5282,-2289 5282,-2347\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5235\\\" y=\\\"-2321.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5235\\\" y=\\\"-2306.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_activation0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_batchnorm0_fwd -->\\n<g id=\\\"edge82\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_activation0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_batchnorm0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5235,-2278.58C5235,-2265.28 5235,-2250.63 5235,-2238.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5235,-2288.89 5230.5,-2278.89 5235,-2283.89 5235,-2278.89 5235,-2278.89 5235,-2278.89 5235,-2283.89 5239.5,-2278.89 5235,-2288.89 5235,-2288.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5262.5\\\" y=\\\"-2259.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_conv0_fwd -->\\n<g id=\\\"node84\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_conv0_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5216,-2456 5122,-2456 5122,-2398 5216,-2398 5216,-2456\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5169\\\" y=\\\"-2430.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5169\\\" y=\\\"-2415.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x3/2x2, 128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_conv0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_activation0 -->\\n<g id=\\\"edge83\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_conv0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_activation0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5191.73,-2389.16C5200.18,-2375.45 5209.64,-2360.12 5217.55,-2347.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5186.34,-2397.89 5187.76,-2387.01 5188.97,-2393.63 5191.59,-2389.38 5191.59,-2389.38 5191.59,-2389.38 5188.97,-2393.63 5195.42,-2391.74 5186.34,-2397.89 5186.34,-2397.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5233.5\\\" y=\\\"-2368.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_batchnorm1_fwd -->\\n<g id=\\\"node85\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_batchnorm1_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5245,-2565 4873,-2565 4873,-2507 5245,-2507 5245,-2565\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5059\\\" y=\\\"-2532.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage2_batchnorm1_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_batchnorm1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_conv0_fwd -->\\n<g id=\\\"edge84\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_batchnorm1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_conv0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5095.27,-2499.72C5109.76,-2485.62 5126.22,-2469.61 5139.91,-2456.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5087.9,-2506.89 5091.93,-2496.69 5091.49,-2503.4 5095.07,-2499.91 5095.07,-2499.91 5095.07,-2499.91 5091.49,-2503.4 5098.21,-2503.14 5087.9,-2506.89 5087.9,-2506.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5151.5\\\" y=\\\"-2477.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_activation1 -->\\n<g id=\\\"node86\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_activation1</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"5120,-2674 5026,-2674 5026,-2616 5120,-2616 5120,-2674\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5073\\\" y=\\\"-2648.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5073\\\" y=\\\"-2633.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_activation1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_batchnorm1_fwd -->\\n<g id=\\\"edge85\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_activation1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_batchnorm1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5067.97,-2605.58C5066.23,-2592.28 5064.32,-2577.63 5062.7,-2565.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5069.32,-2615.89 5063.56,-2606.56 5068.67,-2610.93 5068.02,-2605.97 5068.02,-2605.97 5068.02,-2605.97 5068.67,-2610.93 5072.49,-2605.39 5069.32,-2615.89 5069.32,-2615.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5097.5\\\" y=\\\"-2586.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_conv1_fwd -->\\n<g id=\\\"node87\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_conv1_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5151,-2826 5057,-2826 5057,-2768 5151,-2768 5151,-2826\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5104\\\" y=\\\"-2800.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5104\\\" y=\\\"-2785.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x3/1x1, 128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_conv1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_activation1 -->\\n<g id=\\\"edge86\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_conv1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_activation1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5096.12,-2757.85C5090.72,-2731.75 5083.69,-2697.73 5078.8,-2674.09\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5098.17,-2767.79 5091.74,-2758.9 5097.16,-2762.89 5096.15,-2757.99 5096.15,-2757.99 5096.15,-2757.99 5097.16,-2762.89 5100.55,-2757.08 5098.17,-2767.79 5098.17,-2767.79\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5124.5\\\" y=\\\"-2717.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_conv2_fwd -->\\n<g id=\\\"node88\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_conv2_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5314,-2750 5220,-2750 5220,-2692 5314,-2692 5314,-2750\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5267\\\" y=\\\"-2724.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5267\\\" y=\\\"-2709.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1x1/2x2, 128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_conv2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_activation0 -->\\n<g id=\\\"edge87\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_conv2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_activation0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5273.36,-2681.77C5277.06,-2656.27 5281,-2621.97 5281,-2591.5 5281,-2591.5 5281,-2591.5 5281,-2426 5281,-2397.97 5276.36,-2390.62 5265,-2365 5262.37,-2359.07 5259.02,-2353.05 5255.52,-2347.4\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5271.86,-2691.77 5268.89,-2681.21 5272.6,-2686.82 5273.34,-2681.88 5273.34,-2681.88 5273.34,-2681.88 5272.6,-2686.82 5277.79,-2682.55 5271.86,-2691.77 5271.86,-2691.77\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5308.5\\\" y=\\\"-2532.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">64x56x56</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2__plus0 -->\\n<g id=\\\"node89\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage2__plus0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5310.5,-2935 4995.5,-2935 4995.5,-2877 5310.5,-2877 5310.5,-2935\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5153\\\" y=\\\"-2902.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage2__plus0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2__plus0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_conv1_fwd -->\\n<g id=\\\"edge88\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2__plus0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_conv1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5130.45,-2867.91C5128.88,-2864.93 5127.38,-2861.93 5126,-2859 5121.08,-2848.5 5116.55,-2836.58 5112.91,-2826.07\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5135.31,-2876.73 5126.55,-2870.14 5132.9,-2872.35 5130.49,-2867.97 5130.49,-2867.97 5130.49,-2867.97 5132.9,-2872.35 5134.43,-2865.8 5135.31,-2876.73 5135.31,-2876.73\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5156.5\\\" y=\\\"-2847.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2__plus0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_conv2_fwd -->\\n<g id=\\\"edge89\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2__plus0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_conv2_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5180.6,-2868.39C5182.81,-2865.24 5184.97,-2862.08 5187,-2859 5211.17,-2822.31 5236.28,-2778.16 5251.77,-2750.09\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5174.71,-2876.63 5176.87,-2865.88 5177.62,-2872.56 5180.53,-2868.5 5180.53,-2868.5 5180.53,-2868.5 5177.62,-2872.56 5184.19,-2871.12 5174.71,-2876.63 5174.71,-2876.63\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5226.5\\\" y=\\\"-2847.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_batchnorm2_fwd -->\\n<g id=\\\"node90\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_batchnorm2_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5198,-3044 4826,-3044 4826,-2986 5198,-2986 5198,-3044\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5012\\\" y=\\\"-3011.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage2_batchnorm2_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_batchnorm2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2__plus0 -->\\n<g id=\\\"edge90\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_batchnorm2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2__plus0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5057.22,-2979.68C5076.1,-2965.36 5097.77,-2948.91 5115.71,-2935.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5049.05,-2985.89 5054.29,-2976.26 5053.03,-2982.86 5057.01,-2979.84 5057.01,-2979.84 5057.01,-2979.84 5053.03,-2982.86 5059.73,-2983.43 5049.05,-2985.89 5049.05,-2985.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5121.5\\\" y=\\\"-2956.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_activation2 -->\\n<g id=\\\"node91\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_activation2</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"5059,-3153 4965,-3153 4965,-3095 5059,-3095 5059,-3153\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5012\\\" y=\\\"-3127.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5012\\\" y=\\\"-3112.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_activation2&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_batchnorm2_fwd -->\\n<g id=\\\"edge91\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_activation2&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_batchnorm2_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5012,-3084.58C5012,-3071.28 5012,-3056.63 5012,-3044.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5012,-3094.89 5007.5,-3084.89 5012,-3089.89 5012,-3084.89 5012,-3084.89 5012,-3084.89 5012,-3089.89 5016.5,-3084.89 5012,-3094.89 5012,-3094.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5042.5\\\" y=\\\"-3065.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_conv3_fwd -->\\n<g id=\\\"node92\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_conv3_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5059,-3262 4965,-3262 4965,-3204 5059,-3204 5059,-3262\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5012\\\" y=\\\"-3236.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5012\\\" y=\\\"-3221.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x3/1x1, 128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_conv3_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_activation2 -->\\n<g id=\\\"edge92\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_conv3_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_activation2</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5012,-3193.58C5012,-3180.28 5012,-3165.63 5012,-3153.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5012,-3203.89 5007.5,-3193.89 5012,-3198.89 5012,-3193.89 5012,-3193.89 5012,-3193.89 5012,-3198.89 5016.5,-3193.89 5012,-3203.89 5012,-3203.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5042.5\\\" y=\\\"-3174.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_batchnorm3_fwd -->\\n<g id=\\\"node93\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_batchnorm3_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5198,-3371 4826,-3371 4826,-3313 5198,-3313 5198,-3371\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5012\\\" y=\\\"-3338.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage2_batchnorm3_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_batchnorm3_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_conv3_fwd -->\\n<g id=\\\"edge93\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_batchnorm3_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_conv3_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5012,-3302.58C5012,-3289.28 5012,-3274.63 5012,-3262.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5012,-3312.89 5007.5,-3302.89 5012,-3307.89 5012,-3302.89 5012,-3302.89 5012,-3302.89 5012,-3307.89 5016.5,-3302.89 5012,-3312.89 5012,-3312.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5042.5\\\" y=\\\"-3283.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_activation3 -->\\n<g id=\\\"node94\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_activation3</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"5072,-3480 4978,-3480 4978,-3422 5072,-3422 5072,-3480\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5025\\\" y=\\\"-3454.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5025\\\" y=\\\"-3439.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_activation3&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_batchnorm3_fwd -->\\n<g id=\\\"edge94\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_activation3&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_batchnorm3_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5020.33,-3411.58C5018.72,-3398.28 5016.94,-3383.63 5015.44,-3371.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5021.58,-3421.89 5015.91,-3412.5 5020.98,-3416.92 5020.38,-3411.96 5020.38,-3411.96 5020.38,-3411.96 5020.98,-3416.92 5024.85,-3411.42 5021.58,-3421.89 5021.58,-3421.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5050.5\\\" y=\\\"-3392.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_conv4_fwd -->\\n<g id=\\\"node95\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_conv4_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5094,-3589 5000,-3589 5000,-3531 5094,-3531 5094,-3589\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5047\\\" y=\\\"-3563.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5047\\\" y=\\\"-3548.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x3/1x1, 128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2_conv4_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_activation3 -->\\n<g id=\\\"edge95\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2_conv4_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_activation3</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5039.18,-3520.98C5036.43,-3507.57 5033.38,-3492.75 5030.82,-3480.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5041.22,-3530.89 5034.8,-3522 5040.21,-3525.99 5039.21,-3521.09 5039.21,-3521.09 5039.21,-3521.09 5040.21,-3525.99 5043.61,-3520.19 5041.22,-3530.89 5041.22,-3530.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5068.5\\\" y=\\\"-3501.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2__plus1 -->\\n<g id=\\\"node96\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage2__plus1</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5276.5,-3698 4961.5,-3698 4961.5,-3640 5276.5,-3640 5276.5,-3698\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5119\\\" y=\\\"-3665.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage2__plus1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2__plus1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2__plus0 -->\\n<g id=\\\"edge97\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2__plus1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2__plus0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5180.34,-3634.08C5204.24,-3616.06 5226,-3591.33 5226,-3561 5226,-3561 5226,-3561 5226,-3014 5226,-2983.42 5205.4,-2955.13 5186.08,-2935.33\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5172.19,-3639.93 5177.69,-3630.45 5176.25,-3637.02 5180.31,-3634.11 5180.31,-3634.11 5180.31,-3634.11 5176.25,-3637.02 5182.94,-3637.76 5172.19,-3639.93 5172.19,-3639.93\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5256.5\\\" y=\\\"-3283.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage2__plus1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_conv4_fwd -->\\n<g id=\\\"edge96\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage2__plus1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2_conv4_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5094.47,-3631.55C5085.18,-3617.74 5074.75,-3602.24 5066.04,-3589.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5100.08,-3639.89 5090.77,-3634.1 5097.29,-3635.74 5094.5,-3631.59 5094.5,-3631.59 5094.5,-3631.59 5097.29,-3635.74 5098.23,-3629.08 5100.08,-3639.89 5100.08,-3639.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5118.5\\\" y=\\\"-3610.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_batchnorm0_fwd -->\\n<g id=\\\"node97\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_batchnorm0_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5305,-3807 4933,-3807 4933,-3749 5305,-3749 5305,-3807\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5119\\\" y=\\\"-3774.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage3_batchnorm0_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_batchnorm0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2__plus1 -->\\n<g id=\\\"edge98\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_batchnorm0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage2__plus1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5119,-3738.58C5119,-3725.28 5119,-3710.63 5119,-3698.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5119,-3748.89 5114.5,-3738.89 5119,-3743.89 5119,-3738.89 5119,-3738.89 5119,-3738.89 5119,-3743.89 5123.5,-3738.89 5119,-3748.89 5119,-3748.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5149.5\\\" y=\\\"-3719.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_activation0 -->\\n<g id=\\\"node98\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_activation0</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"5166,-3916 5072,-3916 5072,-3858 5166,-3858 5166,-3916\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5119\\\" y=\\\"-3890.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5119\\\" y=\\\"-3875.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_activation0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_batchnorm0_fwd -->\\n<g id=\\\"edge99\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_activation0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_batchnorm0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5119,-3847.58C5119,-3834.28 5119,-3819.63 5119,-3807.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5119,-3857.89 5114.5,-3847.89 5119,-3852.89 5119,-3847.89 5119,-3847.89 5119,-3847.89 5119,-3852.89 5123.5,-3847.89 5119,-3857.89 5119,-3857.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5149.5\\\" y=\\\"-3828.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_conv0_fwd -->\\n<g id=\\\"node99\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_conv0_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5100,-4025 5006,-4025 5006,-3967 5100,-3967 5100,-4025\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5053\\\" y=\\\"-3999.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5053\\\" y=\\\"-3984.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x3/2x2, 256</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_conv0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_activation0 -->\\n<g id=\\\"edge100\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_conv0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_activation0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5075.73,-3958.16C5084.18,-3944.45 5093.64,-3929.12 5101.55,-3916.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5070.34,-3966.89 5071.76,-3956.01 5072.97,-3962.63 5075.59,-3958.38 5075.59,-3958.38 5075.59,-3958.38 5072.97,-3962.63 5079.42,-3960.74 5070.34,-3966.89 5070.34,-3966.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5120.5\\\" y=\\\"-3937.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_batchnorm1_fwd -->\\n<g id=\\\"node100\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_batchnorm1_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5130,-4134 4758,-4134 4758,-4076 5130,-4076 5130,-4134\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4944\\\" y=\\\"-4101.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage3_batchnorm1_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_batchnorm1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_conv0_fwd -->\\n<g id=\\\"edge101\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_batchnorm1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_conv0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4979.94,-4068.72C4994.3,-4054.62 5010.61,-4038.61 5024.17,-4025.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4972.64,-4075.89 4976.62,-4065.67 4976.21,-4072.38 4979.77,-4068.88 4979.77,-4068.88 4979.77,-4068.88 4976.21,-4072.38 4982.93,-4072.09 4972.64,-4075.89 4972.64,-4075.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5036.5\\\" y=\\\"-4046.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_activation1 -->\\n<g id=\\\"node101\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_activation1</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"5012,-4243 4918,-4243 4918,-4185 5012,-4185 5012,-4243\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4965\\\" y=\\\"-4217.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"4965\\\" y=\\\"-4202.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_activation1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_batchnorm1_fwd -->\\n<g id=\\\"edge102\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_activation1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_batchnorm1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4957.54,-4174.98C4954.91,-4161.57 4952,-4146.75 4949.55,-4134.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4959.48,-4184.89 4953.14,-4175.94 4958.52,-4179.98 4957.56,-4175.07 4957.56,-4175.07 4957.56,-4175.07 4958.52,-4179.98 4961.97,-4174.21 4959.48,-4184.89 4959.48,-4184.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4986.5\\\" y=\\\"-4155.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_conv1_fwd -->\\n<g id=\\\"node102\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_conv1_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5023,-4395 4929,-4395 4929,-4337 5023,-4337 5023,-4395\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4976\\\" y=\\\"-4369.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"4976\\\" y=\\\"-4354.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x3/1x1, 256</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_conv1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_activation1 -->\\n<g id=\\\"edge103\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_conv1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_activation1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4973.18,-4326.51C4971.27,-4300.45 4968.79,-4266.62 4967.06,-4243.09\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4973.93,-4336.79 4968.71,-4327.14 4973.57,-4331.8 4973.2,-4326.81 4973.2,-4326.81 4973.2,-4326.81 4973.57,-4331.8 4977.69,-4326.48 4973.93,-4336.79 4973.93,-4336.79\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5003.5\\\" y=\\\"-4286.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_conv2_fwd -->\\n<g id=\\\"node103\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_conv2_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5186,-4319 5092,-4319 5092,-4261 5186,-4261 5186,-4319\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5139\\\" y=\\\"-4293.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5139\\\" y=\\\"-4278.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1x1/2x2, 256</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_conv2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_activation0 -->\\n<g id=\\\"edge104\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_conv2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_activation0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5151.24,-4251.18C5158.39,-4225.81 5166,-4191.49 5166,-4160.5 5166,-4160.5 5166,-4160.5 5166,-3995 5166,-3967.18 5163.44,-3959.35 5152,-3934 5149.25,-3927.91 5145.63,-3921.81 5141.8,-3916.13\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5148.44,-4260.78 5146.92,-4249.92 5149.84,-4255.98 5151.24,-4251.18 5151.24,-4251.18 5151.24,-4251.18 5149.84,-4255.98 5155.56,-4252.44 5148.44,-4260.78 5148.44,-4260.78\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5196.5\\\" y=\\\"-4101.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x28x28</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3__plus0 -->\\n<g id=\\\"node104\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage3__plus0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5182.5,-4504 4867.5,-4504 4867.5,-4446 5182.5,-4446 5182.5,-4504\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5025\\\" y=\\\"-4471.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage3__plus0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3__plus0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_conv1_fwd -->\\n<g id=\\\"edge105\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3__plus0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_conv1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5002.45,-4436.91C5000.88,-4433.93 4999.38,-4430.93 4998,-4428 4993.08,-4417.5 4988.55,-4405.58 4984.91,-4395.07\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5007.31,-4445.73 4998.55,-4439.14 5004.9,-4441.35 5002.49,-4436.97 5002.49,-4436.97 5002.49,-4436.97 5004.9,-4441.35 5006.43,-4434.8 5007.31,-4445.73 5007.31,-4445.73\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5028.5\\\" y=\\\"-4416.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3__plus0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_conv2_fwd -->\\n<g id=\\\"edge106\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3__plus0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_conv2_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5052.6,-4437.39C5054.81,-4434.24 5056.97,-4431.08 5059,-4428 5083.17,-4391.31 5108.28,-4347.16 5123.77,-4319.09\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5046.71,-4445.63 5048.87,-4434.88 5049.62,-4441.56 5052.53,-4437.5 5052.53,-4437.5 5052.53,-4437.5 5049.62,-4441.56 5056.19,-4440.12 5046.71,-4445.63 5046.71,-4445.63\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5098.5\\\" y=\\\"-4416.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_batchnorm2_fwd -->\\n<g id=\\\"node105\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_batchnorm2_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5070,-4613 4698,-4613 4698,-4555 5070,-4555 5070,-4613\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4884\\\" y=\\\"-4580.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage3_batchnorm2_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_batchnorm2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3__plus0 -->\\n<g id=\\\"edge107\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_batchnorm2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3__plus0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4929.22,-4548.68C4948.1,-4534.36 4969.77,-4517.91 4987.71,-4504.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4921.05,-4554.89 4926.29,-4545.26 4925.03,-4551.86 4929.01,-4548.84 4929.01,-4548.84 4929.01,-4548.84 4925.03,-4551.86 4931.73,-4552.43 4921.05,-4554.89 4921.05,-4554.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4993.5\\\" y=\\\"-4525.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_activation2 -->\\n<g id=\\\"node106\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_activation2</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"4931,-4722 4837,-4722 4837,-4664 4931,-4664 4931,-4722\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4884\\\" y=\\\"-4696.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"4884\\\" y=\\\"-4681.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_activation2&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_batchnorm2_fwd -->\\n<g id=\\\"edge108\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_activation2&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_batchnorm2_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4884,-4653.58C4884,-4640.28 4884,-4625.63 4884,-4613.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4884,-4663.89 4879.5,-4653.89 4884,-4658.89 4884,-4653.89 4884,-4653.89 4884,-4653.89 4884,-4658.89 4888.5,-4653.89 4884,-4663.89 4884,-4663.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4914.5\\\" y=\\\"-4634.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_conv3_fwd -->\\n<g id=\\\"node107\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_conv3_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"4931,-4831 4837,-4831 4837,-4773 4931,-4773 4931,-4831\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4884\\\" y=\\\"-4805.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"4884\\\" y=\\\"-4790.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x3/1x1, 256</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_conv3_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_activation2 -->\\n<g id=\\\"edge109\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_conv3_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_activation2</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4884,-4762.58C4884,-4749.28 4884,-4734.63 4884,-4722.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4884,-4772.89 4879.5,-4762.89 4884,-4767.89 4884,-4762.89 4884,-4762.89 4884,-4762.89 4884,-4767.89 4888.5,-4762.89 4884,-4772.89 4884,-4772.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4914.5\\\" y=\\\"-4743.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_batchnorm3_fwd -->\\n<g id=\\\"node108\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_batchnorm3_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5070,-4940 4698,-4940 4698,-4882 5070,-4882 5070,-4940\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4884\\\" y=\\\"-4907.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage3_batchnorm3_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_batchnorm3_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_conv3_fwd -->\\n<g id=\\\"edge110\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_batchnorm3_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_conv3_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4884,-4871.58C4884,-4858.28 4884,-4843.63 4884,-4831.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4884,-4881.89 4879.5,-4871.89 4884,-4876.89 4884,-4871.89 4884,-4871.89 4884,-4871.89 4884,-4876.89 4888.5,-4871.89 4884,-4881.89 4884,-4881.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4914.5\\\" y=\\\"-4852.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_activation3 -->\\n<g id=\\\"node109\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_activation3</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"5069,-5049 4975,-5049 4975,-4991 5069,-4991 5069,-5049\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5022\\\" y=\\\"-5023.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5022\\\" y=\\\"-5008.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_activation3&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_batchnorm3_fwd -->\\n<g id=\\\"edge111\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_activation3&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_batchnorm3_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4977.49,-4984.49C4959.07,-4970.21 4937.98,-4953.85 4920.49,-4940.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4985.74,-4990.89 4975.08,-4988.32 4981.79,-4987.82 4977.84,-4984.76 4977.84,-4984.76 4977.84,-4984.76 4981.79,-4987.82 4980.6,-4981.2 4985.74,-4990.89 4985.74,-4990.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4992.5\\\" y=\\\"-4961.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_conv4_fwd -->\\n<g id=\\\"node110\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_conv4_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5069,-5158 4975,-5158 4975,-5100 5069,-5100 5069,-5158\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5022\\\" y=\\\"-5132.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5022\\\" y=\\\"-5117.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x3/1x1, 256</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3_conv4_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_activation3 -->\\n<g id=\\\"edge112\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3_conv4_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_activation3</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5022,-5089.58C5022,-5076.28 5022,-5061.63 5022,-5049.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5022,-5099.89 5017.5,-5089.89 5022,-5094.89 5022,-5089.89 5022,-5089.89 5022,-5089.89 5022,-5094.89 5026.5,-5089.89 5022,-5099.89 5022,-5099.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5052.5\\\" y=\\\"-5070.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3__plus1 -->\\n<g id=\\\"node111\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage3__plus1</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5217.5,-5267 4902.5,-5267 4902.5,-5209 5217.5,-5209 5217.5,-5267\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5060\\\" y=\\\"-5234.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage3__plus1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3__plus1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3__plus0 -->\\n<g id=\\\"edge114\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3__plus1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3__plus0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5080.2,-5199.8C5089.29,-5179.72 5098,-5154.17 5098,-5130 5098,-5130 5098,-5130 5098,-4583 5098,-4552.42 5077.4,-4524.13 5058.08,-4504.33\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5075.9,-5208.9 5076.11,-5197.93 5078.04,-5204.38 5080.18,-5199.86 5080.18,-5199.86 5080.18,-5199.86 5078.04,-5204.38 5084.24,-5201.78 5075.9,-5208.9 5075.9,-5208.9\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5128.5\\\" y=\\\"-4852.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage3__plus1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_conv4_fwd -->\\n<g id=\\\"edge113\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage3__plus1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3_conv4_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5027.23,-5200.2C5025.57,-5197.21 5024.13,-5194.13 5023,-5191 5019.26,-5180.64 5018.39,-5168.63 5018.68,-5158.02\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5032.73,-5208.85 5023.57,-5202.83 5030.05,-5204.63 5027.36,-5200.41 5027.36,-5200.41 5027.36,-5200.41 5030.05,-5204.63 5031.16,-5198 5032.73,-5208.85 5032.73,-5208.85\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5053.5\\\" y=\\\"-5179.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_batchnorm0_fwd -->\\n<g id=\\\"node112\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_batchnorm0_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5246,-5376 4874,-5376 4874,-5318 5246,-5318 5246,-5376\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5060\\\" y=\\\"-5343.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage4_batchnorm0_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_batchnorm0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3__plus1 -->\\n<g id=\\\"edge115\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_batchnorm0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage3__plus1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5060,-5307.58C5060,-5294.28 5060,-5279.63 5060,-5267.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5060,-5317.89 5055.5,-5307.89 5060,-5312.89 5060,-5307.89 5060,-5307.89 5060,-5307.89 5060,-5312.89 5064.5,-5307.89 5060,-5317.89 5060,-5317.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5090.5\\\" y=\\\"-5288.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_activation0 -->\\n<g id=\\\"node113\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_activation0</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"5107,-5485 5013,-5485 5013,-5427 5107,-5427 5107,-5485\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5060\\\" y=\\\"-5459.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5060\\\" y=\\\"-5444.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_activation0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_batchnorm0_fwd -->\\n<g id=\\\"edge116\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_activation0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_batchnorm0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5060,-5416.58C5060,-5403.28 5060,-5388.63 5060,-5376.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5060,-5426.89 5055.5,-5416.89 5060,-5421.89 5060,-5416.89 5060,-5416.89 5060,-5416.89 5060,-5421.89 5064.5,-5416.89 5060,-5426.89 5060,-5426.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5090.5\\\" y=\\\"-5397.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_conv0_fwd -->\\n<g id=\\\"node114\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_conv0_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5024,-5594 4930,-5594 4930,-5536 5024,-5536 5024,-5594\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4977\\\" y=\\\"-5568.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"4977\\\" y=\\\"-5553.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x3/2x2, 512</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_conv0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_activation0 -->\\n<g id=\\\"edge117\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_conv0_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_activation0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5004.97,-5527.94C5015.76,-5514.03 5027.91,-5498.36 5038.05,-5485.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4998.81,-5535.89 5001.38,-5525.23 5001.87,-5531.94 5004.94,-5527.99 5004.94,-5527.99 5004.94,-5527.99 5001.87,-5531.94 5008.49,-5530.74 4998.81,-5535.89 4998.81,-5535.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5054.5\\\" y=\\\"-5506.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_batchnorm1_fwd -->\\n<g id=\\\"node115\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_batchnorm1_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5071,-5703 4699,-5703 4699,-5645 5071,-5645 5071,-5703\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4885\\\" y=\\\"-5670.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage4_batchnorm1_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_batchnorm1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_conv0_fwd -->\\n<g id=\\\"edge118\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_batchnorm1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_conv0_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4916.01,-5636.94C4927.96,-5623.03 4941.43,-5607.36 4952.67,-5594.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4909.17,-5644.89 4912.28,-5634.37 4912.43,-5641.1 4915.69,-5637.3 4915.69,-5637.3 4915.69,-5637.3 4912.43,-5641.1 4919.1,-5640.24 4909.17,-5644.89 4909.17,-5644.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4961\\\" y=\\\"-5615.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x7x7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_activation1 -->\\n<g id=\\\"node116\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_activation1</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"4948,-5812 4854,-5812 4854,-5754 4948,-5754 4948,-5812\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4901\\\" y=\\\"-5786.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"4901\\\" y=\\\"-5771.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_activation1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_batchnorm1_fwd -->\\n<g id=\\\"edge119\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_activation1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_batchnorm1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4895.31,-5743.98C4893.31,-5730.57 4891.09,-5715.75 4889.23,-5703.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4896.8,-5753.89 4890.87,-5744.66 4896.06,-5748.94 4895.32,-5744 4895.32,-5744 4895.32,-5744 4896.06,-5748.94 4899.77,-5743.33 4896.8,-5753.89 4896.8,-5753.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4918\\\" y=\\\"-5724.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x7x7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_conv1_fwd -->\\n<g id=\\\"node117\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_conv1_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"4984,-5964 4890,-5964 4890,-5906 4984,-5906 4984,-5964\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4937\\\" y=\\\"-5938.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"4937\\\" y=\\\"-5923.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x3/1x1, 512</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_conv1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_activation1 -->\\n<g id=\\\"edge120\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_conv1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_activation1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4927.84,-5895.85C4921.58,-5869.75 4913.41,-5835.73 4907.74,-5812.09\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4930.23,-5905.79 4923.52,-5897.11 4929.06,-5900.92 4927.89,-5896.06 4927.89,-5896.06 4927.89,-5896.06 4929.06,-5900.92 4932.27,-5895.01 4930.23,-5905.79 4930.23,-5905.79\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4950\\\" y=\\\"-5855.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x7x7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_conv2_fwd -->\\n<g id=\\\"node118\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_conv2_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5139,-5888 5045,-5888 5045,-5830 5139,-5830 5139,-5888\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5092\\\" y=\\\"-5862.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5092\\\" y=\\\"-5847.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1x1/2x2, 512</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_conv2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_activation0 -->\\n<g id=\\\"edge121\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_conv2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_activation0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5098.82,-5819.78C5102.78,-5794.3 5107,-5760 5107,-5729.5 5107,-5729.5 5107,-5729.5 5107,-5564 5107,-5535.39 5092.74,-5505.84 5079.93,-5485.06\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5097.2,-5829.78 5094.35,-5819.19 5098,-5824.85 5098.8,-5819.91 5098.8,-5819.91 5098.8,-5819.91 5098,-5824.85 5103.24,-5820.63 5097.2,-5829.78 5097.2,-5829.78\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5137.5\\\" y=\\\"-5670.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256x14x14</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4__plus0 -->\\n<g id=\\\"node119\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage4__plus0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5133.5,-6073 4818.5,-6073 4818.5,-6015 5133.5,-6015 5133.5,-6073\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4976\\\" y=\\\"-6040.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage4__plus0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4__plus0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_conv1_fwd -->\\n<g id=\\\"edge122\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4__plus0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_conv1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4957.32,-6005.27C4956.14,-6002.5 4955.02,-5999.72 4954,-5997 4950.06,-5986.5 4946.56,-5974.68 4943.78,-5964.27\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4961.48,-6014.63 4953.3,-6007.32 4959.45,-6010.06 4957.41,-6005.49 4957.41,-6005.49 4957.41,-6005.49 4959.45,-6010.06 4961.53,-6003.66 4961.48,-6014.63 4961.48,-6014.63\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4978\\\" y=\\\"-5985.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x7x7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4__plus0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_conv2_fwd -->\\n<g id=\\\"edge123\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4__plus0&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_conv2_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4999.24,-6006.34C5021.3,-5971.53 5054,-5919.94 5074.11,-5888.22\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4993.87,-6014.81 4995.42,-6003.95 4996.55,-6010.58 4999.22,-6006.36 4999.22,-6006.36 4999.22,-6006.36 4996.55,-6010.58 5003.03,-6008.77 4993.87,-6014.81 4993.87,-6014.81\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5039\\\" y=\\\"-5985.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x7x7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_batchnorm2_fwd -->\\n<g id=\\\"node120\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_batchnorm2_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5017,-6182 4645,-6182 4645,-6124 5017,-6124 5017,-6182\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4831\\\" y=\\\"-6149.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage4_batchnorm2_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_batchnorm2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4__plus0 -->\\n<g id=\\\"edge124\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_batchnorm2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4__plus0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4877.28,-6117.85C4896.83,-6103.42 4919.33,-6086.82 4937.91,-6073.11\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4869.1,-6123.89 4874.47,-6114.33 4873.12,-6120.92 4877.14,-6117.95 4877.14,-6117.95 4877.14,-6117.95 4873.12,-6120.92 4879.82,-6121.57 4869.1,-6123.89 4869.1,-6123.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4937\\\" y=\\\"-6094.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x7x7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_activation2 -->\\n<g id=\\\"node121\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_activation2</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"4878,-6291 4784,-6291 4784,-6233 4878,-6233 4878,-6291\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4831\\\" y=\\\"-6265.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"4831\\\" y=\\\"-6250.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_activation2&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_batchnorm2_fwd -->\\n<g id=\\\"edge125\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_activation2&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_batchnorm2_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4831,-6222.58C4831,-6209.28 4831,-6194.63 4831,-6182.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4831,-6232.89 4826.5,-6222.89 4831,-6227.89 4831,-6222.89 4831,-6222.89 4831,-6222.89 4831,-6227.89 4835.5,-6222.89 4831,-6232.89 4831,-6232.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4855\\\" y=\\\"-6203.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x7x7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_conv3_fwd -->\\n<g id=\\\"node122\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_conv3_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"4878,-6400 4784,-6400 4784,-6342 4878,-6342 4878,-6400\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4831\\\" y=\\\"-6374.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"4831\\\" y=\\\"-6359.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x3/1x1, 512</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_conv3_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_activation2 -->\\n<g id=\\\"edge126\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_conv3_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_activation2</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4831,-6331.58C4831,-6318.28 4831,-6303.63 4831,-6291.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4831,-6341.89 4826.5,-6331.89 4831,-6336.89 4831,-6331.89 4831,-6331.89 4831,-6331.89 4831,-6336.89 4835.5,-6331.89 4831,-6341.89 4831,-6341.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4855\\\" y=\\\"-6312.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x7x7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_batchnorm3_fwd -->\\n<g id=\\\"node123\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_batchnorm3_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5017,-6509 4645,-6509 4645,-6451 5017,-6451 5017,-6509\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4831\\\" y=\\\"-6476.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage4_batchnorm3_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_batchnorm3_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_conv3_fwd -->\\n<g id=\\\"edge127\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_batchnorm3_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_conv3_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4831,-6440.58C4831,-6427.28 4831,-6412.63 4831,-6400.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4831,-6450.89 4826.5,-6440.89 4831,-6445.89 4831,-6440.89 4831,-6440.89 4831,-6440.89 4831,-6445.89 4835.5,-6440.89 4831,-6450.89 4831,-6450.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4855\\\" y=\\\"-6421.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x7x7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_activation3 -->\\n<g id=\\\"node124\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_activation3</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"4948,-6618 4854,-6618 4854,-6560 4948,-6560 4948,-6618\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4901\\\" y=\\\"-6592.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"4901\\\" y=\\\"-6577.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_activation3&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_batchnorm3_fwd -->\\n<g id=\\\"edge128\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_activation3&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_batchnorm3_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4876.9,-6551.16C4867.93,-6537.45 4857.9,-6522.12 4849.51,-6509.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4882.61,-6559.89 4873.37,-6553.98 4879.87,-6555.7 4877.13,-6551.52 4877.13,-6551.52 4877.13,-6551.52 4879.87,-6555.7 4880.9,-6549.06 4882.61,-6559.89 4882.61,-6559.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4895\\\" y=\\\"-6530.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x7x7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_conv4_fwd -->\\n<g id=\\\"node125\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_conv4_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5017,-6727 4923,-6727 4923,-6669 5017,-6669 5017,-6727\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4970\\\" y=\\\"-6701.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Convolution</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"4970\\\" y=\\\"-6686.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">3x3/1x1, 512</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4_conv4_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_activation3 -->\\n<g id=\\\"edge129\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4_conv4_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_activation3</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4946.24,-6660.16C4937.4,-6646.45 4927.52,-6631.12 4919.25,-6618.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4951.87,-6668.89 4942.67,-6662.92 4949.16,-6664.68 4946.45,-6660.48 4946.45,-6660.48 4946.45,-6660.48 4949.16,-6664.68 4950.23,-6658.04 4951.87,-6668.89 4951.87,-6668.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4964\\\" y=\\\"-6639.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x7x7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4__plus1 -->\\n<g id=\\\"node126\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_stage4__plus1</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5170.5,-6836 4855.5,-6836 4855.5,-6778 5170.5,-6778 5170.5,-6836\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5013\\\" y=\\\"-6803.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_stage4__plus1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4__plus1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4__plus0 -->\\n<g id=\\\"edge131\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4__plus1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4__plus0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5030.12,-6768.2C5037.73,-6748.08 5045,-6722.66 5045,-6699 5045,-6699 5045,-6699 5045,-6152 5045,-6121.72 5025.24,-6093.2 5006.87,-6073.24\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5026.39,-6777.62 5025.89,-6766.67 5028.23,-6772.97 5030.07,-6768.32 5030.07,-6768.32 5030.07,-6768.32 5028.23,-6772.97 5034.25,-6769.98 5026.39,-6777.62 5026.39,-6777.62\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5069\\\" y=\\\"-6421.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x7x7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_stage4__plus1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_conv4_fwd -->\\n<g id=\\\"edge130\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_stage4__plus1&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4_conv4_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4987.25,-6768.97C4985.69,-6766 4984.24,-6762.99 4983,-6760 4978.72,-6749.7 4975.78,-6737.82 4973.79,-6727.29\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4992.25,-6777.7 4983.38,-6771.26 4989.77,-6773.36 4987.28,-6769.02 4987.28,-6769.02 4987.28,-6769.02 4989.77,-6773.36 4991.19,-6766.78 4992.25,-6777.7 4992.25,-6777.7\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5007\\\" y=\\\"-6748.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x7x7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_batchnorm2_fwd -->\\n<g id=\\\"node127\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_batchnorm2_fwd</title>\\n<polygon fill=\\\"#bebada\\\" stroke=\\\"black\\\" points=\\\"5178.5,-6945 4847.5,-6945 4847.5,-6887 5178.5,-6887 5178.5,-6945\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5013\\\" y=\\\"-6912.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_batchnorm2_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_batchnorm2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4__plus1 -->\\n<g id=\\\"edge132\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_batchnorm2_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_stage4__plus1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5013,-6876.58C5013,-6863.28 5013,-6848.63 5013,-6836.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5013,-6886.89 5008.5,-6876.89 5013,-6881.89 5013,-6876.89 5013,-6876.89 5013,-6876.89 5013,-6881.89 5017.5,-6876.89 5013,-6886.89 5013,-6886.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5037\\\" y=\\\"-6857.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x7x7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_relu1_fwd -->\\n<g id=\\\"node128\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_relu1_fwd</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"5060,-7054 4966,-7054 4966,-6996 5060,-6996 5060,-7054\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5013\\\" y=\\\"-7028.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5013\\\" y=\\\"-7013.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_relu1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_batchnorm2_fwd -->\\n<g id=\\\"edge133\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_relu1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_batchnorm2_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5013,-6985.58C5013,-6972.28 5013,-6957.63 5013,-6945.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5013,-6995.89 5008.5,-6985.89 5013,-6990.89 5013,-6985.89 5013,-6985.89 5013,-6985.89 5013,-6990.89 5017.5,-6985.89 5013,-6995.89 5013,-6995.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5037\\\" y=\\\"-6966.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x7x7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_pool1_fwd -->\\n<g id=\\\"node129\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_pool1_fwd</title>\\n<polygon fill=\\\"#80b1d3\\\" stroke=\\\"black\\\" points=\\\"5060,-7163 4966,-7163 4966,-7105 5060,-7105 5060,-7163\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5013\\\" y=\\\"-7137.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Pooling</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5013\\\" y=\\\"-7122.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">avg, 1x1/1x1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_pool1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_relu1_fwd -->\\n<g id=\\\"edge134\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_pool1_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_relu1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5013,-7094.58C5013,-7081.28 5013,-7066.63 5013,-7054.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5013,-7104.89 5008.5,-7094.89 5013,-7099.89 5013,-7094.89 5013,-7094.89 5013,-7094.89 5013,-7099.89 5017.5,-7094.89 5013,-7104.89 5013,-7104.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5037\\\" y=\\\"-7075.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x7x7</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_flatten0_flatten0 -->\\n<g id=\\\"node130\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_resnetv21_flatten0_flatten0</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"5176.5,-7272 4849.5,-7272 4849.5,-7214 5176.5,-7214 5176.5,-7272\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5013\\\" y=\\\"-7239.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_resnetv21_flatten0_flatten0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_resnetv21_flatten0_flatten0&#45;&gt;dssmrecommendernetwork0_resnetv21_pool1_fwd -->\\n<g id=\\\"edge135\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_resnetv21_flatten0_flatten0&#45;&gt;dssmrecommendernetwork0_resnetv21_pool1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5013,-7203.58C5013,-7190.28 5013,-7175.63 5013,-7163.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5013,-7213.89 5008.5,-7203.89 5013,-7208.89 5013,-7203.89 5013,-7203.89 5013,-7203.89 5013,-7208.89 5017.5,-7203.89 5013,-7213.89 5013,-7213.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5037\\\" y=\\\"-7184.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512x1x1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense4_fwd -->\\n<g id=\\\"node131\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_dense4_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5064.5,-7381 4961.5,-7381 4961.5,-7323 5064.5,-7323 5064.5,-7381\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5013\\\" y=\\\"-7355.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">FullyConnected</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5013\\\" y=\\\"-7340.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense4_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_flatten0_flatten0 -->\\n<g id=\\\"edge136\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_dense4_fwd&#45;&gt;dssmrecommendernetwork0_resnetv21_flatten0_flatten0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5013,-7312.58C5013,-7299.28 5013,-7284.63 5013,-7272.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5013,-7322.89 5008.5,-7312.89 5013,-7317.89 5013,-7312.89 5013,-7312.89 5013,-7312.89 5013,-7317.89 5017.5,-7312.89 5013,-7322.89 5013,-7322.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5023.5\\\" y=\\\"-7293.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">512</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense4_relu_fwd -->\\n<g id=\\\"node132\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_dense4_relu_fwd</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"5060,-7490 4966,-7490 4966,-7432 5060,-7432 5060,-7490\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5013\\\" y=\\\"-7464.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5013\\\" y=\\\"-7449.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense4_relu_fwd&#45;&gt;dssmrecommendernetwork0_dense4_fwd -->\\n<g id=\\\"edge137\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_dense4_relu_fwd&#45;&gt;dssmrecommendernetwork0_dense4_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5013,-7421.58C5013,-7408.28 5013,-7393.63 5013,-7381.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5013,-7431.89 5008.5,-7421.89 5013,-7426.89 5013,-7421.89 5013,-7421.89 5013,-7421.89 5013,-7426.89 5017.5,-7421.89 5013,-7431.89 5013,-7431.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5023.5\\\" y=\\\"-7402.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_concat1 -->\\n<g id=\\\"node133\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_concat1</title>\\n<polygon fill=\\\"#fdb462\\\" stroke=\\\"black\\\" points=\\\"5123.5,-7599 4902.5,-7599 4902.5,-7541 5123.5,-7541 5123.5,-7599\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5013\\\" y=\\\"-7566.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_concat1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_concat1&#45;&gt;dssmrecommendernetwork0_dense3_relu_fwd -->\\n<g id=\\\"edge138\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_concat1&#45;&gt;dssmrecommendernetwork0_dense3_relu_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M4975.17,-7534.1C4959.79,-7519.91 4942.24,-7503.73 4927.68,-7490.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"4982.52,-7540.89 4972.12,-7537.41 4978.85,-7537.5 4975.17,-7534.11 4975.17,-7534.11 4975.17,-7534.11 4978.85,-7537.5 4978.22,-7530.8 4982.52,-7540.89 4982.52,-7540.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"4972.5\\\" y=\\\"-7511.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_concat1&#45;&gt;dssmrecommendernetwork0_dense4_relu_fwd -->\\n<g id=\\\"edge139\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_concat1&#45;&gt;dssmrecommendernetwork0_dense4_relu_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5013,-7530.58C5013,-7517.28 5013,-7502.63 5013,-7490.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5013,-7540.89 5008.5,-7530.89 5013,-7535.89 5013,-7530.89 5013,-7530.89 5013,-7530.89 5013,-7535.89 5017.5,-7530.89 5013,-7540.89 5013,-7540.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5023.5\\\" y=\\\"-7511.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dropout1_fwd -->\\n<g id=\\\"node134\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_dropout1_fwd</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5140.5,-7708 4885.5,-7708 4885.5,-7650 5140.5,-7650 5140.5,-7708\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5013\\\" y=\\\"-7675.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_dropout1_fwd</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dropout1_fwd&#45;&gt;dssmrecommendernetwork0_concat1 -->\\n<g id=\\\"edge140\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_dropout1_fwd&#45;&gt;dssmrecommendernetwork0_concat1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5013,-7639.58C5013,-7626.28 5013,-7611.63 5013,-7599.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5013,-7649.89 5008.5,-7639.89 5013,-7644.89 5013,-7639.89 5013,-7639.89 5013,-7639.89 5013,-7644.89 5017.5,-7639.89 5013,-7649.89 5013,-7649.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5023.5\\\" y=\\\"-7620.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense5_fwd -->\\n<g id=\\\"node135\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_dense5_fwd</title>\\n<polygon fill=\\\"#fb8072\\\" stroke=\\\"black\\\" points=\\\"5066.5,-7817 4963.5,-7817 4963.5,-7759 5066.5,-7759 5066.5,-7817\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5015\\\" y=\\\"-7791.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">FullyConnected</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5015\\\" y=\\\"-7776.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense5_fwd&#45;&gt;dssmrecommendernetwork0_dropout1_fwd -->\\n<g id=\\\"edge141\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_dense5_fwd&#45;&gt;dssmrecommendernetwork0_dropout1_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5014.28,-7748.58C5014.03,-7735.28 5013.76,-7720.63 5013.53,-7708.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5014.47,-7758.89 5009.79,-7748.97 5014.38,-7753.89 5014.29,-7748.89 5014.29,-7748.89 5014.29,-7748.89 5014.38,-7753.89 5018.79,-7748.8 5014.47,-7758.89 5014.47,-7758.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5025.5\\\" y=\\\"-7729.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">256</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense5_relu_fwd -->\\n<g id=\\\"node136\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_dense5_relu_fwd</title>\\n<polygon fill=\\\"#ffffb3\\\" stroke=\\\"black\\\" points=\\\"5093,-7926 4999,-7926 4999,-7868 5093,-7868 5093,-7926\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5046\\\" y=\\\"-7900.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">Activation</text>\\n<text text-anchor=\\\"middle\\\" x=\\\"5046\\\" y=\\\"-7885.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">relu</text>\\n</g>\\n<!-- dssmrecommendernetwork0_dense5_relu_fwd&#45;&gt;dssmrecommendernetwork0_dense5_fwd -->\\n<g id=\\\"edge142\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_dense5_relu_fwd&#45;&gt;dssmrecommendernetwork0_dense5_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5034.98,-7857.98C5031.1,-7844.57 5026.81,-7829.75 5023.2,-7817.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5037.86,-7867.89 5030.75,-7859.53 5036.46,-7863.08 5035.07,-7858.28 5035.07,-7858.28 5035.07,-7858.28 5036.46,-7863.08 5039.39,-7857.03 5037.86,-7867.89 5037.86,-7867.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5043.5\\\" y=\\\"-7838.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_expand_dims1 -->\\n<g id=\\\"node137\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_expand_dims1</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5275,-8035 5017,-8035 5017,-7977 5275,-7977 5275,-8035\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5146\\\" y=\\\"-8002.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_expand_dims1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_expand_dims1&#45;&gt;dssmrecommendernetwork0_dense5_relu_fwd -->\\n<g id=\\\"edge143\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_expand_dims1&#45;&gt;dssmrecommendernetwork0_dense5_relu_fwd</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5112.66,-7969.33C5099.58,-7955.33 5084.77,-7939.49 5072.45,-7926.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5119.73,-7976.89 5109.61,-7972.65 5116.31,-7973.23 5112.9,-7969.58 5112.9,-7969.58 5112.9,-7969.58 5116.31,-7973.23 5116.19,-7966.51 5119.73,-7976.89 5119.73,-7976.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5112.5\\\" y=\\\"-7947.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128</text>\\n</g>\\n<!-- dssmrecommendernetwork0_batch_dot0 -->\\n<g id=\\\"node138\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_batch_dot0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5265.5,-8144 5026.5,-8144 5026.5,-8086 5265.5,-8086 5265.5,-8144\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5146\\\" y=\\\"-8111.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_batch_dot0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_batch_dot0&#45;&gt;dssmrecommendernetwork0_expand_dims0 -->\\n<g id=\\\"edge144\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_batch_dot0&#45;&gt;dssmrecommendernetwork0_expand_dims0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5201.9,-8081.12C5222.11,-8070.58 5245.51,-8059.86 5268,-8053 5341.32,-8030.64 5426.36,-8019.14 5494.94,-8013.24\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5192.84,-8085.94 5199.56,-8077.27 5197.26,-8083.59 5201.67,-8081.24 5201.67,-8081.24 5201.67,-8081.24 5197.26,-8083.59 5203.79,-8085.21 5192.84,-8085.94 5192.84,-8085.94\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5285\\\" y=\\\"-8056.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_batch_dot0&#45;&gt;dssmrecommendernetwork0_expand_dims1 -->\\n<g id=\\\"edge145\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_batch_dot0&#45;&gt;dssmrecommendernetwork0_expand_dims1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5146,-8075.58C5146,-8062.28 5146,-8047.63 5146,-8035.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5146,-8085.89 5141.5,-8075.89 5146,-8080.89 5146,-8075.89 5146,-8075.89 5146,-8075.89 5146,-8080.89 5150.5,-8075.89 5146,-8085.89 5146,-8085.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5163\\\" y=\\\"-8056.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_norm0 -->\\n<g id=\\\"node139\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_norm0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5731.5,-8144 5516.5,-8144 5516.5,-8086 5731.5,-8086 5731.5,-8144\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5624\\\" y=\\\"-8111.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_norm0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_norm0&#45;&gt;dssmrecommendernetwork0_expand_dims0 -->\\n<g id=\\\"edge146\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_norm0&#45;&gt;dssmrecommendernetwork0_expand_dims0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5624,-8075.58C5624,-8062.28 5624,-8047.63 5624,-8035.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5624,-8085.89 5619.5,-8075.89 5624,-8080.89 5624,-8075.89 5624,-8075.89 5624,-8075.89 5624,-8080.89 5628.5,-8075.89 5624,-8085.89 5624,-8085.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5641\\\" y=\\\"-8056.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_norm1 -->\\n<g id=\\\"node140\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_norm1</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5498.5,-8144 5283.5,-8144 5283.5,-8086 5498.5,-8086 5498.5,-8144\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5391\\\" y=\\\"-8111.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_norm1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_norm1&#45;&gt;dssmrecommendernetwork0_expand_dims1 -->\\n<g id=\\\"edge147\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_norm1&#45;&gt;dssmrecommendernetwork0_expand_dims1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5349.52,-8079.92C5336.22,-8070.2 5321.03,-8060.28 5306,-8053 5291.95,-8046.19 5276.72,-8040.21 5261.41,-8035.01\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5357.55,-8085.92 5346.84,-8083.53 5353.54,-8082.93 5349.54,-8079.93 5349.54,-8079.93 5349.54,-8079.93 5353.54,-8082.93 5352.24,-8076.33 5357.55,-8085.92 5357.55,-8085.92\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5348\\\" y=\\\"-8056.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">128x1</text>\\n</g>\\n<!-- dssmrecommendernetwork0__mul0 -->\\n<g id=\\\"node141\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0__mul0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5498,-8253 5284,-8253 5284,-8195 5498,-8195 5498,-8253\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5391\\\" y=\\\"-8220.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0__mul0</text>\\n</g>\\n<!-- dssmrecommendernetwork0__mul0&#45;&gt;dssmrecommendernetwork0_norm0 -->\\n<g id=\\\"edge148\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0__mul0&#45;&gt;dssmrecommendernetwork0_norm0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5461.6,-8190.58C5493.87,-8175.76 5531.74,-8158.37 5562.79,-8144.11\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5452.22,-8194.89 5459.43,-8186.62 5456.76,-8192.8 5461.31,-8190.71 5461.31,-8190.71 5461.31,-8190.71 5456.76,-8192.8 5463.18,-8194.8 5452.22,-8194.89 5452.22,-8194.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5525.5\\\" y=\\\"-8165.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1</text>\\n</g>\\n<!-- dssmrecommendernetwork0__mul0&#45;&gt;dssmrecommendernetwork0_norm1 -->\\n<g id=\\\"edge149\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0__mul0&#45;&gt;dssmrecommendernetwork0_norm1</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5391,-8184.58C5391,-8171.28 5391,-8156.63 5391,-8144.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5391,-8194.89 5386.5,-8184.89 5391,-8189.89 5391,-8184.89 5391,-8184.89 5391,-8184.89 5391,-8189.89 5395.5,-8184.89 5391,-8194.89 5391,-8194.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5394.5\\\" y=\\\"-8165.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1</text>\\n</g>\\n<!-- dssmrecommendernetwork0__plusscalar0 -->\\n<g id=\\\"node142\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0__plusscalar0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5498,-8362 5252,-8362 5252,-8304 5498,-8304 5498,-8362\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5375\\\" y=\\\"-8329.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0__plusscalar0</text>\\n</g>\\n<!-- dssmrecommendernetwork0__plusscalar0&#45;&gt;dssmrecommendernetwork0__mul0 -->\\n<g id=\\\"edge150\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0__plusscalar0&#45;&gt;dssmrecommendernetwork0__mul0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5380.69,-8293.98C5382.69,-8280.57 5384.91,-8265.75 5386.77,-8253.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5379.2,-8303.89 5376.23,-8293.33 5379.94,-8298.94 5380.68,-8294 5380.68,-8294 5380.68,-8294 5379.94,-8298.94 5385.13,-8294.66 5379.2,-8303.89 5379.2,-8303.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5387.5\\\" y=\\\"-8274.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_expand_dims2 -->\\n<g id=\\\"node143\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_expand_dims2</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5479,-8471 5221,-8471 5221,-8413 5479,-8413 5479,-8471\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5350\\\" y=\\\"-8438.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_expand_dims2</text>\\n</g>\\n<!-- dssmrecommendernetwork0_expand_dims2&#45;&gt;dssmrecommendernetwork0__plusscalar0 -->\\n<g id=\\\"edge151\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_expand_dims2&#45;&gt;dssmrecommendernetwork0__plusscalar0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5358.88,-8402.98C5362.02,-8389.57 5365.48,-8374.75 5368.39,-8362.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5356.57,-8412.89 5354.46,-8402.13 5357.71,-8408.02 5358.84,-8403.15 5358.84,-8403.15 5358.84,-8403.15 5357.71,-8408.02 5363.23,-8404.17 5356.57,-8412.89 5356.57,-8412.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5367.5\\\" y=\\\"-8383.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1</text>\\n</g>\\n<!-- dssmrecommendernetwork0__div0 -->\\n<g id=\\\"node144\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0__div0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5269.5,-8580 5060.5,-8580 5060.5,-8522 5269.5,-8522 5269.5,-8580\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5165\\\" y=\\\"-8547.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0__div0</text>\\n</g>\\n<!-- dssmrecommendernetwork0__div0&#45;&gt;dssmrecommendernetwork0_batch_dot0 -->\\n<g id=\\\"edge152\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0__div0&#45;&gt;dssmrecommendernetwork0_batch_dot0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5156.47,-8511.74C5152.65,-8491.43 5149,-8465.98 5149,-8443 5149,-8443 5149,-8443 5149,-8223 5149,-8196.24 5148.05,-8165.73 5147.22,-8144.15\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5158.4,-8521.56 5152.05,-8512.62 5157.44,-8516.66 5156.47,-8511.75 5156.47,-8511.75 5156.47,-8511.75 5157.44,-8516.66 5160.89,-8510.88 5158.4,-8521.56 5158.4,-8521.56\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5159.5\\\" y=\\\"-8329.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1x1</text>\\n</g>\\n<!-- dssmrecommendernetwork0__div0&#45;&gt;dssmrecommendernetwork0_expand_dims2 -->\\n<g id=\\\"edge153\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0__div0&#45;&gt;dssmrecommendernetwork0_expand_dims2</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5222.38,-8516.81C5247.71,-8502.16 5277.17,-8485.12 5301.4,-8471.11\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5213.61,-8521.89 5220.01,-8512.98 5217.93,-8519.38 5222.26,-8516.88 5222.26,-8516.88 5222.26,-8516.88 5217.93,-8519.38 5224.52,-8520.78 5213.61,-8521.89 5213.61,-8521.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5279.5\\\" y=\\\"-8492.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1x1</text>\\n</g>\\n<!-- dssmrecommendernetwork0_squeeze0 -->\\n<g id=\\\"node145\\\" class=\\\"node\\\"><title>dssmrecommendernetwork0_squeeze0</title>\\n<polygon fill=\\\"#fccde5\\\" stroke=\\\"black\\\" points=\\\"5279,-8689 5051,-8689 5051,-8631 5279,-8631 5279,-8689\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5165\\\" y=\\\"-8656.3\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">dssmrecommendernetwork0_squeeze0</text>\\n</g>\\n<!-- dssmrecommendernetwork0_squeeze0&#45;&gt;dssmrecommendernetwork0__div0 -->\\n<g id=\\\"edge154\\\" class=\\\"edge\\\"><title>dssmrecommendernetwork0_squeeze0&#45;&gt;dssmrecommendernetwork0__div0</title>\\n<path fill=\\\"none\\\" stroke=\\\"black\\\" d=\\\"M5165,-8620.58C5165,-8607.28 5165,-8592.63 5165,-8580.3\\\"/>\\n<polygon fill=\\\"black\\\" stroke=\\\"black\\\" points=\\\"5165,-8630.89 5160.5,-8620.89 5165,-8625.89 5165,-8620.89 5165,-8620.89 5165,-8620.89 5165,-8625.89 5169.5,-8620.89 5165,-8630.89 5165,-8630.89\\\"/>\\n<text text-anchor=\\\"middle\\\" x=\\\"5175.5\\\" y=\\\"-8601.8\\\" font-family=\\\"Times,serif\\\" font-size=\\\"14.00\\\">1x1</text>\\n</g>\\n</g>\\n</svg>\\n\",\n      \"text/plain\": [\n       \"<graphviz.dot.Digraph at 0x7ff0a64a48d0>\"\n      ]\n     },\n     \"metadata\": {},\n     \"execution_count\": 11\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"We can print the summary of the network using dummy data. We can see it is already training on 32M parameters!\"\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 12,\n   \"source\": [\n    \"user  = mx.np.array([[200], [100]], ctx)\\n\",\n    \"query = mx.np.array([[10, 20, 0, 0, 0], [40, 50, 0, 0, 0]], ctx) # Example of an encoded text\\n\",\n    \"title = mx.np.array([[10, 20, 0, 0, 0], [40, 50, 0, 0, 0]], ctx) # Example of an encoded text\\n\",\n    \"image = mx.np.random.uniform(size=(2,3, 224,224), ctx=ctx) # Example of an encoded image\\n\",\n    \"\\n\",\n    \"\\n\",\n    \"network.summary(user, query, title, image)\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"stream\",\n     \"name\": \"stdout\",\n     \"text\": [\n      \"--------------------------------------------------------------------------------\\n\",\n      \"        Layer (type)                                Output Shape         Param #\\n\",\n      \"================================================================================\\n\",\n      \"               Input    (2, 1), (2, 5), (2, 5), (2, 3, 224, 224)               0\\n\",\n      \"         Embedding-1                                 (2, 1, 128)        12800000\\n\",\n      \"        Activation-2  <Symbol dssmrecommendernetwork0_dense0_relu_fwd>               0\\n\",\n      \"        Activation-3                                    (2, 128)               0\\n\",\n      \"             Dense-4                                    (2, 128)           16512\\n\",\n      \"         Embedding-5                                 (2, 5, 128)         3840000\\n\",\n      \"              LSTM-6                                 (5, 2, 256)          659456\\n\",\n      \"        Activation-7  <Symbol dssmrecommendernetwork0_dense1_relu_fwd>               0\\n\",\n      \"        Activation-8                                    (2, 128)               0\\n\",\n      \"             Dense-9                                    (2, 128)           32896\\n\",\n      \"          Dropout-10                                    (2, 256)               0\\n\",\n      \"       Activation-11  <Symbol dssmrecommendernetwork0_dense2_relu_fwd>               0\\n\",\n      \"       Activation-12                                    (2, 128)               0\\n\",\n      \"            Dense-13                                    (2, 128)           32896\\n\",\n      \"        Embedding-14                                 (2, 5, 128)         3840000\\n\",\n      \"             LSTM-15                                 (5, 2, 256)          659456\\n\",\n      \"       Activation-16  <Symbol dssmrecommendernetwork0_dense3_relu_fwd>               0\\n\",\n      \"       Activation-17                                    (2, 128)               0\\n\",\n      \"            Dense-18                                    (2, 128)           32896\\n\",\n      \"        BatchNorm-19                            (2, 3, 224, 224)              12\\n\",\n      \"           Conv2D-20                           (2, 64, 112, 112)            9408\\n\",\n      \"        BatchNorm-21                           (2, 64, 112, 112)             256\\n\",\n      \"       Activation-22                           (2, 64, 112, 112)               0\\n\",\n      \"        MaxPool2D-23                             (2, 64, 56, 56)               0\\n\",\n      \"        BatchNorm-24                             (2, 64, 56, 56)             256\\n\",\n      \"           Conv2D-25                             (2, 64, 56, 56)           36864\\n\",\n      \"        BatchNorm-26                             (2, 64, 56, 56)             256\\n\",\n      \"           Conv2D-27                             (2, 64, 56, 56)           36864\\n\",\n      \"     BasicBlockV2-28                             (2, 64, 56, 56)               0\\n\",\n      \"        BatchNorm-29                             (2, 64, 56, 56)             256\\n\",\n      \"           Conv2D-30                             (2, 64, 56, 56)           36864\\n\",\n      \"        BatchNorm-31                             (2, 64, 56, 56)             256\\n\",\n      \"           Conv2D-32                             (2, 64, 56, 56)           36864\\n\",\n      \"     BasicBlockV2-33                             (2, 64, 56, 56)               0\\n\",\n      \"        BatchNorm-34                             (2, 64, 56, 56)             256\\n\",\n      \"           Conv2D-35                            (2, 128, 28, 28)            8192\\n\",\n      \"           Conv2D-36                            (2, 128, 28, 28)           73728\\n\",\n      \"        BatchNorm-37                            (2, 128, 28, 28)             512\\n\",\n      \"           Conv2D-38                            (2, 128, 28, 28)          147456\\n\",\n      \"     BasicBlockV2-39                            (2, 128, 28, 28)               0\\n\",\n      \"        BatchNorm-40                            (2, 128, 28, 28)             512\\n\",\n      \"           Conv2D-41                            (2, 128, 28, 28)          147456\\n\",\n      \"        BatchNorm-42                            (2, 128, 28, 28)             512\\n\",\n      \"           Conv2D-43                            (2, 128, 28, 28)          147456\\n\",\n      \"     BasicBlockV2-44                            (2, 128, 28, 28)               0\\n\",\n      \"        BatchNorm-45                            (2, 128, 28, 28)             512\\n\",\n      \"           Conv2D-46                            (2, 256, 14, 14)           32768\\n\",\n      \"           Conv2D-47                            (2, 256, 14, 14)          294912\\n\",\n      \"        BatchNorm-48                            (2, 256, 14, 14)            1024\\n\",\n      \"           Conv2D-49                            (2, 256, 14, 14)          589824\\n\",\n      \"     BasicBlockV2-50                            (2, 256, 14, 14)               0\\n\",\n      \"        BatchNorm-51                            (2, 256, 14, 14)            1024\\n\",\n      \"           Conv2D-52                            (2, 256, 14, 14)          589824\\n\",\n      \"        BatchNorm-53                            (2, 256, 14, 14)            1024\\n\",\n      \"           Conv2D-54                            (2, 256, 14, 14)          589824\\n\",\n      \"     BasicBlockV2-55                            (2, 256, 14, 14)               0\\n\",\n      \"        BatchNorm-56                            (2, 256, 14, 14)            1024\\n\",\n      \"           Conv2D-57                              (2, 512, 7, 7)          131072\\n\",\n      \"           Conv2D-58                              (2, 512, 7, 7)         1179648\\n\",\n      \"        BatchNorm-59                              (2, 512, 7, 7)            2048\\n\",\n      \"           Conv2D-60                              (2, 512, 7, 7)         2359296\\n\",\n      \"     BasicBlockV2-61                              (2, 512, 7, 7)               0\\n\",\n      \"        BatchNorm-62                              (2, 512, 7, 7)            2048\\n\",\n      \"           Conv2D-63                              (2, 512, 7, 7)         2359296\\n\",\n      \"        BatchNorm-64                              (2, 512, 7, 7)            2048\\n\",\n      \"           Conv2D-65                              (2, 512, 7, 7)         2359296\\n\",\n      \"     BasicBlockV2-66                              (2, 512, 7, 7)               0\\n\",\n      \"        BatchNorm-67                              (2, 512, 7, 7)            2048\\n\",\n      \"       Activation-68                              (2, 512, 7, 7)               0\\n\",\n      \"  GlobalAvgPool2D-69                              (2, 512, 1, 1)               0\\n\",\n      \"          Flatten-70                                    (2, 512)               0\\n\",\n      \"       Activation-71  <Symbol dssmrecommendernetwork0_dense4_relu_fwd>               0\\n\",\n      \"       Activation-72                                    (2, 128)               0\\n\",\n      \"            Dense-73                                    (2, 128)           65664\\n\",\n      \"          Dropout-74                                    (2, 256)               0\\n\",\n      \"       Activation-75  <Symbol dssmrecommendernetwork0_dense5_relu_fwd>               0\\n\",\n      \"       Activation-76                                    (2, 128)               0\\n\",\n      \"            Dense-77                                    (2, 128)           32896\\n\",\n      \"DSSMRecommenderNetwork-78                                      (2, 1)               0\\n\",\n      \"================================================================================\\n\",\n      \"Parameters in forward computation graph, duplicate included\\n\",\n      \"   Total params: 33195468\\n\",\n      \"   Trainable params: 33187520\\n\",\n      \"   Non-trainable params: 7948\\n\",\n      \"Shared params in forward computation graph: 0\\n\",\n      \"Unique parameters in model: 33195468\\n\",\n      \"--------------------------------------------------------------------------------\\n\"\n     ]\n    }\n   ],\n   \"metadata\": {\n    \"collapsed\": true\n   }\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 13,\n   \"source\": [\n    \"network(user, query, title, image)\"\n   ],\n   \"outputs\": [\n    {\n     \"output_type\": \"execute_result\",\n     \"data\": {\n      \"text/plain\": [\n       \"\\n\",\n       \"[[0.34404233]\\n\",\n       \" [0.3254302 ]]\\n\",\n       \"<NDArray 2x1 @gpu(0)>\"\n      ]\n     },\n     \"metadata\": {},\n     \"execution_count\": 13\n    }\n   ],\n   \"metadata\": {}\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"source\": [\n    \"The output is the similarity, if we wanted to train it on real data, we would need to minimize the Cosine loss, 1 - cosine_similarity.\"\n   ],\n   \"metadata\": {}\n  }\n ],\n \"metadata\": {\n  \"kernelspec\": {\n   \"display_name\": \"Python 3\",\n   \"language\": \"python\",\n   \"name\": \"python3\"\n  },\n  \"language_info\": {\n   \"codemirror_mode\": {\n    \"name\": \"ipython\",\n    \"version\": 3\n   },\n   \"file_extension\": \".py\",\n   \"mimetype\": \"text/x-python\",\n   \"name\": \"python\",\n   \"nbconvert_exporter\": \"python\",\n   \"pygments_lexer\": \"ipython3\",\n   \"version\": \"3.6.4\"\n  }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}"
  },
  {
    "path": "example/recommenders/matrix_fact.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport logging\nimport math\nimport random\n\nimport mxnet as mx\nfrom mxnet import gluon, autograd, nd\nimport numpy as np\n\nlogging.basicConfig(level=logging.DEBUG)\n\ndef evaluate_network(network, data_iterator, ctx):\n    loss_acc = 0.\n    l2 = gluon.loss.L2Loss()\n    for idx, (users, items, scores) in enumerate(data_iterator):\n        users_ = gluon.utils.split_and_load(users, ctx)\n        items_ = gluon.utils.split_and_load(items, ctx)\n        scores_ =gluon.utils.split_and_load(scores, ctx)\n        preds = [network(u, i) for u, i in zip(users_, items_)]\n        losses = [l2(p, s).asnumpy() for p, s in zip(preds, scores_)]         \n        loss_acc += sum(losses).mean()/len(ctx)\n    return loss_acc/(idx+1)\n\ndef train(network, train_data, test_data, epochs, learning_rate=0.01, optimizer='sgd', ctx=mx.gpu(0), num_epoch_lr=5, factor=0.2):\n\n    np.random.seed(123)  # Fix random seed for consistent demos\n    mx.np.random.seed(123)  # Fix random seed for consistent demos\n    random.seed(123)  # Fix random seed for consistent demos\n\n    schedule = mx.lr_scheduler.FactorScheduler(step=len(train_data)*len(ctx)*num_epoch_lr, factor=factor)\n\n    trainer = gluon.Trainer(network.collect_params(), optimizer,\n                            {'learning_rate':learning_rate, 'wd':0.0001, 'lr_scheduler':schedule})  \n                            #update_on_kvstore=False)\n\n    l2 = gluon.loss.L2Loss()\n\n    network.hybridize()\n    \n    losses_output = []\n    for e in range(epochs):\n        loss_acc = 0.\n        for idx, (users, items, scores) in enumerate(train_data):\n            \n            users_ = gluon.utils.split_and_load(users, ctx)\n            items_ = gluon.utils.split_and_load(items, ctx)\n            scores_ =gluon.utils.split_and_load(scores, ctx)\n\n            with autograd.record():\n                preds = [network(u, i) for u, i in zip(users_, items_)]\n                losses = [l2(p, s) for p, s in zip(preds, scores_)]\n\n            [l.backward() for l in losses]\n            loss_acc += sum([l.asnumpy() for l in losses]).mean()/len(ctx)\n            trainer.update(users.shape[0])\n\n        test_loss = evaluate_network(network, test_data, ctx)\n        train_loss = loss_acc/(idx+1)\n        print(\"Epoch [{}], Training RMSE {:.4f}, Test RMSE {:.4f}\".format(e, train_loss, test_loss))\n        losses_output.append((train_loss, test_loss))\n    return losses_output"
  },
  {
    "path": "example/recommenders/movielens_data.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"MovieLens data handling: download, parse, and expose as DataIter\n\"\"\"\n\nimport os\nimport mxnet as mx\nfrom mxnet import gluon\n\ndef load_mldataset(filename):\n    \"\"\"Not particularly fast code to parse the text file and load it into three NDArray's\n    and product an NDArrayIter\n    \"\"\"\n    user = []\n    item = []\n    score = []\n    with open(filename) as f:\n        for line in f:\n            tks = line.strip().split('\\t')\n            if len(tks) != 4:\n                continue\n            user.append(int(tks[0]))\n            item.append(int(tks[1]))\n            score.append(float(tks[2]))\n    user = mx.np.array(user)\n    item = mx.np.array(item)\n    score = mx.np.array(score)\n    return gluon.data.ArrayDataset(user, item, score)\n\ndef ensure_local_data(prefix):\n    if not os.path.exists(f\"{prefix}.zip\"):\n        print(f\"Downloading MovieLens data: {prefix}\")\n        # MovieLens 100k dataset from https://grouplens.org/datasets/movielens/\n        # This dataset is copy right to GroupLens Research Group at the University of Minnesota,\n        # and licensed under their usage license.\n        # For full text of the usage license, see http://files.grouplens.org/datasets/movielens/ml-100k-README.txt\n        os.system(f\"wget http://files.grouplens.org/datasets/movielens/{prefix}.zip\")\n        os.system(f\"unzip {prefix}.zip\")\n\n\ndef get_dataset(prefix='ml-100k'):\n    \"\"\"Returns a pair of NDArrayDataIter, one for train, one for test.\n    \"\"\"\n    ensure_local_data(prefix)\n    return (load_mldataset(f'./{prefix}/u1.base'),\n            load_mldataset(f'./{prefix}/u1.test'))\n\ndef max_id(fname):\n    mu = 0\n    mi = 0\n    for line in open(fname):\n        tks = line.strip().split('\\t')\n        if len(tks) != 4:\n            continue\n        mu = max(mu, int(tks[0]))\n        mi = max(mi, int(tks[1]))\n    return mu + 1, mi + 1\n"
  },
  {
    "path": "include/mxnet/api_registry.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file api_registry.h\n * \\brief This file contains utilities related to\n *  the MXNet's global function registry.\n */\n// Acknowledgement: This file originates from incubator-tvm\n#ifndef MXNET_API_REGISTRY_H_\n#define MXNET_API_REGISTRY_H_\n\n#include <string>\n#include <utility>\n#include \"runtime/registry.h\"\n\nnamespace mxnet {\n/*!\n * \\brief Register an API function globally.\n * It simply redirects to MXNET_REGISTER_GLOBAL\n *\n * \\code\n *   MXNET_REGISTER_API(MyPrint)\n *   .set_body([](MXNetArgs args, MXNetRetValue* rv) {\n *     // my code.\n *   });\n * \\endcode\n */\n#define MXNET_REGISTER_API(OpName) MXNET_REGISTER_GLOBAL(OpName)\n\n}  // namespace mxnet\n#endif  // MXNET_API_REGISTRY_H_\n"
  },
  {
    "path": "include/mxnet/base.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file base.h\n * \\brief configuration of MXNet as well as basic data structure.\n */\n#ifndef MXNET_BASE_H_\n#define MXNET_BASE_H_\n\n#include \"dmlc/base.h\"\n#include <string>\n#include \"dmlc/io.h\"\n#include \"dmlc/type_traits.h\"\n#include \"dmlc/parameter.h\"\n#include \"mshadow/tensor.h\"\n// nnvm headers for symbolic construction.\n#include \"nnvm/op.h\"\n#include \"nnvm/symbolic.h\"\n#include \"libinfo.h\"\n#include \"tuple.h\"\n\n/*!\n * \\brief define dllexport for Visual Studio\n */\n#ifdef _MSC_VER\n#ifdef MXNET_EXPORTS\n#define MXNET_API __declspec(dllexport)\n#else\n#define MXNET_API __declspec(dllimport)\n#endif\n#else\n#define MXNET_API\n#endif\n\n/*!\n * \\brief define prediction only\n */\n#ifndef MXNET_PREDICT_ONLY\n#define MXNET_PREDICT_ONLY 0\n#endif\n\n/*! \\brief major version */\n#define MXNET_MAJOR 2\n/*! \\brief minor version */\n#define MXNET_MINOR 0\n/*! \\brief patch version */\n#define MXNET_PATCH 0\n/*! \\brief mxnet version */\n#define MXNET_VERSION (MXNET_MAJOR * 10000 + MXNET_MINOR * 100 + MXNET_PATCH)\n/*! \\brief helper for making version number */\n#define MXNET_MAKE_VERSION(major, minor, patch) ((major)*10000 + (minor)*100 + patch)\n/*!\n * \\brief define function name as profiler message\n */\n#define PROFILER_MESSAGE_FUNCNAME (__FUNCTION__)\n\n/*! \\brief namespace of mxnet */\nnamespace mxnet {\n/*! \\brief mxnet cpu */\ntypedef mshadow::cpu cpu;\n/*! \\brief mxnet gpu */\ntypedef mshadow::gpu gpu;\n/*! \\brief index type usually use unsigned */\ntypedef mshadow::index_t index_t;\n/*! \\brief index type for blas library.*/\ntypedef mshadow::lapack_index_t lapack_index_t;\n/*! \\brief data type that will be used to store ndarray */\ntypedef mshadow::default_real_t real_t;\n/*! \\brief operator structure from NNVM */\nusing Op = nnvm::Op;\n\n/*! \\brief Context information about the execution environment */\nstruct Context {\n  /*! \\brief Type of device */\n  enum DeviceType {\n    kCPU       = cpu::kDevMask,\n    kGPU       = gpu::kDevMask,\n    kCPUPinned = 3,\n    kCPUShared = 5,\n  };\n  /*! \\brief the device type we run the op on */\n  DeviceType dev_type;\n  /*! \\brief device id we are going to run it on */\n  int32_t dev_id;\n  /*! \\brief default constructor */\n  Context() : dev_type(kCPU), dev_id(0) {}\n  /*!\n   * \\brief Get corresponding device mask\n   * \\return cpu::kDevMask or gpu::kDevMask\n   */\n  inline DeviceType dev_mask() const {\n    if (dev_type == kCPUPinned || dev_type == kCPUShared)\n      return kCPU;\n    return dev_type;\n  }\n  /*!\n   * \\brief Returns dev_id for kGPU and kCPUPinned, 0 otherwise\n   */\n  inline int real_dev_id() const {\n    if (dev_type == kCPUPinned || dev_type == kGPU)\n      return dev_id;\n    return 0;\n  }\n  /*!\n   * \\brief Comparator, used to enable Context as std::map key.\n   * \\param b another context to compare\n   * \\return compared result\n   */\n  inline bool operator<(const Context& b) const;\n  /*!\n   * \\brief check if current context equals another one\n   * \\param b another context to compare\n   * \\return whether dev mask and id are same\n   */\n  inline bool operator==(const Context& b) const {\n    return dev_type == b.dev_type && dev_id == b.dev_id;\n  }\n  /*!\n   * \\brief check if current context not equals another one\n   * \\param b another context to compare\n   * \\return whether they are not the same\n   */\n  inline bool operator!=(const Context& b) const {\n    return !(*this == b);\n  }\n  /*!\n   * \\brief save the content into binary stream\n   * \\param strm the output stream\n   */\n  inline void Save(dmlc::Stream* strm) const {\n    strm->Write(&dev_type, sizeof(dev_type));\n    strm->Write(&dev_id, sizeof(dev_id));\n  }\n  /*!\n   * \\brief load the content from binary stream\n   * \\param strm the output stream\n   * \\return whether the load is successful\n   */\n  inline bool Load(dmlc::Stream* strm) {\n    if (strm->Read(&dev_type, sizeof(dev_type)) != sizeof(dev_type))\n      return false;\n    if (strm->Read(&dev_id, sizeof(int32_t)) != sizeof(int32_t))\n      return false;\n    return true;\n  }\n  /*! \\brief the maximal device type */\n  static const int32_t kMaxDevType = 6;\n  /*! \\brief the maximal device index */\n  static const int32_t kMaxDevID = 16;\n  /*!\n   * \\brief Create a new context.\n   * \\param dev_type device type.\n   * \\param dev_id device id. -1 for current device.\n   */\n  inline static Context Create(DeviceType dev_type, int32_t dev_id = -1);\n  /*! \\return CPU Context */\n  inline static Context CPU(int32_t dev_id = 0);\n  /*!\n   * Create a GPU context.\n   * \\param dev_id the device id.\n   * \\return GPU Context. -1 for current GPU.\n   */\n  inline static Context GPU(int32_t dev_id = -1);\n  /*!\n   * Get the number of GPUs available.\n   * \\return The number of GPUs that are available.\n   */\n  inline static int32_t GetGPUCount();\n  /*!\n   * Is the cuda driver installed and visible to the system.\n   * \\return Whether the driver is present.\n   */\n  inline static bool GPUDriverPresent();\n  /*!\n   * Get the number of streams that a GPU Worker has available to operations.\n   * \\return The number of streams that are available.\n   */\n  inline static int32_t GetGPUStreamsPerWorker();\n  /*!\n   * \\brief get the free and total available memory on a GPU\n   * \\param dev the GPU number to query\n   * \\param free_mem pointer to the uint64_t holding free GPU memory\n   * \\param total_mem pointer to the uint64_t holding total GPU memory\n   * \\return No return value\n   */\n  inline static void GetGPUMemoryInformation(int dev, uint64_t* free, uint64_t* total);\n  /*!\n   * Create a pinned CPU context.\n   * \\param dev_id the device id for corresponding GPU.\n   * \\return Pinned CPU context. -1 for current GPU.\n   */\n  inline static Context CPUPinned(int32_t dev_id = -1);\n  /*!\n   * Create a CPU shared memory context.\n   * \\param dev_id dummy device id.\n   * \\return CPU shared memory context.\n   */\n  inline static Context CPUShared(int32_t dev_id = 0);\n  /*!\n   * Create a context from string of the format [cpu|gpu|cpu_pinned](n)\n   * \\param str the string pattern\n   * \\return Context\n   */\n  inline static Context FromString(const std::string& str);\n\n private:\n#if MXNET_USE_CUDA\n  static void CudaLibChecks();\n#endif\n#if MXNET_USE_CUDNN\n  static void CuDNNLibChecks();\n#endif\n};\n\n#if MXNET_USE_CUDA\n/*! \\brief Holds an auxiliary mshadow gpu stream that can be synced with a primary stream. */\nclass GPUAuxStream {\n public:\n  /*!\n   * \\brief constructor.\n   * \\param primary_stream gpu stream that is synced with the created auxiliary stream.\n   */\n  explicit GPUAuxStream(mshadow::Stream<gpu>* primary_stream)\n      : primary_stream_(primary_stream),\n        aux_stream_(primary_stream),\n        gpu_stream_sync_event_(nullptr) {\n    if (Context::GetGPUStreamsPerWorker() >= 2) {\n      // Create auxiliary stream on the same device with the same properties as the primary stream\n      bool primary_has_blas_handle =\n          primary_stream->blas_handle_ownership_ == mshadow::Stream<gpu>::OwnHandle;\n      bool primary_has_dnn_handle =\n          primary_stream->dnn_handle_ownership_ == mshadow::Stream<gpu>::OwnHandle;\n      aux_stream_ = mshadow::NewStream<gpu>(\n          primary_has_blas_handle, primary_has_dnn_handle, primary_stream->dev_id);\n      MSHADOW_CUDA_CALL(cudaEventCreateWithFlags(&gpu_stream_sync_event_, cudaEventDisableTiming));\n    }\n  }\n  /*! \\brief destructor */\n  ~GPUAuxStream() {\n    // If the aux_stream_ == primary_stream_, then we created no new streams to destroy.\n    if (aux_stream_ != primary_stream_) {\n      MSHADOW_CATCH_ERROR(mshadow::DeleteStream<gpu>(aux_stream_));\n      MSHADOW_CATCH_ERROR(cudaEventDestroy(gpu_stream_sync_event_));\n    }\n  }\n  /*!\n   * \\brief Makes future aux stream work wait on the completion of existing primary stream work.\n   */\n  void PreAuxStreamUseSync() {\n    // If the aux_stream_ == primary_stream_, then no synchronization is necessary.\n    if (aux_stream_ != primary_stream_)\n      StreamSync(primary_stream_, aux_stream_, gpu_stream_sync_event_);\n  }\n  /*!\n   * \\brief Makes future primary stream work wait on the completion of existing aux stream work.\n   */\n  void PostAuxStreamUseSync() {\n    // If the aux_stream_ == primary_stream_, then no synchronization is necessary.\n    if (aux_stream_ != primary_stream_)\n      StreamSync(aux_stream_, primary_stream_, gpu_stream_sync_event_);\n  }\n  /*! \\brief Getter for created auxiliary stream. */\n  mshadow::Stream<gpu>* GetStream() {\n    return aux_stream_;\n  }\n  /*!\n   * \\brief Make future work enqueued to `s2` wait on completion of current work enqueued to `s1`.\n   * \\param s1 stream with work that must be completed before future s2 work can begin.\n   * \\param s2 stream whose future work is made to wait on the completion of existing s1 work.\n   * \\param event used to pass s1 state to s2.\n   */\n  static void StreamSync(mshadow::Stream<gpu>* s1, mshadow::Stream<gpu>* s2, cudaEvent_t event) {\n    MSHADOW_CUDA_CALL(cudaEventRecord(event, s1->stream_));\n    MSHADOW_CUDA_CALL(cudaStreamWaitEvent(s2->stream_, event, 0));\n  }\n\n private:\n  mshadow::Stream<gpu>* primary_stream_;\n  mshadow::Stream<gpu>* aux_stream_;\n  cudaEvent_t gpu_stream_sync_event_;\n};\n\n/*!\n * \\brief Provides automatic coordination of an auxilary stream with a primary one.\n * This object, upon construction, prepares an aux stream for use by syncing it with enqueued\n * primary-stream work.  Object destruction will sync again so future primary-stream work\n * will wait on enqueued aux-stream work.  If MXNET_GPU_WORKER_NSTREAMS == 1, then this defaults\n * simply: the primary stream will equal the aux stream and the syncs will be executed as nops.\n * See ./src/operator/cudnn/cudnn_convolution-inl.h for a usage example.\n */\nclass SyncedGPUAuxStream {\n public:\n  /*!\n   * \\brief constructor.\n   * \\param gpu_aux_stream auxilary gpu stream that is managed by this RAII object.\n   */\n  explicit SyncedGPUAuxStream(GPUAuxStream* gpu_aux_stream) : gpu_aux_stream_(gpu_aux_stream) {\n    gpu_aux_stream_->PreAuxStreamUseSync();\n  }\n  /*! \\brief destructor */\n  ~SyncedGPUAuxStream() {\n    gpu_aux_stream_->PostAuxStreamUseSync();\n  }\n  /*! \\brief copy constructor deleted to prevent unexpected synchronizations. */\n  SyncedGPUAuxStream(const SyncedGPUAuxStream&) = delete;\n  /*! \\brief copy assignment operator deleted to prevent unexpected synchronizations. */\n  void operator=(const SyncedGPUAuxStream&) = delete;\n  /*! \\brief move constructor permitted as alternative to copying. */\n  SyncedGPUAuxStream(SyncedGPUAuxStream&&) = default;\n  /*! \\brief move assignment operator permitted as alternative to copy assignment. */\n  SyncedGPUAuxStream& operator=(SyncedGPUAuxStream&&) = default;\n  /*! \\brief Getter for underlying mshadow::Stream<gpu>. */\n  inline mshadow::Stream<gpu>* GetStream() const {\n    return gpu_aux_stream_->GetStream();\n  }\n\n private:\n  GPUAuxStream* gpu_aux_stream_;\n};\n#endif  // MXNET_USE_CUDA\n\n/*!\n * \\brief execution time context.\n *  The information needed in runtime for actual execution.\n */\nstruct RunContext {\n  /*! \\brief base Context */\n  Context ctx;\n  /*!\n   * \\brief the stream of the device, can be nullptr or Stream<gpu>* in GPU mode\n   */\n  void* stream;\n  /*!\n   * \\brief the auxiliary stream of the device, can be nullptr or Stream<gpu>* in GPU mode\n   */\n  void* aux_stream;\n  /*!\n   * \\brief pointer to the cuda event pool used by the dependency engine\n   */\n  void* event_pool = nullptr;\n  /*!\n   * \\brief get mshadow stream from Context\n   * \\return the mshadow stream\n   * \\tparam xpu the device type of the stream\n   */\n  template <typename xpu>\n  inline mshadow::Stream<xpu>* get_stream() const {\n    return static_cast<mshadow::Stream<xpu>*>(stream);\n  }\n#if MXNET_USE_CUDA\n  /*!\n   * \\brief get an RAII object that transparently handles the syncing of the auxiliary stream.\n   * \\return the aux stream auto-syncing object\n   */\n  inline SyncedGPUAuxStream get_gpu_aux_stream() const {\n    return SyncedGPUAuxStream(static_cast<GPUAuxStream*>(aux_stream));\n  }\n#endif\n  /*! \\brief get the base Context from RunContext */\n  inline const Context& get_ctx() const {\n    return ctx;\n  }\n};\n}  // namespace mxnet\n\n//! \\cond Doxygen_Suppress\nnamespace mxnet {\n// implementing Context\ninline bool Context::operator<(const Context& b) const {\n  if (dev_type == b.dev_type) {\n    return dev_id < b.dev_id;\n  } else {\n    return dev_type < b.dev_type;\n  }\n}\ninline Context Context::Create(DeviceType dev_type, int32_t dev_id) {\n  Context ctx;\n  ctx.dev_type = dev_type;\n  ctx.dev_id   = dev_id < 0 ? 0 : dev_id;\n  if (dev_type & kGPU) {\n#if MXNET_USE_CUDA\n    CudaLibChecks();\n#endif\n#if MXNET_USE_CUDNN\n    CuDNNLibChecks();\n#endif\n    if (dev_id < 0) {\n#if MXNET_USE_CUDA\n      CHECK_EQ(cudaGetDevice(&ctx.dev_id), cudaSuccess);\n#else\n      LOG(FATAL) << \"Please compile with CUDA enabled for cuda features\";\n#endif\n    }\n  }\n  return ctx;\n}\ninline Context Context::CPU(int32_t dev_id) {\n  return Create(kCPU, dev_id);\n}\n\ninline Context Context::CPUPinned(int32_t dev_id) {\n  return Create(kCPUPinned, dev_id);\n}\n\ninline Context Context::CPUShared(int32_t dev_id) {\n  return Create(kCPUShared, dev_id);\n}\n\ninline Context Context::GPU(int32_t dev_id) {\n  return Create(kGPU, dev_id);\n}\n\ninline bool Context::GPUDriverPresent() {\n#if MXNET_USE_CUDA\n  int cuda_driver_version = 0;\n  CHECK_EQ(cudaDriverGetVersion(&cuda_driver_version), cudaSuccess);\n  return cuda_driver_version > 0;\n#else\n  return false;\n#endif\n}\n\ninline int32_t Context::GetGPUCount() {\n#if MXNET_USE_CUDA\n  if (!GPUDriverPresent()) {\n    return 0;\n  }\n  int32_t count;\n  cudaError_t e = cudaGetDeviceCount(&count);\n  // TODO(junwu): Remove e == cudaErrorInsufficientDriver\n  // This is skipped for working around wheel build system with older CUDA driver.\n  if (e == cudaErrorNoDevice || e == cudaErrorInsufficientDriver) {\n    return 0;\n  }\n  CHECK_EQ(e, cudaSuccess) << \" CUDA: \" << cudaGetErrorString(e);\n  return count;\n#else\n  return 0;\n#endif\n}\n\ninline int32_t Context::GetGPUStreamsPerWorker() {\n  // The default number of streams available if the user has not set MXNET_GPU_WORKER_NSTREAMS.\n  const int32_t default_num_streams = 1;\n  // The get_aux_stream() interface can supply one additional stream beyond the standard one.\n  static int32_t num_streams =\n      dmlc::GetEnv(\"MXNET_GPU_WORKER_NSTREAMS\", default_num_streams) >= 2 ? 2 : 1;\n  return num_streams;\n}\n\ninline void Context::GetGPUMemoryInformation(int dev, uint64_t* free_mem, uint64_t* total_mem) {\n#if MXNET_USE_CUDA\n\n  size_t memF, memT;\n  cudaError_t e;\n\n  int curDevice;\n  e = cudaGetDevice(&curDevice);\n  CHECK_EQ(e, cudaSuccess) << \" CUDA: \" << cudaGetErrorString(e);\n\n  e = cudaSetDevice(dev);\n  CHECK_EQ(e, cudaSuccess) << \" CUDA: \" << cudaGetErrorString(e);\n\n  e = cudaMemGetInfo(&memF, &memT);\n  CHECK_EQ(e, cudaSuccess) << \" CUDA: \" << cudaGetErrorString(e);\n\n  e = cudaSetDevice(curDevice);\n  CHECK_EQ(e, cudaSuccess) << \" CUDA: \" << cudaGetErrorString(e);\n\n  *free_mem  = static_cast<uint64_t>(memF);\n  *total_mem = static_cast<uint64_t>(memT);\n\n#else\n  LOG(FATAL) << \"This call is only supported for MXNet built with CUDA support.\";\n#endif\n}\n\ninline Context Context::FromString(const std::string& str) {\n  Context ret;\n  try {\n    const std::string::size_type l = str.find('(');\n    CHECK_NE(l, std::string::npos);\n    const std::string::size_type r = str.find(')');\n    CHECK_EQ(r, str.length() - 1);\n\n    const std::string type = str.substr(0, l);\n    int id                 = std::stoi(str.substr(l + 1, r - l - 1));\n    if (type == \"cpu\") {\n      ret = CPU(id);\n    } else if (type == \"gpu\") {\n      ret = GPU(id);\n    } else if (type == \"cpu_pinned\") {\n      ret = CPUPinned(id);\n    } else if (type == \"cpu_shared\") {\n      ret = CPUShared(id);\n    } else {\n      LOG(FATAL) << \"Invalid context string \" << str;\n    }\n  } catch (...) {\n    LOG(FATAL) << \"Invalid context string \" << str;\n  }\n  return ret;\n}\n\ninline std::ostream& operator<<(std::ostream& out, const Context& ctx) {\n  if (ctx.dev_type == Context::kCPU) {\n    out << \"cpu(\";\n  } else if (ctx.dev_type == Context::kGPU) {\n    out << \"gpu(\";\n  } else if (ctx.dev_type == Context::kCPUPinned) {\n    out << \"cpu_pinned(\";\n  } else if (ctx.dev_type == Context::kCPUShared) {\n    out << \"cpu_shared(\";\n  } else {\n    out << \"unknown(\";\n  }\n  out << ctx.dev_id << \")\";\n  return out;\n}\n\n// describe op registration point\n#define STRINGIZE_DETAIL(x) #x\n#define STRINGIZE(x)        STRINGIZE_DETAIL(x)\n#define MXNET_DESCRIBE(...) describe(__VA_ARGS__ \"\\n\\nFrom:\" __FILE__ \":\" STRINGIZE(__LINE__))\n#define ADD_FILELINE        \"\\n\\nDefined in \" __FILE__ \":L\" STRINGIZE(__LINE__)\n\n#if MXNET_USE_ONEDNN == 1 || MXNET_USE_INTGEMM == 1\nconstexpr size_t kDNNLAlign = 64;\n#endif\n\n}  // namespace mxnet\n\nnamespace std {\ntemplate <>\nstruct hash<mxnet::Context> {\n  size_t operator()(const mxnet::Context& ctx) const {\n    size_t res = 0;\n    res        = dmlc::HashCombine(res, static_cast<size_t>(ctx.dev_type));\n    res        = dmlc::HashCombine(res, static_cast<size_t>(ctx.dev_id));\n    return res;\n  }\n};\n\n#if __cplusplus < 201402L && !defined(_MSC_VER)\ntemplate <typename T, typename... Args>\ninline std::unique_ptr<T> make_unique(Args&&... args) {\n  return std::unique_ptr<T>(new T(std::forward<Args>(args)...));\n}\n#endif\n}  // namespace std\n\n#include \"./tensor_blob.h\"\n//! \\endcond\n#endif  // MXNET_BASE_H_\n"
  },
  {
    "path": "include/mxnet/c_api.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file c_api.h\n * \\brief C API of mxnet\n */\n#ifndef MXNET_C_API_H_\n#define MXNET_C_API_H_\n\n/*! \\brief Inhibit C++ name-mangling for MXNet functions. */\n#ifdef __cplusplus\nextern \"C\" {\n#endif  // __cplusplus\n\n/*! \\brief Keep the default value in C++ */\n#ifdef __cplusplus\n#define DEFAULT(x) = x\n#else\n#define DEFAULT(x)\n#endif  // __cplusplus\n\n#include <stdint.h>\n\n#include <stdint.h>\n#include <stddef.h>\n#include <stdbool.h>\n\n/*! \\brief MXNET_DLL prefix for windows */\n#ifdef _WIN32\n#ifdef MXNET_EXPORTS\n#define MXNET_DLL __declspec(dllexport)\n#else\n#define MXNET_DLL __declspec(dllimport)\n#endif\n#else\n#define MXNET_DLL\n#endif\n\n#ifndef MXNET_BRANCH\n#define MXNET_BRANCH \"NotProvided\"\n#endif\n\n#ifndef MXNET_COMMIT_HASH\n#define MXNET_COMMIT_HASH \"NotProvided\"\n#endif\n\n/*! \\brief manually define unsigned int */\ntypedef uint32_t mx_uint;\n/*! \\brief manually define float */\ntypedef float mx_float;\n/*! \\brief data type to store dim size */\ntypedef int64_t dim_t;\n// all the handles are simply void *\n// will be casted internally to specific pointers types\n// these typedefs are mainly used for readablity reasons\n/*! \\brief handle to NDArray */\ntypedef void* NDArrayHandle;\n/*! \\brief handle to a mxnet narray function that changes NDArray */\ntypedef const void* FunctionHandle;\n/*! \\brief handle to a function that takes param and creates symbol */\ntypedef void* AtomicSymbolCreator;\n/*! \\brief handle to cached operator */\ntypedef void* CachedOpHandle;\n/*! \\brief handle to a symbol that can be bind as operator */\ntypedef void* SymbolHandle;\n/*! \\brief handle to a AtomicSymbol */\ntypedef void* AtomicSymbolHandle;\n/*! \\brief handle to an Executor */\ntypedef void* ExecutorHandle;\n/*! \\brief handle a dataiter creator */\ntypedef void* DataIterCreator;\n/*! \\brief handle to a DataIterator */\ntypedef void* DataIterHandle;\n/*! \\brief handle a dataset creator */\ntypedef void* DatasetCreator;\n/*! \\brief handle to a Dataset */\ntypedef void* DatasetHandle;\n/*! \\brief handle to a BatchifyFunction creator*/\ntypedef void* BatchifyFunctionCreator;\n/*! \\brief handle to a BatchifyFunction */\ntypedef void* BatchifyFunctionHandle;\n/*! \\brief handle to KVStore */\ntypedef void* KVStoreHandle;\n/*! \\brief handle to RecordIO */\ntypedef void* RecordIOHandle;\n/*! \\brief handle to MXRtc*/\ntypedef void* RtcHandle;\n/*! \\brief handle to rtc cuda module*/\ntypedef void* CudaModuleHandle;\n/*! \\brief handle to rtc cuda kernel*/\ntypedef void* CudaKernelHandle;\n/*! \\brief handle to a Profile object (domain, duration, counter, etc.) */\ntypedef void* ProfileHandle;\n/*! \\brief handle to DLManagedTensor*/\ntypedef void* DLManagedTensorHandle;\n/*! \\brief handle to Context */\ntypedef const void* ContextHandle;\n/*! \\brief handle to Engine FnProperty */\ntypedef const void* EngineFnPropertyHandle;\n/*! \\brief handle to Engine VarHandle */\ntypedef void* EngineVarHandle;\n\n/*! \\brief Engine asynchronous operation */\ntypedef void (*EngineAsyncFunc)(void*, void*, void*, void*);\n/*! \\brief Engine synchronous operation */\ntypedef void (*EngineSyncFunc)(void*, void*);\n/*! \\brief Callback to free the param for EngineAsyncFunc/EngineSyncFunc */\ntypedef void (*EngineFuncParamDeleter)(void*);\n/*! \\brief Monitor callback called at operator level for cached op */\ntypedef void (*CachedOpMonitorCallback)(const char*, const char*, NDArrayHandle);\n\nstruct NativeOpInfo {\n  void (*forward)(int, float**, int*, unsigned**, int*, void*);\n  void (*backward)(int, float**, int*, unsigned**, int*, void*);\n  void (*infer_shape)(int, int*, unsigned**, void*);\n  void (*list_outputs)(char***, void*);\n  void (*list_arguments)(char***, void*);\n  // all functions also pass a payload void* pointer\n  void* p_forward;\n  void* p_backward;\n  void* p_infer_shape;\n  void* p_list_outputs;\n  void* p_list_arguments;\n};\n\nstruct NDArrayOpInfo {\n  bool (*forward)(int, void**, int*, void*);\n  bool (*backward)(int, void**, int*, void*);\n  bool (*infer_shape)(int, int*, unsigned**, void*);\n  bool (*list_outputs)(char***, void*);\n  bool (*list_arguments)(char***, void*);\n  bool (*declare_backward_dependency)(const int*, const int*, const int*, int*, int**, void*);\n  // all functions also pass a payload void* pointer\n  void* p_forward;\n  void* p_backward;\n  void* p_infer_shape;\n  void* p_list_outputs;\n  void* p_list_arguments;\n  void* p_declare_backward_dependency;\n};\n\ntypedef int (*MXGenericCallback)(void);\n\nstruct MXCallbackList {\n  int num_callbacks;\n  int (**callbacks)(void);\n  void** contexts;\n};\n\nstruct LibFeature {\n  const char* name;\n  bool enabled;\n};\n\nenum CustomOpCallbacks { kCustomOpDelete, kCustomOpForward, kCustomOpBackward };\n\nenum CustomOpPropCallbacks {\n  kCustomOpPropDelete,\n  kCustomOpPropListArguments,\n  kCustomOpPropListOutputs,\n  kCustomOpPropListAuxiliaryStates,\n  kCustomOpPropInferShape,\n  kCustomOpPropDeclareBackwardDependency,\n  kCustomOpPropCreateOperator,\n  kCustomOpPropInferType,\n  kCustomOpPropInferStorageType,\n  kCustomOpPropBackwardInferStorageType\n};\n\ntypedef int (*CustomOpFBFunc)(int /*size*/,\n                              void** /*ptrs*/,\n                              int* /*tags*/,\n                              const int* /*reqs*/,\n                              const int /*is_train*/,\n                              void* /*state*/);\ntypedef int (*CustomOpDelFunc)(void* /*state*/);\ntypedef int (*CustomOpListFunc)(char*** /*args*/, void* /*state*/);\ntypedef int (*CustomOpInferShapeFunc)(int /*num_input*/,\n                                      int* /*ndims*/,\n                                      int** /*shapes*/,\n                                      void* /*state*/);\ntypedef int (*CustomOpInferStorageTypeFunc)(int /*num_input*/, int* /*stypes*/, void* /*state*/);\ntypedef int (*CustomOpBackwardInferStorageTypeFunc)(int /*num_input*/,\n                                                    int* /*stypes*/,\n                                                    int* /*tags*/,\n                                                    void* /*state*/);\ntypedef int (*CustomOpInferTypeFunc)(int /*num_input*/, int* /*types*/, void* /*state*/);\ntypedef int (*CustomOpBwdDepFunc)(const int* /*out_grad*/,\n                                  const int* /*in_data*/,\n                                  const int* /*out_data*/,\n                                  int* /*num_deps*/,\n                                  int** /*rdeps*/,\n                                  void* /*state*/);\ntypedef int (*CustomOpCreateFunc)(const char* /*ctx*/,\n                                  int /*num_inputs*/,\n                                  unsigned** /*shapes*/,\n                                  const int* /*ndims*/,\n                                  const int* /*dtypes*/,\n                                  struct MXCallbackList* /*ret*/,\n                                  void* /*state*/);\ntypedef int (*CustomOpPropCreator)(const char* /*op_type*/,\n                                   const int /*num_kwargs*/,\n                                   const char** /*keys*/,\n                                   const char** /*values*/,\n                                   struct MXCallbackList* /*ret*/);\n\nenum CustomFunctionCallbacks { kCustomFunctionBackward, kCustomFunctionDelete };\n\ntypedef int (*CustomFunctionBwdFunc)(int /*num_ograds*/,\n                                     int /*num_igrads*/,\n                                     void** /*ptrs*/,\n                                     const int* /*reqs*/,\n                                     const int /*is_train*/,\n                                     void* /*state*/);\ntypedef int (*CustomFunctionDelFunc)(void* /*state*/);\n\n/*!\n * \\brief return str message of the last error\n *  all function in this file will return 0 when success\n *  and -1 when an error occured,\n *  MXGetLastError can be called to retrieve the error\n *\n *  this function is threadsafe and can be called by different thread\n *  \\return error info\n */\nMXNET_DLL const char* MXGetLastError();\n\n//-------------------------------------\n// Part 0: Global State setups\n//-------------------------------------\n\n/*!\n * \\brief Load library dynamically\n * \\param path to the library .so file\n * \\param 0 for quiet, 1 for verbose\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXLoadLib(const char* path, unsigned verbose, void** lib);\n\n/*!\n * \\brief Get list of features supported on the runtime\n * \\param libFeature pointer to array of LibFeature\n * \\param size of the array\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXLibInfoFeatures(const struct LibFeature** libFeature, size_t* size);\n\n/*!\n * \\brief return whether the mxnet library is compiled with cxx11 abi\n * \\return whether mxnet is built with cxx11 abi\n */\nMXNET_DLL int MXLibInfoCompiledWithCXX11ABI(int* result);\n\n/*!\n * \\brief Seed all global random number generators in mxnet.\n * \\param seed the random number seed.\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXRandomSeed(int seed);\n\n/*!\n * \\brief Seed the global random number generator of the given device.\n * \\param seed the random number seed.\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXRandomSeedContext(int seed, int dev_type, int dev_id);\n\n/*!\n * \\brief Change floating-point calculations when dealing with denormalized values.\n * Currently this option is only supported in CPU backend.\n * Flushing denormalized values to zero is enabled by default.\n *\n * \\param value state of flush-to-zero and denormals-are-zero to set.\n * \\param prev_state state of flush-to-zero and denormals-are-zero before setting new state.\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXSetFlushDenorms(bool value, bool* prev_state);\n\n/*!\n * \\brief Notify the engine about a shutdown,\n *  This can help engine to print less messages into display.\n *\n *  User do not have to call this function.\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXNotifyShutdown();\n\n/*!\n * \\brief Set up configuration of profiler for the process passed as profile_process in keys\n * \\param num_params Number of parameters\n * \\param keys array of parameter keys\n * \\param vals array of parameter values\n * \\param kvstoreHandle handle to kvstore\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXSetProcessProfilerConfig(int num_params,\n                                         const char* const* keys,\n                                         const char* const* vals,\n                                         KVStoreHandle kvstoreHandle);\n\n/*!\n * \\brief Set up configuration of profiler for worker/current process\n * \\param num_params Number of parameters\n * \\param keys array of parameter keys\n * \\param vals array of parameter values\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXSetProfilerConfig(int num_params, const char* const* keys, const char* const* vals);\n\n/*!\n * \\brief Set up state of profiler for either worker or server process\n * \\param state indicate the working state of profiler,\n *  profiler not running when state == 0,\n *  profiler running when state == 1\n * \\param profile_process an int,\n * when 0 command is for worker/current process,\n * when 1 command is for server process\n * \\param kvstoreHandle handle to kvstore, needed for server process profiling\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXSetProcessProfilerState(int state,\n                                        int profile_process,\n                                        KVStoreHandle kvStoreHandle);\n\n/*!\n * \\brief Set up state of profiler for current process\n * \\param state indicate the working state of profiler,\n *  profiler not running when state == 0,\n *  profiler running when state == 1\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXSetProfilerState(int state);\n\n/*!\n * \\brief Set the scope of profiler for current process\n * \\param scope indicate the working scope of profiler\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXSetProfilerScope(const char* scope);\n\n/*!\n * \\brief Save profile and stop profiler\n * \\param finished true if stat output should stop after this point\n * \\param profile_process an int,\n * when 0 command is for worker/current process,\n * when 1 command is for server process\n * \\param kvstoreHandle handle to kvstore\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXDumpProcessProfile(int finished, int profile_process, KVStoreHandle kvStoreHandle);\n\n/*!\n * \\brief Save profile and stop profiler for worker/current process\n * \\param finished true if stat output should stop after this point\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXDumpProfile(int finished);\n\n/*!\n * \\brief Print sorted aggregate stats to the a string\n *        How aggregate stats are stored will not change\n * \\param out_str will receive a pointer to the output string\n * \\param reset clear the aggregate stats after printing\n * \\param format whether to return in tabular or json format\n * \\param sort_by sort by total, avg, min, max, or count\n * \\param ascending whether to sort ascendingly\n * \\return 0 when success, -1 when failure happens.\n * \\note\n */\nMXNET_DLL int MXAggregateProfileStatsPrint(const char** out_str,\n                                           int reset,\n                                           int format,\n                                           int sort_by,\n                                           int ascending);\n\n/*!\n * \\brief Pause profiler tuning collection\n * \\param paused If nonzero, profiling pauses. Otherwise, profiling resumes/continues\n * \\param profile_process integer which denotes whether to process worker or server process\n * \\param kvstoreHandle handle to kvstore\n * \\return 0 when success, -1 when failure happens.\n * \\note pausing and resuming is global and not recursive\n */\nMXNET_DLL int MXProcessProfilePause(int paused, int profile_process, KVStoreHandle kvStoreHandle);\n\n/*!\n * \\brief Pause profiler tuning collection for worker/current process\n * \\param paused If nonzero, profiling pauses. Otherwise, profiling resumes/continues\n * \\return 0 when success, -1 when failure happens.\n * \\note pausing and resuming is global and not recursive\n */\nMXNET_DLL int MXProfilePause(int paused);\n\n/*!\n * \\brief Create profiling domain\n * \\param domain String representing the domain name to create\n * \\param out Return domain object\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXProfileCreateDomain(const char* domain, ProfileHandle* out);\n\n/*!\n * \\brief Create profile task\n * \\param name Name of the task\n * \\param domain Domain of the task\n * \\param out Output handle\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXProfileCreateTask(ProfileHandle domain, const char* task_name, ProfileHandle* out);\n\n/*!\n * \\brief Create profile frame\n * \\param name Name of the frame\n * \\param domain Domain of the frame\n * \\param out Output handle\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXProfileCreateFrame(ProfileHandle domain,\n                                   const char* frame_name,\n                                   ProfileHandle* out);\n\n/*!\n * \\brief Create profile event\n * \\param name Name of the event\n * \\param out Output handle\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXProfileCreateEvent(const char* event_name, ProfileHandle* out);\n\n/*!\n * \\brief Create profile counter\n * \\param name Name of the counter\n * \\param domain Domain of the counter\n * \\param out Output handle\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXProfileCreateCounter(ProfileHandle domain,\n                                     const char* counter_name,\n                                     ProfileHandle* out);\n\n/*!\n * \\brief Destroy a frame\n * \\param frame_handle Handle to frame to destroy\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXProfileDestroyHandle(ProfileHandle frame_handle);\n\n/*!\n * \\brief Start timing the duration of a profile duration object such as an event, task or frame\n * \\param duration_handle handle to the duration object\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXProfileDurationStart(ProfileHandle duration_handle);\n\n/*!\n * \\brief Stop timing the duration of a profile duration object such as an event, task or frame\n * \\param duration_handle handle to the duration object\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXProfileDurationStop(ProfileHandle duration_handle);\n\n/*!\n * \\brief Set a counter, given its handle\n * \\param counter_handle Handle to counter to set\n * \\param value Value to set the counter to (64-bit unsigned integer)\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXProfileSetCounter(ProfileHandle counter_handle, uint64_t value);\n\n/*!\n * \\brief Adjust a counter by the given amount, given its handle\n * \\param counter_handle Handle to counter to adjust\n * \\param value Value to adjust the counter by (64-bit signed integer)\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXProfileAdjustCounter(ProfileHandle counter_handle, int64_t value);\n\n/*!\n * \\brief Mark a single instant in time\n * \\param domain Domain of the marker\n * \\param instant_marker_name Name of the marker\n * \\param scope Scope of marker ('global', 'process', 'thread', 'task', 'marker')\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXProfileSetMarker(ProfileHandle domain,\n                                 const char* instant_marker_name,\n                                 const char* scope);\n\n/*!\n * \\brief Set the number of OMP threads to use\n * \\param thread_num Number of OMP threads desired\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXSetNumOMPThreads(int thread_num);\n\n/*!\n * \\brief set bulk execution limit\n * \\param bulk_size new bulk_size\n * \\param prev_bulk_size previous bulk_size\n */\nMXNET_DLL int MXEngineSetBulkSize(int bulk_size, int* prev_bulk_size);\n\n/*!\n * \\brief Get the number of GPUs.\n * \\param pointer to int that will hold the number of GPUs available.\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXGetGPUCount(int* out);\n\n/*!\n * \\brief get the free and total available memory on a GPU\n *  Note: Deprecated, use MXGetGPUMemoryInformation64 instead.\n * \\param dev the GPU number to query\n * \\param free_mem pointer to the integer holding free GPU memory\n * \\param total_mem pointer to the integer holding total GPU memory\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXGetGPUMemoryInformation(int dev, int* free_mem, int* total_mem);\n\n/*!\n * \\brief get the free and total available memory on a GPU\n * \\param dev the GPU number to query\n * \\param free_mem pointer to the uint64_t holding free GPU memory\n * \\param total_mem pointer to the uint64_t holding total GPU memory\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXGetGPUMemoryInformation64(int dev, uint64_t* free_mem, uint64_t* total_mem);\n\n/*!\n * \\brief get the MXNet library version as an integer\n * \\param pointer to the integer holding the version number\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXGetVersion(int* out);\n\n/*!\n * \\brief get the MXNet library branch at build time, usually provided by cmake\n * \\param pointer to the string holding the branch name\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXGetBranch(const char** out);\n\n/*!\n * \\brief get the MXNet library commit hash at build time, usually provided by cmake\n * \\param pointer to the string holding the commit hash\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXGetCommitHash(const char** out);\n\n/*!\n * \\brief Load TVM operator from the binary library\n * \\param libpath TVM operators lib file\n * \\return 0 when success, -1 when failure happens\n */\n#if MXNET_USE_TVM_OP\nMXNET_DLL int MXLoadTVMOp(const char* libpath);\n\nstruct OtherOptionEntity {\n  int val;\n};\n\nstruct OtherOptionSpace {\n  OtherOptionEntity* entities;\n  int entities_size;\n};\n\nstruct ConfigSpace {\n  int entity_map_size;\n  char** entity_map_key;\n  OtherOptionEntity* entity_map_val;\n  int space_map_size;\n  char** space_map_key;\n  OtherOptionSpace* space_map_val;\n};\n\ntypedef struct ConfigSpaces {\n  int spaces_size;\n  char** spaces_key;\n  ConfigSpace* spaces_val;\n} ConfigSpaces;\n\nMXNET_DLL int MXLoadTVMConfig(ConfigSpaces config);\n#endif  // MXNET_USE_TVM_OP\n\n//-------------------------------------\n// Part 1: NDArray creation and deletion\n//-------------------------------------\n/*!\n * \\brief create a NDArray handle that is not initialized\n *  can be used to pass in as mutate variables\n *  to hold the result of NDArray\n * \\param out the returning handle\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayCreateNone(NDArrayHandle* out);\n\n/*!\n * \\brief create a NDArray with specified shape and data type\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=0 (by default)\n * \\param shape the pointer to the shape\n * \\param ndim the dimension of the shape\n * \\param dev_type device type, specify device we want to take\n * \\param dev_id the device id of the specific device\n * \\param delay_alloc whether to delay allocation until\n *    the narray is first mutated\n * \\param dtype data type of created array\n * \\param out the returning handle\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayCreate(const uint32_t* shape,\n                              uint32_t ndim,\n                              int dev_type,\n                              int dev_id,\n                              int delay_alloc,\n                              int dtype,\n                              NDArrayHandle* out);\n#define MXNDArrayCreateEx MXNDArrayCreate  // backward compatibility for external deps\n\n/*!\n * \\brief create a NDArray with specified shape and data type\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=1 (not default) i.e. Large Tensor Support\n * \\param shape the pointer to int64_t shape\n * \\param ndim the dimension of the shape\n * \\param dev_type device type, specify device we want to take\n * \\param dev_id the device id of the specific device\n * \\param delay_alloc whether to delay allocation until\n *    the narray is first mutated\n * \\param dtype data type of created array\n * \\param out the returning handle\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayCreate64(const int64_t* shape,\n                                int ndim,\n                                int dev_type,\n                                int dev_id,\n                                int delay_alloc,\n                                int dtype,\n                                NDArrayHandle* out);\n\n/*!\n * \\brief create an empty sparse NDArray with specified shape and data type\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=0 (by default)\n * \\param storage_type the storage type of the ndarray\n * \\param shape the pointer to the shape\n * \\param ndim the dimension of the shape\n * \\param dev_type device type, specify device we want to take\n * \\param dev_id the device id of the specific device\n * \\param delay_alloc whether to delay allocation until\n *        the narray is first mutated\n * \\param dtype data type of created array\n * \\param num_aux the number of aux data to support this ndarray\n * \\param aux_type data type of the aux data for the created array\n * \\param aux_ndims the dimension of the shapes of aux data\n * \\param aux_shape the shapes of aux data\n * \\param out the returning handle\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayCreateSparseEx(int storage_type,\n                                      const uint32_t* shape,\n                                      uint32_t ndim,\n                                      int dev_type,\n                                      int dev_id,\n                                      int delay_alloc,\n                                      int dtype,\n                                      uint32_t num_aux,\n                                      int* aux_type,\n                                      uint32_t* aux_ndims,\n                                      const uint32_t* aux_shape,\n                                      NDArrayHandle* out);\n\n/*!\n * \\brief create an empty sparse NDArray with specified shape and data type\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=1 (not default) i.e. Large Tensor Support\n * \\param storage_type the storage type of the ndarray\n * \\param shape the pointer to the shape\n * \\param ndim the dimension of the shape\n * \\param dev_type device type, specify device we want to take\n * \\param dev_id the device id of the specific device\n * \\param delay_alloc whether to delay allocation until\n *        the narray is first mutated\n * \\param dtype data type of created array\n * \\param num_aux the number of aux data to support this ndarray\n * \\param aux_type data type of the aux data for the created array\n * \\param aux_ndims the dimension of the shapes of aux data\n * \\param aux_shape the shapes of aux data\n * \\param out the returning handle\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayCreateSparseEx64(int storage_type,\n                                        const int64_t* shape,\n                                        int ndim,\n                                        int dev_type,\n                                        int dev_id,\n                                        int delay_alloc,\n                                        int dtype,\n                                        uint32_t num_aux,\n                                        int* aux_type,\n                                        int* aux_ndims,\n                                        const int64_t* aux_shape,\n                                        NDArrayHandle* out);\n\n/*!\n * \\brief create a NDArray handle that is loaded from raw bytes.\n * \\param buf the head of the raw bytes\n * \\param size size of the raw bytes\n * \\param out the returning handle\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayLoadFromRawBytes(const void* buf, size_t size, NDArrayHandle* out);\n/*!\n * \\brief save the NDArray into raw bytes.\n * \\param handle the NDArray handle\n * \\param out_size size of the raw bytes\n * \\param out_buf the head of returning memory bytes.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArraySaveRawBytes(NDArrayHandle handle, size_t* out_size, const char** out_buf);\n/*!\n * \\brief Save list of narray into the file.\n * \\param fname name of the file.\n * \\param num_args number of arguments to save.\n * \\param args the array of NDArrayHandles to be saved.\n * \\param keys the name of the NDArray, optional, can be NULL\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayLegacySave(const char* fname,\n                                  uint32_t num_args,\n                                  NDArrayHandle* args,\n                                  const char** keys);\n/*!\n * \\brief Save list of narray into the file.\n * \\param fname name of the file.\n * \\param num_args number of arguments to save.\n * \\param args the array of NDArrayHandles to be saved.\n * \\param keys the name of the NDArray, optional, can be NULL\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArraySave(const char* fname,\n                            uint32_t num_args,\n                            NDArrayHandle* args,\n                            const char** keys);\n/*!\n * \\brief Load list of narray from the file.\n * \\param fname name of the file.\n * \\param out_size number of narray loaded.\n * \\param out_arr head of the returning narray handles.\n * \\param out_name_size size of output name arrray.\n * \\param out_names the names of returning NDArrays, can be NULL\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayLoad(const char* fname,\n                            uint32_t* out_size,\n                            NDArrayHandle** out_arr,\n                            uint32_t* out_name_size,\n                            const char*** out_names);\n\n/*!\n * \\brief Load list / dictionary of narrays from file content loaded into memory.\n * This will load a list of ndarrays in a similar\n * manner to MXNDArrayLoad, however, it loads from\n * buffer containing the contents of a file, rather than\n * from a specified file.\n * \\param ndarray_buffer pointer to the start of the ndarray file content\n * \\param size size of the file\n * \\param out_size number of narray loaded.\n * \\param out_arr head of the returning narray handles.\n * \\param out_name_size size of output name arrray.\n * \\param out_names the names of returning NDArrays, can be NULL\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayLoadFromBuffer(const void* ndarray_buffer,\n                                      size_t size,\n                                      uint32_t* out_size,\n                                      NDArrayHandle** out_arr,\n                                      uint32_t* out_name_size,\n                                      const char*** out_names);\n\n/*!\n * \\brief Perform a synchronize copy from a contiguous CPU memory region.\n *\n *  This function will call WaitToWrite before the copy is performed.\n *  This is useful to copy data from existing memory region that are\n *  not wrapped by NDArray(thus dependency not being tracked).\n *\n * \\param handle the NDArray handle\n * \\param data the data source to copy from.\n * \\param size the memory size we want to copy from.\n */\nMXNET_DLL int MXNDArraySyncCopyFromCPU(NDArrayHandle handle, const void* data, size_t size);\n/*!\n * \\brief Perform a synchronize copyto a contiguous CPU memory region.\n *\n *  This function will call WaitToRead before the copy is performed.\n *  This is useful to copy data from existing memory region that are\n *  not wrapped by NDArray(thus dependency not being tracked).\n *\n * \\param handle the NDArray handle\n * \\param data the data source to copy into.\n * \\param size the memory size we want to copy into.\n */\nMXNET_DLL int MXNDArraySyncCopyToCPU(NDArrayHandle handle, void* data, size_t size);\n\n/*!\n * \\brief Copy src.data() to dst.data() if i = -1, else dst.aux_data(i) if i >= 0\n * This function blocks. Do not use it in performance critical code.\n * \\param handle_dst handle of a dst ndarray whose data/aux_data has been allocated\n * \\param handle_src handle of a src ndarray which has default storage type\n * \\param i dst data blob indicator\n */\nMXNET_DLL int MXNDArraySyncCopyFromNDArray(NDArrayHandle handle_dst,\n                                           const NDArrayHandle handle_src,\n                                           const int i);\n\n/*!\n * \\brief check whether the NDArray format is valid\n * \\param full_check if `True`, rigorous check, O(N) operations\n *    Otherwise basic check, O(1) operations\n */\nMXNET_DLL int MXNDArraySyncCheckFormat(NDArrayHandle handle, const bool full_check);\n\n/*!\n * \\brief Wait until all the pending writes with respect NDArray are finished.\n *  Always call this before read data out synchronizely.\n * \\param handle the NDArray handle\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayWaitToRead(NDArrayHandle handle);\n\n/*!\n * \\brief Wait until all the pending read/write with respect NDArray are finished.\n *  Always call this before write data into NDArray synchronizely.\n * \\param handle the NDArray handle\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayWaitToWrite(NDArrayHandle handle);\n\n/*!\n * \\brief wait until all delayed operations in\n *   the system is completed\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayWaitAll();\n\n/*!\n * \\brief free the narray handle\n * \\param handle the handle to be freed\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayFree(NDArrayHandle handle);\n\n/*!\n * \\brief Slice the NDArray along axis 0.\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=0 (by default)\n * \\param handle the handle to the NDArray\n * \\param slice_begin The beginning index of slice\n * \\param slice_end The ending index of slice\n * \\param out The NDArrayHandle of sliced NDArray\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArraySlice(NDArrayHandle handle,\n                             uint32_t slice_begin,\n                             uint32_t slice_end,\n                             NDArrayHandle* out);\n\n/*!\n * \\brief Slice the NDArray along axis 0.\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=1 (not default) i.e. Large Tensor Support\n * \\param handle the handle to the NDArray\n * \\param slice_begin The beginning index of slice\n * \\param slice_end The ending index of slice\n * \\param out The NDArrayHandle of sliced NDArray\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArraySlice64(NDArrayHandle handle,\n                               int64_t slice_begin,\n                               int64_t slice_end,\n                               NDArrayHandle* out);\n\n/*!\n * \\brief Index the NDArray along axis 0.\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=0 (by default)\n * \\param handle the handle to the NDArray\n * \\param idx the index\n * \\param out The NDArrayHandle of output NDArray\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayAt(NDArrayHandle handle, uint32_t idx, NDArrayHandle* out);\n\n/*!\n * \\brief Index the NDArray along axis 0.\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=1 (not default) i.e. Large Tensor Support\n * \\param handle the handle to the NDArray\n * \\param idx the index\n * \\param out The NDArrayHandle of output NDArray\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayAt64(NDArrayHandle handle, int64_t idx, NDArrayHandle* out);\n\n/*!\n * \\brief get the storage type of the array\n */\nMXNET_DLL int MXNDArrayGetStorageType(NDArrayHandle handle, int* out_storage_type);\n\n/*!\n * \\brief Reshape the NDArray.\n * \\param handle the handle to the narray\n * \\param ndim number of dimensions of new shape\n * \\param dims new shape\n * \\param out the NDArrayHandle of reshaped NDArray\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayReshape(NDArrayHandle handle, int ndim, int* dims, NDArrayHandle* out);\n\n/*!\n * \\brief Reshape the NDArray.\n * \\param handle the handle to the narray\n * \\param ndim number of dimensions of new shape\n * \\param dims new shape\n * \\param out the NDArrayHandle of reshaped NDArray\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayReshape64(NDArrayHandle handle,\n                                 int ndim,\n                                 dim_t* dims,\n                                 bool reverse,\n                                 NDArrayHandle* out);\n\n/*!\n * \\brief get the shape of the array\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=0 (by default)\n * \\param handle the handle to the narray\n * \\param out_dim the output dimension\n * \\param out_pdata pointer holder to get data pointer of the shape\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayGetShape(NDArrayHandle handle, int* out_dim, const int** out_pdata);\n\n/*!\n * \\brief get the shape of the array\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=1 (not default) i.e. Large Tensor Support\n * \\param handle the handle to the narray\n * \\param out_dim the output dimension\n * \\param out_pdata pointer holder to get data pointer of the shape\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayGetShape64(NDArrayHandle handle, int* out_dim, const int64_t** out_pdata);\n\n/*!\n * \\brief get the content of the data in NDArray\n * \\param handle the handle to the ndarray\n * \\param out_pdata pointer holder to get pointer of data\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayGetData(NDArrayHandle handle, void** out_pdata);\n/*!\n * \\brief Create a reference view of NDArray that\n *  represents as DLManagedTensor\n *  Notice: MXNet uses asynchronous execution. Please call MXNDArrayWaitToRead or\n *          MXNDArrayWaitToWrite before calling MXNDArrayToDLPack.\n * \\param handle the handle to the ndarray\n * \\param out_dlpack pointer holder to get pointer of DLManagedTensor\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayToDLPack(NDArrayHandle handle, DLManagedTensorHandle* out_dlpack);\n\n/*!\n * \\brief Create a NDArray backed by a dlpack tensor.\n *\n * This allows us to create a NDArray using the memory\n * allocated by an external deep learning framework\n * that is DLPack compatible.\n *\n * The memory is retained until the NDArray went out of scope.\n *\n * \\param dlpack the pointer of the input DLManagedTensor\n * \\param transient_handle whether the handle will be destructed before calling the deleter\n * \\param out_handle pointer holder to get pointer of NDArray\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayFromDLPack(DLManagedTensorHandle dlpack,\n                                  const bool transient_handle,\n                                  NDArrayHandle* out_handle);\n\n/*!\n * \\brief Delete a dlpack tensor\n * \\param dlpack the pointer of the input DLManagedTensor\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayCallDLPackDeleter(DLManagedTensorHandle dlpack);\n\n/*!\n * \\brief get the type of the data in NDArray\n * \\param handle the handle to the narray\n * \\param out_dtype pointer holder to get type of data\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayGetDType(NDArrayHandle handle, int* out_dtype);\n\n/*!\n * \\brief get the type of the ith aux data in NDArray\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=0 (by default)\n * \\param handle the handle to the narray\n * \\param i the index of the aux data\n * \\param out_type pointer holder to get type of aux data\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayGetAuxType(NDArrayHandle handle, uint32_t i, int* out_type);\n\n/*!\n * \\brief get the type of the ith aux data in NDArray\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=1 (not default) i.e. Large Tensor Support\n * \\param handle the handle to the narray\n * \\param i the index of the aux data\n * \\param out_type pointer holder to get type of aux data\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayGetAuxType64(NDArrayHandle handle, int64_t i, int* out_type);\n\n/*!\n * \\brief Get a deep copy of the ith aux data blob\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=0 (by default)\n * in the form of an NDArray of default storage type.\n * This function blocks. Do not use it in performance critical code.\n */\nMXNET_DLL int MXNDArrayGetAuxNDArray(NDArrayHandle handle, uint32_t i, NDArrayHandle* out);\n\n/*!\n * \\brief Get a deep copy of the ith aux data blob\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=1 (not default) i.e. Large Tensor Support\n * in the form of an NDArray of default storage type.\n * This function blocks. Do not use it in performance critical code.\n */\nMXNET_DLL int MXNDArrayGetAuxNDArray64(NDArrayHandle handle, int64_t i, NDArrayHandle* out);\n\n/*!\n * \\brief Get a deep copy of the data blob\n * in the form of an NDArray of default storage type.\n * This function blocks. Do not use it in performance critical code.\n */\nMXNET_DLL int MXNDArrayGetDataNDArray(NDArrayHandle handle, NDArrayHandle* out);\n/*!\n * \\brief get the context of the NDArray\n * \\param handle the handle to the narray\n * \\param out_dev_type the output device type\n * \\param out_dev_id the output device id\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayGetContext(NDArrayHandle handle, int* out_dev_type, int* out_dev_id);\n/*!\n * \\brief return gradient buffer attached to this NDArray\n * \\param handle NDArray handle\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayGetGrad(NDArrayHandle handle, NDArrayHandle* out);\n/*!\n * \\brief detach and ndarray from computation graph by clearing entry_\n * \\param handle NDArray handle\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayDetach(NDArrayHandle handle, NDArrayHandle* out);\n/*!\n * \\brief set the flag for gradient array state.\n * \\param handle NDArray handle\n * \\param state the new state.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArraySetGradState(NDArrayHandle handle, int state);\n/*!\n * \\brief set the flag for gradient array state.\n * \\param handle NDArray handle\n * \\param state the new state.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayGetGradState(NDArrayHandle handle, int* out);\n//--------------------------------\n// Part 2: functions on NDArray\n//--------------------------------\n/*!\n * \\brief list all the available functions handles\n *   most user can use it to list all the needed functions\n * \\param out_size the size of returned array\n * \\param out_array the output function array\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXListFunctions(uint32_t* out_size, FunctionHandle** out_array);\n\n/*!\n * \\brief get the function handle by name\n * \\param name the name of the function\n * \\param out the corresponding function handle\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXGetFunction(const char* name, FunctionHandle* out);\n/*!\n * \\brief Get the information of the function handle.\n * \\param fun The function handle.\n * \\param name The returned name of the function.\n * \\param description The returned description of the function.\n * \\param num_args Number of arguments.\n * \\param arg_names Name of the arguments.\n * \\param arg_type_infos Type information about the arguments.\n * \\param arg_descriptions Description information about the arguments.\n * \\param return_type Return type of the function.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXFuncGetInfo(FunctionHandle fun,\n                            const char** name,\n                            const char** description,\n                            uint32_t* num_args,\n                            const char*** arg_names,\n                            const char*** arg_type_infos,\n                            const char*** arg_descriptions,\n                            const char** return_type DEFAULT(NULL));\n/*!\n * \\brief get the argument requirements of the function\n * \\param fun input function handle\n * \\param num_use_vars how many NDArrays to be passed in as used_vars\n * \\param num_scalars scalar variable is needed\n * \\param num_mutate_vars how many NDArrays to be passed in as mutate_vars\n * \\param type_mask the type mask of this function\n * \\return 0 when success, -1 when failure happens\n * \\sa MXFuncInvoke\n */\nMXNET_DLL int MXFuncDescribe(FunctionHandle fun,\n                             uint32_t* num_use_vars,\n                             uint32_t* num_scalars,\n                             uint32_t* num_mutate_vars,\n                             int* type_mask);\n/*!\n * \\brief invoke a function, the array size of passed in arguments\n *   must match the values in the\n * \\param fun the function\n * \\param use_vars the normal arguments passed to function\n * \\param scalar_args the scalar qarguments\n * \\param mutate_vars the mutate arguments\n * \\param num_params number of keyword parameters\n * \\param param_keys keys for keyword parameters\n * \\param param_vals values for keyword parameters\n * \\return 0 when success, -1 when failure happens\n * \\sa MXFuncDescribeArgs\n */\nMXNET_DLL int MXFuncInvoke(FunctionHandle fun,\n                           NDArrayHandle* use_vars,\n                           float* scalar_args,\n                           NDArrayHandle* mutate_vars,\n                           int num_params,\n                           char** param_keys,\n                           char** param_vals);\n/*!\n * \\brief invoke a nnvm op and imperative function\n * \\param creator the op\n * \\param num_inputs number of input NDArrays\n * \\param inputs input NDArrays\n * \\param num_outputs number of output NDArrays\n * \\param outputs output NDArrays\n * \\param num_params number of keyword parameters\n * \\param param_keys keys for keyword parameters\n * \\param param_vals values for keyword parameters\n * \\param out_stypes output ndarrays' stypes\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXImperativeInvoke(AtomicSymbolCreator creator,\n                                 int num_inputs,\n                                 NDArrayHandle* inputs,\n                                 int* num_outputs,\n                                 NDArrayHandle** outputs,\n                                 int num_params,\n                                 const char** param_keys,\n                                 const char** param_vals,\n                                 const int** out_stypes);\n/*!\n * \\brief set whether to record operator for autograd\n * \\param is_recording 1 when recording, 0 when not recording.\n * \\param prev returns the previous status before this set.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXAutogradSetIsRecording(int is_recording, int* prev);\n/*!\n * \\brief set whether to record operator for autograd\n * \\param is_training 1 when training, 0 when testing\n * \\param prev returns the previous status before this set.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXAutogradSetIsTraining(int is_training, int* prev);\n/*!\n * \\brief get whether autograd recording is on\n * \\param curr returns the current status.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXAutogradIsRecording(bool* curr);\n/*!\n * \\brief get whether training mode is on\n * \\param curr returns the current status.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXAutogradIsTraining(bool* curr);\n/*!\n * \\brief set what optimization constraints to apply\n * \\param constraints state composed of OptConstraint flags.\n * \\param prev returns the previous status before this set.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSetOptimizationConstraints(unsigned int constraints, unsigned int* prev);\n/*!\n * \\brief get current optimization constraints\n * \\param curr returns the current status\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXGetOptimizationConstraints(unsigned int* curr);\n/*!\n * \\brief get whether numpy compatibility is on\n * \\param curr returns the current status\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXIsNumpyShape(int* curr);\n/*!\n * \\brief set numpy compatibility switch\n * \\param is_np_shape 1 when numpy shape semantics is thread local on,\n *        2 when numpy shape semantics is global on and 0 when off\n * \\param prev returns the previous status before this set\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSetIsNumpyShape(int is_np_shape, int* prev);\n/*!\n * \\brief get numpy default data type\n * \\param curr returns the current status\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXIsNumpyDefaultDtype(bool* curr);\n/*!\n * \\brief set numpy default data type\n * \\param dtype_flag false when default dtype is flaot32,\n *                   true when default dtype is flaot64.\n * \\param prev returns the previous status before this set\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSetIsNumpyDefaultDtype(bool dtype_flag, bool* prev);\n/*!\n * \\brief mark NDArrays as variables to compute gradient for autograd\n * \\param num_var number of variable NDArrays\n * \\param var_handles variable NDArrays\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXAutogradMarkVariables(uint32_t num_var,\n                                      NDArrayHandle* var_handles,\n                                      uint32_t* reqs_array,\n                                      NDArrayHandle* grad_handles);\n/*!\n * \\brief unmark nonleaf NDArrays to free the memory\n * \\param num_var number of variable NDArrays\n * \\param var_handles variable NDArrays\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXAutogradDropGrads(uint32_t num_var, NDArrayHandle* var_handles);\n/*!\n * \\brief compute the gradient of outputs w.r.t variabels\n * \\param num_output number of output NDArray\n * \\param output_handles output NDArrays\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXAutogradComputeGradient(uint32_t num_output, NDArrayHandle* output_handles);\n/*!\n * \\brief compute the gradient of outputs w.r.t variabels\n * \\param num_output number of output NDArray\n * \\param output_handles output NDArrays\n * \\param ograd_handles head gradient for NDArrays\n * \\param retain_graph whether to keep the graph after backward\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXAutogradBackward(uint32_t num_output,\n                                 NDArrayHandle* output_handles,\n                                 NDArrayHandle* ograd_handles,\n                                 int retain_graph);\n/*!\n * \\brief compute the gradient of outputs w.r.t variabels\n * \\param num_output number of output NDArray\n * \\param output_handles output NDArrays\n * \\param ograd_handles head gradient for NDArrays\n * \\param num_variables number of variables\n * \\param\n * \\param retain_graph whether to keep the graph after backward\n * \\param is_train whether to do backward for training or inference\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXAutogradBackwardEx(uint32_t num_output,\n                                   NDArrayHandle* output_handles,\n                                   NDArrayHandle* ograd_handles,\n                                   uint32_t num_variables,\n                                   NDArrayHandle* var_handles,\n                                   int retain_graph,\n                                   int create_graph,\n                                   int is_train,\n                                   NDArrayHandle** grad_handles,\n                                   int** grad_stypes);\n/*\n * \\brief get the graph constructed by autograd.\n * \\param handle ndarray handle\n * \\param out output symbol handle\n */\nMXNET_DLL int MXAutogradGetSymbol(NDArrayHandle handle, SymbolHandle* out);\n\n/*!\n * \\brief create cached operator, allows to choose thread_safe version\n * of cachedop\n */\nMXNET_DLL int MXCreateCachedOp(SymbolHandle handle,\n                               int num_flags,\n                               const char** keys,\n                               const char** vals,\n                               CachedOpHandle* out,\n                               bool thread_safe DEFAULT(false));\n\n/*!\n * \\brief free cached operator\n */\nMXNET_DLL int MXFreeCachedOp(CachedOpHandle handle);\n\n/*!\n * \\brief get optimized graph from the cached op\n */\nMXNET_DLL int MXCachedOpGetOptimizedSymbol(CachedOpHandle handle, SymbolHandle* out);\n\n/*!\n * \\brief invoke a cached op\n * \\param handle the handle to the cached op\n * \\param num_inputs number of input NDArrays\n * \\param inputs input NDArrays\n * \\param num_outputs number of output NDArrays\n * \\param default_dev_type the default context type\n * \\param default_dev_id the default context device id\n * \\param outputs output NDArrays\n * \\param out_stypes output ndarrays' stypes\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXInvokeCachedOp(CachedOpHandle handle,\n                               int num_inputs,\n                               NDArrayHandle* inputs,\n                               int default_dev_type,\n                               int default_dev_id,\n                               int* num_outputs,\n                               NDArrayHandle** outputs,\n                               const int** out_stypes);\n\n/*!\n * \\brief cached op set monitor callback\n */\nMXNET_DLL int MXCachedOpRegisterOpHook(CachedOpHandle handle,\n                                       CachedOpMonitorCallback callback,\n                                       bool monitor_all);\n\n/*!\n * \\brief Get current status of deferred compute mode\n * \\param curr returns the current status.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArrayIsDeferredCompute(int* curr);\n\n/*!\n * \\brief set whether to enable deferred compute mode\n * \\param deferred_compute_enabled 1 to enable, 0 to disable.\n * \\param prev returns the previous status before this set.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArraySetIsDeferredCompute(int deferred_compute_enabled, int* prev);\n\n/*!\n * \\brief Associate variables with deferred compute arrays\n * \\param arrays ndarray handles to be matched with variables\n * \\param variables symbol handles of variables to be matched with ndarrays\n * \\param num number of arrays and variables respectively\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNDArraySetDeferredComputeVariable(NDArrayHandle* arrays,\n                                                  SymbolHandle* variables,\n                                                  int num);\n\n/*!\n * \\brief Convert the graph constructed during deferred computation mode to a Symbol.\n * \\param output_handles ndarray handles of outputs\n * \\param out grouped output symbol handle\n *\n * Construct a Symbol for the deferred computation graph. output_handles\n * specifies the outputs of interest which the returned symbol will compute.\n */\nMXNET_DLL int MXNDArrayGetDeferredComputeSymbol(NDArrayHandle* output_handles,\n                                                int num_outputs,\n                                                SymbolHandle* out);\n\n/*!\n * \\brief Clear the deferred compute info associated with the ndarrays.\n * \\param arrays ndarray handles of deferred compute outputs\n * \\param num number of ndarrays\n * \\return 0 when success, -1 otherwise\n */\nMXNET_DLL int MXNDArrayClearDeferredCompute(NDArrayHandle* arrays, int num);\n\n//--------------------------------------------\n// Part 3: symbolic configuration generation\n//--------------------------------------------\n/*!\n * \\brief list all the available operator names, include entries.\n * \\param out_size the size of returned array\n * \\param out_array the output operator name array.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXListAllOpNames(uint32_t* out_size, const char*** out_array);\n\n/*!\n * \\brief list all the available AtomicSymbolEntry\n * \\param out_size the size of returned array\n * \\param out_array the output AtomicSymbolCreator array\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolListAtomicSymbolCreators(uint32_t* out_size, AtomicSymbolCreator** out_array);\n\n/*!\n * \\brief Get the name of an atomic symbol.\n * \\param creator the AtomicSymbolCreator.\n * \\param name The returned name of the creator.\n */\nMXNET_DLL int MXSymbolGetAtomicSymbolName(AtomicSymbolCreator creator, const char** name);\n\n/*!\n * \\brief Get the input symbols of the graph.\n * \\param sym The graph.\n * \\param inputs The input symbols of the graph.\n * \\param input_size the number of input symbols returned.\n */\nMXNET_DLL int MXSymbolGetInputSymbols(SymbolHandle sym, SymbolHandle** inputs, int* input_size);\n\n/*!\n * \\brief Cut a subgraph whose nodes are marked with a subgraph attribute.\n * The input graph will be modified. A variable node will be created for each\n * edge that connects to nodes outside the subgraph. The outside nodes that\n * connect to the subgraph will be returned.\n * \\param sym The graph.\n * \\param inputs The nodes that connect to the subgraph.\n * \\param input_size The number of such nodes.\n */\nMXNET_DLL int MXSymbolCutSubgraph(SymbolHandle sym, SymbolHandle** inputs, int* input_size);\n\n/*!\n * \\brief Get the detailed information about atomic symbol.\n * \\param creator the AtomicSymbolCreator.\n * \\param name The returned name of the creator.\n * \\param description The returned description of the symbol.\n * \\param num_args Number of arguments.\n * \\param arg_names Name of the arguments.\n * \\param arg_type_infos Type informations about the arguments.\n * \\param arg_descriptions Description information about the arguments.\n * \\param key_var_num_args The keyword argument for specifying variable number of arguments.\n *            When this parameter has non-zero length, the function allows variable number\n *            of positional arguments, and will need the caller to pass it in in\n *            MXSymbolCreateAtomicSymbol,\n *            With key = key_var_num_args, and value = number of positional arguments.\n * \\param return_type Return type of the function, can be Symbol or Symbol[]\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolGetAtomicSymbolInfo(AtomicSymbolCreator creator,\n                                          const char** name,\n                                          const char** description,\n                                          uint32_t* num_args,\n                                          const char*** arg_names,\n                                          const char*** arg_type_infos,\n                                          const char*** arg_descriptions,\n                                          const char** key_var_num_args,\n                                          const char** return_type DEFAULT(NULL));\n/*!\n * \\brief Create an AtomicSymbol.\n *\n * A Symbol is said to be atomic if it is not composed of other Symbols. Atomic\n * Symbols can be composed.\n *\n * \\param creator the AtomicSymbolCreator\n * \\param num_param the number of parameters\n * \\param keys the keys to the params\n * \\param vals the vals of the params\n * \\param out pointer to the created symbol handle\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolCreateAtomicSymbol(AtomicSymbolCreator creator,\n                                         uint32_t num_param,\n                                         const char** keys,\n                                         const char** vals,\n                                         SymbolHandle* out);\n/*!\n * \\brief Create a Variable Symbol.\n * \\param name name of the variable\n * \\param out pointer to the created symbol handle\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolCreateVariable(const char* name, SymbolHandle* out);\n/*!\n * \\brief Create a Symbol by grouping list of symbols together\n * \\param num_symbols number of symbols to be grouped\n * \\param symbols array of symbol handles\n * \\param out pointer to the created symbol handle\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolCreateGroup(uint32_t num_symbols, SymbolHandle* symbols, SymbolHandle* out);\n/*!\n * \\brief Load a symbol from a json file.\n * \\param fname the file name.\n * \\param out the output symbol.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolCreateFromFile(const char* fname, SymbolHandle* out);\n/*!\n * \\brief Load a symbol from a json string.\n * \\param json the json string.\n * \\param out the output symbol.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolCreateFromJSON(const char* json, SymbolHandle* out);\n/*!\n * \\brief Remove the operators amp_cast and amp_multicast\n * \\param sym_handle the input symbol.\n * \\param ret_sym_handle the output symbol.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolRemoveAmpCast(SymbolHandle sym_handle, SymbolHandle* ret_sym_handle);\n/*!\n * \\brief Save a symbol into a json file.\n * \\param symbol the input symbol.\n * \\param fname the file name.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolSaveToFile(SymbolHandle symbol, const char* fname);\n/*!\n * \\brief Save a symbol into a json string\n * \\param symbol the input symbol.\n * \\param out_json output json string.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolSaveToJSON(SymbolHandle symbol, const char** out_json);\n/*!\n * \\brief Free the symbol handle.\n * \\param symbol the symbol\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolFree(SymbolHandle symbol);\n/*!\n * \\brief Copy the symbol to another handle\n * \\param symbol the source symbol\n * \\param out used to hold the result of copy\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolCopy(SymbolHandle symbol, SymbolHandle* out);\n/*!\n * \\brief Print the content of symbol, used for debug.\n * \\param symbol the symbol\n * \\param out_str pointer to hold the output string of the printing.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolPrint(SymbolHandle symbol, const char** out_str);\n/*!\n * \\brief Get string name from symbol\n * \\param symbol the source symbol\n * \\param out The result name.\n * \\param success Whether the result is contained in out.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolGetName(SymbolHandle symbol, const char** out, int* success);\n/*!\n * \\brief Get string attribute from symbol\n * \\param symbol the source symbol\n * \\param key The key of the symbol.\n * \\param out The result attribute, can be NULL if the attribute do not exist.\n * \\param success Whether the result is contained in out.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolGetAttr(SymbolHandle symbol, const char* key, const char** out, int* success);\n/*!\n * \\brief Set string attribute from symbol.\n *  NOTE: Setting attribute to a symbol can affect the semantics(mutable/immutable) of symbolic\n * graph.\n *\n *  Safe recommendaton: use  immutable graph\n *  - Only allow set attributes during creation of new symbol as optional parameter\n *\n *  Mutable graph (be careful about the semantics):\n *  - Allow set attr at any point.\n *  - Mutating an attribute of some common node of two graphs can cause confusion from user.\n *\n * \\param symbol the source symbol\n * \\param key The key of the symbol.\n * \\param value The value to be saved.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolSetAttr(SymbolHandle symbol, const char* key, const char* value);\n/*!\n * \\brief Get all attributes from symbol, including all descendents.\n * \\param symbol the source symbol\n * \\param out_size The number of output attributes\n * \\param out 2*out_size strings representing key value pairs.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolListAttr(SymbolHandle symbol, uint32_t* out_size, const char*** out);\n/*!\n * \\brief Get all attributes from symbol, excluding descendents.\n * \\param symbol the source symbol\n * \\param out_size The number of output attributes\n * \\param out 2*out_size strings representing key value pairs.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolListAttrShallow(SymbolHandle symbol, uint32_t* out_size, const char*** out);\n/*!\n * \\brief List arguments in the symbol.\n * \\param symbol the symbol\n * \\param out_size output size\n * \\param out_str_array pointer to hold the output string array\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolListArguments(SymbolHandle symbol,\n                                    uint32_t* out_size,\n                                    const char*** out_str_array);\n\n/*!\n * \\brief List returns in the symbol.\n * \\param symbol the symbol\n * \\param out_size output size\n * \\param out_str_array pointer to hold the output string array\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolListOutputs(SymbolHandle symbol,\n                                  uint32_t* out_size,\n                                  const char*** out_str_array);\n\n/*!\n * \\brief Get number of outputs of the symbol.\n * \\param symbol The symbol\n * \\param out_size number of outputs\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolGetNumOutputs(SymbolHandle symbol, uint32_t* output_count);\n\n/*!\n * \\brief Get a symbol that contains all the internals.\n * \\param symbol The symbol\n * \\param out The output symbol whose outputs are all the internals.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolGetInternals(SymbolHandle symbol, SymbolHandle* out);\n/*!\n * \\brief Get a symbol that contains all the inputs.\n * \\param symbol The symbol\n * \\param out The output symbol whose outputs are all the internals.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolGetInputs(SymbolHandle symbol, SymbolHandle* out);\n/*!\n * \\brief Get a symbol that contains only direct children.\n * \\param symbol The symbol\n * \\param out The output symbol whose outputs are the direct children.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolGetChildren(SymbolHandle symbol, SymbolHandle* out);\n/*!\n * \\brief Get index-th outputs of the symbol.\n * \\param symbol The symbol\n * \\param index the Index of the output.\n * \\param out The output symbol whose outputs are the index-th symbol.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolGetOutput(SymbolHandle symbol, uint32_t index, SymbolHandle* out);\n\n/*!\n * \\brief List auxiliary states in the symbol.\n * \\param symbol the symbol\n * \\param out_size output size\n * \\param out_str_array pointer to hold the output string array\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolListAuxiliaryStates(SymbolHandle symbol,\n                                          uint32_t* out_size,\n                                          const char*** out_str_array);\n\n/*!\n * \\brief Compose the symbol on other symbols.\n *\n *  This function will change the sym hanlde.\n *  To achieve function apply behavior, copy the symbol first\n *  before apply.\n *\n * \\param sym the symbol to apply\n * \\param name the name of symbol\n * \\param num_args number of arguments\n * \\param keys the key of keyword args (optional)\n * \\param args arguments to sym\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolCompose(SymbolHandle sym,\n                              const char* name,\n                              uint32_t num_args,\n                              const char** keys,\n                              SymbolHandle* args);\n/*!\n * \\brief Get the gradient graph of the symbol\n *\n * \\param sym the symbol to get gradient\n * \\param num_wrt number of arguments to get gradient\n * \\param wrt the name of the arguments to get gradient\n * \\param out the returned symbol that has gradient\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolGrad(SymbolHandle sym, uint32_t num_wrt, const char** wrt, SymbolHandle* out);\n\n/*!\n * \\brief infer shape of unknown input shapes given the known one.\n *  The shapes are packed into a CSR matrix represented by arg_ind_ptr and arg_shape_data\n *  The call will be treated as a kwargs call if key != NULL or num_args==0, otherwise it is\n *  positional. This api is available when MXNet is built with flag USE_INT64_TENSOR_SIZE=0 (by\n *  default)\n *\n * \\param sym symbol handle\n * \\param num_args number of input arguments.\n * \\param keys the key of keyword args (optional)\n * \\param arg_ind_ptr the head pointer of the rows in CSR\n * \\param arg_shape_data the content of the CSR\n * \\param in_shape_size sizeof the returning array of in_shapes\n * \\param in_shape_ndim returning array of shape dimensions of eachs input shape.\n * \\param in_shape_data returning array of pointers to head of the input shape.\n * \\param out_shape_size sizeof the returning array of out_shapes\n * \\param out_shape_ndim returning array of shape dimensions of each output shape.\n * \\param out_shape_data returning array of pointers to head of the output shape.\n * \\param aux_shape_size sizeof the returning array of aux_shapes\n * \\param aux_shape_ndim returning array of shape dimensions of each auxiliary shape.\n * \\param aux_shape_data returning array of pointers to head of the auxiliary shape.\n * \\param complete whether infer shape completes or more information is needed.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolInferShape(SymbolHandle sym,\n                                 uint32_t num_args,\n                                 const char** keys,\n                                 const uint32_t* arg_ind_ptr,\n                                 const int* arg_shape_data,\n                                 uint32_t* in_shape_size,\n                                 const int** in_shape_ndim,\n                                 const int*** in_shape_data,\n                                 uint32_t* out_shape_size,\n                                 const int** out_shape_ndim,\n                                 const int*** out_shape_data,\n                                 uint32_t* aux_shape_size,\n                                 const int** aux_shape_ndim,\n                                 const int*** aux_shape_data,\n                                 int* complete);\n\n/*!\n * \\brief infer shape of unknown input shapes given the known one.\n *  The shapes are packed into a CSR matrix represented by arg_ind_ptr and arg_shape_data\n *\n *  The call will be treated as a kwargs call if key != NULL or num_args==0, otherwise it is\n * positional. This api is available when MXNet is built with flag USE_INT64_TENSOR_SIZE=1 (not\n * default) i.e. Large Tensor Support\n *\n * \\param sym symbol handle\n * \\param num_args number of input arguments.\n * \\param keys the key of keyword args (optional)\n * \\param arg_ind_ptr the head pointer of the rows in CSR\n * \\param arg_shape_data the content of the CSR\n * \\param in_shape_size sizeof the returning array of in_shapes\n * \\param in_shape_ndim returning array of shape dimensions of each input shape.\n * \\param in_shape_data returning array of pointers to head of the input shape.\n * \\param out_shape_size sizeof the returning array of out_shapes\n * \\param out_shape_ndim returning array of shape dimensions of each output shape.\n * \\param out_shape_data returning array of pointers to head of the output shape.\n * \\param aux_shape_size sizeof the returning array of aux_shapes\n * \\param aux_shape_ndim returning array of shape dimensions of each auxiliary shape.\n * \\param aux_shape_data returning array of pointers to head of the auxiliary shape.\n * \\param complete whether infer shape completes or more information is needed.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolInferShape64(SymbolHandle sym,\n                                   uint32_t num_args,\n                                   const char** keys,\n                                   const int64_t* arg_ind_ptr,\n                                   const int64_t* arg_shape_data,\n                                   size_t* in_shape_size,\n                                   const int** in_shape_ndim,\n                                   const int64_t*** in_shape_data,\n                                   size_t* out_shape_size,\n                                   const int** out_shape_ndim,\n                                   const int64_t*** out_shape_data,\n                                   size_t* aux_shape_size,\n                                   const int** aux_shape_ndim,\n                                   const int64_t*** aux_shape_data,\n                                   int* complete);\n\n/*!\n * \\brief partially infer shape of unknown input shapes given the known one.\n *\n *  Return partially inferred results if not all shapes could be inferred.\n *  The shapes are packed into a CSR matrix represented by arg_ind_ptr and arg_shape_data\n *  The call will be treated as a kwargs call if key != NULL or num_args==0, otherwise it is\n * positional. This api is available when MXNet is built with flag USE_INT64_TENSOR_SIZE=0 (by\n * default)\n *\n * \\param sym symbol handle\n * \\param num_args number of input arguments.\n * \\param keys the key of keyword args (optional)\n * \\param arg_ind_ptr the head pointer of the rows in CSR\n * \\param arg_shape_data the content of the CSR\n * \\param in_shape_size sizeof the returning array of in_shapes\n * \\param in_shape_ndim returning array of shape dimensions of each input shape.\n * \\param in_shape_data returning array of pointers to head of the input shape.\n * \\param out_shape_size sizeof the returning array of out_shapes\n * \\param out_shape_ndim returning array of shape dimensions of each output shape.\n * \\param out_shape_data returning array of pointers to head of the output shape.\n * \\param aux_shape_size sizeof the returning array of aux_shapes\n * \\param aux_shape_ndim returning array of shape dimensions of each auxiliary shape.\n * \\param aux_shape_data returning array of pointers to head of the auxiliary shape.\n * \\param complete whether infer shape completes or more information is needed.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolInferShapePartial(SymbolHandle sym,\n                                        uint32_t num_args,\n                                        const char** keys,\n                                        const uint32_t* arg_ind_ptr,\n                                        const int* arg_shape_data,\n                                        uint32_t* in_shape_size,\n                                        const int** in_shape_ndim,\n                                        const int*** in_shape_data,\n                                        uint32_t* out_shape_size,\n                                        const int** out_shape_ndim,\n                                        const int*** out_shape_data,\n                                        uint32_t* aux_shape_size,\n                                        const int** aux_shape_ndim,\n                                        const int*** aux_shape_data,\n                                        int* complete);\n\n/*!\n * \\brief partially infer shape of unknown input shapes given the known one.\n *\n *  Return partially inferred results if not all shapes could be inferred.\n *  The shapes are packed into a CSR matrix represented by arg_ind_ptr and arg_shape_data\n *  The call will be treated as a kwargs call if key != NULL or num_args==0, otherwise it is\n * positional. This api is available when MXNet is built with flag USE_INT64_TENSOR_SIZE=1 (not\n * default) i.e. Large Tensor Support\n *\n * \\param sym symbol handle\n * \\param num_args number of input arguments.\n * \\param keys the key of keyword args (optional)\n * \\param arg_ind_ptr the head pointer of the rows in CSR\n * \\param arg_shape_data the content of the CSR\n * \\param in_shape_size sizeof the returning array of in_shapes\n * \\param in_shape_ndim returning array of shape dimensions of each input shape.\n * \\param in_shape_data returning array of pointers to head of the input shape.\n * \\param out_shape_size sizeof the returning array of out_shapes\n * \\param out_shape_ndim returning array of shape dimensions of each output shape.\n * \\param out_shape_data returning array of pointers to head of the output shape.\n * \\param aux_shape_size sizeof the returning array of aux_shapes\n * \\param aux_shape_ndim returning array of shape dimensions of each auxiliary shape.\n * \\param aux_shape_data returning array of pointers to head of the auxiliary shape.\n * \\param complete whether infer shape completes or more information is needed.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolInferShapePartial64(SymbolHandle sym,\n                                          uint32_t num_args,\n                                          const char** keys,\n                                          const int64_t* arg_ind_ptr,\n                                          const int64_t* arg_shape_data,\n                                          size_t* in_shape_size,\n                                          const int** in_shape_ndim,\n                                          const int64_t*** in_shape_data,\n                                          size_t* out_shape_size,\n                                          const int** out_shape_ndim,\n                                          const int64_t*** out_shape_data,\n                                          size_t* aux_shape_size,\n                                          const int** aux_shape_ndim,\n                                          const int64_t*** aux_shape_data,\n                                          int* complete);\n\n/*!\n * \\brief infer type of unknown input types given the known one.\n *  The types are packed into a CSR matrix represented by arg_ind_ptr and arg_type_data\n *  The call will be treated as a kwargs call if key != NULL or num_args==0, otherwise it is\n * positional.\n *\n * \\param sym symbol handle\n * \\param num_args numbe of input arguments.\n * \\param keys the key of keyword args (optional)\n * \\param arg_type_data the content of the CSR\n * \\param in_type_size sizeof the returning array of in_types\n * \\param in_type_data returning array of pointers to head of the input type.\n * \\param out_type_size sizeof the returning array of out_types\n * \\param out_type_data returning array of pointers to head of the output type.\n * \\param aux_type_size sizeof the returning array of aux_types\n * \\param aux_type_data returning array of pointers to head of the auxiliary type.\n * \\param complete whether infer type completes or more information is needed.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolInferType(SymbolHandle sym,\n                                uint32_t num_args,\n                                const char** keys,\n                                const int* arg_type_data,\n                                uint32_t* in_type_size,\n                                const int** in_type_data,\n                                uint32_t* out_type_size,\n                                const int** out_type_data,\n                                uint32_t* aux_type_size,\n                                const int** aux_type_data,\n                                int* complete);\n\n/*!\n * \\brief partially infer type of unknown input types given the known one.\n *\n *  Return partially inferred results if not all types could be inferred.\n *  The types are packed into a CSR matrix represented by arg_ind_ptr and arg_type_data\n *  The call will be treated as a kwargs call if key != NULL or num_args==0, otherwise it is\n * positional.\n *\n * \\param sym symbol handle\n * \\param num_args numbe of input arguments.\n * \\param keys the key of keyword args (optional)\n * \\param arg_type_data the content of the CSR\n * \\param in_type_size sizeof the returning array of in_types\n * \\param in_type_data returning array of pointers to head of the input type.\n * \\param out_type_size sizeof the returning array of out_types\n * \\param out_type_data returning array of pointers to head of the output type.\n * \\param aux_type_size sizeof the returning array of aux_types\n * \\param aux_type_data returning array of pointers to head of the auxiliary type.\n * \\param complete whether infer type completes or more information is needed.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXSymbolInferTypePartial(SymbolHandle sym,\n                                       uint32_t num_args,\n                                       const char** keys,\n                                       const int* arg_type_data,\n                                       uint32_t* in_type_size,\n                                       const int** in_type_data,\n                                       uint32_t* out_type_size,\n                                       const int** out_type_data,\n                                       uint32_t* aux_type_size,\n                                       const int** aux_type_data,\n                                       int* complete);\n\n/*!\n * \\brief Convert a symbol into a quantized symbol where FP32 operators are replaced with INT8\n * \\param sym_handle symbol to be converted\n * \\param ret_sym_handle quantized symbol result\n * \\param dev_type device type\n * \\param num_excluded_sym_names number of layers excluded from being quantized in the input symbol\n * \\param excluded_sym_names node names to be excluded from being quantized\n * \\param num_excluded_op_names number of operators excluded from being quantized in the input\n * symbol\n * \\param excluded_op_names operator names to be excluded from being quantized\n * \\param num_offline number of parameters that are quantized offline\n * \\param offline_params array of c strings representing the names of params quantized offline\n * \\param quantized_dtype the quantized destination type for input data\n * \\param calib_quantize **Deprecated**. quantize op will always be calibrated if could\n * \\param quantize_mode quantize mode to be used in quantize pass\n * \\param quantize_granularity quantize granularity, tensor-wise or channel-wise\n * \\param out_num_calib_names return the number of nodes to be calibrated\n * \\param out_calib_names return the node names to be calibrated\n */\nMXNET_DLL int MXQuantizeSymbol(SymbolHandle sym_handle,\n                               SymbolHandle* ret_sym_handle,\n                               const int* dev_type,\n                               const uint32_t num_excluded_sym_names,\n                               const char** excluded_sym_names,\n                               const uint32_t num_excluded_op_names,\n                               const char** excluded_op_names,\n                               const uint32_t num_offline,\n                               const char** offline_params,\n                               const char* quantized_dtype,\n                               const bool calib_quantize,\n                               const char* quantize_mode,\n                               const char* quantize_granularity,\n                               uint32_t* out_num_calib_names,\n                               const char*** out_calib_names);\n\n/*!\n * \\brief Convert a symbol into a mixed precision symbol with cast operators for target dtype\n * casting\n * \\param sym_handle symbol to be converted\n * \\param ret_sym_handle mixed precision symbol result\n * \\param target_dtype target_dtype for mixed precision symbol\n * \\param cast_params_offline whether to cast parameters offline to target_dtype\n * \\param offline_param_cast_attr_p attibute that will hold the dtype a parameter should be offline\n *                                  cast to (when cast_params_offline is true)\n * \\param num_inputs number of model inputs\n * \\param input_names_p names of model inputs\n * \\param num_all_args number of all model arguments\n * \\param all_arg_names_p names of all model arguments\n * \\param all_arg_types_p dtypes of all model arguments\n * \\param num_target_dtype_ops number of ops to be casted to target_dtype\n * \\param target_dtype_ops_p op names to be casted to target_dtype\n * \\param num_fp32_ops number of ops to be casted to FP32\n * \\param fp32_ops_p op names to be casted to fp32\n * \\param num_widest_dtype_ops number of ops to be casted to widest dtype\n * \\param widest_dtype_ops_p op names to be casted to widest dtype\n * \\param num_excluded_symbols number of symbols to be excluded from casting\n * \\param excluded_syms_p symbol names to be excluded from casting\n */\nMXNET_DLL int MXReducePrecisionSymbol(SymbolHandle sym_handle,\n                                      SymbolHandle* ret_sym_handle,\n                                      const int target_dtype,\n                                      const int cast_params_offline,\n                                      const char* const offline_param_cast_attr_p,\n                                      const uint32_t num_inputs,\n                                      const char** const input_names_p,\n                                      const uint32_t num_all_args,\n                                      const char** const all_arg_names_p,\n                                      const int* all_arg_types_p,\n                                      const uint32_t num_target_dtype_ops,\n                                      const char** const target_dtype_ops_p,\n                                      const uint32_t num_fp32_ops,\n                                      const char** const fp32_ops_p,\n                                      const uint32_t num_widest_dtype_ops,\n                                      const char** const widest_dtype_ops_p);\n\n/*!\n * \\brief Set calibration table to node attributes in the sym\n * \\param sym_handle symbol whose node attributes are to be set by calibration table\n * \\param num_layers number of layers in the calibration table\n * \\param layer names stored as keys in the calibration table\n * \\param low_quantiles low quantiles of layers stored in the calibration table\n * \\param high_quantiles high quantiles of layers stored in the calibration table\n * \\param ret_sym_handle returned symbol\n */\nMXNET_DLL int MXSetCalibTableToQuantizedSymbol(SymbolHandle qsym_handle,\n                                               const uint32_t num_layers,\n                                               const char** layer_names,\n                                               const float* low_quantiles,\n                                               const float* high_quantiles,\n                                               SymbolHandle* ret_sym_handle);\n\n/*!\n * \\brief Run subgraph pass based on the backend provided\n * \\param sym_handle symbol to be converted\n * \\param backend backend names for subgraph pass\n * \\param ret_sym_handle returned symbol\n */\nMXNET_DLL int MXGenBackendSubgraph(SymbolHandle sym_handle,\n                                   const char* backend,\n                                   SymbolHandle* ret_sym_handle);\n\n/*!\n * \\brief Generate atomic symbol (able to be composed) from a source symbol\n * \\param sym_handle source symbol\n * \\param ret_sym_handle returned atomic symbol\n */\nMXNET_DLL int MXGenAtomicSymbolFromSymbol(SymbolHandle sym_handle, SymbolHandle* ret_sym_handle);\n/*!\n * \\brief Partitions symbol for given backend, potentially creating subgraphs\n * \\param sym_handle symbol to be partitioned\n * \\param dev_type context device type\n * \\param backend_name backend name\n * \\param ret_sym_handle partitioned symbol returned\n * \\param len number of args\n * \\param in_args_handle args array\n * \\param num_options number of key value pairs\n * \\param keys keys for options\n * \\param vals values corresponding to keys\n * \\param num_input_shapes number of input shapes\n * \\param input_shape_names names of the input shapes\n * \\param input_shape_data pointer to the contiguous data shapes\n * \\param input_shape_idx array of per shape starting idx, the shape length for the i-th input shape\n * is calculate as input_shape_idx[i+1] - input_shape_idx[i]\n * \\param num_input_dtypes number of input data types\n * \\param input_dtype_names array of names of the input data types\n * \\param input_dtypes array of values of the input data types\n * \\param num_input_stypesnumber of input storage types\n * \\param input_stype_names array of names of the input storage types\n * \\param input_stypes array of values of input storage types\n * \\param skip_infer if the optimization should skip the attribute inferences\n * (to use if the backend does not require shape inference)\n * \\param new_args_cnt pointer a number to store the number of new args\n * \\param new_args_handle pointer on array to store the new args handles\n * \\param new_arg_names_handle pointer on array to store the new args names\n * \\param new_aux_cnt pointer a number to store the number of new aux\n * \\param new_aux_handle pointer on array to store the new aux handles\n * \\param new_aux_names_handle pointer on array to store the new aux names\n */\nMXNET_DLL int MXOptimizeForBackend(SymbolHandle sym_handle,\n                                   const char* backend_name,\n                                   const int dev_type,\n                                   SymbolHandle* ret_sym_handle,\n                                   const mx_uint args_len,\n                                   NDArrayHandle* in_args_handle,\n                                   const mx_uint aux_len,\n                                   NDArrayHandle* in_aux_handle,\n                                   const mx_uint num_options,\n                                   const char** keys,\n                                   const char** vals,\n                                   const uint32_t num_input_shapes,\n                                   const char** input_shape_names,\n                                   const int64_t* input_shape_data,\n                                   const uint32_t* input_shape_idx,\n                                   const uint32_t num_input_dtypes,\n                                   const char** input_dtype_names,\n                                   const int* input_dtypes,\n                                   const uint32_t num_input_stypes,\n                                   const char** input_stype_names,\n                                   const int* input_stypes,\n                                   bool skip_infer,\n                                   int* new_args_cnt,\n                                   NDArrayHandle** new_args_handle,\n                                   char*** new_arg_names_handle,\n                                   int* new_aux_cnt,\n                                   NDArrayHandle** new_aux_handle,\n                                   char*** new_aux_names_handle);\n\n//--------------------------------------------\n// Part 5: IO Interface\n//--------------------------------------------\n/*!\n * \\brief List all the available iterator entries\n * \\param out_size the size of returned iterators\n * \\param out_array the output iteratos entries\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXListDataIters(uint32_t* out_size, DataIterCreator** out_array);\n/*!\n * \\brief Init an iterator, init with parameters\n * the array size of passed in arguments\n * \\param handle of the iterator creator\n * \\param num_param number of parameter\n * \\param keys parameter keys\n * \\param vals parameter values\n * \\param out resulting iterator\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXDataIterCreateIter(DataIterCreator handle,\n                                   uint32_t num_param,\n                                   const char** keys,\n                                   const char** vals,\n                                   DataIterHandle* out);\n/*!\n * \\brief Get the detailed information about data iterator.\n * \\param creator the DataIterCreator.\n * \\param name The returned name of the creator.\n * \\param description The returned description of the symbol.\n * \\param num_args Number of arguments.\n * \\param arg_names Name of the arguments.\n * \\param arg_type_infos Type informations about the arguments.\n * \\param arg_descriptions Description information about the arguments.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXDataIterGetIterInfo(DataIterCreator creator,\n                                    const char** name,\n                                    const char** description,\n                                    uint32_t* num_args,\n                                    const char*** arg_names,\n                                    const char*** arg_type_infos,\n                                    const char*** arg_descriptions);\n/*!\n * \\brief Free the handle to the IO module\n * \\param handle the handle pointer to the data iterator\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXDataIterFree(DataIterHandle handle);\n/*!\n * \\brief Move iterator to next position\n * \\param handle the handle to iterator\n * \\param out return value of next\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXDataIterNext(DataIterHandle handle, int* out);\n/*!\n * \\brief Call iterator.Reset\n * \\param handle the handle to iterator\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXDataIterBeforeFirst(DataIterHandle handle);\n\n/*!\n * \\brief Call iterator.GetLenHint. Note that some iterators don't provide length.\n * \\param handle the handle to iterator\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXDataIterGetLenHint(DataIterHandle handle, int64_t* len);\n/*!\n * \\brief Get the handle to the NDArray of underlying data\n * \\param handle the handle pointer to the data iterator\n * \\param out handle to underlying data NDArray\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXDataIterGetData(DataIterHandle handle, NDArrayHandle* out);\n/*!\n * \\brief Get the image index by array.\n * \\param handle the handle pointer to the data iterator\n * \\param out_index output index of the array.\n * \\param out_size output size of the array.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXDataIterGetIndex(DataIterHandle handle, uint64_t** out_index, uint64_t* out_size);\n/*!\n * \\brief Get the padding number in current data batch\n * \\param handle the handle pointer to the data iterator\n * \\param pad pad number ptr\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXDataIterGetPadNum(DataIterHandle handle, int* pad);\n\n/*!\n * \\brief Get the handle to the NDArray of underlying label\n * \\param handle the handle pointer to the data iterator\n * \\param out the handle to underlying label NDArray\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXDataIterGetLabel(DataIterHandle handle, NDArrayHandle* out);\n/*!\n * \\brief Get the handles to specified underlying ndarrays of index\n * \\param handle the handle pointer to the data iterator\n * \\param num_outputs the length of outputs\n * \\param out the handle to an array of NDArrays that stores pointers to handles\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXDataIterGetItems(DataIterHandle handle, int* num_outputs, NDArrayHandle** outputs);\n\n/*!\n * \\brief List all the available dataset entries\n * \\param out_size the size of returned datasets\n * \\param out_array the output dataset entries\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXListDatasets(uint32_t* out_size, DatasetCreator** out_array);\n/*!\n * \\brief Init an dataset, init with parameters\n * the array size of passed in arguments\n * \\param handle of the dataset creator\n * \\param num_param number of parameter\n * \\param keys parameter keys\n * \\param vals parameter values\n * \\param out resulting dataset\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXDatasetCreateDataset(DatasetCreator handle,\n                                     uint32_t num_param,\n                                     const char** keys,\n                                     const char** vals,\n                                     DatasetHandle* out);\n/*!\n * \\brief Get the detailed information about dataset.\n * \\param creator the DatasetCreator.\n * \\param name The returned name of the creator.\n * \\param description The returned description of the symbol.\n * \\param num_args Number of arguments.\n * \\param arg_names Name of the arguments.\n * \\param arg_type_infos Type informations about the arguments.\n * \\param arg_descriptions Description information about the arguments.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXDatasetGetDatasetInfo(DatasetCreator creator,\n                                      const char** name,\n                                      const char** description,\n                                      uint32_t* num_args,\n                                      const char*** arg_names,\n                                      const char*** arg_type_infos,\n                                      const char*** arg_descriptions);\n/*!\n * \\brief Free the handle to the IO module\n * \\param handle the handle pointer to the dataset\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXDatasetFree(DatasetHandle handle);\n/*!\n * \\brief Get dataset overal length(size)\n * \\param handle the handle to dataset\n * \\param out return value of GetLen\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXDatasetGetLen(DatasetHandle handle, uint64_t* out);\n/*!\n * \\brief Get Output NDArray given specified indices\n * \\param handle the handle to dataset\n * \\param index the index of the dataset item to be retrieved\n * \\param num_outputs the number of output ndarrays\n * \\param outputs the pointers to handles of ndarrays\n * \\param is_scalar if not zeros then output should be casted to scalars\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXDatasetGetItems(DatasetHandle handle,\n                                uint64_t index,\n                                int* num_outputs,\n                                NDArrayHandle** outputs);\n\n/*!\n * \\brief List all the available batchify function entries\n * \\param out_size the size of returned batchify functions\n * \\param out_array the output batchify function entries\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXListBatchifyFunctions(uint32_t* out_size, BatchifyFunctionCreator** out_array);\n/*!\n * \\brief Init an batchify function, init with parameters\n * the array size of passed in arguments\n * \\param handle of the batchify function creator\n * \\param num_param number of parameter\n * \\param keys parameter keys\n * \\param vals parameter values\n * \\param out resulting batchify function\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXBatchifyFunctionCreateFunction(BatchifyFunctionCreator handle,\n                                               uint32_t num_param,\n                                               const char** keys,\n                                               const char** vals,\n                                               BatchifyFunctionHandle* out);\n/*!\n * \\brief Get the detailed information about batchify function.\n * \\param creator the batchifyFunctionCreator.\n * \\param name The returned name of the creator.\n * \\param description The returned description of the symbol.\n * \\param num_args Number of arguments.\n * \\param arg_names Name of the arguments.\n * \\param arg_type_infos Type informations about the arguments.\n * \\param arg_descriptions Description information about the arguments.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXBatchifyFunctionGetFunctionInfo(BatchifyFunctionCreator creator,\n                                                const char** name,\n                                                const char** description,\n                                                uint32_t* num_args,\n                                                const char*** arg_names,\n                                                const char*** arg_type_infos,\n                                                const char*** arg_descriptions);\n/*!\n * \\brief Invoke the Batchify Function\n * \\param handle the handle pointer to the batchify function\n * \\param batch_size the batch size\n * \\param num_output the number of ndarrays for output\n * \\param inputs the pointers to input ndarrays\n * \\param ouptuts the pointers to output ndarrays\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXBatchifyFunctionInvoke(BatchifyFunctionHandle handle,\n                                       int batch_size,\n                                       int num_output,\n                                       NDArrayHandle* inputs,\n                                       NDArrayHandle** outputs);\n/*!\n * \\brief Free the handle to the IO module\n * \\param handle the handle pointer to the batchify function\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXBatchifyFunctionFree(BatchifyFunctionHandle handle);\n//--------------------------------------------\n// Part 6: basic KVStore interface\n//--------------------------------------------\n/*!\n * \\brief Initialized ps-lite environment variables\n * \\param num_vars number of variables to initialize\n * \\param keys environment keys\n * \\param vals environment values\n */\nMXNET_DLL int MXInitPSEnv(uint32_t num_vars, const char** keys, const char** vals);\n\n/*!\n * \\brief Create a kvstore\n * \\param type the type of KVStore\n * \\param out The output type of KVStore\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreCreate(const char* type, KVStoreHandle* out);\n\n/*!\n * \\brief Set parameters to use low-bit compressed gradients\n * \\param handle handle to the kvstore\n * \\param keys keys for compression parameters\n * \\param vals values for compression parameters\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreSetGradientCompression(KVStoreHandle handle,\n                                              uint32_t num_params,\n                                              const char** keys,\n                                              const char** vals);\n\n/*!\n * \\brief Delete a KVStore handle.\n * \\param handle handle to the kvstore\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreFree(KVStoreHandle handle);\n/*!\n * \\brief Init a list of (key,value) pairs in kvstore\n * \\param handle handle to the kvstore\n * \\param num the number of key-value pairs\n * \\param keys the list of keys\n * \\param vals the list of values\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreInit(KVStoreHandle handle,\n                            uint32_t num,\n                            const int* keys,\n                            NDArrayHandle* vals);\n\n/*!\n * \\brief Init a list of (key,value) pairs in kvstore, where each key is a string\n * \\param handle handle to the kvstore\n * \\param num the number of key-value pairs\n * \\param keys the list of keys\n * \\param vals the list of values\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreInitEx(KVStoreHandle handle,\n                              uint32_t num,\n                              const char** keys,\n                              NDArrayHandle* vals);\n\n/*!\n * \\brief Push a list of (key,value) pairs to kvstore\n * \\param handle handle to the kvstore\n * \\param num the number of key-value pairs\n * \\param keys the list of keys\n * \\param vals the list of values\n * \\param priority the priority of the action\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStorePush(KVStoreHandle handle,\n                            uint32_t num,\n                            const int* keys,\n                            NDArrayHandle* vals,\n                            int priority);\n/*!\n * \\brief Push a list of (key,value) pairs to kvstore, where each key is a string\n * \\param handle handle to the kvstore\n * \\param num the number of key-value pairs\n * \\param keys the list of keys\n * \\param vals the list of values\n * \\param priority the priority of the action\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStorePushEx(KVStoreHandle handle,\n                              uint32_t num,\n                              const char** keys,\n                              NDArrayHandle* vals,\n                              int priority);\n/*!\n * \\brief pull a list of (key, value) pairs from the kvstore\n * \\param handle handle to the kvstore\n * \\param num the number of key-value pairs\n * \\param keys the list of keys\n * \\param vals the list of values\n * \\param priority the priority of the action\n * \\param ignore_sparse whether to ignore sparse arrays in the request\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStorePullWithSparse(KVStoreHandle handle,\n                                      uint32_t num,\n                                      const int* keys,\n                                      NDArrayHandle* vals,\n                                      int priority,\n                                      bool ignore_sparse);\n/*!\n * \\brief pull a list of (key, value) pairs from the kvstore, where each key is a string\n * \\param handle handle to the kvstore\n * \\param num the number of key-value pairs\n * \\param keys the list of keys\n * \\param vals the list of values\n * \\param priority the priority of the action\n * \\param ignore_sparse whether to ignore sparse arrays in the request\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStorePullWithSparseEx(KVStoreHandle handle,\n                                        uint32_t num,\n                                        const char** keys,\n                                        NDArrayHandle* vals,\n                                        int priority,\n                                        bool ignore_sparse);\n/*!\n * \\brief pull a list of (key, value) pairs from the kvstore\n * \\param handle handle to the kvstore\n * \\param num the number of key-value pairs\n * \\param keys the list of keys\n * \\param vals the list of values\n * \\param priority the priority of the action\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStorePull(KVStoreHandle handle,\n                            uint32_t num,\n                            const int* keys,\n                            NDArrayHandle* vals,\n                            int priority);\n/*!\n * \\brief pull a list of (key, value) pairs from the kvstore, where each key is a string\n * \\param handle handle to the kvstore\n * \\param num the number of key-value pairs\n * \\param keys the list of keys\n * \\param vals the list of values\n * \\param priority the priority of the action\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStorePullEx(KVStoreHandle handle,\n                              uint32_t num,\n                              const char** keys,\n                              NDArrayHandle* vals,\n                              int priority);\n\n/*!\n * \\brief pull a list of (key, value) pairs from the kvstore, where each key is an integer.\n *        The NDArray pulled back will be in row_sparse storage with only the specified\n *        row_ids present based row_ids (others rows are zeros).\n * \\param handle handle to the kvstore\n * \\param num the number of key-value pairs\n * \\param keys the list of keys\n * \\param vals the list of values\n * \\param row_ids the list of row_id NDArrays\n * \\param priority the priority of the action\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStorePullRowSparse(KVStoreHandle handle,\n                                     uint32_t num,\n                                     const int* keys,\n                                     NDArrayHandle* vals,\n                                     const NDArrayHandle* row_ids,\n                                     int priority);\n/*!\n * \\brief pull a list of (key, value) pairs from the kvstore, where each key is a string.\n *        The NDArray pulled back will be in row_sparse storage with only the specified\n *        row_ids present based row_ids (others rows are zeros).\n * \\param handle handle to the kvstore\n * \\param num the number of key-value pairs\n * \\param keys the list of keys\n * \\param vals the list of values\n * \\param row_ids the list of row_id NDArrays\n * \\param priority the priority of the action\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStorePullRowSparseEx(KVStoreHandle handle,\n                                       uint32_t num,\n                                       const char** keys,\n                                       NDArrayHandle* vals,\n                                       const NDArrayHandle* row_ids,\n                                       int priority);\n\n/*!\n * \\brief broadcast a list of (key, value) pairs from the kvstore\n * \\param handle handle to the kvstore\n * \\param vnum the number of key-value pairs corresponding to vkeys\n * \\param vkeys the list of keys for the values to be pushed\n * \\param onum the number of key-value pairs corresponding to okeys\n * \\param okeys the list of keys for the values to be pulled\n * \\param vals the list of values\n * \\param outs the list of outputs\n * \\param priority the priority of the action\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreBroadcast(KVStoreHandle handle,\n                                 mx_uint vnum,\n                                 const int* vkeys,\n                                 mx_uint onum,\n                                 const int* okeys,\n                                 NDArrayHandle* vals,\n                                 NDArrayHandle* outs,\n                                 int priority);\n/*!\n * \\brief broadcast a list of (key, value) pairs from the kvstore,\n * where each key is a string\n * \\param handle handle to the kvstore\n * \\param vnum the number of key-value pairs corresponding to vkeys\n * \\param vkeys the list of keys for the values to be pushed\n * \\param onum the number of key-value pairs corresponding to okeys\n * \\param okeys the list of keys for the values to be pulled\n * \\param vals the list of values\n * \\param outs the list of outputs\n * \\param priority the priority of the action\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreBroadcastEx(KVStoreHandle handle,\n                                   mx_uint vnum,\n                                   const char** vkeys,\n                                   mx_uint onum,\n                                   const char** okeys,\n                                   NDArrayHandle* vals,\n                                   NDArrayHandle* outs,\n                                   int priority);\n\n/*!\n * \\brief push and pull a list of (key, value) pairs from the kvstore\n * \\param handle handle to the kvstore\n * \\param vnum the number of key-value pairs corresponding to vkeys\n * \\param vkeys the list of keys for the values to be pushed\n * \\param onum the number of key-value pairs corresponding to okeys\n * \\param okeys the list of keys for the values to be pulled\n * \\param vals the list of values\n * \\param outs the list of outputs\n * \\param priority the priority of the action\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStorePushPull(KVStoreHandle handle,\n                                mx_uint vnum,\n                                const int* vkeys,\n                                mx_uint onum,\n                                const int* okeys,\n                                NDArrayHandle* vals,\n                                NDArrayHandle* outs,\n                                int priority);\n/*!\n * \\brief push and pull a list of (key, value) pairs from the kvstore,\n * where each key is a string\n * \\param handle handle to the kvstore\n * \\param vnum the number of key-value pairs corresponding to vkeys\n * \\param vkeys the list of keys for the values to be pushed\n * \\param onum the number of key-value pairs corresponding to okeys\n * \\param okeys the list of keys for the values to be pulled\n * \\param vals the list of values\n * \\param outs the list of outputs\n * \\param priority the priority of the action\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStorePushPullEx(KVStoreHandle handle,\n                                  mx_uint vnum,\n                                  const char** vkeys,\n                                  mx_uint onum,\n                                  const char** okeys,\n                                  NDArrayHandle* vals,\n                                  NDArrayHandle* outs,\n                                  int priority);\n\n/*!\n * \\brief user-defined updater for the kvstore\n * It's this updater's responsibility to delete \\a recv and \\a local\n * \\param the key\n * \\param recv the pushed value on this key\n * \\param local the value stored on local on this key\n * \\param handle The additional handle to the updater\n */\ntypedef void(MXKVStoreUpdater)(int key, NDArrayHandle recv, NDArrayHandle local, void* handle);\n/*!\n * \\brief user-defined updater for the kvstore with string keys\n * It's this updater's responsibility to delete \\a recv and \\a local\n * \\param the key\n * \\param recv the pushed value on this key\n * \\param local the value stored on local on this key\n * \\param handle The additional handle to the updater\n */\ntypedef void(MXKVStoreStrUpdater)(const char* key,\n                                  NDArrayHandle recv,\n                                  NDArrayHandle local,\n                                  void* handle);\n/*!\n * \\brief register a push updater\n * \\param handle handle to the KVStore\n * \\param updater udpater function\n * \\param updater_handle The additional handle used to invoke the updater\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreSetUpdater(KVStoreHandle handle,\n                                  MXKVStoreUpdater updater,\n                                  void* updater_handle);\n/*!\n * \\brief register a push updater with int keys and one with string keys\n * \\param handle handle to the KVStore\n * \\param updater updater function with int keys\n * \\param str_updater updater function with string keys\n * \\param updater_handle The additional handle used to invoke the updater\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreSetUpdaterEx(KVStoreHandle handle,\n                                    MXKVStoreUpdater updater,\n                                    MXKVStoreStrUpdater str_updater,\n                                    void* updater_handle);\n/*!\n * \\brief get the type of the kvstore\n * \\param handle handle to the KVStore\n * \\param type a string type\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreGetType(KVStoreHandle handle, const char** type);\n//--------------------------------------------\n// Part 6: advanced KVStore for multi-machines\n//--------------------------------------------\n\n/**\n * \\brief return The rank of this node in its group, which is in [0, GroupSize).\n *\n * \\param handle handle to the KVStore\n * \\param ret the node rank\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreGetRank(KVStoreHandle handle, int* ret);\n\n/**\n * \\brief return The number of nodes in this group, which is\n * - number of workers if if `IsWorkerNode() == true`,\n * - number of servers if if `IsServerNode() == true`,\n * - 1 if `IsSchedulerNode() == true`,\n * \\param handle handle to the KVStore\n * \\param ret the group size\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreGetGroupSize(KVStoreHandle handle, int* ret);\n\n/**\n * \\brief return whether or not this process is a worker node.\n * \\param ret 1 for yes, 0 for no\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreIsWorkerNode(int* ret);\n\n/**\n * \\brief return whether or not this process is a server node.\n * \\param ret 1 for yes, 0 for no\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreIsServerNode(int* ret);\n\n/**\n * \\brief return whether or not this process is a scheduler node.\n * \\param ret 1 for yes, 0 for no\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreIsSchedulerNode(int* ret);\n\n/**\n * \\brief global barrier among all worker machines\n *\n * \\param handle handle to the KVStore\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreBarrier(KVStoreHandle handle);\n\n/**\n * \\brief whether to do barrier when finalize\n *\n * \\param handle handle to the KVStore\n * \\param barrier_before_exit whether to do barrier when kvstore finalize\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreSetBarrierBeforeExit(KVStoreHandle handle, const int barrier_before_exit);\n\n/**\n * \\brief the prototype of a server controller\n * \\param head the head of the command\n * \\param body the body of the command\n * \\param controller_handle helper handle for implementing controller\n */\ntypedef void(MXKVStoreServerController)(int head, const char* body, void* controller_handle);\n\n/**\n * \\brief Run as server (or scheduler)\n * \\param handle handle to the KVStore\n * \\param controller the user-defined server controller\n * \\param controller_handle helper handle for implementing controller\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreRunServer(KVStoreHandle handle,\n                                 MXKVStoreServerController controller,\n                                 void* controller_handle);\n\n/**\n * \\brief Send a command to all server nodes\n * \\param handle handle to the KVStore\n * \\param cmd_id the head of the command\n * \\param cmd_body the body of the command\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXKVStoreSendCommmandToServers(KVStoreHandle handle,\n                                             int cmd_id,\n                                             const char* cmd_body);\n\n/**\n * \\brief Get the number of ps dead node(s) specified by {node_id}\n *\n * \\param handle handle to the KVStore\n * \\param node_id Can be a node group or a single node.\n *                kScheduler = 1, kServerGroup = 2, kWorkerGroup = 4\n * \\param number Ouptut number of dead nodes\n * \\param timeout_sec A node fails to send heartbeart in {timeout_sec} seconds\n *                    will be presumed as 'dead'\n */\nMXNET_DLL int MXKVStoreGetNumDeadNode(KVStoreHandle handle,\n                                      const int node_id,\n                                      int* number,\n                                      const int timeout_sec DEFAULT(60));\n\n/**\n * \\brief Create a RecordIO writer object\n * \\param uri path to file\n * \\param out handle pointer to the created object\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXRecordIOWriterCreate(const char* uri, RecordIOHandle* out);\n\n/**\n * \\brief Delete a RecordIO writer object\n * \\param handle handle to RecordIO object\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXRecordIOWriterFree(RecordIOHandle handle);\n\n/**\n * \\brief Write a record to a RecordIO object\n * \\param handle handle to RecordIO object\n * \\param buf buffer to write\n * \\param size size of buffer\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXRecordIOWriterWriteRecord(RecordIOHandle handle, const char* buf, size_t size);\n\n/**\n * \\brief Get the current writer pointer position\n * \\param handle handle to RecordIO object\n * \\param pos handle to output position\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXRecordIOWriterTell(RecordIOHandle handle, size_t* pos);\n\n/**\n * \\brief Create a RecordIO reader object\n * \\param uri path to file\n * \\param out handle pointer to the created object\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXRecordIOReaderCreate(const char* uri, RecordIOHandle* out);\n\n/**\n * \\brief Delete a RecordIO reader object\n * \\param handle handle to RecordIO object\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXRecordIOReaderFree(RecordIOHandle handle);\n\n/**\n * \\brief Write a record to a RecordIO object\n * \\param handle handle to RecordIO object\n * \\param buf pointer to return buffer\n * \\param size point to size of buffer\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXRecordIOReaderReadRecord(RecordIOHandle handle, char const** buf, size_t* size);\n\n/**\n * \\brief Set the current reader pointer position\n * \\param handle handle to RecordIO object\n * \\param pos target position\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXRecordIOReaderSeek(RecordIOHandle handle, size_t pos);\n\n/**\n * \\brief Get the current writer pointer position\n * \\param handle handle to RecordIO object\n * \\param pos handle to output position\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXRecordIOReaderTell(RecordIOHandle handle, size_t* pos);\n\n/**\n * \\brief Create a MXRtc object\n */\nMXNET_DLL int MXRtcCreate(char* name,\n                          uint32_t num_input,\n                          uint32_t num_output,\n                          char** input_names,\n                          char** output_names,\n                          NDArrayHandle* inputs,\n                          NDArrayHandle* outputs,\n                          char* kernel,\n                          RtcHandle* out);\n\n/**\n * \\brief Run cuda kernel\n */\nMXNET_DLL int MXRtcPush(RtcHandle handle,\n                        uint32_t num_input,\n                        uint32_t num_output,\n                        NDArrayHandle* inputs,\n                        NDArrayHandle* outputs,\n                        uint32_t gridDimX,\n                        uint32_t gridDimY,\n                        uint32_t gridDimZ,\n                        uint32_t blockDimX,\n                        uint32_t blockDimY,\n                        uint32_t blockDimZ);\n\n/**\n * \\brief Delete a MXRtc object\n */\nMXNET_DLL int MXRtcFree(RtcHandle handle);\n/*\n * \\brief register custom operators from frontend.\n * \\param op_type name of custom op\n * \\param creator\n */\nMXNET_DLL int MXCustomOpRegister(const char* op_type, CustomOpPropCreator creator);\n/*\n * \\brief record custom function for backward later.\n * \\param num_inputs number of input NDArrays.\n * \\param inputs handle to input NDArrays.\n * \\param num_outputs number of output NDArrays.\n * \\param outputs handle to output NDArrays.\n * \\param callbacks callbacks for backward function.\n */\nMXNET_DLL int MXCustomFunctionRecord(int num_inputs,\n                                     NDArrayHandle* inputs,\n                                     int num_outputs,\n                                     NDArrayHandle* outputs,\n                                     struct MXCallbackList* callbacks);\n/*\n * \\brief create cuda rtc module\n * \\param source cuda source code\n * \\param num_options number of compiler flags\n * \\param options compiler flags\n * \\param num_exports number of exported function names\n * \\param exported function names\n * \\param out handle to created module\n */\nMXNET_DLL int MXRtcCudaModuleCreate(const char* source,\n                                    int num_options,\n                                    const char** options,\n                                    int num_exports,\n                                    const char** exports,\n                                    CudaModuleHandle* out);\n/*\n * \\brief delete cuda rtc module\n * \\param handle handle to cuda module\n */\nMXNET_DLL int MXRtcCudaModuleFree(CudaModuleHandle handle);\n/*\n * \\brief get kernel from module\n * \\param handle handle to cuda module\n * \\param name name of kernel function\n * \\param num_args number of arguments\n * \\param is_ndarray whether argument is ndarray\n * \\param is_const whether argument is constant\n * \\param arg_types data type of arguments\n * \\param out created kernel\n */\nMXNET_DLL int MXRtcCudaKernelCreate(CudaModuleHandle handle,\n                                    const char* name,\n                                    int num_args,\n                                    int* is_ndarray,\n                                    int* is_const,\n                                    int* arg_types,\n                                    CudaKernelHandle* out);\n/*\n * \\brief delete kernel\n * \\param handle handle to previously created kernel\n */\nMXNET_DLL int MXRtcCudaKernelFree(CudaKernelHandle handle);\n/*\n * \\brief launch cuda kernel\n * \\param handle handle to kernel\n * \\param dev_id (GPU) device id\n * \\param args pointer to arguments\n * \\param grid_dim_x grid dimension x\n * \\param grid_dim_y grid dimension y\n * \\param grid_dim_z grid dimension z\n * \\param block_dim_x block dimension x\n * \\param block_dim_y block dimension y\n * \\param block_dim_z block dimension z\n * \\param shared_mem size of dynamically allocated shared memory\n */\nMXNET_DLL int MXRtcCudaKernelCall(CudaKernelHandle handle,\n                                  int dev_id,\n                                  void** args,\n                                  uint32_t grid_dim_x,\n                                  uint32_t grid_dim_y,\n                                  uint32_t grid_dim_z,\n                                  uint32_t block_dim_x,\n                                  uint32_t block_dim_y,\n                                  uint32_t block_dim_z,\n                                  uint32_t shared_mem);\n/*!\n * \\brief Get shared memory handle from NDArray\n * \\param handle NDArray handle.\n * \\param shared_pid output PID\n * \\param shared_id output shared memory id.\n */\nMXNET_DLL int MXNDArrayGetSharedMemHandle(NDArrayHandle handle, int* shared_pid, int* shared_id);\n\n/*!\n * \\brief Release all unreferenced memory from the devices storage managers memory pool\n * \\param dev_type device type, specify device we want to take\n * \\param dev_id the device id of the specific device\n */\nMXNET_DLL int MXStorageEmptyCache(int dev_type, int dev_id);\n\n/*!\n * \\brief Reconstruct NDArray from shared memory handle\n * \\param shared_pid shared PID\n * \\param shared_id shared memory id\n * \\param shape pointer to NDArray dimensions\n * \\param ndim number of NDArray dimensions\n * \\param dtype data type of NDArray\n * \\param out constructed NDArray\n */\nMXNET_DLL int MXNDArrayCreateFromSharedMem(int shared_pid,\n                                           int shared_id,\n                                           const int* shape,\n                                           int ndim,\n                                           int dtype,\n                                           NDArrayHandle* out);\n\n/*!\n * \\brief Push an asynchronous operation to the engine.\n * \\param async_func Execution function whici takes a parameter on_complete\n *                   that must be called when the execution ompletes.\n * \\param func_param The parameter set on calling async_func, can be NULL.\n * \\param deleter The callback to free func_param, can be NULL.\n * \\param ctx_handle Execution context.\n * \\param const_vars_handle The variables that current operation will use\n *                          but not mutate.\n * \\param num_const_vars The number of const_vars_handle.\n * \\param mutable_vars_handle The variables that current operation will mutate.\n * \\param num_mutable_vars The number of mutable_vars_handle.\n * \\param prop_handle Property of the function.\n * \\param priority Priority of the action, as hint to the engine.\n * \\param opr_name The operation name.\n * \\param wait Whether this is a WaitForVar operation.\n */\nMXNET_DLL int MXEnginePushAsync(EngineAsyncFunc async_func,\n                                void* func_param,\n                                EngineFuncParamDeleter deleter,\n                                ContextHandle ctx_handle,\n                                EngineVarHandle const_vars_handle,\n                                int num_const_vars,\n                                EngineVarHandle mutable_vars_handle,\n                                int num_mutable_vars,\n                                EngineFnPropertyHandle prop_handle DEFAULT(NULL),\n                                int priority DEFAULT(0),\n                                const char* opr_name DEFAULT(NULL),\n                                bool wait DEFAULT(false));\n\n/*!\n * \\brief Push a synchronous operation to the engine.\n * \\param sync_func Execution function that executes the operation.\n * \\param func_param The parameter set on calling sync_func, can be NULL.\n * \\param deleter The callback to free func_param, can be NULL.\n * \\param ctx_handle Execution context.\n * \\param const_vars_handle The variables that current operation will use\n *                          but not mutate.\n * \\param num_const_vars The number of const_vars_handle.\n * \\param mutable_vars_handle The variables that current operation will mutate.\n * \\param num_mutable_vars The number of mutable_vars_handle.\n * \\param prop_handle Property of the function.\n * \\param priority Priority of the action, as hint to the engine.\n * \\param opr_name The operation name.\n */\nMXNET_DLL int MXEnginePushSync(EngineSyncFunc sync_func,\n                               void* func_param,\n                               EngineFuncParamDeleter deleter,\n                               ContextHandle ctx_handle,\n                               EngineVarHandle const_vars_handle,\n                               int num_const_vars,\n                               EngineVarHandle mutable_vars_handle,\n                               int num_mutable_vars,\n                               EngineFnPropertyHandle prop_handle DEFAULT(NULL),\n                               int priority DEFAULT(0),\n                               const char* opr_name DEFAULT(NULL));\n/*!\n * \\brief Create an NDArray from source sharing the same data chunk.\n * \\param src source NDArray\n * \\param out new NDArray sharing the same data chunck with src\n */\nMXNET_DLL int MXShallowCopyNDArray(NDArrayHandle src, NDArrayHandle* out);\n/*!\n * \\brief Create an Symbol from source sharing the same graph structure.\n * \\param src source Symbol\n * \\param out new Symbol sharing the same graph structure with src\n */\nMXNET_DLL int MXShallowCopySymbol(SymbolHandle src, SymbolHandle* out);\n\n/*!\n * \\brief Push an asynchronous operation to the engine.\n * \\param async_func Execution function whici takes a parameter on_complete\n *                   that must be called when the execution ompletes.\n * \\param func_param The parameter set on calling async_func, can be NULL.\n * \\param deleter The callback to free func_param, can be NULL.\n * \\param ctx_handle Execution context.\n * \\param const_nds_handle The NDArrays that current operation will use\n *                          but not mutate.\n * \\param num_const_nds The number of const_nds_handle.\n * \\param mutable_nds_handle The NDArrays that current operation will mutate.\n * \\param num_mutable_nds The number of mutable_nds_handle.\n * \\param prop_handle Property of the function.\n * \\param priority Priority of the action, as hint to the engine.\n * \\param opr_name The operation name.\n * \\param wait Whether this is a WaitForVar operation.\n */\nMXNET_DLL int MXEnginePushAsyncND(EngineAsyncFunc async_func,\n                                  void* func_param,\n                                  EngineFuncParamDeleter deleter,\n                                  ContextHandle ctx_handle,\n                                  NDArrayHandle* const_nds_handle,\n                                  int num_const_nds,\n                                  NDArrayHandle* mutable_nds_handle,\n                                  int num_mutable_nds,\n                                  EngineFnPropertyHandle prop_handle DEFAULT(NULL),\n                                  int priority DEFAULT(0),\n                                  const char* opr_name DEFAULT(NULL),\n                                  bool wait DEFAULT(false));\n\n/*!\n * \\brief Push a synchronous operation to the engine.\n * \\param sync_func Execution function that executes the operation.\n * \\param func_param The parameter set on calling sync_func, can be NULL.\n * \\param deleter The callback to free func_param, can be NULL.\n * \\param ctx_handle Execution context.\n * \\param const_nds_handle The NDArrays that current operation will use\n *                          but not mutate.\n * \\param num_const_nds The number of const_nds_handle.\n * \\param mutable_nds_handle The NDArrays that current operation will mutate.\n * \\param num_mutable_nds The number of mutable_nds_handle.\n * \\param prop_handle Property of the function.\n * \\param priority Priority of the action, as hint to the engine.\n * \\param opr_name The operation name.\n */\nMXNET_DLL int MXEnginePushSyncND(EngineSyncFunc sync_func,\n                                 void* func_param,\n                                 EngineFuncParamDeleter deleter,\n                                 ContextHandle ctx_handle,\n                                 NDArrayHandle* const_nds_handle,\n                                 int num_const_nds,\n                                 NDArrayHandle* mutable_nds_handle,\n                                 int num_mutable_nds,\n                                 EngineFnPropertyHandle prop_handle DEFAULT(NULL),\n                                 int priority DEFAULT(0),\n                                 const char* opr_name DEFAULT(NULL));\n\n/*!\n * \\brief This function checks if any dynamic shape op is present in the symbol.\n * \\param sym_handle handler of the input symbol.\n * \\param has_dynamic_shape Flag to indicate if the symbol contains dynamic shape op.\n */\nMXNET_DLL int MXCheckDynamicShapeOp(SymbolHandle sym_handle, bool* has_dynamic_shape);\n\n/*!\n * \\brief Synchronize the consumer stream with the producer stream where the NDArray lives.\n * \\param handle NDArray handle of producer.\n * \\param stream A pointer to a stream from consumer.\n */\nMXNET_DLL int MXPushStreamDep(NDArrayHandle handle, int stream);\n\n/*!\n * \\brief Get current stream pointer based on current device type and id\n * \\param device_id Current device id.\n * \\param stream A pointer pointing to current stream.\n */\nMXNET_DLL int MXGetCurrentStream(int device_id, int* stream);\n\n/*!\n * \\brief Push a new NVTX range. Requires building with CUDA and NVTX.\n * \\param name Name of the range.\n * \\param color Color used to display the range in the visual profiling tools.\n *              Encoded as 256*256*R + 256*G + B.\n */\nMXNET_DLL int MXNVTXRangePush(const char* name, mx_uint color);\n\n/*!\n * \\brief End the NVTX range. Requires building with CUDA and NVTX.\n */\nMXNET_DLL int MXNVTXRangePop();\n\n/*!\n * \\brief Start CUDA profiling session. Requires building with CUDA and NVTX.\n */\nMXNET_DLL int MXCUDAProfilerStart();\n\n/*!\n * \\brief End CUDA profiling session. Requires building with CUDA and NVTX.\n */\nMXNET_DLL int MXCUDAProfilerStop();\n\n/*!\n * \\brief Turns on or off Layout Optimization\n */\nMXNET_DLL int MXSetOptimizeLayout(bool val);\n\n/*!\n * \\brief Get current Layout Optimization status\n */\nMXNET_DLL int MXGetOptimizeLayout(bool* val);\n\n#ifdef __cplusplus\n}\n#endif  // __cplusplus\n\n#endif  // MXNET_C_API_H_\n"
  },
  {
    "path": "include/mxnet/c_api_error.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file c_api_error.h\n * \\brief Error handling for C API.\n\n *  WARNING: THIS IS NOT A C API. THE FILE IS TEMPORARILY KEPT UNDER THE NAME\n *  C_API_ERROR.H FOR BACKWARDS COMPATIBILITY REASONS. DO NOT RELY ON THIS FILE\n *  WHEN WRITING NEW CODE.\n */\n#include <string>\n\n#ifndef MXNET_C_API_ERROR_H_\n#define MXNET_C_API_ERROR_H_\n\n/*!\n * \\brief Macros to guard beginning and end section of all functions\n * every function starts with API_BEGIN()\n * and finishes with API_END() or API_END_HANDLE_ERROR()\n * The finally clause contains procedure to cleanup states when an error happens.\n */\n#define MX_API_BEGIN() \\\n  try {                \\\n    on_enter_api(__FUNCTION__);\n#define MX_API_END()                       \\\n  }                                        \\\n  catch (const std::exception& _except_) { \\\n    on_exit_api();                         \\\n    return MXAPIHandleException(_except_); \\\n  }                                        \\\n  on_exit_api();                           \\\n  return 0;  // NOLINT(*)\n#define MX_API_END_HANDLE_ERROR(Finalize)  \\\n  }                                        \\\n  catch (const std::exception& _except_) { \\\n    Finalize;                              \\\n    on_exit_api();                         \\\n    return MXAPIHandleException(_except_); \\\n  }                                        \\\n  on_exit_api();                           \\\n  return 0;  // NOLINT(*)\n\n/*!\n * \\brief Set the last error message needed by C API\n * \\param msg The error message to set.\n */\nvoid MXAPISetLastError(const char* msg);\n/*!\n * \\brief handle exception throwed out\n * \\param e the exception\n * \\return the return value of API after exception is handled\n */\nint MXAPIHandleException(const std::exception& e);\n\nnamespace mxnet {\nextern void on_enter_api(const char* function);\nextern void on_exit_api();\n}\n#endif  // MXNET_C_API_ERROR_H_\n"
  },
  {
    "path": "include/mxnet/c_api_test.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file c_api_test.h\n * \\brief C API of mxnet for ease of testing backend in Python\n */\n#ifndef MXNET_C_API_TEST_H_\n#define MXNET_C_API_TEST_H_\n\n/*! \\brief Inhibit C++ name-mangling for MXNet functions. */\n#ifdef __cplusplus\nextern \"C\" {\n#endif  // __cplusplus\n\n#include <mxnet/c_api.h>\n\n/*!\n * \\brief This API partitions a graph only by the operator names\n * provided by users. This will attach a DefaultSubgraphProperty\n * to the input graph for partitioning. This function should be\n * used only for the testing purpose.\n */\nMXNET_DLL int MXBuildSubgraphByOpNames(SymbolHandle sym_handle,\n                                       const char* prop_name,\n                                       const uint32_t num_ops,\n                                       const char** op_names,\n                                       SymbolHandle* ret_sym_handle);\n\n/*!\n * \\brief Given a subgraph property name, use the provided op names\n * as the op_names attribute for that subgraph property, instead of\n * the predefined one. This is only for the purpose of testing.\n */\nMXNET_DLL int MXSetSubgraphPropertyOpNames(const char* prop_name,\n                                           const uint32_t num_ops,\n                                           const char** op_names);\n\n/*!\n * \\brief Given a subgraph property name, use the provided op names\n * as the op_names attribute for that subgraph property, instead of\n * the predefined one. This is only for the purpose of testing.\n * Compared to MXSetSubgraphPropertyOpNames(), this API will add\n * op_names to the backend property.\n */\nMXNET_DLL int MXSetSubgraphPropertyOpNamesV2(const char* prop_name,\n                                             const uint32_t num_ops,\n                                             const char** op_names);\n/*!\n * \\brief Given a subgraph property name, delete the op name set\n * in the SubgraphPropertyOpNameSet.\n */\nMXNET_DLL int MXRemoveSubgraphPropertyOpNames(const char* prop_name);\n/*!\n * \\brief Given a subgraph property name, remove op_names attribute of\n * the in the SubgraphBackend property.\n */\nMXNET_DLL int MXRemoveSubgraphPropertyOpNamesV2(const char* prop_name);\n\n/*!\n * \\brief Get the value of an environment variable as seen by the backend.\n * \\param name The name of the environment variable\n * \\param value The returned value of the environment variable\n */\nMXNET_DLL int MXGetEnv(const char* name, const char** value);\n\n/*!\n * \\brief Set the value of an environment variable from the backend.\n * \\param name The name of the environment variable\n * \\param value The desired value to set the environment variable `name`\n */\nMXNET_DLL int MXSetEnv(const char* name, const char* value);\n\n/*!\n * \\brief Get the maximum SM architecture supported by the nvrtc compiler\n * \\param max_arch The maximum supported architecture (e.g. would be 80, if Ampere)\n * \\return 0 when success, -1 when failure happens.\n */\nMXNET_DLL int MXGetMaxSupportedArch(uint32_t* max_arch);\n\n#ifdef __cplusplus\n}\n#endif  // __cplusplus\n\n#endif  // MXNET_C_API_TEST_H_\n"
  },
  {
    "path": "include/mxnet/engine.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file engine.h\n * \\brief Engine that schedules all the operations according to dependency.\n */\n#ifndef MXNET_ENGINE_H_\n#define MXNET_ENGINE_H_\n\n#if DMLC_USE_CXX11\n#include <algorithm>\n#include <memory>\n#include <functional>\n#endif\n#include <utility>\n#include <vector>\n#include \"./base.h\"\n\nnamespace mxnet {\n\n// forward declare engine\nclass Engine;\n\n/*! \\brief namespace of engine internal types. */\nnamespace engine {\n#if MXNET_USE_CUDA\n/* \\brief The class wrapping CUDA event with timing disabled. */\nclass CUDAEvent final {\n public:\n  explicit CUDAEvent(Context const& ctx);\n\n  CUDAEvent(CUDAEvent&& other) : event_(other.event_), dev_id_(other.dev_id_) {\n    other.event_ = nullptr;\n  }\n\n  CUDAEvent(const CUDAEvent& other) = delete;\n  void operator=(const CUDAEvent& other) = delete;\n\n  ~CUDAEvent();\n\n  inline std::weak_ptr<cudaEvent_t> GetEvent() noexcept {\n    return event_;\n  }\n\n private:\n  std::shared_ptr<cudaEvent_t> event_;\n  int dev_id_;\n};\n\nclass CUDAEventPool final {\n public:\n  explicit CUDAEventPool(Context const& ctx) : counter_(0) {\n    for (size_t i = 0; i < kPoolSize; ++i) {\n      events_.emplace_back(ctx);\n    }\n  }\n\n  inline std::weak_ptr<cudaEvent_t> GetEvent(size_t i) noexcept {\n    return events_.at(i).GetEvent();\n  }\n\n  inline std::pair<std::weak_ptr<cudaEvent_t>, uint64_t> GetNextEvent() noexcept {\n    uint64_t c = counter_++;\n    return {events_.at((c) % kPoolSize).GetEvent(), c};\n  }\n\n  inline uint64_t GetCounterValue() noexcept {\n    return counter_.load();\n  }\n\n private:\n  static constexpr size_t kPoolSize = 64;\n  std::vector<CUDAEvent> events_;\n  std::atomic<uint64_t> counter_;\n};\n\n/*! \\brief full event info for the sync object.*/\nstruct EventInfo {\n  std::weak_ptr<cudaEvent_t> event;\n  cudaStream_t stream;\n  uint64_t pool_index;\n};\n/*! \\brief struct containing cuda events and variables needed for the dependencies.*/\nstruct SyncObject {\n  // vector can carry multiple reader events\n  std::vector<EventInfo> reader_events;\n  // vector should carry only 1 writer event\n  std::vector<EventInfo> writer_event;\n  std::mutex mutex;\n};\n#endif\n\n/*! \\brief base class of engine variables.*/\nstruct Var {\n  virtual size_t version() {\n    return version_;\n  }\n  virtual ~Var() = default;\n  /*!\n   * \\brief cast variable to derived type T\n   * \\tparam T the type we want to cast into.\n   * \\return A casted variable.\n   */\n  template <typename T>\n  inline T* Cast();\n  /*!\n   * \\brief version number of the var. Every time the object it is associated with\n   * is modified, the version number is incremented by 1.\n   */\n  size_t version_{0};\n#if MXNET_USE_CUDA\n  /*!\n   * \\brief struct containing cuda events and variables needed for the dependencies.\n   */\n  SyncObject sync_object;\n#endif\n};  // struct Var\n\n/*! \\brief Internal representation of operator.  */\nstruct Opr;\n/*! \\brief Variable pointer type, usually hold by user used to specify dependencies. */\ntypedef Var* VarHandle;\n/*! \\brief Operator pointer type, usually hold by user.*/\ntypedef Opr* OprHandle;\n/*!\n * \\brief OnStart callback to the engine,\n *  called by AsyncFn before the action\n */\nclass CallbackOnStart {\n public:\n  // use implicit copy and assign\n  /*! \\brief involve the callback */\n  inline void operator()(const dmlc::Error* error = nullptr) const {\n    if (callback_ != nullptr)\n      (*callback_)(engine_, param_, error);\n  }\n\n private:\n  /*! \\brief engine can see content of callback */\n  friend class ::mxnet::Engine;\n  /*! \\brief the real callback */\n  void (*callback_)(Engine*, void*, const dmlc::Error*);\n  /*! \\brief the engine class passed to callback */\n  Engine* engine_;\n  /*! \\brief the parameter set on callback */\n  void* param_;\n};\n/*!\n * \\brief OnComplete Callback to the engine,\n *  called by AsyncFn when action completes\n */\nclass CallbackOnComplete {\n public:\n  // use implicit copy and assign\n  /*! \\brief involve the callback */\n  inline void operator()(const dmlc::Error* error = nullptr) const {\n    (*callback_)(engine_, param_, error);\n  }\n\n private:\n  /*! \\brief engine can see content of callback */\n  friend class ::mxnet::Engine;\n  /*! \\brief the real callback */\n  void (*callback_)(Engine*, void*, const dmlc::Error*);\n  /*! \\brief the engine class passed to callback */\n  Engine* engine_;\n  /*! \\brief the parameter set on callback */\n  void* param_;\n};\n}  // namespace engine\n\n#if DMLC_USE_CXX11\n/*! \\brief Function property, used to hint what action is pushed to engine. */\nenum class FnProperty {\n  /*! \\brief Normal operation */\n  kNormal,\n  /*! \\brief Copy operation from GPU to other devices */\n  kCopyFromGPU,\n  /*! \\brief Copy operation from CPU to other devices */\n  kCopyToGPU,\n  /*! \\brief Prioritized sync operation on CPU */\n  kCPUPrioritized,\n  /*! \\brief Asynchronous function call */\n  kAsync,\n  /*! \\brief Delete variable call */\n  kDeleteVar,\n  /*! \\brief Prioritized sync operation on GPU */\n  kGPUPrioritized,\n  /*! \\brief Operation not to be skipped even with associated exception */\n  kNoSkip\n};  // enum class FnProperty\n\n/*!\n * \\brief Dependency engine that schedules operations.\n */\nclass MXNET_API Engine {\n public:\n  /*! \\brief on start*/\n  typedef engine::CallbackOnStart CallbackOnStart;\n  /*! \\brief callback on complete*/\n  typedef engine::CallbackOnComplete CallbackOnComplete;\n  /*! \\brief Synchronous operation to pass to engine. */\n  typedef std::function<void(RunContext)> SyncFn;\n  /*! \\brief Asynchronous operation to pass to engine. */\n  typedef std::function<void(RunContext, CallbackOnStart, CallbackOnComplete)> AsyncFn;\n  /*! \\brief Variable pointer */\n  typedef engine::VarHandle VarHandle;\n  /*! \\brief Operator pointer */\n  typedef engine::OprHandle OprHandle;\n  /*!\n   * \\brief Notify the engine about a shutdown,\n   *  This can help engine to print less messages into display.\n   *\n   *  User do not have to call this function.\n   * \\return 0 when success, -1 when failure happens.\n   */\n  virtual void NotifyShutdown() = 0;\n  /*!\n   *\\brief Stop all workers in the engine\n   */\n  virtual void Stop() {\n    LOG(FATAL) << \"Engine cannot be stopped\";\n  }\n  /*!\n   * \\brief Restart all workers in the engine\n   */\n  virtual void Start() {\n    LOG(FATAL) << \"Engine cannot be restarted\";\n  }\n  /*!\n   * \\brief Allocate a new variable, the variable can then\n   *        be used to schedule the operation concurrently via dependency\n   *        patterns.\n   * \\return The new variable allocated.\n   */\n  virtual VarHandle NewVariable() = 0;\n  /*!\n   * \\brief Create a new operator. The returned operator could be saved\n   *        externally so that it could be resued for scheduling.\n   * \\param fn The execution function.\n   * \\param const_vars The variables that current operation will use but not\n   *                   mutate.\n   * \\param mutable_vars The variables that current operation will mutate.\n   * \\param prop Property of the function.\n   * \\param opr_name The operator name.\n   * \\param wait Whether this is a WaitForVar operation\n   * \\return The new operator allocated.\n   */\n  virtual OprHandle NewOperator(AsyncFn fn,\n                                std::vector<VarHandle> const& const_vars,\n                                std::vector<VarHandle> const& mutable_vars,\n                                FnProperty prop      = FnProperty::kNormal,\n                                const char* opr_name = nullptr,\n                                bool wait            = false) = 0;\n  /*!\n   * \\brief Delete the given operator.\n   * \\param op The operator to delete.\n   *\n   * The delete will not happen immediately, but will wait until all the\n   * operations using this operator are completed.\n   */\n  virtual void DeleteOperator(OprHandle op) = 0;\n  /*!\n   * \\brief Push an operator to the engine.\n   * \\param op The operator to push.\n   * \\param exec_ctx Execution context.\n   * \\param priority Priority of the action, as hint to the engine.\n   * \\param profiling The variable indicate whether to profile this operator.\n   */\n  virtual void Push(OprHandle op, Context exec_ctx, int priority = 0, bool profiling = false) = 0;\n  /*!\n   * \\brief Push an asynchronous operation to the engine.\n   * \\param exec_fun Execution function, this function takes a parameter\n   *                 on_complete that must be called when the execution\n   *                 completes.\n   * \\param exec_ctx Execution context.\n   * \\param const_vars The variables that current operation will use but not\n   *                   mutate.\n   * \\param mutable_vars The variables that current operation will mutate.\n   * \\param prop Property of the function.\n   * \\param priority Priority of the action, as hint to the engine.\n   * \\param opr_name The operator name.\n   * \\param wait Whether this is a WaitForVar operation\n   */\n  virtual void PushAsync(AsyncFn exec_fun,\n                         Context exec_ctx,\n                         std::vector<VarHandle> const& const_vars,\n                         std::vector<VarHandle> const& mutable_vars,\n                         FnProperty prop      = FnProperty::kNormal,\n                         int priority         = 0,\n                         const char* opr_name = nullptr,\n                         bool wait            = false) = 0;\n  /*!\n   * \\brief Schedule the deletion of a variable.\n   *\n   * The delete will not happen immediately, but will wait until all the\n   * operations depending on var are completed.\n   *\n   * \\param delete_fn A function that will be called after the variable is\n   *                   deleted.\n   * \\param exec_ctx Execution context.\n   * \\param var The variable to be deleted.\n   */\n  virtual void DeleteVariable(SyncFn delete_fn, Context exec_ctx, VarHandle var) = 0;\n  /*!\n   * \\brief Wait for a variable.\n   * \\param var The variable we should wait for. This function returns when the\n   *            variable is ready.\n   */\n  virtual void WaitForVar(VarHandle var) = 0;\n  /*!\n   * \\brief Wait until all the activity of engine finishes.\n   */\n  virtual void WaitForAll() = 0;\n  /*!\\brief Throw if threre are associated exception with var */\n  virtual void Throw(VarHandle var) = 0;\n  /*!\\brief virtual destructor */\n  virtual ~Engine() noexcept(false) {}\n  /*!\n   * \\return Engine singleton.\n   */\n  static Engine* Get();\n  /*!\n   * \\brief Get shared pointer reference to engine singleton.\n   *  Most user should not call this function.\n   *  This function is called by another singleton X who requires\n   *  engine to be destructed after X.\n   *\n   * \\return A shared pointer to Engine singleton.\n   */\n  static const std::shared_ptr<Engine>& _GetSharedRef();\n  /*!\n   * \\brief Push an synchronous operation to the engine.\n   * \\param exec_fn Execution function that executes the operation.\n   * \\param exec_ctx Execution context.\n   * \\param const_vars The variables that current operation will use but not\n   *                   mutate.\n   * \\param mutable_vars The variables that current operation will mutate.\n   * \\param prop Property of the function.\n   * \\param priority Priority of the action, as hint to the engine.\n   * \\param opr_name The operator name.\n   * \\tparam SyncFn the synchronous function to be pushed.\n   */\n  virtual void PushSync(SyncFn exec_fn,\n                        Context exec_ctx,\n                        std::vector<VarHandle> const& const_vars,\n                        std::vector<VarHandle> const& mutable_vars,\n                        FnProperty prop      = FnProperty::kNormal,\n                        int priority         = 0,\n                        const char* opr_name = nullptr) {\n    this->PushAsync(\n        [exec_fn](RunContext ctx, CallbackOnStart on_start, CallbackOnComplete on_complete) {\n          on_start();\n          exec_fn(ctx);\n          on_complete();\n        },\n        exec_ctx,\n        const_vars,\n        mutable_vars,\n        prop,\n        priority,\n        opr_name);\n  }\n\n  /*!\n   * \\brief factory function to create OnStart callback.\n   * \\param callback th static callback function.\n   * \\param param the paramter passed to callback.\n   */\n  inline CallbackOnStart CreateOnStart(void (*callback)(Engine*, void*, const dmlc::Error*),\n                                       void* param) {\n    CallbackOnStart ret;\n    ret.callback_ = callback;\n    ret.engine_   = this;\n    ret.param_    = param;\n    return ret;\n  }\n\n  /*!\n   * \\brief factory function to create OnComplete callback.\n   * \\param callback th static callback function.\n   * \\param param the paramter passed to callback.\n   */\n  inline CallbackOnComplete CreateCallback(void (*callback)(Engine*, void*, const dmlc::Error*),\n                                           void* param) {\n    CallbackOnComplete ret;\n    ret.callback_ = callback;\n    ret.engine_   = this;\n    ret.param_    = param;\n    return ret;\n  }\n  // For each var vector, sort it and remove the duplicated vars.\n  // Also remove vars from read_vars if it also appears in write_vars\n  inline void DeduplicateVarHandle(std::vector<engine::VarHandle>* read_vars,\n                                   std::vector<engine::VarHandle>* write_vars) {\n    std::sort(write_vars->begin(), write_vars->end());\n    write_vars->resize(std::unique(write_vars->begin(), write_vars->end()) - write_vars->begin());\n    std::sort(read_vars->begin(), read_vars->end());\n    read_vars->resize(std::unique(read_vars->begin(), read_vars->end()) - read_vars->begin());\n    auto wit  = write_vars->begin();\n    auto rtop = read_vars->begin();\n    for (auto rit = read_vars->begin(); rit != read_vars->end(); ++rit) {\n      while (wit != write_vars->end() && *wit < *rit)\n        ++wit;\n      if (wit == write_vars->end() || *wit != *rit) {\n        *rtop = *rit;\n        ++rtop;\n      }\n    }\n    read_vars->resize(rtop - read_vars->begin());\n  }\n  /*! \\brief query current limit for bulk size */\n  virtual int bulk_size() const {\n    return 0;\n  }\n  /*! \\brief set maximum limit for bulk size */\n  virtual int set_bulk_size(int) {\n    return 0;\n  }\n};      // class Engine\n#endif  // DMLC_USE_CXX11\n}  // namespace mxnet\n#endif  // MXNET_ENGINE_H_\n"
  },
  {
    "path": "include/mxnet/executor.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file executor.h\n * \\brief Symbolic executor interface of mxnet.\n * \\author Min Lin, Bing Xu\n */\n#ifndef MXNET_EXECUTOR_H_\n#define MXNET_EXECUTOR_H_\n\n#include <dmlc/base.h>\n#include <vector>\n#include <memory>\n#include <map>\n#include <string>\n#include <utility>\n#include \"./base.h\"\n#include \"./c_api.h\"\n#include \"./ndarray.h\"\n#include \"./operator.h\"\n\n// check c++11\n#if DMLC_USE_CXX11 == 0\n#error \"CXX11 was required for symbolic module\"\n#endif\n\nnamespace mxnet {\n/*! \\brief use symbolic graph from NNVM */\nusing nnvm::Symbol;\n\n/*!\n * \\brief Executor of a computation graph.\n *  Executor can be created by Binding a symbol.\n */\nclass Executor {\n public:\n  /*! \\brief destructor */\n  virtual ~Executor() {}\n  /*!\n   * \\brief Perform a Forward operation of Operator\n   *  After this operation, user can get the result by using function head.\n   */\n  virtual void Forward(bool is_train) = 0;\n  /*!\n   * \\brief Perform a Partial Forward operation of Operator.\n   *  Only issue operation specified by step.\n   *  The caller must keep calling PartialForward with increasing steps, until step_left=0.\n   * \\param is_train Whether this is training phase.\n   * \\param step current step, user can always start from 0\n   * \\param step_left Number of steps left to finish the forward.\n   */\n  virtual void PartialForward(bool is_train, int step, int* step_left) = 0;\n  /*!\n   * \\brief Perform a Backward operation of the Operator.\n   *  This must be called after Forward.\n   *  After this operation, NDArrays specified by grad_in_args_store will be updated accordingly.\n   *  User is allowed to pass in an empty Array if the head node is\n   *  loss function and head gradeitn is not needed.\n   *\n   * \\param head_grads the gradient of head nodes to be backproped.\n   */\n  virtual void Backward(const std::vector<NDArray>& head_grads, bool is_train = true) = 0;\n  /*!\n   * \\brief print the execution plan info to output stream.\n   * \\param os the output stream we like to print to.\n   */\n  virtual void Print(std::ostream& os) const {}  // NOLINT(*)\n  /*!\n   * \\brief get array of outputs in the executor.\n   * \\return array of outputs in the executor.\n   */\n  virtual const std::vector<NDArray>& outputs() const = 0;\n  /*!\n   * \\brief get input argument map, key is arg name, value is arg's NDArray.\n   * \\return input argument map in the executor.\n   */\n  virtual const std::unordered_map<std::string, NDArray>& in_arg_map() const = 0;\n  /*!\n   * \\brief get input argument graident map, key is arg name, value is gradient's NDArray.\n   * \\return input argument gradient map in the executor.\n   */\n  virtual const std::unordered_map<std::string, NDArray>& arg_grad_map() const = 0;\n  /*!\n   * \\brief get aux state map, key is arg name, value is aux state's NDArray.\n   * \\return aux state map in the executor.\n   */\n  virtual const std::unordered_map<std::string, NDArray>& aux_state_map() const = 0;\n  /*!\n   * \\brief Return a new executor with the same symbol and shared memory,\n   *  but different input/output shapes.\n   *\n   * \\param partial_shaping Whether to allow changing the shape of unspecified arguments.\n   * \\param allow_up_sizing Whether to allow allocating new ndarrays that's larger than the\n   *  original.\n   * \\param default_ctx the default context of binding.\n   * \\param ctx_map Context mapping group to context.\n   * \\param provided_arg_shapes New shape for arguments.\n   * \\param in_args the NDArray that stores the input arguments.\n   * \\param arg_grads NDArray that is used to store the gradient output of the input arguments.\n   * \\param aux_states NDArray that is used as internal states.\n   * \\return a new executor.\n   */\n  virtual Executor* Reshape(\n      const bool partial_shaping,\n      const bool allow_up_sizing,\n      const Context& default_ctx,\n      const std::map<std::string, Context>& ctx_map,\n      const std::unordered_map<std::string, mxnet::TShape>& provided_arg_shapes,\n      std::vector<NDArray>* in_args,\n      std::vector<NDArray>* arg_grads,\n      std::vector<NDArray>* aux_states) = 0;\n  /*!\n   * \\brief Create an operator by bind symbol with context and arguments.\n   *  If user do not want to compute the gradients of i-th argument, grad_req_type[i] can be\n   * kNullOp.\n   *\n   * \\param default_ctx the default context of binding.\n   * \\param group2ctx Context mapping group to context.\n   * \\param symbol the symbol that specifies the output of Forward pass.\n   * \\param in_args the NDArray that stores the input arguments to the symbol.\n   * \\param arg_grad_store NDArray that is used to store the gradient\n   *  output of the input arguments.\n   * \\param grad_req_type requirment type of gradient saving. Can only be in\n   *  {kNullOp, kAddTo, kWriteTo}.\n   * \\param aux_states NDArray that is used as internal state in op\n   * \\param shared_exec input executor to share memory with.\n   * \\return a new executor.\n   */\n  static Executor* Bind(nnvm::Symbol symbol,\n                        const Context& default_ctx,\n                        const std::map<std::string, Context>& group2ctx,\n                        const std::vector<NDArray>& in_args,\n                        const std::vector<NDArray>& arg_grad_store,\n                        const std::vector<OpReqType>& grad_req_type,\n                        const std::vector<NDArray>& aux_states,\n                        Executor* shared_exec = nullptr);\n\n  static Executor* SimpleBind(\n      nnvm::Symbol symbol,\n      const Context& default_ctx,\n      const std::map<std::string, Context>& group2ctx,\n      const std::vector<Context>& in_arg_ctxes,\n      const std::vector<Context>& arg_grad_ctxes,\n      const std::vector<Context>& aux_state_ctxes,\n      const std::unordered_map<std::string, mxnet::TShape>& arg_shape_map,\n      const std::unordered_map<std::string, int>& arg_dtype_map,\n      const std::unordered_map<std::string, int>& arg_stype_map,\n      const std::vector<OpReqType>& grad_req_types,\n      const std::unordered_set<std::string>& param_names,\n      std::vector<NDArray>* in_args,\n      std::vector<NDArray>* arg_grads,\n      std::vector<NDArray>* aux_states,\n      std::unordered_map<std::string, NDArray>* shared_data_arrays = nullptr,\n      Executor* shared_exec                                        = nullptr);\n\n  /*!\n   * \\brief the prototype of user-defined monitor callback\n   */\n  typedef std::function<void(const char*, void*)> MonitorCallback;\n  /*!\n   * \\brief Install a callback to notify the completion of operation.\n   */\n  virtual void SetMonitorCallback(const MonitorCallback& callback, bool monitor_all = false) {}\n};  // class executor\n}  // namespace mxnet\n#endif  // MXNET_EXECUTOR_H_\n"
  },
  {
    "path": "include/mxnet/expr_operator.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file expr_operator.h\n * \\brief Common operators defined for Expr.\n *\n * \\note Most of the operator defined here perform simple constant folding\n *   when the type is int32 or int64 for simplifying the index expressions.\n */\n// Acknowledgement: This file originates from incubator-tvm\n// Acknowledgement: Most operator APIs originate from Halide.\n#ifndef MXNET_EXPR_OPERATOR_H_\n#define MXNET_EXPR_OPERATOR_H_\n\n#include <mxnet/ir/expr.h>\n\nnamespace mxnet {\n\ntemplate <typename ValueType>\ninline PrimExpr MakeConstScalar(MXNetDataType t, ValueType value) {\n  if (t.is_int())\n    return IntImm(t, static_cast<int64_t>(value));\n  if (t.is_float())\n    return FloatImm(t, static_cast<double>(value));\n  // customized type and uint is not supported for MXNet for now\n  LOG(FATAL) << \"cannot make const for type \" << t;\n  return PrimExpr();\n}\n\ntemplate <typename ValueType>\ninline PrimExpr make_const(MXNetDataType t, ValueType value) {\n  if (t.lanes() == 1) {\n    return MakeConstScalar(t, value);\n  } else {\n    LOG(FATAL) << \"MXNetDataType::lanes() != 1 is not supported \";\n  }\n  return PrimExpr();\n}\n\n}  // namespace mxnet\n\n#endif  // MXNET_EXPR_OPERATOR_H_\n"
  },
  {
    "path": "include/mxnet/graph_attr_types.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file graph_attr_types.h\n * \\brief Data structures that can appear in graph attributes.\n */\n#ifndef MXNET_GRAPH_ATTR_TYPES_H_\n#define MXNET_GRAPH_ATTR_TYPES_H_\n\n#include <mxnet/op_attr_types.h>\n#include <vector>\n\nnamespace mxnet {\n\n/*!\n * \\brief The result holder of storage type of each NodeEntry in the graph.\n * \\note Stored under graph.attrs[\"storage_type\"], provided by Pass \"InferStorageType\"\n *\n * \\code\n *  Graph g = ApplyPass(src_graph, \"InferStorageType\");\n *  const StorageVector& stypes = g.GetAttr<StorageTypeVector>(\"storage_type\");\n *  // get storage type by entry id\n *  int entry_type = stypes[g.indexed_graph().entry_id(my_entry)];\n * \\endcode\n *\n * \\sa FInferStorageType\n */\nusing StorageTypeVector = std::vector<int>;\n\n/*!\n+ * \\brief The result holder of dispatch mode of each Node in the graph.\n+ * \\note Stored under graph.attrs[\"dispatch_mode\"], provided by Pass \"InferStorageType\"\n+ *\n+ * \\code\n+ *  Graph g = ApplyPass(src_graph, \"InferStorageType\");\n+ *  const DispatchModeVector& dispatch_modes = g.GetAttr<DispatchModeVector>(\"dispatch_mode\");\n+ *  // get dispatch mode by entry node id\n+ *  int node_type = dispatch_modes[nid];\n+ * \\endcode\n+ *\n+ * \\sa FInferStorageType\n+ */\nusing DispatchModeVector = std::vector<DispatchMode>;\n\n}  // namespace mxnet\n\n#endif  // MXNET_GRAPH_ATTR_TYPES_H_\n"
  },
  {
    "path": "include/mxnet/imperative.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_IMPERATIVE_H_\n#define MXNET_IMPERATIVE_H_\n\n#include <mxnet/op_attr_types.h>\n#include <mxnet/graph_attr_types.h>\n#include <mxnet/c_api.h>\n#include <nnvm/symbolic.h>\n#include <nnvm/op.h>\n#include <nnvm/graph.h>\n#include <vector>\n#include <atomic>\n#include <utility>\n#include <string>\n#include <unordered_map>\n\n#include \"./ndarray.h\"\n\nnamespace mxnet {\n\nconstexpr char OPT_CONSTRAINT_ATTR[] = \"__opt_constraint__\";\nenum class OptConstraint : unsigned int {\n  None       = 0,\n  DisableAMP = 1 << 0\n  // DisableQuantization = 1 << 1\n};\nusing OptConstraint_int_t = std::underlying_type_t<OptConstraint>;\n\n/*! \\brief there are three numpy shape flags based on priority.\n * GlobalOn\n *   turn on numpy shape flag globally, it includes thread local.\n *   The flag can be seen in any thread.\n * ThreadLocalOn\n *   only turn on thread local numpy shape flag, it cannot be seen\n *   in other threads.\n * Off\n *   turn off numpy shape flag globally.\n * */\nenum NumpyShape { Off, ThreadLocalOn, GlobalOn };\ntypedef NumpyShape NumpyDefaultDtype;\n\n/*! \\brief runtime functions for NDArray */\nclass Imperative {\n public:\n  /*! \\brief */\n  class AGInfo {\n   public:\n    Context ctx;\n    OpReqType grad_req;\n    OpStatePtr state;\n    std::vector<NDArray> outputs;\n    std::vector<NDArray> out_grads;  // used to hold gradient arrays the user is\n                                     // interested in (marked variables)\n    bool fresh_out_grad;\n\n    AGInfo() : grad_req(kNullOp), fresh_out_grad(false) {}\n\n    static void Clear(const nnvm::ObjectPtr& node) {\n      if (node == nullptr || node->info.empty())\n        return;\n      AGInfo& info = Get(node);\n      if (info.grad_req != kNullOp)\n        return;\n      node->info.clear();\n    }\n\n    static AGInfo& Get(const nnvm::ObjectPtr& node) {\n      return dmlc::get<AGInfo>(node->info);\n    }\n\n    static AGInfo& Create(const nnvm::ObjectPtr& node) {\n      node->info.construct<AGInfo>();\n      return Get(node);\n    }\n\n    static bool IsNone(const NDArray& arr) {\n      return arr.autograd_entry_.node == nullptr || arr.autograd_entry_.node->info.empty();\n    }\n\n    static bool IsVariable(const nnvm::ObjectPtr& node) {\n      AGInfo& info = Get(node);\n      return info.grad_req != kNullOp && info.outputs.size() == 1 && info.out_grads.size() == 1;\n    }\n  };\n\n  /*! \\brief DCInfo datastructure to enable deferred computation */\n  class DCInfo {\n   public:\n    explicit DCInfo(const std::vector<NDArray*>& inputs, const std::vector<NDArray*>& outputs);\n\n    /*! \\brief Compute the outputs of the associated operator. */\n    static void Compute(const NDArray& arr);\n\n    static DCInfo& Get(const nnvm::ObjectPtr& node) {\n      return dmlc::get<DCInfo>(node->info);\n    }\n\n    static bool IsNone(const NDArray& arr) {\n      return arr.deferredcompute_entry_.node == nullptr ||\n             arr.deferredcompute_entry_.node->info.empty();\n    }\n\n    static bool IsComputed(const NDArray& arr) {\n      return IsNone(arr) || dmlc::get<DCInfo>(arr.deferredcompute_entry_.node->info).is_computed_;\n    }\n\n    static DCInfo& Create(const nnvm::ObjectPtr& node,\n                          const std::vector<NDArray*>& inputs,\n                          const std::vector<NDArray*>& outputs);\n\n    static void Clear(const nnvm::ObjectPtr& node) {\n      if (node == nullptr || node->info.empty())\n        return;\n      node->info.clear();\n    }\n\n   private:\n    friend class Imperative;\n\n    /*! \\brief Copies of input NDArrays\n     *\n     * If respective input NDArray is deallocated on the frontend, we still need\n     * to keep a copy around to facilitate deferred computation of this array.\n     * The copies share the chunk.\n     *\n     * They are automatically deallocated after computation finished.\n     */\n    std::vector<NDArray> inputs_;\n\n    /*! \\brief Handles of input NDArrays used by frontend\n     *\n     * Frontend may request conversion to Symbol, specifying a list of NDArray\n     * handles corresponding to inputs and outputs of the Symbol. We store the\n     * handles used by frontend to facilitate matching in\n     * GetDeferredComputeSymbol.\n     *\n     * Note that the frontend may have deallocated the NDArray* and the\n     * input_handles stored here may point to invalid memory.\n     */\n    std::vector<const NDArray*> input_handles_;\n\n    /*! \\brief Copies of output NDArrays\n     *\n     * If respective output NDArray is deallocated on the frontend, we still\n     * need to keep a copy around to facilitate deferred computation of arrays\n     * relying on the output array. The copies share the chunk.\n     *\n     * They are automatically deallocated after computation finished.\n     */\n    std::vector<NDArray> outputs_;\n\n    /*! \\brief Remember if the outputs associated with this DCInfo have been computed already */\n    bool is_computed_ = false;\n  };\n\n  /*! \\brief whether operator recording is on. */\n  bool is_training() const {\n    return is_train_;\n  }\n  /*! \\brief turn on or turn off operator recording for autograd. */\n  bool set_is_training(bool is_train) {\n    bool old  = is_train_;\n    is_train_ = is_train;\n    return old;\n  }\n  /*! \\brief whether operator recording is on. */\n  bool is_recording() const {\n    return is_recording_;\n  }\n  /*! \\brief turn on or turn off operator recording for autograd. */\n  bool set_is_recording(bool is_recording) {\n    bool old      = is_recording_;\n    is_recording_ = is_recording;\n    return old;\n  }\n  /*! \\brief whether deferred compute mode is on. */\n  bool is_deferred_compute() const {\n    return is_deferred_compute_;\n  }\n  /*! \\brief turn on or turn off operator recording for autograd. */\n  bool set_is_deferred_compute(bool is_deferred_compute) {\n    bool old             = is_deferred_compute_;\n    is_deferred_compute_ = is_deferred_compute;\n    return old;\n  }\n  /*! \\brief return current numpy compatibility status,\n   *  GlobalOn(2), ThreadLocalOn(1), Off(0).\n   * */\n  int is_np_shape() const {\n    if (is_np_shape_global_) {\n      return NumpyShape::GlobalOn;\n    }\n    return is_np_shape_thread_local_ ? NumpyShape::ThreadLocalOn : NumpyShape::Off;\n  }\n  /*! \\brief specify numpy compatibility off, thread local on or global on. */\n  bool set_is_np_shape(int is_np_shape) {\n    NumpyShape flag = static_cast<NumpyShape>(is_np_shape);\n    bool old        = this->is_np_shape();\n    switch (flag) {\n      case GlobalOn:\n        is_np_shape_global_       = true;\n        is_np_shape_thread_local_ = true;\n        break;\n      case ThreadLocalOn:\n        is_np_shape_thread_local_ = true;\n        break;\n      case Off:\n        is_np_shape_global_       = false;\n        is_np_shape_thread_local_ = false;\n        break;\n    }\n    return old;\n  }\n  /*! \\brief return current numpy default dtype compatibility status.\n   * */\n  bool is_np_default_dtype() const {\n    if (is_np_default_dtype_global_) {\n      return true;\n    }\n    return false;\n  }\n  /*! \\brief specify numpy default dtype off or global on. */\n  bool set_is_np_default_dtype(bool is_np_default_dtype) {\n    bool old = this->is_np_default_dtype();\n    if (is_np_default_dtype) {\n      is_np_default_dtype_global_ = true;\n    } else {\n      is_np_default_dtype_global_ = false;\n    }\n    return old;\n  }\n  /*! \\brief return current optimization constraints. */\n  OptConstraint get_opt_constraints() const {\n    return opt_constraints_;\n  }\n  /*! \\brief set optimization constraints. */\n  OptConstraint set_opt_constraints(OptConstraint constraints) {\n    OptConstraint old = opt_constraints_;\n    opt_constraints_  = constraints;\n    return old;\n  }\n  /*! \\brief to record operator, return corresponding node. */\n  void RecordOp(nnvm::NodeAttrs&& attrs,\n                const std::vector<NDArray*>& inputs,\n                const std::vector<NDArray*>& outputs,\n                const OpStatePtr& state           = OpStatePtr(),\n                std::vector<bool>* p_save_inputs  = nullptr,\n                std::vector<bool>* p_save_outputs = nullptr);\n  /*! \\brief to record operator, return corresponding node. */\n  void RecordDeferredCompute(nnvm::NodeAttrs&& attrs,\n                             const std::vector<NDArray*>& inputs,\n                             const std::vector<NDArray*>& outputs);\n  /*! \\brief obtain symbol representation of deferred compute session. */\n  nnvm::Symbol GetDeferredComputeSymbol(const std::vector<NDArray*>& outputs);\n  /*! \\brief associate arrays with variables for deferred compute */\n  void SetDeferredComputeVariable(NDArrayHandle* arrays, SymbolHandle* variables, const int num);\n  /*! \\brief clear info node associated with array */\n  void DeferredComputeClear(NDArrayHandle* arrays, const int num);\n  /*! \\brief */\n  OpStatePtr Invoke(const Context& default_ctx,\n                    const nnvm::NodeAttrs& attrs,\n                    const std::vector<NDArray*>& inputs,\n                    const std::vector<NDArray*>& outputs);\n  /*! \\brief */\n  OpStatePtr InvokeOp(const Context& ctx,\n                      const nnvm::NodeAttrs& attrs,\n                      const std::vector<NDArray*>& inputs,\n                      const std::vector<NDArray*>& outputs,\n                      const std::vector<OpReqType>& req,\n                      const DispatchMode dispatch_mode,\n                      OpStatePtr state = OpStatePtr());\n  /*! \\brief mark variables for computing gradients. */\n  void MarkVariables(const std::vector<NDArray*>& variables,\n                     const std::vector<uint32_t>& grad_reqs,\n                     const std::vector<NDArray*>& gradients);\n  /*! \\brief unmark nonleaf variables to free the memory. */\n  void DropGrads(const std::vector<NDArray*>& variables);\n  /*! \\brief compute the gradient of outputs w.r.t variables. */\n  std::vector<NDArray*> Backward(const std::vector<NDArray*>& outputs,\n                                 const std::vector<NDArray*>& ograds,\n                                 const std::vector<NDArray*>& variables,\n                                 bool is_train,\n                                 bool retain_graph,\n                                 bool create_graph);\n  /*! \\brief Return the marked nonleaf nodes. */\n  std::vector<nnvm::ObjectPtr> ListNonleafVariables(const nnvm::Symbol& sym) const;\n  /*! \\return AutogradRuntime singleton */\n  static Imperative* Get();\n  /*! \\brief Should op execution bulking be employed during inference. */\n  static bool PreferBulkExecInference() {\n    return dmlc::GetEnv(\"MXNET_EXEC_BULK_EXEC_INFERENCE\", true);\n  }\n  /*! \\brief Should op execution bulking be employed during training. */\n  static bool PreferBulkExecTrain() {\n    return dmlc::GetEnv(\"MXNET_EXEC_BULK_EXEC_TRAIN\", true);\n  }\n  /*! \\brief The max number of op nodes in a bulk during forward pass of training. */\n  static int BulkExecMaxNodeTrainFwd() {\n    return dmlc::GetEnv(\"MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_FWD\",\n                        dmlc::GetEnv(\"MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN\", 15));\n  }\n  /*! \\brief The max number of op nodes in a bulk during backward pass of training. */\n  static int BulkExecMaxNodeTrainBwd() {\n    return dmlc::GetEnv(\"MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_BWD\",\n                        dmlc::GetEnv(\"MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN\", 15));\n  }\n\n private:\n  friend class NDArray;\n  /*! \\brief make constructor protected. */\n  Imperative() {\n    if (PreferBulkExecTrain())\n      backward_bulk_size_ = BulkExecMaxNodeTrainBwd();\n  }\n  /*! \\brief find the input/output ndarrays that are needed for backward */\n  void GetBackwardDependency(const nnvm::ObjectPtr& node,\n                             uint32_t num_inputs,\n                             uint32_t num_outputs,\n                             std::vector<bool>* p_save_inputs,\n                             std::vector<bool>* p_save_outputs);\n  /*! \\brief indicate whether is training. */\n#if DMLC_CXX11_THREAD_LOCAL\n  static thread_local bool is_train_;\n  static thread_local bool is_recording_;\n  static thread_local bool is_deferred_compute_;\n  static thread_local OptConstraint opt_constraints_;\n  // TOOD(junwu): Added numpy compatibility switch for backward compatibility.\n  // Delete it in the next major release.\n  static thread_local bool is_np_shape_thread_local_;\n#else\n  static MX_THREAD_LOCAL bool is_train_;\n  static MX_THREAD_LOCAL bool is_recording_;\n  static MX_THREAD_LOCAL bool is_deferred_compute_;\n  static MX_THREAD_LOCAL OptConstraint opt_constraints_;\n  // TOOD(junwu): Added numpy compatibility switch for backward compatibility.\n  // Delete it in the next major release.\n  static MX_THREAD_LOCAL bool is_np_shape_thread_local_;\n#endif\n  bool is_np_shape_global_{false};\n  bool is_np_default_dtype_global_{false};\n  /*! \\brief node count used for naming */\n  std::atomic<uint64_t> node_count_{0};\n  /*! \\brief variable count used for naming */\n  std::atomic<uint64_t> variable_count_{0};\n  /*! \\brief default backward bulk size */\n  int backward_bulk_size_{0};\n};\n\n}  // namespace mxnet\n#endif  // MXNET_IMPERATIVE_H_\n"
  },
  {
    "path": "include/mxnet/io.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file io.h\n * \\brief mxnet io data structure and data iterator\n */\n#ifndef MXNET_IO_H_\n#define MXNET_IO_H_\n\n#include <vector>\n#include <string>\n#include <utility>\n#include <queue>\n#include \"dmlc/data.h\"\n#include \"dmlc/registry.h\"\n#include \"./base.h\"\n#include \"./ndarray.h\"\n\nnamespace mxnet {\n/*!\n * \\brief iterator type\n * \\tparam DType data type\n */\ntemplate <typename DType>\nclass IIterator : public dmlc::DataIter<DType> {\n public:\n  /*!\n   * \\brief set the parameters and init iter\n   * \\param kwargs key-value pairs\n   */\n  virtual void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) = 0;\n  /*! \\brief reset the iterator */\n  virtual void BeforeFirst(void) = 0;\n  /*! \\brief move to next item */\n  virtual bool Next(void) = 0;\n  /*! \\brief get current data */\n  virtual const DType& Value(void) const = 0;\n  /*! \\brief constructor */\n  virtual ~IIterator(void) {}\n  /*! \\brief store the name of each data, it could be used for making NDArrays */\n  std::vector<std::string> data_names;\n  /*! \\brief set data name to each attribute of data */\n  inline void SetDataName(const std::string data_name) {\n    data_names.push_back(data_name);\n  }\n  /*! \\brief request iterator length hint for current epoch.\n   * Note that the returned value can be < 0, indicating\n   * that the length of iterator is unknown unless you went through all data.\n   */\n  virtual int64_t GetLenHint(void) const {\n    return -1;\n  }\n};  // class IIterator\n\n/*! \\brief a single data instance */\nstruct DataInst {\n  /*! \\brief unique id for instance */\n  unsigned index;\n  /*! \\brief content of data */\n  std::vector<TBlob> data;\n  /*! \\brief extra data to be fed to the network */\n  std::string extra_data;\n};  // struct DataInst\n\n/*!\n * \\brief DataBatch of NDArray, returned by Iterator\n */\nstruct DataBatch {\n  /*! \\brief content of dense data, if this DataBatch is dense */\n  std::vector<NDArray> data;\n  /*! \\brief index of image data */\n  std::vector<uint64_t> index;\n  /*! \\brief extra data to be fed to the network */\n  std::string extra_data;\n  /*! \\brief num of example padded to batch */\n  int num_batch_padd;\n};  // struct DataBatch\n\n/*! \\brief typedef the factory function of data iterator */\ntypedef std::function<IIterator<DataBatch>*()> DataIteratorFactory;\n/*!\n * \\brief Registry entry for DataIterator factory functions.\n */\nstruct DataIteratorReg : public dmlc::FunctionRegEntryBase<DataIteratorReg, DataIteratorFactory> {};\n//--------------------------------------------------------------\n// The following part are API Registration of Iterators\n//--------------------------------------------------------------\n/*!\n * \\brief Macro to register Iterators\n *\n * \\code\n * // example of registering a mnist iterator\n * REGISTER_IO_ITER(MNISTIter)\n * .describe(\"Mnist data iterator\")\n * .set_body([]() {\n *     return new PrefetcherIter(new MNISTIter());\n *   });\n * \\endcode\n */\n#define MXNET_REGISTER_IO_ITER(name) \\\n  DMLC_REGISTRY_REGISTER(::mxnet::DataIteratorReg, DataIteratorReg, name)\n\n/*!\n * \\brief A random accessable dataset which provides GetLen() and GetItem().\n * Unlike DataIter, it's a static lookup storage which is friendly to random access.\n * The dataset itself should NOT contain data processing, which should be applied during\n * data augmentation or transformation processes.\n */\nclass Dataset {\n public:\n  /*!\n   *  \\brief Get the size of the dataset\n   */\n  virtual uint64_t GetLen(void) const = 0;\n  /*!\n   *  \\brief Get the ndarray items given index in dataset\n   *  \\param idx the integer index for required data\n   *  \\param ret the returned ndarray items\n   */\n  virtual bool GetItem(uint64_t idx, std::vector<NDArray>* ret) = 0;\n  // virtual destructor\n  virtual ~Dataset(void) {}\n};  // class Dataset\n\n/*! \\brief typedef the factory function of dataset */\ntypedef std::function<Dataset*(const std::vector<std::pair<std::string, std::string> >&)>\n    DatasetFactory;\n/*!\n * \\brief Registry entry for Dataset factory functions.\n */\nstruct DatasetReg : public dmlc::FunctionRegEntryBase<DatasetReg, DatasetFactory> {};\n//--------------------------------------------------------------\n// The following part are API Registration of Datasets\n//--------------------------------------------------------------\n/*!\n * \\brief Macro to register Datasets\n *\n * \\code\n * // example of registering an image sequence dataset\n * REGISTER_IO_ITE(ImageSequenceDataset)\n * .describe(\"image sequence dataset\")\n * .set_body([]() {\n *     return new ImageSequenceDataset();\n *   });\n * \\endcode\n */\n#define MXNET_REGISTER_IO_DATASET(name) \\\n  DMLC_REGISTRY_REGISTER(::mxnet::DatasetReg, DatasetReg, name)\n\nclass BatchifyFunction {\n public:\n  /*! \\brief Destructor */\n  virtual ~BatchifyFunction(void) {}\n  /*! \\brief The batchify logic */\n  virtual bool Batchify(const std::vector<std::vector<NDArray> >& inputs,\n                        std::vector<NDArray>* outputs) = 0;\n};  // class BatchifyFunction\n\nusing BatchifyFunctionPtr = std::shared_ptr<BatchifyFunction>;\n\n/*! \\brief typedef the factory function of data sampler */\ntypedef std::function<BatchifyFunction*(const std::vector<std::pair<std::string, std::string> >&)>\n    BatchifyFunctionFactory;\n/*!\n * \\brief Registry entry for DataSampler factory functions.\n */\nstruct BatchifyFunctionReg\n    : public dmlc::FunctionRegEntryBase<BatchifyFunctionReg, BatchifyFunctionFactory> {};\n//--------------------------------------------------------------\n// The following part are API Registration of Batchify Function\n//--------------------------------------------------------------\n/*!\n * \\brief Macro to register Batchify Functions\n *\n * \\code\n * // example of registering a Batchify Function\n * MXNET_REGISTER_IO_BATCHIFY_FUNCTION(StackBatchify)\n * .describe(\"Stack Batchify Function\")\n * .set_body([]() {\n *     return new StackBatchify();\n *   });\n * \\endcode\n */\n#define MXNET_REGISTER_IO_BATCHIFY_FUNCTION(name) \\\n  DMLC_REGISTRY_REGISTER(::mxnet::BatchifyFunctionReg, BatchifyFunctionReg, name)\n}  // namespace mxnet\n#endif  // MXNET_IO_H_\n"
  },
  {
    "path": "include/mxnet/ir/expr.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file expr.h\n * \\brief Base expr nodes in MXNet.\n */\n// Acknowledgement: This file originates from incubator-tvm\n#ifndef MXNET_IR_EXPR_H_\n#define MXNET_IR_EXPR_H_\n\n#include <mxnet/runtime/object.h>\n#include <mxnet/node/node.h>\n#include <mxnet/node/container.h>\n#include <mxnet/runtime/data_type.h>\n#include <string>\n\nnamespace mxnet {\n\n/*!\n * \\brief Base type of all the expressions.\n * \\sa Expr\n */\nclass BaseExprNode : public Object {\n public:\n  static constexpr const char* _type_key = \"Expr\";\n  MXNET_DECLARE_BASE_OBJECT_INFO(BaseExprNode, Object);\n};\n\n/*!\n * \\brief Managed reference to BaseExprNode.\n * \\sa BaseExprNode\n */\nclass BaseExpr : public ObjectRef {\n public:\n  /*! \\brief Cosntructor */\n  BaseExpr() {}\n  /*!\n   * \\brief Cosntructor from object ptr.\n   * \\param ptr The object pointer.\n   */\n  explicit BaseExpr(runtime::ObjectPtr<Object> ptr) : ObjectRef(ptr) {}\n  /*! \\brief The container type. */\n  using ContainerType = BaseExprNode;\n};\n\n/*!\n * \\brief Base node of all primitive expressions.\n *\n *  A primitive expression deals with low-level\n *  POD data types and handles without\n *  doing life-cycle management for objects.\n *\n *  PrimExpr is used in the low-level code\n *  optimizations and integer analysis.\n *\n * \\sa PrimExpr\n */\nclass PrimExprNode : public BaseExprNode {\n public:\n  /*!\n   * \\brief The runtime data type of the primitive expression.\n   *\n   * MXNetDataType(dtype) provides coarse grained type information\n   * during compile time and runtime. It is eagerly built in\n   * PrimExpr expression construction and can be used for\n   * quick type checking.\n   *\n   * dtype is sufficient to decide the Type of the PrimExpr\n   * when it corresponds to POD value types such as i32.\n   *\n   * When dtype is MXNetDataType::Handle(), the expression could corresponds to\n   * a more fine-grained Type, and we can get the type by running lazy type inference.\n   */\n  MXNetDataType dtype;\n\n  static constexpr const char* _type_key = \"PrimExpr\";\n  MXNET_DECLARE_BASE_OBJECT_INFO(PrimExprNode, BaseExprNode);\n};\n\n/*!\n * \\brief Reference to PrimExprNode.\n * \\sa PrimExprNode\n */\nclass PrimExpr : public BaseExpr {\n public:\n  /*! \\brief Cosntructor */\n  PrimExpr() {}\n  /*!\n   * \\brief Cosntructor from object ptr.\n   * \\param ptr The object pointer.\n   */\n  explicit PrimExpr(runtime::ObjectPtr<Object> ptr) : BaseExpr(ptr) {}\n  /*!\n   * \\brief construct from integer.\n   * \\param value The value to be constructed.\n   */\n  MXNET_DLL PrimExpr(int32_t value);  // NOLINT(*)\n  /*!\n   * \\brief construct from float.\n   * \\param value The value to be constructed.\n   */\n  MXNET_DLL PrimExpr(float value);  // NOLINT(*)\n  /*!\n   * \\brief construct from string.\n   * \\param str The value to be constructed.\n   */\n  MXNET_DLL PrimExpr(std::string str);  // NOLINT(*)\n\n  /*! \\return the data type of this expression. */\n  MXNetDataType dtype() const {\n    return static_cast<const PrimExprNode*>(get())->dtype;\n  }\n  /*! \\brief The container type. */\n  using ContainerType = PrimExprNode;\n};\n\n/*!\n * \\brief Constant integer literals in the program.\n * \\sa IntImm\n */\nclass IntImmNode : public PrimExprNode {\n public:\n  /*! \\brief the Internal value. */\n  int64_t value;\n\n  static constexpr const char* _type_key = \"IntImm\";\n  MXNET_DECLARE_FINAL_OBJECT_INFO(IntImmNode, PrimExprNode)\n};\n\n/*!\n * \\brief Managed reference class to IntImmNode.\n *\n * \\sa IntImmNode\n */\nclass IntImm : public PrimExpr {\n public:\n  /*!\n   * \\brief Constructor\n   */\n  IntImm() {}\n  /*!\n   * \\brief constructor from node.\n   */\n  explicit IntImm(runtime::ObjectPtr<Object> node) : PrimExpr(node) {}\n  /*!\n   * \\brief Constructor.\n   * \\param dtype The data type of the value.\n   * \\param value The internal value.\n   */\n  MXNET_DLL IntImm(MXNetDataType dtype, int64_t value);\n  /*!\n   * \\brief Get pointer to the internal value.\n   * \\return the content of the integer.\n   */\n  const IntImmNode* operator->() const {\n    return static_cast<const IntImmNode*>(get());\n  }\n  /*! \\brief type indicate the container type */\n  using ContainerType = IntImmNode;\n};\n\n/*!\n * \\brief Constant floating point literals in the program.\n * \\sa FloatImm\n */\nclass FloatImmNode : public PrimExprNode {\n public:\n  /*! \\brief The constant value content. */\n  double value;\n\n  static constexpr const char* _type_key = \"FloatImm\";\n  MXNET_DECLARE_FINAL_OBJECT_INFO(FloatImmNode, PrimExprNode)\n};\n\n/*!\n * \\brief Managed reference class to FloatImmNode.\n *\n * \\sa FloatImmNode\n */\nclass FloatImm : public PrimExpr {\n public:\n  /*!\n   * \\brief Constructor\n   */\n  FloatImm() {}\n  /*!\n   * \\brief constructor from node.\n   */\n  explicit FloatImm(runtime::ObjectPtr<Object> node) : PrimExpr(node) {}\n  /*!\n   * \\brief Constructor.\n   * \\param dtype The data type of the value.\n   * \\param value The internal value.\n   */\n  MXNET_DLL FloatImm(MXNetDataType dtype, double value);\n  /*!\n   * \\brief Get pointer to the container.\n   * \\return The pointer.\n   */\n  const FloatImmNode* operator->() const {\n    return static_cast<const FloatImmNode*>(get());\n  }\n  /*! \\brief type indicate the container type */\n  using ContainerType = FloatImmNode;\n};\n\n}  // namespace mxnet\n#endif  // MXNET_IR_EXPR_H_\n"
  },
  {
    "path": "include/mxnet/kvstore.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file kvstore.h\n * \\brief key-value store interface for mxnet\n */\n#ifndef MXNET_KVSTORE_H_\n#define MXNET_KVSTORE_H_\n#include <dmlc/io.h>\n#include <vector>\n#include <utility>\n#include <unordered_map>\n#include <string>\n#include <functional>\n#include <atomic>\n#include \"../../src/kvstore/gradient_compression.h\"\n#include \"./ndarray.h\"\n#if MXNET_USE_DIST_KVSTORE\n#include \"ps/ps.h\"\n#endif  // MXNET_USE_DIST_KVSTORE\n\nnamespace mxnet {\n\n/*!\n * \\brief enum to denote types of commands kvstore sends to server regarding profiler\n * kSetConfig sets profiler configs. Similar to mx.profiler.set_config()\n * kState allows changing state of profiler to stop or run\n * kPause allows pausing and resuming of profiler\n * kDump asks profiler to dump output\n */\nenum class KVStoreServerProfilerCommand { kSetConfig, kState, kPause, kDump };\n\n/*!\n * \\brief distributed key-value store\n *\n * A distributed key-value store for data synchronization over multiple\n * devices/machines. It support user-defined updater.\n */\nclass KVStore {\n public:\n  /*! \\brief virtual destructor */\n  virtual ~KVStore() {}\n\n  /*!\n   * \\brief Factory function to create a new KVStore.\n   * \\param type The type of the kvstore,\n   *   - 'local' or 'local_update_cpu' or 'local_allreduce_cpu'\n   *       multi-devices on a single machine. can be also\n   *   - 'device' or 'local_allreduce_device' : same to local but use gpus for kv\n   *       allreduce\n   *   - 'dist_*' : multi-machines\n   * \\return a new created KVStore.\n   */\n  static KVStore* Create(const char* type = \"local\");\n\n  /**\n   * \\brief return the type\n   */\n  inline const std::string& type() {\n    return type_;\n  }\n\n  /**\n   * \\brief Set parameters to use low-bit compressed gradients\n   * \\param compression_type type of compression\n   * \\param threshold threshold for 2bit compression\n   */\n  virtual void SetGradientCompression(\n      const std::vector<std::pair<std::string, std::string>>& kwargs) = 0;\n\n  /*!\n   * \\brief Initialize a list of key-value pair to the store.\n   *\n   * One must initialize the key before \\ref Push and \\ref Pull, and a key\n   * should be only initialized once\n   *\n   * It returns after data have been initialized successfully.\n   *\n   * For multiple workers, all workers must call \\ref Init. But only worker 0\n   * (get_rank() == 0)'s values are used for initialization. So others' values\n   * can be empty (but not keys). This function blocks until all workers are\n   * finished. That means, any worker can push and pull on the keys now.\n   *\n   * \\param keys a list of unique keys\n   * \\param values a list of values\n   */\n  virtual void Init(const std::vector<int>& keys, const std::vector<NDArray>& values) = 0;\n  /*!\n   * \\brief Initialize a list of key-value pair to the store.\n   * \\param keys a list of unique keys in string format\n   * \\param values a list of values\n   */\n  virtual void Init(const std::vector<std::string>& str_keys,\n                    const std::vector<NDArray>& values) = 0;\n  /*!\n   * \\brief push a list of key-value pairs into the store\n   *\n   * If a key appears mulitple times in \\a keys, then the according values will\n   * be aggregated (summed) before pushing.\n   *\n   * The (aggregated) values are merged into the store one by one\n   *\n   * \\code\n   * updater(key, value, &value_in_store);\n   * \\endcode\n   *\n   * One can set a user-defined updater by \\ref set_updater. The default updater\n   * is Assign.\n   *\n   * This function returns after adding a push operator to the engine. Any\n   * following operator requiring writing value will be blocked until the\n   * actual push is finished. One can wait the push is finished by\n   *\n   * - when type == \"local\"\n   * \\code\n   * for (auto& v : values) v.WaitToWrite()\n   * \\endcode\n   *\n   * - when type == \"dist\"\n   * \\code\n   * Wait(keys);\n   * \\endcode\n   *\n   * One must call Init() on every key before. And the value NDArray should be\n   * always has the same shape as being inited.\n   *\n   * \\param keys the list of keys\n   * \\param values the list of values\n   * \\param priority Priority of the action.\n   */\n  virtual void Push(const std::vector<int>& keys,\n                    const std::vector<NDArray>& values,\n                    int priority = 0) = 0;\n\n  /*!\n   * \\brief push a list of key-value pairs into the store\n   * \\param keys the list of keys in string format\n   * \\param values the list of values\n   * \\param priority Priority of the action.\n   */\n  virtual void Push(const std::vector<std::string>& str_keys,\n                    const std::vector<NDArray>& values,\n                    int priority = 0) = 0;\n  /*!\n   * \\brief pull a list of key-value pairs from the store\n   *\n   * One must call Init() on \\a key before. And \\a value should be pre-allocated\n   *\n   * This function returns after adding a pull operator to the engine. Any\n   * following operator requiring reading value will be blocked until the\n   * actual pull is finished. One can wait the pull is finished by\n   *\n   * - when type == \"local\"\n   * \\code\n   * for (auto& v : values) v.WaitToRead()\n   * \\endcode\n   *\n   * - when type == \"dist\"\n   * \\code\n   * Wait(keys);\n   * \\endcode\n   *\n   * \\param keys the list of keys\n   * \\param values the list of buffers for the pulled data, they should be preallocated\n   * \\param priority Priority of the action.\n   * \\param ignore_sparse whether to ignore sparse arrays in the request\n   */\n  virtual void Pull(const std::vector<int>& keys,\n                    const std::vector<NDArray*>& values,\n                    int priority       = 0,\n                    bool ignore_sparse = true) = 0;\n  /*!\n   * \\brief pull a list of key-value pairs from the store\n   * \\param keys the list of keys in string format\n   * \\param values the list of buffers for the pulled data, they should be preallocated\n   * \\param priority Priority of the action.\n   * \\param ignore_sparse whether to ignore sparse arrays in the request\n   */\n  virtual void Pull(const std::vector<std::string>& str_keys,\n                    const std::vector<NDArray*>& values,\n                    int priority       = 0,\n                    bool ignore_sparse = true) = 0;\n\n  /*!\n   * \\brief broadcast a list of key-value pairs from the store\n   * \\param vkeys the list of keys to be pushed\n   * \\param okeys the list of keys to be pulled. Should be the same set of keys in vkeys.\n   * \\param values the list of values to be pushed\n   * \\param outs the list of buffers for the pulled data, they should be preallocated\n   * \\param priority Priority of the action.\n   */\n  virtual void Broadcast(const std::vector<int>& vkeys,\n                         const std::vector<int>& okeys,\n                         const std::vector<NDArray>& values,\n                         const std::vector<NDArray*>& outs,\n                         int priority = 0) = 0;\n\n  /*!\n   * \\brief broadcast a list of key-value pairs from the store\n   * \\param vkeys the list of keys to be pushed in string format\n   * \\param okeys the list of keys to be pulled in string format. Should be the same set of keys in\n   * vkeys.\n   * \\param values the list of values to be pushed\n   * \\param outs the list of buffers for the pulled data, they should be preallocated\n   * \\param priority Priority of the action.\n   */\n  virtual void Broadcast(const std::vector<std::string>& str_vkeys,\n                         const std::vector<std::string>& str_okeys,\n                         const std::vector<NDArray>& values,\n                         const std::vector<NDArray*>& outs,\n                         int priority = 0) = 0;\n\n  /*!\n   * \\brief push and pull a list of key-value pairs from the store\n   * \\param vkeys the list of keys to be pushed\n   * \\param okeys the list of keys to be pulled. Should be the same set of keys in vkeys.\n   * \\param values the list of values to be pushed\n   * \\param outs the list of buffers for the pulled data, they should be preallocated\n   * \\param priority Priority of the action.\n   */\n  virtual void PushPull(const std::vector<int>& vkeys,\n                        const std::vector<int>& okeys,\n                        const std::vector<NDArray>& values,\n                        const std::vector<NDArray*>& outs,\n                        int priority = 0) = 0;\n\n  /*!\n   * \\brief push and pull a list of key-value pairs from the store\n   * \\param vkeys the list of keys to be pushed in string format\n   * \\param okeys the list of keys to be pulled in string format. Should be the same set of keys in\n   * vkeys.\n   * \\param values the list of values to be pushed\n   * \\param outs the list of buffers for the pulled data, they should be preallocated\n   * \\param priority Priority of the action.\n   */\n  virtual void PushPull(const std::vector<std::string>& str_vkeys,\n                        const std::vector<std::string>& str_okeys,\n                        const std::vector<NDArray>& values,\n                        const std::vector<NDArray*>& outs,\n                        int priority = 0) = 0;\n  /*!\n   * \\brief pull a list of key-value pairs from the store.\n   *        The NDArray pulled back will be in row_sparse storage with only the\n   *        specified row_ids present (others rows are zeros).\n   * \\param keys the list of keys\n   * \\param values the list of buffers - row_id pairs\n   * \\param priority the priority of the action.\n   */\n  virtual void PullRowSparse(const std::vector<int>& str_keys,\n                             const std::vector<std::pair<NDArray*, NDArray>>& val_rowids,\n                             int priority = 0) = 0;\n\n  /*!\n   * \\brief pull a list of key-value pairs from the store, where each key is a string.\n   *        The NDArray pulled back will be in row_sparse storage with only the\n   *        specified row_ids present (others rows are zeros).\n   * \\param keys the list of keys in string format\n   * \\param values the list of buffers - row_id pairs\n   * \\param priority the priority of the action.\n   */\n  virtual void PullRowSparse(const std::vector<std::string>& str_keys,\n                             const std::vector<std::pair<NDArray*, NDArray>>& val_rowids,\n                             int priority = 0) = 0;\n\n  /**\n   * \\brief the prototype of user-defined updater\n   */\n  typedef std::function<void(int, const NDArray&, NDArray*)> Updater;\n  /**\n   * \\brief the prototype of user-defined updater with string keys\n   */\n  typedef std::function<void(const std::string&, const NDArray&, NDArray*)> StrUpdater;\n  /*!\n   * \\brief set an updater\n   *\n   * Given a key, assume \\a x is the received (pushed) value and \\a y is the\n   * value stored on the store node. The store updates \\a y by `h(x, &y)`. The\n   * default \\a h is ASSIGN, namely `*y = x`.\n   *\n   * \\param updater user-defined updater, default is assign\n   */\n  virtual void set_updater(const Updater& updater) {\n    CHECK(updater) << \"invalid updater\";\n    updater_ = updater;\n  }\n\n  /*!\n   * \\brief set an updater with string keys\n   *\n   * Given a string key, assume \\a x is the received (pushed) value and \\a y is the\n   * value stored on the store node. The store updates \\a y by `h(x, &y)`. The\n   * default \\a h is ASSIGN, namely `*y = x`.\n   *\n   * \\param updater user-defined string updater, default is assign\n   */\n  virtual void set_updater(const StrUpdater& updater) {\n    CHECK(updater) << \"invalid updater\";\n    str_updater_ = updater;\n  }\n\n  /******************************************************\n   * the following are used for multi-machines.\n   ******************************************************/\n\n  /**\n   * \\brief initalize ps-lite environment variables\n   * \\param envs key-value environment variables\n   */\n  static void InitPSEnv(const std::unordered_map<std::string, std::string>& envs) {\n#if MXNET_USE_DIST_KVSTORE\n    ps::Environment::Init(envs);\n#else\n    LOG(FATAL) << \"compile with USE_DIST_KVSTORE=1 to init parameter server's environment\";\n#endif  // MXNET_USE_DIST_KVSTORE\n  }\n\n  /**\n   * \\return whether or not this process is a worker node.\n   *\n   * Always returns true when type == \"local\"\n   */\n  static bool IsWorkerNode() {\n#if MXNET_USE_DIST_KVSTORE\n    const char* role_str = ps::Environment::Get()->find(\"DMLC_ROLE\");\n    return (role_str == nullptr) || (!strcmp(role_str, \"worker\"));\n#else\n    return true;\n#endif  // MXNET_USE_DIST_KVSTORE\n  }\n\n  /**\n   * \\return whether or not this process is a server node.\n   *\n   * Always returns false when type == \"local\"\n   */\n  static bool IsServerNode() {\n#if MXNET_USE_DIST_KVSTORE\n    const char* role_str = ps::Environment::Get()->find(\"DMLC_ROLE\");\n    return (role_str != nullptr) && (!strcmp(role_str, \"server\"));\n#else\n    return false;\n#endif  // MXNET_USE_DIST_KVSTORE\n  }\n\n  void set_barrier_before_exit(const bool barrier_before_exit) {\n#if MXNET_USE_DIST_KVSTORE\n    if (!IsWorkerNode())\n      LOG(FATAL) << \"barrier_before_exit takes effect only on worker nodes\";\n    barrier_before_exit_ = barrier_before_exit;\n#else\n    LOG(FATAL) << \"compile with USE_DIST_KVSTORE=1 to enable barrier\";\n#endif\n  }\n\n  /**\n   * \\return whether or not this process is a scheduler node.\n   *\n   * Always returns false when type == \"local\"\n   */\n  static bool IsSchedulerNode() {\n#if MXNET_USE_DIST_KVSTORE\n    const char* role_str = ps::Environment::Get()->find(\"DMLC_ROLE\");\n    return (role_str != nullptr) && (!strcmp(role_str, \"scheduler\"));\n#else\n    return false;\n#endif  // MXNET_USE_DIST_KVSTORE\n  }\n\n  /*!\n   * \\return The rank of this node in its group, which is in [0,\n   * GroupSize).\n   *\n   * Always return 0 when type == \"local\"\n   */\n  virtual int get_rank() const {\n    return 0;\n  }\n\n  /*!\n   * \\return The number of worker nodes\n   */\n  virtual int get_group_size() const {\n    return 1;\n  }\n\n  /*!\n   * \\return the number of dead node(s) specified by {node_id}\n   * \\param node_id can be a node group or a single node\n   * \\param timeout a node fails to send heartbeart in {timeout} seconds\n   *        will be presumed as 'dead'\n   *\n   * Always return 0 when type == \"local\"\n   */\n  virtual int get_num_dead_node(int node_id, int timeout = 60) const {\n    return 0;\n  }\n\n  /*!\n   * \\brief global barrier among all worker machines\n   *\n   * But note that, this functions only blocks the main thread of workers until\n   * all of them are reached this point. It doesn't guarantee that all\n   * operations issued before are actually finished, such as \\ref Push and \\ref Pull.\n   */\n  virtual void Barrier() {}\n\n  /**\n   * \\brief Send a command to all server nodes\n   *\n   * Send a command to all server nodes, which will make each server node run\n   * \\a controller\n   *\n   * This function returns after the command has been executed in all server nodes\n   *\n   * \\param cmd_id the head of the command\n   * \\param cmd_body the body of the command\n   */\n  virtual void SendCommandToServers(int cmd_id, const std::string& cmd_body) {}\n\n  /**\n   * \\brief Sends server profiler commands to all server nodes\n   * Only the worker with rank=0 sends the command which will be received by all servers\n   * \\param type ProfilerCommand type\n   * \\param params parameters for that command in the form of a string\n   */\n  virtual void SetServerProfilerCommand(const KVStoreServerProfilerCommand type,\n                                        const std::string& params) {\n    LOG(INFO) << \"Unable to pass server the profiler command. If you are using \"\n              << \"distributed kvstore, you need to compile with USE_DIST_KVSTORE=1.\"\n              << \"If you are training on single machine, then there is no server process\"\n              << \"to profile. Please profile the worker process instead.\";\n  }\n\n  /**\n   * \\brief the prototype of a server controller\n   */\n  typedef std::function<void(int, const std::string&)> Controller;\n\n  /**\n   * \\brief Run as server (or scheduler)\n   *\n   * The behavior of a server:\n   * \\code\n   * while(receive(x)) {\n   *   if (IsCommand(x)) controller(x)\n   *   else if (IsKeyValue(x)) updater(x)\n   * }\n   * \\endcode\n   *\n   * \\param controller the user-defined server controller\n   */\n  virtual void RunServer(const Controller& controller) {}\n\n protected:\n  /**\n   * \\brief the user-defined updater\n   */\n  Updater updater_;\n\n  /**\n   * \\brief the user-defined updater with string keys\n   */\n  StrUpdater str_updater_;\n\n  /**\n   * \\brief the kvstore type\n   */\n  std::string type_;\n\n  /** \\brief Gradient compression object starts with GC_NONE mode\n   * Used if SetGradientCompression sets the type.\n   * Currently there is no support for un-setting gradient compression\n   */\n  std::shared_ptr<kvstore::GradientCompression> gradient_compression_;\n\n  /**\n   * \\brief whether to do barrier when finalize\n   */\n  std::atomic<bool> barrier_before_exit_{true};\n};\n\n}  // namespace mxnet\n#endif  // MXNET_KVSTORE_H_\n"
  },
  {
    "path": "include/mxnet/lib_api.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file lib_api.h\n * \\brief APIs to interact with libraries\n * This API specifies function prototypes to\n * register custom ops, partitioner, and passes\n * for library authors\n * See example/extension/lib_custom_op/README.md\n * See example/extension/lib_subgraph/README.md\n * See example/extension/lib_pass/README.md\n */\n\n#ifndef MXNET_LIB_API_H_\n#define MXNET_LIB_API_H_\n\n#include <stdint.h>\n#include <stdlib.h>\n#include <string.h>\n#include <vector>\n#include <map>\n#include <unordered_set>\n#include <unordered_map>\n#include <string>\n#include <iostream>\n#include <utility>\n#include <stdexcept>\n#include <functional>\n#include <random>\n#include <sstream>\n\n#if defined(__NVCC__)\n#include <cuda_runtime.h>\n#include <curand_kernel.h>\n#endif\n\n/* Make sure to update the version number everytime you make changes */\n#define MX_LIBRARY_VERSION 11\n\n/*!\n * \\brief For loading multiple custom op libraries in Linux, exporting same symbol multiple\n * times may lead to undefined behaviour, so we need to set symbol visibility to hidden\n * see https://labjack.com/news/simple-cpp-symbol-visibility-demo for details\n */\n#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)\n#define PRIVATE_SYMBOL\n#else\n#define PRIVATE_SYMBOL __attribute__((visibility(\"hidden\")))\n#endif\n\n/*\n * Import from DLPack https://github.com/dmlc/dlpack/blob/master/include/dlpack/dlpack.h\n */\n#ifndef DLPACK_VERSION\n#ifdef __cplusplus\n#define DLPACK_EXTERN_C extern \"C\"\n#else\n#define DLPACK_EXTERN_C\n#endif\n\n/*! \\brief The current version of dlpack */\n#define DLPACK_VERSION 020\n\n/*! \\brief DLPACK_DLL prefix for windows */\n#ifdef _WIN32\n#ifdef DLPACK_EXPORTS\n#define DLPACK_DLL __declspec(dllexport)\n#else\n#define DLPACK_DLL __declspec(dllimport)\n#endif\n#else\n#define DLPACK_DLL\n#endif\n\n#include <stdint.h>\n#include <stddef.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n/*!\n * \\brief The device type in DLContext.\n */\ntypedef enum {\n  /*! \\brief CPU device */\n  kDLCPU = 1,\n  /*! \\brief CUDA GPU device */\n  kDLGPU = 2,\n  /*!\n   * \\brief Pinned CUDA GPU device by cudaMallocHost\n   * \\note kDLCPUPinned = kDLCPU | kDLGPU\n   */\n  kDLCPUPinned = 3,\n  /*! \\brief OpenCL devices. */\n  kDLOpenCL = 4,\n  /*! \\brief Vulkan buffer for next generation graphics. */\n  kDLVulkan = 7,\n  /*! \\brief Metal for Apple GPU. */\n  kDLMetal = 8,\n  /*! \\brief Verilog simulator buffer */\n  kDLVPI = 9,\n  /*! \\brief ROCm GPUs for AMD GPUs */\n  kDLROCM = 10,\n  /*!\n   * \\brief Reserved extension device type,\n   * used for quickly test extension device\n   * The semantics can differ depending on the implementation.\n   */\n  kDLExtDev = 12,\n} DLDeviceType;\n\n/*!\n * \\brief A Device context for Tensor and operator.\n */\ntypedef struct {\n  /*! \\brief The device type used in the device. */\n  DLDeviceType device_type;\n  /*! \\brief The device index */\n  int device_id;\n} DLContext;\n\n/*!\n * \\brief The type code options DLDataType.\n */\ntypedef enum {\n  kDLInt   = 0U,\n  kDLUInt  = 1U,\n  kDLFloat = 2U,\n} DLDataTypeCode;\n\n/*!\n * \\brief The data type the tensor can hold.\n *\n *  Examples\n *   - float: type_code = 2, bits = 32, lanes=1\n *   - float4(vectorized 4 float): type_code = 2, bits = 32, lanes=4\n *   - int8: type_code = 0, bits = 8, lanes=1\n */\ntypedef struct {\n  /*!\n   * \\brief Type code of base types.\n   * We keep it uint8_t instead of DLDataTypeCode for minimal memory\n   * footprint, but the value should be one of DLDataTypeCode enum values.\n   * */\n  uint8_t code;\n  /*!\n   * \\brief Number of bits, common choices are 8, 16, 32.\n   */\n  uint8_t bits;\n  /*! \\brief Number of lanes in the type, used for vector types. */\n  uint16_t lanes;\n} DLDataType;\n\n/*!\n * \\brief Plain C Tensor object, does not manage memory.\n */\ntypedef struct {\n  /*!\n   * \\brief The opaque data pointer points to the allocated data. This will be\n   * CUDA device pointer or cl_mem handle in OpenCL. This pointer is always\n   * aligns to 256 bytes as in CUDA.\n   *\n   * For given DLTensor, the size of memory required to store the contents of\n   * data is calculated as follows:\n   *\n   * \\code{.c}\n   * static inline size_t GetDataSize(const DLTensor* t) {\n   *   size_t size = 1;\n   *   for (tvm_index_t i = 0; i < t->ndim; ++i) {\n   *     size *= t->shape[i];\n   *   }\n   *   size *= (t->dtype.bits * t->dtype.lanes + 7) / 8;\n   *   return size;\n   * }\n   * \\endcode\n   */\n  void* data;\n  /*! \\brief The device context of the tensor */\n  DLContext ctx;\n  /*! \\brief Number of dimensions */\n  int ndim;\n  /*! \\brief The data type of the pointer*/\n  DLDataType dtype;\n  /*! \\brief The shape of the tensor */\n  int64_t* shape;\n  /*!\n   * \\brief strides of the tensor (in number of elements, not bytes)\n   *  can be nullptr, indicating tensor is compact and row-majored.\n   */\n  int64_t* strides;\n  /*! \\brief The offset in bytes to the beginning pointer to data */\n  uint64_t byte_offset;\n} DLTensor;\n#ifdef __cplusplus\n}  // DLPACK_EXTERN_C\n#endif\n#endif\n\nnamespace mxnet {\nnamespace ext {\n\n/* \\brief Class to store error messages from extensions to pass to MXNet */\nclass MXerrorMsgs {\n public:\n  /* \\brief get singleton pointer to class */\n  static MXerrorMsgs* get();\n\n  /* \\brief add a new error message */\n  std::stringstream& add(const char* file, int line);\n\n  /* \\brief return number of error messages */\n  int size();\n\n  /* \\brief get error message at index */\n  const std::string* get(int idx);\n\n private:\n  /*! \\brief constructor */\n  MXerrorMsgs() {}\n  /*! \\brief destructor */\n  ~MXerrorMsgs() {}\n  /*! \\brief map of entries in registry */\n  std::vector<std::stringstream> messages;\n};\n\n// Add a new error message, example: MX_ERROR_MSG << \"my error msg\";\n#define MX_ERROR_MSG mxnet::ext::MXerrorMsgs::get()->add(__FILE__, __LINE__)\n\n/*!\n * \\brief Tensor data type, consistent with mshadow data type\n */\nenum MXDType {\n  kFloat32 = 0,\n  kFloat64 = 1,\n  kFloat16 = 2,\n  kUint8   = 3,\n  kInt32   = 4,\n  kInt8    = 5,\n  kInt64   = 6,\n  kUNSET   = 100,\n};\n\n/*\n * MXTensor storage type.\n */\nenum MXStorageType {\n  // dense\n  kDefaultStorage = 0,\n  // row sparse\n  kRowSparseStorage = 1,\n  // csr\n  kCSRStorage = 2,\n};\n\n/*!\n * \\brief Context info passing from MXNet OpContext\n * dev_type is string repr of supported context, currently only \"cpu\" and \"gpu\"\n * dev_id is the device index where the tensor locates\n */\nstruct MXContext {\n  MXContext();\n  explicit MXContext(std::string dev_type_, int dev_id_);\n  explicit MXContext(const char* dev_type_, int dev_id_);\n  static MXContext CPU();\n  static MXContext GPU();\n  static MXContext CPU(int dev_id);\n  static MXContext GPU(int dev_id);\n\n  std::string dev_type;\n  int dev_id;\n};\n\nenum MXReturnValue {\n  MX_FAIL    = 0,\n  MX_SUCCESS = 1,\n};\n\n// For sparse tensors, read/write the data from NDarray via pointers.\nstruct MXSparse {\n  // Pointer to data.\n  void* data{nullptr};\n  // length of (non-zero) data.\n  int64_t data_len;\n\n  // To store aux data for sparse.\n  // For CSR, indices stores the col index of non-zero elements.\n  // For row sparse, indices store row index of rows which have non-zero elements.\n  int64_t* indices;\n  int64_t indices_len;\n\n  // For CSR, indptr gives the start and end index of data for each row.\n  // For row sparse, indptr is not used.\n  int64_t* indptr = nullptr;\n  int64_t indptr_len;\n\n  void set(void* data_ptr,\n           const int64_t* dims,\n           int ndims,\n           void* idx,\n           int64_t num_idx,\n           void* idx_ptr       = nullptr,\n           int64_t num_idx_ptr = 0);\n};\n\n/*!\n * \\brief Tensor data structure used by custom operator\n */\nstruct MXTensor {\n  MXTensor();\n  MXTensor(const MXTensor& oth);\n  MXTensor(void* data_ptr,\n           std::vector<int64_t> shape,\n           MXDType dtype,\n           size_t vID,\n           MXContext mx_ctx,\n           MXStorageType stype = kDefaultStorage);\n\n  /*! \\brief populate internal tensor fields */\n  void setTensor(void* dptr,\n                 MXDType type,\n                 const int64_t* dims,\n                 int ndims,\n                 size_t vID,\n                 MXContext mx_ctx,\n                 MXStorageType storage_type);\n\n  /*! \\brief populate DLTensor fields */\n  void setDLTensor();\n\n  /*! \\brief helper function to cast data pointer */\n  template <typename data_type>\n  inline data_type* data() {\n    return reinterpret_cast<data_type*>(data_ptr);\n  }\n\n  /*! \\brief helper function to get data size */\n  int64_t size() const;\n\n  /*! \\brief helper function to compare two MXTensors */\n  bool isSame(const MXTensor& oth) const;\n\n  // For dense, data_ptr points to 1D flattened tensor data\n  // For sparse, data_ptr points to MXSparse\n  void* data_ptr;\n\n  // shape is in [2,3,4] format to represent high-dim tensor\n  std::vector<int64_t> shape;\n\n  // type can only be MXDType enum types\n  MXDType dtype;\n\n  // version number updated if the tensor has changed since the last use by custom op\n  size_t verID;\n\n  // context of MXTensor representing which device the tensor data is located\n  MXContext ctx;\n\n  // corresponding DLTensor repr of MXTensor\n  // easy way to reuse functions taking DLTensor\n  DLTensor dltensor;\n\n  // storage type\n  MXStorageType stype;\n};\n\n/*! \\brief resource malloc function to allocate memory inside Forward/Backward functions */\ntypedef void* (*xpu_malloc_t)(void*, int);\n/*! \\brief sparse alloc function to allocate memory inside Forward/Backward functions */\ntypedef void (*sparse_malloc_t)(void*, int, int, int, void**, int64_t**, int64_t**);\n/*! \\brief resource malloc function to allocate ndarrays for graph passes */\ntypedef void (*nd_malloc_t)(const void* _ndarray_alloc,\n                            const int64_t* shapes,\n                            int num_shapes,\n                            const char* dev_str,\n                            int dev_id,\n                            int dtype,\n                            const char* name,\n                            int isArg,\n                            void** data);\n/*! \\brief GPU stream pointer, is void* when not compiled with CUDA */\n#if defined(__NVCC__)\ntypedef cudaStream_t mx_stream_t;\ntypedef curandStatePhilox4_32_10_t mx_gpu_rand_t;\n#else\ntypedef void* mx_stream_t;\ntypedef void* mx_gpu_rand_t;\n#endif\ntypedef std::mt19937 mx_cpu_rand_t;\n\n/*! \\brief MXNet initialized random states for each device, used for parallelism */\n/* Each thread should generate random number unique sequence out of different states */\n#define MX_NUM_CPU_RANDOM_STATES 1024\n#define MX_NUM_GPU_RANDOM_STATES 32768\n\n/* \\brief Class to help allocate new args/aux params in graph passes */\nclass PassResource {\n public:\n  PassResource(std::unordered_map<std::string, MXTensor>* new_args,\n               std::unordered_map<std::string, MXTensor>* new_aux,\n               nd_malloc_t nd_malloc,\n               const void* nd_alloc);\n\n  // allocate new arg param, adds to args map, returns newly allocated tensor\n  MXTensor* alloc_arg(const std::string& name,\n                      const std::vector<int64_t>& shapes,\n                      const MXContext& ctx,\n                      MXDType dtype) const;\n\n  // allocate new aux param, adds to aux map, returns newly allocated tensor\n  MXTensor* alloc_aux(const std::string& name,\n                      const std::vector<int64_t>& shapes,\n                      const MXContext& ctx,\n                      MXDType dtype) const;\n\n private:\n  std::unordered_map<std::string, MXTensor>* new_args_;\n  std::unordered_map<std::string, MXTensor>* new_aux_;\n  nd_malloc_t nd_malloc_;\n  const void* nd_alloc_;\n};\n\n/*!\n * \\brief provide resource APIs memory allocation mechanism to Forward/Backward functions\n */\nclass OpResource {\n public:\n  OpResource(xpu_malloc_t cpu_malloc_fp,\n             void* cpu_alloc_fp,\n             xpu_malloc_t gpu_malloc_fp,\n             void* gpu_alloc_fp,\n             void* stream,\n             sparse_malloc_t sparse_malloc_fp,\n             void* sparse_alloc_fp,\n             void* rng_cpu_states,\n             void* rng_gpu_states);\n\n  /*! \\brief allocate cpu memory controlled by MXNet */\n  void* alloc_cpu(int size) const;\n\n  /*! \\brief allocate gpu memory controlled by MXNet */\n  void* alloc_gpu(int size) const;\n\n  /*! \\brief return the cuda stream object with correct type */\n  inline mx_stream_t get_cuda_stream() const {\n    return static_cast<mx_stream_t>(cuda_stream);\n  }\n\n  /*! \\brief allocate sparse memory controlled by MXNet */\n  void alloc_sparse(MXSparse* sparse, int index, int indices_len, int indptr_len = 0) const;\n\n  /*! \\brief get pointer to initialized and seeded random number states located on CPU */\n  /* Access each state by states[id], but this id should be <= MX_NUM_CPU_RANDOM_STATES */\n  mx_cpu_rand_t* get_cpu_rand_states() const;\n\n  /*! \\brief get pointer to initialized and seeded random number states located on GPU */\n  /* Access each state by states[id], but this id should be <= MX_NUM_GPU_RANDOM_STATES */\n  /* Note that if you are using cpu build, it will return a nullptr */\n  inline mx_gpu_rand_t* get_gpu_rand_states() const {\n    return static_cast<mx_gpu_rand_t*>(rand_gpu_states);\n  }\n\n private:\n  /*! \\brief allocation lambda function */\n  xpu_malloc_t cpu_malloc, gpu_malloc;\n  /*! \\brief lambda function to return allocated memory handle */\n  void *cpu_alloc, *gpu_alloc;\n  /*! \\brief cuda stream passed from MXNet */\n  void* cuda_stream;\n  /*! \\brief sparse allocation lambda function */\n  sparse_malloc_t sparse_malloc;\n  /*! \\brief lambda function to return allocated sparse memory handle */\n  void* sparse_alloc;\n  /*! \\brief cpu and gpu rng fully inited and seeded states */\n  void *rand_cpu_states, *rand_gpu_states;\n};\n\n/*! \\brief attribute key to help passing serialized subgraph through subgraph op attribute */\n#define MX_STR_SUBGRAPH_SYM_JSON \"subgraph_sym_json\"\n/*! \\brief dtype attribute key for ops after type propagation */\n#define MX_STR_DTYPE \"__ext_dtype__\"\n/*! \\brief shape attribute key for ops after shape propagation */\n#define MX_STR_SHAPE \"__ext_shape__\"\n/*! \\brief extra input attribute key for ops */\n#define MX_STR_EXTRA_INPUTS \"__ext_extra_inputs__\"\n\n/* \\brief get shape value from list of shapes string\n *\n * Examples:\n *\n * getShapeAt(\"[[1]]\", 0) returns \"[1]\"\n * getShapeAt(\"[[1],[2,3]]\", 1) returns \"[2,3]\"\n */\nstd::string getShapeAt(const std::string& shape, unsigned index);\n\n/* \\brief get dtype value from list of dtypes string\n *\n * Examples:\n *\n * getDtypeAt(\"[1]\", 0) returns \"1\"\n * getDtypeAt(\"[1,2]\", 1) returns \"2\"\n */\nstd::string getDtypeAt(const std::string& dtype, unsigned index);\n\n/*!\n * \\brief Json utility to parse serialized subgraph symbol\n */\n/*! \\brief Types of JSON objects */\nenum JsonType { ERR, STR, NUM, LIST, MAP };\n\n/*! \\brief definition of JSON objects */\nstruct JsonVal {\n  JsonVal();  // default constructor\n  // construct a JSON object by type\n  explicit JsonVal(JsonType t);\n  // construct a string JSON object\n  explicit JsonVal(std::string s);\n  // construct a number JSON object\n  explicit JsonVal(int n);\n  // complex constructor\n  JsonVal(JsonType t, int n, std::string s);\n  bool operator<(const JsonVal& o) const;\n\n  // convert JSON object back to JSON-compatible string\n  std::string dump() const;\n\n  // convert JSON-compatible string to JSON object\n  static JsonVal parse(const std::string& json);\n\n  // parse a string JSON object\n  static JsonVal parse_string(const std::string& json, unsigned int* idx);\n\n  // parse a number JSON object\n  static JsonVal parse_num(const std::string& json, unsigned int* idx);\n\n  // parse a list of JSON objects\n  static JsonVal parse_list(const std::string& json, unsigned int* idx);\n\n  // parse a map of JSON objects\n  static JsonVal parse_map(const std::string& json, unsigned int* idx);\n\n  // generic parse function\n  static JsonVal parse(const std::string& json, unsigned int* idx);\n\n  // debug function to convert data structure to a debugstring\n  std::string toString() const;\n\n  JsonType type;\n  int num;\n  std::string str;\n  std::vector<JsonVal> list;\n  std::map<JsonVal, JsonVal> map;\n};\n\n/*!\n * \\brief Graph utility to parse serialized subgraph symbol\n */\nclass Node;\nclass Graph;\n\n// Representation of an input/output to a node\nstruct NodeEntry {\n  Node* node;  // other node thats producing/consuming inputs/outputs\n  int entry;   // entry index from other node (ie. output index from producing node)\n};\n\n// Representation of a node in the graph\nclass Node {\n public:\n  Node();\n\n  // internally set passResource to enable tensor allocation for graph passes\n  void _setPassResource(PassResource* res_);\n\n  /* \\brief allocate an arg tensor for this node */\n  void alloc_arg(const std::vector<int64_t>& shapes, const MXContext& ctx, MXDType dtype);\n\n  /* \\brief allocate an aux tensor for this node */\n  void alloc_aux(const std::vector<int64_t>& shapes, const MXContext& ctx, MXDType dtype);\n\n  std::string op;                                      // operator name (ie. Convolution)\n  std::string name;                                    // unique node name (ie. conv_0 or conv_1)\n  MXTensor* tensor;                                    // tensor data for input nodes\n  std::vector<NodeEntry> inputs;                       // set of inputs to the node\n  std::vector<NodeEntry> outputs;                      // set of outputs from the node\n  std::vector<Graph*> subgraphs;                       // set of subgraphs within this node\n  std::unordered_map<std::string, std::string> attrs;  // node attributes\n\n private:\n  PassResource* res;\n};\n\n// Representation of the graph\nclass Graph {\n public:\n  Graph();\n\n  /* \\brief deleted nodes when deleting the graph */\n  ~Graph();\n\n  /* \\brief create a graph object from an unparsed string */\n  static Graph* fromString(const std::string& json);\n\n  /* \\brief create a graph object from a parsed JSON object */\n  static Graph* fromJson(JsonVal val);\n\n  /* \\brief convert graph object back to JSON object */\n  JsonVal toJson() const;\n\n  /* \\brief convert graph object to JSON string */\n  std::string toString() const;\n\n  /* \\brief visits a node \"n\" */\n  void _dfs_util(Node* n,\n                 std::unordered_set<Node*>* to_visit,\n                 std::function<void(Node*)> handler) const;\n\n  /* \\brief post-order DFS graph traversal */\n  void DFS(std::function<void(Node*)> handler) const;\n\n  /* \\brief sort graph nodes in topological order */\n  std::vector<Node*> topological_sort() const;\n\n  /* \\brief print out graph details */\n  void print(int indent = 0) const;\n\n  /* \\brief add a new node to this graph */\n  Node* addNode(const std::string& name, const std::string& op);\n\n  /* \\brief get node at index in graph */\n  Node* getNode(size_t idx);\n\n  /* \\brief get const node at index in const graph */\n  const Node* getNode(size_t idx) const;\n\n  /* \\brief get attribute on graph */\n  const JsonVal& getAttr(const std::string& key) const;\n\n  /* \\brief get number of nodes in the graph */\n  size_t size() const;\n\n  // internally set passResource to enable tensor allocation for graph passes\n  void _setPassResource(PassResource* res_);\n\n  // internally set arg/aux params when available\n  void _setParams(std::unordered_map<std::string, mxnet::ext::MXTensor>* args,\n                  std::unordered_map<std::string, mxnet::ext::MXTensor>* aux);\n\n  std::vector<Node*> inputs;\n  std::vector<NodeEntry> outputs;\n  std::map<std::string, JsonVal> attrs;\n\n private:\n  std::vector<Node*> nodes;\n  PassResource* res;\n};\n\n/* \\brief An abstract class for library authors creating custom\n * partitioners. Optional, can just implement supportedOps instead\n */\nclass CustomOpSelector {\n public:\n  /* \\brief Select a node to include in subgraph, return true to include node\n   * nodeID - index of node in graph\n   */\n  virtual bool Select(int nodeID) = 0;\n  /* \\brief Select an input node from current node to include in subgraph\n   * return true to include node\n   * nodeID - index of node in graph\n   * input_nodeID - index of input node in graph\n   */\n  virtual bool SelectInput(int nodeID, int input_nodeID) = 0;\n  /* \\brief Select an output node from current node to include in subgraph\n   * return true to include node\n   * nodeID - index of node in graph\n   * output_nodeID - index of output node in graph\n   */\n  virtual bool SelectOutput(int nodeID, int output_nodeID) = 0;\n  /* \\brief Review nodes to include in subgraph\n   * return set of candidate nodes to keep in subgraph\n   * candidates - indices of nodes to include in subgraph\n   * keep - indices of nodes to keep in subgraph\n   */\n  virtual void Filter(const std::vector<int>& candidates, std::vector<int>* keep) {\n    keep->insert(keep->end(), candidates.begin(), candidates.end());\n  }\n  /* \\brief Reset any selector state, called after growing subgraph, before filter\n   * Called after finished calling SelectInput/SelectOutput and growing subgraph\n   */\n  virtual void Reset() {}\n};\n\n/*!\n * \\brief An abstract class for library authors creating stateful op\n * custom library should override Forward and destructor, and has an\n * option to implement Backward\n */\nclass CustomStatefulOp {\n public:\n  CustomStatefulOp();\n  virtual ~CustomStatefulOp();\n\n  template <class A, typename... Ts>\n  static CustomStatefulOp* create(Ts... args) {\n    CustomStatefulOp* op = new A(args...);\n    op->created          = true;\n    return op;\n  }\n\n  bool wasCreated() {\n    return created;\n  }\n\n  virtual MXReturnValue Forward(std::vector<MXTensor>* inputs,\n                                std::vector<MXTensor>* outputs,\n                                const OpResource& op_res) = 0;\n  virtual MXReturnValue Backward(std::vector<MXTensor>* inputs,\n                                 std::vector<MXTensor>* outputs,\n                                 const OpResource& op_res) {\n    MX_ERROR_MSG << \"Error! Operator does not support backward\" << std::endl;\n    return MX_FAIL;\n  }\n\n  bool ignore_warn;\n\n private:\n  bool created;\n};\n\n/*! \\brief Custom Operator function templates */\ntypedef MXReturnValue (*fcomp_t)(const std::unordered_map<std::string, std::string>& attributes,\n                                 std::vector<MXTensor>* inputs,\n                                 std::vector<MXTensor>* outputs,\n                                 const OpResource& res);\ntypedef MXReturnValue (*parseAttrs_t)(\n    const std::unordered_map<std::string, std::string>& attributes,\n    int* num_inputs,\n    int* num_outputs);\ntypedef MXReturnValue (*inferType_t)(const std::unordered_map<std::string, std::string>& attributes,\n                                     std::vector<int>* in_types,\n                                     std::vector<int>* out_types);\ntypedef MXReturnValue (*inferSType_t)(\n    const std::unordered_map<std::string, std::string>& attributes,\n    std::vector<int>* in_storage_types,\n    std::vector<int>* out_storage_types);\ntypedef MXReturnValue (*inferShape_t)(\n    const std::unordered_map<std::string, std::string>& attributes,\n    std::vector<std::vector<unsigned int> >* in_shapes,\n    std::vector<std::vector<unsigned int> >* out_shapes);\ntypedef MXReturnValue (*mutateInputs_t)(\n    const std::unordered_map<std::string, std::string>& attributes,\n    std::vector<int>* input_indices);\ntypedef MXReturnValue (*createOpState_t)(\n    const std::unordered_map<std::string, std::string>& attributes,\n    const MXContext& ctx,\n    const std::vector<std::vector<unsigned int> >& in_shapes,\n    const std::vector<int> in_types,\n    CustomStatefulOp**);\n\n/*!\n * \\brief Class to hold custom operator registration\n */\nclass CustomOp {\n public:\n  explicit CustomOp(const char* op_name);\n\n  CustomOp& setForward(fcomp_t fcomp, const char* ctx);\n\n  CustomOp& setBackward(fcomp_t fgrad, const char* ctx);\n\n  CustomOp& setParseAttrs(parseAttrs_t func);\n\n  CustomOp& setInferType(inferType_t func);\n\n  CustomOp& setInferSType(inferSType_t func);\n\n  CustomOp& setInferShape(inferShape_t func);\n\n  CustomOp& setMutateInputs(mutateInputs_t func);\n\n  CustomOp& setCreateOpState(createOpState_t func, const char* ctx);\n\n  CustomOp& setIsSubgraphOp();\n\n  void mapToVector();\n\n  /*! \\brief operator name */\n  const char* name;\n\n  /*! \\brief operator functions */\n  parseAttrs_t parse_attrs;\n  inferType_t infer_type;\n  inferSType_t infer_storage_type;\n  inferShape_t infer_shape;\n  mutateInputs_t mutate_inputs;\n  bool isSGop;\n\n  /*! \\brief vector repr of ctx map to be easily loaded from c_api */\n  std::vector<const char*> forward_ctx_cstr, backward_ctx_cstr, create_op_ctx_cstr;\n  std::vector<fcomp_t> forward_fp, backward_fp;\n  std::vector<createOpState_t> create_op_fp;\n\n private:\n  void raiseDuplicateContextError();\n\n  /*! \\brief dedup context maps - static string ctx to custom function */\n  std::unordered_map<const char*, fcomp_t> forward_ctx_map, backward_ctx_map;\n  std::unordered_map<const char*, createOpState_t> create_op_ctx_map;\n};\n\n/*! \\brief Custom Pass Create function template */\ntypedef MXReturnValue (*graphPass_t)(mxnet::ext::Graph* graph,\n                                     const std::unordered_map<std::string, std::string>& options);\n\n/*!\n * \\brief An abstract class for graph passes\n */\nclass CustomPass {\n public:\n  CustomPass();\n  explicit CustomPass(const char* pass_name);\n\n  CustomPass& setBody(graphPass_t fn);\n\n  /*! \\brief pass name */\n  const char* name;\n  /*! \\brief pass function */\n  graphPass_t pass;\n};\n\n/*! \\brief Custom Subgraph Create function template */\ntypedef MXReturnValue (*supportedOps_t)(\n    const mxnet::ext::Graph* graph,\n    std::vector<int>* ids,\n    const std::unordered_map<std::string, std::string>& options);\ntypedef MXReturnValue (*createSelector_t)(\n    const mxnet::ext::Graph* graph,\n    CustomOpSelector** sel_inst,\n    const std::unordered_map<std::string, std::string>& options);\ntypedef MXReturnValue (*reviewSubgraph_t)(\n    const mxnet::ext::Graph* subgraph,\n    int subgraph_id,\n    bool* accept,\n    const std::unordered_map<std::string, std::string>& options,\n    std::unordered_map<std::string, std::string>* attrs);\n\n/*!\n * \\brief An abstract class for subgraph property\n */\nclass CustomPartitioner {\n public:\n  CustomPartitioner();\n\n  explicit CustomPartitioner(const char* backend_name);\n\n  CustomPartitioner& addStrategy(const char* prop_name, const char* sg_name);\n\n  CustomPartitioner& setSupportedOps(const char* prop_name, supportedOps_t fn);\n\n  CustomPartitioner& setCreateSelector(const char* prop_name, createSelector_t fn);\n\n  CustomPartitioner& setReviewSubgraph(const char* prop_name, reviewSubgraph_t fn);\n\n  supportedOps_t getSupportedOps(int stg_id);\n\n  createSelector_t getCreateSelector(int stg_id);\n\n  reviewSubgraph_t getReviewSubgraph(int stg_id);\n\n  /*! \\brief partitioner name */\n  const char* name;\n  std::map<std::string, supportedOps_t> supported_map;\n  std::map<std::string, createSelector_t> selector_map;\n  std::map<std::string, reviewSubgraph_t> review_map;\n  /*! \\brief strategy names */\n  std::vector<const char*> strategies;\n  /*! \\brief subgraph operator name */\n  std::vector<const char*> op_names;\n};\n\n/*!\n * \\brief Registry class to registers things (ops, properties)\n *        Singleton class\n */\ntemplate <class T>\nclass Registry {\n public:\n  /*!\n   * \\brief get singleton pointer to class\n   * \\returns pointer to class\n   */\n  static Registry* get() PRIVATE_SYMBOL {\n    static Registry inst;\n    return &inst;\n  }\n  /*!\n   * \\brief add a new entry\n   * \\returns new object associated with registered name\n   */\n  T& add(const char* name) {\n    T* entry = new T(name);\n    entries.push_back(entry);\n    return *entry;\n  }\n  int size() {\n    return entries.size();\n  }\n  T& get(int idx) {\n    return *(entries.at(idx));\n  }\n\n private:\n  /*! \\brief constructor */\n  Registry() {}\n  /*! \\brief destructor */\n  ~Registry() {}\n  /*! \\brief map of entries in registry */\n  std::vector<T*> entries;\n};\n\n/*!\n * \\brief Macros to help with string concat\n * Annoyingly, the concat_ and concat macros are necessary to\n * be able to use __COUNTER__ in an identifier name\n */\n#define MX_STR_CONCAT_(__a, __b) __a##__b\n#define MX_STR_CONCAT(__a, __b)  MX_STR_CONCAT_(__a, __b)\n\n/*! \\brief convert a token to a string */\n#define MX_STRINGIFY(x) #x\n#define MX_TOSTRING(x)  MX_STRINGIFY(x)\n\n/*! \\brief declare a variable with custom name */\n#define MX_REGISTER_NAME_(Name) MXNet##_CustomOp##_##Name\n#define MX_REGISTER_DEF_(Name)  mxnet::ext::CustomOp MX_REGISTER_NAME_(Name)\n\n#define MX_REGISTER_PROP_NAME_(Name) MXNet##_CustomSubProp##_##Name\n#define MX_REGISTER_PROP_DEF_(Name)  mxnet::ext::CustomPartitioner MX_REGISTER_PROP_NAME_(Name)\n\n#define MX_REGISTER_PASS_NAME_(Name) MXNet##_CustomPass##_##Name\n#define MX_REGISTER_PASS_DEF_(Name)  mxnet::ext::CustomPass MX_REGISTER_PASS_NAME_(Name)\n\n/*! \\brief assign a var to a value */\n#define REGISTER_OP(Name)                              \\\n  MX_STR_CONCAT(MX_REGISTER_DEF_(Name), __COUNTER__) = \\\n      mxnet::ext::Registry<mxnet::ext::CustomOp>::get()->add(MX_TOSTRING(Name))\n\n#define REGISTER_PARTITIONER(Name)                          \\\n  MX_STR_CONCAT(MX_REGISTER_PROP_DEF_(Name), __COUNTER__) = \\\n      mxnet::ext::Registry<mxnet::ext::CustomPartitioner>::get()->add(MX_TOSTRING(Name))\n\n#define REGISTER_PASS(Name)                                 \\\n  MX_STR_CONCAT(MX_REGISTER_PASS_DEF_(Name), __COUNTER__) = \\\n      mxnet::ext::Registry<mxnet::ext::CustomPass>::get()->add(MX_TOSTRING(Name))\n\n/* -------------- BELOW ARE CTYPE FUNCTIONS PROTOTYPES --------------- */\n\n/*!\n * \\brief Following are the C type APIs implemented in the external library\n * Each API has a #define string that is used to lookup the function in the library\n * Followed by the function declaration\n */\n#define MXLIB_OPREGSIZE_STR \"_opRegSize\"\ntypedef int (*opRegSize_t)(void);\n\n#define MXLIB_OPREGGET_STR \"_opRegGet\"\ntypedef int (*opRegGet_t)(int idx,\n                          const char** name,\n                          int* isSGop,\n                          const char*** forward_ctx,\n                          mxnet::ext::fcomp_t** forward_fp,\n                          int* forward_count,\n                          const char*** backward_ctx,\n                          mxnet::ext::fcomp_t** backward_fp,\n                          int* backward_count,\n                          const char*** create_op_ctx,\n                          mxnet::ext::createOpState_t** create_op_fp,\n                          int* create_op_count,\n                          mxnet::ext::parseAttrs_t* parse,\n                          mxnet::ext::inferType_t* type,\n                          mxnet::ext::inferSType_t* stype,\n                          mxnet::ext::inferShape_t* shape,\n                          mxnet::ext::mutateInputs_t* mutate);\n\n#define MXLIB_OPCALLFREE_STR \"_opCallFree\"\ntypedef int (*opCallFree_t)(void* ptr);\n\n#define MXLIB_OPCALLPARSEATTRS_STR \"_opCallParseAttrs\"\ntypedef int (*opCallParseAttrs_t)(parseAttrs_t parseAttrs,\n                                  const char* const* keys,\n                                  const char* const* vals,\n                                  int num,\n                                  int* num_in,\n                                  int* num_out);\n\n#define MXLIB_OPCALLINFERSHAPE_STR \"_opCallInferShape\"\ntypedef int (*opCallInferShape_t)(inferShape_t inferShape,\n                                  const char* const* keys,\n                                  const char* const* vals,\n                                  int num,\n                                  unsigned int** inshapes,\n                                  int* indims,\n                                  int num_in,\n                                  unsigned int*** mod_inshapes,\n                                  int** mod_indims,\n                                  unsigned int*** outshapes,\n                                  int** outdims,\n                                  int num_out);\n\n#define MXLIB_OPCALLINFERTYPE_STR \"_opCallInferType\"\ntypedef int (*opCallInferType_t)(inferType_t inferType,\n                                 const char* const* keys,\n                                 const char* const* vals,\n                                 int num,\n                                 int* intypes,\n                                 int num_in,\n                                 int* outtypes,\n                                 int num_out);\n\n#define MXLIB_OPCALLINFERSTYPE_STR \"_opCallInferSType\"\ntypedef int (*opCallInferSType_t)(inferSType_t inferSType,\n                                  const char* const* keys,\n                                  const char* const* vals,\n                                  int num,\n                                  int* intypes,\n                                  int num_in,\n                                  int* outtypes,\n                                  int num_out);\n\n#define MXLIB_OPCALLFCOMP_STR \"_opCallFCompute\"\ntypedef int (*opCallFComp_t)(fcomp_t fcomp,\n                             const char* const* keys,\n                             const char* const* vals,\n                             int num,\n                             const int64_t** inshapes,\n                             int* indims,\n                             void** indata,\n                             int* intypes,\n                             size_t* inIDs,\n                             const char** indev_type,\n                             int* indev_id,\n                             int num_in,\n                             const int64_t** outshapes,\n                             int* outdims,\n                             void** outdata,\n                             int* outtypes,\n                             size_t* outIDs,\n                             const char** outdev_type,\n                             int* outdev_id,\n                             int num_out,\n                             xpu_malloc_t cpu_malloc,\n                             void* cpu_alloc,\n                             xpu_malloc_t gpu_malloc,\n                             void* gpu_alloc,\n                             void* cuda_stream,\n                             sparse_malloc_t sparse_malloc,\n                             void* sparse_alloc,\n                             int* instypes,\n                             int* outstypes,\n                             void** in_indices,\n                             void** out_indices,\n                             void** in_indptr,\n                             void** out_indptr,\n                             int64_t* in_indices_shapes,\n                             int64_t* out_indices_shapes,\n                             int64_t* in_indptr_shapes,\n                             int64_t* out_indptr_shapes,\n                             void* rng_cpu_states,\n                             void* rng_gpu_states);\n\n#define MXLIB_OPCALLMUTATEINPUTS_STR \"_opCallMutateInputs\"\ntypedef int (*opCallMutateInputs_t)(mutateInputs_t mutate,\n                                    const char* const* keys,\n                                    const char* const* vals,\n                                    int num,\n                                    int** mutate_indices,\n                                    int* indices_size);\n\n#define MXLIB_OPCALLCREATEOPSTATE_STR \"_opCallCreateOpState\"\ntypedef int (*opCallCreateOpState_t)(createOpState_t create_op,\n                                     const char* const* keys,\n                                     const char* const* vals,\n                                     int num,\n                                     const char* dev_type,\n                                     int dev_id,\n                                     unsigned int** inshapes,\n                                     int* indims,\n                                     int num_in,\n                                     const int* intypes,\n                                     void** state_op);\n\n#define MXLIB_OPCALLDESTROYOPSTATE_STR \"_opCallDestroyOpState\"\ntypedef int (*opCallDestroyOpState_t)(void* state_op);\n\n#define MXLIB_OPCALLFSTATEFULCOMP_STR \"_opCallFStatefulCompute\"\ntypedef int (*opCallFStatefulComp_t)(int is_forward,\n                                     void* state_op,\n                                     const int64_t** inshapes,\n                                     int* indims,\n                                     void** indata,\n                                     int* intypes,\n                                     size_t* inIDs,\n                                     const char** indev_type,\n                                     int* indev_id,\n                                     int num_in,\n                                     const int64_t** outshapes,\n                                     int* outdims,\n                                     void** outdata,\n                                     int* outtypes,\n                                     size_t* outIDs,\n                                     const char** outdev_type,\n                                     int* outdev_id,\n                                     int num_out,\n                                     xpu_malloc_t cpu_malloc,\n                                     void* cpu_alloc,\n                                     xpu_malloc_t gpu_malloc,\n                                     void* gpu_alloc,\n                                     void* stream,\n                                     sparse_malloc_t sparse_malloc,\n                                     void* sparse_alloc,\n                                     int* instypes,\n                                     int* outstypes,\n                                     void** in_indices,\n                                     void** out_indices,\n                                     void** in_indptr,\n                                     void** out_indptr,\n                                     int64_t* in_indices_shapes,\n                                     int64_t* out_indices_shapes,\n                                     int64_t* in_indptr_shapes,\n                                     int64_t* out_indptr_shapes,\n                                     void* rng_cpu_states,\n                                     void* rng_gpu_states);\n\n#define MXLIB_PARTREGSIZE_STR \"_partRegSize\"\ntypedef int (*partRegSize_t)(void);\n\n#define MXLIB_PARTREGGETCOUNT_STR \"_partRegGetCount\"\ntypedef int (*partRegGetCount_t)(int idx, const char** name);\n\n#define MXLIB_PARTREGGET_STR \"_partRegGet\"\ntypedef void (*partRegGet_t)(int part_idx,\n                             int stg_idx,\n                             const char** strategy,\n                             supportedOps_t* supportedOps,\n                             createSelector_t* createSelector,\n                             reviewSubgraph_t* reviewSubgraph,\n                             const char** op_name);\n\n#define MXLIB_PARTCALLSUPPORTEDOPS_STR \"_partCallSupportedOps\"\ntypedef int (*partCallSupportedOps_t)(supportedOps_t supportedOps,\n                                      const char* json,\n                                      int num_ids,\n                                      int* ids,\n                                      const char* const* opt_keys,\n                                      const char* const* opt_vals,\n                                      int num_opts);\n\n#define MXLIB_PARTCALLCREATESELECTOR_STR \"_partCallCreateSelector\"\ntypedef int (*partCallCreateSelector_t)(createSelector_t createSelector,\n                                        const char* json,\n                                        void** selector,\n                                        const char* const* opt_keys,\n                                        const char* const* opt_vals,\n                                        int num_opts);\n\n#define MXLIB_PARTCALLSELECT_STR \"_partCallSelect\"\ntypedef void (*partCallSelect_t)(void* sel_inst, int nodeID, int* selected);\n\n#define MXLIB_PARTCALLSELECTINPUT_STR \"_partCallSelectInput\"\ntypedef void (*partCallSelectInput_t)(void* sel_inst, int nodeID, int input_nodeID, int* selected);\n\n#define MXLIB_PARTCALLSELECTOUTPUT_STR \"_partCallSelectOutput\"\ntypedef void (*partCallSelectOutput_t)(void* sel_inst,\n                                       int nodeID,\n                                       int output_nodeID,\n                                       int* selected);\n\n#define MXLIB_PARTCALLFILTER_STR \"_partCallFilter\"\ntypedef void (*partCallFilter_t)(void* sel_inst,\n                                 int* candidates,\n                                 int num_candidates,\n                                 int** keep,\n                                 int* num_keep);\n\n#define MXLIB_PARTCALLRESET_STR \"_partCallReset\"\ntypedef void (*partCallReset_t)(void* sel_inst);\n\n#define MXLIB_PARTCALLREVIEWSUBGRAPH_STR \"_partCallReviewSubgraph\"\ntypedef int (*partCallReviewSubgraph_t)(reviewSubgraph_t reviewSubgraph,\n                                        const char* json,\n                                        int subgraph_id,\n                                        int* accept,\n                                        const char* const* opt_keys,\n                                        const char* const* opt_vals,\n                                        int num_opts,\n                                        char*** attr_keys,\n                                        char*** attr_vals,\n                                        int* num_attrs,\n                                        const char* const* arg_names,\n                                        int num_args,\n                                        void* const* arg_data,\n                                        const int64_t* const* arg_shapes,\n                                        const int* arg_dims,\n                                        const int* arg_types,\n                                        const size_t* arg_IDs,\n                                        const char* const* arg_dev_type,\n                                        const int* arg_dev_id,\n                                        const char* const* aux_names,\n                                        int num_aux,\n                                        void* const* aux_data,\n                                        const int64_t* const* aux_shapes,\n                                        const int* aux_dims,\n                                        const int* aux_types,\n                                        const size_t* aux_IDs,\n                                        const char* const* aux_dev_type,\n                                        const int* aux_dev_id);\n\n#define MXLIB_PASSREGSIZE_STR \"_passRegSize\"\ntypedef int (*passRegSize_t)(void);\n\n#define MXLIB_PASSREGGET_STR \"_passRegGet\"\ntypedef void (*passRegGet_t)(int pass_idx, graphPass_t* graphPass, const char** pass_name);\n\n#define MXLIB_PASSCALLGRAPHPASS_STR \"_passCallGraphPass\"\ntypedef int (*passCallGraphPass_t)(graphPass_t graphPass,\n                                   const char* in_graph,\n                                   char** out_graph,\n                                   const char* const* opt_keys,\n                                   const char* const* opt_vals,\n                                   int num_opts,\n                                   const char* pass_name,\n                                   const char* const* arg_names,\n                                   int num_args,\n                                   void* const* arg_data,\n                                   const int64_t* const* arg_shapes,\n                                   const int* arg_dims,\n                                   const int* arg_types,\n                                   const size_t* arg_IDs,\n                                   const char* const* arg_dev_type,\n                                   const int* arg_dev_id,\n                                   const char* const* aux_names,\n                                   int num_aux,\n                                   void* const* aux_data,\n                                   const int64_t* const* aux_shapes,\n                                   const int* aux_dims,\n                                   const int* aux_types,\n                                   const size_t* aux_IDs,\n                                   const char* const* aux_dev_type,\n                                   const int* aux_dev_id,\n                                   nd_malloc_t nd_malloc,\n                                   const void* nd_alloc);\n\n#define MXLIB_INITIALIZE_STR \"initialize\"\ntypedef int (*initialize_t)(int version);\n\n#define MXLIB_OPVERSION_STR \"_opVersion\"\ntypedef int (*opVersion_t)();\n\n#define MXLIB_MSGSIZE_STR \"_msgSize\"\ntypedef int (*msgSize_t)(void);\n\n#define MXLIB_MSGGET_STR \"_msgGet\"\ntypedef int (*msgGet_t)(int idx, const char** msg);\n\n/*! \\brief StatefulOp wrapper class to pass to backend OpState */\nclass CustomStatefulOpWrapper {\n public:\n  ~CustomStatefulOpWrapper();\n  explicit CustomStatefulOpWrapper(CustomStatefulOp* inst, opCallDestroyOpState_t destroy)\n      : instance(inst), destroy_(destroy) {}\n  CustomStatefulOp* get_instance() {\n    return instance;\n  }\n\n private:\n  CustomStatefulOp* instance;\n  opCallDestroyOpState_t destroy_;\n};\n\n#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)\n#define MX_INT_RET  __declspec(dllexport) int __cdecl\n#define MX_VOID_RET __declspec(dllexport) void __cdecl\n#else\n#define MX_INT_RET  int\n#define MX_VOID_RET void\n#endif\n\n}  // namespace ext\n}  // namespace mxnet\n\nextern \"C\" {\n/*! \\brief returns MXNet library version */\nMX_INT_RET _opVersion();\n\n/*! \\brief returns number of ops registered in this library */\nMX_INT_RET _opRegSize();\n\n/*! \\brief returns operator registration at specified index */\nMX_VOID_RET _opRegGet(int idx,\n                      const char** name,\n                      int* isSGop,\n                      const char*** forward_ctx,\n                      mxnet::ext::fcomp_t** forward_fp,\n                      int* forward_count,\n                      const char*** backward_ctx,\n                      mxnet::ext::fcomp_t** backward_fp,\n                      int* backward_count,\n                      const char*** create_op_ctx,\n                      mxnet::ext::createOpState_t** create_op_fp,\n                      int* create_op_count,\n                      mxnet::ext::parseAttrs_t* parse,\n                      mxnet::ext::inferType_t* type,\n                      mxnet::ext::inferSType_t* stype,\n                      mxnet::ext::inferShape_t* shape,\n                      mxnet::ext::mutateInputs_t* mutate);\n\n/*! \\brief calls free from the external library for library allocated arrays */\nMX_VOID_RET _opCallFree(void* ptr);\n\n/*! \\brief returns status of calling parse attributes function for operator from library */\nMX_INT_RET _opCallParseAttrs(mxnet::ext::parseAttrs_t parseAttrs,\n                             const char* const* keys,\n                             const char* const* vals,\n                             int num,\n                             int* num_in,\n                             int* num_out);\n\n/*! \\brief returns status of calling inferShape function for operator from library */\nMX_INT_RET _opCallInferShape(mxnet::ext::inferShape_t inferShape,\n                             const char* const* keys,\n                             const char* const* vals,\n                             int num,\n                             unsigned int** inshapes,\n                             int* indims,\n                             int num_in,\n                             unsigned int*** mod_inshapes,\n                             int** mod_indims,\n                             unsigned int*** outshapes,\n                             int** outdims,\n                             int num_out);\n\n/*! \\brief returns status of calling inferType function for operator from library */\nMX_INT_RET _opCallInferType(mxnet::ext::inferType_t inferType,\n                            const char* const* keys,\n                            const char* const* vals,\n                            int num,\n                            int* intypes,\n                            int num_in,\n                            int* outtypes,\n                            int num_out);\n\n/*! \\brief returns status of calling inferSType function for operator from library */\nMX_INT_RET _opCallInferSType(mxnet::ext::inferSType_t inferSType,\n                             const char* const* keys,\n                             const char* const* vals,\n                             int num,\n                             int* instypes,\n                             int num_in,\n                             int* outstypes,\n                             int num_out);\n\n/*! \\brief returns status of calling Forward/Backward function for operator from library */\nMX_INT_RET _opCallFCompute(mxnet::ext::fcomp_t fcomp,\n                           const char* const* keys,\n                           const char* const* vals,\n                           int num,\n                           const int64_t** inshapes,\n                           int* indims,\n                           void** indata,\n                           int* intypes,\n                           size_t* inIDs,\n                           const char** indev_type,\n                           int* indev_id,\n                           int num_in,\n                           const int64_t** outshapes,\n                           int* outdims,\n                           void** outdata,\n                           int* outtypes,\n                           size_t* outIDs,\n                           const char** outdev_type,\n                           int* outdev_id,\n                           int num_out,\n                           mxnet::ext::xpu_malloc_t cpu_malloc,\n                           void* cpu_alloc,\n                           mxnet::ext::xpu_malloc_t gpu_malloc,\n                           void* gpu_alloc,\n                           void* cuda_stream,\n                           mxnet::ext::sparse_malloc_t sparse_malloc,\n                           void* sparse_alloc,\n                           int* instypes,\n                           int* outstypes,\n                           void** in_indices,\n                           void** out_indices,\n                           void** in_indptr,\n                           void** out_indptr,\n                           int64_t* in_indices_shapes,\n                           int64_t* out_indices_shapes,\n                           int64_t* in_indptr_shapes,\n                           int64_t* out_indptr_shapes,\n                           void* rng_cpu_states,\n                           void* rng_gpu_states);\n\n/*! \\brief returns status of calling mutateInputs function for operator from library */\nMX_INT_RET _opCallMutateInputs(mxnet::ext::mutateInputs_t mutate,\n                               const char* const* keys,\n                               const char* const* vals,\n                               int num,\n                               int** mutate_indices,\n                               int* indices_size);\n\n/*! \\brief returns status of calling createStatefulOp function for operator from library */\nMX_INT_RET _opCallCreateOpState(mxnet::ext::createOpState_t create_op,\n                                const char* const* keys,\n                                const char* const* vals,\n                                int num,\n                                const char* dev_type,\n                                int dev_id,\n                                unsigned int** inshapes,\n                                int* indims,\n                                int num_in,\n                                const int* intypes,\n                                void** state_op);\n\n/*! \\brief returns status of deleting StatefulOp instance for operator from library */\nMX_VOID_RET _opCallDestroyOpState(void* state_op);\n\n/*! \\brief returns status of calling Stateful Forward/Backward for operator from library */\nMX_INT_RET _opCallFStatefulCompute(int is_forward,\n                                   void* state_op,\n                                   const int64_t** inshapes,\n                                   int* indims,\n                                   void** indata,\n                                   int* intypes,\n                                   size_t* inIDs,\n                                   const char** indev_type,\n                                   int* indev_id,\n                                   int num_in,\n                                   const int64_t** outshapes,\n                                   int* outdims,\n                                   void** outdata,\n                                   int* outtypes,\n                                   size_t* outIDs,\n                                   const char** outdev_type,\n                                   int* outdev_id,\n                                   int num_out,\n                                   mxnet::ext::xpu_malloc_t cpu_malloc,\n                                   void* cpu_alloc,\n                                   mxnet::ext::xpu_malloc_t gpu_malloc,\n                                   void* gpu_alloc,\n                                   void* stream,\n                                   mxnet::ext::sparse_malloc_t sparse_malloc,\n                                   void* sparse_alloc,\n                                   int* instypes,\n                                   int* outstypes,\n                                   void** in_indices,\n                                   void** out_indices,\n                                   void** in_indptr,\n                                   void** out_indptr,\n                                   int64_t* in_indices_shapes,\n                                   int64_t* out_indices_shapes,\n                                   int64_t* in_indptr_shapes,\n                                   int64_t* out_indptr_shapes,\n                                   void* rng_cpu_states,\n                                   void* rng_gpu_states);\n\n/*! \\brief returns number of partitioners registered in this library */\nMX_INT_RET _partRegSize();\n\n/* returns number of strategies registered for partitioner\n * at specified index */\nMX_INT_RET _partRegGetCount(int idx, const char** name);\n\n/*! \\brief returns partitioner registration at specified index */\nMX_VOID_RET _partRegGet(int part_idx,\n                        int stg_idx,\n                        const char** strategy,\n                        mxnet::ext::supportedOps_t* supportedOps,\n                        mxnet::ext::createSelector_t* createSelector,\n                        mxnet::ext::reviewSubgraph_t* reviewSubgraph,\n                        const char** op_name);\n\n/*! \\brief returns status of calling supported ops function from library */\nMX_INT_RET _partCallSupportedOps(mxnet::ext::supportedOps_t supportedOps,\n                                 const char* json,\n                                 int num_ids,\n                                 int* ids,\n                                 const char* const* opt_keys,\n                                 const char* const* opt_vals,\n                                 int num_opts);\n\n/*! \\brief returns status of calling create selector function from library */\nMX_INT_RET _partCallCreateSelector(mxnet::ext::createSelector_t createSelector,\n                                   const char* json,\n                                   void** selector,\n                                   const char* const* opt_keys,\n                                   const char* const* opt_vals,\n                                   int num_opts);\n\n/*! \\brief returns status of calling select function from library */\nMX_VOID_RET _partCallSelect(void* sel_inst, int nodeID, int* selected);\n\n/*! \\brief returns status of calling select input function from library */\nMX_VOID_RET _partCallSelectInput(void* sel_inst, int nodeID, int input_nodeID, int* selected);\n\n/*! \\brief returns status of calling select output function from library */\nMX_VOID_RET _partCallSelectOutput(void* sel_inst, int nodeID, int output_nodeID, int* selected);\n\n/*! \\brief returns status of calling filter function from library */\nMX_VOID_RET _partCallFilter(void* sel_inst,\n                            int* candidates,\n                            int num_candidates,\n                            int** keep,\n                            int* num_keep);\n\n/*! \\brief returns status of calling reset selector function from library */\nMX_VOID_RET _partCallReset(void* sel_inst);\n\n/*! \\brief returns status of calling review subgraph function from library */\nMX_INT_RET _partCallReviewSubgraph(mxnet::ext::reviewSubgraph_t reviewSubgraph,\n                                   const char* json,\n                                   int subgraph_id,\n                                   int* accept,\n                                   const char* const* opt_keys,\n                                   const char* const* opt_vals,\n                                   int num_opts,\n                                   char*** attr_keys,\n                                   char*** attr_vals,\n                                   int* num_attrs,\n                                   const char* const* arg_names,\n                                   int num_args,\n                                   void* const* arg_data,\n                                   const int64_t* const* arg_shapes,\n                                   const int* arg_dims,\n                                   const int* arg_types,\n                                   const size_t* arg_IDs,\n                                   const char* const* arg_dev_type,\n                                   const int* arg_dev_id,\n                                   const char* const* aux_names,\n                                   int num_aux,\n                                   void* const* aux_data,\n                                   const int64_t* const* aux_shapes,\n                                   const int* aux_dims,\n                                   const int* aux_types,\n                                   const size_t* aux_IDs,\n                                   const char* const* aux_dev_type,\n                                   const int* aux_dev_id);\n\n/*! \\brief returns number of graph passes registered in this library */\nMX_INT_RET _passRegSize();\n\n/*! \\brief returns pass registration at specified index */\nMX_VOID_RET _passRegGet(int pass_idx, mxnet::ext::graphPass_t* graphPass, const char** pass_name);\n\n/*! \\brief returns status of calling graph pass function from library */\nMX_INT_RET _passCallGraphPass(mxnet::ext::graphPass_t graphPass,\n                              const char* json,\n                              char** out_graph,\n                              const char* const* opt_keys,\n                              const char* const* opt_vals,\n                              int num_opts,\n                              const char* pass_name,\n                              const char* const* arg_names,\n                              int num_args,\n                              void* const* arg_data,\n                              const int64_t* const* arg_shapes,\n                              const int* arg_dims,\n                              const int* arg_types,\n                              const size_t* arg_IDs,\n                              const char* const* arg_dev_type,\n                              const int* arg_dev_id,\n                              const char* const* aux_names,\n                              int num_aux,\n                              void* const* aux_data,\n                              const int64_t* const* aux_shapes,\n                              const int* aux_dims,\n                              const int* aux_types,\n                              const size_t* aux_IDs,\n                              const char* const* aux_dev_type,\n                              const int* aux_dev_id,\n                              mxnet::ext::nd_malloc_t nd_malloc,\n                              const void* nd_alloc);\n\n/*!\n * \\brief Checks if the MXNet version is supported by the library.\n * If supported, initializes the library.\n * \\param version MXNet version number passed to library and defined as:\n *                MXNET_VERSION = (MXNET_MAJOR*10000 + MXNET_MINOR*100 + MXNET_PATCH)\n * \\return Non-zero value on error i.e. library incompatible with passed MXNet version\n */\n#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)\n__declspec(dllexport) mxnet::ext::MXReturnValue __cdecl\n#else\nmxnet::ext::MXReturnValue\n#endif\n    initialize(int version);\n\nMX_INT_RET _msgSize();\n\n/*! \\brief returns operator registration at specified index */\nMX_VOID_RET _msgGet(int idx, const char** msg);\n}  // extern \"C\"\n\n#endif  // MXNET_LIB_API_H_\n"
  },
  {
    "path": "include/mxnet/libinfo.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file libinfo.h\n * \\author larroy\n * \\brief get features of the MXNet library at runtime\n */\n\n#pragma once\n\n#include <string>\n#include <vector>\n#include <array>\n#include <memory>\n#include \"dmlc/base.h\"\n#include \"mshadow/base.h\"\n#include \"c_api.h\"\n\n/*!\n *\\brief whether to use opencv support\n */\n#ifndef MXNET_USE_OPENCV\n#define MXNET_USE_OPENCV 0\n#endif\n\n/*!\n *\\brief whether to use cuda support\n */\n#ifndef MXNET_USE_CUDA\n#define MXNET_USE_CUDA MSHADOW_USE_CUDA\n#endif\n\n/*!\n *\\brief whether to use cudnn library for convolution\n */\n#ifndef MXNET_USE_CUDNN\n#define MXNET_USE_CUDNN MSHADOW_USE_CUDNN\n#endif\n\n#ifndef MXNET_USE_CUTENSOR\n#define MXNET_USE_CUTENSOR MSHADOW_USE_CUTENSOR\n#endif\n\n#ifndef MXNET_USE_NVML\n#define MXNET_USE_NVML 0\n#endif\n\n#ifndef MXNET_USE_NCCL\n#define MXNET_USE_NCCL 0\n#endif\n\n/*!\n *\\brief whether to use cusolver library\n */\n#ifndef MXNET_USE_CUSOLVER\n#define MXNET_USE_CUSOLVER MSHADOW_USE_CUSOLVER\n#endif\n\n/*! \\brief Error message for using gpu when MXNET_USE_CUDA==0 */\n#define MXNET_GPU_NOT_ENABLED_ERROR \"GPU is not enabled\"\n\n#ifndef MXNET_USE_TENSORRT\n#define MXNET_USE_TENSORRT 0\n#endif\n\n#ifndef MXNET_USE_BLAS_ATLAS\n#define MXNET_USE_BLAS_ATLAS 0\n#endif\n\n#ifndef MXNET_USE_BLAS_OPEN\n#define MXNET_USE_BLAS_OPEN 0\n#endif\n\n#ifndef MXNET_USE_BLAS_MKL\n#define MXNET_USE_BLAS_MKL 0\n#endif\n\n#ifndef MXNET_USE_BLAS_APPLE\n#define MXNET_USE_BLAS_APPLE 0\n#endif\n\n#ifndef MXNET_USE_LAPACK\n#define MXNET_USE_LAPACK 0\n#endif\n\n#ifndef MXNET_USE_ONEDNN\n#define MXNET_USE_ONEDNN 0\n#endif\n\n#ifndef MXNET_USE_OPENMP\n#define MXNET_USE_OPENMP 0\n#endif\n\n#ifndef MXNET_USE_F16C\n#define MXNET_USE_F16C MSHADOW_USE_F16C\n#endif\n\n#ifndef MXNET_USE_DIST_KVSTORE\n#define MXNET_USE_DIST_KVSTORE 0\n#endif\n\n#ifndef MXNET_USE_SIGNAL_HANDLER\n#define MXNET_USE_SIGNAL_HANDLER 0\n#endif\n\n#ifndef MXNET_USE_INT64_TENSOR_SIZE\n#define MXNET_USE_INT64_TENSOR_SIZE MSHADOW_INT64_TENSOR_SIZE\n#endif\n\n#ifndef MXNET_USE_TVM_OP\n#define MXNET_USE_TVM_OP 0\n#endif\n\nnamespace mxnet {\nnamespace features {\n// Check compile flags such as CMakeLists.txt\n\n/// Compile time features\n// ATTENTION: When changing this enum, match the strings in the implementation file!\nenum : unsigned {\n  // NVIDIA, CUDA\n  CUDA = 0,\n  CUDNN,\n  NCCL,\n  TENSORRT,\n  CUTENSOR,\n\n  // CPU Features / optimizations\n  CPU_SSE,\n  CPU_SSE2,\n  CPU_SSE3,\n  CPU_SSE4_1,\n  CPU_SSE4_2,\n  CPU_SSE4A,  // AMD extensions to SSE4\n  CPU_AVX,\n  CPU_AVX2,\n\n  // Multiprocessing / CPU / System\n  OPENMP,\n  SSE,\n  F16C,\n  JEMALLOC,\n\n  // Math libraries & BLAS\n  // Flavour of BLAS\n  BLAS_OPEN,\n  BLAS_ATLAS,\n  // Intel(R) Math Kernel Library\n  BLAS_MKL,\n  BLAS_APPLE,\n  // Other math libraries:\n  // Linear Algebra PACKage\n  LAPACK,\n  // oneAPI Deep Neural Network Library (oneDNN)\n  ONEDNN,\n\n  // Image processing\n  OPENCV,\n\n  // Misc\n  DIST_KVSTORE,\n  INT64_TENSOR_SIZE,\n\n  // Signal handler to print stack traces on exceptions\n  SIGNAL_HANDLER,\n  DEBUG,\n\n  // TVM operator\n  TVM_OP,\n\n  // size indicator\n  MAX_FEATURES\n};\n\nstruct EnumNames {\n  static const std::vector<std::string> names;\n};\n\nstruct LibInfo {\n  LibInfo();\n  static LibInfo* getInstance();\n  const std::array<LibFeature, MAX_FEATURES>& getFeatures() {\n    return m_lib_features;\n  }\n\n private:\n  std::array<LibFeature, MAX_FEATURES> m_lib_features;\n  static std::unique_ptr<LibInfo> m_inst;\n};\n\n/*!\n * \\return true if the given feature is supported\n */\nbool is_enabled(unsigned feat);\n\n}  // namespace features\n}  // namespace mxnet\n"
  },
  {
    "path": "include/mxnet/ndarray.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file ndarray.h\n * \\brief NDArray interface that handles array arithematics.\n */\n#ifndef MXNET_NDARRAY_H_\n#define MXNET_NDARRAY_H_\n\n#include <dmlc/base.h>\n#include <dmlc/io.h>\n#include <dmlc/logging.h>\n#include <dmlc/registry.h>\n#include <dmlc/type_traits.h>\n#include <nnvm/node.h>\n\n#include <algorithm>\n#include <map>\n#include <memory>\n#include <string>\n#include <vector>\n#include \"./base.h\"\n#include \"./engine.h\"\n#include \"./storage.h\"\n// check c++11\n#if DMLC_USE_CXX11 == 0\n#error \"cxx11 was required for ndarray module\"\n#endif\n\nnamespace dnnl {\nstruct memory;\n}  // namespace dnnl\n\nnamespace mxnet {\n// enum for storage types\nnamespace csr {\nenum CSRAuxType { kIndPtr, kIdx };\n}\n\nnamespace rowsparse {\nenum RowSparseAuxType { kIdx };\n}\n\nenum NDArrayStorageType {\n  kUndefinedStorage = -1,  // undefined storage\n  kDefaultStorage,         // dense\n  kRowSparseStorage,       // row sparse\n  kCSRStorage,             // csr\n};\n\nenum NDArrayFormatErr {\n  kNormalErr,     // normal\n  kCSRShapeErr,   // shape mismatch for csr\n  kCSRIndPtrErr,  // indptr error for csr\n  kCSRIdxErr,     // idx error for csr\n  kRSPShapeErr,   // shape mismatch for row sparse\n  kRSPIdxErr,     // indices error for row sparse\n};\n\nclass DNNLMemory;\n\n/*!\n * \\brief ndarray interface\n */\nclass NDArray {\n public:\n  /*! \\brief default constructor */\n  NDArray() : autograd_entry_(nullptr) {}\n  /*!\n   * \\brief constructs a new dynamic NDArray\n   * \\param shape the shape of array\n   * \\param ctx context of NDArray\n   * \\param delay_alloc whether delay the allocation\n   * \\param dtype data type of this ndarray\n   */\n  NDArray(const mxnet::TShape& shape,\n          Context ctx,\n          bool delay_alloc = false,\n          int dtype        = mshadow::default_type_flag)\n      : ptr_(std::make_shared<Chunk>(shape, ctx, delay_alloc, dtype)),\n        shape_(shape),\n        dtype_(dtype),\n        storage_type_(kDefaultStorage),\n        autograd_entry_(nullptr) {}\n  /*! \\brief constructor for NDArray with storage type\n   */\n  NDArray(const NDArrayStorageType stype,\n          const mxnet::TShape& shape,\n          Context ctx,\n          bool delay_alloc                     = true,\n          int dtype                            = mshadow::default_type_flag,\n          const std::vector<int>& aux_types    = {},\n          const mxnet::ShapeVector& aux_shapes = {},\n          const mxnet::TShape& storage_shape   = mxnet::TShape(mshadow::Shape1(0))) {\n    ReInit(stype, shape, ctx, dtype, delay_alloc, &aux_types, &aux_shapes, &storage_shape);\n  }\n  /*!\n   * \\brief constructs a new dynamic NDArray whose shape is unknown,\n   *        hence the NDArray is inherently lazily created\n   * \\param ctx context of NDArray\n   * \\param dtype data type of this ndarray\n   */\n  explicit NDArray(Context ctx, int dtype = mshadow::default_type_flag)\n      : ptr_(std::make_shared<Chunk>(mxnet::TShape(mshadow::Shape1(0)), ctx, true, dtype)),\n        shape_(),\n        dtype_(dtype),\n        storage_type_(kDefaultStorage),\n        autograd_entry_(nullptr) {}\n  /*!\n   * \\brief constructing a static NDArray that shares data with TBlob\n   *  Use with caution: allocate ONLY ONE NDArray for each TBlob,\n   *  make sure the memory region is available through out the life of NDArray\n   * \\param data the memory content of static data\n   * \\param dev_id the device id this tensor sits at\n   */\n  NDArray(const TBlob& data, int dev_id)\n      : ptr_(std::make_shared<Chunk>(data, dev_id)),\n        shape_(data.shape_),\n        dtype_(data.type_flag_),\n        storage_type_(kDefaultStorage),\n        autograd_entry_(nullptr) {}\n\n  /*!\n   * \\brief constructing a static NDArray that shares data with TBlob which is with deleter\n   *  Use with caution: allocate ONLY ONE NDArray for each TBlob,\n   *  make sure the memory region is available through out the life of NDArray\n   * \\param data the memory content of static data\n   * \\param dev_id the device id this tensor sits at\n   * \\param deleter the function pointer of custom deleter\n   */\n  NDArray(const TBlob& data, int dev_id, const std::function<void()>& deleter)\n      : ptr_(new Chunk(data, dev_id),\n             [deleter](Chunk* p) {\n               deleter();  // call custom deleter\n               delete p;   // delete Chunk object\n             }),\n        shape_(data.shape_),\n        dtype_(data.type_flag_),\n        storage_type_(kDefaultStorage),\n        autograd_entry_(nullptr) {}\n\n  /*! \\brief create ndarray from shared memory */\n  NDArray(int shared_pid, int shared_id, const mxnet::TShape& shape, int dtype)\n      : ptr_(std::make_shared<Chunk>(shared_pid, shared_id, shape, dtype)),\n        shape_(shape),\n        dtype_(dtype),\n        storage_type_(kDefaultStorage),\n        autograd_entry_(nullptr) {}\n\n  /*!\n   * \\brief constructing a static NDArray of non-default storage that shares data with TBlob\n   *  Use with caution: allocate ONLY ONE NDArray for each TBlob,\n   *  make sure the memory region is available through out the life of NDArray\n   * \\param stype the storage type of NDArray\n   * \\param shape the shape of NDArray\n   * \\param data the memory content of static data\n   * \\param aux_data the memory content of static aux data\n   * \\param dev_id the device id this tensor sits at\n   */\n  NDArray(const NDArrayStorageType stype,\n          const mxnet::TShape& shape,\n          const TBlob& data,\n          const std::vector<TBlob>& aux_data,\n          int dev_id)\n      : ptr_(std::make_shared<Chunk>(stype, data, aux_data, dev_id)),\n        shape_(shape),\n        dtype_(data.type_flag_),\n        storage_type_(stype),\n        autograd_entry_(nullptr) {}\n  /*!\n   * \\brief initialize the NDArray, assuming it is not assigned a meaningful shape before\n   * \\param shape the shape of the NDArray\n   */\n  void Init(const mxnet::TShape& shape) {\n    ptr_->Init(shape, this->dtype_);\n    this->shape_ = shape;\n  }\n\n  void InitDetached(const NDArray* src) {\n    *this           = *src;\n    autograd_entry_ = nnvm::NodeEntry(nullptr);\n  }\n  inline void ReInit() {\n    ptr_ = nullptr;\n    Init(kUndefinedStorage, TShape(), -1);\n  }\n  void ReInit(const NDArrayStorageType stype,\n              const mxnet::TShape& shape,\n              Context ctx,\n              int dtype,\n              bool delay_alloc                     = true,\n              const std::vector<int>* aux_types    = nullptr,\n              const mxnet::ShapeVector* aux_shapes = nullptr,\n              const mxnet::TShape* storage_shape   = nullptr);\n\n  void SelfReorder2Default();\n  /*!\n   * \\brief set the correct shape of NDArray directly from the storage_shape of its own chunk.\n   */\n  void SetShapeFromChunk() const;\n  /*\n   * This indicates whether an array is a view of another array (created by\n   * reshape or slice). If an array is a view and the data is stored in\n   * DNNL format, we need to convert the data to the default format when\n   * data in the view is accessed.\n   */\n  inline bool IsView() const {\n    // View only works on the default storage\n    if (storage_type() != kDefaultStorage)\n      return false;\n    // If the array reuses memory, its shape may be different from the storage\n    // shape. However, we shouldn't consider it as a view.\n    if (reuse_)\n      return false;\n    return byte_offset_ > 0 || shape() != ptr_->storage_shape;\n  }\n\n  /* \\brief Check whether the two arrays are the same array */\n  inline bool IsSame(const NDArray& other) const {\n    return ptr_ == other.ptr_ && shape_ == other.shape_ && byte_offset_ == other.byte_offset_ &&\n           dtype_ == other.dtype_;\n  }\n\n  /*!\n   * \\return the shape of current NDArray.\n   */\n  inline const mxnet::TShape& shape() const {\n    return shape_;\n  }\n  /*!\n   * \\return the shape of underlying chunk which stores the NDArray data/value.\n   *  It is only intended for non-default storage. For row-sparse storage, it is the shape of\n   *  the tensor which stores the non-zero values.\n   */\n  inline const mxnet::TShape& storage_shape() const {\n    CHECK(ptr_ != nullptr);\n    CHECK_NE(storage_type(), kDefaultStorage)\n        << \"storage_shape() is not intended for kDefaultStorage.\";\n    return ptr_->storage_shape;\n  }\n\n  /*!\n   * \\brief get the shape of aux_data(index)\n   * \\param index the index of the aux data\n   * \\return the shape of aux data at given index\n   */\n  inline const mxnet::TShape& aux_shape(size_t index) const {\n    CHECK_NE(storage_type(), kDefaultStorage) << \"aux_shape() is not intended for kDefaultStorage.\";\n    return ptr_->aux_shapes[index];\n  }\n\n  /* \\return the shapes of all aux data */\n  const mxnet::ShapeVector& aux_shapes() const {\n    CHECK_NE(storage_type(), kDefaultStorage)\n        << \"aux_shapes() is not intended for kDefaultStorage.\";\n    return ptr_->aux_shapes;\n  }\n\n  /*! returns the dtypes of all aux data */\n  const std::vector<int>& aux_types() const {\n    CHECK_NE(storage_type(), kDefaultStorage) << \"aux_types() is not intended for kDefaultStorage.\";\n    return ptr_->aux_types;\n  }\n\n  /*!\n   * \\brief For a sparse operation on a csr matrix for example,\n   * the size of the column index array\n   * is an estimated value in the beginning for allocating enough capacity\n   * for the final result. After the operation is done, the exact size of\n   * the shape is known and need to be reset using this function.\n   */\n  inline void set_aux_shape(size_t index, const mxnet::TShape& shape) const {\n    CHECK_NE(storage_type(), kDefaultStorage)\n        << \"set_aux_shape() is not intended for kDefaultStorage.\";\n    ptr_->set_aux_shape(index, shape);\n  }\n\n  /*!\n   * \\return the data TBlob\n   */\n  inline const TBlob& data() const {\n    if (storage_type() == kDefaultStorage)\n      CheckAndAlloc();\n    SetTBlob();\n    return tblob_;\n  }\n  /*!\n   * \\return the gradient ndarray.\n   */\n  NDArray grad() const;\n\n  /*!\n   * \\return the aux TBlob\n   */\n  inline TBlob aux_data(size_t i) const {\n    auto stype = storage_type();\n    TBlob res;\n    auto shape = aux_shape(i);\n    auto type  = aux_type(i);\n    MSHADOW_TYPE_SWITCH(type, DType, {\n      auto dptr = static_cast<DType*>(ptr_->aux_handles[i].dptr);\n      CHECK(stype == kRowSparseStorage || stype == kCSRStorage)\n          << \"Unexpected storage type: \" << stype;\n      res = TBlob(dptr, shape, ptr_->aux_handles[i].ctx.dev_mask(), type);\n    });\n    return res;\n  }\n  /*!\n   * \\return the context of NDArray, this function is only valid when the NDArray is not empty\n   */\n  inline Context ctx() const {\n    CHECK(!is_none());\n    return ptr_->shandle.ctx;\n  }\n  /*!\n   * \\return the data type of NDArray, this function is only valid when the NDArray is not empty\n   */\n  inline int dtype() const {\n    return dtype_;\n  }\n  inline int aux_type(size_t i) const {\n    CHECK(!is_none());\n    return ptr_->aux_types[i];\n  }\n\n  inline NDArrayStorageType storage_type() const {\n    return storage_type_;\n  }\n  /*! \\return whether this ndarray is not initialized */\n  inline bool is_none() const {\n    return ptr_.get() == nullptr;\n  }\n  /*! \\return updated grad state in autograd_entry_ */\n  bool fresh_out_grad() const;\n  /*! \\return updated grad state in autograd_entry_ */\n  void set_fresh_out_grad(bool state) const;\n  /*! \\brief Returns true if a sparse ndarray's aux_data and storage are initialized\n   * Throws an exception if the indices array shape is inconsistent\n   * Returns false if the indices array is empty(nnz = 0) for csr/row_sparse\n   */\n  inline bool storage_initialized() const {\n    if (is_none())\n      return false;\n    auto stype = storage_type();\n    CHECK_NE(stype, kDefaultStorage)\n        << \"storage_initialized() is not intended for kDefaultStorage.\";\n    if (stype == kRowSparseStorage) {\n      CHECK_EQ(aux_shape(rowsparse::kIdx)[0], storage_shape()[0])\n          << \"inconsistent storage shape \" << storage_shape() << \" vs. aux shape \"\n          << aux_shape(rowsparse::kIdx);\n      return aux_shape(rowsparse::kIdx).Size() != 0;\n    } else if (stype == kCSRStorage) {\n      CHECK_EQ(aux_shape(csr::kIdx)[0], storage_shape()[0])\n          << \"inconsistent storage shape \" << storage_shape() << \" vs. aux shape \"\n          << aux_shape(csr::kIdx);\n      return aux_shape(csr::kIdx).Size() != 0;\n    } else {\n      LOG(FATAL) << \"Unknown storage type\";\n    }\n    return true;\n  }\n  /*! \\brief get storage handle */\n  inline Storage::Handle storage_handle() const {\n    CHECK(!is_none());\n    CHECK_EQ(storage_type(), kDefaultStorage);\n    CheckAndAlloc();\n    return ptr_->shandle;\n  }\n  /*! \\brief assign profiler scope and name to the storage handles */\n  void AssignStorageInfo(const std::string& profiler_scope, const std::string& name);\n  /*!\n   * \\brief Block until all the pending write operations with respect\n   *    to current NDArray are finished, and read can be performed.\n   *\n   * If the array has not been computed yet (deferred compute), this will\n   * trigger computation.\n   */\n  void WaitToRead() const;\n  /*!\n   * \\brief Block until all the pending read/write operations with respect\n   *    to current NDArray are finished, and write can be performed.\n   *\n   * If the array has not been computed yet (deferred compute), this will\n   * trigger computation.\n   */\n  void WaitToWrite() const;\n  /*!\n   * \\brief Synchronize the destination stream provided by consumer with the\n   *    source stream that current NDArray lives on.\n   * \\param stream a pointer to the stream provided by consumer.\n   */\n  void StreamSync(int stream) const;\n  /*! \\return the associated variable of the ndarray.*/\n  inline Engine::VarHandle var() const {\n    return ptr_->var;\n  }\n  /*! \\return byte offset in chunk of the ndarray*/\n  inline size_t byte_offset() const {\n    return byte_offset_;\n  }\n  /*! \\brief return var version of the NDArray*/\n  inline size_t version() const {\n    return var()->version();\n  }\n  /*!\n   * \\brief save the content into binary stream\n   * \\param strm the output stream\n   */\n  void Save(dmlc::Stream* strm) const;\n  /*!\n   * \\brief load ndarrays before supporting sparse ndarrays\n   * \\param strm the output stream\n   * \\param magic the magic number used for version control\n   */\n  bool LegacyLoad(dmlc::Stream* strm, const uint32_t magic);\n  /*!\n   * \\brief load the content from binary stream\n   * \\param strm the output stream\n   * \\return whether the load is successful\n   */\n  bool Load(dmlc::Stream* strm);\n  /*!\n   * \\brief set all the elements in ndarray to be scalar\n   * \\param scalar the scalar to set\n   * \\return reference of self\n   */\n  NDArray& operator=(real_t scalar);\n  /*!\n   * \\brief elementwise add to current space\n   *  this mutate the current NDArray\n   * \\param src the data to add\n   * \\return reference of self\n   */\n  NDArray& operator+=(const NDArray& src);\n  /*!\n   * \\brief elementwise add to current space\n   *  this mutate the current NDArray\n   * \\param src the data to add\n   * \\return reference of self\n   */\n  NDArray& operator+=(const real_t& src);\n  /*!\n   * \\brief elementwise subtract from current ndarray\n   * this mutate the current NDArray\n   * \\param src the data to subtract\n   * \\return reference of self\n   */\n  NDArray& operator-=(const NDArray& src);\n  /*!\n   * \\brief elementwise subtract from current ndarray\n   * this mutate the current NDArray\n   * \\param src the data to subtract\n   * \\return reference of self\n   */\n  NDArray& operator-=(const real_t& src);\n  /*!\n   * \\brief elementwise multiplication to current ndarray\n   *  this mutate the current NDArray\n   * \\param src the data to subtract\n   * \\return reference of self\n   */\n  NDArray& operator*=(const NDArray& src);\n  /*!\n   * \\brief elementwise multiplication to current ndarray\n   *  this mutate the current NDArray\n   * \\param src the data to subtract\n   * \\return reference of self\n   */\n  NDArray& operator*=(const real_t& src);\n  /*!\n   * \\brief elementwise division from current ndarray\n   *  this mutate the current NDArray\n   * \\param src the data to subtract\n   * \\return reference of self\n   */\n  NDArray& operator/=(const NDArray& src);\n  /*!\n   * \\brief elementwise division from current ndarray\n   *  this mutate the current NDArray\n   * \\param src the data to subtract\n   * \\return reference of self\n   */\n  NDArray& operator/=(const real_t& src);\n  /*!\n   * \\brief return a new copy this NDArray\n   * \\param ctx the new context of this NDArray\n   * \\return the new copy\n   */\n  NDArray Copy(Context ctx) const;\n  /*!\n   * \\brief Do a synchronize copy from a contiguous CPU memory region.\n   *\n   *  This function will call WaitToWrite before the copy is performed.\n   *  This is useful to copy data from existing memory region that are\n   *  not wrapped by NDArray(thus dependency not being tracked).\n   *\n   * \\param data the data source to copy from.\n   * \\param size the size of the source array, in sizeof(DType) not raw btyes.\n   */\n  void SyncCopyFromCPU(const void* data, size_t size) const;\n\n  /*!\n   * \\brief Copy from src.data()/aux_data(i) to this->data()/aux_data(j)\n   */\n  void SyncCopyFromNDArray(const NDArray& src, int i = -1, int j = -1);\n\n  /*!\n   * \\brief Do a synchronize copy to a contiguous CPU memory region.\n   *\n   *  This function will call WaitToRead before the copy is performed.\n   *  This is useful to copy data from existing memory region that are\n   *  not wrapped by NDArray(thus dependency not being tracked).\n   *\n   * \\param data the data source to copyinto.\n   * \\param size the memory size we want to copy into, in sizeof(DType) not raw btyes.\n   */\n  void SyncCopyToCPU(void* data, size_t size) const;\n  /*!\n   * \\brief check whether the NDArray format is valid\n   * \\param full_check if `True`, rigorous check, O(N) operations\n   *    Otherwise basic check, O(1) operations\n   */\n  void SyncCheckFormat(const bool full_check) const;\n  /*!\n   * \\brief Slice a NDArray\n   * \\param begin begin index in first dim (inclusive)\n   * \\param end end index in first dim (exclusive)\n   * \\return sliced NDArray\n   */\n  NDArray Slice(index_t begin, index_t end) const;\n  /*!\n   * \\brief Slice a NDArray. Supports recording with autograd\n   * \\param begin begin index in first dim (inclusive)\n   * \\param end end index in first dim (exclusive)\n   * \\return sliced NDArray\n   */\n  NDArray SliceWithRecord(index_t begin, index_t end);\n  /*!\n   * \\brief Index a NDArray\n   * \\param idx the index\n   * \\return idx-th sub array NDArray\n   */\n  NDArray At(index_t idx) const;\n  /*!\n   * \\brief Index a NDArray\n   * \\param idx the index\n   * \\return idx-th sub array NDArray\n   */\n  NDArray AtWithRecord(index_t idx);\n  /*!\n   * \\brief Generate a deep copy of aux_data(i) returned as\n   * a default storage type NDArray\n   */\n  NDArray aux_ndarray(size_t i) const;\n\n  /*!\n   * \\brief Generate a deep copy of data() returned as a\n   * default storage type NDArray\n   */\n  NDArray data_ndarray() const;\n\n  /*!\n   * \\brief Create a NDArray that shares memory with current one\n   *  The new array must have smaller memory size than the current array.\n   * \\param shape new shape\n   * \\param dtype The data type.\n   * \\return NDArray in new shape and type.\n   */\n  inline NDArray AsArray(const mxnet::TShape& shape, int dtype) const {\n    CHECK_EQ(storage_type(), kDefaultStorage) << \"AsArray is intended only for kDefaultStorage.\";\n    CHECK_GE(ptr_->shandle.size, shape.Size() * mshadow::mshadow_sizeof(dtype))\n        << \"NDArray.AsArray: target memory size is bigger\";\n    // We can't reuse memory in a view.\n    CHECK(!IsView());\n    NDArray ret = *this;\n    ret.shape_  = shape;\n    ret.dtype_  = dtype;\n    ret.reuse_  = true;\n    return ret;\n  }\n\n  inline void InitAsArray(const NDArray& src, const mxnet::TShape& shape, int dtype) {\n    CHECK_EQ(src.storage_type(), kDefaultStorage)\n        << \"AsArray is intended only for kDefaultStorage.\";\n    CHECK_GE(src.ptr_->shandle.size, shape.Size() * mshadow::mshadow_sizeof(dtype))\n        << \"NDArray.AsArray: target memory size is bigger than what was allocated.\";\n    // We can't reuse memory in a view.\n    CHECK(!src.IsView());\n    *this  = src;\n    shape_ = shape;\n    dtype_ = dtype;\n    reuse_ = true;\n  }\n\n  /*!\n   * \\brief Create a reference view of NDArray that\n   *  represents as DLManagedTensor.\n   * \\return A DLManagedTensor\n   */\n  DLManagedTensor* ToDLPack() const;\n\n  /*!\n   * \\brief Create a NDArray backed by a dlpack tensor.\n   *\n   * This allows us to create a NDArray using the memory\n   * allocated by an external deep learning framework\n   * that is DLPack compatible.\n   *\n   * The memory is retained until the NDArray went out of scope.\n   *\n   * \\return The created NDArray view.\n   */\n  static NDArray FromDLPack(const DLManagedTensor* tensor, bool transient_handle);\n\n  /*!\n   * \\brief Update ndarray chunk storage handles using existing ndarray storage handles\n   * Also update the aux_handle, aux_shapes and aux_types.\n   * This is specifically used for custom op to update the inputs and outputs from\n   * the temporary ndarray which stores intermediate custom op results.\n   * Should be used with caution elsewhere. Supports only CSR and RSP formats.\n   */\n  inline void SparseUpdateChunk(const NDArray& arr) const {\n    CHECK(shape_ == arr.shape_) << \"ndarray shape is different from the target\";\n    CHECK(dtype_ == arr.dtype_) << \"ndarray dtype is different from the target\";\n    auto stype = arr.storage_type();\n    CHECK(stype == kCSRStorage || stype == kRowSparseStorage)\n        << \"Only to be used with CSR and RSP storage types\";\n    // swap shandles between src and dst\n    Storage::Handle shandle_dst = arr.ptr_->shandle;\n    arr.ptr_->shandle           = ptr_->shandle;\n    ptr_->shandle               = shandle_dst;\n\n    ptr_->storage_shape = arr.ptr_->storage_shape;\n    ptr_->storage_type  = arr.ptr_->storage_type;\n    ptr_->ctx           = arr.ptr_->ctx;\n\n    // swap aux_handles between src and dst\n    size_t aux_idx = 0;\n    CHECK(ptr_->aux_handles.size() == arr.ptr_->aux_handles.size())\n        << \"ndarray number of aux_handles is different from target\";\n    for (auto& aux_handle : arr.ptr_->aux_handles) {\n      Storage::Handle aux_dst    = ptr_->aux_handles[aux_idx];\n      ptr_->aux_handles[aux_idx] = aux_handle;\n      aux_handle                 = aux_dst;\n      aux_idx++;\n    }\n    ptr_->aux_types  = arr.ptr_->aux_types;\n    ptr_->aux_shapes = arr.ptr_->aux_shapes;\n  }\n\n  /*!\n   * \\brief Get an reshaped NDArray\n   * \\param shape new shape\n   * \\return NDArray in new shape\n   */\n  NDArray Reshape(const mxnet::TShape& shape) const;\n  /*!\n   * \\brief Get an reshaped NDArray. Supports autograd recording\n   * \\param shape new shape\n   * \\return NDArray in new shape\n   */\n  NDArray ReshapeWithRecord(const mxnet::TShape& shape);\n  /*!\n   * \\brief Return a copy of this NDArray without autograd and deferred compute\n   * history\n   */\n  NDArray Detach() const {\n    NDArray ret(*this);\n    ret.autograd_entry_        = nnvm::NodeEntry(nullptr);\n    ret.deferredcompute_entry_ = nnvm::NodeEntry(nullptr);\n    return ret;\n  }\n\n  nnvm::Symbol get_autograd_symbol() const;\n  /*!\n   * \\brief Allocate the space if it is delayed allocated.\n   * This is an internal function used by system that normal user should not use\n   */\n  inline void CheckAndAlloc() const {\n    CHECK_EQ(storage_type(), kDefaultStorage);\n    ptr_->CheckAndAlloc();\n  }\n\n  /*!\n   * \\brief Allocate the space if the allocation has been delayed\n   * or the requested size is bigger than the available one.\n   * This function can only be called by ndarray of default\n   * storage type and effectively changes the ndarray's shape_.\n   * Note: This function is named as this to avoid overload conflict\n   * with CheckAndAlloc(const mxnet::ShapeVector &aux_shapes), since\n   * mxnet::TShape tmp = some_shape is equivalent to mxnet::TShape tmp = {some_shape}.\n   */\n  void ReshapeAndAlloc(const mxnet::TShape& shape) {\n    CHECK_EQ(storage_type(), kDefaultStorage);\n    CHECK(!is_none());\n    shape_ = shape;\n    ptr_->CheckAndAlloc(shape.Size() * mshadow::mshadow_sizeof(dtype_));\n  }\n\n  /* !\n   * \\brief Alloc memory for non-default storage\n   * aux_shape is only known at run time\n   */\n  inline void CheckAndAlloc(const mxnet::ShapeVector& aux_shapes) const {\n    CHECK_NE(storage_type(), kDefaultStorage)\n        << \"CheckAndAlloc(aux_shapes) is not intended for kDefaultStorage\";\n    ptr_->CheckAndAlloc(shape_, aux_shapes, dtype_);\n  }\n  inline void CheckAndAllocData(const mxnet::TShape& storage_shape) const {\n    CHECK_NE(storage_type(), kDefaultStorage)\n        << \"CheckAndAllocData is not intended for kDefaultStorage\";\n    ptr_->CheckAndAllocData(storage_shape, dtype_);\n  }\n  inline void CheckAndAllocAuxData(size_t i, const mxnet::TShape& aux_shape) const {\n    CHECK_NE(storage_type(), kDefaultStorage)\n        << \"CheckAndAllocAuxData is not intended for kDefaultStorage\";\n    ptr_->CheckAndAllocAuxData(i, aux_shape);\n  }\n\n#if MXNET_USE_ONEDNN == 1\n  /*\n   * Create NDArray from dnnl memory.\n   * dnnl_mem The dnnl memory to be managed.\n   */\n  explicit NDArray(const std::shared_ptr<dnnl::memory>& dnnl_mem);\n  /*\n   * Create NDArray from dnnl memory descriptor.\n   * mem_pd The dnnl memory descriptor to be created.\n   */\n  explicit NDArray(const void* md);\n  /*\n   * Test if the data is stored in one of special DNNL formats.\n   */\n  bool IsDNNLData() const {\n    return ptr_->IsDNNL();\n  }\n  /*\n   * Test if the data is stored in one of default MXNet formats.\n   */\n  bool IsDefaultData() const {\n    return ptr_->IsDefault();\n  }\n  /*\n   * All functions below return a raw pointer to dnnl memory. Actually there\n   * is a shared pointer that hold the memory either in NDArray or in DNNL\n   * stream. As long as we call these functions inside an operator, the return\n   * memory is always valid.\n   */\n\n  /*\n   * This function returns dnnl::memory with the default primitive_desc.\n   */\n  const dnnl::memory* GetDNNLData() const;\n  /*\n   * This function returns dnnl::memory with the given primitive_desc\n   * as long as the array size meets the required size in the given primitive_desc.\n   */\n  const dnnl::memory* GetDNNLData(const void* md) const;\n  /*\n   * This function returns dnnl::memory with the given primitive_desc.\n   * The returned dnnl::memory will have the same physical layout as\n   * the given primitive_desc.\n   */\n  const dnnl::memory* GetDNNLDataReorder(const void* md) const;\n\n  /*\n   * This function copies data from dnnl memory.\n   */\n  void CopyFrom(const dnnl::memory& mem);\n  /*\n   * This function allocates memory for array and creates dnnl memory\n   * with the specified format.\n   */\n  dnnl::memory* CreateDNNLData(const void* md);\n\n  /*\n   * These are the async version of the methods above.\n   * It changes the layout of this NDArray, but it happens after all accesses to\n   * the array are complete.\n   */\n  void Reorder2DefaultAsync() const;\n  void DNNLDataReorderAsync(const void* md) const;\n\n  /*\n   * This creates a new NDArray with the reordered data.\n   * It doesn't affect the data of the original NDArray.\n   */\n  NDArray Reorder2Default() const;\n\n  /*\n   * This creates a new NDArray using f32 with the reordered data.\n   * It doesn't affect the data of the original NDArray.\n   */\n  NDArray Reorder2DefaultFloatFormat() const;\n\n  void InvalidateDNNLData();\n\n  /*\n   * This function is used inside operators to reshape an array.\n   * It doesn't change the layout of the original array and allocate memory from\n   * the temporary buffer. The returned array is only valid inside the current\n   * invocation of this operator.\n   * This is different from Reshape. Reshape will cause data in the array to be\n   * converted to the default layout and allocate memory from malloc directly,\n   * which can be expensive.\n   * It's used by FullyConnected right now.\n   */\n  NDArray DNNLDataReshape(const mxnet::TShape& shape) const;\n\n  /*!\n   * \\ Fix dnnl memory descriptor mismatch from NDArray.\n   */\n  void UpdateDNNLMemDesc(const void* desc);\n#endif\n\n  /*!\n   * \\brief Save list of ndarray into the Stream.x\n   * \\param fo The stream of output.\n   * \\param data the NDArrays to be saved.\n   * \\param names the name of the NDArray, optional, can be zero length.\n   */\n  static void Save(dmlc::Stream* fo,\n                   const std::vector<NDArray>& data,\n                   const std::vector<std::string>& names);\n  /*!\n   * \\brief Load list of ndarray into from the stream.\n   * \\param fi The stream of the input file.\n   * \\param data the NDArrays to be loaded\n   * \\param keys the name of the NDArray, if saved in the file.\n   */\n  static void Load(dmlc::Stream* fi, std::vector<NDArray>* data, std::vector<std::string>* keys);\n\n private:\n  friend class Imperative;\n  /*! \\brief the real data chunk that backs NDArray */\n  // shandle is used to store the actual values in the NDArray\n  // aux_handles store the aux data(such as indices) if it's needed by non-default storage.\n  struct Chunk {\n    /*! \\brief storage handle from storage engine.\n               for non-default storage, shandle stores the data(value) array.\n     */\n    Storage::Handle shandle;\n    /*! \\brief storage handles for aux data (e.g index)\n               for row_sparse, aux_handles[0] = indices\n               for csr, aux_handles[0] = indptr, aux_handles[1] = indices\n    */\n    std::vector<Storage::Handle> aux_handles;\n\n#if MXNET_USE_ONEDNN == 1\n    /*! This is created when data is stored in DNNL format.\n     */\n    std::shared_ptr<DNNLMemory> dnnl_mem_;\n#endif\n    /*! \\brief variable from engine */\n    Engine::VarHandle var;\n    /*!\n     * \\brief if this is true, this means the data do not come\n     * from Storage, and do not need to be freed\n     */\n    /*! \\brief construct from static data */\n    bool static_data;\n    /*! \\brief whether data allocation is delayed. This doesn't indicate whether aux data\n               allocation is delayed. */\n    bool delay_alloc;\n    // the type of the storage. The storage_type is never kUndefinedStorage once the chunk\n    // is constructed.\n    NDArrayStorageType storage_type = kDefaultStorage;\n    /*! \\brief type of aux */\n    std::vector<int> aux_types;\n    // context of data\n    Context ctx;\n    // The shape of the chunk data.\n    // This might not be the same shape as the NDArray, since the storage may be sparse.\n    // The default value for storage_shape is {0} when an empty non-default NDArray is created.\n    mxnet::TShape storage_shape;\n    // The shape of aux data. The default value for the shape depends on the type of storage.\n    // If aux_shapes[i].Size() is zero, aux data i is empty.\n    mxnet::ShapeVector aux_shapes;\n    /*! \\brief Reference to the storage to ensure proper destruct order */\n    std::shared_ptr<Storage> storage_ref_;\n    /*! \\brief Reference to the engine to ensure we cleanup without calling a destructed engine */\n    std::weak_ptr<Engine> engine_ref_;\n\n    /*! \\brief default constructor */\n    Chunk()\n        : static_data(true),\n          delay_alloc(false),\n          storage_ref_(Storage::_GetSharedRef()),\n          engine_ref_(Engine::_GetSharedRef()) {}\n\n    /*! \\brief construct a new chunk */\n    Chunk(mxnet::TShape shape, Context ctx_, bool delay_alloc_, int dtype)\n        : static_data(false),\n          delay_alloc(true),\n          ctx(ctx_),\n          storage_ref_(Storage::_GetSharedRef()),\n          engine_ref_(Engine::_GetSharedRef()) {\n      storage_shape = shape;\n      if (shape_is_known(storage_shape)) {\n        shandle.size = shape.Size() * mshadow::mshadow_sizeof(dtype);\n      }\n      var         = Engine::Get()->NewVariable();\n      shandle.ctx = ctx_;\n      if (!delay_alloc_) {\n        this->CheckAndAlloc();\n      }\n    }\n\n    Chunk(const TBlob& data, int dev_id)\n        : static_data(true),\n          delay_alloc(false),\n          storage_ref_(Storage::_GetSharedRef()),\n          engine_ref_(Engine::_GetSharedRef()) {\n      CHECK(storage_type == kDefaultStorage);\n      var = Engine::Get()->NewVariable();\n      if (data.dev_mask() == cpu::kDevMask) {\n        ctx = Context::CPU();\n      } else {\n        CHECK_EQ(data.dev_mask(), gpu::kDevMask);\n        ctx = Context::GPU(dev_id);\n      }\n      // init shandle\n      shandle.ctx   = ctx;\n      shandle.dptr  = data.dptr_;\n      shandle.size  = data.shape_.Size() * mshadow::mshadow_sizeof(data.type_flag_);\n      storage_shape = data.shape_;\n    }\n\n    Chunk(int shared_pid, int shared_id, const mxnet::TShape& shape, int dtype)\n        : static_data(false),\n          delay_alloc(false),\n          storage_ref_(Storage::_GetSharedRef()),\n          engine_ref_(Engine::_GetSharedRef()) {\n      var                = Engine::Get()->NewVariable();\n      ctx                = Context::CPUShared(0);\n      shandle.size       = shape.Size() * mshadow::mshadow_sizeof(dtype);\n      shandle.ctx        = ctx;\n      shandle.shared_pid = shared_pid;\n      shandle.shared_id  = shared_id;\n      Storage::Get()->Alloc(&shandle);\n      storage_shape = shape;\n    }\n    // Constructor for a non-default storage chunk\n    Chunk(NDArrayStorageType storage_type_,\n          const mxnet::TShape& storage_shape_,\n          Context ctx_,\n          bool delay_alloc_,\n          int dtype,\n          const std::vector<int>& aux_types_,\n          const mxnet::ShapeVector& aux_shapes_)\n        : static_data(false),\n          delay_alloc(delay_alloc_),\n          storage_type(storage_type_),\n          aux_types(aux_types_),\n          ctx(ctx_),\n          storage_shape(storage_shape_),\n          aux_shapes(aux_shapes_),\n          storage_ref_(Storage::_GetSharedRef()),\n          engine_ref_(Engine::_GetSharedRef()) {\n      shandle.ctx = ctx;\n      var         = Engine::Get()->NewVariable();\n      // aux_handles always reflect the correct number of aux data\n      for (size_t i = 0; i < aux_shapes.size(); i++) {\n        CheckAndAllocAuxData(i, aux_shapes[i]);\n        // this line is needed in case when aux_shapes[i].Size() = 0\n        // aux_handles[i] will not be updated and take only default value.\n        aux_handles[i].ctx = ctx;\n      }\n      if (!delay_alloc) {\n        CheckAndAllocData(storage_shape, dtype);\n      }\n    }\n\n    Chunk(const NDArrayStorageType storage_type_,\n          const TBlob& data,\n          const std::vector<TBlob>& aux_data,\n          int dev_id)\n        : static_data(true),\n          delay_alloc(false),\n          storage_type(storage_type_),\n          storage_ref_(Storage::_GetSharedRef()),\n          engine_ref_(Engine::_GetSharedRef()) {\n      using namespace mshadow;\n      CHECK_NE(storage_type, kDefaultStorage);\n      // init var\n      var = Engine::Get()->NewVariable();\n      // init ctx\n      if (data.dev_mask() == cpu::kDevMask) {\n        ctx = Context::CPU();\n      } else {\n        CHECK_EQ(data.dev_mask(), gpu::kDevMask);\n        ctx = Context::GPU(dev_id);\n      }\n      // init shandle\n      shandle.ctx   = ctx;\n      shandle.dptr  = data.dptr_;\n      shandle.size  = data.shape_.Size() * mshadow_sizeof(data.type_flag_);\n      storage_shape = data.shape_;\n      // init aux handles\n      for (const auto& aux : aux_data) {\n        Storage::Handle aux_handle;\n        aux_handle.ctx  = ctx;\n        aux_handle.dptr = aux.dptr_;\n        aux_handle.size = aux.shape_.Size() * mshadow_sizeof(aux.type_flag_);\n        aux_handles.push_back(aux_handle);\n        aux_types.emplace_back(aux.type_flag_);\n        aux_shapes.emplace_back(aux.shape_);\n      }\n    }\n\n    /*! \\brief set the shape for ith aux data, and update storage shape if necessary */\n    inline void set_aux_shape(const size_t i, const mxnet::TShape& shape) {\n      aux_shapes[i] = shape;\n      if (storage_shape.ndim() >= 0) {\n        if (storage_type == kRowSparseStorage && i == rowsparse::kIdx) {\n          storage_shape[0] = shape[0];\n        } else if (storage_type == kCSRStorage && i == csr::kIdx) {\n          storage_shape[0] = shape[0];\n        }\n      }\n    }\n\n    /*! \\brief check if delay alloc is on, do alloc if not yet done */\n    inline void CheckAndAlloc(void) {\n      if (delay_alloc) {\n        Storage::Get()->Alloc(&shandle);\n#if MXNET_USE_ONEDNN == 1\n        dnnl_mem_ = nullptr;\n#endif\n        delay_alloc = false;\n      }\n    }\n\n    /*! \\brief Check and alloc memory for a dense ndarray */\n    // size is the number of bytes\n    void CheckAndAlloc(uint64_t dbytes) {\n      CHECK_EQ(kDefaultStorage, storage_type)\n          << \"CheckAndAlloc(dbytes) is only intended for kDefaultStorage\";\n      dbytes = std::max(dbytes, static_cast<uint64_t>(shandle.size));\n      if (delay_alloc) {\n        shandle.size = dbytes;\n        Storage::Get()->Alloc(&shandle);\n#if MXNET_USE_ONEDNN == 1\n        dnnl_mem_ = nullptr;\n#endif\n        delay_alloc = false;\n      } else if (shandle.size < dbytes) {\n        // free storage\n        Storage::Get()->Free(shandle);\n        // init storage\n        shandle.size = dbytes;\n        Storage::Get()->Alloc(&shandle);\n#if MXNET_USE_ONEDNN == 1\n        dnnl_mem_ = nullptr;\n#endif\n      }\n    }\n    /*! \\brief initialize the shape and dtype, assuming it is not initialized before. */\n    void Init(const mxnet::TShape& shape, int dtype) {\n      auto size     = shape.Size();\n      storage_shape = shape;\n      shandle.size  = size * mshadow::mshadow_sizeof(dtype);\n      this->CheckAndAlloc();\n    }\n    inline void CheckAndAlloc(const mxnet::TShape& shape,\n                              const mxnet::ShapeVector& aux_shapes,\n                              int dtype) {\n      // calculate size, perform allocation\n      if (kRowSparseStorage == storage_type) {\n        // For row sparse, aux_shape indicates the number of rows to allocate\n        auto aux_shape = aux_shapes[rowsparse::kIdx];\n        CheckAndAllocAuxData(rowsparse::kIdx, aux_shape);\n        mxnet::TShape storage_shape(shape);\n        storage_shape[0] = aux_shape[0];\n        CheckAndAllocData(storage_shape, dtype);\n      } else if (kCSRStorage == storage_type) {\n        CheckAndAllocAuxData(csr::kIndPtr, aux_shapes[csr::kIndPtr]);\n        CheckAndAllocAuxData(csr::kIdx, aux_shapes[csr::kIdx]);\n        CheckAndAllocData(aux_shapes[csr::kIdx], dtype);\n      } else {\n        LOG(FATAL) << \"Storage type \" << storage_type << \" not implemented for CheckAndAlloc\";\n      }\n    }\n    // create storage handle for data based on shape and dtype, assuming ctx is set\n    // storage shape is also updated\n    // if data is already allocated, try reuse the storage. Otherwise, free the current one\n    // and allocate new storage\n    void CheckAndAllocData(const mxnet::TShape& shape, int dtype);\n\n#if MXNET_USE_ONEDNN == 1\n    // Have DNNL memory reference to the data in the default storage\n    // or create memory for DNNL.\n    void SetDNNLMem(const mxnet::TShape& shape, int dtype);\n    // If the data is stored in DNNL layout, we reorder data in dnnl_mem_ and\n    // save the result in shandle.\n    void Reorder2Default();\n    // Reroder data to a specified layout.\n    void DNNLDataReorder(const void* md);\n    bool IsDNNL() const;\n    bool IsDefault() const;\n#endif\n\n    // create storage handle for aux data based on shape\n    // this function assumes ctx, aux shapes and aux types are set\n    // aux shape is also updated\n    // if aux data is already allocated, try reuse the storage. Otherwise, free the current one\n    // and allocate new storage\n    inline void CheckAndAllocAuxData(size_t i, const mxnet::TShape& shape) {\n      CHECK_EQ(shape.ndim(), 1) << \"shape must be 1D in CheckAndAllocAuxData\";\n      CHECK_NE(storage_type, kUndefinedStorage)\n          << \"storage type cannot be kUndefinedStorage in CheckAndAllocAuxData\";\n      CHECK_NE(storage_type, kDefaultStorage)\n          << \"storage type cannot be kDefaultStorage in CheckAndAllocAuxData\";\n      if (aux_handles.size() <= i) {\n        aux_handles.resize(i + 1);\n      }\n      size_t aux_bytes = shape.Size() * mshadow::mshadow_sizeof(aux_types[i]);\n      if (aux_handles[i].size < aux_bytes) {\n        // free storage\n        Storage::Get()->Free(aux_handles[i]);\n        // init aux storage\n        aux_handles[i] = Storage::Get()->Alloc(aux_bytes, ctx);\n      }\n      // init shape\n      set_aux_shape(i, shape);\n    }\n    /*! \\brief destructor */\n    ~Chunk();\n  };  // struct Chunk\n\n  /*!\n   * \\brief initialize the NDArray\n   */\n  inline void Init(const NDArrayStorageType stype, const mxnet::TShape& shape, int dtype) {\n    shape_          = shape;\n    dtype_          = dtype;\n    storage_type_   = stype;\n    reuse_          = false;\n    byte_offset_    = 0;\n    autograd_entry_ = nnvm::NodeEntry(nullptr);\n  }\n\n  void SetTBlob() const;\n\n  /*! \\brief internal data of NDArray */\n  std::shared_ptr<Chunk> ptr_{nullptr};\n  /*! \\brief shape of current NDArray\n   *  \\note const methods WaitToRead, WaitToWrite will set shape, if shape is\n   *        previously unknown and array is deferred computed.\n   */\n  mutable mxnet::TShape shape_;\n  /*! \\brief byte offset in chunk */\n  size_t byte_offset_ = 0;\n  /*! \\brief type of data */\n  int dtype_ = -1;\n  /*! \\brief whether the NDArray uses memory of another NDArray. */\n  bool reuse_ = false;\n  /*! \\brief storage type of data */\n  NDArrayStorageType storage_type_ = kUndefinedStorage;\n  /*! \\brief node entry for autograd */\n  nnvm::NodeEntry autograd_entry_;\n  /*! \\brief node entry for deferred computation tracking */\n  nnvm::NodeEntry deferredcompute_entry_;\n  /*!\n   * \\brief internal TBlob\n   * \\note When user access tblob_ by some const methods like\n   *     NDArray::data(), the dptr in tblob_ still need to be updated\n   *     in case that allocation happens. So we make it mutable for\n   *     this situation.\n   */\n  mutable TBlob tblob_;\n};  // class NDArray\n\n/*!\n * \\return the number of aux data used for given storage type\n */\nsize_t num_aux_data(NDArrayStorageType stype);\n\n/*!\n * \\brief issue an copy operation from one NDArray to another\n *  the two ndarray can sit on different devices\n *  this operation will be scheduled by the engine\n *\n * \\param from the ndarray we want to copy data from\n * \\param to the target ndarray\n * \\param priority Priority of the action.\n * \\note The function name explicitly marks the order of from and to\n *     due to different possible convention carried by copy function.\n */\nvoid CopyFromTo(const NDArray& from, const NDArray* to, int priority = 0);\n\n/*!\n * \\brief issue an copy operation from one NDArray to another\n *  the two ndarray can sit on different devices\n *  this operation will be scheduled by the engine\n *\n * \\param from the ndarray we want to copy data from\n * \\param to the target ndarray\n * \\param priority Priority of the action.\n * \\param is_opr whether it is invoked by an operator. For example, false if invoked from\n       KVStore, true if invoked from `_copyto` operator.\n * \\note The function name explicitly marks the order of from and to\n *     due to different possible convention carried by copy function.\n */\nvoid CopyFromTo(const NDArray& from, const NDArray& to, int priority = 0, bool is_opr = false);\n\n/*!\n * \\brief Perform elementwise sum over each data from source, store result into out.\n * \\param source the ndarray we want to sum\n * \\param out the target ndarray\n * \\param priority Priority of the action.\n */\nvoid ElementwiseSum(const std::vector<NDArray>& source, NDArray* out, int priority = 0);\n\n/*!\n * \\brief elementwise add\n * \\param lhs left operand\n * \\param rhs right operand\n * \\return a new result ndarray\n */\nNDArray operator+(const NDArray& lhs, const NDArray& rhs);\n/*!\n * \\brief elementwise add\n * \\param lhs left operand\n * \\param rhs right operand\n * \\return a new result ndarray\n */\nNDArray operator+(const NDArray& lhs, const real_t& rhs);\n/*!\n * \\brief elementwise subtraction\n * \\param lhs left operand\n * \\param rhs right operand\n * \\return a new result ndarray\n */\nNDArray operator-(const NDArray& lhs, const NDArray& rhs);\n/*!\n * \\brief elementwise subtraction\n * \\param lhs left operand\n * \\param rhs right operand\n * \\return a new result ndarray\n */\nNDArray operator-(const NDArray& lhs, const real_t& rhs);\n/*!\n * \\brief elementwise multiplication\n * \\param lhs left operand\n * \\param rhs right operand\n * \\return a new result ndarray\n */\nNDArray operator*(const NDArray& lhs, const NDArray& rhs);\n/*!\n * \\brief elementwise multiplication\n * \\param lhs left operand\n * \\param rhs right operand\n * \\return a new result ndarray\n */\nNDArray operator*(const NDArray& lhs, const real_t& rhs);\n/*!\n * \\brief elementwise division\n * \\param lhs left operand\n * \\param rhs right operand\n * \\return a new result ndarray\n */\nNDArray operator/(const NDArray& lhs, const NDArray& rhs);\n/*!\n * \\brief elementwise division\n * \\param lhs left operand\n * \\param rhs right operand\n * \\return a new result ndarray\n */\nNDArray operator/(const NDArray& lhs, const real_t& rhs);\n\n/*!\n * \\brief Seed all random number generator in mxnet.\n * \\param seed the seed to set to global random number generators.\n */\nvoid RandomSeed(uint32_t seed);\n/*!\n * \\brief Seed the random number generator of the device.\n * \\param seed the seed to set to global random number generators.\n */\nvoid RandomSeed(Context ctx, uint32_t seed);\n/*!\n * \\brief Sample uniform distribution for each elements of out.\n * \\param begin lower bound of distribution.\n * \\param end upper bound of distribution.\n * \\param out output NDArray.\n */\nvoid SampleUniform(real_t begin, real_t end, NDArray* out);\n/*!\n * \\brief Sample gaussian distribution for each elements of out.\n * \\param mu mean of gaussian distribution.\n * \\param sigma standard deviation of gaussian distribution.\n * \\param out output NDArray.\n */\nvoid SampleGaussian(real_t mu, real_t sigma, NDArray* out);\n/*!\n * \\brief Sample gamma distribution for each elements of out.\n * \\param alpha parameter (shape) of the gamma distribution\n * \\param beta parameter (scale) of the gamma distribution\n * \\param out output NDArray.\n */\nvoid SampleGamma(real_t alpha, real_t beta, NDArray* out);\n/*!\n * \\brief Sample exponential distribution for each elements of out.\n * \\param lambda parameter (rate) of the exponential distribution\n * \\param out output NDArray.\n */\nvoid SampleExponential(real_t lambda, NDArray* out);\n/*!\n * \\brief Sample Poisson distribution for each elements of out.\n * \\param lambda parameter (rate) of the Poisson distribution\n * \\param out output NDArray.\n */\nvoid SamplePoisson(real_t lambda, NDArray* out);\n/*!\n * \\brief Sample negative binomial distribution for each elements of out.\n * \\param k failure limit\n * \\param p success probability\n * \\param out output NDArray.\n */\nvoid SampleNegBinomial(int32_t k, real_t p, NDArray* out);\n/*!\n * \\brief Sample generalized negative binomial distribution for each elements of out.\n * \\param mu parameter (mean) of the distribution\n * \\param alpha parameter (over dispersion) of the distribution\n * \\param out output NDArray.\n */\nvoid SampleGenNegBinomial(real_t mu, real_t alpha, NDArray* out);\n\n//--------------------------------------------------------------\n// The following part are API Registration of NDArray functions.\n//--------------------------------------------------------------\n\n/*! \\brief definition of NDArray function */\ntypedef std::function<void(NDArray** used_vars,\n                           real_t* scalars,\n                           NDArray** mutate_vars,\n                           int num_params,\n                           char** param_keys,\n                           char** param_vals)>\n    NDArrayAPIFunction;\n/*! \\brief mask information on how functions can be exposed */\nenum NDArrayFunctionTypeMask {\n  /*! \\brief all the use_vars should go before scalar */\n  kNDArrayArgBeforeScalar = 1,\n  /*! \\brief all the scalar should go before use_vars */\n  kScalarArgBeforeNDArray = 1 << 1,\n  /*!\n   * \\brief whether this function allows the handles in the target to\n   *  be empty NDArray that are not yet initialized, and will initialize\n   *  them when the function is invoked.\n   *\n   *  most function should support this, except copy between different\n   *  devices, which requires the NDArray to be pre-initialized with context\n   */\n  kAcceptEmptyMutateTarget = 1 << 2\n};\n/*! \\brief Registry entry for NDArrayFunction */\nstruct NDArrayFunctionReg\n    : public dmlc::FunctionRegEntryBase<NDArrayFunctionReg, NDArrayAPIFunction> {\n  /*! \\brief number of variable used by this function */\n  unsigned num_use_vars;\n  /*! \\brief number of variable mutated by this function */\n  unsigned num_mutate_vars;\n  /*! \\brief number of scalars used by this function */\n  unsigned num_scalars;\n  /*! \\brief information on how function should be called from API */\n  int type_mask;\n  /*!\n   * \\brief constructor\n   */\n  NDArrayFunctionReg() : num_use_vars(0), num_mutate_vars(0), num_scalars(0), type_mask(0) {}\n  /*!\n   * \\brief set the function body to a NDArray setvalue function\n   *  this will also auto set the parameters correctly\n   * \\param fsetvalue function body to set\n   * \\return ref to the registered entry, used to set properties\n   */\n  inline NDArrayFunctionReg& set_function(void (*fsetvalue)(const real_t& rhs, NDArray* out)) {\n    body            = [fsetvalue](NDArray** used_vars,\n                       real_t* s,\n                       NDArray** mutate_vars,\n                       int num_params,\n                       char** param_keys,\n                       char** param_vals) { (*fsetvalue)(s[0], mutate_vars[0]); };\n    num_mutate_vars = 1;\n    num_scalars     = 1;\n    this->add_argument(\"src\", \"real_t\", \"Source input to the function.\");\n    return *this;\n  }\n  /*!\n   * \\brief set the function body to a ternary NDArray function\n   *  this will also auto set the parameters correctly\n   * \\param fternary function body to set\n   * \\return ref to the registered entry, used to set properties\n   */\n  inline NDArrayFunctionReg& set_function(\n      void (*fternary)(const NDArray& lhs, const NDArray& mhs, const NDArray& rhs, NDArray* out)) {\n    body = [fternary](NDArray** used_vars,\n                      real_t* s,\n                      NDArray** mutate_vars,\n                      int num_params,\n                      char** param_keys,\n                      char** param_vals) {\n      (*fternary)(*used_vars[0], *used_vars[1], *used_vars[2], mutate_vars[0]);\n    };\n    num_use_vars    = 3;\n    num_mutate_vars = 1;\n    type_mask       = kNDArrayArgBeforeScalar | kAcceptEmptyMutateTarget;\n    this->add_argument(\"lhs\", \"NDArray\", \"Left operand to the function.\");\n    this->add_argument(\"mhs\", \"NDArray\", \"Middle operand to the function.\");\n    this->add_argument(\"rhs\", \"NDArray\", \"Right operand to the function.\");\n    return *this;\n  }\n  /*!\n   * \\brief set the function body to a binary NDArray function\n   *  this will also auto set the parameters correctly\n   * \\param fbinary function body to set\n   * \\return ref to the registered entry, used to set properties\n   */\n  inline NDArrayFunctionReg& set_function(void (*fbinary)(const NDArray& lhs,\n                                                          const NDArray& rhs,\n                                                          NDArray* out)) {\n    body = [fbinary](NDArray** used_vars,\n                     real_t* s,\n                     NDArray** mutate_vars,\n                     int num_params,\n                     char** param_keys,\n                     char** param_vals) {\n      (*fbinary)(*used_vars[0], *used_vars[1], mutate_vars[0]);\n    };\n    num_use_vars    = 2;\n    num_mutate_vars = 1;\n    type_mask       = kNDArrayArgBeforeScalar | kAcceptEmptyMutateTarget;\n    this->add_argument(\"lhs\", \"NDArray\", \"Left operand to the function.\");\n    this->add_argument(\"rhs\", \"NDArray\", \"Right operand to the function.\");\n    return *this;\n  }\n  /*!\n   * \\brief set the function body to a binary NDArray function\n   *  this will also auto set the parameters correctly\n   * \\param fscalar function body to set\n   * \\return ref to the registered entry, used to set properties\n   */\n  inline NDArrayFunctionReg& set_function(void (*fscalar)(const NDArray& lhs,\n                                                          const real_t& rhs,\n                                                          NDArray* out)) {\n    body            = [fscalar](NDArray** used_vars,\n                     real_t* s,\n                     NDArray** mutate_vars,\n                     int num_params,\n                     char** param_keys,\n                     char** param_vals) { (*fscalar)(*used_vars[0], s[0], mutate_vars[0]); };\n    num_use_vars    = 1;\n    num_mutate_vars = 1;\n    num_scalars     = 1;\n    type_mask       = kNDArrayArgBeforeScalar | kAcceptEmptyMutateTarget;\n    this->add_argument(\"lhs\", \"NDArray\", \"Left operand to the function.\");\n    this->add_argument(\"rhs\", \"real_t\", \"Right operand to the function.\");\n    return *this;\n  }\n  /*!\n   * \\brief set the function body to a unary NDArray function\n   *  this will also auto set the parameters correctly\n   * \\param funary function body to set\n   * \\return ref to the registered entry, used to set properties\n   */\n  inline NDArrayFunctionReg& set_function(void (*funary)(const NDArray& src, NDArray* out)) {\n    body            = [funary](NDArray** used_vars,\n                    real_t* s,\n                    NDArray** mutate_vars,\n                    int num_params,\n                    char** param_keys,\n                    char** param_vals) { (*funary)(*used_vars[0], mutate_vars[0]); };\n    num_use_vars    = 1;\n    num_mutate_vars = 1;\n    type_mask       = kNDArrayArgBeforeScalar | kAcceptEmptyMutateTarget;\n    this->add_argument(\"src\", \"NDArray\", \"Source input to the function.\");\n    return *this;\n  }\n  /*!\n   * \\brief set the function body to a unary NDArray function\n   *  this will also auto set the parameters correctly\n   * \\param fgeneric function body to set\n   * \\return ref to the registered entry, used to set properties\n   */\n  inline NDArrayFunctionReg& set_function(\n      void (*fgeneric)(NDArray** used_vars,\n                       real_t* s,\n                       NDArray** mutate_vars,\n                       const std::map<std::string, std::string>& param)) {\n    body = [fgeneric](NDArray** used_vars,\n                      real_t* s,\n                      NDArray** mutate_vars,\n                      int num_params,\n                      char** param_keys,\n                      char** param_vals) {\n      std::map<std::string, std::string> param;\n      for (int i = 0; i < num_params; ++i) {\n        param[param_keys[i]] = param_vals[i];\n      }\n      fgeneric(used_vars, s, mutate_vars, param);\n    };\n    return *this;\n  }\n  /*!\n   * \\brief set the number of mutate variables\n   * \\param n number of mutate variablesx\n   * \\return ref to the registered entry, used to set properties\n   */\n  inline NDArrayFunctionReg& set_num_use_vars(unsigned n) {\n    num_use_vars = n;\n    return *this;\n  }\n  /*!\n   * \\brief set the number of mutate variables\n   * \\param n number of mutate variablesx\n   * \\return ref to the registered entry, used to set properties\n   */\n  inline NDArrayFunctionReg& set_num_mutate_vars(unsigned n) {\n    num_mutate_vars = n;\n    return *this;\n  }\n  /*!\n   * \\brief set the number of scalar arguments\n   * \\param n number of scalar arguments\n   * \\return ref to the registered entry, used to set properties\n   */\n  inline NDArrayFunctionReg& set_num_scalars(unsigned n) {\n    num_scalars = n;\n    return *this;\n  }\n  /*!\n   * \\brief set type mask\n   * \\param tmask typemask\n   * \\return ref to the registered entry, used to set properties\n   */\n  inline NDArrayFunctionReg& set_type_mask(int tmask) {\n    type_mask = tmask;\n    return *this;\n  }\n};  // NDArrayFunctionReg\n\n/*!\n * \\brief Macro to register NDArray function\n *\n * Example: the following code is example to register a plus\n * \\code\n *\n * REGISTER_NDARRAY_FUN(Plus)\n * .set_function(Plus);\n *\n * \\endcode\n */\n#define MXNET_REGISTER_NDARRAY_FUN(name) \\\n  DMLC_REGISTRY_REGISTER(::mxnet::NDArrayFunctionReg, NDArrayFunctionReg, name)\n\n}  // namespace mxnet\n\nnamespace dmlc {\n/*!\\brief traits */\nDMLC_DECLARE_TRAITS(has_saveload, mxnet::NDArray, true);\n}  // namespace dmlc\n#endif  // MXNET_NDARRAY_H_\n"
  },
  {
    "path": "include/mxnet/node/container.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*!\n * \\file container.h\n * \\brief Array container\n */\n// Acknowledgement: This file originates from incubator-tvm\n#ifndef MXNET_NODE_CONTAINER_H_\n#define MXNET_NODE_CONTAINER_H_\n\n#include <mxnet/node/node.h>\n\n#include <type_traits>\n#include <vector>\n#include <initializer_list>\n#include <unordered_map>\n#include <utility>\n#include <string>\n\nnamespace mxnet {\n\n/*! \\brief array node content in array */\nclass ArrayNode : public Object {\n public:\n  /*! \\brief the data content */\n  std::vector<ObjectRef> data;\n\n  static constexpr const char* _type_key = \"Array\";\n  MXNET_DECLARE_FINAL_OBJECT_INFO(ArrayNode, Object)\n};\n\n/*!\n * \\brief iterator adapter that adapts TIter to return another type.\n * \\tparam Converter a struct that contains converting function\n * \\tparam TIter the content iterator type.\n */\ntemplate <typename Converter, typename TIter>\nclass IterAdapter {\n public:\n  using difference_type   = typename std::iterator_traits<TIter>::difference_type;\n  using value_type        = typename Converter::ResultType;\n  using pointer           = typename Converter::ResultType*;\n  using reference         = typename Converter::ResultType&;  // NOLINT(*)\n  using iterator_category = typename std::iterator_traits<TIter>::iterator_category;\n\n  explicit IterAdapter(TIter iter) : iter_(iter) {}\n  inline IterAdapter& operator++() {\n    ++iter_;\n    return *this;\n  }\n  inline IterAdapter operator+(difference_type offset) const {\n    return IterAdapter(iter_ + offset);\n  }\n\n  template <typename T = IterAdapter>\n  typename std::enable_if<std::is_same<iterator_category, std::random_access_iterator_tag>::value,\n                          typename T::difference_type>::type inline\n  operator-(const IterAdapter& rhs) const {\n    return iter_ - rhs.iter_;\n  }\n\n  inline bool operator==(IterAdapter other) const {\n    return iter_ == other.iter_;\n  }\n  inline bool operator!=(IterAdapter other) const {\n    return !(*this == other);\n  }\n  inline const value_type operator*() const {\n    return Converter::convert(*iter_);\n  }\n\n private:\n  TIter iter_;\n};\n\n/*!\n * \\brief Array container of NodeRef in DSL graph.\n *  Array implements copy on write semantics, which means array is mutable\n *  but copy will happen when array is referenced in more than two places.\n *\n * operator[] only provide const acces, use Set to mutate the content.\n * \\tparam T The content NodeRef type.\n */\ntemplate <typename T,\n          typename = typename std::enable_if<std::is_base_of<ObjectRef, T>::value>::type>\nclass Array : public ObjectRef {\n public:\n  /*!\n   * \\brief default constructor\n   */\n  Array() {\n    data_ = make_object<ArrayNode>();\n  }\n  /*!\n   * \\brief move constructor\n   * \\param other source\n   */\n  Array(Array<T>&& other) {  // NOLINT(*)\n    data_ = std::move(other.data_);\n  }\n  /*!\n   * \\brief copy constructor\n   * \\param other source\n   */\n  Array(const Array<T>& other) {  // NOLINT(*)\n    data_ = std::move(other.data_);\n  }\n  /*!\n   * \\brief constructor from pointer\n   * \\param n the container pointer\n   */\n  explicit Array(runtime::ObjectPtr<Object> n) : ObjectRef(n) {}\n  /*!\n   * \\brief constructor from iterator\n   * \\param begin begin of iterator\n   * \\param end end of iterator\n   * \\tparam IterType The type of iterator\n   */\n  template <typename IterType>\n  Array(IterType begin, IterType end) {\n    assign(begin, end);\n  }\n  /*!\n   * \\brief constructor from initializer list\n   * \\param init The initalizer list\n   */\n  Array(std::initializer_list<T> init) {  // NOLINT(*)\n    assign(init.begin(), init.end());\n  }\n  /*!\n   * \\brief constructor from vector\n   * \\param init The vector\n   */\n  Array(const std::vector<T>& init) {  // NOLINT(*)\n    assign(init.begin(), init.end());\n  }\n  /*!\n   * \\brief Constructs a container with n elements. Each element is a copy of val\n   * \\param n The size of the container\n   * \\param val The init value\n   */\n  explicit Array(size_t n, const T& val) {\n    auto tmp_node = make_object<ArrayNode>();\n    for (size_t i = 0; i < n; ++i) {\n      tmp_node->data.push_back(val);\n    }\n    data_ = std::move(tmp_node);\n  }\n  /*!\n   * \\brief move assign operator\n   * \\param other The source of assignment\n   * \\return reference to self.\n   */\n  Array<T>& operator=(Array<T>&& other) {\n    data_ = std::move(other.data_);\n    return *this;\n  }\n  /*!\n   * \\brief copy assign operator\n   * \\param other The source of assignment\n   * \\return reference to self.\n   */\n  Array<T>& operator=(const Array<T>& other) {\n    data_ = other.data_;\n    return *this;\n  }\n  /*!\n   * \\brief reset the array to content from iterator.\n   * \\param begin begin of iterator\n   * \\param end end of iterator\n   * \\tparam IterType The type of iterator\n   */\n  template <typename IterType>\n  void assign(IterType begin, IterType end) {\n    auto n = make_object<ArrayNode>();\n    for (IterType it = begin; it != end; ++it) {\n      n->data.push_back(T(*it));\n    }\n    data_ = std::move(n);\n  }\n  /*!\n   * \\brief Read i-th element from array.\n   * \\param i The index\n   * \\return the i-th element.\n   */\n  inline const T operator[](size_t i) const {\n    return DowncastNoCheck<T>(static_cast<const ArrayNode*>(data_.get())->data[i]);\n  }\n  /*! \\return The size of the array */\n  inline size_t size() const {\n    if (data_.get() == nullptr)\n      return 0;\n    return static_cast<const ArrayNode*>(data_.get())->data.size();\n  }\n  /*!\n   * \\brief copy on write semantics\n   *  Do nothing if current handle is the unique copy of the array.\n   *  Otherwise make a new copy of the array to ensure the current handle\n   *  hold a unique copy.\n   *\n   * \\return Handle to the internal node container(which ganrantees to be unique)\n   */\n  inline ArrayNode* CopyOnWrite() {\n    if (data_.get() == nullptr || !data_.unique()) {\n      runtime::ObjectPtr<ArrayNode> n = make_object<ArrayNode>();\n      n->data                         = static_cast<ArrayNode*>(data_.get())->data;\n      runtime::ObjectPtr<Object>(std::move(n)).swap(data_);\n    }\n    return static_cast<ArrayNode*>(data_.get());\n  }\n  /*!\n   * \\brief push a new item to the back of the list\n   * \\param item The item to be pushed.\n   */\n  inline void push_back(const T& item) {\n    ArrayNode* n = this->CopyOnWrite();\n    n->data.push_back(item);\n  }\n  /*!\n   * \\brief Resize the array.\n   * \\param size The new size.\n   */\n  inline void resize(size_t size) {\n    ArrayNode* n = this->CopyOnWrite();\n    n->data.resize(size);\n  }\n  /*!\n   * \\brief set i-th element of the array.\n   * \\param i The index\n   * \\param value The value to be setted.\n   */\n  inline void Set(size_t i, const T& value) {\n    ArrayNode* n = this->CopyOnWrite();\n    n->data[i]   = value;\n  }\n  /*! \\return whether array is empty */\n  inline bool empty() const {\n    return size() == 0;\n  }\n  /*!\n   * \\brief Helper function to apply fmutate to mutate an array.\n   * \\param fmutate The transformation function T -> T.\n   * \\tparam F the type of the mutation function.\n   * \\note This function performs copy on write optimization.\n   */\n  template <typename F>\n  inline void MutateByApply(F fmutate) {\n    ArrayNode* ptr = static_cast<ArrayNode*>(data_.get());\n    if (ptr == nullptr)\n      return;\n    if (data_.unique()) {\n      // Copy on write optimization.\n      // Perform inplace update because this is an unique copy.\n      for (size_t i = 0; i < ptr->data.size(); ++i) {\n        // It is important to use move here\n        // to make prevent the element's ref count from increasing\n        // so fmutate itself can perform copy-on-write optimization\n        T old_elem   = DowncastNoCheck<T>(std::move(ptr->data[i]));\n        T new_elem   = fmutate(std::move(old_elem));\n        ptr->data[i] = std::move(new_elem);\n      }\n    } else {\n      // lazily trigger copy if there is element change.\n      runtime::ObjectPtr<ArrayNode> copy;\n      for (size_t i = 0; i < ptr->data.size(); ++i) {\n        T old_elem = DowncastNoCheck<T>(ptr->data[i]);\n        T new_elem = fmutate(old_elem);\n        if (!new_elem.same_as(ptr->data[i])) {\n          // copy the old array\n          if (copy == nullptr) {\n            copy = runtime::make_object<ArrayNode>(*ptr);\n          }\n          copy->data[i] = std::move(new_elem);\n        }\n      }\n      // replace the data with the new copy.\n      if (copy != nullptr) {\n        data_ = std::move(copy);\n      }\n    }\n  }\n\n  /*! \\brief specify container node */\n  using ContainerType = ArrayNode;\n\n  struct ValueConverter {\n    using ResultType = T;\n    static inline T convert(const ObjectRef& n) {\n      return DowncastNoCheck<T>(n);\n    }\n  };\n  using iterator = IterAdapter<ValueConverter, std::vector<ObjectRef>::const_iterator>;\n\n  using reverse_iterator =\n      IterAdapter<ValueConverter, std::vector<ObjectRef>::const_reverse_iterator>;\n\n  /*! \\return begin iterator */\n  inline iterator begin() const {\n    return iterator(static_cast<const ArrayNode*>(data_.get())->data.begin());\n  }\n  /*! \\return end iterator */\n  inline iterator end() const {\n    return iterator(static_cast<const ArrayNode*>(data_.get())->data.end());\n  }\n  /*! \\return rbegin iterator */\n  inline reverse_iterator rbegin() const {\n    return reverse_iterator(static_cast<const ArrayNode*>(data_.get())->data.rbegin());\n  }\n  /*! \\return rend iterator */\n  inline reverse_iterator rend() const {\n    return reverse_iterator(static_cast<const ArrayNode*>(data_.get())->data.rend());\n  }\n};\n\n}  // namespace mxnet\n#endif  // MXNET_NODE_CONTAINER_H_\n"
  },
  {
    "path": "include/mxnet/node/node.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*!\n * \\file node.h\n * \\brief Definitions and helper macros for IR/AST nodes.\n *\n *  The node folder contains base utilities for IR/AST nodes,\n *  invariant of which specific language dialect.\n *\n *  We implement AST/IR nodes as sub-classes of runtime::Object.\n *  The base class Node is just an alias of runtime::Object.\n *\n *  Besides the runtime type checking provided by Object,\n *  node folder contains additional functionalities such as\n *  reflection and serialization, which are important features\n *  for building a compiler infra.\n */\n// Acknowledgement: This file originates from incubator-tvm\n#ifndef MXNET_NODE_NODE_H_\n#define MXNET_NODE_NODE_H_\n\n#include <mxnet/runtime/c_runtime_api.h>\n#include <mxnet/runtime/object.h>\n#include <mxnet/runtime/memory.h>\n\n#include <string>\n#include <vector>\n#include <utility>\n#include <type_traits>\n\nnamespace mxnet {\n\nusing runtime::Object;\nusing runtime::TypeIndex;\n// We strictly restrict ObjectPtr to ::mxnet::runtime\n// as it may conflict with ::nnvm::ObjectPtr\n// using runtime::ObjectPtr;\nusing runtime::Downcast;\nusing runtime::GetRef;\nusing runtime::make_object;\nusing runtime::ObjectEqual;\nusing runtime::ObjectHash;\nusing runtime::ObjectRef;\n\n}  // namespace mxnet\n\n#endif  // MXNET_NODE_NODE_H_\n"
  },
  {
    "path": "include/mxnet/op_attr_types.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file op_attr_types.h\n * \\brief Additional operator attributes\n *  beside the ones provided by NNVM\n */\n#ifndef MXNET_OP_ATTR_TYPES_H_\n#define MXNET_OP_ATTR_TYPES_H_\n\n#include <mshadow/tensor.h>\n#include <nnvm/op_attr_types.h>\n\n#include <vector>\n#include <functional>\n#include <string>\n\n#include \"./base.h\"\n#include \"./ndarray.h\"\n#include \"./engine.h\"\n#include \"./resource.h\"\n\nnamespace mxnet {\n\nusing nnvm::NodeAttrs;\n\n/*! \\brief operation request type to Forward and Backward */\nenum OpReqType {\n  /*! \\brief no operation, do not write anything */\n  kNullOp,\n  /*! \\brief write gradient to provided space */\n  kWriteTo,\n  /*!\n   * \\brief perform an inplace write,\n   * This option only happen when\n   * Target shares memory with one of input arguments.\n   */\n  kWriteInplace,\n  /*! \\brief add to the provided space */\n  kAddTo\n};\n\n/*!\n * \\brief All the possible information needed by Operator.\n *  This is the superset of RunContext.\n *  We use this data structure to bookkeep everything needed by Forward and Backward.\n * \\sa Resource\n */\nstruct OpContext {\n  /*! \\brief whether there is a backward phase to compute gradients. */\n  bool need_grad;\n  /*! \\brief whether it is training phase */\n  bool is_train;\n  /*! \\brief RunContext related resources */\n  RunContext run_ctx;\n  /*! \\brief the callback when operation completes, used by asynchronize ops */\n  engine::CallbackOnComplete async_on_complete;\n  /*! \\brief Resources requested by the operator */\n  std::vector<Resource> requested;\n  /*!\n   * \\brief get mshadow stream from Context\n   * \\return the mshadow stream\n   * \\tparam xpu the device type of the stream\n   */\n  template <typename xpu>\n  inline mshadow::Stream<xpu>* get_stream() const {\n    return run_ctx.get_stream<xpu>();\n  }\n#if MXNET_USE_CUDA\n  /*!\n   * \\brief get auxilary gpu stream auto-syncing object from Context\n   * \\return the aux stream auto-syncing object\n   */\n  inline SyncedGPUAuxStream get_gpu_aux_stream() const {\n    return run_ctx.get_gpu_aux_stream();\n  }\n#endif\n};\n\n/*! \\brief the execution type of the operator */\nenum class ExecType {\n  /*! \\brief Forward/Backward are synchronous calls */\n  kSync,\n  /*!\n   * \\brief Forward/Backward are asynchronous,\n   *  will call OpContext.async_on_complete when operation finishes.\n   */\n  kAsync,\n  /*!\n   * \\brief Cross device copy operation, this is a special operator that indicates it will copy\n   * across devices. For example the input and output for this type of operator can potentially\n   * reside on different devices.  In the current implementation, a copy operator is specially\n   * handled by an executor. This flag is used for special case treatment and future extension of\n   * different copy ops.\n   */\n  kCrossDeviceCopy,\n  /*!\n   * \\brief A subgraph execution should happen in the main thread, instead of\n   *  in the execution engine.\n   */\n  kSubgraphExec,\n};\n\n/*! \\brief the dispatch mode of the operator */\nenum class DispatchMode {\n  kUndefined = -1,\n  // dispatch on FCompute or FStatefulCompute\n  kFCompute,\n  // dispatch on FComputeEx or FStatefulComputeEx, if available\n  kFComputeEx,\n  // dispatch on FCompute or FStatefulCompute, and performs storage fallback\n  kFComputeFallback,\n  // special dispatch mode for variables\n  kVariable,\n};\n\n/*! \\brief the quantization type of the operator */\nenum class QuantizeType {\n  // This operator doesn't support quantization\n  kNone = 0,\n  // This operator can get huge benefit from quantization, thus must be quantized\n  kMust,\n  // This operator support quantization, but will be decided depending on the connection\n  kSupport,\n};\n\n/*!\n * \\brief Operator state. This is a pointer type, its content is mutable\n *  even if OpStatePtr is const.\n */\nclass OpStatePtr {\n public:\n  /* \\brief Create a OpStatePtr with state of type T.\n   * \\param args Arguments passed to T's constructor.\n   */\n  template <typename T, typename... Args>\n  static OpStatePtr Create(Args&&... args) {\n    OpStatePtr ret;\n    auto state = new T(std::forward<Args>(args)...);\n    auto var   = Engine::Get()->NewVariable();\n    ret.ptr_.reset(new OpState(var, state), [](OpState* p) {\n      Engine::Get()->DeleteVariable([](RunContext s) {}, Context::CPU(), p->var);\n      delete reinterpret_cast<T*>(p->state);\n      delete p;\n    });\n\n    return ret;\n  }\n  /* \\brief Get engine variable associated with this state */\n  engine::VarHandle get_var() const {\n    return ptr_->var;\n  }\n  /* \\brief Get state of type T */\n  template <typename T>\n  T& get_state() const {\n    return *reinterpret_cast<T*>(ptr_->state);\n  }\n  /* \\brief clear state */\n  void reset() {\n    ptr_.reset();\n  }\n  /* \\brief checks whether the managed object is managed only by the current\n            OpStatePtr instance */\n  bool unique() const {\n    return ptr_.unique();\n  }\n  /* \\brief Whether state is empty */\n  explicit operator bool() const {\n    return ptr_ ? true : false;\n  }\n\n private:\n  /* \\brief state structure */\n  struct OpState {\n    engine::VarHandle var;\n    void* state;\n\n    OpState(engine::VarHandle var_, void* state_) : var(var_), state(state_) {}\n    OpState(const OpState& other) = delete;\n    OpState& operator=(const OpState& other) = delete;\n  };\n  /* \\brief shared pointer to state */\n  std::shared_ptr<OpState> ptr_;\n};\n\n/*!\n * \\brief Create a Layer style, forward/backward operator.\n *  This is easy to write code that contains state.\n *  OpStatePtr is a pointer type, it's content is mutable even if\n *  OpStatePtr is constant.\n *\n *\n *  This is not the only way to register an op execution function.\n *  More simpler or specialized operator form can be registered\n *\n *  \\note Register under \"FCreateLayerOp\"\n */\nusing FCreateOpState = std::function<OpStatePtr(const NodeAttrs& attrs,\n                                                Context ctx,\n                                                const mxnet::ShapeVector& in_shape,\n                                                const std::vector<int>& in_type)>;\n\n/*!\n * \\brief Whether the operator always produces the same\n *        output given the same input.\n *        This enables certain optimizations\n *        like common expression elimination.\n *\n * \\note Register under \"THasDeterministicOutput\"\n */\nusing THasDeterministicOutput = bool;\n\n/*!\n * \\brief Execution mode of this operator.\n */\nusing FExecType = std::function<ExecType(const NodeAttrs& attrs)>;\n/*!\n * \\brief Resiger a compute function for stateful operator.\n *  OpStatePtr is a pointer type, it's content is mutable even if\n *  OpStatePtr is constant.\n *\n * \\note Register under \"FStatefulCompute<cpu>\" and \"FStatefulCompute<gpu>\"\n */\nusing FStatefulCompute = std::function<void(const OpStatePtr& state,\n                                            const OpContext& ctx,\n                                            const std::vector<TBlob>& inputs,\n                                            const std::vector<OpReqType>& req,\n                                            const std::vector<TBlob>& outputs)>;\n/*!\n * \\brief Resiger a compute function for stateful operator using NDArray interface.\n *  OpStatePtr is a pointer type, it's content is mutable even if\n *  OpStatePtr is constant.\n *\n * \\note Register under \"FStatefulComputeEx<cpu>\" and \"FStatefulComputeEx<gpu>\"\n */\nusing FStatefulComputeEx = std::function<void(const OpStatePtr& state,\n                                              const OpContext& ctx,\n                                              const std::vector<NDArray>& inputs,\n                                              const std::vector<OpReqType>& req,\n                                              const std::vector<NDArray>& outputs)>;\n/*!\n * \\brief The resource request from the operator.\n *        An operator could register ResourceRequestEx, or ResourceRequest, or neither.\n *\n * \\note Register under \"FResourceRequest\"\n */\nusing FResourceRequest = std::function<std::vector<ResourceRequest>(const NodeAttrs& n)>;\n/*!\n * \\brief The resource request from the operator.\n *        An operator could register ResourceRequestEx, or ResourceRequest, or neither.\n *        If an operator registers both ResourceRequestEx and ResourceRequest,\n *        ResourceRequest is ignored.\n *\n * \\note Register under \"FResourceRequestEx\"\n */\nusing FResourceRequestEx =\n    std::function<std::vector<ResourceRequest>(const NodeAttrs& n,\n                                               const int dev_mask,\n                                               const DispatchMode dispatch_mode)>;\n/*!\n * \\brief Register an operator called as a NDArray function\n *\n * \\note Register under \"FNDArrayFunction\"\n */\nusing FNDArrayFunction = std::function<void(const nnvm::NodeAttrs& attrs,\n                                            const std::vector<NDArray>& inputs,\n                                            std::vector<NDArray>* outputs)>;\n/*!\n * \\brief Register a compute function for simple stateless forward only operator\n *\n * \\note Register under \"FCompute<cpu>\" and \"FCompute<gpu>\"\n */\nusing FCompute = std::function<void(const nnvm::NodeAttrs& attrs,\n                                    const OpContext& ctx,\n                                    const std::vector<TBlob>& inputs,\n                                    const std::vector<OpReqType>& req,\n                                    const std::vector<TBlob>& outputs)>;\n/*!\n * \\brief Register an NDArray compute function for simple stateless forward only operator\n * \\note Register under \"FComputeEx<xpu>\" and \"FComputeEx<xpu>\"\n *       Dispatched only when inferred dispatch_mode is FDispatchComputeEx\n */\nusing FComputeEx = std::function<void(const nnvm::NodeAttrs& attrs,\n                                      const OpContext& ctx,\n                                      const std::vector<NDArray>& inputs,\n                                      const std::vector<OpReqType>& req,\n                                      const std::vector<NDArray>& outputs)>;\n\n/*!\n * \\brief Register a storage and dispatch mode inference function based on\n *        storage types of the inputs and outputs, and the dev_mask for the operator.\n *\n * \\note Register under \"FInferStorageType\"\n */\nusing FInferStorageType = std::function<bool(const NodeAttrs& attrs,\n                                             const int dev_mask,\n                                             DispatchMode* dispatch_mode,\n                                             std::vector<int>* in_attrs,\n                                             std::vector<int>* out_attrs)>;\n\n/*!\n * \\brief Register a quantized node creation function based on the attrs of the node\n * \\note Register under \"FQuantizedOp\" for non-quantized operators\n */\nusing FQuantizable = std::function<QuantizeType(const NodeAttrs& attrs)>;\n\n/*!\n * \\brief Register a quantized node creation function based on the attrs of the node\n * \\note Register under \"FQuantizedOp\" for non-quantized operators\n */\nusing FQuantizedOp = std::function<nnvm::ObjectPtr(const NodeAttrs& attrs)>;\n\n/*!\n * \\brief Register a function to determine if the output of a quantized operator\n * needs to be requantized. This is usually used for the operators\n * taking int8 data types while accumulating in int32, e.g. quantized_conv.\n * \\note Register under \"FNeedRequantize\" for non-quantized operators\n */\nusing FNeedRequantize = std::function<bool(const NodeAttrs& attrs)>;\n\n/*!\n * \\brief Register a function to determine if the input of a quantized operator\n * needs to be quantized. This is usually used for the quantized operators\n * which can handle fp32 inputs directly.\n */\nusing FAvoidQuantizeInput = std::function<\n    bool(const NodeAttrs& attrs, const size_t index, const std::string quantize_granularity)>;\n\n/*!\n * \\brief Register a function to determine if the input of a quantized operator\n * needs to be quantized asymmetrically.\n */\nusing FNeedAsymQuantizeInput = std::function<bool(const NodeAttrs& attrs, const size_t index)>;\n\n/*!\n * \\brief Register a function to determine if the output of a quantized operator\n * needs to be dequantized. This is usually used for the quantized operators\n * which can produce fp32 outputs directly.\n */\nusing FAvoidDequantizeOutput = std::function<bool(const NodeAttrs& attrs, const size_t index)>;\n\n/*!\n * \\brief Register a function to determine if the input of a quantized operator\n * needs to be calibrated. This is usually used for the quantized operators\n * which need calibration on its input.\n */\nusing FNeedCalibrateInput = std::function<std::vector<int>(const NodeAttrs& attrs)>;\n\n/*!\n * \\brief Register a function to determine if the output of a quantized operator\n * needs to be calibrated. This is usually used for the quantized operators\n * which need calibration on its output.\n */\nusing FNeedCalibrateOutput = std::function<std::vector<int>(const NodeAttrs& attrs)>;\n\n#if MXNET_USE_CUDA\n\n/*!\n * \\brief Register a function to determine if\n * the operator implementation is compatible\n * with CUDA graphs. This requires the execution\n * to stay the same as long as the shape and type\n * of input stays the same.\n */\nusing FIsCUDAGraphsCompatible = std::function<bool(const NodeAttrs& attrs, const bool is_train)>;\n\n#endif\n\n}  // namespace mxnet\n\n#endif  // MXNET_OP_ATTR_TYPES_H_\n"
  },
  {
    "path": "include/mxnet/operator.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file operator.h\n * \\brief Operator interface of mxnet.\n * \\author Naiyan Wang\n */\n#ifndef MXNET_OPERATOR_H_\n#define MXNET_OPERATOR_H_\n\n#include <dmlc/base.h>\n#include <dmlc/json.h>\n#include <dmlc/logging.h>\n#include <dmlc/registry.h>\n#include <nnvm/node.h>\n#include <vector>\n#include <map>\n#include <string>\n#include <utility>\n#include \"./base.h\"\n#include \"./resource.h\"\n#include \"./op_attr_types.h\"\n\nnamespace mxnet {\n/*!\n * \\brief Operator interface.\n *  Operator defines basic operation unit of optimized computation graph in mxnet.\n *  This interface relies on pre-allocated memory in TBlob, the caller need to set\n *  the memory region in TBlob correctly before calling Forward and Backward.\n *\n *  Operator is generated by OperatorProperty.\n *  To add new operator(aka. layers of neural nets) to mxnet, developer need to create\n *  a new OperatorProperty and its corresponding Operator.\n *\n * \\sa TBlob, mxnet::TShape, OperatorProperty\n */\nclass Operator {\n public:\n  /*! \\brief destructor */\n  virtual ~Operator() {}\n  /*!\n   * \\brief perform a forward operation of Operator, save the output to TBlob.\n   * \\param ctx runtime context available to this call\n   * \\param in_data array of input data, it is const\n   * \\param req the request types of saving operation, can only be kWriteTo or kWriteInplace.\n   * \\param out_data array of output data, pointer is used to indicate that this is holder\n   *        the space of TBlob in out_data must be pre-allocated with InferShape\n   * \\param aux_states Auxiliary states of operator. Normally operator doesn't\n   *        need, epecial case like Batch Norm requires.\n   * \\sa OpReqType, OpContext\n   */\n  virtual void Forward(const OpContext& ctx,\n                       const std::vector<TBlob>& in_data,\n                       const std::vector<OpReqType>& req,\n                       const std::vector<TBlob>& out_data,\n                       const std::vector<TBlob>& aux_states) = 0;\n  /*!\n   * \\brief Perform a Backward Operation, write gradient to the in_grad.\n   *\n   * \\note\n   * Convention:\n   *   out_grad.size() == OperatorProperty.NumVisibleOutputs()\n   *   out_data.size() == OperatorProperty.NumOutputs()\n   * out_data can contain additional invisible returns that remembers the\n   * state carried from the Forward pass. For example mask in the dropout.\n   * The gradients are passed from visible returns in this function.\n   *\n   * \\par\n   * Not all the TBlobs in the arguments will be available\n   * if you override the DeclareBackwardDependency of corresponding OperatorProperty class.\n   * Only the dependencies you declared will be available at corresponding position,\n   * the rest of the parameters are simply dummy where you will get a nullptr.\n   * You will be safe if you use the default DeclareBackwardDependency.\n   * But only declare what you need will give engine more chance for optimization.\n   *\n   * \\param ctx runtime context available to this call\n   * \\param out_grad the gradient value we get from of the Operator.\n   * \\param in_data the array of input data.\n   * \\param out_data the array of output data.\n   * \\param req request types of the saving operation, can be all types.\n   * \\param in_grad the array of gradient we need to write to.\n   * \\param aux_states Auxiliary states of operator. Normally operator doesn't need\n   * \\sa OperatorProperty, OpReqType, OpContext\n   */\n  virtual void Backward(const OpContext& ctx,\n                        const std::vector<TBlob>& out_grad,\n                        const std::vector<TBlob>& in_data,\n                        const std::vector<TBlob>& out_data,\n                        const std::vector<OpReqType>& req,\n                        const std::vector<TBlob>& in_grad,\n                        const std::vector<TBlob>& aux_states) {\n    LOG(FATAL) << \"Backward is not implemented\";\n  }\n  /*! \\return [Deprecated] execution type of the operator */\n  virtual ExecType exec_type()  // NOLINT(*) exec_type has been moved to OperatorProperty\n      const final {             // NOLINT(*) exec_type has been moved to OperatorProperty\n    return ExecType::kSync;\n  }\n};\n\n#if DMLC_USE_CXX11\n// OperatorProperty allows C++11, while Operator do not rely on it.\n/*!\n * \\brief OperatorProperty is a object that stores all information about Operator.\n * It also contains method to generate context(device) specific operators.\n *\n * It also contains various functions that can be optimally overriden to\n * provide optimization chance for computation engine.\n */\nclass OperatorProperty {\n public:\n  /*!\n   * \\brief virtual destructor\n   */\n  virtual ~OperatorProperty() {}\n  /*!\n   *  \\brief Initialize the Operator by setting the parameters\n   *  This function need to be called before all other functions.\n   *  \\param kwargs the keyword arguments parameters\n   */\n  virtual void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) = 0;\n  /*!\n   * \\brief Get a map representation of internal parameters.\n   *  This can be used by Init to recover the state of OperatorProperty.\n   */\n  virtual std::map<std::string, std::string> GetParams() const = 0;\n  /*!\n   * \\brief Get input arguments of the Operator.\n   * \\return vector of arguments.\n   */\n  virtual std::vector<std::string> ListArguments() const {\n    return {\"data\"};\n  }\n  /*!\n   * \\brief Get name of output values of Operator\n   * \\return name of output values.\n   */\n  virtual std::vector<std::string> ListOutputs() const {\n    return {\"output\"};\n  }\n  /*!\n   * \\brief Get name of auxiliary states of Operator\n   * \\return name of return values.\n   */\n  virtual std::vector<std::string> ListAuxiliaryStates() const {\n    return {};\n  }\n  /*! \\return number of real return values of the Operator */\n  virtual int NumOutputs() const {\n    return this->ListOutputs().size();\n  }\n  /*!\n   * \\brief get number of visible return values during Symbol creation.\n   *  If NumVisibleOutputs() = k, and NumOutputs() = n.\n   *  The first k returns will be presented in the resulting symbol.\n   *\n   *  The rest of the returns can be used for auxiliary states for Backward.\n   *  For example, Dropout will return [data, mask], with NumVisibleOutputs() == 1.\n   *  So when user call sym = Dropout(input), only data is presented in sym.\n   *  But all the returns will be presented in out_data parameter of Backward if requested.\n   *\n   * \\return number of default return values\n   */\n  virtual int NumVisibleOutputs() const {\n    return NumOutputs();\n  }\n  /*!\n   * \\brief infer the shapes of outputs and unknown input arguments\n   * \\param in_shape the shape of input arguments of the operator\n   *     this should be of same length as the vector returned by DescribeArgs\n   *     in_shape allows unknown elements, which are checked by shape.ndim() == 0.\n   *     For unknown shapes, InferShape will try to fill in the correct Shape in in_shape\n   *     For known shapes, InferShape will check shape consistency\n   *\n   *     common practice: set the shape of data input, and usually weight's shape can be inferred\n   *\n   * \\param out_shape the shape of outputs of the operator\n   *     InferShape will modify the vector to fill output mxnet::TShape\n   * \\param aux_shape the shape of auxiliary states of the operator\n   *     InferShape will modify the vector to fill output mxnet::TShape\n   * \\return true if the shape inference is successful, false if there is not enough information.\n   * \\throws dmlc::Error if the known arg_shapes are inconsistent.\n   */\n  virtual bool InferShape(mxnet::ShapeVector* in_shape,\n                          mxnet::ShapeVector* out_shape,\n                          mxnet::ShapeVector* aux_shape) const = 0;\n  /*!\n   * \\brief infer the data types of outputs and unknown input arguments\n   * \\param in_type the type of input arguments of the operator\n   *     this should be of same length as the vector returned by DescribeArgs\n   *     in_type allows unknown elements, which are checked by type.ndim() == 0.\n   *     For unknown types, Infertype will try to fill in the correct type in in_type\n   *     For known types, Infertype will check type consistency\n   *\n   *     common practice: set the type of data input, and usually weight's type can be inferred\n   *\n   * \\param out_type the type of outputs of the operator\n   *     Infertype will modify the vector to fill output Ttype\n   * \\param aux_type the type of auxiliary states of the operator\n   *     Infertype will modify the vector to fill output Ttype\n   * \\return true if the type inference is successful, false if there is not enough information.\n   * \\throws dmlc::Error if the known arg_types are inconsistent.\n   */\n  virtual bool InferType(std::vector<int>* in_type,\n                         std::vector<int>* out_type,\n                         std::vector<int>* aux_type) const {\n    CHECK_LE(in_type->size(), this->ListArguments().size());\n    int n_in = this->ListArguments().size();\n    for (unsigned i = 0; i < in_type->size(); ++i) {\n      CHECK(in_type->at(i) == mshadow::default_type_flag || in_type->at(i) == -1)\n          << \"Unsupported data type \" << in_type->at(i);\n    }\n    in_type->clear();\n    for (int i = 0; i < n_in; ++i)\n      in_type->push_back(mshadow::default_type_flag);\n\n    int n_out = this->ListOutputs().size();\n    out_type->clear();\n    for (int i = 0; i < n_out; ++i)\n      out_type->push_back(mshadow::default_type_flag);\n\n    int n_aux = this->ListAuxiliaryStates().size();\n    aux_type->clear();\n    for (int i = 0; i < n_aux; ++i)\n      aux_type->push_back(mshadow::default_type_flag);\n    return true;\n  }\n  /*!\n   * \\brief Copy this OperatorProperty.\n   * \\return a pointer to the copied OperatorProperty\n   */\n  virtual OperatorProperty* Copy() const = 0;\n  /*!\n   * \\brief Create a Operator on specific context\n   */\n  virtual Operator* CreateOperator(Context ctx) const = 0;\n  /*!\n   * \\brief Create a Operator on specific context and input shape/type\n   * \\param ctx context of this operator\n   * \\param in_shape shape of the input ndarrays\n   * \\param in_type dtype of the input ndarrays\n   * \\return the created operator\n   */\n  virtual Operator* CreateOperatorEx(Context ctx,\n                                     mxnet::ShapeVector* in_shape,\n                                     std::vector<int>* in_type) const {\n    std::vector<int> out_type, aux_type;\n    mxnet::ShapeVector out_shape, aux_shape;\n    out_type.resize(this->ListOutputs().size());\n    out_shape.resize(this->ListOutputs().size());\n    aux_type.resize(this->ListAuxiliaryStates().size());\n    aux_shape.resize(this->ListAuxiliaryStates().size());\n    CHECK(InferType(in_type, &out_type, &aux_type));\n    CHECK(InferShape(in_shape, &out_shape, &aux_shape));\n    return CreateOperator(ctx);\n  }\n  /*!\n   * \\brief return the type string of the Operator\n   *  subclasses override this function.\n   * \\return The type string.\n   */\n  virtual std::string TypeString() const = 0;\n  //--------------------------------------------------------\n  // All the below functions are optional to override.\n  //--------------------------------------------------------\n  /*!\n   * \\brief Declare additional resource required in forward pass.\n   *  These additional resources will be presented in OpContext.requested\n   *  in the same order of the returned Resource.\n   * \\param in_shape The input shape to the operator, corresponds to shapes of in_data.\n   * \\return Additional resource request\n   */\n  virtual std::vector<ResourceRequest> ForwardResource(const mxnet::ShapeVector& in_shape) const {\n    return std::vector<ResourceRequest>();\n  }\n  /*!\n   * \\brief Declare additional resource required in backward pass.\n   *  These additional resources will be presented in OpContext.requested\n   *  in the same order of the returned Resource.\n   * \\param in_shape The input shape to the operator, corresponds to shapes of in_data.\n   * \\return Additional resource request\n   */\n  virtual std::vector<ResourceRequest> BackwardResource(const mxnet::ShapeVector& in_shape) const {\n    return std::vector<ResourceRequest>();\n  }\n  /*!\n   * \\brief Declare the input requirement of Backward pass.\n   *\n   *  Only the returned list of variables will be used in Backward.\n   *  This function is used for memory optimization.\n   *  It is advised to override and only return what is actually needed.\n   *  If this function is not overriden, all the variables will be valid in Backward.\n   *\n   * \\code\n   *  // The following code declares Backward need out_grad[0], in_data[0],in_data[1]\n   *  vector<int> BackwardInputs(const vector<int> &out_grad,\n   *                             const vector<int> &in_data,\n   *                             const vector<int> &out_data) const {\n   *    return {out_grad[0], in_data[0], in_data[1]};\n   *  }\n   * \\endcode\n   * \\param out_grad gradient of outputs in backward pass.\n   * \\param in_data the input data in forward pass.\n   * \\param out_data the output data in forward pass.\n   * \\return an integer vector indicating the input requirments\n   * \\sa BackwardInputs\n   */\n  virtual std::vector<int> DeclareBackwardDependency(const std::vector<int>& out_grad,\n                                                     const std::vector<int>& in_data,\n                                                     const std::vector<int>& out_data) const {\n    // By default requires to see all the things.\n    // remember to override this function to get a better performance.\n    std::vector<int> ret = out_grad;\n    ret.insert(ret.end(), in_data.begin(), in_data.end());\n    ret.insert(ret.end(), out_data.begin(), out_data.end());\n    return ret;\n  }\n  /*!\n   * \\brief Get possible forward inplace options.\n   *  This function enables optimization to reuse memory of inputs in output.\n   *  Only override when necessary, by default in-place is disabled.\n   *\n   *  The reason for void* type in the out_data is to distinguish the order\n   *  of mappings between the two, compiler will report error when\n   *  in_data and out_data's order in the pair get reversed.\n   *\n   * \\code\n   *  // The following code says out_data[0] can share data with in_data[0]\n   *  vector<pair<int, void*> > ForwardInplaceOption(const vector<int> &in_data,\n   *                                                 const vector<void*> &out_data) const {\n   *    return {{in_data[0], out_data[0]}};\n   *  }\n   * \\endcode\n   * \\param in_data The input data in forward pass.\n   * \\param out_data The output data in forward pass.\n   * \\return list of pair of that maps input->output,\n   *   indicating possible in place operations.\n   */\n  virtual std::vector<std::pair<int, void*> > ForwardInplaceOption(\n      const std::vector<int>& in_data,\n      const std::vector<void*>& out_data) const {\n    return std::vector<std::pair<int, void*> >();\n  }\n  /*!\n   * \\brief Get possible backward inplace options.\n   *  This function enables optimization to reuse memory of inputs in output.\n   *  Only override when necessary, by default in-place is disabled.\n   *\n   *  The reason for void* type in the in_grad is to distinguish the order\n   *  of mappings between the two, compiler will report error when\n   *  in_data and out_data's order in the pair get reversed.\n   *\n   * \\code\n   *  // The following code says in_grad[0] can share data with in_data[0]\n   *  vector<pair<int,int> > BackwardInplaceOption(\n   *                 const std::vector<int> &out_grad,\n   *                 const std::vector<int> &in_data,\n   *                 const std::vector<int> &out_data,\n   *                 const std::vector<int> &in_grad) const {\n   *    return {in_data[0], in_grad[0]}};\n   *  }\n   * \\endcode\n   * \\param in_data The input data in forward pass.\n   * \\param out_data The output data in forward pass.\n   * \\param in_grad Gradient of inputs in backward pass.\n   * \\param out_grad Gradient of outputs in backward pass.\n   * \\return list of pair of that maps input->output,\n   *   indicating possible in place operations.\n   */\n  virtual std::vector<std::pair<int, void*> > BackwardInplaceOption(\n      const std::vector<int>& out_grad,\n      const std::vector<int>& in_data,\n      const std::vector<int>& out_data,\n      const std::vector<void*>& in_grad) const {\n    return std::vector<std::pair<int, void*> >();\n  }\n  /*!\n   * \\brief Get Backward Input Dependency for generic types of data.\n   *  Normally T can be pointer of Symbol::DataEntry, or NDArray.\n   *  This function will select the result list of T according to DeclareBackwardDependency.\n   *\n   * \\param in_data the input data in forward pass.\n   * \\param out_data the output data in forward pass.\n   * \\param out_grad gradient of outputs in backward pass.\n   * \\tparam T the generic type parameter.\n   * \\return vector of inputs the Backward Operation depends on.\n   * \\sa DeclareBackwardDependency\n   */\n  template <typename T>\n  inline std::vector<T> BackwardInputs(const std::vector<T>& out_grad,\n                                       const std::vector<T>& in_data,\n                                       const std::vector<T>& out_data) const {\n    int counter = 0;\n    std::vector<int> out_grad_index(out_grad.size());\n    std::vector<int> in_data_index(in_data.size());\n    std::vector<int> out_data_index(out_data.size());\n    for (size_t i = 0; i < out_grad_index.size(); ++i) {\n      out_grad_index[i] = counter++;\n    }\n    for (size_t i = 0; i < in_data_index.size(); ++i) {\n      in_data_index[i] = counter++;\n    }\n    for (size_t i = 0; i < out_data_index.size(); ++i) {\n      out_data_index[i] = counter++;\n    }\n    std::vector<T> all_data;\n    all_data.insert(all_data.end(), out_grad.begin(), out_grad.end());\n    all_data.insert(all_data.end(), in_data.begin(), in_data.end());\n    all_data.insert(all_data.end(), out_data.begin(), out_data.end());\n\n    std::vector<int> ret_index =\n        this->DeclareBackwardDependency(out_grad_index, in_data_index, out_data_index);\n\n    std::vector<T> ret(ret_index.size());\n    for (size_t i = 0; i < ret_index.size(); ++i) {\n      ret[i] = all_data[ret_index[i]];\n    }\n    return ret;\n  }\n  /*!\n   * \\brief create OperatorProperty\n   * \\param type_name the type string of the OperatorProperty\n   * \\return a new constructed OperatorProperty\n   */\n  static OperatorProperty* Create(const char* type_name);\n  /*! \\return execution type of the operator */\n  virtual ExecType exec_type() const {\n    return ExecType::kSync;\n  }\n};\n\n/*! \\brief typedef the factory function of operator property */\ntypedef std::function<OperatorProperty*()> OperatorPropertyFactory;\n/*!\n * \\brief Registry entry for OperatorProperty factory functions.\n */\nstruct OperatorPropertyReg\n    : public dmlc::FunctionRegEntryBase<OperatorPropertyReg, OperatorPropertyFactory> {\n  /*!\n   * \\brief Set key_var_num_args\n   *  When this is set, the API caller is required to pass in a\n   *  argument with key=key_num_args.c_str(), and value=num_args.\n   *  num_args is number of positional argument when calling the function.\n   *\n   *  This is used to pass in length of positional arguments\n   *  for operators that can take variable length of input.\n   *  Most operators do not need to set this property.\n   *\n   * \\param key the key name to be set\n   */\n  inline OperatorPropertyReg& set_key_var_num_args(const std::string& key) {  // NOLINT(*)\n    this->key_var_num_args = key;\n    return *this;\n  }\n  /*!\n   * \\brief Check if TypeString of the type matches the registered name\n   */\n  inline OperatorPropertyReg& check_name() {\n    OperatorProperty* p = this->body();\n    std::string type    = p->TypeString();\n    delete p;\n    CHECK_EQ(this->name, type) << \"Register Name and TypeString mismatch, name=\\\"\" << this->name\n                               << \"\\\",\"\n                               << \" but TypeString=\\\"\" << type << \"\\\"\";\n    return *this;\n  }\n\n  /*! \\brief The key num_args name. */\n  std::string key_var_num_args;\n};\n\n//---------------------------------------------------------------------------------\n// The following part are API Registration of Operators\n// See also MXNET_REGISTER_SIMPLE_OP in operator_util.h for registering simple ops.\n//---------------------------------------------------------------------------------\n/*!\n * \\brief Macro to register OperatorProperty\n *\n * \\code\n * // example of registering a fully connected operator\n * REGISTER_OP_PROPERTY(FullyConnected, FullyConnectedOpProp)\n * .describe(\"Fully connected layer\");\n *\n * \\endcode\n */\n#define MXNET_REGISTER_OP_PROPERTY(name, OperatorPropertyType)                    \\\n  DMLC_REGISTRY_REGISTER(::mxnet::OperatorPropertyReg, OperatorPropertyReg, name) \\\n      .set_body([]() { return new OperatorPropertyType(); })                      \\\n      .set_return_type(\"NDArray-or-Symbol\")                                       \\\n      .check_name()\n\n#endif  // DMLC_USE_CXX11\n}  // namespace mxnet\n#endif  // MXNET_OPERATOR_H_\n"
  },
  {
    "path": "include/mxnet/operator_util.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file operator_util.h\n * \\brief Utility functions and registries to help quickly build new operators.\n *  [Deprecated]\n *  Use the register functions in this file when possible to simplify operator creations.\n *  Operators registered in this file will be exposed to both NDArray API and symbolic API.\n *\n * \\author Tianqi Chen\n */\n#ifndef MXNET_OPERATOR_UTIL_H_\n#define MXNET_OPERATOR_UTIL_H_\n\n#ifdef _MSC_VER\n#pragma warning(disable : 4503)  // disable warning: decorated name length exceeded.\n#endif\n\n#include <dmlc/registry.h>\n#include <dmlc/parameter.h>\n#include <map>\n#include <vector>\n#include <string>\n#include <utility>\n#include \"./base.h\"\n#include \"./operator.h\"\n\n#if DMLC_USE_CXX11\n#include <functional>\n#endif\n\nnamespace mxnet {\n/*! \\brief namespace of arguments */\nnamespace op {\n/*! \\brief super class of all gradient function argument */\nstruct GradFunctionArgument {\n  /*! \\brief The real data */\n  TBlob data;\n};\n\n/*! \\brief First input to the function */\nstruct Input0 : GradFunctionArgument {};\n/*! \\brief Second input to the function */\nstruct Input1 : GradFunctionArgument {};\n\n/*! \\brief Ouput value of the function to the function */\nstruct OutputValue : GradFunctionArgument {};\n/*! \\brief Gradient of output value */\nstruct OutputGrad : GradFunctionArgument {};\n\n/*!\n * \\brief Environment arguments that is used by the function.\n * These can be things like scalar arguments when add a value with scalar.\n */\nstruct EnvArguments {\n  /*! \\brief scalar argument, if enabled */\n  real_t scalar;\n  /*! \\brief keyword arguments */\n  std::vector<std::pair<std::string, std::string> > kwargs;\n  /*! \\brief pointer to the resources requested */\n  std::vector<Resource> resource;\n};\n\n/*!\n * \\brief source function that generate output based on env\n *  The result container is pre-allocated with the correct shape.\n * \\param env The Environment arguments.\n * \\param ret The containter to store return value.\n * \\param req The requirement to stroe the ret.\n * \\param ctx Runtime context to execute the function.\n */\ntypedef void (*SourceFunction)(const EnvArguments& env, TBlob* ret, OpReqType req, RunContext ctx);\n\n/*!\n * \\brief Shape inference function to get the correct shape.\n * \\param env The Environment arguments.\n * \\return The inferred result shape.\n */\ntypedef mxnet::TShape (*SourceShapeFunction)(const EnvArguments& env);\n\n/*!\n * \\brief Unary function that takes a src and save result to ret.\n *  The result container is pre-allocated with the correct shape.\n * \\param src The source data.\n * \\param env The Environment arguments.\n * \\param ret The containter to store return value.\n * \\param req The requirement to stroe the ret.\n * \\param ctx Runtime context to execute the function.\n */\ntypedef void (*UnaryFunction)(const TBlob& src,\n                              const EnvArguments& env,\n                              TBlob* ret,\n                              OpReqType req,\n                              RunContext ctx);\n/*!\n * \\brief Shape inference function to get the correct shape given source.\n * \\param src The source shape\n * \\param env The Environment arguments.\n * \\return The inferred result shape.\n */\ntypedef mxnet::TShape (*UnaryShapeFunction)(const mxnet::TShape& src, const EnvArguments& env);\n\n/*!\n * \\brief Gradient function that takes output value of function and computes gradient wrt to input.\n * \\param out_grad the gradient wrt to output of the function.\n * \\param env The Environment arguments.\n * \\param in_grad The container to store result input gradient.\n * \\param req The requirement to store the ret value.\n * \\param ctx Runtime context to execute the function.\n */\ntypedef void (*UnaryGradFunctionT0)(const OutputGrad& out_grad,\n                                    const EnvArguments& env,\n                                    TBlob* in_grad,\n                                    OpReqType req,\n                                    RunContext ctx);\n/*!\n * \\brief Gradient function that takes output value of function and computes gradient wrt to input.\n * \\param out_grad the gradient wrt to output of the function.\n * \\param out_value the value of the function.\n * \\param env The Environment arguments.\n * \\param in_grad The container to store result input gradient.\n * \\param req The requirement to store the ret value.\n * \\param ctx Runtime context to execute the function.\n */\ntypedef void (*UnaryGradFunctionT1)(const OutputGrad& out_grad,\n                                    const OutputValue& out_value,\n                                    const EnvArguments& env,\n                                    TBlob* in_grad,\n                                    OpReqType req,\n                                    RunContext ctx);\n/*!\n * \\brief Gradient function that takes input value of function and computes gradient wrt to input.\n * \\param out_grad the gradient wrt to output of the function.\n * \\param in_data0 the input value of the function.\n * \\param env The Environment arguments.\n * \\param in_grad The container to store result input gradient.\n * \\param req The requirement to store the ret value.\n * \\param ctx Runtime context to execute the function.\n */\ntypedef void (*UnaryGradFunctionT2)(const OutputGrad& out_grad,\n                                    const Input0& in_data0,\n                                    const EnvArguments& env,\n                                    TBlob* in_grad,\n                                    OpReqType req,\n                                    RunContext ctx);\n/*!\n * \\brief Binary function that takes lhs, rhs and save result to ret.\n *  The result container is pre-allocated with the correct shape.\n * \\param lhs The left operand\n * \\param rhs The right operand\n * \\param env The Environment arguments.\n * \\param ret The containter to store return value.\n * \\param req The requirement to stroe the ret.\n * \\param ctx Runtime context to execute the function.\n */\ntypedef void (*BinaryFunction)(const TBlob& lhs,\n                               const TBlob& rhs,\n                               const EnvArguments& env,\n                               TBlob* ret,\n                               OpReqType req,\n                               RunContext ctx);\n\n/*!\n * \\brief Shape inference function to get the correct shape given source shapes.\n * \\param lhs The shape of left operand.\n * \\param rhs The shape of right operand.\n * \\param env The Environment arguments.\n * \\return The inferred result shape.\n */\ntypedef mxnet::TShape (*BinaryShapeFunction)(const mxnet::TShape& lhs,\n                                             const mxnet::TShape& rhs,\n                                             const EnvArguments& env);\n/*!\n * \\brief Gradient function that takes only output gradient and computes gradient wrt to input.\n *  We support total gradient as a whole to make it easy to combine a few ops.\n * \\param out_grad the gradient wrt to output of the function.\n * \\param env The Environment arguments.\n * \\param lhs_grad The container to store result of lhs gradient.\n * \\param rhs_grad The container to store result of lhs gradient.\n * \\param req_lhs_grad The requirement to store the lhs_grad\n * \\param req_rhs_grad The requirement to store the rhs_grad\n * \\param ctx Runtime context to execute the function.\n */\ntypedef void (*BinaryGradFunctionT0)(const OutputGrad& out_grad,\n                                     const EnvArguments& env,\n                                     TBlob* lhs_grad,\n                                     TBlob* rhs_grad,\n                                     OpReqType req_lhs_grad,\n                                     OpReqType req_rhs_grad,\n                                     RunContext ctx);\n/*!\n * \\brief Gradient function that takes inputs of function anod computes gradient wrt to input.\n * \\param out_grad the gradient wrt to output of the function.\n * \\param lhs The left operand to the function.\n * \\param rhs The right operand to the function.\n * \\param env The Environment arguments.\n * \\param lhs_grad The container to store result of lhs gradient.\n * \\param rhs_grad The container to store result of lhs gradient.\n * \\param req_lhs_grad The requirement to store the lhs_grad\n * \\param req_rhs_grad The requirement to store the rhs_grad\n * \\param ctx Runtime context to execute the function.\n */\ntypedef void (*BinaryGradFunctionT1)(const OutputGrad& out_grad,\n                                     const Input0& lhs,\n                                     const Input1& rhs,\n                                     const EnvArguments& env,\n                                     TBlob* lhs_grad,\n                                     TBlob* rhs_grad,\n                                     OpReqType req_lhs_grad,\n                                     OpReqType req_rhs_grad,\n                                     RunContext ctx);\n\n/*! \\brief options in the registry to set inplace of operator */\nenum SimpleOpInplaceOption {\n  /*! \\brief do not allow inplace in arguments */\n  kNoInplace,\n  /*! \\brief in unary forward, allow inplace in with out */\n  kInplaceInOut,\n  /*! \\brief in unary backward, allow inplace out_grad with in_grad */\n  kInplaceOutIn,\n  /*! \\brief in binary forward, allow inplace left operand with out */\n  kInplaceLhsOut,\n  /*! \\brief in binary backward, allow inplace out_grad with lhs_grad */\n  kInplaceOutLhs\n};\n\n/*! \\brief options in the registry to set symbolic registration */\nenum SimpleOpScalarOption { kScalarBeforeArray, kArrayBeforeScalar };\n\n/*! \\brief options in the registry to set symbolic registration */\nenum SimpleOpRegOption { kNotRegisterSymbolic, kRegisterSymbolic };\n\n/*! \\brief registry entry to register simple operators via functions. */\nclass SimpleOpRegEntry {\n public:\n  /*! \\brief declare self type */\n  typedef SimpleOpRegEntry TSelf;\n  /*! \\brief name of the operator */\n  std::string name;\n  /*!\n   * \\brief set a seperate name for symbol\n   *  This must be called before set_function.\n   *  Default: this is set to be same as the name of operator.\n   * \\param symbol_name the name of symbolic operator.\n   */\n  virtual TSelf& set_symbol_op_name(char const* symbol_name) = 0;\n  /*!\n   * \\brief set number of scalar arguments needed to be passed in env\n   *  A function cannot have both kwargs and scalar arguments.\n   *  Default: this is set to false\n   * \\param enable_scalar whether to enable scalar argument\n   * \\param type_mask the position of the scalar argument.\n   */\n  virtual TSelf& set_enable_scalar(bool enable_scalar,\n                                   SimpleOpScalarOption type_mask = kArrayBeforeScalar) = 0;\n  /*!\n   * \\brief set whether to enable kwargs\n   *  A function cannot have both kwargs and scalar arguments.\n   *  Default: this is set to false\n   * \\param enable_kwargs whether to enable kwargs\n   */\n  virtual TSelf& set_enable_kwargs(bool enable_kwargs) = 0;\n  /*!\n   * \\brief set resource request\n   *  By default there is no resource request.\n   *  The resource will be presented in both forward and backward.\n   * \\param reqs the request.\n   */\n  virtual TSelf& set_resource_request(const std::vector<ResourceRequest>& reqs) = 0;\n  /*!\n   * \\brief set resource request\n   *  By default there is no resource request.\n   *  The resource will be presented in both forward and backward.\n   * \\param req the request.\n   */\n  virtual TSelf& set_resource_request(ResourceRequest req) = 0;\n  /*!\n   * \\brief set source inference function.\n   * \\param fshapeinfer The source function that peforms the operation.\n   */\n  virtual TSelf& set_shape_function(SourceShapeFunction fshapeinfer) = 0;\n  /*!\n   * \\brief set shape inference function.\n   *  Default: out_shape = in_shape\n   * \\param fshapeinfer The unary function that peforms the operation.\n   */\n  virtual TSelf& set_shape_function(UnaryShapeFunction fshapeinfer) = 0;\n  /*!\n   * \\brief set shape inference function to be the binary inference function\n   *  Default: out_shape = lhs_shape, and lhs_shape must equal rhs_shape.\n   * \\param fshapeinfer The binary function that peforms the operation.\n   */\n  virtual TSelf& set_shape_function(BinaryShapeFunction fshapeinfer) = 0;\n  /*!\n   * \\brief set function of the function to be fsource\n   * \\param dev_mask The device mask of the function can act on.\n   * \\param fsource The unary function that peforms the operation.\n   * \\param register_symbolic Whether register a symbolic operator as well.\n   */\n  virtual TSelf& set_function(int dev_mask,\n                              SourceFunction fsource,\n                              SimpleOpRegOption register_symbolic = kRegisterSymbolic) = 0;\n  /*!\n   * \\brief set function of the function to be funary\n   * \\param dev_mask The device mask of the function can act on.\n   * \\param funary The unary function that peforms the operation.\n   * \\param inplace_in_out Whether do inplace optimization on in and out.\n   * \\param register_symbolic Whether register a symbolic operator as well.\n   */\n  virtual TSelf& set_function(int dev_mask,\n                              UnaryFunction funary,\n                              SimpleOpInplaceOption inplace_in_out,\n                              SimpleOpRegOption register_symbolic = kRegisterSymbolic) = 0;\n  /*!\n   * \\brief set function of the function to be funary\n   * \\param dev_mask The device mask of the function can act on.\n   * \\param fbinary The binary function that peforms the operation.\n   * \\param inplace_lhs_out Whether do inplace optimization on lhs and out.\n   * \\param register_symbolic Whether register a symbolic operator as well.\n   */\n  virtual TSelf& set_function(int dev_mask,\n                              BinaryFunction fbinary,\n                              SimpleOpInplaceOption inplace_lhs_out,\n                              SimpleOpRegOption register_symbolic = kRegisterSymbolic) = 0;\n  /*!\n   * \\brief set gradient of the function of this function.\n   * \\param dev_mask The device mask of the function can act on.\n   * \\param fgrad The gradient function to be set.\n   * \\param inplace_out_in_grad whether out_grad and in_grad can share memory.\n   */\n  virtual TSelf& set_gradient(int dev_mask,\n                              UnaryGradFunctionT0 fgrad,\n                              SimpleOpInplaceOption inplace_out_in_grad) = 0;\n  /*!\n   * \\brief set gradient of the function of this function.\n   * \\param dev_mask The device mask of the function can act on.\n   * \\param fgrad The gradient function to be set.\n   * \\param inplace_out_in_grad whether out_grad and in_grad can share memory.\n   */\n  virtual TSelf& set_gradient(int dev_mask,\n                              UnaryGradFunctionT1 fgrad,\n                              SimpleOpInplaceOption inplace_out_in_grad) = 0;\n  /*!\n   * \\brief set gradient of the function of this function.\n   * \\param dev_mask The device mask of the function can act on.\n   * \\param fgrad The gradient function to be set.\n   * \\param inplace_out_in_grad whether out_grad and in_grad can share memory.\n   */\n  virtual TSelf& set_gradient(int dev_mask,\n                              UnaryGradFunctionT2 fgrad,\n                              SimpleOpInplaceOption inplace_out_in_grad) = 0;\n  /*!\n   * \\brief set gradient of the function of this function.\n   * \\param dev_mask The device mask of the function can act on.\n   * \\param fgrad The gradient function to be set.\n   * \\param inplace_out_lhs_grad whether out_grad and lhs_grad can share memory.\n   */\n  virtual TSelf& set_gradient(int dev_mask,\n                              BinaryGradFunctionT0 fgrad,\n                              SimpleOpInplaceOption inplace_out_lhs_grad) = 0;\n  /*!\n   * \\brief set gradient of the function of this function.\n   * \\param dev_mask The device mask of the function can act on.\n   * \\param fgrad The gradient function to be set.\n   * \\param inplace_out_lhs_grad whether out_grad and lhs_grad can share memory.\n   */\n  virtual TSelf& set_gradient(int dev_mask,\n                              BinaryGradFunctionT1 fgrad,\n                              SimpleOpInplaceOption inplace_out_lhs_grad) = 0;\n  /*!\n   * \\brief Describe the function.\n   * \\param description The description of the function.\n   * \\return reference to self.\n   */\n  virtual TSelf& describe(const std::string& description) = 0;\n  /*!\n   * \\brief Describe the function.\n   * \\param args argument information.\n   *  Add additional arguments to the function.\n   * \\return reference to self.\n   */\n  virtual TSelf& add_arguments(const std::vector<dmlc::ParamFieldInfo>& args) = 0;\n  /*! \\brief virtual destructor */\n  virtual ~SimpleOpRegEntry() {}\n};\n\n/*! \\brief registry for TBlob functions */\nclass SimpleOpRegistry {\n public:\n  /*!\n   * \\brief Internal function to register a name function under name.\n   * \\param name name of the function\n   * \\return ref to the registered entry, used to set properties\n   */\n  SimpleOpRegEntry& __REGISTER_OR_FIND__(char const* name);\n  /*!\n   * \\brief Find the entry with corresponding name.\n   * \\param name name of the function\n   * \\return the corresponding function, can be nullptr\n   */\n  inline static const SimpleOpRegEntry* Find(const std::string& name) {\n    return Get()->fmap_.at(name);\n  }\n  /*! \\return global singleton of the registry */\n  static SimpleOpRegistry* Get();\n\n private:\n  // destructor\n  ~SimpleOpRegistry();\n  /*! \\brief internal registry map */\n  std::map<std::string, SimpleOpRegEntry*> fmap_;\n};\n\n/*!\n * \\brief assign the expression to out according to request\n * \\param out the data to be assigned\n * \\param req the assignment request\n * \\param exp the expression\n * \\tparam OType output type\n * \\tparam Exp expression type\n */\n#define ASSIGN_DISPATCH(out, req, exp) \\\n  {                                    \\\n    switch (req) {                     \\\n      case kNullOp:                    \\\n        break;                         \\\n      case kWriteTo:                   \\\n      case kWriteInplace:              \\\n        (out) = (exp);                 \\\n        break;                         \\\n      case kAddTo:                     \\\n        (out) += (exp);                \\\n        break;                         \\\n      default:                         \\\n        LOG(FATAL) << \"not reached\";   \\\n    }                                  \\\n  }\n\n/*!\n * \\brief Maximum ndim supported for special operators like broadcasting with non contiguous lhs/rhs\n */\n#define MXNET_SPECIAL_MAX_NDIM 5\n\n//--------------------------------------------------------------\n// The following part are API Registration of Simple Operators\n//--------------------------------------------------------------\n/*!\n * \\brief Macro to register simple operator to both imperative and symbolic API.\n *\n * see src/operator/elementwise_unary_op-inl.h for example\n *\n * \\code\n * // example of registering a sigmoid operator on GPU\n * // MySigmoid is of type UnaryFunction,\n * // MySigmoidGrad is of type UnaryGradFunctionT2\n *\n * MXNET_REGISTER_SIMPLE_OP(sigmoid, cpu)\n * .set_function(MySigmoid<gpu>, true)\n * .set_gradient(MySigmoidGrad<gpu>, true)\n * .describe(\"Sigmoid function\");\n *\n * \\endcode\n */\n#define MXNET_REGISTER_SIMPLE_OP(Name, DEV)                                               \\\n  static ::mxnet::op::SimpleOpRegEntry& __make_##SimpleOpRegEntry##_##Name##__##DEV##__ = \\\n      ::mxnet::op::SimpleOpRegistry::Get()->__REGISTER_OR_FIND__(#Name)\n\n}  // namespace op\n}  // namespace mxnet\n#endif  // MXNET_OPERATOR_UTIL_H_\n"
  },
  {
    "path": "include/mxnet/random_generator.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file random_generator.h\n * \\brief Parallel random number generator.\n */\n#ifndef MXNET_RANDOM_GENERATOR_H_\n#define MXNET_RANDOM_GENERATOR_H_\n\n#include <random>\n#include <new>\n#include \"./base.h\"\n\n#if MXNET_USE_CUDA\n#include <curand_kernel.h>\n#include <math.h>\n#endif  // MXNET_USE_CUDA\n\nnamespace mxnet {\nnamespace common {\nnamespace random {\n\ntemplate <typename Device, typename DType MSHADOW_DEFAULT_DTYPE>\nclass RandGenerator;\n\ntemplate <typename DType>\nclass RandGenerator<cpu, DType> {\n public:\n  // at least how many random numbers should be generated by one CPU thread.\n  static const int kMinNumRandomPerThread;\n  // store how many global random states for CPU.\n  static const int kNumRandomStates;\n\n  // implementation class for random number generator\n  // TODO(alexzai): move impl class to separate file - tracked in MXNET-948\n  class Impl {\n   public:\n    typedef\n        typename std::conditional<std::is_floating_point<DType>::value, DType, double>::type FType;\n    explicit Impl(RandGenerator<cpu, DType>* gen, int state_idx)\n        : engine_(gen->states_ + state_idx) {}\n\n    Impl(const Impl&) = delete;\n    Impl& operator=(const Impl&) = delete;\n\n    MSHADOW_XINLINE int rand() {\n      return engine_->operator()();\n    }\n\n    MSHADOW_XINLINE int64_t rand_int64() {\n      return static_cast<int64_t>(engine_->operator()() << 31) + engine_->operator()();\n    }\n\n    MSHADOW_XINLINE FType uniform() {\n      typedef typename std::conditional<std::is_integral<DType>::value,\n                                        std::uniform_int_distribution<DType>,\n                                        std::uniform_real_distribution<FType>>::type GType;\n      GType dist_uniform;\n      return dist_uniform(*engine_);\n    }\n\n    MSHADOW_XINLINE FType normal() {\n      std::normal_distribution<FType> dist_normal;\n      return dist_normal(*engine_);\n    }\n\n   private:\n    std::mt19937* engine_;\n  };  // class RandGenerator<cpu, DType>::Impl\n\n  static void AllocState(RandGenerator<cpu, DType>* inst) {\n    inst->states_ = new std::mt19937[kNumRandomStates];\n  }\n\n  static void FreeState(RandGenerator<cpu, DType>* inst) {\n    delete[] inst->states_;\n  }\n\n  MSHADOW_XINLINE void Seed(mshadow::Stream<cpu>*, uint32_t seed) {\n    for (int i = 0; i < kNumRandomStates; ++i)\n      (states_ + i)->seed(seed + i);\n  }\n\n  // export global random states, used by c++ custom operator\n  MSHADOW_XINLINE void* GetStates() {\n    return static_cast<void*>(states_);\n  }\n\n private:\n  std::mt19937* states_;\n};  // class RandGenerator<cpu, DType>\n\ntemplate <typename DType>\nconst int RandGenerator<cpu, DType>::kMinNumRandomPerThread = 64;\n\ntemplate <typename DType>\nconst int RandGenerator<cpu, DType>::kNumRandomStates = 1024;\n\n#if MXNET_USE_CUDA\n\ntemplate <typename DType>\nclass RandGenerator<gpu, DType> {\n public:\n  // at least how many random numbers should be generated by one GPU thread.\n  static const int kMinNumRandomPerThread;\n  // store how many global random states for GPU.\n  static const int kNumRandomStates;\n\n  // uniform number generation in Cuda made consistent with stl (include 0 but exclude 1)\n  // by using 1.0-curand_uniform().\n  // Needed as some samplers in sampler.h won't be able to deal with\n  // one of the boundary cases.\n  // TODO(alexzai): move impl class to separate file - tracked in MXNET-948\n  class Impl {\n   public:\n    Impl& operator=(const Impl&) = delete;\n    Impl(const Impl&)            = delete;\n\n    // Copy state to local memory for efficiency.\n    __device__ explicit Impl(RandGenerator<gpu, DType>* gen, int state_idx)\n        : global_gen_(gen), global_state_idx_(state_idx), state_(*(gen->states_ + state_idx)) {}\n\n    __device__ ~Impl() {\n      // store the curand state back into global memory\n      global_gen_->states_[global_state_idx_] = state_;\n    }\n\n    MSHADOW_FORCE_INLINE __device__ int rand() {\n      return curand(&state_);\n    }\n\n    MSHADOW_FORCE_INLINE __device__ int64_t rand_int64() {\n      return static_cast<int64_t>(curand(&state_) << 31) + curand(&state_);\n    }\n\n    MSHADOW_FORCE_INLINE __device__ float uniform() {\n      return static_cast<float>(1.0) - curand_uniform(&state_);\n    }\n\n    MSHADOW_FORCE_INLINE __device__ float normal() {\n      return curand_normal(&state_);\n    }\n\n   private:\n    RandGenerator<gpu, DType>* global_gen_;\n    int global_state_idx_;\n    curandStatePhilox4_32_10_t state_;\n  };  // class RandGenerator<gpu, DType>::Impl\n\n  static void AllocState(RandGenerator<gpu, DType>* inst);\n\n  static void FreeState(RandGenerator<gpu, DType>* inst);\n\n  void Seed(mshadow::Stream<gpu>* s, uint32_t seed);\n\n  // export global random states, used by c++ custom operator\n  void* GetStates();\n\n private:\n  curandStatePhilox4_32_10_t* states_;\n};  // class RandGenerator<gpu, DType>\n\ntemplate <>\nclass RandGenerator<gpu, double> {\n public:\n  // uniform number generation in Cuda made consistent with stl (include 0 but exclude 1)\n  // by using 1.0-curand_uniform().\n  // Needed as some samplers in sampler.h won't be able to deal with\n  // one of the boundary cases.\n  // TODO(alexzai): move impl class to separate file - tracked in MXNET-948\n  class Impl {\n   public:\n    Impl& operator=(const Impl&) = delete;\n    Impl(const Impl&)            = delete;\n\n    // Copy state to local memory for efficiency.\n    __device__ explicit Impl(RandGenerator<gpu, double>* gen, int state_idx)\n        : global_gen_(gen), global_state_idx_(state_idx), state_(*(gen->states_ + state_idx)) {}\n\n    __device__ ~Impl() {\n      // store the curand state back into global memory\n      global_gen_->states_[global_state_idx_] = state_;\n    }\n\n    MSHADOW_FORCE_INLINE __device__ int rand() {\n      return curand(&state_);\n    }\n\n    MSHADOW_FORCE_INLINE __device__ int64_t rand_int64() {\n      return static_cast<int64_t>(curand(&state_) << 31) + curand(&state_);\n    }\n\n    MSHADOW_FORCE_INLINE __device__ double uniform() {\n      return static_cast<float>(1.0) - curand_uniform_double(&state_);\n    }\n\n    MSHADOW_FORCE_INLINE __device__ double normal() {\n      return curand_normal_double(&state_);\n    }\n\n   private:\n    RandGenerator<gpu, double>* global_gen_;\n    int global_state_idx_;\n    curandStatePhilox4_32_10_t state_;\n  };  // class RandGenerator<gpu, double>::Impl\n\n private:\n  curandStatePhilox4_32_10_t* states_;\n};  // class RandGenerator<gpu, double>\n\n#endif  // MXNET_USE_CUDA\n\n}  // namespace random\n}  // namespace common\n}  // namespace mxnet\n#endif  // MXNET_RANDOM_GENERATOR_H_\n"
  },
  {
    "path": "include/mxnet/resource.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file resource.h\n * \\brief Global resource allocation handling.\n */\n#ifndef MXNET_RESOURCE_H_\n#define MXNET_RESOURCE_H_\n\n#include <dmlc/logging.h>\n#include <string>\n#include \"./base.h\"\n#include \"./engine.h\"\n#include \"./random_generator.h\"\n\nnamespace mxnet {\n\n/*!\n * \\brief The resources that can be requested by Operator\n */\nstruct ResourceRequest {\n  /*! \\brief Resource type, indicating what the pointer type is */\n  enum Type {\n    /*! \\brief mshadow::Random<xpu> object */\n    kRandom,\n    /*! \\brief A dynamic temp space that can be arbitrary size */\n    kTempSpace,\n    /*! \\brief common::RandGenerator<xpu> object, which can be used in GPU kernel functions */\n    kParallelRandom\n#if MXNET_USE_CUDNN == 1\n    ,\n    /*! \\brief cudnnDropoutDescriptor_t object for GPU dropout kernel functions */\n    kCuDNNDropoutDesc\n#endif  // MXNET_USE_CUDNN == 1\n  };\n  /*! \\brief type of resources */\n  Type type;\n  /*! \\brief default constructor */\n  ResourceRequest() {}\n  /*!\n   * \\brief constructor, allow implicit conversion\n   * \\param type type of resources\n   */\n  ResourceRequest(Type type)  // NOLINT(*)\n      : type(type) {}\n};\n\nnamespace {\n/// \\brief Given a path, extract the filename.\ninline std::string __extract_fname(const std::string& path) {\n  std::size_t last_dir_pos = path.find_last_of(\"/\\\\\");\n  if (last_dir_pos == std::string::npos) {\n    return path;\n  }\n  return path.substr(last_dir_pos + 1);\n}\n}  // anonymous namespace\n\n#if (defined(__GNUC__) || defined(__GNUG__)) && !defined(__clang__)\n#define MXNET_RESOURCE_DEFAULT_NAME_FARG(tag)                          \\\n  std::string(tag) + \" (\" + __extract_fname(__builtin_FILE()) + \" +\" + \\\n      std::to_string(__builtin_LINE()) + \")\"\n#else  // !__GNUC__ || __clang__\n#define MXNET_RESOURCE_DEFAULT_NAME_FARG(tag) \\\n  std::string(tag) + \" (\" + __extract_fname(__FILE__) + \" +\" + std::to_string(__LINE__) + \")\"\n#endif  // __GNUC__ && !__clang__\n\n/*!\n * \\brief Resources used by mxnet operations.\n *  A resource is something special other than NDArray,\n *  but will still participate\n */\nstruct Resource {\n  /*! \\brief The original request */\n  ResourceRequest req;\n  /*! \\brief engine variable */\n  engine::VarHandle var;\n  /*! \\brief identifier of id information, used for debug purpose */\n  int32_t id;\n  /*!\n   * \\brief pointer to the resource, do not use directly,\n   *  access using member functions\n   */\n  void* ptr_;\n  /*! \\brief default constructor */\n  Resource() : id(0) {}\n  /*!\n   * \\brief Get random number generator.\n   * \\param stream The stream to use in the random number generator.\n   * \\return the mshadow random number generator requested.\n   * \\tparam xpu the device type of random number generator.\n   */\n  template <typename xpu, typename DType>\n  inline mshadow::Random<xpu, DType>* get_random(mshadow::Stream<xpu>* stream) const {\n    CHECK_EQ(req.type, ResourceRequest::kRandom);\n    mshadow::Random<xpu, DType>* ret = static_cast<mshadow::Random<xpu, DType>*>(ptr_);\n    ret->set_stream(stream);\n    return ret;\n  }\n\n  /*!\n   * \\brief Get parallel random number generator.\n   * \\tparam xpu the device type of random number generator.\n   * \\tparam DType the return type.\n   * \\return the parallel random number generator. for gpu, it is allocated on global memory.\n   */\n  template <typename xpu, typename DType>\n  inline common::random::RandGenerator<xpu, DType>* get_parallel_random() const {\n    CHECK_EQ(req.type, ResourceRequest::kParallelRandom);\n    return static_cast<common::random::RandGenerator<xpu, DType>*>(ptr_);\n  }\n\n  /*!\n   * \\brief Get space requested as mshadow Tensor.\n   *  The caller can request arbitrary size.\n   *\n   *  This space can be shared with other calls to this->get_space.\n   *  So the caller need to serialize the calls when using the conflicted space.\n   *  The old space can get freed, however, this will incur a synchronization,\n   *  when running on device, so the launched kernels that depend on the temp space\n   *  can finish correctly.\n   *\n   * \\param shape   the shape of returning tensor.\n   * \\param stream  the stream of returning tensor.\n   * \\param name    the name of the operator requesting the resource.\n   * \\return the mshadow tensor requested.\n   * \\tparam xpu   the device type of random number generator.\n   * \\tparam ndim  the number of dimension of the tensor requested.\n   */\n  template <typename xpu, int ndim>\n  inline mshadow::Tensor<xpu, ndim, real_t> get_space(\n      mshadow::Shape<ndim> shape,\n      mshadow::Stream<xpu>* stream,\n      const std::string& name = MXNET_RESOURCE_DEFAULT_NAME_FARG(\"temp_space\")) const {\n    return get_space_typed<xpu, ndim, real_t>(shape, stream, name);\n  }\n  /*!\n   * \\brief Get cpu space requested as mshadow Tensor.\n   *  The caller can request arbitrary size.\n   *\n   * \\param shape the Shape of returning tensor.\n   * \\return the mshadow tensor requested.\n   * \\tparam ndim the number of dimension of the tensor requested.\n   */\n  template <int ndim>\n  inline mshadow::Tensor<cpu, ndim, real_t> get_host_space(mshadow::Shape<ndim> shape) const {\n    return get_host_space_typed<cpu, ndim, real_t>(shape);\n  }\n  /*!\n   * \\brief Get space requested as mshadow Tensor in specified type.\n   *  The caller can request arbitrary size.\n   *\n   * \\param shape   the shape of returning tensor.\n   * \\param stream  the stream of returning tensor.\n   * \\param name    the name of the operator requesting the resource.\n   * \\return the mshadow tensor requested.\n   * \\tparam xpu   the device type of random number generator.\n   * \\tparam ndim  the number of dimension of the tensor requested.\n   */\n  template <typename xpu, int ndim, typename DType>\n  inline mshadow::Tensor<xpu, ndim, DType> get_space_typed(\n      mshadow::Shape<ndim> shape,\n      mshadow::Stream<xpu>* stream,\n      const std::string& name = MXNET_RESOURCE_DEFAULT_NAME_FARG(\"temp_space\")) const {\n    CHECK_EQ(req.type, ResourceRequest::kTempSpace);\n    return mshadow::Tensor<xpu, ndim, DType>(\n        reinterpret_cast<DType*>(get_space_internal(shape.Size() * sizeof(DType), name)),\n        shape,\n        shape[ndim - 1],\n        stream);\n  }\n#if MXNET_USE_CUDNN == 1\n  /*!\n   * \\brief Get cuDNN dropout descriptor from shared state space.\n   *\n   * \\param dropout_desc  reference to previously created cuDNN dropout descriptor.\n   * \\param stream  the stream of returning tensor.\n   * \\param dropout the ratio of inputs to keep.\n   * \\param name    the name of the operator requesting the resource.\n   * \\return the mshadow tensor requested.\n   */\n  void get_cudnn_dropout_desc(\n      cudnnDropoutDescriptor_t* dropout_desc,\n      mshadow::Stream<gpu>* stream,\n      const float dropout,\n      const std::string& name = MXNET_RESOURCE_DEFAULT_NAME_FARG(\"cudnn_dropout_state\")) const;\n#endif  // MXNET_USE_CUDNN == 1\n\n  /*!\n   * \\brief Get CPU space as mshadow Tensor in specified type.\n   * The caller can request arbitrary size.\n   *\n   * \\param shape the Shape of returning tensor\n   * \\return the mshadow tensor requested\n   * \\tparam ndim the number of dimnesion of tensor requested\n   * \\tparam DType request data type\n   */\n  template <int ndim, typename DType>\n  inline mshadow::Tensor<cpu, ndim, DType> get_host_space_typed(mshadow::Shape<ndim> shape) const {\n    return mshadow::Tensor<cpu, ndim, DType>(\n        reinterpret_cast<DType*>(get_host_space_internal(shape.Size() * sizeof(DType))),\n        shape,\n        shape[ndim - 1],\n        nullptr);\n  }\n  /*!\n   * \\brief internal function to get space from resources.\n   * \\param size the Size of the space.\n   * \\param name the Name of the operator requesting the resource.\n   * \\return The allocated space.\n   */\n  void* get_space_internal(size_t size, const std::string& name) const;\n  /*!\n   * \\brief internal function to get cpu space from resources.\n   * \\param size The size of space.\n   * \\return The allocated space\n   */\n  void* get_host_space_internal(size_t size) const;\n};\n\n/*! \\brief Global resource manager */\nclass ResourceManager {\n public:\n  /*!\n   * \\brief Get resource of requested type.\n   * \\param ctx the context of the request.\n   * \\param req the resource request.\n   * \\return the requested resource.\n   * \\note The returned resource's ownership is\n   *       still hold by the manager singleton.\n   */\n  virtual Resource Request(Context ctx, const ResourceRequest& req) = 0;\n  /*!\n   * \\brief Seed all the allocated random number generators.\n   * \\param seed the seed to the random number generators on all devices.\n   */\n  virtual void SeedRandom(uint32_t seed) = 0;\n  /*!\n   * \\brief Seed the random number generators of the given context.\n   * \\param seed the seed to the random number generators.\n   */\n  virtual void SeedRandom(Context ctx, uint32_t seed) = 0;\n  /*! \\brief virtual destructor */\n  virtual ~ResourceManager() DMLC_THROW_EXCEPTION {}\n  /*!\n   * \\return Resource manager singleton.\n   */\n  static ResourceManager* Get();\n};\n}  // namespace mxnet\n#endif  // MXNET_RESOURCE_H_\n"
  },
  {
    "path": "include/mxnet/rtc.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_RTC_H_\n#define MXNET_RTC_H_\n#include \"./base.h\"\n#if MXNET_USE_CUDA\n#include <nvrtc.h>\n#include <cuda.h>\n\n#include <vector>\n#include <string>\n#include <memory>\n#include <utility>\n#include <unordered_map>\n#include <unordered_set>\n#include \"./ndarray.h\"\n\nnamespace mxnet {\nnamespace rtc {\n\n/*! \\brief Cuda runtime compile module. */\nclass CudaModule {\n private:\n  /*! \\brief Structure for holding internal info. */\n  struct Chunk {\n    /*!\n     * \\brief Constructs cuda module.\n     * \\param source cuda source code.\n     * \\param exports export symbols before mangling.\n     */\n    Chunk(const char* source,\n          const std::vector<std::string>& options,\n          const std::vector<std::string>& exports);\n    /*! \\brief deconstrutor */\n    ~Chunk();\n    /*!\n     * \\brief Get handle to cuda kernel from loaded module\n     * \\param mangled_name mangled kernel name\n     * \\param ctx context to run kernel on\n     * \\return loaded function handle\n     */\n    CUfunction GetFunction(const std::string& mangled_name, const Context& ctx);\n    /*! \\brief nvrtc program handle. */\n    nvrtcProgram prog_;\n    /*! \\brief compiled cuda PTX */\n    std::vector<char> ptx_;\n    /*! \\brief lazily loaded cuda module */\n    std::unordered_map<int, CUmodule> mod_;\n    /*! \\brief exported names */\n    std::unordered_set<std::string> exports_;\n  };\n  /*! \\brief pointer to Chunk */\n  std::shared_ptr<Chunk> ptr_;\n\n public:\n  /*! \\brief cuda kernel argument descriptor */\n  struct ArgType {\n    /*! \\brief whether argument is NDArray */\n    bool is_ndarray;\n    /*! \\brief whether argument is constant (input) */\n    bool is_const;\n    /*! \\brief data type of argument */\n    mshadow::TypeFlag dtype;\n  };\n  /*! \\brief Cuda kernel */\n  class Kernel {\n   public:\n    /*! \\brief Launch the kernel */\n    void Launch(const Context& ctx,\n                const std::vector<dmlc::any>& args,\n                uint32_t grid_dim_x,\n                uint32_t grid_dim_y,\n                uint32_t grid_dim_z,\n                uint32_t block_dim_x,\n                uint32_t block_dim_y,\n                uint32_t block_dim_z,\n                uint32_t shared_mem);\n    /*! \\brief kernel interface signature */\n    const std::vector<ArgType>& signature() {\n      return signature_;\n    }\n\n   private:\n    friend class CudaModule;\n    /*!\n     * \\brief constructor\n     * \\param mod module of this kernel\n     * \\param mangled_name mangled kernel name\n     * \\param signature kernel argument signature\n     */\n    Kernel(const std::shared_ptr<Chunk>& mod,\n           const std::string& mangled_name,\n           const std::vector<ArgType>& signature);\n    /*! \\brief mangled kernel name */\n    std::string mangled_name_;\n    /*! \\brief kernel argument signature */\n    std::vector<ArgType> signature_;\n    /*! \\brief module of this kernel */\n    std::shared_ptr<Chunk> mod_;\n    /*! \\brief cached kernel function on each device */\n    std::unordered_map<int, CUfunction> func_;\n  };\n  /*!\n   * \\brief CudaModule constructor\n   * \\param source cuda source code.\n   * \\param exports export symbols before mangling.\n   */\n  CudaModule(const char* source,\n             const std::vector<std::string>& options,\n             const std::vector<std::string>& exports)\n      : ptr_(std::make_shared<Chunk>(source, options, exports)) {}\n  /*!\n   * \\brief Get cuda kernal from module by name\n   * \\param name kernel name\n   * \\param signature kernel signature\n   * \\return shared pointer to cuda kernel\n   */\n  std::shared_ptr<Kernel> GetKernel(const std::string& name, const std::vector<ArgType>& signature);\n};\n\n}  // namespace rtc\n}  // namespace mxnet\n\n#endif  // MXNET_USE_CUDA\n#endif  // MXNET_RTC_H_\n"
  },
  {
    "path": "include/mxnet/runtime/c_runtime_api.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*\n * \\file c_runtime_api.h\n * \\brief MXNet runtime library.\n */\n// Acknowledgement: This file originates from incubator-tvm\n#ifndef MXNET_RUNTIME_C_RUNTIME_API_H_\n#define MXNET_RUNTIME_C_RUNTIME_API_H_\n\n#include <dlpack/dlpack.h>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n#include <mxnet/c_api.h>\n#include <stdint.h>\n#include <stddef.h>\n\n/*!\n * \\brief The type code in MXNetType\n * \\note MXNetType is used in two places.\n */\ntypedef enum {\n  // The type code of other types are compatible with DLPack.\n  // The next few fields are extension types\n  // that is used by MXNet API calls.\n  kHandle        = 3U,\n  kNull          = 4U,\n  kMXNetType     = 5U,\n  kMXNetContext  = 6U,\n  kObjectHandle  = 7U,\n  kStr           = 8U,\n  kBytes         = 9U,\n  kPyArg         = 10U,\n  kNDArrayHandle = 11U,\n  // Extension codes for other frameworks to integrate MXNet PackedFunc.\n  // To make sure each framework's id do not conflict, use first and\n  // last sections to mark ranges.\n  // Open an issue at the repo if you need a section of code.\n  kExtBegin  = 15U,\n  kNNVMFirst = 16U,\n  kNNVMLast  = 20U,\n  // The following section of code is used for non-reserved types.\n  kExtReserveEnd = 64U,\n  kExtEnd        = 128U,\n  // The rest of the space is used for custom, user-supplied datatypes\n  kCustomBegin = 129U,\n} MXNetTypeCode;\n\n/*!\n * \\brief Union type of values\n *  being passed through API and function calls.\n */\ntypedef union {\n  int64_t v_int64;\n  double v_float64;\n  void* v_handle;\n  const char* v_str;\n  uint64_t v_uint64;\n  DLDataType v_type;\n} MXNetValue;\n\n/*!\n * \\brief Byte array type used to pass in byte array\n *  When kBytes is used as data type.\n */\ntypedef struct {\n  const char* data;\n  size_t size;\n} MXNetByteArray;\n\n/*! \\brief Handle to packed function handle. */\ntypedef void* MXNetFunctionHandle;\n/*! \\brief Handle to Object. */\ntypedef void* MXNetObjectHandle;\n\n/*!\n * \\brief Free the function when it is no longer needed.\n * \\param func The function handle\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNetFuncFree(MXNetFunctionHandle func);\n\n/*!\n * \\brief Call a Packed MXNet Function.\n *\n * \\param func node handle of the function.\n * \\param arg_values The arguments\n * \\param type_codes The type codes of the arguments\n * \\param num_args Number of arguments.\n *\n * \\param ret_val The return value.\n * \\param ret_type_code the type code of return value.\n *\n * \\return 0 when success, -1 when failure happens\n * \\note MXNet calls always exchanges with type bits=64, lanes=1\n *\n * \\note API calls always exchanges with type bits=64, lanes=1\n *   If API call returns container handles (e.g. FunctionHandle)\n *   these handles should be managed by the front-end.\n *   The front-end need to call free function (e.g. MXNetFuncFree)\n *   to free these handles.\n */\nMXNET_DLL int MXNetFuncCall(MXNetFunctionHandle func,\n                            MXNetValue* arg_values,\n                            int* type_codes,\n                            int num_args,\n                            MXNetValue* ret_val,\n                            int* ret_type_code);\n\n/*!\n * \\brief Get a global function.\n *\n * \\param name The name of the function.\n * \\param out the result function pointer, NULL if it does not exist.\n *\n * \\note The function handle of global function is managed by MXNet runtime,\n *  So MXNetFuncFree is should not be called when it get deleted.\n */\nMXNET_DLL int MXNetFuncGetGlobal(const char* name, MXNetFunctionHandle* out);\n\n/*!\n * \\brief List all the globally registered function name\n * \\param out_size The number of functions\n * \\param out_array The array of function names.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNetFuncListGlobalNames(int* out_size, const char*** out_array);\n\n/*!\n * \\brief Free the object.\n *\n * \\param obj The object handle.\n * \\note Internally we decrease the reference counter of the object.\n *       The object will be freed when every reference to the object are removed.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNetObjectFree(MXNetObjectHandle obj);\n\n/*!\n * \\brief Get the type_index from an object.\n *\n * \\param obj The object handle.\n * \\param out_tindex the output type index.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNetObjectGetTypeIndex(MXNetObjectHandle obj, unsigned* out_tindex);\n\n/*!\n * \\brief Convert type key to type index.\n * \\param type_key The key of the type.\n * \\param out_tindex the corresponding type index.\n * \\return 0 when success, -1 when failure happens\n */\nMXNET_DLL int MXNetObjectTypeKey2Index(const char* type_key, unsigned* out_tindex);\n\n#ifdef __cplusplus\n}  // extern \"C\"\n#endif\n#endif  // MXNET_RUNTIME_C_RUNTIME_API_H_\n"
  },
  {
    "path": "include/mxnet/runtime/container.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file container.h\n * \\brief Common POD(plain old data) container types.\n */\n// Acknowledgement: This file originates from incubator-tvm\n#ifndef MXNET_RUNTIME_CONTAINER_H_\n#define MXNET_RUNTIME_CONTAINER_H_\n#include <dmlc/logging.h>\n#include <mxnet/runtime/memory.h>\n#include <mxnet/runtime/object.h>\n\n#include <initializer_list>\n#include <type_traits>\n#include <utility>\n#include <vector>\n\nnamespace mxnet {\nnamespace runtime {\n\nclass ADTBuilder;\n/*!\n * \\brief Base template for classes with array like memory layout.\n *\n *        It provides general methods to access the memory. The memory\n *        layout is ArrayType + [ElemType]. The alignment of ArrayType\n *        and ElemType is handled by the memory allocator.\n *\n * \\tparam ArrayType The array header type, contains object specific metadata.\n * \\tparam ElemType The type of objects stored in the array right after\n * ArrayType.\n *\n * \\code\n * // Example usage of the template to define a simple array wrapper\n * class ArrayObj : public InplaceArrayBase<ArrayObj, Elem> {\n * public:\n *  // Wrap EmplaceInit to initialize the elements\n *  template <typename Iterator>\n *  void Init(Iterator begin, Iterator end) {\n *   size_t num_elems = std::distance(begin, end);\n *   auto it = begin;\n *   this->size = 0;\n *   for (size_t i = 0; i < num_elems; ++i) {\n *     InplaceArrayBase::EmplaceInit(i, *it++);\n *     this->size++;\n *   }\n *  }\n * }\n *\n * void test_function() {\n *   vector<Elem> fields;\n *   auto ptr = make_inplace_array_object<ArrayObj, Elem>(fields.size());\n *   ptr->Init(fields.begin(), fields.end());\n *\n *   // Access the 0th element in the array.\n *   assert(ptr->operator[](0) == fields[0]);\n * }\n *\n * \\endcode\n */\ntemplate <typename ArrayType, typename ElemType>\nclass InplaceArrayBase {\n public:\n  /*!\n   * \\brief Access element at index\n   * \\param idx The index of the element.\n   * \\return Const reference to ElemType at the index.\n   */\n  const ElemType& operator[](size_t idx) const {\n    size_t size = Self()->GetSize();\n    CHECK_LT(idx, size) << \"Index \" << idx << \" out of bounds \" << size << \"\\n\";\n    return *(reinterpret_cast<ElemType*>(AddressOf(idx)));\n  }\n\n  /*!\n   * \\brief Access element at index\n   * \\param idx The index of the element.\n   * \\return Reference to ElemType at the index.\n   */\n  ElemType& operator[](size_t idx) {\n    size_t size = Self()->GetSize();\n    CHECK_LT(idx, size) << \"Index \" << idx << \" out of bounds \" << size << \"\\n\";\n    return *(reinterpret_cast<ElemType*>(AddressOf(idx)));\n  }\n\n  /*!\n   * \\brief Destroy the Inplace Array Base object\n   */\n  ~InplaceArrayBase() {\n    if (!(std::is_standard_layout<ElemType>::value && std::is_trivial<ElemType>::value)) {\n      size_t size = Self()->GetSize();\n      for (size_t i = 0; i < size; ++i) {\n        ElemType* fp = reinterpret_cast<ElemType*>(AddressOf(i));\n        fp->ElemType::~ElemType();\n      }\n    }\n  }\n\n protected:\n  friend class ADTBuilder;\n  /*!\n   * \\brief Construct a value in place with the arguments.\n   *\n   * \\tparam Args Type parameters of the arguments.\n   * \\param idx Index of the element.\n   * \\param args Arguments to construct the new value.\n   *\n   * \\note Please make sure ArrayType::GetSize returns 0 before first call of\n   * EmplaceInit, and increment GetSize by 1 each time EmplaceInit succeeds.\n   */\n  template <typename... Args>\n  void EmplaceInit(size_t idx, Args&&... args) {\n    void* field_ptr = AddressOf(idx);\n    new (field_ptr) ElemType(std::forward<Args>(args)...);\n  }\n\n private:\n  /*!\n   * \\brief Return the self object for the array.\n   *\n   * \\return Pointer to ArrayType.\n   */\n  inline ArrayType* Self() const {\n    return static_cast<ArrayType*>(const_cast<InplaceArrayBase*>(this));\n  }\n\n  /*!\n   * \\brief Return the raw pointer to the element at idx.\n   *\n   * \\param idx The index of the element.\n   * \\return Raw pointer to the element.\n   */\n  void* AddressOf(size_t idx) const {\n    static_assert(\n        alignof(ArrayType) % alignof(ElemType) == 0 && sizeof(ArrayType) % alignof(ElemType) == 0,\n        \"The size and alignment of ArrayType should respect \"\n        \"ElemType's alignment.\");\n\n    size_t kDataStart = sizeof(ArrayType);\n    ArrayType* self   = Self();\n    char* data_start  = reinterpret_cast<char*>(self) + kDataStart;\n    return data_start + idx * sizeof(ElemType);\n  }\n};\n\n/*! \\brief An object representing a structure or enumeration. */\nclass ADTObj : public Object, public InplaceArrayBase<ADTObj, ObjectRef> {\n public:\n  /*! \\brief The tag representing the constructor used. */\n  uint32_t tag;\n  /*! \\brief Number of fields in the ADT object. */\n  uint32_t size{0};\n  // The fields of the structure follows directly in memory.\n\n  static constexpr const char* _type_key      = \"MXNet.ADT\";\n  static constexpr const uint32_t _type_index = TypeIndex::kMXNetADT;\n  MXNET_DECLARE_FINAL_OBJECT_INFO(ADTObj, Object)\n\n private:\n  /*!\n   * \\return The number of elements in the array.\n   */\n  size_t GetSize() const {\n    return size;\n  }\n\n  /*!\n   * \\brief Initialize the elements in the array.\n   *\n   * \\tparam Iterator Iterator type of the array.\n   * \\param begin The begin iterator.\n   * \\param end The end iterator.\n   */\n  template <typename Iterator>\n  void Init(Iterator begin, Iterator end) {\n    size_t num_elems = std::distance(begin, end);\n    this->size       = 0;\n    auto it          = begin;\n    for (size_t i = 0; i < num_elems; ++i) {\n      InplaceArrayBase::EmplaceInit(i, *it++);\n      // Only increment size after the initialization succeeds\n      this->size++;\n    }\n  }\n\n  friend class ADT;\n  friend InplaceArrayBase<ADTObj, ObjectRef>;\n};\n\n/*! \\brief reference to algebraic data type objects. */\nclass ADT : public ObjectRef {\n public:\n  /*!\n   * \\brief construct an ADT object reference.\n   * \\param tag The tag of the ADT object.\n   * \\param fields The fields of the ADT object.\n   * \\return The constructed ADT object reference.\n   */\n  ADT(uint32_t tag, std::vector<ObjectRef> fields) : ADT(tag, fields.begin(), fields.end()){};\n\n  /*!\n   * \\brief construct an ADT object reference.\n   * \\param tag The tag of the ADT object.\n   * \\param begin The begin iterator to the start of the fields array.\n   * \\param end The end iterator to the end of the fields array.\n   * \\return The constructed ADT object reference.\n   */\n  template <typename Iterator>\n  ADT(uint32_t tag, Iterator begin, Iterator end) {\n    size_t num_elems = std::distance(begin, end);\n    auto ptr         = make_inplace_array_object<ADTObj, ObjectRef>(num_elems);\n    ptr->tag         = tag;\n    ptr->Init(begin, end);\n    data_ = std::move(ptr);\n  }\n\n  /*!\n   * \\brief construct an ADT object reference.\n   * \\param tag The tag of the ADT object.\n   * \\param init The initializer list of fields.\n   * \\return The constructed ADT object reference.\n   */\n  ADT(uint32_t tag, std::initializer_list<ObjectRef> init) : ADT(tag, init.begin(), init.end()){};\n\n  /*!\n   * \\brief Access element at index.\n   *\n   * \\param idx The array index\n   * \\return const ObjectRef\n   */\n  const ObjectRef& operator[](size_t idx) const {\n    return operator->()->operator[](idx);\n  }\n\n  /*!\n   * \\brief Return the ADT tag.\n   */\n  size_t tag() const {\n    return operator->()->tag;\n  }\n\n  /*!\n   * \\brief Return the number of fields.\n   */\n  size_t size() const {\n    return operator->()->size;\n  }\n\n  /*!\n   * \\brief Construct a tuple object.\n   *\n   * \\tparam Args Type params of tuple feilds.\n   * \\param args Tuple fields.\n   * \\return ADT The tuple object reference.\n   */\n  template <typename... Args>\n  static ADT Tuple(Args&&... args) {\n    return ADT(0, std::forward<Args>(args)...);\n  }\n\n  MXNET_DEFINE_OBJECT_REF_METHODS(ADT, ObjectRef, ADTObj)\n};\n\n}  // namespace runtime\n}  // namespace mxnet\n\n#endif  // MXNET_RUNTIME_CONTAINER_H_\n"
  },
  {
    "path": "include/mxnet/runtime/container_ext.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file container_ext.h\n * \\brief Common POD(plain old data) container types extension.\n */\n// Acknowledgement: This file originates from dgl\n#ifndef MXNET_RUNTIME_CONTAINER_EXT_H_\n#define MXNET_RUNTIME_CONTAINER_EXT_H_\n#include <dmlc/logging.h>\n#include <mxnet/runtime/memory.h>\n#include <mxnet/runtime/object.h>\n\n#include <string_view>\n#include <string>\n#include <initializer_list>\n#include <type_traits>\n#include <utility>\n#include <vector>\n#include <unordered_map>\n\nnamespace mxnet {\nnamespace runtime {\n\n// Forward declare MXNetArgValue\nclass MXNetArgValue;\n\n/*! \\brief String-aware ObjectRef hash functor */\nstruct ObjectRefHash {\n  /*!\n   * \\brief Calculate the hash code of an ObjectRef\n   * \\param a The given ObjectRef\n   * \\return Hash code of a, string hash for strings and pointer address otherwise.\n   */\n  size_t operator()(const ObjectRef& a) const;\n};\n\n/*! \\brief String-aware ObjectRef equal functor */\nstruct ObjectRefEqual {\n  /*!\n   * \\brief Check if the two ObjectRef are equal\n   * \\param a One ObjectRef\n   * \\param b The other ObjectRef\n   * \\return String equality if both are strings, pointer address equality otherwise.\n   */\n  bool operator()(const ObjectRef& a, const ObjectRef& b) const;\n};\n\n/*! \\brief Shared content of all specializations of hash map */\nclass MapObj : public Object {\n public:\n  /*! \\brief Type of the keys in the hash map */\n  using key_type = ObjectRef;\n  /*! \\brief Type of the values in the hash map */\n  using mapped_type = ObjectRef;\n  /*! \\brief Type of the actual underlying container */\n  using ContainerType = std::unordered_map<ObjectRef, ObjectRef, ObjectRefHash, ObjectRefEqual>;\n  /*! \\brief Iterator class */\n  using iterator = ContainerType::iterator;\n  /*! \\brief Iterator class */\n  using const_iterator = ContainerType::const_iterator;\n  /*! \\brief Type of value stored in the hash map */\n  using KVType = ContainerType::value_type;\n\n  static_assert(std::is_standard_layout<KVType>::value, \"KVType is not standard layout\");\n  static_assert(sizeof(KVType) == 16 || sizeof(KVType) == 8, \"sizeof(KVType) incorrect\");\n\n  static constexpr const uint32_t _type_index = runtime::TypeIndex::kMXNetMap;\n  static constexpr const char* _type_key      = \"MXNet.Map\";\n  MXNET_DECLARE_FINAL_OBJECT_INFO(MapObj, Object);\n\n  /*!\n   * \\brief Number of elements in the MapObj\n   * \\return The result\n   */\n  size_t size() const {\n    return data_.size();\n  }\n  /*!\n   * \\brief Count the number of times a key exists in the hash map\n   * \\param key The indexing key\n   * \\return The result, 0 or 1\n   */\n  size_t count(const key_type& key) const {\n    return data_.count(key);\n  }\n  /*!\n   * \\brief Index value associated with a key, throw exception if the key does not exist\n   * \\param key The indexing key\n   * \\return The const reference to the value\n   */\n  const mapped_type& at(const key_type& key) const {\n    return data_.at(key);\n  }\n  /*!\n   * \\brief Index value associated with a key, throw exception if the key does not exist\n   * \\param key The indexing key\n   * \\return The mutable reference to the value\n   */\n  mapped_type& at(const key_type& key) {\n    return data_.at(key);\n  }\n  /*! \\return begin iterator */\n  iterator begin() {\n    return data_.begin();\n  }\n  /*! \\return const begin iterator */\n  const_iterator begin() const {\n    return data_.begin();\n  }\n  /*! \\return end iterator */\n  iterator end() {\n    return data_.end();\n  }\n  /*! \\return end iterator */\n  const_iterator end() const {\n    return data_.end();\n  }\n  /*!\n   * \\brief Index value associated with a key\n   * \\param key The indexing key\n   * \\return The iterator of the entry associated with the key, end iterator if not exists\n   */\n  const_iterator find(const key_type& key) const {\n    return data_.find(key);\n  }\n  /*!\n   * \\brief Index value associated with a key\n   * \\param key The indexing key\n   * \\return The iterator of the entry associated with the key, end iterator if not exists\n   */\n  iterator find(const key_type& key) {\n    return data_.find(key);\n  }\n  /*!\n   * \\brief Erase the entry associated with the iterator\n   * \\param position The iterator\n   */\n  void erase(const iterator& position) {\n    data_.erase(position);\n  }\n  /*!\n   * \\brief Erase the entry associated with the key, do nothing if not exists\n   * \\param key The indexing key\n   */\n  void erase(const key_type& key) {\n    data_.erase(key);\n  }\n  /*!\n   * \\brief Create an empty container\n   * \\return The object created\n   */\n  static ObjectPtr<MapObj> Empty() {\n    return make_object<MapObj>();\n  }\n\n protected:\n  /*!\n   * \\brief Create the map using contents from the given iterators.\n   * \\param first Begin of iterator\n   * \\param last End of iterator\n   * \\tparam IterType The type of iterator\n   * \\return ObjectPtr to the map created\n   */\n  template <typename IterType>\n  static ObjectPtr<Object> CreateFromRange(IterType first, IterType last) {\n    ObjectPtr<MapObj> p = make_object<MapObj>();\n    p->data_            = ContainerType(first, last);\n    return p;\n  }\n  /*!\n   * \\brief InsertMaybeReHash an entry into the given hash map\n   * \\param kv The entry to be inserted\n   * \\param map The pointer to the map, can be changed if re-hashing happens\n   */\n  static void InsertMaybeReHash(const KVType& kv, ObjectPtr<Object>* map) {\n    MapObj* map_node          = static_cast<MapObj*>(map->get());\n    map_node->data_[kv.first] = kv.second;\n  }\n  /*!\n   * \\brief Create an empty container with elements copying from another MapObj\n   * \\param from The source container\n   * \\return The object created\n   */\n  static ObjectPtr<MapObj> CopyFrom(MapObj* from) {\n    ObjectPtr<MapObj> p = make_object<MapObj>();\n    p->data_            = ContainerType(from->data_.begin(), from->data_.end());\n    return p;\n  }\n  /*! \\brief The real container storing data */\n  ContainerType data_;\n  template <typename, typename, typename, typename>\n  friend class Map;\n};\n\n/*!\n * \\brief Map container of NodeRef->NodeRef in DSL graph.\n *  Map implements copy on write semantics, which means map is mutable\n *  but copy will happen when array is referenced in more than two places.\n *\n * operator[] only provide const acces, use Set to mutate the content.\n * \\tparam K The key NodeRef type.\n * \\tparam V The value NodeRef type.\n */\ntemplate <typename K,\n          typename V,\n          typename = typename std::enable_if<std::is_base_of<ObjectRef, K>::value>::type,\n          typename = typename std::enable_if<std::is_base_of<ObjectRef, V>::value>::type>\nclass Map : public ObjectRef {\n public:\n  using key_type    = K;\n  using mapped_type = V;\n  class iterator;\n  /*!\n   * \\brief default constructor\n   */\n  Map() {\n    data_ = MapObj::Empty();\n  }\n  /*!\n   * \\brief move constructor\n   * \\param other source\n   */\n  Map(Map<K, V>&& other) {\n    data_ = std::move(other.data_);\n  }\n  /*!\n   * \\brief copy constructor\n   * \\param other source\n   */\n  Map(const Map<K, V>& other) : ObjectRef(other.data_) {}\n  /*!\n   * \\brief copy assign operator\n   * \\param other The source of assignment\n   * \\return reference to self.\n   */\n  Map<K, V>& operator=(Map<K, V>&& other) {\n    data_ = std::move(other.data_);\n    return *this;\n  }\n  /*!\n   * \\brief move assign operator\n   * \\param other The source of assignment\n   * \\return reference to self.\n   */\n  Map<K, V>& operator=(const Map<K, V>& other) {\n    data_ = other.data_;\n    return *this;\n  }\n  /*!\n   * \\brief constructor from pointer\n   * \\param n the container pointer\n   */\n  explicit Map(ObjectPtr<Object> n) : ObjectRef(n) {}\n  /*!\n   * \\brief constructor from iterator\n   * \\param begin begin of iterator\n   * \\param end end of iterator\n   * \\tparam IterType The type of iterator\n   */\n  template <typename IterType>\n  Map(IterType begin, IterType end) {\n    data_ = MapObj::CreateFromRange(begin, end);\n  }\n  /*!\n   * \\brief constructor from initializer list\n   * \\param init The initalizer list\n   */\n  Map(std::initializer_list<std::pair<K, V>> init) {\n    data_ = MapObj::CreateFromRange(init.begin(), init.end());\n  }\n  /*!\n   * \\brief constructor from unordered_map\n   * \\param init The unordered_map\n   */\n  template <typename Hash, typename Equal>\n  Map(const std::unordered_map<K, V, Hash, Equal>& init) {  // NOLINT(*)\n    data_ = MapObj::CreateFromRange(init.begin(), init.end());\n  }\n  /*!\n   * \\brief Read element from map.\n   * \\param key The key\n   * \\return the corresonding element.\n   */\n  const V at(const K& key) const {\n    return DowncastNoCheck<V>(GetMapObj()->at(key));\n  }\n  /*!\n   * \\brief Read element from map.\n   * \\param key The key\n   * \\return the corresonding element.\n   */\n  const V operator[](const K& key) const {\n    return this->at(key);\n  }\n  /*! \\return The size of the array */\n  size_t size() const {\n    MapObj* n = GetMapObj();\n    return n == nullptr ? 0 : n->size();\n  }\n  /*! \\return The number of elements of the key */\n  size_t count(const K& key) const {\n    MapObj* n = GetMapObj();\n    return n == nullptr ? 0 : GetMapObj()->count(key);\n  }\n  /*! \\return whether array is empty */\n  bool empty() const {\n    return size() == 0;\n  }\n  /*!\n   * \\brief set the Map.\n   * \\param key The index key.\n   * \\param value The value to be setted.\n   */\n  void Set(const K& key, const V& value) {\n    CopyOnWrite();\n    MapObj::InsertMaybeReHash(MapObj::KVType(key, value), &data_);\n  }\n  /*! \\return begin iterator */\n  iterator begin() const {\n    return iterator(GetMapObj()->begin());\n  }\n  /*! \\return end iterator */\n  iterator end() const {\n    return iterator(GetMapObj()->end());\n  }\n  /*! \\return find the key and returns the associated iterator */\n  iterator find(const K& key) const {\n    return iterator(GetMapObj()->find(key));\n  }\n\n  void erase(const K& key) {\n    CopyOnWrite()->erase(key);\n  }\n\n  /*!\n   * \\brief copy on write semantics\n   *  Do nothing if current handle is the unique copy of the array.\n   *  Otherwise make a new copy of the array to ensure the current handle\n   *  hold a unique copy.\n   *\n   * \\return Handle to the internal node container(which ganrantees to be unique)\n   */\n  MapObj* CopyOnWrite() {\n    if (data_.get() == nullptr) {\n      data_ = MapObj::Empty();\n    } else if (!data_.unique()) {\n      data_ = MapObj::CopyFrom(GetMapObj());\n    }\n    return GetMapObj();\n  }\n  /*! \\brief specify container node */\n  using ContainerType = MapObj;\n\n  /*! \\brief Iterator of the hash map */\n  class iterator {\n   public:\n    using iterator_category = std::bidirectional_iterator_tag;\n    using difference_type   = int64_t;\n    using value_type        = const std::pair<K, V>;\n    using pointer           = value_type*;\n    using reference         = value_type;\n\n    iterator() : itr() {}\n\n    /*! \\brief Compare iterators */\n    bool operator==(const iterator& other) const {\n      return itr == other.itr;\n    }\n    /*! \\brief Compare iterators */\n    bool operator!=(const iterator& other) const {\n      return itr != other.itr;\n    }\n    /*! \\brief De-reference iterators is not allowed */\n    pointer operator->() const = delete;\n    /*! \\brief De-reference iterators */\n    reference operator*() const {\n      auto& kv = *itr;\n      return std::make_pair(DowncastNoCheck<K>(kv.first), DowncastNoCheck<V>(kv.second));\n    }\n    /*! \\brief Prefix self increment, e.g. ++iter */\n    iterator& operator++() {\n      ++itr;\n      return *this;\n    }\n    /*! \\brief Suffix self increment */\n    iterator operator++(int) {\n      iterator copy = *this;\n      ++(*this);\n      return copy;\n    }\n\n   private:\n    iterator(const MapObj::iterator& itr)  // NOLINT(*)\n        : itr(itr) {}\n\n    template <typename, typename, typename, typename>\n    friend class Map;\n\n    MapObj::iterator itr;\n  };\n\n private:\n  /*! \\brief Return data_ as type of pointer of MapObj */\n  MapObj* GetMapObj() const {\n    return static_cast<MapObj*>(data_.get());\n  }\n};\n\n/*!\n * \\brief Merge two Maps.\n * \\param lhs the first Map to merge.\n * \\param rhs the second Map to merge.\n * @return The merged Array. Original Maps are kept unchanged.\n */\ntemplate <typename K,\n          typename V,\n          typename = typename std::enable_if<std::is_base_of<ObjectRef, K>::value>::type,\n          typename = typename std::enable_if<std::is_base_of<ObjectRef, V>::value>::type>\ninline Map<K, V> Merge(Map<K, V> lhs, const Map<K, V>& rhs) {\n  for (const auto& p : rhs) {\n    lhs.Set(p.first, p.second);\n  }\n  return std::move(lhs);\n}\n\n/*! \\brief An object representing string. It's POD type. */\nclass StringObj : public Object {\n public:\n  /*! \\brief The pointer to string data. */\n  const char* data;\n\n  /*! \\brief The length of the string object. */\n  uint64_t size;\n\n  static constexpr const uint32_t _type_index = TypeIndex::kMXNetString;\n  static constexpr const char* _type_key      = \"MXNet.String\";\n  MXNET_DECLARE_FINAL_OBJECT_INFO(StringObj, Object);\n\n private:\n  /*! \\brief String object which is moved from std::string container. */\n  class FromStd;\n\n  friend class String;\n};\n\n/*!\n * \\brief Reference to string objects.\n *\n * \\code\n *\n * // Example to create runtime String reference object from std::string\n * std::string s = \"hello world\";\n *\n * // You can create the reference from existing std::string\n * String ref{std::move(s)};\n *\n * // You can rebind the reference to another string.\n * ref = std::string{\"hello world2\"};\n *\n * // You can use the reference as hash map key\n * std::unordered_map<String, int32_t> m;\n * m[ref] = 1;\n *\n * // You can compare the reference object with other string objects\n * assert(ref == \"hello world\", true);\n *\n * // You can convert the reference to std::string again\n * string s2 = (string)ref;\n *\n * \\endcode\n */\nclass String : public ObjectRef {\n public:\n  /*!\n   * \\brief Construct an empty string.\n   */\n  String() : String(std::string()) {}\n  /*!\n   * \\brief Construct a new String object\n   *\n   * \\param other The moved/copied std::string object\n   *\n   * \\note If user passes const reference, it will trigger copy. If it's rvalue,\n   * it will be moved into other.\n   */\n  String(std::string other);  // NOLINT(*)\n\n  /*!\n   * \\brief Construct a new String object\n   *\n   * \\param other a char array.\n   */\n  String(const char* other)  // NOLINT(*)\n      : String(std::string(other)) {}\n\n  /*!\n   * \\brief Change the value the reference object points to.\n   *\n   * \\param other The value for the new String\n   *\n   */\n  inline String& operator=(std::string other);\n\n  /*!\n   * \\brief Change the value the reference object points to.\n   *\n   * \\param other The value for the new String\n   */\n  inline String& operator=(const char* other);\n\n  /*!\n   * \\brief Compares this String object to other\n   *\n   * \\param other The String to compare with.\n   *\n   * \\return zero if both char sequences compare equal. negative if this appear\n   * before other, positive otherwise.\n   */\n  int compare(const String& other) const {\n    return memncmp(data(), other.data(), size(), other.size());\n  }\n\n  /*!\n   * \\brief Compares this String object to other\n   *\n   * \\param other The string to compare with.\n   *\n   * \\return zero if both char sequences compare equal. negative if this appear\n   * before other, positive otherwise.\n   */\n  int compare(const std::string& other) const {\n    return memncmp(data(), other.data(), size(), other.size());\n  }\n\n  /*!\n   * \\brief Compares this to other\n   *\n   * \\param other The character array to compare with.\n   *\n   * \\return zero if both char sequences compare equal. negative if this appear\n   * before other, positive otherwise.\n   */\n  int compare(const char* other) const {\n    return memncmp(data(), other, size(), std::stold(other));\n  }\n\n  /*!\n   * \\brief Returns a pointer to the char array in the string.\n   *\n   * \\return const char*\n   */\n  const char* c_str() const {\n    return get()->data;\n  }\n\n  /*!\n   * \\brief Return the length of the string\n   *\n   * \\return size_t string length\n   */\n  size_t size() const {\n    const auto* ptr = get();\n    return ptr->size;\n  }\n\n  /*!\n   * \\brief Return the length of the string\n   *\n   * \\return size_t string length\n   */\n  size_t length() const {\n    return size();\n  }\n\n  /*!\n   * \\brief Retun if the string is empty\n   *\n   * \\return true if empty, false otherwise.\n   */\n  bool empty() const {\n    return size() == 0;\n  }\n\n  /*!\n   * \\brief Return the data pointer\n   *\n   * \\return const char* data pointer\n   */\n  const char* data() const {\n    return get()->data;\n  }\n\n  /*!\n   * \\brief Convert String to an std::string object\n   *\n   * \\return std::string\n   */\n  operator std::string() const {\n    return std::string{get()->data, size()};\n  }\n\n  /*!\n   * \\brief Check if a MXNetArgValue can be converted to String, i.e. it can be std::string or\n   * String \\param val The value to be checked \\return A boolean indicating if val can be converted\n   * to String\n   */\n  inline static bool CanConvertFrom(const MXNetArgValue& val);\n\n  /*!\n   * \\brief Hash the binary bytes\n   * \\param data The data pointer\n   * \\param size The size of the bytes.\n   * \\return the hash value.\n   */\n  static size_t HashBytes(const char* data, size_t size) {\n    // This function falls back to string copy with c++11 compiler and is\n    // recommended to be compiled with c++14\n    return std::hash<std::string_view>()(std::string_view(data, size));\n  }\n\n  MXNET_DEFINE_NOTNULLABLE_OBJECT_REF_METHODS(String, ObjectRef, StringObj);\n\n private:\n  /*!\n   * \\brief Compare two char sequence\n   *\n   * \\param lhs Pointers to the char array to compare\n   * \\param rhs Pointers to the char array to compare\n   * \\param lhs_count Length of the char array to compare\n   * \\param rhs_count Length of the char array to compare\n   * \\return int zero if both char sequences compare equal. negative if this\n   * appear before other, positive otherwise.\n   */\n  static int memncmp(const char* lhs, const char* rhs, size_t lhs_count, size_t rhs_count);\n\n  /*!\n   * \\brief Concatenate two char sequences\n   *\n   * \\param lhs Pointers to the lhs char array\n   * \\param lhs_size The size of the lhs char array\n   * \\param rhs Pointers to the rhs char array\n   * \\param rhs_size The size of the rhs char array\n   *\n   * \\return The concatenated char sequence\n   */\n  static String Concat(const char* lhs, size_t lhs_size, const char* rhs, size_t rhs_size) {\n    std::string ret(lhs, lhs_size);\n    ret.append(rhs, rhs_size);\n    return String(ret);\n  }\n\n  // Overload + operator\n  friend String operator+(const String& lhs, const String& rhs);\n  friend String operator+(const String& lhs, const std::string& rhs);\n  friend String operator+(const std::string& lhs, const String& rhs);\n  friend String operator+(const String& lhs, const char* rhs);\n  friend String operator+(const char* lhs, const String& rhs);\n\n  friend struct mxnet::runtime::ObjectRefEqual;\n};\n\n/*! \\brief An object representing string moved from std::string. */\nclass StringObj::FromStd : public StringObj {\n public:\n  /*!\n   * \\brief Construct a new FromStd object\n   *\n   * \\param other The moved/copied std::string object\n   *\n   * \\note If user passes const reference, it will trigger copy. If it's rvalue,\n   * it will be moved into other.\n   */\n  explicit FromStd(std::string other) : data_container{other} {}\n\n private:\n  /*! \\brief Container that holds the memory. */\n  std::string data_container;\n\n  friend class String;\n};\n\ninline String::String(std::string other) {\n  auto ptr  = make_object<StringObj::FromStd>(std::move(other));\n  ptr->size = ptr->data_container.size();\n  ptr->data = ptr->data_container.data();\n  data_     = std::move(ptr);\n}\n\ninline String& String::operator=(std::string other) {\n  String replace{std::move(other)};\n  data_.swap(replace.data_);\n  return *this;\n}\n\ninline String& String::operator=(const char* other) {\n  return operator=(std::string(other));\n}\n\ninline String operator+(const String& lhs, const String& rhs) {\n  size_t lhs_size = lhs.size();\n  size_t rhs_size = rhs.size();\n  return String::Concat(lhs.data(), lhs_size, rhs.data(), rhs_size);\n}\n\ninline String operator+(const String& lhs, const std::string& rhs) {\n  size_t lhs_size = lhs.size();\n  size_t rhs_size = rhs.size();\n  return String::Concat(lhs.data(), lhs_size, rhs.data(), rhs_size);\n}\n\ninline String operator+(const std::string& lhs, const String& rhs) {\n  size_t lhs_size = lhs.size();\n  size_t rhs_size = rhs.size();\n  return String::Concat(lhs.data(), lhs_size, rhs.data(), rhs_size);\n}\n\ninline String operator+(const char* lhs, const String& rhs) {\n  size_t lhs_size = std::stold(lhs);\n  size_t rhs_size = rhs.size();\n  return String::Concat(lhs, lhs_size, rhs.data(), rhs_size);\n}\n\ninline String operator+(const String& lhs, const char* rhs) {\n  size_t lhs_size = lhs.size();\n  size_t rhs_size = std::stold(rhs);\n  return String::Concat(lhs.data(), lhs_size, rhs, rhs_size);\n}\n\n// Overload < operator\ninline bool operator<(const String& lhs, const std::string& rhs) {\n  return lhs.compare(rhs) < 0;\n}\n\ninline bool operator<(const std::string& lhs, const String& rhs) {\n  return rhs.compare(lhs) > 0;\n}\n\ninline bool operator<(const String& lhs, const String& rhs) {\n  return lhs.compare(rhs) < 0;\n}\n\ninline bool operator<(const String& lhs, const char* rhs) {\n  return lhs.compare(rhs) < 0;\n}\n\ninline bool operator<(const char* lhs, const String& rhs) {\n  return rhs.compare(lhs) > 0;\n}\n\n// Overload > operator\ninline bool operator>(const String& lhs, const std::string& rhs) {\n  return lhs.compare(rhs) > 0;\n}\n\ninline bool operator>(const std::string& lhs, const String& rhs) {\n  return rhs.compare(lhs) < 0;\n}\n\ninline bool operator>(const String& lhs, const String& rhs) {\n  return lhs.compare(rhs) > 0;\n}\n\ninline bool operator>(const String& lhs, const char* rhs) {\n  return lhs.compare(rhs) > 0;\n}\n\ninline bool operator>(const char* lhs, const String& rhs) {\n  return rhs.compare(lhs) < 0;\n}\n\n// Overload <= operator\ninline bool operator<=(const String& lhs, const std::string& rhs) {\n  return lhs.compare(rhs) <= 0;\n}\n\ninline bool operator<=(const std::string& lhs, const String& rhs) {\n  return rhs.compare(lhs) >= 0;\n}\n\ninline bool operator<=(const String& lhs, const String& rhs) {\n  return lhs.compare(rhs) <= 0;\n}\n\ninline bool operator<=(const String& lhs, const char* rhs) {\n  return lhs.compare(rhs) <= 0;\n}\n\ninline bool operator<=(const char* lhs, const String& rhs) {\n  return rhs.compare(lhs) >= 0;\n}\n\n// Overload >= operator\ninline bool operator>=(const String& lhs, const std::string& rhs) {\n  return lhs.compare(rhs) >= 0;\n}\n\ninline bool operator>=(const std::string& lhs, const String& rhs) {\n  return rhs.compare(lhs) <= 0;\n}\n\ninline bool operator>=(const String& lhs, const String& rhs) {\n  return lhs.compare(rhs) >= 0;\n}\n\ninline bool operator>=(const String& lhs, const char* rhs) {\n  return lhs.compare(rhs) >= 0;\n}\n\ninline bool operator>=(const char* lhs, const String& rhs) {\n  return rhs.compare(rhs) <= 0;\n}\n\n// Overload == operator\ninline bool operator==(const String& lhs, const std::string& rhs) {\n  return lhs.compare(rhs) == 0;\n}\n\ninline bool operator==(const std::string& lhs, const String& rhs) {\n  return rhs.compare(lhs) == 0;\n}\n\ninline bool operator==(const String& lhs, const String& rhs) {\n  return lhs.compare(rhs) == 0;\n}\n\ninline bool operator==(const String& lhs, const char* rhs) {\n  return lhs.compare(rhs) == 0;\n}\n\ninline bool operator==(const char* lhs, const String& rhs) {\n  return rhs.compare(lhs) == 0;\n}\n\n// Overload != operator\ninline bool operator!=(const String& lhs, const std::string& rhs) {\n  return lhs.compare(rhs) != 0;\n}\n\ninline bool operator!=(const std::string& lhs, const String& rhs) {\n  return rhs.compare(lhs) != 0;\n}\n\ninline bool operator!=(const String& lhs, const String& rhs) {\n  return lhs.compare(rhs) != 0;\n}\n\ninline bool operator!=(const String& lhs, const char* rhs) {\n  return lhs.compare(rhs) != 0;\n}\n\ninline bool operator!=(const char* lhs, const String& rhs) {\n  return rhs.compare(lhs) != 0;\n}\n\ninline std::ostream& operator<<(std::ostream& out, const String& input) {\n  out.write(input.data(), input.size());\n  return out;\n}\n\ninline int String::memncmp(const char* lhs, const char* rhs, size_t lhs_count, size_t rhs_count) {\n  if (lhs == rhs && lhs_count == rhs_count)\n    return 0;\n\n  for (size_t i = 0; i < lhs_count && i < rhs_count; ++i) {\n    if (lhs[i] < rhs[i])\n      return -1;\n    if (lhs[i] > rhs[i])\n      return 1;\n  }\n  if (lhs_count < rhs_count) {\n    return -1;\n  } else if (lhs_count > rhs_count) {\n    return 1;\n  } else {\n    return 0;\n  }\n}\n\ninline size_t ObjectRefHash::operator()(const ObjectRef& a) const {\n  if (const auto* str = a.as<StringObj>()) {\n    return String::HashBytes(str->data, str->size);\n  }\n  return ObjectHash()(a);\n}\n\ninline bool ObjectRefEqual::operator()(const ObjectRef& a, const ObjectRef& b) const {\n  if (a.same_as(b)) {\n    return true;\n  }\n  if (const auto* str_a = a.as<StringObj>()) {\n    if (const auto* str_b = b.as<StringObj>()) {\n      return String::memncmp(str_a->data, str_b->data, str_a->size, str_b->size) == 0;\n    }\n  }\n  return false;\n}\n\n}  // namespace runtime\n}  // namespace mxnet\n\n#endif  // MXNET_RUNTIME_CONTAINER_EXT_H_\n"
  },
  {
    "path": "include/mxnet/runtime/data_type.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * \\file data_type.h\n * \\brief Primitive runtime data type.\n */\n// Acknowledgement: This file originates from incubator-tvm\n// Acknowledgement: MXNetDataType structure design originates from Halide.\n#ifndef MXNET_RUNTIME_DATA_TYPE_H_\n#define MXNET_RUNTIME_DATA_TYPE_H_\n\n#include <mxnet/runtime/c_runtime_api.h>\n#include <dmlc/logging.h>\n#include <type_traits>\n\nnamespace mxnet {\nnamespace runtime {\n/*!\n * \\brief Runtime primitive data type.\n *\n *  This class is a thin wrapper of DLDataType.\n *  We also make use of MXNetDataType in compiler to store quick hint\n */\nclass MXNetDataType {\n public:\n  /*! \\brief Type code for the MXNetDataType. */\n  enum TypeCode {\n    kInt    = kDLInt,\n    kUInt   = kDLUInt,\n    kFloat  = kDLFloat,\n    kHandle = MXNetTypeCode::kHandle,\n  };\n  /*! \\brief default constructor */\n  MXNetDataType() {}\n  /*!\n   * \\brief Constructor\n   * \\param dtype The DLDataType\n   */\n  explicit MXNetDataType(DLDataType dtype) : data_(dtype) {}\n  /*!\n   * \\brief Constructor\n   * \\param code The type code.\n   * \\param bits The number of bits in the type.\n   * \\param lanes The number of lanes.\n   */\n  MXNetDataType(int code, int bits, int lanes) {\n    data_.code  = static_cast<uint8_t>(code);\n    data_.bits  = static_cast<uint8_t>(bits);\n    data_.lanes = static_cast<uint16_t>(lanes);\n  }\n  /*! \\return The type code. */\n  int code() const {\n    return static_cast<int>(data_.code);\n  }\n  /*! \\return number of bits in the data. */\n  int bits() const {\n    return static_cast<int>(data_.bits);\n  }\n  /*! \\return number of bytes to store each scalar. */\n  int bytes() const {\n    return (bits() + 7) / 8;\n  }\n  /*! \\return number of lanes in the data. */\n  int lanes() const {\n    return static_cast<int>(data_.lanes);\n  }\n  /*! \\return whether type is a scalar type. */\n  bool is_scalar() const {\n    return lanes() == 1;\n  }\n  /*! \\return whether type is a scalar type. */\n  bool is_bool() const {\n    return code() == MXNetDataType::kUInt && bits() == 1;\n  }\n  /*! \\return whether type is a float type. */\n  bool is_float() const {\n    return code() == MXNetDataType::kFloat;\n  }\n  /*! \\return whether type is an int type. */\n  bool is_int() const {\n    return code() == MXNetDataType::kInt;\n  }\n  /*! \\return whether type is an uint type. */\n  bool is_uint() const {\n    return code() == MXNetDataType::kUInt;\n  }\n  /*! \\return whether type is a handle type. */\n  bool is_handle() const {\n    return code() == MXNetDataType::kHandle;\n  }\n  /*! \\return whether type is a vector type. */\n  bool is_vector() const {\n    return lanes() > 1;\n  }\n  /*!\n   * \\brief Create a new data type by change lanes to a specified value.\n   * \\param lanes The target number of lanes.\n   * \\return the result type.\n   */\n  MXNetDataType with_lanes(int lanes) const {\n    return MXNetDataType(data_.code, data_.bits, lanes);\n  }\n  /*!\n   * \\brief Create a new data type by change bits to a specified value.\n   * \\param bits The target number of bits.\n   * \\return the result type.\n   */\n  MXNetDataType with_bits(int bits) const {\n    return MXNetDataType(data_.code, bits, data_.lanes);\n  }\n  /*!\n   * \\brief Get the scalar version of the type.\n   * \\return the result type.\n   */\n  MXNetDataType element_of() const {\n    return with_lanes(1);\n  }\n  /*!\n   * \\brief Equal comparator.\n   * \\param other The data type to compre against.\n   * \\return The comparison resilt.\n   */\n  bool operator==(const MXNetDataType& other) const {\n    return data_.code == other.data_.code && data_.bits == other.data_.bits &&\n           data_.lanes == other.data_.lanes;\n  }\n  /*!\n   * \\brief NotEqual comparator.\n   * \\param other The data type to compre against.\n   * \\return The comparison resilt.\n   */\n  bool operator!=(const MXNetDataType& other) const {\n    return !operator==(other);\n  }\n  /*!\n   * \\brief Converter to DLDataType\n   * \\return the result.\n   */\n  operator DLDataType() const {\n    return data_;\n  }\n\n  /*!\n   * \\brief Construct an int type.\n   * \\param bits The number of bits in the type.\n   * \\param lanes The number of lanes.\n   * \\return The constructed data type.\n   */\n  static MXNetDataType Int(int bits, int lanes = 1) {\n    return MXNetDataType(kDLInt, bits, lanes);\n  }\n  /*!\n   * \\brief Construct an uint type.\n   * \\param bits The number of bits in the type.\n   * \\param lanes The number of lanes\n   * \\return The constructed data type.\n   */\n  static MXNetDataType UInt(int bits, int lanes = 1) {\n    return MXNetDataType(kDLUInt, bits, lanes);\n  }\n  /*!\n   * \\brief Construct an uint type.\n   * \\param bits The number of bits in the type.\n   * \\param lanes The number of lanes\n   * \\return The constructed data type.\n   */\n  static MXNetDataType Float(int bits, int lanes = 1) {\n    return MXNetDataType(kDLFloat, bits, lanes);\n  }\n  /*!\n   * \\brief Construct a bool type.\n   * \\param lanes The number of lanes\n   * \\return The constructed data type.\n   */\n  static MXNetDataType Bool(int lanes = 1) {\n    return MXNetDataType::UInt(1, lanes);\n  }\n  /*!\n   * \\brief Construct a handle type.\n   * \\param bits The number of bits in the type.\n   * \\param lanes The number of lanes\n   * \\return The constructed data type.\n   */\n  static MXNetDataType Handle(int bits = 64, int lanes = 1) {\n    return MXNetDataType(kHandle, bits, lanes);\n  }\n\n private:\n  DLDataType data_;\n};\n\n}  // namespace runtime\n\nusing MXNetDataType = runtime::MXNetDataType;\n\n}  // namespace mxnet\n#endif  //  MXNET_RUNTIME_DATA_TYPE_H_\n"
  },
  {
    "path": "include/mxnet/runtime/ffi_helper.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file ffi_helper\n * \\brief Helper class to support additional objects in FFI.\n */\n// Acknowledgement: This file originates from incubator-tvm\n#ifndef MXNET_RUNTIME_FFI_HELPER_H_\n#define MXNET_RUNTIME_FFI_HELPER_H_\n\n#include <mxnet/runtime/object.h>\n#include <mxnet/runtime/container.h>\n#include <mxnet/runtime/memory.h>\n#include <limits>\n\nnamespace mxnet {\nnamespace runtime {\n\n/*! \\brief Ellipsis. */\nclass EllipsisObj : public Object {\n public:\n  static constexpr const uint32_t _type_index = TypeIndex::kEllipsis;\n  static constexpr const char* _type_key      = \"MXNet.Ellipsis\";\n  MXNET_DECLARE_FINAL_OBJECT_INFO(EllipsisObj, Object)\n};\n\ninline ObjectRef CreateEllipsis() {\n  return ObjectRef(make_object<EllipsisObj>());\n}\n\n/*! \\brief Slice. */\nclass SliceObj : public Object {\n public:\n  int64_t start;\n  int64_t stop;\n  int64_t step;\n\n  static constexpr const uint32_t _type_index = TypeIndex::kSlice;\n  static constexpr const char* _type_key      = \"MXNet.Slice\";\n  MXNET_DECLARE_FINAL_OBJECT_INFO(SliceObj, Object)\n};\n\nclass Slice : public ObjectRef {\n public:\n  explicit inline Slice(int64_t start,\n                        int64_t stop,\n                        int64_t step,\n                        ObjectPtr<SliceObj>&& data = make_object<SliceObj>()) {\n    data->start = start;\n    data->stop  = stop;\n    data->step  = step;\n    data_       = std::move(data);\n  }\n\n  explicit inline Slice(int64_t stop) : Slice(kNoneValue, stop, kNoneValue) {}\n\n  // constant to represent None.\n  static constexpr int64_t kNoneValue = std::numeric_limits<int64_t>::min();\n\n  MXNET_DEFINE_OBJECT_REF_METHODS(Slice, ObjectRef, SliceObj)\n};\n\nint64_t inline SliceNoneValue() {\n  return Slice::kNoneValue;\n}\n\nclass IntegerObj : public Object {\n public:\n  int64_t value;\n  static constexpr const uint32_t _type_index = TypeIndex::kInteger;\n  static constexpr const char* _type_key      = \"MXNet.Integer\";\n  MXNET_DECLARE_FINAL_OBJECT_INFO(IntegerObj, Object)\n};\n\nclass Integer : public ObjectRef {\n public:\n  explicit Integer(int64_t value, ObjectPtr<IntegerObj>&& data = make_object<IntegerObj>()) {\n    data->value = value;\n    data_       = std::move(data);\n  }\n  MXNET_DEFINE_OBJECT_REF_METHODS(Integer, ObjectRef, IntegerObj)\n};\n\nclass FloatObj : public Object {\n public:\n  double value;\n  static constexpr const uint32_t _type_index = TypeIndex::kFloat;\n  static constexpr const char* _type_key      = \"MXNet.Float\";\n  MXNET_DECLARE_FINAL_OBJECT_INFO(FloatObj, Object)\n};\n\nclass Float : public ObjectRef {\n public:\n  explicit Float(double value, ObjectPtr<FloatObj>&& data = make_object<FloatObj>()) {\n    data->value = value;\n    data_       = std::move(data);\n  }\n  MXNET_DEFINE_OBJECT_REF_METHODS(Float, ObjectRef, FloatObj)\n};\n\n//  Helper functions for fast FFI implementations\n/*!\n * \\brief A builder class that helps to incrementally build ADT.\n */\nclass ADTBuilder {\n public:\n  /*! \\brief default constructor */\n  ADTBuilder() = default;\n\n  explicit inline ADTBuilder(uint32_t tag, uint32_t size)\n      : data_(make_inplace_array_object<ADTObj, ObjectRef>(size)) {\n    data_->size = size;\n  }\n\n  template <typename... Args>\n  void inline EmplaceInit(size_t idx, Args&&... args) {\n    data_->EmplaceInit(idx, std::forward<Args>(args)...);\n  }\n\n  ADT inline Get() {\n    return ADT(std::move(data_));\n  }\n\n private:\n  friend class ADT;\n  ObjectPtr<ADTObj> data_;\n};\n}  // namespace runtime\n}  // namespace mxnet\n#endif  // MXNET_RUNTIME_FFI_HELPER_H_\n"
  },
  {
    "path": "include/mxnet/runtime/memory.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*!\n * \\file runtime/memory.h\n * \\brief Runtime memory management.\n */\n// Acknowledgement: This file originates from incubator-tvm\n#ifndef MXNET_RUNTIME_MEMORY_H_\n#define MXNET_RUNTIME_MEMORY_H_\n\n#include <cstdlib>\n#include <utility>\n#include <type_traits>\n#include \"object.h\"\n\nnamespace mxnet {\nnamespace runtime {\n/*!\n * \\brief Allocate an object using default allocator.\n * \\param args arguments to the constructor.\n * \\tparam T the node type.\n * \\return The ObjectPtr to the allocated object.\n */\ntemplate <typename T, typename... Args>\ninline ObjectPtr<T> make_object(Args&&... args);\n\n// Detail implementations after this\n//\n// The current design allows swapping the\n// allocator pattern when necessary.\n//\n// Possible future allocator optimizations:\n// - Arena allocator that gives ownership of memory to arena (deleter_= nullptr)\n// - Thread-local object pools: one pool per size and alignment requirement.\n// - Can specialize by type of object to give the specific allocator to each object.\n\n/*!\n * \\brief Base class of object allocators that implements make.\n *  Use curiously recurring template pattern.\n *\n * \\tparam Derived The derived class.\n */\ntemplate <typename Derived>\nclass ObjAllocatorBase {\n public:\n  /*!\n   * \\brief Make a new object using the allocator.\n   * \\tparam T The type to be allocated.\n   * \\tparam Args The constructor signature.\n   * \\param args The arguments.\n   */\n  template <typename T, typename... Args>\n  inline ObjectPtr<T> make_object(Args&&... args) {\n    using Handler = typename Derived::template Handler<T>;\n    static_assert(std::is_base_of<Object, T>::value, \"make can only be used to create Object\");\n    T* ptr           = Handler::New(static_cast<Derived*>(this), std::forward<Args>(args)...);\n    ptr->type_index_ = T::RuntimeTypeIndex();\n    ptr->deleter_    = Handler::Deleter();\n    return ObjectPtr<T>(ptr);\n  }\n\n  /*!\n   * \\tparam ArrayType The type to be allocated.\n   * \\tparam ElemType The type of array element.\n   * \\tparam Args The constructor signature.\n   * \\param num_elems The number of array elements.\n   * \\param args The arguments.\n   */\n  template <typename ArrayType, typename ElemType, typename... Args>\n  inline ObjectPtr<ArrayType> make_inplace_array(size_t num_elems, Args&&... args) {\n    using Handler = typename Derived::template ArrayHandler<ArrayType, ElemType>;\n    static_assert(std::is_base_of<Object, ArrayType>::value,\n                  \"make_inplace_array can only be used to create Object\");\n    ArrayType* ptr =\n        Handler::New(static_cast<Derived*>(this), num_elems, std::forward<Args>(args)...);\n    ptr->type_index_ = ArrayType::RuntimeTypeIndex();\n    ptr->deleter_    = Handler::Deleter();\n    return ObjectPtr<ArrayType>(ptr);\n  }\n};\n\n// Simple allocator that uses new/delete.\nclass SimpleObjAllocator : public ObjAllocatorBase<SimpleObjAllocator> {\n public:\n  template <typename T>\n  class Handler {\n   public:\n    using StorageType = typename std::aligned_storage<sizeof(T), alignof(T)>::type;\n\n    template <typename... Args>\n    static T* New(SimpleObjAllocator*, Args&&... args) {\n      // NOTE: the first argument is not needed for SimpleObjAllocator\n      // It is reserved for special allocators that needs to recycle\n      // the object to itself (e.g. in the case of object pool).\n      //\n      // In the case of an object pool, an allocator needs to create\n      // a special chunk memory that hides reference to the allocator\n      // and call allocator's release function in the deleter.\n\n      // NOTE2: Use inplace new to allocate\n      // This is used to get rid of warning when deleting a virtual\n      // class with non-virtual destructor.\n      // We are fine here as we captured the right deleter during construction.\n      // This is also the right way to get storage type for an object pool.\n      StorageType* data = new StorageType();\n      new (data) T(std::forward<Args>(args)...);\n      return reinterpret_cast<T*>(data);\n    }\n\n    static Object::FDeleter Deleter() {\n      return Deleter_;\n    }\n\n   private:\n    static void Deleter_(Object* objptr) {\n      // NOTE: this is important to cast back to T*\n      // because objptr and tptr may not be the same\n      // depending on how sub-class allocates the space.\n      T* tptr = static_cast<T*>(objptr);\n      // It is important to do tptr->T::~T(),\n      // so that we explicitly call the specific destructor\n      // instead of tptr->~T(), which could mean the intention\n      // call a virtual destructor(which may not be available and is not required).\n      tptr->T::~T();\n      delete reinterpret_cast<StorageType*>(tptr);\n    }\n  };\n\n  // Array handler that uses new/delete.\n  template <typename ArrayType, typename ElemType>\n  class ArrayHandler {\n   public:\n    using StorageType = typename std::aligned_storage<sizeof(ArrayType), alignof(ArrayType)>::type;\n    // for now only support elements that aligns with array header.\n    static_assert(alignof(ArrayType) % alignof(ElemType) == 0 &&\n                      sizeof(ArrayType) % alignof(ElemType) == 0,\n                  \"element alignment constraint\");\n\n    template <typename... Args>\n    static ArrayType* New(SimpleObjAllocator*, size_t num_elems, Args&&... args) {\n      // NOTE: the first argument is not needed for ArrayObjAllocator\n      // It is reserved for special allocators that needs to recycle\n      // the object to itself (e.g. in the case of object pool).\n      //\n      // In the case of an object pool, an allocator needs to create\n      // a special chunk memory that hides reference to the allocator\n      // and call allocator's release function in the deleter.\n      // NOTE2: Use inplace new to allocate\n      // This is used to get rid of warning when deleting a virtual\n      // class with non-virtual destructor.\n      // We are fine here as we captured the right deleter during construction.\n      // This is also the right way to get storage type for an object pool.\n      size_t unit              = sizeof(StorageType);\n      size_t requested_size    = num_elems * sizeof(ElemType) + sizeof(ArrayType);\n      size_t num_storage_slots = (requested_size + unit - 1) / unit;\n      StorageType* data        = new StorageType[num_storage_slots];\n      new (data) ArrayType(std::forward<Args>(args)...);\n      return reinterpret_cast<ArrayType*>(data);\n    }\n\n    static Object::FDeleter Deleter() {\n      return Deleter_;\n    }\n\n   private:\n    static void Deleter_(Object* objptr) {\n      // NOTE: this is important to cast back to ArrayType*\n      // because objptr and tptr may not be the same\n      // depending on how sub-class allocates the space.\n      ArrayType* tptr = static_cast<ArrayType*>(objptr);\n      // It is important to do tptr->ArrayType::~ArrayType(),\n      // so that we explicitly call the specific destructor\n      // instead of tptr->~ArrayType(), which could mean the intention\n      // call a virtual destructor(which may not be available and is not required).\n      tptr->ArrayType::~ArrayType();\n      StorageType* p = reinterpret_cast<StorageType*>(tptr);\n      delete[] p;\n    }\n  };\n};\n\ntemplate <typename T, typename... Args>\ninline ObjectPtr<T> make_object(Args&&... args) {\n  return SimpleObjAllocator().make_object<T>(std::forward<Args>(args)...);\n}\n\ntemplate <typename ArrayType, typename ElemType, typename... Args>\ninline ObjectPtr<ArrayType> make_inplace_array_object(size_t num_elems, Args&&... args) {\n  return SimpleObjAllocator().make_inplace_array<ArrayType, ElemType>(num_elems,\n                                                                      std::forward<Args>(args)...);\n}\n\n}  // namespace runtime\n}  // namespace mxnet\n#endif  // MXNET_RUNTIME_MEMORY_H_\n"
  },
  {
    "path": "include/mxnet/runtime/ndarray.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file runtime/ndarray.h\n * \\brief A device-independent managed NDArray abstraction.\n */\n// Acknowledgement: This file originates from incubator-tvm\n#ifndef MXNET_RUNTIME_NDARRAY_H_\n#define MXNET_RUNTIME_NDARRAY_H_\n\nnamespace mxnet {\nnamespace runtime {\n\n/*!\n * \\brief The type trait indicates subclass of TVM's NDArray.\n *  For irrelavant classes, code = -1.\n *  For TVM NDArray itself, code = 0.\n *  All subclasses of NDArray should override code > 0.\n */\ntemplate <typename T>\nstruct array_type_info {\n  /*! \\brief the value of the traits */\n  static const int code = -1;\n};\n\n}  // namespace runtime\n}  // namespace mxnet\n#endif  // MXNET_RUNTIME_NDARRAY_H_\n"
  },
  {
    "path": "include/mxnet/runtime/ndarray_handle.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file ndarray_handle.h\n * \\brief NDArray handle types\n */\n#ifndef MXNET_RUNTIME_NDARRAY_HANDLE_H_\n#define MXNET_RUNTIME_NDARRAY_HANDLE_H_\n#include <mxnet/ndarray.h>\n#include <mxnet/runtime/object.h>\n\nnamespace mxnet {\n\nclass NDArrayHandleObj : public Object {\n public:\n  /*! \\brief the Internal value. */\n  NDArray value;\n\n  static constexpr const char* _type_key = \"MXNet.NDArrayHandle\";\n  MXNET_DECLARE_FINAL_OBJECT_INFO(NDArrayHandleObj, Object)\n};\n\nclass NDArrayHandle : public ObjectRef {\n public:\n  explicit NDArrayHandle(NDArray* value) {\n    runtime::ObjectPtr<NDArrayHandleObj> node = make_object<NDArrayHandleObj>();\n    node->value                               = *value;\n    data_                                     = std::move(node);\n  }\n  inline NDArray* getArray() const {\n    return static_cast<NDArray*>(&(static_cast<NDArrayHandleObj*>(data_.get())->value));\n  }\n  MXNET_DEFINE_OBJECT_REF_METHODS(NDArrayHandle, ObjectRef, NDArrayHandleObj)\n};\n\n};  // namespace mxnet\n\n#endif  // MXNET_RUNTIME_NDARRAY_HANDLE_H_\n"
  },
  {
    "path": "include/mxnet/runtime/object.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*!\n * \\file object.h\n * \\brief A managed object in MXNet runtime.\n */\n// Acknowledgement: This file originates from incubator-tvm\n#ifndef MXNET_RUNTIME_OBJECT_H_\n#define MXNET_RUNTIME_OBJECT_H_\n\n#include <dmlc/logging.h>\n#include <type_traits>\n#include <string>\n#include <utility>\n#include \"c_runtime_api.h\"\n\n/*!\n * \\brief Whether or not use atomic reference counter.\n *  If the reference counter is not atomic,\n *  an object cannot be owned by multiple threads.\n *  We can, however, move an object across threads\n */\n#ifndef MXNET_OBJECT_ATOMIC_REF_COUNTER\n#define MXNET_OBJECT_ATOMIC_REF_COUNTER 1\n#endif\n\n#if MXNET_OBJECT_ATOMIC_REF_COUNTER\n#include <atomic>\n#endif  // MXNET_OBJECT_ATOMIC_REF_COUNTER\n\nnamespace mxnet {\nnamespace runtime {\n\n/*! \\brief list of the type index. */\nenum TypeIndex {\n  /*! \\brief Root object type. */\n  kRoot         = 0,\n  kMXNetTensor  = 1,\n  kMXNetClosure = 2,\n  kMXNetADT     = 3,\n  kMXNetMap     = 4,\n  kMXNetString  = 5,\n  kEllipsis     = 6,\n  kSlice        = 7,\n  kInteger      = 8,\n  kFloat        = 9,\n  kStaticIndexEnd,\n  /*! \\brief Type index is allocated during runtime. */\n  kDynamic = kStaticIndexEnd\n};\n\n/*!\n * \\brief base class of all object containers.\n *\n * Sub-class of objects should declare the following static constexpr fields:\n *\n * - _type_index:\n *      Static type index of the object, if assigned to TypeIndex::kDynamic\n *      the type index will be assigned during runtime.\n *      Runtime type index can be accessed by ObjectType::TypeIndex();\n * - _type_key:\n *       The unique string identifier of tyep type.\n * - _type_final:\n *       Whether the type is terminal type(there is no subclass of the type in the object system).\n *       This field is automatically set by marco MXNET_DECLARE_FINAL_OBJECT_INFO\n *       It is still OK to sub-class a terminal object type T and construct it using make_object.\n *       But IsInstance check will only show that the object type is T(instead of the sub-class).\n *\n * The following two fields are necessary for base classes that can be sub-classed.\n *\n * - _type_child_slots:\n *       Number of reserved type index slots for child classes.\n *       Used for runtime optimization for type checking in IsInstance.\n *       If an object's type_index is within range of [type_index, type_index + _type_child_slots]\n *       Then the object can be quickly decided as sub-class of the current object class.\n *       If not, a fallback mechanism is used to check the global type table.\n *       Recommendation: set to estimate number of children needed.\n * - _type_child_slots_can_overflow:\n *       Whether we can add additional child classes even if the number of child classes\n *       exceeds the _type_child_slots. A fallback mechanism to check global type table will be\n * used. Recommendation: set to false for optimal runtime speed if we know exact number of children.\n *\n * Two macros are used to declare helper functions in the object:\n * - Use MXNET_DECLARE_BASE_OBJECT_INFO for object classes that can be sub-classed.\n * - Use MXNET_DECLARE_FINAL_OBJECT_INFO for object classes that cannot be sub-classed.\n *\n * New objects can be created using make_object function.\n * Which will automatically populate the type_index and deleter of the object.\n *\n * \\sa make_object\n * \\sa ObjectPtr\n * \\sa ObjectRef\n *\n * \\code\n *\n *  // Create a base object\n *  class BaseObj : public Object {\n *   public:\n *    // object fields\n *    int field0;\n *\n *    // object properties\n *    static constexpr const uint32_t _type_index = TypeIndex::kDynamic;\n *    static constexpr const char* _type_key = \"test.BaseObj\";\n *    MXNET_DECLARE_BASE_OBJECT_INFO(BaseObj, Object);\n *  };\n *\n *  class ObjLeaf : public ObjBase {\n *   public:\n *    // fields\n *    int child_field0;\n *    // object properties\n *    static constexpr const uint32_t _type_index = TypeIndex::kDynamic;\n *    static constexpr const char* _type_key = \"test.LeafObj\";\n *    MXNET_DECLARE_BASE_OBJECT_INFO(LeaffObj, Object);\n *  };\n *\n *  // The following code should be put into a cc file.\n *  MXNET_REGISTER_OBJECT_TYPE(ObjBase);\n *  MXNET_REGISTER_OBJECT_TYPE(ObjLeaf);\n *\n *  // Usage example.\n *  void TestObjects() {\n *    // create an object\n *    ObjectRef leaf_ref(make_object<LeafObj>());\n *    // cast to a specific instance\n *    const LeafObj* leaf_ptr = leaf_ref.as<LeafObj>();\n *    CHECK(leaf_ptr != nullptr);\n *    // can also cast to the base class.\n *    CHECK(leaf_ref.as<BaseObj>() != nullptr);\n *  }\n *\n * \\endcode\n */\nclass Object {\n public:\n  /*!\n   * \\brief Object deleter\n   * \\param self pointer to the Object.\n   */\n  typedef void (*FDeleter)(Object* self);\n  /*! \\return The internal runtime type index of the object. */\n  uint32_t type_index() const {\n    return type_index_;\n  }\n  /*!\n   * \\return the type key of the object.\n   * \\note this operation is expensive, can be used for error reporting.\n   */\n  std::string GetTypeKey() const {\n    return TypeIndex2Key(type_index_);\n  }\n  /*!\n   * \\return A hash value of the return of GetTypeKey.\n   */\n  size_t GetTypeKeyHash() const {\n    return TypeIndex2KeyHash(type_index_);\n  }\n  /*!\n   * Check if the object is an instance of TargetType.\n   * \\tparam TargetType The target type to be checked.\n   * \\return Whether the target type is true.\n   */\n  template <typename TargetType>\n  inline bool IsInstance() const;\n\n  /*!\n   * \\brief Get the type key of the corresponding index from runtime.\n   * \\param tindex The type index.\n   * \\return the result.\n   */\n  MXNET_DLL static std::string TypeIndex2Key(uint32_t tindex);\n  /*!\n   * \\brief Get the type key hash of the corresponding index from runtime.\n   * \\param tindex The type index.\n   * \\return the related key-hash.\n   */\n  MXNET_DLL static size_t TypeIndex2KeyHash(uint32_t tindex);\n  /*!\n   * \\brief Get the type index of the corresponding key from runtime.\n   * \\param key The type key.\n   * \\return the result.\n   */\n  MXNET_DLL static uint32_t TypeKey2Index(const std::string& key);\n\n#if MXNET_OBJECT_ATOMIC_REF_COUNTER\n  using RefCounterType = std::atomic<int32_t>;\n#else\n  using RefCounterType = int32_t;\n#endif\n\n  static constexpr const char* _type_key = \"Object\";\n\n  static uint32_t _GetOrAllocRuntimeTypeIndex() {\n    return TypeIndex::kRoot;\n  }\n  static uint32_t RuntimeTypeIndex() {\n    return TypeIndex::kRoot;\n  }\n\n  // Default object type properties for sub-classes\n  static constexpr bool _type_final                    = false;\n  static constexpr uint32_t _type_child_slots          = 0;\n  static constexpr bool _type_child_slots_can_overflow = true;\n  // NOTE: the following field is not type index of Object\n  // but was intended to be used by sub-classes as default value.\n  // The type index of Object is TypeIndex::kRoot\n  static constexpr uint32_t _type_index = TypeIndex::kDynamic;\n\n  // Default constructor and copy constructor\n  Object() {}\n  // Override the copy and assign constructors to do nothing.\n  // This is to make sure only contents, but not deleter and ref_counter\n  // are copied when a child class copies itself.\n  // This will enable us to use make_object<ObjectClass>(*obj_ptr)\n  // to copy an existing object.\n  Object(const Object& other) {  // NOLINT(*)\n  }\n  Object(Object&& other) {  // NOLINT(*)\n  }\n  Object& operator=(const Object& other) {  // NOLINT(*)\n    return *this;\n  }\n  Object& operator=(Object&& other) {  // NOLINT(*)\n    return *this;\n  }\n\n protected:\n  // The fields of the base object cell.\n  /*! \\brief Type index(tag) that indicates the type of the object. */\n  uint32_t type_index_{0};\n  /*! \\brief The internal reference counter */\n  RefCounterType ref_counter_{0};\n  /*!\n   * \\brief deleter of this object to enable customized allocation.\n   * If the deleter is nullptr, no deletion will be performed.\n   * The creator of the object must always set the deleter field properly.\n   */\n  FDeleter deleter_ = nullptr;\n  // Invariant checks.\n  static_assert(sizeof(int32_t) == sizeof(RefCounterType) &&\n                    alignof(int32_t) == sizeof(RefCounterType),\n                \"RefCounter ABI check.\");\n\n  /*!\n   * \\brief Get the type index using type key.\n   *\n   *  When the function is first time called for a type,\n   *  it will register the type to the type table in the runtime.\n   *  If the static_tindex is TypeIndex::kDynamic, the function will\n   *  allocate a runtime type index.\n   *  Otherwise, we will populate the type table and return the static index.\n   *\n   * \\param key the type key.\n   * \\param static_tindex The current _type_index field.\n   *                      can be TypeIndex::kDynamic.\n   * \\param parent_tindex The index of the parent.\n   * \\param type_child_slots Number of slots reserved for its children.\n   * \\param type_child_slots_can_overflow Whether to allow child to overflow the slots.\n   * \\return The allocated type index.\n   */\n  MXNET_DLL static uint32_t GetOrAllocRuntimeTypeIndex(const std::string& key,\n                                                       uint32_t static_tindex,\n                                                       uint32_t parent_tindex,\n                                                       uint32_t type_child_slots,\n                                                       bool type_child_slots_can_overflow);\n\n  // reference counter related operations\n  /*! \\brief developer function, increases reference counter. */\n  inline void IncRef();\n  /*!\n   * \\brief developer function, decrease reference counter.\n   * \\note The deleter will be called when ref_counter_ becomes zero.\n   */\n  inline void DecRef();\n\n private:\n  /*!\n   * \\return The usage count of the cell.\n   * \\note We use stl style naming to be consistent with known API in shared_ptr.\n   */\n  inline int use_count() const;\n  /*!\n   * \\brief Check of this object is derived from the parent.\n   * \\param parent_tindex The parent type index.\n   * \\return The derivation results.\n   */\n  MXNET_DLL bool DerivedFrom(uint32_t parent_tindex) const;\n  // friend classes\n  template <typename>\n  friend class ObjAllocatorBase;\n  template <typename>\n  friend class ObjectPtr;\n  friend class MXNetRetValue;\n  friend class ObjectInternal;\n};\n\n/*!\n * \\brief Get a reference type from a raw object ptr type\n *\n *  It is always important to get a reference type\n *  if we want to return a value as reference or keep\n *  the object alive beyond the scope of the function.\n *\n * \\param ptr The object pointer\n * \\tparam RefType The reference type\n * \\tparam ObjectType The object type\n * \\return The corresponding RefType\n */\ntemplate <typename RefType, typename ObjectType>\ninline RefType GetRef(const ObjectType* ptr);\n\n/*!\n * \\brief Downcast a base reference type to a more specific type.\n *\n * \\param ref The inptut reference\n * \\return The corresponding SubRef.\n * \\tparam SubRef The target specific reference type.\n * \\tparam BaseRef the current reference type.\n */\ntemplate <typename SubRef, typename BaseRef>\ninline SubRef Downcast(BaseRef ref);\n\n/*!\n * \\brief A custom smart pointer for Object.\n * \\tparam T the content data type.\n * \\sa make_object\n */\ntemplate <typename T>\nclass ObjectPtr {\n public:\n  /*! \\brief default constructor */\n  ObjectPtr() {}\n  /*! \\brief default constructor */\n  ObjectPtr(std::nullptr_t) {}  // NOLINT(*)\n  /*!\n   * \\brief copy constructor\n   * \\param other The value to be moved\n   */\n  ObjectPtr(const ObjectPtr<T>& other)  // NOLINT(*)\n      : ObjectPtr(other.data_) {}\n  /*!\n   * \\brief copy constructor\n   * \\param other The value to be moved\n   */\n  template <typename U>\n  ObjectPtr(const ObjectPtr<U>& other)  // NOLINT(*)\n      : ObjectPtr(other.data_) {\n    static_assert(std::is_base_of<T, U>::value,\n                  \"can only assign of child class ObjectPtr to parent\");\n  }\n  /*!\n   * \\brief move constructor\n   * \\param other The value to be moved\n   */\n  ObjectPtr(ObjectPtr<T>&& other)  // NOLINT(*)\n      : data_(other.data_) {\n    other.data_ = nullptr;\n  }\n  /*!\n   * \\brief move constructor\n   * \\param other The value to be moved\n   */\n  template <typename Y>\n  ObjectPtr(ObjectPtr<Y>&& other)  // NOLINT(*)\n      : data_(other.data_) {\n    static_assert(std::is_base_of<T, Y>::value,\n                  \"can only assign of child class ObjectPtr to parent\");\n    other.data_ = nullptr;\n  }\n  /*! \\brief destructor */\n  ~ObjectPtr() {\n    this->reset();\n  }\n  /*!\n   * \\brief Swap this array with another Object\n   * \\param other The other Object\n   */\n  void swap(ObjectPtr<T>& other) {  // NOLINT(*)\n    std::swap(data_, other.data_);\n  }\n  /*!\n   * \\return Get the content of the pointer\n   */\n  T* get() const {\n    return static_cast<T*>(data_);\n  }\n  /*!\n   * \\return The pointer\n   */\n  T* operator->() const {\n    return get();\n  }\n  /*!\n   * \\return The reference\n   */\n  T& operator*() const {  // NOLINT(*)\n    return *get();\n  }\n  /*!\n   * \\brief copy assignmemt\n   * \\param other The value to be assigned.\n   * \\return reference to self.\n   */\n  ObjectPtr<T>& operator=(const ObjectPtr<T>& other) {  // NOLINT(*)\n    // takes in plane operator to enable copy elison.\n    // copy-and-swap idiom\n    ObjectPtr(other).swap(*this);  // NOLINT(*)\n    return *this;\n  }\n  /*!\n   * \\brief move assignmemt\n   * \\param other The value to be assigned.\n   * \\return reference to self.\n   */\n  ObjectPtr<T>& operator=(ObjectPtr<T>&& other) {  // NOLINT(*)\n    // copy-and-swap idiom\n    ObjectPtr(std::move(other)).swap(*this);  // NOLINT(*)\n    return *this;\n  }\n  /*! \\brief reset the content of ptr to be nullptr */\n  void reset() {\n    if (data_ != nullptr) {\n      data_->DecRef();\n      data_ = nullptr;\n    }\n  }\n  /*! \\return The use count of the ptr, for debug purposes */\n  int use_count() const {\n    return data_ != nullptr ? data_->use_count() : 0;\n  }\n  /*! \\return whether the reference is unique */\n  bool unique() const {\n    return data_ != nullptr && data_->use_count() == 1;\n  }\n  /*! \\return Whether two ObjectPtr do not equal each other */\n  bool operator==(const ObjectPtr<T>& other) const {\n    return data_ == other.data_;\n  }\n  /*! \\return Whether two ObjectPtr equals each other */\n  bool operator!=(const ObjectPtr<T>& other) const {\n    return data_ != other.data_;\n  }\n  /*! \\return Whether the pointer is nullptr */\n  bool operator==(std::nullptr_t null) const {\n    return data_ == nullptr;\n  }\n  /*! \\return Whether the pointer is not nullptr */\n  bool operator!=(std::nullptr_t null) const {\n    return data_ != nullptr;\n  }\n\n private:\n  /*! \\brief internal pointer field */\n  Object* data_{nullptr};\n  /*!\n   * \\brief constructor from Object\n   * \\param data The data pointer\n   */\n  explicit ObjectPtr(Object* data) : data_(data) {\n    if (data != nullptr) {\n      data_->IncRef();\n    }\n  }\n  // friend classes\n  friend class Object;\n  friend class ObjectRef;\n  friend struct ObjectHash;\n  template <typename>\n  friend class ObjectPtr;\n  template <typename>\n  friend class ObjAllocatorBase;\n  friend class MXNetPODValue_;\n  friend class MXNetArgsSetter;\n  friend class MXNetRetValue;\n  friend class MXNetArgValue;\n  template <typename RefType, typename ObjType>\n  friend RefType GetRef(const ObjType* ptr);\n  template <typename BaseType, typename ObjType>\n  friend ObjectPtr<BaseType> GetObjectPtr(ObjType* ptr);\n};\n\n/*! \\brief Base class of all object reference */\nclass ObjectRef {\n public:\n  /*! \\brief default constructor */\n  ObjectRef() = default;\n  /*! \\brief Constructor from existing object ptr */\n  explicit ObjectRef(ObjectPtr<Object> data) : data_(data) {}\n  /*!\n   * \\brief Comparator\n   * \\param other Another object ref.\n   * \\return the compare result.\n   */\n  bool same_as(const ObjectRef& other) const {\n    return data_ == other.data_;\n  }\n  /*!\n   * \\brief Comparator\n   * \\param other Another object ref.\n   * \\return the compare result.\n   */\n  bool operator==(const ObjectRef& other) const {\n    return data_ == other.data_;\n  }\n  /*!\n   * \\brief Comparator\n   * \\param other Another object ref.\n   * \\return the compare result.\n   */\n  bool operator!=(const ObjectRef& other) const {\n    return data_ != other.data_;\n  }\n  /*!\n   * \\brief Comparator\n   * \\param other Another object ref by address.\n   * \\return the compare result.\n   */\n  bool operator<(const ObjectRef& other) const {\n    return data_.get() < other.data_.get();\n  }\n  /*! \\return whether the expression is null */\n  bool defined() const {\n    return data_ != nullptr;\n  }\n  /*! \\return the internal object pointer */\n  const Object* get() const {\n    return data_.get();\n  }\n  /*! \\return the internal object pointer */\n  const Object* operator->() const {\n    return get();\n  }\n  /*! \\return whether the reference is unique */\n  bool unique() const {\n    return data_.unique();\n  }\n  /*!\n   * \\brief Try to downcast the internal Object to a\n   *  raw pointer of a corresponding type.\n   *\n   *  The function will return a nullptr if the cast failed.\n   *\n   * if (const Add *add = node_ref.As<Add>()) {\n   *   // This is an add node\n   * }\n   * \\tparam ObjectType the target type, must be a subtype of Object/\n   */\n  template <typename ObjectType>\n  inline const ObjectType* as() const;\n\n  /*! \\brief type indicate the container type. */\n  using ContainerType = Object;\n  // Default type properties for the reference class.\n  static constexpr bool _type_is_nullable = true;\n\n protected:\n  /*! \\brief Internal pointer that backs the reference. */\n  ObjectPtr<Object> data_;\n  /*! \\return return a mutable internal ptr, can be used by sub-classes. */\n  Object* get_mutable() const {\n    return data_.get();\n  }\n  /*!\n   * \\brief Internal helper function downcast a ref without check.\n   * \\note Only used for internal dev purposes.\n   * \\tparam T The target reference type.\n   * \\return The casted result.\n   */\n  template <typename T>\n  static T DowncastNoCheck(ObjectRef ref) {\n    return T(std::move(ref.data_));\n  }\n  /*!\n   * \\brief Internal helper function get data_ as ObjectPtr of ObjectType.\n   * \\note only used for internal dev purpose.\n   * \\tparam ObjectType The corresponding object type.\n   * \\return the corresponding type.\n   */\n  template <typename ObjectType>\n  static ObjectPtr<ObjectType> GetDataPtr(const ObjectRef& ref) {\n    return ObjectPtr<ObjectType>(ref.data_.data_);\n  }\n  // friend classes.\n  friend struct ObjectHash;\n  friend class MXNetRetValue;\n  friend class MXNetArgsSetter;\n  template <typename SubRef, typename BaseRef>\n  friend SubRef Downcast(BaseRef ref);\n};\n\n/*!\n * \\brief Get an object ptr type from a raw object ptr.\n *\n * \\param ptr The object pointer\n * \\tparam BaseType The reference type\n * \\tparam ObjectType The object type\n * \\return The corresponding RefType\n */\ntemplate <typename BaseType, typename ObjectType>\ninline ObjectPtr<BaseType> GetObjectPtr(ObjectType* ptr);\n\n/*! \\brief ObjectRef hash functor */\nstruct ObjectHash {\n  size_t operator()(const ObjectRef& a) const {\n    return operator()(a.data_);\n  }\n\n  template <typename T>\n  size_t operator()(const ObjectPtr<T>& a) const {\n    return std::hash<Object*>()(a.get());\n  }\n};\n\n/*! \\brief ObjectRef equal functor */\nstruct ObjectEqual {\n  bool operator()(const ObjectRef& a, const ObjectRef& b) const {\n    return a.same_as(b);\n  }\n\n  template <typename T>\n  size_t operator()(const ObjectPtr<T>& a, const ObjectPtr<T>& b) const {\n    return a == b;\n  }\n};\n\n/*!\n * \\brief helper macro to declare a base object type that can be inheritated.\n * \\param TypeName The name of the current type.\n * \\param ParentType The name of the ParentType\n */\n#define MXNET_DECLARE_BASE_OBJECT_INFO(TypeName, ParentType)                                     \\\n  static uint32_t RuntimeTypeIndex() {                                                           \\\n    return TypeName::_type_index != ::mxnet::runtime::TypeIndex::kDynamic ?                      \\\n               TypeName::_type_index :                                                           \\\n               _GetOrAllocRuntimeTypeIndex();                                                    \\\n  }                                                                                              \\\n  static uint32_t _GetOrAllocRuntimeTypeIndex() {                                                \\\n    static uint32_t tidx = GetOrAllocRuntimeTypeIndex(TypeName::_type_key,                       \\\n                                                      TypeName::_type_index,                     \\\n                                                      ParentType::_GetOrAllocRuntimeTypeIndex(), \\\n                                                      TypeName::_type_child_slots,               \\\n                                                      TypeName::_type_child_slots_can_overflow); \\\n    return tidx;                                                                                 \\\n  }\n\n/*!\n * \\brief helper macro to declare type information in a final class.\n * \\param TypeName The name of the current type.\n * \\param ParentType The name of the ParentType\n */\n#define MXNET_DECLARE_FINAL_OBJECT_INFO(TypeName, ParentType) \\\n  static const constexpr bool _type_final      = true;        \\\n  static const constexpr int _type_child_slots = 0;           \\\n  MXNET_DECLARE_BASE_OBJECT_INFO(TypeName, ParentType)\n\n/*!\n * \\brief Helper macro to register the object type to runtime.\n *  Makes sure that the runtime type table is correctly populated.\n *\n *  Use this macro in the cc file for each terminal class.\n */\n#define MXNET_REGISTER_OBJECT_TYPE(TypeName)                                  \\\n  static DMLC_ATTRIBUTE_UNUSED uint32_t __make_Object_tidx##_##TypeName##__ = \\\n      TypeName::_GetOrAllocRuntimeTypeIndex()\n\n#define MXNET_DEFINE_DEFAULT_COPY_MOVE_AND_ASSIGN(TypeName) \\\n  TypeName(const TypeName& other) = default;                \\\n  TypeName(TypeName&& other)      = default;                \\\n  TypeName& operator=(const TypeName& other) = default;     \\\n  TypeName& operator=(TypeName&& other) = default;\n\n#define MXNET_DEFINE_OBJECT_REF_METHODS(TypeName, ParentType, ObjectName)                       \\\n  TypeName() {}                                                                                 \\\n  explicit TypeName(::mxnet::runtime::ObjectPtr<::mxnet::runtime::Object> n) : ParentType(n) {} \\\n  const ObjectName* operator->() const {                                                        \\\n    return static_cast<const ObjectName*>(data_.get());                                         \\\n  }                                                                                             \\\n  operator bool() const {                                                                       \\\n    return data_ != nullptr;                                                                    \\\n  }                                                                                             \\\n  using ContainerType = ObjectName;\n\n#define MXNET_DEFINE_OBJECT_REF_METHODS_MUT(TypeName, ParentType, ObjectName)                   \\\n  TypeName() {}                                                                                 \\\n  explicit TypeName(::mxnet::runtime::ObjectPtr<::mxnet::runtime::Object> n) : ParentType(n) {} \\\n  ObjectName* operator->() {                                                                    \\\n    return static_cast<ObjectName*>(data_.get());                                               \\\n  }                                                                                             \\\n  operator bool() const {                                                                       \\\n    return data_ != nullptr;                                                                    \\\n  }                                                                                             \\\n  using ContainerType = ObjectName;\n\n#define MXNET_DEFINE_NOTNULLABLE_OBJECT_REF_METHODS(TypeName, ParentType, ObjectName)           \\\n  explicit TypeName(::mxnet::runtime::ObjectPtr<::mxnet::runtime::Object> n) : ParentType(n) {} \\\n  MXNET_DEFINE_DEFAULT_COPY_MOVE_AND_ASSIGN(TypeName);                                          \\\n  const ObjectName* operator->() const {                                                        \\\n    return static_cast<const ObjectName*>(data_.get());                                         \\\n  }                                                                                             \\\n  const ObjectName* get() const {                                                               \\\n    return operator->();                                                                        \\\n  }                                                                                             \\\n  static constexpr bool _type_is_nullable = false;                                              \\\n  using ContainerType                     = ObjectName;\n\n// Implementations details below\n// Object reference counting.\n#if MXNET_OBJECT_ATOMIC_REF_COUNTER\n\ninline void Object::IncRef() {\n  ref_counter_.fetch_add(1, std::memory_order_relaxed);\n}\n\ninline void Object::DecRef() {\n  if (ref_counter_.fetch_sub(1, std::memory_order_release) == 1) {\n    std::atomic_thread_fence(std::memory_order_acquire);\n    if (this->deleter_ != nullptr) {\n      (*this->deleter_)(this);\n    }\n  }\n}\n\ninline int Object::use_count() const {\n  return ref_counter_.load(std::memory_order_relaxed);\n}\n\n#else\n\ninline void Object::IncRef() {\n  ++ref_counter_;\n}\n\ninline void Object::DecRef() {\n  if (--ref_counter == 0) {\n    if (this->deleter_ != nullptr) {\n      (*this->deleter_)(this);\n    }\n  }\n}\n\ninline int Object::use_count() const {\n  return ref_counter_;\n}\n\n#endif  // MXNET_OBJECT_ATOMIC_REF_COUNTER\n\ntemplate <typename TargetType>\ninline bool Object::IsInstance() const {\n  const Object* self = this;\n  // NOTE: the following code can be optimized by\n  // compiler dead-code elimination for already known constants.\n  if (self != nullptr) {\n    // Everything is a subclass of object.\n    if (std::is_same<TargetType, Object>::value)\n      return true;\n    if (TargetType::_type_final) {\n      // if the target type is a final type\n      // then we only need to check the equivalence.\n      return self->type_index_ == TargetType::RuntimeTypeIndex();\n    } else {\n      // if target type is a non-leaf type\n      // Check if type index falls into the range of reserved slots.\n      uint32_t begin = TargetType::RuntimeTypeIndex();\n      // The condition will be optimized by constant-folding.\n      if (TargetType::_type_child_slots != 0) {\n        uint32_t end = begin + TargetType::_type_child_slots;\n        if (self->type_index_ >= begin && self->type_index_ < end)\n          return true;\n      } else {\n        if (self->type_index_ == begin)\n          return true;\n      }\n      if (!TargetType::_type_child_slots_can_overflow)\n        return false;\n      // Invariance: parent index is always smaller than the child.\n      if (self->type_index_ < TargetType::RuntimeTypeIndex())\n        return false;\n      // The rare slower-path, check type hierachy.\n      return self->DerivedFrom(TargetType::RuntimeTypeIndex());\n    }\n  } else {\n    return false;\n  }\n}\n\ntemplate <typename ObjectType>\ninline const ObjectType* ObjectRef::as() const {\n  if (data_ != nullptr && data_->IsInstance<ObjectType>()) {\n    return static_cast<ObjectType*>(data_.get());\n  } else {\n    return nullptr;\n  }\n}\n\ntemplate <typename RefType, typename ObjType>\ninline RefType GetRef(const ObjType* ptr) {\n  static_assert(std::is_base_of<typename RefType::ContainerType, ObjType>::value,\n                \"Can only cast to the ref of same container type\");\n  if (!RefType::_type_is_nullable) {\n    CHECK(ptr != nullptr);\n  }\n  return RefType(ObjectPtr<Object>(const_cast<Object*>(static_cast<const Object*>(ptr))));\n}\n\ntemplate <typename BaseType, typename ObjType>\ninline ObjectPtr<BaseType> GetObjectPtr(ObjType* ptr) {\n  static_assert(std::is_base_of<BaseType, ObjType>::value,\n                \"Can only cast to the ref of same container type\");\n  return ObjectPtr<BaseType>(static_cast<Object*>(ptr));\n}\n\ntemplate <typename SubRef, typename BaseRef>\ninline SubRef Downcast(BaseRef ref) {\n  if (ref.defined()) {\n    CHECK(ref->template IsInstance<typename SubRef::ContainerType>())\n        << \"Downcast from \" << ref->GetTypeKey() << \" to \" << SubRef::ContainerType::_type_key\n        << \" failed.\";\n  } else {\n    CHECK(SubRef::_type_is_nullable) << \"Downcast from nullptr to not nullable reference of \"\n                                     << SubRef::ContainerType::_type_key;\n  }\n  return SubRef(std::move(ref.data_));\n}\n\n}  // namespace runtime\n\ntemplate <typename T>\nusing NodePtr = runtime::ObjectPtr<T>;\n\n}  // namespace mxnet\n\n#endif  // MXNET_RUNTIME_OBJECT_H_\n"
  },
  {
    "path": "include/mxnet/runtime/packed_func.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file runtime/packed_func.h\n * \\brief Type-erased function used across MXNET API.\n */\n// Acknowledgement: This file originates from incubator-tvm\n#ifndef MXNET_RUNTIME_PACKED_FUNC_H_\n#define MXNET_RUNTIME_PACKED_FUNC_H_\n\n#include <dmlc/logging.h>\n#include <mxnet/runtime/c_runtime_api.h>\n#include <mxnet/runtime/object.h>\n#include <mxnet/runtime/ndarray.h>\n#include <mxnet/runtime/container.h>\n#include <mxnet/runtime/container_ext.h>\n#include <mxnet/runtime/ndarray_handle.h>\n#include <mxnet/runtime/ffi_helper.h>\n#include <mxnet/runtime/data_type.h>\n#include <mxnet/runtime/py_arg.h>\n#include <mxnet/node/container.h>\n#include <mxnet/ir/expr.h>\n#include <mxnet/ndarray.h>\n#include <mxnet/base.h>\n#include <functional>\n#include <tuple>\n#include <vector>\n#include <string>\n#include <limits>\n#include <memory>\n#include <utility>\n#include <type_traits>\n#include <sstream>\n\nnamespace mxnet {\n// forward declarations\n// class Integer;\n// class Expr;\n\nnamespace runtime {\n\n/*!\n * \\brief convert a string to TVM type.\n * \\param s The string to be converted.\n * \\return The corresponding tvm type.\n */\ninline DLDataType String2DLDataType(std::string s);\n\n// forward declarations\nclass MXNetArgs;\nclass MXNetArgValue;\nclass MXNetRetValue;\nclass MXNetArgsSetter;\n\n/*!\n * \\brief Packed function is a type-erased function.\n *  The arguments are passed by packed format.\n *\n *  This is an useful unified interface to call generated functions,\n *  It is the unified function function type of TVM.\n *  It corresponds to TVMFunctionHandle in C runtime API.\n */\nclass PackedFunc {\n public:\n  /*!\n   * \\brief The internal std::function\n   * \\param args The arguments to the function.\n   * \\param rv The return value.\n   *\n   * \\code\n   *   // Example code on how to implemented FType\n   *   void MyPackedFunc(MXNetArgs args, MXNetRetValue* rv) {\n   *     // automatically convert arguments to desired type.\n   *     int a0 = args[0];\n   *     float a1 = args[1];\n   *     ...\n   *     // automatically assign values to rv\n   *     std::string my_return_value = \"x\";\n   *     *rv = my_return_value;\n   *   }\n   * \\endcode\n   */\n  using FType = std::function<void(MXNetArgs args, MXNetRetValue* rv)>;\n  /*! \\brief default constructor */\n  PackedFunc() {}\n  /*! \\brief constructor from null */\n  PackedFunc(std::nullptr_t null) {}  // NOLINT(*)\n  /*!\n   * \\brief constructing a packed function from a std::function.\n   * \\param body the internal container of packed function.\n   */\n  explicit PackedFunc(FType body) : body_(body) {}\n  /*!\n   * \\brief Call packed function by directly passing in unpacked format.\n   * \\param args Arguments to be passed.\n   * \\tparam Args arguments to be passed.\n   *\n   * \\code\n   *   // Example code on how to call packed function\n   *   void CallPacked(PackedFunc f) {\n   *     // call like normal functions by pass in arguments\n   *     // return value is automatically converted back\n   *     int rvalue = f(1, 2.0);\n   *   }\n   * \\endcode\n   */\n  template <typename... Args>\n  inline MXNetRetValue operator()(Args&&... args) const;\n  /*!\n   * \\brief Call the function in packed format.\n   * \\param args The arguments\n   * \\param rv The return value.\n   */\n  inline void CallPacked(MXNetArgs args, MXNetRetValue* rv) const;\n  /*! \\return the internal body function */\n  inline FType body() const;\n  /*! \\return Whether the packed function is nullptr */\n  bool operator==(std::nullptr_t null) const {\n    return body_ == nullptr;\n  }\n  /*! \\return Whether the packed function is not nullptr */\n  bool operator!=(std::nullptr_t null) const {\n    return body_ != nullptr;\n  }\n\n private:\n  /*! \\brief internal container of packed function */\n  FType body_;\n};\n\n/*!\n * \\brief Please refer to \\ref TypedPackedFuncAnchor \"TypedPackedFunc<R(Args..)>\"\n */\ntemplate <typename FType>\nclass TypedPackedFunc;\n\n/*!\n * \\anchor TypedPackedFuncAnchor\n * \\brief A PackedFunc wrapper to provide typed function signature.\n * It is backed by a PackedFunc internally.\n *\n * TypedPackedFunc enables compile time type checking.\n * TypedPackedFunc works with the runtime system:\n * - It can be passed as an argument of PackedFunc.\n * - It can be assigned to MXNetRetValue.\n * - It can be directly converted to a type-erased PackedFunc.\n *\n * Developers should prefer TypedPackedFunc over PackedFunc in C++ code\n * as it enables compile time checking.\n * We can construct a TypedPackedFunc from a lambda function\n * with the same signature.\n *\n * \\code\n *  // user defined lambda function.\n *  auto addone = [](int x)->int {\n *    return x + 1;\n *  };\n *  // We can directly convert\n *  // lambda function to TypedPackedFunc\n *  TypedPackedFunc<int(int)> ftyped(addone);\n *  // invoke the function.\n *  int y = ftyped(1);\n *  // Can be directly converted to PackedFunc\n *  PackedFunc packed = ftype;\n * \\endcode\n * \\tparam R The return value of the function.\n * \\tparam Args The argument signature of the function.\n */\ntemplate <typename R, typename... Args>\nclass TypedPackedFunc<R(Args...)> {\n public:\n  /*! \\brief short hand for this function type */\n  using TSelf = TypedPackedFunc<R(Args...)>;\n  /*! \\brief default constructor */\n  TypedPackedFunc() {}\n  /*! \\brief constructor from null */\n  TypedPackedFunc(std::nullptr_t null) {}  // NOLINT(*)\n  /*!\n   * \\brief construct by wrap a PackedFunc\n   *\n   * Example usage:\n   * \\code\n   * PackedFunc packed([](MXNetArgs args, MXNetRetValue *rv) {\n   *   int x = args[0];\n   *   *rv = x + 1;\n   *  });\n   * // construct from packed function\n   * TypedPackedFunc<int(int)> ftyped(packed);\n   * // call the typed version.\n   * CHECK_EQ(ftyped(1), 2);\n   * \\endcode\n   *\n   * \\param packed The packed function\n   */\n  inline TypedPackedFunc(PackedFunc packed);  // NOLINT(*)\n  /*!\n   * \\brief constructor from MXNetRetValue\n   * \\param value The MXNetRetValue\n   */\n  inline TypedPackedFunc(const MXNetRetValue& value);  // NOLINT(*)\n  /*!\n   * \\brief constructor from MXNetArgValue\n   * \\param value The MXNetArgValue\n   */\n  inline TypedPackedFunc(const MXNetArgValue& value);  // NOLINT(*)\n  /*!\n   * \\brief construct from a lambda function with the same signature.\n   *\n   * Example usage:\n   * \\code\n   * auto typed_lambda = [](int x)->int { return x + 1; }\n   * // construct from packed function\n   * TypedPackedFunc<int(int)> ftyped(typed_lambda);\n   * // call the typed version.\n   * CHECK_EQ(ftyped(1), 2);\n   * \\endcode\n   *\n   * \\param typed_lambda typed lambda function.\n   * \\tparam FLambda the type of the lambda function.\n   */\n  template <typename FLambda,\n            typename = typename std::enable_if<\n                std::is_convertible<FLambda,\n                                    std::function<R(Args...)>>::value>::type>\n  TypedPackedFunc(const FLambda& typed_lambda) {  // NOLINT(*)\n    this->AssignTypedLambda(typed_lambda);\n  }\n  /*!\n   * \\brief copy assignment operator from typed lambda\n   *\n   * Example usage:\n   * \\code\n   * // construct from packed function\n   * TypedPackedFunc<int(int)> ftyped;\n   * ftyped = [](int x) { return x + 1; }\n   * // call the typed version.\n   * CHECK_EQ(ftyped(1), 2);\n   * \\endcode\n   *\n   * \\param typed_lambda typed lambda function.\n   * \\tparam FLambda the type of the lambda function.\n   * \\returns reference to self.\n   */\n  template <typename FLambda,\n            typename = typename std::enable_if<\n                std::is_convertible<FLambda,\n                                    std::function<R(Args...)>>::value>::type>\n  TSelf& operator=(FLambda typed_lambda) {  // NOLINT(*)\n    this->AssignTypedLambda(typed_lambda);\n    return *this;\n  }\n  /*!\n   * \\brief copy assignment operator from PackedFunc.\n   * \\param packed The packed function.\n   * \\returns reference to self.\n   */\n  TSelf& operator=(PackedFunc packed) {\n    packed_ = packed;\n    return *this;\n  }\n  /*!\n   * \\brief Invoke the operator.\n   * \\param args The arguments\n   * \\returns The return value.\n   */\n  inline R operator()(Args... args) const;\n  /*!\n   * \\brief convert to PackedFunc\n   * \\return the internal PackedFunc\n   */\n  operator PackedFunc() const {\n    return packed();\n  }\n  /*!\n   * \\return reference the internal PackedFunc\n   */\n  const PackedFunc& packed() const {\n    return packed_;\n  }\n  /*! \\return Whether the packed function is nullptr */\n  bool operator==(std::nullptr_t null) const {\n    return packed_ == nullptr;\n  }\n  /*! \\return Whether the packed function is not nullptr */\n  bool operator!=(std::nullptr_t null) const {\n    return packed_ != nullptr;\n  }\n\n private:\n  friend class MXNetRetValue;\n  /*! \\brief The internal packed function */\n  PackedFunc packed_;\n  /*!\n   * \\brief Assign the packed field using a typed lambda function.\n   *\n   * \\param flambda The lambda function.\n   * \\tparam FLambda The lambda function type.\n   * \\note We capture the lambda when possible for maximum efficiency.\n   */\n  template <typename FLambda>\n  inline void AssignTypedLambda(FLambda flambda);\n};\n\n/*! \\brief Arguments into TVM functions. */\nclass MXNetArgs {\n public:\n  const MXNetValue* values;\n  const int* type_codes;\n  int num_args;\n  /*!\n   * \\brief constructor\n   * \\param values The argument values\n   * \\param type_codes The argument type codes\n   * \\param num_args number of arguments.\n   */\n  MXNetArgs(const MXNetValue* values, const int* type_codes, int num_args)\n      : values(values), type_codes(type_codes), num_args(num_args) {}\n  /*! \\return size of the arguments */\n  inline int size() const;\n  /*!\n   * \\brief Get i-th argument\n   * \\param i the index.\n   * \\return the ith argument.\n   */\n  inline MXNetArgValue operator[](int i) const;\n};\n\n/*!\n * \\brief Convert type code to its name\n * \\param type_code The type code .\n * \\return The name of type code.\n */\ninline const char* TypeCode2Str(int type_code);\n\n/*!\n * \\brief convert a string to TVM type.\n * \\param s The string to be converted.\n * \\return The corresponding tvm type.\n */\n// inline TVMType String2TVMType(std::string s);\n\n// macro to check type code.\n#define MXNET_CHECK_TYPE_CODE(CODE, T) \\\n  CHECK_EQ(CODE, T) << \" expected \" << TypeCode2Str(T) << \" but get \" << TypeCode2Str(CODE)\n\n/*!\n * \\brief Type traits to mark if a class is tvm extension type.\n *\n * To enable extension type in C++ must be registered via marco.\n * TVM_REGISTER_EXT_TYPE(TypeName) after defining this with this traits.\n *\n * Extension class can be passed and returned via PackedFunc in all tvm runtime.\n * Internally extension class is stored as T*.\n *\n * \\tparam T the typename\n */\ntemplate <typename T>\nstruct extension_type_info {\n  static const int code = 0;\n};\n\n/*!\n * \\brief Type traits for runtime type check during FFI conversion.\n * \\tparam T the type to be checked.\n */\ntemplate <typename T>\nstruct ObjectTypeChecker {\n  static bool Check(const Object* ptr) {\n    using ContainerType = typename T::ContainerType;\n    if (ptr == nullptr)\n      return T::_type_is_nullable;\n    return ptr->IsInstance<ContainerType>();\n  }\n  static std::string TypeName() {\n    using ContainerType = typename T::ContainerType;\n    return ContainerType::_type_key;\n  }\n};\n\n/*!\n * \\brief Internal base class to\n *  handle conversion to POD values.\n */\nclass MXNetPODValue_ {\n public:\n  operator double() const {\n    // Allow automatic conversion from int to float\n    // This avoids errors when user pass in int from\n    // the frontend while the API expects a float.\n    if (type_code_ == kDLInt) {\n      return static_cast<double>(value_.v_int64);\n    }\n    MXNET_CHECK_TYPE_CODE(type_code_, kDLFloat);\n    return value_.v_float64;\n  }\n  operator int64_t() const {\n    MXNET_CHECK_TYPE_CODE(type_code_, kDLInt);\n    return value_.v_int64;\n  }\n  operator uint64_t() const {\n    MXNET_CHECK_TYPE_CODE(type_code_, kDLUInt);\n    return value_.v_uint64;\n  }\n  operator int() const {\n    MXNET_CHECK_TYPE_CODE(type_code_, kDLInt);\n    CHECK_LE(value_.v_int64, std::numeric_limits<int>::max());\n    return static_cast<int>(value_.v_int64);\n  }\n  operator bool() const {\n    MXNET_CHECK_TYPE_CODE(type_code_, kDLInt);\n    return value_.v_int64 != 0;\n  }\n  operator void*() const {\n    if (type_code_ == kNull)\n      return nullptr;\n    MXNET_CHECK_TYPE_CODE(type_code_, kHandle);\n    return value_.v_handle;\n  }\n  operator ObjectRef() const {\n    if (type_code_ == kNull) {\n      return ObjectRef(ObjectPtr<Object>(nullptr));\n    }\n    MXNET_CHECK_TYPE_CODE(type_code_, kObjectHandle);\n    return ObjectRef(ObjectPtr<Object>(static_cast<Object*>(value_.v_handle)));\n  }\n  template <typename TObjectRef,\n            typename = typename std::enable_if<std::is_class<TObjectRef>::value>::type>\n  inline bool IsObjectRef() const;\n  template <typename TObjectRef>\n  inline TObjectRef AsObjectRef() const;\n  int type_code() const {\n    return type_code_;\n  }\n\n  /*!\n   * \\brief return handle as specific pointer type.\n   * \\tparam T the data type.\n   * \\return The pointer type.\n   */\n  template <typename T>\n  T* ptr() const {\n    return static_cast<T*>(value_.v_handle);\n  }\n\n protected:\n  friend class MXNetArgsSetter;\n  friend class MXNetRetValue;\n  MXNetPODValue_() : type_code_(kNull) {}\n  MXNetPODValue_(MXNetValue value, int type_code) : value_(value), type_code_(type_code) {}\n\n  /*! \\brief The value */\n  MXNetValue value_;\n  /*! \\brief the type code */\n  int type_code_;\n};\n\n/*!\n * \\brief A single argument value to PackedFunc.\n *  Containing both type_code and MXNetValue\n *\n *  Provides utilities to do type cast into other types.\n */\nclass MXNetArgValue : public MXNetPODValue_ {\n public:\n  /*! \\brief default constructor */\n  MXNetArgValue() {}\n  /*!\n   * \\brief constructor\n   * \\param value of the function\n   * \\param type_code The type code.\n   */\n  MXNetArgValue(MXNetValue value, int type_code) : MXNetPODValue_(value, type_code) {}\n  // reuse converter from parent\n  using MXNetPODValue_::operator double;\n  using MXNetPODValue_::operator int64_t;\n  using MXNetPODValue_::operator uint64_t;\n  using MXNetPODValue_::operator int;\n  using MXNetPODValue_::operator bool;\n  using MXNetPODValue_::operator void*;\n  using MXNetPODValue_::operator ObjectRef;\n  using MXNetPODValue_::AsObjectRef;\n  using MXNetPODValue_::IsObjectRef;\n\n  // conversion operator.\n  operator std::string() const {\n    if (type_code_ == kBytes) {\n      MXNetByteArray* arr = static_cast<MXNetByteArray*>(value_.v_handle);\n      return std::string(arr->data, arr->size);\n    } else {\n      MXNET_CHECK_TYPE_CODE(type_code_, kStr);\n      return std::string(value_.v_str);\n    }\n  }\n  operator DLDataType() const {\n    if (type_code_ == kStr) {\n      return String2DLDataType(operator std::string());\n    }\n    // None type\n    if (type_code_ == kNull) {\n      DLDataType t;\n      t.code  = kHandle;\n      t.bits  = 0;\n      t.lanes = 0;\n      return t;\n    }\n    MXNET_CHECK_TYPE_CODE(type_code_, kMXNetType);\n    return value_.v_type;\n  }\n  operator MXNetDataType() const {\n    return MXNetDataType(operator DLDataType());\n  }\n  operator ::mxnet::NDArray*() const {\n    if (type_code_ == kNull) {\n      return nullptr;\n    }\n    MXNET_CHECK_TYPE_CODE(type_code_, kNDArrayHandle);\n    return reinterpret_cast<::mxnet::NDArray*>(value_.v_handle);\n  }\n  template <typename FType>\n  operator TypedPackedFunc<FType>() const {\n    return TypedPackedFunc<FType>(operator PackedFunc());\n  }\n  const MXNetValue& value() const {\n    return value_;\n  }\n  template <typename T, typename = typename std::enable_if<std::is_class<T>::value>::type>\n  inline operator T() const;\n};\n\n/*!\n * \\brief Return Value container,\n *  Unlike MXNetArgValue, which only holds reference and do not delete\n *  the underlying container during destruction.\n *\n *  MXNetRetValue holds value and will manage the underlying containers\n *  when it stores a complicated data type.\n */\nclass MXNetRetValue : public MXNetPODValue_ {\n public:\n  /*! \\brief default constructor */\n  MXNetRetValue() {}\n  /*!\n   * \\brief move constructor from anoter return value.\n   * \\param other The other return value.\n   */\n  MXNetRetValue(MXNetRetValue&& other) : MXNetPODValue_(other.value_, other.type_code_) {\n    other.value_.v_handle = nullptr;\n    other.type_code_      = kNull;\n  }\n  /*! \\brief destructor */\n  ~MXNetRetValue() {\n    this->Clear();\n  }\n  // reuse converter from parent\n  using MXNetPODValue_::operator double;\n  using MXNetPODValue_::operator int64_t;\n  using MXNetPODValue_::operator uint64_t;\n  using MXNetPODValue_::operator int;\n  using MXNetPODValue_::operator bool;\n  using MXNetPODValue_::operator void*;\n  using MXNetPODValue_::operator ObjectRef;\n  using MXNetPODValue_::AsObjectRef;\n  using MXNetPODValue_::IsObjectRef;\n\n  MXNetRetValue(const MXNetRetValue& other) : MXNetPODValue_() {\n    this->Assign(other);\n  }\n  // conversion operators\n  operator std::string() const {\n    if (type_code_ == kBytes) {\n      return *ptr<std::string>();\n    }\n    MXNET_CHECK_TYPE_CODE(type_code_, kStr);\n    return *ptr<std::string>();\n  }\n  operator DLDataType() const {\n    if (type_code_ == kStr) {\n      return String2DLDataType(operator std::string());\n    }\n    MXNET_CHECK_TYPE_CODE(type_code_, kMXNetType);\n    return value_.v_type;\n  }\n  operator MXNetDataType() const {\n    return MXNetDataType(operator DLDataType());\n  }\n  template <typename FType>\n  operator TypedPackedFunc<FType>() const {\n    return TypedPackedFunc<FType>(operator PackedFunc());\n  }\n  // Assign operators\n  MXNetRetValue& operator=(MXNetRetValue&& other) {\n    this->Clear();\n    value_           = other.value_;\n    type_code_       = other.type_code_;\n    other.type_code_ = kNull;\n    return *this;\n  }\n  MXNetRetValue& operator=(double value) {\n    this->SwitchToPOD(kDLFloat);\n    value_.v_float64 = value;\n    return *this;\n  }\n  MXNetRetValue& operator=(std::nullptr_t value) {\n    this->SwitchToPOD(kNull);\n    value_.v_handle = value;\n    return *this;\n  }\n  MXNetRetValue& operator=(void* value) {\n    this->SwitchToPOD(kHandle);\n    value_.v_handle = value;\n    return *this;\n  }\n  MXNetRetValue& operator=(int64_t value) {\n    this->SwitchToPOD(kDLInt);\n    value_.v_int64 = value;\n    return *this;\n  }\n  MXNetRetValue& operator=(int value) {\n    this->SwitchToPOD(kDLInt);\n    value_.v_int64 = value;\n    return *this;\n  }\n  MXNetRetValue& operator=(bool value) {\n    this->SwitchToPOD(kDLInt);\n    value_.v_int64 = value;\n    return *this;\n  }\n  MXNetRetValue& operator=(std::string value) {\n    this->SwitchToClass(kStr, value);\n    return *this;\n  }\n  MXNetRetValue& operator=(DLDataType t) {\n    this->SwitchToPOD(kMXNetType);\n    value_.v_type = t;\n    return *this;\n  }\n  MXNetRetValue& operator=(const MXNetDataType& other) {\n    return operator=(other.operator DLDataType());\n  }\n  MXNetRetValue& operator=(MXNetByteArray value) {\n    this->SwitchToClass(kBytes, std::string(value.data, value.size));\n    return *this;\n  }\n  MXNetRetValue& operator=(ObjectRef other) {\n    if (other.as<NDArrayHandleObj>()) {\n      return operator=(Downcast<NDArrayHandle, ObjectRef>(other));\n    }\n    return operator=(std::move(other.data_));\n  }\n  template <typename T>\n  MXNetRetValue& operator=(ObjectPtr<T> other) {\n    SwitchToObject(kObjectHandle, std::move(other));\n    return *this;\n  }\n  template <typename FType>\n  MXNetRetValue& operator=(const TypedPackedFunc<FType>& f) {\n    return operator=(f.packed());\n  }\n  MXNetRetValue& operator=(const MXNetRetValue& other) {  // NOLINT(*0\n    this->Assign(other);\n    return *this;\n  }\n  MXNetRetValue& operator=(const MXNetArgValue& other) {\n    this->Assign(other);\n    return *this;\n  }\n  MXNetRetValue& operator=(NDArray* value) {\n    this->SwitchToPOD(kNDArrayHandle);\n    value_.v_handle = reinterpret_cast<void*>(value);\n    return *this;\n  }\n  MXNetRetValue& operator=(NDArrayHandle value) {\n    this->SwitchToPOD(kNDArrayHandle);\n    NDArray* arr    = new NDArray(value->value);\n    value_.v_handle = reinterpret_cast<void*>(arr);\n    return *this;\n  }\n  MXNetRetValue& operator=(const PythonArg& value) {\n    this->SwitchToPOD(kPyArg);\n    value_.v_int64 = value.offset();\n    return *this;\n  }\n  template <typename T, typename = typename std::enable_if<extension_type_info<T>::code != 0>::type>\n  MXNetRetValue& operator=(const T& other) {\n    this->SwitchToClass<T>(extension_type_info<T>::code, other);\n    return *this;\n  }\n  /*!\n   * \\brief Move the value back to front-end via C API.\n   *  This marks the current container as null.\n   *  The managed resources is moved to front-end and\n   *  the front end should take charge in managing them.\n   *\n   * \\param ret_value The return value.\n   * \\param ret_type_code The return type code.\n   */\n  void MoveToCHost(MXNetValue* ret_value, int* ret_type_code) {\n    // cannot move str; need specially handle.\n    CHECK(type_code_ != kStr && type_code_ != kBytes);\n    *ret_value     = value_;\n    *ret_type_code = type_code_;\n    type_code_     = kNull;\n  }\n  /*! \\return The value field, if the data is POD */\n  const MXNetValue& value() const {\n    CHECK(type_code_ != kObjectHandle && type_code_ != kStr)\n        << \"MXNetRetValue.value can only be used for POD data\";\n    return value_;\n  }\n  // ObjectRef related extenstions: in tvm/packed_func_ext.h\n  template <typename T, typename = typename std::enable_if<std::is_class<T>::value>::type>\n  inline operator T() const;\n\n private:\n  template <typename T>\n  void Assign(const T& other) {\n    switch (other.type_code()) {\n      case kStr: {\n        SwitchToClass<std::string>(kStr, other);\n        break;\n      }\n      case kBytes: {\n        SwitchToClass<std::string>(kBytes, other);\n        break;\n      }\n      case kObjectHandle: {\n        *this = other.operator ObjectRef();\n        break;\n      }\n      default: {\n        if (other.type_code() < kExtBegin) {\n          SwitchToPOD(other.type_code());\n          value_ = other.value_;\n        } else {\n          LOG(FATAL) << \"Does not support ext type\";\n        }\n        break;\n      }\n    }\n  }\n  // get the internal container.\n  void SwitchToPOD(int type_code) {\n    if (type_code_ != type_code) {\n      this->Clear();\n      type_code_ = type_code;\n    }\n  }\n  template <typename T>\n  void SwitchToClass(int type_code, T v) {\n    if (type_code_ != type_code) {\n      this->Clear();\n      type_code_      = type_code;\n      value_.v_handle = new T(v);\n    } else {\n      *static_cast<T*>(value_.v_handle) = v;\n    }\n  }\n  void SwitchToObject(int type_code, ObjectPtr<Object> other) {\n    if (other.data_ != nullptr) {\n      this->Clear();\n      type_code_ = type_code;\n      // move the handle out\n      value_.v_handle = other.data_;\n      other.data_     = nullptr;\n    } else {\n      SwitchToPOD(kNull);\n    }\n  }\n  void Clear() {\n    if (type_code_ == kNull)\n      return;\n    switch (type_code_) {\n      case kStr:\n        delete ptr<std::string>();\n        break;\n      case kObjectHandle: {\n        static_cast<Object*>(value_.v_handle)->DecRef();\n        break;\n      }\n    }\n    if (type_code_ > kExtBegin) {\n      LOG(FATAL) << \"Does not support ext type\";\n    }\n    type_code_ = kNull;\n  }\n};\n\ninline DLDataType String2DLDataType(std::string s) {\n  DLDataType t;\n  // handle None type\n  if (s.length() == 0) {\n    t.bits  = 0;\n    t.lanes = 0;\n    t.code  = kHandle;\n    return t;\n  }\n  t.bits           = 32;\n  t.lanes          = 1;\n  const char* scan = nullptr;\n  if (s.substr(0, 3) == \"int\") {\n    t.code = kDLInt;\n    scan   = s.c_str() + 3;\n  } else if (s.substr(0, 4) == \"uint\") {\n    t.code = kDLUInt;\n    scan   = s.c_str() + 4;\n  } else if (s.substr(0, 5) == \"float\") {\n    t.code = kDLFloat;\n    scan   = s.c_str() + 5;\n  } else if (s.substr(0, 6) == \"handle\") {\n    t.code = kHandle;\n    t.bits = 64;  // handle uses 64 bit by default.\n    scan   = s.c_str() + 6;\n  } else if (s == \"bool\") {\n    t.code  = kDLUInt;\n    t.bits  = 1;\n    t.lanes = 1;\n    return t;\n  } else if (s.substr(0, 6) == \"custom\") {\n    LOG(FATAL) << \"custom MXNetDataType is not supported\";\n    // t.code = ParseCustomDatatype(s, &scan);\n  } else {\n    scan = s.c_str();\n    LOG(FATAL) << \"unknown type \" << s;\n  }\n  char* xdelim;  // emulate sscanf(\"%ux%u\", bits, lanes)\n  uint8_t bits = static_cast<uint8_t>(strtoul(scan, &xdelim, 10));\n  if (bits != 0)\n    t.bits = bits;\n  char* endpt = xdelim;\n  if (*xdelim == 'x') {\n    t.lanes = static_cast<uint16_t>(strtoul(xdelim + 1, &endpt, 10));\n  }\n  CHECK(endpt == s.c_str() + s.length()) << \"unknown type \" << s;\n  return t;\n}\n\n// implementation details\ninline const char* TypeCode2Str(int type_code) {\n  switch (type_code) {\n    case kDLInt:\n      return \"int\";\n    case kDLUInt:\n      return \"uint\";\n    case kDLFloat:\n      return \"float\";\n    case kStr:\n      return \"str\";\n    case kBytes:\n      return \"bytes\";\n    case kHandle:\n      return \"handle\";\n    case kNull:\n      return \"NULL\";\n    case kObjectHandle:\n      return \"ObjectCell\";\n    case kNDArrayHandle:\n      return \"NDArray\";\n    default:\n      LOG(FATAL) << \"unknown type_code=\" << static_cast<int>(type_code);\n      return \"\";\n  }\n}\n\ninline int String2MXNetTypeWithBool(const std::string& s) {\n  if (s == \"float32\") {\n    return mshadow::kFloat32;\n  } else if (s == \"float64\") {\n    return mshadow::kFloat64;\n  } else if (s == \"float16\") {\n    return mshadow::kFloat16;\n  } else if (s == \"bfloat16\") {\n    return mshadow::kBfloat16;\n  } else if (s == \"uint8\") {\n    return mshadow::kUint8;\n  } else if (s == \"int8\") {\n    return mshadow::kInt8;\n  } else if (s == \"int32\") {\n    return mshadow::kInt32;\n  } else if (s == \"int64\") {\n    return mshadow::kInt64;\n  } else if (s == \"bool\") {\n    return mshadow::kBool;\n  } else if (s == \"int16\") {\n    return mshadow::kInt16;\n  } else if (s == \"uint16\") {\n    return mshadow::kUint16;\n  } else if (s == \"uint32\") {\n    return mshadow::kUint32;\n  } else if (s == \"uint64\") {\n    return mshadow::kUint64;\n  } else {\n    LOG(FATAL) << \"unknown type \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\ninline int String2MXNetType(const std::string& s) {\n  if (s == \"float32\") {\n    return mshadow::kFloat32;\n  } else if (s == \"float64\") {\n    return mshadow::kFloat64;\n  } else if (s == \"float16\") {\n    return mshadow::kFloat16;\n  } else if (s == \"bfloat16\") {\n    return mshadow::kBfloat16;\n  } else if (s == \"uint8\") {\n    return mshadow::kUint8;\n  } else if (s == \"int8\") {\n    return mshadow::kInt8;\n  } else if (s == \"int32\") {\n    return mshadow::kInt32;\n  } else if (s == \"int64\") {\n    return mshadow::kInt64;\n  } else if (s == \"int16\") {\n    return mshadow::kInt16;\n  } else if (s == \"uint16\") {\n    return mshadow::kUint16;\n  } else if (s == \"uint32\") {\n    return mshadow::kUint32;\n  } else if (s == \"uint64\") {\n    return mshadow::kUint64;\n  } else {\n    LOG(FATAL) << \"unknown type \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\ninline std::ostream& operator<<(std::ostream& os, DLDataType t) {  // NOLINT(*)\n  if (t.bits == 1 && t.lanes == 1 && t.code == kDLUInt) {\n    os << \"bool\";\n    return os;\n  }\n  if (t.code < kCustomBegin) {\n    os << TypeCode2Str(t.code);\n  } else {\n    LOG(FATAL) << \"custom MXNetDataType is not supported\";\n    // os << \"custom[\" << GetCustomTypeName(t.code) << \"]\";\n  }\n  if (t.code == kHandle)\n    return os;\n  os << static_cast<int>(t.bits);\n  if (t.lanes != 1) {\n    os << 'x' << static_cast<int>(t.lanes);\n  }\n  return os;\n}\n\ninline std::ostream& operator<<(std::ostream& os, const MXNetDataType& dtype) {  // NOLINT(*)\n  return os << dtype.operator DLDataType();\n}\n\ninline MXNetArgValue MXNetArgs::operator[](int i) const {\n  CHECK_LT(i, num_args) << \"not enough argument passed, \" << num_args << \" passed\"\n                        << \" but request arg[\" << i << \"].\";\n  return MXNetArgValue(values[i], type_codes[i]);\n}\n\ninline int MXNetArgs::size() const {\n  return num_args;\n}\n\ninline void PackedFunc::CallPacked(MXNetArgs args, MXNetRetValue* rv) const {\n  body_(args, rv);\n}\n\ninline PackedFunc::FType PackedFunc::body() const {\n  return body_;\n}\n\n// internal namespace\nnamespace detail {\n\ntemplate <bool stop, std::size_t I, typename F>\nstruct for_each_dispatcher {\n  template <typename T, typename... Args>\n  static void run(const F& f, T&& value, Args&&... args) {  // NOLINT(*)\n    f(I, std::forward<T>(value));\n    for_each_dispatcher<sizeof...(Args) == 0, (I + 1), F>::run(f, std::forward<Args>(args)...);\n  }\n};\n\ntemplate <std::size_t I, typename F>\nstruct for_each_dispatcher<true, I, F> {\n  static void run(const F& f) {}  // NOLINT(*)\n};\n\ntemplate <typename F, typename... Args>\ninline void for_each(const F& f, Args&&... args) {  // NOLINT(*)\n  for_each_dispatcher<sizeof...(Args) == 0, 0, F>::run(f, std::forward<Args>(args)...);\n}\n}  // namespace detail\n\n/* \\brief argument settter to PackedFunc */\nclass MXNetArgsSetter {\n public:\n  MXNetArgsSetter(MXNetValue* values, int* type_codes) : values_(values), type_codes_(type_codes) {}\n  // setters for POD types\n  template <typename T, typename = typename std::enable_if<std::is_integral<T>::value>::type>\n  void operator()(size_t i, T value) const {\n    values_[i].v_int64 = static_cast<int64_t>(value);\n    type_codes_[i]     = kDLInt;\n  }\n  void operator()(size_t i, uint64_t value) const {\n    values_[i].v_int64 = static_cast<int64_t>(value);\n    CHECK_LE(value, static_cast<uint64_t>(std::numeric_limits<int64_t>::max()));\n    type_codes_[i] = kDLInt;\n  }\n  void operator()(size_t i, double value) const {\n    values_[i].v_float64 = value;\n    type_codes_[i]       = kDLFloat;\n  }\n  void operator()(size_t i, std::nullptr_t value) const {\n    values_[i].v_handle = value;\n    type_codes_[i]      = kNull;\n  }\n  void operator()(size_t i, const MXNetArgValue& value) const {\n    values_[i]     = value.value_;\n    type_codes_[i] = value.type_code_;\n  }\n  void operator()(size_t i, void* value) const {\n    values_[i].v_handle = value;\n    type_codes_[i]      = kHandle;\n  }\n  void operator()(size_t i, const char* value) const {\n    values_[i].v_str = value;\n    type_codes_[i]   = kStr;\n  }\n  // setters for container type\n  // They must be reference(instead of const ref)\n  // to make sure they are alive in the tuple(instead of getting converted)\n  void operator()(size_t i, const std::string& value) const {  // NOLINT(*)\n    values_[i].v_str = value.c_str();\n    type_codes_[i]   = kStr;\n  }\n  void operator()(size_t i, DLDataType value) const {\n    values_[i].v_type = value;\n    type_codes_[i]    = kMXNetType;\n  }\n  void operator()(size_t i, MXNetDataType dtype) const {\n    operator()(i, dtype.operator DLDataType());\n  }\n  void operator()(size_t i, const MXNetByteArray& value) const {  // NOLINT(*)\n    values_[i].v_handle = const_cast<MXNetByteArray*>(&value);\n    type_codes_[i]      = kBytes;\n  }\n  template <typename FType>\n  void operator()(size_t i, const TypedPackedFunc<FType>& value) const {  // NOLINT(*)\n    operator()(i, value.packed());\n  }\n  void operator()(size_t i, const ObjectRef& value) const {  // NOLINT(*)\n    if (value.defined()) {\n      values_[i].v_handle = value.data_.data_;\n      type_codes_[i]      = kObjectHandle;\n    } else {\n      type_codes_[i] = kNull;\n    }\n  }\n  void operator()(size_t i, const MXNetRetValue& value) const {  // NOLINT(*)\n    if (value.type_code() == kStr) {\n      values_[i].v_str = value.ptr<std::string>()->c_str();\n      type_codes_[i]   = kStr;\n    } else {\n      CHECK_NE(value.type_code(), kBytes) << \"not handled.\";\n      values_[i]     = value.value_;\n      type_codes_[i] = value.type_code();\n    }\n  }\n\n private:\n  /*! \\brief The values fields */\n  MXNetValue* values_;\n  /*! \\brief The type code fields */\n  int* type_codes_;\n};\n\ntemplate <typename... Args>\ninline MXNetRetValue PackedFunc::operator()(Args&&... args) const {\n  const int kNumArgs   = sizeof...(Args);\n  const int kArraySize = kNumArgs > 0 ? kNumArgs : 1;\n  MXNetValue values[kArraySize];\n  int type_codes[kArraySize];\n  detail::for_each(MXNetArgsSetter(values, type_codes), std::forward<Args>(args)...);\n  MXNetRetValue rv;\n  body_(MXNetArgs(values, type_codes, kNumArgs), &rv);\n  return rv;\n}\n\nnamespace detail {\ntemplate <typename R, int nleft, int index, typename F>\nstruct unpack_call_dispatcher {\n  template <typename... Args>\n  static void run(const F& f,\n                  const MXNetArgs& args_pack,\n                  MXNetRetValue* rv,\n                  Args&&... unpacked_args) {\n    unpack_call_dispatcher<R, nleft - 1, index + 1, F>::run(\n        f, args_pack, rv, std::forward<Args>(unpacked_args)..., args_pack[index]);\n  }\n};\n\ntemplate <typename R, int index, typename F>\nstruct unpack_call_dispatcher<R, 0, index, F> {\n  template <typename... Args>\n  static void run(const F& f,\n                  const MXNetArgs& args_pack,\n                  MXNetRetValue* rv,\n                  Args&&... unpacked_args) {\n    *rv = R(f(std::forward<Args>(unpacked_args)...));\n  }\n};\n\ntemplate <int index, typename F>\nstruct unpack_call_dispatcher<void, 0, index, F> {\n  template <typename... Args>\n  static void run(const F& f,\n                  const MXNetArgs& args_pack,\n                  MXNetRetValue* rv,\n                  Args&&... unpacked_args) {\n    f(std::forward<Args>(unpacked_args)...);\n  }\n};\n\ntemplate <typename R, int nargs, typename F>\ninline void unpack_call(const F& f, const MXNetArgs& args, MXNetRetValue* rv) {\n  unpack_call_dispatcher<R, nargs, 0, F>::run(f, args, rv);\n}\n\ntemplate <typename R, typename... Args>\ninline R call_packed(const PackedFunc& pf, Args&&... args) {\n  return R(pf(std::forward<Args>(args)...));\n}\n\ntemplate <typename R>\nstruct typed_packed_call_dispatcher {\n  template <typename... Args>\n  static inline R run(const PackedFunc& pf, Args&&... args) {\n    return pf(std::forward<Args>(args)...);\n  }\n};\n\ntemplate <>\nstruct typed_packed_call_dispatcher<void> {\n  template <typename... Args>\n  static inline void run(const PackedFunc& pf, Args&&... args) {\n    pf(std::forward<Args>(args)...);\n  }\n};\n}  // namespace detail\n\ntemplate <typename R, typename... Args>\nTypedPackedFunc<R(Args...)>::TypedPackedFunc(PackedFunc packed) : packed_(packed) {}\n\ntemplate <typename R, typename... Args>\nTypedPackedFunc<R(Args...)>::TypedPackedFunc(const MXNetRetValue& value)\n    : packed_(value.operator PackedFunc()) {}\n\ntemplate <typename R, typename... Args>\nTypedPackedFunc<R(Args...)>::TypedPackedFunc(const MXNetArgValue& value)\n    : packed_(value.operator PackedFunc()) {}\n\ntemplate <typename R, typename... Args>\ntemplate <typename FType>\ninline void TypedPackedFunc<R(Args...)>::AssignTypedLambda(FType flambda) {\n  packed_ = PackedFunc([flambda](const MXNetArgs& args, MXNetRetValue* rv) {\n    detail::unpack_call<R, sizeof...(Args)>(flambda, args, rv);\n  });\n}\n\ntemplate <typename R, typename... Args>\ninline R TypedPackedFunc<R(Args...)>::operator()(Args... args) const {\n  return detail::typed_packed_call_dispatcher<R>::run(packed_, std::forward<Args>(args)...);\n}\n\n// extension and node type handling\nnamespace detail {\ntemplate <typename T, typename TSrc, bool is_ext, bool is_nd>\nstruct MXNetValueCast {\n  static T Apply(const TSrc* self) {\n    static_assert(!is_ext && !is_nd, \"The default case accepts only non-extensions\");\n    return self->template AsObjectRef<T>();\n  }\n};\n\n}  // namespace detail\n\n/*!\n * \\brief Type trait to specify special value conversion rules from\n *        MXNetArgValue and MXNetRetValue.\n *\n *  The trait can be specialized to add type specific conversion logic\n *  from the TVMArgvalue and TVMRetValue.\n *\n * \\tparam TObjectRef the specific ObjectRefType.\n */\ntemplate <typename TObjectRef>\nstruct PackedFuncValueConverter {\n  /*!\n   * \\brief Convert a TObjectRef from an argument value.\n   * \\param val The argument value.\n   * \\return the converted result.\n   */\n  static TObjectRef From(const MXNetArgValue& val) {\n    return val.AsObjectRef<TObjectRef>();\n  }\n  /*!\n   * \\brief Convert a TObjectRef from a return value.\n   * \\param val The argument value.\n   * \\return the converted result.\n   */\n  static TObjectRef From(const MXNetRetValue& val) {\n    return val.AsObjectRef<TObjectRef>();\n  }\n};\n\ntemplate <>\nstruct PackedFuncValueConverter<::mxnet::runtime::String> {\n  static String From(const MXNetArgValue& val) {\n    if (val.IsObjectRef<mxnet::runtime::String>()) {\n      return val.AsObjectRef<mxnet::runtime::String>();\n    } else {\n      return mxnet::runtime::String(val.operator std::string());\n    }\n  }\n\n  static String From(const MXNetRetValue& val) {\n    if (val.IsObjectRef<mxnet::runtime::String>()) {\n      return val.AsObjectRef<mxnet::runtime::String>();\n    } else {\n      return mxnet::runtime::String(val.operator std::string());\n    }\n  }\n};\n\ntemplate <typename TObjectRef>\ninline TObjectRef MXNetPODValue_::AsObjectRef() const {\n  static_assert(std::is_base_of<ObjectRef, TObjectRef>::value,\n                \"Conversion only works for ObjectRef\");\n  using ContainerType = typename TObjectRef::ContainerType;\n\n  if (type_code_ == kNull) {\n    CHECK(TObjectRef::_type_is_nullable)\n        << \"Expect a not null value of \" << ContainerType::_type_key;\n    return TObjectRef(ObjectPtr<Object>(nullptr));\n  }\n  if (type_code_ == kObjectHandle) {\n    // normal object type check.\n    Object* ptr = static_cast<Object*>(value_.v_handle);\n    CHECK(ObjectTypeChecker<TObjectRef>::Check(ptr))\n        << \"Expect \" << ObjectTypeChecker<TObjectRef>::TypeName() << \" but get \"\n        << ptr->GetTypeKey();\n    return TObjectRef(GetObjectPtr<Object>(ptr));\n  } else {\n    MXNET_CHECK_TYPE_CODE(type_code_, kObjectHandle);\n    return TObjectRef(ObjectPtr<Object>(nullptr));\n  }\n}\n\ntemplate <typename T, typename>\ninline MXNetArgValue::operator T() const {\n  return PackedFuncValueConverter<T>::From(*this);\n}\n\ntemplate <typename TObjectRef, typename>\ninline bool MXNetPODValue_::IsObjectRef() const {\n  using ContainerType = typename TObjectRef::ContainerType;\n  return type_code_ == kObjectHandle &&\n         ObjectTypeChecker<TObjectRef>::Check(static_cast<Object*>(value_.v_handle));\n}\n\ninline bool String::CanConvertFrom(const MXNetArgValue& val) {\n  return val.type_code() == kStr || val.IsObjectRef<mxnet::runtime::String>();\n}\n\n}  // namespace runtime\n}  // namespace mxnet\n#endif  // MXNET_RUNTIME_PACKED_FUNC_H_\n"
  },
  {
    "path": "include/mxnet/runtime/py_arg.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * \\file py_arg.h\n * \\brief Python runtime arguments specifier.\n */\n#ifndef MXNET_RUNTIME_PY_ARG_H_\n#define MXNET_RUNTIME_PY_ARG_H_\n\nnamespace mxnet {\nnamespace runtime {\n\nclass PythonArg {\n public:\n  explicit PythonArg(int offset) : offset_(offset) {}\n  int offset() const {\n    return offset_;\n  }\n\n private:\n  int offset_;\n};\n\n}  // namespace runtime\n\n}  // namespace mxnet\n#endif  //  MXNET_RUNTIME_PY_ARG_H_\n"
  },
  {
    "path": "include/mxnet/runtime/registry.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file registry.h\n * \\brief This file defines the TVM global function registry.\n *\n *  The registered functions will be made available to front-end\n *  as well as backend users.\n *\n *  The registry stores type-erased functions.\n *  Each registered function is automatically exposed\n *  to front-end language(e.g. python).\n *\n *  Front-end can also pass callbacks as PackedFunc, or register\n *  then into the same global registry in C++.\n *  The goal is to mix the front-end language and the TVM back-end.\n *\n * \\code\n *   // register the function as MyAPIFuncName\n *   TVM_REGISTER_GLOBAL(MyAPIFuncName)\n *   .set_body([](TVMArgs args, TVMRetValue* rv) {\n *     // my code.\n *   });\n * \\endcode\n */\n// Acknowledgement: This file originates from incubator-tvm\n#ifndef MXNET_RUNTIME_REGISTRY_H_\n#define MXNET_RUNTIME_REGISTRY_H_\n\n#include <string>\n#include <vector>\n#include \"packed_func.h\"\n\nnamespace mxnet {\nnamespace runtime {\n\n/*! \\brief Registry for global function */\nclass Registry {\n public:\n  /*!\n   * \\brief set the body of the function to be f\n   * \\param f The body of the function.\n   */\n  MXNET_DLL Registry& set_body(PackedFunc f);  // NOLINT(*)\n  /*!\n   * \\brief set the body of the function to be f\n   * \\param f The body of the function.\n   */\n  Registry& set_body(PackedFunc::FType f) {  // NOLINT(*)\n    return set_body(PackedFunc(f));\n  }\n  /*!\n   * \\brief set the body of the function to be TypedPackedFunc.\n   *\n   * \\code\n   *\n   * TVM_REGISTER_API(\"addone\")\n   * .set_body_typed<int(int)>([](int x) { return x + 1; });\n   *\n   * \\endcode\n   *\n   * \\param f The body of the function.\n   * \\tparam FType the signature of the function.\n   * \\tparam FLambda The type of f.\n   */\n  template <typename FType, typename FLambda>\n  Registry& set_body_typed(FLambda f) {\n    return set_body(TypedPackedFunc<FType>(f).packed());\n  }\n\n  /*!\n   * \\brief set the body of the function to the given function pointer.\n   *        Note that this doesn't work with lambdas, you need to\n   *        explicitly give a type for those.\n   *        Note that this will ignore default arg values and always require all arguments to be\n   * provided.\n   *\n   * \\code\n   *\n   * int multiply(int x, int y) {\n   *   return x * y;\n   * }\n   *\n   * TVM_REGISTER_API(\"multiply\")\n   * .set_body_typed(multiply); // will have type int(int, int)\n   *\n   * \\endcode\n   *\n   * \\param f The function to forward to.\n   * \\tparam R the return type of the function (inferred).\n   * \\tparam Args the argument types of the function (inferred).\n   */\n  template <typename R, typename... Args>\n  Registry& set_body_typed(R (*f)(Args...)) {\n    return set_body(TypedPackedFunc<R(Args...)>(f));\n  }\n\n  /*!\n   * \\brief set the body of the function to be the passed method pointer.\n   *        Note that this will ignore default arg values and always require all arguments to be\n   * provided.\n   *\n   * \\code\n   *\n   * // node subclass:\n   * struct Example {\n   *    int doThing(int x);\n   * }\n   * TVM_REGISTER_API(\"Example_doThing\")\n   * .set_body_method(&Example::doThing); // will have type int(Example, int)\n   *\n   * \\endcode\n   *\n   * \\param f the method pointer to forward to.\n   * \\tparam T the type containing the method (inferred).\n   * \\tparam R the return type of the function (inferred).\n   * \\tparam Args the argument types of the function (inferred).\n   */\n  template <typename T, typename R, typename... Args>\n  Registry& set_body_method(R (T::*f)(Args...)) {\n    return set_body_typed<R(T, Args...)>([f](T target, Args... params) -> R {\n      // call method pointer\n      return (target.*f)(params...);\n    });\n  }\n\n  /*!\n   * \\brief set the body of the function to be the passed method pointer.\n   *        Note that this will ignore default arg values and always require all arguments to be\n   * provided.\n   *\n   * \\code\n   *\n   * // node subclass:\n   * struct Example {\n   *    int doThing(int x);\n   * }\n   * TVM_REGISTER_API(\"Example_doThing\")\n   * .set_body_method(&Example::doThing); // will have type int(Example, int)\n   *\n   * \\endcode\n   *\n   * \\param f the method pointer to forward to.\n   * \\tparam T the type containing the method (inferred).\n   * \\tparam R the return type of the function (inferred).\n   * \\tparam Args the argument types of the function (inferred).\n   */\n  template <typename T, typename R, typename... Args>\n  Registry& set_body_method(R (T::*f)(Args...) const) {\n    return set_body_typed<R(T, Args...)>([f](const T target, Args... params) -> R {\n      // call method pointer\n      return (target.*f)(params...);\n    });\n  }\n\n  /*!\n   * \\brief set the body of the function to be the passed method pointer.\n   *        Used when calling a method on a Node subclass through a ObjectRef subclass.\n   *        Note that this will ignore default arg values and always require all arguments to be\n   * provided.\n   *\n   * \\code\n   *\n   * // node subclass:\n   * struct ExampleNode: BaseNode {\n   *    int doThing(int x);\n   * }\n   *\n   * // noderef subclass\n   * struct Example;\n   *\n   * TVM_REGISTER_API(\"Example_doThing\")\n   * .set_body_method<Example>(&ExampleNode::doThing); // will have type int(Example, int)\n   *\n   * // note that just doing:\n   * // .set_body_method(&ExampleNode::doThing);\n   * // wouldn't work, because ExampleNode can't be taken from a TVMArgValue.\n   *\n   * \\endcode\n   *\n   * \\param f the method pointer to forward to.\n   * \\tparam TObjectRef the node reference type to call the method on\n   * \\tparam TNode the node type containing the method (inferred).\n   * \\tparam R the return type of the function (inferred).\n   * \\tparam Args the argument types of the function (inferred).\n   */\n  template <typename TObjectRef,\n            typename TNode,\n            typename R,\n            typename... Args,\n            typename = typename std::enable_if<std::is_base_of<ObjectRef, TObjectRef>::value>::type>\n  Registry& set_body_method(R (TNode::*f)(Args...)) {\n    return set_body_typed<R(TObjectRef, Args...)>([f](TObjectRef ref, Args... params) {\n      TNode* target = ref.operator->();\n      // call method pointer\n      return (target->*f)(params...);\n    });\n  }\n\n  /*!\n   * \\brief set the body of the function to be the passed method pointer.\n   *        Used when calling a method on a Node subclass through a ObjectRef subclass.\n   *        Note that this will ignore default arg values and always require all arguments to be\n   * provided.\n   *\n   * \\code\n   *\n   * // node subclass:\n   * struct ExampleNode: BaseNode {\n   *    int doThing(int x);\n   * }\n   *\n   * // noderef subclass\n   * struct Example;\n   *\n   * TVM_REGISTER_API(\"Example_doThing\")\n   * .set_body_method<Example>(&ExampleNode::doThing); // will have type int(Example, int)\n   *\n   * // note that just doing:\n   * // .set_body_method(&ExampleNode::doThing);\n   * // wouldn't work, because ExampleNode can't be taken from a TVMArgValue.\n   *\n   * \\endcode\n   *\n   * \\param f the method pointer to forward to.\n   * \\tparam TObjectRef the node reference type to call the method on\n   * \\tparam TNode the node type containing the method (inferred).\n   * \\tparam R the return type of the function (inferred).\n   * \\tparam Args the argument types of the function (inferred).\n   */\n  template <typename TObjectRef,\n            typename TNode,\n            typename R,\n            typename... Args,\n            typename = typename std::enable_if<std::is_base_of<ObjectRef, TObjectRef>::value>::type>\n  Registry& set_body_method(R (TNode::*f)(Args...) const) {\n    return set_body_typed<R(TObjectRef, Args...)>([f](TObjectRef ref, Args... params) {\n      const TNode* target = ref.operator->();\n      // call method pointer\n      return (target->*f)(params...);\n    });\n  }\n\n  /*!\n   * \\brief Register a function with given name\n   * \\param name The name of the function.\n   * \\param override Whether allow oveeride existing function.\n   * \\return Reference to theregistry.\n   */\n  MXNET_DLL static Registry& Register(const std::string& name, bool override = false);  // NOLINT(*)\n  /*!\n   * \\brief Erase global function from registry, if exist.\n   * \\param name The name of the function.\n   * \\return Whether function exist.\n   */\n  MXNET_DLL static bool Remove(const std::string& name);\n  /*!\n   * \\brief Get the global function by name.\n   * \\param name The name of the function.\n   * \\return pointer to the registered function,\n   *   nullptr if it does not exist.\n   */\n  MXNET_DLL static const PackedFunc* Get(const std::string& name);  // NOLINT(*)\n  /*!\n   * \\brief Get the names of currently registered global function.\n   * \\return The names\n   */\n  MXNET_DLL static std::vector<std::string> ListNames();\n\n  // Internal class.\n  struct Manager;\n\n protected:\n  /*! \\brief name of the function */\n  std::string name_;\n  /*! \\brief internal packed function */\n  PackedFunc func_;\n  friend struct Manager;\n};\n\n/*! \\brief helper macro to supress unused warning */\n#if defined(__GNUC__)\n#define MXNET_ATTRIBUTE_UNUSED __attribute__((unused))\n#else\n#define MXNET_ATTRIBUTE_UNUSED\n#endif\n\n#define MXNET_STR_CONCAT_(__x, __y) __x##__y\n#define MXNET_STR_CONCAT(__x, __y)  MXNET_STR_CONCAT_(__x, __y)\n\n#define MXNET_FUNC_REG_VAR_DEF \\\n  static MXNET_ATTRIBUTE_UNUSED ::mxnet::runtime::Registry& __mk_##MXNET\n\n/*!\n * \\brief Register a function globally.\n * \\code\n *   TVM_REGISTER_GLOBAL(\"MyPrint\")\n *   .set_body([](TVMArgs args, TVMRetValue* rv) {\n *   });\n * \\endcode\n */\n#define MXNET_REGISTER_GLOBAL(OpName)                     \\\n  MXNET_STR_CONCAT(MXNET_FUNC_REG_VAR_DEF, __COUNTER__) = \\\n      ::mxnet::runtime::Registry::Register(OpName)\n\n}  // namespace runtime\n}  // namespace mxnet\n#endif  // MXNET_RUNTIME_REGISTRY_H_\n"
  },
  {
    "path": "include/mxnet/storage.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file storage.h\n * \\brief Storage manager across multiple devices.\n */\n#ifndef MXNET_STORAGE_H_\n#define MXNET_STORAGE_H_\n\n#include <memory>\n#include <string>\n#include <vector>\n#include \"./base.h\"\n\nnamespace mxnet {\n\n#define MXNET_STORAGE_DEFAULT_PROFILER_SCOPE_CSTR \"<unk>:\"\n#define MXNET_STORAGE_DEFAULT_NAME_CSTR           \"unknown\"\n\n/*!\n * \\brief Storage manager across multiple devices.\n */\nclass Storage {\n public:\n  /*!\n   * \\brief Storage sync object.\n   */\n  struct SyncObj {\n#if MXNET_USE_CUDA\n    /*!\n     * \\brief All the events from the engine variable.\n     */\n    std::vector<std::weak_ptr<cudaEvent_t>> events;\n#endif\n  };\n  /*!\n   * \\brief Storage handle.\n   */\n  struct Handle {\n    /*!\n     * \\brief Pointer to the data.\n     */\n    void* dptr{nullptr};\n    /*!\n     * \\brief Size of the storage.\n     */\n    size_t size{0};\n    /*!\n     * \\brief Context information about device and ID.\n     */\n    Context ctx;\n    /*!\n     * \\brief Id for IPC shared memory\n     */\n    int shared_pid{-1};\n    int shared_id{-1};\n    /*!\n     * \\brief Attributes for tracking storage allocations.\n     */\n    std::string profiler_scope{MXNET_STORAGE_DEFAULT_PROFILER_SCOPE_CSTR};\n    std::string name{MXNET_STORAGE_DEFAULT_NAME_CSTR};\n    /*!\n     * \\brief Used to pass events back and forth between the engine Var\n     * and the storage manager.\n     */\n    SyncObj sync_obj;\n  };\n  /*!\n   * \\brief Allocate a new contiguous memory for a given size.\n   * \\param size Total size of memory in bytes.\n   * \\param ctx Context information about the device and ID.\n   * \\param failsafe Return a handle with a null dptr if out of memory, rather than exit.\n   * \\return Handle struct.\n   */\n  Handle Alloc(size_t size, Context ctx, bool failsafe = false) {\n    Handle hd;\n    hd.size = size;\n    hd.ctx  = ctx;\n    this->Alloc(&hd, failsafe);\n    return hd;\n  }\n  /*!\n   * \\brief Allocate a new contiguous memory for a given size.\n   * \\param handle handle initialized with size and ctx\n   */\n  virtual void Alloc(Handle* handle, bool failsafe = false) = 0;\n  /*!\n   * \\brief Increase ref counter on shared memory.\n   * \\param handle handle to shared memory.\n   */\n  virtual void SharedIncrementRefCount(Handle handle) = 0;\n  /*!\n   * \\brief Free storage.\n   * \\param handle Handle struct.\n   */\n  virtual void Free(Handle handle) = 0;\n  /*!\n   * \\brief Free storage directly, without putting it into memory pool.\n   *  This can synchronization of all previous runned device functions.\n   *\n   *  This function is suitable for conatiner structure with requirement on upsizing\n   *  in the beginning phase of the iteration.\n   *\n   * \\param handle Handle struct.\n   */\n  virtual void DirectFree(Handle handle) = 0;\n  /*!\n   * \\brief Release all memory from device if using a pooled storage manager\n   *\n   * This release all memory from pool storage managers such as\n   * GPUPooledStorageManager and GPUPooledRoundedStorageManager.\n   * For non-pool memory managers this has no effect.\n   */\n  virtual void ReleaseAll(Context ctx) = 0;\n  /*!\n   * \\brief Destructor.\n   */\n  virtual ~Storage() {}\n  /*!\n   * \\brief Returns mutex used by storage manager\n   */\n  std::mutex& GetMutex(Context::DeviceType dev) {\n    if (dev == Context::kCPU) {\n      return cpu_mutex_;\n    } else {\n      return gpu_mutex_;\n    }\n  }\n  /*!\n   * \\return Storage singleton.\n   */\n  static Storage* Get();\n  /*!\n   * \\brief Get shared pointer reference to storage singleton.\n   *  Most user should not call this function.\n   *  This function is called by another singleton X who requires\n   *  Storage to be destructed after X.\n   *\n   * \\return A shared pointer to Storage singleton.\n   */\n  static const std::shared_ptr<Storage>& _GetSharedRef();\n\n private:\n  std::mutex cpu_mutex_;\n  std::mutex gpu_mutex_;\n};  // class Storage\n}  // namespace mxnet\n#endif  // MXNET_STORAGE_H_\n"
  },
  {
    "path": "include/mxnet/tensor_blob.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file tensor_blob.h\n * \\brief TBlob class that holds common representation of\n *  arbirary dimension tensor, can be used to transformed\n *  to normal fixed dimenson tensor\n * \\author Tianqi Chen\n */\n#ifndef MXNET_TENSOR_BLOB_H_\n#define MXNET_TENSOR_BLOB_H_\n\n#include <dmlc/logging.h>\n#include <dmlc/json.h>\n#include <dlpack/dlpack.h>\n#include <vector>\n#include <iostream>\n#include <utility>\n#include <algorithm>\n#include \"./base.h\"\n\nnamespace mxnet {\n\n// redefine DLPack enumeration to be backward compatible.\nconstexpr const int kCPU = kDLCPU;\nconstexpr const int kGPU = kDLGPU;\n// extension type code under TVM function.\n// Currently NNVM reserved 16 to 19 type code from TVM\n// 16, 17, 18 is used by NNVM compiler already.\n// Pick code 19 for MXNet NDArray\nconstexpr const int kTVMNDArrayTypeCode = 19;\n\n/* Forward declaration for friend declaration in TBlob */\nclass NDArray;\n\n/*!\n * \\brief tensor blob class that can be used to hold tensor of any dimension,\n *  any device and any data type,\n *  This is a weak type that can be used to transfer data through interface\n *  TBlob itself doesn't involve any arithmetic operations,\n *  but it can be converted to tensor of fixed dimension for further operations\n *\n *  Like tensor, this data structure is like a pointer class and do not\n *  implicit allocated, de-allocate space.\n *  This data structure can be helpful to hold tensors of different dimensions\n *  and wait for further processing\n */\nclass TBlob {\n  friend class NDArray;\n\n public:\n  /*! \\brief pointer to the data */\n  void* dptr_;\n  /*! \\brief shape of the tensor */\n  mxnet::TShape shape_;\n  /*! \\brief type flag of the tensor blob */\n  int type_flag_;\n\n  /*! \\brief default constructor, default copy assign will work */\n  TBlob(void) : dptr_(nullptr), type_flag_(mshadow::DataType<real_t>::kFlag) {\n    SetDLTensor(cpu::kDevMask, 0);\n  }\n  /*!\n   * \\brief constructor that construct TBlob from contiguous memory\n   * \\param dptr the pointer to the memory\n   * \\param shape the shape of the data\n   * \\param dev_mask the device mask, can be cpu::kDevMask or gpu::kDevMask\n   * \\param dev_id the device id\n   */\n  template <typename DType>\n  TBlob(DType* dptr, const mxnet::TShape& shape, int dev_mask, int dev_id = -1)\n      : dptr_(dptr), shape_(shape), type_flag_(mshadow::DataType<DType>::kFlag) {\n    SetDLTensor(dev_mask, dev_id);\n  }\n  /*!\n   * \\brief constructor that construct TBlob from contiguous memory\n   * \\param dptr the pointer to the memory\n   * \\param shape the shape of the data\n   * \\param dev_mask the device mask, can be cpu::kDevMask or gpu::kDevMask\n   * \\param type_flag the type flag. Can be one of enum mshadow::dtype\n   * \\param dev_id the device id\n   */\n  TBlob(void* dptr, const mxnet::TShape& shape, int dev_mask, int type_flag, int dev_id = -1)\n      : dptr_(dptr), shape_(shape), type_flag_(type_flag) {\n    SetDLTensor(dev_mask, dev_id);\n  }\n  /*!\n   * \\brief constructor that construct TBlob from DLTensor\n   * \\param DLTensor Object\n   */\n  explicit TBlob(const DLTensor& dltensor)\n      : dptr_(dltensor.data),\n        shape_(mxnet::TShape(dltensor.shape, dltensor.shape + dltensor.ndim)),\n        type_flag_(DLDataTypeTransform(dltensor.dtype)),\n        dltensor_(dltensor) {\n    // compactness check for DLTensor\n    if (dltensor.strides != nullptr) {\n      // check strides\n      const int& ndim        = dltensor.ndim;\n      const int64_t* shape   = dltensor.shape;\n      const int64_t* strides = dltensor.strides;\n      if (ndim >= 1) {\n        bool err = false;\n        if (strides[ndim - 1] != 1) {\n          err = true;\n        } else {\n          for (int i = ndim - 2; i >= 0; --i) {\n            if (strides[i] != shape[i + 1] * strides[i + 1]) {\n              err = true;\n              break;\n            }\n          }\n        }\n        if (err) {\n          LOG(FATAL) << \"Unsupported DLPack because MXNet only support compact tensor now\";\n        }\n      }\n    }\n  }\n  /*!\n   * \\brief constructor from tensor\n   * \\param src source tensor\n   * \\tparam Device which device the tensor is on\n   * \\tparam dim tensor dimension\n   * \\tparam DType the type of elements in the tensor\n   */\n  template <typename Device, int dim, typename DType>\n  TBlob(const mshadow::Tensor<Device, dim, DType>& src) {  // NOLINT(*)\n    *this = src;\n  }\n  /*!\n   * \\brief constructor from TBlob (copy constructor)\n   * \\param src source TBlob\n   */\n  TBlob(const TBlob& src) : dptr_(src.dptr_), shape_(src.shape_), type_flag_(src.type_flag_) {\n    this->SetDLTensor(src.dev_mask(), src.dev_id());\n  }\n  /*!\n   * \\brief assignment from tensor\n   * \\param src source tensor\n   * \\tparam Device which device the tensor is on\n   * \\tparam dim tensor dimension\n   * \\tparam DType the type of elements in the tensor\n   * \\return reference of self\n   */\n  template <typename Device, int dim, typename DType>\n  inline TBlob& operator=(const mshadow::Tensor<Device, dim, DType>& src) {\n    dptr_      = src.dptr_;\n    shape_     = src.shape_;\n    type_flag_ = mshadow::DataType<DType>::kFlag;\n    SetDLTensor(Device::kDevMask, -1);\n    return *this;\n  }\n  /*!\n   * \\brief assignment from TBlob (copy assignment)\n   * \\param src source TBlob\n   * \\return reference of self\n   */\n  inline TBlob& operator=(const TBlob& src) {\n    dptr_      = src.dptr_;\n    shape_     = src.shape_;\n    type_flag_ = src.type_flag_;\n    SetDLTensor(src.dev_mask(), src.dev_id());\n    return *this;\n  }\n  /*!\n   * \\return whether the tensor's memory is continuous\n   */\n  inline bool CheckContiguous(void) const {\n    return true;\n  }\n  /*!\n   * \\brief reshape to shape\n   * \\param shape desired shape\n   * \\return reshaped blob\n   */\n  inline TBlob reshape(const mxnet::TShape& shape) const {\n    CHECK_EQ(this->shape_.Size(), shape.Size())\n        << \"Shape size mismatch \" << this->shape_.Size() << \" v.s. \" << shape.Size();\n    TBlob ret(this->dptr_, shape, this->dev_mask(), this->type_flag_, this->dev_id());\n    return ret;\n  }\n  /*!\n   * \\brief flatten the tensor to 2 dimension, collapse the higher dimensions together\n   * \\param stream the possible stream target tensor should reside on\n   * \\tparam Device which device the tensor is on\n   * \\tparam DType the type of elements in the tensor\n   * \\return tensor after flatten\n   */\n  template <typename Device, typename DType>\n  inline mshadow::Tensor<Device, 2, DType> FlatTo2D(\n      mshadow::Stream<Device>* stream = nullptr) const {\n    CHECK(Device::kDevMask == this->dev_mask())\n        << \"TBlob.get: device type do not match specified type\";\n    CHECK(mshadow::DataType<DType>::kFlag == type_flag_)\n        << \"TBlob.get_with_shape: data type do not match specified type. \"\n        << \"Expected: \" << mshadow::dtype_string(type_flag_) << \" v.s. given \"\n        << mshadow::dtype_string(mshadow::DataType<DType>::kFlag);\n    return mshadow::Tensor<Device, 2, DType>(static_cast<DType*>(dptr_), shape_.FlatTo2D(), stream);\n  }\n  /*!\n   * \\brief flatten the tensor to 1 dimension, collapse all the dimensions together.\n   * \\param stream the possible stream target tensor should reside on\n   * \\tparam Device which device the tensor is on\n   * \\tparam DType the type of elements in the tensor\n   * \\return tensor after flatten\n   */\n  template <typename Device, typename DType>\n  inline mshadow::Tensor<Device, 1, DType> FlatTo1D(\n      mshadow::Stream<Device>* stream = nullptr) const {\n    return this->get_with_shape<Device, 1, DType>(mshadow::Shape1(shape_.Size()), stream);\n  }\n  /*! \\brief return number of dimension of the tensor inside */\n  inline int ndim(void) const {\n    return shape_.ndim();\n  }\n  /*!\n   * \\brief return size of i-th dimension, start counting from highest dimension.\n   * return type needs to be a signed integer.\n   * \\param idx the dimension count from the highest dimensin\n   * \\return the size. -1 means unknown size to support zero-size tensor.\n   */\n  inline index_t size(index_t idx) const {\n    return shape_[idx];\n  }\n  /*! \\brief total number of elements in the tensor */\n  inline size_t Size(void) const {\n    return shape_.Size();\n  }\n  /*! \\brief get pointer in dtype */\n  template <typename DType>\n  inline DType* dptr() const {\n    CHECK(mshadow::DataType<DType>::kFlag == type_flag_)\n        << \"TBlob.get_with_shape: data type do not match specified type. \"\n        << \"Expected: \" << mshadow::dtype_string(type_flag_) << \" v.s. given \"\n        << mshadow::dtype_string(mshadow::DataType<DType>::kFlag);\n    return static_cast<DType*>(dptr_);\n  }\n  /*! \\brief device mask of the corresponding device */\n  inline int dev_mask() const {\n    return dltensor_.ctx.device_type;\n  }\n  /*! \\brief device index of the corresponding device */\n  inline int dev_id() const {\n    return dltensor_.ctx.device_id;\n  }\n  /*!\n   * \\brief return the corresponding DLTensor\n   * \\return the address of internal DLTensor\n   */\n  inline const DLTensor& dltensor() const {\n    return dltensor_;\n  }\n\n  /*!\n   * \\brief fetch the tensor, with respect to specific dimension\n   * if dim do not match the stored dimension, an error will be issued\n   * \\return the tensor requested\n   * \\param stream the possible stream target tensor should reside on\n   * \\tparam Device which device the tensor is on\n   * \\tparam dim dimension of the tensor\n   * \\tparam DType the type of elements in the tensor\n   */\n  template <typename Device, int dim, typename DType>\n  inline mshadow::Tensor<Device, dim, DType> get(mshadow::Stream<Device>* stream = nullptr) const {\n    CHECK(Device::kDevMask == this->dev_mask())\n        << \"TBlob.get: device type do not match specified type\";\n    return mshadow::Tensor<Device, dim, DType>(\n        dptr<DType>(), shape_.get<dim>(), shape_[shape_.ndim() - 1], stream);\n  }\n  /*!\n   * \\brief fetch a tensor in given shape\n   *  If size do not match the stored size, an error will be issued\n   * \\return the tensor requested\n   * \\param shape the shape required\n   * \\param stream the possible stream target tensor should reside on\n   * \\tparam Device which device the tensor is on\n   * \\tparam dim dimension of the tensor\n   * \\tparam DType the type of elements in the tensor\n   */\n  template <typename Device, int dim, typename DType>\n  inline mshadow::Tensor<Device, dim, DType> get_with_shape(\n      const mshadow::Shape<dim>& shape,\n      mshadow::Stream<Device>* stream = nullptr) const {\n    CHECK(Device::kDevMask == this->dev_mask())\n        << \"TBlob.get: device type do not match specified type\";\n    CHECK_EQ(this->CheckContiguous(), true) << \"TBlob.get_reshape: must be contiguous\";\n    CHECK_EQ(this->shape_.Size(), static_cast<size_t>(shape.Size()))\n        << \"TBlob.get_with_shape: new and old shape do not match total elements\";\n    return mshadow::Tensor<Device, dim, DType>(dptr<DType>(), shape, shape[dim - 1], stream);\n  }\n  /*!\n   * \\brief flatten the tensor to 3 dimension,\n   *  collapse the dimension before and after specified axis.\n   * \\param axis The axis specified.\n   * \\param stream the possible stream target tensor should reside on\n   * \\tparam Device which device the tensor is on\n   * \\tparam DType the type of elements in the tensor\n   * \\return tensor after flatten\n   */\n  template <typename Device, typename DType>\n  inline mshadow::Tensor<Device, 3, DType> FlatTo3D(\n      int axis,\n      mshadow::Stream<Device>* stream = nullptr) const {\n    return this->get_with_shape<Device, 3, DType>(this->shape_.FlatTo3D(axis), stream);\n  }\n  /*!\n   * \\brief flatten the tensor to 3 dimension,\n   *  collapse the dimension: [0, axis_begin), [axis_begin, axis_end], (axis_end, ndim).\n   * \\param axis_begin The beginning axis specified.\n   * \\param axis_end The ending axis specified.\n   * \\param stream the possible stream target tensor should reside on\n   * \\tparam Device which device the tensor is on\n   * \\tparam DType the type of elements in the tensor\n   * \\return tensor after flatten\n   */\n  template <typename Device, typename DType>\n  inline mshadow::Tensor<Device, 3, DType>\n  FlatTo3D(int axis_begin, int axis_end, mshadow::Stream<Device>* stream = nullptr) const {\n    return this->get_with_shape<Device, 3, DType>(this->shape_.FlatTo3D(axis_begin, axis_end),\n                                                  stream);\n  }\n  /*!\n   * \\brief flatten the tensor to specified number of dimensions,\n   *  collapse the highest dimensions or pad with higher dimensions\n   * \\param stream the possible stream target tensor should reside on\n   * \\tparam Device which device the tensor is on\n   * \\tparam dim desired number of dimensions of returned tensor\n   * \\tparam DType the type of elements in the tensor\n   * \\return tensor after flatten\n   */\n  template <typename Device, int dim, typename DType>\n  inline mshadow::Tensor<Device, dim, DType> FlatToKD(\n      mshadow::Stream<Device>* stream = nullptr) const {\n    mshadow::Shape<dim> shape;\n    shape[0] = 1;\n    // Pad higher dimensions in case dim > ndim()\n    for (int i = 0; i < dim - ndim(); ++i) {\n      shape[i] = 1;\n    }\n    // Collapse higher dimensions in case dim < ndim()\n    for (int i = 0; i < ndim() - dim + 1; ++i) {\n      shape[0] *= shape_[i];\n    }\n    // Preserve lower dimensions.\n    for (int i = std::max(0, ndim() - dim + 1); i < ndim(); ++i) {\n      shape[i - ndim() + dim] = shape_[i];\n    }\n    return this->get_with_shape<Device, dim, DType>(shape, stream);\n  }\n\n private:\n  static DLDataType DTypeTransform(int type_flag) {\n    switch (type_flag) {\n      case mshadow::kFloat32:\n        return DLDataType{kDLFloat, 32, 1};\n      case mshadow::kFloat64:\n        return DLDataType{kDLFloat, 64, 1};\n      case mshadow::kFloat16:\n        return DLDataType{kDLFloat, 16, 1};\n      case mshadow::kBfloat16:\n        return DLDataType{kDLBfloat, 16, 1};\n      case mshadow::kUint8:\n        return DLDataType{kDLUInt, 8, 1};\n      case mshadow::kInt32:\n        return DLDataType{kDLInt, 32, 1};\n      case mshadow::kInt8:\n        return DLDataType{kDLInt, 8, 1};\n      case mshadow::kInt64:\n        return DLDataType{kDLInt, 64, 1};\n      case mshadow::kBool:\n        return DLDataType{kDLUInt, 1, 1};\n      case mshadow::kInt16:\n        return DLDataType{kDLInt, 16, 1};\n      case mshadow::kUint16:\n        return DLDataType{kDLUInt, 16, 1};\n      case mshadow::kUint32:\n        return DLDataType{kDLUInt, 32, 1};\n      case mshadow::kUint64:\n        return DLDataType{kDLUInt, 64, 1};\n      default: {\n        LOG(FATAL) << \"Unknown type_flag=\" << type_flag;\n        return DLDataType();\n      }\n    }\n  }\n  static int DLDataTypeTransform(DLDataType dldata_type) {\n    if (dldata_type.lanes != 1) {\n      LOG(FATAL) << \"Unsupported DLDataType whose lanes != 1\";\n    }\n    switch (dldata_type.code) {\n      case kDLFloat:\n        switch (dldata_type.bits) {\n          case 16:\n            return mshadow::kFloat16;\n          case 32:\n            return mshadow::kFloat32;\n          case 64:\n            return mshadow::kFloat64;\n        }\n        break;\n      case kDLBfloat:\n        switch (dldata_type.bits) {\n          case 16:\n            return mshadow::kBfloat16;\n        }\n        break;\n      case kDLUInt:\n        switch (dldata_type.bits) {\n          case 1:\n            return mshadow::kBool;\n          case 8:\n            return mshadow::kUint8;\n          case 16:\n            return mshadow::kUint16;\n          case 32:\n            return mshadow::kUint32;\n          case 64:\n            return mshadow::kUint64;\n        }\n        break;\n      case kDLInt:\n        switch (dldata_type.bits) {\n          case 8:\n            return mshadow::kInt8;\n          case 16:\n            return mshadow::kInt16;\n          case 32:\n            return mshadow::kInt32;\n          case 64:\n            return mshadow::kInt64;\n        }\n        break;\n    }\n    LOG(FATAL) << \"Unknown DLDataType{\" << dldata_type.code << \", \" << dldata_type.bits << \", \"\n               << dldata_type.lanes << \"}\";\n    return mshadow::kFloat32;\n  }\n\n  inline void SetDLTensor(int dev_mask, int dev_id) {\n    dltensor_.data        = dptr_;\n    dltensor_.ctx         = DLContext{static_cast<DLDeviceType>(dev_mask), dev_id};\n    dltensor_.ndim        = shape_.ndim();\n    dltensor_.dtype       = DTypeTransform(type_flag_);\n    dltensor_.shape       = shape_.data();\n    dltensor_.strides     = nullptr;\n    dltensor_.byte_offset = 0;\n  }\n\n private:\n  /*! \\brief corresponding DLTensor of this TBlob */\n  DLTensor dltensor_;\n};\n}  // namespace mxnet\n\nnamespace dmlc {\n// Add a few patches to support mxnet::TShape in dmlc/parameter.\nDMLC_DECLARE_TYPE_NAME(mxnet::TShape, \"Shape(tuple)\");\nDMLC_DECLARE_TYPE_NAME(mxnet::Tuple<int>, \"Shape(tuple)\");\nDMLC_DECLARE_TYPE_NAME(mxnet::Tuple<dmlc::optional<int>>, \"Shape(tuple)\");\nDMLC_DECLARE_TYPE_NAME(nnvm::Tuple<int>, \"Shape(tuple)\");\nDMLC_DECLARE_TYPE_NAME(nnvm::Tuple<dmlc::optional<int>>, \"Shape(tuple)\");\n\nnamespace parameter {\n\ntemplate <>\nclass FieldEntry<mxnet::TShape> : public FieldEntryBase<FieldEntry<mxnet::TShape>, mxnet::TShape> {\n public:\n  FieldEntry() : enforce_nonzero_(false), expect_ndim_(0) {}\n  // parent class\n  typedef FieldEntryBase<FieldEntry<mxnet::TShape>, mxnet::TShape> Parent;\n\n  virtual void Check(void* head) const {\n    Parent::Check(head);\n    mxnet::TShape& v = this->Get(head);\n    if (expect_ndim_ != 0 && v.ndim() != expect_ndim_) {\n      std::ostringstream os;\n      os << \"value \" << v << \"for Parameter \" << this->key_\n         << \" has wrong dimensions, expected dimension=\" << expect_ndim_;\n      throw dmlc::ParamError(os.str());\n    }\n    if (enforce_nonzero_) {\n      for (int i = 0; i < v.ndim(); ++i) {\n        if (v[i] == 0U) {\n          std::ostringstream os;\n          os << \"value \" << v << \"for Parameter \" << this->key_\n             << \" is invalid, the input shape must be nonzero in all dimensions\";\n          throw dmlc::ParamError(os.str());\n        }\n      }\n    }\n  }\n  inline FieldEntry<mxnet::TShape>& enforce_nonzero() {\n    this->enforce_nonzero_ = true;\n    return this->self();\n  }\n  inline FieldEntry<mxnet::TShape>& set_expect_ndim(int ndim) {\n    expect_ndim_ = ndim;\n    return this->self();\n  }\n\n private:\n  // whether all the entries need to be nonzero\n  bool enforce_nonzero_;\n  // expected number of dimension, default = 0 means no restriction.\n  int expect_ndim_;\n};\n\n}  // namespace parameter\n}  // namespace dmlc\n\n#endif  // MXNET_TENSOR_BLOB_H_\n"
  },
  {
    "path": "include/mxnet/tuple.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*!\n * \\file mxnet/tuple.h\n * \\brief Data structure Tuple and TShape to store dynamic sized shapes.\n */\n#ifndef MXNET_TUPLE_H_\n#define MXNET_TUPLE_H_\n\n#include <vector>\n#include <type_traits>\n#include <algorithm>\n#include <utility>\n#include <iostream>\n#include <string>\n#include \"nnvm/op_attr_types.h\"\n#include \"nnvm/graph_attr_types.h\"\n#include \"nnvm/graph.h\"\n#include \"nnvm/pass.h\"\n#include \"runtime/object.h\"\n#include \"runtime/ffi_helper.h\"\n#include \"node/container.h\"\n#include \"ir/expr.h\"\n\nnamespace mxnet {\n\n/*!\n * \\brief A dynamic sized array data structure that is optimized for storing\n * small number of elements with same type.\n *\n *  Data will be stored in stack when number of elements is small.\n *  It is suitable to hold shape of Tensor.\n *\n *  The ndim of a valid tuple is an integer in range [0, inf).\n *  ndim = 0 means the tuple is empty.\n *\n * \\tparam ValueType The type of data stored inside tuple.\n * \\sa TShape\n */\ntemplate <typename ValueType>\nclass Tuple {\n public:\n  /*! \\brief default constructor */\n  Tuple() = default;\n  /*! \\brief destructor */\n  inline ~Tuple() {\n    delete[] data_heap_;\n  }\n  /*!\n   * constructor to construct a tuple with all `value`.\n   * \\param ndim the number of dimension\n   * \\param value the dimension size for all dims\n   */\n  inline Tuple(const int ndim, const dim_t value) {  // NOLINT(*)\n    this->SetDim(ndim);\n    if (ndim > 0) {\n      std::fill_n(begin(), ndim, value);\n    }\n  }\n  /*!\n   * \\brief copy constructor from another tuple\n   * \\param s the source tuple\n   */\n  inline Tuple(const Tuple<ValueType>& s) {\n    if (s.ndim() == -1) {\n      this->SetDim(-1);\n    } else {\n      this->assign(s.begin(), s.end());\n    }\n  }\n  /*!\n   * \\brief constructor from initializer list\n   * \\param init the initializer_list\n   */\n  inline Tuple(std::initializer_list<ValueType> init) {\n    this->assign(init.begin(), init.end());\n  }\n  /*!\n   * \\brief constructor from vector\n   * \\param init the vector\n   */\n  inline Tuple(std::vector<ValueType> init) {  // NOLINT(runtime/explicit)\n    this->assign(init.begin(), init.end());\n  }\n  /*!\n   * \\brief move constructor from Tuple\n   * \\param src the source shape\n   */\n\n  inline Tuple(Tuple<ValueType>&& src) {  // NOLINT(runtime/explicit)\n    this->swap(src);\n  }\n  /*!\n   * \\brief construct the Tuple from content of iterator\n   * \\param begin the beginning of iterator\n   * \\param end end the end of the iterator\n   * \\tparam RandomAccessIterator iterator type\n   */\n  template <typename RandomAccessIterator>\n  inline Tuple(RandomAccessIterator begin, RandomAccessIterator end) {\n    this->assign(begin, end);\n  }\n\n  inline explicit Tuple(const runtime::ObjectRef& src) {\n    using namespace runtime;\n    ADT adt = Downcast<ADT, ObjectRef>(src);\n    this->SetDim(adt.size());\n    for (int i = 0; i < ndim_; ++i) {\n      this->begin()[i] = Downcast<Integer, ObjectRef>(adt[i])->value;\n    }\n  }\n\n  /*!\n   * \\brief Assign content to tuple from iterator.\n   * \\param begin the beginning of iterator\n   * \\param end end the end of the iterator\n   * \\tparam RandomAccessIterator iterator type\n   */\n  template <typename RandomAccessIterator>\n  inline void assign(RandomAccessIterator begin, RandomAccessIterator end) {\n    this->SetDim(end - begin);\n    CHECK_GE(ndim(), 0);\n    std::copy(begin, end, this->begin());\n  }\n  /*!\n   * \\brief Swap current object with other\n   * \\param other another object to be swapped.\n   */\n  inline void swap(Tuple<ValueType>& other) {  // NOLINT(*)\n    std::swap(ndim_, other.ndim_);\n    std::swap(num_heap_allocated_, other.num_heap_allocated_);\n    std::swap(data_stack_, other.data_stack_);\n    std::swap(data_heap_, other.data_heap_);\n  }\n  /*!\n   * \\brief assignment from another tuple.\n   * \\param src source tuple\n   * \\return reference of self\n   */\n  inline Tuple<ValueType>& operator=(const Tuple<ValueType>& src) {\n    if (src.ndim() == -1) {\n      this->SetDim(-1);\n    } else {\n      this->assign(src.begin(), src.end());\n    }\n    return *this;\n  }\n  /*!\n   * \\brief assignment from rvalue of another tuple.\n   * \\param src source tuple\n   * \\return reference of self\n   */\n  inline Tuple<ValueType>& operator=(Tuple<ValueType>&& src) {\n    Tuple<ValueType>(std::move(src)).swap(*this);\n    return *this;\n  }\n  /*!\n   * \\brief assignment from initializer list\n   * \\param init the source initializer list\n   * \\return reference of self\n   */\n  inline Tuple<ValueType>& operator=(std::initializer_list<ValueType> init) {\n    this->assign(init.begin(), init.end());\n    return *this;\n  }\n  /*!\n   * \\return whether two tuple equals\n   * \\param s the tuple to compare against\n   */\n  inline bool operator==(const Tuple<ValueType>& s) const {\n    if (ndim_ != s.ndim_)\n      return false;\n    if (ndim() == -1)\n      return true;\n    return std::equal(begin(), end(), s.begin());\n  }\n  /*!\n   * \\return whether two tuple not equal\n   * \\param s the tuple to compare against\n   */\n  inline bool operator!=(const Tuple<ValueType>& s) const {\n    return !(*this == s);\n  }\n  /*! \\return the begin data pointer to content of the tuple */\n  inline const ValueType* begin() const {\n    return ndim_ <= kStackCache ? data_stack_ : data_heap_;\n  }\n  /*! \\return the begin data pointer to content of the tuple */\n  inline ValueType* begin() {\n    return ndim_ <= kStackCache ? data_stack_ : data_heap_;\n  }\n  /*! \\return the data pointer to end of the tuple */\n  inline const ValueType* end() const {\n    return ndim_ <= kStackCache ? (data_stack_ + ndim_) : (data_heap_ + ndim_);\n  }\n  /*! \\return the data pointer to end the tuple */\n  inline ValueType* end() {\n    return ndim_ <= kStackCache ? (data_stack_ + ndim_) : (data_heap_ + ndim_);\n  }\n  /*! \\return number of dimension of the tuple */\n  inline int ndim() const {\n    return ndim_;\n  }\n  /*!\n   * \\brief get corresponding index\n   * \\param i dimension index\n   * \\return the corresponding dimension size\n   */\n  inline ValueType& operator[](int i) {\n// it fixes the false alarm of assuming signed overflow does not occur\n// when assuming that (X - c) > X is always false [-Werror=strict-overflow]\n#pragma GCC diagnostic push\n#pragma GCC diagnostic ignored \"-Wstrict-overflow\"\n    CHECK(i >= 0 && i < ndim()) << \"index = \" << i << \" must be in range [0, \" << ndim() << \")\";\n#pragma GCC diagnostic pop\n    return begin()[i];\n  }\n  /*!\n   * \\brief get corresponding index\n   * \\param i dimension index\n   * \\return the corresponding dimension size\n   */\n  inline const ValueType& operator[](int i) const {\n// it fixes the false alarm of assuming signed overflow does not occur\n// when assuming that (X - c) > X is always false [-Werror=strict-overflow]\n#pragma GCC diagnostic push\n#pragma GCC diagnostic ignored \"-Wstrict-overflow\"\n    CHECK(i >= 0 && i < ndim()) << \"index = \" << i << \" must be in range [0, \" << ndim() << \")\";\n#pragma GCC diagnostic pop\n    return begin()[i];\n  }\n  /*!\n   * \\brief Save Tuple to JSON.\n   * \\param writer JSONWriter\n   */\n  inline void Save(dmlc::JSONWriter* writer) const {\n    std::vector<ValueType> tmp(begin(), end());\n    writer->Write(tmp);\n  }\n  /*!\n   * \\brief Load Tuple from JSON.\n   * \\param reader JSONReader\n   */\n  inline void Load(dmlc::JSONReader* reader) {\n    std::vector<ValueType> tmp;\n    reader->Read(&tmp);\n    this->assign(tmp.begin(), tmp.end());\n  }\n  /*!\n   * \\brief allow output string of tuple to ostream\n   * \\param os the output stream\n   * \\param t the tuple\n   * \\return the ostream\n   */\n  friend std::ostream& operator<<(std::ostream& os, const Tuple<ValueType>& t) {\n    if (t.ndim() == -1) {\n      // If t is an unknown shape, return string \"None\".\n      // This is consistent with returning unknown shape in Python and generating\n      // C++ operator APIs by OpWrapperGenerator.py (defaultString) in cpp-package.\n      os << \"None\";\n      return os;\n    }\n    os << '[';\n    const ValueType* begin = t.begin();\n    const ValueType* end   = t.end();\n    for (const ValueType* it = begin; it != end; ++it) {\n      if (it != begin)\n        os << ',';\n      os << *it;\n    }\n    os << ']';\n    return os;\n  }\n  /*!\n   * \\brief read tuple from the istream\n   * \\param is the input stream\n   * \\param t The tuple\n   * \\return the istream\n   */\n  friend std::istream& operator>>(std::istream& is, Tuple<ValueType>& t) {\n    // get (\n    while (true) {\n      char ch = is.peek();\n      if (isdigit(ch) || ch == '-') {\n        ValueType idx;\n        if (is >> idx) {\n          t.assign(&idx, &idx + 1);\n        }\n        return is;\n      }\n      is.get();\n      if (ch == '(' || ch == '[')\n        break;\n      if (!isspace(ch)) {\n        if (ch == 'N') {\n          std::string tmp_val;\n          is >> tmp_val;\n          if (tmp_val == \"one\") {  // is stores \"None\"\n            t.SetDim(-1);\n            return is;\n          }\n        }\n        is.setstate(std::ios::failbit);\n        return is;\n      }\n    }\n    // Handle empty tuple. A tensor whose shape is an empty tuple\n    // represents a scalar with ndim = 0.\n    while (isspace(is.peek())) {\n      is.get();\n    }\n    if (is.peek() == ')' || is.peek() == ']') {\n      is.get();\n      t.SetDim(0);\n      return is;\n    }\n    // Handle non-empty tuple\n    ValueType idx;\n    std::vector<ValueType> tmp;\n    while (is >> idx) {\n      tmp.push_back(idx);\n      char ch;\n      do {\n        ch = is.get();\n      } while (isspace(ch));\n      if (std::is_integral<ValueType>::value && ch == 'L') {\n        ch = is.get();\n      }\n      if (ch == ',') {\n        while (true) {\n          ch = is.peek();\n          if (isspace(ch)) {\n            is.get();\n            continue;\n          }\n          if (ch == ')' || ch == ']') {\n            is.get();\n            break;\n          }\n          break;\n        }\n        if (ch == ')' || ch == ']')\n          break;\n      } else if (ch == ')' || ch == ']') {\n        break;\n      } else {\n        is.setstate(std::ios::failbit);\n        return is;\n      }\n    }\n    t.assign(tmp.begin(), tmp.end());\n    return is;\n  }\n  /*!\n   * \\brief save the content into binary stream\n   * \\param strm the output stream\n   * \\tparam DType data type that save to\n   * \\tparam TStream any stream type that have write\n   */\n  template <typename DType = ValueType, typename TStream>\n  inline void Save(TStream* strm) const;\n  /*!\n   * \\brief load the content from binary stream\n   * \\param strm the output stream\n   * \\tparam DType data type that load from\n   * \\tparam TStream any stream type that have write\n   * \\return whether the load is successful\n   */\n  template <typename DType = ValueType, typename TStream>\n  inline bool Load(TStream* strm);\n\n protected:\n  // stack cache size\n  static const int kStackCache = 8;\n  /*! \\brief number of dimension of the tuple */\n  int ndim_{0};\n  /*! \\brief number of cells allocated in data_heap_ */\n  int num_heap_allocated_{0};\n  /*! \\brief in stack space used to store shape when it is small */\n  ValueType data_stack_[kStackCache];\n  /*! \\brief space to store shape when dimension is big*/\n  ValueType* data_heap_{nullptr};\n  // internal function to change the dimension\n  inline void SetDim(int ndim) {\n    CHECK_GE(ndim, -1) << \"ndim cannot be less than -1, received \" << ndim;\n    if (ndim > kStackCache && ndim > num_heap_allocated_) {\n      delete[] data_heap_;\n      data_heap_          = new ValueType[ndim];\n      num_heap_allocated_ = ndim;\n    } else if (ndim <= 0 && data_heap_ != nullptr) {\n      delete[] data_heap_;\n      data_heap_          = nullptr;\n      num_heap_allocated_ = 0;\n    }\n    ndim_ = ndim;\n  }\n};\n\n/*! brief check if a shape's ndim is known. */\ninline bool ndim_is_known(const int ndim) {\n  CHECK_GE(ndim, -1) << \"shape ndim must be >= -1, while received \" << ndim;\n  return ndim != -1;\n}\n\n/*! brief check if a shape's dim size is known. */\ninline bool dim_size_is_known(const dim_t dim_size) {\n  CHECK_GE(dim_size, -1) << \"shape dim size must be >= -1, while received \" << dim_size;\n  return dim_size != -1;\n}\n\n/*!\n * \\brief A Shape class that is used to represent shape of each tensor.\n *\n * The ndim of a valid shape is an integer in range [-1, inf).\n * ndim = -1 means the shape information is unknown and need to be inferred.\n * ndim = 0 means the tensor with the shape is a scalar.\n *\n * The dimension size of a valid shape is an integer in range [-1, inf).\n * dim_size = -1 means the size of that dimension is unknown and need to be inferred.\n * dim_size = 0 means that dimension is empty.\n *\n * The definition of ndim = 0 and dim_size = 0 is consistent with NumPy.\n */\nclass TShape : public Tuple<dim_t> {\n public:\n  /*! \\brief default constructor */\n  TShape() {\n    this->SetDim(-1);\n  }\n  /*!\n   * constructor to construct a shape with all `value`.\n   * \\param ndim the number of dimension\n   * \\param value the dimension size for all dims\n   */\n  inline TShape(const int ndim, const dim_t value) {  // NOLINT(*)\n    this->SetDim(ndim);\n    if (ndim > 0) {\n      std::fill_n(begin(), ndim, value);\n    }\n  }\n  /*!\n   * \\brief copy constructor of TShape\n   * \\param s source shape.\n   */\n  inline TShape(const Tuple<dim_t>& s) {  // NOLINT(*)\n    if (s.ndim() == -1) {\n      this->SetDim(-1);\n    } else {\n      this->assign(s.begin(), s.end());\n    }\n  }\n  /*!\n   * \\brief constructor from initializer list\n   * \\param init the initializer_list\n   */\n  inline TShape(std::initializer_list<dim_t> init) {\n    this->assign(init.begin(), init.end());\n  }\n  /*!\n   * \\brief move constructor.\n   * \\param s source shape.\n   */\n  inline TShape(Tuple<dim_t>&& s) {  // NOLINT(*)\n    this->swap(s);\n  }\n  /*!\n   * \\brief construct the Tuple from content of iterator.\n   * This function is enforced with template arguments of random access iterator types.\n   * This is necessary to distinguish from another constructor: TShape(const int, const dim_t).\n   * \\param begin the beginning of iterator\n   * \\param end end the end of the iterator\n   * \\tparam RandomAccessIterator iterator type\n   */\n  template <typename RandomAccessIterator,\n            typename std::enable_if<\n                std::is_same<typename std::iterator_traits<RandomAccessIterator>::iterator_category,\n                             std::random_access_iterator_tag>::value,\n                int>::type = 0>\n  inline TShape(RandomAccessIterator begin, RandomAccessIterator end) {\n    this->assign(begin, end);\n  }\n\n  inline explicit TShape(const ObjectRef& src) : Tuple(src) {}\n  /*!\n   * \\brief assignment function from tshape\n   * \\param src source shape.\n   * \\return self.\n   */\n  inline TShape& operator=(const Tuple<dim_t>& src) {\n    if (src.ndim() == -1) {\n      this->SetDim(-1);\n    } else {\n      this->assign(src.begin(), src.end());\n    }\n    return *this;\n  }\n  /*!\n   * \\brief move assignment function from tshape\n   * \\param src source shape.\n   * \\return self.\n   */\n  inline TShape& operator=(Tuple<dim_t>&& src) {  // NOLINT(*)\n    TShape(std::move(src)).swap(*this);           // NOLINT(*)\n    return *this;\n  }\n  /*! \\return total number of elements in the shape */\n  inline size_t Size() const {\n    CHECK(ndim_is_known(this->ndim())) << \"Shape is unknown.\";\n    dim_t size         = 1;\n    const dim_t *start = begin(), *fin = end();\n    for (const dim_t* it = start; it != fin; ++it) {\n      CHECK(dim_size_is_known(*it)) << \"Shape dim size cannot be a negative value \" << *it;\n      size *= *it;\n    }\n    return size;\n  }\n  /*!\n   * \\return product shape in [dimstart,dimend)\n   * \\param dimstart start dimension\n   * \\param dimend end dimension\n   */\n  inline size_t ProdShape(int dimstart, int dimend) const {\n    CHECK(ndim_is_known(this->ndim())) << \"Shape is unknown.\";\n    CHECK_GE(dimstart, 0) << \"dimstart must be >= 0, while received \" << dimstart;\n    CHECK_LE(dimend, this->ndim())\n        << \"dimend must be <= \" << this->ndim() << \", while received \" << dimend;\n    dim_t num      = 1;\n    const dim_t* d = this->data();\n    for (int i = dimstart; i < dimend; ++i) {\n      CHECK(dim_size_is_known(d[i])) << \"Shape dim size must be known, while received \" << d[i];\n      num *= d[i];\n    }\n    return num;\n  }\n  /*! \\return the begin data pointer to content of the tuple */\n  inline const dim_t* data() const {\n    return begin();\n  }\n  /*! \\return the begin data pointer to content of the tuple */\n  inline dim_t* data() {\n    return begin();\n  }\n#ifdef MSHADOW_XINLINE\n  template <int dim>\n  inline TShape(const mshadow::Shape<dim>& s) {  // NOLINT(*)\n    this->assign(s.shape_, s.shape_ + dim);\n  }\n\n  template <int dim>\n  inline TShape(mshadow::Shape<dim>&& s) {  // NOLINT(*)\n    this->assign(s.shape_, s.shape_ + dim);\n  }\n  /*!\n   * \\brief assignment from shape\n   * \\param shape source shape\n   * \\tparam dim shape dimension\n   * \\return reference of self\n   */\n  template <int dim>\n  inline TShape& operator=(const mshadow::Shape<dim>& shape) {\n    this->assign(shape.shape_, shape.shape_ + dim);\n    return *this;\n  }\n  /*!\n   * \\brief get the shape of tensor specifying dim\n   * \\return the shape requested\n   * \\tparam dim dimension of the tensor\n   */\n  template <int dim>\n  inline mshadow::Shape<dim> get() const {\n    CHECK_EQ(dim, ndim()) << \"dimension do not match target dimension \" << dim << \" vs \" << ndim();\n    const dim_t* d = this->data();\n    mshadow::Shape<dim> s;\n    for (int i = 0; i < dim; ++i) {\n      s[i] = d[i];\n    }\n    return s;\n  }\n  /*!\n   * flatten the higher dimension to second dimension, return a 2D shape\n   * \\return the flat 2d shape\n   */\n  inline mshadow::Shape<2> FlatTo2D(void) const {\n    mshadow::Shape<2> s;\n    CHECK(ndim_is_known(ndim())) << \"shape must have a valid ndim\";\n    if (ndim() == 0)\n      return mshadow::Shape2(1, 1);\n    const dim_t* d = this->data();\n    s.shape_[1]    = d[ndim() - 1];\n    dim_t ymax     = 1;\n    for (int i = 1; i < ndim(); ++i) {\n      ymax *= d[i - 1];\n    }\n    s.shape_[0] = ymax;\n    return s;\n  }\n  /*!\n   * flatten the shape into three parts: [0, axis_begin), [axis_begin, axis_end], (axis_end, ndim)\n   * \\param axis_begin The beginning axis specified.\n   * \\param axis_end The ending axis specified.\n   * \\return the flat 3d shape\n   */\n  inline mshadow::Shape<3> FlatTo3D(int axis_begin, int axis_end) const {\n    CHECK(axis_end >= axis_begin);\n    mshadow::Shape<3> s;\n    CHECK(ndim_is_known(ndim())) << \"shape must have a valid ndim\";\n    if (ndim() == 0)\n      return mshadow::Shape3(1, 1, 1);\n    const dim_t* d = this->data();\n    s.shape_[0]    = 1;\n    s.shape_[1]    = 1;\n    s.shape_[2]    = 1;\n\n    for (int i = 0; i < axis_begin; ++i) {\n      s.shape_[0] *= d[i];\n    }\n    for (int i = axis_begin; i <= axis_end; ++i) {\n      s.shape_[1] *= d[i];\n    }\n    for (int i = axis_end + 1; i < ndim(); ++i) {\n      s.shape_[2] *= d[i];\n    }\n    return s;\n  }\n  /*!\n   * flatten the axis before and after the specified axis, so it becomes 3D tensor\n   * \\param axis The axis specified.\n   * \\return the flat 3d shape\n   */\n  inline mshadow::Shape<3> FlatTo3D(int axis) const {\n    return FlatTo3D(axis, axis);\n  }\n  inline bool operator==(const TShape& s) const {\n    if (ndim() != s.ndim())\n      return false;\n    return std::equal(begin(), end(), s.begin());\n  }\n  inline bool operator!=(const TShape& s) const {\n    return !(*this == s);\n  }\n  /*!\n   * \\return whether two shape equals\n   * \\param s the shape to compare against\n   * \\tparam dim dimension of the shape\n   */\n  template <int dim>\n  inline bool operator==(const mshadow::Shape<dim>& s) const {\n    if (ndim_ != dim)\n      return false;\n    const dim_t* d = dim <= kStackCache ? data_stack_ : data_heap_;\n    for (size_t i = 0; i < dim; ++i) {\n      if (d[i] != s.shape_[i])\n        return false;\n    }\n    return true;\n  }\n  /*!\n   * \\return whether two shape not equals\n   * \\param s the shape to compare against\n   * \\tparam dim dimension of the shape\n   */\n  template <int dim>\n  inline bool operator!=(const mshadow::Shape<dim>& s) const {\n    return !(*this == s);\n  }\n#endif\n};\n\n/*! brief check if a shape's ndim is known. */\ninline bool ndim_is_known(const TShape& x) {\n  return ndim_is_known(x.ndim());\n}\n\n/*! brief check if a shape's dim size is known. */\ninline bool dim_size_is_known(const TShape& x, const int idx) {\n  CHECK(idx >= 0 && idx < x.ndim())\n      << \"idx = \" << idx << \" exceeds shape dimension range [0, \" << x.ndim() << \")\";\n  return dim_size_is_known(x[idx]);\n}\n\n/*! brief check if shape is known using the NumPy compatible definition.\n * zero-dim and zero-size tensors are valid. -1 means unknown.*/\ninline bool shape_is_known(const TShape& x) {\n  if (!ndim_is_known(x))\n    return false;\n  for (int i = 0; i < x.ndim(); ++i) {\n    if (!dim_size_is_known(x, i))\n      return false;\n  }\n  return true;\n}\n\ninline bool shape_is_known(const std::vector<TShape>& shapes) {\n  for (const TShape& shape : shapes) {\n    if (!shape_is_known(shape))\n      return false;\n  }\n  return true;\n}\n\n/*! \\brief helper function to cast type of container elements */\ntemplate <typename SrcIter, typename DstIter>\ninline DstIter ShapeTypeCast(const SrcIter begin, const SrcIter end, DstIter dst_begin) {\n  typedef typename std::iterator_traits<SrcIter>::value_type SrcDType;\n  typedef typename std::iterator_traits<DstIter>::value_type DstDType;\n  auto cast = [](const SrcDType& dim) { return static_cast<DstDType>(dim); };\n  return std::transform(begin, end, dst_begin, cast);\n}\n\n/*! \\brief helper function to transform a container to TShape with type cast */\ntemplate <typename SrcIter>\ninline TShape ShapeTypeCast(const SrcIter begin, const SrcIter end) {\n  size_t ndim = std::distance(begin, end);\n  TShape res(ndim, -1);\n  ShapeTypeCast(begin, end, res.begin());\n  return res;\n}\n\n/*! \\tparam ValueType The type of data stored inside tuple. */\ntemplate <typename ValueType>\ntemplate <typename DType, typename TStream>\ninline void Tuple<ValueType>::Save(TStream* strm) const {\n  strm->Write(&ndim_, sizeof(ndim_));\n  if (typeid(DType) == typeid(ValueType)) {\n    strm->Write(begin(), sizeof(ValueType) * ndim_);\n  } else {\n    std::vector<DType> buffer(ndim_);\n    ShapeTypeCast(begin(), end(), buffer.data());\n    strm->Write(buffer.data(), sizeof(DType) * ndim_);\n  }\n}\n\n/*! \\tparam ValueType The type of data stored inside tuple. */\ntemplate <typename ValueType>\ntemplate <typename DType, typename TStream>\ninline bool Tuple<ValueType>::Load(TStream* strm) {\n  if (strm->Read(&ndim_, sizeof(ndim_)) != sizeof(ndim_))\n    return false;\n  this->SetDim(ndim_);\n  size_t nread = sizeof(DType) * ndim_;\n  if (typeid(DType) == typeid(ValueType)) {\n    if (strm->Read(begin(), nread) != nread)\n      return false;\n  } else {\n    std::vector<DType> buffer(ndim_);\n    if (strm->Read(buffer.data(), nread) != nread)\n      return false;\n    ShapeTypeCast(buffer.begin(), buffer.end(), begin());\n  }\n  return true;\n}\n\n}  // namespace mxnet\n\nnamespace std {\n/*! \\brief hash function for Tuple. */\ntemplate <typename T>\nstruct hash<mxnet::Tuple<T>> {\n  /*! \\brief hash a Tuple into unsigned int */\n  size_t operator()(const mxnet::Tuple<T>& val) const {\n    std::hash<int> hash_int;\n    size_t res = hash_int(val.ndim());\n    for (int i = 0; i < val.ndim(); ++i) {\n      res = dmlc::HashCombine(res, val[i]);\n    }\n    return res;\n  }\n};\n\n/*! \\brief hash function for TShape. */\ntemplate <>\nstruct hash<mxnet::TShape> {\n  /*! \\brief hash a TShape into unsigned int */\n  size_t operator()(const mxnet::TShape& val) const {\n    std::hash<int> hash_int;\n    size_t res = hash_int(val.ndim());\n    for (int i = 0; i < val.ndim(); ++i) {\n      res = dmlc::HashCombine(res, val[i]);\n    }\n    return res;\n  }\n};\n}  // namespace std\n\nnamespace dmlc {\n/*! \\brief description for optional TShape */\nDMLC_DECLARE_TYPE_NAME(optional<mxnet::TShape>, \"Shape or None\");\nDMLC_DECLARE_TYPE_NAME(optional<mxnet::Tuple<int>>, \"Shape or None\");\n// avoid low version of MSVC\n#if !(defined(_MSC_VER) && _MSC_VER < 1900)\ntemplate <typename T>\nstruct type_name_helper<mxnet::Tuple<T>> {\n  static inline std::string value() {\n    return \"tuple of <\" + type_name<T>() + \">\";\n  }\n};\n#endif\n}  // namespace dmlc\n\nnamespace mxnet {\n/*!\n * \\brief The result holder of shape of each NodeEntry in the graph.\n * \\note Stored under graph.attrs[\"shape\"], provided by Pass \"InferShape\"\n *\n * \\code\n *  Graph g = ApplyPass(src_graph, \"InferShape\");\n *  const ShapeVector& shapes = g.GetAttr<ShapeVector>(\"shape\");\n *  // get shape by entry id\n *  TShape entry_shape = shapes[g.indexed_graph().entry_id(my_entry)];\n * \\endcode\n *\n * \\sa FInferShape\n */\nusing ShapeVector = std::vector<mxnet::TShape>;\n\n/*!\n * \\brief Shape inference function.\n *  Update the shapes given the input shape information.\n *  TShape.ndim() == -1 means the shape is still unknown.\n *\n * \\note Register under \"FInferShape\",\n *  by default do not update any shapes.\n *\n *  FInferShape is needed by shape inference\n */\nusing FInferShape = nnvm::FInferNodeEntryAttr<mxnet::TShape>;\n\n}  // namespace mxnet\n\n#endif  // MXNET_TUPLE_H_\n"
  },
  {
    "path": "licenses/BOOST1_0",
    "content": "Boost Software License - Version 1.0 - August 17th, 2003\n\nPermission is hereby granted, free of charge, to any person or organization\nobtaining a copy of the software and accompanying documentation covered by\nthis license (the \"Software\") to use, reproduce, display, distribute,\nexecute, and transmit the Software, and to prepare derivative works of the\nSoftware, and to permit third-parties to whom the Software is furnished to\ndo so, all subject to the following:\n\nThe copyright notices in the Software and this entire statement, including\nthe above license grant, this restriction and the following disclaimer,\nmust be included in all copies of the Software, in whole or in part, and\nall derivative works of the Software, unless such copies or derivative\nworks are solely in the form of machine-executable object code generated by\na source language processor.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT\nSHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE\nFOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,\nARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n"
  },
  {
    "path": "licenses/BSD2",
    "content": "Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:\n\n1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
  },
  {
    "path": "licenses/BSD3-cmake",
    "content": "CMake - Cross Platform Makefile Generator\nCopyright 2000-2020 Kitware, Inc. and Contributors\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions\nare met:\n\n* Redistributions of source code must retain the above copyright\n  notice, this list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright\n  notice, this list of conditions and the following disclaimer in the\n  documentation and/or other materials provided with the distribution.\n\n* Neither the name of Kitware, Inc. nor the names of Contributors\n  may be used to endorse or promote products derived from this\n  software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nHOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
  },
  {
    "path": "licenses/MIT",
    "content": "The MIT License\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE."
  },
  {
    "path": "licenses/OFL1_1",
    "content": "Copyright (c) <dates>, <Copyright Holder> (<URL|email>),\r\nwith Reserved Font Name <Reserved Font Name>.\r\nCopyright (c) <dates>, <additional Copyright Holder> (<URL|email>),\r\nwith Reserved Font Name <additional Reserved Font Name>.\r\nCopyright (c) <dates>, <additional Copyright Holder> (<URL|email>).\r\n\r\nThis Font Software is licensed under the SIL Open Font License, Version 1.1.\r\nThis license is copied below, and is also available with a FAQ at:\r\nhttp://scripts.sil.org/OFL\r\n\r\n\r\n-----------------------------------------------------------\r\nSIL OPEN FONT LICENSE Version 1.1 - 26 February 2007\r\n-----------------------------------------------------------\r\n\r\nPREAMBLE\r\nThe goals of the Open Font License (OFL) are to stimulate worldwide\r\ndevelopment of collaborative font projects, to support the font creation\r\nefforts of academic and linguistic communities, and to provide a free and\r\nopen framework in which fonts may be shared and improved in partnership\r\nwith others.\r\n\r\nThe OFL allows the licensed fonts to be used, studied, modified and\r\nredistributed freely as long as they are not sold by themselves. The\r\nfonts, including any derivative works, can be bundled, embedded, \r\nredistributed and/or sold with any software provided that any reserved\r\nnames are not used by derivative works. The fonts and derivatives,\r\nhowever, cannot be released under any other type of license. The\r\nrequirement for fonts to remain under this license does not apply\r\nto any document created using the fonts or their derivatives.\r\n\r\nDEFINITIONS\r\n\"Font Software\" refers to the set of files released by the Copyright\r\nHolder(s) under this license and clearly marked as such. This may\r\ninclude source files, build scripts and documentation.\r\n\r\n\"Reserved Font Name\" refers to any names specified as such after the\r\ncopyright statement(s).\r\n\r\n\"Original Version\" refers to the collection of Font Software components as\r\ndistributed by the Copyright Holder(s).\r\n\r\n\"Modified Version\" refers to any derivative made by adding to, deleting,\r\nor substituting -- in part or in whole -- any of the components of the\r\nOriginal Version, by changing formats or by porting the Font Software to a\r\nnew environment.\r\n\r\n\"Author\" refers to any designer, engineer, programmer, technical\r\nwriter or other person who contributed to the Font Software.\r\n\r\nPERMISSION & CONDITIONS\r\nPermission is hereby granted, free of charge, to any person obtaining\r\na copy of the Font Software, to use, study, copy, merge, embed, modify,\r\nredistribute, and sell modified and unmodified copies of the Font\r\nSoftware, subject to the following conditions:\r\n\r\n1) Neither the Font Software nor any of its individual components,\r\nin Original or Modified Versions, may be sold by itself.\r\n\r\n2) Original or Modified Versions of the Font Software may be bundled,\r\nredistributed and/or sold with any software, provided that each copy\r\ncontains the above copyright notice and this license. These can be\r\nincluded either as stand-alone text files, human-readable headers or\r\nin the appropriate machine-readable metadata fields within text or\r\nbinary files as long as those fields can be easily viewed by the user.\r\n\r\n3) No Modified Version of the Font Software may use the Reserved Font\r\nName(s) unless explicit written permission is granted by the corresponding\r\nCopyright Holder. This restriction only applies to the primary font name as\r\npresented to the users.\r\n\r\n4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font\r\nSoftware shall not be used to promote, endorse or advertise any\r\nModified Version, except to acknowledge the contribution(s) of the\r\nCopyright Holder(s) and the Author(s) or with their explicit written\r\npermission.\r\n\r\n5) The Font Software, modified or unmodified, in part or in whole,\r\nmust be distributed entirely under this license, and must not be\r\ndistributed under any other license. The requirement for fonts to\r\nremain under this license does not apply to any document created\r\nusing the Font Software.\r\n\r\nTERMINATION\r\nThis license becomes null and void if any of the above conditions are\r\nnot met.\r\n\r\nDISCLAIMER\r\nTHE FONT SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\r\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF\r\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT\r\nOF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE\r\nCOPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\r\nINCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL\r\nDAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\r\nFROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM\r\nOTHER DEALINGS IN THE FONT SOFTWARE.\r\n"
  },
  {
    "path": "plugin/opencv/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\n\"\"\"Opencv plugin for mxnet\"\"\"\nfrom .opencv import *\n\n"
  },
  {
    "path": "plugin/opencv/cv_api.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file cv_api.h\n * \\brief C API for opencv\n * \\author Junyuan Xie\n */\n#include <dmlc/base.h>\n#include <mxnet/base.h>\n#include <mxnet/ndarray.h>\n#include <opencv2/opencv.hpp>\n#include \"cv_api.h\"\n#include \"../../src/c_api/c_api_common.h\"\n\nusing namespace mxnet;\n// http://www.64lines.com/jpeg-width-height\n// Gets the JPEG size from the array of data passed to the function, file reference:\n// http://www.obrador.com/essentialjpeg/headerinfo.htm\nbool get_jpeg_size(const unsigned char* data, mx_uint data_size, mx_uint* width, mx_uint* height) {\n  // Check for valid JPEG image\n  mx_uint i = 0;  // Keeps track of the position within the file\n  if (data[i] == 0xFF && data[i + 1] == 0xD8 && data[i + 2] == 0xFF && data[i + 3] == 0xE0) {\n    i += 4;\n    // Check for valid JPEG header (null terminated JFIF)\n    if (data[i + 2] == 'J' && data[i + 3] == 'F' && data[i + 4] == 'I' && data[i + 5] == 'F' &&\n        data[i + 6] == 0x00) {\n      // Retrieve the block length of the first block since\n      // the first block will not contain the size of file\n      uint16_t block_length = data[i] * 256 + data[i + 1];\n      while (i < data_size) {\n        i += block_length;  // Increase the file index to get to the next block\n        if (i >= data_size)\n          return false;  // Check to protect against segmentation faults\n        if (data[i] != 0xFF)\n          return false;  // Check that we are truly at the start of another block\n        if (data[i + 1] == 0xC0) {\n          // 0xFFC0 is the \"Start of frame\" marker which contains the file size\n          // The structure of the 0xFFC0 block is quite simple\n          // [0xFFC0][ushort length][uchar precision][ushort x][ushort y]\n          *height = data[i + 5] * 256 + data[i + 6];\n          *width  = data[i + 7] * 256 + data[i + 8];\n          return true;\n        } else {\n          i += 2;                                      // Skip the block marker\n          block_length = data[i] * 256 + data[i + 1];  // Go to the next block\n        }\n      }\n      return false;  // If this point is reached then no size was found\n    } else {\n      return false;  // Not a valid JFIF string\n    }\n  } else {\n    return false;  // Not a valid SOI header\n  }\n}\n\nbool get_png_size(const unsigned char* data, mx_uint data_size, mx_uint* width, mx_uint* height) {\n  if (data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E && data[3] == 0x47) {\n    unsigned char const* p = data + 16;\n    *width                 = ((p[0] * 256 + p[1]) * 256 + p[2]) * 256 + p[3];\n    p += 4;\n    *height = ((p[0] * 256 + p[1]) * 256 + p[2]) * 256 + p[3];\n    return true;\n  } else {\n    return false;\n  }\n}\n\nMXNET_DLL int MXCVImdecode(const unsigned char* img,\n                           const mx_uint len,\n                           const int flag,\n                           NDArrayHandle* out) {\n  API_BEGIN();\n  mx_uint dims[3];\n  CHECK_GE(flag, 0) << \"flag must be 0 (grayscale) or 1 (colored).\";\n  dims[2] = flag == 0 ? 1 : 3;\n  if (get_jpeg_size(img, len, dims + 1, dims)) {\n  } else if (get_png_size(img, len, dims + 1, dims)) {\n  } else {\n    LOG(FATAL) << \"Only supports png and jpg.\";\n  }\n  NDArray ndout(mxnet::TShape(dims, dims + 3), Context::CPU(), true, mshadow::kUint8);\n  unsigned char* img_cpy = new unsigned char[len];\n  memcpy(img_cpy, img, sizeof(unsigned char) * len);\n  Engine::Get()->PushSync(\n      [=](RunContext ctx) {\n        ndout.CheckAndAlloc();\n        cv::Mat buf(1, len, CV_8U, img_cpy);\n        cv::Mat dst(dims[0], dims[1], flag == 0 ? CV_8U : CV_8UC3, ndout.data().dptr_);\n#if (CV_MAJOR_VERSION > 3 || (CV_MAJOR_VERSION == 3 && CV_MINOR_VERSION >= 3))\n        cv::imdecode(buf, flag | cv::IMREAD_IGNORE_ORIENTATION, &dst);\n#else\n        cv::imdecode(buf, flag, &dst);\n#endif\n        CHECK(!dst.empty());\n        delete[] img_cpy;\n      },\n      ndout.ctx(),\n      {},\n      {ndout.var()});\n  NDArray* tmp = new NDArray();\n  *tmp         = ndout;\n  *out         = tmp;\n  API_END();\n}\n\nMXNET_DLL int MXCVResize(NDArrayHandle src,\n                         const mx_uint w,\n                         const mx_uint h,\n                         const int interpolation,\n                         NDArrayHandle* out) {\n  API_BEGIN();\n  NDArray ndsrc = *static_cast<NDArray*>(src);\n  CHECK_EQ(ndsrc.shape().ndim(), 3);\n  CHECK_EQ(ndsrc.ctx(), Context::CPU());\n  CHECK_EQ(ndsrc.dtype(), mshadow::kUint8);\n\n  mx_uint dims[3] = {h, w, ndsrc.shape()[2]};\n  NDArray ndout(mxnet::TShape(dims, dims + 3), Context::CPU(), true, mshadow::kUint8);\n\n  Engine::Get()->PushSync(\n      [=](RunContext ctx) {\n        ndout.CheckAndAlloc();\n        cv::Mat buf(\n            ndsrc.shape()[0], ndsrc.shape()[1], dims[2] == 3 ? CV_8UC3 : CV_8U, ndsrc.data().dptr_);\n        cv::Mat dst(h, w, dims[2] == 3 ? CV_8UC3 : CV_8U, ndout.data().dptr_);\n        cv::resize(buf, dst, cv::Size(w, h), 0, 0, interpolation);\n        CHECK(!dst.empty());\n      },\n      ndout.ctx(),\n      {ndsrc.var()},\n      {ndout.var()});\n  NDArray* tmp = new NDArray();\n  *tmp         = ndout;\n  *out         = tmp;\n  API_END();\n}\n\nMXNET_DLL int MXCVcopyMakeBorder(NDArrayHandle src,\n                                 const int top,\n                                 const int bot,\n                                 const int left,\n                                 const int right,\n                                 const int type,\n                                 const double value,\n                                 NDArrayHandle* out) {\n  API_BEGIN();\n  NDArray ndsrc = *static_cast<NDArray*>(src);\n  CHECK_EQ(ndsrc.shape().ndim(), 3);\n  CHECK_EQ(ndsrc.ctx(), Context::CPU());\n  CHECK_EQ(ndsrc.dtype(), mshadow::kUint8);\n\n  int h = ndsrc.shape()[0], w = ndsrc.shape()[1], c = ndsrc.shape()[2];\n  mx_uint dims[3] = {top + h + bot, left + w + right, c};\n  NDArray ndout(mxnet::TShape(dims, dims + 3), Context::CPU(), true, mshadow::kUint8);\n\n  Engine::Get()->PushSync(\n      [=](RunContext ctx) {\n        ndout.CheckAndAlloc();\n        cv::Mat buf(h, w, c == 3 ? CV_8UC3 : CV_8U, ndsrc.data().dptr_);\n        cv::Mat dst(top + h + bot, left + w + right, c == 3 ? CV_8UC3 : CV_8U, ndout.data().dptr_);\n        cv::copyMakeBorder(buf, dst, top, bot, left, right, type, cv::Scalar(value));\n        CHECK(!dst.empty());\n      },\n      ndout.ctx(),\n      {ndsrc.var()},\n      {ndout.var()});\n  NDArray* tmp = new NDArray();\n  *tmp         = ndout;\n  *out         = tmp;\n  API_END();\n}\n"
  },
  {
    "path": "plugin/opencv/cv_api.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file cv_api.h\n * \\brief C API for opencv\n * \\author Junyuan Xie\n */\n#ifndef PLUGIN_OPENCV_CV_API_H_\n#define PLUGIN_OPENCV_CV_API_H_\n\n#include <mxnet/c_api.h>\n\nMXNET_DLL int MXCVImdecode(const unsigned char* img,\n                           const mx_uint len,\n                           const int flag,\n                           NDArrayHandle* out);\n\nMXNET_DLL int MXCVResize(NDArrayHandle src,\n                         const mx_uint w,\n                         const mx_uint h,\n                         const int interpolation,\n                         NDArrayHandle* out);\n\nMXNET_DLL int MXCVcopyMakeBorder(NDArrayHandle src,\n                                 const int top,\n                                 const int bot,\n                                 const int left,\n                                 const int right,\n                                 const int type,\n                                 const double value,\n                                 NDArrayHandle* out);\n\n#endif  // PLUGIN_OPENCV_CV_API_H_\n"
  },
  {
    "path": "plugin/opencv/opencv.mk",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nOPENCV_SRC = $(wildcard plugin/opencv/*.cc)\nPLUGIN_OBJ += $(patsubst %.cc, build/%.o, $(OPENCV_SRC))\nOPENCV_CUSRC = $(wildcard plugin/opencv/*.cu)\nPLUGIN_CUOBJ += $(patsubst %.cu, build/%_gpu.o, $(OPENCV_CUSRC))\n"
  },
  {
    "path": "plugin/opencv/opencv.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=too-many-arguments,no-member,invalid-name\n\n\"\"\"Opencv plugin for mxnet\"\"\"\nimport random\nimport ctypes\nimport cv2\nimport mxnet as mx\nfrom mxnet.base import _LIB\nfrom mxnet.base import mx_uint, NDArrayHandle, check_call\n\ndef imdecode(str_img, flag=1):\n    \"\"\"Decode image from str buffer.\n    Wrapper for cv2.imdecode that uses mx.nd.NDArray\n\n    Parameters\n    ----------\n    str_img : str\n        str buffer read from image file\n    flag : int\n        same as flag for cv2.imdecode\n    Returns\n    -------\n    img : NDArray\n        decoded image in (width, height, channels)\n        with BGR color channel order\n    \"\"\"\n    hdl = NDArrayHandle()\n    check_call(_LIB.MXCVImdecode(ctypes.c_char_p(str_img),\n                                 mx_uint(len(str_img)),\n                                 flag, ctypes.byref(hdl)))\n    return mx.nd.NDArray(hdl)\n\ndef resize(src, size, interpolation=cv2.INTER_LINEAR):\n    \"\"\"Decode image from str buffer.\n    Wrapper for cv2.imresize that uses mx.nd.NDArray\n\n    Parameters\n    ----------\n    src : NDArray\n        image in (width, height, channels)\n    size : tuple\n        target size in (width, height)\n    interpolation : int\n        same as interpolation for cv2.imresize\n\n    Returns\n    -------\n    img : NDArray\n        resized image\n    \"\"\"\n    hdl = NDArrayHandle()\n    check_call(_LIB.MXCVResize(src.handle, mx_uint(size[0]), mx_uint(size[1]),\n                               interpolation, ctypes.byref(hdl)))\n    return mx.nd.NDArray(hdl)\n\ndef copyMakeBorder(src, top, bot, left, right, border_type=cv2.BORDER_CONSTANT, value=0):\n    \"\"\"Pad image border\n    Wrapper for cv2.copyMakeBorder that uses mx.nd.NDArray\n\n    Parameters\n    ----------\n    src : NDArray\n        Image in (width, height, channels).\n        Others are the same with cv2.copyMakeBorder\n\n    Returns\n    -------\n    img : NDArray\n        padded image\n    \"\"\"\n    hdl = NDArrayHandle()\n    check_call(_LIB.MXCVcopyMakeBorder(src.handle, ctypes.c_int(top), ctypes.c_int(bot),\n                                       ctypes.c_int(left), ctypes.c_int(right),\n                                       ctypes.c_int(border_type), ctypes.c_double(value),\n                                       ctypes.byref(hdl)))\n    return mx.nd.NDArray(hdl)\n\n\ndef scale_down(src_size, size):\n    \"\"\"Scale down crop size if it's bigger than image size\"\"\"\n    w, h = size\n    sw, sh = src_size\n    if sh < h:\n        w, h = float(w*sh)/h, sh\n    if sw < w:\n        w, h = sw, float(h*sw)/w\n    return int(w), int(h)\n\ndef fixed_crop(src, x0, y0, w, h, size=None, interpolation=cv2.INTER_CUBIC):\n    \"\"\"Crop src at fixed location, and (optionally) resize it to size\"\"\"\n    out = mx.nd.crop(src, begin=(y0, x0, 0), end=(y0+h, x0+w, int(src.shape[2])))\n    if size is not None and (w, h) != size:\n        out = resize(out, size, interpolation=interpolation)\n    return out\n\ndef random_crop(src, size):\n    \"\"\"Randomly crop src with size. Upsample result if src is smaller than size\"\"\"\n    h, w, _ = src.shape\n    new_w, new_h = scale_down((w, h), size)\n\n    x0 = random.randint(0, w - new_w)\n    y0 = random.randint(0, h - new_h)\n\n    out = fixed_crop(src, x0, y0, new_w, new_h, size)\n    return out, (x0, y0, new_w, new_h)\n\ndef color_normalize(src, mean, std):\n    \"\"\"Normalize src with mean and std\"\"\"\n    src -= mean\n    src /= std\n    return src\n\ndef random_size_crop(src, size, min_area=0.25, ratio=(3.0/4.0, 4.0/3.0)):\n    \"\"\"Randomly crop src with size. Randomize area and aspect ratio\"\"\"\n    h, w, _ = src.shape\n    area = w*h\n    for _ in range(10):\n        new_area = random.uniform(min_area, 1.0) * area\n        new_ratio = random.uniform(*ratio)\n        new_w = int(new_area*new_ratio)\n        new_h = int(new_area/new_ratio)\n\n        if random.uniform(0., 1.) < 0.5:\n            new_w, new_h = new_h, new_w\n\n        if new_w > w or new_h > h:\n            continue\n\n        x0 = random.randint(0, w - new_w)\n        y0 = random.randint(0, h - new_h)\n\n        out = fixed_crop(src, x0, y0, new_w, new_h, size)\n        return out, (x0, y0, new_w, new_h)\n\n    return random_crop(src, size)\n\nclass ImageListIter(mx.io.DataIter):\n    \"\"\"An example image iterator using opencv plugin\"\"\"\n    def __init__(self, root, flist, batch_size, size, mean=None):\n        mx.io.DataIter.__init__(self)\n        self.root = root\n        self.list = [line.strip() for line in open(flist).readlines()]\n        self.cur = 0\n        self.batch_size = batch_size\n        self.size = size\n        if mean is not None:\n            self.mean = mx.nd.array(mean)\n        else:\n            self.mean = None\n\n    def reset(self):\n        \"\"\"Reset iterator position to 0\"\"\"\n        self.cur = 0\n\n    def next(self):\n        \"\"\"Move iterator position forward\"\"\"\n        batch = mx.nd.zeros((self.batch_size, self.size[1], self.size[0], 3))\n        i = self.cur\n        for i in range(self.cur, min(len(self.list), self.cur+self.batch_size)):\n            str_img = open(self.root+self.list[i]+'.jpg').read()\n            img = imdecode(str_img, 1)\n            img, _ = random_crop(img, self.size)\n            batch[i - self.cur] = img\n        batch = mx.nd.transpose(batch, axes=(0, 3, 1, 2))\n        ret = mx.io.DataBatch(data=[batch],\n                              label=[],\n                              pad=self.batch_size-(i-self.cur),\n                              index=None)\n        self.cur = i\n        return ret\n\n\n\n\n"
  },
  {
    "path": "plugin/sframe/iter_sframe.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file iter_sframe_image.cc\n * \\brief\n * \\author Bing Xu\n */\n\n#include <mxnet/io.h>\n#include <dmlc/base.h>\n#include <dmlc/io.h>\n#include <dmlc/omp.h>\n#include <dmlc/logging.h>\n#include <dmlc/parameter.h>\n#include <string>\n#include <memory>\n#include <unity/lib/image_util.hpp>\n#include <unity/lib/gl_sframe.hpp>\n#include <unity/lib/gl_sarray.hpp>\n#include \"../../src/io/inst_vector.h\"\n#include \"../../src/io/image_recordio.h\"\n#include \"../../src/io/image_augmenter.h\"\n#include \"../../src/io/iter_prefetcher.h\"\n#include \"../../src/io/iter_normalize.h\"\n#include \"../../src/io/iter_batchloader.h\"\n\nnamespace mxnet {\nnamespace io {\n\nstruct SFrameParam : public dmlc::Parameter<SFrameParam> {\n  /*! \\brief sframe path */\n  std::string path_sframe;\n  std::string data_field;\n  std::string label_field;\n  mxnet::TShape data_shape;\n  mxnet::TShape label_shape;\n  DMLC_DECLARE_PARAMETER(SFrameParam) {\n    DMLC_DECLARE_FIELD(path_sframe)\n        .set_default(\"\")\n        .describe(\"Dataset Param: path to image dataset sframe\");\n    DMLC_DECLARE_FIELD(data_field)\n        .set_default(\"data\")\n        .describe(\"Dataset Param: data column in sframe\");\n    DMLC_DECLARE_FIELD(label_field)\n        .set_default(\"label\")\n        .describe(\"Dataset Param: label column in sframe\");\n    DMLC_DECLARE_FIELD(data_shape).describe(\"Dataset Param: input data instance shape\");\n    DMLC_DECLARE_FIELD(label_shape).describe(\"Dataset Param: input label instance shape\");\n  }\n};  // struct SFrameImageParam\n\nclass SFrameIterBase : public IIterator<DataInst> {\n public:\n  SFrameIterBase() {}\n\n  void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override {\n    param_.InitAllowUnknown(kwargs);\n    sframe_ = graphlab::gl_sframe(param_.path_sframe)[{param_.data_field, param_.label_field}];\n    range_it_.reset(new graphlab::gl_sframe_range(sframe_.range_iterator()));\n    this->BeforeFirst();\n  }\n\n  virtual ~SFrameIterBase() {}\n\n  virtual void BeforeFirst() {\n    idx_        = 0;\n    *range_it_  = sframe_.range_iterator();\n    current_it_ = range_it_->begin();\n  }\n\n  virtual const DataInst& Value(void) const {\n    return out_;\n  }\n\n  virtual bool Next() = 0;\n\n protected:\n  /*! \\brief index of instance */\n  index_t idx_;\n  /*! \\brief output of sframe iterator */\n  DataInst out_;\n  /*! \\brief temp space */\n  InstVector tmp_;\n  /*! \\brief sframe iter parameter */\n  SFrameParam param_;\n  /*! \\brief sframe object*/\n  graphlab::gl_sframe sframe_;\n  /*! \\brief sframe range iterator */\n  std::unique_ptr<graphlab::gl_sframe_range> range_it_;\n  /*! \\brief current iterator in range iterator */\n  graphlab::gl_sframe_range::iterator current_it_;\n\n protected:\n  /*! \\brief copy data */\n  template <int dim>\n  void Copy_(mshadow::Tensor<cpu, dim> tensor, const graphlab::flex_vec& vec) {\n    CHECK_EQ(tensor.shape_.Size(), vec.size());\n    CHECK_EQ(tensor.CheckContiguous(), true);\n    mshadow::Tensor<cpu, 1> flatten(tensor.dptr_, mshadow::Shape1(tensor.shape_.Size()));\n    for (index_t i = 0; i < vec.size(); ++i) {\n      flatten[i] = static_cast<float>(vec[i]);\n    }\n  }\n};  // class SFrameIterBase\n\nclass SFrameImageIter : public SFrameIterBase {\n public:\n  SFrameImageIter() : augmenter_(new ImageAugmenter()), prnd_(new common::RANDOM_ENGINE(8964)) {}\n\n  void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override {\n    Parent::Init(kwargs);\n    augmenter_->Init(kwargs);\n    CHECK_EQ(Parent::param_.data_shape.ndim(), 3) << \"Image shpae must be (channel, height, width)\";\n  }\n\n  bool Next(void) override {\n    if (Parent::current_it_ == Parent::range_it_->end()) {\n      return false;\n    }\n    graphlab::image_type gl_img = (*Parent::current_it_)[0];\n    graphlab::flex_vec gl_label = (*Parent::current_it_)[1];\n    // TODO(bing): check not decoded\n    // TODO(bing): check img shape\n    CHECK_EQ(gl_label.size(), Parent::param_.label_shape.Size()) << \"Label shape does not match\";\n    const unsigned char* raw_data = gl_img.get_image_data();\n    cv::Mat res;\n    cv::Mat buf(1, gl_img.m_image_data_size, CV_8U, const_cast<unsigned char*>(raw_data));\n    res                  = cv::imdecode(buf, -1);\n    res                  = augmenter_->Process(res, prnd_.get());\n    const int n_channels = res.channels();\n    if (!tmp_.Size()) {\n      tmp_.Push(\n          Parent::idx_++, Parent::param_.data_shape.get<3>(), Parent::param_.label_shape.get<1>());\n    }\n    mshadow::Tensor<cpu, 3> data = Parent::tmp_.data().Back();\n    std::vector<int> swap_indices;\n    if (n_channels == 1)\n      swap_indices = {0};\n    if (n_channels == 3)\n      swap_indices = {2, 1, 0};\n    for (int i = 0; i < res.rows; ++i) {\n      uchar* im_data = res.ptr<uchar>(i);\n      for (int j = 0; j < res.cols; ++j) {\n        for (int k = 0; k < n_channels; ++k) {\n          data[k][i][j] = im_data[swap_indices[k]];\n        }\n        im_data += n_channels;\n      }\n    }\n    mshadow::Tensor<cpu, 1> label = Parent::tmp_.label().Back();\n    Parent::Copy_<1>(label, gl_label);\n    res.release();\n    out_ = Parent::tmp_[0];\n    ++current_it_;\n    return true;\n  }\n\n private:\n  /*! \\brief parent type */\n  typedef SFrameIterBase Parent;\n  /*! \\brief image augmenter */\n  std::unique_ptr<ImageAugmenter> augmenter_;\n  /*! \\brief randim generator*/\n  std::unique_ptr<common::RANDOM_ENGINE> prnd_;\n};  // class SFrameImageIter\n\nclass SFrameDataIter : public SFrameIterBase {\n public:\n  bool Next() override {\n    if (Parent::current_it_ == Parent::range_it_->end()) {\n      return false;\n    }\n    graphlab::flex_vec gl_data  = (*Parent::current_it_)[0];\n    graphlab::flex_vec gl_label = (*Parent::current_it_)[1];\n    CHECK_EQ(gl_data.size(), Parent::param_.data_shape.Size()) << \"Data shape does not match\";\n    CHECK_EQ(gl_label.size(), Parent::param_.label_shape.Size()) << \"Label shape does not match\";\n    if (!Parent::tmp_.Size()) {\n      Parent::tmp_.Push(\n          Parent::idx_++, Parent::param_.data_shape.get<3>(), Parent::param_.label_shape.get<1>());\n    }\n    mshadow::Tensor<cpu, 3> data = Parent::tmp_.data().Back();\n    Parent::Copy_<3>(data, gl_data);\n    mshadow::Tensor<cpu, 1> label = Parent::tmp_.label().Back();\n    Parent::Copy_<1>(label, gl_label);\n    out_ = Parent::tmp_[0];\n    ++current_it_;\n    return true;\n  }\n\n private:\n  /*! \\brief parent type */\n  typedef SFrameIterBase Parent;\n};  // class SFrameDataIter\n\nDMLC_REGISTER_PARAMETER(SFrameParam);\n\nMXNET_REGISTER_IO_ITER(SFrameImageIter)\n    .describe(\"Naive SFrame image iterator prototype\")\n    .add_arguments(SFrameParam::__FIELDS__())\n    .add_arguments(BatchParam::__FIELDS__())\n    .add_arguments(PrefetcherParam::__FIELDS__())\n    .add_arguments(ImageAugmentParam::__FIELDS__())\n    .add_arguments(ImageNormalizeParam::__FIELDS__())\n    .set_body([]() {\n      return new PrefetcherIter(new BatchLoader(new ImageNormalizeIter(new SFrameImageIter())));\n    });\n\nMXNET_REGISTER_IO_ITER(SFrameDataIter)\n    .describe(\"Naive SFrame data iterator prototype\")\n    .add_arguments(SFrameParam::__FIELDS__())\n    .add_arguments(BatchParam::__FIELDS__())\n    .add_arguments(PrefetcherParam::__FIELDS__())\n    .set_body([]() { return new PrefetcherIter(new BatchLoader(new SFrameDataIter())); });\n\n}  // namespace io\n}  // namespace mxnet\n"
  },
  {
    "path": "plugin/sframe/plugin.mk",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nSFRMAE_SRC = plugin/sframe/iter_sframe.cc\nPLUGIN_OBJ += build/plugin/sframe/iter_sframe.o\nCFLAGS += -I$(SFRAME_PATH)/oss_src/unity/lib/\nCFLAGS += -I$(SFRAME_PATH)/oss_src/\nLDFLAGS += -L$(SFRAME_PATH)/release/oss_src/unity/python/sframe/\nLDFLAGS += -lunity_shared\nLDFLAGS += -lboost_system\n"
  },
  {
    "path": "plugin/torch/torch.mk",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nCFLAGS += -I$(TORCH_PATH)/install/include -I$(TORCH_PATH)/install/include/TH -I$(TORCH_PATH)/install/include/THC/ -DMXNET_USE_TORCH=1\nLDFLAGS += -L$(TORCH_PATH)/install/lib -lluajit -lluaT -lTH -lTHC\n\nTORCH_SRC = $(wildcard plugin/torch/*.cc)\nPLUGIN_OBJ += $(patsubst %.cc, build/%.o, $(TORCH_SRC))\nTORCH_CUSRC = $(wildcard plugin/torch/*.cu)\nPLUGIN_CUOBJ += $(patsubst %.cu, build/%_gpu.o, $(TORCH_CUSRC))\n"
  },
  {
    "path": "plugin/torch/torch_base.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file torch_base.cc\n * \\brief torch_state\n * \\author Junyuan Xie\n */\n#include \"./torch_base.h\"\n\nnamespace mxnet {\nTorchState::TorchState() {\n  this->L = luaL_newstate();\n\n  luaL_openlibs(L);\n  luaL_loadstring(L,\n                  \"require 'torch'\\n\"\n                  \"require 'nn'\\n\"\n#if MXNET_USE_CUDA\n                  \"require 'cutorch'\\n\"\n                  \"require 'cunn'\\n\"\n#if MXNET_USE_CUDNN\n                  \"require 'cudnn'\\n\"\n#endif  // MXNET_USE_CUDNN\n#endif  // MXNET_USE_CUDA\n  );    // NOLINT(*)\n  int err = lua_pcall(L, 0, 0, 0);\n  CHECK_EQ(err, 0) << lua_tostring(L, -1);\n}\n\nTorchState* TorchState::ThreadSharedLuaState() {\n  thread_local TorchState* state = nullptr;\n  if (!state) {\n    state = new TorchState();\n  }\n  return state;\n}\n\ntemplate <>\nvoid TorchState::SetStream(mshadow::Stream<mshadow::cpu>* s) {\n  return;\n}\n\n#if MXNET_USE_CUDA\ntemplate <>\nvoid TorchState::SetStream(mshadow::Stream<mshadow::gpu>* s) {\n  CudaState()->currentStream = mshadow::Stream<gpu>::GetStream(s);\n}\n#endif  // MXNET_USE_CUDA\n}  // namespace mxnet\n"
  },
  {
    "path": "plugin/torch/torch_base.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file torch_base.h\n * \\brief Torch interface.\n * \\author Junyuan Xie\n */\n#ifndef PLUGIN_TORCH_TORCH_BASE_H_\n#define PLUGIN_TORCH_TORCH_BASE_H_\n#include <mxnet/base.h>\n\nextern \"C\" {\n#include <lua.h>\n#include <luaT.h>\n#include <lualib.h>\n#include <TH/THStorage.h>\n#include <TH/THTensor.h>\n}\n\n#if MXNET_USE_CUDA\nextern \"C\" {\n#include <THC/THCStorage.h>\n#include <THC/THCTensor.h>\n#include <THC/THCTensorCopy.h>\n}\n#endif  // MXNET_USE_CUDA\n\n#include <vector>\n\nnamespace mxnet {\n\nclass TorchState {\n public:\n  lua_State* L;\n  TorchState();\n  static TorchState* ThreadSharedLuaState();\n\n#if MXNET_USE_CUDA\n  THCState* CudaState() {\n    lua_getglobal(L, \"cutorch\");\n    CHECK(!lua_isnil(L, -1));\n    lua_getfield(L, -1, \"_state\");\n    CHECK(!lua_isnil(L, -1));\n    THCState* state = reinterpret_cast<THCState*>(lua_touserdata(L, -1));\n    lua_pop(L, 2);\n    return state;\n  }\n#endif  // MXNET_USE_CUDA\n\n  template <typename xpu>\n  void SetStream(mshadow::Stream<xpu>* s);\n\n  void PrintState() {\n    int i;\n    int top = lua_gettop(L);\n    LOG(INFO) << \"Stack height: \" << top;\n    for (i = 1; i <= top; i++) { /* repeat for each level */\n      int t = lua_type(L, i);\n      switch (t) {\n        case LUA_TSTRING: /* strings */\n          LOG(INFO) << i << \": '\" << lua_tostring(L, i) << \"'\";\n          break;\n        case LUA_TBOOLEAN: /* booleans */\n          LOG(INFO) << i << \": \" << (lua_toboolean(L, i) ? \"true\" : \"false\");\n          break;\n        case LUA_TNUMBER: /* numbers */\n          LOG(INFO) << i << \": \" << lua_tonumber(L, i);\n          break;\n        default: /* other values */\n          LOG(INFO) << i << \": \" << lua_typename(L, t);\n          break;\n      }\n    }\n  }\n\n  int Deserialize(THCharStorage* chunk) {  // read only to the chunk\n    CHECK_NE(chunk, NULL);\n    lua_getglobal(L, \"Deserialize\");\n    luaT_pushudata(L, chunk, \"torch.CharStorage\");\n    THCharStorage_retain(chunk);  // keep it because read only\n    int err = lua_pcall(L, 1, 1, 0);\n    CHECK_EQ(err, 0);\n    return 1;\n  }\n\n  int Serialize(THCharStorage** chunk) {\n    lua_getglobal(L, \"Serialize\");\n    lua_pushvalue(L, -2);\n    int err = lua_pcall(L, 1, 1, 0);\n    CHECK_EQ(err, 0) << \"Serialize failed \" << lua_tostring(L, -1);\n    THCharStorage_free(*chunk);  // free the original\n    *chunk = reinterpret_cast<THCharStorage*>(luaT_toudata(L, -1, \"torch.CharStorage\"));\n    THCharStorage_retain(*chunk);  // keep the chunk even when lua side deletes\n    lua_pop(L, 2);\n    return 0;\n  }\n};\n\ntypedef void* THGeneralTensor;\ntypedef void* THGeneralStorage;\n\nclass TorchTensor {\n public:\n  static const char* TensorType(int dev_mask) {\n    switch (dev_mask) {\n      case cpu::kDevMask:\n        return \"torch.FloatTensor\";\n      case gpu::kDevMask:\n        return \"torch.CudaTensor\";\n      default:\n        LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR;\n        return NULL;\n    }\n  }\n\n  static const char* ModuleType(int dev_mask) {\n    switch (dev_mask) {\n      case cpu::kDevMask:\n        return \":float()\";\n      case gpu::kDevMask:\n        return \":cuda()\";\n      default:\n        LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR;\n        return NULL;\n    }\n  }\n\n  static const char* TensorType(TBlob data) {\n    return TensorType(data.dev_mask());\n  }\n\n  static const char* ModuleType(TBlob data) {\n    return TensorType(data.dev_mask());\n  }\n\n  static THGeneralTensor TBlobToTHTensor(TorchState* torchState, TBlob data) {\n    size_t size            = data.Size();\n    THGeneralTensor tensor = NULL;\n    THLongStorage* thshape = THLongStorage_newWithSize(data.ndim());\n    for (int i = 0; i < data.ndim(); ++i) {\n      THLongStorage_set(thshape, i, data.shape_[i]);\n    }\n    CHECK_EQ(data.type_flag_, mshadow::kFloat32) << \"Torch Interface only support float32\";\n    switch (data.dev_mask()) {\n      case cpu::kDevMask: {\n        THFloatStorage* storage =\n            THFloatStorage_newWithData(static_cast<real_t*>(data.dptr_), size);\n        THFloatStorage_clearFlag(storage, TH_STORAGE_FREEMEM);\n        tensor = (THGeneralTensor)THFloatTensor_newWithStorage(storage, 0, thshape, NULL);\n        THFloatStorage_free(storage);\n        break;\n      }\n#if MXNET_USE_CUDA\n      case gpu::kDevMask: {\n        THCState* state = torchState->CudaState();\n        THCudaStorage* storage =\n            THCudaStorage_newWithData(state, static_cast<real_t*>(data.dptr_), size);\n        // a bug in cutorch\n        THFloatStorage_clearFlag(reinterpret_cast<THFloatStorage*>(storage), TH_STORAGE_FREEMEM);\n        tensor = (THGeneralTensor)THCudaTensor_newWithStorage(state, storage, 0, thshape, NULL);\n        THCudaStorage_free(state, storage);\n        break;\n      }\n#endif\n      default:\n        LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR;\n    }\n    THLongStorage_free(thshape);\n\n    return tensor;\n  }\n\n  static void FreeInternal(TorchState* torchState, THGeneralTensor tensor, int dev_mask) {\n    switch (dev_mask) {\n      case cpu::kDevMask: {\n        THFloatStorage* original = static_cast<THFloatTensor*>(tensor)->storage;\n        THFloatStorage_free(original);\n        break;\n      }\n#if MXNET_USE_CUDA\n      case gpu::kDevMask: {\n        THCState* state         = torchState->CudaState();\n        THCudaStorage* original = static_cast<THCudaTensor*>(tensor)->storage;\n        THCudaStorage_free(state, original);\n        break;\n      }\n#endif\n      default:\n        LOG(FATAL) << \"Unknown device type \" << dev_mask;\n    }\n  }\n\n  static void SetInternal(TorchState* torchState, THGeneralTensor tensor, const TBlob& blob) {\n    size_t size = blob.Size();\n    switch (blob.dev_mask()) {\n      case cpu::kDevMask: {\n        THFloatStorage* storage =\n            THFloatStorage_newWithData(static_cast<real_t*>(blob.dptr_), size);\n        THFloatStorage_clearFlag(storage, TH_STORAGE_FREEMEM);\n        THFloatStorage* original                     = static_cast<THFloatTensor*>(tensor)->storage;\n        static_cast<THFloatTensor*>(tensor)->storage = storage;\n        THFloatStorage_free(original);\n        break;\n      }\n#if MXNET_USE_CUDA\n      case gpu::kDevMask: {\n        THCState* state = torchState->CudaState();\n        THCudaStorage* storage =\n            THCudaStorage_newWithData(state, static_cast<real_t*>(blob.dptr_), size);\n        // TODO(min): torch bug Cuda version not implemented\n        THFloatStorage_clearFlag(reinterpret_cast<THFloatStorage*>(storage), TH_STORAGE_FREEMEM);\n        THCudaStorage* original                     = static_cast<THCudaTensor*>(tensor)->storage;\n        static_cast<THCudaTensor*>(tensor)->storage = storage;\n        THCudaStorage_free(state, original);\n        break;\n      }\n#endif\n      default:\n        LOG(FATAL) << \"Unknown device type \" << blob.dev_mask();\n    }\n  }\n\n  static std::vector<THGeneralTensor> TBlobVectorAsTable(\n      TorchState* torchState,\n      const std::vector<TBlob>::const_iterator begin,\n      const std::vector<TBlob>::const_iterator end) {\n    lua_State* L = torchState->L;\n    std::vector<THGeneralTensor> res;\n    int num = end - begin;\n    if (num > 1) {\n      lua_createtable(L, num, 0);\n      int index = 1;\n      for (std::vector<TBlob>::const_iterator it = begin; it != end; ++it) {\n        THGeneralTensor th = TorchTensor::TBlobToTHTensor(torchState, *it);\n        res.push_back(th);\n        luaT_pushudata(L, th, TorchTensor::TensorType(*it));\n        lua_rawseti(L, -2, index++);\n      }\n    } else if (num == 0) {\n      lua_pushnil(L);\n    } else {\n      THGeneralTensor th = TorchTensor::TBlobToTHTensor(torchState, *begin);\n      res.push_back(th);\n      luaT_pushudata(L, th, TorchTensor::TensorType(*begin));\n    }\n    return res;\n  }\n\n  static void CopyIfDifferent(TorchState* torchState, TBlob dst, THGeneralTensor th_dst) {\n    lua_State* L = torchState->L;\n    if (luaT_isudata(L, -1, TorchTensor::TensorType(cpu::kDevMask))) {\n      CHECK_EQ(dst.dev_mask(), cpu::kDevMask) << \"Device type mismatch.\";\n      THFloatTensor* src =\n          static_cast<THFloatTensor*>(luaT_toudata(L, -1, TorchTensor::TensorType(cpu::kDevMask)));\n      if (src->storage != static_cast<THFloatTensor*>(th_dst)->storage) {\n        THFloatTensor_copy(static_cast<THFloatTensor*>(th_dst), src);\n      }\n#if MXNET_USE_CUDA\n    } else if (luaT_isudata(L, -1, TorchTensor::TensorType(gpu::kDevMask))) {\n      CHECK_EQ(dst.dev_mask(), gpu::kDevMask) << \"Device type mismatch.\";\n      THCudaTensor* src =\n          static_cast<THCudaTensor*>(luaT_toudata(L, -1, TorchTensor::TensorType(gpu::kDevMask)));\n      if (src->storage != static_cast<THCudaTensor*>(th_dst)->storage) {\n        THCudaTensor_copy(torchState->CudaState(), static_cast<THCudaTensor*>(th_dst), src);\n      }\n#endif  // MXNET_USE_CUDA\n    } else {\n      LOG(FATAL) << \"Unsupported Torch tensor type \" << luaT_typename(L, -1);\n    }\n  }\n\n  static void CheckOutput(TorchState* torchState,\n                          std::vector<TBlob>::const_iterator begin,\n                          std::vector<TBlob>::const_iterator end,\n                          std::vector<THGeneralTensor>::const_iterator th_begin,\n                          std::vector<THGeneralTensor>::const_iterator th_end) {\n    lua_State* L = torchState->L;\n    int num      = end - begin;\n    CHECK_EQ(th_end - th_begin, num);\n    if (num == 0) {\n    } else if (num == 1) {\n      CopyIfDifferent(torchState, *begin, *th_begin);\n    } else {\n      CHECK(lua_istable(L, -1));\n      lua_pushnil(L);\n      for (; begin != end; ++begin, ++th_begin) {\n        CHECK(lua_next(L, -2));\n        CopyIfDifferent(torchState, *begin, *th_begin);\n        lua_pop(L, 1);\n      }\n      lua_pop(L, 1);\n    }\n  }\n};\n\n}  // namespace mxnet\n#endif  // PLUGIN_TORCH_TORCH_BASE_H_\n"
  },
  {
    "path": "plugin/torch/torch_criterion-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file torch_module-inl.h\n * \\brief torch module operator\n * \\author Min Lin\n */\n#ifndef PLUGIN_TORCH_TORCH_CRITERION_INL_H_\n#define PLUGIN_TORCH_TORCH_CRITERION_INL_H_\n\n#include <dmlc/logging.h>\n#include <dmlc/parameter.h>\n#include <mxnet/operator.h>\n#include <stdio.h>\n#include <cstring>\n#include <map>\n#include <string>\n#include <vector>\n#include <utility>\n#include \"../../src/operator/operator_common.h\"\n#include \"./torch_base.h\"\n\nnamespace mxnet {\nnamespace op {\nstruct TorchCriterionParam : public dmlc::Parameter<TorchCriterionParam> {\n  std::string lua_string;\n  mxnet::TShape label_shape;\n  float grad_scale;\n  DMLC_DECLARE_PARAMETER(TorchCriterionParam) {\n    DMLC_DECLARE_FIELD(lua_string)\n        .describe(\"lua string that is called to generate the torch criterion object\");\n    DMLC_DECLARE_FIELD(label_shape)\n        .set_default(mxnet::TShape())\n        .enforce_nonzero()\n        .describe(\"Shape of label (without batch size).\");\n    DMLC_DECLARE_FIELD(grad_scale)\n        .set_default(1.0f)\n        .describe(\"Scale the gradient by a float factor (a.k.a weight of this loss).\");\n  }\n};\n\n/**\n * \\brief This is the implementation of activation operator.\n * \\tparam xpu The device that the op will be executed on.\n */\ntemplate <typename xpu>\nclass TorchCriterionOp : public Operator {\n private:\n  TorchCriterionParam param_;\n  TorchState* torchState_;\n  int lua_reference_;\n\n public:\n  explicit TorchCriterionOp(TorchCriterionParam p) {\n    this->param_      = p;\n    this->torchState_ = new TorchState();\n    lua_State* L      = torchState_->L;\n    CHECK_EQ(lua_gettop(L), 0);\n    std::string exec =\n        std::string(\"return \") + p.lua_string + TorchTensor::ModuleType(xpu::kDevMask);\n    CHECK_EQ(luaL_loadstring(L, exec.c_str()), 0);\n    int err = lua_pcall(L, 0, 1, 0);\n    CHECK_EQ(err, 0) << lua_tostring(L, -1);\n    // serialize\n    this->lua_reference_ = lua_ref(L, LUA_REGISTRYINDEX);\n  }\n\n  ~TorchCriterionOp() {\n    delete this->torchState_;\n  }\n\n  virtual void Forward(const OpContext& ctx,\n                       const std::vector<TBlob>& in_data,\n                       const std::vector<OpReqType>& req,\n                       const std::vector<TBlob>& out_data,\n                       const std::vector<TBlob>& aux_args) {\n    using namespace mshadow;\n    lua_State* L = torchState_->L;\n    CHECK_EQ(lua_gettop(L), 0);\n    CHECK_EQ(in_data.size(), 2);\n    CHECK_EQ(out_data.size(), 1);\n    Stream<xpu>* s = ctx.get_stream<xpu>();\n    torchState_->SetStream(s);\n    lua_rawgeti(L, LUA_REGISTRYINDEX, lua_reference_);\n    // call forward\n    // | self\n    lua_getfield(L, -1, \"forward\");\n    // | self | forward\n    lua_pushvalue(L, -2);\n    // | self | forward | self\n    for (index_t i = 0; i < in_data.size(); ++i) {\n      THGeneralTensor th = TorchTensor::TBlobToTHTensor(torchState_, in_data[i]);\n      luaT_pushudata(L, th, TorchTensor::TensorType(in_data[i]));\n    }\n    // | self | forward | self | pred | label\n    int err = lua_pcall(L, 3, 1, 0);\n    CHECK_EQ(err, 0) << lua_tostring(L, -1);\n    CHECK(lua_isnumber(L, -1)) << \"Criterion must return a number\";\n    real_t loss = static_cast<real_t>(lua_tonumber(L, -1));\n    lua_pop(L, 1);\n    Tensor<xpu, 2> out = out_data[0].FlatTo2D<xpu, real_t>(s);\n    Assign(out, req[0], loss * param_.grad_scale);\n    lua_pop(L, 1);\n    CHECK_EQ(lua_gettop(L), 0);\n  }\n\n  virtual void Backward(const OpContext& ctx,\n                        const std::vector<TBlob>& out_grad,\n                        const std::vector<TBlob>& in_data,\n                        const std::vector<TBlob>& out_data,\n                        const std::vector<OpReqType>& req,\n                        const std::vector<TBlob>& in_grad,\n                        const std::vector<TBlob>& aux_args) {\n    using namespace mshadow;\n    lua_State* L = torchState_->L;\n    CHECK_EQ(lua_gettop(L), 0);\n    CHECK_EQ(in_data.size(), 2);\n    CHECK_EQ(out_data.size(), 1);\n    CHECK_EQ(req[0], kWriteTo) << \"Torch Criterion only supports write to in_grad\";\n    CHECK_EQ(req[1], kNullOp) << \"Torch Criterion cannot back prop to label\";\n    Stream<xpu>* s = ctx.get_stream<xpu>();\n    torchState_->SetStream(s);\n    lua_rawgeti(L, LUA_REGISTRYINDEX, lua_reference_);\n    THGeneralTensor th = TorchTensor::TBlobToTHTensor(torchState_, in_grad[0]);\n    luaT_pushudata(L, th, TorchTensor::TensorType(in_grad[0]));\n    lua_setfield(L, -2, \"gradInput\");\n    lua_getfield(L, -1, \"backward\");\n    // | self | backward\n    lua_pushvalue(L, -2);\n    // | self | backward | self\n    for (index_t i = 0; i < in_data.size(); ++i) {\n      th = TorchTensor::TBlobToTHTensor(torchState_, in_data[i]);\n      luaT_pushudata(L, th, TorchTensor::TensorType(in_data[i]));\n    }\n    // | self | forward | self | pred | label\n    int err = lua_pcall(L, 3, 0, 0);\n    CHECK_EQ(err, 0) << lua_tostring(L, -1);\n    Tensor<xpu, 2> grad = in_grad[0].FlatTo2D<xpu, real_t>(s);\n    grad *= param_.grad_scale * in_grad[0].shape_[0];\n    lua_pop(L, 1);\n    CHECK_EQ(lua_gettop(L), 0);\n  }\n};  // class TorchCriterionOp\n\n// Decalre Factory function, used for dispatch specialization\ntemplate <typename xpu>\nOperator* CreateOp(TorchCriterionParam type);\n\n#if DMLC_USE_CXX11\nclass TorchCriterionProp : public OperatorProperty {\n public:\n  std::vector<std::string> ListArguments() const override {\n    return {\"data\", \"label\"};\n  }\n\n  virtual std::vector<std::string> ListOutputs() const {\n    return {\"output\"};\n  }\n\n  void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override {\n    param_.Init(kwargs);\n  }\n\n  std::map<std::string, std::string> GetParams() const override {\n    return param_.__DICT__();\n  }\n\n  bool InferShape(mxnet::ShapeVector* in_shape,\n                  mxnet::ShapeVector* out_shape,\n                  mxnet::ShapeVector* aux_shape) const override {\n    using namespace mshadow;\n    CHECK_EQ(in_shape->size(), 2);\n    const mxnet::TShape& dshape = in_shape->at(0);\n    if (dshape.ndim() == 0)\n      return false;\n    std::vector<index_t> lshape;\n    lshape.push_back(dshape[0]);\n    lshape.insert(lshape.end(),\n                  param_.label_shape.data(),\n                  param_.label_shape.data() + param_.label_shape.ndim());\n    mxnet::TShape shape(lshape.begin(), lshape.end());\n    SHAPE_ASSIGN_CHECK(*in_shape, 1, shape);\n    out_shape->clear();\n    out_shape->push_back(Shape1(dshape[0]));\n    return true;\n  }\n\n  OperatorProperty* Copy() const override {\n    auto ptr    = new TorchCriterionProp();\n    ptr->param_ = param_;\n    return ptr;\n  }\n\n  std::string TypeString() const override {\n    return \"TorchCriterion\";\n  }\n\n  // decalre dependency and inplace optimization options\n  std::vector<int> DeclareBackwardDependency(const std::vector<int>& out_grad,\n                                             const std::vector<int>& in_data,\n                                             const std::vector<int>& out_data) const override {\n    std::vector<int> dep;\n    dep.insert(dep.end(), in_data.begin(), in_data.end());\n    // Ensure that the backward and forward cannot be called at the same time\n    dep.insert(dep.end(), out_data.begin(), out_data.end());\n    return dep;\n  }\n\n  Operator* CreateOperator(Context ctx) const override;\n\n private:\n  TorchCriterionParam param_;\n};\n#endif  // DMLC_USE_CXX11\n}  // namespace op\n}  // namespace mxnet\n#endif  // PLUGIN_TORCH_TORCH_CRITERION_INL_H_\n"
  },
  {
    "path": "plugin/torch/torch_criterion.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file activation.cc\n * \\brief activation op\n * \\author Junyuan Xie\n */\n#include \"./torch_criterion-inl.h\"\n#include \"../../src/operator/mshadow_op.h\"\n\nnamespace mxnet {\nnamespace op {\ntemplate <>\nOperator* CreateOp<cpu>(TorchCriterionParam param) {\n  return new TorchCriterionOp<cpu>(param);\n}\n\n// DO_BIND_DISPATCH comes from operator_common.h\nOperator* TorchCriterionProp::CreateOperator(Context ctx) const {\n  DO_BIND_DISPATCH(CreateOp, param_);\n}\n\nDMLC_REGISTER_PARAMETER(TorchCriterionParam);\n\nMXNET_REGISTER_OP_PROPERTY(TorchCriterion, TorchCriterionProp)\n    .describe(\"Criterions from torch.\")\n    .add_arguments(TorchCriterionParam::__FIELDS__());\n\n}  // namespace op\n}  // namespace mxnet\n"
  },
  {
    "path": "plugin/torch/torch_criterion.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file activation.cc\n * \\brief activation op\n * \\author Bing Xu\n */\n#include \"./torch_criterion-inl.h\"\n#include \"../../src/operator/mshadow_op.h\"\n\nnamespace mxnet {\nnamespace op {\ntemplate <>\nOperator* CreateOp<gpu>(TorchCriterionParam param) {\n  return new TorchCriterionOp<gpu>(param);\n}\n\n}  // namespace op\n}  // namespace mxnet\n"
  },
  {
    "path": "plugin/torch/torch_function.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file torch_base.cc\n * \\brief torch_state\n * \\author Junyuan Xie\n */\n#include \"./torch_function.h\"\n\nnamespace mxnet {\n\n// Construction or extraction functions\nMXNET_REGISTER_TORCH_CONSTRUCTOR_FUN(_th_eye, eye);\nMXNET_REGISTER_TORCH_CONSTRUCTOR_FUN(_th_ones, ones);\nMXNET_REGISTER_TORCH_CONSTRUCTOR_FUN(_th_rand, rand);\nMXNET_REGISTER_TORCH_CONSTRUCTOR_FUN(_th_randn, randn);\nMXNET_REGISTER_TORCH_CONSTRUCTOR_FUN(_th_randperm, randperm);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_tril, tril);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_triu, triu);\nMXNET_REGISTER_TORCH_CONSTRUCTOR_FUN(_th_zeros, zeros);\n\n// Element-wise Mathematical Operations\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_abs, abs);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_sign, sign);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_acos, acos);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_asin, asin);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_atan, atan);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_ceil, ceil);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_cos, cos);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_cosh, cosh);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_exp, exp);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_floor, floor);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_log, log);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_log1p, log1p);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_pow, pow)\n    .add_argument(\"n\",\n                  \"float\",\n                  \"pow(x, n) returns x^n, element-wise. \"\n                  \"pow(n, x) returns n^x, element-wise.\");\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_round, round);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_sin, sin);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_sinh, sinh);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_sqrt, sqrt);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_tan, tan);\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_tanh, tanh);\n\n// Basic operations\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_add_scalar, add)\n    .add_argument(\"value\", \"float\", \"Add value to all elements in x\");\nMXNET_REGISTER_TORCH_BINARY_FUN_WITH_ARG(_th_add, add);\nMXNET_REGISTER_TORCH_BINARY_FUN(_th_add_axpy, add);\n\n// MXNET_REGISTER_TORCH_UNARY_FUN(_th_csub_scalar, csub);\n// MXNET_REGISTER_TORCH_BINARY_FUN_WITH_ARG(_th_csub, csub);\n\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_mul_scalar, mul)\n    .add_argument(\"value\", \"float\", \"Multiply value to all elements in x\");\nMXNET_REGISTER_TORCH_BINARY_FUN_WITH_ARG(_th_cmul, cmul);\n\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_clamp, clamp);\nMXNET_REGISTER_TORCH_BINARY_FUN_WITH_ARG(_th_cpow, cpow);\nMXNET_REGISTER_TORCH_TENARY_FUN(_th_addcmul, addcmul);\n\nMXNET_REGISTER_TORCH_UNARY_FUN(_th_div_scalar, div)\n    .add_argument(\"value\", \"float\", \"Divide all elements in x by value\");\nMXNET_REGISTER_TORCH_BINARY_FUN_WITH_ARG(_th_cdiv, cdiv);\nMXNET_REGISTER_TORCH_TENARY_FUN(_th_addcdiv, addcdiv);\n\nMXNET_REGISTER_TORCH_TENARY_FUN(_th_addmv, addmv);\nMXNET_REGISTER_TORCH_TENARY_FUN(_th_addr, addr);\nMXNET_REGISTER_TORCH_TENARY_FUN(_th_addmm, addmm);\nMXNET_REGISTER_TORCH_TENARY_FUN(_th_addbmm, addbmm);\nMXNET_REGISTER_TORCH_TENARY_FUN(_th_baddbmm, baddbmm);\n\nstruct TorchMMShape {\n  static std::vector<mshadow::TShape> GetShape(NDArray** u,\n                                               const std::map<std::string, std::string>& param) {\n    CHECK_EQ(u[0]->shape().ndim(), 2);\n    CHECK_EQ(u[1]->shape().ndim(), 2);\n    CHECK_EQ(u[0]->shape()[1], u[1]->shape()[0]);\n    index_t shape[] = {u[0]->shape()[0], u[1]->shape()[1]};\n    mshadow::TShape tshape(shape, shape + 2);\n    return {tshape};\n  }\n  static constexpr const char* fname = \"mm\";\n  static const int num_inputs        = 2;\n  static const int num_outputs       = 1;\n};\nMXNET_REGISTER_TORCH_FUN(_th_mm, TorchMMShape);\n\nstruct TorchMVShape {\n  static std::vector<mshadow::TShape> GetShape(NDArray** u,\n                                               const std::map<std::string, std::string>& param) {\n    CHECK_EQ(u[0]->shape().ndim(), 2);\n    CHECK_EQ(u[1]->shape().ndim(), 1);\n    CHECK_EQ(u[0]->shape()[1], u[1]->shape()[0]);\n    index_t shape[] = {u[0]->shape()[0]};\n    mshadow::TShape tshape(shape, shape + 1);\n    return {tshape};\n  }\n  static constexpr const char* fname = \"mv\";\n  static const int num_inputs        = 2;\n  static const int num_outputs       = 1;\n};\nMXNET_REGISTER_TORCH_FUN(_th_mv, TorchMVShape);\n\nstruct TorchBMMShape {\n  static std::vector<mshadow::TShape> GetShape(NDArray** u,\n                                               const std::map<std::string, std::string>& param) {\n    CHECK_EQ(u[0]->shape().ndim(), 3);\n    CHECK_EQ(u[1]->shape().ndim(), 3);\n    CHECK_EQ(u[0]->shape()[0], u[1]->shape()[0]);\n    CHECK_EQ(u[0]->shape()[2], u[1]->shape()[1]);\n    index_t shape[] = {u[0]->shape()[1], u[1]->shape()[2]};\n    mshadow::TShape tshape(shape, shape + 2);\n    return {tshape};\n  }\n  static constexpr const char* fname = \"bmm\";\n  static const int num_inputs        = 2;\n  static const int num_outputs       = 1;\n};\nMXNET_REGISTER_TORCH_FUN(_th_bmm, TorchBMMShape);\n\nstruct TorchGERShape {\n  static std::vector<mshadow::TShape> GetShape(NDArray** u,\n                                               const std::map<std::string, std::string>& param) {\n    CHECK_EQ(u[0]->shape().ndim(), 1);\n    CHECK_EQ(u[1]->shape().ndim(), 1);\n    index_t shape[] = {u[0]->shape()[0], u[1]->shape()[0]};\n    mshadow::TShape tshape(shape, shape + 2);\n    return {tshape};\n  }\n  static constexpr const char* fname = \"ger\";\n  static const int num_inputs        = 2;\n  static const int num_outputs       = 1;\n};\nMXNET_REGISTER_TORCH_FUN(_th_ger, TorchGERShape);\n\n}  // namespace mxnet\n"
  },
  {
    "path": "plugin/torch/torch_function.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file torch_function.h\n * \\brief Torch interface.\n * \\author Junyuan Xie\n */\n#ifndef PLUGIN_TORCH_TORCH_FUNCTION_H_\n#define PLUGIN_TORCH_TORCH_FUNCTION_H_\n#include \"./torch_base.h\"\n#include <mxnet/base.h>\n#include <mxnet/ndarray.h>\n#include <stdio.h>\n#include <stdlib.h>\n#include <string>\n#include <map>\n#include <algorithm>\n#include <vector>\n\nnamespace mxnet {\n\ntemplate <typename xpu, typename OP>\nvoid TorchRunOp(std::vector<NDArray> arr_in,\n                std::vector<NDArray> arr_out,\n                const std::map<std::string, std::string>& param,\n                RunContext ctx) {\n  TorchState* torchState = TorchState::ThreadSharedLuaState();\n  torchState->SetStream(ctx.get_stream<xpu>());\n  lua_State* L = torchState->L;\n\n  lua_getglobal(L, \"torch\");\n  lua_getfield(L, -1, OP::fname);\n  int idx = 0;\n  std::vector<NDArray> arr(arr_out.begin(), arr_out.end());\n  arr.insert(arr.end(), arr_in.begin(), arr_in.end());\n  std::string format = param.at(\"format\");\n  std::istringstream args(param.at(\"args\"));\n  for (size_t i = 0; i < format.size(); ++i) {\n    std::string val;\n    std::getline(args, val, ',');\n    switch (format[i]) {\n      case 'n': {\n        CHECK(idx < arr.size()) << \"Too few NDArray arguments for Torch.\" << OP::fname;\n        luaT_pushudata(L,\n                       TorchTensor::TBlobToTHTensor(torchState, arr[idx].data()),\n                       TorchTensor::TensorType(arr[idx].data()));\n        idx++;\n        break;\n      }\n      case 'i':\n        lua_pushinteger(L, std::stoi(val));\n        break;\n      case 'f':\n        lua_pushnumber(L, std::stof(val));\n        break;\n      case 's':\n        lua_pushstring(L, val.c_str());\n        break;\n      case 'b':\n        lua_pushboolean(L, std::stoi(val));\n        break;\n      default:\n        LOG(FATAL) << \"Unknown argument type \" << format[i] << \" for Torch.\" << OP::fname;\n    }\n  }\n  CHECK_EQ(lua_pcall(L, format.size(), 0, 0), 0) << \"Lua Error: \" << lua_tostring(L, -1);\n}\n\ntemplate <typename OP>\nvoid TorchOp(NDArray** u,\n             real_t* s,\n             NDArray** out,\n             const std::map<std::string, std::string>& param) {\n  std::vector<mshadow::TShape> shapes = OP::GetShape(u, param);\n  CHECK_EQ(shapes.size(), OP::num_outputs) << \"Too many output shapes for TorchOp \" << OP::fname;\n  Context ctx;\n  int type_flag;\n  if (OP::num_inputs) {\n    ctx       = u[0]->ctx();\n    type_flag = u[0]->dtype();\n    for (int i = 0; i < OP::num_inputs; ++i) {\n      CHECK_EQ(ctx, u[i]->ctx()) << \"Context of all oprands must be the same.\";\n      CHECK_EQ(type_flag, u[i]->dtype()) << \"Data type of all oprands must be the same.\";\n    }\n  } else {\n    CHECK(param.count(\"ctx\")) << \"Must provide keyword argument ctx for TorchOp with 0 inputs\";\n    std::string str_ctx(param.at(\"ctx\"));\n    int id;\n    char tmp[4];\n    sscanf(str_ctx.c_str(), \"%3s(%d)\", tmp, &id);\n    std::string dev(tmp);\n    if (dev == \"cpu\") {\n      ctx = Context::Create(Context::kCPU, id);\n    } else if (dev == \"gpu\") {\n      ctx = Context::Create(Context::kGPU, id);\n    } else {\n      LOG(FATAL) << \"Unknown device type \" << dev;\n    }\n\n    if (param.count(\"dtype\")) {\n      std::stringstream str_dtype(param.at(\"dtype\"));\n      str_dtype >> type_flag;\n    } else {\n      type_flag = mshadow::default_type_flag;\n    }\n  }\n  std::vector<NDArray> arr_in, arr_out;\n  std::vector<Engine::VarHandle> var_in, var_out, var_const;\n  for (int i = 0; i < OP::num_inputs; ++i) {\n    arr_in.push_back(*(u[i]));\n    var_in.push_back(u[i]->var());\n  }\n  for (int i = 0; i < OP::num_outputs; ++i) {\n    if (out[i]->is_none()) {\n      *(out[i]) = NDArray(shapes[i], ctx, false, type_flag);\n    }\n    arr_out.push_back(*(out[i]));\n    var_out.push_back(out[i]->var());\n  }\n  std::sort(var_in.begin(), var_in.end());\n  var_in.resize(std::unique(var_in.begin(), var_in.end()) - var_in.begin());\n  std::sort(var_out.begin(), var_out.end());\n  var_out.resize(std::unique(var_out.begin(), var_out.end()) - var_out.begin());\n  std::set_difference(var_in.begin(),\n                      var_in.end(),\n                      var_out.begin(),\n                      var_out.end(),\n                      std::inserter(var_const, var_const.begin()));\n  switch (ctx.dev_mask()) {\n    case mshadow::cpu::kDevMask: {\n      Engine::Get()->PushSync(\n          [arr_in, arr_out, param](RunContext rctx) {\n            TorchRunOp<mshadow::cpu, OP>(arr_in, arr_out, param, rctx);\n          },\n          ctx,\n          var_const,\n          var_out);\n      break;\n    }\n#if MXNET_USE_CUDA\n    case gpu::kDevMask: {\n      Engine::Get()->PushSync(\n          [arr_in, arr_out, param](RunContext rctx) {\n            TorchRunOp<mshadow::gpu, OP>(arr_in, arr_out, param, rctx);\n          },\n          ctx,\n          var_const,\n          var_out);\n      break;\n    }\n#endif\n    default:\n      LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR;\n  }\n}\n\nstruct TorchFirstShape {\n  static std::vector<mshadow::TShape> GetShape(NDArray** u,\n                                               const std::map<std::string, std::string>& param) {\n    return {u[0]->shape()};\n  }\n};\n\nstruct TorchConstructorShape {\n  static std::vector<mshadow::TShape> GetShape(NDArray** u,\n                                               const std::map<std::string, std::string>& param) {\n    std::vector<index_t> shape;\n    std::string format = param.at(\"format\");\n    std::istringstream args(param.at(\"args\"));\n    std::string val;\n    std::getline(args, val, ',');\n    CHECK_LE(format.size(), 5) << \"Only support up to 4 dimensions.\";\n    for (size_t i = 1; i < format.size(); ++i) {\n      CHECK_EQ(format[i], 'i') << \"Only take integer arguments.\";\n      std::getline(args, val, ',');\n      shape.push_back(std::stoi(val));\n    }\n    mshadow::TShape tshape(shape.begin(), shape.end());\n    return {tshape};\n  }\n  static const int num_inputs  = 0;\n  static const int num_outputs = 1;\n};\n\n#define MXNET_REGISTER_TORCH_FUN(name, OP)  \\\n  MXNET_REGISTER_NDARRAY_FUN(name)          \\\n      .set_function(TorchOp<OP>)            \\\n      .set_num_use_vars(OP::num_inputs)     \\\n      .set_num_mutate_vars(OP::num_outputs) \\\n      .set_type_mask(kAcceptEmptyMutateTarget)\n\n#define MXNET_REGISTER_TORCH_UNARY_FUN(name, func)                   \\\n  struct TorchUnaryOpDesc_##name##_##func : public TorchFirstShape { \\\n    static constexpr const char* fname = #func;                      \\\n    static const int num_inputs        = 1;                          \\\n    static const int num_outputs       = 1;                          \\\n  };                                                                 \\\n  MXNET_REGISTER_TORCH_FUN(name, TorchUnaryOpDesc_##name##_##func)   \\\n      .add_argument(\"x\", \"NDArray\", \"Input NDArray\")\n\n#define MXNET_REGISTER_TORCH_BINARY_FUN(name, func)                   \\\n  struct TorchBinaryOpDesc_##name##_##func : public TorchFirstShape { \\\n    static constexpr const char* fname = #func;                       \\\n    static const int num_inputs        = 2;                           \\\n    static const int num_outputs       = 1;                           \\\n  };                                                                  \\\n  MXNET_REGISTER_TORCH_FUN(name, TorchBinaryOpDesc_##name##_##func)\n\n#define MXNET_REGISTER_TORCH_BINARY_FUN_WITH_ARG(name, func) \\\n  MXNET_REGISTER_TORCH_BINARY_FUN(name, func)                \\\n      .add_argument(\"x1\", \"NDArray\", \"First Input NDArray\")  \\\n      .add_argument(\"x2\", \"NDArray\", \"Second Input NDArray\")\n\n#define MXNET_REGISTER_TORCH_TENARY_FUN(name, func)                   \\\n  struct TorchTenaryOpDesc_##name##_##func : public TorchFirstShape { \\\n    static constexpr const char* fname = #func;                       \\\n    static const int num_inputs        = 3;                           \\\n    static const int num_outputs       = 1;                           \\\n  };                                                                  \\\n  MXNET_REGISTER_TORCH_FUN(name, TorchTenaryOpDesc_##name##_##func)\n\n#define MXNET_REGISTER_TORCH_CONSTRUCTOR_FUN(name, func)                         \\\n  struct TorchConstructorOpDesc_##name##_##func : public TorchConstructorShape { \\\n    static constexpr const char* fname = #func;                                  \\\n  };                                                                             \\\n  MXNET_REGISTER_TORCH_FUN(name, TorchConstructorOpDesc_##name##_##func)\n\n}  // namespace mxnet\n#endif  // PLUGIN_TORCH_TORCH_FUNCTION_H_\n"
  },
  {
    "path": "plugin/torch/torch_module-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file torch_module-inl.h\n * \\brief torch module operator\n * \\author Min Lin\n */\n#ifndef PLUGIN_TORCH_TORCH_MODULE_INL_H_\n#define PLUGIN_TORCH_TORCH_MODULE_INL_H_\n\n#include <dmlc/logging.h>\n#include <dmlc/parameter.h>\n#include <mxnet/operator.h>\n#include <stdio.h>\n#include <cstring>\n#include <map>\n#include <string>\n#include <vector>\n#include <utility>\n#include \"../../src/operator/operator_common.h\"\n#include \"./torch_base.h\"\n\nnamespace mxnet {\nnamespace op {\nstruct TorchModuleParam : public dmlc::Parameter<TorchModuleParam> {\n  std::string lua_string;\n  uint32_t num_data;\n  uint32_t num_params;\n  uint32_t num_outputs;\n  DMLC_DECLARE_PARAMETER(TorchModuleParam) {\n    DMLC_DECLARE_FIELD(lua_string)\n        .describe(\"lua string that is called to generate the torch module object\");\n    DMLC_DECLARE_FIELD(num_data).describe(\"the number of input data\");\n    DMLC_DECLARE_FIELD(num_params).describe(\"the number of parameters\");\n    DMLC_DECLARE_FIELD(num_outputs).describe(\"the number of outputs\");\n  }\n};\n\n/**\n * \\brief This is the implementation of activation operator.\n * \\tparam xpu The device that the op will be executed on.\n */\ntemplate <typename xpu>\nclass TorchModuleOp : public Operator {\n private:\n  TorchModuleParam param_;\n  TorchState* torchState_;\n  int lua_reference_;\n\n public:\n  explicit TorchModuleOp(TorchModuleParam p, TorchState* torchState) : torchState_(torchState) {\n    this->param_ = p;\n    lua_State* L = torchState_->L;\n    CHECK_EQ(lua_gettop(L), 0);\n    std::string exec =\n        std::string(\"return \") + p.lua_string + TorchTensor::ModuleType(xpu::kDevMask);\n    CHECK_EQ(luaL_loadstring(L, exec.c_str()), 0);\n    int err = lua_pcall(L, 0, 1, 0);\n    CHECK_EQ(err, 0) << lua_tostring(L, -1);\n    // Get number of parameters\n    uint32_t param_num = 0;\n    lua_getfield(L, -1, \"parameters\");\n    lua_pushvalue(L, -2);\n    CHECK_EQ(lua_pcall(L, 1, LUA_MULTRET, 0), 0);\n    if (lua_gettop(L) == 1) {\n      param_num = 0;\n    } else {\n      CHECK_EQ(lua_gettop(L), 3);\n      param_num = lua_objlen(L, -2);\n      lua_pop(L, 2);\n    }\n    CHECK_EQ(param_num, param_.num_params);\n    // Free the parameters allocated by torch so it doesn't take up memory.\n    if (param_.num_params != 0) {\n      // get the parameters into the stack\n      lua_getfield(L, -1, \"parameters\");\n      lua_pushvalue(L, -2);\n      int err = lua_pcall(L, 1, 1, 0);\n      CHECK_EQ(err, 0);\n      // iterate the parameters table to free tblobs inside\n      lua_pushnil(L);\n      while (lua_next(L, -2)) {\n        CHECK(luaT_isudata(L, -1, TorchTensor::TensorType(xpu::kDevMask)));\n        void* udata = luaT_toudata(L, -1, TorchTensor::TensorType(xpu::kDevMask));\n        TorchTensor::FreeInternal(torchState_, static_cast<THGeneralTensor>(udata), xpu::kDevMask);\n        lua_pop(L, 1);\n      }\n      lua_pop(L, 1);  // pop the parameter table\n    }\n    this->lua_reference_ = luaL_ref(L, LUA_REGISTRYINDEX);\n  }\n\n  virtual void Forward(const OpContext& ctx,\n                       const std::vector<TBlob>& in_data,\n                       const std::vector<OpReqType>& req,\n                       const std::vector<TBlob>& out_data,\n                       const std::vector<TBlob>& aux_args) {\n    lua_State* L = torchState_->L;\n\n    CHECK_EQ(lua_gettop(L), 0);\n    CHECK_EQ(in_data.size(), param_.num_params + param_.num_data);\n    CHECK_EQ(out_data.size(), param_.num_outputs);\n    mshadow::Stream<xpu>* s = ctx.get_stream<xpu>();\n    torchState_->SetStream(s);\n    // Deserialize self table\n\n    lua_rawgeti(L, LUA_REGISTRYINDEX, lua_reference_);\n\n    std::vector<THGeneralTensor> th_output = TorchTensor::TBlobVectorAsTable(\n        torchState_, out_data.begin(), out_data.begin() + param_.num_outputs);\n    // set the output field\n    lua_setfield(L, -2, \"output\");\n    // set the parameters\n    if (param_.num_params != 0) {\n      // get the parameters into the stack\n      lua_getfield(L, -1, \"parameters\");\n      lua_pushvalue(L, -2);\n      int err = lua_pcall(L, 1, 1, 0);\n      CHECK_EQ(err, 0);\n      // iterate the parameters table to put tblobs inside\n      lua_pushnil(L);\n      std::vector<TBlob>::const_iterator it = in_data.begin() + param_.num_data;\n      while (lua_next(L, -2)) {\n        CHECK(luaT_isudata(L, -1, TorchTensor::TensorType(*it)));\n        void* udata = luaT_toudata(L, -1, TorchTensor::TensorType(*it));\n        TorchTensor::SetInternal(torchState_, static_cast<THGeneralTensor>(udata), *(it));\n        it++;\n        lua_pop(L, 1);\n      }\n      lua_pop(L, 1);  // pop the parameter table\n    }\n    // call updateOutput\n    // | self\n    lua_getfield(L, -1, \"updateOutput\");\n    // | self | updateOutput\n    lua_pushvalue(L, -2);\n    // | self | updateOutput | self\n    TorchTensor::TBlobVectorAsTable(\n        torchState_, in_data.begin(), in_data.begin() + param_.num_data);\n    // | self | updateOutput | self | inputs\n    int err = lua_pcall(L, 2, 1, 0);  // doesn't need the output\n    CHECK_EQ(err, 0) << lua_tostring(L, -1);\n    TorchTensor::CheckOutput(torchState_,\n                             out_data.begin(),\n                             out_data.begin() + param_.num_outputs,\n                             th_output.begin(),\n                             th_output.end());\n    lua_pop(L, 2);\n    CHECK_EQ(lua_gettop(L), 0);\n  }\n\n  virtual void Backward(const OpContext& ctx,\n                        const std::vector<TBlob>& out_grad,\n                        const std::vector<TBlob>& in_data,\n                        const std::vector<TBlob>& out_data,\n                        const std::vector<OpReqType>& req,\n                        const std::vector<TBlob>& in_grad,\n                        const std::vector<TBlob>& aux_args) {\n    lua_State* L = torchState_->L;\n    CHECK_EQ(lua_gettop(L), 0);\n    CHECK_EQ(in_data.size(), param_.num_params + param_.num_data);\n    CHECK_EQ(out_data.size(), param_.num_outputs);\n    CHECK_EQ(out_grad.size(), param_.num_outputs);\n    CHECK_EQ(in_grad.size(), param_.num_params + param_.num_data);\n    mshadow::Stream<xpu>* s = ctx.get_stream<xpu>();\n    torchState_->SetStream(s);\n    lua_rawgeti(L, LUA_REGISTRYINDEX, lua_reference_);\n    TorchTensor::TBlobVectorAsTable(torchState_, out_data.begin(), out_data.end());\n    lua_setfield(L, -2, \"output\");\n    std::vector<THGeneralTensor> th_grad = TorchTensor::TBlobVectorAsTable(\n        torchState_, in_grad.begin(), in_grad.begin() + param_.num_data);\n    lua_setfield(L, -2, \"gradInput\");\n    if (param_.num_params != 0) {\n      // get the parameters into the stack\n      lua_getfield(L, -1, \"parameters\");\n      lua_pushvalue(L, -2);\n      int err = lua_pcall(L, 1, LUA_MULTRET, 0);\n      CHECK_EQ(err, 0) << lua_tostring(L, -1);\n      // iterate the parameters table to put tblobs inside\n      lua_pushnil(L);\n      std::vector<TBlob>::const_iterator it = in_data.begin() + param_.num_data;\n      while (lua_next(L, -3)) {\n        TorchTensor::SetInternal(\n            torchState_,\n            static_cast<THGeneralTensor>(luaT_toudata(L, -1, TorchTensor::TensorType(*it))),\n            *it);\n        it++;\n        lua_pop(L, 1);\n      }\n      // iterate the grad of params\n      lua_pushnil(L);\n      it = in_grad.begin() + param_.num_data;\n\n      while (lua_next(L, -2)) {\n        TorchTensor::SetInternal(\n            torchState_,\n            static_cast<THGeneralTensor>(luaT_toudata(L, -1, TorchTensor::TensorType(*it))),\n            *it);\n        it++;\n        lua_pop(L, 1);\n      }\n      lua_pop(L, 2);  // pop the parameters\n    }\n    lua_getfield(L, -1, \"zeroGradParameters\");\n    lua_pushvalue(L, -2);\n    CHECK_EQ(lua_pcall(L, 1, 0, 0), 0);\n    TorchTensor::TBlobVectorAsTable(\n        torchState_, in_data.begin(), in_data.begin() + param_.num_data);\n    TorchTensor::TBlobVectorAsTable(torchState_, out_grad.begin(), out_grad.end());\n    // call\n    lua_getfield(L, -3, \"accGradParameters\");\n    lua_pushvalue(L, -4);\n    lua_pushvalue(L, -4);\n    lua_pushvalue(L, -4);\n    lua_pushnumber(L, 1);\n    int err = lua_pcall(L, 4, 0, 0);  // doesn't need the output\n    CHECK_EQ(err, 0) << lua_tostring(L, -1);\n    lua_getfield(L, -3, \"updateGradInput\");\n    lua_pushvalue(L, -4);\n    lua_pushvalue(L, -4);\n    lua_pushvalue(L, -4);\n    err = lua_pcall(L, 3, 1, 0);  // doesn't need the output\n    CHECK_EQ(err, 0) << lua_tostring(L, -1);\n    TorchTensor::CheckOutput(torchState_,\n                             in_grad.begin(),\n                             in_grad.begin() + param_.num_data,\n                             th_grad.begin(),\n                             th_grad.end());\n    lua_pop(L, 4);\n    CHECK_EQ(lua_gettop(L), 0);\n  }\n};  // class TorchModuleOp\n\n// Declare Factory function, used for dispatch specialization\ntemplate <typename xpu>\nOperator* CreateOp(TorchModuleParam type, TorchState* torchState);\n\n#if DMLC_USE_CXX11\nclass TorchModuleProp : public OperatorProperty {\n protected:\n  mutable std::vector<std::string> arguments_;\n  mutable TorchState* torchState_;\n  mutable int lua_reference_;\n\n  void InitTorchState() const {\n    this->torchState_ = new TorchState();\n    lua_State* L      = torchState_->L;\n    std::string exec  = std::string(\"return \") + param_.lua_string;\n    CHECK_EQ(luaL_loadstring(L, exec.c_str()), 0);\n    int err = lua_pcall(L, 0, LUA_MULTRET, 0);\n    CHECK_EQ(lua_gettop(L), 1);\n    CHECK_EQ(err, 0) << lua_tostring(L, -1);\n    lua_getfield(L, -1, \"float\");\n    lua_pushvalue(L, -2);\n    err = lua_pcall(L, 1, 1, 0);\n    CHECK_EQ(err, 0);\n    lua_reference_ = lua_ref(L, LUA_REGISTRYINDEX);\n    lua_pop(L, 1);\n\n    CHECK_EQ(lua_gettop(L), 0);\n  }\n\n public:\n  TorchModuleProp() : OperatorProperty(), torchState_(NULL), lua_reference_(-1) {}\n\n  std::vector<std::string> ListArguments() const override {\n    if (!torchState_) {\n      InitTorchState();\n    }\n    lua_State* L = torchState_->L;\n\n    if (arguments_.size() == 0) {\n      for (uint32_t i = 0; i < param_.num_data; ++i) {\n        std::string data = \"data_\" + std::to_string(i);\n        arguments_.push_back(data);\n      }\n      std::string lua_code =\n          \"return function(module)\\n\"\n          \"          local params = module:parameters()\\n\"\n          \"          local dict = {}\\n\"\n          \"          if params == nil then\\n\"\n          \"             return {}\\n\"\n          \"          end\\n\"\n          \"          for id, p in ipairs(params) do\\n\"\n          \"             dict[p] = string.format('param_%d', id)\\n\"\n          \"          end\\n\"\n          \"          for key, value in pairs(module) do\\n\"\n          \"             if dict[value] then\\n\"\n          \"                dict[value] = key\\n\"\n          \"             end\\n\"\n          \"          end\\n\"\n          \"          local ret = {}\\n\"\n          \"          for _, p in ipairs(params) do\\n\"\n          \"             table.insert(ret, dict[p])\\n\"\n          \"          end\\n\"\n          \"          return ret\\n\"\n          \"end\\n\";\n      luaL_loadstring(L, lua_code.c_str());\n      int err = lua_pcall(L, 0, 1, 0);  // return the function\n      CHECK_EQ(err, 0) << lua_tostring(L, -1);\n      lua_rawgeti(L, LUA_REGISTRYINDEX, lua_reference_);\n      err = lua_pcall(L, 1, 1, 0);  // call the function\n      CHECK_EQ(err, 0) << lua_tostring(L, -1);\n      lua_pushnil(L);\n      while (lua_next(L, -2)) {\n        arguments_.push_back(lua_tostring(L, -1));\n        lua_pop(L, 1);\n      }\n      lua_pop(L, 1);\n    }\n    return arguments_;\n  }\n\n  virtual std::vector<std::string> ListOutputs() const {\n    if (param_.num_outputs > 1) {\n      std::vector<std::string> ret;\n      std::string output = \"output\";\n      for (uint32_t i = 0; i < param_.num_outputs; ++i) {\n        ret.push_back(output + std::to_string(i));\n      }\n      return ret;\n    } else {\n      return {\"output\"};\n    }\n  }\n  void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override {\n    param_.Init(kwargs);\n  }\n  std::map<std::string, std::string> GetParams() const override {\n    return param_.__DICT__();\n  }\n\n  bool InferShape(mxnet::ShapeVector* in_shape,\n                  mxnet::ShapeVector* out_shape,\n                  mxnet::ShapeVector* aux_shape) const override {\n    if (torchState_ == nullptr) {\n      this->InitTorchState();\n    }\n    lua_State* L = torchState_->L;\n\n    CHECK_EQ(lua_gettop(L), 0);\n    lua_rawgeti(L, LUA_REGISTRYINDEX, lua_reference_);\n    CHECK_EQ(in_shape->size(), param_.num_data + param_.num_params);\n    CHECK_EQ(out_shape->size(), param_.num_outputs);\n    CHECK_EQ(aux_shape->size(), 0);\n    lua_getfield(L, -1, \"updateOutput\");\n    lua_pushvalue(L, -2);  // self\n    if (param_.num_data == 1) {\n      THLongStorage* thshape = THLongStorage_newWithSize((*in_shape)[0].ndim());\n      for (uint32_t i = 0; i < (*in_shape)[0].ndim(); ++i) {\n        THLongStorage_set(thshape, i, (*in_shape)[0][i]);\n      }\n      THFloatTensor* in_data = THFloatTensor_newWithSize(thshape, NULL);\n      THLongStorage_free(thshape);\n      luaT_pushudata(L, in_data, TorchTensor::TensorType(mshadow::cpu::kDevMask));\n    } else if (param_.num_data > 1) {\n      lua_createtable(L, param_.num_data, 0);\n      for (uint32_t data_index = 0; data_index < param_.num_data; ++data_index) {\n        THLongStorage* thshape = THLongStorage_newWithSize((*in_shape)[data_index].ndim());\n        for (uint32_t i = 0; i < (*in_shape)[data_index].ndim(); ++i) {\n          THLongStorage_set(thshape, i, (*in_shape)[data_index][i]);\n        }\n        THFloatTensor* in_data = THFloatTensor_newWithSize(thshape, NULL);\n        THLongStorage_free(thshape);\n        luaT_pushudata(L, in_data, TorchTensor::TensorType(mshadow::cpu::kDevMask));\n        lua_rawseti(L, -2, data_index);\n      }\n    }\n    int err = lua_pcall(L, 2, 0, 0);\n    CHECK_EQ(err, 0) << lua_tostring(L, -1);\n    if (param_.num_params != 0) {\n      lua_getfield(L, -1, \"parameters\");\n      lua_pushvalue(L, -2);\n      int err = lua_pcall(L, 1, LUA_MULTRET, 0);\n      CHECK_EQ(err, 0);\n      CHECK_EQ(lua_gettop(L), 3);\n      lua_pushnil(L);\n      int index = param_.num_data;\n      while (lua_next(L, -3)) {\n        THFloatTensor* param = reinterpret_cast<THFloatTensor*>(\n            luaT_toudata(L, -1, TorchTensor::TensorType(mshadow::cpu::kDevMask)));\n        long int* size       = param->size;  // NOLINT(*)\n        (*in_shape)[index++] = mxnet::TShape(size, size + THFloatTensor_nDimension(param));\n        lua_pop(L, 1);\n      }\n      lua_pop(L, 2);\n    }\n    lua_getfield(L, -1, \"output\");\n    if (param_.num_outputs == 0) {\n    } else if (param_.num_outputs == 1) {\n      THFloatTensor* output = reinterpret_cast<THFloatTensor*>(\n          luaT_toudata(L, -1, TorchTensor::TensorType(mshadow::cpu::kDevMask)));\n      long int* size  = output->size;  // NOLINT(*)\n      (*out_shape)[0] = mxnet::TShape(size, size + THFloatTensor_nDimension(output));\n    } else {\n      for (uint32_t data_index = 0; data_index < param_.num_outputs; ++data_index) {\n        lua_pushnil(L);\n        int index = 0;\n        while (lua_next(L, -2)) {\n          THFloatTensor* out = reinterpret_cast<THFloatTensor*>(\n              luaT_toudata(L, -1, TorchTensor::TensorType(mshadow::cpu::kDevMask)));\n          long int* size        = out->size;  // NOLINT(*)\n          (*out_shape)[index++] = mxnet::TShape(size, size + THFloatTensor_nDimension(out));\n        }\n      }\n    }\n    lua_pop(L, 2);\n    CHECK_EQ(lua_gettop(L), 0);\n    return true;\n  }\n\n  OperatorProperty* Copy() const override {\n    auto ptr    = new TorchModuleProp();\n    ptr->param_ = param_;\n    return ptr;\n  }\n\n  std::string TypeString() const override {\n    return \"TorchModule\";\n  }\n\n  // decalre dependency and inplace optimization options\n  std::vector<int> DeclareBackwardDependency(const std::vector<int>& out_grad,\n                                             const std::vector<int>& in_data,\n                                             const std::vector<int>& out_data) const override {\n    std::vector<int> dep;\n    dep.insert(dep.end(), out_grad.begin(), out_grad.end());\n    dep.insert(dep.end(), out_data.begin(), out_data.end());\n    dep.insert(dep.end(), in_data.begin(), in_data.end());\n    return dep;\n  }\n\n  Operator* CreateOperator(Context ctx) const override;\n\n private:\n  TorchModuleParam param_;\n};\n#endif  // DMLC_USE_CXX11\n}  // namespace op\n}  // namespace mxnet\n#endif  // PLUGIN_TORCH_TORCH_MODULE_INL_H_\n"
  },
  {
    "path": "plugin/torch/torch_module.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file activation.cc\n * \\brief activation op\n * \\author Bing Xu\n */\n#include \"./torch_module-inl.h\"\n#include \"../../src/operator/mshadow_op.h\"\n\nnamespace mxnet {\nnamespace op {\ntemplate <>\nOperator* CreateOp<cpu>(TorchModuleParam param, TorchState* torchState) {\n  return new TorchModuleOp<cpu>(param, torchState);\n}\n\n// DO_BIND_DISPATCH comes from operator_common.h\nOperator* TorchModuleProp::CreateOperator(Context ctx) const {\n  DO_BIND_DISPATCH(CreateOp, param_, torchState_);\n}\n\nDMLC_REGISTER_PARAMETER(TorchModuleParam);\n\nMXNET_REGISTER_OP_PROPERTY(TorchModule, TorchModuleProp)\n    .describe(\"Modules from torch.\")\n    .add_arguments(TorchModuleParam::__FIELDS__());\n\n}  // namespace op\n}  // namespace mxnet\n"
  },
  {
    "path": "plugin/torch/torch_module.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file activation.cc\n * \\brief activation op\n * \\author Bing Xu\n */\n#include \"./torch_module-inl.h\"\n#include \"../../src/operator/mshadow_op.h\"\n\nnamespace mxnet {\nnamespace op {\ntemplate <>\nOperator* CreateOp<gpu>(TorchModuleParam param, TorchState* torchState) {\n  return new TorchModuleOp<gpu>(param, torchState);\n}\n\n}  // namespace op\n}  // namespace mxnet\n"
  },
  {
    "path": "plugin/warpctc/warpctc-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file warpctc-inl.h\n * \\brief warpctc operator\n * \\author Liang Xiang\n */\n#ifndef PLUGIN_WARPCTC_WARPCTC_INL_H_\n#define PLUGIN_WARPCTC_WARPCTC_INL_H_\n\n#include <dmlc/logging.h>\n#include <dmlc/parameter.h>\n#include <mxnet/operator.h>\n#include <stdio.h>\n#include <ctc.h>\n#include <cstring>\n#include <map>\n#include <string>\n#include <vector>\n#include <utility>\n#include <iostream>\n#include \"../../src/operator/operator_common.h\"\n\nnamespace mxnet {\nnamespace op {\n\nnamespace warpctc_enum {\nenum CTCOpInputs { kData, kLabel };\nenum CTCOpOutputs { kOut };\nenum CTCTemp { kTmp };\n}  // namespace warpctc_enum\n\nstruct WarpCTCParam : public dmlc::Parameter<WarpCTCParam> {\n  int label_length;\n  int input_length;\n  DMLC_DECLARE_PARAMETER(WarpCTCParam) {\n    DMLC_DECLARE_FIELD(label_length).set_default(0).describe(\"Real label length\");\n    DMLC_DECLARE_FIELD(input_length).set_default(0).describe(\"Input length\");\n  }\n};\n\ntemplate <typename xpu>\nclass WarpCTCOp : public Operator {\n private:\n  WarpCTCParam param_;\n\n public:\n  explicit WarpCTCOp(WarpCTCParam p) {\n    this->param_ = p;\n  }\n\n  ~WarpCTCOp() {}\n\n  inline void throw_on_error(ctcStatus_t status, const char* message) {\n    if (status != CTC_STATUS_SUCCESS) {\n      throw std::runtime_error(message + (\", stat = \" + std::string(ctcGetStatusString(status))));\n    }\n  }\n\n  virtual void Forward(const OpContext& ctx,\n                       const std::vector<TBlob>& in_data,\n                       const std::vector<OpReqType>& req,\n                       const std::vector<TBlob>& out_data,\n                       const std::vector<TBlob>& aux_args) {\n    using namespace mshadow;\n    using namespace mshadow::expr;\n    CHECK_EQ(in_data.size(), 2) << \"CTCOutput Input: [data, label]\";\n    CHECK_EQ(out_data.size(), 1) << \"CTCOutput Output: [output]\";\n\n    Stream<xpu>* s                    = ctx.get_stream<xpu>();\n    TBlob data                        = in_data[warpctc_enum::kData];\n    TBlob out                         = out_data[warpctc_enum::kOut];\n    Tensor<xpu, 2, float> data_tensor = data.FlatTo2D<xpu, float>(s);\n    Tensor<xpu, 2, float> out_tensor  = out.FlatTo2D<xpu, float>(s);\n    Softmax(out_tensor, data_tensor);\n  }\n\n  std::vector<int> labelLengths(const int* flat_labels,\n                                int minibatch,\n                                int size,\n                                int blank,\n                                int* total_length) {\n    CHECK_EQ(param_.label_length * minibatch, size)\n        << \"label size should = label_length * minibatch\";\n    std::vector<int> ret(minibatch, 0);\n    for (int i = 0; i < size; i++) {\n      if (flat_labels[i] == blank) {\n        continue;\n      }\n      int b = i / param_.label_length;\n      ret[b]++;\n      (*total_length)++;\n    }\n    return ret;\n  }\n\n  void removeBlank(const int* flat_labels, int* cpu_labels, int size, int blank) {\n    int k = 0;\n    for (int i = 0; i < size; i++) {\n      if (flat_labels[i] != blank) {\n        cpu_labels[k] = flat_labels[i];\n        k += 1;\n      }\n    }\n  }\n\n  virtual void Backward(const OpContext& ctx,\n                        const std::vector<TBlob>& out_grad,\n                        const std::vector<TBlob>& in_data,\n                        const std::vector<TBlob>& out_data,\n                        const std::vector<OpReqType>& req,\n                        const std::vector<TBlob>& in_grad,\n                        const std::vector<TBlob>& aux_args) {\n    using namespace mshadow;\n    Stream<xpu>* s = ctx.get_stream<xpu>();\n    TBlob data     = in_data[warpctc_enum::kData];\n    TBlob label    = in_data[warpctc_enum::kLabel];\n    CHECK_EQ(data.shape_.ndim(), 2) << \"input data shape should be 2 (t*n, p)\";\n    ctcOptions info;  // please updated to latest baidu/warp-ctc NOLINT(*)\n    if (data.dev_mask() == cpu::kDevMask) {\n      info.loc         = CTC_CPU;\n      info.num_threads = 1;\n    } else if (data.dev_mask() == gpu::kDevMask) {\n#if MXNET_USE_CUDA\n      info.loc    = CTC_GPU;\n      info.stream = ctx.get_stream<gpu>()->stream_;\n    } else {\n#endif\n      LOG(FATAL) << \"Unknown device type \" << data.dev_mask();\n    }\n    info.blank_label = 0;\n\n    int T             = param_.input_length;\n    int minibatch     = data.shape_[0] / T;\n    int alphabet_size = data.shape_[1];\n    std::vector<int> input_lengths;\n    for (int i = 0; i < minibatch; i++) {\n      input_lengths.push_back(T);\n    }\n\n#if MXNET_USE_CUDA\n    cudaError_t cuda_status;\n#endif\n    float* activations  = static_cast<float*>(data.dptr_);\n    int* flat_labels    = static_cast<int*>(label.dptr_);\n    int* cpu_raw_labels = flat_labels;\n    float* grads        = static_cast<float*>(in_grad[warpctc_enum::kData].dptr_);\n    if (data.dev_mask() == gpu::kDevMask) {\n#if MXNET_USE_CUDA\n      cpu_raw_labels = reinterpret_cast<int*>(malloc(sizeof(int) * label.Size()));\n      cuda_status    = cudaMemcpyAsync(cpu_raw_labels,\n                                    flat_labels,\n                                    label.Size() * sizeof(int),\n                                    cudaMemcpyDeviceToHost,\n                                    ctx.get_stream<gpu>()->stream_);\n      CHECK_EQ(cuda_status, cudaSuccess) << \"cuda memcpy label error\";\n#endif\n    }\n\n    int total_label_length = 0;\n    std::vector<int> label_lengths =\n        labelLengths(cpu_raw_labels, minibatch, label.Size(), 0, &total_label_length);\n    int* cpu_labels = reinterpret_cast<int*>(malloc(sizeof(int) * total_label_length));\n    removeBlank(cpu_raw_labels, cpu_labels, label.Size(), 0);\n\n    size_t alloc_bytes;\n    throw_on_error(get_workspace_size(label_lengths.data(),\n                                      input_lengths.data(),\n                                      alphabet_size,\n                                      input_lengths.size(),\n                                      info,\n                                      &alloc_bytes),\n                   \"Error: get_workspace_size in inf_test\");\n\n    Tensor<xpu, 1> ctc_workspace =\n        ctx.requested[warpctc_enum::kTmp].get_space<xpu>(mshadow::Shape1(alloc_bytes), s);\n\n    std::vector<float> costs(minibatch);\n    throw_on_error(compute_ctc_loss(activations,\n                                    grads,\n                                    cpu_labels,\n                                    label_lengths.data(),\n                                    input_lengths.data(),\n                                    alphabet_size,\n                                    minibatch,\n                                    costs.data(),\n                                    ctc_workspace.dptr_,\n                                    info),\n                   \"Error: compute_ctc_loss\");\n\n    if (data.dev_mask() == cpu::kDevMask) {\n      free(cpu_labels);\n    } else if (data.dev_mask() == gpu::kDevMask) {\n#if MXNET_USE_CUDA\n      free(cpu_raw_labels);\n      free(cpu_labels);\n#endif\n    }\n  }\n};\n\ntemplate <typename xpu>\nOperator* CreateOp(WarpCTCParam type);\n\n#if DMLC_USE_CXX11\nclass WarpCTCProp : public OperatorProperty {\n public:\n  std::vector<std::string> ListArguments() const override {\n    return {\"data\", \"label\"};\n  }\n\n  virtual std::vector<std::string> ListOutputs() const {\n    return {\"output\"};\n  }\n\n  void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override {\n    param_.Init(kwargs);\n  }\n\n  std::map<std::string, std::string> GetParams() const override {\n    return param_.__DICT__();\n  }\n\n  bool InferShape(mxnet::ShapeVector* in_shape,\n                  mxnet::ShapeVector* out_shape,\n                  mxnet::ShapeVector* aux_shape) const override {\n    using namespace mshadow;\n    CHECK_EQ(in_shape->size(), 2) << \"Input:[data, label]\";\n    const mxnet::TShape& dshape = in_shape->at(0);\n    if (dshape.ndim() == 0)\n      return false;\n    mxnet::TShape label_shape(dshape.ndim() - 1, 1);\n    label_shape[0] = param_.label_length * (dshape[0] / param_.input_length);\n    SHAPE_ASSIGN_CHECK(*in_shape, warpctc_enum::kLabel, label_shape);\n\n    out_shape->clear();\n    out_shape->push_back(dshape);\n    return true;\n  }\n\n  virtual bool InferType(std::vector<int>* in_type,\n                         std::vector<int>* out_type,\n                         std::vector<int>* aux_type) const {\n    CHECK_LE(in_type->size(), this->ListArguments().size());\n    in_type->clear();\n    in_type->push_back(mshadow::kFloat32);\n    in_type->push_back(mshadow::kInt32);\n    out_type->clear();\n    out_type->push_back(mshadow::kFloat32);\n    return true;\n  }\n\n  std::vector<ResourceRequest> BackwardResource(const mxnet::ShapeVector& in_shape) const override {\n    return {ResourceRequest::kTempSpace};\n  }\n\n  OperatorProperty* Copy() const override {\n    auto ptr    = new WarpCTCProp();\n    ptr->param_ = param_;\n    return ptr;\n  }\n\n  std::string TypeString() const override {\n    return \"WarpCTC\";\n  }\n\n  std::vector<int> DeclareBackwardDependency(const std::vector<int>& out_grad,\n                                             const std::vector<int>& in_data,\n                                             const std::vector<int>& out_data) const override {\n    return {\n        in_data[warpctc_enum::kData], in_data[warpctc_enum::kLabel], out_data[warpctc_enum::kOut]};\n  }\n\n  Operator* CreateOperator(Context ctx) const override;\n\n private:\n  WarpCTCParam param_;\n};\n#endif  // DMLC_USE_CXX11\n\n}  // namespace op\n}  // namespace mxnet\n\n#endif  // PLUGIN_WARPCTC_WARPCTC_INL_H_\n"
  },
  {
    "path": "plugin/warpctc/warpctc.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file warpctc.cc\n * \\brief warpctc op\n * \\author Liang Xiang\n */\n\n#include \"./warpctc-inl.h\"\n#include \"../../src/operator/mshadow_op.h\"\n\nnamespace mxnet {\nnamespace op {\ntemplate <>\nOperator* CreateOp<cpu>(WarpCTCParam param) {\n  return new WarpCTCOp<cpu>(param);\n}\n\nOperator* WarpCTCProp::CreateOperator(Context ctx) const {\n  DO_BIND_DISPATCH(CreateOp, param_);\n}\n\nDMLC_REGISTER_PARAMETER(WarpCTCParam);\n\nMXNET_REGISTER_OP_PROPERTY(WarpCTC, WarpCTCProp)\n    .add_argument(\"data\", \"NDArray-or-Symbol\", \"Input data.\")\n    .add_argument(\"label\", \"NDArray-or-Symbol\", \"Input label.\")\n    .describe(\"warp ctc.\")\n    .add_arguments(WarpCTCParam::__FIELDS__());\n\n}  // namespace op\n}  // namespace mxnet\n"
  },
  {
    "path": "plugin/warpctc/warpctc.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file warpctc.cc\n * \\brief warpctc op\n * \\author Liang Xiang\n */\n#include \"./warpctc-inl.h\"\n#include <stdio.h>\n#include \"../../src/operator/mshadow_op.h\"\n\nnamespace mxnet {\nnamespace op {\ntemplate <>\nOperator* CreateOp<gpu>(WarpCTCParam param) {\n  return new WarpCTCOp<gpu>(param);\n}\n\n}  // namespace op\n}  // namespace mxnet\n"
  },
  {
    "path": "plugin/warpctc/warpctc.mk",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nCFLAGS += -I$(WARPCTC_PATH)/include\nLDFLAGS += -L$(WARPCTC_PATH)/build -lwarpctc\n\nWARPCTC_SRC = $(wildcard plugin/warpctc/*.cc)\nPLUGIN_OBJ += $(patsubst %.cc, build/%.o, $(WARPCTC_SRC))\nWARPCTC_CUSRC = $(wildcard plugin/warpctc/*.cu)\nPLUGIN_CUOBJ += $(patsubst %.cu, build/%_gpu.o, $(WARPCTC_CUSRC))\n"
  },
  {
    "path": "prospector.yaml",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\noutput-format: text\n\nstrictness: medium\ntest-warnings: true\ndoc-warnings: false\nmax-line-length: 120\n\nignore-paths:\n    - .github\n    - 3rdparty\n    - benchmark\n    - cd\n    - ci\n    - cmake\n    - config\n    - contrib\n    - cpp-package\n    - docker\n    - docs\n    - example\n    - include\n    - licences\n    - plugin\n    - src\n    - tools\n    - build\n    # - python # enabled\n    # - tests  # enabled\n\n    \npylint:\n    disable:\n        - consider-using-enumerate\n        - consider-using-with\n        - unspecified-encoding\n        - consider-using-f-string\n        - simplifiable-if-expression\n        - undefined-variable\n        - deprecated-method\n        - unused-import\n        - wrong-import-position\n        - bare-except\n        - import-outside-toplevel\n        - protected-access\n        - no-value-for-parameter\n        - unused-private-member\n        - import-error\n        - unused-wildcard-import\n        - arguments-differ\n        - logging-format-interpolation\n        - unused-variable\n        - logging-too-many-args\n        - pointless-string-statement\n        - useless-suppression\n        - trailing-newlines\n        - use-maxsplit-arg\n        - redefined-builtin\n        - singleton-comparison\n        - misplaced-comparison-constant\n        - unsubscriptable-object\n        - too-many-locals\n        - too-many-statements\n        - invalid-sequence-index\n        - chained-comparison\n        - pointless-statement\n        - unbalanced-tuple-unpacking\n        - no-else-return\n        - super-with-arguments\n        - use-list-literal\n        - logging-not-lazy\n        - unreachable\n        - too-many-arguments\n        - multiple-imports\n        - bad-indentation\n        - invalid-envvar-default\n        - unused-argument\n        - line-too-long\n        - no-self-use\n        - attribute-defined-outside-init\n        - bad-option-value\n        - global-statement\n        - fixme\n        - no-member\n        - no-name-in-module\n        - superfluous-parens\n        - useless-super-delegation\n        - len-as-condition\n        - invalid-unary-operand-type\n        - consider-using-dict-comprehension\n        - consider-using-set-comprehension\n        - try-except-raise\n        - useless-object-inheritance\n        - c-extension-no-member\n        - deprecated-lambda\n        - too-few-public-methods\n        - too-many-branches\n        - too-many-instance-attributes\n        - too-many-public-methods\n        - too-many-lines\n        - duplicate-code\n        - cyclic-import\n        - cell-var-from-loop\n        - raise-missing-from\n        - unnecessary-comprehension\n        - unidiomatic-typecheck\n        - consider-using-in\n        - unsupported-assignment-operation\n        - unnecessary-pass\n        - reimported\n        - unexpected-keyword-arg\n        - arguments-renamed\n        - consider-using-dict-items\n        - consider-iterating-dictionary\n        - undefined-loop-variable\n        - no-else-continue\n        - too-many-nested-blocks\n        - comparison-with-itself\n        - unnecessary-lambda\n        - too-many-function-args\n        - use-dict-literal\n        - redefined-argument-from-local\n        - function-redefined\n        - bad-staticmethod-argument\n        - consider-using-generator\n        - abstract-method\n        - relative-beyond-top-level\n        - use-a-generator\n        - no-else-break\n        - using-constant-test\n        - use-symbolic-message-instead\n        - bad-inline-option\n        - invalid-name\n        - consider-using-min-builtin\n        - consider-using-max-builtin\n        - trailing-comma-tuple\n        - inconsistent-return-statements\n        - global-variable-not-assigned\n        - literal-comparison\n        - expression-not-assigned\n        - used-before-assignment\n        - disallowed-name\n        - not-callable\n        - implicit-str-concat\n        - self-assigning-variable\n        - dangerous-default-value\n        - eval-used\n        - consider-using-from-import\n        - redundant-u-string-prefix\n    enable: \n        - indexing-exception\n        - old-raise-syntax\n        - undefined-variable\n    \n    options:\n        # Good variable names which should always be accepted, separated by a comma\n        good-names: i,j,_,a,b,op,x,y,wd,lr,kv,k,v,s,p,h,c,m,n,X,t,g,f\n        # Bad variable names which should always be refused, separated by a comma\n        #bad-names:\n        # Colon-delimited sets of names that determine each other's naming style when\n        # the name regexes allow several styles.\n        #name-group:\n        # Include a hint for the correct naming format with invalid-name\n        include-naming-hint: no\n        # List of decorators that produce properties, such as abc.abstractproperty. Add\n        # to this list to register other decorators that produce valid properties.\n        property-classes: abc.abstractproperty\n        # Regular expression matching correct module names\n        module-rgx: (([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$\n        # Naming hint for module names\n        module-name-hint: (([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$\n        # Regular expression matching correct constant names\n        const-rgx: (([A-Z_][A-Z0-9_]*)|(__.*__))$\n        # Naming hint for constant names\n        const-name-hint: (([A-Z_][A-Z0-9_]*)|(__.*__))$\n        # Regular expression matching correct inline iteration names\n        inlinevar-rgx: \"[A-Za-z_][A-Za-z0-9_]*$\"\n        # Naming hint for inline iteration names\n        inlinevar-name-hint: \"[A-Za-z_][A-Za-z0-9_]*$\"\n        # Regular expression matching correct method names\n        method-rgx: \"[a-z_][a-z0-9_]{2,30}$\"\n        # Naming hint for method names\n        method-name-hint: \"[a-z_][a-z0-9_]{2,30}$\"\n        # Regular expression matching correct class attribute names\n        class-attribute-rgx: ([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$\n        # Naming hint for class attribute names\n        class-attribute-name-hint: ([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$\n        # Regular expression matching correct argument names\n        argument-rgx: \"[a-z_][a-z0-9_]{2,30}$\"\n        # Naming hint for argument names\n        argument-name-hint: \"[a-z_][a-z0-9_]{2,30}$\"\n        # Regular expression matching correct attribute names\n        attr-rgx: \"[a-z_][a-z0-9_]{2,30}$\"\n        # Naming hint for attribute names\n        attr-name-hint: \"[a-z_][a-z0-9_]{2,30}$\"\n        # Regular expression matching correct variable names\n        variable-rgx: \"[a-z_][a-z0-9_]{2,30}$\"\n        # Naming hint for variable names\n        variable-name-hint: \"[a-z_][a-z0-9_]{2,30}$\"\n        # Regular expression matching correct function names\n        function-rgx: \"[a-z_][a-z0-9_]{2,30}$\"\n        # Naming hint for function names\n        function-name-hint: \"[a-z_][a-z0-9_]{2,30}$\"\n        # Regular expression matching correct class names\n        class-rgx: \"[A-Za-z_][a-zA-Z0-9]+$\"\n        # Naming hint for class names\n        class-name-hint: \"[A-Z_][a-zA-Z0-9]+$\"\n        # Regular expression which should only match function or class names that do\n        # not require a docstring.\n        no-docstring-rgx: ^_\n        # Minimum line length for functions/classes that require docstrings, shorter\n        # ones are exempt.\n        docstring-min-length: 10\n\nmccabe:\n    disable:\n        - MC0001 # A statement is too complex \n\npep8:\n    disable:\n        # Descriptions and examples for each of the rules in Flake8 https://www.flake8rules.com/\n        - E305  # Expected 2 blank lines after end of function or class\n        - E306  # Expected 1 blank line before a nested definition\n        - E501  # Line too long (139 > 120 characters)\n        - E117  # Over-indented\n        - E722  # Do not use bare except, specify exception instead\n        - E741  # Do not use variables named 'I', 'O', or 'l'\n        - E712  # Comparison to true should be 'if cond is true:' or 'if cond:'\n        - W605  # Invalid escape sequence 'x'\n        - E704  # Multiple statements on one line (def)\n        - F811  # Redefinition of unused name from line N\n        - F632  # Use ==/!= to compare str, bytes, and int literals\n        - F821  # Undefined name name\n        - F524  # .format(...) missing argument\n        - E116  # Unexpected indentation (comment)\n        - E114  # Indentation is not a multiple of four (comment)\n        - N807  # Function name should not start and end with ‘__’\n        - E129  # Visually indented line with same indent as next logical line\n        - E131  # Continuation line unaligned for hanging indent\n        - E713  # Test for membership should be 'not in'\n        - E115  # Expected an indented block (comment)\n        - E714  # Test for object identity should be 'is not'\n        - E711  # Comparison to none should be 'if cond is none:'\n        - E101  # Indentation contains mixed spaces and tabs\n        - E721  # Do not compare types, use 'isinstance()'\n\npyflakes:\n    disable:\n        # Descriptions and examples for each of the rules in Flake8 https://www.flake8rules.com/\n        - F401  # Module imported but unused \n        - F405  # Name may be undefined, or defined from star imports: module\n        - F841  # Local variable name is assigned to but never used\n        - E713  # Test for membership should be 'not in'\n        - E114  # Indentation is not a multiple of four (comment)\n        - E116  # Unexpected indentation (comment)\n        - E711  # Comparison to none should be 'if cond is none:'\n        - E115  # Expected an indented block (comment)\n        - N807  # Function name should not start and end with ‘__’\n        - E101  # Indentation contains mixed spaces and tabs\n        - F811  # Redefinition of unused name from line N\n        - F632  # Use ==/!= to compare str, bytes, and int literals\n        - F821  # undefined name name\n        - F524  # .format(...) missing argument\n"
  },
  {
    "path": "pytest.ini",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n[pytest]\nmarkers =\n    seed: set the python, numpy and mxnet random seeds to a specified value for test reproducibility\n    serial: mark a test that requires more resources to run that are thus only suitable for serial run.\n    remote_required: mark a test that requires internet access.\n    gpu: mark a test that requires GPU.\n    integration: mark an integration test\n    onnx_coverage: ONNX coverage test\n    garbage_expected: this test leaks ndarray references. The tested functionality is broken or there is a Python bug.\n\nenv =\n    MXNET_HOME=tests/data\n\ntimeout = 1200\nfaulthandler_timeout = 1200\n"
  },
  {
    "path": "python/.gitignore",
    "content": "dist\n*.egg-info\nbuild\n*.cpp"
  },
  {
    "path": "python/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\nMXNet Python Package\n====================\nThis directory and nested files contain MXNet Python package and language binding.\n\n## Installation\nTo install MXNet Python package, visit MXNet [Install Instruction](https://mxnet.apache.org/get_started)\n\n\n## Running the unit tests\n\nFor running unit tests, you will need the [pytest PyPi package](https://pypi.python.org/pypi/pytest). To install:\n```bash\npip install --upgrade pytest\n```\n\nOnce ```pytest``` is installed, run the following from MXNet root directory (please make sure the installation path of ```pytest``` is included in your ```$PATH``` environment variable):\n```\npytest tests/python/unittest\npytest tests/python/train\n\n```\n"
  },
  {
    "path": "python/mxnet/__init__.py",
    "content": "#!/usr/bin/env python\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"MXNet: a concise, fast and flexible framework for deep learning.\"\"\"\n\nfrom .context import Context, current_context\nfrom .device import Device, current_device, cpu, gpu, cpu_pinned\nfrom . import engine, error\nfrom .base import MXNetError\nfrom .util import is_np_shape, set_np_shape, np_shape, use_np_shape\nfrom .util import is_np_array, np_array, use_np_array, use_np\nfrom .util import is_np_default_dtype, np_default_dtype, use_np_default_dtype\nfrom . import base\n\n# version info\n__version__ = base.__version__\n\nfrom . import contrib\nfrom . import ndarray\n# use mx.nd as short for mx.ndarray\nfrom . import ndarray as nd\nfrom . import numpy\n# use mx.np as short for mx.numpy\nfrom . import numpy as np\nfrom . import numpy_extension\n# use mx.npx as short for mx.numpy_extension\nfrom . import numpy_extension as npx\nfrom . import name\n# use mx.sym as short for mx.symbol\nfrom . import symbol as sym\n# use mx.np_symbol as short for mx.symbol.numpy\nfrom .symbol.numpy import _symbol as np_symbol\nfrom . import symbol\nfrom . import symbol_doc\nfrom . import io\nfrom . import recordio\nfrom . import operator\n# use mx.rnd as short for mx.random\nfrom . import random as rnd\nfrom . import random\nfrom . import optimizer\nfrom . import model\nfrom . import notebook\nfrom . import initializer\n# use mx.init as short for mx.initializer\nfrom . import initializer as init\nfrom . import visualization\n# use mx.viz as short for mx.visualization\nfrom . import visualization as viz\nfrom . import callback\n# from . import misc\nfrom . import lr_scheduler\n# Runtime compile module\nfrom . import rtc\n# Attribute scope to add attributes to symbolic graphs\nfrom .attribute import AttrScope\n\nfrom . import profiler\nfrom . import log\n\nfrom . import image\n# use mx.img as short for mx.image\nfrom . import image as img\n\nfrom . import test_utils\n\nfrom . import gluon\n\nfrom . import _deferred_compute\n\n# With the native kvstore module (such as 'dist_sync_device'), the module launches a separate\n# process when role is set to \"server\". This should be done after other modules are initialized.\n# Otherwise this may result in errors when unpickling custom LR scheduler/optimizers.\n# For example, the LRScheduler in gluoncv depends on a specific version of MXNet, and\n# checks the __version__ attr of MXNet, which is not set on kvstore server due to the\n# fact that kvstore-server module is imported before the __version__ attr is set.\n# use mx.kv as short for mx.kvstore\nfrom . import kvstore\nfrom . import kvstore as kv\nfrom .kvstore import kvstore_server\n\n# Dynamic library module should be done after ndarray and symbol are initialized\nfrom . import library\nfrom . import tvmop\n\nfrom . import numpy_op_signature\nfrom . import numpy_dispatch_protocol\nfrom . import numpy_op_fallback\n\nfrom . import _global_var\n\nfrom . import _api_internal\nfrom . import api\nfrom . import container\n\nnpx.set_np()\n"
  },
  {
    "path": "python/mxnet/_api_internal.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Namespace of internal API\n\nThe functions in this namespace are automatically exported from C++ side via PackedFunc\nthat is registered by \"MXNET_REGISTER_*\" macro. This way makes calling Python functions from C++\nside very easily.\n\nEach string starts with \"_\" in the \"MXNET_REGISTER_*\" macro is an internal API.\n\nAcknowledgement: This file originates from incubator-tvm\n\"\"\"\n"
  },
  {
    "path": "python/mxnet/_ctypes/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"ctypes module\"\n"
  },
  {
    "path": "python/mxnet/_ctypes/_api_internal.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"CachedOp APIs exposed from C++.\"\"\"\n\nimport mxnet._ffi\n\nmxnet._ffi._init_api(\"cached_op\", __name__)\n"
  },
  {
    "path": "python/mxnet/_ctypes/cached_op.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=invalid-name, protected-access, too-many-arguments\n# pylint: disable=global-statement, unused-import\n\"\"\"CachedOp API.\"\"\"\n\nimport ctypes\n\nfrom ..base import _LIB\nfrom ..base import c_handle_array\nfrom ..base import NDArrayHandle, CachedOpHandle, SymbolHandle\nfrom ..base import check_call\nfrom .. import _global_var\nfrom ..ndarray._internal import NDArrayBase\nfrom . import _api_internal\n\ndef _monitor_callback_wrapper(callback):\n    \"\"\"A wrapper for the user-defined handle.\"\"\"\n    def callback_handle(name, opr_name, array, _):\n        \"\"\" ctypes function \"\"\"\n        callback(name, opr_name, array)\n    return callback_handle\n\nclass CachedOp(object):\n    \"\"\"Cached operator handle.\"\"\"\n    __slots__ = [\"handle\", \"is_np_sym\", \"_monitor_callback\"]\n\n    def __init__(self, sym, flags=(), thread_safe=False):\n        self._monitor_callback = None\n\n        from ..symbol.numpy._symbol import _Symbol\n        self.is_np_sym = bool(isinstance(sym, _Symbol))\n\n        flags = {key: str(value) for key, value in flags}\n        self.handle = CachedOpHandle(_api_internal.create(\n            sym.handle,\n            flags,\n            thread_safe\n        ))\n\n    def __del__(self):\n        _api_internal.free(self.handle)\n\n    def get_optimized_symbol(self):\n        \"\"\"Get an optimized version of the symbol from the cached op.\n\n        Returns\n        -------\n        symbol : Symbol\n            Optimized symbol from the executor.\n        \"\"\"\n        from ..symbol import Symbol\n        sym_handle = SymbolHandle(_api_internal.get_optimized_symbol(self.handle))\n        ret = Symbol(sym_handle)\n        return ret\n\n    def __call__(self, *args, **kwargs):\n        \"\"\"ctypes implementation of imperative invoke wrapper\"\"\"\n        # New FFI only supports numpy ndarray\n        default_device = kwargs.pop('default_device', None)\n        if not default_device:\n            default_device = kwargs.pop('default_ctx', None)\n        out = kwargs.pop('out', None)\n        if kwargs:\n            raise TypeError(\n                \"CachedOp.__call__ got unexpected keyword argument(s): \" + \\\n                ', '.join(kwargs.keys()))\n        if self.is_np_sym:\n            if len(args) == 1 and args[0] is None:\n                args = []\n            type_id = default_device.device_typeid if default_device else None\n            device_id = default_device.device_id if default_device else None\n            out_arg = out if out is not None and not isinstance(out, NDArrayBase) else (out, )\n            output_vars = _api_internal.invoke(\n                self.handle,\n                len(args),\n                *args,\n                type_id,\n                device_id,\n                *out_arg\n            )\n            if out is not None:\n                return out\n            if isinstance(output_vars, NDArrayBase):\n                return output_vars\n            else:\n                return list(output_vars)\n        else:\n            if out is not None:\n                original_output = out\n                if isinstance(out, NDArrayBase):\n                    out = (out,)\n                num_output = ctypes.c_int(len(out))\n                output_vars = c_handle_array(out)\n                output_vars = ctypes.cast(output_vars, ctypes.POINTER(NDArrayHandle))\n            else:\n                original_output = None\n                output_vars = ctypes.POINTER(NDArrayHandle)()\n                num_output = ctypes.c_int(0)\n\n            # return output stypes to avoid the c_api call for checking\n            # a handle's stype in _ndarray_cls\n            out_stypes = ctypes.POINTER(ctypes.c_int)()\n\n            # (None, ) -> []\n            if len(args) == 1 and args[0] is None:\n                args = []\n                assert default_device is not None, 'default_device is required if no input is provided'\n            else:\n                default_device = args[0].device if default_device is None else default_device\n\n            check_call(_LIB.MXInvokeCachedOp(\n                self.handle,\n                ctypes.c_int(len(args)),\n                c_handle_array(args),\n                ctypes.c_int(default_device.device_typeid),\n                ctypes.c_int(default_device.device_id),\n                ctypes.byref(num_output),\n                ctypes.byref(output_vars),\n                ctypes.byref(out_stypes)))\n\n            if original_output is not None:\n                return original_output\n            create_ndarray_fn = _global_var._ndarray_cls\n            if num_output.value == 1:\n                return create_ndarray_fn(ctypes.cast(output_vars[0], NDArrayHandle),\n                                         stype=out_stypes[0])\n            else:\n                return [create_ndarray_fn(ctypes.cast(output_vars[i], NDArrayHandle),\n                                          stype=out_stypes[i]) for i in range(num_output.value)]\n\n    def _register_op_hook(self, callback, monitor_all=False):\n        \"\"\"Install callback for monitor.\n\n        Parameters\n        ----------\n        callback : function\n            Takes a string for node_name, string for op_name and a NDArrayHandle.\n        monitor_all : bool, default False\n            If true, monitor both input _imperative_invoked output, otherwise monitor output only.\n        \"\"\"\n        cb_type = ctypes.CFUNCTYPE(None, ctypes.c_char_p, ctypes.c_char_p, NDArrayHandle, ctypes.c_void_p)\n        if callback:\n            self._monitor_callback = cb_type(_monitor_callback_wrapper(callback))\n        callback_ptr = ctypes.cast(self._monitor_callback, ctypes.c_void_p)\n        _api_internal.register_op_hook(self.handle, callback_ptr, monitor_all)\n"
  },
  {
    "path": "python/mxnet/_ctypes/ndarray.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=invalid-name, protected-access, too-many-arguments\n# pylint: disable=global-statement, unused-import\n\"\"\"NDArray configuration API.\"\"\"\n\nimport ctypes\n\nfrom ..base import _LIB\nfrom ..base import c_str_array, c_handle_array\nfrom ..base import NDArrayHandle\nfrom ..base import check_call\nfrom .. import _global_var\n\nclass NDArrayBase(object):\n    \"\"\"Base data structure for ndarray\"\"\"\n    __slots__ = [\"handle\", \"writable\", \"_alive\"]\n    # pylint: disable= no-member\n\n    def __init__(self, handle, writable=True):\n        \"\"\"initialize a new NDArray\n\n        Parameters\n        ----------\n        handle : NDArrayHandle\n            NDArray handle of C API\n        \"\"\"\n        if handle is not None:\n            assert isinstance(handle, NDArrayHandle)\n        self.handle = handle\n        self.writable = writable\n        self._alive = True\n\n    def __del__(self):\n        check_call(_LIB.MXNDArrayFree(self.handle))\n        self._alive = False\n\n    def __reduce__(self):\n        return (_global_var._ndarray_cls, (None,), self.__getstate__())\n\n\ndef _imperative_invoke(handle, ndargs, keys, vals, out, is_np_op, output_is_list):\n    \"\"\"ctypes implementation of imperative invoke wrapper\"\"\"\n    if out is not None:\n        original_output = out\n        if isinstance(out, NDArrayBase):\n            out = (out,)\n        num_output = ctypes.c_int(len(out))\n        output_vars = c_handle_array(out)\n        output_vars = ctypes.cast(output_vars, ctypes.POINTER(NDArrayHandle))\n    else:\n        original_output = None\n        output_vars = ctypes.POINTER(NDArrayHandle)()\n        num_output = ctypes.c_int(0)\n\n    # return output stypes to avoid the c_api call for checking\n    # a handle's stype in _ndarray_cls\n    out_stypes = ctypes.POINTER(ctypes.c_int)()\n\n    check_call(_LIB.MXImperativeInvoke(\n        ctypes.c_void_p(handle),\n        ctypes.c_int(len(ndargs)),\n        c_handle_array(ndargs),\n        ctypes.byref(num_output),\n        ctypes.byref(output_vars),\n        ctypes.c_int(len(keys)),\n        c_str_array(keys),\n        c_str_array([str(s) for s in vals]),\n        ctypes.byref(out_stypes)))\n\n    create_ndarray_fn = _global_var._np_ndarray_cls if is_np_op else _global_var._ndarray_cls\n    if original_output is not None:\n        return original_output\n    if num_output.value == 1 and not output_is_list:\n        return create_ndarray_fn(ctypes.cast(output_vars[0], NDArrayHandle),\n                                 stype=out_stypes[0])\n    else:\n        return [create_ndarray_fn(ctypes.cast(output_vars[i], NDArrayHandle),\n                                  stype=out_stypes[i]) for i in range(num_output.value)]\n"
  },
  {
    "path": "python/mxnet/_ctypes/space.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"ConfigSpace ctypes API.\"\"\"\nimport ctypes\n\nfrom ..base import _LIB\nfrom ..base import c_str_array, c_array\nfrom ..base import check_call\n\nclass COtherOptionEntity(ctypes.Structure):\n    \"\"\"ctypes data structure for OtherOptionEntity\"\"\"\n    _fields_ = [(\"val\", ctypes.c_int)]\n\n\nclass COtherOptionSpace(ctypes.Structure):\n    \"\"\"ctypes data structure for OtherOptionSpace\"\"\"\n    _fields_ = [(\"entities\", ctypes.POINTER(COtherOptionEntity)),\n                (\"entities_size\", ctypes.c_int)]\n\n\nclass CConfigSpace(ctypes.Structure):\n    \"\"\"ctypes data structure for ConfigSpace\"\"\"\n    _fields_ = [(\"entity_map_size\", ctypes.c_int),\n                (\"entity_map_key\", ctypes.POINTER(ctypes.c_char_p)),\n                (\"entity_map_val\", ctypes.POINTER(COtherOptionEntity)),\n                (\"space_map_size\", ctypes.c_int),\n                (\"space_map_key\", ctypes.POINTER(ctypes.c_char_p)),\n                (\"space_map_val\", ctypes.POINTER(COtherOptionSpace))]\n\n\nclass CConfigSpaces(ctypes.Structure):\n    \"\"\"ctypes data structure for ConfigSpaces\"\"\"\n    _fields_ = [(\"spaces_size\", ctypes.c_int),\n                (\"spaces_key\", ctypes.POINTER(ctypes.c_char_p)),\n                (\"spaces_val\", ctypes.POINTER(CConfigSpace))]\n\n\ndef c_other_option_entity(x):\n    \"\"\"constructor for OtherOptionEntity\"\"\"\n    ret = COtherOptionEntity()\n    ret.val = x.val\n    return ret\n\n\ndef c_other_option_space(x):\n    \"\"\"constructor for OtherOptionSpace\"\"\"\n    ret = COtherOptionSpace()\n    ret.entities = c_array(COtherOptionEntity,\n                           [c_other_option_entity(e) for e in x.entities])\n    ret.entities_size = len(x.entities)\n    return ret\n\n\ndef c_config_space(x):\n    \"\"\"constructor for ConfigSpace\"\"\"\n    ret = CConfigSpace()\n    ret.entity_map_key = c_str_array(x._entity_map.keys())\n    ret.entity_map_val = c_array(COtherOptionEntity,\n                                 [c_other_option_entity(e) for e in x._entity_map.values()])\n    ret.entity_map_size = len(x._entity_map)\n    ret.space_map_key = c_str_array(x.space_map.keys())\n    ret.space_map_val = c_array(COtherOptionSpace,\n                                [c_other_option_space(v) for v in x.space_map.values()])\n    ret.space_map_size = len(x.space_map)\n    return ret\n\n\ndef c_config_spaces(x):\n    \"\"\"constructor for ConfigSpaces\"\"\"\n    ret = CConfigSpaces()\n    ret.spaces_size = len(x.spaces)\n    ret.spaces_key = c_str_array(x.spaces.keys())\n    ret.spaces_val = c_array(CConfigSpace, [c_config_space(c) for c in x.spaces.values()])\n    return ret\n\n\ndef _set_tvm_op_config(x):\n    \"\"\"ctypes implementation of populating the config singleton\"\"\"\n    check_call(_LIB.MXLoadTVMConfig(c_config_spaces(x)))\n    return x\n"
  },
  {
    "path": "python/mxnet/_ctypes/symbol.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=invalid-name, protected-access, too-many-arguments,  global-statement\n\"\"\"Symbolic configuration API.\"\"\"\n\nimport ctypes\nfrom ..base import _LIB\nfrom ..base import c_str_array, c_handle_array, c_str, mx_uint\nfrom ..base import SymbolHandle\nfrom ..base import check_call\n\n# The symbol class to be used (Cython or Ctypes)\n_symbol_cls = None\n_np_symbol_cls = None\n\nclass SymbolBase(object):\n    \"\"\"Symbol is symbolic graph.\"\"\"\n    __slots__ = [\"handle\", \"_alive\"]\n    # pylint: disable=no-member\n    def __init__(self, handle):\n        \"\"\"Initialize the function with handle\n\n        Parameters\n        ----------\n        handle : SymbolHandle\n            the handle to the underlying C++ Symbol\n        \"\"\"\n        self.handle = handle\n        self._alive = True\n\n    def __del__(self):\n        check_call(_LIB.NNSymbolFree(self.handle))\n        self._alive = False\n\n    def _compose(self, *args, **kwargs):\n        \"\"\"Compose symbol on inputs.\n\n        This call mutates the current symbol.\n\n        Parameters\n        ----------\n        args:\n            provide positional arguments\n\n        kwargs:\n            provide keyword arguments\n\n        Returns\n        -------\n        the resulting symbol\n        \"\"\"\n        name = kwargs.pop('name', None)\n\n        if name:\n            name = c_str(name)\n        if len(args) != 0 and len(kwargs) != 0:\n            raise TypeError('compose only accept input Symbols \\\n                either as positional or keyword arguments, not both')\n\n        for arg in args:\n            if not isinstance(arg, SymbolBase):\n                raise TypeError('Compose expect `Symbol` as arguments')\n        for val in kwargs.values():\n            if not isinstance(val, SymbolBase):\n                raise TypeError('Compose expect `Symbol` as arguments')\n\n        num_args = len(args) + len(kwargs)\n        if len(kwargs) != 0:\n            keys = c_str_array(kwargs.keys())\n            args = c_handle_array(kwargs.values())\n        else:\n            keys = None\n            args = c_handle_array(kwargs.values())\n        check_call(_LIB.NNSymbolCompose(\n            self.handle, name, num_args, keys, args))\n\n    def _set_attr(self, **kwargs):\n        \"\"\"Set the attribute of the symbol.\n\n        Parameters\n        ----------\n        **kwargs\n            The attributes to set\n        \"\"\"\n        keys = c_str_array(kwargs.keys())\n        vals = c_str_array([str(s) for s in kwargs.values()])\n        num_args = mx_uint(len(kwargs))\n        check_call(_LIB.MXSymbolSetAttrs(\n            self.handle, num_args, keys, vals))\n\n    def _set_handle(self, handle):\n        \"\"\"Set handle.\"\"\"\n        self.handle = handle\n\n    def __reduce__(self):\n        return (_symbol_cls, (None,), self.__getstate__())\n\n\ndef _set_symbol_class(cls):\n    \"\"\"Set the symbolic class to be cls\"\"\"\n    global _symbol_cls\n    _symbol_cls = cls\n\n\ndef _set_np_symbol_class(cls):\n    \"\"\"Set the numpy-compatible symbolic class to be cls\"\"\"\n    global _np_symbol_cls\n    _np_symbol_cls = cls\n\n\ndef _symbol_creator(handle, args, kwargs, keys, vals, name, is_np_op, output_is_list=False):\n    sym_handle = SymbolHandle()\n    check_call(_LIB.MXSymbolCreateAtomicSymbol(\n        ctypes.c_void_p(handle),\n        mx_uint(len(keys)),\n        c_str_array(keys),\n        c_str_array([str(v) for v in vals]),\n        ctypes.byref(sym_handle)))\n\n    if args and kwargs:\n        raise TypeError(\n            'Operators with variable length input can only accept input'\n            'Symbols either as positional or keyword arguments, not both')\n    create_symbol_fn = _np_symbol_cls if is_np_op else _symbol_cls\n    s = create_symbol_fn(sym_handle)\n    if args:\n        s._compose(*args, name=name)\n    elif kwargs:\n        s._compose(name=name, **kwargs)\n    else:\n        s._compose(name=name)\n    if is_np_op:\n        # Determine whether the symbol is a list.\n        if s.num_outputs > 1:\n            return list(s)\n        elif output_is_list:\n            return [s]\n    return s\n"
  },
  {
    "path": "python/mxnet/_cy3/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\nThis folder is by default empty and will hold DLLs generated by cython.\n"
  },
  {
    "path": "python/mxnet/_cy3/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for cython generated modules for python3\"\"\"\n"
  },
  {
    "path": "python/mxnet/_deferred_compute.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Deferred Compute for NDArray.\"\"\"\n\nimport ctypes\nimport contextlib\n\nfrom .base import _LIB, check_call, SymbolHandle, _as_list\nfrom .symbol import Symbol\n\n__all__ = []\n\ndef is_deferred_compute():\n    \"\"\"Get status of deferred compute mode.\"\"\"\n    curr = ctypes.c_bool()\n    check_call(_LIB.MXNDArrayIsDeferredCompute(ctypes.byref(curr)))\n    return curr.value\n\ndef set_deferred_compute(state):\n    \"\"\"Enable / Disable deferred compute mode.\n\n    Parameters\n    ----------\n    state: bool\n\n    Returns\n    -------\n    Previous deferred compute state.\n    \"\"\"\n    prev = ctypes.c_int()\n    check_call(_LIB.MXNDArraySetIsDeferredCompute(ctypes.c_int(state), ctypes.byref(prev)))\n    return bool(prev.value)\n\n\n@contextlib.contextmanager\ndef context(state=True):\n    \"\"\"Set deferred compute state to `state` within context. Reset afterwards to previous value.\"\"\"\n    # Like other MXNet context manager, this bleeds state across concurrent\n    # code: \"Context managers that have state should use Context Variables\n    # instead of threading.local() to prevent their state from bleeding to\n    # other code unexpectedly, when used in concurrent code.\"\n    # https://github.com/apache/incubator-mxnet/issues/17495#issuecomment-585461965\n    val = set_deferred_compute(state)\n    try:\n        yield\n    finally:\n        set_deferred_compute(val)\n\n\ndef get_symbol(output_arrays, *, sym_cls=Symbol):\n    \"\"\"Get symbolic representation of computation recorded in deferred compute mode.\n\n    Parameters\n    ----------\n    output_arrays: NDArray or List[NDArray]\n    sym_cls: class used to construct Symbol\n\n    Returns\n    -------\n    Symbol of sym_cls\n    \"\"\"\n    output_arrays = _as_list(output_arrays)\n    # Prepare ctypes array types\n    output_handles_type = ctypes.c_void_p * len(output_arrays)\n    # Convert handles\n    output_handles = output_handles_type(*[array.handle for array in output_arrays])\n    handle = SymbolHandle()\n    check_call(_LIB.MXNDArrayGetDeferredComputeSymbol(output_handles, len(output_arrays),\n                                                      ctypes.byref(handle)))\n    return sym_cls(handle)\n\n\ndef set_variable(arrays, variables):\n    \"\"\"Associate variables with arrays.\n\n    Parameters\n    ----------\n    arrays: NDArray or List[NDArray]\n    variables: Symbol or List[Symbol] of variables\n    \"\"\"\n\n    arrays = _as_list(arrays)\n    variables = _as_list(variables)\n\n    # Prepare ctypes array types\n    arrays_type = variables_type = ctypes.c_void_p * len(arrays)\n\n    # Convert handles\n    arrays = arrays_type(*[array.handle for array in arrays])\n    variables = variables_type(*[symbol.handle for symbol in variables])\n\n    check_call(_LIB.MXNDArraySetDeferredComputeVariable(arrays, variables, len(arrays)))\n\n\ndef clear(arrays):\n    \"\"\"Clear the dc info node associated with output variables.\n\n    Parameters\n    ----------\n    arrays: NDArray or List[NDArray]\n    \"\"\"\n\n    arrays = _as_list(arrays)\n\n    # Prepare ctypes array types\n    arrays_type = ctypes.c_void_p * len(arrays)\n\n    # Convert handles\n    arrays = arrays_type(*[array.handle for array in arrays])\n\n    check_call(_LIB.MXNDArrayClearDeferredCompute(arrays, len(arrays)))\n"
  },
  {
    "path": "python/mxnet/_ffi/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"\nAcknowledgement: This file originates from incubator-tvm\n\"\"\"\nfrom .function import _init_api, get_global_func\nfrom .node_generic import convert_to_node\n"
  },
  {
    "path": "python/mxnet/_ffi/_ctypes/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"\nctypes specific implementation of FFI\nAcknowledgement: This file originates from incubator-tvm\n\"\"\"\n"
  },
  {
    "path": "python/mxnet/_ffi/_ctypes/function.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n# coding: utf-8\n# pylint: disable=invalid-name, protected-access, too-many-branches, global-statement, unused-import\n\"\"\"\nFunction configuration API.\nAcknowledgement: This file originates from incubator-tvm\n\"\"\"\nimport ctypes\nfrom numbers import Number, Integral\nimport numpy as onp\n\nfrom ...base import get_last_ffi_error, _LIB, check_call, _MAX_VALUE_64_BIT_SIGNED_, _MAX_VALUE_64_BIT_UNSIGNED_\nfrom ..base import c_str\nfrom .types import MXNetValue, TypeCode\nfrom .types import RETURN_SWITCH\nfrom ..._ctypes.ndarray import NDArrayBase\nfrom .object import ObjectBase, PyNativeObject, _set_class_object\nfrom . import object as _object\n\nObjectHandle = ctypes.c_void_p\nFunctionHandle = ctypes.c_void_p\n\ndef _make_packed_func(handle, is_global):\n    \"\"\"Make a packed function class\"\"\"\n    obj = _CLASS_PACKED_FUNC.__new__(_CLASS_PACKED_FUNC)\n    obj.is_global = is_global\n    obj.handle = handle\n    return obj\n\ndef _get_global_func(name, allow_missing=False):\n    handle = FunctionHandle()\n    check_call(_LIB.MXNetFuncGetGlobal(c_str(name), ctypes.byref(handle)))\n    if handle.value:\n        return _make_packed_func(handle, False)\n\n    if allow_missing:\n        return None\n\n    raise ValueError(f\"Cannot find global function {name}\")\n\ndef _make_mxnet_args(args, temp_args):\n    \"\"\"Pack arguments into c args mxnet call accept\"\"\"\n    num_args = len(args)\n    values = (MXNetValue * num_args)()\n    type_codes = (ctypes.c_int * num_args)()\n    for i, arg in enumerate(args):\n        if isinstance(arg, NDArrayBase):\n            values[i].v_handle = arg.handle\n            type_codes[i] = TypeCode.NDARRAYHANDLE\n        elif isinstance(arg, Integral):\n            if arg > _MAX_VALUE_64_BIT_UNSIGNED_:\n                raise OverflowError(\"Integer out of bounds\")\n            if arg > _MAX_VALUE_64_BIT_SIGNED_:\n                values[i].v_uint64 = arg\n                type_codes[i] = TypeCode.UINT\n            else:\n                values[i].v_int64 = arg\n                type_codes[i] = TypeCode.INT\n        elif isinstance(arg, ObjectBase):\n            values[i].v_handle = arg.handle\n            type_codes[i] = TypeCode.OBJECT_HANDLE\n        elif arg is None:\n            values[i].v_handle = None\n            type_codes[i] = TypeCode.NULL\n        elif isinstance(arg, PyNativeObject):\n            values[i].v_handle = arg.__mxnet_object__.handle\n            type_codes[i] = TypeCode.OBJECT_HANDLE\n        elif isinstance(arg, Number):\n            values[i].v_float64 = arg\n            type_codes[i] = TypeCode.FLOAT\n        elif isinstance(arg, str):\n            values[i].v_str = c_str(arg)\n            type_codes[i] = TypeCode.STR\n        elif isinstance(arg, (list, tuple, dict)):\n            arg = _FUNC_CONVERT_TO_NODE(arg)\n            values[i].v_handle = arg.handle\n            type_codes[i] = TypeCode.OBJECT_HANDLE\n            temp_args.append(arg)\n        elif isinstance(arg, ctypes.c_void_p):\n            values[i].v_handle = arg\n            type_codes[i] = TypeCode.HANDLE\n        elif isinstance(arg, type):\n            values[i].v_str = c_str(onp.dtype(arg).name)\n            type_codes[i] = TypeCode.STR\n        else:\n            raise TypeError(f\"Don't know how to handle type {type(arg)}\")\n    return values, type_codes, num_args\n\n\nclass FunctionBase(object):\n    \"\"\"Function base.\"\"\"\n    __slots__ = [\"handle\", \"is_global\"]\n    # pylint: disable=no-member\n    def __init__(self, handle, is_global):\n        \"\"\"Initialize the function with handle\n\n        Parameters\n        ----------\n        handle : FunctionHandle\n            the handle to the underlying function.\n\n        is_global : bool\n            Whether this is a global function in python\n        \"\"\"\n        self.handle = handle\n        self.is_global = is_global\n\n    def __del__(self):\n        if not self.is_global and _LIB is not None:\n            if _LIB.MXNetFuncFree(self.handle) != 0:\n                raise get_last_ffi_error()\n\n    def __call__(self, *args):\n        \"\"\"Call the function with positional arguments\n\n        args : list\n           The positional arguments to the function call.\n        \"\"\"\n        temp_args = []\n        values, tcodes, num_args = _make_mxnet_args(args, temp_args)\n        ret_val = MXNetValue()\n        ret_tcode = ctypes.c_int()\n        if _LIB.MXNetFuncCall(\n                self.handle, values, tcodes, ctypes.c_int(num_args),\n                ctypes.byref(ret_val), ctypes.byref(ret_tcode)) != 0:\n            raise get_last_ffi_error()\n        _ = temp_args\n        _ = args\n        return (RETURN_SWITCH[ret_tcode.value](ret_val) if ret_tcode.value != TypeCode.PYARG\n                else RETURN_SWITCH[ret_tcode.value](ret_val, args))\n\n\ndef __init_handle_by_constructor__(fconstructor, args):\n    \"\"\"Initialize handle by constructor\"\"\"\n    temp_args = []\n    values, tcodes, num_args = _make_mxnet_args(args, temp_args)\n    ret_val = MXNetValue()\n    ret_tcode = ctypes.c_int()\n    if _LIB.MXNetFuncCall(\n            fconstructor.handle, values, tcodes, ctypes.c_int(num_args),\n            ctypes.byref(ret_val), ctypes.byref(ret_tcode)) != 0:\n        raise get_last_ffi_error()\n    _ = temp_args\n    _ = args\n    assert ret_tcode.value == TypeCode.OBJECT_HANDLE\n    handle = ret_val.v_handle\n    return handle\n\n_object.__init_by_constructor__ = __init_handle_by_constructor__\n\n_CLASS_PACKED_FUNC = None\n_FUNC_CONVERT_TO_NODE = None\n\ndef _set_class_packed_func(packed_func_class):\n    \"\"\"Initialize packed function defined in cython\"\"\"\n    global _CLASS_PACKED_FUNC\n    _CLASS_PACKED_FUNC = packed_func_class\n\ndef _set_node_generic(func_convert_to_node):\n    \"\"\"Initialize packed function type conversion function in cython\"\"\"\n    global _FUNC_CONVERT_TO_NODE\n    _FUNC_CONVERT_TO_NODE = func_convert_to_node\n"
  },
  {
    "path": "python/mxnet/_ffi/_ctypes/object.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n# pylint: disable=invalid-name\n\"\"\"\nRuntime Object api\nAcknowledgement: This file originates from incubator-tvm\n\"\"\"\nimport ctypes\nfrom ...base import _LIB, check_call\nfrom .types import RETURN_SWITCH, TypeCode\n\nObjectHandle = ctypes.c_void_p\n__init_by_constructor__ = None\n\n\"\"\"Maps object type to its constructor\"\"\"\nOBJECT_TYPE = {}\n\n_CLASS_OBJECT = None\n\ndef _set_class_object(object_class):\n    \"\"\"Initialize object class defined in python\"\"\"\n    global _CLASS_OBJECT\n    _CLASS_OBJECT = object_class\n\ndef _register_object(index, cls):\n    \"\"\"register object class\"\"\"\n    # if issubclass(cls, NDArrayBase):\n    #     _register_ndarray(index, cls)\n    #     return\n    OBJECT_TYPE[index] = cls\n\n\ndef _return_object(x):\n    handle = x.v_handle\n    if not isinstance(handle, ObjectHandle):\n        handle = ObjectHandle(handle)\n    tindex = ctypes.c_uint()\n    check_call(_LIB.MXNetObjectGetTypeIndex(handle, ctypes.byref(tindex)))\n    cls = OBJECT_TYPE.get(tindex.value, _CLASS_OBJECT)\n    if issubclass(cls, PyNativeObject):\n        obj = _CLASS_OBJECT.__new__(_CLASS_OBJECT)\n        obj.handle = handle\n        return cls.__from_mxnet_object__(cls, obj)\n    # Avoid calling __init__ of cls, instead directly call __new__\n    # This allows child class to implement their own __init__\n    obj = cls.__new__(cls)\n    obj.handle = handle\n    return obj\n\nRETURN_SWITCH[TypeCode.OBJECT_HANDLE] = _return_object\n\nclass PyNativeObject:\n    \"\"\"Base class of all MXNet objects that also subclass python's builtin types.\"\"\"\n\n    __slots__ = []\n\n    def __init_mxnet_object_by_constructor__(self, fconstructor, *args):\n        \"\"\"Initialize the internal mxnet_object by calling constructor function.\n\n        Parameters\n        ----------\n        fconstructor : Function\n            Constructor function.\n\n        args: list of objects\n            The arguments to the constructor\n\n        Note\n        ----\n        We have a special calling convention to call constructor functions.\n        So the return object is directly set into the object\n        \"\"\"\n        # pylint: disable=assigning-non-slot\n        obj = _CLASS_OBJECT.__new__(_CLASS_OBJECT)\n        obj.__init_handle_by_constructor__(fconstructor, *args)\n        self.__mxnet_object__ = obj\n\nclass ObjectBase(object):\n    \"\"\"Base object for all object types\"\"\"\n    __slots__ = [\"handle\"]\n\n    def __del__(self):\n        if _LIB is not None:\n            check_call(_LIB.MXNetObjectFree(self.handle))\n\n    def __init_handle_by_constructor__(self, fconstructor, *args):\n        \"\"\"Initialize the handle by calling constructor function.\n\n        Parameters\n        ----------\n        fconstructor : Function\n            Constructor function.\n\n        args: list of objects\n            The arguments to the constructor\n\n        Note\n        ----\n        We have a special calling convention to call constructor functions.\n        So the return handle is directly set into the Node object\n        instead of creating a new Node.\n        \"\"\"\n        # assign handle first to avoid error raising\n        self.handle = None\n        handle = __init_by_constructor__(fconstructor, args)\n        if not isinstance(handle, ObjectHandle):\n            handle = ObjectHandle(handle)\n        self.handle = handle\n\n    def same_as(self, other):\n        \"\"\"Check object identity.\n\n        Parameters\n        ----------\n        other : object\n            The other object to compare against.\n\n        Returns\n        -------\n        result : bool\n             The comparison result.\n        \"\"\"\n        if not isinstance(other, ObjectBase):\n            return False\n        if self.handle is None:\n            return other.handle is None\n        return self.handle.value == other.handle.value\n"
  },
  {
    "path": "python/mxnet/_ffi/_ctypes/types.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"The C Types used in API.\nAcknowledgement: This file originates from incubator-tvm\n\"\"\"\n# pylint: disable=invalid-name\nimport ctypes\nfrom ..base import py_str\nfrom ...base import NDArrayHandle\nfrom ... import _global_var\n\n\nclass TypeCode(object):\n    \"\"\"Type code used in API calls\"\"\"\n    INT = 0\n    UINT = 1\n    FLOAT = 2\n    HANDLE = 3\n    NULL = 4\n    MXNET_TYPE = 5\n    MXNET_CONTEXT = 6\n    OBJECT_HANDLE = 7\n    STR = 8\n    BYTES = 9\n    PYARG = 10\n    NDARRAYHANDLE = 11\n    EXT_BEGIN = 15\n\n\nclass MXNetValue(ctypes.Union):\n    \"\"\"MXNetValue in C API\"\"\"\n    _fields_ = [(\"v_int64\", ctypes.c_int64),\n                (\"v_float64\", ctypes.c_double),\n                (\"v_handle\", ctypes.c_void_p),\n                (\"v_str\", ctypes.c_char_p),\n                (\"v_uint64\", ctypes.c_uint64)]\n\nRETURN_SWITCH = {\n    TypeCode.INT: lambda x: x.v_int64,\n    TypeCode.UINT: lambda x: x.v_uint64,\n    TypeCode.FLOAT: lambda x: x.v_float64,\n    TypeCode.NULL: lambda x: None,\n    TypeCode.STR: lambda x: py_str(x.v_str),\n    TypeCode.NDARRAYHANDLE: lambda x: _global_var._np_ndarray_cls(handle=NDArrayHandle(x.v_handle)),\n    TypeCode.HANDLE: lambda x: x.v_handle,\n    TypeCode.PYARG: lambda x, args: args[x.v_int64],\n}\n"
  },
  {
    "path": "python/mxnet/_ffi/_cy3/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"cython3 namespace\nAcknowledgement: This file originates from incubator-tvm\n\"\"\"\n"
  },
  {
    "path": "python/mxnet/_ffi/_cython/base.pxi",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Acknowledgement: This file originates from incubator-tvm\"\"\"\n\nfrom libcpp.vector cimport vector\nfrom cpython.version cimport PY_MAJOR_VERSION\nfrom cpython cimport pycapsule\nfrom libc.stdint cimport int32_t, int64_t, uint64_t, uint8_t, uint16_t, uint32_t\nimport ctypes\nfrom ...base import get_last_ffi_error, _MAX_VALUE_64_BIT_SIGNED_, _MAX_VALUE_64_BIT_UNSIGNED_\n\ncdef enum MXNetTypeCode:\n    kInt = 0\n    kUInt = 1\n    kFloat = 2\n    kHandle = 3\n    kNull = 4\n    kMXNetType = 5\n    kMXNetContext = 6\n    kObjectHandle = 7\n    kStr = 8\n    kBytes = 9\n    kPyArg = 10\n    kNDArrayHandle = 11\n    kExtBegin = 15\n\ncdef extern from \"mxnet/runtime/c_runtime_api.h\":\n    ctypedef struct MXNetValue:\n        int64_t v_int64\n        double v_float64\n        void* v_handle\n        const char* v_str\n        uint64_t v_uint64\n\nctypedef void* MXNetRetValueHandle\nctypedef void* MXNetFunctionHandle\nctypedef void* ObjectHandle\n\n\ncdef extern from \"mxnet/runtime/c_runtime_api.h\":\n    int MXNetFuncCall(MXNetFunctionHandle func,\n                      MXNetValue* arg_values,\n                      int* type_codes,\n                      int num_args,\n                      MXNetValue* ret_val,\n                      int* ret_type_code)\n    int MXNetFuncFree(MXNetFunctionHandle func)\n    int MXNetObjectFree(ObjectHandle obj)\n    int MXNetObjectGetTypeIndex(ObjectHandle obj, unsigned* out_index)\n    int MXNetFuncGetGlobal(const char* name,\n                           MXNetFunctionHandle* out)\n\ncdef inline py_str(const char* x):\n    if PY_MAJOR_VERSION < 3:\n        return x\n    else:\n        return x.decode(\"utf-8\")\n\n\ncdef inline c_str(pystr):\n    \"\"\"Create ctypes char * from a python string\n    Parameters\n    ----------\n    string : string type\n        python string\n\n    Returns\n    -------\n    str : c_char_p\n        A char pointer that can be passed to C API\n    \"\"\"\n    return pystr.encode(\"utf-8\")\n\n\ncdef inline CALL(int ret):\n    if ret != 0:\n        raise get_last_ffi_error()\n\n\ncdef inline object ctypes_handle(void* chandle):\n    \"\"\"Cast C handle to ctypes handle.\"\"\"\n    return ctypes.cast(<unsigned long long>chandle, ctypes.c_void_p)\n\n\ncdef inline void* c_handle(object handle):\n    \"\"\"Cast C types handle to c handle.\"\"\"\n    cdef unsigned long long v_ptr\n    v_ptr = handle.value\n    return <void*>(v_ptr)\n"
  },
  {
    "path": "python/mxnet/_ffi/_cython/core.pyx",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Acknowledgement: This file originates from incubator-tvm\"\"\"\n\ninclude \"./base.pxi\"\ninclude \"./ndarray.pxi\"\ninclude \"./object.pxi\"\ninclude \"./function.pxi\"\n"
  },
  {
    "path": "python/mxnet/_ffi/_cython/function.pxi",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Acknowledgement: This file originates from incubator-tvm\"\"\"\n\nimport ctypes\nimport numpy as onp\nimport traceback\nfrom ...ndarray._internal import NDArrayBase\nfrom numbers import Number, Integral\n\n\ncdef inline int make_arg(object arg,\n                         MXNetValue* value,\n                         int* tcode,\n                         list temp_args) except -1:\n    \"\"\"Pack arguments into c args mxnet call accept\"\"\"\n    cdef unsigned long long ptr\n\n    if isinstance(arg, NDArrayBase):\n        value[0].v_handle = <void*><size_t>(arg._get_handle())\n        tcode[0] = kNDArrayHandle\n    elif isinstance(arg, Integral):\n        if arg > _MAX_VALUE_64_BIT_UNSIGNED_:\n            raise OverflowError(\"Integer out of bounds\")\n        elif arg > _MAX_VALUE_64_BIT_SIGNED_:\n            value[0].v_uint64 = arg\n            tcode[0] = kUInt\n        else:\n            value[0].v_int64 = arg\n            tcode[0] = kInt\n    elif isinstance(arg, ObjectBase):\n        value[0].v_handle = (<ObjectBase>arg).chandle\n        tcode[0] = kObjectHandle\n    elif isinstance(arg, float):\n        value[0].v_float64 = arg\n        tcode[0] = kFloat\n    elif isinstance(arg, PyNativeObject):\n        value[0].v_handle = (<ObjectBase>(arg.__mxnet_object__)).chandle\n        tcode[0] = kObjectHandle\n    elif isinstance(arg, str):\n        tstr = c_str(arg)\n        value[0].v_str = tstr\n        tcode[0] = kStr\n        temp_args.append(tstr)\n    elif isinstance(arg, (list, tuple, dict)):\n        arg = _FUNC_CONVERT_TO_NODE(arg)\n        value[0].v_handle = (<ObjectBase>arg).chandle\n        tcode[0] = kObjectHandle\n        temp_args.append(arg)\n    elif arg is None:\n        value[0].v_handle = NULL\n        tcode[0] = kNull\n    elif isinstance(arg, Number):\n        value[0].v_float64 = arg\n        tcode[0] = kFloat\n    elif isinstance(arg, ctypes.c_void_p):\n        value[0].v_handle = c_handle(arg)\n        tcode[0] = kHandle\n    elif isinstance(arg, type):\n        tstr = c_str(onp.dtype(arg).name)\n        value[0].v_str = tstr\n        tcode[0] = kStr\n        temp_args.append(tstr)\n    else:\n        raise TypeError(\"Don't know how to handle type %s\" % type(arg))\n    return 0\n\n\ncdef inline object make_ret(MXNetValue value, int tcode):\n    \"\"\"convert result to return value.\"\"\"\n    if tcode == kNDArrayHandle:\n        return c_make_array(value.v_handle)\n    elif tcode == kNull:\n        return None\n    elif tcode == kObjectHandle:\n        return make_ret_object(value.v_handle)\n    elif tcode == kInt:\n        return value.v_int64\n    elif tcode == kFloat:\n        return value.v_float64\n    elif tcode == kStr:\n        return py_str(value.v_str)\n    elif tcode == kHandle:\n        return <unsigned long long>(value.v_handle)\n    raise ValueError(\"Unhandled type code %d\" % tcode)\n\n\ncdef inline int FuncCall3(void* chandle,\n                          tuple args,\n                          int nargs,\n                          MXNetValue* ret_val,\n                          int* ret_tcode) except -1:\n    cdef MXNetValue[3] values\n    cdef int[3] tcodes\n    nargs = len(args)\n    temp_args = []\n    for i in range(nargs):\n        make_arg(args[i], &values[i], &tcodes[i], temp_args)\n    CALL(MXNetFuncCall(chandle, &values[0], &tcodes[0],\n                     nargs, ret_val, ret_tcode))\n    return 0\n\n\ncdef inline int FuncCall(void* chandle,\n                         tuple args,\n                         MXNetValue* ret_val,\n                         int* ret_tcode) except -1:\n    cdef int nargs\n    nargs = len(args)\n    if nargs <= 3:\n        FuncCall3(chandle, args, nargs, ret_val, ret_tcode)\n        return 0\n\n    cdef vector[MXNetValue] values\n    cdef vector[int] tcodes\n    values.resize(nargs)\n    tcodes.resize(nargs)\n\n    temp_args = []\n    for i in range(nargs):\n        make_arg(args[i], &values[i], &tcodes[i], temp_args)\n    CALL(MXNetFuncCall(chandle, &values[0], &tcodes[0],\n                     nargs, ret_val, ret_tcode))\n    return 0\n\n\ncdef inline int ConstructorCall(void* constructor_handle,\n                                int type_code,\n                                tuple args,\n                                void** handle) except -1:\n    \"\"\"Call contructor of a handle function\"\"\"\n    cdef MXNetValue ret_val\n    cdef int ret_tcode\n    FuncCall(constructor_handle, args, &ret_val, &ret_tcode)\n    assert ret_tcode == type_code\n    handle[0] = ret_val.v_handle\n    return 0\n\n\ncdef class FunctionBase:\n    cdef MXNetFunctionHandle chandle\n    cdef int is_global\n\n    cdef inline _set_handle(self, handle):\n        if handle is None:\n            self.chandle = NULL\n        else:\n            self.chandle = c_handle(handle)\n\n    property is_global:\n        def __get__(self):\n            return self.c_is_global != 0\n\n        def __set__(self, value):\n            self.c_is_global = value\n\n    property handle:\n        def __get__(self):\n            if self.chandle == NULL:\n                return None\n            else:\n                return ctypes.cast(<unsigned long long>self.chandle, ctypes.c_void_p)\n        def __set__(self, value):\n            self._set_handle(value)\n\n    def __init__(self, handle, is_global):\n        self._set_handle(handle)\n        self.c_is_global = is_global\n\n    def __dealloc__(self):\n        if self.is_global == 0:\n            CALL(MXNetFuncFree(self.chandle))\n\n    def __call__(self, *args):\n        cdef MXNetValue ret_val\n        cdef int ret_tcode\n        FuncCall(self.chandle, args, &ret_val, &ret_tcode)\n        if ret_tcode == kPyArg:\n            return args[ret_val.v_int64]\n        else:\n            return make_ret(ret_val, ret_tcode)\n\ncdef object make_packed_func(MXNetFunctionHandle chandle, int is_global):\n    obj = _CLASS_PACKED_FUNC.__new__(_CLASS_PACKED_FUNC)\n    (<FunctionBase>obj).chandle = chandle\n    (<FunctionBase>obj).is_global = is_global\n    return obj\n\ndef _get_global_func(name, allow_missing=False):\n    cdef MXNetFunctionHandle chandle\n    CALL(MXNetFuncGetGlobal(c_str(name), &chandle))\n    if chandle != NULL:\n        return make_packed_func(chandle, True)\n\n    if allow_missing:\n        return None\n\n    raise ValueError(\"Cannot find global function %s\" % name)\n\n_CLASS_OBJECT = None\n_CLASS_PACKED_FUNC = None\n_FUNC_CONVERT_TO_NODE = None\n\ndef _set_class_object(obj_class):\n    \"\"\"Initialize object class defined in cython\"\"\"\n    global _CLASS_OBJECT\n    _CLASS_OBJECT = obj_class\n\ndef _set_class_packed_func(func_class):\n    \"\"\"Initialize packed function defined in cython\"\"\"\n    global _CLASS_PACKED_FUNC\n    _CLASS_PACKED_FUNC = func_class\n\ndef _set_node_generic(func_convert_to_node):\n    \"\"\"Initialize packed function type conversion function in cython\"\"\"\n    global _FUNC_CONVERT_TO_NODE\n    _FUNC_CONVERT_TO_NODE = func_convert_to_node\n"
  },
  {
    "path": "python/mxnet/_ffi/_cython/ndarray.pxi",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Acknowledgement: This file originates from incubator-tvm\"\"\"\n\nimport ctypes\nfrom ... import _global_var\n\ncdef c_make_array(void* handle):\n    return _global_var._np_ndarray_cls(handle=<unsigned long long>handle)\n"
  },
  {
    "path": "python/mxnet/_ffi/_cython/object.pxi",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"\nMaps object type to its constructor\nAcknowledgement: This file originates from incubator-tvm\n\"\"\"\ncdef list OBJECT_TYPE = []\n\ndef _register_object(int index, object cls):\n    \"\"\"register object class\"\"\"\n    global OBJECT_TYPE\n    while len(OBJECT_TYPE) <= index:\n        OBJECT_TYPE.append(None)\n    OBJECT_TYPE[index] = cls\n\n\ncdef inline object make_ret_object(void* chandle):\n    global OBJECT_TYPE\n    global _CLASS_OBJECT\n    cdef unsigned tindex\n    cdef object cls\n    object_type = OBJECT_TYPE\n    CALL(MXNetObjectGetTypeIndex(chandle, &tindex))\n    if tindex < len(OBJECT_TYPE):\n        cls = OBJECT_TYPE[tindex]\n        if cls is not None:\n            if issubclass(cls, PyNativeObject):\n                obj = _CLASS_OBJECT.__new__(_CLASS_OBJECT)\n                (<ObjectBase>obj).chandle = chandle\n                return cls.__from_mxnet_object__(cls, obj)\n            obj = cls.__new__(cls)\n        else:\n            obj = _CLASS_OBJECT.__new__(_CLASS_OBJECT)\n    else:\n        obj = _CLASS_OBJECT.__new__(_CLASS_OBJECT)\n    (<ObjectBase>obj).chandle = chandle\n    return obj\n\nclass PyNativeObject:\n    \"\"\"Base class of all MXNet objects that also subclass python's builtin types.\"\"\"\n    __slots__ = []\n\n    def __init_mxnet_object_by_constructor__(self, fconstructor, *args):\n        \"\"\"Initialize the internal mxnet_object by calling constructor function.\n\n        Parameters\n        ----------\n        fconstructor : Function\n            Constructor function.\n\n        args: list of objects\n            The arguments to the constructor\n\n        Note\n        ----\n        We have a special calling convention to call constructor functions.\n        So the return object is directly set into the object\n        \"\"\"\n        obj = _CLASS_OBJECT.__new__(_CLASS_OBJECT)\n        obj.__init_handle_by_constructor__(fconstructor, *args)\n        self.__mxnet_object__ = obj\n\ncdef class ObjectBase:\n    cdef void* chandle\n\n    cdef inline _set_handle(self, handle):\n        cdef unsigned long long ptr\n        if handle is None:\n            self.chandle = NULL\n        else:\n            ptr = handle.value\n            self.chandle = <void*>(ptr)\n\n    property handle:\n        def __get__(self):\n            if self.chandle == NULL:\n                return None\n            else:\n                return ctypes_handle(self.chandle)\n\n        def __set__(self, value):\n            self._set_handle(value)\n\n    def __dealloc__(self):\n        CALL(MXNetObjectFree(self.chandle))\n\n    def __init_handle_by_constructor__(self, fconstructor, *args):\n        \"\"\"Initialize the handle by calling constructor function.\n\n        Parameters\n        ----------\n        fconstructor : Function\n            Constructor function.\n\n        args: list of objects\n            The arguments to the constructor\n\n        Note\n        ----\n        We have a special calling convention to call constructor functions.\n        So the return handle is directly set into the Node object\n        instead of creating a new Node.\n        \"\"\"\n        # avoid error raised during construction.\n        self.chandle = NULL\n        cdef void* chandle\n        ConstructorCall(\n            (<FunctionBase>fconstructor).chandle,\n            kObjectHandle, args, &chandle)\n        self.chandle = chandle\n\n    def same_as(self, other):\n        \"\"\"Check object identity.\n\n        Parameters\n        ----------\n        other : object\n            The other object to compare against.\n\n        Returns\n        -------\n        result : bool\n             The comparison result.\n        \"\"\"\n        if not isinstance(other, ObjectBase):\n            return False\n        return self.chandle == (<ObjectBase>other).chandle\n"
  },
  {
    "path": "python/mxnet/_ffi/base.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n# coding: utf-8\n# pylint: disable=invalid-name\n\"\"\"Base library for MXNet FFI.\nAcknowledgement: This file originates from incubator-tvm\n\"\"\"\nimport sys\nimport ctypes\nimport numpy as np\n\nstring_types = (str,)\ninteger_types = (int, np.int32)\nnumeric_types = integer_types + (float, np.float32)\n# this function is needed for python3\n# to convert ctypes.char_p .value back to python str\nif sys.platform == \"win32\":\n    encoding = 'cp' + str(ctypes.cdll.kernel32.GetACP())\n    py_str = lambda x: x.decode(encoding)\nelse:\n    py_str = lambda x: x.decode('utf-8')\n\n#----------------------------\n# helper function in ctypes.\n#----------------------------\ndef c_str(string):\n    \"\"\"Create ctypes char * from a python string\n    Parameters\n    ----------\n    string : string type\n        python string\n\n    Returns\n    -------\n    str : c_char_p\n        A char pointer that can be passed to C API\n    \"\"\"\n    return ctypes.c_char_p(string.encode('utf-8'))\n\n\ndef c_array(ctype, values):\n    \"\"\"Create ctypes array from a python array\n\n    Parameters\n    ----------\n    ctype : ctypes data type\n        data type of the array we want to convert to\n\n    values : tuple or list\n        data content\n\n    Returns\n    -------\n    out : ctypes array\n        Created ctypes array\n    \"\"\"\n    return (ctype * len(values))(*values)\n"
  },
  {
    "path": "python/mxnet/_ffi/function.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=invalid-name, unused-import\n\"\"\"\nFunction namespace.\nAcknowledgement: This file originates from incubator-tvm\n\"\"\"\nimport os\nimport sys\nimport ctypes\nfrom ..base import _LIB, check_call\nfrom .base import py_str, c_str\n\ntry:\n    if int(os.environ.get(\"MXNET_ENABLE_CYTHON\", True)) == 0:\n        from ._ctypes.function import FunctionBase as _FunctionBase\n        from ._ctypes.function import _set_class_packed_func, _get_global_func\n        # To set RETURN_SWITCH for OBJECT_HANDLE\n        from . import object\n    else:\n        from ._cy3.core import FunctionBase as _FunctionBase\n        from ._cy3.core import _set_class_packed_func, _get_global_func\nexcept ImportError:\n    if int(os.environ.get(\"MXNET_ENFORCE_CYTHON\", False)) != 0:\n        raise ImportError(\"Cython Module cannot be loaded but MXNET_ENFORCE_CYTHON=1\")\n    from ._ctypes.function import FunctionBase as _FunctionBase\n    from ._ctypes.function import _set_class_packed_func, _get_global_func\n    # To set RETURN_SWITCH for OBJECT_HANDLE\n    from . import object\n\n\nclass Function(_FunctionBase):\n    \"\"\"The PackedFunc object used in TVM.\n\n    Function plays an key role to bridge front and backend in TVM.\n    Function provide a type-erased interface, you can call function with positional arguments.\n\n    The compiled module returns Function.\n    TVM backend also registers and exposes its API as Functions.\n    For example, the developer function exposed in tvm.ir_pass are actually\n    C++ functions that are registered as PackedFunc\n\n    The following are list of common usage scenario of tvm.Function.\n\n    - Automatic exposure of C++ API into python\n    - To call PackedFunc from python side\n    - To call python callbacks to inspect results in generated code\n    - Bring python hook into C++ backend\n\n    See Also\n    --------\n    tvm.register_func: How to register global function.\n    tvm.get_global_func: How to get global function.\n    \"\"\"\n\n\ndef get_global_func(name, allow_missing=False):\n    \"\"\"Get a global function by name\n\n    Parameters\n    ----------\n    name : str\n        The name of the global function\n\n    allow_missing : bool\n        Whether allow missing function or raise an error.\n\n    Returns\n    -------\n    func : tvm.Function\n        The function to be returned, None if function is missing.\n    \"\"\"\n    return _get_global_func(name, allow_missing)\n\n\ndef list_global_func_names():\n    \"\"\"Get list of global functions registered.\n\n    Returns\n    -------\n    names : list\n       List of global functions names.\n    \"\"\"\n    plist = ctypes.POINTER(ctypes.c_char_p)()\n    size = ctypes.c_uint()\n\n    check_call(_LIB.MXNetFuncListGlobalNames(ctypes.byref(size),\n                                             ctypes.byref(plist)))\n    fnames = []\n    for i in range(size.value):\n        fnames.append(py_str(plist[i]))\n    return fnames\n\n\ndef _get_api(f):\n    flocal = f\n    flocal.is_global = True\n    return flocal\n\n\ndef _init_api(namespace, target_module_name=None):\n    \"\"\"Initialize api for a given module name\n\n    namespace : str\n       The namespace of the source registry\n\n    target_module_name : str\n       The target module name if different from namespace\n    \"\"\"\n    target_module_name = (\n        target_module_name if target_module_name else namespace)\n    if namespace.startswith(\"mxnet.\"):\n        _init_api_prefix(target_module_name, namespace[6:])\n    else:\n        _init_api_prefix(target_module_name, namespace)\n\n\ndef _init_api_prefix(module_name, prefix):\n    module = sys.modules[module_name]\n\n    for name in list_global_func_names():\n        if prefix == \"api\":\n            fname = name\n            if name.startswith(\"_\"):\n                target_module = sys.modules[\"mxnet._api_internal\"]\n            else:\n                target_module = module\n        else:\n            if not name.startswith(prefix):\n                continue\n            fname = name[len(prefix)+1:]\n            target_module = module\n\n        if fname.find(\".\") != -1:\n            continue\n        f = get_global_func(name)\n        ff = _get_api(f)\n        ff.__name__ = fname\n        ff.__doc__ = (f\"MXNet PackedFunc {fname}. \")\n        setattr(target_module, ff.__name__, ff)\n\n_set_class_packed_func(Function)\n"
  },
  {
    "path": "python/mxnet/_ffi/node_generic.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Common implementation of Node generic related logic\nAcknowledgement: This file originates from incubator-tvm\"\"\"\n# pylint: disable=unused-import\nfrom numbers import Number, Integral\nfrom .. import _api_internal\nfrom ..ndarray._internal import NDArrayBase\nfrom .object import _ObjectBase, PyNativeObject, _set_node_generic\nfrom .base import string_types\n\ndef _scalar_type_inference(value):\n    if hasattr(value, 'dtype'):\n        dtype = str(value.dtype)\n    elif isinstance(value, bool):\n        dtype = 'bool'\n    elif isinstance(value, float):\n        # We intentionally convert the float to float32 since it's more common in DL.\n        dtype = 'float32'\n    elif isinstance(value, int):\n        # We intentionally convert the python int to int32 since it's more common in DL.\n        dtype = 'int32'\n    else:\n        raise NotImplementedError('Cannot automatically inference the type.'\n                                  ' value={}'.format(value))\n    return dtype\n\n\ndef convert_to_node(value):\n    \"\"\"Convert a python value to corresponding node type.\n\n    Parameters\n    ----------\n    value : str\n        The value to be inspected.\n\n    Returns\n    -------\n    node : Node\n        The corresponding node value.\n    \"\"\"\n    if isinstance(value, (_ObjectBase, NDArrayBase, PyNativeObject)):\n        return value\n    elif isinstance(value, Integral):\n        return _api_internal._Integer(value)\n    elif isinstance(value, float):\n        return _api_internal._Float(value)\n    elif isinstance(value, string_types):\n        return _api_internal._String(value)\n    elif isinstance(value, (list, tuple)):\n        value = [convert_to_node(x) for x in value]\n        return _api_internal._ADT(*value)\n    elif isinstance(value, dict):\n        vlist = []\n        for item in value.items():\n            if (not isinstance(item[0], (_ObjectBase, NDArrayBase, PyNativeObject)) and\n                    not isinstance(item[0], string_types)):\n                raise ValueError(\"key of map must already been a container type\")\n            vlist.append(item[0])\n            vlist.append(convert_to_node(item[1]))\n        return _api_internal._Map(*vlist)\n    raise ValueError(f\"don't know how to convert type {type(value)} to node\")\n\n\ndef const(value, dtype=None):\n    \"\"\"Construct a constant value for a given type.\n\n    Parameters\n    ----------\n    value : int or float\n        The input value\n\n    dtype : str or None, optional\n        The data type.\n\n    Returns\n    -------\n    expr : Expr\n        Constant expression corresponds to the value.\n    \"\"\"\n    if dtype is None:\n        dtype = _scalar_type_inference(value)\n    return _api_internal._const(value, dtype)\n\n_set_node_generic(convert_to_node)\n"
  },
  {
    "path": "python/mxnet/_ffi/object.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n# pylint: disable=invalid-name\n\"\"\"Runtime Object API\nAcknowledgement: This file originates from incubator-tvm\"\"\"\nimport os\nimport ctypes\nfrom ..base import _LIB, check_call, c_str\n\ntry:\n    # pylint: disable=wrong-import-position,unused-import\n    if int(os.environ.get(\"MXNET_ENABLE_CYTHON\", True)) == 0:\n        from ._ctypes.function import _set_class_object, _set_node_generic\n        from ._ctypes.object import ObjectBase as _ObjectBase\n        from ._ctypes.object import _register_object, PyNativeObject\n    else:\n        from ._cy3.core import _set_class_object, _set_node_generic\n        from ._cy3.core import ObjectBase as _ObjectBase\n        from ._cy3.core import _register_object, PyNativeObject\nexcept ImportError:\n    if int(os.environ.get(\"MXNET_ENFORCE_CYTHON\", False)) != 0:\n        raise ImportError(\"Cython Module cannot be loaded but MXNET_ENFORCE_CYTHON=1\")\n    from ._ctypes.function import _set_class_object, _set_node_generic\n    from ._ctypes.object import ObjectBase as _ObjectBase\n    from ._ctypes.object import _register_object, PyNativeObject\n\n\ndef _new_object(cls):\n    \"\"\"Helper function for pickle\"\"\"\n    return cls.__new__(cls)\n\n\nclass Object(_ObjectBase):\n    \"\"\"Base class for all mxnet's runtime objects.\"\"\"\n\n\ndef register_object(type_key=None):\n    \"\"\"register object type.\n\n    Parameters\n    ----------\n    type_key : str or cls\n        The type key of the node\n\n    Examples\n    --------\n    The following code registers MyObject\n    using type key \"test.MyObject\"\n\n    .. code-block:: python\n\n      @register_object(\"test.MyObject\")\n      class MyObject(Object):\n          pass\n    \"\"\"\n    object_name = type_key if isinstance(type_key, str) else type_key.__name__\n\n    def register(cls):\n        \"\"\"internal register function\"\"\"\n        if hasattr(cls, \"_type_index\"):\n            tindex = cls._type_index\n        else:\n            tidx = ctypes.c_uint()\n            check_call(_LIB.MXNetObjectTypeKey2Index(\n                c_str(object_name), ctypes.byref(tidx)))\n            tindex = tidx.value\n        _register_object(tindex, cls)\n        return cls\n\n    if isinstance(type_key, str):\n        return register\n\n    return register(type_key)\n\n\ndef getitem_helper(obj, elem_getter, length, idx):\n    \"\"\"Helper function to implement a pythonic getitem function.\n\n    Parameters\n    ----------\n    obj: object\n        The original object\n\n    elem_getter : function\n        A simple function that takes index and return a single element.\n\n    length : int\n        The size of the array\n\n    idx : int or slice\n        The argument passed to getitem\n\n    Returns\n    -------\n    result : object\n        The result of getitem\n    \"\"\"\n    if isinstance(idx, slice):\n        start = idx.start if idx.start is not None else 0\n        stop = idx.stop if idx.stop is not None else length\n        step = idx.step if idx.step is not None else 1\n        if start < 0:\n            start += length\n        if stop < 0:\n            stop += length\n        return [elem_getter(obj, i) for i in range(start, stop, step)]\n\n    if idx < -length or idx >= length:\n        raise IndexError(\"Index out of range. size: {}, got index {}\"\n                         .format(length, idx))\n    if idx < 0:\n        idx += length\n    return elem_getter(obj, idx)\n\n\n_set_class_object(Object)\n"
  },
  {
    "path": "python/mxnet/_ffi/runtime_ctypes.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Common runtime ctypes.\nAcknowledgement: This file originates from incubator-tvm\n\"\"\"\n# pylint: disable=invalid-name\nimport ctypes\n\n\nclass TVMByteArray(ctypes.Structure):\n    \"\"\"Temp data structure for byte array.\"\"\"\n    _fields_ = [(\"data\", ctypes.POINTER(ctypes.c_byte)),\n                (\"size\", ctypes.c_size_t)]\n"
  },
  {
    "path": "python/mxnet/_global_var.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"global variables for ffi\"\"\"\n\n_ndarray_cls = None\n_np_ndarray_cls = None\n\n\ndef _set_ndarray_class(cls):\n    global _ndarray_cls\n    _ndarray_cls = cls\n\n\ndef _set_np_ndarray_class(cls):\n    global _np_ndarray_cls\n    _np_ndarray_cls = cls\n"
  },
  {
    "path": "python/mxnet/_numpy_op_doc.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: skip-file\n\n\"\"\"Doc placeholder for numpy ops with prefix _np.\"\"\"\n\n\ndef _np_sometrue(a, axis=None, keepdims=False, out=None):\n    \"\"\"\n    Check whether some values are true.\n\n    Refer to `any` for full documentation.\n\n    See Also\n    --------\n    any : equivalent function; see for details.\n    \"\"\"\n    pass\n\n\ndef _npx_nonzero(a):\n    \"\"\"\n    Return the indices of the elements that are non-zero.\n\n    Returns a ndarray with ndim is 2. Each row contains the indices\n    of the non-zero elements. The values in `a` are always tested and returned in\n    row-major, C-style order.\n\n    The result of this is always a 2-D array, with a row for\n    each non-zero element.\n\n    Parameters\n    ----------\n    a : array_like\n        Input array.\n\n    Returns\n    -------\n    array : ndarray\n        Indices of elements that are non-zero.\n\n    Notes\n    -----\n    This function differs from the original numpy.nonzero in the following aspects:\n        - Does not support python numeric.\n        - The return value is same as numpy.transpose(numpy.nonzero(a)).\n\n    Examples\n    --------\n    >>> x = np.array([[3, 0, 0], [0, 4, 0], [5, 6, 0]])\n    >>> x\n    array([[3, 0, 0],\n           [0, 4, 0],\n           [5, 6, 0]])\n    >>> npx.nonzero(x)\n    array([[0, 0],\n           [1, 1],\n           [2, 0],\n           [2, 1]], dtype=int64)\n\n    >>> np.transpose(npx.nonzero(x))\n    array([[0, 1, 2, 2],\n           [0, 1, 0, 1]], dtype=int64)\n    \"\"\"\n    pass\n\n\ndef _np_repeat(a, repeats, axis=None):\n    \"\"\"\n    Repeat elements of an array.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array.\n    repeats : int\n        The number of repetitions for each element.\n    axis : int, optional\n        The axis along which to repeat values.  By default, use the\n        flattened input array, and return a flat output array.\n\n    Returns\n    -------\n    repeated_array : ndarray\n        Output array which has the same shape as `a`, except along\n        the given axis.\n\n    Notes\n    -----\n    Unlike the official NumPy ``repeat`` operator, this operator currently\n    does not support array of ints for the parameter `repeats`.\n\n    Examples\n    --------\n    >>> x = np.arange(4).reshape(2, 2)\n    >>> x\n    array([[0., 1.],\n           [2., 3.]])\n    >>> np.repeat(x, repeats=3)\n    array([0., 0., 0., 1., 1., 1., 2., 2., 2., 3., 3., 3.])\n    >>> np.repeat(x, repeats=3, axis=0)\n    array([[0., 1.],\n           [0., 1.],\n           [0., 1.],\n           [2., 3.],\n           [2., 3.],\n           [2., 3.]])\n    >>> np.repeat(x, repeats=3, axis=1)\n    array([[0., 0., 0., 1., 1., 1.],\n           [2., 2., 2., 3., 3., 3.]])\n    \"\"\"\n    pass\n\n\ndef _np_dot(a, b, out=None):\n    \"\"\"\n    Dot product of two arrays. Specifically,\n\n    - If both `a` and `b` are 1-D arrays, it is inner product of vectors\n\n    - If both `a` and `b` are 2-D arrays, it is matrix multiplication,\n\n    - If either `a` or `b` is 0-D (scalar), it is equivalent to :func:`multiply`\n      and using ``np.multiply(a, b)`` or ``a * b`` is preferred.\n\n    - If `a` is an N-D array and `b` is a 1-D array, it is a sum product over\n      the last axis of `a` and `b`.\n\n    - If `a` is an N-D array and `b` is a 2-D array, it is a\n      sum product over the last axis of `a` and the second-to-last axis of `b`::\n\n        dot(a, b)[i,j,k] = sum(a[i,j,:] * b[:,k])\n\n    Parameters\n    ----------\n    a : ndarray\n        First argument.\n    b : ndarray\n        Second argument.\n\n    out : ndarray, optional\n        Output argument. It must have the same shape and type as the expected output.\n\n    Returns\n    -------\n    output : ndarray\n        Returns the dot product of `a` and `b`.  If `a` and `b` are both\n        scalars or both 1-D arrays then a scalar is returned; otherwise\n        an array is returned.\n        If `out` is given, then it is returned\n\n    Examples\n    --------\n    >>> a = np.array(3)\n    >>> b = np.array(4)\n    >>> np.dot(a, b)\n    array(12.)\n\n    For 2-D arrays it is the matrix product:\n\n    >>> a = np.array([[1, 0], [0, 1]])\n    >>> b = np.array([[4, 1], [2, 2]])\n    >>> np.dot(a, b)\n    array([[4., 1.],\n           [2., 2.]])\n\n    >>> a = np.arange(3*4*5*6).reshape((3,4,5,6))\n    >>> b = np.arange(5*6)[::-1].reshape((6,5))\n    >>> np.dot(a, b)[2,3,2,2]\n    array(29884.)\n    >>> np.sum(a[2,3,2,:] * b[:,2])\n    array(29884.)\n    \"\"\"\n    pass\n\n\ndef _np_copy(a, out=None):\n    \"\"\"\n    Return an array copy of the given object.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data.\n    out : ndarray or None, optional\n        Alternative output array in which to place the result. It must have\n        the same shape and dtype as the expected output.\n\n    Returns\n    -------\n    arr : ndarray\n        Array interpretation of `a`.\n\n    Notes\n    -------\n    This function differs from the original `numpy.copy\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.copy.html>`_ in\n    the following aspects:\n\n    - Input type does not support Python native iterables(list, tuple, ...).\n    - ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be the same as the expected output.\n    - ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the same as the expected output.\n    - Does not support \"order\" parameter.\n\n    Examples\n    --------\n    Create an array x, with a reference y and a copy z:\n\n    >>> x = np.array([1, 2, 3])\n    >>> y = x\n    >>> z = np.copy(x)\n\n    Note that, when ``x`` is modified, ``y`` is also modified, but not ``z``:\n\n    >>> x[0] = 10\n    >>> x[0] == y[0]\n    array([1.])\n    >>> x[0] == z[0]\n    array([0.])\n    \"\"\"\n    pass\n\n\ndef _np_reshape(a, newshape, order='C', out=None):\n    \"\"\"\n    Gives a new shape to an array without changing its data.\n    This function always returns a copy of the input array if\n    ``out`` is not provided.\n\n    Parameters\n    ----------\n    a : ndarray\n        Array to be reshaped.\n    newshape : int or tuple of ints\n        The new shape should be compatible with the original shape. If\n        an integer, then the result will be a 1-D array of that length.\n        One shape dimension can be -1. In this case, the value is\n        inferred from the length of the array and remaining dimensions.\n    order : {'C'}, optional\n        Read the elements of `a` using this index order, and place the\n        elements into the reshaped array using this index order.  'C'\n        means to read / write the elements using C-like index order,\n        with the last axis index changing fastest, back to the first\n        axis index changing slowest. Other order types such as 'F'/'A'\n        may be added in the future.\n\n    Returns\n    -------\n    reshaped_array : ndarray\n        It will be always a copy of the original array. This behavior is different\n        from the official NumPy ``reshape`` operator where views of the original array may be\n        generated.\n\n    See Also\n    --------\n    ndarray.reshape : Equivalent method.\n\n    Examples\n    --------\n    >>> a = np.arange(6).reshape((3, 2))\n    >>> a\n    array([[0., 1.],\n           [2., 3.],\n           [4., 5.]])\n\n    >>> np.reshape(a, (2, 3)) # C-like index ordering\n    array([[0., 1., 2.],\n           [3., 4., 5.]])\n\n    >>> np.reshape(np.ravel(a), (2, 3)) # equivalent to C ravel then C reshape\n    array([[0., 1., 2.],\n           [3., 4., 5.]])\n\n    >>> a = np.array([[1,2,3], [4,5,6]])\n    >>> np.reshape(a, 6)\n    array([1., 2., 3., 4., 5., 6.])\n\n    >>> np.reshape(a, (3,-1))       # the unspecified value is inferred to be 2\n    array([[1., 2.],\n           [3., 4.],\n           [5., 6.]])\n    \"\"\"\n\n\ndef _np_squeeze(a, axis=None, out=None):\n    \"\"\"\n    Remove single-dimensional entries from the shape of an array.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data.\n    axis : None or int or tuple of ints, optional\n        Selects a subset of the single-dimensional entries in the\n        shape. If an axis is selected with shape entry greater than\n        one, an error is raised.\n    out : ndarray, optional\n        Array into which the output is placed. It must have the same size\n        and dtype as the input array.\n\n    Returns\n    -------\n    squeezed : ndarray\n        The input array, but with all or a subset of the\n        dimensions of length 1 removed. It always returns a copy of `a`.\n\n    Raises\n    ------\n    MXNetError\n        If `axis` is not `None`, and an axis being squeezed is not of length 1\n\n    See Also\n    --------\n    expand_dims : The inverse operation, adding singleton dimensions\n    reshape : Insert, remove, and combine dimensions, and resize existing ones\n\n    Examples\n    --------\n    >>> x = np.array([[[0], [1], [2]]])\n    >>> x.shape\n    (1, 3, 1)\n    >>> np.squeeze(x).shape\n    (3,)\n    >>> np.squeeze(x, axis=0).shape\n    (3, 1)\n    >>> np.squeeze(x, axis=1).shape\n    Traceback (most recent call last):\n    ...\n    mxnet.base.MXNetError: cannot select an axis to squeeze out which has size=3 not equal to one\n    >>> np.squeeze(x, axis=2).shape\n    (1, 3)\n    \"\"\"\n    pass\n\n\ndef _np_prod(a, axis=None, dtype=None, out=None, keepdims=False):\n    \"\"\"\n    Return the product of array elements over a given axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which a product is performed.\n        The default (`axis` = `None`) is perform a product over all\n        the dimensions of the input array. `axis` may be negative, in\n        which case it counts from the last to the first axis.\n        If this is a tuple of ints, a product is performed on multiple\n        axes, instead of a single axis or all the axes as before.\n    dtype : data-type, optional\n        The data-type of the returned array, as well as of the accumulator\n        in which the elements are multiplied.  By default, if `a` is of\n        integer type, `dtype` is the default platform integer. (Note: if\n        the type of `a` is unsigned, then so is `dtype`.)  Otherwise,\n        the dtype is the same as that of `a`.\n    out : ndarray, optional\n        Alternative output array in which to place the result. It must have\n        the same shape as the expected output, but the type of the\n        output values will be cast if necessary.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the original `arr`.\n\n    Returns\n    -------\n    product_along_axis : ndarray, see `dtype` parameter above.\n        An array shaped as `a` but with the specified axis removed.\n        Returns a reference to `out` if specified.\n\n    See Also\n    --------\n    ndarray.prod : equivalent method\n\n    Notes\n    -----\n    Arithmetic is modular when using integer types, and no error is\n    raised on overflow.  That means that, on a 32-bit platform:\n\n    >>> x = np.array([536870910, 536870910, 536870910, 536870910])\n    >>> np.prod(x) #random\n    array(8.307675e+34)\n\n    Examples\n    --------\n    By default, calculate the product of all elements:\n\n    >>> np.prod(np.array([1.,2.]))\n    array(2.)\n\n    Even when the input array is two-dimensional:\n\n    >>> np.prod(np.array([1.,2.,3.,4.]).reshape((2,2)))\n    array(24.)\n\n    But we can also specify the axis over which to multiply:\n\n    >>> np.prod(np.array([1.,2.,3.,4.]).reshape((2,2)), axis=1)\n    array([  2.,  12.])\n\n    If the type of `x` is unsigned, then the output type is\n    the unsigned platform integer:\n\n    >>> x = np.array([1, 2, 3], dtype=np.uint8)\n    >>> np.prod(x).dtype == np.uint8\n    True\n\n    If `x` is of a signed integer type, then the output type\n    is the default platform integer:\n\n    >>> x = np.array([1, 2, 3], dtype=np.int8)\n    >>> np.prod(x).dtype == np.int8\n    True\n    \"\"\"\n    pass\n\n\ndef _np_product(a, axis=None, dtype=None, out=None, keepdims=False):\n    \"\"\"\n    Return the product of array elements over a given axis.\n\n    See Also\n    --------\n    prod : equivalent function; see for details.\n    \"\"\"\n    pass\n\n\ndef _np_moveaxis(a, source, destination):\n    \"\"\"Move axes of an array to new positions.\n    Other axes remain in their original order.\n\n    Parameters\n    ----------\n    a : ndarray\n        The array whose axes should be reordered.\n        source : int or sequence of int\n        Original positions of the axes to move. These must be unique.\n        destination : int or sequence of int\n        Destination positions for each of the original axes. These must also be\n        unique.\n\n    Returns\n    -------\n    result : ndarray\n        Array with moved axes. This array is a view of the input array.\n\n    See Also\n    --------\n        transpose: Permute the dimensions of an array.\n        swapaxes: Interchange two axes of an array.\n\n    Examples\n    --------\n    >>> x = np.zeros((3, 4, 5))\n    >>> np.moveaxis(x, 0, -1).shape\n    (4, 5, 3)\n    >>> np.moveaxis(x, -1, 0).shape\n    (5, 3, 4)\n    These all achieve the same result:\n    >>> np.transpose(x).shape\n    (5, 4, 3)\n    >>> np.swapaxes(x, 0, -1).shape\n    (5, 4, 3)\n    >>> np.moveaxis(x, [0, 1], [-1, -2]).shape\n    (5, 4, 3)\n    >>> np.moveaxis(x, [0, 1, 2], [-1, -2, -3]).shape\n    (5, 4, 3)\n    \"\"\"\n    pass\n\ndef _np__random_shuffle(x):\n    \"\"\"\n    Modify a sequence in-place by shuffling its contents.\n\n    This function only shuffles the array along the first axis of a\n    multi-dimensional array. The order of sub-arrays is changed but\n    their contents remain the same.\n\n    Parameters\n    ----------\n    x: ndarray\n        The array or list to be shuffled.\n\n    Returns\n    -------\n    None\n\n    Examples\n    --------\n    >>> arr = np.arange(10)\n    >>> np.random.shuffle(arr)\n    >>> arr\n    array([5., 1., 0., 6., 7., 3., 9., 8., 4., 2.])  # random\n\n    Multi-dimensional arrays are only shuffled along the first axis:\n\n    >>> arr = np.arange(9).reshape((3, 3))\n    >>> np.random.shuffle(arr)\n    >>> arr\n    array([[6., 7., 8.], # random\n           [3., 4., 5.],\n           [0., 1., 2.]])\n    \"\"\"\n    pass\n\n\ndef _npx_constraint_check(x, msg):\n    \"\"\"\n    This operator will check if all the elements in a boolean tensor is true.\n    If not, ValueError exception will be raised in the backend with given error message.\n    In order to evaluate this operator, one should multiply the origin tensor by the return value\n    of this operator to force this operator become part of the computation graph,\n    otherwise the check would not be working under symoblic mode.\n\n    Parameters\n    ----------\n    x : ndarray\n        A boolean tensor.\n    msg : string\n        The error message in the exception.\n\n    Returns\n    -------\n    out : ndarray\n        If all the elements in the input tensor are true,\n        array(True) will be returned, otherwise ValueError exception would\n        be raised before anything got returned.\n\n    Examples\n    --------\n    >>> loc = np.zeros((2,2))\n    >>> scale = np.array(#some_value)\n    >>> constraint = (scale > 0)\n    >>> np.random.normal(loc,\n                     scale * npx.constraint_check(constraint, 'Scale should be larger than zero'))\n\n    If elements in the scale tensor are all bigger than zero, npx.constraint_check would return\n    `np.array(True)`, which will not change the value of `scale` when multiplied by.\n    If some of the elements in the scale tensor violate the constraint,\n    i.e. there exists `False` in the boolean tensor `constraint`,\n    a `ValueError` exception with given message 'Scale should be larger than zero' would be raised.\n    \"\"\"\n    pass\n\n\ndef _npx_reshape(a, newshape, reverse=False, order='C'):\n    \"\"\"\n    Gives a new shape to an array without changing its data.\n    This function always returns a copy of the input array if\n    ``out`` is not provided.\n\n    Parameters\n    ----------\n    a : ndarray\n        Array to be reshaped.\n    newshape : int or tuple of ints\n        The new shape should be compatible with the original shape.\n        If an integer, then the result will be a 1-D array of that length.\n        One shape dimension can be -1. In this case, the value is inferred\n        from the length of the array and remaining dimensions.\n        -2 to -6 are used for data manipulation.\n\n        - -2 copy this dimension from the input to the output shape.\n        - -3 will skip current dimension if and only if the current dim size is one.\n        - -4 copy all remain of the input dimensions to the output shape.\n        - -5 use the product of two consecutive dimensions of the input\n          shape as the output.\n        - -6 split one dimension of the input into two dimensions passed\n          subsequent to -6 in the new shape.\n\n    reverse : bool, optional\n        If set to true, the special values will be inferred from right to left.\n    order : {'C'}, optional\n        Read the elements of `a` using this index order, and place the\n        elements into the reshaped array using this index order.  'C'\n        means to read / write the elements using C-like index order,\n        with the last axis index changing fastest, back to the first\n        axis index changing slowest. Other order types such as 'F'/'A'\n        may be added in the future.\n\n    Returns\n    -------\n    reshaped_array : ndarray\n        It will be always a copy of the original array. This behavior is different\n        from the official NumPy ``reshape`` operator where views of the original array may be\n        generated.\n\n    Examples\n    --------\n    >>> x = np.ones((2, 3, 8))\n    >>> npx.reshape(x, (-2, -2, 2, -1)).shape\n    (2, 3, 2, 4)\n    >>> x = np.ones((8, 3, 3, 3, 4, 4))\n    >>> npx.reshape(x, (-6, 2, -1, -4)).shape\n    (2, 4, 3, 3, 3, 4, 4)\n    >>> x = np.ones((8, 3, 3, 3, 4, 4))\n    >>> npx.reshape(x, (-5, -4)).shape\n    (24, 3, 3, 4, 4)\n    >>> x = np.ones((8, 1, 1, 1, 3))\n    >>> npx.reshape(x, (-2, -3, -3, -3, -2)).shape\n    (8, 3)\n    >>> x = np.ones((8, 3, 3, 3, 3, 8))\n    >>> npx.reshape(x, (-4, -5), reverse=True).shape\n    (8, 3, 3, 3, 24)\n    >>> x = np.ones((8, 3, 2, 4, 8))\n    >>> npx.reshape(x, (-4, -1, 2, -6), reverse=True).shape\n    (8, 3, 2, 4, 4, 2)\n    \"\"\"\n    pass\n\n\ndef _npx_index_add(a, ind, val):\n    \"\"\"\n    Add values to input according to given indexes.\n    If exists repeate positions to be updated, the update value will be accumulated.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data. The array to be updated.\n    ind : ndarray\n        Indexes for indicating update positions.\n        For example, array([[0, 1], [2, 3], [4, 5]] indicates here are two positions to\n        be updated, which is (0, 2, 4) and (1, 3, 5).\n        Note: - 'ind' cannot be empty array '[]', for that case, please use operator 'add' instead.\n              - 0 <= ind.ndim <= 2.\n              - ind.dtype should be 'int32' or 'int64'\n    val : ndarray\n        Input data. The array to update the input 'a'.\n\n    Returns\n    -------\n    out : ndarray\n        The output array.\n\n    Examples\n    --------\n    >>> a = np.zeros((2, 3, 4))\n    >>> ind = np.array([[0, 0], [0, 0], [0, 1]], dtype='int32')\n    >>> val = np.arange(2).reshape(2) + 1\n    >>> b = npx.index_add(a, ind, val)\n    >>> b\n    array([[[1., 2., 0., 0.],\n            [0., 0., 0., 0.],\n            [0., 0., 0., 0.]],\n\n           [[0., 0., 0., 0.],\n            [0., 0., 0., 0.],\n            [0., 0., 0., 0.]]])\n    \n    >>> ind = np.array([[0, 0], [0, 0], [0, 0]], dtype='int32')  # accumulate values in repeated positions\n    >>> b = npx.index_add(a, ind, val)\n    >>> b\n    array([[[3., 0., 0., 0.],\n            [0., 0., 0., 0.],\n            [0., 0., 0., 0.]],\n\n           [[0., 0., 0., 0.],\n            [0., 0., 0., 0.],\n            [0., 0., 0., 0.]]])\n    \n    >>> ind=np.array([[0, 0], [0, 1]], dtype='int32') \n    >>> val = np.arange(8).reshape(2, 4) \n    >>> b = npx.index_add(a, ind, val)\n    >>> b\n    array([[[0., 1., 2., 3.],\n            [4., 5., 6., 7.],\n            [0., 0., 0., 0.]],\n\n           [[0., 0., 0., 0.],\n            [0., 0., 0., 0.],\n            [0., 0., 0., 0.]]])\n    \n    >>> val = np.arange(4).reshape(4)  # brocast 'val'\n    >>> b = npx.index_add(a, ind, val)\n    >>> b\n    array([[[0., 1., 2., 3.],\n            [0., 1., 2., 3.],\n            [0., 0., 0., 0.]],\n\n        [[0., 0., 0., 0.],\n            [0., 0., 0., 0.],\n            [0., 0., 0., 0.]]])\n    \"\"\"\n    pass\n\n\ndef _npx_index_update(a, ind, val):\n    \"\"\"\n    Update values to input according to given indexes.\n    If multiple indices refer to the same location it is undefined which update is chosen; it may choose\n    the order of updates arbitrarily and nondeterministically (e.g., due to concurrent updates on some\n    hardware platforms). Recommend not to use repeate positions.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data. The array to be updated.\n        Support dtype: 'float32', 'float64', 'int32', 'int64'.\n    ind : ndarray\n        Indexes for indicating update positions.\n        For example, array([[0, 1], [2, 3], [4, 5]] indicates here are two positions to\n        be updated, which is (0, 2, 4) and (1, 3, 5).\n        Note: - 'ind' cannot be empty array '[]', for that case, please use operator 'add' instead.\n              - 0 <= ind.ndim <= 2.\n              - ind.dtype should be 'int32' or 'int64'\n    val : ndarray\n        Input data. The array to update the input 'a'.\n        Support dtype: 'float32', 'float64', 'int32', 'int64'.\n\n    Returns\n    -------\n    out : ndarray\n        The output array.\n\n    Examples\n    --------\n    >>> a = np.zeros((2, 3, 4))\n    >>> ind = np.array([[0, 0], [0, 0], [0, 1]], dtype='int32')\n    >>> val = np.arange(2).reshape(2) + 1\n    >>> b = npx.index_update(a, ind, val)\n    >>> b\n    array([[[1., 2., 0., 0.],\n            [0., 0., 0., 0.],\n            [0., 0., 0., 0.]],\n\n           [[0., 0., 0., 0.],\n            [0., 0., 0., 0.],\n            [0., 0., 0., 0.]]])\n\n    >>> ind=np.array([[0, 0], [0, 1]], dtype='int32') \n    >>> val = np.arange(8).reshape(2, 4) \n    >>> b = npx.index_update(a, ind, val)\n    >>> b\n    array([[[0., 1., 2., 3.],\n            [4., 5., 6., 7.],\n            [0., 0., 0., 0.]],\n\n           [[0., 0., 0., 0.],\n            [0., 0., 0., 0.],\n            [0., 0., 0., 0.]]])\n    \n    >>> val = np.arange(4).reshape(4)  # brocast 'val'\n    >>> b = npx.index_update(a, ind, val)\n    >>> b\n    array([[[0., 1., 2., 3.],\n            [0., 1., 2., 3.],\n            [0., 0., 0., 0.]],\n\n        [[0., 0., 0., 0.],\n            [0., 0., 0., 0.],\n            [0., 0., 0., 0.]]])\n    \"\"\"\n    pass\n\n\ndef _np_diag(array, k=0):\n    \"\"\"\n    Extracts a diagonal or constructs a diagonal array.\n    - 1-D arrays: constructs a 2-D array with the input as its diagonal, all other elements are zero.\n    - 2-D arrays: extracts the k-th Diagonal\n\n    Parameters\n    ----------\n    array : ndarray\n        The array to apply diag method.\n    k : offset\n        extracts or constructs kth diagonal given input array\n\n    Examples\n    --------\n    >>> x = np.arange(9).reshape((3,3))\n    >>> x\n    array([[0, 1, 2],\n           [3, 4, 5],\n           [6, 7, 8]])\n    >>> np.diag(x)\n    array([0, 4, 8])\n    >>> np.diag(x, k=1)\n    array([1, 5])\n    >>> np.diag(x, k=-1)\n    array([3, 7])\n\n    >>> np.diag(np.diag(x))\n    array([[0, 0, 0],\n           [0, 4, 0],\n           [0, 0, 8]])\n    \"\"\"\n    pass\n\n\ndef _np_diagonal(a, offset=0, axis1=0, axis2=1):\n    \"\"\"\n    If a is 2-D, returns the diagonal of a with the given offset, i.e., the collection of elements of\n    the form a[i, i+offset]. If a has more than two dimensions, then the axes specified by axis1 and\n    axis2 are used to determine the 2-D sub-array whose diagonal is returned. The shape of the\n    resulting array can be determined by removing axis1 and axis2 and appending an index to the\n    right equal to the size of the resulting diagonals.\n\n    Parameters\n    ----------\n    a : Symbol\n        Input data from which diagonal are taken.\n    offset: int, Optional\n        Offset of the diagonal from the main diagonal\n    axis1: int, Optional\n        Axis to be used as the first axis of the 2-D sub-arrays\n    axis2: int, Optional\n        Axis to be used as the second axis of the 2-D sub-arrays\n\n    Returns\n    -------\n    out : Symbol\n        Output result\n\n    Raises\n    -------\n    ValueError:  If the dimension of a is less than 2.\n\n    Examples\n    --------\n    >>> a = np.arange(4).reshape(2,2)\n    >>> a\n    array([[0, 1],\n        [2, 3]])\n    >>> np.diagonal(a)\n    array([0, 3])\n    >>> np.diagonal(a, 1)\n    array([1])\n\n    >>> a = np.arange(8).reshape(2,2,2)\n    >>>a\n    array([[[0, 1],\n            [2, 3]],\n            [[4, 5],\n            [6, 7]]])\n    >>> np.diagonal(a, 0, 0, 1)\n    array([[0, 6],\n            [1, 7]])\n    \"\"\"\n    pass\n\n\ndef _np_diagflat(array, k=0):\n    \"\"\"\n    Create a two-dimensional array with the flattened input as a diagonal.\n    Parameters\n    ----------\n    arr : ndarray\n        Input data, which is flattened and set as the `k`-th\n        diagonal of the output.\n    k : int, optional\n        Diagonal to set; 0, the default, corresponds to the \"main\" diagonal,\n        a positive (negative) `k` giving the number of the diagonal above\n        (below) the main.\n    Returns\n    -------\n    out : ndarray\n        The 2-D output array.\n    See Also\n    --------\n    diag : MATLAB work-alike for 1-D and 2-D arrays.\n    diagonal : Return specified diagonals.\n    trace : Sum along diagonals.\n    Examples\n    --------\n    >>> np.diagflat([[1,2], [3,4]])\n    array([[1, 0, 0, 0],\n           [0, 2, 0, 0],\n           [0, 0, 3, 0],\n           [0, 0, 0, 4]])\n    >>> np.diagflat([1,2], 1)\n    array([[0, 1, 0],\n           [0, 0, 2],\n           [0, 0, 0]])\n    \"\"\"\n    pass\n"
  },
  {
    "path": "python/mxnet/amp/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Automatic mixed precision module.\"\"\"\n\nfrom .amp import *\n"
  },
  {
    "path": "python/mxnet/amp/amp.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Functions for enabling AMP (automatic mixed precision).\"\"\"\n__all__ = ['init', 'init_trainer', 'scale_loss', 'unscale', 'convert_model',\n           'convert_hybrid_block', 'list_lp16_ops', 'list_fp32_ops',\n           'list_lp16_fp32_ops', 'list_conditional_fp32_ops',\n           'list_widest_type_cast', 'list_loss_output_functions', 'list_lp16_use_fp32_params',\n           'convert_symbol']\n\nfrom array import array\nimport ctypes\nimport inspect\nimport logging\nimport contextlib\nimport sys\nimport numpy as np\n\nfrom mxnet import numpy\nfrom .. import symbol\nfrom ..device import gpu\nfrom ..symbol import Symbol\nfrom ..symbol import contrib as symbol_contrib\nfrom .. import ndarray\nfrom ..ndarray import NDArray, dtype_np_to_mx, get_dtype_type, get_dtype_name, bfloat16\nfrom . import lists\nfrom ..gluon import Block, HybridBlock, trainer\nfrom .. import base\nfrom ..base import (_NP_OP_PREFIX, _NP_OP_SUBMODULE_LIST, _NP_EXT_OP_PREFIX,\n                    _NP_EXT_OP_SUBMODULE_LIST, _NP_INTERNAL_OP_PREFIX,\n                    c_str_array, c_str, c_array_buf, SymbolHandle, check_call, _LIB)\nfrom .. import optimizer as opt\nfrom .loss_scaler import LossScaler\nfrom ..operator import get_all_registered_operators_grouped\nfrom ..util import wrap_ctx_to_device_func\n\nOFFLINE_CAST_DTYPE_ATTR = '__amp_dtype__'\n\nfloat_types_gpu = (np.float16, np.float32)\nfloat_types_cpu = (bfloat16, np.float32)\n\ndef _cast_symbol_NDArray(s, dtype, is_numpy_module=False):\n    if isinstance(s, Symbol):\n        amp_cast = symbol.numpy._internal.amp_cast if is_numpy_module else symbol.amp_cast\n        return amp_cast(s, dtype=dtype)\n    if isinstance(s, NDArray):\n        amp_cast = ndarray.numpy._internal.amp_cast if is_numpy_module else ndarray.amp_cast\n        if s.dtype != dtype and (s.dtype in float_types_gpu and s.context.device_type != 'cpu' or\n                                 s.dtype in float_types_cpu and s.context.device_type == 'cpu'):\n            return amp_cast(s, dtype=dtype)\n    return s\n\ndef _get_nd_fun_to_wrap(name, module, submodule_dict):\n    module_internal = getattr(module, \"_internal\")\n    prefix = base._get_op_name_prefix(name)\n    if prefix:\n        if prefix != '_random_' or name.endswith('_like'):\n            func_name = name[len(prefix):]\n            cur_module = submodule_dict[prefix]\n        else:\n            func_name = name\n            cur_module = module_internal\n    elif name.startswith('_'):\n        func_name = name\n        cur_module = module_internal\n    else:\n        func_name = name\n        cur_module = module\n    return func_name, [cur_module]\n\ndef _get_np_fun_to_wrap(name, ns_prefix):\n    for pre, mod, subs in ((_NP_OP_PREFIX, 'numpy', _NP_OP_SUBMODULE_LIST),\n                           (_NP_EXT_OP_PREFIX, 'numpy_extension', _NP_EXT_OP_SUBMODULE_LIST),\n                           (_NP_INTERNAL_OP_PREFIX, 'numpy._internal', [])):\n        if name.startswith(pre):\n            nm = name[len(pre):]\n            for sub in subs:\n                if nm.startswith(sub):\n                    func, modules = nm[len(sub):], [sys.modules[f'{ns_prefix}.{mod}.{sub[1:-1]}']]\n                    break\n            else:\n                func, modules = nm, [sys.modules[f'{ns_prefix}.{mod}']]\n                break\n    else:\n        assert False, f'Unable to find target module for {name} in {ns_prefix}'\n    if name.startswith(_NP_INTERNAL_OP_PREFIX) and ns_prefix == 'mxnet.ndarray':\n        if hasattr(ndarray.numpy._api_internal, func):\n            modules.append(ndarray.numpy._api_internal)\n    return func, modules\n\ndef _wrap_module_functions(module, is_numpy_module, target_dtype, get_aliases, get_cond_aliases,\n                           get_fun_to_wrap, target_precision_ops=None, conditional_fp32_ops=None,\n                           fp32_ops=None):\n\n    nd_mod = ndarray.numpy._internal if is_numpy_module else ndarray\n    sy_mod = symbol.numpy._internal if is_numpy_module else symbol\n\n    def _ndarray_wrapper(f, target_dtype, fp32_param=None, cond_arg=None):\n        def _new_fun(*args, **kwargs):\n            if cond_arg is not None:\n                if (cond_arg[0] not in kwargs or\n                        kwargs[cond_arg[0]] not in cond_arg[1]):\n                    return f(*args, **kwargs)\n            if fp32_param:\n                new_args = []\n                for i, x in enumerate(args):\n                    if fp32_param[i]:\n                        new_args.append(x)\n                    else:\n                        new_args.append(_cast_symbol_NDArray(x, target_dtype, is_numpy_module))\n            else:\n                new_args = list(map(\n                    lambda x: _cast_symbol_NDArray(x, target_dtype, is_numpy_module), args))\n            args = tuple(new_args)\n            if fp32_param:\n                new_kwargs = {}\n                for k, v in kwargs.items():\n                    if k in fp32_param:\n                        new_kwargs[k] = v\n                    else:\n                        new_kwargs[k] = _cast_symbol_NDArray(v, target_dtype, is_numpy_module)\n                    kwargs = new_kwargs\n            else:\n                kwargs = {k: _cast_symbol_NDArray(v, target_dtype, is_numpy_module)\n                          for k, v in kwargs.items()}\n            return f(*args, **kwargs)\n        _new_fun.__name__ = f.__name__\n        _new_fun.__module__ = f.__module__\n        _new_fun.__doc__ = f.__doc__\n        return _new_fun\n\n    def _symbol_wrapper(f, target_dtype, fp32_param=None, cond_arg=None):\n        def _new_fun(*args, **kwargs):\n            if cond_arg is not None:\n                if (cond_arg[0] not in kwargs or\n                        kwargs[cond_arg[0]] not in cond_arg[1]):\n                    return f(*args, **kwargs)\n            sym = f(*args, **kwargs)\n            inputs = sym.get_children()\n            aux = sym.list_auxiliary_states()\n            if fp32_param:\n                new_inputs = []\n                for i, x in enumerate(inputs):\n                    if (x.name in aux) or fp32_param[i]:\n                        new_inputs.append(x)\n                    else:\n                        new_inputs.append(_cast_symbol_NDArray(x, target_dtype, is_numpy_module))\n                inputs = new_inputs\n            else:\n                inputs = list(map(lambda x: _cast_symbol_NDArray(x, target_dtype, is_numpy_module)\n                                  if x.name not in aux else x, inputs))\n            atomic_sym = sym._gen_atomic_symbol()\n            wrapped_sym = atomic_sym(*inputs)\n            wrapped_sym._set_attr(name=sym.name)\n            return wrapped_sym\n        _new_fun.__name__ = f.__name__\n        _new_fun.__module__ = f.__module__\n        _new_fun.__doc__ = f.__doc__\n        return _new_fun\n\n    def _symbol_widest_wrapper(f):\n        def _new_fun(*args, **kwargs):\n            symbols = []\n            is_symbol = False\n            args = list(args)\n            for i, arg in enumerate(args):\n                if isinstance(arg, (Symbol, NDArray)):\n                    symbols.append((args, i, arg))\n                    is_symbol = is_symbol or isinstance(arg, Symbol)\n            for k, arg in kwargs.items():\n                if isinstance(arg, (Symbol, NDArray)):\n                    symbols.append((kwargs, k, arg))\n                    is_symbol = is_symbol or isinstance(arg, Symbol)\n            if not is_symbol:\n                # NDArray case\n                widest_type = target_dtype\n                for _, _, arg in symbols:\n                    if isinstance(arg, NDArray):\n                        if arg.dtype == np.float32:\n                            widest_type = np.float32\n                for arr, index, arg in symbols:\n                    if arg.dtype != widest_type and arg.dtype == target_dtype:\n                        arr[index] = nd_mod.amp_cast(arg, dtype=widest_type)\n            else:\n                # Symbol case\n                sym_to_check = list(map(lambda x: x[2], symbols))\n                casted_syms = sy_mod.amp_multicast(*sym_to_check, num_outputs=len(sym_to_check))\n                symbols = list(map(lambda x_y: (x_y[0][0], x_y[0][1], x_y[1]),\n                                   zip(symbols, casted_syms)))\n                for arr, index, arg in symbols:\n                    arr[index] = arg\n\n            return f(*args, **kwargs)\n        _new_fun.__name__ = f.__name__\n        _new_fun.__module__ = f.__module__\n        _new_fun.__doc__ = f.__doc__\n        return _new_fun\n\n    _wrapper = _symbol_wrapper if module in (symbol, Symbol, symbol_contrib) else _ndarray_wrapper\n\n    fp32_param_list = list_lp16_use_fp32_params(target_dtype)\n    wrap_list = target_precision_ops if target_precision_ops is not None \\\n                    else list_lp16_ops(target_dtype)\n    for fun_name in get_aliases(wrap_list):\n        fun_name, modules = get_fun_to_wrap(fun_name, module)\n        for cur_module in modules:\n            f_to_wrap = getattr(cur_module, fun_name)\n            fp32_param = fp32_param_list[fun_name] if (fp32_param_list and fun_name in fp32_param_list) else None\n            setattr(cur_module, fun_name, _wrapper(f_to_wrap, target_dtype, fp32_param=fp32_param))\n            if not is_numpy_module and cur_module == module:\n                setattr(module.op, fun_name, _wrapper(f_to_wrap, target_dtype, fp32_param=fp32_param))\n\n    wrap_list = fp32_ops if fp32_ops is not None else list_fp32_ops(target_dtype)\n    for fun_name in get_aliases(wrap_list):\n        fun_name, modules = get_fun_to_wrap(fun_name, module)\n        for cur_module in modules:\n            f_to_wrap = getattr(cur_module, fun_name)\n            setattr(cur_module, fun_name, _wrapper(f_to_wrap, np.float32))\n            if not is_numpy_module and cur_module == module:\n                setattr(module.op, fun_name, _wrapper(f_to_wrap, np.float32))\n\n    wrap_list = conditional_fp32_ops if conditional_fp32_ops is not None \\\n                    else list_conditional_fp32_ops(target_dtype)\n    for fun_name, arg, arg_values in get_cond_aliases(wrap_list):\n        fun_name, modules = get_fun_to_wrap(fun_name, module)\n        for cur_module in modules:\n            f_to_wrap = getattr(cur_module, fun_name)\n            setattr(cur_module, fun_name, _wrapper(f_to_wrap, np.float32, cond_arg=(arg, arg_values)))\n            if not is_numpy_module and cur_module == module:\n                setattr(module.op, fun_name, _wrapper(f_to_wrap, np.float32, cond_arg=(arg, arg_values)))\n\n    for fun_name in get_aliases(list_widest_type_cast(target_dtype)):\n        fun_name, modules = get_fun_to_wrap(fun_name, module)\n        for cur_module in modules:\n            f_to_wrap = getattr(cur_module, fun_name)\n            setattr(cur_module, fun_name, _symbol_widest_wrapper(f_to_wrap))\n            if not is_numpy_module and cur_module == module:\n                setattr(module.op, fun_name, _symbol_widest_wrapper(f_to_wrap))\n\ndef _wrap_loss_output_functions(module, ls, target_dtype):\n    if module == ndarray:\n        def _wrapper(f):\n            def _scaling_wrapper(*args, **kwargs):\n                if 'grad_scale' in kwargs:\n                    kwargs['grad_scale'] = kwargs['grad_scale'] * ls.loss_scale\n                else:\n                    kwargs['grad_scale'] = ls.loss_scale\n                return f(*args, **kwargs)\n            _scaling_wrapper.__name__ = f.__name__\n            _scaling_wrapper.__module__ = f.__module__\n            _scaling_wrapper.__doc__ = f.__doc__\n            return _scaling_wrapper\n    else:\n        def _wrapper(f):\n            def _warning_wrapper(*args, **kwargs):\n                logging.warning(\"%s does not support dynamic loss scaling \"\n                                \"in symbolic and hybridized execution.\", f.__name__)\n                return f(*args, **kwargs)\n            _warning_wrapper.__name__ = f.__name__\n            _warning_wrapper.__module__ = f.__module__\n            _warning_wrapper.__doc__ = f.__doc__\n            return _warning_wrapper\n\n    for fun_name in list_loss_output_functions(target_dtype):\n        try:\n            f_to_wrap = getattr(module, fun_name)\n            setattr(module, fun_name, _wrapper(f_to_wrap))\n        except AttributeError:\n            pass\n\n_amp_initialized = False\n_amp_loss_scale_initialized = False\n_loss_scaler = None\n\n@contextlib.contextmanager\ndef scale_loss(loss, optimizer_or_trainer):\n    assert optimizer_or_trainer._amp_loss_scaler is not None, \\\n        'Loss scaler is not initialized, did you forget to call amp.init_trainer()?'\n    optimizer_or_trainer._scale = (optimizer_or_trainer._amp_original_scale /\n                                   optimizer_or_trainer._amp_loss_scaler.loss_scale)\n    if isinstance(loss, (list, tuple)):\n        yield [l * optimizer_or_trainer._amp_loss_scaler.loss_scale for l in loss]\n    else:\n        yield optimizer_or_trainer._amp_loss_scaler.loss_scale * loss\n\ndef warn_if_model_exists():\n    for f in inspect.stack():\n        for k, v in f.frame.f_locals.items():\n            if isinstance(v, Block):\n                logging.warning('Block %s created in [%s:%d] before AMP init.',\n                                k, f.filename, f.lineno)\n                return\n\ndef init(target_dtype='float16', target_precision_ops=None,\n         conditional_fp32_ops=None, fp32_ops=None, layout_optimization=False):\n    \"\"\"Initialize AMP (automatic mixed precision).\n\n    This needs to be done before model creation.\n\n    Parameters\n    ----------\n    target_dtype : {'float16', 'bfloat16'}\n        Target low precision type for AMP. Currently only float16 and bfloat16 are supported.\n    target_precision_ops : list of string\n        Override the list of functions casted to target_dtype. Entries in this list\n        are names of the functions casted to target_dtype.\n    conditional_fp32_ops : list of (string, string, list of string)\n        Override the list of functions conditionally casted to FP32. The format\n        of the list is (name of the function, name of the parameter, list of\n        values of the parameter that make the function be casted to FP32).\n    fp32_ops : list of string\n        Override the list of functions casted to FP32. Entries in this list\n        are names of the functions casted to FP32.\n    \"\"\"\n    global _amp_initialized\n    global _loss_scaler\n    if not _amp_initialized:\n        assert target_dtype in ['float16', np.float16, 'bfloat16', bfloat16], \\\n               \"AMP currently supports only float16 or bfloat16 as a target_dtype\"\n        _amp_initialized = True\n        log_msg = \"Using AMP\"\n        if layout_optimization:\n            log_msg += \"\\n - layout optimization: enabled\"\n            check_call(_LIB.MXSetOptimizeLayout(ctypes.c_bool(True)))\n        logging.info(log_msg)\n        if target_dtype == \"bfloat16\":\n            target_dtype = bfloat16\n        else:\n            target_dtype = np.dtype(target_dtype)\n\n        warn_if_model_exists()\n\n        ops = get_all_registered_operators_grouped()\n        get_aliases_nd = lambda l: [a for op in l for a in ops[op] if not base._is_np_op(a)]\n        get_aliases_np = lambda l: [a for op in l for a in ops[op] if base._is_np_op(a)]\n        get_aliases_np_pub = lambda l: [a for op in l for a in ops[op]\n                                        if a.startswith(('_np_', '_npx_'))]\n        get_cond_aliases_nd = lambda l: [(a, *rest) for op, *rest in l for a in ops[op]\n                                         if not base._is_np_op(a)]\n        get_cond_aliases_np = lambda l: [(a, *rest) for op, *rest in l for a in ops[op]\n                                         if base._is_np_op(a)]\n        get_cond_aliases_np_pub = lambda l: [(a, *rest) for op, *rest in l for a in ops[op]\n                                             if a.startswith(('_np_', '_npx_'))]\n        sy_submodules = {p:getattr(symbol, p[1:-1]) for p in base._OP_NAME_PREFIX_LIST}\n        get_sy_fun = lambda fun, mod: _get_nd_fun_to_wrap(fun, mod, sy_submodules)\n        nd_submodules = {p:getattr(ndarray, p[1:-1]) for p in base._OP_NAME_PREFIX_LIST}\n        get_nd_fun = lambda fun, mod: _get_nd_fun_to_wrap(fun, mod, nd_submodules)\n        get_np_sy_fun = lambda fun, mod: _get_np_fun_to_wrap(fun, \"mxnet.symbol\")\n        get_np_nd_fun = lambda fun, mod: _get_np_fun_to_wrap(fun, \"mxnet.ndarray\")\n        get_np_fun = lambda fun, mode: _get_np_fun_to_wrap(fun, \"mxnet\")\n        todo = [\n            (symbol, False, get_aliases_nd, get_cond_aliases_nd, get_sy_fun),\n            (ndarray, False, get_aliases_nd, get_cond_aliases_nd, get_nd_fun),\n            (symbol.numpy, True, get_aliases_np, get_cond_aliases_np, get_np_sy_fun),\n            (ndarray.numpy, True, get_aliases_np, get_cond_aliases_np, get_np_nd_fun),\n            (numpy, True, get_aliases_np_pub, get_cond_aliases_np_pub, get_np_fun),\n        ]\n        _loss_scaler = LossScaler()\n        for module, is_numpy, get_aliases, get_cond_aliases, get_fun in todo:\n            _wrap_module_functions(module, is_numpy, target_dtype, get_aliases, get_cond_aliases,\n                                   get_fun, target_precision_ops, conditional_fp32_ops, fp32_ops)\n            _wrap_loss_output_functions(module, _loss_scaler, target_dtype)\n\ndef init_trainer(optimizer_or_trainer):\n    \"\"\"Initialize trainer or optimizer to work with AMP dynamic loss scaling.\n\n    Parameters\n    ----------\n    optimizer_or_trainer : Optimizer or Trainer\n        MXNet Optimizer or Gluon trainer to initialize with AMP\n    \"\"\"\n    global _amp_loss_scale_initialized\n    global _amp_initialized\n    global _loss_scaler\n    assert _amp_initialized, \"AMP not initialized, did you forget to call amp.init()?\"\n    if not _amp_loss_scale_initialized:\n        _amp_loss_scale_initialized = True\n        loss_scaler = _loss_scaler\n    else:\n        loss_scaler = LossScaler()\n    #_wrap_output\n    if isinstance(optimizer_or_trainer, trainer.Trainer):\n        optimizer_or_trainer._amp_loss_scaler = loss_scaler\n        optimizer_or_trainer._amp_original_scale = optimizer_or_trainer._scale\n        trainer.Trainer.amp_loss_scale = property(lambda self: self._amp_loss_scaler.loss_scale)\n    elif isinstance(optimizer_or_trainer, opt.Optimizer):\n        raise TypeError(\"AMP is currently only compatible with Gluon Trainer\")\n    else:\n        raise TypeError(\"optimizer_or_trainer should be a Gluon Trainer or \"\n                        f\"an optimizer, instead is {type(optimizer_or_trainer)}\")\n\ndef unscale(optimizer_or_trainer):\n    \"\"\"Check and unscale the gradients manually. This function should only be used\n    if accessing gradients is necessary, e.g. for gradient clipping.\n\n    Parameters\n    ----------\n    optimizer_or_trainer : Optimizer or Trainer\n        MXNet optimizer or Gluon Trainer used when scaling the gradients\n    \"\"\"\n    if isinstance(optimizer_or_trainer, trainer.Trainer):\n        valid_grads = [p._grad for p in optimizer_or_trainer._params if p._grad is not None]\n        for grads in valid_grads:\n            # TODO(ptredak): make a bulked unscale\n            for g in grads:\n                g[:] *= optimizer_or_trainer._scale\n        optimizer_or_trainer._scale = 1.\n    elif isinstance(optimizer_or_trainer, opt.Optimizer):\n        # TODO(ptredak): make it work with the optimizer\n        raise TypeError(\"AMP is currently only compatible with Gluon Trainer\")\n    else:\n        raise TypeError(\"optimizer_or_trainer should be a Gluon Trainer or \"\n                        f\"an optimizer, instead is {type(optimizer_or_trainer)}\")\n\n\ndef convert_symbol(sym, input_dtypes, param_dtypes, target_dtype, target_dtype_ops=None,\n                   fp32_ops=None, conditional_fp32_ops=None, excluded_sym_names=[],\n                   cast_params_offline=False):\n    \"\"\"Given a symbol object representing a neural network of data type FP32 and target_dtype,\n    add cast layers according to the op lists (target_dtype_ops, fp32_ops,\n    conditional_fp32_ops) if provided, otherwise use the default\n    lists provided by the framework.\n\n    Parameters\n    ----------\n    sym : Symbol\n        FP32 neural network symbol\n    input_dtypes: dict\n        Dictionary mapping names of model inputs to their dtypes\n    param_dtypes: dict\n        Dictionary mapping names of model parameters to their dtypes\n    target_dtype : str or numpy, optional defaults to float16\n        currently only supports float16 and bfloat16. The target dtype indicates to add cast layers\n        when possible so that lower precision computation can be leveraged.\n    target_dtype_ops : list of strs, optional\n        Override the list of operator names casted to the target_dtype.\n        If None, uses the framework's default list to be casted to target_dtype.\n    fp32_ops : list of strs, optional\n        Override the list of operator names casted to FP32.\n        If None, uses the framework's default list to be casted to FP32.\n    conditional_fp32_ops : list of (string, string, list of string), optional\n        Override the list of functions to be casted to FP32.\n        The format of the list is\n        (name of the function, name of the parameter,\n         list of values of the parameter that make the operator to be casted to FP32)\n    excluded_sym_names : list of strs, optional\n        A list of strings that represent the names of symbols that users want to exclude\n        from being casted to LP16 or FP32.\n    data_names : list of strs, optional\n        A list of strings that represent input data tensor names to the model\n    cast_params_offline : bool, default False\n        Whether to cast arg_params and aux_params now, instead of doing it every time at runtime.\n    \"\"\"\n    import json\n\n    assert isinstance(sym, Symbol), \"First argument to convert_symbol should be a Symbol\"\n    assert target_dtype_ops is None or isinstance(target_dtype_ops, list), \\\n        \"target_dtype_ops should be a list of strings\"\n    assert fp32_ops is None or isinstance(fp32_ops, list), \\\n        \"fp32_ops should be a list of strings\"\n    assert conditional_fp32_ops is None or isinstance(conditional_fp32_ops, list), \\\n        \"conditional_fp32_ops should be a list of strings\"\n\n    target_dtype = get_dtype_name(target_dtype)\n    assert target_dtype in ['float16', *bfloat16.names], \\\n        \"Only float16 and bfloat16 types are currently supported as target_dtype\"\n\n    if target_dtype_ops is None:\n        target_dtype_ops = list_lp16_ops(target_dtype)\n    if fp32_ops is None:\n        fp32_ops = list_fp32_ops(target_dtype)\n\n    # conditional ops\n    if conditional_fp32_ops is None:\n        conditional_fp32_ops = list_conditional_fp32_ops(target_dtype)\n    cond_ops = {cond_op[0]: {} for cond_op in conditional_fp32_ops}\n    for cond_op in conditional_fp32_ops:\n        op_name, attr_name, attr_vals = cond_op\n        assert isinstance(op_name, str) and isinstance(attr_name, str) and isinstance(attr_vals, list), \\\n            \"conditional_fp32_ops should be a list of (str, str, list of str)\"\n        cond_ops[op_name].setdefault(attr_name, []).extend(attr_vals)\n\n    nodes_attrs = sym.attr_dict()\n    nodes_op = {n['name']: n['op'] for n in json.loads(sym.tojson())['nodes']}\n    for node_name, node_op in nodes_op.items():\n        if node_op not in cond_ops:\n            continue\n        node_attrs = nodes_attrs[node_name]\n        for attr_name, attr_vals in cond_ops[node_op].items():\n            assert attr_name in node_attrs\n            if node_attrs[attr_name] in attr_vals:\n                excluded_sym_names.append(node_name)\n                break\n\n    excluded_sym_names = set(excluded_sym_names)\n    for node in sym.get_internals():\n        if node.name in excluded_sym_names:\n            excluded_sym_names.remove(node.name)\n            opt_constraints = node.attr('__opt_constraint__')\n            opt_constraints = 0 if opt_constraints is None else int(opt_constraints)\n            opt_constraints |= HybridBlock.OptConstraint.Flag.DisableAMP.value\n            node._set_attr(__opt_constraint__=str(opt_constraints))\n\n    if len(excluded_sym_names) > 0:\n        logging.warning(\"excluded_sym_names are not present in the network. Missing nodes: {}\".format(\n            excluded_sym_names))\n\n    # Op lists should not intersect\n    common_ops = set(target_dtype_ops) & set(fp32_ops)\n    assert len(common_ops) == 0, \"Common ops in target_dtype_ops and fp32_ops: {}\".format(common_ops)\n    common_ops = set(target_dtype_ops) & set(cond_ops)\n    assert len(common_ops) == 0, \"Common ops in target_dtype_ops and conditional_fp32_ops: {}\".format(\n        common_ops)\n    common_ops = set(cond_ops) & set(fp32_ops)\n    assert len(common_ops) == 0, \"Common ops in fp32_ops and conditional_fp32_ops: {}\".format(common_ops)\n\n    combined_ops = set(target_dtype_ops + fp32_ops + list(cond_ops.keys()))\n    original_cond_ops = [cond_op[0] for cond_op in list_conditional_fp32_ops(target_dtype)]\n    all_lp16_fp32_ops = set(list_lp16_ops(target_dtype) + list_fp32_ops(target_dtype) +\n                            list_lp16_fp32_ops(target_dtype) + original_cond_ops)\n\n    illegal_ops = combined_ops - all_lp16_fp32_ops\n    assert len(illegal_ops) == 0, f'''Can only choose ops from one of the four lists\n                            for lp16_ops and fp32_ops\n                            1. amp.list_lp16_ops(target_dtype)\n                            2. amp.list_fp32_ops(target_dtype)\n                            3. amp.list_lp16_fp32_ops(target_dtype)\n                            4. amp.list_conditional_fp32_ops(target_dtype)\n                            Op {illegal_ops} not in any of them'''\n\n    widest_dtype_ops = list_widest_type_cast(target_dtype)\n\n    input_names = list(input_dtypes.keys())\n    all_arg_names, all_arg_types = [], []\n\n    for name, dtype in {**input_dtypes, **param_dtypes}.items():\n        all_arg_names.append(name)\n        all_arg_types.append(dtype_np_to_mx(dtype))\n    out = SymbolHandle()\n    check_call(_LIB.MXReducePrecisionSymbol(sym.handle,\n                                            ctypes.byref(out),\n                                            ctypes.c_int(dtype_np_to_mx(target_dtype)),\n                                            ctypes.c_int(cast_params_offline),\n                                            c_str(OFFLINE_CAST_DTYPE_ATTR),\n                                            ctypes.c_uint(len(input_names)),\n                                            c_str_array(input_names),\n                                            ctypes.c_uint(len(all_arg_names)),\n                                            c_str_array(all_arg_names),\n                                            c_array_buf(ctypes.c_int, array('i', all_arg_types)),\n                                            ctypes.c_uint(len(target_dtype_ops)),\n                                            c_str_array(target_dtype_ops),\n                                            ctypes.c_uint(len(fp32_ops)),\n                                            c_str_array(fp32_ops),\n                                            ctypes.c_uint(len(widest_dtype_ops)),\n                                            c_str_array(widest_dtype_ops)))\n    return type(sym)(out)\n\n\ndef convert_model(sym, arg_params, aux_params, input_dtypes, target_dtype,\n                  target_dtype_ops=None, fp32_ops=None, conditional_fp32_ops=None,\n                  excluded_sym_names=[], cast_params_offline=False):\n    \"\"\"API for converting a model from FP32 model to a mixed precision model.\n    MXNet tries to convert the FP32 model to mixed precision model by adding\n    cast layers using amp_cast and amp_multicast operators which can be used for inference use cases.\n    The decision on which cast layer to add is based on hardcoded lists for Automatic Mixed Precision\n    in MXNet. These lists can be overridden by the user by providing their own lists\n    using : targe_precision_ops, fp32_ops, widest_precision_ops, conditional_fp32_ops\n\n    arg_params : dict\n        Dictionary of name to `NDArray`.\n    aux_params : dict\n        Dictionary of name to `NDArray`.\n    input_dtypes: dict\n        Dictionary mapping names of model inputs to their dtypes\n    target_dtype : str\n        Currently only supports float16 and bfloat 16. The target dtype indicates to add cast layers\n        when possible so that lower precision computation can be leveraged.\n    target_dtype_ops : list of strs\n        Override the list of operator names casted to target_dtype.\n        If None, uses the framework's default list to be casted to target dtype.\n    fp32_ops : list of strs\n        Override the lists of operator names casted to FP32.\n        If None, uses the framework's default list to be casted to FP32.\n    widest_dtype_ops : list of strs\n        A list of op names provided by user which should run in widest precision among its inputs.\n        If None, uses the framework's default list of widest_precision_ops.\n    conditional_fp32_ops : list of (string, string, list of string)\n        Override the list of operators to be casted to FP32.\n        The format of the list is\n        (name of the function, name of the parameter,\n         list of values of the parameter that make the operator to be casted to\n        fp32)\n    excluded_sym_names : list of strs\n        A list of strings that represent the names of symbols that users want to exclude\n        from being executed in lower precision.\n    cast_params_offline : bool, default False\n        Whether to cast arg_params and aux_params now, instead of doing it every time at runtime.\n    \"\"\"\n    assert isinstance(sym, Symbol), \"First argument to convert_model should be a Symbol\"\n    assert isinstance(\n        arg_params, dict), \"Second argument to convert_model should be a dict of name to ndarray\"\n    assert isinstance(\n        aux_params, dict), \"Third argument to convert_model should be a dict of name to ndarray\"\n\n    arg_params = arg_params.copy()\n    aux_params = aux_params.copy()\n    param_dtypes = {name: data.dtype for name, data in arg_params.items()}\n    param_dtypes.update({name: data.dtype for name, data in aux_params.items()})\n    sym = convert_symbol(sym, input_dtypes, param_dtypes, target_dtype, target_dtype_ops,\n                         fp32_ops, conditional_fp32_ops, excluded_sym_names, cast_params_offline)\n\n    # If dtype is set for params, cast the param to that dtype\n    attr_dict = sym.attr_dict()\n    for sym_name in sym.list_arguments():\n        if attr_dict.get(sym_name, {}).get(OFFLINE_CAST_DTYPE_ATTR, '') != '' and sym_name in arg_params:\n            typ = get_dtype_type(attr_dict[sym_name][OFFLINE_CAST_DTYPE_ATTR])\n            if arg_params[sym_name].dtype != typ:\n                arg_params[sym_name] = arg_params[sym_name].astype(typ)\n\n    for sym_name in sym.list_auxiliary_states():\n        if attr_dict.get(sym_name, {}).get(OFFLINE_CAST_DTYPE_ATTR, '') != '' and sym_name in aux_params:\n            typ = get_dtype_type(attr_dict[sym_name][OFFLINE_CAST_DTYPE_ATTR])\n            if aux_params[sym_name].dtype != typ:\n                aux_params[sym_name] = aux_params[sym_name].astype(typ)\n\n    # Return the converted symbol and casted params\n    return sym, arg_params, aux_params\n\n\n@wrap_ctx_to_device_func\ndef convert_hybrid_block(block, data_example, target_dtype, target_dtype_ops=None,\n                         fp32_ops=None, conditional_fp32_ops=None,\n                         excluded_sym_names=[], device=None,\n                         cast_params_offline=False):\n    \"\"\"Given a hybrid block/symbol block representing a FP32 model and a target_dtype,\n    return a block with mixed precision support which can be used for inference use cases.\n\n    Parameters\n    ----------\n    block : HybridBlock or SymbolBlock object\n        FP32 HybridBlock or SymbolBlock object\n    data_example: tuple or list of NDArrays\n        Data example, representing the data that this model will work with during the inference.\n    target_dtype : str or numpy\n        currently only supports float16 and bfloat16. The target dtype indicates to add cast layers\n        when possible so that lower precision computation can be leveraged.\n    target_precision_ops : list of strs\n        Override the list of operator names casted to target_dtype.\n        If None, uses the framework's default list to be casted to FP32.\n    conditional_fp32_ops : list of (str, str, list of str)\n        Override the list of functions to be casted to FP32.\n        The format of the list is\n        (name of the function, name of the parameter,\n         list of values of the parameter that make the operator to be casted to FP32\n    excluded_sym_names : list of strs\n        A list of strings that represent the names of symbols that users want to exclude\n        from being quantized\n    device : Device\n        Device on which model parameters should live. Default value: current device.\n    cast_params_offline : bool, default False\n        Whether to cast arg_params and aux_params now, instead of doing it every time at runtime.\n    \"\"\"\n    from ..gluon import SymbolBlock\n    from ..ndarray import NDArray as ND_NDArray, waitall\n    from ..numpy import ndarray as NP_NDArray\n\n    assert isinstance(block, HybridBlock), \"block input should be a HybridBlock\"\n    if not isinstance(data_example, (list, tuple)):\n        data_example = [data_example]\n    for data in data_example:\n        assert isinstance(data, (ND_NDArray, NP_NDArray)), \"Data example must be composed of \" \\\n            \"mxnet.numpy.ndarray or mxnet.ndarray.NDArray instances\"\n    if not block._active:\n        block.hybridize(static_alloc=False, static_shape=False)\n    block(*data_example)\n    waitall()\n\n    sym, params = block.export(None, remove_amp_cast=False)\n    args, auxs = {}, {}\n    for name, data in params.items():\n        if name.startswith('arg:'):\n            arg_name = name[len('arg:'):]\n            args[arg_name] = data\n        else:\n            assert name.startswith('aux:')\n            aux_name = name[len('aux:'):]\n            auxs[aux_name] = data\n\n    input_names = set(sym.list_arguments()) - (set(args.keys()) | set(auxs.keys()))\n    input_names_ordered = HybridBlock.generate_arg_names(len(data_example))\n    assert input_names == set(input_names_ordered)\n\n    input_dtypes = {name: data.dtype for name, data in zip(input_names_ordered, data_example)}\n    lp_sym, lp_args, lp_auxs = convert_model(sym, args, auxs, input_dtypes, target_dtype,\n                                             target_dtype_ops, fp32_ops, conditional_fp32_ops,\n                                             excluded_sym_names, cast_params_offline)\n\n    inputs = [in_sym for in_sym in lp_sym.get_inputs() if in_sym.name in input_names]\n    param_dict = lp_args\n    param_dict.update(lp_auxs)\n\n    ret = SymbolBlock(lp_sym, inputs)\n    ret.load_dict(param_dict, device=device, cast_dtype=True, dtype_source='saved')\n    return ret\n\n\ndef list_lp16_ops(target_dtype):\n    \"\"\"Get the default list of LP16 ops for AMP\n    \"\"\"\n    if target_dtype in ['float16', np.float16]:\n        return lists.symbol_fp16.FP16_FUNCS\n    else:\n        assert get_dtype_name(target_dtype) in bfloat16.names, \"not supported type\"\n        return lists.symbol_bf16.BF16_FUNCS\n\ndef list_fp32_ops(target_dtype):\n    \"\"\"Get the default list of FP32 ops for AMP\n    \"\"\"\n    if target_dtype in ['float16', np.float16]:\n        return lists.symbol_fp16.FP32_FUNCS\n    else:\n        assert get_dtype_name(target_dtype) in bfloat16.names, \"not supported type\"\n        return lists.symbol_bf16.FP32_FUNCS\n\ndef list_lp16_fp32_ops(target_dtype):\n    \"\"\"Get the default list of ops which run in both LP16 and FP32\n    \"\"\"\n    if target_dtype in ['float16', np.float16]:\n        return lists.symbol_fp16.FP16_FP32_FUNCS\n    else:\n        assert get_dtype_name(target_dtype) in bfloat16.names, \"not supported type\"\n        return lists.symbol_bf16.BF16_FP32_FUNCS\n\ndef list_conditional_fp32_ops(target_dtype):\n    \"\"\"Get the conditional fp32 ops list\n    \"\"\"\n    if target_dtype in ['float16', np.float16]:\n        return lists.symbol_fp16.CONDITIONAL_FP32_FUNCS\n    else:\n        assert get_dtype_name(target_dtype) in bfloat16.names, \"not supported type\"\n        return lists.symbol_bf16.CONDITIONAL_FP32_FUNCS\n\ndef list_widest_type_cast(target_dtype):\n    \"\"\"Get the widest type cast ops list\n    \"\"\"\n    if target_dtype in ['float16', np.float16]:\n        return lists.symbol_fp16.WIDEST_TYPE_CASTS\n    else:\n        assert get_dtype_name(target_dtype) in bfloat16.names, \"not supported type\"\n        return lists.symbol_bf16.WIDEST_TYPE_CASTS\n\ndef list_loss_output_functions(target_dtype):\n    \"\"\"Get loss function list\n    \"\"\"\n    if target_dtype in ['float16', np.float16]:\n        return lists.symbol_fp16.LOSS_OUTPUT_FUNCTIONS\n    else:\n        assert get_dtype_name(target_dtype) in bfloat16.names, \"not supported type\"\n        return lists.symbol_bf16.LOSS_OUTPUT_FUNCTIONS\n\ndef list_lp16_use_fp32_params(target_dtype):\n    \"\"\" Get the params restrict for LP16\n\n    \"\"\"\n    if target_dtype in ['float16', np.float16]:\n        return None\n    else:\n        assert get_dtype_name(target_dtype) in bfloat16.names, \"not supported type\"\n        return lists.symbol_bf16.BF16_USE_FP32_PARAMS\n"
  },
  {
    "path": "python/mxnet/amp/lists/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Lists of functions whitelisted/blacklisted for automatic mixed precision.\"\"\"\n\nfrom . import symbol_fp16\nfrom . import symbol_bf16\n"
  },
  {
    "path": "python/mxnet/amp/lists/symbol_bf16.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Lists of functions whitelisted/blacklisted for automatic mixed precision in symbol API.\"\"\"\n\nfrom ...runtime import Features\n\n# Functions that should be cast to lower precision\nBF16_FUNCS = [\n    'Convolution',\n    'Deconvolution',\n    'FullyConnected'\n]\nif Features.instance.is_enabled('ONEDNN'):\n    BF16_FUNCS.extend([\n        '_sg_onednn_conv',\n        '_sg_onednn_fully_connected',\n        '_sg_onednn_selfatt_qk',\n        '_sg_onednn_selfatt_qk_split',\n        '_sg_onednn_selfatt_valatt'\n    ])\n\n\n# Functions that should not be casted, either because\n# they are irrelevant (not used in the network itself\n# like image transformations or optimizers) or they\n# are dtype neutral (can work in both bf16 and fp32)\nBF16_FP32_FUNCS = [\n    '_contrib_AdaptiveAvgPooling2D',\n    'Activation',\n    'BatchNorm',\n    'LayerNorm',\n    'LRN',\n    'softmax',\n    'log_softmax',\n    #'masked_softmax', TODO: fix segfault appearing for a 4D input tensor\n    'Pooling',\n    '_npi_mean',\n    '_npi_sum',\n    '_npi_square',\n    '_npi_sqrt',\n    '_npi_exp',\n    '_npi_tanh',\n    '_npi_transpose',\n    '_npx_reshape',\n    '_npi_where',\n    #'_contrib_quantize_asym', # used in rnn, which is hard to convert to bf16\n    '_contrib_quantize_v2',\n    #'_contrib_quantize', # not used anymore\n    'sum',\n    'mean',\n    '_copy',\n    'Reshape',\n    'Flatten',\n    'transpose',\n    'expand_dims',\n    'slice',\n    'stack',\n    'space_to_depth',\n    '_split_v2',\n\n    # no oneDNN support:\n    'Cast',\n    'where',\n    'take',\n]\n# 'RNN', # GetEnv(\"MXNET_USE_ONEDNN_RNN\", 1)\n\n# Functions with multiple inputs, that need the same\n# type of all their inputs\nWIDEST_TYPE_CASTS = [\n    'Concat',\n    'dot',\n    'batch_dot',\n    'broadcast_add',\n    'broadcast_sub',\n    'broadcast_mul',\n    'broadcast_div',\n    'elemwise_add',\n    'add_n',\n    '_npi_dot',\n    '_npi_add',\n    '_npi_multiply',\n    '_npi_subtract',\n    '_npi_true_divide',\n]\nif Features.instance.is_enabled('ONEDNN'):\n    WIDEST_TYPE_CASTS.extend([\n        '_sg_onednn_batch_dot',\n        '_sg_onednn_batch_norm',\n    ])\n\n# Functions that when running with Bfloat16, the params that still need float32.\nBF16_USE_FP32_PARAMS = {\n    'BatchNormWithReLU': [\"\", \"gamma\", \"beta\", \"moving_mean\", \"moving_var\"],\n    'BatchNorm': [\"\", \"gamma\", \"beta\", \"moving_mean\", \"moving_var\"],\n}\n\n# Functions that have to be cast to FP32 due to possible\n# overflows\nFP32_FUNCS = [\n    'amp_cast',\n    'amp_multicast',\n    'masked_softmax',\n    'BilinearSampler',\n    'BlockGrad',\n    'CTCLoss',\n    'Correlation',\n    'Crop',\n    'Custom',\n    'Dropout',\n    'Embedding',\n    'GridGenerator',\n    'GroupNorm',\n    'IdentityAttachKLSparseReg',\n    'InstanceNorm',\n    'L2Normalization',\n    'LinearRegressionOutput',\n    'LogisticRegressionOutput',\n    'MAERegressionOutput',\n    'MakeLoss',\n    'Pad',\n    'RNN',\n    'ROIPooling',\n    'SVMOutput',\n    'SequenceLast',\n    'SequenceMask',\n    'SequenceReverse',\n    'SliceChannel',\n    'SoftmaxActivation',\n    'SoftmaxOutput',\n    'SpatialTransformer',\n    'SwapAxis',\n    'UpSampling',\n    '_CachedOp',\n    '_CachedOpThreadSafe',\n    '_CrossDeviceCopy',\n    '_CustomFunction',\n    '_NDArray',\n    '_Native',\n    '_NoGradient',\n    '_adabelief_update',\n    '_adamw_update',\n    '_arange',\n    '_cond',\n    '_contrib_BilinearResize2D',\n    '_contrib_DeformablePSROIPooling',\n    '_contrib_MultiBoxDetection',\n    '_contrib_MultiBoxPrior',\n    '_contrib_MultiBoxTarget',\n    '_contrib_MultiProposal',\n    '_contrib_PSROIPooling',\n    '_contrib_Proposal',\n    '_contrib_ROIAlign',\n    '_contrib_RROIAlign',\n    '_contrib_SyncBatchNorm',\n    '_contrib_allclose',\n    '_contrib_arange_like',\n    '_contrib_bipartite_matching',\n    '_contrib_boolean_mask',\n    '_contrib_box_decode',\n    '_contrib_box_encode',\n    '_contrib_box_iou',\n    '_contrib_box_nms',\n    '_contrib_calibrate_entropy',\n    '_contrib_count_sketch',\n    '_contrib_dequantize',\n    '_contrib_dgl_adjacency',\n    '_contrib_dgl_csr_neighbor_non_uniform_sample',\n    '_contrib_dgl_csr_neighbor_uniform_sample',\n    '_contrib_dgl_graph_compact',\n    '_contrib_dgl_subgraph',\n    '_contrib_div_sqrt_dim',\n    '_contrib_dynamic_reshape',\n    '_contrib_edge_id',\n    '_contrib_fft',\n    '_contrib_getnnz',\n    '_contrib_gradientmultiplier',\n    '_contrib_group_adagrad_update',\n    '_contrib_hawkesll',\n    '_contrib_index_array',\n    '_contrib_index_copy',\n    '_contrib_interleaved_matmul_encdec_qk',\n    '_contrib_interleaved_matmul_encdec_valatt',\n    '_contrib_interleaved_matmul_selfatt_qk',\n    '_contrib_interleaved_matmul_selfatt_valatt',\n    '_contrib_intgemm_fully_connected',\n    '_contrib_intgemm_maxabsolute',\n    '_contrib_intgemm_prepare_data',\n    '_contrib_intgemm_prepare_weight',\n    '_contrib_intgemm_take_weight',\n    '_contrib_quadratic',\n    '_contrib_quantize',\n    '_contrib_quantize_asym',\n    '_contrib_quantized_act',\n    '_contrib_quantized_batch_norm',\n    '_contrib_quantized_batch_norm_relu',\n    '_contrib_quantized_concat',\n    '_contrib_quantized_conv',\n    '_contrib_quantized_elemwise_add',\n    '_contrib_quantized_elemwise_mul',\n    '_contrib_quantized_embedding',\n    '_contrib_quantized_flatten',\n    '_contrib_quantized_fully_connected',\n    '_contrib_quantized_pooling',\n    '_contrib_quantized_reshape',\n    '_contrib_quantized_rnn',\n    '_contrib_quantized_transpose',\n    '_contrib_requantize',\n    '_contrib_round_ste',\n    '_contrib_sign_ste',\n    '_contrib_sldwin_atten_context',\n    '_contrib_sldwin_atten_mask_like',\n    '_contrib_sldwin_atten_score',\n    '_copyto',\n    '_cvcopyMakeBorder',\n    '_cvimdecode',\n    '_cvimread',\n    '_cvimresize',\n    '_div_scalar',\n    '_equal',\n    '_equal_scalar',\n    '_eye',\n    '_foreach',\n    '_full',\n    '_grad_add',\n    '_greater',\n    '_greater_equal',\n    '_greater_equal_scalar',\n    '_greater_scalar',\n    '_histogram',\n    '_hypot',\n    '_hypot_scalar',\n    '_identity_with_attr_like_rhs',\n    '_image_adjust_lighting',\n    '_image_crop',\n    '_image_flip_left_right',\n    '_image_flip_top_bottom',\n    '_image_normalize',\n    '_image_random_brightness',\n    '_image_random_color_jitter',\n    '_image_random_contrast',\n    '_image_random_crop',\n    '_image_random_flip_left_right',\n    '_image_random_flip_top_bottom',\n    '_image_random_hue',\n    '_image_random_lighting',\n    '_image_random_resized_crop',\n    '_image_random_saturation',\n    '_image_resize',\n    '_image_to_tensor',\n    '_imdecode',\n    '_lesser',\n    '_lesser_equal',\n    '_lesser_equal_scalar',\n    '_lesser_scalar',\n    '_linalg_det',\n    '_linalg_extractdiag',\n    '_linalg_extracttrian',\n    '_linalg_gelqf',\n    '_linalg_gemm',\n    '_linalg_gemm2',\n    '_linalg_inverse',\n    '_linalg_makediag',\n    '_linalg_maketrian',\n    '_linalg_potrf',\n    '_linalg_potri',\n    '_linalg_slogdet',\n    '_linalg_sumlogdiag',\n    '_linalg_syevd',\n    '_linalg_syrk',\n    '_linalg_trmm',\n    '_linalg_trsm',\n    '_linspace',\n    '_logical_and',\n    '_logical_and_scalar',\n    '_logical_or',\n    '_logical_or_scalar',\n    '_logical_xor',\n    '_logical_xor_scalar',\n    '_maximum',\n    '_maximum_scalar',\n    '_minimum',\n    '_minimum_scalar',\n    '_minus_scalar',\n    '_mod',\n    '_mod_scalar',\n    '_mp_adabelief_update',\n    '_mp_adamw_update',\n    '_mul_scalar',\n    '_multi_adabelief_update',\n    '_multi_adamw_update',\n    '_multi_lamb_update',\n    '_multi_lans_update',\n    '_multi_mp_adabelief_update',\n    '_multi_mp_adamw_update',\n    '_multi_mp_lamb_update',\n    '_multi_mp_lans_update',\n    '_not_equal',\n    '_not_equal_scalar',\n    '_np_reshape',\n    '_npi_absolute',\n    '_npi_add_scalar',\n    '_npi_advanced_indexing',\n    '_npi_advanced_indexing_multiple',\n    '_npi_all',\n    '_npi_any',\n    '_npi_arange',\n    '_npi_arccos',\n    '_npi_arccosh',\n    '_npi_arcsin',\n    '_npi_arcsinh',\n    '_npi_arctan',\n    '_npi_arctan2',\n    '_npi_arctan2_scalar',\n    '_npi_arctanh',\n    '_npi_argmax',\n    '_npi_argmin',\n    '_npi_around',\n    '_npi_atleast_1d',\n    '_npi_atleast_2d',\n    '_npi_atleast_3d',\n    '_npi_average',\n    '_npi_bernoulli',\n    '_npi_bincount',\n    '_npi_bitwise_and',\n    '_npi_bitwise_and_scalar',\n    '_npi_bitwise_left_shift',\n    '_npi_bitwise_left_shift_scalar',\n    '_npi_bitwise_not',\n    '_npi_bitwise_or',\n    '_npi_bitwise_or_scalar',\n    '_npi_bitwise_right_shift',\n    '_npi_bitwise_right_shift_scalar',\n    '_npi_bitwise_xor',\n    '_npi_bitwise_xor_scalar',\n    '_npi_blackman',\n    '_npi_boolean_mask_assign_scalar',\n    '_npi_boolean_mask_assign_tensor',\n    '_npi_broadcast_to',\n    '_npi_cbrt',\n    '_npi_ceil',\n    '_npi_choice',\n    '_npi_cholesky',\n    '_npi_column_stack',\n    '_npi_copy',\n    '_npi_copysign',\n    '_npi_copysign_scalar',\n    '_npi_cos',\n    '_npi_cosh',\n    '_npi_cross',\n    '_npi_cumsum',\n    '_npi_degrees',\n    '_npi_delete',\n    '_npi_diag',\n    '_npi_diag_indices_from',\n    '_npi_diagflat',\n    '_npi_diagonal',\n    '_npi_diff',\n    '_npi_dsplit',\n    '_npi_dstack',\n    '_npi_ediff1d',\n    '_npi_eig',\n    '_npi_eigh',\n    '_npi_eigvals',\n    '_npi_eigvalsh',\n    '_npi_einsum',\n    '_npi_equal',\n    '_npi_equal_scalar',\n    '_npi_expm1',\n    '_npi_exponential',\n    '_npi_eye',\n    '_npi_fill_diagonal',\n    '_npi_fix',\n    '_npi_flip',\n    '_npi_floor',\n    '_npi_floor_divide',\n    '_npi_floor_divide_scalar',\n    '_npi_fmax',\n    '_npi_fmax_scalar',\n    '_npi_fmin',\n    '_npi_fmin_scalar',\n    '_npi_fmod',\n    '_npi_fmod_scalar',\n    '_npi_full',\n    '_npi_full_like',\n    '_npi_gamma',\n    '_npi_gcd',\n    '_npi_gcd_scalar',\n    '_npi_greater',\n    '_npi_greater_equal',\n    '_npi_greater_equal_scalar',\n    '_npi_greater_scalar',\n    '_npi_gumbel',\n    '_npi_hamming',\n    '_npi_hanning',\n    '_npi_hsplit',\n    '_npi_hstack',\n    '_npi_hypot',\n    '_npi_identity',\n    '_npi_indices',\n    '_npi_insert_scalar',\n    '_npi_insert_slice',\n    '_npi_insert_tensor',\n    '_npi_interp',\n    '_npi_isfinite',\n    '_npi_isinf',\n    '_npi_isnan',\n    '_npi_isneginf',\n    '_npi_isposinf',\n    '_npi_kron',\n    '_npi_laplace',\n    '_npi_lcm',\n    '_npi_lcm_scalar',\n    '_npi_ldexp',\n    '_npi_ldexp_scalar',\n    '_npi_less',\n    '_npi_less_equal',\n    '_npi_less_equal_scalar',\n    '_npi_less_scalar',\n    '_npi_linspace',\n    '_npi_log',\n    '_npi_log10',\n    '_npi_log1p',\n    '_npi_log2',\n    '_npi_logaddexp',\n    '_npi_logaddexp_scalar',\n    '_npi_logical_and',\n    '_npi_logical_and_scalar',\n    '_npi_logical_not',\n    '_npi_logical_or',\n    '_npi_logical_or_scalar',\n    '_npi_logical_xor',\n    '_npi_logical_xor_scalar',\n    '_npi_logistic',\n    '_npi_logspace',\n    '_npi_lstsq',\n    '_npi_matmul',\n    '_npi_matrix_rank',\n    '_npi_matrix_rank_none_tol',\n    '_npi_max',\n    '_npi_min',\n    '_npi_mod',\n    '_npi_mod_scalar',\n    '_npi_moveaxis',\n    '_npi_multinomial',\n    '_npi_multiply_scalar',\n    '_npi_nan_to_num',\n    '_npi_negative',\n    '_npi_norm',\n    '_npi_normal',\n    '_npi_normal_n',\n    '_npi_not_equal',\n    '_npi_not_equal_scalar',\n    '_npi_ones',\n    '_npi_pad',\n    '_npi_pareto',\n    '_npi_percentile',\n    '_npi_pinv',\n    '_npi_pinv_scalar_rcond',\n    '_npi_polyval',\n    '_npi_power',\n    '_npi_power_scalar',\n    '_npi_powerd',\n    '_npi_prod',\n    '_npi_qr',\n    '_npi_radians',\n    '_npi_rarctan2_scalar',\n    '_npi_rayleigh',\n    '_npi_rbitwise_left_shift_scalar',\n    '_npi_rbitwise_right_shift_scalar',\n    '_npi_rcopysign_scalar',\n    '_npi_reciprocal',\n    '_npi_repeats',\n    '_npi_rfloor_divide_scalar',\n    '_npi_rfmod_scalar',\n    '_npi_rint',\n    '_npi_rldexp_scalar',\n    '_npi_rmod_scalar',\n    '_npi_roll',\n    '_npi_rollaxis',\n    '_npi_rot90',\n    '_npi_rpower_scalar',\n    '_npi_rsubtract_scalar',\n    '_npi_rtrue_divide_scalar',\n    '_npi_share_memory',\n    '_npi_sign',\n    '_npi_sin',\n    '_npi_sinh',\n    '_npi_solve',\n    '_npi_squeeze',\n    '_npi_std',\n    '_npi_subtract_scalar',\n    '_npi_svd',\n    '_npi_tan',\n    '_npi_tensordot',\n    '_npi_tensordot_int_axes',\n    '_npi_tensorinv',\n    '_npi_tensorsolve',\n    '_npi_trace',\n    '_npi_tri',\n    '_npi_tril',\n    '_npi_tril_indices',\n    '_npi_triu',\n    '_npi_true_divide_scalar',\n    '_npi_trunc',\n    '_npi_uniform',\n    '_npi_uniform_n',\n    '_npi_unique',\n    '_npi_var',\n    '_npi_vstack',\n    '_npi_weibull',\n    '_npi_where_lscalar',\n    '_npi_where_rscalar',\n    '_npi_where_scalar2',\n    '_npi_zeros',\n    '_npx_cond',\n    '_npx_constraint_check',\n    '_npx_deformable_convolution',\n    '_npx_foreach',\n    '_npx_index_add',\n    '_npx_index_update',\n    '_npx_modulated_deformable_convolution',\n    '_npx_nonzero',\n    '_npx_quantized_reshape',\n    '_npx_quantized_transpose',\n    '_npx_relu',\n    '_npx_sigmoid',\n    '_npx_while_loop',\n    '_onehot_encode',\n    '_ones',\n    '_plus_scalar',\n    '_power',\n    '_power_scalar',\n    '_random_binomial',\n    '_random_exponential',\n    '_random_exponential_like',\n    '_random_gamma',\n    '_random_gamma_like',\n    '_random_generalized_negative_binomial',\n    '_random_generalized_negative_binomial_like',\n    '_random_negative_binomial',\n    '_random_negative_binomial_like',\n    '_random_normal',\n    '_random_normal_like',\n    '_random_pdf_dirichlet',\n    '_random_pdf_exponential',\n    '_random_pdf_gamma',\n    '_random_pdf_generalized_negative_binomial',\n    '_random_pdf_negative_binomial',\n    '_random_pdf_normal',\n    '_random_pdf_poisson',\n    '_random_pdf_uniform',\n    '_random_poisson',\n    '_random_poisson_like',\n    '_random_randint',\n    '_random_uniform',\n    '_random_uniform_like',\n    '_ravel_multi_index',\n    '_rdiv_scalar',\n    '_rminus_scalar',\n    '_rmod_scalar',\n    '_rnn_param_concat',\n    '_rpower_scalar',\n    '_sample_binomial',\n    '_sample_categorical',\n    '_sample_exponential',\n    '_sample_gamma',\n    '_sample_generalized_negative_binomial',\n    '_sample_multinomial',\n    '_sample_negative_binomial',\n    '_sample_normal',\n    '_sample_poisson',\n    '_sample_uniform',\n    '_sample_unique_zipfian',\n    '_scatter_set_nd',\n    '_set_value',\n    '_shuffle',\n    '_slice_assign',\n    '_slice_assign_scalar',\n    '_sparse_adagrad_update',\n    '_sparse_retain',\n    '_square_sum',\n    '_unravel_index',\n    '_while_loop',\n    '_zeros',\n    '_zeros_without_dtype',\n    'abs',\n    'adam_update',\n    'all_finite',\n    'arccos',\n    'arccosh',\n    'arcsin',\n    'arcsinh',\n    'arctan',\n    'arctanh',\n    'argmax',\n    'argmax_channel',\n    'argmin',\n    'argsort',\n    'batch_take',\n    'broadcast_axis',\n    'broadcast_equal',\n    'broadcast_greater',\n    'broadcast_greater_equal',\n    'broadcast_hypot',\n    'broadcast_lesser',\n    'broadcast_lesser_equal',\n    'broadcast_like',\n    'broadcast_logical_and',\n    'broadcast_logical_or',\n    'broadcast_logical_xor',\n    'broadcast_maximum',\n    'broadcast_minimum',\n    'broadcast_mod',\n    'broadcast_not_equal',\n    'broadcast_power',\n    'broadcast_to',\n    'cast_storage',\n    'cbrt',\n    'ceil',\n    'clip',\n    'col2im',\n    'cos',\n    'cosh',\n    'degrees',\n    'depth_to_space',\n    'diag',\n    'digamma',\n    'elemwise_div',\n    'elemwise_mul',\n    'elemwise_sub',\n    'erf',\n    'erfinv',\n    'exp',\n    'expm1',\n    'fill_element_0index',\n    'fix',\n    'floor',\n    'ftml_update',\n    'ftrl_update',\n    'gamma',\n    'gammaln',\n    'gather_nd',\n    'hard_sigmoid',\n    'im2col',\n    'khatri_rao',\n    'lamb_update_phase1',\n    'lamb_update_phase2',\n    'log',\n    'log10',\n    'log1p',\n    'log2',\n    'log_sigmoid',\n    'logical_not',\n    'make_loss',\n    'masked_log_softmax',\n    'max',\n    'min',\n    'mish',\n    'moments',\n    'mp_lamb_update_phase1',\n    'mp_lamb_update_phase2',\n    'mp_nag_mom_update',\n    'mp_sgd_mom_update',\n    'mp_sgd_update',\n    'multi_all_finite',\n    'multi_lars',\n    'multi_mp_sgd_mom_update',\n    'multi_mp_sgd_update',\n    'multi_sgd_mom_update',\n    'multi_sgd_update',\n    'multi_sum_sq',\n    'nag_mom_update',\n    'nanprod',\n    'nansum',\n    'negative',\n    'norm',\n    'one_hot',\n    'ones_like',\n    'pick',\n    'preloaded_multi_mp_sgd_mom_update',\n    'preloaded_multi_mp_sgd_update',\n    'preloaded_multi_sgd_mom_update',\n    'preloaded_multi_sgd_update',\n    'prod',\n    'radians',\n    'rcbrt',\n    'reciprocal',\n    'relu',\n    'repeat',\n    'reset_arrays',\n    'reshape_like',\n    'reverse',\n    'rint',\n    'rmsprop_update',\n    'rmspropalex_update',\n    'round',\n    'rsqrt',\n    'scatter_nd',\n    'sgd_mom_update',\n    'sgd_update',\n    'shape_array',\n    'sigmoid',\n    'sign',\n    'signsgd_update',\n    'signum_update',\n    'sin',\n    'sinh',\n    'size_array',\n    'slice_axis',\n    'slice_like',\n    'smooth_l1',\n    'softmax_cross_entropy',\n    'softmin',\n    'softsign',\n    'sort',\n    'sqrt',\n    'square',\n    'squeeze',\n    'tan',\n    'tanh',\n    'tile',\n    'topk',\n    'trunc',\n    'zeros_like',\n]\n\n# Functions that have to be cast to FP32 only for\n# some values of their parameters\nCONDITIONAL_FP32_FUNCS = [\n    ('LeakyReLU', 'act_type', ['selu']),\n]\n\nLOSS_OUTPUT_FUNCTIONS = [\n    'SoftmaxOutput'\n]\n"
  },
  {
    "path": "python/mxnet/amp/lists/symbol_fp16.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Lists of functions whitelisted/blacklisted for automatic mixed precision in symbol API.\"\"\"\n\nfrom ...runtime import Features\n\n\n# Functions that should be cast to lower precision\nFP16_FUNCS = [\n    '_linalg_gemm',\n    '_linalg_gemm2',\n    '_npi_einsum',\n    '_npi_matmul',\n    'Convolution',\n    'Deconvolution',\n    'FullyConnected',\n    'RNN',\n    ]\n\n# Functions that should not be casted, either because\n# they are irrelevant (not used in the network itself\n# like image transformations or optimizers) or they\n# are dtype neutral (can work in both fp16 and fp32)\nFP16_FP32_FUNCS = [\n    'BatchNorm',\n    'BilinearSampler',\n    'BlockGrad',\n    'Cast',\n    'cast_storage',\n    '_contrib_allclose',\n    '_contrib_arange_like',\n    '_contrib_dynamic_reshape',\n    '_contrib_intgemm_fully_connected',\n    '_contrib_intgemm_maxabsolute',\n    '_contrib_intgemm_prepare_data',\n    '_contrib_intgemm_prepare_weight',\n    '_contrib_intgemm_take_weight',\n    '_contrib_quantized_batch_norm',\n    '_contrib_quantized_batch_norm_relu',\n    '_contrib_quantized_elemwise_mul',\n    '_contrib_quantized_embedding',\n    '_contrib_mrcnn_mask_target',\n    '_contrib_round_ste',\n    '_contrib_sign_ste',\n    'Crop',\n    'Dropout',\n    'Embedding',\n    'Flatten',\n    'GridGenerator',\n    'Pad',\n    'Pooling',\n    'ROIPooling',\n    'Reshape',\n    'SequenceLast',\n    'SequenceMask',\n    'SequenceReverse',\n    'SliceChannel',\n    'SpatialTransformer',\n    'SwapAxis',\n    'UpSampling',\n    '_CachedOp',\n    '_CachedOpThreadSafe',\n    '_CrossDeviceCopy',\n    '_CustomFunction',\n    '_FusedOp',\n    '_FusedOpHelper',\n    '_FusedOpOutHelper',\n    '_NoGradient',\n    '_adabelief_update',\n    '_adamw_update',\n    '_arange',\n    '_cond',\n    '_contrib_AdaptiveAvgPooling2D',\n    '_contrib_BilinearResize2D',\n    '_contrib_bipartite_matching',\n    '_contrib_dequantize',\n    '_contrib_div_sqrt_dim',\n    '_contrib_boolean_mask',\n    '_contrib_getnnz',\n    '_contrib_gradientmultiplier',\n    '_contrib_group_adagrad_update',\n    '_contrib_index_array',\n    '_contrib_index_copy',\n    '_contrib_quadratic',\n    '_contrib_quantize',\n    '_contrib_quantize_asym',\n    '_contrib_quantize_v2',\n    '_contrib_quantized_concat',\n    '_contrib_quantized_conv',\n    '_contrib_quantized_flatten',\n    '_contrib_quantized_fully_connected',\n    '_contrib_quantized_pooling',\n    '_contrib_quantized_elemwise_add',\n    '_contrib_quantized_act',\n    '_contrib_quantized_reshape',\n    '_contrib_quantized_rnn',\n    '_contrib_quantized_transpose',\n    '_npx_quantized_reshape',\n    '_npx_quantized_transpose',\n    '_image_crop',\n    '_linspace',\n    '_contrib_requantize',\n    '_copy',\n    '_copyto',\n    '_cvcopyMakeBorder',\n    '_cvimdecode',\n    '_cvimread',\n    '_cvimresize',\n    '_div_scalar',\n    '_equal_scalar',\n    '_eye',\n    '_foreach',\n    '_while_loop',\n    '_full',\n    '_grad_add',\n    '_greater_scalar',\n    '_greater_equal_scalar',\n    '_histogram',\n    '_hypot_scalar',\n    '_identity_with_attr_like_rhs',\n    '_image_adjust_lighting',\n    '_image_flip_left_right',\n    '_image_flip_top_bottom',\n    '_image_normalize',\n    '_image_random_brightness',\n    '_image_random_color_jitter',\n    '_image_random_contrast',\n    '_image_random_crop',\n    '_image_random_resized_crop',\n    '_image_random_flip_left_right',\n    '_image_random_flip_top_bottom',\n    '_image_random_hue',\n    '_image_random_lighting',\n    '_image_random_saturation',\n    '_image_resize',\n    '_image_to_tensor',\n    '_imdecode',\n    '_lesser_scalar',\n    '_lesser_equal_scalar',\n    '_logical_and_scalar',\n    '_logical_or_scalar',\n    '_logical_xor_scalar',\n    '_maximum_scalar',\n    '_minimum_scalar',\n    '_minus_scalar',\n    '_mod_scalar',\n    '_mp_adabelief_update',\n    '_mp_adamw_update',\n    '_mul_scalar',\n    '_multi_adabelief_update',\n    '_multi_adamw_update',\n    '_multi_lamb_update',\n    '_multi_lans_update',\n    '_multi_mp_adabelief_update',\n    '_multi_mp_adamw_update',\n    '_multi_mp_lamb_update',\n    '_multi_mp_lans_update',\n    '_not_equal_scalar',\n    '_np_reshape',\n    '_npi_absolute',\n    '_npi_add',\n    '_npi_add_scalar',\n    '_npi_advanced_indexing',\n    '_npi_advanced_indexing_multiple',\n    '_npi_all',\n    '_npi_any',\n    '_npi_arange',\n    '_npi_arccosh',\n    '_npi_arcsinh',\n    '_npi_arctan',\n    '_npi_arctan2',\n    '_npi_arctan2_scalar',\n    '_npi_argmax',\n    '_npi_argmin',\n    '_npi_around',\n    '_npi_atleast_1d',\n    '_npi_atleast_2d',\n    '_npi_atleast_3d',\n    '_npi_bernoulli',\n    '_npi_bincount',\n    '_npi_bitwise_and',\n    '_npi_bitwise_and_scalar',\n    '_npi_bitwise_not',\n    '_npi_bitwise_or',\n    '_npi_bitwise_or_scalar',\n    '_npi_bitwise_xor',\n    '_npi_bitwise_xor_scalar',\n    '_npi_bitwise_left_shift',\n    '_npi_bitwise_left_shift_scalar',\n    '_npi_bitwise_right_shift',\n    '_npi_bitwise_right_shift_scalar',\n    '_npi_rbitwise_left_shift_scalar',\n    '_npi_rbitwise_right_shift_scalar',\n    '_npi_blackman',\n    '_npi_boolean_mask_assign_scalar',\n    '_npi_boolean_mask_assign_tensor',\n    '_npi_broadcast_to',\n    '_npi_cbrt',\n    '_npi_ceil',\n    '_npi_choice',\n    '_npi_copy',\n    '_npi_copysign_scalar',\n    '_npi_cos',\n    '_npi_degrees',\n    '_npi_delete',\n    '_npi_diag',\n    '_npi_diag_indices_from',\n    '_npi_diagflat',\n    '_npi_diagonal',\n    '_npi_diff',\n    '_npi_dsplit',\n    '_npi_equal_scalar',\n    '_npi_exponential',\n    '_npi_eye',\n    '_npi_fill_diagonal',\n    '_npi_fix',\n    '_npi_flip',\n    '_npi_floor',\n    '_npi_fmax_scalar',\n    '_npi_fmin_scalar',\n    '_npi_fmod_scalar',\n    '_npi_full',\n    '_npi_full_like',\n    '_npi_gamma',\n    '_npi_greater_equal_scalar',\n    '_npi_greater_scalar',\n    '_npi_gumbel',\n    '_npi_hamming',\n    '_npi_hanning',\n    '_npi_hsplit',\n    '_npi_identity',\n    '_npi_indices',\n    '_npi_insert_scalar',\n    '_npi_insert_slice',\n    '_npi_insert_tensor',\n    '_npi_interp',\n    '_npi_isinf',\n    '_npi_isfinite',\n    '_npi_isnan',\n    '_npi_isneginf',\n    '_npi_isposinf',\n    '_npi_laplace',\n    '_npi_less_equal_scalar',\n    '_npi_less_scalar',\n    '_npi_logistic',\n    '_npi_lcm',\n    '_npi_lcm_scalar',\n    '_npi_gcd',\n    '_npi_gcd_scalar',\n    '_npi_linspace',\n    '_npi_logical_not',\n    '_npi_logical_and_scalar',\n    '_npi_logical_or_scalar',\n    '_npi_logical_xor_scalar',\n    '_npi_logspace',\n    '_npi_max',\n    '_npi_min',\n    '_npi_mod',\n    '_npi_mod_scalar',\n    '_npi_moveaxis',\n    '_npi_multinomial',\n    '_npi_multiply',\n    '_npi_multiply_scalar',\n    '_npi_floor_divide',\n    '_npi_floor_divide_scalar',\n    '_npi_rfloor_divide_scalar',\n    '_npi_nan_to_num',\n    '_npi_negative',\n    '_npi_normal',\n    '_npi_normal_n',\n    '_npi_not_equal_scalar',\n    '_npi_ones',\n    '_npi_pad',\n    '_npi_pareto',\n    '_npi_percentile',\n    '_npi_powerd',\n    '_npi_radians',\n    '_npi_rarctan2_scalar',\n    '_npi_rayleigh',\n    '_npi_rcopysign_scalar',\n    '_npi_repeats',\n    '_npi_rfmod_scalar',\n    '_npi_rint',\n    '_npi_rmod_scalar',\n    '_npi_roll',\n    '_npi_rollaxis',\n    '_npi_rot90',\n    '_npi_rsubtract_scalar',\n    '_npi_rtrue_divide_scalar',\n    '_npi_share_memory',\n    '_npi_sign',\n    '_npi_sin',\n    '_npi_sqrt',\n    '_npi_squeeze',\n    '_npi_subtract',\n    '_npi_subtract_scalar',\n    '_npi_tanh',\n    '_npi_transpose',\n    '_npi_tri',\n    '_npi_tril',\n    '_npi_tril_indices',\n    '_npi_triu',\n    '_npi_true_divide',\n    '_npi_true_divide_scalar',\n    '_npi_trunc',\n    '_npi_uniform',\n    '_npi_uniform_n',\n    '_npi_unique',\n    '_npi_weibull',\n    '_npi_where_lscalar',\n    '_npi_where_rscalar',\n    '_npi_where_scalar2',\n    '_npi_zeros',\n    '_npx_constraint_check',\n    '_npx_nonzero',\n    '_npx_relu',\n    '_npx_reshape',\n    '_npx_sigmoid',\n    '_npx_cond',\n    '_npx_foreach',\n    '_npx_while_loop',\n    '_onehot_encode',\n    '_ones',\n    '_plus_scalar',\n    '_random_exponential',\n    '_random_exponential_like',\n    '_random_gamma',\n    '_random_gamma_like',\n    '_random_binomial',\n    '_random_binomial_like',\n    '_random_generalized_negative_binomial',\n    '_random_generalized_negative_binomial_like',\n    '_random_negative_binomial',\n    '_random_negative_binomial_like',\n    '_random_normal',\n    '_random_normal_like',\n    '_random_poisson',\n    '_random_poisson_like',\n    '_random_randint',\n    '_random_uniform',\n    '_random_uniform_like',\n    '_ravel_multi_index',\n    '_rminus_scalar',\n    '_rmod_scalar',\n    '_rnn_param_concat',\n    '_sample_exponential',\n    '_sample_gamma',\n    '_sample_binomial',\n    '_sample_generalized_negative_binomial',\n    '_sample_categorical',\n    '_sample_multinomial',\n    '_sample_negative_binomial',\n    '_sample_normal',\n    '_sample_poisson',\n    '_sample_uniform',\n    '_sample_unique_zipfian',\n    '_scatter_set_nd',\n    '_set_value',\n    '_shuffle',\n    '_slice_assign',\n    '_slice_assign_scalar',\n    '_sparse_adagrad_update',\n    '_sparse_retain',\n    '_split_v2',\n    '_unravel_index',\n    '_zeros',\n    '_zeros_without_dtype',\n    'abs',\n    'adam_update',\n    'all_finite',\n    'amp_cast',\n    'amp_multicast',\n    'arccosh',\n    'arcsinh',\n    'arctan',\n    'argmax',\n    'argmax_channel',\n    'argmin',\n    'batch_take',\n    'broadcast_axis',\n    'broadcast_like',\n    'broadcast_to',\n    'cbrt',\n    'ceil',\n    'clip',\n    'col2im',\n    'cos',\n    'degrees',\n    'depth_to_space',\n    'diag',\n    'erf',\n    'expand_dims',\n    'fill_element_0index',\n    'fix',\n    'floor',\n    'ftml_update',\n    'ftrl_update',\n    'gather_nd',\n    'hard_sigmoid',\n    'im2col',\n    'lamb_update_phase1',\n    'lamb_update_phase2',\n    'logical_not',\n    'log_sigmoid',\n    'max',\n    'min',\n    'mish',\n    'mp_lamb_update_phase1',\n    'mp_lamb_update_phase2',\n    'mp_nag_mom_update',\n    'mp_sgd_mom_update',\n    'mp_sgd_update',\n    'multi_all_finite',\n    'multi_lars',\n    'multi_mp_sgd_mom_update',\n    'multi_mp_sgd_update',\n    'multi_sgd_mom_update',\n    'multi_sgd_update',\n    'multi_sum_sq',\n    'nag_mom_update',\n    'negative',\n    'one_hot',\n    'ones_like',\n    'pick',\n    'preloaded_multi_mp_sgd_mom_update',\n    'preloaded_multi_mp_sgd_update',\n    'preloaded_multi_sgd_mom_update',\n    'preloaded_multi_sgd_update',\n    'radians',\n    'relu',\n    'repeat',\n    'reset_arrays',\n    'reshape_like',\n    'reverse',\n    'rint',\n    'rmsprop_update',\n    'rmspropalex_update',\n    'round',\n    'scatter_nd',\n    'sgd_mom_update',\n    'sgd_update',\n    'shape_array',\n    'sigmoid',\n    'sign',\n    'signsgd_update',\n    'signum_update',\n    'sin',\n    'size_array',\n    'slice',\n    'slice_axis',\n    'slice_like',\n    'softsign',\n    'sort',\n    'space_to_depth',\n    'sqrt',\n    'squeeze',\n    'take',\n    'tanh',\n    'tile',\n    'transpose',\n    'trunc',\n    'zeros_like',\n    ]\n\n# Functions that have to be cast to FP32 due to possible\n# overflows\nFP32_FUNCS = [\n    'IdentityAttachKLSparseReg',\n    'arccos',\n    'arcsin',\n    'cosh',\n    'erfinv',\n    'sinh',\n    'tan',\n    'arctanh',\n    '_contrib_calibrate_entropy',\n    '_contrib_MultiBoxDetection',\n    '_contrib_MultiBoxPrior',\n    '_contrib_MultiBoxTarget',\n    '_npi_arccos',\n    '_npi_arcsin',\n    '_npi_arctanh',\n    '_npi_cosh',\n    '_npi_sinh',\n    '_npi_tan',\n\n    # Exponents\n    '_npi_exp',\n    '_npi_expm1',\n    '_npi_ldexp',\n    '_npi_ldexp_scalar',\n    '_npi_logaddexp',\n    '_npi_logaddexp_scalar',\n    '_npi_log',\n    '_npi_log10',\n    '_npi_log1p',\n    '_npi_log2',\n    '_npi_rldexp_scalar',\n    'exp',\n    'expm1',\n    'log',\n    'log10',\n    'log2',\n    'log1p',\n\n    # Powers\n    'broadcast_power',\n    'square',\n    'reciprocal',\n    '_rdiv_scalar',\n    'rsqrt',\n    'rcbrt',\n    '_power',\n    '_power_scalar',\n    '_rpower_scalar',\n    '_square_sum',\n    '_contrib_hawkesll',\n    '_npi_power',\n    '_npi_power_scalar',\n    '_npi_reciprocal',\n    '_npi_rpower_scalar',\n    '_npi_square',\n\n    # Reductions\n    '_npi_average',\n    '_npi_cumsum',\n    '_npi_mean',\n    '_npi_polyval',\n    '_npi_prod',\n    '_npi_std',\n    '_npi_sum',\n    '_npi_trace',\n    '_npi_var',\n    'sum',\n    'nansum',\n    'prod',\n    'nanprod',\n    'mean',\n    'norm',\n    'softmin',\n    'khatri_rao',\n    'moments',\n\n    # Misc\n    '_npi_cholesky',\n    '_npi_eig',\n    '_npi_eigh',\n    '_npi_eigvals',\n    '_npi_eigvalsh',\n    '_npi_lstsq',\n    '_npi_matrix_rank',\n    '_npi_matrix_rank_none_tol',\n    '_npi_norm',\n    '_npi_pinv',\n    '_npi_pinv_scalar_rcond',\n    '_npi_qr',\n    '_npi_solve',\n    '_npi_svd',\n    '_npi_tensorinv',\n    '_npi_tensorsolve',\n    'digamma',\n    'gamma',\n    'gammaln',\n    '_linalg_gelqf',\n    '_linalg_potrf',\n    '_linalg_potri',\n    '_linalg_sumlogdiag',\n    '_linalg_syevd',\n    '_linalg_syrk',\n    '_linalg_trmm',\n    '_linalg_trsm',\n    '_linalg_makediag',\n    '_linalg_extractdiag',\n    '_linalg_maketrian',\n    '_linalg_extracttrian',\n    '_linalg_inverse',\n    '_linalg_det',\n    '_linalg_slogdet',\n    '_NDArray',\n    '_Native',\n    '_contrib_count_sketch',\n    '_contrib_SyncBatchNorm',\n    '_contrib_fft',\n    'argsort',\n    'topk',\n\n    # Neural network\n    'SoftmaxOutput',\n    'softmax',\n    'log_softmax',\n    'masked_softmax',\n    'masked_log_softmax',\n    'InstanceNorm',\n    'LayerNorm',\n    'GroupNorm',\n    'L2Normalization',\n    'LRN',\n    'SoftmaxActivation',\n    'LinearRegressionOutput',\n    'LogisticRegressionOutput',\n    'MAERegressionOutput',\n    'SVMOutput',\n    'softmax_cross_entropy',\n    'smooth_l1',\n    'MakeLoss',\n    'make_loss',\n    'Custom',\n    'CTCLoss',\n    '_npx_deformable_convolution',\n    '_npx_modulated_deformable_convolution',\n    '_contrib_DeformablePSROIPooling',\n    '_contrib_sldwin_atten_score',\n    '_contrib_sldwin_atten_mask_like',\n    '_contrib_sldwin_atten_context',\n    ]\n\nif Features().is_enabled('ONEDNN'):\n    FP32_FUNCS.extend([\n        '_sg_onednn_conv',\n        '_sg_onednn_fully_connected',\n        '_sg_onednn_selfatt_qk',\n        '_sg_onednn_selfatt_qk_split',\n        '_sg_onednn_selfatt_valatt',\n        '_sg_onednn_batch_dot',\n        '_sg_onednn_batch_norm',\n        '_sg_pow_mul_scalar'\n    ])\n\n# Functions that have to be cast to FP32 only for\n# some values of their parameters\nCONDITIONAL_FP32_FUNCS = [\n    ('Activation', 'act_type', ['softrelu']),\n    ('LeakyReLU', 'act_type', ['elu', 'selu']),\n    ]\n\n# Functions with multiple inputs, that need the same\n# type of all their inputs\nWIDEST_TYPE_CASTS = [\n    '_equal',\n    '_greater',\n    '_greater_equal',\n    '_hypot',\n    '_lesser',\n    '_lesser_equal',\n    '_logical_and',\n    '_logical_or',\n    '_logical_xor',\n    '_maximum',\n    '_minimum',\n    '_mod',\n    '_not_equal',\n    '_npi_column_stack',\n    '_npi_copysign',\n    '_npi_cross',\n    '_npi_dot',\n    '_npi_ediff1d',\n    '_npi_equal',\n    '_npi_fmax',\n    '_npi_fmin',\n    '_npi_fmod',\n    '_npi_greater',\n    '_npi_greater_equal',\n    '_npi_hypot',\n    '_npi_kron',\n    '_npi_less',\n    '_npi_less_equal',\n    '_npi_logical_and',\n    '_npi_logical_or',\n    '_npi_logical_xor',\n    '_npi_not_equal',\n    '_npi_dstack',\n    '_npi_hstack',\n    '_npi_tensordot',\n    '_npi_tensordot_int_axes',\n    '_npi_vstack',\n    '_npi_where',\n    '_npx_index_add',\n    '_npx_index_update',\n    'Concat',\n    '_contrib_RROIAlign',\n    'Correlation',\n    'add_n',\n    'batch_dot',\n    'broadcast_add',\n    'broadcast_div',\n    'broadcast_equal',\n    'broadcast_greater',\n    'broadcast_greater_equal',\n    'broadcast_hypot',\n    'broadcast_lesser',\n    'broadcast_lesser_equal',\n    'broadcast_logical_and',\n    'broadcast_logical_or',\n    'broadcast_logical_xor',\n    'broadcast_maximum',\n    'broadcast_minimum',\n    'broadcast_mod',\n    'broadcast_mul',\n    'broadcast_not_equal',\n    'broadcast_sub',\n    'dot',\n    'elemwise_add',\n    'elemwise_div',\n    'elemwise_mul',\n    'elemwise_sub',\n    'stack',\n    '_contrib_MultiProposal',\n    '_contrib_PSROIPooling',\n    '_contrib_Proposal',\n    '_contrib_ROIAlign',\n    '_contrib_box_decode',\n    '_contrib_box_encode',\n    '_contrib_box_iou',\n    '_contrib_box_nms',\n    '_contrib_dgl_adjacency',\n    '_contrib_dgl_csr_neighbor_non_uniform_sample',\n    '_contrib_dgl_csr_neighbor_uniform_sample',\n    '_contrib_dgl_graph_compact',\n    '_contrib_dgl_subgraph',\n    '_contrib_edge_id',\n    '_contrib_interleaved_matmul_encdec_qk',\n    '_contrib_interleaved_matmul_encdec_valatt',\n    '_contrib_interleaved_matmul_selfatt_qk',\n    '_contrib_interleaved_matmul_selfatt_valatt',\n    'where',\n\n    '_random_pdf_gamma',\n    '_random_pdf_exponential',\n    '_random_pdf_uniform',\n    '_random_pdf_negative_binomial',\n    '_random_pdf_generalized_negative_binomial',\n    '_random_pdf_dirichlet',\n    '_random_pdf_normal',\n    '_random_pdf_poisson',\n    ]\n\nLOSS_OUTPUT_FUNCTIONS = [\n    'SoftmaxOutput',\n    'LinearRegressionOutput',\n    'LogisticRegressionOutput',\n    'MAERegressionOutput',\n    ]\n"
  },
  {
    "path": "python/mxnet/amp/loss_scaler.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Dynamic loss scaler for AMP.\"\"\"\nimport logging\n\nfrom .. import autograd as ag\nfrom .. import ndarray\nfrom ..util import is_np_array\n\nclass LossScaler(object):\n    \"\"\"Dynamic loss scaler for AMP.\n\n    Properties\n    ----------\n    loss_scale : float\n        The current loss scale\n    \"\"\"\n    def __init__(self):\n        self._loss_scale = 2.**16\n        self._next_loss_scale = self._loss_scale\n        self._max_loss_scale = 2.**24\n        self._scale_seq_len = 2000\n        self._unskipped = 0\n\n    @property\n    def loss_scale(self):\n        return self._loss_scale\n\n    def has_overflow(self, params):\n        \"\"\"Check gradients for overflow.\"\"\"\n        if is_np_array():\n            all_finite_f = ndarray.numpy._internal.multi_all_finite\n            ones_f = lambda ctx: ndarray.numpy.ones((1,), device=ctx)\n        else:\n            all_finite_f = ndarray.multi_all_finite\n            ones_f = lambda ctx: ndarray.ones((1,), ctx=ctx)\n        with ag.pause():\n            chunk_size = 200\n            valid_params = [p._grad[0] for p in params if p._grad is not None]\n            gpu_output = ones_f(valid_params[0].context)\n            nb_params = len(valid_params)\n            for idx in range(0, nb_params, chunk_size):\n                all_finite_f(*valid_params[idx:idx+chunk_size],\n                             num_arrays=len(valid_params[idx:idx+chunk_size]),\n                             init_output=False, out=gpu_output)\n        has_overflow = not bool(gpu_output.asnumpy())\n        self._loss_scale = self._next_loss_scale\n        if has_overflow:\n            self._next_loss_scale = self._loss_scale / 2.\n            self._unskipped = 0\n            logging.info(\"AMP: decreasing loss scale to %f\", self._next_loss_scale)\n        else:\n            self._unskipped += 1\n        if self._unskipped == self._scale_seq_len:\n            self._unskipped = 0\n            self._next_loss_scale = min(self._max_loss_scale, self._loss_scale * 2.)\n            logging.info(\"AMP: increasing loss scale to %f\", self._next_loss_scale)\n        return has_overflow\n"
  },
  {
    "path": "python/mxnet/api.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Functions defined in MXNet.\nAcknowledgement: This file originates from incubator-tvm\"\"\"\n\nfrom ._ffi.function import _init_api\n\n_init_api(\"mxnet.api\")\n"
  },
  {
    "path": "python/mxnet/attribute.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Attribute scoping support for symbolic API.\"\"\"\nimport contextvars\nfrom collections import defaultdict\n\nfrom .base import string_types\n\nclass AttrScope:\n    \"\"\"Attribute manager for scoping.\n\n    User can also inherit this object to change naming behavior.\n\n    Parameters\n    ----------\n    kwargs\n        The attributes to set for all symbol creations in the scope.\n    \"\"\"\n    _subgraph_names = defaultdict(int)\n\n    def __init__(self, **kwargs):\n        self._old_scope = None\n        for value in kwargs.values():\n            if not isinstance(value, string_types):\n                raise ValueError(\"Attributes need to be string\")\n        self._attr = kwargs\n\n    def get(self, attr):\n        \"\"\"\n        Get the attribute dict given the attribute set by the symbol.\n\n        Parameters\n        ----------\n        attr : dict of string to string\n            The attribute passed in by user during symbol creation.\n\n        Returns\n        -------\n        attr : dict of string to string\n            Updated attributes to add other scope related attributes.\n        \"\"\"\n        if self._attr:\n            ret = self._attr.copy()\n            if attr:\n                ret.update(attr)\n            return ret\n        else:\n            return attr if attr else {}\n\n    def __enter__(self):  # pylint: disable=protected-access\n        attr = _current.get()._attr.copy()\n        attr.update(self._attr)\n        self._attr = attr\n        # Token can't be pickled and Token.old_value is Token.MISSING if _current.get() uses default value\n        self._old_scope = _current.get()\n        _current.set(self)\n        return self\n\n    def __exit__(self, ptype, value, trace):\n        assert self._old_scope\n        _current.set(self._old_scope)\n\n\n_current = contextvars.ContextVar('namemanager', default=AttrScope())\n\n\ndef current():\n    \"\"\"Returns the current name manager.\"\"\"\n    return _current.get()\n"
  },
  {
    "path": "python/mxnet/autograd.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Autograd for NDArray.\"\"\"\n\nfrom array import array\nfrom threading import Lock\nimport traceback\nimport ctypes\nfrom ctypes import c_int, c_void_p, CFUNCTYPE, POINTER, cast\nfrom .base import _LIB, check_call, string_types, mx_uint\nfrom .base import NDArrayHandle, c_array, c_handle_array, c_array_buf, MXCallbackList, SymbolHandle\nfrom .ndarray import NDArray, _ndarray_cls\nfrom .ndarray import _GRAD_REQ_MAP\nfrom .symbol import Symbol\nfrom .util import is_np_array\n\n\ndef set_recording(is_recording): #pylint: disable=redefined-outer-name\n    \"\"\"Set status to recording/not recording. When recording, graph will be constructed\n    for gradient computation.\n\n    Parameters\n    ----------\n    is_recording: bool\n\n    Returns\n    -------\n    previous state before this set.\n    \"\"\"\n    prev = ctypes.c_int()\n    check_call(_LIB.MXAutogradSetIsRecording(\n        ctypes.c_int(is_recording), ctypes.byref(prev)))\n    return bool(prev.value)\n\ndef set_training(train_mode): #pylint: disable=redefined-outer-name\n    \"\"\"Set status to training/predicting. This affects ctx.is_train in operator\n    running context. For example, Dropout will drop inputs randomly when\n    train_mode=True while simply passing through if train_mode=False.\n\n    Parameters\n    ----------\n    train_mode: bool\n\n    Returns\n    -------\n    previous state before this set.\n    \"\"\"\n    prev = ctypes.c_int()\n    check_call(_LIB.MXAutogradSetIsTraining(\n        ctypes.c_int(train_mode), ctypes.byref(prev)))\n    return bool(prev.value)\n\ndef is_recording():\n    \"\"\"Get status on recording/not recording.\n\n    Returns\n    -------\n    Current state of recording.\n    \"\"\"\n    curr = ctypes.c_bool()\n    check_call(_LIB.MXAutogradIsRecording(ctypes.byref(curr)))\n    return curr.value\n\ndef is_training():\n    \"\"\"Get status on training/predicting.\n\n    Returns\n    -------\n    Current state of training/predicting.\n    \"\"\"\n    curr = ctypes.c_bool()\n    check_call(_LIB.MXAutogradIsTraining(ctypes.byref(curr)))\n    return curr.value\n\n\nclass _RecordingStateScope(object):\n    \"\"\"Scope for managing training state.\n\n    Example::\n\n        with _RecordingStateScope(True, True):\n            y = model(x)\n            backward([y])\n\n    \"\"\"\n    def __init__(self, is_record, train_mode): #pylint: disable=redefined-outer-name\n        self._enter_is_record = is_record\n        self._enter_train_mode = train_mode\n        self._prev_is_record = None\n        self._prev_train_mode = None\n\n    def __enter__(self):\n        if self._enter_is_record is not None:\n            self._prev_is_record = set_recording(self._enter_is_record)\n        if self._enter_train_mode is not None:\n            self._prev_train_mode = set_training(self._enter_train_mode)\n\n    def __exit__(self, ptype, value, trace):\n        if self._enter_is_record is not None and self._prev_is_record != self._enter_is_record:\n            set_recording(self._prev_is_record)\n        if self._enter_train_mode is not None and self._prev_train_mode != self._enter_train_mode:\n            set_training(self._prev_train_mode)\n\n\ndef record(train_mode=True): #pylint: disable=redefined-outer-name\n    \"\"\"Returns an autograd recording scope context to be used in 'with' statement\n    and captures code that needs gradients to be calculated.\n\n    .. note:: When forwarding with train_mode=False, the corresponding backward\n              should also use train_mode=False, otherwise gradient is undefined.\n\n    Example::\n\n        with autograd.record():\n            y = model(x)\n            backward([y])\n        metric.update(...)\n        optim.step(...)\n\n    Parameters\n    ----------\n    train_mode: bool, default True\n        Whether the forward pass is in training or predicting mode. This controls the behavior\n        of some layers such as Dropout, BatchNorm.\n    \"\"\"\n    return _RecordingStateScope(True, train_mode)\n\n\ndef pause(train_mode=False): #pylint: disable=redefined-outer-name\n    \"\"\"Returns a scope context to be used in 'with' statement for codes that do not need\n    gradients to be calculated.\n\n    Example::\n\n        with autograd.record():\n            y = model(x)\n            backward([y])\n            with autograd.pause():\n                # testing, IO, gradient updates...\n\n    Parameters\n    ----------\n    train_mode: bool, default False\n        Whether to do forward for training or predicting.\n    \"\"\"\n    return _RecordingStateScope(False, train_mode)\n\n\ndef train_mode():\n    \"\"\"Returns a scope context to be used in 'with' statement\n    in which forward pass behavior is set to training mode,\n    without changing the recording states.\n\n    Example::\n\n        y = model(x)\n        with autograd.train_mode():\n            y = dropout(y)\n\n    \"\"\"\n    return _RecordingStateScope(None, True)\n\n\ndef predict_mode():\n    \"\"\"Returns a scope context to be used in 'with' statement\n    in which forward pass behavior is set to inference mode,\n    without changing the recording states.\n\n    Example::\n\n        with autograd.record():\n            y = model(x)\n            with autograd.predict_mode():\n                y = sampling(y)\n            backward([y])\n    \"\"\"\n    return _RecordingStateScope(None, False)\n\n\ndef mark_variables(variables, gradients, grad_reqs='write'):\n    \"\"\"Mark NDArrays as variables to compute gradient for autograd.\n\n    This is equivalent to the function .attach_grad() in a variable, but with this\n    call we can set the gradient to any value.\n\n    Parameters\n    ----------\n    variables: NDArray or list of NDArray\n    gradients: NDArray or list of NDArray\n    grad_reqs: str or list of str\n    \"\"\"\n    if isinstance(variables, NDArray):\n        assert isinstance(gradients, NDArray)\n        variables = [variables]\n        gradients = [gradients]\n\n    if isinstance(grad_reqs, string_types):\n        grad_reqs = [_GRAD_REQ_MAP[grad_reqs]]*len(variables)\n    else:\n        grad_reqs = [_GRAD_REQ_MAP[i] for i in grad_reqs]\n\n    check_call(_LIB.MXAutogradMarkVariables(\n        len(variables),\n        c_handle_array(variables),\n        c_array_buf(mx_uint, array('I', grad_reqs)),\n        c_handle_array(gradients)))\n\n\ndef _parse_head(heads, head_grads):\n    \"\"\"parse head gradient for backward and grad.\"\"\"\n    if isinstance(heads, NDArray):\n        heads = [heads]\n    if isinstance(head_grads, NDArray):\n        head_grads = [head_grads]\n\n    head_handles = c_handle_array(heads)\n\n    if head_grads is None:\n        hgrad_handles = ctypes.c_void_p(0)\n    else:\n        msg = \"heads and head_grads must be lists of the same length: {} vs. {}\"\n        assert len(heads) == len(head_grads), msg.format(len(heads), len(head_grads))\n        hgrad_handles = c_array(NDArrayHandle,\n                                [i.handle if i is not None else NDArrayHandle(0)\n                                 for i in head_grads])\n    return head_handles, hgrad_handles\n\n\ndef backward(heads, head_grads=None, retain_graph=False, train_mode=True): #pylint: disable=redefined-outer-name\n    \"\"\"Compute the gradients of heads w.r.t previously marked variables.\n\n    Parameters\n    ----------\n    heads: NDArray or list of NDArray\n        Output NDArray(s)\n    head_grads: NDArray or list of NDArray or None\n        Gradients with respect to heads.\n    train_mode: bool, optional\n        Whether to do backward for training or predicting.\n    \"\"\"\n    head_handles, hgrad_handles = _parse_head(heads, head_grads)\n\n    check_call(_LIB.MXAutogradBackwardEx(\n        len(head_handles),\n        head_handles,\n        hgrad_handles,\n        0,\n        ctypes.c_void_p(0),\n        ctypes.c_int(retain_graph),\n        ctypes.c_int(0),\n        ctypes.c_int(train_mode),\n        ctypes.c_void_p(0),\n        ctypes.c_void_p(0)))\n\n\ndef grad(heads, variables, head_grads=None, retain_graph=None, create_graph=False,\n         train_mode=True):  #pylint: disable=redefined-outer-name\n    \"\"\"Compute the gradients of heads w.r.t variables. Gradients will be\n    returned as new NDArrays instead of stored into `variable.grad`.\n    Supports recording gradient graph for computing higher order gradients.\n\n    .. note::\n\n      Currently only a very limited set of operators support higher order \\\n      gradients.\n\n    Parameters\n    ----------\n    heads: NDArray or list of NDArray\n        Output NDArray(s)\n    variables: NDArray or list of NDArray\n        Input variables to compute gradients for.\n    head_grads: NDArray or list of NDArray or None\n        Gradients with respect to heads.\n    retain_graph: bool\n        Whether to keep computation graph to differentiate again, instead\n        of clearing history and release memory. Defaults to the same value\n        as create_graph.\n    create_graph: bool\n        Whether to record gradient graph for computing higher order\n    train_mode: bool, optional\n        Whether to do backward for training or prediction.\n\n    Returns\n    -------\n    NDArray or list of NDArray:\n        Gradients with respect to variables.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((1,))\n    >>> x.attach_grad()\n    >>> with mx.autograd.record():\n    ...     z = mx.nd.elemwise_add(mx.nd.exp(x), x)\n    >>> dx = mx.autograd.grad(z, [x], create_graph=True)\n    >>> print(dx)\n    [\n    [ 3.71828175]\n    <NDArray 1 @cpu(0)>]\n    \"\"\"\n    head_handles, hgrad_handles = _parse_head(heads, head_grads)\n\n    if isinstance(variables, NDArray):\n        variables = [variables]\n    else:\n        assert len(variables), \"variables cannot be an empty list.\"\n    var_handles = c_handle_array(variables)\n\n    retain_graph = retain_graph if retain_graph is not None else create_graph\n    grad_vars = ctypes.POINTER(NDArrayHandle)()\n    grad_stypes = ctypes.POINTER(ctypes.c_int)()\n\n    check_call(_LIB.MXAutogradBackwardEx(\n        len(head_handles),\n        head_handles,\n        hgrad_handles,\n        len(var_handles),\n        var_handles,\n        ctypes.c_int(retain_graph),\n        ctypes.c_int(create_graph),\n        ctypes.c_int(train_mode),\n        ctypes.byref(grad_vars),\n        ctypes.byref(grad_stypes)))\n\n    ret = [_ndarray_cls(ctypes.cast(grad_vars[i], NDArrayHandle),\n                        stype=grad_stypes[i])\n           for i in range(len(var_handles))]\n    if isinstance(variables, NDArray):\n        return ret[0]\n    return ret\n\n\ndef get_symbol(x):\n    \"\"\"Retrieve recorded computation history as `Symbol`.\n\n    Parameters\n    ----------\n    x : NDArray\n        Array representing the head of computation graph.\n\n    Returns\n    -------\n    Symbol\n        The retrieved Symbol.\n    \"\"\"\n    assert isinstance(x, NDArray), \\\n       f\"get_symbol: Invalid argument type, expecting {NDArray}, got {type(x)}\"\n    hdl = SymbolHandle()\n    check_call(_LIB.MXAutogradGetSymbol(x.handle, ctypes.byref(hdl)))\n    return Symbol(hdl)\n\n\nclass Function(object):\n    \"\"\"Customize differentiation in autograd.\n\n    If you don't want to use the gradients computed by the default\n    chain-rule, you can use Function to customize differentiation for\n    computation. You define your computation in\n    the forward method and provide the customized differentiation\n    in the backward method. During gradient computation, autograd will\n    use the user-defined backward function instead of the default chain-rule.\n    You can also cast to numpy array and back for some operations in\n    forward and backward.\n\n    For example, a stable sigmoid function can be defined as::\n\n        class sigmoid(mx.autograd.Function):\n            def forward(self, x):\n                y = 1 / (1 + mx.nd.exp(-x))\n                self.save_for_backward(y)\n                return y\n\n            def backward(self, dy):\n                # backward takes as many inputs as forward's return value,\n                # and returns as many NDArrays as forward's arguments.\n                y, = self.saved_tensors\n                return dy * y * (1-y)\n\n    Then, the function can be used in the following way::\n\n        func = sigmoid()\n        x = mx.nd.random.uniform(shape=(10,))\n        x.attach_grad()\n\n        with mx.autograd.record():\n            m = func(x)\n            m.backward()\n        dx = x.grad.asnumpy()\n\n    \"\"\"\n    _bwd_functype = CFUNCTYPE(c_int, c_int, c_int, POINTER(c_void_p),\n                              POINTER(c_int), c_int, c_void_p)\n    _del_functype = CFUNCTYPE(c_int, c_void_p)\n    class _Registry(object):\n        \"\"\"CustomOp registry.\"\"\"\n        def __init__(self):\n            self.ref_holder = {}\n            self.counter = 0\n            self.lock = Lock()\n\n        def inc(self):\n            \"\"\"Get index for new entry.\"\"\"\n            self.lock.acquire()\n            cur = self.counter\n            self.counter += 1\n            self.lock.release()\n            return cur\n\n    _registry = _Registry()\n\n    def __init__(self):\n        self._used = False\n        self.saved_tensors = ()\n\n    def save_for_backward(self, *args):\n        self.saved_tensors = args\n\n    def __call__(self, *inputs):\n        assert not self._used, \\\n            \"Each Function instance can only be called once. \"\\\n            \"Please create another instance.\"\n        self._used = True\n\n        prev_recording = set_recording(False)\n        outputs = self.forward(*inputs)\n        set_recording(prev_recording)\n\n        if not prev_recording:\n            return outputs\n\n        ret_outputs = outputs\n        if isinstance(outputs, NDArray):\n            outputs = (outputs,)\n\n        key = Function._registry.inc()\n        if is_np_array():\n            from .numpy import ndarray\n            array_cls = ndarray\n        else:\n            array_cls = NDArray\n\n        def backward_entry(num_ograds, num_igrads, ptrs, reqs, is_train, _):\n            \"\"\"entry point for backward.\"\"\"\n            # pylint: disable=W0613\n            try:\n                output_grads = [array_cls(ctypes.cast(i, NDArrayHandle), writable=False) \\\n                                for i in ptrs[:num_ograds]]\n                input_grads = [array_cls(ctypes.cast(i, NDArrayHandle), writable=True) \\\n                               for i in ptrs[num_ograds:num_ograds+num_igrads]]\n                reqs = [reqs[i] for i in range(num_igrads)]\n                rets = self.backward(*output_grads)\n                if isinstance(rets, array_cls):\n                    rets = (rets,)\n                assert len(rets) == len(input_grads), \\\n                    f\"{self.__class__.name}.backward must return exactly the same number \" \\\n                    \"of NDArrays as the number of NDArrays arguments to forward.\" \\\n                    f\"Expecting {len(input_grads)} got {len(rets)}\"\n                for igrad, ret, req in zip(input_grads, rets, reqs):\n                    assert isinstance(ret, array_cls), \\\n                        f\"autograd.Function.backward must return NDArrays, not {type(ret)}\"\n                    if req == 0:  # null\n                        return True\n                    elif req in (1, 2):  # write or inplace\n                        igrad[:] = ret\n                    elif req == 'add':\n                        igrad[:] += ret\n            except Exception:  # pylint: disable=broad-except\n                print(f'Error in Function.backward: {traceback.format_exc()}')\n                return False\n            return True\n\n        def delete_entry(_):\n            \"\"\"C Callback for CustomFunction::delete\"\"\"\n            try:\n                del Function._registry.ref_holder[key]\n            except Exception:  # pylint: disable=broad-except\n                print(f'Error in autograd.Function.delete: {traceback.format_exc()}')\n                return False\n            return True\n\n        callbacks = [Function._bwd_functype(backward_entry),\n                     Function._del_functype(delete_entry)]\n        callbacks = [cast(i, CFUNCTYPE(c_int)) for i in callbacks]\n        context = MXCallbackList(c_int(len(callbacks)),\n                                 cast(c_array(CFUNCTYPE(c_int), callbacks),\n                                      POINTER(CFUNCTYPE(c_int))),\n                                 cast(c_array(c_void_p, [None]*len(callbacks)),\n                                      POINTER(c_void_p)))\n        Function._registry.ref_holder[key] = context\n        check_call(_LIB.MXCustomFunctionRecord(\n            c_int(len(inputs)),\n            c_handle_array(inputs),\n            c_int(len(outputs)),\n            c_handle_array(outputs),\n            ctypes.byref(context)))\n\n        return ret_outputs\n\n    def forward(self, *inputs):\n        \"\"\"Forward computation.\"\"\"\n        raise NotImplementedError\n\n    def backward(self, *output_grads):\n        \"\"\"Backward computation.\n\n        Takes as many inputs as forward's outputs,\n        and returns as many NDArrays as forward's inputs.\n        \"\"\"\n        raise NotImplementedError\n"
  },
  {
    "path": "python/mxnet/base.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=invalid-name, no-member, trailing-comma-tuple, bad-mcs-classmethod-argument, unnecessary-pass, too-many-lines, wrong-import-position\n\"\"\"ctypes library of mxnet and helper functions.\"\"\"\n\nimport re\nimport atexit\nimport ctypes\nimport os\nimport sys\nimport inspect\nimport platform\nimport numpy as _np\n\nfrom . import libinfo\n\n__all__ = ['MXNetError']\n#----------------------------\n# library loading\n#----------------------------\n\n# pylint: disable=pointless-statement\ntry:\n    basestring\n    long\nexcept NameError:\n    basestring = str\n    long = int\n# pylint: enable=pointless-statement\n\ninteger_types = (int, long, _np.int32, _np.int64)\nnumeric_types = (float, int, long, _np.generic)\nstring_types = basestring,\nerror_types = {}\n\n# Upper bound of uint64\n_MAX_VALUE_64_BIT_SIGNED_ = 9_223_372_036_854_775_807\n# Upper bound of int64\n_MAX_VALUE_64_BIT_UNSIGNED_ = 18_446_744_073_709_551_615\n# Upper bound of float32\n_MAX_VALUE_FLOAT32_REPRESENT_ = 16_777_216\n\n# this function is needed for python3\n# to convert ctypes.char_p .value back to python str\npy_str = lambda x: x.decode('utf-8')\n\n\ndef data_dir_default():\n    \"\"\"\n\n    :return: default data directory depending on the platform and environment variables\n    \"\"\"\n    system = platform.system()\n    if system == 'Windows':\n        return os.path.join(os.environ.get('APPDATA'), 'mxnet')\n    else:\n        return os.path.join(os.path.expanduser(\"~\"), '.mxnet')\n\n\ndef data_dir():\n    \"\"\"\n\n    :return: data directory in the filesystem for storage, for example when downloading models\n    \"\"\"\n    return os.getenv('MXNET_HOME', data_dir_default())\n\n\nclass _NullType(object):\n    \"\"\"Placeholder for arguments\"\"\"\n    def __repr__(self):\n        return '_Null'\n\n\n_Null = _NullType()\n\n\nclass MXNetError(RuntimeError):\n    \"\"\"Default error thrown by MXNet functions.\n\n    MXNetError will be raised if you do not give any error type specification,\n    \"\"\"\n\ndef register_error(func_name=None, cls=None):\n    \"\"\"Register an error class so it can be recognized by the ffi error handler.\n\n    Parameters\n    ----------\n    func_name : str or function or class\n        The name of the error function.\n\n    cls : function\n        The function to create the class\n\n    Returns\n    -------\n    fregister : function\n        Register function if f is not specified.\n\n    Examples\n    --------\n    .. code-block:: python\n\n      @mxnet.error.register_error\n      class MyError(RuntimeError):\n          pass\n\n      err_inst = mxnet.error.create_ffi_error(\"MyError: xyz\")\n      assert isinstance(err_inst, MyError)\n    \"\"\"\n    if callable(func_name):\n        cls = func_name\n        func_name = cls.__name__\n\n    def register(mycls):\n        \"\"\"internal register function\"\"\"\n        err_name = func_name if isinstance(func_name, str) else mycls.__name__\n        error_types[err_name] = mycls\n        return mycls\n    if cls is None:\n        return register\n    return register(cls)\n\n\ndef _valid_error_name(name):\n    \"\"\"Check whether name is a valid error name.\"\"\"\n    return all(x.isalnum() or x in \"_.\" for x in name)\n\n\ndef _find_error_type(line):\n    \"\"\"Find the error name given the first line of the error message.\n\n    Parameters\n    ----------\n    line : str\n        The first line of error message.\n\n    Returns\n    -------\n    name : str The error name\n    \"\"\"\n    end_pos = line.find(\":\")\n    if end_pos == -1:\n        return None\n    err_name = line[:end_pos]\n    if _valid_error_name(err_name):\n        return err_name\n    return None\n\n\ndef c2pyerror(err_msg):\n    \"\"\"Translate C API error message to python style.\n\n    Parameters\n    ----------\n    err_msg : str\n        The error message.\n\n    Returns\n    -------\n    new_msg : str\n        Translated message.\n\n    err_type : str\n        Detected error type.\n    \"\"\"\n    arr = err_msg.split(\"\\n\")\n    if arr[-1] == \"\":\n        arr.pop()\n    err_type = _find_error_type(arr[0])\n    trace_mode = False\n    stack_trace = []\n    message = []\n    for line in arr:\n        if trace_mode:\n            if line.startswith(\"  \"):\n                stack_trace.append(line)\n            else:\n                trace_mode = False\n        if not trace_mode:\n            if line.startswith(\"Stack trace\"):\n                trace_mode = True\n            else:\n                message.append(line)\n    out_msg = \"\"\n    if stack_trace:\n        out_msg += \"Traceback (most recent call last):\\n\"\n        out_msg += \"\\n\".join(reversed(stack_trace)) + \"\\n\"\n    out_msg += \"\\n\".join(message)\n    return out_msg, err_type\n\n@register_error\nclass NotImplementedForSymbol(MXNetError):\n    \"\"\"Error: Not implemented for symbol\"\"\"\n    def __init__(self, function, alias, *args):\n        super(NotImplementedForSymbol, self).__init__()\n        self.function = function.__name__\n        self.alias = alias\n        self.args = [str(type(a)) for a in args]\n\n    def __str__(self):\n        msg = 'Function {}'.format(self.function)\n        if self.alias:\n            msg += ' (namely operator \"{}\")'.format(self.alias)\n        if self.args:\n            msg += ' with arguments ({})'.format(', '.join(self.args))\n        msg += ' is not implemented for Symbol and only available in NDArray.'\n        return msg\n\n\ndef get_last_ffi_error():\n    \"\"\"Create error object given result of MXGetLastError.\n\n    Returns\n    -------\n    err : object\n        The error object based on the err_msg\n    \"\"\"\n    c_err_msg = py_str(_LIB.MXGetLastError())\n    py_err_msg, err_type = c2pyerror(c_err_msg)\n    if err_type is not None and err_type.startswith(\"mxnet.error.\"):\n        err_type = err_type[10:]\n    return error_types.get(err_type, MXNetError)(py_err_msg)\n\n\ndef check_call(ret):\n    \"\"\"Check the return value of C API call.\n\n    This function will raise an exception when an error occurs.\n    Wrap every API call with this function.\n\n    Parameters\n    ----------\n    ret : int\n        return value from API calls.\n    \"\"\"\n    if ret != 0:\n        raise get_last_ffi_error()\n\n\nclass NotSupportedForSparseNDArray(MXNetError):\n    \"\"\"Error: Not supported for SparseNDArray\"\"\"\n    def __init__(self, function, alias, *args):\n        super(NotSupportedForSparseNDArray, self).__init__()\n        self.function = function.__name__\n        self.alias = alias\n        self.args = [str(type(a)) for a in args]\n\n    def __str__(self):\n        msg = 'Function {}'.format(self.function)\n        if self.alias:\n            msg += ' (namely operator \"{}\")'.format(self.alias)\n        if self.args:\n            msg += ' with arguments ({})'.format(', '.join(self.args))\n        msg += ' is not supported for SparseNDArray and only available in NDArray.'\n        return msg\n\n\nclass MXCallbackList(ctypes.Structure):\n    \"\"\"Structure that holds Callback information. Passed to CustomOpProp.\"\"\"\n    _fields_ = [\n        ('num_callbacks', ctypes.c_int),\n        ('callbacks', ctypes.POINTER(ctypes.CFUNCTYPE(ctypes.c_int))),\n        ('contexts', ctypes.POINTER(ctypes.c_void_p))\n        ]\n\n\n# pylint: disable=line-too-long\ndef _load_lib():\n    \"\"\"Load library by searching possible path.\"\"\"\n    lib_path = libinfo.find_lib_path()\n    try:\n        if sys.version_info >= (3, 8) and os.name == \"nt\":\n            # use LOAD_WITH_ALTERED_SEARCH_PATH, For simplicity, let's just fill the numbers.\n            # pylint: disable=E1123\n            lib = ctypes.CDLL(lib_path[0], winmode=0x00000008)\n        else:\n            lib = ctypes.CDLL(lib_path[0], ctypes.RTLD_LOCAL)\n        # DMatrix functions\n        lib.MXGetLastError.restype = ctypes.c_char_p\n    except OSError as e:\n        if \"libcudnn\" in e.args[0]:\n            e.args = (e.args[0]+'\\nNotes: Starting from version 1.8.0, cuDNN and NCCL should be installed by users in advance. \\\n                      \\nPlease follow the instructions in https://docs.nvidia.com/deeplearning/cudnn/install-guide/index.html to install cuDNN.',)\n            raise OSError(e) from None\n        if \"libnccl\" in e.args[0]:\n            e.args = (e.args[0]+'\\nNotes: Starting from version 1.8.0, cuDNN and NCCL should be installed by users in advance. \\\n                      \\nPlease follow the instructions in https://docs.nvidia.com/deeplearning/nccl/install-guide/index.html to install NCCL.',)\n            raise OSError(e) from None\n        if \"libquadmath\" in e.args[0]:\n            e.args = (e.args[0]+'\\nNotes: As libquadmath.so.0 is a GPL library and MXNet part of the Apache Software Foundation, \\\n                      \\nMXNet must not redistribute libquadmath.so.0 as part of the Pypi package and users must manually install it. \\\n                      \\nOn Debian based systems, including Ubuntu, run sudo apt install libquadmath0 to install the shared library. \\\n                      \\nOn RHEL based systems, including CentOS, run sudo yum install libquadmath to install the shared library. ')\n            raise OSError(e) from None\n        raise\n    else:\n        return lib\n\n\n# version number\n__version__ = libinfo.__version__\n# library instance of mxnet\n_LIB = _load_lib()\n\ncheck_call(_LIB.MXSetFlushDenorms(ctypes.c_bool(True),\n                                  ctypes.byref(ctypes.c_bool())))\n# type definitions\nmx_int = ctypes.c_int\nmx_uint = ctypes.c_uint\nmx_int64 = ctypes.c_int64\nmx_float = ctypes.c_float\nmx_float_p = ctypes.POINTER(mx_float)\nmx_real_t = _np.float32\nNDArrayHandle = ctypes.c_void_p\nFunctionHandle = ctypes.c_void_p\nOpHandle = ctypes.c_void_p\nCachedOpHandle = ctypes.c_void_p\nSymbolHandle = ctypes.c_void_p\nDataIterCreatorHandle = ctypes.c_void_p\nDataIterHandle = ctypes.c_void_p\nDatasetHandle = ctypes.c_void_p\nBatchifyFunctionhandle = ctypes.c_void_p\nKVStoreHandle = ctypes.c_void_p\nRecordIOHandle = ctypes.c_void_p\nRtcHandle = ctypes.c_void_p\nCudaModuleHandle = ctypes.c_void_p\nCudaKernelHandle = ctypes.c_void_p\nProfileHandle = ctypes.c_void_p\n\n\n#----------------------------\n# helper function definition\n#----------------------------\ndef c_str(string):\n    \"\"\"Create ctypes char * from a Python string.\n\n    Parameters\n    ----------\n    string : string type\n        Python string.\n\n    Returns\n    -------\n    str : c_char_p\n        A char pointer that can be passed to C API.\n\n    Examples\n    --------\n    >>> x = mx.base.c_str(\"Hello, World\")\n    >>> print(x.value)\n    b\"Hello, World\"\n    \"\"\"\n    return ctypes.c_char_p(string.encode('utf-8'))\n\ndef c_str_array(strings):\n    \"\"\"Create ctypes const char ** from a list of Python strings.\n\n    Parameters\n    ----------\n    strings : list of string\n        Python strings.\n\n    Returns\n    -------\n    (ctypes.c_char_p * len(strings))\n        A const char ** pointer that can be passed to C API.\n    \"\"\"\n    arr = (ctypes.c_char_p * len(strings))()\n    arr[:] = [s.encode('utf-8') for s in strings]\n    return arr\n\n\ndef c_array(ctype, values):\n    \"\"\"Create ctypes array from a Python array.\n\n    Parameters\n    ----------\n    ctype : ctypes data type\n        Data type of the array we want to convert to, such as mx_float.\n\n    values : tuple or list\n        Data content.\n\n    Returns\n    -------\n    out : ctypes array\n        Created ctypes array.\n\n    Examples\n    --------\n    >>> x = mx.base.c_array(mx.base.mx_float, [1, 2, 3])\n    >>> print len(x)\n    3\n    >>> x[1]\n    2.0\n    \"\"\"\n    out = (ctype * len(values))()\n    out[:] = values\n    return out\n\n\ndef c_array_buf(ctype, buf):\n    \"\"\"Create ctypes array from a Python buffer.\n    For primitive types, using the buffer created with array.array is faster\n    than a c_array call.\n\n    Parameters\n    ----------\n    ctype : ctypes data type\n        Data type of the array we want to convert to, such as mx_float.\n\n    buf : buffer type\n        Data content.\n\n    Returns\n    -------\n    out : ctypes array\n        Created ctypes array.\n\n    Examples\n    --------\n    >>> x = mx.base.c_array_buf(mx.base.mx_float, array.array('i', [1, 2, 3]))\n    >>> print len(x)\n    3\n    >>> x[1]\n    2.0\n    \"\"\"\n    return (ctype * len(buf)).from_buffer(buf)\n\n\ndef c_handle_array(objs):\n    \"\"\"Create ctypes const void ** from a list of MXNet objects with handles.\n\n    Parameters\n    ----------\n    objs : list of NDArray/Symbol.\n        MXNet objects.\n\n    Returns\n    -------\n    (ctypes.c_void_p * len(objs))\n        A void ** pointer that can be passed to C API.\n    \"\"\"\n    arr = (ctypes.c_void_p * len(objs))()\n    arr[:] = [o.handle for o in objs]\n    return arr\n\n\ndef ctypes2buffer(cptr, length):\n    \"\"\"Convert ctypes pointer to buffer type.\n\n    Parameters\n    ----------\n    cptr : ctypes.POINTER(ctypes.c_char)\n        Pointer to the raw memory region.\n    length : int\n        The length of the buffer.\n\n    Returns\n    -------\n    buffer : bytearray\n        The raw byte memory buffer.\n    \"\"\"\n    if not isinstance(cptr, ctypes.POINTER(ctypes.c_char)):\n        raise TypeError('expected char pointer')\n    res = bytearray(length)\n    rptr = (ctypes.c_char * length).from_buffer(res)\n    if not ctypes.memmove(rptr, cptr, length):\n        raise RuntimeError('memmove failed')\n    return res\n\n\ndef ctypes2numpy_shared(cptr, shape):\n    \"\"\"Convert a ctypes pointer to a numpy array.\n\n    The resulting NumPy array shares the memory with the pointer.\n\n    Parameters\n    ----------\n    cptr : ctypes.POINTER(mx_float)\n        pointer to the memory region\n\n    shape : tuple\n        Shape of target `NDArray`.\n\n    Returns\n    -------\n    out : numpy_array\n        A numpy array : numpy array.\n    \"\"\"\n    if not isinstance(cptr, ctypes.POINTER(mx_float)):\n        raise RuntimeError('expected float pointer')\n    size = 1\n    for s in shape:\n        size *= s\n    dbuffer = (mx_float * size).from_address(ctypes.addressof(cptr.contents))\n    return _np.frombuffer(dbuffer, dtype=_np.float32).reshape(shape)\n\n\ndef build_param_doc(arg_names, arg_types, arg_descs, remove_dup=True):\n    \"\"\"Build argument docs in python style.\n\n    arg_names : list of str\n        Argument names.\n\n    arg_types : list of str\n        Argument type information.\n\n    arg_descs : list of str\n        Argument description information.\n\n    remove_dup : boolean, optional\n        Whether remove duplication or not.\n\n    Returns\n    -------\n    docstr : str\n        Python docstring of parameter sections.\n    \"\"\"\n    param_keys = set()\n    param_str = []\n    for key, type_info, desc in zip(arg_names, arg_types, arg_descs):\n        if key in param_keys and remove_dup:\n            continue\n        if key == 'num_args':\n            continue\n        param_keys.add(key)\n        ret = f'{key} : {type_info}'\n        if len(desc) != 0:\n            ret += '\\n    ' + desc\n        param_str.append(ret)\n    doc_str = ('Parameters\\n' +\n               '----------\\n' +\n               '{}\\n')\n    doc_str = doc_str.format('\\n'.join(param_str))\n    return doc_str\n\n\ndef _notify_shutdown():\n    \"\"\"Notify MXNet about a shutdown.\"\"\"\n    check_call(_LIB.MXNotifyShutdown())\n\n\natexit.register(_notify_shutdown)\n\n\ndef add_fileline_to_docstring(module, incursive=True):\n    \"\"\"Append the definition position to each function contained in module.\n\n    Examples\n    --------\n    # Put the following codes at the end of a file\n    add_fileline_to_docstring(__name__)\n    \"\"\"\n\n    def _add_fileline(obj):\n        \"\"\"Add fileinto to a object.\n        \"\"\"\n        if obj.__doc__ is None or 'From:' in obj.__doc__:\n            return\n        fname = inspect.getsourcefile(obj)\n        if fname is None:\n            return\n        try:\n            line = inspect.getsourcelines(obj)[-1]\n        except IOError:\n            return\n        obj.__doc__ += f'\\n\\nFrom:{fname}:{line}'\n\n    if isinstance(module, str):\n        module = sys.modules[module]\n    for _, obj in inspect.getmembers(module):\n        if inspect.isbuiltin(obj):\n            continue\n        if inspect.isfunction(obj):\n            _add_fileline(obj)\n        if inspect.ismethod(obj):\n            _add_fileline(obj.__func__)\n        if inspect.isclass(obj) and incursive:\n            add_fileline_to_docstring(obj, False)\n\n\ndef _as_list(obj):\n    \"\"\"A utility function that converts the argument to a list if it is not already.\n\n    Parameters\n    ----------\n    obj : object\n\n    Returns\n    -------\n    If `obj` is a list or tuple, return it. Otherwise, return `[obj]` as a\n    single-element list.\n\n    \"\"\"\n    if isinstance(obj, (list, tuple)):\n        return obj\n    else:\n        return [obj]\n\n\n_OP_NAME_PREFIX_LIST = ['_contrib_', '_linalg_', '_sparse_', '_image_', '_random_']\n\n\ndef _get_op_name_prefix(op_name):\n    \"\"\"\n    Check whether the given op_name starts with any words in `_OP_NAME_PREFIX_LIST`.\n    If found, return the prefix; else, return an empty string.\n    \"\"\"\n    for prefix in _OP_NAME_PREFIX_LIST:\n        if op_name.startswith(prefix):\n            return prefix\n    return \"\"\n\n\n# pylint: enable=invalid-name\ndef _init_op_module(root_namespace, module_name, make_op_func):\n    \"\"\"\n    Registers op functions created by `make_op_func` under\n    `root_namespace.module_name.[submodule_name]`,\n    where `submodule_name` is one of `_OP_SUBMODULE_NAME_LIST`.\n\n    Parameters\n    ----------\n    root_namespace : str\n        Top level module name, `mxnet` in the current cases.\n    module_name : str\n        Second level module name, `ndarray` and `symbol` in the current cases.\n    make_op_func : function\n        Function for creating op functions for `ndarray` and `symbol` modules.\n    \"\"\"\n    plist = ctypes.POINTER(ctypes.c_char_p)()\n    size = ctypes.c_uint()\n\n    check_call(_LIB.MXListAllOpNames(ctypes.byref(size),\n                                     ctypes.byref(plist)))\n    op_names = []\n    for i in range(size.value):\n        op_name = py_str(plist[i])\n        if not _is_np_op(op_name):\n            op_names.append(op_name)\n\n    module_op = sys.modules[f\"{root_namespace}.{module_name}.op\"]\n    module_internal = sys.modules[f\"{root_namespace}.{module_name}._internal\"]\n    # contrib module in the old format (deprecated)\n    # kept here for backward compatibility\n    # use mx.nd.contrib or mx.sym.contrib from now on\n    contrib_module_name_old = f\"{root_namespace}.contrib.{module_name}\"\n    contrib_module_old = sys.modules[contrib_module_name_old]\n    submodule_dict = {}\n    for op_name_prefix in _OP_NAME_PREFIX_LIST:\n        submodule_dict[op_name_prefix] =\\\n            sys.modules[f\"{root_namespace}.{module_name}.{op_name_prefix[1:-1]}\"]\n    for name in op_names:\n        hdl = OpHandle()\n        check_call(_LIB.NNGetOpHandle(c_str(name), ctypes.byref(hdl)))\n        op_name_prefix = _get_op_name_prefix(name)\n        module_name_local = module_name\n        if len(op_name_prefix) > 0:\n            if op_name_prefix != '_random_' or name.endswith('_like'):\n                func_name = name[len(op_name_prefix):]\n                cur_module = submodule_dict[op_name_prefix]\n                module_name_local = f\"{root_namespace}.{module_name}.{op_name_prefix[1:-1]}\"\n            else:\n                func_name = name\n                cur_module = module_internal\n        elif name.startswith('_'):\n            func_name = name\n            cur_module = module_internal\n        else:\n            func_name = name\n            cur_module = module_op\n\n        function = make_op_func(hdl, name, func_name)\n        function.__module__ = module_name_local\n        setattr(cur_module, function.__name__, function)\n        cur_module.__all__.append(function.__name__)\n\n        if op_name_prefix == '_contrib_':\n            hdl = OpHandle()\n            check_call(_LIB.NNGetOpHandle(c_str(name), ctypes.byref(hdl)))\n            func_name = name[len(op_name_prefix):]\n\n            function = make_op_func(hdl, name, func_name)\n            function.__module__ = contrib_module_name_old\n            setattr(contrib_module_old, function.__name__, function)\n            contrib_module_old.__all__.append(function.__name__)\n\n\ndef _generate_op_module_signature(root_namespace, module_name, op_code_gen_func):\n    \"\"\"\n    Generate op functions created by `op_code_gen_func` and write to the source file\n    of `root_namespace.module_name.[submodule_name]`,\n    where `submodule_name` is one of `_OP_SUBMODULE_NAME_LIST`.\n\n    Parameters\n    ----------\n    root_namespace : str\n        Top level module name, `mxnet` in the current cases.\n    module_name : str\n        Second level module name, `ndarray` and `symbol` in the current cases.\n    op_code_gen_func : function\n        Function for creating op functions for `ndarray` and `symbol` modules.\n    \"\"\"\n    license_lines = [\n        '# Licensed to the Apache Software Foundation (ASF) under one',\n        '# or more contributor license agreements.  See the NOTICE file',\n        '# distributed with this work for additional information',\n        '# regarding copyright ownership.  The ASF licenses this file',\n        '# to you under the Apache License, Version 2.0 (the',\n        '# \"License\"); you may not use this file except in compliance',\n        '# with the License.  You may obtain a copy of the License at',\n        '#',\n        '#   http://www.apache.org/licenses/LICENSE-2.0',\n        '#',\n        '# Unless required by applicable law or agreed to in writing,',\n        '# software distributed under the License is distributed on an',\n        '# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY',\n        '# KIND, either express or implied.  See the License for the',\n        '# specific language governing permissions and limitations',\n        '# under the License.',\n        '',\n    ]\n    license_str = os.linesep.join(license_lines)\n    def get_module_file(module_name):\n        \"\"\"Return the generated module file based on module name.\"\"\"\n        path = os.path.dirname(__file__)\n        module_path = module_name.split('.')\n        module_path[-1] = 'gen_' + module_path[-1]\n        file_name = os.path.join(path, '..', *module_path) + '.py'\n        module_file = open(file_name, 'w', encoding=\"utf-8\")\n        dependencies = {'symbol': ['from ._internal import SymbolBase',\n                                   'from ..base import _Null'],\n                        'ndarray': ['from ._internal import NDArrayBase',\n                                    'from ..base import _Null']}\n        module_file.write('# coding: utf-8')\n        module_file.write(license_str)\n        module_file.write('# File content is auto-generated. Do not modify.' + os.linesep)\n        module_file.write('# pylint: skip-file' + os.linesep)\n        module_file.write(os.linesep.join(dependencies[module_name.split('.')[1]]))\n        return module_file\n\n    def write_all_str(module_file, module_all_list):\n        \"\"\"Write the proper __all__ based on available operators.\"\"\"\n        module_file.write(os.linesep)\n        module_file.write(os.linesep)\n        all_str = '__all__ = [' + ', '.join([f\"'{s}'\" for s in module_all_list]) + ']'\n        module_file.write(all_str)\n\n    plist = ctypes.POINTER(ctypes.c_char_p)()\n    size = ctypes.c_uint()\n\n    check_call(_LIB.MXListAllOpNames(ctypes.byref(size),\n                                     ctypes.byref(plist)))\n    op_names = []\n    for i in range(size.value):\n        op_name = py_str(plist[i])\n        if not _is_np_op(op_name):\n            op_names.append(op_name)\n\n    module_op_file = get_module_file(f\"{root_namespace}.{module_name}.op\")\n    module_op_all = []\n    module_internal_file = get_module_file(f\"{root_namespace}.{module_name}._internal\")\n    module_internal_all = []\n    submodule_dict = {}\n    for op_name_prefix in _OP_NAME_PREFIX_LIST:\n        submodule_dict[op_name_prefix] =\\\n            (get_module_file(f\"{root_namespace}.{module_name}.{op_name_prefix[1:-1]}\"), [])\n    for name in op_names:\n        hdl = OpHandle()\n        check_call(_LIB.NNGetOpHandle(c_str(name), ctypes.byref(hdl)))\n        op_name_prefix = _get_op_name_prefix(name)\n        if len(op_name_prefix) > 0:\n            func_name = name[len(op_name_prefix):]\n            cur_module_file, cur_module_all = submodule_dict[op_name_prefix]\n        elif name.startswith('_'):\n            func_name = name\n            cur_module_file = module_internal_file\n            cur_module_all = module_internal_all\n        else:\n            func_name = name\n            cur_module_file = module_op_file\n            cur_module_all = module_op_all\n\n        code, _ = op_code_gen_func(hdl, name, func_name, True)\n        cur_module_file.write(os.linesep)\n        cur_module_file.write(code)\n        cur_module_all.append(func_name)\n\n    for (submodule_f, submodule_all) in submodule_dict.values():\n        write_all_str(submodule_f, submodule_all)\n        submodule_f.close()\n    write_all_str(module_op_file, module_op_all)\n    module_op_file.close()\n    write_all_str(module_internal_file, module_internal_all)\n    module_internal_file.close()\n\nctypes.pythonapi.PyCapsule_New.restype = ctypes.py_object\nctypes.pythonapi.PyCapsule_GetPointer.restype = ctypes.c_void_p\n\n\n_NP_OP_PREFIX = '_np_'\n_NP_OP_SUBMODULE_LIST = ['_random_', '_linalg_']\n_NP_OP_IMPLEMENTED_SET = {'_np_reshape'}\n\n_NP_EXT_OP_PREFIX = '_npx_'\n_NP_EXT_OP_SUBMODULE_LIST = ['_image_', '_random_']\n_NP_EXT_OP_IMPLEMENTED_SET = {'_npx_softmax', '_npx_log_softmax', '_npx_masked_softmax',\n                              '_npx_masked_log_softmax', '_npx_activation',\n                              '_npx_batch_norm', '_npx_fully_connected', '_npx_pick',\n                              '_npx_convolution', '_npx_deconvolution', '_npx_pooling',\n                              '_npx_dropout', '_npx_one_hot', '_npx_rnn', '_npx_embedding',\n                              '_npx_topk', '_npx_layer_norm', '_npx_leaky_relu', '_npx_batch_dot',\n                              '_npx_broadcast_like', '_npx_arange_like', '_npx_group_norm',\n                              '_npx_foreach', '_npx_while_loop', '_npx_cond'}\n\n_NP_INTERNAL_OP_PREFIX = '_npi_'\n\n_NP_OUTPUT_IS_LIST_OPERATORS = {'_npi_split', '_npi_hsplit'}\n\n\ndef _is_np_op(op_name):\n    return op_name.startswith(_NP_OP_PREFIX) or op_name.startswith(_NP_EXT_OP_PREFIX)\\\n           or op_name.startswith(_NP_INTERNAL_OP_PREFIX)\n\n\ndef _output_is_list(op_name):\n    \"\"\" Whether the output of the operator is a list.\n\n    Parameters\n    ----------\n    op_name : Name of the operator\n\n    Returns\n    -------\n\n    \"\"\"\n    if _is_np_op(op_name):\n        return op_name in _NP_OUTPUT_IS_LIST_OPERATORS\n    return False\n\n\ndef _get_op_submodule_name(op_name, op_name_prefix, submodule_name_list):\n    \"\"\"Get the submodule name of a specific op\"\"\"\n    assert op_name.startswith(op_name_prefix)\n    for submodule_name in submodule_name_list:\n        if op_name[len(op_name_prefix):].startswith(submodule_name):\n            return submodule_name\n    return \"\"\n\n\ndef _init_np_op_module(root_module_name, np_module_name, mx_module_name, make_op_func):\n    \"\"\"\n    Register numpy operators in namespaces `mxnet.numpy`, `mxnet.ndarray.numpy`\n    and `mxnet.symbol.numpy`. They are used in imperative mode, Gluon APIs w/o hybridization,\n    and Gluon APIs w/ hybridization, respectively. Essentially, operators with the same name\n    registered in three namespaces, respectively share the same functionality in C++ backend.\n    Different namespaces are needed for dispatching operator calls in Gluon's `HybridBlock` by `F`.\n\n    Parameters\n    ----------\n    root_module_name : str\n        Top level module name, `mxnet` in the current cases.\n    np_module_name : str\n        Second level module name, `numpy` or `numpy_extension` in the current case.\n    make_op_func : function\n        Function for creating op functions.\n    \"\"\"\n    from . import _numpy_op_doc as _np_op_doc\n    if np_module_name == 'numpy':\n        op_name_prefix = _NP_OP_PREFIX\n        submodule_name_list = _NP_OP_SUBMODULE_LIST\n        op_implemented_set = _NP_OP_IMPLEMENTED_SET\n    elif np_module_name == 'numpy_extension':\n        op_name_prefix = _NP_EXT_OP_PREFIX\n        submodule_name_list = _NP_EXT_OP_SUBMODULE_LIST\n        op_implemented_set = _NP_EXT_OP_IMPLEMENTED_SET\n    elif np_module_name == 'numpy._internal':\n        op_name_prefix = _NP_INTERNAL_OP_PREFIX\n        submodule_name_list = []\n        op_implemented_set = set()\n    else:\n        raise ValueError('unsupported np module name {}'.format(np_module_name))\n\n    plist = ctypes.POINTER(ctypes.c_char_p)()\n    size = ctypes.c_uint()\n    check_call(_LIB.MXListAllOpNames(ctypes.byref(size), ctypes.byref(plist)))\n    op_names = []\n    for i in range(size.value):\n        name = py_str(plist[i])\n        if mx_module_name != 'symbol':\n            if name.startswith(op_name_prefix) and name not in op_implemented_set:\n                op_names.append(name)\n        else:\n            if name.startswith(op_name_prefix):\n                op_names.append(name)\n\n    if mx_module_name is None:\n        # register np/npx ops for imperative programming\n        op_module_name = f\"{root_module_name}.{np_module_name}._op\" # e.g. mxnet.numpy._op\n        op_submodule_name = f\"{root_module_name}.{np_module_name}\" # e.g. mxnet.numpy.random\n    elif mx_module_name in ('ndarray', 'symbol'):\n        # register numpy internal ops and np/npx ops for use in Gluon\n        # np internal ops are registered in mxnet.ndarray/symbol.numpy._internal\n        # np ops are registered in mxnet.ndarray/symbol.numpy._op\n        # npx ops are registered in mxnet.ndarray/symbol.numpy_extension._op\n        op_module_name = f\"{root_module_name}.{mx_module_name}.{np_module_name}\"\n        if op_name_prefix != _NP_INTERNAL_OP_PREFIX:\n            op_module_name += '._op'\n        # e.g. mxnet.symbol.numpy.random\n        op_submodule_name = f\"{root_module_name}.{mx_module_name}.{np_module_name}\"\n    else:\n        raise ValueError('unsupported mxnet module {}'.format(mx_module_name))\n    op_submodule_name += '.{}'\n\n    op_module = sys.modules[op_module_name]\n    submodule_dict = {}\n    for submodule_name in submodule_name_list:\n        submodule_dict[submodule_name] = sys.modules[op_submodule_name.format(submodule_name[1:-1])]\n    for name in op_names:\n        hdl = OpHandle()\n        check_call(_LIB.NNGetOpHandle(c_str(name), ctypes.byref(hdl)))\n        submodule_name = _get_op_submodule_name(name, op_name_prefix, submodule_name_list)\n        if len(submodule_name) > 0:\n            func_name = name[(len(op_name_prefix) + len(submodule_name)):]\n            cur_module = submodule_dict[submodule_name]\n            module_name_local = op_submodule_name.format(submodule_name[1:-1])\n        else:\n            func_name = name[len(op_name_prefix):]\n            cur_module = op_module\n            module_name_local =\\\n                op_module_name[:-len('._op')] if op_module_name.endswith('._op') else op_module_name\n\n        function = make_op_func(hdl, name, func_name)\n        function.__module__ = module_name_local\n        setattr(cur_module, function.__name__, function)\n        cur_module.__all__.append(function.__name__)\n\n        if hasattr(_np_op_doc, name):\n            function.__doc__ = getattr(_np_op_doc, name).__doc__\n        else:\n            function.__doc__ = re.sub('NDArray', 'ndarray', function.__doc__)\n"
  },
  {
    "path": "python/mxnet/callback.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Callback functions that can be used to track various status during epoch.\"\"\"\n\nimport logging\nimport math\nimport time\nfrom .model import save_checkpoint\n\ndef do_checkpoint(prefix, period=1):\n    \"\"\"A callback that saves a model checkpoint every few epochs.\n    Each checkpoint is made up of a couple of binary files: a model description file and a\n    parameters (weights and biases) file. The model description file is named\n    `prefix`--symbol.json and the parameters file is named `prefix`-`epoch_number`.params\n\n    Parameters\n    ----------\n    prefix : str\n        Prefix for the checkpoint filenames.\n    period : int, optional\n        Interval (number of epochs) between checkpoints. Default `period` is 1.\n\n    Returns\n    -------\n    callback : function\n        A callback function that can be passed as `epoch_end_callback` to fit.\n\n    Example\n    -------\n    >>> module.fit(iterator, num_epoch=n_epoch,\n    ... epoch_end_callback  = mx.callback.do_checkpoint(\"mymodel\", 1))\n    Start training with [cpu(0)]\n    Epoch[0] Resetting Data Iterator\n    Epoch[0] Time cost=0.100\n    Saved checkpoint to \"mymodel-0001.params\"\n    Epoch[1] Resetting Data Iterator\n    Epoch[1] Time cost=0.060\n    Saved checkpoint to \"mymodel-0002.params\"\n    \"\"\"\n    period = int(max(1, period))\n    def _callback(iter_no, sym, arg, aux):\n        \"\"\"The checkpoint function.\"\"\"\n        if (iter_no + 1) % period == 0:\n            save_checkpoint(prefix, iter_no + 1, sym, arg, aux)\n    return _callback\n\n\ndef log_train_metric(period, auto_reset=False):\n    \"\"\"Callback to log the training evaluation result every period.\n\n    Parameters\n    ----------\n    period : int\n        The number of batch to log the training evaluation metric.\n    auto_reset : bool\n        Reset the metric after each log.\n\n    Returns\n    -------\n    callback : function\n        The callback function that can be passed as iter_epoch_callback to fit.\n    \"\"\"\n    def _callback(param):\n        \"\"\"The checkpoint function.\"\"\"\n        if param.nbatch % period == 0 and param.eval_metric is not None:\n            name_value = param.eval_metric.get_name_value()\n            for name, value in name_value:\n                logging.info('Iter[%d] Batch[%d] Train-%s=%f',\n                             param.epoch, param.nbatch, name, value)\n            if auto_reset:\n                param.eval_metric.reset()\n    return _callback\n\n\nclass Speedometer(object):\n    \"\"\"Logs training speed and evaluation metrics periodically.\n\n    Parameters\n    ----------\n    batch_size: int\n        Batch size of data.\n    frequent: int\n        Specifies how frequently training speed and evaluation metrics\n        must be logged. Default behavior is to log once every 50 batches.\n    auto_reset : bool\n        Reset the evaluation metrics after each log.\n\n    Example\n    -------\n    >>> # Print training speed and evaluation metrics every ten batches. Batch size is one.\n    >>> module.fit(iterator, num_epoch=n_epoch,\n    ... batch_end_callback=mx.callback.Speedometer(1, 10))\n    Epoch[0] Batch [10] Speed: 1910.41 samples/sec  Train-accuracy=0.200000\n    Epoch[0] Batch [20] Speed: 1764.83 samples/sec  Train-accuracy=0.400000\n    Epoch[0] Batch [30] Speed: 1740.59 samples/sec  Train-accuracy=0.500000\n    \"\"\"\n    def __init__(self, batch_size, frequent=50, auto_reset=True):\n        self.batch_size = batch_size\n        self.frequent = frequent\n        self.init = False\n        self.tic = 0\n        self.last_count = 0\n        self.auto_reset = auto_reset\n\n    def __call__(self, param):\n        \"\"\"Callback to Show speed.\"\"\"\n        count = param.nbatch\n        if self.last_count > count:\n            self.init = False\n        self.last_count = count\n\n        if self.init:\n            if count % self.frequent == 0:\n                # #11504\n                try:\n                    speed = self.frequent * self.batch_size / (time.time() - self.tic)\n                except ZeroDivisionError:\n                    speed = float('inf')\n                if param.eval_metric is not None:\n                    name_value = param.eval_metric.get_name_value()\n                    if self.auto_reset:\n                        param.eval_metric.reset()\n                        msg = 'Epoch[%d] Batch [%d-%d]\\tSpeed: %.2f samples/sec'\n                        msg += '\\t%s=%f'*len(name_value)\n                        logging.info(msg, param.epoch, count-self.frequent, count, speed, *sum(name_value, ()))\n                    else:\n                        msg = 'Epoch[%d] Batch [0-%d]\\tSpeed: %.2f samples/sec'\n                        msg += '\\t%s=%f'*len(name_value)\n                        logging.info(msg, param.epoch, count, speed, *sum(name_value, ()))\n                else:\n                    logging.info(\"Iter[%d] Batch [%d]\\tSpeed: %.2f samples/sec\",\n                                 param.epoch, count, speed)\n                self.tic = time.time()\n        else:\n            self.init = True\n            self.tic = time.time()\n\n\nclass ProgressBar(object):\n    \"\"\"Displays a progress bar, indicating the percentage of batches processed within each epoch.\n\n    Parameters\n    ----------\n    total: int\n        total number of batches per epoch\n    length: int\n        number of chars to define maximum length of progress bar\n\n    Examples\n    --------\n    >>> progress_bar = mx.callback.ProgressBar(total=2)\n    >>> mod.fit(data, num_epoch=5, batch_end_callback=progress_bar)\n    [========--------] 50.0%\n    [================] 100.0%\n    \"\"\"\n    def __init__(self, total, length=80):\n        self.bar_len = length\n        self.total = total\n\n    def __call__(self, param):\n        \"\"\"Callback to Show progress bar.\"\"\"\n        count = param.nbatch\n        filled_len = int(round(self.bar_len * count / float(self.total)))\n        percents = math.ceil(100.0 * count / float(self.total))\n        prog_bar = '=' * filled_len + '-' * (self.bar_len - filled_len)\n        logging.info('[%s] %s%s\\r', prog_bar, percents, '%')\n\n\nclass LogValidationMetricsCallback(object):\n    \"\"\"Just logs the eval metrics at the end of an epoch.\"\"\"\n\n    def __call__(self, param):\n        if not param.eval_metric:\n            return\n        name_value = param.eval_metric.get_name_value()\n        for name, value in name_value:\n            logging.info('Epoch[%d] Validation-%s=%f', param.epoch, name, value)\n"
  },
  {
    "path": "python/mxnet/container.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n# pylint: disable=undefined-variable\n\"\"\"\nContainer data structures.\nAcknowledgement: This file originates from incubator-tvm\n\"\"\"\nfrom ._ffi.object import Object, register_object, getitem_helper, PyNativeObject\nfrom ._ffi.function import _init_api\n\n@register_object(\"MXNet.ADT\")\nclass ADT(Object):\n    \"\"\"Algebatic data type(ADT) object.\n\n    Parameters\n    ----------\n    tag : int\n        The tag of ADT.\n\n    fields : list[Object] or tuple[Object]\n        The source tuple.\n    \"\"\"\n    def __init__(self, tag, fields):\n        for f in fields:\n            assert isinstance(f, (Object)), \"Expect object\" \\\n            \", but received : {0}\".format(type(f))\n        self.__init_handle_by_constructor__(_ADT, tag, *fields)\n\n    @property\n    def tag(self):\n        return _GetADTTag(self)\n\n    def __getitem__(self, idx):\n        return getitem_helper(\n            self, _GetADTFields, len(self), idx)\n\n    def __len__(self):\n        return _GetADTSize(self)\n\n@register_object(\"MXNet.Map\")\nclass Map(Object):\n    \"\"\"Map container of MXNet.\n\n    You do not need to create Map explicitly.\n    Normally python dict will be converted automaticall to Map during mxnet function call.\n    You can use convert to create a dict[Object-> Object] into a Map\n    \"\"\"\n\n    def __getitem__(self, k):\n        return _MapGetItem(self, k)\n\n    def __contains__(self, k):\n        return _MapCount(self, k) != 0\n\n    def items(self):\n        \"\"\"Get the items from the map\"\"\"\n        akvs = _MapItems(self)\n        return [(akvs[i], akvs[i+1]) for i in range(0, len(akvs), 2)]\n\n    def __len__(self):\n        return _MapSize(self)\n\n    def get(self, key, default=None):\n        \"\"\"Get an element with a default value.\n\n        Parameters\n        ----------\n        key : object\n            The attribute key.\n\n        default : object\n            The default object.\n\n        Returns\n        -------\n        value: object\n            The result value.\n        \"\"\"\n        return self[key] if key in self else default\n\n@register_object(\"MXNet.String\")\nclass String(str, PyNativeObject):\n    \"\"\"String object, represented as a python str.\n\n    Parameters\n    ----------\n    content : str\n        The content string used to construct the object.\n    \"\"\"\n\n    __slots__ = [\"__mxnet_object__\"]\n\n    def __new__(cls, content):\n        \"\"\"Construct from string content.\"\"\"\n        val = str.__new__(cls, content)\n        val.__init_mxnet_object_by_constructor__(_String, content)\n        return val\n\n    # pylint: disable=no-self-argument\n    def __from_mxnet_object__(cls, obj):\n        \"\"\"Construct from a given mxnet object.\"\"\"\n        content = _GetFFIString(obj)\n        val = str.__new__(cls, content)\n        val.__mxnet_object__ = obj\n        return val\n\n_init_api(\"mxnet.container\")\n"
  },
  {
    "path": "python/mxnet/context.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Context management API of mxnet.\"\"\"\nfrom warnings import warn\nfrom .device import Device, _current, cpu, gpu, cpu_pinned  # pylint: disable=unused-import\nfrom .device import num_gpus, gpu_memory_info  # pylint: disable=unused-import\n\n\ndef Context(*args, **kwargs):\n    \"\"\"This class has been deprecated. Please refer to ``device.Device``.\"\"\"\n    warn('Directly use Context class to construct a device will be deprecated. '\n         'Please use Device class instead. ', DeprecationWarning)\n    return Device(*args, **kwargs)\n\ndef current_context():\n    \"\"\"This function has been deprecated. Please refer to ``device.current_device``.\"\"\"\n    warn('Directly use current_context to get current device will be deprecated. '\n         'Please use current_device method instead. ', DeprecationWarning)\n    return _current.get()\n"
  },
  {
    "path": "python/mxnet/contrib/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Experimental contributions\"\"\"\n\nfrom . import symbol\nfrom . import ndarray\n\nfrom . import symbol as sym\nfrom . import ndarray as nd\n\nfrom . import tensorboard\n\nfrom . import text\nfrom . import onnx\nfrom . import io\nfrom . import quantization\nfrom . import quantization as quant\nfrom . import tensorrt\n"
  },
  {
    "path": "python/mxnet/contrib/io.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Contrib data iterators for common data formats.\"\"\"\nfrom ..io import DataIter, DataDesc\nfrom .. import ndarray as nd\n\n\nclass DataLoaderIter(DataIter):\n    \"\"\"Returns an iterator for ``mx.gluon.data.Dataloader`` so gluon dataloader\n    can be used in symbolic module.\n\n    Parameters\n    ----------\n    loader : mxnet.gluon.data.Dataloader\n        Gluon dataloader instance\n    data_name : str, optional\n        The data name.\n    label_name : str, optional\n        The label name.\n    dtype : str, optional\n        The dtype specifier, can be float32 or float16\n\n    Examples\n    --------\n    >>> import mxnet as mx\n    >>> from mxnet.gluon.data.vision import MNIST\n    >>> from mxnet.gluon.data import DataLoader\n    >>> train_dataset = MNIST(train=True)\n    >>> train_data = mx.gluon.data.DataLoader(train_dataset, 32, shuffle=True, num_workers=4)\n    >>> dataiter = mx.io.DataloaderIter(train_data)\n    >>> for batch in dataiter:\n    ...     batch.data[0].shape\n    ...\n    (32L, 28L, 28L, 1L)\n    \"\"\"\n    def __init__(self, loader, data_name='data', label_name='softmax_label', dtype='float32'):\n        super(DataLoaderIter, self).__init__()\n        self._loader = loader\n        self._iter = iter(self._loader)\n        data, label = next(self._iter)\n        self.batch_size = data.shape[0]\n        self.dtype = dtype\n        self.provide_data = [DataDesc(data_name, data.shape, dtype)]\n        self.provide_label = [DataDesc(label_name, label.shape, dtype)]\n        self._current_batch = None\n        self.reset()\n\n    def reset(self):\n        self._iter = iter(self._loader)\n\n    def iter_next(self):\n        try:\n            self._current_batch = next(self._iter)\n        except StopIteration:\n            self._current_batch = None\n        return self._current_batch is not None\n\n    def getdata(self):\n        if self.getpad():\n            dshape = self._current_batch[0].shape\n            ret = nd.empty(shape=([self.batch_size] + list(dshape[1:])))\n            ret[:dshape[0]] = self._current_batch[0].astype(self.dtype)\n            return [ret]\n        return [self._current_batch[0].astype(self.dtype)]\n\n    def getlabel(self):\n        if self.getpad():\n            lshape = self._current_batch[1].shape\n            ret = nd.empty(shape=([self.batch_size] + list(lshape[1:])))\n            ret[:lshape[0]] = self._current_batch[1].astype(self.dtype)\n            return [ret]\n        return [self._current_batch[1].astype(self.dtype)]\n\n    def getpad(self):\n        return self.batch_size - self._current_batch[0].shape[0]\n\n    def getindex(self):\n        return None\n"
  },
  {
    "path": "python/mxnet/contrib/ndarray.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"NDArray namespace used to register contrib functions\"\"\"\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/contrib/onnx/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Module for ONNX model format support for Apache MXNet.\"\"\"\n\nfrom ...onnx import export_model as export_model_\n\ndef export_model(*args, **kwargs):\n    print('Calling mxnet.contrib.onnx.export_model...')\n    print('Please be advised that the ONNX module has been moved to mxnet.onnx and '\n          'mxnet.onnx.export_model is the preferred path. The current path will be deprecated '\n          'in the upcoming MXNet v1.10 release.')\n    return export_model_(*args, **kwargs)\n"
  },
  {
    "path": "python/mxnet/contrib/quantization.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Quantization module for generating quantized (INT8) models from FP32 models.\"\"\"\n\nimport abc\nimport ctypes\nimport logging\nimport os\nimport warnings\nimport numpy as np\nimport mxnet as mx\nfrom ..base import _LIB, check_call, py_str\nfrom ..base import c_array, c_str, mx_uint, mx_real_t, c_str_array\nfrom ..base import SymbolHandle\nfrom ..symbol import Symbol\nfrom .. import ndarray\nfrom ..io import DataDesc\nfrom ..device import cpu, Device\nfrom ..util import is_np_array, wrap_ctx_to_device_func\n\n\ndef _multilist_iterator(arg, func):\n    \"\"\"Iterate over multidiemnsional list and returns new list\n    with same dimensions, but applied `func` function on list elements.\n    E.g. _multilist_iterator([1, 2, [3, 4]], lambda x: x**2) = [1, 4, [9, 16]]\n    \"\"\"\n    ret = []\n    if isinstance(arg, list):\n        for el in arg:\n            ret.append(_multilist_iterator(el, func))\n    else:\n        return func(arg)\n\n    return ret\n\ndef _quantize_params(qsym, params, min_max_dict):\n    \"\"\"Given a quantized symbol and a dict of params that have not been quantized,\n    generate quantized params. Currently only supports quantizing the arg_params\n    with names of `weight` or `bias`, not aux_params. If `qsym` contains symbols\n    that are excluded from being quantized, their corresponding params will\n    not be quantized, but saved together with quantized params of the symbols that\n    have been quantized.\n\n    Parameters\n    ----------\n    qsym : Symbol\n        Quantized symbol from FP32 symbol.\n    params : dict of str->NDArray\n    min_max_dict : dict of min/max pairs of layers' output\n    \"\"\"\n    inputs_name = qsym.list_arguments()\n    quantized_params = {}\n    if is_np_array():\n        quantize_fn = mx.npx.contrib_quantize\n        min_fn = lambda arr: mx.np.array([mx.np.min(arr)])\n        max_fn = lambda arr: mx.np.array([mx.np.max(arr)])\n        array_cls = mx.np\n    else:\n        quantize_fn = mx.nd.contrib.quantize\n        min_fn = mx.nd.min\n        max_fn = mx.nd.max\n        array_cls = mx.nd\n\n    for name in inputs_name:\n        if name.endswith(('weight_quantize', 'bias_quantize')):\n            original_name = name[:-len('_quantize')]\n            param = params[original_name]\n            # pylint: disable=unbalanced-tuple-unpacking\n            param_min = min_fn(param)\n            param_max = max_fn(param)\n            val, vmin, vmax = quantize_fn(data=param,\n                                          min_range=param_min,\n                                          max_range=param_max,\n                                          out_type='int8')\n            quantized_params[name] = val\n            quantized_params[name+'_min'] = vmin\n            quantized_params[name+'_max'] = vmax\n        elif name in params:\n            quantized_params[name] = params[name]\n        elif name.endswith(('_min')):\n            output = name[: - len('_min')]\n            if output in min_max_dict:\n                quantized_params[name] = array_cls.array([min_max_dict[output][0]])\n        elif name.endswith(('_max')):\n            output = name[: - len('_min')]\n            if output in min_max_dict:\n                quantized_params[name] = array_cls.array([min_max_dict[output][1]])\n    return quantized_params\n\n\ndef _quantize_symbol(sym, device, excluded_symbols=None, excluded_operators=None,\n                     offline_params=None, quantized_dtype='int8', quantize_mode='smart',\n                     quantize_granularity='tensor-wise'):\n    \"\"\"Given a symbol object representing a neural network of data type FP32,\n    quantize it into a INT8 network.\n\n    Parameters\n    ----------\n    sym : Symbol\n        FP32 neural network symbol.\n    device : Device\n        Defines the device that users want to run quantized symbol.\n    excluded_symbols : list of strings\n        A list of strings representing the names of the symbols that users want to excluding\n        from being quantized.\n    excluded_operators : list of strings\n        A list of strings representing the names of the operators that users want to excluding\n        from being quantized.\n    offline_params : list of strs\n        Names of the parameters that users want to quantize offline. It's always recommended to\n        quantize parameters offline so that quantizing parameters during the inference can be\n        avoided.\n    quantized_dtype : str\n        The quantized destination type for input data.\n    quantize_mode : str\n        The mode that quantization pass to apply.\n    quantize_granularity : str\n        The granularity of quantization, currently supports 'tensor-wise' and 'channel-wise'\n        quantization. The default value is 'tensor-wise'.\n    \"\"\"\n    num_excluded_symbols = 0\n    if excluded_symbols is not None:\n        assert isinstance(excluded_symbols, list)\n        num_excluded_symbols = len(excluded_symbols)\n    else:\n        excluded_symbols = []\n\n    num_excluded_ops = 0\n    if excluded_operators is not None:\n        assert isinstance(excluded_operators, list)\n        num_excluded_ops = len(excluded_operators)\n    else:\n        excluded_operators = []\n\n    num_offline = 0\n    offline = []\n    if offline_params is not None:\n        num_offline = len(offline_params)\n        for k in offline_params:\n            offline.append(c_str(k))\n\n    out = SymbolHandle()\n    size = mx_uint()\n    calib_str = ctypes.POINTER(ctypes.c_char_p)()\n    check_call(_LIB.MXQuantizeSymbol(sym.handle,\n                                     ctypes.byref(out),\n                                     ctypes.byref(ctypes.c_int(device.device_typeid)),\n                                     mx_uint(num_excluded_symbols),\n                                     c_str_array(excluded_symbols),\n                                     mx_uint(num_excluded_ops),\n                                     c_str_array(excluded_operators),\n                                     mx_uint(num_offline),\n                                     c_array(ctypes.c_char_p, offline),\n                                     c_str(quantized_dtype),\n                                     ctypes.c_bool(True),\n                                     c_str(quantize_mode),\n                                     c_str(quantize_granularity),\n                                     ctypes.byref(size),\n                                     ctypes.byref(calib_str)))\n    calib_layers = []\n    calib_layers = [py_str(calib_str[i]) for i in range(size.value)]\n    return Symbol(out), calib_layers\n\n\nclass CalibrationCollector(object):\n    \"\"\"Base class for all other collectors used with quantization\"\"\"\n    __metaclass__ = abc.ABCMeta\n\n    def __init__(self):\n        self.include_layers = None\n        self.min_max_dict = {}\n\n    @abc.abstractmethod\n    def collect(self, name, op_name, arr):\n        \"\"\"Function which is registered to Block as monitor callback. Names of layers\n        requiring calibration are stored in `self.include_layers` variable.\n\n        Parameters\n        ----------\n        name : str\n            Node name from which collected data comes from.\n        op_name : str\n            Operator name from which collected data comes from. Single operator\n            can have multiple input/ouput nodes - each should have different name.\n        arr : NDArray\n            NDArray containing data of monitored node.\n        \"\"\"\n\n    def post_collect(self):\n        \"\"\" Function called after collecting parameters. Returns dictionary of min and max values\n        for each calibrated layer. If not overriden, returns content of `self.min_max_dict`.\n        \"\"\"\n        return self.min_max_dict\n\n\nclass _LayerHistogramCollector(CalibrationCollector):\n    \"\"\"Saves layer histogram in a dict with layer names as keys and lists of NDArrays as\n    values. The collected histogram will be used for calculating the optimal thresholds for\n    quantization using KL divergence.\n    \"\"\"\n    def __init__(self, quantized_dtype, num_bins=8001, include_layers=None, logger=None):\n        super(_LayerHistogramCollector, self).__init__()\n        self.hist_dict = {}\n        self.num_bins = num_bins\n        self.include_layers = include_layers\n        self.logger = logger\n        self.quantized_dtype = quantized_dtype\n\n    def collect(self, name, op_name, arr):\n        \"\"\"Callback function for collecting layer output NDArrays.\"\"\"\n        if name not in self.include_layers:\n            return\n        arr = arr.copyto(cpu()).asnumpy()\n        if self.logger:\n            self.logger.debug(f\"Collecting layer {name} histogram of shape {arr.shape}\")\n        min_range = np.min(arr)\n        max_range = np.max(arr)\n        th = max(abs(min_range), abs(max_range))\n        if name in self.hist_dict:\n            self.hist_dict[name] = self.combine_histogram(self.hist_dict[name], arr, min_range, max_range, th)\n        else:\n            hist, hist_edges = np.histogram(arr, bins=self.num_bins, range=(-th, th))\n            self.hist_dict[name] = (hist, hist_edges, min_range, max_range, th)\n\n    def post_collect(self):\n        min_max_dict = self.get_optimal_thresholds(self.hist_dict, self.quantized_dtype, logger=self.logger)\n        return min_max_dict\n\n    @staticmethod\n    def combine_histogram(old_hist, arr, new_min, new_max, new_th):\n        \"\"\"Collect layer histogram for arr and combine it with old histogram.\"\"\"\n        (old_hist, old_hist_edges, old_min, old_max, old_th) = old_hist\n        if new_th <= old_th:\n            hist, _ = np.histogram(arr, bins=len(old_hist), range=(-old_th, old_th))\n            return (old_hist + hist, old_hist_edges, min(old_min, new_min), max(old_max, new_max), old_th)\n        else:\n            # Need to generate new histogram with new_th\n            old_num_bins = len(old_hist)\n            old_step = 2 * old_th / old_num_bins\n            half_increased_bins = int((new_th - old_th) // old_step + 1)\n            new_num_bins = half_increased_bins * 2 + old_num_bins\n            new_th = half_increased_bins * old_step + old_th\n            hist, hist_edges = np.histogram(arr, bins=new_num_bins, range=(-new_th, new_th))\n            hist[half_increased_bins:new_num_bins - half_increased_bins] += old_hist\n            return (hist, hist_edges, min(old_min, new_min), max(old_max, new_max), new_th)\n\n    # pylint: disable=line-too-long\n    @staticmethod\n    def get_optimal_threshold(hist_data, quantized_dtype, num_quantized_bins=255):\n        \"\"\"Given a dataset, find the optimal threshold for quantizing it.\n        The reference distribution is `q`, and the candidate distribution is `p`.\n        `q` is a truncated version of the original distribution.\n\n        Ref: http://on-demand.gputechconf.com/gtc/2017/presentation/s7310-8-bit-inference-with-tensorrt.pdf\n        \"\"\"\n        (hist, hist_edges, min_val, max_val, _) = hist_data\n        num_bins = len(hist)\n        assert (num_bins % 2 == 1)\n        if min_val >= 0 and quantized_dtype in ['auto', 'uint8']:\n            # We need to move negative bins to positive bins to fit uint8 range.\n            num_quantized_bins = num_quantized_bins * 2 + 1\n        hist = ndarray.array(hist, ctx=cpu())\n        hist_edges = ndarray.array(hist_edges, ctx=cpu())\n        threshold, divergence = ndarray.contrib.calibrate_entropy(hist=hist,\n                                                                  hist_edges=hist_edges,\n                                                                  num_quantized_bins=num_quantized_bins)\n        threshold = threshold.asnumpy()\n        divergence = divergence.asnumpy()\n        return min_val, max_val, threshold, divergence\n    # pylint: enable=line-too-long\n\n    @staticmethod\n    def get_optimal_thresholds(hist_dict, quantized_dtype, num_quantized_bins=255, logger=None):\n        \"\"\"Given a ndarray dict, find the optimal threshold for quantizing each value of the key.\"\"\"\n        assert isinstance(hist_dict, dict)\n        if logger is not None:\n            logger.info('Calculating optimal thresholds for quantization using KL divergence'\n                        f' with num_quantized_bins={num_quantized_bins}')\n        th_dict = {}\n        # copy hist_dict keys since the keys() only returns a view in python3\n        layer_names = list(hist_dict.keys())\n        for name in layer_names:\n            assert name in hist_dict\n            min_val, max_val, th, divergence = \\\n                _LayerHistogramCollector.get_optimal_threshold(hist_dict[name], quantized_dtype,\n                                                               num_quantized_bins=num_quantized_bins)\n            if min_val >= 0 and quantized_dtype in ['auto', 'uint8']:\n                th_dict[name] = (0, th)\n            else:\n                th_dict[name] = (-th, th)\n            del hist_dict[name]  # release the memory\n            if logger:\n                logger.debug(f\"layer={name}, min_val={min_val}, max_val={max_val}, th={th}, divergence={divergence}\")\n        return th_dict\n\n\nclass _LayerOutputMinMaxCollector(CalibrationCollector):\n    \"\"\"Saves layer output min and max values in a dict with layer names as keys.\n    The collected min and max values will be directly used as thresholds for quantization.\n    \"\"\"\n    def __init__(self, quantized_dtype, include_layers=None, logger=None):\n        super(_LayerOutputMinMaxCollector, self).__init__()\n        self.min_max_dict = {}\n        self.quantized_dtype = quantized_dtype\n        self.include_layers = include_layers\n        self.logger = logger\n\n    def collect(self, name, op_name, arr):\n        \"\"\"Callback function for collecting min and max values from an NDArray.\"\"\"\n        if name not in self.include_layers:\n            return\n        arr = arr.copyto(cpu()).asnumpy()\n        min_range = np.min(arr)\n        max_range = np.max(arr)\n        if name in self.min_max_dict:\n            cur_min_max = self.min_max_dict[name]\n            self.min_max_dict[name] = (min(cur_min_max[0], min_range),\n                                       max(cur_min_max[1], max_range))\n        else:\n            self.min_max_dict[name] = (min_range, max_range)\n        if self.logger:\n            self.logger.debug(f\"Collecting layer {name} min_range={min_range}, max_range={max_range}\")\n\n\ndef _calibrate_quantized_sym(qsym, min_max_dict):\n    \"\"\"Given a dictionary containing the thresholds for quantizing the layers,\n    set the thresholds into the quantized symbol as the params of requantize operators.\n    \"\"\"\n    if min_max_dict is None or len(min_max_dict) == 0:\n        return qsym\n    num_layer_outputs = len(min_max_dict)\n    layer_output_names = []\n    min_vals = []\n    max_vals = []\n    for k, v in min_max_dict.items():\n        layer_output_names.append(k)\n        min_vals.append(v[0])\n        max_vals.append(v[1])\n\n    calibrated_sym = SymbolHandle()\n    check_call(_LIB.MXSetCalibTableToQuantizedSymbol(qsym.handle,\n                                                     mx_uint(num_layer_outputs),\n                                                     c_str_array(layer_output_names),\n                                                     c_array(ctypes.c_float, min_vals),\n                                                     c_array(ctypes.c_float, max_vals),\n                                                     ctypes.byref(calibrated_sym)))\n    return Symbol(calibrated_sym)\n\n\ndef _collect_layer_statistics(sym_block, data, collector, num_inputs, num_calib_batches=None, logger=None):\n    if not isinstance(data, mx.gluon.data.DataLoader):\n        raise ValueError(f'Only supports data as a type of DataLoader, while received type {str(type(data))}')\n    sym_block.register_op_hook(collector.collect, monitor_all=True)\n    num_batches = 0\n    for batch in data:\n        if not isinstance(batch, list):\n            batch = [batch]\n        batch = _multilist_iterator(batch, lambda b: b.as_in_context(mx.cpu()))\n        sym_block(*batch[:num_inputs])\n        num_batches += 1\n        if num_calib_batches is not None and num_batches >= num_calib_batches:\n            break\n    if logger is not None:\n        logger.info(f\"Collected statistics from {num_batches} batches\")\n    return num_batches\n\n\ndef _generate_list_of_data_desc(data_shapes, data_types):\n    \"\"\"Convert list of tuples to list of DataDesc.\"\"\"\n    def flatten_list(arg):\n        ret = []\n        for el in arg:\n            if isinstance(el, list):\n                ret += flatten_list(el)\n            else:\n                ret.append(el)\n        return ret\n\n    flattened_data_types = flatten_list(data_types)\n    flattened_data_shapes = flatten_list(data_shapes)\n\n    if all(isinstance(x, DataDesc) for x in flattened_data_shapes):\n        return data_shapes\n\n    assert len(flattened_data_types) == len(flattened_data_shapes)\n\n    # pass integral type as reference\n    counter = [0]\n    def get_data_desc(data_shape, counter=counter, data_types=flattened_data_types):\n        if isinstance(data_shape, DataDesc):\n            return data_shape\n        elif isinstance(data_shape, tuple):\n            desc = DataDesc(name='data' + str(counter[0]), shape=data_shape,\n                                        dtype=data_types[counter[0]])\n            counter[0] += 1\n            return desc\n        else:\n            raise ValueError('data_shapes must be either a list of DataDesc or a list of Tuple')\n\n\n    if len(data_shapes) == 1 and not isinstance(data_shapes[0], list):\n        data_descs = [DataDesc(name='data', shape=data_shapes[0], dtype=data_types[0])]\n    else:\n        data_descs = _multilist_iterator(data_shapes, get_data_desc)\n\n    return data_descs\n\n@wrap_ctx_to_device_func\ndef quantize_model(sym, arg_params, aux_params, data_names=('data',),\n                   device=cpu(), excluded_sym_names=None, excluded_op_names=None, calib_mode='entropy',\n                   calib_data=None, num_calib_batches=None,\n                   quantized_dtype='int8', quantize_mode='smart',\n                   quantize_granularity='tensor-wise', logger=None):\n    \"\"\"User-level API for generating a quantized model from a FP32 model w/ or w/o calibration.\n    The backend quantized operators are only enabled for Linux systems. Please do not run\n    inference using the quantized models on Windows for now.\n    The quantization implementation adopts the TensorFlow's approach:\n    https://www.tensorflow.org/lite/performance/post_training_quantization.\n    The calibration implementation borrows the idea of Nvidia's 8-bit Inference with TensorRT:\n    http://on-demand.gputechconf.com/gtc/2017/presentation/s7310-8-bit-inference-with-tensorrt.pdf\n    and adapts the method to MXNet.\n\n    .. _`quantize_model_params`:\n\n    Parameters\n    ----------\n    sym : Symbol\n        Defines the structure of a neural network for FP32 data types.\n    arg_params : dict\n        Dictionary of name to `NDArray`.\n    aux_params : dict\n        Dictionary of name to `NDArray`.\n    data_names : list of strings\n        Data names required for creating a Module object to run forward propagation on the\n        calibration dataset.\n    device : Device\n        Defines the device that users want to run forward propagation on the calibration\n        dataset for collecting layer output statistics. Currently, only supports single device.\n    excluded_sym_names : list of strings\n        A list of strings representing the names of the symbols that users want to excluding\n        from being quantized.\n    excluded_op_names : list of strings\n        A list of strings representing the names of the operators that users want to excluding\n        from being quantized.\n    calib_mode : str\n        If calib_mode='none', no calibration will be used and the thresholds for\n        requantization after the corresponding layers will be calculated at runtime by\n        calling min and max operators. The quantized models generated in this\n        mode are normally 10-20% slower than those with calibrations during inference.\n        If calib_mode='naive', the min and max values of the layer outputs from a calibration\n        dataset will be directly taken as the thresholds for quantization.\n        If calib_mode='entropy' (default mode), the thresholds for quantization will be\n        derived such that the KL divergence between the distributions of FP32 layer outputs and\n        quantized layer outputs is minimized based upon the calibration dataset.\n    calib_data : DataLoader\n        A DataLoader initialized by the calibration dataset.\n    num_calib_batches : int or None\n        The maximum number of batches that user would like to use for calibration. If not provided,\n        the whole calibration dataset will be used.\n    quantized_dtype : str\n        The quantized destination type for input data. Currently support 'int8', 'uint8' and 'auto'.\n        'auto' means automatically select output type according to calibration result.\n        Default value is 'int8'.\n    quantize_mode : str\n        The mode that quantization pass to apply. Support 'full' and 'smart'.\n        'full' means quantize all operator if possible.\n        'smart' means quantization pass will smartly choice which operator should be quantized.\n    quantize_granularity : str\n        The granularity of quantization, currently supports 'tensor-wise' and 'channel-wise'\n        quantization. The default value is 'tensor-wise'.\n    logger : Object\n        A logging object for printing information during the process of quantization.\n\n    Returns\n    -------\n    quantized_model : tuple\n        A tuple of quantized symbol, quantized arg_params, and aux_params.\n    \"\"\"\n    warnings.warn('WARNING: This will be deprecated please use quantize_net with Gluon models')\n    if excluded_sym_names is None:\n        excluded_sym_names = []\n    if not isinstance(excluded_sym_names, list):\n        raise ValueError('excluded_sym_names must be a list of strings representing'\n                         ' the names of the symbols that will not be quantized,'\n                         f' while received type {str(type(excluded_sym_names))}')\n\n    if excluded_op_names is None:\n        excluded_op_names = []\n    if not isinstance(excluded_op_names, list):\n        raise ValueError('excluded_op_names must be a list of strings representing'\n                         ' the names of the operators that will not be quantized,'\n                         f' while received type {str(type(excluded_op_names))}')\n\n    if logger:\n        os.environ['MXNET_QUANTIZATION_VERBOSE'] = '1'\n        logger.info('Quantizing symbol')\n    if quantized_dtype not in ('int8', 'uint8', 'auto'):\n        raise ValueError(f'unknown quantized_dtype {quantized_dtype} received,'\n                         ' expected `int8`, `uint8` or `auto`')\n    if quantize_granularity not in ('tensor-wise', 'channel-wise'):\n        raise ValueError(f'unkonwn quantize_granularity {quantize_granularity} received,'\n                         ' expected `tensor-wise` or `channel-wise`.')\n    qsym, calib_layers = _quantize_symbol(sym, device, excluded_symbols=excluded_sym_names,\n                                          excluded_operators=excluded_op_names,\n                                          offline_params=list(arg_params.keys()),\n                                          quantized_dtype=quantized_dtype,\n                                          quantize_mode=quantize_mode,\n                                          quantize_granularity=quantize_granularity)\n    min_max_dict = {}\n    if calib_mode is not None and calib_mode != 'none':\n        if not isinstance(device, Device):\n            raise ValueError(f'currently only supports single device, while received {str(device)}')\n        if calib_data is None:\n            raise ValueError(f'calib_data must be provided when calib_mode={calib_mode}')\n        if not isinstance(calib_data, mx.gluon.data.DataLoader):\n            raise ValueError(f'calib_data must be of DataLoader type when calib_mode={calib_mode},'\n                             f' while received type {str(type(calib_data))}')\n\n        inputs = [mx.sym.var(dname) for dname in data_names]\n        param_dict = arg_params\n        param_dict.update(aux_params)\n        sym_block = mx.gluon.SymbolBlock(sym, inputs)\n        sym_block.load_dict(param_dict)\n\n        if calib_mode == 'entropy':\n            collector = _LayerHistogramCollector(quantized_dtype=quantized_dtype,\n                                                 include_layers=calib_layers,\n                                                 logger=logger)\n        elif calib_mode == 'naive':\n            collector = _LayerOutputMinMaxCollector(quantized_dtype=quantized_dtype,\n                                                    include_layers=calib_layers,\n                                                    logger=logger)\n\n        else:\n            raise ValueError(f'unknown calibration mode {calib_mode} received,'\n                             ' expected `none`, `naive`, or `entropy`')\n\n        num_batches = _collect_layer_statistics(sym_block, calib_data, collector,\n                                                len(inputs), num_calib_batches, logger)\n        if logger:\n            logger.info(f'Collected layer output min/max values from FP32 model using {num_batches} batches')\n            logger.info('Performing calibration post collecting operations')\n\n        min_max_dict = collector.post_collect()\n        qsym = _calibrate_quantized_sym(qsym, min_max_dict)\n\n    if logger:\n        logger.info('Quantizing parameters')\n    qarg_params = _quantize_params(qsym, arg_params, min_max_dict)\n\n    if is_np_array():\n        qsym = qsym.as_np_ndarray()\n\n    return qsym, qarg_params, aux_params\n\n@wrap_ctx_to_device_func\ndef quantize_model_onednn(sym, arg_params, aux_params, data_names=('data',),\n                          device=cpu(), excluded_sym_names=None, excluded_op_names=None,\n                          calib_mode='entropy', calib_data=None, num_calib_batches=None,\n                          quantized_dtype='int8', quantize_mode='smart',\n                          quantize_granularity='tensor-wise', logger=None):\n    \"\"\"User-level API for generating a fusion + quantized model from a FP32 model\n    w/ or w/o calibration with oneDNN.\n    The backend quantized operators are only enabled for Linux systems. Please do not run\n    inference using the quantized models on Windows for now.\n\n    Parameters\n    ----------\n    all\n        :ref:`As in quantize_model<quantize_model_params>`\n\n\n    Returns\n    -------\n    quantized_model: tuple\n        A tuple of quantized symbol, quantized arg_params, and aux_params.\n    \"\"\"\n    if not isinstance(device, Device):\n        raise ValueError(f'currently only supports single device, while received {str(device)}')\n    if device.device_type != 'cpu':\n        raise ValueError(\n            'quantize_model_onednn only support Intel cpu platform with oneDNN Backend')\n\n    sym = sym.optimize_for(backend='ONEDNN_QUANTIZE')\n\n    qsym, qarg_params, aux_params = quantize_model(sym=sym, arg_params=arg_params, aux_params=aux_params,\n                                                   data_names=data_names, device=device,\n                                                   excluded_sym_names=excluded_sym_names,\n                                                   excluded_op_names=excluded_op_names,\n                                                   calib_mode=calib_mode, calib_data=calib_data,\n                                                   num_calib_batches=num_calib_batches,\n                                                   quantized_dtype=quantized_dtype, quantize_mode=quantize_mode,\n                                                   quantize_granularity=quantize_granularity, logger=logger)\n\n    qsym = qsym.optimize_for(backend='ONEDNN_QUANTIZE')\n\n    return qsym, qarg_params, aux_params\n\ndef quantize_graph(sym, arg_params, aux_params, device=cpu(),\n                   excluded_sym_names=None, excluded_op_names=None,\n                   calib_mode='entropy', quantized_dtype='int8',\n                   quantize_mode='full', quantize_granularity='tensor-wise',\n                   LayerOutputCollector=None, logger=None):\n    \"\"\"User-level API for generating a quantized model from a FP32 model w/o calibration\n    and a collector for naive or entropy calibration.\n    The backend quantized operators are only enabled for Linux systems. Please do not run\n    inference using the quantized models on Windows for now.\n\n    Parameters\n    ----------\n    sym : Symbol\n        Defines the structure of a neural network for FP32 data types.\n    device : Device\n        Defines the device that users want to run forward propagation on the calibration\n        dataset for collecting layer output statistics. Currently, only supports single device.\n    arg_params : dict\n        Dictionary of name to `NDArray`.\n    aux_params : dict\n        Dictionary of name to `NDArray`.\n    excluded_sym_names : list of strings\n        A list of strings representing the names of the symbols that users want to excluding\n        from being quantized.\n    excluded_op_names : list of strings\n        A list of strings representing the names of the operators that users want to excluding\n    calib_mode : str\n        If calib_mode='none', no calibration will be used and the thresholds for\n        requantization after the corresponding layers will be calculated at runtime by\n        calling min and max operators. The quantized models generated in this\n        mode are normally 10-20% slower than those with calibrations during inference.\n        If calib_mode='naive', the min and max values of the layer outputs from a calibration\n        dataset will be directly taken as the thresholds for quantization.\n        If calib_mode='entropy' (default mode), the thresholds for quantization will be\n        derived such that the KL divergence between the distributions of FP32 layer outputs and\n        quantized layer outputs is minimized based upon the calibration dataset.\n    quantized_dtype : str\n        The quantized destination type for input data. Currently support 'int8'\n        , 'uint8' and 'auto'. 'auto' means automatically select output type according to calibration result.\n        Default value is 'int8'.\n    quantize_mode : str\n        The mode that quantization pass to apply. Support 'full' and 'smart'.\n        'full' means quantize all operator if possible.\n        'smart' means quantization pass will smartly choice which operator should be quantized.\n    quantize_granularity : str\n        The granularity of quantization, currently supports 'tensor-wise' and 'channel-wise'\n        quantization. The default value is 'tensor-wise'.\n    LayerOutputCollector : subclass of CalibrationCollector\n        For custom calibration method usage.\n        Passed object's include_layers attribute will be feed with names of layers which needs calibration\n    logger : Object\n        A logging object for printing information during the process of quantization.\n    Returns\n    -------\n    quantized_model : tuple\n        A tuple of quantized symbol, quantized arg_params, aux_params and collector.\n    \"\"\"\n    if excluded_sym_names is None:\n        excluded_sym_names = []\n    if not isinstance(excluded_sym_names, list):\n        raise ValueError('excluded_sym_names must be a list of strings representing'\n                         ' the names of the symbols that will not be quantized,'\n                         f' while received type {str(type(excluded_sym_names))}')\n    if not isinstance(device, Device):\n        raise ValueError(f'currently only supports single device, while received {str(device)}')\n    if logger:\n        os.environ['MXNET_QUANTIZATION_VERBOSE'] = '1'\n        logger.info('Quantizing graph')\n    if quantized_dtype not in ('int8', 'uint8', 'auto'):\n        raise ValueError(f'unknown quantized_dtype {quantized_dtype} received,'\n                         ' expected `int8`, `uint8` or `auto`')\n    if quantize_granularity not in ('tensor-wise', 'channel-wise'):\n        raise ValueError(f'unkonwn quantize_granularity {quantize_granularity} received,'\n                         ' expected `tensor-wise` or `channel-wise`.')\n    qsym, calib_layers = _quantize_symbol(sym, device, excluded_symbols=excluded_sym_names,\n                                          excluded_operators=excluded_op_names,\n                                          offline_params=list(arg_params.keys()),\n                                          quantized_dtype=quantized_dtype,\n                                          quantize_mode=quantize_mode,\n                                          quantize_granularity=quantize_granularity)\n\n    collector = None\n    if calib_mode is not None and calib_mode != 'none':\n        if calib_mode == 'entropy':\n            collector = _LayerHistogramCollector(quantized_dtype=quantized_dtype,\n                                                 include_layers=calib_layers, logger=logger)\n            if logger:\n                logger.info(\n                    'Create a layer output collector for entropy calibration.')\n        elif calib_mode == 'naive':\n            collector = _LayerOutputMinMaxCollector(quantized_dtype=quantized_dtype,\n                                                    include_layers=calib_layers, logger=logger)\n            if logger:\n                logger.info(\n                    'Create a layer output minmax collector for naive calibration')\n        elif calib_mode == 'custom' and LayerOutputCollector is not None:\n            if not isinstance(LayerOutputCollector, CalibrationCollector):\n                raise ValueError('LayerOutputCollecotr must be a subclass of a CalibrationCollector class,'\n                                 f' but it is {LayerOutputCollector.__class__}')\n            collector = LayerOutputCollector\n\n            # Inject layer names that need calibration to collector\n            if hasattr(collector, \"include_layers\"):\n                if collector.include_layers is not None:\n                    logger.info('Custom collector has set include_layers attribute. '\n                                'Calibration layers not passed')\n                else:\n                    collector.include_layers = calib_layers\n            if logger:\n                logger.info(\n                    'Create a custom layer output minmax collector for calibration')\n        else:\n            raise ValueError(f'unknown calibration mode {calib_mode} received,'\n                             ' expected `none`, `naive`, `entropy` or `custom`')\n        if logger:\n            logger.info('Collector created, please use set_monitor_callback'\n                        ' to collect calibration information.')\n\n    if logger:\n        logger.info('Quantizing parameters')\n    qarg_params = _quantize_params(qsym, arg_params, min_max_dict={})\n\n    if is_np_array():\n        qsym = qsym.as_np_ndarray()\n\n    return qsym, qarg_params, aux_params, collector, calib_layers\n\ndef calib_graph(qsym, arg_params, aux_params, collector,\n                calib_mode='entropy', logger=None):\n    \"\"\"User-level API for calibrating a quantized model using a filled collector.\n    The backend quantized operators are only enabled for Linux systems. Please do not run\n    inference using the quantized models on Windows for now.\n\n    Parameters\n    ----------\n    qsym : Symbol\n        Defines the structure of a neural network for INT8 data types.\n    arg_params : dict\n        Dictionary of name to `NDArray`.\n    aux_params : dict\n        Dictionary of name to `NDArray`.\n    collector : function\n        layer collector for naive or entropy calibration.\n    calib_mode : str\n        If calib_mode='none', no calibration will be used and the thresholds for\n        requantization after the corresponding layers will be calculated at runtime by\n        calling min and max operators. The quantized models generated in this\n        mode are normally 10-20% slower than those with calibrations during inference.\n        If calib_mode='naive', the min and max values of the layer outputs from a calibration\n        dataset will be directly taken as the thresholds for quantization.\n        If calib_mode='entropy' (default mode), the thresholds for quantization will be\n        derived such that the KL divergence between the distributions of FP32 layer outputs and\n        quantized layer outputs is minimized based upon the calibration dataset.\n    quantized_dtype : str\n        The quantized destination type for input data. Currently support 'int8'\n        , 'uint8' and 'auto'. 'auto' means automatically select output type according to calibration result.\n        Default value is 'int8'.\n    logger : Object\n        A logging object for printing information during the process of quantization.\n    Returns\n    -------\n    quantized_model : tuple\n        A tuple of calibrated symbol, quantized arg_params, aux_params.\n    \"\"\"\n    min_max_dict = {}\n    if calib_mode is not None and calib_mode != 'none':\n        if calib_mode in ('entropy', 'naive', 'custom'):\n            min_max_dict = collector.post_collect()\n\n        else:\n            raise ValueError(f'unknown calibration mode {calib_mode} received,'\n                             ' expected `none`, `naive`, `entropy` or `custom`')\n        qsym = _calibrate_quantized_sym(qsym, min_max_dict)\n    else:\n        raise ValueError('Please set calibration mode to naive, entropy or custom (with custom CalibrationCollector)')\n\n    if logger:\n        logger.info('Quantizing parameters')\n    qarg_params = _quantize_params(qsym, arg_params, min_max_dict)\n\n    if is_np_array():\n        qsym = qsym.as_np_ndarray()\n\n    return qsym, qarg_params, aux_params\n\n@wrap_ctx_to_device_func\ndef quantize_net(network, quantized_dtype='auto', quantize_mode='full', quantize_granularity='tensor-wise',\n                 exclude_layers=None, exclude_layers_match=None, exclude_operators=None,\n                 calib_data=None, data_shapes=None, calib_mode='none',\n                 num_calib_batches=None, device=cpu(), LayerOutputCollector=None, logger=None):\n    \"\"\"User-level API for Gluon users to generate a quantized SymbolBlock from a FP32 HybridBlock w/ or w/o calibration.\n    The backend quantized operators are only enabled for Linux systems. Please do not run\n    inference using the quantized models on Windows for now.\n\n    Parameters\n    ----------\n    network : Gluon HybridBlock\n        Defines the structure of a neural network for FP32 data types.\n    quantized_dtype : str\n        The quantized destination type for input data. Currently support 'int8'\n        , 'uint8' and 'auto'. 'auto' means automatically select output type according to calibration result.\n        Default value is 'int8'.\n    quantize_mode : str\n        The mode that quantization pass to apply. Support 'full' and 'smart'.\n        'full' means quantize all operator if possible.\n        'smart' means quantization pass will smartly choice which operator should be quantized.\n    quantize_granularity: str\n        The granularity of quantization, currently supports 'tensor-wise' and 'channel-wise'\n        quantization. The default value is 'tensor-wise'.\n    exclude_layers : list of strings\n        A list of strings representing the names of the symbols that users want to excluding\n    exclude_layers_match : list of strings\n        A list of strings wildcard matching the names of the symbols that users want to excluding\n        from being quantized.\n    exclude_operators : list of strings\n        A list of strings representing the names of the operators that users want to excluding\n    calib_data : gluon.DataLoader\n        A iterable data loading object.\n    data_shapes : list of DataDesc or list of tuple\n        A list of data shapes. Required if calib_data is not provided. In case of tuples,\n        the names of inputs are generated.\n    calib_mode : str\n        If calib_mode='none', no calibration will be used and the thresholds for\n        requantization after the corresponding layers will be calculated at runtime by\n        calling min and max operators. The quantized models generated in this\n        mode are normally 10-20% slower than those with calibrations during inference.\n        If calib_mode='naive', the min and max values of the layer outputs from a calibration\n        dataset will be directly taken as the thresholds for quantization.\n        If calib_mode='entropy' (default mode), the thresholds for quantization will be\n        derived such that the KL divergence between the distributions of FP32 layer outputs and\n        quantized layer outputs is minimized based upon the calibration dataset.\n        If calib_mode='custom', the provided LayerOutputCollector will be used to determine\n        the thresholds for quantization. For more information refer to CalibrationCollector\n        documentation.\n    num_calib_batches : int or None\n        The maximum number of batches that user would like to use for calibration. If not provided,\n        the whole calibration dataset will be used.\n    device : Device\n        Defines the device that users want to run forward propagation on the calibration\n        dataset for collecting layer output statistics. Currently, only supports single device.\n    LayerOutputCollector : subclass of CalibrationCollector\n        For `custom` calibration method usage.\n        Passed object's include_layers attribute will be feed with names of layers which needs calibration\n    logger : Object\n        A logging object for printing information during the process of quantization.\n\n    Returns\n    -------\n    network : Gluon SymbolBlock\n        Defines the structure of a neural network for INT8 data types.\n    \"\"\"\n    from ..gluon import SymbolBlock\n\n    if device != mx.cpu():\n        raise ValueError('Quantization currently supports only CPU device')\n    backend = 'ONEDNN_QUANTIZE'\n\n    network.hybridize(static_alloc=False, static_shape=False)\n    data_types = None\n    if data_shapes is None:\n        if calib_data is None:\n            raise ValueError('At least one of data_shapes or calib_data has to be provided.')\n\n        if isinstance(calib_data, mx.gluon.data.DataLoader):\n            x = iter(calib_data)\n            batch = next(x)\n            if isinstance(batch, list):\n                data_shapes = _multilist_iterator(batch, lambda x: x.shape)\n                data_types = _multilist_iterator(batch, lambda x: x.dtype)\n            else:\n                data_shapes = [batch.shape]\n                data_types = [batch.dtype]\n        else:\n            raise ValueError('calib_data expects mx.gluon.data.DataLoader')\n\n    if data_types is None:\n        data_types = _multilist_iterator(data_shapes, lambda x: mx_real_t)\n\n    data_descs = _generate_list_of_data_desc(data_shapes, data_types)\n\n    num_inputs = len(data_descs)\n    data_nd = []\n    arr_fn = mx.np if is_np_array() else mx.nd\n    data_nd = _multilist_iterator(data_descs, lambda d, F=arr_fn: F.zeros(shape=d.shape, dtype=d.dtype))\n\n    while True:\n        try:\n            network(*data_nd)\n        except (ValueError, TypeError) as err:\n            if logger:\n                logger.warning(err)\n                logger.warning(\"Deduced input data descriptors failed to run forward pass.\"\n                               \" Trying again with one less input.\")\n            del data_nd[-1]\n            num_inputs -= 1\n            data_shapes = [b.shape for b in data_nd]\n            data_types = [b.dtype for b in data_nd]\n            data_descs = _generate_list_of_data_desc(data_shapes, data_types)\n            continue\n        else:\n            break\n\n    symnet, params = network.export(None)\n    symnet = symnet.optimize_for(backend=backend)\n\n    if is_np_array():\n        symnet = symnet.as_np_ndarray()\n\n    args, auxs = dict(), dict()\n    for k, v in params.items():\n        ptype, pname = k[:3], k[4:]\n        if ptype == \"arg\":\n            args[pname] = v\n        else:\n            auxs[pname] = v\n\n    if exclude_layers is None:\n        exclude_layers = []\n    if exclude_layers_match is None:\n        exclude_layers_match = []\n    if exclude_operators is None:\n        exclude_operators = []\n    for name_match in exclude_layers_match:\n        for layers in list(symnet.get_internals()):\n            if layers.name.find(name_match) != -1:\n                exclude_layers.append(layers.name)\n    if logger:\n        logger.info(f'These layers have been excluded {exclude_layers}')\n\n    qsym, qarg_params, aux_params, collector, _ = quantize_graph(\n        sym=symnet, arg_params=args, aux_params=auxs, device=device,\n        excluded_sym_names=exclude_layers, excluded_op_names=exclude_operators,\n        calib_mode=calib_mode, quantized_dtype=quantized_dtype, quantize_mode=quantize_mode,\n        quantize_granularity=quantize_granularity, LayerOutputCollector=LayerOutputCollector,\n        logger=logger)\n\n    if calib_mode is not None and calib_mode != 'none':\n        if not isinstance(device, Device):\n            raise ValueError(\n                f'currently only supports single device, while received {str(device)}')\n        if calib_data is None:\n            raise ValueError(\n                f'calib_data must be provided when calib_mode={calib_mode}')\n        if calib_mode in ['naive', 'entropy', 'custom']:\n            inputs = _multilist_iterator(data_descs, lambda dd: mx.sym.var(dd.name))\n            calib_net = SymbolBlock(symnet, inputs)\n            for k, v in calib_net.collect_params().items():\n               v.grad_req = 'null'\n\n            calib_net.load_dict(params, cast_dtype=True, dtype_source='saved')\n            calib_net.hybridize(static_alloc=False, static_shape=False)\n            num_batches = _collect_layer_statistics(calib_net, calib_data, collector, num_inputs,\n                                                    num_calib_batches, logger)\n\n            if logger:\n                logger.info(f'Collected layer output values from FP32 model using {num_batches} batches')\n\n            qsym, qarg_params, aux_params = calib_graph(\n                qsym=qsym, arg_params=args, aux_params=auxs, collector=collector,\n                calib_mode=calib_mode, logger=logger)\n        else:\n            raise ValueError('calib_mode has to be one of: naive, entropy, custom')\n    elif calib_mode is not None and calib_mode == 'none':\n        inputs = _multilist_iterator(data_descs, lambda dd: mx.sym.var(dd.name))\n\n    net = SymbolBlock(qsym, inputs)\n    for k, v in net.collect_params().items():\n        v.grad_req = 'null'\n\n    all_params = {(f'arg:{k}'): v.as_in_context(cpu()) for k, v in qarg_params.items()}\n    all_params.update({(f'aux:{k}'): v.as_in_context(cpu()) for k, v in aux_params.items()})\n    net.load_dict(all_params, cast_dtype=True, dtype_source='saved')\n    net.optimize_for(data_nd, backend=backend, skip_infer=True)\n    return net\n"
  },
  {
    "path": "python/mxnet/contrib/symbol.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Symbol namespace used to register contrib functions\"\"\"\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/contrib/tensorboard.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"TensorBoard functions that can be used to log various status during epoch.\"\"\"\n\nimport logging\n\n\nclass LogMetricsCallback(object):\n    \"\"\"Log metrics periodically in TensorBoard.\n    This callback works almost same as `callback.Speedometer`, but write TensorBoard event file\n    for visualization. For more usage, please refer https://github.com/dmlc/tensorboard\n\n    Parameters\n    ----------\n    logging_dir : str\n        TensorBoard event file directory.\n        After that, use `tensorboard --logdir=path/to/logs` to launch TensorBoard visualization.\n    prefix : str\n        Prefix for a metric name of `scalar` value.\n        You might want to use this param to leverage TensorBoard plot feature,\n        where TensorBoard plots different curves in one graph when they have same `name`.\n        The follow example shows the usage(how to compare a train and eval metric in a same graph).\n\n    Examples\n    --------\n    >>> # log train and eval metrics under different directories.\n    >>> training_log = 'logs/train'\n    >>> evaluation_log = 'logs/eval'\n    >>> # in this case, each training and evaluation metric pairs has same name,\n    >>> # you can add a prefix to make it separate.\n    >>> batch_end_callbacks = [mx.contrib.tensorboard.LogMetricsCallback(training_log)]\n    >>> eval_end_callbacks = [mx.contrib.tensorboard.LogMetricsCallback(evaluation_log)]\n    >>> # run\n    >>> model.fit(train,\n    >>>     ...\n    >>>     batch_end_callback = batch_end_callbacks,\n    >>>     eval_end_callback  = eval_end_callbacks)\n    >>> # Then use `tensorboard --logdir=logs/` to launch TensorBoard visualization.\n    \"\"\"\n    def __init__(self, logging_dir, prefix=None):\n        self.prefix = prefix\n        try:\n            from mxboard import SummaryWriter\n            self.summary_writer = SummaryWriter(logging_dir)\n        except ImportError:\n            logging.error('You can install mxboard via `pip install mxboard`.')\n\n    def __call__(self, param):\n        \"\"\"Callback to log training speed and metrics in TensorBoard.\"\"\"\n        if param.eval_metric is None:\n            return\n        name_value = param.eval_metric.get_name_value()\n        for name, value in name_value:\n            if self.prefix is not None:\n                name = f'{self.prefix}-{name}'\n            self.summary_writer.add_scalar(name, value, global_step=param.epoch)\n"
  },
  {
    "path": "python/mxnet/contrib/tensorrt.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\" Module to enable the use of TensorRT optimized graphs.\"\"\"\nimport os\n\ndef set_use_fp16(status):\n    \"\"\"\n    Set an environment variable which will enable or disable the use of FP16 precision in\n    TensorRT\n    Note: The mode FP16 force the whole TRT node to be executed in FP16\n    :param status: Boolean, True if TensorRT should run in FP16, False for FP32\n    \"\"\"\n    os.environ[\"MXNET_TENSORRT_USE_FP16\"] = str(int(status))\n\ndef get_use_fp16():\n    \"\"\"\n    Get an environment variable which describes if TensorRT is currently running in FP16\n    :return: Boolean, true if TensorRT is running in FP16, False for FP32\n    \"\"\"\n    return bool(int(os.environ.get(\"MXNET_TENSORRT_USE_FP16\", 1)) == 1)\n\ndef init_tensorrt_params(sym, arg_params, aux_params):\n    \"\"\"\n    Set weights in attributes of TensorRT nodes\n    :param sym: Symbol, the symbol graph should contains some TensorRT nodes\n    :param arg_params: arg_params\n    :param aux_params: aux_params\n    :return arg_params, aux_params: remaining params that are not in TensorRT nodes\n    \"\"\"\n    arg_params = arg_params.copy()\n    aux_params = aux_params.copy()\n    for s in sym.get_internals():\n        new_params_names = \"\"\n        tensorrt_params = {}\n        if 'subgraph_params_names' in s.list_attr():\n            keys = s.list_attr()['subgraph_params_names'].split(';')\n            for k in keys:\n                if k in arg_params:\n                    new_params_names += k + \";\"\n                    tensorrt_params['subgraph_param_' + k] = arg_params[k]\n                    arg_params.pop(k)\n                elif k in aux_params:\n                    new_params_names += k + \";\"\n                    tensorrt_params['subgraph_param_' + k] = aux_params[k]\n                    aux_params.pop(k)\n            new_attrs = {}\n            for k, v in tensorrt_params.items():\n                new_attrs[k] = str(v.handle.value)\n            if len(new_attrs) > 0:\n                s._set_attr(**new_attrs)\n                s._set_attr(subgraph_params_names=new_params_names[:-1])\n    return arg_params, aux_params\n"
  },
  {
    "path": "python/mxnet/contrib/text/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"This module includes utilities for indexing and embedding text.\"\"\"\n\nfrom . import utils\nfrom . import vocab\nfrom . import embedding\n"
  },
  {
    "path": "python/mxnet/contrib/text/_constants.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\n\"\"\"Read text files and load embeddings.\"\"\"\n\nUNKNOWN_IDX = 0\n\nAPACHE_REPO_URL = 'https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/'\n\nGLOVE_PRETRAINED_FILE_SHA1 = \\\n    {'glove.42B.300d.zip': 'f8e722b39578f776927465b71b231bae2ae8776a',\n     'glove.6B.zip': 'b64e54f1877d2f735bdd000c1d7d771e25c7dfdc',\n     'glove.840B.300d.zip': '8084fbacc2dee3b1fd1ca4cc534cbfff3519ed0d',\n     'glove.twitter.27B.zip': 'dce69c404025a8312c323197347695e81fd529fc'}\n\nGLOVE_PRETRAINED_ARCHIVE_SHA1 = \\\n    {'glove.42B.300d.txt': '876767977d6bd4d947c0f84d44510677bc94612a',\n     'glove.6B.50d.txt': '21bf566a9d27f84d253e0cd4d4be9dcc07976a6d',\n     'glove.6B.100d.txt': '16b1dbfaf35476790bd9df40c83e2dfbd05312f1',\n     'glove.6B.200d.txt': '17d0355ddaa253e298ede39877d1be70f99d9148',\n     'glove.6B.300d.txt': '646443dd885090927f8215ecf7a677e9f703858d',\n     'glove.840B.300d.txt': '294b9f37fa64cce31f9ebb409c266fc379527708',\n     'glove.twitter.27B.25d.txt':\n         '767d80889d8c8a22ae7cd25e09d0650a6ff0a502',\n     'glove.twitter.27B.50d.txt':\n         '9585f4be97e286339bf0112d0d3aa7c15a3e864d',\n     'glove.twitter.27B.100d.txt':\n         '1bbeab8323c72332bd46ada0fc3c99f2faaa8ca8',\n     'glove.twitter.27B.200d.txt':\n         '7921c77a53aa5977b1d9ce3a7c4430cbd9d1207a'}\n\nFAST_TEXT_ARCHIVE_SHA1 = \\\n    {'crawl-300d-2M.zip': 'bb40313d15837ceecc1e879bc954e9be04b17c3c',\n     'wiki.aa.zip': '0d85feb259e17d5258f38b2b615a2b87cd628427',\n     'wiki.ab.zip': '7a8c555b9cf3837c9b31c901e9e0142209990365',\n     'wiki.ace.zip': '51555fccbe53b726f6c86a84d704c026a78dd02f',\n     'wiki.ady.zip': '725d2c30c03001c941ac4084549c55c7f8e1d766',\n     'wiki.af.zip': '1a18d34e1b60433b837f5850750a44ca3845323d',\n     'wiki.ak.zip': 'daecc2303cfd05bc6c33b24d78c14e0d7f33e3a7',\n     'wiki.als.zip': '38851192e0b556e566be6c3c93370abf9867e525',\n     'wiki.am.zip': '4576e0121448564b07f448e05e287236343f17c1',\n     'wiki.ang.zip': '9c03da3b06d4becef5d387b9a61438b9362fc36a',\n     'wiki.an.zip': '170f60bdd161cf8e4b5e018acd7d36e8bfc457a6',\n     'wiki.arc.zip': 'c8dad8b00865bf736b087e7b323999ab404bda29',\n     'wiki.ar.zip': '34e9869daa463fdc5609040ff33a03e67512e9fd',\n     'wiki.arz.zip': '2d2790e11e401d46e1bce2970ee5264d5678a32b',\n     'wiki.ast.zip': '1136515e2de556c077324bcd42ffe7f40c8d94c6',\n     'wiki.as.zip': 'f9efde3e4ccda4a1e93fa275a3210f74036e9e46',\n     'wiki.av.zip': '9f8568a3e094a48de4a3b6bea3bdb6fd7e875a08',\n     'wiki.ay.zip': 'f09a422cedc6a0f15fbf30d290febe8057de83db',\n     'wiki.azb.zip': 'd8895581050b9fdb5a10dfec3e27910a150b6faf',\n     'wiki.az.zip': '2a34c2db872597ba3e345ce8b7db138241f9efbf',\n     'wiki.bar.zip': 'd6e40135a6f4ba7a07fab11633034eccb1b05d0a',\n     'wiki.bat_smg.zip': '5d08bd04f0515a36723776c0682b3de0f11d4264',\n     'wiki.ba.zip': '412ac2f3bf9a605e56e2b0990bb0baed41ddf3b0',\n     'wiki.bcl.zip': 'd3717cda357e08390cb57a64e07f5c7b7768d5be',\n     'wiki.be.zip': 'b691e63b8080af23cc37f5f2b21b3154e464c425',\n     'wiki.bg.zip': '08509a510a95e2a8905c19d83faf40d614d2268b',\n     'wiki.bh.zip': 'a812600c6454b779d442b7680e3867e15d895095',\n     'wiki.bi.zip': 'd0d4a3f57419424815f77b3951ef9c7336f6adf5',\n     'wiki.bjn.zip': '0d81879ff7611380896eac6059bb677a5b3fe308',\n     'wiki.bm.zip': 'f3a2a1a8dbc94973a74343c059595a310a66665b',\n     'wiki.bn.zip': 'b3bc70520edf3963c2217873ff5c2537d3545650',\n     'wiki.bo.zip': '2be9fe7701d6a8501461df7bd98fee26859cf83a',\n     'wiki.bpy.zip': 'd44b9267bb4f86e3e43972a6a952cc0ccf90dd3c',\n     'wiki.br.zip': '4bfa66f1ea5aa5cad736eccaa211f6025596bcd6',\n     'wiki.bs.zip': '40c560c5994ab50485d08eeaffd88740f30236ab',\n     'wiki.bug.zip': 'bc7cd87bb067ac477000259cd4f95f45bfb6e4df',\n     'wiki.bxr.zip': '8396fd67ef53f3123540766788a0db54734c4f1a',\n     'wiki.ca.zip': '8f5d3caf0f5d223b2771ec44f7e620e396974fb2',\n     'wiki.cbk_zam.zip': '0af3be50823b564433455d10c8753df88461458f',\n     'wiki.cdo.zip': '19024215aa0c13872c027fc6127b5d7506198b5f',\n     'wiki.ceb.zip': '96374428bf36a43983ba4307d7f6fb5ab52a6c6a',\n     'wiki.ce.zip': 'b27f1a8da448bc9315e15d4261519c64f00de8eb',\n     'wiki.cho.zip': '20944e34c2b58f14adb849dd5a6f5168c7affdea',\n     'wiki.chr.zip': 'b7f41ee3fa76e933e0b5ad6b793c507fc19afe98',\n     'wiki.chy.zip': '4ef66004a609c724fd7d8aab2877f7634323d43f',\n     'wiki.ch.zip': '7f73678b685c9b5f5d6eea9bc00322cfc18d40cb',\n     'wiki.ckb.zip': 'b7db2805526ad8bed878af257b32ca9ba814855f',\n     'wiki.co.zip': '1b9e19b11763cb87ca00520dbdd6ada565547c9c',\n     'wiki.crh.zip': '792003bae25c4471d25721440002c983fa5af020',\n     'wiki.cr.zip': '875e4aa0de8a829e57f6c8e13d43cac5103210de',\n     'wiki.csb.zip': 'fa776014c4c83487d7cb2485bd08eaf6739d9dca',\n     'wiki.cs.zip': 'dca18cb80460522cd281ccc3c9922cf2b3c08b81',\n     'wiki.cu.zip': 'ed23b48ba3193181a358d7a73005afa7655a4fc3',\n     'wiki.cv.zip': '27ccd50942c9c218e00365ee293fa0c3087a7646',\n     'wiki.cy.zip': '78940d5be2969b82c99f785bda2ac5f4e18e149c',\n     'wiki.da.zip': 'a45077d9d73328bd6a96efdba1b31ed9a3639dcd',\n     'wiki.de.zip': '0d9e4bf80100b46237dcb73cfefe390103e7e827',\n     'wiki.diq.zip': '0eef7d9e2f0ce3f100a22dc8fcede9449e466528',\n     'wiki.dsb.zip': '903cd80550931effba1d4e52a19c22592837d11c',\n     'wiki.dv.zip': '3fa06719641ff33ac8a5439d330a8108521da1e7',\n     'wiki.dz.zip': '8bf3937971c3c996493c30b264cb8268627d7bd6',\n     'wiki.ee.zip': 'e66bc50013d884fe69f4f67ba44af2e34fe97927',\n     'wiki.el.zip': '3015f358036658fb126d42fa794d67a90c5b91ad',\n     'wiki.eml.zip': '5be541be6115af5914ac2b8118a09232b771123b',\n     'wiki.en.zip': '7f83d578a31a8168423c77ea25ad381494a5e920',\n     'wiki.eo.zip': 'e7612df98c37cb872f0edc3c3e21dcd2f80a4d69',\n     'wiki.es.zip': '1b7668b23db26810ea433173ce0c11281e801f74',\n     'wiki.et.zip': 'aa31004e7b8ebf359e166b8ea6b8e6f77fac190f',\n     'wiki.eu.zip': '8d7699451cbac4d69750caa8d58b4740cc72e0ca',\n     'wiki.ext.zip': '3aeb4d77c48eb503b26ceb2a76a0a7d841124a71',\n     'wiki.fa.zip': '08b6e805c8623fba526143d46f4685549c4380a6',\n     'wiki.ff.zip': '64f690eda733a6fb4f794e42eb6ff05f09ec1d38',\n     'wiki.fiu_vro.zip': '35c3fdcec0f0dc1ce303212967ea59936641daee',\n     'wiki.fi.zip': '252299a2a59cc0ac07ba25f9458afc26bbac669f',\n     'wiki.fj.zip': '004d1279c27324d02b961341cf0d6ee06dbe8966',\n     'wiki.fo.zip': '12f1d6360d4867cdebcc93be87c024a4709d1af5',\n     'wiki.frp.zip': '8a0f636b5440a9aab38014efada9edfdf94150d5',\n     'wiki.frr.zip': '7c9e7b8109b98aa39b303dd77d837b37e96d4113',\n     'wiki.fr.zip': 'd906e68760153d771e5982009b0150e913254b2d',\n     'wiki.fur.zip': 'd5d2ae08696ed074a581eac563a60eb85467a792',\n     'wiki.fy.zip': '342609d29882fae0a3b402d8ea1478606be0d93b',\n     'wiki.gag.zip': 'f2b91f89dd9b9a1301727476f7823b7260b5f129',\n     'wiki.gan.zip': 'd3ad3c1151555266e1feb9f98b066ee31ee5f410',\n     'wiki.ga.zip': '798b0c26783c7af05d9c4f899ca9fddafeb1e0a1',\n     'wiki.gd.zip': '49085fa182a528bdc51f10e99bef33c88c1e3112',\n     'wiki.glk.zip': '9e16727ffcc691483b69ecbcd331b1df2efa4bcd',\n     'wiki.gl.zip': 'c71c7e6601b2cbdc7930982fbeea636deddd107d',\n     'wiki.gn.zip': '493ccb583211217ccd23e0a43f42ba773bd94f78',\n     'wiki.gom.zip': '45bbd49750ddb7df5afe01fcfd5dda2958934dfa',\n     'wiki.got.zip': '669d018f72827fb965e5ef37e224e21f4682b2e5',\n     'wiki.gu.zip': '4afe874f7830f693e9f83508fc3fb444b33aebdf',\n     'wiki.gv.zip': '9411197eebc07775949d9bb6e440780a68502a5c',\n     'wiki.hak.zip': 'cd1e14bd5d50fa764883b148bda5b821375531e0',\n     'wiki.haw.zip': 'cacd4eb4e476bdd842e8014764b8ae380b346ed2',\n     'wiki.ha.zip': '14acc50950b451f40fe028fd08d042af44732398',\n     'wiki.he.zip': 'a9e2cd13bc2e55d83820c529bac1f518a7198bc0',\n     'wiki.hif.zip': 'dcdd488239deb0ede807cff263ddc972009c21f5',\n     'wiki.hi.zip': '15899ec17985bc0e1db1df497e1b4a51bba1982b',\n     'wiki.ho.zip': 'fde454bb4f3841ea5dde2bbf879138305a4d0b36',\n     'wiki.hr.zip': 'f5d33ba967f7c56538fa9f5f0093f6d634e9db44',\n     'wiki.hsb.zip': '64dc13c7645d2b65b8ba252bd8dfb1c616e8923a',\n     'wiki.ht.zip': 'cf50a5cadcf91aba9ab58d095d65f348e2375d12',\n     'wiki.hu.zip': 'b27f293caedf81a2d09204b11f52a7c8d7443643',\n     'wiki.hy.zip': '641b8666bc2168998989fae1b20a09d3428766bb',\n     'wiki.hz.zip': '1639f9f096de6fac84336a784a391ce73e523d62',\n     'wiki.ia.zip': '37640aaf8a25c02883190951337b5a6f0157d781',\n     'wiki.id.zip': '56ee0c7a38a6d232706932493eaa37b2a87667ee',\n     'wiki.ie.zip': '7c3a5d7f96c801570e2305f45a40d401fcc038b9',\n     'wiki.ig.zip': '405ebc2e8a959163c9f2f8dd015a0bcefd440111',\n     'wiki.ii.zip': '1ec1c7d95d61eeca2dbbd8e432caf88524aaf28e',\n     'wiki.ik.zip': 'e9d088c0d8d0ab420d6d0469c6a0fdb668f1833c',\n     'wiki.ilo.zip': 'cbc9754978ce55e86da2eb3db20579f4a1f19947',\n     'wiki.io.zip': '9e5ab1fd5c4f1094d111f501129e0eecccec69a0',\n     'wiki.is.zip': '0744e63636cf794e0a406c922827628a3dd415b7',\n     'wiki.it.zip': '29f4eb6a5d7dcf45b02b4d08a4a70dfae4c41200',\n     'wiki.iu.zip': 'fb2e8de825d554257768d363a3a09f711afb001b',\n     'wiki.jam.zip': '077cfb6de9d025aee4a5b2ea9ce15ada02f10a4f',\n     'wiki.ja.zip': '7940f6c2bc490c04902f0faf0562b92cae7136bf',\n     'wiki.jbo.zip': '3d086b6c9a369f197516cd0dc699a94612f45c6a',\n     'wiki.jv.zip': '2f68cb3436b27a25ddfa40fab3e2cd44574b437e',\n     'wiki.kaa.zip': '9fd5df362b7cb615f2267084d8b3fb8608be2693',\n     'wiki.kab.zip': '96abf1440ad21de58d7274d3a16885ef4a2efda4',\n     'wiki.ka.zip': '72ddb2382c87184fc05a93e89ed8aa4f54a62a0a',\n     'wiki.kbd.zip': '81dfc3c6f8581c2aa15342c84688b4ba59b81cc6',\n     'wiki.kg.zip': '4d07cabef6f804fc6432d3f630675ed4cbbdd49e',\n     'wiki.ki.zip': '59b5c31df227ff9454ad8b3a1d16b065620dbddf',\n     'wiki.kj.zip': '751b80c4a4d82dd217d3d2b3905eb39b349874d7',\n     'wiki.kk.zip': '7fb733a2405f421a7c49b756381a52965a8af205',\n     'wiki.kl.zip': '05a9d5c9bf12d8845356f88b546418d2e40f79c6',\n     'wiki.km.zip': 'da0a67028fa0244a2e7257ae259c2f7a7544dc66',\n     'wiki.kn.zip': '6cead946350b31fb2f353085fd00b8ea9c9ecc77',\n     'wiki.koi.zip': '0c61f83434404267527eaf583e89b4d8bb3a6a65',\n     'wiki.ko.zip': 'c0825282faf1e7af6820bd8b28d06c77760dcbe4',\n     'wiki.krc.zip': '0df3c3f0f89521299dab741be3d698b2c94c194e',\n     'wiki.kr.zip': '71651f046cef420fb28ca15e35720bb7747c4586',\n     'wiki.ksh.zip': '8b9ab88baa49e72e40a5a80bef98f3ea2afbdd07',\n     'wiki.ks.zip': '02af37f12753662c9e7bcac3b8786dfd2f298710',\n     'wiki.ku.zip': 'ca1d370b327ceca025884bf83139456024a3a978',\n     'wiki.kv.zip': '28b3617c5566f3182f14bf11a906456b227840ba',\n     'wiki.kw.zip': '075a02e8eaae26897c23898fb4d36f4e41e4d1d0',\n     'wiki.ky.zip': '771601a934cd4d0a98e5059f6389d2496e8dcf7c',\n     'wiki.lad.zip': '2788ba3f275d72299e877c96cde106bd8590f405',\n     'wiki.la.zip': '759f6365874442ab8e04d992b047f53ad74231a6',\n     'wiki.lbe.zip': 'c8105f1cf8a3d46ccfacff1d40a581f442b3c4a1',\n     'wiki.lb.zip': 'dac5af52364f2c0d3a0c794411465d1254f2fb48',\n     'wiki.lez.zip': '17331cb779dee8cb60f2734213af80d57acfcfad',\n     'wiki.lg.zip': 'fd4e2d67d1f098474053abc9a1984dfe4a2854b7',\n     'wiki.lij.zip': 'c29157f5e4d2b37c01cf6e389f03ddafef6acdb2',\n     'wiki.li.zip': '10490e49a12230af2127543da69c427f92c6508f',\n     'wiki.lmo.zip': 'cc44163572deddd78af6b006394f623cb21934fc',\n     'wiki.ln.zip': 'bf52699c5cbf79bedb2e2856d8a720189b6864f3',\n     'wiki.lo.zip': '3fd8a70d8e26071a365f10016875a4a4f15ffcee',\n     'wiki.lrc.zip': 'e262b4fcc55cba48d997cd06d006b82a5abe09a9',\n     'wiki.ltg.zip': 'df6a83f2fab35f9a2f97fd8d857cb1cfa59f331f',\n     'wiki.lt.zip': 'a738a3f29a6a5481082a7a9a41b2040b9cf537e4',\n     'wiki.lv.zip': '8e328d99aacaa021fcc51425caebc063e22e6cf4',\n     'wiki.mai.zip': 'e909de86c27eced2cb5f02f550da7fc2502b5eda',\n     'wiki.map_bms.zip': '192bf6b88f955746abb398893868482730585e3a',\n     'wiki.mdf.zip': '3d0d5da3c85bef8ae52f0fd17e314a1960a26d36',\n     'wiki.mg.zip': 'fe66055b63ce8771bf43f8dd543bbd967f8ea8b3',\n     'wiki.mhr.zip': '33514c98da3bd9602851db96fa3dd8192aac0674',\n     'wiki.mh.zip': 'dc77309103c6cfed7ff095b3f9f158e1ae437e71',\n     'wiki.min.zip': '8b925eea6df0411ee09baef5801d807cfec8cfa4',\n     'wiki.mi.zip': 'd57831e8d7cb2ec260fc9d83d4281f0bacfb29a5',\n     'wiki.mk.zip': 'b1fc2d85527e99530a93e3bbc5fa9fcde89910f3',\n     'wiki.ml.zip': 'b9d53b8e76a05f5e959afd190da3015b36793297',\n     'wiki.mn.zip': '715bf0ee67b48ec872659380fcf63ad006ddcc7e',\n     'wiki.mo.zip': 'fb273fe373eb61310051d94ad6911320f573d0ec',\n     'wiki.mrj.zip': 'b0d1e43e37e1718c8e05fd81a511095636def361',\n     'wiki.mr.zip': '67e942a7742cc957298c8cd0cd0af0531dc936d7',\n     'wiki.ms.zip': 'e218f113702b039fc8e80a77b894cd9fa4eff77d',\n     'wiki.mt.zip': 'd68d5b636eac07b2e1307186c2c05b9a80e39658',\n     'wiki.multi.ar.zip': '31c7b742c63c3367e9bce5c4dca37d5ceb33f1a6',\n     'wiki.multi.bg.zip': '8991e8123bce7fd6c8e4510c71ede5715ae36f01',\n     'wiki.multi.ca.zip': '0786e071438150485d394a4bf2e976d3a1b313ff',\n     'wiki.multi.cs.zip': '7237f291146e69f0fc7002a0e175c7fd003d44e8',\n     'wiki.multi.da.zip': '5591c20015191101aee190c02738c99073a8fe76',\n     'wiki.multi.de.zip': '986160e51a08f4a93f1573d17352e375cbaedd6d',\n     'wiki.multi.el.zip': '570eb12811ce61f6176f263eff3e945be69e7da0',\n     'wiki.multi.en.zip': '2c3ef35d8338d4a905e7d10645572ab7a6730d44',\n     'wiki.multi.es.zip': 'c1db7c7175665a7230f92ed038b78de780e060e9',\n     'wiki.multi.et.zip': '54d0515865c754331b445dd9ba0ae7ed79b770aa',\n     'wiki.multi.fi.zip': 'c94abc803a42b89cd75b278114b1f2cf4e2f3ecd',\n     'wiki.multi.fr.zip': 'd4904b79eaf8ae386a7011ad84afc9b4238c9928',\n     'wiki.multi.he.zip': '370ec2a379eecc2d2e984cde3e0f6d0a027eade7',\n     'wiki.multi.hr.zip': 'd3f25ae76b040ffa09e964f6edc55488f6086394',\n     'wiki.multi.hu.zip': '4b64bcdf0fc1f01bbd8427bd7bf6b46319308e7a',\n     'wiki.multi.id.zip': '3ad5f590d5c847b35a334f1bdb48b9c466f5de68',\n     'wiki.multi.it.zip': '18746450e665e96c33f2e2026986f643a27e0945',\n     'wiki.multi.mk.zip': '1d899f1449d8729b7dbae226f05151a656694626',\n     'wiki.multi.nl.zip': 'ff0a04dbb07c2cdbc61d5a241175e30ed46b48d4',\n     'wiki.multi.no.zip': 'd1af729024181e64f58ae37ab233fc53811e2601',\n     'wiki.multi.pl.zip': '91c3984c4f3158b1cb1ff11d8cc4f9240631266e',\n     'wiki.multi.pt.zip': 'a1782c4fa4337008f82c0e2bf78e4323d145be29',\n     'wiki.multi.ro.zip': 'b1a0840d084009ce00c47a3c24c984648dbe8785',\n     'wiki.multi.ru.zip': '540607ba4334dab6089de463f974861aac8a35ae',\n     'wiki.multi.sk.zip': '2a2bb39e011cf2bf6dcb8cb6c482b8eb9764eea3',\n     'wiki.multi.sl.zip': '99442dab442dc196c107868db9174c78e270db1e',\n     'wiki.multi.sv.zip': 'b40be83d2d7c27633c712aea62ceec0d409cc03a',\n     'wiki.multi.tr.zip': 'e2bffab1616f54d180ba3d8bfe5e94ec9a489184',\n     'wiki.multi.uk.zip': 'e97f64d9ba2b58a5e80c9b896b87340aba1e0eb0',\n     'wiki.multi.vi.zip': '532fa24d8787a8906fb04a88e74a713b00cb33ec',\n     'wiki.mus.zip': '1bb0cad10889b8a3bfa36c36c7da1f2fb2237bb8',\n     'wiki.mwl.zip': 'e3d1fd1fa6290521d403e84eba577e552e330844',\n     'wiki.myv.zip': '64a6505691441778766b7941b5e7f45a624a64a5',\n     'wiki.my.zip': '491ce8dbf174d4abff758db4950f49eda90883d9',\n     'wiki.mzn.zip': '76abf410749fd4516ead20ced891b54245fcd4a3',\n     'wiki.nah.zip': '0496592cdd70eaf61b257fb5345843d38f425592',\n     'wiki.nap.zip': 'f0df66cdbef5734f0afeb806cda631722fb426d8',\n     'wiki.na.zip': '2456e4776b5e985cfaedfac244e0b40cff4e613c',\n     'wiki.nds_nl.zip': 'ffd10e05b749281634eb7a758102d8d6ff42760e',\n     'wiki.nds.zip': '2455e9fa4294828b25b32bdad7307a105f9fbe1d',\n     'wiki-news-300d-1M-subword.zip': '697f4c8f37443be3aee7b96abe28fd7ebec95ef3',\n     'wiki-news-300d-1M.zip': '567ef9c2e207be25da23e61312e6ba620da30466',\n     'wiki.new.zip': 'a781885678cc1079d4be221c414339eb9bee8d19',\n     'wiki.ne.zip': '180b068343288cda40d012aaa99d29459d341eb4',\n     'wiki.ng.zip': '6db8111ab700f7b0841af87f1f1453341048014e',\n     'wiki.nl.zip': '582420f290947cf38503b7f4b8ea9bb21918005e',\n     'wiki.nn.zip': '4a0e30376b361ee19800e6d897a865572e330f84',\n     'wiki.nov.zip': 'ac98c0300302019ff855698561708abd81730db3',\n     'wiki.no.zip': '6893a7912ab3756e31d09ef1f9023c27c0b047f8',\n     'wiki.nrm.zip': 'bd27aadf25a165ebbac486437ea6a06b710fdda6',\n     'wiki.nso.zip': 'c55dfebb83351c952831db34e779e0a380212f05',\n     'wiki.nv.zip': 'cf122e5ee041287917c594a2cb6cd247978f1ec0',\n     'wiki.ny.zip': '9086021a60babd7e87afa469dbadb004523f5fd2',\n     'wiki.oc.zip': '15075544cf837135127d8688cd06fb8e4c8b7f3d',\n     'wiki.olo.zip': '523628bb652e1563b4dd5a94b518addf10699f74',\n     'wiki.om.zip': 'a29360ab3930d889c4eb5b385589f84c1ff9f06e',\n     'wiki.or.zip': 'a782e649ae5307dece445b0c11b15ffb9ce88297',\n     'wiki.os.zip': '0d76ca005afd48b87dea5c9784c4c48bb51d3e3e',\n     'wiki.pag.zip': 'b046ef71badc9d7eec161e3aec2ffc3abb7bad20',\n     'wiki.pam.zip': 'abed25ef407e05209f2653d571bba5bc7c66e7b3',\n     'wiki.pap.zip': '5d099bfc65c85f824634a191ce33e8e42f947ded',\n     'wiki.pa.zip': '2066ed0016720b9f8779f55f2cc2de08511025f6',\n     'wiki.pcd.zip': '66914c99e5531c0484448b84568971362cdad0f6',\n     'wiki.pdc.zip': '6ed181fa1f8782917ae7849490c0a5cb0b0b9b29',\n     'wiki.pfl.zip': '8d271226af8509962b15a96c4d6e41d9aabd972c',\n     'wiki.pih.zip': '365955dbecb17027435fe487ab92a7a267fa25bd',\n     'wiki.pi.zip': 'eeb863545392c92cff0f3e3d9c3f61539d3fa1dd',\n     'wiki.pl.zip': '2b0cae8af2637bc24b958e6757149d1b9f8c8fea',\n     'wiki.pms.zip': '9eff2e96e1cb9bf02adf816c4feb5aa3cd1a384f',\n     'wiki.pnb.zip': '23f77d1d9469f5b2c342984288cb3092d53d8dee',\n     'wiki.pnt.zip': '84cc9532d2fd7b322bcba91e01ac36c9a719e23a',\n     'wiki.ps.zip': '18c9ffb2a81cbc25299b26e35170a29b7de9309c',\n     'wiki.pt.zip': '37752109a44829de5ea10b173d7c0cecc0b1a0d7',\n     'wiki.qu.zip': '5582c07eeeaec10d9382b3ab90d2921fc97fa2e0',\n     'wiki.rmy.zip': 'a106ab536001e92e7a9708417faee9418f4058d0',\n     'wiki.rm.zip': '67a324941f2b895a418fbd89314a18bfda19b1de',\n     'wiki.rn.zip': 'ce17294909c046e90bb0131632e1d795d1771816',\n     'wiki.roa_rup.zip': 'a9a378e90cd46353283c92cfb7d34dd485a018d2',\n     'wiki.roa_tara.zip': '953fe4cf1667cbb9b3b8e11666885bfedf74b411',\n     'wiki.ro.zip': '6bbb0f9452398416d9183e00e6cd091a02fb351f',\n     'wiki.rue.zip': 'e9f9b8ab63c7722b4b68e8c465b1c69436132553',\n     'wiki.ru.zip': 'f8f68aa5792941d7750b545e56f1ff5127e88cc2',\n     'wiki.rw.zip': '018b9fb76fca5ce7a3e1f266df33fcc1bbc50493',\n     'wiki.sah.zip': 'f6c94dbd3b719b154217388310fab72e5a69f823',\n     'wiki.sa.zip': '4dc78b48d651056546d14b659c6598770c6bce77',\n     'wiki.scn.zip': '218ba35c042cb3e179988bac9acf51cccf37422b',\n     'wiki.sco.zip': 'daa8cedbb223e87d48f720aed9ce63dd0c81c632',\n     'wiki.sc.zip': '909cc5160cad60fda34ab89c2b87ae4229402eeb',\n     'wiki.sd.zip': '5468ed141bf2f1d9b1f8d7b31fee926b496ea9db',\n     'wiki.se.zip': '0eb962f8768d88ffcbde3aac833e134a263c2055',\n     'wiki.sg.zip': '651035aa74dc2f515253444f48aa9911094f9d27',\n     'wiki.sh.zip': 'cf3057b61bd5bca6f47640801681d451aee210cf',\n     'wiki.simple.zip': '367737535e39defb0e713a7ff2374cb932c5a9bc',\n     'wiki.si.zip': 'cebb2f4011b0d679fe856c5950076e3c48496ecc',\n     'wiki.sk.zip': '6c43758d0c0f52351210c558cc33266a65709068',\n     'wiki.sl.zip': 'd0239eefc830e5919bef8d9173a884e9e7371e7a',\n     'wiki.sm.zip': '2e3cf33f17b449c8f81cc9ea4c84d542cfd23a14',\n     'wiki.sn.zip': '4d3844ee350ee0065e5fe910a3f669ef863a2fc9',\n     'wiki.so.zip': '9da45db9b21d1f27c4f73152539c1e4fc9b1c49c',\n     'wiki.sq.zip': '0db976ec147df49e648cf8256562371d0ae6f2f0',\n     'wiki.srn.zip': '120e229d522cc22008c50e0eb74b23d9f6eca51d',\n     'wiki.sr.zip': '63b67391158bdd7a642f7d8412771c22e1041744',\n     'wiki.ss.zip': '4368f7931f6730a6e8cb9b5794906f2d827582a8',\n     'wiki.stq.zip': 'fb1ba577bf6fb7f7fcdc52bf392e63ed8492465d',\n     'wiki.st.zip': 'b7e96392b3880c19e210fd42bc72e3f76c07a4c3',\n     'wiki.su.zip': '4c4880cfca1ff954c88e44a32f201218eb2be146',\n     'wiki.sv.zip': 'e2b10091585f795dd18289c4a65a1da591a78196',\n     'wiki.sw.zip': '726631d8998ba1647d040e6b70f4bad7b8d8c367',\n     'wiki.szl.zip': 'a70de974cff95cad0443f5faa6c8412c92998100',\n     'wiki.ta.zip': '6bafd0bb523f654038393ba191012527745b940b',\n     'wiki.tcy.zip': 'b4bd573eaf9fd87300a25648b38a053161d12c39',\n     'wiki.tet.zip': '7e5608958977164e544850a5a169f5d55cd47a20',\n     'wiki.te.zip': '948e5a6ec13ac95b595c3f52a6e7b9642a56c530',\n     'wiki.tg.zip': '5b46429024d6819f6b511a4924b90c958615d40e',\n     'wiki.th.zip': 'b8ee0878cec41b4ab1055a17d0ed669de1ed9afd',\n     'wiki.ti.zip': 'd55abb74bb3ff195d2293ee9e77886111ee50e52',\n     'wiki.tk.zip': '20263f39a31a1d55343f9dea7aecaa2860aefde8',\n     'wiki.tl.zip': '2f2b809017249f8c4f8d5eb62979b58f16e8732b',\n     'wiki.tn.zip': '0aa11b07b1ad6437bc1e9b6476d51ddd35dad994',\n     'wiki.to.zip': '6b90b32ae258a56e67b42736675236b91163b3ad',\n     'wiki.tpi.zip': 'ca9591e621ae667a1521d0bb5275435d45e974cc',\n     'wiki.tr.zip': '3b6f86c2a115c7adec1b073b1f5624890e680148',\n     'wiki.ts.zip': '8a00b16f2881977ad6f8c8665316c27fcab9b842',\n     'wiki.tt.zip': '8d2f559bf1e09180d6dc4b127d61815a27670a20',\n     'wiki.tum.zip': '5b3f6f3d8cae4d9534cd1fd3afc2f64ec8342b8d',\n     'wiki.tw.zip': '7c189fabfcdb2973178c25d35fd10e46ee7148aa',\n     'wiki.tyv.zip': '5e3811a19bbf961a5361ac37ff3502287c9ab022',\n     'wiki.ty.zip': 'a7f31f8cabf4282533773aa7e63f294315cc85ea',\n     'wiki.udm.zip': '643df5ab0914535e46e6839845d0ab585c81a119',\n     'wiki.ug.zip': 'a5388269893ac4c7da28b2284f3536ca0f3c9341',\n     'wiki.uk.zip': 'fdc9b0a0ab806e5845e9d89b8887ec9d555a0547',\n     'wiki.ur.zip': '75579eb5609ea31d79bc2d1bd81d01f48e01bc7c',\n     'wiki.uz.zip': 'aa149200f8c6e3e8bb5aa3c67112675d136900b8',\n     'wiki.vec.zip': '58c4c9528154e256fbefeb97b8c1675356079f74',\n     'wiki.vep.zip': '966b371afcc383058a5fbc6ee8f822620f03feac',\n     'wiki.ve.zip': '6450e3ec2c78980c5a41d71ff159aa27918dda75',\n     'wiki.vi.zip': 'bfa287fbb358a66b4f9576585df3e46607e1595c',\n     'wiki.vls.zip': '7335bfda43890f42e045b8a5de25d1a8629fe012',\n     'wiki.vo.zip': 'c2ca18bea165cb1253c1d88fa9958a25088fc84b',\n     'wiki.war.zip': '5cda8fdd64e3acf5488ad361b68a63fb23747559',\n     'wiki.wa.zip': '2e538c10a0e9f43ea5875c90a8ce01a07c4695a7',\n     'wiki.wo.zip': 'f54c65ab63f98ffec7b3fb5bdd51a814034bd673',\n     'wiki.wuu.zip': '68d9ad802836737392d62056231bf1b7a58594c9',\n     'wiki.xal.zip': 'fb39fed41ccba2e4e58ab7714a53aae3695dbe04',\n     'wiki.xh.zip': 'd37caa4d94e66588879231d0826798d8aa4b0a44',\n     'wiki.xmf.zip': '956c43bca0d88e9348099cde43d58898e43d9f27',\n     'wiki.yi.zip': '151c1670c48e976e4202272b066d7080a8c83615',\n     'wiki.yo.zip': 'fdbd0fc6e35bb04c3aef1fa6f0262ba261b11199',\n     'wiki.za.zip': '11f6a5dcb49c4d0571d5ac4fb3d7dda1d378fc06',\n     'wiki.zea.zip': '22159a722c5c0390bad9206eb75e6e166efe38e9',\n     'wiki.zh_classical.zip': 'c689d61d2254caf1ecec0909249523b09a737717',\n     'wiki.zh_min_nan.zip': '0516a413565484d924a4c8b50c690d39344cdb64',\n     'wiki.zh_yue.zip': '464f4c1c2039194cbae7502ed3a2eeff4df9e34f',\n     'wiki.zh.zip': '2374ec566f6411b9bb570077636695fe9768a5ba',\n     'wiki.zu.zip': 'a6d0325dab37cd551e6d7f6c783dd13f4c71db2f'}\n\nFAST_TEXT_FILE_SHA1 = \\\n    {'crawl-300d-2M.vec': '9b556504d099a6c01f3dd76b88775d02cb2f1946',\n     'wiki.aa.vec': '5cce30fc85471572c498f278bbe495184577363e',\n     'wiki.ab.vec': '9d89a403a9a866d3da8dd8cfab849f59ee499343',\n     'wiki.ace.vec': '85d00074f7a08626f39da6a0c8a5cfa250096ab9',\n     'wiki.ady.vec': '9d17d74f0348224cdebf8a831e61af0825f8952d',\n     'wiki.af.vec': '999e64bcd8dab8de42cb1feceeca360def35324d',\n     'wiki.ak.vec': '6092b8af335c2dc93e8df2bbf1d715f01e637bb4',\n     'wiki.als.vec': '96052e96870695cca50857b5fde5f9f42219139a',\n     'wiki.am.vec': 'dff7fcdd8f5ba0638ab9e1758a89800766156d72',\n     'wiki.ang.vec': 'a7c30e02422d97d23a0701279c5c1c03159130a5',\n     'wiki.an.vec': '5b4c2b1de5c04e4e0be83841410ca84c47305d21',\n     'wiki.arc.vec': 'fd3ad743103f80cde9cfc048d7ca509e50efb35a',\n     'wiki.ar.vec': 'c46e2142f799cc385bd25f0c0a8943ca565505a4',\n     'wiki.arz.vec': '5e904087043b91f4945dd708f4230fdf51360132',\n     'wiki.ast.vec': '89a90357101953b7c292697fd050c00fe5c38ac5',\n     'wiki.as.vec': 'cad5883b5147cbe6cdbf604f65cabdb675a59258',\n     'wiki.av.vec': '99976a63ca8c4231f808fd4314f0433db35e290d',\n     'wiki.ay.vec': 'be359dad25b2c742d3abfa94c5f5db13f86c730e',\n     'wiki.azb.vec': 'e23af0a436b97434813c3cb14ed114cc5b352faa',\n     'wiki.az.vec': '9581d55d9056ad398a153c37b502f3a07867d091',\n     'wiki.bar.vec': '96130f1f2e5bffdd06c202ad4472e5234020980a',\n     'wiki.bat_smg.vec': 'cb3aef58da2011183b39fca64cabf3d9d7a62f4b',\n     'wiki.ba.vec': '22147ee16b2d163cc88d09a035264fd0c10dab68',\n     'wiki.bcl.vec': 'd4117b5c443438ddfa608b10a5be2c2501817e7e',\n     'wiki.be.vec': '6cf81322cd7b046a7f02ec4c4960ad27045383fa',\n     'wiki.bg.vec': '7c1cc6d0c52b038e4b7173259b0c009f242cf486',\n     'wiki.bh.vec': 'ab2d29017afa015c49566a6d9bf75393c23ac4c0',\n     'wiki.bi.vec': '15785220cd6e6c86cc87e7d3f3322a5541a4fe5d',\n     'wiki.bjn.vec': '5f134cf288e8042dcd048a3ee76159aab42c7288',\n     'wiki.bm.vec': 'f36a19c95e90865f6518d4487e59f363b47bd865',\n     'wiki.bn.vec': '6fc3bfd9af455719f55bee0bea31b11afc70cf06',\n     'wiki.bo.vec': '2e9358e03dcfa09da23d2e1499d84b10348fd8a9',\n     'wiki.bpy.vec': 'c2bb15487c4bdb8fa869772694300ae1fee73896',\n     'wiki.br.vec': 'df44e16abd2017e2a1b6c6588ee02779b19907f6',\n     'wiki.bs.vec': 'c4943a290819ceae1611dd11179b40aab0df0471',\n     'wiki.bug.vec': '942d8f7dadde5faa33aa72862501434f48e29f60',\n     'wiki.bxr.vec': 'eaf767690c6b194605ae778719212e3874873d4c',\n     'wiki.ca.vec': 'f5971edee11c939f6a7accfd33a9a45caa54141a',\n     'wiki.cbk_zam.vec': '6fef47b4559eec402ce371de20dfb018acd6347d',\n     'wiki.cdo.vec': '95e8196bf76323dbabab1b8a49ba4d677af3ccea',\n     'wiki.ceb.vec': 'b8516a55537b8f80c927d77d95cdf7e4ff849a05',\n     'wiki.ce.vec': '1d94b0168a773895b23889f7f07d7cf56c11a360',\n     'wiki.cho.vec': 'cec6778f025fa9ae4134046c6c3a6291bd9c63f9',\n     'wiki.chr.vec': '8501bf86b41074ed6c8d15b9209ef7ce83122e70',\n     'wiki.ch.vec': '46803f3a1734f6a7b0d8cb053bbb86a6915d02e9',\n     'wiki.chy.vec': '26c87688551ffe3a0c7a5952e894306651e62131',\n     'wiki.ckb.vec': 'adb2fef309f1d93f429442b9c16c1564192c58f3',\n     'wiki.co.vec': 'af876a918594e5541207bc12f17bfc4268df7b93',\n     'wiki.crh.vec': 'c0d2310a1207fcacc94b25b149420b33bf835015',\n     'wiki.cr.vec': '61dd9f044b7dfa56dcf1c3c07c7504c569420528',\n     'wiki.csb.vec': '649cb2692f08414987c875dc331022567d367497',\n     'wiki.cs.vec': 'f3ec1502aeee6a550d8cf784273fa62f61419a4e',\n     'wiki.cu.vec': 'ddadb14ea00ea1dda716ee33732497ec049b526f',\n     'wiki.cv.vec': '9cdb0bee5a0fea030def85597dba7108f21b0424',\n     'wiki.cy.vec': '32d976a9bfc4dd6e39328c906eead0f597bd9e25',\n     'wiki.da.vec': '526947dab1ffbc1465c7a766f2bca4de50676b08',\n     'wiki.de.vec': '2ed2696afe55f023b0040b238d9a47e5fedfe48b',\n     'wiki.diq.vec': '77f3c370d1d77806fafe368cf788af550ff607dd',\n     'wiki.dsb.vec': 'e49a647a441fbf011ac5411dd6005e8725b9a65d',\n     'wiki.dv.vec': 'e135ba97c711a021bc3317db2b95db5212c17658',\n     'wiki.dz.vec': '24888f0b2cd156360bfb5e9e905240163ba798d8',\n     'wiki.ee.vec': 'afd1670655daa7ffba51187a415fdd0b43f1d487',\n     'wiki.el.vec': '6f034271390feaa6f9d7d16f933ddef637755979',\n     'wiki.eml.vec': 'de6be7a2ffdda226eec730dd54b4c614bd7f5dca',\n     'wiki.en.vec': 'c1e418f144ceb332b4328d27addf508731fa87df',\n     'wiki.eo.vec': 'b56998fd69f66755b722a9481a9bdaf10f62c9aa',\n     'wiki.es.vec': '2f41401aa0925167176bcd7a6770423d891dfef5',\n     'wiki.et.vec': '64d56b66c02d5e49b1b66a85854d67d2dd9ebd41',\n     'wiki.eu.vec': '5e72f4ef93666971fea5d2180b354e0a0821ba91',\n     'wiki.ext.vec': '456c5632b13a0f136cd180ebe2dda67b83f78397',\n     'wiki.fa.vec': '09b6cc685c895c66b853af9617787d3ab0891e2c',\n     'wiki.ff.vec': '12b09d695f5fb8de4b5da9d36a73eb178b293a04',\n     'wiki.fiu_vro.vec': '168a71a2b1c478e6810fa5dce9612d8bf8a273dc',\n     'wiki.fi.vec': '91d19baae994d7e556b5b5938be2dc6013f9c706',\n     'wiki.fj.vec': '36d36dc14001a109926bfc633594f6a2f7401697',\n     'wiki.fo.vec': 'eead8ddc7bb74b12b16784723abf802bb51f844d',\n     'wiki.frp.vec': '0eb70a613ccf807c7308c1f62535f0606465029d',\n     'wiki.frr.vec': 'cde62af939cb2de35e341cef2c74813802a58ed4',\n     'wiki.fr.vec': 'b092229005a65d8683a4112852fe6eb8161a6917',\n     'wiki.fur.vec': 'd4a595cffa1abcdcf4229ba15277179ce5d20bc6',\n     'wiki.fy.vec': 'd4beef537b7ff142a3986513879ff51a9ec14a7b',\n     'wiki.gag.vec': 'c82ec7a5d081f0673661824f4fc34345dee255f0',\n     'wiki.gan.vec': '7e53a33b7bd5b0360ea4cb452145616c09445029',\n     'wiki.ga.vec': 'caaa5b2167a499893313ac1aa38416a6a0fe9a24',\n     'wiki.gd.vec': 'f4b513598a1bf0f0d5b6521ea8ce363e9596cb97',\n     'wiki.glk.vec': '20a7759075916e10531f5b3577302353cef565cd',\n     'wiki.gl.vec': '8888bb8f3d70b36729b9ae479fe3765e0c083862',\n     'wiki.gn.vec': '98594af7897c5a1f35885ddecc77556a7e7ae981',\n     'wiki.gom.vec': '5a1193d9e5d49d06354c14e2b7c01bea176e13f1',\n     'wiki.got.vec': 'dfa06de83a0e3099027c57b84561d7d990ea8310',\n     'wiki.gu.vec': 'f9e13452eb63d92bea44c7c3db8fba9945c7000e',\n     'wiki.gv.vec': '993a7ee31bdacc91763dad656aa6c2947b873473',\n     'wiki.hak.vec': '9e83512d34c7f81739492bf0abbb25ff1ef88573',\n     'wiki.ha.vec': '677a24efeeb1bcb8c0a931407775f18b18e875ae',\n     'wiki.haw.vec': '58fea5aa1b37723797d26fb3d050ce6176757240',\n     'wiki.he.vec': '55534560247394669e3f5c169136770c93bc2708',\n     'wiki.hif.vec': '49697cf784814d3f1a47559724028e0fc0940d36',\n     'wiki.hi.vec': '8049bb8604bc049d48bd934e27b0e184c480a413',\n     'wiki.ho.vec': '9c75a09e099213aa8cd1f1020b223427537cbdd8',\n     'wiki.hr.vec': '0c96f9af092cf8a84b03aec1426cd23921671489',\n     'wiki.hsb.vec': '3dc7830544c58535bed308c552d609e13b973502',\n     'wiki.ht.vec': '5039dfb58a074ac046813f2dae81159be8c5213f',\n     'wiki.hu.vec': 'cd777e9efca3d4bd97c89f01690cfa4840d9c46f',\n     'wiki.hy.vec': '21f9259d04cfd22db446a45d3622af225f00cf20',\n     'wiki.hz.vec': '2a94b1390d68027748a05169fbc0c11a9a183456',\n     'wiki.ia.vec': '2a348dc924638efc20c34785852b0837364aed76',\n     'wiki.id.vec': 'c49d5c9bec89114599427f6c12a5bda2e5523dfd',\n     'wiki.ie.vec': '01b0d11c0e7397418e73853d220e97bdcf7a8961',\n     'wiki.ig.vec': 'd2d1643b4fb1a18a4d002cf2969073f7f201b3b2',\n     'wiki.ii.vec': '41c6cd68b3ebe4ece2a06c37b06dca5d07c9fb3a',\n     'wiki.ik.vec': 'af31cbec7b839f50fa70553ec63c58f7067d3ea8',\n     'wiki.ilo.vec': 'c0e43835a3f4e0033ea5d7c6ff189982b2f26a05',\n     'wiki.io.vec': 'af0c480c5872bff31d82e767c1116da2a6be0c00',\n     'wiki.is.vec': 'ae0b018f92b3e218f2dacb2045a8f0a0446788a5',\n     'wiki.it.vec': 'ac4a985e85ffae48047034e2603d804bf126caa9',\n     'wiki.iu.vec': '5d51b2ba215005216ae003f4a6d6ef39fb30ca2e',\n     'wiki.jam.vec': '6d51e384c56330097c2531fdbf4e74418909e388',\n     'wiki.ja.vec': '7a2b1af1e46d795410692a002e40fa3085135f69',\n     'wiki.jbo.vec': 'c90481946aa4b6b304528292612ae620f6549f3e',\n     'wiki.jv.vec': '2ff7927d3ff04b8208133497b3778ede00ea463f',\n     'wiki.kaa.vec': 'd990d3b9bd511d2d630f923099a6b9110231b2ed',\n     'wiki.kab.vec': 'e3b73d41267d8d4cd42f6cc5a0c05dc4e021bf74',\n     'wiki.ka.vec': '8b92b73f27f9b77818211e053a33985589de7c62',\n     'wiki.kbd.vec': 'f5b8dbe47a7fae702232b5680b070ef6e865539e',\n     'wiki.kg.vec': '1550647b6059e6eb649b100e31c53bd0661117b2',\n     'wiki.ki.vec': 'c4e373e2ea13f7fa1e95b0733365e4b3fc8b2cc8',\n     'wiki.kj.vec': 'c27e563683f9c96ff6f680a6d6bb9e9e2f9960d0',\n     'wiki.kk.vec': '6343b2b31bad2e13d03a110b91c38fab4adc01cd',\n     'wiki.kl.vec': 'e5def7fb1b56c5956b6e951e912d53ba0ff089f8',\n     'wiki.km.vec': '64f7fff1df90b1f7241b232e901f76223a3719e0',\n     'wiki.kn.vec': '32763f4f860f0d081f3aabf3e7d17b7858e7d877',\n     'wiki.koi.vec': '4001f0617fe0fdd3b22116b304f497b7b16c6e4c',\n     'wiki.ko.vec': '042c85a788c2778cca538cf716b8a78f0d7fa823',\n     'wiki.krc.vec': '0c6ef043d51e5f337a309804f1db180fa0bb2cb8',\n     'wiki.kr.vec': '25d5b4d5911a819c48328c48fb346417d07d4070',\n     'wiki.ksh.vec': '4c3bb4f12073532b6fb7cc6c2be5e53319ef5b65',\n     'wiki.ks.vec': '5056a87c4ee2d8bf0792436fc6b2b61648014de9',\n     'wiki.ku.vec': '4d3a2401527dd9ba6be2b0cd31f6cd3edebadce9',\n     'wiki.kv.vec': '164dc44d701b9d606a45f0b0446076adc3858dca',\n     'wiki.kw.vec': 'f9eaa35a7e4f077f6de85c7801f74582f91b52c1',\n     'wiki.ky.vec': '13b0ae3f23822317a0243bd9182105c631c834b3',\n     'wiki.lad.vec': 'c510e520cde97050bf1cbeb36f2b90e6348ceed4',\n     'wiki.la.vec': '9ea6286a0581084533db8d6ee96e0b7d15166543',\n     'wiki.lbe.vec': '283619d93255571f14fd4545bb0577979171b990',\n     'wiki.lb.vec': 'b146f23628c84e64314a35a5b6cc65a33777e22d',\n     'wiki.lez.vec': '8e579b984a500ad89fc66767bfd7319766bd669b',\n     'wiki.lg.vec': 'b096f5248dfbb343dc4696c97ea253510e1c4ef9',\n     'wiki.lij.vec': '4ff5bb405c820e4119f0636efc301da15a08c00a',\n     'wiki.li.vec': '0fb9ec4ac93676d8ef651692062bc3d7f6ae0843',\n     'wiki.lmo.vec': 'a89414d9ceee4823622258f18936f67faf7e06e7',\n     'wiki.ln.vec': '70b6a286b42958e25cb80824e0d8f1aee2de6dde',\n     'wiki.lo.vec': '7c83f82b80c49b8eab21f62ecdb3681b8bda40a6',\n     'wiki.lrc.vec': 'c1ae4fb79a19d44bfe8f601f0a30fbec841fa612',\n     'wiki.ltg.vec': 'ec2f13d1290bd54afcaa74569e66e43e9bfef264',\n     'wiki.lt.vec': '58d3ebef24e5e31be1a8318b45c08ebb16ad775a',\n     'wiki.lv.vec': 'ef6b549f96e22718f513d47a611d3d6bc001a164',\n     'wiki.mai.vec': '7f513ff36e485b19f91f83b30c32dd82e9e497f6',\n     'wiki.map_bms.vec': 'e7deab5fdd38fa3331b1bcb4a16432b38c512e21',\n     'wiki.mdf.vec': 'b16099ce0283a241339716eac41cfd99fdea7f36',\n     'wiki.mg.vec': '0808252740909d6129f672584311263e7b2adadc',\n     'wiki.mhr.vec': '39f62e292336cabc364f0d1913540b881b406393',\n     'wiki.mh.vec': '7d2d8bff722fe0a5d869d9da11792a406aff3dc3',\n     'wiki.min.vec': '3bb0fa596cf27a1d165c55684bebdc8d40cb8ad7',\n     'wiki.mi.vec': 'e8acf9c7c2ab840a192c563aa776201a88e4ca89',\n     'wiki.mk.vec': '85a3d3f13fa88ffde023d2326c65bdded4983dff',\n     'wiki.ml.vec': '2b70fe76e8cf199a18551de782784a21e8db0b66',\n     'wiki.mn.vec': '7cef7ecdf9d98484d9b598b25d0e717dba6acfd9',\n     'wiki.mo.vec': 'cc54b661aefabdf516b49d24acb51273b3acf210',\n     'wiki.mrj.vec': 'aa1c1ecba1ffd6b42c8d9659a8a04ab328ae1650',\n     'wiki.mr.vec': '2cd6cf88bfdfb24850d345749ce0cfea8d65829e',\n     'wiki.ms.vec': '458e1a079799a54cdc0a7b78c7fa1729d2683a6d',\n     'wiki.mt.vec': '81f4c1d84dd4cc4276d59cb903fcc9aba46be981',\n     'wiki.multi.ar.vec': 'f1f12cc9d629382af574a3db74fe49c2fd615c8f',\n     'wiki.multi.bg.vec': '22470e664e4b35761a33c64433ea2f0c12140673',\n     'wiki.multi.ca.vec': 'bc8d98b4d86d740d1985d73d211d887d561bcdd7',\n     'wiki.multi.cs.vec': '17358b62e63f96b0479d6a70e9235a0421493884',\n     'wiki.multi.da.vec': 'ebc75f428714d26fb1fa31accce49ad3b31e273b',\n     'wiki.multi.de.vec': 'b9a63406aedf4446b467b94d12674bfe4723b52d',\n     'wiki.multi.el.vec': '03d33db85bf83f35b943ce93b18c02fa98a0bc05',\n     'wiki.multi.en.vec': '696719afdbe470ee4a2eb668229486dba1df19cc',\n     'wiki.multi.es.vec': '98c9e35564ec57fee5dbc6155890150452f45d3f',\n     'wiki.multi.et.vec': 'db10189093387e853f2fd3978770e1cc7bc07820',\n     'wiki.multi.fi.vec': '746916885a1c7d4ec3f139a32cf267f9e15f5363',\n     'wiki.multi.fr.vec': 'fe1535827b631d934beb02f8d36ba901b2c94a46',\n     'wiki.multi.he.vec': '6dd112f018165317da22971a2b6fdb2a15dafa91',\n     'wiki.multi.hr.vec': 'ff9f23cf595ec8dd93cd93c6b48049730c34253b',\n     'wiki.multi.hu.vec': '6da405c9b048f3cbb990bfb29ef149f0430aa2e7',\n     'wiki.multi.id.vec': '34edadab182682198c37ade8538530c545635742',\n     'wiki.multi.it.vec': 'c55802bd73d46a6fc86771097670e02a70b5d46d',\n     'wiki.multi.mk.vec': 'cec8550503ebca0bdc7ad11f2c15085b7072a990',\n     'wiki.multi.nl.vec': 'c3f45a5fe8a8bc213cdf35dce51651b752ca60c4',\n     'wiki.multi.no.vec': '105236df530c8fc2ce5b1e2550a2059bbc46fc28',\n     'wiki.multi.pl.vec': '676eb5acb22982c0c9a7d6e4c90d26730c6d120e',\n     'wiki.multi.pt.vec': '625b0a5384873c79a5dcfff5ee3fde49a3a65013',\n     'wiki.multi.ro.vec': '82bd59674509b69f988f9870e3a291836ba43e84',\n     'wiki.multi.ru.vec': 'a7d9c5f2ab2abb448a5111d352caa921adabe830',\n     'wiki.multi.sk.vec': '98d849ee77f0320472cc5afa002bfde129be7089',\n     'wiki.multi.sl.vec': 'fb5cfb8a9c44380d74fb21ddd204e820c4e05c31',\n     'wiki.multi.sv.vec': '95d6cc3ba23dffff9be6adb467b617dd57780cb2',\n     'wiki.multi.tr.vec': 'ecb0e353eaccba3fcacc6994d93065934ef429e9',\n     'wiki.multi.uk.vec': '35f4f5a1ead8bd66bcaf865021fc3aae94456ab6',\n     'wiki.multi.vi.vec': 'b1abe06360e1d65a0db65dd41ead7b2f9d651ea0',\n     'wiki.mus.vec': 'fa1066f7bd09df4589993ca498c19aeb6cf986fd',\n     'wiki.mwl.vec': '3d10a218242b94fcc3981aa3beb012b701827a55',\n     'wiki.my.vec': 'e7c7989e32b23ca1a9caf534cc65ecaf9e1b9112',\n     'wiki.myv.vec': '7de0927fd3d65677de7f770b3bd57c73b58df85d',\n     'wiki.mzn.vec': 'aefad49237808acab99e1ca8eeaaf531666f261d',\n     'wiki.nah.vec': 'c52e01cf4479fb7ec91ef39f298e8f97aeb6496e',\n     'wiki.nap.vec': '6c9bd8ce1e85ee679b25189fd6f6d36afb119b6c',\n     'wiki.na.vec': '8a592eb3dbe5693372714dff495d01cabc3ea215',\n     'wiki.nds_nl.vec': '1cd96d12e78e5cd3f65ca2773a17696bda387b9f',\n     'wiki.nds.vec': '7bf293149c08226e05bcf0442ac6e601162b9ffd',\n     'wiki.ne.vec': '1045d7876f947cd4602d9ca79f7c4323a5d3a52d',\n     'wiki-news-300d-1M-subword.vec': '717a3058e0ba5ef3cde52c3df0d4f0f60b0a113a',\n     'wiki-news-300d-1M.vec': '11cac9efe6f599e659be182f5766d6fbd5b1cab9',\n     'wiki.new.vec': '51f6c0b4ef1aee9fad4ab1cb69a7479db35e39a5',\n     'wiki.ng.vec': 'c3016cc07d40bd43bea84b7c600244ff3d2a928e',\n     'wiki.nl.vec': 'd796ee27e37b7d1d464e03c265c31ab62b52533e',\n     'wiki.nn.vec': '35aeab89ffeca0377accbbd3bf18b81913c75448',\n     'wiki.no.vec': 'd52e8019d7cc48569c8c3b514d2b1bd10261b5c0',\n     'wiki.nov.vec': '5455c6e8463b1c43dd073e3e177702fb9a1dd834',\n     'wiki.nrm.vec': 'b4cb941b126b26fa045c5fc75a490a31a969101c',\n     'wiki.nso.vec': 'a906271509c2b343df35d1471509492bbfa883aa',\n     'wiki.nv.vec': 'f5a6ea213bfe95c82cb22b53b4965df8b67ffeab',\n     'wiki.ny.vec': '3aec3dcaea6c35f8254c407621644f87df37e411',\n     'wiki.oc.vec': 'cc1833492899d75571148c2c305591f53d63f0b1',\n     'wiki.olo.vec': 'cbadb4cada4dc579d0becdac93dfb479d76bf6c8',\n     'wiki.om.vec': '91789a8d9f9284f7e71e4bb8d9a60eae4af4adca',\n     'wiki.or.vec': 'a6b120fe536b6c0133b077dca0043c3bc97eef0b',\n     'wiki.os.vec': '791b26cc300e9a1f0a08c7b2213a264e41ce30d6',\n     'wiki.pag.vec': '03f71faf060c4eb33802275279967349c0337553',\n     'wiki.pam.vec': '8fbd31e70d0ca0c61eb1a152efaa8ecb29180967',\n     'wiki.pap.vec': '8cd98267cc55a4f9de80212e29651ddf7a9e83fd',\n     'wiki.pa.vec': '4939d0db77a5b28d7d5aab0fab4f999d93b2053e',\n     'wiki.pcd.vec': 'd2e8e7321b6f1bce94c563cb8ef8af2b45cc3e48',\n     'wiki.pdc.vec': '401e24d0fb9b0ae9e06a5c700684361f58727fcf',\n     'wiki.pfl.vec': '0ad9b7f3ae13f909f12835107432fee4c4ed3031',\n     'wiki.pih.vec': '4ae6ef2a9c6c88e9322eda900e0f58be5592a29b',\n     'wiki.pi.vec': 'd388db284357042f4260e1a567cb489b05bb8e0b',\n     'wiki.pl.vec': 'd031adb6f83eda0364a861dcbf5ef779b5951c0b',\n     'wiki.pms.vec': 'e30bda8d33d61db43243c157b9ac2feeaff316c8',\n     'wiki.pnb.vec': '35f38862d3d83012d6db7baa8a4105e3e0a416e7',\n     'wiki.pnt.vec': '38134772012d68f247e34daf220d9d4ed3e7f489',\n     'wiki.ps.vec': '64f1bec5d5b937289199ceae2e1da6557ce48852',\n     'wiki.pt.vec': '7f11ebdb0cbf5929b38319f1e977d2c13bcd741b',\n     'wiki.qu.vec': '58de8c8290e8bc8f2a6a677312e28457113437b2',\n     'wiki.rm.vec': '5d3144b47a0dd98648a6df0636384ab2a010ad7b',\n     'wiki.rmy.vec': '3d36d3485961900c23355a0f7c2ba656a8558c29',\n     'wiki.rn.vec': '80b6171b78dd932f59f70dbef074abb906af4eee',\n     'wiki.roa_rup.vec': 'e31a44353cd84b976586c8df35a2ab58318120f0',\n     'wiki.roa_tara.vec': 'b3fcb01ff0bac53a0ba08c5c0c411f26ee83a95a',\n     'wiki.ro.vec': 'c088ea2752d5ec8b42e32410c191a14839ae8a1f',\n     'wiki.rue.vec': 'fe539e0ea0bbbfd3ee06bd0c5521a035c7361ec5',\n     'wiki.ru.vec': '7514a2c60ee4118abb451ed32a0d61cb52dec384',\n     'wiki.rw.vec': 'af2ec410da6519a86ba21004c8b4c7fde768a91c',\n     'wiki.sah.vec': '202470467194a1cbdcd571b14ef68371a29b38d9',\n     'wiki.sa.vec': '7fed78d1d7674453b9876ee99aeeeba85ea46699',\n     'wiki.scn.vec': 'bde043a235551e1643506774c5d9b61ecf2fc424',\n     'wiki.sco.vec': '4625a5ad90a57f994be9b3aa4f8f3ecda941a821',\n     'wiki.sc.vec': 'dba8dc7754ef04b1ba0cd702d94eea9575cde91c',\n     'wiki.sd.vec': '36852d1253496e598fbd9b9009f07f454a6bea5b',\n     'wiki.se.vec': 'f46b35ee6b893c2f12dd1b929bbc2b8120cbcd8d',\n     'wiki.sg.vec': '90ece136bef7ad6e4e97776a1c7238499544405d',\n     'wiki.sh.vec': '016691ecb26ace442731d92b1265e5c6c3d8ca5f',\n     'wiki.simple.vec': '55267c50fbdf4e4ae0fbbda5c73830a379d68795',\n     'wiki.si.vec': 'd05ed6a0bc1ee56e5d2e5f881d47372095f6eb0c',\n     'wiki.sk.vec': '98759aacf7352d49a51390fae02030776510ae13',\n     'wiki.sl.vec': 'b26997c0ed1de26a47b11efdc26ac1e7f189fa54',\n     'wiki.sm.vec': '88c2c57ca483626b052403418cb4372d72352bc9',\n     'wiki.sn.vec': '8dbb1019dcc8f842a8c0f550295ae697f8e1b7e0',\n     'wiki.so.vec': '294756b60b03fe57cb08abd8d677d6a717b40bc8',\n     'wiki.sq.vec': 'd07ffed553f5eb4756d0a1548a7ba9a51a52f7c6',\n     'wiki.srn.vec': 'faee05e550f5b08809a9ae5586ac4b08c9a1c359',\n     'wiki.sr.vec': '3cf09f476f55a92fdd2880f7ba336656ab232736',\n     'wiki.ss.vec': '488546a3b2f88f549c50ae9f32f1997cc441b039',\n     'wiki.stq.vec': '1bf88af29f1d86cac16042a5bea6b1651c96a8c1',\n     'wiki.st.vec': '963646055d12873b1c83b0eef8649ecaf473d42e',\n     'wiki.su.vec': '25e864495acb6d280bab0e62480f68550c9ceed4',\n     'wiki.sv.vec': 'eab83ae36701139696477b91b6e8d292ef175053',\n     'wiki.sw.vec': '8e70d207dbbd14e60a48e260a23fbf284a8e9f06',\n     'wiki.szl.vec': '0573cf888ec70b459b0596d34814fe60fd69f190',\n     'wiki.ta.vec': 'b66b5358527b1f3a6a421ab26464a3c1e75e18af',\n     'wiki.tcy.vec': '388b1d89642fcc790b688e9643b3d19e14d66f40',\n     'wiki.tet.vec': 'f38fe0e76b9b08ff652689eeee42c4fdadd9a47e',\n     'wiki.te.vec': 'e71dcf3cc45da1bcdae5e431324025bd2026d0c8',\n     'wiki.tg.vec': '6a5cd5bfe571ca0359b66d21bf6950553213f42d',\n     'wiki.th.vec': '1d6e0d525392a1042d017534f6c320c5a0afd345',\n     'wiki.ti.vec': 'c769fbc99bbb4138a40231e573685c7948d4a4c4',\n     'wiki.tk.vec': '33ae577f77d339ab7a0dff88855b8d5c974d0aef',\n     'wiki.tl.vec': 'd508e229ced7201510999e76d583de3ff2339d8b',\n     'wiki.tn.vec': '39f45f3fa86645bb25c54150204abcd51cc1048c',\n     'wiki.to.vec': '64d512665b55e9ef9a3915e8167347be79310fa0',\n     'wiki.tpi.vec': '407b96d235f54f3e0be9dc23a3bab89c6593a621',\n     'wiki.tr.vec': '13234aa1bf5f99e81d933482b3b83c3e4bf6c85e',\n     'wiki.ts.vec': '00f8229e2f230afd388221c0f823a1de9fc0e443',\n     'wiki.tt.vec': '913bb3a11da6f8142b3bbec3ef065162d9350f1d',\n     'wiki.tum.vec': 'bfbe43364724af882a520d2edcc2ce049c7357cd',\n     'wiki.tw.vec': 'f329b667d70d9f0b753e55e1b1579b5a5191d3bd',\n     'wiki.ty.vec': 'b881f60b8c75a71864d9847a17961d368f3058fc',\n     'wiki.tyv.vec': 'e8f9a36dc58e4108c553f96e247a877a099ab5ba',\n     'wiki.udm.vec': '336a8526f22e177faac69573661dc9c3ce36591f',\n     'wiki.ug.vec': '586d2febafaf17c9187c599ffd7b96e559103c34',\n     'wiki.uk.vec': '77f7737b9f88eac2b3e130ea8abb8886336fd0c6',\n     'wiki.ur.vec': 'cb8132102152a958df72bd3e25f1a72abb4c9c76',\n     'wiki.uz.vec': '11c3a76dae12b454f693811e33ae2e60015743e2',\n     'wiki.vec.vec': 'ae4b055fba21974e56beecab3a95f9dc24a62fd0',\n     'wiki.vep.vec': 'a38a781fde24f4d7b52aa8bc450b9949dd4e1808',\n     'wiki.ve.vec': 'b7d2947501de1c30a9f8496d5efae20c051104e1',\n     'wiki.vi.vec': 'bc84245b52b2e212e28dc6856c0693ce9845a9c5',\n     'wiki.vls.vec': '07e8636908c057b9870ce4b98c7130d460cf882a',\n     'wiki.vo.vec': 'c830988b6965bfce2f932b1be193f7d1f755f411',\n     'wiki.war.vec': '1f5d443d6f612b59a53820dd6f39fd886a6ad30f',\n     'wiki.wa.vec': '18f9ca1a585e1d18c3630029141a2e19d7d34a8e',\n     'wiki.wo.vec': '2ad96a7a9e640bc0dbcf316b1f414b92802dcb8e',\n     'wiki.wuu.vec': 'e1cbae1d3ad52329d0f36ada764016fbacf07049',\n     'wiki.xal.vec': 'b738222d84cb8c8fdb2b30a7219aa5d3bdc2f61c',\n     'wiki.xh.vec': 'bf37f741b0b75953281d11df2b4d80100df9e666',\n     'wiki.xmf.vec': 'dc1923cfd1a7002d5d60426b60e6756854ab4a14',\n     'wiki.yi.vec': '299d61958b7dcc38774768f1489121384726d860',\n     'wiki.yo.vec': 'e35c8aff2924ba07936be9d0d94bd298f09702a4',\n     'wiki.za.vec': 'e3a0e58bd2e5b1891c71f1f7e37ff71997a20361',\n     'wiki.zea.vec': 'ee12db26aab3f2b3b2745a298ef414e7aeb5a058',\n     'wiki.zh_classical.vec': '840981c83dd8e5cb02d1cd695e2fe0870941316c',\n     'wiki.zh_min_nan.vec': 'f91ccb013e200bb7ed560082ddf4bdd9c2f315bb',\n     'wiki.zh.vec': '117ab34faa80e381641fbabf3a24bc8cfba44050',\n     'wiki.zh_yue.vec': 'd2ac1ab9eb1a908797644f83f259c90cb3c1a350',\n     'wiki.zu.vec': '4b244b9697a8280e6646842c5fc81bb3a6bc8ec7'}\n"
  },
  {
    "path": "python/mxnet/contrib/text/embedding.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=consider-iterating-dictionary\n# pylint: disable=super-init-not-called\n\n\"\"\"Text token embeddings.\"\"\"\n\nimport logging\nimport os\nimport tarfile\nimport warnings\nimport zipfile\n\nfrom . import _constants as C\nfrom . import vocab\nfrom ... import ndarray as nd\nfrom ... import registry\nfrom ... import base\nfrom ...util import is_np_array\nfrom ... import numpy as _mx_np\nfrom ... import numpy_extension as _mx_npx\n\n\ndef register(embedding_cls):\n    \"\"\"Registers a new token embedding.\n\n\n    Once an embedding is registered, we can create an instance of this embedding with\n    :func:`~mxnet.contrib.text.embedding.create`.\n\n\n    Examples\n    --------\n    >>> @mxnet.contrib.text.embedding.register\n    ... class MyTextEmbed(mxnet.contrib.text.embedding._TokenEmbedding):\n    ...     def __init__(self, pretrained_file_name='my_pretrain_file'):\n    ...         pass\n    >>> embed = mxnet.contrib.text.embedding.create('MyTokenEmbed')\n    >>> print(type(embed))\n    <class '__main__.MyTokenEmbed'>\n    \"\"\"\n\n    register_text_embedding = registry.get_register_func(_TokenEmbedding, 'token embedding')\n    return register_text_embedding(embedding_cls)\n\n\ndef create(embedding_name, **kwargs):\n    \"\"\"Creates an instance of token embedding.\n\n\n    Creates a token embedding instance by loading embedding vectors from an externally hosted\n    pre-trained token embedding file, such as those of GloVe and FastText. To get all the valid\n    `embedding_name` and `pretrained_file_name`, use\n    `mxnet.contrib.text.embedding.get_pretrained_file_names()`.\n\n\n    Parameters\n    ----------\n    embedding_name : str\n        The token embedding name (case-insensitive).\n\n\n    Returns\n    -------\n    An instance of `mxnet.contrib.text.glossary._TokenEmbedding`:\n        A token embedding instance that loads embedding vectors from an externally hosted\n        pre-trained token embedding file.\n    \"\"\"\n\n    create_text_embedding = registry.get_create_func(_TokenEmbedding, 'token embedding')\n    return create_text_embedding(embedding_name, **kwargs)\n\n\ndef get_pretrained_file_names(embedding_name=None):\n    \"\"\"Get valid token embedding names and their pre-trained file names.\n\n\n    To load token embedding vectors from an externally hosted pre-trained token embedding file,\n    such as those of GloVe and FastText, one should use\n    `mxnet.contrib.text.embedding.create(embedding_name, pretrained_file_name)`.\n    This method returns all the valid names of `pretrained_file_name` for the specified\n    `embedding_name`. If `embedding_name` is set to None, this method returns all the valid\n    names of `embedding_name` with their associated `pretrained_file_name`.\n\n\n    Parameters\n    ----------\n    embedding_name : str or None, default None\n        The pre-trained token embedding name.\n\n\n    Returns\n    -------\n    dict or list:\n        A list of all the valid pre-trained token embedding file names (`pretrained_file_name`)\n        for the specified token embedding name (`embedding_name`). If the text embeding name is\n        set to None, returns a dict mapping each valid token embedding name to a list of valid\n        pre-trained files (`pretrained_file_name`). They can be plugged into\n        `mxnet.contrib.text.embedding.create(embedding_name,\n        pretrained_file_name)`.\n    \"\"\"\n\n    text_embedding_reg = registry.get_registry(_TokenEmbedding)\n\n    if embedding_name is not None:\n        if embedding_name not in text_embedding_reg:\n            raise KeyError(f'Cannot find `embedding_name` {embedding_name}. Use '\n                           '`get_pretrained_file_names('\n                           'embedding_name=None).keys()` to get all the valid embedding '\n                           'names.')\n        return list(text_embedding_reg[embedding_name].pretrained_file_name_sha1.keys())\n    else:\n        return {embedding_name: list(embedding_cls.pretrained_file_name_sha1.keys())\n                for embedding_name, embedding_cls in registry.get_registry(_TokenEmbedding).items()}\n\n\nclass _TokenEmbedding(vocab.Vocabulary):\n    \"\"\"Token embedding base class.\n\n\n    To load token embeddings from an externally hosted pre-trained token embedding file, such as\n    those of GloVe and FastText, use\n    :func:`~mxnet.contrib.text.embedding.create(embedding_name, pretrained_file_name)`.\n    To get all the available `embedding_name` and `pretrained_file_name`, use\n    :func:`~mxnet.contrib.text.embedding.get_pretrained_file_names()`.\n\n    Alternatively, to load embedding vectors from a custom pre-trained token embedding file, use\n    :class:`~mxnet.contrib.text.embedding.CustomEmbedding`.\n\n    Moreover, to load composite embedding vectors, such as to concatenate embedding vectors, use\n    :class:`~mxnet.contrib.text.embedding.CompositeEmbedding`.\n\n    For every unknown token, if its representation `self.unknown_token` is encountered in the\n    pre-trained token embedding file, index 0 of `self.idx_to_vec` maps to the pre-trained token\n    embedding vector loaded from the file; otherwise, index 0 of `self.idx_to_vec` maps to the\n    token embedding vector initialized by `init_unknown_vec`.\n\n    If a token is encountered multiple times in the pre-trained token embedding file, only the\n    first-encountered token embedding vector will be loaded and the rest will be skipped.\n\n    The indexed tokens in a text token embedding may come from a vocabulary or from the loaded\n    embedding vectors. In the former case, only the indexed tokens in a vocabulary are associated\n    with the loaded embedding vectors, such as loaded from a pre-trained token embedding file. In\n    the later case, all the tokens from the loaded embedding vectors, such as loaded from a\n    pre-trained token embedding file, are taken as the indexed tokens of the embedding.\n\n\n    Attributes\n    ----------\n    token_to_idx : dict mapping str to int\n        A dict mapping each token to its index integer.\n    idx_to_token : list of strs\n        A list of indexed tokens where the list indices and the token indices are aligned.\n    unknown_token : hashable object\n        The representation for any unknown token. In other words, any unknown token will be indexed\n        as the same representation.\n    reserved_tokens : list of strs or None\n        A list of reserved tokens that will always be indexed.\n    vec_len : int\n        The length of the embedding vector for each token.\n    idx_to_vec : mxnet.ndarray.NDArray\n        For all the indexed tokens in this embedding, this NDArray maps each token's index to an\n        embedding vector. The largest valid index maps to the initialized embedding vector for every\n        reserved token, such as an unknown_token token and a padding token.\n    \"\"\"\n\n    def __init__(self, **kwargs):\n        super(_TokenEmbedding, self).__init__(**kwargs)\n\n    @classmethod\n    def _get_download_file_name(cls, pretrained_file_name):\n        return pretrained_file_name\n\n    @classmethod\n    def _get_pretrained_file_url(cls, pretrained_file_name):\n        repo_url = os.environ.get('MXNET_GLUON_REPO', C.APACHE_REPO_URL)\n        embedding_cls = cls.__name__.lower()\n\n        url_format = '{repo_url}gluon/embeddings/{cls}/{file_name}'\n        return url_format.format(repo_url=repo_url, cls=embedding_cls,\n                                 file_name=cls._get_download_file_name(pretrained_file_name))\n\n    @classmethod\n    def _get_pretrained_file(cls, embedding_root, pretrained_file_name):\n        from ...gluon.utils import check_sha1, download\n        embedding_cls = cls.__name__.lower()\n        embedding_root = os.path.expanduser(embedding_root)\n        url = cls._get_pretrained_file_url(pretrained_file_name)\n\n        embedding_dir = os.path.join(embedding_root, embedding_cls)\n        pretrained_file_path = os.path.join(embedding_dir, pretrained_file_name)\n        downloaded_file = os.path.basename(url)\n        downloaded_file_path = os.path.join(embedding_dir, downloaded_file)\n\n        expected_file_hash = cls.pretrained_file_name_sha1[pretrained_file_name]\n\n        if hasattr(cls, 'pretrained_archive_name_sha1'):\n            expected_downloaded_hash = \\\n                cls.pretrained_archive_name_sha1[downloaded_file]\n        else:\n            expected_downloaded_hash = expected_file_hash\n\n        if not os.path.exists(pretrained_file_path) \\\n           or not check_sha1(pretrained_file_path, expected_file_hash):\n            download(url, downloaded_file_path, sha1_hash=expected_downloaded_hash)\n\n            ext = os.path.splitext(downloaded_file)[1]\n            if ext == '.zip':\n                with zipfile.ZipFile(downloaded_file_path, 'r') as zf:\n                    zf.extractall(embedding_dir)\n            elif ext == '.gz':\n                with tarfile.open(downloaded_file_path, 'r:gz') as tar:\n                    tar.extractall(path=embedding_dir)\n        return pretrained_file_path\n\n    def _load_embedding(self, pretrained_file_path, elem_delim, init_unknown_vec, encoding='utf8'):\n        \"\"\"Load embedding vectors from the pre-trained token embedding file.\n\n\n        For every unknown token, if its representation `self.unknown_token` is encountered in the\n        pre-trained token embedding file, index 0 of `self.idx_to_vec` maps to the pre-trained token\n        embedding vector loaded from the file; otherwise, index 0 of `self.idx_to_vec` maps to the\n        text embedding vector initialized by `init_unknown_vec`.\n\n        If a token is encountered multiple times in the pre-trained text embedding file, only the\n        first-encountered token embedding vector will be loaded and the rest will be skipped.\n        \"\"\"\n\n        pretrained_file_path = os.path.expanduser(pretrained_file_path)\n\n        if not os.path.isfile(pretrained_file_path):\n            raise ValueError('`pretrained_file_path` must be a valid path to '\n                             'the pre-trained token embedding file.')\n\n        logging.info('Loading pre-trained token embedding vectors from %s', pretrained_file_path)\n        vec_len = None\n        all_elems = []\n        tokens = set()\n        loaded_unknown_vec = None\n        line_num = 0\n        with open(pretrained_file_path, 'r', encoding=encoding) as f:\n            for line in f:\n                line_num += 1\n                elems = line.rstrip().split(elem_delim)\n\n                assert len(elems) > 1, f'At line {line_num} of the pre-trained text embedding file: the ' \\\n                                       f'data format of the pre-trained token embedding file {pretrained_file_path} ' \\\n                                       'is unexpected.'\n\n                token, elems = elems[0], [float(i) for i in elems[1:]]\n\n                if token == self.unknown_token and loaded_unknown_vec is None:\n                    loaded_unknown_vec = elems\n                    tokens.add(self.unknown_token)\n                elif token in tokens:\n                    warnings.warn(f'At line {line_num} of the pre-trained token embedding file: the '\n                                  f'embedding vector for token {token} has been loaded and a duplicate '\n                                  'embedding for the  same token is seen and skipped.')\n                elif len(elems) == 1:\n                    warnings.warn(f'At line {line_num} of the pre-trained text embedding file: token {token} '\n                                  f'with 1-dimensional vector {elems} is likely a header and is '\n                                  'skipped.')\n                else:\n                    if vec_len is None:\n                        vec_len = len(elems)\n                        # Reserve a vector slot for the unknown token at the very beggining because\n                        # the unknown index is 0.\n                        all_elems.extend([0] * vec_len)\n                    else:\n                        assert len(elems) == vec_len, \\\n                            f'At line {line_num} of the pre-trained token embedding file: the dimension ' \\\n                            f'of token {token} is {len(elems)} but the dimension of previous tokens is {vec_len}. ' \\\n                            'Dimensions of all the tokens must be the same.'\n                    all_elems.extend(elems)\n                    self._idx_to_token.append(token)\n                    self._token_to_idx[token] = len(self._idx_to_token) - 1\n                    tokens.add(token)\n\n        self._vec_len = vec_len\n        array_fn = _mx_np.array if is_np_array() else nd.array\n        self._idx_to_vec = array_fn(all_elems).reshape((-1, self.vec_len))\n\n        if loaded_unknown_vec is None:\n            init_val = init_unknown_vec(shape=self.vec_len)\n            self._idx_to_vec[C.UNKNOWN_IDX] =\\\n                init_val.as_np_ndarray() if is_np_array() else init_val\n        else:\n            self._idx_to_vec[C.UNKNOWN_IDX] = array_fn(loaded_unknown_vec)\n\n    def _index_tokens_from_vocabulary(self, vocabulary):\n        self._token_to_idx = vocabulary.token_to_idx.copy() \\\n            if vocabulary.token_to_idx is not None else None\n        self._idx_to_token = vocabulary.idx_to_token[:] \\\n            if vocabulary.idx_to_token is not None else None\n        self._unknown_token = vocabulary.unknown_token\n        self._reserved_tokens = vocabulary.reserved_tokens[:] \\\n            if vocabulary.reserved_tokens is not None else None\n\n    def _set_idx_to_vec_by_embeddings(self, token_embeddings, vocab_len, vocab_idx_to_token):\n        \"\"\"Sets the mapping between token indices and token embedding vectors.\n\n\n        Parameters\n        ----------\n        token_embeddings : instance or list `mxnet.contrib.text.embedding._TokenEmbedding`\n            One or multiple pre-trained token embeddings to load. If it is a list of multiple\n            embeddings, these embedding vectors will be concatenated for each token.\n        vocab_len : int\n            Length of vocabulary whose tokens are indexed in the token embedding.\n        vocab_idx_to_token: list of str\n            A list of indexed tokens in the vocabulary. These tokens are indexed in the token\n            embedding.\n        \"\"\"\n\n        new_vec_len = sum(embed.vec_len for embed in token_embeddings)\n        zeros_fn = _mx_np.zeros if is_np_array() else nd.zeros\n        new_idx_to_vec = zeros_fn(shape=(vocab_len, new_vec_len))\n\n        col_start = 0\n        # Concatenate all the embedding vectors in token_embeddings.\n        for embed in token_embeddings:\n            col_end = col_start + embed.vec_len\n            # Cancatenate vectors of the unknown token.\n            new_idx_to_vec[0, col_start:col_end] = embed.idx_to_vec[0]\n            new_idx_to_vec[1:, col_start:col_end] = embed.get_vecs_by_tokens(vocab_idx_to_token[1:])\n            col_start = col_end\n\n        self._vec_len = new_vec_len\n        self._idx_to_vec = new_idx_to_vec\n\n    def _build_embedding_for_vocabulary(self, vocabulary):\n        if vocabulary is not None:\n            assert isinstance(vocabulary, vocab.Vocabulary), \\\n                'The argument `vocabulary` must be an instance of ' \\\n                'mxnet.contrib.text.vocab.Vocabulary.'\n\n            # Set _idx_to_vec so that indices of tokens from vocabulary are associated with the\n            # loaded token embedding vectors.\n            self._set_idx_to_vec_by_embeddings([self], len(vocabulary), vocabulary.idx_to_token)\n\n            # Index tokens from vocabulary.\n            self._index_tokens_from_vocabulary(vocabulary)\n\n    @property\n    def vec_len(self):\n        return self._vec_len\n\n    @property\n    def idx_to_vec(self):\n        return self._idx_to_vec\n\n    def get_vecs_by_tokens(self, tokens, lower_case_backup=False):\n        \"\"\"Look up embedding vectors of tokens.\n\n\n        Parameters\n        ----------\n        tokens : str or list of strs\n            A token or a list of tokens.\n        lower_case_backup : bool, default False\n            If False, each token in the original case will be looked up; if True, each token in the\n            original case will be looked up first, if not found in the keys of the property\n            `token_to_idx`, the token in the lower case will be looked up.\n\n\n        Returns\n        -------\n        mxnet.ndarray.NDArray:\n            The embedding vector(s) of the token(s). According to numpy conventions, if `tokens` is\n            a string, returns a 1-D NDArray of shape `self.vec_len`; if `tokens` is a list of\n            strings, returns a 2-D NDArray of shape=(len(tokens), self.vec_len).\n        \"\"\"\n\n        to_reduce = False\n        if not isinstance(tokens, list):\n            tokens = [tokens]\n            to_reduce = True\n\n        if not lower_case_backup:\n            indices = [self.token_to_idx.get(token, C.UNKNOWN_IDX) for token in tokens]\n        else:\n            indices = [self.token_to_idx[token] if token in self.token_to_idx\n                       else self.token_to_idx.get(token.lower(), C.UNKNOWN_IDX)\n                       for token in tokens]\n\n        if is_np_array():\n            embedding_fn = _mx_npx.embedding\n            array_fn = _mx_np.array\n        else:\n            embedding_fn = nd.Embedding\n            array_fn = nd.array\n        vecs = embedding_fn(array_fn(indices), self.idx_to_vec, self.idx_to_vec.shape[0],\n                            self.idx_to_vec.shape[1])\n\n        return vecs[0] if to_reduce else vecs\n\n    def update_token_vectors(self, tokens, new_vectors):\n        \"\"\"Updates embedding vectors for tokens.\n\n\n        Parameters\n        ----------\n        tokens : str or a list of strs\n            A token or a list of tokens whose embedding vector are to be updated.\n        new_vectors : mxnet.ndarray.NDArray\n            An NDArray to be assigned to the embedding vectors of `tokens`. Its length must be equal\n            to the number of `tokens` and its width must be equal to the dimension of embeddings of\n            the glossary. If `tokens` is a singleton, it must be 1-D or 2-D. If `tokens` is a list\n            of multiple strings, it must be 2-D.\n        \"\"\"\n\n        assert self.idx_to_vec is not None, 'The property `idx_to_vec` has not been properly set.'\n\n        if not isinstance(tokens, list) or len(tokens) == 1:\n            assert isinstance(new_vectors, nd.NDArray) and len(new_vectors.shape) in [1, 2], \\\n                '`new_vectors` must be a 1-D or 2-D NDArray if `tokens` is a singleton.'\n            if not isinstance(tokens, list):\n                tokens = [tokens]\n            if len(new_vectors.shape) == 1:\n                expand_dims_fn = _mx_np.expand_dims if is_np_array() else nd.expand_dims\n                new_vectors = expand_dims_fn(new_vectors, axis=0)\n\n        else:\n            assert isinstance(new_vectors, nd.NDArray) and len(new_vectors.shape) == 2, \\\n                '`new_vectors` must be a 2-D NDArray if `tokens` is a list of multiple strings.'\n        assert new_vectors.shape == (len(tokens), self.vec_len), \\\n            'The length of new_vectors must be equal to the number of tokens and the width of' \\\n            'new_vectors must be equal to the dimension of embeddings of the glossary.'\n\n        indices = []\n        for token in tokens:\n            if token in self.token_to_idx:\n                indices.append(self.token_to_idx[token])\n            else:\n                raise ValueError(f'Token {token} is unknown. To update the embedding vector for an '\n                                 'unknown token, please specify it explicitly as the '\n                                 f'`unknown_token` {self.idx_to_token[C.UNKNOWN_IDX]} in `tokens`. '\n                                 'This is to avoid unintended updates.')\n\n        array_fn = _mx_np.array if is_np_array() else nd.array\n        self._idx_to_vec[array_fn(indices)] = new_vectors\n\n    @classmethod\n    def _check_pretrained_file_names(cls, pretrained_file_name):\n        \"\"\"Checks if a pre-trained token embedding file name is valid.\n\n\n        Parameters\n        ----------\n        pretrained_file_name : str\n            The pre-trained token embedding file.\n        \"\"\"\n\n        embedding_name = cls.__name__.lower()\n        if pretrained_file_name not in cls.pretrained_file_name_sha1:\n            raise KeyError(f'Cannot find pretrained file {pretrained_file_name} for token embedding {embedding_name}. Valid '\n                           f'pretrained files for embedding {embedding_name}: {\", \".join(cls.pretrained_file_name_sha1.keys())}')\n\n\n@register\nclass GloVe(_TokenEmbedding):\n    \"\"\"The GloVe word embedding.\n\n\n    GloVe is an unsupervised learning algorithm for obtaining vector representations for words.\n    Training is performed on aggregated global word-word co-occurrence statistics from a corpus, and\n    the resulting representations showcase interesting linear substructures of the word vector\n    space. (Source from https://nlp.stanford.edu/projects/glove/)\n\n    References\n    ----------\n\n    GloVe: Global Vectors for Word Representation.\n    Jeffrey Pennington, Richard Socher, and Christopher D. Manning.\n    https://nlp.stanford.edu/pubs/glove.pdf\n\n    Website:\n\n    https://nlp.stanford.edu/projects/glove/\n\n    To get the updated URLs to the externally hosted pre-trained token embedding\n    files, visit https://nlp.stanford.edu/projects/glove/\n\n    License for pre-trained embeddings:\n\n        https://fedoraproject.org/wiki/Licensing/PDDL\n\n\n    Parameters\n    ----------\n    pretrained_file_name : str, default 'glove.840B.300d.txt'\n        The name of the pre-trained token embedding file.\n    embedding_root : str, default $MXNET_HOME/embeddings\n        The root directory for storing embedding-related files.\n    init_unknown_vec : callback\n        The callback used to initialize the embedding vector for the unknown token.\n    vocabulary : :class:`~mxnet.contrib.text.vocab.Vocabulary`, default None\n        It contains the tokens to index. Each indexed token will be associated with the loaded\n        embedding vectors, such as loaded from a pre-trained token embedding file. If None, all the\n        tokens from the loaded embedding vectors, such as loaded from a pre-trained token embedding\n        file, will be indexed.\n    \"\"\"\n\n    # Map a pre-trained token embedding archive file and its SHA-1 hash.\n    pretrained_archive_name_sha1 = C.GLOVE_PRETRAINED_FILE_SHA1\n\n    # Map a pre-trained token embedding file and its SHA-1 hash.\n    pretrained_file_name_sha1 = C.GLOVE_PRETRAINED_ARCHIVE_SHA1\n\n    @classmethod\n    def _get_download_file_name(cls, pretrained_file_name):\n        # Map a pre-trained embedding file to its archive to download.\n        src_archive = {archive.split('.')[1]: archive for archive in\n                       GloVe.pretrained_archive_name_sha1.keys()}\n        archive = src_archive[pretrained_file_name.split('.')[1]]\n        return archive\n\n    def __init__(self, pretrained_file_name='glove.840B.300d.txt',\n                 embedding_root=os.path.join(base.data_dir(), 'embeddings'),\n                 init_unknown_vec=nd.zeros, vocabulary=None, **kwargs):\n        GloVe._check_pretrained_file_names(pretrained_file_name)\n\n        super(GloVe, self).__init__(**kwargs)\n        pretrained_file_path = GloVe._get_pretrained_file(embedding_root, pretrained_file_name)\n\n        self._load_embedding(pretrained_file_path, ' ', init_unknown_vec)\n\n        if vocabulary is not None:\n            self._build_embedding_for_vocabulary(vocabulary)\n\n\n@register\nclass FastText(_TokenEmbedding):\n    \"\"\"The fastText word embedding.\n\n\n    FastText is an open-source, free, lightweight library that allows users to learn text\n    representations and text classifiers. It works on standard, generic hardware. Models can later\n    be reduced in size to even fit on mobile devices. (Source from https://fasttext.cc/)\n\n    References\n    ----------\n\n    Enriching Word Vectors with Subword Information.\n    Piotr Bojanowski, Edouard Grave, Armand Joulin, and Tomas Mikolov.\n    https://arxiv.org/abs/1607.04606\n\n    Bag of Tricks for Efficient Text Classification.\n    Armand Joulin, Edouard Grave, Piotr Bojanowski, and Tomas Mikolov.\n    https://arxiv.org/abs/1607.01759\n\n    FastText.zip: Compressing text classification models.\n    Armand Joulin, Edouard Grave, Piotr Bojanowski, Matthijs Douze, Herve Jegou,\n    and Tomas Mikolov.\n    https://arxiv.org/abs/1612.03651\n\n    For 'wiki.multi' embeddings:\n    Word Translation Without Parallel Data\n    Alexis Conneau, Guillaume Lample, Marc'Aurelio Ranzato, Ludovic Denoyer,\n    and Herve Jegou.\n    https://arxiv.org/abs/1710.04087\n\n    Website:\n\n    https://fasttext.cc/\n\n    To get the updated URLs to the externally hosted pre-trained token embedding files, visit\n    https://github.com/facebookresearch/fastText/blob/master/docs/pretrained-vectors.md\n\n    License for pre-trained embeddings:\n\n        https://creativecommons.org/licenses/by-sa/3.0/\n\n\n    Parameters\n    ----------\n    pretrained_file_name : str, default 'wiki.en.vec'\n        The name of the pre-trained token embedding file.\n    embedding_root : str, default $MXNET_HOME/embeddings\n        The root directory for storing embedding-related files.\n    init_unknown_vec : callback\n        The callback used to initialize the embedding vector for the unknown token.\n    vocabulary : :class:`~mxnet.contrib.text.vocab.Vocabulary`, default None\n        It contains the tokens to index. Each indexed token will be associated with the loaded\n        embedding vectors, such as loaded from a pre-trained token embedding file. If None, all the\n        tokens from the loaded embedding vectors, such as loaded from a pre-trained token embedding\n        file, will be indexed.\n    \"\"\"\n\n    # Map a pre-trained token embedding archive file and its SHA-1 hash.\n    pretrained_archive_name_sha1 = C.FAST_TEXT_ARCHIVE_SHA1\n\n    # Map a pre-trained token embedding file and its SHA-1 hash.\n    pretrained_file_name_sha1 = C.FAST_TEXT_FILE_SHA1\n\n    @classmethod\n    def _get_download_file_name(cls, pretrained_file_name):\n        # Map a pre-trained embedding file to its archive to download.\n        return '.'.join(pretrained_file_name.split('.')[:-1])+'.zip'\n\n    def __init__(self, pretrained_file_name='wiki.simple.vec',\n                 embedding_root=os.path.join(base.data_dir(), 'embeddings'),\n                 init_unknown_vec=nd.zeros, vocabulary=None, **kwargs):\n        FastText._check_pretrained_file_names(pretrained_file_name)\n\n        super(FastText, self).__init__(**kwargs)\n        pretrained_file_path = FastText._get_pretrained_file(embedding_root, pretrained_file_name)\n\n        self._load_embedding(pretrained_file_path, ' ', init_unknown_vec)\n\n        if vocabulary is not None:\n            self._build_embedding_for_vocabulary(vocabulary)\n\n\nclass CustomEmbedding(_TokenEmbedding):\n    \"\"\"User-defined token embedding.\n\n    This is to load embedding vectors from a user-defined pre-trained text embedding file.\n\n    Denote by '[ed]' the argument `elem_delim`. Denote by [v_ij] the j-th element of the token\n    embedding vector for [token_i], the expected format of a custom pre-trained token embedding file\n    is:\n\n    '[token_1][ed][v_11][ed][v_12][ed]...[ed][v_1k]\\\\\\\\n[token_2][ed][v_21][ed][v_22][ed]...[ed]\n    [v_2k]\\\\\\\\n...'\n\n    where k is the length of the embedding vector `vec_len`.\n\n\n    Parameters\n    ----------\n    pretrained_file_path : str\n        The path to the custom pre-trained token embedding file.\n    elem_delim : str, default ' '\n        The delimiter for splitting a token and every embedding vector element value on the same\n        line of the custom pre-trained token embedding file.\n    encoding : str, default 'utf8'\n        The encoding scheme for reading the custom pre-trained token embedding file.\n    init_unknown_vec : callback\n        The callback used to initialize the embedding vector for the unknown token.\n    vocabulary : :class:`~mxnet.contrib.text.vocab.Vocabulary`, default None\n        It contains the tokens to index. Each indexed token will be associated with the loaded\n        embedding vectors, such as loaded from a pre-trained token embedding file. If None, all the\n        tokens from the loaded embedding vectors, such as loaded from a pre-trained token embedding\n        file, will be indexed.\n    \"\"\"\n\n    def __init__(self, pretrained_file_path, elem_delim=' ', encoding='utf8',\n                 init_unknown_vec=nd.zeros, vocabulary=None, **kwargs):\n        super(CustomEmbedding, self).__init__(**kwargs)\n        self._load_embedding(pretrained_file_path, elem_delim, init_unknown_vec, encoding)\n\n        if vocabulary is not None:\n            self._build_embedding_for_vocabulary(vocabulary)\n\n\nclass CompositeEmbedding(_TokenEmbedding):\n    \"\"\"Composite token embeddings.\n\n\n    For each indexed token in a vocabulary, multiple embedding vectors, such as concatenated\n    multiple embedding vectors, will be associated with it. Such embedding vectors can be loaded\n    from externally hosted or custom pre-trained token embedding files, such as via token embedding\n    instances.\n\n\n    Parameters\n    ----------\n    vocabulary : :class:`~mxnet.contrib.text.vocab.Vocabulary`\n        For each indexed token in a vocabulary, multiple embedding vectors, such as concatenated\n        multiple embedding vectors, will be associated with it.\n    token_embeddings : instance or list of `mxnet.contrib.text.embedding._TokenEmbedding`\n        One or multiple pre-trained token embeddings to load. If it is a list of multiple\n        embeddings, these embedding vectors will be concatenated for each token.\n    \"\"\"\n    def __init__(self, vocabulary, token_embeddings):\n\n        # Sanity checks.\n        assert isinstance(vocabulary, vocab.Vocabulary), \\\n            'The argument `vocabulary` must be an instance of ' \\\n            'mxnet.contrib.text.indexer.Vocabulary.'\n\n        if not isinstance(token_embeddings, list):\n            token_embeddings = [token_embeddings]\n\n        for embed in token_embeddings:\n            assert isinstance(embed, _TokenEmbedding), \\\n                'The argument `token_embeddings` must be an instance or a list of instances ' \\\n                'of `mxnet.contrib.text.embedding.TextEmbedding` whose embedding vectors will be' \\\n                'loaded or concatenated-then-loaded to map to the indexed tokens.'\n\n        # Index tokens.\n        self._index_tokens_from_vocabulary(vocabulary)\n\n        # Set _idx_to_vec so that indices of tokens from keys of `counter` are associated with token\n        # embedding vectors from `token_embeddings`.\n        self._set_idx_to_vec_by_embeddings(token_embeddings, len(self), self.idx_to_token)\n"
  },
  {
    "path": "python/mxnet/contrib/text/utils.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\n\"\"\"Provide utilities for text data processing.\"\"\"\n\nimport collections\nimport re\n\n\ndef count_tokens_from_str(source_str, token_delim=' ', seq_delim='\\n',\n                          to_lower=False, counter_to_update=None):\n    \"\"\"Counts tokens in the specified string.\n\n    For token_delim=\\'<td>\\' and seq_delim=\\'<sd>\\', a specified string of two sequences of\n    tokens may look like::\n\n    <td>token1<td>token2<td>token3<td><sd><td>token4<td>token5<td><sd>\n\n    <td> and <sd> are regular expressions. Make use of \\\\\\\\ to allow special characters as\n    delimiters. The list of\n    special characters can be found at https://docs.python.org/3/library/re.html.\n\n    Parameters\n    ----------\n    source_str : str\n        A source string of tokens.\n    token_delim : str, default ' '\n        A token delimiter.\n    seq_delim : str, default '\\\\\\\\n'\n        A sequence delimiter.\n    to_lower : bool, default False\n        Whether to convert the source source_str to the lower case.\n    counter_to_update : collections.Counter or None, default None\n        The collections.Counter instance to be updated with the token counts of `source_str`. If\n        None, return a new collections.Counter instance counting tokens from `source_str`.\n\n\n    Returns\n    -------\n    collections.Counter\n        The `counter_to_update` collections.Counter instance after being updated with the token\n        counts of `source_str`. If `counter_to_update` is None, return a new collections.Counter\n        instance counting tokens from `source_str`.\n\n\n    Examples\n    --------\n    >>> source_str = ' Life is great ! \\\\n life is good . \\\\n'\n    >>> count_tokens_from_str(token_line, ' ', '\\\\n', True)\n    Counter({'!': 1, '.': 1, 'good': 1, 'great': 1, 'is': 2, 'life': 2})\n\n\n    >>> source_str = '*Life*is*great*!*\\\\n*life*is*good*.*\\\\n'\n    >>> count_tokens_from_str(token_line, '\\\\*', '\\\\n', True)\n    Counter({'is': 2, 'life': 2, '!': 1, 'great': 1, 'good': 1, '.': 1})\n    \"\"\"\n\n    source_str = filter(None,\n                        re.split(token_delim + '|' + seq_delim, source_str))\n    if to_lower:\n        source_str = [t.lower() for t in source_str]\n\n    if counter_to_update is None:\n        return collections.Counter(source_str)  # pylint: disable=too-many-function-args\n    else:\n        counter_to_update.update(source_str)\n        return counter_to_update\n"
  },
  {
    "path": "python/mxnet/contrib/text/vocab.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=consider-iterating-dictionary\n\n\"\"\"Text token indexer.\"\"\"\n\nimport collections\n\nfrom . import _constants as C\n\n\nclass Vocabulary(object):\n    \"\"\"Indexing for text tokens.\n\n\n    Build indices for the unknown token, reserved tokens, and input counter keys. Indexed tokens can\n    be used by token embeddings.\n\n\n    Parameters\n    ----------\n    counter : collections.Counter or None, default None\n        Counts text token frequencies in the text data. Its keys will be indexed according to\n        frequency thresholds such as `most_freq_count` and `min_freq`. Keys of `counter`,\n        `unknown_token`, and values of `reserved_tokens` must be of the same hashable type.\n        Examples: str, int, and tuple.\n    most_freq_count : None or int, default None\n        The maximum possible number of the most frequent tokens in the keys of `counter` that can be\n        indexed. Note that this argument does not count any token from `reserved_tokens`. Suppose\n        that there are different keys of `counter` whose frequency are the same, if indexing all of\n        them will exceed this argument value, such keys will be indexed one by one according to\n        their __cmp__() order until the frequency threshold is met. If this argument is None or\n        larger than its largest possible value restricted by `counter` and `reserved_tokens`, this\n        argument has no effect.\n    min_freq : int, default 1\n        The minimum frequency required for a token in the keys of `counter` to be indexed.\n    unknown_token : hashable object, default '&lt;unk&gt;'\n        The representation for any unknown token. In other words, any unknown token will be indexed\n        as the same representation. Keys of `counter`, `unknown_token`, and values of\n        `reserved_tokens` must be of the same hashable type. Examples: str, int, and tuple.\n    reserved_tokens : list of hashable objects or None, default None\n        A list of reserved tokens that will always be indexed, such as special symbols representing\n        padding, beginning of sentence, and end of sentence. It cannot contain `unknown_token`, or\n        duplicate reserved tokens. Keys of `counter`, `unknown_token`, and values of\n        `reserved_tokens` must be of the same hashable type. Examples: str, int, and tuple.\n\n\n    Attributes\n    ----------\n    unknown_token : hashable object\n        The representation for any unknown token. In other words, any unknown token will be indexed\n        as the same representation.\n    reserved_tokens : list of strs or None\n        A list of reserved tokens that will always be indexed.\n    \"\"\"\n\n    def __init__(self, counter=None, most_freq_count=None, min_freq=1, unknown_token='<unk>',\n                 reserved_tokens=None):\n\n        # Sanity checks.\n        assert min_freq > 0, '`min_freq` must be set to a positive value.'\n\n        if reserved_tokens is not None:\n            reserved_token_set = set(reserved_tokens)\n            assert unknown_token not in reserved_token_set, \\\n                '`reserved_token` cannot contain `unknown_token`.'\n            assert len(reserved_token_set) == len(reserved_tokens), \\\n                '`reserved_tokens` cannot contain duplicate reserved tokens.'\n\n        self._index_unknown_and_reserved_tokens(unknown_token, reserved_tokens)\n\n        if counter is not None:\n            self._index_counter_keys(counter, unknown_token, reserved_tokens, most_freq_count,\n                                     min_freq)\n\n    def _index_unknown_and_reserved_tokens(self, unknown_token, reserved_tokens):\n        \"\"\"Indexes unknown and reserved tokens.\"\"\"\n\n        self._unknown_token = unknown_token\n        # Thus, constants.UNKNOWN_IDX must be 0.\n        self._idx_to_token = [unknown_token]\n\n        if reserved_tokens is None:\n            self._reserved_tokens = None\n        else:\n            self._reserved_tokens = reserved_tokens[:]\n            self._idx_to_token.extend(reserved_tokens)\n\n        self._token_to_idx = {token: idx for idx, token in enumerate(self._idx_to_token)}\n\n    def _index_counter_keys(self, counter, unknown_token, reserved_tokens, most_freq_count,\n                            min_freq):\n        \"\"\"Indexes keys of `counter`.\n\n\n        Indexes keys of `counter` according to frequency thresholds such as `most_freq_count` and\n        `min_freq`.\n        \"\"\"\n\n        assert isinstance(counter, collections.Counter), \\\n            '`counter` must be an instance of collections.Counter.'\n\n        unknown_and_reserved_tokens = set(reserved_tokens) if reserved_tokens is not None else set()\n        unknown_and_reserved_tokens.add(unknown_token)\n\n        token_freqs = sorted(counter.items(), key=lambda x: x[0])\n        token_freqs.sort(key=lambda x: x[1], reverse=True)\n\n        token_cap = len(unknown_and_reserved_tokens) + (\n            len(counter) if most_freq_count is None else most_freq_count)\n\n        for token, freq in token_freqs:\n            if freq < min_freq or len(self._idx_to_token) == token_cap:\n                break\n            if token not in unknown_and_reserved_tokens:\n                self._idx_to_token.append(token)\n                self._token_to_idx[token] = len(self._idx_to_token) - 1\n\n    def __len__(self):\n        return len(self.idx_to_token)\n\n    @property\n    def token_to_idx(self):\n        \"\"\"\n        dict mapping str to int: A dict mapping each token to its index integer.\n        \"\"\"\n        return self._token_to_idx\n\n    @property\n    def idx_to_token(self):\n        \"\"\"\n        list of strs:  A list of indexed tokens where the list indices and the token indices are aligned.\n        \"\"\"\n        return self._idx_to_token\n\n    @property\n    def unknown_token(self):\n        return self._unknown_token\n\n    @property\n    def reserved_tokens(self):\n        return self._reserved_tokens\n\n    def to_indices(self, tokens):\n        \"\"\"Converts tokens to indices according to the vocabulary.\n\n\n        Parameters\n        ----------\n        tokens : str or list of strs\n            A source token or tokens to be converted.\n\n\n        Returns\n        -------\n        int or list of ints\n            A token index or a list of token indices according to the vocabulary.\n        \"\"\"\n\n        to_reduce = False\n        if not isinstance(tokens, list):\n            tokens = [tokens]\n            to_reduce = True\n\n        indices = [self.token_to_idx[token] if token in self.token_to_idx\n                   else C.UNKNOWN_IDX for token in tokens]\n\n        return indices[0] if to_reduce else indices\n\n    def to_tokens(self, indices):\n        \"\"\"Converts token indices to tokens according to the vocabulary.\n\n\n        Parameters\n        ----------\n        indices : int or list of ints\n            A source token index or token indices to be converted.\n\n\n        Returns\n        -------\n        str or list of strs\n            A token or a list of tokens according to the vocabulary.\n        \"\"\"\n\n        to_reduce = False\n        if not isinstance(indices, list):\n            indices = [indices]\n            to_reduce = True\n\n        max_idx = len(self.idx_to_token) - 1\n\n        tokens = []\n        for idx in indices:\n            if not isinstance(idx, int) or idx > max_idx:\n                raise ValueError(f'Token index {idx} in the provided `indices` is invalid.')\n            tokens.append(self.idx_to_token[idx])\n\n        return tokens[0] if to_reduce else tokens\n"
  },
  {
    "path": "python/mxnet/cuda/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Provides python interface to CUDA-related functions of the MXNet library\"\"\"\n\nfrom ..base import _LIB, check_call\nfrom . import nvtx\n\ndef cuda_profiler_start():\n    \"\"\"Starts the CUDA profiler\"\"\"\n    check_call(_LIB.MXCUDAProfilerStart())\n\ndef cuda_profiler_stop():\n    \"\"\"Stops the CUDA profiler\"\"\"\n    check_call(_LIB.MXCUDAProfilerStop())\n"
  },
  {
    "path": "python/mxnet/cuda/nvtx.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Utilities for NVTX usage in MXNet\"\"\"\n\nfrom ..base import _LIB, mx_uint, c_str, check_call\n\n# Palette of colors\nRED = 0xFF0000\nGREEN = 0x00FF00\nBLUE = 0x0000FF\nYELLOW = 0xB58900\nORANGE = 0xCB4B16\nRED1 = 0xDC322F\nMAGENTA = 0xD33682\nVIOLET = 0x6C71C4\nBLUE1 = 0x268BD2\nCYAN = 0x2AA198\nGREEN1 = 0x859900\n\ndef range_push(name, color=ORANGE):\n    \"\"\"Starts a new named NVTX range.\"\"\"\n    check_call(_LIB.MXNVTXRangePush(\n        c_str(name),\n        mx_uint(color)))\n\ndef range_pop():\n    \"\"\"Ends a NVTX range.\"\"\"\n    check_call(_LIB.MXNVTXRangePop())\n\nclass range:\n    def __init__(self, name, color=ORANGE):\n        self.name = name\n        self.color = color\n\n    def __enter__(self):\n        range_push(self.name, self.color)\n\n    def __exit__(self, exc_type, exc_val, exc_tb):\n        range_pop()\n"
  },
  {
    "path": "python/mxnet/cython/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"cython\"\"\"\n"
  },
  {
    "path": "python/mxnet/cython/base.pyi",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n\n#   http://www.apache.org/licenses/LICENSE-2.0\n\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nfrom ..base import get_last_ffi_error\n\nfrom libcpp.vector cimport vector\nfrom libcpp.string cimport string\nfrom libcpp cimport bool as _bool\nfrom cpython.version cimport PY_MAJOR_VERSION\n\nctypedef void* SymbolHandle\nctypedef void* NDArrayHandle\nctypedef void* OpHandle\nctypedef void* CachedOpHandle\nctypedef void* MonitorCallbackHandle\nctypedef unsigned nn_uint\nctypedef void (*CachedOpMonitorCallback)(const char*,\n                                         const char*,\n                                         NDArrayHandle)\n\ncdef py_str(const char* x):\n    if PY_MAJOR_VERSION < 3:\n        return x\n    else:\n        return x.decode(\"utf-8\")\n\ncdef c_str(pystr):\n    \"\"\"Create ctypes char * from a python string\n    Parameters\n    ----------\n    string : string type\n        python string\n\n    Returns\n    -------\n    str : c_char_p\n        A char pointer that can be passed to C API\n    \"\"\"\n    return pystr.encode(\"utf-8\")\n\n\ncdef CALL(int ret):\n    if ret != 0:\n        raise get_last_ffi_error()\n\n\ncdef const char** CBeginPtr(vector[const char*]& vec):\n    if (vec.size() != 0):\n        return &vec[0]\n    else:\n        return NULL\n\ncdef vector[const char*] SVec2Ptr(vector[string]& vec):\n    cdef vector[const char*] svec\n    svec.resize(vec.size())\n    for i in range(vec.size()):\n        svec[i] = vec[i].c_str()\n    return svec\n\n\ncdef extern from \"nnvm/c_api.h\":\n    const char* NNGetLastError();\n    int NNGetOpHandle(const char *op_name,\n                      OpHandle *handle);\n    int NNGetOpInfo(OpHandle op,\n                    const char **name,\n                    const char **description,\n                    nn_uint *num_doc_args,\n                    const char ***arg_names,\n                    const char ***arg_type_infos,\n                    const char ***arg_descriptions,\n                    const char **return_type);\n    int NNSymbolFree(SymbolHandle symbol);\n    int NNSymbolGetNumOutputs(SymbolHandle sym,\n                              nn_uint* output_count);\n    int NNSymbolCompose(SymbolHandle sym,\n                        const char* name,\n                        nn_uint num_args,\n                        const char** keys,\n                        SymbolHandle* args);\n\n\ncdef extern from \"mxnet/c_api.h\":\n    int MXListAllOpNames(nn_uint *out_size,\n                         const char ***out_array);\n    int MXSymbolGetAtomicSymbolInfo(OpHandle creator,\n                                    const char **name,\n                                    const char **description,\n                                    nn_uint *num_doc_args,\n                                    const char ***arg_names,\n                                    const char ***arg_type_infos,\n                                    const char ***arg_descriptions,\n                                    const char **key_var_args,\n                                    const char **return_type);\n    int MXSymbolCreateAtomicSymbol(OpHandle op,\n                                   nn_uint num_param,\n                                   const char **keys,\n                                   const char **vals,\n                                   SymbolHandle *out);\n    int MXSymbolSetAttr(SymbolHandle symbol,\n                        const char* key,\n                        const char* value);\n    int MXImperativeInvoke(OpHandle creator,\n                           int num_inputs,\n                           NDArrayHandle *inputs,\n                           int *num_outputs,\n                           NDArrayHandle **outputs,\n                           int num_params,\n                           const char **param_keys,\n                           const char **param_vals,\n                           const int **out_stypes);\n    int MXNDArrayFree(NDArrayHandle handle);\n"
  },
  {
    "path": "python/mxnet/cython/ndarray.pyx",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\nimport sys as _sys\nimport ctypes as _ctypes\nimport numpy as np\nfrom ..ndarray_doc import _build_doc\nfrom libc.stdint cimport uint32_t, int64_t\nfrom ..base import _LIB\nfrom .. import _global_var\n\ninclude \"./base.pyi\"\n\ncdef class NDArrayBase:\n    \"\"\"Symbol is symbolic graph.\"\"\"\n    # handle for symbolic operator.\n    cdef NDArrayHandle chandle\n    cdef int cwritable\n    cdef public bint _alive\n\n    cdef _set_handle(self, handle):\n        cdef unsigned long long ptr\n        if handle is None:\n            self.chandle = NULL\n        else:\n            if isinstance(handle, (int, long)):\n                ptr = handle\n            else:\n                ptr = handle.value\n            self.chandle = <SymbolHandle>(ptr)\n\n    property handle:\n        def __get__(self):\n            if self.chandle == NULL:\n                return None\n            else:\n                return _ctypes.cast(<unsigned long long>self.chandle, _ctypes.c_void_p)\n        def __set__(self, value):\n            self._set_handle(value)\n    property writable:\n        def __get__(self):\n            return bool(self.cwritable)\n\n    def __init__(self, handle, writable=True):\n        self._set_handle(handle)\n        self.cwritable = writable\n        self._alive = True\n\n    def __dealloc__(self):\n        CALL(MXNDArrayFree(self.chandle))\n        self._alive = False\n\n    def __reduce__(self):\n        return (_global_var._ndarray_cls, (None,), self.__getstate__())\n\n    def _get_handle(self):\n        return <size_t>self.chandle\n\n\ncdef NewArray(NDArrayHandle handle, int stype=-1, int is_np_array=0):\n    \"\"\"Create a new array given handle\"\"\"\n    create_array_fn = _global_var._np_ndarray_cls if is_np_array else _global_var._ndarray_cls\n    return create_array_fn(_ctypes.cast(<unsigned long long>handle, _ctypes.c_void_p), stype=stype)\n\n\ndef _imperative_invoke(handle, ndargs, keys, vals, out, is_np_op=0, output_is_list=0):\n    \"\"\"cython implementation of imperative invoke wrapper\"\"\"\n    cdef unsigned long long ihandle = handle\n    cdef OpHandle chandle = <OpHandle>ihandle\n    cdef vector[string] ckeys\n    cdef vector[string] cvals\n    cdef vector[NDArrayHandle] ndvars\n    cdef vector[NDArrayHandle] output_vars\n    cdef NDArrayHandle* p_output_vars\n    cdef NDArrayHandle ret_handle\n    cdef int num_output\n    cdef const int* p_output_stypes\n\n    for i in ndargs:\n        ndvars.push_back((<NDArrayBase>i).chandle)\n    for i in keys:\n        ckeys.push_back(c_str(i))\n    for i in vals:\n        cvals.push_back(c_str(str(i)))\n\n    original_output = None\n    if out is not None:\n        original_output = out\n        if isinstance(out, NDArrayBase):\n            output_vars.push_back((<NDArrayBase>out).chandle)\n        else:\n            for i in out:\n                output_vars.push_back((<NDArrayBase>i).chandle)\n\n    num_output = output_vars.size()\n    if output_vars.size() == 0:\n        p_output_vars = NULL\n    else:\n        p_output_vars = &output_vars[0]\n\n    cdef vector[const char*] param_keys = SVec2Ptr(ckeys)\n    cdef vector[const char*] param_vals = SVec2Ptr(cvals)\n\n    CALL(MXImperativeInvoke(\n        chandle,\n        <int>ndvars.size(),\n        &ndvars[0] if ndvars.size() != 0 else NULL,\n        &num_output,\n        &p_output_vars,\n        <int>param_keys.size(),\n        CBeginPtr(param_keys),\n        CBeginPtr(param_vals),\n        &p_output_stypes))\n\n    if original_output is not None:\n        return original_output\n    if num_output == 1 and not output_is_list:\n        return NewArray(p_output_vars[0], p_output_stypes[0], is_np_op)\n    else:\n        return [NewArray(p_output_vars[i], p_output_stypes[i], is_np_op) for i in range(num_output)]\n"
  },
  {
    "path": "python/mxnet/cython/symbol.pyx",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\nimport sys as _sys\nimport ctypes as _ctypes\nimport numpy as _numpy\n\nfrom numbers import Number as _Number\nfrom ..name import NameManager\nfrom ..attribute import AttrScope\nfrom ..symbol_doc import _build_doc\n\ninclude \"./base.pyi\"\n\ncdef class SymbolBase:\n    \"\"\"Symbol is symbolic graph.\"\"\"\n    # handle for symbolic operator.\n    cdef SymbolHandle chandle\n    cdef public bint _alive\n\n    cdef _set_handle(self, handle):\n        cdef unsigned long long ptr\n        if handle is None:\n            self.chandle = NULL\n        else:\n            ptr = handle.value\n            self.chandle = <SymbolHandle>(ptr)\n\n    property handle:\n        def __get__(self):\n            if self.chandle == NULL:\n                return None\n            else:\n                return _ctypes.cast(<unsigned long long>self.chandle, _ctypes.c_void_p)\n        def __set__(self, value):\n            self._set_handle(value)\n\n    def __init__(self, handle):\n        self._set_handle(handle)\n        self._alive = True\n\n    def __dealloc__(self):\n        CALL(NNSymbolFree(self.chandle))\n        self._alive = False\n\n    def _set_attr(self, **kwargs):\n        \"\"\"Set the attribute of the symbol.\n\n        Parameters\n        ----------\n        **kwargs\n            The attributes to set\n        \"\"\"\n        SymbolSetAttr(self.chandle, kwargs)\n\n    def __reduce__(self):\n        return (_symbol_cls, (None,), self.__getstate__())\n\n\ncdef SymbolSetAttr(SymbolHandle handle, dict kwargs):\n    cdef string sparam_key\n    cdef string sparam_val\n    cdef const char* param_key\n    cdef const char* param_val\n    for k, v in kwargs.items():\n        sparam_key = c_str(k)\n        sparam_val = c_str(str(v))\n        param_key = sparam_key.c_str()\n        param_val = sparam_val.c_str()\n        CALL(MXSymbolSetAttr(handle, param_key, param_val))\n\n\n_symbol_cls = SymbolBase\n_np_symbol_cls = None\n\ndef _set_symbol_class(cls):\n    global _symbol_cls\n    _symbol_cls = cls\n\n\ndef _set_np_symbol_class(cls):\n    global _np_symbol_cls\n    _np_symbol_cls = cls\n\n\ncdef NewSymbol(SymbolHandle handle, int is_np_sym=0):\n    \"\"\"Create a new symbol given handle\"\"\"\n    create_symbol_fn = _np_symbol_cls if is_np_sym else _symbol_cls\n    sym = create_symbol_fn(None)\n    (<SymbolBase>sym).chandle = handle\n    return sym\n\n\ndef _symbol_creator(handle, args, kwargs, keys, vals, name, is_np_op=0, output_is_list=0):\n    cdef unsigned long long ihandle = handle\n    cdef OpHandle chandle = <OpHandle>ihandle\n    cdef vector[string] ckeys\n    cdef vector[string] cvals\n    cdef vector[string] sym_keys\n    cdef vector[SymbolHandle] sym_args\n    cdef SymbolHandle ret_handle\n    cdef string cname = c_str(name)\n    cdef nn_uint nout\n\n    for i in keys:\n        ckeys.push_back(c_str(i))\n    for i in vals:\n        cvals.push_back(c_str(str(i)))\n\n    cdef vector[const char*] param_keys = SVec2Ptr(ckeys)\n    cdef vector[const char*] param_vals = SVec2Ptr(cvals)\n\n    CALL(MXSymbolCreateAtomicSymbol(\n        chandle,\n        <nn_uint>param_keys.size(),\n        CBeginPtr(param_keys),\n        CBeginPtr(param_vals),\n        &ret_handle))\n\n    if args and kwargs:\n        raise TypeError(\n            'Operators with variable length input can only accept input'\n            'Symbols either as positional or keyword arguments, not both')\n\n    if args:\n        for i in args:\n            sym_args.push_back((<SymbolBase>i).chandle)\n    elif kwargs:\n        for k, v in kwargs.items():\n            sym_keys.push_back(c_str(k))\n            sym_args.push_back((<SymbolBase>v).chandle)\n\n    cdef vector[const char*] csym_keys = SVec2Ptr(sym_keys)\n\n    CALL(NNSymbolCompose(\n        ret_handle,\n        cname.c_str(),\n        <nn_uint>sym_args.size(),\n        &csym_keys[0] if csym_keys.size() != 0 else NULL,\n        &sym_args[0] if sym_args.size() != 0 else NULL))\n\n    sym = NewSymbol(ret_handle, is_np_op)\n    if is_np_op:\n        CALL(NNSymbolGetNumOutputs(ret_handle, &nout))\n        if nout > 1:\n            return list(sym)\n        elif output_is_list:\n            return [sym]\n    return sym\n"
  },
  {
    "path": "python/mxnet/device.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Device management API of mxnet.\"\"\"\nimport contextvars\nimport ctypes\nfrom .base import _LIB\nfrom .base import check_call\n\n\nclass Device:\n    \"\"\"Constructs a device structure.\n\n    MXNet can run operations on CPU and different GPUs.\n    A Device class describes the device type and ID on which computation should be carried on.\n\n    One can use mx.cpu and mx.gpu for short.\n\n    See also\n    ----------\n    `How to run MXNet on multiple CPU/GPUs <http://mxnet.incubator.apache.org/api/faq/distributed_training>`\n    for more details.\n\n    Parameters\n    ----------\n    device_type : {'cpu', 'gpu'} or Device.\n        String representing the device type.\n\n    device_id : int (default=0)\n        The device id of the device, needed for GPU.\n\n    Note\n    ----\n    Device can also be used as a way to change the default device.\n\n    Examples\n    --------\n    >>> # array on cpu\n    >>> cpu_array = mx.np.ones((2, 3))\n    >>> # switch default Device to GPU(2)\n    >>> with mx.Device(mx.gpu(2)):\n    ...     gpu_array = mx.np.ones((2, 3))\n    >>> gpu_array.device\n    gpu(2)\n\n    One can also explicitly specify the device when creating an array.\n\n    >>> gpu_array = mx.np.ones((2, 3), mx.gpu(1))\n    >>> gpu_array.device\n    gpu(1)\n    \"\"\"\n    devtype2str = {1: 'cpu', 2: 'gpu', 3: 'cpu_pinned', 5: 'cpu_shared'}\n    devstr2type = {'cpu': 1, 'gpu': 2, 'cpu_pinned': 3, 'cpu_shared': 5}\n    def __init__(self, device_type, device_id=0):\n        if isinstance(device_type, Device):\n            self.device_typeid = device_type.device_typeid\n            self.device_id = device_type.device_id\n        else:\n            self.device_typeid = Device.devstr2type[device_type]\n            self.device_id = device_id\n        self._old_ctx = None\n\n    @property\n    def device_type(self):\n        \"\"\"Returns the device type of current device.\n\n        Examples\n        -------\n        >>> mx.device.current_device().device_type\n        'cpu'\n        >>> mx.current_device().device_type\n        'cpu'\n\n        Returns\n        -------\n        device_type : str\n        \"\"\"\n        return Device.devtype2str[self.device_typeid]\n\n    def __hash__(self):\n        \"\"\"Compute hash value of device for dictionary lookup\"\"\"\n        return hash((self.device_typeid, self.device_id))\n\n    def __eq__(self, other):\n        \"\"\"Compares two devices. Two devices are equal if they\n        have the same device type and device id.\n        \"\"\"\n        return isinstance(other, Device) and \\\n            self.device_typeid == other.device_typeid and \\\n            self.device_id == other.device_id\n\n    def __str__(self):\n        return f'{self.device_type}({self.device_id})'\n\n    def __repr__(self):\n        return self.__str__()\n\n    def __enter__(self):\n        # Token can't be pickled and Token.old_value is Token.MISSING if _current.get() uses default value\n        self._old_ctx = _current.get()\n        _current.set(self)\n        return self\n\n    def __exit__(self, ptype, value, trace):\n        _current.set(self._old_ctx)\n\n    def empty_cache(self):\n        \"\"\"Empties the memory cache for the current device.\n\n        MXNet utilizes a memory pool to avoid excessive allocations.\n        Calling empty_cache will empty the memory pool of the\n        device. This will only free the memory of the unreferenced data.\n\n        Examples\n        -------\n        >>> ctx = mx.gpu(0)\n        >>> arr = mx.np.ones((200,200), ctx=ctx)\n        >>> del arr\n        >>> ctx.empty_cache() # forces release of memory allocated for arr\n        \"\"\"\n        dev_type = ctypes.c_int(self.device_typeid)\n        dev_id = ctypes.c_int(self.device_id)\n        check_call(_LIB.MXStorageEmptyCache(dev_type, dev_id))\n\n\ndef cpu(device_id=0):\n    \"\"\"Returns a CPU device.\n\n    This function is a short cut for ``Device('cpu', device_id)``.\n    For most operations, when no device is specified, the default device is `cpu()`.\n\n    Examples\n    ----------\n    >>> with mx.cpu():\n    ...     cpu_array = mx.np.ones((2, 3))\n    >>> cpu_array.device\n    cpu(0)\n    >>> cpu_array = mx.np.ones((2, 3), ctx=mx.cpu())\n    >>> cpu_array.device\n    cpu(0)\n\n    Parameters\n    ----------\n    device_id : int, optional\n        The device id of the device. `device_id` is not needed for CPU.\n        This is included to make interface compatible with GPU.\n\n    Returns\n    -------\n    device : Device\n        The corresponding CPU device.\n    \"\"\"\n    return Device('cpu', device_id)\n\n\ndef cpu_pinned(device_id=0):\n    \"\"\"Returns a CPU pinned memory device. Copying from CPU pinned memory to GPU\n    is faster than from normal CPU memory.\n\n    This function is a short cut for ``Device('cpu_pinned', device_id)``.\n\n    Examples\n    ----------\n    >>> with mx.cpu_pinned():\n    ...     cpu_array = mx.np.ones((2, 3))\n    >>> cpu_array.device\n    cpu_pinned(0)\n    >>> cpu_array = mx.np.ones((2, 3), ctx=mx.cpu_pinned())\n    >>> cpu_array.device\n    cpu_pinned(0)\n\n    Parameters\n    ----------\n    device_id : int, optional\n        The device id of the device. `device_id` is not needed for CPU.\n        This is included to make interface compatible with GPU.\n\n    Returns\n    -------\n    device : Device\n        The corresponding CPU pinned memory device.\n    \"\"\"\n    return Device('cpu_pinned', device_id)\n\n\ndef gpu(device_id=0):\n    \"\"\"Returns a GPU device.\n\n    This function is a short cut for Device('gpu', device_id).\n    The K GPUs on a node are typically numbered as 0,...,K-1.\n\n    Examples\n    ----------\n    >>> cpu_array = mx.np.ones((2, 3))\n    >>> cpu_array.device\n    cpu(0)\n    >>> with mx.gpu(1):\n    ...     gpu_array = mx.np.ones((2, 3))\n    >>> gpu_array.device\n    gpu(1)\n    >>> gpu_array = mx.np.ones((2, 3), ctx=mx.gpu(1))\n    >>> gpu_array.device\n    gpu(1)\n\n    Parameters\n    ----------\n    device_id : int, optional\n        The device id of the device, needed for GPU.\n\n    Returns\n    -------\n    device : Device\n        The corresponding GPU device.\n    \"\"\"\n    return Device('gpu', device_id)\n\n\ndef num_gpus():\n    \"\"\"Query CUDA for the number of GPUs present.\n\n    Raises\n    ------\n    Will raise an exception on any CUDA error.\n\n    Returns\n    -------\n    count : int\n        The number of GPUs.\n\n    \"\"\"\n    count = ctypes.c_int()\n    check_call(_LIB.MXGetGPUCount(ctypes.byref(count)))\n    return count.value\n\n\ndef gpu_memory_info(device_id=0):\n    \"\"\"Query CUDA for the free and total bytes of GPU global memory.\n\n    Parameters\n    ----------\n    device_id : int, optional\n        The device id of the GPU device.\n\n    Raises\n    ------\n    Will raise an exception on any CUDA error.\n\n    Returns\n    -------\n    (free, total) : (int, int)\n    \"\"\"\n    free = ctypes.c_uint64()\n    total = ctypes.c_uint64()\n    dev_id = ctypes.c_int(device_id)\n    check_call(_LIB.MXGetGPUMemoryInformation64(dev_id, ctypes.byref(free), ctypes.byref(total)))\n    return (free.value, total.value)\n\n\n_current = contextvars.ContextVar('namemanager', default=Device('cpu', 0))\n\n\ndef current_device():\n    \"\"\"Returns the current device.\n\n    By default, `mx.cpu()` is used for all the computations\n    and it can be overridden by using `with mx.Device(x)` statement where\n    x can be cpu(device_id) or gpu(device_id).\n\n    Examples\n    -------\n    >>> mx.current_device()\n    cpu(0)\n    >>> with mx.Device('gpu', 1):  # Device changed in `with` block.\n    ...    mx.current_device()  # Computation done here will be on gpu(1).\n    ...\n    gpu(1)\n    >>> mx.current_device() # Back to default device.\n    cpu(0)\n\n    Returns\n    -------\n    default_device : Device\n    \"\"\"\n    return _current.get()\n"
  },
  {
    "path": "python/mxnet/dlpack.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=protected-access\n# pylint: disable=import-error, no-name-in-module, undefined-variable\n\n\"\"\"DLPack API of MXNet.\"\"\"\n\nimport ctypes\nimport enum\n\nfrom mxnet.device import current_device\nfrom .base import _LIB, c_str, check_call, NDArrayHandle, mx_int\n\nDLPackHandle = ctypes.c_void_p\n\nPyCapsuleDestructor = ctypes.CFUNCTYPE(None, ctypes.c_void_p)\n_c_str_dltensor = c_str('dltensor')\n_c_str_used_dltensor = c_str('used_dltensor')\n\ndef _dlpack_deleter(pycapsule):\n    pycapsule = ctypes.c_void_p(pycapsule)\n    if ctypes.pythonapi.PyCapsule_IsValid(pycapsule, _c_str_dltensor):\n        ptr = ctypes.c_void_p(\n            ctypes.pythonapi.PyCapsule_GetPointer(pycapsule, _c_str_dltensor))\n        check_call(_LIB.MXNDArrayCallDLPackDeleter(ptr))\n\n_c_dlpack_deleter = PyCapsuleDestructor(_dlpack_deleter)\n\nclass DLDeviceType(enum.IntEnum):\n    DLCPU = 1,\n    DLGPU = 2,\n    DLCPUPINNED = 3,\n    DLOPENCL = 4,\n    DLVULKAN = 7,\n    DLMETAL = 8,\n    DLVPI = 9,\n    DLROCM = 10,\n    DLEXTDEV = 12,\n\n\nclass DLContext(ctypes.Structure):\n    _fields_ = [(\"device_type\", ctypes.c_int),\n                (\"device_id\", ctypes.c_int)]\n\nclass DLDataType(ctypes.Structure):\n    _fields_ = [(\"type_code\", ctypes.c_uint8),\n                (\"bits\", ctypes.c_uint8),\n                (\"lanes\", ctypes.c_uint16)]\n    TYPE_MAP = {\n        \"int32\": (0, 32, 1),\n        \"int64\": (0, 64, 1),\n        \"bool\": (1, 1, 1),\n        \"uint8\": (1, 8, 1),\n        \"uint32\": (1, 32, 1),\n        \"uint64\": (1, 64, 1),\n        'float16': (2, 16, 1),\n        \"float32\": (2, 32, 1),\n        \"float64\": (2, 64, 1),\n    }\n\n\nclass DLTensor(ctypes.Structure):\n    _fields_ = [(\"data\", ctypes.c_void_p),\n                (\"ctx\", DLContext),\n                (\"ndim\", ctypes.c_int),\n                (\"dtype\", DLDataType),\n                (\"shape\", ctypes.POINTER(ctypes.c_int64)),\n                (\"strides\", ctypes.POINTER(ctypes.c_int64)),\n                (\"byte_offset\", ctypes.c_uint64)]\n\nclass DLManagedTensor(ctypes.Structure):\n    pass\n\n\nDeleterFunc = ctypes.CFUNCTYPE(None, ctypes.POINTER(DLManagedTensor))\n\n\nDLManagedTensor._fields_ = [(\"dl_tensor\", DLTensor),           # pylint: disable=protected-access\n                            (\"manager_ctx\", ctypes.c_void_p),\n                            (\"deleter\", DeleterFunc)]\n\n@DeleterFunc\ndef dl_managed_tensor_deleter(dl_managed_tensor_handle):\n    void_p = dl_managed_tensor_handle.contents.manager_ctx\n    pyobj = ctypes.cast(void_p, ctypes.py_object)\n    ctypes.pythonapi.Py_DecRef(pyobj)\n\ndef ndarray_from_dlpack(array_cls):\n    \"\"\"Returns a function that returns specified array_cls from dlpack.\n\n    Returns\n    -------\n    fn : dlpack -> array_cls\n    \"\"\"\n    def from_dlpack(dlpack):\n        tp = type(dlpack)\n        if tp.__module__ == \"builtins\" and tp.__name__ == \"PyCapsule\":\n            dlpack = ctypes.py_object(dlpack)        \n        elif hasattr(dlpack, \"__dlpack__\"):\n            device, device_id = dlpack.__dlpack_device__()\n            if device != DLDeviceType.DLGPU:\n                dlpack = ctypes.py_object(dlpack.__dlpack__())\n            else:\n                s = mx_int()\n                check_call(_LIB.MXGetCurrentStream(\n                    ctypes.c_int(device_id), ctypes.byref(s)))\n                dlpack = ctypes.py_object(dlpack.__dlpack__(stream=s.value))\n        else:\n            raise AttributeError(\"Required PyCapsule or object with __dlpack__\")\n        handle = NDArrayHandle()\n        assert ctypes.pythonapi.PyCapsule_IsValid(dlpack, _c_str_dltensor), ValueError(\n            'Invalid DLPack Tensor. DLTensor capsules can be consumed only once.')\n        dlpack_handle = ctypes.c_void_p(ctypes.pythonapi.PyCapsule_GetPointer(dlpack, _c_str_dltensor))\n        check_call(_LIB.MXNDArrayFromDLPack(dlpack_handle, False, ctypes.byref(handle)))\n        # Rename PyCapsule (DLPack)\n        ctypes.pythonapi.PyCapsule_SetName(dlpack, _c_str_used_dltensor)\n        # delete the deleter of the old dlpack\n        ctypes.pythonapi.PyCapsule_SetDestructor(dlpack, None)\n        return array_cls(handle=handle)\n    return from_dlpack\n\n\ndef ndarray_to_dlpack_for_read():\n    \"\"\"Returns a function that returns dlpack for reading from mxnet array.\n\n    Returns\n    -------\n    fn : tensor -> dlpack\n    \"\"\"\n    def to_dlpack_for_read(data):\n        data.wait_to_read()\n        dlpack = DLPackHandle()\n        check_call(_LIB.MXNDArrayToDLPack(data.handle, ctypes.byref(dlpack)))\n        return ctypes.pythonapi.PyCapsule_New(dlpack, _c_str_dltensor, _c_dlpack_deleter)\n    return to_dlpack_for_read\n\ndef ndarray_to_dlpack_for_write():\n    \"\"\"Returns a function that returns dlpack for writing from mxnet array.\n\n    Returns\n    -------\n    fn : tensor -> dlpack\n    \"\"\"\n    def to_dlpack_for_write(data):\n\n        check_call(_LIB.MXNDArrayWaitToWrite(data.handle))\n        dlpack = DLPackHandle()\n        check_call(_LIB.MXNDArrayToDLPack(data.handle, ctypes.byref(dlpack)))\n        return ctypes.pythonapi.PyCapsule_New(dlpack, _c_str_dltensor, _c_dlpack_deleter)\n    return to_dlpack_for_write\n\ndef ndarray_from_numpy(array_cls, array_create_fn):\n    \"\"\"Returns a function that creates array_cls from numpy array.\n\n    Returns\n    -------\n    fn : tensor -> dlpack\n    \"\"\"\n    def from_numpy(ndarray, zero_copy=True):\n        def _make_manager_ctx(obj):\n            pyobj = ctypes.py_object(obj)\n            void_p = ctypes.c_void_p.from_buffer(pyobj)\n            ctypes.pythonapi.Py_IncRef(pyobj)\n            return void_p\n\n        def _make_dl_tensor(array):\n            if str(array.dtype) not in DLDataType.TYPE_MAP:\n                raise ValueError(str(array.dtype) + \" is not supported.\")\n            dl_tensor = DLTensor()\n            dl_tensor.data = array.ctypes.data_as(ctypes.c_void_p)\n            dl_tensor.ctx = DLContext(1, 0)\n            dl_tensor.ndim = array.ndim\n            dl_tensor.dtype = DLDataType.TYPE_MAP[str(array.dtype)]\n            dl_tensor.shape = array.ctypes.shape_as(ctypes.c_int64)\n            dl_tensor.strides = None\n            dl_tensor.byte_offset = 0\n            return dl_tensor\n\n        def _make_dl_managed_tensor(array):\n            c_obj = DLManagedTensor()\n            c_obj.dl_tensor = _make_dl_tensor(array)\n            c_obj.manager_ctx = _make_manager_ctx(array)\n            c_obj.deleter = dl_managed_tensor_deleter\n            return c_obj\n\n        if not zero_copy:\n            return array_create_fn(ndarray, dtype=ndarray.dtype)\n\n        if not ndarray.flags['C_CONTIGUOUS']:\n            raise ValueError(\"Only c-contiguous arrays are supported for zero-copy\")\n\n        ndarray.flags['WRITEABLE'] = False\n        c_obj = _make_dl_managed_tensor(ndarray)\n        handle = NDArrayHandle()\n        check_call(_LIB.MXNDArrayFromDLPack(ctypes.byref(c_obj), True, ctypes.byref(handle)))\n        return array_cls(handle=handle)\n    return from_numpy\n"
  },
  {
    "path": "python/mxnet/engine.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Engine properties management.\"\"\"\n\nimport ctypes\nfrom .base import _LIB, check_call\n\n\ndef set_bulk_size(size):\n    \"\"\"Set size limit on bulk execution.\n\n    Bulk execution bundles many operators to run together.\n    This can improve performance when running a lot of small\n    operators sequentially.\n\n    Parameters\n    ----------\n    size : int\n        Maximum number of operators that can be bundled in a bulk.\n\n    Returns\n    -------\n    int\n        Previous bulk size.\n    \"\"\"\n    prev = ctypes.c_int()\n    check_call(_LIB.MXEngineSetBulkSize(\n        ctypes.c_int(size), ctypes.byref(prev)))\n    return prev.value\n\n\nclass _BulkScope(object):\n    \"\"\"Scope object for bulk execution.\"\"\"\n    def __init__(self, size):\n        self._size = size\n        self._old_size = None\n\n    def __enter__(self):\n        self._old_size = set_bulk_size(self._size)\n        return self\n\n    def __exit__(self, ptype, value, trace):\n        set_bulk_size(self._old_size)\n\n\ndef bulk(size):\n    \"\"\"Bulk execution bundles many operators to run together.\n    This can improve performance when running a lot of small\n    operators sequentially.\n\n    Returns a scope for managing bulk size::\n\n        with mx.engine.bulk(10):\n            x = mx.nd.zeros((1,))\n            for i in range(100):\n                x += 1\n    \"\"\"\n    return _BulkScope(size)\n"
  },
  {
    "path": "python/mxnet/error.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Structured error classes in MXNet.\n\nEach error class takes an error message as its input.\nSee the example sections for for suggested message conventions.\nTo make the code more readable, we recommended developers to\ncopy the examples and raise errors with the same message convention.\n\"\"\"\nfrom .base import MXNetError, register_error\n\n__all__ = ['MXNetError', 'register']\n\nregister = register_error\n\n@register_error\nclass InternalError(MXNetError):\n    \"\"\"Internal error in the system.\n\n    Examples\n    --------\n    .. code :: c++\n\n        // Example code C++\n        LOG(FATAL) << \"InternalError: internal error detail.\";\n\n    .. code :: python\n\n        # Example code in python\n        raise InternalError(\"internal error detail\")\n    \"\"\"\n    def __init__(self, msg):\n        # Patch up additional hint message.\n        if \"MXNet hint:\" not in msg:\n            msg += (\"\\nMXNet hint: You hit an internal error. Please open an issue in \"\n                    \"https://github.com/apache/mxnet/issues/new/choose\"\n                    \" to report it.\")\n        super(InternalError, self).__init__(msg)\n\n\nregister_error(\"ValueError\", ValueError)\nregister_error(\"TypeError\", TypeError)\nregister_error(\"AttributeError\", AttributeError)\nregister_error(\"IndexError\", IndexError)\nregister_error(\"NotImplementedError\", NotImplementedError)\nregister_error(\"InternalError\", InternalError)\nregister_error(\"IOError\", IOError)\nregister_error(\"FloatingPointError\", FloatingPointError)\nregister_error(\"RuntimeError\", RuntimeError)\n"
  },
  {
    "path": "python/mxnet/executor.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=invalid-name, protected-access, too-many-locals, too-many-arguments\n\"\"\"Symbolic Executor component of MXNet.\"\"\"\n\nimport numpy as np\nfrom . import ndarray\n\nclass Executor:\n    \"\"\"Executor is the object providing efficient symbolic and imperative graph\n    execution and optimization.\n\n    Examples\n    --------\n    >>> # typical approach to create an executor is to bind symbol\n    >>> a = mx.sym.var('a')\n    >>> b = mx.sym.var('b')\n    >>> c = 2 * a + b\n    >>> texec = c._bind(mx.cpu(), {'a': mx.nd.array([1,2]), 'b':mx.nd.array([2,3])})\n    \"\"\"\n    def __init__(self, sym, device, args, args_grad, grad_req, aux_states, static_alloc=False):\n        self.outputs = None\n        self._input_names = sym.list_inputs()\n        self._aux_names = sym.list_auxiliary_states()\n        self._arg_names = sym.list_arguments()\n        self._output_names = sym.list_outputs()\n        self._device = device\n        self._grad_req = grad_req\n        self.static_alloc = static_alloc\n        # grad_req\n        self._requires_grad = False\n        if isinstance(grad_req, dict):\n            for k, v in grad_req.items():\n                if k in self._input_names and v != 'null':\n                    self._requires_grad = True\n        else:\n            assert isinstance(grad_req, str)\n            self._requires_grad = grad_req != 'null'\n\n        # args grad\n        self._args_grad = args_grad\n        if not self._args_grad:\n            self._args_grad = None\n\n        # args\n        self._args = [None] * len(self._input_names)\n        if isinstance(args, dict):\n            for k, v in args.items():\n                try:\n                    i = self._input_names.index(k)\n                    self._args[i] = v.copyto(device)\n                # ignore provided arg which is not present in\n                # input_names\n                except ValueError:\n                    pass\n        else:\n            assert isinstance(args, (list, tuple))\n            for i, arg in enumerate(args):\n                name = self._arg_names[i]\n                index = self._input_names.index(name)\n                self._args[index] = arg.copyto(device)\n\n        # aux states\n        if aux_states:\n            if isinstance(aux_states, dict):\n                for k, v in aux_states.items():\n                    if k in self._aux_names:\n                        i = self._input_names.index(k)\n                        self._args[i] = v.copyto(device)\n            else:\n                assert isinstance(aux_states, (list, tuple))\n                for i, v in enumerate(aux_states):\n                    index = self._input_names.index(self._aux_names[i])\n                    self._args[index] = v.copyto(device)\n\n        # arg grad\n        if self._args_grad:\n            if isinstance(self._args_grad, dict):\n                for k, g in self._args_grad.items():\n                    try:\n                        i = self._input_names.index(k)\n                        # get req\n                        if isinstance(grad_req, str):\n                            req = grad_req\n                        else:\n                            assert isinstance(grad_req, dict)\n                            req = grad_req[k]\n                        if req != 'null':\n                            with self._device:\n                                self._args[i].attach_grad(req, stype=g.stype)\n                                self._args[i].grad[:] = g\n                    # ignore provided arg which is not present in\n                    # input_names\n                    except ValueError:\n                        pass\n            else:\n                assert isinstance(self._args_grad, (list, tuple))\n                for i, g in enumerate(self._args_grad):\n                    # get req\n                    if isinstance(grad_req, str):\n                        req = grad_req\n                    else:\n                        assert isinstance(grad_req, dict)\n                        req = grad_req[self._input_names[i]]\n                    if req != 'null':\n                        with self._device:\n                            self._args[i].attach_grad(req, stype=g.stype)\n                            self._args[i].grad[:] = g\n        self._cached_op = ndarray.CachedOp(sym, flags=[(\"static_alloc\", self.static_alloc)])\n\n    def get_optimized_symbol(self):\n        \"\"\"Get an optimized version of the symbol from the executor.\n\n        Returns\n        -------\n        symbol : Symbol\n            Optimized symbol from the executor.\n        \"\"\"\n        return self._cached_op.get_optimized_symbol()\n\n\n    def forward(self, is_train=False, **kwargs):\n        \"\"\"Calculate the outputs specified by the bound symbol.\n\n        Parameters\n        ----------\n        is_train: bool, optional\n            Whether this forward is for evaluation purpose. If True,\n            a backward call is expected to follow.\n\n        **kwargs\n            Additional specification of input arguments.\n\n        Examples\n        --------\n        >>> # doing forward by specifying data\n        >>> texec.forward(is_train=True, data=mydata)\n        >>> # doing forward by not specifying things, but copy to the executor before hand\n        >>> mydata.copyto(texec.arg_dict['data'])\n        >>> texec.forward(is_train=True)\n        >>> # doing forward by specifying data and get outputs\n        >>> outputs = texec.forward(is_train=True, data=mydata)\n        >>> print(outputs[0].asnumpy())\n        \"\"\"\n        if kwargs:\n            for name, array in kwargs.items():\n                if name in self._input_names:\n                    index = self._input_names.index(name)\n                    with self._device:\n                        arr = ndarray.array(array, dtype=array.dtype)\n                        if self._args[index] is None:\n                            self._args[index] = arr\n                            # get req\n                            if isinstance(self._grad_req, str):\n                                req = self._grad_req\n                            else:\n                                assert isinstance(self._grad_req, dict)\n                                req = self._grad_req[name]\n                            if req != 'null':\n                                with self._device:\n                                    self._args[index].attach_grad(req)\n                        else:\n                            self._args[index][:] = arr\n\n        from . import autograd\n        default_device = None if self._input_names else self._device\n        with autograd.record(train_mode=is_train):\n            self.outputs = self._cached_op(*self._args,\n                                           default_device=default_device)\n        if not isinstance(self.outputs, (list, tuple)):\n            self.outputs = [self.outputs]\n        return self.outputs\n\n    def backward(self, out_grads=None):\n        \"\"\"Do backward pass to get the gradient of arguments.\n\n        Parameters\n        ----------\n        out_grads : NDArray or list of NDArray or dict of str to NDArray, optional\n            Gradient on the outputs to be propagated back.\n            This parameter is only needed when bind is called\n            on outputs that are not a loss function.\n        is_train : bool, default True\n            Whether this backward is for training or inference. Note that in rare\n            cases you want to call backward with is_train=False to get gradient\n            during inference.\n\n        \"\"\"\n        from . import autograd\n        if out_grads is not None:\n            if not isinstance(out_grads, (list, tuple)):\n                out_grads = [out_grads]\n            out_grads = [o.copyto(self._device) for o in out_grads]\n\n        if self._requires_grad:\n            if self.outputs is None:\n                self.forward()\n            autograd.backward(self.outputs, head_grads=out_grads)\n\n            if isinstance(self._args_grad, dict):\n                for k, v in self._args_grad.items():\n                    try:\n                        i = self._input_names.index(k)\n                        if self._args[i].grad is not None:\n                            v[:] = self._args[i].grad\n                    # ignore provided arg grad which is not present in\n                    # input_names\n                    except ValueError:\n                        pass\n            else:\n                assert isinstance(self._args_grad, (list, tuple))\n                for arg, out in zip(self._args, self._args_grad):\n                    if arg.grad is not None:\n                        out[:] = arg.grad\n\n    @property\n    def aux_arrays(self):\n        \"\"\"the auxilary argument array\"\"\"\n        assert isinstance(self._args, list)\n        aux_array = []\n        for name in self._aux_names:\n            index = self._input_names.index(name)\n            aux_array.append(self._args[index])\n        return aux_array\n\n    @property\n    def arg_arrays(self):\n        \"\"\"the argument array\"\"\"\n        assert isinstance(self._args, list)\n        arg_array = []\n        for name in self._arg_names:\n            index = self._input_names.index(name)\n            arg_array.append(self._args[index])\n        return arg_array\n\n    @property\n    def grad_arrays(self):\n        \"\"\"the gradient array\"\"\"\n        if isinstance(self._args_grad, (list, tuple)):\n            return list(self._args_grad)\n\n        arr = [None] * len(self._arg_names)\n        if self._args_grad:\n            assert isinstance(self._args_grad, dict)\n            for k, _ in self._args_grad.items():\n                try:\n                    i = self._input_names.index(k)\n                    j = self._arg_names.index(k)\n                    arr[j] = self._args[i].grad\n                # ignore provided arg grad which is not present in\n                # input_names\n                except ValueError:\n                    pass\n        return arr\n\n    @property\n    def arg_dict(self):\n        \"\"\"Get dictionary representation of argument arrrays.\n\n        Returns\n        -------\n        arg_dict : dict of str to NDArray\n            The dictionary that maps the names of arguments to NDArrays.\n\n        Raises\n        ------\n        ValueError : if there are duplicated names in the arguments.\n        \"\"\"\n        ret = {}\n        for k, v in zip(self._input_names, self._args):\n            if k in self._arg_names:\n                ret[k] = v\n        return ret\n\n    @property\n    def aux_dict(self):\n        \"\"\"Get dictionary representation of auxiliary states arrays.\n\n        Returns\n        -------\n        aux_dict : dict of str to NDArray\n            The dictionary that maps name of auxiliary states to NDArrays.\n\n        Raises\n        ------\n        ValueError : if there are duplicated names in the auxiliary states.\n        \"\"\"\n        ret = {}\n        for k, v in zip(self._input_names, self._args):\n            if k in self._aux_names:\n                ret[k] = v\n        return ret\n\n    @property\n    def grad_dict(self):\n        \"\"\"Get dictionary representation of gradient arrays.\n\n        Returns\n        -------\n        grad_dict : dict of str to NDArray\n            The dictionary that maps name of arguments to gradient arrays.\n        \"\"\"\n        ret = {}\n        for k, v in zip(self._input_names, self._args):\n            if k in self._arg_names:\n                ret[k] = v.grad\n        return ret\n\n    @property\n    def output_dict(self):\n        \"\"\"Get dictionary representation of output arrays.\n\n        Returns\n        -------\n        output_dict : dict of str to NDArray\n            The dictionary that maps name of output names to NDArrays.\n\n        Raises\n        ------\n        ValueError : if there are duplicated names in the outputs.\n        \"\"\"\n        ret = {}\n        for k, v in zip(self._output_names, self.outputs):\n            ret[k] = v\n        return ret\n\n    def copy_params_from(self, arg_params, aux_params=None, allow_extra_params=False):\n        \"\"\"Copy parameters from arg_params, aux_params into executor's internal array.\n\n        Parameters\n        ----------\n        arg_params : dict of str to NDArray\n            Parameters, dict of name to NDArray of arguments.\n\n        aux_params : dict of str to NDArray, optional\n            Parameters, dict of name to NDArray of auxiliary states.\n\n        allow_extra_params : boolean, optional\n            Whether allow extra parameters that are not needed by symbol.\n            If this is True, no error will be thrown when arg_params or aux_params\n            contain extra parameters that is not needed by the executor.\n\n        Raises\n        ------\n        ValueError\n            If there is additional parameters in the dict but ``allow_extra_params=False``.\n\n        Examples\n        --------\n        >>> # set parameters with existing model checkpoint\n        >>> model_prefix = 'mx_mlp'\n        >>> sym, arg_params, aux_params = mx.model.load_checkpoint(model_prefix, 0)\n        >>> texec.copy_params_from(arg_params, aux_params)\n        \"\"\"\n        for name, array in arg_params.items():\n            if name in self.arg_dict:\n                dst = self.arg_dict[name]\n                array.astype(dst.dtype).copyto(dst)\n            elif not allow_extra_params:\n                raise ValueError(f'Find name \\\"{name}\\\" that is not in the arguments')\n\n        if aux_params is None:\n            return\n\n        for name, array in aux_params.items():\n            if name in self.aux_dict:\n                dst = self.aux_dict[name]\n                array.astype(dst.dtype).copyto(dst)\n            elif not allow_extra_params:\n                raise ValueError(f'Find name {name} that is not in the auxiliary states')\n"
  },
  {
    "path": "python/mxnet/gluon/.gitignore",
    "content": "!data\n"
  },
  {
    "path": "python/mxnet/gluon/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Neural network module.\"\"\"\n\nfrom . import metric\n\nfrom .parameter import *\n\nfrom .block import *\n\nfrom . import nn\n\nfrom . import rnn\n\nfrom .trainer import *\n\nfrom . import loss\n\nfrom . import utils\n\nfrom . import data\n\nfrom . import model_zoo\n\nfrom . import contrib\n\nfrom . import probability\n"
  },
  {
    "path": "python/mxnet/gluon/block.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable= arguments-differ, too-many-lines, reimported\n\"\"\"Base container class for all neural network models.\"\"\"\n__all__ = ['Block', 'HybridBlock', 'SymbolBlock']\n\nimport enum\nimport ctypes\nimport copy\nimport warnings\nimport weakref\nfrom collections import OrderedDict, defaultdict\nimport contextlib\nimport contextvars\n\nimport re\nimport json\nimport numpy as np\n\nfrom ..base import mx_real_t, MXNetError, NDArrayHandle, SymbolHandle, py_str, check_call, _LIB\nfrom .. import symbol, ndarray, initializer, autograd, _deferred_compute as dc, name as _name, \\\n    profiler as _profiler, device as _device\nfrom ..symbol.numpy import _symbol as np_symbol\nfrom ..symbol import Symbol, fromjson\nfrom ..ndarray import NDArray, get_dtype_name\nfrom .parameter import Parameter, DeferredInitializationError\nfrom .utils import _indent, _brief_print_list, HookHandle, shape_is_known\nfrom .utils import _check_same_symbol_type, _check_all_np_ndarrays, _check_block_input_np_ndarrays\nfrom .. import numpy_extension as _mx_npx\nfrom .. import numpy as _mx_np, ndarray as nd\nfrom .. util import is_np_array, np_shape, np_array, wrap_ctx_to_device_func\n\n\n_naming_counter = contextvars.ContextVar('namecounter')\n_prefix = contextvars.ContextVar('prefix', default='')\n\n\n@contextlib.contextmanager\ndef _block_scope(block):\n    \"\"\"Append the classname of the current Block to the symbolic and memory profiler name scopes.\"\"\"\n    name = type(block).__name__.lower()\n    counter = _naming_counter.get(None)\n    if counter is not None:\n        count = counter.get(name, 0)\n        counter[name] = count + 1\n        name = f'{name}{count}'\n    counter_token = _naming_counter.set({})\n    prefix_token = _prefix.set(_prefix.get() + name + '_')\n    with _name.Prefix(_prefix.get()):\n        with _profiler.scope(name + ':'):\n            yield\n    _naming_counter.reset(counter_token)\n    _prefix.reset(prefix_token)\n\n\ndef _gather_type_device_info(args):\n    \"\"\"Analyze the elements inside the nested args object and find:\n        - If there exists ndarray\n        - If there exists symbol\n        - All devices appearing in args\n\n    Parameters\n    ----------\n    args : list or NDArray or Symbol\n        Could be a nested architecture.\n\n    Returns\n    -------\n    has_symbol : bool\n        Whether the elements in args contains symbols\n    has_ndarray : bool\n        Whether the elements in args contains ndarrays\n    device_set : set of mxnet.device.Device\n        Contains all possible devices of the inner ndarrays in args. Can be empty if there is no\n        ndarray inside args.\n    first_device : mxnet.device.Device or None\n        Device of the first appeared NDArray (for backward-compatibility)\n    \"\"\"\n    if isinstance(args, NDArray):\n        return False, True, {args.device}, args.device\n    elif isinstance(args, Symbol):\n        return True, False, set(), None\n    elif isinstance(args, (list, tuple)):\n        has_symbol = False\n        has_ndarray = False\n        device_set = set()\n        first_device = None\n        for ele in args:\n            ele_has_sym, ele_has_nd, ele_device_set, ele_first_device =\\\n                _gather_type_device_info(ele)\n            has_symbol = has_symbol or ele_has_sym\n            has_ndarray = has_ndarray or ele_has_nd\n            if first_device is None and ele_first_device is not None:\n                first_device = ele_first_device\n            device_set = device_set | ele_device_set\n            if has_symbol and has_ndarray:\n                break\n        return has_symbol, has_ndarray, device_set, first_device\n    else:\n        return False, False, set(), None\n\n\ndef _flatten(args, inout_str):\n    \"\"\"Parse the arguments into a flattened list + an additional format array.\n    The format array stores the structure of the original arguments to help reconstruct the inputs.\n\n    Parameters\n    ----------\n    args : NDArray, Symbol, or (nested) list of Symbol or NDArray\n        We allow None inside the args.\n    inout_str : str\n        The name of the HybridBlock\n\n    Returns\n    -------\n    flat : list of Symbol or NDArray\n        The flatten version of the input args.\n    fmts : (nested) list of ints\n        Stores the format information of the original structured args.\n    \"\"\"\n    if isinstance(args, NDArray):\n        return [args], int(0)\n    if isinstance(args, Symbol):\n        length = len(args.list_outputs())\n        length = length if length > 1 else 0\n        return [args], int(length)\n    if args is None:\n        return [None], int(-1)\n\n    if not isinstance(args, (list, tuple)):\n        raise ValueError(\"When hybridized, the input of HybridBlock {}\"\n                         \" must be (nested) list of Symbol\"\n                         \" or NDArray, \"\n                         \"but got {} of type {}\".format(inout_str, str(args), str(type(args))))\n    flat = []\n    fmts = []\n    for i in args:\n        arg, fmt = _flatten(i, inout_str)\n        flat.extend(arg)\n        fmts.append(fmt)\n    return flat, fmts\n\n\ndef _regroup(args, fmt):\n    \"\"\"Reconstruct the structured arguments based on the flattened version.\n\n    Parameters\n    ----------\n    args : NDArray, Symbol, or (nested) list of Symbol or NDArray\n        We allow None inside the args.\n    fmt : (nested) list of ints\n        Stores the format information of the original structured args.\n\n    Returns\n    -------\n    ret : NDArray, Symbol, or (nested) list of Symbol or NDArray\n\n    \"\"\"\n    def _merger(args, fmt):\n        \"\"\"Recursive call to merge the arguments\"\"\"\n        if isinstance(fmt, int):\n            if fmt < -1:\n                raise ValueError(\"Unsupported encoded format {}.\".format(fmt))\n            if fmt == 0:\n                return args[0], args[1:]\n            if fmt == -1:\n                if args[0] is not None:\n                    raise ValueError('We do not support passing types that are not None'\n                                     ' when the initial HybridBlock has received NoneType and'\n                                     ' has been hybridized.'\n                                     ' Received arg = {}, fmt = {}.'.format(args[0], fmt))\n                return None, args[1:]\n            else:\n                return args[:fmt], args[fmt:]\n\n        if not isinstance(args, (list, tuple)):\n            raise ValueError(\"When hybridized, the output of HybridBlock must be (nested)\"\n                             \" list of Symbol or NDArray, \"\n                             \"but got {} of type {}\".format(args, type(args)))\n        ret = []\n        for i in fmt:\n            res, args = _merger(args, i)\n            ret.append(res)\n        return ret, args\n    return _merger(args, fmt)[0]\n\n\nclass Block:\n    \"\"\"Base class for all neural network layers and models. Your models should\n    subclass this class.\n\n    :py:class:`Block` can be nested recursively in a tree structure. You can create and\n    assign child :py:class:`Block` as regular attributes::\n\n        import mxnet as mx\n        from mxnet.gluon import Block, nn\n\n        class Model(Block):\n            def __init__(self, **kwargs):\n                super(Model, self).__init__(**kwargs)\n                self.dense0 = nn.Dense(20)\n                self.dense1 = nn.Dense(20)\n\n            def forward(self, x):\n                x = mx.npx.relu(self.dense0(x))\n                return mx.npx.relu(self.dense1(x))\n\n        model = Model()\n        model.initialize(device=mx.cpu(0))\n        model(mx.np.zeros((10, 10), device=mx.cpu(0)))\n\n\n    Child :py:class:`Block` assigned this way will be registered and :py:meth:`collect_params`\n    will collect their Parameters recursively. You can also manually register\n    child blocks with :py:meth:`register_child`.\n\n    \"\"\"\n    def __init__(self):\n        self._children = OrderedDict()\n        self._reg_params = {}\n        self._forward_hooks = OrderedDict()\n        self._forward_pre_hooks = OrderedDict()\n\n    def __repr__(self):\n        s = '{name}(\\n{modstr}\\n)'\n        modstr = '\\n'.join(['  ({key}): {block}'.format(key=key,\n                                                        block=_indent(block.__repr__(), 2))\n                            for key, block in self.__dict__.items() if isinstance(block, Block)])\n        return s.format(name=self.__class__.__name__, modstr=modstr)\n\n    def __setattr__(self, name, value):\n        \"\"\"Registers parameters.\"\"\"\n\n        if hasattr(self, name):\n            existing = getattr(self, name)\n            if isinstance(existing, (Parameter, Block)) and not isinstance(value, type(existing)):\n                raise TypeError('Changing attribute type for {name} from {type1} to {type2}' \\\n                                'is not allowed.'.format(\n                                    name=name, type1=type(existing), type2=type(value)))\n\n        if isinstance(value, Block):\n            self.register_child(value, name)\n        elif isinstance(value, Parameter):\n            self._reg_params[name] = value\n\n        super(Block, self).__setattr__(name, value)\n\n    def _check_container_with_block(self):\n        children = set(self._children.values())\n        def _find_unregistered_block_in_container(data):\n            # Find whether a nested container structure contains Blocks\n            if isinstance(data, (list, tuple)):\n                for ele in data:\n                    if _find_unregistered_block_in_container(ele):\n                        return True\n                return False\n            elif isinstance(data, dict):\n                for _, v in data.items():\n                    if _find_unregistered_block_in_container(v):\n                        return True\n                return False\n            elif isinstance(data, Block):\n                return not data in (c() for c in children)\n            else:\n                return False\n        for k, v in self.__dict__.items():\n            if isinstance(v, (list, tuple, dict)) and not (k.startswith('__') or k == '_children'):\n                if _find_unregistered_block_in_container(v):\n                    warnings.warn('\"{name}\" is an unregistered container with Blocks. '\n                                  'Note that Blocks inside the list, tuple or dict will not be '\n                                  'registered automatically. Make sure to register them using '\n                                  'register_child() or switching to '\n                                  'nn.Sequential/nn.HybridSequential instead. '\n                                  .format(name=self.__class__.__name__ + \".\" + k), stacklevel=3)\n\n    def _alias(self):\n        return self.__class__.__name__.lower()\n\n    @property\n    def params(self):\n        \"\"\"Returns this :py:class:`Block`'s parameter dictionary (does not include its\n        children's parameters).\"\"\"\n        return self._reg_params\n\n    def collect_params(self, select=None):\n        \"\"\"Returns a :py:class:`Dict` containing this :py:class:`Block` and all of its\n        children's Parameters(default), also can returns the select :py:class:`Dict`\n        which match some given regular expressions.\n\n        For example, collect the specified parameters in ['conv1.weight', 'conv1.bias', 'fc.weight',\n        'fc.bias']::\n\n            model.collect_params('conv1.weight|conv1.bias|fc.weight|fc.bias')\n\n        or collect all parameters whose names end with 'weight' or 'bias', this can be done\n        using regular expressions::\n\n            model.collect_params('.*weight|.*bias')\n\n        Parameters\n        ----------\n        select : str\n            regular expressions\n\n        Returns\n        -------\n        The selected :py:class:`Dict`\n        \"\"\"\n        # We need to check here because blocks inside containers are not supported.\n        self._check_container_with_block()\n        return self._collect_params_with_prefix(select=select)\n\n    def _collect_params_with_prefix(self, prefix='', select=None):\n        if prefix:\n            prefix += '.'\n        if select is None:\n            ret = {prefix + key : val for key, val in self._reg_params.items()}\n        else:\n            pattern = re.compile(select)\n            ret = {prefix + key : val for key, val in self._reg_params.items() if pattern.match(prefix + key)}\n\n        for name, child in self._children.items():\n            ret.update(child()._collect_params_with_prefix(prefix + name, select))\n        return ret\n\n    def save_parameters(self, filename, deduplicate=False):\n        \"\"\"Save parameters to file.\n\n        Saved parameters can only be loaded with `load_parameters`. Note that this\n        method only saves parameters, not model structure. If you want to save\n        model structures, please use :py:meth:`HybridBlock.export`.\n\n        Parameters\n        ----------\n        filename : str\n            Path to file.\n        deduplicate : bool, default False\n            If True, save shared parameters only once. Otherwise, if a Block\n            contains multiple sub-blocks that share parameters, each of the\n            shared parameters will be separately saved for every sub-block.\n\n        References\n        ----------\n        `Saving and Loading Gluon Models \\\n        <https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/blocks/save_load_params.html>`_\n        \"\"\"\n        params = self._collect_params_with_prefix()\n\n        if deduplicate:\n            # Shared parameters are stored only a single time as of MXNet 1.6.\n            # Shared parameters are registered under multiple prefixes returned by\n            # _collect_params_with_prefix. We select a single one and only store\n            # it. In load_parameters it is sufficient for a shared parameter to\n            # only set it for a single prefix.\n            reverse_params = {v: k for k, v in params.items()}\n            params = {v: k for k, v in reverse_params.items()}\n\n        arg_dict = {key: val._reduce() for key, val in params.items()}\n        if is_np_array():\n            _mx_npx.savez(filename, **arg_dict)\n        else:\n            ndarray.save(filename, arg_dict)\n\n    @wrap_ctx_to_device_func\n    def load_parameters(self, filename, device=None, allow_missing=False,\n                        ignore_extra=False, cast_dtype=False, dtype_source='current'):\n        \"\"\"Load parameters from file previously saved by `save_parameters`.\n\n        Parameters\n        ----------\n        filename : str\n            Path to parameter file.\n        device : Device or list of Device, default cpu()\n            Device(s) to initialize loaded parameters on.\n        allow_missing : bool, default False\n            Whether to silently skip loading parameters not represents in the file.\n        ignore_extra : bool, default False\n            Whether to silently ignore parameters from the file that are not\n            present in this Block.\n        cast_dtype : bool, default False\n            Cast the data type of the NDArray loaded from the checkpoint to the dtype\n            provided by the Parameter if any.\n        dtype_source : str, default 'current'\n            must be in {'current', 'saved'}\n            Only valid if cast_dtype=True, specify the source of the dtype for casting\n            the parameters\n        References\n        ----------\n        `Saving and Loading Gluon Models \\\n        <https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/blocks/save_load_params.html>`_\n        \"\"\"\n        if is_np_array():\n            # failure may happen when loading parameters saved as NDArrays within\n            # NumPy semantics. Check the failure type and recover from it if it happens.\n            try:\n                loaded = _mx_npx.load(filename)\n            except MXNetError as e:\n                err_msg = str(e)\n                if 'is_np_shape' in err_msg:\n                    # Loading failure due to parameters saved without numpy semantics.\n                    # Temporarily disable numpy semantics and load parameters. After it's\n                    # done, resume the numpy semantics. This is fine because the cases\n                    # numpy ndarray covers is a superset of the legacy ndarray's.\n                    with np_array(False):\n                        with np_shape(False):\n                            loaded_nds = ndarray.load(filename)\n                    assert isinstance(loaded_nds, dict),\\\n                        'expecting a dict type, got {}'.format(str(type(loaded_nds)))\n                    loaded = {k: loaded_nds[k].as_np_ndarray() for k in loaded_nds}\n                else:\n                    raise ValueError(err_msg)\n        else:\n            loaded = ndarray.load(filename)\n\n        if not loaded:\n            return\n        full_dict = {'params': loaded, 'filename': filename}\n        self.load_dict(full_dict, device, allow_missing, ignore_extra, cast_dtype, dtype_source)\n\n    def load_dict(self, param_dict, device=None, allow_missing=False,\n                  ignore_extra=False, cast_dtype=False, dtype_source=\"current\"):\n        \"\"\"Load parameters from dict\n\n        Parameters\n        ----------\n        param_dict : dict\n            Dictionary containing model parameters\n        device : Device, optional\n            Device context on which the memory is allocated. Default is\n            `mxnet.device.current_device()`.\n        allow_missing : bool, default False\n            Whether to silently skip loading parameters not represented in the file.\n        ignore_extra : bool, default False\n            Whether to silently ignore parameters from the file that are not\n            present in this dict.\n        cast_dtype : bool, default False\n            Cast the data type of the NDArray loaded from the checkpoint to the dtype\n            provided by the Parameter if any\n        dtype_source : str, default 'current'\n            must be in {'current', 'saved'}\n            Only valid if cast_dtype=True, specify the source of the dtype for casting\n            the parameters\n        \"\"\"\n        if isinstance(param_dict.get('filename'), str):\n            # pass from load_parameters\n            filename = param_dict['filename']\n            param_dict = param_dict['params']\n        else:\n            filename = None\n        params = self.collect_params()\n        error_str = f\"file: {filename}\" if filename else \"param_dict\"\n        loaded = {k[4:] if k.startswith('arg:') or k.startswith('aux:') else k: v \\\n                  for k, v in param_dict.items()}\n\n        if not allow_missing:\n            params_inv = defaultdict(list)\n            for k, v in params.items():\n                params_inv[v].append(k)\n\n            for name, param in params.items():\n                assert any(p in loaded for p in params_inv[param]), \\\n                f\"Parameter '{name}' is missing in '{error_str}', which contains parameters: {_brief_print_list(loaded.keys())}. \" \\\n                    \"Set allow_missing=True to ignore missing parameters.\"\n\n        if device is None:\n            device = _device.current_device()\n        for name in loaded:\n            if not ignore_extra and name not in params:\n                raise ValueError(\n                    f\"Parameter '{name}' loaded from '{error_str}' is not present in Dict, \" \\\n                    f\"which contains parameters {_brief_print_list(params.keys())}. Set ignore_extra=True to ignore. \")\n            if name in params:\n                param = loaded[name]\n                if isinstance(param, np.ndarray):\n                    param = _mx_np.array(param) if is_np_array() else nd.array(param)\n                params[name]._load_init(param, device, cast_dtype=cast_dtype, dtype_source=dtype_source)\n\n    def register_child(self, block, name=None):\n        \"\"\"Registers block as a child of self. :py:class:`Block` s assigned to self as\n        attributes will be registered automatically.\"\"\"\n        if name is None:\n            name = str(len(self._children))\n        self._children[name] = weakref.ref(block)\n\n    def register_forward_pre_hook(self, hook):\n        r\"\"\"Registers a forward pre-hook on the block.\n\n        The hook function is called immediately before :func:`forward`.\n        It should not modify the input or output.\n\n        Parameters\n        ----------\n        hook : callable\n            The forward hook function of form `hook(block, input) -> None`.\n\n        Returns\n        -------\n        :class:`mxnet.gluon.utils.HookHandle`\n        \"\"\"\n        handle = HookHandle()\n        handle.attach(self._forward_pre_hooks, hook)\n        return handle\n\n    def register_forward_hook(self, hook):\n        r\"\"\"Registers a forward hook on the block.\n\n        The hook function is called immediately after :func:`forward`.\n        It should not modify the input or output.\n\n        Parameters\n        ----------\n        hook : callable\n            The forward hook function of form `hook(block, input, output) -> None`.\n\n        Returns\n        -------\n        :class:`mxnet.gluon.utils.HookHandle`\n        \"\"\"\n        handle = HookHandle()\n        handle.attach(self._forward_hooks, hook)\n        return handle\n\n    def apply(self, fn):\n        r\"\"\"Applies ``fn`` recursively to every child block as well as self.\n\n        Parameters\n        ----------\n        fn : callable\n            Function to be applied to each submodule, of form `fn(block)`.\n\n        Returns\n        -------\n        this block\n        \"\"\"\n        for cld in self._children.values():\n            cld().apply(fn)\n        fn(self)\n        return self\n\n    @wrap_ctx_to_device_func\n    def initialize(self, init=initializer.Uniform(), device=None, verbose=False,\n                   force_reinit=False):\n        \"\"\"Initializes :py:class:`Parameter` s of this :py:class:`Block` and its children.\n\n        Parameters\n        ----------\n        init : Initializer\n            Global default Initializer to be used when :py:meth:`Parameter.init` is ``None``.\n            Otherwise, :py:meth:`Parameter.init` takes precedence.\n        device : Device or list of Device\n            Keeps a copy of Parameters on one or many device(s).\n        verbose : bool, default False\n            Whether to verbosely print out details on initialization.\n        force_reinit : bool, default False\n            Whether to force re-initialization if parameter is already initialized.\n        \"\"\"\n        params = self.collect_params()\n        if verbose:\n            init.set_verbosity(verbose=verbose)\n        for v in params.values():\n            v.initialize(None, device, init, force_reinit=force_reinit)\n\n    def save(self, prefix):\n        \"\"\"Save the model architecture and parameters to load again later\n\n        Saves the model architecture as a nested dictionary where each Block\n        in the model is a dictionary and its children are sub-dictionaries.\n\n        Each Block is uniquely identified by Block class name and a unique ID.\n        We save each Block's parameter UUID to restore later in order to match\n        the saved parameters.\n\n        Recursively traverses a Block's children in order (since its an\n        OrderedDict) and uses the unique ID to denote that specific Block.\n\n        Assumes that the model is created in an identical order every time.\n        If the model is not able to be recreated deterministically do not\n        use this set of APIs to save/load your model.\n\n        For HybridBlocks, the cached_graph is saved (Symbol & inputs) if\n        it has already been hybridized.\n\n        Parameters\n        ----------\n        prefix : str\n            The prefix to use in filenames for saving this model:\n            <prefix>-model.json and <prefix>-model.params\n        \"\"\"\n        # create empty model structure\n        model = {}\n        def _save_cached_graphs(blk, structure, index=0):\n            # create new entry for this block\n            mdl = {}\n            # encode unique name based on block type and ID\n            name = type(blk).__name__.lower()\n            structure[name+str(index)] = mdl\n            index += 1\n            if isinstance(blk, HybridBlock):\n                if blk._cached_graph:\n                    # save in/out formats\n                    mdl['in_format'] = blk._in_format\n                    mdl['out_format'] = blk._out_format\n                    # save cached graph & input symbols\n                    syms, out = blk._cached_graph\n                    mdl_syms = []\n                    for sym in syms:\n                        mdl_syms.append(sym.tojson())\n                    mdl['inputs'] = mdl_syms\n                    mdl['symbol'] = out.tojson()\n                    mdl['hybridized'] = True\n                else:\n                    mdl['hybridized'] = False\n            # save param uuids\n            pmap = {}\n            mdl['params'] = pmap\n            pnames = list(blk.params.keys())\n            for p in pnames:\n                param = blk.params[p]\n                pmap[p] = param._uuid\n            # recursively save children\n            for child in blk._children.values():\n                index = _save_cached_graphs(child(), mdl, index)\n            # return latest index (ie. block count)\n            return index\n\n        # save top-level block\n        _save_cached_graphs(self, model)\n        # save model\n        with open(prefix+'-model.json', 'w') as fp:\n            json.dump(model, fp)\n        # save params\n        self.save_parameters('MyModel-model.params')\n\n    def load(self, prefix):\n        \"\"\"Load a model saved using the `save` API\n\n        Reconfigures a model using the saved configuration. This function\n        does not regenerate the model architecture. It resets each Block's\n        parameter UUIDs as they were when saved in order to match the names of the\n        saved parameters.\n\n        This function assumes the Blocks in the model were created in the same\n        order they were when the model was saved. This is because each Block is\n        uniquely identified by Block class name and a unique ID in order (since\n        its an OrderedDict) and uses the unique ID to denote that specific Block.\n\n        Assumes that the model is created in an identical order every time.\n        If the model is not able to be recreated deterministically do not\n        use this set of APIs to save/load your model.\n\n        For HybridBlocks, the cached_graph (Symbol & inputs) and settings are\n        restored if it had been hybridized before saving.\n\n        Parameters\n        ----------\n        prefix : str\n            The prefix to use in filenames for loading this model:\n            <prefix>-model.json and <prefix>-model.params\n        \"\"\"\n        # load model json from file\n        with open(prefix+'-model.json') as fp:\n            model = json.load(fp)\n\n        def _load_cached_graphs(blk, structure, index=0):\n            # get block name\n            name = type(blk).__name__.lower()\n            # lookup previous encoded name based on block type and ID\n            mdl = structure[name+str(index)]\n            index += 1\n            if isinstance(blk, HybridBlock):\n                if mdl['hybridized']:\n                    # restore in/out formats\n                    blk._in_format = mdl['in_format']\n                    blk._out_format = mdl['out_format']\n                    # get saved symbol\n                    out = fromjson(mdl['symbol'])\n                    syms = []\n                    # recreate inputs for this symbol\n                    for inp in mdl['inputs']:\n                        syms.append(fromjson(inp))\n                    # reset cached_graph and active status\n                    blk._cached_graph = (syms, out)\n                    blk._active = True\n            # reload param uuids\n            pmap = mdl['params']\n            for p, uuid in pmap.items():\n                param = blk.params[p]\n                param._uuid = uuid\n            # recursively reload children\n            for child in blk._children.values():\n                index = _load_cached_graphs(child(), mdl, index)\n            # return latest index (ie. block count)\n            return index\n\n        # load top-level block\n        _load_cached_graphs(self, model)\n        # load params\n        self.load_parameters('MyModel-model.params')\n\n    def hybridize(self, active=True, **kwargs):\n        \"\"\" Please refer description of HybridBlock hybridize().\n        \"\"\"\n        for cld in self._children.values():\n            cld().hybridize(active, **kwargs)\n\n    def cast(self, dtype):\n        \"\"\"Cast this Block to use another data type.\n\n        Parameters\n        ----------\n        dtype : str or numpy.dtype\n            The new data type.\n        \"\"\"\n        for child in self._children.values():\n            child().cast(dtype)\n        for _, param in self.params.items():\n            param.cast(dtype)\n\n    def zero_grad(self):\n        \"\"\"Sets all Parameters' gradient buffer to 0.\"\"\"\n        # collect gradient arrays for each device\n        arrays = defaultdict(list)\n        params = self.collect_params()\n        for p in params.values():\n            if p.grad_req == 'null' or p._grad is None:\n                continue\n            for g in p.list_grad():\n                if g.stype == 'row_sparse':\n                    ndarray.zeros_like(g, out=g)\n                else:\n                    if is_np_array():\n                        arrays[g.device].append(g.as_nd_ndarray())\n                    else:\n                        arrays[g.device].append(g)\n\n        if len(arrays) == 0:\n            return\n\n        for arr in arrays.values():\n            ndarray.reset_arrays(*arr, num_arrays=len(arr))\n\n    def reset_device(self, device):\n        \"\"\"Re-assign all Parameters to other devices.\n\n        Parameters\n        ----------\n        device : Device or list of Device, default :py:meth:`device.current_device()`.\n            Assign Parameter to given device. If device is a list of Device, a\n            copy will be made for each device.\n        \"\"\"\n        params = self.collect_params()\n        for i in params.values():\n            i.reset_device(device)\n\n    def reset_ctx(self, ctx):\n        \"\"\"This function has been deprecated. Please refer to ``Block.reset_device``.\"\"\"\n        warnings.warn('Block.reset_ctx has been renamed to'\n                      ' Block.reset_device', DeprecationWarning)\n        self.reset_device(ctx)\n\n    def setattr(self, name, value):\n        \"\"\"Set an attribute to a new value for all Parameters.\n\n        For example, set grad_req to null if you don't need gradient w.r.t a\n        model's Parameters::\n\n            model.setattr('grad_req', 'null')\n\n        or change the learning rate multiplier::\n\n            model.setattr('lr_mult', 0.5)\n\n        Parameters\n        ----------\n        name : str\n            Name of the attribute.\n        value : valid type for attribute name\n            The new value for the attribute.\n        \"\"\"\n        params = self.collect_params()\n        for i in params.values():\n            setattr(i, name, value)\n\n    def share_parameters(self, shared):\n        \"\"\"Share parameters recursively inside the model.\n\n        For example, if you want ``dense1`` to share ``dense0``'s weights, you can do::\n\n            dense0 = nn.Dense(20)\n            dense1 = nn.Dense(20)\n            dense1.share_parameters(dense0.collect_params())\n\n        which equals to\n            dense1.weight = dense0.weight\n            dense1.bias = dense0.bias\n\n        Note that unlike the `load_parameters` or `load_dict` functions,\n        `share_parameters` results in the `Parameter` object being shared (or\n        tied) between the models, whereas `load_parameters` or `load_dict` only\n        set the value of the data dictionary of a model. If you call\n        `load_parameters` or `load_dict` after `share_parameters`, the loaded\n        value will be reflected in all networks that use the shared (or tied)\n        `Parameter` object.\n\n        Parameters\n        ----------\n        shared : Dict\n            Dict of the shared parameters.\n\n        Returns\n        -------\n        this block\n        \"\"\"\n        if shared is None:\n            return self\n        if not isinstance(shared, (dict, OrderedDict)):\n            raise ValueError(\"'shared' should be in type of Dict. Get type {}!\".format(type(shared)))\n        shared_set = set(shared.keys())\n        self._shared_parameters(shared, shared_set)\n        if len(shared_set) > 0:\n            for name in shared_set:\n                warnings.warn(\"Parameter name {} is not in the current model!\".format(name))\n        return self\n\n    def _shared_parameters(self, shared, shared_set, prefix=\"\"):\n        if prefix:\n            prefix += '.'\n        for name in self._reg_params:\n            key = prefix + name\n            if shared.get(key) is not None:\n                setattr(self, name, shared[key])\n                shared_set.remove(key)\n        for name, child in self._children.items():\n            child()._shared_parameters(shared, shared_set, prefix + name)\n\n    def __call__(self, *args):\n        \"\"\"Calls forward. Only accepts positional arguments.\"\"\"\n        for hook in self._forward_pre_hooks.values():\n            hook(self, args)\n\n        out = self.forward(*args)\n\n        for hook in self._forward_hooks.values():\n            hook(self, args, out)\n        if _mx_npx.is_np_array():\n            _check_all_np_ndarrays(out)\n        return out\n\n    def forward(self, *args):\n        \"\"\"Overrides to implement forward computation using :py:class:`NDArray`. Only\n        accepts positional arguments.\n\n        Parameters\n        ----------\n        *args : list of NDArray\n            Input tensors.\n        \"\"\"\n        # pylint: disable= invalid-name\n        raise NotImplementedError\n\n    def register_op_hook(self, callback, monitor_all=False):\n        \"\"\"Install callback monitor.\n\n        Parameters\n        ----------\n        callback : function\n            Function called to inspect the values of the intermediate outputs\n            of blocks after hybridization. It takes 3 parameters:\n            name of the tensor being inspected (str)\n            name of the operator producing or consuming that tensor (str)\n            tensor being inspected (NDArray).\n        monitor_all : bool, default False\n            If True, monitor both input and output, otherwise monitor output only.\n        \"\"\"\n        for cld in self._children.values():\n            cld().register_op_hook(callback, monitor_all)\n\n    def summary(self, *inputs):\n        \"\"\"Print the summary of the model's output and parameters.\n\n        The network must have been initialized, and must not have been hybridized.\n\n        Parameters\n        ----------\n        inputs : object\n            Any input that the model supports. For any tensor in the input, only\n            :class:`mxnet.ndarray.NDArray` is supported.\n        \"\"\"\n        summary = OrderedDict()\n        seen = set()\n        hooks = []\n\n        def _get_shape_str(args):\n            def flatten(args):\n                if not isinstance(args, (list, tuple)):\n                    return [args], int(0)\n                flat = []\n                fmts = []\n                for i in args:\n                    arg, fmt = flatten(i)\n                    flat.extend(arg)\n                    fmts.append(fmt)\n                return flat, fmts\n\n            def regroup(args, fmt):\n                if isinstance(fmt, int):\n                    if fmt == 0:\n                        return args[0], args[1:]\n                    return args[:fmt], args[fmt:]\n                ret = []\n                for i in fmt:\n                    res, args = regroup(args, i)\n                    ret.append(res)\n                return ret, args\n\n            flat_args, fmts = flatten(args)\n            flat_arg_shapes = [x.shape if isinstance(x, ndarray.NDArray) else x\n                               for x in flat_args]\n            shapes = regroup(flat_arg_shapes, fmts)[0]\n            if isinstance(shapes, list):\n                shape_str = str(shapes)[1:-1]\n            else:\n                shape_str = str(shapes)\n            return shape_str.replace('L', '')\n\n        def _register_summary_hook(block):\n            assert not isinstance(block, HybridBlock) or not block._active, \\\n                    '\"{}\" must not be hybridized to print summary.'.format(type(block).__name__)\n            def _summary_hook(block, _, outputs):\n                class_name = block.__class__.__name__\n                block_idx = len(summary) - 1\n\n                m_key = f'{class_name}-{block_idx+1}'\n                summary[m_key] = OrderedDict()\n                summary[m_key]['output_shape'] = _get_shape_str(outputs)\n\n                params = 0\n                summary[m_key]['trainable'] = 0\n                summary[m_key]['shared'] = 0\n                for p in block.params.values():\n                    params += p.data().size\n                    summary[m_key]['trainable'] += 0 if p.grad_req == 'null' else p.data().size\n                    if p in seen:\n                        summary[m_key]['shared'] += p.data().size\n                    else:\n                        seen.add(p)\n                summary[m_key]['n_params'] = params\n\n            from .nn.basic_layers import Sequential, HybridSequential\n            if not isinstance(block, (Sequential, HybridSequential)):\n                hooks.append(block.register_forward_hook(_summary_hook))\n\n        summary['Input'] = OrderedDict()\n        summary['Input']['output_shape'] = _get_shape_str(inputs)\n        summary['Input']['n_params'] = 0\n        summary['Input']['trainable'] = 0\n        summary['Input']['shared'] = 0\n\n        try:\n            self.apply(_register_summary_hook)\n            self(*inputs)\n\n            line_format = '{:>20}  {:>42} {:>15}'\n            print('-'*80)\n            print(line_format.format('Layer (type)', 'Output Shape', 'Param #'))\n            print('='*80)\n            total_params = 0\n            trainable_params = 0\n            shared_params = 0\n            for layer in summary:\n                print(line_format.format(layer,\n                                         str(summary[layer]['output_shape']),\n                                         summary[layer]['n_params']))\n                total_params += summary[layer]['n_params']\n                trainable_params += summary[layer]['trainable']\n                shared_params += summary[layer]['shared']\n            print('='*80)\n            print('Parameters in forward computation graph, duplicate included')\n            print('   Total params: ' + str(total_params))\n            print('   Trainable params: ' + str(trainable_params))\n            print('   Non-trainable params: ' + str(total_params - trainable_params))\n            print('Shared params in forward computation graph: ' + str(shared_params))\n            print('Unique parameters in model: ' + str(total_params - shared_params))\n            print('-'*80)\n        finally:\n            for h in hooks:\n                h.detach()\n\n\nclass HybridBlock(Block):\n    \"\"\"`HybridBlock` supports forwarding with both Symbol and NDArray.\n\n    `HybridBlock` is similar to `Block`, with a few differences::\n\n        import mxnet as mx\n        from mxnet.gluon import HybridBlock, nn\n\n        class Model(HybridBlock):\n            def __init__(self, **kwargs):\n                super(Model, self).__init__(**kwargs)\n                self.dense0 = nn.Dense(20)\n                self.dense1 = nn.Dense(20)\n\n            def forward(self, x):\n                x = mx.npx.relu(self.dense0(x))\n                return mx.npx.relu(self.dense1(x))\n\n        model = Model()\n        model.initialize(device=mx.cpu(0))\n        model.hybridize()\n        model(mx.np.zeros((10, 10), device=mx.cpu(0)))\n\n    Forward computation in :py:class:`HybridBlock` must be static to work with :py:class:`Symbol` s,\n    i.e. you cannot call :py:meth:`NDArray.asnumpy`, :py:attr:`NDArray.shape`,\n    :py:attr:`NDArray.dtype`, `NDArray` indexing (`x[i]`) etc on tensors.\n    Also, you cannot use branching or loop logic that bases on non-constant\n    expressions like random numbers or intermediate results, since they change\n    the graph structure for each iteration.\n\n    Before activating with :py:meth:`hybridize()`, :py:class:`HybridBlock` works just like normal\n    :py:class:`Block`. After activation, :py:class:`HybridBlock` will create a symbolic graph\n    representing the forward computation and cache it. On subsequent forwards,\n    the cached graph will be used instead of :py:meth:`forward`.\n\n    Please see references for detailed tutorial.\n\n    References\n    ----------\n        `Hybridize - A Hybrid of Imperative and Symbolic Programming\n        <https://mxnet.apache.org/versions/master/api/python/docs/tutorials/packages/gluon/blocks/hybridize.html>`_\n    \"\"\"\n    class OptConstraint:\n        class Flag(enum.Flag):\n            DisableAMP = enum.auto()\n\n        def __init__(self, flag) -> None:\n            self.flag = flag\n            self.enter_state = None\n\n        def __enter__(self):\n            self.enter_state = HybridBlock.OptConstraint.Flag(get_optimization_constraints())\n            target_state = self.enter_state | self.flag\n            set_optimization_constraints(target_state)\n\n        def __exit__(self, ptype, value, trace):\n            set_optimization_constraints(self.enter_state)\n\n        @staticmethod\n        def disable_all():\n            opt_flag = HybridBlock.OptConstraint.Flag()\n            for flag in HybridBlock.OptConstraint.Flag:\n                opt_flag |= flag\n\n        @staticmethod\n        def disable_amp():\n            return HybridBlock.OptConstraint(HybridBlock.OptConstraint.Flag.DisableAMP)\n\n    def __init__(self):\n        super(HybridBlock, self).__init__()\n        assert hasattr(self, \"hybrid_forward\") is False, (\n            \"'forward' instead of 'hybrid_forward' interface needs to be used starting from Gluon2.0.\"\n            \"Please follow MXNet2.0 Migration Guide to use new APIs.\")\n        self._cached_graph = ()\n        self._cached_op = None\n        self._out_format = None\n        self._in_format = None\n        self._called_infer_shape_already = False\n        self._active = False\n        self._flags = []\n        self._callback = None\n        self._monitor_all = False\n        self._backend = None\n        self._backend_opts = {}\n        self._partition_if_dynamic = True\n        self._first_forward = True\n\n    def __setattr__(self, name, value):\n        \"\"\"Registers parameters.\"\"\"\n        super(HybridBlock, self).__setattr__(name, value)\n        if isinstance(value, HybridBlock):\n            if self._active:\n                warnings.warn(\"Currently the model has been hybridized. Automatically deactivate the hybridization \\\n                               when changing the children blocks.\")\n                self._active = False\n            self._clear_cached_op()\n\n    @staticmethod\n    def generate_arg_names(arg_num):\n        return ['data'] if arg_num == 1 else ['data{}'.format(i) for i in range(arg_num)]\n\n    def _get_graph(self, *args):\n        if not self._cached_graph:\n            flatten_args, self._in_format = _flatten(args, \"input\")\n            flatten_args = [ele.detach() if ele is not None else None for ele in flatten_args]\n            real_args = [ele for ele in flatten_args if ele is not None]\n            if len(real_args) == 0:\n                raise ValueError('All args are None and we do not support such a case.'\n                                 ' Received args={}'.format(args))\n            arg_names = HybridBlock.generate_arg_names(len(real_args))\n            symbol_inputs = [\n                symbol.var(name).as_np_ndarray()\n                if isinstance(arg, _mx_np.ndarray) else symbol.var(name)\n                for arg, name in zip(real_args, arg_names)\n            ]\n            dc.set_variable(real_args, symbol_inputs)\n            args = _regroup(flatten_args, self._in_format)\n            with autograd.pause(), dc.context():\n                out = super().__call__(*args)\n            flatten_out, self._out_format = _flatten(out, \"output\")\n            symbol_outputs = dc.get_symbol(flatten_out, sym_cls=type(symbol_inputs[0]))\n            dc.clear(flatten_out)\n            self._cached_graph = symbol_inputs, symbol_outputs\n        return self._cached_graph\n\n    def _build_cache(self, *args, update_graph=True):\n        data, out = self._get_graph(*args)\n        data_names = {data.name: i for i, data in enumerate(data)}\n        params = {p.var().name: p for p in self.collect_params().values()}\n        param_serialization_names = {p.var().name: n for n, p in self.collect_params().items()}\n        param_names = set(params.keys())\n        input_names = out.list_inputs()\n        expected_names = set(input_names)\n        for name in expected_names:\n            assert name in param_names or name in data_names, \\\n                f\"Unknown input to HybridBlock: {name}\"\n\n        used_data_names = [i for i in data_names if i in expected_names]\n        if len(used_data_names) != len(data_names):\n            unused = ', '.join([f'{i}-th' for name, i in data_names.items()\n                                if name not in expected_names])\n            warnings.warn(f\"The {unused} input to HybridBlock is not used by \"\n                          \"any computation. Is this intended?\", stacklevel=4)\n\n        used_param_names = [i for i in param_names if i in expected_names]\n        if len(used_param_names) != len(param_names):\n            unused = ', '.join(list(param_names - set(used_param_names)))\n            warnings.warn(f\"Parameter {unused} is not used by any computation. \"\n                          \"Is this intended?\", stacklevel=4)\n\n        args, _ = _flatten(args, \"input\")\n        try:\n            for name in input_names:\n                if name in params:\n                    params[name].data()\n        except DeferredInitializationError:\n            self._deferred_infer_shape(*args)\n            for name in input_names:\n                if name in params:\n                    params[name]._finish_deferred_init()\n\n        arg_dict, aux_dict = dict(), dict()\n        if self._backend:\n            # set device for inputs\n            _, _, device_set, _ = _gather_type_device_info(list(args))\n            device = device_set.pop() if len(device_set) > 0 else None\n            # get list of params in the order of out.list_arguments\n            input_shapes = dict()\n            for name in out.list_arguments():\n                if name in data_names.keys() and data_names[name] < len(args):\n                    if isinstance(args[data_names[name]], NDArray):\n                        arg_dict[name] = args[data_names[name]]\n                    elif (isinstance(args[data_names[name]], symbol.Symbol) and\n                          '__shape__' in args[data_names[name]].list_attr()):\n                        shape_str = args[data_names[name]].list_attr()['__shape__']\n                        input_shapes[name] = tuple(map(int, shape_str.strip('()').split(',')))\n                elif name in params:\n                    arg_dict[name] = params[name].data()\n\n            for name in out.list_auxiliary_states():\n                if name in data_names.keys() and data_names[name] < len(args):\n                    if isinstance(args[data_names[name]], NDArray):\n                        aux_dict[name] = args[data_names[name]]\n                    elif (isinstance(args[data_names[name]], symbol.Symbol) and\n                          '__shape__' in args[data_names[name]].list_attr()):\n                        shape_str = args[data_names[name]].list_attr()['__shape__']\n                        input_shapes[name] = tuple(map(int, shape_str.strip('()').split(',')))\n                elif name in params:\n                    aux_dict[name] = params[name].data()\n\n            # Partition the graph\n            out = out.optimize_for(self._backend, arg_dict, aux_dict, device, input_shapes, **self._backend_opts)\n\n            #update cached graph with partitioned graph\n            if update_graph:\n                self._cached_graph = data, out\n\n        input_names = out.list_inputs()\n        data_indices = []\n        param_indices = []\n\n        # In the default case, _cached_ops_args contains all the parameters from params (the sets are identical)\n        # In the case of Partition API optimized graph _cached_ops_args might contain some parameters from params,\n        # might contain some new parameters created during optimization and added to `arg_dict/aux_dict`,\n        # and might not contain some parameters that were deleted during optimization.\n        self._cached_op_args = []\n        for i, name in enumerate(input_names):\n            triple = None\n            if name in data_names:\n                data_indices.append(i)\n                triple = (True, name, data_names[name])\n            else:\n                param_indices.append(i)\n                if name in params:\n                    param = params[name]\n                    serialization_name = param_serialization_names[name]  # HybridBlock.export\n                else:\n                    # The param is missing from the original params dictionary, which means the param must have\n                    # been added by the Partition API backend\n                    if name in arg_dict or name:\n                        param_data = arg_dict[name]\n                    elif name in aux_dict:\n                        param_data = aux_dict[name]\n                    else:\n                        raise RuntimeError('A parameter was added to the graph during optimization but it was not '\n                                           'added to the parameter dicts.\\n'\n                                           'Please check the backend.')\n\n                    param = Parameter(name, dtype=param_data.dtype)\n                    param._var_name = name\n                    serialization_name = name  # HybridBlock.export\n                    param._load_init(param_data, param_data.device)\n                triple = (False, serialization_name, param)\n\n            self._cached_op_args.append(triple)\n\n        for i in range(len(self._flags) - 1, -1, -1):\n            kv = self._flags[i]\n            if kv[0] in ['data_indices', 'param_indices']:\n                self._flags.remove(kv)\n        self._flags = [('data_indices', data_indices), ('param_indices', param_indices)] + self._flags\n        self._cached_op = ndarray.CachedOp(out, self._flags)\n\n    def _deferred_infer_shape(self, *args):\n        try:\n            self.infer_shape(*args)\n        except Exception as e:\n            error_msg = \"Deferred initialization failed because shape\"\\\n                        \" cannot be inferred. {}\".format(e)\n            raise ValueError(error_msg)\n\n    def _call_cached_op(self, *args):\n        if self._cached_op is None:\n            self._build_cache(*args)\n\n        if self._first_forward and self._partition_if_dynamic:\n            self._first_forward = False\n            # partition static shape ops if the graph contains any dynamic shape op\n            _, out = self._cached_graph\n            is_dynamic = out.has_dynamic_shape_op()\n            if is_dynamic:\n                self._backend = 'static_shape'\n                self._backend_opts = {k : v for k, v in self._flags}\n                self._build_cache(*args, update_graph=False)\n\n        assert self._cached_op, \"Gluon failed to build the cache. \" \\\n                                \"This should never happen. \" \\\n                                \"Please submit an issue on Github\" \\\n                                \" https://github.com/apache/mxnet.\"\n        if self._callback:\n            self._cached_op._register_op_hook(self._callback, self._monitor_all)\n            if len(self._flags) >= 2 and (self._flags[1] or self._flags[0]):\n                warnings.warn(\"register_op_hook is experimental when static_alloc=True / static_shape=True \"\n                              \" and may not work correctly\")\n\n        args, fmt = _flatten(args, \"input\")\n        if fmt != self._in_format:\n            # Do not raise in the case that the fmt or stored_fmt ends with None and\n            # We are relying on the default values.\n            if len(self._in_format) > len(fmt):\n                valid = all([self._in_format[i] == -1\n                             for i in range(len(fmt), len(self._in_format))])\n                valid = valid and (fmt == self._in_format[:len(fmt)])\n            elif len(self._in_format) < len(fmt):\n                valid = all([fmt[i] == -1\n                             for i in range(len(self._in_format), len(fmt))])\n                valid = valid and (fmt[:len(self._in_format)] == self._in_format)\n            else:\n                valid = False\n            if not valid:\n                raise ValueError(\"The argument structure of HybridBlock does not match\"\n                                 \" the cached version. Stored format = {}, input format = {}\"\n                                 .format(fmt, self._in_format))\n\n        args_without_none = [ele for ele in args if ele is not None]\n        cargs = [args_without_none[i] if is_arg else i.data()\n                 for is_arg, name, i in self._cached_op_args]\n        out = self._cached_op(*cargs)\n        if isinstance(out, NDArray):\n            out = [out]\n        return _regroup(out, self._out_format)\n\n    def optimize_for(self, x, *args, backend=None, clear=False,\n                     partition_if_dynamic=True,\n                     static_alloc=False,\n                     static_shape=False,\n                     inline_limit=2,\n                     forward_bulk_size=None,\n                     backward_bulk_size=None,\n                     **kwargs):\n        \"\"\"Partitions the current HybridBlock and optimizes it for a given backend\n        without executing a forward pass. Modifies the HybridBlock in-place.\n\n        Immediately partitions a HybridBlock using the specified backend. Combines\n        the work done in the hybridize API with part of the work done in the forward\n        pass without calling the CachedOp. Can be used in place of hybridize,\n        afterwards `export` can be called or inference can be run. See README.md in\n        example/extensions/lib_subgraph/README.md for more details.\n\n        Examples\n        --------\n        # partition and then export to file\n        block.optimize_for(x, backend='myPart')\n        block.export('partitioned')\n\n        # partition and then run inference\n        block.optimize_for(x, backend='myPart')\n        block(x)\n\n        Parameters\n        ----------\n        x : NDArray\n            first input to model\n        *args : NDArray\n            other inputs to model\n        backend : str\n            The name of backend, as registered in `SubgraphBackendRegistry`, default None\n        backend_opts : dict of user-specified options to pass to the backend for partitioning, optional\n            Passed on to `PrePartition` and `PostPartition` functions of `SubgraphProperty`\n        clear : bool, default False\n            clears any previous optimizations\n        partition_if_dynamic : bool, default False\n            whether to partition the graph when dynamic shape op exists\n        static_alloc : bool, default False\n            Statically allocate memory to improve speed. Memory usage may increase.\n        static_shape : bool, default False\n            Optimize for invariant input shapes between iterations. Must also\n            set static_alloc to True. Change of input shapes is still allowed\n            but slower.\n        inline_limit : optional int, default 2\n            Maximum number of operators that can be inlined.\n        forward_bulk_size : optional int, default None\n            Segment size of bulk execution during forward pass.\n        backward_bulk_size : optional int, default None\n            Segment size of bulk execution during backward pass.\n        **kwargs: The backend options, optional\n            Passed on to `PrePartition` and `PostPartition` functions of `SubgraphProperty`\n        \"\"\"\n        self._backend = backend\n        if len(kwargs) > 0:\n            self._backend_opts = kwargs\n\n        if clear or not self._active:\n            self.hybridize(True, partition_if_dynamic, static_alloc, static_shape,\n                           inline_limit, forward_bulk_size, backward_bulk_size)\n\n        # do part of forward API call\n        has_symbol, has_ndarray, device_set, _ = _gather_type_device_info([x] + list(args))\n        if not has_symbol and not has_ndarray:\n            raise ValueError('In HybridBlock, there must be one NDArray or one Symbol in the input.'\n                             ' Please check the type of the args.\\n')\n        if len(device_set) > 1:\n            raise ValueError('Found multiple devices in the input, '\n                             'After hybridized, the HybridBlock only supports one input '\n                             'device. You can print the ele.device in the '\n                             'input arguments to inspect their devices. '\n                             'Find all devices = {}'.format(device_set))\n\n        self._build_cache(x, *args)\n        assert self._cached_op, \"Gluon failed to build the cache. \" \\\n                                \"This should never happen. \" \\\n                                \"Please submit an issue on Github\" \\\n                                \" https://github.com/apache/mxnet.\"\n        # do not actually call the cached_op\n\n        self._first_forward = True\n        # clear the backend\n        self._backend = None\n        self._backend_opts = {}\n\n    def _clear_cached_op(self):\n        self._cached_graph = ()\n        self._cached_op = None\n        self._first_forward = True\n\n    def register_child(self, block, name=None):\n        if not isinstance(block, HybridBlock):\n            raise ValueError(\n                \"Children of HybridBlock must also be HybridBlock, \" \\\n                f\"but {str(block)} has type {str(type(block))}. If you are using Sequential, \" \\\n                \"please try HybridSequential instead.\")\n        super(HybridBlock, self).register_child(block, name)\n        if self._active:\n            warnings.warn(\"Currently the model has been hybridized. Automatically deactivate the hybridization \\\n                           when adding new children block.\")\n            self._active = False\n        self._clear_cached_op()\n\n    def hybridize(self, active=True,\n                  partition_if_dynamic=True,\n                  static_alloc=False,\n                  static_shape=False,\n                  inline_limit=2,\n                  forward_bulk_size=None,\n                  backward_bulk_size=None):\n        \"\"\"Activates or deactivates :py:class:`HybridBlock` s recursively. Has no effect on\n        non-hybrid children.\n\n        Parameters\n        ----------\n        active : bool, default True\n            Whether to turn hybrid on or off.\n        partition_if_dynamic : bool, default False\n            whether to partition the graph when dynamic shape op exists\n        static_alloc : bool, default False\n            Statically allocate memory to improve speed. Memory usage may increase.\n        static_shape : bool, default False\n            Optimize for invariant input shapes between iterations. Must also\n            set static_alloc to True. Change of input shapes is still allowed\n            but slower.\n        inline_limit : optional int, default 2\n            Maximum number of operators that can be inlined.\n        forward_bulk_size : optional int, default None\n            Segment size of bulk execution during forward pass.\n        backward_bulk_size : optional int, default None\n            Segment size of bulk execution during backward pass.\n        \"\"\"\n\n        self._active = active\n        self._partition_if_dynamic = partition_if_dynamic\n        self._flags = [(\"static_alloc\", static_alloc), (\"static_shape\", static_shape),\n                       (\"inline_limit\", inline_limit)]\n        if forward_bulk_size is not None:\n            self._flags.append((\"forward_bulk_size\", forward_bulk_size))\n        if backward_bulk_size is not None:\n            self._flags.append((\"backward_bulk_size\", backward_bulk_size))\n        self._clear_cached_op()\n        if active and self._forward_hooks or self._forward_pre_hooks:\n            warnings.warn('\"{block}\" is being hybridized while still having forward hook/pre-hook. '\n                          'If \"{block}\" is a child of HybridBlock, the hooks will not take effect.'\n                          .format(block=self))\n        super(HybridBlock, self).hybridize(active,\n                                           static_alloc=static_alloc,\n                                           static_shape=static_shape,\n                                           inline_limit=inline_limit,\n                                           forward_bulk_size=forward_bulk_size,\n                                           backward_bulk_size=backward_bulk_size)\n\n    def cast(self, dtype):\n        if self._active:\n            warnings.warn(\"Currently the model has been hybridized. Automatically deactivate the hybridization \\\n                           when cast the block to use another data type.\")\n            self._active = False\n        self._clear_cached_op()\n        super(HybridBlock, self).cast(dtype)\n\n    def _infer_attrs(self, infer_fn, attr, *args):\n        \"\"\"Generic infer attributes.\"\"\"\n        inputs, out = self._get_graph(*args)\n        args, _ = _flatten(args, \"input\")\n        args_without_none = [ele for ele in args if ele is not None]\n        with warnings.catch_warnings(record=True) as w:\n            arg_attrs, _, aux_attrs = getattr(out, infer_fn)(\n                **{i.name: getattr(j, attr) for i, j in zip(inputs, args_without_none)})\n            if arg_attrs is None:\n                raise ValueError(w[0].message)\n        sdict = {i: j for i, j in zip(out.list_arguments(), arg_attrs)}\n        sdict.update({name : attr for name, attr in \\\n             zip(out.list_auxiliary_states(), aux_attrs)})\n        for i in self.collect_params().values():\n            setattr(i, attr, sdict[i.var().name])\n\n    def infer_shape(self, *args):\n        \"\"\"Infers shape of Parameters from inputs.\"\"\"\n        # pylint: disable=unused-argument\n        # In Gluon 2, users must implement infer_shape, if any deferred\n        # initialized parameters are associated with the HybridBlock\n        params = [p for p in self._reg_params.values() if not shape_is_known(p.shape)]\n        if params:\n            params_str = \", \".join(\"{} ({})\".format(p.name, p.shape) for p in params)\n            raise RuntimeError(\n                \"{name} has parameters with unknown shape. You need to either specify the shape \"\n                \"in __init__ or implement {name}.infer_shape to set the parameter shapes \"\n                \"based on the first input. Parameters with unknown shapes are {params}\".format(\n                    name=type(self).__name__, params=params_str))\n\n    def infer_type(self, *args):\n        \"\"\"Infers data type of Parameters from inputs.\"\"\"\n        self._infer_attrs('infer_type', 'dtype', *args)\n\n    def export(self, path, epoch=0, remove_amp_cast=True):\n        \"\"\"Export HybridBlock to json format that can be loaded by\n        `gluon.SymbolBlock.imports` or the C++ interface.\n\n        .. note:: When there are only one input, it will have name `data`. When there\n                  Are more than one inputs, they will be named as `data0`, `data1`, etc.\n\n        Parameters\n        ----------\n        path : str or None\n            Path to save model. Two files `path-symbol.json` and `path-xxxx.params`\n            will be created, where xxxx is the 4 digits epoch number.\n            If None, do not export to file but return Python Symbol object and\n            corresponding dictionary of parameters.\n        epoch : int\n            Epoch number of saved model.\n        remove_amp_cast : bool, optional\n            Whether to remove the amp_cast and amp_multicast operators, before saving the model.\n\n        Returns\n        -------\n        symbol_filename : str\n            Filename to which model symbols were saved, including `path` prefix.\n        params_filename : str\n            Filename to which model parameters were saved, including `path` prefix.\n        \"\"\"\n        if not self._cached_graph:\n            raise RuntimeError(\n                \"Please first call block.hybridize() and then run forward with \"\n                \"this block at least once before calling export.\")\n        sym = copy.copy(self._cached_graph[1])\n\n        # Deduplicate params (shared parameters use the same input symbol)\n        reverse_params = {v: k for k, v in self.collect_params().items()}\n        params = {v: k for k, v in reverse_params.items()}\n\n        # In export we have global information on the structure of the graph\n        # can rename the symbol inputs to human-readable, deterministic names.\n        # That's not true in general, which is why internally random unique identifiers are used.\n        rename_map = {param.var().name: name for name, param in params.items()}\n        for var in sym.get_inputs():\n            if var.name in rename_map:\n                var._set_attr(name=rename_map[var.name])\n        \n        path_string = path if path is not None else \"\"\n        sym_filename = f'{path_string}-symbol.json'\n        if path is not None:\n            sym.save(sym_filename, remove_amp_cast=remove_amp_cast)\n\n        arg_names = set(sym.list_arguments())\n        aux_names = set(sym.list_auxiliary_states())\n        arg_dict = {}\n        for is_arg, name, param in self._cached_op_args:\n            if not is_arg:\n                if name in arg_names:\n                    arg_dict['arg:{}'.format(name)] = param._reduce()\n                else:\n                    if name not in aux_names:\n                        warnings.warn('Parameter \"{name}\" is not found in the graph. '\n                                      .format(name=name), stacklevel=3)\n                    else:\n                        arg_dict[f'aux:{name}'] = param._reduce()\n        params_filename = f'{path_string}-{epoch:04d}.params'\n\n        if path is not None:\n            if is_np_array():\n                _mx_npx.savez(params_filename, **arg_dict)\n            else:\n                ndarray.save(params_filename, arg_dict)\n            return (sym_filename, params_filename if arg_dict else None)\n\n        if remove_amp_cast:\n            handle = SymbolHandle()\n            check_call(_LIB.MXSymbolRemoveAmpCast(sym.handle, ctypes.byref(handle)))\n            sym = type(sym)(handle)\n        return sym, arg_dict\n\n    def register_op_hook(self, callback, monitor_all=False):\n        \"\"\"Install op hook for block recursively.\n\n        Parameters\n        ----------\n        callback : function\n            Function called to inspect the values of the intermediate outputs\n            of blocks after hybridization. It takes 3 parameters:\n            name of the tensor being inspected (str)\n            name of the operator producing or consuming that tensor (str)\n            tensor being inspected (NDArray).\n        monitor_all : bool, default False\n            If True, monitor both input and output, otherwise monitor output only.\n        \"\"\"\n        def c_callback(name, op_name, array):\n            \"\"\"wrapper for user callback\"\"\"\n            array = ctypes.cast(array, NDArrayHandle)\n            array = NDArray(array, writable=False)\n            name = py_str(name)\n            op_name = py_str(op_name)\n            callback(name, op_name, array)\n\n        self._callback = c_callback\n        self._monitor_all = monitor_all\n        for cld in self._children.values():\n            cld()._callback = c_callback\n            cld()._monitor_all = monitor_all\n\n    def __call__(self, x, *args):\n        _check_block_input_np_ndarrays([x, *args])\n        assert self.forward is not HybridBlock.forward, (\n            'Must define {name}.forward. '\n            'Defining {name}.hybrid_forward is deprecated.'.format(name=type(self).__name__))\n\n        _, has_ndarray, device_set, first_device = _gather_type_device_info([x] + list(args))\n        if not has_ndarray:\n            raise ValueError('In HybridBlock, there must be one NDArray in the input.'\n                             ' Please check the type of the args.\\n')\n        if self._active and not dc.is_deferred_compute():\n            # Do not call CachedOp if not hybridized or inside deferred compute mode.\n            if len(device_set) > 1:\n                raise ValueError('Find multiple devices in the input, '\n                                 'After hybridized, the HybridBlock only supports one input '\n                                 'device. You can print the ele.device in the '\n                                 'input arguments to inspect their devices. '\n                                 'Find all devices = {}'.format(device_set))\n\n        if not self._called_infer_shape_already:\n            self.infer_shape(x, *args)\n            for p in self._reg_params.values():\n                p._finish_deferred_init()\n            self._called_infer_shape_already = True\n\n        if not self._active:\n            # Normal imperative computation of forward()\n            return super().__call__(x, *args)\n\n        if dc.is_deferred_compute():\n            # Deferred compute is already enabled. This typically means that the current\n            # HybridBlock is a child block of a HybridBlock that has been hybridized.\n            return super().__call__(x, *args)\n\n        with first_device:\n            return self._call_cached_op(x, *args)\n\n    def forward(self, x, *args):\n        \"\"\"Overrides the forward computation. Arguments must be\n        :py:class:`mxnet.numpy.ndarray`.\"\"\"\n\n        raise NotImplementedError\n\n    def reset_device(self, device):\n        \"\"\"Re-assign all Parameters to other devices. If the Block is hybridized, it will reset the _cached_op_args.\n\n        Parameters\n        ----------\n        device : Device or list of Device, default :py:meth:`device.current_device()`.\n            Assign Parameter to given device. If device is a list of Device, a\n            copy will be made for each device.\n        \"\"\"\n        params = self.collect_params()\n        if self._cached_op:\n            for p in self._cached_op_args:\n                # resetting parameters creating by the partitioning backend\n                if p.name not in params:\n                    p.reset_device(device)\n        for p in params.values():\n            p.reset_device(device)\n\n    def reset_ctx(self, ctx):\n        \"\"\"This function has been deprecated. Please refer to ``HybridBlock.reset_device``.\"\"\"\n        warnings.warn('HybridBlock.reset_ctx has been renamed to'\n                      ' HybridBlock.reset_device', DeprecationWarning)\n        self.reset_device(ctx)\n\n\nclass SymbolBlock(HybridBlock):\n    \"\"\"Construct block from symbol. This is useful for using pre-trained models\n    as feature extractors. For example, you may want to extract the output\n    from fc2 layer in AlexNet.\n\n    Parameters\n    ----------\n    outputs : Symbol or list of Symbol\n        The desired output for SymbolBlock.\n    inputs : Symbol or list of Symbol\n        The Variables in output's argument that should be used as inputs.\n    params : dict\n        Parameter dictionary for arguments and auxililary states of outputs\n        that are not inputs.\n\n    Examples\n    --------\n    >>> # To extract the feature from fc1 and fc2 layers of AlexNet:\n    >>> alexnet = gluon.model_zoo.vision.alexnet(pretrained=True, device=mx.cpu())\n    >>> inputs = mx.sym.var('data')\n    >>> out = alexnet(inputs)\n    >>> internals = out.get_internals()\n    >>> print(internals.list_outputs())\n    ['data', ..., 'features_9_act_fwd_output', ..., 'features_11_act_fwd_output', ...]\n    >>> outputs = [internals['features_9_act_fwd_output'],\n                   internals['features_11_act_fwd_output']]\n    >>> # Create SymbolBlock that shares parameters with alexnet\n    >>> feat_model = gluon.SymbolBlock(outputs, inputs, params=alexnet.collect_params())\n    >>> x = mx.nd.random.normal(shape=(16, 3, 224, 224))\n    >>> print(feat_model(x))\n    \"\"\"\n    @staticmethod\n    @wrap_ctx_to_device_func\n    def imports(symbol_file, input_names, param_file=None, device=None, allow_missing=False,\n                ignore_extra=False):\n        \"\"\"Import model previously saved by `gluon.HybridBlock.export`\n        as a `gluon.SymbolBlock` for use in Gluon.\n\n        Parameters\n        ----------\n        symbol_file : str\n            Path to symbol file.\n        input_names : list of str\n            List of input variable names\n        param_file : str, optional\n            Path to parameter file.\n        device : Device, default None\n            The device to initialize `gluon.SymbolBlock` on.\n        allow_missing : bool, default False\n            Whether to silently skip loading parameters not represents in the file.\n        ignore_extra : bool, default False\n            Whether to silently ignore parameters from the file that are not\n            present in this Block.\n\n        Returns\n        -------\n        gluon.SymbolBlock\n            `gluon.SymbolBlock` loaded from symbol and parameter files.\n\n        Examples\n        --------\n        >>> net1 = gluon.model_zoo.vision.resnet18_v1(pretrained=True)\n        >>> net1.hybridize()\n        >>> x = mx.nd.random.normal(shape=(1, 3, 32, 32))\n        >>> out1 = net1(x)\n        >>> net1.export('net1', epoch=1)\n        >>>\n        >>> net2 = gluon.SymbolBlock.imports(\n        ...     'net1-symbol.json', ['data'], 'net1-0001.params')\n        >>> out2 = net2(x)\n        \"\"\"\n        if is_np_array():\n            sym = np_symbol.load(symbol_file)\n        else:\n            sym = symbol.load(symbol_file)\n        if isinstance(input_names, str):\n            input_names = [input_names]\n        if param_file is None:\n            # Get a valid type inference by using fp32\n            inputs = [symbol.var(i, dtype=mx_real_t) for i in input_names]\n        else:\n            # Do not specify type, rely on saved params type instead\n            inputs = [symbol.var(i).as_np_ndarray() if is_np_array() else symbol.var(i) for i in input_names]\n        ret = SymbolBlock(sym, inputs)\n        if param_file is not None:\n            ret.load_parameters(param_file, device, allow_missing, ignore_extra, True, 'saved')\n        return ret\n\n    def __repr__(self):\n        s = '{name}(\\n{modstr}\\n)'\n        modstr = '\\n'.join(['{block} : {numinputs} -> {numoutputs}'.format(block=self._cached_graph[1],\n                                                                           numinputs=len(self._cached_graph[0]),\n                                                                           numoutputs=len(self._cached_graph[1].\n                                                                                          list_outputs()))])\n        return s.format(name=self.__class__.__name__,\n                        modstr=modstr)\n\n    def __init__(self, outputs, inputs, params=None):\n        super(SymbolBlock, self).__init__()\n\n        if isinstance(inputs, symbol.Symbol) and len(inputs.list_outputs()) == 1:\n            inputs = [inputs]\n        if isinstance(outputs, (list, tuple)) and len(outputs) == 1:\n            outputs = outputs[0]\n\n        syms, self._in_format = _flatten(inputs, \"input\")\n        out, self._out_format = _flatten(outputs, \"output\")\n        input_names = set()\n        for i in syms:\n            assert len(i.get_internals().list_outputs()) == 1, \\\n                f\"Input symbols must be variable, but {str(i)} is an output of operators\"\n            input_names.add(i.name)\n\n        # check if any symbol is row_sparse\n        row_sparse_storage = ndarray.ndarray._STORAGE_TYPE_STR_TO_ID['row_sparse']\n\n        for i in out:\n            for j in i.get_internals():\n                assert(j.attr(\"__storage_type__\") != str(row_sparse_storage)), \\\n                    f\"SymbolBlock doesn't support Parameter '{j.name}' because its storage \" \\\n                    \"type is 'row_sparse'.\"\n        if len(out) > 1:\n            out = symbol.Group(out, _check_same_symbol_type(out))\n        else:\n            out = out[0]\n\n        # Infer type of parameters. Without this, every parameter will be created with\n        # default type i.e., fp32\n        arg_params = out.list_arguments()\n        aux_params = out.list_auxiliary_states()\n\n        arg_types, aux_types = _infer_param_types(syms, out, arg_params, aux_params)\n\n        if params is None:\n            params = {}\n        unused_params = set(params.keys()) - set(arg_params) - set(aux_params)\n        if len(unused_params) > 0:\n            raise ValueError('{} params are unused by the model.'.format(unused_params))\n        self._reg_params = params\n\n        for i, arg in enumerate(arg_params):\n            if arg in self._reg_params:\n                self._reg_params[arg]._check_and_setattr(allow_deferred_init=True, dtype=arg_types[i])\n                if self._reg_params[arg]._var is None:\n                    self._reg_params[arg]._var_name = arg\n            elif arg not in input_names:\n                self._reg_params[arg] = Parameter(name=arg, allow_deferred_init=True, dtype=arg_types[i])\n                self._reg_params[arg]._var_name = arg\n        for i, aux in enumerate(aux_params):\n            if aux in self._reg_params:\n                self._reg_params[aux]._check_and_setattr(grad_req='null', allow_deferred_init=True,\n                                                         dtype=aux_types[i])\n                if self._reg_params[aux]._var is None:\n                    self._reg_params[aux]._var_name = aux\n            elif aux not in input_names:\n                self._reg_params[aux] = Parameter(name=aux, grad_req='null',\n                                                  allow_deferred_init=True, dtype=aux_types[i])\n                self._reg_params[aux]._var_name = aux\n\n        self._cached_graph = syms, out\n\n    def infer_shape(self, *args):\n        \"\"\"Infers shape of Parameters from inputs.\"\"\"\n        self._infer_attrs('infer_shape', 'shape', *args)\n\n    def __call__(self, x, *args):\n        \"\"\"Calls forward. Only accepts positional arguments.\"\"\"\n        for hook in self._forward_pre_hooks.values():\n            hook(self, [x, *args])\n\n        out = self.forward(x, *args)\n\n        for hook in self._forward_hooks.values():\n            hook(self, [x, *args], out)\n\n        return out\n\n    def forward(self, x, *args):\n        if dc.is_deferred_compute():\n            raise RuntimeError('Calling a SymbolBlock from within HybridBlock '\n                               'is not yet supported in Gluon 2.')\n\n        if isinstance(x, NDArray):\n            with x.device:\n                return self._call_cached_op(x, *args)\n\n        assert isinstance(x, Symbol), \\\n            \"HybridBlock requires the first argument to forward be either \" \\\n            f\"Symbol or NDArray, but got {type(x)}\"\n        args, in_fmt = _flatten([x] + list(args), \"input\")\n        assert in_fmt == self._in_format, \"Invalid input format\"\n        ret = copy.copy(self._cached_graph[1])\n        ret._compose(**{k.name: v for k, v in zip(self._cached_graph[0], args)})\n        return _regroup(list(ret), self._out_format)\n\n    def _clear_cached_op(self):\n        tmp = self._cached_graph\n        super(SymbolBlock, self)._clear_cached_op()\n        self._cached_graph = tmp\n\n    def cast(self, dtype):\n        self._clear_cached_op()\n        super(SymbolBlock, self).cast(dtype)\n        if get_dtype_name(dtype) == 'float16':\n            # correct BatchNorm types back to float32 due to its special requirement\n            out = self._cached_graph[1]\n            params_list = out.get_internals().list_inputs()\n            for node in params_list:\n                if node.endswith('running_var'):\n                    prefix = node[:-11]\n                    sibs = [prefix + t for t in ('running_mean', 'gamma', 'beta')]\n                    is_bn = all(p in params_list for p in sibs)\n                    if is_bn:\n                        self.params.get(node).cast('float32')\n                        for sib in sibs:\n                            self.params.get(sib).cast('float32')\n                if node.endswith('moving_var'):\n                    # another convention used\n                    prefix = node[:-10]\n                    sibs = [prefix + t for t in ('moving_mean', 'gamma', 'beta')]\n                    is_bn = all(p in params_list for p in sibs)\n                    if is_bn:\n                        self.params.get(node).cast('float32')\n                        for sib in sibs:\n                            self.params.get(sib).cast('float32')\n\ndef _infer_param_types(in_params, out_params, arg_params, aux_params, default_dtype=mx_real_t):\n    \"\"\"Utility function that helps in inferring DType of args and auxs params\n    from given input param.\n\n    Parameters\n    ----------\n    in_params: List of Symbol\n        List of input symbol variables.\n    out_params: Symbol\n        Output symbol variable.\n    arg_params: List of Str\n        List of names of argument parametrs.\n    aux_params: List of Str\n        List of names of auxiliary parameters.\n    default_dtype: numpy.dtype or str, default 'float32'\n        Default data type for arg_params and aux_params, if unable to infer the type.\n\n    Returns\n    -------\n    arg_types: List of numpy.dtype\n        List of arg_params type. Order is same as arg_params.\n        Defaults to 'float32', if unable to infer type.\n    aux_types: List of numpy.dtype\n        List of aux_params type. Order is same as aux_params.\n        Defaults to 'float32', if unable to infer type.\n    \"\"\"\n    arg_types = None\n    aux_types = None\n\n    # Get Input symbol details. This will be used to infer types of\n    # other parameters.\n    input_sym_names = [in_param.name for in_param in in_params]\n\n    # Try to infer input types. If not successful, we will set default dtype.\n    # If successful, we will try to infer other params in the graph.\n    input_sym_arg_types = []\n    can_infer_input_type = True\n    for in_param in in_params:\n        input_sym_arg_type = in_param.infer_type()[0]\n        if not input_sym_arg_type or len(input_sym_arg_type) < 1:\n            can_infer_input_type = False\n            break\n        else:\n            input_sym_arg_types.append(in_param.infer_type()[0][0])\n\n    # Try to infer types of other parameters.\n    if can_infer_input_type:\n        params = {k:v for k, v in zip(input_sym_names, input_sym_arg_types)}\n        try:\n            arg_types, _, aux_types = out_params.infer_type(**params)\n        except MXNetError:\n            # Cannot infer type with current input\n            arg_types, aux_types = None, None\n\n    if arg_types is None or len(arg_types) != len(arg_params):\n        arg_types = []\n        for _ in arg_params:\n            arg_types.append(default_dtype)\n\n    if aux_types is None or len(aux_types) != len(aux_params):\n        aux_types = []\n        for _ in aux_params:\n            aux_types.append(default_dtype)\n\n    return (arg_types, aux_types)\n\n\ndef set_optimization_constraints(state):\n    prev_state = ctypes.c_uint()\n    check_call(_LIB.MXSetOptimizationConstraints(ctypes.c_uint(state.value), ctypes.byref(prev_state)))\n    return HybridBlock.OptConstraint.Flag(prev_state.value)\n\n\ndef get_optimization_constraints():\n    curr = ctypes.c_uint()\n    check_call(_LIB.MXGetOptimizationConstraints(ctypes.byref(curr)))\n    return HybridBlock.OptConstraint.Flag(curr.value)\n"
  },
  {
    "path": "python/mxnet/gluon/contrib/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Contrib neural network module.\"\"\"\n\nfrom . import data\n\nfrom . import estimator\n"
  },
  {
    "path": "python/mxnet/gluon/contrib/data/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Contrib datasets.\"\"\"\n\nfrom . import vision\n"
  },
  {
    "path": "python/mxnet/gluon/contrib/data/_constants.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\n\"\"\"Read text files and load embeddings.\"\"\"\n\nEOS_TOKEN = '<eos>'\n"
  },
  {
    "path": "python/mxnet/gluon/contrib/data/vision/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\r\n# or more contributor license agreements.  See the NOTICE file\r\n# distributed with this work for additional information\r\n# regarding copyright ownership.  The ASF licenses this file\r\n# to you under the Apache License, Version 2.0 (the\r\n# \"License\"); you may not use this file except in compliance\r\n# with the License.  You may obtain a copy of the License at\r\n#\r\n#   http://www.apache.org/licenses/LICENSE-2.0\r\n#\r\n# Unless required by applicable law or agreed to in writing,\r\n# software distributed under the License is distributed on an\r\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\r\n# KIND, either express or implied.  See the License for the\r\n# specific language governing permissions and limitations\r\n# under the License.\r\n\r\n# coding: utf-8\r\n# pylint: disable=wildcard-import\r\n\"\"\"Contrib vision utilities.\"\"\"\r\nfrom .transforms import *\r\nfrom .dataloader import *\r\n"
  },
  {
    "path": "python/mxnet/gluon/contrib/data/vision/dataloader.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable= arguments-differ, wildcard-import\n\"Contrib Vision DataLoaders.\"\nimport logging\nimport numpy as np\n\nfrom ..... import ndarray as nd\nfrom .....util import is_np_array\nfrom ..... import numpy as _mx_np   # pylint: disable=reimported\nfrom ....nn import HybridSequential, Sequential, HybridBlock, Block\nfrom ....data.vision import transforms\nfrom ....data import DataLoader\nfrom .transforms import bbox\n\n__all__ = ['create_image_augment', 'ImageDataLoader', 'ImageBboxDataLoader']\n\ndef create_image_augment(data_shape, resize=0, rand_crop=False, rand_resize=False, rand_mirror=False,\n                         mean=None, std=None, brightness=0, contrast=0, saturation=0, hue=0,\n                         pca_noise=0, rand_gray=0, inter_method=2, dtype='float32'):\n    \"\"\"Creates an augmenter block.\n\n    Parameters\n    ----------\n    data_shape : tuple of int\n        Shape for output data\n    resize : int\n        Resize shorter edge if larger than 0 at the begining\n    rand_crop : bool\n        Whether to enable random cropping other than center crop\n    rand_resize : bool\n        Whether to enable random sized cropping, require rand_crop to be enabled\n    rand_gray : float\n        [0, 1], probability to convert to grayscale for all channels, the number\n        of channels will not be reduced to 1\n    rand_mirror : bool\n        Whether to apply horizontal flip to image with probability 0.5\n    mean : np.ndarray or None\n        Mean pixel values for [r, g, b]\n    std : np.ndarray or None\n        Standard deviations for [r, g, b]\n    brightness : float\n        Brightness jittering range (percent)\n    contrast : float\n        Contrast jittering range (percent)\n    saturation : float\n        Saturation jittering range (percent)\n    hue : float\n        Hue jittering range (percent)\n    pca_noise : float\n        Pca noise level (percent)\n    inter_method : int, default=2(Area-based)\n        Interpolation method for all resizing operations\n\n        Possible values:\n        0: Nearest Neighbors Interpolation.\n        1: Bilinear interpolation.\n        2: Bicubic interpolation over 4x4 pixel neighborhood.\n        3: Area-based (resampling using pixel area relation). It may be a\n        preferred method for image decimation, as it gives moire-free\n        results. But when the image is zoomed, it is similar to the Nearest\n        Neighbors method. (used by default).\n        4: Lanczos interpolation over 8x8 pixel neighborhood.\n        10: Random select from interpolation method metioned above.\n        Note:\n        When shrinking an image, it will generally look best with AREA-based\n        interpolation, whereas, when enlarging an image, it will generally look best\n        with Bicubic (slow) or Bilinear (faster but still looks OK).\n\n    Examples\n    --------\n    >>> # An example of creating multiple augmenters\n    >>> augs = mx.gluon.contrib.data.create_image_augment(data_shape=(3, 300, 300), rand_mirror=True,\n    ...    mean=True, brightness=0.125, contrast=0.125, rand_gray=0.05,\n    ...    saturation=0.125, pca_noise=0.05, inter_method=10)\n    \"\"\"\n    if inter_method == 10:\n        inter_method = np.random.randint(0, 5)\n    augmenter = HybridSequential()\n    if resize > 0:\n        augmenter.add(transforms.image.Resize(resize, interpolation=inter_method))\n    crop_size = (data_shape[2], data_shape[1])\n    if rand_resize:\n        assert rand_crop\n        augmenter.add(transforms.image.RandomResizedCrop(crop_size, interpolation=inter_method))\n    elif rand_crop:\n        augmenter.add(transforms.image.RandomCrop(crop_size, interpolation=inter_method))\n    else:\n        augmenter.add(transforms.image.CenterCrop(crop_size, interpolation=inter_method))\n\n    if rand_mirror:\n        augmenter.add(transforms.image.RandomFlipLeftRight(0.5))\n\n    augmenter.add(transforms.Cast())\n\n    if brightness or contrast or saturation or hue:\n        augmenter.add(transforms.image.RandomColorJitter(brightness, contrast, saturation, hue))\n\n    if pca_noise > 0:\n        augmenter.add(transforms.image.RandomLighting(pca_noise))\n\n    if rand_gray > 0:\n        augmenter.add(transforms.image.RandomGray(rand_gray))\n\n    if mean is True:\n        mean = [123.68, 116.28, 103.53]\n    elif mean is not None:\n        assert isinstance(mean, (tuple, list))\n\n    if std is True:\n        std = [58.395, 57.12, 57.375]\n    elif std is not None:\n        assert isinstance(std, (tuple, list))\n\n    augmenter.add(transforms.image.ToTensor())\n\n    if mean is not None or std is not None:\n        augmenter.add(transforms.image.Normalize(mean, std))\n\n    augmenter.add(transforms.Cast(dtype))\n\n    return augmenter\n\nclass ImageDataLoader(object):\n    \"\"\"Image data loader with a large number of augmentation choices.\n    This loader supports reading from both .rec files and raw image files.\n\n    To load input images from .rec files, use `path_imgrec` parameter and to load from raw image\n    files, use `path_imglist` and `path_root` parameters.\n\n    To use data partition (for distributed training) or shuffling, specify `path_imgidx` parameter.\n\n    Parameters\n    ----------\n    batch_size : int\n        Number of examples per batch.\n    data_shape : tuple\n        Data shape in (channels, height, width) format.\n        For now, only RGB image with 3 channels is supported.\n    path_imgrec : str\n        Path to image record file (.rec).\n        Created with tools/im2rec.py or bin/im2rec.\n    path_imglist : str\n        Path to image list (.lst).\n        Created with tools/im2rec.py or with custom script.\n        Format: Tab separated record of index, one or more labels and relative_path_from_root.\n    imglist: list\n        A list of images with the label(s).\n        Each item is a list [imagelabel: float or list of float, imgpath].\n    path_root : str\n        Root folder of image files.\n        Whether to shuffle all images at the start of each iteration or not.\n        Can be slow for HDD.\n    part_index : int\n        Partition index.\n    num_parts : int\n        Total number of partitions.\n    dtype : str\n        Label data type. Default: float32. Other options: int32, int64, float64\n    last_batch : {'keep', 'discard', 'rollover'}\n        How to handle the last batch if batch_size does not evenly divide\n        `len(dataset)`.\n\n        keep - A batch with less samples than previous batches is returned.\n        discard - The last batch is discarded if its incomplete.\n        rollover - The remaining samples are rolled over to the next epoch.\n    kwargs : ...\n        More arguments for creating augmenter. See mx.gluon.contrib.vision.dataloader.create_image_augment.\n    \"\"\"\n    def __init__(self, batch_size, data_shape, path_imgrec=None, path_imglist=None, path_root='.',\n                 part_index=0, num_parts=1, aug_list=None, imglist=None,\n                 dtype='float32', shuffle=False, sampler=None,\n                 last_batch=None, batch_sampler=None, batchify_fn=None,\n                 num_workers=0, pin_memory=False, pin_device_id=0,\n                 prefetch=None, thread_pool=False, timeout=120, try_nopython=None,\n                 **kwargs):\n        assert path_imgrec or path_imglist or (isinstance(imglist, list))\n        assert dtype in ['int32', 'float32', 'int64', 'float64'], dtype + ' label not supported'\n        logging.info('Using %s workers for decoding...', str(num_workers))\n        logging.info('Set `num_workers` variable to a larger number to speed up loading'\n                     ' (it requires shared memory to work and may occupy more memory).')\n        class_name = self.__class__.__name__\n        if path_imgrec:\n            logging.info('%s: loading recordio %s...',\n                         class_name, path_imgrec)\n            from ....data.vision.datasets import ImageRecordDataset\n            dataset = ImageRecordDataset(path_imgrec, flag=1)\n        elif path_imglist:\n            logging.info('%s: loading image list %s...', class_name, path_imglist)\n            from ....data.vision.datasets import ImageListDataset\n            dataset = ImageListDataset(path_root, path_imglist, flag=1)\n        elif isinstance(imglist, list):\n            logging.info('%s: loading image list...', class_name)\n            from ....data.vision.datasets import ImageListDataset\n            dataset = ImageListDataset(path_root, imglist, flag=1)\n        else:\n            raise ValueError('Either path_imgrec, path_imglist, or imglist must be provided')\n\n        if num_parts > 1:\n            dataset = dataset.shard(num_parts, part_index)\n\n        if aug_list is None:\n            # apply default transforms\n            augmenter = create_image_augment(data_shape, **kwargs)\n        elif isinstance(aug_list, list):\n            if all([isinstance(a, HybridBlock) for a in aug_list]):\n                augmenter = HybridSequential()\n            else:\n                augmenter = Sequential()\n            for aug in aug_list:\n                augmenter.add(aug)\n        elif isinstance(aug_list, Block):\n            augmenter = aug_list\n        else:\n            raise ValueError('aug_list must be a list of Blocks or Block')\n        augmenter.hybridize()\n        self._iter = DataLoader(dataset.transform_first(augmenter), batch_size=batch_size,\n                                shuffle=shuffle, sampler=sampler, last_batch=last_batch,\n                                batch_sampler=batch_sampler, batchify_fn=batchify_fn,\n                                num_workers=num_workers, pin_memory=pin_memory,\n                                pin_device_id=pin_device_id, prefetch=prefetch,\n                                thread_pool=thread_pool, timeout=timeout, try_nopython=try_nopython)\n\n    def __iter__(self):\n        return iter(self._iter)\n\n    def __len__(self):\n        return len(self._iter)\n\ndef create_bbox_augment(data_shape, rand_crop=0, rand_pad=0, rand_gray=0,\n                        rand_mirror=False, mean=None, std=None, brightness=0, contrast=0,\n                        saturation=0, pca_noise=0, hue=0, inter_method=2,\n                        max_aspect_ratio=2, area_range=(0.3, 3.0),\n                        max_attempts=50, pad_val=(127, 127, 127), dtype='float32'):\n    \"\"\"Create augmenters for bbox/object detection.\n\n    Parameters\n    ----------\n    data_shape : tuple of int\n        Shape for output data\n    rand_crop : float\n        [0, 1], probability to apply random cropping\n    rand_pad : float\n        [0, 1], probability to apply random padding\n    rand_gray : float\n        [0, 1], probability to convert to grayscale for all channels\n    rand_mirror : bool\n        Whether to apply horizontal flip to image with probability 0.5\n    mean : np.ndarray or None\n        Mean pixel values for [r, g, b]\n    std : np.ndarray or None\n        Standard deviations for [r, g, b]\n    brightness : float\n        Brightness jittering range (percent)\n    contrast : float\n        Contrast jittering range (percent)\n    saturation : float\n        Saturation jittering range (percent)\n    hue : float\n        Hue jittering range (percent)\n    pca_noise : float\n        Pca noise level (percent)\n    inter_method : int, default=2(Area-based)\n        Interpolation method for all resizing operations\n\n        Possible values:\n        0: Nearest Neighbors Interpolation.\n        1: Bilinear interpolation.\n        2: Area-based (resampling using pixel area relation). It may be a\n        preferred method for image decimation, as it gives moire-free\n        results. But when the image is zoomed, it is similar to the Nearest\n        Neighbors method. (used by default).\n        3: Bicubic interpolation over 4x4 pixel neighborhood.\n        4: Lanczos interpolation over 8x8 pixel neighborhood.\n        10: Random select from interpolation method metioned above.\n        Note:\n        When shrinking an image, it will generally look best with AREA-based\n        interpolation, whereas, when enlarging an image, it will generally look best\n        with Bicubic (slow) or Bilinear (faster but still looks OK).\n    max_aspect_ratio : float\n        The cropped area of the image must have an aspect ratio = width / height\n        within this range.\n    area_range : tuple of floats\n        The cropped area of the image must contain a fraction of the supplied\n        image within in this range.\n    max_attempts : int\n        Number of attempts at generating a cropped/padded region of the image of the\n        specified constraints. After max_attempts failures, return the original image.\n    pad_val: float\n        Pixel value to be filled when padding is enabled. pad_val will automatically\n        be subtracted by mean and divided by std if applicable.\n\n    Examples\n    --------\n    >>> # An example of creating multiple augmenters\n    >>> augs = mx.gluon.contrib.data.create_bbox_augment(data_shape=(3, 300, 300), rand_crop=0.5,\n    ...    rand_pad=0.5, rand_mirror=True, mean=True, brightness=0.125, contrast=0.125,\n    ...    saturation=0.125, pca_noise=0.05, inter_method=10, min_object_covered=[0.3, 0.5, 0.9],\n    ...    area_range=(0.3, 3.0))\n    \"\"\"\n    if inter_method == 10:\n        inter_method = np.random.randint(0, 5)\n    augmenter = Sequential()\n    if rand_crop > 0:\n        augmenter.add(bbox.ImageBboxRandomCropWithConstraints(\n            p=rand_crop, min_scale=area_range[0], max_scale=1.0,\n            max_aspect_ratio=max_aspect_ratio, max_trial=max_attempts))\n\n    if rand_mirror > 0:\n        augmenter.add(bbox.ImageBboxRandomFlipLeftRight(0.5))\n\n    if rand_pad > 0:\n        augmenter.add(bbox.ImageBboxRandomExpand(\n            p=rand_pad, max_ratio=area_range[1], fill=pad_val))\n\n    # force resize\n    augmenter.add(bbox.ImageBboxResize(data_shape[2], data_shape[1], interp=inter_method))\n\n    if brightness or contrast or saturation or hue:\n        augmenter.add(transforms.image.RandomColorJitter(\n            brightness=brightness, contrast=contrast, saturation=saturation, hue=hue))\n\n    if pca_noise > 0:\n        augmenter.add(transforms.image.RandomLighting(pca_noise))\n\n    if rand_gray > 0:\n        augmenter.add(transforms.image.RandomGray(rand_gray))\n\n    if mean is True:\n        mean = [123.68, 116.28, 103.53]\n    elif mean is not None:\n        assert isinstance(mean, (tuple, list))\n\n    if std is True:\n        std = [58.395, 57.12, 57.375]\n    elif std is not None:\n        assert isinstance(std, (tuple, list))\n\n    augmenter.add(transforms.image.ToTensor())\n    if mean is not None or std is not None:\n        augmenter.add(transforms.image.Normalize(mean, std))\n\n    augmenter.add(transforms.Cast(dtype))\n\n    return augmenter\n\n\nclass ImageBboxDataLoader(object):\n    \"\"\"Image iterator with a large number of augmentation choices for detection.\n\n    Parameters\n    ----------\n    batch_size : int\n        Number of examples per batch.\n    data_shape : tuple\n        Data shape in (channels, height, width) format.\n        For now, only RGB image with 3 channels is supported.\n    path_imgrec : str\n        Path to image record file (.rec).\n        Created with tools/im2rec.py or bin/im2rec.\n    path_imglist : str\n        Path to image list (.lst).\n        Created with tools/im2rec.py or with custom script.\n        Format: Tab separated record of index, one or more labels and relative_path_from_root.\n    imglist: list\n        A list of images with the label(s).\n        Each item is a list [imagelabel: float or list of float, imgpath].\n    path_root : str\n        Root folder of image files.\n    shuffle : bool\n        Whether to shuffle all images at the start of each iteration or not.\n        Can be slow for HDD.\n    aug_list : list or None\n        Augmenter list for generating distorted images\n    part_index : int\n        Partition index.\n    num_parts : int\n        Total number of partitions.\n    last_batch : {'keep', 'discard', 'rollover'}\n        How to handle the last batch if batch_size does not evenly divide\n        `len(dataset)`.\n\n        keep - A batch with less samples than previous batches is returned.\n        discard - The last batch is discarded if its incomplete.\n        rollover - The remaining samples are rolled over to the next epoch.\n    kwargs : ...\n        More arguments for creating augmenter. See mx.gluon.contrib.data.create_bbox_augment.\n    \"\"\"\n    def __init__(self, batch_size, data_shape, path_imgrec=None, path_imglist=None, path_root='.',\n                 part_index=0, num_parts=1, aug_list=None, imglist=None,\n                 coord_normalized=True, dtype='float32', shuffle=False, sampler=None,\n                 last_batch=None, batch_sampler=None, batchify_fn=None,\n                 num_workers=0, pin_memory=False, pin_device_id=0,\n                 prefetch=None, thread_pool=False, timeout=120, try_nopython=None,\n                 **kwargs):\n        assert path_imgrec or path_imglist or (isinstance(imglist, list))\n        assert dtype in ['int32', 'float32', 'int64', 'float64'], dtype + ' label not supported'\n        logging.info('Using %s workers for decoding...', str(num_workers))\n        logging.info('Set `num_workers` variable to a larger number to speed up loading'\n                     ' (it requires shared memory to work and may occupy more memory).')\n        class_name = self.__class__.__name__\n        if path_imgrec:\n            logging.info('%s: loading recordio %s...',\n                         class_name, path_imgrec)\n            from ....data.vision.datasets import ImageRecordDataset\n            dataset = ImageRecordDataset(path_imgrec, flag=1)\n        elif path_imglist:\n            logging.info('%s: loading image list %s...', class_name, path_imglist)\n            from ....data.vision.datasets import ImageListDataset\n            dataset = ImageListDataset(path_root, path_imglist, flag=1)\n        elif isinstance(imglist, list):\n            logging.info('%s: loading image list...', class_name)\n            from ....data.vision.datasets import ImageListDataset\n            dataset = ImageListDataset(path_root, imglist, flag=1)\n        else:\n            raise ValueError('Either path_imgrec, path_imglist, or imglist must be provided')\n\n        if num_parts > 1:\n            dataset = dataset.shard(num_parts, part_index)\n\n        if aug_list is None:\n            # apply default transforms\n            augmenter = create_bbox_augment(data_shape, **kwargs)\n        elif isinstance(aug_list, list):\n            if all([isinstance(a, HybridBlock) for a in aug_list]):\n                augmenter = HybridSequential()\n            else:\n                augmenter = Sequential()\n            for aug in aug_list:\n                augmenter.add(aug)\n        elif isinstance(aug_list, Block):\n            augmenter = aug_list\n        else:\n            raise ValueError('aug_list must be a list of Blocks')\n        augmenter.hybridize()\n        wrapper_aug = Sequential()\n        wrapper_aug.add(BboxLabelTransform(coord_normalized))\n        wrapper_aug.add(augmenter)\n\n        if batchify_fn is None:\n            from ....data.batchify import Stack, Pad, Group\n            pad_batchify = Pad(val=-1)\n            pad_batchify._warned = True\n            batchify_fn = Group(Stack(), pad_batchify)  # stack image, pad bbox\n        self._iter = DataLoader(dataset.transform(wrapper_aug), batch_size=batch_size,\n                                shuffle=shuffle, sampler=sampler, last_batch=last_batch,\n                                batch_sampler=batch_sampler, batchify_fn=batchify_fn,\n                                num_workers=num_workers, pin_memory=pin_memory,\n                                pin_device_id=pin_device_id, prefetch=prefetch,\n                                thread_pool=thread_pool, timeout=timeout, try_nopython=try_nopython)\n\n    def __iter__(self):\n        return iter(self._iter)\n\n    def __len__(self):\n        return len(self._iter)\n\nclass BboxLabelTransform(Block):\n    \"\"\"Transform to convert 1-D bbox label to 2-D as in shape Nx5.\n\n    Parameters\n    ----------\n    coord_normalized : bool\n        Whether the coordinates(x0, y0, x1, y1) are normalized to (0, 1).\n\n    \"\"\"\n    def __init__(self, coord_normalized=True):\n        super(BboxLabelTransform, self).__init__()\n        self._coord_normalized = coord_normalized\n\n    def forward(self, img, label):\n        \"\"\"transform 1-D bbox label to Nx5 ndarray\"\"\"\n        if self._coord_normalized:\n            height = img.shape[0]\n            width = img.shape[1]\n        else:\n            height = width = None\n        if not isinstance(label, np.ndarray):\n            label = label.asnumpy()\n        label = label.flatten()\n        header_len = int(label[0])  # label header\n        label_width = int(label[1])  # the label width for each object, >= 5\n        if label_width < 5:\n            raise ValueError(\n                \"Label info for each object should >= 5, given {}\".format(label_width))\n        min_len = header_len + 5\n        if len(label) < min_len:\n            raise ValueError(\n                \"Expected label length >= {}, got {}\".format(min_len, len(label)))\n        if (len(label) - header_len) % label_width:\n            raise ValueError(\n                \"Broken label of size {}, cannot reshape into (N, {}) \"\n                \"if header length {} is excluded\".format(len(label), label_width, header_len))\n        bbox_label = label[header_len:].reshape(-1, label_width)\n        # swap columns, requires [xmin-ymin-xmax-ymax-id-extra0-extra1-xxx]\n        ids = bbox_label[:, 0].copy()\n        bbox_label[:, :4] = bbox_label[:, 1:5]\n        bbox_label[:, 4] = ids\n        # restore to absolute coordinates\n        if width is not None:\n            bbox_label[:, (0, 2)] *= width\n        if height is not None:\n            bbox_label[:, (1, 3)] *= height\n        array_fn = _mx_np.array if is_np_array() else nd.array\n        return img, array_fn(bbox_label)\n"
  },
  {
    "path": "python/mxnet/gluon/contrib/data/vision/transforms/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Contrib vision transforms.\"\"\"\nfrom .bbox import *\n"
  },
  {
    "path": "python/mxnet/gluon/contrib/data/vision/transforms/bbox/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Gluon contrib vision bbox transform\"\"\"\nfrom .bbox import *\n"
  },
  {
    "path": "python/mxnet/gluon/contrib/data/vision/transforms/bbox/bbox.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable= arguments-differ, wildcard-import\n\"Bounding box transforms.\"\nimport random\n\nfrom .......base import numeric_types\nfrom ......block import Block\nfrom .......util import is_np_array\nfrom ....... import ndarray as nd, numpy_extension as npx, numpy as np\nfrom .utils import _check_bbox_shape, bbox_crop, bbox_translate\nfrom .utils import bbox_resize, bbox_random_crop_with_constraints\n\n__all__ = ['ImageBboxRandomFlipLeftRight', 'ImageBboxCrop',\n           'ImageBboxRandomCropWithConstraints', 'ImageBboxResize']\n\n\nclass ImageBboxRandomFlipLeftRight(Block):\n    \"\"\"Randomly flip the input image and bbox left to right with a probability\n    of p(0.5 by default).\n\n    Parameters\n    ----------\n    p : float\n        The probability to preceed with random cropping logic.\n\n    Inputs:\n        - **data**: input tensor with (Hi x Wi x C) shape.\n        - **bbox**: input tensor with shape (N, 4+) where N is the number of bounding boxes.\n            The second axis represents attributes of the bounding box.\n            Specifically, these are :math:`(x_{min}, y_{min}, x_{max}, y_{max})`,\n            we allow additional attributes other than coordinates, which stay intact\n            during bounding box transformations.\n\n    Outputs:\n        - **out**: output tensor with same shape as `data`.\n        - **bbox**: input tensor with same shape as `bbox`.\n    \"\"\"\n    def __init__(self, p=0.5):\n        super(ImageBboxRandomFlipLeftRight, self).__init__()\n        self.p = p\n\n    def forward(self, img, bbox):\n        _check_bbox_shape(bbox)\n        if self.p <= 0:\n            return img, bbox\n        elif self.p >= 1:\n            img = self._flip_image(img)\n            bbox = self._flip_bbox(img, bbox)\n            return img, bbox\n        else:\n            if self.p < random.random():\n                return img, bbox\n            else:\n                img = self._flip_image(img)\n                bbox = self._flip_bbox(img, bbox)\n                return img, bbox\n\n    def _flip_image(self, img):\n        if is_np_array():\n            return npx.image.flip_left_right(img)\n        else:\n            return nd.image.flip_left_right(img)\n\n    def _flip_bbox(self, img, bbox):\n        width = img.shape[-2]\n        xmax = width - bbox[:, 0]\n        xmin = width - bbox[:, 2]\n        bbox[:, 0] = xmin\n        bbox[:, 2] = xmax\n        return bbox\n\n\nclass ImageBboxCrop(Block):\n    \"\"\"Crops the image `src` and `bbox` to the given `crop`.\n\n    Parameters\n    ----------\n    crop_box : tuple\n        Tuple of length 4. :math:`(x_{min}, y_{min}, width, height)`\n    allow_outside_center : bool\n        If `False`, remove bounding boxes which have centers outside cropping area.\n\n\n    Inputs:\n        - **data**: input tensor with (Hi x Wi x C) shape.\n        - **bbox**: input tensor with shape (N, 4+) where N is the number of bounding boxes.\n            The second axis represents attributes of the bounding box.\n            Specifically, these are :math:`(x_{min}, y_{min}, x_{max}, y_{max})`,\n            we allow additional attributes other than coordinates, which stay intact\n            during bounding box transformations.\n\n    Outputs:\n        - **out**: output tensor with (H x W x C) shape.\n        - **bbox**: output tensor with shape (M, 4+) where M <= N is the number of valid bounding\n            boxes after cropping. :math:`(x_{min}, y_{min}, x_{max}, y_{max})`\n\n    \"\"\"\n    def __init__(self, crop, allow_outside_center=False):\n        super(ImageBboxCrop, self).__init__()\n        assert len(crop) == 4, \"expect crop to be (x_min, y_min, x_max, y_max)\"\n        self.xmin = crop[0]\n        self.ymin = crop[1]\n        self.width = crop[2]\n        self.height = crop[3]\n        assert self.xmin >= 0\n        assert self.ymin >= 0\n        assert self.width > 0\n        assert self.height > 0\n        self.xmax = self.width + self.xmin\n        self.ymax = self.height + self.ymin\n        self._allow_outside_center = allow_outside_center\n\n    def forward(self, img, bbox):\n        if self.xmax >= img.shape[-2] or self.ymax >= img.shape[-3]:\n            return img, bbox\n        if is_np_array():\n            new_img = npx.image.crop(img, self.xmin, self.ymin, self.width, self.height)\n            new_bbox = np.array(bbox_crop(bbox.asnumpy(),\n                                          (self.xmin, self.ymin, self.width, self.height),\n                                          self._allow_outside_center))\n        else:\n            new_img = nd.image.crop(img, self.xmin, self.ymin, self.width, self.height)\n            new_bbox = nd.array(bbox_crop(bbox.asnumpy(),\n                                          (self.xmin, self.ymin, self.width, self.height),\n                                          self._allow_outside_center))\n        return new_img, new_bbox\n\n\nclass ImageBboxRandomCropWithConstraints(Block):\n    \"\"\"Crop an image randomly with bounding box constraints.\n\n    Please check `mx.gluon.contrib.data.transforms.bbox.utils.bbox_random_crop_with_constraints`\n    for implementation details.\n\n    Parameters\n    ----------\n    p : float\n        The probability to preceed with random cropping logic.\n    min_scale : float\n        The minimum ratio between a cropped region and the original image.\n        The default value is :obj:`0.3`.\n    max_scale : float\n        The maximum ratio between a cropped region and the original image.\n        The default value is :obj:`1`.\n    max_aspect_ratio : float\n        The maximum aspect ratio of cropped region.\n        The default value is :obj:`2`.\n    constraints : iterable of tuples\n        An iterable of constraints.\n        Each constraint should be :obj:`(min_iou, max_iou)` format.\n        If means no constraint if set :obj:`min_iou` or :obj:`max_iou` to :obj:`None`.\n        If this argument defaults to :obj:`None`, :obj:`((0.1, None), (0.3, None),\n        (0.5, None), (0.7, None), (0.9, None), (None, 1))` will be used.\n    max_trial : int\n        Maximum number of trials for each constraint before exit no matter what.\n\n    Inputs:\n        - **data**: input tensor with (Hi x Wi x C) shape.\n        - **bbox**: input tensor with shape (N, 4+) where N is the number of bounding boxes.\n            The second axis represents attributes of the bounding box.\n            Specifically, these are :math:`(x_{min}, y_{min}, x_{max}, y_{max})`,\n            we allow additional attributes other than coordinates, which stay intact\n            during bounding box transformations.\n\n    Outputs:\n        - **out**: Cropped image with shape (H x W x C)\n        - **bbox**: Cropped bounding boxes with shape :obj:`(M, 4+)` where M <= N.\n            Tuple of length 4 as :math:`(x_{min}, y_{min}, x_{max}, y_{max})`.\n    \"\"\"\n    def __init__(self, p=0.5, min_scale=0.3, max_scale=1,\n                 max_aspect_ratio=2, constraints=None,\n                 max_trial=50):\n        super(ImageBboxRandomCropWithConstraints, self).__init__()\n        self.p = p\n        self._args = {\n            \"min_scale\": min_scale,\n            \"max_scale\": max_scale,\n            \"max_aspect_ratio\": max_aspect_ratio,\n            \"constraints\": constraints,\n            \"max_trial\": max_trial\n        }\n\n    def forward(self, img, bbox):\n        if random.random() > self.p:\n            return img, bbox\n        im_size = (img.shape[-2], img.shape[-3])\n        new_bbox, crop = bbox_random_crop_with_constraints(bbox.asnumpy(), im_size, **self._args)\n        if crop == (0, 0, im_size[0], im_size[1]):\n            return img, bbox\n        if is_np_array():\n            new_img = npx.image.crop(img, x=crop[0], y=crop[1], width=crop[2], height=crop[3])\n            new_bbox = np.array(new_bbox)\n        else:\n            new_img = nd.image.crop(img, x=crop[0], y=crop[1], width=crop[2], height=crop[3])\n            new_bbox = nd.array(new_bbox)\n        return new_img, new_bbox\n\n\nclass ImageBboxRandomExpand(Block):\n    \"\"\"Randomly expand image to a larger region with padded pixels.\n    Apply tranlation to bounding boxes accordingly.\n\n    Parameters\n    ----------\n    p : float\n        The probability to preceed with random cropping logic.\n    max_ratio : float\n        The minimum expansion ratio. If `max_ratio` is 2, the range of\n        output image size is 1x ~ 2x of the original input size.\n    fill : float or tuple of float\n        The value(s) for the pixels in expanded regions. Can be scalar or tuple,\n        note the if tuple is provided, its size must match the image channels, typically 3.\n    keep_ratio : bool\n        If `True`, the output must have the same aspect ratio as input, otherwise the output\n        can have arbitrary aspect ratio.\n\n    Inputs:\n        - **data**: input tensor with (Hi x Wi x C) shape.\n        - **bbox**: input tensor with shape (N, 4+) where N is the number of bounding boxes.\n            The second axis represents attributes of the bounding box.\n            Specifically, these are :math:`(x_{min}, y_{min}, x_{max}, y_{max})`,\n            we allow additional attributes other than coordinates, which stay intact\n            during bounding box transformations.\n\n    Outputs:\n        - **out**: Cropped image with shape (H x W x C)\n        - **bbox**: Cropped bounding boxes with shape :obj:`(N, 4+)`.\n            Tuple of length 4 as :math:`(x_{min}, y_{min}, x_{max}, y_{max})`.\n\n    \"\"\"\n    def __init__(self, p=0.5, max_ratio=4, fill=0, keep_ratio=True):\n        super(ImageBboxRandomExpand, self).__init__()\n        self.p = p\n        self._max_ratio = max_ratio\n        self._fill = fill\n        self._keep_ratio = keep_ratio\n\n    def forward(self, img, bbox):\n        if self._max_ratio <= 1 or random.random() > self.p:\n            return img, bbox\n        if len(img.shape) != 3:\n            raise NotImplementedError('ImageBboxRandomExpand only support images in HWC format')\n\n        h, w, c = img.shape\n        ratio_x = random.uniform(1, self._max_ratio)\n        if self._keep_ratio:\n            ratio_y = ratio_x\n        else:\n            ratio_y = random.uniform(1, self._max_ratio)\n\n        oh, ow = int(h * ratio_y), int(w * ratio_x)\n        off_y = random.randint(0, oh - h)\n        off_x = random.randint(0, ow - w)\n\n        # make canvas\n        if is_np_array():\n            F = np\n        else:\n            F = nd\n        if isinstance(self._fill, numeric_types):\n            dst = F.full(shape=(oh, ow, c), val=self._fill, dtype=img.dtype)\n        else:\n            fill = F.array(self._fill, dtype=img.dtype, ctx=img.device)\n            if not c == fill.size:\n                raise ValueError(\"Channel and fill size mismatch, {} vs {}\".format(c, fill.size))\n            dst = F.tile(fill.reshape((1, c)), reps=(oh * ow, 1)).reshape((oh, ow, c))\n\n        dst[off_y:off_y+h, off_x:off_x+w, :] = img\n\n        # translate bbox\n        new_bbox = bbox_translate(bbox.asnumpy(), off_x, off_y)\n        if is_np_array():\n            new_bbox = np.array(new_bbox)\n        else:\n            new_bbox = nd.array(new_bbox)\n\n        return dst, new_bbox\n\n\nclass ImageBboxResize(Block):\n    \"\"\"Apply resize to image and bounding boxes.\n\n    Parameters\n    ----------\n    width : int\n        The target output width.\n    height : int\n        The target output height.\n\n    Inputs:\n        - **data**: input tensor with (Hi x Wi x C) shape.\n        - **bbox**: input tensor with shape (N, 4+) where N is the number of bounding boxes.\n            The second axis represents attributes of the bounding box.\n            Specifically, these are :math:`(x_{min}, y_{min}, x_{max}, y_{max})`,\n            we allow additional attributes other than coordinates, which stay intact\n            during bounding box transformations.\n\n    Outputs:\n        - **out**: Cropped image with shape (H x W x C)\n        - **bbox**: Cropped bounding boxes with shape :obj:`(M, 4+)` where M <= N.\n            Tuple of length 4 as :math:`(x_{min}, y_{min}, x_{max}, y_{max})`.\n\n    \"\"\"\n    def __init__(self, width, height, interp=1):\n        super(ImageBboxResize, self).__init__()\n        self._size = (width, height)\n        self._interp = interp\n\n    def forward(self, img, bbox):\n        if len(img.shape) != 3:\n            raise NotImplementedError('ImageBboxResize only support images in HWC format')\n\n        if self._interp == -1:\n            # random interpolation mode\n            interp = random.randint(0, 5)\n        else:\n            interp = self._interp\n\n        if is_np_array():\n            new_img = npx.image.resize(img, self._size, False, interp)\n            new_bbox = np.array(bbox_resize(bbox.asnumpy(),\n                                            (img.shape[-2], img.shape[-3]), self._size))\n        else:\n            new_img = nd.image.resize(img, self._size, False, interp)\n            new_bbox = nd.array(bbox_resize(bbox.asnumpy(),\n                                            (img.shape[-2], img.shape[-3]), self._size))\n        return new_img, new_bbox\n"
  },
  {
    "path": "python/mxnet/gluon/contrib/data/vision/transforms/bbox/utils.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable= arguments-differ, wildcard-import\n\"Bounding box utilities.\"\nfrom __future__ import division\nimport random\n\nimport numpy as np\n\ndef _check_bbox_shape(bbox):\n    assert len(bbox.shape) == 2, \"bbox requires shape of (N, 4+), given: {}\".format(bbox.shape)\n    assert bbox.shape[1] >= 4, \"bbox requires shape of (N, 4+), given: {}\".format(bbox.shape)\n\ndef bbox_crop(bbox, crop_box=None, allow_outside_center=True):\n    \"\"\"Crop bounding boxes according to slice area.\n    This method is mainly used with image cropping to ensure bonding boxes fit\n    within the cropped image.\n\n    Parameters\n    ----------\n    bbox : numpy.ndarray\n        Numpy.ndarray with shape (N, 4+) where N is the number of bounding boxes.\n        The second axis represents attributes of the bounding box.\n        Specifically, these are :math:`(x_{min}, y_{min}, x_{max}, y_{max})`,\n        we allow additional attributes other than coordinates, which stay intact\n        during bounding box transformations.\n    crop_box : tuple\n        Tuple of length 4. :math:`(x_{min}, y_{min}, width, height)`\n    allow_outside_center : bool\n        If `False`, remove bounding boxes which have centers outside cropping area.\n    Returns\n    -------\n    numpy.ndarray\n        Cropped bounding boxes with shape (M, 4+) where M <= N.\n    \"\"\"\n    bbox = bbox.copy()\n    if crop_box is None:\n        return bbox\n    if not len(crop_box) == 4:\n        raise ValueError(\n            \"Invalid crop_box parameter, requires length 4, given {}\".format(str(crop_box)))\n    if sum([int(c is None) for c in crop_box]) == 4:\n        return bbox\n\n    l, t, w, h = crop_box\n\n    left = l if l else 0\n    top = t if t else 0\n    right = left + (w if w else np.inf)\n    bottom = top + (h if h else np.inf)\n    crop_bbox = np.array((left, top, right, bottom))\n\n    if allow_outside_center:\n        mask = np.ones(bbox.shape[0], dtype=bool)\n    else:\n        centers = (bbox[:, :2] + bbox[:, 2:4]) / 2\n        mask = ((crop_bbox[:2] <= centers) * (centers < crop_bbox[2:])).all(axis=1)\n\n    # transform borders\n    bbox[:, :2] = np.maximum(bbox[:, :2], crop_bbox[:2])\n    bbox[:, 2:4] = np.minimum(bbox[:, 2:4], crop_bbox[2:4])\n    bbox[:, :2] -= crop_bbox[:2]\n    bbox[:, 2:4] -= crop_bbox[:2]\n\n    mask = (mask * (bbox[:, :2] < bbox[:, 2:4]).all(axis=1))\n    bbox = bbox[mask]\n    return bbox\n\ndef bbox_flip(bbox, size, flip_x=False, flip_y=False):\n    \"\"\"Flip bounding boxes according to image flipping directions.\n\n    Parameters\n    ----------\n    bbox : numpy.ndarray\n        Numpy.ndarray with shape (N, 4+) where N is the number of bounding boxes.\n        The second axis represents attributes of the bounding box.\n        Specifically, these are :math:`(x_{min}, y_{min}, x_{max}, y_{max})`,\n        we allow additional attributes other than coordinates, which stay intact\n        during bounding box transformations.\n    size : tuple\n        Tuple of length 2: (width, height).\n    flip_x : bool\n        Whether flip horizontally.\n    flip_y : bool\n        Whether flip vertically.\n\n    Returns\n    -------\n    numpy.ndarray\n        Flipped bounding boxes with original shape.\n    \"\"\"\n    if not len(size) == 2:\n        raise ValueError(\"size requires length 2 tuple, given {}\".format(len(size)))\n    width, height = size\n    bbox = bbox.copy()\n    if flip_y:\n        ymax = height - bbox[:, 1]\n        ymin = height - bbox[:, 3]\n        bbox[:, 1] = ymin\n        bbox[:, 3] = ymax\n    if flip_x:\n        xmax = width - bbox[:, 0]\n        xmin = width - bbox[:, 2]\n        bbox[:, 0] = xmin\n        bbox[:, 2] = xmax\n    return bbox\n\ndef bbox_resize(bbox, in_size, out_size):\n    \"\"\"Resize bouding boxes according to image resize operation.\n\n    Parameters\n    ----------\n    bbox : numpy.ndarray\n        Numpy.ndarray with shape (N, 4+) where N is the number of bounding boxes.\n        The second axis represents attributes of the bounding box.\n        Specifically, these are :math:`(x_{min}, y_{min}, x_{max}, y_{max})`,\n        we allow additional attributes other than coordinates, which stay intact\n        during bounding box transformations.\n    in_size : tuple\n        Tuple of length 2: (width, height) for input.\n    out_size : tuple\n        Tuple of length 2: (width, height) for output.\n\n    Returns\n    -------\n    numpy.ndarray\n        Resized bounding boxes with original shape.\n    \"\"\"\n    if not len(in_size) == 2:\n        raise ValueError(\"in_size requires length 2 tuple, given {}\".format(len(in_size)))\n    if not len(out_size) == 2:\n        raise ValueError(\"out_size requires length 2 tuple, given {}\".format(len(out_size)))\n\n    bbox = bbox.copy().astype(float)\n    x_scale = out_size[0] / in_size[0]\n    y_scale = out_size[1] / in_size[1]\n    bbox[:, 1] = y_scale * bbox[:, 1]\n    bbox[:, 3] = y_scale * bbox[:, 3]\n    bbox[:, 0] = x_scale * bbox[:, 0]\n    bbox[:, 2] = x_scale * bbox[:, 2]\n    return bbox\n\ndef bbox_translate(bbox, x_offset=0, y_offset=0):\n    \"\"\"Translate bounding boxes by offsets.\n\n    Parameters\n    ----------\n    bbox : numpy.ndarray\n        Numpy.ndarray with shape (N, 4+) where N is the number of bounding boxes.\n        The second axis represents attributes of the bounding box.\n        Specifically, these are :math:`(x_{min}, y_{min}, x_{max}, y_{max})`,\n        we allow additional attributes other than coordinates, which stay intact\n        during bounding box transformations.\n    x_offset : int or float\n        Offset along x axis.\n    y_offset : int or float\n        Offset along y axis.\n\n    Returns\n    -------\n    numpy.ndarray\n        Translated bounding boxes with original shape.\n    \"\"\"\n    bbox = bbox.copy()\n    bbox[:, :2] += (x_offset, y_offset)\n    bbox[:, 2:4] += (x_offset, y_offset)\n    return bbox\n\ndef bbox_iou(bbox_a, bbox_b, offset=0):\n    \"\"\"Calculate Intersection-Over-Union(IOU) of two bounding boxes.\n\n    Parameters\n    ----------\n    bbox_a : numpy.ndarray\n        An ndarray with shape :math:`(N, 4)`.\n    bbox_b : numpy.ndarray\n        An ndarray with shape :math:`(M, 4)`.\n    offset : float or int, default is 0\n        The ``offset`` is used to control the whether the width(or height) is computed as\n        (right - left + ``offset``).\n        Note that the offset must be 0 for normalized bboxes, whose ranges are in ``[0, 1]``.\n\n    Returns\n    -------\n    numpy.ndarray\n        An ndarray with shape :math:`(N, M)` indicates IOU between each pairs of\n        bounding boxes in `bbox_a` and `bbox_b`.\n\n    \"\"\"\n    if bbox_a.shape[1] < 4 or bbox_b.shape[1] < 4:\n        raise IndexError(\"Bounding boxes axis 1 must have at least length 4\")\n\n    tl = np.maximum(bbox_a[:, None, :2], bbox_b[:, :2])\n    br = np.minimum(bbox_a[:, None, 2:4], bbox_b[:, 2:4])\n\n    area_i = np.prod(br - tl + offset, axis=2) * (tl < br).all(axis=2)\n    area_a = np.prod(bbox_a[:, 2:4] - bbox_a[:, :2] + offset, axis=1)\n    area_b = np.prod(bbox_b[:, 2:4] - bbox_b[:, :2] + offset, axis=1)\n    return area_i / (area_a[:, None] + area_b - area_i)\n\n\ndef bbox_xywh_to_xyxy(xywh):\n    \"\"\"Convert bounding boxes from format (xmin, ymin, w, h) to (xmin, ymin, xmax, ymax)\n\n    Parameters\n    ----------\n    xywh : list, tuple or numpy.ndarray\n        The bbox in format (x, y, w, h).\n        If numpy.ndarray is provided, we expect multiple bounding boxes with\n        shape `(N, 4)`.\n\n    Returns\n    -------\n    tuple or numpy.ndarray\n        The converted bboxes in format (xmin, ymin, xmax, ymax).\n        If input is numpy.ndarray, return is numpy.ndarray correspondingly.\n\n    \"\"\"\n    if isinstance(xywh, (tuple, list)):\n        if not len(xywh) == 4:\n            raise IndexError(\n                \"Bounding boxes must have 4 elements, given {}\".format(len(xywh)))\n        w, h = np.maximum(xywh[2] - 1, 0), np.maximum(xywh[3] - 1, 0)\n        return xywh[0], xywh[1], xywh[0] + w, xywh[1] + h\n    elif isinstance(xywh, np.ndarray):\n        if not xywh.size % 4 == 0:\n            raise IndexError(\n                \"Bounding boxes must have n * 4 elements, given {}\".format(xywh.shape))\n        xyxy = np.hstack((xywh[:, :2], xywh[:, :2] + np.maximum(0, xywh[:, 2:4] - 1)))\n        return xyxy\n    else:\n        raise TypeError(\n            'Expect input xywh a list, tuple or numpy.ndarray, given {}'.format(type(xywh)))\n\n\ndef bbox_xyxy_to_xywh(xyxy):\n    \"\"\"Convert bounding boxes from format (xmin, ymin, xmax, ymax) to (x, y, w, h).\n\n    Parameters\n    ----------\n    xyxy : list, tuple or numpy.ndarray\n        The bbox in format (xmin, ymin, xmax, ymax).\n        If numpy.ndarray is provided, we expect multiple bounding boxes with\n        shape `(N, 4)`.\n\n    Returns\n    -------\n    tuple or numpy.ndarray\n        The converted bboxes in format (x, y, w, h).\n        If input is numpy.ndarray, return is numpy.ndarray correspondingly.\n\n    \"\"\"\n    if isinstance(xyxy, (tuple, list)):\n        if not len(xyxy) == 4:\n            raise IndexError(\n                \"Bounding boxes must have 4 elements, given {}\".format(len(xyxy)))\n        x1, y1 = xyxy[0], xyxy[1]\n        w, h = xyxy[2] - x1 + 1, xyxy[3] - y1 + 1\n        return x1, y1, w, h\n    elif isinstance(xyxy, np.ndarray):\n        if not xyxy.size % 4 == 0:\n            raise IndexError(\n                \"Bounding boxes must have n * 4 elements, given {}\".format(xyxy.shape))\n        return np.hstack((xyxy[:, :2], xyxy[:, 2:4] - xyxy[:, :2] + 1))\n    else:\n        raise TypeError(\n            'Expect input xywh a list, tuple or numpy.ndarray, given {}'.format(type(xyxy)))\n\n\ndef bbox_clip_xyxy(xyxy, width, height):\n    \"\"\"Clip bounding box with format (xmin, ymin, xmax, ymax) to specified boundary.\n\n    All bounding boxes will be clipped to the new region `(0, 0, width, height)`.\n\n    Parameters\n    ----------\n    xyxy : list, tuple or numpy.ndarray\n        The bbox in format (xmin, ymin, xmax, ymax).\n        If numpy.ndarray is provided, we expect multiple bounding boxes with\n        shape `(N, 4)`.\n    width : int or float\n        Boundary width.\n    height : int or float\n        Boundary height.\n\n    Returns\n    -------\n    type\n        Description of returned object.\n\n    \"\"\"\n    if isinstance(xyxy, (tuple, list)):\n        if not len(xyxy) == 4:\n            raise IndexError(\n                \"Bounding boxes must have 4 elements, given {}\".format(len(xyxy)))\n        x1 = np.minimum(width - 1, np.maximum(0, xyxy[0]))\n        y1 = np.minimum(height - 1, np.maximum(0, xyxy[1]))\n        x2 = np.minimum(width - 1, np.maximum(0, xyxy[2]))\n        y2 = np.minimum(height - 1, np.maximum(0, xyxy[3]))\n        return x1, y1, x2, y2\n    elif isinstance(xyxy, np.ndarray):\n        if not xyxy.size % 4 == 0:\n            raise IndexError(\n                \"Bounding boxes must have n * 4 elements, given {}\".format(xyxy.shape))\n        x1 = np.minimum(width - 1, np.maximum(0, xyxy[:, 0]))\n        y1 = np.minimum(height - 1, np.maximum(0, xyxy[:, 1]))\n        x2 = np.minimum(width - 1, np.maximum(0, xyxy[:, 2]))\n        y2 = np.minimum(height - 1, np.maximum(0, xyxy[:, 3]))\n        return np.hstack((x1, y1, x2, y2))\n    else:\n        raise TypeError(\n            'Expect input xywh a list, tuple or numpy.ndarray, given {}'.format(type(xyxy)))\n\ndef bbox_random_crop_with_constraints(bbox, size, min_scale=0.3, max_scale=1,\n                                      max_aspect_ratio=2, constraints=None,\n                                      max_trial=50):\n    \"\"\"Crop an image randomly with bounding box constraints.\n\n    This data augmentation is used in training of\n    Single Shot Multibox Detector [#]_. More details can be found in\n    data augmentation section of the original paper.\n    .. [#] Wei Liu, Dragomir Anguelov, Dumitru Erhan, Christian Szegedy,\n       Scott Reed, Cheng-Yang Fu, Alexander C. Berg.\n       SSD: Single Shot MultiBox Detector. ECCV 2016.\n\n    Parameters\n    ----------\n    bbox : numpy.ndarray\n        Numpy.ndarray with shape (N, 4+) where N is the number of bounding boxes.\n        The second axis represents attributes of the bounding box.\n        Specifically, these are :math:`(x_{min}, y_{min}, x_{max}, y_{max})`,\n        we allow additional attributes other than coordinates, which stay intact\n        during bounding box transformations.\n    size : tuple\n        Tuple of length 2 of image shape as (width, height).\n    min_scale : float\n        The minimum ratio between a cropped region and the original image.\n        The default value is :obj:`0.3`.\n    max_scale : float\n        The maximum ratio between a cropped region and the original image.\n        The default value is :obj:`1`.\n    max_aspect_ratio : float\n        The maximum aspect ratio of cropped region.\n        The default value is :obj:`2`.\n    constraints : iterable of tuples\n        An iterable of constraints.\n        Each constraint should be :obj:`(min_iou, max_iou)` format.\n        If means no constraint if set :obj:`min_iou` or :obj:`max_iou` to :obj:`None`.\n        If this argument defaults to :obj:`None`, :obj:`((0.1, None), (0.3, None),\n        (0.5, None), (0.7, None), (0.9, None), (None, 1))` will be used.\n    max_trial : int\n        Maximum number of trials for each constraint before exit no matter what.\n\n    Returns\n    -------\n    numpy.ndarray\n        Cropped bounding boxes with shape :obj:`(M, 4+)` where M <= N.\n    tuple\n        Tuple of length 4 as (x_offset, y_offset, new_width, new_height).\n\n    \"\"\"\n    # default params in paper\n    if constraints is None:\n        constraints = (\n            (0.1, None),\n            (0.3, None),\n            (0.5, None),\n            (0.7, None),\n            (0.9, None),\n            (None, 1),\n        )\n\n    w, h = size\n\n    candidates = [(0, 0, w, h)]\n    for min_iou, max_iou in constraints:\n        min_iou = -np.inf if min_iou is None else min_iou\n        max_iou = np.inf if max_iou is None else max_iou\n\n        for _ in range(max_trial):\n            scale = random.uniform(min_scale, max_scale)\n            aspect_ratio = random.uniform(\n                max(1 / max_aspect_ratio, scale * scale),\n                min(max_aspect_ratio, 1 / (scale * scale)))\n            crop_h = int(h * scale / np.sqrt(aspect_ratio))\n            crop_w = int(w * scale * np.sqrt(aspect_ratio))\n\n            crop_t = random.randrange(h - crop_h)\n            crop_l = random.randrange(w - crop_w)\n            crop_bb = np.array((crop_l, crop_t, crop_l + crop_w, crop_t + crop_h))\n\n            if len(bbox) == 0:\n                top, bottom = crop_t, crop_t + crop_h\n                left, right = crop_l, crop_l + crop_w\n                return bbox, (left, top, right-left, bottom-top)\n\n            iou = bbox_iou(bbox, crop_bb[np.newaxis])\n            if min_iou <= iou.min() and iou.max() <= max_iou:\n                top, bottom = crop_t, crop_t + crop_h\n                left, right = crop_l, crop_l + crop_w\n                candidates.append((left, top, right-left, bottom-top))\n                break\n\n    # random select one\n    while candidates:\n        crop = candidates.pop(np.random.randint(0, len(candidates)))\n        new_bbox = bbox_crop(bbox, crop, allow_outside_center=False)\n        if new_bbox.size < 1:\n            continue\n        new_crop = (crop[0], crop[1], crop[2], crop[3])\n        return new_bbox, new_crop\n    return bbox, (0, 0, w, h)\n"
  },
  {
    "path": "python/mxnet/gluon/contrib/estimator/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=wildcard-import\n\"\"\"Gluon Estimator Module\"\"\"\nfrom . import estimator\nfrom . import event_handler\nfrom . import batch_processor\nfrom .estimator import *\nfrom .event_handler import *\nfrom .batch_processor import *\n"
  },
  {
    "path": "python/mxnet/gluon/contrib/estimator/batch_processor.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import, unused-argument, too-many-ancestors\n\"\"\"Gluon Batch Processor for Estimators\"\"\"\n\nfrom ...utils import split_and_load\nfrom .... import autograd\nfrom .... import npx\n\n__all__ = ['BatchProcessor']\n\nclass BatchProcessor(object):\n    \"\"\"BatchProcessor Class for plug and play fit_batch & evaluate_batch\n\n    During training or validation, data are divided into minibatches for processing. This\n    class aims at providing hooks of training or validating on a minibatch of data. Users\n    may provide customized fit_batch() and evaluate_batch() methods by inheriting from\n    this class and overriding class methods.\n\n    :py:class:`BatchProcessor` can be used to replace fit_batch() and evaluate_batch()\n    in the base estimator class\n    \"\"\"\n\n    def __init__(self):\n        pass\n\n    def _get_data_and_label(self, batch, ctx, batch_axis=0):\n        data = batch[0]\n        label = batch[1]\n        data = split_and_load(data, ctx_list=ctx, batch_axis=batch_axis)\n        label = split_and_load(label, ctx_list=ctx, batch_axis=batch_axis)\n        return data, label\n\n    def evaluate_batch(self, estimator,\n                       val_batch,\n                       batch_axis=0):\n        \"\"\"Evaluate the estimator model on a batch of validation data.\n\n        Parameters\n        ----------\n        estimator : Estimator\n            Reference to the estimator\n        val_batch : tuple\n            Data and label of a batch from the validation data loader.\n        batch_axis : int, default 0\n            Batch axis to split the validation data into devices.\n        \"\"\"\n        data, label = self._get_data_and_label(val_batch, estimator.device, batch_axis)\n        pred = [estimator.val_net(x) for x in data]\n        loss = [estimator.val_loss(y_hat, y) for y_hat, y in zip(pred, label)]\n\n        return data, label, pred, loss\n\n    def fit_batch(self, estimator,\n                  train_batch,\n                  batch_axis=0):\n        \"\"\"Trains the estimator model on a batch of training data.\n\n        Parameters\n        ----------\n        estimator : Estimator\n            Reference to the estimator\n        train_batch : tuple\n            Data and label of a batch from the training data loader.\n        batch_axis : int, default 0\n            Batch axis to split the training data into devices.\n\n        Returns\n        -------\n        data: List of NDArray\n            Sharded data from the batch. Data is sharded with\n            `gluon.split_and_load`.\n        label: List of NDArray\n            Sharded label from the batch. Labels are sharded with\n            `gluon.split_and_load`.\n        pred: List of NDArray\n            Prediction on each of the sharded inputs.\n        loss: List of NDArray\n            Loss on each of the sharded inputs.\n        \"\"\"\n        data, label = self._get_data_and_label(train_batch, estimator.device, batch_axis)\n\n        with autograd.record():\n            pred = [estimator.net(x) for x in data]\n            loss = [estimator.loss(y_hat, y) for y_hat, y in zip(pred, label)]\n\n        for l in loss:\n            l.backward()\n\n        npx.waitall()\n\n        return data, label, pred, loss\n"
  },
  {
    "path": "python/mxnet/gluon/contrib/estimator/estimator.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import, unused-variable\n\"\"\"Gluon Estimator\"\"\"\n\nimport copy\nimport logging\nimport sys\nimport warnings\n\nfrom .event_handler import MetricHandler, ValidationHandler, LoggingHandler, StoppingHandler, GradientUpdateHandler\nfrom .event_handler import TrainBegin, EpochBegin, BatchBegin, BatchEnd, EpochEnd, TrainEnd\nfrom .event_handler import _check_event_handlers\nfrom .utils import _check_metrics, _suggest_metric_for_loss, _check_handler_metric_ref\nfrom ...data import DataLoader\nfrom ...loss import Loss as gluon_loss\nfrom ...trainer import Trainer\nfrom ...utils import split_and_load\nfrom ....device import Device, cpu, gpu, num_gpus\nfrom ...metric import Loss as metric_loss\nfrom .batch_processor import BatchProcessor\n\n__all__ = ['Estimator']\n\n\nclass Estimator(object):\n    \"\"\"Estimator Class for easy model training\n\n    :py:class:`Estimator` can be used to facilitate the training & validation process\n\n\n    Parameters\n    ----------\n    net : gluon.Block\n        The model used for training.\n    loss : gluon.loss.Loss\n        Loss (objective) function to calculate during training.\n    train_metrics : EvalMetric or list of EvalMetric\n        Training metrics for evaluating models on training dataset.\n    val_metrics : EvalMetric or list of EvalMetric\n        Validation metrics for evaluating models on validation dataset.\n    initializer : Initializer\n        Initializer to initialize the network.\n    trainer : Trainer\n        Trainer to apply optimizer on network parameters.\n    device : Device or list of Device\n        Device(s) to run the training on.\n    val_net : gluon.Block\n        The model used for validation. The validation model does not necessarily belong to\n        the same model class as the training model. But the two models typically share the\n        same architecture. Therefore the validation model can reuse parameters of the\n        training model.\n\n        The code example of consruction of val_net sharing the same network parameters as\n        the training net is given below:\n\n        >>> net = _get_train_network()\n        >>> val_net = _get_test_network()\n        >>> val_net.share_parameters(net.collect_params())\n        >>> net.initialize(device=device)\n        >>> est = Estimator(net, loss, val_net=val_net)\n\n        Proper namespace match is required for weight sharing between two networks. Most networks\n        inheriting :py:class:`Block` can share their parameters correctly. An exception is\n        Sequential networks that Block scope must be specified for correct weight sharing. For\n        the  naming in mxnet Gluon API, please refer to the site\n        (https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/blocks/naming.html)\n        for future information.\n    val_loss : gluon.loss.loss\n        Loss (objective) function to calculate during validation. If set val_loss\n        None, it will use the same loss function as self.loss\n    batch_processor: BatchProcessor\n        BatchProcessor provides customized fit_batch() and evaluate_batch() methods\n    \"\"\"\n\n    logger = None\n    \"\"\"logging.Logger object associated with the Estimator.\n\n    The logger is used for all logs generated by this estimator and its\n    handlers. A new logging.Logger is created during Estimator construction and\n    configured to write all logs with level logging.INFO or higher to\n    sys.stdout.\n\n    You can modify the logging settings using the standard Python methods. For\n    example, to save logs to a file in addition to printing them to stdout\n    output, you can attach a logging.FileHandler to the logger.\n\n    >>> est = Estimator(net, loss)\n    >>> import logging\n    >>> est.logger.addHandler(logging.FileHandler(filename))\n\n    \"\"\"\n\n    def __init__(self, net,\n                 loss,\n                 train_metrics=None,\n                 val_metrics=None,\n                 initializer=None,\n                 trainer=None,\n                 device=None,\n                 val_net=None,\n                 val_loss=None,\n                 batch_processor=None):\n        self.net = net\n        self.loss = self._check_loss(loss)\n        self._train_metrics = _check_metrics(train_metrics)\n        self._val_metrics = _check_metrics(val_metrics)\n        self._add_default_training_metrics()\n        self._add_validation_metrics()\n        self.val_loss = self.loss\n        if val_loss is not None:\n            self.val_loss = self._check_loss(val_loss)\n        self.val_net = self.net\n        if val_net is not None:\n            self.val_net = val_net\n\n        self.logger = logging.Logger(name='Estimator', level=logging.INFO)\n        self.logger.addHandler(logging.StreamHandler(sys.stdout))\n\n        self.device = self._check_devices(device)\n        self._initialize(initializer)\n        self.trainer = self._check_trainer(trainer)\n        self.batch_processor = self._check_batch_processor(batch_processor)\n\n    def _check_loss(self, loss):\n        if not isinstance(loss, gluon_loss):\n            raise ValueError(\"loss must be a Loss, \"\n                             \"refer to gluon.loss.Loss:{}\".format(loss))\n        return loss\n\n    def _check_context(self, context):\n        \"\"\"This function has been deprecated. Please refer to ``Estimator._check_devices``.\"\"\"\n        warnings.warn('Estimator._check_context has been renamed to'\n                      ' Estimator._check_devices', DeprecationWarning)\n        return self._check_devices(context)\n\n    def _check_devices(self, devices):\n        # infer available devices\n        gpus = num_gpus()\n        available_gpus = [gpu(i) for i in range(gpus)]\n\n        if devices:\n            # check devices values, only accept Device or a list of Device\n            if isinstance(devices, Device):\n                devices = [devices]\n            elif isinstance(devices, list) and all([isinstance(c, Device) for c in devices]):\n                devices = devices\n            else:\n                raise ValueError(\"devices must be a Device or a list of Device, \"\n                                 \"for example mx.cpu() or [mx.gpu(0), mx.gpu(1)], \"\n                                 \"refer to mxnet.Device:{}\".format(devices))\n            for device in devices:\n                assert device in available_gpus or str(device).startswith('cpu'), \\\n                    \"{} is not available, please make sure \" \\\n                    \"your device is in one of: mx.cpu(), {}\".format(\n                        device, ', '.join([str(device) for device in available_gpus]))\n        else:\n            # provide default device\n            if gpus > 0:\n                # only use 1 GPU by default\n                if gpus > 1:\n                    warnings.warn(\"You have multiple GPUs, gpu(0) will be used by default.\"\n                                  \"To utilize all your GPUs, specify device as a list of gpus, \"\n                                  \"e.g. devices=[mx.gpu(0), mx.gpu(1)] \")\n                devices = [gpu(0)]\n            else:\n                devices = [cpu()]\n        return devices\n\n    def _check_batch_processor(self, batch_processor):\n        # check whether the batch processor contains fit_batch() and evaluate_batch() methods\n        if batch_processor is not None:\n            model_fit = getattr(batch_processor, 'fit_batch', None)\n            model_evaluate = getattr(batch_processor, 'evaluate_batch', None)\n            if not callable(model_fit) or not callable(model_evaluate):\n                raise ValueError('Customized Batch Processor must contain fit_batch()'\n                                 ' and evaluate_batch() methods')\n        else:\n            batch_processor = BatchProcessor()\n        return batch_processor\n\n    def _initialize(self, initializer):\n        # initialize the network\n        if not self._is_initialized():\n            # net is partially or not initialized,\n            # initialize with user specified initializer\n            # if initializer is None, default initializer will be used\n            # do not re-init layers already initialized\n            if initializer:\n                self.net.initialize(init=initializer, device=self.device)\n            else:\n                self.net.initialize(device=self.device)\n        elif initializer:\n            # net is fully initialized, and user passed not None initializer\n            # do not force reinitialize, give warning\n            warnings.warn(\"Network already fully initialized, skipping initialization. \"\n                          \"You don't need to pass initializer if you already \"\n                          \"initialized your net. \"\n                          \"You can use net.initialize(init=your_initializer, force_reinit=True)\"\n                          \"to force re-initialize.\")\n\n    def _check_trainer(self, trainer):\n        # handle trainer\n        if not trainer:\n            warnings.warn(\"No trainer specified, default SGD optimizer \"\n                          \"with learning rate 0.001 is used.\")\n            trainer = Trainer(self.net.collect_params(),\n                              'sgd', {'learning_rate': 0.001})\n        elif not isinstance(trainer, Trainer):\n            raise ValueError(\"Trainer must be a Gluon Trainer instance, refer to \"\n                             \"gluon.Trainer:{}\".format(trainer))\n        return trainer\n\n    def _is_initialized(self):\n        param_dict = self.net.collect_params()\n        for param in param_dict:\n            try:\n                param_dict[param].list_device()\n            except RuntimeError:\n                return False\n        return True\n\n    def _get_data_and_label(self, batch, device, batch_axis=0):\n        data = batch[0]\n        label = batch[1]\n        data = split_and_load(data, device, batch_axis=batch_axis)\n        label = split_and_load(label, device, batch_axis=batch_axis)\n        return data, label\n\n    def _add_default_training_metrics(self):\n        if not self._train_metrics:\n            suggested_metric = _suggest_metric_for_loss(self.loss)\n            if suggested_metric:\n                self._train_metrics = [suggested_metric]\n            loss_name = type(self.loss).__name__\n            self._train_metrics.append(metric_loss(loss_name))\n\n        for metric in self._train_metrics:\n            # add training prefix to the metric name\n            # it is useful for event handlers to distinguish them from validation metrics\n            metric.name = 'training ' + metric.name\n\n    def _add_validation_metrics(self):\n        if not self._val_metrics:\n            self._val_metrics = [copy.deepcopy(metric) for metric in self._train_metrics]\n\n        for metric in self._val_metrics:\n            # add validation prefix to the metric name\n            # it is useful for event handlers to distinguish them from training metrics\n            if 'training' in metric.name:\n                metric.name = metric.name.replace('training', 'validation')\n            else:\n                metric.name = 'validation ' + metric.name\n\n    @property\n    def train_metrics(self):\n        return self._train_metrics\n\n    @property\n    def val_metrics(self):\n        return self._val_metrics\n\n    def evaluate(self,\n                 val_data,\n                 batch_axis=0,\n                 event_handlers=None):\n        \"\"\"Evaluate model on validation data.\n\n        This function calls :py:func:`evaluate_batch` on each of the batches from the\n        validation data loader. Thus, for custom use cases, it's possible to inherit the\n        estimator class and override :py:func:`evaluate_batch`.\n\n        Parameters\n        ----------\n        val_data : DataLoader\n            Validation data loader with data and labels.\n        batch_axis : int, default 0\n            Batch axis to split the validation data into devices.\n        event_handlers : EventHandler or list of EventHandler\n            List of :py:class:`EventHandlers` to apply during validation. Besides\n            event handlers specified here, a default MetricHandler and a LoggingHandler\n            will be added if not specified explicitly.\n        \"\"\"\n        if not isinstance(val_data, DataLoader):\n            raise ValueError(\"Estimator only support input as Gluon DataLoader. Alternatively, you \"\n                             \"can transform your DataIter or any NDArray into Gluon DataLoader. \"\n                             \"Refer to gluon.data.DataLoader\")\n\n        for metric in self.val_metrics:\n            metric.reset()\n        estimator_ref = self\n\n        event_handlers = self._prepare_default_validation_handlers(event_handlers)\n\n        _, epoch_begin, batch_begin, batch_end, \\\n        epoch_end, _ = self._categorize_handlers(event_handlers)\n\n        estimator_ref = self\n\n        for handler in epoch_begin:\n            handler.epoch_begin(estimator_ref)\n\n        for _, batch in enumerate(val_data):\n            for handler in batch_begin:\n                handler.batch_begin(estimator_ref, batch=batch)\n\n            _, label, pred, loss = \\\n            self.batch_processor.evaluate_batch(estimator_ref, batch,\n                                                batch_axis)\n\n            for handler in batch_end:\n                handler.batch_end(estimator_ref, batch=batch, pred=pred, label=label, loss=loss)\n\n        for handler in epoch_end:\n            handler.epoch_end(estimator_ref)\n\n    def fit(self, train_data,\n            val_data=None,\n            epochs=None,\n            event_handlers=None,\n            batches=None,\n            batch_axis=0):\n        \"\"\"Trains the model with a given :py:class:`DataLoader` for a specified\n        number of epochs or batches. The batch size is inferred from the\n        data loader's batch_size.\n\n        This function calls :py:func:`fit_batch` on each of the batches from the\n        training data loader. Thus, for custom use cases, it's possible to inherit the\n        estimator class and override :py:func:`fit_batch`.\n\n        Parameters\n        ----------\n        train_data : DataLoader\n            Training data loader with data and labels.\n        val_data : DataLoader, default None\n            Validation data loader with data and labels.\n        epochs : int, default None\n            Number of epochs to iterate on the training data.\n            You can only specify one and only one type of iteration(epochs or batches).\n        event_handlers : EventHandler or list of EventHandler\n            List of :py:class:`EventHandlers` to apply during training. Besides\n            the event handlers specified here, a StoppingHandler,\n            LoggingHandler and MetricHandler will be added by default if not\n            yet specified manually. If validation data is provided, a\n            ValidationHandler is also added if not already specified.\n        batches : int, default None\n            Number of batches to iterate on the training data.\n            You can only specify one and only one type of iteration(epochs or batches).\n        batch_axis : int, default 0\n            Batch axis to split the training data into devices.\n        \"\"\"\n        if not isinstance(train_data, DataLoader):\n            raise ValueError(\"Estimator only support input as Gluon DataLoader. Alternatively, you \"\n                             \"can transform your DataIter or any NDArray into Gluon DataLoader. \"\n                             \"Refer to gluon.data.dataloader\")\n\n        # must specify one and only one of epochs or batches\n        if (not epochs) == (not batches):\n            raise ValueError(\n                \"Fit only support exactly one type of iteration, \"\n                \"train by number of epochs or number of batches.\"\n                \"Please specify one and only one of: epochs or batches.\")\n\n        self.max_epoch = epochs\n        self.max_batch = batches\n        self.batch_axis = batch_axis\n\n        # provide default handlers\n        event_handlers = self._prepare_default_handlers(val_data, event_handlers)\n\n        train_begin, epoch_begin, batch_begin, \\\n        batch_end, epoch_end, train_end = self._categorize_handlers(event_handlers)\n\n        # pass a reference to all event handlers\n        estimator_ref = self\n        # training begin\n        for handler in train_begin:\n            handler.train_begin(estimator_ref)\n\n        while True:\n            # epoch begin\n            for handler in epoch_begin:\n                handler.epoch_begin(estimator_ref)\n\n            for batch in train_data:\n                # batch begin\n                for handler in batch_begin:\n                    handler.batch_begin(estimator_ref, batch=batch)\n\n                _, label, pred, loss = self.batch_processor.fit_batch(estimator_ref,\n                                                                      batch, batch_axis)\n                # batch end\n\n                batch_end_result = []\n                for handler in batch_end:\n                    batch_end_result.append(handler.batch_end(estimator_ref, batch=batch,\n                                                              pred=pred, label=label, loss=loss))\n                # if any handler signaled to stop\n                if any(batch_end_result):\n                    break\n\n            # epoch end\n            epoch_end_result = []\n            for handler in epoch_end:\n                epoch_end_result.append(handler.epoch_end(estimator_ref))\n            # if any handler signaled to stop\n            if any(epoch_end_result):\n                break\n\n        # train end\n        for handler in train_end:\n            handler.train_end(estimator_ref)\n\n    def _prepare_default_handlers(self, val_data, event_handlers):\n        event_handlers = _check_event_handlers(event_handlers)\n        added_default_handlers = []\n\n        # no need to add to default handler check as StoppingHandler does not use metrics\n        added_default_handlers.append(StoppingHandler(self.max_epoch, self.max_batch))\n\n        if not any(isinstance(handler, GradientUpdateHandler) for handler in event_handlers):\n            added_default_handlers.append(GradientUpdateHandler())\n\n        if not any(isinstance(handler, MetricHandler) for handler in event_handlers):\n            added_default_handlers.append(MetricHandler(metrics=self.train_metrics))\n\n        if not any(isinstance(handler, ValidationHandler) for handler in event_handlers):\n            # no validation handler\n            if val_data:\n                # add default validation handler if validation data found\n                added_default_handlers.append(ValidationHandler(val_data=val_data,\n                                                                eval_fn=self.evaluate))\n\n        if not any(isinstance(handler, LoggingHandler) for handler in event_handlers):\n            added_default_handlers.append(LoggingHandler(metrics=self.train_metrics))\n\n        # if there is a mix of user defined event handlers and default event handlers\n        # they should have the same set of metrics\n        mixing_handlers = event_handlers and added_default_handlers\n\n        event_handlers.extend(added_default_handlers)\n\n        if mixing_handlers:\n            # check if all handlers have the same set of references to metrics\n            known_metrics = set(self.train_metrics + self.val_metrics)\n            for handler in event_handlers:\n                _check_handler_metric_ref(handler, known_metrics)\n\n        event_handlers.sort(key=lambda handler: getattr(handler, 'priority', 0))\n        return event_handlers\n\n    def _prepare_default_validation_handlers(self, event_handlers):\n        event_handlers = _check_event_handlers(event_handlers)\n        added_default_handlers = []\n\n        # add default logging handler and metric handler for validation\n        if not any(isinstance(handler, MetricHandler) for handler in event_handlers):\n            added_default_handlers.append(MetricHandler(metrics=self.val_metrics))\n\n        if not any(isinstance(handler, LoggingHandler) for handler in event_handlers):\n            added_default_handlers.append(LoggingHandler(metrics=self.val_metrics))\n\n        mixing_handlers = event_handlers and added_default_handlers\n        event_handlers.extend(added_default_handlers)\n\n        # check if all handlers refer to well-defined validation metrics\n        if mixing_handlers:\n            known_metrics = set(self.val_metrics)\n            for handler in event_handlers:\n                _check_handler_metric_ref(handler, known_metrics)\n\n        event_handlers.sort(key=lambda handler: getattr(handler, 'priority', 0))\n        return event_handlers\n\n    def _categorize_handlers(self, event_handlers):\n        \"\"\"\n        categorize handlers into 6 event lists to avoid calling empty methods\n        for example, only event handlers with train_begin method\n        implemented will be called at train begin\n        \"\"\"\n\n        train_begin = []\n        epoch_begin = []\n        batch_begin = []\n        batch_end = []\n        epoch_end = []\n        train_end = []\n        for handler in event_handlers:\n            if isinstance(handler, TrainBegin):\n                train_begin.append(handler)\n            if isinstance(handler, EpochBegin):\n                epoch_begin.append(handler)\n            if isinstance(handler, BatchBegin):\n                batch_begin.append(handler)\n            if isinstance(handler, BatchEnd):\n                batch_end.append(handler)\n            if isinstance(handler, EpochEnd):\n                epoch_end.append(handler)\n            if isinstance(handler, TrainEnd):\n                train_end.append(handler)\n        return train_begin, epoch_begin, batch_begin, batch_end, epoch_end, train_end\n"
  },
  {
    "path": "python/mxnet/gluon/contrib/estimator/event_handler.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import, unused-argument, too-many-ancestors\n\"\"\"Gluon EventHandlers for Estimators\"\"\"\n\nimport os\nimport time\nimport warnings\n\nimport numpy as np\n\nfrom ...metric import CompositeEvalMetric, EvalMetric\nfrom ...metric import Loss as metric_loss\nfrom .utils import _check_metrics\n\n__all__ = ['TrainBegin', 'TrainEnd', 'EpochBegin', 'EpochEnd', 'BatchBegin', 'BatchEnd',\n           'StoppingHandler', 'MetricHandler', 'ValidationHandler',\n           'LoggingHandler', 'CheckpointHandler', 'EarlyStoppingHandler', 'GradientUpdateHandler']\n\n\nclass EventHandler(object):\n    pass\n\n\ndef _check_event_handlers(handlers):\n    if isinstance(handlers, EventHandler):\n        handlers = [handlers]\n    else:\n        handlers = handlers or []\n        if not all([isinstance(handler, EventHandler) for handler in handlers]):\n            raise ValueError(\"handlers must be an EventHandler or a list of EventHandler, \"\n                             \"got: {}\".format(handlers))\n    return handlers\n\n\nclass TrainBegin(EventHandler):\n    def train_begin(self, estimator, *args, **kwargs):\n        pass\n\n\nclass TrainEnd(EventHandler):\n    def train_end(self, estimator, *args, **kwargs):\n        pass\n\n\nclass EpochBegin(EventHandler):\n    def epoch_begin(self, estimator, *args, **kwargs):\n        pass\n\n\nclass EpochEnd(EventHandler):\n    def epoch_end(self, estimator, *args, **kwargs):\n        return False\n\n\nclass BatchBegin(EventHandler):\n    def batch_begin(self, estimator, *args, **kwargs):\n        pass\n\n\nclass BatchEnd(EventHandler):\n    def batch_end(self, estimator, *args, **kwargs):\n        return False\n\n\nclass StoppingHandler(TrainBegin, BatchEnd, EpochEnd):\n    \"\"\"Stop conditions to stop training\n    Stop training if maximum number of batches or epochs\n    reached.\n\n    Parameters\n    ----------\n    max_epoch : int, default None\n        Number of maximum epochs to train.\n    max_batch : int, default None\n        Number of maximum batches to train.\n\n    \"\"\"\n\n    def __init__(self, max_epoch=None, max_batch=None):\n        self.max_epoch = max_epoch\n        self.max_batch = max_batch\n        self.current_batch = 0\n        self.current_epoch = 0\n        self.stop_training = False\n\n    def train_begin(self, estimator, *args, **kwargs):\n        self.max_epoch = estimator.max_epoch\n        self.max_batch = estimator.max_batch\n        self.current_batch = 0\n        self.current_epoch = 0\n\n    def batch_end(self, estimator, *args, **kwargs):\n        self.current_batch += 1\n        if self.current_batch == self.max_batch:\n            self.stop_training = True\n        return self.stop_training\n\n    def epoch_end(self, estimator, *args, **kwargs):\n        self.current_epoch += 1\n        if self.current_epoch == self.max_epoch:\n            self.stop_training = True\n        return self.stop_training\n\n\nclass MetricHandler(EpochBegin, BatchEnd):\n    \"\"\"Metric Handler that update metric values at batch end\n\n    :py:class:`MetricHandler` takes model predictions and true labels\n    and update the metrics, it also update metric wrapper for loss with loss values.\n    Validation loss and metrics will be handled by :py:class:`ValidationHandler`\n\n    Parameters\n    ----------\n    metrics : List of EvalMetrics\n        Metrics to be updated at batch end.\n    priority : scalar\n        Priority level of the MetricHandler. Priority level is sorted in ascending\n        order. The lower the number is, the higher priority level the handler is.\n    \"\"\"\n\n    def __init__(self, metrics, priority=-1000):\n        self.metrics = _check_metrics(metrics)\n        # order to be called among all callbacks\n        # metrics need to be calculated before other callbacks can access them\n        self.priority = priority\n\n    def epoch_begin(self, estimator, *args, **kwargs):\n        for metric in self.metrics:\n            metric.reset()\n\n    def batch_end(self, estimator, *args, **kwargs):\n        pred = kwargs['pred']\n        label = kwargs['label']\n        loss = kwargs['loss']\n        for metric in self.metrics:\n            if isinstance(metric, metric_loss):\n                # metric wrapper for loss values\n                metric.update(0, loss)\n            else:\n                metric.update(label, pred)\n\n\nclass ValidationHandler(TrainBegin, BatchEnd, EpochEnd):\n    \"\"\"Validation Handler that evaluate model on validation dataset\n\n    :py:class:`ValidationHandler` takes validation dataset, an evaluation function,\n    metrics to be evaluated, and how often to run the validation. You can provide custom\n    evaluation function or use the one provided my :py:class:`Estimator`\n\n    Parameters\n    ----------\n    val_data : DataLoader\n        Validation data set to run evaluation.\n    eval_fn : function\n        A function defines how to run evaluation and\n        calculate loss and metrics.\n    epoch_period : int, default 1\n        How often to run validation at epoch end, by default\n        :py:class:`ValidationHandler` validate every epoch.\n    batch_period : int, default None\n        How often to run validation at batch end, by default\n        :py:class:`ValidationHandler` does not validate at batch end.\n    priority: scalar, default -1000\n        Priority level of the ValidationHandler. Priority level is sorted in\n        ascending order. The lower the number is, the higher priority level the\n        handler is.\n    event_handlers : EventHandler or list of EventHandlers\n        List of :py:class:`EventHandler` to apply during validaiton. This argument\n        is used by self.eval_fn function in order to process customized event\n        handlers.\n    \"\"\"\n\n    def __init__(self,\n                 val_data,\n                 eval_fn,\n                 epoch_period=1,\n                 batch_period=None,\n                 priority=-1000,\n                 event_handlers=None):\n        self.val_data = val_data\n        self.eval_fn = eval_fn\n        self.epoch_period = epoch_period\n        self.batch_period = batch_period\n        self.current_batch = 0\n        self.current_epoch = 0\n        # order to be called among all callbacks\n        # validation metrics need to be calculated before other callbacks can access them\n        self.priority = priority\n        self.event_handlers = event_handlers\n\n    def train_begin(self, estimator, *args, **kwargs):\n        # reset epoch and batch counter\n        self.current_batch = 0\n        self.current_epoch = 0\n\n    def batch_end(self, estimator, *args, **kwargs):\n        self.current_batch += 1\n        if self.batch_period and self.current_batch % self.batch_period == 0:\n            self.eval_fn(val_data=self.val_data, batch_axis=estimator.batch_axis,\n                         event_handlers=self.event_handlers)\n\n    def epoch_end(self, estimator, *args, **kwargs):\n        self.current_epoch += 1\n        if self.epoch_period and self.current_epoch % self.epoch_period == 0:\n            self.eval_fn(val_data=self.val_data, batch_axis=estimator.batch_axis,\n                         event_handlers=self.event_handlers)\n\n\nclass LoggingHandler(TrainBegin, TrainEnd, EpochBegin, EpochEnd, BatchBegin, BatchEnd):\n    \"\"\"Basic Logging Handler that applies to every Gluon estimator by default.\n\n    :py:class:`LoggingHandler` logs hyper-parameters, training statistics,\n    and other useful information during training\n\n    Parameters\n    ----------\n    log_interval: int or str, default 'epoch'\n        Logging interval during training.\n        log_interval='epoch': display metrics every epoch\n        log_interval=integer k: display metrics every interval of k batches\n    metrics : list of EvalMetrics\n        Metrics to be logged, logged at batch end, epoch end, train end.\n    priority : scalar, default np.Inf\n        Priority level of the LoggingHandler. Priority level is sorted in\n        ascending order. The lower the number is, the higher priority level the\n        handler is.\n    \"\"\"\n\n    def __init__(self, log_interval='epoch',\n                 metrics=None,\n                 priority=np.Inf):\n        super(LoggingHandler, self).__init__()\n        if not isinstance(log_interval, int) and log_interval != 'epoch':\n            raise ValueError(\"log_interval must be either an integer or string 'epoch'\")\n        self.metrics = _check_metrics(metrics)\n        self.batch_index = 0\n        self.current_epoch = 0\n        self.processed_samples = 0\n        # logging handler need to be called at last to make sure all states are updated\n        # it will also shut down logging at train end\n        self.priority = priority\n        self.log_interval = log_interval\n        self.log_interval_time = 0\n\n    def train_begin(self, estimator, *args, **kwargs):\n        self.train_start = time.time()\n        trainer = estimator.trainer\n        optimizer = trainer.optimizer.__class__.__name__\n        lr = trainer.learning_rate\n        estimator.logger.info(\"Training begin: using optimizer %s \"\n                              \"with current learning rate %.4f \",\n                              optimizer, lr)\n        if estimator.max_epoch:\n            estimator.logger.info(\"Train for %d epochs.\", estimator.max_epoch)\n        else:\n            estimator.logger.info(\"Train for %d batches.\", estimator.max_batch)\n        # reset all counters\n        self.current_epoch = 0\n        self.batch_index = 0\n        self.processed_samples = 0\n        self.log_interval_time = 0\n\n    def train_end(self, estimator, *args, **kwargs):\n        train_time = time.time() - self.train_start\n        msg = f'Train finished using total {train_time}s with {self.current_epoch} epochs. '\n        # log every result in train stats including train/validation loss & metrics\n        for metric in self.metrics:\n            name, value = metric.get()\n            msg += f'{name}: {value:.4f}, '\n        estimator.logger.info(msg.rstrip(', '))\n\n    def batch_begin(self, estimator, *args, **kwargs):\n        if isinstance(self.log_interval, int):\n            self.batch_start = time.time()\n\n    def batch_end(self, estimator, *args, **kwargs):\n        if isinstance(self.log_interval, int):\n            batch_time = time.time() - self.batch_start\n            msg = f'[Epoch {self.current_epoch}][Batch {self.batch_index}]'\n            self.processed_samples += kwargs['batch'][0].shape[0]\n            msg += f'[Samples {self.processed_samples}] '\n            self.log_interval_time += batch_time\n            if self.batch_index % self.log_interval == 0:\n                msg += f'time/interval: {self.log_interval_time:.3f}s '\n                self.log_interval_time = 0\n                for metric in self.metrics:\n                    # only log current training loss & metric after each interval\n                    name, value = metric.get()\n                    msg += f'{name}: {value:.4f}, '\n                estimator.logger.info(msg.rstrip(', '))\n        self.batch_index += 1\n\n    def epoch_begin(self, estimator, *args, **kwargs):\n        if isinstance(self.log_interval, int) or self.log_interval == 'epoch':\n            is_training = False\n            # use the name hack defined in __init__() of estimator class\n            for metric in self.metrics:\n                if 'training' in metric.name:\n                    is_training = True\n            self.epoch_start = time.time()\n            if is_training:\n                estimator.logger.info(\"[Epoch %d] Begin, current learning rate: %.4f\",\n                                      self.current_epoch, estimator.trainer.learning_rate)\n            else:\n                estimator.logger.info(\"Validation Begin\")\n\n    def epoch_end(self, estimator, *args, **kwargs):\n        if isinstance(self.log_interval, int) or self.log_interval == 'epoch':\n            epoch_time = time.time() - self.epoch_start\n            msg = f'[Epoch {self.current_epoch}] Finished in {epoch_time:.3f}s, '\n            for monitor in self.metrics:\n                name, value = monitor.get()\n                msg += f'{name}: {value:.4f}, '\n            estimator.logger.info(msg.rstrip(', '))\n        self.current_epoch += 1\n        self.batch_index = 0\n\n\nclass CheckpointHandler(TrainBegin, BatchEnd, EpochEnd):\n    \"\"\"Save the model after user define period\n\n    :py:class:`CheckpointHandler` saves the network architecture after first batch if the model\n    can be fully hybridized, saves model parameters and trainer states after user defined period,\n    default saves every epoch.\n\n    Parameters\n    ----------\n    model_dir : str\n        File directory to save all the model related files including model architecture,\n        model parameters, and trainer states.\n    model_prefix : str default 'model'\n        Prefix to add for all checkpoint file names.\n    monitor: EvalMetric, default None\n        The metrics to monitor and determine if model has improved\n    verbose: int, default 0\n        Verbosity mode, 1 means inform user every time a checkpoint is saved\n    save_best: bool, default False\n        If True, monitor must not be None, :py:class:`CheckpointHandler` will save the\n        model parameters and trainer states with the best monitored value.\n    mode: str, default 'auto'\n        One of {auto, min, max}, if `save_best=True`, the comparison to make\n        and determine if the monitored value has improved. if 'auto' mode,\n        :py:class:`CheckpointHandler` will try to use min or max based on\n        the monitored metric name.\n    epoch_period: int, default 1\n        Epoch intervals between saving the network. By default, checkpoints are\n        saved every epoch.\n    batch_period: int, default None\n        Batch intervals between saving the network.\n        By default, checkpoints are not saved based on the number of batches.\n    max_checkpoints : int, default 5\n        Maximum number of checkpoint files to keep in the model_dir, older checkpoints\n        will be removed. Best checkpoint file is not counted.\n    resume_from_checkpoint : bool, default False\n        Whether to resume training from checkpoint in model_dir. If True and checkpoints\n        found, :py:class:`CheckpointHandler` will load net parameters and trainer states,\n        and train the remaining of epochs and batches.\n    \"\"\"\n\n    def __init__(self,\n                 model_dir,\n                 model_prefix='model',\n                 monitor=None,\n                 verbose=0,\n                 save_best=False,\n                 mode='auto',\n                 epoch_period=1,\n                 batch_period=None,\n                 max_checkpoints=5,\n                 resume_from_checkpoint=False):\n        self.monitor = monitor\n        self.verbose = verbose\n        if not os.path.exists(model_dir):\n            os.makedirs(model_dir)\n        self.model_dir = model_dir\n        self.model_prefix = model_prefix\n        self.save_best = save_best\n        if self.save_best and not isinstance(self.monitor, EvalMetric):\n            raise ValueError(\"To save best model only, please provide one of the metric objects \"\n                             \"from estimator.train_metrics and estimator.val_metrics as monitor.\")\n        self.epoch_period = epoch_period\n        self.batch_period = batch_period\n        self.current_batch = 0\n        self.current_epoch = 0\n        self.max_checkpoints = max_checkpoints\n        self.resume_from_checkpoint = resume_from_checkpoint\n        self.saved_checkpoints = []\n        if self.save_best:\n            if mode not in ['auto', 'min', 'max']:\n                warnings.warn(f'ModelCheckpoint mode {mode} is unknown, '\n                              'fallback to auto mode. CheckpointHandler will use'\n                              'max mode for f1 and accuracy metric comparison and '\n                              'use min mode other wise',\n                              RuntimeWarning)\n                mode = 'auto'\n\n            if mode == 'min':\n                self.monitor_op = np.less\n                self.best = np.Inf\n            elif mode == 'max':\n                self.monitor_op = np.greater\n                self.best = -np.Inf\n            else:\n                # use greater for accuracy and f1 and less otherwise\n                if 'acc' or 'f1' in self.monitor.get()[0].lower():\n                    warnings.warn(\"`greater` operator will be used to determine if {} has improved. \"\n                                  \"Please specify `mode='min'` to use the `less` operator. \"\n                                  \"Specify `mode='max' to disable this warning.`\"\n                                  .format(self.monitor.get()[0]))\n                    self.monitor_op = np.greater\n                else:\n                    warnings.warn(\"`less` operator will be used to determine if {} has improved. \"\n                                  \"Please specify `mode='max'` to use the `greater` operator. \"\n                                  \"Specify `mode='min' to disable this warning.`\"\n                                  .format(self.monitor.get()[0]))\n                    self.monitor_op = np.less\n\n    def train_begin(self, estimator, *args, **kwargs):\n        # reset all counters\n        self.current_epoch = 0\n        self.current_batch = 0\n        if self.save_best:\n            self.best = np.Inf if self.monitor_op == np.less else -np.Inf  # pylint: disable=comparison-with-callable\n        if self.resume_from_checkpoint:\n            error_msg = \"To use resume from checkpoint, you must only specify \" \\\n                        \"the same type of period you used for training.\" \\\n                        \"For example, if you are training based on number of epochs,\" \\\n                        \"you must save only based on epochs, and set batch_period to None.\"\n            if estimator.max_batch:\n                assert self.batch_period, error_msg\n                assert not self.epoch_period, error_msg\n            if estimator.max_epoch:\n                assert self.epoch_period, error_msg\n                assert not self.batch_period, error_msg\n\n            self._resume_from_checkpoint(estimator)\n\n    def batch_end(self, estimator, *args, **kwargs):\n        # only save symbol once after first batch\n        if self.current_batch == 0:\n            self._save_symbol(estimator)\n        if self.batch_period and (self.current_batch + 1) % self.batch_period == 0:\n            self._save_checkpoint(estimator)\n        self.current_batch += 1\n\n    def epoch_end(self, estimator, *args, **kwargs):\n        if self.epoch_period and (self.current_epoch + 1) % self.epoch_period == 0:\n            self._save_checkpoint(estimator)\n        self.current_epoch += 1\n\n    def _save_checkpoint(self, estimator):\n        # if resumed from checkpoint, increment checkpoint number\n        if self.resume_from_checkpoint:\n            save_epoch_number = self.current_epoch + self.trained_epoch + 1\n            if estimator.max_epoch:\n                # checkpoint saved at epoch end, batch number already incremented\n                save_batch_number = self.current_batch + self.trained_batch\n            else:\n                save_batch_number = self.current_batch + self.trained_batch + 1\n        else:\n            save_epoch_number = self.current_epoch\n            save_batch_number = self.current_batch\n        prefix = f\"{self.model_prefix}-epoch{save_epoch_number}batch{save_batch_number}\"\n        self._save_params_and_trainer(estimator, prefix)\n        if self.verbose > 0:\n            estimator.logger.info(f'[Epoch {self.current_epoch}] CheckpointHandler: trained total {self.current_batch + 1} batches, '\n                                  f'saving model at {self.model_dir} with prefix: {prefix}')\n\n        if self.save_best:\n            monitor_name, monitor_value = self.monitor.get()\n            # check if monitor exists in train stats\n            if np.isnan(monitor_value):\n                warnings.warn(RuntimeWarning(\n                    'Skipping save best because %s is not updated, make sure you pass one of the '\n                    'metric objects estimator.train_metrics and estimator.val_metrics as monitor',\n                    monitor_name))\n            else:\n                if self.monitor_op(monitor_value, self.best):\n                    prefix = self.model_prefix + '-best'\n                    self._save_params_and_trainer(estimator, prefix)\n                    if self.verbose > 0:\n                        estimator.logger.info('[Epoch %d] CheckpointHandler: '\n                                              '%s improved from %0.5f to %0.5f, '\n                                              'updating best model at %s with prefix: %s',\n                                              self.current_epoch, monitor_name,\n                                              self.best, monitor_value, self.model_dir, prefix)\n                    self.best = monitor_value\n                else:\n                    if self.verbose > 0:\n                        estimator.logger.info('[Epoch %d] CheckpointHandler: '\n                                              '%s did not improve from %0.5f, '\n                                              'skipping updating best model',\n                                              self.current_batch, monitor_name,\n                                              self.best)\n\n    def _save_symbol(self, estimator):\n        symbol_file = os.path.join(self.model_dir, self.model_prefix + '-symbol.json')\n        if hasattr(estimator.net, '_cached_graph') and estimator.net._cached_graph:\n            sym = estimator.net._cached_graph[1]\n            sym.save(symbol_file)\n        else:\n            estimator.logger.info(\n                \"Model architecture(symbol file) is not saved, please use HybridBlock \"\n                \"to construct your model, and call net.hybridize() before passing to \"\n                \"Estimator in order to save model architecture as %s.\",\n                symbol_file)\n\n    def _save_params_and_trainer(self, estimator, file_prefix):\n        param_file = os.path.join(self.model_dir, file_prefix + '.params')\n        trainer_file = os.path.join(self.model_dir, file_prefix + '.states')\n        estimator.net.save_parameters(param_file)\n        estimator.trainer.save_states(trainer_file)\n\n        # only count checkpoints with epoch or batch number in file name\n        if 'best' not in file_prefix:\n            self.saved_checkpoints.append(file_prefix)\n        # remove old checkpoint when max number of checkpoints reached\n        if len(self.saved_checkpoints) > self.max_checkpoints:\n            prefix = self.saved_checkpoints.pop(0)\n            for fname in os.listdir(self.model_dir):\n                if fname.startswith(prefix):\n                    os.remove(os.path.join(self.model_dir, fname))\n\n    def _resume_from_checkpoint(self, estimator):\n        prefix = self.model_prefix + '-epoch'\n        self.trained_epoch = self._find_max_iteration(\n            dir=self.model_dir,\n            prefix=prefix,\n            start='epoch',\n            end='batch',\n            saved_checkpoints=self.saved_checkpoints)\n        prefix += str(self.trained_epoch)\n        self.trained_batch = self._find_max_iteration(\n            dir=self.model_dir,\n            prefix=prefix,\n            start='batch',\n            end='.params')\n\n        if self.trained_epoch == -1:\n            msg = \"CheckpointHandler: No checkpoint found, training from scratch for \"\n            if estimator.max_batch:\n                msg += f\"{estimator.max_batch} batches\"\n            else:\n                msg += f\"{estimator.max_epoch} epochs\"\n            estimator.logger.info(msg)\n        else:\n            msg = f\"CheckpointHandler: Checkpoint resumed from epoch {self.trained_epoch} batch {self.trained_batch}, \" \\\n                  \"continue to train for \"\n            # change maximum number of epoch or batch to train if resumed from epoch checkpoint\n            if estimator.max_epoch:\n                if self.trained_epoch >= estimator.max_epoch - 1:\n                    raise ValueError(f\"Found checkpoint with maximum number of epoch {estimator.max_epoch} reached, please specify \"\n                                     \"resume_from_checkpoint=False (default value) if you wan to train from scratch.\")\n                estimator.max_epoch = estimator.max_epoch - self.trained_epoch - 1\n                msg += f\"{estimator.max_epoch} epochs \"\n            if estimator.max_batch:\n                if self.trained_batch >= estimator.max_batch - 1:\n                    raise ValueError(f\"Found checkpoint with maximum number of batch {self.trained_batch} reached, please specify\"\n                                     \"resume_from_checkpoint=False (default value) if you wan to train from scratch.\")\n                estimator.max_batch = estimator.max_batch - self.trained_batch - 1\n                msg += f\"{estimator.max_batch} batches \"\n            # load checkpoint\n            param_file = \"{}-epoch{}batch{}.params\".format(self.model_prefix, self.trained_epoch, self.trained_batch)\n            param_file = os.path.join(self.model_dir, param_file)\n            trainer_file = \"{}-epoch{}batch{}.states\".format(self.model_prefix, self.trained_epoch, self.trained_batch)\n            trainer_file = os.path.join(self.model_dir, trainer_file)\n            assert os.path.exists(param_file), f\"Failed to load checkpoint, {param_file} does not exist\"\n            assert os.path.exists(trainer_file), f\"Failed to load checkpoint, {trainer_file} does not exist\"\n            estimator.net.load_parameters(param_file, ctx=estimator.device)\n            estimator.trainer.load_states(trainer_file)\n            estimator.logger.warning(msg)\n\n    def _find_max_iteration(self, dir, prefix, start, end, saved_checkpoints=None):\n        error_msg = \"Error parsing checkpoint file, please check your \" \\\n                    \"checkpoints have the format: \" \\\n                    \"{model_name}-epoch{epoch_number}batch{batch_number}.params, \" \\\n                    \"there should also be a .states file for each .params file \"\n        max_iter = -1\n        for fname in os.listdir(dir):\n            if fname.startswith(prefix) and '.params' in fname:\n                if saved_checkpoints:\n                    # save prefix of existing checkpoints\n                    saved_checkpoints.append(fname[:fname.find('.params')])\n                try:\n                    # find trained number of epoch\n                    iter = int(fname[fname.find(start) + len(start): fname.find(end)])\n                    if iter > max_iter:\n                        max_iter = iter\n                except ValueError:\n                    raise ValueError(error_msg)\n        return max_iter\n\n\nclass EarlyStoppingHandler(TrainBegin, EpochEnd, TrainEnd):\n    \"\"\"Early stop training if monitored value is not improving\n\n    Parameters\n    ----------\n    monitor: EvalMetric\n        The metric to monitor, and stop training if this metric does not improve.\n    min_delta: float, default 0\n        Minimal change in monitored value to be considered as an improvement.\n    patience: int, default 0\n        Number of epochs to wait for improvement before terminate training.\n    mode: str, default 'auto'\n        One of {auto, min, max}, if `save_best_only=True`, the comparison to make\n        and determine if the monitored value has improved. if 'auto' mode, checkpoint\n        handler will try to use min or max based on the monitored metric name.\n    baseline: float\n        Baseline value to compare the monitored value with.\n    \"\"\"\n\n    def __init__(self,\n                 monitor,\n                 min_delta=0,\n                 patience=0,\n                 mode='auto',\n                 baseline=None):\n        super(EarlyStoppingHandler, self).__init__()\n\n        if not isinstance(monitor, EvalMetric):\n            raise ValueError(\n                \"Please provide one of the metric objects from estimator.train_metrics and \"\n                \"estimator.val_metrics as monitor.\")\n        if isinstance(monitor, CompositeEvalMetric):\n            raise ValueError(\"CompositeEvalMetric is not supported for EarlyStoppingHandler, \"\n                             \"please specify a simple metric instead.\")\n        self.monitor = monitor\n        self.baseline = baseline\n        self.patience = patience\n        self.min_delta = min_delta\n        self.wait = 0\n        self.stopped_epoch = 0\n        self.current_epoch = 0\n        self.stop_training = False\n\n        if mode not in ['auto', 'min', 'max']:\n            warnings.warn(f'EarlyStopping mode {mode} is unknown, '\n                          'fallback to auto mode. CheckpointHandler will use'\n                          'max mode for f1 and accuracy metric comparison and '\n                          'use min mode other wise',\n                          RuntimeWarning)\n            mode = 'auto'\n\n        if mode == 'min':\n            self.monitor_op = np.less\n        elif mode == 'max':\n            self.monitor_op = np.greater\n        else:\n            if 'acc' or 'f1' in self.monitor.get()[0].lower():\n                warnings.warn(\"`greater` operator will be used to determine if {} has improved. \"\n                              \"Please specify `mode='min'` to use the `less` operator. \"\n                              \"Specify `mode='max' to disable this warning.`\"\n                              .format(self.monitor.get()[0]))\n                self.monitor_op = np.greater\n            else:\n                warnings.warn(\"`less` operator will be used to determine if {} has improved. \"\n                              \"Please specify `mode='max'` to use the `greater` operator. \"\n                              \"Specify `mode='min' to disable this warning.`\"\n                              .format(self.monitor.get()[0]))\n                self.monitor_op = np.less\n\n        if self.monitor_op == np.greater:  # pylint: disable=comparison-with-callable\n            self.min_delta *= 1\n        else:\n            self.min_delta *= -1\n\n    def train_begin(self, estimator, *args, **kwargs):\n        self.wait = 0\n        self.stopped_epoch = 0\n        self.current_epoch = 0\n        self.stop_training = False\n        if self.baseline is not None:\n            self.best = self.baseline\n        else:\n            self.best = np.Inf if self.monitor_op == np.less else -np.Inf  # pylint: disable=comparison-with-callable\n\n    def epoch_end(self, estimator, *args, **kwargs):\n        monitor_name, monitor_value = self.monitor.get()\n        if np.isnan(monitor_value):\n            warnings.warn(RuntimeWarning(\n                '%s is not updated, make sure you pass one of the metric objects from'\n                'estimator.train_metrics and estimator.val_metrics as monitor.', monitor_name))\n        else:\n            if self.monitor_op(monitor_value - self.min_delta, self.best):\n                self.best = monitor_value\n                self.wait = 0\n            else:\n                self.wait += 1\n                if self.wait >= self.patience:\n                    self.stopped_epoch = self.current_epoch\n                    self.stop_training = True\n        self.current_epoch += 1\n        return self.stop_training\n\n    def train_end(self, estimator, *args, **kwargs):\n        if self.stopped_epoch > 0:\n            estimator.logger.info('[Epoch %d] EarlyStoppingHanlder: '\n                                  'early stopping due to %s not improving',\n                                  self.stopped_epoch, self.monitor.get()[0])\n\nclass GradientUpdateHandler(BatchEnd):\n    \"\"\"Gradient Update Handler that apply gradients on network weights\n\n    :py:class:`GradientUpdateHandler` takes the priority level. It updates weight parameters\n    at the end of each batch\n\n    Parameters\n    ----------\n    priority : scalar, default -2000\n        priority level of the gradient update handler. Priority level is sorted in ascending\n        order. The lower the number is, the higher priority level the handler is.\n    \"\"\"\n    def __init__(self, priority=-2000):\n        self.priority = priority\n\n    def batch_end(self, estimator, *args, **kwargs):\n        loss = kwargs['loss']\n        batch_size = 0\n        if not isinstance(loss, list):\n            loss = [loss]\n        if isinstance(loss, list):\n            for l in loss:\n                batch_size += l.shape[0]\n\n        estimator.trainer.step(batch_size)\n"
  },
  {
    "path": "python/mxnet/gluon/contrib/estimator/utils.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import, unused-variable\n\"\"\"Gluon Estimator Utility Functions\"\"\"\n\nfrom ...loss import SoftmaxCrossEntropyLoss\nfrom ...metric import Accuracy, EvalMetric, CompositeEvalMetric\n\ndef _check_metrics(metrics):\n    if isinstance(metrics, CompositeEvalMetric):\n        metrics = [m for metric in metrics.metrics for m in _check_metrics(metric)]\n    elif isinstance(metrics, EvalMetric):\n        metrics = [metrics]\n    else:\n        metrics = metrics or []\n        if not all([isinstance(metric, EvalMetric) for metric in metrics]):\n            raise ValueError(\"metrics must be a Metric or a list of Metric, \"\n                             \"refer to mxnet.gluon.metric.EvalMetric: {}\".format(metrics))\n    return metrics\n\ndef _check_handler_metric_ref(handler, known_metrics):\n    for attribute in dir(handler):\n        if any(keyword in attribute for keyword in ['metric' or 'monitor']):\n            reference = getattr(handler, attribute)\n            if not reference:\n                continue\n            elif isinstance(reference, list):\n                for metric in reference:\n                    _check_metric_known(handler, metric, known_metrics)\n            else:\n                _check_metric_known(handler, reference, known_metrics)\n\ndef _check_metric_known(handler, metric, known_metrics):\n    if metric not in known_metrics:\n        raise ValueError(\n            'Event handler {} refers to a metric instance {} outside of '\n            'the known training and validation metrics. Please use the metrics from '\n            'estimator.train_metrics and estimator.val_metrics '\n            'instead.'.format(type(handler).__name__,\n                              metric))\n\ndef _suggest_metric_for_loss(loss):\n    if isinstance(loss, SoftmaxCrossEntropyLoss):\n        return Accuracy()\n    return None\n"
  },
  {
    "path": "python/mxnet/gluon/data/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Dataset utilities.\"\"\"\n\nfrom .dataset import *\n\nfrom .sampler import *\n\nfrom .dataloader import *\n\nfrom . import vision\n\nfrom . import _internal\n"
  },
  {
    "path": "python/mxnet/gluon/data/_internal.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"C++ Datasets for common data formats.\"\"\"\nimport sys\nimport ctypes\n\nfrom .dataset import Dataset\nfrom .sampler import Sampler\nfrom ...base import _LIB\nfrom ...base import c_str_array, mx_uint, py_str\nfrom ...base import DatasetHandle, NDArrayHandle, BatchifyFunctionhandle\nfrom ...base import check_call, build_param_doc as _build_param_doc\nfrom ...ndarray import NDArray\nfrom ...ndarray import _ndarray_cls\nfrom ...numpy.multiarray import _np_ndarray_cls\nfrom ...util import is_np_array, default_array\nfrom ...io import io as _io\n\n\nclass MXDataset(Dataset):\n    \"\"\"A python wrapper a C++ dataset.\n\n    Parameters\n    ----------\n    handle : DatasetHandle, required\n        The handle to the underlying C++ Dataset.\n\n    \"\"\"\n    def __init__(self, handle, **kwargs):\n        super(MXDataset, self).__init__()\n        self.handle = handle\n        self._kwargs = kwargs\n        # get dataset size\n        length = ctypes.c_uint64(0)\n        check_call(_LIB.MXDatasetGetLen(self.handle, ctypes.byref(length)))\n        self._len = length.value\n\n    def __del__(self):\n        check_call(_LIB.MXDatasetFree(self.handle))\n\n    def __len__(self):\n        return self._len\n\n    def __getitem__(self, idx):\n        orig_idx = idx\n        if idx < 0:\n            idx += self._len\n        # check bound\n        if idx < 0 or idx >= self._len:\n            raise IndexError(\"Index {} out of bound: (0, {})\".format(orig_idx, self._len))\n        create_ndarray_fn = _np_ndarray_cls if is_np_array() else _ndarray_cls\n        output_vars = ctypes.POINTER(NDArrayHandle)()\n        num_output = ctypes.c_int(0)\n        check_call(_LIB.MXDatasetGetItems(self.handle,\n                                          ctypes.c_uint64(idx),\n                                          ctypes.byref(num_output),\n                                          ctypes.byref(output_vars)))\n        out = [create_ndarray_fn(ctypes.cast(output_vars[i], NDArrayHandle),\n                                 False) for i in range(num_output.value)]\n        for i in range(num_output.value):\n            if out[i].size == 1:\n                out[i] = out[i].asnumpy()\n        if len(out) > 1:\n            return tuple(out)\n        return out[0]\n\n\nclass MXSampler(Sampler):\n    \"\"\"MXNet internal sampler implemented in c++.\n\n    Parameters\n    ----------\n    name : str\n        Name of the sampler.\n\n    \"\"\"\n    def __init__(self, name, **kwargs):\n        try:\n            creator = getattr(_io, name)\n        except AttributeError:\n            raise ValueError('{} is not a valid MXDataIter class'.format(name))\n        self._iter = creator(**kwargs)\n\n    def __len__(self):\n        try:\n            size = len(self._iter)\n        except TypeError:\n            raise TypeError('Iterator {} does not provide length info'.format(self._iter))\n        return size\n\n    def __iter__(self):\n        for item in self._iter:\n            ret = item.data[0].asnumpy().flatten().tolist()\n            pad = item.pad\n            if pad > 0:\n                # remove padded values\n                ret = ret[:-pad]\n            elif len(ret) == 1:\n                ret = ret[0]\n            yield ret\n        self._iter.reset()\n\n\nclass MXBatchifyFunction(object):\n    \"\"\"MXNet batchify function implemented in C++.\n\n    Parameters\n    ----------\n    handle : ctypes.c_void\n        Object handle.\n\n    \"\"\"\n    def __init__(self, handle, **kwargs):\n        self._kwargs = kwargs\n        self.handle = handle\n\n    def __del__(self):\n        if self.handle is not None:\n            check_call(_LIB.MXBatchifyFunctionFree(self.handle))\n\n    def __getstate__(self):\n        \"\"\"Override pickling behavior.\"\"\"\n        # pickling pointer is not allowed\n        d = dict({'creator_name': self._kwargs['creator_name'],\n                  '_kwargs': self._kwargs})\n        return d\n\n    def __setstate__(self, d):\n        \"\"\"Restore from pickled.\"\"\"\n        creator = d['_kwargs']['creator_name']\n        d['_kwargs'].pop('creator_name')\n        other = getattr(sys.modules[__name__], creator)(**d['_kwargs'])\n        self.handle = other.handle\n        self._kwargs = other._kwargs\n        other.handle = None\n\n    def __call__(self, data, num_out=1):\n        if isinstance(data[0], NDArray):\n            create_ndarray_fn = _np_ndarray_cls if is_np_array() else _ndarray_cls\n            num_output = ctypes.c_int(num_out)\n            input_arrs = (NDArrayHandle * len(data))()\n            for i, d in enumerate(data):\n                input_arrs[i] = d.handle\n            input_vars = ctypes.cast(input_arrs, ctypes.POINTER(NDArrayHandle))\n            batch_size = ctypes.c_int(len(data) // num_output.value)\n            output_vars = ctypes.POINTER(NDArrayHandle)()\n            check_call(_LIB.MXBatchifyFunctionInvoke(self.handle,\n                                                     batch_size,\n                                                     num_output,\n                                                     input_vars,\n                                                     ctypes.byref(output_vars)))\n            out = [create_ndarray_fn(ctypes.cast(output_vars[i], NDArrayHandle), \\\n                False) for i in range(num_output.value)]\n            if len(out) == 1:\n                out = out[0]\n            return out\n        elif isinstance(data[0], (list, tuple)):\n            return self.__call__([j for sub in data for j in sub], num_out=len(data[0]))\n        else:\n            data = [default_array(i) for i in data]\n            return self.__call__(data, num_out=num_out)\n\ndef _make_internal_datasets(handle):\n    \"\"\"Create an io iterator by handle.\"\"\"\n    name = ctypes.c_char_p()\n    desc = ctypes.c_char_p()\n    num_args = mx_uint()\n    arg_names = ctypes.POINTER(ctypes.c_char_p)()\n    arg_types = ctypes.POINTER(ctypes.c_char_p)()\n    arg_descs = ctypes.POINTER(ctypes.c_char_p)()\n\n    check_call(_LIB.MXDatasetGetDatasetInfo( \\\n            handle, ctypes.byref(name), ctypes.byref(desc), \\\n            ctypes.byref(num_args), \\\n            ctypes.byref(arg_names), \\\n            ctypes.byref(arg_types), \\\n            ctypes.byref(arg_descs)))\n    iter_name = py_str(name.value)\n\n    narg = int(num_args.value)\n    param_str = _build_param_doc(\n        [py_str(arg_names[i]) for i in range(narg)],\n        [py_str(arg_types[i]) for i in range(narg)],\n        [py_str(arg_descs[i]) for i in range(narg)])\n\n    doc_str = (f'{desc.value}\\n\\n' +\n               f'{param_str}\\n' +\n               'Returns\\n' +\n               '-------\\n' +\n               'MXDataset\\n'+\n               '    The result dataset.')\n\n    def creator(*args, **kwargs):\n        \"\"\"Create a dataset.\n        The parameters listed below can be passed in as keyword arguments.\n\n        Parameters\n        ----------\n        name : string, required.\n            Name of the resulting dataset.\n\n        Returns\n        -------\n        dataset: Dataset\n            The resulting dataset.\n        \"\"\"\n        param_keys = []\n        param_vals = []\n\n        for k, val in kwargs.items():\n            # convert ndarray to handle\n            if hasattr(val, 'handle'):\n                val = val.handle.value\n            if isinstance(val, (tuple, list)):\n                val = [vv.handle.value if hasattr(vv, 'handle') else vv for vv in val]\n            param_keys.append(k)\n            param_vals.append(str(val))\n        # create atomic symbol\n        param_keys = c_str_array(param_keys)\n        param_vals = c_str_array(param_vals)\n        dataset_handle = DatasetHandle()\n        check_call(_LIB.MXDatasetCreateDataset(\n            handle,\n            mx_uint(len(param_keys)),\n            param_keys, param_vals,\n            ctypes.byref(dataset_handle)))\n\n        if len(args):\n            raise TypeError(f'{iter_name} can only accept keyword arguments')\n\n        return MXDataset(dataset_handle, **kwargs)\n\n    creator.__name__ = iter_name\n    creator.__doc__ = doc_str\n    return creator\n\ndef _init_internal_dataset_module():\n    \"\"\"List and add all the datasets to current module.\"\"\"\n    plist = ctypes.POINTER(ctypes.c_void_p)()\n    size = ctypes.c_uint()\n    check_call(_LIB.MXListDatasets(ctypes.byref(size), ctypes.byref(plist)))\n    module_obj = sys.modules[__name__]\n    for i in range(size.value):\n        hdl = ctypes.c_void_p(plist[i])\n        dataset = _make_internal_datasets(hdl)\n        setattr(module_obj, dataset.__name__, dataset)\n\n_init_internal_dataset_module()\n\ndef _make_internal_batchify_functions(handle):\n    \"\"\"Create an io iterator by handle.\"\"\"\n    name = ctypes.c_char_p()\n    desc = ctypes.c_char_p()\n    num_args = mx_uint()\n    arg_names = ctypes.POINTER(ctypes.c_char_p)()\n    arg_types = ctypes.POINTER(ctypes.c_char_p)()\n    arg_descs = ctypes.POINTER(ctypes.c_char_p)()\n\n    check_call(_LIB.MXBatchifyFunctionGetFunctionInfo( \\\n            handle, ctypes.byref(name), ctypes.byref(desc), \\\n            ctypes.byref(num_args), \\\n            ctypes.byref(arg_names), \\\n            ctypes.byref(arg_types), \\\n            ctypes.byref(arg_descs)))\n    bf_name = py_str(name.value)\n\n    narg = int(num_args.value)\n    param_str = _build_param_doc(\n        [py_str(arg_names[i]) for i in range(narg)],\n        [py_str(arg_types[i]) for i in range(narg)],\n        [py_str(arg_descs[i]) for i in range(narg)])\n\n    doc_str = (f'{desc.value}\\n\\n' +\n               f'{param_str}\\n' +\n               'Returns\\n' +\n               '-------\\n' +\n               'MXBatchifyFunction\\n'+\n               '    The result batchify function.')\n\n    def creator(*args, **kwargs):\n        \"\"\"Create an iterator.\n        The parameters listed below can be passed in as keyword arguments.\n\n        Parameters\n        ----------\n        name : string, required.\n            Name of the resulting batchify function.\n\n        Returns\n        -------\n        batchify_func: BatchifyFunction\n            The resulting batchify function.\n        \"\"\"\n        param_keys = []\n        param_vals = []\n\n        for k, val in kwargs.items():\n            # convert ndarray to handle\n            if hasattr(val, 'handle'):\n                val = val.handle.value\n            if isinstance(val, (tuple, list)):\n                val = [vv.handle.value if hasattr(vv, 'handle') else vv for vv in val]\n            param_keys.append(k)\n            param_vals.append(str(val))\n        # create atomic symbol\n        param_keys = c_str_array(param_keys)\n        param_vals = c_str_array(param_vals)\n        batchify_fn_handle = BatchifyFunctionhandle()\n        check_call(_LIB.MXBatchifyFunctionCreateFunction(\n            handle,\n            mx_uint(len(param_keys)),\n            param_keys, param_vals,\n            ctypes.byref(batchify_fn_handle)))\n\n        if len(args):\n            raise TypeError(f'{bf_name} can only accept keyword arguments')\n\n        return MXBatchifyFunction(batchify_fn_handle, creator_name=bf_name, **kwargs)\n\n    creator.__name__ = bf_name\n    creator.__doc__ = doc_str\n    return creator\n\ndef _init_internal_batchify_function_module():\n    \"\"\"List and add all the batchify_functions to current module.\"\"\"\n    plist = ctypes.POINTER(ctypes.c_void_p)()\n    size = ctypes.c_uint()\n    check_call(_LIB.MXListBatchifyFunctions(ctypes.byref(size), ctypes.byref(plist)))\n    module_obj = sys.modules[__name__]\n    for i in range(size.value):\n        hdl = ctypes.c_void_p(plist[i])\n        bf = _make_internal_batchify_functions(hdl)\n        setattr(module_obj, bf.__name__, bf)\n\n_init_internal_batchify_function_module()\n"
  },
  {
    "path": "python/mxnet/gluon/data/batchify.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=reimported, consider-using-enumerate\n\"\"\"Batchify function.\"\"\"\nimport math\nimport warnings\nimport numpy as np\n\nfrom ...device import Device, cpu\nfrom ... import ndarray as nd\nfrom ... import numpy as _np\nfrom ...util import is_np_array\n\nclass Stack(object):\n    r\"\"\"Stack the input data samples to construct the batch.\n    The N input samples must have the same shape/length and will be stacked to construct a batch.\n    Examples\n    --------\n    >>> from mxnet.gluon.data import batchify\n    >>> # Stack multiple lists\n    >>> a = [1, 2, 3, 4]\n    >>> b = [4, 5, 6, 8]\n    >>> c = [8, 9, 1, 2]\n    >>> batchify.Stack()([a, b, c])\n    [[1. 2. 3. 4.]\n     [4. 5. 6. 8.]\n     [8. 9. 1. 2.]]\n    <NDArray 3x4 @cpu(0)>\n    >>> # Stack multiple numpy.ndarrays\n    >>> import numpy as np\n    >>> a = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])\n    >>> b = np.array([[5, 6, 7, 8], [1, 2, 3, 4]])\n    >>> batchify.Stack()([a, b])\n    [[[1. 2. 3. 4.]\n      [5. 6. 7. 8.]]\n     [[5. 6. 7. 8.]\n      [1. 2. 3. 4.]]]\n    <NDArray 2x2x4 @cpu(0)>\n    >>> # Stack multiple NDArrays\n    >>> import mxnet as mx\n    >>> a = nd.array([[1, 2, 3, 4], [5, 6, 7, 8]])\n    >>> b = nd.array([[5, 6, 7, 8], [1, 2, 3, 4]])\n    >>> batchify.Stack()([a, b])\n    [[[1. 2. 3. 4.]\n      [5. 6. 7. 8.]]\n     [[5. 6. 7. 8.]\n      [1. 2. 3. 4.]]]\n    <NDArray 2x2x4 @cpu(0)>\n    \"\"\"\n    def __init__(self, use_shared_mem=False):\n        self._use_shared_mem = use_shared_mem\n\n    def __call__(self, data):\n        \"\"\"Batchify the input data\n        Parameters\n        ----------\n        data : list\n            The input data samples\n        Returns\n        -------\n        batch_data : NDArray\n        \"\"\"\n        _arr = _np if is_np_array() else nd\n        _arr_cls = _arr.ndarray if is_np_array() else _arr.NDArray\n        if isinstance(data[0], _arr_cls):\n            dtype = data[0].dtype\n            if self._use_shared_mem:\n                out = _arr.empty((len(data),) + data[0].shape, dtype=dtype,\n                                 ctx=Device('cpu_shared', 0))\n                return _arr.stack(data, out=out) if is_np_array() else _arr.stack(*data, out=out)\n            else:\n                return _arr.stack(data) if is_np_array() else _arr.stack(*data)\n        elif isinstance(data[0], (tuple, list)):\n            data = zip(*data)\n            return [self.__call__(i) for i in data]\n        else:\n            out = np.asarray(data)\n            dtype = out.dtype\n            if self._use_shared_mem:\n                return _arr.array(out, ctx=Device('cpu_shared', 0), dtype=dtype)\n            else:\n                return _arr.array(out, dtype=dtype)\n\n    def __mx_handle__(self):\n        from ._internal import StackBatchify\n        return StackBatchify()\n\ndef _pad_arrs_to_max_length(arrs, pad_val, use_shared_mem, dtype, round_to=None):\n    \"\"\"Inner Implementation of the Pad batchify\n    Parameters\n    ----------\n    arrs : list\n    pad_val : number\n    use_shared_mem : bool, default False\n    round_to : int\n\n    Returns\n    -------\n    ret : NDArray\n    \"\"\"\n    _arr = _np if is_np_array() else nd\n    _arr_cls = _np.ndarray if is_np_array() else nd.NDArray\n    if isinstance(arrs[0], _arr_cls):\n        dtype = arrs[0].dtype if dtype is None else dtype\n        arrs = [arr.asnumpy() for arr in arrs]\n    elif not isinstance(arrs[0], np.ndarray):\n        arrs = [np.asarray(ele) for ele in arrs]\n        dtype = arrs[0][0].dtype if dtype is None else dtype\n    else:\n        dtype = arrs[0].dtype if dtype is None else dtype\n\n    ret_shape = list(arrs[0].shape)\n    for pad_axis in range(len(ret_shape)):\n        curr_lengths = [ele.shape[pad_axis] for ele in arrs]\n        max_size = max(curr_lengths)\n        if round_to is not None:\n            max_size = round_to * math.ceil(max_size / round_to)\n        ret_shape[pad_axis] = max_size\n    ret_shape = (len(arrs), ) + tuple(ret_shape)\n\n    ret = np.full(shape=ret_shape, fill_value=pad_val, dtype=dtype)\n\n    for i, arr in enumerate(arrs):\n        if arr.shape == ret_shape[1:]:\n            ret[i] = arr\n        else:\n            slices = [slice(None) for _ in range(arr.ndim)]\n            for pad_axis in range(arr.ndim):\n                slices[pad_axis] = slice(0, arr.shape[pad_axis])\n                assert slices[pad_axis].start != slices[pad_axis].stop\n            slices = [slice(i, i + 1)] + slices\n            ret[tuple(slices)] = arr\n\n\n    device = Device('cpu_shared', 0) if use_shared_mem else cpu()\n    ret = _arr.array(ret, ctx=device, dtype=dtype)\n\n    return ret\n\n\nclass Pad(object):\n    \"\"\"Pad the input ndarrays along the specific padding axis and stack them to get the output.\n    Input of the function will be N samples. Each sample should contain a single element that\n    can be 1) numpy.ndarray, 2) mxnet.nd.NDArray, 3) list of numbers.\n    You can set the `pad_val` to determine the padding value.\n\n    The arrays will be padded to the largest dimensions(at most 5 dimensions to pad) and then\n    stacked to form the final output.\n\n    Parameters\n    ----------\n    val : float or int, default None\n        The padding value.\n    dtype : str or numpy.dtype, default None\n        The value type of the output. If it is set to None, the input data type is used.\n    round_to : int, default None\n        If specified, the padded dimension will be rounded to be multiple of this argument.\n\n    Examples\n    --------\n    >>> from mxnet.gluon.data import batchify\n    >>> # Inputs are multiple lists\n    >>> a = [1, 2, 3, 4]\n    >>> b = [4, 5, 6]\n    >>> c = [8, 2]\n    >>> batchify.Pad()([a, b, c])\n    [[ 1  2  3  4]\n     [ 4  5  6  0]\n     [ 8  2  0  0]]\n    <NDArray 3x4 @cpu(0)>\n    >>> # Also output the lengths\n    >>> a = [1, 2, 3, 4]\n    >>> b = [4, 5, 6]\n    >>> c = [8, 2]\n    >>> # Inputs are multiple ndarrays\n    >>> import numpy as np\n    >>> a = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])\n    >>> b = np.array([[5, 8], [1, 2]])\n    >>> batchify.Pad(val=-1)([a, b])\n    [[[ 1  2  3  4]\n      [ 5  6  7  8]]\n     [[ 5  8 -1 -1]\n      [ 1  2 -1 -1]]]\n    <NDArray 2x2x4 @cpu(0)>\n    >>> # Inputs are multiple NDArrays\n    >>> import mxnet as mx\n    >>> a = nd.array([[1, 2, 3, 4], [5, 6, 7, 8]])\n    >>> b = nd.array([[5, 8], [1, 2]])\n    >>> batchify.Pad(val=-1)([a, b])\n    [[[ 1.  2.  3.  4.]\n      [ 5.  6.  7.  8.]]\n     [[ 5.  8. -1. -1.]\n      [ 1.  2. -1. -1.]]]\n    <NDArray 2x2x4 @cpu(0)>\n    \"\"\"\n    def __init__(self, val=None, dtype=None, round_to=None, use_shared_mem=False):\n        self._pad_val = 0 if val is None else val\n        self._dtype = dtype\n        self._warned = False\n        self._round_to = round_to\n        self._use_shared_mem = use_shared_mem\n\n    def __call__(self, data):\n        \"\"\"Batchify the input data.\n\n        The input can be list of numpy.ndarray, list of numbers or list of\n        mxnet.nd.NDArray. Inputting mxnet.nd.NDArray is discouraged as each\n        array need to be converted to numpy for efficient padding.\n        The arrays will be padded to the largest dimension at `axis` and then\n        stacked to form the final output.\n\n        Parameters\n        ----------\n        data : List[np.ndarray] or List[List[dtype]] or List[nd.NDArray]\n            List of samples to pad and stack.\n        Returns\n        -------\n        batch_data: NDArray\n            Data in the minibatch. Shape is (N, ...)\n        \"\"\"\n        _arr = _np if is_np_array() else nd\n        _arr_cls = _arr.ndarray if is_np_array() else _arr.NDArray\n        if isinstance(data[0], _arr_cls) and not self._warned:\n            self._warned = True\n            warnings.warn(\n                'Using Pad with NDArrays is discouraged for speed reasons. '\n                'Instead you should pad your data while it is still a list '\n                'and before converting to an NDArray. '\n                'Alternatively you can consider inputting a numpy.ndarray.')\n        if isinstance(data[0], (_arr_cls, np.ndarray, list)):\n            padded_arr = _pad_arrs_to_max_length(data, self._pad_val,\n                                                 self._use_shared_mem,\n                                                 self._dtype, self._round_to)\n            return padded_arr\n        else:\n            raise NotImplementedError(\n                \"Pad() does not support multiple items, use Group(Pad(), Pad(), ...) instead\")\n\n    def __mx_handle__(self):\n        from ._internal import PadBatchify\n        return PadBatchify(pad_val=self._pad_val, dtype=self._dtype if self._dtype is not None else -1)\n\ndef _append_arrs(arrs, use_shared_mem=False, expand=False, batch_axis=0):\n    \"\"\"Internal impl for returning appened arrays as list.\"\"\"\n    _arr = _np if is_np_array() else nd\n    if isinstance(arrs[0], _arr.NDArray):\n        if use_shared_mem:\n            out = [x.as_in_context(Device('cpu_shared', 0)) for x in arrs]\n        else:\n            out = arrs\n    else:\n        if use_shared_mem:\n            out = [_arr.array(x, ctx=Device('cpu_shared', 0)) for x in arrs]\n        else:\n            out = [_arr.array(x) for x in arrs]\n\n    # add batch axis\n    if expand:\n        out = [x.expand_dims(axis=batch_axis) for x in out]\n    return out\n\n\nclass Append(object):\n    r\"\"\"Loosely return list of the input data samples.\n    There is no constraint of shape for any of the input samples, however, you will\n    only be able to apply single batch operations since the output have different shapes.\n    Examples\n    --------\n    >>> a = [1, 2, 3, 4]\n    >>> b = [4, 5, 6]\n    >>> c = [8, 2]\n    >>> batchify.Append()([a, b, c])\n    [\n    [[1. 2. 3. 4.]]\n    <NDArray 1x4 @cpu_shared(0)>,\n    [[4. 5. 6.]]\n    <NDArray 1x3 @cpu_shared(0)>,\n    [[8. 2.]]\n    <NDArray 1x2 @cpu_shared(0)>\n    ]\n    \"\"\"\n\n    def __init__(self, expand=True, batch_axis=0, use_shared_mem=False):\n        self._expand = expand\n        self._batch_axis = batch_axis\n        self._use_shared_mem = use_shared_mem\n\n    def __call__(self, data):\n        \"\"\"Batchify the input data.\n        Parameters\n        ----------\n        data : list\n            The input data samples\n        Returns\n        -------\n        batch_data : NDArray\n        \"\"\"\n        return _append_arrs(data, use_shared_mem=self._use_shared_mem,\n                            expand=self._expand, batch_axis=self._batch_axis)\n\nclass Group(object):\n    \"\"\"Wrap multiple batchify functions together. The input functions will be applied\n    to the corresponding input fields.\n    Each data sample should be a list or tuple containing multiple attributes. The `i`th batchify\n    function stored in `Group` will be applied on the `i`th attribute. For example, each\n    data sample is (nd_data, label). You can wrap two batchify functions using\n    `Group(DataBatchify, LabelBatchify)` to batchify nd_data and label correspondingly.\n    Parameters\n    ----------\n    fn : list or tuple or callable\n        The batchify functions to wrap.\n    *args : tuple of callable\n        The additional batchify functions to wrap.\n    Examples\n    --------\n    >>> a = ([1, 2, 3, 4], 0)\n    >>> b = ([5, 7], 1)\n    >>> c = ([1, 2, 3, 4, 5, 6, 7], 0)\n    >>> f1, f2 = Group(Pad(val=0),\n    ...                Stack())([a, b])\n    >>> f1\n    <BLANKLINE>\n    [[1. 2. 3. 4.]\n     [5. 7. 0. 0.]]\n    <NDArray 2x4 @cpu_shared(0)>\n    >>> f2\n    <BLANKLINE>\n    [0 1]\n    <NDArray 2 @cpu_shared(0)>\n    \"\"\"\n    def __init__(self, fn, *args):\n        self._handle = None\n        if isinstance(fn, (list, tuple)):\n            assert len(args) == 0, 'Input pattern not understood. The input of Group can be ' \\\n                                   'Group(A, B, C) or Group([A, B, C]) or Group((A, B, C)). ' \\\n                                   f'Received fn={str(fn)}, args={str(args)}'\n            self._fn = fn\n        else:\n            self._fn = (fn, ) + args\n        for i, ele_fn in enumerate(self._fn):\n            assert hasattr(ele_fn, '__call__'), 'Batchify functions must be callable! ' \\\n                                                f'type(fn[{i}]) = {str(type(ele_fn))}'\n\n    def __call__(self, data):\n        \"\"\"Batchify the input data.\n        Parameters\n        ----------\n        data : list\n            The samples to batchfy. Each sample should contain N attributes.\n        Returns\n        -------\n        ret : tuple\n            A tuple of length N. Contains the batchified result of each attribute in the input.\n        \"\"\"\n        assert len(data[0]) == len(self._fn),\\\n            'The number of attributes in each data sample should contains' \\\n            ' {} elements'.format(len(self._fn))\n        ret = []\n        for i, ele_fn in enumerate(self._fn):\n            ret.append(ele_fn([ele[i] for ele in data]))\n        return tuple(ret)\n\n    def __mx_handle__(self):\n        if self._handle  is None:\n            from ._internal import GroupBatchify\n            try:\n                mx_fn = [fn.__mx_handle__() for fn in self._fn]\n                self._handle = GroupBatchify(functions=mx_fn)\n            except Exception as e:\n                raise NotImplementedError(\n                    \"GroupBatchify requires all internal batchify functions supported by backend.\"\n                    + str(e))\n        return self._handle\n\nclass AsList(object):\n    \"\"\"Simply forward the list of input data.\n    This is particularly useful when the Dataset contains textual data\n    and in conjonction with the `Group` batchify function.\n    Examples\n    --------\n    >>> a = ([1, 2, 3, 4], \"I am using MXNet\")\n    >>> b = ([5, 7, 2, 5], \"Gluon rocks!\")\n    >>> c = ([1, 2, 3, 4], \"Batchification!\")\n    >>> _, l = Group(Stack(), AsList())([a, b, c])\n    >>> l\n    ['I am using MXNet', 'Gluon rocks!', 'Batchification!']\n    \"\"\"\n    def __call__(self, data):\n        \"\"\"\n        Parameters\n        ----------\n        data : list\n            The list of samples\n        Returns\n        -------\n        ret : list\n            The input list\n        \"\"\"\n        return list(data)\n"
  },
  {
    "path": "python/mxnet/gluon/data/dataloader.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=ungrouped-imports\n\"\"\"Dataset generator.\"\"\"\n__all__ = ['DataLoader']\n\nimport pickle\nimport logging\nimport io\nimport sys\nimport signal\nimport multiprocessing\nimport multiprocessing.queues\nfrom multiprocessing.reduction import ForkingPickler\nfrom multiprocessing.pool import ThreadPool\nimport threading\nimport numpy as np\n\ntry:\n    import multiprocessing.resource_sharer\nexcept ImportError:\n    pass\n\nfrom . import sampler as _sampler\nfrom . import batchify as _batchify\nfrom ... import ndarray as nd, context\nfrom ...util import is_np_shape, is_np_array, set_np\nfrom ... import numpy as _mx_np  # pylint: disable=reimported\n\nif sys.platform == 'darwin' or sys.platform == 'win32':\n    def rebuild_ndarray(*args):\n        \"\"\"Rebuild ndarray from pickled shared memory\"\"\"\n        # pylint: disable=no-value-for-parameter\n        return nd.NDArray(nd.ndarray._new_from_shared_mem(*args))\n\n    def reduce_ndarray(data):\n        \"\"\"Reduce ndarray to shared memory handle\"\"\"\n        return rebuild_ndarray, data._to_shared_mem()\nelse:\n    def rebuild_ndarray(pid, fd, shape, dtype):\n        \"\"\"Rebuild ndarray from pickled shared memory\"\"\"\n        # pylint: disable=no-value-for-parameter\n        fd = fd.detach()\n        return nd.NDArray(nd.ndarray._new_from_shared_mem(pid, fd, shape, dtype))\n\n    def reduce_ndarray(data):\n        \"\"\"Reduce ndarray to shared memory handle\"\"\"\n        # keep a local ref before duplicating fd\n        data = data.as_in_context(context.Context('cpu_shared', 0))\n        pid, fd, shape, dtype = data._to_shared_mem()\n        fd = multiprocessing.reduction.DupFd(fd)\n        return rebuild_ndarray, (pid, fd, shape, dtype)\n\nForkingPickler.register(nd.NDArray, reduce_ndarray)\n\nif sys.platform == 'darwin' or sys.platform == 'win32':\n    def rebuild_np_ndarray(*args):\n        \"\"\"Rebuild ndarray from pickled shared memory\"\"\"\n        # pylint: disable=no-value-for-parameter\n        return _mx_np.ndarray(nd.ndarray._new_from_shared_mem(*args))\n\n    def reduce_np_ndarray(data):\n        \"\"\"Reduce ndarray to shared memory handle\"\"\"\n        return rebuild_np_ndarray, data._to_shared_mem()\nelse:\n    def rebuild_np_ndarray(pid, fd, shape, dtype):\n        \"\"\"Rebuild ndarray from pickled shared memory\"\"\"\n        # pylint: disable=no-value-for-parameter\n        fd = fd.detach()\n        return _mx_np.ndarray(nd.ndarray._new_from_shared_mem(pid, fd, shape, dtype))\n\n    def reduce_np_ndarray(data):\n        \"\"\"Reduce ndarray to shared memory handle\"\"\"\n        # keep a local ref before duplicating fd\n        data = data.as_in_context(context.Context('cpu_shared', 0))\n        pid, fd, shape, dtype = data._to_shared_mem()\n        fd = multiprocessing.reduction.DupFd(fd)\n        return rebuild_np_ndarray, (pid, fd, shape, dtype)\n\nForkingPickler.register(_mx_np.ndarray, reduce_np_ndarray)\n\n\nclass ConnectionWrapper(object):\n    \"\"\"Connection wrapper for multiprocessing that supports sending\n    NDArray via shared memory.\"\"\"\n\n    def __init__(self, conn):\n        self._conn = conn\n\n    def send(self, obj):\n        \"\"\"Send object\"\"\"\n        buf = io.BytesIO()\n        ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(obj)\n        self.send_bytes(buf.getvalue())\n\n    def recv(self):\n        \"\"\"Receive object\"\"\"\n        buf = self.recv_bytes()\n        return pickle.loads(buf)\n\n    def __getattr__(self, name):\n        \"\"\"Emmulate conn\"\"\"\n        attr = self.__dict__.get('_conn', None)\n        return getattr(attr, name)\n\n\nclass Queue(multiprocessing.queues.Queue):\n    \"\"\"Wrapper for multiprocessing queue that dumps NDArray with shared memory.\"\"\"\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, ctx=multiprocessing.get_context(), **kwargs)\n        self._reader = ConnectionWrapper(self._reader)\n        self._writer = ConnectionWrapper(self._writer)\n        self._send = self._writer.send\n        self._recv = self._reader.recv\n\n\nclass SimpleQueue(multiprocessing.queues.SimpleQueue):\n    \"\"\"Wrapper for multiprocessing SimpleQueue that dumps NDArray with shared memory.\n       SimpleQueue don't use threading internally.\n    \"\"\"\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, ctx=multiprocessing.get_context(), **kwargs)\n        self._reader = ConnectionWrapper(self._reader)\n        self._writer = ConnectionWrapper(self._writer)\n        self._send = self._writer.send\n        self._recv = self._reader.recv\n\ndef default_batchify_fn(data):\n    \"\"\"Collate data into batch.\"\"\"\n    if isinstance(data[0], nd.NDArray):\n        return _mx_np.stack(data) if is_np_array() else nd.stack(*data)\n    elif isinstance(data[0], tuple):\n        data = zip(*data)\n        return [default_batchify_fn(i) for i in data]\n    else:\n        data = np.asarray(data)\n        array_fn = _mx_np.array if is_np_array() else nd.array\n        return array_fn(data, dtype=data.dtype)\n\n\ndef default_mp_batchify_fn(data):\n    \"\"\"Collate data into batch. Use shared memory for stacking.\"\"\"\n    if isinstance(data[0], nd.NDArray):\n        empty_fn = _mx_np.empty if is_np_array() else nd.empty\n        out = empty_fn((len(data),) + data[0].shape, dtype=data[0].dtype,\n                       ctx=context.Context('cpu_shared', 0))\n        if is_np_array():\n            return _mx_np.stack(data, out=out)\n        else:\n            return nd.stack(*data, out=out)\n    elif isinstance(data[0], tuple):\n        data = zip(*data)\n        return [default_mp_batchify_fn(i) for i in data]\n    else:\n        data = np.asarray(data)\n        array_fn = _mx_np.array if is_np_array() else nd.array\n        return array_fn(data, dtype=data.dtype,\n                        ctx=context.Context('cpu_shared', 0))\n\n\ndef _as_in_context(data, ctx):\n    \"\"\"Move data into new context.\"\"\"\n    if isinstance(data, nd.NDArray):\n        return data.as_in_context(ctx)\n    elif isinstance(data, (list, tuple)):\n        return [_as_in_context(d, ctx) for d in data]\n    return data\n\n\ndef worker_loop_v1(dataset, key_queue, data_queue, batchify_fn):\n    \"\"\"Worker loop for multiprocessing DataLoader.\"\"\"\n    while True:\n        idx, samples = key_queue.get()\n        if idx is None:\n            break\n        batch = batchify_fn([dataset[i] for i in samples])\n        data_queue.put((idx, batch))\n\ndef fetcher_loop_v1(data_queue, data_buffer, pin_memory=False,\n                    pin_device_id=0, data_buffer_lock=None):\n    \"\"\"Fetcher loop for fetching data from queue and put in reorder dict.\"\"\"\n    while True:\n        idx, batch = data_queue.get()\n        if idx is None:\n            break\n        if pin_memory:\n            batch = _as_in_context(batch, context.cpu_pinned(pin_device_id))\n        else:\n            batch = _as_in_context(batch, context.cpu())\n        if data_buffer_lock is not None:\n            with data_buffer_lock:\n                data_buffer[idx] = batch\n        else:\n            data_buffer[idx] = batch\n\n\nclass _MultiWorkerIterV1(object):\n    \"\"\"Internal multi-worker iterator for DataLoader.\"\"\"\n    def __init__(self, num_workers, dataset, batchify_fn, batch_sampler,\n                 pin_memory=False, pin_device_id=0, worker_fn=worker_loop_v1):\n        assert num_workers > 0, \"_MultiWorkerIter is not for {} workers\".format(num_workers)\n        self._num_workers = num_workers\n        self._dataset = dataset\n        self._batchify_fn = batchify_fn\n        self._batch_sampler = batch_sampler\n        self._key_queue = Queue()\n        self._data_queue = SimpleQueue()\n\n        self._data_buffer = {}\n        self._data_buffer_lock = threading.Lock()\n\n        self._rcvd_idx = 0\n        self._sent_idx = 0\n        self._iter = iter(self._batch_sampler)\n        self._shutdown = False\n\n        workers = []\n        for _ in range(self._num_workers):\n            worker = multiprocessing.Process(\n                target=worker_fn,\n                args=(self._dataset, self._key_queue, self._data_queue, self._batchify_fn))\n            worker.daemon = True\n            worker.start()\n            workers.append(worker)\n        self._workers = workers\n\n        self._fetcher = threading.Thread(\n            target=fetcher_loop_v1,\n            args=(self._data_queue, self._data_buffer, pin_memory,\n                  pin_device_id, self._data_buffer_lock))\n        self._fetcher.daemon = True\n        self._fetcher.start()\n\n        # pre-fetch\n        for _ in range(2 * self._num_workers):\n            self._push_next()\n\n    def __len__(self):\n        return len(self._batch_sampler)\n\n    def __del__(self):\n        self.shutdown()\n\n    def _push_next(self):\n        \"\"\"Assign next batch workload to workers.\"\"\"\n        r = next(self._iter, None)\n        if r is None:\n            return\n        self._key_queue.put((self._sent_idx, r))\n        self._sent_idx += 1\n\n    def __next__(self):\n        assert not self._shutdown, \"call __next__ after shutdown is forbidden\"\n        if self._rcvd_idx == self._sent_idx:\n            assert not self._data_buffer, \"Data buffer should be empty at this moment\"\n            self.shutdown()\n            raise StopIteration\n\n        while True:\n            if self._rcvd_idx in self._data_buffer:\n                with self._data_buffer_lock:\n                    batch = self._data_buffer.pop(self._rcvd_idx)\n                self._rcvd_idx += 1\n                self._push_next()\n                return batch\n\n    def next(self):\n        return self.__next__()\n\n    def __iter__(self):\n        return self\n\n    def shutdown(self):\n        \"\"\"Shutdown internal workers by pushing terminate signals.\"\"\"\n        if not self._shutdown:\n            # send shutdown signal to the fetcher and join data queue first\n            # Remark:   loop_fetcher need to be joined prior to the workers.\n            #           otherwise, the fetcher may fail at getting data\n            self._data_queue.put((None, None))\n            self._fetcher.join()\n            # send shutdown signal to all worker processes\n            for _ in range(self._num_workers):\n                self._key_queue.put((None, None))\n            # force shut down any alive worker processes\n            for w in self._workers:\n                if w.is_alive():\n                    w.terminate()\n            self._shutdown = True\n\n\nclass DataLoaderV1(object):\n    \"\"\"Loads data from a dataset and returns mini-batches of data.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Source dataset. Note that numpy and mxnet arrays can be directly used\n        as a Dataset.\n    batch_size : int\n        Size of mini-batch.\n    shuffle : bool\n        Whether to shuffle the samples.\n    sampler : Sampler\n        The sampler to use. Either specify sampler or shuffle, not both.\n    last_batch : {'keep', 'discard', 'rollover'}\n        How to handle the last batch if batch_size does not evenly divide\n        `len(dataset)`:\n        - ``keep`` - A batch with less samples than previous batches is returned.\n        - ``discard`` - The last batch is discarded if its incomplete.\n        - ``rollover`` - The remaining samples are rolled over to the next epoch.\n    batch_sampler : Sampler\n        A sampler that returns mini-batches. Do not specify batch_size,\n        shuffle, sampler, and last_batch if batch_sampler is specified.\n    batchify_fn : callable\n        Callback function to allow users to specify how to merge samples\n        into a batch. Defaults to ``default_batchify_fn``.\n\n        .. code-block:: python\n\n            def default_batchify_fn(data):\n                if isinstance(data[0], nd.NDArray):\n                    return nd.stack(*data)\n                elif isinstance(data[0], tuple):\n                    data = zip(*data)\n                    return [default_batchify_fn(i) for i in data]\n                else:\n                    data = np.asarray(data)\n                    return nd.array(data, dtype=data.dtype)\n\n    num_workers : int, default 0\n        The number of multiprocessing workers to use for data preprocessing.\n    pin_memory : boolean, default False\n        If ``True``, the dataloader will copy NDArrays into pinned memory\n        before returning them. Copying from CPU pinned memory to GPU is faster\n        than from normal CPU memory.\n    pin_device_id : int, default 0\n        The device id to use for allocating pinned memory if pin_memory is ``True``\n    \"\"\"\n    def __init__(self, dataset, batch_size=None, shuffle=False, sampler=None,\n                 last_batch=None, batch_sampler=None, batchify_fn=None,\n                 num_workers=0, pin_memory=False, pin_device_id=0):\n        self._dataset = dataset\n        self._pin_memory = pin_memory\n        self._pin_device_id = pin_device_id\n\n        if batch_sampler is None:\n            if batch_size is None:\n                raise ValueError(\"batch_size must be specified unless \" \\\n                                 \"batch_sampler is specified\")\n            if sampler is None:\n                if shuffle:\n                    sampler = _sampler.RandomSampler(len(dataset))\n                else:\n                    sampler = _sampler.SequentialSampler(len(dataset))\n            elif shuffle:\n                raise ValueError(\"shuffle must not be specified if sampler is specified\")\n\n            batch_sampler = _sampler.BatchSampler(\n                sampler, batch_size, last_batch if last_batch else 'keep')\n        elif batch_size is not None or shuffle or sampler is not None or \\\n                last_batch is not None:\n            raise ValueError(\"batch_size, shuffle, sampler and last_batch must \" \\\n                             \"not be specified if batch_sampler is specified.\")\n\n        self._batch_sampler = batch_sampler\n        self._num_workers = num_workers if num_workers >= 0 else 0\n        if batchify_fn is None:\n            if num_workers > 0:\n                self._batchify_fn = _batchify.Stack(use_shared_mem=True)\n            else:\n                self._batchify_fn = _batchify.Stack()\n        else:\n            self._batchify_fn = batchify_fn\n\n    def __iter__(self):\n        if self._num_workers == 0:\n            def same_process_iter():\n                for batch in self._batch_sampler:\n                    ret = self._batchify_fn([self._dataset[idx] for idx in batch])\n                    if self._pin_memory:\n                        ret = _as_in_context(ret, context.cpu_pinned(self._pin_device_id))\n                    yield ret\n            return same_process_iter()\n\n        # multi-worker\n        return _MultiWorkerIterV1(self._num_workers, self._dataset,\n                                  self._batchify_fn, self._batch_sampler,\n                                  self._pin_memory, self._pin_device_id)\n\n    def __len__(self):\n        return len(self._batch_sampler)\n\n\ndef _thread_worker_initializer(active_shape, active_array):\n    \"\"\"Initializer for ThreadPool.\"\"\"\n    set_np(shape=active_shape, array=active_array)\n\n\n_worker_dataset = None\ndef _worker_initializer(dataset, active_shape, active_array):\n    \"\"\"Initialier for processing pool.\"\"\"\n    # global dataset is per-process based and only available in worker processes\n    # this is only necessary to handle MXIndexedRecordIO because otherwise dataset\n    # can be passed as argument\n    global _worker_dataset\n    _worker_dataset = dataset\n    set_np(shape=active_shape, array=active_array)\n\ndef _worker_fn(samples, batchify_fn, dataset=None):\n    \"\"\"Function for processing data in worker process.\"\"\"\n    # pylint: disable=unused-argument\n    # it is required that each worker process has to fork a new MXIndexedRecordIO handle\n    # preserving dataset as global variable can save tons of overhead and is safe in new process\n    global _worker_dataset\n    batch = batchify_fn([_worker_dataset[i] for i in samples])\n    buf = io.BytesIO()\n    ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(batch)\n    return buf.getvalue()\n\ndef _thread_worker_fn(samples, batchify_fn, dataset):\n    \"\"\"Threadpool worker function for processing data.\"\"\"\n    return batchify_fn([dataset[i] for i in samples])\n\nclass _MultiWorkerIter(object):\n    \"\"\"Internal multi-worker iterator for DataLoader.\"\"\"\n    def __init__(self, worker_pool, batchify_fn, batch_sampler, pin_memory=False,\n                 pin_device_id=0, worker_fn=_worker_fn, prefetch=0, dataset=None,\n                 data_loader=None, timeout=120):\n        self._worker_pool = worker_pool\n        self._batchify_fn = batchify_fn\n        self._batch_sampler = batch_sampler\n        self._data_buffer = {}\n        self._rcvd_idx = 0\n        self._sent_idx = 0\n        self._iter = iter(self._batch_sampler)\n        self._worker_fn = worker_fn\n        self._pin_memory = pin_memory\n        self._pin_device_id = pin_device_id\n        self._dataset = dataset\n        self._data_loader = data_loader\n        self._timeout = timeout\n        # pre-fetch\n        for _ in range(prefetch):\n            self._push_next()\n\n    def __len__(self):\n        return len(self._batch_sampler)\n\n    def _push_next(self):\n        \"\"\"Assign next batch workload to workers.\"\"\"\n        r = next(self._iter, None)\n        if r is None:\n            return\n        async_ret = self._worker_pool.apply_async(\n            self._worker_fn, (r, self._batchify_fn, self._dataset))\n        self._data_buffer[self._sent_idx] = async_ret\n        self._sent_idx += 1\n\n    def __next__(self):\n        self._push_next()\n        if self._rcvd_idx == self._sent_idx:\n            assert not self._data_buffer, \"Data buffer should be empty at this moment\"\n            raise StopIteration\n\n        assert self._rcvd_idx < self._sent_idx, \"rcvd_idx must be smaller than sent_idx\"\n        assert self._rcvd_idx in self._data_buffer, \"fatal error with _push_next, rcvd_idx missing\"\n        ret = self._data_buffer.pop(self._rcvd_idx)\n        try:\n            if self._dataset is None:\n                batch = pickle.loads(ret.get(self._timeout))\n            else:\n                batch = ret.get(self._timeout)\n            if self._pin_memory:\n                batch = _as_in_context(batch, context.cpu_pinned(self._pin_device_id))\n            self._rcvd_idx += 1\n            return batch\n        except multiprocessing.context.TimeoutError:\n            msg = '''Worker timed out after {} seconds. This might be caused by \\n\n            - Slow transform. Please increase timeout to allow slower data loading in each worker.\n            '''.format(self._timeout)\n            if not isinstance(self._worker_pool, multiprocessing.pool.ThreadPool):\n                msg += '''- Insufficient shared_memory if `timeout` is large enough.\n            Please consider reduce `num_workers` or increase shared_memory in system.\n            '''\n            print(msg)\n            raise\n        except Exception:\n            self._worker_pool.terminate()\n            raise\n\n    def next(self):\n        return self.__next__()\n\n    def __iter__(self):\n        return self\n\n\nclass DataLoader(object):\n    \"\"\"Loads data from a dataset and returns mini-batches of data.\n\n    Parameters\n    ----------\n    dataset : Dataset\n        Source dataset. Note that numpy and mxnet arrays can be directly used\n        as a Dataset.\n    batch_size : int\n        Size of mini-batch.\n    shuffle : bool\n        Whether to shuffle the samples.\n    sampler : Sampler\n        The sampler to use. Either specify sampler or shuffle, not both.\n    last_batch : {'keep', 'discard', 'rollover'}\n        How to handle the last batch if batch_size does not evenly divide\n        ``len(dataset)``.\n\n        keep - A batch with less samples than previous batches is returned.\n        discard - The last batch is discarded if its incomplete.\n        rollover - The remaining samples are rolled over to the next epoch.\n    batch_sampler : Sampler\n        A sampler that returns mini-batches. Do not specify batch_size,\n        shuffle, sampler, and last_batch if batch_sampler is specified.\n    batchify_fn : callable\n        Callback function to allow users to specify how to merge samples\n        into a batch. Defaults to `gluon.data.batchify.Stack()`.\n\n        .. code-block:: python\n\n            def default_batchify_fn(data):\n                if isinstance(data[0], nd.NDArray):\n                    return nd.stack(*data)\n                elif isinstance(data[0], np.ndarray):\n                    return np.stack(*data)\n                elif isinstance(data[0], tuple):\n                    data = zip(*data)\n                    return [default_batchify_fn(i) for i in data]\n                else:\n                    data = np.asarray(data)\n                    return np.ndarray(data, dtype=data.dtype)\n\n    num_workers : int, default 0\n        The number of multiprocessing workers to use for data preprocessing.\n    pin_memory : boolean, default False\n        If ``True``, the dataloader will copy NDArrays into pinned memory\n        before returning them. Copying from CPU pinned memory to GPU is faster\n        than from normal CPU memory.\n    pin_device_id : int, default 0\n        The device id to use for allocating pinned memory if pin_memory is ``True``\n    prefetch : int, default is `num_workers * 2`\n        The number of prefetching batches only works if `num_workers` > 0.\n        If `prefetch` > 0, it allow worker process to prefetch certain batches before\n        acquiring data from iterators.\n        Note that using large prefetching batch will provide smoother bootstrapping performance,\n        but will consume more shared_memory. Using smaller number may forfeit the purpose of using\n        multiple worker processes, try reduce `num_workers` in this case.\n        By default it defaults to `num_workers * 2`.\n    thread_pool : bool, default False\n        If ``True``, use threading pool instead of multiprocessing pool. Using threadpool\n        can avoid shared memory usage. If `DataLoader` is more IO bounded or GIL is not a killing\n        problem, threadpool version may achieve better performance than multiprocessing.\n    timeout : int, default is 120\n        The timeout in seconds for each worker to fetch a batch data. Only modify this number\n        unless you are experiencing timeout and you know it's due to slow data loading.\n        Sometimes full `shared_memory` will cause all workers to hang and causes timeout. In these\n        cases please reduce `num_workers` or increase system `shared_memory` size instead.\n    try_nopython : bool or None, default is None\n        Try compile python dataloading pipeline into pure MXNet c++ implementation. The benefit is\n        potentially faster iteration, no `shared_memory` usage, and less processes managed by python.\n        The compilation is not gauranteed to support all use cases, but it will fallback to python in\n        case of failure. You can set `try_nopython` to `False` to disable auto-detection of the\n        compilation feature or leave it to `None` to allow MXNet to determine it automatically.\n        If you request `try_nopython` to `True` and the compilation fails, it will raise a\n        RuntimeError with the failure reason.\n\n    \"\"\"\n    def __init__(self, dataset, batch_size=None, shuffle=False, sampler=None,\n                 last_batch=None, batch_sampler=None, batchify_fn=None,\n                 num_workers=0, pin_memory=False, pin_device_id=0,\n                 prefetch=None, thread_pool=False, timeout=120, try_nopython=None):\n        self._dataset = dataset\n        self._pin_memory = pin_memory\n        self._pin_device_id = pin_device_id\n        self._thread_pool = thread_pool\n        self._timeout = timeout\n        self._mx_iter = None\n        assert timeout > 0, \"timeout must be positive, given {}\".format(timeout)\n\n        if batch_sampler is None:\n            if batch_size is None:\n                raise ValueError(\"batch_size must be specified unless \" \\\n                                 \"batch_sampler is specified\")\n            if sampler is None:\n                if shuffle:\n                    sampler = _sampler.RandomSampler(len(dataset))\n                else:\n                    sampler = _sampler.SequentialSampler(len(dataset))\n            elif shuffle:\n                raise ValueError(\"shuffle must not be specified if sampler is specified\")\n\n            batch_sampler = _sampler.BatchSampler(\n                sampler, batch_size, last_batch if last_batch else 'keep')\n        elif batch_size is not None or shuffle or sampler is not None or \\\n                last_batch is not None:\n            raise ValueError(\"batch_size, shuffle, sampler and last_batch must \" \\\n                             \"not be specified if batch_sampler is specified.\")\n\n        self._batch_sampler = batch_sampler\n        self._num_workers = num_workers if num_workers >= 0 else 0\n        self._worker_pool = None\n        self._prefetch = max(0, int(prefetch) if prefetch is not None else 2 * self._num_workers)\n        if batchify_fn is None:\n            if num_workers > 0:\n                self._batchify_fn = _batchify.Stack(use_shared_mem=True)\n            else:\n                self._batchify_fn = _batchify.Stack()\n        else:\n            self._batchify_fn = batchify_fn\n\n        if num_workers > 0 and (try_nopython or try_nopython is None):\n            # check for capability to use mx backend threadedLoader\n            use_mx_iter, mx_iter_args = _check_mx_loader_capability(\n                self._dataset, self._batch_sampler, self._batchify_fn)\n            if not use_mx_iter:\n                if try_nopython:\n                    raise RuntimeError(mx_iter_args)\n        else:\n            use_mx_iter = False\n\n        if use_mx_iter:\n            logging.info(\"Using MXNet backend ThreadedDataLoader with %s workers \"\n                         \"instead of python dataloader.\", self._num_workers)\n            self._mx_iter = _MXThreadedDataLoader(\n                num_workers=self._num_workers,\n                pin_memory=self._pin_memory,\n                pin_device_id=self._pin_device_id,\n                prefetch=self._prefetch, **mx_iter_args)\n        else:\n            nd.waitall()\n            import gc\n            gc.collect()\n            nd.waitall()\n            if self._num_workers > 0:\n                if self._thread_pool:\n                    self._worker_pool = ThreadPool(self._num_workers,\n                                                   initializer=_thread_worker_initializer,\n                                                   initargs=(is_np_shape(), is_np_array()))\n                else:\n                    # set ignore keyboard interupt signal before forking processes\n                    original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)\n                    self._worker_pool = multiprocessing.Pool(\n                        self._num_workers, initializer=_worker_initializer,\n                        initargs=[self._dataset, is_np_shape(), is_np_array()])\n                    # resume keyboard interupt signal in main process\n                    signal.signal(signal.SIGINT, original_sigint_handler)\n\n    def __iter__(self):\n        if self._mx_iter is not None:\n            return iter(self._mx_iter)\n\n        if self._num_workers == 0:\n            def same_process_iter():\n                for batch in self._batch_sampler:\n                    ret = self._batchify_fn([self._dataset[idx] for idx in batch])\n                    if self._pin_memory:\n                        ret = _as_in_context(ret, context.cpu_pinned(self._pin_device_id))\n                    yield ret\n            return same_process_iter()\n\n        # multi-worker\n        return _MultiWorkerIter(self._worker_pool, self._batchify_fn, self._batch_sampler,\n                                pin_memory=self._pin_memory, pin_device_id=self._pin_device_id,\n                                worker_fn=_thread_worker_fn if self._thread_pool else _worker_fn,\n                                prefetch=self._prefetch,\n                                dataset=self._dataset if self._thread_pool else None,\n                                data_loader=self, timeout=self._timeout)\n\n    def __len__(self):\n        return len(self._batch_sampler)\n\n    def __del__(self):\n        if self._worker_pool:\n            # manually terminate due to a bug that pool is not automatically terminated\n            # https://bugs.python.org/issue34172\n            assert isinstance(self._worker_pool, multiprocessing.pool.Pool)\n            self._worker_pool.terminate()\n\ndef _check_mx_loader_capability(dataset, batch_sampler, batchify_fn):\n    from ._internal import MXDataset, MXSampler\n    from ._internal import MXBatchifyFunction\n    mx_loader_args = {}\n    error_template = \"MXNet backend loader compatibility: \" \\\n        \"[dataset - {}][batchify_fn - {}][batch sampler - {}]\"\n\n    # supported dataset\n    if isinstance(dataset, MXDataset):\n        mx_loader_args['dataset'] = dataset\n    elif hasattr(dataset, '__mx_handle__'):\n        try:\n            mx_loader_args['dataset'] = dataset.__mx_handle__()\n        except NotImplementedError:\n            return False, error_template.format('fail', 'unknown', 'unknown')\n    else:\n        return False, error_template.format('fail', 'unknown', 'unknown')\n\n    # supported batchify functions\n    if hasattr(batchify_fn, '__mx_handle__'):\n        mx_loader_args['batchify_fn'] = batchify_fn.__mx_handle__()\n    elif isinstance(batchify_fn, MXBatchifyFunction):\n        mx_loader_args['batchify_fn'] = batchify_fn\n    else:\n        return False, error_template.format('pass', 'fail', 'unknown')\n\n    # supported sampler\n    if isinstance(batch_sampler, _sampler.BatchSampler):\n        if isinstance(batch_sampler._sampler, _sampler.SequentialSampler):\n            mx_loader_args['batch_sampler'] = MXSampler(\n                'SequentialSampler', length=batch_sampler._sampler._length,\n                start=batch_sampler._sampler._start,\n                batch_size=batch_sampler._batch_size,\n                last_batch=batch_sampler._last_batch)\n        elif isinstance(batch_sampler._sampler, _sampler.RandomSampler):\n            mx_loader_args['batch_sampler'] = MXSampler(\n                'RandomSampler', length=batch_sampler._sampler._length,\n                batch_size=batch_sampler._batch_size,\n                last_batch=batch_sampler._last_batch)\n        else:\n            return False, error_template.format('pass', 'pass', 'fail')\n    elif isinstance(batch_sampler, MXSampler):\n        mx_loader_args['batch_sampler'] = batch_sampler\n    else:\n        return False, error_template.format('pass', 'pass', 'fail')\n    # all good\n    return True, mx_loader_args\n\n\nclass _MXThreadedDataLoader(object):\n    \"\"\"MXNet internal C++ threaded Data Iterator in form of DataLoader\n\n    parameters\n    ----------\n    dataset : Dataset\n        Source dataset. Note that numpy and mxnet arrays can be directly used\n        as a Dataset.\n    batch_sampler : Sampler\n        A sampler that returns mini-batches.\n    batchify_fn : callable\n        Callback function to allow users to specify how to merge samples\n        into a batch. Defaults to `gluon.data.batchify.Stack()`::\n    num_workers : int, default 0\n        The number of multiprocessing workers to use for data preprocessing.\n    pin_memory : boolean, default False\n        If ``True``, the dataloader will copy NDArrays into pinned memory\n        before returning them. Copying from CPU pinned memory to GPU is faster\n        than from normal CPU memory.\n    pin_device_id : int, default 0\n        The device id to use for allocating pinned memory if pin_memory is ``True``\n    prefetch : int, default is `num_workers * 2`\n        The number of prefetching batches only works if `num_workers` > 0.\n        If `prefetch` > 0, it allow worker process to prefetch certain batches before\n        acquiring data from iterators.\n        Note that using large prefetching batch will provide smoother bootstrapping performance,\n        but will consume more shared_memory. Using smaller number may forfeit the purpose of using\n        multiple worker processes, try reduce `num_workers` in this case.\n        By default it defaults to `num_workers * 2`, maximum prefetch size is `16`.\n    \"\"\"\n    def __init__(self, dataset, batch_sampler, batchify_fn,\n                 num_workers=0, pin_memory=False, pin_device_id=0,\n                 prefetch=4):\n        from ._internal import MXDataset, MXSampler, MXBatchifyFunction\n        from ...io.io import ThreadedDataLoader\n        assert isinstance(dataset, MXDataset)\n        assert isinstance(batch_sampler, MXSampler)\n        assert isinstance(batchify_fn, MXBatchifyFunction)\n        self._dataset = dataset\n        self._batch_sampler = batch_sampler\n        self._batchify_fn = batchify_fn\n        if num_workers == 0:\n            num_workers = 1  # different convention for single thread\n        if prefetch == 0:\n            prefetch = 1  # at least one buffer required\n        pin_device_id = pin_device_id if pin_memory else -1\n        ctx = 'cpu_pinned' if pin_memory else 'cpu'\n        self._iter = ThreadedDataLoader(num_workers=num_workers, dataset=dataset,\n                                        sampler=batch_sampler, batchify_fn=batchify_fn,\n                                        prefetch_buffer=prefetch, ctx=ctx,\n                                        device_id=pin_device_id)\n\n    def __iter__(self):\n        while self._iter.iter_next():\n            self._iter.first_batch = None\n            items = self._iter.getitems()\n            pad = self._iter.getpad()\n            if pad > 0:\n                items = tuple([x[:-pad] for x in items])\n            if len(items) < 2:\n                items = items[0]\n            yield items\n        self._iter.reset()\n\n    def __len__(self):\n        return len(self._iter)\n"
  },
  {
    "path": "python/mxnet/gluon/data/dataset.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=\n\"\"\"Dataset container.\"\"\"\n__all__ = ['Dataset', 'SimpleDataset', 'ArrayDataset',\n           'RecordFileDataset']\n\nimport os\n\nfrom ... import recordio, ndarray\nfrom ...util import default_array\n\n\nclass Dataset(object):\n    \"\"\"Abstract dataset class. All datasets should have this interface.\n\n    Subclasses need to override `__getitem__`, which returns the i-th\n    element, and `__len__`, which returns the total number elements.\n\n    .. note:: An mxnet or numpy array can be directly used as a dataset.\n    \"\"\"\n    def __getitem__(self, idx):\n        raise NotImplementedError\n\n    def __len__(self):\n        raise NotImplementedError\n\n    def filter(self, fn):\n        \"\"\"Returns a new dataset with samples filtered by the\n        filter function `fn`.\n\n        Note that if the Dataset is the result of a lazily transformed one with\n        transform(lazy=False), the filter is eagerly applied to the transformed\n        samples without materializing the transformed result. That is, the\n        transformation will be applied again whenever a sample is retrieved after\n        filter().\n\n        Parameters\n        ----------\n        fn : callable\n            A filter function that takes a sample as input and\n            returns a boolean. Samples that return False are discarded.\n\n        Returns\n        -------\n        Dataset\n            The filtered dataset.\n        \"\"\"\n        from . import FilterSampler\n        return _SampledDataset(self, FilterSampler(fn, self))\n\n    def shard(self, num_shards, index):\n        \"\"\"Returns a new dataset includes only 1/num_shards of this dataset.\n\n        For distributed training, be sure to shard before you randomize the dataset\n        (such as shuffle), if you want each worker to reach a unique subset.\n\n        Parameters\n        ----------\n        num_shards : int\n            A integer representing the number of data shards.\n        index : int\n            A integer representing the index of the current shard.\n\n        Returns\n        -------\n        Dataset\n            The result dataset.\n        \"\"\"\n        assert index < num_shards, f'Shard index of out bound: {index} out of {num_shards}'\n        assert num_shards > 0, 'Number of shards must be greater than 0'\n        assert index >= 0, 'Index must be non-negative'\n        length = len(self)\n        shard_len = length // num_shards\n        rest = length % num_shards\n        # Compute the start index for this partition\n        start = shard_len * index + min(index, rest)\n        # Compute the end index for this partition\n        end = start + shard_len + (index < rest)\n        from . import SequentialSampler\n        return _SampledDataset(self, SequentialSampler(end - start, start))\n\n    def take(self, count):\n        \"\"\"Returns a new dataset with at most `count` number of samples in it.\n\n        Parameters\n        ----------\n        count : int or None\n            A integer representing the number of elements of this dataset that\n            should be taken to form the new dataset. If count is None, or if count\n            is greater than the size of this dataset, the new dataset will contain\n            all elements of this dataset.\n\n        Returns\n        -------\n        Dataset\n            The result dataset.\n        \"\"\"\n        if count is None or count > len(self):\n            count = len(self)\n        from . import SequentialSampler\n        return _SampledDataset(self, SequentialSampler(count))\n\n    def sample(self, sampler):\n        \"\"\"Returns a new dataset with elements sampled by the sampler.\n\n        Parameters\n        ----------\n        sampler : Sampler\n            A Sampler that returns the indices of sampled elements.\n\n        Returns\n        -------\n        Dataset\n            The result dataset.\n        \"\"\"\n        from . import Sampler\n        if not isinstance(sampler, Sampler):\n            raise TypeError(f'Invalid sampler type: {type(sampler)}. Expected gluon.data.Sampler instead.')\n        return _SampledDataset(self, sampler)\n\n    def transform(self, fn, lazy=True):\n        \"\"\"Returns a new dataset with each sample transformed by the\n        transformer function `fn`.\n\n        Parameters\n        ----------\n        fn : callable\n            A transformer function that takes a sample as input and\n            returns the transformed sample.\n        lazy : bool, default True\n            If False, transforms all samples at once. Otherwise,\n            transforms each sample on demand. Note that if `fn`\n            is stochastic, you must set lazy to True or you will\n            get the same result on all epochs.\n\n        Returns\n        -------\n        Dataset\n            The transformed dataset.\n        \"\"\"\n        trans = _LazyTransformDataset(self, fn)\n        if lazy:\n            return trans\n        return SimpleDataset([i for i in trans])\n\n    def transform_first(self, fn, lazy=True):\n        \"\"\"Returns a new dataset with the first element of each sample\n        transformed by the transformer function `fn`.\n\n        This is mostly applicable when each sample contains two components\n        - features and label, i.e., (X, y), and you only want to transform\n        the first element X (i.e., the features) while keeping the label y\n        unchanged.\n\n        Parameters\n        ----------\n        fn : callable\n            A transformer function that takes the first element of a sample\n            as input and returns the transformed element.\n        lazy : bool, default True\n            If False, transforms all samples at once. Otherwise,\n            transforms each sample on demand. Note that if `fn`\n            is stochastic, you must set lazy to True or you will\n            get the same result on all epochs.\n\n        Returns\n        -------\n        Dataset\n            The transformed dataset.\n        \"\"\"\n        return self.transform(_TransformFirstClosure(fn), lazy)\n\n\nclass SimpleDataset(Dataset):\n    \"\"\"Simple Dataset wrapper for lists and arrays.\n\n    Parameters\n    ----------\n    data : dataset-like object\n        Any object that implements `len()` and `[]`.\n    \"\"\"\n    def __init__(self, data):\n        self._data = data\n        self._handle = None\n\n    def __len__(self):\n        return len(self._data)\n\n    def __getitem__(self, idx):\n        return self._data[idx]\n\n    def __mx_handle__(self):\n        if self._handle is None:\n            import numpy as np\n            from ._internal import NDArrayDataset\n            if isinstance(self._data, (np.ndarray, ndarray.NDArray)):\n                self._handle = NDArrayDataset(arr=default_array(self._data))\n            else:\n                raise NotImplementedError(\n                    \"C++ handle for general type object is not supported, \"\n                    \"given {}, expect np.ndarray\".format(type(self._data)))\n        return self._handle\n\n\nclass _LazyTransformDataset(Dataset):\n    \"\"\"Lazily transformed dataset.\"\"\"\n    def __init__(self, data, fn):\n        self._data = data\n        self._fn = fn\n        self.handle = None\n\n    def __len__(self):\n        return len(self._data)\n\n    def __getitem__(self, idx):\n        item = self._data[idx]\n        if isinstance(item, tuple):\n            return self._fn(*item)\n        return self._fn(item)\n\n    def __mx_handle__(self):\n        if self.handle is None:\n            from ..block import HybridBlock\n            from ._internal import LazyTransformDataset\n            from ...base import numeric_types\n            if not hasattr(self._data, '__mx_handle__'):\n                raise NotImplementedError(\"{} don't support backend\".format(self._data))\n            if isinstance(self._fn, HybridBlock):\n                item = self._data[0]\n                self._fn.hybridize()\n                if isinstance(item, tuple):\n                    ret = self._fn(*item)\n                    is_scalar = [int(isinstance(x, numeric_types)) for x in ret]\n                else:\n                    ret = self._fn(item)\n                    is_scalar = [int(isinstance(ret, numeric_types))]\n                cached_op = self._fn._cached_op\n                self.handle = LazyTransformDataset(cached_op=cached_op,\n                                                   dataset=self._data.__mx_handle__(),\n                                                   scalar_outputs=tuple(is_scalar))\n            elif isinstance(self._fn, _TransformFirstClosure):\n                if not isinstance(self._fn._fn, HybridBlock):\n                    raise NotImplementedError(\"Block not supported.\")\n                item = self._data[0][0]\n                self._fn._fn.hybridize()\n                ret = self._fn._fn(item)\n                is_scalar = [int(isinstance(ret, numeric_types))]\n                cached_op = self._fn._fn._cached_op\n                self.handle = LazyTransformDataset(cached_op=cached_op,\n                                                   dataset=self._data.__mx_handle__(),\n                                                   scalar_outputs=tuple(is_scalar),\n                                                   transform_indices=(0,))\n            else:\n                raise NotImplementedError(\n                    \"C++ handle Not implemented for transforms that are not hybridizable\")\n        return self.handle\n\n\nclass _TransformFirstClosure(object):\n    \"\"\"Use callable object instead of nested function, it can be pickled.\"\"\"\n    def __init__(self, fn):\n        self._fn = fn\n\n    def __call__(self, x, *args):\n        if args:\n            return (self._fn(x),) + args\n        return self._fn(x)\n\nclass _FilteredDataset(Dataset):\n    \"\"\"Dataset with a filter applied\"\"\"\n    def __init__(self, dataset, fn):\n        self._dataset = dataset\n        self._indices = [i for i, sample in enumerate(dataset) if fn(sample)]\n        self.handle = None\n\n    def __len__(self):\n        return len(self._indices)\n\n    def __getitem__(self, idx):\n        return self._dataset[self._indices[idx]]\n\n    def __mx_handle__(self):\n        if self.handle is None:\n            from ._internal import MXDataset, IndexedDataset\n            if hasattr(self._dataset, '__mx_handle__'):\n                dataset = self._dataset.__mx_handle__()\n            elif isinstance(self._dataset, MXDataset):\n                dataset = self._dataset\n            else:\n                raise NotImplementedError('{} not supported.'.format(self._dataset))\n            self.handle = IndexedDataset(base=dataset,\n                                         indices=self._indices)\n        return self.handle\n\n\nclass _SampledDataset(Dataset):\n    \"\"\"Dataset with elements chosen by a sampler\"\"\"\n    def __init__(self, dataset, sampler):\n        self._dataset = dataset\n        self._sampler = sampler\n        self._indices = list(iter(sampler))\n        self.handle = None\n\n    def __len__(self):\n        return len(self._sampler)\n\n    def __getitem__(self, idx):\n        return self._dataset[self._indices[idx]]\n\n    def __mx_handle__(self):\n        if self.handle is None:\n            from ._internal import MXDataset, IndexedDataset\n            if hasattr(self._dataset, '__mx_handle__'):\n                dataset = self._dataset.__mx_handle__()\n            elif isinstance(self._dataset, MXDataset):\n                dataset = self._dataset\n            else:\n                raise NotImplementedError('{} not supported.'.format(self._dataset))\n            self.handle = IndexedDataset(base=dataset,\n                                         indices=self._indices)\n        return self.handle\n\n\nclass ArrayDataset(Dataset):\n    \"\"\"A dataset that combines multiple dataset-like objects, e.g.\n    Datasets, lists, arrays, etc.\n\n    The i-th sample is defined as `(x1[i], x2[i], ...)`.\n\n    Parameters\n    ----------\n    *args : one or more dataset-like objects\n        The data arrays.\n    \"\"\"\n    def __init__(self, *args):\n        assert len(args) > 0, \"Needs at least 1 arrays\"\n        self._length = len(args[0])\n        self._data = []\n        for i, data in enumerate(args):\n            assert len(data) == self._length, \\\n                f\"All arrays must have the same length; array[0] has length {self._length} \" \\\n                f\"while array[{i+1}] has {len(data)}.\"\n            if isinstance(data, ndarray.NDArray) and len(data.shape) == 1:\n                data = data.asnumpy()\n            self._data.append(data)\n        self.handle = None\n\n    def __getitem__(self, idx):\n        if len(self._data) == 1:\n            return self._data[0][idx]\n        else:\n            return tuple(data[idx] for data in self._data)\n\n    def __len__(self):\n        return self._length\n\n    def __mx_handle__(self):\n        if self.handle is None:\n            from ._internal import MXDataset, NDArrayDataset, GroupDataset\n            datasets = []\n            for data in self._data:\n                if isinstance(data, MXDataset):\n                    datasets.append(data)\n                elif hasattr(data, '__mx_handle__'):\n                    datasets.append(data.__mx_handle__())\n                else:\n                    datasets.append(NDArrayDataset(arr=default_array(data)))\n            self.handle = GroupDataset(datasets=datasets)\n        return self.handle\n\n\nclass RecordFileDataset(Dataset):\n    \"\"\"A dataset wrapping over a RecordIO (.rec) file.\n\n    Each sample is a string representing the raw content of an record.\n\n    Parameters\n    ----------\n    filename : str\n        Path to rec file.\n    \"\"\"\n    def __init__(self, filename):\n        self.idx_file = os.path.splitext(filename)[0] + '.idx'\n        self.filename = filename\n        self._record = recordio.MXIndexedRecordIO(self.idx_file, self.filename, 'r')\n\n    def __getitem__(self, idx):\n        return self._record.read_idx(self._record.keys[idx])\n\n    def __len__(self):\n        return len(self._record.keys)\n\n    def __mx_handle__(self):\n        from ._internal import RecordFileDataset as _RecordFileDataset\n        return _RecordFileDataset(rec_file=self.filename, idx_file=self.idx_file)\n\n\nclass _DownloadedDataset(Dataset):\n    \"\"\"Base class for MNIST, cifar10, etc.\"\"\"\n    def __init__(self, root, transform):\n        super(_DownloadedDataset, self).__init__()\n        if transform is not None:\n            raise DeprecationWarning(\n                'Directly apply transform to dataset is deprecated. '\n                'Please use dataset.transform() or dataset.transform_first() instead...')\n        self._transform = transform\n        self._data = None\n        self._label = None\n        root = os.path.expanduser(root)\n        self._root = root\n        if not os.path.isdir(root):\n            os.makedirs(root)\n        self._get_data()\n        self.handle = None\n\n    def __getitem__(self, idx):\n        if self._transform is not None:\n            return self._transform(self._data[idx], self._label[idx])\n        return self._data[idx], self._label[idx]\n\n    def __len__(self):\n        return len(self._label)\n\n    def _get_data(self):\n        raise NotImplementedError\n\n    def __mx_handle__(self):\n        if self.handle is None:\n            from ._internal import NDArrayDataset, GroupDataset\n            self.handle = GroupDataset(\n                datasets=(NDArrayDataset(arr=default_array(self._data)),\n                          NDArrayDataset(arr=default_array(self._label))))\n        return self.handle\n"
  },
  {
    "path": "python/mxnet/gluon/data/sampler.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=\n\"\"\"Dataset sampler.\"\"\"\n__all__ = ['Sampler', 'SequentialSampler', 'RandomSampler', 'FilterSampler', 'BatchSampler',\n           'IntervalSampler']\n\nimport numpy as np\n\nclass Sampler(object):\n    \"\"\"Base class for samplers.\n\n    All samplers should subclass `Sampler` and define `__iter__` and `__len__`\n    methods.\n    \"\"\"\n    def __iter__(self):\n        raise NotImplementedError\n\n    def __len__(self):\n        raise NotImplementedError\n\n\nclass SequentialSampler(Sampler):\n    \"\"\"Samples elements from [start, start+length) sequentially.\n\n    Parameters\n    ----------\n    length : int\n        Length of the sequence.\n    start : int, default is 0\n        The start of the sequence index.\n    \"\"\"\n    def __init__(self, length, start=0):\n        self._length = length\n        self._start = start\n\n    def __iter__(self):\n        return iter(range(self._start, self._start + self._length))\n\n    def __len__(self):\n        return self._length\n\nclass RandomSampler(Sampler):\n    \"\"\"Samples elements from [0, length) randomly without replacement.\n\n    Parameters\n    ----------\n    length : int\n        Length of the sequence.\n    \"\"\"\n    def __init__(self, length):\n        self._length = length\n\n    def __iter__(self):\n        indices = np.arange(self._length)\n        np.random.shuffle(indices)\n        return iter(indices)\n\n    def __len__(self):\n        return self._length\n\nclass FilterSampler(Sampler):\n    \"\"\"Samples elements from a Dataset for which `fn` returns True.\n\n    Parameters\n    ----------\n    fn : callable\n        A callable function that takes a sample and returns a boolean\n    dataset : Dataset\n        The dataset to filter.\n    \"\"\"\n    def __init__(self, fn, dataset):\n        self._fn = fn\n        self._dataset = dataset\n        self._indices = [i for i, sample in enumerate(dataset) if fn(sample)]\n\n    def __iter__(self):\n        return iter(self._indices)\n\n    def __len__(self):\n        return len(self._indices)\n\n\nclass BatchSampler(Sampler):\n    \"\"\"Wraps over another `Sampler` and return mini-batches of samples.\n\n    Parameters\n    ----------\n    sampler : Sampler\n        The source Sampler.\n    batch_size : int\n        Size of mini-batch.\n    last_batch : {'keep', 'discard', 'rollover'}\n        Specifies how the last batch is handled if batch_size does not evenly\n        divide sequence length.\n\n        If 'keep', the last batch will be returned directly, but will contain\n        less element than `batch_size` requires.\n\n        If 'discard', the last batch will be discarded.\n\n        If 'rollover', the remaining elements will be rolled over to the next\n        iteration.\n\n    Examples\n    --------\n    >>> sampler = gluon.data.SequentialSampler(10)\n    >>> batch_sampler = gluon.data.BatchSampler(sampler, 3, 'keep')\n    >>> list(batch_sampler)\n    [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]\n    \"\"\"\n    def __init__(self, sampler, batch_size, last_batch='keep'):\n        self._sampler = sampler\n        self._batch_size = batch_size\n        self._last_batch = last_batch\n        self._prev = []\n\n    def __iter__(self):\n        batch, self._prev = self._prev, []\n        for i in self._sampler:\n            batch.append(i)\n            if len(batch) == self._batch_size:\n                yield batch\n                batch = []\n        if batch:\n            if self._last_batch == 'keep':\n                yield batch\n            elif self._last_batch == 'discard':\n                return\n            elif self._last_batch == 'rollover':\n                self._prev = batch\n            else:\n                raise ValueError(\n                    \"last_batch must be one of 'keep', 'discard', or 'rollover', \" \\\n                    f\"but got {self._last_batch}\")\n\n    def __len__(self):\n        if self._last_batch == 'keep':\n            return (len(self._sampler) + self._batch_size - 1) // self._batch_size\n        if self._last_batch == 'discard':\n            return len(self._sampler) // self._batch_size\n        if self._last_batch == 'rollover':\n            return (len(self._prev) + len(self._sampler)) // self._batch_size\n        raise ValueError(\n            \"last_batch must be one of 'keep', 'discard', or 'rollover', \" \\\n            f\"but got {self._last_batch}\")\n\n\nclass IntervalSampler(Sampler):\n    \"\"\"Samples elements from [0, length) at fixed intervals.\n\n    Parameters\n    ----------\n    length : int\n        Length of the sequence.\n    interval : int\n        The number of items to skip between two samples.\n    rollover : bool, default True\n        Whether to start again from the first skipped item after reaching the end.\n        If true, this sampler would start again from the first skipped item until all items\n        are visited.\n        Otherwise, iteration stops when end is reached and skipped items are ignored.\n\n    Examples\n    --------\n    >>> sampler = contrib.data.IntervalSampler(13, interval=3)\n    >>> list(sampler)\n    [0, 3, 6, 9, 12, 1, 4, 7, 10, 2, 5, 8, 11]\n    >>> sampler = contrib.data.IntervalSampler(13, interval=3, rollover=False)\n    >>> list(sampler)\n    [0, 3, 6, 9, 12]\n    \"\"\"\n    def __init__(self, length, interval, rollover=True):\n        assert interval <= length, \\\n            \"Interval {} must be smaller than or equal to length {}\".format(interval, length)\n        self._length = length\n        self._interval = interval\n        self._rollover = rollover\n\n    def __iter__(self):\n        for i in range(self._interval if self._rollover else 1):\n            for j in range(i, self._length, self._interval):\n                yield j\n\n    def __len__(self):\n        return self._length\n"
  },
  {
    "path": "python/mxnet/gluon/data/vision/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Vision utilities.\"\"\"\n\nfrom .datasets import *\n\nfrom . import transforms\n"
  },
  {
    "path": "python/mxnet/gluon/data/vision/datasets.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=\n\"\"\"Dataset container.\"\"\"\n__all__ = ['MNIST', 'FashionMNIST', 'CIFAR10', 'CIFAR100',\n           'ImageRecordDataset', 'ImageFolderDataset', 'ImageListDataset']\n\nimport os\nimport gzip\nimport tarfile\nimport struct\nimport warnings\nimport numpy as np\n\nfrom .. import dataset\nfrom ...utils import download, check_sha1, _get_repo_file_url\nfrom .... import ndarray as nd, image, recordio, base\nfrom .... import numpy as _mx_np  # pylint: disable=reimported\nfrom ....util import is_np_array, default_array\nfrom ....base import numeric_types\n\n\nclass MNIST(dataset._DownloadedDataset):\n    \"\"\"MNIST handwritten digits dataset from http://yann.lecun.com/exdb/mnist\n\n    Each sample is an image (in 3D NDArray) with shape (28, 28, 1).\n\n    Parameters\n    ----------\n    root : str, default $MXNET_HOME/datasets/mnist\n        Path to temp folder for storing data.\n    train : bool, default True\n        Whether to load the training or testing set.\n    transform : function, default None\n        DEPRECATED FUNCTION ARGUMENTS.\n        A user defined callback that transforms each sample. For example::\n\n            transform=lambda data, label: (data.astype(np.float32)/255, label)\n\n    \"\"\"\n    def __init__(self, root=os.path.join(base.data_dir(), 'datasets', 'mnist'),\n                 train=True, transform=None):\n        self._train = train\n        self._train_data = ('train-images-idx3-ubyte.gz',\n                            '6c95f4b05d2bf285e1bfb0e7960c31bd3b3f8a7d')\n        self._train_label = ('train-labels-idx1-ubyte.gz',\n                             '2a80914081dc54586dbdf242f9805a6b8d2a15fc')\n        self._test_data = ('t10k-images-idx3-ubyte.gz',\n                           'c3a25af1f52dad7f726cce8cacb138654b760d48')\n        self._test_label = ('t10k-labels-idx1-ubyte.gz',\n                            '763e7fa3757d93b0cdec073cef058b2004252c17')\n        self._namespace = 'mnist'\n        super(MNIST, self).__init__(root, transform)\n\n    def _get_data(self):\n        if self._train:\n            data, label = self._train_data, self._train_label\n        else:\n            data, label = self._test_data, self._test_label\n\n        namespace = 'gluon/dataset/'+self._namespace\n        data_file = download(_get_repo_file_url(namespace, data[0]),\n                             path=self._root,\n                             sha1_hash=data[1])\n        label_file = download(_get_repo_file_url(namespace, label[0]),\n                              path=self._root,\n                              sha1_hash=label[1])\n\n        with gzip.open(label_file, 'rb') as fin:\n            struct.unpack(\">II\", fin.read(8))\n            label = np.frombuffer(fin.read(), dtype=np.uint8).astype(np.int32)\n            if is_np_array():\n                label = _mx_np.array(label, dtype=label.dtype)\n\n        with gzip.open(data_file, 'rb') as fin:\n            struct.unpack(\">IIII\", fin.read(16))\n            data = np.frombuffer(fin.read(), dtype=np.uint8)\n            data = data.reshape(len(label), 28, 28, 1)\n\n        array_fn = _mx_np.array if is_np_array() else nd.array\n        self._data = array_fn(data, dtype=data.dtype)\n        self._label = label\n\n\nclass FashionMNIST(MNIST):\n    \"\"\"A dataset of Zalando's article images consisting of fashion products,\n    a drop-in replacement of the original MNIST dataset from\n    https://github.com/zalandoresearch/fashion-mnist\n\n    Each sample is an image (in 3D NDArray) with shape (28, 28, 1).\n\n    Parameters\n    ----------\n    root : str, default $MXNET_HOME/datasets/fashion-mnist'\n        Path to temp folder for storing data.\n    train : bool, default True\n        Whether to load the training or testing set.\n    transform : function, default None\n        DEPRECATED FUNCTION ARGUMENTS.\n        A user defined callback that transforms each sample. For example::\n\n            transform=lambda data, label: (data.astype(np.float32)/255, label)\n\n    \"\"\"\n    def __init__(self, root=os.path.join(base.data_dir(), 'datasets', 'fashion-mnist'),\n                 train=True, transform=None):\n        self._train = train\n        self._train_data = ('train-images-idx3-ubyte.gz',\n                            '0cf37b0d40ed5169c6b3aba31069a9770ac9043d')\n        self._train_label = ('train-labels-idx1-ubyte.gz',\n                             '236021d52f1e40852b06a4c3008d8de8aef1e40b')\n        self._test_data = ('t10k-images-idx3-ubyte.gz',\n                           '626ed6a7c06dd17c0eec72fa3be1740f146a2863')\n        self._test_label = ('t10k-labels-idx1-ubyte.gz',\n                            '17f9ab60e7257a1620f4ad76bbbaf857c3920701')\n        self._namespace = 'fashion-mnist'\n        super(MNIST, self).__init__(root, transform) # pylint: disable=bad-super-call\n\n\nclass CIFAR10(dataset._DownloadedDataset):\n    \"\"\"CIFAR10 image classification dataset from https://www.cs.toronto.edu/~kriz/cifar.html\n\n    Each sample is an image (in 3D NDArray) with shape (32, 32, 3).\n\n    Parameters\n    ----------\n    root : str, default $MXNET_HOME/datasets/cifar10\n        Path to temp folder for storing data.\n    train : bool, default True\n        Whether to load the training or testing set.\n    transform : function, default None\n        DEPRECATED FUNCTION ARGUMENTS.\n        A user defined callback that transforms each sample. For example::\n\n            transform=lambda data, label: (data.astype(np.float32)/255, label)\n\n    \"\"\"\n    def __init__(self, root=os.path.join(base.data_dir(), 'datasets', 'cifar10'),\n                 train=True, transform=None):\n        self._train = train\n        self._archive_file = ('cifar-10-binary.tar.gz', 'fab780a1e191a7eda0f345501ccd62d20f7ed891')\n        self._train_data = [('data_batch_1.bin', 'aadd24acce27caa71bf4b10992e9e7b2d74c2540'),\n                            ('data_batch_2.bin', 'c0ba65cce70568cd57b4e03e9ac8d2a5367c1795'),\n                            ('data_batch_3.bin', '1dd00a74ab1d17a6e7d73e185b69dbf31242f295'),\n                            ('data_batch_4.bin', 'aab85764eb3584312d3c7f65fd2fd016e36a258e'),\n                            ('data_batch_5.bin', '26e2849e66a845b7f1e4614ae70f4889ae604628')]\n        self._test_data = [('test_batch.bin', '67eb016db431130d61cd03c7ad570b013799c88c')]\n        self._namespace = 'cifar10'\n        super(CIFAR10, self).__init__(root, transform)\n\n    def _read_batch(self, filename):\n        with open(filename, 'rb') as fin:\n            data = np.frombuffer(fin.read(), dtype=np.uint8).reshape(-1, 3072+1)\n\n        return data[:, 1:].reshape(-1, 3, 32, 32).transpose(0, 2, 3, 1), \\\n               data[:, 0].astype(np.int32)\n\n    def _get_data(self):\n        if any(not os.path.exists(path) or not check_sha1(path, sha1)\n               for path, sha1 in ((os.path.join(self._root, name), sha1)\n                                  for name, sha1 in self._train_data + self._test_data)):\n            namespace = 'gluon/dataset/'+self._namespace\n            filename = download(_get_repo_file_url(namespace, self._archive_file[0]),\n                                path=self._root,\n                                sha1_hash=self._archive_file[1])\n\n            with tarfile.open(filename) as tar:\n                tar.extractall(self._root)\n\n        if self._train:\n            data_files = self._train_data\n        else:\n            data_files = self._test_data\n        data, label = zip(*(self._read_batch(os.path.join(self._root, name))\n                            for name, _ in data_files))\n        data = np.concatenate(data)\n        label = np.concatenate(label)\n\n        array_fn = _mx_np.array if is_np_array() else nd.array\n        self._data = array_fn(data, dtype=data.dtype)\n        self._label = array_fn(label, dtype=label.dtype) if is_np_array() else label\n\n\nclass CIFAR100(CIFAR10):\n    \"\"\"CIFAR100 image classification dataset from https://www.cs.toronto.edu/~kriz/cifar.html\n\n    Each sample is an image (in 3D NDArray) with shape (32, 32, 3).\n\n    Parameters\n    ----------\n    root : str, default $MXNET_HOME/datasets/cifar100\n        Path to temp folder for storing data.\n    fine_label : bool, default False\n        Whether to load the fine-grained (100 classes) or coarse-grained (20 super-classes) labels.\n    train : bool, default True\n        Whether to load the training or testing set.\n    transform : function, default None\n        DEPRECATED FUNCTION ARGUMENTS.\n        A user defined callback that transforms each sample. For example::\n\n            transform=lambda data, label: (data.astype(np.float32)/255, label)\n\n    \"\"\"\n    def __init__(self, root=os.path.join(base.data_dir(), 'datasets', 'cifar100'),\n                 fine_label=False, train=True, transform=None):\n        self._train = train\n        self._archive_file = ('cifar-100-binary.tar.gz', 'a0bb982c76b83111308126cc779a992fa506b90b')\n        self._train_data = [('train.bin', 'e207cd2e05b73b1393c74c7f5e7bea451d63e08e')]\n        self._test_data = [('test.bin', '8fb6623e830365ff53cf14adec797474f5478006')]\n        self._fine_label = fine_label\n        self._namespace = 'cifar100'\n        super(CIFAR10, self).__init__(root, transform) # pylint: disable=bad-super-call\n\n    def _read_batch(self, filename):\n        with open(filename, 'rb') as fin:\n            data = np.frombuffer(fin.read(), dtype=np.uint8).reshape(-1, 3072+2)\n\n        return data[:, 2:].reshape(-1, 3, 32, 32).transpose(0, 2, 3, 1), \\\n               data[:, 0+self._fine_label].astype(np.int32)\n\n\nclass ImageRecordDataset(dataset.RecordFileDataset):\n    \"\"\"A dataset wrapping over a RecordIO file containing images.\n\n    Each sample is an image and its corresponding label.\n\n    Parameters\n    ----------\n    filename : str\n        Path to rec file.\n    flag : {0, 1}, default 1\n        If 0, always convert images to greyscale. \\\n        If 1, always convert images to colored (RGB).\n    transform : function, default None\n        DEPRECATED FUNCTION ARGUMENTS.\n        A user defined callback that transforms each sample. For example::\n\n            transform=lambda data, label: (data.astype(np.float32)/255, label)\n\n    \"\"\"\n    def __init__(self, filename, flag=1, transform=None):\n        super(ImageRecordDataset, self).__init__(filename)\n        if transform is not None:\n            raise DeprecationWarning(\n                'Directly apply transform to dataset is deprecated. '\n                'Please use dataset.transform() or dataset.transform_first() instead...')\n        self._flag = flag\n        self._transform = transform\n\n    def __getitem__(self, idx):\n        record = super(ImageRecordDataset, self).__getitem__(idx)\n        header, img = recordio.unpack(record)\n        if self._transform is not None:\n            return self._transform(image.imdecode(img, self._flag), header.label)\n        return image.imdecode(img, self._flag), header.label\n\n    def __mx_handle__(self):\n        from .._internal import ImageRecordFileDataset as _ImageRecordFileDataset\n        return _ImageRecordFileDataset(rec_file=self.filename, idx_file=self.idx_file,\n                                       flag=self._flag)\n\n\nclass ImageFolderDataset(dataset.Dataset):\n    \"\"\"A dataset for loading image files stored in a folder structure.\n\n    like::\n\n        root/car/0001.jpg\n        root/car/xxxa.jpg\n        root/car/yyyb.jpg\n        root/bus/123.jpg\n        root/bus/023.jpg\n        root/bus/wwww.jpg\n\n    Parameters\n    ----------\n    root : str\n        Path to root directory.\n    flag : {0, 1}, default 1\n        If 0, always convert loaded images to greyscale (1 channel).\n        If 1, always convert loaded images to colored (3 channels).\n    transform : callable, default None\n        DEPRECATED FUNCTION ARGUMENTS.\n        A function that takes data and label and transforms them::\n\n            transform = lambda data, label: (data.astype(np.float32)/255, label)\n\n    Attributes\n    ----------\n    synsets : list\n        List of class names. `synsets[i]` is the name for the integer label `i`\n    items : list of tuples\n        List of all images in (filename, label) pairs.\n    \"\"\"\n    def __init__(self, root, flag=1, transform=None):\n        self._root = os.path.expanduser(root)\n        self._flag = flag\n        if transform is not None:\n            raise DeprecationWarning(\n                'Directly apply transform to dataset is deprecated. '\n                'Please use dataset.transform() or dataset.transform_first() instead...')\n        self._transform = transform\n        self._exts = ['.jpg', '.jpeg', '.png']\n        self._list_images(self._root)\n        self._handle = None\n\n    def _list_images(self, root):\n        self.synsets = []\n        self.items = []\n\n        for folder in sorted(os.listdir(root)):\n            path = os.path.join(root, folder)\n            if not os.path.isdir(path):\n                warnings.warn(f'Ignoring {path}, which is not a directory.', stacklevel=3)\n                continue\n            label = len(self.synsets)\n            self.synsets.append(folder)\n            for filename in sorted(os.listdir(path)):\n                filename = os.path.join(path, filename)\n                ext = os.path.splitext(filename)[1]\n                if ext.lower() not in self._exts:\n                    warnings.warn(f'Ignoring {filename} of type {ext}. Only support {\", \".join(self._exts)}')\n                    continue\n                self.items.append((filename, label))\n\n    def __getitem__(self, idx):\n        img = image.imread(self.items[idx][0], self._flag)\n        label = self.items[idx][1]\n        if self._transform is not None:\n            return self._transform(img, label)\n        return img, label\n\n    def __len__(self):\n        return len(self.items)\n\n    def __mx_handle__(self):\n        if self._handle is None:\n            from .._internal import ImageSequenceDataset, NDArrayDataset, GroupDataset\n            path_sep = '|'\n            im_names = path_sep.join([x[0] for x in self.items])\n            label = default_array([x[1] for x in self.items])\n            self._handle = GroupDataset(datasets=(\n                ImageSequenceDataset(img_list=im_names, path_sep=path_sep, flag=self._flag),\n                NDArrayDataset(arr=label)))\n        return self._handle\n\n\nclass ImageListDataset(dataset.Dataset):\n    \"\"\"A dataset for loading image files specified by a list of entries.\n\n    like::\n\n        # if written to text file *.lst\n        0\\t0\\troot/car/0001.jpg\n        1\\t0\\troot/car/xxxa.jpg\n        2\\t0\\troot/car/yyyb.jpg\n        3\\t1\\troot/bus/123.jpg\n        4\\t1\\troot/bus/023.jpg\n        5\\t1\\troot/bus/wwww.jpg\n\n        # if as a pure list, each item is a list [imagelabel: float or list of float, imgpath]\n        [[0, root/car/0001.jpg]\n         [0, root/car/xxxa.jpg]\n         [0, root/car/yyyb.jpg]\n         [1, root/bus/123.jpg]\n         [1, root/bus/023.jpg]\n         [1, root/bus/wwww.jpg]]\n\n    Parameters\n    ----------\n    root : str\n        Path to root directory.\n    imglist : str or list\n        Specify the path of imglist file or a list directly\n    flag : {0, 1}, default 1\n        If 0, always convert loaded images to greyscale (1 channel).\n        If 1, always convert loaded images to colored (3 channels).\n\n    Attributes\n    ----------\n    items : list of tuples\n        List of all images in (filename, label) pairs.\n    \"\"\"\n    def __init__(self, root='.', imglist=None, flag=1):\n        self._root = os.path.expanduser(root)\n        self._flag = flag\n        self._imglist = {}\n        self._imgkeys = []\n        self._handle = None\n        array_fn = _mx_np.array if is_np_array() else nd.array\n        if isinstance(imglist, str):\n            # read from file\n            fname = os.path.join(self._root, imglist)\n            with open(fname, 'rt') as fin:\n                for line in iter(fin.readline, ''):\n                    line = line.strip().split('\\t')\n                    label = array_fn(line[1:-1])\n                    key = int(line[0])\n                    self._imglist[key] = (label, os.path.join(self._root, line[-1]))\n                    self._imgkeys.append(key)\n        elif isinstance(imglist, list):\n            index = 1\n            for img in imglist:\n                key = str(index)\n                index += 1\n                if len(img) > 2:\n                    label = array_fn(img[:-1])\n                elif isinstance(img[0], numeric_types):\n                    label = array_fn([img[0]])\n                else:\n                    label = array_fn(img[0])\n                assert isinstance(img[-1], str)\n                self._imglist[key] = (label, os.path.join(self._root, img[-1]))\n                self._imgkeys.append(key)\n        else:\n            raise ValueError(\n                \"imglist must be filename or list of valid entries, given {}\".format(\n                    type(imglist)))\n\n    def __getitem__(self, idx):\n        key = self._imgkeys[idx]\n        img = image.imread(self._imglist[key][1], self._flag)\n        label = self._imglist[key][0]\n        return img, label\n\n    def __len__(self):\n        return len(self._imgkeys)\n\n    def __mx_handle__(self):\n        if self._handle is None:\n            from .._internal import ImageSequenceDataset, NDArrayDataset, GroupDataset\n            path_sep = '|'\n            im_names = path_sep.join([self._imglist[x][1] for x in self._imgkeys])\n            label = default_array(np.array([self._imglist[x][0].asnumpy() for x in self._imgkeys]))\n            self._handle = GroupDataset(datasets=(\n                ImageSequenceDataset(img_list=im_names, path_sep=path_sep, flag=self._flag),\n                NDArrayDataset(arr=label)))\n        return self._handle\n"
  },
  {
    "path": "python/mxnet/gluon/data/vision/transforms/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable= arguments-differ, wildcard-import\n\"Vision transforms.\"\n\nimport warnings\nimport random\n\nfrom ....block import Block, HybridBlock\nfrom ....nn import Sequential, HybridSequential\nfrom .....util import use_np\nfrom ..... import np, npx\n\nfrom . image import *\nfrom .image import _append_return\n\n\nclass Compose(Sequential):\n    \"\"\"Sequentially composes multiple transforms.\n\n    Parameters\n    ----------\n    transforms : list of transform Blocks.\n        The list of transforms to be composed.\n\n\n    Inputs:\n        - **data**: input tensor with shape of the first transform Block requires.\n\n    Outputs:\n        - **out**: output tensor with shape of the last transform Block produces.\n\n    Examples\n    --------\n    >>> transformer = transforms.Compose([transforms.Resize(300),\n    ...                                   transforms.CenterCrop(256),\n    ...                                   transforms.ToTensor()])\n    >>> image = mx.nd.random.uniform(0, 255, (224, 224, 3)).astype(dtype=np.uint8)\n    >>> transformer(image)\n    <NDArray 3x256x256 @cpu(0)>\n    \"\"\"\n    def __init__(self, transforms):\n        super(Compose, self).__init__()\n        transforms.append(None)\n        hybrid = []\n        for i in transforms:\n            if isinstance(i, HybridBlock):\n                hybrid.append(i)\n                continue\n            elif len(hybrid) == 1:\n                self.add(hybrid[0])\n                hybrid = []\n            elif len(hybrid) > 1:\n                hblock = HybridSequential()\n                for j in hybrid:\n                    hblock.add(j)\n                hblock.hybridize()\n                self.add(hblock)\n                hybrid = []\n\n            if i is not None:\n                self.add(i)\n\n\nclass HybridCompose(HybridSequential):\n    \"\"\"Sequentially composes multiple transforms. This is the Hybrid version of Compose.\n\n    Parameters\n    ----------\n    transforms : list of transform Blocks.\n        The list of transforms to be composed.\n\n\n    Inputs:\n        - **data**: input tensor with shape of the first transform Block requires.\n\n    Outputs:\n        - **out**: output tensor with shape of the last transform Block produces.\n\n    Examples\n    --------\n    >>> transformer = transforms.HybridCompose([transforms.Resize(300),\n    ...                                   transforms.CenterCrop(256),\n    ...                                   transforms.ToTensor()])\n    >>> image = mx.nd.random.uniform(0, 255, (224, 224, 3)).astype(dtype=np.uint8)\n    >>> transformer(image)\n    <NDArray 3x256x256 @cpu(0)>\n    \"\"\"\n    def __init__(self, transforms):\n        super(HybridCompose, self).__init__()\n        for i in transforms:\n            if not isinstance(i, HybridBlock):\n                raise ValueError(\"{} is not a HybridBlock, try use `Compose` instead\".format(i))\n            self.add(i)\n        self.hybridize()\n\n\n@use_np\nclass Cast(HybridBlock):\n    \"\"\"Cast inputs to a specific data type\n\n    Parameters\n    ----------\n    dtype : str, default 'float32'\n        The target data type, in string or `numpy.dtype`.\n\n\n    Inputs:\n        - **data**: input tensor with arbitrary shape and dtype.\n\n    Outputs:\n        - **out**: output tensor with the same shape as `data` and data type as dtype.\n    \"\"\"\n    def __init__(self, dtype='float32'):\n        super(Cast, self).__init__()\n        self._dtype = dtype\n\n    def forward(self, *args):\n        return tuple(x.astype(self._dtype) for x in args)\n\n\nclass RandomApply(Sequential):\n    \"\"\"Apply a list of transformations randomly given probability\n\n    Parameters\n    ----------\n    transforms\n        List of transformations.\n    p : float\n        Probability of applying the transformations.\n\n\n    Inputs:\n        - **data**: input tensor.\n\n    Outputs:\n        - **out**: transformed image.\n    \"\"\"\n\n    def __init__(self, transforms, p=0.5):\n        super(RandomApply, self).__init__()\n        self.transforms = transforms\n        self.p = p\n\n    def forward(self, x, *args):\n        if self.p < random.random():\n            return x\n        x = self.transforms(x)\n        return _append_return(x, *args)\n\n\nclass HybridRandomApply(HybridSequential):\n    \"\"\"Apply a list of transformations randomly given probability\n\n    Parameters\n    ----------\n    transforms\n        List of transformations which must be HybridBlocks.\n    p : float\n        Probability of applying the transformations.\n\n\n    Inputs:\n        - **data**: input tensor.\n\n    Outputs:\n        - **out**: transformed image.\n    \"\"\"\n\n    def __init__(self, transforms, p=0.5):\n        super(HybridRandomApply, self).__init__()\n        assert isinstance(transforms, HybridBlock)\n        self.transforms = transforms\n        self.p = p\n\n    def forward(self, x, *args):\n        cond = lambda p: p < np.random.uniform(low=0, high=1, size=1)\n        return npx.cond(cond, x, self.transforms(x), self.p)\n"
  },
  {
    "path": "python/mxnet/gluon/data/vision/transforms/image.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable= arguments-differ\n\"Image transforms.\"\nimport numpy as onp\n\nfrom ....block import Block, HybridBlock\nfrom ..... import image\nfrom .....base import numeric_types\nfrom .....util import use_np\nfrom ..... import np, npx\n\n__all__ = ['ToTensor', 'Normalize', 'Rotate', 'RandomRotation',\n           'RandomResizedCrop', 'CropResize', 'CropResize', 'RandomCrop',\n           'CenterCrop', 'Resize', 'RandomFlipLeftRight', 'RandomFlipTopBottom',\n           'RandomBrightness', 'RandomContrast', 'RandomSaturation', 'RandomHue',\n           'RandomColorJitter', 'RandomLighting', 'RandomGray']\n\ndef _append_return(*args):\n    \"\"\"Append multiple args together.\n    This allows many transform functions to bypass additional arguments.\n    \"\"\"\n    if args:\n        if len(args) == 1:\n            return args[0]\n        return tuple(args)\n    return None\n\n\n@use_np\nclass ToTensor(HybridBlock):\n    \"\"\"Converts an image NDArray or batch of image NDArray to a tensor NDArray.\n\n    Converts an image NDArray of shape (H x W x C) in the range\n    [0, 255] to a float32 tensor NDArray of shape (C x H x W) in\n    the range [0, 1].\n\n    If batch input, converts a batch image NDArray of shape (N x H x W x C) in the\n    range [0, 255] to a float32 tensor NDArray of shape (N x C x H x W).\n\n    Inputs:\n        - **data**: input tensor with (H x W x C) or (N x H x W x C) shape and uint8 type.\n\n    Outputs:\n        - **out**: output tensor with (C x H x W) or (N x C x H x W) shape and float32 type.\n\n    Examples\n    --------\n    >>> transformer = vision.transforms.ToTensor()\n    >>> image = mx.nd.random.uniform(0, 255, (4, 2, 3)).astype(dtype=np.uint8)\n    >>> transformer(image)\n    [[[ 0.85490197  0.72156864]\n      [ 0.09019608  0.74117649]\n      [ 0.61960787  0.92941177]\n      [ 0.96470588  0.1882353 ]]\n     [[ 0.6156863   0.73725492]\n      [ 0.46666667  0.98039216]\n      [ 0.44705883  0.45490196]\n      [ 0.01960784  0.8509804 ]]\n     [[ 0.39607844  0.03137255]\n      [ 0.72156864  0.52941179]\n      [ 0.16470589  0.7647059 ]\n      [ 0.05490196  0.70588237]]]\n    <NDArray 3x4x2 @cpu(0)>\n    \"\"\"\n    def __init__(self):\n        super(ToTensor, self).__init__()\n\n    def forward(self, x, *args):\n        return _append_return(npx.image.to_tensor(x), *args)\n\n\n@use_np\nclass Normalize(HybridBlock):\n    \"\"\"Normalize an tensor of shape (C x H x W) or (N x C x H x W) with mean and\n    standard deviation.\n\n    Given mean `(m1, ..., mn)` and std `(s1, ..., sn)` for `n` channels,\n    this transform normalizes each channel of the input tensor with::\n\n        output[i] = (input[i] - mi) / si\n\n    If mean or std is scalar, the same value will be applied to all channels.\n\n    Parameters\n    ----------\n    mean : float or tuple of floats\n        The mean values.\n    std : float or tuple of floats\n        The standard deviation values.\n\n\n    Inputs:\n        - **data**: input tensor with (C x H x W) or (N x C x H x W) shape.\n\n    Outputs:\n        - **out**: output tensor with the shape as `data`.\n\n    Examples\n    --------\n    >>> transformer = transforms.Normalize(mean=(0, 1, 2), std=(3, 2, 1))\n    >>> image = mx.nd.random.uniform(0, 1, (3, 4, 2))\n    >>> transformer(image)\n    [[[ 0.18293785  0.19761486]\n      [ 0.23839645  0.28142193]\n      [ 0.20092112  0.28598186]\n      [ 0.18162774  0.28241724]]\n     [[-0.2881726  -0.18821815]\n      [-0.17705294 -0.30780914]\n      [-0.2812064  -0.3512327 ]\n      [-0.05411351 -0.4716435 ]]\n     [[-1.0363373  -1.7273437 ]\n      [-1.6165586  -1.5223348 ]\n      [-1.208275   -1.1878313 ]\n      [-1.4711051  -1.5200229 ]]]\n    <NDArray 3x4x2 @cpu(0)>\n    \"\"\"\n    def __init__(self, mean=0.0, std=1.0):\n        super(Normalize, self).__init__()\n        self._mean = mean\n        self._std = std\n\n    def forward(self, x, *args):\n        return _append_return(npx.image.normalize(x, self._mean, self._std), *args)\n\n\n@use_np\nclass Rotate(Block):\n    \"\"\"Rotate the input image by a given angle. Keeps the original image shape.\n\n    Parameters\n    ----------\n    rotation_degrees : float32\n        Desired rotation angle in degrees.\n    zoom_in : bool\n        Zoom in image so that no padding is present in final output.\n    zoom_out : bool\n        Zoom out image so that the entire original image is present in final output.\n\n\n    Inputs:\n        - **data**: input tensor with (C x H x W) or (N x C x H x W) shape.\n\n    Outputs:\n        - **out**: output tensor with (C x H x W) or (N x C x H x W) shape.\n    \"\"\"\n    def __init__(self, rotation_degrees, zoom_in=False, zoom_out=False):\n        super(Rotate, self).__init__()\n        self._args = (rotation_degrees, zoom_in, zoom_out)\n\n    def forward(self, x, *args):\n        if onp.dtype(x.dtype) is not onp.dtype(onp.float32):\n            raise TypeError(\"This transformation only supports float32. \"\n                            \"Consider calling it after ToTensor, given: {}\".format(x.dtype))\n        return _append_return(image.imrotate(x, *self._args), *args)\n\n\n@use_np\nclass RandomRotation(Block):\n    \"\"\"Random rotate the input image by a random angle.\n       Keeps the original image shape and aspect ratio.\n\n    Parameters\n    ----------\n    angle_limits: tuple\n        Tuple of 2 elements containing the upper and lower limit\n        for rotation angles in degree.\n    zoom_in : bool\n        Zoom in image so that no padding is present in final output.\n    zoom_out : bool\n        Zoom out image so that the entire original image is present in final output.\n    rotate_with_proba : float32\n\n\n    Inputs:\n        - **data**: input tensor with (C x H x W) or (N x C x H x W) shape.\n\n    Outputs:\n        - **out**: output tensor with (C x H x W) or (N x C x H x W) shape.\n    \"\"\"\n    def __init__(self, angle_limits, zoom_in=False, zoom_out=False, rotate_with_proba=1.0):\n        super(RandomRotation, self).__init__()\n        lower, upper = angle_limits\n        if lower >= upper:\n            raise ValueError(\"`angle_limits` must be an ordered tuple\")\n        if rotate_with_proba < 0 or rotate_with_proba > 1:\n            raise ValueError(\"Probability of rotating the image should be between 0 and 1\")\n        self._args = (angle_limits, zoom_in, zoom_out)\n        self._rotate_with_proba = rotate_with_proba\n\n    def forward(self, x, *args):\n        if onp.random.random() > self._rotate_with_proba:\n            return x\n        if onp.dtype(x.dtype) is not onp.dtype(onp.float32):\n            raise TypeError(\"This transformation only supports float32. \"\n                            \"Consider calling it after ToTensor\")\n        return _append_return(image.random_rotate(x, *self._args), *args)\n\n\n@use_np\nclass RandomResizedCrop(HybridBlock):\n    \"\"\"Crop the input image with random scale and aspect ratio.\n\n    Makes a crop of the original image with random size (default: 0.08\n    to 1.0 of the original image size) and random aspect ratio (default:\n    3/4 to 4/3), then resize it to the specified size.\n\n    Parameters\n    ----------\n    size : int or tuple of (W, H)\n        Size of the final output.\n    scale : tuple of two floats\n        If scale is `(min_area, max_area)`, the cropped image's area will\n        range from min_area to max_area of the original image's area\n    ratio : tuple of two floats\n        Range of aspect ratio of the cropped image before resizing.\n    interpolation : int\n        Interpolation method for resizing. By default uses bilinear\n        interpolation. See OpenCV's resize function for available choices.\n\n\n    Inputs:\n        - **data**: input tensor with (Hi x Wi x C) shape.\n\n    Outputs:\n        - **out**: output tensor with (H x W x C) shape.\n    \"\"\"\n    def __init__(self, size, scale=(0.08, 1.0), ratio=(3.0/4.0, 4.0/3.0),\n                 interpolation=1):\n        super(RandomResizedCrop, self).__init__()\n        if isinstance(size, numeric_types):\n            size = (size, size)\n        if isinstance(scale, numeric_types):\n            scale = (scale, 1.0)\n        self._kwargs = {'width': size[0], 'height': size[1],\n                        'area': scale, 'ratio': ratio,\n                        'interp': interpolation, 'max_trial': 10}\n\n    def forward(self, x, *args):\n        return _append_return(npx.image.random_resized_crop(x, **self._kwargs), *args)\n\n\n@use_np\nclass CropResize(HybridBlock):\n    r\"\"\"Crop the input image with and optionally resize it.\n\n    Makes a crop of the original image then optionally resize it to the specified size.\n\n    Parameters\n    ----------\n    x : int\n        Left boundary of the cropping area\n    y : int\n        Top boundary of the cropping area\n    w : int\n        Width of the cropping area\n    h : int\n        Height of the cropping area\n    size : int or tuple of (w, h)\n        Optional, resize to new size after cropping\n    interpolation : int, optional\n        Interpolation method for resizing. By default uses bilinear\n        interpolation. See OpenCV's resize function for available choices.\n        https://docs.opencv.org/2.4/modules/imgproc/doc/geometric_transformations.html?highlight=resize#resize\n        Note that the Resize on gpu use contrib.bilinearResize2D operator\n        which only support bilinear interpolation(1).\n\n\n    Inputs:\n        - **data**: input tensor with (H x W x C) or (N x H x W x C) shape.\n\n    Outputs:\n        - **out**: input tensor with (H x W x C) or (N x H x W x C) shape.\n\n    Examples\n    --------\n    >>> transformer = vision.transforms.CropResize(x=0, y=0, width=100, height=100)\n    >>> image = mx.nd.random.uniform(0, 255, (224, 224, 3)).astype(dtype=np.uint8)\n    >>> transformer(image)\n    <NDArray 100x100x3 @cpu(0)>\n    >>> image = mx.nd.random.uniform(0, 255, (3, 224, 224, 3)).astype(dtype=np.uint8)\n    >>> transformer(image)\n    <NDArray 3x100x100x3 @cpu(0)>\n    >>> transformer = vision.transforms.CropResize(x=0, y=0, width=100, height=100, size=(50, 50), interpolation=1)\n    >>> transformer(image)\n    <NDArray 3x50x50 @cpu(0)>\n    \"\"\"\n    def __init__(self, x, y, width, height, size=None, interpolation=None):\n        super(CropResize, self).__init__()\n        self._x = x\n        self._y = y\n        self._width = width\n        self._height = height\n        self._size = size\n        self._interpolation = interpolation\n\n    def forward(self, x, *args):\n        out = npx.image.crop(x, self._x, self._y, self._width, self._height)\n        if self._size:\n            out = npx.image.resize(out, self._size, False, self._interpolation)\n        return _append_return(out, *args)\n\n@use_np\nclass RandomCrop(HybridBlock):\n    \"\"\"Randomly crop `src` with `size` (width, height).\n    Padding is optional.\n    Upsample result if `src` is smaller than `size`\n    .\n    Parameters\n    ----------\n    size : int or tuple of (W, H)\n        Size of the final output.\n    pad: int or tuple\n        if int, size of the zero-padding\n        if tuple, number of values padded to the edges of each axis.\n            ((before_1, after_1), ... (before_N, after_N)) unique pad widths for each axis.\n            ((before, after),) yields same before and after pad for each axis.\n            (pad,) or int is a shortcut for before = after = pad width for all axes.\n    pad_value : int\n        The value to use for padded pixels\n    interpolation : int\n        Interpolation method for resizing. By default uses bilinear\n        interpolation. See OpenCV's resize function for available choices.\n    Inputs:\n        - **data**: input tensor with (Hi x Wi x C) shape.\n    Outputs:\n        - **out**: output tensor with ((H+2*pad) x (W+2*pad) x C) shape.\n    \"\"\"\n\n    def __init__(self, size, pad=None, pad_value=0, interpolation=1):\n        super(RandomCrop, self).__init__()\n        if isinstance(size, numeric_types):\n            size = (size, size)\n        self._args = ((0, 1), (0, 1), size[0], size[1], interpolation)\n        self._pad_value = pad_value\n        if isinstance(pad, int):\n            self.nd_pad = (0, 0, 0, 0, pad, pad, pad, pad, 0, 0)  # workaround as 5D\n            self.np_pad = ((pad, pad), (pad, pad), (0, 0))\n        elif pad is not None:\n            assert len(pad) >= 4\n            self.nd_pad = tuple([0] * 4 + list(pad) + [0] * (6 - len(pad)))\n            self.np_pad = ((pad[0], pad[1]), (pad[2], pad[3]), (0, 0))\n        else:\n            self.nd_pad = pad\n            self.np_pad = pad\n\n    def forward(self, x, *args):\n        if self.np_pad:\n            x = np.pad(x, pad_width=self.np_pad, mode='constant', constant_values=self._pad_value)\n        # pylint: disable=too-many-function-args\n        return _append_return(npx.image.random_crop(x, *self._args), *args)\n\n@use_np\nclass CenterCrop(HybridBlock):\n    \"\"\"Crops the image `src` to the given `size` by trimming on all four\n    sides and preserving the center of the image. Upsamples if `src` is\n    smaller than `size`.\n\n    Parameters\n    ----------\n    size : int or tuple of (W, H)\n        Size of output image.\n    interpolation : int\n        Interpolation method for resizing. By default uses bilinear\n        interpolation. See OpenCV's resize function for available choices.\n\n\n    Inputs:\n        - **data**: input tensor with (Hi x Wi x C) shape.\n\n    Outputs:\n        - **out**: output tensor with (H x W x C) shape.\n\n    Examples\n    --------\n    >>> transformer = vision.transforms.CenterCrop(size=(1000, 500))\n    >>> image = mx.nd.random.uniform(0, 255, (2321, 3482, 3)).astype(dtype=np.uint8)\n    >>> transformer(image)\n    <NDArray 500x1000x3 @cpu(0)>\n    \"\"\"\n    def __init__(self, size, interpolation=1):\n        super(CenterCrop, self).__init__()\n        if isinstance(size, numeric_types):\n            size = (size, size)\n        self._args = (size[0], size[1], interpolation)\n\n    def forward(self, x, *args):\n        # pylint: disable=too-many-function-args\n        return _append_return(npx.image.random_crop(x, (0.5, 0.5), (0.5, 0.5), *self._args), *args)\n\n\n@use_np\nclass Resize(HybridBlock):\n    \"\"\"Resize an image or a batch of image NDArray to the given size.\n    Should be applied before `mxnet.gluon.data.vision.transforms.ToTensor`.\n\n    Parameters\n    ----------\n    size : int or tuple of (W, H)\n        Size of output image.\n    keep_ratio : bool\n        Whether to resize the short edge or both edges to `size`,\n        if size is give as an integer.\n    interpolation : int\n        Interpolation method for resizing. By default uses bilinear\n        interpolation. See OpenCV's resize function for available choices.\n        Note that the Resize on gpu use contrib.bilinearResize2D operator\n        which only support bilinear interpolation(1).\n\n\n    Inputs:\n        - **data**: input tensor with (H x W x C) or (N x H x W x C) shape.\n\n    Outputs:\n        - **out**: output tensor with (H x W x C) or (N x H x W x C) shape.\n\n    Examples\n    --------\n    >>> transformer = vision.transforms.Resize(size=(1000, 500))\n    >>> image = mx.nd.random.uniform(0, 255, (224, 224, 3)).astype(dtype=np.uint8)\n    >>> transformer(image)\n    <NDArray 500x1000x3 @cpu(0)>\n    >>> image = mx.nd.random.uniform(0, 255, (3, 224, 224, 3)).astype(dtype=np.uint8)\n    >>> transformer(image)\n    <NDArray 3x500x1000x3 @cpu(0)>\n    \"\"\"\n    def __init__(self, size, keep_ratio=False, interpolation=1):\n        super(Resize, self).__init__()\n        self._keep = keep_ratio\n        self._size = size\n        self._interpolation = interpolation\n\n    def forward(self, x, *args):\n        return _append_return(npx.image.resize(x, self._size, self._keep, self._interpolation), *args)\n\n@use_np\nclass RandomFlipLeftRight(HybridBlock):\n    \"\"\"Randomly flip the input image left to right with a probability\n    of p(0.5 by default).\n\n    Inputs:\n        - **data**: input tensor with (H x W x C) shape.\n\n    Outputs:\n        - **out**: output tensor with same shape as `data`.\n    \"\"\"\n    def __init__(self, p=0.5):\n        super(RandomFlipLeftRight, self).__init__()\n        self.p = p\n\n    def forward(self, x, *args):\n        if self.p <= 0:\n            return _append_return(x, *args)\n\n        if self.p >= 1:\n            return _append_return(npx.image.flip_left_right(x), *args)\n        return _append_return(npx.image.random_flip_left_right(x, p=self.p), *args)\n\n\n@use_np\nclass RandomFlipTopBottom(HybridBlock):\n    \"\"\"Randomly flip the input image top to bottom with a probability\n    of p(0.5 by default).\n\n    Inputs:\n        - **data**: input tensor with (H x W x C) shape.\n\n    Outputs:\n        - **out**: output tensor with same shape as `data`.\n    \"\"\"\n    def __init__(self, p=0.5):\n        super(RandomFlipTopBottom, self).__init__()\n        self.p = p\n\n    def forward(self, x, *args):\n        if self.p <= 0:\n            return _append_return(x, *args)\n\n        if self.p >= 1:\n            return _append_return(npx.image.flip_top_bottom(x), *args)\n        return _append_return(npx.image.random_flip_top_bottom(x, p=self.p), *args)\n\n\n@use_np\nclass RandomBrightness(HybridBlock):\n    \"\"\"Randomly jitters image brightness with a factor\n    chosen from `[max(0, 1 - brightness), 1 + brightness]`.\n\n    Parameters\n    ----------\n    brightness: float\n        How much to jitter brightness. brightness factor is randomly\n        chosen from `[max(0, 1 - brightness), 1 + brightness]`.\n\n\n    Inputs:\n        - **data**: input tensor with (H x W x C) shape.\n\n    Outputs:\n        - **out**: output tensor with same shape as `data`.\n    \"\"\"\n    def __init__(self, brightness):\n        super(RandomBrightness, self).__init__()\n        self._args = (max(0, 1-brightness), 1+brightness)\n\n    def forward(self, x, *args):\n        return _append_return(npx.image.random_brightness(x, *self._args), *args)\n\n\n@use_np\nclass RandomContrast(HybridBlock):\n    \"\"\"Randomly jitters image contrast with a factor\n    chosen from `[max(0, 1 - contrast), 1 + contrast]`.\n\n    Parameters\n    ----------\n    contrast: float\n        How much to jitter contrast. contrast factor is randomly\n        chosen from `[max(0, 1 - contrast), 1 + contrast]`.\n\n\n    Inputs:\n        - **data**: input tensor with (H x W x C) shape.\n\n    Outputs:\n        - **out**: output tensor with same shape as `data`.\n    \"\"\"\n    def __init__(self, contrast):\n        super(RandomContrast, self).__init__()\n        self._args = (max(0, 1-contrast), 1+contrast)\n\n    def forward(self, x, *args):\n        return _append_return(npx.image.random_contrast(x, *self._args), *args)\n\n\n@use_np\nclass RandomSaturation(HybridBlock):\n    \"\"\"Randomly jitters image saturation with a factor\n    chosen from `[max(0, 1 - saturation), 1 + saturation]`.\n\n    Parameters\n    ----------\n    saturation: float\n        How much to jitter saturation. saturation factor is randomly\n        chosen from `[max(0, 1 - saturation), 1 + saturation]`.\n\n\n    Inputs:\n        - **data**: input tensor with (H x W x C) shape.\n\n    Outputs:\n        - **out**: output tensor with same shape as `data`.\n    \"\"\"\n    def __init__(self, saturation):\n        super(RandomSaturation, self).__init__()\n        self._args = (max(0, 1-saturation), 1+saturation)\n\n    def forward(self, x, *args):\n        return _append_return(npx.image.random_saturation(x, *self._args), *args)\n\n\n@use_np\nclass RandomHue(HybridBlock):\n    \"\"\"Randomly jitters image hue with a factor\n    chosen from `[max(0, 1 - hue), 1 + hue]`.\n\n    Parameters\n    ----------\n    hue: float\n        How much to jitter hue. hue factor is randomly\n        chosen from `[max(0, 1 - hue), 1 + hue]`.\n\n\n    Inputs:\n        - **data**: input tensor with (H x W x C) shape.\n\n    Outputs:\n        - **out**: output tensor with same shape as `data`.\n    \"\"\"\n    def __init__(self, hue):\n        super(RandomHue, self).__init__()\n        self._args = (max(0, 1-hue), 1+hue)\n\n    def forward(self, x, *args):\n        return _append_return(npx.image.random_hue(x, *self._args), *args)\n\n\n@use_np\nclass RandomColorJitter(HybridBlock):\n    \"\"\"Randomly jitters the brightness, contrast, saturation, and hue\n    of an image.\n\n    Parameters\n    ----------\n    brightness : float\n        How much to jitter brightness. brightness factor is randomly\n        chosen from `[max(0, 1 - brightness), 1 + brightness]`.\n    contrast : float\n        How much to jitter contrast. contrast factor is randomly\n        chosen from `[max(0, 1 - contrast), 1 + contrast]`.\n    saturation : float\n        How much to jitter saturation. saturation factor is randomly\n        chosen from `[max(0, 1 - saturation), 1 + saturation]`.\n    hue : float\n        How much to jitter hue. hue factor is randomly\n        chosen from `[max(0, 1 - hue), 1 + hue]`.\n\n\n    Inputs:\n        - **data**: input tensor with (H x W x C) shape.\n\n    Outputs:\n        - **out**: output tensor with same shape as `data`.\n    \"\"\"\n    def __init__(self, brightness=0, contrast=0, saturation=0, hue=0):\n        super(RandomColorJitter, self).__init__()\n        self._args = (brightness, contrast, saturation, hue)\n\n    def forward(self, x, *args):\n        return _append_return(npx.image.random_color_jitter(x, *self._args), *args)\n\n\n@use_np\nclass RandomLighting(HybridBlock):\n    \"\"\"Add AlexNet-style PCA-based noise to an image.\n\n    Parameters\n    ----------\n    alpha : float\n        Intensity of the image.\n\n\n    Inputs:\n        - **data**: input tensor with (H x W x C) shape.\n\n    Outputs:\n        - **out**: output tensor with same shape as `data`.\n    \"\"\"\n    def __init__(self, alpha):\n        super(RandomLighting, self).__init__()\n        self._alpha = alpha\n\n    def forward(self, x, *args):\n        return _append_return(npx.image.random_lighting(x, self._alpha), *args)\n\n\n@use_np\nclass RandomGray(HybridBlock):\n    \"\"\"Randomly convert to gray image.\n\n    Parameters\n    ----------\n    p : float\n        Probability to convert to grayscale\n    \"\"\"\n    def __init__(self, p=0.5):\n        super(RandomGray, self).__init__()\n        self.p = p\n\n    def forward(self, x, *args):\n        mat = np.concatenate((np.full((3, 1), 0.2989),\n                              np.full((3, 1), 0.5870),\n                              np.full((3, 1), 0.114)), axis=1)\n        x = x.astype(dtype='float32')\n        gray = np.where(self.p < np.random.uniform(), x, np.dot(x, mat))\n        return _append_return(gray, *args)\n"
  },
  {
    "path": "python/mxnet/gluon/loss.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=arguments-differ\n\"\"\" losses for training neural networks \"\"\"\n__all__ = ['Loss', 'L2Loss', 'L1Loss',\n           'SigmoidBinaryCrossEntropyLoss', 'SigmoidBCELoss',\n           'SoftmaxCrossEntropyLoss', 'SoftmaxCELoss',\n           'KLDivLoss', 'CTCLoss', 'HuberLoss', 'HingeLoss',\n           'SquaredHingeLoss', 'LogisticLoss', 'TripletLoss', 'PoissonNLLLoss', 'CosineEmbeddingLoss', 'SDMLLoss']\n\nimport numpy as _np\nfrom ..base import numeric_types\nfrom .block import HybridBlock\nfrom ..util import use_np\nfrom .. import np, npx\n\n\ndef _apply_weighting(loss, weight=None, sample_weight=None):\n    \"\"\"Apply weighting to loss.\n\n    Parameters\n    ----------\n    loss : Symbol\n        The loss to be weighted.\n    weight : float or None\n        Global scalar weight for loss.\n    sample_weight : Symbol or None\n        Per sample weighting. Must be broadcastable to\n        the same shape as loss. For example, if loss has\n        shape (64, 10) and you want to weight each sample\n        in the batch separately, `sample_weight` should have\n        shape (64, 1).\n\n    Returns\n    -------\n    loss : Symbol\n        Weighted loss\n    \"\"\"\n    if sample_weight is not None:\n        loss = loss * sample_weight\n\n    if weight is not None:\n        assert isinstance(weight, numeric_types), \"weight must be a number\"\n        loss = loss * weight\n\n    return loss\n\n\ndef _batch_mean(loss, batch_axis):\n    \"\"\"Return mean on the specified batch axis, not keeping the axis\"\"\"\n    axes = list(range(loss.ndim))\n    del axes[batch_axis]\n    return np.mean(loss, axis=axes)\n\ndef _batch_sum(loss, batch_axis):\n    \"\"\"Return sum on the specified batch axis, not keeping the axis\"\"\"\n    axes = list(range(loss.ndim))\n    del axes[batch_axis]\n    return np.sum(loss, axis=axes)\n\n\n\n@use_np\nclass Loss(HybridBlock):\n    \"\"\"Base class for loss.\n\n    Parameters\n    ----------\n    weight : float or None\n        Global scalar weight for loss.\n    batch_axis : int, default 0\n        The axis that represents mini-batch.\n    \"\"\"\n\n    def __init__(self, weight, batch_axis, **kwargs):\n        super(Loss, self).__init__(**kwargs)\n        self._weight = weight\n        self._batch_axis = batch_axis\n\n    def __repr__(self):\n        s = '{name}(batch_axis={_batch_axis}, w={_weight})'\n        return s.format(name=self.__class__.__name__, **self.__dict__)\n\n    def forward(self, x, *args):\n        \"\"\"Overrides to construct symbolic graph for this `Block`.\n\n        Parameters\n        ----------\n        x : Symbol or NDArray\n            The first input tensor.\n        *args : list of Symbol or list of NDArray\n            Additional input tensors.\n\n        \"\"\"\n        # pylint: disable= invalid-name\n        raise NotImplementedError\n\n\n@use_np\nclass L2Loss(Loss):\n    r\"\"\"Calculates the mean squared error between `label` and `pred`.\n\n    .. math:: L = \\frac{1}{2} \\sum_i \\vert {label}_i - {pred}_i \\vert^2.\n\n    `label` and `pred` can have arbitrary shape as long as they have the same\n    number of elements.\n\n    Parameters\n    ----------\n    weight : float or None\n        Global scalar weight for loss.\n    batch_axis : int, default 0\n        The axis that represents mini-batch.\n\n\n    Inputs:\n        - **pred**: prediction tensor with arbitrary shape\n        - **label**: target tensor with the same size as pred.\n        - **sample_weight**: element-wise weighting tensor. Must be broadcastable\n          to the same shape as pred. For example, if pred has shape (64, 10)\n          and you want to weigh each sample in the batch separately,\n          sample_weight should have shape (64, 1).\n\n    Outputs:\n        - **loss**: loss tensor with shape (batch_size,). Dimenions other than\n          batch_axis are averaged out.\n    \"\"\"\n\n    def __init__(self, weight=1., batch_axis=0, **kwargs):\n        super(L2Loss, self).__init__(weight, batch_axis, **kwargs)\n\n    def forward(self, pred, label, sample_weight=None):\n        label = npx.reshape_like(label, pred)\n        loss = np.square(label - pred)\n        loss = _apply_weighting(loss, self._weight / 2, sample_weight)\n        return _batch_mean(loss, self._batch_axis)\n\n\n\n@use_np\nclass L1Loss(Loss):\n    r\"\"\"Calculates the mean absolute error between `label` and `pred`.\n\n    .. math:: L = \\sum_i \\vert {label}_i - {pred}_i \\vert.\n\n    `label` and `pred` can have arbitrary shape as long as they have the same\n    number of elements.\n\n    Parameters\n    ----------\n    weight : float or None\n        Global scalar weight for loss.\n    batch_axis : int, default 0\n        The axis that represents mini-batch.\n\n\n    Inputs:\n        - **pred**: prediction tensor with arbitrary shape\n        - **label**: target tensor with the same size as pred.\n        - **sample_weight**: element-wise weighting tensor. Must be broadcastable\n          to the same shape as pred. For example, if pred has shape (64, 10)\n          and you want to weigh each sample in the batch separately,\n          sample_weight should have shape (64, 1).\n\n    Outputs:\n        - **loss**: loss tensor with shape (batch_size,). Dimenions other than\n          batch_axis are averaged out.\n    \"\"\"\n\n    def __init__(self, weight=None, batch_axis=0, **kwargs):\n        super(L1Loss, self).__init__(weight, batch_axis, **kwargs)\n\n    def forward(self, pred, label, sample_weight=None):\n        label = npx.reshape_like(label, pred)\n        loss = np.abs(label - pred)\n        loss = _apply_weighting(loss, self._weight, sample_weight)\n        return _batch_mean(loss, self._batch_axis)\n\n\n\n@use_np\nclass SigmoidBinaryCrossEntropyLoss(Loss):\n    r\"\"\"The cross-entropy loss for binary classification. (alias: SigmoidBCELoss)\n\n    BCE loss is useful when training logistic regression. If `from_sigmoid`\n    is False (default), this loss computes:\n\n    .. math::\n\n        prob = \\frac{1}{1 + \\exp(-{pred})}\n\n        L = - \\sum_i {label}_i * \\log({prob}_i) * pos\\_weight +\n            (1 - {label}_i) * \\log(1 - {prob}_i)\n\n    If `from_sigmoid` is True, this loss computes:\n\n    .. math::\n\n        L = - \\sum_i {label}_i * \\log({pred}_i) * pos\\_weight +\n            (1 - {label}_i) * \\log(1 - {pred}_i)\n\n    A tensor `pos_weight > 1` decreases the false negative count, hence increasing\n    the recall.\n    Conversely setting `pos_weight < 1` decreases the false positive count and\n    increases the precision.\n\n    `pred` and `label` can have arbitrary shape as long as they have the same\n    number of elements.\n\n    Parameters\n    ----------\n    from_sigmoid : bool, default is `False`\n        Whether the input is from the output of sigmoid. Set this to false will make\n        the loss calculate sigmoid and BCE together, which is more numerically\n        stable through log-sum-exp trick.\n    weight : float or None\n        Global scalar weight for loss.\n    batch_axis : int, default 0\n        The axis that represents mini-batch.\n\n\n    Inputs:\n        - **pred**: prediction tensor with arbitrary shape\n        - **label**: target tensor with values in range `[0, 1]`. Must have the\n          same size as `pred`.\n        - **sample_weight**: element-wise weighting tensor. Must be broadcastable\n          to the same shape as pred. For example, if pred has shape (64, 10)\n          and you want to weigh each sample in the batch separately,\n          sample_weight should have shape (64, 1).\n        - **pos_weight**: a weighting tensor of positive examples. Must be a vector with length\n          equal to the number of classes.For example, if pred has shape (64, 10),\n          pos_weight should have shape (1, 10).\n\n    Outputs:\n        - **loss**: loss tensor with shape (batch_size,). Dimenions other than\n          batch_axis are averaged out.\n    \"\"\"\n\n    def __init__(self, from_sigmoid=False, weight=None, batch_axis=0, **kwargs):\n        super(SigmoidBinaryCrossEntropyLoss, self).__init__(\n            weight, batch_axis, **kwargs)\n        self._from_sigmoid = from_sigmoid\n\n    def forward(self, pred, label, sample_weight=None, pos_weight=None):\n        label = npx.reshape_like(label, pred)\n        if not self._from_sigmoid:\n            if pos_weight is None:\n                # We use the stable formula: max(x, 0) - x * z + log(1 + exp(-abs(x)))\n                loss = npx.relu(pred) - pred * label + \\\n                    npx.activation(-np.abs(pred), act_type='softrelu')\n            else:\n                # We use the stable formula: x - x * z + (1 + z * pos_weight - z) * \\\n                #    (log(1 + exp(-abs(x))) + max(-x, 0))\n                log_weight = 1 + np.multiply(pos_weight - 1, label)\n                loss = pred - pred * label + log_weight * \\\n                       (npx.activation(-np.abs(pred), act_type='softrelu') + npx.relu(-pred))\n        else:\n            eps = 1e-12\n            if pos_weight is None:\n                loss = -(np.log(pred + eps) * label\n                         + np.log(1. - pred + eps) * (1. - label))\n            else:\n                loss = -(np.multiply(np.log(pred + eps) * label, pos_weight)\n                         + np.log(1. - pred + eps) * (1. - label))\n        loss = _apply_weighting(loss, self._weight, sample_weight)\n        return _batch_mean(loss, self._batch_axis)\n\n\nSigmoidBCELoss = SigmoidBinaryCrossEntropyLoss\n\n\n\n\n@use_np\nclass SoftmaxCrossEntropyLoss(Loss):\n    r\"\"\"Computes the softmax cross entropy loss. (alias: SoftmaxCELoss)\n\n    If `sparse_label` is `True` (default), label should contain integer\n    category indicators:\n\n    .. math::\n\n        \\DeclareMathOperator{softmax}{softmax}\n\n        p = \\softmax({pred})\n\n        L = -\\sum_i \\log p_{i,{label}_i}\n\n    `label`'s shape should be `pred`'s shape with the `axis` dimension removed.\n    i.e. for `pred` with shape (1,2,3,4) and `axis = 2`, `label`'s shape should\n    be (1,2,4).\n\n    If `sparse_label` is `False`, `label` should contain probability distribution\n    and `label`'s shape should be the same with `pred`:\n\n    .. math::\n\n        p = \\softmax({pred})\n\n        L = -\\sum_i \\sum_j {label}_j \\log p_{ij}\n\n    Parameters\n    ----------\n    axis : int, default -1\n        The axis to sum over when computing softmax and entropy.\n    sparse_label : bool, default True\n        Whether label is an integer array instead of probability distribution.\n    from_logits : bool, default False\n        Whether input is a log probability (usually from log_softmax) instead\n        of unnormalized numbers.\n    weight : float or None\n        Global scalar weight for loss.\n    batch_axis : int, default 0\n        The axis that represents mini-batch.\n\n\n    Inputs:\n        - **pred**: the prediction tensor, where the `batch_axis` dimension\n          ranges over batch size and `axis` dimension ranges over the number\n          of classes.\n        - **label**: the truth tensor. When `sparse_label` is True, `label`'s\n          shape should be `pred`'s shape with the `axis` dimension removed.\n          i.e. for `pred` with shape (1,2,3,4) and `axis = 2`, `label`'s shape\n          should be (1,2,4) and values should be integers between 0 and 2. If\n          `sparse_label` is False, `label`'s shape must be the same as `pred`\n          and values should be floats in the range `[0, 1]`.\n        - **sample_weight**: element-wise weighting tensor. Must be broadcastable\n          to the same shape as pred. For example, if pred has shape (64, 10)\n          and you want to weigh each sample in the batch separately,\n          sample_weight should have shape (64, 1).\n\n    Outputs:\n        - **loss**: loss tensor with shape (batch_size,). Dimenions other than\n          batch_axis are averaged out.\n    \"\"\"\n\n    def __init__(self, axis=-1, sparse_label=True, from_logits=False, weight=None,\n                 batch_axis=0, **kwargs):\n        super(SoftmaxCrossEntropyLoss, self).__init__(\n            weight, batch_axis, **kwargs)\n        self._axis = axis\n        self._sparse_label = sparse_label\n        self._from_logits = from_logits\n\n    def forward(self, pred, label, sample_weight=None):\n        if not self._from_logits:\n            pred = npx.log_softmax(pred, axis=self._axis)\n        if self._sparse_label:\n            loss = -npx.pick(pred, label, axis=self._axis, keepdims=True)\n        else:\n            label = npx.reshape_like(label, pred)\n            loss = -(pred * label).sum(axis=self._axis, keepdims=True)\n        loss = _apply_weighting(loss, self._weight, sample_weight)\n        return _batch_mean(loss, self._batch_axis)\n\n\nSoftmaxCELoss = SoftmaxCrossEntropyLoss\n\n\n@use_np\nclass KLDivLoss(Loss):\n    r\"\"\"The Kullback-Leibler divergence loss.\n\n    KL divergence measures the distance between contiguous distributions. It\n    can be used to minimize information loss when approximating a distribution.\n    If `from_logits` is True (default), loss is defined as:\n\n    .. math::\n\n        L = \\sum_i {label}_i * \\big[\\log({label}_i) - {pred}_i\\big]\n\n    If `from_logits` is False, loss is defined as:\n\n    .. math::\n\n        \\DeclareMathOperator{softmax}{softmax}\n\n        prob = \\softmax({pred})\n\n        L = \\sum_i {label}_i * \\big[\\log({label}_i) - \\log({prob}_i)\\big]\n\n\n    `label` and `pred` can have arbitrary shape as long as they have the same\n    number of elements.\n\n    Parameters\n    ----------\n    from_logits : bool, default is `True`\n        Whether the input is log probability (usually from log_softmax) instead\n        of unnormalized numbers.\n    axis : int, default -1\n        The dimension along with to compute softmax. Only used when `from_logits`\n        is False.\n    weight : float or None\n        Global scalar weight for loss.\n    batch_axis : int, default 0\n        The axis that represents mini-batch.\n\n\n    Inputs:\n        - **pred**: prediction tensor with arbitrary shape. If `from_logits` is\n          True, `pred` should be log probabilities. Otherwise, it should be\n          unnormalized predictions, i.e. from a dense layer.\n        - **label**: truth tensor with values in range `(0, 1)`. Must have\n          the same size as `pred`.\n        - **sample_weight**: element-wise weighting tensor. Must be broadcastable\n          to the same shape as pred. For example, if pred has shape (64, 10)\n          and you want to weigh each sample in the batch separately,\n          sample_weight should have shape (64, 1).\n\n    Outputs:\n        - **loss**: loss tensor with shape (batch_size,). Dimenions other than\n          batch_axis are averaged out.\n\n\n    References\n    ----------\n        `Kullback-Leibler divergence\n        <https://en.wikipedia.org/wiki/Kullback-Leibler_divergence>`_\n    \"\"\"\n\n    def __init__(self, from_logits=True, axis=-1, weight=None, batch_axis=0,\n                 **kwargs):\n        super(KLDivLoss, self).__init__(weight, batch_axis, **kwargs)\n        self._from_logits = from_logits\n        self._axis = axis\n\n    def forward(self, pred, label, sample_weight=None):\n        if not self._from_logits:\n            pred = npx.log_softmax(pred, self._axis)\n        loss = label * (np.log(label + 1e-12) - pred)\n        loss = _apply_weighting(loss, self._weight, sample_weight)\n        return _batch_mean(loss, self._batch_axis)\n\n\n@use_np\nclass CTCLoss(Loss):\n    r\"\"\"Connectionist Temporal Classification Loss.\n\n\n    Parameters\n    ----------\n    layout : str, default 'NTC'\n        Layout of prediction tensor. 'N', 'T', 'C' stands for batch size,\n        sequence length, and alphabet_size respectively.\n    label_layout : str, default 'NT'\n        Layout of the labels. 'N', 'T' stands for batch size, and sequence\n        length respectively.\n    weight : float or None\n        Global scalar weight for loss.\n\n\n    Inputs:\n        - **pred**: unnormalized prediction tensor (before softmax).\n          Its shape depends on `layout`. If `layout` is 'TNC', pred\n          should have shape `(sequence_length, batch_size, alphabet_size)`.\n          Note that in the last dimension, index `alphabet_size-1` is reserved\n          for internal use as blank label. So `alphabet_size` is one plus the\n          actual alphabet size.\n\n        - **label**: zero-based label tensor. Its shape depends on `label_layout`.\n          If `label_layout` is 'TN', `label` should have shape\n          `(label_sequence_length, batch_size)`.\n\n        - **pred_lengths**: optional (default None), used for specifying the\n          length of each entry when different `pred` entries in the same batch\n          have different lengths. `pred_lengths` should have shape `(batch_size,)`.\n\n        - **label_lengths**: optional (default None), used for specifying the\n          length of each entry when different `label` entries in the same batch\n          have different lengths. `label_lengths` should have shape `(batch_size,)`.\n\n    Outputs:\n        - **loss**: output loss has shape `(batch_size,)`.\n\n\n    **Example**: suppose the vocabulary is `[a, b, c]`, and in one batch we\n    have three sequences 'ba', 'cbb', and 'abac'. We can index the labels as\n    `{'a': 0, 'b': 1, 'c': 2, blank: 3}`. Then `alphabet_size` should be 4,\n    where label 3 is reserved for internal use by `CTCLoss`. We then need to\n    pad each sequence with `-1` to make a rectangular `label` tensor::\n\n        [[1, 0, -1, -1],\n         [2, 1,  1, -1],\n         [0, 1,  0,  2]]\n\n\n    References\n    ----------\n        `Connectionist Temporal Classification: Labelling Unsegmented\n        Sequence Data with Recurrent Neural Networks\n        <http://www.cs.toronto.edu/~graves/icml_2006.pdf>`_\n    \"\"\"\n\n    def __init__(self, layout='NTC', label_layout='NT', weight=None, **kwargs):\n        assert layout in ['NTC', 'TNC'],\\\n            f\"Only 'NTC' and 'TNC' layouts for pred are supported. Got: {layout}\"\n        assert label_layout in ['NT', 'TN'],\\\n            f\"Only 'NT' and 'TN' layouts for label are supported. Got: {label_layout}\"\n        self._layout = layout\n        self._label_layout = label_layout\n        batch_axis = label_layout.find('N')\n        super(CTCLoss, self).__init__(weight, batch_axis, **kwargs)\n\n    def forward(self, pred, label, pred_lengths=None, label_lengths=None, sample_weight=None):\n        if self._layout == 'NTC':\n            pred = np.swapaxes(pred, 0, 1)\n        if self._batch_axis == 1:\n            label = np.swapaxes(label, 0, 1)\n        loss = npx.ctc_loss(pred, label, pred_lengths, label_lengths,\n                            use_data_lengths=pred_lengths is not None,\n                            use_label_lengths=label_lengths is not None,\n                            blank_label='last')\n        return _apply_weighting(loss, self._weight, sample_weight)\n\n\n@use_np\nclass HuberLoss(Loss):\n    r\"\"\"Calculates smoothed L1 loss that is equal to L1 loss if absolute error\n    exceeds rho but is equal to L2 loss otherwise. Also called SmoothedL1 loss.\n\n    .. math::\n        L = \\sum_i \\begin{cases} \\frac{1}{2 {rho}} ({label}_i - {pred}_i)^2 &\n                           \\text{ if } |{label}_i - {pred}_i| < {rho} \\\\\n                           |{label}_i - {pred}_i| - \\frac{{rho}}{2} &\n                           \\text{ otherwise }\n            \\end{cases}\n\n    `label` and `pred` can have arbitrary shape as long as they have the same\n    number of elements.\n\n    Parameters\n    ----------\n    rho : float, default 1\n        Threshold for trimmed mean estimator.\n    weight : float or None\n        Global scalar weight for loss.\n    batch_axis : int, default 0\n        The axis that represents mini-batch.\n\n\n    Inputs:\n        - **pred**: prediction tensor with arbitrary shape\n        - **label**: target tensor with the same size as pred.\n        - **sample_weight**: element-wise weighting tensor. Must be broadcastable\n          to the same shape as pred. For example, if pred has shape (64, 10)\n          and you want to weigh each sample in the batch separately,\n          sample_weight should have shape (64, 1).\n\n    Outputs:\n        - **loss**: loss tensor with shape (batch_size,). Dimenions other than\n          batch_axis are averaged out.\n    \"\"\"\n\n    def __init__(self, rho=1, weight=None, batch_axis=0, **kwargs):\n        super(HuberLoss, self).__init__(weight, batch_axis, **kwargs)\n        self._rho = rho\n\n    def forward(self, pred, label, sample_weight=None):\n        label = npx.reshape_like(label, pred)\n        loss = np.abs(label - pred)\n        loss = np.where(loss > self._rho, loss - 0.5 * self._rho,\n                        (0.5 / self._rho) * np.square(loss))\n        loss = _apply_weighting(loss, self._weight, sample_weight)\n        return _batch_mean(loss, self._batch_axis)\n\n\n@use_np\nclass HingeLoss(Loss):\n    r\"\"\"Calculates the hinge loss function often used in SVMs:\n\n    .. math::\n        L = \\sum_i max(0, {margin} - {pred}_i \\cdot {label}_i)\n\n    where `pred` is the classifier prediction and `label` is the target tensor\n    containing values -1 or 1. `label` and `pred` must have the same number of\n    elements.\n\n    Parameters\n    ----------\n    margin : float\n        The margin in hinge loss. Defaults to 1.0\n    weight : float or None\n        Global scalar weight for loss.\n    batch_axis : int, default 0\n        The axis that represents mini-batch.\n\n\n    Inputs:\n        - **pred**: prediction tensor with arbitrary shape.\n        - **label**: truth tensor with values -1 or 1. Must have the same size\n          as pred.\n        - **sample_weight**: element-wise weighting tensor. Must be broadcastable\n          to the same shape as pred. For example, if pred has shape (64, 10)\n          and you want to weigh each sample in the batch separately,\n          sample_weight should have shape (64, 1).\n\n    Outputs:\n        - **loss**: loss tensor with shape (batch_size,). Dimenions other than\n          batch_axis are averaged out.\n    \"\"\"\n\n    def __init__(self, margin=1, weight=None, batch_axis=0, **kwargs):\n        super(HingeLoss, self).__init__(weight, batch_axis, **kwargs)\n        self._margin = margin\n\n    def forward(self, pred, label, sample_weight=None):\n        label = npx.reshape_like(label, pred)\n        loss = npx.relu(self._margin - pred * label)\n        loss = _apply_weighting(loss, self._weight, sample_weight)\n        return _batch_mean(loss, self._batch_axis)\n\n\n@use_np\nclass SquaredHingeLoss(Loss):\n    r\"\"\"Calculates the soft-margin loss function used in SVMs:\n\n    .. math::\n        L = \\sum_i max(0, {margin} - {pred}_i \\cdot {label}_i)^2\n\n    where `pred` is the classifier prediction and `label` is the target tensor\n    containing values -1 or 1. `label` and `pred` can have arbitrary shape as\n    long as they have the same number of elements.\n\n    Parameters\n    ----------\n    margin : float\n        The margin in hinge loss. Defaults to 1.0\n    weight : float or None\n        Global scalar weight for loss.\n    batch_axis : int, default 0\n        The axis that represents mini-batch.\n\n\n    Inputs:\n        - **pred**: prediction tensor with arbitrary shape\n        - **label**: truth tensor with values -1 or 1. Must have the same size\n          as pred.\n        - **sample_weight**: element-wise weighting tensor. Must be broadcastable\n          to the same shape as pred. For example, if pred has shape (64, 10)\n          and you want to weigh each sample in the batch separately,\n          sample_weight should have shape (64, 1).\n\n    Outputs:\n        - **loss**: loss tensor with shape (batch_size,). Dimenions other than\n          batch_axis are averaged out.\n    \"\"\"\n\n    def __init__(self, margin=1, weight=None, batch_axis=0, **kwargs):\n        super(SquaredHingeLoss, self).__init__(weight, batch_axis, **kwargs)\n        self._margin = margin\n\n    def forward(self, pred, label, sample_weight=None):\n        label = npx.reshape_like(label, pred)\n        loss = np.square(npx.relu(self._margin - pred * label))\n        loss = _apply_weighting(loss, self._weight, sample_weight)\n        return _batch_mean(loss, self._batch_axis)\n\n\n@use_np\nclass LogisticLoss(Loss):\n    r\"\"\"Calculates the logistic loss (for binary losses only):\n\n    .. math::\n        L = \\sum_i \\log(1 + \\exp(- {pred}_i \\cdot {label}_i))\n\n    where `pred` is the classifier prediction and `label` is the target tensor\n    containing values -1 or 1 (0 or 1 if `label_format` is binary).\n    `label` and `pred` can have arbitrary shape as long as they have the same number of elements.\n\n    Parameters\n    ----------\n    weight : float or None\n        Global scalar weight for loss.\n    batch_axis : int, default 0\n        The axis that represents mini-batch.\n    label_format : str, default 'signed'\n        Can be either 'signed' or 'binary'. If the label_format is 'signed', all label values should\n        be either -1 or 1. If the label_format is 'binary', all label values should be either\n        0 or 1.\n\n    Inputs:\n        - **pred**: prediction tensor with arbitrary shape.\n        - **label**: truth tensor with values -1/1 (label_format is 'signed')\n          or 0/1 (label_format is 'binary'). Must have the same size as pred.\n        - **sample_weight**: element-wise weighting tensor. Must be broadcastable\n          to the same shape as pred. For example, if pred has shape (64, 10)\n          and you want to weigh each sample in the batch separately,\n          sample_weight should have shape (64, 1).\n\n    Outputs:\n        - **loss**: loss tensor with shape (batch_size,). Dimenions other than\n          batch_axis are averaged out.\n    \"\"\"\n\n    def __init__(self, weight=None, batch_axis=0, label_format='signed', **kwargs):\n        super(LogisticLoss, self).__init__(weight, batch_axis, **kwargs)\n        self._label_format = label_format\n        if self._label_format not in [\"signed\", \"binary\"]:\n            raise ValueError(f\"label_format can only be signed or binary, received {label_format}.\")\n\n    def forward(self, pred, label, sample_weight=None):\n        label = npx.reshape_like(label, pred)\n        if self._label_format == 'signed':\n            label = (label + 1.0) / 2.0  # Transform label to be either 0 or 1\n        # Use a stable formula in computation\n        loss = npx.relu(pred) - pred * label + \\\n            npx.activation(-np.abs(pred), act_type='softrelu')\n        loss = _apply_weighting(loss, self._weight, sample_weight)\n        return _batch_mean(loss, self._batch_axis)\n\n\n@use_np\nclass TripletLoss(Loss):\n    r\"\"\"Calculates triplet loss given three input tensors and a positive margin.\n    Triplet loss measures the relative similarity between a positive\n    example, a negative example, and prediction:\n\n    .. math::\n        L = \\sum_i \\max(\\Vert {pos_i}_i - {pred} \\Vert_2^2 -\n                        \\Vert {neg_i}_i - {pred} \\Vert_2^2 + {margin}, 0)\n\n    `positive`, `negative`, and 'pred' can have arbitrary shape as long as they\n    have the same number of elements.\n\n    Parameters\n    ----------\n    margin : float\n        Margin of separation between correct and incorrect pair.\n    weight : float or None\n        Global scalar weight for loss.\n    batch_axis : int, default 0\n        The axis that represents mini-batch.\n\n\n    Inputs:\n        - **pred**: prediction tensor with arbitrary shape\n        - **positive**: positive example tensor with arbitrary shape. Must have\n          the same size as pred.\n        - **negative**: negative example tensor with arbitrary shape Must have\n          the same size as pred.\n\n    Outputs:\n        - **loss**: loss tensor with shape (batch_size,).\n    \"\"\"\n\n    def __init__(self, margin=1, weight=None, batch_axis=0, **kwargs):\n        super(TripletLoss, self).__init__(weight, batch_axis, **kwargs)\n        self._margin = margin\n\n    @use_np\n    def forward(self, pred, positive, negative, sample_weight=None):\n        positive = npx.reshape_like(positive, pred)\n        negative = npx.reshape_like(negative, pred)\n        loss = _batch_sum(np.square(positive - pred) - np.square(negative - pred), self._batch_axis)\n        loss = npx.relu(loss + self._margin)\n        return _apply_weighting(loss, self._weight, sample_weight)\n\n\n@use_np\nclass PoissonNLLLoss(Loss):\n    r\"\"\"For a target (Random Variable) in a Poisson distribution, the function calculates the Negative\n    Log likelihood loss.\n    PoissonNLLLoss measures the loss accrued from a poisson regression prediction made by the model.\n\n    .. math::\n        L = \\text{pred} - \\text{target} * \\log(\\text{pred}) +\\log(\\text{target!})\n\n    `target`, 'pred' can have arbitrary shape as long as they have the same number of elements.\n\n    Parameters\n    ----------\n    from_logits : boolean, default True\n        indicating whether log(predicted) value has already been computed. If True, the loss is computed as\n        :math:`\\exp(\\text{pred}) - \\text{target} * \\text{pred}`, and if False, then loss is computed as\n        :math:`\\text{pred} - \\text{target} * \\log(\\text{pred}+\\text{epsilon})`.The default value\n    weight : float or None\n        Global scalar weight for loss.\n    batch_axis : int, default 0\n        The axis that represents mini-batch.\n    compute_full: boolean, default False\n        Indicates whether to add an approximation(Stirling factor) for the Factorial term in the formula for the loss.\n        The Stirling factor is:\n        :math:`\\text{target} * \\log(\\text{target}) - \\text{target} + 0.5 * \\log(2 * \\pi * \\text{target})`\n    epsilon: float, default 1e-08\n        This is to avoid calculating log(0) which is not defined.\n\n\n    Inputs:\n        - **pred**:   Predicted value\n        - **target**: Random variable(count or number) which belongs to a Poisson distribution.\n        - **sample_weight**: element-wise weighting tensor. Must be broadcastable\n          to the same shape as pred. For example, if pred has shape (64, 10)\n          and you want to weigh each sample in the batch separately,\n          sample_weight should have shape (64, 1).\n\n    Outputs:\n        - **loss**: Average loss (shape=(1,1)) of the loss tensor with shape (batch_size,).\n    \"\"\"\n\n    def __init__(self, weight=None, from_logits=True, batch_axis=0, compute_full=False, **kwargs):\n        super(PoissonNLLLoss, self).__init__(weight, batch_axis, **kwargs)\n        self._from_logits = from_logits\n        self._compute_full = compute_full\n\n    def forward(self, pred, target, sample_weight=None, epsilon=1e-08):\n        target = npx.reshape_like(target, pred)\n        if self._from_logits:\n            loss = np.exp(pred) - target * pred\n        else:\n            loss = pred - target * np.log(pred + epsilon)\n        if self._compute_full:\n            # Using numpy's pi value\n            stirling_factor = target * \\\n                np.log(target) - target + 0.5 * np.log(2 * target * _np.pi)\n            target_gt_1 = target > 1\n            stirling_factor = stirling_factor * target_gt_1\n            loss = loss + stirling_factor\n        loss = _apply_weighting(loss, self._weight, sample_weight)\n        return _batch_mean(loss, self._batch_axis)\n\n\n@use_np\nclass CosineEmbeddingLoss(Loss):\n    r\"\"\"For a target label 1 or -1, vectors input1 and input2, the function computes the cosine distance\n    between the vectors. This can be interpreted as how similar/dissimilar two input vectors are.\n\n    .. math::\n\n        L = \\sum_i \\begin{cases} 1 - {cos\\_sim({input1}_i, {input2}_i)} & \\text{ if } {label}_i = 1\\\\\n                         {cos\\_sim({input1}_i, {input2}_i)} & \\text{ if } {label}_i = -1 \\end{cases}\\\\\n        cos\\_sim(input1, input2) = \\frac{{input1}_i.{input2}_i}{||{input1}_i||.||{input2}_i||}\n\n    `input1`, `input2` can have arbitrary shape as long as they have the same number of elements.\n\n    Parameters\n    ----------\n    weight : float or None\n        Global scalar weight for loss.\n    batch_axis : int, default 0\n        The axis that represents mini-batch.\n    margin : float\n        Margin of separation between correct and incorrect pair.\n\n\n    Inputs:\n        - **input1**: a tensor with arbitrary shape\n        - **input2**: another tensor with same shape as pred to which input1 is\n          compared for similarity and loss calculation\n        - **label**: A 1-D tensor indicating for each pair input1 and input2, target label is 1 or -1\n        - **sample_weight**: element-wise weighting tensor. Must be broadcastable\n          to the same shape as input1. For example, if input1 has shape (64, 10)\n          and you want to weigh each sample in the batch separately,\n          sample_weight should have shape (64, 1).\n\n    Outputs:\n        - **loss**: The loss tensor with shape (batch_size,).\n    \"\"\"\n\n    def __init__(self, weight=None, batch_axis=0, margin=0, **kwargs):\n        super(CosineEmbeddingLoss, self).__init__(weight, batch_axis, **kwargs)\n        self._margin = margin\n\n    def forward(self, input1, input2, label, sample_weight=None):\n        input1 = npx.reshape_like(input1, input2)\n        cos_sim = self._cosine_similarity(input1, input2)\n        label = npx.reshape_like(label, cos_sim)\n        loss = np.where(label == 1,\n                        1 - cos_sim,\n                        np.clip(cos_sim - self._margin, 0, 1 - self._margin))\n\n        loss = _apply_weighting(loss, self._weight, sample_weight)\n        return _batch_mean(loss, self._batch_axis)\n\n    def _cosine_similarity(self, x, y, axis=-1):\n        # Calculates the cosine similarity between 2 vectors\n        x_norm = npx.reshape(npx.norm(x, axis=axis), (-1, 1))\n        y_norm = npx.reshape(npx.norm(y, axis=axis), (-1, 1))\n        x_dot_y = npx.reshape(np.sum(x * y, axis=axis), (-1, 1))\n        eps_arr = np.full((1, 1), 1e-12)\n        return (x_dot_y / np.maximum(x_norm * y_norm, eps_arr))\n\n\n@use_np\nclass SDMLLoss(Loss):\n    r\"\"\"Calculates Batchwise Smoothed Deep Metric Learning (SDML) Loss given two input tensors and a smoothing weight\n    SDM Loss learns similarity between paired samples by using unpaired samples in the minibatch\n    as potential negative examples.\n\n    The loss is described in greater detail in\n    \"Large Scale Question Paraphrase Retrieval with Smoothed Deep Metric Learning.\"\n    - by Bonadiman, Daniele, Anjishnu Kumar, and Arpit Mittal.  arXiv preprint arXiv:1905.12786 (2019).\n    URL: https://arxiv.org/pdf/1905.12786.pdf\n\n    According to the authors, this loss formulation achieves comparable or higher accuracy to\n    Triplet Loss but converges much faster.\n    The loss assumes that the items in both tensors in each minibatch\n    are aligned such that x1[0] corresponds to x2[0] and all other datapoints in the minibatch are unrelated.\n    `x1` and  `x2` are minibatches of vectors.\n\n    Parameters\n    ----------\n    smoothing_parameter : float\n        Probability mass to be distributed over the minibatch. Must be < 1.0.\n    weight : float or None\n        Global scalar weight for loss.\n    batch_axis : int, default 0\n        The axis that represents mini-batch.\n\n    Inputs:\n        - **x1**: Minibatch of data points with shape (batch_size, vector_dim)\n        - **x2**: Minibatch of data points with shape (batch_size, vector_dim)\n          Each item in x2 is a positive sample for the same index in x1.\n          That is, x1[0] and x2[0] form a positive pair, x1[1] and x2[1] form a positive pair - and so on.\n          All data points in different rows should be decorrelated\n\n    Outputs:\n        - **loss**: loss tensor with shape (batch_size,).\n    \"\"\"\n\n    def __init__(self, smoothing_parameter=0.3, weight=1., batch_axis=0, **kwargs):\n        super(SDMLLoss, self).__init__(weight, batch_axis, **kwargs)\n        self.kl_loss = KLDivLoss(from_logits=True)\n        # Smoothing probability mass\n        self.smoothing_parameter = smoothing_parameter\n\n    def _compute_distances(self, x1, x2):\n        \"\"\"\n        This function computes the euclidean distance between every vector\n        in the two batches in input.\n        \"\"\"\n        # expanding x1 form [batch_size, dim] to [batch_size, 1, dim]\n        # and x2 to [1, batch_size, dim]\n        x1_ = np.expand_dims(x1, 1)\n        x2_ = np.expand_dims(x2, 0)\n        # pointwise squared differences\n        squared_diffs = (x1_ - x2_)**2\n        # sum of squared differences distance\n        return squared_diffs.sum(axis=2)\n\n\n    # pylint: disable=too-many-function-args\n    def _compute_labels(self, batch_size):\n        \"\"\"\n        The function creates the label matrix for the loss.\n        It is an identity matrix of size [BATCH_SIZE x BATCH_SIZE]\n        labels:\n            [[1, 0]\n             [0, 1]]\n\n        after the proces the labels are smoothed by a small amount to\n        account for errors.\n\n        labels:\n            [[0.9, 0.1]\n             [0.1, 0.9]]\n\n\n        Pereyra, Gabriel, et al. \"Regularizing neural networks by penalizing\n        confident output distributions.\" arXiv preprint arXiv:1701.06548 (2017).\n        \"\"\"\n\n        gold = np.eye(batch_size)\n        labels = gold * (1 - self.smoothing_parameter) + (1 - gold) * self.smoothing_parameter / (batch_size - 1)\n        return labels\n\n    def forward(self, x1, x2):\n        \"\"\"\n        the function computes the kl divergence between the negative distances\n        (internally it compute a softmax casting into probabilities) and the\n        identity matrix.\n\n        This assumes that the two batches are aligned therefore the more similar\n        vector should be the one having the same id.\n\n        Batch1                                Batch2\n\n        President of France                   French President\n        President of US                       American President\n\n        Given the question president of France in batch 1 the model will\n        learn to predict french president comparing it with all the other\n        vectors in batch 2\n        \"\"\"\n        batch_size = x1.shape[0]\n        labels = self._compute_labels(batch_size)\n        distances = self._compute_distances(x1, x2)\n        log_probabilities = npx.log_softmax(-distances, axis=1)\n        # multiply for the number of labels to obtain the correct loss (gluon kl_loss averages instead of sum)\n        # PR#18423:multiply for the number of labels should multiply x1.shape[1] rather than x1.shape[0])\n        # After PR#18423, it is no need to multiply it anymore.\n        return self.kl_loss(log_probabilities, labels.to_device(distances.device))\n"
  },
  {
    "path": "python/mxnet/gluon/metric.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=no-member, too-many-lines\n\n\"\"\"Online evaluation metric module.\"\"\"\nimport math\nfrom collections import OrderedDict\n\nfrom .. import numpy\nfrom ..util import use_np\n\nfrom ..base import numeric_types, string_types\nfrom .. import ndarray, npx\nfrom .. import registry\n\n\ndef check_label_shapes(labels, preds, wrap=False, shape=False):\n    \"\"\"Helper function for checking shape of label and prediction\n\n    Parameters\n    ----------\n    labels : list of `NDArray`\n        The labels of the data.\n\n    preds : list of `NDArray`\n        Predicted values.\n\n    wrap : boolean\n        If True, wrap labels/preds in a list if they are single NDArray\n\n    shape : boolean\n        If True, check the shape of labels and preds;\n        Otherwise only check their length.\n    \"\"\"\n    if not shape:\n        label_shape, pred_shape = len(labels), len(preds)\n    else:\n        label_shape, pred_shape = labels.shape, preds.shape\n\n    if label_shape != pred_shape:\n        raise ValueError(\"Shape of labels {} does not match shape of \"\n                         \"predictions {}\".format(label_shape, pred_shape))\n\n    if wrap:\n        if isinstance(labels, ndarray.ndarray.NDArray):\n            labels = [labels]\n        if isinstance(preds, ndarray.ndarray.NDArray):\n            preds = [preds]\n\n    return labels, preds\n\nclass EvalMetric(object):\n    \"\"\"Base class for all evaluation metrics.\n\n    .. note::\n\n        This is a base class that provides common metric interfaces.\n        One should not use this class directly, but instead create new metric\n        classes that extend it.\n\n    Parameters\n    ----------\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n    \"\"\"\n    def __init__(self, name, output_names=None,\n                 label_names=None, **kwargs):\n        self.name = str(name)\n        self.output_names = output_names\n        self.label_names = label_names\n        self._kwargs = kwargs\n        self.reset()\n\n    def __str__(self):\n        return \"EvalMetric: {}\".format(dict(self.get_name_value()))\n\n    def get_config(self):\n        \"\"\"Save configurations of metric. Can be recreated\n        from configs with metric.create(``**config``)\n        \"\"\"\n        config = self._kwargs.copy()\n        config.update({\n            'metric': self.__class__.__name__,\n            'name': self.name,\n            'output_names': self.output_names,\n            'label_names': self.label_names})\n        return config\n\n    def update_dict(self, label, pred):\n        \"\"\"Update the internal evaluation with named label and pred\n\n        Parameters\n        ----------\n        labels : OrderedDict of str -> NDArray\n            name to array mapping for labels.\n\n        preds : OrderedDict of str -> NDArray\n            name to array mapping of predicted outputs.\n        \"\"\"\n        if self.output_names is not None:\n            pred = [pred[name] for name in self.output_names]\n        else:\n            pred = list(pred.values())\n\n        if self.label_names is not None:\n            label = [label[name] for name in self.label_names]\n        else:\n            label = list(label.values())\n\n        self.update(label, pred)\n\n    def update(self, labels, preds):\n        \"\"\"Updates the internal evaluation result.\n\n        Parameters\n        ----------\n        labels : list of `NDArray`\n            The labels of the data.\n\n        preds : list of `NDArray`\n            Predicted values.\n        \"\"\"\n        raise NotImplementedError()\n\n    def reset(self):\n        \"\"\"Resets the internal evaluation result to initial state.\"\"\"\n        self.num_inst = 0\n        self.sum_metric = 0.0\n\n    def get(self):\n        \"\"\"Gets the current evaluation result.\n\n        Returns\n        -------\n        names : list of str\n           Name of the metrics.\n        values : list of float\n           Value of the evaluations.\n        \"\"\"\n        if self.num_inst == 0:\n            return (self.name, float('nan'))\n        else:\n            res = self.sum_metric / self.num_inst\n            if isinstance(res, numpy.ndarray) and len(res.shape) == 0:\n                # currently calling ' c = mxnet.numpy.array([1,2,3]).sum() ' would get\n                # ' array(6.) ', a ndarray with shape ()\n                # In this case, returning a 'float' in .get() is more explicit.\n                res = res.item()\n            return (self.name, res)\n\n    def get_name_value(self):\n        \"\"\"Returns zipped name and value pairs.\n\n        Returns\n        -------\n        list of tuples\n            A (name, value) tuple list.\n        \"\"\"\n        name, value = self.get()\n        if not isinstance(name, list):\n            name = [name]\n        if not isinstance(value, list):\n            value = [value]\n        return list(zip(name, value))\n\n# pylint: disable=invalid-name\nregister = registry.get_register_func(EvalMetric, 'metric')\nalias = registry.get_alias_func(EvalMetric, 'metric')\n_create = registry.get_create_func(EvalMetric, 'metric')\n# pylint: enable=invalid-name\n\n\ndef create(metric, *args, **kwargs):\n    \"\"\"Creates evaluation metric from metric names or instances of EvalMetric\n    or a custom metric function.\n\n    Parameters\n    ----------\n    metric : str or callable\n        Specifies the metric to create.\n        This argument must be one of the below:\n\n        - Name of a metric.\n        - An instance of `EvalMetric`.\n        - A list, each element of which is a metric or a metric name.\n        - An evaluation function that computes custom metric for a given batch of\n          labels and predictions.\n    *args : list\n        Additional arguments to metric constructor.\n        Only used when metric is str.\n    **kwargs : dict\n        Additional arguments to metric constructor.\n        Only used when metric is str\n\n    Examples\n    --------\n    >>> def custom_metric(label, pred):\n    ...     return np.mean(np.abs(label - pred))\n    ...\n    >>> metric1 = mx.gluon.metric.create('acc')\n    >>> metric2 = mx.gluon.metric.create(custom_metric)\n    >>> metric3 = mx.gluon.metric.create([metric1, metric2, 'rmse'])\n    \"\"\"\n    if callable(metric):\n        return CustomMetric(metric, *args, **kwargs)\n    elif isinstance(metric, list):\n        composite_metric = CompositeEvalMetric()\n        for child_metric in metric:\n            composite_metric.add(create(child_metric, *args, **kwargs))\n        return composite_metric\n\n    return _create(metric, *args, **kwargs)\n\n\n@register\n@alias('composite')\nclass CompositeEvalMetric(EvalMetric):\n    \"\"\"Manages multiple evaluation metrics.\n\n    Parameters\n    ----------\n    metrics : list of EvalMetric\n        List of child metrics.\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n\n    Examples\n    --------\n    >>> predicts = [mx.np.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])]\n    >>> labels   = [mx.np.array([0, 1, 1])]\n    >>> eval_metrics_1 = mx.gluon.metric.Accuracy()\n    >>> eval_metrics_2 = mx.gluon.metric.F1()\n    >>> eval_metrics = mx.gluon.metric.CompositeEvalMetric()\n    >>> for child_metric in [eval_metrics_1, eval_metrics_2]:\n    >>>     eval_metrics.add(child_metric)\n    >>> eval_metrics.update(labels = labels, preds = predicts)\n    >>> eval_metrics.get()\n    (['accuracy', 'f1'], [0.6666666666666666, 0.8])\n    \"\"\"\n\n    def __init__(self, metrics=None, name='composite',\n                 output_names=None, label_names=None):\n        super(CompositeEvalMetric, self).__init__(\n            name, output_names=output_names, label_names=label_names)\n        if metrics is None:\n            metrics = []\n        self.metrics = [create(i) for i in metrics]\n\n    def add(self, metric):\n        \"\"\"Adds a child metric.\n\n        Parameters\n        ----------\n        metric\n            A metric instance.\n        \"\"\"\n        self.metrics.append(create(metric))\n\n    def get_metric(self, index):\n        \"\"\"Returns a child metric.\n\n        Parameters\n        ----------\n        index : int\n            Index of child metric in the list of metrics.\n        \"\"\"\n        try:\n            return self.metrics[index]\n        except IndexError:\n            return ValueError(\"Metric index {} is out of range 0 and {}\".format(\n                index, len(self.metrics)))\n\n    def update_dict(self, labels, preds): # pylint: disable=arguments-differ\n        if self.label_names is not None:\n            labels = OrderedDict([i for i in labels.items()\n                                  if i[0] in self.label_names])\n        if self.output_names is not None:\n            preds = OrderedDict([i for i in preds.items()\n                                 if i[0] in self.output_names])\n\n        for metric in self.metrics:\n            metric.update_dict(labels, preds)\n\n    def update(self, labels, preds):\n        \"\"\"Updates the internal evaluation result.\n\n        Parameters\n        ----------\n        labels : list of `NDArray`\n            The labels of the data.\n\n        preds : list of `NDArray`\n            Predicted values.\n        \"\"\"\n        for metric in self.metrics:\n            metric.update(labels, preds)\n\n    def reset(self):\n        \"\"\"Resets the internal evaluation result to initial state.\"\"\"\n        try:\n            for metric in self.metrics:\n                metric.reset()\n        except AttributeError:\n            pass\n\n    def get(self):\n        \"\"\"Returns the current evaluation result.\n\n        Returns\n        -------\n        names : list of str\n           Name of the metrics.\n        values : list of float\n           Value of the evaluations.\n        \"\"\"\n        names = []\n        values = []\n        for metric in self.metrics:\n            name, value = metric.get()\n            if isinstance(name, string_types):\n                name = [name]\n            if isinstance(value, numeric_types):\n                value = [value]\n            names.extend(name)\n            values.extend(value)\n        return (names, values)\n\n    def get_config(self):\n        config = super(CompositeEvalMetric, self).get_config()\n        config.update({'metrics': [i.get_config() for i in self.metrics]})\n        return config\n\n\n########################\n# CLASSIFICATION METRICS\n########################\n\n\n@register\n@alias('acc')\n@use_np\nclass Accuracy(EvalMetric):\n    \"\"\"Computes accuracy classification score.\n\n    The accuracy score is defined as\n\n    .. math::\n\n        \\\\text{accuracy}(y, \\\\hat{y}) = \\\\frac{1}{n} \\\\sum_{i=0}^{n-1}\n        \\\\text{1}(\\\\hat{y_i} == y_i)\n\n    Parameters\n    ----------\n    axis : int, default=1\n        The axis that represents classes\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n\n    Examples\n    --------\n    >>> predicts = [mx.np.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])]\n    >>> labels   = [mx.np.array([0, 1, 1])]\n    >>> acc = mx.gluon.metric.Accuracy()\n    >>> acc.update(preds = predicts, labels = labels)\n    >>> acc.get()\n    ('accuracy', 0.6666666666666666)\n    \"\"\"\n    def __init__(self, axis=1, name='accuracy',\n                 output_names=None, label_names=None):\n        super(Accuracy, self).__init__(\n            name, axis=axis,\n            output_names=output_names, label_names=label_names)\n        self.axis = axis\n\n    def update(self, labels, preds):\n        \"\"\"Updates the internal evaluation result.\n\n        Parameters\n        ----------\n        labels : list of `NDArray`\n            The labels of the data with class indices as values, one per sample.\n\n        preds : list of `NDArray`\n            Prediction values for samples. Each prediction value can either be the class index,\n            or a vector of likelihoods for all classes.\n        \"\"\"\n        labels, preds = check_label_shapes(labels, preds, True)\n\n        for label, pred_label in zip(labels, preds):\n            pred_label = pred_label.as_np_ndarray().to_device(label.device)\n            label = label.as_np_ndarray()\n            if pred_label.shape != label.shape:\n                pred_label = pred_label.argmax(axis=self.axis)\n            pred_label = pred_label.astype('int32')\n            label = label.astype('int32')\n            # flatten before checking shapes to avoid shape miss match\n            label = label.reshape(-1)\n            pred_label = pred_label.reshape(-1)\n\n            check_label_shapes(label, pred_label)\n\n            num_correct = (pred_label == label).sum().astype('float64')\n            self.sum_metric += num_correct\n            self.num_inst += len(pred_label)\n\n\n@register\n@alias('top_k_accuracy', 'top_k_acc')\n@use_np\nclass TopKAccuracy(EvalMetric):\n    \"\"\"Computes top k predictions accuracy.\n\n    `TopKAccuracy` differs from Accuracy in that it considers the prediction\n    to be ``True`` as long as the ground truth label is in the top K\n    predicated labels.\n\n    If `top_k` = ``1``, then `TopKAccuracy` is identical to `Accuracy`.\n\n    Parameters\n    ----------\n    top_k : int\n        Whether targets are in top k predictions.\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n\n    Examples\n    --------\n    >>> np.random.seed(999)\n    >>> top_k = 3\n    >>> labels = [mx.np.array([2, 6, 9, 2, 3, 4, 7, 8, 9, 6])]\n    >>> predicts = [mx.np.array(np.random.rand(10, 10))]\n    >>> acc = mx.gluon.metric.TopKAccuracy(top_k=top_k)\n    >>> acc.update(labels, predicts)\n    >>> acc.get()\n    ('top_k_accuracy', 0.3)\n    \"\"\"\n\n    def __init__(self, top_k=1, name='top_k_accuracy',\n                 output_names=None, label_names=None):\n        super(TopKAccuracy, self).__init__(\n            name, top_k=top_k,\n            output_names=output_names, label_names=label_names)\n        self.top_k = top_k\n        assert(self.top_k > 1), 'Please use Accuracy if top_k is no more than 1'\n        self.name += f'_{self.top_k}'\n\n    def update(self, labels, preds):\n        \"\"\"Updates the internal evaluation result.\n\n        Parameters\n        ----------\n        labels : list of `NDArray`\n            The labels of the data.\n\n        preds : list of `NDArray`\n            Predicted values.\n        \"\"\"\n        labels, preds = check_label_shapes(labels, preds, True)\n\n        for label, pred_label in zip(labels, preds):\n            assert(len(pred_label.shape) <= 2), 'Predictions should be no more than 2 dims'\n            # Using argpartition here instead of argsort is safe because\n            # we do not care about the order of top k elements. It is\n            # much faster, which is important since that computation is\n            # single-threaded due to Python GIL.\n            pred_label = pred_label.as_np_ndarray().to_device(label.device).astype('float32')\n            pred_label = numpy.argpartition(pred_label, -self.top_k).to_device(label.device)\n            label = label.as_np_ndarray().astype('int32')\n            check_label_shapes(label, pred_label)\n            num_samples = pred_label.shape[0]\n            num_dims = len(pred_label.shape)\n            if num_dims == 1:\n                num_correct = (pred_label.reshape(-1) == label.reshape(-1)).sum()\n                self.sum_metric += num_correct.astype('float64')\n            elif num_dims == 2:\n                num_classes = pred_label.shape[1]\n                top_k = min(num_classes, self.top_k)\n                for j in range(top_k):\n                    num_correct = (pred_label[:, num_classes - 1 - j].reshape(-1) == label.reshape(-1)).sum()\n                    self.sum_metric += num_correct.astype('float64')\n            self.num_inst += num_samples\n\n\ndef predict_with_threshold(pred, threshold=0.5):\n    \"\"\"Do thresholding of predictions in binary and multilabel cases.\n\n    Parameters\n    ----------\n    preds : ndarray\n        predictions in shape of (batch_size, ...) or (batch_size, ..., num_categories)\n\n    preds : float or ndarray\n        threshold（s) in shape of float or (num_categories)\n    \"\"\"\n    if isinstance(threshold, float):\n        return pred > threshold\n    elif isinstance(threshold, (numpy.ndarray, ndarray.ndarray.NDArray)):\n        num_classes = pred.shape[-1]\n        assert threshold.shape[-1] == num_classes, \\\n                f\"shape mismatch: {pred.shape[-1]} vs. {threshold.shape[-1]}\"\n        return pred > threshold\n    else:\n        raise ValueError(\"{} is a wrong type for threshold!\".format(type(threshold)))\n\n\ndef one_hot(idx, num):\n    return (numpy.arange(num).astype(idx) == idx[:, None]).astype('int32')\n\n\n@use_np\nclass _ClassificationMetrics(object):\n    \"\"\"Private container class for classification metric statistics.\n\n    True/false positive and true/false negative counts are sufficient statistics for various classification metrics.\n    This class provides the machinery to track those statistics across mini-batches of\n    (label, prediction) pairs.\n\n    Parameters\n    ----------\n    class_type : str, default \"binary\"\n        \"binary\": f1 for binary classification.\n        \"multiclass\": f1 for multiclassification problem.\n        \"multilabel\": f1 for multilabel classification.\n    beta : float, default 1\n        weight of precision in harmonic mean.\n    threshold : float, default 0.5\n        threshold for deciding whether the predictions are positive or negative.\n\n    \"\"\"\n\n    def __init__(self, class_type=\"binary\", threshold=0.5, beta=1):\n        self.class_type = class_type\n        self.threshold = threshold\n        self.beta = beta\n        self.reset_stats()\n\n    def _set(self, num, device):\n        if self.num_classes is None:\n            self.num_classes = num\n            self.true_positives = numpy.zeros(num, dtype='float64').to_device(device)\n            self.false_negatives = numpy.zeros(num, dtype='float64').to_device(device)\n            self.false_positives = numpy.zeros(num, dtype='float64').to_device(device)\n            self.true_negatives = numpy.zeros(num, dtype='float64').to_device(device)\n        else:\n            assert self.num_classes == num, \\\n                \"Input number of classes has changed from {} to {}\".format(self.num_classes, num)\n\n    def update_stats(self, label, pred):\n        \"\"\"Update various binary classification counts for a single (label, pred) pair.\n\n        Parameters\n        ----------\n        label : `NDArray`\n            The labels of the data.\n\n        pred : `NDArray`\n            Predicted values.\n        \"\"\"\n        pred = pred.as_np_ndarray().to_device(label.device)\n        label = label.as_np_ndarray().astype('int32')\n        if self.class_type == \"binary\":\n            self._set(1, label.device)\n            if label.max() > 1:\n                raise ValueError(\"Wrong label for binary classification.\")\n            if pred.shape == label.shape:\n                pass\n            elif pred.shape[-1] > 2:\n                raise ValueError(\"The shape of prediction {} is wrong for binary classification.\".format(pred.shape))\n            elif pred.shape[-1] == 2:\n                pred = pred.reshape(-1, 2)[:, 1]\n            pred_label = predict_with_threshold(pred, self.threshold).reshape(-1)\n            label = label.reshape(-1)\n\n        elif self.class_type == \"multiclass\":\n            num = pred.shape[-1]\n            self._set(num, label.device)\n            assert label.max() < num, \"pred contains fewer classes than label!\"\n            pred_label = one_hot(pred.argmax(axis=-1).reshape(-1), num)\n            label = one_hot(label.reshape(-1), num)\n\n        elif self.class_type == \"multilabel\":\n            num = pred.shape[-1]\n            self._set(num, label.device)\n            assert pred.shape == label.shape, \\\n                \"The shape of label should be same as that of prediction for multilabel classification.\"\n            pred_label = predict_with_threshold(pred, self.threshold).reshape(-1, num)\n            label = label.reshape(-1, num)\n        else:\n            raise ValueError(\n                \"Wrong class_type {}! Only supports ['binary', 'multiclass', 'multilabel']\".format(self.class_type))\n\n        check_label_shapes(label, pred_label)\n\n        pred_true = (pred_label == 1)\n        pred_false = (pred_label == 0)\n        label_true = (label == 1)\n        label_false = (label == 0)\n\n        true_pos = (pred_true * label_true).sum(0)\n        false_pos = (pred_true * label_false).sum(0)\n        false_neg = (pred_false * label_true).sum(0)\n        true_neg = (pred_false * label_false).sum(0)\n        self.true_positives += true_pos\n        self.false_positives += false_pos\n        self.false_negatives += false_neg\n        self.true_negatives += true_neg\n\n    @property\n    def precision(self):\n        if self.num_classes is not None:\n            return self.true_positives / numpy.maximum(self.true_positives + self.false_positives, 1e-12)\n        else:\n            return 0.\n\n    @property\n    def micro_precision(self):\n        if self.num_classes is not None:\n            return self.true_positives.sum() / \\\n                numpy.maximum(self.true_positives.sum() + self.false_positives.sum(), 1e-12)\n        else:\n            return 0.\n\n    @property\n    def recall(self):\n        if self.num_classes is not None:\n            return self.true_positives / numpy.maximum(self.true_positives + self.false_negatives, 1e-12)\n        else:\n            return 0.\n\n    @property\n    def micro_recall(self):\n        if self.num_classes is not None:\n            return self.true_positives.sum() / \\\n                numpy.maximum(self.true_positives.sum() + self.false_negatives.sum(), 1e-12)\n        else:\n            return 0.\n\n    @property\n    def fscore(self):\n        return (1 + self.beta ** 2) * self.precision * self.recall / \\\n            numpy.maximum(self.beta ** 2 * self.precision + self.recall, 1e-12)\n\n    @property\n    def micro_fscore(self):\n        if self.micro_precision + self.micro_recall > 0:\n            return (1 + self.beta ** 2) * self.micro_precision * self.micro_recall / \\\n                (self.beta ** 2 * self.micro_precision + self.micro_recall)\n        else:\n            return 0.\n\n    def binary_matthewscc(self):\n        \"\"\"Calculate the Matthew's Correlation Coefficent\"\"\"\n        if not self.total_examples:\n            return 0.\n\n        true_pos = float(self.true_positives)\n        false_pos = float(self.false_positives)\n        false_neg = float(self.false_negatives)\n        true_neg = float(self.true_negatives)\n\n        terms = [(true_pos + false_pos),\n                 (true_pos + false_neg),\n                 (true_neg + false_pos),\n                 (true_neg + false_neg)]\n        denom = 1.\n        for t in filter(lambda t: t != 0., terms):\n            denom *= t\n        return ((true_pos * true_neg) - (false_pos * false_neg)) / math.sqrt(denom)\n\n    @property\n    def total_examples(self):\n        if self.num_classes is None:\n            return 0\n        return int(self.false_negatives[0] + self.false_positives[0] + \\\n               self.true_negatives[0] + self.true_positives[0])\n\n    def reset_stats(self):\n        self.num_classes = None\n        self.true_positives = None\n        self.false_negatives = None\n        self.false_positives = None\n        self.true_negatives = None\n\n\n@register\n@use_np\nclass F1(EvalMetric):\n    \"\"\"Computes the F1 score of a binary classification problem.\n\n    The F1 score is equivalent to harmonic mean of the precision and recall,\n    where the best value is 1.0 and the worst value is 0.0. The formula for F1 score is::\n\n        F1 = 2 * (precision * recall) / (precision + recall)\n\n    The formula for precision and recall is::\n\n        precision = true_positives / (true_positives + false_positives)\n        recall    = true_positives / (true_positives + false_negatives)\n\n    .. note::\n\n        This F1 score only supports binary classification.\n\n    Parameters\n    ----------\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n    class_type : str, default \"binary\"\n        \"binary\": f1 for binary classification.\n        \"multiclass\": f1 for multiclassification problem.\n        \"multilabel\": f1 for multilabel classification.\n    threshold : float, default 0.5\n        threshold for postive confidence value.\n    average : str, default 'micro'\n        Strategy to be used for aggregating across mini-batches.\n            \"macro\": Calculate metrics for each label and return unweighted mean of f1.\n            \"micro\": Calculate metrics globally by counting the total TP, FN and FP.\n            None: Return f1 scores for each class (numpy.ndarray) .\n\n    Examples\n    --------\n    >>> predicts = [mx.np.array([[0.3, 0.7], [0., 1.], [0.4, 0.6]])]\n    >>> labels   = [mx.np.array([0., 1., 1.])]\n    >>> f1 = mx.gluon.metric.F1()\n    >>> f1.update(preds = predicts, labels = labels)\n    >>> f1.get()\n    ('f1', 0.8)\n    \"\"\"\n\n    def __init__(self, name='f1',\n                 output_names=None, label_names=None, class_type=\"binary\", threshold=0.5, average=\"micro\"):\n        self.average = average\n        self.metrics = _ClassificationMetrics(class_type=class_type, threshold=threshold)\n        EvalMetric.__init__(self, name=name,\n                            output_names=output_names, label_names=label_names)\n\n    def update(self, labels, preds):\n        \"\"\"Updates the internal evaluation result.\n\n        Parameters\n        ----------\n        labels : list of `NDArray`\n            The labels of the data.\n\n        preds : list of `NDArray`\n            Predicted values.\n        \"\"\"\n        labels, preds = check_label_shapes(labels, preds, True)\n\n        for label, pred in zip(labels, preds):\n            self.metrics.update_stats(label, pred)\n\n        if self.average == \"micro\":\n            self.sum_metric = self.metrics.micro_fscore * self.metrics.total_examples\n        elif self.average == \"macro\":\n            self.sum_metric = self.metrics.fscore.mean() * self.metrics.total_examples\n        else:\n            self.sum_metric = self.metrics.fscore * self.metrics.total_examples\n        self.num_inst = self.metrics.total_examples\n\n    def reset(self):\n        \"\"\"Resets the internal evaluation result to initial state.\"\"\"\n        self.sum_metric = 0.\n        self.num_inst = 0\n        self.metrics.reset_stats()\n\n\n@register\n@use_np\nclass Fbeta(F1):\n    \"\"\"Computes the Fbeta score of a binary classification problem.\n\n    The Fbeta score is equivalent to harmonic mean of the precision and recall,\n    where the best value is 1.0 and the worst value is 0.0. The formula for Fbeta score is::\n\n        Fbeta = (1 + beta ** 2) * (precision * recall) / (beta ** 2 * precision + recall)\n\n    The formula for precision and recall is::\n\n        precision = true_positives / (true_positives + false_positives)\n        recall    = true_positives / (true_positives + false_negatives)\n\n    .. note::\n\n        This Fbeta score only supports binary classification.\n\n    Parameters\n    ----------\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n    class_type : str, default \"binary\"\n        \"binary\": f1 for binary classification.\n        \"multiclass\": f1 for multiclassification problem.\n        \"multilabel\": f1 for multilabel classification.\n    beta : float, default 1\n        weight of precision in harmonic mean.\n    threshold : float, default 0.5\n        threshold for postive confidence value.\n    average : str, default 'micro'\n        Strategy to be used for aggregating across mini-batches.\n            \"macro\": Calculate metrics for each label and return unweighted mean of f1.\n            \"micro\": Calculate metrics globally by counting the total TP, FN and FP.\n            None: Return f1 scores for each class.\n\n    Examples\n    --------\n    >>> predicts = [mx.np.array([[0.3, 0.7], [0., 1.], [0.4, 0.6]])]\n    >>> labels   = [mx.np.array([0., 1., 1.])]\n    >>> fbeta = mx.gluon.metric.Fbeta(beta=2)\n    >>> fbeta.update(preds = predicts, labels = labels)\n    >>> fbeta.get()\n    ('fbeta', 0.9090909090909091)\n    \"\"\"\n\n    def __init__(self, name='fbeta',\n                 output_names=None, label_names=None, class_type=\"binary\", beta=1, threshold=0.5, average=\"micro\"):\n        super(Fbeta, self).__init__(\n            name=name, output_names=output_names, label_names=label_names,\n            class_type=class_type, threshold=threshold, average=average)\n        self.metrics = _ClassificationMetrics(class_type=class_type, threshold=threshold, beta=beta)\n\n\n@register\n@use_np\nclass BinaryAccuracy(EvalMetric):\n    \"\"\"Computes the accuracy of a binary or multilabel classification problem.\n\n    Parameters\n    ----------\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n    threshold : float or ndarray, default 0.5\n        threshold for deciding whether the predictions are positive or negative.\n\n    Examples\n    --------\n    >>> predicts = [mx.np.array([0.7, 1, 0.55])]\n    >>> labels   = [mx.np.array([0., 1., 0.])]\n    >>> bacc = mx.gluon.metric.BinaryAccuracy(threshold=0.6)\n    >>> bacc.update(preds = predicts, labels = labels)\n    >>> bacc.get()\n    ('binary_accuracy', 0.6666666666666666)\n    \"\"\"\n\n    def __init__(self, name='binary_accuracy',\n                 output_names=None, label_names=None, threshold=0.5):\n        self.threshold = threshold\n        EvalMetric.__init__(self, name=name,\n                            output_names=output_names, label_names=label_names)\n\n    def update(self, labels, preds):\n        \"\"\"Updates the internal evaluation result.\n\n        Parameters\n        ----------\n        labels : list of `NDArray`\n            Each label denotes positive/negative for each class.\n\n        preds : list of `NDArray`\n            Each prediction value is a confidence value of being positive for each class.\n        \"\"\"\n        labels, preds = check_label_shapes(labels, preds, True)\n\n        for label, pred_label in zip(labels, preds):\n            pred_label = predict_with_threshold(pred_label, self.threshold)\n\n            pred_label = pred_label.as_np_ndarray().astype('int32').to_device(label.device)\n            label = label.as_np_ndarray().astype('int32')\n            # flatten before checking shapes to avoid shape miss match\n            label = label.reshape(-1)\n            pred_label = pred_label.reshape(-1)\n\n            check_label_shapes(label, pred_label)\n\n            num_correct = (pred_label == label).sum().astype('float64')\n            self.sum_metric += num_correct\n            self.num_inst += len(pred_label)\n\n\n@register\n@use_np\nclass MCC(EvalMetric):\n    \"\"\"Computes the Matthews Correlation Coefficient of a binary classification problem.\n\n    While slower to compute than F1 the MCC can give insight that F1 or Accuracy cannot.\n    For instance, if the network always predicts the same result\n    then the MCC will immeadiately show this. The MCC is also symetric with respect\n    to positive and negative categorization, however, there needs to be both\n    positive and negative examples in the labels or it will always return 0.\n    MCC of 0 is uncorrelated, 1 is completely correlated, and -1 is negatively correlated.\n\n    .. math::\n\n        \\\\text{MCC} = \\\\frac{ TP \\\\times TN - FP \\\\times FN }\n        {\\\\sqrt{ (TP + FP) ( TP + FN ) ( TN + FP ) ( TN + FN ) } }\n\n    where 0 terms in the denominator are replaced by 1.\n\n    .. note::\n\n        This version of MCC only supports binary classification.  See PCC.\n\n    Parameters\n    ----------\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n\n    Examples\n    --------\n    >>> # In this example the network almost always predicts positive\n    >>> false_positives = 1000\n    >>> false_negatives = 1\n    >>> true_positives = 10000\n    >>> true_negatives = 1\n    >>> predicts = [mx.np.array(\n        [[.3, .7]]*false_positives +\n        [[.7, .3]]*true_negatives +\n        [[.7, .3]]*false_negatives +\n        [[.3, .7]]*true_positives\n    )]\n    >>> labels  = [mx.np.array(\n        [0.]*(false_positives + true_negatives) +\n        [1.]*(false_negatives + true_positives)\n    )]\n    >>> f1 = mx.gluon.metric.F1()\n    >>> f1.update(preds = predicts, labels = labels)\n    >>> mcc = mx.gluon.metric.MCC()\n    >>> mcc.update(preds = predicts, labels = labels)\n    >>> f1.get()\n    ('f1', 0.95233560306652054)\n    >>> mcc.get()\n    ('mcc', 0.01917751877733392)\n    \"\"\"\n\n    def __init__(self, name='mcc',\n                 output_names=None, label_names=None):\n        self._metrics = _ClassificationMetrics()\n        EvalMetric.__init__(self, name=name,\n                            output_names=output_names, label_names=label_names)\n\n    def update(self, labels, preds):\n        \"\"\"Updates the internal evaluation result.\n\n        Parameters\n        ----------\n        labels : list of `NDArray`\n            The labels of the data.\n\n        preds : list of `NDArray`\n            Predicted values.\n        \"\"\"\n        labels, preds = check_label_shapes(labels, preds, True)\n\n        for label, pred in zip(labels, preds):\n            self._metrics.update_stats(label, pred)\n\n        self.sum_metric = self._metrics.binary_matthewscc() * self._metrics.total_examples\n        self.num_inst = self._metrics.total_examples\n\n    def reset(self):\n        \"\"\"Resets the internal evaluation result to initial state.\"\"\"\n        self.sum_metric = 0.\n        self.num_inst = 0.\n        self._metrics.reset_stats()\n\n\n####################\n# REGRESSION METRICS\n####################\n\n\n@register\n@use_np\nclass MAE(EvalMetric):\n    \"\"\"Computes Mean Absolute Error (MAE) loss.\n\n    The mean absolute error is given by\n\n    .. math::\n\n        \\\\frac{\\\\sum_i^n |y_i - \\\\hat{y}_i|}{n}\n\n    Parameters\n    ----------\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n\n    Examples\n    --------\n    >>> predicts = [mx.np.array([3, -0.5, 2, 7])]\n    >>> labels = [mx.np.array([2.5, 0.0, 2, 8])]\n    >>> mean_absolute_error = mx.gluon.metric.MAE()\n    >>> mean_absolute_error.update(labels = labels, preds = predicts)\n    >>> mean_absolute_error.get()\n    ('mae', 0.5)\n    \"\"\"\n\n    def __init__(self, name='mae',\n                 output_names=None, label_names=None):\n        super(MAE, self).__init__(\n            name, output_names=output_names, label_names=label_names)\n\n    def update(self, labels, preds):\n        \"\"\"Updates the internal evaluation result.\n\n        Parameters\n        ----------\n        labels : list of `NDArray`\n            The labels of the data.\n\n        preds : list of `NDArray`\n            Predicted values.\n        \"\"\"\n        labels, preds = check_label_shapes(labels, preds, True)\n\n        for label, pred in zip(labels, preds):\n            label = label.as_np_ndarray()\n            pred = pred.as_np_ndarray().to_device(label.device)\n\n            num_inst = label.shape[0]\n            mae = numpy.abs(label - pred).reshape(num_inst, -1).mean(axis=-1).sum()\n\n            self.sum_metric += mae\n            self.num_inst += num_inst\n\n\n@register\n@use_np\nclass MSE(EvalMetric):\n    \"\"\"Computes Mean Squared Error (MSE) loss.\n\n    The mean squared error is given by\n\n    .. math::\n\n        \\\\frac{\\\\sum_i^n (y_i - \\\\hat{y}_i)^2}{n}\n\n    Parameters\n    ----------\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n\n    Examples\n    --------\n    >>> predicts = [mx.np.array([3, -0.5, 2, 7])]\n    >>> labels = [mx.np.array([2.5, 0.0, 2, 8])]\n    >>> mean_squared_error = mx.gluon.metric.MSE()\n    >>> mean_squared_error.update(labels = labels, preds = predicts)\n    >>> mean_squared_error.get()\n    ('mse', 0.375)\n    \"\"\"\n    def __init__(self, name='mse',\n                 output_names=None, label_names=None):\n        super(MSE, self).__init__(\n            name, output_names=output_names, label_names=label_names)\n\n    def update(self, labels, preds):\n        \"\"\"Updates the internal evaluation result.\n\n        Parameters\n        ----------\n        labels : list of `NDArray`\n            The labels of the data.\n\n        preds : list of `NDArray`\n            Predicted values.\n        \"\"\"\n        labels, preds = check_label_shapes(labels, preds, True)\n\n        for label, pred in zip(labels, preds):\n            label = label.as_np_ndarray()\n            pred = pred.as_np_ndarray().to_device(label.device)\n\n            num_inst = label.shape[0]\n            mse = ((label - pred)**2.0).reshape(num_inst, -1).mean(axis=-1).sum()\n\n            self.sum_metric += mse\n            self.num_inst += num_inst\n\n\n@register\n@use_np\nclass RMSE(MSE):\n    \"\"\"Computes Root Mean Squred Error (RMSE) loss.\n\n    The root mean squared error is given by\n\n    .. math::\n\n        \\\\sqrt{\\\\frac{\\\\sum_i^n (y_i - \\\\hat{y}_i)^2}{n}}\n\n    Parameters\n    ----------\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n\n    Examples\n    --------\n    >>> predicts = [mx.np.array([3, -0.5, 2, 7])]\n    >>> labels = [mx.np.array([2.5, 0.0, 2, 8])]\n    >>> root_mean_squared_error = mx.gluon.metric.RMSE()\n    >>> root_mean_squared_error.update(labels = labels, preds = predicts)\n    >>> root_mean_squared_error.get()\n    ('rmse', 0.612372457981)\n    \"\"\"\n    def __init__(self, name='rmse',\n                 output_names=None, label_names=None):\n        super(RMSE, self).__init__(\n            name, output_names=output_names, label_names=label_names)\n\n    def get(self):\n        if self.num_inst == 0:\n            return (self.name, float('nan'))\n        else:\n            return (self.name, math.sqrt(self.sum_metric / self.num_inst))\n\n\n@register\n@use_np\nclass MeanPairwiseDistance(EvalMetric):\n    \"\"\"Computes Mean Pairwise Distance.\n\n    The mean pairwise distance is given by\n\n    .. math::\n\n        \\\\sqrt{\\\\frac{(\\\\sum_i^n (y_i - \\\\hat{y}_i)^p)^\\\\frac{1}{p}}{n}}\n\n    Parameters\n    ----------\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n    p : float, default 2\n        calculating distance using the p-norm\n\n    Examples\n    --------\n    >>> predicts = [mx.np.array([[1., 2.], [3., 4.]])]\n    >>> labels = [mx.np.array([[1., 0.], [4., 2.]])]\n    >>> mpd = mx.gluon.metric.MeanPairwiseDistance()\n    >>> mpd.update(labels = labels, preds = predicts)\n    >>> mpd.get()\n    ('mpd', 2.1180338859558105)\n    \"\"\"\n    def __init__(self, name='mpd',\n                 output_names=None, label_names=None, p=2):\n        super(MeanPairwiseDistance, self).__init__(\n            name, output_names=output_names, label_names=label_names)\n        self.p = p\n\n    def update(self, labels, preds):\n        \"\"\"Updates the internal evaluation result.\n\n        Parameters\n        ----------\n        labels : list of `NDArray`\n            The labels of the data.\n\n        preds : list of `NDArray`\n            Predicted values.\n        \"\"\"\n        labels, preds = check_label_shapes(labels, preds, True)\n\n        for label, pred in zip(labels, preds):\n            label = label.as_np_ndarray()\n            pred = pred.as_np_ndarray().to_device(label.device)\n\n            label = label.reshape(label.shape[0], -1)\n            pred = pred.reshape(pred.shape[0], -1)\n\n            dis = (((label - pred) ** self.p).sum(axis=-1)) ** (1./self.p)\n            dis = dis.sum()\n            num_inst = label.shape[0]\n\n            self.sum_metric += dis\n            self.num_inst += num_inst\n\n\n@register\n@use_np\nclass MeanCosineSimilarity(EvalMetric):\n    r\"\"\"Computes Mean Cosine Similarity.\n\n    The mean cosine similarity is given by\n\n    .. math::\n\n        cos_sim(label, pred) = \\frac{{label}.{pred}}{max(||label||.||pred||, eps)}\n\n    Calculation happens on the last dimension of label and pred.\n\n    Parameters\n    ----------\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n    eps : float, default 1e-8\n        small vale to avoid division by zero.\n\n    Examples\n    --------\n    >>> predicts = [mx.np.array([[1., 0.], [1., 1.]])]\n    >>> labels = [mx.np.array([[3., 4.], [2., 2.]])]\n    >>> mcs = mx.gluon.metric.MeanCosineSimilarity()\n    >>> mcs.update(labels = labels, preds = predicts)\n    >>> mcs.get()\n    ('cos_sim', 0.8)\n    \"\"\"\n    def __init__(self, name='cos_sim',\n                 output_names=None, label_names=None, eps=1e-8):\n        super(MeanCosineSimilarity, self).__init__(\n            name, output_names=output_names, label_names=label_names)\n        self.eps = eps\n\n    def update(self, labels, preds):\n        \"\"\"Updates the internal evaluation result.\n\n        Parameters\n        ----------\n        labels : list of `NDArray`\n            The labels of the data.\n\n        preds : list of `NDArray`\n            Predicted values.\n        \"\"\"\n        labels, preds = check_label_shapes(labels, preds, True)\n\n        for label, pred in zip(labels, preds):\n            label = label.as_np_ndarray()\n            pred = pred.as_np_ndarray().to_device(label.device)\n\n            if len(label.shape) == 1:\n                label = label.reshape(1, label.shape[0])\n            if len(pred.shape) == 1:\n                pred = pred.reshape(1, pred.shape[0])\n\n            sim = (label * pred).sum(axis=-1)\n            n_p = numpy.linalg.norm(pred, axis=-1)\n            n_l = numpy.linalg.norm(label, axis=-1)\n            sim = sim / numpy.maximum(n_l * n_p, self.eps)\n            sim = sim.sum()\n            num_inst = len(label.reshape(-1, label.shape[-1])) # numpy.prod(label.shape[:-1]) is not supported\n            self.sum_metric += sim\n            self.num_inst += num_inst\n\n\n@register\n@alias('ce')\n@use_np\nclass CrossEntropy(EvalMetric):\n    \"\"\"Computes Cross Entropy loss.\n\n    The cross entropy over a batch of sample size :math:`N` is given by\n\n    .. math::\n\n       -\\\\sum_{n=1}^{N}\\\\sum_{k=1}^{K}t_{nk}\\\\log (y_{nk}),\n\n    where :math:`t_{nk}=1` if and only if sample :math:`n` belongs to class :math:`k`.\n    :math:`y_{nk}` denotes the probability of sample :math:`n` belonging to\n    class :math:`k`.\n\n    Parameters\n    ----------\n    eps : float, default 1e-12\n        Use small constant for the case that predicted value is 0.\n    ignore_label : int or None, default None\n        Index of invalid label to ignore when\n        counting. By default, sets to -1.\n        If set to `None`, it will include all entries.\n    axis : int, default -1\n        The axis from prediction that was used to\n        compute softmax. By default use the last axis.\n    from_logits : boolean, default False\n        Whether `pred` is expected to be a logits tensor.\n        By default, we assume that `pred` encodes a probability distribution.\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n\n    Examples\n    --------\n    >>> predicts = [mx.np.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])]\n    >>> labels   = [mx.np.array([0, 1, 1])]\n    >>> ce = mx.gluon.metric.CrossEntropy()\n    >>> ce.update(labels, predicts)\n    >>> ce.get()\n    ('cross-entropy', 0.57159948348999023)\n    \"\"\"\n    def __init__(self, eps=1e-12, ignore_label=None, axis=-1, from_logits=False,\n                 name='cross-entropy', output_names=None, label_names=None):\n        super(CrossEntropy, self).__init__(\n            name, output_names=output_names, label_names=label_names)\n        self.ignore_label = ignore_label\n        self.axis = axis\n        self.from_logits = from_logits\n        self.eps = eps\n\n    def update(self, labels, preds):\n        \"\"\"Updates the internal evaluation result.\n\n        Parameters\n        ----------\n        labels : list of `NDArray`\n            The labels of the data.\n\n        preds : list of `NDArray`\n            Predicted values.\n        \"\"\"\n        labels, preds = check_label_shapes(labels, preds, True)\n\n        loss = 0.\n        num = 0\n        for label, pred in zip(labels, preds):\n            assert label.size == pred.size/pred.shape[-1], \\\n                f\"shape mismatch: {label.shape} vs. {pred.shape}\"\n            label = label.reshape((label.size,))\n            if self.from_logits:\n                pred = npx.softmax(pred, axis=self.axis)\n            pred = npx.pick(pred.to_device(label.device), label.astype(dtype='int32'), axis=self.axis)\n            if self.ignore_label is not None:\n                ignore = (label == self.ignore_label).astype(pred.dtype)\n                num -= ignore.sum()\n                pred = pred * (1 - ignore) + ignore\n            loss -= numpy.log(numpy.maximum(self.eps, pred)).sum()\n            num += pred.size\n        self.sum_metric += loss\n        self.num_inst += num\n\n\n@register\n@use_np\nclass Perplexity(CrossEntropy):\n    \"\"\"Computes perplexity.\n\n    Perplexity is a measurement of how well a probability distribution\n    or model predicts a sample. A low perplexity indicates the model\n    is good at predicting the sample.\n\n    The perplexity of a model q is defined as\n\n    .. math::\n\n        b^{\\\\big(-\\\\frac{1}{N} \\\\sum_{i=1}^N \\\\log_b q(x_i) \\\\big)}\n        = \\\\exp \\\\big(-\\\\frac{1}{N} \\\\sum_{i=1}^N \\\\log q(x_i)\\\\big)\n\n    where we let `b = e`.\n\n    :math:`q(x_i)` is the predicted value of its ground truth\n    label on sample :math:`x_i`.\n\n    For example, we have three samples :math:`x_1, x_2, x_3` and their labels\n    are :math:`[0, 1, 1]`.\n    Suppose our model predicts :math:`q(x_1) = p(y_1 = 0 | x_1) = 0.3`\n    and :math:`q(x_2) = 1.0`,\n    :math:`q(x_3) = 0.6`. The perplexity of model q is\n    :math:`exp\\\\big(-(\\\\log 0.3 + \\\\log 1.0 + \\\\log 0.6) / 3\\\\big) = 1.77109762852`.\n\n    Parameters\n    ----------\n    eps : float, default 1e-12\n        Use small constant for the case that predicted value is 0.\n    ignore_label : int or None, default None\n        Index of invalid label to ignore when\n        counting. By default, sets to -1.\n        If set to `None`, it will include all entries.\n    axis : int (default -1)\n        The axis from prediction that was used to\n        compute softmax. By default use the last axis.\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n\n    Examples\n    --------\n    >>> predicts = [mx.np.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])]\n    >>> labels   = [mx.np.array([0, 1, 1])]\n    >>> perp = mx.gluon.metric.Perplexity(ignore_label=None)\n    >>> perp.update(labels, predicts)\n    >>> perp.get()\n    ('Perplexity', 1.7710976285155853)\n    \"\"\"\n    def __init__(self, eps=1e-12, ignore_label=None, axis=-1, from_logits=False,\n                 name='perplexity', output_names=None, label_names=None):\n        super(Perplexity, self).__init__(\n            eps=eps, ignore_label=ignore_label, axis=axis, from_logits=from_logits,\n            name=name, output_names=output_names, label_names=label_names)\n\n    def get(self):\n        if self.num_inst == 0:\n            return (self.name, float('nan'))\n        else:\n            return (self.name, math.exp(self.sum_metric/self.num_inst))\n\n\n@register\n@alias('pearsonr')\n@use_np\nclass PearsonCorrelation(EvalMetric):\n    \"\"\"Computes Pearson correlation.\n\n    The pearson correlation is given by\n\n    .. math::\n\n        \\\\frac{cov(y, \\\\hat{y})}{\\\\sigma{y}\\\\sigma{\\\\hat{y}}}\n\n    Parameters\n    ----------\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n\n    Examples\n    --------\n    >>> predicts = [mx.np.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])]\n    >>> labels   = [mx.np.array([[1, 0], [0, 1], [0, 1]])]\n    >>> pr = mx.gluon.metric.PearsonCorrelation()\n    >>> pr.update(labels, predicts)\n    >>> pr.get()\n    ('pearsonr', 0.42163704544016178)\n    \"\"\"\n    def __init__(self, name='pearsonr',\n                 output_names=None, label_names=None):\n        super(PearsonCorrelation, self).__init__(\n            name, output_names=output_names, label_names=label_names)\n        self.reset()\n\n    def reset(self):\n        self._sse_p = 0\n        self._mean_p = 0\n        self._sse_l = 0\n        self._mean_l = 0\n        self._pred_nums = 0\n        self._label_nums = 0\n        self._conv = 0\n\n        self.num_inst = 0\n        self.sum_metric = 0.0\n\n    def update_variance(self, new_values, *aggregate):\n        #Welford's online algorithm for variance update\n        count, mean, m_2 = aggregate\n        count += len(new_values)\n        delta = new_values - mean\n        mean += numpy.sum(delta / count)\n        delta_2 = new_values - mean\n        m_2 += numpy.sum(delta * delta_2)\n        return count, mean, m_2\n\n    def update_cov(self, label, pred):\n        self._conv = self._conv + numpy.sum((label - self._mean_l) * (pred - self._mean_p))\n\n    def update(self, labels, preds):\n        \"\"\"Updates the internal evaluation result.\n\n        Parameters\n        ----------\n        labels : list of `NDArray`\n            The labels of the data.\n        preds : list of `NDArray`\n            Predicted values.\n        \"\"\"\n        labels, preds = check_label_shapes(labels, preds, True)\n        for label, pred in zip(labels, preds):\n            check_label_shapes(label, pred, False, True)\n            label = label.as_np_ndarray().reshape(-1).astype(numpy.float64)\n            pred = pred.as_np_ndarray().to_device(label.device).reshape(-1).astype(numpy.float64)\n\n            self.num_inst += 1\n            self._label_nums, self._mean_l, self._sse_l = \\\n                self.update_variance(label, self._label_nums, self._mean_l, self._sse_l)\n            self.update_cov(label, pred)\n            self._pred_nums, self._mean_p, self._sse_p = \\\n                self.update_variance(pred, self._pred_nums, self._mean_p, self._sse_p)\n\n    def get(self):\n        if self.num_inst == 0:\n            return (self.name, float('nan'))\n\n        n = self._label_nums\n        pearsonr = self._conv / ((n-1) * numpy.sqrt(self._sse_p / (n - 1)) * numpy.sqrt(self._sse_l / (n - 1)))\n        return (self.name, float(pearsonr))\n\n@register\n@use_np\nclass PCC(EvalMetric):\n    \"\"\"PCC is a multiclass equivalent for the Matthews correlation coefficient derived\n    from a discrete solution to the Pearson correlation coefficient.\n\n    .. math::\n\n        \\\\text{PCC} = \\\\frac {\\\\sum _{k}\\\\sum _{l}\\\\sum _{m}C_{kk}C_{lm}-C_{kl}C_{mk}}\n        {{\\\\sqrt {\\\\sum _{k}(\\\\sum _{l}C_{kl})(\\\\sum _{k'|k'\\\\neq k}\\\\sum _{l'}C_{k'l'})}}\n         {\\\\sqrt {\\\\sum _{k}(\\\\sum _{l}C_{lk})(\\\\sum _{k'|k'\\\\neq k}\\\\sum _{l'}C_{l'k'})}}}\n\n    defined in terms of a K x K confusion matrix C.\n\n    When there are more than two labels the PCC will no longer range between -1 and +1.\n    Instead the minimum value will be between -1 and 0 depending on the true distribution.\n    The maximum value is always +1.\n\n    Parameters\n    ----------\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n\n    Examples\n    --------\n    >>> # In this example the network almost always predicts positive\n    >>> false_positives = 1000\n    >>> false_negatives = 1\n    >>> true_positives = 10000\n    >>> true_negatives = 1\n    >>> predicts = [mx.np.array(\n        [[.3, .7]]*false_positives +\n        [[.7, .3]]*true_negatives +\n        [[.7, .3]]*false_negatives +\n        [[.3, .7]]*true_positives\n    )]\n    >>> labels  = [mx.np.array(\n        [0]*(false_positives + true_negatives) +\n        [1]*(false_negatives + true_positives)\n    )]\n    >>> f1 = mx.gluon.metric.F1()\n    >>> f1.update(preds = predicts, labels = labels)\n    >>> pcc = mx.gluon.metric.PCC()\n    >>> pcc.update(preds = predicts, labels = labels)\n    >>> f1.get()\n    ('f1', 0.95233560306652054)\n    >>> pcc.get()\n    ('pcc', 0.01917751877733392)\n    \"\"\"\n    def __init__(self, name='pcc',\n                 output_names=None, label_names=None):\n        self.k = 2\n        super(PCC, self).__init__(\n            name=name, output_names=output_names, label_names=label_names)\n\n    def _grow(self, inc):\n        self.lcm = numpy.pad(\n            self.lcm, ((0, inc), (0, inc)), 'constant', constant_values=(0))\n        self.k += inc\n\n    def _calc_mcc(self, cmat):\n        n = cmat.sum()\n        x = cmat.sum(axis=1)\n        y = cmat.sum(axis=0)\n        cov_xx = numpy.sum(x * (n - x))\n        cov_yy = numpy.sum(y * (n - y))\n        if cov_xx == 0 or cov_yy == 0:\n            return float('nan')\n        # i = cmat.diagonal() # mxnet.numpy.ndarray.diagonal() is currently not available.\n        i = cmat[numpy.arange(self.k), numpy.arange(self.k)]\n        cov_xy = numpy.sum(i * n - x * y)\n        return cov_xy / (cov_xx * cov_yy) ** 0.5\n\n    def update(self, labels, preds):\n        \"\"\"Updates the internal evaluation result.\n\n        Parameters\n        ----------\n        labels : list of `NDArray`\n            The labels of the data.\n\n        preds : list of `NDArray`\n            Predicted values.\n        \"\"\"\n        labels, preds = check_label_shapes(labels, preds, True)\n\n        # update the confusion matrix\n        for label, pred in zip(labels, preds):\n            label = label.astype('int32', copy=False).as_np_ndarray()\n            pred = pred.as_np_ndarray().to_device(label.device)\n            if pred.shape != label.shape:\n                pred = pred.argmax(axis=1).astype(label, copy=False)\n            else:\n                pred = pred.astype('int32', copy=False)\n            n = int(max(pred.max(), label.max()))\n            if n >= self.k:\n                self._grow(n + 1 - self.k)\n            bcm = numpy.zeros((self.k, self.k), dtype='float64')\n            for i, j in zip(pred, label):\n                bcm[i, j] += 1\n            self.lcm += bcm\n        self.num_inst += 1\n\n    @property\n    def sum_metric(self):\n        return self._calc_mcc(self.lcm) * self.num_inst\n\n    def reset(self):\n        \"\"\"Resets the internal evaluation result to initial state.\"\"\"\n        self.num_inst = 0.\n        self.lcm = numpy.zeros((self.k, self.k), dtype='float64')\n\n\n@register\n@use_np\nclass Loss(EvalMetric):\n    \"\"\"Dummy metric for directly printing loss.\n\n    Parameters\n    ----------\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n    \"\"\"\n    def __init__(self, name='loss',\n                 output_names=None, label_names=None):\n        super(Loss, self).__init__(\n            name, output_names=output_names, label_names=label_names)\n\n    def update(self, _, preds):\n\n        if isinstance(preds, ndarray.ndarray.NDArray):\n            preds = [preds]\n\n        for pred in preds:\n            loss = pred.sum().item()\n            self.sum_metric += loss\n            self.num_inst += pred.size\n\n\n@register\nclass Torch(Loss):\n    \"\"\"Dummy metric for torch criterions.\"\"\"\n    def __init__(self, name='torch',\n                 output_names=None, label_names=None):\n        super(Torch, self).__init__(\n            name, output_names=output_names, label_names=label_names)\n\n\n@register\n@use_np\nclass CustomMetric(EvalMetric):\n    \"\"\"Computes a customized evaluation metric.\n\n    The `feval` function can return a `tuple` of (sum_metric, num_inst) or return\n    an `int` sum_metric.\n\n    Parameters\n    ----------\n    feval : callable(label, pred)\n        Customized evaluation function.\n    name : str, optional\n        The name of the metric. (the default is None).\n    allow_extra_outputs : bool, optional\n        If true, the prediction outputs can have extra outputs.\n        This is useful in RNN, where the states are also produced\n        in outputs for forwarding. (the default is False).\n    name : str\n        Name of this metric instance for display.\n    output_names : list of str, or None\n        Name of predictions that should be used when updating with update_dict.\n        By default include all predictions.\n    label_names : list of str, or None\n        Name of labels that should be used when updating with update_dict.\n        By default include all labels.\n\n    Examples\n    --------\n    >>> predicts = [mx.np.array(np.array([3, -0.5, 2, 7]).reshape(4,1))]\n    >>> labels = [mx.np.array(np.array([2.5, 0.0, 2, 8]).reshape(4,1))]\n    >>> feval = lambda x, y : (x + y).mean()\n    >>> eval_metrics = mx.gluon.metric.CustomMetric(feval=feval)\n    >>> eval_metrics.update(labels, predicts)\n    >>> eval_metrics.get()\n    ('custom(<lambda>)', 6.0)\n    \"\"\"\n    def __init__(self, feval, name=None, allow_extra_outputs=False,\n                 output_names=None, label_names=None):\n        if name is None:\n            name = feval.__name__\n            if name.find('<') != -1:\n                name = f'custom({name})'\n        super(CustomMetric, self).__init__(\n            name, feval=feval,\n            allow_extra_outputs=allow_extra_outputs,\n            output_names=output_names, label_names=label_names)\n        self._feval = feval\n        self._allow_extra_outputs = allow_extra_outputs\n\n    def update(self, labels, preds):\n        \"\"\"Updates the internal evaluation result.\n\n        Parameters\n        ----------\n        labels : list of `NDArray`\n            The labels of the data.\n\n        preds : list of `NDArray`\n            Predicted values.\n        \"\"\"\n        if not self._allow_extra_outputs:\n            labels, preds = check_label_shapes(labels, preds, True)\n\n        for pred, label in zip(preds, labels):\n            label = label.as_np_ndarray()\n            pred = pred.as_np_ndarray().to_device(label.device)\n\n            reval = self._feval(label, pred)\n            if isinstance(reval, tuple):\n                (sum_metric, num_inst) = reval\n                self.sum_metric += sum_metric\n                self.num_inst += num_inst\n            else:\n                self.sum_metric += reval\n                self.num_inst += 1\n\n    def get_config(self):\n        raise NotImplementedError(\"CustomMetric cannot be serialized\")\n\n\n# pylint: disable=invalid-name\ndef np(numpy_feval, name=None, allow_extra_outputs=False):\n    \"\"\"Creates a custom evaluation metric that receives its inputs as numpy arrays.\n\n    Parameters\n    ----------\n    numpy_feval : callable(label, pred)\n        Custom evaluation function that receives labels and predictions for a minibatch\n        as numpy arrays and returns the corresponding custom metric as a floating point number.\n    name : str, optional\n        Name of the custom metric.\n    allow_extra_outputs : bool, optional\n        Whether prediction output is allowed to have extra outputs. This is useful in cases\n        like RNN where states are also part of output which can then be fed back to the RNN\n        in the next step. By default, extra outputs are not allowed.\n\n    Returns\n    -------\n    float\n        Custom metric corresponding to the provided labels and predictions.\n\n    Example\n    -------\n    >>> def custom_metric(label, pred):\n    ...     return np.mean(np.abs(label-pred))\n    ...\n    >>> metric = mx.gluon.metric.np(custom_metric)\n    \"\"\"\n    def feval(label, pred):\n        \"\"\"Internal eval function.\"\"\"\n        return numpy_feval(label, pred)\n    feval.__name__ = numpy_feval.__name__\n    return CustomMetric(feval, name, allow_extra_outputs)\n# pylint: enable=invalid-name\n"
  },
  {
    "path": "python/mxnet/gluon/model_zoo/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Predefined and pretrained models.\"\"\"\n\nfrom . import model_store\n\nfrom . import vision\n"
  },
  {
    "path": "python/mxnet/gluon/model_zoo/model_store.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Model zoo for pre-trained models.\"\"\"\n__all__ = ['get_model_file', 'purge']\nimport os\nimport zipfile\nimport logging\nimport uuid\n\nfrom ..utils import download, check_sha1, replace_file, TemporaryDirectory\nfrom ... import base\n\n_model_sha1 = {name: checksum for checksum, name in [\n    ('44335d1f0046b328243b32a26a4fbd62d9057b45', 'alexnet'),\n    ('f27dbf2dbd5ce9a80b102d89c7483342cd33cb31', 'densenet121'),\n    ('b6c8a95717e3e761bd88d145f4d0a214aaa515dc', 'densenet161'),\n    ('2603f878403c6aa5a71a124c4a3307143d6820e9', 'densenet169'),\n    ('1cdbc116bc3a1b65832b18cf53e1cb8e7da017eb', 'densenet201'),\n    ('ed47ec45a937b656fcc94dabde85495bbef5ba1f', 'inceptionv3'),\n    ('9f83e440996887baf91a6aff1cccc1c903a64274', 'mobilenet0.25'),\n    ('8e9d539cc66aa5efa71c4b6af983b936ab8701c3', 'mobilenet0.5'),\n    ('529b2c7f4934e6cb851155b22c96c9ab0a7c4dc2', 'mobilenet0.75'),\n    ('6b8c5106c730e8750bcd82ceb75220a3351157cd', 'mobilenet1.0'),\n    ('36da4ff1867abccd32b29592d79fc753bca5a215', 'mobilenetv2_1.0'),\n    ('e2be7b72a79fe4a750d1dd415afedf01c3ea818d', 'mobilenetv2_0.75'),\n    ('aabd26cd335379fcb72ae6c8fac45a70eab11785', 'mobilenetv2_0.5'),\n    ('ae8f9392789b04822cbb1d98c27283fc5f8aa0a7', 'mobilenetv2_0.25'),\n    ('a0666292f0a30ff61f857b0b66efc0228eb6a54b', 'resnet18_v1'),\n    ('48216ba99a8b1005d75c0f3a0c422301a0473233', 'resnet34_v1'),\n    ('0aee57f96768c0a2d5b23a6ec91eb08dfb0a45ce', 'resnet50_v1'),\n    ('d988c13d6159779e907140a638c56f229634cb02', 'resnet101_v1'),\n    ('671c637a14387ab9e2654eafd0d493d86b1c8579', 'resnet152_v1'),\n    ('a81db45fd7b7a2d12ab97cd88ef0a5ac48b8f657', 'resnet18_v2'),\n    ('9d6b80bbc35169de6b6edecffdd6047c56fdd322', 'resnet34_v2'),\n    ('ecdde35339c1aadbec4f547857078e734a76fb49', 'resnet50_v2'),\n    ('18e93e4f48947e002547f50eabbcc9c83e516aa6', 'resnet101_v2'),\n    ('f2695542de38cf7e71ed58f02893d82bb409415e', 'resnet152_v2'),\n    ('264ba4970a0cc87a4f15c96e25246a1307caf523', 'squeezenet1.0'),\n    ('33ba0f93753c83d86e1eb397f38a667eaf2e9376', 'squeezenet1.1'),\n    ('dd221b160977f36a53f464cb54648d227c707a05', 'vgg11'),\n    ('ee79a8098a91fbe05b7a973fed2017a6117723a8', 'vgg11_bn'),\n    ('6bc5de58a05a5e2e7f493e2d75a580d83efde38c', 'vgg13'),\n    ('7d97a06c3c7a1aecc88b6e7385c2b373a249e95e', 'vgg13_bn'),\n    ('e660d4569ccb679ec68f1fd3cce07a387252a90a', 'vgg16'),\n    ('7f01cf050d357127a73826045c245041b0df7363', 'vgg16_bn'),\n    ('ad2f660d101905472b83590b59708b71ea22b2e5', 'vgg19'),\n    ('f360b758e856f1074a85abd5fd873ed1d98297c3', 'vgg19_bn')]}\n\napache_repo_url = 'https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/'\n_url_format = '{repo_url}gluon/models/{file_name}.zip'\n\ndef short_hash(name):\n    if name not in _model_sha1:\n        raise ValueError('Pretrained model for {name} is not available.'.format(name=name))\n    return _model_sha1[name][:8]\n\ndef get_model_file(name, root=os.path.join(base.data_dir(), 'models')):\n    r\"\"\"Return location for the pretrained on local file system.\n\n    This function will download from online model zoo when model cannot be found or has mismatch.\n    The root directory will be created if it doesn't exist.\n\n    Parameters\n    ----------\n    name : str\n        Name of the model.\n    root : str, default $MXNET_HOME/models\n        Location for keeping the model parameters.\n\n    Returns\n    -------\n    file_path\n        Path to the requested pretrained model file.\n    \"\"\"\n    file_name = '{name}-{short_hash}'.format(name=name,\n                                             short_hash=short_hash(name))\n    root = os.path.expanduser(root)\n    file_path = os.path.join(root, file_name+'.params')\n    sha1_hash = _model_sha1[name]\n    if os.path.exists(file_path):\n        if check_sha1(file_path, sha1_hash):\n            return file_path\n        else:\n            logging.warning('Mismatch in the content of model file detected. Downloading again.')\n    else:\n        logging.info('Model file not found. Downloading to %s.', file_path)\n\n    os.makedirs(root, exist_ok=True)\n\n    repo_url = os.environ.get('MXNET_GLUON_REPO', apache_repo_url)\n    if repo_url[-1] != '/':\n        repo_url = repo_url + '/'\n\n    random_uuid = str(uuid.uuid4())\n    temp_zip_file_path = os.path.join(root, file_name+'.zip'+random_uuid)\n    download(_url_format.format(repo_url=repo_url, file_name=file_name),\n             path=temp_zip_file_path, overwrite=True)\n    with zipfile.ZipFile(temp_zip_file_path) as zf:\n        with TemporaryDirectory(dir=root) as temp_dir:\n            zf.extractall(temp_dir)\n            temp_file_path = os.path.join(temp_dir, file_name+'.params')\n            replace_file(temp_file_path, file_path)\n    os.remove(temp_zip_file_path)\n\n    if check_sha1(file_path, sha1_hash):\n        return file_path\n    else:\n        raise ValueError('Downloaded file has different hash. Please try again.')\n\ndef purge(root=os.path.join(base.data_dir(), 'models')):\n    r\"\"\"Purge all pretrained model files in local file store.\n\n    Parameters\n    ----------\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    root = os.path.expanduser(root)\n    files = os.listdir(root)\n    for f in files:\n        if f.endswith(\".params\"):\n            os.remove(os.path.join(root, f))\n"
  },
  {
    "path": "python/mxnet/gluon/model_zoo/vision/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import, arguments-differ\nr\"\"\"Module for pre-defined neural network models.\n\nThis module contains definitions for the following model architectures:\n-  `AlexNet`_\n-  `DenseNet`_\n-  `Inception V3`_\n-  `ResNet V1`_\n-  `ResNet V2`_\n-  `SqueezeNet`_\n-  `VGG`_\n-  `MobileNet`_\n-  `MobileNetV2`_\n\nYou can construct a model with random weights by calling its constructor:\n\n.. code::\n\n    from mxnet.gluon.model_zoo import vision\n    resnet18 = vision.resnet18_v1()\n    alexnet = vision.alexnet()\n    squeezenet = vision.squeezenet1_0()\n    densenet = vision.densenet_161()\n\nWe provide pre-trained models for all the listed models.\nThese models can constructed by passing ``pretrained=True``:\n\n.. code::\n\n    from mxnet.gluon.model_zoo import vision\n    resnet18 = vision.resnet18_v1(pretrained=True)\n    alexnet = vision.alexnet(pretrained=True)\n\nAll pre-trained models expect input images normalized in the same way,\ni.e. mini-batches of 3-channel RGB images of shape (N x 3 x H x W),\nwhere N is the batch size, and H and W are expected to be at least 224.\nThe images have to be loaded in to a range of [0, 1] and then normalized\nusing ``mean = [0.485, 0.456, 0.406]`` and ``std = [0.229, 0.224, 0.225]``.\nThe transformation should preferrably happen at preprocessing. You can use\n``mx.image.color_normalize`` for such transformation::\n\n    image = image/255\n    normalized = mx.image.color_normalize(image,\n                                          mean=mx.np.array([0.485, 0.456, 0.406]),\n                                          std=mx.np.array([0.229, 0.224, 0.225]))\n\n.. _AlexNet: https://arxiv.org/abs/1404.5997\n.. _DenseNet: https://arxiv.org/abs/1608.06993\n.. _Inception V3: http://arxiv.org/abs/1512.00567\n.. _ResNet V1: https://arxiv.org/abs/1512.03385\n.. _ResNet V2: https://arxiv.org/abs/1603.05027\n.. _SqueezeNet: https://arxiv.org/abs/1602.07360\n.. _VGG: https://arxiv.org/abs/1409.1556\n.. _MobileNet: https://arxiv.org/abs/1704.04861\n.. _MobileNetV2: https://arxiv.org/abs/1801.04381\n\"\"\"\n\nfrom .alexnet import *\n\nfrom .densenet import *\n\nfrom .inception import *\n\nfrom .resnet import *\n\nfrom .squeezenet import *\n\nfrom .vgg import *\n\nfrom .mobilenet import *\n\n\ndef get_model(name, **kwargs):\n    \"\"\"Returns a pre-defined model by name\n\n    Parameters\n    ----------\n    name : str\n        Name of the model.\n    pretrained : bool\n        Whether to load the pretrained weights for model.\n    classes : int\n        Number of classes for the output layer.\n    ctx : Context, default CPU\n        The context in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n\n    Returns\n    -------\n    gluon.HybridBlock\n        The model.\n    \"\"\"\n    models = {'resnet18_v1': resnet18_v1,\n              'resnet34_v1': resnet34_v1,\n              'resnet50_v1': resnet50_v1,\n              'resnet101_v1': resnet101_v1,\n              'resnet152_v1': resnet152_v1,\n              'resnet18_v2': resnet18_v2,\n              'resnet34_v2': resnet34_v2,\n              'resnet50_v2': resnet50_v2,\n              'resnet101_v2': resnet101_v2,\n              'resnet152_v2': resnet152_v2,\n              'vgg11': vgg11,\n              'vgg13': vgg13,\n              'vgg16': vgg16,\n              'vgg19': vgg19,\n              'vgg11_bn': vgg11_bn,\n              'vgg13_bn': vgg13_bn,\n              'vgg16_bn': vgg16_bn,\n              'vgg19_bn': vgg19_bn,\n              'alexnet': alexnet,\n              'densenet121': densenet121,\n              'densenet161': densenet161,\n              'densenet169': densenet169,\n              'densenet201': densenet201,\n              'squeezenet1.0': squeezenet1_0,\n              'squeezenet1.1': squeezenet1_1,\n              'inceptionv3': inception_v3,\n              'mobilenet1.0': mobilenet1_0,\n              'mobilenet0.75': mobilenet0_75,\n              'mobilenet0.5': mobilenet0_5,\n              'mobilenet0.25': mobilenet0_25,\n              'mobilenetv2_1.0': mobilenet_v2_1_0,\n              'mobilenetv2_0.75': mobilenet_v2_0_75,\n              'mobilenetv2_0.5': mobilenet_v2_0_5,\n              'mobilenetv2_0.25': mobilenet_v2_0_25\n             }\n    name = name.lower()\n    if name not in models:\n        raise ValueError(\n            \"Model {} is not supported. Available options are\\n\\t{}\".format(name, '\\n\\t'.join(sorted(models.keys()))))\n    return models[name](**kwargs)\n"
  },
  {
    "path": "python/mxnet/gluon/model_zoo/vision/alexnet.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable= arguments-differ\n\"\"\"Alexnet, implemented in Gluon.\"\"\"\n__all__ = ['AlexNet', 'alexnet']\n\nimport os\n\nfrom ....device import cpu\nfrom ...block import HybridBlock\nfrom ... import nn\nfrom .... import base\nfrom ....util import use_np, wrap_ctx_to_device_func\n\n# Net\n@use_np\nclass AlexNet(HybridBlock):\n    r\"\"\"AlexNet model from the `\"One weird trick...\" <https://arxiv.org/abs/1404.5997>`_ paper.\n\n    Parameters\n    ----------\n    classes : int, default 1000\n        Number of classes for the output layer.\n    \"\"\"\n    def __init__(self, classes=1000, **kwargs):\n        super(AlexNet, self).__init__(**kwargs)\n        self.features = nn.HybridSequential()\n        self.features.add(nn.Conv2D(64, kernel_size=11, strides=4,\n                                    padding=2, activation='relu'))\n        self.features.add(nn.MaxPool2D(pool_size=3, strides=2))\n        self.features.add(nn.Conv2D(192, kernel_size=5, padding=2,\n                                    activation='relu'))\n        self.features.add(nn.MaxPool2D(pool_size=3, strides=2))\n        self.features.add(nn.Conv2D(384, kernel_size=3, padding=1,\n                                    activation='relu'))\n        self.features.add(nn.Conv2D(256, kernel_size=3, padding=1,\n                                    activation='relu'))\n        self.features.add(nn.Conv2D(256, kernel_size=3, padding=1,\n                                    activation='relu'))\n        self.features.add(nn.MaxPool2D(pool_size=3, strides=2))\n        self.features.add(nn.Flatten())\n        self.features.add(nn.Dense(4096, activation='relu'))\n        self.features.add(nn.Dropout(0.5))\n        self.features.add(nn.Dense(4096, activation='relu'))\n        self.features.add(nn.Dropout(0.5))\n\n        self.output = nn.Dense(classes)\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.output(x)\n        return x\n\n# Constructor\n@wrap_ctx_to_device_func\ndef alexnet(pretrained=False, device=cpu(),\n            root=os.path.join(base.data_dir(), 'models'), **kwargs):\n    r\"\"\"AlexNet model from the `\"One weird trick...\" <https://arxiv.org/abs/1404.5997>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default $MXNET_HOME/models\n        Location for keeping the model parameters.\n    \"\"\"\n    net = AlexNet(**kwargs)\n    if pretrained:\n        from ..model_store import get_model_file\n        net.load_parameters(get_model_file('alexnet', root=root), device=device)\n    return net\n"
  },
  {
    "path": "python/mxnet/gluon/model_zoo/vision/densenet.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable= arguments-differ\n\"\"\"DenseNet, implemented in Gluon.\"\"\"\n__all__ = ['DenseNet', 'densenet121', 'densenet161', 'densenet169', 'densenet201']\n\nimport os\n\nfrom ....device import cpu\nfrom ...block import HybridBlock\nfrom ... import nn\nfrom .... import base\nfrom ....util import use_np, wrap_ctx_to_device_func\n\n# Helpers\ndef _make_dense_block(num_layers, bn_size, growth_rate, dropout):\n    out = nn.HybridSequential()\n    for _ in range(num_layers):\n        out.add(_make_dense_layer(growth_rate, bn_size, dropout))\n    return out\n\ndef _make_dense_layer(growth_rate, bn_size, dropout):\n    new_features = nn.HybridSequential()\n    new_features.add(nn.BatchNorm())\n    new_features.add(nn.Activation('relu'))\n    new_features.add(nn.Conv2D(bn_size * growth_rate, kernel_size=1, use_bias=False))\n    new_features.add(nn.BatchNorm())\n    new_features.add(nn.Activation('relu'))\n    new_features.add(nn.Conv2D(growth_rate, kernel_size=3, padding=1, use_bias=False))\n    if dropout:\n        new_features.add(nn.Dropout(dropout))\n\n    out = nn.HybridConcatenate(axis=1)\n    out.add(nn.Identity())\n    out.add(new_features)\n\n    return out\n\ndef _make_transition(num_output_features):\n    out = nn.HybridSequential()\n    out.add(nn.BatchNorm())\n    out.add(nn.Activation('relu'))\n    out.add(nn.Conv2D(num_output_features, kernel_size=1, use_bias=False))\n    out.add(nn.AvgPool2D(pool_size=2, strides=2))\n    return out\n\n# Net\n@use_np\nclass DenseNet(HybridBlock):\n    r\"\"\"Densenet-BC model from the\n    `\"Densely Connected Convolutional Networks\" <https://arxiv.org/pdf/1608.06993.pdf>`_ paper.\n\n    Parameters\n    ----------\n    num_init_features : int\n        Number of filters to learn in the first convolution layer.\n    growth_rate : int\n        Number of filters to add each layer (`k` in the paper).\n    block_config : list of int\n        List of integers for numbers of layers in each pooling block.\n    bn_size : int, default 4\n        Multiplicative factor for number of bottle neck layers.\n        (i.e. bn_size * k features in the bottleneck layer)\n    dropout : float, default 0\n        Rate of dropout after each dense layer.\n    classes : int, default 1000\n        Number of classification classes.\n    \"\"\"\n    def __init__(self, num_init_features, growth_rate, block_config,\n                 bn_size=4, dropout=0, classes=1000, **kwargs):\n\n        super(DenseNet, self).__init__(**kwargs)\n        self.features = nn.HybridSequential()\n        self.features.add(nn.Conv2D(num_init_features, kernel_size=7,\n                                    strides=2, padding=3, use_bias=False))\n        self.features.add(nn.BatchNorm())\n        self.features.add(nn.Activation('relu'))\n        self.features.add(nn.MaxPool2D(pool_size=3, strides=2, padding=1))\n        # Add dense blocks\n        num_features = num_init_features\n        for i, num_layers in enumerate(block_config):\n            self.features.add(_make_dense_block(num_layers, bn_size, growth_rate, dropout))\n            num_features = num_features + num_layers * growth_rate\n            if i != len(block_config) - 1:\n                self.features.add(_make_transition(num_features // 2))\n                num_features = num_features // 2\n        self.features.add(nn.BatchNorm())\n        self.features.add(nn.Activation('relu'))\n        self.features.add(nn.AvgPool2D(pool_size=7))\n        self.features.add(nn.Flatten())\n\n        self.output = nn.Dense(classes)\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.output(x)\n        return x\n\n\n# Specification\ndensenet_spec = {121: (64, 32, [6, 12, 24, 16]),\n                 161: (96, 48, [6, 12, 36, 24]),\n                 169: (64, 32, [6, 12, 32, 32]),\n                 201: (64, 32, [6, 12, 48, 32])}\n\n\n# Constructor\n@wrap_ctx_to_device_func\ndef get_densenet(num_layers, pretrained=False, device=cpu(),\n                 root=os.path.join(base.data_dir(), 'models'), **kwargs):\n    r\"\"\"Densenet-BC model from the\n    `\"Densely Connected Convolutional Networks\" <https://arxiv.org/pdf/1608.06993.pdf>`_ paper.\n\n    Parameters\n    ----------\n    num_layers : int\n        Number of layers for the variant of densenet. Options are 121, 161, 169, 201.\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default $MXNET_HOME/models\n        Location for keeping the model parameters.\n    \"\"\"\n    num_init_features, growth_rate, block_config = densenet_spec[num_layers]\n    net = DenseNet(num_init_features, growth_rate, block_config, **kwargs)\n    if pretrained:\n        from ..model_store import get_model_file\n        net.load_parameters(get_model_file(f'densenet{num_layers}', root=root), device=device)\n    return net\n\ndef densenet121(**kwargs):\n    r\"\"\"Densenet-BC 121-layer model from the\n    `\"Densely Connected Convolutional Networks\" <https://arxiv.org/pdf/1608.06993.pdf>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_densenet(121, **kwargs)\n\ndef densenet161(**kwargs):\n    r\"\"\"Densenet-BC 161-layer model from the\n    `\"Densely Connected Convolutional Networks\" <https://arxiv.org/pdf/1608.06993.pdf>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_densenet(161, **kwargs)\n\ndef densenet169(**kwargs):\n    r\"\"\"Densenet-BC 169-layer model from the\n    `\"Densely Connected Convolutional Networks\" <https://arxiv.org/pdf/1608.06993.pdf>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_densenet(169, **kwargs)\n\ndef densenet201(**kwargs):\n    r\"\"\"Densenet-BC 201-layer model from the\n    `\"Densely Connected Convolutional Networks\" <https://arxiv.org/pdf/1608.06993.pdf>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_densenet(201, **kwargs)\n"
  },
  {
    "path": "python/mxnet/gluon/model_zoo/vision/inception.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable= arguments-differ\n\"\"\"Inception, implemented in Gluon.\"\"\"\n__all__ = ['Inception3', 'inception_v3']\n\nimport os\n\nfrom ....device import cpu\nfrom ...block import HybridBlock\nfrom ... import nn\nfrom .... import base\nfrom ....util import use_np, wrap_ctx_to_device_func\n\n# Helpers\ndef _make_basic_conv(**kwargs):\n    out = nn.HybridSequential()\n    out.add(nn.Conv2D(use_bias=False, **kwargs))\n    out.add(nn.BatchNorm(epsilon=0.001))\n    out.add(nn.Activation('relu'))\n    return out\n\ndef _make_branch(use_pool, *conv_settings):\n    out = nn.HybridSequential()\n    if use_pool == 'avg':\n        out.add(nn.AvgPool2D(pool_size=3, strides=1, padding=1))\n    elif use_pool == 'max':\n        out.add(nn.MaxPool2D(pool_size=3, strides=2))\n    setting_names = ['channels', 'kernel_size', 'strides', 'padding']\n    for setting in conv_settings:\n        kwargs = {}\n        for i, value in enumerate(setting):\n            if value is not None:\n                kwargs[setting_names[i]] = value\n        out.add(_make_basic_conv(**kwargs))\n    return out\n\ndef _make_A(pool_features):\n    out = nn.HybridConcatenate(axis=1)\n    out.add(_make_branch(None,\n                         (64, 1, None, None)))\n    out.add(_make_branch(None,\n                         (48, 1, None, None),\n                         (64, 5, None, 2)))\n    out.add(_make_branch(None,\n                         (64, 1, None, None),\n                         (96, 3, None, 1),\n                         (96, 3, None, 1)))\n    out.add(_make_branch('avg',\n                         (pool_features, 1, None, None)))\n    return out\n\ndef _make_B():\n    out = nn.HybridConcatenate(axis=1)\n    out.add(_make_branch(None,\n                         (384, 3, 2, None)))\n    out.add(_make_branch(None,\n                         (64, 1, None, None),\n                         (96, 3, None, 1),\n                         (96, 3, 2, None)))\n    out.add(_make_branch('max'))\n    return out\n\ndef _make_C(channels_7x7):\n    out = nn.HybridConcatenate(axis=1)\n    out.add(_make_branch(None,\n                         (192, 1, None, None)))\n    out.add(_make_branch(None,\n                         (channels_7x7, 1, None, None),\n                         (channels_7x7, (1, 7), None, (0, 3)),\n                         (192, (7, 1), None, (3, 0))))\n    out.add(_make_branch(None,\n                         (channels_7x7, 1, None, None),\n                         (channels_7x7, (7, 1), None, (3, 0)),\n                         (channels_7x7, (1, 7), None, (0, 3)),\n                         (channels_7x7, (7, 1), None, (3, 0)),\n                         (192, (1, 7), None, (0, 3))))\n    out.add(_make_branch('avg',\n                         (192, 1, None, None)))\n    return out\n\ndef _make_D():\n    out = nn.HybridConcatenate(axis=1)\n    out.add(_make_branch(None,\n                         (192, 1, None, None),\n                         (320, 3, 2, None)))\n    out.add(_make_branch(None,\n                         (192, 1, None, None),\n                         (192, (1, 7), None, (0, 3)),\n                         (192, (7, 1), None, (3, 0)),\n                         (192, 3, 2, None)))\n    out.add(_make_branch('max'))\n    return out\n\ndef _make_E():\n    out = nn.HybridConcatenate(axis=1)\n    out.add(_make_branch(None,\n                         (320, 1, None, None)))\n\n    branch_3x3 = nn.HybridSequential()\n    out.add(branch_3x3)\n    branch_3x3.add(_make_branch(None,\n                                (384, 1, None, None)))\n    branch_3x3_split = nn.HybridConcatenate(axis=1)\n    branch_3x3_split.add(_make_branch(None,\n                                      (384, (1, 3), None, (0, 1))))\n    branch_3x3_split.add(_make_branch(None,\n                                      (384, (3, 1), None, (1, 0))))\n    branch_3x3.add(branch_3x3_split)\n\n    branch_3x3dbl = nn.HybridSequential()\n    out.add(branch_3x3dbl)\n    branch_3x3dbl.add(_make_branch(None,\n                                   (448, 1, None, None),\n                                   (384, 3, None, 1)))\n    branch_3x3dbl_split = nn.HybridConcatenate(axis=1)\n    branch_3x3dbl.add(branch_3x3dbl_split)\n    branch_3x3dbl_split.add(_make_branch(None,\n                                         (384, (1, 3), None, (0, 1))))\n    branch_3x3dbl_split.add(_make_branch(None,\n                                         (384, (3, 1), None, (1, 0))))\n\n    out.add(_make_branch('avg',\n                         (192, 1, None, None)))\n    return out\n\ndef make_aux(classes):\n    out = nn.HybridSequential()\n    out.add(nn.AvgPool2D(pool_size=5, strides=3))\n    out.add(_make_basic_conv(channels=128, kernel_size=1))\n    out.add(_make_basic_conv(channels=768, kernel_size=5))\n    out.add(nn.Flatten())\n    out.add(nn.Dense(classes))\n    return out\n\n# Net\n@use_np\nclass Inception3(HybridBlock):\n    r\"\"\"Inception v3 model from\n    `\"Rethinking the Inception Architecture for Computer Vision\"\n    <http://arxiv.org/abs/1512.00567>`_ paper.\n\n    Parameters\n    ----------\n    classes : int, default 1000\n        Number of classification classes.\n    \"\"\"\n    def __init__(self, classes=1000, **kwargs):\n        super(Inception3, self).__init__(**kwargs)\n        # self.use_aux_logits = use_aux_logits\n        self.features = nn.HybridSequential()\n        self.features.add(_make_basic_conv(channels=32, kernel_size=3, strides=2))\n        self.features.add(_make_basic_conv(channels=32, kernel_size=3))\n        self.features.add(_make_basic_conv(channels=64, kernel_size=3, padding=1))\n        self.features.add(nn.MaxPool2D(pool_size=3, strides=2))\n        self.features.add(_make_basic_conv(channels=80, kernel_size=1))\n        self.features.add(_make_basic_conv(channels=192, kernel_size=3))\n        self.features.add(nn.MaxPool2D(pool_size=3, strides=2))\n        self.features.add(_make_A(32))\n        self.features.add(_make_A(64))\n        self.features.add(_make_A(64))\n        self.features.add(_make_B())\n        self.features.add(_make_C(128))\n        self.features.add(_make_C(160))\n        self.features.add(_make_C(160))\n        self.features.add(_make_C(192))\n        self.features.add(_make_D())\n        self.features.add(_make_E())\n        self.features.add(_make_E())\n        self.features.add(nn.AvgPool2D(pool_size=8))\n        self.features.add(nn.Dropout(0.5))\n\n        self.output = nn.Dense(classes)\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.output(x)\n        return x\n\n# Constructor\n@wrap_ctx_to_device_func\ndef inception_v3(pretrained=False, device=cpu(),\n                 root=os.path.join(base.data_dir(), 'models'), **kwargs):\n    r\"\"\"Inception v3 model from\n    `\"Rethinking the Inception Architecture for Computer Vision\"\n    <http://arxiv.org/abs/1512.00567>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default $MXNET_HOME/models\n        Location for keeping the model parameters.\n    \"\"\"\n    net = Inception3(**kwargs)\n    if pretrained:\n        from ..model_store import get_model_file\n        net.load_parameters(get_model_file('inceptionv3', root=root), device=device)\n    return net\n"
  },
  {
    "path": "python/mxnet/gluon/model_zoo/vision/mobilenet.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable= arguments-differ\n\"\"\"MobileNet and MobileNetV2, implemented in Gluon.\"\"\"\n__all__ = ['MobileNet', 'MobileNetV2', 'mobilenet1_0', 'mobilenet_v2_1_0', 'mobilenet0_75',\n           'mobilenet_v2_0_75', 'mobilenet0_5', 'mobilenet_v2_0_5', 'mobilenet0_25',\n           'mobilenet_v2_0_25', 'get_mobilenet', 'get_mobilenet_v2']\n\n__modify__ = 'dwSun'\n__modified_date__ = '18/04/18'\n\nimport os\n\nfrom ... import nn\nfrom ....device import cpu\nfrom ...block import HybridBlock\nfrom .... import base, np\nfrom ....util import use_np, wrap_ctx_to_device_func\n\n\n# Helpers\n@use_np\nclass RELU6(nn.HybridBlock):\n    \"\"\"Relu6 used in MobileNetV2.\"\"\"\n\n    def __init__(self, **kwargs):\n        super(RELU6, self).__init__(**kwargs)\n\n    def forward(self, x):\n        return np.clip(x, 0, 6)\n\n\n# pylint: disable= too-many-arguments\ndef _add_conv(out, channels=1, kernel=1, stride=1, pad=0,\n              num_group=1, active=True, relu6=False):\n    out.add(nn.Conv2D(channels, kernel, stride, pad, groups=num_group, use_bias=False))\n    out.add(nn.BatchNorm(scale=True))\n    if active:\n        out.add(RELU6() if relu6 else nn.Activation('relu'))\n\n\ndef _add_conv_dw(out, dw_channels, channels, stride, relu6=False):\n    _add_conv(out, channels=dw_channels, kernel=3, stride=stride,\n              pad=1, num_group=dw_channels, relu6=relu6)\n    _add_conv(out, channels=channels, relu6=relu6)\n\n\n@use_np\nclass LinearBottleneck(nn.HybridBlock):\n    r\"\"\"LinearBottleneck used in MobileNetV2 model from the\n    `\"Inverted Residuals and Linear Bottlenecks:\n    Mobile Networks for Classification, Detection and Segmentation\"\n    <https://arxiv.org/abs/1801.04381>`_ paper.\n\n    Parameters\n    ----------\n    in_channels : int\n        Number of input channels.\n    channels : int\n        Number of output channels.\n    t : int\n        Layer expansion ratio.\n    stride : int\n        stride\n    \"\"\"\n\n    def __init__(self, in_channels, channels, t, stride, **kwargs):\n        super(LinearBottleneck, self).__init__(**kwargs)\n        self.use_shortcut = stride == 1 and in_channels == channels\n        self.out = nn.HybridSequential()\n\n        _add_conv(self.out, in_channels * t, relu6=True)\n        _add_conv(self.out, in_channels * t, kernel=3, stride=stride,\n                  pad=1, num_group=in_channels * t, relu6=True)\n        _add_conv(self.out, channels, active=False, relu6=True)\n\n    def forward(self, x):\n        out = self.out(x)\n        if self.use_shortcut:\n            out = np.add(out, x)\n        return out\n\n\n# Net\n@use_np\nclass MobileNet(HybridBlock):\n    r\"\"\"MobileNet model from the\n    `\"MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications\"\n    <https://arxiv.org/abs/1704.04861>`_ paper.\n\n    Parameters\n    ----------\n    multiplier : float, default 1.0\n        The width multiplier for controling the model size. Only multipliers that are no\n        less than 0.25 are supported. The actual number of channels is equal to the original\n        channel size multiplied by this multiplier.\n    classes : int, default 1000\n        Number of classes for the output layer.\n    \"\"\"\n\n    def __init__(self, multiplier=1.0, classes=1000, **kwargs):\n        super(MobileNet, self).__init__(**kwargs)\n        self.features = nn.HybridSequential()\n        _add_conv(self.features, channels=int(32 * multiplier), kernel=3, pad=1, stride=2)\n        dw_channels = [int(x * multiplier) for x in [32, 64] + [128] * 2\n                       + [256] * 2 + [512] * 6 + [1024]]\n        channels = [int(x * multiplier) for x in [64] + [128] * 2 + [256] * 2\n                    + [512] * 6 + [1024] * 2]\n        strides = [1, 2] * 3 + [1] * 5 + [2, 1]\n        for dwc, c, s in zip(dw_channels, channels, strides):\n            _add_conv_dw(self.features, dw_channels=dwc, channels=c, stride=s)\n        self.features.add(nn.GlobalAvgPool2D())\n        self.features.add(nn.Flatten())\n\n        self.output = nn.Dense(classes)\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.output(x)\n        return x\n\n\n@use_np\nclass MobileNetV2(nn.HybridBlock):\n    r\"\"\"MobileNetV2 model from the\n    `\"Inverted Residuals and Linear Bottlenecks:\n    Mobile Networks for Classification, Detection and Segmentation\"\n    <https://arxiv.org/abs/1801.04381>`_ paper.\n\n    Parameters\n    ----------\n    multiplier : float, default 1.0\n        The width multiplier for controling the model size. The actual number of channels\n        is equal to the original channel size multiplied by this multiplier.\n    classes : int, default 1000\n        Number of classes for the output layer.\n    \"\"\"\n\n    def __init__(self, multiplier=1.0, classes=1000, **kwargs):\n        super(MobileNetV2, self).__init__(**kwargs)\n        self.features = nn.HybridSequential()\n        _add_conv(self.features, int(32 * multiplier), kernel=3,\n                  stride=2, pad=1, relu6=True)\n\n        in_channels_group = [int(x * multiplier) for x in [32] + [16] + [24] * 2\n                             + [32] * 3 + [64] * 4 + [96] * 3 + [160] * 3]\n        channels_group = [int(x * multiplier) for x in [16] + [24] * 2 + [32] * 3\n                          + [64] * 4 + [96] * 3 + [160] * 3 + [320]]\n        ts = [1] + [6] * 16\n        strides = [1, 2] * 2 + [1, 1, 2] + [1] * 6 + [2] + [1] * 3\n\n        for in_c, c, t, s in zip(in_channels_group, channels_group, ts, strides):\n            self.features.add(LinearBottleneck(in_channels=in_c, channels=c,\n                                               t=t, stride=s))\n\n        last_channels = int(1280 * multiplier) if multiplier > 1.0 else 1280\n        _add_conv(self.features, last_channels, relu6=True)\n\n        self.features.add(nn.GlobalAvgPool2D())\n\n        self.output = nn.HybridSequential()\n        self.output.add(\n            nn.Conv2D(classes, 1, use_bias=False),\n            nn.Flatten()\n        )\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.output(x)\n        return x\n\n\n# Constructor\n@wrap_ctx_to_device_func\ndef get_mobilenet(multiplier, pretrained=False, device=cpu(),\n                  root=os.path.join(base.data_dir(), 'models'), **kwargs):\n    r\"\"\"MobileNet model from the\n    `\"MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications\"\n    <https://arxiv.org/abs/1704.04861>`_ paper.\n\n    Parameters\n    ----------\n    multiplier : float\n        The width multiplier for controling the model size. Only multipliers that are no\n        less than 0.25 are supported. The actual number of channels is equal to the original\n        channel size multiplied by this multiplier.\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default $MXNET_HOME/models\n        Location for keeping the model parameters.\n    \"\"\"\n    net = MobileNet(multiplier, **kwargs)\n\n    if pretrained:\n        from ..model_store import get_model_file\n        version_suffix = '{0:.2f}'.format(multiplier)\n        if version_suffix in ('1.00', '0.50'):\n            version_suffix = version_suffix[:-1]\n        net.load_parameters(\n            get_model_file(f'mobilenet{version_suffix}', root=root), device=device)\n    return net\n\n\n@wrap_ctx_to_device_func\ndef get_mobilenet_v2(multiplier, pretrained=False, device=cpu(),\n                     root=os.path.join(base.data_dir(), 'models'), **kwargs):\n    r\"\"\"MobileNetV2 model from the\n    `\"Inverted Residuals and Linear Bottlenecks:\n    Mobile Networks for Classification, Detection and Segmentation\"\n    <https://arxiv.org/abs/1801.04381>`_ paper.\n\n    Parameters\n    ----------\n    multiplier : float\n        The width multiplier for controling the model size. Only multipliers that are no\n        less than 0.25 are supported. The actual number of channels is equal to the original\n        channel size multiplied by this multiplier.\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default $MXNET_HOME/models\n        Location for keeping the model parameters.\n    \"\"\"\n    net = MobileNetV2(multiplier, **kwargs)\n\n    if pretrained:\n        from ..model_store import get_model_file\n        version_suffix = '{0:.2f}'.format(multiplier)\n        if version_suffix in ('1.00', '0.50'):\n            version_suffix = version_suffix[:-1]\n        net.load_parameters(\n            get_model_file(f'mobilenetv2_{version_suffix}', root=root), device=device)\n    return net\n\n\n@wrap_ctx_to_device_func\ndef mobilenet1_0(**kwargs):\n    r\"\"\"MobileNet model from the\n    `\"MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications\"\n    <https://arxiv.org/abs/1704.04861>`_ paper, with width multiplier 1.0.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    \"\"\"\n    return get_mobilenet(1.0, **kwargs)\n\n\n@wrap_ctx_to_device_func\ndef mobilenet_v2_1_0(**kwargs):\n    r\"\"\"MobileNetV2 model from the\n    `\"Inverted Residuals and Linear Bottlenecks:\n    Mobile Networks for Classification, Detection and Segmentation\"\n    <https://arxiv.org/abs/1801.04381>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    \"\"\"\n    return get_mobilenet_v2(1.0, **kwargs)\n\n\n@wrap_ctx_to_device_func\ndef mobilenet0_75(**kwargs):\n    r\"\"\"MobileNet model from the\n    `\"MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications\"\n    <https://arxiv.org/abs/1704.04861>`_ paper, with width multiplier 0.75.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    \"\"\"\n    return get_mobilenet(0.75, **kwargs)\n\n\n@wrap_ctx_to_device_func\ndef mobilenet_v2_0_75(**kwargs):\n    r\"\"\"MobileNetV2 model from the\n    `\"Inverted Residuals and Linear Bottlenecks:\n    Mobile Networks for Classification, Detection and Segmentation\"\n    <https://arxiv.org/abs/1801.04381>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    \"\"\"\n    return get_mobilenet_v2(0.75, **kwargs)\n\n\n@wrap_ctx_to_device_func\ndef mobilenet0_5(**kwargs):\n    r\"\"\"MobileNet model from the\n    `\"MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications\"\n    <https://arxiv.org/abs/1704.04861>`_ paper, with width multiplier 0.5.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    \"\"\"\n    return get_mobilenet(0.5, **kwargs)\n\n\n@wrap_ctx_to_device_func\ndef mobilenet_v2_0_5(**kwargs):\n    r\"\"\"MobileNetV2 model from the\n    `\"Inverted Residuals and Linear Bottlenecks:\n    Mobile Networks for Classification, Detection and Segmentation\"\n    <https://arxiv.org/abs/1801.04381>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    \"\"\"\n    return get_mobilenet_v2(0.5, **kwargs)\n\n\n@wrap_ctx_to_device_func\ndef mobilenet0_25(**kwargs):\n    r\"\"\"MobileNet model from the\n    `\"MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications\"\n    <https://arxiv.org/abs/1704.04861>`_ paper, with width multiplier 0.25.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    \"\"\"\n    return get_mobilenet(0.25, **kwargs)\n\n\n@wrap_ctx_to_device_func\ndef mobilenet_v2_0_25(**kwargs):\n    r\"\"\"MobileNetV2 model from the\n    `\"Inverted Residuals and Linear Bottlenecks:\n    Mobile Networks for Classification, Detection and Segmentation\"\n    <https://arxiv.org/abs/1801.04381>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    \"\"\"\n    return get_mobilenet_v2(0.25, **kwargs)\n"
  },
  {
    "path": "python/mxnet/gluon/model_zoo/vision/resnet.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable= arguments-differ\n\"\"\"ResNets, implemented in Gluon.\"\"\"\n\n__all__ = ['ResNetV1', 'ResNetV2',\n           'BasicBlockV1', 'BasicBlockV2',\n           'BottleneckV1', 'BottleneckV2',\n           'resnet18_v1', 'resnet34_v1', 'resnet50_v1', 'resnet101_v1', 'resnet152_v1',\n           'resnet18_v2', 'resnet34_v2', 'resnet50_v2', 'resnet101_v2', 'resnet152_v2',\n           'get_resnet']\n\nimport os\n\nfrom ....device import cpu\nfrom ...block import HybridBlock\nfrom ... import nn\nfrom .... import base\nfrom .... util import use_np, wrap_ctx_to_device_func\nfrom .... import npx\n\n# Helpers\ndef _conv3x3(channels, stride, in_channels):\n    return nn.Conv2D(channels, kernel_size=3, strides=stride, padding=1,\n                     use_bias=False, in_channels=in_channels)\n\n\n# Blocks\n@use_np\nclass BasicBlockV1(HybridBlock):\n    r\"\"\"BasicBlock V1 from `\"Deep Residual Learning for Image Recognition\"\n    <http://arxiv.org/abs/1512.03385>`_ paper.\n    This is used for ResNet V1 for 18, 34 layers.\n\n    Parameters\n    ----------\n    channels : int\n        Number of output channels.\n    stride : int\n        Stride size.\n    downsample : bool, default False\n        Whether to downsample the input.\n    in_channels : int, default 0\n        Number of input channels. Default is 0, to infer from the graph.\n    \"\"\"\n    def __init__(self, channels, stride, downsample=False, in_channels=0, **kwargs):\n        super(BasicBlockV1, self).__init__(**kwargs)\n        self.body = nn.HybridSequential()\n        self.body.add(_conv3x3(channels, stride, in_channels))\n        self.body.add(nn.BatchNorm())\n        self.body.add(nn.Activation('relu'))\n        self.body.add(_conv3x3(channels, 1, channels))\n        self.body.add(nn.BatchNorm())\n        if downsample:\n            self.downsample = nn.HybridSequential()\n            self.downsample.add(nn.Conv2D(channels, kernel_size=1, strides=stride,\n                                          use_bias=False, in_channels=in_channels))\n            self.downsample.add(nn.BatchNorm())\n        else:\n            self.downsample = None\n\n    def forward(self, x):\n        residual = x\n\n        x = self.body(x)\n\n        if self.downsample:\n            residual = self.downsample(residual)\n\n        x = npx.activation(residual+x, act_type='relu')\n\n        return x\n\n\n@use_np\nclass BottleneckV1(HybridBlock):\n    r\"\"\"Bottleneck V1 from `\"Deep Residual Learning for Image Recognition\"\n    <http://arxiv.org/abs/1512.03385>`_ paper.\n    This is used for ResNet V1 for 50, 101, 152 layers.\n\n    Parameters\n    ----------\n    channels : int\n        Number of output channels.\n    stride : int\n        Stride size.\n    downsample : bool, default False\n        Whether to downsample the input.\n    in_channels : int, default 0\n        Number of input channels. Default is 0, to infer from the graph.\n    \"\"\"\n    def __init__(self, channels, stride, downsample=False, in_channels=0, **kwargs):\n        super(BottleneckV1, self).__init__(**kwargs)\n        self.body = nn.HybridSequential()\n        self.body.add(nn.Conv2D(channels//4, kernel_size=1, strides=stride))\n        self.body.add(nn.BatchNorm())\n        self.body.add(nn.Activation('relu'))\n        self.body.add(_conv3x3(channels//4, 1, channels//4))\n        self.body.add(nn.BatchNorm())\n        self.body.add(nn.Activation('relu'))\n        self.body.add(nn.Conv2D(channels, kernel_size=1, strides=1))\n        self.body.add(nn.BatchNorm())\n        if downsample:\n            self.downsample = nn.HybridSequential()\n            self.downsample.add(nn.Conv2D(channels, kernel_size=1, strides=stride,\n                                          use_bias=False, in_channels=in_channels))\n            self.downsample.add(nn.BatchNorm())\n        else:\n            self.downsample = None\n\n    def forward(self, x):\n        residual = x\n\n        x = self.body(x)\n\n        if self.downsample:\n            residual = self.downsample(residual)\n\n        x = npx.activation(x + residual, act_type='relu')\n        return x\n\n\n@use_np\nclass BasicBlockV2(HybridBlock):\n    r\"\"\"BasicBlock V2 from\n    `\"Identity Mappings in Deep Residual Networks\"\n    <https://arxiv.org/abs/1603.05027>`_ paper.\n    This is used for ResNet V2 for 18, 34 layers.\n\n    Parameters\n    ----------\n    channels : int\n        Number of output channels.\n    stride : int\n        Stride size.\n    downsample : bool, default False\n        Whether to downsample the input.\n    in_channels : int, default 0\n        Number of input channels. Default is 0, to infer from the graph.\n    \"\"\"\n    def __init__(self, channels, stride, downsample=False, in_channels=0, **kwargs):\n        super(BasicBlockV2, self).__init__(**kwargs)\n        self.bn1 = nn.BatchNorm()\n        self.conv1 = _conv3x3(channels, stride, in_channels)\n        self.bn2 = nn.BatchNorm()\n        self.conv2 = _conv3x3(channels, 1, channels)\n        if downsample:\n            self.downsample = nn.Conv2D(channels, 1, stride, use_bias=False,\n                                        in_channels=in_channels)\n        else:\n            self.downsample = None\n\n    def forward(self, x):\n        residual = x\n        x = self.bn1(x)\n        x = npx.activation(x, act_type='relu')\n        if self.downsample:\n            residual = self.downsample(x)\n        x = self.conv1(x)\n\n        x = self.bn2(x)\n        x = npx.activation(x, act_type='relu')\n        x = self.conv2(x)\n\n        return x + residual\n\n\n@use_np\nclass BottleneckV2(HybridBlock):\n    r\"\"\"Bottleneck V2 from\n    `\"Identity Mappings in Deep Residual Networks\"\n    <https://arxiv.org/abs/1603.05027>`_ paper.\n    This is used for ResNet V2 for 50, 101, 152 layers.\n\n    Parameters\n    ----------\n    channels : int\n        Number of output channels.\n    stride : int\n        Stride size.\n    downsample : bool, default False\n        Whether to downsample the input.\n    in_channels : int, default 0\n        Number of input channels. Default is 0, to infer from the graph.\n    \"\"\"\n    def __init__(self, channels, stride, downsample=False, in_channels=0, **kwargs):\n        super(BottleneckV2, self).__init__(**kwargs)\n        self.bn1 = nn.BatchNorm()\n        self.conv1 = nn.Conv2D(channels//4, kernel_size=1, strides=1, use_bias=False)\n        self.bn2 = nn.BatchNorm()\n        self.conv2 = _conv3x3(channels//4, stride, channels//4)\n        self.bn3 = nn.BatchNorm()\n        self.conv3 = nn.Conv2D(channels, kernel_size=1, strides=1, use_bias=False)\n        if downsample:\n            self.downsample = nn.Conv2D(channels, 1, stride, use_bias=False,\n                                        in_channels=in_channels)\n        else:\n            self.downsample = None\n\n    def forward(self, x):\n        residual = x\n        x = self.bn1(x)\n        x = npx.activation(x, act_type='relu')\n        if self.downsample:\n            residual = self.downsample(x)\n        x = self.conv1(x)\n\n        x = self.bn2(x)\n        x = npx.activation(x, act_type='relu')\n        x = self.conv2(x)\n\n        x = self.bn3(x)\n        x = npx.activation(x, act_type='relu')\n        x = self.conv3(x)\n\n        return x + residual\n\n\n# Nets\n@use_np\nclass ResNetV1(HybridBlock):\n    r\"\"\"ResNet V1 model from\n    `\"Deep Residual Learning for Image Recognition\"\n    <http://arxiv.org/abs/1512.03385>`_ paper.\n\n    Parameters\n    ----------\n    block : gluon.HybridBlock\n        Class for the residual block. Options are BasicBlockV1, BottleneckV1.\n    layers : list of int\n        Numbers of layers in each block\n    channels : list of int\n        Numbers of channels in each block. Length should be one larger than layers list.\n    classes : int, default 1000\n        Number of classification classes.\n    thumbnail : bool, default False\n        Enable thumbnail.\n    \"\"\"\n    def __init__(self, block, layers, channels, classes=1000, thumbnail=False, **kwargs):\n        super(ResNetV1, self).__init__(**kwargs)\n        assert len(layers) == len(channels) - 1\n        self.features = nn.HybridSequential()\n        if thumbnail:\n            self.features.add(_conv3x3(channels[0], 1, 0))\n        else:\n            self.features.add(nn.Conv2D(channels[0], 7, 2, 3, use_bias=False))\n            self.features.add(nn.BatchNorm())\n            self.features.add(nn.Activation('relu'))\n            self.features.add(nn.MaxPool2D(3, 2, 1))\n\n        for i, num_layer in enumerate(layers):\n            stride = 1 if i == 0 else 2\n            self.features.add(self._make_layer(block, num_layer, channels[i+1],\n                                               stride, in_channels=channels[i]))\n        self.features.add(nn.GlobalAvgPool2D())\n\n        self.output = nn.Dense(classes, in_units=channels[-1])\n\n    def _make_layer(self, block, layers, channels, stride, in_channels=0):\n        layer = nn.HybridSequential()\n        layer.add(block(channels, stride, channels != in_channels, in_channels=in_channels))\n        for _ in range(layers-1):\n            layer.add(block(channels, 1, False, in_channels=channels))\n        return layer\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.output(x)\n\n        return x\n\n\n@use_np\nclass ResNetV2(HybridBlock):\n    r\"\"\"ResNet V2 model from\n    `\"Identity Mappings in Deep Residual Networks\"\n    <https://arxiv.org/abs/1603.05027>`_ paper.\n\n    Parameters\n    ----------\n    block : gluon.HybridBlock\n        Class for the residual block. Options are BasicBlockV1, BottleneckV1.\n    layers : list of int\n        Numbers of layers in each block\n    channels : list of int\n        Numbers of channels in each block. Length should be one larger than layers list.\n    classes : int, default 1000\n        Number of classification classes.\n    thumbnail : bool, default False\n        Enable thumbnail.\n    \"\"\"\n    def __init__(self, block, layers, channels, classes=1000, thumbnail=False, **kwargs):\n        super(ResNetV2, self).__init__(**kwargs)\n        assert len(layers) == len(channels) - 1\n        self.features = nn.HybridSequential()\n        self.features.add(nn.BatchNorm(scale=False, center=False))\n        if thumbnail:\n            self.features.add(_conv3x3(channels[0], 1, 0))\n        else:\n            self.features.add(nn.Conv2D(channels[0], 7, 2, 3, use_bias=False))\n            self.features.add(nn.BatchNorm())\n            self.features.add(nn.Activation('relu'))\n            self.features.add(nn.MaxPool2D(3, 2, 1))\n\n        in_channels = channels[0]\n        for i, num_layer in enumerate(layers):\n            stride = 1 if i == 0 else 2\n            self.features.add(self._make_layer(block, num_layer, channels[i+1],\n                                               stride, in_channels=in_channels))\n            in_channels = channels[i+1]\n        self.features.add(nn.BatchNorm())\n        self.features.add(nn.Activation('relu'))\n        self.features.add(nn.GlobalAvgPool2D())\n        self.features.add(nn.Flatten())\n\n        self.output = nn.Dense(classes, in_units=in_channels)\n\n    def _make_layer(self, block, layers, channels, stride, in_channels=0):\n        layer = nn.HybridSequential()\n        layer.add(block(channels, stride, channels != in_channels, in_channels=in_channels))\n        for _ in range(layers-1):\n            layer.add(block(channels, 1, False, in_channels=channels))\n        return layer\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.output(x)\n        return x\n\n\n# Specification\nresnet_spec = {18: ('basic_block', [2, 2, 2, 2], [64, 64, 128, 256, 512]),\n               34: ('basic_block', [3, 4, 6, 3], [64, 64, 128, 256, 512]),\n               50: ('bottle_neck', [3, 4, 6, 3], [64, 256, 512, 1024, 2048]),\n               101: ('bottle_neck', [3, 4, 23, 3], [64, 256, 512, 1024, 2048]),\n               152: ('bottle_neck', [3, 8, 36, 3], [64, 256, 512, 1024, 2048])}\n\nresnet_net_versions = [ResNetV1, ResNetV2]\nresnet_block_versions = [{'basic_block': BasicBlockV1, 'bottle_neck': BottleneckV1},\n                         {'basic_block': BasicBlockV2, 'bottle_neck': BottleneckV2}]\n\n\n# Constructor\n@wrap_ctx_to_device_func\ndef get_resnet(version, num_layers, pretrained=False, device=cpu(),\n               root=os.path.join(base.data_dir(), 'models'), **kwargs):\n    r\"\"\"ResNet V1 model from `\"Deep Residual Learning for Image Recognition\"\n    <http://arxiv.org/abs/1512.03385>`_ paper.\n    ResNet V2 model from `\"Identity Mappings in Deep Residual Networks\"\n    <https://arxiv.org/abs/1603.05027>`_ paper.\n\n    Parameters\n    ----------\n    version : int\n        Version of ResNet. Options are 1, 2.\n    num_layers : int\n        Numbers of layers. Options are 18, 34, 50, 101, 152.\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default $MXNET_HOME/models\n        Location for keeping the model parameters.\n    \"\"\"\n    assert num_layers in resnet_spec, \\\n        f\"Invalid number of layers: {num_layers}. Options are {str(resnet_spec.keys())}\"\n    block_type, layers, channels = resnet_spec[num_layers]\n    assert version >= 1 and version <= 2, \\\n        f\"Invalid resnet version: {version}. Options are 1 and 2.\"\n    resnet_class = resnet_net_versions[version-1]\n    block_class = resnet_block_versions[version-1][block_type]\n    net = resnet_class(block_class, layers, channels, **kwargs)\n    if pretrained:\n        from ..model_store import get_model_file\n        net.load_parameters(get_model_file(f'resnet{num_layers}_v{version}',\n                                           root=root), device=device)\n    return net\n\n@wrap_ctx_to_device_func\ndef resnet18_v1(**kwargs):\n    r\"\"\"ResNet-18 V1 model from `\"Deep Residual Learning for Image Recognition\"\n    <http://arxiv.org/abs/1512.03385>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_resnet(1, 18, **kwargs)\n\n@wrap_ctx_to_device_func\ndef resnet34_v1(**kwargs):\n    r\"\"\"ResNet-34 V1 model from `\"Deep Residual Learning for Image Recognition\"\n    <http://arxiv.org/abs/1512.03385>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_resnet(1, 34, **kwargs)\n\n@wrap_ctx_to_device_func\ndef resnet50_v1(**kwargs):\n    r\"\"\"ResNet-50 V1 model from `\"Deep Residual Learning for Image Recognition\"\n    <http://arxiv.org/abs/1512.03385>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_resnet(1, 50, **kwargs)\n\n@wrap_ctx_to_device_func\ndef resnet101_v1(**kwargs):\n    r\"\"\"ResNet-101 V1 model from `\"Deep Residual Learning for Image Recognition\"\n    <http://arxiv.org/abs/1512.03385>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_resnet(1, 101, **kwargs)\n\n@wrap_ctx_to_device_func\ndef resnet152_v1(**kwargs):\n    r\"\"\"ResNet-152 V1 model from `\"Deep Residual Learning for Image Recognition\"\n    <http://arxiv.org/abs/1512.03385>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_resnet(1, 152, **kwargs)\n\n@wrap_ctx_to_device_func\ndef resnet18_v2(**kwargs):\n    r\"\"\"ResNet-18 V2 model from `\"Identity Mappings in Deep Residual Networks\"\n    <https://arxiv.org/abs/1603.05027>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_resnet(2, 18, **kwargs)\n\n@wrap_ctx_to_device_func\ndef resnet34_v2(**kwargs):\n    r\"\"\"ResNet-34 V2 model from `\"Identity Mappings in Deep Residual Networks\"\n    <https://arxiv.org/abs/1603.05027>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_resnet(2, 34, **kwargs)\n\n@wrap_ctx_to_device_func\ndef resnet50_v2(**kwargs):\n    r\"\"\"ResNet-50 V2 model from `\"Identity Mappings in Deep Residual Networks\"\n    <https://arxiv.org/abs/1603.05027>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_resnet(2, 50, **kwargs)\n\n@wrap_ctx_to_device_func\ndef resnet101_v2(**kwargs):\n    r\"\"\"ResNet-101 V2 model from `\"Identity Mappings in Deep Residual Networks\"\n    <https://arxiv.org/abs/1603.05027>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_resnet(2, 101, **kwargs)\n\n@wrap_ctx_to_device_func\ndef resnet152_v2(**kwargs):\n    r\"\"\"ResNet-152 V2 model from `\"Identity Mappings in Deep Residual Networks\"\n    <https://arxiv.org/abs/1603.05027>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_resnet(2, 152, **kwargs)\n"
  },
  {
    "path": "python/mxnet/gluon/model_zoo/vision/squeezenet.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable= arguments-differ\n\"\"\"SqueezeNet, implemented in Gluon.\"\"\"\n__all__ = ['SqueezeNet', 'squeezenet1_0', 'squeezenet1_1']\n\nimport os\n\nfrom ....device import cpu\nfrom ...block import HybridBlock\nfrom ... import nn\nfrom .... import base\nfrom ....util import use_np, wrap_ctx_to_device_func\n\n# Helpers\ndef _make_fire(squeeze_channels, expand1x1_channels, expand3x3_channels):\n    out = nn.HybridSequential()\n    out.add(_make_fire_conv(squeeze_channels, 1))\n\n    paths = nn.HybridConcatenate(axis=1)\n    paths.add(_make_fire_conv(expand1x1_channels, 1))\n    paths.add(_make_fire_conv(expand3x3_channels, 3, 1))\n    out.add(paths)\n\n    return out\n\ndef _make_fire_conv(channels, kernel_size, padding=0):\n    out = nn.HybridSequential()\n    out.add(nn.Conv2D(channels, kernel_size, padding=padding))\n    out.add(nn.Activation('relu'))\n    return out\n\n# Net\n@use_np\nclass SqueezeNet(HybridBlock):\n    r\"\"\"SqueezeNet model from the `\"SqueezeNet: AlexNet-level accuracy with 50x fewer parameters\n    and <0.5MB model size\" <https://arxiv.org/abs/1602.07360>`_ paper.\n    SqueezeNet 1.1 model from the `official SqueezeNet repo\n    <https://github.com/DeepScale/SqueezeNet/tree/master/SqueezeNet_v1.1>`_.\n    SqueezeNet 1.1 has 2.4x less computation and slightly fewer parameters\n    than SqueezeNet 1.0, without sacrificing accuracy.\n\n    Parameters\n    ----------\n    version : str\n        Version of squeezenet. Options are '1.0', '1.1'.\n    classes : int, default 1000\n        Number of classification classes.\n    \"\"\"\n    def __init__(self, version, classes=1000, **kwargs):\n        super(SqueezeNet, self).__init__(**kwargs)\n        assert version in ['1.0', '1.1'], (\"Unsupported SqueezeNet version {version}:\"\n                                           \"1.0 or 1.1 expected\".format(version=version))\n        self.features = nn.HybridSequential()\n        if version == '1.0':\n            self.features.add(nn.Conv2D(96, kernel_size=7, strides=2))\n            self.features.add(nn.Activation('relu'))\n            self.features.add(nn.MaxPool2D(pool_size=3, strides=2, ceil_mode=True))\n            self.features.add(_make_fire(16, 64, 64))\n            self.features.add(_make_fire(16, 64, 64))\n            self.features.add(_make_fire(32, 128, 128))\n            self.features.add(nn.MaxPool2D(pool_size=3, strides=2, ceil_mode=True))\n            self.features.add(_make_fire(32, 128, 128))\n            self.features.add(_make_fire(48, 192, 192))\n            self.features.add(_make_fire(48, 192, 192))\n            self.features.add(_make_fire(64, 256, 256))\n            self.features.add(nn.MaxPool2D(pool_size=3, strides=2, ceil_mode=True))\n            self.features.add(_make_fire(64, 256, 256))\n        else:\n            self.features.add(nn.Conv2D(64, kernel_size=3, strides=2))\n            self.features.add(nn.Activation('relu'))\n            self.features.add(nn.MaxPool2D(pool_size=3, strides=2, ceil_mode=True))\n            self.features.add(_make_fire(16, 64, 64))\n            self.features.add(_make_fire(16, 64, 64))\n            self.features.add(nn.MaxPool2D(pool_size=3, strides=2, ceil_mode=True))\n            self.features.add(_make_fire(32, 128, 128))\n            self.features.add(_make_fire(32, 128, 128))\n            self.features.add(nn.MaxPool2D(pool_size=3, strides=2, ceil_mode=True))\n            self.features.add(_make_fire(48, 192, 192))\n            self.features.add(_make_fire(48, 192, 192))\n            self.features.add(_make_fire(64, 256, 256))\n            self.features.add(_make_fire(64, 256, 256))\n        self.features.add(nn.Dropout(0.5))\n\n        self.output = nn.HybridSequential()\n        self.output.add(nn.Conv2D(classes, kernel_size=1))\n        self.output.add(nn.Activation('relu'))\n        self.output.add(nn.AvgPool2D(13))\n        self.output.add(nn.Flatten())\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.output(x)\n        return x\n\n# Constructor\n@wrap_ctx_to_device_func\ndef get_squeezenet(version, pretrained=False, device=cpu(),\n                   root=os.path.join(base.data_dir(), 'models'), **kwargs):\n    r\"\"\"SqueezeNet model from the `\"SqueezeNet: AlexNet-level accuracy with 50x fewer parameters\n    and <0.5MB model size\" <https://arxiv.org/abs/1602.07360>`_ paper.\n    SqueezeNet 1.1 model from the `official SqueezeNet repo\n    <https://github.com/DeepScale/SqueezeNet/tree/master/SqueezeNet_v1.1>`_.\n    SqueezeNet 1.1 has 2.4x less computation and slightly fewer parameters\n    than SqueezeNet 1.0, without sacrificing accuracy.\n\n    Parameters\n    ----------\n    version : str\n        Version of squeezenet. Options are '1.0', '1.1'.\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default $MXNET_HOME/models\n        Location for keeping the model parameters.\n    \"\"\"\n    net = SqueezeNet(version, **kwargs)\n    if pretrained:\n        from ..model_store import get_model_file\n        net.load_parameters(get_model_file(f'squeezenet{version}', root=root), device=device)\n    return net\n\n@wrap_ctx_to_device_func\ndef squeezenet1_0(**kwargs):\n    r\"\"\"SqueezeNet 1.0 model from the `\"SqueezeNet: AlexNet-level accuracy with 50x fewer parameters\n    and <0.5MB model size\" <https://arxiv.org/abs/1602.07360>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_squeezenet('1.0', **kwargs)\n\n@wrap_ctx_to_device_func\ndef squeezenet1_1(**kwargs):\n    r\"\"\"SqueezeNet 1.1 model from the `official SqueezeNet repo\n    <https://github.com/DeepScale/SqueezeNet/tree/master/SqueezeNet_v1.1>`_.\n    SqueezeNet 1.1 has 2.4x less computation and slightly fewer parameters\n    than SqueezeNet 1.0, without sacrificing accuracy.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_squeezenet('1.1', **kwargs)\n"
  },
  {
    "path": "python/mxnet/gluon/model_zoo/vision/vgg.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable= arguments-differ\n\"\"\"VGG, implemented in Gluon.\"\"\"\n__all__ = ['VGG',\n           'vgg11', 'vgg13', 'vgg16', 'vgg19',\n           'vgg11_bn', 'vgg13_bn', 'vgg16_bn', 'vgg19_bn',\n           'get_vgg']\n\nimport os\n\nfrom ....device import cpu\nfrom ....initializer import Xavier\nfrom ...block import HybridBlock\nfrom ... import nn\nfrom .... import base\nfrom ....util import use_np, wrap_ctx_to_device_func\n\n\n@use_np\nclass VGG(HybridBlock):\n    r\"\"\"VGG model from the `\"Very Deep Convolutional Networks for Large-Scale Image Recognition\"\n    <https://arxiv.org/abs/1409.1556>`_ paper.\n\n    Parameters\n    ----------\n    layers : list of int\n        Numbers of layers in each feature block.\n    filters : list of int\n        Numbers of filters in each feature block. List length should match the layers.\n    classes : int, default 1000\n        Number of classification classes.\n    batch_norm : bool, default False\n        Use batch normalization.\n    \"\"\"\n    def __init__(self, layers, filters, classes=1000, batch_norm=False, **kwargs):\n        super(VGG, self).__init__(**kwargs)\n        assert len(layers) == len(filters)\n        self.features = self._make_features(layers, filters, batch_norm)\n        self.features.add(nn.Dense(4096, activation='relu',\n                                   weight_initializer='normal',\n                                   bias_initializer='zeros'))\n        self.features.add(nn.Dropout(rate=0.5))\n        self.features.add(nn.Dense(4096, activation='relu',\n                                   weight_initializer='normal',\n                                   bias_initializer='zeros'))\n        self.features.add(nn.Dropout(rate=0.5))\n        self.output = nn.Dense(classes,\n                               weight_initializer='normal',\n                               bias_initializer='zeros')\n\n    def _make_features(self, layers, filters, batch_norm):\n        featurizer = nn.HybridSequential()\n        for i, num in enumerate(layers):\n            for _ in range(num):\n                featurizer.add(nn.Conv2D(filters[i], kernel_size=3, padding=1,\n                                         weight_initializer=Xavier(rnd_type='gaussian',\n                                                                   factor_type='out',\n                                                                   magnitude=2),\n                                         bias_initializer='zeros'))\n                if batch_norm:\n                    featurizer.add(nn.BatchNorm())\n                featurizer.add(nn.Activation('relu'))\n            featurizer.add(nn.MaxPool2D(strides=2))\n        return featurizer\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.output(x)\n        return x\n\n\n# Specification\nvgg_spec = {11: ([1, 1, 2, 2, 2], [64, 128, 256, 512, 512]),\n            13: ([2, 2, 2, 2, 2], [64, 128, 256, 512, 512]),\n            16: ([2, 2, 3, 3, 3], [64, 128, 256, 512, 512]),\n            19: ([2, 2, 4, 4, 4], [64, 128, 256, 512, 512])}\n\n\n# Constructors\n@wrap_ctx_to_device_func\ndef get_vgg(num_layers, pretrained=False, device=cpu(),\n            root=os.path.join(base.data_dir(), 'models'), **kwargs):\n    r\"\"\"VGG model from the `\"Very Deep Convolutional Networks for Large-Scale Image Recognition\"\n    <https://arxiv.org/abs/1409.1556>`_ paper.\n\n    Parameters\n    ----------\n    num_layers : int\n        Number of layers for the variant of densenet. Options are 11, 13, 16, 19.\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default $MXNET_HOME/models\n        Location for keeping the model parameters.\n    \"\"\"\n    layers, filters = vgg_spec[num_layers]\n    net = VGG(layers, filters, **kwargs)\n    if pretrained:\n        from ..model_store import get_model_file\n        batch_norm_suffix = '_bn' if kwargs.get('batch_norm') else ''\n        net.load_parameters(get_model_file(f'vgg{num_layers}{batch_norm_suffix}',\n                                           root=root), device=device)\n    return net\n\n@wrap_ctx_to_device_func\ndef vgg11(**kwargs):\n    r\"\"\"VGG-11 model from the `\"Very Deep Convolutional Networks for Large-Scale Image Recognition\"\n    <https://arxiv.org/abs/1409.1556>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_vgg(11, **kwargs)\n\n@wrap_ctx_to_device_func\ndef vgg13(**kwargs):\n    r\"\"\"VGG-13 model from the `\"Very Deep Convolutional Networks for Large-Scale Image Recognition\"\n    <https://arxiv.org/abs/1409.1556>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_vgg(13, **kwargs)\n\n@wrap_ctx_to_device_func\ndef vgg16(**kwargs):\n    r\"\"\"VGG-16 model from the `\"Very Deep Convolutional Networks for Large-Scale Image Recognition\"\n    <https://arxiv.org/abs/1409.1556>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_vgg(16, **kwargs)\n\n@wrap_ctx_to_device_func\ndef vgg19(**kwargs):\n    r\"\"\"VGG-19 model from the `\"Very Deep Convolutional Networks for Large-Scale Image Recognition\"\n    <https://arxiv.org/abs/1409.1556>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    return get_vgg(19, **kwargs)\n\n@wrap_ctx_to_device_func\ndef vgg11_bn(**kwargs):\n    r\"\"\"VGG-11 model with batch normalization from the\n    `\"Very Deep Convolutional Networks for Large-Scale Image Recognition\"\n    <https://arxiv.org/abs/1409.1556>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    kwargs['batch_norm'] = True\n    return get_vgg(11, **kwargs)\n\n@wrap_ctx_to_device_func\ndef vgg13_bn(**kwargs):\n    r\"\"\"VGG-13 model with batch normalization from the\n    `\"Very Deep Convolutional Networks for Large-Scale Image Recognition\"\n    <https://arxiv.org/abs/1409.1556>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    kwargs['batch_norm'] = True\n    return get_vgg(13, **kwargs)\n\n@wrap_ctx_to_device_func\ndef vgg16_bn(**kwargs):\n    r\"\"\"VGG-16 model with batch normalization from the\n    `\"Very Deep Convolutional Networks for Large-Scale Image Recognition\"\n    <https://arxiv.org/abs/1409.1556>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    kwargs['batch_norm'] = True\n    return get_vgg(16, **kwargs)\n\n@wrap_ctx_to_device_func\ndef vgg19_bn(**kwargs):\n    r\"\"\"VGG-19 model with batch normalization from the\n    `\"Very Deep Convolutional Networks for Large-Scale Image Recognition\"\n    <https://arxiv.org/abs/1409.1556>`_ paper.\n\n    Parameters\n    ----------\n    pretrained : bool, default False\n        Whether to load the pretrained weights for model.\n    device : Device, default CPU\n        The device in which to load the pretrained weights.\n    root : str, default '$MXNET_HOME/models'\n        Location for keeping the model parameters.\n    \"\"\"\n    kwargs['batch_norm'] = True\n    return get_vgg(19, **kwargs)\n"
  },
  {
    "path": "python/mxnet/gluon/nn/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Neural network layers.\"\"\"\n\nfrom ..block import *\n\nfrom .basic_layers import *\n\nfrom .conv_layers import *\n\nfrom .activations import *\n"
  },
  {
    "path": "python/mxnet/gluon/nn/activations.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable= arguments-differ\n\"\"\"Basic neural network layers.\"\"\"\n__all__ = ['Activation', 'LeakyReLU', 'PReLU', 'ELU', 'SELU', 'Swish', 'GELU', 'SiLU']\n\nfrom ... import initializer, npx\nfrom ..block import HybridBlock\nfrom ..parameter import Parameter\nfrom ...util import use_np\n\n\n@use_np\nclass Activation(HybridBlock):\n    r\"\"\"Applies an activation function to input.\n\n    Parameters\n    ----------\n    activation : str\n        Name of activation function to use.\n        See :func:`~mxnet.ndarray.Activation` for available choices.\n\n\n    Inputs:\n        - **data**: input tensor with arbitrary shape.\n\n    Outputs:\n        - **out**: output tensor with the same shape as `data`.\n    \"\"\"\n    def __init__(self, activation, **kwargs):\n        self._act_type = activation\n        super(Activation, self).__init__(**kwargs)\n\n    def _alias(self):\n        return self._act_type\n\n    def forward(self, x):\n        return npx.activation(x, act_type=self._act_type, name='fwd')\n\n    def __repr__(self):\n        s = '{name}({_act_type})'\n        return s.format(name=self.__class__.__name__,\n                        **self.__dict__)\n\n\n@use_np\nclass LeakyReLU(HybridBlock):\n    r\"\"\"Leaky version of a Rectified Linear Unit.\n\n    It allows a small gradient when the unit is not active\n\n    .. math::\n\n        f\\left(x\\right) = \\left\\{\n            \\begin{array}{lr}\n               \\alpha x & : x \\lt 0 \\\\\n                      x & : x \\geq 0 \\\\\n            \\end{array}\n        \\right.\\\\\n\n    Parameters\n    ----------\n    alpha : float\n        slope coefficient for the negative half axis. Must be >= 0.\n\n\n    Inputs:\n        - **data**: input tensor with arbitrary shape.\n\n    Outputs:\n        - **out**: output tensor with the same shape as `data`.\n    \"\"\"\n    def __init__(self, alpha, **kwargs):\n        assert alpha >= 0, \"Slope coefficient for LeakyReLU must be no less than 0.\"\n        super(LeakyReLU, self).__init__(**kwargs)\n        self._alpha = alpha\n\n    def forward(self, x):\n        return npx.leaky_relu(x, act_type='leaky', slope=self._alpha, name='fwd')\n\n    def __repr__(self):\n        s = '{name}({alpha})'\n        return s.format(name=self.__class__.__name__,\n                        alpha=self._alpha)\n\n\n@use_np\nclass PReLU(HybridBlock):\n    r\"\"\"Parametric leaky version of a Rectified Linear Unit.\n    <https://arxiv.org/abs/1502.01852>`_ paper.\n\n    It learns a gradient when the unit is not active\n\n    .. math::\n\n        f\\left(x\\right) = \\left\\{\n            \\begin{array}{lr}\n               \\alpha x & : x \\lt 0 \\\\\n                      x & : x \\geq 0 \\\\\n            \\end{array}\n        \\right.\\\\\n\n    where alpha is a learned parameter.\n\n    Parameters\n    ----------\n    alpha_initializer : Initializer\n        Initializer for the `embeddings` matrix.\n    in_channels : int, default 1\n        Number of channels (alpha parameters) to learn. Can either be 1\n        or `n` where `n` is the size of the second dimension of the input\n        tensor.\n\n    Inputs:\n        - **data**: input tensor with arbitrary shape.\n\n    Outputs:\n        - **out**: output tensor with the same shape as `data`.\n    \"\"\"\n    def __init__(self, alpha_initializer=initializer.Constant(0.25),\n                 in_channels=1, **kwargs):\n        super(PReLU, self).__init__(**kwargs)\n        self.alpha = Parameter('alpha', shape=(in_channels,), init=alpha_initializer)\n\n    def forward(self, x):\n        device = x.device\n        return npx.leaky_relu(x, gamma=self.alpha.data(device), act_type='prelu', name='fwd')\n\n\n@use_np\nclass ELU(HybridBlock):\n    r\"\"\"\n    Exponential Linear Unit (ELU)\n        \"Fast and Accurate Deep Network Learning by Exponential Linear Units\", Clevert et al, 2016\n        https://arxiv.org/abs/1511.07289\n        Published as a conference paper at ICLR 2016\n\n    Parameters\n    ----------\n    alpha : float\n        The alpha parameter as described by Clevert et al, 2016\n\n\n    Inputs:\n        - **data**: input tensor with arbitrary shape.\n\n    Outputs:\n        - **out**: output tensor with the same shape as `data`.\n    \"\"\"\n\n    def __init__(self, alpha=1.0, **kwargs):\n        super(ELU, self).__init__(**kwargs)\n        self._alpha = alpha\n\n    def forward(self, x):\n        return npx.leaky_relu(x, act_type='elu', slope=self._alpha)\n\n\n@use_np\nclass SELU(HybridBlock):\n    r\"\"\"\n    Scaled Exponential Linear Unit (SELU)\n        \"Self-Normalizing Neural Networks\", Klambauer et al, 2017\n        https://arxiv.org/abs/1706.02515\n\n\n    Inputs:\n        - **data**: input tensor with arbitrary shape.\n\n    Outputs:\n        - **out**: output tensor with the same shape as `data`.\n    \"\"\"\n    def __init__(self, **kwargs):\n        super(SELU, self).__init__(**kwargs)\n\n    def forward(self, x):\n        return npx.leaky_relu(x, act_type='selu', name='fwd')\n\n\n@use_np\nclass GELU(HybridBlock):\n    r\"\"\"\n    Gaussian Exponential Linear Unit (GELU)\n        \"Gaussian Error Linear Units (GELUs)\", Hendrycks et al, 2016\n        https://arxiv.org/abs/1606.08415\n\n    Parameters\n    ----------\n    approximation : string\n        Which approximation of GELU calculation to use (erf or tanh).\n\n    Inputs:\n        - **data**: input tensor with arbitrary shape.\n\n    Outputs:\n        - **out**: output tensor with the same shape as `data`.\n    \"\"\"\n    def __init__(self, approximation='erf', **kwargs):\n        if approximation not in ['erf', 'tanh']:\n            raise ValueError(\"Unsupported approximation! Supported values are 'erf' and 'tanh', \"\n                             \"but got '{}'\".format(approximation))\n        self._act_algorithm = 'gelu_' + approximation\n        super(GELU, self).__init__(**kwargs)\n\n    def forward(self, x):\n        return npx.leaky_relu(x, act_type=self._act_algorithm, name='fwd')\n\n\n@use_np\nclass Swish(HybridBlock):\n    r\"\"\"\n    Swish Activation function (SiLU with a hyperparameter)\n        https://arxiv.org/pdf/1710.05941.pdf\n\n    Parameters\n    ----------\n    beta : float\n        swish(x) = x * sigmoid(beta*x)\n\n\n    Inputs:\n        - **data**: input tensor with arbitrary shape.\n\n    Outputs:\n        - **out**: output tensor with the same shape as `data`.\n    \"\"\"\n\n    def __init__(self, beta=1.0, **kwargs):\n        super(Swish, self).__init__(**kwargs)\n        self._beta = beta\n\n    def forward(self, x):\n        return x * npx.sigmoid(self._beta * x)\n\n\n@use_np\nclass SiLU(HybridBlock):\n    r\"\"\"\n    Sigmoid Linear Units\n        Originally proposed \"Gaussian Error Linear Units (GELUs)\", Hendrycks et al, 2016\n        https://arxiv.org/abs/1606.08415\n\n    Parameters\n    ----------\n    beta : float\n        silu(x) = x * sigmoid(x)\n\n\n    Inputs:\n        - **data**: input tensor with arbitrary shape.\n\n    Outputs:\n        - **out**: output tensor with the same shape as `data`.\n    \"\"\"\n\n    def __init__(self, **kwargs):\n        super(SiLU, self).__init__(**kwargs)\n\n    def forward(self, x):\n        return x * npx.sigmoid(x)\n"
  },
  {
    "path": "python/mxnet/gluon/nn/basic_layers.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable= arguments-differ\n\"\"\"Basic neural network layers.\"\"\"\n__all__ = ['Sequential', 'HybridSequential', 'Dense', 'Dropout', 'Embedding',\n           'BatchNorm', 'SyncBatchNorm', 'InstanceNorm', 'LayerNorm', 'GroupNorm',\n           'Flatten', 'Lambda', 'HybridLambda', 'Concatenate', 'HybridConcatenate', 'Identity']\nimport warnings\nimport uuid\nimport numpy as _np\n\nfrom .activations import Activation\nfrom ..block import Block, HybridBlock\nfrom ..utils import _indent\nfrom ... import np, npx, device as _device\nfrom ...util import use_np\nfrom ..parameter import Parameter\nfrom ...ndarray import get_dtype_name\n\nclass Sequential(Block):\n    \"\"\"Stacks Blocks sequentially.\n\n    Example::\n\n        net = nn.Sequential()\n        net.add(nn.Dense(10, activation='relu'))\n        net.add(nn.Dense(20))\n    \"\"\"\n    def __init__(self):\n        super(Sequential, self).__init__()\n        self._layers = []\n\n    def add(self, *blocks):\n        \"\"\"Adds block on top of the stack.\"\"\"\n        for block in blocks:\n            self._layers.append(block)\n            self.register_child(block)\n\n    def forward(self, x, *args):\n        for block in self._children.values():\n            x = block()(x, *args)\n            args = []\n            if isinstance(x, (tuple, list)):\n                args = x[1:]\n                x = x[0]\n        if args:\n            x = tuple([x] + list(args))\n        return x\n\n    def __repr__(self):\n        s = '{name}(\\n{modstr}\\n)'\n        modstr = '\\n'.join(['  ({key}): {block}'.format(key=key,\n                                                        block=_indent(block().__repr__(), 2))\n                            for key, block in self._children.items()])\n        return s.format(name=self.__class__.__name__, modstr=modstr)\n\n    def __getitem__(self, key):\n        layers = list(self._children.values())[key]\n        if isinstance(layers, list):\n            net = type(self)()\n            net.add(*(l() for l in layers))\n            return net\n        else:\n            return layers()\n\n    def __len__(self):\n        return len(self._children)\n\n    def hybridize(self, active=True, **kwargs):\n        \"\"\"Activates or deactivates `HybridBlock` s recursively. Has no effect on\n        non-hybrid children.\n\n        Parameters\n        ----------\n        active : bool, default True\n            Whether to turn hybrid on or off.\n        **kwargs : string\n            Additional flags for hybridized operator.\n        \"\"\"\n        if self._children and all(isinstance(c(), HybridBlock) for c in self._children.values()):\n            warnings.warn(\n                f\"All children of this Sequential layer '{repr(self)}'\\n are HybridBlocks. Consider \"\n                \"using HybridSequential for the best performance.\", stacklevel=2)\n        super(Sequential, self).hybridize(active, **kwargs)\n\n\n@use_np\nclass HybridSequential(HybridBlock):\n    \"\"\"Stacks HybridBlocks sequentially.\n\n    Example::\n\n        net = nn.HybridSequential()\n        net.add(nn.Dense(10, activation='relu'))\n        net.add(nn.Dense(20))\n        net.hybridize()\n    \"\"\"\n    def __init__(self):\n        super().__init__()\n        self._layers = []\n\n    def add(self, *blocks):\n        \"\"\"Adds block on top of the stack.\"\"\"\n        for block in blocks:\n            self._layers.append(block)\n            self.register_child(block)\n\n    def forward(self, x, *args):\n        for block in self._children.values():\n            x = block()(x, *args)\n            args = []\n            if isinstance(x, (tuple, list)):\n                args = x[1:]\n                x = x[0]\n        if args:\n            x = tuple([x] + list(args))\n        return x\n\n    def __repr__(self):\n        s = '{name}(\\n{modstr}\\n)'\n        modstr = '\\n'.join(['  ({key}): {block}'.format(key=key,\n                                                        block=_indent(block().__repr__(), 2))\n                            for key, block in self._children.items()])\n        return s.format(name=self.__class__.__name__, modstr=modstr)\n\n    def __getitem__(self, key):\n        layers = list(self._children.values())[key]\n        if isinstance(layers, list):\n            net = type(self)()\n            net.add(*(l() for l in layers))\n            return net\n        else:\n            return layers()\n\n    def __len__(self):\n        return len(self._children)\n\n\n@use_np\nclass Dense(HybridBlock):\n    r\"\"\"Just your regular densely-connected NN layer.\n\n    `Dense` implements the operation:\n    `output = activation(dot(input, weight.T) + bias)`\n    where `activation` is the element-wise activation function\n    passed as the `activation` argument, `weight` is a weights matrix\n    created by the layer, and `bias` is a bias vector created by the layer\n    (only applicable if `use_bias` is `True`).\n\n    Parameters\n    ----------\n    units : int\n        Dimensionality of the output space.\n    activation : str\n        Activation function to use. See help on `Activation` layer.\n        If you don't specify anything, no activation is applied\n        (ie. \"linear\" activation: `a(x) = x`).\n    use_bias : bool, default True\n        Whether the layer uses a bias vector.\n    flatten: bool, default True\n        Whether the input tensor should be flattened.\n        If true, all but the first axis of input data are collapsed together.\n        If false, all but the last axis of input data are kept the same, and the transformation\n        applies on the last axis.\n    dtype : str or np.dtype, default 'float32'\n        Data type of output embeddings.\n    weight_initializer : str or `Initializer`\n        Initializer for the `kernel` weights matrix.\n    bias_initializer: str or `Initializer`\n        Initializer for the bias vector.\n    in_units : int, optional\n        Size of the input data. If not specified, initialization will be\n        deferred to the first time `forward` is called and `in_units`\n        will be inferred from the shape of input data.\n\n\n    Inputs:\n        - **data**: if `flatten` is True, `data` should be a tensor with shape\n          `(batch_size, x1, x2, ..., xn)`, where x1 * x2 * ... * xn is equal to\n          `in_units`. If `flatten` is False, `data` should have shape\n          `(x1, x2, ..., xn, in_units)`.\n\n    Outputs:\n        - **out**: if `flatten` is True, `out` will be a tensor with shape\n          `(batch_size, units)`. If `flatten` is False, `out` will have shape\n          `(x1, x2, ..., xn, units)`.\n    \"\"\"\n    def __init__(self, units, activation=None, use_bias=True, flatten=True,\n                 dtype='float32', weight_initializer=None, bias_initializer='zeros',\n                 in_units=0, **kwargs):\n        super(Dense, self).__init__(**kwargs)\n        self._flatten = flatten\n        self._units = units\n        self._in_units = in_units\n        self.weight = Parameter('weight', shape=(units, in_units),\n                                init=weight_initializer, dtype=dtype,\n                                allow_deferred_init=True)\n        if use_bias:\n            self.bias = Parameter('bias', shape=(units,),\n                                  init=bias_initializer, dtype=dtype,\n                                  allow_deferred_init=True)\n        else:\n            self.bias = None\n        if activation is not None:\n            self.act = Activation(activation)\n        else:\n            self.act = None\n\n    def forward(self, x):\n        device = x.device\n        act = npx.fully_connected(x, self.weight.data(device),\n                                  self.bias.data(device) if self.bias is not None else None,\n                                  no_bias=self.bias is None,\n                                  num_hidden=self._units, flatten=self._flatten, name='fwd')\n        if self.act is not None:\n            act = self.act(act)\n        return act\n\n    def infer_shape(self, x, *args):\n        if self._flatten:\n            num_input = 1\n            for i in range(1, x.ndim):\n                num_input *= x.shape[i]\n            self.weight.shape = (self.weight.shape[0], num_input)\n        else:\n            self.weight.shape = (self.weight.shape[0], x.shape[x.ndim - 1])\n\n    def __repr__(self):\n        s = '{name}({layout}, {act})'\n        shape = self.weight.shape\n        return s.format(name=self.__class__.__name__,\n                        act=self.act if self.act else 'linear',\n                        layout='{0} -> {1}'.format(shape[1] if shape[1] else None, shape[0]))\n\n\n@use_np\nclass Dropout(HybridBlock):\n    \"\"\"Applies Dropout to the input.\n\n    Dropout consists in randomly setting a fraction `rate` of input units\n    to 0 at each update during training time, which helps prevent overfitting.\n\n    Parameters\n    ----------\n    rate : float\n        Fraction of the input units to drop. Must be a number between 0 and 1.\n    axes : tuple of int, default ()\n        The axes on which dropout mask is shared. If empty, regular dropout is applied.\n\n\n    Inputs:\n        - **data**: input tensor with arbitrary shape.\n\n    Outputs:\n        - **out**: output tensor with the same shape as `data`.\n\n    References\n    ----------\n        `Dropout: A Simple Way to Prevent Neural Networks from Overfitting\n        <http://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf>`_\n    \"\"\"\n    def __init__(self, rate, axes=(), **kwargs):\n        super(Dropout, self).__init__(**kwargs)\n        self._rate = rate\n        self._axes = axes\n\n    def forward(self, x):\n        if self._rate > 0:\n            return npx.dropout(x, p=self._rate, axes=self._axes, name='fwd', cudnn_off=False)\n        else:\n            return np.copy(x)\n\n    def __repr__(self):\n        s = '{name}(p = {_rate}, axes={_axes})'\n        return s.format(name=self.__class__.__name__,\n                        **self.__dict__)\n\n\n@use_np\nclass _BatchNorm(HybridBlock):\n    \"\"\"Abstract BatchNorm layer (private, used as implementation base).\n    Batch normalization layer (Ioffe and Szegedy, 2014).\n    Normalizes the input at each batch, i.e. applies a transformation\n    that maintains the mean activation close to 0 and the activation\n    standard deviation close to 1.\n\n    Parameters\n    ----------\n    axis : int, default 1\n        The axis that should be normalized. This is typically the channels\n        (C) axis. For instance, after a `Conv2D` layer with `layout='NCHW'`,\n        set `axis=1` in `BatchNorm`. If `layout='NHWC'`, then set `axis=3`.\n    momentum: float, default 0.9\n        Momentum for the moving average.\n    epsilon: float, default 1e-5\n        Small float added to variance to avoid dividing by zero.\n    center: bool, default True\n        If True, add offset of `beta` to normalized tensor.\n        If False, `beta` is ignored.\n    scale: bool, default True\n        If True, multiply by `gamma`. If False, `gamma` is not used.\n        When the next layer is linear (also e.g. `nn.relu`),\n        this can be disabled since the scaling\n        will be done by the next layer.\n    use_global_stats: bool, default False\n        If True, use global moving statistics instead of local batch-norm. This will force\n        change batch-norm into a scale shift operator.\n        If False, use local batch-norm.\n    beta_initializer: str or `Initializer`, default 'zeros'\n        Initializer for the beta weight.\n    gamma_initializer: str or `Initializer`, default 'ones'\n        Initializer for the gamma weight.\n    running_mean_initializer: str or `Initializer`, default 'zeros'\n        Initializer for the running mean.\n    running_variance_initializer: str or `Initializer`, default 'ones'\n        Initializer for the running variance.\n    in_channels : int, default 0\n        Number of channels (feature maps) in input data. If not specified,\n        initialization will be deferred to the first time `forward` is called\n        and `in_channels` will be inferred from the shape of input data.\n\n\n    Inputs:\n        - **data**: input tensor with arbitrary shape.\n\n    Outputs:\n        - **out**: output tensor with the same shape as `data`.\n    \"\"\"\n    def __init__(self, axis=1, momentum=0.9, epsilon=1e-5, center=True, scale=True,\n                 use_global_stats=False,\n                 beta_initializer='zeros', gamma_initializer='ones',\n                 running_mean_initializer='zeros', running_variance_initializer='ones',\n                 in_channels=0, **kwargs):\n        super(_BatchNorm, self).__init__(**kwargs)\n        self._kwargs = {'axis': axis, 'eps': epsilon, 'momentum': momentum,\n                        'fix_gamma': not scale, 'use_global_stats': use_global_stats}\n        self._axis = axis\n        if in_channels != 0:\n            self.in_channels = in_channels\n\n        self.gamma = Parameter('gamma', grad_req='write' if scale else 'null',\n                               shape=(in_channels,), init=gamma_initializer,\n                               allow_deferred_init=True,\n                               differentiable=scale)\n        self.beta = Parameter('beta', grad_req='write' if center else 'null',\n                              shape=(in_channels,), init=beta_initializer,\n                              allow_deferred_init=True,\n                              differentiable=center)\n        self.running_mean = Parameter('running_mean', grad_req='null',\n                                      shape=(in_channels,),\n                                      init=running_mean_initializer,\n                                      allow_deferred_init=True,\n                                      differentiable=False)\n        self.running_var = Parameter('running_var', grad_req='null',\n                                     shape=(in_channels,),\n                                     init=running_variance_initializer,\n                                     allow_deferred_init=True,\n                                     differentiable=False)\n\n    def cast(self, dtype):\n        if get_dtype_name(dtype) == 'float16':\n            dtype = 'float32'\n        super(_BatchNorm, self).cast(dtype)\n\n    def forward(self, x):\n        device = x.device\n        return npx.batch_norm(x, self.gamma.data(device), self.beta.data(device),\n                                  self.running_mean.data(device),\n                                  self.running_var.data(device),\n                                  name='fwd', **self._kwargs)\n\n    def infer_shape(self, x, *args):\n        channel_axis = self._axis if self._axis >= 0 else self._axis + x.ndim\n        channel_count = x.shape[channel_axis]\n        self.gamma.shape = (channel_count,)\n        self.beta.shape = (channel_count,)\n        self.running_mean.shape = (channel_count,)\n        self.running_var.shape = (channel_count,)\n\n    def __repr__(self):\n        s = '{name}({content}'\n        in_channels = self.gamma.shape[0]\n        s += ', in_channels={0}'.format(in_channels if in_channels else None)\n        s += ')'\n        return s.format(name=self.__class__.__name__,\n                        content=', '.join(['='.join([k, v.__repr__()])\n                                           for k, v in self._kwargs.items()]))\n\nclass BatchNorm(_BatchNorm):\n    \"\"\"Batch normalization layer (Ioffe and Szegedy, 2014).\n    Normalizes the input at each batch, i.e. applies a transformation\n    that maintains the mean activation close to 0 and the activation\n    standard deviation close to 1.\n\n    Parameters\n    ----------\n    axis : int, default 1\n        The axis that should be normalized. This is typically the channels\n        (C) axis. For instance, after a `Conv2D` layer with `layout='NCHW'`,\n        set `axis=1` in `BatchNorm`. If `layout='NHWC'`, then set `axis=3`.\n    momentum: float, default 0.9\n        Momentum for the moving average.\n    epsilon: float, default 1e-5\n        Small float added to variance to avoid dividing by zero.\n    center: bool, default True\n        If True, add offset of `beta` to normalized tensor.\n        If False, `beta` is ignored.\n    scale: bool, default True\n        If True, multiply by `gamma`. If False, `gamma` is not used.\n        When the next layer is linear (also e.g. `nn.relu`),\n        this can be disabled since the scaling\n        will be done by the next layer.\n    use_global_stats: bool, default False\n        If True, use global moving statistics instead of local batch-norm. This will force\n        change batch-norm into a scale shift operator.\n        If False, use local batch-norm.\n    beta_initializer: str or `Initializer`, default 'zeros'\n        Initializer for the beta weight.\n    gamma_initializer: str or `Initializer`, default 'ones'\n        Initializer for the gamma weight.\n    running_mean_initializer: str or `Initializer`, default 'zeros'\n        Initializer for the running mean.\n    running_variance_initializer: str or `Initializer`, default 'ones'\n        Initializer for the running variance.\n    in_channels : int, default 0\n        Number of channels (feature maps) in input data. If not specified,\n        initialization will be deferred to the first time `forward` is called\n        and `in_channels` will be inferred from the shape of input data.\n\n\n    Inputs:\n        - **data**: input tensor with arbitrary shape.\n\n    Outputs:\n        - **out**: output tensor with the same shape as `data`.\n    \"\"\"\n    def __init__(self, axis=1, momentum=0.9, epsilon=1e-5, center=True, scale=True,\n                 use_global_stats=False,\n                 beta_initializer='zeros', gamma_initializer='ones',\n                 running_mean_initializer='zeros', running_variance_initializer='ones',\n                 in_channels=0, **kwargs):\n        super(BatchNorm, self).__init__(\n            axis=axis, momentum=momentum, epsilon=epsilon, center=center,\n            scale=scale,\n            use_global_stats=use_global_stats,\n            beta_initializer=beta_initializer,\n            gamma_initializer=gamma_initializer,\n            running_mean_initializer=running_mean_initializer,\n            running_variance_initializer=running_variance_initializer,\n            in_channels=in_channels, **kwargs)\n\n\n@use_np\nclass Embedding(HybridBlock):\n    r\"\"\"Turns non-negative integers (indexes/tokens) into dense vectors\n    of fixed size. eg. [4, 20] -> [[0.25, 0.1], [0.6, -0.2]]\n\n    .. note::\n        if `sparse_grad` is set to True, the gradient w.r.t weight will be\n        sparse. Only a subset of optimizers support sparse gradients, including SGD,\n        AdaGrad and Adam. By default lazy updates is turned on, which may perform\n        differently from standard updates. For more details, please check the\n        Optimization API at:\n        https://mxnet.apache.org/versions/master/api/python/docs/api/optimizer/index.html\n\n    Parameters\n    ----------\n    input_dim : int\n        Size of the vocabulary, i.e. maximum integer index + 1.\n    output_dim : int\n        Dimension of the dense embedding.\n    dtype : str or np.dtype, default 'float32'\n        Data type of output embeddings.\n    weight_initializer : Initializer\n        Initializer for the `embeddings` matrix.\n    sparse_grad: bool\n        If True, gradient w.r.t. weight will be a 'row_sparse' NDArray.\n\n    Inputs:\n        - **data**: (N-1)-D tensor with shape: `(x1, x2, ..., xN-1)`.\n\n    Output:\n        - **out**: N-D tensor with shape: `(x1, x2, ..., xN-1, output_dim)`.\n    \"\"\"\n    def __init__(self, input_dim, output_dim, dtype='float32',\n                 weight_initializer=None, sparse_grad=False, **kwargs):\n        super(Embedding, self).__init__(**kwargs)\n        assert not sparse_grad, \"Currently, sparse feature is not supported in Gluon2.0\"\n        grad_stype = 'row_sparse' if sparse_grad else 'default'\n        self._kwargs = {'input_dim': input_dim, 'output_dim': output_dim,\n                        'dtype': dtype, 'sparse_grad': sparse_grad}\n        self.weight = Parameter('weight', shape=(input_dim, output_dim),\n                                init=weight_initializer, dtype=dtype,\n                                allow_deferred_init=True, grad_stype=grad_stype)\n\n    def forward(self, x):\n        device = x.device\n        return npx.embedding(x, self.weight.data(device), name='fwd', **self._kwargs)\n\n    def __repr__(self):\n        s = '{block_name}({input_dim} -> {output_dim}, {dtype})'\n        return s.format(block_name=self.__class__.__name__,\n                        **self._kwargs)\n\n\n@use_np\nclass Flatten(HybridBlock):\n    r\"\"\"Flattens the input to two dimensional.\n\n    Inputs:\n        - **data**: input tensor with arbitrary shape `(N, x1, x2, ..., xn)`\n\n    Output:\n        - **out**: 2D tensor with shape: `(N, x1 \\cdot x2 \\cdot ... \\cdot xn)`\n    \"\"\"\n    def __init__(self, **kwargs):\n        super(Flatten, self).__init__(**kwargs)\n\n    def forward(self, x):\n        return npx.batch_flatten(x)\n\n    def __repr__(self):\n        return self.__class__.__name__\n\n\n@use_np\nclass InstanceNorm(HybridBlock):\n    r\"\"\"\n    Applies instance normalization to the n-dimensional input array.\n    This operator takes an n-dimensional input array where (n>2) and normalizes\n    the input using the following formula:\n\n    .. math::\n\n      \\bar{C} = \\{i \\mid i \\neq 0, i \\neq axis\\}\n\n      out = \\frac{x - mean[data, \\bar{C}]}{ \\sqrt{Var[data, \\bar{C}]} + \\epsilon}\n       * gamma + beta\n\n    Parameters\n    ----------\n    axis : int, default 1\n        The axis that will be excluded in the normalization process. This is typically the channels\n        (C) axis. For instance, after a `Conv2D` layer with `layout='NCHW'`,\n        set `axis=1` in `InstanceNorm`. If `layout='NHWC'`, then set `axis=3`. Data will be\n        normalized along axes excluding the first axis and the axis given.\n    epsilon: float, default 1e-5\n        Small float added to variance to avoid dividing by zero.\n    center: bool, default True\n        If True, add offset of `beta` to normalized tensor.\n        If False, `beta` is ignored.\n    scale: bool, default True\n        If True, multiply by `gamma`. If False, `gamma` is not used.\n        When the next layer is linear (also e.g. `nn.relu`),\n        this can be disabled since the scaling\n        will be done by the next layer.\n    beta_initializer: str or `Initializer`, default 'zeros'\n        Initializer for the beta weight.\n    gamma_initializer: str or `Initializer`, default 'ones'\n        Initializer for the gamma weight.\n    in_channels : int, default 0\n        Number of channels (feature maps) in input data. If not specified,\n        initialization will be deferred to the first time `forward` is called\n        and `in_channels` will be inferred from the shape of input data.\n\n\n    Inputs:\n        - **data**: input tensor with arbitrary shape.\n\n    Outputs:\n        - **out**: output tensor with the same shape as `data`.\n\n    References\n    ----------\n        `Instance Normalization: The Missing Ingredient for Fast Stylization\n        <https://arxiv.org/abs/1607.08022>`_\n\n    Examples\n    --------\n    >>> # Input of shape (2,1,2)\n    >>> x = mx.np.array([[[ 1.1,  2.2]],\n    ...                 [[ 3.3,  4.4]]])\n    >>> # Instance normalization is calculated with the above formula\n    >>> layer = InstanceNorm()\n    >>> layer.initialize(device=mx.cpu(0))\n    >>> layer(x)\n    [[[-0.99998355  0.99998331]]\n     [[-0.99998319  0.99998361]]]\n    \"\"\"\n    def __init__(self, axis=1, epsilon=1e-5, center=True, scale=False,\n                 beta_initializer='zeros', gamma_initializer='ones',\n                 in_channels=0, **kwargs):\n        super(InstanceNorm, self).__init__(**kwargs)\n        self._kwargs = {'eps': epsilon, 'axis': axis, 'center': center, 'scale': scale}\n        self._axis = axis\n        self._epsilon = epsilon\n        self.gamma = Parameter('gamma', grad_req='write' if scale else 'null',\n                               shape=(in_channels,), init=gamma_initializer,\n                               allow_deferred_init=True)\n        self.beta = Parameter('beta', grad_req='write' if center else 'null',\n                              shape=(in_channels,), init=beta_initializer,\n                              allow_deferred_init=True)\n\n    def forward(self, x):\n        device = x.device\n        if self._axis == 1:\n            return npx.instance_norm(x, self.gamma.data(device), self.beta.data(device),\n                                     name='fwd', eps=self._epsilon)\n        x = x.swapaxes(1, self._axis)\n        return npx.instance_norm(x, self.gamma.data(device), self.beta.data(device),\n                                 name='fwd', eps=self._epsilon).swapaxes(1, self._axis)\n\n    def infer_shape(self, x, *args):\n        self.gamma.shape = (x.shape[1],)\n        self.beta.shape = (x.shape[1],)\n\n    def __repr__(self):\n        s = '{name}({content}'\n        in_channels = self.gamma.shape[0]\n        s += ', in_channels={0}'.format(in_channels)\n        s += ')'\n        return s.format(name=self.__class__.__name__,\n                        content=', '.join(['='.join([k, v.__repr__()])\n                                           for k, v in self._kwargs.items()]))\n\n\n@use_np\nclass LayerNorm(HybridBlock):\n    r\"\"\"\n    Applies layer normalization to the n-dimensional input array.\n    This operator takes an n-dimensional input array and normalizes\n    the input using the given axis:\n\n    .. math::\n\n      out = \\frac{x - mean[data, axis]}{ \\sqrt{Var[data, axis] + \\epsilon}} * gamma + beta\n\n    Parameters\n    ----------\n    axis : int, default -1\n        The axis that should be normalized. This is typically the axis of the channels.\n    epsilon: float, default 1e-5\n        Small float added to variance to avoid dividing by zero.\n    center: bool, default True\n        If True, add offset of `beta` to normalized tensor.\n        If False, `beta` is ignored.\n    scale: bool, default True\n        If True, multiply by `gamma`. If False, `gamma` is not used.\n    beta_initializer: str or `Initializer`, default 'zeros'\n        Initializer for the beta weight.\n    gamma_initializer: str or `Initializer`, default 'ones'\n        Initializer for the gamma weight.\n    in_channels : int, default 0\n        Number of channels (feature maps) in input data. If not specified,\n        initialization will be deferred to the first time `forward` is called\n        and `in_channels` will be inferred from the shape of input data.\n\n\n    Inputs:\n        - **data**: input tensor with arbitrary shape.\n\n    Outputs:\n        - **out**: output tensor with the same shape as `data`.\n\n    References\n    ----------\n        `Layer Normalization\n        <https://arxiv.org/pdf/1607.06450.pdf>`_\n\n    Examples\n    --------\n    >>> # Input of shape (2, 5)\n    >>> x = mx.np.array([[1, 2, 3, 4, 5], [1, 1, 2, 2, 2]])\n    >>> # Layer normalization is calculated with the above formula\n    >>> layer = LayerNorm()\n    >>> layer.initialize(device=mx.cpu(0))\n    >>> layer(x)\n    [[-1.41421    -0.707105    0.          0.707105    1.41421   ]\n     [-1.2247195  -1.2247195   0.81647956  0.81647956  0.81647956]]\n    \"\"\"\n    def __init__(self, axis=-1, epsilon=1e-5, center=True, scale=True,\n                 beta_initializer='zeros', gamma_initializer='ones',\n                 in_channels=0):\n        super(LayerNorm, self).__init__()\n        self._kwargs = {'eps': epsilon, 'axis': axis, 'center': center, 'scale': scale}\n        self._axis = axis\n        self._epsilon = epsilon\n        self._center = center\n        self._scale = scale\n        self.gamma = Parameter('gamma', grad_req='write' if scale else 'null',\n                               shape=(in_channels,), init=gamma_initializer,\n                               allow_deferred_init=True)\n        self.beta = Parameter('beta', grad_req='write' if center else 'null',\n                              shape=(in_channels,), init=beta_initializer,\n                              allow_deferred_init=True)\n\n    def forward(self, data):\n        device = data.device\n        return npx.layer_norm(data, gamma=self.gamma.data(device),\n                              beta=self.beta.data(device), axis=self._axis, eps=self._epsilon)\n\n    def infer_shape(self, data, *args):\n        channel_axis = self._axis if self._axis >= 0 else self._axis + data.ndim\n        channel_count = data.shape[channel_axis]\n        self.gamma.shape = (channel_count,)\n        self.beta.shape = (channel_count,)\n\n    def __repr__(self):\n        s = '{name}({content}'\n        in_channels = self.gamma.shape[0]\n        s += ', in_channels={0}'.format(in_channels)\n        s += ')'\n        return s.format(name=self.__class__.__name__,\n                        content=', '.join(['='.join([k, v.__repr__()])\n                                           for k, v in self._kwargs.items()]))\n\n\n@use_np\nclass GroupNorm(HybridBlock):\n    r\"\"\"\n    Applies group normalization to the n-dimensional input array.\n    This operator takes an n-dimensional input array where the leftmost 2 axis are\n    `batch` and `channel` respectively:\n\n    .. math::\n\n      x = x.reshape((N, num_groups, C // num_groups, ...))\n      axis = (2, ...)\n      out = \\frac{x - mean[x, axis]}{ \\sqrt{Var[x, axis] + \\epsilon}} * gamma + beta\n\n    Parameters\n    ----------\n    num_groups: int, default 1\n        Number of groups to separate the channel axis into.\n    epsilon: float, default 1e-5\n        Small float added to variance to avoid dividing by zero.\n    center: bool, default True\n        If True, add offset of `beta` to normalized tensor.\n        If False, `beta` is ignored.\n    scale: bool, default True\n        If True, multiply by `gamma`. If False, `gamma` is not used.\n    beta_initializer: str or `Initializer`, default 'zeros'\n        Initializer for the beta weight.\n    gamma_initializer: str or `Initializer`, default 'ones'\n        Initializer for the gamma weight.\n\n\n    Inputs:\n        - **data**: input tensor with shape (N, C, ...).\n\n    Outputs:\n        - **out**: output tensor with the same shape as `data`.\n\n    References\n    ----------\n        `Group Normalization\n        <https://arxiv.org/pdf/1803.08494.pdf>`_\n\n    Examples\n    --------\n    >>> # Input of shape (2, 3, 4)\n    >>> x = mx.np.array([[[ 0,  1,  2,  3],\n                          [ 4,  5,  6,  7],\n                          [ 8,  9, 10, 11]],\n                         [[12, 13, 14, 15],\n                          [16, 17, 18, 19],\n                          [20, 21, 22, 23]]])\n    >>> # Group normalization is calculated with the above formula\n    >>> layer = GroupNorm()\n    >>> layer.initialize(device=mx.cpu(0))\n    >>> layer(x)\n    [[[-1.5932543 -1.3035717 -1.0138891 -0.7242065]\n      [-0.4345239 -0.1448413  0.1448413  0.4345239]\n      [ 0.7242065  1.0138891  1.3035717  1.5932543]]\n     [[-1.5932543 -1.3035717 -1.0138891 -0.7242065]\n      [-0.4345239 -0.1448413  0.1448413  0.4345239]\n      [ 0.7242065  1.0138891  1.3035717  1.5932543]]]\n    \"\"\"\n    def __init__(self, num_groups=1, epsilon=1e-5, center=True, scale=True,\n                 beta_initializer='zeros', gamma_initializer='ones',\n                 in_channels=0):\n        super(GroupNorm, self).__init__()\n        self._kwargs = {'eps': epsilon, 'num_groups': num_groups, 'center': center, 'scale': scale}\n        self._num_groups = num_groups\n        self._epsilon = epsilon\n        self._center = center\n        self._scale = scale\n        self.gamma = Parameter('gamma', grad_req='write' if scale else 'null',\n                               shape=(in_channels,), init=gamma_initializer,\n                               allow_deferred_init=True)\n        self.beta = Parameter('beta', grad_req='write' if center else 'null',\n                              shape=(in_channels,), init=beta_initializer,\n                              allow_deferred_init=True)\n\n    def forward(self, data):\n        device = data.device\n        norm_data = npx.group_norm(data, gamma=self.gamma.data(device), beta=self.beta.data(device),\n                                   num_groups=self._num_groups, eps=self._epsilon)\n        return norm_data\n\n    def infer_shape(self, data, *args):\n        self.gamma.shape = (data.shape[1],)\n        self.beta.shape = (data.shape[1],)\n\n    def __repr__(self):\n        s = '{name}({content}'\n        in_channels = self.gamma.shape[0]\n        s += ', in_channels={0}'.format(in_channels)\n        s += ')'\n        return s.format(name=self.__class__.__name__,\n                        content=', '.join(['='.join([k, v.__repr__()])\n                                           for k, v in self._kwargs.items()]))\n\n\nclass Lambda(Block):\n    r\"\"\"Wraps an operator or an expression as a Block object.\n\n\n    Parameters\n    ----------\n    function : str or function\n        Function used in lambda must be one of the following:\n        1) the name of an operator that is available in ndarray. For example::\n\n            block = Lambda('tanh')\n\n        2) a function that conforms to ``def function(*args)``. For example::\n\n            block = Lambda(lambda x: npx.leaky_relu(x, slope=0.1))\n\n    Inputs:\n        - ** *args **: one or more input data. Their shapes depend on the function.\n\n    Output:\n        - ** *outputs **: one or more output data. Their shapes depend on the function.\n    \"\"\"\n    def __init__(self, function):\n        super(Lambda, self).__init__()\n        if isinstance(function, str):\n            if hasattr(np, function):\n                self._func_impl = getattr(np, function)\n            elif hasattr(npx, function):\n                self._func_impl = getattr(npx, function)\n            else:\n                raise Exception(f\"Function name {function} is not found in np/npx.\")\n            self._func_name = function\n        elif callable(function):\n            self._func_impl = function\n        else:\n            raise ValueError(\n                \"Unrecognized function in lambda: {} of type {}\"\n                .format(function, type(function)))\n\n    def forward(self, *args):\n        return self._func_impl(*args)\n\n    def __repr__(self):\n        return '{name}({function})'.format(name=self.__class__.__name__,\n                                           function=self._func_impl.__name__)\n\n\n@use_np\nclass HybridLambda(HybridBlock):\n    r\"\"\"Wraps an operator or an expression as a HybridBlock object.\n\n    Parameters\n    ----------\n    function : str or function\n        Function used in lambda must be one of the following:\n        1) The name of an operator that is available in both symbol and ndarray. For example::\n\n            block = HybridLambda('tanh')\n\n        2) A function that conforms to ``def function(F, data, *args)``. For example::\n\n            block = HybridLambda(lambda F, x: F.LeakyReLU(x, slope=0.1))\n\n    Inputs:\n        - ** *args **: one or more input data. First argument must be symbol or ndarray. Their \\\n            shapes depend on the function.\n\n    Output:\n        - ** *outputs **: one or more output data. Their shapes depend on the function.\n\n    \"\"\"\n    def __init__(self, function):\n        super(HybridLambda, self).__init__()\n        if isinstance(function, str):\n            if hasattr(np, function):\n                self._func = getattr(np, function)\n            elif hasattr(npx, function):\n                self._func = getattr(npx, function)\n            else:\n                raise Exception(f\"Function name {function} is not found in np/npx.\")\n            self._func_name = function\n        elif callable(function):\n            self._func = function\n            self._func_name = function.__name__\n        else:\n            raise ValueError(\n                \"Unrecognized function in lambda: {} of type {}\"\n                .format(function, type(function)))\n\n    def forward(self, x, *args):\n        return self._func(x, *args)\n\n    def __repr__(self):\n        return '{name}({function})'.format(name=self.__class__.__name__,\n                                           function=self._func_name)\n\n\n@use_np\nclass Concatenate(Sequential):\n    \"\"\"Lays `Block` s concurrently.\n\n    This block feeds its input to all children blocks, and\n    produce the output by concatenating all the children blocks' outputs\n    on the specified axis.\n\n    Example::\n\n        net = Concatenate()\n        net.add(nn.Dense(10, activation='relu'))\n        net.add(nn.Dense(20))\n        net.add(Identity())\n\n    Parameters\n    ----------\n    axis : int, default -1\n        The axis on which to concatenate the outputs.\n    \"\"\"\n    def __init__(self, axis=-1):\n        super(Concatenate, self).__init__()\n        self.axis = axis\n\n    def forward(self, x):\n        out = []\n        for block in self._children.values():\n            out.append(block()(x))\n        out = np.concatenate(out, axis=self.axis)\n        return out\n\n\n@use_np\nclass HybridConcatenate(HybridSequential):\n    \"\"\"Lays `HybridBlock` s concurrently.\n\n    This block feeds its input to all children blocks, and\n    produce the output by concatenating all the children blocks' outputs\n    on the specified axis.\n\n    Example::\n\n        net = HybridConcatenate()\n        net.add(nn.Dense(10, activation='relu'))\n        net.add(nn.Dense(20))\n        net.add(Identity())\n\n    Parameters\n    ----------\n    axis : int, default -1\n        The axis on which to concatenate the outputs.\n    \"\"\"\n    def __init__(self, axis=-1):\n        super().__init__()\n        self.axis = axis\n\n    def forward(self, x):\n        out = []\n        for block in self._children.values():\n            out.append(block()(x))\n        out = np.concatenate(out, axis=self.axis)\n        return out\n\n\n@use_np\nclass Identity(HybridBlock):\n    \"\"\"Block that passes through the input directly.\n\n    This block can be used in conjunction with HybridConcatenate\n    block for residual connection.\n\n    Example::\n\n        net = HybridConcatenate()\n        net.add(nn.Dense(10, activation='relu'))\n        net.add(nn.Dense(20))\n        net.add(Identity())\n    \"\"\"\n    def __init__(self):\n        super(Identity, self).__init__()\n\n    def forward(self, x):\n        return x\n\n\n@use_np\nclass SyncBatchNorm(BatchNorm):\n    \"\"\"Cross-GPU Synchronized Batch normalization (SyncBN)\n\n    Standard BN [1]_ implementation only normalize the data within each device.\n    SyncBN normalizes the input within the whole mini-batch.\n    We follow the implementation described in the paper [2]_.\n\n    Note: Current implementation of SyncBN does not support FP16 training.\n    For FP16 inference, use standard nn.BatchNorm instead of SyncBN.\n\n    Parameters\n    ----------\n    in_channels : int, default 0\n        Number of channels (feature maps) in input data. If not specified,\n        initialization will be deferred to the first time `forward` is called\n        and `in_channels` will be inferred from the shape of input data.\n    num_devices : int, default number of visible GPUs\n    momentum: float, default 0.9\n        Momentum for the moving average.\n    epsilon: float, default 1e-5\n        Small float added to variance to avoid dividing by zero.\n    center: bool, default True\n        If True, add offset of `beta` to normalized tensor.\n        If False, `beta` is ignored.\n    scale: bool, default True\n        If True, multiply by `gamma`. If False, `gamma` is not used.\n        When the next layer is linear (also e.g. `nn.relu`),\n        this can be disabled since the scaling\n        will be done by the next layer.\n    use_global_stats: bool, default False\n        If True, use global moving statistics instead of local batch-norm. This will force\n        change batch-norm into a scale shift operator.\n        If False, use local batch-norm.\n    beta_initializer: str or `Initializer`, default 'zeros'\n        Initializer for the beta weight.\n    gamma_initializer: str or `Initializer`, default 'ones'\n        Initializer for the gamma weight.\n    running_mean_initializer: str or `Initializer`, default 'zeros'\n        Initializer for the running mean.\n    running_variance_initializer: str or `Initializer`, default 'ones'\n        Initializer for the running variance.\n\n\n    Inputs:\n        - **data**: input tensor with arbitrary shape.\n    Outputs:\n        - **out**: output tensor with the same shape as `data`.\n\n    Reference:\n        .. [1] Ioffe, Sergey, and Christian Szegedy. \"Batch normalization: Accelerating \\\n          deep network training by reducing internal covariate shift.\" *ICML 2015*\n        .. [2] Hang Zhang, Kristin Dana, Jianping Shi, Zhongyue Zhang, Xiaogang Wang, \\\n          Ambrish Tyagi, and Amit Agrawal. \"Context Encoding for Semantic Segmentation.\" *CVPR 2018*\n    \"\"\"\n    def __init__(self, in_channels=0, num_devices=None, momentum=0.9, epsilon=1e-5,\n                 center=True, scale=True, use_global_stats=False, beta_initializer='zeros',\n                 gamma_initializer='ones', running_mean_initializer='zeros',\n                 running_variance_initializer='ones', **kwargs):\n        super(SyncBatchNorm, self).__init__(\n            axis=1, momentum=momentum, epsilon=epsilon,\n            center=center, scale=scale,\n            use_global_stats=use_global_stats,\n            beta_initializer=beta_initializer,\n            gamma_initializer=gamma_initializer,\n            running_mean_initializer=running_mean_initializer,\n            running_variance_initializer=running_variance_initializer,\n            in_channels=in_channels, **kwargs)\n        num_devices = self._get_num_devices() if num_devices is None else num_devices\n        self._kwargs = {'eps': epsilon, 'momentum': momentum,\n                        'fix_gamma': not scale, 'use_global_stats': use_global_stats,\n                        'ndev': num_devices, 'key': uuid.uuid4()}\n\n    def _get_num_devices(self):\n        warnings.warn(\"Caution using SyncBatchNorm: \"\n                      \"if not using all the GPUs, please mannually set num_devices\",\n                      UserWarning)\n        num_devices = _device.num_gpus()\n        num_devices = num_devices if num_devices > 0 else 1\n        return num_devices\n\n    def forward(self, x):\n        device = x.device\n        return npx.sync_batch_norm(x, self.gamma.data(device), self.beta.data(device),\n                                   self.running_mean.data(device), self.running_var.data(device),\n                                   name='fwd', **self._kwargs)\n"
  },
  {
    "path": "python/mxnet/gluon/nn/conv_layers.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable= arguments-differ, too-many-lines\n\"\"\"Convolutional neural network layers.\"\"\"\n__all__ = ['Conv1D', 'Conv2D', 'Conv3D',\n           'Conv1DTranspose', 'Conv2DTranspose', 'Conv3DTranspose',\n           'MaxPool1D', 'MaxPool2D', 'MaxPool3D',\n           'AvgPool1D', 'AvgPool2D', 'AvgPool3D',\n           'GlobalMaxPool1D', 'GlobalMaxPool2D', 'GlobalMaxPool3D',\n           'GlobalAvgPool1D', 'GlobalAvgPool2D', 'GlobalAvgPool3D',\n           'ReflectionPad2D', 'DeformableConvolution', 'ModulatedDeformableConvolution',\n           'PixelShuffle1D', 'PixelShuffle2D', 'PixelShuffle3D']\n\nfrom ..block import HybridBlock\nfrom ..parameter import Parameter\nfrom ... import np, npx\nfrom ...base import numeric_types\nfrom .activations import Activation\nfrom ...util import use_np\n\n@use_np\nclass _Conv(HybridBlock):\n    \"\"\"Abstract nD convolution layer (private, used as implementation base).\n\n    This layer creates a convolution kernel that is convolved\n    with the layer input to produce a tensor of outputs.\n    If `use_bias` is `True`, a bias vector is created and added to the outputs.\n    Finally, if `activation` is not `None`,\n    it is applied to the outputs as well.\n\n    Parameters\n    ----------\n    channels : int\n        The dimensionality of the output space\n        i.e. the number of output channels in the convolution.\n    kernel_size : int or tuple/list of n ints\n        Specifies the dimensions of the convolution window.\n    strides: int or tuple/list of n ints,\n        Specifies the strides of the convolution.\n    padding : int or tuple/list of n ints,\n        If padding is non-zero, then the input is implicitly zero-padded\n        on both sides for padding number of points\n    dilation: int or tuple/list of n ints,\n        Specifies the dilation rate to use for dilated convolution.\n    groups : int\n        Controls the connections between inputs and outputs.\n        At groups=1, all inputs are convolved to all outputs.\n        At groups=2, the operation becomes equivalent to having two convolution\n        layers side by side, each seeing half the input channels, and producing\n        half the output channels, and both subsequently concatenated.\n    layout : str,\n        Dimension ordering of data and weight. Can be 'NCW', 'NWC', 'NCHW',\n        'NHWC', 'NCDHW', 'NDHWC', etc. 'N', 'C', 'H', 'W', 'D' stands for\n        batch, channel, height, width and depth dimensions respectively.\n        Convolution is performed over 'D', 'H', and 'W' dimensions.\n    in_channels : int, default 0\n        The number of input channels to this layer. If not specified,\n        initialization will be deferred to the first time `forward` is called\n        and `in_channels` will be inferred from the shape of input data.\n    activation : str\n        Activation function to use. See :func:`~mxnet.npx.activation`.\n        If you don't specify anything, no activation is applied\n        (ie. \"linear\" activation: `a(x) = x`).\n    use_bias: bool\n        Whether the layer uses a bias vector.\n    weight_initializer : str or `Initializer`\n        Initializer for the `weight` weights matrix.\n    bias_initializer: str or `Initializer`\n        Initializer for the bias vector.\n    \"\"\"\n    def __init__(self, channels, kernel_size, strides, padding, dilation,\n                 groups, layout, in_channels=0, activation=None, use_bias=True,\n                 weight_initializer=None, bias_initializer='zeros',\n                 op_name='convolution', adj=None):\n        super(_Conv, self).__init__()\n        self._channels = channels\n        self._in_channels = in_channels\n        self._kernel_size = kernel_size\n        self._layout = layout\n        self._groups = groups\n        if isinstance(strides, numeric_types):\n            strides = (strides,)*len(kernel_size)\n        if isinstance(padding, numeric_types):\n            padding = (padding,)*len(kernel_size)\n        if isinstance(dilation, numeric_types):\n            dilation = (dilation,)*len(kernel_size)\n        self._op_name = op_name\n        self._kwargs = {\n            'kernel': kernel_size, 'stride': strides, 'dilate': dilation,\n            'pad': padding, 'num_filter': channels, 'num_group': groups,\n            'no_bias': not use_bias, 'layout': layout}\n        if adj is not None:\n            self._kwargs['adj'] = adj\n\n        self.weight = Parameter('weight', shape=self.pre_infer(),\n                                init=weight_initializer,\n                                allow_deferred_init=True)\n        if use_bias:\n            self.bias = Parameter('bias', shape=(channels,),\n                                  init=bias_initializer,\n                                  allow_deferred_init=True)\n        else:\n            self.bias = None\n\n        if activation is not None:\n            self.act = Activation(activation)\n        else:\n            self.act = None\n\n    def forward(self, x):\n        device = x.device\n        if self.bias is None:\n            act = getattr(npx, self._op_name)(x, self.weight.data(device), **self._kwargs)\n        else:\n            act = getattr(npx, self._op_name)(x, self.weight.data(device), self.bias.data(device),\n                                              **self._kwargs)\n        if self.act is not None:\n            act = self.act(act)\n        return act\n\n    def pre_infer(self):\n        \"\"\"\n        Pre-infer the shape of weight parameter based on kernel size, group size and channels\n        \"\"\"\n        wshape = [-1]*(len(self._kernel_size) + 2)\n        if self._op_name == \"convolution\":\n            if len(self._kernel_size) == 1:\n                wshape[self._layout.find('N')] = self._channels // self._groups\n                wshape[self._layout.find('W')] = self._kernel_size[0]\n                wshape[0] *= self._groups\n            elif len(self._kernel_size) == 2:\n                wshape[self._layout.find('N')] = self._channels // self._groups\n                wshape[self._layout.find('H')] = self._kernel_size[0]\n                wshape[self._layout.find('W')] = self._kernel_size[1]\n                wshape[0] *= self._groups\n            else:\n                assert len(self._kernel_size) == 3, \"kernel_size must be 1, 2 or 3\"\n                wshape[self._layout.find('N')] = self._channels // self._groups\n                wshape[self._layout.find('D')] = self._kernel_size[0]\n                wshape[self._layout.find('H')] = self._kernel_size[1]\n                wshape[self._layout.find('W')] = self._kernel_size[2]\n                wshape[0] *= self._groups\n        else:\n            assert self._op_name == \"deconvolution\", \\\n                \"Only support operator name with convolution and deconvolution\"\n            if len(self._kernel_size) == 1:\n                wshape[self._layout.find('C')] = self._channels // self._groups\n                wshape[self._layout.find('W')] = self._kernel_size[0]\n            elif len(self._kernel_size) == 2:\n                wshape[self._layout.find('C')] = self._channels // self._groups\n                wshape[self._layout.find('H')] = self._kernel_size[0]\n                wshape[self._layout.find('W')] = self._kernel_size[1]\n            else:\n                assert len(self._kernel_size) == 3, \"kernel_size must be 1, 2 or 3\"\n                wshape[self._layout.find('C')] = self._channels // self._groups\n                wshape[self._layout.find('D')] = self._kernel_size[0]\n                wshape[self._layout.find('H')] = self._kernel_size[1]\n                wshape[self._layout.find('W')] = self._kernel_size[2]\n        return tuple(wshape)\n\n    def infer_shape(self, x):\n        dshape1 = x.shape[self._layout.find('C')]\n        wshape = self.weight.shape\n        if self._op_name == \"convolution\":\n            wshape_list = list(wshape)\n            wshape_list[self._layout.find('C')] = dshape1 // self._groups\n        else:\n            assert self._op_name == \"deconvolution\", \\\n                \"Only support operator name with convolution and deconvolution\"\n            wshape_list = list(wshape)\n            wshape_list[self._layout.find('N')] = dshape1\n        self.weight.shape = tuple(wshape_list)\n\n    def _alias(self):\n        return 'conv'\n\n    def __repr__(self):\n        s = '{name}({mapping}, kernel_size={kernel}, stride={stride}'\n        len_kernel_size = len(self._kwargs['kernel'])\n        if self._kwargs['pad'] != (0,) * len_kernel_size:\n            s += ', padding={pad}'\n        if self._kwargs['dilate'] != (1,) * len_kernel_size:\n            s += ', dilation={dilate}'\n        if hasattr(self, 'out_pad') and self.out_pad != (0,) * len_kernel_size:\n            s += ', output_padding={out_pad}'.format(out_pad=self.out_pad)\n        if self._kwargs['num_group'] != 1:\n            s += ', groups={num_group}'\n        if self.bias is None:\n            s += ', bias=False'\n        if self.act:\n            s += ', {}'.format(self.act)\n        s += ')'\n        shape = self.weight.shape\n        if 'Transpose' in self.__class__.__name__:\n            mapping = '{1} -> {0}'\n        else:\n            mapping = '{0} -> {1}'\n        return s.format(name=self.__class__.__name__,\n                        mapping=mapping.format(shape[1] if shape[1] else None, shape[0]),\n                        **self._kwargs)\n\n\nclass Conv1D(_Conv):\n    r\"\"\"1D convolution layer (e.g. temporal convolution).\n\n    This layer creates a convolution kernel that is convolved\n    with the layer input over a single spatial (or temporal) dimension\n    to produce a tensor of outputs.\n    If `use_bias` is True, a bias vector is created and added to the outputs.\n    Finally, if `activation` is not `None`,\n    it is applied to the outputs as well.\n\n    If `in_channels` is not specified, `Parameter` initialization will be\n    deferred to the first time `forward` is called and `in_channels` will be\n    inferred from the shape of input data.\n\n\n    Parameters\n    ----------\n    channels : int\n        The dimensionality of the output space, i.e. the number of output\n        channels (filters) in the convolution.\n    kernel_size :int or tuple/list of 1 int\n        Specifies the dimensions of the convolution window.\n    strides : int or tuple/list of 1 int,\n        Specify the strides of the convolution.\n    padding : int or a tuple/list of 1 int,\n        If padding is non-zero, then the input is implicitly zero-padded\n        on both sides for padding number of points\n    dilation : int or tuple/list of 1 int\n        Specifies the dilation rate to use for dilated convolution.\n    groups : int\n        Controls the connections between inputs and outputs.\n        At groups=1, all inputs are convolved to all outputs.\n        At groups=2, the operation becomes equivalent to having two conv\n        layers side by side, each seeing half the input channels, and producing\n        half the output channels, and both subsequently concatenated.\n    layout: str, default 'NCW'\n        Dimension ordering of data and weight. Only supports 'NCW' layout for now.\n        'N', 'C', 'W' stands for batch, channel, and width (time) dimensions\n        respectively. Convolution is applied on the 'W' dimension.\n    in_channels : int, default 0\n        The number of input channels to this layer. If not specified,\n        initialization will be deferred to the first time `forward` is called\n        and `in_channels` will be inferred from the shape of input data.\n    activation : str\n        Activation function to use. See :func:`~mxnet.npx.activation`.\n        If you don't specify anything, no activation is applied\n        (ie. \"linear\" activation: `a(x) = x`).\n    use_bias : bool\n        Whether the layer uses a bias vector.\n    weight_initializer : str or `Initializer`\n        Initializer for the `weight` weights matrix.\n    bias_initializer : str or `Initializer`\n        Initializer for the bias vector.\n\n\n    Inputs:\n        - **data**: 3D input tensor with shape `(batch_size, in_channels, width)`\n          when `layout` is `NCW`. For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 3D output tensor with shape `(batch_size, channels, out_width)`\n          when `layout` is `NCW`. out_width is calculated as::\n\n              out_width = floor((width+2*padding-dilation*(kernel_size-1)-1)/stride)+1\n    \"\"\"\n    def __init__(self, channels, kernel_size, strides=1, padding=0, dilation=1,\n                 groups=1, layout='NCW', activation=None, use_bias=True,\n                 weight_initializer=None, bias_initializer='zeros',\n                 in_channels=0, **kwargs):\n        assert layout == 'NCW', \"Only supports 'NCW' layout for now\"\n        if isinstance(kernel_size, numeric_types):\n            kernel_size = (kernel_size,)\n        assert len(kernel_size) == 1, \"kernel_size must be a number or a list of 1 ints\"\n        op_name = 'convolution'\n        super(Conv1D, self).__init__(\n            channels, kernel_size, strides, padding, dilation, groups, layout,\n            in_channels, activation, use_bias, weight_initializer, bias_initializer,\n            op_name, **kwargs)\n\n\nclass Conv2D(_Conv):\n    r\"\"\"2D convolution layer (e.g. spatial convolution over images).\n\n    This layer creates a convolution kernel that is convolved\n    with the layer input to produce a tensor of\n    outputs. If `use_bias` is True,\n    a bias vector is created and added to the outputs. Finally, if\n    `activation` is not `None`, it is applied to the outputs as well.\n\n    If `in_channels` is not specified, `Parameter` initialization will be\n    deferred to the first time `forward` is called and `in_channels` will be\n    inferred from the shape of input data.\n\n    Parameters\n    ----------\n    channels : int\n        The dimensionality of the output space, i.e. the number of output\n        channels (filters) in the convolution.\n    kernel_size :int or tuple/list of 2 int\n        Specifies the dimensions of the convolution window.\n    strides : int or tuple/list of 2 int,\n        Specify the strides of the convolution.\n    padding : int or a tuple/list of 2 int,\n        If padding is non-zero, then the input is implicitly zero-padded\n        on both sides for padding number of points\n    dilation : int or tuple/list of 2 int\n        Specifies the dilation rate to use for dilated convolution.\n    groups : int\n        Controls the connections between inputs and outputs.\n        At groups=1, all inputs are convolved to all outputs.\n        At groups=2, the operation becomes equivalent to having two conv\n        layers side by side, each seeing half the input channels, and producing\n        half the output channels, and both subsequently concatenated.\n    layout : str, default 'NCHW'\n        Dimension ordering of data and weight. Only supports 'NCHW' and 'NHWC'\n        layout for now. 'N', 'C', 'H', 'W' stands for batch, channel, height,\n        and width dimensions respectively. Convolution is applied on the 'H' and\n        'W' dimensions.\n    in_channels : int, default 0\n        The number of input channels to this layer. If not specified,\n        initialization will be deferred to the first time `forward` is called\n        and `in_channels` will be inferred from the shape of input data.\n    activation : str\n        Activation function to use. See :func:`~mxnet.npx.activation`.\n        If you don't specify anything, no activation is applied\n        (ie. \"linear\" activation: `a(x) = x`).\n    use_bias : bool\n        Whether the layer uses a bias vector.\n    weight_initializer : str or `Initializer`\n        Initializer for the `weight` weights matrix.\n    bias_initializer : str or `Initializer`\n        Initializer for the bias vector.\n\n\n    Inputs:\n        - **data**: 4D input tensor with shape\n          `(batch_size, in_channels, height, width)` when `layout` is `NCHW`.\n          For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 4D output tensor with shape\n          `(batch_size, channels, out_height, out_width)` when `layout` is `NCHW`.\n          out_height and out_width are calculated as::\n\n              out_height = floor((height+2*padding[0]-dilation[0]*(kernel_size[0]-1)-1)/stride[0])+1\n              out_width = floor((width+2*padding[1]-dilation[1]*(kernel_size[1]-1)-1)/stride[1])+1\n    \"\"\"\n    def __init__(self, channels, kernel_size, strides=(1, 1), padding=(0, 0),\n                 dilation=(1, 1), groups=1, layout='NCHW',\n                 activation=None, use_bias=True, weight_initializer=None,\n                 bias_initializer='zeros', in_channels=0, **kwargs):\n        assert layout in ('NCHW', 'NHWC'), \"Only supports 'NCHW' and 'NHWC' layout for now\"\n        if isinstance(kernel_size, numeric_types):\n            kernel_size = (kernel_size,)*2\n        assert len(kernel_size) == 2, \"kernel_size must be a number or a list of 2 ints\"\n        op_name = 'convolution'\n        super(Conv2D, self).__init__(\n            channels, kernel_size, strides, padding, dilation, groups, layout,\n            in_channels, activation, use_bias, weight_initializer, bias_initializer,\n            op_name, **kwargs)\n\n\nclass Conv3D(_Conv):\n    \"\"\"3D convolution layer (e.g. spatial convolution over volumes).\n\n    This layer creates a convolution kernel that is convolved\n    with the layer input to produce a tensor of\n    outputs. If `use_bias` is `True`,\n    a bias vector is created and added to the outputs. Finally, if\n    `activation` is not `None`, it is applied to the outputs as well.\n\n    If `in_channels` is not specified, `Parameter` initialization will be\n    deferred to the first time `forward` is called and `in_channels` will be\n    inferred from the shape of input data.\n\n    Parameters\n    ----------\n    channels : int\n        The dimensionality of the output space, i.e. the number of output\n        channels (filters) in the convolution.\n    kernel_size :int or tuple/list of 3 int\n        Specifies the dimensions of the convolution window.\n    strides : int or tuple/list of 3 int,\n        Specify the strides of the convolution.\n    padding : int or a tuple/list of 3 int,\n        If padding is non-zero, then the input is implicitly zero-padded\n        on both sides for padding number of points\n    dilation : int or tuple/list of 3 int\n        Specifies the dilation rate to use for dilated convolution.\n    groups : int\n        Controls the connections between inputs and outputs.\n        At groups=1, all inputs are convolved to all outputs.\n        At groups=2, the operation becomes equivalent to having two conv\n        layers side by side, each seeing half the input channels, and producing\n        half the output channels, and both subsequently concatenated.\n    layout : str, default 'NCDHW'\n        Dimension ordering of data and weight. Only supports 'NCDHW' and 'NDHWC'\n        layout for now. 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height,\n        width and depth dimensions respectively. Convolution is applied on the 'D',\n        'H' and 'W' dimensions.\n    in_channels : int, default 0\n        The number of input channels to this layer. If not specified,\n        initialization will be deferred to the first time `forward` is called\n        and `in_channels` will be inferred from the shape of input data.\n    activation : str\n        Activation function to use. See :func:`~mxnet.npx.activation`.\n        If you don't specify anything, no activation is applied\n        (ie. \"linear\" activation: `a(x) = x`).\n    use_bias : bool\n        Whether the layer uses a bias vector.\n    weight_initializer : str or `Initializer`\n        Initializer for the `weight` weights matrix.\n    bias_initializer : str or `Initializer`\n        Initializer for the bias vector.\n\n\n    Inputs:\n        - **data**: 5D input tensor with shape\n          `(batch_size, in_channels, depth, height, width)` when `layout` is `NCDHW`.\n          For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 5D output tensor with shape\n          `(batch_size, channels, out_depth, out_height, out_width)` when `layout` is `NCDHW`.\n          out_depth, out_height and out_width are calculated as::\n\n              out_depth = floor((depth+2*padding[0]-dilation[0]*(kernel_size[0]-1)-1)/stride[0])+1\n              out_height = floor((height+2*padding[1]-dilation[1]*(kernel_size[1]-1)-1)/stride[1])+1\n              out_width = floor((width+2*padding[2]-dilation[2]*(kernel_size[2]-1)-1)/stride[2])+1\n    \"\"\"\n    def __init__(self, channels, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0),\n                 dilation=(1, 1, 1), groups=1, layout='NCDHW', activation=None,\n                 use_bias=True, weight_initializer=None, bias_initializer='zeros',\n                 in_channels=0, **kwargs):\n        assert layout in ('NCDHW', 'NDHWC'), \"Only supports 'NCDHW' and 'NDHWC' layout for now\"\n        if isinstance(kernel_size, numeric_types):\n            kernel_size = (kernel_size,)*3\n        assert len(kernel_size) == 3, \"kernel_size must be a number or a list of 3 ints\"\n        op_name = 'convolution'\n        super(Conv3D, self).__init__(\n            channels, kernel_size, strides, padding, dilation, groups, layout,\n            in_channels, activation, use_bias, weight_initializer, bias_initializer,\n            op_name, **kwargs)\n\n\nclass Conv1DTranspose(_Conv):\n    \"\"\"Transposed 1D convolution layer (sometimes called Deconvolution).\n\n    The need for transposed convolutions generally arises\n    from the desire to use a transformation going in the opposite direction\n    of a normal convolution, i.e., from something that has the shape of the\n    output of some convolution to something that has the shape of its input\n    while maintaining a connectivity pattern that is compatible with\n    said convolution.\n\n    If `in_channels` is not specified, `Parameter` initialization will be\n    deferred to the first time `forward` is called and `in_channels` will be\n    inferred from the shape of input data.\n\n    Parameters\n    ----------\n    channels : int\n        The dimensionality of the output space, i.e. the number of output\n        channels (filters) in the convolution.\n    kernel_size :int or tuple/list of 1 int\n        Specifies the dimensions of the convolution window.\n    strides : int or tuple/list of 1 int\n        Specify the strides of the convolution.\n    padding : int or a tuple/list of 1 int,\n        If padding is non-zero, then the input is implicitly zero-padded\n        on both sides for padding number of points\n    output_padding: int or a tuple/list of 1 int\n        Controls the amount of implicit zero-paddings on both sides of the\n        output for output_padding number of points for each dimension.\n    dilation : int or tuple/list of 1 int\n        Controls the spacing between the kernel points; also known as the\n        a trous algorithm\n    groups : int\n        Controls the connections between inputs and outputs.\n        At groups=1, all inputs are convolved to all outputs.\n        At groups=2, the operation becomes equivalent to having two conv\n        layers side by side, each seeing half the input channels, and producing\n        half the output channels, and both subsequently concatenated.\n    layout : str, default 'NCW'\n        Dimension ordering of data and weight. Only supports 'NCW' layout for now.\n        'N', 'C', 'W' stands for batch, channel, and width (time) dimensions\n        respectively. Convolution is applied on the 'W' dimension.\n    in_channels : int, default 0\n        The number of input channels to this layer. If not specified,\n        initialization will be deferred to the first time `forward` is called\n        and `in_channels` will be inferred from the shape of input data.\n    activation : str\n        Activation function to use. See :func:`~mxnet.npx.activation`.\n        If you don't specify anything, no activation is applied\n        (ie. \"linear\" activation: `a(x) = x`).\n    use_bias : bool\n        Whether the layer uses a bias vector.\n    weight_initializer : str or `Initializer`\n        Initializer for the `weight` weights matrix.\n    bias_initializer : str or `Initializer`\n        Initializer for the bias vector.\n\n\n    Inputs:\n        - **data**: 3D input tensor with shape `(batch_size, in_channels, width)`\n          when `layout` is `NCW`. For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 3D output tensor with shape `(batch_size, channels, out_width)`\n          when `layout` is `NCW`. out_width is calculated as::\n\n              out_width = (width-1)*strides-2*padding+kernel_size+output_padding\n    \"\"\"\n    def __init__(self, channels, kernel_size, strides=1, padding=0, output_padding=0,\n                 dilation=1, groups=1, layout='NCW', activation=None, use_bias=True,\n                 weight_initializer=None, bias_initializer='zeros',\n                 in_channels=0, **kwargs):\n        assert layout == 'NCW', \"Only supports 'NCW' layout for now\"\n        if isinstance(kernel_size, numeric_types):\n            kernel_size = (kernel_size,)\n        if isinstance(output_padding, numeric_types):\n            output_padding = (output_padding,)\n        assert len(kernel_size) == 1, \"kernel_size must be a number or a list of 1 ints\"\n        assert len(output_padding) == 1, \"output_padding must be a number or a list of 1 ints\"\n        op_name = 'deconvolution'\n        super(Conv1DTranspose, self).__init__(\n            channels, kernel_size, strides, padding, dilation, groups, layout,\n            in_channels, activation, use_bias, weight_initializer,\n            bias_initializer, op_name=op_name, adj=output_padding, **kwargs)\n        self.outpad = output_padding\n\n\nclass Conv2DTranspose(_Conv):\n    \"\"\"Transposed 2D convolution layer (sometimes called Deconvolution).\n\n    The need for transposed convolutions generally arises\n    from the desire to use a transformation going in the opposite direction\n    of a normal convolution, i.e., from something that has the shape of the\n    output of some convolution to something that has the shape of its input\n    while maintaining a connectivity pattern that is compatible with\n    said convolution.\n\n    If `in_channels` is not specified, `Parameter` initialization will be\n    deferred to the first time `forward` is called and `in_channels` will be\n    inferred from the shape of input data.\n\n\n    Parameters\n    ----------\n    channels : int\n        The dimensionality of the output space, i.e. the number of output\n        channels (filters) in the convolution.\n    kernel_size :int or tuple/list of 2 int\n        Specifies the dimensions of the convolution window.\n    strides : int or tuple/list of 2 int\n        Specify the strides of the convolution.\n    padding : int or a tuple/list of 2 int,\n        If padding is non-zero, then the input is implicitly zero-padded\n        on both sides for padding number of points\n    output_padding: int or a tuple/list of 2 int\n        Controls the amount of implicit zero-paddings on both sides of the\n        output for output_padding number of points for each dimension.\n    dilation : int or tuple/list of 2 int\n        Controls the spacing between the kernel points; also known as the\n        a trous algorithm\n    groups : int\n        Controls the connections between inputs and outputs.\n        At groups=1, all inputs are convolved to all outputs.\n        At groups=2, the operation becomes equivalent to having two conv\n        layers side by side, each seeing half the input channels, and producing\n        half the output channels, and both subsequently concatenated.\n    layout : str, default 'NCHW'\n        Dimension ordering of data and weight. Only supports 'NCHW' and 'NHWC'\n        layout for now. 'N', 'C', 'H', 'W' stands for batch, channel, height,\n        and width dimensions respectively. Convolution is applied on the 'H' and\n        'W' dimensions.\n    in_channels : int, default 0\n        The number of input channels to this layer. If not specified,\n        initialization will be deferred to the first time `forward` is called\n        and `in_channels` will be inferred from the shape of input data.\n    activation : str\n        Activation function to use. See :func:`~mxnet.npx.activation`.\n        If you don't specify anything, no activation is applied\n        (ie. \"linear\" activation: `a(x) = x`).\n    use_bias : bool\n        Whether the layer uses a bias vector.\n    weight_initializer : str or `Initializer`\n        Initializer for the `weight` weights matrix.\n    bias_initializer : str or `Initializer`\n        Initializer for the bias vector.\n\n\n    Inputs:\n        - **data**: 4D input tensor with shape\n          `(batch_size, in_channels, height, width)` when `layout` is `NCHW`.\n          For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 4D output tensor with shape\n          `(batch_size, channels, out_height, out_width)` when `layout` is `NCHW`.\n          out_height and out_width are calculated as::\n\n              out_height = (height-1)*strides[0]-2*padding[0]+kernel_size[0]+output_padding[0]\n              out_width = (width-1)*strides[1]-2*padding[1]+kernel_size[1]+output_padding[1]\n    \"\"\"\n    def __init__(self, channels, kernel_size, strides=(1, 1), padding=(0, 0),\n                 output_padding=(0, 0), dilation=(1, 1), groups=1, layout='NCHW',\n                 activation=None, use_bias=True, weight_initializer=None,\n                 bias_initializer='zeros', in_channels=0, **kwargs):\n        assert layout in ('NCHW', 'NHWC'), \"Only supports 'NCHW' and 'NHWC' layout for now\"\n        if isinstance(kernel_size, numeric_types):\n            kernel_size = (kernel_size,)*2\n        if isinstance(output_padding, numeric_types):\n            output_padding = (output_padding,)*2\n        assert len(kernel_size) == 2, \"kernel_size must be a number or a list of 2 ints\"\n        assert len(output_padding) == 2, \"output_padding must be a number or a list of 2 ints\"\n        op_name = 'deconvolution'\n        super(Conv2DTranspose, self).__init__(\n            channels, kernel_size, strides, padding, dilation, groups, layout,\n            in_channels, activation, use_bias, weight_initializer,\n            bias_initializer, op_name=op_name, adj=output_padding, **kwargs)\n        self.outpad = output_padding\n\n\nclass Conv3DTranspose(_Conv):\n    \"\"\"Transposed 3D convolution layer (sometimes called Deconvolution).\n\n    The need for transposed convolutions generally arises\n    from the desire to use a transformation going in the opposite direction\n    of a normal convolution, i.e., from something that has the shape of the\n    output of some convolution to something that has the shape of its input\n    while maintaining a connectivity pattern that is compatible with\n    said convolution.\n\n    If `in_channels` is not specified, `Parameter` initialization will be\n    deferred to the first time `forward` is called and `in_channels` will be\n    inferred from the shape of input data.\n\n\n    Parameters\n    ----------\n    channels : int\n        The dimensionality of the output space, i.e. the number of output\n        channels (filters) in the convolution.\n    kernel_size :int or tuple/list of 3 int\n        Specifies the dimensions of the convolution window.\n    strides : int or tuple/list of 3 int\n        Specify the strides of the convolution.\n    padding : int or a tuple/list of 3 int,\n        If padding is non-zero, then the input is implicitly zero-padded\n        on both sides for padding number of points\n    output_padding: int or a tuple/list of 3 int\n        Controls the amount of implicit zero-paddings on both sides of the\n        output for output_padding number of points for each dimension.\n    dilation : int or tuple/list of 3 int\n        Controls the spacing between the kernel points; also known as the\n        a trous algorithm.\n    groups : int\n        Controls the connections between inputs and outputs.\n        At groups=1, all inputs are convolved to all outputs.\n        At groups=2, the operation becomes equivalent to having two conv\n        layers side by side, each seeing half the input channels, and producing\n        half the output channels, and both subsequently concatenated.\n    layout : str, default 'NCDHW'\n        Dimension ordering of data and weight. Only supports 'NCDHW' and 'NDHWC'\n        layout for now. 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height,\n        width and depth dimensions respectively. Convolution is applied on the 'D',\n        'H' and 'W' dimensions.\n    in_channels : int, default 0\n        The number of input channels to this layer. If not specified,\n        initialization will be deferred to the first time `forward` is called\n        and `in_channels` will be inferred from the shape of input data.\n    activation : str\n        Activation function to use. See :func:`~mxnet.npx.activation`.\n        If you don't specify anything, no activation is applied\n        (ie. \"linear\" activation: `a(x) = x`).\n    use_bias : bool\n        Whether the layer uses a bias vector.\n    weight_initializer : str or `Initializer`\n        Initializer for the `weight` weights matrix.\n    bias_initializer : str or `Initializer`\n        Initializer for the bias vector.\n\n\n    Inputs:\n        - **data**: 5D input tensor with shape\n          `(batch_size, in_channels, depth, height, width)` when `layout` is `NCDHW`.\n          For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 5D output tensor with shape\n          `(batch_size, channels, out_depth, out_height, out_width)` when `layout` is `NCDHW`.\n          out_depth, out_height and out_width are calculated as::\n\n            out_depth = (depth-1)*strides[0]-2*padding[0]+kernel_size[0]+output_padding[0]\n            out_height = (height-1)*strides[1]-2*padding[1]+kernel_size[1]+output_padding[1]\n            out_width = (width-1)*strides[2]-2*padding[2]+kernel_size[2]+output_padding[2]\n    \"\"\"\n    def __init__(self, channels, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0),\n                 output_padding=(0, 0, 0), dilation=(1, 1, 1), groups=1, layout='NCDHW',\n                 activation=None, use_bias=True, weight_initializer=None,\n                 bias_initializer='zeros', in_channels=0, **kwargs):\n        assert layout in ('NCDHW', 'NDHWC'), \"Only supports 'NCDHW' and 'NDHWC' layout for now\"\n        if isinstance(kernel_size, numeric_types):\n            kernel_size = (kernel_size,)*3\n        if isinstance(output_padding, numeric_types):\n            output_padding = (output_padding,)*3\n        assert len(kernel_size) == 3, \"kernel_size must be a number or a list of 3 ints\"\n        assert len(output_padding) == 3, \"output_padding must be a number or a list of 3 ints\"\n        op_name = 'deconvolution'\n        super(Conv3DTranspose, self).__init__(\n            channels, kernel_size, strides, padding, dilation, groups, layout,\n            in_channels, activation, use_bias, weight_initializer, bias_initializer,\n            op_name=op_name, adj=output_padding, **kwargs)\n        self.outpad = output_padding\n\n\n@use_np\nclass _Pooling(HybridBlock):\n    \"\"\"Abstract class for different pooling layers.\"\"\"\n    def __init__(self, pool_size, strides, padding, ceil_mode, global_pool,\n                 pool_type, layout, count_include_pad=None, **kwargs):\n        super(_Pooling, self).__init__(**kwargs)\n        if strides is None:\n            strides = pool_size\n        if isinstance(strides, numeric_types):\n            strides = (strides,)*len(pool_size)\n        if isinstance(padding, numeric_types):\n            padding = (padding,)*len(pool_size)\n        self._kwargs = {\n            'kernel': pool_size, 'stride': strides, 'pad': padding,\n            'global_pool': global_pool, 'pool_type': pool_type,\n            'layout': layout,\n            'pooling_convention': 'full' if ceil_mode else 'valid'}\n        if count_include_pad is not None:\n            self._kwargs['count_include_pad'] = count_include_pad\n\n    def _alias(self):\n        return 'pool'\n\n    def forward(self, x):\n        return npx.pooling(x, name='fwd', **self._kwargs)\n\n    def __repr__(self):\n        s = '{name}(size={kernel}, stride={stride}, padding={pad}, ceil_mode={ceil_mode}'\n        s += ', global_pool={global_pool}, pool_type={pool_type}, layout={layout})'\n        return s.format(name=self.__class__.__name__,\n                        ceil_mode=self._kwargs['pooling_convention'] == 'full',\n                        **self._kwargs)\n\n\nclass MaxPool1D(_Pooling):\n    \"\"\"Max pooling operation for one dimensional data.\n\n\n    Parameters\n    ----------\n    pool_size: int\n        Size of the max pooling windows.\n    strides: int, or None\n        Factor by which to downscale. E.g. 2 will halve the input size.\n        If `None`, it will default to `pool_size`.\n    padding: int\n        If padding is non-zero, then the input is implicitly\n        zero-padded on both sides for padding number of points.\n    layout : str, default 'NCW'\n        Dimension ordering of data and out ('NCW' or 'NWC').\n        'N', 'C', 'W' stands for batch, channel, and width (time) dimensions\n        respectively. Pooling is applied on the W dimension.\n    ceil_mode : bool, default False\n        When `True`, will use ceil instead of floor to compute the output shape.\n\n\n    Inputs:\n        - **data**: 3D input tensor with shape `(batch_size, in_channels, width)`\n          when `layout` is `NCW`. For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 3D output tensor with shape `(batch_size, channels, out_width)`\n          when `layout` is `NCW`. out_width is calculated as::\n\n              out_width = floor((width+2*padding-pool_size)/strides)+1\n\n          When `ceil_mode` is `True`, ceil will be used instead of floor in this\n          equation.\n    \"\"\"\n    def __init__(self, pool_size=2, strides=None, padding=0, layout='NCW',\n                 ceil_mode=False, **kwargs):\n        assert layout in ('NCW', 'NWC'),\\\n            \"Only NCW and NWC layouts are valid for 1D Pooling\"\n        if isinstance(pool_size, numeric_types):\n            pool_size = (pool_size,)\n        assert len(pool_size) == 1, \"pool_size must be a number or a list of 1 ints\"\n        super(MaxPool1D, self).__init__(\n            pool_size, strides, padding, ceil_mode, False, 'max', layout, **kwargs)\n\n\nclass MaxPool2D(_Pooling):\n    \"\"\"Max pooling operation for two dimensional (spatial) data.\n\n\n    Parameters\n    ----------\n    pool_size: int or list/tuple of 2 ints,\n        Size of the max pooling windows.\n    strides: int, list/tuple of 2 ints, or None.\n        Factor by which to downscale. E.g. 2 will halve the input size.\n        If `None`, it will default to `pool_size`.\n    padding: int or list/tuple of 2 ints,\n        If padding is non-zero, then the input is implicitly\n        zero-padded on both sides for padding number of points.\n    layout : str, default 'NCHW'\n        Dimension ordering of data and out ('NCHW' or 'NHWC').\n        'N', 'C', 'H', 'W' stands for batch, channel, height, and width\n        dimensions respectively. padding is applied on 'H' and 'W' dimension.\n    ceil_mode : bool, default False\n        When `True`, will use ceil instead of floor to compute the output shape.\n\n\n    Inputs:\n        - **data**: 4D input tensor with shape\n          `(batch_size, in_channels, height, width)` when `layout` is `NCHW`.\n          For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 4D output tensor with shape\n          `(batch_size, channels, out_height, out_width)` when `layout` is `NCHW`.\n          out_height and out_width are calculated as::\n\n              out_height = floor((height+2*padding[0]-pool_size[0])/strides[0])+1\n              out_width = floor((width+2*padding[1]-pool_size[1])/strides[1])+1\n\n          When `ceil_mode` is `True`, ceil will be used instead of floor in this\n          equation.\n    \"\"\"\n    def __init__(self, pool_size=(2, 2), strides=None, padding=0, layout='NCHW',\n                 ceil_mode=False, **kwargs):\n        assert layout in ('NCHW', 'NHWC'),\\\n            \"Only NCHW and NHWC layouts are valid for 2D Pooling\"\n        if isinstance(pool_size, numeric_types):\n            pool_size = (pool_size,)*2\n        assert len(pool_size) == 2, \"pool_size must be a number or a list of 2 ints\"\n        super(MaxPool2D, self).__init__(\n            pool_size, strides, padding, ceil_mode, False, 'max', layout, **kwargs)\n\n\nclass MaxPool3D(_Pooling):\n    \"\"\"Max pooling operation for 3D data (spatial or spatio-temporal).\n\n\n    Parameters\n    ----------\n    pool_size: int or list/tuple of 3 ints,\n        Size of the max pooling windows.\n    strides: int, list/tuple of 3 ints, or None.\n        Factor by which to downscale. E.g. 2 will halve the input size.\n        If `None`, it will default to `pool_size`.\n    padding: int or list/tuple of 3 ints,\n        If padding is non-zero, then the input is implicitly\n        zero-padded on both sides for padding number of points.\n    layout : str, default 'NCDHW'\n        Dimension ordering of data and out ('NCDHW' or 'NDHWC').\n        'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and\n        depth dimensions respectively. padding is applied on 'D', 'H' and 'W'\n        dimension.\n    ceil_mode : bool, default False\n        When `True`, will use ceil instead of floor to compute the output shape.\n\n\n    Inputs:\n        - **data**: 5D input tensor with shape\n          `(batch_size, in_channels, depth, height, width)` when `layout` is `NCW`.\n          For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 5D output tensor with shape\n          `(batch_size, channels, out_depth, out_height, out_width)` when `layout` is `NCDHW`.\n          out_depth, out_height and out_width are calculated as::\n\n              out_depth = floor((depth+2*padding[0]-pool_size[0])/strides[0])+1\n              out_height = floor((height+2*padding[1]-pool_size[1])/strides[1])+1\n              out_width = floor((width+2*padding[2]-pool_size[2])/strides[2])+1\n\n          When `ceil_mode` is `True`, ceil will be used instead of floor in this\n          equation.\n    \"\"\"\n    def __init__(self, pool_size=(2, 2, 2), strides=None, padding=0,\n                 ceil_mode=False, layout='NCDHW', **kwargs):\n        assert layout in ('NCDHW', 'NDHWC'),\\\n            \"Only NCDHW and NDHWC layouts are valid for 3D Pooling\"\n        if isinstance(pool_size, numeric_types):\n            pool_size = (pool_size,)*3\n        assert len(pool_size) == 3, \"pool_size must be a number or a list of 3 ints\"\n        super(MaxPool3D, self).__init__(\n            pool_size, strides, padding, ceil_mode, False, 'max', layout, **kwargs)\n\n\nclass AvgPool1D(_Pooling):\n    \"\"\"Average pooling operation for temporal data.\n\n    Parameters\n    ----------\n    pool_size: int\n        Size of the average pooling windows.\n    strides: int, or None\n        Factor by which to downscale. E.g. 2 will halve the input size.\n        If `None`, it will default to `pool_size`.\n    padding: int\n        If padding is non-zero, then the input is implicitly\n        zero-padded on both sides for padding number of points.\n    layout : str, default 'NCW'\n        Dimension ordering of data and out ('NCW' or 'NWC').\n        'N', 'C', 'W' stands for batch, channel, and width (time) dimensions\n        respectively. padding is applied on 'W' dimension.\n    ceil_mode : bool, default False\n        When `True`, will use ceil instead of floor to compute the output shape.\n    count_include_pad : bool, default True\n        When 'False', will exclude padding elements when computing the average value.\n\n\n    Inputs:\n        - **data**: 3D input tensor with shape `(batch_size, in_channels, width)`\n          when `layout` is `NCW`. For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 3D output tensor with shape `(batch_size, channels, out_width)`\n          when `layout` is `NCW`. out_width is calculated as::\n\n              out_width = floor((width+2*padding-pool_size)/strides)+1\n\n          When `ceil_mode` is `True`, ceil will be used instead of floor in this\n          equation.\n    \"\"\"\n    def __init__(self, pool_size=2, strides=None, padding=0, layout='NCW',\n                 ceil_mode=False, count_include_pad=True, **kwargs):\n        assert layout in ('NCW', 'NWC'),\\\n            \"Only NCW and NWC layouts are valid for 1D Pooling\"\n        if isinstance(pool_size, numeric_types):\n            pool_size = (pool_size,)\n        assert len(pool_size) == 1, \"pool_size must be a number or a list of 1 ints\"\n        super(AvgPool1D, self).__init__(\n            pool_size, strides, padding, ceil_mode, False, 'avg', layout, count_include_pad,\n            **kwargs)\n\n\nclass AvgPool2D(_Pooling):\n    \"\"\"Average pooling operation for spatial data.\n\n    Parameters\n    ----------\n    pool_size: int or list/tuple of 2 ints,\n        Size of the average pooling windows.\n    strides: int, list/tuple of 2 ints, or None.\n        Factor by which to downscale. E.g. 2 will halve the input size.\n        If `None`, it will default to `pool_size`.\n    padding: int or list/tuple of 2 ints,\n        If padding is non-zero, then the input is implicitly\n        zero-padded on both sides for padding number of points.\n    layout : str, default 'NCHW'\n        Dimension ordering of data and out ('NCHW' or 'NHWC').\n        'N', 'C', 'H', 'W' stands for batch, channel, height, and width\n        dimensions respectively. padding is applied on 'H' and 'W' dimension.\n    ceil_mode : bool, default False\n        When True, will use ceil instead of floor to compute the output shape.\n    count_include_pad : bool, default True\n        When 'False', will exclude padding elements when computing the average value.\n\n\n    Inputs:\n        - **data**: 4D input tensor with shape\n          `(batch_size, in_channels, height, width)` when `layout` is `NCHW`.\n          For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 4D output tensor with shape\n          `(batch_size, channels, out_height, out_width)` when `layout` is `NCHW`.\n          out_height and out_width are calculated as::\n\n              out_height = floor((height+2*padding[0]-pool_size[0])/strides[0])+1\n              out_width = floor((width+2*padding[1]-pool_size[1])/strides[1])+1\n\n          When `ceil_mode` is `True`, ceil will be used instead of floor in this\n          equation.\n    \"\"\"\n    def __init__(self, pool_size=(2, 2), strides=None, padding=0,\n                 ceil_mode=False, layout='NCHW', count_include_pad=True, **kwargs):\n        assert layout in ('NCHW', 'NHWC'),\\\n            \"Only NCHW and NHWC layouts are valid for 2D Pooling\"\n        if isinstance(pool_size, numeric_types):\n            pool_size = (pool_size,)*2\n        assert len(pool_size) == 2, \"pool_size must be a number or a list of 2 ints\"\n        super(AvgPool2D, self).__init__(\n            pool_size, strides, padding, ceil_mode, False, 'avg', layout, count_include_pad,\n            **kwargs)\n\n\nclass AvgPool3D(_Pooling):\n    \"\"\"Average pooling operation for 3D data (spatial or spatio-temporal).\n\n    Parameters\n    ----------\n    pool_size: int or list/tuple of 3 ints,\n        Size of the average pooling windows.\n    strides: int, list/tuple of 3 ints, or None.\n        Factor by which to downscale. E.g. 2 will halve the input size.\n        If `None`, it will default to `pool_size`.\n    padding: int or list/tuple of 3 ints,\n        If padding is non-zero, then the input is implicitly\n        zero-padded on both sides for padding number of points.\n    layout : str, default 'NCDHW'\n        Dimension ordering of data and out ('NCDHW' or 'NDHWC').\n        'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and\n        depth dimensions respectively. padding is applied on 'D', 'H' and 'W'\n        dimension.\n    ceil_mode : bool, default False\n        When True, will use ceil instead of floor to compute the output shape.\n    count_include_pad : bool, default True\n        When 'False', will exclude padding elements when computing the average value.\n\n\n    Inputs:\n        - **data**: 5D input tensor with shape\n          `(batch_size, in_channels, depth, height, width)` when `layout` is `NCDHW`.\n          For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 5D output tensor with shape\n          `(batch_size, channels, out_depth, out_height, out_width)` when `layout` is `NCDHW`.\n          out_depth, out_height and out_width are calculated as::\n\n              out_depth = floor((depth+2*padding[0]-pool_size[0])/strides[0])+1\n              out_height = floor((height+2*padding[1]-pool_size[1])/strides[1])+1\n              out_width = floor((width+2*padding[2]-pool_size[2])/strides[2])+1\n\n          When `ceil_mode` is `True,` ceil will be used instead of floor in this\n          equation.\n    \"\"\"\n    def __init__(self, pool_size=(2, 2, 2), strides=None, padding=0,\n                 ceil_mode=False, layout='NCDHW', count_include_pad=True, **kwargs):\n        assert layout in ('NCDHW', 'NDHWC'),\\\n            \"Only NCDHW and NDHWC layouts are valid for 3D Pooling\"\n        if isinstance(pool_size, numeric_types):\n            pool_size = (pool_size,)*3\n        assert len(pool_size) == 3, \"pool_size must be a number or a list of 3 ints\"\n        super(AvgPool3D, self).__init__(\n            pool_size, strides, padding, ceil_mode, False, 'avg', layout, count_include_pad,\n            **kwargs)\n\n\nclass GlobalMaxPool1D(_Pooling):\n    \"\"\"Gloabl max pooling operation for one dimensional (temporal) data.\n\n\n    Parameters\n    ----------\n    layout : str, default 'NCW'\n        Dimension ordering of data and out ('NCW' or 'NWC').\n        'N', 'C', 'W' stands for batch, channel, and width (time) dimensions\n        respectively. Pooling is applied on the W dimension.\n\n\n    Inputs:\n        - **data**: 3D input tensor with shape `(batch_size, in_channels, width)`\n          when `layout` is `NCW`. For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 3D output tensor with shape `(batch_size, channels, 1)`\n          when `layout` is `NCW`.\n    \"\"\"\n    def __init__(self, layout='NCW', **kwargs):\n        assert layout in ('NCW', 'NWC'),\\\n            \"Only NCW and NWC layouts are valid for 1D Pooling\"\n        super(GlobalMaxPool1D, self).__init__(\n            (1,), None, 0, True, True, 'max', layout, **kwargs)\n\n\nclass GlobalMaxPool2D(_Pooling):\n    \"\"\"Global max pooling operation for two dimensional (spatial) data.\n\n\n    Parameters\n    ----------\n    layout : str, default 'NCHW'\n        Dimension ordering of data and out ('NCHW' or 'NHWC').\n        'N', 'C', 'H', 'W' stands for batch, channel, height, and width\n        dimensions respectively. padding is applied on 'H' and 'W' dimension.\n\n\n    Inputs:\n        - **data**: 4D input tensor with shape\n          `(batch_size, in_channels, height, width)` when `layout` is `NCHW`.\n          For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 4D output tensor with shape\n          `(batch_size, channels, 1, 1)` when `layout` is `NCHW`.\n    \"\"\"\n    def __init__(self, layout='NCHW', **kwargs):\n        assert layout in ('NCHW', 'NHWC'),\\\n            \"Only NCHW and NHWC layouts are valid for 2D Pooling\"\n        super(GlobalMaxPool2D, self).__init__(\n            (1, 1), None, 0, True, True, 'max', layout, **kwargs)\n\n\nclass GlobalMaxPool3D(_Pooling):\n    \"\"\"Global max pooling operation for 3D data (spatial or spatio-temporal).\n\n\n    Parameters\n    ----------\n    layout : str, default 'NCDHW'\n        Dimension ordering of data and out ('NCDHW' or 'NDHWC').\n        'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and\n        depth dimensions respectively. padding is applied on 'D', 'H' and 'W'\n        dimension.\n\n\n    Inputs:\n        - **data**: 5D input tensor with shape\n          `(batch_size, in_channels, depth, height, width)` when `layout` is `NCW`.\n          For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 5D output tensor with shape\n          `(batch_size, channels, 1, 1, 1)` when `layout` is `NCDHW`.\n    \"\"\"\n    def __init__(self, layout='NCDHW', **kwargs):\n        assert layout in ('NCDHW', 'NDHWC'),\\\n            \"Only NCDHW and NDHWC layouts are valid for 3D Pooling\"\n        super(GlobalMaxPool3D, self).__init__(\n            (1, 1, 1), None, 0, True, True, 'max', layout, **kwargs)\n\n\nclass GlobalAvgPool1D(_Pooling):\n    \"\"\"Global average pooling operation for temporal data.\n\n    Parameters\n    ----------\n    layout : str, default 'NCW'\n        Dimension ordering of data and out ('NCW' or 'NWC').\n        'N', 'C', 'W' stands for batch, channel, and width (time) dimensions\n        respectively. padding is applied on 'W' dimension.\n\n\n    Inputs:\n        - **data**: 3D input tensor with shape `(batch_size, in_channels, width)`\n          when `layout` is `NCW`. For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 3D output tensor with shape `(batch_size, channels, 1)`.\n    \"\"\"\n    def __init__(self, layout='NCW', **kwargs):\n        assert layout in ('NCW', 'NWC'),\\\n            \"Only NCW and NWC layouts are valid for 1D Pooling\"\n        super(GlobalAvgPool1D, self).__init__(\n            (1,), None, 0, True, True, 'avg', layout, **kwargs)\n\n\nclass GlobalAvgPool2D(_Pooling):\n    \"\"\"Global average pooling operation for spatial data.\n\n    Parameters\n    ----------\n    layout : str, default 'NCHW'\n        Dimension ordering of data and out ('NCHW' or 'NHWC').\n        'N', 'C', 'H', 'W' stands for batch, channel, height, and width\n        dimensions respectively.\n\n\n    Inputs:\n        - **data**: 4D input tensor with shape\n          `(batch_size, in_channels, height, width)` when `layout` is `NCHW`.\n          For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 4D output tensor with shape\n          `(batch_size, channels, 1, 1)` when `layout` is `NCHW`.\n    \"\"\"\n    def __init__(self, layout='NCHW', **kwargs):\n        assert layout in ('NCHW', 'NHWC'),\\\n            \"Only NCHW and NHWC layouts are valid for 2D Pooling\"\n        super(GlobalAvgPool2D, self).__init__(\n            (1, 1), None, 0, True, True, 'avg', layout, **kwargs)\n\n\nclass GlobalAvgPool3D(_Pooling):\n    \"\"\"Global average pooling operation for 3D data (spatial or spatio-temporal).\n\n    Parameters\n    ----------\n    layout : str, default 'NCDHW'\n        Dimension ordering of data and out ('NCDHW' or 'NDHWC').\n        'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and\n        depth dimensions respectively. padding is applied on 'D', 'H' and 'W'\n        dimension.\n\n\n    Inputs:\n        - **data**: 5D input tensor with shape\n          `(batch_size, in_channels, depth, height, width)` when `layout` is `NCDHW`.\n          For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 5D output tensor with shape\n          `(batch_size, channels, 1, 1, 1)` when `layout` is `NCDHW`.\n    \"\"\"\n    def __init__(self, layout='NCDHW', **kwargs):\n        assert layout in ('NCDHW', 'NDHWC'),\\\n            \"Only NCDHW and NDHWC layouts are valid for 3D Pooling\"\n        super(GlobalAvgPool3D, self).__init__(\n            (1, 1, 1), None, 0, True, True, 'avg', layout, **kwargs)\n\n\n@use_np\nclass ReflectionPad2D(HybridBlock):\n    r\"\"\"Pads the input tensor using the reflection of the input boundary.\n\n    Parameters\n    ----------\n    padding: int\n        An integer padding size\n\n\n    Inputs:\n        - **data**: input tensor with the shape :math:`(N, C, H_{in}, W_{in})`.\n\n    Outputs:\n        - **out**: output tensor with the shape :math:`(N, C, H_{out}, W_{out})`, where\n\n          .. math::\n\n            H_{out} = H_{in} + 2 \\cdot padding\n\n            W_{out} = W_{in} + 2 \\cdot padding\n\n\n    Examples\n    --------\n    >>> m = nn.ReflectionPad2D(3)\n    >>> input = mx.np.random.normal(size=(16, 3, 224, 224))\n    >>> output = m(input)\n    \"\"\"\n    def __init__(self, padding=0, **kwargs):\n        super(ReflectionPad2D, self).__init__(**kwargs)\n        if isinstance(padding, numeric_types):\n            padding = (0, 0, 0, 0, padding, padding, padding, padding)\n        assert(len(padding) == 8)\n        self._padding = padding\n\n    def forward(self, x):\n        \"\"\"\n        Use pad operator in numpy extension module,\n        which has backward support for reflect mode\n        \"\"\"\n        return npx.pad(x, mode='reflect', pad_width=self._padding)\n\n\n@use_np\nclass DeformableConvolution(HybridBlock):\n    \"\"\"2-D Deformable Convolution v_1 (Dai, 2017).\n    Normal Convolution uses sampling points in a regular grid, while the sampling\n    points of Deformablem Convolution can be offset. The offset is learned with a\n    separate convolution layer during the training. Both the convolution layer for\n    generating the output features and the offsets are included in this gluon layer.\n\n    Parameters\n    ----------\n    channels : int,\n        The dimensionality of the output space\n        i.e. the number of output channels in the convolution.\n    kernel_size : int or tuple/list of 2 ints, (Default value = (1,1))\n        Specifies the dimensions of the convolution window.\n    strides : int or tuple/list of 2 ints, (Default value = (1,1))\n        Specifies the strides of the convolution.\n    padding : int or tuple/list of 2 ints, (Default value = (0,0))\n        If padding is non-zero, then the input is implicitly zero-padded\n        on both sides for padding number of points.\n    dilation : int or tuple/list of 2 ints, (Default value = (1,1))\n        Specifies the dilation rate to use for dilated convolution.\n    groups : int, (Default value = 1)\n        Controls the connections between inputs and outputs.\n        At groups=1, all inputs are convolved to all outputs.\n        At groups=2, the operation becomes equivalent to having two convolution\n        layers side by side, each seeing half the input channels, and producing\n        half the output channels, and both subsequently concatenated.\n    num_deformable_group : int, (Default value = 1)\n        Number of deformable group partitions.\n    layout : str, (Default value = NCHW)\n        Dimension ordering of data and weight. Can be 'NCW', 'NWC', 'NCHW',\n        'NHWC', 'NCDHW', 'NDHWC', etc. 'N', 'C', 'H', 'W', 'D' stands for\n        batch, channel, height, width and depth dimensions respectively.\n        Convolution is performed over 'D', 'H', and 'W' dimensions.\n    use_bias : bool, (Default value = True)\n        Whether the layer for generating the output features uses a bias vector.\n    in_channels : int, (Default value = 0)\n        The number of input channels to this layer. If not specified,\n        initialization will be deferred to the first time `forward` is called\n        and input channels will be inferred from the shape of input data.\n    activation : str, (Default value = None)\n        Activation function to use. See :func:`~mxnet.npx.activation`.\n        If you don't specify anything, no activation is applied\n        (ie. \"linear\" activation: `a(x) = x`).\n    weight_initializer : str or `Initializer`, (Default value = None)\n        Initializer for the `weight` weights matrix for the convolution layer\n        for generating the output features.\n    bias_initializer : str or `Initializer`, (Default value = zeros)\n        Initializer for the bias vector for the convolution layer\n        for generating the output features.\n    offset_weight_initializer : str or `Initializer`, (Default value = zeros)\n        Initializer for the `weight` weights matrix for the convolution layer\n        for generating the offset.\n    offset_bias_initializer : str or `Initializer`, (Default value = zeros),\n        Initializer for the bias vector for the convolution layer\n        for generating the offset.\n    offset_use_bias: bool, (Default value = True)\n        Whether the layer for generating the offset uses a bias vector.\n\n    Inputs:\n        - **data**: 4D input tensor with shape\n          `(batch_size, in_channels, height, width)` when `layout` is `NCHW`.\n          For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 4D output tensor with shape\n          `(batch_size, channels, out_height, out_width)` when `layout` is `NCHW`.\n          out_height and out_width are calculated as::\n\n              out_height = floor((height+2*padding[0]-dilation[0]*(kernel_size[0]-1)-1)/stride[0])+1\n              out_width = floor((width+2*padding[1]-dilation[1]*(kernel_size[1]-1)-1)/stride[1])+1\n    \"\"\"\n\n    def __init__(self, channels, kernel_size=(1, 1), strides=(1, 1), padding=(0, 0), dilation=(1, 1), groups=1,\n                 num_deformable_group=1, layout='NCHW', use_bias=True, in_channels=0, activation=None,\n                 weight_initializer=None, bias_initializer='zeros',\n                 offset_weight_initializer='zeros', offset_bias_initializer='zeros', offset_use_bias=True,\n                 op_name='DeformableConvolution', adj=None):\n        super(DeformableConvolution, self).__init__()\n        self._channels = channels\n        self._in_channels = in_channels\n\n        assert layout in ('NCHW', 'NHWC'), \"Only supports 'NCHW' and 'NHWC' layout for now\"\n        if isinstance(kernel_size, numeric_types):\n            kernel_size = (kernel_size,) * 2\n        if isinstance(strides, numeric_types):\n            strides = (strides,) * len(kernel_size)\n        if isinstance(padding, numeric_types):\n            padding = (padding,) * len(kernel_size)\n        if isinstance(dilation, numeric_types):\n            dilation = (dilation,) * len(kernel_size)\n        self._op_name = op_name\n        self._kernel_size = kernel_size\n        self._layout = layout\n        self._groups = groups\n\n        offset_channels = 2 * kernel_size[0] * kernel_size[1] * num_deformable_group\n        self._offset_channels = offset_channels\n        self._kwargs_offset = {\n            'kernel': kernel_size, 'stride': strides, 'dilate': dilation,\n            'pad': padding, 'num_filter': offset_channels, 'num_group': groups,\n            'no_bias': not offset_use_bias, 'layout': layout}\n\n        self._kwargs_deformable_conv = {\n            'kernel': kernel_size, 'stride': strides, 'dilate': dilation,\n            'pad': padding, 'num_filter': channels, 'num_group': groups,\n            'num_deformable_group': num_deformable_group,\n            'no_bias': not use_bias, 'layout': layout}\n\n        if adj:\n            self._kwargs_offset['adj'] = adj\n            self._kwargs_deformable_conv['adj'] = adj\n\n        self.offset_weight = Parameter('offset_weight', shape=self.pre_infer_offset_weight(),\n                                       init=offset_weight_initializer,\n                                       allow_deferred_init=True)\n\n        if offset_use_bias:\n            self.offset_bias = Parameter('offset_bias', shape=(offset_channels,),\n                                         init=offset_bias_initializer,\n                                         allow_deferred_init=True)\n        else:\n            self.offset_bias = None\n\n        self.deformable_conv_weight = Parameter('deformable_conv_weight',\n                                                shape=self.pre_infer_weight(),\n                                                init=weight_initializer,\n                                                allow_deferred_init=True)\n\n        if use_bias:\n            self.deformable_conv_bias = Parameter('deformable_conv_bias', shape=(channels,),\n                                                  init=bias_initializer,\n                                                  allow_deferred_init=True)\n        else:\n            self.deformable_conv_bias = None\n\n        if activation:\n            self.act = Activation(activation)\n        else:\n            self.act = None\n\n    def forward(self, x):\n        device = x.device\n        if self.offset_bias is None:\n            offset = npx.convolution(x, self.offset_weight.data(device), cudnn_off=True, **self._kwargs_offset)\n        else:\n            offset = npx.convolution(x, self.offset_weight.data(device), self.offset_bias.data(device),\n                                     cudnn_off=True, **self._kwargs_offset)\n\n        if self.deformable_conv_bias is None:\n            act = npx.deformable_convolution(data=x, offset=offset,\n                                             weight=self.deformable_conv_weight.data(device),\n                                             name='fwd', **self._kwargs_deformable_conv)\n        else:\n            act = npx.deformable_convolution(data=x, offset=offset,\n                                             weight=self.deformable_conv_weight.data(device),\n                                             bias=self.deformable_conv_bias.data(device), name='fwd',\n                                             **self._kwargs_deformable_conv)\n\n        if self.act:\n            act = self.act(act)\n        return act\n\n\n    def pre_infer_offset_weight(self):\n        \"\"\"\n        Pre-infer the shape of offsite weight parameter based on kernel size,\n        group size and offset channels\n        \"\"\"\n        wshape = [-1]*(len(self._kernel_size) + 2)\n        wshape[self._layout.find('N')] = self._offset_channels // self._groups\n        wshape[self._layout.find('H')] = self._kernel_size[0]\n        wshape[self._layout.find('W')] = self._kernel_size[1]\n        wshape[0] *= self._groups\n        return tuple(wshape)\n\n    def pre_infer_weight(self):\n        \"\"\"\n        Pre-infer the shape of weight parameter based on kernel size, group size and channels\n        \"\"\"\n        wshape = [-1]*(len(self._kernel_size) + 2)\n        wshape[self._layout.find('N')] = self._channels // self._groups\n        wshape[self._layout.find('H')] = self._kernel_size[0]\n        wshape[self._layout.find('W')] = self._kernel_size[1]\n        wshape[0] *= self._groups\n        return tuple(wshape)\n\n    def infer_shape(self, x):\n        dshape1 = x.shape[self._layout.find('C')]\n        wshape = self.deformable_conv_weight.shape\n        wshape_offset = self.offset_weight.shape\n        wshape_list = list(wshape)\n        wshape_offset_list = list(wshape_offset)\n        wshape_list[self._layout.find('C')] = dshape1 // self._groups\n        wshape_offset_list[self._layout.find('C')] = dshape1 // self._groups\n        self.deformable_conv_weight.shape = tuple(wshape_list)\n        self.offset_weight.shape = tuple(wshape_offset_list)\n\n    def _alias(self):\n        return 'deformable_conv'\n\n    def __repr__(self):\n        s = '{name}({mapping}, kernel_size={kernel}, stride={stride}'\n        len_kernel_size = len(self._kwargs_deformable_conv['kernel'])\n        if self._kwargs_deformable_conv['pad'] != (0,) * len_kernel_size:\n            s += ', padding={pad}'\n        if self._kwargs_deformable_conv['dilate'] != (1,) * len_kernel_size:\n            s += ', dilation={dilate}'\n        if hasattr(self, 'out_pad') and self.out_pad != (0,) * len_kernel_size:\n            s += ', output_padding={out_pad}'.format(out_pad=self.out_pad)\n        if self._kwargs_deformable_conv['num_group'] != 1:\n            s += ', groups={num_group}'\n        if self.deformable_conv_bias is None:\n            s += ', bias=False'\n        if self.act:\n            s += ', {}'.format(self.act)\n        s += ')'\n        shape = self.deformable_conv_weight.shape\n        return s.format(name=self.__class__.__name__,\n                        mapping='{0} -> {1}'.format(shape[1] if shape[1] else None, shape[0]),\n                        **self._kwargs_deformable_conv)\n\n\n@use_np\nclass ModulatedDeformableConvolution(HybridBlock):\n    \"\"\"2-D Deformable Convolution v2 (Dai, 2018).\n\n    The modulated deformable convolution operation is described in https://arxiv.org/abs/1811.11168\n\n    Parameters\n    ----------\n    channels : int,\n        The dimensionality of the output space\n        i.e. the number of output channels in the convolution.\n    kernel_size : int or tuple/list of 2 ints, (Default value = (1,1))\n        Specifies the dimensions of the convolution window.\n    strides : int or tuple/list of 2 ints, (Default value = (1,1))\n        Specifies the strides of the convolution.\n    padding : int or tuple/list of 2 ints, (Default value = (0,0))\n        If padding is non-zero, then the input is implicitly zero-padded\n        on both sides for padding number of points.\n    dilation : int or tuple/list of 2 ints, (Default value = (1,1))\n        Specifies the dilation rate to use for dilated convolution.\n    groups : int, (Default value = 1)\n        Controls the connections between inputs and outputs.\n        At groups=1, all inputs are convolved to all outputs.\n        At groups=2, the operation becomes equivalent to having two convolution\n        layers side by side, each seeing half the input channels, and producing\n        half the output channels, and both subsequently concatenated.\n    num_deformable_group : int, (Default value = 1)\n        Number of deformable group partitions.\n    layout : str, (Default value = NCHW)\n        Dimension ordering of data and weight. Can be 'NCW', 'NWC', 'NCHW',\n        'NHWC', 'NCDHW', 'NDHWC', etc. 'N', 'C', 'H', 'W', 'D' stands for\n        batch, channel, height, width and depth dimensions respectively.\n        Convolution is performed over 'D', 'H', and 'W' dimensions.\n    use_bias : bool, (Default value = True)\n        Whether the layer for generating the output features uses a bias vector.\n    in_channels : int, (Default value = 0)\n        The number of input channels to this layer. If not specified,\n        initialization will be deferred to the first time `forward` is called\n        and input channels will be inferred from the shape of input data.\n    activation : str, (Default value = None)\n        Activation function to use. See :func:`~mxnet.ndarray.Activation`.\n        If you don't specify anything, no activation is applied\n        (ie. \"linear\" activation: `a(x) = x`).\n    weight_initializer : str or `Initializer`, (Default value = None)\n        Initializer for the `weight` weights matrix for the convolution layer\n        for generating the output features.\n    bias_initializer : str or `Initializer`, (Default value = zeros)\n        Initializer for the bias vector for the convolution layer\n        for generating the output features.\n    offset_weight_initializer : str or `Initializer`, (Default value = zeros)\n        Initializer for the `weight` weights matrix for the convolution layer\n        for generating the offset.\n    offset_bias_initializer : str or `Initializer`, (Default value = zeros),\n        Initializer for the bias vector for the convolution layer\n        for generating the offset.\n    offset_use_bias: bool, (Default value = True)\n        Whether the layer for generating the offset uses a bias vector.\n\n    Inputs:\n        - **data**: 4D input tensor with shape\n          `(batch_size, in_channels, height, width)` when `layout` is `NCHW`.\n          For other layouts shape is permuted accordingly.\n\n    Outputs:\n        - **out**: 4D output tensor with shape\n          `(batch_size, channels, out_height, out_width)` when `layout` is `NCHW`.\n          out_height and out_width are calculated as::\n\n              out_height = floor((height+2*padding[0]-dilation[0]*(kernel_size[0]-1)-1)/stride[0])+1\n              out_width = floor((width+2*padding[1]-dilation[1]*(kernel_size[1]-1)-1)/stride[1])+1\n    \"\"\"\n\n    def __init__(self, channels, kernel_size=(1, 1), strides=(1, 1), padding=(0, 0), dilation=(1, 1), groups=1,\n                 num_deformable_group=1, layout='NCHW', use_bias=True, in_channels=0, activation=None,\n                 weight_initializer=None, bias_initializer='zeros',\n                 offset_weight_initializer='zeros', offset_bias_initializer='zeros', offset_use_bias=True,\n                 op_name='ModulatedDeformableConvolution', adj=None):\n        super(ModulatedDeformableConvolution, self).__init__()\n        self._channels = channels\n        self._in_channels = in_channels\n\n        assert layout in ('NCHW', 'NHWC'), \"Only supports 'NCHW' and 'NHWC' layout for now\"\n        if isinstance(kernel_size, numeric_types):\n            kernel_size = (kernel_size,) * 2\n        if isinstance(strides, numeric_types):\n            strides = (strides,) * len(kernel_size)\n        if isinstance(padding, numeric_types):\n            padding = (padding,) * len(kernel_size)\n        if isinstance(dilation, numeric_types):\n            dilation = (dilation,) * len(kernel_size)\n        self._op_name = op_name\n\n        offset_channels = num_deformable_group * 3 * kernel_size[0] * kernel_size[1]\n        self.offset_split_index = num_deformable_group * 2 * kernel_size[0] * kernel_size[1]\n        self._layout = layout\n        self._groups = groups\n        self._offset_channels = offset_channels\n        self._kernel_size = kernel_size\n        self._kwargs_offset = {\n            'kernel': kernel_size, 'stride': strides, 'dilate': dilation,\n            'pad': padding, 'num_filter': offset_channels, 'num_group': groups,\n            'no_bias': not offset_use_bias, 'layout': layout}\n\n        self._kwargs_deformable_conv = {\n            'kernel': kernel_size, 'stride': strides, 'dilate': dilation,\n            'pad': padding, 'num_filter': channels, 'num_group': groups,\n            'num_deformable_group': num_deformable_group,\n            'no_bias': not use_bias, 'layout': layout}\n\n        if adj:\n            self._kwargs_offset['adj'] = adj\n            self._kwargs_deformable_conv['adj'] = adj\n\n        self.deformable_conv_weight = Parameter('deformable_conv_weight',\n                                                shape=self.pre_infer_weight(),\n                                                init=weight_initializer,\n                                                allow_deferred_init=True)\n\n        if use_bias:\n            self.deformable_conv_bias = Parameter('deformable_conv_bias', shape=(channels,),\n                                                  init=bias_initializer,\n                                                  allow_deferred_init=True)\n        else:\n            self.deformable_conv_bias = None\n\n        self.offset_weight = Parameter('offset_weight', shape=self.pre_infer_offset_weight(),\n                                       init=offset_weight_initializer,\n                                       allow_deferred_init=True)\n\n        if offset_use_bias:\n            self.offset_bias = Parameter('offset_bias', shape=(offset_channels,),\n                                         init=offset_bias_initializer,\n                                         allow_deferred_init=True)\n        else:\n            self.offset_bias = None\n\n        if activation:\n            self.act = Activation(activation)\n        else:\n            self.act = None\n\n    def forward(self, x):\n        device = x.device\n        if self.offset_bias is None:\n            offset = npx.convolution(x, self.offset_weight.data(device),\n                                     cudnn_off=True, **self._kwargs_offset)\n        else:\n            offset = npx.convolution(x, self.offset_weight.data(device),\n                                     self.offset_bias.data(device), cudnn_off=True, **self._kwargs_offset)\n\n        offset_t = npx.slice_axis(offset, axis=1, begin=0, end=self.offset_split_index)\n        mask = npx.slice_axis(offset, axis=1, begin=self.offset_split_index, end=None)\n        mask = npx.sigmoid(mask) * 2\n\n        if self.deformable_conv_bias is None:\n            act = npx.modulated_deformable_convolution(data=x, offset=offset_t, mask=mask,\n                                                       weight=self.deformable_conv_weight.data(device),\n                                                       name='fwd', **self._kwargs_deformable_conv)\n        else:\n            act = npx.modulated_deformable_convolution(data=x, offset=offset_t, mask=mask,\n                                                       weight=self.deformable_conv_weight.data(device),\n                                                       bias=self.deformable_conv_bias.data(device), name='fwd',\n                                                       **self._kwargs_deformable_conv)\n\n        if self.act:\n            act = self.act(act)\n        return act\n\n    def pre_infer_offset_weight(self):\n        \"\"\"\n        Pre-infer the shape of offsite weight parameter based on kernel size,\n        group size and offset channels\n        \"\"\"\n        wshape = [-1]*(len(self._kernel_size) + 2)\n        wshape[self._layout.find('N')] = self._offset_channels // self._groups\n        wshape[self._layout.find('H')] = self._kernel_size[0]\n        wshape[self._layout.find('W')] = self._kernel_size[1]\n        wshape[0] *= self._groups\n        return tuple(wshape)\n\n    def pre_infer_weight(self):\n        \"\"\"\n        Pre-infer the shape of weight parameter based on kernel size, group size and channels\n        \"\"\"\n        wshape = [-1]*(len(self._kernel_size) + 2)\n        wshape[self._layout.find('N')] = self._channels // self._groups\n        wshape[self._layout.find('H')] = self._kernel_size[0]\n        wshape[self._layout.find('W')] = self._kernel_size[1]\n        wshape[0] *= self._groups\n        return tuple(wshape)\n\n    def infer_shape(self, x):\n        dshape1 = x.shape[self._layout.find('C')]\n        wshape = self.deformable_conv_weight.shape\n        wshape_offset = self.offset_weight.shape\n        wshape_list = list(wshape)\n        wshape_offset_list = list(wshape_offset)\n        wshape_list[self._layout.find('C')] = dshape1 // self._groups\n        wshape_offset_list[self._layout.find('C')] = dshape1 // self._groups\n        self.deformable_conv_weight.shape = tuple(wshape_list)\n        self.offset_weight.shape = tuple(wshape_offset_list)\n\n    def _alias(self):\n        return 'modulated_deformable_conv'\n\n\n@use_np\nclass PixelShuffle1D(HybridBlock):\n\n    r\"\"\"Pixel-shuffle layer for upsampling in 1 dimension.\n\n    Pixel-shuffling is the operation of taking groups of values along\n    the *channel* dimension and regrouping them into blocks of pixels\n    along the ``W`` dimension, thereby effectively multiplying that dimension\n    by a constant factor in size.\n\n    For example, a feature map of shape :math:`(fC, W)` is reshaped\n    into :math:`(C, fW)` by forming little value groups of size :math:`f`\n    and arranging them in a grid of size :math:`W`.\n\n    Parameters\n    ----------\n    factor : int or 1-tuple of int\n        Upsampling factor, applied to the ``W`` dimension.\n\n    Inputs:\n        - **data**: Tensor of shape ``(N, f*C, W)``.\n    Outputs:\n        - **out**: Tensor of shape ``(N, C, W*f)``.\n\n    Examples\n    --------\n    >>> pxshuf = PixelShuffle1D(2)\n    >>> x = mx.np.zeros((1, 8, 3))\n    >>> pxshuf(x).shape\n    (1, 4, 6)\n    \"\"\"\n\n    def __init__(self, factor):\n        super(PixelShuffle1D, self).__init__()\n        self._factor = int(factor)\n\n    def forward(self, x):\n        \"\"\"Perform pixel-shuffling on the input.\"\"\"\n        f = self._factor                                             # (N, C*f, W)\n        x = npx.reshape(x, (-2, -6, -1, f, -2))  # (N, C, f, W)\n        x = np.transpose(x, (0, 1, 3, 2))     # (N, C, W, f)\n        x = npx.reshape(x, (-2, -2, -5))         # (N, C, W*f)\n        return x\n\n    def __repr__(self):\n        return \"{}({})\".format(self.__class__.__name__, self._factor)\n\n\n@use_np\nclass PixelShuffle2D(HybridBlock):\n\n    r\"\"\"Pixel-shuffle layer for upsampling in 2 dimensions.\n\n    Pixel-shuffling is the operation of taking groups of values along\n    the *channel* dimension and regrouping them into blocks of pixels\n    along the ``H`` and ``W`` dimensions, thereby effectively multiplying\n    those dimensions by a constant factor in size.\n\n    For example, a feature map of shape :math:`(f^2 C, H, W)` is reshaped\n    into :math:`(C, fH, fW)` by forming little :math:`f \\times f` blocks\n    of pixels and arranging them in an :math:`H \\times W` grid.\n\n    Pixel-shuffling together with regular convolution is an alternative,\n    learnable way of upsampling an image by arbitrary factors. It is reported\n    to help overcome checkerboard artifacts that are common in upsampling with\n    transposed convolutions (also called deconvolutions). See the paper\n    `Real-Time Single Image and Video Super-Resolution Using an Efficient\n    Sub-Pixel Convolutional Neural Network <https://arxiv.org/abs/1609.05158>`_\n    for further details.\n\n    Parameters\n    ----------\n    factor : int or 2-tuple of int\n        Upsampling factors, applied to the ``H`` and ``W`` dimensions,\n        in that order.\n\n    Inputs:\n        - **data**: Tensor of shape ``(N, f1*f2*C, H, W)``.\n    Outputs:\n        - **out**: Tensor of shape ``(N, C, H*f1, W*f2)``.\n\n    Examples\n    --------\n    >>> pxshuf = PixelShuffle2D((2, 3))\n    >>> x = mx.np.zeros((1, 12, 3, 5))\n    >>> pxshuf(x).shape\n    (1, 2, 6, 15)\n    \"\"\"\n\n    def __init__(self, factor):\n        super(PixelShuffle2D, self).__init__()\n        try:\n            self._factors = (int(factor),) * 2\n        except TypeError:\n            self._factors = tuple(int(fac) for fac in factor)\n            assert len(self._factors) == 2, \"wrong length {}\".format(len(self._factors))\n\n    def forward(self, x):\n        \"\"\"Perform pixel-shuffling on the input.\"\"\"\n        f1, f2 = self._factors\n                                                      # (N, f1*f2*C, H, W)\n        x = npx.reshape(x, (-2, -6, -1, f1 * f2, -2, -2))  # (N, C, f1*f2, H, W)\n        x = npx.reshape(x, (-2, -2, -6, f1, f2, -2, -2))    # (N, C, f1, f2, H, W)\n        x = np.transpose(x, (0, 1, 4, 2, 5, 3))        # (N, C, H, f1, W, f2)\n        x = npx.reshape(x, (-2, -2, -5, -5))              # (N, C, H*f1, W*f2)\n        return x\n\n    def __repr__(self):\n        return \"{}({})\".format(self.__class__.__name__, self._factors)\n\n\n@use_np\nclass PixelShuffle3D(HybridBlock):\n\n    r\"\"\"Pixel-shuffle layer for upsampling in 3 dimensions.\n\n    Pixel-shuffling (or voxel-shuffling in 3D) is the operation of taking\n    groups of values along the *channel* dimension and regrouping them into\n    blocks of voxels along the ``D``, ``H`` and ``W`` dimensions, thereby\n    effectively multiplying those dimensions by a constant factor in size.\n\n    For example, a feature map of shape :math:`(f^3 C, D, H, W)` is reshaped\n    into :math:`(C, fD, fH, fW)` by forming little :math:`f \\times f \\times f`\n    blocks of voxels and arranging them in a :math:`D \\times H \\times W` grid.\n\n    Pixel-shuffling together with regular convolution is an alternative,\n    learnable way of upsampling an image by arbitrary factors. It is reported\n    to help overcome checkerboard artifacts that are common in upsampling with\n    transposed convolutions (also called deconvolutions). See the paper\n    `Real-Time Single Image and Video Super-Resolution Using an Efficient\n    Sub-Pixel Convolutional Neural Network <https://arxiv.org/abs/1609.05158>`_\n    for further details.\n\n    Parameters\n    ----------\n    factor : int or 3-tuple of int\n        Upsampling factors, applied to the ``D``, ``H`` and ``W``\n        dimensions, in that order.\n\n    Inputs:\n        - **data**: Tensor of shape ``(N, f1*f2*f3*C, D, H, W)``.\n    Outputs:\n        - **out**: Tensor of shape ``(N, C, D*f1, H*f2, W*f3)``.\n\n    Examples\n    --------\n    >>> pxshuf = PixelShuffle3D((2, 3, 4))\n    >>> x = mx.np.zeros((1, 48, 3, 5, 7))\n    >>> pxshuf(x).shape\n    (1, 2, 6, 15, 28)\n    \"\"\"\n\n    def __init__(self, factor):\n        super(PixelShuffle3D, self).__init__()\n        try:\n            self._factors = (int(factor),) * 3\n        except TypeError:\n            self._factors = tuple(int(fac) for fac in factor)\n            assert len(self._factors) == 3, \"wrong length {}\".format(len(self._factors))\n\n    def forward(self, x):\n        \"\"\"Perform pixel-shuffling on the input.\"\"\"\n        # `transpose` doesn't support 8D, need other implementation\n        f1, f2, f3 = self._factors\n                                                              # (N, C*f1*f2*f3, D, H, W)\n        x = npx.reshape(x, (-2, -6, -1, f1 * f2 * f3, -2, -2, -2))  # (N, C, f1*f2*f3, D, H, W)\n        x = np.swapaxes(x, 2, 3)                               # (N, C, D, f1*f2*f3, H, W)\n        x = npx.reshape(x, (-2, -2, -2, -6, f1, f2*f3, -2, -2))      # (N, C, D, f1, f2*f3, H, W)\n        x = npx.reshape(x, (-2, -2, -5, -2, -2, -2))                 # (N, C, D*f1, f2*f3, H, W)\n        x = np.swapaxes(x, 3, 4)                               # (N, C, D*f1, H, f2*f3, W)\n        x = npx.reshape(x, (-2, -2, -2, -2, -6, f2, f3, -2))         # (N, C, D*f1, H, f2, f3, W)\n        x = npx.reshape(x, (-2, -2, -2, -5, -2, -2))                 # (N, C, D*f1, H*f2, f3, W)\n        x = np.swapaxes(x, 4, 5)                               # (N, C, D*f1, H*f2, W, f3)\n        x = npx.reshape(x, (-2, -2, -2, -2, -5))                    # (N, C, D*f1, H*f2, W*f3)\n        return x\n\n    def __repr__(self):\n        return \"{}({})\".format(self.__class__.__name__, self._factors)\n"
  },
  {
    "path": "python/mxnet/gluon/parameter.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=unnecessary-pass, too-many-lines\n\"\"\"Neural network parameter.\"\"\"\n\n__all__ = ['DeferredInitializationError', 'Parameter', 'Constant',\n           'tensor_types']\n\n\nimport uuid\nimport warnings\nimport weakref\nimport numpy as np\n\nfrom ..base import mx_real_t, MXNetError\nfrom .. import symbol, ndarray, initializer, device as _device, _deferred_compute as dc\nfrom ..device import Device, cpu\nfrom .. import autograd\nfrom .utils import shape_is_known\nfrom ..util import is_np_shape, is_np_array, wrap_ctx_to_device_func\nfrom .. import numpy as _mx_np  # pylint: disable=reimported\n\n# pylint: disable= invalid-name\ntensor_types = (symbol.Symbol, ndarray.NDArray)\n# pylint: enable= invalid-name\n\nclass DeferredInitializationError(MXNetError):\n    \"\"\"Error for unfinished deferred initialization.\"\"\"\n    pass\n\nclass Parameter(object):\n    \"\"\"A Container holding parameters (weights) of Blocks.\n\n    :py:class:`Parameter` holds a copy of the parameter on each :py:class:`Device` after\n    it is initialized with ``Parameter.initialize(...)``. If :py:attr:`grad_req` is\n    not ``'null'``, it will also hold a gradient array on each :py:class:`Device`::\n\n        device = mx.gpu(0)\n        x = mx.np.zeros((16, 100), device=device)\n        w = mx.gluon.Parameter('fc_weight', shape=(64, 100), init=mx.init.Xavier())\n        b = mx.gluon.Parameter('fc_bias', shape=(64,), init=mx.init.Zero())\n        w.initialize(device=device)\n        b.initialize(device=device)\n        out = mx.npx.fully_connected(x, w.data(device), b.data(device), num_hidden=64)\n\n    Parameters\n    ----------\n    name : str, default 'weight'\n        Name of this parameter. It decides the corresponding default initializer.\n    grad_req : {'write', 'add', 'null'}, default 'write'\n        Specifies how to update gradient to grad arrays.\n\n        - ``'write'`` means everytime gradient is written to grad :py:class:`NDArray`.\n        - ``'add'`` means everytime gradient is added to the grad :py:class:`NDArray`. You need\n          to manually call ``zero_grad()`` to clear the gradient buffer before each\n          iteration when using this option.\n        - 'null' means gradient is not requested for this parameter. gradient arrays\n          will not be allocated.\n    shape : int or tuple of int, default None\n        Shape of this parameter. By default shape is not specified. Parameter with\n        unknown shape can be used for :py:class:`Symbol` API, but ``init`` will throw an error\n        when using :py:class:`NDArray` API.\n    dtype : numpy.dtype or str, default 'float32'\n        Data type of this parameter. For example, ``numpy.float32`` or ``'float32'``.\n    lr_mult : float, default 1.0\n        Learning rate multiplier. Learning rate will be multiplied by lr_mult\n        when updating this parameter with optimizer.\n    wd_mult : float, default 1.0\n        Weight decay multiplier (L2 regularizer coefficient). Works similar to lr_mult.\n    init : Initializer, default None\n        Initializer of this parameter. Will use the global initializer by default.\n    stype: {'default', 'row_sparse', 'csr'}, defaults to 'default'.\n        The storage type of the parameter.\n    grad_stype: {'default', 'row_sparse', 'csr'}, defaults to 'default'.\n        The storage type of the parameter's gradient.\n\n    Attributes\n    ----------\n    grad_req : {'write', 'add', 'null'}\n        This can be set before or after initialization. Setting ``grad_req`` to ``'null'``\n        with ``x.grad_req = 'null'`` saves memory and computation when you don't\n        need gradient w.r.t x.\n    lr_mult : float\n        Local learning rate multiplier for this Parameter. The actual learning rate\n        is calculated with ``learning_rate * lr_mult``. You can set it with\n        ``param.lr_mult = 2.0``\n    wd_mult : float\n        Local weight decay multiplier for this Parameter.\n    \"\"\"\n    def __init__(self, name='weight', grad_req='write', shape=None, dtype=mx_real_t,\n                 lr_mult=1.0, wd_mult=1.0, init=None, allow_deferred_init=False,\n                 differentiable=True, stype='default', grad_stype='default'):\n        self._var = None\n        self._uuid = str(uuid.uuid4())\n        self._var_name = None\n        self._data = None\n        self._grad = None\n        self._device_list = None\n        self._device_map = None\n        self._trainer = None\n        self._deferred_init = ()\n        self._differentiable = differentiable\n        self._allow_deferred_init = allow_deferred_init\n        self._grad_req = None\n        if isinstance(shape, int):\n            shape = (shape,)\n        self._shape = shape\n        self._name = name\n        self._dtype = dtype\n        self.lr_mult = lr_mult\n        self.wd_mult = wd_mult\n        self.grad_req = grad_req\n        self.init = init\n        # sparse related storage type information\n        valid_stypes = ['default', 'row_sparse', 'csr']\n        assert grad_stype in valid_stypes, \"grad_stype for Parameter must be \" \\\n            f\"one of 'default', 'row_sparse', or 'csr', but got '{grad_stype}'\"\n        assert stype in valid_stypes, \"stype for Parameter must be \" \\\n            f\"one of 'default', 'row_sparse', or 'csr', but got '{stype}'\"\n        self._grad_stype = grad_stype\n        self._stype = stype\n\n    def __repr__(self):\n        s = 'Parameter (shape={shape}, dtype={dtype})'\n        return s.format(shape=self.shape, dtype=self.dtype)\n\n    @property\n    def grad_req(self):\n        return self._grad_req\n\n    @property\n    def name(self):\n        return self._name\n\n    @grad_req.setter\n    def grad_req(self, req):\n        assert req in ['write', 'add', 'null'], \\\n            f\"grad_req must be one of 'write', 'add', or 'null', but got '{req}'\"\n        if not self._differentiable:\n            req = 'null'\n        if self._grad_req == req:\n            return\n        self._grad_req = req\n        if req == 'null' and self._grad is not None:\n            self._grad = None\n            self._data = [i.detach() for i in self._data]\n        elif self._data is not None:\n            self._init_grad()\n\n    @property\n    def dtype(self):\n        \"\"\"The type of the parameter.\n\n        Setting the dtype value is equivalent to casting the value of the parameter\n        \"\"\"\n        return self._dtype\n\n    @dtype.setter\n    def dtype(self, dtype):\n        self.cast(dtype)\n\n    @property\n    def shape(self):\n        \"\"\"The shape of the parameter.\n\n        By default, an unknown dimension size is 0. However, when the NumPy semantic\n        is turned on, unknown dimension size is -1.\n        \"\"\"\n        if self._shape is None:\n            return None\n        elif is_np_shape():\n            # Parameters shouldn't be zero-size. If one of its dimension is 0,\n            # it means the parameter isn't initialized. In the NumPy semantics,\n            # the unknown dimension should be marked with -1.\n            return tuple(i if i != 0 else -1 for i in self._shape)\n        else:\n            return self._shape\n\n    @shape.setter\n    def shape(self, new_shape):\n        if self._shape is None:\n            self._shape = new_shape\n            return\n\n        assert len(self._shape) == len(new_shape) and \\\n            all(j in (-1, 0, i) for i, j in zip(new_shape, self._shape)), \\\n            f\"Expected shape {str(new_shape)} is incompatible with given shape {str(self._shape)} for Parameter {str(self.name)}.\" \n            # -1 means unknown dim size in np_shape mode\n\n        self._shape = new_shape\n\n    def _set_trainer(self, trainer):\n        \"\"\" Set the trainer this parameter is associated with. \"\"\"\n        # trainer cannot be replaced for sparse params\n        if self._stype != 'default' and self._trainer and trainer and self._trainer() is not trainer:\n            raise RuntimeError(\n                f\"Failed to set the trainer for Parameter '{self.name}' because it was already set. \" \\\n                f\"More than one trainers for a {self._stype} Parameter is not supported.\")\n        if trainer is not None:\n            self._trainer = weakref.ref(trainer)\n        else:\n            self._trainer = trainer\n\n    def _check_and_get(self, arr_list, device):\n        if arr_list is not None:\n            if device is list:\n                return arr_list\n            if device is None:\n                if len(arr_list) == 1:\n                    return arr_list[0]\n                else:\n                    device = _device.current_device()\n            device_list = self._device_map[device.device_typeid&1]\n            if device.device_id < len(device_list):\n                idx = device_list[device.device_id]\n                if idx is not None:\n                    return arr_list[idx]\n            raise RuntimeError(\n                f\"Parameter '{self.name}' was not initialized on device {str(device)}. \"\n                f\"It was only initialized on {str(self._device_list)}.\")\n        if self._deferred_init:\n            raise DeferredInitializationError(\n                f\"Parameter '{self.name}' has not been initialized yet because initialization was \" \\\n                \"deferred. Actual initialization happens during the first forward pass. \" \\\n                \"Please pass one batch of data through the network before accessing Parameters. \" \\\n                \"You can also avoid deferred initialization by specifying in_units, \" \\\n                \"num_features, etc., for network layers.\")\n        raise RuntimeError(\n            f\"Parameter '{self.name}' has not been initialized. Note that \" \\\n            \"you should initialize parameters and create Trainer \" \\\n            \"with Block.collect_params() instead of Block.params \" \\\n            \"because the later does not include Parameters of \" \\\n            \"nested child Blocks\")\n\n    @wrap_ctx_to_device_func\n    def _get_row_sparse(self, arr_list, device, row_id):\n        \"\"\" Get row_sparse data from row_sparse parameters based on row_id. \"\"\"\n        # get row sparse params based on row ids\n        if not isinstance(row_id, ndarray.NDArray):\n            raise TypeError(f\"row_id must have NDArray type, but {type(row_id)} is given\")\n        trainer = self._trainer() if self._trainer else None\n        if not trainer:\n            raise RuntimeError(f\"Cannot get row_sparse data for Parameter '{self.name}' when no \" \\\n                               \"Trainer is created with it.\")\n        results = self._check_and_get(arr_list, device)\n\n        # fetch row sparse params from the trainer\n        trainer._row_sparse_pull(self, results, row_id)\n        return results\n\n    @wrap_ctx_to_device_func\n    def _load_init(self, data, device, cast_dtype=False, dtype_source='current'):\n        \"\"\"\n        (Re)initializes by loading from data.\n        Parameters\n        ----------\n        data : NDArray\n            The data to load\n        device : Device or list of Device\n            Device(s) initialize loaded parameters on.\n        cast_dtype : bool, default False\n            Cast the data type of the parameter\n        dtype_source : str, default 'current'\n            must be in {'current', 'saved'}\n            Only valid if cast_dtype=True, specify the source of the dtype for casting\n            the parameters\n        \"\"\"\n        if cast_dtype:\n            assert dtype_source in ['current', 'saved']\n        if self.shape:\n            unknown_dim_size = -1 if is_np_shape() else 0\n            for self_dim, data_dim in zip(self.shape, data.shape):\n                assert self_dim in (unknown_dim_size, data_dim), \\\n                    f\"Failed loading Parameter '{self.name}' from saved params: \" \\\n                    f\"shape incompatible expected {str(self.shape)} vs saved {str(data.shape)}\"\n            self.shape = tuple(i if i != unknown_dim_size else j\n                               for i, j in zip(self.shape, data.shape))\n        if self.dtype:\n            if cast_dtype and self.dtype != data.dtype:\n                if dtype_source == 'current':\n                    data = data.astype(self.dtype, copy=False)\n                elif dtype_source == 'saved':\n                    self.dtype = data.dtype\n            else:\n                assert self.dtype == data.dtype, \\\n                f\"Failed loading Parameter '{self.name}' from saved params: \" \\\n                f\"dtype incompatible expected {str(self.dtype)} vs saved {str(data.dtype)}. \" \\\n                \"Set cast_dtype=True to cast the dtype of saved params.\"\n        if self._stype != data.stype:\n            data = data.tostype(self._stype)\n        if isinstance(device, Device):\n            device = [device]\n        if self._data is None:\n            if self._deferred_init:\n                assert device is None or set(device) == set(self._deferred_init[1]), \\\n                    f\"Failed to load Parameter '{self.name}' on {str(device)} because it was \" \\\n                    f\"previous initialized on {str(self.list_device())}.\"\n                device = self._deferred_init[1]\n            elif device is None:\n                device = [cpu()]\n            self._init_impl(data, device)\n        else:\n            assert device is None or set(device) == set(self.list_device()), \\\n                f\"Failed to load Parameter '{self.name}' on {str(device)} because it was \" \\\n                f\"previous initialized on {str(self.list_device())}.\"\n            self.set_data(data)\n        self._deferred_init = ()\n\n    def _finish_deferred_init(self):\n        \"\"\"Finishes deferred initialization.\"\"\"\n        if not self._deferred_init:\n            return\n        init, device, default_init, data = self._deferred_init\n        self._deferred_init = ()\n\n        assert shape_is_known(self.shape), \\\n            f\"Cannot initialize Parameter '{self.name}' because it has \" \\\n            f\"invalid shape: {str(self.shape)}. Please specify in_units, \" \\\n            \"in_channels, etc for `Block`s.\"\n\n        with autograd.pause(), dc.context(False):\n            if data is None:\n                if is_np_array():\n                    kwargs = {'shape': self.shape, 'dtype': self.dtype, 'device': cpu()}\n                    if self._stype != 'default':\n                        raise ValueError(\"Currently stype {} is not supported in NumPy interface and Gluon2.0\"\n                                         .format(self._stype))\n                    zeros_fn = _mx_np.zeros\n                else:\n                    kwargs = {'shape': self.shape, 'dtype': self.dtype, 'ctx': cpu()}\n                    kwargs['stype'] = self._stype\n                    zeros_fn = ndarray.zeros\n                data = zeros_fn(**kwargs)\n                initializer.create(default_init)(\n                    initializer.InitDesc(self.name, {'__init__': init}), data)\n\n            self._init_impl(data, device)\n\n    def _init_impl(self, data, device_list):\n        \"\"\"Sets data and grad.\"\"\"\n        self._device_list = list(device_list)\n        self._device_map = [[], []]\n        for i, device in enumerate(self._device_list):\n            dev_list = self._device_map[device.device_typeid&1]\n            while len(dev_list) <= device.device_id:\n                dev_list.append(None)\n            dev_list[device.device_id] = i\n\n        self._data = [data.copyto(device) for device in self._device_list]\n        self._init_grad()\n\n    def _init_grad(self):\n        \"\"\"Initialize grad buffers.\"\"\"\n        if self.grad_req == 'null':\n            self._grad = None\n            return\n\n        if is_np_array():\n            if self._grad_stype != 'default':\n                raise ValueError(\"Currently stype {} is not supported in NumPy interface and Gluon2.0\"\n                                 .format(self._grad_stype))\n            self._grad = [_mx_np.zeros(shape=i.shape, dtype=i.dtype, device=i.device)\n                          for i in self._data]\n        else:\n            self._grad = [ndarray.zeros(shape=i.shape, dtype=i.dtype, ctx=i.context,\n                                        stype=self._grad_stype) for i in self._data]\n\n        autograd.mark_variables(self._check_and_get(self._data, list),\n                                self._grad, self.grad_req)\n\n    def _reduce(self):\n        \"\"\"Reduce data from multiple device to cpu.\"\"\"\n        device = cpu()\n        if self._stype == 'default':\n            block = self.list_data()\n            if len(block) > 1:\n                if is_np_array():\n                    data = sum([w.copyto(device) for w in block]) / len(block)\n                else:\n                    data = ndarray.add_n(*(w.copyto(device) for w in block)) / len(block)\n            else:\n                data = self.data().copyto(device)\n        else:\n            # fetch all rows for 'row_sparse' param\n            all_row_ids = ndarray.arange(0, self.shape[0], dtype='int64', ctx=device)\n            data = ndarray.zeros(self.shape, stype='row_sparse', ctx=device)\n            trainer = self._trainer() if self._trainer else None\n            if not trainer:\n                raise RuntimeError(f\"Cannot reduce row_sparse data for Parameter '{self.name}' when no \" \\\n                                   \"Trainer is created with it.\")\n            trainer._row_sparse_pull(self, data, all_row_ids, full_idx=True)\n        return data\n\n    @wrap_ctx_to_device_func\n    def initialize(self, init=None, device=None, default_init=initializer.Uniform(),\n                   force_reinit=False):\n        \"\"\"Initializes parameter and gradient arrays. Only used for :py:class:`NDArray` API.\n\n        Parameters\n        ----------\n        init : Initializer\n            The initializer to use. Overrides :py:meth:`Parameter.init` and default_init.\n        device : Device or list of Device, default :py:meth:`device.current_device()`.\n            Assign Parameter to given device. If device is a list of Device, a\n            copy will be made for each device.\n\n            .. note::\n                Copies are independent arrays. User is responsible for keeping\n                their values consistent when updating.\n                Normally :py:class:`gluon.Trainer` does this for you.\n\n        default_init : Initializer\n            Default initializer is used when both :py:func:`init`\n            and :py:meth:`Parameter.init` are ``None``.\n        force_reinit : bool, default False\n            Whether to force re-initialization if parameter is already initialized.\n        Examples\n        --------\n        >>> weight = mx.gluon.Parameter('weight', shape=(2, 2))\n        >>> weight.initialize(device=mx.cpu(0))\n        >>> weight.data()\n        [[-0.01068833  0.01729892]\n         [ 0.02042518 -0.01618656]]\n        <NDArray 2x2 @cpu(0)>\n        >>> weight.grad()\n        [[ 0.  0.]\n         [ 0.  0.]]\n        <NDArray 2x2 @cpu(0)>\n        >>> weight.initialize(device=[mx.gpu(0), mx.gpu(1)])\n        >>> weight.data(mx.gpu(0))\n        [[-0.00873779 -0.02834515]\n         [ 0.05484822 -0.06206018]]\n        <NDArray 2x2 @gpu(0)>\n        >>> weight.data(mx.gpu(1))\n        [[-0.00873779 -0.02834515]\n         [ 0.05484822 -0.06206018]]\n        <NDArray 2x2 @gpu(1)>\n        \"\"\"\n        if self._data is not None and not force_reinit:\n            warnings.warn(f\"Parameter '{self.name}' is already initialized, ignoring. \" \\\n                          \"Set force_reinit=True to re-initialize.\",\n                          stacklevel=2)\n            return\n        self._data = self._grad = None\n        if device is None:\n            device = [_device.current_device()]\n        if isinstance(device, Device):\n            device = [device]\n        if isinstance(self.init, initializer.RNNFused):\n            self.init.set_initializer(init if init else default_init)\n            init = default_init = self.init\n        if init is None:\n            init = default_init if self.init is None else self.init\n        if not shape_is_known(self.shape):\n            if self._allow_deferred_init:\n                self._deferred_init = (init, device, default_init, None)\n                return\n            raise ValueError(f\"Cannot initialize Parameter '{self.name}' because it has \" \\\n                             f\"invalid shape: {str(self.shape)}.\")\n\n        self._deferred_init = (init, device, default_init, None)\n        self._finish_deferred_init()\n\n    def reset_device(self, device):\n        \"\"\"Re-assign Parameter to other devices.\n\n        Parameters\n        ----------\n        device : Device or list of Device, default ``device.current_device()``.\n            Assign Parameter to given device. If device is a list of Device, a\n            copy will be made for each device.\n        \"\"\"\n        if device is None:\n            device = [_device.current_device()]\n        if isinstance(device, Device):\n            device = [device]\n        if self._data:\n            data = self._reduce()\n            with autograd.pause():\n                self._init_impl(data, device)\n        elif self._deferred_init:\n            init, _, default_init, data = self._deferred_init\n            self._deferred_init = (init, device, default_init, data)\n        else:\n            raise ValueError(f\"Cannot reset device for Parameter '{self.name}' because it \"\n                             \"has not been initialized.\")\n\n    def reset_ctx(self, ctx):\n        \"\"\"This function has been deprecated. Please refer to ``Parameter.reset_device``.\"\"\"\n        warnings.warn('Parameter.reset_ctx has been renamed to'\n                      ' Parameter.reset_device', DeprecationWarning)\n        self.reset_device(ctx)\n\n    def set_data(self, data):\n        \"\"\"Sets this parameter's value on all devices.\"\"\"\n        self.shape = data.shape\n\n        if self._data is None:\n            assert self._deferred_init, \\\n                f\"Parameter '{self.name}' has not been initialized\"\n            self._deferred_init = self._deferred_init[:3] + (data,)\n            return\n\n        # if update_on_kvstore, we need to make sure the copy stored in kvstore is in sync\n        trainer = self._trainer() if self._trainer else None\n        if trainer and trainer._kv_initialized and trainer._update_on_kvstore:\n            if self not in trainer._params_to_init:\n                trainer._reset_kvstore()\n\n        for arr in self._check_and_get(self._data, list):\n            arr[:] = data\n\n    def row_sparse_data(self, row_id):\n        \"\"\"Returns a copy of the 'row_sparse' parameter on the same device as row_id's.\n        The copy only retains rows whose ids occur in provided row ids.\n        The parameter must have been initialized on this device before.\n\n        Parameters\n        ----------\n        row_id: NDArray\n            Row ids to retain for the 'row_sparse' parameter.\n\n        Returns\n        -------\n        NDArray on row_id's device\n        \"\"\"\n        if self._stype != 'row_sparse':\n            raise RuntimeError(f\"Cannot return a copy of Parameter {self.name} via row_sparse_data() \" \\\n                               f\"because its storage type is {self._stype}. Please use data() instead.\")\n        return self._get_row_sparse(self._data, row_id.device, row_id)\n\n    def list_row_sparse_data(self, row_id):\n        \"\"\"Returns copies of the 'row_sparse' parameter on all devices, in the same order\n        as creation. The copy only retains rows whose ids occur in provided row ids.\n        The parameter must have been initialized before.\n\n        Parameters\n        ----------\n        row_id: NDArray\n            Row ids to retain for the 'row_sparse' parameter.\n\n        Returns\n        -------\n        list of NDArrays\n        \"\"\"\n        if self._stype != 'row_sparse':\n            raise RuntimeError(f\"Cannot return copies of Parameter '{self.name}' on all devices via \" \\\n                               f\"list_row_sparse_data() because its storage type is {self._stype}. Please \" \\\n                               \"use data() instead.\")\n        return self._get_row_sparse(self._data, list, row_id)\n\n    @wrap_ctx_to_device_func\n    def data(self, device=None):\n        \"\"\"Returns a copy of this parameter on one device. Must have been\n        initialized on this device before. For sparse parameters, use\n        :py:meth:`Parameter.row_sparse_data` instead.\n\n        Parameters\n        ----------\n        device : Device\n            Desired device.\n\n        Returns\n        -------\n        NDArray on device\n        \"\"\"\n        if self._stype != 'default':\n            raise RuntimeError(f\"Cannot return a copy of Parameter '{self.name}' on device {str(device)} via data() \" \\\n                               f\"because its storage type is {self._stype}. Please use row_sparse_data() instead.\")\n        data = self._check_and_get(self._data, device)\n        dc.set_variable(data, self.var())\n        return data\n\n    def list_data(self):\n        \"\"\"Returns copies of this parameter on all devices, in the same order\n        as creation. For sparse parameters, use :py:meth:`Parameter.list_row_sparse_data`\n        instead.\n\n        Returns\n        -------\n        list of NDArrays\n        \"\"\"\n        if self._stype != 'default':\n            raise RuntimeError(f\"Cannot return copies of Parameter '{self.name}' on all devices via \" \\\n                               f\"list_data() because its storage type is {self._stype}. Please use \" \\\n                               \"row_sparse_data() instead.\")\n        return self._check_and_get(self._data, list)\n\n    def grad(self, device=None):\n        \"\"\"Returns a gradient buffer for this parameter on one device.\n\n        Parameters\n        ----------\n        device : Device\n            Desired device.\n        \"\"\"\n        if self._data is not None and self._grad is None:\n            raise RuntimeError(\n                f\"Cannot get gradient array for Parameter '{self.name}' \" \\\n                \"because grad_req='null'\")\n        return self._check_and_get(self._grad, device)\n\n    def list_grad(self):\n        \"\"\"Returns gradient buffers on all devices, in the same order\n        as :py:meth:`values`.\"\"\"\n        if self._data is not None and self._grad is None:\n            raise RuntimeError(\n                f\"Cannot get gradient array for Parameter '{self.name}' \" \\\n                \"because grad_req='null'\")\n        return self._check_and_get(self._grad, list)\n\n    def list_ctx(self):\n        \"\"\"This function has been deprecated. Please refer to ``Parameter.list_device``.\"\"\"\n        warnings.warn('Parameter.list_ctx has been renamed to'\n                      ' Parameter.list_device', DeprecationWarning)\n        return self.list_device()\n\n    def list_device(self):\n        \"\"\"Returns a list of devices this parameter is initialized on.\"\"\"\n        if self._data is None:\n            if self._deferred_init:\n                return self._deferred_init[1]\n            raise RuntimeError(f\"Parameter '{self.name}' has not been initialized\")\n        return self._device_list\n\n    def zero_grad(self):\n        \"\"\"Sets gradient buffer on all devices to 0. No action is taken if\n        parameter is uninitialized or doesn't require gradient.\"\"\"\n        if self._grad is None:\n            return\n        for i in self._grad:\n            ndarray.zeros_like(i, out=i)\n\n    def var(self):\n        \"\"\"Returns a symbol representing this parameter.\"\"\"\n        if self._var is None:\n            if self._var_name is None:  # _var_name is set manually in SymbolBlock.import\n                # The variable name is required by the storage profiler.\n                self._var_name = self._uuid.replace('-', '_') + '_' + self._name\n            self._var = symbol.var(self._var_name, shape=self.shape, dtype=self.dtype,\n                                   lr_mult=self.lr_mult, wd_mult=self.wd_mult,\n                                   init=self.init, stype=self._stype)\n            if is_np_array():\n                self._var = self._var.as_np_ndarray()\n        return self._var\n\n    def cast(self, dtype):\n        \"\"\"Cast data and gradient of this Parameter to a new data type.\n\n        Parameters\n        ----------\n        dtype : str or numpy.dtype\n            The new data type.\n        \"\"\"\n        self._dtype = dtype\n        self._var = None  # Clear Symbol Variable as it caches the dtype\n        if self._data is None:\n            return\n        with autograd.pause():\n            self._data = [i.astype(dtype) for i in self._data]\n            if self._grad is None:\n                return\n            self._grad = [i.astype(dtype) for i in self._grad]\n            autograd.mark_variables(self._data, self._grad, self.grad_req)\n\n    def _check_and_setattr(self, **kwargs):\n        \"\"\"check and set attributes for parameter\"\"\"\n        for k, v in kwargs.items():\n            if hasattr(self, k) and getattr(self, k) is not None:\n                existing = getattr(self, k)\n                if k == 'shape' and len(v) == len(existing):\n                    inferred_shape = []\n                    matched = True\n                    for dim1, dim2 in zip(v, existing):\n                        if dim1 != dim2 and dim1 > 0 and dim2 > 0:\n                            matched = False\n                            break\n                        elif dim1 == dim2:\n                            inferred_shape.append(dim1)\n                        elif dim1 in (0, -1):  # -1 means unknown dim size in np_shape mode\n                            inferred_shape.append(dim2)\n                        else:\n                            inferred_shape.append(dim1)\n\n                    if matched:\n                        self._shape = tuple(inferred_shape)\n                        continue\n                elif k == 'dtype' and np.dtype(v) == np.dtype(existing):\n                    continue\n\n                assert v is None or v == existing, \\\n                    f\"Cannot retrieve Parameter '{self.name}' because desired attribute \" \\\n                    f\"does not match with stored for attribute '{k}': \" \\\n                    f\"desired '{str(v)}' vs stored '{str(getattr(self, k))}'.\"\n            else:\n                setattr(self, k, v)\n\nclass Constant(Parameter):\n    \"\"\"A constant parameter for holding immutable tensors.\n    `Constant`s are ignored by `autograd` and `Trainer`, thus their values\n    will not change during training. But you can still update their values\n    manually with the `set_data` method.\n\n    `Constant` s can be created with either::\n\n        const = mx.gluon.Constant([[1,2],[3,4]])\n\n    or::\n\n        class Block(gluon.Block):\n            def __init__(self, **kwargs):\n                super(Block, self).__init__(**kwargs)\n                self.const = mx.gluon.Constant([[1,2],[3,4]])\n\n    Parameters\n    ----------\n    value : array-like\n        Initial value for the constant.\n    \"\"\"\n    def __init__(self, value):\n        if not isinstance(value, ndarray.NDArray):\n            array_fn = _mx_np.array if is_np_array() else ndarray.array\n            value = array_fn(value)\n        self.value = value\n\n        class Init(initializer.Initializer):\n            def _init_weight(self, _, arr):\n                value.copyto(arr)\n        init_name = 'Constant_{}'.format(id(self))\n        initializer.alias(init_name)(Init)\n\n        super(Constant, self).__init__(\n            name='const', grad_req='null', shape=value.shape, dtype=value.dtype,\n            init=init_name)\n\n    def __repr__(self):\n        s = 'Constant (shape={shape}, dtype={dtype})'\n        return s.format(shape=self.shape, dtype=self.dtype)\n\n    @property\n    def grad_req(self):\n        return 'null'\n\n    @grad_req.setter\n    def grad_req(self, req):\n        if req != 'null':\n            warnings.warn('Constant parameter \"{}\" does not support '\n                          'grad_req other than \"null\", and new value \"{}\" '\n                          'is ignored.'.format(self.name, req))\n"
  },
  {
    "path": "python/mxnet/gluon/probability/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Probability module\"\"\"\n\nfrom .block import *\n\nfrom .distributions import *\n\nfrom .transformation import *\n"
  },
  {
    "path": "python/mxnet/gluon/probability/block/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Stochastic block.\"\"\"\n\nfrom .stochastic_block import *\n"
  },
  {
    "path": "python/mxnet/gluon/probability/block/stochastic_block.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=abstract-method\n\"\"\"Stochastic block class.\"\"\"\n__all__ = ['StochasticBlock', 'StochasticSequential']\n\nfrom functools import wraps\nfrom ...block import HybridBlock\nfrom ...utils import _indent\n\n\nclass StochasticBlock(HybridBlock):\n    \"\"\"`StochasticBlock` extends `HybridBlock` to support accumulating loss\n    in the forward phase, which is extremely useful in building Bayesian Neural Network,\n    where the loss function is composed of a classification loss and a KL loss.\n\n    \"\"\"\n\n    def __init__(self, **kwargs):\n        super(StochasticBlock, self).__init__(**kwargs)\n        self._losses = []\n        self._losscache = []\n        # Recording whether collectLoss is invoked.\n        self._flag = False\n\n    def add_loss(self, loss):\n        self._losscache.append(loss)\n\n    @staticmethod\n    def collectLoss(func):\n        \"\"\"To accumulate loss during the forward phase, one could first decorate\n        forward with `StochasticBlock.collectLoss,\n        and then collect the loss tensor `x` by calling self.add_loss(x).\n        For example, in the following forward function,\n        we generate samples from a Gaussian parameterized by `loc` and `scale` and\n        accumulate the KL-divergence between it and its prior into the block's loss storage.:\n        @StochasticBlock.collectLoss\n        def forward(self, loc, scale):\n            qz = mgp.Normal(loc, scale)\n            # prior\n            pz = mgp.Normal(np.zeros_like(loc), np.ones_like(scale))\n            self.add_loss(mgp.kl_divergence(qz, pz))\n            return qz.sample()\n        \"\"\"\n        @wraps(func)\n        def inner(self, *args, **kwargs):\n            # Loss from forward\n            func_out = func(self, *args, **kwargs)\n            collected_loss = self._losscache\n            self._losscache = []\n            self._flag = True\n            return (func_out, collected_loss)\n\n        return inner\n\n    def __call__(self, *args, **kwargs):\n\t\t# pylint: disable=arguments-differ\n        self._flag = False\n        out = super().__call__(*args, **kwargs)\n        if not self._flag:\n            raise ValueError(\"The forward function should be decorated by \" +\n                             \"StochasticBlock.collectLoss\")\n        self._losses = out[1]\n        return out[0]\n\n    @property\n    def losses(self):\n        return self._losses\n\n\nclass StochasticSequential(StochasticBlock):\n    \"\"\"Stack StochasticBlock sequentially.\n    \"\"\"\n\n    def __init__(self, **kwargs):\n        super(StochasticSequential, self).__init__(**kwargs)\n        self._layers = []\n\n    def add(self, *blocks):\n        \"\"\"Adds block on top of the stack.\"\"\"\n        for block in blocks:\n            self._layers.append(block)\n            self.register_child(block)\n\n    @StochasticBlock.collectLoss\n    def forward(self, x, *args):\n        # pylint: disable=arguments-differ\n        for block in self._children.values():\n            x = block()(x, *args)\n            args = []\n            if isinstance(x, (tuple, list)):\n                args = x[1:]\n                x = x[0]\n        if args:\n            x = tuple([x] + list(args))\n        for block in self._layers:\n            if hasattr(block, '_losses'):\n                self.add_loss(block._losses)\n        return x\n\n    def __repr__(self):\n        s = '{name}(\\n{modstr}\\n)'\n        modstr = '\\n'.join(['  ({key}): {block}'.format(key=key,\n                                                        block=_indent(block().__repr__(), 2))\n                            for key, block in self._children.items()])\n        return s.format(name=self.__class__.__name__, modstr=modstr)\n\n    def __getitem__(self, key):\n        layers = list(self._children.values())[key]\n        if isinstance(layers, list):\n            net = type(self)()\n            net.add(*(l() for l in layers))\n            return net\n        else:\n            return layers()\n\n    def __len__(self):\n        return len(self._children)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Distribution classes.\"\"\"\n\nfrom .distribution import *\n\nfrom .exp_family import *\n\nfrom .exponential import *\n\nfrom .weibull import *\n\nfrom .pareto import *\n\nfrom .uniform import *\n\nfrom .normal import *\n\nfrom .laplace import *\n\nfrom .cauchy import *\n\nfrom .half_cauchy import *\n\nfrom .poisson import *\n\nfrom .geometric import *\n\nfrom .negative_binomial import *\n\nfrom .gamma import *\n\nfrom .dirichlet import *\n\nfrom .beta import *\n\nfrom .chi2 import *\n\nfrom .fishersnedecor import *\n\nfrom .studentT import *\n\nfrom .half_normal import *\n\nfrom .independent import *\n\nfrom .bernoulli import *\n\nfrom .binomial import *\n\nfrom .relaxed_bernoulli import *\n\nfrom .gumbel import *\n\nfrom .categorical import *\n\nfrom .one_hot_categorical import *\n\nfrom .relaxed_one_hot_categorical import *\n\nfrom .multinomial import *\n\nfrom .multivariate_normal import *\n\nfrom .transformed_distribution import *\n\nfrom .divergence import *\n\nfrom .utils import *\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/bernoulli.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Bernoulli class.\"\"\"\n__all__ = ['Bernoulli']\n\nfrom .exp_family import ExponentialFamily\nfrom .utils import prob2logit, logit2prob, cached_property, sample_n_shape_converter\nfrom .constraint import Boolean, Interval, Real\nfrom .... import np, npx\n\n\nclass Bernoulli(ExponentialFamily):\n    r\"\"\"Create a bernoulli distribution object.\n\n    Parameters\n    ----------\n    prob : Tensor or scalar, default None\n        Probability of sampling `1`.\n    logit : Tensor or scalar, default None\n        The log-odds of sampling `1`.\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    support = Boolean()\n    arg_constraints = {'prob': Interval(0, 1),\n                       'logit': Real()}\n\n    def __init__(self, prob=None, logit=None, validate_args=None):\n        if (prob is None) == (logit is None):\n            raise ValueError(\n                \"Either `prob` or `logit` must be specified, but not both. \" +\n                \"Received prob={}, logit={}\".format(prob, logit))\n\n        if prob is not None:\n            self.prob = prob\n        else:\n            self.logit = logit\n\n        super(Bernoulli, self).__init__(\n            event_dim=0, validate_args=validate_args)\n\n    @cached_property\n    def prob(self):\n        \"\"\"Get the probability of sampling `1`.\n\n        Returns\n        -------\n        Tensor\n            Parameter tensor.\n        \"\"\"\n        # pylint: disable=method-hidden\n        return logit2prob(self.logit, True)\n\n    @cached_property\n    def logit(self):\n        \"\"\"Get the log-odds of sampling `1`.\n\n        Returns\n        -------\n        Tensor\n            Parameter tensor.\n        \"\"\"\n        # pylint: disable=method-hidden\n        return prob2logit(self.prob, True)\n\n    @property\n    def mean(self):\n        return self.prob\n\n    @property\n    def variance(self):\n        return self.prob * (1 - self.prob)\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        F = self.F\n        if 'prob' in self.__dict__:\n            new_instance.prob = np.broadcast_to(self.prob, batch_shape)\n        else:\n            new_instance.logit = np.broadcast_to(self.logit, batch_shape)\n        super(Bernoulli, new_instance).__init__(F=F,\n                                                event_dim=self.event_dim,\n                                                validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        if self.prob is None:\n            logit = self.logit\n            return logit * (value - 1) - np.log(np.exp(-logit) + 1)\n        else:\n            # Parameterized by probability\n            eps = 1e-12\n            return (np.log(self.prob + eps) * value\n                    + np.log1p(-self.prob + eps) * (1 - value))\n\n    def sample(self, size=None):\n        return npx.random.bernoulli(self.prob, self.logit, size)\n\n    def sample_n(self, size=None):\n        return npx.random.bernoulli(self.prob, self.logit, sample_n_shape_converter(size))\n\n    @property\n    def _natural_params(self):\n        return (self.logit,)\n\n    def _log_normalizer(self, x):\n        # pylint: disable=arguments-differ\n        return np.log(1 + np.exp(x))\n\n    def entropy(self):\n        logit = self.logit\n        prob = self.prob\n        return -(logit * (prob - 1) - np.log(np.exp(-logit) + 1))\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/beta.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Beta Distribution.\"\"\"\n__all__ = ['Beta']\n\nfrom .exp_family import ExponentialFamily\nfrom .constraint import UnitInterval, Positive\nfrom .utils import sample_n_shape_converter, gammaln, digamma, _clip_prob\nfrom .... import np\n\n\nclass Beta(ExponentialFamily):\n    r\"\"\"Create a Beta distribution object.\n\n    Parameters\n    ----------\n    alpha : Tensor or scalar\n       The first shape parameter\n    beta : Tensor or scalar\n        The second shape parameter\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    has_grad = False\n    support = UnitInterval()\n    arg_constraints = {'alpha': Positive(),\n                       'beta': Positive()}\n\n    def __init__(self, alpha, beta, validate_args=None):\n        self.alpha = alpha\n        self.beta = beta\n        super(Beta, self).__init__(\n            event_dim=0, validate_args=validate_args)\n\n    def sample(self, size=None):\n        X = np.random.gamma(self.alpha, 1, size=size)\n        Y = np.random.gamma(self.beta, 1, size=size)\n        out = X / (X + Y)\n        return _clip_prob(out)\n\n    def sample_n(self, size=None):\n        return self.sample(sample_n_shape_converter(size))\n\n    @property\n    def mean(self):\n        a = self.alpha\n        b = self.beta\n        return a / (a + b)\n\n    @property\n    def variance(self):\n        a = self.alpha\n        b = self.beta\n        return (a * b /\n                ((a + b) ** 2 * (a + b + 1)))\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        lgamma = gammaln()\n        log = np.log\n        log1p = np.log1p\n        a = self.alpha\n        b = self.beta\n        lgamma_term = lgamma(a + b) - lgamma(a) - lgamma(b)\n        return (a - 1) * log(value) + (b - 1) * log1p(-value) + lgamma_term\n\n    def entropy(self):\n        lgamma = gammaln()\n        dgamma = digamma()\n        a = self.alpha\n        b = self.beta\n        lgamma_term = lgamma(a + b) - lgamma(a) - lgamma(b)\n        return (-lgamma_term - (a - 1) * dgamma(a) - (b - 1) * dgamma(b) +\n                (a + b - 2) * dgamma(a + b))\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/binomial.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Binomial distribution class.\"\"\"\n__all__ = ['Binomial']\n\nfrom .distribution import Distribution\nfrom .utils import prob2logit, logit2prob, cached_property, sample_n_shape_converter\nfrom .utils import gammaln\nfrom .constraint import Interval, Real, NonNegativeInteger\nfrom .... import np, npx\n\n\nclass Binomial(Distribution):\n    r\"\"\"Create a binomial distribution object.\n\n    Parameters\n    ----------\n    n : scalar\n        Non-negative interger of Bernoulli trials to stop.\n    prob : Tensor or scalar, default None\n        Probability of sampling `1`.\n    logit : Tensor or scalar, default None\n        The log-odds of sampling `1`.\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    support = NonNegativeInteger()\n    arg_constraints = {'prob': Interval(0, 1),\n                       'logit': Real()}\n\n    def __init__(self, n=1, prob=None, logit=None, validate_args=None):\n        if (n < 0) or (n % 1 != 0):\n            raise ValueError(\n                \"Expect `n` to be non-negative integer, received n={}\".format(n))\n        if (prob is None) == (logit is None):\n            raise ValueError(\n                \"Either `prob` or `logit` must be specified, but not both. \" +\n                \"Received prob={}, logit={}\".format(prob, logit))\n\n        if prob is not None:\n            self.prob = prob\n        else:\n            self.logit = logit\n        self.n = n\n        super(Binomial, self).__init__(\n            event_dim=0, validate_args=validate_args)\n\n    @cached_property\n    def prob(self):\n        \"\"\"Get the probability of sampling `1`.\n\n        Returns\n        -------\n        Tensor\n            Parameter tensor.\n        \"\"\"\n        # pylint: disable=method-hidden\n        return logit2prob(self.logit, True)\n\n    @cached_property\n    def logit(self):\n        \"\"\"Get the log-odds of sampling `1`.\n\n        Returns\n        -------\n        Tensor\n            Parameter tensor.\n        \"\"\"\n        # pylint: disable=method-hidden\n        return prob2logit(self.prob, True)\n\n    @property\n    def mean(self):\n        return self.n * self.prob\n\n    @property\n    def variance(self):\n        p = self.prob\n        return self.n * p * (1 - p)\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        if 'prob' in self.__dict__:\n            new_instance.prob = np.broadcast_to(self.prob, batch_shape)\n        else:\n            new_instance.logit = np.broadcast_to(self.logit, batch_shape)\n        new_instance.n = self.n\n        super(Binomial, new_instance).__init__(event_dim=self.event_dim,\n                                               validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        lgamma = gammaln()\n        binomal_coef = lgamma(self.n + 1) - lgamma(1 +\n                                                   value) - lgamma(self.n - value + 1)\n        # log(prob) may have numerical issue.\n        unnormalized_log_prob = (value * np.log(self.prob) +\n                                 (self.n - value) * np.log1p(-self.prob))\n        return binomal_coef + unnormalized_log_prob\n\n    def sample(self, size=None):\n        if size is not None:\n            logit = np.broadcast_to(self.logit, size)\n        else:\n            logit = self.logit\n        expanded_logit = np.repeat(\n            np.expand_dims(logit, -1), int(self.n), -1)\n        return npx.random.bernoulli(logit=expanded_logit).sum(-1)\n\n    def sample_n(self, size=None):\n        logit = self.logit\n        expanded_logit = np.repeat(\n            np.expand_dims(logit, -1), int(self.n), -1)\n        return npx.random.bernoulli(\n            logit=expanded_logit,\n            size=sample_n_shape_converter(size)\n        ).sum(-1)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/categorical.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Categorical class.\"\"\"\n__all__ = ['Categorical']\n\nfrom .distribution import Distribution\nfrom .utils import prob2logit, logit2prob, cached_property, sample_n_shape_converter\nfrom .constraint import Simplex, Real, IntegerInterval\nfrom .... import np, npx\n\n\nclass Categorical(Distribution):\n    \"\"\"Create a categorical distribution object.\n\n    Parameters\n    ----------\n    num_events : Int\n        Number of events.\n    prob : Tensor\n        Probabilities of each event.\n    logit : Tensor\n        The log-odds of each event\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    has_enumerate_support = True\n    arg_constraints = {'prob': Simplex(),\n                       'logit': Real()}\n\n    def __init__(self, num_events, prob=None, logit=None, validate_args=None):\n        if (num_events > 0):\n            num_events = int(num_events)\n            self.num_events = num_events\n        else:\n            raise ValueError(\"`num_events` should be greater than zero. \" +\n                             \"Received num_events={}\".format(num_events))\n        if (prob is None) == (logit is None):\n            raise ValueError(\n                \"Either `prob` or `logit` must be specified, but not both. \" +\n                \"Received prob={}, logit={}\".format(prob, logit))\n\n        if prob is not None:\n            self.prob = prob\n        else:\n            self.logit = logit\n\n        super(Categorical, self).__init__(\n            event_dim=0, validate_args=validate_args)\n\n    @cached_property\n    def prob(self):\n        # pylint: disable=method-hidden\n        \"\"\"Get the probability of sampling each class.\n\n        Returns\n        -------\n        Tensor\n            Parameter tensor.\n        \"\"\"\n        return logit2prob(self.logit, False)\n\n    @cached_property\n    def logit(self):\n        # pylint: disable=method-hidden\n        \"\"\"Get the log probability of sampling each class.\n\n        Returns\n        -------\n        Tensor\n            Parameter tensor.\n        \"\"\"\n        return prob2logit(self.prob, False)\n\n    @property\n    def support(self):\n        return IntegerInterval(0, self.num_events)\n\n    def log_prob(self, value):\n        \"\"\"Compute the log-likelihood of `value`\n\n        Parameters\n        ----------\n        value : Tensor\n            samples from Categorical distribution\n\n        Returns\n        -------\n        Tensor\n            log-likelihood of `value`\n        \"\"\"\n        if self._validate_args:\n            self._validate_samples(value)\n        logit = self.logit\n        indices = np.expand_dims(value, -1).astype('int')\n        expanded_logit = logit * np.ones_like(logit + indices)  # pylint: disable=too-many-function-args\n        return npx.pick(expanded_logit, indices).squeeze()\n\n    def sample(self, size=None):\n        \"\"\"Sample from categorical distribution.\n        Given logit/prob of size `(batch_size, num_events)`,\n        `batch_size` samples will be drawn.\n        If `size` is given, `np.broadcast(size, batch_size)` samples will be drawn.\n\n        Parameters\n        ----------\n        size : int or tuple of ints\n\n        Returns\n        -------\n        out : Tensor\n            Samples from the categorical distribution.\n        \"\"\"\n        if size is None:\n            size = ()\n            logit = self.logit\n        else:\n            if isinstance(size, int):\n                logit = np.broadcast_to(self.logit, (size,) + (-2,))\n            else:\n                logit = np.broadcast_to(self.logit, size + (-2,))\n        gumbel_samples = np.random.gumbel(logit)\n        return np.argmax(gumbel_samples, axis=-1)\n\n    def sample_n(self, size=None):\n        size = sample_n_shape_converter(size)\n        gumbel_samples = np.random.gumbel(self.logit, size=size)\n        return np.argmax(gumbel_samples, axis=-1)\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        new_instance.prob = np.broadcast_to(self.prob, batch_shape + (-2,))\n        new_instance.logit = np.broadcast_to(self.logit, batch_shape + (-2,))\n        new_instance.num_events = self.num_events\n        super(Categorical, new_instance).__init__(event_dim=self.event_dim,\n                                                  validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n\n    def enumerate_support(self):\n        num_events = self.num_events\n        value = npx.arange_like(self.logit) % num_events\n        return np.moveaxis(value, -1, 0)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/cauchy.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Cauchy distribution\"\"\"\n\n__all__ = ['Cauchy']\n\nfrom numbers import Number\nfrom numpy import nan, pi\nfrom .constraint import Real\nfrom .distribution import Distribution\nfrom .utils import sample_n_shape_converter\nfrom .... import np\n\n\nclass Cauchy(Distribution):\n    r\"\"\"Create a relaxed Cauchy distribution object.\n\n    Parameters\n    ----------\n    loc : Tensor or scalar, default 0\n        mode or median of the distribution\n    scale : Tensor or scalar, default 1\n        half width at half maximum\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    has_grad = True\n    support = Real()\n    arg_constraints = {'loc': Real(), 'scale': Real()}\n\n    def __init__(self, loc=0.0, scale=1.0, validate_args=None):\n        self.loc = loc\n        self.scale = scale\n        super(Cauchy, self).__init__(\n            event_dim=0, validate_args=validate_args)\n\n    @property\n    def mean(self):\n        return nan\n\n    @property\n    def variance(self):\n        return nan\n\n    def sample(self, size=None):\n        # TODO: Implement sampling op in the backend.\n        # `np.zeros_like` does not support scalar at this moment.\n        if (isinstance(self.loc, Number), isinstance(self.scale, Number)) == (True, True):\n            u = np.random.uniform(size=size)\n        else:\n            u = np.random.uniform(np.zeros_like(  # pylint: disable=too-many-function-args\n                self.loc + self.scale), size=size)\n        return self.icdf(u)\n\n    def sample_n(self, size=None):\n        return self.sample(sample_n_shape_converter(size))\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        return (-np.log(pi) - np.log(self.scale) -\n                np.log(1 + ((value - self.loc) / self.scale) ** 2))\n\n    def cdf(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        return np.arctan((value - self.loc) / self.scale) / pi + 0.5\n\n    def icdf(self, value):\n        return np.tan(pi * (value - 0.5)) * self.scale + self.loc\n\n    def entropy(self):\n        return np.log(4 * pi) + np.log(self.scale)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/chi2.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Chi-sqaure distribution\"\"\"\n__all__ = ['Chi2']\n\nfrom .gamma import Gamma\nfrom .constraint import Positive\n\n\nclass Chi2(Gamma):\n    r\"\"\"Create a Chi2 distribution object.\n    Chi2(df) is equivalent to Gamma(shape=df / 2, scale=2)\n\n    Parameters\n    ----------\n    df : Tensor or scalar, default 0\n        Shape parameter of the distribution.\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    arg_constraints = {'df': Positive()}\n\n    def __init__(self, df, validate_args=None):\n        super(Chi2, self).__init__(df / 2, 2, validate_args)\n\n    @property\n    def df(self):\n        return self.shape * 2\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/constraint.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Base class and implementations of constraint\"\"\"\n__all__ = [\"Constraint\", \"Real\", \"Boolean\",\n           \"Interval\", \"OpenInterval\", \"HalfOpenInterval\", \"UnitInterval\",\n           \"IntegerInterval\", \"IntegerOpenInterval\", \"IntegerHalfOpenInterval\",\n           \"GreaterThan\", \"GreaterThanEq\", \"IntegerGreaterThan\", \"IntegerGreaterThanEq\",\n           \"LessThan\", \"LessThanEq\", \"IntegerLessThan\", \"IntegerLessThanEq\",\n           \"Positive\", \"NonNegative\", \"PositiveInteger\", \"NonNegativeInteger\",\n           \"Simplex\", \"LowerTriangular\", \"LowerCholesky\", \"PositiveDefinite\",\n           \"Cat\", \"Stack\"]\n\nfrom .utils import constraint_check\nfrom .... import np\n\n\nclass Constraint(object):\n    \"\"\"Base class for constraints.\n\n    A constraint object represents a region over which a variable\n    is valid.\n    \"\"\"\n\n    def check(self, value):\n        \"\"\"Check if `value` satisfies the constraint,\n        return the origin value if valid,\n        raise `ValueError` with given message otherwise.\n\n        Parameters\n        ----------\n        value : Tensor\n            Input tensor to be checked.\n        \"\"\"\n        raise NotImplementedError\n\n\nclass _Dependent(Constraint):\n    \"\"\"\n    Placeholder for variables whose support depends on other variables.\n    \"\"\"\n\n    def check(self, value):\n        raise ValueError('Cannot validate dependent constraint')\n\n\ndef is_dependent(constraint):\n    return isinstance(constraint, _Dependent)\n\n\nclass _DependentProperty(property, _Dependent):\n    \"\"\"\n    Decorator that extends @property to act like a `_Dependent` constraint when\n    called on a class and act like a property when called on an object.\n    Example::\n        class Uniform(Distribution):\n            def __init__(self, low, high):\n                self.low = low\n                self.high = high\n            @constraint.dependent_property\n            def support(self):\n                return constraint.Interval(self.low, self.high)\n    \"\"\"\n    pass # pylint: disable=unnecessary-pass\n\n\nclass Real(Constraint):\n    \"\"\"\n    Constrain to be a real number. (exclude `np.nan`)\n    \"\"\"\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be a real tensor\".format(\n            value)\n        # False when value has NANs\n        condition = (value == value) # pylint: disable=comparison-with-itself\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass Boolean(Constraint):\n    \"\"\"\n    Constrain to `{0, 1}`.\n    \"\"\"\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be either 0 or 1.\".format(\n            value)\n        condition = (value == 0) | (value == 1)\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass Interval(Constraint):\n    \"\"\"\n    Constrain to a real interval `[lower_bound, upper_bound]`\n    \"\"\"\n\n    def __init__(self, lower_bound, upper_bound):\n        super(Interval, self).__init__()\n        self._lower_bound = lower_bound\n        self._upper_bound = upper_bound\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be >= {} and <= {}.\".format(\n            value, self._lower_bound, self._upper_bound)\n        condition = (value >= self._lower_bound) & (value <= self._upper_bound)\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass OpenInterval(Constraint):\n    \"\"\"\n    Constrain to a real interval `(lower_bound, upper_bound)`\n    \"\"\"\n\n    def __init__(self, lower_bound, upper_bound):\n        super(OpenInterval, self).__init__()\n        self._lower_bound = lower_bound\n        self._upper_bound = upper_bound\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be > {} and < {}.\".format(\n            value, self._lower_bound, self._upper_bound)\n        condition = (value > self._lower_bound) & (value < self._upper_bound)\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass HalfOpenInterval(Constraint):\n    \"\"\"\n    Constrain to a real interval `[lower_bound, upper_bound)`\n    \"\"\"\n\n    def __init__(self, lower_bound, upper_bound):\n        super(HalfOpenInterval, self).__init__()\n        self._lower_bound = lower_bound\n        self._upper_bound = upper_bound\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be >= {} and < {}.\".format(\n            value, self._lower_bound, self._upper_bound)\n        condition = (value >= self._lower_bound) & (value < self._upper_bound)\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass IntegerInterval(Constraint):\n    \"\"\"\n    Constrain to an integer interval `[lower_bound, upper_bound]`\n    \"\"\"\n\n    def __init__(self, lower_bound, upper_bound):\n        super(IntegerInterval, self).__init__()\n        self._lower_bound = lower_bound\n        self._upper_bound = upper_bound\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be integer and be >= {} and <= {}.\".format(\n            value, self._lower_bound, self._upper_bound)\n        condition = value % 1 == 0\n        condition = condition & (value >= self._lower_bound) & (\n            value <= self._upper_bound)\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass IntegerOpenInterval(Constraint):\n    \"\"\"\n    Constrain to an integer interval `(lower_bound, upper_bound)`\n    \"\"\"\n\n    def __init__(self, lower_bound, upper_bound):\n        super(IntegerOpenInterval, self).__init__()\n        self._lower_bound = lower_bound\n        self._upper_bound = upper_bound\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be integer and be > {} and < {}.\".format(\n            value, self._lower_bound, self._upper_bound)\n        condition = value % 1 == 0\n        condition = condition & (value > self._lower_bound) & (\n            value < self._upper_bound)\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass IntegerHalfOpenInterval(Constraint):\n    \"\"\"\n    Constrain to an integer interval `[lower_bound, upper_bound)`\n    \"\"\"\n\n    def __init__(self, lower_bound, upper_bound):\n        super(IntegerHalfOpenInterval, self).__init__()\n        self._lower_bound = lower_bound\n        self._upper_bound = upper_bound\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be integer and be >= {} and < {}.\".format(\n            value, self._lower_bound, self._upper_bound)\n        condition = value % 1 == 0\n        condition = condition & (value >= self._lower_bound) & (\n            value < self._upper_bound)\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass GreaterThan(Constraint):\n    \"\"\"\n    Constrain to be greater than `lower_bound`.\n    \"\"\"\n\n    def __init__(self, lower_bound):\n        super(GreaterThan, self).__init__()\n        self._lower_bound = lower_bound\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be greater than {}\".format(\n            value, self._lower_bound)\n        condition = value > self._lower_bound\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass UnitInterval(Interval):\n    \"\"\"\n    Constrain to an unit interval `[0, 1]`\n    \"\"\"\n\n    def __init__(self):\n        super(UnitInterval, self).__init__(0, 1)\n\n\nclass GreaterThanEq(Constraint):\n    \"\"\"\n    Constrain to be greater than or equal to `lower_bound`.\n    \"\"\"\n\n    def __init__(self, lower_bound):\n        super(GreaterThanEq, self).__init__()\n        self._lower_bound = lower_bound\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be greater than or equal to {}\".format(\n            value, self._lower_bound)\n        condition = value >= self._lower_bound\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass LessThan(Constraint):\n    \"\"\"\n    Constrain to be less than `upper_bound`.\n    \"\"\"\n\n    def __init__(self, upper_bound):\n        super(LessThan, self).__init__()\n        self._upper_bound = upper_bound\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be less than {}\".format(\n            value, self._upper_bound)\n        condition = value < self._upper_bound\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass LessThanEq(Constraint):\n    \"\"\"\n    Constrain to be less than `upper_bound`.\n    \"\"\"\n\n    def __init__(self, upper_bound):\n        super(LessThanEq, self).__init__()\n        self._upper_bound = upper_bound\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be less than or equal to {}\".format(\n            value, self._upper_bound)\n        condition = value <= self._upper_bound\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass IntegerGreaterThan(Constraint):\n    \"\"\"\n    Constrain to be integer and be greater than `lower_bound`.\n    \"\"\"\n\n    def __init__(self, lower_bound):\n        super(IntegerGreaterThan, self).__init__()\n        self._lower_bound = lower_bound\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be integer and be greater than {}\".format(\n            value, self._lower_bound)\n        condition = value % 1 == 0\n        condition = np.bitwise_and(condition, value > self._lower_bound)\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass IntegerGreaterThanEq(Constraint):\n    \"\"\"\n    Constrain to be integer and be greater than or equal to `lower_bound`.\n    \"\"\"\n\n    def __init__(self, lower_bound):\n        super(IntegerGreaterThanEq, self).__init__()\n        self._lower_bound = lower_bound\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be integer and\" \\\n                  \" be greater than or equal to {}\".format(\n                      value, self._lower_bound)\n        condition = value % 1 == 0\n        condition = np.bitwise_and(condition, value >= self._lower_bound)\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass IntegerLessThan(Constraint):\n    \"\"\"\n    Constrain to be integer and be less than `upper_bound`.\n    \"\"\"\n\n    def __init__(self, upper_bound):\n        super(IntegerLessThan, self).__init__()\n        self._upper_bound = upper_bound\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be integer and be less than {}\".format(\n            value, self._upper_bound)\n        condition = value % 1 == 0\n        condition = condition & (value < self._upper_bound)\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass IntegerLessThanEq(Constraint):\n    \"\"\"\n    Constrain to be integer and be less than or equal to `upper_bound`.\n    \"\"\"\n\n    def __init__(self, upper_bound):\n        super(IntegerLessThanEq, self).__init__()\n        self._upper_bound = upper_bound\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be integer and\" \\\n                  \" be less than or equal to {}\".format(\n                      value, self._upper_bound)\n        condition = value % 1 == 0\n        condition = condition & (value <= self._upper_bound)\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass Positive(GreaterThan):\n    \"\"\"\n    Constrain to be greater than zero.\n    \"\"\"\n\n    def __init__(self):\n        super(Positive, self).__init__(0)\n\n\nclass NonNegative(GreaterThanEq):\n    \"\"\"\n    Constrain to be greater than or equal to zero.\n    \"\"\"\n\n    def __init__(self):\n        super(NonNegative, self).__init__(0)\n\n\nclass PositiveInteger(IntegerGreaterThan):\n    \"\"\"\n    Constrain to be positive integer.\n    \"\"\"\n\n    def __init__(self):\n        super(PositiveInteger, self).__init__(0)\n\n\nclass NonNegativeInteger(IntegerGreaterThanEq):\n    \"\"\"\n    Constrain to be non-negative integer.\n    \"\"\"\n\n    def __init__(self):\n        super(NonNegativeInteger, self).__init__(0)\n\n\nclass Simplex(Constraint):\n    \"\"\"\n    Constraint to the simplex that rightmost dimension lies on a simplex.\n    `x >= 0` and `x.sum(-1) == 1`.\n    \"\"\"\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be >= 0 and\" \\\n                  \" its rightmost dimension should sum up to 1\".format(value)\n        condition = np.all(value >= 0, axis=-1)\n        condition = condition & (np.abs(value.sum(-1) - 1) < 1e-6)\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass LowerTriangular(Constraint):\n    \"\"\"\n    Constraint to square lower triangular matrices.\n    \"\"\"\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be\" \\\n                  \" square lower triangular matrices\".format(value)\n        condition = np.tril(value) == value\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass LowerCholesky(Constraint):\n    \"\"\"\n    Constraint to square lower triangular matrices with real and positive diagonal entries.\n    \"\"\"\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be\" \\\n                  \" square lower triangular matrices\" \\\n                  \" with real and positive diagonal entries\".format(value)\n        condition = np.all(np.tril(value) == value, axis=-1)\n        condition = condition & (np.diagonal(value, axis1=-2, axis2=-1) > 0)\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass PositiveDefinite(Constraint):\n    \"\"\"\n    Constraint to positive-definite matrices.\n    \"\"\"\n\n    def check(self, value):\n        err_msg = \"Constraint violated: {} should be\" \\\n                  \" positive definite matrices\".format(value)\n        eps = 1e-5\n        condition = np.all(\n            np.abs(value - value.mT) < eps, axis=-1)\n        condition = condition & (np.linalg.eigvals(value) > 0)\n        _value = constraint_check()(condition, err_msg) * value\n        return _value\n\n\nclass Cat(Constraint):\n    \"\"\"\n    Constraint functor that applies a sequence of constraints\n    `constraint_seq` at the submatrices at `axis`, each of size `lengths[axis]`,\n    in compatible with :func:`np.concatenate`.\n    \"\"\"\n\n    def __init__(self, constraint_seq, axis=0, lengths=None):\n        assert all(isinstance(c, Constraint) for c in constraint_seq)\n        self._constraint_seq = list(constraint_seq)\n        if lengths is None:\n            lengths = [1] * len(self._constraint_seq)\n        self._lengths = list(lengths)\n        assert len(self._lengths) == len(self._constraint_seq),\\\n            \"The number of lengths {} should be equal to number\" \\\n            \" of constraints {}\".format(\n                len(self._lengths), len(self._constraint_seq))\n        self._axis = axis\n\n    def check(self, value):\n        _values = []\n        start = 0\n        for length in self._lengths:\n            v = np.take(value, indices=np.arange(\n                start, start + length), axis=self._axis)\n            _values.append(v)\n            start = start + length\n        _value = np.concatenate(_values, self._axis)\n        return _value\n\n\nclass Stack(Constraint):\n    \"\"\"\n    Constraint functor that applies a sequence of constraints\n    `constraint_seq` at the submatrices at `axis`,\n    in compatible with :func:`np.stack`.\n\n    Stack is currently only supported in imperative mode.\n    \"\"\"\n\n    def __init__(self, constraint_seq, axis=0):\n        assert all(isinstance(c, Constraint) for c in constraint_seq)\n        self._constraint_seq = list(constraint_seq)\n        self._axis = axis\n\n    def check(self, value):\n        size = value.shape[self._axis]\n        value_array = np.split(value, size, axis=self._axis)\n        value_array = [constraint.check(np.squeeze(v)) for v, constraint\n                       in zip(value_array, self._constraint_seq)]\n        _value = np.stack(value_array, self._axis)\n        return _value\n\n\ndependent_property = _DependentProperty\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/dirichlet.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Dirichlet Distribution.\"\"\"\n__all__ = ['Dirichlet']\n\nfrom .exp_family import ExponentialFamily\nfrom .constraint import Positive, Simplex\nfrom .utils import gammaln, digamma, sample_n_shape_converter, _clip_float_eps\nfrom .... import np\n\n\nclass Dirichlet(ExponentialFamily):\n    r\"\"\"Create a Dirichlet distribution object.\n\n    Parameters\n    ----------\n    alpha : Tensor or scalar\n       Shape parameter of the distribution\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    has_grad = False\n    support = Simplex()\n    arg_constraints = {'alpha': Positive()}\n\n    def __init__(self, alpha, validate_args=None):\n        self.alpha = alpha\n        super(Dirichlet, self).__init__(\n            event_dim=1, validate_args=validate_args)\n\n    def sample(self, size=None):\n        if size is None:\n            size = ()\n            alpha = self.alpha\n        else:\n            if isinstance(size, int):\n                alpha = np.broadcast_to(self.alpha, (size,) + (-2,))\n            else:\n                alpha = np.broadcast_to(self.alpha, size + (-2,))\n        gamma_samples = np.random.gamma(alpha, 1)\n        s = gamma_samples.sum(-1, keepdims=True)\n        return _clip_float_eps(gamma_samples / s)\n\n    def sample_n(self, size=None):\n        alpha = self.alpha\n        if size is None:\n            return self.sample()\n        gamma_samples = np.random.gamma(\n            alpha, 1, sample_n_shape_converter(size))\n        s = gamma_samples.sum(-1, keepdims=True)\n        return _clip_float_eps(gamma_samples / s)\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        lgamma = gammaln()\n        alpha = self.alpha\n        return (np.log(value) * (alpha - 1.0)).sum(-1) +\\\n            lgamma(alpha.sum(-1)) - lgamma(alpha).sum(-1)\n\n    @property\n    def mean(self):\n        alpha = self.alpha\n        return alpha / alpha.sum(-1, keepdims=True)\n\n    @property\n    def variance(self):\n        a = self.alpha\n        s = a.sum(-1, keepdims=True)\n        return a * (s - a) / ((s + 1) * s ** 2)\n\n    def entropy(self):\n        lgamma = gammaln()\n        dgamma = digamma()\n        a0 = self.alpha.sum(-1)\n        log_B_alpha = lgamma(self.alpha).sum(-1) - lgamma(a0)\n        return (log_B_alpha + (self.alpha - 1).sum(-1) * dgamma(a0) -\n                ((self.alpha - 1) * dgamma(self.alpha)).sum(-1))\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/distribution.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Base distribution class.\"\"\"\n__all__ = ['Distribution']\n\nfrom numbers import Number\nfrom .utils import cached_property\nfrom .... import np\n\n\nclass Distribution(object):\n    r\"\"\"Base class for distribution.\n\n    Parameters\n    ----------\n    event_dim : int, default None\n        Variable indicating the dimension of the distribution's support.\n    validate_args : bool, default None\n        Whether to validate the distribution parameters\n    \"\"\"\n\n    # Variable indicating whether the sampling method has\n    # pathwise gradient.\n    has_grad = False\n    support = None\n    has_enumerate_support = False\n    arg_constraints = {}\n    _validate_args = False\n\n    @staticmethod\n    def set_default_validate_args(value):\n        if value not in [True, False]:\n            raise ValueError\n        Distribution._validate_args = value\n\n    def __init__(self, event_dim=None, validate_args=None):\n        self.event_dim = event_dim\n        if validate_args is not None:\n            self._validate_args = validate_args\n        if self._validate_args:\n            for param, constraint in self.arg_constraints.items():\n                if param not in self.__dict__ and isinstance(getattr(type(self), param),\n                                                             cached_property):\n                    # skip param that is decorated by cached_property\n                    continue\n                setattr(self, param, constraint.check(getattr(self, param)))\n        super(Distribution, self).__init__()\n\n    def log_prob(self, value):\n        r\"\"\"\n        Returns the log of the probability density/mass function evaluated at `value`.\n        \"\"\"\n        raise NotImplementedError()\n\n    def pdf(self, value):\n        r\"\"\"\n        Returns the probability density/mass function evaluated at `value`.\n        \"\"\"\n        return np.exp(self.log_prob(value))\n\n    def cdf(self, value):\n        r\"\"\"\n        Returns the cumulative density/mass function evaluated at `value`.\n        \"\"\"\n        raise NotImplementedError\n\n    def icdf(self, value):\n        r\"\"\"\n        Returns the inverse cumulative density/mass function evaluated at `value`.\n        \"\"\"\n        raise NotImplementedError\n\n    def sample(self, size=None):\n        r\"\"\"\n        Generates a `shape` shaped sample.\n        \"\"\"\n        raise NotImplementedError\n\n    def sample_n(self, size):\n        r\"\"\"\n        Generate samples of (n + parameter_shape) from the distribution.\n        \"\"\"\n        raise NotImplementedError\n\n    def broadcast_to(self, batch_shape):\n        r\"\"\"\n        Returns a new distribution instance with parameters expanded\n        to `batch_shape`. This method calls `numpy.broadcast_to` on\n        the parameters.\n\n        Parameters\n        ----------\n        batch_shape : Tuple\n            The batch shape of the desired distribution.\n\n        \"\"\"\n        raise NotImplementedError\n\n    def enumerate_support(self):\n        r\"\"\"\n        Returns a tensor that contains all values supported\n        by a discrete distribution.\n        \"\"\"\n        raise NotImplementedError\n\n    @property\n    def arg_constraints(self):\n        \"\"\"\n        Returns a dictionary from parameter names to\n        :class:`~mxnet.gluon.probability.distributions.constraint.Constraint` objects that\n        should be satisfied by each parameter of this distribution. Args that\n        are not ndarray/symbol need not appear in this dict.\n        \"\"\"\n        # pylint: disable=function-redefined\n        raise NotImplementedError\n\n    @property\n    def mean(self):\n        r\"\"\"\n        Returns the mean of the distribution.\n        \"\"\"\n        raise NotImplementedError\n\n    @property\n    def variance(self):\n        r\"\"\"\n        Returns the variance of the distribution.\n        \"\"\"\n        raise NotImplementedError\n\n    @property\n    def stddev(self):\n        \"\"\"\n        Returns the standard deviation of the distribution.\n        \"\"\"\n        return self.variance.sqrt()\n\n    @property\n    def support(self):\n        r\"\"\"\n        Returns a function representing the distribution's support.\n        \"\"\"\n        # pylint: disable=function-redefined\n        raise NotImplementedError\n\n    def entropy(self):\n        r\"\"\"\n        Returns entropy of distribution.\n        \"\"\"\n        raise NotImplementedError\n\n    def perplexity(self):\n        r\"\"\"\n        Returns perplexity of distribution.\n        \"\"\"\n        return np.exp(self.entropy())\n\n    def __repr__(self):\n        mode = self.F\n        args_string = ''\n        if 'symbol' not in mode.__name__:\n            for k in self.arg_constraints:\n                try:\n                    v = self.__dict__[k]\n                except KeyError:\n                    # TODO: Some of the keys in `arg_constraints` are cached_properties, which\n                    # are set as instance property only after they are called (hence won't\n                    # be in self.__dict__). In case they have not been called yet, we set shape\n                    # to `None` - as a quick fix, since it is not known.\n                    shape_v = None\n                else:\n                    if isinstance(v, Number):\n                        shape_v = ()\n                    else:\n                        shape_v = v.shape\n                args_string += '{}: size {}'.format(k, shape_v) + ', '\n        args_string += ', '.join(['F: {}'.format(mode.__name__),\n                                  'event_dim: {}'.format(self.event_dim)])\n        return self.__class__.__name__ + '(' + args_string + ')'\n\n    def _validate_samples(self, value):\n        \"\"\"\n        Validate samples for methods like `log_prob`, `cdf`.\n        Check if `value` lies in `self.support`\n        \"\"\"\n        return self.support.check(value)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/divergence.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"KL divergence functions.\"\"\"\n__all__ = ['register_kl', 'kl_divergence', 'empirical_kl']\n\nimport math\nimport numpy as _np\n\nfrom .... import np\nfrom .utils import gammaln, digamma\nfrom .exponential import Exponential\nfrom .pareto import Pareto\nfrom .uniform import Uniform\nfrom .normal import Normal\nfrom .laplace import Laplace\nfrom .cauchy import Cauchy\nfrom .poisson import Poisson\nfrom .geometric import Geometric\nfrom .gamma import Gamma\nfrom .dirichlet import Dirichlet\nfrom .beta import Beta\nfrom .half_normal import HalfNormal\nfrom .bernoulli import Bernoulli\nfrom .binomial import Binomial\nfrom .gumbel import Gumbel\nfrom .categorical import Categorical\nfrom .one_hot_categorical import OneHotCategorical\nfrom .multivariate_normal import MultivariateNormal\n\n\ndef empirical_kl(p, q, n_samples=1):\n    r\"\"\"Estimate KL(p||q) through monte-carlo estimation, i.e. approximate\n    KL(p||q) with:\n\n        1/M * \\Sum_{i=1}^{M} log(p(x_i) / q(x_i)), x_i ~ p(x)\n\n    Parameters\n    ----------\n    p : Distribution\n    q : Distribution\n    n_samples : int, optional\n        Number of monte-carlo samples, by default 1\n    \"\"\"\n    samples = p.sample_n(n_samples)\n    return (p.log_prob(samples) - q.log_prob(samples)).mean(0)\n\n\ndef register_kl(typeP, typeQ):\n    \"\"\"Decorator for registering custom implementation of kl divergence between\n    distribution `typeP` and `typeQ`\n\n    Returns\n    ------- function\n    \"\"\"\n    func_name = \"_kl_\" + str(typeP.__name__) \\\n                + \"_\" + str(typeQ.__name__)\n\n    def decorator(func):\n        func_arg_num = func.__code__.co_argcount\n        if (func_arg_num != 2):\n            raise TypeError('Expect kl_divergence implementation '\n                            + 'to have exactly two arguments, but got {}'.format(func_arg_num))\n        if not hasattr(_KL_storage, func_name):\n            setattr(_KL_storage, func_name, func)\n        else:\n            # Behavior TBD.\n            print(\"Error: Duplicate definition\")\n        return func\n    return decorator\n\n\ndef kl_divergence(p, q):\n    r\"\"\"\n    Return the kl divergence between p and q,\n    this method will automatically dispatch\n    to the corresponding function based on q's type.\n\n    Parameters\n    ----------\n    p : Distribution\n        lhs distribution.\n    q : Distribution\n        rhs distribution.\n\n    Returns\n    -------\n    Tensor\n        KL(p||q)\n    \"\"\"\n    func = _dispatch_kl(p.__class__.__name__, q.__class__.__name__)\n    return func(p, q) # pylint: disable=not-callable\n\n\ndef _dispatch_kl(type_p, type_q):\n    r\"\"\"KL divergence methods should be registered\n    with distribution name,\n    i.e. the implementation of KL(P(\\theta)||Q(\\theta))\n    should be named after _kl_{P}_{Q}\n\n    Parameters\n    ----------\n    type_q : Typename of a distribution\n    type_q : Typename of a distribution\n\n\n    Returns\n    -------\n    Get a class method with function name.\n    \"\"\"\n    func_name = \"_kl_\" + str(type_p) + \"_\" + str(type_q)\n    func_impl = getattr(_KL_storage, func_name, None)\n    if (not callable(func_impl)):\n        raise NotImplementedError(\n            \"KL divergence between {} and {} is not implemented.\".format(type_p, type_q))\n    return func_impl\n\n\nclass _KL_storage():\n    r\"\"\"Class for storing the definition of kl divergence\n    between distributions.\n    All the class methods should be static\n    \"\"\"\n\n    @staticmethod\n    def _kl_Normal_Normal(p, q):\n        var_ratio = (p.scale / q.scale) ** 2\n        t1 = ((p.loc - q.loc) / q.scale) ** 2\n        return 0.5 * (var_ratio + t1 - 1 - np.log(var_ratio))\n\n\n@register_kl(Bernoulli, Bernoulli)\ndef _kl_bernoulli_bernoulli(p, q):\n    prob_p = p.prob\n    prob_q = q.prob\n    t1 = prob_p * np.log(prob_p / prob_q)\n    t2 = (1 - prob_p) * np.log((1 - prob_p) / (1 - prob_q))\n    return t1 + t2\n\n\n@register_kl(Categorical, Categorical)\ndef _kl_categorical_categorical(p, q):\n    return (p.prob * (p.logit - q.logit)).sum(-1)\n\n\n@register_kl(OneHotCategorical, OneHotCategorical)\ndef _kl_onehotcategorical_onehotcategorical(p, q):\n    return _kl_categorical_categorical(p._categorical, q._categorical)\n\n\n@register_kl(Uniform, Uniform)\ndef _kl_uniform_uniform(p, q):\n    result = np.log((q.high - q.low) / (p.high - p.low))\n    result = np.where((q.low > p.low) | (q.high < p.high), _np.inf, result)\n    return result\n\n\n@register_kl(Cauchy, Cauchy)\ndef _kl_cauchy_cauchy(p, q):\n    t1 = np.log((p.scale + q.scale) ** 2 + (p.loc - q.loc) ** 2)\n    t2 = np.log(4 * p.scale * q.scale)\n    return t1 - t2\n\n\n@register_kl(Laplace, Laplace)\ndef _kl_laplace_laplace(p, q):\n    scale_ratio = p.scale / q.scale\n    loc_abs_diff = np.abs(p.loc - q.loc)\n    t1 = -np.log(scale_ratio)\n    t2 = loc_abs_diff / q.scale\n    t3 = scale_ratio * np.exp(-loc_abs_diff / p.scale)\n    return t1 + t2 + t3 - 1\n\n\n@register_kl(Poisson, Poisson)\ndef _kl_poisson_poisson(p, q):\n    t1 = p.rate * (np.log(p.rate) - np.log(q.rate))\n    t2 = (p.rate - q.rate)\n    return t1 - t2\n\n\n@register_kl(Geometric, Geometric)\ndef _kl_geometric_geometric(p, q):\n    return (-p.entropy() - np.log1p(-q.prob) / p.prob - q.logit)\n\n\n@register_kl(Exponential, Exponential)\ndef _kl_exponential_exponential(p, q):\n    scale_ratio = p.scale / q.scale\n    t1 = -np.log(scale_ratio)\n    return t1 + scale_ratio - 1\n\n\n@register_kl(Pareto, Pareto)\ndef _kl_pareto_pareto(p, q):\n    scale_ratio = p.scale / q.scale\n    alpha_ratio = q.alpha / p.alpha\n    t1 = q.alpha * np.log(scale_ratio)\n    t2 = -np.log(alpha_ratio)\n    result = t1 + t2 + alpha_ratio - 1\n    result = np.where(p.support._lower_bound <\n                      q.support._lower_bound, _np.nan, result)\n    return result\n\n\n@register_kl(Gumbel, Gumbel)\ndef _kl_gumbel_gumbel(p, q):\n    lgamma = gammaln()\n    _euler_gamma = _np.euler_gamma\n    ct1 = p.scale / q.scale\n    ct2 = q.loc / q.scale\n    ct3 = p.loc / q.scale\n    t1 = -np.log(ct1) - ct2 + ct3\n    t2 = ct1 * _euler_gamma\n    t3 = np.exp(ct2 + lgamma(1 + ct1) - ct3)\n    return t1 + t2 + t3 - (1 + _euler_gamma)\n\n\n@register_kl(Gamma, Gamma)\ndef _kl_gamma_gamma(p, q):\n    lgamma = gammaln()\n    dgamma = digamma()\n    return (\n        q.shape * np.log(q.scale / p.scale) +\n        lgamma(q.shape) - lgamma(p.shape) +\n        (p.shape - q.shape) * dgamma(p.shape) +\n        (p.shape * p.scale) * (1 / q.scale - 1 / p.scale)\n    )\n\n\n@register_kl(Beta, Beta)\ndef _kl_beta_beta(p, q):\n    lgamma = gammaln()\n    dgamma = digamma()\n    sum_params_p = p.beta + p.alpha\n    sum_params_q = q.beta + q.alpha\n    t1 = lgamma(q.alpha) + lgamma(q.beta) + lgamma(sum_params_p)\n    t2 = lgamma(p.alpha) + lgamma(p.beta) + lgamma(sum_params_q)\n    t3 = (p.beta - q.beta) * dgamma(p.beta)\n    t4 = (p.alpha - q.alpha) * dgamma(p.alpha)\n    t5 = (sum_params_q - sum_params_p) * dgamma(sum_params_p)\n    return t1 - t2 + t3 + t4 + t5\n\n# http://bariskurt.com/kullback-leibler-divergence-between-two-dirichlet-and-beta-distributions/\n\n\n@register_kl(Dirichlet, Dirichlet)\ndef _kl_dirichlet_dirichlet(p, q):\n    lgamma = gammaln()\n    dgamma = digamma()\n    sum_p_concentration = p.alpha.sum(-1)\n    sum_q_concentration = q.alpha.sum(-1)\n    t1 = lgamma(sum_p_concentration) - lgamma(sum_q_concentration)\n    t2 = (lgamma(p.alpha) - lgamma(q.alpha)).sum(-1)\n    t3 = p.alpha - q.alpha\n    t4 = dgamma(p.alpha) - np.expand_dims(dgamma(sum_p_concentration), -1)\n    return t1 - t2 + (t3 * t4).sum(-1)\n\n\n@register_kl(HalfNormal, HalfNormal)\ndef _kl_halfNormal_halfNormal(p, q):\n    var_ratio = (p.scale / q.scale) ** 2\n    t1 = ((p.loc - q.loc) / q.scale) ** 2\n    return 0.5 * (var_ratio + t1 - 1 - np.log(var_ratio))\n\n\n@register_kl(Binomial, Binomial)\ndef _kl_binomial_binomial(p, q):\n    kl = p.n * (p.prob * (p.logit - q.logit) +\n                np.log1p(-p.prob) - np.log1p(-q.prob))\n    kl = np.where(p.n > q.n, _np.inf, kl)\n    return kl\n\n\n@register_kl(MultivariateNormal, MultivariateNormal)\ndef _kl_mvn_mvn(p, q):\n    log_det = (lambda mvn:\n               np.log(\n                   np.diagonal(mvn.scale_tril, axis1=-2, axis2=-1)\n               ).sum(-1)\n               )\n    # log(det(\\Sigma_1) / det(\\Sigma_2))\n    term1 = log_det(q) - log_det(p)\n\n    # tr(inv(\\Sigma_2) * \\Sigma_1)\n    term2 = np.trace(np.matmul(q.precision, p.cov), axis1=-2, axis2=-1)\n\n    # (\\mu_2 - \\mu_1).T * inv(\\Sigma_2) * (\\mu_2 - \\mu_1)\n    diff = q.loc - p.loc\n    term3 = np.einsum(\n        '...i,...i->...',\n        diff,\n        # Batch matrix vector multiply\n        np.einsum('...jk,...j->...k', q.precision, diff)\n    ) * -0.5\n    n = np.ones_like(diff).sum(-1)  # pylint: disable=too-many-function-args\n    return 0.5 * (term1 + term2 + term3 - n)\n\n\n@register_kl(Uniform, Normal)\ndef _kl_uniform_normal(p, q):\n    common_term = p.high - p.low\n    t1 = np.log(math.sqrt(math.pi * 2) * q.scale / common_term)\n    t2 = (common_term) ** 2 / 12\n    t3 = ((p.high + p.low - 2 * q.loc) / 2) ** 2\n    return t1 + 0.5 * (t2 + t3) / (q.scale ** 2)\n\n\n@register_kl(Uniform, Gumbel)\ndef _kl_uniform_gumbel(p, q):\n    common_term = q.scale / (p.high - p.low)\n    high_loc_diff = (p.high - q.loc) / q.scale\n    low_loc_diff = (p.low - q.loc) / q.scale\n    t1 = np.log(common_term) + 0.5 * (high_loc_diff + low_loc_diff)\n    t2 = common_term * (np.exp(-high_loc_diff) - np.exp(-low_loc_diff))\n    return t1 - t2\n\n\n@register_kl(Exponential, Gumbel)\ndef _kl_exponential_gumbel(p, q):\n    scale_rate_prod = q.scale / p.scale\n    loc_scale_ratio = q.loc / q.scale\n    t1 = np.log(scale_rate_prod) - 1\n    t2 = np.exp(loc_scale_ratio) * scale_rate_prod / (scale_rate_prod + 1)\n    t3 = scale_rate_prod ** -1\n    return t1 - loc_scale_ratio + t2 + t3\n\n\n@register_kl(Exponential, Normal)\ndef _kl_exponential_normal(p, q):\n    var_normal = q.variance\n    rate_sqr = p.scale ** (-2)\n    t1 = 0.5 * np.log(rate_sqr * var_normal * 2 * _np.pi)\n    t2 = rate_sqr ** -1\n    t3 = q.loc * p.scale\n    t4 = (q.loc ** 2) * 0.5\n    return t1 - 1 + (t2 - t3 + t4) / var_normal\n\n\n@register_kl(Exponential, Gamma)\ndef _kl_exponential_gamma(p, q):\n    lgamma = gammaln()\n    ratio = p.scale / q.scale\n    t1 = -q.shape * np.log(ratio)\n    return t1 + ratio + lgamma(q.shape) + q.shape * _np.euler_gamma - (1 + _np.euler_gamma)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/exp_family.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Exponential family class\"\"\"\n__all__ = ['ExponentialFamily']\n\nfrom .distribution import Distribution\n\n\nclass ExponentialFamily(Distribution):\n    r\"\"\"\n    ExponentialFamily inherits from Distribution. ExponentialFamily is a base\n    class for distributions whose density function has the form:\n    p_F(x;\\theta) = exp(\n        <t(x), \\theta> -\n        F(\\theta) +\n        k(x)\n    ) where\n    t(x): sufficient statistics\n    \\theta: natural parameters\n    F(\\theta): log_normalizer\n    k(x): carrier measure\n    \"\"\"\n\n    @property\n    def _natural_params(self):\n        r\"\"\"\n        Return a tuple that stores natural parameters of the distribution.\n        \"\"\"\n        raise NotImplementedError\n\n    def _log_normalizer(self, *natural_params):\n        r\"\"\"\n        Return the log_normalizer F(\\theta) based the natural parameters.\n        \"\"\"\n        raise NotImplementedError\n\n    def _mean_carrier_measure(self, x):\n        r\"\"\"\n        Return the mean of carrier measure k(x) based on input x,\n        this method is required for calculating the entropy.\n        \"\"\"\n        raise NotImplementedError\n\n    def entropy(self):\n        r\"\"\"\n        Return the entropy of a distribution.\n        The entropy of distributions in exponential families\n        could be computed by:\n        H(P) = F(\\theta) - <\\theta, F(\\theta)'> - E_p[k(x)]\n        \"\"\"\n        raise NotImplementedError\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/exponential.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Exponential Distribution.\"\"\"\n__all__ = ['Exponential']\n\nfrom .exp_family import ExponentialFamily\nfrom .constraint import Positive\nfrom .utils import sample_n_shape_converter, cached_property\nfrom .... import np\n\n\nclass Exponential(ExponentialFamily):\n    r\"\"\"Create a Exponential distribution object parameterized by `scale`.\n\n    Parameters\n    ----------\n    scale : Tensor or scalar\n       Scale of the distribution. (scale = 1 /rate)\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    has_grad = True\n    support = Positive()\n    arg_constraints = {'scale': Positive()}\n\n    def __init__(self, scale=1.0, validate_args=None):\n        self.scale = scale\n        super(Exponential, self).__init__(\n            event_dim=0, validate_args=validate_args)\n\n    @cached_property\n    def rate(self):\n        return 1 / self.scale\n\n    @property\n    def mean(self):\n        return self.scale\n\n    @property\n    def variance(self):\n        return self.scale ** 2\n\n    @property\n    def stddev(self):\n        return self.scale\n\n    def sample(self, size=None):\n        return np.random.exponential(self.scale, size=size)\n\n    def sample_n(self, size=None):\n        return np.random.exponential(self.scale,\n                                     size=sample_n_shape_converter(size))\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        new_instance.scale = np.broadcast_to(self.scale, batch_shape)\n        super(Exponential, new_instance).__init__(event_dim=self.event_dim,\n                                                  validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        return np.log(self.rate) - self.rate * value\n\n    def cdf(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        return 1 - np.exp(-self.rate * value)\n\n    def icdf(self, value):\n        return - self.scale * np.log(1 - value)\n\n    def entropy(self):\n        return 1.0 + np.log(self.scale)\n\n    @property\n    def _natural_params(self):\n        return (-self.rate,)\n\n    def _log_normalizer(self, x):\n        # pylint: disable=arguments-differ\n        return -np.log(-x)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/fishersnedecor.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Snedecor's F Distribution.\"\"\"\n__all__ = ['FisherSnedecor']\n\nfrom numpy import nan\nfrom .distribution import Distribution\nfrom .gamma import Gamma\nfrom .constraint import Positive\nfrom .utils import gammaln\nfrom .... import np\n\n\n\nclass FisherSnedecor(Distribution):\n    r\"\"\"Create a FisherSnedecor distribution object, often known as F distribution.\n\n    Parameters\n    ----------\n    df1 : Tensor or scalar\n        degree of freedom parameter 1\n    scale : Tensor or scalar\n        degree of freedom parameter 2\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    support = Positive()\n    arg_constraints = {'df1': Positive(), 'df2': Positive()}\n\n    def __init__(self, df1, df2, validate_args=None):\n        self.df1 = df1\n        self.df2 = df2\n        self._gamma1 = Gamma(0.5 * self.df1, 1 / self.df1)\n        self._gamma2 = Gamma(0.5 * self.df2, 1 / self.df2)\n        super(FisherSnedecor, self).__init__(\n            event_dim=0, validate_args=validate_args)\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        new_instance.df1 = np.broadcast_to(self.df1, batch_shape)\n        new_instance.df2 = np.broadcast_to(self.df2, batch_shape)\n        new_instance._gamma1 = self._gamma1.broadcast_to(batch_shape)\n        new_instance._gamma2 = self._gamma2.broadcast_to(batch_shape)\n        super(FisherSnedecor, new_instance).__init__(event_dim=0, validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n\n    @property\n    def mean(self):\n        # mean is only defined for df2 > 2\n        df2 = np.where(self.df2 <= 2, nan, self.df2)\n        return df2 / (df2 - 2)\n\n    @property\n    def variance(self):\n        # variance is only define for df2 > 4\n        df2 = np.where(self.df2 <= 4, nan, self.df2)\n        df1 = self.df1\n        numerator = 2 * df2 ** 2 * (df1 + df2 - 2)\n        denominator = df1 * (df2 - 2) ** 2 * (df2 - 4)\n        return numerator / denominator\n\n    def sample(self, size=None):\n        X1 = self._gamma1.sample(size)\n        X2 = self._gamma2.sample(size)\n        return X1 / X2\n\n    def sample_n(self, size=None):\n        X1 = self._gamma1.sample_n(size)\n        X2 = self._gamma2.sample_n(size)\n        return X1 / X2\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        lgamma = gammaln()\n        log = np.log\n        ct1 = self.df1 / 2\n        ct2 = self.df2 / 2\n        ct3 = self.df1 / self.df2\n        t1 = lgamma(ct1 + ct2) - lgamma(ct1) - \\\n            lgamma(ct2)  # Beta(df1/2, df2/2)\n        t2 = log(ct3) * ct1 + (ct1 - 1) * log(value)\n        t3 = (ct1 + ct2) * log(ct3 * value + 1)\n        return t1 + t2 - t3\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/gamma.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Gamma Distribution.\"\"\"\n__all__ = ['Gamma']\n\nfrom .exp_family import ExponentialFamily\nfrom .constraint import Real, Positive\nfrom .utils import sample_n_shape_converter, gammaln, digamma\nfrom .... import np\n\n\n\nclass Gamma(ExponentialFamily):\n    r\"\"\"Create a Gamma distribution object.\n\n    Parameters\n    ----------\n    shape : Tensor or scalar\n        shape parameter of the distribution, often represented by `k` or `\\alpha`\n    scale : Tensor or scalar, default 1\n        scale parameter of the distribution, often represented by `\\theta`,\n        `\\theta` = 1 / `\\beta`, where `\\beta` stands for the rate parameter.\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    # TODO: Implement implicit reparameterization gradient for Gamma.\n    has_grad = False\n    support = Real()\n    arg_constraints = {'shape': Positive(), 'scale': Positive()}\n\n    def __init__(self, shape, scale=1.0, validate_args=None):\n        self.shape = shape\n        self.scale = scale\n        super(Gamma, self).__init__(\n            event_dim=0, validate_args=validate_args)\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        log_fn = np.log\n        lgamma = gammaln()\n        # alpha (concentration)\n        a = self.shape\n        # beta (rate)\n        b = 1 / self.scale\n        return a * log_fn(b) + (a - 1) * log_fn(value) - b * value - lgamma(a)\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        new_instance.shape = np.broadcast_to(self.shape, batch_shape)\n        new_instance.scale = np.broadcast_to(self.scale, batch_shape)\n        super(Gamma, new_instance).__init__(event_dim=self.event_dim,\n                                            validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n\n    def sample(self, size=None):\n        return np.random.gamma(self.shape, 1, size) * self.scale\n\n    def sample_n(self, size=None):\n        return np.random.gamma(self.shape, 1, sample_n_shape_converter(size)) * self.scale\n\n    @property\n    def mean(self):\n        return self.shape * self.scale\n\n    @property\n    def variance(self):\n        return self.shape * (self.scale ** 2)\n\n    def entropy(self):\n        lgamma = gammaln()\n        dgamma = digamma()\n        return (self.shape + np.log(self.scale) + lgamma(self.shape) +\n                (1 - self.shape) * dgamma(self.shape))\n\n    @property\n    def _natural_params(self):\n        return (self.shape - 1, -1 / self.scale)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/geometric.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Geometric distribution class.\"\"\"\n__all__ = ['Geometric']\n\nfrom numbers import Number\nfrom .distribution import Distribution\nfrom .utils import prob2logit, logit2prob, cached_property, sample_n_shape_converter\nfrom .constraint import NonNegativeInteger, Interval, Real\nfrom .... import np\n\n\nclass Geometric(Distribution):\n    r\"\"\"Create a geometric distribution object.\n\n    Parameters\n    ----------\n    prob : Tensor or scalar, default None\n        Probability of sampling `1`.\n    logit : Tensor or scalar, default None\n        The log-odds of sampling `1`.\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    support = NonNegativeInteger()\n    arg_constraints = {'prob': Interval(0, 1),\n                       'logit': Real()}\n\n    def __init__(self, prob=None, logit=None, validate_args=None):\n        if (prob is None) == (logit is None):\n            raise ValueError(\n                \"Either `prob` or `logit` must be specified, but not both. \" +\n                \"Received prob={}, logit={}\".format(prob, logit))\n\n        if prob is not None:\n            self.prob = prob\n        else:\n            self.logit = logit\n        super(Geometric, self).__init__(\n            event_dim=0, validate_args=validate_args)\n\n    @cached_property\n    def prob(self):\n        \"\"\"Get the probability of sampling `1`.\n\n        Returns\n        -------\n        Tensor\n            Parameter tensor.\n        \"\"\"\n        # pylint: disable=method-hidden\n        return logit2prob(self.logit, True)\n\n    @cached_property\n    def logit(self):\n        \"\"\"Get the log-odds of sampling `1`.\n\n        Returns\n        -------\n        Tensor\n            Parameter tensor.\n        \"\"\"\n        # pylint: disable=method-hidden\n        return prob2logit(self.prob, True)\n\n    @property\n    def mean(self):\n        return 1 / self.prob - 1\n\n    @property\n    def variance(self):\n        return (1 / self.prob - 1) / self.prob\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        if 'prob' in self.__dict__:\n            new_instance.prob = np.broadcast_to(self.prob, batch_shape)\n        else:\n            new_instance.logit = np.broadcast_to(self.logit, batch_shape)\n        super(Geometric, new_instance).__init__(event_dim=self.event_dim,\n                                                validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        prob = self.prob\n        return value * np.log1p(-prob) + np.log(prob)\n\n    def sample(self, size=None):\n        if isinstance(self.prob, Number):\n            shape_tensor = np.zeros(())\n        else:\n            shape_tensor = np.zeros_like(self.prob)  # pylint: disable=too-many-function-args\n        u = np.random.uniform(shape_tensor, size=size)\n        samples = np.floor(\n            np.log(u) / np.log1p(-self.prob)\n        )\n        return samples\n\n    def sample_n(self, size=None):\n        return self.sample(sample_n_shape_converter(size))\n\n    def entropy(self):\n        logit = self.logit\n        prob = self.prob\n        return -(logit * (prob - 1) - np.log1p(np.exp(-logit))) / prob\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/gumbel.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Gumbel Distribution.\"\"\"\n__all__ = ['Gumbel']\n\nimport math\nfrom numpy import euler_gamma # Euler-Mascheroni constant\nfrom .distribution import Distribution\nfrom .constraint import Real, Positive\nfrom .utils import sample_n_shape_converter\nfrom .... import np\n\n\nclass Gumbel(Distribution):\n    r\"\"\"Create a Gumble distribution object\n\n    Parameters\n    ----------\n    loc : Tensor or scalar, default 0\n        Location parameter of the distribution.\n    scale : Tensor or scalar, default 1\n        Scale parameter of the distribution\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    has_grad = True\n    support = Real()\n    arg_constraints = {'loc': Real(),\n                       'scale': Positive()}\n\n    def __init__(self, loc, scale=1, validate_args=None):\n        self.loc = loc\n        self.scale = scale\n        super(Gumbel, self).__init__(\n            event_dim=0, validate_args=validate_args)\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        # Standardized sample\n        y = (self.loc - value) / self.scale\n        return (y - np.exp(y)) - np.log(self.scale)\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        F = self.F\n        new_instance.loc = np.broadcast_to(self.loc, batch_shape)\n        new_instance.scale = np.broadcast_to(self.scale, batch_shape)\n        super(Gumbel, new_instance).__init__(F=F,\n                                             event_dim=self.event_dim,\n                                             validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n\n    def cdf(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        y = (value - self.loc) / self.scale\n        exp_fn = np.exp\n        return exp_fn(-exp_fn(-y))\n\n    def icdf(self, value):\n        log_fn = np.log\n        return self.loc + self.scale * (-log_fn(-log_fn(value)))\n\n    def sample(self, size=None):\n        return np.random.gumbel(self.loc, self.scale, size)\n\n    def sample_n(self, size=None):\n        return np.random.gumbel(self.loc, self.scale, sample_n_shape_converter(size))\n\n    @property\n    def mean(self):\n        return self.loc + self.scale * euler_gamma\n\n    @property\n    def stddev(self):\n        return (math.pi / math.sqrt(6)) * self.scale\n\n    @property\n    def variance(self):\n        return self.stddev ** 2\n\n    def entropy(self):\n        return np.log(self.scale) + (1 + euler_gamma)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/half_cauchy.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Half-cauchy Distribution\"\"\"\n__all__ = [\"HalfCauchy\"]\n\nimport math\nfrom numpy import inf\nfrom .transformed_distribution import TransformedDistribution\nfrom ..transformation import AbsTransform\nfrom .cauchy import Cauchy\nfrom .constraint import Positive\nfrom .... import np\n\n\nclass HalfCauchy(TransformedDistribution):\n    r\"\"\"Create a half cauchy object, where\n        X ~ Cauchy(0, scale)\n        Y = |X| ~ HalfCauchy(scale)\n\n    Parameters\n    ----------\n    scale : Tensor or scalar, default 1\n        Scale of the full Cauchy distribution.\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    has_grad = True\n    support = Positive()\n    arg_constraints = {'scale': Positive()}\n\n    def __init__(self, scale=1.0, validate_args=None):\n        base_dist = Cauchy(0, scale)\n        self.scale = scale\n        super(HalfCauchy, self).__init__(\n            base_dist, AbsTransform(), validate_args=validate_args)\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        log_prob = self._base_dist.log_prob(value) + math.log(2)\n        log_prob = np.where(value < 0, -inf, log_prob)\n        return log_prob\n\n    def cdf(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        return 2 * self._base_dist.cdf(value) - 1\n\n    def icdf(self, value):\n        return self._base_dist.icdf((value + 1) / 2)\n\n    def entropy(self):\n        return self._base_dist.entropy() - math.log(2)\n\n    @property\n    def mean(self):\n        return self.scale * math.sqrt(2 / math.pi)\n\n    @property\n    def variance(self):\n        return np.power(self.scale, 2) * (1 - 2 / math.pi)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/half_normal.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Half-normal Distribution\"\"\"\n__all__ = [\"HalfNormal\"]\n\nimport math\nfrom numpy import inf\nfrom .transformed_distribution import TransformedDistribution\nfrom ..transformation import AbsTransform\nfrom .normal import Normal\nfrom .constraint import Positive\nfrom .... import np\n\n\nclass HalfNormal(TransformedDistribution):\n    r\"\"\"Create a half normal object, where\n        X ~ Normal(0, scale)\n        Y = |X| ~ HalfNormal(scale)\n\n    Parameters\n    ----------\n    scale : Tensor or scalar, default 1\n        Scale of the full Normal distribution.\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    has_grad = True\n    support = Positive()\n    arg_constraints = {'scale': Positive()}\n\n    def __init__(self, scale=1.0, validate_args=None):\n        base_dist = Normal(0, scale)\n        self.scale = scale\n        super(HalfNormal, self).__init__(\n            base_dist, AbsTransform(), validate_args=validate_args)\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        log_prob = self._base_dist.log_prob(value) + math.log(2)\n        log_prob = np.where(value < 0, -inf, log_prob)\n        return log_prob\n\n    def cdf(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        return 2 * self._base_dist.cdf(value) - 1\n\n    def icdf(self, value):\n        return self._base_dist.icdf((value + 1) / 2)\n\n    @property\n    def loc(self):\n        return self._base_dist.loc\n\n    @property\n    def mean(self):\n        return self.scale * math.sqrt(2 / math.pi)\n\n    @property\n    def variance(self):\n        return np.power(self.scale, 2) * (1 - 2 / math.pi)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/independent.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Independent class.\"\"\"\n__all__ = ['Independent']\n\nfrom .distribution import Distribution\nfrom .constraint import dependent_property\nfrom .utils import sum_right_most\n\n\nclass Independent(Distribution):\n    r\"\"\"\n    Reinterprets some collection of independent, non-identical distributions as\n    a single multivariate random variable (convert some `batch_dim` to `event_dim`).\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    arg_constraints = {}\n\n    def __init__(self, base_distribution, reinterpreted_batch_ndims, validate_args=None):\n        event_dim = reinterpreted_batch_ndims + base_distribution.event_dim\n        self.base_dist = base_distribution\n        self.reinterpreted_batch_ndims = reinterpreted_batch_ndims\n        super(Independent, self).__init__(event_dim=event_dim,\n                                          validate_args=validate_args)\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        # we use -2 to copy the sizes of reinterpreted batch dimensions\n        reinterpreted_axes = (-2,) * self.reinterpreted_batch_ndims\n        new_instance.base_dist = self.base_dist.broadcast_to(\n            batch_shape + reinterpreted_axes)\n        new_instance.reinterpreted_batch_ndims = self.reinterpreted_batch_ndims\n        super(Independent, new_instance).__init__(event_dim=self.event_dim,\n                                                  validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n\n    @property\n    def has_enumerate_support(self):\n        if self.reinterpreted_batch_ndims > 0:\n            return False\n        return self.base_dist.has_enumerate_support\n\n    @dependent_property\n    def support(self):\n        return self.base_dist.support\n\n    @property\n    def mean(self):\n        return self.base_dist.mean\n\n    @property\n    def variance(self):\n        return self.base_dist.variance\n\n    def sample(self, size=None):\n        return self.base_dist.sample(size)\n\n    def sample_n(self, size):\n        return self.base_dist.sample_n(size)\n\n    def log_prob(self, value):\n        log_prob = self.base_dist.log_prob(value)\n        return sum_right_most(log_prob, self.reinterpreted_batch_ndims)\n\n    def entropy(self):\n        entropy = self.base_dist.entropy()\n        return sum_right_most(entropy, self.reinterpreted_batch_ndims)\n\n    def enumerate_support(self):\n        if self.reinterpreted_batch_ndims > 0:\n            raise NotImplementedError(\n                \"Enumeration over cartesian product is not implemented\")\n        return self.base_dist.enumerate_support()\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/laplace.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Laplace distribution\"\"\"\n__all__ = ['Laplace']\n\nfrom .constraint import Real, Positive\nfrom .distribution import Distribution\nfrom .utils import sample_n_shape_converter\nfrom .... import np\n\n\nclass Laplace(Distribution):\n    r\"\"\"Create a laplace distribution object.\n\n    Parameters\n    ----------\n    loc : Tensor or scalar, default 0\n        mean of the distribution.\n    scale : Tensor or scalar, default 1\n        scale of the distribution\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    has_grad = False\n    support = Real()\n    arg_constraints = {'loc': Real(), 'scale': Positive()}\n\n    def __init__(self, loc=0.0, scale=1.0, validate_args=None):\n        self.loc = loc\n        self.scale = scale\n        super(Laplace, self).__init__(\n            event_dim=0, validate_args=validate_args)\n\n    def log_prob(self, value):\n        \"\"\"Compute the log likelihood of `value`.\n\n        Parameters\n        ----------\n        value : Tensor\n            Input data.\n\n        Returns\n        -------\n        Tensor\n            Log likelihood of the input.\n        \"\"\"\n        if self._validate_args:\n            self._validate_samples(value)\n        return -np.log(2 * self.scale) - np.abs(value - self.loc) / self.scale\n\n    def sample(self, size=None):\n        r\"\"\"Generate samples of `size` from the normal distribution\n        parameterized by `self._loc` and `self._scale`\n\n        Parameters\n        ----------\n        size : Tuple, Scalar, or None\n            Size of samples to be generated. If size=None, the output shape\n            will be `broadcast(loc, scale).shape`\n\n        Returns\n        -------\n        Tensor\n            Samples from Normal distribution.\n        \"\"\"\n        return np.random.laplace(self.loc, self.scale, size)\n\n    def sample_n(self, size=None):\n        r\"\"\"Generate samples of (batch_size + broadcast(loc, scale).shape)\n        from the normal distribution parameterized by `self._loc` and `self._scale`\n\n        Parameters\n        ----------\n        size : Tuple, Scalar, or None\n            Size of independent batch to be generated from the distribution.\n\n        Returns\n        -------\n        Tensor\n            Samples from Normal distribution.\n        \"\"\"\n        return np.random.laplace(self.loc, self.scale, sample_n_shape_converter(size))\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        new_instance.loc = np.broadcast_to(self.loc, batch_shape)\n        new_instance.scale = np.broadcast_to(self.scale, batch_shape)\n        super(Laplace, new_instance).__init__(event_dim=self.event_dim,\n                                              validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n\n    def cdf(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        value = value - self.loc\n        return 0.5 - 0.5 * np.sign(value) * np.expm1(-np.abs(value) / self.scale)\n\n    def icdf(self, value):\n        value = value - 0.5\n        return self.loc - self.scale * np.sign(value) * np.log1p(-2 * np.abs(value))\n\n    @property\n    def mean(self):\n        return self.loc\n\n    @property\n    def stddev(self):\n        return (2 ** 0.5) * self.scale\n\n    @property\n    def variance(self):\n        return 2 * (self.scale ** 2)\n\n    def entropy(self):\n        return 1 + np.log(2 * self.scale)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/multinomial.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Multinomial Distribution\"\"\"\n__all__ = ['Multinomial']\n\nfrom numbers import Number\nfrom .distribution import Distribution\nfrom .one_hot_categorical import OneHotCategorical\nfrom .utils import cached_property, logit2prob, prob2logit, gammaln\nfrom .constraint import Simplex, Real, IntegerInterval\n\n\nclass Multinomial(Distribution):\n    r\"\"\"Create a multinomial distribution object.\n\n    Parameters\n    ----------\n    num_events : int\n        number of events.\n    prob : Tensor\n        probability of each event.\n    logit : Tensor\n        unnormalized probability of each event.\n    total_count : int\n        number of trials.\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    arg_constraints = {'prob': Simplex(), 'logit': Real()}\n\n    def __init__(self, num_events,\n                 prob=None, logit=None, total_count=1, validate_args=None):\n        if not isinstance(total_count, Number):\n            raise ValueError(\"Expect `total_conut` to be scalar value\")\n        self.total_count = total_count\n        if (prob is None) == (logit is None):\n            raise ValueError(\n                \"Either `prob` or `logit` must be specified, but not both. \" +\n                \"Received prob={}, logit={}\".format(prob, logit))\n        if prob is not None:\n            self.prob = prob\n        else:\n            self.logit = logit\n        self._categorical = OneHotCategorical(\n            num_events, prob, logit, validate_args)\n        super(Multinomial, self).__init__(\n            event_dim=1, validate_args=validate_args)\n\n    @property\n    def mean(self):\n        return self.prob * self.total_count\n\n    @property\n    def variance(self):\n        return self.total_count * self.prob * (1 - self.prob)\n\n    @cached_property\n    def prob(self):\n        # pylint: disable=method-hidden\n        return logit2prob(self.logit, False)\n\n    @cached_property\n    def logit(self):\n        # pylint: disable=method-hidden\n        return prob2logit(self.prob, False)\n\n    @property\n    def support(self):\n        return IntegerInterval(0, self.total_count)\n\n    def sample(self, size=None):\n        if size is not None:\n            categorical = self._categorical.broadcast_to(size)\n        else:\n            categorical = self._categorical\n        return categorical.sample_n(self.total_count).sum(0)\n\n    def sample_n(self, size=None):\n        if isinstance(size, Number):\n            size = (size,)\n        size = () if size is None else size\n        return self._categorical.sample_n((self.total_count,) + size).sum(0)\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        lgamma = gammaln()\n        log_factorial_n = lgamma(value.sum(-1) + 1)\n        log_factorial_x = lgamma(value + 1).sum(-1)\n        log_power = (self.logit * value).sum(-1)\n        return log_factorial_n - log_factorial_x + log_power\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        new_instance._categorical = self._categorical.broadcast_to(batch_shape)\n        new_instance.num_events = self.num_events\n        new_instance.total_conut = self.total_count\n        super(Multinomial, new_instance).__init__(event_dim=self.event_dim,\n                                                  validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/multivariate_normal.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Multivariate Normal Distribution\"\"\"\n__all__ = ['MultivariateNormal']\n\nimport math\nfrom .distribution import Distribution\nfrom .constraint import Real, PositiveDefinite, LowerCholesky\nfrom .utils import cached_property\nfrom .... import np\n\n\nclass MultivariateNormal(Distribution):\n    r\"\"\"Create a multivaraite Normal distribution object.\n\n    Parameters\n    ----------\n    loc : Tensor\n        mean of the distribution.\n    cov : Tensor\n        covariance matrix of the distribution\n    precision : Tensor\n        precision matrix of the distribution\n    scale_tril : Tensor\n        lower-triangular factor of the covariance\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    has_grad = True\n    support = Real()\n    arg_constraints = {'loc': Real(),\n                       'cov': PositiveDefinite(),\n                       'precision': PositiveDefinite(),\n                       'scale_tril': LowerCholesky()}\n\n    def __init__(self, loc, cov=None, precision=None, scale_tril=None, validate_args=None):\n        if (cov is not None) + (precision is not None) + (scale_tril is not None) != 1:\n            raise ValueError(\"Exactly one onf `cov` or `precision` or \" +\n                             \"`scale_tril` may be specified\")\n        self.loc = loc\n        if cov is not None:\n            self.cov = cov\n        elif precision is not None:\n            self.precision = precision\n        else:\n            self.scale_tril = scale_tril\n        super(MultivariateNormal, self).__init__(\n            event_dim=1, validate_args=validate_args)\n\n    def _precision_to_scale_tril(self, P):\n        \"\"\"\n        P = inv(L * L.T) = inv(L.T) * inv(L)\n        flip(P) = flip(inv(L.T)) * flip(inv(L))\n        flip(inv(L.T)) = Cholesky(flip(P))\n        L = flip(Cholesky(flip(P))).T\n        \"\"\"\n        L_flip_inv_T = np.linalg.cholesky(np.flip(P, (-1, -2)))\n        L = np.linalg.inv(np.flip(L_flip_inv_T, (-1, -2)).mT)\n        return L\n\n    @cached_property\n    def scale_tril(self):\n        # pylint: disable=method-hidden\n        if 'cov' in self.__dict__:\n            return np.linalg.cholesky(self.cov)\n        return self._precision_to_scale_tril(self.precision)\n\n    @cached_property\n    def cov(self):\n        # pylint: disable=method-hidden\n        if 'scale_tril' in self.__dict__:\n            return np.matmul(self.scale_tril, self.scale_tril.mT)\n        return np.linalg.inv(self.precision)\n\n    @cached_property\n    def precision(self):\n        # pylint: disable=method-hidden\n        if 'cov' in self.__dict__:\n            return np.linalg.inv(self.cov)\n        scale_tril_inv = np.linalg.inv(self.scale_tril)\n        return np.matmul(scale_tril_inv.mT, scale_tril_inv)\n\n    @property\n    def mean(self):\n        return self.loc\n\n    @property\n    def variance(self):\n        return (self.scale_tril ** 2).sum(-1)\n\n    def sample(self, size=None):\n        # symbol does not support `np.broadcast`\n        shape_tensor = self.loc + self.scale_tril.sum(-1)\n        if size is not None:\n            if isinstance(size, int):\n                size = (size,)\n            shape_tensor = np.broadcast_to(shape_tensor, size + (-2,))\n        noise = np.random.normal(np.zeros_like(  # pylint: disable=too-many-function-args\n            shape_tensor), np.ones_like(shape_tensor))  # pylint: disable=too-many-function-args\n        samples = self.loc + \\\n            np.einsum('...jk,...j->...k', self.scale_tril, noise)\n        return samples\n\n    def sample_n(self, size=None):\n        if size is None:\n            return self.sample()\n        # symbol does not support `np.broadcast`\n        shape_tensor = self.loc + self.scale_tril[..., 0]\n        if isinstance(size, int):\n            size = (size,)\n        noise = np.random.normal(np.zeros_like(shape_tensor), np.ones_like(shape_tensor),  # pylint: disable=too-many-function-args\n                                 (-2,) + size)\n        samples = self.loc + \\\n            np.einsum('...jk,...j->...k', self.scale_tril, noise)\n        return samples\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        diff = value - self.loc\n        # diff.T * inv(\\Sigma) * diff\n        M = np.einsum(\n            '...i,...i->...',\n            diff,\n            np.einsum('...jk,...j->...k', self.precision,\n                      diff)  # Batch matrix vector multiply\n        ) * -0.5\n        #   (2 * \\pi)^{-k/2} * det(\\Sigma)^{-1/2}\n        # = det(2 * \\pi * L * L.T)^{-1/2}\n        # = det(\\sqrt(2 * \\pi) * L)^{-1}\n        half_log_det = np.log(\n            np.diagonal(np.sqrt(2 * math.pi) *\n                        self.scale_tril, axis1=-2, axis2=-1)\n        ).sum(-1)\n        return M - half_log_det\n\n    def entropy(self):\n        #   det(2 * \\pi * e * \\Sigma)\n        # = det(\\sqrt(2 * \\pi * e) * L)^2\n        return np.log(np.diagonal(\n            np.sqrt(2 * math.pi * math.e) * self.scale_tril,\n            axis1=-2, axis2=-1\n        )).sum(-1)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/negative_binomial.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Negative binomial distribution class.\"\"\"\n__all__ = ['NegativeBinomial']\n\nfrom .distribution import Distribution\nfrom .poisson import Poisson\nfrom .gamma import Gamma\nfrom .utils import prob2logit, logit2prob, cached_property\nfrom .utils import gammaln\nfrom .constraint import GreaterThanEq, Interval, Real, NonNegativeInteger\nfrom .... import np\n\n\nclass NegativeBinomial(Distribution):\n    r\"\"\"Create a negative binomial distribution object.\n\n    Parameters\n    ----------\n    n : Tensor or scalar\n        Non-negative number of negative Bernoulli trials to stop.\n    prob : Tensor or scalar, default None\n        Probability of sampling `1`.\n    logit : Tensor or scalar, default None\n        The log-odds of sampling `1`.\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    support = NonNegativeInteger()\n    arg_constraints = {'n': GreaterThanEq(0),\n                       'prob': Interval(0, 1),\n                       'logit': Real()}\n\n    def __init__(self, n, prob=None, logit=None, validate_args=None):\n        if (prob is None) == (logit is None):\n            raise ValueError(\n                \"Either `prob` or `logit` must be specified, but not both. \" +\n                \"Received prob={}, logit={}\".format(prob, logit))\n\n        if prob is not None:\n            self.prob = prob\n        else:\n            self.logit = logit\n        self.n = n\n        super(NegativeBinomial, self).__init__(\n            event_dim=0, validate_args=validate_args)\n\n    @cached_property\n    def prob(self):\n        \"\"\"Get the probability of sampling `1`.\n\n        Returns\n        -------\n        Tensor\n            Parameter tensor.\n        \"\"\"\n        # pylint: disable=method-hidden\n        return logit2prob(self.logit, True)\n\n    @cached_property\n    def logit(self):\n        \"\"\"Get the log-odds of sampling `1`.\n\n        Returns\n        -------\n        Tensor\n            Parameter tensor.\n        \"\"\"\n        # pylint: disable=method-hidden\n        return prob2logit(self.prob, True)\n\n    @property\n    def mean(self):\n        return self.n * np.exp(self.logit)\n\n    @property\n    def variance(self):\n        prob = self.prob\n        return self.n * prob / (1 - prob) ** 2\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        if 'prob' in self.__dict__:\n            new_instance.prob = np.broadcast_to(self.prob, batch_shape)\n        else:\n            new_instance.logit = np.broadcast_to(self.logit, batch_shape)\n        new_instance.n = np.broadcast_to(self.n, batch_shape)\n        super(NegativeBinomial, new_instance).__init__(event_dim=self.event_dim,\n                                                       validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        lgamma = gammaln()\n        binomal_coef = lgamma(value + self.n) - \\\n            lgamma(1 + value) - lgamma(self.n)\n        # log(prob) may have numerical issue.\n        unnormalized_log_prob = self.n * \\\n            np.log(self.prob) + value * np.log1p(-self.prob)\n        return binomal_coef + unnormalized_log_prob\n\n    def sample(self, size=None):\n        # Sample via Poisson-Gamma mixture\n        rate = Gamma(shape=self.n, scale=np.exp(\n            self.logit)).sample(size)\n        return Poisson(rate).sample()\n\n    def sample_n(self, size=None):\n        # Sample via Poisson-Gamma mixture\n        rate = Gamma(shape=self.n, scale=np.exp(\n            self.logit)).sample_n(size)\n        return Poisson(rate).sample()\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/normal.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Normal distribution\"\"\"\n__all__ = ['Normal']\n\nimport math\nfrom .constraint import Real, Positive\nfrom .exp_family import ExponentialFamily\nfrom .utils import erf, erfinv\nfrom .... import np, npx\n\n\nclass Normal(ExponentialFamily):\n    r\"\"\"Create a Normal distribution object.\n\n    Parameters\n    ----------\n    loc : Tensor or scalar, default 0\n        mean of the distribution.\n    scale : Tensor or scalar, default 1\n        standard deviation of the distribution\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    has_grad = True\n    support = Real()\n    arg_constraints = {'loc': Real(), 'scale': Positive()}\n\n    def __init__(self, loc=0.0, scale=1.0, validate_args=None):\n        self.loc = loc\n        self.scale = scale\n        super(Normal, self).__init__(\n            event_dim=0, validate_args=validate_args)\n\n    def log_prob(self, value):\n        \"\"\"Compute the log likelihood of `value`.\n\n        Parameters\n        ----------\n        value : Tensor\n            Input data.\n\n        Returns\n        -------\n        Tensor\n            Log likelihood of the input.\n        \"\"\"\n        if self._validate_args:\n            self._validate_samples(value)\n        log_scale = np.log(self.scale)\n        log_prob = -((value - self.loc) ** 2) / (2 * self.variance)\n        log_prob = log_prob - log_scale\n        log_prob = log_prob - np.log(np.sqrt(2 * math.pi))\n        return log_prob\n\n    def sample(self, size=None):\n        r\"\"\"Generate samples of `size` from the normal distribution\n        parameterized by `self._loc` and `self._scale`\n\n        Parameters\n        ----------\n        size : Tuple, Scalar, or None\n            Size of samples to be generated. If size=None, the output shape\n            will be `broadcast(loc, scale).shape`\n\n        Returns\n        -------\n        Tensor\n            Samples from Normal distribution.\n        \"\"\"\n        return np.random.normal(self.loc, self.scale, size)\n\n    def sample_n(self, size=None):\n        r\"\"\"Generate samples of (batch_size + broadcast(loc, scale).shape)\n        from the normal distribution parameterized by `self._loc` and `self._scale`\n\n        Parameters\n        ----------\n        size : Tuple, Scalar, or None\n            Size of independent batch to be generated from the distribution.\n\n        Returns\n        -------\n        Tensor\n            Samples from Normal distribution.\n        \"\"\"\n        return npx.random.normal_n(self.loc, self.scale, size)\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        new_instance.loc = np.broadcast_to(self.loc, batch_shape)\n        new_instance.scale = np.broadcast_to(self.scale, batch_shape)\n        super(Normal, new_instance).__init__(event_dim=self.event_dim,\n                                             validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n\n    def cdf(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        erf_func = erf()\n        standarized_samples = ((value - self.loc) /\n                               (math.sqrt(2) * self.scale))\n        erf_term = erf_func(standarized_samples)\n        return 0.5 * (1 + erf_term)\n\n    def icdf(self, value):\n        erfinv_func = erfinv()\n        return self.loc + self.scale * erfinv_func(2 * value - 1) * math.sqrt(2)\n\n    @property\n    def mean(self):\n        return self.loc\n\n    @property\n    def stddev(self):\n        return self.scale\n\n    @property\n    def variance(self):\n        return self.scale ** 2\n\n    def entropy(self):\n        return 0.5 + 0.5 * math.log(2 * math.pi) + np.log(self.scale)\n\n    @property\n    def _natural_params(self):\n        r\"\"\"Return the natural parameters of normal distribution,\n        which are (\\frac{\\mu}{\\sigma^2}, -0.5 / (\\sigma^2))\n\n        Returns\n        -------\n        Tuple\n            Natural parameters of normal distribution.\n        \"\"\"\n        return (self.loc / (self.scale ** 2),\n                -0.5 * np.reciprocal(self.scale ** 2))\n\n    def _log_normalizer(self, x, y):\n        # pylint: disable=arguments-differ\n        return -0.25 * np.pow(x, 2) / y + 0.5 * np.log(-math.pi / y)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/one_hot_categorical.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"One-hot Categorical Distribution\"\"\"\n__all__ = ['OneHotCategorical']\n\nfrom .distribution import Distribution\nfrom .categorical import Categorical\nfrom .utils import cached_property\nfrom .constraint import Simplex, Real\nfrom .... import npx\n\n\nclass OneHotCategorical(Distribution):\n    \"\"\"Create a one-hot categorical distribution object.\n\n    Parameters\n    ----------\n    num_events : Int\n        Number of events.\n    prob : Tensor\n        Probabilities of each event.\n    logit : Tensor\n        The log-odds of each event\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    arg_constraints = {'prob': Simplex(), 'logit': Real()}\n\n    def __init__(self, num_events, prob=None, logit=None, validate_args=None):\n        if (num_events > 0):\n            num_events = int(num_events)\n            self.num_events = num_events\n        else:\n            raise ValueError(\"`num_events` should be greater than zero. \" +\n                             \"Received num_events={}\".format(num_events))\n        self._categorical = Categorical(\n            num_events, prob, logit, validate_args)\n        super(OneHotCategorical, self).__init__(\n            event_dim=1, validate_args=validate_args)\n\n    @cached_property\n    def prob(self):\n        return self._categorical.prob\n\n    @cached_property\n    def logit(self):\n        return self._categorical.logit\n\n    @property\n    def mean(self):\n        return self._categorical.prob\n\n    @property\n    def variance(self):\n        prob = self.prob\n        return prob * (1 - prob)\n\n    def sample(self, size=None):\n        indices = self._categorical.sample(size)\n        return npx.one_hot(indices, self.num_events)\n\n    def sample_n(self, size=None):\n        indices = self._categorical.sample_n(size)\n        return npx.one_hot(indices, self.num_events)\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        logit = self.logit\n        return (value * logit).sum(-1)\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        new_instance._categorical = self._categorical.broadcast_to(batch_shape)\n        new_instance.num_events = self.num_events\n        super(OneHotCategorical, new_instance).__init__(event_dim=self.event_dim,\n                                                        validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n\n    def enumerate_support(self):\n        value = self._categorical.enumerate_support()\n        return npx.one_hot(value, self.num_events)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/pareto.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Pareto Distribution.\"\"\"\n__all__ = ['Pareto']\n\nfrom .transformed_distribution import TransformedDistribution\nfrom .exponential import Exponential\nfrom .constraint import Positive, dependent_property, GreaterThan\nfrom ..transformation import ExpTransform, AffineTransform\nfrom .utils import sample_n_shape_converter\nfrom .... import np\n\n\nclass Pareto(TransformedDistribution):\n    r\"\"\"Create a Pareto Type I distribution object.\n\n    Parameters\n    ----------\n    alpha : Tensor or scalar\n        shape parameter of the distribution.\n    scale : Tensor or scalar, default 1\n        scale parameter of the distribution.\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    has_grad = True\n    arg_constraints = {'scale': Positive(),\n                       'alpha': Positive()}\n\n    def __init__(self, alpha, scale=1.0, validate_args=None):\n        self.alpha = alpha\n        self.scale = scale\n        base_dist = Exponential(1 / self.alpha)\n        super(Pareto, self).__init__(base_dist, [\n            ExpTransform(), AffineTransform(0, self.scale)])\n\n    def sample(self, size=None):\n        return self.scale * (np.random.pareto(self.alpha, size) + 1)\n\n    def sample_n(self, size=None):\n        return self.scale * (np.random.pareto(self.alpha, sample_n_shape_converter(size)) + 1)\n\n    @dependent_property\n    def support(self):\n        return GreaterThan(self.scale)\n\n    @property\n    def mean(self):\n        a = np.clip(self.alpha, 1, None)\n        return a * self.scale / (a - 1)\n\n    @property\n    def variance(self):\n        a = np.clip(self.alpha, 2, None)\n        return (self.scale ** 2) * a / ((a - 1) ** 2 * (a - 2))\n\n    def entropy(self):\n        return np.log(self.scale / self.alpha) + 1 / self.alpha + 1\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/poisson.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Poisson distribution.\"\"\"\n__all__ = ['Poisson']\n\nfrom numbers import Number\nfrom .exp_family import ExponentialFamily\nfrom .constraint import Positive, NonNegativeInteger\nfrom .utils import gammaln\nfrom .... import np, npx\n\n\nclass Poisson(ExponentialFamily):\n    r\"\"\"Create a Poisson distribution object.\n\n    Parameters\n    ----------\n    rate : Tensor or scalar, default 1\n        rate parameter of the distribution.\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    arg_constraints = {'rate': Positive()}\n    support = NonNegativeInteger()\n\n    def __init__(self, rate=1.0, validate_args=None):\n        self.rate = rate\n        super(Poisson, self).__init__(\n            event_dim=0, validate_args=validate_args)\n\n    @property\n    def mean(self):\n        return self.rate\n\n    @property\n    def variance(self):\n        return self.rate\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        new_instance.rate = np.broadcast_to(self.rate, batch_shape)\n        super(Poisson, new_instance).__init__(event_dim=self.event_dim,\n                                              validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n\n    def sample(self, size=None):\n        lam = self.rate\n        if size is None:\n            size = ()\n        if isinstance(lam, Number):\n            # Scalar case\n            return npx.scalar_poisson(lam, size)\n        else:\n            # Tensor case\n            shape_tensor = np.ones(size)\n            # shape = () currently not supported\n            return npx.tensor_poisson(lam * shape_tensor)\n\n    def sample_n(self, size=None):\n        lam = self.rate\n        if isinstance(lam, Number):\n            # Scalar case\n            if size is None:\n                size = ()\n            return npx.scalar_poisson(lam, size)\n        else:\n            return np.moveaxis(npx.tensor_poisson(lam, size), -1, 0)\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        lgamma = gammaln()\n        rate = self.rate\n        return value * np.log(rate) - rate - lgamma(value + 1)\n\n    @property\n    def _natural_params(self):\n        return (np.log(self.rate),)\n\n    def _log_normalizer(self, x):\n        # pylint: disable=arguments-differ\n        return np.exp(x)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/relaxed_bernoulli.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Relaxed Bernoulli class.\"\"\"\n__all__ = ['RelaxedBernoulli']\n\nfrom .distribution import Distribution\nfrom .transformed_distribution import TransformedDistribution\nfrom ..transformation import SigmoidTransform\nfrom .utils import prob2logit, logit2prob, cached_property\nfrom .constraint import OpenInterval, Real, Interval\nfrom .... import np\n\n\nclass _LogitRelaxedBernoulli(Distribution):\n    r\"\"\"Helper class for creating an unnormalized relaxed Bernoulli object.\n\n    Parameters\n    ----------\n    T : scalar, default None\n        Relaxation temperature\n    prob : Tensor or scalar, default None\n        Probability of sampling `1`.\n    logit : Tensor or scalar, default None\n        The log-odds of sampling `1`.\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    has_grad = True\n    support = Real()\n    arg_constraints = {'prob': Interval(0, 1),\n                       'logit': Real()}\n\n    def __init__(self, T, prob=None, logit=None, validate_args=None):\n        self.T = T\n        if (prob is None) == (logit is None):\n            raise ValueError(\n                \"Either `prob` or `logit` must be specified, but not both. \" +\n                \"Received prob={}, logit={}\".format(prob, logit))\n        if prob is not None:\n            self.prob = prob\n        else:\n            self.logit = logit\n        super(_LogitRelaxedBernoulli, self).__init__(\n            event_dim=0, validate_args=validate_args\n        )\n\n    @cached_property\n    def prob(self):\n        # pylint: disable=method-hidden\n        return logit2prob(self.logit, True)\n\n    @cached_property\n    def logit(self):\n        # pylint: disable=method-hidden\n        return prob2logit(self.prob, True)\n\n    def sample(self, size=None):\n        logit = self.logit\n        return np.random.logistic(loc=logit, scale=1, size=size) / self.T\n\n    def log_prob(self, value):\n        # log-likelihood of `value` from (Logistic(logit, 1) / T)\n        diff = self.logit - self.T * value\n        return np.log(self.T) + diff - 2 * np.log1p(np.exp(diff))\n\n\nclass RelaxedBernoulli(TransformedDistribution):\n    r\"\"\"Create a relaxed Bernoulli distribution object.\n\n    Parameters\n    ----------\n    T : scalar, default None\n        Relaxation temperature\n    prob : Tensor or scalar, default None\n        Probability of sampling `1`.\n    logit : Tensor or scalar, default None\n        The log-odds of sampling `1`.\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    has_grad = True\n    support = OpenInterval(0, 1)\n    arg_constraints = {'prob': Interval(0, 1),\n                       'logit': Real()}\n\n    def __init__(self, T, prob=None, logit=None, validate_args=None):\n        base_dist = _LogitRelaxedBernoulli(T, prob, logit, validate_args)\n        super(RelaxedBernoulli, self).__init__(base_dist, SigmoidTransform())\n\n    @property\n    def T(self):\n        return self._base_dist.T\n\n    @property\n    def prob(self):\n        return self._base_dist.prob\n\n    @property\n    def logit(self):\n        return self._base_dist.logit\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        if 'prob' in self.__dict__:\n            new_instance.prob = np.broadcast_to(self.prob, batch_shape)\n        else:\n            new_instance.logit = np.broadcast_to(self.logit, batch_shape)\n        super(RelaxedBernoulli, new_instance).__init__(event_dim=self.event_dim,\n                                                       validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/relaxed_one_hot_categorical.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Relaxed Bernoulli class.\"\"\"\n__all__ = ['RelaxedOneHotCategorical']\n\nfrom math import lgamma\nfrom .distribution import Distribution\nfrom .transformed_distribution import TransformedDistribution\nfrom ..transformation import ExpTransform\nfrom .utils import prob2logit, logit2prob, cached_property\nfrom .constraint import Real, Simplex\nfrom .... import np, npx\n\n\nclass _LogRelaxedOneHotCategorical(Distribution):\n    \"\"\"Helper class for creating the log of a\n    categorical distribution object.\n\n    Parameters\n    ----------\n    T : scalar, default None\n        Relaxation temperature\n    num_events : Int\n        Number of events.\n    prob : Tensor\n        Probabilities of each event.\n    logit : Tensor\n        The log-odds of each event\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    has_grad = True\n    arg_constraints = {'prob': Simplex(),\n                       'logit': Real()}\n\n    def __init__(self, T, num_events, prob=None, logit=None, validate_args=None):\n        self.T = T\n        if (num_events > 0):\n            num_events = int(num_events)\n            self.num_events = num_events\n        else:\n            raise ValueError(\"`num_events` should be greater than zero. \" +\n                             \"Received num_events={}\".format(num_events))\n        if (prob is None) == (logit is None):\n            raise ValueError(\n                \"Either `prob` or `logit` must be specified, but not both. \" +\n                \"Received prob={}, logit={}\".format(prob, logit))\n\n        if prob is not None:\n            self.prob = prob\n        else:\n            self.logit = logit\n\n        super(_LogRelaxedOneHotCategorical, self).__init__(\n            event_dim=1, validate_args=validate_args)\n\n    @cached_property\n    def prob(self):\n        \"\"\"Get the probability of sampling each class.\n\n        Returns\n        -------\n        Tensor\n            Parameter tensor.\n        \"\"\"\n        # pylint: disable=method-hidden\n        return logit2prob(self.logit, False)\n\n    @cached_property\n    def logit(self):\n        \"\"\"Get the log probability of sampling each class.\n\n        Returns\n        -------\n        Tensor\n            Parameter tensor.\n        \"\"\"\n        # pylint: disable=method-hidden\n        return prob2logit(self.prob, False)\n\n    def log_prob(self, value):\n        \"\"\"Compute the log-likelihood of `value`\n\n        Parameters\n        ----------\n        value : Tensor\n            samples from Relaxed Categorical distribution\n\n        Returns\n        -------\n        Tensor\n            log-likelihood of `value`\n        \"\"\"\n        K = self.num_events  # Python scalar\n        logit = self.logit\n        y = logit - value * self.T\n        log_sum_exp = np.log(np.exp(y).sum(-1, keepdims=True) + 1e-20)\n        log_scale = lgamma(K) - np.log(self.T) * (-(K - 1))\n        return (y - log_sum_exp).sum(-1) + log_scale\n\n    def sample(self, size=None):\n        if size is None:\n            size = ()\n            logit = self.logit\n        else:\n            if isinstance(size, int):\n                logit = np.broadcast_to(self.logit, (size) + (-2,))\n            else:\n                logit = np.broadcast_to(self.logit, size + (-2,))\n        scores = np.random.gumbel(logit) / self.T\n        return np.log(npx.softmax(scores, axis=-1) + 1e-20)\n\n\nclass RelaxedOneHotCategorical(TransformedDistribution):\n    \"\"\"Create a relaxed one hot categorical distribution object.\n\n    Parameters\n    ----------\n    T : scalar, default None\n        Relaxation temperature\n    num_events : Int\n        Number of events.\n    prob : Tensor\n        Probabilities of each event.\n    logit : Tensor\n        The log-odds of each event\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    has_grad = True\n    arg_constraints = {'prob': Simplex(),\n                       'logit': Real()}\n\n    def __init__(self, T, num_events, prob=None, logit=None, validate_args=None):\n        base_dist = _LogRelaxedOneHotCategorical(\n            T, num_events, prob, logit, validate_args)\n        super(RelaxedOneHotCategorical, self).__init__(\n            base_dist, ExpTransform())\n\n    @property\n    def T(self):\n        return self._base_dist.T\n\n    @property\n    def prob(self):\n        return self._base_dist.prob\n\n    @property\n    def logit(self):\n        return self._base_dist.logit\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/studentT.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Student T distribution\"\"\"\n__all__ = ['StudentT']\n\nfrom numpy import nan, inf, pi\nfrom .distribution import Distribution\nfrom .constraint import Real, Positive\nfrom .chi2 import Chi2\nfrom .utils import gammaln, digamma, sample_n_shape_converter\nfrom .... import np\n\n\nclass StudentT(Distribution):\n    r\"\"\"Create a studentT distribution object, often known as t distribution.\n\n    Parameters\n    ----------\n    df : Tensor or scalar\n        degree of freedom.\n    loc : Tensor or scalar, default 0\n        mean of the distribution.\n    scale : Tensor or scalar, default 1\n        scale of the distribution\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    support = Real()\n    arg_constraints = {'df': Positive(), 'loc': Real(), 'scale': Real()}\n\n    def __init__(self, df, loc=0.0, scale=1.0, validate_args=None):\n        self.df = df\n        self.loc = loc\n        self.scale = scale\n        self._chi2 = Chi2(self.df)\n        super(StudentT, self).__init__(\n            event_dim=0, validate_args=validate_args)\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        new_instance.loc = np.broadcast_to(self.loc, batch_shape)\n        new_instance.scale = np.broadcast_to(self.scale, batch_shape)\n        new_instance.df = np.broadcast_to(self.df, batch_shape)\n        new_instance._chi2 = self._chi2.broadcast_to(batch_shape)\n        super(StudentT, new_instance).__init__(\n            event_dim=0, validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n\n    @property\n    def mean(self):\n        # mean is only defined for df > 1\n        m = np.where(self.df <= 1, nan, self.loc)\n        return m\n\n    @property\n    def variance(self):\n        df = self.df\n        v = self.scale ** 2 * self.df / (self.df - 2)\n        v = np.where(df <= 2, inf, v)\n        v = np.where(df <= 1, nan, v)\n        return v\n\n    def sample(self, size=None):\n        X = np.random.normal(size=size)\n        Z = self._chi2.sample(size)\n        Y = X * np.sqrt(self.df / Z)\n        return self.loc + Y * self.scale\n\n    def sample_n(self, size=None):\n        return self.sample(sample_n_shape_converter(size))\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        lgamma = gammaln()\n        df = self.df\n        value = (value - self.loc) / self.scale\n        return (\n            lgamma((df + 1) / 2) - lgamma(df / 2) -\n            np.log(self.scale) - 0.5 * np.log(df * pi)\n            - 0.5 * (df + 1) * np.log1p(value ** 2 / df)\n        )\n\n    def entropy(self):\n        lgamma = gammaln()\n        dgamma = digamma()\n        log_fn = np.log\n        lbeta = lgamma(0.5 * self.df) + lgamma(0.5) - \\\n            lgamma(0.5 * (self.df + 1))\n        return (log_fn(self.scale) +\n                0.5 * (self.df + 1) *\n                (dgamma(0.5 * (self.df + 1)) - dgamma(0.5 * self.df)) +\n                0.5 * log_fn(self.df) + lbeta)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/transformed_distribution.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Transformed distribution\"\"\"\n__all__ = ['TransformedDistribution']\n\nfrom ..transformation import Transformation\nfrom .distribution import Distribution\nfrom .utils import sum_right_most\nfrom .... import np\n\n\nclass TransformedDistribution(Distribution):\n    \"\"\"A distribution generated by applying a sequence of transformations to\n    a base distribution/\n\n    Parameters\n    ----------\n    base_dist : Distribution\n        Base distribution\n    transforms : Transformation or List\n        Transformation to be applied\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    def __init__(self, base_dist, transforms, validate_args=None):\n        self._base_dist = base_dist\n        if isinstance(transforms, Transformation):\n            transforms = [transforms, ]\n        self._transforms = transforms\n        event_dim = max([self._base_dist.event_dim] +\n                        [t.event_dim for t in self._transforms])\n        super(TransformedDistribution, self).__init__(\n            event_dim=event_dim, validate_args=validate_args)\n\n    def sample(self, size=None):\n        x = self._base_dist.sample(size)\n        for t in self._transforms:\n            x = t(x)\n        return x\n\n    def sample_n(self, size=None):\n        x = self._base_dist.sample_n(size)\n        for t in self._transforms:\n            x = t(x)\n        return x\n\n    def log_prob(self, value):\n        \"\"\"\n        Compute log-likelihood of `value` with `log_det_jacobian` and\n        log-likelihood of the base distribution according to the following conclusion:\n\n        Given that Y = T(X),\n        log(p(y)) = log(p(x)) - log(|dy/dx|)\n        \"\"\"\n        log_prob = 0.0\n        y = value  # T_n(T_{n-1}(...T_1(x)))\n        # Reverse `_transforms` to transform to the base distribution.\n        for t in reversed(self._transforms):\n            x = t.inv(y)\n            log_prob = log_prob - sum_right_most(t.log_det_jacobian(x, y),\n                                                 self.event_dim - t.event_dim)\n            y = x\n        log_prob = log_prob + sum_right_most(self._base_dist.log_prob(y),\n                                             self.event_dim - self._base_dist.event_dim)\n        return log_prob\n\n    def cdf(self, value):\n        \"\"\"\n        Compute the cumulative distribution function(CDF) p(Y < `value`)\n        \"\"\"\n        sign = np.ones_like(value)  # pylint: disable=too-many-function-args\n        for t in reversed(self._transforms):\n            value = t.inv(value)\n            sign = sign * t.sign\n        value = self._base_dist.cdf(value)\n        return sign * (value - 0.5) + 0.5\n\n    def icdf(self, value):\n        sign = np.ones_like(value)  # pylint: disable=too-many-function-args\n        for t in self._transforms:\n            sign = sign * t.sign\n        value = sign * (value - 0.5) + 0.5  # value or (1 - value)\n        samples_base = self._base_dist.icdf(value)\n        for t in self._transforms:\n            samples_base = t(samples_base)\n        return samples_base\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/uniform.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Uniform distribution\"\"\"\n__all__ = ['Uniform']\n\nfrom .distribution import Distribution\nfrom .constraint import Real, Interval\nfrom .utils import sample_n_shape_converter\nfrom .... import np\n\n\nclass Uniform(Distribution):\n    r\"\"\"Create a uniform distribution object.\n\n    Parameters\n    ----------\n    low : Tensor or scalar, default 0\n        lower range of the distribution.\n    high : Tensor or scalar, default 1\n        upper range of the distribution.\n    \"\"\"\n    # pylint: disable=abstract-method\n\n    # Reparameterization gradient for Uniform is currently not implemented\n    # in the backend at this moment.\n    has_grad = False\n    arg_constraints = {'low': Real(), 'high': Real()}\n\n    def __init__(self, low=0.0, high=1.0, validate_args=None):\n        self.low = low\n        self.high = high\n        super(Uniform, self).__init__(\n            event_dim=0, validate_args=validate_args)\n\n    def log_prob(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        def type_converter(x):\n            return float(x) if isinstance(x, bool) else x.astype('float')\n        lower_bound = type_converter(self.low < value)\n        upper_bound = type_converter(self.high > value)\n        # 0 if value \\in [low, high], -inf otherwise.\n        out_of_support_value = np.log(lower_bound * upper_bound)\n        return out_of_support_value - np.log(self.high - self.low)\n\n    def sample(self, size=None):\n        return np.random.uniform(self.low, self.high, size=size)\n\n    def sample_n(self, size=None):\n        return np.random.uniform(self.low, self.high,\n                                 size=sample_n_shape_converter(size))\n\n    @property\n    def support(self):\n        return Interval(self.low, self.high)\n\n    def broadcast_to(self, batch_shape):\n        new_instance = self.__new__(type(self))\n        new_instance.low = np.broadcast_to(self.low, batch_shape)\n        new_instance.high = np.broadcast_to(self.high, batch_shape)\n        super(Uniform, new_instance).__init__(event_dim=self.event_dim,\n                                              validate_args=False)\n        new_instance._validate_args = self._validate_args\n        return new_instance\n\n    def cdf(self, value):\n        if self._validate_args:\n            self._validate_samples(value)\n        x = (value - self.low) / (self.high - self.low)\n        return x.clip(0, 1)\n\n    def icdf(self, value):\n        return value * (self.high - self.low) + self.low\n\n    def entropy(self):\n        return np.log(self.high - self.low)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/utils.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Distribution utilities\"\"\"\n__all__ = ['prob2logit', 'logit2prob', 'cached_property', 'sample_n_shape_converter',\n           'constraint_check', 'digamma', 'gammaln', 'erfinv', 'erf']\n\nfrom functools import update_wrapper\nfrom numbers import Number\nimport numpy as onp\ntry:\n    import scipy.special as sc\nexcept ImportError:\n    sc = None\nfrom .... import np, npx\n\n\ndef constraint_check():\n    \"\"\"Unified check_constraint interface for both scalar and tensor\n    \"\"\"\n    def _check(condition, err_msg):\n        if isinstance(condition, bool):\n            if not condition:\n                raise ValueError(err_msg)\n            return 1.0\n        return npx.constraint_check(condition, err_msg)\n    return _check\n\n\ndef digamma():\n    \"\"\"Unified digamma interface for both scalar and tensor\n    \"\"\"\n    def compute(value):\n        \"\"\"Return digamma(value)\n        \"\"\"\n        if isinstance(value, Number):\n            if sc is not None:\n                return sc.digamma(value, dtype='float32')\n            else:\n                raise ValueError('Numbers are not supported as input if scipy is not installed')\n        return npx.digamma(value)\n    return compute\n\n\ndef gammaln():\n    \"\"\"Unified gammaln interface for both scalar and tensor\n    \"\"\"\n    def compute(value):\n        \"\"\"Return log(gamma(value))\n        \"\"\"\n        if isinstance(value, Number):\n            if sc is not None:\n                return sc.gammaln(value, dtype='float32')\n            else:\n                raise ValueError('Numbers are not supported as input if scipy is not installed')\n        return npx.gammaln(value)\n    return compute\n\n\ndef erf():\n    \"\"\"Unified erf interface for both scalar and tensor\n    \"\"\"\n    def compute(value):\n        if isinstance(value, Number):\n            if sc is not None:\n                return sc.erf(value)\n            else:\n                raise ValueError('Numbers are not supported as input if scipy is not installed')\n        return npx.erf(value)\n    return compute\n\n\ndef erfinv():\n    \"\"\"Unified erfinv interface for both scalar and tensor\n    \"\"\"\n    def compute(value):\n        if isinstance(value, Number):\n            if sc is not None:\n                return sc.erfinv(value)\n            else:\n                raise ValueError('Numbers are not supported as input if scipy is not installed')\n        return npx.erfinv(value)\n    return compute\n\n\ndef sample_n_shape_converter(size):\n    \"\"\"Convert `size` to the proper format for performing sample_n.\n    \"\"\"\n    if size is None:\n        return size\n    if size == ():\n        size = None\n    else:\n        if isinstance(size, int):\n            size = (size,)\n        size = (-2,) + size\n    return size\n\n\ndef sum_right_most(x, ndim):\n    \"\"\"Sum along the right most `ndim` dimensions of `x`,\n\n    Parameters\n    ----------\n    x : Tensor\n        Input tensor.\n    ndim : Int\n        Number of dimensions to be summed.\n\n    Returns\n    -------\n    Tensor\n    \"\"\"\n    if ndim == 0:\n        return x\n    axes = list(range(-ndim, 0))\n    return x.sum(axes)\n\n\ndef _clip_prob(prob):\n    eps = onp.finfo('float32').eps\n    return np.clip(prob, eps, 1 - eps)\n\n\ndef _clip_float_eps(value):\n    eps = onp.finfo('float32').eps\n    return np.maximum(value, eps)\n\n\ndef prob2logit(prob, binary=True):\n    r\"\"\"Convert probability to logit form.\n    For the binary case, the logit stands for log(p / (1 - p)).\n    Whereas for the multinomial case, the logit denotes log(p).\n    \"\"\"\n    _clipped_prob = _clip_prob(prob)\n    if binary:\n        return np.log(_clipped_prob) - np.log1p(-_clipped_prob)\n    # The clipped prob would cause numerical error in the categorical case,\n    # no idea about the reason behind.\n    return np.log(_clipped_prob)\n\n\ndef logit2prob(logit, binary=True):\n    r\"\"\"Convert logit into probability form.\n    For the binary case, `sigmoid()` is applied on the logit tensor.\n    Whereas for the multinomial case, `softmax` is applied along the last\n    dimension of the logit tensor.\n    \"\"\"\n    if binary:\n        return npx.sigmoid(logit)\n    return npx.softmax(logit)\n\n\nclass _CachedProperty(object):\n    r\"\"\"Use as a decorator for loading class attribute, but caches the value.\"\"\"\n\n    def __init__(self, func):\n        self._func = func\n        update_wrapper(self, self._func)\n\n    def __get__(self, instance, cls=None):\n        if instance is None:\n            return self\n        value = self._func(instance)\n        setattr(instance, self._func.__name__, value)\n        return value\n\n\ncached_property = _CachedProperty\n"
  },
  {
    "path": "python/mxnet/gluon/probability/distributions/weibull.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Weibull Distribution.\"\"\"\n__all__ = ['Weibull']\n\n# Euler-Mascheroni constant\nfrom numpy import euler_gamma\nfrom .transformed_distribution import TransformedDistribution\nfrom .exponential import Exponential\nfrom .constraint import Positive\nfrom ..transformation import PowerTransform, AffineTransform\nfrom .utils import sample_n_shape_converter, gammaln\nfrom .... import np, npx\n\n\nclass Weibull(TransformedDistribution):\n    r\"\"\"Create a two parameter Weibull distribution object.\n\n    Parameters\n    ----------\n    concentration : Tensor or scalar\n        Concentration/shape parameter of the distribution.\n    scale : Tensor or scalar, default 1\n        scale parameter of the distribution.\n    \"\"\"\n    # pylint: disable=abstract-method\n    has_grad = True\n    support = Positive()\n    arg_constraints = {'scale': Positive(),\n                       'concentration': Positive()}\n\n    def __init__(self, concentration, scale=1.0, validate_args=None):\n        self.concentration = concentration\n        self.scale = scale\n        base_dist = Exponential()\n        super(Weibull, self).__init__(base_dist, [PowerTransform(1 / self.concentration),\n                                                  AffineTransform(0, self.scale)])\n\n    def sample(self, size=None):\n        return self.scale * np.random.weibull(self.concentration, size)\n\n    def sample_n(self, size=None):\n        return self.scale * np.random.weibull(self.concentration,\n                                              sample_n_shape_converter(size))\n\n    @property\n    def mean(self):\n        return self.scale * np.exp(npx.gammaln(1 + 1 / self.concentration))\n\n    @property\n    def variance(self):\n        exp = np.exp\n        lgamma = gammaln()\n        term1 = exp(lgamma(1 + 2 / self.concentration))\n        term2 = exp(2 * lgamma(1 + 1 / self.concentration))\n        return (self.scale ** 2) * (term1 - term2)\n\n    def entropy(self):\n        return (euler_gamma * (1 - 1 / self.concentration) +\n                np.log(self.scale / self.concentration) + 1)\n"
  },
  {
    "path": "python/mxnet/gluon/probability/transformation/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Transformation classes.\"\"\"\n\nfrom .transformation import *\nfrom .domain_map import *\n"
  },
  {
    "path": "python/mxnet/gluon/probability/transformation/domain_map.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Classes for registering and storing bijection/transformations from\nunconstrained space to a given domain.\n\"\"\"\n\nfrom numbers import Number\nfrom .transformation import (\n    ExpTransform, AffineTransform, SigmoidTransform, ComposeTransform)\nfrom ..distributions.constraint import (Constraint, Positive, GreaterThan, GreaterThanEq,\n                                        LessThan, Interval, HalfOpenInterval)\n\n\n__all__ = ['domain_map', 'biject_to', 'transform_to']\n\n\nclass domain_map():\n    \"\"\"\n    Abstract Class for registering and storing mappings from domain\n    to bijections/transformations\n    \"\"\"\n    def __init__(self):\n        # constraint -> constraint -> transformation\n        self._storage = {}\n        super(domain_map, self).__init__()\n\n    def register(self, constraint, factory=None):\n        \"\"\"Register a bijection/transformation from unconstrained space to the domain\n        specified by `constraint`.\n\n        Parameters\n        ----------\n        constraint : Type or Object\n            A class of constraint or an object of constraint\n        factory : callable\n            A function that outputs a `transformation` given a `constraint`,\n            by default None.\n        \"\"\"\n        # Decorator mode\n        if factory is None:\n            return lambda factory: self.register(constraint, factory)\n\n        if isinstance(constraint, Constraint):\n            constraint = type(constraint)\n\n        if not isinstance(constraint, type) or not issubclass(constraint, Constraint):\n            raise TypeError('Expected constraint to be either a Constraint subclass or instance, '\n                            'but got {}'.format(constraint))\n\n        self._storage[constraint] = factory\n        return factory\n\n    def __call__(self, constraint):\n        try:\n            factory = self._storage[type(constraint)]\n        except KeyError:\n            raise NotImplementedError(\n                'Cannot transform {} constraints'.format(type(constraint).__name__))\n        return factory(constraint)\n\n\nbiject_to = domain_map()\ntransform_to = domain_map()\n\n\n@biject_to.register(Positive)\n@transform_to.register(Positive)\ndef _transform_to_positive(constraint):\n    # Although `constraint` is not used in this factory function,\n    # we decide to keep it for the purpose of consistency.\n    # pylint: disable=unused-argument\n    return ExpTransform()\n\n\n@biject_to.register(GreaterThan)\n@biject_to.register(GreaterThanEq)\n@transform_to.register(GreaterThan)\n@transform_to.register(GreaterThanEq)\ndef _transform_to_greater_than(constraint):\n    return ComposeTransform([ExpTransform(),\n                             AffineTransform(constraint._lower_bound, 1)])\n\n\n@biject_to.register(LessThan)\n@transform_to.register(LessThan)\ndef _transform_to_less_than(constraint):\n    return ComposeTransform([ExpTransform(),\n                             AffineTransform(constraint._upper_bound, -1)])\n\n\n@biject_to.register(Interval)\n@biject_to.register(HalfOpenInterval)\n@transform_to.register(Interval)\n@transform_to.register(HalfOpenInterval)\ndef _transform_to_interval(constraint):\n    # Handle the special case of the unit interval.\n    lower_is_0 = isinstance(constraint._lower_bound,\n                            Number) and constraint._lower_bound == 0\n    upper_is_1 = isinstance(constraint._upper_bound,\n                            Number) and constraint._upper_bound == 1\n    if lower_is_0 and upper_is_1:\n        return SigmoidTransform()\n\n    loc = constraint._lower_bound\n    scale = constraint._upper_bound - constraint._lower_bound\n    return ComposeTransform([SigmoidTransform(),\n                             AffineTransform(loc, scale)])\n"
  },
  {
    "path": "python/mxnet/gluon/probability/transformation/transformation.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=abstract-method\n# pylint: disable=arguments-differ\n\"\"\"Transformation Classes\"\"\"\n__all__ = [\"Transformation\", \"TransformBlock\", \"ComposeTransform\", \"ExpTransform\",\n           \"AffineTransform\", \"PowerTransform\", \"AbsTransform\", 'SigmoidTransform',\n           'SoftmaxTransform']\n\nimport weakref\nfrom ..distributions.utils import _clip_prob, cached_property, sum_right_most\nfrom ...block import HybridBlock\nfrom .... import np, npx\n\n\nclass Transformation(object):\n    r\"\"\"Abstract class for implementing invertible transformation\n    with computable log  det jacobians\n\n    Attributes\n    ----------\n    bijective : bool\n\n    \"\"\"\n    bijective = False\n    event_dim = 0\n\n    def __init__(self):\n        self._inv = None\n        super(Transformation, self).__init__()\n\n    @property\n    def sign(self):\n        \"\"\"\n        Returns the sign of the determinant of the Jacobian.\n        \"\"\"\n        raise NotImplementedError\n\n    @property\n    def inv(self):\n        inv = None\n        if self._inv is not None:\n            inv = self._inv()\n        if inv is None:\n            inv = _InverseTransformation(self)\n            self._inv = weakref.ref(inv)\n        return inv\n\n    def __call__(self, x):\n        return self._forward_compute(x)\n\n    def _inv_call(self, y):\n        return self._inverse_compute(y)\n\n    def _forward_compute(self, x):\n        raise NotImplementedError\n\n    def _inverse_compute(self, x):\n        raise NotImplementedError\n\n    def log_det_jacobian(self, x, y):\n        \"\"\"\n        Compute the value of log(|dy/dx|)\n        \"\"\"\n        raise NotImplementedError\n\n\nclass _InverseTransformation(Transformation):\n    \"\"\"\n    A private class representing the invert of `Transformation`,\n    which should be accessed through `Transformation.inv` property.\n    \"\"\"\n\n    def __init__(self, forward_transformation):\n        super(_InverseTransformation, self).__init__()\n        self._inv = forward_transformation\n\n    @property\n    def inv(self):\n        return self._inv\n\n    @property\n    def sign(self):\n        return self._inv.sign\n\n    @property\n    def event_dim(self):\n        return self._inv.event_dim\n\n    def __call__(self, x):\n        return self._inv._inverse_compute(x)\n\n    def log_det_jacobian(self, x, y):\n        return -self._inv.log_det_jacobian(y, x)\n\n\nclass TransformBlock(Transformation, HybridBlock):\n    \"\"\"Transform with learnable parameters should inherit from this class\n    rather than `Transformation`.\n    For example: normalization flow.\n    \"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n\n\nclass ComposeTransform(Transformation):\n    r\"\"\"\n    Composes multiple transforms in a chain.\n    \"\"\"\n    def __init__(self, parts):\n        super(ComposeTransform, self).__init__()\n        self._parts = parts\n\n    def _forward_compute(self, x):\n        for t in self._parts:\n            x = t(x)\n        return x\n\n    # @cached_property is, in essence, @property with lazy evaluation.\n    # pylint: disable=invalid-overridden-method\n    @cached_property\n    def sign(self):\n        sign = 1\n        for p in self._parts:\n            sign = sign * p.sign\n        return sign\n\n    @cached_property\n    def event_dim(self):\n        return max(p.event_dim for p in self._parts) if self._parts else 0\n\n    @property\n    def inv(self):\n        inv = None\n        if self._inv is not None:\n            inv = self._inv()\n        if inv is None:\n            inv = ComposeTransform([t.inv for t in reversed(self._parts)])\n            self._inv = weakref.ref(inv)\n            inv._inv = weakref.ref(self)\n        return inv\n\n    def log_det_jacobian(self, x, y):\n        if not self._parts:\n            return np.zeros_like(x)  # pylint: disable=too-many-function-args\n        result = 0\n        x_prime = None\n        for t in self._parts[:-1]:\n            x_prime = t(x)\n            result = result + sum_right_most(t.log_det_jacobian(x, x_prime),\n                                             self.event_dim - t.event_dim)\n            x = x_prime\n        t_last = self._parts[-1]\n        result = result + sum_right_most(t_last.log_det_jacobian(x, y),\n                                         self.event_dim - t_last.event_dim)\n\n        return result\n\n\nclass ExpTransform(Transformation):\n    r\"\"\"\n    Perform the exponential transform: y = exp{x}.\n    \"\"\"\n    bijective = True\n    sign = 1\n\n    def _forward_compute(self, x):\n        return np.exp(x)\n\n    def _inverse_compute(self, y):\n        return np.log(y)\n\n    def log_det_jacobian(self, x, y):\n        return x\n\n\nclass AffineTransform(Transformation):\n    r\"\"\"\n    Perform *pointwise* affine transform: y = loc + scale * x.\n    \"\"\"\n    bijective = True\n\n    def __init__(self, loc, scale, event_dim=0):\n        super(AffineTransform, self).__init__()\n        self._loc = loc\n        self._scale = scale\n        self.event_dim = event_dim\n\n    def _forward_compute(self, x):\n        return self._loc + self._scale * x\n\n    def _inverse_compute(self, y):\n        return (y - self._loc) / self._scale\n\n    def log_det_jacobian(self, x, y):\n        # element-wise abs(log(dy/dx))\n        value = np.ones_like(x) * np.log(np.abs(self._scale))  # pylint: disable=too-many-function-args\n        return sum_right_most(value, self.event_dim)\n\n    @property\n    def sign(self):\n        return np.sign(self._scale)\n\n\nclass PowerTransform(Transformation):\n    r\"\"\"\n    Perform *pointwise* power transform: y = pow(x, exponent).\n    \"\"\"\n    bijective = True\n    sign = 1\n\n    def __init__(self, exponent):\n        super(PowerTransform, self).__init__()\n        self._exponent = exponent\n\n    def _forward_compute(self, x):\n        return np.power(x, self._exponent)\n\n    def _inverse_compute(self, y):\n        return np.power(y, 1 / self._exponent)\n\n    def log_det_jacobian(self, x, y):\n        log_fn = np.log\n        abs_fn = np.abs\n        return log_fn(abs_fn(self._exponent * y / x))\n\n\nclass SigmoidTransform(Transformation):\n    r\"\"\"\n    Perform *pointwise* sigmoid transform: y = 1 / (1 + exp(-x)).\n    \"\"\"\n    bijective = True\n    sign = 1\n\n    def _forward_compute(self, x):\n        return _clip_prob(npx.sigmoid(x))\n\n    def _inverse_compute(self, y):\n        clipped_prob = _clip_prob(y)\n        return np.log(clipped_prob) - np.log1p(-clipped_prob)\n\n    def log_det_jacobian(self, x, y):\n        softplus_fn = lambda x: np.log(1 + np.exp(x))\n        return -softplus_fn(-x) - softplus_fn(x)\n\n\nclass SoftmaxTransform(Transformation):\n    event_dim = 1\n\n    def _forward_compute(self, x):\n        return npx.softmax(x, -1)\n\n    def _inverse_compute(self, y):\n        return np.log(y)\n\n\nclass AbsTransform(Transformation):\n    def _forward_compute(self, x):\n        return np.abs(x)\n\n    def _inverse_compute(self, y):\n        return y\n"
  },
  {
    "path": "python/mxnet/gluon/rnn/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Recurrent neural network module.\"\"\"\n\nfrom .rnn_cell import *\nfrom .conv_rnn_cell import *\nfrom .rnn_layer import *\n"
  },
  {
    "path": "python/mxnet/gluon/rnn/conv_rnn_cell.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=arguments-differ, too-many-lines\n# coding: utf-8\n\"\"\"Definition of various recurrent neural network cells.\"\"\"\n__all__ = ['Conv1DRNNCell', 'Conv2DRNNCell', 'Conv3DRNNCell',\n           'Conv1DLSTMCell', 'Conv2DLSTMCell', 'Conv3DLSTMCell',\n           'Conv1DGRUCell', 'Conv2DGRUCell', 'Conv3DGRUCell']\n\n\nfrom math import floor\n\nfrom ...base import numeric_types\nfrom .rnn_cell import HybridRecurrentCell\nfrom ..parameter import Parameter\nfrom ... import np, npx\nfrom ...util import use_np\n\n\ndef _get_conv_out_size(dimensions, kernels, paddings, dilations):\n    return tuple(int(floor(x+2*p-d*(k-1)-1)+1) if x else 0 for x, k, p, d in\n                 zip(dimensions, kernels, paddings, dilations))\n\n\n@use_np\nclass _BaseConvRNNCell(HybridRecurrentCell):\n    \"\"\"Abstract base class for convolutional RNNs\"\"\"\n    def __init__(self, input_shape, hidden_channels,\n                 i2h_kernel, h2h_kernel,\n                 i2h_pad, i2h_dilate, h2h_dilate,\n                 i2h_weight_initializer, h2h_weight_initializer,\n                 i2h_bias_initializer, h2h_bias_initializer,\n                 dims,\n                 conv_layout, activation):\n        super(_BaseConvRNNCell, self).__init__()\n\n        self._hidden_channels = hidden_channels\n        self._input_shape = input_shape\n        self._conv_layout = conv_layout\n        self._activation = activation\n\n        # Convolution setting\n        assert all(isinstance(spec, int) or len(spec) == dims\n                   for spec in [i2h_kernel, i2h_pad, i2h_dilate,\n                                h2h_kernel, h2h_dilate]), \\\n               \"For {dims}D convolution, the convolution settings can only be either int \" \\\n               \"or list/tuple of length {dims}\".format(dims=dims)\n\n        self._i2h_kernel = (i2h_kernel,) * dims if isinstance(i2h_kernel, numeric_types) \\\n                           else i2h_kernel\n        self._stride = (1,) * dims\n        self._i2h_pad = (i2h_pad,) * dims if isinstance(i2h_pad, numeric_types) \\\n                        else i2h_pad\n        self._i2h_dilate = (i2h_dilate,) * dims if isinstance(i2h_dilate, numeric_types) \\\n                           else i2h_dilate\n        self._h2h_kernel = (h2h_kernel,) * dims if isinstance(h2h_kernel, numeric_types) \\\n                           else h2h_kernel\n        assert all(k % 2 == 1 for k in self._h2h_kernel), \\\n            f\"Only support odd number, get h2h_kernel= {str(h2h_kernel)}\"\n        self._h2h_dilate = (h2h_dilate,) * dims if isinstance(h2h_dilate, numeric_types) \\\n                           else h2h_dilate\n\n        self._channel_axis, \\\n        self._in_channels, \\\n        i2h_param_shape, \\\n        h2h_param_shape, \\\n        self._h2h_pad, \\\n        self._state_shape = self._decide_shapes()\n\n        self.i2h_weight = Parameter('i2h_weight', shape=i2h_param_shape,\n                                    init=i2h_weight_initializer,\n                                    allow_deferred_init=True)\n        self.h2h_weight = Parameter('h2h_weight', shape=h2h_param_shape,\n                                    init=h2h_weight_initializer,\n                                    allow_deferred_init=True)\n        self.i2h_bias = Parameter('i2h_bias', shape=(hidden_channels*self._num_gates,),\n                                  init=i2h_bias_initializer,\n                                  allow_deferred_init=True)\n        self.h2h_bias = Parameter('h2h_bias', shape=(hidden_channels*self._num_gates,),\n                                  init=h2h_bias_initializer,\n                                  allow_deferred_init=True)\n\n    def _decide_shapes(self):\n        channel_axis = self._conv_layout.find('C')\n        input_shape = self._input_shape\n        in_channels = input_shape[channel_axis - 1]\n        hidden_channels = self._hidden_channels\n        if channel_axis == 1:\n            dimensions = input_shape[1:]\n        else:\n            dimensions = input_shape[:-1]\n\n        total_out = hidden_channels * self._num_gates\n\n        i2h_param_shape = (total_out,)\n        h2h_param_shape = (total_out,)\n        state_shape = (hidden_channels,)\n        conv_out_size = _get_conv_out_size(dimensions,\n                                           self._i2h_kernel,\n                                           self._i2h_pad,\n                                           self._i2h_dilate)\n        h2h_pad = tuple(d*(k-1)//2 for d, k in zip(self._h2h_dilate, self._h2h_kernel))\n        if channel_axis == 1:\n            i2h_param_shape += (in_channels,) + self._i2h_kernel\n            h2h_param_shape += (hidden_channels,) + self._h2h_kernel\n            state_shape += conv_out_size\n        else:\n            i2h_param_shape += self._i2h_kernel + (in_channels,)\n            h2h_param_shape += self._h2h_kernel + (hidden_channels,)\n            state_shape = conv_out_size + state_shape\n\n        return channel_axis, in_channels, i2h_param_shape, \\\n               h2h_param_shape, h2h_pad, state_shape\n\n    def __repr__(self):\n        s = '{name}({mapping}'\n        if hasattr(self, '_activation'):\n            s += ', {_activation}'\n        s += ', {_conv_layout}'\n        s += ')'\n        attrs = self.__dict__\n        shape = self.i2h_weight.shape\n        in_channels = shape[1 if self._channel_axis == 1 else -1]\n        mapping = ('{0} -> {1}'.format(in_channels if in_channels else None, shape[0]))\n        return s.format(name=self.__class__.__name__,\n                        mapping=mapping,\n                        **attrs)\n\n    @property\n    def _num_gates(self):\n        return len(self._gate_names)\n\n    def _conv_forward(self, inputs, states):\n        device = inputs.device\n        i2h = npx.convolution(data=inputs,\n                              num_filter=self._hidden_channels*self._num_gates,\n                              kernel=self._i2h_kernel,\n                              stride=self._stride,\n                              pad=self._i2h_pad,\n                              dilate=self._i2h_dilate,\n                              weight=self.i2h_weight.data(device),\n                              bias=self.i2h_bias.data(device),\n                              layout=self._conv_layout)\n        h2h = npx.convolution(data=states[0].to_device(device),\n                              num_filter=self._hidden_channels*self._num_gates,\n                              kernel=self._h2h_kernel,\n                              dilate=self._h2h_dilate,\n                              pad=self._h2h_pad,\n                              stride=self._stride,\n                              weight=self.h2h_weight.data(device),\n                              bias=self.h2h_bias.data(device),\n                              layout=self._conv_layout)\n        return i2h, h2h\n\n    def state_info(self, batch_size=0):\n        raise NotImplementedError(\"_BaseConvRNNCell is abstract class for convolutional RNN\")\n\n    def forward(self, inputs, states):\n        raise NotImplementedError(\"_BaseConvRNNCell is abstract class for convolutional RNN\")\n\n    # pylint: disable=unused-argument\n    def infer_shape(self, i, x, is_bidirect):\n        channel_axis = self._conv_layout.find('C')\n        shape_c = x.shape[-len(self._i2h_kernel)-1:][channel_axis-1]\n        wshape = self.i2h_weight.shape\n        wshape_list = list(wshape)\n        wshape_list[self._conv_layout.find('C')] = shape_c\n        self.i2h_weight.shape = tuple(wshape_list)\n\n\n@use_np\nclass _ConvRNNCell(_BaseConvRNNCell):\n    def __init__(self, input_shape, hidden_channels,\n                 i2h_kernel, h2h_kernel, i2h_pad, i2h_dilate, h2h_dilate,\n                 i2h_weight_initializer, h2h_weight_initializer,\n                 i2h_bias_initializer, h2h_bias_initializer,\n                 dims, conv_layout, activation):\n        super(_ConvRNNCell, self).__init__(input_shape=input_shape,\n                                           hidden_channels=hidden_channels,\n                                           activation=activation,\n                                           i2h_kernel=i2h_kernel,\n                                           i2h_pad=i2h_pad, i2h_dilate=i2h_dilate,\n                                           h2h_kernel=h2h_kernel, h2h_dilate=h2h_dilate,\n                                           i2h_weight_initializer=i2h_weight_initializer,\n                                           h2h_weight_initializer=h2h_weight_initializer,\n                                           i2h_bias_initializer=i2h_bias_initializer,\n                                           h2h_bias_initializer=h2h_bias_initializer,\n                                           dims=dims,\n                                           conv_layout=conv_layout)\n\n    def state_info(self, batch_size=0):\n        return [{'shape': (batch_size,)+self._state_shape, '__layout__': self._conv_layout}]\n\n    def _alias(self):\n        return 'conv_rnn'\n\n    @property\n    def _gate_names(self):\n        return ('',)\n\n    def forward(self, inputs, states):\n        i2h, h2h = self._conv_forward(inputs, states)\n        output = self._get_activation(i2h + h2h, self._activation)\n        return output, [output]\n\n\nclass Conv1DRNNCell(_ConvRNNCell):\n    r\"\"\"1D Convolutional RNN cell.\n\n    .. math::\n\n        h_t = tanh(W_i \\ast x_t + R_i \\ast h_{t-1} + b_i)\n\n    Parameters\n    ----------\n    input_shape : tuple of int\n        Input tensor shape at each time step for each sample, excluding dimension of the batch size\n        and sequence length. Must be consistent with `conv_layout`.\n        For example, for layout 'NCW' the shape should be (C, W).\n    hidden_channels : int\n        Number of output channels.\n    i2h_kernel : int or tuple of int\n        Input convolution kernel sizes.\n    h2h_kernel : int or tuple of int\n        Recurrent convolution kernel sizes. Only odd-numbered sizes are supported.\n    i2h_pad : int or tuple of int, default (0,)\n        Pad for input convolution.\n    i2h_dilate : int or tuple of int, default (1,)\n        Input convolution dilate.\n    h2h_dilate : int or tuple of int, default (1,)\n        Recurrent convolution dilate.\n    i2h_weight_initializer : str or Initializer\n        Initializer for the input weights matrix, used for the input convolutions.\n    h2h_weight_initializer : str or Initializer\n        Initializer for the recurrent weights matrix, used for the input convolutions.\n    i2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the input convolution bias vectors.\n    h2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the recurrent convolution bias vectors.\n    conv_layout : str, default 'NCW'\n        Layout for all convolution inputs, outputs and weights. Options are 'NCW' and 'NWC'.\n    activation : str or gluon.Block, default 'tanh'\n        Type of activation function.\n        If argument type is string, it's equivalent to nn.Activation(act_type=str). See\n        :func:`~mxnet.ndarray.Activation` for available choices.\n        Alternatively, other activation blocks such as nn.LeakyReLU can be used.\n    \"\"\"\n    def __init__(self, input_shape, hidden_channels,\n                 i2h_kernel, h2h_kernel,\n                 i2h_pad=(0,), i2h_dilate=(1,), h2h_dilate=(1,),\n                 i2h_weight_initializer=None, h2h_weight_initializer=None,\n                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',\n                 conv_layout='NCW', activation='tanh'):\n        super(Conv1DRNNCell, self).__init__(input_shape=input_shape,\n                                            hidden_channels=hidden_channels,\n                                            i2h_kernel=i2h_kernel, h2h_kernel=h2h_kernel,\n                                            i2h_pad=i2h_pad,\n                                            i2h_dilate=i2h_dilate, h2h_dilate=h2h_dilate,\n                                            i2h_weight_initializer=i2h_weight_initializer,\n                                            h2h_weight_initializer=h2h_weight_initializer,\n                                            i2h_bias_initializer=i2h_bias_initializer,\n                                            h2h_bias_initializer=h2h_bias_initializer,\n                                            dims=1,\n                                            conv_layout=conv_layout,\n                                            activation=activation)\n\n\nclass Conv2DRNNCell(_ConvRNNCell):\n    r\"\"\"2D Convolutional RNN cell.\n\n    .. math::\n\n        h_t = tanh(W_i \\ast x_t + R_i \\ast h_{t-1} + b_i)\n\n    Parameters\n    ----------\n    input_shape : tuple of int\n        Input tensor shape at each time step for each sample, excluding dimension of the batch size\n        and sequence length. Must be consistent with `conv_layout`.\n        For example, for layout 'NCHW' the shape should be (C, H, W).\n    hidden_channels : int\n        Number of output channels.\n    i2h_kernel : int or tuple of int\n        Input convolution kernel sizes.\n    h2h_kernel : int or tuple of int\n        Recurrent convolution kernel sizes. Only odd-numbered sizes are supported.\n    i2h_pad : int or tuple of int, default (0, 0)\n        Pad for input convolution.\n    i2h_dilate : int or tuple of int, default (1, 1)\n        Input convolution dilate.\n    h2h_dilate : int or tuple of int, default (1, 1)\n        Recurrent convolution dilate.\n    i2h_weight_initializer : str or Initializer\n        Initializer for the input weights matrix, used for the input convolutions.\n    h2h_weight_initializer : str or Initializer\n        Initializer for the recurrent weights matrix, used for the input convolutions.\n    i2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the input convolution bias vectors.\n    h2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the recurrent convolution bias vectors.\n    conv_layout : str, default 'NCHW'\n        Layout for all convolution inputs, outputs and weights. Options are 'NCHW' and 'NHWC'.\n    activation : str or gluon.Block, default 'tanh'\n        Type of activation function.\n        If argument type is string, it's equivalent to nn.Activation(act_type=str). See\n        :func:`~mxnet.ndarray.Activation` for available choices.\n        Alternatively, other activation blocks such as nn.LeakyReLU can be used.\n    \"\"\"\n    def __init__(self, input_shape, hidden_channels,\n                 i2h_kernel, h2h_kernel,\n                 i2h_pad=(0, 0), i2h_dilate=(1, 1), h2h_dilate=(1, 1),\n                 i2h_weight_initializer=None, h2h_weight_initializer=None,\n                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',\n                 conv_layout='NCHW', activation='tanh'):\n        super(Conv2DRNNCell, self).__init__(input_shape=input_shape,\n                                            hidden_channels=hidden_channels,\n                                            i2h_kernel=i2h_kernel, h2h_kernel=h2h_kernel,\n                                            i2h_pad=i2h_pad,\n                                            i2h_dilate=i2h_dilate, h2h_dilate=h2h_dilate,\n                                            i2h_weight_initializer=i2h_weight_initializer,\n                                            h2h_weight_initializer=h2h_weight_initializer,\n                                            i2h_bias_initializer=i2h_bias_initializer,\n                                            h2h_bias_initializer=h2h_bias_initializer,\n                                            dims=2,\n                                            conv_layout=conv_layout,\n                                            activation=activation)\n\n\nclass Conv3DRNNCell(_ConvRNNCell):\n    r\"\"\"3D Convolutional RNN cells\n\n    .. math::\n\n        h_t = tanh(W_i \\ast x_t + R_i \\ast h_{t-1} + b_i)\n\n    Parameters\n    ----------\n    input_shape : tuple of int\n        Input tensor shape at each time step for each sample, excluding dimension of the batch size\n        and sequence length. Must be consistent with `conv_layout`.\n        For example, for layout 'NCDHW' the shape should be (C, D, H, W).\n    hidden_channels : int\n        Number of output channels.\n    i2h_kernel : int or tuple of int\n        Input convolution kernel sizes.\n    h2h_kernel : int or tuple of int\n        Recurrent convolution kernel sizes. Only odd-numbered sizes are supported.\n    i2h_pad : int or tuple of int, default (0, 0, 0)\n        Pad for input convolution.\n    i2h_dilate : int or tuple of int, default (1, 1, 1)\n        Input convolution dilate.\n    h2h_dilate : int or tuple of int, default (1, 1, 1)\n        Recurrent convolution dilate.\n    i2h_weight_initializer : str or Initializer\n        Initializer for the input weights matrix, used for the input convolutions.\n    h2h_weight_initializer : str or Initializer\n        Initializer for the recurrent weights matrix, used for the input convolutions.\n    i2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the input convolution bias vectors.\n    h2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the recurrent convolution bias vectors.\n    conv_layout : str, default 'NCDHW'\n        Layout for all convolution inputs, outputs and weights. Options are 'NCDHW' and 'NDHWC'.\n    activation : str or gluon.Block, default 'tanh'\n        Type of activation function.\n        If argument type is string, it's equivalent to nn.Activation(act_type=str). See\n        :func:`~mxnet.ndarray.Activation` for available choices.\n        Alternatively, other activation blocks such as nn.LeakyReLU can be used.\n    \"\"\"\n    def __init__(self, input_shape, hidden_channels,\n                 i2h_kernel, h2h_kernel,\n                 i2h_pad=(0, 0, 0),\n                 i2h_dilate=(1, 1, 1), h2h_dilate=(1, 1, 1),\n                 i2h_weight_initializer=None, h2h_weight_initializer=None,\n                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',\n                 conv_layout='NCDHW', activation='tanh'):\n        super(Conv3DRNNCell, self).__init__(input_shape=input_shape,\n                                            hidden_channels=hidden_channels,\n                                            i2h_kernel=i2h_kernel, h2h_kernel=h2h_kernel,\n                                            i2h_pad=i2h_pad,\n                                            i2h_dilate=i2h_dilate, h2h_dilate=h2h_dilate,\n                                            i2h_weight_initializer=i2h_weight_initializer,\n                                            h2h_weight_initializer=h2h_weight_initializer,\n                                            i2h_bias_initializer=i2h_bias_initializer,\n                                            h2h_bias_initializer=h2h_bias_initializer,\n                                            dims=3,\n                                            conv_layout=conv_layout,\n                                            activation=activation)\n\n\n@use_np\nclass _ConvLSTMCell(_BaseConvRNNCell):\n    def __init__(self, input_shape, hidden_channels,\n                 i2h_kernel, h2h_kernel,\n                 i2h_pad, i2h_dilate, h2h_dilate,\n                 i2h_weight_initializer, h2h_weight_initializer,\n                 i2h_bias_initializer, h2h_bias_initializer,\n                 dims, conv_layout, activation):\n        super(_ConvLSTMCell, self).__init__(input_shape=input_shape,\n                                            hidden_channels=hidden_channels,\n                                            i2h_kernel=i2h_kernel, h2h_kernel=h2h_kernel,\n                                            i2h_pad=i2h_pad,\n                                            i2h_dilate=i2h_dilate, h2h_dilate=h2h_dilate,\n                                            i2h_weight_initializer=i2h_weight_initializer,\n                                            h2h_weight_initializer=h2h_weight_initializer,\n                                            i2h_bias_initializer=i2h_bias_initializer,\n                                            h2h_bias_initializer=h2h_bias_initializer,\n                                            dims=dims,\n                                            conv_layout=conv_layout,\n                                            activation=activation)\n\n    def state_info(self, batch_size=0):\n        return [{'shape': (batch_size,)+self._state_shape, '__layout__': self._conv_layout},\n                {'shape': (batch_size,)+self._state_shape, '__layout__': self._conv_layout}]\n\n    def _alias(self):\n        return 'conv_lstm'\n\n    @property\n    def _gate_names(self):\n        return ['_i', '_f', '_c', '_o']\n\n    def forward(self, inputs, states):\n        i2h, h2h = self._conv_forward(inputs, states)\n        gates = i2h + h2h\n        slice_gates = npx.slice_channel(gates, num_outputs=4, axis=self._channel_axis)\n        in_gate = npx.activation(slice_gates[0], act_type=\"sigmoid\")\n        forget_gate = npx.activation(slice_gates[1], act_type=\"sigmoid\")\n        in_transform = self._get_activation(slice_gates[2], self._activation)\n        out_gate = npx.activation(slice_gates[3], act_type=\"sigmoid\")\n        next_c = forget_gate * states[1].to_device(inputs.device) + in_gate * in_transform\n        next_h = np.multiply(out_gate, self._get_activation(next_c, self._activation))\n\n        return next_h, [next_h, next_c]\n\n\nclass Conv1DLSTMCell(_ConvLSTMCell):\n    r\"\"\"1D Convolutional LSTM network cell.\n\n    `\"Convolutional LSTM Network: A Machine Learning Approach for Precipitation Nowcasting\"\n    <https://arxiv.org/abs/1506.04214>`_ paper. Xingjian et al. NIPS2015\n\n    .. math::\n        \\begin{array}{ll}\n        i_t = \\sigma(W_i \\ast x_t + R_i \\ast h_{t-1} + b_i) \\\\\n        f_t = \\sigma(W_f \\ast x_t + R_f \\ast h_{t-1} + b_f) \\\\\n        o_t = \\sigma(W_o \\ast x_t + R_o \\ast h_{t-1} + b_o) \\\\\n        c^\\prime_t = tanh(W_c \\ast x_t + R_c \\ast h_{t-1} + b_c) \\\\\n        c_t = f_t \\circ c_{t-1} + i_t \\circ c^\\prime_t \\\\\n        h_t = o_t \\circ tanh(c_t) \\\\\n        \\end{array}\n\n    Parameters\n    ----------\n    input_shape : tuple of int\n        Input tensor shape at each time step for each sample, excluding dimension of the batch size\n        and sequence length. Must be consistent with `conv_layout`.\n        For example, for layout 'NCW' the shape should be (C, W).\n    hidden_channels : int\n        Number of output channels.\n    i2h_kernel : int or tuple of int\n        Input convolution kernel sizes.\n    h2h_kernel : int or tuple of int\n        Recurrent convolution kernel sizes. Only odd-numbered sizes are supported.\n    i2h_pad : int or tuple of int, default (0,)\n        Pad for input convolution.\n    i2h_dilate : int or tuple of int, default (1,)\n        Input convolution dilate.\n    h2h_dilate : int or tuple of int, default (1,)\n        Recurrent convolution dilate.\n    i2h_weight_initializer : str or Initializer\n        Initializer for the input weights matrix, used for the input convolutions.\n    h2h_weight_initializer : str or Initializer\n        Initializer for the recurrent weights matrix, used for the input convolutions.\n    i2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the input convolution bias vectors.\n    h2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the recurrent convolution bias vectors.\n    conv_layout : str, default 'NCW'\n        Layout for all convolution inputs, outputs and weights. Options are 'NCW' and 'NWC'.\n    activation : str or gluon.Block, default 'tanh'\n        Type of activation function used in c^\\prime_t.\n        If argument type is string, it's equivalent to nn.Activation(act_type=str). See\n        :func:`~mxnet.ndarray.Activation` for available choices.\n        Alternatively, other activation blocks such as nn.LeakyReLU can be used.\n    \"\"\"\n    def __init__(self, input_shape, hidden_channels,\n                 i2h_kernel, h2h_kernel,\n                 i2h_pad=(0,),\n                 i2h_dilate=(1,), h2h_dilate=(1,),\n                 i2h_weight_initializer=None, h2h_weight_initializer=None,\n                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',\n                 conv_layout='NCW', activation='tanh'):\n        super(Conv1DLSTMCell, self).__init__(input_shape=input_shape,\n                                             hidden_channels=hidden_channels,\n                                             i2h_kernel=i2h_kernel, h2h_kernel=h2h_kernel,\n                                             i2h_pad=i2h_pad,\n                                             i2h_dilate=i2h_dilate, h2h_dilate=h2h_dilate,\n                                             i2h_weight_initializer=i2h_weight_initializer,\n                                             h2h_weight_initializer=h2h_weight_initializer,\n                                             i2h_bias_initializer=i2h_bias_initializer,\n                                             h2h_bias_initializer=h2h_bias_initializer,\n                                             dims=1,\n                                             conv_layout=conv_layout,\n                                             activation=activation)\n\n\nclass Conv2DLSTMCell(_ConvLSTMCell):\n    r\"\"\"2D Convolutional LSTM network cell.\n\n    `\"Convolutional LSTM Network: A Machine Learning Approach for Precipitation Nowcasting\"\n    <https://arxiv.org/abs/1506.04214>`_ paper. Xingjian et al. NIPS2015\n\n    .. math::\n        \\begin{array}{ll}\n        i_t = \\sigma(W_i \\ast x_t + R_i \\ast h_{t-1} + b_i) \\\\\n        f_t = \\sigma(W_f \\ast x_t + R_f \\ast h_{t-1} + b_f) \\\\\n        o_t = \\sigma(W_o \\ast x_t + R_o \\ast h_{t-1} + b_o) \\\\\n        c^\\prime_t = tanh(W_c \\ast x_t + R_c \\ast h_{t-1} + b_c) \\\\\n        c_t = f_t \\circ c_{t-1} + i_t \\circ c^\\prime_t \\\\\n        h_t = o_t \\circ tanh(c_t) \\\\\n        \\end{array}\n\n    Parameters\n    ----------\n    input_shape : tuple of int\n        Input tensor shape at each time step for each sample, excluding dimension of the batch size\n        and sequence length. Must be consistent with `conv_layout`.\n        For example, for layout 'NCHW' the shape should be (C, H, W).\n    hidden_channels : int\n        Number of output channels.\n    i2h_kernel : int or tuple of int\n        Input convolution kernel sizes.\n    h2h_kernel : int or tuple of int\n        Recurrent convolution kernel sizes. Only odd-numbered sizes are supported.\n    i2h_pad : int or tuple of int, default (0, 0)\n        Pad for input convolution.\n    i2h_dilate : int or tuple of int, default (1, 1)\n        Input convolution dilate.\n    h2h_dilate : int or tuple of int, default (1, 1)\n        Recurrent convolution dilate.\n    i2h_weight_initializer : str or Initializer\n        Initializer for the input weights matrix, used for the input convolutions.\n    h2h_weight_initializer : str or Initializer\n        Initializer for the recurrent weights matrix, used for the input convolutions.\n    i2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the input convolution bias vectors.\n    h2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the recurrent convolution bias vectors.\n    conv_layout : str, default 'NCHW'\n        Layout for all convolution inputs, outputs and weights. Options are 'NCHW' and 'NHWC'.\n    activation : str or gluon.Block, default 'tanh'\n        Type of activation function used in c^\\prime_t.\n        If argument type is string, it's equivalent to nn.Activation(act_type=str). See\n        :func:`~mxnet.ndarray.Activation` for available choices.\n        Alternatively, other activation blocks such as nn.LeakyReLU can be used.\n    \"\"\"\n    def __init__(self, input_shape, hidden_channels,\n                 i2h_kernel, h2h_kernel,\n                 i2h_pad=(0, 0),\n                 i2h_dilate=(1, 1), h2h_dilate=(1, 1),\n                 i2h_weight_initializer=None, h2h_weight_initializer=None,\n                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',\n                 conv_layout='NCHW', activation='tanh'):\n        super(Conv2DLSTMCell, self).__init__(input_shape=input_shape,\n                                             hidden_channels=hidden_channels,\n                                             i2h_kernel=i2h_kernel, h2h_kernel=h2h_kernel,\n                                             i2h_pad=i2h_pad,\n                                             i2h_dilate=i2h_dilate, h2h_dilate=h2h_dilate,\n                                             i2h_weight_initializer=i2h_weight_initializer,\n                                             h2h_weight_initializer=h2h_weight_initializer,\n                                             i2h_bias_initializer=i2h_bias_initializer,\n                                             h2h_bias_initializer=h2h_bias_initializer,\n                                             dims=2,\n                                             conv_layout=conv_layout,\n                                             activation=activation)\n\n\nclass Conv3DLSTMCell(_ConvLSTMCell):\n    r\"\"\"3D Convolutional LSTM network cell.\n\n    `\"Convolutional LSTM Network: A Machine Learning Approach for Precipitation Nowcasting\"\n    <https://arxiv.org/abs/1506.04214>`_ paper. Xingjian et al. NIPS2015\n\n    .. math::\n        \\begin{array}{ll}\n        i_t = \\sigma(W_i \\ast x_t + R_i \\ast h_{t-1} + b_i) \\\\\n        f_t = \\sigma(W_f \\ast x_t + R_f \\ast h_{t-1} + b_f) \\\\\n        o_t = \\sigma(W_o \\ast x_t + R_o \\ast h_{t-1} + b_o) \\\\\n        c^\\prime_t = tanh(W_c \\ast x_t + R_c \\ast h_{t-1} + b_c) \\\\\n        c_t = f_t \\circ c_{t-1} + i_t \\circ c^\\prime_t \\\\\n        h_t = o_t \\circ tanh(c_t) \\\\\n        \\end{array}\n\n    Parameters\n    ----------\n    input_shape : tuple of int\n        Input tensor shape at each time step for each sample, excluding dimension of the batch size\n        and sequence length. Must be consistent with `conv_layout`.\n        For example, for layout 'NCDHW' the shape should be (C, D, H, W).\n    hidden_channels : int\n        Number of output channels.\n    i2h_kernel : int or tuple of int\n        Input convolution kernel sizes.\n    h2h_kernel : int or tuple of int\n        Recurrent convolution kernel sizes. Only odd-numbered sizes are supported.\n    i2h_pad : int or tuple of int, default (0, 0, 0)\n        Pad for input convolution.\n    i2h_dilate : int or tuple of int, default (1, 1, 1)\n        Input convolution dilate.\n    h2h_dilate : int or tuple of int, default (1, 1, 1)\n        Recurrent convolution dilate.\n    i2h_weight_initializer : str or Initializer\n        Initializer for the input weights matrix, used for the input convolutions.\n    h2h_weight_initializer : str or Initializer\n        Initializer for the recurrent weights matrix, used for the input convolutions.\n    i2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the input convolution bias vectors.\n    h2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the recurrent convolution bias vectors.\n    conv_layout : str, default 'NCDHW'\n        Layout for all convolution inputs, outputs and weights. Options are 'NCDHW' and 'NDHWC'.\n    activation : str or gluon.Block, default 'tanh'\n        Type of activation function used in c^\\prime_t.\n        If argument type is string, it's equivalent to nn.Activation(act_type=str). See\n        :func:`~mxnet.ndarray.Activation` for available choices.\n        Alternatively, other activation blocks such as nn.LeakyReLU can be used.\n    \"\"\"\n    def __init__(self, input_shape, hidden_channels,\n                 i2h_kernel, h2h_kernel,\n                 i2h_pad=(0, 0, 0),\n                 i2h_dilate=(1, 1, 1), h2h_dilate=(1, 1, 1),\n                 i2h_weight_initializer=None, h2h_weight_initializer=None,\n                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',\n                 conv_layout='NCDHW', activation='tanh'):\n        super(Conv3DLSTMCell, self).__init__(input_shape=input_shape,\n                                             hidden_channels=hidden_channels,\n                                             i2h_kernel=i2h_kernel, h2h_kernel=h2h_kernel,\n                                             i2h_pad=i2h_pad,\n                                             i2h_dilate=i2h_dilate, h2h_dilate=h2h_dilate,\n                                             i2h_weight_initializer=i2h_weight_initializer,\n                                             h2h_weight_initializer=h2h_weight_initializer,\n                                             i2h_bias_initializer=i2h_bias_initializer,\n                                             h2h_bias_initializer=h2h_bias_initializer,\n                                             dims=3,\n                                             conv_layout=conv_layout,\n                                             activation=activation)\n\n\n@use_np\nclass _ConvGRUCell(_BaseConvRNNCell):\n    def __init__(self, input_shape, hidden_channels,\n                 i2h_kernel, h2h_kernel, i2h_pad, i2h_dilate, h2h_dilate,\n                 i2h_weight_initializer, h2h_weight_initializer,\n                 i2h_bias_initializer, h2h_bias_initializer,\n                 dims, conv_layout, activation):\n        super(_ConvGRUCell, self).__init__(input_shape=input_shape,\n                                           hidden_channels=hidden_channels,\n                                           i2h_kernel=i2h_kernel, h2h_kernel=h2h_kernel,\n                                           i2h_pad=i2h_pad,\n                                           i2h_dilate=i2h_dilate, h2h_dilate=h2h_dilate,\n                                           i2h_weight_initializer=i2h_weight_initializer,\n                                           h2h_weight_initializer=h2h_weight_initializer,\n                                           i2h_bias_initializer=i2h_bias_initializer,\n                                           h2h_bias_initializer=h2h_bias_initializer,\n                                           dims=dims,\n                                           conv_layout=conv_layout,\n                                           activation=activation)\n\n    def state_info(self, batch_size=0):\n        return [{'shape': (batch_size,)+self._state_shape, '__layout__': self._conv_layout}]\n\n    def _alias(self):\n        return 'conv_gru'\n\n    @property\n    def _gate_names(self):\n        return ['_r', '_z', '_o']\n\n    def forward(self, inputs, states):\n        i2h, h2h = self._conv_forward(inputs, states)\n\n        i2h_r, i2h_z, i2h = npx.slice_channel(i2h, num_outputs=3,\n                                              axis=self._channel_axis)\n        h2h_r, h2h_z, h2h = npx.slice_channel(h2h, num_outputs=3,\n                                              axis=self._channel_axis)\n\n        reset_gate = npx.activation(i2h_r + h2h_r, act_type=\"sigmoid\")\n        update_gate = npx.activation(i2h_z + h2h_z, act_type=\"sigmoid\")\n\n        next_h_tmp = self._get_activation(i2h + reset_gate * h2h, self._activation)\n\n        next_h = (1. - update_gate) * next_h_tmp + update_gate * \\\n            states[0].to_device(inputs.device)\n\n        return next_h, [next_h]\n\n\nclass Conv1DGRUCell(_ConvGRUCell):\n    r\"\"\"1D Convolutional Gated Rectified Unit (GRU) network cell.\n\n    .. math::\n        \\begin{array}{ll}\n        r_t = \\sigma(W_r \\ast x_t + R_r \\ast h_{t-1} + b_r) \\\\\n        z_t = \\sigma(W_z \\ast x_t + R_z \\ast h_{t-1} + b_z) \\\\\n        n_t = tanh(W_i \\ast x_t + b_i + r_t \\circ (R_n \\ast h_{t-1} + b_n)) \\\\\n        h^\\prime_t = (1 - z_t) \\circ n_t + z_t \\circ h \\\\\n        \\end{array}\n\n    Parameters\n    ----------\n    input_shape : tuple of int\n        Input tensor shape at each time step for each sample, excluding dimension of the batch size\n        and sequence length. Must be consistent with `conv_layout`.\n        For example, for layout 'NCW' the shape should be (C, W).\n    hidden_channels : int\n        Number of output channels.\n    i2h_kernel : int or tuple of int\n        Input convolution kernel sizes.\n    h2h_kernel : int or tuple of int\n        Recurrent convolution kernel sizes. Only odd-numbered sizes are supported.\n    i2h_pad : int or tuple of int, default (0,)\n        Pad for input convolution.\n    i2h_dilate : int or tuple of int, default (1,)\n        Input convolution dilate.\n    h2h_dilate : int or tuple of int, default (1,)\n        Recurrent convolution dilate.\n    i2h_weight_initializer : str or Initializer\n        Initializer for the input weights matrix, used for the input convolutions.\n    h2h_weight_initializer : str or Initializer\n        Initializer for the recurrent weights matrix, used for the input convolutions.\n    i2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the input convolution bias vectors.\n    h2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the recurrent convolution bias vectors.\n    conv_layout : str, default 'NCW'\n        Layout for all convolution inputs, outputs and weights. Options are 'NCW' and 'NWC'.\n    activation : str or gluon.Block, default 'tanh'\n        Type of activation function used in n_t.\n        If argument type is string, it's equivalent to nn.Activation(act_type=str). See\n        :func:`~mxnet.ndarray.Activation` for available choices.\n        Alternatively, other activation blocks such as nn.LeakyReLU can be used.\n    \"\"\"\n    def __init__(self, input_shape, hidden_channels,\n                 i2h_kernel, h2h_kernel,\n                 i2h_pad=(0,),\n                 i2h_dilate=(1,), h2h_dilate=(1,),\n                 i2h_weight_initializer=None, h2h_weight_initializer=None,\n                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',\n                 conv_layout='NCW', activation='tanh'):\n        super(Conv1DGRUCell, self).__init__(input_shape=input_shape,\n                                            hidden_channels=hidden_channels,\n                                            i2h_kernel=i2h_kernel, h2h_kernel=h2h_kernel,\n                                            i2h_pad=i2h_pad,\n                                            i2h_dilate=i2h_dilate, h2h_dilate=h2h_dilate,\n                                            i2h_weight_initializer=i2h_weight_initializer,\n                                            h2h_weight_initializer=h2h_weight_initializer,\n                                            i2h_bias_initializer=i2h_bias_initializer,\n                                            h2h_bias_initializer=h2h_bias_initializer,\n                                            dims=1,\n                                            conv_layout=conv_layout,\n                                            activation=activation)\n\n\nclass Conv2DGRUCell(_ConvGRUCell):\n    r\"\"\"2D Convolutional Gated Rectified Unit (GRU) network cell.\n\n    .. math::\n        \\begin{array}{ll}\n        r_t = \\sigma(W_r \\ast x_t + R_r \\ast h_{t-1} + b_r) \\\\\n        z_t = \\sigma(W_z \\ast x_t + R_z \\ast h_{t-1} + b_z) \\\\\n        n_t = tanh(W_i \\ast x_t + b_i + r_t \\circ (R_n \\ast h_{t-1} + b_n)) \\\\\n        h^\\prime_t = (1 - z_t) \\circ n_t + z_t \\circ h \\\\\n        \\end{array}\n\n    Parameters\n    ----------\n    input_shape : tuple of int\n        Input tensor shape at each time step for each sample, excluding dimension of the batch size\n        and sequence length. Must be consistent with `conv_layout`.\n        For example, for layout 'NCHW' the shape should be (C, H, W).\n    hidden_channels : int\n        Number of output channels.\n    i2h_kernel : int or tuple of int\n        Input convolution kernel sizes.\n    h2h_kernel : int or tuple of int\n        Recurrent convolution kernel sizes. Only odd-numbered sizes are supported.\n    i2h_pad : int or tuple of int, default (0, 0)\n        Pad for input convolution.\n    i2h_dilate : int or tuple of int, default (1, 1)\n        Input convolution dilate.\n    h2h_dilate : int or tuple of int, default (1, 1)\n        Recurrent convolution dilate.\n    i2h_weight_initializer : str or Initializer\n        Initializer for the input weights matrix, used for the input convolutions.\n    h2h_weight_initializer : str or Initializer\n        Initializer for the recurrent weights matrix, used for the input convolutions.\n    i2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the input convolution bias vectors.\n    h2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the recurrent convolution bias vectors.\n    conv_layout : str, default 'NCHW'\n        Layout for all convolution inputs, outputs and weights. Options are 'NCHW' and 'NHWC'.\n    activation : str or gluon.Block, default 'tanh'\n        Type of activation function used in n_t.\n        If argument type is string, it's equivalent to nn.Activation(act_type=str). See\n        :func:`~mxnet.ndarray.Activation` for available choices.\n        Alternatively, other activation blocks such as nn.LeakyReLU can be used.\n    \"\"\"\n    def __init__(self, input_shape, hidden_channels,\n                 i2h_kernel, h2h_kernel,\n                 i2h_pad=(0, 0),\n                 i2h_dilate=(1, 1), h2h_dilate=(1, 1),\n                 i2h_weight_initializer=None, h2h_weight_initializer=None,\n                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',\n                 conv_layout='NCHW', activation='tanh'):\n        super(Conv2DGRUCell, self).__init__(input_shape=input_shape,\n                                            hidden_channels=hidden_channels,\n                                            i2h_kernel=i2h_kernel, h2h_kernel=h2h_kernel,\n                                            i2h_pad=i2h_pad,\n                                            i2h_dilate=i2h_dilate, h2h_dilate=h2h_dilate,\n                                            i2h_weight_initializer=i2h_weight_initializer,\n                                            h2h_weight_initializer=h2h_weight_initializer,\n                                            i2h_bias_initializer=i2h_bias_initializer,\n                                            h2h_bias_initializer=h2h_bias_initializer,\n                                            dims=2,\n                                            conv_layout=conv_layout,\n                                            activation=activation)\n\n\nclass Conv3DGRUCell(_ConvGRUCell):\n    r\"\"\"3D Convolutional Gated Rectified Unit (GRU) network cell.\n\n    .. math::\n        \\begin{array}{ll}\n        r_t = \\sigma(W_r \\ast x_t + R_r \\ast h_{t-1} + b_r) \\\\\n        z_t = \\sigma(W_z \\ast x_t + R_z \\ast h_{t-1} + b_z) \\\\\n        n_t = tanh(W_i \\ast x_t + b_i + r_t \\circ (R_n \\ast h_{t-1} + b_n)) \\\\\n        h^\\prime_t = (1 - z_t) \\circ n_t + z_t \\circ h \\\\\n        \\end{array}\n\n    Parameters\n    ----------\n    input_shape : tuple of int\n        Input tensor shape at each time step for each sample, excluding dimension of the batch size\n        and sequence length. Must be consistent with `conv_layout`.\n        For example, for layout 'NCDHW' the shape should be (C, D, H, W).\n    hidden_channels : int\n        Number of output channels.\n    i2h_kernel : int or tuple of int\n        Input convolution kernel sizes.\n    h2h_kernel : int or tuple of int\n        Recurrent convolution kernel sizes. Only odd-numbered sizes are supported.\n    i2h_pad : int or tuple of int, default (0, 0, 0)\n        Pad for input convolution.\n    i2h_dilate : int or tuple of int, default (1, 1, 1)\n        Input convolution dilate.\n    h2h_dilate : int or tuple of int, default (1, 1, 1)\n        Recurrent convolution dilate.\n    i2h_weight_initializer : str or Initializer\n        Initializer for the input weights matrix, used for the input convolutions.\n    h2h_weight_initializer : str or Initializer\n        Initializer for the recurrent weights matrix, used for the input convolutions.\n    i2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the input convolution bias vectors.\n    h2h_bias_initializer : str or Initializer, default zeros\n        Initializer for the recurrent convolution bias vectors.\n    conv_layout : str, default 'NCDHW'\n        Layout for all convolution inputs, outputs and weights. Options are 'NCDHW' and 'NDHWC'.\n    activation : str or gluon.Block, default 'tanh'\n        Type of activation function used in n_t.\n        If argument type is string, it's equivalent to nn.Activation(act_type=str). See\n        :func:`~mxnet.ndarray.Activation` for available choices.\n        Alternatively, other activation blocks such as nn.LeakyReLU can be used.\n    \"\"\"\n    def __init__(self, input_shape, hidden_channels,\n                 i2h_kernel, h2h_kernel,\n                 i2h_pad=(0, 0, 0),\n                 i2h_dilate=(1, 1, 1), h2h_dilate=(1, 1, 1),\n                 i2h_weight_initializer=None, h2h_weight_initializer=None,\n                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',\n                 conv_layout='NCDHW', activation='tanh'):\n        super(Conv3DGRUCell, self).__init__(input_shape=input_shape,\n                                            hidden_channels=hidden_channels,\n                                            i2h_kernel=i2h_kernel, h2h_kernel=h2h_kernel,\n                                            i2h_pad=i2h_pad,\n                                            i2h_dilate=i2h_dilate, h2h_dilate=h2h_dilate,\n                                            i2h_weight_initializer=i2h_weight_initializer,\n                                            h2h_weight_initializer=h2h_weight_initializer,\n                                            i2h_bias_initializer=i2h_bias_initializer,\n                                            h2h_bias_initializer=h2h_bias_initializer,\n                                            dims=3,\n                                            conv_layout=conv_layout,\n                                            activation=activation)\n"
  },
  {
    "path": "python/mxnet/gluon/rnn/rnn_cell.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=no-member, invalid-name, protected-access, no-self-use\n# pylint: disable=too-many-branches, too-many-arguments, no-self-use\n# pylint: disable=too-many-lines, arguments-differ\n\"\"\"Definition of various recurrent neural network cells.\"\"\"\n__all__ = ['RecurrentCell', 'HybridRecurrentCell',\n           'RNNCell', 'LSTMCell', 'GRUCell',\n           'SequentialRNNCell', 'HybridSequentialRNNCell', 'DropoutCell',\n           'ModifierCell', 'ZoneoutCell', 'ResidualCell',\n           'BidirectionalCell', 'VariationalDropoutCell', 'LSTMPCell']\n\nfrom ... import np, npx, cpu\nfrom ...util import use_np\nfrom ...base import string_types, numeric_types, _as_list\nfrom ..block import Block, HybridBlock\nfrom ..parameter import Parameter\nfrom ..utils import _indent\nfrom .. import tensor_types\nfrom ..nn import LeakyReLU\n\n\ndef _cells_state_info(cells, batch_size):\n    return sum([c().state_info(batch_size) for c in cells], [])\n\ndef _cells_begin_state(cells, **kwargs):\n    return sum([c().begin_state(**kwargs) for c in cells], [])\n\ndef _get_begin_state(cell, begin_state, inputs, batch_size):\n    if begin_state is None:\n        device = inputs.device if isinstance(inputs, tensor_types) else inputs[0].device\n        with device:\n            begin_state = cell.begin_state(func=np.zeros, batch_size=batch_size)\n    return begin_state\n\ndef _format_sequence(length, inputs, layout, merge, in_layout=None):\n    assert inputs is not None, \\\n        \"unroll(inputs=None) has been deprecated. \" \\\n        \"Please create input variables outside unroll.\"\n\n    axis = layout.find('T')\n    batch_axis = layout.find('N')\n    batch_size = 0\n    in_axis = in_layout.find('T') if in_layout is not None else axis\n    if isinstance(inputs, np.ndarray):\n        batch_size = inputs.shape[batch_axis]\n        if merge is False:\n            assert length is None or length == inputs.shape[in_axis]\n            inputs = _as_list(npx.slice_channel(inputs, axis=in_axis,\n                                                num_outputs=inputs.shape[in_axis],\n                                                squeeze_axis=1))\n    else:\n        assert isinstance(inputs, (list, tuple)), \\\n            \"Only support MXNet numpy ndarray or list of MXNet numpy ndarrays as inputs\"\n        assert length is None or len(inputs) == length\n        batch_size = inputs[0].shape[0]\n        if merge is True:\n            inputs = np.stack(inputs, axis=axis)\n            in_axis = axis\n\n    if isinstance(inputs, np.ndarray) and axis != in_axis:\n        inputs = np.swapaxes(inputs, axis, in_axis)\n\n    return inputs, axis, batch_size\n\ndef _mask_sequence_variable_length(data, length, valid_length, time_axis, merge):\n    assert valid_length is not None\n    if not isinstance(data, tensor_types):\n        data = np.stack(data, axis=time_axis)\n    outputs = npx.sequence_mask(data, sequence_length=valid_length, use_sequence_length=True,\n                                axis=time_axis)\n    if not merge:\n        outputs = _as_list(npx.slice_channel(outputs, num_outputs=length, axis=time_axis,\n                                             squeeze_axis=True))\n    return outputs\n\ndef _reverse_sequences(sequences, unroll_step, valid_length=None):\n    if valid_length is None:\n        reversed_sequences = list(reversed(sequences))\n    else:\n        reversed_sequences = npx.sequence_reverse(np.stack(sequences, axis=0),\n                                                  sequence_length=valid_length,\n                                                  use_sequence_length=True)\n        if unroll_step > 1:\n            reversed_sequences = npx.slice_channel(reversed_sequences, axis=0,\n                                                   num_outputs=unroll_step, squeeze_axis=True)\n        else:\n            reversed_sequences = [reversed_sequences[0]]\n\n    return reversed_sequences\n\n\n@use_np\nclass RecurrentCell(Block):\n    \"\"\"Abstract base class for RNN cells\n\n    \"\"\"\n    def __init__(self):\n        super(RecurrentCell, self).__init__()\n        self._modified = False\n        self.reset()\n\n    def reset(self):\n        \"\"\"Reset before re-using the cell for another graph.\"\"\"\n        self._init_counter = -1\n        self._counter = -1\n        for cell in self._children.values():\n            cell().reset()\n\n    def state_info(self, batch_size=0):\n        \"\"\"shape and layout information of states\"\"\"\n        raise NotImplementedError()\n\n    def begin_state(self, batch_size=0, func=np.zeros, **kwargs):\n        \"\"\"Initial state for this cell.\n\n        Parameters\n        ----------\n        func : callable, default symbol.zeros\n            Function for creating initial state.\n\n            For Symbol API, func can be `symbol.zeros`, `symbol.uniform`,\n            `symbol.var etc`. Use `symbol.var` if you want to directly\n            feed input as states.\n\n            For NDArray API, func can be `ndarray.zeros`, `ndarray.ones`, etc.\n        batch_size: int, default 0\n            Only required for NDArray API. Size of the batch ('N' in layout)\n            dimension of input.\n\n        **kwargs :\n            Additional keyword arguments passed to func. For example\n            `mean`, `std`, `dtype`, etc.\n\n        Returns\n        -------\n        states : nested list of Symbol\n            Starting states for the first RNN step.\n        \"\"\"\n        assert not self._modified, \\\n            \"After applying modifier cells (e.g. ZoneoutCell) the base \" \\\n            \"cell cannot be called directly. Call the modifier cell instead.\"\n        states = []\n        for info in self.state_info(batch_size):\n            if info is not None:\n                info.update(kwargs)\n            else:\n                info = kwargs\n            state = func(shape=info.pop(\"shape\", ()),\n                         device=info.pop(\"device\", cpu()),\n                         dtype=info.pop(\"dtype\", \"float32\"))\n            states.append(state)\n        return states\n\n    def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None,\n               valid_length=None):\n        \"\"\"Unrolls an RNN cell across time steps.\n\n        Parameters\n        ----------\n        length : int\n            Number of steps to unroll.\n        inputs : Symbol, list of Symbol, or None\n            If `inputs` is a single Symbol (usually the output\n            of Embedding symbol), it should have shape\n            (batch_size, length, ...) if `layout` is 'NTC',\n            or (length, batch_size, ...) if `layout` is 'TNC'.\n\n            If `inputs` is a list of symbols (usually output of\n            previous unroll), they should all have shape\n            (batch_size, ...).\n        begin_state : nested list of Symbol, optional\n            Input states created by `begin_state()`\n            or output state of another cell.\n            Created from `begin_state()` if `None`.\n        layout : str, optional\n            `layout` of input symbol. Only used if inputs\n            is a single Symbol.\n        merge_outputs : bool, optional\n            If `False`, returns outputs as a list of Symbols.\n            If `True`, concatenates output across time steps\n            and returns a single symbol with shape\n            (batch_size, length, ...) if layout is 'NTC',\n            or (length, batch_size, ...) if layout is 'TNC'.\n            If `None`, output whatever is faster.\n        valid_length : Symbol, NDArray or None\n            `valid_length` specifies the length of the sequences in the batch without padding.\n            This option is especially useful for building sequence-to-sequence models where\n            the input and output sequences would potentially be padded.\n            If `valid_length` is None, all sequences are assumed to have the same length.\n            If `valid_length` is a Symbol or NDArray, it should have shape (batch_size,).\n            The ith element will be the length of the ith sequence in the batch.\n            The last valid state will be return and the padded outputs will be masked with 0.\n            Note that `valid_length` must be smaller or equal to `length`.\n\n        Returns\n        -------\n        outputs : list of Symbol or Symbol\n            Symbol (if `merge_outputs` is True) or list of Symbols\n            (if `merge_outputs` is False) corresponding to the output from\n            the RNN from this unrolling.\n\n        states : list of Symbol\n            The new state of this RNN after this unrolling.\n            The type of this symbol is same as the output of `begin_state()`.\n        \"\"\"\n        # pylint: disable=too-many-locals\n        self.reset()\n\n        inputs, axis, batch_size = _format_sequence(length, inputs, layout, False)\n        begin_state = _get_begin_state(self, begin_state, inputs, batch_size)\n\n        states = begin_state\n        outputs = []\n        all_states = []\n        for i in range(length):\n            output, states = self(inputs[i], states)\n            outputs.append(output)\n            if valid_length is not None:\n                all_states.append(states)\n        if valid_length is not None:\n            states = [npx.sequence_last(np.stack(ele_list, axis=0),\n                                        sequence_length=valid_length,\n                                        use_sequence_length=True,\n                                        axis=0)\n                      for ele_list in zip(*all_states)]\n            outputs = _mask_sequence_variable_length(outputs, length, valid_length, axis, True)\n        outputs, _, _ = _format_sequence(length, outputs, layout, merge_outputs)\n\n        return outputs, states\n\n    #pylint: disable=no-self-use\n    def _get_activation(self, inputs, activation, **kwargs):\n        \"\"\"Get activation function. Convert if is string\"\"\"\n        func = {'tanh': np.tanh,\n                'relu': npx.relu,\n                'sigmoid': npx.sigmoid,\n                'softsign': npx.softsign}.get(activation)\n        if func:\n            return func(inputs, **kwargs)\n        elif isinstance(activation, string_types):\n            return npx.activation(inputs, act_type=activation, **kwargs)\n        elif isinstance(activation, LeakyReLU):\n            return npx.leaky_relu(inputs, act_type='leaky', slope=activation._alpha, **kwargs)\n        return activation(inputs, **kwargs)\n\n    def forward(self, inputs, states):\n        \"\"\"Unrolls the recurrent cell for one time step.\n\n        Parameters\n        ----------\n        inputs : sym.Variable\n            Input symbol, 2D, of shape (batch_size * num_units).\n        states : list of sym.Variable\n            RNN state from previous step or the output of begin_state().\n\n        Returns\n        -------\n        output : Symbol\n            Symbol corresponding to the output from the RNN when unrolling\n            for a single time step.\n        states : list of Symbol\n            The new state of this RNN after this unrolling.\n            The type of this symbol is same as the output of `begin_state()`.\n            This can be used as an input state to the next time step\n            of this RNN.\n\n        See Also\n        --------\n        begin_state: This function can provide the states for the first time step.\n        unroll: This function unrolls an RNN for a given number of (>=1) time steps.\n        \"\"\"\n        # pylint: disable= arguments-differ\n        self._counter += 1\n        return super(RecurrentCell, self).forward(inputs, states)\n\n@use_np\nclass HybridRecurrentCell(RecurrentCell, HybridBlock):\n    \"\"\"HybridRecurrentCell supports hybridize.\"\"\"\n    def __init__(self):\n        super(HybridRecurrentCell, self).__init__()\n\n    def forward(self, x, *args, **kwargs):\n        raise NotImplementedError\n\n\n@use_np\nclass RNNCell(HybridRecurrentCell):\n    r\"\"\"Elman RNN recurrent neural network cell.\n\n    Each call computes the following function:\n\n    .. math::\n\n        h_t = \\tanh(w_{ih} * x_t + b_{ih}  +  w_{hh} * h_{(t-1)} + b_{hh})\n\n    where :math:`h_t` is the hidden state at time `t`, and :math:`x_t` is the hidden\n    state of the previous layer at time `t` or :math:`input_t` for the first layer.\n    If nonlinearity='relu', then `ReLU` is used instead of `tanh`.\n\n    Parameters\n    ----------\n    hidden_size : int\n        Number of units in output symbol\n    activation : str or Symbol, default 'tanh'\n        Type of activation function.\n    i2h_weight_initializer : str or Initializer\n        Initializer for the input weights matrix, used for the linear\n        transformation of the inputs.\n    h2h_weight_initializer : str or Initializer\n        Initializer for the recurrent weights matrix, used for the linear\n        transformation of the recurrent state.\n    i2h_bias_initializer : str or Initializer, default 'zeros'\n        Initializer for the bias vector.\n    h2h_bias_initializer : str or Initializer, default 'zeros'\n        Initializer for the bias vector.\n    input_size: int, default 0\n        The number of expected features in the input x.\n        If not specified, it will be inferred from input.\n\n\n    Inputs:\n        - **data**: input tensor with shape `(batch_size, input_size)`.\n        - **states**: a list of one initial recurrent state tensor with shape\n          `(batch_size, num_hidden)`.\n\n    Outputs:\n        - **out**: output tensor with shape `(batch_size, num_hidden)`.\n        - **next_states**: a list of one output recurrent state tensor with the\n          same shape as `states`.\n    \"\"\"\n    def __init__(self, hidden_size, activation='tanh',\n                 i2h_weight_initializer=None, h2h_weight_initializer=None,\n                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',\n                 input_size=0):\n        super(RNNCell, self).__init__()\n        self._hidden_size = hidden_size\n        self._activation = activation\n        self._input_size = input_size\n        self.i2h_weight = Parameter('i2h_weight', shape=(hidden_size, input_size),\n                                    init=i2h_weight_initializer,\n                                    allow_deferred_init=True)\n        self.h2h_weight = Parameter('h2h_weight', shape=(hidden_size, hidden_size),\n                                    init=h2h_weight_initializer,\n                                    allow_deferred_init=True)\n        self.i2h_bias = Parameter('i2h_bias', shape=(hidden_size,),\n                                  init=i2h_bias_initializer,\n                                  allow_deferred_init=True)\n        self.h2h_bias = Parameter('h2h_bias', shape=(hidden_size,),\n                                  init=h2h_bias_initializer,\n                                  allow_deferred_init=True)\n\n    def state_info(self, batch_size=0):\n        return [{'shape': (batch_size, self._hidden_size), '__layout__': 'NC'}]\n\n    def _alias(self):\n        return 'rnn'\n\n    def __repr__(self):\n        s = '{name}({mapping}'\n        if hasattr(self, '_activation'):\n            s += ', {_activation}'\n        s += ')'\n        shape = self.i2h_weight.shape\n        mapping = '{0} -> {1}'.format(shape[1] if shape[1] else None, shape[0])\n        return s.format(name=self.__class__.__name__,\n                        mapping=mapping,\n                        **self.__dict__)\n\n    def forward(self, inputs, states):\n        device = inputs.device\n        i2h = npx.fully_connected(inputs, weight=self.i2h_weight.data(device),\n                                  bias=self.i2h_bias.data(device),\n                                  num_hidden=self._hidden_size,\n                                  no_bias=False)\n        h2h = npx.fully_connected(states[0].to_device(device),\n                                  weight=self.h2h_weight.data(device),\n                                  bias=self.h2h_bias.data(device),\n                                  num_hidden=self._hidden_size,\n                                  no_bias=False)\n        i2h_plus_h2h = i2h + h2h\n        output = self._get_activation(i2h_plus_h2h, self._activation)\n\n        return output, [output]\n\n    def infer_shape(self, i, x, is_bidirect):\n        if i == 0:\n            self.i2h_weight.shape = (self._hidden_size, x.shape[x.ndim-1])\n        else:\n            nh = self._hidden_size\n            if is_bidirect:\n                nh *= 2\n            self.i2h_weight.shape = (self._hidden_size, nh)\n\n\n@use_np\nclass LSTMCell(HybridRecurrentCell):\n    r\"\"\"Long-Short Term Memory (LSTM) network cell.\n\n    Each call computes the following function:\n\n    .. math::\n        \\begin{array}{ll}\n        i_t = sigmoid(W_{ii} x_t + b_{ii} + W_{hi} h_{(t-1)} + b_{hi}) \\\\\n        f_t = sigmoid(W_{if} x_t + b_{if} + W_{hf} h_{(t-1)} + b_{hf}) \\\\\n        g_t = \\tanh(W_{ig} x_t + b_{ig} + W_{hc} h_{(t-1)} + b_{hg}) \\\\\n        o_t = sigmoid(W_{io} x_t + b_{io} + W_{ho} h_{(t-1)} + b_{ho}) \\\\\n        c_t = f_t * c_{(t-1)} + i_t * g_t \\\\\n        h_t = o_t * \\tanh(c_t)\n        \\end{array}\n\n    where :math:`h_t` is the hidden state at time `t`, :math:`c_t` is the\n    cell state at time `t`, :math:`x_t` is the hidden state of the previous\n    layer at time `t` or :math:`input_t` for the first layer, and :math:`i_t`,\n    :math:`f_t`, :math:`g_t`, :math:`o_t` are the input, forget, cell, and\n    out gates, respectively.\n\n    Parameters\n    ----------\n    hidden_size : int\n        Number of units in output symbol.\n    i2h_weight_initializer : str or Initializer\n        Initializer for the input weights matrix, used for the linear\n        transformation of the inputs.\n    h2h_weight_initializer : str or Initializer\n        Initializer for the recurrent weights matrix, used for the linear\n        transformation of the recurrent state.\n    i2h_bias_initializer : str or Initializer, default 'zeros'\n        Initializer for the bias vector.\n    h2h_bias_initializer : str or Initializer, default 'zeros'\n        Initializer for the bias vector.\n    input_size: int, default 0\n        The number of expected features in the input x.\n        If not specified, it will be inferred from input.\n    activation : str, default 'tanh'\n        Activation type to use. See nd/symbol Activation\n        for supported types.\n    recurrent_activation : str, default 'sigmoid'\n        Activation type to use for the recurrent step. See nd/symbol Activation\n        for supported types.\n\n    Inputs:\n        - **data**: input tensor with shape `(batch_size, input_size)`.\n        - **states**: a list of two initial recurrent state tensors. Each has shape\n          `(batch_size, num_hidden)`.\n\n    Outputs:\n        - **out**: output tensor with shape `(batch_size, num_hidden)`.\n        - **next_states**: a list of two output recurrent state tensors. Each has\n          the same shape as `states`.\n    \"\"\"\n    # pylint: disable=too-many-instance-attributes\n    def __init__(self, hidden_size,\n                 i2h_weight_initializer=None, h2h_weight_initializer=None,\n                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',\n                 input_size=0, activation='tanh', recurrent_activation='sigmoid'):\n        super(LSTMCell, self).__init__()\n\n        self._hidden_size = hidden_size\n        self._input_size = input_size\n        self.i2h_weight = Parameter('i2h_weight', shape=(4*hidden_size, input_size),\n                                    init=i2h_weight_initializer,\n                                    allow_deferred_init=True)\n        self.h2h_weight = Parameter('h2h_weight', shape=(4*hidden_size, hidden_size),\n                                    init=h2h_weight_initializer,\n                                    allow_deferred_init=True)\n        self.i2h_bias = Parameter('i2h_bias', shape=(4*hidden_size,),\n                                  init=i2h_bias_initializer,\n                                  allow_deferred_init=True)\n        self.h2h_bias = Parameter('h2h_bias', shape=(4*hidden_size,),\n                                  init=h2h_bias_initializer,\n                                  allow_deferred_init=True)\n        self._activation = activation\n        self._recurrent_activation = recurrent_activation\n\n\n    def state_info(self, batch_size=0):\n        return [{'shape': (batch_size, self._hidden_size), '__layout__': 'NC'},\n                {'shape': (batch_size, self._hidden_size), '__layout__': 'NC'}]\n\n    def _alias(self):\n        return 'lstm'\n\n    def __repr__(self):\n        s = '{name}({mapping})'\n        shape = self.i2h_weight.shape\n        mapping = '{0} -> {1}'.format(shape[1] if shape[1] else None, shape[0])\n        return s.format(name=self.__class__.__name__,\n                        mapping=mapping,\n                        **self.__dict__)\n\n    def forward(self, inputs, states):\n        # pylint: disable=too-many-locals\n        device = inputs.device\n        i2h = npx.fully_connected(inputs, weight=self.i2h_weight.data(device),\n                                  bias=self.i2h_bias.data(device),\n                                  num_hidden=self._hidden_size*4, no_bias=False)\n        h2h = npx.fully_connected(states[0].to_device(device),\n                                  weight=self.h2h_weight.data(device),\n                                  bias=self.h2h_bias.data(device),\n                                  num_hidden=self._hidden_size*4, no_bias=False)\n        gates = i2h + h2h\n        slice_gates = npx.slice_channel(gates, num_outputs=4)\n        in_gate = self._get_activation(slice_gates[0], self._recurrent_activation)\n        forget_gate = self._get_activation(slice_gates[1], self._recurrent_activation)\n        in_transform = self._get_activation(slice_gates[2], self._activation)\n        out_gate = self._get_activation(slice_gates[3], self._recurrent_activation)\n        next_c = np.multiply(forget_gate, states[1].to_device(device)) + \\\n                 np.multiply(in_gate, in_transform)\n        next_h = np.multiply(out_gate, npx.activation(next_c, act_type=self._activation))\n\n        return next_h, [next_h, next_c]\n\n    def infer_shape(self, i, x, is_bidirect):\n        if i == 0:\n            self.i2h_weight.shape = (4*self._hidden_size, x.shape[x.ndim-1])\n        else:\n            nh = self._hidden_size\n            if is_bidirect:\n                nh *= 2\n            self.i2h_weight.shape = (4*self._hidden_size, nh)\n\n@use_np\nclass GRUCell(HybridRecurrentCell):\n    r\"\"\"Gated Rectified Unit (GRU) network cell.\n    Note: this is an implementation of the cuDNN version of GRUs\n    (slight modification compared to Cho et al. 2014; the reset gate :math:`r_t`\n    is applied after matrix multiplication).\n\n    Each call computes the following function:\n\n    .. math::\n        \\begin{array}{ll}\n        r_t = sigmoid(W_{ir} x_t + b_{ir} + W_{hr} h_{(t-1)} + b_{hr}) \\\\\n        i_t = sigmoid(W_{ii} x_t + b_{ii} + W_{hi} h_{(t-1)} + b_{hi}) \\\\\n        n_t = \\tanh(W_{in} x_t + b_{in} + r_t * (W_{hn} h_{(t-1)} + b_{hn})) \\\\\n        h_t = (1 - i_t) * n_t + i_t * h_{(t-1)} \\\\\n        \\end{array}\n\n    where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the hidden\n    state of the previous layer at time `t` or :math:`input_t` for the first layer,\n    and :math:`r_t`, :math:`i_t`, :math:`n_t` are the reset, input, and new gates, respectively.\n\n    Parameters\n    ----------\n    hidden_size : int\n        Number of units in output symbol.\n    i2h_weight_initializer : str or Initializer\n        Initializer for the input weights matrix, used for the linear\n        transformation of the inputs.\n    h2h_weight_initializer : str or Initializer\n        Initializer for the recurrent weights matrix, used for the linear\n        transformation of the recurrent state.\n    i2h_bias_initializer : str or Initializer, default 'zeros'\n        Initializer for the bias vector.\n    h2h_bias_initializer : str or Initializer, default 'zeros'\n        Initializer for the bias vector.\n    input_size: int, default 0\n        The number of expected features in the input x.\n        If not specified, it will be inferred from input.\n    activation : str, default 'tanh'\n        Activation type to use. See nd/symbol Activation\n        for supported types.\n    recurrent_activation : str, default 'sigmoid'\n        Activation type to use for the recurrent step. See nd/symbol Activation\n        for supported types.\n\n\n    Inputs:\n        - **data**: input tensor with shape `(batch_size, input_size)`.\n        - **states**: a list of one initial recurrent state tensor with shape\n          `(batch_size, num_hidden)`.\n\n    Outputs:\n        - **out**: output tensor with shape `(batch_size, num_hidden)`.\n        - **next_states**: a list of one output recurrent state tensor with the\n          same shape as `states`.\n    \"\"\"\n    def __init__(self, hidden_size,\n                 i2h_weight_initializer=None, h2h_weight_initializer=None,\n                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',\n                 input_size=0, activation='tanh', recurrent_activation='sigmoid'):\n        super(GRUCell, self).__init__()\n        self._hidden_size = hidden_size\n        self._input_size = input_size\n        self.i2h_weight = Parameter('i2h_weight', shape=(3*hidden_size, input_size),\n                                    init=i2h_weight_initializer,\n                                    allow_deferred_init=True)\n        self.h2h_weight = Parameter('h2h_weight', shape=(3*hidden_size, hidden_size),\n                                    init=h2h_weight_initializer,\n                                    allow_deferred_init=True)\n        self.i2h_bias = Parameter('i2h_bias', shape=(3*hidden_size,),\n                                  init=i2h_bias_initializer,\n                                  allow_deferred_init=True)\n        self.h2h_bias = Parameter('h2h_bias', shape=(3*hidden_size,),\n                                  init=h2h_bias_initializer,\n                                  allow_deferred_init=True)\n        self._activation = activation\n        self._recurrent_activation = recurrent_activation\n\n    def state_info(self, batch_size=0):\n        return [{'shape': (batch_size, self._hidden_size), '__layout__': 'NC'}]\n\n    def _alias(self):\n        return 'gru'\n\n    def __repr__(self):\n        s = '{name}({mapping})'\n        shape = self.i2h_weight.shape\n        mapping = '{0} -> {1}'.format(shape[1] if shape[1] else None, shape[0])\n        return s.format(name=self.__class__.__name__,\n                        mapping=mapping,\n                        **self.__dict__)\n\n    def forward(self, inputs, states):\n        # pylint: disable=too-many-locals\n        device = inputs.device\n        prev_state_h = states[0].to_device(device)\n        i2h = npx.fully_connected(inputs,\n                                  weight=self.i2h_weight.data(device),\n                                  bias=self.i2h_bias.data(device),\n                                  num_hidden=self._hidden_size * 3,\n                                  no_bias=False)\n        h2h = npx.fully_connected(prev_state_h,\n                                  weight=self.h2h_weight.data(device),\n                                  bias=self.h2h_bias.data(device),\n                                  num_hidden=self._hidden_size * 3,\n                                  no_bias=False)\n\n        i2h_r, i2h_z, i2h = npx.slice_channel(i2h, num_outputs=3)\n        h2h_r, h2h_z, h2h = npx.slice_channel(h2h, num_outputs=3)\n\n        reset_gate = self._get_activation(i2h_r + h2h_r,\n                                          self._recurrent_activation)\n        update_gate = self._get_activation(i2h_z + h2h_z,\n                                           self._recurrent_activation)\n        next_h_tmp = self._get_activation(i2h + np.multiply(reset_gate, h2h),\n                                          self._activation)\n        ones = np.ones(update_gate.shape)\n        next_h = np.multiply((ones - update_gate), next_h_tmp) + np.multiply(update_gate, prev_state_h)\n\n        return next_h, [next_h]\n\n    def infer_shape(self, i, x, is_bidirect):\n        if i == 0:\n            self.i2h_weight.shape = (3*self._hidden_size, x.shape[x.ndim-1])\n        else:\n            nh = self._hidden_size\n            if is_bidirect:\n                nh *= 2\n            self.i2h_weight.shape = (3*self._hidden_size, nh)\n\n@use_np\nclass SequentialRNNCell(RecurrentCell):\n    \"\"\"Sequentially stacking multiple RNN cells.\"\"\"\n    def __init__(self):\n        super(SequentialRNNCell, self).__init__()\n        self._layers = []\n\n    def __repr__(self):\n        s = '{name}(\\n{modstr}\\n)'\n        return s.format(name=self.__class__.__name__,\n                        modstr='\\n'.join(['({i}): {m}'.format(i=i, m=_indent(m().__repr__(), 2))\n                                          for i, m in self._children.items()]))\n\n    def add(self, cell):\n        \"\"\"Appends a cell into the stack.\n\n        Parameters\n        ----------\n        cell : RecurrentCell\n            The cell to add.\n        \"\"\"\n        self._layers.append(cell)\n        self.register_child(cell)\n\n    def state_info(self, batch_size=0):\n        return _cells_state_info(self._children.values(), batch_size)\n\n    def begin_state(self, **kwargs):\n        assert not self._modified, \\\n            \"After applying modifier cells (e.g. ZoneoutCell) the base \" \\\n            \"cell cannot be called directly. Call the modifier cell instead.\"\n        return _cells_begin_state(self._children.values(), **kwargs)\n\n    def __call__(self, inputs, states):\n        self._counter += 1\n        next_states = []\n        p = 0\n        assert all(not isinstance(cell(), BidirectionalCell) for cell in self._children.values())\n        for cell in self._children.values():\n            assert not isinstance(cell(), BidirectionalCell)\n            n = len(cell().state_info())\n            state = states[p:p+n]\n            p += n\n            inputs, state = cell()(inputs, state)\n            next_states.append(state)\n        return inputs, sum(next_states, [])\n\n    def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None,\n               valid_length=None):\n        # pylint: disable=too-many-locals\n        self.reset()\n\n        inputs, _, batch_size = _format_sequence(length, inputs, layout, None)\n        num_cells = len(self._children)\n        begin_state = _get_begin_state(self, begin_state, inputs, batch_size)\n\n        p = 0\n        next_states = []\n        for i, cell in enumerate(self._children.values()):\n            n = len(cell().state_info())\n            states = begin_state[p:p+n]\n            p += n\n            inputs, states = cell().unroll(length, inputs=inputs, begin_state=states,\n                                           layout=layout,\n                                           merge_outputs=None if i < num_cells-1 else merge_outputs,\n                                           valid_length=valid_length)\n            next_states.extend(states)\n\n        return inputs, next_states\n\n    def __getitem__(self, i):\n        return self._children[str(i)]()\n\n    def __len__(self):\n        return len(self._children)\n\n    def forward(self, *args, **kwargs):\n        # pylint: disable=missing-docstring\n        raise NotImplementedError\n\n    def infer_shape(self, _, x, is_bidirect):\n        for i, child in enumerate(self._layers):\n            child.infer_shape(i, x, is_bidirect)\n\n\n@use_np\nclass HybridSequentialRNNCell(HybridRecurrentCell):\n    \"\"\"Sequentially stacking multiple HybridRNN cells.\"\"\"\n    def __init__(self):\n        super(HybridSequentialRNNCell, self).__init__()\n        self._layers = []\n\n    def __repr__(self):\n        s = '{name}(\\n{modstr}\\n)'\n        return s.format(name=self.__class__.__name__,\n                        modstr='\\n'.join(['({i}): {m}'.format(i=i, m=_indent(m().__repr__(), 2))\n                                          for i, m in self._children.items()]))\n\n    def add(self, cell):\n        \"\"\"Appends a cell into the stack.\n\n        Parameters\n        ----------\n        cell : RecurrentCell\n            The cell to add.\n        \"\"\"\n        self._layers.append(cell)\n        self.register_child(cell)\n\n    def state_info(self, batch_size=0):\n        return _cells_state_info(self._children.values(), batch_size)\n\n    def begin_state(self, **kwargs):\n        assert not self._modified, \\\n            \"After applying modifier cells (e.g. ZoneoutCell) the base \" \\\n            \"cell cannot be called directly. Call the modifier cell instead.\"\n        return _cells_begin_state(self._children.values(), **kwargs)\n\n    def __call__(self, inputs, states):\n        self._counter += 1\n        next_states = []\n        p = 0\n        assert all(not isinstance(cell(), BidirectionalCell) for cell in self._children.values())\n        for cell in self._children.values():\n            n = len(cell().state_info())\n            state = states[p:p+n]\n            p += n\n            inputs, state = cell()(inputs, state)\n            next_states.append(state)\n        return inputs, sum(next_states, [])\n\n    def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None,\n               valid_length=None):\n        self.reset()\n\n        inputs, _, batch_size = _format_sequence(length, inputs, layout, None)\n        num_cells = len(self._children)\n        begin_state = _get_begin_state(self, begin_state, inputs, batch_size)\n\n        p = 0\n        next_states = []\n        for i, cell in enumerate(self._children.values()):\n            n = len(cell().state_info())\n            states = begin_state[p:p+n]\n            p += n\n            inputs, states = cell().unroll(length, inputs=inputs, begin_state=states,\n                                           layout=layout,\n                                           merge_outputs=None if i < num_cells-1 else merge_outputs,\n                                           valid_length=valid_length)\n            next_states.extend(states)\n\n        return inputs, next_states\n\n    def __getitem__(self, i):\n        return self._children[str(i)]()\n\n    def __len__(self):\n        return len(self._children)\n\n    def forward(self, inputs, states):\n        return self.__call__(inputs, states)\n\n    # pylint: disable=unused-argument\n    def infer_shape(self, _, x, is_bidirect):\n        for i, child in enumerate(self._layers):\n            child.infer_shape(i, x, False)\n\n\n@use_np\nclass DropoutCell(HybridRecurrentCell):\n    \"\"\"Applies dropout on input.\n\n    Parameters\n    ----------\n    rate : float\n        Percentage of elements to drop out, which\n        is 1 - percentage to retain.\n    axes : tuple of int, default ()\n        The axes on which dropout mask is shared. If empty, regular dropout is applied.\n\n\n    Inputs:\n        - **data**: input tensor with shape `(batch_size, size)`.\n        - **states**: a list of recurrent state tensors.\n\n    Outputs:\n        - **out**: output tensor with shape `(batch_size, size)`.\n        - **next_states**: returns input `states` directly.\n    \"\"\"\n    def __init__(self, rate, axes=()):\n        super(DropoutCell, self).__init__()\n        assert isinstance(rate, numeric_types), \"rate must be a number\"\n        self._rate = rate\n        self._axes = axes\n\n    def __repr__(self):\n        s = '{name}(rate={_rate}, axes={_axes})'\n        return s.format(name=self.__class__.__name__,\n                        **self.__dict__)\n\n    def state_info(self, batch_size=0):\n        return []\n\n    def _alias(self):\n        return 'dropout'\n\n    def forward(self, inputs, states):\n        if self._rate > 0:\n            inputs = npx.dropout(data=inputs, p=self._rate, axes=self._axes)\n        return inputs, states\n\n    def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None,\n               valid_length=None):\n        self.reset()\n\n        inputs, _, _ = _format_sequence(length, inputs, layout, merge_outputs)\n        if isinstance(inputs, tensor_types):\n            return self.forward(inputs, begin_state if begin_state else [])\n        return super(DropoutCell, self).unroll(\n            length, inputs, begin_state=begin_state, layout=layout,\n            merge_outputs=merge_outputs, valid_length=None)\n\n\n@use_np\nclass ModifierCell(HybridRecurrentCell):\n    \"\"\"Base class for modifier cells. A modifier\n    cell takes a base cell, apply modifications\n    on it (e.g. Zoneout), and returns a new cell.\n\n    After applying modifiers the base cell should\n    no longer be called directly. The modifier cell\n    should be used instead.\n    \"\"\"\n    def __init__(self, base_cell):\n        assert not base_cell._modified, \\\n            f\"Cell {base_cell.name} is already modified. One cell cannot be modified twice\"\n        base_cell._modified = True\n        super(ModifierCell, self).__init__()\n        self.base_cell = base_cell\n\n    @property\n    def params(self):\n        return self.base_cell.params\n\n    def state_info(self, batch_size=0):\n        return self.base_cell.state_info(batch_size)\n\n    def begin_state(self, func=np.zeros, **kwargs):\n        assert not self._modified, \\\n            \"After applying modifier cells (e.g. DropoutCell) the base \" \\\n            \"cell cannot be called directly. Call the modifier cell instead.\"\n        self.base_cell._modified = False\n        begin = self.base_cell.begin_state(func=func, **kwargs)\n        self.base_cell._modified = True\n        return begin\n\n    def forward(self, inputs, states):\n        raise NotImplementedError\n\n    def __repr__(self):\n        s = '{name}({base_cell})'\n        return s.format(name=self.__class__.__name__,\n                        **self.__dict__)\n\n\n@use_np\nclass ZoneoutCell(ModifierCell):\n    \"\"\"Applies Zoneout on base cell.\"\"\"\n    def __init__(self, base_cell, zoneout_outputs=0., zoneout_states=0.):\n        assert not isinstance(base_cell, BidirectionalCell), \\\n            \"BidirectionalCell doesn't support zoneout since it doesn't support step. \" \\\n            \"Please add ZoneoutCell to the cells underneath instead.\"\n        assert not isinstance(base_cell, SequentialRNNCell) or not base_cell._bidirectional, \\\n            \"Bidirectional SequentialRNNCell doesn't support zoneout. \" \\\n            \"Please add ZoneoutCell to the cells underneath instead.\"\n        super(ZoneoutCell, self).__init__(base_cell)\n        self.zoneout_outputs = zoneout_outputs\n        self.zoneout_states = zoneout_states\n        self._prev_output = None\n\n    def __repr__(self):\n        s = '{name}(p_out={zoneout_outputs}, p_state={zoneout_states}, {base_cell})'\n        return s.format(name=self.__class__.__name__,\n                        **self.__dict__)\n\n    def _alias(self):\n        return 'zoneout'\n\n    def reset(self):\n        super(ZoneoutCell, self).reset()\n        self._prev_output = None\n\n    def forward(self, inputs, states):\n        device = inputs.device\n        cell, p_outputs, p_states = self.base_cell, self.zoneout_outputs, self.zoneout_states\n        next_output, next_states = cell(inputs, states)\n        mask = (lambda p, like: npx.dropout(np.ones(like.shape), p=p))\n\n        prev_output = self._prev_output\n        if prev_output is None:\n            prev_output = np.zeros(next_output.shape)\n\n        output = (np.where(mask(p_outputs, next_output), next_output, prev_output)\n                  if p_outputs != 0. else next_output)\n        states = ([np.where(mask(p_states, new_s), new_s, old_s.to_device(device)) for new_s, old_s in\n                   zip(next_states, states)] if p_states != 0. else next_states)\n\n        self._prev_output = output\n\n        return output, states\n\n    def infer_shape(self, i, x, is_bidirect):\n        self.base_cell.infer_shape(i, x, is_bidirect)\n\n@use_np\nclass ResidualCell(ModifierCell):\n    \"\"\"\n    Adds residual connection as described in Wu et al, 2016\n    (https://arxiv.org/abs/1609.08144).\n    Output of the cell is output of the base cell plus input.\n    \"\"\"\n\n    def __init__(self, base_cell):\n        # pylint: disable=useless-super-delegation\n        super(ResidualCell, self).__init__(base_cell)\n\n    def forward(self, inputs, states):\n        output, states = self.base_cell(inputs, states)\n        output = output + inputs\n        return output, states\n\n    def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None,\n               valid_length=None):\n        self.reset()\n\n        self.base_cell._modified = False\n        outputs, states = self.base_cell.unroll(length, inputs=inputs, begin_state=begin_state,\n                                                layout=layout, merge_outputs=merge_outputs,\n                                                valid_length=valid_length)\n        self.base_cell._modified = True\n\n        merge_outputs = isinstance(outputs, tensor_types) if merge_outputs is None else \\\n                        merge_outputs\n        inputs, axis, _ = _format_sequence(length, inputs, layout, merge_outputs)\n        if valid_length is not None:\n            # mask the padded inputs to zero\n            inputs = _mask_sequence_variable_length(inputs, length, valid_length, axis,\n                                                    merge_outputs)\n        if merge_outputs:\n            outputs = outputs + inputs\n        else:\n            outputs = [i + j for i, j in zip(outputs, inputs)]\n\n        return outputs, states\n\n    def infer_shape(self, i, x, is_bidirect):\n        self.base_cell.infer_shape(i, x, is_bidirect)\n\n\n@use_np\nclass BidirectionalCell(HybridRecurrentCell):\n    \"\"\"Bidirectional RNN cell.\n\n    Parameters\n    ----------\n    l_cell : RecurrentCell\n        Cell for forward unrolling\n    r_cell : RecurrentCell\n        Cell for backward unrolling\n    \"\"\"\n    def __init__(self, l_cell, r_cell):\n        super(BidirectionalCell, self).__init__()\n        self.l_cell = l_cell\n        self.r_cell = r_cell\n\n    def __call__(self, inputs, states):\n        raise NotImplementedError(\"Bidirectional cannot be stepped. Please use unroll\")\n\n    def __repr__(self):\n        s = '{name}(forward={l_cell}, backward={r_cell})'\n        return s.format(name=self.__class__.__name__,\n                        l_cell=self._children['l_cell'](),\n                        r_cell=self._children['r_cell']())\n\n    def state_info(self, batch_size=0):\n        return _cells_state_info(self._children.values(), batch_size)\n\n    def begin_state(self, **kwargs):\n        assert not self._modified, \\\n            \"After applying modifier cells (e.g. DropoutCell) the base \" \\\n            \"cell cannot be called directly. Call the modifier cell instead.\"\n        return _cells_begin_state(self._children.values(), **kwargs)\n\n    def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None,\n               valid_length=None):\n        # pylint: disable=too-many-locals\n        self.reset()\n\n        inputs, axis, batch_size = _format_sequence(length, inputs, layout, False)\n        reversed_inputs = list(_reverse_sequences(inputs, length, valid_length))\n        begin_state = _get_begin_state(self, begin_state, inputs, batch_size)\n\n        states = begin_state\n        l_cell, r_cell = [c() for c in self._children.values()]\n        l_outputs, l_states = l_cell.unroll(length, inputs=inputs,\n                                            begin_state=states[:len(l_cell.state_info(batch_size))],\n                                            layout=layout, merge_outputs=merge_outputs,\n                                            valid_length=valid_length)\n        r_outputs, r_states = r_cell.unroll(length,\n                                            inputs=reversed_inputs,\n                                            begin_state=states[len(l_cell.state_info(batch_size)):],\n                                            layout=layout, merge_outputs=False,\n                                            valid_length=valid_length)\n        reversed_r_outputs = _reverse_sequences(r_outputs, length, valid_length)\n\n        if merge_outputs is None:\n            merge_outputs = isinstance(l_outputs, tensor_types)\n            l_outputs, _, _ = _format_sequence(None, l_outputs, layout, merge_outputs)\n            reversed_r_outputs, _, _ = _format_sequence(None, reversed_r_outputs, layout,\n                                                        merge_outputs)\n\n        if merge_outputs:\n            reversed_r_outputs = np.stack(reversed_r_outputs, axis=axis)\n            outputs = np.concatenate([l_outputs, reversed_r_outputs], axis=2)\n\n        else:\n            outputs = [np.concatenate([l_o, r_o], axis=1)\n                       for i, (l_o, r_o) in enumerate(zip(l_outputs, reversed_r_outputs))]\n        if valid_length is not None:\n            outputs = _mask_sequence_variable_length(outputs, length, valid_length, axis,\n                                                     merge_outputs)\n        states = l_states + r_states\n        return outputs, states\n\n    #pylint: disable=W0613\n    def infer_shape(self, i, x, is_bidirect):\n        l_cell, r_cell = [c() for c in self._children.values()]\n        l_cell.infer_shape(i, x, True)\n        r_cell.infer_shape(i, x, True)\n\n@use_np\nclass VariationalDropoutCell(ModifierCell):\n    \"\"\"\n    Applies Variational Dropout on base cell.\n    https://arxiv.org/pdf/1512.05287.pdf\n\n    Variational dropout uses the same dropout mask across time-steps. It can be applied to RNN\n    inputs, outputs, and states. The masks for them are not shared.\n\n    The dropout mask is initialized when stepping forward for the first time and will remain\n    the same until .reset() is called. Thus, if using the cell and stepping manually without calling\n    .unroll(), the .reset() should be called after each sequence.\n\n    Parameters\n    ----------\n    base_cell : RecurrentCell\n        The cell on which to perform variational dropout.\n    drop_inputs : float, default 0.\n        The dropout rate for inputs. Won't apply dropout if it equals 0.\n    drop_states : float, default 0.\n        The dropout rate for state inputs on the first state channel.\n        Won't apply dropout if it equals 0.\n    drop_outputs : float, default 0.\n        The dropout rate for outputs. Won't apply dropout if it equals 0.\n    \"\"\"\n    def __init__(self, base_cell, drop_inputs=0., drop_states=0., drop_outputs=0.):\n        assert not drop_states or not isinstance(base_cell, BidirectionalCell), \\\n            \"BidirectionalCell doesn't support variational state dropout. \" \\\n            \"Please add VariationalDropoutCell to the cells underneath instead.\"\n        assert not drop_states \\\n               or not isinstance(base_cell, SequentialRNNCell) or not base_cell._bidirectional, \\\n            \"Bidirectional SequentialRNNCell doesn't support variational state dropout. \" \\\n            \"Please add VariationalDropoutCell to the cells underneath instead.\"\n        super(VariationalDropoutCell, self).__init__(base_cell)\n        self.drop_inputs = drop_inputs\n        self.drop_states = drop_states\n        self.drop_outputs = drop_outputs\n        self.drop_inputs_mask = None\n        self.drop_states_mask = None\n        self.drop_outputs_mask = None\n\n    def _alias(self):\n        return 'vardrop'\n\n    def reset(self):\n        super(VariationalDropoutCell, self).reset()\n        self.drop_inputs_mask = None\n        self.drop_states_mask = None\n        self.drop_outputs_mask = None\n\n    def _initialize_input_masks(self, inputs, states):\n        if self.drop_states and self.drop_states_mask is None:\n            self.drop_states_mask = npx.dropout(np.ones(states[0].shape),\n                                                p=self.drop_states)\n\n        if self.drop_inputs and self.drop_inputs_mask is None:\n            self.drop_inputs_mask = npx.dropout(np.ones(inputs.shape),\n                                                p=self.drop_inputs)\n\n    def _initialize_output_mask(self, output):\n        if self.drop_outputs and self.drop_outputs_mask is None:\n            self.drop_outputs_mask = npx.dropout(np.ones(output.shape),\n                                                 p=self.drop_outputs)\n\n\n    def forward(self, inputs, states):\n        device = inputs.device\n        cell = self.base_cell\n        self._initialize_input_masks(inputs, states)\n\n        if self.drop_states:\n            states = list(states)\n            # state dropout only needs to be applied on h, which is always the first state.\n            states[0] = states[0].to_device(device) * self.drop_states_mask\n\n        if self.drop_inputs:\n            inputs = inputs * self.drop_inputs_mask\n\n        next_output, next_states = cell(inputs, states)\n\n        self._initialize_output_mask(next_output)\n        if self.drop_outputs:\n            next_output = next_output * self.drop_outputs_mask\n\n        return next_output, next_states\n\n    def __repr__(self):\n        s = '{name}(p_out = {drop_outputs}, p_state = {drop_states})'\n        return s.format(name=self.__class__.__name__,\n                        **self.__dict__)\n\n    def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None,\n               valid_length=None):\n        \"\"\"Unrolls an RNN cell across time steps.\n\n        Parameters\n        ----------\n        length : int\n            Number of steps to unroll.\n        inputs : Symbol, list of Symbol, or None\n            If `inputs` is a single Symbol (usually the output\n            of Embedding symbol), it should have shape\n            (batch_size, length, ...) if `layout` is 'NTC',\n            or (length, batch_size, ...) if `layout` is 'TNC'.\n\n            If `inputs` is a list of symbols (usually output of\n            previous unroll), they should all have shape\n            (batch_size, ...).\n        begin_state : nested list of Symbol, optional\n            Input states created by `begin_state()`\n            or output state of another cell.\n            Created from `begin_state()` if `None`.\n        layout : str, optional\n            `layout` of input symbol. Only used if inputs\n            is a single Symbol.\n        merge_outputs : bool, optional\n            If `False`, returns outputs as a list of Symbols.\n            If `True`, concatenates output across time steps\n            and returns a single symbol with shape\n            (batch_size, length, ...) if layout is 'NTC',\n            or (length, batch_size, ...) if layout is 'TNC'.\n            If `None`, output whatever is faster.\n        valid_length : Symbol, NDArray or None\n            `valid_length` specifies the length of the sequences in the batch without padding.\n            This option is especially useful for building sequence-to-sequence models where\n            the input and output sequences would potentially be padded.\n            If `valid_length` is None, all sequences are assumed to have the same length.\n            If `valid_length` is a Symbol or NDArray, it should have shape (batch_size,).\n            The ith element will be the length of the ith sequence in the batch.\n            The last valid state will be return and the padded outputs will be masked with 0.\n            Note that `valid_length` must be smaller or equal to `length`.\n\n        Returns\n        -------\n        outputs : list of Symbol or Symbol\n            Symbol (if `merge_outputs` is True) or list of Symbols\n            (if `merge_outputs` is False) corresponding to the output from\n            the RNN from this unrolling.\n\n        states : list of Symbol\n            The new state of this RNN after this unrolling.\n            The type of this symbol is same as the output of `begin_state()`.\n        \"\"\"\n\n        # Dropout on inputs and outputs can be performed on the whole sequence\n        # only when state dropout is not present.\n        if self.drop_states:\n            return super(VariationalDropoutCell, self).unroll(length, inputs, begin_state,\n                                                              layout, merge_outputs,\n                                                              valid_length=valid_length)\n\n        self.reset()\n\n        inputs, axis, batch_size = _format_sequence(length, inputs, layout, True)\n        states = _get_begin_state(self, begin_state, inputs, batch_size)\n\n        if self.drop_inputs:\n            inputs = npx.dropout(inputs, p=self.drop_inputs, axes=(axis,))\n\n        outputs, states = self.base_cell.unroll(length, inputs, states, layout, merge_outputs=True,\n                                                valid_length=valid_length)\n        if self.drop_outputs:\n            outputs = npx.dropout(outputs, p=self.drop_outputs, axes=(axis,))\n        merge_outputs = isinstance(outputs, tensor_types) if merge_outputs is None else \\\n            merge_outputs\n        outputs, _, _ = _format_sequence(length, outputs, layout, merge_outputs)\n        if valid_length is not None:\n            outputs = _mask_sequence_variable_length(outputs, length, valid_length, axis,\n                                                     merge_outputs)\n        return outputs, states\n\n    def infer_shape(self, i, x, is_bidirect):\n        self.base_cell.infer_shape(i, x, is_bidirect)\n\n@use_np\nclass LSTMPCell(HybridRecurrentCell):\n    r\"\"\"Long-Short Term Memory Projected (LSTMP) network cell.\n    (https://arxiv.org/abs/1402.1128)\n\n    Each call computes the following function:\n\n    .. math::\n        \\begin{array}{ll}\n        i_t = sigmoid(W_{ii} x_t + b_{ii} + W_{ri} r_{(t-1)} + b_{ri}) \\\\\n        f_t = sigmoid(W_{if} x_t + b_{if} + W_{rf} r_{(t-1)} + b_{rf}) \\\\\n        g_t = \\tanh(W_{ig} x_t + b_{ig} + W_{rc} r_{(t-1)} + b_{rg}) \\\\\n        o_t = sigmoid(W_{io} x_t + b_{io} + W_{ro} r_{(t-1)} + b_{ro}) \\\\\n        c_t = f_t * c_{(t-1)} + i_t * g_t \\\\\n        h_t = o_t * \\tanh(c_t) \\\\\n        r_t = W_{hr} h_t\n        \\end{array}\n\n    where :math:`r_t` is the projected recurrent activation at time `t`,\n    :math:`h_t` is the hidden state at time `t`, :math:`c_t` is the\n    cell state at time `t`, :math:`x_t` is the input at time `t`, and :math:`i_t`,\n    :math:`f_t`, :math:`g_t`, :math:`o_t` are the input, forget, cell, and\n    out gates, respectively.\n\n    Parameters\n    ----------\n\n    hidden_size : int\n        Number of units in cell state symbol.\n    projection_size : int\n        Number of units in output symbol.\n    i2h_weight_initializer : str or Initializer\n        Initializer for the input weights matrix, used for the linear\n        transformation of the inputs.\n    h2h_weight_initializer : str or Initializer\n        Initializer for the recurrent weights matrix, used for the linear\n        transformation of the hidden state.\n    h2r_weight_initializer : str or Initializer\n        Initializer for the projection weights matrix, used for the linear\n        transformation of the recurrent state.\n    i2h_bias_initializer : str or Initializer, default 'lstmbias'\n        Initializer for the bias vector. By default, bias for the forget\n        gate is initialized to 1 while all other biases are initialized\n        to zero.\n    h2h_bias_initializer : str or Initializer\n        Initializer for the bias vector.\n    Inputs:\n        - **data**: input tensor with shape `(batch_size, input_size)`.\n        - **states**: a list of two initial recurrent state tensors, with shape\n          `(batch_size, projection_size)` and `(batch_size, hidden_size)` respectively.\n    Outputs:\n        - **out**: output tensor with shape `(batch_size, num_hidden)`.\n        - **next_states**: a list of two output recurrent state tensors. Each has\n          the same shape as `states`.\n    \"\"\"\n    def __init__(self, hidden_size, projection_size,\n                 i2h_weight_initializer=None, h2h_weight_initializer=None,\n                 h2r_weight_initializer=None,\n                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',\n                 input_size=0):\n        super(LSTMPCell, self).__init__()\n\n        self._hidden_size = hidden_size\n        self._input_size = input_size\n        self._projection_size = projection_size\n        self.i2h_weight = Parameter('i2h_weight', shape=(4*hidden_size, input_size),\n                                    init=i2h_weight_initializer,\n                                    allow_deferred_init=True)\n        self.h2h_weight = Parameter('h2h_weight', shape=(4*hidden_size, projection_size),\n                                    init=h2h_weight_initializer,\n                                    allow_deferred_init=True)\n        self.h2r_weight = Parameter('h2r_weight', shape=(projection_size, hidden_size),\n                                    init=h2r_weight_initializer,\n                                    allow_deferred_init=True)\n        self.i2h_bias = Parameter('i2h_bias', shape=(4*hidden_size,),\n                                  init=i2h_bias_initializer,\n                                  allow_deferred_init=True)\n        self.h2h_bias = Parameter('h2h_bias', shape=(4*hidden_size,),\n                                  init=h2h_bias_initializer,\n                                  allow_deferred_init=True)\n\n    def state_info(self, batch_size=0):\n        return [{'shape': (batch_size, self._projection_size), '__layout__': 'NC'},\n                {'shape': (batch_size, self._hidden_size), '__layout__': 'NC'}]\n\n    def _alias(self):\n        return 'lstmp'\n\n    def __repr__(self):\n        s = '{name}({mapping})'\n        shape = self.i2h_weight.shape\n        proj_shape = self.h2r_weight.shape\n        mapping = '{0} -> {1} -> {2}'.format(shape[1] if shape[1] else None,\n                                             shape[0], proj_shape[0])\n        return s.format(name=self.__class__.__name__,\n                        mapping=mapping,\n                        **self.__dict__)\n\n    # pylint: disable= arguments-differ\n    def forward(self, inputs, states):\n        device = inputs.device\n        i2h = npx.fully_connected(inputs, weight=self.i2h_weight.data(device),\n                                  bias=self.i2h_bias.data(device),\n                                  num_hidden=self._hidden_size*4, no_bias=False)\n        h2h = npx.fully_connected(states[0].to_device(device),\n                                  weight=self.h2h_weight.data(device),\n                                  bias=self.h2h_bias.data(device),\n                                  num_hidden=self._hidden_size*4, no_bias=False)\n        gates = i2h + h2h\n        slice_gates = npx.slice_channel(gates, num_outputs=4)\n        in_gate = npx.activation(slice_gates[0], act_type=\"sigmoid\")\n        forget_gate = npx.activation(slice_gates[1], act_type=\"sigmoid\")\n        in_transform = npx.activation(slice_gates[2], act_type=\"tanh\")\n        out_gate = npx.activation(slice_gates[3], act_type=\"sigmoid\")\n        next_c = forget_gate * states[1].to_device(device) + in_gate * in_transform\n        hidden = np.multiply(out_gate, npx.activation(next_c, act_type=\"tanh\"))\n        next_r = npx.fully_connected(hidden, num_hidden=self._projection_size,\n                                     weight=self.h2r_weight.data(device), no_bias=True)\n\n        return next_r, [next_r, next_c]\n\n    def infer_shape(self, i, x, is_bidirect):\n        if i == 0:\n            self.i2h_weight.shape = (4*self._hidden_size, x.shape[x.ndim-1])\n        else:\n            nh = self._projection_size\n            if is_bidirect:\n                nh *= 2\n            self.i2h_weight.shape = (4*self._hidden_size, nh)\n\n\ndef dynamic_unroll(cell, inputs, begin_state, drop_inputs=0, drop_outputs=0,\n                   layout='TNC', valid_length=None):\n    \"\"\"Unrolls an RNN cell across time steps.\n\n    Currently, 'TNC' is a preferred layout. unroll on the input of this layout\n    runs much faster.\n\n    Parameters\n    ----------\n    cell : an object whose base class is RNNCell.\n        The RNN cell to run on the input sequence.\n    inputs : Symbol\n        It should have shape (batch_size, length, ...) if `layout` is 'NTC',\n        or (length, batch_size, ...) if `layout` is 'TNC'.\n    begin_state : nested list of Symbol\n        The initial states of the RNN sequence.\n    drop_inputs : float, default 0.\n        The dropout rate for inputs. Won't apply dropout if it equals 0.\n    drop_outputs : float, default 0.\n        The dropout rate for outputs. Won't apply dropout if it equals 0.\n    layout : str, optional\n        `layout` of input symbol. Only used if inputs\n        is a single Symbol.\n    valid_length : Symbol, NDArray or None\n        `valid_length` specifies the length of the sequences in the batch without padding.\n        This option is especially useful for building sequence-to-sequence models where\n        the input and output sequences would potentially be padded.\n        If `valid_length` is None, all sequences are assumed to have the same length.\n        If `valid_length` is a Symbol or NDArray, it should have shape (batch_size,).\n        The ith element will be the length of the ith sequence in the batch.\n        The last valid state will be return and the padded outputs will be masked with 0.\n        Note that `valid_length` must be smaller or equal to `length`.\n\n    Returns\n    -------\n    outputs : Symbol\n        the output of the RNN from this unrolling.\n\n    states : list of Symbol\n        The new state of this RNN after this unrolling.\n        The type of this symbol is same as the output of `begin_state`.\n\n    Examples\n    --------\n    >>> seq_len = 3\n    >>> batch_size = 2\n    >>> input_size = 5\n    >>> cell = mx.gluon.rnn.LSTMCell(input_size)\n    >>> cell.initialize(device=mx.cpu())\n    >>> rnn_data = mx.np.normal(loc=0, scale=1, shape=(seq_len, batch_size, input_size))\n    >>> state_shape = (batch_size, input_size)\n    >>> states = [mx.np.normal(loc=0, scale=1, shape=state_shape) for i in range(2)]\n    >>> valid_length = mx.np.array([2, 3])\n    >>> output, states = mx.gluon.rnn.rnn_cell.dynamic_unroll(cell, rnn_data, states,\n    ...                                                       valid_length=valid_length,\n    ...                                                       layout='TNC')\n    >>> print(output)\n    [[[ 0.00767238  0.00023103  0.03973929 -0.00925503 -0.05660512]\n      [ 0.00881535  0.05428379 -0.02493718 -0.01834097  0.02189514]]\n     [[-0.00676967  0.01447039  0.01287002 -0.00574152 -0.05734247]\n      [ 0.01568508  0.02650866 -0.04270559 -0.04328435  0.00904011]]\n     [[ 0.          0.          0.          0.          0.        ]\n      [ 0.01055336  0.02734251 -0.03153727 -0.03742751 -0.01378113]]]\n     <NDArray 3x2x5 @cpu(0)>\n    \"\"\"\n\n    # Merge is always True, so we don't need length.\n    inputs, axis, _ = _format_sequence(0, inputs, layout, True)\n    if axis != 0:\n        axes = list(range(len(layout)))\n        tmp = axes[0]\n        axes[0] = axes[axis]\n        axes[axis] = tmp\n        inputs = np.transpose(inputs, axes=axes)\n    states = begin_state\n\n    if drop_inputs:\n        inputs = npx.dropout(inputs, p=drop_inputs, axes=(axis,))\n\n    if valid_length is None:\n        outputs, states = npx.foreach(cell, inputs, states + [valid_length])\n    else:\n        zeros = []\n        for s in states:\n            zeros.append(np.zeros(s.shape))\n        states = list(_as_list(states))\n        states.append(np.zeros((1)))\n        class loop_body(HybridBlock):\n            \"\"\"Loop body for foreach operator\"\"\"\n            def __init__(self, cell):\n                super(loop_body, self).__init__()\n                self.cell = cell\n\n            def forward(self, inputs, states):\n                valid_len = states.pop()\n                cell_states = states[:-1]\n                iter_no = states[-1]\n                out, new_states = self.cell(inputs, cell_states)\n                for i, state in enumerate(cell_states):\n                    cond = npx.broadcast_greater(valid_len, iter_no)\n                    cond_broad = np.broadcast_to(cond, new_states[i].T.shape).T\n                    new_states[i] = np.where(cond_broad, new_states[i], state)\n                new_states.append(iter_no + 1)\n                new_states.append(valid_len)\n                return out, new_states\n        body = loop_body(cell)\n        outputs, states = npx.foreach(body, inputs, states + [valid_length])\n        states.pop()\n    if drop_outputs:\n        outputs = npx.dropout(outputs, p=drop_outputs, axes=(axis,))\n    if valid_length is not None:\n        if axis != 0:\n            outputs = np.transpose(outputs, axes)\n        outputs = npx.sequence_mask(outputs, sequence_length=valid_length,\n                                    use_sequence_length=True, axis=axis)\n        # the last state is the iteration number. We don't need it.\n        return outputs, states[:-1]\n    else:\n        if axis != 0:\n            outputs = np.transpose(outputs, axes)\n        return outputs, states\n"
  },
  {
    "path": "python/mxnet/gluon/rnn/rnn_layer.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=no-member, invalid-name, protected-access, no-self-use\n# pylint: disable=too-many-branches, too-many-arguments, no-self-use\n# pylint: disable=too-many-lines, arguments-differ\n\"\"\"Definition of various recurrent neural network layers.\"\"\"\n\n__all__ = ['RNN', 'LSTM', 'GRU']\n\nfrom ... import np, npx, initializer, cpu\nfrom .. import HybridBlock, tensor_types\nfrom ..parameter import Parameter\nfrom ...util import use_np\n\n\n@use_np\nclass _RNNLayer(HybridBlock):\n    \"\"\"Implementation of recurrent layers.\"\"\"\n    def __init__(self, hidden_size, num_layers, layout,\n                 dropout, bidirectional, input_size,\n                 i2h_weight_initializer, h2h_weight_initializer,\n                 i2h_bias_initializer, h2h_bias_initializer,\n                 mode, projection_size, h2r_weight_initializer,\n                 lstm_state_clip_min, lstm_state_clip_max, lstm_state_clip_nan,\n                 dtype, use_sequence_length=False, **kwargs):\n        super(_RNNLayer, self).__init__(**kwargs)\n        assert layout in ('TNC', 'NTC'), \\\n            f\"Invalid layout {layout}; must be one of ['TNC' or 'NTC']\"\n        self._hidden_size = hidden_size\n        self._projection_size = projection_size if projection_size else None\n        self._num_layers = num_layers\n        self._mode = mode\n        self._layout = layout\n        self._dropout = dropout\n        self._dir = 2 if bidirectional else 1\n        self._input_size = input_size\n        self._lstm_state_clip_min = lstm_state_clip_min\n        self._lstm_state_clip_max = lstm_state_clip_max\n        self._lstm_state_clip_nan = lstm_state_clip_nan\n        self._dtype = dtype\n        self._use_sequence_length = use_sequence_length\n        self.skip_states = None\n\n        self._gates = {'rnn_relu': 1, 'rnn_tanh': 1, 'lstm': 4, 'gru': 3}[mode]\n\n        param_initializer = initializer.RNNFused(\n            mode, num_layers, hidden_size,\n            bidirectional, projection_size,\n            i2h_weight_initializer=i2h_weight_initializer,\n            h2h_weight_initializer=h2h_weight_initializer,\n            i2h_bias_initializer=i2h_bias_initializer,\n            h2h_bias_initializer=h2h_bias_initializer,\n            h2r_weight_initializer=h2r_weight_initializer)\n\n        self.rnn_param = Parameter('rnn_param', shape=(-1,), init=param_initializer,\n                                   allow_deferred_init=True, dtype=dtype)\n\n    def __repr__(self):\n        s = '{name}({mapping}, {_layout}'\n        if self._num_layers != 1:\n            s += ', num_layers={_num_layers}'\n        if self._dropout != 0:\n            s += ', dropout={_dropout}'\n        if self._dir == 2:\n            s += ', bidirectional'\n        s += ')'\n        mapping = '{0} -> {1}'.format(self._input_size if self._input_size else None, self._hidden_size)\n        return s.format(name=self.__class__.__name__,\n                        mapping=mapping,\n                        **self.__dict__)\n\n    def state_info(self, batch_size=0):\n        raise NotImplementedError\n\n    def cast(self, dtype):\n        super(_RNNLayer, self).cast(dtype)\n        self._dtype = dtype\n\n    def begin_state(self, batch_size=0, func=np.zeros, **kwargs):\n        \"\"\"Initial state for this cell.\n\n        Parameters\n        ----------\n        batch_size: int\n            Only required for `NDArray` API. Size of the batch ('N' in layout).\n            Dimension of the input.\n        func : callable, default `ndarray.zeros`\n            Function for creating initial state.\n\n            For Symbol API, func can be `symbol.zeros`, `symbol.uniform`,\n            `symbol.var` etc. Use `symbol.var` if you want to directly\n            feed input as states.\n\n            For NDArray API, func can be `ndarray.zeros`, `ndarray.ones`, etc.\n\n        **kwargs :\n            Additional keyword arguments passed to func. For example\n            `mean`, `std`, `dtype`, etc.\n\n        Returns\n        -------\n        states : nested list of Symbol\n            Starting states for the first RNN step.\n        \"\"\"\n        states = []\n        for info in self.state_info(batch_size):\n            if info is not None:\n                info.update(kwargs)\n            else:\n                info = kwargs\n            state = func(shape=info.pop(\"shape\", ()),\n                         device=info.pop(\"device\", cpu()),\n                         dtype=info.pop(\"dtype\", \"float32\"))\n            states.append(state)\n        return states\n\n    def __call__(self, inputs, states=None, sequence_length=None, **kwargs):\n        self.skip_states = states is None\n        if states is None:\n            batch_size = inputs.shape[self._layout.find('N')]\n            states = self.begin_state(batch_size, device=inputs.device, dtype=inputs.dtype)\n        if isinstance(states, tensor_types):\n            states = [states]\n\n        if self._use_sequence_length:\n            return super(_RNNLayer, self).__call__(inputs, states, sequence_length, **kwargs)\n        else:\n            return super(_RNNLayer, self).__call__(inputs, states, **kwargs)\n\n    def forward(self, inputs, states, sequence_length=None):\n        batch_size = inputs.shape[self._layout.find('N')]\n\n        for state, info in zip(states, self.state_info(batch_size)):\n            if state.shape != info['shape']:\n                raise ValueError(\n                    f\"Invalid recurrent state shape. Expecting {str(info['shape'])}, got {str(state.shape)}.\")\n        out = self._forward_kernel(inputs, states, sequence_length)\n\n        # out is (output, state)\n        return out[0] if self.skip_states else out\n\n    def infer_shape(self, inputs, *args):\n        assert inputs.ndim == 3, \\\n            \"Input data should be rank-3 tensor of dim [sequence length, batch size, input size]\"\n        self._input_size = inputs.shape[2]\n        ng, ni, nh = self._gates, inputs.shape[2], self._hidden_size\n\n        size = nh * self._dir * ng\n        size1 = (ni + nh + 2) * size  # first layer size\n        size2 = (nh * self._dir + nh + 2) * size  # second layer size\n        if self._projection_size:\n            size1 = (ni + self._projection_size + 2) * size  # first layer size\n            size2 = (self._projection_size * self._dir + \\\n                self._projection_size + 2) * size  # second layer size\n        param_size = size1 + (self._num_layers - 1) * size2\n        if self._projection_size:\n            param_size += self._projection_size * nh * self._num_layers * self._dir\n        self.rnn_param.shape = (param_size, )\n\n    def _forward_kernel(self, inputs, states, sequence_length):\n        \"\"\" forward using CUDNN or CPU kenrel\"\"\"\n        device = inputs.device\n        if self._layout == 'NTC':\n            inputs = np.swapaxes(inputs, 0, 1)\n\n        if self._use_sequence_length:\n            rnn_args = states + [sequence_length]\n        else:\n            rnn_args = states\n\n        rnn_args_device = []\n        for args in rnn_args:\n            new_args = args.to_device(device)\n            rnn_args_device.append(new_args)\n\n        rnn = npx.rnn(inputs, self.rnn_param.data(device), *rnn_args_device,\n                      use_sequence_length=self._use_sequence_length,\n                      state_size=self._hidden_size, projection_size=self._projection_size,\n                      num_layers=self._num_layers, bidirectional=self._dir == 2,\n                      p=self._dropout, state_outputs=True, mode=self._mode,\n                      lstm_state_clip_min=self._lstm_state_clip_min,\n                      lstm_state_clip_max=self._lstm_state_clip_max,\n                      lstm_state_clip_nan=self._lstm_state_clip_nan)\n\n        if self._mode == 'lstm':\n            outputs, states = rnn[0], [rnn[1], rnn[2]]\n        else:\n            outputs, states = rnn[0], [rnn[1]]\n\n        if self._layout == 'NTC':\n            outputs = np.swapaxes(outputs, 0, 1)\n\n        return outputs, states\n\n\nclass RNN(_RNNLayer):\n    r\"\"\"Applies a multi-layer Elman RNN with `tanh` or `ReLU` non-linearity to an input sequence.\n\n    For each element in the input sequence, each layer computes the following\n    function:\n\n    .. math::\n        h_t = \\tanh(w_{ih} * x_t + b_{ih}  +  w_{hh} * h_{(t-1)} + b_{hh})\n\n    where :math:`h_t` is the hidden state at time `t`, and :math:`x_t` is the output\n    of the previous layer at time `t` or :math:`input_t` for the first layer.\n    If nonlinearity='relu', then `ReLU` is used instead of `tanh`.\n\n    Parameters\n    ----------\n    hidden_size: int\n        The number of features in the hidden state h.\n    num_layers: int, default 1\n        Number of recurrent layers.\n    activation: {'relu' or 'tanh'}, default 'relu'\n        The activation function to use.\n    layout : str, default 'TNC'\n        The format of input and output tensors. T, N and C stand for\n        sequence length, batch size, and feature dimensions respectively.\n    dropout: float, default 0\n        If non-zero, introduces a dropout layer on the outputs of each\n        RNN layer except the last layer.\n    bidirectional: bool, default False\n        If `True`, becomes a bidirectional RNN.\n    i2h_weight_initializer : str or Initializer\n        Initializer for the input weights matrix, used for the linear\n        transformation of the inputs.\n    h2h_weight_initializer : str or Initializer\n        Initializer for the recurrent weights matrix, used for the linear\n        transformation of the recurrent state.\n    i2h_bias_initializer : str or Initializer\n        Initializer for the bias vector.\n    h2h_bias_initializer : str or Initializer\n        Initializer for the bias vector.\n    input_size: int, default 0\n        The number of expected features in the input x.\n        If not specified, it will be inferred from input.\n    dtype : str, default 'float32'\n        Type to initialize the parameters and default states to\n\n\n    Inputs:\n        - **data**: input tensor with shape `(sequence_length, batch_size, input_size)`\n          when `layout` is \"TNC\". For other layouts, dimensions are permuted accordingly\n          using transpose() operator which adds performance overhead. Consider creating\n          batches in TNC layout during data batching step.\n\n        - **states**: initial recurrent state tensor with shape\n          `(num_layers, batch_size, num_hidden)`. If `bidirectional` is True,\n          shape will instead be `(2*num_layers, batch_size, num_hidden)`. If\n          `states` is None, zeros will be used as default begin states.\n\n    Outputs:\n        - **out**: output tensor with shape `(sequence_length, batch_size, num_hidden)`\n          when `layout` is \"TNC\". If `bidirectional` is True, output shape will instead\n          be `(sequence_length, batch_size, 2*num_hidden)`\n        - **out_states**: output recurrent state tensor with the same shape as `states`.\n          If `states` is None `out_states` will not be returned.\n\n\n    Examples\n    --------\n    >>> layer = mx.gluon.rnn.RNN(100, 3)\n    >>> layer.initialize()\n    >>> input = mx.np.random.uniform(size=(5, 3, 10))\n    >>> # by default zeros are used as begin state\n    >>> output = layer(input)\n    >>> # manually specify begin state.\n    >>> h0 = mx.np.random.uniform(size=(3, 3, 100))\n    >>> output, hn = layer(input, h0)\n    \"\"\"\n    def __init__(self, hidden_size, num_layers=1, activation='relu',\n                 layout='TNC', dropout=0, bidirectional=False,\n                 i2h_weight_initializer=None, h2h_weight_initializer=None,\n                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',\n                 input_size=0, dtype='float32', **kwargs):\n        super(RNN, self).__init__(hidden_size, num_layers, layout,\n                                  dropout, bidirectional, input_size,\n                                  i2h_weight_initializer, h2h_weight_initializer,\n                                  i2h_bias_initializer, h2h_bias_initializer,\n                                  'rnn_'+activation, None, None, None, None, False,\n                                  dtype, **kwargs)\n\n    def state_info(self, batch_size=0):\n        return [{'shape': (self._num_layers * self._dir, batch_size, self._hidden_size),\n                 '__layout__': 'LNC', 'dtype': self._dtype}]\n\n\nclass LSTM(_RNNLayer):\n    r\"\"\"Applies a multi-layer long short-term memory (LSTM) RNN to an input sequence.\n\n    For each element in the input sequence, each layer computes the following\n    function:\n\n    .. math::\n        \\begin{array}{ll}\n        i_t = sigmoid(W_{ii} x_t + b_{ii} + W_{hi} h_{(t-1)} + b_{hi}) \\\\\n        f_t = sigmoid(W_{if} x_t + b_{if} + W_{hf} h_{(t-1)} + b_{hf}) \\\\\n        g_t = \\tanh(W_{ig} x_t + b_{ig} + W_{hc} h_{(t-1)} + b_{hg}) \\\\\n        o_t = sigmoid(W_{io} x_t + b_{io} + W_{ho} h_{(t-1)} + b_{ho}) \\\\\n        c_t = f_t * c_{(t-1)} + i_t * g_t \\\\\n        h_t = o_t * \\tanh(c_t)\n        \\end{array}\n\n    where :math:`h_t` is the hidden state at time `t`, :math:`c_t` is the\n    cell state at time `t`, :math:`x_t` is the hidden state of the previous\n    layer at time `t` or :math:`input_t` for the first layer, and :math:`i_t`,\n    :math:`f_t`, :math:`g_t`, :math:`o_t` are the input, forget, cell, and\n    out gates, respectively.\n\n    Parameters\n    ----------\n    hidden_size: int\n        The number of features in the hidden state h.\n    num_layers: int, default 1\n        Number of recurrent layers.\n    layout : str, default 'TNC'\n        The format of input and output tensors. T, N and C stand for\n        sequence length, batch size, and feature dimensions respectively.\n    dropout: float, default 0\n        If non-zero, introduces a dropout layer on the outputs of each\n        RNN layer except the last layer.\n    bidirectional: bool, default False\n        If `True`, becomes a bidirectional RNN.\n    i2h_weight_initializer : str or Initializer\n        Initializer for the input weights matrix, used for the linear\n        transformation of the inputs.\n    h2h_weight_initializer : str or Initializer\n        Initializer for the recurrent weights matrix, used for the linear\n        transformation of the recurrent state.\n    i2h_bias_initializer : str or Initializer, default 'lstmbias'\n        Initializer for the bias vector. By default, bias for the forget\n        gate is initialized to 1 while all other biases are initialized\n        to zero.\n    h2h_bias_initializer : str or Initializer\n        Initializer for the bias vector.\n    projection_size: int, default None\n        The number of features after projection.\n    h2r_weight_initializer : str or Initializer, default None\n        Initializer for the projected recurrent weights matrix, used for the linear\n        transformation of the recurrent state to the projected space.\n    state_clip_min : float or None, default None\n        Minimum clip value of LSTM states. This option must be used together with\n        state_clip_max. If None, clipping is not applied.\n    state_clip_max : float or None, default None\n        Maximum clip value of LSTM states. This option must be used together with\n        state_clip_min. If None, clipping is not applied.\n    state_clip_nan : boolean, default False\n        Whether to stop NaN from propagating in state by clipping it to min/max.\n        If the clipping range is not specified, this option is ignored.\n    dtype : str, default 'float32'\n        Type to initialize the parameters and default states to\n    input_size: int, default 0\n        The number of expected features in the input x.\n        If not specified, it will be inferred from input.\n\n\n    Inputs:\n        - **data**: input tensor with shape `(sequence_length, batch_size, input_size)`\n          when `layout` is \"TNC\". For other layouts, dimensions are permuted accordingly\n          using transpose() operator which adds performance overhead. Consider creating\n          batches in TNC layout during data batching step.\n        - **states**: a list of two initial recurrent state tensors. Each has shape\n          `(num_layers, batch_size, num_hidden)`. If `bidirectional` is True,\n          shape will instead be `(2*num_layers, batch_size, num_hidden)`. If\n          `states` is None, zeros will be used as default begin states.\n\n    Outputs:\n        - **out**: output tensor with shape `(sequence_length, batch_size, num_hidden)`\n          when `layout` is \"TNC\". If `bidirectional` is True, output shape will instead\n          be `(sequence_length, batch_size, 2*num_hidden)`\n        - **out_states**: a list of two output recurrent state tensors with the same\n          shape as in `states`. If `states` is None `out_states` will not be returned.\n\n\n    Examples\n    --------\n    >>> layer = mx.gluon.rnn.LSTM(100, 3)\n    >>> layer.initialize()\n    >>> input = mx.np.random.uniform(size=(5, 3, 10))\n    >>> # by default zeros are used as begin state\n    >>> output = layer(input)\n    >>> # manually specify begin state.\n    >>> h0 = mx.np.random.uniform(size=(3, 3, 100))\n    >>> c0 = mx.np.random.uniform(size=(3, 3, 100))\n    >>> output, hn = layer(input, [h0, c0])\n    \"\"\"\n    def __init__(self, hidden_size, num_layers=1, layout='TNC',\n                 dropout=0, bidirectional=False, input_size=0,\n                 i2h_weight_initializer=None, h2h_weight_initializer=None,\n                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',\n                 projection_size=None, h2r_weight_initializer=None,\n                 state_clip_min=None, state_clip_max=None, state_clip_nan=False,\n                 dtype='float32', **kwargs):\n        super(LSTM, self).__init__(hidden_size, num_layers, layout,\n                                   dropout, bidirectional, input_size,\n                                   i2h_weight_initializer, h2h_weight_initializer,\n                                   i2h_bias_initializer, h2h_bias_initializer,\n                                   'lstm', projection_size, h2r_weight_initializer,\n                                   state_clip_min, state_clip_max, state_clip_nan,\n                                   dtype, **kwargs)\n\n    def state_info(self, batch_size=0):\n        if self._projection_size is None:\n            return [{'shape': (self._num_layers * self._dir, batch_size, self._hidden_size),\n                     '__layout__': 'LNC', 'dtype': self._dtype},\n                    {'shape': (self._num_layers * self._dir, batch_size, self._hidden_size),\n                     '__layout__': 'LNC', 'dtype': self._dtype}]\n        else:\n            return [{'shape': (self._num_layers * self._dir, batch_size, self._projection_size),\n                     '__layout__': 'LNC', 'dtype': self._dtype},\n                    {'shape': (self._num_layers * self._dir, batch_size, self._hidden_size),\n                     '__layout__': 'LNC', 'dtype': self._dtype}]\n\n\nclass GRU(_RNNLayer):\n    r\"\"\"Applies a multi-layer gated recurrent unit (GRU) RNN to an input sequence.\n    Note: this is an implementation of the cuDNN version of GRUs\n    (slight modification compared to Cho et al. 2014; the reset gate :math:`r_t`\n    is applied after matrix multiplication).\n\n    For each element in the input sequence, each layer computes the following\n    function:\n\n    .. math::\n        \\begin{array}{ll}\n        r_t = sigmoid(W_{ir} x_t + b_{ir} + W_{hr} h_{(t-1)} + b_{hr}) \\\\\n        i_t = sigmoid(W_{ii} x_t + b_{ii} + W_{hi} h_{(t-1)} + b_{hi}) \\\\\n        n_t = \\tanh(W_{in} x_t + b_{in} + r_t * (W_{hn} h_{(t-1)} + b_{hn})) \\\\\n        h_t = (1 - i_t) * n_t + i_t * h_{(t-1)} \\\\\n        \\end{array}\n\n    where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the hidden\n    state of the previous layer at time `t` or :math:`input_t` for the first layer,\n    and :math:`r_t`, :math:`i_t`, :math:`n_t` are the reset, input, and new gates, respectively.\n\n    Parameters\n    ----------\n    hidden_size: int\n        The number of features in the hidden state h\n    num_layers: int, default 1\n        Number of recurrent layers.\n    layout : str, default 'TNC'\n        The format of input and output tensors. T, N and C stand for\n        sequence length, batch size, and feature dimensions respectively.\n    dropout: float, default 0\n        If non-zero, introduces a dropout layer on the outputs of each\n        RNN layer except the last layer\n    bidirectional: bool, default False\n        If True, becomes a bidirectional RNN.\n    i2h_weight_initializer : str or Initializer\n        Initializer for the input weights matrix, used for the linear\n        transformation of the inputs.\n    h2h_weight_initializer : str or Initializer\n        Initializer for the recurrent weights matrix, used for the linear\n        transformation of the recurrent state.\n    i2h_bias_initializer : str or Initializer\n        Initializer for the bias vector.\n    h2h_bias_initializer : str or Initializer\n        Initializer for the bias vector.\n    dtype : str, default 'float32'\n        Type to initialize the parameters and default states to\n    input_size: int, default 0\n        The number of expected features in the input x.\n        If not specified, it will be inferred from input.\n\n\n    Inputs:\n        - **data**: input tensor with shape `(sequence_length, batch_size, input_size)`\n          when `layout` is \"TNC\". For other layouts, dimensions are permuted accordingly\n          using transpose() operator which adds performance overhead. Consider creating\n          batches in TNC layout during data batching step.\n        - **states**: initial recurrent state tensor with shape\n          `(num_layers, batch_size, num_hidden)`. If `bidirectional` is True,\n          shape will instead be `(2*num_layers, batch_size, num_hidden)`. If\n          `states` is None, zeros will be used as default begin states.\n\n    Outputs:\n        - **out**: output tensor with shape `(sequence_length, batch_size, num_hidden)`\n          when `layout` is \"TNC\". If `bidirectional` is True, output shape will instead\n          be `(sequence_length, batch_size, 2*num_hidden)`\n        - **out_states**: output recurrent state tensor with the same shape as `states`.\n          If `states` is None `out_states` will not be returned.\n\n\n    Examples\n    --------\n    >>> layer = mx.gluon.rnn.GRU(100, 3)\n    >>> layer.initialize()\n    >>> input = mx.np.random.uniform(size=(5, 3, 10))\n    >>> # by default zeros are used as begin state\n    >>> output = layer(input)\n    >>> # manually specify begin state.\n    >>> h0 = mx.np.random.uniform(size=(3, 3, 100))\n    >>> output, hn = layer(input, h0)\n    \"\"\"\n    def __init__(self, hidden_size, num_layers=1, layout='TNC',\n                 dropout=0, bidirectional=False, input_size=0,\n                 i2h_weight_initializer=None, h2h_weight_initializer=None,\n                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',\n                 dtype='float32', **kwargs):\n        super(GRU, self).__init__(hidden_size, num_layers, layout,\n                                  dropout, bidirectional, input_size,\n                                  i2h_weight_initializer, h2h_weight_initializer,\n                                  i2h_bias_initializer, h2h_bias_initializer,\n                                  'gru', None, None, None, None, False,\n                                  dtype, **kwargs)\n\n    def state_info(self, batch_size=0):\n        return [{'shape': (self._num_layers * self._dir, batch_size, self._hidden_size),\n                 '__layout__': 'LNC', 'dtype': self._dtype}]\n"
  },
  {
    "path": "python/mxnet/gluon/trainer.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=line-too-long\n\"\"\"Parameter optimizer.\"\"\"\n__all__ = ['Trainer']\n\nimport warnings\nfrom collections import OrderedDict\n\nfrom .. import optimizer as opt\nfrom ..model import _create_kvstore, _create_sparse_kvstore\nfrom .parameter import Parameter\nfrom ..kvstore import KVStore\n\n\nclass Trainer(object):\n    \"\"\"Applies an `Optimizer` on a set of Parameters. Trainer should\n    be used together with `autograd`.\n\n    .. note::\n\n        For the following cases, updates will always happen on kvstore,\n        i.e., you cannot set update_on_kvstore=False.\n\n        - dist kvstore with sparse weights or sparse gradients\n        - dist async kvstore\n        - `optimizer.lr_scheduler` is not None\n\n    Parameters\n    ----------\n    params : Dict\n        The set of parameters to optimize.\n    optimizer : str or Optimizer\n        The optimizer to use. See\n        `help <https://mxnet.apache.org/api/python/docs/api/optimizer/index.html>`_\n        on Optimizer for a list of available optimizers.\n    optimizer_params : dict\n        Key-word arguments to be passed to optimizer constructor. For example,\n        `{'learning_rate': 0.1}`. All optimizers accept learning_rate, wd (weight decay),\n        clip_gradient, and lr_scheduler. See each optimizer's\n        constructor for a list of additional supported arguments.\n    kvstore : str or KVStore\n        kvstore type for multi-gpu and distributed training. See help on\n        :func:`mxnet.kvstore.create` for more information.\n    compression_params : dict\n        Specifies type of gradient compression and additional arguments depending\n        on the type of compression being used. For example, 2bit compression requires a threshold.\n        Arguments would then be {'type':'2bit', 'threshold':0.5}\n        See mxnet.KVStore.set_gradient_compression method for more details on gradient compression.\n    update_on_kvstore : bool, default None\n        Whether to perform parameter updates on kvstore. If None and optimizer.aggregate_num <= 1,\n        then trainer will choose the more suitable option depending on the type of kvstore.\n        If None and optimizer.aggregate_num > 1, `update_on_kvstore` is set to False.\n        If the `update_on_kvstore` argument is provided,\n        environment variable `MXNET_UPDATE_ON_KVSTORE` will be ignored.\n\n    Properties\n    ----------\n    learning_rate : float\n        The current learning rate of the optimizer. Given an Optimizer object\n        optimizer, its learning rate can be accessed as optimizer.learning_rate.\n    \"\"\"\n    def __init__(self, params, optimizer, optimizer_params=None, kvstore='device',\n                 compression_params=None, update_on_kvstore=None):\n        param_list = []\n        if isinstance(params, (dict, OrderedDict)):\n            for key in sorted(list(params.keys())):\n                param_list.append(params[key])\n            params = param_list\n        if not isinstance(params, (list, tuple)):\n            raise ValueError(\n                \"First argument must be a list or dict of Parameters, \" \\\n                f\"got {type(params)}.\")\n        self._params = []\n        # parameters to initialize on the kvstore\n        self._contains_sparse_weight = False\n        self._contains_sparse_grad = False\n        self._param2idx = {}\n        for i, param in enumerate(params):\n            if not isinstance(param, Parameter):\n                raise ValueError(\n                    \"First argument must be a list or dict of Parameters, \" \\\n                    f\"got list of {type(param)}.\")\n            if param._uuid in self._param2idx:\n                # Shared parameters have same uuid; only need to store one of the shared versions\n                continue\n            self._param2idx[param._uuid] = i\n            self._params.append(param)\n            param._set_trainer(self)\n            if param._stype != 'default':\n                self._contains_sparse_weight = True\n            if param._grad_stype != 'default':\n                self._contains_sparse_grad = True\n        self._compression_params = compression_params\n        self._devices = self._check_devices()\n        optimizer_params = optimizer_params if optimizer_params else {}\n        self._init_optimizer(optimizer, optimizer_params)\n        self._scale = self._optimizer.rescale_grad\n        if self._optimizer.aggregate_num > 1 and update_on_kvstore is not None:\n            if update_on_kvstore:\n                raise ValueError(\"Cannot set update_on_kvstore=True \"\n                                 \"when optimizer.aggregate_num > 1.\")\n        if update_on_kvstore is None and self._optimizer.aggregate_num > 1:\n            update_on_kvstore = False\n        self._kvstore_params = {'kvstore': kvstore, 'update_on_kvstore': update_on_kvstore}\n        self._kv_initialized = False\n        self._kvstore = None\n        self._update_on_kvstore = None\n        self._distributed = None\n        self._params_to_init = []\n        self._reset_kvstore()\n\n    def _check_contexts(self):\n        \"\"\"This function has been deprecated. Please refer to ``Trainer._check_devices``.\"\"\"\n        warnings.warn('Trainer._check_contexts has been renamed to'\n                      ' Trainer._check_devices', DeprecationWarning)\n        return self._check_devices()\n\n    def _check_devices(self):\n        devices = None\n        for param in self._params:\n            device = param.list_device()\n            assert devices is None or devices == device, \\\n                \"All Parameters must be initialized on the same set of devices, \" \\\n                f\"but Parameter {param.name} is initialized on {str(device)} while previous Parameters \" \\\n                f\"are initialized on {str(devices)}.\"\n            devices = device\n        return devices\n\n    def _init_optimizer(self, optimizer, optimizer_params):\n        param_dict = {i: param for i, param in enumerate(self._params)}\n        if isinstance(optimizer, opt.Optimizer):\n            assert not optimizer_params, \\\n                \"optimizer_params must be None if optimizer is an instance of \" \\\n                \"Optimizer instead of str\"\n            self._optimizer = optimizer\n            # param_dict must not be deep copied, so that if user mutate the lr_mult\n            # or wd_mult of some parameters, it takes effect.\n            self._optimizer.param_dict = param_dict\n        else:\n            self._optimizer = opt.create(optimizer, param_dict=param_dict,\n                                         **optimizer_params)\n        self._updaters = [opt.get_updater(self._optimizer) \\\n                            for _ in self._devices]\n\n    def _init_params(self):\n        \"\"\"Initialize parameters in the KVStore.\n\n        Parameters with incomplete initialization are ignored.\n\n        \"\"\"\n        assert self._kv_initialized, \"Cannot initialize parameters in KVStore \" \\\n                                     \"when KVStore is not initialized.\"\n        params_to_init = []\n        if self._kvstore:\n            for param in self._params_to_init:\n                if param._deferred_init:\n                    params_to_init.append(param)\n                else:\n                    param_arrays = param._check_and_get(param._data, list)\n                    idx = self._param2idx[param._uuid]\n                    if param._stype != 'default':\n                        self._kvstore.init(idx, param_arrays[0])\n                    else:\n                        self._kvstore.broadcast(idx, param_arrays[0], param_arrays)\n\n        self._params_to_init = params_to_init\n\n    def _reset_kvstore(self):\n        \"\"\"Reset kvstore.\"\"\"\n        if self._kvstore and 'dist' in self._kvstore.type:\n            raise RuntimeError(\"Cannot reset distributed KVStore.\")\n        self._kv_initialized = False\n        self._kvstore = None\n        self._distributed = None\n        self._update_on_kvstore = None\n        self._params_to_init = [param for param in self._params]\n\n    def _init_kvstore(self):\n        \"\"\"Create kvstore.\"\"\"\n        config = self._kvstore_params\n        # configure kvstore, update_on_kvstore and self._distributed on three cases:\n        if self._contains_sparse_weight:\n            # If weight is sparse, kvstore must be present and the weight must be updated on kvstore.\n            # The training loop is the following:\n            #    - row_sparse_pull(sparse_weight)\n            #    - forward()\n            #    - backward()\n            #    - push_and_update(grad)\n            #    - pull(weight)\n            kvstore, update_on_kvstore = _create_sparse_kvstore(config['kvstore'])\n            self._distributed = 'dist' in kvstore.type\n            # raise err if user provides unsupported configs\n            if config['update_on_kvstore'] is False:\n                raise ValueError(\"Cannot set update_on_kvstore=False when sparse weights \"\n                                 \"are present.\")\n\n        elif self._contains_sparse_grad:\n            # For single node training with dense weight and sparse grad,\n            # we prefer update_on_kvstore=False because this is usually faster.\n            # This means we push and pull sparse gradients, and we do not store weight in kvstore.\n            # The training loop is the following:\n            #    - forward()\n            #    - backward()\n            #    - push(grad)\n            #    - pull(grad)\n            #    - update(grad, weight)\n            #\n            # For multi-node training with dense weight and sparse grad,\n            # only update_on_kvstore=True is supported, due to the fact that\n            # kv.row_sparse_pull(grad) is not implemented.\n            # Therefore, we push sparse gradients and pull dense weights.\n            # The training loop contains:\n            #    - forward()\n            #    - backward()\n            #    - push_and_update(grad)\n            #    - pull(weight)\n            arg_arrays = {param._uuid: param.data(self._devices[0]) for param in self._params}\n            kvstore, _ = _create_kvstore(config['kvstore'], len(self._devices), arg_arrays)\n            self._distributed = 'dist' in kvstore.type if kvstore else False\n            update_on_kvstore = self._distributed\n            # raise err if user provides unsupported configs\n            if config['update_on_kvstore'] is not None:\n                if config['update_on_kvstore'] is False and self._distributed:\n                    raise ValueError(\"Cannot set update_on_kvstore=False on dist kvstore \"\n                                     \"when sparse gradients are present.\")\n                update_on_kvstore = config['update_on_kvstore']\n            # raise err if a custom kvstore is used for sparse training\n            if kvstore is not None and not isinstance(kvstore, KVStore):\n                raise ValueError(\"Cannot use {} for multi-device training with sparse gradients\"\n                                 .format(type(kvstore)))\n\n        else:\n            # Training with dense weight and dense gradients.\n            # The only unsupported mode is async with update_on_kvstore=False\n            arg_arrays = {param._uuid: param.data(self._devices[0]) for param in self._params}\n            kvstore, update_on_kvstore = _create_kvstore(config['kvstore'], len(self._devices),\n                                                         arg_arrays)\n            self._distributed = 'dist' in kvstore.type if kvstore else False\n            if self._distributed and 'async' in kvstore.type:\n                update_on_kvstore = True\n                # raise err if user provides unsupported configs\n                if config['update_on_kvstore'] is False:\n                    raise ValueError(\"Please set update_on_kvstore=True \"\n                                     \"when training in async mode.\")\n            if config['update_on_kvstore'] is not None:\n                update_on_kvstore = config['update_on_kvstore']\n            # raise err if update_on_kvstore is set to True with kvstores that do not support optimizers\n            if update_on_kvstore and not kvstore.is_capable('optimizer'):\n                if config['update_on_kvstore']:\n                    raise ValueError(\"Please set update_on_kvstore=False \"\n                                     \"when training with {}\".format(type(kvstore)))\n                update_on_kvstore = False\n\n        # set grad compression and optimizers\n        if kvstore:\n            if self._compression_params:\n                kvstore.set_gradient_compression(self._compression_params)\n            if update_on_kvstore:\n                # optimizer preferably needs to be set before init for multiprecision\n                kvstore.set_optimizer(self._optimizer)\n            self._kvstore = kvstore\n            self._update_on_kvstore = update_on_kvstore\n        else:\n            self._kvstore = None\n            self._update_on_kvstore = None\n\n        self._kv_initialized = True\n\n    @property\n    def learning_rate(self):\n        if not isinstance(self._optimizer, opt.Optimizer):\n            raise UserWarning(\"Optimizer has to be defined before its learning \"\n                              \"rate can be accessed.\")\n\n        return self._optimizer.learning_rate\n\n    @property\n    def optimizer(self):\n        if isinstance(self._optimizer, opt.Optimizer):\n            return self._optimizer\n        else:\n            raise UserWarning(\"Optimizer has not been initialized yet\")\n\n    def set_learning_rate(self, lr):\n        \"\"\"Sets a new learning rate of the optimizer.\n\n        Parameters\n        ----------\n        lr : float\n            The new learning rate of the optimizer.\n        \"\"\"\n        if not isinstance(self._optimizer, opt.Optimizer):\n            raise UserWarning(\"Optimizer has to be defined before its learning \"\n                              \"rate is mutated.\")\n\n        self._optimizer.set_learning_rate(lr)\n\n    def _row_sparse_pull(self, parameter, out, row_id, full_idx=False):\n        \"\"\"Internal method to invoke pull operations on KVStore. If `full_idx` is set to True,\n        `kv.pull` is preferred instead of `kv.row_sparse_pull`.\n        \"\"\"\n        # initialize kv and params if not already\n        if not self._kv_initialized:\n            self._init_kvstore()\n        if self._params_to_init:\n            self._init_params()\n        idx = self._param2idx[parameter._uuid]\n        if full_idx and 'dist' not in self._kvstore.type:\n            assert row_id.size == out.shape[0]\n            self._kvstore.pull(idx, out=out, priority=-idx, ignore_sparse=False)\n        else:\n            self._kvstore.row_sparse_pull(idx, out=out, row_ids=row_id, priority=-idx)\n\n    def _check_and_rescale_grad(self, scale):\n        if self._update_on_kvstore and self._distributed and self._kv_initialized:\n            if self._optimizer.rescale_grad != scale:\n                raise UserWarning('Possible change in the `batch_size` from previous '\n                                  '`step` detected. Optimizer gradient normalizing '\n                                  'factor will not change w.r.t new batch_size when '\n                                  'update_on_kvstore=True and when distributed kvstore '\n                                  'is used.')\n        self._optimizer.rescale_grad = scale\n\n    def step(self, batch_size, ignore_stale_grad=False):\n        \"\"\"Makes one step of parameter update. Should be called after\n        `autograd.backward()` and outside of `record()` scope.\n\n        For normal parameter updates, `step()` should be used, which internally calls\n        `allreduce_grads()` and then `update()`. However, if you need to get the reduced\n        gradients to perform certain transformation, such as in gradient clipping, then\n        you may want to manually call `allreduce_grads()` and `update()` separately.\n\n        Parameters\n        ----------\n        batch_size : int\n            Batch size of data processed. Gradient will be normalized by `1/batch_size`.\n            Set this to 1 if you normalized loss manually with `loss = mean(loss)`.\n        ignore_stale_grad : bool, optional, default=False\n            If true, ignores Parameters with stale gradient (gradient that has not\n            been updated by `backward` after last step) and skip update.\n        \"\"\"\n        rescale_grad = self._scale / batch_size\n        self._check_and_rescale_grad(rescale_grad)\n\n        if not self._kv_initialized:\n            self._init_kvstore()\n        if self._params_to_init:\n            self._init_params()\n\n        self._allreduce_grads()\n        self._update(ignore_stale_grad)\n\n    def allreduce_grads(self):\n        \"\"\"For each parameter, reduce the gradients from different devices.\n\n        Should be called after `autograd.backward()`, outside of `record()` scope,\n        and before `trainer.update()`.\n\n        For normal parameter updates, `step()` should be used, which internally calls\n        `allreduce_grads()` and then `update()`. However, if you need to get the reduced\n        gradients to perform certain transformation, such as in gradient clipping, then\n        you may want to manually call `allreduce_grads()` and `update()` separately.\n        \"\"\"\n        if not self._kv_initialized:\n            self._init_kvstore()\n        if self._params_to_init:\n            self._init_params()\n        assert not (self._kvstore and self._update_on_kvstore), \\\n                'allreduce_grads() when parameters are updated on kvstore ' \\\n                'is not supported. Try setting `update_on_kvstore` ' \\\n                'to False when creating trainer.'\n\n        self._allreduce_grads()\n\n    def _allreduce_grads(self):\n        # nothing to reduce\n        if not self._kvstore:\n            return\n        for i, param in enumerate(self._params):\n            if param.grad_req != 'null':\n                idx = self._param2idx[param._uuid]\n                grad_list = param.list_grad()\n                # sparse gradients, call push and pull separately\n                if grad_list[0].stype != 'default':\n                    self._kvstore.push(idx, grad_list, priority=-i)\n                    if param._stype == 'default':\n                        if self._update_on_kvstore:\n                            pull_list = param.list_data()\n                        else:\n                            pull_list = param.list_grad()\n                        self._kvstore.pull(idx, pull_list, priority=-i,\n                                           ignore_sparse=self._distributed)\n                else:\n                    # allreduce dense gradients if not update_on_kvstore,\n                    # otherwise push dense gradients, pull dense weights\n                    if self._update_on_kvstore:\n                        self._kvstore.pushpull(idx, grad_list, out=param.list_data(), priority=-i)\n                    else:\n                        self._kvstore.pushpull(idx, grad_list, priority=-i)\n\n    def update(self, batch_size, ignore_stale_grad=False):\n        \"\"\"Makes one step of parameter update.\n\n        Should be called after `autograd.backward()` and outside of `record()` scope,\n        and after `trainer.update()`.\n\n\n        For normal parameter updates, `step()` should be used, which internally calls\n        `allreduce_grads()` and then `update()`. However, if you need to get the reduced\n        gradients to perform certain transformation, such as in gradient clipping, then\n        you may want to manually call `allreduce_grads()` and `update()` separately.\n\n        Parameters\n        ----------\n        batch_size : int\n            Batch size of data processed. Gradient will be normalized by `1/batch_size`.\n            Set this to 1 if you normalized loss manually with `loss = mean(loss)`.\n        ignore_stale_grad : bool, optional, default=False\n            If true, ignores Parameters with stale gradient (gradient that has not\n            been updated by `backward` after last step) and skip update.\n        \"\"\"\n        if not self._kv_initialized:\n            self._init_kvstore()\n        if self._params_to_init:\n            self._init_params()\n        assert not (self._kvstore and self._update_on_kvstore), \\\n                'update() when parameters are updated on kvstore ' \\\n                'is not supported. Try setting `update_on_kvstore` ' \\\n                'to False when creating trainer.'\n\n        self._check_and_rescale_grad(self._scale / batch_size)\n        self._update(ignore_stale_grad)\n\n    def _update(self, ignore_stale_grad=False):\n        loss_scaler = getattr(self, '_amp_loss_scaler', None)\n        if loss_scaler is not None:\n            if loss_scaler.has_overflow(self._params):\n                return  # skip on overflow\n\n        updates = [[] for _ in self._updaters]\n\n        for i, param in enumerate(self._params):\n            if param.grad_req == 'null':\n                continue\n\n            if not ignore_stale_grad:\n                for data in param._check_and_get(param._data, list):\n                    if not data._fresh_grad:\n                        raise UserWarning(\n                            f\"Gradient of Parameter `{param.name}` on device {str(data.device)} has not been updated \"\n                            \"by backward since last `step`. This could mean a bug in your \"\n                            \"model that made it only use a subset of the Parameters (Blocks) \"\n                            \"for this iteration. If you are intentionally only using a subset, \"\n                            \"call step with ignore_stale_grad=True to suppress this \"\n                            \"warning and skip updating of Parameters with stale gradient\")\n\n            if self._kvstore and self._update_on_kvstore:\n                continue\n\n            for upd, arr, grad in zip(updates, param.list_data(), param.list_grad()):\n                if not ignore_stale_grad or arr._fresh_grad:\n                    upd.append((i, grad, arr))\n                    arr._fresh_grad = False\n\n        if not (self._kvstore and self._update_on_kvstore):\n            for updater, upd in zip(self._updaters, updates):\n                if upd:\n                    i, g, w = zip(*upd)\n                    updater(i, g, w)\n\n    def save_states(self, fname):\n        \"\"\"Saves trainer states (e.g. optimizer, momentum) to a file.\n\n\n        Parameters\n        ----------\n        fname : str\n            Path to output states file.\n\n        Note\n        ----\n        `optimizer.param_dict`, which contains Parameter information (such as\n        `lr_mult` and `wd_mult`) will not be saved.\n        \"\"\"\n        assert self._optimizer is not None\n\n        if not self._kv_initialized:\n            self._init_kvstore()\n        if self._params_to_init:\n            self._init_params()\n\n        if self._update_on_kvstore:\n            assert not self._params_to_init, \"Cannot save trainer states when some \" \\\n                                             \"parameters are not yet initialized in kvstore.\"\n            self._kvstore.save_optimizer_states(fname, dump_optimizer=True)\n        else:\n            with open(fname, 'wb') as fout:\n                fout.write(self._updaters[0].get_states(dump_optimizer=True))\n\n    def load_states(self, fname):\n        \"\"\"Loads trainer states (e.g. optimizer, momentum) from a file.\n\n        Parameters\n        ----------\n        fname : str\n            Path to input states file.\n\n        Note\n        ----\n        `optimizer.param_dict`, which contains Parameter information (such as\n        `lr_mult` and `wd_mult`) will not be loaded from the file, but rather set\n        based on current Trainer's parameters.\n        \"\"\"\n        if not self._kv_initialized:\n            self._init_kvstore()\n        if self._params_to_init:\n            self._init_params()\n\n        if self._update_on_kvstore:\n            self._kvstore.load_optimizer_states(fname)\n            self._optimizer = self._kvstore._updater.optimizer\n        else:\n            with open(fname, 'rb') as f:\n                states = f.read()\n            for updater in self._updaters:\n                updater.set_states(states)\n                updater.optimizer = self._updaters[0].optimizer\n            self._optimizer = self._updaters[0].optimizer\n        param_dict = {i: param for i, param in enumerate(self._params)}\n        self._optimizer.param_dict = param_dict\n"
  },
  {
    "path": "python/mxnet/gluon/utils.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=\n\"\"\"Parallelization utility optimizer.\"\"\"\n\n__all__ = ['split_data', 'split_and_load', 'clip_global_norm',\n           'check_sha1', 'download', 'replace_file']\n\nimport os\nimport sys\nimport hashlib\nimport uuid\nimport warnings\nimport collections\nimport weakref\nimport requests\n\nimport numpy as np\n\nfrom .. import ndarray\nfrom ..util import is_np_shape, is_np_array, TemporaryDirectory\nfrom .. import numpy as _mx_np  # pylint: disable=reimported\n\n\ndef split_data(data, num_slice, batch_axis=0, even_split=True):\n    \"\"\"Splits an NDArray into `num_slice` slices along `batch_axis`.\n    Usually used for data parallelism where each slices is sent\n    to one device (i.e. GPU).\n\n    Parameters\n    ----------\n    data : NDArray\n        A batch of data.\n    num_slice : int\n        Number of desired slices.\n    batch_axis : int, default 0\n        The axis along which to slice.\n    even_split : bool, default True\n        Whether to force all slices to have the same number of elements.\n        If `True`, an error will be raised when `num_slice` does not evenly\n        divide `data.shape[batch_axis]`.\n\n    Returns\n    -------\n    list of NDArray\n        Return value is a list even if `num_slice` is 1.\n    \"\"\"\n    size = data.shape[batch_axis]\n    if even_split and size % num_slice != 0:\n        raise ValueError(\n            f\"data with shape {str(data.shape)} cannot be evenly split into {num_slice} slices \" \\\n            f\"along axis {batch_axis}. Use a batch size that's multiple of {num_slice} \" \\\n            f\"or set even_split=False to allow uneven partitioning of data.\")\n\n    n_each_section, extras = divmod(size, num_slice)\n    section_sizes = [0] + (extras * [n_each_section + 1] +\n                           (num_slice - extras) * [n_each_section])\n    div_points = np.array(section_sizes).cumsum()\n    if is_np_array():\n        slices = _mx_np.split(data, indices_or_sections=list(div_points[1: -1]), axis=batch_axis)\n    else:\n        slices = []\n        for i in range(num_slice):\n            st = div_points[i]\n            end = div_points[i + 1]\n            slices.append(ndarray.slice_axis(data, axis=batch_axis, begin=st, end=end))\n    return slices\n\n\ndef split_and_load(data, ctx_list, batch_axis=0, even_split=True):\n    \"\"\"Splits an NDArray into `len(ctx_list)` slices along `batch_axis` and loads\n    each slice to one context in `ctx_list`.\n\n    Parameters\n    ----------\n    data : NDArray or ndarray\n        A batch of data.\n    ctx_list : list of Context\n        A list of Contexts.\n    batch_axis : int, default 0\n        The axis along which to slice.\n    even_split : bool, default True\n        Whether to force all slices to have the same number of elements.\n\n    Returns\n    -------\n    list of NDArrays or ndarrays\n        Each corresponds to a context in `ctx_list`.\n    \"\"\"\n    array_fn = _mx_np.array if is_np_array() else ndarray.array\n    if not isinstance(data, ndarray.NDArray):\n        data = array_fn(data, ctx=ctx_list[0])\n    if len(ctx_list) == 1:\n        return [data.as_in_context(ctx_list[0])]\n\n    slices = split_data(data, len(ctx_list), batch_axis, even_split)\n    return [i.as_in_context(ctx) for i, ctx in zip(slices, ctx_list)]\n\n\ndef clip_global_norm(arrays, max_norm, check_isfinite=True):\n    \"\"\"Rescales NDArrays so that the sum of their 2-norm is smaller than `max_norm`.\n\n    Parameters\n    ----------\n    arrays : list of NDArray\n    max_norm : float\n    check_isfinite : bool, default True\n         If True, check that the total_norm is finite (not nan or inf). This\n         requires a blocking .asscalar() call.\n\n    Returns\n    -------\n    NDArray or float\n      Total norm. Return type is NDArray of shape (1,) if check_isfinite is\n      False. Otherwise a float is returned.\n\n    \"\"\"\n    # group arrays by ctx\n    def group_by_ctx(arr_list):\n        groups = collections.defaultdict(list)\n        for arr in arr_list:\n            ctx = arr.device\n            groups[ctx].append(arr)\n        return groups\n    def multi_sum_sq(*args, ctx=None):\n        sum = _mx_np.array([0], device=ctx)\n        for arg in args:\n            sum += _mx_np.square(arg).sum().item()\n        return sum\n    arrays_groups = group_by_ctx(arrays)\n    all_ctx_sum = _mx_np.array([0])\n    ctx = arrays[0].device\n    for group in arrays_groups:\n        sum_sq = multi_sum_sq(*arrays_groups[group], ctx=ctx)\n        all_ctx_sum += sum_sq\n    # global reduce\n    total_norm = _mx_np.sqrt(all_ctx_sum)\n    if check_isfinite:\n        if not np.isfinite(total_norm.item()):\n            warnings.warn(\n                UserWarning('nan or inf is detected. '\n                            'Clipping results will be undefined.'), stacklevel=2)\n    scale = max_norm / (total_norm + 1e-8)\n    scale = _mx_np.min(_mx_np.concatenate([scale, _mx_np.ones(1, device=ctx)], axis=0))\n    for arr in arrays:\n        arr *= scale.item()\n    if check_isfinite:\n        return total_norm.item()\n    else:\n        return total_norm\n\n\ndef _indent(s_, numSpaces):\n    \"\"\"Indent string\n    \"\"\"\n    s = s_.split('\\n')\n    if len(s) == 1:\n        return s_\n    first = s.pop(0)\n    s = [first] + [(numSpaces * ' ') + line for line in s]\n    s = '\\n'.join(s)\n    return s\n\n\ndef check_sha1(filename, sha1_hash):\n    \"\"\"Check whether the sha1 hash of the file content matches the expected hash.\n\n    Parameters\n    ----------\n    filename : str\n        Path to the file.\n    sha1_hash : str\n        Expected sha1 hash in hexadecimal digits.\n\n    Returns\n    -------\n    bool\n        Whether the file content matches the expected hash.\n    \"\"\"\n    sha1 = hashlib.sha1()\n    with open(filename, 'rb') as f:\n        while True:\n            data = f.read(1048576)\n            if not data:\n                break\n            sha1.update(data)\n\n    return sha1.hexdigest() == sha1_hash\n\n\nif not sys.platform.startswith('win32'):\n    # refer to https://github.com/untitaker/python-atomicwrites\n    def replace_file(src, dst):\n        \"\"\"Implement atomic os.replace with linux and OSX.\n\n        Parameters\n        ----------\n        src : source file path\n        dst : destination file path\n        \"\"\"\n        try:\n            os.rename(src, dst)\n        except OSError:\n            try:\n                os.remove(src)\n            except OSError:\n                pass\n            finally:\n                raise OSError(\n                    'Moving downloaded temp file - {}, to {} failed. \\\n                    Please retry the download.'.format(src, dst))\nelse:\n    import ctypes\n\n    _MOVEFILE_REPLACE_EXISTING = 0x1\n    # Setting this value guarantees that a move performed as a copy\n    # and delete operation is flushed to disk before the function returns.\n    # The flush occurs at the end of the copy operation.\n    _MOVEFILE_WRITE_THROUGH = 0x8\n    _windows_default_flags = _MOVEFILE_WRITE_THROUGH\n\n    def _str_to_unicode(x):\n        \"\"\"Handle text decoding. Internal use only\"\"\"\n        if not isinstance(x, str):\n            return x.decode(sys.getfilesystemencoding())\n        return x\n\n    def _handle_errors(rv, src):\n        \"\"\"Handle WinError. Internal use only\"\"\"\n        if not rv:\n            msg = ctypes.FormatError(ctypes.GetLastError())\n            # if the MoveFileExW fails(e.g. fail to acquire file lock), removes the tempfile\n            try:\n                os.remove(src)\n            except OSError:\n                pass\n            finally:\n                raise OSError(msg)\n\n    def replace_file(src, dst):\n        \"\"\"Implement atomic os.replace with windows.\n\n        refer to https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-movefileexw\n        The function fails when one of the process(copy, flush, delete) fails.\n\n        Parameters\n        ----------\n        src : source file path\n        dst : destination file path\n        \"\"\"\n        _handle_errors(ctypes.windll.kernel32.MoveFileExW(\n            _str_to_unicode(src), _str_to_unicode(dst),\n            _windows_default_flags | _MOVEFILE_REPLACE_EXISTING\n        ), src)\n\n\ndef download(url, path=None, overwrite=False, sha1_hash=None, retries=5, verify_ssl=True):\n    \"\"\"Download a given URL\n\n    Parameters\n    ----------\n    url : str\n        URL to download\n    path : str, optional\n        Destination path to store downloaded file. By default stores to the\n        current directory with same name as in url.\n    overwrite : bool, optional\n        Whether to overwrite destination file if already exists.\n    sha1_hash : str, optional\n        Expected sha1 hash in hexadecimal digits. Will ignore existing file when hash is specified\n        but doesn't match.\n    retries : integer, default 5\n        The number of times to attempt the download in case of failure or non 200 return codes\n    verify_ssl : bool, default True\n        Verify SSL certificates.\n\n    Returns\n    -------\n    str\n        The file path of the downloaded file.\n    \"\"\"\n    if path is None:\n        fname = url.split('/')[-1]\n        # Empty filenames are invalid\n        assert fname, 'Can\\'t construct file-name from this URL. ' \\\n            'Please set the `path` option manually.'\n    else:\n        path = os.path.expanduser(path)\n        if os.path.isdir(path):\n            fname = os.path.join(path, url.split('/')[-1])\n        else:\n            fname = path\n    assert retries >= 0, \"Number of retries should be at least 0, currently it's {}\".format(\n        retries)\n\n    if not verify_ssl:\n        warnings.warn(\n            'Unverified HTTPS request is being made (verify_ssl=False). '\n            'Adding certificate verification is strongly advised.')\n\n    if overwrite or not os.path.exists(fname) or (sha1_hash and not check_sha1(fname, sha1_hash)):\n        dirname = os.path.dirname(os.path.abspath(os.path.expanduser(fname)))\n        if not os.path.exists(dirname):\n            os.makedirs(dirname, exist_ok=True)\n        while retries + 1 > 0:\n            # Disable pyling too broad Exception\n            # pylint: disable=W0703\n            try:\n                print('Downloading {} from {}...'.format(fname, url))\n                r = requests.get(url, stream=True, verify=verify_ssl)\n                if r.status_code != 200:\n                    raise RuntimeError('Failed downloading url {}'.format(url))\n                # create uuid for temporary files\n                random_uuid = str(uuid.uuid4())\n                with open('{}.{}'.format(fname, random_uuid), 'wb') as f:\n                    for chunk in r.iter_content(chunk_size=1024):\n                        if chunk: # filter out keep-alive new chunks\n                            f.write(chunk)\n                # if the target file exists(created by other processes)\n                # and have the same hash with target file\n                # delete the temporary file\n                if not os.path.exists(fname) or (sha1_hash and not check_sha1(fname, sha1_hash)):\n                    # atmoic operation in the same file system\n                    replace_file('{}.{}'.format(fname, random_uuid), fname)\n                else:\n                    try:\n                        os.remove('{}.{}'.format(fname, random_uuid))\n                    except OSError:\n                        pass\n                    finally:\n                        warnings.warn(\n                            'File {} exists in file system so the downloaded file is deleted'.format(fname))\n                if sha1_hash and not check_sha1(fname, sha1_hash):\n                    raise UserWarning(\n                        'File {} is downloaded but the content hash does not match.'\n                        ' The repo may be outdated or download may be incomplete. '\n                        'If the \"repo_url\" is overridden, consider switching to '\n                        'the default repo.'.format(fname))\n                break\n            except Exception as e:\n                retries -= 1\n                if retries <= 0:\n                    raise e\n\n                print('download failed due to {}, retrying, {} attempt{} left'\n                      .format(repr(e), retries, 's' if retries > 1 else ''))\n\n    return fname\n\ndef _get_repo_url():\n    \"\"\"Return the base URL for Gluon dataset and model repository.\"\"\"\n    default_repo = 'https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/'\n    repo_url = os.environ.get('MXNET_GLUON_REPO', default_repo)\n    if repo_url[-1] != '/':\n        repo_url = repo_url+'/'\n    return repo_url\n\ndef _get_repo_file_url(namespace, filename):\n    \"\"\"Return the URL for hosted file in Gluon repository.\n\n    Parameters\n    ----------\n    namespace : str\n        Namespace of the file.\n    filename : str\n        Name of the file\n    \"\"\"\n    return '{base_url}{namespace}/{filename}'.format(base_url=_get_repo_url(),\n                                                     namespace=namespace,\n                                                     filename=filename)\n\ndef _brief_print_list(lst, limit=7):\n    \"\"\"Print at most `limit` elements of list.\"\"\"\n    lst = list(lst)\n    if len(lst) > limit:\n        return _brief_print_list(lst[:limit//2], limit) + ', ..., ' + \\\n            _brief_print_list(lst[-limit//2:], limit)\n    return ', '.join([f\"'{str(i)}'\" for i in lst])\n\n\nclass HookHandle(object):\n    \"\"\"A handle that can attach/detach a hook.\"\"\"\n\n    def __init__(self):\n        self._hooks_dict_ref = None\n        self._id = None\n\n    def attach(self, hooks_dict, hook):\n        assert not self._hooks_dict_ref, 'The same handle cannot be attached twice.'\n        self._id = id(hook)\n        hooks_dict[self._id] = hook\n        self._hooks_dict_ref = weakref.ref(hooks_dict)\n\n    def detach(self):\n        hooks_dict = self._hooks_dict_ref()\n        if hooks_dict is not None and self._id in hooks_dict:\n            del hooks_dict[self._id]\n\n    def __getstate__(self):\n        return (self._hooks_dict_ref(), self._id)\n\n    def __setstate__(self, state):\n        if state[0] is None:\n            self._hooks_dict_ref = weakref.ref(collections.OrderedDict())\n        else:\n            self._hooks_dict_ref = weakref.ref(state[0])\n        self._id = state[1]\n\n    def __enter__(self):\n        return self\n\n    def __exit__(self, ptype, value, trace):\n        self.detach()\n\n\ndef shape_is_known(shape):\n    \"\"\"Check whether a shape is completely known with or without np semantics.\n\n    Please see the doc of is_np_shape for more details.\n    \"\"\"\n    if shape is None:\n        return False\n    unknown_dim_size = -1 if is_np_shape() else 0\n    if len(shape) == 0:\n        return unknown_dim_size == -1\n    for dim_size in shape:\n        if dim_size == unknown_dim_size:\n            return False\n        assert dim_size > unknown_dim_size, \"shape dimension size cannot be less than {}, while \" \\\n                                            \"received {}\".format(unknown_dim_size, dim_size)\n    return True\n\n\ndef _check_same_symbol_type(symbols):\n    \"\"\"Check whether all the symbols in the list are of the same type.\n    Raise type error if the types are different. Return the class of\n    the symbols.\"\"\"\n    from ..symbol.numpy import _Symbol as np_symbol\n    from ..symbol import Symbol as nd_symbol\n    is_np_sym = isinstance(symbols[0], np_symbol)\n    for s in symbols[1:]:\n        if is_np_sym != isinstance(s, np_symbol):\n            raise TypeError('Found both classic symbol (mx.sym.Symbol) and numpy symbol '\n                            '(mx.sym.np._Symbol) in outputs. This will prevent you from building '\n                            'a computation graph by grouping them since different types of symbols '\n                            'are not allowed to be grouped in Gluon to form a computation graph. '\n                            'You will need to convert them to the same type of symbols, either '\n                            'classic or numpy following this rule: if you want numpy ndarray '\n                            'output(s) from the computation graph, please convert all the classic '\n                            'symbols in the list to numpy symbols by calling `as_np_ndarray()` '\n                            'on each of them; if you want classic ndarray output(s) from the '\n                            'computation graph, please convert all the numpy symbols in the list '\n                            'to classic symbols by calling `as_nd_ndarray()` on each of them.')\n    return np_symbol if is_np_sym else nd_symbol\n\n\ndef _check_all_np_ndarrays(out):\n    \"\"\"Check if ndarrays/symbols in out are all np.ndarray/np._Symbol.\"\"\"\n    from ..numpy import ndarray as np_ndarray\n    from ..symbol.numpy import _Symbol as np_symbol\n    from ..symbol import Symbol as nd_symbol\n    from ..ndarray import NDArray as nd_ndarray\n\n    # pylint: disable=no-else-raise\n    if isinstance(out, (nd_ndarray, nd_symbol)) and not isinstance(out, (np_ndarray, np_symbol)):\n        raise TypeError(\"Block's output ndarrays/symbols must be of type `mxnet.numpy.ndarray`\"\n                        \" or `mxnet.symbol.numpy._Symbol`, while got output type {}\"\n                        .format(str(type(out))))\n    elif isinstance(out, (list, tuple)):\n        for i in out:\n            _check_all_np_ndarrays(i)\n    # pylint: enable=no-else-raise\n\n\ndef _check_block_input_np_ndarrays(inputs):\n    \"\"\"Check if block's inputs are numpy ndarrays.\"\"\"\n    from ..numpy import ndarray as np_ndarray\n    from ..symbol import Symbol as nd_symbol\n    from ..ndarray import NDArray as nd_ndarray\n\n    # pylint: disable=no-else-raise\n    if isinstance(inputs, (nd_ndarray, nd_symbol)) and not isinstance(inputs, (np_ndarray)):\n        raise TypeError(\"Block's inputs must be of type `mxnet.numpy.ndarray`, \"\n                        \"while got output type {}\"\n                        .format(str(type(inputs))))\n    elif isinstance(inputs, (list, tuple)):\n        for i in inputs:\n            _check_block_input_np_ndarrays(i)\n    # pylint: enable=no-else-raise\n\n\n# pylint: disable=too-many-nested-blocks\ndef split_rnn_params(param, mode, num_layers, input_size, hidden_size, bidirectional=False, projection_size=None):\n    \"\"\"Split rnn layer parameter into weight and bias in different layer.\n\n    Parameters\n    ----------\n    param : ndarray\n        The parameter of rnn layer.\n    mode : str\n        Mode of rnn. Supported modes: rnn_relu, rnn_tanh, lstm, gru\n    num_layers : int, default 1\n        Number of recurrent layers.\n    input_size: int, default 0\n        The number of expected features in the input x.\n        If not specified, it will be inferred from input.\n    hidden_size: int\n        The number of features in the hidden state h.\n    bidirectional: bool, default False\n        If `True`, becomes a bidirectional RNN.\n    projection_size: int, default None\n        The number of features after projection.\n    \"\"\"\n    gates = {'rnn_relu': 1, 'rnn_tanh': 1, 'lstm': 4, 'gru': 3}[mode]\n    dir = 2 if bidirectional else 1\n    param_dict = {}\n    begin = 0\n    if not projection_size:\n        for p in ['weight', 'bias']:\n            for l in range(num_layers):\n                for d in ['l', 'r'][:dir]:\n                    for g in ['i2h', 'h2h']:\n                        ni = input_size\n                        if l != 0:\n                            ni = hidden_size * dir\n                        if g == 'h2h':\n                            ni = hidden_size\n                        shape0 = gates * hidden_size\n                        if p == 'weight':\n                            cur_len = shape0 * ni\n                            param_dict['{}{}_{}_{}'.format(d, l, g, p)] = \\\n                                param[begin:begin+cur_len].reshape(shape0, ni)\n                        else:\n                            cur_len = shape0\n                            param_dict['{}{}_{}_{}'.format(d, l, g, p)] = \\\n                                param[begin:begin+cur_len].reshape(shape0,)\n                        begin += cur_len\n    else:\n        for p in ['weight', 'bias']:\n            for l in range(num_layers):\n                for d in ['l', 'r'][:dir]:\n                    for g in ['i2h', 'h2h', 'h2r']:\n                        if g != 'h2r' or p != 'bias':\n                            if g == 'h2r':\n                                cur_len = projection_size * hidden_size\n                                param_dict['{}{}_{}_{}'.format(d, l, g, p)] = \\\n                                    param[begin:begin+cur_len]. \\\n                                        reshape(projection_size, hidden_size)\n                            else:\n                                ni = input_size\n                                if l != 0:\n                                    ni = projection_size * dir\n                                if g == 'h2h':\n                                    ni = projection_size\n                                shape0 = gates * hidden_size\n                                if p == 'weight':\n                                    cur_len = shape0 * ni\n                                    param_dict['{}{}_{}_{}'.format(d, l, g, p)] = \\\n                                        param[begin:begin+cur_len].reshape(shape0, ni)\n                                else:\n                                    cur_len = shape0\n                                    param_dict['{}{}_{}_{}'.format(d, l, g, p)] = \\\n                                        param[begin:begin+cur_len].reshape(shape0,)\n                            begin += cur_len\n    return param_dict\n"
  },
  {
    "path": "python/mxnet/image/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\"Image Iterators and image augmentation functions\"\"\"\n\nfrom . import image\nfrom .image import *\n\nfrom . import detection\nfrom . import detection as det\nfrom .detection import *\n"
  },
  {
    "path": "python/mxnet/image/detection.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=unused-import, too-many-lines\n\"\"\"Read images and perform augmentations for object detection.\"\"\"\n\n\nimport json\nimport logging\nimport random\nimport warnings\n\nimport numpy as np\n\nfrom ..base import numeric_types\nfrom .. import ndarray as nd\nfrom ..ndarray._internal import _cvcopyMakeBorder as copyMakeBorder\nfrom .. import io\nfrom .image import RandomOrderAug, ColorJitterAug, LightingAug, ColorNormalizeAug\nfrom .image import ResizeAug, ForceResizeAug, CastAug, HueJitterAug, RandomGrayAug\nfrom .image import fixed_crop, ImageIter, Augmenter\nfrom ..util import is_np_array\nfrom .. import numpy as _mx_np  # pylint: disable=reimported\n\n\nclass DetAugmenter(object):\n    \"\"\"Detection base augmenter\"\"\"\n    def __init__(self, **kwargs):\n        self._kwargs = kwargs\n        for k, v in self._kwargs.items():\n            if isinstance(v, nd.NDArray):\n                v = v.asnumpy()\n            if isinstance(v, np.ndarray):\n                v = v.tolist()\n                self._kwargs[k] = v\n\n    def dumps(self):\n        \"\"\"Saves the Augmenter to string\n\n        Returns\n        -------\n        str\n            JSON formatted string that describes the Augmenter.\n        \"\"\"\n        return json.dumps([self.__class__.__name__.lower(), self._kwargs])\n\n    def __call__(self, src, label):\n        \"\"\"Abstract implementation body\"\"\"\n        raise NotImplementedError(\"Must override implementation.\")\n\n\nclass DetBorrowAug(DetAugmenter):\n    \"\"\"Borrow standard augmenter from image classification.\n    Which is good once you know label won't be affected after this augmenter.\n\n    Parameters\n    ----------\n    augmenter : mx.image.Augmenter\n        The borrowed standard augmenter which has no effect on label\n    \"\"\"\n    def __init__(self, augmenter):\n        if not isinstance(augmenter, Augmenter):\n            raise TypeError('Borrowing from invalid Augmenter')\n        super(DetBorrowAug, self).__init__(augmenter=augmenter.dumps())\n        self.augmenter = augmenter\n\n    def dumps(self):\n        \"\"\"Override the default one to avoid duplicate dump.\"\"\"\n        return [self.__class__.__name__.lower(), self.augmenter.dumps()]\n\n    def __call__(self, src, label):\n        \"\"\"Augmenter implementation body\"\"\"\n        src = self.augmenter(src)\n        return (src, label)\n\n\nclass DetRandomSelectAug(DetAugmenter):\n    \"\"\"Randomly select one augmenter to apply, with chance to skip all.\n\n    Parameters\n    ----------\n    aug_list : list of DetAugmenter\n        The random selection will be applied to one of the augmenters\n    skip_prob : float\n        The probability to skip all augmenters and return input directly\n    \"\"\"\n    def __init__(self, aug_list, skip_prob=0):\n        super(DetRandomSelectAug, self).__init__(skip_prob=skip_prob)\n        if not isinstance(aug_list, (list, tuple)):\n            aug_list = [aug_list]\n        for aug in aug_list:\n            if not isinstance(aug, DetAugmenter):\n                raise ValueError('Allow DetAugmenter in list only')\n        if not aug_list:\n            skip_prob = 1  # disabled\n\n        self.aug_list = aug_list\n        self.skip_prob = skip_prob\n\n    def dumps(self):\n        \"\"\"Override default.\"\"\"\n        return [self.__class__.__name__.lower(), [x.dumps() for x in self.aug_list]]\n\n    def __call__(self, src, label):\n        \"\"\"Augmenter implementation body\"\"\"\n        if random.random() < self.skip_prob:\n            return (src, label)\n        else:\n            random.shuffle(self.aug_list)\n            return self.aug_list[0](src, label)\n\n\nclass DetHorizontalFlipAug(DetAugmenter):\n    \"\"\"Random horizontal flipping.\n\n    Parameters\n    ----------\n    p : float\n        chance [0, 1] to flip\n    \"\"\"\n    def __init__(self, p):\n        super(DetHorizontalFlipAug, self).__init__(p=p)\n        self.p = p\n\n    def __call__(self, src, label):\n        \"\"\"Augmenter implementation\"\"\"\n        if random.random() < self.p:\n            src = nd.flip(src, axis=1)\n            self._flip_label(label)\n        return (src, label)\n\n    def _flip_label(self, label):\n        \"\"\"Helper function to flip label.\"\"\"\n        tmp = 1.0 - label[:, 1]\n        label[:, 1] = 1.0 - label[:, 3]\n        label[:, 3] = tmp\n\n\nclass DetRandomCropAug(DetAugmenter):\n    \"\"\"Random cropping with constraints\n\n    Parameters\n    ----------\n    min_object_covered : float, default=0.1\n        The cropped area of the image must contain at least this fraction of\n        any bounding box supplied. The value of this parameter should be non-negative.\n        In the case of 0, the cropped area does not need to overlap any of the\n        bounding boxes supplied.\n    min_eject_coverage : float, default=0.3\n        The minimum coverage of cropped sample w.r.t its original size. With this\n        constraint, objects that have marginal area after crop will be discarded.\n    aspect_ratio_range : tuple of floats, default=(0.75, 1.33)\n        The cropped area of the image must have an aspect ratio = width / height\n        within this range.\n    area_range : tuple of floats, default=(0.05, 1.0)\n        The cropped area of the image must contain a fraction of the supplied\n        image within in this range.\n    max_attempts : int, default=50\n        Number of attempts at generating a cropped/padded region of the image of the\n        specified constraints. After max_attempts failures, return the original image.\n    \"\"\"\n    def __init__(self, min_object_covered=0.1, aspect_ratio_range=(0.75, 1.33),\n                 area_range=(0.05, 1.0), min_eject_coverage=0.3, max_attempts=50):\n        if not isinstance(aspect_ratio_range, (tuple, list)):\n            assert isinstance(aspect_ratio_range, numeric_types)\n            logging.info('Using fixed aspect ratio: %s in DetRandomCropAug',\n                         str(aspect_ratio_range))\n            aspect_ratio_range = (aspect_ratio_range, aspect_ratio_range)\n        if not isinstance(area_range, (tuple, list)):\n            assert isinstance(area_range, numeric_types)\n            logging.info('Using fixed area range: %s in DetRandomCropAug', area_range)\n            area_range = (area_range, area_range)\n        super(DetRandomCropAug, self).__init__(min_object_covered=min_object_covered,\n                                               aspect_ratio_range=aspect_ratio_range,\n                                               area_range=area_range,\n                                               min_eject_coverage=min_eject_coverage,\n                                               max_attempts=max_attempts)\n        self.min_object_covered = min_object_covered\n        self.min_eject_coverage = min_eject_coverage\n        self.max_attempts = max_attempts\n        self.aspect_ratio_range = aspect_ratio_range\n        self.area_range = area_range\n        self.enabled = False\n        if (area_range[1] <= 0 or area_range[0] > area_range[1]):\n            warnings.warn('Skip DetRandomCropAug due to invalid area_range: %s', area_range)\n        elif (aspect_ratio_range[0] > aspect_ratio_range[1] or aspect_ratio_range[0] <= 0):\n            warnings.warn('Skip DetRandomCropAug due to invalid aspect_ratio_range: %s',\n                          aspect_ratio_range)\n        else:\n            self.enabled = True\n\n    def __call__(self, src, label):\n        \"\"\"Augmenter implementation body\"\"\"\n        crop = self._random_crop_proposal(label, src.shape[0], src.shape[1])\n        if crop:\n            x, y, w, h, label = crop\n            src = fixed_crop(src, x, y, w, h, None)\n        return (src, label)\n\n    def _calculate_areas(self, label):\n        \"\"\"Calculate areas for multiple labels\"\"\"\n        heights = np.maximum(0, label[:, 3] - label[:, 1])\n        widths = np.maximum(0, label[:, 2] - label[:, 0])\n        return heights * widths\n\n\n    def _intersect(self, label, xmin, ymin, xmax, ymax):\n        \"\"\"Calculate intersect areas, normalized.\"\"\"\n        left = np.maximum(label[:, 0], xmin)\n        right = np.minimum(label[:, 2], xmax)\n        top = np.maximum(label[:, 1], ymin)\n        bot = np.minimum(label[:, 3], ymax)\n        invalid = np.where(np.logical_or(left >= right, top >= bot))[0]\n        out = label.copy()\n        out[:, 0] = left\n        out[:, 1] = top\n        out[:, 2] = right\n        out[:, 3] = bot\n        out[invalid, :] = 0\n        return out\n\n    def _check_satisfy_constraints(self, label, xmin, ymin, xmax, ymax, width, height):\n        \"\"\"Check if constrains are satisfied\"\"\"\n        if (xmax - xmin) * (ymax - ymin) < 2:\n            return False  # only 1 pixel\n        x1 = float(xmin) / width\n        y1 = float(ymin) / height\n        x2 = float(xmax) / width\n        y2 = float(ymax) / height\n        object_areas = self._calculate_areas(label[:, 1:])\n        valid_objects = np.where(object_areas * width * height > 2)[0]\n        if valid_objects.size < 1:\n            return False\n        intersects = self._intersect(label[valid_objects, 1:], x1, y1, x2, y2)\n        coverages = self._calculate_areas(intersects) / object_areas[valid_objects]\n        coverages = coverages[np.where(coverages > 0)[0]]\n        return coverages.size > 0 and np.amin(coverages) > self.min_object_covered\n\n    def _update_labels(self, label, crop_box, height, width):\n        \"\"\"Convert labels according to crop box\"\"\"\n        xmin = float(crop_box[0]) / width\n        ymin = float(crop_box[1]) / height\n        w = float(crop_box[2]) / width\n        h = float(crop_box[3]) / height\n        out = label.copy()\n        out[:, (1, 3)] -= xmin\n        out[:, (2, 4)] -= ymin\n        out[:, (1, 3)] /= w\n        out[:, (2, 4)] /= h\n        out[:, 1:5] = np.maximum(0, out[:, 1:5])\n        out[:, 1:5] = np.minimum(1, out[:, 1:5])\n        coverage = self._calculate_areas(out[:, 1:]) * w * h / self._calculate_areas(label[:, 1:])\n        valid = np.logical_and(out[:, 3] > out[:, 1], out[:, 4] > out[:, 2])\n        valid = np.logical_and(valid, coverage > self.min_eject_coverage)\n        valid = np.where(valid)[0]\n        if valid.size < 1:\n            return None\n        out = out[valid, :]\n        return out\n\n    def _random_crop_proposal(self, label, height, width):\n        \"\"\"Propose cropping areas\"\"\"\n        from math import sqrt\n\n        if not self.enabled or height <= 0 or width <= 0:\n            return ()\n        min_area = self.area_range[0] * height * width\n        max_area = self.area_range[1] * height * width\n        for _ in range(self.max_attempts):\n            ratio = random.uniform(*self.aspect_ratio_range)\n            if ratio <= 0:\n                continue\n            h = int(round(sqrt(min_area / ratio)))\n            max_h = int(round(sqrt(max_area / ratio)))\n            if round(max_h * ratio) > width:\n                # find smallest max_h satifying round(max_h * ratio) <= width\n                max_h = int((width + 0.4999999) / ratio)\n            if max_h > height:\n                max_h = height\n            if h > max_h:\n                h = max_h\n            if h < max_h:\n                # generate random h in range [h, max_h]\n                h = random.randint(h, max_h)\n            w = int(round(h * ratio))\n            assert w <= width\n\n            # trying to fix rounding problems\n            area = w * h\n            if area < min_area:\n                h += 1\n                w = int(round(h * ratio))\n                area = w * h\n            if area > max_area:\n                h -= 1\n                w = int(round(h * ratio))\n                area = w * h\n            if not (min_area <= area <= max_area and 0 <= w <= width and 0 <= h <= height):\n                continue\n\n            y = random.randint(0, max(0, height - h))\n            x = random.randint(0, max(0, width - w))\n            if self._check_satisfy_constraints(label, x, y, x + w, y + h, width, height):\n                new_label = self._update_labels(label, (x, y, w, h), height, width)\n                if new_label is not None:\n                    return (x, y, w, h, new_label)\n        return ()\n\n\nclass DetRandomPadAug(DetAugmenter):\n    \"\"\"Random padding augmenter.\n\n    Parameters\n    ----------\n    aspect_ratio_range : tuple of floats, default=(0.75, 1.33)\n        The padded area of the image must have an aspect ratio = width / height\n        within this range.\n    area_range : tuple of floats, default=(1.0, 3.0)\n        The padded area of the image must be larger than the original area\n    max_attempts : int, default=50\n        Number of attempts at generating a padded region of the image of the\n        specified constraints. After max_attempts failures, return the original image.\n    pad_val: float or tuple of float, default=(128, 128, 128)\n        pixel value to be filled when padding is enabled.\n    \"\"\"\n    def __init__(self, aspect_ratio_range=(0.75, 1.33), area_range=(1.0, 3.0),\n                 max_attempts=50, pad_val=(128, 128, 128)):\n        if not isinstance(pad_val, (list, tuple)):\n            assert isinstance(pad_val, numeric_types)\n            pad_val = (pad_val)\n        if not isinstance(aspect_ratio_range, (list, tuple)):\n            assert isinstance(aspect_ratio_range, numeric_types)\n            logging.info('Using fixed aspect ratio: %s in DetRandomPadAug',\n                         str(aspect_ratio_range))\n            aspect_ratio_range = (aspect_ratio_range, aspect_ratio_range)\n        if not isinstance(area_range, (tuple, list)):\n            assert isinstance(area_range, numeric_types)\n            logging.info('Using fixed area range: %s in DetRandomPadAug', area_range)\n            area_range = (area_range, area_range)\n        super(DetRandomPadAug, self).__init__(aspect_ratio_range=aspect_ratio_range,\n                                              area_range=area_range, max_attempts=max_attempts,\n                                              pad_val=pad_val)\n        self.pad_val = pad_val\n        self.aspect_ratio_range = aspect_ratio_range\n        self.area_range = area_range\n        self.max_attempts = max_attempts\n        self.enabled = False\n        if (area_range[1] <= 1.0 or area_range[0] > area_range[1]):\n            warnings.warn('Skip DetRandomPadAug due to invalid parameters: %s', area_range)\n        elif (aspect_ratio_range[0] <= 0 or aspect_ratio_range[0] > aspect_ratio_range[1]):\n            warnings.warn('Skip DetRandomPadAug due to invalid aspect_ratio_range: %s',\n                          aspect_ratio_range)\n        else:\n            self.enabled = True\n\n    def __call__(self, src, label):\n        \"\"\"Augmenter body\"\"\"\n        height, width, _ = src.shape\n        pad = self._random_pad_proposal(label, height, width)\n        if pad:\n            x, y, w, h, label = pad\n            src = copyMakeBorder(src, y, h-y-height, x, w-x-width, 16, values=self.pad_val)\n        return (src, label)\n\n    def _update_labels(self, label, pad_box, height, width):\n        \"\"\"Update label according to padding region\"\"\"\n        out = label.copy()\n        out[:, (1, 3)] = (out[:, (1, 3)] * width + pad_box[0]) / pad_box[2]\n        out[:, (2, 4)] = (out[:, (2, 4)] * height + pad_box[1]) / pad_box[3]\n        return out\n\n    def _random_pad_proposal(self, label, height, width):\n        \"\"\"Generate random padding region\"\"\"\n        from math import sqrt\n        if not self.enabled or height <= 0 or width <= 0:\n            return ()\n        min_area = self.area_range[0] * height * width\n        max_area = self.area_range[1] * height * width\n        for _ in range(self.max_attempts):\n            ratio = random.uniform(*self.aspect_ratio_range)\n            if ratio <= 0:\n                continue\n            h = int(round(sqrt(min_area / ratio)))\n            max_h = int(round(sqrt(max_area / ratio)))\n            if round(h * ratio) < width:\n                h = int((width + 0.499999) / ratio)\n            if h < height:\n                h = height\n            if h > max_h:\n                h = max_h\n            if h < max_h:\n                h = random.randint(h, max_h)\n            w = int(round(h * ratio))\n            if (h - height) < 2 or (w - width) < 2:\n                continue  # marginal padding is not helpful\n\n            y = random.randint(0, max(0, h - height))\n            x = random.randint(0, max(0, w - width))\n            new_label = self._update_labels(label, (x, y, w, h), height, width)\n            return (x, y, w, h, new_label)\n        return ()\n\n\ndef CreateMultiRandCropAugmenter(min_object_covered=0.1, aspect_ratio_range=(0.75, 1.33),\n                                 area_range=(0.05, 1.0), min_eject_coverage=0.3,\n                                 max_attempts=50, skip_prob=0):\n    \"\"\"Helper function to create multiple random crop augmenters.\n\n    Parameters\n    ----------\n    min_object_covered : float or list of float, default=0.1\n        The cropped area of the image must contain at least this fraction of\n        any bounding box supplied. The value of this parameter should be non-negative.\n        In the case of 0, the cropped area does not need to overlap any of the\n        bounding boxes supplied.\n    min_eject_coverage : float or list of float, default=0.3\n        The minimum coverage of cropped sample w.r.t its original size. With this\n        constraint, objects that have marginal area after crop will be discarded.\n    aspect_ratio_range : tuple of floats or list of tuple of floats, default=(0.75, 1.33)\n        The cropped area of the image must have an aspect ratio = width / height\n        within this range.\n    area_range : tuple of floats or list of tuple of floats, default=(0.05, 1.0)\n        The cropped area of the image must contain a fraction of the supplied\n        image within in this range.\n    max_attempts : int or list of int, default=50\n        Number of attempts at generating a cropped/padded region of the image of the\n        specified constraints. After max_attempts failures, return the original image.\n\n    Examples\n    --------\n    >>> # An example of creating multiple random crop augmenters\n    >>> min_object_covered = [0.1, 0.3, 0.5, 0.7, 0.9]  # use 5 augmenters\n    >>> aspect_ratio_range = (0.75, 1.33)  # use same range for all augmenters\n    >>> area_range = [(0.1, 1.0), (0.2, 1.0), (0.2, 1.0), (0.3, 0.9), (0.5, 1.0)]\n    >>> min_eject_coverage = 0.3\n    >>> max_attempts = 50\n    >>> aug = mx.image.det.CreateMultiRandCropAugmenter(min_object_covered=min_object_covered,\n            aspect_ratio_range=aspect_ratio_range, area_range=area_range,\n            min_eject_coverage=min_eject_coverage, max_attempts=max_attempts,\n            skip_prob=0)\n    >>> aug.dumps()  # show some details\n\n    \"\"\"\n    def align_parameters(params):\n        \"\"\"Align parameters as pairs\"\"\"\n        out_params = []\n        num = 1\n        for p in params:\n            if not isinstance(p, list):\n                p = [p]\n            out_params.append(p)\n            num = max(num, len(p))\n        # align for each param\n        for k, p in enumerate(out_params):\n            if len(p) != num:\n                assert len(p) == 1\n                out_params[k] = p * num\n        return out_params\n\n    aligned_params = align_parameters([min_object_covered, aspect_ratio_range, area_range,\n                                       min_eject_coverage, max_attempts])\n    augs = []\n    for moc, arr, ar, mec, ma in zip(*aligned_params):\n        augs.append(DetRandomCropAug(min_object_covered=moc, aspect_ratio_range=arr,\n                                     area_range=ar, min_eject_coverage=mec, max_attempts=ma))\n    return DetRandomSelectAug(augs, skip_prob=skip_prob)\n\n\ndef CreateDetAugmenter(data_shape, resize=0, rand_crop=0, rand_pad=0, rand_gray=0,\n                       rand_mirror=False, mean=None, std=None, brightness=0, contrast=0,\n                       saturation=0, pca_noise=0, hue=0, inter_method=2, min_object_covered=0.1,\n                       aspect_ratio_range=(0.75, 1.33), area_range=(0.05, 3.0),\n                       min_eject_coverage=0.3, max_attempts=50, pad_val=(127, 127, 127)):\n    \"\"\"Create augmenters for detection.\n\n    Parameters\n    ----------\n    data_shape : tuple of int\n        Shape for output data\n    resize : int\n        Resize shorter edge if larger than 0 at the begining\n    rand_crop : float\n        [0, 1], probability to apply random cropping\n    rand_pad : float\n        [0, 1], probability to apply random padding\n    rand_gray : float\n        [0, 1], probability to convert to grayscale for all channels\n    rand_mirror : bool\n        Whether to apply horizontal flip to image with probability 0.5\n    mean : np.ndarray or None\n        Mean pixel values for [r, g, b]\n    std : np.ndarray or None\n        Standard deviations for [r, g, b]\n    brightness : float\n        Brightness jittering range (percent)\n    contrast : float\n        Contrast jittering range (percent)\n    saturation : float\n        Saturation jittering range (percent)\n    hue : float\n        Hue jittering range (percent)\n    pca_noise : float\n        Pca noise level (percent)\n    inter_method : int, default=2(Area-based)\n        Interpolation method for all resizing operations\n\n        Possible values:\n        0: Nearest Neighbors Interpolation.\n        1: Bilinear interpolation.\n        2: Area-based (resampling using pixel area relation). It may be a\n        preferred method for image decimation, as it gives moire-free\n        results. But when the image is zoomed, it is similar to the Nearest\n        Neighbors method. (used by default).\n        3: Bicubic interpolation over 4x4 pixel neighborhood.\n        4: Lanczos interpolation over 8x8 pixel neighborhood.\n        9: Cubic for enlarge, area for shrink, bilinear for others\n        10: Random select from interpolation method metioned above.\n        Note:\n        When shrinking an image, it will generally look best with AREA-based\n        interpolation, whereas, when enlarging an image, it will generally look best\n        with Bicubic (slow) or Bilinear (faster but still looks OK).\n    min_object_covered : float\n        The cropped area of the image must contain at least this fraction of\n        any bounding box supplied. The value of this parameter should be non-negative.\n        In the case of 0, the cropped area does not need to overlap any of the\n        bounding boxes supplied.\n    min_eject_coverage : float\n        The minimum coverage of cropped sample w.r.t its original size. With this\n        constraint, objects that have marginal area after crop will be discarded.\n    aspect_ratio_range : tuple of floats\n        The cropped area of the image must have an aspect ratio = width / height\n        within this range.\n    area_range : tuple of floats\n        The cropped area of the image must contain a fraction of the supplied\n        image within in this range.\n    max_attempts : int\n        Number of attempts at generating a cropped/padded region of the image of the\n        specified constraints. After max_attempts failures, return the original image.\n    pad_val: float\n        Pixel value to be filled when padding is enabled. pad_val will automatically\n        be subtracted by mean and divided by std if applicable.\n\n    Examples\n    --------\n    >>> # An example of creating multiple augmenters\n    >>> augs = mx.image.CreateDetAugmenter(data_shape=(3, 300, 300), rand_crop=0.5,\n    ...    rand_pad=0.5, rand_mirror=True, mean=True, brightness=0.125, contrast=0.125,\n    ...    saturation=0.125, pca_noise=0.05, inter_method=10, min_object_covered=[0.3, 0.5, 0.9],\n    ...    area_range=(0.3, 3.0))\n    >>> # dump the details\n    >>> for aug in augs:\n    ...    aug.dumps()\n    \"\"\"\n    auglist = []\n\n    if resize > 0:\n        auglist.append(DetBorrowAug(ResizeAug(resize, inter_method)))\n\n    if rand_crop > 0:\n        crop_augs = CreateMultiRandCropAugmenter(min_object_covered, aspect_ratio_range,\n                                                 area_range, min_eject_coverage,\n                                                 max_attempts, skip_prob=(1 - rand_crop))\n        auglist.append(crop_augs)\n\n    if rand_mirror > 0:\n        auglist.append(DetHorizontalFlipAug(0.5))\n\n    # apply random padding as late as possible to save computation\n    if rand_pad > 0:\n        pad_aug = DetRandomPadAug(aspect_ratio_range,\n                                  (1.0, area_range[1]), max_attempts, pad_val)\n        auglist.append(DetRandomSelectAug([pad_aug], 1 - rand_pad))\n\n    # force resize\n    auglist.append(DetBorrowAug(ForceResizeAug((data_shape[2], data_shape[1]), inter_method)))\n\n    auglist.append(DetBorrowAug(CastAug()))\n\n    if brightness or contrast or saturation:\n        auglist.append(DetBorrowAug(ColorJitterAug(brightness, contrast, saturation)))\n\n    if hue:\n        auglist.append(DetBorrowAug(HueJitterAug(hue)))\n\n    if pca_noise > 0:\n        eigval = np.array([55.46, 4.794, 1.148])\n        eigvec = np.array([[-0.5675, 0.7192, 0.4009],\n                           [-0.5808, -0.0045, -0.8140],\n                           [-0.5836, -0.6948, 0.4203]])\n        auglist.append(DetBorrowAug(LightingAug(pca_noise, eigval, eigvec)))\n\n    if rand_gray > 0:\n        auglist.append(DetBorrowAug(RandomGrayAug(rand_gray)))\n\n    if mean is True:\n        mean = np.array([123.68, 116.28, 103.53])\n    elif mean is not None:\n        assert isinstance(mean, np.ndarray) and mean.shape[0] in [1, 3]\n\n    if std is True:\n        std = np.array([58.395, 57.12, 57.375])\n    elif std is not None:\n        assert isinstance(std, np.ndarray) and std.shape[0] in [1, 3]\n\n    if mean is not None or std is not None:\n        auglist.append(DetBorrowAug(ColorNormalizeAug(mean, std)))\n\n    return auglist\n\n\nclass ImageDetIter(ImageIter):\n    \"\"\"Image iterator with a large number of augmentation choices for detection.\n\n    Parameters\n    ----------\n    aug_list : list or None\n        Augmenter list for generating distorted images\n    batch_size : int\n        Number of examples per batch.\n    data_shape : tuple\n        Data shape in (channels, height, width) format.\n        For now, only RGB image with 3 channels is supported.\n    path_imgrec : str\n        Path to image record file (.rec).\n        Created with tools/im2rec.py or bin/im2rec.\n    path_imglist : str\n        Path to image list (.lst).\n        Created with tools/im2rec.py or with custom script.\n        Format: Tab separated record of index, one or more labels and relative_path_from_root.\n    imglist: list\n        A list of images with the label(s).\n        Each item is a list [imagelabel: float or list of float, imgpath].\n    path_root : str\n        Root folder of image files.\n    path_imgidx : str\n        Path to image index file. Needed for partition and shuffling when using .rec source.\n    shuffle : bool\n        Whether to shuffle all images at the start of each iteration or not.\n        Can be slow for HDD.\n    part_index : int\n        Partition index.\n    num_parts : int\n        Total number of partitions.\n    data_name : str\n        Data name for provided symbols.\n    label_name : str\n        Name for detection labels\n    last_batch_handle : str, optional\n        How to handle the last batch.\n        This parameter can be 'pad'(default), 'discard' or 'roll_over'.\n        If 'pad', the last batch will be padded with data starting from the begining\n        If 'discard', the last batch will be discarded\n        If 'roll_over', the remaining elements will be rolled over to the next iteration\n    kwargs : ...\n        More arguments for creating augmenter. See mx.image.CreateDetAugmenter.\n    \"\"\"\n    def __init__(self, batch_size, data_shape,\n                 path_imgrec=None, path_imglist=None, path_root=None, path_imgidx=None,\n                 shuffle=False, part_index=0, num_parts=1, aug_list=None, imglist=None,\n                 data_name='data', label_name='label', last_batch_handle='pad', **kwargs):\n        super(ImageDetIter, self).__init__(batch_size=batch_size, data_shape=data_shape,\n                                           path_imgrec=path_imgrec, path_imglist=path_imglist,\n                                           path_root=path_root, path_imgidx=path_imgidx,\n                                           shuffle=shuffle, part_index=part_index,\n                                           num_parts=num_parts, aug_list=[], imglist=imglist,\n                                           data_name=data_name, label_name=label_name,\n                                           last_batch_handle=last_batch_handle)\n\n        if aug_list is None:\n            self.auglist = CreateDetAugmenter(data_shape, **kwargs)\n        else:\n            self.auglist = aug_list\n\n        # went through all labels to get the proper label shape\n        label_shape = self._estimate_label_shape()\n        self.provide_label = [(label_name, (self.batch_size, label_shape[0], label_shape[1]))]\n        self.label_shape = label_shape\n\n    def _check_valid_label(self, label):\n        \"\"\"Validate label and its shape.\"\"\"\n        if len(label.shape) != 2 or label.shape[1] < 5:\n            msg = f\"Label with shape (1+, 5+) required, {str(label)} received.\"\n            raise RuntimeError(msg)\n        valid_label = np.where(np.logical_and(label[:, 0] >= 0, label[:, 3] > label[:, 1],\n                                              label[:, 4] > label[:, 2]))[0]\n        if valid_label.size < 1:\n            raise RuntimeError('Invalid label occurs.')\n\n    def _estimate_label_shape(self):\n        \"\"\"Helper function to estimate label shape\"\"\"\n        max_count, label = 0, None\n        self.reset()\n        try:\n            while True:\n                label, _ = self.next_sample()\n                label = self._parse_label(label)\n                max_count = max(max_count, label.shape[0])\n        except StopIteration:\n            pass\n        self.reset()\n        return (max_count, label.shape[1] if label is not None else 5)\n\n    def _parse_label(self, label):\n        \"\"\"Helper function to parse object detection label.\n\n        Format for raw label:\n        n \\t k \\t ... \\t [id \\t xmin\\t ymin \\t xmax \\t ymax \\t ...] \\t [repeat]\n        where n is the width of header, 2 or larger\n        k is the width of each object annotation, can be arbitrary, at least 5\n        \"\"\"\n        if isinstance(label, nd.NDArray):\n            label = label.asnumpy()\n        raw = label.ravel()\n        if raw.size < 7:\n            raise RuntimeError(\"Label shape is invalid: \" + str(raw.shape))\n        header_width = int(raw[0])\n        obj_width = int(raw[1])\n        if (raw.size - header_width) % obj_width != 0:\n            msg = f\"Label shape {str(raw.shape)} inconsistent with annotation width {obj_width}.\"\n            raise RuntimeError(msg)\n        out = np.reshape(raw[header_width:], (-1, obj_width))\n        # remove bad ground-truths\n        valid = np.where(np.logical_and(out[:, 3] > out[:, 1], out[:, 4] > out[:, 2]))[0]\n        if valid.size < 1:\n            raise RuntimeError('Encounter sample with no valid label.')\n        return out[valid, :]\n\n    def reshape(self, data_shape=None, label_shape=None):\n        \"\"\"Reshape iterator for data_shape or label_shape.\n\n        Parameters\n        ----------\n        data_shape : tuple or None\n            Reshape the data_shape to the new shape if not None\n        label_shape : tuple or None\n            Reshape label shape to new shape if not None\n        \"\"\"\n        if data_shape is not None:\n            self.check_data_shape(data_shape)\n            self.provide_data = [(self.provide_data[0][0], (self.batch_size,) + data_shape)]\n            self.data_shape = data_shape\n        if label_shape is not None:\n            self.check_label_shape(label_shape)\n            self.provide_label = [(self.provide_label[0][0], (self.batch_size,) + label_shape)]\n            self.label_shape = label_shape\n\n    def _batchify(self, batch_data, batch_label, start=0):\n        \"\"\"Override the helper function for batchifying data\"\"\"\n        i = start\n        batch_size = self.batch_size\n        array_fn = _mx_np.array if is_np_array() else nd.array\n        try:\n            while i < batch_size:\n                label, s = self.next_sample()\n                data = self.imdecode(s)\n                try:\n                    self.check_valid_image([data])\n                    label = self._parse_label(label)\n                    data, label = self.augmentation_transform(data, label)\n                    self._check_valid_label(label)\n                except RuntimeError as e:\n                    logging.debug('Invalid image, skipping:  %s', str(e))\n                    continue\n                for datum in [data]:\n                    assert i < batch_size, 'Batch size must be multiples of augmenter output length'\n                    batch_data[i] = self.postprocess_data(datum)\n                    num_object = label.shape[0]\n                    batch_label[i][0:num_object] = array_fn(label)\n                    if num_object < batch_label[i].shape[0]:\n                        batch_label[i][num_object:] = -1\n                    i += 1\n        except StopIteration:\n            if not i:\n                raise StopIteration\n\n        return i\n\n    def next(self):\n        \"\"\"Override the function for returning next batch.\"\"\"\n        batch_size = self.batch_size\n        c, h, w = self.data_shape\n        # if last batch data is rolled over\n        if self._cache_data is not None:\n            # check both the data and label have values\n            assert self._cache_label is not None, \"_cache_label didn't have values\"\n            assert self._cache_idx is not None, \"_cache_idx didn't have values\"\n            batch_data = self._cache_data\n            batch_label = self._cache_label\n            i = self._cache_idx\n        else:\n            if is_np_array():\n                zeros_fn = _mx_np.zeros\n                empty_fn = _mx_np.empty\n            else:\n                zeros_fn = nd.zeros\n                empty_fn = nd.empty\n            batch_data = zeros_fn((batch_size, c, h, w))\n            batch_label = empty_fn(self.provide_label[0][1])\n            batch_label[:] = -1\n            i = self._batchify(batch_data, batch_label)\n        # calculate the padding\n        pad = batch_size - i\n        # handle padding for the last batch\n        if pad != 0:\n            if self.last_batch_handle == 'discard':\n                raise StopIteration\n            # if the option is 'roll_over', throw StopIteration and cache the data\n            if self.last_batch_handle == 'roll_over' and \\\n                self._cache_data is None:\n                self._cache_data = batch_data\n                self._cache_label = batch_label\n                self._cache_idx = i\n                raise StopIteration\n\n            _ = self._batchify(batch_data, batch_label, i)\n            if self.last_batch_handle == 'pad':\n                self._allow_read = False\n            else:\n                self._cache_data = None\n                self._cache_label = None\n                self._cache_idx = None\n\n        return io.DataBatch([batch_data], [batch_label], pad=pad)\n\n    def augmentation_transform(self, data, label):  # pylint: disable=arguments-differ\n        \"\"\"Override Transforms input data with specified augmentations.\"\"\"\n        for aug in self.auglist:\n            data, label = aug(data, label)\n        return (data, label)\n\n    def check_label_shape(self, label_shape):\n        \"\"\"Checks if the new label shape is valid\"\"\"\n        if not len(label_shape) == 2:\n            raise ValueError('label_shape should have length 2')\n        if label_shape[0] < self.label_shape[0]:\n            msg = f'Attempts to reduce label count from {self.label_shape[0]} to {label_shape[0]}, not allowed.'\n            raise ValueError(msg)\n        if label_shape[1] != self.provide_label[0][1][2]:\n            msg = f'label_shape object width inconsistent: {self.provide_label[0][1][2]} vs {label_shape[1]}.'\n            raise ValueError(msg)\n\n    def draw_next(self, color=None, thickness=2, mean=None, std=None, clip=True,\n                  waitKey=None, window_name='draw_next', id2labels=None):\n        \"\"\"Display next image with bounding boxes drawn.\n\n        Parameters\n        ----------\n        color : tuple\n            Bounding box color in RGB, use None for random color\n        thickness : int\n            Bounding box border thickness\n        mean : True or numpy.ndarray\n            Compensate for the mean to have better visual effect\n        std : True or numpy.ndarray\n            Revert standard deviations\n        clip : bool\n            If true, clip to [0, 255] for better visual effect\n        waitKey : None or int\n            Hold the window for waitKey milliseconds if set, skip ploting if None\n        window_name : str\n            Plot window name if waitKey is set.\n        id2labels : dict\n            Mapping of labels id to labels name.\n\n        Returns\n        -------\n            numpy.ndarray\n\n        Examples\n        --------\n        >>> # use draw_next to get images with bounding boxes drawn\n        >>> iterator = mx.image.ImageDetIter(1, (3, 600, 600), path_imgrec='train.rec')\n        >>> for image in iterator.draw_next(waitKey=None):\n        ...     # display image\n        >>> # or let draw_next display using cv2 module\n        >>> for image in iterator.draw_next(waitKey=0, window_name='disp'):\n        ...     pass\n        \"\"\"\n        try:\n            import cv2\n        except ImportError as e:\n            warnings.warn('Unable to import cv2, skip drawing: %s', str(e))\n            return\n        count = 0\n        try:\n            while True:\n                label, s = self.next_sample()\n                data = self.imdecode(s)\n                try:\n                    self.check_valid_image([data])\n                    label = self._parse_label(label)\n                except RuntimeError as e:\n                    logging.debug('Invalid image, skipping:  %s', str(e))\n                    continue\n                count += 1\n                data, label = self.augmentation_transform(data, label)\n                image = data.asnumpy()\n\n                # revert color_normalize\n                if std is True:\n                    std = np.array([58.395, 57.12, 57.375])\n                elif std is not None:\n                    assert isinstance(std, np.ndarray) and std.shape[0] in [1, 3]\n                if std is not None:\n                    image *= std\n\n                if mean is True:\n                    mean = np.array([123.68, 116.28, 103.53])\n                elif mean is not None:\n                    assert isinstance(mean, np.ndarray) and mean.shape[0] in [1, 3]\n                if mean is not None:\n                    image += mean\n\n                # swap RGB\n                image[:, :, (0, 1, 2)] = image[:, :, (2, 1, 0)]\n                if clip:\n                    image = np.maximum(0, np.minimum(255, image))\n                if color:\n                    color = color[::-1]\n                image = image.astype(np.uint8)\n                height, width, _ = image.shape\n                for i in range(label.shape[0]):\n                    x1 = int(label[i, 1] * width)\n                    if x1 < 0:\n                        continue\n                    y1 = int(label[i, 2] * height)\n                    x2 = int(label[i, 3] * width)\n                    y2 = int(label[i, 4] * height)\n                    bc = np.random.rand(3) * 255 if not color else color\n                    cv2.rectangle(image, (x1, y1), (x2, y2), bc, thickness)\n                    if id2labels is not None:\n                        cls_id = int(label[i, 0])\n                        if cls_id in id2labels:\n                            cls_name = id2labels[cls_id]\n                            text = \"{:s}\".format(cls_name)\n                            font = cv2.FONT_HERSHEY_SIMPLEX\n                            font_scale = 0.5\n                            text_height = cv2.getTextSize(text, font, font_scale, 2)[0][1]\n                            tc = (255, 255, 255)\n                            tpos = (x1 + 5, y1 + text_height + 5)\n                            cv2.putText(image, text, tpos, font, font_scale, tc, 2)\n                if waitKey is not None:\n                    cv2.imshow(window_name, image)\n                    cv2.waitKey(waitKey)\n                yield image\n        except StopIteration:\n            if not count:\n                return\n\n    def sync_label_shape(self, it, verbose=False):\n        \"\"\"Synchronize label shape with the input iterator. This is useful when\n        train/validation iterators have different label padding.\n\n        Parameters\n        ----------\n        it : ImageDetIter\n            The other iterator to synchronize\n        verbose : bool\n            Print verbose log if true\n\n        Returns\n        -------\n        ImageDetIter\n            The synchronized other iterator, the internal label shape is updated as well.\n\n        Examples\n        --------\n        >>> train_iter = mx.image.ImageDetIter(32, (3, 300, 300), path_imgrec='train.rec')\n        >>> val_iter = mx.image.ImageDetIter(32, (3, 300, 300), path.imgrec='val.rec')\n        >>> train_iter.label_shape\n        (30, 6)\n        >>> val_iter.label_shape\n        (25, 6)\n        >>> val_iter = train_iter.sync_label_shape(val_iter, verbose=False)\n        >>> train_iter.label_shape\n        (30, 6)\n        >>> val_iter.label_shape\n        (30, 6)\n        \"\"\"\n        assert isinstance(it, ImageDetIter), 'Synchronize with invalid iterator.'\n        train_label_shape = self.label_shape\n        val_label_shape = it.label_shape\n        assert train_label_shape[1] == val_label_shape[1], \"object width mismatch.\"\n        max_count = max(train_label_shape[0], val_label_shape[0])\n        if max_count > train_label_shape[0]:\n            self.reshape(None, (max_count, train_label_shape[1]))\n        if max_count > val_label_shape[0]:\n            it.reshape(None, (max_count, val_label_shape[1]))\n        if verbose and max_count > min(train_label_shape[0], val_label_shape[0]):\n            logging.info('Resized label_shape to (%d, %d).', max_count, train_label_shape[1])\n        return it\n"
  },
  {
    "path": "python/mxnet/image/image.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=no-member, too-many-lines, redefined-builtin, protected-access, unused-import, invalid-name\n# pylint: disable=too-many-arguments, too-many-locals, no-name-in-module, too-many-branches, too-many-statements\n\"\"\"Read individual image files and perform augmentations.\"\"\"\n\n\nimport sys\nimport os\nimport random\nimport logging\nimport json\nimport warnings\n\nfrom numbers import Number\n\nimport numpy as np\n\nfrom .. import numpy as _mx_np  # pylint: disable=reimported\n\n\ntry:\n    import cv2\nexcept ImportError:\n    cv2 = None\n\nfrom ..base import numeric_types\nfrom .. import ndarray as nd\nfrom ..ndarray import _internal\nfrom .. import io\nfrom .. import recordio\nfrom .. util import is_np_array\nfrom ..ndarray.numpy import _internal as _npi\n\n\ndef imread(filename, *args, **kwargs):\n    \"\"\"Read and decode an image to an NDArray.\n\n    .. note:: `imread` uses OpenCV (not the CV2 Python library).\n       MXNet must have been built with USE_OPENCV=1 for `imdecode` to work.\n\n    Parameters\n    ----------\n    filename : str\n        Name of the image file to be loaded.\n    flag : {0, 1}, default 1\n        1 for three channel color output. 0 for grayscale output.\n    to_rgb : bool, default True\n        True for RGB formatted output (MXNet default).\n        False for BGR formatted output (OpenCV default).\n    out : NDArray, optional\n        Output buffer. Use `None` for automatic allocation.\n\n    Returns\n    -------\n    NDArray\n        An `NDArray` containing the image.\n\n    Example\n    -------\n    >>> mx.img.imread(\"flower.jpg\")\n    <NDArray 224x224x3 @cpu(0)>\n\n    Set `flag` parameter to 0 to get grayscale output\n\n    >>> mx.img.imread(\"flower.jpg\", flag=0)\n    <NDArray 224x224x1 @cpu(0)>\n\n    Set `to_rgb` parameter to 0 to get output in OpenCV format (BGR)\n\n    >>> mx.img.imread(\"flower.jpg\", to_rgb=0)\n    <NDArray 224x224x3 @cpu(0)>\n    \"\"\"\n    if is_np_array():\n        read_fn = _npi.cvimread\n    else:\n        read_fn = _internal._cvimread\n    return read_fn(filename, *args, **kwargs)\n\n\ndef imresize(src, w, h, *args, **kwargs):\n    r\"\"\"Resize image with OpenCV.\n\n    .. note:: `imresize` uses OpenCV (not the CV2 Python library). MXNet must have been built\n       with USE_OPENCV=1 for `imresize` to work.\n\n    Parameters\n    ----------\n    src : NDArray\n        source image\n    w : int, required\n        Width of resized image.\n    h : int, required\n        Height of resized image.\n    interp : int, optional, default=1\n        Interpolation method (default=cv2.INTER_LINEAR).\n        Possible values:\n        0: Nearest Neighbors Interpolation.\n        1: Bilinear interpolation.\n        2: Bicubic interpolation over 4x4 pixel neighborhood.\n        3: Area-based (resampling using pixel area relation). It may be a\n        preferred method for image decimation, as it gives moire-free\n        results. But when the image is zoomed, it is similar to the Nearest\n        Neighbors method. (used by default).\n        4: Lanczos interpolation over 8x8 pixel neighborhood.\n        9: Cubic for enlarge, area for shrink, bilinear for others\n        10: Random select from interpolation method metioned above.\n        Note:\n        When shrinking an image, it will generally look best with AREA-based\n        interpolation, whereas, when enlarging an image, it will generally look best\n        with Bicubic (slow) or Bilinear (faster but still looks OK).\n        More details can be found in the documentation of OpenCV, please refer to\n        http://docs.opencv.org/master/da/d54/group__imgproc__transform.html.\n\n    out : NDArray, optional\n        The output NDArray to hold the result.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Example\n    -------\n    >>> with open(\"flower.jpeg\", 'rb') as fp:\n    ...     str_image = fp.read()\n    ...\n    >>> image = mx.img.imdecode(str_image)\n    >>> image\n    <NDArray 2321x3482x3 @cpu(0)>\n    >>> new_image = mx.img.resize(image, 240, 360)\n    >>> new_image\n    <NDArray 240x360x3 @cpu(0)>\n    \"\"\"\n    resize_fn = _npi.cvimresize if is_np_array() else _internal._cvimresize\n    return resize_fn(src, w, h, *args, **kwargs)\n\n\ndef imdecode(buf, *args, **kwargs):\n    \"\"\"Decode an image to an NDArray.\n\n    .. note:: `imdecode` uses OpenCV (not the CV2 Python library).\n       MXNet must have been built with USE_OPENCV=1 for `imdecode` to work.\n\n    Parameters\n    ----------\n    buf : str/bytes/bytearray or numpy.ndarray\n        Binary image data as string or numpy ndarray.\n    flag : int, optional, default=1\n        1 for three channel color output. 0 for grayscale output.\n    to_rgb : int, optional, default=1\n        1 for RGB formatted output (MXNet default). 0 for BGR formatted output (OpenCV default).\n    out : NDArray, optional\n        Output buffer. Use `None` for automatic allocation.\n\n    Returns\n    -------\n    NDArray\n        An `NDArray` containing the image.\n\n    Example\n    -------\n    >>> with open(\"flower.jpg\", 'rb') as fp:\n    ...     str_image = fp.read()\n    ...\n    >>> image = mx.img.imdecode(str_image)\n    >>> image\n    <NDArray 224x224x3 @cpu(0)>\n\n    Set `flag` parameter to 0 to get grayscale output\n\n    >>> with open(\"flower.jpg\", 'rb') as fp:\n    ...     str_image = fp.read()\n    ...\n    >>> image = mx.img.imdecode(str_image, flag=0)\n    >>> image\n    <NDArray 224x224x1 @cpu(0)>\n\n    Set `to_rgb` parameter to 0 to get output in OpenCV format (BGR)\n\n    >>> with open(\"flower.jpg\", 'rb') as fp:\n    ...     str_image = fp.read()\n    ...\n    >>> image = mx.img.imdecode(str_image, to_rgb=0)\n    >>> image\n    <NDArray 224x224x3 @cpu(0)>\n    \"\"\"\n    if not isinstance(buf, nd.NDArray):\n        if not isinstance(buf, (bytes, bytearray, np.ndarray)):\n            raise ValueError('buf must be of type bytes, bytearray or numpy.ndarray,'\n                             'if you would like to input type str, please convert to bytes')\n        array_fn = _mx_np.array if is_np_array() else nd.array\n        buf = array_fn(np.frombuffer(buf, dtype=np.uint8), dtype=np.uint8)\n\n    cvimdecode = _npi.cvimdecode if is_np_array() else _internal._cvimdecode\n    return cvimdecode(buf, *args, **kwargs)\n\n\ndef scale_down(src_size, size):\n    \"\"\"Scales down crop size if it's larger than image size.\n\n    If width/height of the crop is larger than the width/height of the image,\n    sets the width/height to the width/height of the image.\n\n    Parameters\n    ----------\n    src_size : tuple of int\n        Size of the image in (width, height) format.\n    size : tuple of int\n        Size of the crop in (width, height) format.\n\n    Returns\n    -------\n    tuple of int\n        A tuple containing the scaled crop size in (width, height) format.\n\n    Example\n    --------\n    >>> src_size = (640,480)\n    >>> size = (720,120)\n    >>> new_size = mx.img.scale_down(src_size, size)\n    >>> new_size\n    (640,106)\n    \"\"\"\n    w, h = size\n    sw, sh = src_size\n    if sh < h:\n        w, h = float(w * sh) / h, sh\n    if sw < w:\n        w, h = sw, float(h * sw) / w\n    return int(w), int(h)\n\n\ndef copyMakeBorder(src, top, bot, left, right, *args, **kwargs):\n    \"\"\"Pad image border with OpenCV.\n\n    Parameters\n    ----------\n    src : NDArray\n        source image\n    top : int, required\n        Top margin.\n    bot : int, required\n        Bottom margin.\n    left : int, required\n        Left margin.\n    right : int, required\n        Right margin.\n    type : int, optional, default='0'\n        Filling type (default=cv2.BORDER_CONSTANT).\n        0 - cv2.BORDER_CONSTANT - Adds a constant colored border.\n        1 - cv2.BORDER_REFLECT - Border will be mirror reflection of the\n        border elements, like this : fedcba|abcdefgh|hgfedcb\n        2 - cv2.BORDER_REFLECT_101 or cv.BORDER_DEFAULT - Same as above,\n        but with a slight change, like this : gfedcb|abcdefgh|gfedcba\n        3 - cv2.BORDER_REPLICATE - Last element is replicated throughout,\n        like this: aaaaaa|abcdefgh|hhhhhhh\n        4 - cv2.BORDER_WRAP - it will look like this : cdefgh|abcdefgh|abcdefg\n    value : double, optional, default=0\n        (Deprecated! Use ``values`` instead.) Fill with single value.\n    values : tuple of <double>, optional, default=[]\n        Fill with value(RGB[A] or gray), up to 4 channels.\n\n    out : NDArray, optional\n        The output NDArray to hold the result.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Example\n    --------\n    >>> with open(\"flower.jpeg\", 'rb') as fp:\n    ...     str_image = fp.read()\n    ...\n    >>> image = mx.img.imdecode(str_image)\n    >>> image\n    <NDArray 2321x3482x3 @cpu(0)>\n    >>> new_image = mx_border = mx.image.copyMakeBorder(mx_img, 1, 2, 3, 4, type=0)\n    >>> new_image\n    <NDArray 2324x3489x3 @cpu(0)>\n    \"\"\"\n    return _internal._cvcopyMakeBorder(src, top, bot, left, right, *args, **kwargs)\n\n\ndef _get_interp_method(interp, sizes=()):\n    \"\"\"Get the interpolation method for resize functions.\n    The major purpose of this function is to wrap a random interp method selection\n    and a auto-estimation method.\n\n    Parameters\n    ----------\n    interp : int\n        interpolation method for all resizing operations\n\n        Possible values:\n        0: Nearest Neighbors Interpolation.\n        1: Bilinear interpolation.\n        2: Bicubic interpolation over 4x4 pixel neighborhood.\n        3: Area-based (resampling using pixel area relation). It may be a\n        preferred method for image decimation, as it gives moire-free\n        results. But when the image is zoomed, it is similar to the Nearest\n        Neighbors method. (used by default).\n        4: Lanczos interpolation over 8x8 pixel neighborhood.\n        9: Cubic for enlarge, area for shrink, bilinear for others\n        10: Random select from interpolation method metioned above.\n        Note:\n        When shrinking an image, it will generally look best with AREA-based\n        interpolation, whereas, when enlarging an image, it will generally look best\n        with Bicubic (slow) or Bilinear (faster but still looks OK).\n        More details can be found in the documentation of OpenCV, please refer to\n        http://docs.opencv.org/master/da/d54/group__imgproc__transform.html.\n    sizes : tuple of int\n        (old_height, old_width, new_height, new_width), if None provided, auto(9)\n        will return Area(2) anyway.\n\n    Returns\n    -------\n    int\n        interp method from 0 to 4\n    \"\"\"\n    if interp == 9:\n        if sizes:\n            assert len(sizes) == 4\n            oh, ow, nh, nw = sizes\n            if nh > oh and nw > ow:\n                return 2\n            elif nh < oh and nw < ow:\n                return 3\n            else:\n                return 1\n        else:\n            return 2\n    if interp == 10:\n        return random.randint(0, 4)\n    if interp not in (0, 1, 2, 3, 4):\n        raise ValueError(f'Unknown interp method {interp}')\n    return interp\n\n\ndef resize_short(src, size, interp=2):\n    \"\"\"Resizes shorter edge to size.\n\n    .. note:: `resize_short` uses OpenCV (not the CV2 Python library).\n       MXNet must have been built with OpenCV for `resize_short` to work.\n\n    Resizes the original image by setting the shorter edge to size\n    and setting the longer edge accordingly.\n    Resizing function is called from OpenCV.\n\n    Parameters\n    ----------\n    src : NDArray\n        The original image.\n    size : int\n        The length to be set for the shorter edge.\n    interp : int, optional, default=2\n        Interpolation method used for resizing the image.\n        Possible values:\n        0: Nearest Neighbors Interpolation.\n        1: Bilinear interpolation.\n        2: Bicubic interpolation over 4x4 pixel neighborhood.\n        3: Area-based (resampling using pixel area relation). It may be a\n        preferred method for image decimation, as it gives moire-free\n        results. But when the image is zoomed, it is similar to the Nearest\n        Neighbors method. (used by default).\n        4: Lanczos interpolation over 8x8 pixel neighborhood.\n        9: Cubic for enlarge, area for shrink, bilinear for others\n        10: Random select from interpolation method metioned above.\n        Note:\n        When shrinking an image, it will generally look best with AREA-based\n        interpolation, whereas, when enlarging an image, it will generally look best\n        with Bicubic (slow) or Bilinear (faster but still looks OK).\n        More details can be found in the documentation of OpenCV, please refer to\n        http://docs.opencv.org/master/da/d54/group__imgproc__transform.html.\n\n    Returns\n    -------\n    NDArray\n        An 'NDArray' containing the resized image.\n\n    Example\n    -------\n    >>> with open(\"flower.jpeg\", 'rb') as fp:\n    ...     str_image = fp.read()\n    ...\n    >>> image = mx.img.imdecode(str_image)\n    >>> image\n    <NDArray 2321x3482x3 @cpu(0)>\n    >>> size = 640\n    >>> new_image = mx.img.resize_short(image, size)\n    >>> new_image\n    <NDArray 2321x3482x3 @cpu(0)>\n    \"\"\"\n    h, w, _ = src.shape\n    if h > w:\n        new_h, new_w = size * h // w, size\n    else:\n        new_h, new_w = size, size * w // h\n    return imresize(src, new_w, new_h, interp=_get_interp_method(interp, (h, w, new_h, new_w)))\n\n\ndef fixed_crop(src, x0, y0, w, h, size=None, interp=2):\n    \"\"\"Crop src at fixed location, and (optionally) resize it to size.\n\n    Parameters\n    ----------\n    src : NDArray\n        Input image\n    x0 : int\n        Left boundary of the cropping area\n    y0 : int\n        Top boundary of the cropping area\n    w : int\n        Width of the cropping area\n    h : int\n        Height of the cropping area\n    size : tuple of (w, h)\n        Optional, resize to new size after cropping\n    interp : int, optional, default=2\n        Interpolation method. See resize_short for details.\n\n    Returns\n    -------\n    NDArray\n        An `NDArray` containing the cropped image.\n    \"\"\"\n    out = src[y0:y0+h, x0:x0+w]\n    if size is not None and (w, h) != size:\n        sizes = (h, w, size[1], size[0])\n        out = imresize(out, *size, interp=_get_interp_method(interp, sizes))\n    return out\n\n\ndef random_crop(src, size, interp=2):\n    \"\"\"Randomly crop `src` with `size` (width, height).\n    Upsample result if `src` is smaller than `size`.\n\n    Parameters\n    ----------\n    src: Source image `NDArray`\n    size: Size of the crop formatted as (width, height). If the `size` is larger\n           than the image, then the source image is upsampled to `size` and returned.\n    interp: int, optional, default=2\n        Interpolation method. See resize_short for details.\n    Returns\n    -------\n    NDArray\n        An `NDArray` containing the cropped image.\n    Tuple\n        A tuple (x, y, width, height) where (x, y) is top-left position of the crop in the\n        original image and (width, height) are the dimensions of the cropped image.\n\n    Example\n    -------\n    >>> im = mx.nd.array(cv2.imread(\"flower.jpg\"))\n    >>> cropped_im, rect  = mx.image.random_crop(im, (100, 100))\n    >>> print cropped_im\n    <NDArray 100x100x1 @cpu(0)>\n    >>> print rect\n    (20, 21, 100, 100)\n    \"\"\"\n\n    h, w, _ = src.shape\n    new_w, new_h = scale_down((w, h), size)\n\n    x0 = random.randint(0, w - new_w)\n    y0 = random.randint(0, h - new_h)\n\n    out = fixed_crop(src, x0, y0, new_w, new_h, size, interp)\n    return out, (x0, y0, new_w, new_h)\n\n\ndef center_crop(src, size, interp=2):\n    \"\"\"Crops the image `src` to the given `size` by trimming on all four\n    sides and preserving the center of the image. Upsamples if `src` is smaller\n    than `size`.\n\n    .. note:: This requires MXNet to be compiled with USE_OPENCV.\n\n    Parameters\n    ----------\n    src : NDArray\n        Binary source image data.\n    size : list or tuple of int\n        The desired output image size.\n    interp : int, optional, default=2\n        Interpolation method. See resize_short for details.\n\n    Returns\n    -------\n    NDArray\n        The cropped image.\n    Tuple\n        (x, y, width, height) where x, y are the positions of the crop in the\n        original image and width, height the dimensions of the crop.\n\n    Example\n    -------\n    >>> with open(\"flower.jpg\", 'rb') as fp:\n    ...     str_image = fp.read()\n    ...\n    >>> image = mx.image.imdecode(str_image)\n    >>> image\n    <NDArray 2321x3482x3 @cpu(0)>\n    >>> cropped_image, (x, y, width, height) = mx.image.center_crop(image, (1000, 500))\n    >>> cropped_image\n    <NDArray 500x1000x3 @cpu(0)>\n    >>> x, y, width, height\n    (1241, 910, 1000, 500)\n    \"\"\"\n\n    h, w, _ = src.shape\n    new_w, new_h = scale_down((w, h), size)\n\n    x0 = int((w - new_w) / 2)\n    y0 = int((h - new_h) / 2)\n\n    out = fixed_crop(src, x0, y0, new_w, new_h, size, interp)\n    return out, (x0, y0, new_w, new_h)\n\n\ndef color_normalize(src, mean, std=None):\n    \"\"\"Normalize src with mean and std.\n\n    Parameters\n    ----------\n    src : NDArray\n        Input image\n    mean : NDArray\n        RGB mean to be subtracted\n    std : NDArray\n        RGB standard deviation to be divided\n\n    Returns\n    -------\n    NDArray\n        An `NDArray` containing the normalized image.\n    \"\"\"\n    if mean is not None:\n        src -= mean\n    if std is not None:\n        src /= std\n    return src\n\n\ndef random_size_crop(src, size, area, ratio, interp=2, **kwargs):\n    \"\"\"Randomly crop src with size. Randomize area and aspect ratio.\n\n    Parameters\n    ----------\n    src : NDArray\n        Input image\n    size : tuple of (int, int)\n        Size of the crop formatted as (width, height).\n    area : float in (0, 1] or tuple of (float, float)\n        If tuple, minimum area and maximum area to be maintained after cropping\n        If float, minimum area to be maintained after cropping, maximum area is set to 1.0\n    ratio : tuple of (float, float)\n        Aspect ratio range as (min_aspect_ratio, max_aspect_ratio)\n    interp: int, optional, default=2\n        Interpolation method. See resize_short for details.\n    Returns\n    -------\n    NDArray\n        An `NDArray` containing the cropped image.\n    Tuple\n        A tuple (x, y, width, height) where (x, y) is top-left position of the crop in the\n        original image and (width, height) are the dimensions of the cropped image.\n\n    \"\"\"\n    h, w, _ = src.shape\n    src_area = h * w\n\n    if 'min_area' in kwargs:\n        warnings.warn('`min_area` is deprecated. Please use `area` instead.',\n                      DeprecationWarning)\n        area = kwargs.pop('min_area')\n    assert not kwargs, \"unexpected keyword arguments for `random_size_crop`.\"\n\n    if isinstance(area, numeric_types):\n        area = (area, 1.0)\n    for _ in range(10):\n        target_area = random.uniform(area[0], area[1]) * src_area\n        log_ratio = (np.log(ratio[0]), np.log(ratio[1]))\n        new_ratio = np.exp(random.uniform(*log_ratio))\n\n        new_w = int(round(np.sqrt(target_area * new_ratio)))\n        new_h = int(round(np.sqrt(target_area / new_ratio)))\n\n        if new_w <= w and new_h <= h:\n            x0 = random.randint(0, w - new_w)\n            y0 = random.randint(0, h - new_h)\n\n            out = fixed_crop(src, x0, y0, new_w, new_h, size, interp)\n            return out, (x0, y0, new_w, new_h)\n\n    # fall back to center_crop\n    return center_crop(src, size, interp)\n\n\ndef imrotate(src, rotation_degrees, zoom_in=False, zoom_out=False):\n    \"\"\"Rotates the input image(s) of a specific rotation degree.\n\n    Parameters\n    ----------\n    src : NDArray\n        Input image (format CHW) or batch of images (format NCHW),\n        in both case is required a float32 data type.\n    rotation_degrees: scalar or NDArray\n        Wanted rotation in degrees. In case of `src` being a single image\n        a scalar is needed, otherwise a mono-dimensional vector of angles\n        or a scalar.\n    zoom_in: bool\n        If True input image(s) will be zoomed in a way so that no padding\n        will be shown in the output result.\n    zoom_out: bool\n        If True input image(s) will be zoomed in a way so that the whole\n        original image will be contained in the output result.\n    Returns\n    -------\n    NDArray\n        An `NDArray` containing the rotated image(s).\n    \"\"\"\n    if zoom_in and zoom_out:\n        raise ValueError(\"`zoom_in` and `zoom_out` cannot be both True\")\n    if np.dtype(src.dtype) is not np.dtype(np.float32):\n        raise TypeError(\"Only `float32` images are supported by this function\")\n    # handles the case in which a single image is passed to this function\n    expanded = False\n    if src.ndim == 3:\n        expanded = True\n        src = _mx_np.expand_dims(src, 0) if is_np_array() else src.expand_dims(axis=0)\n        if not isinstance(rotation_degrees, Number):\n            raise TypeError(\"When a single image is passed the rotation angle is \"\n                            \"required to be a scalar.\")\n    elif src.ndim != 4:\n        raise ValueError(\"Only 3D and 4D are supported by this function\")\n\n    # when a scalar is passed we wrap it into an array\n    if isinstance(rotation_degrees, Number):\n        rotation_degrees = nd.array([rotation_degrees] * len(src),\n                                    ctx=src.ctx)\n\n    if len(src) != len(rotation_degrees):\n        raise ValueError(\n            \"The number of images must be equal to the number of rotation angles\"\n        )\n\n    rotation_degrees = rotation_degrees.as_in_context(src.ctx)\n    rotation_rad = np.pi * rotation_degrees / 180\n    # reshape the rotations angle in order to be broadcasted\n    # over the `src` tensor\n    rotation_rad = rotation_rad.expand_dims(axis=1).expand_dims(axis=2)\n    _, _, h, w = src.shape\n\n    # Generate a grid centered at the center of the image\n    hscale = (float(h - 1) / 2)\n    wscale = (float(w - 1) / 2)\n    h_matrix = (\n        nd.repeat(nd.arange(h, ctx=src.ctx).astype('float32').reshape(h, 1), w, axis=1) - hscale\n    ).expand_dims(axis=0)\n    w_matrix = (\n        nd.repeat(nd.arange(w, ctx=src.ctx).astype('float32').reshape(1, w), h, axis=0) - wscale\n    ).expand_dims(axis=0)\n    # perform rotation on the grid\n    c_alpha = nd.cos(rotation_rad)\n    s_alpha = nd.sin(rotation_rad)\n    w_matrix_rot = w_matrix * c_alpha - h_matrix * s_alpha\n    h_matrix_rot = w_matrix * s_alpha + h_matrix * c_alpha\n    # NOTE: grid normalization must be performed after the rotation\n    #       to keep the aspec ratio\n    w_matrix_rot = w_matrix_rot / wscale\n    h_matrix_rot = h_matrix_rot / hscale\n\n    h, w = nd.array([h], ctx=src.ctx), nd.array([w], ctx=src.ctx)\n    # compute the scale factor in case `zoom_in` or `zoom_out` are True\n    if zoom_in or zoom_out:\n        rho_corner = nd.sqrt(h * h + w * w)\n        ang_corner = nd.arctan(h / w)\n        corner1_x_pos = nd.abs(rho_corner * nd.cos(ang_corner + nd.abs(rotation_rad)))\n        corner1_y_pos = nd.abs(rho_corner * nd.sin(ang_corner + nd.abs(rotation_rad)))\n        corner2_x_pos = nd.abs(rho_corner * nd.cos(ang_corner - nd.abs(rotation_rad)))\n        corner2_y_pos = nd.abs(rho_corner * nd.sin(ang_corner - nd.abs(rotation_rad)))\n        max_x = nd.maximum(corner1_x_pos, corner2_x_pos)\n        max_y = nd.maximum(corner1_y_pos, corner2_y_pos)\n        if zoom_out:\n            scale_x = max_x / w\n            scale_y = max_y / h\n            globalscale = nd.maximum(scale_x, scale_y)\n        else:\n            scale_x = w / max_x\n            scale_y = h / max_y\n            globalscale = nd.minimum(scale_x, scale_y)\n        globalscale = globalscale.expand_dims(axis=3)\n    else:\n        globalscale = 1\n    grid = nd.concat(w_matrix_rot.expand_dims(axis=1),\n                     h_matrix_rot.expand_dims(axis=1), dim=1)\n    grid = grid * globalscale\n    if is_np_array():\n        src = src.as_nd_ndarray()\n    rot_img = nd.BilinearSampler(src, grid)\n    if is_np_array():\n        rot_img = rot_img.as_np_ndarray()\n    if expanded:\n        return rot_img[0]\n    return rot_img\n\n\ndef random_rotate(src, angle_limits, zoom_in=False, zoom_out=False):\n    \"\"\"Random rotates `src` by an angle included in angle limits.\n\n    Parameters\n    ----------\n    src : NDArray\n        Input image (format CHW) or batch of images (format NCHW),\n        in both case is required a float32 data type.\n    angle_limits: tuple\n        Tuple of 2 elements containing the upper and lower limit\n        for rotation angles in degree.\n    zoom_in: bool\n        If True input image(s) will be zoomed in a way so that no padding\n        will be shown in the output result.\n    zoom_out: bool\n        If True input image(s) will be zoomed in a way so that the whole\n        original image will be contained in the output result.\n    Returns\n    -------\n    NDArray\n        An `NDArray` containing the rotated image(s).\n    \"\"\"\n    if src.ndim == 3:\n        rotation_degrees = np.random.uniform(*angle_limits)\n    else:\n        n = src.shape[0]\n        rotation_degrees = nd.array(np.random.uniform(\n            *angle_limits,\n            size=n\n        ))\n    return imrotate(src, rotation_degrees,\n                    zoom_in=zoom_in, zoom_out=zoom_out)\n\n\nclass Augmenter(object):\n    \"\"\"Image Augmenter base class\"\"\"\n    def __init__(self, **kwargs):\n        self._kwargs = kwargs\n        for k, v in self._kwargs.items():\n            if isinstance(v, nd.NDArray):\n                v = v.asnumpy()\n            if isinstance(v, np.ndarray):\n                v = v.tolist()\n                self._kwargs[k] = v\n\n    def dumps(self):\n        \"\"\"Saves the Augmenter to string\n\n        Returns\n        -------\n        str\n            JSON formatted string that describes the Augmenter.\n        \"\"\"\n        return json.dumps([self.__class__.__name__.lower(), self._kwargs])\n\n    def __call__(self, src):\n        \"\"\"Abstract implementation body\"\"\"\n        raise NotImplementedError(\"Must override implementation.\")\n\n\nclass SequentialAug(Augmenter):\n    \"\"\"Composing a sequential augmenter list.\n\n    Parameters\n    ----------\n    ts : list of augmenters\n        A series of augmenters to be applied in sequential order.\n    \"\"\"\n    def __init__(self, ts):\n        super(SequentialAug, self).__init__()\n        self.ts = ts\n\n    def dumps(self):\n        \"\"\"Override the default to avoid duplicate dump.\"\"\"\n        return [self.__class__.__name__.lower(), [x.dumps() for x in self.ts]]\n\n    def __call__(self, src):\n        \"\"\"Augmenter body\"\"\"\n        for aug in self.ts:\n            src = aug(src)\n        return src\n\n\nclass ResizeAug(Augmenter):\n    \"\"\"Make resize shorter edge to size augmenter.\n\n    Parameters\n    ----------\n    size : int\n        The length to be set for the shorter edge.\n    interp : int, optional, default=2\n        Interpolation method. See resize_short for details.\n    \"\"\"\n    def __init__(self, size, interp=2):\n        super(ResizeAug, self).__init__(size=size, interp=interp)\n        self.size = size\n        self.interp = interp\n\n    def __call__(self, src):\n        \"\"\"Augmenter body\"\"\"\n        return resize_short(src, self.size, self.interp)\n\n\nclass ForceResizeAug(Augmenter):\n    \"\"\"Force resize to size regardless of aspect ratio\n\n    Parameters\n    ----------\n    size : tuple of (int, int)\n        The desired size as in (width, height)\n    interp : int, optional, default=2\n        Interpolation method. See resize_short for details.\n    \"\"\"\n    def __init__(self, size, interp=2):\n        super(ForceResizeAug, self).__init__(size=size, interp=interp)\n        self.size = size\n        self.interp = interp\n\n    def __call__(self, src):\n        \"\"\"Augmenter body\"\"\"\n        sizes = (src.shape[0], src.shape[1], self.size[1], self.size[0])\n        return imresize(src, *self.size, interp=_get_interp_method(self.interp, sizes))\n\n\nclass RandomCropAug(Augmenter):\n    \"\"\"Make random crop augmenter\n\n    Parameters\n    ----------\n    size : int\n        The length to be set for the shorter edge.\n    interp : int, optional, default=2\n        Interpolation method. See resize_short for details.\n    \"\"\"\n    def __init__(self, size, interp=2):\n        super(RandomCropAug, self).__init__(size=size, interp=interp)\n        self.size = size\n        self.interp = interp\n\n    def __call__(self, src):\n        \"\"\"Augmenter body\"\"\"\n        return random_crop(src, self.size, self.interp)[0]\n\n\nclass RandomSizedCropAug(Augmenter):\n    \"\"\"Make random crop with random resizing and random aspect ratio jitter augmenter.\n\n    Parameters\n    ----------\n    size : tuple of (int, int)\n        Size of the crop formatted as (width, height).\n    area : float in (0, 1] or tuple of (float, float)\n        If tuple, minimum area and maximum area to be maintained after cropping\n        If float, minimum area to be maintained after cropping, maximum area is set to 1.0\n    ratio : tuple of (float, float)\n        Aspect ratio range as (min_aspect_ratio, max_aspect_ratio)\n    interp: int, optional, default=2\n        Interpolation method. See resize_short for details.\n    \"\"\"\n    def __init__(self, size, area, ratio, interp=2, **kwargs):\n        super(RandomSizedCropAug, self).__init__(size=size, area=area,\n                                                 ratio=ratio, interp=interp)\n        self.size = size\n        if 'min_area' in kwargs:\n            warnings.warn('`min_area` is deprecated. Please use `area` instead.',\n                          DeprecationWarning)\n            self.area = kwargs.pop('min_area')\n        else:\n            self.area = area\n        self.ratio = ratio\n        self.interp = interp\n        assert not kwargs, \"unexpected keyword arguments for `RandomSizedCropAug`.\"\n\n    def __call__(self, src):\n        \"\"\"Augmenter body\"\"\"\n        return random_size_crop(src, self.size, self.area, self.ratio, self.interp)[0]\n\n\nclass CenterCropAug(Augmenter):\n    \"\"\"Make center crop augmenter.\n\n    Parameters\n    ----------\n    size : list or tuple of int\n        The desired output image size.\n    interp : int, optional, default=2\n        Interpolation method. See resize_short for details.\n    \"\"\"\n    def __init__(self, size, interp=2):\n        super(CenterCropAug, self).__init__(size=size, interp=interp)\n        self.size = size\n        self.interp = interp\n\n    def __call__(self, src):\n        \"\"\"Augmenter body\"\"\"\n        return center_crop(src, self.size, self.interp)[0]\n\n\nclass RandomOrderAug(Augmenter):\n    \"\"\"Apply list of augmenters in random order\n\n    Parameters\n    ----------\n    ts : list of augmenters\n        A series of augmenters to be applied in random order\n    \"\"\"\n    def __init__(self, ts):\n        super(RandomOrderAug, self).__init__()\n        self.ts = ts\n\n    def dumps(self):\n        \"\"\"Override the default to avoid duplicate dump.\"\"\"\n        return [self.__class__.__name__.lower(), [x.dumps() for x in self.ts]]\n\n    def __call__(self, src):\n        \"\"\"Augmenter body\"\"\"\n        random.shuffle(self.ts)\n        for t in self.ts:\n            src = t(src)\n        return src\n\n\nclass BrightnessJitterAug(Augmenter):\n    \"\"\"Random brightness jitter augmentation.\n\n    Parameters\n    ----------\n    brightness : float\n        The brightness jitter ratio range, [0, 1]\n    \"\"\"\n    def __init__(self, brightness):\n        super(BrightnessJitterAug, self).__init__(brightness=brightness)\n        self.brightness = brightness\n\n    def __call__(self, src):\n        \"\"\"Augmenter body\"\"\"\n        alpha = 1.0 + random.uniform(-self.brightness, self.brightness)\n        src *= alpha\n        return src\n\n\nclass ContrastJitterAug(Augmenter):\n    \"\"\"Random contrast jitter augmentation.\n\n    Parameters\n    ----------\n    contrast : float\n        The contrast jitter ratio range, [0, 1]\n    \"\"\"\n    def __init__(self, contrast):\n        super(ContrastJitterAug, self).__init__(contrast=contrast)\n        self.contrast = contrast\n        self.coef = nd.array([[[0.299, 0.587, 0.114]]])\n\n    def __call__(self, src):\n        \"\"\"Augmenter body\"\"\"\n        alpha = 1.0 + random.uniform(-self.contrast, self.contrast)\n        gray = src * self.coef\n        gray = (3.0 * (1.0 - alpha) / gray.size) * nd.sum(gray)\n        src *= alpha\n        src += gray\n        return src\n\n\nclass SaturationJitterAug(Augmenter):\n    \"\"\"Random saturation jitter augmentation.\n\n    Parameters\n    ----------\n    saturation : float\n        The saturation jitter ratio range, [0, 1]\n    \"\"\"\n    def __init__(self, saturation):\n        super(SaturationJitterAug, self).__init__(saturation=saturation)\n        self.saturation = saturation\n        self.coef = nd.array([[[0.299, 0.587, 0.114]]])\n\n    def __call__(self, src):\n        \"\"\"Augmenter body\"\"\"\n        alpha = 1.0 + random.uniform(-self.saturation, self.saturation)\n        gray = src * self.coef\n        gray = nd.sum(gray, axis=2, keepdims=True)\n        gray *= (1.0 - alpha)\n        src *= alpha\n        src += gray\n        return src\n\n\nclass HueJitterAug(Augmenter):\n    \"\"\"Random hue jitter augmentation.\n\n    Parameters\n    ----------\n    hue : float\n        The hue jitter ratio range, [0, 1]\n    \"\"\"\n    def __init__(self, hue):\n        super(HueJitterAug, self).__init__(hue=hue)\n        self.hue = hue\n        self.tyiq = np.array([[0.299, 0.587, 0.114],\n                              [0.596, -0.274, -0.321],\n                              [0.211, -0.523, 0.311]])\n        self.ityiq = np.array([[1.0, 0.956, 0.621],\n                               [1.0, -0.272, -0.647],\n                               [1.0, -1.107, 1.705]])\n\n    def __call__(self, src):\n        \"\"\"Augmenter body.\n        Using approximate linear transfomation described in:\n        https://beesbuzz.biz/code/hsv_color_transforms.php\n        \"\"\"\n        alpha = random.uniform(-self.hue, self.hue)\n        u = np.cos(alpha * np.pi)\n        w = np.sin(alpha * np.pi)\n        bt = np.array([[1.0, 0.0, 0.0],\n                       [0.0, u, -w],\n                       [0.0, w, u]])\n        t = np.dot(np.dot(self.ityiq, bt), self.tyiq).T\n        src = nd.dot(src, nd.array(t))\n        return src\n\n\nclass ColorJitterAug(RandomOrderAug):\n    \"\"\"Apply random brightness, contrast and saturation jitter in random order.\n\n    Parameters\n    ----------\n    brightness : float\n        The brightness jitter ratio range, [0, 1]\n    contrast : float\n        The contrast jitter ratio range, [0, 1]\n    saturation : float\n        The saturation jitter ratio range, [0, 1]\n    \"\"\"\n    def __init__(self, brightness, contrast, saturation):\n        ts = []\n        if brightness > 0:\n            ts.append(BrightnessJitterAug(brightness))\n        if contrast > 0:\n            ts.append(ContrastJitterAug(contrast))\n        if saturation > 0:\n            ts.append(SaturationJitterAug(saturation))\n        super(ColorJitterAug, self).__init__(ts)\n\n\nclass LightingAug(Augmenter):\n    \"\"\"Add PCA based noise.\n\n    Parameters\n    ----------\n    alphastd : float\n        Noise level\n    eigval : 3x1 np.array\n        Eigen values\n    eigvec : 3x3 np.array\n        Eigen vectors\n    \"\"\"\n    def __init__(self, alphastd, eigval, eigvec):\n        super(LightingAug, self).__init__(alphastd=alphastd, eigval=eigval, eigvec=eigvec)\n        self.alphastd = alphastd\n        self.eigval = eigval\n        self.eigvec = eigvec\n\n    def __call__(self, src):\n        \"\"\"Augmenter body\"\"\"\n        alpha = np.random.normal(0, self.alphastd, size=(3,))\n        rgb = np.dot(self.eigvec * alpha, self.eigval)\n        src += nd.array(rgb)\n        return src\n\n\nclass ColorNormalizeAug(Augmenter):\n    \"\"\"Mean and std normalization.\n\n    Parameters\n    ----------\n    mean : NDArray\n        RGB mean to be subtracted\n    std : NDArray\n        RGB standard deviation to be divided\n    \"\"\"\n    def __init__(self, mean, std):\n        super(ColorNormalizeAug, self).__init__(mean=mean, std=std)\n        self.mean = mean if mean is None or isinstance(mean, nd.NDArray) else nd.array(mean)\n        self.std = std if std is None or isinstance(std, nd.NDArray) else nd.array(std)\n\n    def __call__(self, src):\n        \"\"\"Augmenter body\"\"\"\n        return color_normalize(src, self.mean, self.std)\n\n\nclass RandomGrayAug(Augmenter):\n    \"\"\"Randomly convert to gray image.\n\n    Parameters\n    ----------\n    p : float\n        Probability to convert to grayscale\n    \"\"\"\n    def __init__(self, p):\n        super(RandomGrayAug, self).__init__(p=p)\n        self.p = p\n        self.mat = nd.array([[0.21, 0.21, 0.21],\n                             [0.72, 0.72, 0.72],\n                             [0.07, 0.07, 0.07]])\n\n    def __call__(self, src):\n        \"\"\"Augmenter body\"\"\"\n        if random.random() < self.p:\n            src = nd.dot(src, self.mat)\n        return src\n\n\nclass HorizontalFlipAug(Augmenter):\n    \"\"\"Random horizontal flip.\n\n    Parameters\n    ----------\n    p : float\n        Probability to flip image horizontally\n    \"\"\"\n    def __init__(self, p):\n        super(HorizontalFlipAug, self).__init__(p=p)\n        self.p = p\n\n    def __call__(self, src):\n        \"\"\"Augmenter body\"\"\"\n        if random.random() < self.p:\n            src = nd.flip(src, axis=1)\n        return src\n\n\nclass CastAug(Augmenter):\n    \"\"\"Cast to float32\"\"\"\n    def __init__(self, typ='float32'):\n        super(CastAug, self).__init__(type=typ)\n        self.typ = typ\n\n    def __call__(self, src):\n        \"\"\"Augmenter body\"\"\"\n        src = src.astype(self.typ)\n        return src\n\n\ndef CreateAugmenter(data_shape, resize=0, rand_crop=False, rand_resize=False, rand_mirror=False,\n                    mean=None, std=None, brightness=0, contrast=0, saturation=0, hue=0,\n                    pca_noise=0, rand_gray=0, inter_method=2):\n    \"\"\"Creates an augmenter list.\n\n    Parameters\n    ----------\n    data_shape : tuple of int\n        Shape for output data\n    resize : int\n        Resize shorter edge if larger than 0 at the begining\n    rand_crop : bool\n        Whether to enable random cropping other than center crop\n    rand_resize : bool\n        Whether to enable random sized cropping, require rand_crop to be enabled\n    rand_gray : float\n        [0, 1], probability to convert to grayscale for all channels, the number\n        of channels will not be reduced to 1\n    rand_mirror : bool\n        Whether to apply horizontal flip to image with probability 0.5\n    mean : np.ndarray or None\n        Mean pixel values for [r, g, b]\n    std : np.ndarray or None\n        Standard deviations for [r, g, b]\n    brightness : float\n        Brightness jittering range (percent)\n    contrast : float\n        Contrast jittering range (percent)\n    saturation : float\n        Saturation jittering range (percent)\n    hue : float\n        Hue jittering range (percent)\n    pca_noise : float\n        Pca noise level (percent)\n    inter_method : int, default=2(Area-based)\n        Interpolation method for all resizing operations\n\n        Possible values:\n        0: Nearest Neighbors Interpolation.\n        1: Bilinear interpolation.\n        2: Bicubic interpolation over 4x4 pixel neighborhood.\n        3: Area-based (resampling using pixel area relation). It may be a\n        preferred method for image decimation, as it gives moire-free\n        results. But when the image is zoomed, it is similar to the Nearest\n        Neighbors method. (used by default).\n        4: Lanczos interpolation over 8x8 pixel neighborhood.\n        9: Cubic for enlarge, area for shrink, bilinear for others\n        10: Random select from interpolation method metioned above.\n        Note:\n        When shrinking an image, it will generally look best with AREA-based\n        interpolation, whereas, when enlarging an image, it will generally look best\n        with Bicubic (slow) or Bilinear (faster but still looks OK).\n\n    Examples\n    --------\n    >>> # An example of creating multiple augmenters\n    >>> augs = mx.image.CreateAugmenter(data_shape=(3, 300, 300), rand_mirror=True,\n    ...    mean=True, brightness=0.125, contrast=0.125, rand_gray=0.05,\n    ...    saturation=0.125, pca_noise=0.05, inter_method=10)\n    >>> # dump the details\n    >>> for aug in augs:\n    ...    aug.dumps()\n    \"\"\"\n    auglist = []\n\n    if resize > 0:\n        auglist.append(ResizeAug(resize, inter_method))\n\n    crop_size = (data_shape[2], data_shape[1])\n    if rand_resize:\n        assert rand_crop\n        auglist.append(RandomSizedCropAug(crop_size, 0.08, (3.0 / 4.0, 4.0 / 3.0), inter_method))\n    elif rand_crop:\n        auglist.append(RandomCropAug(crop_size, inter_method))\n    else:\n        auglist.append(CenterCropAug(crop_size, inter_method))\n\n    if rand_mirror:\n        auglist.append(HorizontalFlipAug(0.5))\n\n    auglist.append(CastAug())\n\n    if brightness or contrast or saturation:\n        auglist.append(ColorJitterAug(brightness, contrast, saturation))\n\n    if hue:\n        auglist.append(HueJitterAug(hue))\n\n    if pca_noise > 0:\n        eigval = np.array([55.46, 4.794, 1.148])\n        eigvec = np.array([[-0.5675, 0.7192, 0.4009],\n                           [-0.5808, -0.0045, -0.8140],\n                           [-0.5836, -0.6948, 0.4203]])\n        auglist.append(LightingAug(pca_noise, eigval, eigvec))\n\n    if rand_gray > 0:\n        auglist.append(RandomGrayAug(rand_gray))\n\n    if mean is True:\n        mean = nd.array([123.68, 116.28, 103.53])\n    elif mean is not None:\n        assert isinstance(mean, (np.ndarray, nd.NDArray)) and mean.shape[0] in [1, 3]\n\n    if std is True:\n        std = nd.array([58.395, 57.12, 57.375])\n    elif std is not None:\n        assert isinstance(std, (np.ndarray, nd.NDArray)) and std.shape[0] in [1, 3]\n\n    if mean is not None or std is not None:\n        auglist.append(ColorNormalizeAug(mean, std))\n\n    return auglist\n\n\nclass ImageIter(io.DataIter):\n    \"\"\"Image data iterator with a large number of augmentation choices.\n    This iterator supports reading from both .rec files and raw image files.\n\n    To load input images from .rec files, use `path_imgrec` parameter and to load from raw image\n    files, use `path_imglist` and `path_root` parameters.\n\n    To use data partition (for distributed training) or shuffling, specify `path_imgidx` parameter.\n\n    Parameters\n    ----------\n    batch_size : int\n        Number of examples per batch.\n    data_shape : tuple\n        Data shape in (channels, height, width) format.\n        For now, only RGB image with 3 channels is supported.\n    label_width : int, optional\n        Number of labels per example. The default label width is 1.\n    path_imgrec : str\n        Path to image record file (.rec).\n        Created with tools/im2rec.py or bin/im2rec.\n    path_imglist : str\n        Path to image list (.lst).\n        Created with tools/im2rec.py or with custom script.\n        Format: Tab separated record of index, one or more labels and relative_path_from_root.\n    imglist: list\n        A list of images with the label(s).\n        Each item is a list [imagelabel: float or list of float, imgpath].\n    path_root : str\n        Root folder of image files.\n    path_imgidx : str\n        Path to image index file. Needed for partition and shuffling when using .rec source.\n    shuffle : bool\n        Whether to shuffle all images at the start of each iteration or not.\n        Can be slow for HDD.\n    part_index : int\n        Partition index.\n    num_parts : int\n        Total number of partitions.\n    data_name : str\n        Data name for provided symbols.\n    label_name : str\n        Label name for provided symbols.\n    dtype : str\n        Label data type. Default: float32. Other options: int32, int64, float64\n    last_batch_handle : str, optional\n        How to handle the last batch.\n        This parameter can be 'pad'(default), 'discard' or 'roll_over'.\n        If 'pad', the last batch will be padded with data starting from the begining\n        If 'discard', the last batch will be discarded\n        If 'roll_over', the remaining elements will be rolled over to the next iteration\n    kwargs : ...\n        More arguments for creating augmenter. See mx.image.CreateAugmenter.\n    \"\"\"\n\n    def __init__(self, batch_size, data_shape, label_width=1,\n                 path_imgrec=None, path_imglist=None, path_root=None, path_imgidx=None,\n                 shuffle=False, part_index=0, num_parts=1, aug_list=None, imglist=None,\n                 data_name='data', label_name='softmax_label', dtype='float32',\n                 last_batch_handle='pad', **kwargs):\n        super(ImageIter, self).__init__()\n        assert path_imgrec or path_imglist or (isinstance(imglist, list))\n        assert dtype in ['int32', 'float32', 'int64', 'float64'], dtype + ' label not supported'\n        num_threads = os.environ.get('MXNET_CPU_WORKER_NTHREADS', 1)\n        logging.info('Using %s threads for decoding...', str(num_threads))\n        logging.info('Set enviroment variable MXNET_CPU_WORKER_NTHREADS to a'\n                     ' larger number to use more threads.')\n        class_name = self.__class__.__name__\n        if path_imgrec:\n            logging.info('%s: loading recordio %s...',\n                         class_name, path_imgrec)\n            if path_imgidx:\n                self.imgrec = recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, 'r')\n                self.imgidx = list(self.imgrec.keys)\n            else:\n                self.imgrec = recordio.MXRecordIO(path_imgrec, 'r')\n                self.imgidx = None\n        else:\n            self.imgrec = None\n\n        array_fn = _mx_np.array if is_np_array() else nd.array\n        if path_imglist:\n            logging.info('%s: loading image list %s...', class_name, path_imglist)\n            with open(path_imglist) as fin:\n                imglist = {}\n                imgkeys = []\n                for line in iter(fin.readline, ''):\n                    line = line.strip().split('\\t')\n                    label = array_fn(line[1:-1], dtype=dtype)\n                    key = int(line[0])\n                    imglist[key] = (label, line[-1])\n                    imgkeys.append(key)\n                self.imglist = imglist\n        elif isinstance(imglist, list):\n            logging.info('%s: loading image list...', class_name)\n            result = {}\n            imgkeys = []\n            index = 1\n            for img in imglist:\n                key = str(index)\n                index += 1\n                if len(img) > 2:\n                    label = array_fn(img[:-1], dtype=dtype)\n                elif isinstance(img[0], numeric_types):\n                    label = array_fn([img[0]], dtype=dtype)\n                else:\n                    label = array_fn(img[0], dtype=dtype)\n                result[key] = (label, img[-1])\n                imgkeys.append(str(key))\n            self.imglist = result\n        else:\n            self.imglist = None\n        self.path_root = path_root\n\n        self.check_data_shape(data_shape)\n        self.provide_data = [(data_name, (batch_size,) + data_shape)]\n        if label_width > 1:\n            self.provide_label = [(label_name, (batch_size, label_width))]\n        else:\n            self.provide_label = [(label_name, (batch_size,))]\n        self.batch_size = batch_size\n        self.data_shape = data_shape\n        self.label_width = label_width\n        self.shuffle = shuffle\n        if self.imgrec is None:\n            self.seq = imgkeys\n        elif shuffle or num_parts > 1 or path_imgidx:\n            assert self.imgidx is not None\n            self.seq = self.imgidx\n        else:\n            self.seq = None\n\n        if num_parts > 1:\n            assert part_index < num_parts\n            N = len(self.seq)\n            C = N // num_parts\n            self.seq = self.seq[part_index * C:(part_index + 1) * C]\n        if aug_list is None:\n            self.auglist = CreateAugmenter(data_shape, **kwargs)\n        else:\n            self.auglist = aug_list\n        self.cur = 0\n        self._allow_read = True\n        self.last_batch_handle = last_batch_handle\n        self.num_image = len(self.seq) if self.seq is not None else None\n        self._cache_data = None\n        self._cache_label = None\n        self._cache_idx = None\n        self.reset()\n\n    def reset(self):\n        \"\"\"Resets the iterator to the beginning of the data.\"\"\"\n        if self.seq is not None and self.shuffle:\n            random.shuffle(self.seq)\n        if self.last_batch_handle != 'roll_over' or \\\n            self._cache_data is None:\n            if self.imgrec is not None:\n                self.imgrec.reset()\n            self.cur = 0\n            if self._allow_read is False:\n                self._allow_read = True\n\n    def hard_reset(self):\n        \"\"\"Resets the iterator and ignore roll over data\"\"\"\n        if self.seq is not None and self.shuffle:\n            random.shuffle(self.seq)\n        if self.imgrec is not None:\n            self.imgrec.reset()\n        self.cur = 0\n        self._allow_read = True\n        self._cache_data = None\n        self._cache_label = None\n        self._cache_idx = None\n\n    def next_sample(self):\n        \"\"\"Helper function for reading in next sample.\"\"\"\n        if self._allow_read is False:\n            raise StopIteration\n        if self.seq is not None:\n            if self.cur < self.num_image:\n                idx = self.seq[self.cur]\n            else:\n                if self.last_batch_handle != 'discard':\n                    self.cur = 0\n                raise StopIteration\n            self.cur += 1\n            if self.imgrec is not None:\n                s = self.imgrec.read_idx(idx)\n                header, img = recordio.unpack(s)\n                if self.imglist is None:\n                    return header.label, img\n                else:\n                    return self.imglist[idx][0], img\n            else:\n                label, fname = self.imglist[idx]\n                return label, self.read_image(fname)\n        else:\n            s = self.imgrec.read()\n            if s is None:\n                if self.last_batch_handle != 'discard':\n                    self.imgrec.reset()\n                raise StopIteration\n            header, img = recordio.unpack(s)\n            return header.label, img\n\n    def _batchify(self, batch_data, batch_label, start=0):\n        \"\"\"Helper function for batchifying data\"\"\"\n        i = start\n        batch_size = self.batch_size\n        try:\n            while i < batch_size:\n                label, s = self.next_sample()\n                data = self.imdecode(s)\n                try:\n                    self.check_valid_image(data)\n                except RuntimeError as e:\n                    logging.debug('Invalid image, skipping:  %s', str(e))\n                    continue\n                data = self.augmentation_transform(data)\n                assert i < batch_size, 'Batch size must be multiples of augmenter output length'\n                batch_data[i] = self.postprocess_data(data)\n                batch_label[i] = label\n                i += 1\n        except StopIteration:\n            if not i:\n                raise StopIteration\n        return i\n\n    def next(self):\n        \"\"\"Returns the next batch of data.\"\"\"\n        batch_size = self.batch_size\n        c, h, w = self.data_shape\n        # if last batch data is rolled over\n        if self._cache_data is not None:\n            # check both the data and label have values\n            assert self._cache_label is not None, \"_cache_label didn't have values\"\n            assert self._cache_idx is not None, \"_cache_idx didn't have values\"\n            batch_data = self._cache_data\n            batch_label = self._cache_label\n            i = self._cache_idx\n            # clear the cache data\n        else:\n            if is_np_array():\n                zeros_fn = _mx_np.zeros\n                empty_fn = _mx_np.empty\n            else:\n                zeros_fn = nd.zeros\n                empty_fn = nd.empty\n            batch_data = zeros_fn((batch_size, c, h, w))\n            batch_label = empty_fn(self.provide_label[0][1])\n            i = self._batchify(batch_data, batch_label)\n        # calculate the padding\n        pad = batch_size - i\n        # handle padding for the last batch\n        if pad != 0:\n            if self.last_batch_handle == 'discard':\n                raise StopIteration\n            # if the option is 'roll_over', throw StopIteration and cache the data\n            if self.last_batch_handle == 'roll_over' and \\\n                self._cache_data is None:\n                self._cache_data = batch_data\n                self._cache_label = batch_label\n                self._cache_idx = i\n                raise StopIteration\n\n            _ = self._batchify(batch_data, batch_label, i)\n            if self.last_batch_handle == 'pad':\n                self._allow_read = False\n            else:\n                self._cache_data = None\n                self._cache_label = None\n                self._cache_idx = None\n\n        return io.DataBatch([batch_data], [batch_label], pad=pad)\n\n    def check_data_shape(self, data_shape):\n        \"\"\"Checks if the input data shape is valid\"\"\"\n        if not len(data_shape) == 3:\n            raise ValueError('data_shape should have length 3, with dimensions CxHxW')\n        if not data_shape[0] == 3:\n            raise ValueError('This iterator expects inputs to have 3 channels.')\n\n    def check_valid_image(self, data):\n        \"\"\"Checks if the input data is valid\"\"\"\n        if len(data[0].shape) == 0:\n            raise RuntimeError('Data shape is wrong')\n\n    def imdecode(self, s):\n        \"\"\"Decodes a string or byte string to an NDArray.\n        See mx.img.imdecode for more details.\"\"\"\n        def locate():\n            \"\"\"Locate the image file/index if decode fails.\"\"\"\n            if self.seq is not None:\n                idx = self.seq[(self.cur % self.num_image) - 1]\n            else:\n                idx = (self.cur % self.num_image) - 1\n            if self.imglist is not None:\n                _, fname = self.imglist[idx]\n                msg = \"filename: {}\".format(fname)\n            else:\n                msg = \"index: {}\".format(idx)\n            return \"Broken image \" + msg\n        try:\n            img = imdecode(s)\n        except Exception as e:\n            raise RuntimeError(\"{}, {}\".format(locate(), e))\n        return img\n\n    def read_image(self, fname):\n        \"\"\"Reads an input image `fname` and returns the decoded raw bytes.\n        Examples\n        --------\n        >>> dataIter.read_image('Face.jpg') # returns decoded raw bytes.\n        \"\"\"\n        with open(os.path.join(self.path_root, fname), 'rb') as fin:\n            img = fin.read()\n        return img\n\n    def augmentation_transform(self, data):\n        \"\"\"Transforms input data with specified augmentation.\"\"\"\n        for aug in self.auglist:\n            data = aug(data)\n        return data\n\n    def postprocess_data(self, datum):\n        \"\"\"Final postprocessing step before image is loaded into the batch.\"\"\"\n        if is_np_array():\n            return datum.transpose(2, 0, 1)\n        else:\n            return nd.transpose(datum, axes=(2, 0, 1))\n"
  },
  {
    "path": "python/mxnet/initializer.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Weight initializer.\"\"\"\n\nimport re\nimport logging\nimport warnings\nimport json\nfrom math import sqrt\nimport numpy as np\nfrom .base import string_types\nfrom .ndarray import NDArray, load\nfrom . import random\nfrom . import registry\nfrom . import ndarray\nfrom . util import is_np_array\nfrom . import numpy as _mx_np  # pylint: disable=reimported\n\n\n# inherit str for backward compatibility\nclass InitDesc(str):\n    \"\"\"\n    Descriptor for the initialization pattern.\n\n    Parameters\n    ----------\n    name : str\n        Name of variable.\n    attrs : dict of str to str\n        Attributes of this variable taken from ``Symbol.attr_dict``.\n    global_init : Initializer\n        Global initializer to fallback to.\n    \"\"\"\n    def __new__(cls, name, attrs=None, global_init=None):\n        ret = super(InitDesc, cls).__new__(cls, name)\n        ret.attrs = attrs or {}\n        ret.global_init = global_init\n        return ret\n\n\nclass Initializer(object):\n    \"\"\"The base class of an initializer.\"\"\"\n    def __init__(self, **kwargs):\n        self._kwargs = kwargs\n        self._verbose = False\n        self._print_func = None\n\n    def set_verbosity(self, verbose=False, print_func=None):\n        \"\"\"Switch on/off verbose mode\n\n        Parameters\n        ----------\n        verbose : bool\n            switch on/off verbose mode\n        print_func : function\n            A function that computes statistics of initialized arrays.\n            Takes an `NDArray` and returns an `str`. Defaults to mean\n            absolute value str((abs(x)/size(x)).asscalar()).\n        \"\"\"\n        self._verbose = verbose\n        if print_func is None:\n            def asum_stat(x):\n                \"\"\"returns |x|/size(x), async execution.\"\"\"\n                return str((ndarray.norm(x)/sqrt(x.size)).asscalar())\n            print_func = asum_stat\n        self._print_func = print_func\n        return self\n\n    def _verbose_print(self, desc, init, arr):\n        \"\"\"Internal verbose print function\n\n        Parameters\n        ----------\n        desc : InitDesc or str\n            name of the array\n        init : str\n            initializer pattern\n        arr : NDArray\n            initialized array\n        \"\"\"\n        if self._verbose and self._print_func:\n            logging.info('Initialized %s as %s: %s', desc, init, self._print_func(arr))\n\n    def dumps(self):\n        \"\"\"Saves the initializer to string\n\n        Returns\n        -------\n        str\n            JSON formatted string that describes the initializer.\n\n        Examples\n        --------\n        >>> # Create initializer and retrieve its parameters\n        ...\n        >>> init = mx.init.Normal(0.5)\n        >>> init.dumps()\n        '[\"normal\", {\"sigma\": 0.5}]'\n        >>> init = mx.init.Xavier(factor_type=\"in\", magnitude=2.34)\n        >>> init.dumps()\n        '[\"xavier\", {\"rnd_type\": \"uniform\", \"magnitude\": 2.34, \"factor_type\": \"in\"}]'\n        \"\"\"\n        return json.dumps([self.__class__.__name__.lower(), self._kwargs])\n\n    def __call__(self, desc, arr):\n        \"\"\"Initialize an array\n\n        Parameters\n        ----------\n        desc : InitDesc\n            Initialization pattern descriptor.\n\n        arr : NDArray\n            The array to be initialized.\n        \"\"\"\n        if not isinstance(desc, InitDesc):\n            self._legacy_init(desc, arr)\n            return\n\n        if desc.global_init is None:\n            desc.global_init = self\n        init = desc.attrs.get('__init__', \"\")\n\n        if init:\n            # when calling Variable initializer\n            create(init)._init_weight(desc, arr)\n            self._verbose_print(desc, init, arr)\n        else:\n            # register nnvm::FSetInputVariableAttrs in the backend for new patterns\n            # don't add new cases here.\n            if desc.endswith('weight'):\n                self._init_weight(desc, arr)\n                self._verbose_print(desc, 'weight', arr)\n            elif desc.endswith('bias'):\n                self._init_bias(desc, arr)\n                self._verbose_print(desc, 'bias', arr)\n            elif desc.endswith('gamma'):\n                self._init_gamma(desc, arr)\n                self._verbose_print(desc, 'gamma', arr)\n            elif desc.endswith('beta'):\n                self._init_beta(desc, arr)\n                self._verbose_print(desc, 'beta', arr)\n            elif desc.endswith('min'):\n                self._init_zero(desc, arr)\n                self._verbose_print(desc, 'min', arr)\n            elif desc.endswith('max'):\n                self._init_one(desc, arr)\n                self._verbose_print(desc, 'max', arr)\n            elif desc.endswith('weight_quantize'):\n                self._init_quantized_weight(desc, arr)\n                self._verbose_print(desc, 'weight_quantize', arr)\n            elif desc.endswith('bias_quantize'):\n                self._init_quantized_bias(desc, arr)\n                self._verbose_print(desc, 'bias_quantize', arr)\n            else:\n                self._init_default(desc, arr)\n\n    def _legacy_init(self, name, arr):\n        \"\"\"Legacy initialization method.\n\n        Parameters\n        ----------\n        name : str\n            Name of corresponding NDArray.\n\n        arr : NDArray\n            NDArray to be initialized.\n        \"\"\"\n        warnings.warn(\n            \"\\033[91mCalling initializer with init(str, NDArray) has been deprecated.\" \\\n            \"please use init(mx.init.InitDesc(...), NDArray) instead.\\033[0m\",\n            DeprecationWarning, stacklevel=3)\n        if not isinstance(name, string_types):\n            raise TypeError('name must be string')\n        if not isinstance(arr, NDArray):\n            raise TypeError('arr must be NDArray')\n        if name.startswith('upsampling'):\n            self._init_bilinear(name, arr)\n        elif name.startswith('stn_loc') and name.endswith('weight'):\n            self._init_zero(name, arr)\n        elif name.startswith('stn_loc') and name.endswith('bias'):\n            self._init_loc_bias(name, arr)\n        elif name.endswith('bias'):\n            self._init_bias(name, arr)\n        elif name.endswith('gamma'):\n            self._init_gamma(name, arr)\n        elif name.endswith('beta'):\n            self._init_beta(name, arr)\n        elif name.endswith('weight'):\n            self._init_weight(name, arr)\n        elif name.endswith(\"moving_mean\"):\n            self._init_zero(name, arr)\n        elif name.endswith(\"moving_var\"):\n            self._init_one(name, arr)\n        elif name.endswith(\"moving_inv_var\"):\n            self._init_zero(name, arr)\n        elif name.endswith(\"moving_avg\"):\n            self._init_zero(name, arr)\n        elif name.endswith('min'):\n            self._init_zero(name, arr)\n        elif name.endswith('max'):\n            self._init_one(name, arr)\n        else:\n            self._init_default(name, arr)\n\n    def _init_bilinear(self, _, arr):\n        weight = np.zeros(np.prod(arr.shape), dtype='float32')\n        shape = arr.shape\n        f = np.ceil(shape[3] / 2.)\n        c = (2 * f - 1 - f % 2) / (2. * f)\n        for i in range(np.prod(shape)):\n            x = i % shape[3]\n            y = (i // shape[3]) % shape[2]\n            weight[i] = (1 - abs(x / f - c)) * (1 - abs(y / f - c))\n        arr[:] = weight.reshape(shape)\n\n    def _init_loc_bias(self, _, arr):\n        shape = arr.shape\n        assert(shape[0] == 6)\n        arr[:] = np.array([1.0, 0, 0, 0, 1.0, 0])\n\n    def _init_zero(self, _, arr):\n        arr[:] = 0.0\n\n    def _init_one(self, _, arr):\n        arr[:] = 1.0\n\n    def _init_bias(self, _, arr):\n        arr[:] = 0.0\n\n    def _init_quantized_bias(self, _, arr):\n        arr[:] = 0\n\n    def _init_gamma(self, _, arr):\n        arr[:] = 1.0\n\n    def _init_beta(self, _, arr):\n        arr[:] = 0.0\n\n    def _init_weight(self, name, arr):\n        \"\"\"Abstract method to Initialize weight.\"\"\"\n        raise NotImplementedError(\"Must override it\")\n\n    def _init_quantized_weight(self, _, arr):\n        _arr = random.randint(-127, 127, dtype='int32').asnumpy()\n        arr[:] = np.int8(_arr)\n\n    def _init_default(self, name, _):\n        raise ValueError(\n            f'Unknown initialization pattern for {name}. ' \\\n            'Default initialization is now limited to '\\\n            '\"weight\", \"bias\", \"gamma\" (1.0), and \"beta\" (0.0).' \\\n            'Please use mx.sym.Variable(init=mx.init.*) to set initialization pattern')\n\n    def __eq__(self, other):\n        if not isinstance(other, Initializer):\n            return NotImplemented\n        # pylint: disable=unidiomatic-typecheck\n        return type(self) is type(other) and self._kwargs == other._kwargs\n\n# pylint: disable=invalid-name\n_register = registry.get_register_func(Initializer, 'initializer')\nalias = registry.get_alias_func(Initializer, 'initializer')\ncreate = registry.get_create_func(Initializer, 'initializer')\n# pylint: enable=invalid-name\n\ndef register(klass):\n    \"\"\"Registers a custom initializer.\n\n    Custom initializers can be created by extending `mx.init.Initializer` and implementing the\n    required functions like `_init_weight` and `_init_bias`. The created initializer must be\n    registered using `mx.init.register` before it can be called by name.\n\n    Parameters\n    ----------\n    klass : class\n        A subclass of `mx.init.Initializer` that needs to be registered as a custom initializer.\n\n    Example\n    -------\n    >>> # Create and register a custom initializer that\n    ... # initializes weights to 0.1 and biases to 1.\n    ...\n    >>> @mx.init.register\n    ... @alias('myinit')\n    ... class CustomInit(mx.init.Initializer):\n    ...   def __init__(self):\n    ...     super(CustomInit, self).__init__()\n    ...   def _init_weight(self, _, arr):\n    ...     arr[:] = 0.1\n    ...   def _init_bias(self, _, arr):\n    ...     arr[:] = 1\n    ...\n    >>> # block is an instance of 'mxnet.gluon.Block'\n    ...\n    >>> block.initialize(CustomInit())\n    \"\"\"\n    return _register(klass)\n\n\nclass Load(object):\n    \"\"\"Initializes variables by loading data from file or dict.\n\n    **Note** Load will drop ``arg:`` or ``aux:`` from name and\n    initialize the variables that match with the prefix dropped.\n\n    Parameters\n    ----------\n    param: str or dict of str->`NDArray`\n        Parameter file or dict mapping name to NDArray.\n    default_init: Initializer\n        Default initializer when name is not found in `param`.\n    verbose: bool\n        Flag for enabling logging of source when initializing.\n\n    \"\"\"\n    def __init__(self, param, default_init=None, verbose=False):\n        if isinstance(param, str):\n            param = load(param)\n        assert isinstance(param, dict)\n        self.param = {}\n        for name, arr in param.items():\n            if name.startswith('arg:') or name.startswith('aux:'):\n                self.param[name[4:]] = arr\n            else:\n                self.param[name] = arr\n        self.default_init = default_init\n        self.verbose = verbose\n\n    def __call__(self, name, arr):\n        if name in self.param:\n            assert arr.shape == self.param[name].shape, \\\n                f'Parameter {name} cannot be initialized from loading. ' + \\\n                f'Shape mismatch, target {str(arr.shape)} vs loaded {self.param[name].shape}'\n            arr[:] = self.param[name]\n            if self.verbose:\n                logging.info('Initialized %s by loading', name)\n        else:\n            assert self.default_init is not None, \\\n                f\"Cannot Initialize {name}. Not found in loaded param \" + \\\n                \"and no default Initializer is provided.\"\n            self.default_init(name, arr)\n            if self.verbose:\n                logging.info('Initialized %s by default', name)\n\n\nclass Mixed(object):\n    \"\"\"Initialize parameters using multiple initializers.\n\n    Parameters\n    ----------\n    patterns: list of str\n        List of regular expressions matching parameter names.\n    initializers: list of Initializer\n        List of initializers corresponding to `patterns`.\n\n    Example\n    -------\n    >>> # Given 'block', an instance of 'mxnet.gluon.Block', initialize biases to zero\n    ... # and every other parameter to random values with uniform distribution.\n    ...\n    >>> init = mx.initializer.Mixed(['bias', '.*'], [mx.init.Zero(), mx.init.Uniform(0.1)])\n    >>> block.initialize(init)\n    >>>\n    >>> for dictionary in module.get_params():\n    ...     for key in dictionary:\n    ...         print(key)\n    ...         print(dictionary[key].asnumpy())\n    ...\n    fullyconnected1_weight\n    [[ 0.0097627   0.01856892  0.04303787]]\n    fullyconnected1_bias\n    [ 0.]\n\n    \"\"\"\n    def __init__(self, patterns, initializers):\n        assert len(patterns) == len(initializers)\n        self.map = list(zip([re.compile(p) for p in patterns], initializers))\n\n    def __call__(self, name, arr):\n        for prog, init in self.map:\n            if prog.match(name):\n                init(name, arr)\n                return\n        raise ValueError('Parameter name %s did not match any pattern. Consider' +\n                         'add a \".*\" pattern at the and with default Initializer.')\n\n@register\n@alias(\"zeros\")\nclass Zero(Initializer):\n    \"\"\"Initializes weights to zero.\n\n    Example\n    -------\n    >>> # Given 'block', an instance of 'mxnet.gluon.Block', initialize weights to zero.\n    ...\n    >>> init = mx.initializer.Zero()\n    >>> module.initialize(init)\n    >>> for dictionary in module.get_params():\n    ...     for key in dictionary:\n    ...         print(key)\n    ...         print(dictionary[key].asnumpy())\n    ...\n    fullyconnected0_weight\n    [[ 0.  0.  0.]]\n    \"\"\"\n    def __init__(self):\n        super(Zero, self).__init__()\n\n    def _init_weight(self, _, arr):\n        arr[:] = 0\n\n@register\n@alias(\"ones\")\nclass One(Initializer):\n    \"\"\"Initializes weights to one.\n\n    Example\n    -------\n    >>> # Given 'block', an instance of 'mxnet.gluon.Block', initialize weights to one.\n    ...\n    >>> init = mx.initializer.One()\n    >>> module.initialize(init)\n    >>> for dictionary in module.get_params():\n    ...     for key in dictionary:\n    ...         print(key)\n    ...         print(dictionary[key].asnumpy())\n    ...\n    fullyconnected0_weight\n    [[ 1.  1.  1.]]\n    \"\"\"\n    def __init__(self):\n        super(One, self).__init__()\n\n    def _init_weight(self, _, arr):\n        arr[:] = 1\n\n@register\nclass Constant(Initializer):\n    \"\"\"Initializes the weights to a given value.\n    The value passed in can be a scalar or a NDarray that matches the shape\n    of the parameter to be set.\n\n    Parameters\n    ----------\n    value : float, NDArray\n        Value to set.\n    \"\"\"\n    def __init__(self, value):\n        super(Constant, self).__init__(value=value)\n        self.value = value\n\n    def _init_weight(self, _, arr):\n        arr[:] = self.value\n\n    def dumps(self):\n        val = self._kwargs['value']\n        if not np.isscalar(val):\n            self._kwargs['value'] = val.tolist() if isinstance(val, np.ndarray) else val.asnumpy().tolist()\n        return json.dumps([self.__class__.__name__.lower(), self._kwargs])\n\n@register\nclass Uniform(Initializer):\n    \"\"\"Initializes weights with random values uniformly sampled from a given range.\n\n    Parameters\n    ----------\n    scale : float, optional\n        The bound on the range of the generated random values.\n        Values are generated from the range [-`scale`, `scale`].\n        Default scale is 0.07.\n\n    Example\n    -------\n    >>> # Given 'block', an instance of 'mxnet.gluon.Block', initialize weights\n    >>> # to random values uniformly sampled between -0.1 and 0.1.\n    ...\n    >>> init = mx.init.Uniform(0.1)\n    >>> module.initialize(init)\n    >>> for dictionary in module.get_params():\n    ...     for key in dictionary:\n    ...         print(key)\n    ...         print(dictionary[key].asnumpy())\n    ...\n    fullyconnected0_weight\n    [[ 0.01360891 -0.02144304  0.08511933]]\n    \"\"\"\n    def __init__(self, scale=0.07):\n        super(Uniform, self).__init__(scale=scale)\n        self.scale = scale\n\n    def _init_weight(self, _, arr):\n        uniform_fn = _mx_np.random.uniform if is_np_array() else random.uniform\n        uniform_fn(-self.scale, self.scale, arr.shape, dtype=arr.dtype, out=arr)\n\n@register\nclass Normal(Initializer):\n    \"\"\"Initializes weights with random values sampled from a normal distribution\n    with a mean of zero and standard deviation of `sigma`.\n\n    Parameters\n    ----------\n    sigma : float, optional\n        Standard deviation of the normal distribution.\n        Default standard deviation is 0.01.\n\n    Example\n    -------\n    >>> # Given 'block', an instance of 'mxnet.gluon.Block', initialize weights\n    >>> # to random values sampled from a normal distribution.\n    ...\n    >>> init = mx.init.Normal(0.5)\n    >>> module.initialize(init)\n    >>> for dictionary in module.get_params():\n    ...     for key in dictionary:\n    ...         print(key)\n    ...         print(dictionary[key].asnumpy())\n    ...\n    fullyconnected0_weight\n    [[-0.3214761  -0.12660924  0.53789419]]\n    \"\"\"\n    def __init__(self, sigma=0.01):\n        super(Normal, self).__init__(sigma=sigma)\n        self.sigma = sigma\n\n    def _init_weight(self, _, arr):\n        normal_fn = _mx_np.random.normal if is_np_array() else random.normal\n        normal_fn(0, self.sigma, arr.shape, dtype=arr.dtype, out=arr)\n\n@register\nclass Orthogonal(Initializer):\n    \"\"\"Initialize weight as orthogonal matrix.\n\n    This initializer implements *Exact solutions to the nonlinear dynamics of\n    learning in deep linear neural networks*, available at\n    https://arxiv.org/abs/1312.6120.\n\n    Parameters\n    ----------\n    scale : float optional\n        Scaling factor of weight.\n\n    rand_type: string optional\n        Use \"uniform\" or \"normal\" random number to initialize weight.\n\n    \"\"\"\n    def __init__(self, scale=1.414, rand_type=\"uniform\"):\n        super(Orthogonal, self).__init__(scale=scale, rand_type=rand_type)\n        self.scale = scale\n        self.rand_type = rand_type\n\n    def _init_weight(self, _, arr):\n        nout = arr.shape[0]\n        nin = np.prod(arr.shape[1:])\n        if self.rand_type == \"uniform\":\n            tmp = random.uniform(-1.0, 1.0, shape=(nout, nin)).asnumpy()\n        elif self.rand_type == \"normal\":\n            tmp = random.normal(0.0, 1.0, shape=(nout, nin)).asnumpy()\n        u, _, v = np.linalg.svd(tmp, full_matrices=False) # pylint: disable=invalid-name\n        if u.shape == tmp.shape:\n            res = u\n        else:\n            res = v\n        res = self.scale * res.reshape(arr.shape)\n        arr[:] = res\n\n@register\nclass Xavier(Initializer):\n    \"\"\"Returns an initializer performing \"Xavier\" initialization for weights.\n\n    This initializer is designed to keep the scale of gradients roughly the same\n    in all layers.\n\n    By default, `rnd_type` is ``'uniform'`` and `factor_type` is ``'avg'``,\n    the initializer fills the weights with random numbers in the range\n    of :math:`[-c, c]`, where :math:`c = \\\\sqrt{\\\\frac{3.}{0.5 * (n_{in} + n_{out})}}`.\n    :math:`n_{in}` is the number of neurons feeding into weights, and :math:`n_{out}` is\n    the number of neurons the result is fed to.\n\n    If `rnd_type` is ``'uniform'`` and `factor_type` is ``'in'``,\n    the :math:`c = \\\\sqrt{\\\\frac{3.}{n_{in}}}`.\n    Similarly when `factor_type` is ``'out'``, the :math:`c = \\\\sqrt{\\\\frac{3.}{n_{out}}}`.\n\n    If `rnd_type` is ``'gaussian'`` and `factor_type` is ``'avg'``,\n    the initializer fills the weights with numbers from normal distribution with\n    a standard deviation of :math:`\\\\sqrt{\\\\frac{3.}{0.5 * (n_{in} + n_{out})}}`.\n\n    Parameters\n    ----------\n    rnd_type: str, optional\n        Random generator type, can be ``'gaussian'`` or ``'uniform'``.\n\n    factor_type: str, optional\n        Can be ``'avg'``, ``'in'``, or ``'out'``.\n\n    magnitude: float, optional\n        Scale of random number.\n    \"\"\"\n    def __init__(self, rnd_type=\"uniform\", factor_type=\"avg\", magnitude=3):\n        super(Xavier, self).__init__(rnd_type=rnd_type, factor_type=factor_type,\n                                     magnitude=magnitude)\n        self.rnd_type = rnd_type\n        self.factor_type = factor_type\n        self.magnitude = float(magnitude)\n\n\n    def _init_weight(self, name, arr):\n        shape = arr.shape\n        hw_scale = 1.\n        if len(shape) < 2:\n            raise ValueError('Xavier initializer cannot be applied to vector {0}. It requires at'\n                             ' least 2D.'.format(name))\n        if len(shape) > 2:\n            hw_scale = np.prod(shape[2:])\n        fan_in, fan_out = shape[1] * hw_scale, shape[0] * hw_scale\n        factor = 1.\n        if self.factor_type == \"avg\":\n            factor = (fan_in + fan_out) / 2.0\n        elif self.factor_type == \"in\":\n            factor = fan_in\n        elif self.factor_type == \"out\":\n            factor = fan_out\n        else:\n            raise ValueError(\"Incorrect factor type\")\n        scale = np.sqrt(self.magnitude / factor)\n        if self.rnd_type == \"uniform\":\n            uniform_fn = _mx_np.random.uniform if is_np_array() else random.uniform\n            uniform_fn(-scale, scale, arr.shape, dtype=arr.dtype, out=arr)\n        elif self.rnd_type == \"gaussian\":\n            normal_fn = _mx_np.random.normal if is_np_array() else random.normal\n            normal_fn(0, scale, arr.shape, dtype=arr.dtype, out=arr)\n        else:\n            raise ValueError(\"Unknown random type\")\n\n@register\nclass MSRAPrelu(Xavier):\n    \"\"\"Initialize the weight according to a MSRA paper.\n\n    This initializer implements *Delving Deep into Rectifiers: Surpassing\n    Human-Level Performance on ImageNet Classification*, available at\n    https://arxiv.org/abs/1502.01852.\n\n    This initializer is proposed for initialization related to ReLu activation,\n    it makes some changes on top of Xavier method.\n\n    Parameters\n    ----------\n    factor_type: str, optional\n        Can be ``'avg'``, ``'in'``, or ``'out'``.\n\n    slope: float, optional\n        initial slope of any PReLU (or similar) nonlinearities.\n    \"\"\"\n    def __init__(self, factor_type=\"avg\", slope=0.25):\n        magnitude = 2. / (1 + slope ** 2)\n        super(MSRAPrelu, self).__init__(\"gaussian\", factor_type, magnitude)\n        self._kwargs = {'factor_type': factor_type, 'slope': slope}\n\n@register\nclass Bilinear(Initializer):\n    \"\"\"Initialize weight for upsampling layers.\"\"\"\n    def __init__(self):\n        super(Bilinear, self).__init__()\n\n    def _init_weight(self, _, arr):\n        weight = np.zeros(np.prod(arr.shape), dtype='float32')\n        shape = arr.shape\n        f = np.ceil(shape[3] / 2.)\n        c = (2 * f - 1 - f % 2) / (2. * f)\n        for i in range(np.prod(shape)):\n            x = i % shape[3]\n            y = (i // shape[3]) % shape[2]\n            weight[i] = (1 - abs(x / f - c)) * (1 - abs(y / f - c))\n        arr[:] = weight.reshape(shape)\n\n\n@register\nclass LSTMBias(Initializer):\n    \"\"\"Initialize all biases of an LSTMCell to 0.0 except for\n    the forget gate whose bias is set to custom value.\n\n    Parameters\n    ----------\n    forget_bias: float, default 1.0\n        bias for the forget gate. Jozefowicz et al. 2015 recommends\n        setting this to 1.0.\n    \"\"\"\n    def __init__(self, forget_bias=1.0):\n        super(LSTMBias, self).__init__(forget_bias=forget_bias)\n        self.forget_bias = forget_bias\n\n    def _init_weight(self, name, arr):\n        arr[:] = 0.0\n        # in the case of LSTMCell the forget gate is the second\n        # gate of the 4 LSTM gates, we modify the according values.\n        num_hidden = int(arr.shape[0] / 4)\n        arr[num_hidden:2*num_hidden] = self.forget_bias\n\n\n@register\nclass RNNFused(Initializer):\n    \"\"\"Initialize RNN fused parameter with bias part initialized to 0.0 and\n    weight initialized with random values uniformly sampled from a given range.\n\n    Parameters\n    ----------\n    mode : {'gru', 'lstm', 'rnn_relu', 'rnn_tanh'}, required\n        the type of RNN to compute\n    num_layers : int (non-negative), required\n        number of stacked layers\n    state_size : int (non-negative), required\n        size of the state for each layer\n    bidirectional : boolean, optional, default=0\n        whether to use bidirectional recurrent layers\n    projection_size : int or None, optional, default='None'\n        size of project size\n    scale : float, optional\n        The bound on the range of the generated random values for weights.\n        Values are generated from the range [-`scale`, `scale`].\n        Default scale is 0.07.\n    \"\"\"\n    def __init__(self, mode, num_layers, state_size, bidirectional=False,\n                 projection_size=None, i2h_weight_initializer=None,\n                 h2h_weight_initializer=None, i2h_bias_initializer=None,\n                 h2h_bias_initializer=None, h2r_weight_initializer=None):\n        super(RNNFused, self).__init__(mode=mode, num_layers=num_layers,\n                                       state_size=state_size,\n                                       bidirectional=bidirectional,\n                                       projection_size=projection_size,\n                                       i2h_weight_initializer=i2h_weight_initializer,\n                                       h2h_weight_initializer=h2h_weight_initializer,\n                                       i2h_bias_initializer=i2h_bias_initializer,\n                                       h2h_bias_initializer=h2h_bias_initializer,\n                                       h2r_weight_initializer=h2r_weight_initializer)\n        self.gates = {'rnn_relu': 1, 'rnn_tanh': 1, 'lstm': 4, 'gru': 3}[mode]\n        self.num_layers = num_layers\n        self.num_hidden = state_size\n        self.dir = 2 if bidirectional else 1\n        self.projection_size = projection_size\n        self._i2h_weight_initializer = i2h_weight_initializer\n        self._h2h_weight_initializer = h2h_weight_initializer\n        self._i2h_bias_initializer = i2h_bias_initializer\n        self._h2h_bias_initializer = h2h_bias_initializer\n        self._h2r_weight_initializer = h2r_weight_initializer\n\n    # pylint: disable=too-many-nested-blocks\n    def _init_weight(self, name, arr):\n        arr_len = arr.shape[0]\n        size = self.num_hidden * self.dir * self.gates\n        if not self.projection_size:\n            # second layer size\n            size2 = (self.num_hidden * self.dir + self.num_hidden + 2) * size\n            input_size = (arr_len - (self.num_layers - 1) * size2) // \\\n                size - 2 - self.num_hidden\n        else:\n            # second layer size\n            size2 = (self.projection_size * self.dir + self.projection_size + 2) * size\n            size_projection = self.projection_size * self.num_hidden * self.num_layers * self.dir\n            input_size = (arr_len - size_projection - (self.num_layers - 1) * size2) // \\\n                size - 2 - self.projection_size\n        begin = 0\n        if not self.projection_size:\n            for param in ['weight', 'bias']:\n                for layer_num in range(self.num_layers):\n                    for _ in range(self.dir):\n                        for connect in ['i2h', 'h2h']:\n                            num_inputs = input_size\n                            if layer_num != 0:\n                                num_inputs = self.num_hidden * self.dir\n                            if connect == 'h2h':\n                                num_inputs = self.num_hidden\n                            shape0 = self.gates * self.num_hidden\n                            if param == 'weight':\n                                cur_len = shape0 * num_inputs\n                            else:\n                                cur_len = shape0\n                            self._init_util(param, connect, arr[begin:begin+cur_len])\n                            begin += cur_len\n        else:\n            for param in ['weight', 'bias']:\n                for layer_num in range(self.num_layers):\n                    for _ in range(self.dir):\n                        for connect in ['i2h', 'h2h', 'h2r']:\n                            if connect != 'h2r' or param != 'bias':\n                                if connect == 'h2r':\n                                    cur_len = self.projection_size * self.num_hidden\n                                else:\n                                    num_inputs = input_size\n                                    if layer_num != 0:\n                                        num_inputs = self.projection_size * self.dir\n                                    if connect == 'h2h':\n                                        num_inputs = self.projection_size\n                                    shape0 = self.gates * self.num_hidden\n                                    if param == 'weight':\n                                        cur_len = shape0 * num_inputs\n                                    else:\n                                        cur_len = shape0\n                                self._init_util(param, connect, arr[begin:begin+cur_len])\n                                begin += cur_len\n\n    def _init_util(self, param, connect, arr):\n        name = \"_{}_{}_initializer\".format(connect, param)\n        init = getattr(self, name)\n        create(init)(InitDesc(name, {'__init__': init}), arr)\n\n    def set_initializer(self, init):\n        self._i2h_weight_initializer = \\\n            init if not self._i2h_weight_initializer else 'uniform'\n        self._h2h_weight_initializer = \\\n            init if not self._h2h_weight_initializer else 'uniform'\n        self._i2h_bias_initializer = \\\n            init if not self._i2h_bias_initializer else 'zero'\n        self._h2h_bias_initializer = \\\n            init if not self._i2h_bias_initializer else 'zero'\n        self._h2r_weight_initializer = \\\n            init if not self._h2r_weight_initializer else 'uniform'\n"
  },
  {
    "path": "python/mxnet/io/__init__.py",
    "content": "#!/usr/bin/env python\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import\n\"\"\" Data iterators for common data formats and utility functions.\"\"\"\n\nfrom . import io\nfrom .io import CSVIter, DataBatch, DataDesc, DataIter, ImageDetRecordIter, ImageRecordInt8Iter, ImageRecordIter,\\\n    ImageRecordIter_v1, ImageRecordUInt8Iter, ImageRecordUInt8Iter_v1, LibSVMIter, MNISTIter, MXDataIter, NDArrayIter,\\\n    PrefetchingIter, ResizeIter\n\nfrom . import utils\nfrom .utils import _init_data, _getdata_by_idx, _has_instance\n"
  },
  {
    "path": "python/mxnet/io/io.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=unnecessary-pass\n\"\"\"Data iterators for common data formats.\"\"\"\nfrom collections import namedtuple\n\nimport sys\nimport ctypes\nimport logging\nimport threading\nimport numpy as np\n\nfrom ..base import _LIB\nfrom ..base import c_str_array, mx_uint, py_str\nfrom ..base import DataIterHandle, NDArrayHandle\nfrom ..base import mx_real_t\nfrom ..base import check_call, build_param_doc as _build_param_doc\nfrom ..ndarray import NDArray\nfrom ..ndarray.sparse import CSRNDArray\nfrom ..util import is_np_array\nfrom ..ndarray import array\nfrom ..ndarray import concat, tile\n\nfrom .utils import _init_data, _has_instance, _getdata_by_idx\n\nclass DataDesc(namedtuple('DataDesc', ['name', 'shape'])):\n    \"\"\"DataDesc is used to store name, shape, type and layout\n    information of the data or the label.\n\n    The `layout` describes how the axes in `shape` should be interpreted,\n    for example for image data setting `layout=NCHW` indicates\n    that the first axis is number of examples in the batch(N),\n    C is number of channels, H is the height and W is the width of the image.\n\n    For sequential data, by default `layout` is set to ``NTC``, where\n    N is number of examples in the batch, T the temporal axis representing time\n    and C is the number of channels.\n\n    Parameters\n    ----------\n    cls : DataDesc\n         The class.\n    name : str\n         Data name.\n    shape : tuple of int\n         Data shape.\n    dtype : np.dtype, optional\n         Data type.\n    layout : str, optional\n         Data layout.\n    \"\"\"\n    def __new__(cls, name, shape, dtype=mx_real_t, layout='NCHW'): # pylint: disable=super-on-old-class\n        ret = super(cls, DataDesc).__new__(cls, name, shape)\n        ret.dtype = dtype\n        ret.layout = layout\n        return ret\n\n    def __repr__(self):\n        return f\"DataDesc[{self.name},{self.shape},{self.dtype},{self.layout}]\"\n\n    @staticmethod\n    def get_batch_axis(layout):\n        \"\"\"Get the dimension that corresponds to the batch size.\n\n        When data parallelism is used, the data will be automatically split and\n        concatenated along the batch-size dimension. Axis can be -1, which means\n        the whole array will be copied for each data-parallelism device.\n\n        Parameters\n        ----------\n        layout : str\n            layout string. For example, \"NCHW\".\n\n        Returns\n        -------\n        int\n            An axis indicating the batch_size dimension.\n        \"\"\"\n        if layout is None:\n            return 0\n        return layout.find('N')\n\n    @staticmethod\n    def get_list(shapes, types):\n        \"\"\"Get DataDesc list from attribute lists.\n\n        Parameters\n        ----------\n        shapes : a tuple of (name, shape)\n        types : a tuple of  (name, np.dtype)\n        \"\"\"\n        if types is not None:\n            type_dict = dict(types)\n            return [DataDesc(x[0], x[1], type_dict[x[0]]) for x in shapes]\n        else:\n            return [DataDesc(x[0], x[1]) for x in shapes]\n\nclass DataBatch(object):\n    \"\"\"A data batch.\n\n    MXNet's data iterator returns a batch of data for each `next` call.\n    This data contains `batch_size` number of examples.\n\n    If the input data consists of images, then shape of these images depend on\n    the `layout` attribute of `DataDesc` object in `provide_data` parameter.\n\n    If `layout` is set to 'NCHW' then, images should be stored in a 4-D matrix\n    of shape ``(batch_size, num_channel, height, width)``.\n    If `layout` is set to 'NHWC' then, images should be stored in a 4-D matrix\n    of shape ``(batch_size, height, width, num_channel)``.\n    The channels are often in RGB order.\n\n    Parameters\n    ----------\n    data : list of `NDArray`, each array containing `batch_size` examples.\n          A list of input data.\n    label : list of `NDArray`, each array often containing a 1-dimensional array. optional\n          A list of input labels.\n    pad : int, optional\n          The number of examples padded at the end of a batch. It is used when the\n          total number of examples read is not divisible by the `batch_size`.\n          These extra padded examples are ignored in prediction.\n    index : numpy.array, optional\n          The example indices in this batch.\n    bucket_key : int, optional\n          The bucket key, used for bucketing module.\n    provide_data : list of `DataDesc`, optional\n          A list of `DataDesc` objects. `DataDesc` is used to store\n          name, shape, type and layout information of the data.\n          The *i*-th element describes the name and shape of ``data[i]``.\n    provide_label : list of `DataDesc`, optional\n          A list of `DataDesc` objects. `DataDesc` is used to store\n          name, shape, type and layout information of the label.\n          The *i*-th element describes the name and shape of ``label[i]``.\n    \"\"\"\n    def __init__(self, data, label=None, pad=None, index=None,\n                 bucket_key=None, provide_data=None, provide_label=None):\n        if data is not None:\n            assert isinstance(data, (list, tuple)), \"Data must be list of NDArrays\"\n        if label is not None:\n            assert isinstance(label, (list, tuple)), \"Label must be list of NDArrays\"\n        self.data = data\n        self.label = label\n        self.pad = pad\n        self.index = index\n\n        self.bucket_key = bucket_key\n        self.provide_data = provide_data\n        self.provide_label = provide_label\n\n    def __str__(self):\n        data_shapes = [d.shape for d in self.data]\n        if self.label:\n            label_shapes = [l.shape for l in self.label]\n        else:\n            label_shapes = None\n        return \"{}: data shapes: {} label shapes: {}\".format(\n            self.__class__.__name__,\n            data_shapes,\n            label_shapes)\n\nclass DataIter(object):\n    \"\"\"The base class for an MXNet data iterator.\n\n    All I/O in MXNet is handled by specializations of this class. Data iterators\n    in MXNet are similar to standard-iterators in Python. On each call to `next`\n    they return a `DataBatch` which represents the next batch of data. When\n    there is no more data to return, it raises a `StopIteration` exception.\n\n    Parameters\n    ----------\n    batch_size : int, optional\n        The batch size, namely the number of items in the batch.\n\n    See Also\n    --------\n    NDArrayIter : Data-iterator for MXNet NDArray or numpy-ndarray objects.\n    CSVIter : Data-iterator for csv data.\n    LibSVMIter : Data-iterator for libsvm data.\n    ImageIter : Data-iterator for images.\n    \"\"\"\n    def __init__(self, batch_size=0):\n        self.batch_size = batch_size\n\n    def __iter__(self):\n        return self\n\n    def reset(self):\n        \"\"\"Reset the iterator to the begin of the data.\"\"\"\n        pass\n\n    def next(self):\n        \"\"\"Get next data batch from iterator.\n\n        Returns\n        -------\n        DataBatch\n            The data of next batch.\n\n        Raises\n        ------\n        StopIteration\n            If the end of the data is reached.\n        \"\"\"\n        if self.iter_next():\n            return DataBatch(data=self.getdata(), label=self.getlabel(), \\\n                    pad=self.getpad(), index=self.getindex())\n        else:\n            raise StopIteration\n\n    def __next__(self):\n        return self.next()\n\n    def iter_next(self):\n        \"\"\"Move to the next batch.\n\n        Returns\n        -------\n        boolean\n            Whether the move is successful.\n        \"\"\"\n        pass\n\n    def getdata(self):\n        \"\"\"Get data of current batch.\n\n        Returns\n        -------\n        list of NDArray\n            The data of the current batch.\n        \"\"\"\n        pass\n\n    def getlabel(self):\n        \"\"\"Get label of the current batch.\n\n        Returns\n        -------\n        list of NDArray\n            The label of the current batch.\n        \"\"\"\n        pass\n\n    def getindex(self):\n        \"\"\"Get index of the current batch.\n\n        Returns\n        -------\n        index : numpy.array\n            The indices of examples in the current batch.\n        \"\"\"\n        return None\n\n    def getpad(self):\n        \"\"\"Get the number of padding examples in the current batch.\n\n        Returns\n        -------\n        int\n            Number of padding examples in the current batch.\n        \"\"\"\n        pass\n\nclass ResizeIter(DataIter):\n    \"\"\"Resize a data iterator to a given number of batches.\n\n    Parameters\n    ----------\n    data_iter : DataIter\n        The data iterator to be resized.\n    size : int\n        The number of batches per epoch to resize to.\n    reset_internal : bool\n        Whether to reset internal iterator on ResizeIter.reset.\n\n\n    Examples\n    --------\n    >>> nd_iter = mx.io.NDArrayIter(mx.nd.ones((100,10)), batch_size=25)\n    >>> resize_iter = mx.io.ResizeIter(nd_iter, 2)\n    >>> for batch in resize_iter:\n    ...     print(batch.data)\n    [<NDArray 25x10 @cpu(0)>]\n    [<NDArray 25x10 @cpu(0)>]\n    \"\"\"\n    def __init__(self, data_iter, size, reset_internal=True):\n        super(ResizeIter, self).__init__()\n        self.data_iter = data_iter\n        self.size = size\n        self.reset_internal = reset_internal\n        self.cur = 0\n        self.current_batch = None\n\n        self.provide_data = data_iter.provide_data\n        self.provide_label = data_iter.provide_label\n        self.batch_size = data_iter.batch_size\n        if hasattr(data_iter, 'default_bucket_key'):\n            self.default_bucket_key = data_iter.default_bucket_key\n\n    def reset(self):\n        self.cur = 0\n        if self.reset_internal:\n            self.data_iter.reset()\n\n    def iter_next(self):\n        if self.cur == self.size:\n            return False\n        try:\n            self.current_batch = self.data_iter.next()\n        except StopIteration:\n            self.data_iter.reset()\n            self.current_batch = self.data_iter.next()\n\n        self.cur += 1\n        return True\n\n    def getdata(self):\n        return self.current_batch.data\n\n    def getlabel(self):\n        return self.current_batch.label\n\n    def getindex(self):\n        return self.current_batch.index\n\n    def getpad(self):\n        return self.current_batch.pad\n\nclass PrefetchingIter(DataIter):\n    \"\"\"Performs pre-fetch for other data iterators.\n\n    This iterator will create another thread to perform ``iter_next`` and then\n    store the data in memory. It potentially accelerates the data read, at the\n    cost of more memory usage.\n\n    Parameters\n    ----------\n    iters : DataIter or list of DataIter\n        The data iterators to be pre-fetched.\n    rename_data : None or list of dict\n        The *i*-th element is a renaming map for the *i*-th iter, in the form of\n        {'original_name' : 'new_name'}. Should have one entry for each entry\n        in iter[i].provide_data.\n    rename_label : None or list of dict\n        Similar to ``rename_data``.\n\n    Examples\n    --------\n    >>> iter1 = mx.io.NDArrayIter({'data':mx.nd.ones((100,10))}, batch_size=25)\n    >>> iter2 = mx.io.NDArrayIter({'data':mx.nd.ones((100,10))}, batch_size=25)\n    >>> piter = mx.io.PrefetchingIter([iter1, iter2],\n    ...                               rename_data=[{'data': 'data_1'}, {'data': 'data_2'}])\n    >>> print(piter.provide_data)\n    [DataDesc[data_1,(25, 10L),<type 'numpy.float32'>,NCHW],\n     DataDesc[data_2,(25, 10L),<type 'numpy.float32'>,NCHW]]\n    \"\"\"\n    def __init__(self, iters, rename_data=None, rename_label=None):\n        super(PrefetchingIter, self).__init__()\n        if not isinstance(iters, list):\n            iters = [iters]\n        self.n_iter = len(iters)\n        assert self.n_iter > 0\n        self.iters = iters\n        self.rename_data = rename_data\n        self.rename_label = rename_label\n        self.batch_size = self.provide_data[0][1][0]\n        self.data_ready = [threading.Event() for i in range(self.n_iter)]\n        self.data_taken = [threading.Event() for i in range(self.n_iter)]\n        for i in self.data_taken:\n            i.set()\n        self.started = True\n        self.current_batch = [None for i in range(self.n_iter)]\n        self.next_batch = [None for i in range(self.n_iter)]\n        def prefetch_func(self, i):\n            \"\"\"Thread entry\"\"\"\n            while True:\n                self.data_taken[i].wait()\n                if not self.started:\n                    break\n                try:\n                    self.next_batch[i] = self.iters[i].next()\n                except StopIteration:\n                    self.next_batch[i] = None\n                self.data_taken[i].clear()\n                self.data_ready[i].set()\n        self.prefetch_threads = [threading.Thread(target=prefetch_func, args=[self, i]) \\\n                                 for i in range(self.n_iter)]\n        for thread in self.prefetch_threads:\n            thread.setDaemon(True)\n            thread.start()\n\n    def __del__(self):\n        self.started = False\n        for i in self.data_taken:\n            i.set()\n        for thread in self.prefetch_threads:\n            thread.join()\n\n    @property\n    def provide_data(self):\n        if self.rename_data is None:\n            return sum([i.provide_data for i in self.iters], [])\n        else:\n            return sum([[\n                DataDesc(r[x.name], x.shape, x.dtype)\n                if isinstance(x, DataDesc) else DataDesc(*x)\n                for x in i.provide_data\n            ] for r, i in zip(self.rename_data, self.iters)], [])\n\n    @property\n    def provide_label(self):\n        if self.rename_label is None:\n            return sum([i.provide_label for i in self.iters], [])\n        else:\n            return sum([[\n                DataDesc(r[x.name], x.shape, x.dtype)\n                if isinstance(x, DataDesc) else DataDesc(*x)\n                for x in i.provide_label\n            ] for r, i in zip(self.rename_label, self.iters)], [])\n\n    def reset(self):\n        for i in self.data_ready:\n            i.wait()\n        for i in self.iters:\n            i.reset()\n        for i in self.data_ready:\n            i.clear()\n        for i in self.data_taken:\n            i.set()\n\n    def iter_next(self):\n        for i in self.data_ready:\n            i.wait()\n        if self.next_batch[0] is None:\n            for i in self.next_batch:\n                assert i is None, \"Number of entry mismatches between iterators\"\n            return False\n        else:\n            for batch in self.next_batch:\n                assert batch.pad == self.next_batch[0].pad, \\\n                    \"Number of entry mismatches between iterators\"\n            self.current_batch = DataBatch(sum([batch.data for batch in self.next_batch], []),\n                                           sum([batch.label for batch in self.next_batch], []),\n                                           self.next_batch[0].pad,\n                                           self.next_batch[0].index,\n                                           provide_data=self.provide_data,\n                                           provide_label=self.provide_label)\n            for i in self.data_ready:\n                i.clear()\n            for i in self.data_taken:\n                i.set()\n            return True\n\n    def next(self):\n        if self.iter_next():\n            return self.current_batch\n        else:\n            raise StopIteration\n\n    def getdata(self):\n        return self.current_batch.data\n\n    def getlabel(self):\n        return self.current_batch.label\n\n    def getindex(self):\n        return self.current_batch.index\n\n    def getpad(self):\n        return self.current_batch.pad\n\n\nclass NDArrayIter(DataIter):\n    \"\"\"Returns an iterator for ``mx.nd.NDArray``, ``numpy.ndarray``, ``h5py.Dataset``\n    ``mx.nd.sparse.CSRNDArray`` or ``scipy.sparse.csr_matrix``.\n\n    Examples\n    --------\n    >>> data = np.arange(40).reshape((10,2,2))\n    >>> labels = np.ones([10, 1])\n    >>> dataiter = mx.io.NDArrayIter(data, labels, 3, True, last_batch_handle='discard')\n    >>> for batch in dataiter:\n    ...     print batch.data[0].asnumpy()\n    ...     batch.data[0].shape\n    ...\n    [[[ 36.  37.]\n      [ 38.  39.]]\n     [[ 16.  17.]\n      [ 18.  19.]]\n     [[ 12.  13.]\n      [ 14.  15.]]]\n    (3L, 2L, 2L)\n    [[[ 32.  33.]\n      [ 34.  35.]]\n     [[  4.   5.]\n      [  6.   7.]]\n     [[ 24.  25.]\n      [ 26.  27.]]]\n    (3L, 2L, 2L)\n    [[[  8.   9.]\n      [ 10.  11.]]\n     [[ 20.  21.]\n      [ 22.  23.]]\n     [[ 28.  29.]\n      [ 30.  31.]]]\n    (3L, 2L, 2L)\n    >>> dataiter.provide_data # Returns a list of `DataDesc`\n    [DataDesc[data,(3, 2L, 2L),<type 'numpy.float32'>,NCHW]]\n    >>> dataiter.provide_label # Returns a list of `DataDesc`\n    [DataDesc[softmax_label,(3, 1L),<type 'numpy.float32'>,NCHW]]\n\n    In the above example, data is shuffled as `shuffle` parameter is set to `True`\n    and remaining examples are discarded as `last_batch_handle` parameter is set to `discard`.\n\n    Usage of `last_batch_handle` parameter:\n\n    >>> dataiter = mx.io.NDArrayIter(data, labels, 3, True, last_batch_handle='pad')\n    >>> batchidx = 0\n    >>> for batch in dataiter:\n    ...     batchidx += 1\n    ...\n    >>> batchidx  # Padding added after the examples read are over. So, 10/3+1 batches are created.\n    4\n    >>> dataiter = mx.io.NDArrayIter(data, labels, 3, True, last_batch_handle='discard')\n    >>> batchidx = 0\n    >>> for batch in dataiter:\n    ...     batchidx += 1\n    ...\n    >>> batchidx # Remaining examples are discarded. So, 10/3 batches are created.\n    3\n    >>> dataiter = mx.io.NDArrayIter(data, labels, 3, False, last_batch_handle='roll_over')\n    >>> batchidx = 0\n    >>> for batch in dataiter:\n    ...     batchidx += 1\n    ...\n    >>> batchidx # Remaining examples are rolled over to the next iteration.\n    3\n    >>> dataiter.reset()\n    >>> dataiter.next().data[0].asnumpy()\n    [[[ 36.  37.]\n      [ 38.  39.]]\n     [[ 0.  1.]\n      [ 2.  3.]]\n     [[ 4.  5.]\n      [ 6.  7.]]]\n    (3L, 2L, 2L)\n\n    `NDArrayIter` also supports multiple input and labels.\n\n    >>> data = {'data1':np.zeros(shape=(10,2,2)), 'data2':np.zeros(shape=(20,2,2))}\n    >>> label = {'label1':np.zeros(shape=(10,1)), 'label2':np.zeros(shape=(20,1))}\n    >>> dataiter = mx.io.NDArrayIter(data, label, 3, True, last_batch_handle='discard')\n\n    `NDArrayIter` also supports ``mx.nd.sparse.CSRNDArray``\n    with `last_batch_handle` set to `discard`.\n\n    >>> csr_data = mx.nd.array(np.arange(40).reshape((10,4))).tostype('csr')\n    >>> labels = np.ones([10, 1])\n    >>> dataiter = mx.io.NDArrayIter(csr_data, labels, 3, last_batch_handle='discard')\n    >>> [batch.data[0] for batch in dataiter]\n    [\n    <CSRNDArray 3x4 @cpu(0)>,\n    <CSRNDArray 3x4 @cpu(0)>,\n    <CSRNDArray 3x4 @cpu(0)>]\n\n    Parameters\n    ----------\n    data: array or list of array or dict of string to array\n        The input data.\n    label: array or list of array or dict of string to array, optional\n        The input label.\n    batch_size: int\n        Batch size of data.\n    shuffle: bool, optional\n        Whether to shuffle the data.\n        Only supported if no h5py.Dataset inputs are used.\n    last_batch_handle : str, optional\n        How to handle the last batch. This parameter can be 'pad', 'discard' or\n        'roll_over'.\n        If 'pad', the last batch will be padded with data starting from the begining\n        If 'discard', the last batch will be discarded\n        If 'roll_over', the remaining elements will be rolled over to the next iteration and\n        note that it is intended for training and can cause problems if used for prediction.\n    data_name : str, optional\n        The data name.\n    label_name : str, optional\n        The label name.\n    \"\"\"\n    def __init__(self, data, label=None, batch_size=1, shuffle=False,\n                 last_batch_handle='pad', data_name='data',\n                 label_name='softmax_label'):\n        super(NDArrayIter, self).__init__(batch_size)\n\n        self.data = _init_data(data, allow_empty=False, default_name=data_name)\n        self.label = _init_data(label, allow_empty=True, default_name=label_name)\n\n        if ((_has_instance(self.data, CSRNDArray) or\n             _has_instance(self.label, CSRNDArray)) and\n                (last_batch_handle != 'discard')):\n            raise NotImplementedError(\"`NDArrayIter` only supports ``CSRNDArray``\" \\\n                                      \" with `last_batch_handle` set to `discard`.\")\n\n        self.idx = np.arange(self.data[0][1].shape[0])\n        self.shuffle = shuffle\n        self.last_batch_handle = last_batch_handle\n        self.batch_size = batch_size\n        self.cursor = -self.batch_size\n        self.num_data = self.idx.shape[0]\n        # shuffle\n        self.reset()\n\n        self.data_list = [x[1] for x in self.data] + [x[1] for x in self.label]\n        self.num_source = len(self.data_list)\n        # used for 'roll_over'\n        self._cache_data = None\n        self._cache_label = None\n\n    @property\n    def provide_data(self):\n        \"\"\"The name and shape of data provided by this iterator.\"\"\"\n        return [\n            DataDesc(k, tuple([self.batch_size] + list(v.shape[1:])), v.dtype)\n            for k, v in self.data\n        ]\n\n    @property\n    def provide_label(self):\n        \"\"\"The name and shape of label provided by this iterator.\"\"\"\n        batch_axis = self.layout.find('N')\n        return [\n            DataDesc(k, tuple(list(v.shape[:batch_axis]) + \\\n                              [self.batch_size] + list(v.shape[batch_axis + 1:])),\n                     v.dtype, layout=self.layout)\n            for k, v in self.label\n        ]\n\n    def hard_reset(self):\n        \"\"\"Ignore roll over data and set to start.\"\"\"\n        if self.shuffle:\n            self._shuffle_data()\n        self.cursor = -self.batch_size\n        self._cache_data = None\n        self._cache_label = None\n\n    def reset(self):\n        \"\"\"Resets the iterator to the beginning of the data.\"\"\"\n        if self.shuffle:\n            self._shuffle_data()\n        # the range below indicate the last batch\n        if self.last_batch_handle == 'roll_over' and \\\n            self.num_data - self.batch_size < self.cursor < self.num_data:\n            # (self.cursor - self.num_data) represents the data we have for the last batch\n            self.cursor = self.cursor - self.num_data - self.batch_size\n        else:\n            self.cursor = -self.batch_size\n\n    def iter_next(self):\n        \"\"\"Increments the coursor by batch_size for next batch\n        and check current cursor if it exceed the number of data points.\"\"\"\n        self.cursor += self.batch_size\n        return self.cursor < self.num_data\n\n    def next(self):\n        \"\"\"Returns the next batch of data.\"\"\"\n        if not self.iter_next():\n            raise StopIteration\n        data = self.getdata()\n        label = self.getlabel()\n        # iter should stop when last batch is not complete\n        if data[0].shape[0] != self.batch_size:\n        # in this case, cache it for next epoch\n            self._cache_data = data\n            self._cache_label = label\n            raise StopIteration\n        return DataBatch(data=data, label=label, \\\n            pad=self.getpad(), index=None)\n\n    def _getdata(self, data_source, start=None, end=None):\n        \"\"\"Load data from underlying arrays.\"\"\"\n        assert start is not None or end is not None, 'should at least specify start or end'\n        start = start if start is not None else 0\n        if end is None:\n            end = data_source[0][1].shape[0] if data_source else 0\n        s = slice(start, end)\n        return [\n            x[1][s]\n            if isinstance(x[1], (np.ndarray, NDArray)) else\n            # h5py (only supports indices in increasing order)\n            array(x[1][sorted(self.idx[s])][[\n                list(self.idx[s]).index(i)\n                for i in sorted(self.idx[s])\n            ]]) for x in data_source\n        ]\n\n    def _concat(self, first_data, second_data):\n        \"\"\"Helper function to concat two NDArrays.\"\"\"\n        if (not first_data) or (not second_data):\n            return first_data if first_data else second_data\n        assert len(first_data) == len(\n            second_data), 'data source should contain the same size'\n        return [\n            concat(\n                first_data[i],\n                second_data[i],\n                dim=0\n            ) for i in range(len(first_data))\n        ]\n\n    def _tile(self, data, repeats):\n        if not data:\n            return []\n        res = []\n        for datum in data:\n            reps = [1] * len(datum.shape)\n            reps[0] = repeats\n            res.append(tile(datum, reps))\n        return res\n\n    def _batchify(self, data_source):\n        \"\"\"Load data from underlying arrays, internal use only.\"\"\"\n        assert self.cursor < self.num_data, 'DataIter needs reset.'\n        # first batch of next epoch with 'roll_over'\n        if self.last_batch_handle == 'roll_over' and \\\n            -self.batch_size < self.cursor < 0:\n            assert self._cache_data is not None or self._cache_label is not None, \\\n                'next epoch should have cached data'\n            cache_data = self._cache_data if self._cache_data is not None else self._cache_label\n            second_data = self._getdata(\n                data_source, end=self.cursor + self.batch_size)\n            if self._cache_data is not None:\n                self._cache_data = None\n            else:\n                self._cache_label = None\n            return self._concat(cache_data, second_data)\n        # last batch with 'pad'\n        elif self.last_batch_handle == 'pad' and \\\n            self.cursor + self.batch_size > self.num_data:\n            pad = self.batch_size - self.num_data + self.cursor\n            first_data = self._getdata(data_source, start=self.cursor)\n            if pad > self.num_data:\n                repeats = pad // self.num_data\n                second_data = self._tile(self._getdata(data_source, end=self.num_data), repeats)\n                if pad % self.num_data != 0:\n                    second_data = self._concat(second_data, self._getdata(data_source, end=pad % self.num_data))\n            else:\n                second_data = self._getdata(data_source, end=pad)\n            return self._concat(first_data, second_data)\n        # normal case\n        else:\n            if self.cursor + self.batch_size < self.num_data:\n                end_idx = self.cursor + self.batch_size\n            # get incomplete last batch\n            else:\n                end_idx = self.num_data\n            return self._getdata(data_source, self.cursor, end_idx)\n\n    def getdata(self):\n        \"\"\"Get data.\"\"\"\n        return self._batchify(self.data)\n\n    def getlabel(self):\n        \"\"\"Get label.\"\"\"\n        return self._batchify(self.label)\n\n    def getpad(self):\n        \"\"\"Get pad value of DataBatch.\"\"\"\n        if self.last_batch_handle == 'pad' and \\\n           self.cursor + self.batch_size > self.num_data:\n            return self.cursor + self.batch_size - self.num_data\n        # check the first batch\n        elif self.last_batch_handle == 'roll_over' and \\\n            -self.batch_size < self.cursor < 0:\n            return -self.cursor\n        else:\n            return 0\n\n    def _shuffle_data(self):\n        \"\"\"Shuffle the data.\"\"\"\n        # shuffle index\n        np.random.shuffle(self.idx)\n        # get the data by corresponding index\n        self.data = _getdata_by_idx(self.data, self.idx)\n        self.label = _getdata_by_idx(self.label, self.idx)\n\nclass MXDataIter(DataIter):\n    \"\"\"A python wrapper a C++ data iterator.\n\n    This iterator is the Python wrapper to all native C++ data iterators, such\n    as `CSVIter`, `ImageRecordIter`, `MNISTIter`, etc. When initializing\n    `CSVIter` for example, you will get an `MXDataIter` instance to use in your\n    Python code. Calls to `next`, `reset`, etc will be delegated to the\n    underlying C++ data iterators.\n\n    Usually you don't need to interact with `MXDataIter` directly unless you are\n    implementing your own data iterators in C++. To do that, please refer to\n    examples under the `src/io` folder.\n\n    Parameters\n    ----------\n    handle : DataIterHandle, required\n        The handle to the underlying C++ Data Iterator.\n    data_name : str, optional\n        Data name. Default to \"data\".\n    label_name : str, optional\n        Label name. Default to \"softmax_label\".\n\n    See Also\n    --------\n    src/io : The underlying C++ data iterator implementation, e.g., `CSVIter`.\n    \"\"\"\n    def __init__(self, handle, data_name='data', label_name='softmax_label', **kwargs):\n        super(MXDataIter, self).__init__()\n        from ..ndarray import _ndarray_cls\n        from ..numpy.multiarray import _np_ndarray_cls\n        self._create_ndarray_fn = _np_ndarray_cls if is_np_array() else _ndarray_cls\n        self.handle = handle\n        self._kwargs = kwargs\n        # debug option, used to test the speed with io effect eliminated\n        self._debug_skip_load = False\n\n        # load the first batch to get shape information\n        self.first_batch = None\n        self.first_batch = self.next()\n        data = self.first_batch.data[0]\n        label = self.first_batch.label[0]\n\n        # properties\n        self.provide_data = [DataDesc(data_name, data.shape, data.dtype)]\n        self.provide_label = [DataDesc(label_name, label.shape, label.dtype)]\n        self.batch_size = data.shape[0]\n\n    def __del__(self):\n        check_call(_LIB.MXDataIterFree(self.handle))\n\n    def debug_skip_load(self):\n        # Set the iterator to simply return always first batch. This can be used\n        # to test the speed of network without taking the loading delay into\n        # account.\n        self._debug_skip_load = True\n        logging.info('Set debug_skip_load to be true, will simply return first batch')\n\n    def reset(self):\n        self._debug_at_begin = True\n        self.first_batch = None\n        check_call(_LIB.MXDataIterBeforeFirst(self.handle))\n\n    def next(self):\n        if self._debug_skip_load and not self._debug_at_begin:\n            return  DataBatch(data=[self.getdata()], label=[self.getlabel()], pad=self.getpad(),\n                              index=self.getindex())\n        if self.first_batch is not None:\n            batch = self.first_batch\n            self.first_batch = None\n            return batch\n        self._debug_at_begin = False\n        next_res = ctypes.c_int(0)\n        check_call(_LIB.MXDataIterNext(self.handle, ctypes.byref(next_res)))\n        if next_res.value:\n            return DataBatch(data=[self.getdata()], label=[self.getlabel()], pad=self.getpad(),\n                             index=self.getindex())\n        else:\n            raise StopIteration\n\n    def iter_next(self):\n        if self.first_batch is not None:\n            return True\n        next_res = ctypes.c_int(0)\n        check_call(_LIB.MXDataIterNext(self.handle, ctypes.byref(next_res)))\n        return next_res.value\n\n    def getdata(self):\n        hdl = NDArrayHandle()\n        check_call(_LIB.MXDataIterGetData(self.handle, ctypes.byref(hdl)))\n        return self._create_ndarray_fn(hdl, False)\n\n    def getlabel(self):\n        hdl = NDArrayHandle()\n        check_call(_LIB.MXDataIterGetLabel(self.handle, ctypes.byref(hdl)))\n        return self._create_ndarray_fn(hdl, False)\n\n    def getindex(self):\n        index_size = ctypes.c_uint64(0)\n        index_data = ctypes.POINTER(ctypes.c_uint64)()\n        check_call(_LIB.MXDataIterGetIndex(self.handle,\n                                           ctypes.byref(index_data),\n                                           ctypes.byref(index_size)))\n        if index_size.value:\n            address = ctypes.addressof(index_data.contents)\n            dbuffer = (ctypes.c_uint64* index_size.value).from_address(address)\n            np_index = np.frombuffer(dbuffer, dtype=np.uint64)\n            return np_index.copy()\n        else:\n            return None\n\n    def getpad(self):\n        pad = ctypes.c_int(0)\n        check_call(_LIB.MXDataIterGetPadNum(self.handle, ctypes.byref(pad)))\n        return pad.value\n\n    def getitems(self):\n        output_vars = ctypes.POINTER(NDArrayHandle)()\n        num_output = ctypes.c_int(0)\n        check_call(_LIB.MXDataIterGetItems(self.handle,\n                                           ctypes.byref(num_output),\n                                           ctypes.byref(output_vars)))\n        out = [self._create_ndarray_fn(ctypes.cast(output_vars[i], NDArrayHandle),\n                                       False) for i in range(num_output.value)]\n        return tuple(out)\n\n    def __len__(self):\n        length = ctypes.c_int64(-1)\n        check_call(_LIB.MXDataIterGetLenHint(self.handle, ctypes.byref(length)))\n        if length.value < 0:\n            return 0\n        return length.value\n\n\ndef _make_io_iterator(handle):\n    \"\"\"Create an io iterator by handle.\"\"\"\n    name = ctypes.c_char_p()\n    desc = ctypes.c_char_p()\n    num_args = mx_uint()\n    arg_names = ctypes.POINTER(ctypes.c_char_p)()\n    arg_types = ctypes.POINTER(ctypes.c_char_p)()\n    arg_descs = ctypes.POINTER(ctypes.c_char_p)()\n\n    check_call(_LIB.MXDataIterGetIterInfo( \\\n            handle, ctypes.byref(name), ctypes.byref(desc), \\\n            ctypes.byref(num_args), \\\n            ctypes.byref(arg_names), \\\n            ctypes.byref(arg_types), \\\n            ctypes.byref(arg_descs)))\n    iter_name = py_str(name.value)\n\n    narg = int(num_args.value)\n    param_str = _build_param_doc(\n        [py_str(arg_names[i]) for i in range(narg)],\n        [py_str(arg_types[i]) for i in range(narg)],\n        [py_str(arg_descs[i]) for i in range(narg)])\n\n    doc_str = (f'{desc.value}\\n\\n' +\n               f'{param_str}\\n' +\n               'Returns\\n' +\n               '-------\\n' +\n               'MXDataIter\\n'+\n               '    The result iterator.')\n\n    def creator(*args, **kwargs):\n        \"\"\"Create an iterator.\n        The parameters listed below can be passed in as keyword arguments.\n\n        Parameters\n        ----------\n        name : string, required.\n            Name of the resulting data iterator.\n\n        Returns\n        -------\n        dataiter: Dataiter\n            The resulting data iterator.\n        \"\"\"\n        param_keys = []\n        param_vals = []\n\n        for k, val in kwargs.items():\n            if iter_name == 'ThreadedDataLoader':\n                # convert ndarray to handle\n                if hasattr(val, 'handle'):\n                    val = val.handle.value\n                elif isinstance(val, (tuple, list)):\n                    val = [vv.handle.value if hasattr(vv, 'handle') else vv for vv in val]\n                elif isinstance(getattr(val, '_iter', None), MXDataIter):\n                    val = val._iter.handle.value\n            param_keys.append(k)\n            param_vals.append(str(val))\n        # create atomic symbol\n        param_keys = c_str_array(param_keys)\n        param_vals = c_str_array(param_vals)\n        iter_handle = DataIterHandle()\n        check_call(_LIB.MXDataIterCreateIter(\n            handle,\n            mx_uint(len(param_keys)),\n            param_keys, param_vals,\n            ctypes.byref(iter_handle)))\n\n        if len(args):\n            raise TypeError(f'{iter_name} can only accept keyword arguments')\n\n        return MXDataIter(iter_handle, **kwargs)\n\n    creator.__name__ = iter_name\n    creator.__doc__ = doc_str\n    return creator\n\ndef _init_io_module():\n    \"\"\"List and add all the data iterators to current module.\"\"\"\n    plist = ctypes.POINTER(ctypes.c_void_p)()\n    size = ctypes.c_uint()\n    check_call(_LIB.MXListDataIters(ctypes.byref(size), ctypes.byref(plist)))\n    module_obj = sys.modules[__name__]\n    for i in range(size.value):\n        hdl = ctypes.c_void_p(plist[i])\n        dataiter = _make_io_iterator(hdl)\n        setattr(module_obj, dataiter.__name__, dataiter)\n\n_init_io_module()\n"
  },
  {
    "path": "python/mxnet/io/utils.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"utility functions for io.py\"\"\"\nfrom collections import OrderedDict\n\nimport numpy as np\ntry:\n    import h5py\nexcept ImportError:\n    h5py = None\n\nfrom ..ndarray.sparse import CSRNDArray\nfrom ..ndarray.sparse import array as sparse_array\nfrom ..ndarray import NDArray\nfrom ..ndarray import array\n\ndef _init_data(data, allow_empty, default_name):\n    \"\"\"Convert data into canonical form.\"\"\"\n    assert (data is not None) or allow_empty\n    if data is None:\n        data = []\n\n    if isinstance(data, (np.ndarray, NDArray, h5py.Dataset)\n                  if h5py else (np.ndarray, NDArray)):\n        data = [data]\n    if isinstance(data, list):\n        if not allow_empty:\n            assert(len(data) > 0)\n        if len(data) == 1:\n            data = OrderedDict([(default_name, data[0])])  # pylint: disable=redefined-variable-type\n        else:\n            data = OrderedDict(  # pylint: disable=redefined-variable-type\n                [(f'_{i}_{default_name}', d) for i, d in enumerate(data)])\n    if not isinstance(data, dict):\n        raise TypeError(\"Input must be NDArray, numpy.ndarray, h5py.Dataset \" +\n                        \"a list of them or dict with them as values\")\n    for k, v in data.items():\n        if not isinstance(v, (NDArray, h5py.Dataset) if h5py else NDArray):\n            try:\n                data[k] = array(v)\n            except:\n                raise TypeError((f\"Invalid type '{type(v)}' for {k}, \") +\n                                \"should be NDArray, numpy.ndarray or h5py.Dataset\")\n\n    return list(sorted(data.items()))\n\n\ndef _has_instance(data, dtype):\n    \"\"\"Return True if ``data`` has instance of ``dtype``.\n    This function is called after _init_data.\n    ``data`` is a list of (str, NDArray)\"\"\"\n    for item in data:\n        _, arr = item\n        if isinstance(arr, dtype):\n            return True\n    return False\n\n\ndef _getdata_by_idx(data, idx):\n    \"\"\"Shuffle the data.\"\"\"\n    shuffle_data = []\n\n    for k, v in data:\n        if (isinstance(v, h5py.Dataset) if h5py else False):\n            shuffle_data.append((k, v))\n        elif isinstance(v, CSRNDArray):\n            shuffle_data.append((k, sparse_array(v.asscipy()[idx], v.context)))\n        else:\n            shuffle_data.append((k, array(v.asnumpy()[idx], v.context)))\n\n    return shuffle_data\n"
  },
  {
    "path": "python/mxnet/kvstore/__init__.py",
    "content": "#!/usr/bin/env python\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Key-value store for distributed communication\"\"\"\nfrom .kvstore import *\nfrom .base import *\nfrom .kvstore_server import *\nfrom .byteps import *\nfrom .horovod import *\n"
  },
  {
    "path": "python/mxnet/kvstore/base.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\" Key value store interface of MXNet for parameter synchronization.\"\"\"\n\nfrom array import array\nimport ctypes\nimport warnings\nfrom ..ndarray import NDArray\nfrom ..base import _LIB, c_str_array, c_handle_array, c_array, c_array_buf, c_str\nfrom ..base import check_call, string_types\nfrom ..base import KVStoreHandle\nfrom ..profiler import set_kvstore_handle\n\n__all__ = ['create', 'KVStoreBase']\n\ndef _ctype_key_value(keys, vals):\n    \"\"\"Returns ctype arrays for the key-value args, and the whether string keys are used.\n    For internal use only.\n    \"\"\"\n    if isinstance(keys, (tuple, list)):\n        assert(len(keys) == len(vals))\n        c_keys = []\n        c_vals = []\n        use_str_keys = None\n        for key, val in zip(keys, vals):\n            c_key_i, c_val_i, str_keys_i = _ctype_key_value(key, val)\n            c_keys += c_key_i\n            c_vals += c_val_i\n            use_str_keys = str_keys_i if use_str_keys is None else use_str_keys\n            assert(use_str_keys == str_keys_i), \"inconsistent types of keys detected.\"\n        c_keys_arr = c_array(ctypes.c_char_p, c_keys) if use_str_keys \\\n                     else c_array(ctypes.c_int, c_keys)\n        c_vals_arr = c_array(ctypes.c_void_p, c_vals)\n        return (c_keys_arr, c_vals_arr, use_str_keys)\n\n    assert(isinstance(keys, (int,) + string_types)), \\\n           \"unexpected type for keys: \" + str(type(keys))\n    use_str_keys = isinstance(keys, string_types)\n    if isinstance(vals, NDArray):\n        c_keys = c_str_array([keys]) if use_str_keys \\\n                 else c_array_buf(ctypes.c_int, array('i', [keys]))\n        return (c_keys, c_handle_array([vals]), use_str_keys)\n    else:\n        for value in vals:\n            assert(isinstance(value, NDArray))\n        c_keys = c_str_array([keys] * len(vals)) if use_str_keys \\\n                 else c_array_buf(ctypes.c_int, array('i', [keys] * len(vals)))\n        return (c_keys, c_handle_array(vals), use_str_keys)\n\ndef _ctype_dict(param_dict):\n    \"\"\"Returns ctype arrays for keys and values(converted to strings) in a dictionary\"\"\"\n    assert(isinstance(param_dict, dict)), \\\n        \"unexpected type for param_dict: \" + str(type(param_dict))\n    c_keys = c_array(ctypes.c_char_p, [c_str(k) for k in param_dict.keys()])\n    c_vals = c_array(ctypes.c_char_p, [c_str(str(v)) for v in param_dict.values()])\n    return (c_keys, c_vals)\n\nclass KVStoreBase(object):\n    \"\"\"An abstract key-value store interface for data parallel training.\"\"\"\n\n    def broadcast(self, key, value, out, priority=0):\n        \"\"\" Broadcast the `value` NDArray at rank 0 to all ranks,\n        and store the result in `out`\n\n        Parameters\n        ----------\n        key : str or int\n            The key.\n\n        value : NDArray\n            The value corresponding to the key to broadcast\n\n        out : NDArray, or list of NDArray\n            Values corresponding to the key to store the result\n\n        priority : int, optional\n            The priority of the operation.\n            Higher priority operations are likely to be executed before other actions.\n        \"\"\"\n        raise NotImplementedError()\n\n    def pushpull(self, key, value, out=None, priority=0):\n        \"\"\" Performs push and pull a single value or a sequence of values from the store.\n\n        This function is coalesced form of push and pull operations.\n\n        `value` is pushed to the kvstore server for summation with the specified keys,\n        and the results are pulled from the server to `out`. If `out` is not specified\n        the pulled values are written to `value`.\n\n        Note that for allreduce based approaches such as horovod, there is no notion of\n        server or store. This function performs allreduce.\n\n        Parameters\n        ----------\n        key : str or int\n            The key.\n\n        value : NDArray, or list of NDArray\n            Values corresponding to the keys.\n\n        out: NDArray, or list of NDArray\n            Values corresponding to the key.\n\n        priority : int, optional\n            The priority of the operation.\n            Higher priority operations are likely to be executed before other actions.\n        \"\"\"\n        raise NotImplementedError()\n\n    def set_optimizer(self, optimizer):\n        \"\"\" Registers an optimizer with the kvstore.\n\n        When using a single machine, this function updates the local optimizer.\n        If using multiple machines and this operation is invoked from a worker node,\n        it will serialized the optimizer with pickle and send it to all servers.\n        The function returns after all servers have been updated.\n\n        Parameters\n        ----------\n        optimizer : KVStoreBase\n            The new optimizer for the store\n        \"\"\"\n        raise NotImplementedError()\n\n    OPTIMIZER = 'optimizer'\n\n    def is_capable(self, capability):\n        \"\"\"Queries if the KVStore type supports certain capability, such as optimizer algorithm,\n        gradient compression, sparsity, etc.\n\n        Parameters\n        ----------\n        capability: str\n            The capability to query\n\n        Returns\n        -------\n        result : bool\n            Whether the capability is supported or not.\n        \"\"\"\n        raise NotImplementedError()\n\n    def save_optimizer_states(self, fname, dump_optimizer=False):\n        \"\"\"Saves the optimizer (updater) state to a file. This is often used when checkpointing\n        the model during training.\n\n        Parameters\n        ----------\n        fname : str\n            Path to the output states file.\n        dump_optimizer : bool, default False\n            Whether to also save the optimizer itself. This would also save optimizer\n            information such as learning rate and weight decay schedules.\n        \"\"\"\n        raise NotImplementedError()\n\n    def load_optimizer_states(self, fname):\n        \"\"\"Loads the optimizer (updater) state from the file.\n\n        Parameters\n        ----------\n        fname : str\n            Path to input states file.\n        \"\"\"\n        raise NotImplementedError()\n\n    @property\n    def type(self):\n        \"\"\" Returns the type of this kvstore backend.\n\n        Returns\n        -------\n        type : str\n            the string type\n        \"\"\"\n        raise NotImplementedError()\n\n    @property\n    def rank(self):\n        \"\"\" Returns the rank of this worker node.\n\n        Returns\n        -------\n        rank : int\n            The rank of this node, which is in range [0, num_workers())\n        \"\"\"\n        raise NotImplementedError()\n\n    @property\n    def num_workers(self):\n        \"\"\"Returns the number of worker nodes.\n\n        Returns\n        -------\n        size :int\n            The number of worker nodes.\n        \"\"\"\n        raise NotImplementedError()\n\n    kv_registry = {}\n\n    @staticmethod\n    def register(klass):\n        \"\"\"Registers a new KVStore.\n        Once a kvstore is registered, we can create an instance of this\n        kvstore with `create` later.\n\n        Examples\n        --------\n        >>> @mx.kvstore.KVStoreBase.register\n        ... class MyKVStore(mx.kvstore.KVStoreBase):\n        ...     pass\n        >>> kv = mx.kv.create('MyKVStore')\n        >>> print(type(kv))\n        <class '__main__.MyKVStore'>\n        \"\"\"\n        assert(isinstance(klass, type))\n        name = klass.__name__.lower()\n        if name in KVStoreBase.kv_registry:\n            warnings.warn(f'WARNING: New kvstore {klass.__module__}.{klass.__name__} is overriding '\n                          'existing kvstore '\n                          f'{KVStoreBase.kv_registry[name].__module__}.{KVStoreBase.kv_registry[name].__name__}')\n        KVStoreBase.kv_registry[name] = klass\n        return klass\n\n@KVStoreBase.register\nclass TestStore(KVStoreBase):\n    \"\"\"A key-value store for testing.\"\"\"\n\n    def broadcast(self, key, value, out, priority=0):\n        \"\"\" Broadcast the `value` NDArray at rank 0 to all ranks,\n        and store the result in `out`\n\n        Parameters\n        ----------\n        key : str or int\n            The key.\n\n        value : NDArray\n            The value corresponding to the key to broadcast\n\n        out : NDArray, or list of NDArray\n            Values corresponding to the key to store the result\n\n        priority : int, optional\n            The priority of the operation.\n            Higher priority operations are likely to be executed before other actions.\n        \"\"\"\n        out = out if isinstance(out, list) else [out]\n        for o in out:\n            o[:] = value\n\n    def pushpull(self, key, value, out=None, priority=0):\n        \"\"\" Performs push and pull a single value or a sequence of values from the store.\n\n        This function is coalesced form of push and pull operations.\n\n        `value` is pushed to the kvstore server for summation with the specified keys,\n        and the results are pulled from the server to `out`. If `out` is not specified\n        the pulled values are written to `value`.\n\n        Parameters\n        ----------\n        key : str or int\n            The key.\n\n        value : NDArray, or list of NDArray\n            Values corresponding to the keys.\n\n        out: NDArray, or list of NDArray\n            Values corresponding to the key.\n\n        priority : int, optional\n            The priority of the operation.\n            Higher priority operations are likely to be executed before other actions.\n        \"\"\"\n        ctx = value[0].context\n        if isinstance(value, NDArray):\n            if out is not None:\n                out = out if isinstance(out, list) else [out]\n                for o in out:\n                    o[:] = value\n        else:\n            reduced_value = sum([val.as_in_context(ctx) for val in value])\n            if out is None:\n                for v in value:\n                    v[:] = reduced_value\n            else:\n                out = out if isinstance(out, list) else [out]\n                for o in out:\n                    o[:] = reduced_value\n\n    @staticmethod\n    def is_capable(capability):\n        \"\"\"Queries if the KVStore type supports certain capability, such as optimizer algorithm,\n        gradient compression, sparsity, etc.\n        If the kvstore does not store weights in server part, then no optimizer is supported,\n        this function will return False.\n\n        Parameters\n        ----------\n        capability: str\n            The capability to query\n\n        Returns\n        -------\n        result : bool\n            Whether the capability is supported or not.\n        \"\"\"\n        if capability.lower() == KVStoreBase.OPTIMIZER:\n            return False\n        else:\n            raise ValueError('Unknown capability: {}'.format(capability))\n\n    @property\n    def type(self):\n        \"\"\" Returns the type of this kvstore.\n\n        Returns\n        -------\n        type : str\n            the string type\n        \"\"\"\n        return 'teststore'\n\n    @property\n    def rank(self):\n        \"\"\" Returns the rank of this worker node.\n\n        Returns\n        -------\n        rank : int\n            The rank of this node, which is in range [0, num_workers())\n        \"\"\"\n        return 0\n\n    @property\n    def num_workers(self):\n        \"\"\"Returns the number of worker nodes.\n\n        Returns\n        -------\n        size :int\n            The number of worker nodes.\n        \"\"\"\n        return 1\n\n    def set_optimizer(self, optimizer):\n        \"\"\" Registers an optimizer with the kvstore.\n\n        When using a single machine, this function updates the local optimizer.\n        If using multiple machines and this operation is invoked from a worker node,\n        it will serialized the optimizer with pickle and send it to all servers.\n        The function returns after all servers have been updated.\n\n        Parameters\n        ----------\n        optimizer : KVStoreBase\n            The new optimizer for the store\n        \"\"\"\n        raise NotImplementedError()\n\n    def save_optimizer_states(self, fname, dump_optimizer=False):\n        \"\"\"Saves the optimizer (updater) state to a file. This is often used when checkpointing\n        the model during training.\n\n        Parameters\n        ----------\n        fname : str\n            Path to the output states file.\n        dump_optimizer : bool, default False\n            Whether to also save the optimizer itself. This would also save optimizer\n            information such as learning rate and weight decay schedules.\n        \"\"\"\n        raise NotImplementedError()\n\n    def load_optimizer_states(self, fname):\n        \"\"\"Loads the optimizer (updater) state from the file.\n\n        Parameters\n        ----------\n        fname : str\n            Path to input states file.\n        \"\"\"\n        raise NotImplementedError()\n\ndef create(name='local'):\n    \"\"\"Creates a new KVStore.\n\n    For single machine training, there are two commonly used types:\n\n    ``local``: Copies all gradients to CPU memory and updates weights there.\n\n    ``device``: Aggregates gradients and updates weights on GPUs. With this setting,\n    the KVStore also attempts to use GPU peer-to-peer communication,\n    potentially accelerating the communication.\n\n    For distributed training, KVStore also supports a number of types:\n\n    ``dist_sync``: Behaves similarly to ``local`` but with one major difference.\n    With ``dist_sync``, batch-size now means the batch size used on each machine.\n    So if there are ``n`` machines and we use batch size ``b``,\n    then ``dist_sync`` behaves like ``local`` with batch size ``n * b``.\n\n    ``dist_device_sync``: Identical to ``dist_sync`` with the difference similar\n    to ``device`` vs ``local``.\n\n    ``dist_async``: Performs asynchronous updates.\n    The weights are updated whenever gradients are received from any machine.\n    No two updates happen on the same weight at the same time. However, the order is not\n    guaranteed.\n\n    ``byteps``: Use byteps as broadcast/pushpull backend.\n    This kind of kvstore doesn't store weights, thus there won't be optimizer in this kvstore server.\n    Byteps doesn't support pure cpu training, so be sure to enable gpu training when using this kvstore.\n\n    Parameters\n    ----------\n    name : {'local', 'device', 'nccl', 'dist_sync', 'dist_device_sync', 'dist_async', 'horovod', 'byteps'}\n        The type of KVStore.\n\n    Returns\n    -------\n    kv : KVStoreBase\n        The created KVStore.\n    \"\"\"\n    if not isinstance(name, string_types):\n        raise TypeError('name must be a string')\n    name = name.lower()\n\n    # first lookup the registry\n    if name in KVStoreBase.kv_registry:\n        return KVStoreBase.kv_registry[name]()\n    else:\n        # fall back to the native kvstore implementation\n        handle = KVStoreHandle()\n        check_call(_LIB.MXKVStoreCreate(c_str(name),\n                                        ctypes.byref(handle)))\n        from .kvstore import KVStore\n        kv = KVStore(handle)\n        set_kvstore_handle(kv.handle)\n        return kv\n"
  },
  {
    "path": "python/mxnet/kvstore/byteps.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\" BytePS backend for MXNet KVStore\"\"\"\nfrom __future__ import absolute_import\n\nfrom ..ndarray import NDArray\nfrom .base import KVStoreBase\n\n__all__ = ['BytePS']\n\n\n@KVStoreBase.register\nclass BytePS(KVStoreBase):\n    \"\"\"BytePS backend for MXNet KVStore interface.\"\"\"\n\n    def __init__(self):\n        \"\"\"Initializes a new KVStore.\"\"\"\n        try:\n            import byteps.mxnet as bps\n            self.handle = bps\n        except ModuleNotFoundError as err:\n            print('Did not find BytePS library. Please install BytePS first')\n            raise err\n        except ImportError as err:\n            print('Did not find BytePS library. Please install BytePS first')\n            raise err\n        self.handle.init()\n\n    def broadcast(self, key, value, out, priority=0):\n        \"\"\" Broadcast the value NDArray at rank 0 to all ranks' out. If out is None,\n        the result is stored in `value`.\n\n        Parameters\n        ----------\n        key : str, or int\n            The keys.\n        value : NDArray, or list of NDArray\n            Values corresponding to the key.\n        out : NDArray, or lise of NDArray\n            Values corresponding to the keys.\n\n        Examples\n        --------\n        >>> # broadcast a single key-value pair\n        >>> shape = (2,3)\n        >>> kv = mx.kv.create('byteps')\n        >>> a = mx.nd.zeros(shape)\n        >>> kv.broadcast('3', mx.nd.ones(shape)*2, out=a)\n        >>> print a.asnumpy()\n        [[ 2.  2.  2.]\n        [ 2.  2.  2.]]\n        \"\"\"\n        # do not accept list or tuple for key/value\n        assert isinstance(key, (str, int))\n\n        # unpack the list if it contains just one NDArray\n        value = value[0] if isinstance(\n            value, list) and len(value) == 1 else value\n        assert isinstance(\n            value, NDArray), \"The type of value can only be NDArray or list of NDArray which has only one element.\"\n        assert value.context.device_type == 'gpu', \"Byteps KVStore only support GPU context for broadcast value.\"\n\n        # optimzation when out = value or out = [value]\n        if isinstance(out, (list, tuple)) and len(out) == 1:\n            inplace = value is out[0]\n        else:\n            inplace = value is out\n\n        if inplace:\n            broadcast_value = value\n        else:\n            broadcast_value = value.copy()\n        # for non-root-rank, assign value with 0, thus the result of pushpull will be\n        # equal to the value of root-rank, thus implementing broadcast.\n        root_rank = 0\n        if self.rank != root_rank:\n            broadcast_value.__imul__(0)\n        self.handle.byteps_declare_tensor(str(key))\n        self.handle.byteps_push_pull(broadcast_value, version=0, priority=priority,\n                                     name=str(key), is_average=False)\n        # Make sure tensors pushed to MXNet engine get processed such that all\n        # workers are synced before starting training.\n        broadcast_value.wait_to_read()\n\n        out = out if isinstance(out, list) else [out]\n        for o in out:\n            broadcast_value.copyto(o)\n\n    def pushpull(self, key, value, out=None, priority=0):\n        \"\"\" Performs push and pull a single value from the store.\n        This function is coalesced form of push and pull operations.\n        `value` is pushed to the kvstore server for the specified keys and the aggregated\n        values are pulled from the server to `out`. If `out` is not specified the pulled\n        values are written to `value`.\n\n        Parameters\n        ----------\n        key : str, or int\n            The key.\n        value : NDArray, or list of NDArray\n            Values corresponding to the key.\n        out: NDArray, or list of NDArray\n            Values corresponding to the key.\n        priority : int, optional\n            The priority of the operation.\n            Higher priority operations are likely to be executed before other actions.\n\n        Examples\n        --------\n        >>> # pushpull a single key-value pair\n        >>> kv.pushpull('3', mx.nd.ones(shape)*8, out=a)\n        >>> print a.asnumpy()\n        [[ 8.  8.  8.]\n        [ 8.  8.  8.]]\n        \"\"\"\n        # the most common operation operates on one NDArray as `value`, and\n        # `out` is set to None, for inplace pushpull.\n\n        assert isinstance(key, (str, int))\n\n        # unpack the list if it contains just one NDArray\n        value = value[0] if isinstance(\n            value, list) and len(value) == 1 else value\n        assert isinstance(\n            value, NDArray), \"The type of value can only be NDArray or list of NDArray which has only one element.\"\n        assert value.context.device_type == 'gpu', \"Byteps KVStore only support GPU context for pushpull value\"\n\n        # optimzation when out = value or out = [value]\n        if isinstance(out, (list, tuple)) and len(out) == 1:\n            inplace = value is out[0]\n        else:\n            inplace = value is out\n\n        if inplace:\n            pushpull_value = value\n        else:\n            pushpull_value = value.copy()\n\n        self.handle.byteps_declare_tensor(str(key))\n        self.handle.byteps_push_pull(pushpull_value, version=0, priority=priority,\n                                     name=str(key), is_average=False)\n\n        if out is not None:\n            out = out if isinstance(out, list) else [out]\n            for o in out:\n                pushpull_value.copyto(o)\n\n    @staticmethod\n    def is_capable(capability):\n        \"\"\"Queries if the KVStore type supports certain capability, such as optimizer algorithm,\n        gradient compression, sparsity, etc.\n        As byteps server does not store weight, this function will return false for any capabilities.\n\n        Parameters\n        ----------\n        capability: str\n            The capability to query\n\n        Returns\n        -------\n        result : bool\n            Whether the capability is supported or not.\n        \"\"\"\n        return False\n\n    @property\n    def type(self):\n        \"\"\" Returns the type of this kvstore.\n\n        Returns\n        -------\n        type : str\n            the string type\n        \"\"\"\n        return 'byteps'\n\n    @property\n    def local_rank(self):\n        \"\"\" Returns the local rank of this worker on the node.\n\n        Returns\n        -------\n        rank : int\n            The local rank of this node, which is in range [0, num_workers_on_current_node())\n        \"\"\"\n        return self.handle.local_rank()\n\n    @property\n    def rank(self):\n        \"\"\" Returns the rank of this worker node.\n\n        Returns\n        -------\n        rank : int\n            The rank of this node, which is in range [0, num_workers())\n        \"\"\"\n        return self.handle.rank()\n\n    @property\n    def num_workers(self):\n        \"\"\"Returns the number of worker nodes.\n\n        Returns\n        -------\n        size :int\n            The number of worker nodes.\n        \"\"\"\n        return self.handle.size()\n\n    def set_optimizer(self, optimizer):\n        \"\"\"\n        Not Implement yet.\n\n        Parameters\n        ----------\n        optimizer : KVStoreBase\n            The new optimizer for the store\n        \"\"\"\n        raise NotImplementedError()\n\n    def save_optimizer_states(self, fname, dump_optimizer=False):\n        \"\"\"\n        Not Implement yet.\n\n        Parameters\n        ----------\n        fname : str\n            Path to the output states file.\n        dump_optimizer : bool, default False\n            Whether to also save the optimizer itself. This would also save optimizer\n            information such as learning rate and weight decay schedules.\n        \"\"\"\n        raise NotImplementedError()\n\n    def load_optimizer_states(self, fname):\n        \"\"\"\n        Not Implement yet.\n\n        Parameters\n        ----------\n        fname : str\n            Path to input states file.\n        \"\"\"\n        raise NotImplementedError()\n"
  },
  {
    "path": "python/mxnet/kvstore/horovod.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\" Key value store interface of MXNet for Horovod \"\"\"\nfrom __future__ import absolute_import\nfrom .base import KVStoreBase\n\n__all__ = ['Horovod']\n\n\n@KVStoreBase.register\nclass Horovod(KVStoreBase):\n    \"\"\"A communication backend using Horovod.\"\"\"\n\n    def __init__(self):\n        import horovod.mxnet as hvd\n        hvd.init()\n\n    @property\n    def type(self):\n        return 'horovod'\n\n    def broadcast(self, key, value, out, priority=0):\n        \"\"\" Broadcast the `value` NDArray at rank 0 to all ranks\n\n        Parameters\n        ----------\n        key : str, or int\n            The key is used to name the tensor for allreduce. Its\n            usage is different from that of parameter servers.\n\n        value : NDArray\n            The tensor that is to be broadcasted.\n\n        out : NDArray, list of NDArray\n            Output tensor that receives value broadcasted from root process\n\n        priority : int, optional\n            The priority of the operation.\n            Higher priority operations are likely to be executed before other actions.\n\n        Examples\n        --------\n        >>> a = mx.nd.ones(shape)\n        >>> b = mx.nd.zeros(shape)\n        >>> kv.broadcast('2', value=a, out=b)\n        >>> print(b.asnumpy)\n        [[ 1.  1.  1.]\n        [ 1.  1.  1.]]\n        \"\"\"\n        import horovod.mxnet as hvd\n\n        out = out if isinstance(out, list) else [out]\n\n        # TODO (lnyuan): need to copy data to each device memory\n        for o in out:\n            o[:] = hvd.broadcast(tensor=value, root_rank=0, name=str(key),\n                                 priority=priority)\n\n    def pushpull(self, key, value, out=None, priority=0):\n        \"\"\" Performs allreduce on a single tensor or a list of tensor objects\n\n        This function performs in-place summation of the input tensor over all the processes.\n\n        The name `pushpull` is a generic term. In Horovod, its action is implemented via\n        ring allreduce. Each operation is identified by the 'key'; if `key` is not provided, an\n        incremented auto-generated name is used. The tensor type and shape must be\n        the same on all processes for a given name. The reduction will not start until all processes\n        are ready to send and receive the tensor.\n\n        Parameters\n        ----------\n        key : str, int, or sequence of str or int\n            Keys used to uniquely tag an operation.\n\n        value : NDArray\n            Tensor value on one process to be summed. If `out` is not specified, the `value` will\n            be modified in-place\n\n        out: NDArray\n            Output tensor after allreduce. If not specified, the input tensor `value` will be\n            modified in-place.\n\n        priority : int, optional\n            The priority of the operation.\n            Higher priority operations are likely to be executed before other actions.\n\n        Examples\n        --------\n        >>> # perform in-place allreduce on tensor a\n        >>> shape = (2, 3)\n        >>> nworker = kv.num_workers # assume there are 8 processes\n        >>> a = mx.nd.ones(shape)\n        >>> kv.pushpull('1', a)\n        >>> print(a.asnumpy())\n        [[ 8.  8.  8.]\n        [ 8.  8.  8.]]\n\n        >>> # perform allreduce on tensor a and output to b\n        >>> a = mx.nd.ones(shape)\n        >>> kv.pushpull('2', a, out=b)\n        >>> print(b.asnumpy())\n        [[ 8.  8.  8.]\n        [ 8.  8.  8.]]\n        \"\"\"\n        import horovod.mxnet as hvd\n\n        if out is None:\n            value = value if isinstance(value, list) else [value]\n            for v in value:\n                hvd.allreduce_(v, average=False, name=str(key),\n                               priority=priority)\n        else:\n            out = out if isinstance(out, list) else [out]\n            value = value if isinstance(value, list) else [value]\n            for o, v in zip(out, value):\n                o[:] = hvd.allreduce(v, average=False, name=str(key),\n                                     priority=priority)\n\n    def set_optimizer(self, optimizer):\n        pass\n\n    @staticmethod\n    def is_capable(capability):\n        return False\n\n    def save_optimizer_states(self, fname, dump_optimizer=False):\n        pass\n\n    def load_optimizer_states(self, fname):\n        pass\n\n    @property\n    def rank(self):\n        import horovod.mxnet as hvd\n        return hvd.rank()\n\n    @property\n    def local_rank(self):\n        import horovod.mxnet as hvd\n        return hvd.local_rank()\n\n    @property\n    def num_workers(self):\n        import horovod.mxnet as hvd\n        return hvd.size()\n"
  },
  {
    "path": "python/mxnet/kvstore/kvstore.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\" Key value store interface of MXNet for parameter synchronization.\"\"\"\n\nimport pickle\nimport ctypes\nimport os\nfrom ..ndarray import NDArray\nfrom ..ndarray import _ndarray_cls\nfrom ..base import _LIB, c_str\nfrom ..base import check_call, mx_uint, py_str\nfrom ..base import NDArrayHandle, KVStoreHandle\nfrom .. import optimizer as opt\nfrom .base import _ctype_key_value, _ctype_dict, KVStoreBase\n\n__all__ = ['KVStore']\n\ndef _updater_wrapper(updater):\n    \"\"\"A wrapper for the user-defined handle.\"\"\"\n    def updater_handle(key, lhs_handle, rhs_handle, _):\n        \"\"\" ctypes function \"\"\"\n        lhs = _ndarray_cls(NDArrayHandle(lhs_handle))\n        rhs = _ndarray_cls(NDArrayHandle(rhs_handle))\n        updater(key, lhs, rhs)\n    return updater_handle\n\ndef _get_kvstore_server_command_type(command):\n    command_types = {'kController': 0,\n                     'kSetMultiPrecision': 1,\n                     'kStopServer': 2,\n                     'kSyncMode': 3,\n                     'kSetGradientCompression': 4,\n                     'kSetProfilerParams': 5}\n    assert (command in command_types), \"Unknown command type to send to server\"\n    return command_types[command]\n\n\nclass KVStore(KVStoreBase):\n    \"\"\"A key-value store for synchronization of values, over multiple devices.\"\"\"\n\n    def __init__(self, handle):\n        \"\"\"Initializes a new KVStore.\n\n        Parameters\n        ----------\n        handle : KVStoreHandle\n            `KVStore` handle of C API.\n        \"\"\"\n        assert isinstance(handle, KVStoreHandle)\n        self.handle = handle\n        self._updater = None\n        self._updater_func = None\n        self._str_updater_func = None\n        self._is_p3 = (os.getenv('DMLC_PS_VAN_TYPE', '') == 'p3')\n\n    def __del__(self):\n        check_call(_LIB.MXKVStoreFree(self.handle))\n\n    def broadcast(self, key, value, out, priority=0):\n        \"\"\" Broadcast the `value` NDArray at rank 0 to all ranks,\n        and store the result in `out`.\n\n        Note that the native KVStore does not support broadcasting the same key more than once.\n\n        Parameters\n        ----------\n        key : str, or int\n            The key.\n\n        value : NDArray, list of NDArray, or list of list of NDArray\n            Values corresponding to the keys.\n\n        out: NDArray or list of NDArray or list of list of NDArray\n            Outputs corresponding to the keys.\n\n        priority : int, optional\n            The priority of the operation.\n            Higher priority operations are likely to be executed before other actions.\n\n        Examples\n        --------\n        >>> # broadcast a single key-value pair\n        >>> shape = (2,3)\n        >>> kv = mx.kv.create('local')\n        >>> a = mx.nd.zeros(shape)\n        >>> kv.broadcast('3', mx.nd.ones(shape)*2, out=a)\n        >>> print a.asnumpy()\n        [[ 2.  2.  2.]\n        [ 2.  2.  2.]]\n\n        \"\"\"\n        cvkeys, cvals, use_str_keys = _ctype_key_value(key, value)\n        cokeys, couts, _ = _ctype_key_value(key, out)\n\n        if use_str_keys:\n            check_call(_LIB.MXKVStoreBroadcastEx(\n                self.handle, mx_uint(len(cvkeys)), cvkeys, mx_uint(len(cokeys)), cokeys,\n                cvals, couts, ctypes.c_int(priority)))\n        else:\n            check_call(_LIB.MXKVStoreBroadcast(\n                self.handle, mx_uint(len(cvkeys)), cvkeys, mx_uint(len(cokeys)), cokeys,\n                cvals, couts, ctypes.c_int(priority)))\n\n\n    def is_capable(self, capability):\n        \"\"\"Queries if the KVStore type supports certain capability, such as optimizer algorithm,\n        gradient compression, sparsity, etc.\n\n        Parameters\n        ----------\n        capability: str\n            The capability to query\n\n        Returns\n        -------\n        result : bool\n            Whether the capability is supported or not.\n        \"\"\"\n        if capability.lower() == KVStoreBase.OPTIMIZER:\n            return not self._is_p3\n        else:\n            raise ValueError('Unknown capability: {}'.format(capability))\n\n    def init(self, key, value):\n        \"\"\" Initializes a single or a sequence of key-value pairs into the store.\n\n        For each key, one must `init` it before calling `push` or `pull`.\n        When multiple workers invoke `init` for the same key, only\n        the value supplied by worker with rank `0` is used. This function returns\n        after data has been initialized successfully.\n\n        Parameters\n        ----------\n        key : str, int, or sequence of str or int\n            The keys.\n        value : NDArray, RowSparseNDArray or sequence of NDArray or RowSparseNDArray\n            Values corresponding to the keys.\n\n        Examples\n        --------\n        >>> # init a single key-value pair\n        >>> shape = (2,3)\n        >>> kv = mx.kv.create('local')\n        >>> kv.init('3', mx.nd.ones(shape)*2)\n        >>> a = mx.nd.zeros(shape)\n        >>> kv.pull('3', out=a)\n        >>> print a.asnumpy()\n        [[ 2.  2.  2.]\n        [ 2.  2.  2.]]\n\n        >>> # init a list of key-value pairs\n        >>> keys = ['5', '7', '9']\n        >>> kv.init(keys, [mx.nd.ones(shape)]*len(keys))\n\n        >>> # init a row_sparse value\n        >>> kv.init('4', mx.nd.ones(shape).tostype('row_sparse'))\n        >>> b = mx.nd.sparse.zeros('row_sparse', shape)\n        >>> kv.row_sparse_pull('4', row_ids=mx.nd.array([0, 1]), out=b)\n        >>> print b\n        <RowSparseNDArray 2x3 @cpu(0)>\n        \"\"\"\n        ckeys, cvals, use_str_keys = _ctype_key_value(key, value)\n        if use_str_keys:\n            check_call(_LIB.MXKVStoreInitEx(self.handle, mx_uint(len(ckeys)), ckeys, cvals))\n        else:\n            check_call(_LIB.MXKVStoreInit(self.handle, mx_uint(len(ckeys)), ckeys, cvals))\n\n    def push(self, key, value, priority=0):\n        \"\"\" Pushes a single or a sequence of key-value pairs into the store.\n\n        This function returns immediately after adding an operator to the engine.\n        The actual operation is executed asynchronously. If there are consecutive\n        pushes to the same key, there is no guarantee on the serialization of pushes.\n        The execution of a push does not guarantee that all previous pushes are\n        finished.\n        There is no synchronization between workers.\n        One can use ``_barrier()`` to sync all workers.\n\n        Parameters\n        ----------\n        key : str, int, or sequence of str or int\n            Keys.\n\n        value : NDArray, RowSparseNDArray, list of NDArray or RowSparseNDArray,\n                or list of list of NDArray or RowSparseNDArray\n            Values corresponding to the keys.\n\n        priority : int, optional\n            The priority of the push operation.\n            Higher priority push operations are likely to be executed before\n            other push actions.\n\n        Examples\n        --------\n        >>> # push a single key-value pair\n        >>> shape = (2,3)\n        >>> kv.push('3', mx.nd.ones(shape)*8)\n        >>> kv.pull('3', out=a) # pull out the value\n        >>> print a.asnumpy()\n        [[ 8.  8.  8.]\n        [ 8.  8.  8.]]\n\n        >>> # aggregate the value and the push\n        >>> gpus = [mx.gpu(i) for i in range(4)]\n        >>> b = [mx.nd.ones(shape, gpu) for gpu in gpus]\n        >>> kv.push('3', b)\n        >>> kv.pull('3', out=a)\n        >>> print a.asnumpy()\n        [[ 4.  4.  4.]\n        [ 4.  4.  4.]]\n\n        >>> # push a list of keys.\n        >>> # single device\n        >>> keys = ['4', '5', '6']\n        >>> kv.push(keys, [mx.nd.ones(shape)]*len(keys))\n        >>> b = [mx.nd.zeros(shape)]*len(keys)\n        >>> kv.pull(keys, out=b)\n        >>> print b[1].asnumpy()\n        [[ 1.  1.  1.]\n        [ 1.  1.  1.]]\n\n        >>> # multiple devices:\n        >>> keys = ['7', '8', '9']\n        >>> b = [[mx.nd.ones(shape, gpu) for gpu in gpus]] * len(keys)\n        >>> kv.push(keys, b)\n        >>> kv.pull(keys, out=b)\n        >>> print b[1][1].asnumpy()\n        [[ 4.  4.  4.]\n        [ 4.  4.  4.]]\n\n        >>> # push a row_sparse value\n        >>> b = mx.nd.sparse.zeros('row_sparse', shape)\n        >>> kv.init('10', mx.nd.sparse.zeros('row_sparse', shape))\n        >>> kv.push('10', mx.nd.ones(shape).tostype('row_sparse'))\n        >>> # pull out the value\n        >>> kv.row_sparse_pull('10', row_ids=mx.nd.array([0, 1]), out=b)\n        >>> print b\n        <RowSparseNDArray 2x3 @cpu(0)>\n        \"\"\"\n        ckeys, cvals, use_str_keys = _ctype_key_value(key, value)\n        if use_str_keys:\n            check_call(_LIB.MXKVStorePushEx(\n                self.handle, mx_uint(len(ckeys)), ckeys, cvals, ctypes.c_int(priority)))\n        else:\n            check_call(_LIB.MXKVStorePush(\n                self.handle, mx_uint(len(ckeys)), ckeys, cvals, ctypes.c_int(priority)))\n\n\n    def pull(self, key, out=None, priority=0, ignore_sparse=True):\n        \"\"\" Pulls a single value or a sequence of values from the store.\n\n        This function returns immediately after adding an operator to the engine.\n        Subsequent attempts to read from the `out` variable will be blocked until the\n        pull operation completes.\n\n        `pull` is executed asynchronously after all previous `pull` calls and only\n        the last `push` call for the same input key(s) are finished.\n\n        The returned values are guaranteed to be the latest values in the store.\n\n        pull with `RowSparseNDArray` is not supported for dist kvstore.\n        Please use ``row_sparse_pull`` instead.\n\n        Parameters\n        ----------\n        key : str, int, or sequence of str or int\n            Keys.\n\n        out: NDArray or list of NDArray or list of list of NDArray\n            Values corresponding to the keys.\n\n        priority : int, optional\n            The priority of the pull operation.\n            Higher priority pull operations are likely to be executed before\n            other pull actions.\n\n        ignore_sparse: bool, optional, default True\n            Whether to ignore sparse arrays in the request.\n\n        Examples\n        --------\n        >>> # pull a single key-value pair\n        >>> shape = (2,3)\n        >>> a = mx.nd.zeros(shape)\n        >>> kv.pull('3', out=a)\n        >>> print a.asnumpy()\n        [[ 2.  2.  2.]\n        [ 2.  2.  2.]]\n\n        >>> # pull into multiple devices\n        >>> b = [mx.nd.ones(shape, gpu) for gpu in gpus]\n        >>> kv.pull('3', out=b)\n        >>> print b[1].asnumpy()\n        [[ 2.  2.  2.]\n        [ 2.  2.  2.]]\n\n        >>> # pull a list of key-value pairs.\n        >>> # On single device\n        >>> keys = ['5', '7', '9']\n        >>> b = [mx.nd.zeros(shape)]*len(keys)\n        >>> kv.pull(keys, out=b)\n        >>> print b[1].asnumpy()\n        [[ 2.  2.  2.]\n        [ 2.  2.  2.]]\n        >>> # On multiple devices\n        >>> keys = ['6', '8', '10']\n        >>> b = [[mx.nd.ones(shape, gpu) for gpu in gpus]] * len(keys)\n        >>> kv.pull(keys, out=b)\n        >>> print b[1][1].asnumpy()\n        [[ 2.  2.  2.]\n        [ 2.  2.  2.]]\n        \"\"\"\n        assert(out is not None)\n        ckeys, cvals, use_str_keys = _ctype_key_value(key, out)\n        if use_str_keys:\n            check_call(_LIB.MXKVStorePullWithSparseEx(self.handle, mx_uint(len(ckeys)), ckeys,\n                                                      cvals, ctypes.c_int(priority),\n                                                      ctypes.c_bool(ignore_sparse)))\n        else:\n            check_call(_LIB.MXKVStorePullWithSparse(self.handle, mx_uint(len(ckeys)), ckeys,\n                                                    cvals, ctypes.c_int(priority),\n                                                    ctypes.c_bool(ignore_sparse)))\n\n    def pushpull(self, key, value, out=None, priority=0):\n        \"\"\" Performs push and pull a single value or a sequence of values from the store.\n\n        This function is coalesced form of push and pull operations. This function returns\n        immediately after adding an operator to the engine. Subsequent attempts to read\n        from the `out` variable will be blocked until the pull operation completes.\n\n        `value` is pushed to the kvstore server for the specified keys and the updated\n        values are pulled from the server to `out`. If `out` is not specified the pulled\n        values are written to `value`. The returned values are guaranteed to be the latest\n        values in the store.\n\n        pushpull with `RowSparseNDArray` is not supported for dist kvstore.\n\n        Parameters\n        ----------\n        key : str, int, or sequence of str or int\n            Keys.\n\n        value : NDArray, list of NDArray, or list of list of NDArray\n            Values corresponding to the keys.\n\n        out: NDArray or list of NDArray or list of list of NDArray, optional\n            Outputs corresponding to the keys.\n\n        priority : int, optional\n            The priority of the operation.\n            Higher priority operations are likely to be executed before other actions.\n\n        Examples\n        --------\n        >>> # pushpull a single key-value pair\n        >>> shape = (2,3)\n        >>> kv.pushpull('3', mx.nd.ones(shape)*8, out=a)\n        >>> print a.asnumpy()\n        [[ 8.  8.  8.]\n        [ 8.  8.  8.]]\n\n        >>> # aggregate the value and then pushpull\n        >>> gpus = [mx.gpu(i) for i in range(4)]\n        >>> b = [mx.nd.ones(shape, gpu) for gpu in gpus]\n        >>> kv.pushpull('3', b, out=a)\n        >>> print a.asnumpy()\n        [[ 4.  4.  4.]\n        [ 4.  4.  4.]]\n\n        >>> # pushpull a list of keys.\n        >>> # single device\n        >>> keys = ['4', '5', '6']\n        >>> b = [mx.nd.zeros(shape)]*len(keys)\n        >>> kv.pushpull(keys, [mx.nd.ones(shape)]*len(keys), out=b)\n        >>> print b[1].asnumpy()\n        [[ 1.  1.  1.]\n        [ 1.  1.  1.]]\n\n        >>> # multiple devices:\n        >>> keys = ['7', '8', '9']\n        >>> b = [[mx.nd.ones(shape, gpu) for gpu in gpus]] * len(keys)\n        >>> kv.pushpull(keys, b)\n        >>> print b[1][1].asnumpy()\n        [[ 4.  4.  4.]\n        [ 4.  4.  4.]]\n\n        \"\"\"\n        cvkeys, cvals, use_str_keys = _ctype_key_value(key, value)\n        if out is not None:\n            cokeys, couts, _ = _ctype_key_value(key, out)\n        else:\n            cokeys = cvkeys\n            couts = cvals\n\n        if use_str_keys:\n            check_call(_LIB.MXKVStorePushPullEx(\n                self.handle, mx_uint(len(cvkeys)), cvkeys, mx_uint(len(cokeys)), cokeys,\n                cvals, couts, ctypes.c_int(priority)))\n        else:\n            check_call(_LIB.MXKVStorePushPull(\n                self.handle, mx_uint(len(cvkeys)), cvkeys, mx_uint(len(cokeys)), cokeys,\n                cvals, couts, ctypes.c_int(priority)))\n\n    def row_sparse_pull(self, key, out=None, priority=0, row_ids=None):\n        \"\"\" Pulls a single RowSparseNDArray value or a sequence of RowSparseNDArray values \\\n        from the store with specified row_ids. When there is only one row_id, KVStoreRowSparsePull \\\n        is invoked just once and the result is broadcast to all the rest of outputs.\n\n        `row_sparse_pull` is executed asynchronously after all previous\n        `pull`/`row_sparse_pull` calls and the last `push` call for the\n        same input key(s) are finished.\n\n        The returned values are guaranteed to be the latest values in the store.\n\n        Parameters\n        ----------\n        key : str, int, or sequence of str or int\n            Keys.\n\n        out: RowSparseNDArray or list of RowSparseNDArray or list of list of RowSparseNDArray\n            Values corresponding to the keys. The stype is expected to be row_sparse\n\n        priority : int, optional\n            The priority of the pull operation.\n            Higher priority pull operations are likely to be executed before\n            other pull actions.\n\n        row_ids : NDArray or list of NDArray\n            The row_ids for which to pull for each value. Each row_id is an 1-D NDArray \\\n            whose values don't have to be unique nor sorted.\n\n        Examples\n        --------\n        >>> shape = (3, 3)\n        >>> kv.init('3', mx.nd.ones(shape).tostype('row_sparse'))\n        >>> a = mx.nd.sparse.zeros('row_sparse', shape)\n        >>> row_ids = mx.nd.array([0, 2], dtype='int64')\n        >>> kv.row_sparse_pull('3', out=a, row_ids=row_ids)\n        >>> print a.asnumpy()\n        [[ 1.  1.  1.]\n        [ 0.  0.  0.]\n        [ 1.  1.  1.]]\n        >>> duplicate_row_ids = mx.nd.array([2, 2], dtype='int64')\n        >>> kv.row_sparse_pull('3', out=a, row_ids=duplicate_row_ids)\n        >>> print a.asnumpy()\n        [[ 0.  0.  0.]\n        [ 0.  0.  0.]\n        [ 1.  1.  1.]]\n        >>> unsorted_row_ids = mx.nd.array([1, 0], dtype='int64')\n        >>> kv.row_sparse_pull('3', out=a, row_ids=unsorted_row_ids)\n        >>> print a.asnumpy()\n        [[ 1.  1.  1.]\n        [ 1.  1.  1.]\n        [ 0.  0.  0.]]\n        \"\"\"\n        assert(out is not None)\n        assert(row_ids is not None)\n        if isinstance(row_ids, NDArray):\n            row_ids = [row_ids]\n        assert(isinstance(row_ids, list)), \\\n            \"row_ids should be NDArray or list of NDArray\"\n        first_out = out\n        # whether row_ids are the same\n        single_rowid = False\n        if len(row_ids) == 1 and isinstance(out, list):\n            single_rowid = True\n            first_out = [out[0]]\n        ckeys, cvals, use_str_keys = _ctype_key_value(key, first_out)\n        _, crow_ids, _ = _ctype_key_value(key, row_ids)\n        assert(len(crow_ids) == len(cvals)), \\\n               \"the number of row_ids doesn't match the number of values\"\n        if use_str_keys:\n            check_call(_LIB.MXKVStorePullRowSparseEx(\n                self.handle, mx_uint(len(ckeys)), ckeys, cvals, crow_ids, ctypes.c_int(priority)))\n        else:\n            check_call(_LIB.MXKVStorePullRowSparse(\n                self.handle, mx_uint(len(ckeys)), ckeys, cvals, crow_ids, ctypes.c_int(priority)))\n        # the result can be copied to other devices without invoking row_sparse_pull\n        # if the indices are the same\n        if single_rowid:\n            for out_i in out[1:]:\n                out[0].copyto(out_i)\n\n    def set_gradient_compression(self, compression_params):\n        \"\"\" Specifies type of low-bit quantization for gradient compression \\\n         and additional arguments depending on the type of compression being used.\n\n        The 1bit compression works as follows: values which is above the threshold in the\n        gradient will be set to +1, whereas values below threshold will be set to -1.\n\n        2bit Gradient Compression takes a positive float `threshold`.\n        The technique works by thresholding values such that positive values in the\n        gradient above threshold will be set to threshold. Negative values whose absolute\n        values are higher than threshold, will be set to the negative of threshold.\n        Values whose absolute values are less than threshold will be set to 0.\n        By doing so, each value in the gradient is in one of three states. 2bits are\n        used to represent these states, and every 16 float values in the original\n        gradient can be represented using one float. This compressed representation\n        can reduce communication costs. The difference between these thresholded values and\n        original values is stored at the sender's end as residual and added to the\n        gradient in the next iteration.\n\n        When kvstore is 'local', gradient compression is used to reduce communication\n        between multiple devices (gpus). Gradient is quantized on each GPU which\n        computed the gradients, then sent to the GPU which merges the gradients. This\n        receiving GPU dequantizes the gradients and merges them. Note that this\n        increases memory usage on each GPU because of the residual array stored.\n\n        When kvstore is 'dist', gradient compression is used to reduce communication\n        from worker to sender. Gradient is quantized on each worker which\n        computed the gradients, then sent to the server which dequantizes\n        this data and merges the gradients from each worker. Note that this\n        increases CPU memory usage on each worker because of the residual array stored.\n        Only worker to server communication is compressed in this setting.\n        If each machine has multiple GPUs, currently this GPU to GPU or GPU to CPU communication\n        is not compressed. Server to worker communication (in the case of pull)\n        is also not compressed.\n\n        To use 2bit compression, we need to specify `type` as `2bit`.\n        Only specifying `type` would use default value for the threshold.\n        To completely specify the arguments for 2bit compression, we would need to pass\n        a dictionary which includes `threshold` like:\n        {'type': '2bit', 'threshold': 0.5}\n\n        Parameters\n        ----------\n        compression_params : dict\n            A dictionary specifying the type and parameters for gradient compression.\n            The key `type` in this dictionary is a\n            required string argument and specifies the type of gradient compression.\n            Currently `type` can be only `1bit` and `2bit`\n            Other keys in this dictionary are optional and specific to the type\n            of gradient compression.\n        \"\"\"\n        if ('device' in self.type) or ('dist' in self.type): # pylint: disable=unsupported-membership-test\n            ckeys, cvals = _ctype_dict(compression_params)\n            check_call(_LIB.MXKVStoreSetGradientCompression(self.handle,\n                                                            mx_uint(len(compression_params)),\n                                                            ckeys, cvals))\n        else:\n            raise Exception('Gradient compression is not supported for this type of kvstore')\n\n    def set_optimizer(self, optimizer):\n        \"\"\" Registers an optimizer with the kvstore.\n\n        When using a single machine, this function updates the local optimizer.\n        If using multiple machines and this operation is invoked from a worker node,\n        it will serialized the optimizer with pickle and send it to all servers.\n        The function returns after all servers have been updated.\n\n        Parameters\n        ----------\n        optimizer : Optimizer\n            The new optimizer for the store\n\n        Examples\n        --------\n\n        >>> kv = mx.kv.create()\n        >>> shape = (2, 2)\n        >>> weight = mx.nd.zeros(shape)\n        >>> kv.init(3, weight)\n        >>> # set the optimizer for kvstore as the default SGD optimizer\n        >>> kv.set_optimizer(mx.optimizer.SGD())\n        >>> grad = mx.nd.ones(shape)\n        >>> kv.push(3, grad)\n        >>> kv.pull(3, out = weight)\n        >>> # weight is updated via gradient descent\n        >>> weight.asnumpy()\n        array([[-0.01, -0.01],\n               [-0.01, -0.01]], dtype=float32)\n        \"\"\"\n        is_worker = ctypes.c_int()\n        check_call(_LIB.MXKVStoreIsWorkerNode(ctypes.byref(is_worker)))\n\n        # pylint: disable=invalid-name\n        if 'dist' in self.type and is_worker.value: # pylint: disable=unsupported-membership-test\n            # send the optimizer to server\n            try:\n                # use ASCII protocol 0, might be slower, but not a big ideal\n                optim_str = py_str(pickle.dumps(optimizer, 0))\n            except:\n                raise\n            cmd = _get_kvstore_server_command_type('kController')\n            self._send_command_to_servers(cmd, optim_str)\n            if optimizer.multi_precision:\n                cmd = _get_kvstore_server_command_type('kSetMultiPrecision')\n                self._send_command_to_servers(cmd, '')\n        else:\n            self._set_updater(opt.get_updater(optimizer))\n\n    @property\n    def type(self):\n        \"\"\" Returns the type of this kvstore.\n\n        Returns\n        -------\n        type : str\n            the string type\n        \"\"\"\n        kv_type = ctypes.c_char_p()\n        check_call(_LIB.MXKVStoreGetType(self.handle, ctypes.byref(kv_type)))\n        return py_str(kv_type.value)\n\n    @property\n    def rank(self):\n        \"\"\" Returns the rank of this worker node.\n\n        Returns\n        -------\n        rank : int\n            The rank of this node, which is in range [0, num_workers())\n        \"\"\"\n        rank = ctypes.c_int()\n        check_call(_LIB.MXKVStoreGetRank(self.handle, ctypes.byref(rank)))\n        return rank.value\n\n    @property\n    def num_workers(self):\n        \"\"\"Returns the number of worker nodes.\n\n        Returns\n        -------\n        size :int\n            The number of worker nodes.\n        \"\"\"\n        size = ctypes.c_int()\n        check_call(_LIB.MXKVStoreGetGroupSize(self.handle, ctypes.byref(size)))\n        return size.value\n\n    def save_optimizer_states(self, fname, dump_optimizer=False):\n        \"\"\"Saves the optimizer (updater) state to a file. This is often used when checkpointing\n        the model during training.\n\n        Parameters\n        ----------\n        fname : str\n            Path to the output states file.\n        dump_optimizer : bool, default False\n            Whether to also save the optimizer itself. This would also save optimizer\n            information such as learning rate and weight decay schedules.\n        \"\"\"\n        assert self._updater is not None, \"Cannot save states for distributed training\"\n        with open(fname, 'wb') as fout:\n            fout.write(self._updater.get_states(dump_optimizer))\n\n    def load_optimizer_states(self, fname):\n        \"\"\"Loads the optimizer (updater) state from the file.\n\n        Parameters\n        ----------\n        fname : str\n            Path to input states file.\n        \"\"\"\n        assert self._updater is not None, \"Cannot load states for distributed training\"\n        self._updater.set_states(open(fname, 'rb').read())\n\n    def _set_updater(self, updater):\n        \"\"\"Sets a push updater into the store.\n\n        This function only changes the local store. When running on multiple machines one must\n        use `set_optimizer`.\n\n        Parameters\n        ----------\n        updater : function\n            The updater function.\n\n        Examples\n        --------\n        >>> def update(key, input, stored):\n        ...     print \"update on key: %d\" % key\n        ...     stored += input * 2\n        >>> kv._set_updater(update)\n        >>> kv.pull('3', out=a)\n        >>> print a.asnumpy()\n        [[ 4.  4.  4.]\n        [ 4.  4.  4.]]\n        >>> kv.push('3', mx.nd.ones(shape))\n        update on key: 3\n        >>> kv.pull('3', out=a)\n        >>> print a.asnumpy()\n        [[ 6.  6.  6.]\n        [ 6.  6.  6.]]\n        \"\"\"\n        self._updater = updater\n        # set updater with int keys\n        _updater_proto = ctypes.CFUNCTYPE(\n            None, ctypes.c_int, NDArrayHandle, NDArrayHandle, ctypes.c_void_p)\n        self._updater_func = _updater_proto(_updater_wrapper(updater))\n        # set updater with str keys\n        _str_updater_proto = ctypes.CFUNCTYPE(\n            None, ctypes.c_char_p, NDArrayHandle, NDArrayHandle, ctypes.c_void_p)\n        self._str_updater_func = _str_updater_proto(_updater_wrapper(updater))\n        check_call(_LIB.MXKVStoreSetUpdaterEx(self.handle, self._updater_func,\n                                              self._str_updater_func, None))\n\n\n    def _barrier(self):\n        \"\"\"Invokes global barrier among all worker nodes.\n\n        For example, assume there are `n` machines. We would like machine `0` to first\n        `init` the values and then have all the workers `pull` the initialized value.\n        Before pulling, we can place invoke `_barrier()` to guarantee that the\n        initialization is finished.\n        \"\"\"\n        check_call(_LIB.MXKVStoreBarrier(self.handle))\n\n    def _send_command_to_servers(self, head, body):\n        \"\"\"Sends a command to all server nodes.\n\n        Sending command to a server node will cause that server node to invoke\n        ``KVStoreServer.controller`` to execute the command.\n\n        This function returns after the command has been executed on all server\n        nodes.\n\n        Parameters\n        ----------\n        head : int\n            the head of the command.\n        body : str\n            the body of the command.\n        \"\"\"\n        check_call(_LIB.MXKVStoreSendCommmandToServers(\n            self.handle, mx_uint(head), c_str(body)))\n"
  },
  {
    "path": "python/mxnet/kvstore/kvstore_server.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"A server node for the key value store.\"\"\"\nimport ctypes\nimport sys\nimport pickle\nimport logging\nfrom ..base import _LIB, check_call\nfrom .base import create\n\n__all__ = ['KVStoreServer']\n\nclass KVStoreServer(object):\n    \"\"\"The key-value store server.\"\"\"\n    def __init__(self, kvstore):\n        \"\"\"Initialize a new KVStoreServer.\n\n        Parameters\n        ----------\n        kvstore : KVStore\n        \"\"\"\n        self.kvstore = kvstore\n        self.handle = kvstore.handle\n        self.init_logginig = False\n\n    def _controller(self):\n        \"\"\"Return the server controller.\"\"\"\n        def server_controller(cmd_id, cmd_body, _):\n            \"\"\"Server controler.\"\"\"\n            if not self.init_logginig:\n                # the reason put the codes here is because we cannot get\n                # kvstore.rank earlier\n                head = '%(asctime)-15s Server[' + str(\n                    self.kvstore.rank) + '] %(message)s'\n                logging.basicConfig(level=logging.DEBUG, format=head)\n                self.init_logginig = True\n\n            if cmd_id == 0:\n                try:\n                    optimizer = pickle.loads(cmd_body)\n                except:\n                    raise\n                self.kvstore.set_optimizer(optimizer)\n            else:\n                print(f\"server {self.kvstore.rank}, unknown command ({cmd_id}, {cmd_body})\")\n        return server_controller\n\n    def run(self):\n        \"\"\"Run the server, whose behavior is like.\n\n\n        >>> while receive(x):\n        ...     if is_command x: controller(x)\n        ...     else if is_key_value x: updater(x)\n        \"\"\"\n        _ctrl_proto = ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.c_char_p, ctypes.c_void_p)\n        check_call(_LIB.MXKVStoreRunServer(self.handle, _ctrl_proto(self._controller()), None))\n\ndef _init_kvstore_server_module():\n    \"\"\"Start server/scheduler.\"\"\"\n    is_worker = ctypes.c_int()\n    check_call(_LIB.MXKVStoreIsWorkerNode(ctypes.byref(is_worker)))\n    if is_worker.value == 0:\n        kvstore = create('dist')\n        server = KVStoreServer(kvstore)\n        server.run()\n        sys.exit()\n\n_init_kvstore_server_module()\n"
  },
  {
    "path": "python/mxnet/libinfo.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Information about mxnet.\"\"\"\nimport os\nimport platform\nimport logging\nimport sys\n\ndef find_lib_path(prefix='libmxnet'):\n    \"\"\"Find MXNet dynamic library files.\n\n    Returns\n    -------\n    lib_path : list(string)\n        List of all found path to the libraries.\n    \"\"\"\n    lib_from_env = os.environ.get('MXNET_LIBRARY_PATH')\n    if lib_from_env:\n        lib_from_env = lib_from_env.replace('libmxnet', prefix)\n        if os.path.isfile(lib_from_env):\n            if not os.path.isabs(lib_from_env):\n                logging.warning(\"MXNET_LIBRARY_PATH should be an absolute path, instead of: %s\",\n                                lib_from_env)\n            else:\n                if os.name == 'nt':\n                    os.environ['PATH'] = os.environ['PATH'] + ';' + os.path.dirname(lib_from_env)\n                return [lib_from_env]\n        else:\n            logging.warning(\"MXNET_LIBRARY_PATH '%s' doesn't exist\", lib_from_env)\n\n    curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))\n    api_path = os.path.join(curr_path, '../../lib/')\n    cmake_build_path = os.path.join(curr_path, '../../build/')\n    dll_path = [curr_path, api_path, cmake_build_path]\n    if os.name == 'nt':\n        dll_path.append(os.path.join(curr_path, '../../build'))\n        vs_configuration = 'Release'\n        if platform.architecture()[0] == '64bit':\n            dll_path.append(os.path.join(curr_path, '../../build', vs_configuration))\n            dll_path.append(os.path.join(curr_path, '../../windows/x64', vs_configuration))\n        else:\n            dll_path.append(os.path.join(curr_path, '../../build', vs_configuration))\n            dll_path.append(os.path.join(curr_path, '../../windows', vs_configuration))\n    elif os.name == \"posix\" and os.environ.get('LD_LIBRARY_PATH', None):\n        dll_path[0:0] = [p.strip() for p in os.environ['LD_LIBRARY_PATH'].split(\":\")]\n    if os.name == 'nt':\n        os.environ['PATH'] = os.path.dirname(__file__) + ';' + os.environ.get('PATH', '')\n        dll_path = [os.path.join(p, prefix + '.dll') for p in dll_path]\n    elif platform.system() == 'Darwin':\n        dll_path = [os.path.join(p, prefix + '.dylib') for p in dll_path] + \\\n                   [os.path.join(p, prefix + '.so') for p in dll_path]\n    else:\n        dll_path.append('../../../')\n        dll_path = [os.path.join(p, prefix + '.so') for p in dll_path]\n    lib_path = [p for p in dll_path if os.path.exists(p) and os.path.isfile(p)]\n    if len(lib_path) == 0:\n        raise RuntimeError('Cannot find the MXNet library.\\n' +\n                           'List of candidates:\\n' + str('\\n'.join(dll_path)))\n    if os.name == 'nt':\n        os.environ['PATH'] = os.environ['PATH'] + ';' + os.path.dirname(lib_path[0])\n        if sys.version_info >= (3, 8):\n            if 'CUDA_PATH' not in os.environ:\n                raise RuntimeError('Cannot find the env CUDA_PATH.Please set CUDA_PATH env with cuda path')\n            os.add_dll_directory(os.path.dirname(lib_path[0]))\n            os.add_dll_directory(os.path.join(os.environ['CUDA_PATH'], 'bin'))\n    return lib_path\n\ndef find_include_path():\n    \"\"\"Find MXNet included header files.\n\n    Returns\n    -------\n    incl_path : string\n        Path to the header files.\n    \"\"\"\n    incl_from_env = os.environ.get('MXNET_INCLUDE_PATH')\n    if incl_from_env:\n        if os.path.isdir(incl_from_env):\n            if not os.path.isabs(incl_from_env):\n                logging.warning(\"MXNET_INCLUDE_PATH should be an absolute path, instead of: %s\",\n                                incl_from_env)\n            else:\n                return incl_from_env\n        else:\n            logging.warning(\"MXNET_INCLUDE_PATH '%s' doesn't exist\", incl_from_env)\n\n    curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))\n    # include path in pip package\n    pip_incl_path = os.path.join(curr_path, 'include/')\n    if os.path.isdir(pip_incl_path):\n        return pip_incl_path\n    else:\n        # include path if build from source\n        src_incl_path = os.path.join(curr_path, '../../include/')\n        if os.path.isdir(src_incl_path):\n            return src_incl_path\n        else:\n            raise RuntimeError('Cannot find the MXNet include path in either ' + pip_incl_path +\n                               ' or ' + src_incl_path + '\\n')\n\n\ndef find_conf_path(prefix='tvmop'):\n    \"\"\"Find TVM op config files.\n\n    Returns\n    -------\n    conf_path : string\n        Path to the config files.\n    \"\"\"\n    conf_from_env = os.environ.get('MXNET_CONF_PATH')\n    if conf_from_env:\n        if os.path.isfile(conf_from_env):\n            if not os.path.isabs(conf_from_env):\n                logging.warning(\"MXNET_CONF_PATH should be an absolute path, instead of: %s\",\n                                conf_from_env)\n            else:\n                return conf_from_env\n        else:\n            logging.warning(\"MXNET_CONF_PATH '%s' doesn't exist\", conf_from_env)\n\n    curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))\n    makefile_path = os.path.join(curr_path, '../../lib/')\n    cmake_build_path = os.path.join(curr_path, '../../build/')\n    candidates_path = [makefile_path, cmake_build_path]\n    candidates_path = [p + prefix + '.conf' for p in candidates_path]\n    conf_path = [p for p in candidates_path if os.path.exists(p) and os.path.isfile(p)]\n    if len(conf_path) == 0:\n        raise RuntimeError('Cannot find the TVM op config.\\n' +\n                           'List of candidates:\\n' + str('\\n'.join(candidates_path)))\n    return conf_path\n\n\n# current version\n__version__ = \"2.0.0\"\n"
  },
  {
    "path": "python/mxnet/library.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Library management API of mxnet.\"\"\"\nimport ctypes\nimport sys\nimport os\nfrom .base import _LIB, check_call, MXNetError, _init_op_module, mx_uint\nfrom .ndarray.register import _make_ndarray_function\nfrom .symbol.register import _make_symbol_function\n\nclass MXlib:\n    \"\"\"Holds a pointed to a loaded shared library and closes it on destruction\"\"\"\n    def __init__(self, handle):\n        self.handle = handle\n    def __del__(self):\n        libdl = ctypes.CDLL(\"libdl.so\")\n        libdl.dlclose(self.handle)\n\n# set of libraries loaded\nloaded_libs = []\n\ndef load(path, verbose=True):\n    \"\"\"Loads library dynamically.\n\n    Parameters\n    ---------\n    path : string\n        Path to library .so/.dll file\n\n    verbose : boolean\n        defaults to True, set to False to avoid printing library info\n\n    Returns\n    ---------\n    void\n    \"\"\"\n    global loaded_libs\n\n    #check if path exists\n    if not os.path.exists(path):\n        raise MXNetError(f\"load path {path} does NOT exist\")\n    #check if path is an absolute path\n    if not os.path.isabs(path):\n        raise MXNetError(f\"load path {path} is not an absolute path\")\n    #check if path is to a library file\n    _, file_ext = os.path.splitext(path)\n    if not file_ext in ['.so', '.dll']:\n        raise MXNetError(f\"load path {path} is NOT a library file\")\n\n    verbose_val = 1 if verbose else 0\n    byt_obj = path.encode('utf-8')\n    chararr = ctypes.c_char_p(byt_obj)\n    lib_ptr = ctypes.c_void_p(0)\n    check_call(_LIB.MXLoadLib(chararr, mx_uint(verbose_val), ctypes.byref(lib_ptr)))\n    # add library pointer to list so it can be closed later\n    loaded_libs.append(MXlib(lib_ptr))\n\n    #regenerate operators\n    _init_op_module('mxnet', 'ndarray', _make_ndarray_function)\n    _init_op_module('mxnet', 'symbol', _make_symbol_function)\n\n    #re-register mx.nd.op into mx.nd\n    mx_nd = sys.modules[\"mxnet.ndarray\"]\n    mx_nd_op = sys.modules[\"mxnet.ndarray.op\"]\n    for op in dir(mx_nd_op):\n        func = getattr(mx_nd_op, op)\n        setattr(mx_nd, op, func)\n\n    #re-register mx.sym.op into mx.sym\n    mx_sym = sys.modules[\"mxnet.symbol\"]\n    mx_sym_op = sys.modules[\"mxnet.symbol.op\"]\n    for op in dir(mx_sym_op):\n        func = getattr(mx_sym_op, op)\n        setattr(mx_sym, op, func)\n\ndef compiled_with_gcc_cxx11_abi():\n    \"\"\"Check if the library is compiled with _GLIBCXX_USE_CXX11_ABI.\n\n    Please see\n    https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html for\n    more information. When building libraries relying on MXNet C++ headers, it\n    is required to use the same C++ ABI in the library as well as in libmxnet.\n\n    Returns\n    -------\n    int\n        1 If compiled with _GLIBCXX_USE_CXX11_ABI=1\n        0 If compiled with _GLIBCXX_USE_CXX11_ABI=0\n       -1 If compiled with a compiler that does not support _GLIBCXX_USE_CXX11_ABI\n\n    \"\"\"\n    ret = ctypes.c_int()\n    check_call(_LIB.MXLibInfoCompiledWithCXX11ABI(ctypes.byref(ret)))\n    return ret.value\n"
  },
  {
    "path": "python/mxnet/log.py",
    "content": "#!/usr/bin/env python\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# -*- coding: utf-8 -*-\n# pylint: disable= protected-access, invalid-name\n\"\"\"Logging utilities.\"\"\"\nimport logging\nimport warnings\n\nCRITICAL = logging.CRITICAL\nERROR = logging.ERROR\nWARNING = logging.WARNING\nINFO = logging.INFO\nDEBUG = logging.DEBUG\nNOTSET = logging.NOTSET\n\n\nclass _Formatter(logging.Formatter):\n    # pylint: disable= no-self-use\n    \"\"\"Customized log formatter.\"\"\"\n\n    def __init__(self):\n        datefmt = '%m%d %H:%M:%S'\n        super(_Formatter, self).__init__(datefmt=datefmt)\n\n    def _get_color(self, level):\n        # pylint: disable= missing-docstring\n        if logging.WARNING <= level:\n            return '\\x1b[31m'\n        elif logging.INFO <= level:\n            return '\\x1b[32m'\n        return '\\x1b[34m'\n\n    def _get_label(self, level):\n        # pylint: disable= missing-docstring\n        if level == logging.CRITICAL:\n            return 'C'\n        elif level == logging.ERROR:\n            return 'E'\n        elif level == logging.WARNING:\n            return 'W'\n        elif level == logging.INFO:\n            return 'I'\n        elif level == logging.DEBUG:\n            return 'D'\n        return 'U'\n\n    def format(self, record):\n        # pylint: disable= missing-docstring\n        fmt = self._get_color(record.levelno)\n        fmt += self._get_label(record.levelno)\n        fmt += '%(asctime)s %(process)d %(pathname)s:%(funcName)s:%(lineno)d'\n        fmt += ']\\x1b[0m'\n        fmt += ' %(message)s'\n        self._style._fmt = fmt # pylint: disable= no-member\n        return super(_Formatter, self).format(record)\n\ndef getLogger(name=None, filename=None, filemode=None, level=WARNING):\n    \"\"\"Gets a customized logger.\n\n    .. note:: `getLogger` is deprecated. Use `get_logger` instead.\n\n    \"\"\"\n    warnings.warn(\"getLogger is deprecated, Use get_logger instead.\",\n                  DeprecationWarning, stacklevel=2)\n    return get_logger(name, filename, filemode, level)\n\ndef get_logger(name=None, filename=None, filemode=None, level=WARNING):\n    \"\"\"Gets a customized logger.\n\n    Parameters\n    ----------\n    name: str, optional\n        Name of the logger.\n    filename: str, optional\n        The filename to which the logger's output will be sent.\n    filemode: str, optional\n        The file mode to open the file (corresponding to `filename`),\n        default is 'a' if `filename` is not ``None``.\n    level: int, optional\n        The `logging` level for the logger.\n        See: https://docs.python.org/2/library/logging.html#logging-levels\n\n    Returns\n    -------\n    Logger\n        A customized `Logger` object.\n\n    Example\n    -------\n    ## get_logger call with default parameters.\n    >>> from mxnet.log import get_logger\n    >>> logger = get_logger(\"Test\")\n    >>> logger.warn(\"Hello World\")\n    W0505 00:29:47 3525 <stdin>:<module>:1] Hello World\n\n    ## get_logger call with WARNING level.\n    >>> import logging\n    >>> logger = get_logger(\"Test2\", level=logging.WARNING)\n    >>> logger.warn(\"Hello World\")\n    W0505 00:30:50 3525 <stdin>:<module>:1] Hello World\n    >>> logger.debug(\"Hello World\") # This doesn't return anything as the level is logging.WARNING.\n\n    ## get_logger call with DEBUG level.\n    >>> logger = get_logger(\"Test3\", level=logging.DEBUG)\n    >>> logger.debug(\"Hello World\") # Logs the debug output as the level is logging.DEBUG.\n    D0505 00:31:30 3525 <stdin>:<module>:1] Hello World\n    \"\"\"\n    logger = logging.getLogger(name)\n    if name is not None and not getattr(logger, '_init_done', None):\n        logger._init_done = True\n        if filename:\n            mode = filemode if filemode else 'a'\n            hdlr = logging.FileHandler(filename, mode)\n        else:\n            hdlr = logging.StreamHandler() # pylint: disable=redefined-variable-type\n            # the `_Formatter` contain some escape character to\n            # represent color, which is not suitable for FileHandler,\n            # (TODO) maybe we can add another Formatter for FileHandler.\n            hdlr.setFormatter(_Formatter())\n        logger.addHandler(hdlr)\n        logger.setLevel(level)\n    return logger\n"
  },
  {
    "path": "python/mxnet/lr_scheduler.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Scheduling learning rate.\"\"\"\nimport logging\nfrom math import cos, pi\n\nclass LRScheduler(object):\n    \"\"\"Base class of a learning rate scheduler.\n\n    A scheduler returns a new learning rate based on the number of updates that have\n    been performed.\n\n    Parameters\n    ----------\n    base_lr : float, optional\n        The initial learning rate.\n    warmup_steps: int\n        number of warmup steps used before this scheduler starts decay\n    warmup_begin_lr: float\n        if using warmup, the learning rate from which it starts warming up\n    warmup_mode: string\n        warmup can be done in two modes.\n        'linear' mode gradually increases lr with each step in equal increments\n        'constant' mode keeps lr at warmup_begin_lr for warmup_steps\n    \"\"\"\n    def __init__(self, base_lr=0.01,\n                 warmup_steps=0, warmup_begin_lr=0, warmup_mode='linear'):\n        self.base_lr = base_lr\n        assert isinstance(warmup_steps, int)\n        self.warmup_steps = warmup_steps\n\n        self.warmup_final_lr = base_lr\n        self.warmup_begin_lr = warmup_begin_lr\n        if self.warmup_begin_lr > self.warmup_final_lr:\n            raise ValueError(\"Base lr has to be higher than warmup_begin_lr\")\n        if self.warmup_steps < 0:\n            raise ValueError(\"Warmup steps has to be positive or 0\")\n        if warmup_mode not in ['linear', 'constant']:\n            raise ValueError(\"Supports only linear and constant modes of warmup\")\n        self.warmup_mode = warmup_mode\n\n    def get_warmup_lr(self, num_update):\n        assert num_update < self.warmup_steps\n        if self.warmup_mode == 'linear':\n            increase = (self.warmup_final_lr - self.warmup_begin_lr) \\\n                       * float(num_update) / float(self.warmup_steps)\n            return self.warmup_begin_lr + increase\n        elif self.warmup_mode == 'constant':\n            return self.warmup_begin_lr\n        else:\n            raise ValueError(f\"Invalid warmup mode {self.warmup_mode}\")\n\n    def __call__(self, num_update):\n        \"\"\"Return a new learning rate.\n\n        The ``num_update`` is the upper bound of the number of updates applied to\n        every weight.\n\n        Assume the optimizer has updated *i*-th weight by *k_i* times, namely\n        ``optimizer.update(i, weight_i)`` is called by *k_i* times. Then::\n\n            num_update = max([k_i for all i])\n\n        Parameters\n        ----------\n        num_update: int\n            the maximal number of updates applied to a weight.\n        \"\"\"\n        raise NotImplementedError(\"must override this\")\n\nclass FactorScheduler(LRScheduler):\n    \"\"\"Reduce the learning rate by a factor for every *n* steps.\n\n    It returns a new learning rate by::\n\n        base_lr * pow(factor, floor(num_update/step))\n\n    Parameters\n    ----------\n    step : int\n        Changes the learning rate for every n updates.\n    factor : float, optional\n        The factor to change the learning rate.\n    stop_factor_lr : float, optional\n        Stop updating the learning rate if it is less than this value.\n    \"\"\"\n    def __init__(self, step, factor=1, stop_factor_lr=1e-8, base_lr=0.01,\n                 warmup_steps=0, warmup_begin_lr=0, warmup_mode='linear'):\n        super(FactorScheduler, self).__init__(base_lr, warmup_steps, warmup_begin_lr, warmup_mode)\n        if step < 1:\n            raise ValueError(\"Schedule step must be greater or equal than 1 round\")\n        if factor > 1.0:\n            raise ValueError(\"Factor must be no more than 1 to make lr reduce\")\n        self.step = step\n        self.factor = factor\n        self.stop_factor_lr = stop_factor_lr\n        self.count = 0\n\n    def __call__(self, num_update):\n        if num_update < self.warmup_steps:\n            return self.get_warmup_lr(num_update)\n\n        # NOTE: use while rather than if  (for continuing training via load_epoch)\n        while num_update > self.count + self.step:\n            self.count += self.step\n            self.base_lr *= self.factor\n            if self.base_lr < self.stop_factor_lr:\n                self.base_lr = self.stop_factor_lr\n                logging.info(\"Update[%d]: now learning rate arrived at %0.5e, will not \"\n                             \"change in the future\", num_update, self.base_lr)\n            else:\n                logging.info(\"Update[%d]: Change learning rate to %0.5e\",\n                             num_update, self.base_lr)\n        return self.base_lr\n\nclass MultiFactorScheduler(LRScheduler):\n    \"\"\"Reduce the learning rate by given a list of steps.\n\n    Assume there exists *k* such that::\n\n       step[k] <= num_update and num_update < step[k+1]\n\n    Then calculate the new learning rate by::\n\n       base_lr * pow(factor, k+1)\n\n    Parameters\n    ----------\n    step: list of int\n        The list of steps to schedule a change\n    factor: float\n        The factor to change the learning rate.\n    warmup_steps: int\n        number of warmup steps used before this scheduler starts decay\n    warmup_begin_lr: float\n        if using warmup, the learning rate from which it starts warming up\n    warmup_mode: string\n        warmup can be done in two modes.\n        'linear' mode gradually increases lr with each step in equal increments\n        'constant' mode keeps lr at warmup_begin_lr for warmup_steps\n    \"\"\"\n    def __init__(self, step, factor=1, base_lr=0.01, warmup_steps=0, warmup_begin_lr=0,\n                 warmup_mode='linear'):\n        super(MultiFactorScheduler, self).__init__(base_lr, warmup_steps,\n                                                   warmup_begin_lr, warmup_mode)\n        assert isinstance(step, list) and len(step) >= 1\n        for i, _step in enumerate(step):\n            if i != 0 and step[i] <= step[i-1]:\n                raise ValueError(\"Schedule step must be an increasing integer list\")\n            if _step < 1:\n                raise ValueError(\"Schedule step must be greater or equal than 1 round\")\n        if factor > 1.0:\n            raise ValueError(\"Factor must be no more than 1 to make lr reduce\")\n        self.step = step\n        self.cur_step_ind = 0\n        self.factor = factor\n        self.count = 0\n\n    def __call__(self, num_update):\n        if num_update < self.warmup_steps:\n            return self.get_warmup_lr(num_update)\n\n        # NOTE: use while rather than if  (for continuing training via load_epoch)\n        while self.cur_step_ind <= len(self.step)-1:\n            if num_update > self.step[self.cur_step_ind]:\n                self.count = self.step[self.cur_step_ind]\n                self.cur_step_ind += 1\n                self.base_lr *= self.factor\n                logging.info(\"Update[%d]: Change learning rate to %0.5e\",\n                             num_update, self.base_lr)\n            else:\n                return self.base_lr\n        return self.base_lr\n\nclass PolyScheduler(LRScheduler):\n    \"\"\" Reduce the learning rate according to a polynomial of given power.\n\n    Calculate the new learning rate, after warmup if any, by::\n\n       final_lr + (start_lr - final_lr) * (1-nup/max_nup)^pwr\n       if nup < max_nup, 0 otherwise.\n\n    Parameters\n    ----------\n        max_update: int\n            maximum number of updates before the decay reaches final learning rate.\n        base_lr: float\n            base learning rate to start from\n        pwr:   int\n            power of the decay term as a function of the current number of updates.\n        final_lr:   float\n            final learning rate after all steps\n        warmup_steps: int\n            number of warmup steps used before this scheduler starts decay\n        warmup_begin_lr: float\n            if using warmup, the learning rate from which it starts warming up\n        warmup_mode: string\n            warmup can be done in two modes.\n            'linear' mode gradually increases lr with each step in equal increments\n            'constant' mode keeps lr at warmup_begin_lr for warmup_steps\n    \"\"\"\n\n    def __init__(self, max_update, base_lr=0.01, pwr=2, final_lr=0,\n                 warmup_steps=0, warmup_begin_lr=0, warmup_mode='linear'):\n        super(PolyScheduler, self).__init__(base_lr, warmup_steps, warmup_begin_lr, warmup_mode)\n        assert isinstance(max_update, int)\n        if max_update < 1:\n            raise ValueError(\"maximum number of updates must be strictly positive\")\n        self.power = pwr\n        self.base_lr_orig = self.base_lr\n        self.max_update = max_update\n        self.final_lr = final_lr\n        self.max_steps = self.max_update - self.warmup_steps\n\n    def __call__(self, num_update):\n        if num_update < self.warmup_steps:\n            return self.get_warmup_lr(num_update)\n        if num_update <= self.max_update:\n            self.base_lr = self.final_lr + (self.base_lr_orig - self.final_lr) * \\\n                pow(1 - float(num_update - self.warmup_steps) / float(self.max_steps), self.power)\n        return self.base_lr\n\nclass CosineScheduler(LRScheduler):\n    \"\"\" Reduce the learning rate according to a cosine function\n\n    Calculate the new learning rate by::\n\n       final_lr + (start_lr - final_lr) * (1+cos(pi * nup/max_nup))/2\n       if nup < max_nup, 0 otherwise.\n\n    Parameters\n    ----------\n        max_update: int\n            maximum number of updates before the decay reaches 0\n        base_lr: float\n            base learning rate\n        final_lr: float\n            final learning rate after all steps\n        warmup_steps: int\n            number of warmup steps used before this scheduler starts decay\n        warmup_begin_lr: float\n            if using warmup, the learning rate from which it starts warming up\n        warmup_mode: string\n            warmup can be done in two modes.\n            'linear' mode gradually increases lr with each step in equal increments\n            'constant' mode keeps lr at warmup_begin_lr for warmup_steps\n    \"\"\"\n\n    def __init__(self, max_update, base_lr=0.01, final_lr=0,\n                 warmup_steps=0, warmup_begin_lr=0, warmup_mode='linear'):\n        super(CosineScheduler, self).__init__(base_lr, warmup_steps, warmup_begin_lr, warmup_mode)\n        assert isinstance(max_update, int)\n        if max_update < 1:\n            raise ValueError(\"maximum number of updates must be strictly positive\")\n        self.base_lr_orig = base_lr\n        self.max_update = max_update\n        self.final_lr = final_lr\n        self.max_steps = self.max_update - self.warmup_steps\n\n    def __call__(self, num_update):\n        if num_update < self.warmup_steps:\n            return self.get_warmup_lr(num_update)\n        if num_update <= self.max_update:\n            self.base_lr = self.final_lr + (self.base_lr_orig - self.final_lr) * \\\n                (1 + cos(pi * (num_update - self.warmup_steps) / self.max_steps)) / 2\n        return self.base_lr\n"
  },
  {
    "path": "python/mxnet/misc.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=invalid-name\n\"\"\"Learning rate scheduler.\"\"\"\n\nimport math\nimport logging\n\nclass LearningRateScheduler(object):\n    \"\"\"Base class of learning rate scheduler.\"\"\"\n    def __init__(self):\n        self.base_lr = 0.01\n\n    def __call__(self, iteration):\n        \"\"\"\n        Call to schedule current learning rate.\n\n        Parameters\n        ----------\n        iteration: int\n            Current iteration count.\n        \"\"\"\n        raise NotImplementedError(\"must override this\")\n\n\nclass FactorScheduler(LearningRateScheduler):\n    \"\"\"Reduce learning rate in factor.\n\n    Parameters\n    ----------\n    step: int\n        Schedule learning rate after every round.\n    factor: float\n        Reduce learning rate factor.\n    \"\"\"\n    def __init__(self, step, factor=0.1):\n        super(FactorScheduler, self).__init__()\n        if step < 1:\n            raise ValueError(\"Schedule step must be greater or equal than 1 round\")\n        if factor >= 1.0:\n            raise ValueError(\"Factor must be less than 1 to make lr reduce\")\n        self.step = step\n        self.factor = factor\n        self.old_lr = self.base_lr\n        self.init = False\n\n    def __call__(self, iteration):\n        \"\"\"\n        Call to schedule current learning rate.\n\n        Parameters\n        ----------\n        iteration: int\n            Current iteration count.\n        \"\"\"\n\n        if not self.init:\n            self.init = True\n            self.old_lr = self.base_lr\n        lr = self.base_lr * math.pow(self.factor, int(iteration / self.step))\n        if lr != self.old_lr:\n            self.old_lr = lr\n            logging.info(\"At Iteration [%d]: Swith to new learning rate %.5f\",\n                         iteration, lr)\n        return lr\n"
  },
  {
    "path": "python/mxnet/model.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=fixme, invalid-name, too-many-arguments, too-many-locals, too-many-lines\n# pylint: disable=too-many-branches, too-many-statements\n\"\"\"MXNet model module\"\"\"\n\nimport os\nimport logging\nfrom collections import namedtuple\nimport numpy as np\n\nfrom . import ndarray as nd\nfrom . import symbol as sym\nfrom . import kvstore as kvs\nfrom .device import cpu\n\nBASE_ESTIMATOR = object\n\ntry:\n    from sklearn.base import BaseEstimator\n    BASE_ESTIMATOR = BaseEstimator\nexcept ImportError:\n    SKLEARN_INSTALLED = False\n\n# Parameter to pass to batch_end_callback\nBatchEndParam = namedtuple('BatchEndParams',\n                           ['epoch',\n                            'nbatch',\n                            'eval_metric',\n                            'locals'])\n\ndef _create_sparse_kvstore(kvstore):\n    \"\"\"Create kvstore assuming some parameters' storage types are row_sparse.\n\n    Parameters\n    ----------\n    kvstore : KVStore or str\n        The kvstore.\n\n    Returns\n    -------\n    kvstore : KVStore\n    update_on_kvstore : bool. Always True.\n    \"\"\"\n    # always update on kvstore\n    if isinstance(kvstore, kvs.KVStore):\n        kv = kvstore\n    elif isinstance(kvstore, str):\n        kv = kvs.create(kvstore)\n    else:\n        raise TypeError(f\"Cannot create '{kvstore}' KVStore with row_sparse parameters. \"\n                        \"The type must be KVStore or str.\")\n    assert kv.is_capable(kvs.KVStoreBase.OPTIMIZER), \\\n        \"KVStore with sparse weight requires optimizer support. \" \\\n        \"However, type(kv) does not support optimizer. \" \\\n        \"Please consider other kvstore backends (e.g. dist_device) instead.\"\n    return (kv, True)\n\ndef _create_kvstore(kvstore, num_device, arg_params):\n    \"\"\"Create kvstore\n    This function select and create a proper kvstore if given the kvstore type.\n\n    Parameters\n    ----------\n    kvstore : KVStore or str\n        The kvstore.\n    num_device : int\n        The number of devices\n    arg_params : dict of str to `NDArray`.\n        Model parameter, dict of name to `NDArray` of net's weights.\n    \"\"\"\n    update_on_kvstore = bool(int(os.getenv('MXNET_UPDATE_ON_KVSTORE', \"1\")))\n    if kvstore is None:\n        kv = None\n    elif isinstance(kvstore, kvs.KVStoreBase):\n        kv = kvstore\n    elif isinstance(kvstore, str):\n        # create kvstore using the string type\n        if num_device == 1 and 'dist' not in kvstore:\n            # no need to use kv for single device and single machine\n            kv = None\n        else:\n            kv = kvs.create(kvstore)\n            if kvstore == 'local':\n                # automatically select a proper local\n                max_size = max(np.prod(param.shape) for param in\n                               arg_params.values())\n                if max_size > 1024 * 1024 * 16:\n                    update_on_kvstore = False\n    else:\n        raise TypeError('kvstore must be KVStore, str or None')\n\n    if kv is None:\n        update_on_kvstore = False\n    else:\n        update_on_kvstore &= kv.is_capable(kvs.KVStoreBase.OPTIMIZER)\n\n    return (kv, update_on_kvstore)\n\ndef _initialize_kvstore(kvstore, param_arrays, arg_params, param_names, update_on_kvstore):\n    \"\"\"Initialize kvstore\"\"\"\n    for idx, param_on_devs in enumerate(param_arrays):\n        name = param_names[idx]\n        if not update_on_kvstore or arg_params[name].stype != 'default':\n            kvstore.init(name, arg_params[name])\n        else:\n            kvstore.broadcast(name, arg_params[name], out=param_on_devs)\n\ndef _update_params_on_kvstore_nccl(param_arrays, grad_arrays, kvstore, param_names):\n    \"\"\"Perform update of param_arrays from grad_arrays on NCCL kvstore.\"\"\"\n    valid_indices = [index for index, grad_list in\n                     enumerate(grad_arrays) if grad_list[0] is not None]\n    valid_grad_arrays = [grad_arrays[i] for i in valid_indices]\n    valid_param_arrays = [param_arrays[i] for i in valid_indices]\n    valid_param_names = [param_names[i] for i in valid_indices]\n    size = len(valid_grad_arrays)\n    start = 0\n    # Use aggregation by default only with NCCL\n    default_batch = '16'\n    batch = int(os.getenv('MXNET_UPDATE_AGGREGATION_SIZE', default_batch))\n    while start < size:\n        end = start + batch if start + batch < size else size\n        # push gradient, priority is negative index\n        # pull back the weights\n        kvstore.pushpull(valid_param_names[start:end], valid_grad_arrays[start:end],\n                         out=valid_param_arrays[start:end], priority=-start)\n        start = end\n\ndef _update_params_on_kvstore(param_arrays, grad_arrays, kvstore, param_names):\n    \"\"\"Perform update of param_arrays from grad_arrays on kvstore.\"\"\"\n    for index, pair in enumerate(zip(param_arrays, grad_arrays)):\n        arg_list, grad_list = pair\n        if grad_list[0] is None:\n            continue\n        name = param_names[index]\n        # push gradient, priority is negative index\n        # pull back the weights\n        if grad_list[0].stype == 'default' and arg_list[0].stype == 'default':\n            kvstore.pushpull(name, grad_list, out=arg_list, priority=-index)\n        else:\n            kvstore.push(name, grad_list, priority=-index)\n            kvstore.pull(name, out=arg_list, priority=-index)\n\ndef _update_params(param_arrays, grad_arrays, updater, num_device,\n                   kvstore=None, param_names=None):\n    \"\"\"Perform update of param_arrays from grad_arrays not on kvstore.\"\"\"\n    updates = [[] for _ in range(num_device)]\n    for i, pair in enumerate(zip(param_arrays, grad_arrays)):\n        arg_list, grad_list = pair\n        if grad_list[0] is None:\n            continue\n        index = i\n        if kvstore:\n            name = param_names[index]\n            # push gradient, priority is negative index\n            if grad_list[0].stype == 'default' and arg_list[0].stype == 'default':\n                kvstore.pushpull(name, grad_list, priority=-index)\n            else:\n                kvstore.push(name, grad_list, priority=-index)\n                kvstore.pull(name, out=grad_list, priority=-index)\n        for k, p in enumerate(zip(arg_list, grad_list)):\n            # faked an index here, to make optimizer create diff\n            # state for the same index but on diff devs, TODO(mli)\n            # use a better solution later\n            w, g = p\n            updates[k].append((index*num_device+k, g, w))\n    for dev_updates in updates:\n        # update params if param_arrays and grad_arrays are not empty\n        if dev_updates:\n            i, w, g = zip(*dev_updates)\n            updater(i, w, g)\n\n\ndef save_checkpoint(prefix, epoch, symbol, arg_params, aux_params, remove_amp_cast=True):\n    \"\"\"Checkpoint the model data into file.\n\n    Parameters\n    ----------\n    prefix : str\n        Prefix of model name.\n    epoch : int\n        The epoch number of the model.\n    symbol : Symbol\n        The input Symbol.\n    arg_params : dict of str to NDArray\n        Model parameter, dict of name to NDArray of net's weights.\n    aux_params : dict of str to NDArray\n        Model parameter, dict of name to NDArray of net's auxiliary states.\n    remove_amp_cast : bool, optional\n        Whether to remove the amp_cast and amp_multicast operators, before saving the model.\n    Notes\n    -----\n    - ``prefix-symbol.json`` will be saved for symbol.\n    - ``prefix-epoch.params`` will be saved for parameters.\n    \"\"\"\n    if symbol is not None:\n        symbol.save(f'{prefix}-symbol.json', remove_amp_cast=remove_amp_cast)\n\n    save_dict = {(f'arg:{k}') : v.as_in_context(cpu()) for k, v in arg_params.items()}\n    save_dict.update({(f'aux:{k}') : v.as_in_context(cpu()) for k, v in aux_params.items()})\n    param_name = f'{prefix}-{epoch:04}.params'\n    nd.save(param_name, save_dict)\n    logging.info('Saved checkpoint to \"{}\"'.format(param_name))\n\n\ndef load_params(prefix, epoch):\n    \"\"\"Load params from a file\n    \"\"\"\n    save_dict = nd.load(f'{prefix}-{epoch:04}.params')\n    arg_params = {}\n    aux_params = {}\n    if not save_dict:\n        logging.warning(\"Params file '%s' is empty\", f'{prefix}-{epoch:04}.params')\n        return (arg_params, aux_params)\n    for k, v in save_dict.items():\n        tp, name = k.split(\":\", 1)\n        if tp == \"arg\":\n            arg_params[name] = v\n        if tp == \"aux\":\n            aux_params[name] = v\n    return (arg_params, aux_params)\n\ndef load_checkpoint(prefix, epoch):\n    \"\"\"Load model checkpoint from file.\n\n    Parameters\n    ----------\n    prefix : str\n        Prefix of model name.\n    epoch : int\n        Epoch number of model we would like to load.\n\n    Returns\n    -------\n    symbol : Symbol\n        The symbol configuration of computation network.\n    arg_params : dict of str to NDArray\n        Model parameter, dict of name to NDArray of net's weights.\n    aux_params : dict of str to NDArray\n        Model parameter, dict of name to NDArray of net's auxiliary states.\n\n    Notes\n    -----\n    - Symbol will be loaded from ``prefix-symbol.json``.\n    - Parameters will be loaded from ``prefix-epoch.params``.\n    \"\"\"\n    symbol = sym.load(f'{prefix}-symbol.json')\n    arg_params, aux_params = load_params(prefix, epoch)\n    return (symbol, arg_params, aux_params)\n"
  },
  {
    "path": "python/mxnet/name.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Automatic naming support for symbolic API.\"\"\"\nimport contextvars\n\n\nclass NameManager:\n    \"\"\"NameManager to do automatic naming.\n\n    Developers can also inherit from this class to change naming behavior.\n    \"\"\"\n    def __init__(self):\n        self._counter = {}\n        self._old_manager = None\n\n    def get(self, name, hint):\n        \"\"\"Get the canonical name for a symbol.\n\n        This is the default implementation.\n        If the user specifies a name,\n        the user-specified name will be used.\n\n        When user does not specify a name, we automatically generate a\n        name based on the hint string.\n\n        Parameters\n        ----------\n        name : str or None\n            The name specified by the user.\n\n        hint : str\n            A hint string, which can be used to generate name.\n\n        Returns\n        -------\n        full_name : str\n            A canonical name for the symbol.\n        \"\"\"\n        if name:\n            return name\n        if hint not in self._counter:\n            self._counter[hint] = 0\n        name = f'{hint}{self._counter[hint]}'\n        self._counter[hint] += 1\n        return name\n\n    def __enter__(self):\n        # Token can't be pickled and Token.old_value is Token.MISSING if _current.get() uses default value\n        self._old_manager = _current.get()\n        _current.set(self)\n        return self\n\n    def __exit__(self, ptype, value, trace):\n        _current.set(self._old_manager)\n\n\nclass Prefix(NameManager):\n    \"\"\"A name manager that attaches a prefix to all names.\n\n    Examples\n    --------\n    >>> import mxnet as mx\n    >>> data = mx.symbol.Variable('data')\n    >>> with mx.name.Prefix('mynet_'):\n            net = mx.symbol.FullyConnected(data, num_hidden=10, name='fc1')\n    >>> net.list_arguments()\n    ['data', 'mynet_fc1_weight', 'mynet_fc1_bias']\n    \"\"\"\n    def __init__(self, prefix):\n        super().__init__()\n        self._prefix = prefix\n\n    def get(self, name, hint):\n        name = super().get(name, hint)\n        return self._prefix + name\n\n\n_current = contextvars.ContextVar('namemanager', default=NameManager())\n\n\ndef current():\n    \"\"\"Returns the current name manager.\"\"\"\n    return _current.get()\n"
  },
  {
    "path": "python/mxnet/ndarray/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"NDArray API of MXNet.\"\"\"\n\nfrom . import _internal, contrib, linalg, op, random, sparse, utils, image, ndarray, numpy\n# pylint: disable=wildcard-import, redefined-builtin\ntry:\n    from .gen_op import * # pylint: disable=unused-wildcard-import\nexcept ImportError:\n    pass\nfrom . import register\nfrom .op import *\nfrom .ndarray import *\n# pylint: enable=wildcard-import\nfrom .utils import load, load_frombuffer, save, zeros, empty, array\nfrom .sparse import _ndarray_cls\nfrom .ndarray import _GRAD_REQ_MAP, dtype_mx_to_np, dtype_np_to_mx, _new_empty_handle\nfrom . import numpy as np\nfrom . import numpy_extension as npx\n\n__all__ = op.__all__ + ndarray.__all__ + utils.__all__ + \\\n          ['contrib', 'linalg', 'random', 'sparse', 'image', 'numpy', 'numpy_extension']\n"
  },
  {
    "path": "python/mxnet/ndarray/_internal.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=wildcard-import, unused-import\n\"\"\"NDArray namespace used to register internal functions.\"\"\"\nimport os as _os\nimport sys as _sys\n\ntry:\n    if int(_os.environ.get(\"MXNET_ENABLE_CYTHON\", True)) == 0:\n        from .._ctypes.ndarray import NDArrayBase\n        from .._ctypes.ndarray import _imperative_invoke\n        from .._ctypes.cached_op import CachedOp\n        from .._global_var import _set_ndarray_class, _set_np_ndarray_class\n    else:\n        from .._cy3.ndarray import NDArrayBase\n        from .._cy3.ndarray import _imperative_invoke\n        from .._ctypes.cached_op import CachedOp\n        from .._global_var import _set_ndarray_class, _set_np_ndarray_class\nexcept ImportError:\n    if int(_os.environ.get(\"MXNET_ENFORCE_CYTHON\", False)) != 0:\n        raise ImportError(\"Cython Module cannot be loaded but MXNET_ENFORCE_CYTHON=1\")\n    from .._ctypes.ndarray import NDArrayBase\n    from .._ctypes.ndarray import _imperative_invoke\n    from .._ctypes.cached_op import CachedOp\n    from .._global_var import _set_ndarray_class, _set_np_ndarray_class\n\nfrom ..base import _Null\ntry:\n    from .gen__internal import * # pylint: disable=unused-wildcard-import\nexcept ImportError:\n    pass\n\n__all__ = ['NDArrayBase', 'CachedOp', '_imperative_invoke', '_set_ndarray_class',\n           '_set_np_ndarray_class']\n"
  },
  {
    "path": "python/mxnet/ndarray/contrib.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import, unused-wildcard-import,redefined-outer-name\n\"\"\"Contrib NDArray API of MXNet.\"\"\"\nimport math\nimport numpy as np\nimport mxnet as mx\nfrom ..device import current_device\nfrom ..random import uniform\nfrom ..base import _as_list\nfrom . import ndarray\ntry:\n    from .gen_contrib import *\nexcept ImportError:\n    pass\n\n__all__ = [\"rand_zipfian\", \"foreach\", \"while_loop\", \"cond\", \"isinf\", \"isfinite\", \"isnan\"]\n\ndef _flatten_list(nested_list):\n    return [item for sublist in nested_list for item in sublist]\n\n# pylint: disable=line-too-long\ndef rand_zipfian(true_classes, num_sampled, range_max, ctx=None):\n    \"\"\"Draw random samples from an approximately log-uniform or Zipfian distribution.\n\n    This operation randomly samples *num_sampled* candidates the range of integers [0, range_max).\n    The elements of sampled_candidates are drawn with replacement from the base distribution.\n\n    The base distribution for this operator is an approximately log-uniform or Zipfian distribution:\n\n    P(class) = (log(class + 2) - log(class + 1)) / log(range_max + 1)\n\n    This sampler is useful when the true classes approximately follow such a distribution.\n    For example, if the classes represent words in a lexicon sorted in decreasing order of \\\n    frequency. If your classes are not ordered by decreasing frequency, do not use this op.\n\n    Additionaly, it also returns the number of times each of the \\\n    true classes and the sampled classes is expected to occur.\n\n    Parameters\n    ----------\n    true_classes : NDArray\n        A 1-D NDArray of the target classes.\n    num_sampled: int\n        The number of classes to randomly sample.\n    range_max: int\n        The number of possible classes.\n    ctx : Context\n        Device context of output. Default is current context.\n\n    Returns\n    -------\n    samples: NDArray\n        The sampled candidate classes in 1-D `int64` dtype.\n    expected_count_true: NDArray\n        The expected count for true classes in 1-D `float64` dtype.\n    expected_count_sample: NDArray\n        The expected count for sampled candidates in 1-D `float64` dtype.\n\n    Examples\n    --------\n    >>> true_cls = mx.nd.array([3])\n    >>> samples, exp_count_true, exp_count_sample = mx.nd.contrib.rand_zipfian(true_cls, 4, 5)\n    >>> samples\n    [1 3 3 3]\n    <NDArray 4 @cpu(0)>\n    >>> exp_count_true\n    [ 0.12453879]\n    <NDArray 1 @cpu(0)>\n    >>> exp_count_sample\n    [ 0.22629439  0.12453879  0.12453879  0.12453879]\n    <NDArray 4 @cpu(0)>\n    \"\"\"\n    if ctx is None:\n        ctx = current_device()\n    log_range = math.log(range_max + 1)\n    rand = uniform(0, log_range, shape=(num_sampled,), dtype='float64', ctx=ctx)\n    # make sure sampled_classes are in the range of [0, range_max)\n    sampled_classes = (rand.exp() - 1).astype('int64') % range_max\n\n    true_cls = true_classes.as_in_context(ctx).astype('float64')\n    expected_count_true = ((true_cls + 2.0) / (true_cls + 1.0)).log() / log_range * num_sampled\n    # cast sampled classes to fp64 to avoid interget division\n    sampled_cls_fp64 = sampled_classes.astype('float64')\n    expected_prob_sampled = ((sampled_cls_fp64 + 2.0) / (sampled_cls_fp64 + 1.0)).log() / log_range\n    expected_count_sampled = expected_prob_sampled * num_sampled\n    return sampled_classes, expected_count_true, expected_count_sampled\n# pylint: enable=line-too-long\n\n\ndef _flatten(args, inout_str):\n    if isinstance(args, ndarray.NDArray):\n        return [args], int(0)\n\n    assert isinstance(args, (list, tuple)), \\\n        f\"{inout_str} must be (nested) list of NDArray, \" \\\n        f\"but got {str(args)} of type {str(type(args))}\"\n    flat = []\n    fmts = []\n    for i in args:\n        arg, fmt = _flatten(i, inout_str)\n        flat.extend(arg)\n        fmts.append(fmt)\n    return flat, fmts\n\n\ndef _regroup(args, fmt):\n    if isinstance(fmt, int):\n        if fmt == 0:\n            return args[0], args[1:]\n        return args[:fmt], args[fmt:]\n\n    assert isinstance(args, (list, tuple)), \\\n        \"output must be (nested) list of NDArray, \" \\\n        f\"but got {str(args)} of type {str(type(args))}\"\n    ret = []\n    for i in fmt:\n        res, args = _regroup(args, i)\n        ret.append(res)\n    return ret, args\n\n\ndef foreach(body, data, init_states):\n    \"\"\"Run a for loop with user-defined computation over NDArrays on dimension 0.\n\n    This operator simulates a for loop and body has the computation for an iteration\n    of the for loop. It runs the computation in body on each slice from the input\n    NDArrays.\n\n    body takes two arguments as input and outputs a tuple of two elements,\n    as illustrated below::\n\n        out, states = body(data1, states)\n\n    data1 can be either an NDArray or a list of NDArrays. If data is an NDArray,\n    data1 is an NDArray. Otherwise, data1 is a list of NDArrays and has the same\n    size as data. states is a list of NDArrays and have the same size as init_states.\n    Similarly, out can be either an NDArray or a list of NDArrays, which are concatenated\n    as the first output of foreach; states from the last execution of body\n    are the second output of foreach.\n\n    The computation done by this operator is equivalent to the pseudo code below\n    when the input data is NDArray::\n\n        states = init_states\n        outs = []\n        for i in data.shape[0]:\n            s = data[i]\n            out, states = body(s, states)\n            outs.append(out)\n        outs = stack(*outs)\n\n\n    Parameters\n    ----------\n    body : a Python function.\n        Define computation in an iteration.\n    data: an NDArray or a list of NDArrays.\n        The input data.\n    init_states: an NDArray or nested lists of NDArrays.\n        The initial values of the loop states.\n\n    Returns\n    -------\n    outputs: an NDArray or nested lists of NDArrays.\n        The output data concatenated from the output of all iterations.\n    states: an NDArray or nested lists of NDArrays.\n        The loop states in the last iteration.\n\n    Examples\n    --------\n    >>> step = lambda data, states: (data + states[0], [states[0] * 2])\n    >>> data = mx.nd.random.uniform(shape=(2, 10))\n    >>> states = [mx.nd.random.uniform(shape=(10))]\n    >>> outs, states = mx.nd.contrib.foreach(step, data, states)\n    \"\"\"\n\n    def check_input(inputs, in_type, msg):\n        is_NDArray_or_list = True\n        if isinstance(inputs, list):\n            for i in inputs:\n                if not isinstance(i, in_type):\n                    is_NDArray_or_list = False\n                    break\n        else:\n            is_NDArray_or_list = isinstance(inputs, in_type)\n        assert is_NDArray_or_list, msg\n\n    flatten, _ = _flatten(data, \"foreach input\")\n    check_input(flatten, ndarray.NDArray,\n                \"data should be an NDArray or a nested list of NDArrays\")\n    flatten, _ = _flatten(init_states, \"foreach states\")\n    check_input(flatten, ndarray.NDArray,\n                \"init_states should be an NDArray or a nested list of NDArrays\")\n\n    not_data_list = isinstance(data, ndarray.NDArray)\n    num_iters = data.shape[0] if not_data_list else data[0].shape[0]\n    states = init_states\n    outputs = []\n    for i in range(num_iters):\n        if not_data_list:\n            eles = data[i]\n        else:\n            eles = [d[i] for d in data]\n        outs, states = body(eles, states)\n        outs, out_fmt = _flatten(outs, \"foreach output\")\n        outputs.append(outs)\n    outputs = zip(*outputs)\n    tmp_outputs = []\n    for out in outputs:\n        tmp_outputs.append(ndarray.op.stack(*out))\n    outputs = tmp_outputs\n    outputs, _ = _regroup(outputs, out_fmt)\n\n    return (outputs, states)\n\ndef while_loop(cond, func, loop_vars, max_iterations=None):\n    \"\"\"Run a while loop with user-defined computation and loop condition.\n\n    This operator simulates a while loop which iterately does customized computation\n    as long as the condition is satisfied.\n\n    `loop_vars` is a list of NDArrays on which the computation uses.\n\n    `cond` is a user-defined function, used as the loop condition.\n    It consumes `loop_vars`, and produces a scalar MXNet NDArray,\n    indicating the termination of the loop.\n    The loop ends when `cond` returns false (zero).\n    The `cond` is variadic, and its signature should be\n    `cond(*loop_vars) => NDArray`.\n\n    `func` is a user-defined function, used as the loop body.\n    It also consumes `loop_vars`, and produces `step_output` and `new_loop_vars` at each step.\n    In each step, `step_output` should contain the same number elements.\n    Through all steps, the i-th element of `step_output` should have the same shape and dtype.\n    Also, `new_loop_vars` should contain the same number of elements as `loop_vars`,\n    and the corresponding element should have the same shape and dtype.\n    The `func` is variadic, and its signature should be\n    `func(*loop_vars) =>\n    (NDArray or nested List[NDArray] step_output, NDArray or nested List[NDArray] new_loop_vars)`.\n\n    `max_iterations` is a scalar that defines the maximum number of iterations allowed.\n\n    This function returns two lists.\n    The first list has the length of `|step_output|`,\n    in which the i-th element are all i-th elements of\n    `step_output` from all steps, stacked along axis 0.\n    The second list has the length of `|loop_vars|`,\n    which represents final states of loop variables.\n\n    .. warning::\n\n       For now, the axis 0 of all NDArrays in the first list are `max_iterations`,\n       due to lack of dynamic shape inference.\n\n    .. warning::\n\n       When `cond` is never satisfied, we assume `step_output` is empty,\n       because it cannot be inferred. This is different from the symbolic version.\n\n    Parameters\n    ----------\n    cond: a Python function.\n        The loop condition.\n    func: a Python function.\n        The loop body.\n    loop_vars: an NDArray or nested lists of NDArrays.\n        The initial values of the loop variables.\n    max_iterations: a python int.\n        Maximum number of iterations.\n\n    Returns\n    ------\n    outputs: an NDArray or nested lists of NDArrays\n        stacked output from each step\n    states: an NDArray or nested lists of NDArrays\n        final state\n\n    Examples\n    --------\n    >>> cond = lambda i, s: i <= 5\n    >>> func = lambda i, s: ([i + s], [i + 1, s + i])\n    >>> loop_vars = (mx.nd.array([0], dtype=\"int64\"), mx.nd.array([1], dtype=\"int64\"))\n    >>> outputs, states = mx.nd.contrib.while_loop(cond, func, loop_vars, max_iterations=10)\n    >>> outputs\n    [\n    [[ 1]\n    [ 2]\n    [ 4]\n    [ 7]\n    [11]\n    [16]\n    [...]  # undefined value\n    [...]\n    [...]\n    [...]]\n    <NDArray 6x1 @cpu(0)>]\n    >>> states\n    [\n    [6]\n    <NDArray 1 @cpu(0)>,\n    [16]\n    <NDArray 1 @cpu(0)>]\n    \"\"\"\n    def _to_python_scalar(inputs, type_, name):\n        \"\"\"Converts \"inputs\", possibly typed mxnet NDArray, a numpy ndarray, other python types,\n        to the given type\n        \"\"\"\n        if isinstance(inputs, ndarray.NDArray):\n            inputs = inputs.asscalar()\n        try:\n            inputs = type_(inputs)\n        except:\n            raise ValueError(f\"Cannot convert {name} to python {type_.__name__}\")\n        return inputs\n\n    def _func_wrapper(loop_vars):\n        \"\"\"This wrapper unifies\n             \"func: loop_vars -> new_loop_vars\"\n         and \"func: loop_vars -> (step_output, new_loop_vars)\"\n        into \"func: loop_vars -> (None or tuple of step_outputs, tuple of new_loop_vars)\n        \"\"\"\n        step_output, new_loop_vars = func(*loop_vars)\n        if step_output is None:\n            step_output = []\n        if new_loop_vars is None:\n            new_loop_vars = []\n        if isinstance(step_output, tuple):\n            step_output = list(step_output)\n        if isinstance(new_loop_vars, tuple):\n            new_loop_vars = list(new_loop_vars)\n        new_loop_vars = _as_list(new_loop_vars)\n        if len(loop_vars) != len(new_loop_vars):\n            raise ValueError(\"The length of loop_vars should be consistent during the loop\")\n        return step_output, new_loop_vars\n\n    if max_iterations is None:\n        raise ValueError(\"max_iterations should be specified\")\n    max_iterations = _to_python_scalar(max_iterations, int, \"max_iteration\")\n    # It should be work as fine if loop_vars are empty I guess,\n    # but it is semantically unnecessary to include this case.\n    if len(loop_vars) == 0:\n        raise ValueError(\"loop_vars should contain at least one element\")\n\n    steps = 0\n    outputs = []\n    # there might not be an iteration.\n    out_fmt = None\n    not_loop_var_list = isinstance(loop_vars, ndarray.NDArray)\n    loop_vars = _as_list(loop_vars)\n    while steps < max_iterations and \\\n            _to_python_scalar(cond(*loop_vars), bool, \"Return value of cond\"): # loop condition\n        step_output, loop_vars = _func_wrapper(loop_vars)\n        step_output, out_fmt = _flatten(step_output, \"while output\")\n        outputs.append(step_output)\n        steps += 1\n        if len(outputs) != steps or len(step_output) != len(outputs[0]):\n            raise ValueError(\"Number of elements in step_output should be the same in each step\")\n    stacked_outputs = []\n    for i_th, items in enumerate(zip(*outputs), 1):\n        # `mx.ndarray.pad` only support 4-D or 5-D inputs for now\n        # so we could not use it.\n        items = [x.expand_dims(0) for x in items]\n        if steps != max_iterations and items:\n            pad_shape = [max_iterations - steps] + list(items[0].shape[1: ])\n            pad = ndarray.empty(\n                shape=pad_shape,\n                ctx=items[0].context,\n                dtype=items[0].dtype,\n            )\n            items = list(items) + [pad]\n        try:\n            stacked_outputs.append(ndarray.op.concat(*items, dim=0))\n        except ValueError:\n            raise ValueError(\"\\n\".join(\n                [f\"Shapes of {i_th}-th elements in step_outputs are inconsistent, which are:\"] +\n                [f\"  Step {i}, shape is {str(x.shape)}\" for i, x in enumerate(items)]\n            ))\n    if out_fmt is not None:\n        stacked_outputs, _ = _regroup(stacked_outputs, out_fmt)\n    if not_loop_var_list:\n        loop_vars = loop_vars[0]\n    return stacked_outputs, loop_vars\n\ndef cond(pred, then_func, else_func):\n    \"\"\"Run an if-then-else using user-defined condition and computation\n\n    This operator simulates a if-like branch which chooses to do one of\n    the two customized computations according to the specified condition.\n\n    `pred` is a scalar MXNet NDArray,\n    indicating which branch of computation should be used.\n\n    `then_func` is a user-defined function, used as computation of the then branch.\n    It produces `outputs`, which is a list of NDArrays.\n    The signature of `then_func` should be\n    `then_func() => NDArray or nested List[NDArray]`.\n\n    `else_func` is a user-defined function, used as computation of the else branch.\n    It produces `outputs`, which is a list of NDArrays.\n    The signature of `else_func` should be\n    `else_func() => NDArray or nested List[NDArray]`.\n\n    The `outputs` produces by `then_func` and `else_func` should have the same number\n    of elements, all of which should be in the same shape, of the same dtype and stype.\n\n    This function returns a list of symbols, representing the computation result.\n\n    Parameters\n    ----------\n    pred: a MXNet NDArray representing a scalar.\n        The branch condition.\n    then_func: a Python function.\n        The computation to be executed if `pred` is true.\n    else_func: a Python function.\n        The computation to be executed if `pred` is false.\n\n    Returns\n    -------\n    outputs: an NDArray or nested lists of NDArrays, representing the result of computation.\n\n    Examples\n    --------\n    >>> a, b = mx.nd.array([1]), mx.nd.array([2])\n    >>> pred = a * b < 5\n    >>> then_func = lambda: (a + 5) * (b + 5)\n    >>> else_func = lambda: (a - 5) * (b - 5)\n    >>> outputs = mx.nd.contrib.cond(pred, then_func, else_func)\n    >>> outputs[0]\n    [42.]\n    <NDArray 1 @cpu(0)>\n    \"\"\"\n    def _to_python_scalar(inputs, type_, name):\n        \"\"\"Converts \"inputs\", possibly typed mxnet NDArray, a numpy ndarray, other python types,\n        to the given type\n        \"\"\"\n        if hasattr(inputs, \"asscalar\"):\n            inputs = inputs.asscalar()\n        try:\n            inputs = type_(inputs)\n        except:\n            raise ValueError(f\"Cannot convert {name} to python {type_.__name__}\")\n        return inputs\n\n    branch = _to_python_scalar(pred, bool, \"pred\")\n    if branch:\n        return then_func()\n    else:\n        return else_func()\n\ndef isinf(data):\n    \"\"\"Performs an element-wise check to determine if the NDArray contains an infinite element\n    or not.\n\n\n    Parameters\n    ----------\n    input : NDArray\n        An N-D NDArray.\n\n    Returns\n    -------\n    output: NDArray\n        The output NDarray, with same shape as input, where 1 indicates the array element is\n        equal to positive or negative infinity and 0 otherwise.\n\n    Examples\n    --------\n    >>> data = mx.nd.array([np.inf, -np.inf, np.NINF, -1])\n    >>> output = mx.nd.contrib.isinf(data)\n    >>> output\n    [1. 1. 1. 0.]\n    <NDArray 4 @cpu(0)>\n    \"\"\"\n    return data.abs() == np.inf\n\ndef isfinite(data):\n    \"\"\"Performs an element-wise check to determine if the NDArray contains an infinite element\n    or not.\n\n\n    Parameters\n    ----------\n    input : NDArray\n        An N-D NDArray.\n\n    Returns\n    -------\n    output: NDArray\n        The output NDarray, with same shape as input, where 1 indicates the array element is\n        finite i.e. not equal to positive or negative infinity and 0 in places where it is\n        positive or negative infinity.\n\n    Examples\n    --------\n    >>> data = mx.nd.array([np.inf, -np.inf, np.NINF, -1])\n    >>> output = mx.nd.contrib.isfinite(data)\n    >>> output\n    [0. 0. 0. 1.]\n    <NDArray 4 @cpu(0)>\n    \"\"\"\n    is_data_not_nan = data == data  # pylint: disable=comparison-with-itself\n    is_data_not_infinite = data.abs() != np.inf\n    return ndarray.logical_and(is_data_not_infinite, is_data_not_nan)\n\ndef isnan(data):\n    \"\"\"Performs an element-wise check to determine if the NDArray contains a NaN element\n    or not.\n\n\n    Parameters\n    ----------\n    data : NDArray\n        An N-D NDArray.\n\n    Returns\n    -------\n    output: NDArray\n        The output NDarray, with same shape as input, where 1 indicates the array element is\n        NaN i.e. Not a Number and 0 otherwise.\n\n    Examples\n    --------\n    >>> data = mx.nd.array([np.nan, -1])\n    >>> output = mx.nd.contrib.isnan(data)\n    >>> output\n    [1. 0.]\n    <NDArray 2 @cpu(0)>\n    \"\"\"\n    return data != data  # pylint: disable=comparison-with-itself\n\ndef _get_rescale_grad(rescale_grad, ctx=mx.cpu()):\n    if not isinstance(rescale_grad, ndarray.NDArray):\n        return ndarray.full(shape=(1,), val=rescale_grad, ctx=ctx)\n    else:\n        return rescale_grad.as_in_context(ctx)\n\ndef adamw_update(weight, grad, mean, var, rescale_grad, lr, eta, beta1=0.9, beta2=0.999,\n                 epsilon=1e-8, wd=0, clip_gradient=-1, out=None, name=None, **kwargs):\n    rescale_grad = _get_rescale_grad(rescale_grad, ctx=weight.context)\n    return ndarray._internal._adamw_update(weight=weight, grad=grad, mean=mean, var=var,\n                                           rescale_grad=rescale_grad, lr=lr, eta=eta,\n                                           beta1=beta1, beta2=beta2, epsilon=epsilon,\n                                           wd=wd, clip_gradient=clip_gradient, out=out,\n                                           name=name, **kwargs)\n\ndef mp_adamw_update(weight, grad, mean, var, weight32, rescale_grad, lr, eta, beta1=0.9,\n                    beta2=0.999, epsilon=1e-8, wd=0, clip_gradient=-1, out=None,\n                    name=None, **kwargs):\n    rescale_grad = _get_rescale_grad(rescale_grad, ctx=weight.context)\n    return ndarray._internal._mp_adamw_update(weight=weight, grad=grad, mean=mean, var=var,\n                                              weight32=weight32,\n                                              rescale_grad=rescale_grad, lr=lr, eta=eta,\n                                              beta1=beta1, beta2=beta2, epsilon=epsilon,\n                                              wd=wd, clip_gradient=clip_gradient, out=out,\n                                              name=name, **kwargs)\n\ndef multi_adamw_update(weights, grads, mean, var, rescale_grad, lrs, wds, etas,\n                       out=None, name=None, size=0, **kwargs):\n    if not size:\n        size = len(weights)\n\n    rescale_grad = _get_rescale_grad(rescale_grad, ctx=weights[0].context)\n    temp_list = _flatten_list(zip(weights, grads, mean, var)) + [rescale_grad]\n    return ndarray._internal._multi_adamw_update(*temp_list,\n                                                 out=out,\n                                                 num_weights=size,\n                                                 lrs=lrs,\n                                                 wds=wds,\n                                                 etas=etas,\n                                                 name=name,\n                                                 **kwargs)\n\ndef multi_mp_adamw_update(weights, grads, mean, var, weights32, rescale_grad, lrs, wds, etas,\n                          out=None, name=None, size=0, **kwargs):\n    if not size:\n        size = len(weights)\n\n    rescale_grad = _get_rescale_grad(rescale_grad, ctx=weights[0].context)\n    temp_list = _flatten_list(zip(weights, grads, mean, var, weights32)) + [rescale_grad]\n    return ndarray._internal._multi_mp_adamw_update(*temp_list,\n                                                    out=out,\n                                                    num_weights=size,\n                                                    lrs=lrs,\n                                                    wds=wds,\n                                                    etas=etas,\n                                                    name=name,\n                                                    **kwargs)\n\ndef multi_lamb_update(weights, grads, mean, var, step_count,\n                      lrs, wds, out=None, num_tensors=0, **kwargs):\n    \"\"\"Given a list of gradients, update weights, mean and variance of multiple tensors\n    following LAMB Optimizer implementation.\n\n    Parameters\n    ----------\n    weights : List of NDArrays containing the input weights of multiple tensors\n\n    grads : List of NDArrays containing input gradients\n\n    mean : List of NDArrays containing mean of multiple tensors to be updated\n\n    var : List of NDArrays containing variance of multiple tensors to be updated\n\n    step_count : List of scalars with the number of update step for each tensor\n\n    lrs : List of learning rates (one for each tensor)\n\n    wds : List of weight decays (one for each tensor)\n\n    out: List of NDArrays where the updated weights will be stored\n\n    num_tensors : Number of NDArrays/tensors in the list\n    \"\"\"\n\n    if not num_tensors:\n        num_tensors = len(weights)\n    temp_list = _flatten_list(zip(weights, grads, mean, var))\n    return ndarray._internal._multi_lamb_update(*temp_list,\n                                                out=out,\n                                                num_tensors=num_tensors,\n                                                step_count=step_count,\n                                                learning_rates=lrs,\n                                                wds=wds,\n                                                **kwargs)\n\ndef multi_mp_lamb_update(weights, grads, mean, var, weights32, step_count,\n                         lrs, wds, out=None, num_tensors=0, **kwargs):\n    \"\"\"Given a list of gradients, update weights, mean and variance of multiple tensors\n    following LAMB Optimizer implementation, and using Mixed-Precision.\n\n    Parameters\n    ----------\n    weights : List of NDArrays containing the input weights of multiple tensors\n\n    grads : List of NDArrays containing input gradients\n\n    mean : List of NDArrays containing mean of multiple tensors to be updated\n\n    var : List of NDArrays containing variance of multiple tensors to be updated\n\n    weights32 : Master copy of weights in FP32\n\n    step_count : List of scalars with the number of update step for each tensor\n\n    lrs : List of learning rates (one for each tensor)\n\n    wds : List of weight decays (one for each tensor)\n\n    out: List of NDArrays where the updated weights will be stored\n\n    num_tensors : Number of NDArrays/tensors in the list\n    \"\"\"\n\n    if not num_tensors:\n        num_tensors = len(weights)\n    temp_list = _flatten_list(zip(weights, grads, mean, var, weights32))\n    return ndarray._internal._multi_mp_lamb_update(*temp_list,\n                                                   out=out,\n                                                   num_tensors=num_tensors,\n                                                   step_count=step_count,\n                                                   learning_rates=lrs,\n                                                   wds=wds,\n                                                   **kwargs)\n\ndef adabelief_update(weight, grad, mean, var, rescale_grad, lr, eta, beta1=0.9, beta2=0.999,\n                     epsilon=1e-8, wd=0, clip_gradient=-1, out=None, name=None, **kwargs):\n    rescale_grad = _get_rescale_grad(rescale_grad, ctx=weight.context)\n    return ndarray._internal._adabelief_update(weight=weight, grad=grad, mean=mean, var=var,\n                                               rescale_grad=rescale_grad, lr=lr, eta=eta,\n                                               beta1=beta1, beta2=beta2, epsilon=epsilon,\n                                               wd=wd, clip_gradient=clip_gradient, out=out,\n                                               name=name, **kwargs)\n\ndef mp_adabelief_update(weight, grad, mean, var, weight32, rescale_grad, lr, eta, beta1=0.9,\n                        beta2=0.999, epsilon=1e-8, wd=0, clip_gradient=-1, out=None,\n                        name=None, **kwargs):\n    rescale_grad = _get_rescale_grad(rescale_grad, ctx=weight.context)\n    return ndarray._internal._mp_adabelief_update(weight=weight, grad=grad, mean=mean, var=var,\n                                                  weight32=weight32,\n                                                  rescale_grad=rescale_grad, lr=lr, eta=eta,\n                                                  beta1=beta1, beta2=beta2, epsilon=epsilon,\n                                                  wd=wd, clip_gradient=clip_gradient, out=out,\n                                                  name=name, **kwargs)\n\ndef multi_adabelief_update(weights, grads, mean, var, rescale_grad, lrs, wds, etas,\n                           out=None, name=None, size=0, **kwargs):\n    if not size:\n        size = len(weights)\n\n    rescale_grad = _get_rescale_grad(rescale_grad, ctx=weights[0].context)\n    temp_list = _flatten_list(zip(weights, grads, mean, var)) + [rescale_grad]\n    return ndarray._internal._multi_adabelief_update(*temp_list,\n                                                     out=out,\n                                                     num_weights=size,\n                                                     lrs=lrs,\n                                                     wds=wds,\n                                                     etas=etas,\n                                                     name=name,\n                                                     **kwargs)\n\ndef multi_mp_adabelief_update(weights, grads, mean, var, weights32, rescale_grad, lrs, wds, etas,\n                              out=None, name=None, size=0, **kwargs):\n    if not size:\n        size = len(weights)\n\n    rescale_grad = _get_rescale_grad(rescale_grad, ctx=weights[0].context)\n    temp_list = _flatten_list(zip(weights, grads, mean, var, weights32)) + [rescale_grad]\n    return ndarray._internal._multi_mp_adabelief_update(*temp_list,\n                                                        out=out,\n                                                        num_weights=size,\n                                                        lrs=lrs,\n                                                        wds=wds,\n                                                        etas=etas,\n                                                        name=name,\n                                                        **kwargs)\n\ndef multi_lans_update(weights, grads, mean, var, step_count,\n                      lrs, wds, out=None, num_tensors=0, **kwargs):\n    \"\"\"Given a list of gradients, update weights, mean and variance of multiple tensors\n    following LANS Optimizer implementation.\n\n    Parameters\n    ----------\n    weights : List of NDArrays containing the input weights of multiple tensors\n\n    grads : List of NDArrays containing input gradients\n\n    mean : List of NDArrays containing mean of multiple tensors to be updated\n\n    var : List of NDArrays containing variance of multiple tensors to be updated\n\n    step_count : List of scalars with the number of update step for each tensor\n\n    lrs : List of learning rates (one for each tensor)\n\n    wds : List of weight decays (one for each tensor)\n\n    out: List of NDArrays where the updated weights will be stored\n\n    num_tensors : Number of NDArrays/tensors in the list\n    \"\"\"\n\n    if not num_tensors:\n        num_tensors = len(weights)\n    temp_list = _flatten_list(zip(weights, grads, mean, var))\n    return ndarray._internal._multi_lans_update(*temp_list,\n                                                out=out,\n                                                num_tensors=num_tensors,\n                                                step_count=step_count,\n                                                learning_rates=lrs,\n                                                wds=wds,\n                                                **kwargs)\n\n\ndef multi_mp_lans_update(weights, grads, mean, var, weights32, step_count,\n                         lrs, wds, out=None, num_tensors=0, **kwargs):\n    \"\"\"Given a list of gradients, update weights, mean and variance of multiple tensors\n    following LANS Optimizer implementation, and using Mixed-Precision.\n\n    Parameters\n    ----------\n    weights : List of NDArrays containing the input weights of multiple tensors\n\n    grads : List of NDArrays containing input gradients\n\n    mean : List of NDArrays containing mean of multiple tensors to be updated\n\n    var : List of NDArrays containing variance of multiple tensors to be updated\n\n    weights32 : Master copy of weights in FP32\n\n    step_count : List of scalars with the number of update step for each tensor\n\n    lrs : List of learning rates (one for each tensor)\n\n    wds : List of weight decays (one for each tensor)\n\n    out: List of NDArrays where the updated weights will be stored\n\n    num_tensors : Number of NDArrays/tensors in the list\n    \"\"\"\n\n    if not num_tensors:\n        num_tensors = len(weights)\n    temp_list = _flatten_list(zip(weights, grads, mean, var, weights32))\n    return ndarray._internal._multi_mp_lans_update(*temp_list,\n                                                   out=out,\n                                                   num_tensors=num_tensors,\n                                                   step_count=step_count,\n                                                   learning_rates=lrs,\n                                                   wds=wds,\n                                                   **kwargs)\n"
  },
  {
    "path": "python/mxnet/ndarray/image.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import, unused-wildcard-import\n\"\"\"Image NDArray API of MXNet.\"\"\"\ntry:\n    from .gen_image import *\nexcept ImportError:\n    pass\n\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/ndarray/linalg.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import, unused-wildcard-import\n\"\"\"Linear Algebra NDArray API of MXNet.\"\"\"\ntry:\n    from .gen_linalg import *\nexcept ImportError:\n    pass\n\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/ndarray/ndarray.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=too-many-lines, protected-access\n# pylint: disable=import-error, no-name-in-module, undefined-variable\n\n\"\"\"NDArray API of MXNet.\"\"\"\n\n\ntry:\n    from __builtin__ import slice as py_slice\nexcept ImportError:\n    from builtins import slice as py_slice\n\nfrom array import array as native_array\nimport ctypes\nimport warnings\nimport operator\nfrom functools import reduce # pylint: disable=redefined-builtin\nimport numpy as np\nfrom ..base import _LIB, numeric_types, integer_types\nfrom ..base import c_array, c_array_buf, c_handle_array, mx_real_t\nfrom ..base import mx_uint, NDArrayHandle, check_call, mx_int, mx_int64\nfrom ..base import ctypes2buffer\nfrom ..dlpack import ndarray_to_dlpack_for_read, ndarray_to_dlpack_for_write\nfrom ..dlpack import ndarray_from_dlpack, ndarray_from_numpy\nfrom ..runtime import Features\nfrom ..device import Device, current_device\nfrom ..util import is_np_array\nfrom . import _internal\nfrom . import op\nfrom ._internal import NDArrayBase\n\n__all__ = [\"NDArray\", \"concatenate\", \"dtype_np_to_mx\", \"dtype_mx_to_np\", \"_GRAD_REQ_MAP\",\n           \"ones\", \"add\", \"arange\", \"linspace\", \"eye\", \"divide\", \"equal\", \"full\", \"greater\",\n           \"greater_equal\", \"imdecode\", \"lesser\", \"lesser_equal\", \"logical_and\", \"logical_or\",\n           \"logical_xor\", \"maximum\", \"minimum\", \"moveaxis\", \"modulo\", \"multiply\", \"not_equal\",\n           \"onehot_encode\", \"power\", \"subtract\", \"true_divide\", \"waitall\", \"_new_empty_handle\",\n           \"histogram\", \"split_v2\", \"to_dlpack_for_read\", \"to_dlpack_for_write\", \"from_dlpack\",\n           \"from_numpy\", \"zeros\", \"indexing_key_expand_implicit_axes\", \"get_indexing_dispatch_code\",\n           \"get_oshape_of_gather_nd_op\", \"bfloat16\", \"get_dtype_type\", \"is_mx_dtype\",\n           \"get_dtype_name\"]\n\n_STORAGE_TYPE_UNDEFINED = -1\n_STORAGE_TYPE_DEFAULT = 0\n_STORAGE_TYPE_ROW_SPARSE = 1\n_STORAGE_TYPE_CSR = 2\n_SIGNED_INT32_UPPER_LIMIT = (2**31 - 1)\n\nbfloat16 = np.dtype([('bfloat16', np.uint16)])\n\n# pylint: disable= no-member\n_DTYPE_NP_TO_MX = {\n    None: -1,\n    np.float32: 0,\n    np.float64: 1,\n    np.float16: 2,\n    np.uint8: 3,\n    np.int32: 4,\n    np.int8: 5,\n    np.int64: 6,\n    np.bool_: 7,\n    np.int16: 8,\n    np.uint16 : 9,\n    np.uint32 : 10,\n    np.uint64 : 11,\n    bfloat16: 12,\n}\n\ndef _register_platform_dependent_mx_dtype():\n    \"\"\"Register platform dependent types to the fixed size counterparts.\"\"\"\n    kind_map = {'i': 'int', 'u': 'uint', 'f': 'float'}\n    for np_type in [\n            np.byte, np.ubyte, np.short, np.ushort, np.intc, np.uintc, np.int_,\n            np.uint, np.longlong, np.ulonglong, np.half, np.float16, np.single,\n            np.double, np.longdouble]:\n        dtype = np.dtype(np_type)\n        kind, size = dtype.kind, dtype.itemsize\n        bits = size * 8\n        fixed_dtype = getattr(np, kind_map[kind]+str(bits))\n        if fixed_dtype in _DTYPE_NP_TO_MX:\n            _DTYPE_NP_TO_MX[np_type] = _DTYPE_NP_TO_MX[fixed_dtype]\n_register_platform_dependent_mx_dtype()\n\n_DTYPE_MX_TO_NP = {\n    -1: None,\n    0: np.float32,\n    1: np.float64,\n    2: np.float16,\n    3: np.uint8,\n    4: np.int32,\n    5: np.int8,\n    6: np.int64,\n    7: np.bool_,\n    8: np.int16,\n    9: np.uint16,\n    10: np.uint32,\n    11: np.uint64,\n    12: bfloat16,\n}\n\ndef get_dtype_type(dtype):\n    if (isinstance(dtype, str) and dtype in bfloat16.names) or np.dtype(dtype) == bfloat16:\n        return bfloat16\n    return np.dtype(dtype).type\n\ndef is_mx_dtype(dtype):\n    return get_dtype_type(dtype) in _DTYPE_NP_TO_MX\n\ndef get_dtype_name(dtype):\n    dtype = np.dtype(get_dtype_type(dtype))\n    return bfloat16.names[0] if dtype == bfloat16 else dtype.name\n\ndef dtype_np_to_mx(dtype):\n    if not is_mx_dtype(dtype):\n        raise TypeError('dtype must be one of: ' + str(_DTYPE_NP_TO_MX))\n    dtype_type = get_dtype_type(dtype)\n    return _DTYPE_NP_TO_MX[dtype_type]\n\ndef dtype_mx_to_np(dtype_idx):\n    return _DTYPE_MX_TO_NP[dtype_idx]\n\n\n_STORAGE_TYPE_STR_TO_ID = {\n    'undefined': _STORAGE_TYPE_UNDEFINED,\n    'default': _STORAGE_TYPE_DEFAULT,\n    'row_sparse': _STORAGE_TYPE_ROW_SPARSE,\n    'csr': _STORAGE_TYPE_CSR,\n}\n\n_STORAGE_TYPE_ID_TO_STR = {\n    _STORAGE_TYPE_UNDEFINED: 'undefined',\n    _STORAGE_TYPE_DEFAULT: 'default',\n    _STORAGE_TYPE_ROW_SPARSE: 'row_sparse',\n    _STORAGE_TYPE_CSR: 'csr',\n}\n\n_GRAD_REQ_MAP = {\n    'null': 0,\n    'write': 1,\n    'add': 3\n}\n# pylint: enable= no-member\n\n# Return code for dispatching indexing function call\n_NDARRAY_UNSUPPORTED_INDEXING = -1\n_NDARRAY_BASIC_INDEXING = 0\n_NDARRAY_ADVANCED_INDEXING = 1\n_NDARRAY_EMPTY_TUPLE_INDEXING = 2\n\n# Return code for 0-d boolean array handler\n_NDARRAY_NO_ZERO_DIM_BOOL_ARRAY = -1\n_NDARRAY_ZERO_DIM_BOOL_ARRAY_FALSE = 0\n_NDARRAY_ZERO_DIM_BOOL_ARRAY_TRUE = 1\n\n# Caching whether MXNet was built with INT64 support or not\n_INT64_TENSOR_SIZE_ENABLED = None\n\ndef _int64_enabled():\n    global _INT64_TENSOR_SIZE_ENABLED\n    if _INT64_TENSOR_SIZE_ENABLED is None:\n        _INT64_TENSOR_SIZE_ENABLED = Features().is_enabled('INT64_TENSOR_SIZE')\n    return _INT64_TENSOR_SIZE_ENABLED\n\ndef _new_empty_handle():\n    \"\"\"Returns a new empty handle.\n\n    Empty handle can be used to hold a result.\n\n    Returns\n    -------\n    handle\n        A new empty `NDArray` handle.\n    \"\"\"\n    hdl = NDArrayHandle()\n    check_call(_LIB.MXNDArrayCreateNone(ctypes.byref(hdl)))\n    return hdl\n\n\ndef _new_alloc_handle(shape, ctx, delay_alloc, dtype=mx_real_t):\n    \"\"\"Return a new handle with specified shape and context.\n\n    Empty handle is only used to hold results.\n\n    Returns\n    -------\n    handle\n        A new empty `NDArray` handle.\n    \"\"\"\n    hdl = NDArrayHandle()\n    if _int64_enabled():\n        check_call(_LIB.MXNDArrayCreate64(\n            c_array_buf(mx_int64, native_array('q', shape)),\n            ctypes.c_int(len(shape)),\n            ctypes.c_int(ctx.device_typeid),\n            ctypes.c_int(ctx.device_id),\n            ctypes.c_int(int(delay_alloc)),\n            ctypes.c_int(int(dtype_np_to_mx(dtype))),\n            ctypes.byref(hdl)))\n    else:\n        # When shape is larger than unit32 then there is an overflow error at python end itself.\n        # It needs to be caught here since the call doesn't even reach backend.\n        size = 1\n        for idx in shape:\n            size = size * idx\n        if size > _SIGNED_INT32_UPPER_LIMIT:\n            raise Exception(\"[_new_alloc_handle] Size of tensor you are trying to allocate is \" +\n                            \"larger than 2^31 elements. Please build with flag \" +\n                            \"USE_INT64_TENSOR_SIZE=1\")\n        check_call(_LIB.MXNDArrayCreate(\n            c_array_buf(mx_uint, native_array('I', shape)),\n            mx_uint(len(shape)),\n            ctypes.c_int(ctx.device_typeid),\n            ctypes.c_int(ctx.device_id),\n            ctypes.c_int(int(delay_alloc)),\n            ctypes.c_int(int(dtype_np_to_mx(dtype))),\n            ctypes.byref(hdl)))\n    return hdl\n\n\ndef _new_from_shared_mem(shared_pid, shared_id, shape, dtype):\n    hdl = NDArrayHandle()\n    check_call(_LIB.MXNDArrayCreateFromSharedMem(\n        ctypes.c_int(shared_pid),\n        ctypes.c_int(shared_id),\n        c_array(mx_int, shape),\n        mx_int(len(shape)),\n        ctypes.c_int(int(dtype_np_to_mx(dtype))),\n        ctypes.byref(hdl)))\n    return hdl\n\n\ndef waitall():\n    \"\"\"Wait for all async operations to finish in MXNet.\n\n    This function is used for benchmarking only.\n\n    .. note::\n\n       If your mxnet code throws an exception, then waitall can cause performance impact.\n    \"\"\"\n    check_call(_LIB.MXNDArrayWaitAll())\n\n\ndef _storage_type(handle):\n    storage_type = ctypes.c_int(0)\n    check_call(_LIB.MXNDArrayGetStorageType(handle, ctypes.byref(storage_type)))\n    return storage_type.value\n\n\nclass NDArray(NDArrayBase):\n    \"\"\"An array object representing a multidimensional, homogeneous array of\nfixed-size items.\n\n    \"\"\"\n    __slots__ = []\n    # make numpy functions return NDArray instead of numpy object array\n    __array_priority__ = 1000.0\n    # Extension type code for TVM function.\n    # See C++ side of definition(kTVMNDArrayTypeCode) at include/mxmet/tensor_blob.h\n    _tvm_tcode = 19\n    # pylint: disable= no-member, undefined-variable\n\n    def as_np_ndarray(self):\n        \"\"\"Convert mxnet.ndarray.NDArray to mxnet.numpy.ndarray.\"\"\"\n        storage_type = self.stype\n        if storage_type != 'default':\n            raise ValueError('cannot convert ndarray of stype {} to numpy ndarray'\n                             .format(str(type(storage_type))))\n        from ..numpy import ndarray\n        hdl = NDArrayHandle()\n        check_call(_LIB.MXShallowCopyNDArray(self.handle, ctypes.byref(hdl)))\n        return ndarray(handle=hdl, writable=self.writable)\n\n    def as_nd_ndarray(self):\n        \"\"\"A convenience function for creating a classic ndarray from the current\n        ndarray with zero copy. For this class, it just returns itself since it is\n        already a classic ndarray.\"\"\"\n        return self\n\n    @property\n    def _tvm_handle(self):\n        return self.handle.value\n\n    def __repr__(self):\n        \"\"\"Returns a string representation of the array.\"\"\"\n        if self._alive:\n            shape_info = 'x'.join([f'{x}' for x in self.shape])\n            return f'\\n{str(self.asnumpy())}\\n<{self.__class__.__name__} {shape_info} @{self.ctx}>'\n        else:\n            return '<FREED {}>'.format(self.__class__.__name__)\n\n    def __reduce__(self):\n        return NDArray, (None,), self.__getstate__()\n\n    def _to_shared_mem(self):\n        shared_pid = ctypes.c_int()\n        shared_id = ctypes.c_int()\n        check_call(_LIB.MXNDArrayGetSharedMemHandle(\n            self.handle, ctypes.byref(shared_pid), ctypes.byref(shared_id)))\n        return shared_pid.value, shared_id.value, self.shape, self.dtype\n\n    def __abs__(self):\n        \"\"\"x.__abs__() <=> abs(x) <=> x.abs() <=> mx.nd.abs(x, y)\"\"\"\n        return self.abs()\n\n    def __add__(self, other):\n        \"\"\"x.__add__(y) <=> x+y <=> mx.nd.add(x, y) \"\"\"\n        return add(self, other)\n\n    def __iadd__(self, other):\n        \"\"\"x.__iadd__(y) <=> x+=y \"\"\"\n        if not self.writable:\n            raise ValueError('trying to add to a readonly NDArray')\n        if isinstance(other, NDArray):\n            return op.broadcast_add(self, other, out=self)\n        elif isinstance(other, numeric_types):\n            return _internal._plus_scalar(self, float(other), out=self)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __radd__(self, other):\n        return self.__add__(other)\n\n    def __sub__(self, other):\n        \"\"\"x.__sub__(y) <=> x-y <=> mx.nd.subtract(x, y) \"\"\"\n        return subtract(self, other)\n\n    def __isub__(self, other):\n        \"\"\"x.__isub__(y) <=> x-=y \"\"\"\n        if not self.writable:\n            raise ValueError('trying to subtract from a readonly NDArray')\n        if isinstance(other, NDArray):\n            return op.broadcast_sub(self, other, out=self)\n        elif isinstance(other, numeric_types):\n            return _internal._minus_scalar(self, float(other), out=self)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __rsub__(self, other):\n        \"\"\"x.__rsub__(y) <=> y-x <=> mx.nd.subtract(y, x) \"\"\"\n        return subtract(other, self)\n\n    def __mul__(self, other):\n        \"\"\"x.__mul__(y) <=> x*y <=> mx.nd.multiply(x, y) \"\"\"\n        return multiply(self, other)\n\n    def __neg__(self):\n        \"\"\"x.__neg__(y) <=> -x \"\"\"\n        return _internal._mul_scalar(self, -1.0)\n\n    def __imul__(self, other):\n        \"\"\"x.__imul__(y) <=> x*=y \"\"\"\n        if not self.writable:\n            raise ValueError('trying to multiply to a readonly NDArray')\n        if isinstance(other, NDArray):\n            return op.broadcast_mul(self, other, out=self)\n        elif isinstance(other, numeric_types):\n            return _internal._mul_scalar(self, float(other), out=self)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __rmul__(self, other):\n        return self.__mul__(other)\n\n    def __div__(self, other):\n        \"\"\"x.__div__(y) <=> x/y <=> mx.nd.divide(x, y) \"\"\"\n        return divide(self, other)\n\n    def __rdiv__(self, other):\n        \"\"\"x.__rdiv__(y) <=> y/x <=> mx.nd.divide(y, x) \"\"\"\n        return divide(other, self)\n\n    def __idiv__(self, other):\n        \"\"\"x.__rdiv__(y) <=> x/=y \"\"\"\n        if not self.writable:\n            raise ValueError('trying to divide from a readonly NDArray')\n        if isinstance(other, NDArray):\n            return op.broadcast_div(self, other, out=self)\n        elif isinstance(other, numeric_types):\n            return _internal._div_scalar(self, float(other), out=self)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __truediv__(self, other):\n        return divide(self, other)\n\n    def __rtruediv__(self, other):\n        return divide(other, self)\n\n    def __itruediv__(self, other):\n        return self.__idiv__(other)\n\n    def __mod__(self, other):\n        \"\"\"x.__mod__(y) <=> x%y <=> mx.nd.modulo(x, y) \"\"\"\n        return modulo(self, other)\n\n    def __rmod__(self, other):\n        \"\"\"x.__rmod__(y) <=> y%x <=> mx.nd.modulo(y, x) \"\"\"\n        return modulo(other, self)\n\n    def __imod__(self, other):\n        \"\"\"x.__rmod__(y) <=> x%=y \"\"\"\n        if not self.writable:\n            raise ValueError('trying to take modulo from a readonly NDArray')\n        if isinstance(other, NDArray):\n            return op.broadcast_mod(self, other, out=self)\n        elif isinstance(other, numeric_types):\n            return _internal._mod_scalar(self, float(other), out=self)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __pow__(self, other):\n        \"\"\"x.__pow__(y) <=> x**y <=> mx.nd.power(x,y) \"\"\"\n        return power(self, other)\n\n    def __rpow__(self, other):\n        \"\"\"x.__pow__(y) <=> y**x <=> mx.nd.power(y,x) \"\"\"\n        return power(other, self)\n\n    def __eq__(self, other):\n        \"\"\"x.__eq__(y) <=> x==y <=> mx.nd.equal(x, y) \"\"\"\n        return equal(self, other)\n\n    def __hash__(self):\n        \"\"\"Default hash function.\"\"\"\n        return id(self)//16\n\n    def __ne__(self, other):\n        \"\"\"x.__ne__(y) <=> x!=y <=> mx.nd.not_equal(x, y) \"\"\"\n        return not_equal(self, other)\n\n    def __gt__(self, other):\n        \"\"\"x.__gt__(y) <=> x>y <=> mx.nd.greater(x, y) \"\"\"\n        return greater(self, other)\n\n    def __ge__(self, other):\n        \"\"\"x.__ge__(y) <=> x>=y <=> mx.nd.greater_equal(x, y) \"\"\"\n        return greater_equal(self, other)\n\n    def __lt__(self, other):\n        \"\"\"x.__lt__(y) <=> x<y <=> mx.nd.lesser(x, y) \"\"\"\n        return lesser(self, other)\n\n    def __le__(self, other):\n        \"\"\"x.__le__(y) <=> x<=y <=> mx.nd.less_equal(x, y) \"\"\"\n        return lesser_equal(self, other)\n\n    def __bool__(self):\n        num_elements = reduce(operator.mul, self.shape, 1)\n        if num_elements == 0:\n            return False\n        elif num_elements == 1:\n            return bool(self.asscalar())\n        else:\n            raise ValueError(\"The truth value of an NDArray with multiple elements \" \\\n                             \"is ambiguous.\")\n\n    __nonzero__ = __bool__\n\n    def __str__(self):\n        \"\"\"Returns a readable string representation of the array.\"\"\"\n        if self.dtype == bfloat16:\n            return super(NDArray, self.astype(float)).__str__()\n        else:\n            return super(NDArray, self).__str__()\n\n    def __len__(self):\n        \"\"\"Number of element along the first axis.\"\"\"\n        return self.shape[0]\n\n    def __getstate__(self):\n        handle = self.handle\n        this = {'handle' : None}\n        if handle is not None:\n            length = ctypes.c_size_t()\n            cptr = ctypes.POINTER(ctypes.c_char)()\n            check_call(_LIB.MXNDArraySaveRawBytes(self.handle,\n                                                  ctypes.byref(length),\n                                                  ctypes.byref(cptr)))\n            this['handle'] = ctypes2buffer(cptr, length.value)\n        return this\n\n    def __setstate__(self, state):\n        # pylint: disable=assigning-non-slot\n        handle = state['handle']\n        if handle is not None:\n            buf = handle\n            handle = NDArrayHandle()\n            ptr = (ctypes.c_char * len(buf)).from_buffer(buf)\n            length = ctypes.c_size_t(len(buf))\n            check_call(_LIB.MXNDArrayLoadFromRawBytes(ptr, length, ctypes.byref(handle)))\n            self.handle = handle\n        else:\n            self.handle = None\n\n    def __setitem__(self, key, value):\n        \"\"\"x.__setitem__(i, y) <=> x[i]=y\n\n        Sets ``self[key]`` to ``value``.\n\n        This functions supports advanced indexing as defined in `the NumPy\n        advanced indexing documentation\n        <https://docs.scipy.org/doc/numpy/reference/arrays.indexing.html#advanced-indexing>`_,\n        with the restriction that boolean array indexing is not supported.\n\n        Parameters\n        ----------\n        key : int, mxnet.ndarray.slice, list, np.ndarray, NDArray, or tuple of all previous types\n            The indexing key.\n        value : scalar or array-like object that can be broadcast to the shape of self[key]\n            The value to set.\n\n        Examples\n        --------\n        >>> x = mx.nd.zeros((2, 3))\n        >>> x[:] = 1\n        >>> x.asnumpy()\n        array([[ 1.,  1.,  1.],\n               [ 1.,  1.,  1.]], dtype=float32)\n        >>> x[:, 1:2] = 2\n        >>> x.asnumpy()\n        array([[ 1.,  2.,  1.],\n               [ 1.,  2.,  1.]], dtype=float32)\n        >>> x[1:2, 1:] = 3\n        >>> x.asnumpy()\n        array([[ 1.,  2.,  1.],\n               [ 1.,  3.,  3.]], dtype=float32)\n        >>> x[1:, 0:2] = mx.nd.zeros((1, 2))\n        >>> x.asnumpy()\n        array([[ 1.,  2.,  1.],\n               [ 0.,  0.,  3.]], dtype=float32)\n        >>> x[1, 2] = 4\n        >>> x.asnumpy()\n        array([[ 1.,  2.,  1.],\n               [ 0.,  0.,  4.]], dtype=float32)\n        >>> x[[0], [1, 2]] = 5\n        >>> x.asnumpy()\n        array([[ 1.,  5.,  5.],\n               [ 0.,  0.,  4.]], dtype=float32)\n        >>> x[::-1, 0:2:2] = [6]\n        >>> x.asnumpy()\n        array([[ 6.,  5.,  5.],\n               [ 6.,  0.,  4.]], dtype=float32)\n        \"\"\"\n        if self.ndim == 0:\n            if not isinstance(key, (tuple, py_slice)):\n                raise IndexError('scalar tensor can only accept `()` and `:` as index')\n            if isinstance(key, tuple) and len(key) != 0:\n                raise IndexError('scalar tensor can only accept `()` and `:` as index')\n            if isinstance(value, numeric_types):\n                self._full(value)\n            elif isinstance(value, NDArray) and value.size == 1:\n                if value.shape != self.shape:\n                    value = value.reshape(self.shape)\n                value.copyto(self)\n            elif isinstance(value, (np.ndarray, np.generic)) and value.size == 1:\n                if isinstance(value, np.generic) or value.shape != self.shape:\n                    value = value.reshape(self.shape)\n                self._sync_copyfrom(value)\n            else:\n                raise ValueError('setting an array element with a sequence.')\n\n        elif self.size == 0:\n            return\n\n        else:\n            key, _ = indexing_key_expand_implicit_axes(key, self.shape)\n            slc_key = tuple(idx for idx in key if idx is not None)\n\n            if len(slc_key) < self.ndim:\n                raise RuntimeError(\n                    'too few indices after normalization: expected `ndim` ({}) '\n                    'but got {}. This is a bug, please report it!'\n                    ''.format(self.ndim, len(slc_key))\n                )\n            if len(slc_key) > self.ndim:\n                raise IndexError(\n                    'too many indices ({}) for array with {} dimensions'\n                    ''.format(len(slc_key), self.ndim)\n                )\n\n            indexing_dispatch_code = get_indexing_dispatch_code(slc_key)\n            if indexing_dispatch_code == _NDARRAY_BASIC_INDEXING:\n                self._set_nd_basic_indexing(key, value)\n            elif indexing_dispatch_code == _NDARRAY_ADVANCED_INDEXING:\n                self._set_nd_advanced_indexing(key, value)\n            else:\n                raise ValueError(\n                    'Indexing NDArray with index {} of type {} is not supported'\n                    ''.format(key, type(key))\n                )\n\n    def __getitem__(self, key):  # pylint: disable=too-many-return-statements\n        \"\"\"x.__getitem__(i) <=> x[i]\n\n        Returns a sliced view of this array if the elements fetched are contiguous in memory;\n        otherwise, returns a newly created NDArray.\n        This functions supports advanced indexing defined in the following reference with\n        some restrictions.\n\n        For basic indexing, i.e., if ``key`` consists only of integers,\n        ``slice``, ``Ellipsis`` (``...``) and ``None``, a mutable view is\n        returned that shares memory with this array if the accessed portion is\n        contiguous in memory.\n        Otherwise, a newly created ``NDArray`` is returned.\n\n        This functions supports advanced indexing as defined in `the NumPy\n        advanced indexing documentation\n        <https://docs.scipy.org/doc/numpy/reference/arrays.indexing.html#advanced-indexing>`_,\n        with the restriction that boolean array indexing is not supported.\n\n        Parameters\n        ----------\n        key : int, mxnet.ndarray.slice, list, np.ndarray, NDArray, or tuple of all previous types\n            Indexing key.\n\n        Examples\n        --------\n        The default is to give explicit indices for all axes:\n\n        >>> x = mx.nd.arange(0, 6).reshape((2, 3))\n        >>> x.asnumpy()\n        array([[ 0.,  1.,  2.],\n               [ 3.,  4.,  5.]], dtype=float32)\n        >>> x[0, :].asnumpy()\n        array([0., 1., 2.], dtype=float32)\n        >>> x[0, :2].asnumpy()\n        array([0., 1.], dtype=float32)\n        >>> x[:, :-1].asnumpy()\n        array([[0., 1.],\n               [3., 4.]], dtype=float32)\n\n        If fewer indices are given, they are automatically supplemented by an\n        appropriate number of ``slice(None)`` (\"``:``\") to the right. For\n        instance, a single integer indexes along the first axis:\n\n        >>> x = mx.nd.arange(0, 6).reshape((2, 3))\n        >>> x[0].asnumpy()\n        array([0., 1., 2.], dtype=float32)\n        >>> x[1:].asnumpy()\n        array([[3., 4., 5.]], dtype=float32)\n\n        To omit a range of axes that should be kept as-is, an `Ellipsis`\n        (\"``...``\") can be used:\n\n        >>> x = mx.nd.arange(0, 16).reshape((2, 2, 2, 2))\n        >>> x[0, ..., 1].asnumpy()\n        array([[1., 3.],\n               [5., 7.]], dtype=float32)\n        >>> x[0, :, :, 1].asnumpy()  # equivalent\n        array([[1., 3.],\n               [5., 7.]], dtype=float32)\n\n        New axes of length 1 can be created by inserting ``None``\n        (`numpy.newaxis`) in the index:\n\n        >>> x = mx.nd.arange(0, 6).reshape((2, 3))\n        >>> x[None, :, :].asnumpy()\n        array([[[0., 1., 2.],\n                [3., 4., 5.]]], dtype=float32)\n        >>> x[None, :, :].shape\n        (1, 2, 3)\n\n        If the indexed portion of the array is contiguous in memory, no data\n        is copied. Instead, a shared-memory view of the original array is\n        returned, and changes to that view affect the original array:\n\n        >>> x = mx.nd.arange(0, 8).reshape((2, 2, 2))\n        >>> y = x[0]  # contiguous\n        >>> y.asnumpy()\n        array([[0., 1.],\n               [2., 3.]], dtype=float32)\n        >>> y[:] = -1\n        >>> x.asnumpy()\n        array([[[-1., -1.],\n                [-1., -1.]],\n        <BLANKLINE>\n               [[ 4.,  5.],\n                [ 6.,  7.]]], dtype=float32)\n        >>> x = mx.nd.arange(0, 8).reshape((2, 2, 2))\n        >>> y = x[1, :1, :]  # contiguous\n        >>> y.asnumpy()\n        array([[4., 5.]], dtype=float32)\n        >>> y[:] = -1\n        >>> x.asnumpy()\n        array([[[ 0.,  1.],\n                [ 2.,  3.]],\n        <BLANKLINE>\n               [[-1., -1.],\n                [ 6.,  7.]]], dtype=float32)\n        >>> x = mx.nd.arange(0, 8).reshape((2, 2, 2))\n        >>> y = x[:, :, 1]  # not contiguous\n        >>> y.asnumpy()\n        array([[1., 3.],\n               [5., 7.]], dtype=float32)\n        >>> y[:] = -1\n        >>> x.asnumpy()\n        array([[[0., 1.],\n                [2., 3.]],\n        <BLANKLINE>\n               [[4., 5.],\n                [6., 7.]]], dtype=float32)\n\n        If the indexing key contains `list`, `numpy.ndarray` or `NDArray`\n        objects, advanced indexing is triggered, which always returns a\n        copy:\n\n        >>> x = mx.nd.arange(0, 8).reshape((2, 2, 2))\n        >>> x[[0, 1]].asnumpy()\n        array([[[0., 1.],\n                [2., 3.]],\n        <BLANKLINE>\n               [[4., 5.],\n                [6., 7.]]], dtype=float32)\n        >>> x[[0, 1], :].asnumpy()  # equivalent\n        array([[[0., 1.],\n                [2., 3.]],\n        <BLANKLINE>\n               [[4., 5.],\n                [6., 7.]]], dtype=float32)\n        >>> y = np.array([0, 1], dtype='int32')\n        >>> x[1:, y].asnumpy()\n        array([[[4., 5.],\n                [6., 7.]]], dtype=float32)\n        >>> y = mx.nd.array([0, 1], dtype='int32')\n        >>> x[1:, y].asnumpy()\n        array([[[4., 5.],\n                [6., 7.]]], dtype=float32)\n        \"\"\"\n        ndim = self.ndim\n        shape = self.shape\n\n        if ndim == 0 and (key == () or key == slice(None, None, None)):\n            return self\n\n        # Handle simple cases for higher speed\n        if isinstance(key, tuple) and len(key) == 0:\n            return self\n        if isinstance(key, tuple) and len(key) == ndim\\\n                and all(isinstance(idx, integer_types) for idx in key):\n            out = self\n            for idx in key:\n                out = out[idx]\n            return out\n        if isinstance(key, integer_types):\n            if key > shape[0] - 1:\n                raise IndexError(\n                    'index {} is out of bounds for axis 0 with size {}'.format(\n                        key, shape[0]))\n            return self._at(key)\n        elif isinstance(key, py_slice):\n            if (key.step is None or key.step == 1):\n                if  key.start is not None or key.stop is not None:\n                    return self._slice(key.start, key.stop)\n                else:\n                    return self\n            elif key.step == 0:\n                raise ValueError(\"slice step cannot be zero\")\n\n        key, _ = indexing_key_expand_implicit_axes(key, self.shape)\n        if len(key) == 0:\n            raise ValueError('indexing key cannot be an empty tuple')\n\n        indexing_dispatch_code = get_indexing_dispatch_code(key)\n        if indexing_dispatch_code == _NDARRAY_BASIC_INDEXING:\n            return self._get_nd_basic_indexing(key)\n        elif indexing_dispatch_code == _NDARRAY_ADVANCED_INDEXING:\n            return self._get_nd_advanced_indexing(key)\n        else:\n            raise RuntimeError\n\n    def _prepare_value_nd(self, value, bcast_shape, squeeze_axes=None):\n        \"\"\"Return a broadcast `NDArray` with same context and dtype as ``self``.\n        For setting item, The returned `ndarray` is squeezed according to squeeze_axes since the\n        value_nd is assigned to not yet expanded space in original array.\n        `value`: numeric types or array like.\n        `bcast_shape`: a shape tuple.\n        `squeeze_axes`: a sequence of axes to squeeze in the value array.\n        \"\"\"\n        if isinstance(value, numeric_types):\n            value_nd = full(bcast_shape, value, ctx=self.ctx, dtype=self.dtype)\n        elif type(value) == self.__class__:  # pylint: disable=unidiomatic-typecheck\n            value_nd = value.as_in_context(self.ctx)\n            if value_nd.dtype != self.dtype:\n                value_nd = value_nd.astype(self.dtype)\n        else:\n            try:\n                value_nd = array(value, ctx=self.ctx, dtype=self.dtype)\n            except:\n                raise TypeError('{} does not support assignment with non-array-like '\n                                'object {} of type {}'.format(self.__class__, value, type(value)))\n\n        # For setitem, if there is None in indices, we need to squeeze the assigned value_nd\n        # since None is also ignored in slicing the  original array.\n        if squeeze_axes and value_nd.ndim > len(bcast_shape):\n            squeeze_axes = tuple([ax for ax in squeeze_axes if ax < len(value_nd.shape)])\n            value_nd = value_nd.squeeze(axis=tuple(squeeze_axes))\n\n        # handle the cases like the following\n        # a = nd.zeros((3, 3)), b = nd.ones((1, 1, 1, 1, 3)), a[0] = b\n        # b cannot broadcast directly to a[0].shape unless its leading 1-size axes are trimmed\n        if value_nd.ndim > len(bcast_shape):\n            squeeze_axes = []\n            for i in range(value_nd.ndim - len(bcast_shape)):\n                if value_nd.shape[i] == 1:\n                    squeeze_axes.append(i)\n                else:\n                    break\n            if squeeze_axes:\n                value_nd = value_nd.squeeze(squeeze_axes)\n\n        if value_nd.shape != bcast_shape:\n            if value_nd.size == 0:\n                value_nd = value_nd.reshape(bcast_shape)\n            else:\n                value_nd = value_nd.broadcast_to(bcast_shape)\n        return value_nd\n\n    # pylint: disable=invalid-name\n    @staticmethod\n    def _basic_indexing_key_to_begin_end_step(idcs, shape, keep_none=True):\n        \"\"\"Map a tuple of ``slice`` and ``None`` (ignored) to begin, end, step tuples.\"\"\"\n        idcs = [idx for idx in idcs if idx is not None]\n        idcs = [idx if isinstance(idx, py_slice) else _int_to_slice(idx)\n                for idx in idcs]\n\n        if keep_none:\n            sss_list = [(slc.start, slc.stop, slc.step) for slc, n in zip(idcs, shape)]\n        else:\n            sss_list = [slc.indices(n) for slc, n in zip(idcs, shape)]\n        return tuple(zip(*sss_list))\n    # pylint: enable=invalid-name\n\n    # pylint: disable=invalid-name\n    @staticmethod\n    def _basic_indexing_key_int_to_slice(idcs):\n        \"\"\"Return the converted indexing tuple and the integer axes.\"\"\"\n        int_axes = []\n        conv_idcs = []\n        for ax, idx in enumerate(idcs):\n            if isinstance(idx, integer_types):\n                conv_idcs.append(_int_to_slice(idx))\n                int_axes.append(ax)\n            else:\n                conv_idcs.append(idx)\n\n        return tuple(conv_idcs), tuple(int_axes)\n    # pylint: enable=invalid-name\n\n    @staticmethod\n    def _new_axes_after_basic_indexing(axes, key):\n        \"\"\"Return indices of ``axes`` after slicing with ``key``.\n\n        This function is used to calculate the positions where new axes should\n        end up after indexing, taking into account the removal of axes by\n        integer indexing.\n\n        The ``key`` sequence should be the exapanded key including slices, integer types\n        and ``None``.\n        \"\"\"\n        steps = [0] + [0 if isinstance(idx, integer_types) else 1 for idx in key]\n        cum_steps = np.cumsum(steps)\n        axes_after = tuple(cum_steps[axes])\n        return axes_after\n\n    @staticmethod\n    def _new_axes_after_advanced_indexing(key, adv_axs, bcast_adv_ndim, adv_are_adjacent):  # pylint: disable=invalid-name\n        \"\"\"\n        Return indices of ``axes`` after slicing with ``key_nd``.\n\n        This function is used to calculate the positions where new axes should\n        end up after indexing, taking into account the removal of axes by\n        integer indexing.\n\n        The ``key`` sequence should be the exapanded key including slices, array like objects,\n        integer types and ``None``.\n        ``adv_axes`` is the sequence of indices of advanced axes.\n        ``bcast_adv_ndim`` is the number of dimensions of advanced indexing subspace.\n        ``adv_are_adjacent`` is a boolean value. Value being True means all advanced indicies are adjacent.\n\n        Note: integer indices are also considered advanced indices here.\n        \"\"\"\n        new_axes = [ax for ax in range(len(key)) if key[ax] is None]\n        adv_axs_set = set(adv_axs)\n        if not adv_are_adjacent:\n            steps = [bcast_adv_ndim] + [0 if ax in adv_axs_set else 1 for ax in range(len(key))]\n        else:\n            steps = [0] + [0 if ax in adv_axs_set else 1 for ax in range(len(key))]\n        cum_steps = np.cumsum(steps)\n        axes_after = tuple(cum_steps[new_axes])\n        return axes_after\n\n    # pylint: disable=invalid-name\n    @staticmethod\n    def _basic_indexing_slice_is_contiguous(slc_key, shape):\n        \"\"\"Whether indexing with the given key results in a contiguous array.\n\n        The rule is: From right to left, if in an axis, a slice produces a\n        proper subset, the later slice must have <=1 elements.\n\n        The ``slc_key`` sequence must have the same length as ``shape`` and\n        only contain `slice` objects.\n        \"\"\"\n        assert len(slc_key) == len(shape)\n        is_subset = False\n        total_sliced_elements = np.prod([_get_slice_len(slc, n)\n                                         for slc, n in zip(slc_key, shape)])\n        if total_sliced_elements in (0, 1):\n            return True\n        for idx, n in zip(reversed(slc_key), reversed(shape)):\n            _, _, step = idx.indices(n)\n            num_elements = _get_slice_len(idx, n)\n            if num_elements == 0:\n                return True\n            elif num_elements > 1 and (step > 1 or step < 0):\n                # We do not support the case of reverse slicing of multiple elements and\n                # forward slicing of #elements > 1 and step > 1\n                return False\n            elif is_subset:\n                if num_elements > 1:\n                    return False\n            else:\n                if num_elements < n:\n                    is_subset = True\n        return True\n    # pylint: enable=invalid-name\n\n    @staticmethod\n    def _basic_indexing_sliced_shape(slc_key, shape):\n        \"\"\"Return the shape after slicing with the given key.\"\"\"\n        assert len(slc_key) == len(shape)\n        sliced_shape = []\n        for slc, n in zip(slc_key, shape):\n            num_elements = _get_slice_len(slc, n)\n            sliced_shape.append(num_elements)\n        return tuple(sliced_shape)\n\n    # pylint: disable=invalid-name\n    @staticmethod\n    def _basic_indexing_contiguous_flat_begin_end(slc_key, shape):\n        \"\"\"Return the flat indices of begin and end for contiguous slicing.\"\"\"\n        assert len(slc_key) == len(shape)\n        flat_begin, flat_end = 0, 0\n        for slc, n in zip(slc_key, shape):\n            flat_begin *= n\n            flat_end *= n\n            begin, _, _ = slc.indices(n)\n            num_elements = _get_slice_len(slc, n)\n            if num_elements == 0:\n                return 0, 0\n            else:\n                flat_begin += begin\n                flat_end += begin + num_elements - 1\n        return flat_begin, flat_end + 1\n    # pylint: enable=invalid-name\n\n    @staticmethod\n    def _drop_int_axes(indexed_shape, int_axes):\n        \"\"\"drop the axis of indexed_shape corresponding to int axes\"\"\"\n        bcast_shape = []\n        for i, size in enumerate(indexed_shape):\n            if i not in int_axes:\n                bcast_shape.append(size)\n        if not bcast_shape:\n            bcast_shape = [1]\n        return tuple(bcast_shape)\n\n    def _set_nd_basic_indexing(self, key, value):\n        \"\"\"This function indexes ``self`` with a tuple of ``slice`` objects only.\"\"\"\n        for idx in key:\n            if idx is not None and not isinstance(idx, (py_slice, integer_types)):\n                raise RuntimeError(\n                    '`key` may only contain `slice` or integer objects in the '\n                    'basic implementation, got object of type {}. '\n                    'This is a bug, please report it!'\n                    ''.format(type(idx)))\n        key_nd = tuple(idx for idx in key if idx is not None)\n        int_axes = [\n            ax for ax in range(len(key_nd)) if isinstance(key_nd[ax], integer_types)\n        ]\n\n        # Check bounds for integer axes\n        for ax in int_axes:  # pylint: disable=invalid-name\n            if not -self.shape[ax] <= key_nd[ax] < self.shape[ax]:\n                raise IndexError(\n                    'index {} is out of bounds for axis {} with size {}'\n                    ''.format(key_nd[ax], ax, self.shape[ax]))\n\n        begin, end, step = self._basic_indexing_key_to_begin_end_step(\n            key, self.shape, keep_none=False\n        )\n        indexed_shape = tuple(\n            _get_dim_size(b, e, s) for b, e, s in zip(begin, end, step)\n        )\n        can_assign_directly = (\n            (indexed_shape == self.shape) and all(s > 0 for s in step)\n        )\n        begin, end, step = self._basic_indexing_key_to_begin_end_step(\n            key, self.shape, keep_none=True\n        )\n        none_axes = [ax for ax in range(len(key)) if key[ax] is None]\n        new_axes = self._new_axes_after_basic_indexing(none_axes, key)\n\n        if can_assign_directly:\n            # Easy case, overwrite whole array.\n            if type(value) == self.__class__:  # pylint: disable=unidiomatic-typecheck\n                if value.handle is not self.handle:\n                    # Need to do this before `broadcast_to`.\n                    bcast_shape = self._drop_int_axes(indexed_shape, int_axes)\n                    value_nd = self._prepare_value_nd(value, bcast_shape=bcast_shape, squeeze_axes=new_axes)\n                    value_nd = value_nd.reshape(indexed_shape)\n                    value_nd.copyto(self)\n\n            elif isinstance(value, numeric_types):\n                if isinstance(value, bool):\n                    self._full(int(value))\n                else:\n                    self._full(value)\n\n            elif isinstance(value, (np.ndarray, np.generic)):\n                tmp_shape = _shape_for_bcast(\n                    value.shape, target_ndim=self.ndim, new_axes=int_axes\n                )\n                value = value.reshape(tmp_shape)\n                if isinstance(value, np.generic) or value.shape != self.shape:\n                    value = np.broadcast_to(value, self.shape)\n                self._sync_copyfrom(value)\n\n            else:\n                # Other array-like\n                # drop the axis of indexed_shape corresponding to int axes\n                bcast_shape = self._drop_int_axes(indexed_shape, int_axes)\n                value_nd = self._prepare_value_nd(value, bcast_shape=bcast_shape, squeeze_axes=new_axes)\n                value_nd = value_nd.reshape(indexed_shape)\n                value_nd.copyto(self)\n\n        elif isinstance(value, numeric_types):\n            self.slice_assign_scalar(float(value), begin, end, step)\n\n        else:\n            # drop the axis of indexed_shape corresponding to int axes\n            bcast_shape = self._drop_int_axes(indexed_shape, int_axes)\n            value_nd = self._prepare_value_nd(value, bcast_shape=bcast_shape, squeeze_axes=new_axes)\n            value_nd = value_nd.reshape(indexed_shape)\n            self.slice_assign(value_nd, begin, end, step)\n\n    def _get_nd_basic_indexing(self, key):\n        \"\"\"This function indexes ``self`` with a tuple of `slice` objects only.\"\"\"\n        key_nd = tuple(idx for idx in key if idx is not None)\n        if len(key_nd) < self.ndim:\n            raise RuntimeError(\n                'too few indices after normalization: expected `ndim` ({}) '\n                'but got {}. This is a bug, please report it!'\n                ''.format(self.ndim, len(key_nd))\n            )\n        if len(key_nd) > self.ndim:\n            raise IndexError(\n                'too many indices ({}) for array with {} dimensions'\n                ''.format(len(key_nd), self.ndim)\n            )\n        slc_key, int_axes = self._basic_indexing_key_int_to_slice(key_nd)\n        none_axes = [ax for ax in range(len(key)) if key[ax] is None]\n        if none_axes:\n            new_axes = self._new_axes_after_basic_indexing(none_axes, key)\n        else:\n            new_axes = []\n\n        # Check bounds for integer axes\n        for ax in int_axes:  # pylint: disable=invalid-name\n            if not -self.shape[ax] <= key_nd[ax] < self.shape[ax]:\n                raise IndexError(\n                    'index {} is out of bounds for axis {} with size {}'\n                    ''.format(key_nd[ax], ax, self.shape[ax]))\n\n        # Convert to begin, end and step, and return immediately if the slice\n        # is empty\n        begin, end, step = self._basic_indexing_key_to_begin_end_step(\n            slc_key, self.shape, keep_none=False\n        )\n\n        if self._basic_indexing_slice_is_contiguous(slc_key, self.shape):\n            # Create a shared-memory view by using low-level flat slicing\n            flat_begin, flat_end = self._basic_indexing_contiguous_flat_begin_end(\n                slc_key, self.shape\n            )\n            handle = NDArrayHandle()\n            flat_self = self.reshape(-1)\n            if _int64_enabled():\n                check_call(\n                    _LIB.MXNDArraySlice64(\n                        flat_self.handle,\n                        ctypes.c_int64(flat_begin),\n                        ctypes.c_int64(flat_end),\n                        ctypes.byref(handle),\n                    )\n                )\n            else:\n                check_call(\n                    _LIB.MXNDArraySlice(\n                        flat_self.handle,\n                        ctypes.c_uint32(flat_begin),\n                        ctypes.c_uint32(flat_end),\n                        ctypes.byref(handle),\n                    )\n                )\n            sliced_shape = self._basic_indexing_sliced_shape(slc_key, self.shape)\n            sliced = NDArray(handle=handle, writable=self.writable).reshape(sliced_shape)\n        else:\n            begin, end, step = self._basic_indexing_key_to_begin_end_step(\n                slc_key, self.shape, keep_none=True\n            )\n            sliced = op.slice(self, begin, end, step)\n\n        # Reshape to final shape due to integer and `None` entries in `key`.\n        final_shape = [sliced.shape[i] for i in range(sliced.ndim)\n                       if i not in int_axes]\n        for ax in new_axes:  # pylint: disable=invalid-name\n            final_shape.insert(ax, 1)\n\n        if len(final_shape) == 0:\n            # Override for single element indexing\n            final_shape = [1]\n        return sliced.reshape(final_shape)\n\n    @staticmethod\n    def _advanced_index_to_array(idx, ax_len, ctx):\n        \"\"\"Convert ``idx`` to `NDArray` for advanced indexing.\n\n        The ``ax_len`` is used to convert `slice` objects to integer arrays.\n        \"\"\"\n        if _int64_enabled():\n            idx_dtype = 'int64'\n        else:\n            idx_dtype = 'int32'\n        if isinstance(idx, NDArray):\n            if idx.dtype != idx_dtype:\n                idx = idx.astype(idx_dtype)\n            return idx.as_in_context(ctx)\n        elif isinstance(idx, (np.ndarray, list, tuple)):\n            return array(idx, ctx, idx_dtype)\n        elif isinstance(idx, integer_types):\n            return array([idx], ctx, idx_dtype)\n        elif isinstance(idx, py_slice):\n            start, stop, step = idx.indices(ax_len)\n            return arange(start, stop, step, ctx=ctx, dtype=idx_dtype)\n        elif isinstance(idx, range):\n            return arange(idx.start, idx.stop, idx.step, ctx=ctx, dtype=idx_dtype)\n        else:\n            raise RuntimeError('illegal index type {}'.format(type(idx)))\n\n    # pylint: disable=invalid-name\n    @staticmethod\n    def _broadcast_advanced_indices(arrays, block_axes):\n        \"\"\"Broadcast arrays according to position in the sequence.\n\n        Here, \"according to position\" means that an array of dimension 1\n        (which is the case for all except ``block_axes``) will have shape\n        ``(1, ..., 1, N, 1, ..., 1)``, where ``N`` is the length, and the\n        position of ``N`` in the shape is the same as the position of the\n        array in the ``arrays`` sequence, plus extra dimensions of the\n        advanced block if it is left of the array.\n\n        The arrays at ``block_axes`` are the advanced indices. They are assumed to\n        be ready for mutual broadcasting to produce the advanced indexing block.\n        It is further assumed that the numbers in ``block_axes`` are consecutive.\n\n        The return value is a tuple containing the arrays with broadcast shapes.\n        \"\"\"\n        block_shape = _broadcast_shapes([arrays[ax] for ax in block_axes])\n        ndim_blk = len(block_shape)\n        ndim_blk_delta = ndim_blk - len(block_axes)\n        ndim_lead = block_axes[0]\n        ndim_trail = len(arrays) - (block_axes[-1] + 1)\n\n        bcast_shape = (\n            tuple(arrays[ax].shape[0] for ax in range(ndim_lead)) +\n            block_shape +\n            tuple(arrays[ax].shape[0] for ax in range(block_axes[-1] + 1, len(arrays)))\n        )\n\n        bcast_arrays = [None] * len(arrays)\n        for ax in block_axes:\n            arr = arrays[ax].broadcast_to(block_shape)\n            shp = (1,) * ndim_lead + block_shape + (1,) * ndim_trail\n            bcast_arrays[ax] = arr.reshape(shp).broadcast_to(bcast_shape)\n\n        for ax in set(range(len(arrays))) - set(block_axes):\n            shp = [1] * len(bcast_shape)\n            if ax < ndim_lead:\n                shp[ax] = arrays[ax].shape[0]\n            else:\n                shp[ax + ndim_blk_delta] = arrays[ax].shape[0]\n            bcast_arrays[ax] = arrays[ax].reshape(shp).broadcast_to(bcast_shape)\n\n        return tuple(bcast_arrays)\n    # pylint: enable=invalid-name\n\n    @staticmethod\n    def _drop_slice_none_at_end(key):\n        \"\"\"Remove ``slice(None)`` at the end of a key.\n\n        This is used for efficiency in advanced indexing, to avoid generating\n        ``arange(n)`` arrays for these axes. The `gather_nd` and `scatter_nd`\n        handle implicit full trailing axes automatically.\n        \"\"\"\n        key = list(key)\n        while isinstance(key[-1], py_slice) and key[-1] == slice(None):\n            key.pop()\n        return tuple(key)\n\n    def _get_index_nd(self, key):\n        \"\"\"\n        Return an index array for use in `scatter_nd` and `gather_nd`,\n        and a list of positions of new_axes in ouptut shape.\n        \"\"\"\n        key_nd = tuple(idx for idx in key if idx is not None)\n        if len(key_nd) < self.ndim:\n            raise RuntimeError(\n                'too few indices after normalization: expected `ndim` ({}) '\n                'but got {}. This is a bug, please report it!'\n                ''.format(self.ndim, len(key_nd))\n            )\n        if len(key_nd) > self.ndim:\n            raise IndexError(\n                'too many indices ({}) for array with {} dimensions'\n                ''.format(len(key_nd), self.ndim)\n            )\n        ndim = len(key_nd)\n\n        # --- Preparation --- #\n\n        # - Make lists for bookkeeping of advanced indices & axes\n        # - Drop trailing `slice(None)` entries in `key` for efficiency\n        # - Determine whether the advanced indices are adjacent in `key`\n        # - Depending on that, make index permutations to move around indices\n\n        adv_axs = [ax for ax, idx in enumerate(key) if _is_advanced_index(idx)]\n        adv_axs_nd = [ax for ax, idx in enumerate(key_nd) if _is_advanced_index(idx)]\n        adv_idcs_are_adjacent = bool(np.all(np.diff(adv_axs) == 1))\n        nonadv_axs_nd = [ax for ax in range(ndim) if ax not in adv_axs_nd]\n        adv_idcs_nd = [key_nd[ax] for ax in adv_axs_nd]\n        idcs_short = self._drop_slice_none_at_end(key_nd)\n        dropped_axs = list(range(len(idcs_short), ndim))\n\n        if adv_idcs_are_adjacent:\n            # The easy case: the advanced block can stay at its position, and no\n            # permutation needs to be done (identity permutation)\n            axs_nd_permut = axs_nd_permut_inv = tuple(range(ndim))\n            idcs_permut_short = idcs_short\n            block_axs_nd = adv_axs_nd\n        else:\n            # The more complicated case: during broadcasting, we need to use the\n            # indices in the *permuted* order, where the advanced block is\n            # at the beginning, while the final index for `gather_nd` is stacked\n            # in the *original* order, so that the association of index with\n            # array axis remains the same.\n\n            # This order is used for broadcasting: advanced block at the beginning\n            idcs_permut_short = (\n                adv_idcs_nd +\n                [key_nd[ax] for ax in range(ndim)\n                 if ax not in adv_axs_nd and ax not in dropped_axs]\n            )\n            block_axs_nd = list(range(len(adv_axs_nd)))\n            axs_nd_permut = adv_axs_nd + nonadv_axs_nd\n            axs_nd_permut_inv = list(np.argsort(axs_nd_permut))\n\n        # --- Conversion, broadcasting and index stacking --- #\n\n        # - Convert all indices in `key` to arrays: integers to 1-element arrays,\n        #   `slice` objects to arrays with explicit indices\n        # - Reshape arrays for broadcasting according to their position in the\n        #   *permuted* key\n        # - Broadcast and stack the indices in the *original* order\n\n        shape_nd_permut = tuple(self.shape[ax] for ax in axs_nd_permut)\n        converted_idcs_short = [\n            self._advanced_index_to_array(idx, ax_len, self.ctx)\n            for idx, ax_len in zip(idcs_permut_short, shape_nd_permut)\n        ]\n        bcast_idcs_permut_short = self._broadcast_advanced_indices(\n            converted_idcs_short, block_axes=block_axs_nd\n        )\n\n        # Get the ndim of advanced indexing subspace\n        converted_advanced_idcs = [\n            self._advanced_index_to_array(idx, ax_len, self.ctx)\n            for idx, ax_len in zip(adv_idcs_nd, [self.shape[ax] for ax in adv_axs_nd])\n        ]\n        bcast_advanced_shape = _broadcast_shapes(converted_advanced_idcs)\n\n        # Undo the permutation to restore the original order\n        bcast_idcs_short = [\n            bcast_idcs_permut_short[ax]\n            for ax in axs_nd_permut_inv\n            if axs_nd_permut[ax] not in dropped_axs\n        ]\n\n        # Calculate where the newaxes are inserted after advanced indexing\n        new_axes_positions = self._new_axes_after_advanced_indexing(key, adv_axs,\\\n                                len(bcast_advanced_shape), adv_idcs_are_adjacent)\n\n                                # if any array is numpy.ndarray, stack in numpy ndarray class.\n        for idcs in bcast_idcs_short:\n            if type(idcs) != NDArray:  # pylint: disable=unidiomatic-typecheck\n                return bcast_idcs_short, new_axes_positions\n\n        return op.stack(*bcast_idcs_short), new_axes_positions\n\n    def _set_nd_advanced_indexing(self, key, value):\n        \"\"\"This function is called by __setitem__ when key is an advanced index.\"\"\"\n        indices, new_axes = self._get_index_nd(key)\n        vshape = get_oshape_of_gather_nd_op(self.shape, indices.shape)\n        value_nd = self._prepare_value_nd(value, bcast_shape=vshape, squeeze_axes=new_axes)\n        self._scatter_set_nd(value_nd, indices)\n\n    def _get_nd_advanced_indexing(self, key):\n        \"\"\"Get item when key is a tuple of any objects of the following types:\n        NDArray, np.ndarray, list, tuple, slice, and integer.\"\"\"\n        slc_key, new_axes = self._get_index_nd(key)\n        sliced = op.gather_nd(self, slc_key)\n\n        # Reshape due to `None` entries in `key`.\n        if new_axes:\n            final_shape = [sliced.shape[i] for i in range(sliced.ndim)]\n            for ax in new_axes:  # pylint: disable=invalid-name\n                final_shape.insert(ax, 1)\n            return sliced.reshape(final_shape)\n        else:\n            return sliced\n\n    def _sync_copyfrom(self, source_array):\n        \"\"\"Performs a synchronized copy from the `source_array` to the current array.\n        This is called through ``x[:] = source_array``, where the `source_array`\n        is a `numpy.ndarray` or array-like object.\n        This function blocks until all the pending read/write operations with respect\n        to the current `NDArray` are finished and carry out the copy operation to the\n        current NDArray.\n\n        Parameters\n        ----------\n        source_array : array_like\n            The data source we would like to copy from.\n\n        Example\n        -------\n        >>> a = mx.nd.array([1, 2])\n        >>> a.asnumpy()\n        array([ 1.,  2.], dtype=float32)\n        >>> a[:] = np.array([3, 4])\n        >> a.asnumpy()\n        array([ 3.,  4.], dtype=float32)\n        \"\"\"\n        if not isinstance(source_array, np.ndarray):\n            try:\n                source_array = np.array(source_array, dtype=self.dtype)\n            except:\n                raise TypeError('array must consist of array-like data,' +\n                                f'type {str(type(array))} is not supported')\n        source_array = np.asarray(source_array, dtype=self.dtype, order='C')\n        if source_array.shape != self.shape:\n            raise ValueError(f'Shape inconsistent: expected {str(source_array.shape)} vs got {str(self.shape)}')\n        check_call(_LIB.MXNDArraySyncCopyFromCPU(\n            self.handle,\n            source_array.ctypes.data_as(ctypes.c_void_p),\n            ctypes.c_size_t(source_array.size)))\n\n    def _slice(self, start, stop):\n        \"\"\"Returns a sliced NDArray that shares memory with the current one.\n        This is called through ``x[start:stop]``.\n\n        Parameters\n        ----------\n        start : int\n            Starting inclusive index of slice in the first dim.\n        stop : int\n            Finishing exclusive index of slice in the first dim.\n\n        Returns\n        -------\n            `NDArray` sharing the memory with the current one sliced from\n            start to stop in the first dim.\n\n        Examples:\n        >>> a = mx.nd.array([[1,2], [3, 4], [5, 6], [7, 8]])\n        >>> a[1:2].asnumpy()\n        array([[ 3.,  4.]], dtype=float32)\n        >>> a[1:1].asnumpy()\n        array([], shape=(0, 2), dtype=float32)\n        \"\"\"\n        handle = NDArrayHandle()\n        start, stop, _ = _get_index_range(start, stop, self.shape[0])\n\n        check_call(_LIB.MXNDArraySlice(\n            self.handle, mx_uint(start), mx_uint(stop), ctypes.byref(handle)))\n        return self.__class__(handle=handle, writable=self.writable)\n\n    def _at(self, idx):\n        \"\"\"Returns a view of the array sliced at `idx` in the first dim.\n        This is called through ``x[idx]``.\n\n        Parameters\n        ----------\n        idx : int\n            index for slicing the `NDArray` in the first dim.\n\n        Returns\n        -------\n        NDArray\n            `NDArray` sharing the memory with the current one sliced at `idx` in the first dim.\n\n        Examples\n        --------\n        >>> a = mx.nd.array([[1,2], [3, 4]])\n        >>> a[1].asnumpy()\n        array([ 3.,  4.], dtype=float32)\n        >>> b = mx.nd.array([1, 2, 3, 4])\n        >>> b[0].asnumpy()\n        array([ 1.], dtype=float32)\n        \"\"\"\n        handle = NDArrayHandle()\n        if idx < 0:\n            length = self.shape[0]\n            idx += length\n            if idx < 0:\n                raise IndexError(f'index {idx-length} is out of bounds for axis 0 with size {length}')\n        if _int64_enabled():\n            check_call(_LIB.MXNDArrayAt64(\n                self.handle, ctypes.c_int64(idx), ctypes.byref(handle)))\n        else:\n            check_call(_LIB.MXNDArrayAt(\n                self.handle, ctypes.c_uint32(idx), ctypes.byref(handle)))\n        return self.__class__(handle=handle, writable=self.writable)\n\n    def reshape(self, *shape, **kwargs):\n        \"\"\"Returns a **view** of this array with a new shape without altering any data.\n\n        Parameters\n        ----------\n        shape : tuple of int, or n ints\n            The new shape should not change the array size, namely\n            ``np.prod(new_shape)`` should be equal to ``np.prod(self.shape)``.\n            Some dimensions of the shape can take special values from the set {0, -1, -2, -3, -4}.\n            The significance of each is explained below:\n\n            - ``0``  copy this dimension from the input to the output shape.\n\n              Example::\n\n              - input shape = (2,3,4), shape = (4,0,2), output shape = (4,3,2)\n              - input shape = (2,3,4), shape = (2,0,0), output shape = (2,3,4)\n\n            - ``-1`` infers the dimension of the output shape by using the remainder of the\n              input dimensions keeping the size of the new array same as that of the input array.\n              At most one dimension of shape can be -1.\n\n              Example::\n\n              - input shape = (2,3,4), shape = (6,1,-1), output shape = (6,1,4)\n              - input shape = (2,3,4), shape = (3,-1,8), output shape = (3,1,8)\n              - input shape = (2,3,4), shape=(-1,), output shape = (24,)\n\n            - ``-2`` copy all/remainder of the input dimensions to the output shape.\n\n              Example::\n\n              - input shape = (2,3,4), shape = (-2,), output shape = (2,3,4)\n              - input shape = (2,3,4), shape = (2,-2), output shape = (2,3,4)\n              - input shape = (2,3,4), shape = (-2,1,1), output shape = (2,3,4,1,1)\n\n            - ``-3`` use the product of two consecutive dimensions of the input shape as the\n              output dimension.\n\n              Example::\n\n              - input shape = (2,3,4), shape = (-3,4), output shape = (6,4)\n              - input shape = (2,3,4,5), shape = (-3,-3), output shape = (6,20)\n              - input shape = (2,3,4), shape = (0,-3), output shape = (2,12)\n              - input shape = (2,3,4), shape = (-3,-2), output shape = (6,4)\n\n            - ``-4`` split one dimension of the input into two dimensions passed subsequent to\n              -4 in shape (can contain -1).\n\n              Example::\n\n              - input shape = (2,3,4), shape = (-4,1,2,-2), output shape =(1,2,3,4)\n              - input shape = (2,3,4), shape = (2,-4,-1,3,-2), output shape = (2,1,3,4)\n\n            - If the argument `reverse` is set to 1, then the special values are inferred from right\n              to left.\n\n              Example::\n\n              - without reverse=1, for input shape = (10,5,4), shape = (-1,0), output shape would be \\\n                (40,5).\n              - with reverse=1, output shape will be (50,4).\n\n        reverse : bool, default False\n            If true then the special values are inferred from right to left. Only supported as\n            keyword argument.\n\n\n        Returns\n        -------\n        NDArray\n            An array with desired shape that shares data with this array.\n\n        Examples\n        --------\n        >>> x = mx.nd.arange(0,6).reshape(2,3)\n        >>> x.asnumpy()\n        array([[ 0.,  1.,  2.],\n               [ 3.,  4.,  5.]], dtype=float32)\n        >>> y = x.reshape(3,2)\n        >>> y.asnumpy()\n        array([[ 0.,  1.],\n               [ 2.,  3.],\n               [ 4.,  5.]], dtype=float32)\n        >>> y = x.reshape(3,-1)\n        >>> y.asnumpy()\n        array([[ 0.,  1.],\n               [ 2.,  3.],\n               [ 4.,  5.]], dtype=float32)\n        >>> y = x.reshape(3,2)\n        >>> y.asnumpy()\n        array([[ 0.,  1.],\n               [ 2.,  3.],\n               [ 4.,  5.]], dtype=float32)\n        >>> y = x.reshape(-3)\n        >>> y.asnumpy()\n        array([ 0.  1.  2.  3.  4.  5.], dtype=float32)\n        >>> y[:] = -1\n        >>> x.asnumpy()\n        array([[-1., -1., -1.],\n               [-1., -1., -1.]], dtype=float32)\n        \"\"\"\n        if len(shape) == 1 and isinstance(shape[0], (list, tuple)):\n            shape = shape[0]\n        elif not shape:\n            shape = kwargs.get('shape')\n            assert shape, \"Shape must be provided.\"\n        if not all(k in ['shape', 'reverse'] for k in kwargs):\n            raise TypeError(\n                \"Got unknown keywords in reshape: {}. \" \\\n                \"Accepted keyword arguments are 'shape' and 'reverse'.\".format(\n                    ', '.join([k for k in kwargs if k not in ['shape', 'reverse']])))\n        reverse = kwargs.get('reverse', False)\n        handle = NDArrayHandle()\n\n        # Actual reshape\n        check_call(_LIB.MXNDArrayReshape64(self.handle,\n                                           len(shape),\n                                           c_array(ctypes.c_int64, shape),\n                                           reverse,\n                                           ctypes.byref(handle)))\n        res = self.__class__(handle=handle, writable=self.writable)\n\n        # Array size should not change\n        if np.prod(res.shape) != np.prod(self.shape):\n            raise ValueError('Cannot reshape array of size {} into shape {}'.format(np.prod(self.shape), shape))\n        return res\n\n    def reshape_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`reshape_like`.\n\n        The arguments are the same as for :py:func:`reshape_like`, with\n        this array as data.\n        \"\"\"\n        return op.reshape_like(self, *args, **kwargs)\n\n    def zeros_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`zeros_like`.\n\n        The arguments are the same as for :py:func:`zeros_like`, with\n        this array as data.\n        \"\"\"\n        return op.zeros_like(self, *args, **kwargs)\n\n    def ones_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`ones_like`.\n\n        The arguments are the same as for :py:func:`ones_like`, with\n        this array as data.\n        \"\"\"\n        return op.ones_like(self, *args, **kwargs)\n\n    def broadcast_axes(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`broadcast_axes`.\n\n        The arguments are the same as for :py:func:`broadcast_axes`, with\n        this array as data.\n        \"\"\"\n        return op.broadcast_axes(self, *args, **kwargs)\n\n    def repeat(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`repeat`.\n\n        The arguments are the same as for :py:func:`repeat`, with\n        this array as data.\n        \"\"\"\n        return op.repeat(self, *args, **kwargs)\n\n    def pad(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`pad`.\n\n        The arguments are the same as for :py:func:`pad`, with\n        this array as data.\n        \"\"\"\n        return op.pad(self, *args, **kwargs)\n\n    def swapaxes(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`swapaxes`.\n\n        The arguments are the same as for :py:func:`swapaxes`, with\n        this array as data.\n        \"\"\"\n        return op.swapaxes(self, *args, **kwargs)\n\n    def split(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`split`.\n\n        The arguments are the same as for :py:func:`split`, with\n        this array as data.\n        \"\"\"\n        return op.split(self, *args, **kwargs)\n\n    def split_v2(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`split_v2`.\n\n        The arguments are the same as for :py:func:`split_v2`, with\n        this array as data.\n        \"\"\"\n        return split_v2(self, *args, **kwargs)\n\n    def slice(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`slice`.\n\n        The arguments are the same as for :py:func:`slice`, with\n        this array as data.\n        \"\"\"\n        return op.slice(self, *args, **kwargs)\n\n    def slice_axis(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`slice_axis`.\n\n        The arguments are the same as for :py:func:`slice_axis`, with\n        this array as data.\n        \"\"\"\n        return op.slice_axis(self, *args, **kwargs)\n\n    def slice_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`slice_like`.\n\n        The arguments are the same as for :py:func:`slice_like`, with\n        this array as data.\n        \"\"\"\n        return op.slice_like(self, *args, **kwargs)\n\n    def take(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`take`.\n\n        The arguments are the same as for :py:func:`take`, with\n        this array as data.\n        \"\"\"\n        return op.take(self, *args, **kwargs)\n\n    def one_hot(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`one_hot`.\n\n        The arguments are the same as for :py:func:`one_hot`, with\n        this array as data.\n        \"\"\"\n        return op.one_hot(self, *args, **kwargs)\n\n    def pick(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`pick`.\n\n        The arguments are the same as for :py:func:`pick`, with\n        this array as data.\n        \"\"\"\n        return op.pick(self, *args, **kwargs)\n\n    def sort(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sort`.\n\n        The arguments are the same as for :py:func:`sort`, with\n        this array as data.\n        \"\"\"\n        return op.sort(self, *args, **kwargs)\n\n    def topk(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`topk`.\n\n        The arguments are the same as for :py:func:`topk`, with\n        this array as data.\n        \"\"\"\n        return op.topk(self, *args, **kwargs)\n\n    def argsort(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`argsort`.\n\n        The arguments are the same as for :py:func:`argsort`, with\n        this array as data.\n        \"\"\"\n        return op.argsort(self, *args, **kwargs)\n\n    def argmax(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`argmax`.\n\n        The arguments are the same as for :py:func:`argmax`, with\n        this array as data.\n        \"\"\"\n        return op.argmax(self, *args, **kwargs)\n\n    def argmax_channel(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`argmax_channel`.\n\n        The arguments are the same as for :py:func:`argmax_channel`, with\n        this array as data.\n        \"\"\"\n        return op.argmax_channel(self, *args, **kwargs)\n\n    def argmin(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`argmin`.\n\n        The arguments are the same as for :py:func:`argmin`, with\n        this array as data.\n        \"\"\"\n        return op.argmin(self, *args, **kwargs)\n\n    def clip(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`clip`.\n\n        The arguments are the same as for :py:func:`clip`, with\n        this array as data.\n        \"\"\"\n        return op.clip(self, *args, **kwargs)\n\n    def abs(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`abs`.\n\n        The arguments are the same as for :py:func:`abs`, with\n        this array as data.\n        \"\"\"\n        return op.abs(self, *args, **kwargs)\n\n    def sign(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sign`.\n\n        The arguments are the same as for :py:func:`sign`, with\n        this array as data.\n        \"\"\"\n        return op.sign(self, *args, **kwargs)\n\n    def flatten(self, inplace=False):\n        \"\"\"Flatten this array without altering any data.\n\n        Parameters\n        ----------\n        inplace : bool, default False\n            If True, this method returns a **view** of this array\n            that shares data with this array. Otherwise, a copy is returned.\n\n        Returns\n        -------\n        NDArray\n            An array with flattened shape `(d1, d2*...*dk)` that shares data with\n            this array with shape `(d1, d2, ..., dk)`.\n\n        Examples\n        --------\n        >>> x = mx.nd.arange(30).reshape(5,2,3)\n        >>> y = x.flatten(inplace=True)\n        >>> z = x.flatten()\n        >>> y.shape\n        (5, 6)\n        >>> y[0].asnumpy()\n        array([0., 1., 2., 3., 4., 5.], dtype=float32)\n        >>> y[:] = -1\n        >>> x[0].asnumpy()\n        array([[-1., -1., -1.],\n               [-1., -1., -1.]], dtype=float32)\n        >>> z[0].asnumpy()\n        array([0., 1., 2., 3., 4., 5.], dtype=float32)\n        \"\"\"\n        return op.flatten(self) if not inplace else self.reshape((0, -1))\n\n    def shape_array(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`shape_array`.\n\n        The arguments are the same as for :py:func:`shape_array`, with\n        this array as data.\n        \"\"\"\n        return op.shape_array(self, *args, **kwargs)\n\n    def size_array(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`size_array`.\n\n        The arguments are the same as for :py:func:`size_array`, with\n        this array as data.\n        \"\"\"\n        return op.size_array(self, *args, **kwargs)\n\n    def expand_dims(self, axis, inplace=False):\n        \"\"\"Adds an additional dimension to the current array without altering any data.\n\n        Parameters\n        ----------\n        axis : int\n            Position where new axis is to be inserted.\n            Suppose that the input NDArray's dimension is ndim,\n            the range of the inserted axis is [-ndim, ndim].\n        inplace : bool, default False\n            If True, this method returns a **view** of this array\n            that shares data with this array. Otherwise, a copy is returned.\n\n        Returns\n        -------\n        NDArray\n            An array with expanded shape `(d1, d2, ..., 1, di, ..., dk)`\n            that shares data with this array with shape `(d1, d2, ..., dk)`,\n            given input axis `i`.\n\n        Examples\n        --------\n        >>> x = mx.nd.arange(6).reshape(2,3)\n        >>> y = x.expand_dims(1, inplace=True)\n        >>> z = x.expand_dims(1)\n        >>> y.shape\n        (2, 1, 3)\n        >>> y[0].asnumpy()\n        array([[0., 1., 2.]], dtype=float32)\n        >>> y[:] = -1\n        >>> x.asnumpy()\n        array([[-1., -1., -1.],\n               [-1., -1., -1.]], dtype=float32)\n        >>> z[0].asnumpy()\n        array([[0., 1., 2.]], dtype=float32)\n        \"\"\"\n        if not inplace:\n            return op.expand_dims(self, axis=axis)\n        else:\n            new_shape = list(self.shape)\n            assert -len(new_shape)-1 <= axis <= len(new_shape), \\\n                    \"axis {} is out of range for {}d array\".format(axis, len(new_shape))\n            if axis < 0:\n                axis += len(new_shape) + 1\n            new_shape.insert(axis, 1)\n            return self.reshape(new_shape)\n\n    def tile(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`tile`.\n\n        The arguments are the same as for :py:func:`tile`, with\n        this array as data.\n        \"\"\"\n        return op.tile(self, *args, **kwargs)\n\n    def transpose(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`transpose`.\n\n        The arguments are the same as for :py:func:`transpose`, with\n        this array as data.\n        \"\"\"\n        return op.transpose(self, *args, **kwargs)\n\n    def flip(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`flip`.\n\n        The arguments are the same as for :py:func:`flip`, with\n        this array as data.\n        \"\"\"\n        return op.flip(self, *args, **kwargs)\n\n    def depth_to_space(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`depth_to_space`.\n\n        The arguments are the same as for :py:func:`depth_to_space`, with\n        this array as data.\n        \"\"\"\n        return op.depth_to_space(self, *args, **kwargs)\n\n    def space_to_depth(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`space_to_depth`.\n\n        The arguments are the same as for :py:func:`space_to_depth`, with\n        this array as data.\n        \"\"\"\n        return op.space_to_depth(self, *args, **kwargs)\n\n    def diag(self, k=0, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`diag`.\n\n        The arguments are the same as for :py:func:`diag`, with\n        this array as data.\n        \"\"\"\n        return op.diag(self, k, **kwargs)\n\n    def sum(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sum`.\n\n        The arguments are the same as for :py:func:`sum`, with\n        this array as data.\n        \"\"\"\n        return op.sum(self, *args, **kwargs)\n\n    def nansum(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`nansum`.\n\n        The arguments are the same as for :py:func:`nansum`, with\n        this array as data.\n        \"\"\"\n        return op.nansum(self, *args, **kwargs)\n\n    def prod(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`prod`.\n\n        The arguments are the same as for :py:func:`prod`, with\n        this array as data.\n        \"\"\"\n        return op.prod(self, *args, **kwargs)\n\n    def nanprod(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`nanprod`.\n\n        The arguments are the same as for :py:func:`nanprod`, with\n        this array as data.\n        \"\"\"\n        return op.nanprod(self, *args, **kwargs)\n\n    def mean(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`mean`.\n\n        The arguments are the same as for :py:func:`mean`, with\n        this array as data.\n        \"\"\"\n        return op.mean(self, *args, **kwargs)\n\n    def max(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`max`.\n\n        The arguments are the same as for :py:func:`max`, with\n        this array as data.\n        \"\"\"\n        return op.max(self, *args, **kwargs)\n\n    def min(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`min`.\n\n        The arguments are the same as for :py:func:`min`, with\n        this array as data.\n        \"\"\"\n        return op.min(self, *args, **kwargs)\n\n    def norm(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`norm`.\n\n        The arguments are the same as for :py:func:`norm`, with\n        this array as data.\n        \"\"\"\n        return op.norm(self, *args, **kwargs)\n\n    def round(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`round`.\n\n        The arguments are the same as for :py:func:`round`, with\n        this array as data.\n        \"\"\"\n        return op.round(self, *args, **kwargs)\n\n    def rint(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`rint`.\n\n        The arguments are the same as for :py:func:`rint`, with\n        this array as data.\n        \"\"\"\n        return op.rint(self, *args, **kwargs)\n\n    def fix(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`fix`.\n\n        The arguments are the same as for :py:func:`fix`, with\n        this array as data.\n        \"\"\"\n        return op.fix(self, *args, **kwargs)\n\n    def floor(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`floor`.\n\n        The arguments are the same as for :py:func:`floor`, with\n        this array as data.\n        \"\"\"\n        return op.floor(self, *args, **kwargs)\n\n    def ceil(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`ceil`.\n\n        The arguments are the same as for :py:func:`ceil`, with\n        this array as data.\n        \"\"\"\n        return op.ceil(self, *args, **kwargs)\n\n    def trunc(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`trunc`.\n\n        The arguments are the same as for :py:func:`trunc`, with\n        this array as data.\n        \"\"\"\n        return op.trunc(self, *args, **kwargs)\n\n    def sin(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sin`.\n\n        The arguments are the same as for :py:func:`sin`, with\n        this array as data.\n        \"\"\"\n        return op.sin(self, *args, **kwargs)\n\n    def cos(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`cos`.\n\n        The arguments are the same as for :py:func:`cos`, with\n        this array as data.\n        \"\"\"\n        return op.cos(self, *args, **kwargs)\n\n    def tan(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`tan`.\n\n        The arguments are the same as for :py:func:`tan`, with\n        this array as data.\n        \"\"\"\n        return op.tan(self, *args, **kwargs)\n\n    def arcsin(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arcsin`.\n\n        The arguments are the same as for :py:func:`arcsin`, with\n        this array as data.\n        \"\"\"\n        return op.arcsin(self, *args, **kwargs)\n\n    def arccos(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arccos`.\n\n        The arguments are the same as for :py:func:`arccos`, with\n        this array as data.\n        \"\"\"\n        return op.arccos(self, *args, **kwargs)\n\n    def arctan(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arctan`.\n\n        The arguments are the same as for :py:func:`arctan`, with\n        this array as data.\n        \"\"\"\n        return op.arctan(self, *args, **kwargs)\n\n    def degrees(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`degrees`.\n\n        The arguments are the same as for :py:func:`degrees`, with\n        this array as data.\n        \"\"\"\n        return op.degrees(self, *args, **kwargs)\n\n    def radians(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`radians`.\n\n        The arguments are the same as for :py:func:`radians`, with\n        this array as data.\n        \"\"\"\n        return op.radians(self, *args, **kwargs)\n\n    def sinh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sinh`.\n\n        The arguments are the same as for :py:func:`sinh`, with\n        this array as data.\n        \"\"\"\n        return op.sinh(self, *args, **kwargs)\n\n    def cosh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`cosh`.\n\n        The arguments are the same as for :py:func:`cosh`, with\n        this array as data.\n        \"\"\"\n        return op.cosh(self, *args, **kwargs)\n\n    def tanh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`tanh`.\n\n        The arguments are the same as for :py:func:`tanh`, with\n        this array as data.\n        \"\"\"\n        return op.tanh(self, *args, **kwargs)\n\n    def arcsinh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arcsinh`.\n\n        The arguments are the same as for :py:func:`arcsinh`, with\n        this array as data.\n        \"\"\"\n        return op.arcsinh(self, *args, **kwargs)\n\n    def arccosh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arccosh`.\n\n        The arguments are the same as for :py:func:`arccosh`, with\n        this array as data.\n        \"\"\"\n        return op.arccosh(self, *args, **kwargs)\n\n    def arctanh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arctanh`.\n\n        The arguments are the same as for :py:func:`arctanh`, with\n        this array as data.\n        \"\"\"\n        return op.arctanh(self, *args, **kwargs)\n\n    def exp(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`exp`.\n\n        The arguments are the same as for :py:func:`exp`, with\n        this array as data.\n        \"\"\"\n        return op.exp(self, *args, **kwargs)\n\n    def expm1(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`expm1`.\n\n        The arguments are the same as for :py:func:`expm1`, with\n        this array as data.\n        \"\"\"\n        return op.expm1(self, *args, **kwargs)\n\n    def log(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log`.\n\n        The arguments are the same as for :py:func:`log`, with\n        this array as data.\n        \"\"\"\n        return op.log(self, *args, **kwargs)\n\n    def log10(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log10`.\n\n        The arguments are the same as for :py:func:`log10`, with\n        this array as data.\n        \"\"\"\n        return op.log10(self, *args, **kwargs)\n\n    def log2(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log2`.\n\n        The arguments are the same as for :py:func:`log2`, with\n        this array as data.\n        \"\"\"\n        return op.log2(self, *args, **kwargs)\n\n    def log1p(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log1p`.\n\n        The arguments are the same as for :py:func:`log1p`, with\n        this array as data.\n        \"\"\"\n        return op.log1p(self, *args, **kwargs)\n\n    def log_sigmoid(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log_sigmoid`.\n\n        The arguments are the same as for :py:func:`log_sigmoid`, with\n        this array as data.\n        \"\"\"\n        return op.log_sigmoid(self, *args, **kwargs)\n\n    def sqrt(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sqrt`.\n\n        The arguments are the same as for :py:func:`sqrt`, with\n        this array as data.\n        \"\"\"\n        return op.sqrt(self, *args, **kwargs)\n\n    def rsqrt(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`rsqrt`.\n\n        The arguments are the same as for :py:func:`rsqrt`, with\n        this array as data.\n        \"\"\"\n        return op.rsqrt(self, *args, **kwargs)\n\n    def cbrt(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`cbrt`.\n\n        The arguments are the same as for :py:func:`cbrt`, with\n        this array as data.\n        \"\"\"\n        return op.cbrt(self, *args, **kwargs)\n\n    def rcbrt(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`rcbrt`.\n\n        The arguments are the same as for :py:func:`rcbrt`, with\n        this array as data.\n        \"\"\"\n        return op.rcbrt(self, *args, **kwargs)\n\n    def square(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`square`.\n\n        The arguments are the same as for :py:func:`square`, with\n        this array as data.\n        \"\"\"\n        return op.square(self, *args, **kwargs)\n\n    def reciprocal(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`reciprocal`.\n\n        The arguments are the same as for :py:func:`reciprocal`, with\n        this array as data.\n        \"\"\"\n        return op.reciprocal(self, *args, **kwargs)\n\n    def relu(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`relu`.\n\n        The arguments are the same as for :py:func:`relu`, with\n        this array as data.\n        \"\"\"\n        return op.relu(self, *args, **kwargs)\n\n    def sigmoid(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sigmoid`.\n\n        The arguments are the same as for :py:func:`sigmoid`, with\n        this array as data.\n        \"\"\"\n        return op.sigmoid(self, *args, **kwargs)\n\n    def softmax(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`softmax`.\n\n        The arguments are the same as for :py:func:`softmax`, with\n        this array as data.\n        \"\"\"\n        return op.softmax(self, *args, **kwargs)\n\n    def log_softmax(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log_softmax`.\n\n        The arguments are the same as for :py:func:`log_softmax`, with\n        this array as data.\n        \"\"\"\n        return op.log_softmax(self, *args, **kwargs)\n\n    def softmin(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`softmin`.\n\n        The arguments are the same as for :py:func:`softmin`, with\n        this array as data.\n        \"\"\"\n        return op.softmin(self, *args, **kwargs)\n\n    def mish(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`mish`.\n\n        The arguments are the same as for :py:func:`mish`, with\n        this array as data.\n        \"\"\"\n        return op.mish(self, *args, **kwargs)\n\n    def squeeze(self, axis=None, inplace=False):\n        \"\"\"Remove dimensions with size 1 from this array without altering any data.\n\n        Parameters\n        ----------\n        axis : int, tuple of int, or None\n            Selects a subset of the single-dimensional entries in the shape.\n            If an axis is selected with shape entry greater than one, an error is raised.\n        inplace : bool, default False\n            If True, this method returns a **view** of this array\n            that shares data with this array. Otherwise, a copy is returned.\n        \"\"\"\n        if not inplace:\n            return op.squeeze(self, axis=axis)\n        else:\n            new_shape = list(self.shape)\n            axes = axis # rename variable for readability\n            if isinstance(axes, int):\n                axes = [axes]\n            if axes:\n                assert len(axes) == len(set(axes)), \\\n                    \"axis {} contains duplicate which is not allowed.\".format(axes)\n                resolved_axes = [i if i >= 0 else i+len(self.shape) for i in axes]\n                for arg_axis, actual_axis in zip(axes, resolved_axes):\n                    assert -len(new_shape) <= arg_axis < len(new_shape), \\\n                        \"axis {} is out of range for {}d array\".format(arg_axis, len(new_shape))\n                    axis_size = new_shape[actual_axis]\n                    assert axis_size == 1, \\\n                        \"Squeeze target axis {} must be size 1, got {}.\".format(arg_axis, axis_size)\n                for i in sorted(resolved_axes, reverse=True):\n                    del new_shape[i]\n            else:\n                for i in reversed(range(len(new_shape))):\n                    if new_shape[i] == 1:\n                        del new_shape[i]\n            if not new_shape:\n                new_shape.append(1)\n\n            return self.reshape(new_shape)\n\n    # pylint: disable= undefined-variable\n    def broadcast_to(self, shape):\n        \"\"\"Broadcasts the input array to a new shape.\n\n        Broadcasting is only allowed on axes with size 1. The new shape cannot change\n        the number of dimensions.\n        For example, you could broadcast from shape (2, 1) to (2, 3), but not from\n        shape (2, 3) to (2, 3, 3).\n\n        Parameters\n        ----------\n        shape : tuple of int\n            The shape of the desired array.\n\n        Returns\n        -------\n        NDArray\n            A NDArray with the desired shape that is not sharing data with this\n            array, even if the new shape is the same as ``self.shape``.\n\n        Examples\n        --------\n        >>> x = mx.nd.arange(0,3).reshape((1,3,1))\n        >>> x.asnumpy()\n        array([[[ 0.],\n                [ 1.],\n                [ 2.]]], dtype=float32)\n        >>> y = x.broadcast_to((2,3,3))\n        >>> y.asnumpy()\n        array([[[ 0.,  0.,  0.],\n                [ 1.,  1.,  1.],\n                [ 2.,  2.,  2.]],\n        <BLANKLINE>\n               [[ 0.,  0.,  0.],\n                [ 1.,  1.,  1.],\n                [ 2.,  2.,  2.]]], dtype=float32)\n        \"\"\"\n        cur_shape = self.shape\n        err_str = 'operands could not be broadcast together with remapped shapes' \\\n                  '[original->remapped]: {} and requested shape {}'.format(cur_shape, shape)\n        if len(shape) < len(cur_shape):\n            raise ValueError(err_str)\n        cur_shape = (1,) * (len(shape) - len(cur_shape)) + cur_shape\n        cur_shape_arr = np.array(cur_shape)\n        broadcasting_axes = np.nonzero(cur_shape_arr != np.array(shape))\n        if (cur_shape_arr[broadcasting_axes] != 1).any():\n            raise ValueError(err_str)\n        if cur_shape != self.shape:\n            return op.broadcast_to(self.reshape(cur_shape), shape=shape)\n        else:\n            return op.broadcast_to(self, shape=tuple(shape))\n    # pylint: enable= undefined-variable\n\n    def broadcast_like(self, other):\n        \"\"\"Broadcasts the input array to the shape of other.\n\n        Broadcasting is only allowed on axes with size 1. The new shape cannot change\n        the number of dimensions.\n        For example, you could broadcast from shape (2, 1) to (2, 3), but not from\n        shape (2, 3) to (2, 3, 3).\n\n        Parameters\n        ----------\n        other : NDArray\n            Array with shape of the desired array.\n\n        Returns\n        -------\n        NDArray\n            A NDArray with the desired shape that is not sharing data with this\n            array, even if the new shape is the same as ``self.shape``.\n\n        Examples\n        --------\n        >>> x = mx.nd.arange(0,3).reshape((1,3,1))\n        >>> x.asnumpy()\n        array([[[ 0.],\n                [ 1.],\n                [ 2.]]], dtype=float32)\n        >>> y = x.broadcast_like(mx.nd.ones((2,3,3)))\n        >>> y.asnumpy()\n        array([[[ 0.,  0.,  0.],\n                [ 1.,  1.,  1.],\n                [ 2.,  2.,  2.]],\n        <BLANKLINE>\n               [[ 0.,  0.,  0.],\n                [ 1.,  1.,  1.],\n                [ 2.,  2.,  2.]]], dtype=float32)\n        \"\"\"\n        return self.broadcast_to(other.shape)\n\n    def wait_to_read(self):\n        \"\"\"Waits until all previous write operations on the current array are finished.\n\n        This method guarantees that all previous write operations that pushed\n        into the backend engine for execution are actually finished.\n\n        Examples\n        --------\n        >>> import time\n        >>> tic = time.time()\n        >>> a = mx.nd.ones((1000,1000))\n        >>> b = mx.nd.dot(a, a)\n        >>> print(time.time() - tic) # doctest: +SKIP\n        0.003854036331176758\n        >>> b.wait_to_read()\n        >>> print(time.time() - tic) # doctest: +SKIP\n        0.0893700122833252\n        \"\"\"\n        check_call(_LIB.MXNDArrayWaitToRead(self.handle))\n\n    @property\n    def ndim(self):\n        \"\"\"Returns the number of dimensions of this array\n\n        Examples\n        --------\n        >>> x = mx.nd.array([1, 2, 3, 4])\n        >>> x.ndim\n        1\n        >>> x = mx.nd.array([[1, 2], [3, 4]])\n        >>> x.ndim\n        2\n        \"\"\"\n        return len(self.shape)\n\n    @property\n    def shape(self):\n        \"\"\"Tuple of array dimensions.\n\n        Examples\n        --------\n        >>> x = mx.nd.array([1, 2, 3, 4])\n        >>> x.shape\n        (4L,)\n        >>> y = mx.nd.zeros((2, 3, 4))\n        >>> y.shape\n        (2L, 3L, 4L)\n        \"\"\"\n        ndim = mx_int()\n        if _int64_enabled():\n            pdata = ctypes.POINTER(mx_int64)()\n            check_call(_LIB.MXNDArrayGetShape64(\n                self.handle, ctypes.byref(ndim), ctypes.byref(pdata)))\n        else:\n            pdata = ctypes.POINTER(mx_int)()\n            check_call(_LIB.MXNDArrayGetShape(\n                self.handle, ctypes.byref(ndim), ctypes.byref(pdata)))\n        if ndim.value == -1:\n            return None\n        else:\n            return tuple(pdata[:ndim.value])  # pylint: disable=invalid-slice-index\n\n\n    @property\n    def size(self):\n        \"\"\"Number of elements in the array.\n\n        Equivalent to the product of the array's dimensions.\n\n        Examples\n        --------\n        >>> import numpy as np\n        >>> x = mx.nd.zeros((3, 5, 2))\n        >>> x.size\n        30\n        >>> np.prod(x.shape)\n        30\n        \"\"\"\n        size = 1\n        for i in self.shape:\n            size *= i\n        return size\n\n    @property\n    def context(self):\n        \"\"\"Device context of the array.\n\n        Examples\n        --------\n        >>> x = mx.nd.array([1, 2, 3, 4])\n        >>> x.context\n        cpu(0)\n        >>> type(x.context)\n        <class 'mxnet.device.Device'>\n        >>> y = mx.nd.zeros((2,3), mx.gpu(0))\n        >>> y.context\n        gpu(0)\n        \"\"\"\n        dev_typeid = ctypes.c_int()\n        dev_id = ctypes.c_int()\n        check_call(_LIB.MXNDArrayGetContext(\n            self.handle, ctypes.byref(dev_typeid), ctypes.byref(dev_id)))\n        return Device(Device.devtype2str[dev_typeid.value], dev_id.value)\n\n    @property\n    def ctx(self):\n        \"\"\"Device context of the array. Has the same meaning as context.\n\n        Examples\n        --------\n        >>> x = mx.nd.array([1, 2, 3, 4])\n        >>> x.ctx\n        cpu(0)\n        >>> type(x.ctx)\n        <class 'mxnet.context.Context'>\n        >>> y = mx.nd.zeros((2,3), mx.gpu(0))\n        >>> y.ctx\n        gpu(0)\n        \"\"\"\n        return self.context\n\n    @property\n    def device(self):\n        \"\"\"Device context of the array. Has the same meaning as context.\n\n        Examples\n        --------\n        >>> x = mx.nd.array([1, 2, 3, 4])\n        >>> x.device\n        cpu(0)\n        >>> type(x.device)\n        <class 'mxnet.device.Device'>\n        >>> y = mx.nd.zeros((2,3), mx.gpu(0))\n        >>> y.device\n        gpu(0)\n        \"\"\"\n        return self.context\n\n    @property\n    def dtype(self):\n        \"\"\"Data-type of the array's elements.\n\n        Returns\n        -------\n        numpy.dtype\n            This NDArray's data type.\n\n        Examples\n        --------\n        >>> x = mx.nd.zeros((2,3))\n        >>> x.dtype\n        <type 'numpy.float32'>\n        >>> y = mx.nd.zeros((2,3), dtype='int32')\n        >>> y.dtype\n        <type 'numpy.int32'>\n        \"\"\"\n        mx_dtype = ctypes.c_int()\n        check_call(_LIB.MXNDArrayGetDType(\n            self.handle, ctypes.byref(mx_dtype)))\n        return dtype_mx_to_np(mx_dtype.value)\n\n    @property\n    def stype(self):\n        \"\"\"Storage-type of the array.\n        \"\"\"\n        return _STORAGE_TYPE_ID_TO_STR[_storage_type(self.handle)]\n\n    @property\n    # pylint: disable= invalid-name, undefined-variable\n    def T(self):\n        \"\"\"Returns a copy of the array with axes transposed.\n\n        Equivalent to ``mx.nd.transpose(self)`` except that\n        self is returned if ``self.ndim < 2``.\n\n        Unlike ``numpy.ndarray.T``, this function returns a copy\n        rather than a view of the array unless ``self.ndim < 2``.\n\n        Examples\n        --------\n        >>> x = mx.nd.arange(0,6).reshape((2,3))\n        >>> x.asnumpy()\n        array([[ 0.,  1.,  2.],\n               [ 3.,  4.,  5.]], dtype=float32)\n        >>> x.T.asnumpy()\n        array([[ 0.,  3.],\n               [ 1.,  4.],\n               [ 2.,  5.]], dtype=float32)\n\n        \"\"\"\n        if len(self.shape) < 2:\n            return self\n        return op.transpose(self)\n    # pylint: enable= invalid-name, undefined-variable\n\n    @property\n    def _fresh_grad(self):\n        \"\"\"Whether this array's corresponding gradient array\n        (registered via `autograd.mark_variables`) has been\n        updated by `autograd.backward` since last reset.\n\n        `_fresh_grad` need to be manually set to False\n        after consuming gradient (usually after updating this\n        array).\n        \"\"\"\n        out = ctypes.c_int()\n        check_call(_LIB.MXNDArrayGetGradState(self.handle, ctypes.byref(out)))\n        return out.value\n\n    @_fresh_grad.setter\n    def _fresh_grad(self, state):\n        check_call(_LIB.MXNDArraySetGradState(self.handle, ctypes.c_int(state)))\n\n    def asnumpy(self):\n        \"\"\"Returns a ``numpy.ndarray`` object with value copied from this array.\n\n        Examples\n        --------\n        >>> x = mx.nd.ones((2,3))\n        >>> y = x.asnumpy()\n        >>> type(y)\n        <type 'numpy.ndarray'>\n        >>> y\n        array([[ 1.,  1.,  1.],\n               [ 1.,  1.,  1.]], dtype=float32)\n        >>> z = mx.nd.ones((2,3), dtype='int32')\n        >>> z.asnumpy()\n        array([[1, 1, 1],\n               [1, 1, 1]], dtype=int32)\n        \"\"\"\n        if self.dtype == bfloat16:\n            return self.astype(np.float32).asnumpy()\n        data = np.empty(self.shape, dtype=self.dtype)\n        check_call(_LIB.MXNDArraySyncCopyToCPU(\n            self.handle,\n            data.ctypes.data_as(ctypes.c_void_p),\n            ctypes.c_size_t(data.size)))\n        return data\n\n    def asscalar(self):\n        \"\"\"Returns a scalar whose value is copied from this array.\n\n        This function is equivalent to ``self.asnumpy()[0]``. This NDArray must have shape (1,).\n\n        Examples\n        --------\n        >>> x = mx.nd.ones((1,), dtype='int32')\n        >>> x.asscalar()\n        1\n        >>> type(x.asscalar())\n        <type 'numpy.int32'>\n        \"\"\"\n        if self.size != 1:\n            raise ValueError(\"The current array is not a scalar\")\n        if self.ndim == 1:\n            return self.asnumpy()[0]\n        else:\n            return self.asnumpy()[()]\n\n    def astype(self, dtype, copy=True):\n        \"\"\"Returns a copy of the array after casting to a specified type.\n\n        Parameters\n        ----------\n        dtype : numpy.dtype or str\n            The type of the returned array.\n        copy : bool\n            Default `True`. By default, astype always returns a newly\n            allocated ndarray on the same context. If this is set to\n            `False`, and the dtype requested is the same as the ndarray's\n            dtype, the ndarray is returned instead of a copy.\n\n        Returns\n        -------\n        NDArray, CSRNDArray or RowSparseNDArray\n            The copied array after casting to the specified type, or\n            the same array if copy=False and dtype is the same as the input\n            array.\n\n        Examples\n        --------\n        >>> x = mx.nd.zeros((2,3), dtype='float32')\n        >>> y = x.astype('int32')\n        >>> y.dtype\n        <type 'numpy.int32'>\n        \"\"\"\n\n        if dtype is None:\n            dtype = mx_real_t\n        if not copy and np.dtype(dtype) == self.dtype:\n            return self\n\n        return op.cast(self, dtype=dtype)\n\n    def copyto(self, other):\n        \"\"\"Copies the value of this array to another array.\n\n        If ``other`` is a ``NDArray`` object, then ``other.shape`` and\n        ``self.shape`` should be the same. This function copies the value from\n        ``self`` to ``other``.\n\n        If ``other`` is a context, a new ``NDArray`` will be first created on\n        the target context, and the value of ``self`` is copied.\n\n        Parameters\n        ----------\n        other : NDArray or Context\n            The destination array or context.\n\n        Returns\n        -------\n        NDArray, CSRNDArray or RowSparseNDArray\n            The copied array. If ``other`` is an ``NDArray``, then the return value\n            and ``other`` will point to the same ``NDArray``.\n\n        Examples\n        --------\n        >>> x = mx.nd.ones((2,3))\n        >>> y = mx.nd.zeros((2,3), mx.gpu(0))\n        >>> z = x.copyto(y)\n        >>> z is y\n        True\n        >>> y.asnumpy()\n        array([[ 1.,  1.,  1.],\n               [ 1.,  1.,  1.]], dtype=float32)\n        >>> y.copyto(mx.gpu(0))\n        <NDArray 2x3 @gpu(0)>\n\n        \"\"\"\n        if isinstance(other, NDArray):\n            if other.handle is self.handle:\n                warnings.warn('You are attempting to copy an array to itself', RuntimeWarning)\n                return False\n            return _internal._copyto(self, out=other)\n        elif isinstance(other, Device):\n            hret = NDArray(_new_alloc_handle(self.shape, other, True, self.dtype))\n            return _internal._copyto(self, out=hret)\n        else:\n            raise TypeError('copyto does not support type ' + str(type(other)))\n\n    def copy(self):\n        \"\"\"Makes a copy of this ``NDArray``, keeping the same context.\n\n        Returns\n        -------\n        NDArray, CSRNDArray or RowSparseNDArray\n            The copied array\n\n        Examples\n        --------\n        >>> x = mx.nd.ones((2,3))\n        >>> y = x.copy()\n        >>> y.asnumpy()\n        array([[ 1.,  1.,  1.],\n               [ 1.,  1.,  1.]], dtype=float32)\n        \"\"\"\n        return self.copyto(self.ctx)\n\n    def slice_assign_scalar(self, value, begin, end, step):\n        \"\"\"\n        Assign the scalar to a cropped subset of this NDArray. Value will broadcast to the shape of the cropped shape\n        and will be cast to the same dtype of the NDArray.\n\n        Parameters\n        ----------\n        value: numeric value\n            Value and this NDArray should be of the same data type.\n            The shape of rhs should be the same as the cropped shape of this NDArray.\n        begin: tuple of begin indices\n        end: tuple of end indices\n        step: tuple of step lenghths\n\n        Returns\n        -------\n        This NDArray.\n\n        Examples\n        --------\n        >>> from mxnet import nd\n        >>> x = nd.ones((2, 2, 2))\n        >>> y = x.slice_assign_scalar(0, (0, 0, None), (1, 1, None), (None, None, None))\n        >>> y\n        [[[0. 0.]\n        [1. 1.]]\n\n        [[1. 1.]\n        [1. 1.]]]\n        <NDArray 2x2x2 @cpu(0)>\n        >>> x\n        [[[0. 0.]\n        [1. 1.]]\n\n        [[1. 1.]\n        [1. 1.]]]\n        <NDArray 2x2x2 @cpu(0)>\n\n        \"\"\"\n        return _internal._slice_assign_scalar(self, value, begin=begin, end=end, step=step, out=self)\n\n    def slice_assign(self, rhs, begin, end, step):\n        \"\"\"\n        Assign the rhs to a cropped subset of this NDarray in place.\n        Returns the view of this NDArray.\n\n        Parameters\n        ----------\n        rhs: NDArray.\n            rhs and this NDArray should be of the same data type, and on the same device.\n            The shape of rhs should be the same as the cropped shape of this NDArray.\n        begin: tuple of begin indices\n        end: tuple of end indices\n        step: tuple of step lenghths\n\n        Returns\n        -------\n        This NDArray.\n\n        Examples\n        --------\n        >>> x = nd.ones((2, 2, 2))\n        >>> assigned = nd.zeros((1, 1, 2))\n        >>> y = x.slice_assign(assigned, (0, 0, None), (1, 1, None), (None, None, None))\n        >>> y\n        [[[0. 0.]\n        [1. 1.]]\n\n        [[1. 1.]\n        [1. 1.]]]\n        <NDArray 2x2x2 @cpu(0)>\n        >>> x\n        [[[0. 0.]\n        [1. 1.]]\n\n        [[1. 1.]\n        [1. 1.]]]\n        <NDArray 2x2x2 @cpu(0)>\n        \"\"\"\n        return _internal._slice_assign(self, rhs, begin=begin, end=end, step=step, out=self)\n\n\n    def as_in_context(self, context):\n        \"\"\"Returns an array on the target device with the same value as this array.\n\n        If the target context is the same as ``self.context``, then ``self`` is\n        returned.  Otherwise, a copy is made.\n\n        Parameters\n        ----------\n        context : Context\n            The target context.\n\n        Returns\n        -------\n        NDArray, CSRNDArray or RowSparseNDArray\n            The target array.\n\n\n        Examples\n        --------\n        >>> x = mx.nd.ones((2,3))\n        >>> y = x.as_in_context(mx.cpu())\n        >>> y is x\n        True\n        >>> z = x.as_in_context(mx.gpu(0))\n        >>> z is x\n        False\n        \"\"\"\n        if self.context == context:\n            return self\n        return self.copyto(context)\n\n    def attach_grad(self, grad_req='write', stype=None):\n        \"\"\"Attach a gradient buffer to this NDArray, so that `backward`\n        can compute gradient with respect to it.\n\n        The gradient is initialized to zeros.\n\n        Parameters\n        ----------\n        grad_req : {'write', 'add', 'null'}\n            How gradient will be accumulated.\n            - 'write': gradient will be overwritten on every backward.\n            - 'add': gradient will be added to existing value on every backward.\n            - 'null': do not compute gradient for this NDArray.\n        stype : str, optional\n            The storage type of the gradient array. Defaults to the same stype of this NDArray.\n        \"\"\"\n        from . import zeros as _zeros\n        if stype is not None:\n            grad = _zeros(self.shape, stype=stype, dtype=self.dtype)\n        else:\n            grad = op.zeros_like(self)  # pylint: disable=undefined-variable\n        grad_req = _GRAD_REQ_MAP[grad_req]\n        check_call(_LIB.MXAutogradMarkVariables(\n            1, ctypes.pointer(self.handle),\n            ctypes.pointer(mx_uint(grad_req)),\n            ctypes.pointer(grad.handle)))\n\n    def drop_grad(self):\n        \"\"\"Free the memory of the marked ndarray.\"\"\"\n        check_call(_LIB.MXAutogradDropGrads(\n            1, ctypes.pointer(self.handle)))\n\n    @property\n    def grad(self):\n        \"\"\"Returns gradient buffer attached to this NDArray.\"\"\"\n        from . import _ndarray_cls\n        hdl = NDArrayHandle()\n        check_call(_LIB.MXNDArrayGetGrad(self.handle, ctypes.byref(hdl)))\n        if hdl.value is None:\n            return None\n        return _ndarray_cls(hdl)\n\n    def detach(self):\n        \"\"\"Returns a new NDArray, detached from the current graph.\"\"\"\n        from . import _ndarray_cls\n        hdl = NDArrayHandle()\n        check_call(_LIB.MXNDArrayDetach(self.handle, ctypes.byref(hdl)))\n        return _ndarray_cls(hdl)\n\n    def backward(self, out_grad=None, retain_graph=False, train_mode=True):\n        \"\"\"Compute the gradients of this NDArray w.r.t variables.\n\n        Parameters\n        ----------\n        out_grad : NDArray, optional\n            Gradient with respect to head.\n        retain_graph : bool, optional\n            Whether to retain the computaion graph for another backward\n            pass on the same graph. By default the computaion history\n            is cleared.\n        train_mode : bool, optional\n            Whether to compute gradient for training or inference.\n        \"\"\"\n        if out_grad is None:\n            ograd_handles = [NDArrayHandle(0)]\n        else:\n            ograd_handles = [out_grad.handle]\n\n        check_call(_LIB.MXAutogradBackwardEx(\n            1, c_handle_array([self]),\n            c_array(NDArrayHandle, ograd_handles),\n            0,\n            ctypes.c_void_p(0),\n            ctypes.c_int(retain_graph),\n            ctypes.c_int(0),\n            ctypes.c_int(train_mode),\n            ctypes.c_void_p(0),\n            ctypes.c_void_p(0)))\n\n    def tostype(self, stype):\n        \"\"\"Return a copy of the array with chosen storage type.\n\n        See Also\n        ----------\n        :meth:`mxnet.ndarray.cast_storage`.\n\n        Returns\n        -------\n        NDArray, CSRNDArray or RowSparseNDArray\n            A copy of the array with the chosen storage stype\n        \"\"\"\n        if stype == 'csr' and len(self.shape) != 2:\n            raise ValueError(\"To convert to a CSR, the NDArray should be 2 Dimensional. Current \"\n                             f\"shape is {str(self.shape)}\")\n\n        return op.cast_storage(self, stype=stype)\n\n    def to_dlpack_for_read(self):\n        \"\"\"Returns a reference view of NDArray that represents as DLManagedTensor until\n        all previous write operations on the current array are finished.\n\n        Returns\n        -------\n        PyCapsule (the pointer of DLManagedTensor)\n            a reference view of NDArray that represents as DLManagedTensor.\n\n        Examples\n        --------\n        >>> x = mx.nd.ones((2,3))\n        >>> y = mx.nd.to_dlpack_for_read(x)\n        >>> type(y)\n        <class 'PyCapsule'>\n        >>> z = mx.nd.from_dlpack(y)\n        >>> z\n        [[1. 1. 1.]\n         [1. 1. 1.]]\n        <NDArray 2x3 @cpu(0)>\n        \"\"\"\n        return to_dlpack_for_read(self)\n\n    def to_dlpack_for_write(self):\n        \"\"\"Returns a reference view of NDArray that represents as DLManagedTensor until\n        all previous read/write operations on the current array are finished.\n\n        Returns\n        -------\n        PyCapsule (the pointer of DLManagedTensor)\n            a reference view of NDArray that represents as DLManagedTensor.\n\n        Examples\n        --------\n        >>> x = mx.nd.ones((2,3))\n        >>> w = mx.nd.to_dlpack_for_write(x)\n        >>> type(w)\n        <class 'PyCapsule'>\n        >>> u = mx.nd.from_dlpack(w)\n        >>> u += 1\n        >>> x\n        [[2. 2. 2.]\n         [2. 2. 2.]]\n        <NDArray 2x3 @cpu(0)>\n        \"\"\"\n        return to_dlpack_for_write(self)\n\n    def _full(self, value):\n        \"\"\"\n        This is added as an NDArray class method in order to support polymorphism in NDArray and numpy.ndarray indexing\n        \"\"\"\n        return _internal._full(self.shape, value=value, ctx=self.ctx, dtype=self.dtype, out=self)\n\n    def _scatter_set_nd(self, value_nd, indices):\n        \"\"\"\n        This is added as an NDArray class method in order to support polymorphism in NDArray and numpy.ndarray indexing\n        \"\"\"\n        return _internal._scatter_set_nd(\n            lhs=self, rhs=value_nd, indices=indices, shape=self.shape, out=self\n        )\n\ndef check_boolean_array_dimension(array_shape, axis, bool_shape):\n    \"\"\"\n    Advanced boolean indexing is implemented through the use of `nonzero`.\n    Size check is necessary to make sure that the boolean array\n    has exactly as many dimensions as it is supposed to work with before the conversion\n    \"\"\"\n    for i, val in enumerate(bool_shape):\n        if array_shape[axis + i] != val:\n            raise IndexError('boolean index did not match indexed array along axis {};'\n                             ' size is {} but corresponding boolean size is {}'\n                             .format(axis + i, array_shape[axis + i], val))\n\ndef indexing_key_expand_implicit_axes(key, shape):\n    \"\"\"\n    Make implicit axes explicit by adding ``slice(None)``\n    and convert boolean array to integer array through `nonzero`.\n\n    Examples\n    --------\n    >>> shape = (3, 4, 5)\n    >>> indexing_key_expand_implicit_axes(np.s_[2, 1, 1], shape)\n    (2, 1, 1)\n    >>> indexing_key_expand_implicit_axes(np.s_[0], shape)\n    (0, slice(None, None, None), slice(None, None, None))\n    >>> indexing_key_expand_implicit_axes(np.s_[0, ...], shape)  # equivalent\n    (0, slice(None, None, None), slice(None, None, None))\n    >>> indexing_key_expand_implicit_axes(np.s_[:2, None, 0, ...], shape)\n    (slice(None, 2, None), None, 0, slice(None, None, None))\n    >>> bool_array = np.array([[True, False, True, False],\n                               [False, True, False, True],\n                               [True, False, True, False]], dtype=np.bool)\n    >>> indexing_key_expand_implicit_axes(np.s_[bool_array, None, 0:2], shape)\n    (array([0, 0, 1, 1, 2, 2], dtype=int64), array([0, 2, 1, 3, 0, 2], dtype=int64), None, slice(None, 2, None))\n    \"\"\"\n    if not isinstance(key, tuple):\n        key = (key,)\n    # We need to loop explicitly since tuple functions like `index()` or\n    # `count()` use `==` internally, which doesn't play well with fancy\n    # indexing.\n    ell_idx = None\n    num_none = 0\n    nonell_key = []\n\n    # For 0-d boolean indices: A new axis is added,\n    # but at the same time no axis is \"used\". So if we have True,\n    # we add a new axis (a bit like with np.newaxis). If it is\n    # False, we add a new axis, but this axis has 0 entries.\n    # prepend is defined to handle this case.\n    # prepend = _NDARRAY_NO_ZERO_DIM_BOOL_ARRAY/-1 means there is no 0-d boolean scalar\n    # prepend = _NDARRAY_ZERO_DIM_BOOL_ARRAY_FALSE/0 means an zero dim must be expanded\n    # prepend = _NDARRAY_ZERO_DIM_BOOL_ARRAY_TRUE/1 means a new axis must be expanded\n    prepend = _NDARRAY_NO_ZERO_DIM_BOOL_ARRAY\n    axis = 0\n    for i, idx in enumerate(key):\n        if idx is Ellipsis:\n            if ell_idx is not None:\n                raise IndexError(\n                    'Cannot use more than one ellipsis (`...`) for indexing'\n                )\n            ell_idx = i\n        else:\n            # convert primitive type boolean value to mx.np.bool type\n            # otherwise will be treated as 1/0\n            if isinstance(idx, bool):\n                idx = array(idx, dtype=np.bool_)\n            if idx is None:\n                num_none += 1\n            if isinstance(idx, NDArrayBase) and idx.ndim == 0 and idx.dtype == np.bool_:\n                if not idx: # array(False) has priority\n                    prepend = _NDARRAY_ZERO_DIM_BOOL_ARRAY_FALSE\n                else:\n                    prepend = _NDARRAY_ZERO_DIM_BOOL_ARRAY_TRUE\n            elif isinstance(idx, NDArrayBase) and idx.ndim == 0 and idx.dtype != np.bool_:\n                # This handles ndarray of zero dim. e.g array(1)\n                # while advoid converting zero dim boolean array\n                # float type will be converted to int\n                nonell_key.append(int(idx.item()))\n                axis += 1\n            elif isinstance(idx, NDArrayBase) and idx.dtype == np.bool_:\n                # Necessary size check before using `nonzero`\n                check_boolean_array_dimension(shape, axis, idx.shape)\n                # If the whole array is false and npx.set_np() is not set_up\n                # the program will throw infer shape error\n                if not is_np_array():\n                    raise ValueError('Cannot perform boolean indexing in legacy mode. Please activate'\n                                     ' numpy semantics by calling `npx.set_np()` in the global scope'\n                                     ' before calling this function.')\n                # Add the arrays from the nonzero result to the index\n                nonell_key.extend(idx.nonzero())\n                axis += idx.ndim\n            else:\n                nonell_key.append(idx)\n                axis += 1\n\n    nonell_key = tuple(nonell_key)\n\n    if ell_idx is None:\n        # This handles the case of \"too few\" indices, e.g., `nd.zeros((2, 3))[0]`,\n        # where the ellipsis is implicitly after the last entry.\n        ell_idx = len(nonell_key)\n\n    ell_ndim = len(shape) + num_none - len(nonell_key)\n    expanded_key = (nonell_key[:ell_idx] +\n                    (slice(None),) * ell_ndim +\n                    nonell_key[ell_idx:])\n\n    return expanded_key, prepend\n\n\ndef _int_to_slice(idx):\n    \"\"\"Return a slice that indexes the same entries as a single int.\"\"\"\n    if idx == -1:\n        # Avoid slice(-1, 0)\n        return slice(-1, None)\n    else:\n        return slice(idx, idx + 1)\n\n\ndef _shape_for_bcast(shape, target_ndim, new_axes):\n    \"\"\"Return shape with added axes for broadcasting in ``target_ndim`` dimensions.\n\n    If ``shape`` is shorter than ``target_ndim``, fixed ``1`` entries are inserted\n    into the returned shape, in locations indexed by ``new_axes``. The rest is\n    filled from the back with ``shape`` while possible.\n    \"\"\"\n    new_shape = [None] * target_ndim\n    if len(shape) < target_ndim:\n        for new_ax in new_axes:\n            new_shape[new_ax] = 1\n\n    # Replace `None` from the right with `shape` entries from the right as\n    # long as possible, thereafter with 1.\n    ax_s = 1\n    for ax in range(1, target_ndim + 1):\n        if new_shape[-ax] is None:\n            try:\n                new_shape[-ax] = shape[-ax_s]\n                ax_s += 1\n            except IndexError:\n                new_shape[-ax] = 1\n\n    return tuple(new_shape)\n\n\ndef _is_advanced_index(idx):\n    \"\"\"Return whether ``idx`` is an advanced index (array-like or integer).\n\n    Note that in contrast to basic indexing, integers are considered advanced\n    indices in the context of advanced indexing as they participate in\n    broadcasting.\n    \"\"\"\n    if isinstance(idx, (NDArray, np.ndarray, integer_types, list, tuple)):\n        return True\n    elif isinstance(idx, py_slice) or idx is None:\n        return False\n    elif isinstance(idx, range):\n        return True\n    else:\n        raise RuntimeError('illegal index type {}'.format(type(idx)))\n\n\ndef get_indexing_dispatch_code(key):\n    \"\"\"Returns a dispatch code for calling basic or advanced indexing functions.\"\"\"\n    assert isinstance(key, tuple)\n\n    for idx in key:\n        if isinstance(idx, (NDArray, np.ndarray, list, tuple, range)):\n            if isinstance(idx, tuple) and len(idx) == 0:\n                return _NDARRAY_EMPTY_TUPLE_INDEXING\n            return _NDARRAY_ADVANCED_INDEXING\n        elif not (isinstance(idx, (py_slice, integer_types)) or idx is None):\n            raise ValueError(\n                'NDArray does not support slicing with key {} of type {}.'\n                ''.format(idx, type(idx))\n            )\n    return _NDARRAY_BASIC_INDEXING\n\n\ndef _get_index_range(start, stop, length, step=1):\n    \"\"\"Given start, stop, step and array length, return\n    absolute values of start, stop, and step for generating index range.\n    The returned values have been compensated by adding length if they\n    are less than zero for all the cases but slice(None, None, -1).\n    Note that the returned value of stop is not necessarily >= 0, since\n    absolute stop is -1 in the case of slice(None, None, -1).\"\"\"\n    if step == 0:\n        raise ValueError('step size cannot be zero')\n    if length < 0:\n        raise ValueError('array length cannot be less than zero')\n    if step is None:\n        step = 1\n    if start is None:\n        if step > 0:\n            start = 0\n        else:\n            start = length - 1\n    elif start < 0:\n        start += length\n        if start < 0:\n            start = 0\n    elif start >= length:\n        start = length\n\n    if stop is None:\n        if step > 0:\n            stop = length\n        else:\n            # this supports case such as ::-1\n            # stop = -1 here refers to the element before index 0,\n            # instead of the last element in the array\n            stop = -1\n    elif stop < 0:\n        stop += length\n        if stop < 0:\n            stop = 0\n    elif stop > length:\n        stop = length\n\n    return start, stop, step\n\n\ndef get_oshape_of_gather_nd_op(dshape, ishape):\n    \"\"\"Given data and index shapes, get the output `NDArray` shape.\n    This basically implements the infer shape logic of op gather_nd.\"\"\"\n    assert len(dshape) > 0 and len(ishape) > 0\n    oshape = list(ishape[1:])\n    if ishape[0] < len(dshape):\n        oshape.extend(dshape[ishape[0]:])\n    return tuple(oshape)\n\n\ndef _get_dim_size(start, stop, step):\n    \"\"\"Given start, stop, and step, calculate the number of elements\n    of this slice.\n    \"\"\"\n    assert step != 0\n    if stop == start:\n        return 0\n    if step > 0:\n        assert start < stop\n        dim_size = (stop - start - 1) // step + 1\n    else:\n        assert stop < start\n        dim_size = (start - stop - 1) // (-step) + 1\n    return dim_size\n\n\ndef _get_slice_len(slc, seq_length):\n    \"\"\"Given a python slice object and the length of the sequence, calculate the number of elements\n     in the slice.\n\n    Parameters\n    ----------\n    slc : py_slice\n        The slice object\n    seq_length : int\n        The length of the object you are going to apply the slice on\n\n    Returns\n    -------\n    ret : int\n        Total number of elements in the slice\n    \"\"\"\n    start, stop, step = slc.indices(seq_length)\n    return max(0, (stop - start + (step - (1 if step > 0 else -1))) // step)\n\n\ndef _get_broadcast_shape(shape1, shape2):\n    \"\"\"Given two shapes that are not identical, find the shape\n    that both input shapes can broadcast to.\"\"\"\n    if shape1 == shape2:\n        return shape1\n\n    length1 = len(shape1)\n    length2 = len(shape2)\n    if length1 > length2:\n        shape = list(shape1)\n    else:\n        shape = list(shape2)\n    i = max(length1, length2) - 1\n    for a, b in zip(shape1[::-1], shape2[::-1]):\n        if a != 1 and b != 1 and a != b:\n            raise ValueError(f'shape1={shape1} is not broadcastable to shape2={shape2}')\n        shape[i] = b if a == 1 else a\n        i -= 1\n    return tuple(shape)\n\n\ndef _broadcast_shapes(seq):\n    \"\"\"Return the broadcast shape of all advanced indices in ``seq``.\n\n    All entries are assumed to have a ``shape`` property.\n    \"\"\"\n    return reduce(_get_broadcast_shape, [x.shape for x in seq], ())\n\n\ndef onehot_encode(indices, out):\n    \"\"\"One-hot encoding indices into matrix out.\n\n    .. note:: `onehot_encode` is deprecated. Use `one_hot` instead.\n\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _internal._onehot_encode(indices, out, out=out)\n    # pylint: enable= no-member, protected-access\n\n\ndef ones(shape, ctx=None, dtype=None, **kwargs):\n    \"\"\"Returns a new array filled with all ones, with the given shape and type.\n\n    Parameters\n    ----------\n    shape : int or tuple of int or list of int\n        The shape of the empty array.\n    ctx : Context, optional\n        An optional device context.\n        Defaults to the current default context (``mxnet.context.current_context()``).\n    dtype : str or numpy.dtype, optional\n        An optional value type (default is `float32`).\n    out : NDArray, optional\n        The output NDArray (default is `None`).\n\n    Returns\n    -------\n    NDArray\n        A new array of the specified shape filled with all ones.\n\n    Examples\n    --------\n    >>> mx.nd.ones(1).asnumpy()\n    array([ 1.], dtype=float32)\n    >>> mx.nd.ones((1,2), mx.gpu(0))\n    <NDArray 1x2 @gpu(0)>\n    >>> mx.nd.ones((1,2), dtype='float16').asnumpy()\n    array([[ 1.,  1.]], dtype=float16)\n    \"\"\"\n    # pylint: disable= unused-argument\n    if ctx is None:\n        ctx = current_device()\n    dtype = mx_real_t if dtype is None else dtype\n    # pylint: disable= no-member, protected-access\n    return _internal._ones(shape=shape, ctx=ctx, dtype=dtype, **kwargs)\n    # pylint: enable= no-member, protected-access\n\n\ndef full(shape, val, ctx=None, dtype=mx_real_t, out=None):\n    \"\"\"Returns a new array of given shape and type, filled with the given value `val`.\n\n    Parameters\n    --------\n    shape : int or tuple of int\n        The shape of the new array.\n    val : scalar\n        Fill value.\n    ctx : Context, optional\n        Device context (default is the current default context).\n    dtype : `str` or `numpy.dtype`, optional\n        The data type of the returned `NDArray`. The default datatype is `float32`.\n    out : NDArray, optional\n        The output NDArray (default is `None`).\n\n    Returns\n    -------\n    NDArray\n        `NDArray` filled with `val`, with the given shape, ctx, and dtype.\n\n    Examples\n    --------\n    >>> mx.nd.full(1, 2.0).asnumpy()\n    array([ 2.], dtype=float32)\n    >>> mx.nd.full((1, 2), 2.0, mx.gpu(0))\n    <NDArray 1x2 @gpu(0)>\n    >>> mx.nd.full((1, 2), 2.0, dtype='float16').asnumpy()\n    array([[ 2.,  2.]], dtype=float16)\n    \"\"\"\n    out = empty(shape, ctx, dtype) if out is None else out\n    out[:] = val\n    return out\n\n\ndef array(source_array, ctx=None, dtype=None):\n    \"\"\"Creates an array from any object exposing the array interface.\n\n    Parameters\n    ----------\n    source_array : array_like\n        An object exposing the array interface, an object whose `__array__`\n        method returns an array, or any (nested) sequence.\n    ctx : Context, optional\n        Device context (default is the current default context).\n    dtype : str or numpy.dtype, optional\n        The data type of the output array. The default dtype is ``source_array.dtype``\n        if `source_array` is an `NDArray`, `float32` otherwise.\n\n    Returns\n    -------\n    NDArray\n        An `NDArray` with the same contents as the `source_array`.\n    \"\"\"\n    if isinstance(source_array, NDArray):\n        dtype = source_array.dtype if dtype is None else dtype\n    else:\n        dtype = mx_real_t if dtype is None else dtype\n        if not isinstance(source_array, np.ndarray):\n            try:\n                source_array = np.array(source_array, dtype=dtype)\n            except:\n                raise TypeError('source_array must be array like object')\n\n    if source_array.shape == ():\n        # In this case we can't assign, so we need to go through an auxiliary array\n        arr = empty((1,), ctx, dtype)\n        arr[:] = source_array\n        return arr.reshape(())\n    elif source_array.size == 0:\n        return empty(source_array.shape, ctx, dtype)\n    else:\n        arr = empty(source_array.shape, ctx, dtype)\n        arr[:] = source_array\n        return arr\n\n\ndef moveaxis(tensor, source, destination):\n    \"\"\"Moves the `source` axis into the `destination` position\n    while leaving the other axes in their original order\n\n    Parameters\n    ----------\n    tensor : mx.nd.array\n        The array which axes should be reordered\n    source : int or sequence of int\n        Original position of the axes to move. Can be negative but must be unique.\n    destination : int or sequence of int\n        Destination position for each of the original axes. Can be negative but must be unique.\n\n    Returns\n    -------\n    result : mx.nd.array\n        Array with moved axes.\n\n    Examples\n    --------\n    >>> X = mx.nd.array([[1, 2, 3], [4, 5, 6]])\n    >>> mx.nd.moveaxis(X, 0, 1).shape\n    (3L, 2L)\n\n    >>> X = mx.nd.zeros((3, 4, 5))\n    >>> mx.nd.moveaxis(X, [0, 1], [-1, -2]).shape\n    (5, 4, 3)\n    \"\"\"\n    try:\n        source = np.core.numeric.normalize_axis_tuple(source, tensor.ndim)\n    except IndexError:\n        raise ValueError('Source should verify 0 <= source < tensor.ndim'\n                         f'Got {source}')\n    try:\n        destination = np.core.numeric.normalize_axis_tuple(destination, tensor.ndim)\n    except IndexError:\n        raise ValueError(f'Destination should verify 0 <= destination < tensor.ndim ({tensor.ndim}).',\n                         f'Got {destination}')\n\n    if len(source) != len(destination):\n        raise ValueError('`source` and `destination` arguments must have '\n                         'the same number of elements')\n\n    order = [n for n in range(tensor.ndim) if n not in source]\n\n    for dest, src in sorted(zip(destination, source)):\n        order.insert(dest, src)\n\n    return op.transpose(tensor, order)\n\n\n# pylint: disable= no-member, protected-access, too-many-arguments, redefined-outer-name\ndef arange(start, stop=None, step=1.0, repeat=1, infer_range=None, ctx=None, dtype=mx_real_t):\n    \"\"\"Returns evenly spaced values within a given interval.\n\n    Values are generated within the half-open interval [`start`, `stop`). In other\n    words, the interval includes `start` but excludes `stop`. The function is\n    similar to the built-in Python function `range` and to `numpy.arange`,\n    but returns an `NDArray`.\n\n    Parameters\n    ----------\n    start : number, optional\n        Start of interval. The default start value is 0.\n    stop : number\n        End of interval.\n    step : number, optional\n        Spacing between values. The default step size is 1.\n    repeat : int, optional\n        Number of times to repeat each element. The default repeat count is 1.\n    infer_range : boolean, optional\n        Infer the stop position from the start, step, repeat, and output tensor size.\n        Deprecated. Only False is supported.\n    ctx : Context, optional\n        Device context. Default context is the current default context.\n    dtype : str or numpy.dtype, optional\n        The data type of the `NDArray`. The default datatype is `np.float32`.\n\n    Returns\n    -------\n    NDArray\n        `NDArray` of evenly spaced values in the specified range.\n\n    Examples\n    --------\n    >>> mx.nd.arange(3).asnumpy()\n    array([ 0.,  1.,  2.], dtype=float32)\n    >>> mx.nd.arange(2, 6).asnumpy()\n    array([ 2.,  3.,  4.,  5.], dtype=float32)\n    >>> mx.nd.arange(2, 6, step=2).asnumpy()\n    array([ 2.,  4.], dtype=float32)\n    >>> mx.nd.arange(2, 6, step=1.5, repeat=2).asnumpy()\n    array([ 2. ,  2. ,  3.5,  3.5,  5. ,  5. ], dtype=float32)\n    >>> mx.nd.arange(2, 6, step=2, repeat=3, dtype='int32').asnumpy()\n    array([2, 2, 2, 4, 4, 4], dtype=int32)\n    \"\"\"\n    if infer_range is not None:\n        warnings.warn('`infer_range` argument has been deprecated',\n                      DeprecationWarning)\n    if ctx is None:\n        ctx = current_device()\n    return _internal._arange(start=start, stop=stop, step=step, repeat=repeat,\n                             infer_range=False, dtype=dtype, ctx=str(ctx))\n# pylint: enable= no-member, protected-access, too-many-arguments\n\n\n# pylint: disable= no-member, protected-access, too-many-arguments\ndef linspace(start, stop, num, endpoint=True, ctx=None, dtype=mx_real_t):\n    \"\"\"Return evenly spaced numbers within a specified interval.\n\n    Values are generated within the half-open interval [`start`, `stop`) or\n    closed interval [start, stop] depending on whether `endpoint` is True or\n    False. The function is similar to `numpy.linspace`, but returns an `NDArray`.\n\n    Parameters\n    ----------\n    start : number\n        Start of interval.\n    stop : number\n        End of interval, unless endpoint is set to False.  In that case,\n        the sequence consists of all but the last of `num + 1` evenly spaced\n        samples, so that stop is excluded. Note that the step size changes\n        when endpoint is False.\n    num : number\n        Number of samples to generate. Must be non-negative.\n    endpoint : bool\n        If True, stop is the last sample. Otherwise, it is not included.\n        The default is True.\n    ctx : Context, optional\n        Device context. Default context is the current default context.\n    dtype : str or numpy.dtype, optional\n        The data type of the `NDArray`. The default datatype is `np.float32`.\n\n    Returns\n    -------\n    NDArray\n        `NDArray` of evenly spaced values in the specified range.\n\n    Examples\n    --------\n    >>> mx.nd.linspace(2.0, 3.0, 5).asnumpy()\n    array([ 2.,  2.25.,  2.5,  2.75,  3.], dtype=float32)\n    >>> mx.nd.linspace(2.0, 3.0, 5, endpoint=False).asnumpy()\n    array([ 2.,  2.2.,  2.4,  2.6,  2.8], dtype=float32)\n    \"\"\"\n    if ctx is None:\n        ctx = current_device()\n    return _internal._linspace(start=start, stop=stop, num=num,\n                               endpoint=endpoint, dtype=dtype, ctx=str(ctx))\n# pylint: disable= no-member, protected-access, too-many-arguments\n\n\n#pylint: disable= too-many-arguments, no-member, protected-access\ndef _ufunc_helper(lhs, rhs, fn_array, fn_scalar, lfn_scalar, rfn_scalar=None):\n    \"\"\" Helper function for element-wise operation.\n    The function will perform numpy-like broadcasting if needed and call different functions.\n\n    Parameters\n    --------\n    lhs : NDArray or numeric value\n        Left-hand side operand.\n\n    rhs : NDArray or numeric value\n        Right-hand operand,\n\n    fn_array : function\n        Function to be called if both lhs and rhs are of ``NDArray`` type.\n\n    fn_scalar : function\n        Function to be called if both lhs and rhs are numeric values.\n\n    lfn_scalar : function\n        Function to be called if lhs is ``NDArray`` while rhs is numeric value\n\n    rfn_scalar : function\n        Function to be called if lhs is numeric value while rhs is ``NDArray``;\n        if none is provided, then the function is commutative, so rfn_scalar is equal to lfn_scalar\n\n    Returns\n    --------\n    NDArray\n        result array\n    \"\"\"\n    if isinstance(lhs, numeric_types):\n        if isinstance(rhs, numeric_types):\n            return fn_scalar(lhs, rhs)\n        else:\n            if rfn_scalar is None:\n                # commutative function\n                return lfn_scalar(rhs, float(lhs))\n            else:\n                return rfn_scalar(rhs, float(lhs))\n    elif isinstance(rhs, numeric_types):\n        return lfn_scalar(lhs, float(rhs))\n    elif isinstance(rhs, NDArray):\n        return fn_array(lhs, rhs)\n    else:\n        raise TypeError(f'type {str(type(rhs))} not supported')\n#pylint: enable= too-many-arguments, no-member, protected-access\n\n\ndef add(lhs, rhs):\n    \"\"\"Returns element-wise sum of the input arrays with broadcasting.\n\n    Equivalent to ``lhs + rhs``, ``mx.nd.broadcast_add(lhs, rhs)`` and\n    ``mx.nd.broadcast_plus(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.array\n        First array to be added.\n    rhs : scalar or mxnet.ndarray.array\n         Second array to be added.\n        If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        The element-wise sum of the input arrays.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))\n    >>> y = mx.nd.arange(2).reshape((2,1))\n    >>> z = mx.nd.arange(2).reshape((1,2))\n    >>> x.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 0.],\n           [ 1.]], dtype=float32)\n    >>> z.asnumpy()\n    array([[ 0.,  1.]], dtype=float32)\n    >>> (x+2).asnumpy()\n    array([[ 3.,  3.,  3.],\n           [ 3.,  3.,  3.]], dtype=float32)\n    >>> (x+y).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 2.,  2.,  2.]], dtype=float32)\n    >>> mx.nd.add(x,y).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 2.,  2.,  2.]], dtype=float32)\n    >>> (z + y).asnumpy()\n    array([[ 0.,  1.],\n           [ 1.,  2.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_add,\n        operator.add,\n        _internal._plus_scalar,\n        None)\n    # pylint: enable= no-member, protected-access\n\n\ndef subtract(lhs, rhs):\n    \"\"\"Returns element-wise difference of the input arrays with broadcasting.\n\n    Equivalent to ``lhs - rhs``, ``mx.nd.broadcast_sub(lhs, rhs)`` and\n    ``mx.nd.broadcast_minus(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.array\n        First array to be subtracted.\n    rhs : scalar or mxnet.ndarray.array\n         Second array to be subtracted.\n        If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        The element-wise difference of the input arrays.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))\n    >>> y = mx.nd.arange(2).reshape((2,1))\n    >>> z = mx.nd.arange(2).reshape((1,2))\n    >>> x.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 0.],\n           [ 1.]], dtype=float32)\n    >>> z.asnumpy()\n    array([[ 0.,  1.]], dtype=float32)\n    >>> (x-2).asnumpy()\n    array([[-1., -1., -1.],\n           [-1., -1., -1.]], dtype=float32)\n    >>> (x-y).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 0.,  0.,  0.]], dtype=float32)\n    >>> mx.nd.subtract(x,y).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 0.,  0.,  0.]], dtype=float32)\n    >>> (z-y).asnumpy()\n    array([[ 0.,  1.],\n           [-1.,  0.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_sub,\n        operator.sub,\n        _internal._minus_scalar,\n        _internal._rminus_scalar)\n    # pylint: enable= no-member, protected-access\n\n\ndef multiply(lhs, rhs):\n    \"\"\"Returns element-wise product of the input arrays with broadcasting.\n\n    Equivalent to ``lhs * rhs`` and ``mx.nd.broadcast_mul(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.array\n        First array to be multiplied.\n    rhs : scalar or mxnet.ndarray.array\n         Second array to be multiplied.\n        If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        The element-wise multiplication of the input arrays.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))\n    >>> y = mx.nd.arange(2).reshape((2,1))\n    >>> z = mx.nd.arange(2).reshape((1,2))\n    >>> x.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 0.],\n           [ 1.]], dtype=float32)\n    >>> z.asnumpy()\n    array([[ 0.,  1.]], dtype=float32)\n    >>> (x*2).asnumpy()\n    array([[ 2.,  2.,  2.],\n           [ 2.,  2.,  2.]], dtype=float32)\n    >>> (x*y).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> mx.nd.multiply(x, y).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> (z*y).asnumpy()\n    array([[ 0.,  0.],\n           [ 0.,  1.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_mul,\n        operator.mul,\n        _internal._mul_scalar,\n        None)\n    # pylint: enable= no-member, protected-access\n\n\ndef divide(lhs, rhs):\n    \"\"\"Returns element-wise division of the input arrays with broadcasting.\n\n    Equivalent to ``lhs / rhs`` and ``mx.nd.broadcast_div(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.array\n        First array in division.\n    rhs : scalar or mxnet.ndarray.array\n         Second array in division.\n        The arrays to be divided. If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        The element-wise division of the input arrays.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))*6\n    >>> y = mx.nd.ones((2,1))*2\n    >>> x.asnumpy()\n    array([[ 6.,  6.,  6.],\n           [ 6.,  6.,  6.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 2.],\n           [ 2.]], dtype=float32)\n    >>> x/2\n    <NDArray 2x3 @cpu(0)>\n    >>> (x/3).asnumpy()\n    array([[ 2.,  2.,  2.],\n           [ 2.,  2.,  2.]], dtype=float32)\n    >>> (x/y).asnumpy()\n    array([[ 3.,  3.,  3.],\n           [ 3.,  3.,  3.]], dtype=float32)\n    >>> mx.nd.divide(x,y).asnumpy()\n    array([[ 3.,  3.,  3.],\n           [ 3.,  3.,  3.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_div,\n        operator.truediv,\n        _internal._div_scalar,\n        _internal._rdiv_scalar)\n    # pylint: enable= no-member, protected-access\n\n\ndef modulo(lhs, rhs):\n    \"\"\"Returns element-wise modulo of the input arrays with broadcasting.\n\n    Equivalent to ``lhs % rhs`` and ``mx.nd.broadcast_mod(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.array\n        First array in modulo.\n    rhs : scalar or mxnet.ndarray.array\n         Second array in modulo.\n        The arrays to be taken modulo. If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        The element-wise modulo of the input arrays.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))*6\n    >>> y = mx.nd.ones((2,1))*4\n    >>> x.asnumpy()\n    array([[ 6.,  6.,  6.],\n           [ 6.,  6.,  6.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 4.],\n           [ 4.]], dtype=float32)\n    >>> x%5\n    <NDArray 2x3 @cpu(0)>\n    >>> (x%5).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> (x%y).asnumpy()\n    array([[ 2.,  2.,  2.],\n           [ 2.,  2.,  2.]], dtype=float32)\n    >>> mx.nd.modulo(x,y).asnumpy()\n    array([[ 2.,  2.,  2.],\n           [ 2.,  2.,  2.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_mod,\n        operator.mod,\n        _internal._mod_scalar,\n        _internal._rmod_scalar)\n    # pylint: enable= no-member, protected-access\n\n\ndef power(base, exp):\n    \"\"\"Returns result of first array elements raised to powers from second array, element-wise\n    with broadcasting.\n\n    Equivalent to ``base ** exp`` and ``mx.nd.broadcast_power(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    base : scalar or NDArray\n         The base array\n    exp : scalar or NDArray\n         The exponent array. If ``base.shape != exp.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    --------\n    NDArray\n        The bases in x raised to the exponents in y.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))*2\n    >>> y = mx.nd.arange(1,3).reshape((2,1))\n    >>> z = mx.nd.arange(1,3).reshape((2,1))\n    >>> x.asnumpy()\n    array([[ 2.,  2.,  2.],\n           [ 2.,  2.,  2.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 1.],\n           [ 2.]], dtype=float32)\n    >>> z.asnumpy()\n    array([[ 1.],\n           [ 2.]], dtype=float32)\n    >>> (x**2).asnumpy()\n    array([[ 4.,  4.,  4.],\n           [ 4.,  4.,  4.]], dtype=float32)\n    >>> (x**y).asnumpy()\n    array([[ 2.,  2.,  2.],\n           [ 4.,  4.,  4.]], dtype=float32)\n    >>> mx.nd.power(x,y).asnumpy()\n    array([[ 2.,  2.,  2.],\n           [ 4.,  4.,  4.]], dtype=float32)\n    >>> (z**y).asnumpy()\n    array([[ 1.],\n           [ 4.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        base,\n        exp,\n        op.broadcast_power,\n        operator.pow,\n        _internal._power_scalar,\n        _internal._rpower_scalar)\n    # pylint: enable= no-member, protected-access\n\n\ndef maximum(lhs, rhs):\n    \"\"\"Returns element-wise maximum of the input arrays with broadcasting.\n\n    Equivalent to ``mx.nd.broadcast_maximum(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.array\n        First array to be compared.\n    rhs : scalar or mxnet.ndarray.array\n         Second array to be compared. If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        The element-wise maximum of the input arrays.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))\n    >>> y = mx.nd.arange(2).reshape((2,1))\n    >>> z = mx.nd.arange(2).reshape((1,2))\n    >>> x.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 0.],\n           [ 1.]], dtype=float32)\n    >>> z.asnumpy()\n    array([[ 0.,  1.]], dtype=float32)\n    >>> mx.nd.maximum(x, 2).asnumpy()\n    array([[ 2.,  2.,  2.],\n           [ 2.,  2.,  2.]], dtype=float32)\n    >>> mx.nd.maximum(x, y).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> mx.nd.maximum(y, z).asnumpy()\n    array([[ 0.,  1.],\n           [ 1.,  1.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_maximum,\n        lambda x, y: x if x > y else y,\n        _internal._maximum_scalar,\n        None)\n    # pylint: enable= no-member, protected-access\n\n\ndef minimum(lhs, rhs):\n    \"\"\"Returns element-wise minimum of the input arrays with broadcasting.\n\n    Equivalent to ``mx.nd.broadcast_minimum(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.array\n        First array to be compared.\n    rhs : scalar or mxnet.ndarray.array\n         Second array to be compared. If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        The element-wise minimum of the input arrays.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))\n    >>> y = mx.nd.arange(2).reshape((2,1))\n    >>> z = mx.nd.arange(2).reshape((1,2))\n    >>> x.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 0.],\n           [ 1.]], dtype=float32)\n    >>> z.asnumpy()\n    array([[ 0.,  1.]], dtype=float32)\n    >>> mx.nd.minimum(x, 2).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> mx.nd.minimum(x, y).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> mx.nd.minimum(z, y).asnumpy()\n    array([[ 0.,  0.],\n           [ 0.,  1.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_minimum,\n        lambda x, y: x if x < y else y,\n        _internal._minimum_scalar,\n        None)\n    # pylint: enable= no-member, protected-access\n\n\ndef equal(lhs, rhs):\n    \"\"\"Returns the result of element-wise **equal to** (==) comparison operation with\n    broadcasting.\n\n    For each element in input arrays, return 1(true) if corresponding elements are same,\n    otherwise return 0(false).\n\n    Equivalent to ``lhs == rhs`` and ``mx.nd.broadcast_equal(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.array\n        First array to be compared.\n    rhs : scalar or mxnet.ndarray.array\n         Second array to be compared. If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        Output array of boolean values.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))\n    >>> y = mx.nd.arange(2).reshape((2,1))\n    >>> z = mx.nd.arange(2).reshape((1,2))\n    >>> x.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 0.],\n           [ 1.]], dtype=float32)\n    >>> z.asnumpy()\n    array([[ 0.,  1.]], dtype=float32)\n    >>> (x == 1).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> (x == y).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> mx.nd.equal(x,y).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> (z == y).asnumpy()\n    array([[ 1.,  0.],\n           [ 0.,  1.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_equal,\n        lambda x, y: 1 if x == y else 0,\n        _internal._equal_scalar,\n        None)\n    # pylint: enable= no-member, protected-access\n\n\ndef not_equal(lhs, rhs):\n    \"\"\"Returns the result of element-wise **not equal to** (!=) comparison operation\n    with broadcasting.\n\n    For each element in input arrays, return 1(true) if corresponding elements are different,\n    otherwise return 0(false).\n\n    Equivalent to ``lhs != rhs`` and ``mx.nd.broadcast_not_equal(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.array\n        First array to be compared.\n    rhs : scalar or mxnet.ndarray.array\n         Second array to be compared. If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        Output array of boolean values.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))\n    >>> y = mx.nd.arange(2).reshape((2,1))\n    >>> z = mx.nd.arange(2).reshape((1,2))\n    >>> x.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 0.],\n           [ 1.]], dtype=float32)\n    >>> z.asnumpy()\n    array([[ 0.,  1.]], dtype=float32)\n    >>> (z == y).asnumpy()\n    array([[ 1.,  0.],\n           [ 0.,  1.]], dtype=float32)\n    >>> (x != 1).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 0.,  0.,  0.]], dtype=float32)\n    >>> (x != y).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 0.,  0.,  0.]], dtype=float32)\n    >>> mx.nd.not_equal(x, y).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 0.,  0.,  0.]], dtype=float32)\n    >>> (z != y).asnumpy()\n    array([[ 0.,  1.],\n           [ 1.,  0.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_not_equal,\n        lambda x, y: 1 if x != y else 0,\n        _internal._not_equal_scalar,\n        None)\n    # pylint: enable= no-member, protected-access\n\n\ndef greater(lhs, rhs):\n    \"\"\"Returns the result of element-wise **greater than** (>) comparison operation\n    with broadcasting.\n\n    For each element in input arrays, return 1(true) if lhs elements are greater than rhs,\n    otherwise return 0(false).\n\n    Equivalent to ``lhs > rhs`` and ``mx.nd.broadcast_greater(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.array\n        First array to be compared.\n    rhs : scalar or mxnet.ndarray.array\n         Second array to be compared. If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        Output array of boolean values.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))\n    >>> y = mx.nd.arange(2).reshape((2,1))\n    >>> z = mx.nd.arange(2).reshape((1,2))\n    >>> x.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 0.],\n           [ 1.]], dtype=float32)\n    >>> z.asnumpy()\n    array([[ 0.,  1.]], dtype=float32)\n    >>> (x > 1).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 0.,  0.,  0.]], dtype=float32)\n    >>> (x > y).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 0.,  0.,  0.]], dtype=float32)\n    >>> mx.nd.greater(x, y).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 0.,  0.,  0.]], dtype=float32)\n    >>> (z > y).asnumpy()\n    array([[ 0.,  1.],\n           [ 0.,  0.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_greater,\n        lambda x, y: 1 if x > y else 0,\n        _internal._greater_scalar,\n        _internal._lesser_scalar)\n    # pylint: enable= no-member, protected-access\n\n\ndef greater_equal(lhs, rhs):\n    \"\"\"Returns the result of element-wise **greater than or equal to** (>=) comparison\n    operation with broadcasting.\n\n    For each element in input arrays, return 1(true) if lhs elements are greater than equal to rhs,\n    otherwise return 0(false).\n\n    Equivalent to ``lhs >= rhs`` and ``mx.nd.broadcast_greater_equal(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.array\n        First array to be compared.\n    rhs : scalar or mxnet.ndarray.array\n         Second array to be compared. If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        Output array of boolean values.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))\n    >>> y = mx.nd.arange(2).reshape((2,1))\n    >>> z = mx.nd.arange(2).reshape((1,2))\n    >>> x.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 0.],\n           [ 1.]], dtype=float32)\n    >>> z.asnumpy()\n    array([[ 0.,  1.]], dtype=float32)\n    >>> (x >= 1).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> (x >= y).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> mx.nd.greater_equal(x, y).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> (z >= y).asnumpy()\n    array([[ 1.,  1.],\n           [ 0.,  1.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_greater_equal,\n        lambda x, y: 1 if x >= y else 0,\n        _internal._greater_equal_scalar,\n        _internal._lesser_equal_scalar)\n    # pylint: enable= no-member, protected-access\n\n\ndef lesser(lhs, rhs):\n    \"\"\"Returns the result of element-wise **lesser than** (<) comparison operation\n    with broadcasting.\n\n    For each element in input arrays, return 1(true) if lhs elements are less than rhs,\n    otherwise return 0(false).\n\n    Equivalent to ``lhs < rhs`` and ``mx.nd.broadcast_lesser(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.array\n        First array to be compared.\n    rhs : scalar or mxnet.ndarray.array\n         Second array to be compared. If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        Output array of boolean values.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))\n    >>> y = mx.nd.arange(2).reshape((2,1))\n    >>> z = mx.nd.arange(2).reshape((1,2))\n    >>> x.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 0.],\n           [ 1.]], dtype=float32)\n    >>> z.asnumpy()\n    array([[ 0.,  1.]], dtype=float32)\n    >>> (x < 1).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 0.,  0.,  0.]], dtype=float32)\n    >>> (x < y).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 0.,  0.,  0.]], dtype=float32)\n    >>> mx.nd.lesser(x, y).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 0.,  0.,  0.]], dtype=float32)\n    >>> (z < y).asnumpy()\n    array([[ 0.,  0.],\n           [ 1.,  0.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_lesser,\n        lambda x, y: 1 if x < y else 0,\n        _internal._lesser_scalar,\n        _internal._greater_scalar)\n    # pylint: enable= no-member, protected-access\n\n\ndef lesser_equal(lhs, rhs):\n    \"\"\"Returns the result of element-wise **lesser than or equal to** (<=) comparison\n    operation with broadcasting.\n\n    For each element in input arrays, return 1(true) if lhs elements are\n    lesser than equal to rhs, otherwise return 0(false).\n\n    Equivalent to ``lhs <= rhs`` and ``mx.nd.broadcast_lesser_equal(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.array\n        First array to be compared.\n    rhs : scalar or mxnet.ndarray.array\n         Second array to be compared. If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        Output array of boolean values.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))\n    >>> y = mx.nd.arange(2).reshape((2,1))\n    >>> z = mx.nd.arange(2).reshape((1,2))\n    >>> x.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 0.],\n           [ 1.]], dtype=float32)\n    >>> z.asnumpy()\n    array([[ 0.,  1.]], dtype=float32)\n    >>> (x <= 1).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> (x <= y).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> mx.nd.lesser_equal(x, y).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> (z <= y).asnumpy()\n    array([[ 1.,  0.],\n           [ 1.,  1.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_lesser_equal,\n        lambda x, y: 1 if x <= y else 0,\n        _internal._lesser_equal_scalar,\n        _internal._greater_equal_scalar)\n    # pylint: enable= no-member, protected-access\n\ndef logical_and(lhs, rhs):\n    \"\"\"Returns the result of element-wise **logical and** comparison\n    operation with broadcasting.\n\n    For each element in input arrays, return 1(true) if lhs elements and rhs elements\n    are true, otherwise return 0(false).\n\n    Equivalent to ``lhs and rhs`` and ``mx.nd.broadcast_logical_and(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.array\n        First input of the function.\n    rhs : scalar or mxnet.ndarray.array\n         Second input of the function. If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        Output array of boolean values.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))\n    >>> y = mx.nd.arange(2).reshape((2,1))\n    >>> z = mx.nd.arange(2).reshape((1,2))\n    >>> x.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 0.],\n           [ 1.]], dtype=float32)\n    >>> z.asnumpy()\n    array([[ 0.,  1.]], dtype=float32)\n    >>> mx.nd.logical_and(x, 1).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> mx.nd.logical_and(x, y).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> mx.nd.logical_and(z, y).asnumpy()\n    array([[ 0.,  0.],\n           [ 0.,  1.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_logical_and,\n        lambda x, y: 1 if x and y else 0,\n        _internal._logical_and_scalar,\n        None)\n    # pylint: enable= no-member, protected-access\n\ndef logical_or(lhs, rhs):\n    \"\"\"Returns the result of element-wise **logical or** comparison\n    operation with broadcasting.\n\n    For each element in input arrays, return 1(true) if lhs elements or rhs elements\n    are true, otherwise return 0(false).\n\n    Equivalent to ``lhs or rhs`` and ``mx.nd.broadcast_logical_or(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.array\n        First input of the function.\n    rhs : scalar or mxnet.ndarray.array\n         Second input of the function. If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        Output array of boolean values.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))\n    >>> y = mx.nd.arange(2).reshape((2,1))\n    >>> z = mx.nd.arange(2).reshape((1,2))\n    >>> x.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 0.],\n           [ 1.]], dtype=float32)\n    >>> z.asnumpy()\n    array([[ 0.,  1.]], dtype=float32)\n    >>> mx.nd.logical_or(x, 1).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> mx.nd.logical_or(x, y).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> mx.nd.logical_or(z, y).asnumpy()\n    array([[ 0.,  1.],\n           [ 1.,  1.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_logical_or,\n        lambda x, y: 1 if x or y else 0,\n        _internal._logical_or_scalar,\n        None)\n    # pylint: enable= no-member, protected-access\n\ndef logical_xor(lhs, rhs):\n    \"\"\"Returns the result of element-wise **logical xor** comparison\n    operation with broadcasting.\n\n    For each element in input arrays, return 1(true) if lhs elements or rhs elements\n    are true, otherwise return 0(false).\n\n    Equivalent to ``bool(lhs) ^ bool(rhs)`` and ``mx.nd.broadcast_logical_xor(lhs, rhs)``.\n\n    .. note::\n\n       If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n       then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.array\n        First input of the function.\n    rhs : scalar or mxnet.ndarray.array\n         Second input of the function. If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        Output array of boolean values.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))\n    >>> y = mx.nd.arange(2).reshape((2,1))\n    >>> z = mx.nd.arange(2).reshape((1,2))\n    >>> x.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 0.],\n           [ 1.]], dtype=float32)\n    >>> z.asnumpy()\n    array([[ 0.,  1.]], dtype=float32)\n    >>> mx.nd.logical_xor(x, y).asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 0.,  0.,  0.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_logical_xor,\n        lambda x, y: 1 if bool(x) ^ bool(y) else 0,\n        _internal._logical_xor_scalar,\n        None)\n    # pylint: enable= no-member, protected-access\n\ndef true_divide(lhs, rhs):\n\n    \"\"\"This function is similar to :meth:`divide`.\n    \"\"\"\n    return divide(lhs, rhs)\n\n\ndef concatenate(arrays, axis=0, always_copy=True):\n    \"\"\"DEPRECATED, use ``concat`` instead\n\n    Parameters\n    ----------\n    arrays : list of `NDArray`\n        Arrays to be concatenate. They must have identical shape except\n        the first dimension. They also must have the same data type.\n    axis : int\n        The axis along which to concatenate.\n    always_copy : bool\n        Default `True`. When not `True`, if the arrays only contain one\n        `NDArray`, that element will be returned directly, avoid copying.\n\n    Returns\n    -------\n    NDArray\n        An `NDArray` that lives on the same context as `arrays[0].context`.\n    \"\"\"\n    # Unsupported in deferred compute mode due to use of inplace operations.\n    from .._deferred_compute import is_deferred_compute  # pylint: disable=wrong-import-position\n    assert not is_deferred_compute(), 'nd.concatenate is deprecated and ' \\\n        'unsupported in deferred compute mode. Use nd.concat instead.'\n\n    assert isinstance(arrays, list)\n    assert len(arrays) > 0\n    assert isinstance(arrays[0], NDArray)\n\n    if not always_copy and len(arrays) == 1:\n        return arrays[0]\n\n    shape_axis = arrays[0].shape[axis]\n    shape_rest1 = arrays[0].shape[0:axis]\n    shape_rest2 = arrays[0].shape[axis+1:]\n    dtype = arrays[0].dtype\n    for arr in arrays[1:]:\n        shape_axis += arr.shape[axis]\n        assert shape_rest1 == arr.shape[0:axis]\n        assert shape_rest2 == arr.shape[axis+1:]\n        assert dtype == arr.dtype\n    ret_shape = shape_rest1 + (shape_axis,) + shape_rest2\n    ret = empty(ret_shape, ctx=arrays[0].ctx, dtype=dtype)\n\n    idx = 0\n    begin = [0 for _ in ret_shape]\n    end = list(ret_shape)\n    for arr in arrays:\n        if axis == 0:\n            ret[idx:idx+arr.shape[0]] = arr\n        else:\n            begin[axis] = idx\n            end[axis] = idx+arr.shape[axis]\n            # pylint: disable=no-member,protected-access\n            _internal._crop_assign(ret, arr, out=ret,\n                                   begin=tuple(begin),\n                                   end=tuple(end))\n            # pylint: enable=no-member,protected-access\n        idx += arr.shape[axis]\n\n    return ret\n\n\n# pylint: disable=redefined-outer-name\ndef imdecode(str_img, clip_rect=(0, 0, 0, 0), out=None, index=0, channels=3, mean=None):\n    \"\"\"DEPRECATED, use mx.img instead\n\n    Parameters\n    ----------\n    str_img : str\n        Binary image data\n    clip_rect : iterable of 4 int\n        Clip decoded image to rectangle (x0, y0, x1, y1).\n    out : NDArray\n        Output buffer. Can be 3 dimensional (c, h, w) or 4 dimensional (n, c, h, w).\n    index : int\n        Output decoded image to i-th slice of 4 dimensional buffer.\n    channels : int\n        Number of channels to output. Decode to grey scale when channels = 1.\n    mean : NDArray\n        Subtract mean from decode image before outputing.\n    \"\"\"\n    # pylint: disable= no-member, protected-access, too-many-arguments\n    if mean is None:\n        mean = NDArray(_new_empty_handle())\n    if out is None:\n        return _internal._imdecode(mean, index,\n                                   clip_rect[0],\n                                   clip_rect[1],\n                                   clip_rect[2],\n                                   clip_rect[3],\n                                   channels,\n                                   len(str_img),\n                                   str_img=str_img)\n    else:\n        return _internal._imdecode(mean, index,\n                                   clip_rect[0],\n                                   clip_rect[1],\n                                   clip_rect[2],\n                                   clip_rect[3],\n                                   channels,\n                                   len(str_img),\n                                   str_img=str_img,\n                                   out=out)\n\n\ndef zeros(shape, ctx=None, dtype=None, **kwargs):\n    \"\"\"Returns a new array filled with all zeros, with the given shape and type.\n\n    Parameters\n    ----------\n    shape : int or tuple of int\n        The shape of the empty array.\n    ctx : Context, optional\n        An optional device context (default is the current default context).\n    dtype : str or numpy.dtype, optional\n        An optional value type (default is `float32`).\n    out : NDArray, optional\n        The output NDArray (default is `None`).\n\n    Returns\n    -------\n    NDArray\n        A created array\n\n    Examples\n    --------\n    >>> mx.nd.zeros(1).asnumpy()\n    array([ 0.], dtype=float32)\n    >>> mx.nd.zeros((1,2), mx.gpu(0))\n    <NDArray 1x2 @gpu(0)>\n    >>> mx.nd.zeros((1,2), mx.gpu(0), 'float16').asnumpy()\n    array([[ 0.,  0.]], dtype=float16)\n    \"\"\"\n    # pylint: disable= unused-argument\n    if ctx is None:\n        ctx = current_device()\n    dtype = mx_real_t if dtype is None else dtype\n    # pylint: disable= no-member, protected-access\n    return _internal._zeros(shape=shape, ctx=ctx, dtype=dtype, **kwargs)\n    # pylint: enable= no-member, protected-access\n\ndef eye(N, M=0, k=0, ctx=None, dtype=None, **kwargs):\n    \"\"\"Return a 2-D array with ones on the diagonal and zeros elsewhere.\n\n    Parameters\n    ----------\n    N: int\n        Number of rows in the output.\n    M: int, optional\n        Number of columns in the output. If 0, defaults to N.\n    k: int, optional\n        Index of the diagonal: 0 (the default) refers to the main diagonal,\n        a positive value refers to an upper diagonal,\n        and a negative value to a lower diagonal.\n    ctx: Context, optional\n        An optional device context (default is the current default context)\n    dtype: str or numpy.dtype, optional\n        An optional value type (default is `float32`)\n\n    Returns\n    -------\n    NDArray\n        A created array\n\n    Examples\n    --------\n    >>> mx.nd.eye(2)\n    [[ 1.  0.]\n     [ 0.  1.]]\n    <NDArray 2x2 @cpu(0)>\n    >>> mx.nd.eye(2, 3, 1)\n    [[ 0.  1.  0.]\n     [ 0.  0.  1.]]\n    <NDArray 2x3 @cpu(0)>\n    \"\"\"\n    # pylint: disable= unused-argument\n    if ctx is None:\n        ctx = current_device()\n    dtype = mx_real_t if dtype is None else dtype\n    # pylint: disable= no-member, protected-access\n    return _internal._eye(N=N, M=M, k=k, ctx=ctx, dtype=dtype, **kwargs)\n    # pylint: enable= no-member, protected-access\n\n\ndef empty(shape, ctx=None, dtype=None):\n    \"\"\"Returns a new array of given shape and type, without initializing entries.\n\n    Parameters\n    ----------\n    shape : int or tuple of int\n        The shape of the empty array.\n    ctx : Context, optional\n        An optional device context (default is the current default context).\n    dtype : str or numpy.dtype, optional\n        An optional value type (default is `float32`).\n\n    Returns\n    -------\n    NDArray\n        A created array.\n\n    \"\"\"\n    if isinstance(shape, int):\n        shape = (shape, )\n    if ctx is None:\n        ctx = current_device()\n    if dtype is None:\n        dtype = mx_real_t\n    return NDArray(handle=_new_alloc_handle(shape, ctx, False, dtype))\n\n\n# pylint: disable= redefined-builtin\ndef histogram(a, bins=10, range=None):\n    \"\"\"Compute the histogram of the input data.\n\n    Parameters\n    ----------\n    a : NDArray\n        Input data. The histogram is computed over the flattened array.\n    bins : int or sequence of scalars\n        If bins is an int, it defines the number of equal-width bins in the\n        given range (10, by default). If bins is a sequence, it defines the bin edges,\n        including the rightmost edge, allowing for non-uniform bin widths.\n    range : (float, float), optional\n        The lower and upper range of the bins. If not provided, range is simply (a.min(), a.max()).\n        Values outside the range are ignored. The first element of the range must be less than or\n        equal to the second. range affects the automatic bin computation as well, the range will\n        be equally divided by the number of bins.\n\n    Returns\n    -------\n    NDArray\n        A created array.\n\n    \"\"\"\n\n    # pylint: disable= no-member, protected-access\n    if isinstance(bins, NDArray):\n        return _internal._histogram(data=a, bins=bins)\n    elif isinstance(bins, integer_types):\n        if range is None:\n            warnings.warn(\"range is not specified, using numpy's result \"\n                          \"to ensure consistency with numpy\")\n            res, bin_bounds = np.histogram(a.asnumpy(), bins=bins)\n            return array(res), array(bin_bounds)\n        return _internal._histogram(data=a, bin_cnt=bins, range=range)\n    raise ValueError(\"bins argument should be either an integer or an NDArray\")\n    # pylint: enable= no-member, protected-access, redefined-builtin\n\ndef split_v2(ary, indices_or_sections, axis=0, squeeze_axis=False):\n    \"\"\"Split an array into multiple sub-arrays.\n\n    Parameters\n    ----------\n    ary : NDArray\n        Array to be divided into sub-arrays.\n    indices_or_sections : int or tuple of ints\n        If `indices_or_sections` is an integer, N, the array will be divided\n        into N equal arrays along `axis`.  If such a split is not possible,\n        an error is raised.\n        If `indices_or_sections` is a 1-D array of sorted integers, the entries\n        indicate where along `axis` the array is split.  For example,\n        ``[2, 3]`` would, for ``axis=0``, result in\n        - ary[:2]\n        - ary[2:3]\n        - ary[3:]\n        If an index exceeds the dimension of the array along `axis`,\n        an empty sub-array is returned correspondingly.\n    axis : int, optional\n        The axis along which to split, default is 0.\n    squeeze_axis: boolean, optional\n        Whether to squeeze the axis of sub-arrays or not, only useful when size\n        of the sub-arrays are 1 on the `axis`. Default is False.\n\n    Returns\n    -------\n    NDArray\n        A created array.\n\n    \"\"\"\n    indices = []\n    axis_size = ary.shape[axis]\n    if isinstance(indices_or_sections, int):\n        sections = indices_or_sections\n        if axis_size % sections:\n            raise ValueError('array split does not result in an equal division')\n        section_size = int(axis_size / sections)\n        indices = [i * section_size for i in range(sections)]\n    elif isinstance(indices_or_sections, tuple):\n        indices = [0] + list(indices_or_sections)\n    else:\n        raise ValueError('indices_or_sections must either int or tuple of ints')\n    return _internal._split_v2(ary, indices, axis, squeeze_axis)\n\nfrom_dlpack = ndarray_from_dlpack(NDArray)\nfrom_dlpack_doc = \"\"\"Returns a NDArray backed by a dlpack tensor.\n\n    Parameters\n    ----------\n    dlpack: PyCapsule (the pointer of DLManagedTensor)\n        input data\n\n    Returns\n    -------\n    NDArray\n        a NDArray backed by a dlpack tensor\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3))\n    >>> y = mx.nd.to_dlpack_for_read(x)\n    >>> type(y)\n    <class 'PyCapsule'>\n    >>> z = mx.nd.from_dlpack(y)\n    >>> type(z)\n    <class 'mxnet.ndarray.ndarray.NDArray'>\n    >>> z\n    [[ 1.  1.  1.]\n     [ 1.  1.  1.]]\n    <NDArray 2x3 @cpu(0)>\n\n    >>> w = mx.nd.to_dlpack_for_write(x)\n    >>> type(w)\n    <class 'PyCapsule'>\n    >>> u = mx.nd.from_dlpack(w)\n    >>> u += 1\n    >>> x\n    [[2. 2. 2.]\n     [2. 2. 2.]]\n    <NDArray 2x3 @cpu(0)>\n    \"\"\"\nfrom_dlpack.__doc__ = from_dlpack_doc\n\nfrom_numpy = ndarray_from_numpy(NDArray, array)\nfrom_numpy_doc = \"\"\"Returns an MXNet's NDArray backed by numpy's ndarray.\n    When `zero_copy` is set to be true,\n    this API consumes numpy's ndarray and produces MXNet's ndarray\n    without having to copy the content. In this case, we disallow\n    users to modify the given numpy ndarray, and it is suggested\n    not to read the numpy ndarray as well for internal correctness.\n\n    Parameters\n    ----------\n    ndarray: NDArray\n        input data\n    zero_copy: bool\n        Whether we use DLPack's zero-copy conversion to convert to MXNet's NDArray.\n        This is only available for c-contiguous arrays, i.e. array.flags[C_CONTIGUOUS] == True.\n\n    Returns\n    -------\n    NDArray\n        a NDArray backed by a dlpack tensor\n\"\"\"\nfrom_numpy.__doc__ = from_numpy_doc\n\n\nto_dlpack_for_read = ndarray_to_dlpack_for_read()\nto_dlpack_for_read_doc = \"\"\"Returns a reference view of NDArray that represents as DLManagedTensor until\nall previous write operations on the current array are finished.\n\nParameters\n----------\ndata: NDArray\n    input data.\n\nReturns\n-------\nPyCapsule (the pointer of DLManagedTensor)\n    a reference view of NDArray that represents as DLManagedTensor.\n\nExamples\n--------\n>>> x = mx.nd.ones((2,3))\n>>> y = mx.nd.to_dlpack_for_read(x)\n>>> type(y)\n<class 'PyCapsule'>\n>>> z = mx.nd.from_dlpack(y)\n>>> z\n[[1. 1. 1.]\n [1. 1. 1.]]\n<NDArray 2x3 @cpu(0)>\n\"\"\"\nto_dlpack_for_read.__doc__ = to_dlpack_for_read_doc\n\nto_dlpack_for_write = ndarray_to_dlpack_for_write()\nto_dlpack_for_write_doc = \"\"\"Returns a reference view of NDArray that represents as\nDLManagedTensor until all previous read/write operations on the current array are finished.\n\nParameters\n----------\ndata: NDArray\n    input data.\n\nReturns\n-------\nPyCapsule : the pointer of DLManagedTensor\n    a reference view of NDArray that represents as DLManagedTensor.\n\nExamples\n--------\n>>> x = mx.nd.ones((2,3))\n>>> w = mx.nd.to_dlpack_for_write(x)\n>>> type(w)\n<class 'PyCapsule'>\n>>> u = mx.nd.from_dlpack(w)\n>>> u += 1\n>>> x\n[[2. 2. 2.]\n [2. 2. 2.]]\n<NDArray 2x3 @cpu(0)>\n\"\"\"\nto_dlpack_for_write.__doc__ = to_dlpack_for_write_doc\n"
  },
  {
    "path": "python/mxnet/ndarray/numpy/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Module for numpy ops under mxnet.ndarray.\"\"\"\n\nfrom . import random\nfrom . import linalg\nfrom . import _op, _internal\nfrom . import _register\nfrom ._op import *  # pylint: disable=wildcard-import\n\n__all__ = _op.__all__\n"
  },
  {
    "path": "python/mxnet/ndarray/numpy/_api_internal.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for numpy internal api.\"\"\"\n\nfrom ..._ffi.function import _init_api\n\n__all__ = []\n\n_init_api(\"_npi\", \"mxnet.ndarray.numpy._api_internal\")\n"
  },
  {
    "path": "python/mxnet/ndarray/numpy/_internal.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for numpy internal ops.\"\"\"\n\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/ndarray/numpy/_op.py",
    "content": "# pylint: disable=C0302\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=unused-argument\n\"\"\"Namespace for numpy operators used in Gluon dispatched by F=ndarray.\"\"\"\n\nimport numpy as _np\nfrom ...base import numeric_types, integer_types\nfrom ...util import _sanity_check_params, set_module\nfrom ...util import wrap_np_unary_func, wrap_np_binary_func\nfrom ...util import is_np_default_dtype, dtype_from_number\nfrom ...device import current_device\nfrom . import _internal as _npi\nfrom . import _api_internal\nfrom ..ndarray import NDArray, get_dtype_name\n\n\n__all__ = ['shape', 'zeros', 'zeros_like', 'ones', 'ones_like', 'full', 'full_like', 'empty_like', 'invert', 'delete',\n           'add', 'broadcast_to', 'subtract', 'multiply', 'divide', 'mod', 'remainder', 'fmod',\n           'power', 'bitwise_not', 'trace', 'transpose', 'copy', 'moveaxis', 'reshape', 'dot',\n           'arctan2', 'sin', 'cos', 'tan', 'sinh', 'cosh', 'tanh', 'log10', 'sqrt', 'cbrt', 'abs', 'insert', 'fabs',\n           'absolute', 'exp', 'expm1', 'arcsin', 'arccos', 'arctan', 'sign', 'log', 'degrees', 'log2', 'matmul',\n           'log1p', 'rint', 'radians', 'reciprocal', 'square', 'negative', 'fix', 'ceil', 'floor', 'histogram',\n           'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'argsort', 'all', 'any', 'sort',\n           'tensordot', 'eye', 'linspace', 'median', 'tril_indices', 'triu_indices_from', 'triu_indices',\n           'logspace', 'expand_dims', 'tile', 'arange', 'array_split', 'split', 'hsplit', 'vsplit', 'dsplit',\n           'concatenate', 'append', 'stack', 'vstack', 'row_stack', 'column_stack', 'hstack', 'dstack',\n           'average', 'mean', 'maximum', 'fmax', 'minimum', 'fmin', 'around', 'round', 'round_', 'flatnonzero',\n           'max', 'min', 'amax', 'amin', 'logical_and', 'logical_or', 'logical_xor',\n           'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'indices', 'copysign', 'ravel', 'unravel_index',\n           'diag_indices_from', 'hanning', 'hamming', 'blackman', 'flip', 'flipud', 'fliplr',\n           'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm', 'gcd',\n           'tril', 'triu', 'tri', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'cross', 'kron',\n           'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum',\n           'true_divide', 'nonzero', 'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'interp',\n           'diff', 'ediff1d', 'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',\n           'atleast_1d', 'atleast_2d', 'atleast_3d', 'fill_diagonal', 'squeeze',\n           'where', 'bincount', 'rollaxis', 'diagflat', 'repeat', 'prod', 'pad', 'cumsum', 'sum', 'diag', 'diagonal',\n           'positive', 'logaddexp', 'floor_divide', 'bitwise_left_shift', 'bitwise_right_shift']\n\n\n@set_module('mxnet.ndarray.numpy')\ndef shape(a):\n    \"\"\"\n    Return the shape of an array.\n\n    Parameters\n    ----------\n    a : array_like\n        Input array.\n\n    Returns\n    -------\n    shape : tuple of ints\n        The elements of the shape tuple give the lengths of the\n        corresponding array dimensions.\n\n    See Also\n    --------\n    ndarray.shape : Equivalent array method.\n\n    Examples\n    --------\n    >>> np.shape(np.eye(3))\n    (3, 3)\n    >>> np.shape([[1, 2]])\n    (1, 2)\n    >>> np.shape([0])\n    (1,)\n    >>> np.shape(0)\n    ()\n    \"\"\"\n    return a.shape\n\n\n@set_module('mxnet.ndarray.numpy')\ndef zeros(shape, dtype=None, order='C', device=None):  # pylint: disable=redefined-outer-name\n    \"\"\"Return a new array of given shape and type, filled with zeros.\n    This function currently only supports storing multi-dimensional data\n    in row-major (C-style).\n\n    Parameters\n    ----------\n    shape : int or tuple of int\n        The shape of the empty array.\n    dtype : str or numpy.dtype, optional\n        An optional value type.\n        - When npx.is_np_default_dtype() returns False, default dtype is float32;\n        - When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Note that this behavior is different from NumPy's `zeros` function where `float64`\n        is the default value, here we can set 'float32' or 'float64' as your default dtype,\n        because `float32` is considered as the default data type in deep learning.\n    order : {'C'}, optional, default: 'C'\n        How to store multi-dimensional data in memory, currently only row-major\n        (C-style) is supported.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    out : ndarray\n        Array of zeros with the given shape, dtype, and device.\n    \"\"\"\n    if order != 'C':\n        raise NotImplementedError\n    # If the following code (4 lines) regarding device is removed\n    # np.zeros((3, 4)) can be as fast as 4.96 us\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    return _api_internal.zeros(shape, dtype, device)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef ones(shape, dtype=None, order='C', device=None):  # pylint: disable=redefined-outer-name\n    \"\"\"Return a new array of given shape and type, filled with ones.\n    This function currently only supports storing multi-dimensional data\n    in row-major (C-style).\n\n    Parameters\n    ----------\n    shape : int or tuple of int\n        The shape of the empty array.\n    dtype : str or numpy.dtype, optional\n        An optional value type.\n        - When npx.is_np_default_dtype() returns False, default dtype is float32;\n        - When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Note that this behavior is different from NumPy's `ones` function where\n        `float64` is the default value.\n    order : {'C'}, optional, default: 'C'\n        How to store multi-dimensional data in memory, currently only row-major\n        (C-style) is supported.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    out : ndarray\n        Array of ones with the given shape, dtype, and device.\n    \"\"\"\n    if order != 'C':\n        raise NotImplementedError\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    return _api_internal.ones(shape, dtype, device)\n\n\n# pylint: disable=too-many-arguments, redefined-outer-name\n@set_module('mxnet.ndarray.numpy')\ndef zeros_like(a, dtype=None, order='C', device=None, out=None):\n    \"\"\"\n    Return an array of zeros with the same shape and type as a given array.\n\n    Parameters\n    ----------\n    a : ndarray\n        The shape and data-type of `a` define these same attributes of\n        the returned array.\n    dtype : data-type, optional\n        Overrides the data type of the result.\n        Temporarily do not support boolean type.\n    order : {'C'}, optional\n        Whether to store multidimensional data in C- or Fortran-contiguous\n        (row- or column-wise) order in memory. Currently only supports C order.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n          Array of zeros with the same shape and type as a.\n\n    See Also\n    --------\n    empty_like : Return an empty array with shape and type of input.\n    ones_like : Return an array of ones with shape and type of input.\n    zeros_like : Return an array of zeros with shape and type of input.\n    full : Return a new array of given shape filled with value.\n\n    Examples\n    --------\n    >>> x = np.arange(6)\n    >>> x = x.reshape((2, 3))\n    >>> x\n    array([[0., 1., 2.],\n           [3., 4., 5.]])\n    >>> np.zeros_like(x)\n    array([[0., 0., 0.],\n           [0., 0., 0.]])\n    >>> np.zeros_like(x, int)\n    array([[0, 0, 0],\n           [0, 0, 0]], dtype=int64)\n    >>> y = np.arange(3, dtype=float)\n    >>> y\n    array([0., 1., 2.], dtype=float64)\n    >>> np.zeros_like(y)\n    array([0., 0., 0.], dtype=float64)\n    \"\"\"\n    if order != 'C':\n        raise NotImplementedError\n    return full_like(a, 0, dtype=dtype, order=order, device=device, out=out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef ones_like(a, dtype=None, order='C', device=None, out=None):\n    \"\"\"\n    Return an array of ones with the same shape and type as a given array.\n\n    Parameters\n    ----------\n    a : ndarray\n        The shape and data-type of `a` define these same attributes of\n        the returned array.\n    dtype : data-type, optional\n        Overrides the data type of the result.\n        Temporarily do not support boolean type.\n    order : {'C'}, optional\n        Whether to store multidimensional data in C- or Fortran-contiguous\n        (row- or column-wise) order in memory. Currently only supports C order.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n        Array of ones with the same shape and type as a.\n\n    See Also\n    --------\n    empty_like : Return an empty array with shape and type of input.\n    zeros_like : Return an array of zeros with shape and type of input.\n    full_like : Return a new array with shape of input filled with value.\n    ones : Return a new array setting values to one.\n\n    Examples\n    --------\n    >>> x = np.arange(6)\n    >>> x = x.reshape((2, 3))\n    >>> x\n    array([[0., 1., 2.],\n           [3., 4., 5.]])\n    >>> np.ones_like(x)\n    array([[1., 1., 1.],\n           [1., 1., 1.]])\n    >>> np.ones_like(x, int)\n    array([[1, 1, 1],\n           [1, 1, 1]], dtype=int64)\n    >>> y = np.arange(3, dtype=float)\n    >>> y\n    array([0., 1., 2.], dtype=float64)\n    >>> np.ones_like(y)\n    array([1., 1., 1.], dtype=float64)\n    \"\"\"\n    return full_like(a, 1, dtype=dtype, order=order, device=device, out=out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef broadcast_to(array, shape):\n    \"\"\"\n    Broadcast an array to a new shape.\n\n    Parameters\n    ----------\n    array : ndarray or scalar\n        The array to broadcast.\n    shape : tuple\n        The shape of the desired array.\n\n    Returns\n    -------\n    broadcast : array\n        A readonly view on the original array with the given shape. It is\n        typically not contiguous. Furthermore, more than one element of a\n        broadcasted array may refer to a single memory location.\n\n    Raises\n    ------\n    MXNetError\n        If the array is not compatible with the new shape according to NumPy's\n        broadcasting rules.\n    \"\"\"\n    if _np.isscalar(array):\n        return full(shape, array)\n    return _api_internal.broadcast_to(array, shape)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef full(shape, fill_value, dtype=None, order='C', device=None, out=None):  # pylint: disable=too-many-arguments\n    \"\"\"\n    Return a new array of given shape and type, filled with `fill_value`.\n\n    Parameters\n    ----------\n    shape : int or sequence of ints\n        Shape of the new array, e.g., ``(2, 3)`` or ``2``.\n    fill_value : scalar or ndarray\n        Fill value.\n    dtype : data-type, optional\n        If dtype is None, the output array data type must be inferred from fill_value.\n        If it’s an int, the output array dtype must be the default integer dtype;\n        If it’s a float, then the output array dtype must be the default floating-point data type;\n        If it’s a bool then the output array must have boolean dtype. Default: None.\n    order : {'C'}, optional\n        Whether to store multidimensional data in C- or Fortran-contiguous\n        (row- or column-wise) order in memory. Currently only supports C order.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n        Array of `fill_value` with the given shape, dtype, and order.\n        If `fill_value` is an ndarray, out will have the same device as `fill_value`\n        regardless of the provided `device`.\n\n    Notes\n    -----\n    This function differs from the original `numpy.full\n    https://docs.scipy.org/doc/numpy/reference/generated/numpy.full.html`_ in\n    the following way(s):\n    - Have an additional `device` argument to specify the device\n    - Have an additional `out` argument\n    - Currently does not support `order` selection\n\n    See Also\n    --------\n    empty : Return a new uninitialized array.\n    ones : Return a new array setting values to one.\n    zeros : Return a new array setting values to zero.\n\n    Examples\n    --------\n    >>> np.full((2, 2), 10)\n    array([[10., 10.],\n           [10., 10.]])\n    >>> np.full((2, 2), 2, dtype=np.int32, device=mx.cpu(0))\n    array([[2, 2],\n           [2, 2]], dtype=int32)\n\n    \"\"\"\n    if order != 'C':\n        raise NotImplementedError\n    if isinstance(fill_value, NDArray):\n        if dtype is None:\n            ret = broadcast_to(fill_value, shape)\n        else:\n            ret = broadcast_to(fill_value, shape).astype(dtype)\n        return ret\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if isinstance(fill_value, bool):\n        fill_value = int(fill_value)\n        dtype = _np.bool if dtype is None else dtype\n    elif isinstance(fill_value, numeric_types):\n        if dtype is None or dtype is float:\n            dtype = dtype_from_number(fill_value)\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    return _api_internal.full(shape, dtype, fill_value, device, out)\n# pylint: enable=too-many-arguments, redefined-outer-name\n\n\n@set_module('mxnet.ndarray.numpy')\ndef full_like(a, fill_value, dtype=None, order='C', device=None, out=None): # pylint: disable=too-many-arguments\n    \"\"\"\n    Return a full array with the same shape and type as a given array.\n\n    Parameters\n    ----------\n    a : ndarray\n        The shape and data-type of `a` define these same attributes of\n        the returned array.\n    fill_value : scalar\n        Fill value.\n    dtype : data-type, optional\n        Overrides the data type of the result.\n        Temporarily do not support boolean type.\n    order : {'C'}, optional\n        Whether to store multidimensional data in C- or Fortran-contiguous\n        (row- or column-wise) order in memory. Currently only supports C order.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n        Array of `fill_value` with the same shape and type as `a`.\n\n    See Also\n    --------\n    empty_like : Return an empty array with shape and type of input.\n    ones_like : Return an array of ones with shape and type of input.\n    zeros_like : Return an array of zeros with shape and type of input.\n    full : Return a new array of given shape filled with value.\n\n    Examples\n    --------\n    >>> x = np.arange(6, dtype=int)\n    >>> np.full_like(x, 1)\n    array([1, 1, 1, 1, 1, 1], dtype=int64)\n    >>> np.full_like(x, 0.1)\n    array([0, 0, 0, 0, 0, 0], dtype=int64)\n    >>> np.full_like(x, 0.1, dtype=np.float64)\n    array([0.1, 0.1, 0.1, 0.1, 0.1, 0.1], dtype=float64)\n    >>> np.full_like(x, np.nan, dtype=np.double)\n    array([nan, nan, nan, nan, nan, nan], dtype=float64)\n    >>> y = np.arange(6, dtype=np.float32)\n    >>> np.full_like(y, 0.1)\n    array([0.1, 0.1, 0.1, 0.1, 0.1, 0.1])\n    \"\"\"\n    if order != 'C':\n        raise NotImplementedError\n    if isinstance(fill_value, bool):\n        fill_value = int(fill_value)\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    return _api_internal.full_like(a, fill_value, dtype, device, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef empty_like(prototype, dtype=None, order='C', subok=False, shape=None): # pylint: disable=W0621\n    \"\"\"\n    Return a new array with the same shape and type as a given array.\n\n    Parameters\n    ----------\n    prototype : ndarray\n        The shape and data-type of `prototype` define these same attributes\n        of the returned array.\n    dtype : data-type, optional\n        Overrides the data type of the result.\n    order : {'C'}, optional\n        Whether to store multidimensional data in C- or Fortran-contiguous\n        (row- or column-wise) order in memory. Currently only supports C order.\n    subok : {False}, optional\n        If True, then the newly created array will use the sub-class\n        type of 'a', otherwise it will be a base-class array. Defaults\n        to False.\n        (Only support False at this moment)\n    shape : int or sequence of ints, optional.\n        Overrides the shape of the result. If order='K' and the number of\n        dimensions is unchanged, will try to keep order, otherwise,\n        order='C' is implied.\n        (Not supported at this moment)\n\n    Returns\n    -------\n    out : ndarray\n        Array of uninitialized (arbitrary) data with the same\n        shape and type as `prototype`.\n\n    See Also\n    --------\n    ones_like : Return an array of ones with shape and type of input.\n    zeros_like : Return an array of zeros with shape and type of input.\n    full_like : Return a new array with shape of input filled with value.\n    empty : Return a new uninitialized array.\n\n    Notes\n    -----\n    This function does *not* initialize the returned array; to do that use\n    `zeros_like` or `ones_like` instead.  It may be marginally faster than\n    the functions that do set the array values.\n\n    Examples\n    --------\n    >>> a = np.array([[1,2,3], [4,5,6]])\n    >>> np.empty_like(a)\n    array([[-5764607523034234880, -2305834244544065442,           4563075075], # uninitialized\n           [          4567052944, -5764607523034234880,      844424930131968]])\n    >>> a = np.array([[1., 2., 3.],[4.,5.,6.]])\n    >>> np.empty_like(a)\n    array([[4.9e-324, 9.9e-324, 1.5e-323], # uninitialized\n           [2.0e-323, 2.5e-323, 3.0e-323]])\n    \"\"\"\n    dtype_list = {_np.float16: 'float16', _np.float32: 'float32', _np.float64: 'float64',\n                  float: 'float64', _np.int8: 'int8', _np.int16: 'int16', _np.int32: 'int32',\n                  _np.int64: 'int64', int:'int64', _np.uint8: 'uint8', _np.uint16: 'uint16',\n                  _np.uint32: 'uint32', _np.uint64: 'uint64', _np.bool: 'bool',\n                  _np.bool_: 'bool_', bool: 'bool', None: 'None'}\n    if order != 'C':\n        raise NotImplementedError(\"Only support C-order at this moment\")\n    if subok:\n        raise NotImplementedError(\"Creating array by using sub-class is not supported at this moment\")\n    if shape is not None:\n        raise NotImplementedError(\"Assigning new shape is not supported at this moment\")\n    try:\n        dtype = dtype if isinstance(dtype, str) else dtype_list[dtype]\n    except:\n        raise NotImplementedError(\"Do not support this dtype at this moment\")\n    return _npi.empty_like_fallback(prototype, dtype=dtype, order=order, subok=subok, shape=shape)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef arange(start, stop=None, step=1, dtype=None, device=None):\n    \"\"\"Return evenly spaced values within a given interval.\n\n    Values are generated within the half-open interval ``[start, stop)``\n    (in other words, the interval including `start` but excluding `stop`).\n    For integer arguments the function is equivalent to the Python built-in\n    `range` function, but returns an ndarray rather than a list.\n\n    Parameters\n    ----------\n    start : number, optional\n        Start of interval. The interval includes this value.  The default\n        start value is 0.\n    stop : number\n        End of interval. The interval does not include this value, except\n        in some cases where `step` is not an integer and floating point\n        round-off affects the length of `out`.\n    step : number, optional\n        Spacing between values. For any output `out`, this is the distance\n        between two adjacent values, ``out[i+1] - out[i]``.  The default\n        step size is 1.  If `step` is specified as a position argument,\n        `start` must also be given.\n    dtype : dtype\n        The type of the output array.\n        - When npx.is_np_default_dtype() returns False, default dtype is float32;\n        - When npx.is_np_default_dtype() returns True, default dtype is float64.\n\n    Returns\n    -------\n    arange : ndarray\n        Array of evenly spaced values.\n\n        For floating point arguments, the length of the result is\n        ``ceil((stop - start)/step)``.  Because of floating point overflow,\n        this rule may result in the last element of `out` being greater\n        than `stop`.\n    \"\"\"\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if stop is None:\n        stop = start\n        start = 0\n    if step is None:\n        step = 1\n    if start is None and stop is None:\n        raise ValueError('start and stop cannot be both None')\n    if step == 0:\n        raise ZeroDivisionError('step cannot be 0')\n    return _api_internal.arange(start, stop, step, dtype, device)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef identity(n, dtype=None, device=None):\n    \"\"\"\n    Return the identity array.\n\n    The identity array is a square array with ones on\n    the main diagonal.\n\n    Parameters\n    ----------\n    n : int\n        Number of rows (and columns) in `n` x `n` output.\n    dtype : data-type, optional\n        Data-type of the output.\n        - When npx.is_np_default_dtype() returns False, default dtype is float32;\n        - When npx.is_np_default_dtype() returns True, default dtype is float64.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    out : ndarray\n        `n` x `n` array with its main diagonal set to one,\n        and all other elements 0.\n\n    Examples\n    --------\n    >>> np.identity(3)\n    array([[1., 0., 0.],\n           [0., 1., 0.],\n           [0., 0., 1.]])\n    \"\"\"\n    if not isinstance(n, int):\n        raise TypeError(\"Input 'n' should be an integer\")\n    if n < 0:\n        raise ValueError(\"Input 'n' cannot be negative\")\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    shape = (n, n)  # pylint: disable=redefined-outer-name\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    return _api_internal.identity(shape, dtype, device)\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.ndarray.numpy')\ndef take(a, indices, axis=None, mode='raise', out=None):\n    r\"\"\"\n    Take elements from an array along an axis.\n\n    When axis is not None, this function does the same thing as \"fancy\"\n    indexing (indexing arrays using arrays); however, it can be easier to use\n    if you need elements along a given axis. A call such as\n    ``np.take(arr, indices, axis=3)`` is equivalent to\n    ``arr[:,:,:,indices,...]``.\n\n    Explained without fancy indexing, this is equivalent to the following use\n    of `ndindex`, which sets each of ``ii``, ``jj``, and ``kk`` to a tuple of\n    indices::\n\n        Ni, Nk = a.shape[:axis], a.shape[axis+1:]\n        Nj = indices.shape\n        for ii in ndindex(Ni):\n            for jj in ndindex(Nj):\n                for kk in ndindex(Nk):\n                    out[ii + jj + kk] = a[ii + (indices[jj],) + kk]\n\n    Parameters\n    ----------\n    a : ndarray\n        The source array.\n    indices : ndarray\n        The indices of the values to extract. Also allow scalars for indices.\n    axis : int, optional\n        The axis over which to select values. By default, the flattened\n        input array is used.\n    out : ndarray, optional\n        If provided, the result will be placed in this array. It should\n        be of the appropriate shape and dtype.\n    mode : {'clip', 'wrap'}, optional\n        Specifies how out-of-bounds indices will behave.\n\n        * 'clip' -- clip to the range (default)\n        * 'wrap' -- wrap around\n\n        'clip' mode means that all indices that are too large are replaced\n        by the index that addresses the last element along that axis. Note\n        that this disables indexing with negative numbers.\n\n    Returns\n    -------\n    out : ndarray\n        The returned array has the same type as `a`.\n\n    Notes\n    -----\n\n    This function differs from the original `numpy.take\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.take.html>`_ in\n    the following way(s):\n\n    - Only ndarray or scalar ndarray is accepted as valid input.\n\n    Examples\n    --------\n    >>> a = np.array([4, 3, 5, 7, 6, 8])\n    >>> indices = np.array([0, 1, 4])\n    >>> np.take(a, indices)\n    array([4., 3., 6.])\n\n    In this example for `a` is an ndarray, \"fancy\" indexing can be used.\n\n    >>> a[indices]\n    array([4., 3., 6.])\n\n    If `indices` is not one dimensional, the output also has these dimensions.\n\n    >>> np.take(a, np.array([[0, 1], [2, 3]]))\n    array([[4., 3.],\n           [5., 7.]])\n    \"\"\"\n    if mode not in ('wrap', 'clip', 'raise'):\n        raise NotImplementedError(\n            \"function take does not support mode '{}'\".format(mode))\n    if axis is None:\n        return _api_internal.take(reshape(a, -1), indices, 0, mode, out)\n    else:\n        return _api_internal.take(a, indices, axis, mode, out)\n# pylint: enable=redefined-outer-name\n\n\n@set_module('mxnet.ndarray.numpy')\ndef insert(arr, obj, values, axis=None):\n    \"\"\"\n    Insert values along the given axis before the given indices.\n\n    Parameters\n    ----------\n    arr : ndarray\n        Input array.\n    obj : int, slice or ndarray of int64\n        Object that defines the index or indices before which `values` is\n        inserted.\n        Support for multiple insertions when `obj` is a single scalar or a\n        sequence with one element (only support int32 and int64 element).\n    values : ndarray\n        Values to insert into `arr`.\n        If the type of values is different from that of arr, values is converted\n        to the type of arr.\n    axis : int, optional\n        Axis along which to insert `values`.  If `axis` is None then `arr`\n        is flattened first.\n\n    Returns\n    -------\n    out : ndarray\n        A copy of `arr` with `values` inserted.  Note that `insert`\n        does not occur in-place: a new array is returned. If\n        `axis` is None, `out` is a flattened array.\n\n    Notes\n    -----\n    - Note that for higher dimensional inserts `obj=0` behaves very different\n    from `obj=[0]` just like `arr[:,0,:] = values` is different from\n    `arr[:,[0],:] = values`.\n    - If obj is a ndarray, it's dtype only supports int64\n\n    Examples\n    --------\n    >>> a = np.array([[1, 1], [2, 2], [3, 3]])\n    >>> a\n    array([[1., 1.],\n           [2., 2.],\n           [3., 3.]])\n    >>> np.insert(a, 1, np.array(5))\n    array([1., 5., 1., 2., 2., 3., 3.])\n    >>> np.insert(a, 1, np.array(5), axis=1)\n    array([[1., 5., 1.],\n           [2., 5., 2.],\n           [3., 5., 3.]])\n\n    Difference between sequence and scalars:\n\n    >>> np.insert(a, np.array([1], dtype=np.int64), np.array([[1],[2],[3]]), axis=1)\n    array([[1., 1., 1.],\n           [2., 2., 2.],\n           [3., 3., 3.]])\n    >>> np.insert(a, 1, np.array([1, 2, 3]), axis=1)\n    array([[1., 1., 1.],\n           [2., 2., 2.],\n           [3., 3., 3.]])\n\n    >>> b = a.flatten()\n    >>> b\n    array([1., 1., 2., 2., 3., 3.])\n    >>> np.insert(b, np.array([2, 2], dtype=np.int64), np.array([5, 6]))\n    array([1., 1., 5., 6., 2., 2., 3., 3.])\n\n    >>> np.insert(b, slice(2, 4), np.array([5, 6]))\n    array([1., 1., 5., 2., 6., 2., 3., 3.])\n\n    # type casting\n    >>> np.insert(b.astype(np.int32), np.array([2, 2],dtype='int64'), np.array([7.13, False]))\n    array([1, 1, 7, 0, 2, 2, 3, 3], dtype=int32)\n\n    >>> x = np.arange(8).reshape(2, 4)\n    >>> idx = np.array([1, 3], dtype=np.int64)\n    >>> np.insert(x, idx, np.array([999]), axis=1)\n    array([[  0., 999.,   1.,   2., 999.,   3.],\n           [  4., 999.,   5.,   6., 999.,   7.]])\n    \"\"\"\n    if isinstance(values, numeric_types):\n        if isinstance(obj, slice):\n            start = obj.start\n            stop = obj.stop\n            step = 1 if obj.step is None else obj.step\n            return _api_internal.insert_slice(arr, values, start, stop, step, axis)\n        elif isinstance(obj, integer_types):\n            return _api_internal.insert_scalar(arr, values, obj, axis)\n        elif isinstance(obj, NDArray):\n            return _api_internal.insert_tensor(arr, obj, values, axis)\n\n    if not isinstance(arr, NDArray):\n        raise TypeError(\"'arr' can not support type {}\".format(str(type(arr))))\n    if not isinstance(values, NDArray):\n        raise TypeError(\"'values' can not support type {}\".format(str(type(values))))\n    if isinstance(obj, slice):\n        start = obj.start\n        stop = obj.stop\n        step = 1 if obj.step is None else obj.step\n        return _api_internal.insert_slice(arr, values, start, stop, step, axis)\n    elif isinstance(obj, integer_types):\n        return _api_internal.insert_scalar(arr, values, obj, axis)\n    elif isinstance(obj, NDArray):\n        return _api_internal.insert_tensor(arr, values, obj, axis)\n    else:\n        raise TypeError(\"'obj' can not support type {}\".format(str(type(obj))))\n\n\n#pylint: disable= too-many-arguments, no-member, protected-access\ndef _ufunc_helper(lhs, rhs, fn_array, fn_scalar, lfn_scalar, rfn_scalar=None, out=None):\n    \"\"\" Helper function for element-wise operation.\n    The function will perform numpy-like broadcasting if needed and call different functions.\n\n    Parameters\n    --------\n    lhs : ndarray or numeric value\n        Left-hand side operand.\n\n    rhs : ndarray or numeric value\n        Right-hand operand,\n\n    fn_array : function\n        Function to be called if both lhs and rhs are of ``ndarray`` type.\n\n    fn_scalar : function\n        Function to be called if both lhs and rhs are numeric values.\n\n    lfn_scalar : function\n        Function to be called if lhs is ``ndarray`` while rhs is numeric value\n\n    rfn_scalar : function\n        Function to be called if lhs is numeric value while rhs is ``ndarray``;\n        if none is provided, then the function is commutative, so rfn_scalar is equal to lfn_scalar\n\n    Returns\n    --------\n    mxnet.numpy.ndarray or scalar\n        result array or scalar\n    \"\"\"\n    from ...numpy import ndarray\n    from ...numpy_extension import from_numpy  # pylint: disable=unused-import\n    if isinstance(lhs, numeric_types):\n        if isinstance(rhs, numeric_types):\n            return fn_scalar(lhs, rhs, out=out)\n        else:\n            if rfn_scalar is None:\n                # commutative function\n                return lfn_scalar(rhs, float(lhs), out=out)\n            else:\n                return rfn_scalar(rhs, float(lhs), out=out)\n    elif isinstance(rhs, numeric_types):\n        return lfn_scalar(lhs, float(rhs), out=out)\n    elif isinstance(lhs, ndarray) and isinstance(rhs, ndarray):\n        return fn_array(lhs, rhs, out=out)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(rhs))))\n#pylint: enable= too-many-arguments, no-member, protected-access\n\n\n@set_module('mxnet.ndarray.numpy')\ndef unique(ar, return_index=False, return_inverse=False, return_counts=False, axis=None):\n    \"\"\"\n    Find the unique elements of an array.\n\n    Returns the sorted unique elements of an array. There are three optional\n    outputs in addition to the unique elements:\n\n    * the indices of the input array that give the unique values\n    * the indices of the unique array that reconstruct the input array\n    * the number of times each unique value comes up in the input array\n\n    Parameters\n    ----------\n    ar : ndarray\n        Input array. Unless `axis` is specified, this will be flattened if it\n        is not already 1-D.\n    return_index : bool, optional\n        If True, also return the indices of `ar` (along the specified axis,\n        if provided, or in the flattened array) that result in the unique array.\n    return_inverse : bool, optional\n        If True, also return the indices of the unique array (for the specified\n        axis, if provided) that can be used to reconstruct `ar`.\n    return_counts : bool, optional\n        If True, also return the number of times each unique item appears\n        in `ar`.\n    axis : int or None, optional\n        The axis to operate on. If None, `ar` will be flattened. If an integer,\n        the subarrays indexed by the given axis will be flattened and treated\n        as the elements of a 1-D array with the dimension of the given axis,\n        see the notes for more details. The default is None.\n\n    Returns\n    -------\n    unique : ndarray\n        The sorted unique values.\n    unique_indices : ndarray, optional\n        The indices of the first occurrences of the unique values in the\n        original array. Only provided if `return_index` is True.\n    unique_inverse : ndarray, optional\n        The indices to reconstruct the original array from the\n        unique array. Only provided if `return_inverse` is True.\n    unique_counts : ndarray, optional\n        The number of times each of the unique values comes up in the\n        original array. Only provided if `return_counts` is True.\n\n    Notes\n    -----\n    When an axis is specified the subarrays indexed by the axis are sorted.\n    This is done by making the specified axis the first dimension of the array\n    and then flattening the subarrays in C order. The flattened subarrays are\n    then viewed as a structured type with each element given a label, with the\n    effect that we end up with a 1-D array of structured types that can be\n    treated in the same way as any other 1-D array. The result is that the\n    flattened subarrays are sorted in lexicographic order starting with the\n    first element.\n\n    This function differs from the original `numpy.unique\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.unique.html>`_ in\n    the following aspects:\n\n    - Only support ndarray as input.\n    - Object arrays or structured arrays are not supported.\n\n    Examples\n    --------\n    >>> np.unique(np.array([1, 1, 2, 2, 3, 3]))\n    array([1., 2., 3.])\n    >>> a = np.array([[1, 1], [2, 3]])\n    >>> np.unique(a)\n    array([1., 2., 3.])\n\n    Return the unique rows of a 2D array\n\n    >>> a = np.array([[1, 0, 0], [1, 0, 0], [2, 3, 4]])\n    >>> np.unique(a, axis=0)\n    array([[1., 0., 0.],\n           [2., 3., 4.]])\n\n    Return the indices of the original array that give the unique values:\n\n    >>> a = np.array([1, 2, 6, 4, 2, 3, 2])\n    >>> u, indices = np.unique(a, return_index=True)\n    >>> u\n    array([1., 2., 3., 4., 6.])\n    >>> indices\n    array([0, 1, 5, 3, 2], dtype=int64)\n    >>> a[indices]\n    array([1., 2., 3., 4., 6.])\n\n    Reconstruct the input array from the unique values:\n\n    >>> a = np.array([1, 2, 6, 4, 2, 3, 2])\n    >>> u, indices = np.unique(a, return_inverse=True)\n    >>> u\n    array([1., 2., 3., 4., 6.])\n    >>> indices\n    array([0, 1, 4, 3, 1, 2, 1], dtype=int64)\n    >>> u[indices]\n    array([1., 2., 6., 4., 2., 3., 2.])\n    \"\"\"\n    ret = list(_api_internal.unique(ar, return_index, return_inverse, return_counts, axis))\n    return ret[0] if len(ret) == 1 else tuple(ret)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef add(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Add arguments element-wise.\n\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalar values\n        The arrays to be added. If x1.shape != x2.shape, they must be broadcastable to\n        a common shape (which may be the shape of one or the other).\n\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    add : ndarray or scalar\n        The sum of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars.\n\n    Notes\n    -----\n    This operator now supports automatic type promotion. The resulting type will be determined\n    according to the following rules:\n        * If both inputs are of floating number types, the output is the more precise type.\n        * If only one of the inputs is floating number type, the result is that type.\n        * If both inputs are of integer types (including boolean), not supported yet.\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.add(x1, x2, out=out)\n    return _api_internal.add(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef subtract(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Subtract arguments element-wise.\n\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalar values\n        The arrays to be subtracted from each other. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which may be the shape\n        of one or the other).\n\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    subtract : ndarray or scalar\n        The difference of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars.\n\n    Notes\n    -----\n    This operator now supports automatic type promotion. The resulting type will be determined\n    according to the following rules:\n        * If both inputs are of floating number types, the output is the more precise type.\n        * If only one of the inputs is floating number type, the result is that type.\n        * If both inputs are of integer types (including boolean), not supported yet.\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.subtract(x1, x2, out=out)\n    return _api_internal.subtract(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef multiply(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Multiply arguments element-wise.\n\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalar values\n        The arrays to be multiplied. If x1.shape != x2.shape, they must be broadcastable to\n        a common shape (which may be the shape of one or the other).\n\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        The multiplication of x1 and x2, element-wise. This is a scalar if both x1 and x2\n        are scalars.\n\n    Notes\n    -----\n    This operator now supports automatic type promotion. The resulting type will be determined\n    according to the following rules:\n        * If both inputs are of floating number types, the output is the more precise type.\n        * If only one of the inputs is floating number type, the result is that type.\n        * If both inputs are of integer types (including boolean), not supported yet.\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.multiply(x1, x2, out=out)\n    return _api_internal.multiply(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef divide(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns a true division of the inputs, element-wise.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Dividend array.\n\n    x2 : ndarray or scalar\n        Divisor array.\n\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        This is a scalar if both x1 and x2 are scalars.\n\n    Notes\n    -----\n    This operator now supports automatic type promotion. The resulting type will be determined\n    according to the following rules:\n        * If both inputs are of floating number types, the output is the more precise type.\n        * If only one of the inputs is floating number type, the result is that type.\n        * If both inputs are of integer types (including boolean), the output is of default dtype.\n          - When npx.is_np_default_dtype() returns False, default dtype is float32;\n          - When npx.is_np_default_dtype() returns True, default dtype is float64.\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.divide(x1, x2, out=out)\n    return _api_internal.true_divide(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef true_divide(x1, x2, out=None):\n    \"\"\"Returns a true division of the inputs, element-wise.\n\n    Instead of the Python traditional 'floor division', this returns a true\n    division.  True division adjusts the output type to present the best\n    answer, regardless of input types.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Dividend array.\n\n    x2 : ndarray or scalar\n        Divisor array.\n\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        This is a scalar if both x1 and x2 are scalars.\n\n    Notes\n    -----\n    This operator now supports automatic type promotion. The resulting type will be determined\n    according to the following rules:\n        * If both inputs are of floating number types, the output is the more precise type.\n        * If only one of the inputs is floating number type, the result is that type.\n        * If both inputs are of integer types (including boolean), the output is of default dtype.\n          - When npx.is_np_default_dtype() returns False, default dtype is float32;\n          - When npx.is_np_default_dtype() returns True, default dtype is float64.\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.true_divide(x1, x2, out=out)\n    return _api_internal.true_divide(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef floor_divide(x1, x2, out=None):\n    \"\"\"Return the largest integer smaller or equal to the division of the inputs.\n    It is equivalent to the Python // operator and pairs with the Python % (remainder),\n    function so that a = a % b + b * (a // b) up to roundoff.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Dividend array.\n    x2 : ndarray or scalar\n        Divisor array.\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        This is a scalar if both x1 and x2 are scalars.\n\n    .. note::\n\n       This operator now supports automatic type promotion. The resulting type will be determined\n       according to the following rules:\n\n       * If both inputs are of floating number types, the output is the more precise type.\n       * If only one of the inputs is floating number type, the result is that type.\n       * If both inputs are of integer types (including boolean), the output is the more\n       precise type\n\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.floor_divide(x1, x2, out=out)\n    return _api_internal.floor_divide(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef mod(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Return element-wise remainder of division.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Dividend array.\n\n    x2 : ndarray or scalar\n        Divisor array.\n\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        This is a scalar if both x1 and x2 are scalars.\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.mod(x1, x2, out=out)\n    return _api_internal.mod(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef fmod(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Return element-wise remainder of division.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Dividend array.\n\n    x2 : ndarray or scalar\n        Divisor array.\n\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        This is a scalar if both x1 and x2 are scalars.\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        _np.fmod(x1, x2, out=out)\n    return _api_internal.fmod(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef delete(arr, obj, axis=None):\n    \"\"\"\n    Return a new array with sub-arrays along an axis deleted. For a one\n    dimensional array, this returns those entries not returned by\n    `arr[obj]`.\n\n    Parameters\n    ----------\n    arr : ndarray\n      Input array.\n    obj : slice, int or ndarray of ints\n      Indicate indices of sub-arrays to remove along the specified axis.\n    axis : int, optional\n      The axis along which to delete the subarray defined by `obj`.\n      If `axis` is None, `obj` is applied to the flattened array.\n\n    Returns\n    -------\n    out : ndarray\n        A copy of `arr` with the elements specified by `obj` removed. Note\n        that `delete` does not occur in-place. If `axis` is None, `out` is\n        a flattened array.\n\n    Examples\n    --------\n    >>> arr = np.array([[1,2,3,4], [5,6,7,8], [9,10,11,12]])\n    >>> arr\n    array([[ 1.,  2.,  3.,  4.],\n           [ 5.,  6.,  7.,  8.],\n           [ 9., 10., 11., 12.]])\n\n    >>> np.delete(arr, 1, 0)\n    array([[ 1.,  2.,  3.,  4.],\n           [ 9., 10., 11., 12.]])\n\n    >>> np.delete(arr, slice(None, None, 2), 1)\n    array([[ 2.,  4.],\n           [ 6.,  8.],\n           [10., 12.]])\n\n    >>> np.delete(arr, np.array([1,3,5]), None)\n    array([ 1.,  3.,  5.,  7.,  8.,  9., 10., 11., 12.])\n    >>> np.delete(arr, np.array([1,1,5]), None)\n    array([ 1.,  3.,  4.,  5.,  7.,  8.,  9., 10., 11., 12.])\n    \"\"\"\n    if not isinstance(arr, NDArray):\n        raise TypeError(\"'arr' can not support type {}\".format(str(type(arr))))\n    if isinstance(obj, slice):\n        start = obj.start\n        stop = obj.stop\n        step = 1 if obj.step is None else obj.step\n        return _api_internal.delete(arr, start, stop, step, axis)\n    elif isinstance(obj, integer_types):\n        return _api_internal.delete(arr, obj, axis)\n    elif isinstance(obj, NDArray):\n        return _api_internal.delete(arr, obj, axis)\n    else:\n        raise TypeError(\"'obj' can not support type {}\".format(str(type(obj))))\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef matmul(a, b, out=None):\n    \"\"\"\n    Matrix product of two arrays.\n\n    Parameters\n    ----------\n    a, b : ndarray\n        Input arrays, scalars not allowed.\n    out : ndarray, optional\n        A location into which the result is stored.\n        If provided, it must have a shape that matches the signature (n,k),(k,m)->(n,m).\n        If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray\n        The matrix product of the inputs.\n        This is a scalar only when both x1, x2 are 1-d vectors.\n\n    Raises\n    ------\n    MXNetError\n        If the last dimension of a is not the same size as the second-to-last dimension of b.\n        If a scalar value is passed in.\n\n    See Also\n    --------\n    tensordot :\n        Sum products over arbitrary axes.\n    dot :\n        alternative matrix product with different broadcasting rules.\n    einsum :\n        Einstein summation convention.\n\n    Notes\n    -----\n    The behavior depends on the arguments in the following way.\n\n    - If both arguments are 2-D they are multiplied like conventional matrices.\n    - If either argument is N-D, N > 2, it is treated as a stack of matrices\n      residing in the last two indexes and broadcast accordingly.\n    - If the first argument is 1-D, it is promoted to a matrix by prepending\n      a 1 to its dimensions. After matrix multiplication the prepended 1 is removed.\n    - If the second argument is 1-D, it is promoted to a matrix by appending a 1\n      to its dimensions. After matrix multiplication the appended 1 is removed.\n\n    matmul differs from dot in two important ways:\n\n    - Multiplication by scalars is not allowed, use multiply instead.\n    - Stacks of matrices are broadcast together as if the matrices were elements,\n    respecting the signature (n,k),(k,m)->(n,m):\n    >>> a = np.ones([9, 5, 7, 4])\n    >>> c = np.ones([9, 5, 4, 3])\n    >>> np.dot(a, c).shape\n    (9, 5, 7, 9, 5, 3)\n    >>> np.matmul(a, c).shape\n    (9, 5, 7, 3)\n    >>> # n is 7, k is 4, m is 3\n\n    Examples\n    --------\n    For 2-D arrays it is the matrix product:\n    >>> a = np.array([[1, 0],\n    ...               [0, 1]])\n    >>> b = np.array([[4, 1],\n    ...               [2, 2]])\n    >>> np.matmul(a, b)\n    array([[4., 1.],\n           [2., 2.]])\n\n    For 2-D mixed with 1-D, the result is the usual.\n    >>> a = np.array([[1, 0],\n    ...               [0, 1]])\n    >>> b = np.array([1, 2])\n    >>> np.matmul(a, b)\n    array([1., 2.])\n    >>> np.matmul(b, a)\n    array([1., 2.])\n\n    Broadcasting is conventional for stacks of arrays\n    >>> a = np.arange(2 * 2 * 4).reshape((2, 2, 4))\n    >>> b = np.arange(2 * 2 * 4).reshape((2, 4, 2))\n    >>> np.matmul(a, b).shape\n    (2, 2, 2)\n    >>> np.matmul(a, b)[0, 1, 1]\n    array(98.)\n    >>> sum(a[0, 1, :] * b[0, :, 1])\n    array(98.)\n\n    Scalar multiplication raises an error.\n    >>> np.matmul([1, 2], 3)\n    Traceback (most recent call last):\n    ...\n    mxnet.base.MXNetError: ... : Multiplication by scalars is not allowed.\n    \"\"\"\n    return _api_internal.matmul(a, b, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef remainder(x1, x2, out=None):\n    \"\"\"\n    Return element-wise remainder of division.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Dividend array.\n\n    x2 : ndarray or scalar\n        Divisor array.\n\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        This is a scalar if both x1 and x2 are scalars.\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        _np.mod(x1, x2, out=out)\n    return _api_internal.mod(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef power(x1, x2, out=None, **kwargs):\n    \"\"\"\n    First array elements raised to powers from second array, element-wise.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        The bases.\n\n    x2 : ndarray or scalar\n        The exponent.\n\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        The bases in x1 raised to the exponents in x2.\n        This is a scalar if both x1 and x2 are scalars.\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.power(x1, x2, out=out)\n    return _api_internal.power(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef all(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Test whether all array elements along a given axis evaluate to True.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array or object that can be converted to an array.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which a logical AND reduction is performed.\n        The default (axis = None) is to perform a logical AND over\n        all the dimensions of the input array.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in\n        the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the input array.\n    out : ndarray, optional\n        Alternate output array in which to place the result. It must have\n        the same shape as the expected output and its type is preserved\n\n    Returns\n    --------\n    all : ndarray, bool\n        A new boolean or array is returned unless out is specified,\n        in which case a reference to out is returned.\n\n    Examples:\n    ---------\n    >>> np.all([[True,False],[True,True]])\n    False\n\n    >>> np.all([[True,False],[True,True]], axis=0)\n    array([ True, False])\n\n    >>> np.all([-1, 4, 5])\n    True\n\n    >>> np.all([1.0, np.nan])\n    True\n\n    >>> o=np.array(False)\n    >>> z=np.all([-1, 4, 5], out=o)\n    >>> id(z), id(o), z\n    (28293632, 28293632, array(True)) # may vary\n    \"\"\"\n    return _api_internal.all(a, axis, keepdims, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef any(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Test whether any array element along a given axis evaluates to True.\n    Returns single boolean unless axis is not None\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array or object that can be converted to an array.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which a logical AND reduction is performed.\n        The default (axis = None) is to perform a logical AND over\n        all the dimensions of the input array.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in\n        the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the input array.\n    out : ndarray, optional\n        Alternate output array in which to place the result. It must have\n        the same shape as the expected output and its type is preserved\n\n    Returns\n    --------\n    any : bool or ndarray\n        A new boolean or ndarray is returned unless out is specified,\n        in which case a reference to out is returned.\n\n    Examples:\n    ---------\n    >>> np.any([[True, False], [True, True]])\n    True\n\n    >>> np.any([[True, False], [False, False]], axis=0)\n    array([ True, False])\n\n    >>> np.any([-1, 0, 5])\n    True\n\n    >>> np.any(np.nan)\n    True\n\n    >>> o=np.array(False)\n    >>> z=np.any([-1, 4, 5], out=o)\n    >>> z, o\n    (array(True), array(True))\n    >>> # Check now that z is a reference to o\n    >>> z is o\n    True\n    >>> id(z), id(o) # identity of z and o              # doctest: +SKIP\n    (191614240, 191614240)\n    \"\"\"\n    return _api_internal.any(a, axis, keepdims, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef argsort(a, axis=-1, descending=False, stable=True):\n    \"\"\"\n    Returns the indices that sort an array `x` along a specified axis.\n\n    Notes\n    -----\n    `argsort` is a standard API in\n    https://data-apis.org/array-api/latest/API_specification/generated/signatures.sorting_functions.argsort.html\n    instead of an official NumPy operator.\n\n    Parameters\n    ----------\n    a : ndarray\n        Array to sort.\n    axis : int or None, optional\n        Axis along which to sort.  The default is -1 (the last axis). If None,\n        the flattened array is used.\n    descending : bool, optional\n        sort order. If `True`, the returned indices sort x in descending order (by value).\n        If `False`, the returned indices sort x in ascending order (by value).Default: False.\n    stable : bool, optional\n        sort stability. If `True`, the returned indices must maintain the relative order\n        of x values which compare as equal. If `False`, the returned indices may or may not\n        maintain the relative order of x values which compare as equal. Default: True.\n\n    Returns\n    -------\n    index_array : ndarray, int\n        Array of indices that sort `a` along the specified `axis`.\n        If `a` is one-dimensional, ``a[index_array]`` yields a sorted `a`.\n        More generally, ``np.take_along_axis(a, index_array, axis=axis)``\n        always yields the sorted `a`, irrespective of dimensionality.\n\n    Notes\n    -----\n    This operator does not support different sorting algorithms.\n\n    Examples\n    --------\n    One dimensional array:\n\n    >>> x = np.array([3, 1, 2])\n    >>> np.argsort(x)\n    array([1, 2, 0])\n\n    Two-dimensional array:\n\n    >>> x = np.array([[0, 3], [2, 2]])\n    >>> x\n    array([[0, 3],\n           [2, 2]])\n    >>> ind = np.argsort(x, axis=0)  # sorts along first axis (down)\n    >>> ind\n    array([[0, 1],\n           [1, 0]])\n    >>> np.take_along_axis(x, ind, axis=0)  # same as np.sort(x, axis=0)\n    array([[0, 2],\n           [2, 3]])\n    >>> ind = np.argsort(x, axis=1)  # sorts along last axis (across)\n    >>> ind\n    array([[0, 1],\n           [0, 1]])\n    >>> np.take_along_axis(x, ind, axis=1)  # same as np.sort(x, axis=1)\n    array([[0, 3],\n           [2, 2]])\n\n    Indices of the sorted elements of a N-dimensional array:\n\n    >>> ind = np.unravel_index(np.argsort(x, axis=None), x.shape)\n    >>> ind\n    (array([0, 1, 1, 0]), array([0, 0, 1, 1]))\n    >>> x[ind]  # same as np.sort(x, axis=None)\n    array([0, 2, 2, 3])\n    \"\"\"\n    return _api_internal.argsort(a, axis, not descending, 'int64')\n\n\n@set_module('mxnet.ndarray.numpy')\ndef sort(a, axis=-1, descending=False, stable=True):\n    \"\"\"\n    Return a sorted copy of an array.\n\n    Notes\n    -----\n    `sort` is a standard API in\n    https://data-apis.org/array-api/latest/API_specification/generated/signatures.sorting_functions.sort.html\n    instead of an official NumPy operator.\n\n    Parameters\n    ----------\n    a : ndarray\n        Array to sort.\n    axis : int or None, optional\n        Axis along which to sort.  The default is -1 (the last axis). If None,\n        the flattened array is used.\n    descending : bool, optional\n        sort order. If `True`, the returned indices sort x in descending order (by value).\n        If `False`, the returned indices sort x in ascending order (by value).Default: False.\n    stable : bool, optional\n        sort stability. If `True`, the returned indices must maintain the relative order\n        of x values which compare as equal. If `False`, the returned indices may or may not\n        maintain the relative order of x values which compare as equal. Default: True.\n\n    Returns\n    -------\n    sorted_array : ndarray\n        Array of the same type and shape as `a`.\n\n    Notes\n    -----\n    This operator does not support different sorting algorithms.\n\n    Examples\n    --------\n    >>> a = np.array([[1,4],[3,1]])\n    >>> np.sort(a)                # sort along the last axis\n    array([[1, 4],\n           [1, 3]])\n    >>> np.sort(a, axis=None)     # sort the flattened array\n    array([1, 1, 3, 4])\n    >>> np.sort(a, axis=0)        # sort along the first axis\n    array([[1, 1],\n           [3, 4]])\n    \"\"\"\n    return _api_internal.sort(a, axis, not descending)\n\n@set_module('mxnet.ndarray.numpy')\ndef dot(a, b, out=None):\n    \"\"\"\n    Dot product of two arrays. Specifically,\n\n    - If both `a` and `b` are 1-D arrays, it is inner product of vectors\n\n    - If both `a` and `b` are 2-D arrays, it is matrix multiplication,\n\n    - If either `a` or `b` is 0-D (scalar), it is equivalent to :func:`multiply`\n      and using ``np.multiply(a, b)`` or ``a * b`` is preferred.\n\n    - If `a` is an N-D array and `b` is a 1-D array, it is a sum product over\n      the last axis of `a` and `b`.\n\n    - If `a` is an N-D array and `b` is a 2-D array, it is a\n      sum product over the last axis of `a` and the second-to-last axis of `b`::\n\n        dot(a, b)[i,j,k] = sum(a[i,j,:] * b[:,k])\n\n    Parameters\n    ----------\n    a : ndarray\n        First argument.\n    b : ndarray\n        Second argument.\n\n    out : ndarray, optional\n        Output argument. It must have the same shape and type as the expected output.\n\n    Returns\n    -------\n    output : ndarray\n        Returns the dot product of `a` and `b`.  If `a` and `b` are both\n        scalars or both 1-D arrays then a scalar is returned; otherwise\n        an array is returned.\n        If `out` is given, then it is returned\n\n    Examples\n    --------\n    >>> a = np.array(3)\n    >>> b = np.array(4)\n    >>> np.dot(a, b)\n    array(12.)\n\n    For 2-D arrays it is the matrix product:\n\n    >>> a = np.array([[1, 0], [0, 1]])\n    >>> b = np.array([[4, 1], [2, 2]])\n    >>> np.dot(a, b)\n    array([[4., 1.],\n           [2., 2.]])\n\n    >>> a = np.arange(3*4*5*6).reshape((3,4,5,6))\n    >>> b = np.arange(5*6)[::-1].reshape((6,5))\n    >>> np.dot(a, b)[2,3,2,2]\n    array(29884.)\n    >>> np.sum(a[2,3,2,:] * b[:,2])\n    array(29884.)\n    \"\"\"\n    return _api_internal.dot(a, b, out)\n\n@set_module('mxnet.ndarray.numpy')\ndef tensordot(a, b, axes=2):\n    r\"\"\"\n    tensordot(a, b, axes=2)\n    Compute tensor dot product along specified axes for arrays >= 1-D.\n    Given two tensors (arrays of dimension greater than or equal to one),\n    `a` and `b`, and an ndarray object containing two ndarray\n    objects, ``(a_axes, b_axes)``, sum the products of `a`'s and `b`'s\n    elements (components) over the axes specified by ``a_axes`` and\n    ``b_axes``. The third argument can be a single non-negative\n    integer_like scalar, ``N``; if it is such, then the last ``N``\n    dimensions of `a` and the first ``N`` dimensions of `b` are summed\n    over.\n    Parameters\n    ----------\n    a, b : ndarray, len(shape) >= 1\n        Tensors to \"dot\".\n    axes : int or (2,) ndarray\n        * integer_like\n        If an int N, sum over the last N axes of `a` and the first N axes\n        of `b` in order. The sizes of the corresponding axes must match.\n        * (2,) ndarray\n        Or, a list of axes to be summed over, first sequence applying to `a`,\n        second to `b`. Both elements ndarray must be of the same length.\n    See Also\n    --------\n    dot, einsum\n    Notes\n    -----\n    Three common use cases are:\n        * ``axes = 0`` : tensor product :math:`a\\otimes b`\n        * ``axes = 1`` : tensor dot product :math:`a\\cdot b`\n        * ``axes = 2`` : (default) tensor double contraction :math:`a:b`\n    When `axes` is integer_like, the sequence for evaluation will be: first\n    the -Nth axis in `a` and 0th axis in `b`, and the -1th axis in `a` and\n    Nth axis in `b` last.\n    When there is more than one axis to sum over - and they are not the last\n    (first) axes of `a` (`b`) - the argument `axes` should consist of\n    two sequences of the same length, with the first axis to sum over given\n    first in both sequences, the second axis second, and so forth.\n    Examples\n    --------\n    >>> a = np.arange(60.).reshape(3,4,5)\n    >>> b = np.arange(24.).reshape(4,3,2)\n    >>> c = np.tensordot(a,b, axes=([1,0],[0,1]))\n    >>> c.shape\n    (5, 2)\n    >>> c\n    array([[ 4400.,  4730.],\n           [ 4532.,  4874.],\n           [ 4664.,  5018.],\n           [ 4796.,  5162.],\n           [ 4928.,  5306.]])\n    \"\"\"\n    return _api_internal.tensordot(a, b, axes)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef histogram(a, bins=10, range=None, normed=None, weights=None, density=None):  # pylint: disable=too-many-arguments\n    \"\"\"\n    Compute the histogram of a set of data.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data. The histogram is computed over the flattened array.\n    bins : int or NDArray\n        If `bins` is an int, it defines the number of equal-width\n        bins in the given range (10, by default). If `bins` is a\n        sequence, it defines a monotonically increasing array of bin edges,\n        including the rightmost edge, allowing for non-uniform bin widths.\n        .. versionadded:: 1.11.0\n        If `bins` is a string, it defines the method used to calculate the\n        optimal bin width, as defined by `histogram_bin_edges`.\n    range : (float, float)\n        The lower and upper range of the bins. Required when `bins` is an integer.\n        Values outside the range are ignored. The first element of the range must\n        be less than or equal to the second.\n    normed : bool, optional\n        Not supported yet, coming soon.\n    weights : array_like, optional\n        Not supported yet, coming soon.\n    density : bool, optional\n        Not supported yet, coming soon.\n    \"\"\"\n    if normed is True:\n        raise NotImplementedError(\"normed is not supported yet...\")\n    if weights is not None:\n        raise NotImplementedError(\"weights is not supported yet...\")\n    if density is True:\n        raise NotImplementedError(\"density is not supported yet...\")\n    if isinstance(bins, numeric_types):\n        if range is None:\n            raise NotImplementedError(\"automatic range is not supported yet...\")\n        return tuple(_api_internal.histogram(a, None, bins, range))\n    if isinstance(bins, (list, tuple)):\n        raise NotImplementedError(\"array_like bins is not supported yet...\")\n    if isinstance(bins, str):\n        raise NotImplementedError(\"string bins is not supported yet...\")\n    if isinstance(bins, NDArray):\n        return tuple(_api_internal.histogram(a, bins, None, None))\n    raise ValueError(\"np.histogram fails with\", locals())\n\n\n@set_module('mxnet.ndarray.numpy')\ndef eye(N, M=None, k=0, dtype=float, **kwargs):\n    \"\"\"\n    Return a 2-D array with ones on the diagonal and zeros elsewhere.\n\n    Parameters\n    ----------\n    N : int\n        Number of rows in the output.\n    M : int, optional\n        Number of columns in the output. If None, defaults to N.\n    k : int, optional\n        Index of the diagonal: 0 (the default) refers to the main diagonal,\n        a positive value refers to an upper diagonal,\n        and a negative value to a lower diagonal.\n    dtype : data-type, optional\n        Data-type of the returned array.\n        - When npx.is_np_default_dtype() returns False, default dtype is float32;\n        - When npx.is_np_default_dtype() returns True, default dtype is float64.\n\n    Returns\n    -------\n    I : ndarray of shape (N,M)\n        An array where all elements are equal to zero,\n        except for the k-th diagonal, whose values are equal to one.\n    \"\"\"\n    _sanity_check_params('eye', ['order'], kwargs)\n    device = kwargs.pop('device', current_device())\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if dtype is None or dtype is float:\n        dtype = _np.float64 if is_np_default_dtype() else _np.float32\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n\n    # To avoid overflow errors, map large positive k values to the just-out-of-range \"num_columns\" value\n    k = minimum(k, M if M is not None else N)\n    # Similarly, map large negative k values to the just-out-of-range \"-num_rows\" value\n    k = maximum(k, -N)\n    return _api_internal.eye(N, M, int(k), device, dtype)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis=0, device=None):  # pylint: disable=too-many-arguments\n    r\"\"\"\n    Return evenly spaced numbers over a specified interval.\n    Returns num evenly spaced samples, calculated over the interval [start, stop].\n    The endpoint of the interval can optionally be excluded.\n\n    Parameters\n    ----------\n    start : int or float\n        The starting value of the sequence.\n    stop : int or float\n        The end value of the sequence, unless endpoint is set to False. In\n        that case, the sequence consists of all but the last of num + 1\n        evenly spaced samples, so that stop is excluded. Note that the step\n        size changes when endpoint is False.\n    num : int, optional\n        Number of samples to generate. Default is 50. Must be non-negative.\n    endpoint : bool, optional\n        If True, stop is the last sample. Otherwise, it is not included.\n        Default is True.\n    retstep : bool, optional\n        If True, return (samples, step), where step is the spacing between samples.\n    dtype : dtype, optional\n        The type of the output array. If dtype is not given, infer the data\n        type from the other input arguments.\n    axis : int, optional\n        The axis in the result to store the samples. Relevant only if start or\n        stop are array-like. By default (0), the samples will be along a new\n        axis inserted at the beginning. Use -1 to get an axis at the end.\n\n    Returns\n    -------\n    samples : ndarray\n        There are num equally spaced samples in the closed interval\n        `[start, stop]` or the half-open interval `[start, stop)`\n        (depending on whether endpoint is True or False).\n    step : float, optional\n        Only returned if retstep is True\n        Size of spacing between samples.\n\n\n    See Also\n    --------\n    arange : Similar to `linspace`, but uses a step size (instead of the\n             number of samples).\n\n    Examples\n    --------\n    >>> np.linspace(2.0, 3.0, num=5)\n    array([2.  , 2.25, 2.5 , 2.75, 3.  ])\n    >>> np.linspace(2.0, 3.0, num=5, endpoint=False)\n    array([2. , 2.2, 2.4, 2.6, 2.8])\n    >>> np.linspace(2.0, 3.0, num=5, retstep=True)\n    (array([2.  , 2.25, 2.5 , 2.75, 3.  ]), 0.25)\n\n    Graphical illustration:\n\n    >>> import matplotlib.pyplot as plt\n    >>> N = 8\n    >>> y = np.zeros(N)\n    >>> x1 = np.linspace(0, 10, N, endpoint=True)\n    >>> x2 = np.linspace(0, 10, N, endpoint=False)\n    >>> plt.plot(x1.asnumpy(), y.asnumpy(), 'o')\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.plot(x2.asnumpy(), (y + 0.5).asnumpy(), 'o')\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.ylim([-0.5, 1])\n    (-0.5, 1)\n    >>> plt.show()\n\n    Notes\n    -----\n\n    This function differs from the original `numpy.linspace\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linspace.html>`_ in\n    the following aspects:\n\n    - `start` and `stop` do not support list, numpy ndarray and mxnet ndarray\n    - axis could only be 0\n    - There could be an additional `device` argument to specify the device, e.g. the i-th\n      GPU.\n    \"\"\"\n    if isinstance(start, (list, _np.ndarray, NDArray)) or \\\n       isinstance(stop, (list, _np.ndarray, NDArray)):\n        raise NotImplementedError('start and stop only support int')\n    if axis != 0:\n        raise NotImplementedError(\"the function only support axis 0\")\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    if dtype is None:\n        dtype = _np.float64 if is_np_default_dtype() else _np.float32\n    if retstep:\n        step = (stop - start) / (num - int(endpoint))\n        return _api_internal.linspace(start, stop, num, endpoint, device, dtype), step\n    else:\n        return _api_internal.linspace(start, stop, num, endpoint, device, dtype)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None, axis=0, device=None):  # pylint: disable=too-many-arguments\n    r\"\"\"Return numbers spaced evenly on a log scale.\n\n    In linear space, the sequence starts at ``base ** start``\n    (`base` to the power of `start`) and ends with ``base ** stop``\n    (see `endpoint` below).\n\n        Non-scalar `start` and `stop` are now supported.\n\n    Parameters\n    ----------\n    start : int or float\n        ``base ** start`` is the starting value of the sequence.\n    stop : int or float\n        ``base ** stop`` is the final value of the sequence, unless `endpoint`\n        is False.  In that case, ``num + 1`` values are spaced over the\n        interval in log-space, of which all but the last (a sequence of\n        length `num`) are returned.\n    num : integer, optional\n        Number of samples to generate.  Default is 50.\n    endpoint : boolean, optional\n        If true, `stop` is the last sample. Otherwise, it is not included.\n        Default is True.\n    base : float, optional\n        The base of the log space. The step size between the elements in\n        ``ln(samples) / ln(base)`` (or ``log_base(samples)``) is uniform.\n        Default is 10.0.\n    dtype : dtype\n        The type of the output array.  If `dtype` is not given, infer the data\n        type from the other input arguments.\n    axis : int, optional\n        The axis in the result to store the samples.  Relevant only if start\n        or stop are array-like.  By default (0), the samples will be along a\n        new axis inserted at the beginning. Now, axis only support axis = 0.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    samples : ndarray\n        `num` samples, equally spaced on a log scale.\n\n    See Also\n    --------\n    arange : Similar to linspace, with the step size specified instead of the\n             number of samples. Note that, when used with a float endpoint, the\n             endpoint may or may not be included.\n    linspace : Similar to logspace, but with the samples uniformly distributed\n               in linear space, instead of log space.\n\n    Notes\n    -----\n    Logspace is equivalent to the code. Now wo only support axis = 0.\n\n    >>> y = np.linspace(start, stop, num=num, endpoint=endpoint)\n    ...\n    >>> power(base, y).astype(dtype)\n    ...\n\n    Examples\n    --------\n    >>> np.logspace(2.0, 3.0, num=4)\n    array([ 100.     ,  215.44347,  464.15887, 1000.     ])\n    >>> np.logspace(2.0, 3.0, num=4, endpoint=False)\n    array([100.     , 177.82794, 316.22775, 562.3413 ])\n    >>> np.logspace(2.0, 3.0, num=4, base=2.0)\n    array([4.       , 5.0396843, 6.349604 , 8.       ])\n    >>> np.logspace(2.0, 3.0, num=4, base=2.0, dtype=np.int32)\n    array([4, 5, 6, 8], dtype=int32)\n    >>> np.logspace(2.0, 3.0, num=4, device=npx.gpu(0))\n    array([ 100.     ,  215.44347,  464.15887, 1000.     ], device=gpu(0))\n    \"\"\"\n    if isinstance(start, (list, tuple, _np.ndarray, NDArray)) or \\\n       isinstance(stop, (list, tuple, _np.ndarray, NDArray)):\n        raise NotImplementedError('start and stop only support int and float')\n    if axis != 0:\n        raise NotImplementedError(\"the function only support axis 0\")\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    return _api_internal.logspace(start, stop, num, endpoint, base, device, dtype)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef expand_dims(a, axis):\n    \"\"\"Expand the shape of an array.\n\n    Insert a new axis that will appear at the `axis` position in the expanded\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array.\n    axis : int\n        Position in the expanded axes where the new axis is placed.\n\n    Returns\n    -------\n    res : ndarray\n        Output array. The number of dimensions is one greater than that of\n        the input array.\n    \"\"\"\n    return _api_internal.expand_dims(a, axis)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef gcd(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns the greatest common divisor of ``|x1|`` and ``|x2|``\n\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalar values\n        The arrays for computing greatest common divisor. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which may be the shape of\n        one or the other).\n\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The greatest common divisor of the absolute value of the inputs\n        This is a scalar if both `x1` and `x2` are scalars.\n\n    See Also\n    --------\n    lcm : The lowest common multiple\n\n    Examples\n    --------\n    >>> np.gcd(12, 20)\n    4\n    >>> np.gcd(np.arange(6, dtype=int), 20)\n    array([20,  1,  2,  1,  4,  5], dtype=int64)\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.gcd(x1, x2, out=out)\n    return _api_internal.gcd(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef lcm(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns the lowest common multiple of ``|x1|`` and ``|x2|``\n\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalar values\n        The arrays for computing lowest common multiple. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which may be the shape of\n        one or the other).\n\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The lowest common multiple of the absolute value of the inputs\n        This is a scalar if both `x1` and `x2` are scalars.\n\n    See Also\n    --------\n    gcd : The greatest common divisor\n\n    Examples\n    --------\n    >>> np.lcm(12, 20)\n    60\n    >>> np.lcm(np.arange(6, dtype=int), 20)\n    array([ 0, 20, 20, 60, 20, 20], dtype=int64)\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.lcm(x1, x2, out=out)\n    return _api_internal.lcm(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef tril(m, k=0):\n    r\"\"\"\n    Lower triangle of an array.\n\n    Return a copy of an array with elements above the `k`-th diagonal zeroed.\n\n    Parameters\n    ----------\n    m : ndarray, shape (M, N)\n        Input array.\n    k : int, optional\n        Diagonal above which to zero elements.  `k = 0` (the default) is the\n        main diagonal, `k < 0` is below it and `k > 0` is above.\n\n    Returns\n    -------\n    tril : ndarray, shape (M, N)\n        Lower triangle of `m`, of same shape and data-type as `m`.\n\n    See Also\n    --------\n    triu : same thing, only for the upper triangle\n\n    Examples\n    --------\n    >>> a = np.array([[1,2,3],[4,5,6],[7,8,9],[10,11,12]])\n    >>> np.tril(a, -1)\n    array([[ 0.,  0.,  0.],\n           [ 4.,  0.,  0.],\n           [ 7.,  8.,  0.],\n           [10., 11., 12.]])\n    \"\"\"\n    return _api_internal.tril(m, k)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef triu(m, k=0):\n    r\"\"\"\n    Upper triangle of an array.\n\n    Return a copy of a matrix with the elements below the `k`-th diagonal\n    zeroed.\n\n    Please refer to the documentation for `tril` for further details.\n\n    See Also\n    --------\n    tril : lower triangle of an array\n\n    Examples\n    --------\n    >>> np.triu(np.array([[1,2,3],[4,5,6],[7,8,9],[10,11,12]]), -1)\n    array([[ 1,  2,  3],\n           [ 4,  5,  6],\n           [ 0,  8,  9],\n           [ 0,  0, 12]])\n    \"\"\"\n    return _api_internal.triu(m, k)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef trace(a, offset=0, axis1=0, axis2=1, out=None):\n    \"\"\"\n    Return the sum along diagonals of the array.\n    If `a` is 2-D, the sum along its diagonal with the given offset\n    is returned, i.e., the sum of elements ``a[i,i+offset]`` for all i.\n    If `a` has more than two dimensions, then the axes specified by axis1 and\n    axis2 are used to determine the 2-D sub-arrays whose traces are returned.\n    The shape of the resulting array is the same as that of `a` with `axis1`\n    and `axis2` removed.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array, from which the diagonals are taken.\n    offset : int, optional\n        Offset of the diagonal from the main diagonal. Can be both positive\n        and negative. Defaults to 0.\n    axis1, axis2 : int, optional\n        Axes to be used as the first and second axis of the 2-D sub-arrays\n        from which the diagonals should be taken. Defaults are the first two\n        axes of `a`.\n    out : ndarray, optional\n        Array into which the output is placed. It must be of the right shape\n        and right type to hold the output.\n\n    Returns\n    -------\n    sum_along_diagonals : ndarray\n        If `a` is 2-D, the sum along the diagonal is returned.  If `a` has\n        larger dimensions, then an array of sums along diagonals is returned.\n\n    Examples\n    --------\n    >>> a = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]])\n    >>> np.trace(a)\n    array(3.)\n    >>> a = np.arange(8).reshape((2, 2, 2))\n    >>> np.trace(a)\n    array([6., 8.])\n    >>> a = np.arange(24).reshape((2, 2, 2, 3))\n    >>> np.trace(a).shape\n    (2, 3)\n    \"\"\"\n    return _api_internal.trace(a, offset, axis1, axis2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef tri(N, M=None, k=0, dtype=None, device=None):\n    r\"\"\"\n    An array with ones at and below the given diagonal and zeros elsewhere.\n\n    Parameters\n    ----------\n    N : int\n        Number of rows in the array.\n    M : int, optional\n        Number of columns in the array.\n        By default, `M` is taken equal to `N`.\n    k : int, optional\n        The sub-diagonal at and below which the array is filled.\n        `k` = 0 is the main diagonal, while `k` < 0 is below it,\n        and `k` > 0 is above.  The default is 0.\n    dtype : dtype, optional\n        Data type of the returned array.  The default is float.\n\n    Returns\n    -------\n    tri : ndarray of shape (N, M)\n        Array with its lower triangle filled with ones and zero elsewhere;\n        in other words ``T[i,j] == 1`` for ``i <= j + k``, 0 otherwise.\n\n    Examples\n    --------\n    >>> np.tri(3, 5, 2, dtype=int)\n    array([[1, 1, 1, 0, 0],\n           [1, 1, 1, 1, 0],\n           [1, 1, 1, 1, 1]])\n\n    >>> np.tri(3, 5, -1)\n    array([[0.,  0.,  0.,  0.,  0.],\n           [1.,  0.,  0.,  0.,  0.],\n           [1.,  1.,  0.,  0.,  0.]])\n    \"\"\"\n    if device is None:\n        device = str(current_device())\n    return _api_internal.tri(N, M, k, dtype, device)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef triu_indices(n, k=0, m=None, device=None):\n    r\"\"\"\n    Return the indices for the upper-triangle of an (n, m) array.\n\n    Parameters\n    ----------\n    n : int\n        The size of the arrays for which the returned indices will\n        be valid.\n    k : int, optional\n        Diagonal offset (see `triu` for details).\n    m : int, optional\n        .. versionadded:: 1.9.0\n\n        The column dimension of the arrays for which the returned\n        arrays will be valid.\n        By default `m` is taken equal to `n`.\n\n\n    Returns\n    -------\n    inds : tuple, shape(2) of ndarrays, shape(`n`)\n        The indices for the triangle. The returned tuple contains two arrays,\n        each with the indices along one dimension of the array.  Can be used\n        to slice a ndarray of shape(`n`, `n`).\n\n    See also\n    --------\n    tril_indices : similar function, for lower-triangular.\n    mask_indices : generic function accepting an arbitrary mask function.\n    triu, tril\n\n    Examples\n    --------\n    Compute two different sets of indices to access 4x4 arrays, one for the\n    upper triangular part starting at the main diagonal, and one starting two\n    diagonals further right:\n\n    >>> iu1 = np.triu_indices(4)\n    >>> iu2 = np.triu_indices(4, 2)\n\n    Here is how they can be used with a sample array:\n\n    >>> a = np.arange(16).reshape(4, 4)\n    >>> a\n    array([[ 0,  1,  2,  3],\n           [ 4,  5,  6,  7],\n           [ 8,  9, 10, 11],\n           [12, 13, 14, 15]])\n\n    Both for indexing:\n\n    >>> a[iu1]\n    array([ 0,  1,  2, ..., 10, 11, 15])\n\n    And for assigning values:\n\n    >>> a[iu1] = -1\n    >>> a\n    array([[-1, -1, -1, -1],\n           [ 4, -1, -1, -1],\n           [ 8,  9, -1, -1],\n           [12, 13, 14, -1]])\n\n    These cover only a small part of the whole array (two diagonals right\n    of the main one):\n\n    >>> a[iu2] = -10\n    >>> a\n    array([[ -1,  -1, -10, -10],\n           [  4,  -1,  -1, -10],\n           [  8,   9,  -1,  -1],\n           [ 12,  13,  14,  -1]])\n        \"\"\"\n    return nonzero(~tri(N=n, M=m, k=k-1, dtype=bool, device=device))\n\n\n\n@set_module('mxnet.ndarray.numpy')\ndef triu_indices_from(arr, k=0):\n    \"\"\"\n    Return the indices for the upper-triangle of arr.\n    See `triu_indices` for full details.\n    Parameters\n    ----------\n    arr : ndarray, shape(N, N)\n        The indices will be valid for square arrays.\n    k : int, optional\n        Diagonal offset (see `triu` for details).\n    Returns\n    -------\n    triu_indices_from : tuple, shape(2) of ndarray, shape(N)\n        Indices for the upper-triangle of `arr`.\n    See Also\n    --------\n    triu_indices, triu\n    \"\"\"\n    if arr.ndim != 2:\n        raise ValueError(\"input array must be 2-d\")\n    return triu_indices(arr.shape[-2], k=k, m=arr.shape[-1])\n\n\ndef _unary_func_helper(x, fn_array, fn_scalar, out=None, **kwargs):\n    \"\"\"Helper function for unary operators with kwargs.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input of the unary operator.\n    fn_array : function\n        Function to be called if x is of ``ndarray`` type.\n    fn_scalar : function\n        Function to be called if x is a Python scalar.\n    out : ndarray\n        The buffer ndarray for storing the result of the unary function.\n\n    Returns\n    -------\n    out : mxnet.numpy.ndarray or scalar\n        Result array or scalar.\n    \"\"\"\n    if isinstance(x, numeric_types):\n        return fn_scalar(x, **kwargs)\n    elif isinstance(x, NDArray):\n        return fn_array(x, out=out, **kwargs)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(x))))\n\n\ndef _pure_unary_func_helper(x, fn_array, fn_scalar, out=None, **kwargs):\n    \"\"\"Helper function for unary operators without support for kwargs.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input of the unary operator.\n    fn_array : function\n        Function to be called if x is of ``ndarray`` type.\n    fn_scalar : function\n        Function to be called if x is a Python scalar.\n    out : ndarray\n        The buffer ndarray for storing the result of the unary function.\n\n    Returns\n    -------\n    out : mxnet.numpy.ndarray or scalar\n        Result array or scalar.\n    \"\"\"\n    if isinstance(x, numeric_types):\n        return fn_scalar(x, **kwargs)\n    elif isinstance(x, NDArray):\n        return fn_array(x, out)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(x))))\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef sin(x, out=None, **kwargs):\n    r\"\"\"\n    Trigonometric sine, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Angle, in radians (:math:`2 \\pi` rad equals 360 degrees).\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a shape that the inputs broadcast to. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output is the same as that of the input if the input is an ndarray.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The sine of each element of x. This is a scalar if `x` is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n\n    Examples\n    --------\n    >>> np.sin(np.pi/2.)\n    1.0\n    >>> np.sin(np.array((0., 30., 45., 60., 90.)) * np.pi / 180.)\n    array([0.        , 0.5       , 0.70710677, 0.86602545, 1.        ])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.sin, _np.sin, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef cos(x, out=None, **kwargs):\n    r\"\"\"\n    Cosine, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Angle, in radians (:math:`2 \\pi` rad equals 360 degrees).\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a shape that the inputs broadcast to. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output is the same as that of the input if the input is an ndarray.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The corresponding cosine values. This is a scalar if x is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n\n    Examples\n    --------\n    >>> np.cos(np.array([0, np.pi/2, np.pi]))\n    array([ 1.000000e+00, -4.371139e-08, -1.000000e+00])\n    >>> # Example of providing the optional output parameter\n    >>> out1 = np.array([0], dtype='f')\n    >>> out2 = np.cos(np.array([0.1]), out1)\n    >>> out2 is out1\n    True\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.cos, _np.cos, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef sinh(x, out=None, **kwargs):\n    \"\"\"\n    Hyperbolic sine, element-wise.\n    Equivalent to ``1/2 * (np.exp(x) - np.exp(-x))`` or ``-1j * np.sin(1j*x)``.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array or scalar.\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a shape that the inputs broadcast to. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output is the same as that of the input if the input is an ndarray.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The corresponding hyperbolic sine values. This is a scalar if `x` is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n\n    Examples\n    --------\n    >>> np.sinh(0)\n    0.0\n    >>> # Example of providing the optional output parameter\n    >>> out1 = np.array([0], dtype='f')\n    >>> out2 = np.sinh(np.array([0.1]), out1)\n    >>> out2 is out1\n    True\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.sinh, _np.sinh, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef cosh(x, out=None, **kwargs):\n    \"\"\"\n    Hyperbolic cosine, element-wise.\n    Equivalent to ``1/2 * (np.exp(x) + np.exp(-x))`` and ``np.cos(1j*x)``.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array or scalar.\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a shape that the inputs broadcast to. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output is the same as that of the input if the input is an ndarray.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The corresponding hyperbolic cosine values. This is a scalar if `x` is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n\n    Examples\n    --------\n    >>> np.cosh(0)\n    1.0\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.cosh, _np.cosh, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef tanh(x, out=None, **kwargs):\n    \"\"\"\n    Compute hyperbolic tangent element-wise.\n    Equivalent to ``np.sinh(x)/np.cosh(x)``.\n\n    Parameters\n    ----------\n    x : ndarray or scalar.\n        Input array.\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a shape that the inputs fill into. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output and input must be the same.\n\n    Returns\n    -------\n    y : ndarray or scalar\n       The corresponding hyperbolic tangent values.\n\n    Notes\n    -----\n    If `out` is provided, the function writes the result into it,\n    and returns a reference to `out`.  (See Examples)\n    - input x does not support complex computation (like imaginary number)\n    >>> np.tanh(np.pi*1j)\n    TypeError: type <type 'complex'> not supported\n\n    Examples\n    --------\n    >>> np.tanh(np.array[0, np.pi]))\n    array([0.       , 0.9962721])\n    >>> np.tanh(np.pi)\n    0.99627207622075\n    >>> # Example of providing the optional output parameter illustrating\n    >>> # that what is returned is a reference to said parameter\n    >>> out1 = np.array(1)\n    >>> out2 = np.tanh(np.array(0.1), out1)\n    >>> out2 is out1\n    True\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.tanh, _np.tanh, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef log10(x, out=None, **kwargs):\n    \"\"\"\n    Return the base 10 logarithm of the input array, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array or scalar.\n    out : ndarray or None\n        A location into which t'absolute', he result is stored. If provided, it\n        must have a shape that the inputs broadcast to. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output is the same as that of the input if the input is an ndarray.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The logarithm to the base 10 of `x`, element-wise. NaNs are\n        returned where x is negative. This is a scalar if `x` is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n\n    Examples\n    --------\n    >>> np.log10(np.array([1e-15, -3.]))\n    array([-15.,  nan])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.log10, _np.log10, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef sqrt(x, out=None, **kwargs):\n    \"\"\"\n    Return the non-negative square-root of an array, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        The values whose square-roots are required.\n    out : ndarray, or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        An array of the same shape as `x`, containing the positive\n        square-root of each element in `x`. This is a scalar if `x` is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n\n    Examples\n    --------\n    >>> np.sqrt(np.array([1,4,9]))\n    array([1., 2., 3.])\n    >>> np.sqrt(np.array([4, -1, _np.inf]))\n    array([ 2., nan, inf])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.sqrt, _np.sqrt, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef cbrt(x, out=None, **kwargs):\n    r\"\"\"\n    Return the cube-root of an array, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray\n        The values whose cube-roots are required.\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape that the\n        inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n        A tuple (possible only as a keyword argument) must have length equal to the number of outputs.\n\n    Returns\n    ----------\n    y : ndarray\n        An array of the same shape as x, containing the cube cube-root of each element in x.\n        If out was provided, y is a reference to it. This is a scalar if x is a scalar.\n\n    Examples\n    ----------\n    >>> np.cbrt([1,8,27])\n    array([ 1.,  2.,  3.])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.cbrt, _np.cbrt, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef abs(x, out=None, **kwargs):\n    r\"\"\"\n    Calculate the absolute value element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    absolute : ndarray\n        An ndarray containing the absolute value of\n        each element in `x`. This is a scalar if `x` is a scalar.\n\n    Examples\n    --------\n    >>> x = np.array([-1.2, 1.2])\n    >>> np.abs(x)\n    array([1.2, 1.2])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.abs, _np.abs, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef fabs(x, out=None, **kwargs):\n    r\"\"\"\n    Calculate the absolute value element-wise.\n\n    This function returns the absolute values (positive magnitude) of the\n    data in `x`. Complex values are not handled, use `absolute` to find the\n    absolute values of complex data.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    absolute : ndarray\n        An ndarray containing the absolute value of\n        each element in `x`. This is a scalar if `x` is a scalar.\n\n    Examples\n    --------\n    >>> np.fabs(-1)\n    1.0\n    >>> np.fabs(np.array([-1.2, 1.2]))s\n    array([ 1.2,  1.2])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.abs, _np.abs, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef absolute(x, out=None, **kwargs):\n    r\"\"\"\n    Calculate the absolute value element-wise.\n    np.abs is a shorthand for this function.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array.\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n        A tuple (possible only as a keyword argument) must have length equal to the number of outputs.\n\n    Returns\n    ----------\n    absolute : ndarray\n        An ndarray containing the absolute value of each element in x.\n\n    Examples\n    ----------\n    >>> x = np.array([-1.2, 1.2])\n    >>> np.absolute(x)\n    array([ 1.2,  1.2])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.abs, _np.abs, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef sign(x, out=None, **kwargs):\n    r\"\"\"\n    Returns an element-wise indication of the sign of a number.\n    The `sign` function returns ``-1 if x < 0, 0 if x==0, 1 if x > 0``. Only supports real number.\n\n    Parameters\n    ----------\n    x : ndarray or a scalar\n        Input values.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray\n        The sign of `x`.\n        This is a scalar if `x` is a scalar.\n\n    Note\n    -------\n    - Only supports real number as input elements.\n    - Input type does not support Python native iterables(list, tuple, ...).\n    - ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be the same as the expected output.\n    - ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the same as the expected output.\n    - ``out`` param does not support scalar input case.\n\n    Examples\n    --------\n    >>> a = np.array([-5., 4.5])\n    >>> np.sign(a)\n    array([-1.,  1.])\n    >>> # Use scalars as inputs:\n    >>> np.sign(4.0)\n    1.0\n    >>> np.sign(0)\n    0\n    >>> # Use ``out`` parameter:\n    >>> b = np.zeros((2, ))\n    >>> np.sign(a, out=b)\n    array([-1.,  1.])\n    >>> b\n    array([-1.,  1.])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.sign, _np.sign, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef exp(x, out=None, **kwargs):\n    r\"\"\"\n    Calculate the exponential of all elements in the input array.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input values.\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array, element-wise exponential of `x`.\n        This is a scalar if `x` is a scalar.\n\n    Examples\n    --------\n    >>> np.exp(1)\n    2.718281828459045\n    >>> x = np.array([-1, 1, -2, 2])\n    >>> np.exp(x)\n    array([0.36787945, 2.7182817 , 0.13533528, 7.389056  ])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.exp, _np.exp, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef expm1(x, out=None, **kwargs):\n    r\"\"\"\n    Calculate `exp(x) - 1` of all elements in the input array.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input values.\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array, element-wise exponential minus one: `out = exp(x) - 1`.\n        This is a scalar if `x` is a scalar.\n\n    Examples\n    --------\n    >>> np.expm1(1)\n    1.718281828459045\n    >>> x = np.array([-1, 1, -2, 2])\n    >>> np.expm1(x)\n    array([-0.63212056,  1.71828183, -0.86466472,  6.3890561])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.expm1, _np.expm1, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef arcsin(x, out=None, **kwargs):\n    r\"\"\"\n    Inverse sine, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        `y`-coordinate on the unit circle.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape as the input.\n        If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    angle : ndarray or scalar\n        Output array is same shape and type as x. This is a scalar if x is a scalar.\n        The inverse sine of each element in `x`, in radians and in the\n        closed interval ``[-pi/2, pi/2]``.\n\n    Examples\n    --------\n    >>> np.arcsin(1)     # pi/2\n    1.5707963267948966\n    >>> np.arcsin(-1)    # -pi/2\n    -1.5707963267948966\n    >>> np.arcsin(0)\n    0.0\n\n    Notes\n    -----\n    `arcsin` is a multivalued function: for each `x` there are infinitely\n    many numbers `z` such that :math:`sin(z) = x`.  The convention is to\n    return the angle `z` whose real part lies in [-pi/2, pi/2].\n    For real-valued input data types, *arcsin* always returns real output.\n    For each value that cannot be expressed as a real number or infinity,\n    it yields ``nan`` and sets the `invalid` floating point error flag.\n    The inverse sine is also known as `asin` or sin^{-1}.\n    The output `ndarray` has the same `device` as the input `ndarray`.\n    This function differs from the original `numpy.arcsin\n    <https://numpy.org/doc/stable/reference/generated/numpy.arcsin.html>`_ in\n    the following aspects:\n    - Only support ndarray or scalar now.\n    - `where` argument is not supported.\n    - Complex input is not supported.\n\n    References\n    ----------\n    Abramowitz, M. and Stegun, I. A., *Handbook of Mathematical Functions*,\n    10th printing, New York: Dover, 1964, pp. 79ff.\n    http://www.math.sfu.ca/~cbm/aands/\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.arcsin, _np.arcsin, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef arccos(x, out=None, **kwargs):\n    r\"\"\"\n    Trigonometric inverse cosine, element-wise.\n    The inverse of cos so that, if y = cos(x), then x = arccos(y).\n\n    Parameters\n    ----------\n    x : ndarray\n        x-coordinate on the unit circle. For real arguments, the domain is [-1, 1].\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape that\n        the inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n        A tuple (possible only as a keyword argument) must have length equal to the number of outputs.\n\n    Returns\n    ----------\n    angle : ndarray\n        The angle of the ray intersecting the unit circle at the given x-coordinate in radians [0, pi].\n        This is a scalar if x is a scalar.\n\n    See also\n    ----------\n    cos, arctan, arcsin\n\n    Notes\n    ----------\n    arccos is a multivalued function: for each x there are infinitely many numbers z such that\n    cos(z) = x. The convention is to return the angle z whose real part lies in [0, pi].\n    For real-valued input data types, arccos always returns real output.\n    For each value that cannot be expressed as a real number or infinity, it yields nan and sets\n    the invalid floating point error flag.\n    The inverse cos is also known as acos or cos^-1.\n\n    Examples\n    ----------\n    >>> np.arccos([1, -1])\n    array([ 0.        ,  3.14159265])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.arccos, _np.arccos, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef arctan(x, out=None, **kwargs):\n    r\"\"\"\n    Trigonometric inverse tangent, element-wise.\n    The inverse of tan, so that if ``y = tan(x)`` then ``x = arctan(y)``.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input values.\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Out has the same shape as `x`. It lies is in\n        ``[-pi/2, pi/2]`` (``arctan(+/-inf)`` returns ``+/-pi/2``).\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    `arctan` is a multi-valued function: for each `x` there are infinitely\n    many numbers `z` such that tan(`z`) = `x`.  The convention is to return\n    the angle `z` whose real part lies in [-pi/2, pi/2].\n    For real-valued input data types, `arctan` always returns real output.\n    For each value that cannot be expressed as a real number or infinity,\n    it yields ``nan`` and sets the `invalid` floating point error flag.\n    For complex-valued input, we do not have support for them yet.\n    The inverse tangent is also known as `atan` or tan^{-1}.\n\n    Examples\n    --------\n    >>> x = np.array([0, 1])\n    >>> np.arctan(x)\n    array([0.       , 0.7853982])\n    >>> np.pi/4\n    0.7853981633974483\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.arctan, _np.arctan, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef log(x, out=None, **kwargs):\n    \"\"\"\n    Natural logarithm, element-wise.\n    The natural logarithm `log` is the inverse of the exponential function,\n    so that `log(exp(x)) = x`. The natural logarithm is logarithm in base\n    `e`.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input value. Elements must be of real value.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray\n        The natural logarithm of `x`, element-wise.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n     Currently only supports data of real values and ``inf`` as input. Returns data of real value, ``inf``, ``-inf`` and\n    ``nan`` according to the input.\n    This function differs from the original `numpy.log\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.log.html>`_ in\n    the following aspects:\n    - Does not support complex number for now\n    - Input type does not support Python native iterables(list, tuple, ...).\n    - ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be the same as the expected output.\n    - ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the same as the expected output.\n    - ``out`` param does not support scalar input case.\n\n    Examples\n    --------\n    >>> a = np.array([1, np.exp(1), np.exp(2), 0], dtype=np.float64)\n    >>> np.log(a)\n    array([  0.,   1.,   2., -inf], dtype=float64)\n    >>> # Using default float32 dtype may lead to slightly different behavior:\n    >>> a = np.array([1, np.exp(1), np.exp(2), 0], dtype=np.float32)\n    >>> np.log(a)\n    array([  0.,  0.99999994,   2., -inf])\n    >>> np.log(1)\n    0.0\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.log, _np.log, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef degrees(x, out=None, **kwargs):\n    \"\"\"\n    Convert angles from radians to degrees.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input value. Elements must be of real value.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray\n        The corresponding degree values; if `out` was supplied this is a\n        reference to it.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -------\n    This function differs from the original `numpy.degrees\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.degrees.html>`_ in\n    the following aspects:\n    - Input type does not support Python native iterables(list, tuple, ...). Only ndarray is supported.\n    - ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be the same as the expected output.\n    - ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the same as the expected output.\n    - ``out`` param does not support scalar input case.\n\n    Examples\n    --------\n    >>> rad = np.arange(12.) * np.pi / 6\n    >>> np.degrees(rad)\n    array([  0.,  30.,  60.,  90., 120., 150., 180., 210., 240., 270., 300., 330.])\n    >>> # Use specified ``out`` ndarray:\n    >>> out = np.zeros((rad.shape))\n    >>> np.degrees(rad, out)\n    array([  0.,  30.,  60.,  90., 120., 150., 180., 210., 240., 270., 300., 330.])\n    >>> out\n    array([  0.,  30.,  60.,  90., 120., 150., 180., 210., 240., 270., 300., 330.])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.degrees, _np.degrees, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef rad2deg(x, out=None, **kwargs):\n    r\"\"\"\n    Convert angles from radians to degrees.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Angles in degrees.\n    out : ndarray or None, optional\n        A location into which the result is stored. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The corresponding angle in radians.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    \"rad2deg(x)\" is \"x *180 / pi\".\n\n    This function differs from the original numpy.arange in the following aspects:\n        - Only support float32 and float64.\n        - `out` must be in the same size of input.\n\n    Examples\n    --------\n    >>> np.rad2deg(np.pi/2)\n    90.0\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.rad2deg, _np.rad2deg, out=out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef rint(x, out=None, **kwargs):\n    \"\"\"\n    Round elements of the array to the nearest integer.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None\n        A location into which the result is stored.\n        If provided, it must have the same shape and type as the input.\n        If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array is same shape and type as x. This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    This function differs from the original `numpy.rint\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.rint.html>`_ in\n    the following way(s):\n    - only ndarray or scalar is accpted as valid input, tuple of ndarray is not supported\n    - broadcasting to `out` of different shape is currently not supported\n    - when input is plain python numerics, the result will not be stored in the `out` param\n\n    Examples\n    --------\n    >>> a = np.array([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0])\n    >>> np.rint(a)\n    array([-2., -2., -0.,  0.,  1.,  2.,  2.])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.rint, _np.rint, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef log2(x, out=None, **kwargs):\n    \"\"\"\n    Base-2 logarithm of x.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input values.\n    out : ndarray or None\n        A location into which the result is stored.\n        If provided, it must have the same shape and type as the input.\n        If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray\n        The logarithm base two of `x`, element-wise.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    This function differs from the original `numpy.log2\n    <https://www.google.com/search?q=numpy+log2>`_ in\n    the following way(s):\n    - only ndarray or scalar is accpted as valid input, tuple of ndarray is not supported\n    - broadcasting to `out` of different shape is currently not supported\n    - when input is plain python numerics, the result will not be stored in the `out` param\n\n    Examples\n    --------\n    >>> x = np.array([0, 1, 2, 2**4])\n    >>> np.log2(x)\n    array([-inf,   0.,   1.,   4.])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.log2, _np.log2, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef log1p(x, out=None, **kwargs):\n    \"\"\"\n    Return the natural logarithm of one plus the input array, element-wise.\n    Calculates ``log(1 + x)``.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a shape that the inputs fill into. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output and input must be the same.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        Natural logarithm of 1 + x, element-wise. This is a scalar\n        if x is a scalar.\n\n    Notes\n    -----\n    For real-valued input, `log1p` is accurate also for `x` so small\n    that `1 + x == 1` in floating-point accuracy.\n    Logarithm is a multivalued function: for each `x` there is an infinite\n    number of `z` such that `exp(z) = 1 + x`. The convention is to return\n    the `z` whose imaginary part lies in `[-pi, pi]`.\n    For real-valued input data types, `log1p` always returns real output.\n    For each value that cannot be expressed as a real number or infinity,\n    it yields ``nan`` and sets the `invalid` floating point error flag.\n    cannot support complex-valued input.\n\n    Examples\n    --------\n    >>> np.log1p(1e-99)\n    1e-99\n    >>> a = np.array([3, 4, 5])\n    >>> np.log1p(a)\n    array([1.3862944, 1.609438 , 1.7917595])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.log1p, _np.log1p, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef radians(x, out=None, **kwargs):\n    \"\"\"\n    Convert angles from degrees to radians.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array in degrees.\n    out : ndarray or None\n        A location into which the result is stored.\n        If provided, it must have the same shape and type as the input.\n        If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray\n        The corresponding radian values. This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    This function differs from the original `numpy.radians\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.radians.html>`_ in\n    the following way(s):\n    - only ndarray or scalar is accpted as valid input, tuple of ndarray is not supported\n    - broadcasting to `out` of different shape is currently not supported\n    - when input is plain python numerics, the result will not be stored in the `out` param\n\n    Examples\n    --------\n    >>> deg = np.arange(12.) * 30.\n    >>> np.radians(deg)\n    array([0.       , 0.5235988, 1.0471976, 1.5707964, 2.0943952, 2.6179938,\n           3.1415927, 3.6651914, 4.1887903, 4.712389 , 5.2359877, 5.7595863],\n           dtype=float32)\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.radians, _np.radians, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef deg2rad(x, out=None, **kwargs):\n    r\"\"\"\n    Convert angles from degrees to radians.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Angles in degrees.\n    out : ndarray or None, optional\n        A location into which the result is stored. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The corresponding angle in radians.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    \"deg2rad(x)\" is \"x * pi / 180\".\n\n    This function differs from the original numpy.arange in the following aspects:\n        - Only support float32 and float64.\n        - `out` must be in the same size of input.\n\n    Examples\n    --------\n    >>> np.deg2rad(180)\n    3.1415927\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.deg2rad, _np.deg2rad, out=out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef reciprocal(x, out=None, **kwargs):\n    r\"\"\"\n    Return the reciprocal of the argument, element-wise.\n    Calculates ``1/x``.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        The values whose reciprocals are required.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape as the input.\n        If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        Output array is same shape and type as x. This is a scalar if x is a scalar.\n\n    Examples\n    --------\n    >>> np.reciprocal(2.)\n    0.5\n    >>> x = np.array([1, 2., 3.33])\n    >>> np.reciprocal(x)\n    array([1.       , 0.5      , 0.3003003])\n\n    Notes\n    -----\n    .. note::\n        This function is not designed to work with integers.\n    For integer arguments with absolute value larger than 1 the result is\n    always zero because of the way Python handles integer division.  For\n    integer zero the result is an overflow.\n    The output `ndarray` has the same `device` as the input `ndarray`.\n    This function differs from the original `numpy.reciprocal\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.reciprocal.html>`_ in\n    the following aspects:\n    - Only support ndarray and scalar now.\n    - `where` argument is not supported.\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.reciprocal, _np.reciprocal, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef square(x, out=None, **kwargs):\n    r\"\"\"\n    Return the element-wise square of the input.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        The values whose squares are required.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape as the input.\n        If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        Output array is same shape and type as x. This is a scalar if x is a scalar.\n\n    Examples\n    --------\n    >>> np.square(2.)\n    4.0\n    >>> x = np.array([1, 2., -1])\n    >>> np.square(x)\n    array([1., 4., 1.])\n\n    Notes\n    -----\n    The output `ndarray` has the same `device` as the input `ndarray`.\n    This function differs from the original `numpy.square\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.square.html>`_ in\n    the following aspects:\n    - Only support ndarray and scalar now.\n    - `where` argument is not supported.\n    - Complex input is not supported.\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.square, _np.square, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef negative(x, out=None, **kwargs):\n    r\"\"\"\n    Numerical negative, element-wise.\n\n    Parameters:\n    ------------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray, None, or tuple of ndarray and None, optional\n          A location into which the result is stored.\n\n    Returns:\n    ---------\n    y : ndarray or scalar\n        Returned array or scalar: y = -x. This is a scalar if x is a scalar.\n\n    Examples:\n    ---------\n    >>> np.negative(1)\n    -1\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.negative, _np.negative, out=out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef positive(x, out=None, **kwargs):\n    r\"\"\"\n    Computes the numerical positive of each element `x_i` (i.e.,`y_i = +x_i`)\n    of the input array x .\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        Returned array or scalar: y = +x. This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    Equivalent to `x.copy()`, but only defined for types that support arithmetic.\n\n    Examples\n    --------\n    >>> x1 = np.array(([1., -1.]))\n    >>> np.positive(x1)\n    array([ 1., -1.])\n    >>> +x1\n    array([ 1., -1.])\n    \"\"\"\n    if out is x:\n        return x\n    return _pure_unary_func_helper(x, _api_internal.copy, _np.positive, out=out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef fix(x, out=None, **kwargs):\n    r\"\"\"\n    Round an array of floats element-wise to nearest integer towards zero.\n    The rounded values are returned as floats.\n\n    Parameters:\n    ----------\n    x : ndarray\n        An array of floats to be rounded\n    out : ndarray, optional\n        Output array\n\n    Returns:\n    -------\n    y : ndarray of floats\n\n    Examples\n    ---------\n    >>> np.fix(3.14)\n    3\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.fix, _np.fix, out=out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef tan(x, out=None, **kwargs):\n    r\"\"\"\n    Compute tangent element-wise.\n    Equivalent to np.sin(x)/np.cos(x) element-wise.\n\n    Parameters:\n    ----------\n    x : ndarray\n        Input array.\n    out : ndarray, None, or tuple of ndarray and None, optional\n          A location into which the result is stored. If provided,\n          it must have a shape that the inputs broadcast to. If not provided or None,\n          a freshly-allocated array is returned. A tuple (possible only as a keyword argument)\n          must have length equal to the number of outputs.\n    where : ndarray, optional\n            Values of True indicate to calculate the ufunc at that position,\n            values of False indicate to leave the value in the output alone.\n\n    Returns:\n    -------\n    y : ndarray\n    The corresponding tangent values. This is a scalar if x is a scalar.\n\n    Examples:\n    ---------\n    >>> np.tan(0.5)\n    0.5463024898437905\n    \"\"\"\n\n    return _pure_unary_func_helper(x, _api_internal.tan, _np.tan, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef ceil(x, out=None, **kwargs):\n    r\"\"\"\n    Return the ceiling of the input, element-wise.\n    The ceil of the ndarray `x` is the smallest integer `i`, such that\n    `i >= x`.  It is often denoted as :math:`\\lceil x \\rceil`.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a same shape that the inputs fill into. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output and input must be the same.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The ceiling of each element in `x`, with `float` dtype.\n        This is a scalar if `x` is a scalar.\n\n    Examples\n    --------\n    >>> a = np.array([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0])\n    >>> np.ceil(a)\n    array([-1., -1., -0.,  1.,  2.,  2.,  2.])\n    >>> #if you use parameter out, x and out must be ndarray.\n    >>> a = np.array(1)\n    >>> np.ceil(np.array(3.5), a)\n    array(4.)\n    >>> a\n    array(4.)\n    \"\"\"\n    if isinstance(x, NDArray) and _np.issubdtype(x.dtype, _np.integer):\n        return x\n    return _pure_unary_func_helper(x, _api_internal.ceil, _np.ceil, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef floor(x, out=None, **kwargs):\n    r\"\"\"\n    Return the floor of the input, element-wise.\n    The floor of the ndarray `x` is the largest integer `i`, such that\n    `i <= x`.  It is often denoted as :math:`\\lfloor x \\rfloor`.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a same shape that the inputs fill into. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output and input must be the same.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The floor of each element in `x`, with `float` dtype.\n        This is a scalar if `x` is a scalar.\n\n    Examples\n    --------\n    >>> a = np.array([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0])\n    >>> np.floor(a)\n    array([-2., -2., -1.,  0.,  1.,  1.,  2.])\n    >>> #if you use parameter out, x and out must be ndarray.\n    >>> a = np.array(1)\n    >>> np.floor(np.array(3.5), a)\n    array(3.)\n    >>> a\n    array(3.)\n    \"\"\"\n    if isinstance(x, NDArray) and _np.issubdtype(x.dtype, _np.integer):\n        return x\n    return _pure_unary_func_helper(x, _api_internal.floor, _np.floor, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef bitwise_not(x, out=None, **kwargs):\n    r\"\"\"\n    Compute bit-wise inversion, or bit-wise NOT, element-wise.\n    Computes the bit-wise NOT of the underlying binary representation of\n    the integers in the input arrays. This ufunc implements the C/Python\n    operator ``~``.\n\n    Parameters\n    ----------\n    x : array_like\n        Only integer and boolean types are handled.\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Result.\n        This is a scalar if `x` is a scalar.\n\n    See Also\n    --------\n    bitwise_and, bitwise_or, bitwise_xor\n    logical_not\n    binary_repr :\n        Return the binary representation of the input number as a string.\n\n    Examples\n    --------\n    We've seen that 13 is represented by ``00001101``.\n    The invert or bit-wise NOT of 13 is then:\n\n    >>> x = np.invert(np.array(13, dtype=np.uint8))\n    >>> x\n    242\n    >>> np.binary_repr(x, width=8)\n    '11110010'\n\n    Notes\n    -----\n    `bitwise_not` is an alias for `invert`:\n\n    >>> np.bitwise_not is np.invert\n    True\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.bitwise_not, _np.bitwise_not, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef invert(x, out=None, **kwargs):\n    r\"\"\"\n    Compute bit-wise inversion, or bit-wise NOT, element-wise.\n    Computes the bit-wise NOT of the underlying binary representation of\n    the integers in the input arrays. This ufunc implements the C/Python\n    operator ``~``.\n\n    Parameters\n    ----------\n    x : array_like\n        Only integer and boolean types are handled.\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Result.\n        This is a scalar if `x` is a scalar.\n\n    See Also\n    --------\n    bitwise_and, bitwise_or, bitwise_xor\n    logical_not\n    binary_repr :\n        Return the binary representation of the input number as a string.\n\n    Examples\n    --------\n    We've seen that 13 is represented by ``00001101``.\n    The invert or bit-wise NOT of 13 is then:\n\n    >>> x = np.invert(np.array(13, dtype=np.uint8))\n    >>> x\n    242\n    >>> np.binary_repr(x, width=8)\n    '11110010'\n\n    Notes\n    -----\n    `bitwise_not` is an alias for `invert`:\n\n    >>> np.bitwise_not is np.invert\n    True\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.bitwise_not, _np.bitwise_not, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef trunc(x, out=None, **kwargs):\n    r\"\"\"\n    Return the truncated value of the input, element-wise.\n    The truncated value of the scalar `x` is the nearest integer `i` which\n    is closer to zero than `x` is. In short, the fractional part of the\n    signed number `x` is discarded.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input data.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The truncated value of each element in `x`.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    This function differs from the original numpy.trunc in the following aspects:\n        - Do not support `where`, a parameter in numpy which indicates where to calculate.\n        - Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n        - Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n        - If `x` is plain python numeric, the result won't be stored in out.\n\n    Examples\n    --------\n    >>> a = np.array([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0])\n    >>> np.trunc(a)\n    array([-1., -1., -0.,  0.,  1.,  1.,  2.])\n    \"\"\"\n    if isinstance(x, NDArray) and _np.issubdtype(x.dtype, _np.integer):\n        return x\n    return _pure_unary_func_helper(x, _api_internal.trunc, _np.trunc, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef logical_not(x, out=None, **kwargs):\n    r\"\"\"\n    Compute the truth value of NOT x element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Logical NOT is applied to the elements of `x`.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n\n    Returns\n    -------\n    y : bool or ndarray of bool\n        Boolean result with the same shape as `x` of the NOT operation\n        on elements of `x`.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    This function differs from the original numpy.logical_not in the following aspects:\n        - Do not support `where`, a parameter in numpy which indicates where to calculate.\n        - Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n        - Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n        - If `x` is plain python numeric, the result won't be stored in out.\n\n    Examples\n    --------\n    >>> x= np.array([True, False, 0, 1])\n    >>> np.logical_not(x)\n    array([False,  True,  True, False])\n\n    >>> x = np.arange(5)\n    >>> np.logical_not(x<3)\n    array([False, False, False,  True,  True])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.logical_not, _np.logical_not, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef arcsinh(x, out=None, **kwargs):\n    r\"\"\"\n    Inverse hyperbolic sine, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n\n    Returns\n    -------\n    arcsinh : ndarray\n        Array of the same shape as `x`.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    `arcsinh` is a multivalued function: for each `x` there are infinitely\n    many numbers `z` such that `sinh(z) = x`.\n\n    For real-valued input data types, `arcsinh` always returns real output.\n    For each value that cannot be expressed as a real number or infinity, it\n    yields ``nan`` and sets the `invalid` floating point error flag.\n\n    This function differs from the original numpy.arcsinh in the following aspects:\n        - Do not support `where`, a parameter in numpy which indicates where to calculate.\n        - Do not support complex-valued input.\n        - Cannot cast type automatically. DType of `out` must be same as the expected one.\n        - Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n        - If `x` is plain python numeric, the result won't be stored in out.\n\n    Examples\n    --------\n    >>> a = np.array([3.2, 5.0])\n    >>> np.arcsinh(a)\n    array([1.8309381, 2.2924316])\n    >>> np.arcsinh(1)\n    0.0\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.arcsinh, _np.arcsinh, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef arccosh(x, out=None, **kwargs):\n    r\"\"\"\n    Inverse hyperbolic cosine, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n\n    Returns\n    -------\n    arccosh : ndarray\n        Array of the same shape as `x`.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    `arccosh` is a multivalued function: for each `x` there are infinitely\n    many numbers `z` such that `cosh(z) = x`.\n\n    For real-valued input data types, `arccosh` always returns real output.\n    For each value that cannot be expressed as a real number or infinity, it\n    yields ``nan`` and sets the `invalid` floating point error flag.\n\n    This function differs from the original numpy.arccosh in the following aspects:\n        - Do not support `where`, a parameter in numpy which indicates where to calculate.\n        - Do not support complex-valued input.\n        - Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n        - Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n        - If `x` is plain python numeric, the result won't be stored in out.\n\n    Examples\n    --------\n    >>> a = np.array([3.2, 5.0])\n    >>> np.arccosh(a)\n    array([1.8309381, 2.2924316])\n    >>> np.arccosh(1)\n    0.0\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.arccosh, _np.arccosh, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef arctanh(x, out=None, **kwargs):\n    r\"\"\"\n    Inverse hyperbolic tangent, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n\n    Returns\n    -------\n    arctanh : ndarray\n        Array of the same shape as `x`.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    `arctanh` is a multivalued function: for each `x` there are infinitely\n    many numbers `z` such that `tanh(z) = x`.\n\n    For real-valued input data types, `arctanh` always returns real output.\n    For each value that cannot be expressed as a real number or infinity, it\n    yields ``nan`` and sets the `invalid` floating point error flag.\n\n    This function differs from the original numpy.arctanh in the following aspects:\n        - Do not support `where`, a parameter in numpy which indicates where to calculate.\n        - Do not support complex-valued input.\n        - Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n        - Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n        - If `x` is plain python numeric, the result won't be stored in out.\n\n    Examples\n    --------\n    >>> a = np.array([0.0, -0.5])\n    >>> np.arctanh(a)\n    array([0., -0.54930615])\n    >>> np.arctanh(0.0)\n    0.0\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.arctanh, _np.arctanh, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef tile(A, reps):\n    r\"\"\"\n    Construct an array by repeating A the number of times given by reps.\n\n    If `reps` has length ``d``, the result will have dimension of\n    ``max(d, A.ndim)``.\n\n    If ``A.ndim < d``, `A` is promoted to be d-dimensional by prepending new\n    axes. So a shape (3,) array is promoted to (1, 3) for 2-D replication,\n    or shape (1, 1, 3) for 3-D replication. If this is not the desired\n    behavior, promote `A` to d-dimensions manually before calling this\n    function.\n\n    If ``A.ndim > d``, `reps` is promoted to `A`.ndim by pre-pending 1's to it.\n    Thus for an `A` of shape (2, 3, 4, 5), a `reps` of (2, 2) is treated as\n    (1, 1, 2, 2).\n\n    Parameters\n    ----------\n    A : ndarray or scalar\n        An input array or a scalar to repeat.\n    reps : a single integer or tuple of integers\n        The number of repetitions of `A` along each axis.\n\n    Returns\n    -------\n    c : ndarray\n        The tiled output array.\n\n    Examples\n    --------\n    >>> a = np.array([0, 1, 2])\n    >>> np.tile(a, 2)\n    array([0., 1., 2., 0., 1., 2.])\n    >>> np.tile(a, (2, 2))\n    array([[0., 1., 2., 0., 1., 2.],\n           [0., 1., 2., 0., 1., 2.]])\n    >>> np.tile(a, (2, 1, 2))\n    array([[[0., 1., 2., 0., 1., 2.]],\n           [[0., 1., 2., 0., 1., 2.]]])\n\n    >>> b = np.array([[1, 2], [3, 4]])\n    >>> np.tile(b, 2)\n    array([[1., 2., 1., 2.],\n           [3., 4., 3., 4.]])\n    >>> np.tile(b, (2, 1))\n    array([[1., 2.],\n           [3., 4.],\n           [1., 2.],\n           [3., 4.]])\n\n    >>> c = np.array([1,2,3,4])\n    >>> np.tile(c,(4,1))\n    array([[1., 2., 3., 4.],\n           [1., 2., 3., 4.],\n           [1., 2., 3., 4.],\n           [1., 2., 3., 4.]])\n\n    Scalar as input:\n\n    >>> np.tile(2, 3)\n    array([2, 2, 2]) # repeating integer `2`\n\n    \"\"\"\n    if isinstance(A, numeric_types):\n        return _np.tile(A, reps)\n    elif isinstance(A, NDArray):\n        return _api_internal.tile(A, reps)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(A))))\n\n\n@set_module('mxnet.ndarray.numpy')\ndef transpose(a, axes=None):\n    \"\"\"\n    Permute the dimensions of an array.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array.\n    axes : list of ints, optional\n        By default, reverse the dimensions,\n        otherwise permute the axes according to the values given.\n\n    Returns\n    -------\n    p : ndarray\n        a with its axes permuted.\n\n    Notes\n    -----\n    This function differs from the original `numpy.transpose\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.transpose.html>`_ in\n    the following way(s):\n\n    - only ndarray is accepted as valid input, python iterables are not supported\n    - the operator always returns an `ndarray` that does not share the memory with the input\n\n    Examples\n    --------\n    >>> x = np.arange(4).reshape((2,2))\n    >>> x\n    array([[0., 1.],\n           [2., 3.]])\n    >>> np.transpose(x)\n    array([[0., 2.],\n           [1., 3.]])\n    >>> x = np.ones((1, 2, 3))\n    >>> np.transpose(x, (1, 0, 2)).shape\n    (2, 1, 3)\n    \"\"\"\n    return _api_internal.transpose(a, axes)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef repeat(a, repeats, axis=None):\n    \"\"\"\n    Repeat elements of an array.\n\n    Parameters\n    ----------\n    a : array_like\n        Input array.\n    repeats : int\n        The number of repetitions for each element.\n    axis : int, optional\n        The axis along which to repeat values.  By default, use the\n        flattened input array, and return a flat output array.\n\n    Returns\n    -------\n    repeated_array : ndarray\n        Output array which has the same shape as `a`, except along\n        the given axis.\n\n    See Also\n    --------\n    tile : Tile an array.\n\n    Examples\n    --------\n    >>> np.repeat(3, 4)\n    array([3, 3, 3, 3])\n    >>> x = np.array([[1,2],[3,4]])\n    >>> np.repeat(x, 2)\n    array([1, 1, 2, 2, 3, 3, 4, 4])\n    >>> np.repeat(x, 3, axis=1)\n    array([[1, 1, 1, 2, 2, 2],\n           [3, 3, 3, 4, 4, 4]])\n    >>> np.repeat(x, [1, 2], axis=0)\n    array([[1, 2],\n           [3, 4],\n           [3, 4]])\n    \"\"\"\n    if isinstance(repeats, numeric_types):\n        repeats = [repeats]\n    return _api_internal.repeats(a, repeats, axis)\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.ndarray.numpy')\ndef split(ary, indices_or_sections, axis=0):\n    \"\"\"\n    Split an array into multiple sub-arrays.\n\n    Parameters\n    ----------\n    ary : ndarray\n        Array to be divided into sub-arrays.\n    indices_or_sections : int or 1-D python tuple, list or set.\n        If `indices_or_sections` is an integer, N, the array will be divided\n        into N equal arrays along `axis`.  If such a split is not possible,\n        an error is raised.\n        If `indices_or_sections` is a 1-D array of sorted integers, the entries\n        indicate where along `axis` the array is split.  For example,\n        ``[2, 3]`` would, for ``axis=0``, result in\n          - ary[:2]\n          - ary[2:3]\n          - ary[3:]\n        If an index exceeds the dimension of the array along `axis`,\n        an empty sub-array is returned correspondingly.\n    axis : int, optional\n        The axis along which to split, default is 0.\n\n    Returns\n    -------\n    sub-arrays : list of ndarrays\n        A list of sub-arrays.\n\n    Raises\n    ------\n    ValueError\n        If `indices_or_sections` is given as an integer, but\n        a split does not result in equal division.\n    \"\"\"\n    if isinstance(indices_or_sections, set):\n        indices_or_sections = list(indices_or_sections)\n    return list(_api_internal.split(ary, indices_or_sections, axis))\n# pylint: enable=redefined-outer-name\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.ndarray.numpy')\ndef array_split(ary, indices_or_sections, axis=0):\n    \"\"\"Split an array into multiple sub-arrays.\n\n    If `indices_or_sections` is an integer, N, the array will be divided\n    into N equal arrays along `axis`.  If such a split is not possible,\n    an array of length l that should be split into n sections, it returns\n    l % n sub-arrays of size l//n + 1 and the rest of size l//n.\n\n    If `indices_or_sections` is a 1-D array of sorted integers, the entries\n        indicate where along `axis` the array is split.  For example,\n        ``[2, 3]`` would, for ``axis=0``, result in\n          - ary[:2]\n          - ary[2:3]\n          - ary[3:]\n    If an index exceeds the dimension of the array along `axis`,\n    an empty sub-array is returned correspondingly.\n\n    Parameters\n    ----------\n    ary : ndarray\n        Array to be divided into sub-arrays.\n    indices_or_sections : int or 1-D Python tuple, list or set.\n        Param used to determine the number and size of the subarray.\n    axis : int, optional\n        The axis along which to split, default is 0.\n\n    Returns\n    -------\n    sub-arrays : list of ndarrays\n        A list of sub-arrays.\n\n    Examples\n    --------\n    >>> x = np.arange(9.0)\n    >>> np.array_split(x, 3)\n    [array([0., 1., 2.]), array([3., 4., 5.]), array([6., 7., 8.])]\n\n    >>> np.array_split(x, [3, 5, 6, 8])\n    [array([0., 1., 2.]), array([3., 4.]), array([5.]), array([6., 7.]), array([])]\n\n    >>> x = np.arange(8.0)\n    >>> np.array_split(x, 3)\n    [array([0.,  1.,  2.]), array([3.,  4.,  5.]), array([6.,  7.])]\n\n    >>> x = np.arange(7.0)\n    >>> np.array_split(x, 3)\n    [array([0.,  1.,  2.]), array([3.,  4.]), array([5.,  6.])]\n    \"\"\"\n    if isinstance(indices_or_sections, set):\n        indices_or_sections = list(indices_or_sections)\n    return list(_api_internal.array_split(ary, indices_or_sections, axis))\n# pylint: enable=redefined-outer-name\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.ndarray.numpy')\ndef hsplit(ary, indices_or_sections):\n    \"\"\"Split an array into multiple sub-arrays horizontally (column-wise).\n\n    This is equivalent to ``split`` with ``axis=0`` if ``ary`` has one\n    dimension, and otherwise that with ``axis=1``.\n\n    Parameters\n    ----------\n    ary : ndarray\n        Array to be divided into sub-arrays.\n    indices_or_sections : int, list of ints or tuple of ints.\n        If `indices_or_sections` is an integer, N, the array will be divided\n        into N equal arrays along `axis`.  If such a split is not possible,\n        an error is raised.\n\n        If `indices_or_sections` is a list of sorted integers, the entries\n        indicate where along `axis` the array is split.\n\n        If an index exceeds the dimension of the array along `axis`,\n        it will raises errors. so index must less than or euqal to\n        the dimension of the array along axis.\n\n    Returns\n    -------\n    sub-arrays : list of ndarrays\n        A list of sub-arrays.\n\n    Notes\n    ------\n    - If `indices_or_sections` is given as an integer, but a split\n      does not result in equal division.It will raises ValueErrors.\n\n    - If indices_or_sections is an integer, and the number is 1, it will\n      raises an error. Because single output from split is not supported yet...\n\n    See Also\n    --------\n    split : Split an array into multiple sub-arrays of equal size.\n\n    Examples\n    --------\n    >>> x = np.arange(16.0).reshape(4, 4)\n    >>> x\n    array([[ 0.,  1.,  2.,  3.],\n           [ 4.,  5.,  6.,  7.],\n           [ 8.,  9., 10., 11.],\n           [12., 13., 14., 15.]])\n    >>> np.hsplit(x, 2)\n    [array([[ 0.,  1.],\n           [ 4.,  5.],\n           [ 8.,  9.],\n           [12., 13.]]),\n    array([[ 2.,  3.],\n           [ 6.,  7.],\n           [10., 11.],\n           [14., 15.]])]\n    >>> np.hsplit(x, [3, 6])\n    [array([[ 0.,  1.,  2.],\n           [ 4.,  5.,  6.],\n           [ 8.,  9., 10.],\n           [12., 13., 14.]]),\n    array([[ 3.],\n           [ 7.],\n           [11.],\n           [15.]]),\n    array([], shape=(4, 0), dtype=float32)]\n\n    With a higher dimensional array the split is still along the second axis.\n\n    >>> x = np.arange(8.0).reshape(2, 2, 2)\n    >>> x\n    array([[[ 0.,  1.],\n            [ 2.,  3.]],\n           [[ 4.,  5.],\n            [ 6.,  7.]]])\n    >>> np.hsplit(x, 2)\n    [array([[[ 0.,  1.]],\n            [[ 4.,  5.]]]),\n     array([[[ 2.,  3.]],\n            [[ 6.,  7.]]])]\n\n    If ``ary`` has one dimension, 'axis' = 0.\n    >>> x = np.arange(4)\n    array([0., 1., 2., 3.])\n    >>> np.hsplit(x, 2)\n    [array([0., 1.]), array([2., 3.])]\n\n    If you want to produce an empty sub-array, you can see an example.\n    >>> np.hsplit(x, [2, 2])\n    [array([0., 1.]), array([], dtype=float32), array([2., 3.])]\n    \"\"\"\n    if isinstance(indices_or_sections, set):\n        indices_or_sections = list(indices_or_sections)\n    return list(_api_internal.hsplit(ary, indices_or_sections))\n# pylint: enable=redefined-outer-name\n\n\n@set_module('mxnet.ndarray.numpy')\ndef vsplit(ary, indices_or_sections):\n    r\"\"\"\n    vsplit(ary, indices_or_sections)\n\n    Split an array into multiple sub-arrays vertically (row-wise).\n\n    ``vsplit`` is equivalent to ``split`` with `axis=0` (default): the array is always split\n    along the first axis regardless of the array dimension.\n\n    Parameters\n    ----------\n    ary : ndarray\n        Array to be divided into sub-arrays.\n    indices_or_sections : int or 1 - D Python tuple, list or set.\n        If `indices_or_sections` is an integer, N, the array will be divided into N equal arrays\n        along axis 0.  If such a split is not possible, an error is raised.\n\n        If `indices_or_sections` is a 1-D array of sorted integers, the entries indicate where\n        along axis 0 the array is split.  For example, ``[2, 3]`` would result in\n\n          - ary[:2]\n          - ary[2:3]\n          - ary[3:]\n\n        If an index exceeds the dimension of the array along axis 0, an error will be thrown.\n\n    Returns\n    -------\n    sub-arrays : list of ndarrays\n        A list of sub-arrays.\n\n    See Also\n    --------\n    split : Split an array into multiple sub-arrays of equal size.\n\n    Notes\n    -------\n    This function differs from the original `numpy.degrees\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.degrees.html>`_ in\n    the following aspects:\n\n    - Currently parameter ``indices_or_sections`` does not support ndarray, but supports scalar,\n    tuple and list.\n    - In ``indices_or_sections``, if an index exceeds the dimension of the array along axis 0,\n    an error will be thrown.\n\n    Examples\n    --------\n    >>> x = np.arange(16.0).reshape(4, 4)\n    >>> x\n    array([[  0.,   1.,   2.,   3.],\n           [  4.,   5.,   6.,   7.],\n           [  8.,   9.,  10.,  11.],\n           [ 12.,  13.,  14.,  15.]])\n    >>> np.vsplit(x, 2)\n    [array([[0., 1., 2., 3.],\n            [4., 5., 6., 7.]]), array([[ 8.,  9., 10., 11.],\n            [12., 13., 14., 15.]])]\n\n    With a higher dimensional array the split is still along the first axis.\n\n    >>> x = np.arange(8.0).reshape(2, 2, 2)\n    >>> x\n    array([[[ 0.,  1.],\n            [ 2.,  3.]],\n           [[ 4.,  5.],\n            [ 6.,  7.]]])\n    >>> np.vsplit(x, 2)\n    [array([[[0., 1.],\n            [2., 3.]]]), array([[[4., 5.],\n            [6., 7.]]])]\n\n    \"\"\"\n    if isinstance(indices_or_sections, set):\n        indices_or_sections = list(indices_or_sections)\n    return list(_api_internal.vsplit(ary, indices_or_sections))\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.ndarray.numpy')\ndef dsplit(ary, indices_or_sections):\n    \"\"\"\n    Split array into multiple sub-arrays along the 3rd axis (depth).\n\n    Please refer to the `split` documentation.  `dsplit` is equivalent\n    to `split` with ``axis=2``, the array is always split along the third\n    axis provided the array dimension is greater than or equal to 3.\n\n    Parameters\n    ----------\n    ary : ndarray\n        Array to be divided into sub-arrays.\n    indices_or_sections : int or 1 - D Python tuple, list or set.\n        If `indices_or_sections` is an integer, N, the array will be divided into N equal arrays\n        along axis 2.  If such a split is not possible, an error is raised.\n\n        If `indices_or_sections` is a 1-D array of sorted integers, the entries indicate where\n        along axis 2 the array is split.  For example, ``[2, 3]`` would result in\n\n          - ary[:, :, :2]\n          - ary[:, :, 2:3]\n          - ary[:, :, 3:]\n\n        If an index exceeds the dimension of the array along axis 2, an error will be thrown.\n\n    Examples\n    --------\n    >>> x = np.arange(16.0).reshape(2, 2, 4)\n    >>> x\n    array([[[ 0.,   1.,   2.,   3.],\n            [ 4.,   5.,   6.,   7.]],\n           [[ 8.,   9.,  10.,  11.],\n            [12.,  13.,  14.,  15.]]])\n    >>> np.dsplit(x, 2)\n    [array([[[ 0.,  1.],\n            [ 4.,  5.]],\n           [[ 8.,  9.],\n            [12., 13.]]]), array([[[ 2.,  3.],\n            [ 6.,  7.]],\n           [[10., 11.],\n            [14., 15.]]])]\n    >>> np.dsplit(x, np.array([3, 6]))\n    [array([[[ 0.,   1.,   2.],\n            [ 4.,   5.,   6.]],\n           [[ 8.,   9.,  10.],\n            [12.,  13.,  14.]]]),\n     array([[[ 3.],\n            [ 7.]],\n           [[11.],\n            [15.]]]),\n    array([], shape=(2, 2, 0), dtype=float64)]\n    \"\"\"\n    if isinstance(indices_or_sections, set):\n        indices_or_sections = list(indices_or_sections)\n    return list(_api_internal.dsplit(ary, indices_or_sections))\n# pylint: enable=redefined-outer-name\n\n\n@set_module('mxnet.ndarray.numpy')\ndef concatenate(seq, axis=0, out=None):\n    \"\"\"\n    Join a sequence of arrays along an existing axis.\n\n    Parameters\n    ----------\n    a1, a2, ... : sequence of ndarray\n        The arrays must have the same shape, except in the dimension\n        corresponding to `axis` (the first, by default).\n    axis : int, optional\n        The axis along which the arrays will be joined.  If axis is None,\n        arrays are flattened before use.  Default is 0.\n    out : ndarray, optional\n        If provided, the destination to place the result. The shape must be\n        correct, matching that of what concatenate would have returned if no\n        out argument were specified.\n\n    Returns\n    -------\n    res : ndarray\n        The concatenated array.\n\n    Examples\n    --------\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> b = np.array([[5, 6]])\n    >>> np.concatenate((a, b), axis=0)\n    array([[1., 2.],\n           [3., 4.],\n           [5., 6.]])\n\n    >>> np.concatenate((a, b), axis=None)\n    array([1., 2., 3., 4., 5., 6.])\n\n    >>> np.concatenate((a, b.T), axis=1)\n    array([[1., 2., 5.],\n           [3., 4., 6.]])\n    \"\"\"\n    return _api_internal.concatenate(*seq, axis, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef append(arr, values, axis=None):  # pylint: disable=redefined-outer-name\n    \"\"\"\n    Append values to the end of an array.\n\n    Parameters\n    ----------\n    arr : ndarray\n        Values are appended to a copy of this array.\n    values : ndarray\n        These values are appended to a copy of `arr`.  It must be of the\n        correct shape (the same shape as `arr`, excluding `axis`).  If\n        `axis` is not specified, `values` can be any shape and will be\n        flattened before use.\n    axis : int, optional\n        The axis along which `values` are appended.  If `axis` is not\n        given, both `arr` and `values` are flattened before use.\n\n    Returns\n    -------\n    append : ndarray\n        A copy of `arr` with `values` appended to `axis`.  Note that\n        `append` does not occur in-place: a new array is allocated and\n        filled.  If `axis` is None, `out` is a flattened array.\n\n    Examples\n    --------\n    >>> np.append(np.array([1, 2, 3]), np.array([[4, 5, 6],[7, 8, 9]]))\n    array([1., 2., 3., 4., 5., 6., 7., 8., 9.])\n\n    When `axis` is specified, `values` must have the correct shape.\n\n    >>> np.append(np.array([[1, 2, 3], [4, 5, 6]]), np.array([[7, 8, 9]]), axis=0)\n    array([[1., 2., 3.],\n           [4., 5., 6.],\n           [7., 8., 9.]])\n    \"\"\"\n    out = None\n    return _api_internal.concatenate(arr, values, axis, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef stack(arrays, axis=0, out=None):\n    \"\"\"Join a sequence of arrays along a new axis.\n        The axis parameter specifies the index of the new axis in the dimensions of the result.\n        For example, if `axis=0` it will be the first dimension and if `axis=-1` it will be the last dimension.\n\n    Parameters\n    ----------\n    arrays : sequence of ndarray\n        Each array must have the same shape.\n    axis : int, optional\n        The axis in the result array along which the input arrays are stacked.\n    out : ndarray, optional\n        If provided, the destination to place the result. The shape must be correct,\n        matching that of what stack would have returned if no out argument were specified.\n\n    Returns\n    -------\n    stacked : ndarray\n        The stacked array has one more dimension than the input arrays.\"\"\"\n    def get_list(arrays):\n        if not hasattr(arrays, '__getitem__') and hasattr(arrays, '__iter__'):\n            raise ValueError(\"expected iterable for arrays but got {}\".format(type(arrays)))\n        return [arr for arr in arrays]\n\n    arrays = get_list(arrays)\n    return _api_internal.stack(*arrays, axis, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef vstack(arrays, out=None):\n    r\"\"\"Stack arrays in sequence vertically (row wise).\n\n    This is equivalent to concatenation along the first axis after 1-D arrays\n    of shape `(N,)` have been reshaped to `(1,N)`. Rebuilds arrays divided by\n    `vsplit`.\n\n    This function makes most sense for arrays with up to 3 dimensions. For\n    instance, for pixel-data with a height (first axis), width (second axis),\n    and r/g/b channels (third axis). The functions `concatenate` and `stack`\n    provide more general stacking and concatenation operations.\n\n    Parameters\n    ----------\n    tup : sequence of ndarrays\n        The arrays must have the same shape along all but the first axis.\n        1-D arrays must have the same length.\n\n    Returns\n    -------\n    stacked : ndarray\n        The array formed by stacking the given arrays, will be at least 2-D.\n\n    Examples\n    --------\n    >>> a = np.array([1, 2, 3])\n    >>> b = np.array([2, 3, 4])\n    >>> np.vstack((a, b))\n    array([[1., 2., 3.],\n            [2., 3., 4.]])\n\n    >>> a = np.array([[1], [2], [3]])\n    >>> b = np.array([[2], [3], [4]])\n    >>> np.vstack((a, b))\n    array([[1.],\n            [2.],\n            [3.],\n            [2.],\n            [3.],\n            [4.]])\n    \"\"\"\n    def get_list(arrays):\n        if not hasattr(arrays, '__getitem__') and hasattr(arrays, '__iter__'):\n            raise ValueError(\"expected iterable for arrays but got {}\".format(type(arrays)))\n        return [arr for arr in arrays]\n\n    arrays = get_list(arrays)\n    return _api_internal.vstack(*arrays)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef row_stack(arrays):\n    r\"\"\"Stack arrays in sequence vertically (row wise).\n    This is equivalent to concatenation along the first axis after 1-D arrays\n    of shape `(N,)` have been reshaped to `(1,N)`. Rebuilds arrays divided by\n    `vsplit`.\n    This function makes most sense for arrays with up to 3 dimensions. For\n    instance, for pixel-data with a height (first axis), width (second axis),\n    and r/g/b channels (third axis). The functions `concatenate` and `stack`\n    provide more general stacking and concatenation operations.\n    Parameters\n    ----------\n    tup : sequence of ndarrays\n        The arrays must have the same shape along all but the first axis.\n        1-D arrays must have the same length.\n    Returns\n    -------\n    stacked : ndarray\n        The array formed by stacking the given arrays, will be at least 2-D.\n    Examples\n    --------\n    >>> a = np.array([1, 2, 3])\n    >>> b = np.array([2, 3, 4])\n    >>> np.vstack((a, b))\n    array([[1., 2., 3.],\n            [2., 3., 4.]])\n    >>> a = np.array([[1], [2], [3]])\n    >>> b = np.array([[2], [3], [4]])\n    >>> np.vstack((a, b))\n    array([[1.],\n            [2.],\n            [3.],\n            [2.],\n            [3.],\n            [4.]])\n    \"\"\"\n    def get_list(arrays):\n        if not hasattr(arrays, '__getitem__') and hasattr(arrays, '__iter__'):\n            raise ValueError(\"expected iterable for arrays but got {}\".format(type(arrays)))\n        return [arr for arr in arrays]\n\n    arrays = get_list(arrays)\n    return _api_internal.vstack(*arrays)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef column_stack(tup):\n    \"\"\"\n    Stack 1-D arrays as columns into a 2-D array.\n    Take a sequence of 1-D arrays and stack them as columns\n    to make a single 2-D array. 2-D arrays are stacked as-is,\n    just like with `hstack`.  1-D arrays are turned into 2-D columns\n    first.\n\n    Returns\n    --------\n    stacked : 2-D array\n        The array formed by stacking the given arrays.\n\n    See Also\n    --------\n    stack, hstack, vstack, concatenate\n\n    Examples\n    --------\n    >>> a = np.array((1,2,3))\n    >>> b = np.array((2,3,4))\n    >>> np.column_stack((a,b))\n    array([[1., 2.],\n           [2., 3.],\n           [3., 4.]])\n    \"\"\"\n    return _api_internal.column_stack(*tup)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef hstack(arrays):\n    \"\"\"\n    Stack arrays in sequence horizontally (column wise).\n    This is equivalent to concatenation along the second axis,\n    except for 1-D arrays where it concatenates along the first axis.\n    Rebuilds arrays divided by hsplit.\n    This function makes most sense for arrays with up to 3 dimensions.\n    For instance, for pixel-data with a height (first axis), width (second axis),\n    and r/g/b channels (third axis). The functions concatenate,\n    stack and block provide more general stacking and concatenation operations.\n\n    Parameters\n    ----------\n    tup : sequence of ndarrays\n        The arrays must have the same shape along all but the second axis, except 1-D arrays which can be any length.\n\n    Returns\n    -------\n    stacked : ndarray\n        The array formed by stacking the given arrays.\n\n    Examples\n    --------\n    >>> from mxnet import np,npx\n    >>> a = np.array((1,2,3))\n    >>> b = np.array((2,3,4))\n    >>> np.hstack((a,b))\n    array([1., 2., 3., 2., 3., 4.])\n    >>> a = np.array([[1],[2],[3]])\n    >>> b = np.array([[2],[3],[4]])\n    >>> np.hstack((a,b))\n    array([[1., 2.],\n           [2., 3.],\n           [3., 4.]])\n    \"\"\"\n    return _api_internal.hstack(*arrays)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef dstack(arrays):\n    \"\"\"\n    Stack arrays in sequence depth wise (along third axis).\n    This is equivalent to concatenation along the third axis after 2-D arrays\n    of shape `(M,N)` have been reshaped to `(M,N,1)` and 1-D arrays of shape\n    `(N,)` have been reshaped to `(1,N,1)`. Rebuilds arrays divided by\n    `dsplit`.\n    This function makes most sense for arrays with up to 3 dimensions. For\n    instance, for pixel-data with a height (first axis), width (second axis),\n    and r/g/b channels (third axis). The functions `concatenate`, `stack` and\n    `block` provide more general stacking and concatenation operations.\n\n    Parameters\n    ----------\n    tup : sequence of arrays\n        The arrays must have the same shape along all but the third axis.\n        1-D or 2-D arrays must have the same shape.\n\n    Returns\n    -------\n    stacked : ndarray\n        The array formed by stacking the given arrays, will be at least 3-D.\n\n    Examples\n    --------\n    >>> a = np.array((1,2,3))\n    >>> b = np.array((2,3,4))\n    >>> np.dstack((a,b))\n    array([[[1, 2],\n            [2, 3],\n            [3, 4]]])\n    >>> a = np.array([[1],[2],[3]])\n    >>> b = np.array([[2],[3],[4]])\n    >>> np.dstack((a,b))\n    array([[[1, 2]],\n           [[2, 3]],\n           [[3, 4]]])\n    \"\"\"\n    return _api_internal.dstack(*arrays)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef maximum(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns element-wise maximum of the input arrays with broadcasting.\n\n    Parameters\n    ----------\n    x1, x2 : scalar or mxnet.numpy.ndarray\n        The arrays holding the elements to be compared. They must have the same shape,\n        or shapes that can be broadcast to a single shape.\n\n    Returns\n    -------\n    out : mxnet.numpy.ndarray or scalar\n        The maximum of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars.\"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.maximum(x1, x2, out=out)\n    return _api_internal.maximum(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef fmax(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns element-wise maximum of the input arrays with broadcasting. (Ignores NaNs)\n\n    Parameters\n    ----------\n    x1, x2 : scalar or mxnet.numpy.ndarray\n        The arrays holding the elements to be compared. They must have the same shape,\n        or shapes that can be broadcast to a single shape.\n\n    Returns\n    -------\n    out : mxnet.numpy.ndarray or scalar\n        The maximum of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars.\"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        _np.fmax(x1, x2, out=out)\n    return _api_internal.fmax(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef minimum(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns element-wise minimum of the input arrays with broadcasting.\n\n    Parameters\n    ----------\n    x1, x2 : scalar or mxnet.numpy.ndarray\n        The arrays holding the elements to be compared. They must have the same shape,\n        or shapes that can be broadcast to a single shape.\n\n    Returns\n    -------\n    out : mxnet.numpy.ndarray or scalar\n        The minimum of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars.\"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.minimum(x1, x2, out=out)\n    return _api_internal.minimum(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef fmin(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns element-wise minimum of the input arrays with broadcasting. (Ignores NaNs)\n\n    Parameters\n    ----------\n    x1, x2 : scalar or mxnet.numpy.ndarray\n        The arrays holding the elements to be compared. They must have the same shape,\n        or shapes that can be broadcast to a single shape.\n\n    Returns\n    -------\n    out : mxnet.numpy.ndarray or scalar\n        The minimum of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars.\"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        _np.fmin(x1, x2, out=out)\n    return _api_internal.fmin(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef max(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Return the maximum of an array or maximum along an axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data.\n    axis : int, optional\n        Axis along which to operate.  By default, flattened input is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result.  Must\n        be of the same shape and buffer length as the expected output.\n        See `doc.ufuncs` (Section \"Output arguments\") for more details.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the original `arr`.\n\n    Returns\n    -------\n    max : ndarray\n        Maximum of `a`. If `axis` is None, the result is an array of dimension 1.\n        If `axis` is given, the result is an array of dimension\n        ``a.ndim - 1``.\n\n    See Also\n    --------\n    min :\n        The minimum value of an array along a given axis, ignoring any nan.\n    maximum :\n        Element-wise maximum of two arrays, ignoring any nan.\n    argmax :\n        Return the indices of the maximum values.\n\n    Notes\n    -----\n    NaN in the orginal `numpy` is denoted as nan and will be ignored.\n\n    Don't use `max` for element-wise comparison of 2 arrays; when\n    ``a.shape[0]`` is 2, ``maximum(a[0], a[1])`` is faster than\n    ``max(a, axis=0)``.\n\n    Examples\n    --------\n    >>> a = np.arange(4).reshape((2,2))\n    >>> a\n    array([[0., 1.],\n        [2., 3.]])\n    >>> np.max(a)            # Maximum of the flattened array\n    array(3.)\n    >>> np.max(a, axis=0)    # Maxima along the first axis\n    array([2., 3.])\n    >>> np.max(a, axis=1)    # Maxima along the second axis\n    array([1., 3.])\n\n    >>> b = np.arange(5, dtype=np.float32)\n    >>> b[2] = np.nan\n    >>> np.max(b)\n    array(4.)\n    \"\"\"\n    return _api_internal.max(a, axis, keepdims, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef min(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Return the minimum of an array or minimum along an axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data.\n    axis : int, optional\n        Axis along which to operate.  By default, flattened input is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result.  Must\n        be of the same shape and buffer length as the expected output.\n        See `doc.ufuncs` (Section \"Output arguments\") for more details.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the original `arr`.\n\n    Returns\n    -------\n    min : ndarray\n        Minimum of `a`. If `axis` is None, the result is an array of dimension 1.\n        If `axis` is given, the result is an array of dimension\n        ``a.ndim - 1``.\n\n    See Also\n    --------\n    max :\n        The maximum value of an array along a given axis, ignoring any nan.\n    minimum :\n        Element-wise minimum of two arrays, ignoring any nan.\n\n    Notes\n    -----\n    NaN in the orginal `numpy` is denoted as nan and will be ignored.\n\n    Don't use `min` for element-wise comparison of 2 arrays; when\n    ``a.shape[0]`` is 2, ``minimum(a[0], a[1])`` is faster than\n    ``min(a, axis=0)``.\n\n    Examples\n    --------\n    >>> a = np.arange(4).reshape((2,2))\n    >>> a\n    array([[0., 1.],\n        [2., 3.]])\n    >>> np.min(a)           # Minimum of the flattened array\n    array(0.)\n    >>> np.min(a, axis=0)   # Minima along the first axis\n    array([0., 1.])\n    >>> np.min(a, axis=1)   # Minima along the second axis\n    array([0., 2.])\n    >>> b = np.arange(5, dtype=np.float32)\n    >>> b[2] = np.nan\n    >>> np.min(b)\n    array(0.) # nan will be ignored\n    \"\"\"\n    return _api_internal.min(a, axis, keepdims, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef amax(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Return the maximum of an array or maximum along an axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data.\n    axis : int, optional\n        Axis along which to operate.  By default, flattened input is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result.  Must\n        be of the same shape and buffer length as the expected output.\n        See `doc.ufuncs` (Section \"Output arguments\") for more details.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the original `arr`.\n\n    Returns\n    -------\n    max : ndarray\n        Maximum of `a`. If `axis` is None, the result is an array of dimension 1.\n        If `axis` is given, the result is an array of dimension\n        ``a.ndim - 1``.\n\n    See Also\n    --------\n    min :\n        The minimum value of an array along a given axis, ignoring any nan.\n    maximum :\n        Element-wise maximum of two arrays, ignoring any nan.\n    argmax :\n        Return the indices of the maximum values.\n\n    Notes\n    -----\n    NaN in the orginal `numpy` is denoted as nan and will be ignored.\n\n    Don't use `max` for element-wise comparison of 2 arrays; when\n    ``a.shape[0]`` is 2, ``maximum(a[0], a[1])`` is faster than\n    ``max(a, axis=0)``.\n\n    Examples\n    --------\n    >>> a = np.arange(4).reshape((2,2))\n    >>> a\n    array([[0., 1.],\n        [2., 3.]])\n    >>> np.max(a)            # Maximum of the flattened array\n    array(3.)\n    >>> np.max(a, axis=0)    # Maxima along the first axis\n    array([2., 3.])\n    >>> np.max(a, axis=1)    # Maxima along the second axis\n    array([1., 3.])\n\n    >>> b = np.arange(5, dtype=np.float32)\n    >>> b[2] = np.nan\n    >>> np.max(b)\n    array(4.)\n    \"\"\"\n    return _api_internal.amax(a, axis, keepdims, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef amin(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Return the minimum of an array or minimum along an axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data.\n    axis : int, optional\n        Axis along which to operate.  By default, flattened input is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result.  Must\n        be of the same shape and buffer length as the expected output.\n        See `doc.ufuncs` (Section \"Output arguments\") for more details.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the original `arr`.\n\n    Returns\n    -------\n    min : ndarray\n        Minimum of `a`. If `axis` is None, the result is an array of dimension 1.\n        If `axis` is given, the result is an array of dimension\n        ``a.ndim - 1``.\n\n    See Also\n    --------\n    max :\n        The maximum value of an array along a given axis, ignoring any nan.\n    minimum :\n        Element-wise minimum of two arrays, ignoring any nan.\n\n    Notes\n    -----\n    NaN in the orginal `numpy` is denoted as nan and will be ignored.\n\n    Don't use `min` for element-wise comparison of 2 arrays; when\n    ``a.shape[0]`` is 2, ``minimum(a[0], a[1])`` is faster than\n    ``min(a, axis=0)``.\n\n    Examples\n    --------\n    >>> a = np.arange(4).reshape((2,2))\n    >>> a\n    array([[0., 1.],\n        [2., 3.]])\n    >>> np.min(a)           # Minimum of the flattened array\n    array(0.)\n    >>> np.min(a, axis=0)   # Minima along the first axis\n    array([0., 1.])\n    >>> np.min(a, axis=1)   # Minima along the second axis\n    array([0., 2.])\n    >>> b = np.arange(5, dtype=np.float32)\n    >>> b[2] = np.nan\n    >>> np.min(b)\n    array(0.) # nan will be ignored\n    \"\"\"\n    return _api_internal.amin(a, axis, keepdims, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef swapaxes(a, axis1, axis2):\n    \"\"\"Interchange two axes of an array.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array.\n    axis1 : int\n        First axis.\n    axis2 : int\n        Second axis.\n\n    Returns\n    -------\n    a_swapped : ndarray\n        Swapped array. This is always a copy of the input array.\n    \"\"\"\n    return _npi.swapaxes(a, dim1=axis1, dim2=axis2)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef clip(a, a_min, a_max, out=None):\n    \"\"\"clip(a, a_min, a_max, out=None)\n\n    Clip (limit) the values in an array.\n    Given an interval, values outside the interval are clipped to\n    the interval edges.  For example, if an interval of ``[0, 1]``\n    is specified, values smaller than 0 become 0, and values larger\n    than 1 become 1.\n\n    Parameters\n    ----------\n    a : ndarray\n        Array containing elements to clip.\n    a_min : scalar or `None`\n        Minimum value. If `None`, clipping is not performed on lower\n        interval edge. Not more than one of `a_min` and `a_max` may be\n        `None`.\n    a_max : scalar or `None`\n        Maximum value. If `None`, clipping is not performed on upper\n        interval edge. Not more than one of `a_min` and `a_max` may be\n        `None`.\n    out : ndarray, optional\n        The results will be placed in this array. It may be the input\n        array for in-place clipping.  `out` must be of the right shape\n        to hold the output.  Its type is preserved.\n\n    Returns\n    -------\n    clipped_array : ndarray\n        An array with the elements of `a`, but where values\n        < `a_min` are replaced with `a_min`, and those > `a_max`\n        with `a_max`.\n\n    Notes\n    -----\n    ndarray `a_min` and `a_max` are not supported.\n\n    Examples\n    --------\n    >>> a = np.arange(10)\n    >>> np.clip(a, 1, 8)\n    array([1., 1., 2., 3., 4., 5., 6., 7., 8., 8.])\n    >>> a\n    array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.])\n    >>> np.clip(a, 3, 6, out=a)\n    array([3., 3., 3., 3., 4., 5., 6., 6., 6., 6.])\n    \"\"\"\n    if a_min is None and a_max is None:\n        raise ValueError('array_clip: must set either max or min')\n    return _api_internal.clip(a, a_min, a_max, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef tril_indices(n, k=0, m=None):\n    \"\"\"\n    Return the indices for the lower-triangle of an (n, m) array.\n\n    Parameters\n    ----------\n    n : int\n        The row dimension of the arrays for which the returned\n        indices will be valid.\n    k : int, optional\n        Diagonal offset (see `tril` for details).\n    m : int, optional\n        .. versionadded:: 1.9.0\n\n        The column dimension of the arrays for which the returned\n        arrays will be valid.\n        By default `m` is taken equal to `n`.\n\n    Returns\n    -------\n    inds : tuple of arrays\n        The indices for the triangle. The returned tuple contains two arrays,\n        each with the indices along one dimension of the array.\n\n    See also\n    --------\n    triu_indices : similar function, for upper-triangular.\n    mask_indices : generic function accepting an arbitrary mask function.\n    tril, triu\n\n    Notes\n    -----\n    .. versionadded:: 1.4.0\n\n    Examples\n    --------\n    Compute two different sets of indices to access 4x4 arrays, one for the\n    lower triangular part starting at the main diagonal, and one starting two\n    diagonals further right:\n\n    >>> il1 = np.tril_indices(4)\n    >>> il2 = np.tril_indices(4, 2)\n\n    Here is how they can be used with a sample array:\n\n    >>> a = np.arange(16).reshape(4, 4)\n    >>> a\n    array([[ 0,  1,  2,  3],\n           [ 4,  5,  6,  7],\n           [ 8,  9, 10, 11],\n           [12, 13, 14, 15]])\n\n    Both for indexing:\n\n    >>> a[il1]\n    array([ 0,  4,  5,  8,  9, 10, 12, 13, 14, 15])\n\n    And for assigning values:\n\n    >>> a[il1] = -1\n    >>> a\n    array([[-1,  1,  2,  3],\n           [-1, -1,  6,  7],\n           [-1, -1, -1, 11],\n           [-1, -1, -1, -1]])\n\n    These cover almost the whole array (two diagonals right of the main one):\n\n    >>> a[il2] = -10\n    >>> a\n    array([[-10, -10, -10,   3],\n           [-10, -10, -10, -10],\n           [-10, -10, -10, -10],\n           [-10, -10, -10, -10]])\n\n    \"\"\"\n    if m is None:\n        m = n\n    return tuple(_api_internal.tril_indices(n, k, m))\n\n\n@set_module('mxnet.ndarray.numpy')\ndef argmax(a, axis=None, out=None, keepdims=False):\n    r\"\"\"\n    Returns the indices of the maximum values along an axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array. Only support ndarrays of dtype `float16`, `float32`, and `float64`.\n    axis : int, optional\n        By default, the index is into the flattened array, otherwise\n        along the specified axis.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n    keepdims : bool\n        If True, the reduced axes (dimensions) must be included in the result as\n        singleton dimensions, and, accordingly, the result must be compatible with\n        the input array. Otherwise, if False, the reduced axes (dimensions) must\n        not be included in the result. Default: False .\n\n    Returns\n    -------\n    index_array : ndarray of indices whose dtype is same as the input ndarray.\n        Array of indices into the array. It has the same shape as `a.shape`\n        with the dimension along `axis` removed.\n\n    Notes\n    -----\n    ``keepdims`` param is part of request in data-api-standard\n    <https://data-apis.org/array-api/latest/API_specification/generated/signatures.searching_functions.argmax.html>`_,\n    which is not the parameter in official NumPy\n\n    In case of multiple occurrences of the maximum values, the indices\n    corresponding to the first occurrence are returned.\n\n    This function differs from the original `numpy.argmax\n    <https://numpy.org/doc/stable/reference/generated/numpy.argmax.html>`_ in\n    the following aspects:\n\n    - Input type does not support Python native iterables(list, tuple, ...).\n    - ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be the same as the expected output.\n    - ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the same as the expected output.\n    - ``out`` param does not support scalar input case.\n\n    Examples\n    --------\n    >>> a = np.arange(6).reshape(2,3) + 10\n    >>> a\n    array([[10., 11., 12.],\n           [13., 14., 15.]])\n    >>> np.argmax(a)\n    array(5.)\n    >>> np.argmax(a, axis=0)\n    array([1., 1., 1.])\n    >>> np.argmax(a, axis=1)\n    array([2., 2.])\n\n    >>> b = np.arange(6)\n    >>> b[1] = 5\n    >>> b\n    array([0., 5., 2., 3., 4., 5.])\n    >>> np.argmax(b)  # Only the first occurrence is returned.\n    array(1.)\n\n    Specify ``out`` ndarray:\n\n    >>> a = np.arange(6).reshape(2,3) + 10\n    >>> b = np.zeros((2,))\n    >>> np.argmax(a, axis=1, out=b)\n    array([2., 2.])\n    >>> b\n    array([2., 2.])\n    \"\"\"\n    return _api_internal.argmax(a, axis, keepdims, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef argmin(a, axis=None, out=None, keepdims=False):\n    r\"\"\"\n    Returns the indices of the maximum values along an axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array. Only support ndarrays of dtype `float16`, `float32`, and `float64`.\n    axis : int, optional\n        By default, the index is into the flattened array, otherwise\n        along the specified axis.\n    out : ndarray or None, optional\n        If provided, the result will be inserted into this array. It should\n        be of the appropriate shape and dtype.\n    keepdims : bool\n        If True, the reduced axes (dimensions) must be included in the result as\n        singleton dimensions, and, accordingly, the result must be compatible with\n        the input array. Otherwise, if False, the reduced axes (dimensions) must\n        not be included in the result. Default: False .\n\n    Returns\n    -------\n    index_array : ndarray of indices whose dtype is same as the input ndarray.\n        Array of indices into the array. It has the same shape as `a.shape`\n        with the dimension along `axis` removed.\n\n    Notes\n    -----\n    ``keepdims`` param is part of request in data-api-standard\n    <https://data-apis.org/array-api/latest/API_specification/generated/signatures.searching_functions.argmin.html>`_,\n    which is not the parameter in official NumPy\n\n    In case of multiple occurrences of the maximum values, the indices\n    corresponding to the first occurrence are returned.\n\n    This function differs from the original `numpy.argmax\n    <https://numpy.org/doc/stable/reference/generated/numpy.argmax.html>`_ in\n    the following aspects:\n\n    - Input type does not support Python native iterables(list, tuple, ...).\n    - ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be the same as the expected output.\n    - ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the same as the expected output.\n    - ``out`` param does not support scalar input case.\n\n    Examples\n    --------\n    >>> a = np.arange(6).reshape(2,3) + 10\n    >>> a\n    array([[10., 11., 12.],\n           [13., 14., 15.]])\n    >>> np.argmin(a)\n    array(0.)\n    >>> np.argmin(a, axis=0)\n    array([0., 0., 0.])\n    >>> np.argmin(a, axis=1)\n    array([0., 0.])\n\n    >>> b = np.arange(6)\n    >>> b[2] = 0\n    >>> b\n    array([0., 1., 0., 3., 4., 5.])\n    >>> np.argmax(b)  # Only the first occurrence is returned.\n    array(0.)\n\n    Specify ``out`` ndarray:\n\n    >>> a = np.arange(6).reshape(2,3) + 10\n    >>> b = np.zeros((2,))\n    >>> np.argmin(a, axis=1, out=b)\n    array([0., 0.])\n    >>> b\n    array([0., 0.])\n    \"\"\"\n    return _api_internal.argmin(a, axis, keepdims, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef average(a, axis=None, weights=None, returned=False, out=None):\n    \"\"\"\n    Compute the weighted average along the specified axis.\n\n    Parameters\n    --------\n    a : ndarray\n        Array containing data to be averaged.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which to average a.\n        The default, axis=None, will average over\n        all of the elements of the input array.\n        If axis is negative it counts from the last to the first axis.\n        New in version 1.7.0.\n        If axis is a tuple of ints, averaging is\n        performed on all of the axes specified in the tuple\n        instead of a single axis or all the axes as before.\n    weights : ndarray, optional\n        An array of weights associated with the values in a, must be the same dtype with a.\n        Each value in a contributes to the average according to its associated weight.\n        The weights array can either be 1-D (in which case its length must be\n        the size of a along the given axis) or of the same shape as a.\n        If weights=None, then all data in a are assumed to have a weight equal to one.\n        The 1-D calculation is: avg = sum(a * weights) / sum(weights)\n        The only constraint on weights is that sum(weights) must not be 0.\n    returned : bool, optional\n        Default is False.\n        If True, the tuple (average, sum_of_weights) is returned,\n        otherwise only the average is returned.\n        If weights=None, sum_of_weights is equivalent to\n        the number of elements over which the average is taken.\n    out : ndarray, optional\n        If provided, the calculation is done into this array.\n\n    Returns\n    --------\n    retval, [sum_of_weights] : ndarray\n        Return the average along the specified axis.\n        When returned is True, return a tuple with the average as the first element\n        and the sum of the weights as the second element. sum_of_weights is of the same type as retval.\n        If a is integral, the result dtype will be current default dtype, otherwise it will be the same\n        as dtype of a. (i.e. When npx.is_np_default_dtype() returns False, default dtype is float32; When\n        npx.is_np_default_dtype() returns True, default dtype is float64.)\n\n    Raises\n    --------\n        MXNetError\n        - When all weights along axis sum to zero.\n        - When the length of 1D weights is not the same as the shape of a along axis.\n        - When given 1D weights, the axis is not specified or is not int.\n        - When the shape of weights and a differ, but weights are not 1D.\n\n    See also\n    --------\n        mean\n\n    Notes\n    --------\n    This function differs from the original `numpy.average`\n    <https://numpy.org/devdocs/reference/generated/numpy.average.html>`_ in\n    the following way(s):\n\n    - Does not guarantee the same behavior with numpy when given float16 dtype and overflow happens\n    - Does not support complex dtype\n    - The dtypes of a and weights must be the same\n    - Integral a results in default dtype.\n      i.e. When npx.is_np_default_dtype() returns False, default dtype is float32;\n      When npx.is_np_default_dtype() returns True, default dtype is float64.\n\n    Examples\n    --------\n    >>> data = np.arange(1, 5)\n    >>> data\n    array([1., 2., 3., 4.])\n    >>> np.average(data)\n    array(2.5)\n    >>> np.average(np.arange(1, 11), weights=np.arange(10, 0, -1))\n    array(4.)\n    >>> data = np.arange(6).reshape((3,2))\n    >>> data\n    array([[0., 1.],\n           [2., 3.],\n           [4., 5.]])\n    >>> weights = np.array([0.25, 0.75])\n    array([0.25, 0.75])\n    >>> np.average(data, axis=1, weights=weights)\n    array([0.75, 2.75, 4.75])\n    \"\"\"\n    out = _api_internal.average(a, weights, axis, returned, weights is not None, out)\n    if isinstance(out, NDArray):\n        return out\n    else:\n        return list(out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef mean(a, axis=None, dtype=None, out=None, keepdims=False):  # pylint: disable=arguments-differ\n    \"\"\"\n    mean(a, axis=None, dtype=None, out=None, keepdims=None)\n    Compute the arithmetic mean along the specified axis.\n    Returns the average of the array elements.\n    The average is taken over the flattened array by default, otherwise over the specified axis.\n    Parameters\n    ----------\n    a : ndarray\n        ndarray containing numbers whose mean is desired.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which the means are computed. The default is to compute the mean of the flattened array.\n        If this is a tuple of ints, a mean is performed over multiple axes,\n        instead of a single axis or all the axes as before.\n    dtype : data-type, optional\n        Type to use in computing the mean.\n        For integer inputs, the default is your current default dtype (i.e. When npx.is_np_default_dtype() returns\n        False, default dtype is float32; When npx.is_np_default_dtype() returns True, default dtype is float64.);\n        For floating point inputs, it is the same as the input dtype.\n    out : ndarray, optional\n        Alternate output array in which to place the result. The default is None; if provided,\n        it must have the same shape and type as the expected output\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in the result\n        as dimensions with size one. With this option, the result will broadcast correctly\n        against the input array.\n        If the default value is passed, then keepdims will not be passed through to the mean\n        method of sub-classes of ndarray, however any non-default value will be. If the sub-class\n        method does not implement keepdims any exceptions will be raised.\n    Returns\n    -------\n    m : ndarray, see dtype parameter above\n        If out=None, returns a new array containing the mean values,\n        otherwise a reference to the output array is returned.\n    Notes\n    -----\n    This function differs from the original `numpy.mean\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.mean.html>`_ in\n    the following way(s):\n    - only ndarray is accepted as valid input, python iterables or scalar is not supported\n    - default data type for integer input is float32 or float64, which depends on your current default dtype.\n      When npx.is_np_default_dtype() returns False, default dtype is float32;\n      When npx.is_np_default_dtype() returns True, default dtype is float64.\n    Examples\n    --------\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> np.mean(a)\n    array(2.5)\n    >>> a = np.zeros((2, 512*512), dtype=np.float32)\n    >>> a[0,:] = 1.0\n    >>> a[1,:] = 0.1\n    >>> np.mean(a)\n    array(0.55)\n    >>> np.mean(a, dtype=np.float64)\n    array(0.55)\n    \"\"\"\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    return _api_internal.mean(a, axis, dtype, keepdims, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):  # pylint: disable=too-many-arguments\n    \"\"\"\n    Compute the standard deviation along the specified axis.\n    Returns the standard deviation, a measure of the spread of a distribution,\n    of the array elements. The standard deviation is computed for the\n    flattened array by default, otherwise over the specified axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Calculate the standard deviation of these values.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which the standard deviation is computed. The\n        default is to compute the standard deviation of the flattened array.\n        .. versionadded:: 1.7.0\n        If this is a tuple of ints, a standard deviation is performed over\n        multiple axes, instead of a single axis or all the axes as before.\n    dtype : dtype, optional\n        Type to use in computing the standard deviation. For arrays of\n        integer type the default is float64, for arrays of float types it is\n        the same as the array type.\n    out : ndarray, optional\n        Alternative output array in which to place the result. It must have\n        the same shape as the expected output but the type (of the calculated\n        values) will be cast if necessary.\n    ddof : int, optional\n        Means Delta Degrees of Freedom.  The divisor used in calculations\n        is ``N - ddof``, where ``N`` represents the number of elements.\n        By default `ddof` is zero.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the input array.\n        If the default value is passed, then `keepdims` will not be\n        passed through to the `std` method of sub-classes of\n        `ndarray`, however any non-default value will be.  If the\n        sub-class' method does not implement `keepdims` any\n        exceptions will be raised.\n\n    Returns\n    -------\n    standard_deviation : ndarray, see dtype parameter above.\n        If `out` is None, return a new array containing the standard deviation,\n        otherwise return a reference to the output array.\n\n    Examples\n    --------\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> np.std(a)\n    1.1180339887498949 # may vary\n    >>> np.std(a, axis=0)\n    array([1.,  1.])\n    >>> np.std(a, axis=1)\n    array([0.5,  0.5])\n    In single precision, std() can be inaccurate:\n    >>> a = np.zeros((2, 512*512), dtype=np.float32)\n    >>> a[0, :] = 1.0\n    >>> a[1, :] = 0.1\n    >>> np.std(a)\n    array(0.45)\n    >>> np.std(a, dtype=np.float64)\n    array(0.45, dtype=float64)\n    \"\"\"\n    return _api_internal.std(a, axis, dtype, ddof, keepdims, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):  # pylint: disable=too-many-arguments\n    \"\"\"\n    Compute the variance along the specified axis.\n    Returns the variance of the array elements, a measure of the spread of a\n    distribution.  The variance is computed for the flattened array by\n    default, otherwise over the specified axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Array containing numbers whose variance is desired.  If `a` is not an\n        array, a conversion is attempted.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which the variance is computed.  The default is to\n        compute the variance of the flattened array.\n        .. versionadded:: 1.7.0\n        If this is a tuple of ints, a variance is performed over multiple axes,\n        instead of a single axis or all the axes as before.\n    dtype : data-type, optional\n        Type to use in computing the variance.\n        For arrays of integer type the default is `float32` or 'float64',\n        When npx.is_np_default_dtype() returns False, default dtype is float32,\n        When npx.is_np_default_dtype() returns True, default dtype is float64;\n        For arrays of float types it is the same as the array type.\n    out : ndarray, optional\n        Alternate output array in which to place the result.  It must have\n        the same shape as the expected output, but the type is cast if\n        necessary.\n    ddof : int, optional\n        \"Delta Degrees of Freedom\": the divisor used in the calculation is\n        ``N - ddof``, where ``N`` represents the number of elements. By\n        default `ddof` is zero.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the input array.\n        If the default value is passed, then `keepdims` will not be\n        passed through to the `var` method of sub-classes of\n        `ndarray`, however any non-default value will be.  If the\n        sub-class' method does not implement `keepdims` any\n        exceptions will be raised.\n\n    Returns\n    -------\n    variance : ndarray, see dtype parameter above\n        If ``out=None``, returns a new array containing the variance;\n        otherwise, a reference to the output array is returned.\n\n    Examples\n    --------\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> np.var(a)\n    array(1.25)\n    >>> np.var(a, axis=0)\n    array([1.,  1.])\n    >>> np.var(a, axis=1)\n    array([0.25,  0.25])\n\n    >>> a = np.zeros((2, 512*512), dtype=np.float32)\n    >>> a[0, :] = 1.0\n    >>> a[1, :] = 0.1\n    >>> np.var(a)\n    array(0.2025)\n    >>> np.var(a, dtype=np.float64)\n    array(0.2025, dtype=float64)\n    >>> ((1-0.55)**2 + (0.1-0.55)**2)/2\n    0.2025\n    \"\"\"\n    return _api_internal.var(a, axis, dtype, ddof, keepdims, out)\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.ndarray.numpy')\ndef indices(dimensions, dtype=None, device=None):\n    \"\"\"Return an array representing the indices of a grid.\n\n    Compute an array where the subarrays contain index values 0,1,...\n    varying only along the corresponding axis.\n\n    Parameters\n    ----------\n    dimensions : sequence of ints\n        The shape of the grid.\n    dtype : data-type, optional\n        The desired data-type for the array. Default is `int64`.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    grid : ndarray\n        The array of grid indices,\n        ``grid.shape = (len(dimensions),) + tuple(dimensions)``.\n\n    Notes\n    -----\n    The output shape is obtained by prepending the number of dimensions\n    in front of the tuple of dimensions, i.e. if `dimensions` is a tuple\n    ``(r0, ..., rN-1)`` of length ``N``, the output shape is\n    ``(N,r0,...,rN-1)``.\n\n    The subarrays ``grid[k]`` contains the N-D array of indices along the\n    ``k-th`` axis. Explicitly::\n\n        grid[k,i0,i1,...,iN-1] = ik\n\n    Examples\n    --------\n    >>> grid = np.indices((2, 3))\n    >>> grid.shape\n    (2, 2, 3)\n    >>> grid[0]        # row indices\n    array([[0, 0, 0],\n           [1, 1, 1]], dtype=int64)\n    >>> grid[1]        # column indices\n    array([[0, 0, 0],\n           [1, 1, 1]], dtype=int64)\n\n    The indices can be used as an index into an array.\n\n    >>> x = np.arange(20).reshape(5, 4)\n    >>> row, col = np.indices((2, 3))\n    >>> x[row, col]\n    array([[0., 1., 2.],\n           [4., 5., 6.]])\n\n    Note that it would be more straightforward in the above example to\n    extract the required elements directly with ``x[:2, :3]``.\n    \"\"\"\n    if isinstance(dimensions, (tuple, list)):\n        if device is None:\n            device = str(current_device())\n        else:\n            device = str(device)\n        if dtype is not None and not isinstance(dtype, str):\n            dtype = get_dtype_name(dtype)\n        return _api_internal.indices(dimensions, dtype, device)\n    else:\n        raise ValueError(\"The dimensions must be sequence of ints\")\n# pylint: enable=redefined-outer-name\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef copysign(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Change the sign of x1 to that of x2, element-wise.\n\n    If `x2` is a scalar, its sign will be copied to all elements of `x1`.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Values to change the sign of.\n    x2 : ndarray or scalar\n        The sign of `x2` is copied to `x1`.\n    out : ndarray or None, optional\n        A location into which the result is stored. It must be of the\n        right shape and right type to hold the output. If not provided\n        or `None`,a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        The values of `x1` with the sign of `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n\n    Notes\n    -------\n    This function differs from the original `numpy.copysign\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.copysign.html>`_ in\n    the following aspects:\n\n    - ``where`` param is not supported.\n\n    Examples\n    --------\n    >>> np.copysign(1.3, -1)\n    -1.3\n    >>> 1/np.copysign(0, 1)\n    inf\n    >>> 1/np.copysign(0, -1)\n    -inf\n\n    >>> a = np.array([-1, 0, 1])\n    >>> np.copysign(a, -1.1)\n    array([-1., -0., -1.])\n    >>> np.copysign(a, np.arange(3)-1)\n    array([-1.,  0.,  1.])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.copysign(x1, x2, out=out)\n    return _api_internal.copysign(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef ravel(x, order='C'):\n    r\"\"\"\n    ravel(x)\n\n    Return a contiguous flattened array.\n    A 1-D array, containing the elements of the input, is returned.  A copy is\n    made only if needed.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array.  The elements in `x` are read in row-major, C-style order and\n        packed as a 1-D array.\n    order : `C`, optional\n        Only support row-major, C-style order.\n\n    Returns\n    -------\n    y : ndarray\n        y is an array of the same subtype as `x`, with shape ``(x.size,)``.\n        Note that matrices are special cased for backward compatibility, if `x`\n        is a matrix, then y is a 1-D ndarray.\n\n    Notes\n    -----\n    This function differs from the original numpy.arange in the following aspects:\n        - Only support row-major, C-style order.\n\n    Examples\n    --------\n    It is equivalent to ``reshape(x, -1)``.\n\n    >>> x = np.array([[1, 2, 3], [4, 5, 6]])\n    >>> print(np.ravel(x))\n    [1. 2. 3. 4. 5. 6.]\n\n    >>> print(x.reshape(-1))\n    [1. 2. 3. 4. 5. 6.]\n\n    >>> print(np.ravel(x.T))\n    [1. 4. 2. 5. 3. 6.]\n    \"\"\"\n    if order == 'F':\n        raise NotImplementedError('order {} is not supported'.format(order))\n    if isinstance(x, numeric_types):\n        return _np.reshape(x, -1)\n    elif isinstance(x, NDArray):\n        return reshape(x, -1)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(x))))\n\n\n@set_module('mxnet.ndarray.numpy')\ndef unravel_index(indices, shape, order='C'): # pylint: disable=redefined-outer-name\n    \"\"\"\n    Converts a flat index or array of flat indices into a tuple of coordinate arrays.\n\n    Parameters:\n    -------------\n    indices : array_like\n            An integer array whose elements are indices into the flattened version of an array of dimensions shape.\n            Before version 1.6.0, this function accepted just one index value.\n    shape : tuple of ints\n            The shape of the array to use for unraveling indices.\n\n    Returns:\n    -------------\n    unraveled_coords : ndarray\n            Each row in the ndarray has the same shape as the indices array.\n            Each column in the ndarray represents the unravelled index\n\n    Examples:\n    -------------\n    >>> np.unravel_index([22, 41, 37], (7,6))\n    ([3. 6. 6.]\n      [4. 5. 1.])\n    >>> np.unravel_index(1621, (6,7,8,9))\n    (3, 1, 4, 1)\n    \"\"\"\n    if order == 'C':\n        if isinstance(indices, numeric_types):\n            return _np.unravel_index(indices, shape)\n        if isinstance(indices, NDArray):\n            return tuple(_api_internal.unravel_index(indices, shape))\n        raise TypeError('Do not support type {} as indices.'.format(str(type(indices))))\n    raise NotImplementedError('Do not support column-major (Fortran-style) order at this moment')\n\n\ndef flatnonzero(a):\n    r\"\"\"\n    Return indices that are non-zero in the flattened version of a.\n\n    This is equivalent to np.nonzero(np.ravel(a))[0].\n\n    Parameters\n    ----------\n    a : array_like\n        Input data.\n\n    Returns\n    -------\n    res : ndarray\n        Output array, containing the indices of the elements of `a.ravel()`\n        that are non-zero.\n\n    See Also\n    --------\n    nonzero : Return the indices of the non-zero elements of the input array.\n    ravel : Return a 1-D array containing the elements of the input array.\n\n    Examples\n    --------\n    >>> x = np.arange(-2, 3)\n    >>> x\n    array([-2, -1,  0,  1,  2])\n    >>> np.flatnonzero(x)\n    array([0, 1, 3, 4])\n\n    Use the indices of the non-zero elements as an index array to extract\n    these elements:\n\n    >>> x.ravel()[np.flatnonzero(x)]\n    array([-2, -1,  1,  2])\n    \"\"\"\n    return nonzero(ravel(a))[0]\n\n\n@set_module('mxnet.ndarray.numpy')\ndef diag_indices_from(arr):\n    \"\"\"\n    This returns a tuple of indices that can be used to access the main diagonal of an array\n    a with a.ndim >= 2 dimensions and shape (n, n, ..., n). For a.ndim = 2 this is\n    the usual diagonal, for a.ndim > 2 this is the set of indices to access\n    a[i, i, ..., i] for i = [0..n-1].\n\n    Parameters:\n    -------------\n    arr : ndarray\n        Input array for acessing the main diagonal. All dimensions\n        should have equal length.\n\n    Return:\n    -------------\n    diag: tuple of ndarray\n        indices of the main diagonal.\n\n    Examples:\n    -------------\n    >>> a = np.arange(16).reshape(4, 4)\n    >>> a\n    array([[ 0,  1,  2,  3],\n        [ 4,  5,  6,  7],\n        [ 8,  9, 10, 11],\n        [12, 13, 14, 15]])\n    >>> idx = np.diag_indices_from(a)\n    >>> idx\n    (array([0, 1, 2, 3]), array([0, 1, 2, 3]))\n    >>> a[idx] = 100\n    >>> a\n    array([[100,   1,   2,   3],\n        [  4, 100,   6,   7],\n        [  8,   9, 100,  11],\n        [ 12,  13,  14, 100]])\n    \"\"\"\n    return tuple(_api_internal.diag_indices_from(arr))\n\n\n@set_module('mxnet.ndarray.numpy')\ndef hanning(M, dtype=None, device=None):\n    r\"\"\"Return the Hanning window.\n\n    The Hanning window is a taper formed by using a weighted cosine.\n\n    Parameters\n    ----------\n    M : int\n        Number of points in the output window. If zero or less, an\n        empty array is returned.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    out : ndarray, shape(M,)\n        The window, with the maximum value normalized to one (the value\n        one appears only if `M` is odd).\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Note that you need select numpy.float32 or float64 in this operator.\n\n    See Also\n    --------\n    blackman, hamming\n\n    Notes\n    -----\n    The Hanning window is defined as\n\n    .. math::  w(n) = 0.5 - 0.5cos\\left(\\frac{2\\pi{n}}{M-1}\\right)\n               \\qquad 0 \\leq n \\leq M-1\n\n    The Hanning was named for Julius von Hann, an Austrian meteorologist.\n    It is also known as the Cosine Bell. Some authors prefer that it be\n    called a Hann window, to help avoid confusion with the very similar\n    Hamming window.\n\n    Most references to the Hanning window come from the signal processing\n    literature, where it is used as one of many windowing functions for\n    smoothing values.  It is also known as an apodization (which means\n    \"removing the foot\", i.e. smoothing discontinuities at the beginning\n    and end of the sampled signal) or tapering function.\n\n    References\n    ----------\n    .. [1] Blackman, R.B. and Tukey, J.W., (1958) The measurement of power\n           spectra, Dover Publications, New York.\n    .. [2] E.R. Kanasewich, \"Time Sequence Analysis in Geophysics\",\n           The University of Alberta Press, 1975, pp. 106-108.\n    .. [3] Wikipedia, \"Window function\",\n           http://en.wikipedia.org/wiki/Window_function\n    .. [4] W.H. Press,  B.P. Flannery, S.A. Teukolsky, and W.T. Vetterling,\n           \"Numerical Recipes\", Cambridge University Press, 1986, page 425.\n\n    Examples\n    --------\n    >>> np.hanning(12)\n    array([0.        , 0.07937324, 0.29229254, 0.5711574 , 0.8274304 ,\n           0.9797465 , 0.97974646, 0.82743025, 0.5711573 , 0.29229245,\n           0.07937312, 0.        ])\n\n    Plot the window and its frequency response:\n\n    >>> import matplotlib.pyplot as plt\n    >>> window = np.hanning(51)\n    >>> plt.plot(window.asnumpy())\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.title(\"Hann window\")\n    Text(0.5, 1.0, 'Hann window')\n    >>> plt.ylabel(\"Amplitude\")\n    Text(0, 0.5, 'Amplitude')\n    >>> plt.xlabel(\"Sample\")\n    Text(0.5, 0, 'Sample')\n    >>> plt.show()\n    \"\"\"\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    return _api_internal.hanning(M, dtype, device)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef hamming(M, dtype=None, device=None):\n    r\"\"\"Return the hamming window.\n\n    The hamming window is a taper formed by using a weighted cosine.\n\n    Parameters\n    ----------\n    M : int\n        Number of points in the output window. If zero or less, an\n        empty array is returned.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    out : ndarray, shape(M,)\n        The window, with the maximum value normalized to one (the value\n        one appears only if `M` is odd).\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Note that you need select numpy.float32 or float64 in this operator.\n\n    See Also\n    --------\n    blackman, hanning\n\n    Notes\n    -----\n    The Hamming window is defined as\n\n    .. math::  w(n) = 0.54 - 0.46cos\\left(\\frac{2\\pi{n}}{M-1}\\right)\n               \\qquad 0 \\leq n \\leq M-1\n\n    The Hamming was named for R. W. Hamming, an associate of J. W. Tukey\n    and is described in Blackman and Tukey. It was recommended for\n    smoothing the truncated autocovariance function in the time domain.\n    Most references to the Hamming window come from the signal processing\n    literature, where it is used as one of many windowing functions for\n    smoothing values.  It is also known as an apodization (which means\n    \"removing the foot\", i.e. smoothing discontinuities at the beginning\n    and end of the sampled signal) or tapering function.\n\n    References\n    ----------\n    .. [1] Blackman, R.B. and Tukey, J.W., (1958) The measurement of power\n           spectra, Dover Publications, New York.\n    .. [2] E.R. Kanasewich, \"Time Sequence Analysis in Geophysics\", The\n           University of Alberta Press, 1975, pp. 109-110.\n    .. [3] Wikipedia, \"Window function\",\n           https://en.wikipedia.org/wiki/Window_function\n    .. [4] W.H. Press,  B.P. Flannery, S.A. Teukolsky, and W.T. Vetterling,\n           \"Numerical Recipes\", Cambridge University Press, 1986, page 425.\n\n    Examples\n    --------\n    >>> np.hamming(12)\n    array([0.08000001, 0.15302339, 0.34890914, 0.6054648 , 0.841236  ,\n           0.9813669 , 0.9813668 , 0.8412359 , 0.6054647 , 0.34890908,\n           0.15302327, 0.08000001])\n\n    Plot the window and its frequency response:\n\n    >>> import matplotlib.pyplot as plt\n    >>> window = np.hamming(51)\n    >>> plt.plot(window.asnumpy())\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.title(\"hamming window\")\n    Text(0.5, 1.0, 'hamming window')\n    >>> plt.ylabel(\"Amplitude\")\n    Text(0, 0.5, 'Amplitude')\n    >>> plt.xlabel(\"Sample\")\n    Text(0.5, 0, 'Sample')\n    >>> plt.show()\n    \"\"\"\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    return _api_internal.hamming(M, dtype, device)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef blackman(M, dtype=None, device=None):\n    r\"\"\"Return the Blackman window.\n\n    The Blackman window is a taper formed by using the first three\n    terms of a summation of cosines. It was designed to have close to the\n    minimal leakage possible.  It is close to optimal, only slightly worse\n    than a Kaiser window.\n\n    Parameters\n    ----------\n    M : int\n        Number of points in the output window. If zero or less, an\n        empty array is returned.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    out : ndarray\n        The window, with the maximum value normalized to one (the value one\n        appears only if the number of samples is odd).\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Note that you need select numpy.float32 or float64 in this operator.\n\n    See Also\n    --------\n    hamming, hanning\n\n    Notes\n    -----\n    The Blackman window is defined as\n\n    .. math::  w(n) = 0.42 - 0.5 \\cos(2\\pi n/{M-1}) + 0.08 \\cos(4\\pi n/{M-1})\n\n    Most references to the Blackman window come from the signal processing\n    literature, where it is used as one of many windowing functions for\n    smoothing values.  It is also known as an apodization (which means\n    \"removing the foot\", i.e. smoothing discontinuities at the beginning\n    and end of the sampled signal) or tapering function. It is known as a\n    \"near optimal\" tapering function, almost as good (by some measures)\n    as the kaiser window.\n\n    References\n    ----------\n    Blackman, R.B. and Tukey, J.W., (1958) The measurement of power spectra,\n    Dover Publications, New York.\n\n    Oppenheim, A.V., and R.W. Schafer. Discrete-Time Signal Processing.\n    Upper Saddle River, NJ: Prentice-Hall, 1999, pp. 468-471.\n\n    Examples\n    --------\n    >>> np.blackman(12)\n    array([-1.4901161e-08,  3.2606423e-02,  1.5990365e-01,  4.1439798e-01,\n            7.3604530e-01,  9.6704686e-01,  9.6704674e-01,  7.3604506e-01,\n            4.1439781e-01,  1.5990359e-01,  3.2606363e-02, -1.4901161e-08])\n\n    Plot the window and its frequency response:\n\n    >>> import matplotlib.pyplot as plt\n    >>> window = np.blackman(51)\n    >>> plt.plot(window.asnumpy())\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.title(\"blackman window\")\n    Text(0.5, 1.0, 'blackman window')\n    >>> plt.ylabel(\"Amplitude\")\n    Text(0, 0.5, 'Amplitude')\n    >>> plt.xlabel(\"Sample\")\n    Text(0.5, 0, 'Sample')\n    >>> plt.show()\n    \"\"\"\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    return _api_internal.blackman(M, dtype, device)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef flip(m, axis=None, out=None):\n    r\"\"\"\n    flip(m, axis=None, out=None)\n\n    Reverse the order of elements in an array along the given axis.\n\n    The shape of the array is preserved, but the elements are reordered.\n\n    Parameters\n    ----------\n    m : ndarray or scalar\n        Input array.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which to flip over. The default,\n        axis=None, will flip over all of the axes of the input array.\n        If axis is negative it counts from the last to the first axis.\n\n        If axis is a tuple of ints, flipping is performed on all of the axes\n        specified in the tuple.\n    out : ndarray or scalar, optional\n        Alternative output array in which to place the result. It must have\n        the same shape and type as the expected output.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        A view of `m` with the entries of axis reversed.  Since a view is\n        returned, this operation is done in constant time.\n\n    Examples\n    --------\n    >>> A = np.arange(8).reshape((2,2,2))\n    >>> A\n    array([[[0, 1],\n            [2, 3]],\n           [[4, 5],\n            [6, 7]]])\n    >>> np.flip(A, 0)\n    array([[[4, 5],\n            [6, 7]],\n           [[0, 1],\n            [2, 3]]])\n    >>> np.flip(A, 1)\n    array([[[2, 3],\n            [0, 1]],\n           [[6, 7],\n            [4, 5]]])\n    >>> np.flip(A)\n    array([[[7, 6],\n            [5, 4]],\n           [[3, 2],\n            [1, 0]]])\n    >>> np.flip(A, (0, 2))\n    array([[[5, 4],\n            [7, 6]],\n           [[1, 0],\n            [3, 2]]])\n    \"\"\"\n    from ...numpy import ndarray\n    if isinstance(m, numeric_types):\n        return _np.flip(m, axis)\n    elif isinstance(m, ndarray):\n        return _api_internal.flip(m, axis, out)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(m))))\n\n\n@set_module('mxnet.ndarray.numpy')\ndef flipud(m):\n    r\"\"\"\n    flipud(*args, **kwargs)\n\n    Flip array in the up/down direction.\n\n    Flip the entries in each column in the up/down direction.\n    Rows are preserved, but appear in a different order than before.\n\n    Parameters\n    ----------\n    m : array_like\n        Input array.\n\n    Returns\n    -------\n    out : array_like\n        A view of `m` with the rows reversed.  Since a view is\n        returned, this operation is :math:`\\mathcal O(1)`.\n\n    See Also\n    --------\n    fliplr : Flip array in the left/right direction.\n    rot90 : Rotate array counterclockwise.\n\n    Notes\n    -----\n    Equivalent to ``m[::-1,...]``.\n    Does not require the array to be two-dimensional.\n\n    Examples\n    --------\n    >>> A = np.diag(np.array([1.0, 2, 3]))\n    >>> A\n    array([[1.,  0.,  0.],\n           [0.,  2.,  0.],\n           [0.,  0.,  3.]])\n    >>> np.flipud(A)\n    array([[0.,  0.,  3.],\n           [0.,  2.,  0.],\n           [1.,  0.,  0.]])\n\n    >>> A = np.random.randn(2,3,5)\n    >>> np.all(np.flipud(A) == A[::-1,...])\n    array(True)\n\n    >>> np.flipud(np.array([1,2]))\n    array([2., 1.])\n    \"\"\"\n    return flip(m, 0)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef fliplr(m):\n    r\"\"\"\n    fliplr(*args, **kwargs)\n\n    Flip array in the left/right direction.\n\n    Flip the entries in each row in the left/right direction.\n    Columns are preserved, but appear in a different order than before.\n\n    Parameters\n    ----------\n    m : array_like\n        Input array, must be at least 2-D.\n\n    Returns\n    -------\n    f : ndarray\n        A view of `m` with the columns reversed.  Since a view\n        is returned, this operation is :math:`\\mathcal O(1)`.\n\n    See Also\n    --------\n    flipud : Flip array in the up/down direction.\n    rot90 : Rotate array counterclockwise.\n\n    Notes\n    -----\n    Equivalent to m[:,::-1]. Requires the array to be at least 2-D.\n\n    Examples\n    --------\n    >>> A = np.diag(np.array([1.,2.,3.]))\n    >>> A\n    array([[1.,  0.,  0.],\n           [0.,  2.,  0.],\n           [0.,  0.,  3.]])\n    >>> np.fliplr(A)\n    array([[0.,  0.,  1.],\n           [0.,  2.,  0.],\n           [3.,  0.,  0.]])\n\n    >>> A = np.random.randn(2,3,5)\n    >>> np.all(np.fliplr(A) == A[:,::-1,...])\n    array(True)\n    \"\"\"\n    return flip(m, 1)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef around(x, decimals=0, out=None, **kwargs):\n    r\"\"\"\n    around(x, decimals=0, out=None)\n\n    Evenly round to the given number of decimals.\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input data.\n    decimals : int, optional\n        Number of decimal places to round to (default: 0).  If\n        decimals is negative, it specifies the number of positions to\n        the left of the decimal point.\n    out : ndarray, optional\n        Alternative output array in which to place the result. It must have\n        the same shape and type as the expected output.\n\n    Returns\n    -------\n    rounded_array : ndarray or scalar\n        An array of the same type as `x`, containing the rounded values.\n        A reference to the result is returned.\n\n    Notes\n    -----\n    For values exactly halfway between rounded decimal values, NumPy\n    rounds to the nearest even value. Thus 1.5 and 2.5 round to 2.0,\n    -0.5 and 0.5 round to 0.0, etc.\n\n    This function differs from the original numpy.prod in the following aspects:\n\n        - Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n        - Cannot support complex-valued number.\n\n    Examples\n    --------\n    >>> np.around([0.37, 1.64])\n    array([ 0.,  2.])\n    >>> np.around([0.37, 1.64], decimals=1)\n    array([ 0.4,  1.6])\n    >>> np.around([.5, 1.5, 2.5, 3.5, 4.5]) # rounds to nearest even value\n    array([ 0.,  2.,  2.,  4.,  4.])\n    >>> np.around([1, 2, 3, 11], decimals=1) # ndarray of ints is returned\n    array([ 1,  2,  3, 11])\n    >>> np.around([1, 2, 3, 11], decimals=-1)\n    array([ 0,  0,  0, 10])\n    \"\"\"\n    from ...numpy import ndarray\n    if isinstance(x, numeric_types):\n        return _np.around(x, decimals, **kwargs)\n    elif isinstance(x, ndarray):\n        return _api_internal.around(x, decimals, out, **kwargs)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(x))))\n\n\n@set_module('mxnet.ndarray.numpy')\ndef round(x, decimals=0, out=None, **kwargs):\n    r\"\"\"\n    round(a, decimals=0, out=None)\n    Round an array to the given number of decimals.\n\n    See Also\n    --------\n    around : equivalent function; see for details.\n    \"\"\"\n    from ...numpy import ndarray\n    if isinstance(x, numeric_types):\n        return _np.around(x, decimals, **kwargs)\n    elif isinstance(x, ndarray):\n        return _api_internal.around(x, decimals, out, **kwargs)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(x))))\n\n\n@set_module('mxnet.ndarray.numpy')\ndef round_(x, decimals=0, out=None, **kwargs):\n    r\"\"\"\n    round_(a, decimals=0, out=None)\n    Round an array to the given number of decimals.\n\n    See Also\n    --------\n    around : equivalent function; see for details.\n    \"\"\"\n    from ...numpy import ndarray\n    if isinstance(x, numeric_types):\n        return _np.around(x, decimals, **kwargs)\n    elif isinstance(x, ndarray):\n        return _npi.around(x, decimals, out=out, **kwargs)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(x))))\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef arctan2(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Element-wise arc tangent of ``x1/x2`` choosing the quadrant correctly.\n\n    The quadrant (i.e., branch) is chosen so that ``arctan2(x1, x2)`` is\n    the signed angle in radians between the ray ending at the origin and\n    passing through the point (1,0), and the ray ending at the origin and\n    passing through the point (`x2`, `x1`).  (Note the role reversal: the\n    \"`y`-coordinate\" is the first function parameter, the \"`x`-coordinate\"\n    is the second.)  By IEEE convention, this function is defined for\n    `x2` = +/-0 and for either or both of `x1` and `x2` = +/-inf (see\n    Notes for specific values).\n\n    This function is not defined for complex-valued arguments; for the\n    so-called argument of complex values, use `angle`.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        `y`-coordinates.\n    x2 : ndarray or scalar\n        `x`-coordinates. `x2` must be broadcastable to match the shape of\n        `x1` or vice versa.\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Array of angles in radians, in the range ``[-pi, pi]``. This is a scalar if\n        `x1` and `x2` are scalars.\n\n    Notes\n    -----\n    *arctan2* is identical to the `atan2` function of the underlying\n    C library.  The following special values are defined in the C\n    standard: [1]_\n\n    ====== ====== ================\n    `x1`   `x2`   `arctan2(x1,x2)`\n    ====== ====== ================\n    +/- 0  +0     +/- 0\n    +/- 0  -0     +/- pi\n        > 0   +/-inf +0 / +pi\n        < 0   +/-inf -0 / -pi\n    +/-inf +inf   +/- (pi/4)\n    +/-inf -inf   +/- (3*pi/4)\n    ====== ====== ================\n\n    Note that +0 and -0 are distinct floating point numbers, as are +inf\n    and -inf.\n\n    This function differs from the original numpy.arange in the following aspects:\n        - Only support float16, float32 and float64.\n\n    References\n    ----------\n    .. [1] ISO/IEC standard 9899:1999, \"Programming language C.\"\n\n    Examples\n    --------\n    Consider four points in different quadrants:\n\n    >>> x = np.array([-1, +1, +1, -1])\n    >>> y = np.array([-1, -1, +1, +1])\n    >>> np.arctan2(y, x) * 180 / np.pi\n    array([-135.,  -45.,   45.,  135.])\n\n    Note the order of the parameters. `arctan2` is defined also when `x2` = 0\n    and at several other special points, obtaining values in\n    the range ``[-pi, pi]``:\n\n    >>> x = np.array([1, -1])\n    >>> y = np.array([0, 0])\n    >>> np.arctan2(x, y)\n    array([ 1.5707964, -1.5707964])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.arctan2(x1, x2, out=out)\n    return _api_internal.arctan2(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef hypot(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Given the \"legs\" of a right triangle, return its hypotenuse.\n\n    Equivalent to ``sqrt(x1**2 + x2**2)``, element-wise.  If `x1` or\n    `x2` is scalar_like (i.e., unambiguously cast-able to a scalar type),\n    it is broadcast for use with each element of the other argument.\n\n    Parameters\n    ----------\n    x1, x2 : ndarray\n        Leg of the triangle(s).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n\n    Returns\n    -------\n    z : ndarray\n        The hypotenuse of the triangle(s).\n        This is a scalar if both `x1` and `x2` are scalars.\n\n    Notes\n    -----\n    This function differs from the original numpy.arange in the following aspects:\n        - Only support float16, float32 and float64.\n\n    Examples\n    --------\n    >>> np.hypot(3*np.ones((3, 3)), 4*np.ones((3, 3)))\n    array([[ 5.,  5.,  5.],\n           [ 5.,  5.,  5.],\n           [ 5.,  5.,  5.]])\n\n    Example showing broadcast of scalar_like argument:\n\n    >>> np.hypot(3*np.ones((3, 3)), [4])\n    array([[ 5.,  5.,  5.],\n           [ 5.,  5.,  5.],\n           [ 5.,  5.,  5.]])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.hypot(x1, x2, out=out)\n    return _api_internal.hypot(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef bitwise_and(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Compute the bit-wise XOR of two arrays element-wise.\n\n    Parameters\n    ----------\n    x1, x2 : ndarray or scalar\n        Only integer and boolean types are handled. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which becomes the shape of the output).\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape that the\n        inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n        Result.\n\n    Examples\n    --------\n    >>> np.bitwise_and(13, 17)\n    1\n\n    >>> np.bitwise_and(14, 13)\n    12\n    >>> np.bitwise_and(np.array([14,3], dtype='int32'), 13)\n    array([12,  1], dtype=int32)\n\n    >>> np.bitwise_and(np.array([11,7], dtype='int32'), np.array([4,25], dtype='int32'))\n    array([0, 1], dtype=int32)\n    >>> np.bitwise_and(np.array([2,5,255], dtype='int32'), np.array([3,14,16], dtype='int32'))\n    array([ 2,  4, 16], dtype=int32)\n    >>> np.bitwise_and(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))\n    array([False,  True])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.bitwise_and(x1, x2, out=out)\n    return _api_internal.bitwise_and(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef bitwise_xor(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Compute the bit-wise XOR of two arrays element-wise.\n\n    Parameters\n    ----------\n    x1, x2 : ndarray or scalar\n        Only integer and boolean types are handled. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which becomes the shape of the output).\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape that the\n        inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n        Result.\n\n    Examples\n    --------\n    >>> np.bitwise_xor(13, 17)\n    28\n\n    >>> np.bitwise_xor(31, 5)\n    26\n    >>> np.bitwise_xor(np.array([31,3], dtype='int32'), 5)\n    array([26,  6])\n\n    >>> np.bitwise_xor(np.array([31,3], dtype='int32'), np.array([5,6], dtype='int32'))\n    array([26,  5])\n    >>> np.bitwise_xor(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))\n    array([ True, False])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.bitwise_xor(x1, x2, out=out)\n    return _api_internal.bitwise_xor(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef bitwise_or(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Compute the bit-wise OR of two arrays element-wise.\n\n    Parameters\n    ----------\n    x1, x2 : ndarray or scalar\n        Only integer and boolean types are handled. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which becomes the shape of the output).\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape that the\n        inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n        Result.\n\n    Examples\n    --------\n    >>> np.bitwise_or(13, 17)\n    29\n\n    >>> np.bitwise_or(31, 5)\n    31\n    >>> np.bitwise_or(np.array([31,3], dtype='int32'), 5)\n    array([31,  7])\n\n    >>> np.bitwise_or(np.array([31,3], dtype='int32'), np.array([5,6], dtype='int32'))\n    array([31,  7])\n    >>> np.bitwise_or(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))\n    array([ True, True])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.bitwise_or(x1, x2, out=out)\n    return _api_internal.bitwise_or(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef ldexp(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns x1 * 2**x2, element-wise.\n    The mantissas `x1` and twos exponents `x2` are used to construct\n    floating point numbers ``x1 * 2**x2``.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Array of multipliers.\n    x2 : ndarray or scalar, int\n        Array of twos exponents.\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The result of ``x1 * 2**x2``.\n        This is a scalar if both `x1` and `x2` are scalars.\n\n    Notes\n    -----\n    Complex dtypes are not supported, they will raise a TypeError.\n    Different from numpy, we allow x2 to be float besides int.\n    `ldexp` is useful as the inverse of `frexp`, if used by itself it is\n    more clear to simply use the expression ``x1 * 2**x2``.\n\n    Examples\n    --------\n    >>> np.ldexp(5, np.arange(4))\n    array([  5.,  10.,  20.,  40.])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.ldexp(x1, x2, out=out)\n    return _api_internal.ldexp(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef logaddexp(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Logarithm of the sum of exponentiations of the inputs.\n\n    Calculates log(exp(x1) + exp(x2)). This function is useful in statistics where\n    the calculated probabilities of events may be so small as to exceed the range of\n    normal floating point numbers. In such cases the logarithm of the calculate\n    probability is stored. This function allows adding probabilities stored\n    in such a fashion.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Array of multipliers.\n    x2 : ndarray or scalar, int\n        Array of twos exponents.\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        Logarithm of exp(x1) + exp(x2). This is a scalar if both x1 and x2 are scalars.\n\n    Examples\n    --------\n    >>> prob1 = np.log(1e-50)\n    >>> prob2 = np.log(2.5e-50)\n    >>> prob12 = np.logaddexp(prob1, prob2)\n    >>> prob12\n    -113.87649168120691\n    >>> np.exp(prob12)\n    3.5000000000000057e-50\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.logaddexp(x1, x2, out=out)\n    return _api_internal.logaddexp(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef vdot(a, b):\n    r\"\"\"\n    Return the dot product of two vectors.\n    Note that `vdot` handles multidimensional arrays differently than `dot`:\n    it does *not* perform a matrix product, but flattens input arguments\n    to 1-D vectors first. Consequently, it should only be used for vectors.\n\n    Parameters\n    ----------\n    a : ndarray\n        First argument to the dot product.\n    b : ndarray\n        Second argument to the dot product.\n\n    Returns\n    -------\n    output : ndarray\n        Dot product of `a` and `b`.\n\n    See Also\n    --------\n    dot : Return the dot product without using the complex conjugate of the\n        first argument.\n\n    Examples\n    --------\n    Note that higher-dimensional arrays are flattened!\n    >>> a = np.array([[1, 4], [5, 6]])\n    >>> b = np.array([[4, 1], [2, 2]])\n    >>> np.vdot(a, b)\n    30\n    >>> np.vdot(b, a)\n    30\n    >>> 1*4 + 4*1 + 5*2 + 6*2\n    30\n    \"\"\"\n    return tensordot(a.flatten(), b.flatten(), 1)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef inner(a, b):\n    r\"\"\"\n    Inner product of two arrays.\n    Ordinary inner product of vectors for 1-D arrays (without complex\n    conjugation), in higher dimensions a sum product over the last axes.\n\n    Parameters\n    ----------\n    a, b : ndarray\n        If `a` and `b` are nonscalar, their last dimensions must match.\n\n    Returns\n    -------\n    out : ndarray\n        `out.shape = a.shape[:-1] + b.shape[:-1]`\n\n    Raises\n    ------\n    ValueError\n        If the last dimension of `a` and `b` has different size.\n\n    See Also\n    --------\n    tensordot : Sum products over arbitrary axes.\n    dot : Generalised matrix product, using second last dimension of `b`.\n    einsum : Einstein summation convention.\n\n    Notes\n    -----\n    For vectors (1-D arrays) it computes the ordinary inner-product::\n        np.inner(a, b) = sum(a[:]*b[:])\n    More generally, if `ndim(a) = r > 0` and `ndim(b) = s > 0`::\n        np.inner(a, b) = np.tensordot(a, b, axes=(-1,-1))\n    or explicitly::\n        np.inner(a, b)[i0,...,ir-1,j0,...,js-1]\n            = sum(a[i0,...,ir-1,:]*b[j0,...,js-1,:])\n    In addition `a` or `b` may be scalars, in which case::\n    np.inner(a,b) = a*b\n\n    Examples\n    --------\n    Ordinary inner product for vectors:\n    >>> a = np.array([1,2,3])\n    >>> b = np.array([0,1,0])\n    >>> np.inner(a, b)\n    2\n    A multidimensional example:\n    >>> a = np.arange(24).reshape((2,3,4))\n    >>> b = np.arange(4)\n    >>> np.inner(a, b)\n    array([[ 14,  38,  62],\n           [ 86, 110, 134]])\n    \"\"\"\n    return tensordot(a, b, [-1, -1])\n\n\n@set_module('mxnet.ndarray.numpy')\ndef outer(a, b):\n    r\"\"\"\n    Compute the outer product of two vectors.\n    Given two vectors, ``a = [a0, a1, ..., aM]`` and\n    ``b = [b0, b1, ..., bN]``,\n    the outer product [1]_ is::\n    [[a0*b0  a0*b1 ... a0*bN ]\n    [a1*b0    .\n    [ ...          .\n    [aM*b0            aM*bN ]]\n\n    Parameters\n    ----------\n    a : (M,) ndarray\n        First input vector.  Input is flattened if\n        not already 1-dimensional.\n    b : (N,) ndarray\n        Second input vector.  Input is flattened if\n        not already 1-dimensional.\n\n    Returns\n    -------\n    out : (M, N) ndarray\n        ``out[i, j] = a[i] * b[j]``\n    See also\n    --------\n    inner\n    einsum : ``einsum('i,j->ij', a.ravel(), b.ravel())`` is the equivalent.\n    ufunc.outer : A generalization to N dimensions and other operations.\n                ``np.multiply.outer(a.ravel(), b.ravel())`` is the equivalent.\n    References\n    ----------\n    .. [1] : G. H. Golub and C. F. Van Loan, *Matrix Computations*, 3rd\n            ed., Baltimore, MD, Johns Hopkins University Press, 1996,\n            pg. 8.\n    Examples\n    --------\n    Make a (*very* coarse) grid for computing a Mandelbrot set:\n    >>> rl = np.outer(np.ones((5,)), np.linspace(-2, 2, 5))\n    >>> rl\n    array([[-2., -1.,  0.,  1.,  2.],\n        [-2., -1.,  0.,  1.,  2.],\n        [-2., -1.,  0.,  1.,  2.],\n        [-2., -1.,  0.,  1.,  2.],\n        [-2., -1.,  0.,  1.,  2.]])\n    \"\"\"\n    return tensordot(a.reshape_view((-1, )), b.reshape_view((-1, )), 0)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef cross(a, b, axisa=-1, axisb=-1, axisc=-1, axis=None): # pylint: disable=too-many-arguments\n    \"\"\"\n    Return the cross product of two (arrays of) vectors.\n\n    The cross product of `a` and `b` in :math:`R^3` is a vector perpendicular\n    to both `a` and `b`.  If `a` and `b` are arrays of vectors, the vectors\n    are defined by the last axis of `a` and `b` by default, and these axis\n    can have dimensions 2 or 3.  Where the dimension of either `a` or `b` is\n    2, the third component of the input vector is assumed to be zero and the\n    cross product calculated accordingly.  In cases where both input vectors\n    have dimension 2, the z-component of the cross product is returned.\n\n    Parameters\n    ----------\n    a : ndarray\n        Components of the first vector(s).\n    b : ndarray\n        Components of the second vector(s).\n    axisa : int, optional\n        Axis of `a` that defines the vector(s).  By default, the last axis.\n    axisb : int, optional\n        Axis of `b` that defines the vector(s).  By default, the last axis.\n    axisc : int, optional\n        Axis of `c` containing the cross product vector(s).  Ignored if\n        both input vectors have dimension 2, as the return is scalar.\n        By default, the last axis.\n    axis : int, optional\n        If defined, the axis of `a`, `b` and `c` that defines the vector(s)\n        and cross product(s).  Overrides `axisa`, `axisb` and `axisc`.\n\n    Returns\n    -------\n    c : ndarray\n        Vector cross product(s).\n\n    Raises\n    ------\n    ValueError\n        When the dimension of the vector(s) in `a` and/or `b` does not\n        equal 2 or 3.\n\n    Notes\n    -----\n    Supports full broadcasting of the inputs.\n\n    Examples\n    --------\n    Vector cross-product.\n\n    >>> x = np.array([1., 2., 3.])\n    >>> y = np.array([4., 5., 6.])\n    >>> np.cross(x, y)\n    array([-3.,  6., -3.])\n\n    One vector with dimension 2.\n\n    >>> x = np.array([1., 2.])\n    >>> y = np.array([4., 5., 6.])\n    >>> np.cross(x, y)\n    array([12., -6., -3.])\n\n    Equivalently:\n\n    >>> x = np.array([1., 2., 0.])\n    >>> y = np.array([4., 5., 6.])\n    >>> np.cross(x, y)\n    array([12., -6., -3.])\n\n    Both vectors with dimension 2.\n\n    >>> x = np.array([1., 2.])\n    >>> y = np.array([4., 5.])\n    >>> np.cross(x, y)\n    array(-3.)\n\n    Multiple vector cross-products. Note that the direction of the cross\n    product vector is defined by the `right-hand rule`.\n\n    >>> x = np.array([[1., 2., 3.], [4., 5., 6.]])\n    >>> y = np.array([[4., 5., 6.], [1., 2., 3.]])\n    >>> np.cross(x, y)\n    array([[-3.,  6., -3.],\n           [ 3., -6.,  3.]])\n\n    The orientation of `c` can be changed using the `axisc` keyword.\n\n    >>> np.cross(x, y, axisc=0)\n    array([[-3.,  3.],\n           [ 6., -6.],\n           [-3.,  3.]])\n\n    Change the vector definition of `x` and `y` using `axisa` and `axisb`.\n\n    >>> x = np.array([[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]])\n    >>> y = np.array([[7., 8., 9.], [4., 5., 6.], [1., 2., 3.]])\n    >>> np.cross(x, y)\n    array([[ -6.,  12.,  -6.],\n           [  0.,   0.,   0.],\n           [  6., -12.,   6.]])\n    >>> np.cross(x, y, axisa=0, axisb=0)\n    array([[-24.,  48., -24.],\n           [-30.,  60., -30.],\n           [-36.,  72., -36.]])\n    \"\"\"\n    if axis is not None:\n        axisa, axisb, axisc = (axis,) * 3\n\n    if isinstance(a, NDArray) and isinstance(b, NDArray):\n        return _api_internal.cross(a, b, axisa, axisb, axisc)\n    else:\n        raise TypeError(\"Input data should be NDarray\")\n\n\n@set_module('mxnet.ndarray.numpy')\ndef kron(a, b):\n    r\"\"\"\n    Kronecker product of two arrays.\n    Computes the Kronecker product, a composite array made of blocks of the\n    second array scaled by the first.\n    Parameters\n    ----------\n    a, b : ndarray\n    Returns\n    -------\n    out : ndarray\n    See Also\n    --------\n    outer : The outer product\n    Notes\n    -----\n    The function assumes that the number of dimensions of `a` and `b`\n    are the same, if necessary prepending the smallest with ones.\n    If `a.shape = (r0,r1,..,rN)` and `b.shape = (s0,s1,...,sN)`,\n    the Kronecker product has shape `(r0*s0, r1*s1, ..., rN*SN)`.\n    The elements are products of elements from `a` and `b`, organized\n    explicitly by::\n        kron(a,b)[k0,k1,...,kN] = a[i0,i1,...,iN] * b[j0,j1,...,jN]\n    where::\n        kt = it * st + jt,  t = 0,...,N\n    In the common 2-D case (N=1), the block structure can be visualized::\n        [[ a[0,0]*b,   a[0,1]*b,  ... , a[0,-1]*b  ],\n        [  ...                              ...   ],\n        [ a[-1,0]*b,  a[-1,1]*b, ... , a[-1,-1]*b ]]\n    Examples\n    --------\n    >>> np.kron([1,10,100], [5,6,7])\n    array([  5,   6,   7,  50,  60,  70, 500, 600, 700])\n    >>> np.kron([5,6,7], [1,10,100])\n    array([  5,  50, 500,   6,  60, 600,   7,  70, 700])\n    \"\"\"\n    return _api_internal.kron(a, b)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef equal(x1, x2, out=None):\n    \"\"\"\n    Return (x1 == x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    not_equal, greater_equal, less_equal, greater, less\n    Examples\n    --------\n    >>> np.equal(np.ones(2, 1)), np.zeros(1, 3))\n    array([[False, False, False],\n           [False, False, False]])\n    >>> np.equal(1, np.ones(1))\n    array([ True])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.equal(x1, x2, out=out)\n    return _api_internal.equal(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef not_equal(x1, x2, out=None):\n    \"\"\"\n    Return (x1 != x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    equal, greater, greater_equal, less, less_equal\n    Examples\n    --------\n    >>> np.not_equal(np.ones(2, 1)), np.zeros(1, 3))\n    array([[ True,  True,  True],\n           [ True,  True,  True]])\n    >>> np.not_equal(1, np.ones(1))\n    array([False])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.not_equal(x1, x2, out=out)\n    return _api_internal.not_equal(x1, x2, out)\n\n\n\n@set_module('mxnet.ndarray.numpy')\ndef greater(x1, x2, out=None):\n    \"\"\"\n    Return the truth value of (x1 > x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    equal, greater, greater_equal, less, less_equal\n    Examples\n    --------\n    >>> np.greater(np.ones(2, 1)), np.zeros(1, 3))\n    array([[ True,  True,  True],\n           [ True,  True,  True]])\n    >>> np.greater(1, np.ones(1))\n    array([False])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.greater(x1, x2, out=out)\n    return _api_internal.greater(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef less(x1, x2, out=None):\n    \"\"\"\n    Return the truth value of (x1 < x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    equal, greater, greater_equal, less, less_equal\n    Examples\n    --------\n    >>> np.less(np.ones(2, 1)), np.zeros(1, 3))\n    array([[ True,  True,  True],\n           [ True,  True,  True]])\n    >>> np.less(1, np.ones(1))\n    array([False])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.less(x1, x2, out=out)\n    return _api_internal.less(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef greater_equal(x1, x2, out=None):\n    \"\"\"\n    Return the truth value of (x1 >= x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    equal, greater, greater_equal, less, less_equal\n    Examples\n    --------\n    >>> np.greater_equal(np.ones(2, 1)), np.zeros(1, 3))\n    array([[ True,  True,  True],\n           [ True,  True,  True]])\n    >>> np.greater_equal(1, np.ones(1))\n    array([True])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.greater_equal(x1, x2, out=out)\n    return _api_internal.greater_equal(x1, x2, out)\n\n\n\n@set_module('mxnet.ndarray.numpy')\ndef less_equal(x1, x2, out=None):\n    \"\"\"\n    Return the truth value of (x1 <= x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    equal, greater, greater_equal, less, less_equal\n    Examples\n    --------\n    >>> np.less_equal(np.ones(2, 1)), np.zeros(1, 3))\n    array([[False, False, False],\n           [False, False, False]])\n    >>> np.less_equal(1, np.ones(1))\n    array([True])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.less_equal(x1, x2, out=out)\n    return _api_internal.less_equal(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef roll(a, shift, axis=None):\n    \"\"\"\n    Roll array elements along a given axis.\n\n    Elements that roll beyond the last position are re-introduced at\n    the first.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array.\n    shift : int or tuple of ints\n        The number of places by which elements are shifted.  If a tuple,\n        then `axis` must be a tuple of the same size, and each of the\n        given axes is shifted by the corresponding number.  If an int\n        while `axis` is a tuple of ints, then the same value is used for\n        all given axes.\n    axis : int or tuple of ints, optional\n        Axis or axes along which elements are shifted.  By default, the\n        array is flattened before shifting, after which the original\n        shape is restored.\n\n    Returns\n    -------\n    res : ndarray\n        Output array, with the same shape as `a`.\n\n    Notes\n    -----\n    Supports rolling over multiple dimensions simultaneously.\n\n    Examples\n    --------\n    >>> x = np.arange(10)\n    >>> np.roll(x, 2)\n    array([8., 9., 0., 1., 2., 3., 4., 5., 6., 7.])\n    >>> np.roll(x, -2)\n    array([2., 3., 4., 5., 6., 7., 8., 9., 0., 1.])\n\n    >>> x2 = np.reshape(x, (2,5))\n    >>> x2\n    array([[0., 1., 2., 3., 4.],\n           [5., 6., 7., 8., 9.]])\n    >>> np.roll(x2, 1)\n    array([[9., 0., 1., 2., 3.],\n           [4., 5., 6., 7., 8.]])\n    >>> np.roll(x2, -1)\n    array([[1., 2., 3., 4., 5.],\n           [6., 7., 8., 9., 0.]])\n    >>> np.roll(x2, 1, axis=0)\n    array([[5., 6., 7., 8., 9.],\n           [0., 1., 2., 3., 4.]])\n    >>> np.roll(x2, -1, axis=0)\n    array([[5., 6., 7., 8., 9.],\n           [0., 1., 2., 3., 4.]])\n    >>> np.roll(x2, 1, axis=1)\n    array([[4., 0., 1., 2., 3.],\n           [9., 5., 6., 7., 8.]])\n    >>> np.roll(x2, -1, axis=1)\n    array([[1., 2., 3., 4., 0.],\n           [6., 7., 8., 9., 5.]])\n   \"\"\"\n    return _api_internal.roll(a, shift, axis)\n\n\n@wrap_np_binary_func\ndef logical_and(x1, x2, out=None):\n    r\"\"\"\n    Compute the truth value of x1 AND x2 element-wise.\n    Parameters\n    ----------\n    x1, x2 : array_like\n        Logical AND is applied to the elements of `x1` and `x2`.\n        If ``x1.shape != x2.shape``, they must be broadcastable to a common\n        shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n    Returns\n    -------\n    y : ndarray or bool\n        Boolean result of the logical AND operation applied to the elements\n        of `x1` and `x2`; the shape is determined by broadcasting.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    logical_or, logical_not, logical_xor, bitwise_or\n    Examples\n    --------\n    >>> np.logical_and(True, False)\n    False\n    >>> np.logical_and(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))\n    array([False,  True])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.logical_and(x1, x2, out=out)\n    return _api_internal.logical_and(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef logical_or(x1, x2, out=None):\n    \"\"\"\n    Compute the truth value of x1 OR x2 element-wise.\n    Parameters\n    ----------\n    x1, x2 : array_like\n        Logical OR is applied to the elements of `x1` and `x2`.\n        If ``x1.shape != x2.shape``, they must be broadcastable to a common\n        shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n    Returns\n    -------\n    y : ndarray or bool\n        Boolean result of the logical OR operation applied to the elements\n        of `x1` and `x2`; the shape is determined by broadcasting.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    logical_and, logical_not, logical_xor, bitwise_or\n    Examples\n    --------\n    >>> np.logical_or(True, False)\n    True\n    >>> np.logical_or(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))\n    array([True,  True])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.logical_or(x1, x2, out=out)\n    return _api_internal.logical_or(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_binary_func\ndef logical_xor(x1, x2, out=None):\n    \"\"\"\n    Compute the truth value of x1 XOR x2 element-wise.\n    Parameters\n    ----------\n    x1, x2 : array_like\n        Logical XOR is applied to the elements of `x1` and `x2`.\n        If ``x1.shape != x2.shape``, they must be broadcastable to a common\n        shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n    Returns\n    -------\n    y : ndarray or bool\n        Boolean result of the logical XOR operation applied to the elements\n        of `x1` and `x2`; the shape is determined by broadcasting.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    logical_and, logical_not, logical_or, bitwise_or\n    Examples\n    --------\n    >>> np.logical_xor(True, False)\n    True\n    >>> np.logical_xor(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))\n    array([ True, False])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.logical_xor(x1, x2, out=out)\n    return _api_internal.logical_xor(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef rot90(m, k=1, axes=(0, 1)):\n    \"\"\"\n    Rotate an array by 90 degrees in the plane specified by axes.\n    Rotation direction is from the first towards the second axis.\n    Parameters\n    ----------\n    m : ndarray\n        Array of two or more dimensions.\n    k : integer\n        Number of times the array is rotated by 90 degrees.\n    axes: (2,) array_like\n        The array is rotated in the plane defined by the axes.\n        Axes must be different.\n\n    Returns\n    -------\n    y : ndarray\n        A rotated view of `m`.\n\n    -----\n    rot90(m, k=1, axes=(1,0)) is the reverse of rot90(m, k=1, axes=(0,1))\n    rot90(m, k=1, axes=(1,0)) is equivalent to rot90(m, k=-1, axes=(0,1))\n    Examples\n    --------\n    >>> m = np.array([[1,2],[3,4]], 'int')\n    >>> m\n    array([[1, 2],\n           [3, 4]], dtype=int64)\n    >>> np.rot90(m)\n    array([[2, 4],\n           [1, 3]], dtype=int64)\n    >>> np.rot90(m, 2)\n    array([[4, 3],\n           [2, 1]], dtype=int64)\n    >>> m = np.arange(8).reshape((2,2,2))\n    >>> np.rot90(m, 1, (1,2))\n    array([[[1., 3.],\n            [0., 2.]],\n\n           [[5., 7.],\n            [4., 6.]]])\n    \"\"\"\n    return _api_internal.rot90(m, k, axes)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef einsum(*operands, **kwargs):\n    r\"\"\"\n    einsum(subscripts, *operands, out=None, optimize=False)\n\n    Evaluates the Einstein summation convention on the operands.\n\n    Using the Einstein summation convention, many common multi-dimensional,\n    linear algebraic array operations can be represented in a simple fashion.\n    In *implicit* mode `einsum` computes these values.\n\n    In *explicit* mode, `einsum` provides further flexibility to compute\n    other array operations that might not be considered classical Einstein\n    summation operations, by disabling, or forcing summation over specified\n    subscript labels.\n\n    See the notes and examples for clarification.\n\n    Parameters\n    ----------\n    subscripts : str\n        Specifies the subscripts for summation as comma separated list of\n        subscript labels. An implicit (classical Einstein summation)\n        calculation is performed unless the explicit indicator '->' is\n        included as well as subscript labels of the precise output form.\n    operands : list of ndarray\n        These are the arrays for the operation.\n    out : ndarray, optional\n        If provided, the calculation is done into this array.\n    optimize : {False, True}, optional\n        Controls if intermediate optimization should occur. No optimization\n        will occur if False. Defaults to False.\n\n    Returns\n    -------\n    output : ndarray\n        The calculation based on the Einstein summation convention.\n\n    Notes\n    -----\n    The Einstein summation convention can be used to compute\n    many multi-dimensional, linear algebraic array operations. `einsum`\n    provides a succinct way of representing these.\n\n    A non-exhaustive list of these operations,\n    which can be computed by `einsum`, is shown below along with examples:\n\n    * Trace of an array, :py:func:`np.trace`.\n    * Return a diagonal, :py:func:`np.diag`.\n    * Array axis summations, :py:func:`np.sum`.\n    * Transpositions and permutations, :py:func:`np.transpose`.\n    * Matrix multiplication and dot product, :py:func:`np.matmul` :py:func:`np.dot`.\n    * Vector inner and outer products, :py:func:`np.inner` :py:func:`np.outer`.\n    * Broadcasting, element-wise and scalar multiplication, :py:func:`np.multiply`.\n    * Tensor contractions, :py:func:`np.tensordot`.\n\n    The subscripts string is a comma-separated list of subscript labels,\n    where each label refers to a dimension of the corresponding operand.\n    Whenever a label is repeated it is summed, so ``np.einsum('i,i', a, b)``\n    is equivalent to :py:func:`np.inner(a,b) <np.inner>`. If a label\n    appears only once, it is not summed, so ``np.einsum('i', a)`` produces a\n    view of ``a`` with no changes. A further example ``np.einsum('ij,jk', a, b)``\n    describes traditional matrix multiplication and is equivalent to\n    :py:func:`np.matmul(a,b) <np.matmul>`. Repeated subscript labels in one\n    operand take the diagonal. For example, ``np.einsum('ii', a)`` is equivalent\n    to :py:func:`np.trace(a) <np.trace>`.\n\n    In *implicit mode*, the chosen subscripts are important\n    since the axes of the output are reordered alphabetically.  This\n    means that ``np.einsum('ij', a)`` doesn't affect a 2D array, while\n    ``np.einsum('ji', a)`` takes its transpose. Additionally,\n    ``np.einsum('ij,jk', a, b)`` returns a matrix multiplication, while,\n    ``np.einsum('ij,jh', a, b)`` returns the transpose of the\n    multiplication since subscript 'h' precedes subscript 'i'.\n\n    In *explicit mode* the output can be directly controlled by\n    specifying output subscript labels.  This requires the\n    identifier '->' as well as the list of output subscript labels.\n    This feature increases the flexibility of the function since\n    summing can be disabled or forced when required. The call\n    ``np.einsum('i->', a)`` is like :py:func:`np.sum(a, axis=-1) <np.sum>`,\n    and ``np.einsum('ii->i', a)`` is like :py:func:`np.diag(a) <np.diag>`.\n    The difference is that `einsum` does not allow broadcasting by default.\n    Additionally ``np.einsum('ij,jh->ih', a, b)`` directly specifies the\n    order of the output subscript labels and therefore returns matrix\n    multiplication, unlike the example above in implicit mode.\n\n    To enable and control broadcasting, use an ellipsis.  Default\n    NumPy-style broadcasting is done by adding an ellipsis\n    to the left of each term, like ``np.einsum('...ii->...i', a)``.\n    To take the trace along the first and last axes,\n    you can do ``np.einsum('i...i', a)``, or to do a matrix-matrix\n    product with the left-most indices instead of rightmost, one can do\n    ``np.einsum('ij...,jk...->ik...', a, b)``.\n\n    When there is only one operand, no axes are summed, and no output\n    parameter is provided, a view into the operand is returned instead\n    of a new array.  Thus, taking the diagonal as ``np.einsum('ii->i', a)``\n    produces a view.\n\n    The ``optimize`` argument which will optimize the contraction order\n    of an einsum expression. For a contraction with three or more operands this\n    can greatly increase the computational efficiency at the cost of a larger\n    memory footprint during computation.\n\n    Typically a 'greedy' algorithm is applied which empirical tests have shown\n    returns the optimal path in the majority of cases. 'optimal' is not supported\n    for now.\n\n    This function differs from the original `numpy.einsum\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.einsum.html>`_ in\n    the following way(s):\n\n    - Does not support 'optimal' strategy\n    - Does not support the alternative subscript like\n        `einsum(op0, sublist0, op1, sublist1, ..., [sublistout])`\n    - Does not produce view in any cases\n\n    Examples\n    --------\n    >>> a = np.arange(25).reshape(5,5)\n    >>> b = np.arange(5)\n    >>> c = np.arange(6).reshape(2,3)\n\n    Trace of a matrix:\n\n    >>> np.einsum('ii', a)\n    array(60.)\n\n    Extract the diagonal (requires explicit form):\n\n    >>> np.einsum('ii->i', a)\n    array([ 0.,  6., 12., 18., 24.])\n\n    Sum over an axis (requires explicit form):\n\n    >>> np.einsum('ij->i', a)\n    array([ 10.,  35.,  60.,  85., 110.])\n    >>> np.sum(a, axis=1)\n    array([ 10.,  35.,  60.,  85., 110.])\n\n    For higher dimensional arrays summing a single axis can be done with ellipsis:\n\n    >>> np.einsum('...j->...', a)\n    array([ 10.,  35.,  60.,  85., 110.])\n\n    Compute a matrix transpose, or reorder any number of axes:\n\n    >>> np.einsum('ji', c)\n    array([[0., 3.],\n           [1., 4.],\n           [2., 5.]])\n    >>> np.einsum('ij->ji', c)\n    array([[0., 3.],\n           [1., 4.],\n           [2., 5.]])\n    >>> np.transpose(c)\n    array([[0., 3.],\n           [1., 4.],\n           [2., 5.]])\n\n    Vector inner products:\n\n    >>> np.einsum('i,i', b, b)\n    array(30.)\n\n    Matrix vector multiplication:\n\n    >>> np.einsum('ij,j', a, b)\n    array([ 30.,  80., 130., 180., 230.])\n    >>> np.dot(a, b)\n    array([ 30.,  80., 130., 180., 230.])\n    >>> np.einsum('...j,j', a, b)\n    array([ 30.,  80., 130., 180., 230.])\n\n    Broadcasting and scalar multiplication:\n\n    >>> np.einsum('..., ...', np.array(3), c)\n    array([[ 0.,  3.,  6.],\n           [ 9., 12., 15.]])\n    >>> np.einsum(',ij', np.array(3), c)\n    array([[ 0.,  3.,  6.],\n           [ 9., 12., 15.]])\n    >>> np.multiply(3, c)\n    array([[ 0.,  3.,  6.],\n           [ 9., 12., 15.]])\n\n    Vector outer product:\n\n    >>> np.einsum('i,j', np.arange(2)+1, b)\n    array([[0., 1., 2., 3., 4.],\n           [0., 2., 4., 6., 8.]])\n\n    Tensor contraction:\n\n    >>> a = np.arange(60.).reshape(3,4,5)\n    >>> b = np.arange(24.).reshape(4,3,2)\n    >>> np.einsum('ijk,jil->kl', a, b)\n    array([[4400., 4730.],\n           [4532., 4874.],\n           [4664., 5018.],\n           [4796., 5162.],\n           [4928., 5306.]])\n\n    Example of ellipsis use:\n\n    >>> a = np.arange(6).reshape((3,2))\n    >>> b = np.arange(12).reshape((4,3))\n    >>> np.einsum('ki,jk->ij', a, b)\n    array([[10., 28., 46., 64.],\n           [13., 40., 67., 94.]])\n    >>> np.einsum('ki,...k->i...', a, b)\n    array([[10., 28., 46., 64.],\n           [13., 40., 67., 94.]])\n    >>> np.einsum('k...,jk', a, b)\n    array([[10., 28., 46., 64.],\n           [13., 40., 67., 94.]])\n\n    Chained array operations. For more complicated contractions, speed ups\n    might be achieved by repeatedly computing a 'greedy' path. Performance\n    improvements can be particularly significant with larger arrays:\n\n    >>> a = np.ones(64).reshape(2,4,8)\n    # Basic `einsum`: ~42.22ms  (benchmarked on 3.4GHz Intel Xeon.)\n    >>> for iteration in range(500):\n    ...     np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a)\n    # Greedy `einsum` (faster optimal path approximation): ~0.117ms\n    >>> for iteration in range(500):\n    ...     np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a, optimize=True)\n    \"\"\"\n    # Grab non-einsum kwargs; do not optimize by default.\n    optimize_arg = kwargs.pop('optimize', False)\n    out = kwargs.pop('out', None)\n\n    subscripts = operands[0]\n    operands = operands[1:]\n    return _api_internal.einsum(*operands, subscripts, out, int(optimize_arg))\n\n\n@set_module('mxnet.ndarray.numpy')\ndef nonzero(a):\n    \"\"\"\n    Return the indices of the elements that are non-zero.\n\n    Returns a tuple of arrays, one for each dimension of `a`,\n    containing the indices of the non-zero elements in that\n    dimension. The values in `a` are always returned in\n    row-major, C-style order.\n\n    To group the indices by element, rather than dimension, use `argwhere`,\n    which returns a row for each non-zero element.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array.\n\n    Returns\n    -------\n    tuple_of_arrays : tuple\n        Indices of elements that are non-zero.\n\n    See Also\n    --------\n    ndarray.nonzero :\n        Equivalent ndarray method.\n\n    Notes\n    -----\n    While the nonzero values can be obtained with ``a[nonzero(a)]``, it is\n    recommended to use ``x[x.astype(bool)]`` or ``x[x != 0]`` instead, which\n    will correctly handle 0-d arrays.\n\n    Examples\n    --------\n    >>> x = np.array([[3, 0, 0], [0, 4, 0], [5, 6, 0]])\n    >>> x\n    array([[3, 0, 0],\n           [0, 4, 0],\n           [5, 6, 0]], dtype=int32)\n    >>> np.nonzero(x)\n    (array([0, 1, 2, 2], dtype=int64), array([0, 1, 0, 1], dtype=int64))\n\n    >>> x[np.nonzero(x)]\n    array([3, 4, 5, 6])\n    >>> np.transpose(np.stack(np.nonzero(x)))\n    array([[0, 0],\n           [1, 1],\n           [2, 0],\n           [2, 1]], dtype=int64)\n\n    A common use for ``nonzero`` is to find the indices of an array, where\n    a condition is True.  Given an array `a`, the condition `a` > 3 is a\n    boolean array and since False is interpreted as 0, np.nonzero(a > 3)\n    yields the indices of the `a` where the condition is true.\n\n    >>> a = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.int32)\n    >>> a > 3\n    array([[False, False, False],\n           [ True,  True,  True],\n           [ True,  True,  True]])\n    >>> np.nonzero(a > 3)\n    (array([1, 1, 1, 2, 2, 2], dtype=int64), array([0, 1, 2, 0, 1, 2], dtype=int64))\n\n    Using this result to index `a` is equivalent to using the mask directly:\n\n    >>> a[np.nonzero(a > 3)]\n    array([4, 5, 6, 7, 8, 9], dtype=int32)\n    >>> a[a > 3]\n    array([4, 5, 6, 7, 8, 9], dtype=int32)\n\n    ``nonzero`` can also be called as a method of the array.\n\n    >>> (a > 3).nonzero()\n    (array([1, 1, 1, 2, 2, 2], dtype=int64), array([0, 1, 2, 0, 1, 2], dtype=int64))\n    \"\"\"\n    out = _api_internal.nonzero(a).transpose()\n    return tuple([out[i] for i in range(len(out))])\n\n\n@set_module('mxnet.ndarray.numpy')\ndef percentile(a, q, axis=None, out=None, overwrite_input=None, interpolation='linear', keepdims=False): # pylint: disable=too-many-arguments\n    \"\"\"\n    Compute the q-th percentile of the data along the specified axis.\n    Returns the q-th percentile(s) of the array elements.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array\n    q : ndarray\n        Percentile or sequence of percentiles to compute.\n    axis : {int, tuple of int, None}, optional\n        Axis or axes along which the percentiles are computed. The default is to\n        compute the percentile(s) along a flattened version of the array.\n    out : ndarray, optional\n        Alternative output array in which to place the result. It must have the same\n        shape and buffer length as the expected output, but the type (of the output)\n        will be cast if necessary.\n    overwrite_input : bool, optional (Not supported yet)\n        If True, then allow the input array a to be modified by intermediate calculations,\n        to save memory. In this case, the contents of the input a after this function\n        completes is undefined.\n    interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}\n        This optional parameter specifies the interpolation method to use when the\n        desired percentile lies between two data points i < j:\n        'linear': i + (j - i) * fraction, where fraction is the fractional part of the\n        index surrounded by i and j.\n        'lower': i.\n        'higher': j.\n        'nearest': i or j, whichever is nearest.\n        'midpoint': (i + j) / 2.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in the result as\n        dimensions with size one. With this option, the result will broadcast\n        correctly against the original array a.\n\n    Returns\n    -------\n    percentile : scalar or ndarray\n        Output array.\n\n    Examples\n    --------\n    >>> a = np.array([[10, 7, 4], [3, 2, 1]])\n    >>> a\n    array([[10,  7,  4],\n        [ 3,  2,  1]])\n    >>> np.percentile(a, np.array(50))\n    array(3.5)\n    >>> np.percentile(a, np.array(50), axis=0)\n    array([6.5, 4.5, 2.5])\n    >>> np.percentile(a, np.array(50), axis=1)\n    array([7.,  2.])\n    >>> np.percentile(a, np.array(50), axis=1, keepdims=True)\n    array([[7.],\n        [2.]])\n\n    >>> m = np.percentile(a, np.array(50), axis=0)\n    >>> out = np.zeros_like(m)\n    >>> np.percentile(a, np.array(50), axis=0, out=out)\n    array([6.5, 4.5, 2.5])\n    >>> m\n    array([6.5, 4.5, 2.5])\n    \"\"\"\n    if overwrite_input is not None:\n        raise NotImplementedError('overwrite_input is not supported yet')\n    return _api_internal.percentile(a, q, axis, interpolation, keepdims, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef median(a, axis=None, out=None, overwrite_input=None, keepdims=False):\n    r\"\"\"\n    Compute the median along the specified axis.\n    Returns the median of the array elements.\n    Parameters\n    ----------\n    a : array_like\n        Input array or object that can be converted to an array.\n    axis : {int, sequence of int, None}, optional\n        Axis or axes along which the medians are computed. The default\n        is to compute the median along a flattened version of the array.\n        A sequence of axes is supported since version 1.9.0.\n    out : ndarray, optional\n        Alternative output array in which to place the result. It must\n        have the same shape and buffer length as the expected output,\n        but the type (of the output) will be cast if necessary.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the original `arr`.\n    Returns\n    -------\n    median : ndarray\n        A new array holding the result. If the input contains integers\n        or floats smaller than ``float32``, then the output data-type is\n        ``np.float32``.  Otherwise, the data-type of the output is the\n        same as that of the input. If `out` is specified, that array is\n        returned instead.\n    See Also\n    --------\n    mean, percentile\n    Examples\n    --------\n    >>> a = np.array([[10, 7, 4], [3, 2, 1]])\n    >>> a\n    array([[10,  7,  4],\n        [ 3,  2,  1]])\n    >>> np.median(a)\n    3.5\n    >>> np.median(a, axis=0)\n    array([6.5, 4.5, 2.5])\n    >>> np.median(a, axis=1)\n    array([7.,  2.])\n    \"\"\"\n    return quantile(a=a, q=0.5, axis=axis, out=out, overwrite_input=overwrite_input,\n                    interpolation='midpoint', keepdims=keepdims)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef quantile(a, q, axis=None, out=None, overwrite_input=None, interpolation='linear', keepdims=False): # pylint: disable=too-many-arguments\n    \"\"\"\n    Compute the q-th quantile of the data along the specified axis.\n    New in version 1.15.0.\n    Parameters\n    ----------\n    a : ndarray\n        Input array or object that can be converted to an array.\n    q : ndarray\n        Quantile or sequence of quantiles to compute, which must be between 0 and 1 inclusive.\n    axis : {int, tuple of int, None}, optional\n        Axis or axes along which the quantiles are computed.\n        The default is to compute the quantile(s) along a flattened version of the array.\n    out : ndarray, optional\n        Alternative output array in which to place the result.\n        It must have the same shape and buffer length as the expected output,\n        but the type (of the output) will be cast if necessary.\n    interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}\n        This optional parameter specifies the interpolation method to use\n        when the desired quantile lies between two data points i < j:\n            linear: i + (j - i) * fraction, where fraction is the fractional part of the index surrounded by i and j.\n            lower: i.\n            higher: j.\n            nearest: i or j, whichever is nearest.\n            midpoint: (i + j) / 2.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in the result as dimensions with size one.\n        With this option, the result will broadcast correctly against the original array a.\n    Returns\n    -------\n    quantile : ndarray\n        If q is a single quantile and axis=None, then the result is a scalar.\n        If multiple quantiles are given, first axis of the result corresponds to the quantiles.\n        The other axes are the axes that remain after the reduction of a.\n        If out is specified, that array is returned instead.\n    See also\n    --------\n    mean\n    Notes\n    -----\n    Given a vector V of length N, the q-th quantile of V is the value q of the way from the minimum\n    to the maximum in a sorted copy of V. The values and distances of the two nearest neighbors\n    as well as the interpolation parameter will determine the quantile if the normalized ranking\n    does not match the location of q exactly. This function is the same as the median if q=0.5,\n    the same as the minimum if q=0.0 and the same as the maximum if q=1.0.\n    This function differs from the original `numpy.quantile\n    <https://numpy.org/devdocs/reference/generated/numpy.quantile.html>`_ in\n    the following aspects:\n    - q must be ndarray type even if it is a scalar\n    - do not support overwrite_input\n    Examples\n    --------\n    >>> a = np.array([[10, 7, 4], [3, 2, 1]])\n    >>> a\n    array([[10., 7., 4.],\n           [3., 2., 1.]])\n    >>> q = np.array(0.5)\n    >>> q\n    array(0.5)\n    >>> np.quantile(a, q)\n    array(3.5)\n    >>> np.quantile(a, q, axis=0)\n    array([6.5, 4.5, 2.5])\n    >>> np.quantile(a, q, axis=1)\n    array([7., 2.])\n    >>> np.quantile(a, q, axis=1, keepdims=True)\n    array([[7.],\n           [2.]])\n    >>> m = np.quantile(a, q, axis=0)\n    >>> out = np.zeros_like(m)\n    >>> np.quantile(a, q, axis=0, out=out)\n    array([6.5, 4.5, 2.5])\n    >>> out\n    array([6.5, 4.5, 2.5])\n    \"\"\"\n    if overwrite_input is not None:\n        raise NotImplementedError('overwrite_input is not supported yet')\n    return _api_internal.percentile(a, q * 100, axis, interpolation, keepdims, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef shares_memory(a, b, max_work=None):\n    \"\"\"\n    Determine if two arrays share memory\n\n    Parameters\n    ----------\n    a, b : ndarray\n        Input arrays\n\n    Returns\n    -------\n    out : bool\n\n    See Also\n    --------\n    may_share_memory\n\n    Examples\n    --------\n    >>> np.may_share_memory(np.array([1,2]), np.array([5,8,9]))\n    False\n\n    This function differs from the original `numpy.shares_memory\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.shares_memory.html>`_ in\n    the following way(s):\n\n    - Does not support `max_work`, it is a dummy argument\n    - Actually it is same as `may_share_memory` in MXNet np\n    \"\"\"\n    return _api_internal.share_memory(a, b).item()\n\n\n@set_module('mxnet.ndarray.numpy')\ndef may_share_memory(a, b, max_work=None):\n    \"\"\"\n    Determine if two arrays might share memory\n\n    A return of True does not necessarily mean that the two arrays\n    share any element.  It just means that they *might*.\n\n    Only the memory bounds of a and b are checked by default.\n\n    Parameters\n    ----------\n    a, b : ndarray\n        Input arrays\n\n    Returns\n    -------\n    out : bool\n\n    See Also\n    --------\n    shares_memory\n\n    Examples\n    --------\n    >>> np.may_share_memory(np.array([1,2]), np.array([5,8,9]))\n    False\n    >>> x = np.zeros([3, 4])\n    >>> np.may_share_memory(x[:,0], x[:,1])\n    True\n\n    This function differs from the original `numpy.may_share_memory\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.may_share_memory.html>`_ in\n    the following way(s):\n\n    - Does not support `max_work`, it is a dummy argument\n    - Actually it is same as `shares_memory` in MXNet np\n    \"\"\"\n    return _api_internal.share_memory(a, b).item()\n\n\n@set_module('mxnet.ndarray.numpy')\ndef interp(x, xp, fp, left=None, right=None, period=None):  # pylint: disable=too-many-arguments\n    \"\"\"\n    One-dimensional linear interpolation.\n    Returns the one-dimensional piecewise linear interpolant to a function\n    with given values at discrete data-points.\n\n    Parameters\n    ----------\n    x : ndarray\n        The x-coordinates of the interpolated values.\n    xp : 1-D array of floats\n        The x-coordinates of the data points, must be increasing if argument\n        `period` is not specified. Otherwise, `xp` is internally sorted after\n        normalizing the periodic boundaries with ``xp = xp % period``.\n    fp : 1-D array of floats\n        The y-coordinates of the data points, same length as `xp`.\n    left : optional float corresponding to fp\n        Value to return for `x < xp[0]`, default is `fp[0]`.\n    right : optional float corresponding to fp\n        Value to return for `x > xp[-1]`, default is `fp[-1]`.\n    period : None or float, optional\n        A period for the x-coordinates. This parameter allows the proper\n        interpolation of angular x-coordinates. Parameters `left` and `right`\n        are ignored if `period` is specified.\n        .. versionadded:: 1.10.0\n\n    Returns\n    -------\n    y : float (corresponding to fp) or ndarray\n        The interpolated values, same shape as `x`.\n    Raises\n    ------\n    ValueError\n        If `xp` and `fp` have different length\n        If `xp` or `fp` are not 1-D sequences\n        If `period == 0`\n\n    Notes\n    -----\n    Does not check that the x-coordinate sequence `xp` is increasing.\n    If `xp` is not increasing, the results are nonsense.\n    A simple check for increasing is::\n        np.all(np.diff(xp) > 0)\n\n    Examples\n    --------\n    >>> xp = [1, 2, 3]\n    >>> fp = [3, 2, 0]\n    >>> np.interp(2.5, xp, fp)\n    1.0\n    >>> np.interp([0, 1, 1.5, 2.72, 3.14], xp, fp)\n    array([ 3. ,  3. ,  2.5 ,  0.56,  0. ])\n    >>> UNDEF = -99.0\n    >>> np.interp(3.14, xp, fp, right=UNDEF)\n    -99.0\n    Plot an interpolant to the sine function:\n    >>> x = np.linspace(0, 2*np.pi, 10)\n    >>> y = np.sin(x)\n    >>> xvals = np.linspace(0, 2*np.pi, 50)\n    >>> yinterp = np.interp(xvals, x, y)\n    >>> import matplotlib.pyplot as plt\n    >>> plt.plot(x, y, 'o')\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.plot(xvals, yinterp, '-x')\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.show()\n    Interpolation with periodic x-coordinates:\n    >>> x = [-180, -170, -185, 185, -10, -5, 0, 365]\n    >>> xp = [190, -190, 350, -350]\n    >>> fp = [5, 10, 3, 4]\n    >>> np.interp(x, xp, fp, period=360)\n    array([7.5, 5., 8.75, 6.25, 3., 3.25, 3.5, 3.75])\n    \"\"\"\n    if not isinstance(x, numeric_types):\n        x = x.astype(float)\n    return _api_internal.interp(xp.astype(float), fp.astype(float), x, left,\n                                right, period)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef diff(a, n=1, axis=-1, prepend=None, append=None):  # pylint: disable=redefined-outer-name\n    r\"\"\"\n    Calculate the n-th discrete difference along the given axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array\n    n : int, optional\n        The number of times values are differenced. If zero, the input is returned as-is.\n    axis : int, optional\n        The axis along which the difference is taken, default is the last axis.\n    prepend, append : ndarray, optional\n        Not supported yet\n\n    Returns\n    -------\n    diff : ndarray\n        The n-th differences.\n        The shape of the output is the same as a except along axis where the dimension is smaller by n.\n        The type of the output is the same as the type of the difference between any two elements of a.\n\n    Examples\n    --------\n    >>> x = np.array([1, 2, 4, 7, 0])\n    >>> np.diff(x)\n    array([ 1,  2,  3, -7])\n    >>> np.diff(x, n=2)\n    array([  1,   1, -10])\n\n    >>> x = np.array([[1, 3, 6, 10], [0, 5, 6, 8]])\n    >>> np.diff(x)\n    array([[2, 3, 4],\n           [5, 1, 2]])\n    >>> np.diff(x, axis=0)\n    array([[-1,  2,  0, -2]])\n\n    Notes\n    -----\n    Optional inputs `prepend` and `append` are not supported yet\n    \"\"\"\n    if (prepend or append):\n        raise NotImplementedError('prepend and append options are not supported yet')\n    return _api_internal.diff(a, n, axis)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef ediff1d(ary, to_end=None, to_begin=None):\n    \"\"\"\n    The differences between consecutive elements of an array.\n\n    Parameters\n    ----------\n    ary : ndarray\n        If necessary, will be flattened before the differences are taken.\n    to_end : ndarray or scalar, optional\n        Number(s) to append at the end of the returned differences.\n    to_begin : ndarray or scalar, optional\n        Number(s) to prepend at the beginning of the returned differences.\n\n    Returns\n    -------\n    ediff1d : ndarray\n        The differences. Loosely, this is ``ary.flat[1:] - ary.flat[:-1]``.\n\n    Examples\n    --------\n    >>> x = np.array([1, 2, 4, 7, 0])\n    >>> np.ediff1d(x)\n    array([ 1.,  2.,  3., -7.])\n\n    >>> np.ediff1d(x, to_begin=-99, to_end=np.array([88, 99]))\n    rray([-99.,   1.,   2.,   3.,  -7.,  88.,  99.])\n\n    The returned array is always 1D.\n\n    >>> y = np.array([[1, 2, 4], [1, 6, 24]])\n    >>> np.ediff1d(y)\n    array([ 1.,  2., -3.,  5., 18.])\n\n    >>> np.ediff1d(x, to_begin=y)\n    array([ 1.,  2.,  4.,  1.,  6., 24.,  1.,  2.,  3., -7.])\n    \"\"\"\n    return _api_internal.ediff1d(ary, to_end, to_begin)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef resize(a, new_shape):\n    \"\"\"\n    Return a new array with the specified shape.\n    If the new array is larger than the original array, then the new\n    array is filled with repeated copies of `a`.  Note that this behavior\n    is different from a.resize(new_shape) which fills with zeros instead\n    of repeated copies of `a`.\n\n    Parameters\n    ----------\n    a : ndarray\n        Array to be resized.\n    new_shape : int or tuple of int\n        Shape of resized array.\n\n    Returns\n    -------\n    reshaped_array : ndarray\n        The new array is formed from the data in the old array, repeated\n        if necessary to fill out the required number of elements.  The\n        data are repeated in the order that they are stored in memory.\n\n    See Also\n    --------\n    ndarray.resize : resize an array in-place.\n\n    Notes\n    -----\n    Warning: This functionality does **not** consider axes separately,\n    i.e. it does not apply interpolation/extrapolation.\n    It fills the return array with the required number of elements, taken\n    from `a` as they are laid out in memory, disregarding strides and axes.\n    (This is in case the new shape is smaller. For larger, see above.)\n    This functionality is therefore not suitable to resize images,\n    or data where each axis represents a separate and distinct entity.\n\n    Examples\n    --------\n    >>> a = np.array([[0, 1], [2, 3]])\n    >>> np.resize(a, (2, 3))\n    array([[0., 1., 2.],\n           [3., 0., 1.]])\n    >>> np.resize(a, (1, 4))\n    array([[0., 1., 2., 3.]])\n    >>> np.resize(a,(2, 4))\n    array([[0., 1., 2., 3.],\n           [0., 1., 2., 3.]])\n    \"\"\"\n    return _npi.resize_fallback(a, new_shape=new_shape)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef fill_diagonal(a, val, wrap=False):\n    \"\"\"\n    Fill the main diagonal of the given array of any dimensionality.\n    For an array `a` with ``a.ndim >= 2``, the diagonal is the list of\n    locations with indices ``a[i, ..., i]`` all identical. This function\n    modifies the input array in-place, it does not return a value.\n\n    Parameters\n    ----------\n    a : array, at least 2-D.\n      Array whose diagonal is to be filled, it gets modified in-place.\n    val : scalar\n      Value to be written on the diagonal, its type must be compatible with\n      that of the array a.\n    wrap : bool\n      For tall matrices in NumPy version up to 1.6.2, the\n      diagonal \"wrapped\" after N columns. You can have this behavior\n      with this option. This affects only tall matrices.\n\n    Examples\n    --------\n    >>> a = np.zeros((3, 3), int)\n    >>> np.fill_diagonal(a, 5)\n    >>> a\n    array([[5, 0, 0],\n           [0, 5, 0],\n           [0, 0, 5]])\n    The same function can operate on a 4-D array:\n    >>> a = np.zeros((3, 3, 3, 3), int)\n    >>> np.fill_diagonal(a, 4)\n    We only show a few blocks for clarity:\n    >>> a[0, 0]\n    array([[4, 0, 0],\n           [0, 0, 0],\n           [0, 0, 0]])\n    >>> a[1, 1]\n    array([[0, 0, 0],\n           [0, 4, 0],\n           [0, 0, 0]])\n    >>> a[2, 2]\n    array([[0, 0, 0],\n           [0, 0, 0],\n           [0, 0, 4]])\n    The wrap option affects only tall matrices:\n    >>> # tall matrices no wrap\n    >>> a = np.zeros((5, 3), int)\n    >>> np.fill_diagonal(a, 4)\n    >>> a\n    array([[4, 0, 0],\n           [0, 4, 0],\n           [0, 0, 4],\n           [0, 0, 0],\n           [0, 0, 0]])\n    >>> # tall matrices wrap\n    >>> a = np.zeros((5, 3), int)\n    >>> np.fill_diagonal(a, 4, wrap=True)\n    >>> a\n    array([[4, 0, 0],\n           [0, 4, 0],\n           [0, 0, 4],\n           [0, 0, 0],\n           [4, 0, 0]])\n    >>> # wide matrices\n    >>> a = np.zeros((3, 5), int)\n    >>> np.fill_diagonal(a, 4, wrap=True)\n    >>> a\n    array([[4, 0, 0, 0, 0],\n           [0, 4, 0, 0, 0],\n           [0, 0, 4, 0, 0]])\n    The anti-diagonal can be filled by reversing the order of elements\n    using either `numpy.flipud` or `numpy.fliplr`.\n    >>> a = np.zeros((3, 3), int);\n    >>> np.fill_diagonal(np.fliplr(a), [1,2,3])  # Horizontal flip\n    >>> a\n    array([[0, 0, 1],\n           [0, 2, 0],\n           [3, 0, 0]])\n    >>> np.fill_diagonal(np.flipud(a), [1,2,3])  # Vertical flip\n    >>> a\n    array([[0, 0, 3],\n           [0, 2, 0],\n           [1, 0, 0]])\n    Note that the order in which the diagonal is filled varies depending\n    on the flip function.\n    \"\"\"\n    if isinstance(val, list):\n        val = [float(v) for v in val]\n    else:\n        val = [float(val)]\n    _api_internal.fill_diagonal(a, val, wrap, a)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef squeeze(x, axis=None):\n    \"\"\"\n    Remove single-dimensional entries from the shape of an array.\n\n    Parameters\n    ----------\n    a : array_like\n        Input data.\n    axis : None or int or tuple of ints, optional\n        .. versionadded:: 1.7.0\n        Selects a subset of the single-dimensional entries in the\n        shape. If an axis is selected with shape entry greater than\n        one, an error is raised.\n\n    Returns\n    -------\n    squeezed : ndarray\n        The input array, but with all or a subset of the\n        dimensions of length 1 removed. This is always `a` itself\n        or a view into `a`.\n\n    Raises\n    ------\n    ValueError\n        If `axis` is not `None`, and an axis being squeezed is not of length 1\n\n    See Also\n    --------\n    expand_dims : The inverse operation, adding singleton dimensions\n    reshape : Insert, remove, and combine dimensions, and resize existing ones\n\n    Examples\n    --------\n    >>> x = np.array([[[0], [1], [2]]])\n    >>> x.shape\n    (1, 3, 1)\n    >>> np.squeeze(x).shape\n    (3,)\n    >>> np.squeeze(x, axis=0).shape\n    (3, 1)\n    >>> np.squeeze(x, axis=1).shape\n    Traceback (most recent call last):\n    ...\n    ValueError: cannot select an axis to squeeze out which has size not equal to one\n    >>> np.squeeze(x, axis=2).shape\n    (1, 3)\n    \"\"\"\n    return _api_internal.squeeze(x, axis)\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.ndarray.numpy')\ndef nan_to_num(x, copy=True, nan=0.0, posinf=None, neginf=None, **kwargs):\n    \"\"\"\n    Replace NaN with zero and infinity with large finite numbers (default\n    behaviour) or with the numbers defined by the user using the `nan`,\n    `posinf` and/or `neginf` keywords.\n\n    If `x` is inexact, NaN is replaced by zero or by the user defined value in\n    `nan` keyword, infinity is replaced by the largest finite floating point\n    values representable by ``x.dtype`` or by the user defined value in\n    `posinf` keyword and -infinity is replaced by the most negative finite\n    floating point values representable by ``x.dtype`` or by the user defined\n    value in `neginf` keyword.\n\n    For complex dtypes, the above is applied to each of the real and\n    imaginary components of `x` separately.\n\n    If `x` is not inexact, then no replacements are made.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input data.\n    copy : bool, optional\n        Whether to create a copy of `x` (True) or to replace values\n        in-place (False). The in-place operation only occurs if\n        casting to an array does not require a copy.\n        Default is True.\n    nan : int, float, optional\n        Value to be used to fill NaN values. If no value is passed\n        then NaN values will be replaced with 0.0.\n    posinf : int, float, optional\n        Value to be used to fill positive infinity values. If no value is\n        passed then positive infinity values will be replaced with a very\n        large number.\n    neginf : int, float, optional\n        Value to be used to fill negative infinity values. If no value is\n        passed then negative infinity values will be replaced with a very\n        small (or negative) number.\n\n        .. versionadded:: 1.13\n\n    Returns\n    -------\n    out : ndarray\n        `x`, with the non-finite values replaced. If `copy` is False, this may\n        be `x` itself.\n\n    Notes\n    -----\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic\n    (IEEE 754). This means that Not a Number is not equivalent to infinity.\n\n    Examples\n    --------\n    >>> np.nan_to_num(np.inf)\n    1.7976931348623157e+308\n    >>> np.nan_to_num(-np.inf)\n    -1.7976931348623157e+308\n    >>> np.nan_to_num(np.nan)\n    0.0\n    >>> x = np.array([np.inf, -np.inf, np.nan, -128, 128])\n    >>> np.nan_to_num(x)\n    array([ 3.4028235e+38, -3.4028235e+38,  0.0000000e+00, -1.2800000e+02,\n            1.2800000e+02])\n    >>> np.nan_to_num(x, nan=-9999, posinf=33333333, neginf=33333333)\n    array([ 3.3333332e+07,  3.3333332e+07, -9.9990000e+03, -1.2800000e+02,\n            1.2800000e+02])\n    >>> y = np.array([[-1, 0, 1],[9999,234,-14222]],dtype=\"float64\")/0\n    array([[-inf,  nan,  inf],\n        [ inf,  inf, -inf]], dtype=float64)\n    >>> np.nan_to_num(y)\n    array([[-1.79769313e+308,  0.00000000e+000,  1.79769313e+308],\n        [ 1.79769313e+308,  1.79769313e+308, -1.79769313e+308]], dtype=float64)\n    >>> np.nan_to_num(y, nan=111111, posinf=222222)\n    array([[-1.79769313e+308,  1.11111000e+005,  2.22222000e+005],\n        [ 2.22222000e+005,  2.22222000e+005, -1.79769313e+308]], dtype=float64)\n    >>> y\n    array([[-inf,  nan,  inf],\n       [ inf,  inf, -inf]], dtype=float64)\n    >>> np.nan_to_num(y, copy=False, nan=111111, posinf=222222)\n    array([[-1.79769313e+308,  1.11111000e+005,  2.22222000e+005],\n       [ 2.22222000e+005,  2.22222000e+005, -1.79769313e+308]], dtype=float64)\n    >>> y\n    array([[-1.79769313e+308,  1.11111000e+005,  2.22222000e+005],\n       [ 2.22222000e+005,  2.22222000e+005, -1.79769313e+308]], dtype=float64)\n    \"\"\"\n    if isinstance(x, numeric_types):\n        return _np.nan_to_num(x, copy, nan, posinf, neginf)\n    elif isinstance(x, NDArray):\n        if x.dtype in ['int8', 'uint8', 'int32', 'int64']:\n            return x\n        if not copy:\n            return _api_internal.nan_to_num(x, copy, nan, posinf, neginf, x)\n        return _api_internal.nan_to_num(x, copy, nan, posinf, neginf, None)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(x))))\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef isnan(x, out=None, **kwargs):\n    \"\"\"\n    Test element-wise for NaN and return result as a boolean array.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or bool\n        True where x is NaN, false otherwise.\n        This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic (IEEE 754).\n\n    This function differs from the original `numpy.isinf\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.isnan.html>`_ in\n    the following aspects:\n    - Does not support complex number for now\n    - Input type does not support Python native iterables(list, tuple, ...).\n    - ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be the same as the expected output.\n    - ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the same as the expected output.\n    - ``out`` param does not support scalar input case.\n\n    Examples\n    --------\n    >>> np.isnan(np.nan)\n    True\n    >>> np.isnan(np.inf)\n    False\n    >>> np.isnan(np.array([np.log(-1.),1.,np.log(0)]))\n    array([ True, False, False])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.isnan, _np.isnan, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef isinf(x, out=None, **kwargs):\n    \"\"\"\n    Test element-wise for positive or negative infinity.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or bool\n        True where x is positive or negative infinity, false otherwise.\n        This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic (IEEE 754).\n    This means that Not a Number is not equivalent to infinity.\n\n    This function differs from the original `numpy.isnan\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.isnan.html>`_ in\n    the following aspects:\n    - Does not support complex number for now\n    - Input type does not support Python native iterables(list, tuple, ...).\n    - ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be the same as the expected output.\n    - ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the same as the expected output.\n    - ``out`` param does not support scalar input case.\n\n    Examples\n    --------\n    >>> np.isinf(np.inf)\n    True\n    >>> np.isinf(np.nan)\n    False\n    >>> np.isinf(np.array([np.inf, -np.inf, 1.0, np.nan]))\n    array([ True,  True, False, False])\n    >>> x = np.array([-np.inf, 0., np.inf])\n    >>> y = np.array([True, True, True], dtype=np.bool_)\n    >>> np.isinf(x, y)\n    array([ True, False,  True])\n    >>> y\n    array([ True, False,  True])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.isinf, _np.isinf, out=out, **kwargs)\n\n\n@wrap_np_unary_func\ndef isposinf(x, out=None, **kwargs):\n    \"\"\"\n    Test element-wise for positive infinity, return result as bool array.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or bool\n        True where x is positive infinity, false otherwise.\n        This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic (IEEE 754).\n    This means that Not a Number is not equivalent to infinity.\n\n    Examples\n    --------\n    >>> np.isposinf(np.inf)\n    True\n    >>> np.isposinf(-np.inf)\n    False\n    >>> np.isposinf(np.nan)\n    False\n    >>> np.isposinf(np.array([-np.inf, 0., np.inf]))\n    array([False, False,  True])\n    >>> x = np.array([-np.inf, 0., np.inf])\n    >>> y = np.array([True, True, True], dtype=np.bool)\n    >>> np.isposinf(x, y)\n    array([False, False,  True])\n    >>> y\n    array([False, False,  True])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.isposinf, _np.isposinf, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef isneginf(x, out=None, **kwargs):\n    \"\"\"\n    Test element-wise for negative infinity, return result as bool array.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or bool\n        True where x is negative infinity, false otherwise.\n        This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic (IEEE 754).\n    This means that Not a Number is not equivalent to infinity.\n\n    Examples\n    --------\n    >>> np.isneginf(-np.inf)\n    True\n    >>> np.isneginf(np.inf)\n    False\n    >>> np.isneginf(float('-inf'))\n    True\n    >>> np.isneginf(np.array([-np.inf, 0., np.inf]))\n    array([ True, False, False])\n    >>> x = np.array([-np.inf, 0., np.inf])\n    >>> y = np.array([True, True, True], dtype=np.bool)\n    >>> np.isneginf(x, y)\n    array([ True, False, False])\n    >>> y\n    array([ True, False, False])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.isneginf, _np.isneginf, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef isfinite(x, out=None, **kwargs):\n    \"\"\"\n    Test element-wise for finiteness (not infinity or not Not a Number).\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or bool\n        True where x is negative infinity, false otherwise.\n        This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    Not a Number, positive infinity and negative infinity are considered to be non-finite.\n\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic (IEEE 754).\n    This means that Not a Number is not equivalent to infinity.\n    Also that positive infinity is not equivalent to negative infinity.\n    But infinity is equivalent to positive infinity. Errors result if the second argument\n    is also supplied when x is a scalar input, or if first and second arguments have different shapes.\n\n    Examples\n    --------\n    >>> np.isfinite(1)\n    True\n    >>> np.isfinite(0)\n    True\n    >>> np.isfinite(np.nan)\n    False\n    >>> np.isfinite(np.inf)\n    False\n    >>> np.isfinite(-np.inf)\n    False\n    >>> np.isfinite(np.array([np.log(-1.),1.,np.log(0)]))\n    array([False,  True, False])\n    >>> x = np.array([-np.inf, 0., np.inf])\n    >>> y = np.array([True, True, True], dtype=np.bool)\n    >>> np.isfinite(x, y)\n    array([False,  True, False])\n    >>> y\n    array([False,  True, False])\n    \"\"\"\n    return _pure_unary_func_helper(x, _api_internal.isfinite, _np.isfinite, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef atleast_1d(*arys):\n    \"\"\"\n    Convert inputs to arrays with at least one dimension.\n\n    Scalar inputs are converted to 1-dimensional arrays, whilst higher-dimensional inputs are preserved.\n\n    Parameters\n    ----------\n    arys1, arys2, ... : ndarray\n        One or more input arrays.\n\n    Returns\n    -------\n    ret : ndarray\n        An array, or list of arrays, each with a.ndim >= 1. Copies are made only if necessary.\n\n    See also\n    --------\n    atleast_2d, atleast_3d\n\n    Examples\n    --------\n    >>> np.atleast_1d(1.0)\n    array([1.])\n    >>> x = np.arange(9.0).reshape(3,3)\n    >>> np.atleast_1d(x)\n    array([[0., 1., 2.],\n           [3., 4., 5.],\n           [6., 7., 8.]])\n    >>> np.atleast_1d(np.array(1), np.array([3, 4]))\n    [array([1.]), array([3., 4.])]\n    \"\"\"\n    if len(arys) == 1:\n        return _api_internal.atleast_1d(*arys)[0]\n    return list(_api_internal.atleast_1d(*arys))\n\n\n@set_module('mxnet.ndarray.numpy')\ndef atleast_2d(*arys):\n    \"\"\"\n    Convert inputs to arrays with at least two dimensions.\n\n    Parameters\n    ----------\n    arys1, arys2, ... : ndarray\n        One or more input arrays.\n\n    Returns\n    -------\n    ret : ndarray\n        An array, or list of arrays, each with a.ndim >= 2. Copies are made only if necessary.\n\n    See also\n    --------\n    atleast_1d, atleast_3d\n\n    Examples\n    --------\n    >>> np.atleast_2d(3.0)\n    array([[3.]])\n    >>> x = np.arange(3.0)\n    >>> np.atleast_2d(x)\n    array([[0., 1., 2.]])\n    >>> np.atleast_2d(np.array(1), np.array([1, 2]), np.array([[1, 2]]))\n    [array([[1.]]), array([[1., 2.]]), array([[1., 2.]])]\n    \"\"\"\n    if len(arys) == 1:\n        return _api_internal.atleast_2d(*arys)[0]\n    return list(_api_internal.atleast_2d(*arys))\n\n\n@set_module('mxnet.ndarray.numpy')\ndef atleast_3d(*arys):\n    \"\"\"\n    Convert inputs to arrays with at least three dimension.\n\n    Parameters\n    ----------\n    arys1, arys2, ... : ndarray\n        One or more input arrays.\n\n    Returns\n    -------\n    ret : ndarray\n        An array, or list of arrays, each with a.ndim >= 3.\n        For example, a 1-D array of shape (N,) becomes a view of shape (1, N, 1),\n        and a 2-D array of shape (M, N) becomes a view of shape (M, N, 1).\n\n    See also\n    --------\n    atleast_1d, atleast_2d\n\n    Examples\n    --------\n    >>> np.atleast_3d(3.0)\n    array([[[3.]]])\n    >>> x = np.arange(3.0)\n    >>> np.atleast_3d(x).shape\n    (1, 3, 1)\n    >>> x = np.arange(12.0).reshape(4,3)\n    >>> np.atleast_3d(x).shape\n    (4, 3, 1)\n    >>> for arr in np.atleast_3d(np.array([1, 2]), np.array([[1, 2]]), np.array([[[1, 2]]])):\n    ...     print(arr, arr.shape)\n    ...\n    [[[1.]\n      [2.]]] (1, 2, 1)\n    [[[1.]\n      [2.]]] (1, 2, 1)\n    [[[1. 2.]]] (1, 1, 2)\n    \"\"\"\n    if len(arys) == 1:\n        return _api_internal.atleast_3d(*arys)[0]\n    return list(_api_internal.atleast_3d(*arys))\n\n\n@set_module('mxnet.ndarray.numpy')\ndef where(condition, x=None, y=None):  # pylint: disable=too-many-return-statements\n    \"\"\"where(condition, [x, y])\n    Return elements chosen from `x` or `y` depending on `condition`.\n\n    .. note::\n        When only `condition` is provided, this function is a shorthand for\n        ``np.asarray(condition).nonzero()``. The rest of this documentation\n        covers only the case where all three arguments are provided.\n\n    Parameters\n    ----------\n    condition : ndarray\n        Where True, yield `x`, otherwise yield `y`.\n    x, y : ndarray\n        Values from which to choose. `x`, `y` and `condition` need to be\n        broadcastable to some shape. `x` and `y` must have the same dtype.\n\n    Returns\n    -------\n    out : ndarray\n        An array with elements from `x` where `condition` is True, and elements\n        from `y` elsewhere.\n\n    Notes\n    -----\n    If all the arrays are 1-D, `where` is equivalent to::\n\n        [xv if c else yv\n        for c, xv, yv in zip(condition, x, y)]\n\n    This function differs from the original `numpy.where\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.where.html>`_ in\n    the following way(s):\n\n    - If `condition` is a scalar, this operator returns x or y directly without broadcasting.\n    - If `condition` is ndarray, while both `x` and `y` are scalars,\n        the output dtype will be `float32`.\n\n    Examples\n    --------\n    >>> a = np.arange(10)\n    >>> a\n    array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.])\n    >>> np.where(a < 5, a, 10*a)\n    array([ 0.,  1.,  2.,  3.,  4., 50., 60., 70., 80., 90.])\n\n    This can be used on multidimensional arrays too:\n\n    >>> cond = np.array([[True, False], [True, True]])\n    >>> x = np.array([[1, 2], [3, 4]])\n    >>> y = np.array([[9, 8], [7, 6]])\n    >>> np.where(cond, x, y)\n    array([[1., 8.],\n           [3., 4.]])\n\n    The shapes of x, y, and the condition are broadcast together:\n\n    >>> x, y = onp.ogrid[:3, :4]\n    >>> x = np.array(x)\n    >>> y = np.array(y)\n    >>> np.where(x < y, x, 10 + y)  # both x and 10+y are broadcast\n    array([[10,  0,  0,  0],\n           [10, 11,  1,  1],\n           [10, 11, 12,  2]], dtype=int64)\n\n    >>> a = np.array([[0, 1, 2],\n    ...               [0, 2, 4],\n    ...               [0, 3, 6]])\n    >>> np.where(a < 4, a, -1)  # -1 is broadcast\n    array([[ 0.,  1.,  2.],\n           [ 0.,  2., -1.],\n           [ 0.,  3., -1.]])\n    \"\"\"\n    if x is None and y is None:\n        return nonzero(condition)\n    else:\n        if isinstance(condition, numeric_types):\n            if condition != 0:\n                return x\n            else:\n                return y\n        else:\n            return _api_internal.where(condition, x, y)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef polyval(p, x):\n    \"\"\"\n    Evaluate a polynomial at specific values.\n    If p is of length N, this function returns the value:\n    p[0]*x**(N-1) + p[1]*x**(N-2) + ... + p[N-2]*x + p[N-1]\n    If x is a sequence, then p(x) is returned for each element of x.\n    If x is another polynomial then the composite polynomial p(x(t)) is returned.\n\n    Parameters\n    ----------\n    p : ndarray\n        1D array of polynomial coefficients (including coefficients equal to zero)\n        from highest degree to the constant term.\n    x : ndarray\n        An array of numbers, at which to evaluate p.\n\n    Returns\n    -------\n    values : ndarray\n        Result array of polynomials\n\n    Notes\n    -----\n    This function differs from the original `numpy.polyval\n    <https://numpy.org/devdocs/reference/generated/numpy.polyval.html>`_ in\n    the following way(s):\n    - Does not support poly1d.\n    - X should be ndarray type even if it contains only one element.\n\n    Examples\n    --------\n    >>> p = np.array([3, 0, 1])\n    array([3., 0., 1.])\n    >>> x = np.array([5])\n    array([5.])\n    >>> np.polyval(p, x)  # 3 * 5**2 + 0 * 5**1 + 1\n    array([76.])\n    >>> x = np.array([5, 4])\n    array([5., 4.])\n    >>> np.polyval(p, x)\n    array([76., 49.])\n    \"\"\"\n    from ...numpy import ndarray\n    if isinstance(p, numeric_types) and isinstance(x, numeric_types):\n        return _np.polyval(p, x)\n    elif isinstance(p, ndarray) and isinstance(x, ndarray):\n        return _api_internal.polyval(p, x)\n    else:\n        raise TypeError('type not supported')\n\n\n@set_module('mxnet.ndarray.numpy')\ndef bincount(x, weights=None, minlength=0):\n    \"\"\"\n    Count number of occurrences of each value in array of non-negative ints.\n\n    Parameters\n    ----------\n    x : ndarray\n        input array, 1 dimension, nonnegative ints.\n    weights: ndarray\n        input weigths same shape as x. (Optional)\n    minlength: int\n        A minimum number of bins for the output. (Optional)\n\n    Returns\n    --------\n    out : ndarray\n        the result of binning the input array. The length of out is equal to amax(x)+1.\n\n    Raises\n    --------\n    Value Error\n        If the input is not 1-dimensional, or contains elements with negative values,\n        or if minlength is negative\n    TypeError\n        If the type of the input is float or complex.\n\n    Examples\n    --------\n    >>> np.bincount(np.arange(5))\n    array([1, 1, 1, 1, 1])\n    >>> np.bincount(np.array([0, 1, 1, 3, 2, 1, 7]))\n    array([1, 3, 1, 1, 0, 0, 0, 1])\n\n    >>> x = np.array([0, 1, 1, 3, 2, 1, 7, 23])\n    >>> np.bincount(x).size == np.amax(x)+1\n    True\n\n    >>> np.bincount(np.arange(5, dtype=float))\n    Traceback (most recent call last):\n    File \"<stdin>\", line 1, in <module>\n    TypeError: array cannot be safely cast to required type\n\n    >>> w = np.array([0.3, 0.5, 0.2, 0.7, 1., -0.6]) # weights\n    >>> x = np.array([0, 1, 1, 2, 2, 2])\n    >>> np.bincount(x,  weights=w)\n    array([ 0.3,  0.7,  1.1])\n    \"\"\"\n    if minlength < 0:\n        raise ValueError(\"Minlength value should greater than 0\")\n    return _api_internal.bincount(x, weights, minlength)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef pad(x, pad_width, mode='constant', **kwargs): # pylint: disable=too-many-arguments\n    \"\"\"\n    Pad an array.\n\n    Parameters\n    ----------\n    array : array_like of rank N\n        The array to pad.\n    pad_width : {sequence, array_like, int}\n        Number of values padded to the edges of each axis.\n        ((before_1, after_1), ... (before_N, after_N)) unique pad widths\n        for each axis.\n        ((before, after),) yields same before and after pad for each axis.\n        (pad,) or int is a shortcut for before = after = pad width for all\n        axes.\n    mode : str or function, optional\n        One of the following string values or a user supplied function.\n        'constant' (default)\n            Pads with a constant value.\n        'edge'\n            Pads with the edge values of array.\n        'linear_ramp'\n            not supported yet\n        'maximum'\n            Pads with the maximum value of all of the\n            vector along each axis.\n        'mean'\n            not supported yet\n        'median'\n            not supported yet\n        'minimum'\n            Pads with the minimum value of all of the\n            vector along each axis.\n        'reflect'\n            Pads with the reflection of the vector mirrored on\n            the first and last values of the vector along each\n            axis.\n        'symmetric'\n            Pads with the reflection of the vector mirrored\n            along the edge of the array.\n        'wrap'\n            not supported yet.\n        'empty'\n            not supported yet.\n        <function>\n            not supported yet.\n    stat_length : not supported yet\n    constant_values : scalar, optional\n        Used in 'constant'.  The values to set the padded values for each\n        axis.\n        Default is 0.\n\n    end_values : not supported yet\n    reflect_type : {'even', 'odd'}, optional\n        only support even now\n\n    Returns\n    -------\n    pad : ndarray\n        Padded array of rank equal to `array` with shape increased\n        according to `pad_width`.\n    \"\"\"\n    # pylint: disable = too-many-return-statements, inconsistent-return-statements\n    if not _np.asarray(pad_width).dtype.kind == 'i':\n        raise TypeError('`pad_width` must be of integral type.')\n    if not isinstance(pad_width, tuple):\n        raise TypeError(\"`pad_width` must be tuple.\")\n    if mode == \"linear_ramp\":\n        raise ValueError(\"mode {'linear_ramp'} is not supported.\")\n    if mode == \"wrap\":\n        raise ValueError(\"mode {'wrap'} is not supported.\")\n    if mode == \"median\":\n        raise ValueError(\"mode {'median'} is not supported.\")\n    if mode == \"mean\":\n        raise ValueError(\"mode {'mean'} is not supported.\")\n    if mode == \"empty\":\n        raise ValueError(\"mode {'empty'} is not supported.\")\n    if callable(mode):\n        raise ValueError(\"mode {'<function>'} is not supported.\")\n\n    allowedkwargs = {\n        'constant': ['constant_values'],\n        'edge': [],\n        'linear_ramp': ['end_values'],\n        'maximum': ['stat_length'],\n        'mean': ['stat_length'],\n        'median': ['stat_length'],\n        'minimum': ['stat_length'],\n        'reflect': ['reflect_type'],\n        'symmetric': ['reflect_type'],\n        'wrap': [],\n        }\n\n    if isinstance(mode, _np.compat.basestring):\n        # Make sure have allowed kwargs appropriate for mode\n        for key in kwargs:\n            if key not in allowedkwargs[mode]:\n                raise ValueError(f'{key} keyword not in allowed keywords {allowedkwargs[mode]}')\n\n    unsupported_kwargs = set(kwargs) - set(allowedkwargs[mode])\n    if unsupported_kwargs:\n        raise ValueError(\"unsupported keyword arguments for mode '{}': {}\"\n                         .format(mode, unsupported_kwargs))\n    if mode == \"constant\":\n        values = kwargs.get(\"constant_values\", 0)\n        if isinstance(values, tuple):\n            raise TypeError(\"unsupported constant_values type: {'tuple'}.\")\n        return _api_internal.pad(x, pad_width, 'constant', values, \"even\")\n    elif mode == \"symmetric\":\n        values = kwargs.get(\"reflect_type\", \"even\")\n        if values != \"even\" and values is not None:\n            raise ValueError(\"unsupported reflect_type '{}'\".format(values))\n        return _api_internal.pad(x, pad_width, 'symmetric', 0, \"even\")\n    elif mode == \"edge\":\n        return _api_internal.pad(x, pad_width, 'edge', 0, \"even\")\n    elif mode == \"reflect\":\n        values = kwargs.get(\"reflect_type\", \"even\")\n        if values != \"even\" and values is not None:\n            raise ValueError(\"unsupported reflect_type '{}'\".format(values))\n        return _api_internal.pad(x, pad_width, 'reflect', 0, \"even\")\n    elif mode == \"maximum\":\n        values = kwargs.get(\"stat_length\", None)\n        if values is not None:\n            raise ValueError(\"unsupported stat_length '{}'\".format(values))\n        return _api_internal.pad(x, pad_width, 'maximum', 0, \"even\")\n    elif mode == \"minimum\":\n        values = kwargs.get(\"stat_length\", None)\n        if values is not None:\n            raise ValueError(\"unsupported stat_length '{}'\".format(values))\n        return _api_internal.pad(x, pad_width, 'minimum', 0, \"even\")\n    return _api_internal.pad(x, pad_width, 'constant', 0, \"even\")\n\n\n@set_module('mxnet.ndarray.numpy')\ndef prod(a, axis=None, dtype=None, out=None, keepdims=False, initial=None): # pylint: disable=too-many-arguments\n    \"\"\"\n    Return the product of array elements over a given axis.\n\n    Parameters\n    ----------\n    a : array_like\n        Input data.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which a product is performed.  The default,\n        axis=None, will calculate the product of all the elements in the\n        input array. If axis is negative it counts from the last to the\n        first axis.\n        .. versionadded:: 1.7.0\n        If axis is a tuple of ints, a product is performed on all of the\n        axes specified in the tuple instead of a single axis or all the\n        axes as before.\n    dtype : dtype, optional\n        The type of the returned array, as well as of the accumulator in\n        which the elements are multiplied.  The dtype of `a` is used by\n        default unless `a` has an integer dtype of less precision than the\n        default platform integer.  In that case, if `a` is signed then the\n        platform integer is used while if `a` is unsigned then an unsigned\n        integer of the same precision as the platform integer is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result. It must have\n        the same shape as the expected output, but the type of the output\n        values will be cast if necessary.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in the\n        result as dimensions with size one. With this option, the result\n        will broadcast correctly against the input array.\n        If the default value is passed, then `keepdims` will not be\n        passed through to the `prod` method of sub-classes of\n        `ndarray`, however any non-default value will be.  If the\n        sub-class' method does not implement `keepdims` any\n        exceptions will be raised.\n    initial : scalar, optional\n        The starting value for this product. See `~numpy.ufunc.reduce` for details.\n    where : not supported\n\n    Returns\n    -------\n    product_along_axis : ndarray, see `dtype` parameter above.\n        An array shaped as `a` but with the specified axis removed.\n        Returns a reference to `out` if specified.\n\n    Examples\n    --------\n    By default, calculate the product of all elements:\n    >>> np.prod([1.,2.])\n    2.0\n    Even when the input array is two-dimensional:\n    >>> np.prod([[1.,2.],[3.,4.]])\n    24.0\n    But we can also specify the axis over which to multiply:\n    >>> np.prod([[1.,2.],[3.,4.]], axis=1)\n    array([  2.,  12.])\n    Or select specific elements to include:\n    >>> np.prod([1., np.nan, 3.], where=[True, False, True])\n    3.0\n    If the type of `x` is unsigned, then the output type is\n    the unsigned platform integer:\n    >>> x = np.array([1, 2, 3], dtype=np.uint8)\n    >>> np.prod(x).dtype == np.uint\n    True\n    If `x` is of a signed integer type, then the output type\n    is the default platform integer:\n    >>> x = np.array([1, 2, 3], dtype=np.int8)\n    >>> np.prod(x).dtype == int\n    True\n    You can also start the product with a value other than one:\n    >>> np.prod([1, 2], initial=5)\n    10\n    \"\"\"\n    return _api_internal.prod(a, axis, dtype, keepdims, initial, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef cumsum(a, axis=None, dtype=None, out=None):\n    \"\"\"\n    Return the cumulative sum of the elements along a given axis.\n\n    Parameters\n    ----------\n    a : array_like\n        Input array.\n    axis : int, optional\n        Axis along which the cumulative sum is computed. The default\n        (None) is to compute the cumsum over the flattened array.\n    dtype : dtype, optional\n        Type of the returned array and of the accumulator in which the\n        elements are summed.  If `dtype` is not specified, it defaults\n        to the dtype of `a`, unless `a` has an integer dtype with a\n        precision less than that of the default platform integer.  In\n        that case, the default platform integer is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result. It must\n        have the same shape and buffer length as the expected output\n        but the type will be cast if necessary. See `doc.ufuncs`\n        (Section \"Output arguments\") for more details.\n\n    Returns\n    -------\n    cumsum_along_axis : ndarray.\n        A new array holding the result is returned unless `out` is\n        specified, in which case a reference to `out` is returned. The\n        result has the same size as `a`, and the same shape as `a` if\n        `axis` is not None or `a` is a 1-d array.\n\n    Examples\n    --------\n    >>> a = np.array([[1,2,3], [4,5,6]])\n    >>> a\n    array([[1, 2, 3],\n           [4, 5, 6]])\n    >>> np.cumsum(a)\n    array([ 1,  3,  6, 10, 15, 21])\n    >>> np.cumsum(a, dtype=float)     # specifies type of output value(s)\n    array([  1.,   3.,   6.,  10.,  15.,  21.])\n    >>> np.cumsum(a,axis=0)      # sum over rows for each of the 3 columns\n    array([[1, 2, 3],\n           [5, 7, 9]])\n    >>> np.cumsum(a,axis=1)      # sum over columns for each of the 2 rows\n    array([[ 1,  3,  6],\n           [ 4,  9, 15]])\n    \"\"\"\n    return _api_internal.cumsum(a, axis, dtype, out)\n\n@set_module('mxnet.ndarray.numpy')\ndef reshape(a, newshape, order='C'):\n    \"\"\"\n    Gives a new shape to an array without changing its data.\n    This function always returns a copy of the input array if\n    ``out`` is not provided.\n\n    Parameters\n    ----------\n    a : ndarray\n        Array to be reshaped.\n\n    newshape : int or tuple of ints\n        The new shape should be compatible with the original shape. If\n        an integer, then the result will be a 1-D array of that length.\n        One shape dimension can be -1. In this case, the value is\n        inferred from the length of the array and remaining dimensions.\n\n    order : {'C'}, optional\n        Read the elements of `a` using this index order, and place the\n        elements into the reshaped array using this index order.  'C'\n        means to read / write the elements using C-like index order,\n        with the last axis index changing fastest, back to the first\n        axis index changing slowest. Other order types such as 'F'/'A'\n        may be added in the future.\n\n    Returns\n    -------\n    reshaped_array : ndarray\n        It will be always a copy of the original array. This behavior is different\n        from the official NumPy ``reshape`` operator where views of the original array may be\n        generated.\n\n    See Also\n    --------\n    ndarray.reshape : Equivalent method.\n\n    Examples\n    --------\n    >>> a = np.arange(6).reshape((3, 2))\n    >>> a\n    array([[0., 1.],\n           [2., 3.],\n           [4., 5.]])\n\n    >>> np.reshape(a, (2, 3)) # C-like index ordering\n    array([[0., 1., 2.],\n           [3., 4., 5.]])\n\n    >>> np.reshape(np.ravel(a), (2, 3)) # equivalent to C ravel then C reshape\n    array([[0., 1., 2.],\n           [3., 4., 5.]])\n\n    >>> a = np.array([[1,2,3], [4,5,6]])\n    >>> np.reshape(a, 6)\n    array([1., 2., 3., 4., 5., 6.])\n\n    >>> np.reshape(a, (3,-1))       # the unspecified value is inferred to be 2\n    array([[1., 2.],\n           [3., 4.],\n           [5., 6.]])\n    \"\"\"\n    return _api_internal.reshape(a, newshape, False, order)\n\n@set_module('mxnet.ndarray.numpy')\ndef moveaxis(a, source, destination):\n    \"\"\"Move axes of an array to new positions.\n    Other axes remain in their original order.\n\n    Parameters\n    ----------\n    a : ndarray\n        The array whose axes should be reordered.\n    source : int or sequence of int\n        Original positions of the axes to move. These must be unique.\n    destination : int or sequence of int\n        Destination positions for each of the original axes. These must also be\n        unique.\n\n    Returns\n    -------\n    result : ndarray\n        Array with moved axes. This array is a view of the input array.\n\n    See Also\n    --------\n        transpose: Permute the dimensions of an array.\n        swapaxes: Interchange two axes of an array.\n\n    Examples\n    --------\n    >>> x = np.zeros((3, 4, 5))\n    >>> np.moveaxis(x, 0, -1).shape\n    (4, 5, 3)\n    >>> np.moveaxis(x, -1, 0).shape\n    (5, 3, 4)\n    These all achieve the same result:\n    >>> np.transpose(x).shape\n    (5, 4, 3)\n    >>> np.swapaxes(x, 0, -1).shape\n    (5, 4, 3)\n    >>> np.moveaxis(x, [0, 1], [-1, -2]).shape\n    (5, 4, 3)\n    >>> np.moveaxis(x, [0, 1, 2], [-1, -2, -3]).shape\n    (5, 4, 3)\n    \"\"\"\n    return _api_internal.moveaxis(a, source, destination)\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.ndarray.numpy')\ndef copy(a):\n    \"\"\"\n    Return an array copy of the given object.\n\n    Parameters\n    ----------\n    a :\n        Input array.\n\n    Returns\n    -------\n    arr : ndarray\n        Array interpretation of a.\n\n    -----\n    Examples\n    --------\n    >>> x = np.array([1, 2, 3])\n    >>> y = x\n    >>> z = np.copy(x)\n    >>> x[0] = 10\n    >>> x[0] == y[0]\n        True\n    >>> x[0] == z[0]\n        False\n    \"\"\"\n    return _api_internal.copy(a)\n\n@set_module('mxnet.ndarray.numpy')\ndef rollaxis(a, axis, start=0):\n    \"\"\"\n    Roll the specified axis backwards, until it lies in a given position.\n    a\n        Input array.\n    axis : integer\n        The axis to roll backwards. The positions of the other axes do not\n        change relative to one another.\n    start: int, optional\n        The axis is rolled until it lies before this position.\n        The default, 0, results in a “complete” roll.\n\n    Returns\n    -------\n    res : ndarray\n        A view after applying rollaxis to `a` is returned.\n\n    -----\n    Examples\n    --------\n    >>> a = np.ones((3,4,5,6))\n    >>> np.rollaxis(a, 3, 1).shape\n    (3, 6, 4, 5)\n    >>> np.rollaxis(a, 2).shape\n    (5, 3, 4, 6)\n    >>> np.rollaxis(a, 1, 4).shape\n    (3, 5, 6, 4)\n    \"\"\"\n    return _api_internal.rollaxis(a, axis, start)\n\n@set_module('mxnet.ndarray.numpy')\ndef diag(v, k=0):\n    \"\"\"\n    Extracts a diagonal or constructs a diagonal array.\n    - 1-D arrays: constructs a 2-D array with the input as its diagonal, all other elements are zero.\n    - 2-D arrays: extracts the k-th Diagonal\n\n    Parameters\n    ----------\n    array : ndarray\n        The array to apply diag method.\n    k : offset\n        extracts or constructs kth diagonal given input array\n\n    Returns\n    ----------\n    out : ndarray\n    The extracted diagonal or constructed diagonal array.\n\n    Examples\n    --------\n    >>> x = np.arange(9).reshape((3,3))\n    >>> x\n    array([[0, 1, 2],\n           [3, 4, 5],\n           [6, 7, 8]])\n    >>> np.diag(x)\n    array([0, 4, 8])\n    >>> np.diag(x, k=1)\n    array([1, 5])\n    >>> np.diag(x, k=-1)\n    array([3, 7])\n\n    >>> np.diag(np.diag(x))\n    array([[0, 0, 0],\n           [0, 4, 0],\n           [0, 0, 8]])\n    \"\"\"\n    return _api_internal.diag(v, k)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef diagflat(v, k=0):\n    \"\"\"\n    Create a two-dimensional array with the flattened input as a diagonal.\n\n    Parameters\n    ----------\n    v : array_like\n        Input data, which is flattened and set as the `k`-th\n        diagonal of the output.\n    k : int, optional\n        Diagonal to set; 0, the default, corresponds to the \"main\" diagonal,\n        a positive (negative) `k` giving the number of the diagonal above\n        (below) the main.\n\n    Returns\n    -------\n    out : ndarray\n        The 2-D output array.\n\n    See Also\n    --------\n    diag : MATLAB work-alike for 1-D and 2-D arrays.\n    diagonal : Return specified diagonals.\n    trace : Sum along diagonals.\n\n    Examples\n    --------\n    >>> np.diagflat([[1,2], [3,4]])\n    array([[1, 0, 0, 0],\n           [0, 2, 0, 0],\n           [0, 0, 3, 0],\n           [0, 0, 0, 4]])\n    >>> np.diagflat([1,2], 1)\n    array([[0, 1, 0],\n           [0, 0, 2],\n           [0, 0, 0]])\n    \"\"\"\n    return _api_internal.diagflat(v, k)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef diagonal(a, offset=0, axis1=0, axis2=1):\n    \"\"\"\n    If a is 2-D, returns the diagonal of a with the given offset, i.e., the collection of elements of\n    the form a[i, i+offset]. If a has more than two dimensions, then the axes specified by axis1 and\n    axis2 are used to determine the 2-D sub-array whose diagonal is returned. The shape of the\n    resulting array can be determined by removing axis1 and axis2 and appending an index to the\n    right equal to the size of the resulting diagonals.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data from which diagonal are taken.\n    offset: int, Optional\n        Offset of the diagonal from the main diagonal\n    axis1: int, Optional\n        Axis to be used as the first axis of the 2-D sub-arrays\n    axis2: int, Optional\n        Axis to be used as the second axis of the 2-D sub-arrays\n\n    Returns\n    -------\n    out : ndarray\n        Output result\n\n    Raises\n    -------\n    ValueError:  If the dimension of a is less than 2.\n\n    Examples\n    --------\n    >>> a = np.arange(4).reshape(2,2)\n    >>> a\n    array([[0, 1],\n        [2, 3]])\n    >>> np.diagonal(a)\n    array([0, 3])\n    >>> np.diagonal(a, 1)\n    array([1])\n\n    >>> a = np.arange(8).reshape(2,2,2)\n    >>>a\n    array([[[0, 1],\n            [2, 3]],\n            [[4, 5],\n            [6, 7]]])\n    >>> np.diagonal(a, 0, 0, 1)\n    array([[0, 6],\n            [1, 7]])\n    \"\"\"\n    return _api_internal.diagonal(a, offset, axis1, axis2)\n\n\n# pylint:disable=redefined-outer-name, too-many-arguments\n@set_module('mxnet.ndarray.numpy')\ndef sum(a, axis=None, dtype=None, out=None, keepdims=None, initial=None, where=None):\n    r\"\"\"\n    Sum of array elements over a given axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data.\n    axis : None or int, optional\n        Axis or axes along which a sum is performed.  The default,\n        axis=None, will sum all of the elements of the input array.  If\n        axis is negative it counts from the last to the first axis.\n    dtype : dtype, optional\n        The type of the returned array and of the accumulator in which the\n        elements are summed. The default type is float32.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the input array.\n\n        If the default value is passed, then `keepdims` will not be\n        passed through to the `sum` method of sub-classes of\n        `ndarray`, however any non-default value will be.  If the\n        sub-classes `sum` method does not implement `keepdims` any\n        exceptions will be raised.\n    initial: Currently only supports None as input, optional\n        Starting value for the sum.\n        Currently not implemented. Please use ``None`` as input or skip this argument.\n    out : ndarray or None, optional\n        Alternative output array in which to place the result. It must have\n        the same shape and dtype as the expected output.\n\n    Returns\n    -------\n    sum_along_axis : ndarray\n        An ndarray with the same shape as `a`, with the specified\n        axis removed. If an output array is specified, a reference to\n        `out` is returned.\n\n    Notes\n    -----\n    - Input type does not support Python native iterables.\n    - \"out\" param: cannot perform auto type change. out ndarray's dtype must be the same as the expected output.\n    - \"initial\" param is not supported yet. Please use None as input.\n    - Arithmetic is modular when using integer types, and no error is raised on overflow.\n    - The sum of an empty array is the neutral element 0:\n\n    >>> a = np.empty(1)\n    >>> np.sum(a)\n    array(0.)\n\n    This function differs from the original `numpy.sum\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.sum.html>`_ in\n    the following aspects:\n\n    - Input type does not support Python native iterables(list, tuple, ...).\n    - \"out\" param: cannot perform auto type cast. out ndarray's dtype must be the same as the expected output.\n    - \"initial\" param is not supported yet. Please use ``None`` as input or skip it.\n    - The default type is float32.\n\n    Examples\n    --------\n    >>> a = np.array([0.5, 1.5])\n    >>> np.sum(a)\n    array(2.)\n    >>> a = np.array([0.5, 0.7, 0.2, 1.5])\n    >>> np.sum(a, dtype=np.int32)\n    array(2, dtype=int32)\n    >>> a = np.array([[0, 1], [0, 5]])\n    >>> np.sum(a)\n    array(6.)\n    >>> np.sum(a, axis=0)\n    array([0., 6.])\n    >>> np.sum(a, axis=1)\n    array([1., 5.])\n\n    With output ndarray:\n\n    >>> a = np.array([[0, 1], [0, 5]])\n    >>> b = np.ones((2,), dtype=np.float32)\n    >>> np.sum(a, axis=0, out=b)\n    array([0., 6.])\n    >>> b\n    array([0., 6.])\n\n    If the accumulator is too small, overflow occurs:\n\n    >>> np.ones(128, dtype=np.int8).sum(dtype=np.int8)\n    array(-128, dtype=int8)\n    \"\"\"\n    if where is not None and where is not True:\n        raise ValueError(\"only where=None or where=True cases are supported for now\")\n    return _api_internal.sum(a, axis, dtype, keepdims, initial, out)\n# pylint:enable=redefined-outer-name, too-many-arguments\n\n\n@set_module('mxnet.ndarray.numpy')\ndef bitwise_left_shift(x1, x2, out=None):\n    r\"\"\"\n    Shift the bits of and integer to the left. Bits are shifted to the left by\n    appending x2 0s at the right of x1. Since the internal representation of numbers\n    is in binary format, this operation is equivalent to ``x1 * 2**x2``\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Input values.\n    x2 : ndarray or scalar\n        Number of zeros to append to x1. Has to be non-negative. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which becomes the shape of the output).\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape that the\n        inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n        Result.\n\n    Examples\n    --------\n    >>> np.binary_repr(5)\n    '101'\n    >>> np.left_shift(5, 2)\n    20\n    >>> np.binary_repr(20)\n    '10100'\n    >>> np.left_shift(5, np.array([1,2,3]))\n    array([10, 20, 40])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.left_shift(x1, x2, out=out)\n    return _api_internal.bitwise_left_shift(x1, x2, out)\n\n\n@set_module('mxnet.ndarray.numpy')\ndef bitwise_right_shift(x1, x2, out=None):\n    r\"\"\"\n    Shift the bits of and integer to the right. Bits are shifted to the right by\n    x2. Because the internal representation of numbers is in binary format,\n    this operation is equivalent to ``x1 / 2**x2``\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Input values.\n    x1 : ndarray or scalar\n        Number of bits to remove at the right of x1. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which becomes the shape of the output).\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape that the\n        inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n        Result.\n\n    Examples\n    --------\n    >>> np.binary_repr(10)\n    '1010'\n    >>> np.right_shift(10, 1)\n    5\n    >>> np.binary_repr(5)\n    '101'\n    >>> np.right_shift(10, np.array([1,2,3]))\n    array([5, 2, 1])\n    \"\"\"\n    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):\n        return _np.right_shift(x1, x2, out=out)\n    return _api_internal.bitwise_right_shift(x1, x2, out)\n"
  },
  {
    "path": "python/mxnet/ndarray/numpy/_register.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Registering numpy ops.\"\"\"\n\nfrom ...base import _init_np_op_module\nfrom ..register import _make_ndarray_function\n\n\n_init_np_op_module(root_module_name='mxnet', np_module_name='numpy',\n                   mx_module_name='ndarray', make_op_func=_make_ndarray_function)\n\n_init_np_op_module(root_module_name='mxnet', np_module_name='numpy._internal',\n                   mx_module_name='ndarray', make_op_func=_make_ndarray_function)\n"
  },
  {
    "path": "python/mxnet/ndarray/numpy/linalg.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for operators used in Gluon dispatched by F=ndarray.\"\"\"\n\nimport numpy as _np\nfrom . import _op as _mx_nd_np\nfrom . import _api_internal\n\n__all__ = ['norm', 'svd', 'cholesky', 'qr', 'inv', 'det', 'slogdet', 'solve', 'tensorinv', 'tensorsolve',\n           'pinv', 'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq', 'matrix_rank']\n\n\ndef matrix_rank(M, tol=None, hermitian=False):\n    \"\"\"\n    Return matrix rank of array using SVD method\n\n    Rank of the array is the number of singular values of the array that are\n    greater than `tol`.\n\n    Parameters\n    M : {(M,), (..., M, N)} ndarray\n        Input vector or stack of matrices.\n    tol : (...) ndarray, float, optional\n        Threshold below which SVD values are considered zero. If `tol` is\n        None, and ``S`` is an array with singular values for `M`, and\n        ``eps`` is the epsilon value for datatype of ``S``, then `tol` is\n        set to ``S.max() * max(M.shape) * eps``.\n    hermitian : bool, optional\n        If True, `M` is assumed to be Hermitian (symmetric if real-valued),\n        enabling a more efficient method for finding singular values.\n        Defaults to False.\n\n    Returns\n    -------\n    rank : (...) ndarray\n        Rank of M.\n\n    Examples\n    --------\n    >>> from mxnet import np\n    >>> np.matrix_rank(np.eye(4)) # Full rank matrix\n    4\n    >>> I=np.eye(4); I[-1,-1] = 0. # rank deficient matrix\n    >>> np.matrix_rank(I)\n    3\n    >>> np.matrix_rank(np.ones((4,))) # 1 dimension - rank 1 unless all 0\n    1\n    >>> np.matrix_rank(np.zeros((4,)))\n    0\n    \"\"\"\n    finfo_eps_32 = _np.finfo(_np.float32).eps\n    finfo_eps_64 = _np.finfo(_np.float64).eps\n    if hermitian is True:\n        raise NotImplementedError(\"hermitian is not supported yet...\")\n    return _api_internal.matrix_rank(M, tol, hermitian, finfo_eps_32, finfo_eps_64)\n\n\ndef lstsq(a, b, rcond='warn'):\n    r\"\"\"\n    Return the least-squares solution to a linear matrix equation.\n\n    Solves the equation :math:`a x = b` by computing a vector `x` that\n    minimizes the squared Euclidean 2-norm :math:`\\| b - a x \\|^2_2`.\n    The equation may be under-, well-, or over-determined (i.e., the\n    number of linearly independent rows of `a` can be less than, equal\n    to, or greater than its number of linearly independent columns).\n    If `a` is square and of full rank, then `x` (but for round-off error)\n    is the \"exact\" solution of the equation.\n\n    Parameters\n    ----------\n    a : (M, N) ndarray\n        \"Coefficient\" matrix.\n    b : {(M,), (M, K)} ndarray\n        Ordinate or \"dependent variable\" values. If `b` is two-dimensional,\n        the least-squares solution is calculated for each of the `K` columns\n        of `b`.\n    rcond : float, optional\n        Cut-off ratio for small singular values of `a`.\n        For the purposes of rank determination, singular values are treated\n        as zero if they are smaller than `rcond` times the largest singular\n        value of `a`\n        The default of ``warn`` or ``-1`` will use the machine precision as\n        `rcond` parameter. The default of ``None`` will use the machine\n        precision times `max(M, N)`.\n\n    Returns\n    -------\n    x : {(N,), (N, K)} ndarray\n        Least-squares solution. If `b` is two-dimensional,\n        the solutions are in the `K` columns of `x`.\n    residuals : {(1,), (K,), (0,)} ndarray\n        Sums of residuals.\n        Squared Euclidean 2-norm for each column in ``b - a*x``.\n        If the rank of `a` is < N or M <= N, this is an empty array.\n        If `b` is 1-dimensional, this is a (1,) shape array.\n        Otherwise the shape is (K,).\n    rank : int\n        Rank of matrix `a`.\n    s : (min(M, N),) ndarray\n        Singular values of `a`.\n\n    Raises\n    ------\n    MXNetError\n        If computation does not converge.\n\n    Notes\n    -----\n    If `b` is a matrix, then all array results are returned as matrices.\n\n    Examples\n    --------\n    >>> x = np.array([0, 1, 2, 3])\n    >>> y = np.array([-1, 0.2, 0.9, 2.1])\n    >>> A = np.vstack([x, np.ones(len(x))]).T\n    >>> A\n    array([[ 0.,  1.],\n           [ 1.,  1.],\n           [ 2.,  1.],\n           [ 3.,  1.]])\n    >>> m, c = np.linalg.lstsq(A, y, rcond=None)[0]\n    >>> m, c\n    (1.0 -0.95) # may vary\n    \"\"\"\n    finfo_eps_32 = _np.finfo(_np.float32).eps\n    finfo_eps_64 = _np.finfo(_np.float64).eps\n    x, residuals, rank, s = _api_internal.lstsq(a, b, rcond, finfo_eps_32, finfo_eps_64)\n    return (x, residuals, rank, s)\n\n\ndef pinv(a, rcond=1e-15, hermitian=False):\n    r\"\"\"\n    Compute the (Moore-Penrose) pseudo-inverse of a matrix.\n\n    Calculate the generalized inverse of a matrix using its\n    singular-value decomposition (SVD) and including all\n    *large* singular values.\n\n    Parameters\n    ----------\n    a : (..., M, N) ndarray\n        Matrix or stack of matrices to be pseudo-inverted.\n    rcond : (...) {float or ndarray of float}, optional\n        Cutoff for small singular values.\n        Singular values less than or equal to\n        ``rcond * largest_singular_value`` are set to zero.\n        Broadcasts against the stack of matrices.\n    hermitian : bool, optional\n        If True, `a` is assumed to be Hermitian (symmetric if real-valued),\n        enabling a more efficient method for finding singular values.\n        Defaults to False.\n\n    Returns\n    -------\n    B : (..., N, M) ndarray\n        The pseudo-inverse of `a`. If `a` is a `matrix` instance, then so\n        is `B`.\n\n    Raises\n    ------\n    MXNetError\n        If the SVD computation does not converge.\n\n    Notes\n    -----\n    The pseudo-inverse of a matrix A, denoted :math:`A^+`, is\n    defined as: \"the matrix that 'solves' [the least-squares problem]\n    :math:`Ax = b`,\" i.e., if :math:`\\\\bar{x}` is said solution, then\n    :math:`A^+` is that matrix such that :math:`\\\\bar{x} = A^+b`.\n\n    It can be shown that if :math:`Q_1 \\\\Sigma Q_2^T = A` is the singular\n    value decomposition of A, then\n    :math:`A^+ = Q_2 \\\\Sigma^+ Q_1^T`, where :math:`Q_{1,2}` are\n    orthogonal matrices, :math:`\\\\Sigma` is a diagonal matrix consisting\n    of A's so-called singular values, (followed, typically, by\n    zeros), and then :math:`\\\\Sigma^+` is simply the diagonal matrix\n    consisting of the reciprocals of A's singular values\n    (again, followed by zeros). [1]_\n\n    References\n    ----------\n    .. [1] G. Strang, *Linear Algebra and Its Applications*, 2nd Ed., Orlando,\n           FL, Academic Press, Inc., 1980, pp. 139-142.\n\n    Examples\n    --------\n    The following example checks that ``a * a+ * a == a`` and\n    ``a+ * a * a+ == a+``:\n    >>> a = np.random.randn(2, 3)\n    >>> pinv_a = np.linalg.pinv(a)\n    >>> (a - np.dot(a, np.dot(pinv_a, a))).sum()\n    array(0.)\n    >>> (pinv_a - np.dot(pinv_a, np.dot(a, pinv_a))).sum()\n    array(0.)\n    \"\"\"\n    if hermitian is True:\n        raise NotImplementedError(\"hermitian is not supported yet...\")\n    return _api_internal.pinv(a, rcond, hermitian)\n\n\n# pylint: disable=too-many-return-statements\ndef norm(x, ord=None, axis=None, keepdims=False):\n    r\"\"\"Matrix or vector norm.\n    This function is able to return one of eight different matrix norms,\n    or one of an infinite number of vector norms (described below), depending\n    on the value of the ``ord`` parameter.\n    Parameters\n    ----------\n    x : ndarray\n        Input array.  If `axis` is None, `x` must be 1-D or 2-D.\n    ord : {non-zero int, inf, -inf, 'fro', 'nuc'}, optional\n        Order of the norm (see table under ``Notes``). inf means numpy's\n        `inf` object.\n    axis : {int, 2-tuple of ints, None}, optional\n        If `axis` is an integer, it specifies the axis of `x` along which to\n        compute the vector norms.  If `axis` is a 2-tuple, it specifies the\n        axes that hold 2-D matrices, and the matrix norms of these matrices\n        are computed.  If `axis` is None then either a vector norm (when `x`\n        is 1-D) or a matrix norm (when `x` is 2-D) is returned.\n    keepdims : bool, optional\n        If this is set to True, the axes which are normed over are left in the\n        result as dimensions with size one.  With this option the result will\n        broadcast correctly against the original `x`.\n    Returns\n    -------\n    n : ndarray\n        Norm of the matrix or vector(s).\n    Notes\n    -----\n    For values of ``ord <= 0``, the result is, strictly speaking, not a\n    mathematical 'norm', but it may still be useful for various numerical\n    purposes.\n    The following norms can be calculated:\n    =====  ============================  ==========================\n    ord    norm for matrices             norm for vectors\n    =====  ============================  ==========================\n    None   Frobenius norm                2-norm\n    'fro'  Frobenius norm                --\n    'nuc'  --                            --\n    inf    max(sum(abs(x), axis=1))      max(abs(x))\n    -inf   min(sum(abs(x), axis=1))      min(abs(x))\n    0      --                            sum(x != 0)\n    1      max(sum(abs(x), axis=0))      as below\n    -1     min(sum(abs(x), axis=0))      as below\n    2      --                            as below\n    -2     --                            as below\n    other  --                            sum(abs(x)**ord)**(1./ord)\n    =====  ============================  ==========================\n    The Frobenius norm is given by [1]_:\n        :math:`||A||_F = [\\sum_{i,j} abs(a_{i,j})^2]^{1/2}`\n    The nuclear norm is the sum of the singular values.\n    When you want to operate norm for matrices,if you ord is (-1, 1, inf, -inf),\n    you must give you axis, it is not support default axis.\n    References\n    ----------\n    .. [1] G. H. Golub and C. F. Van Loan, *Matrix Computations*,\n           Baltimore, MD, Johns Hopkins University Press, 1985, pg. 15\n    Examples\n    --------\n    >>> from mxnet import np\n    >>> a = np.arange(9) - 4\n    >>> a\n    array([-4., -3., -2., -1.,  0.,  1.,  2.,  3.,  4.])\n    >>> b = a.reshape((3, 3))\n    >>> b\n    array([[-4., -3., -2.],\n           [-1.,  0.,  1.],\n           [ 2.,  3.,  4.]])\n    >>> np.linalg.norm(a)\n    array(7.745967)\n    >>> np.linalg.norm(b)\n    array(7.745967)\n    >>> np.linalg.norm(b, 'fro')\n    array(7.745967)\n    >>> np.linalg.norm(a, 'inf')\n    array(4.)\n    >>> np.linalg.norm(b, 'inf', axis=(0, 1))\n    array(9.)\n    >>> np.linalg.norm(a, '-inf')\n    array(0.)\n    >>> np.linalg.norm(b, '-inf', axis=(0, 1))\n    array(2.)\n    >>> np.linalg.norm(a, 1)\n    array(20.)\n    >>> np.linalg.norm(b, 1, axis=(0, 1))\n    array(7.)\n    >>> np.linalg.norm(a, -1)\n    array(0.)\n    >>> np.linalg.norm(b, -1, axis=(0, 1))\n    array(6.)\n    >>> np.linalg.norm(a, 2)\n    array(7.745967)\n    >>> np.linalg.norm(a, -2)\n    array(0.)\n    >>> np.linalg.norm(a, 3)\n    array(5.8480353)\n    >>> np.linalg.norm(a, -3)\n    array(0.)\n    Using the `axis` argument to compute vector norms:\n    >>> c = np.array([[ 1, 2, 3],\n    ...               [-1, 1, 4]])\n    >>> np.linalg.norm(c, axis=0)\n    array([1.4142135, 2.236068 , 5.       ])\n    >>> np.linalg.norm(c, axis=1)\n    array([3.7416573, 4.2426405])\n    >>> np.linalg.norm(c, ord=1, axis=1)\n    array([6., 6.])\n    Using the `axis` argument to compute matrix norms:\n    >>> m = np.arange(8).reshape(2,2,2)\n    >>> np.linalg.norm(m, axis=(1,2))\n    array([ 3.7416573, 11.224973 ])\n    >>> np.linalg.norm(m[0, :, :]), np.linalg.norm(m[1, :, :])\n    (array(3.7416573), array(11.224973))\n    \"\"\"\n    if axis is None and ord is None:\n        return _api_internal.norm(x, 2, None, keepdims, -2)\n    if axis is None or isinstance(axis, (int, tuple)):  # pylint: disable=too-many-nested-blocks\n        if axis is not None:\n            if isinstance(axis, int):\n                axis = (axis, )\n            if len(axis) == 2:\n                if ord in ['inf', '-inf']:\n                    row_axis, col_axis = axis\n                    if not keepdims:\n                        if row_axis > col_axis:\n                            row_axis -= 1\n                    if ord == 'inf':\n                        return _mx_nd_np.sum(_mx_nd_np.abs(x), axis=col_axis, keepdims=keepdims).max(axis=row_axis, keepdims=keepdims)  # pylint: disable=line-too-long\n                    else:\n                        return _mx_nd_np.sum(_mx_nd_np.abs(x), axis=col_axis, keepdims=keepdims).min(axis=row_axis, keepdims=keepdims)  # pylint: disable=line-too-long\n                if ord in [1, -1]:\n                    row_axis, col_axis = axis\n                    if not keepdims:\n                        if row_axis < col_axis:\n                            col_axis -= 1\n                    if ord == 1:\n                        return _mx_nd_np.sum(_mx_nd_np.abs(x), axis=row_axis, keepdims=keepdims).max(axis=col_axis, keepdims=keepdims)  # pylint: disable=line-too-long\n                    elif ord == -1:\n                        return _mx_nd_np.sum(_mx_nd_np.abs(x), axis=row_axis, keepdims=keepdims).min(axis=col_axis, keepdims=keepdims)  # pylint: disable=line-too-long\n                if ord in [2, -2]:\n                    return _api_internal.norm(x, ord, axis, keepdims, 0)\n                if ord is None:\n                    return _api_internal.norm(x, 2, axis, keepdims, 1)\n        if ord == 'inf':\n            return _mx_nd_np.max(_mx_nd_np.abs(x), axis=axis, keepdims=keepdims)\n        elif ord == '-inf':\n            return _mx_nd_np.min(_mx_nd_np.abs(x), axis=axis, keepdims=keepdims)\n        elif ord is None:\n            return _api_internal.norm(x, 2, axis, keepdims, 1)\n        elif ord == 2:\n            return _api_internal.norm(x, 2, axis, keepdims, -1)\n        elif ord == 'nuc':\n            return _api_internal.norm(x, 2, axis, keepdims, 2)\n        elif ord in ['fro', 'f']:\n            return _api_internal.norm(x, 2, axis, keepdims, 1)\n        else:\n            return _api_internal.norm(x, ord, axis, keepdims, -1)\n    else:\n        raise TypeError(\"'axis' must be None, an integer or a tuple of integers.\")\n# pylint: enable=too-many-return-statements\n\n\ndef svd(a):\n    r\"\"\"\n    Singular Value Decomposition.\n\n    When `a` is a 2D array, it is factorized as ``ut @ np.diag(s) @ v``,\n    where `ut` and `v` are 2D orthonormal arrays and `s` is a 1D\n    array of `a`'s singular values. When `a` is higher-dimensional, SVD is\n    applied in stacked mode as explained below.\n\n    Parameters\n    ----------\n    a : (..., M, N) ndarray\n        A real array with ``a.ndim >= 2`` and ``M <= N``.\n\n    Returns\n    -------\n    ut: (..., M, M) ndarray\n        Orthonormal array(s). The first ``a.ndim - 2`` dimensions have the same\n        size as those of the input `a`.\n    s : (..., M) ndarray\n        Vector(s) with the singular values, within each vector sorted in\n        descending order. The first ``a.ndim - 2`` dimensions have the same\n        size as those of the input `a`.\n    v : (..., M, N) ndarray\n        Orthonormal array(s). The first ``a.ndim - 2`` dimensions have the same\n        size as those of the input `a`.\n\n    Notes\n    -----\n\n    The decomposition is performed using LAPACK routine ``_gesvd``.\n\n    SVD is usually described for the factorization of a 2D matrix :math:`A`.\n    The higher-dimensional case will be discussed below. In the 2D case, SVD is\n    written as :math:`A = U^T S V`, where :math:`A = a`, :math:`U^T = ut`,\n    :math:`S= \\mathtt{np.diag}(s)` and :math:`V = v`. The 1D array `s`\n    contains the singular values of `a` and `ut` and `v` are orthonormal. The rows\n    of `v` are the eigenvectors of :math:`A^T A` and the columns of `ut` are\n    the eigenvectors of :math:`A A^T`. In both cases the corresponding\n    (possibly non-zero) eigenvalues are given by ``s**2``.\n\n    The sign of rows of `u` and `v` are determined as described in\n    `Auto-Differentiating Linear Algebra <https://arxiv.org/pdf/1710.08717.pdf>`_.\n\n    If `a` has more than two dimensions, then broadcasting rules apply.\n    This means that SVD is working in \"stacked\" mode: it iterates over\n    all indices of the first ``a.ndim - 2`` dimensions and for each\n    combination SVD is applied to the last two indices. The matrix `a`\n    can be reconstructed from the decomposition with either\n    ``(ut * s[..., None, :]) @ v`` or\n    ``ut @ (s[..., None] * v)``. (The ``@`` operator denotes batch matrix multiplication)\n\n    This function differs from the original `numpy.linalg.svd\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.svd.html>`_ in\n    the following way(s):\n     - The sign of rows of `u` and `v` may differ.\n     - Does not support complex input.\n\n    Examples\n    --------\n    >>> a = np.arange(54).reshape(6, 9)\n    >>> ut, s, v = np.linalg.svd(a)\n    >>> ut.shape, s.shape, v.shape\n    ((6, 6), (6,), (6, 9))\n    >>> s = s.reshape(6, 1)\n    >>> ret = np.dot(ut, s * v)\n    >>> (ret - a > 1e-3).sum()\n    array(0.)\n    >>> (ret - a < -1e-3).sum()\n    array(0.)\n    \"\"\"\n    return tuple(_api_internal.svd(a))\n\n\ndef cholesky(a, upper=False):\n    r\"\"\"\n    Cholesky decomposition.\n\n    Notes\n    -----\n    `upper` param is requested by API standardization in\n    https://data-apis.org/array-api/latest/extensions/generated/signatures.linalg.cholesky.html\n    instead of parameter in official NumPy operator.\n\n    Return the Cholesky decomposition, `L * L.T`, of the square matrix `a`,\n    where `L` is lower-triangular and .T is the transpose operator. `a` must be\n    symmetric and positive-definite. Only `L` is actually returned. Complex-valued\n    input is currently not supported.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Symmetric, positive-definite input matrix.\n    upper : bool\n        If `True`, the result must be the upper-triangular Cholesky factor.\n        If `False`, the result must be the lower-triangular Cholesky factor.\n        Default: `False`.\n\n    Returns\n    -------\n    L : (..., M, M) ndarray\n        Lower-triangular Cholesky factor of `a`.\n\n    Raises\n    ------\n    MXNetError\n        If the decomposition fails, for example, if `a` is not positive-definite.\n\n    Notes\n    -----\n    Broadcasting rules apply.\n\n    The Cholesky decomposition is often used as a fast way of solving\n\n    .. math:: A \\mathbf{x} = \\mathbf{b}\n\n    (when `A` is both symmetric and positive-definite).\n\n    First, we solve for :math:`\\mathbf{y}` in\n\n    .. math:: L \\mathbf{y} = \\mathbf{b},\n\n    and then for :math:`\\mathbf{x}` in\n\n    .. math:: L.T \\mathbf{x} = \\mathbf{y}.\n\n    Examples\n    --------\n    >>> A = np.array([[16, 4], [4, 10]])\n    >>> A\n    array([[16.,  4.],\n           [ 4., 10.]])\n    >>> L = np.linalg.cholesky(A)\n    >>> L\n    array([[4., 0.],\n           [1., 3.]])\n    >>> np.dot(L, L.T)\n    array([[16.,  4.],\n           [ 4., 10.]])\n    \"\"\"\n    return _api_internal.cholesky(a, not upper)\n\n\ndef qr(a, mode='reduced'):\n    r\"\"\"\n    Compute the qr factorization of a matrix a.\n    Factor the matrix a as qr, where q is orthonormal and r is upper-triangular.\n\n    Parameters\n    ----------\n    a : (..., M, N) ndarray\n        Matrix or stack of matrices to be qr factored.\n    mode: {‘reduced’, ‘complete’, ‘r’, ‘raw’, ‘full’, ‘economic’}, optional\n        Only default mode, 'reduced', is implemented. If K = min(M, N), then\n        * 'reduced’ : returns q, r with dimensions (M, K), (K, N) (default)\n\n    Returns\n    -------\n    q : (..., M, K) ndarray\n        A matrix or stack of matrices with K orthonormal columns, with K = min(M, N).\n    r : (..., K, N) ndarray\n        A matrix or stack of upper triangular matrices.\n\n    Raises\n    ------\n    MXNetError\n        If factoring fails.\n\n    Examples\n    --------\n    >>> from mxnet import np\n    >>> a = np.random.uniform(-10, 10, (2, 2))\n    >>> q, r = np.linalg.qr(a)\n    >>> q\n    array([[-0.22121978, -0.97522414],\n           [-0.97522414,  0.22121954]])\n    >>> r\n    array([[-4.4131265 , -7.1255064 ],\n           [ 0.        , -0.28771925]])\n    >>> a = np.random.uniform(-10, 10, (2, 3))\n    >>> q, r = np.linalg.qr(a)\n    >>> q\n    array([[-0.28376842, -0.9588929 ],\n           [-0.9588929 ,  0.28376836]])\n    >>> r\n    array([[-7.242763  , -0.5673361 , -2.624416  ],\n           [ 0.        , -7.297918  , -0.15949416]])\n    >>> a = np.random.uniform(-10, 10, (3, 2))\n    >>> q, r = np.linalg.qr(a)\n    >>> q\n    array([[-0.34515655,  0.10919492],\n           [ 0.14765628, -0.97452265],\n           [-0.92685735, -0.19591334]])\n    >>> r\n    array([[-8.453794,  8.4175  ],\n           [ 0.      ,  5.430561]])\n    \"\"\"\n    if mode is not None and mode != 'reduced':\n        raise NotImplementedError(\"Only default mode='reduced' is implemented.\")\n    return tuple(_api_internal.qr(a))\n\n\ndef inv(a):\n    r\"\"\"\n    Compute the (multiplicative) inverse of a matrix.\n\n    Given a square matrix `a`, return the matrix `ainv` satisfying\n    ``dot(a, ainv) = dot(ainv, a) = eye(a.shape[0])``.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Matrix to be inverted.\n\n    Returns\n    -------\n    ainv : (..., M, M) ndarray\n        (Multiplicative) inverse of the matrix `a`.\n\n    Raises\n    ------\n    MXNetError\n        If `a` is not square or inversion fails.\n\n    Examples\n    --------\n    >>> from mxnet import np\n    >>> a = np.array([[1., 2.], [3., 4.]])\n    array([[-2. ,  1. ],\n           [ 1.5, -0.5]])\n\n    Inverses of several matrices can be computed at once:\n\n    >>> a = np.array([[[1., 2.], [3., 4.]], [[1, 3], [3, 5]]])\n    >>> np.linalg.inv(a)\n    array([[[-2.        ,  1.        ],\n            [ 1.5       , -0.5       ]],\n\n           [[-1.2500001 ,  0.75000006],\n            [ 0.75000006, -0.25000003]]])\n    \"\"\"\n    return _api_internal.inv(a)\n\n\ndef det(a):\n    r\"\"\"\n    Compute the determinant of an array.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Input array to compute determinants for.\n\n    Returns\n    -------\n    det : (...) ndarray\n        Determinant of `a`.\n\n    See Also\n    --------\n    slogdet : Another way to represent the determinant, more suitable\n    for large matrices where underflow/overflow may occur.\n\n    Notes\n    -----\n    Broadcasting rules apply, see the `numpy.linalg` documentation for\n    details.\n    The determinant is computed via LU factorization using the LAPACK\n    routine z/dgetrf.\n\n    Examples\n    --------\n    The determinant of a 2-D array [[a, b], [c, d]] is ad - bc:\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> np.linalg.det(a)\n    -2.0\n\n    Computing determinants for a stack of matrices:\n    >>> a = np.array([ [[1, 2], [3, 4]], [[1, 2], [2, 1]], [[1, 3], [3, 1]] ])\n    >>> a.shape\n    (3, 2, 2)\n\n    >>> np.linalg.det(a)\n    array([-2., -3., -8.])\n    \"\"\"\n    return _api_internal.det(a)\n\n\ndef slogdet(a):\n    r\"\"\"\n    Compute the sign and (natural) logarithm of the determinant of an array.\n    If an array has a very small or very large determinant, then a call to\n    `det` may overflow or underflow. This routine is more robust against such\n    issues, because it computes the logarithm of the determinant rather than\n    the determinant itself.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Input array, has to be a square 2-D array.\n\n    Returns\n    -------\n    sign : (...) ndarray\n        A number representing the sign of the determinant. For a real matrix,\n        this is 1, 0, or -1.\n    logdet : (...) array_like\n        The natural log of the absolute value of the determinant.\n    If the determinant is zero, then `sign` will be 0 and `logdet` will be\n    -Inf. In all cases, the determinant is equal to ``sign * np.exp(logdet)``.\n\n    See Also\n    --------\n    det\n\n    Notes\n    -----\n    Broadcasting rules apply, see the `numpy.linalg` documentation for\n    details.\n    The determinant is computed via LU factorization using the LAPACK\n    routine z/dgetrf.\n\n    Examples\n    --------\n    The determinant of a 2-D array ``[[a, b], [c, d]]`` is ``ad - bc``:\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> (sign, logdet) = np.linalg.slogdet(a)\n    >>> (sign, logdet)\n    (-1., 0.69314718055994529)\n\n    >>> sign * np.exp(logdet)\n    -2.0\n\n    Computing log-determinants for a stack of matrices:\n    >>> a = np.array([ [[1, 2], [3, 4]], [[1, 2], [2, 1]], [[1, 3], [3, 1]] ])\n    >>> a.shape\n    (3, 2, 2)\n\n    >>> sign, logdet = np.linalg.slogdet(a)\n    >>> (sign, logdet)\n    (array([-1., -1., -1.]), array([ 0.69314718,  1.09861229,  2.07944154]))\n\n    >>> sign * np.exp(logdet)\n    array([-2., -3., -8.])\n\n    This routine succeeds where ordinary `det` does not:\n    >>> np.linalg.det(np.eye(500) * 0.1)\n    0.0\n    >>> np.linalg.slogdet(np.eye(500) * 0.1)\n    (1., -1151.2925464970228)\n    \"\"\"\n    return tuple(_api_internal.slogdet(a))\n\n\ndef solve(a, b):\n    r\"\"\"\n    Solve a linear matrix equation, or system of linear scalar equations.\n\n    Computes the \"exact\" solution, `x`, of the well-determined, i.e., full\n    rank, linear matrix equation `ax = b`.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Coefficient matrix.\n    b : {(..., M,), (..., M, K)}, ndarray\n        Ordinate or \"dependent variable\" values.\n\n    Returns\n    -------\n    x : {(..., M,), (..., M, K)} ndarray\n        Solution to the system a x = b.  Returned shape is identical to `b`.\n\n    Raises\n    ------\n    MXNetError\n        If `a` is singular or not square.\n\n    Notes\n    -----\n    Broadcasting rules apply, see the `numpy.linalg` documentation for\n    details.\n\n    The solutions are computed using LAPACK routine ``_gesv``.\n\n    `a` must be square and of full-rank, i.e., all rows (or, equivalently,\n    columns) must be linearly independent; if either is not true, use\n    `lstsq` for the least-squares best \"solution\" of the\n    system/equation.\n\n    Examples\n    --------\n    Solve the system of equations ``3 * x0 + x1 = 9`` and ``x0 + 2 * x1 = 8``:\n\n    >>> a = np.array([[3,1], [1,2]])\n    >>> b = np.array([9,8])\n    >>> x = np.linalg.solve(a, b)\n    >>> x\n    array([2.,  3.])\n\n    Check that the solution is correct:\n\n    >>> np.allclose(np.dot(a, x), b)\n    True\n    \"\"\"\n    return _api_internal.solve(a, b)\n\n\ndef tensorinv(a, ind=2):\n    r\"\"\"\n    Compute the 'inverse' of an N-dimensional array.\n\n    The result is an inverse for `a` relative to the tensordot operation\n    ``tensordot(a, b, ind)``, i. e., up to floating-point accuracy,\n    ``tensordot(tensorinv(a), a, ind)`` is the \"identity\" tensor for the\n    tensordot operation.\n\n    Parameters\n    ----------\n    a : array_like\n        Tensor to 'invert'. Its shape must be 'square', i. e.,\n        ``prod(a.shape[:ind]) == prod(a.shape[ind:])``.\n    ind : int, optional\n        Number of first indices that are involved in the inverse sum.\n        Must be a positive integer, default is 2.\n\n    Returns\n    -------\n    b : ndarray\n        `a`'s tensordot inverse, shape ``a.shape[ind:] + a.shape[:ind]``.\n\n    Raises\n    ------\n    MXNetError\n        If `a` is singular or not 'square' (in the above sense).\n\n    See Also\n    --------\n    tensordot, tensorsolve\n\n    Examples\n    --------\n    >>> a = np.eye(4*6)\n    >>> a.shape = (4, 6, 8, 3)\n    >>> ainv = np.linalg.tensorinv(a, ind=2)\n    >>> ainv.shape\n    (8, 3, 4, 6)\n    >>> b = np.random.randn(4, 6)\n    >>> np.allclose(np.tensordot(ainv, b), np.linalg.tensorsolve(a, b))\n    True\n\n    >>> a = np.eye(4*6)\n    >>> a.shape = (24, 8, 3)\n    >>> ainv = np.linalg.tensorinv(a, ind=1)\n    >>> ainv.shape\n    (8, 3, 24)\n    >>> b = np.random.randn(24)\n    >>> np.allclose(np.tensordot(ainv, b, 1), np.linalg.tensorsolve(a, b))\n    True\n    \"\"\"\n    return _api_internal.tensorinv(a, ind)\n\n\ndef tensorsolve(a, b, axes=None):\n    r\"\"\"\n    Solve the tensor equation ``a x = b`` for x.\n    It is assumed that all indices of `x` are summed over in the product,\n    together with the rightmost indices of `a`, as is done in, for example,\n    ``tensordot(a, x, axes=b.ndim)``.\n\n    Parameters\n    ----------\n    a : ndarray\n        Coefficient tensor, of shape ``b.shape + Q``. `Q`, a tuple, equals\n        the shape of that sub-tensor of `a` consisting of the appropriate\n        number of its rightmost indices, and must be such that\n        ``prod(Q) == prod(b.shape)`` (in which sense `a` is said to be\n        'square').\n    b : ndarray\n        Right-hand tensor, which can be of any shape.\n    axes : tuple of ints, optional\n        Axes in `a` to reorder to the right, before inversion.\n        If None (default), no reordering is done.\n\n    Returns\n    -------\n    x : ndarray, shape Q\n\n    Raises\n    ------\n    MXNetError\n        If `a` is singular or not 'square' (in the above sense).\n\n    See Also\n    --------\n    numpy.tensordot, tensorinv, numpy.einsum\n\n    Examples\n    --------\n    >>> a = np.eye(2*3*4)\n    >>> a.shape = (2*3, 4, 2, 3, 4)\n    >>> b = np.random.randn(2*3, 4)\n    >>> x = np.linalg.tensorsolve(a, b)\n    >>> x.shape\n    (2, 3, 4)\n    >>> np.allclose(np.tensordot(a, x, axes=3), b)\n    True\n    \"\"\"\n    return _api_internal.tensorsolve(a, b, axes)\n\n\ndef eigvals(a):\n    r\"\"\"\n    Compute the eigenvalues of a general matrix.\n\n    Main difference between `eigvals` and `eig`: the eigenvectors aren't\n    returned.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        A real-valued matrix whose eigenvalues will be computed.\n\n    Returns\n    -------\n    w : (..., M,) ndarray\n        The eigenvalues, each repeated according to its multiplicity.\n        They are not necessarily ordered.\n\n    Raises\n    ------\n    MXNetError\n        If the eigenvalue computation does not converge.\n\n    See Also\n    --------\n    eig : eigenvalues and right eigenvectors of general arrays\n    eigh : eigenvalues and eigenvectors of a real symmetric array.\n    eigvalsh : eigenvalues of a real symmetric.\n\n    Notes\n    -----\n    Broadcasting rules apply, see the `numpy.linalg` documentation for\n    details.\n\n    This is implemented using the ``_geev`` LAPACK routines which compute\n    the eigenvalues and eigenvectors of general square arrays.\n\n    This function differs from the original `numpy.linalg.eigvals\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.eigvals.html>`_ in\n    the following way(s):\n     - Does not support complex input and output.\n\n    Examples\n    --------\n    Illustration, using the fact that the eigenvalues of a diagonal matrix\n    are its diagonal elements, that multiplying a matrix on the left\n    by an orthogonal matrix, `Q`, and on the right by `Q.T` (the transpose\n    of `Q`), preserves the eigenvalues of the \"middle\" matrix.  In other words,\n    if `Q` is orthogonal, then ``Q * A * Q.T`` has the same eigenvalues as\n    ``A``:\n    >>> from numpy import linalg as LA\n    >>> x = np.random.random()\n    >>> Q = np.array([[np.cos(x), -np.sin(x)], [np.sin(x), np.cos(x)]])\n    >>> LA.norm(Q[0, :]), LA.norm(Q[1, :]), np.dot(Q[0, :],Q[1, :])\n    (1.0, 1.0, 0.0)\n\n    Now multiply a diagonal matrix by ``Q`` on one side and by ``Q.T`` on the other:\n    >>> D = np.diag((-1,1))\n    >>> LA.eigvals(D)\n    array([-1.,  1.])\n    >>> A = np.dot(Q, D)\n    >>> A = np.dot(A, Q.T)\n    >>> LA.eigvals(A)\n    array([ 1., -1.]) # random\n    \"\"\"\n    return _api_internal.eigvals(a)\n\n\ndef eigvalsh(a, UPLO='L'):\n    r\"\"\"\n    Compute the eigenvalues real symmetric matrix.\n\n    Main difference from eigh: the eigenvectors are not computed.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        A real-valued matrix whose eigenvalues are to be computed.\n    UPLO : {'L', 'U'}, optional\n        Specifies whether the calculation is done with the lower triangular\n        part of `a` ('L', default) or the upper triangular part ('U').\n        Irrespective of this value only the real parts of the diagonal will\n        be considered in the computation to preserve the notion of a Hermitian\n        matrix. It therefore follows that the imaginary part of the diagonal\n        will always be treated as zero.\n\n    Returns\n    -------\n    w : (..., M,) ndarray\n        The eigenvalues in ascending order, each repeated according to\n        its multiplicity.\n\n    Raises\n    ------\n    MXNetError\n        If the eigenvalue computation does not converge.\n\n    See Also\n    --------\n    eig : eigenvalues and right eigenvectors of general arrays\n    eigvals : eigenvalues of a non-symmetric array.\n    eigh : eigenvalues and eigenvectors of a real symmetric array.\n\n    Notes\n    -----\n    Broadcasting rules apply, see the `numpy.linalg` documentation for\n    details.\n\n    The eigenvalues are computed using LAPACK routines ``_syevd``.\n\n    This function differs from the original `numpy.linalg.eigvalsh\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.eigvalsh.html>`_ in\n    the following way(s):\n     - Does not support complex input and output.\n\n    Examples\n    --------\n    >>> from numpy import linalg as LA\n    >>> a = np.array([[ 5.4119368 ,  8.996273  , -5.086096  ],\n                      [ 0.8866155 ,  1.7490431 , -4.6107802 ],\n                      [-0.08034172,  4.4172044 ,  1.4528792 ]])\n    >>> LA.eigvalsh(a, UPLO='L')\n    array([-2.87381886,  5.10144682,  6.38623114]) # in ascending order\n    \"\"\"\n    return _api_internal.eigvalsh(a, UPLO)\n\n\ndef eig(a):\n    r\"\"\"\n    Compute the eigenvalues and right eigenvectors of a square array.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Matrices for which the eigenvalues and right eigenvectors will\n        be computed\n\n    Returns\n    -------\n    w : (..., M) ndarray\n        The eigenvalues, each repeated according to its multiplicity.\n        The eigenvalues are not necessarily ordered.\n    v : (..., M, M) ndarray\n        The normalized (unit \"length\") eigenvectors, such that the\n        column ``v[:,i]`` is the eigenvector corresponding to the\n        eigenvalue ``w[i]``.\n\n    Raises\n    ------\n    MXNetError\n        If the eigenvalue computation does not converge.\n\n    See Also\n    --------\n    eigvals : eigenvalues of a non-symmetric array.\n    eigh : eigenvalues and eigenvectors of a real symmetric array.\n    eigvalsh : eigenvalues of a real symmetric.\n\n    Notes\n    -----\n    This is implemented using the ``_geev`` LAPACK routines which compute\n    the eigenvalues and eigenvectors of general square arrays.\n\n    The number `w` is an eigenvalue of `a` if there exists a vector\n    `v` such that ``dot(a,v) = w * v``. Thus, the arrays `a`, `w`, and\n    `v` satisfy the equations ``dot(a[:,:], v[:,i]) = w[i] * v[:,i]``\n    for :math:`i \\\\in \\\\{0,...,M-1\\\\}`.\n\n    The array `v` of eigenvectors may not be of maximum rank, that is, some\n    of the columns may be linearly dependent, although round-off error may\n    obscure that fact. If the eigenvalues are all different, then theoretically\n    the eigenvectors are linearly independent.\n\n    This function differs from the original `numpy.linalg.eig\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.eig.html>`_ in\n    the following way(s):\n     - Does not support complex input and output.\n\n    Examples\n    --------\n    >>> from numpy import linalg as LA\n    >>> a = np.array([[-1.9147992 ,  6.054115  , 18.046988  ],\n                      [ 0.77563655, -4.860152  ,  2.1012988 ],\n                      [ 2.6083658 ,  2.3705218 ,  0.3192524 ]])\n    >>> w, v = LA.eig(a)\n    >>> w\n    array([ 6.9683027, -7.768063 , -5.655937 ])\n    >>> v\n    array([[ 0.90617794,  0.9543622 ,  0.2492316 ],\n           [ 0.13086087, -0.04077047, -0.9325615 ],\n           [ 0.4021404 , -0.29585576,  0.26117516]])\n    \"\"\"\n    w, v = _api_internal.eig(a)\n    return (w, v)\n\n\ndef eigh(a, UPLO='L'):\n    r\"\"\"\n    Return the eigenvalues and eigenvectors real symmetric matrix.\n\n    Returns two objects, a 1-D array containing the eigenvalues of `a`, and\n    a 2-D square array or matrix (depending on the input type) of the\n    corresponding eigenvectors (in columns).\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        real symmetric matrices whose eigenvalues and eigenvectors are to be computed.\n    UPLO : {'L', 'U'}, optional\n        Specifies whether the calculation is done with the lower triangular\n        part of `a` ('L', default) or the upper triangular part ('U').\n        Irrespective of this value only the real parts of the diagonal will\n        be considered in the computation to preserve the notion of a Hermitian\n        matrix. It therefore follows that the imaginary part of the diagonal\n        will always be treated as zero.\n\n    Returns\n    -------\n    w : (..., M) ndarray\n        The eigenvalues in ascending order, each repeated according to\n        its multiplicity.\n    v : {(..., M, M) ndarray, (..., M, M) matrix}\n        The column ``v[:, i]`` is the normalized eigenvector corresponding\n        to the eigenvalue ``w[i]``.  Will return a matrix object if `a` is\n        a matrix object.\n\n    Raises\n    ------\n    MXNetError\n        If the eigenvalue computation does not converge.\n\n    See Also\n    --------\n    eig : eigenvalues and right eigenvectors of general arrays\n    eigvals : eigenvalues of a non-symmetric array.\n    eigvalsh : eigenvalues of a real symmetric.\n\n    Notes\n    -----\n    The eigenvalues/eigenvectors are computed using LAPACK routines ``_syevd``.\n\n    This function differs from the original `numpy.linalg.eigh\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.eigh.html>`_ in\n    the following way(s):\n     - Does not support complex input and output.\n\n    Examples\n    --------\n    >>> from numpy import linalg as LA\n    >>> a = np.array([[ 6.8189726 , -3.926585  ,  4.3990498 ],\n                      [-0.59656644, -1.9166266 ,  9.54532   ],\n                      [ 2.1093285 ,  0.19688708, -1.1634291 ]])\n    >>> w, v = LA.eigh(a, UPLO='L')\n    >>> w\n    array([-2.175445 , -1.4581827,  7.3725457])\n    >>> v\n    array([[ 0.1805163 , -0.16569263,  0.9695154 ],\n           [ 0.8242942 ,  0.56326365, -0.05721384],\n           [-0.53661287,  0.80949366,  0.23825769]])\n    \"\"\"\n    w, v = _api_internal.eigh(a, UPLO)\n    return (w, v)\n"
  },
  {
    "path": "python/mxnet/ndarray/numpy/random.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for operators used in Gluon dispatched by F=ndarray.\"\"\"\nimport numpy as np\nfrom ...util import is_np_default_dtype\nfrom ...device import current_device\nfrom . import _internal as _npi\nfrom . import _api_internal\nfrom ...util import wrap_ctx_to_device_func\nfrom ..ndarray import get_dtype_name\n\n\n__all__ = ['randint', 'uniform', 'normal', \"choice\", \"rand\", \"multinomial\", \"multivariate_normal\",\n           'logistic', 'gumbel', \"rayleigh\", 'f',\n           'laplace',\n           \"shuffle\", 'gamma', 'beta', 'chisquare', 'exponential', 'lognormal', 'weibull', 'pareto', 'power']\n\n\n@wrap_ctx_to_device_func\ndef randint(low, high=None, size=None, dtype=None, device=None, out=None):\n    r\"\"\"Return random integers from `low` (inclusive) to `high` (exclusive).\n\n    Return random integers from the \"discrete uniform\" distribution of\n    the specified dtype in the \"half-open\" interval [`low`, `high`). If\n    `high` is None (the default), then results are from [0, `low`).\n\n    Parameters\n    ----------\n    low : int\n        Lowest (signed) integer to be drawn from the distribution (unless\n        ``high=None``, in which case this parameter is one above the\n        *highest* such integer).\n    high : int, optional\n        If provided, one above the largest (signed) integer to be drawn\n        from the distribution (see above for behavior if ``high=None``).\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  Default is None, in which case a\n        single value is returned.\n    dtype : dtype, optional\n        Desired dtype of the result. All dtypes are determined by their\n        name, i.e., 'int64', 'int', etc, so byteorder is not available\n        and a specific precision may have different C types depending\n        on the platform. The default value is 'np.int'.\n    device : Device, optional\n        Device context of output. Default is current device.\n    out : ndarray, optional\n        The output ndarray (default is `None`).\n\n    Returns\n    -------\n    out : ndarray of ints\n        `size`-shaped array of random integers from the appropriate\n        distribution, or a single such random int if `size` not provided.\n\n    Examples\n    --------\n    >>> np.random.randint(2, size=10)\n    array([1, 0, 0, 0, 1, 1, 0, 0, 1, 0])\n    >>> np.random.randint(1, size=10)\n    array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n\n    Generate a 2 x 4 array of ints between 0 and 4, inclusive:\n\n    >>> np.random.randint(5, size=(2, 4))\n    array([[4, 0, 2, 1],\n           [3, 2, 2, 0]])\n    \"\"\"\n    if dtype is None:\n        dtype = 'int64'\n    elif not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if size is None:\n        size = ()\n    if high is None:\n        high = low\n        low = 0\n    return _api_internal.randint(low, high, size, dtype, device, out)\n\n\n@wrap_ctx_to_device_func\ndef uniform(low=0.0, high=1.0, size=None, dtype=None, device=None, out=None):\n    r\"\"\"Draw samples from a uniform distribution.\n\n    Samples are uniformly distributed over the half-open interval\n    ``[low, high)`` (includes low, but excludes high).  In other words,\n    any value within the given interval is equally likely to be drawn\n    by `uniform`.\n\n    Parameters\n    ----------\n    low : float, ndarray, optional\n        Lower boundary of the output interval.  All values generated will be\n        greater than or equal to low.  The default value is 0.\n    high : float, ndarray, optional\n        Upper boundary of the output interval.  All values generated will be\n        less than high.  The default value is 1.0.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a scalar tensor containing a single value is returned if\n        ``low`` and ``high`` are both scalars.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n    device : Device, optional\n        Device context of output. Default is current device.\n    out : ``ndarray``, optional\n        Store output to an existing ``ndarray``.\n\n    Returns\n    -------\n    out : ndarray\n        Drawn samples from the parameterized uniform distribution.\n    \"\"\"\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    if size == ():\n        size = None\n    return _api_internal.uniform(low, high, size, device, dtype, out)\n\n\n@wrap_ctx_to_device_func\ndef normal(loc=0.0, scale=1.0, size=None, dtype=None, device=None, out=None):\n    r\"\"\"Draw random samples from a normal (Gaussian) distribution.\n\n    Samples are distributed according to a normal distribution parametrized\n    by *loc* (mean) and *scale* (standard deviation).\n\n\n    Parameters\n    ----------\n    loc : float, optional\n        Mean (centre) of the distribution.\n    scale : float, optional\n        Standard deviation (spread or \"width\") of the distribution.\n    size : int or tuple of ints, optional\n        Output shape. If the given shape is, e.g., `(m, n, k)`, then `m * n * k`\n        samples are drawn. If size is `None` (default), a scalar tensor containing\n        a single value is returned if loc and scale are both scalars.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n    device : Device, optional\n        Device context of output. Default is current device.\n    out : ``ndarray``, optional\n        Store output to an existing ``ndarray``.\n\n    Returns\n    -------\n    out : ndarray\n        Drawn samples from the parameterized normal distribution.\n    \"\"\"\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    if size == ():\n        size = None\n    return _api_internal.normal(loc, scale, size, device, dtype, out)\n\n\n@wrap_ctx_to_device_func\ndef lognormal(mean=0.0, sigma=1.0, size=None, dtype=None, device=None, out=None):\n    r\"\"\"Draw samples from a log-normal distribution.\n\n    Draw samples from a log-normal distribution with specified mean,\n    standard deviation, and array shape.  Note that the mean and standard\n    deviation are not the values for the distribution itself, but of the\n    underlying normal distribution it is derived from.\n\n    Parameters\n    ----------\n    mean : float or array_like of floats, optional\n        Mean value of the underlying normal distribution. Default is 0.\n    sigma : float or array_like of floats, optional\n        Standard deviation of the underlying normal distribution. Must be\n        non-negative. Default is 1.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``mean`` and ``sigma`` are both scalars.\n        Otherwise, ``np.broadcast(mean, sigma).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    device : Device, optional\n        Device context of output. Default is current device.\n    out : ``ndarray``, optional\n        Store output to an existing ``ndarray``.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized log-normal distribution.\n    \"\"\"\n    from . import _op as _mx_np_op\n    return _mx_np_op.exp(normal(loc=mean, scale=sigma, size=size, dtype=dtype, device=device, out=out))\n\n\n@wrap_ctx_to_device_func\ndef logistic(loc=0.0, scale=1.0, size=None, device=None, out=None):\n    r\"\"\"Draw samples from a logistic distribution.\n\n    Samples are drawn from a logistic distribution with specified\n    parameters, loc (location or mean, also median), and scale (>0).\n\n    Parameters\n    ----------\n    loc : float or array_like of floats, optional\n        Parameter of the distribution. Default is 0.\n    scale : float or array_like of floats, optional\n        Parameter of the distribution. Must be non-negative.\n        Default is 1.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``loc`` and ``scale`` are both scalars.\n        Otherwise, ``np.broadcast(loc, scale).size`` samples are drawn.\n    device : Device, optional\n        Device context of output. Default is current device.\n    out : ``ndarray``, optional\n        Store output to an existing ``ndarray``.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized logistic distribution.\n    \"\"\"\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if size == ():\n        size = None\n    return _api_internal.logistic(loc, scale, size, device, out)\n\n\n@wrap_ctx_to_device_func\ndef gumbel(loc=0.0, scale=1.0, size=None, device=None, out=None):\n    r\"\"\"Draw samples from a Gumbel distribution.\n\n    Draw samples from a Gumbel distribution with specified location and\n    scale.\n\n    Parameters\n    ----------\n    loc : float or array_like of floats, optional\n        The location of the mode of the distribution. Default is 0.\n    scale : float or array_like of floats, optional\n        The scale parameter of the distribution. Default is 1. Must be non-\n        negative.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``loc`` and ``scale`` are both scalars.\n        Otherwise, ``np.broadcast(loc, scale).size`` samples are drawn.\n    device : Device, optional\n        Device context of output. Default is current device.\n    out : ``ndarray``, optional\n        Store output to an existing ``ndarray``.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized Gumbel distribution.\n    \"\"\"\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if size == ():\n        size = None\n    return _api_internal.gumbel(loc, scale, size, device, out)\n\n\ndef multinomial(n, pvals, size=None):\n    r\"\"\"multinomial(n, pvals, size=None)\n\n    Draw samples from a multinomial distribution.\n\n    The multinomial distribution is a multivariate generalisation of the binomial distribution.\n    Take an experiment with one of ``p`` possible outcomes. An example of such an experiment is throwing a dice,\n    where the outcome can be 1 through 6. Each sample drawn from the distribution represents n such experiments.\n    Its values, ``X_i = [X_0, X_1, ..., X_p]``, represent the number of times the outcome was ``i``.\n\n    Parameters\n    ----------\n    n : int\n        Number of experiments.\n    pvals : sequence of floats, length p\n        Probabilities of each of the p different outcomes. These should sum to 1.\n    size : int or tuple of ints, optional\n        Output shape. If the given shape is, e.g., ``(m, n, k)``, then ``m * n * k`` samples\n        are drawn. Default is None, in which case a single value is returned.\n\n    Returns\n    -------\n    out : ndarray\n        The drawn samples, of shape size, if that was provided. If not, the shape is ``(N,)``.\n        In other words, each entry ``out[i,j,...,:]`` is an N-dimensional value drawn from the distribution.\n\n    Examples\n    --------\n    Throw a dice 1000 times, and 1000 times again:\n\n    >>> np.random.multinomial(1000, [1/6.]*6, size=2)\n    array([[164, 161, 179, 158, 150, 188],\n           [178, 162, 177, 143, 163, 177]])\n\n    A loaded die is more likely to land on number 6:\n\n    >>> np.random.multinomial(100, [1/7.]*5 + [2/7.])\n    array([19, 14, 12, 11, 21, 23])\n\n    >>> np.random.multinomial(100, [1.0 / 3, 2.0 / 3])\n    array([32, 68])\n    \"\"\"\n    if isinstance(pvals, np.ndarray):\n        raise ValueError('numpy ndarray is not supported!')\n    if any(isinstance(i, list) for i in pvals):\n        raise ValueError('object too deep for desired array')\n    return _api_internal.multinomial(n, pvals, size)\n\n\n@wrap_ctx_to_device_func\ndef rayleigh(scale=1.0, size=None, device=None, out=None):\n    r\"\"\"Draw samples from a Rayleigh distribution.\n\n    The :math:`\\chi` and Weibull distributions are generalizations of the\n    Rayleigh.\n\n    Parameters\n    ----------\n    scale : float, optional\n        Scale, also equals the mode. Must be non-negative. Default is 1.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``scale`` is a scalar.  Otherwise,\n        ``np.array(scale).size`` samples are drawn.\n    device : Device, optional\n        Device context of output. Default is current device.\n    out : ``ndarray``, optional\n        Store output to an existing ``ndarray``.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized Rayleigh distribution.\n    \"\"\"\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if size == ():\n        size = None\n    return _api_internal.rayleigh(scale, size, device, out)\n\n\ndef multivariate_normal(mean, cov, size=None, check_valid=None, tol=None):\n    \"\"\"\n    multivariate_normal(mean, cov, size=None, check_valid=None, tol=None)\n\n    Draw random samples from a multivariate normal distribution.\n\n    The multivariate normal, multinormal or Gaussian distribution is a\n    generalization of the one-dimensional normal distribution to higher\n    dimensions.  Such a distribution is specified by its mean and\n    covariance matrix.  These parameters are analogous to the mean\n    (average or \"center\") and variance (standard deviation, or \"width,\"\n    squared) of the one-dimensional normal distribution.\n\n    This operator is a little different from the one in official NumPy.\n    The official NumPy operator only accepts 1-D ndarray as mean and 2-D ndarray as cov,\n    whereas the operator in MXNet np supports batch operation and auto-broadcasting.\n\n    Both `mean` and `cov` may have any number of leading dimensions, which correspond\n    to a batch shape. They are not necessarily assumed to have the same batch shape,\n    just ones which can be broadcasted.\n\n    Parameters\n    ----------\n    mean : K-D ndarray, of shape (..., N)\n        Mean of the N-dimensional distribution.\n    cov : (K+1)-D ndarray, of shape (..., N, N)\n        Covariance matrix of the distribution. The last two dimensions must be symmetric and\n        positive-semidefinite for proper sampling.\n    size : int or tuple of ints, optional\n        Given a shape of, for example, ``(m,n,k)``,\n        ``m*n*k`` identically distributed batchs of samples are\n        generated, and packed in an `m`-by-`n`-by-`k` arrangement.\n        If no shape is specified, a batch of (`N`-D) sample is returned.\n    check_valid : { 'warn', 'raise', 'ignore' }, optional\n        Behavior when the covariance matrix is not positive semidefinite.\n        (Not supported)\n    tol : float, optional\n        Tolerance when checking the singular values in covariance matrix.\n        cov is cast to double before the check.\n        (Not supported)\n\n    Returns\n    -------\n    out : ndarray\n        The input shape of `mean` and `cov` should satisfy the requirements of broadcasting.\n        If the parameter `size` is not provided,\n        the output shape is ``np.broadcast(mean.shape, cov.shape[:-1])``.\n        Otherwise, the output shape is ``size + np.broadcast(mean.shape, cov.shape[:-1])``\n\n    Examples\n    --------\n    >>> mean = np.array([1, 2])\n    >>> cov = np.array([[1, 0], [0, 1]])\n    >>> x = np.random.multivariate_normal(mean, cov, (3, 3))\n    >>> x.shape\n    (3, 3, 2)\n\n    The following is probably true, given that 0.6 is roughly twice the\n    standard deviation:\n\n    >>> list((x[0,0,:] - mean) < 0.6)\n    [True, True] # random\n\n    # Performs autobroadcasting when the batch shape of\n    # `mean` and `cov` is different but compatible.\n\n    >>> mean = np.zeros((3,2)) # shape (3, 2)\n    >>> cov = np.array([[1, 0], [0, 100]]) # shape (2, 2)\n    >>> x = np.random.multivariate_normal(mean, cov)\n    >>> x\n    array([[-1.6115597 , -8.726251  ],\n           [ 2.2425299 ,  2.8104177 ],\n           [ 0.36229908, -8.386591  ]])\n    \"\"\"\n    if check_valid is not None:\n        raise NotImplementedError('Parameter `check_valid` is not supported')\n    if tol is not None:\n        raise NotImplementedError('Parameter `tol` is not supported')\n    return _npi.mvn_fallback(mean, cov, size=size)\n\n\n@wrap_ctx_to_device_func\ndef choice(a, size=None, replace=True, p=None, device=None, out=None):\n    r\"\"\"Generates a random sample from a given 1-D array\n\n    Parameters\n    -----------\n    a : 1-D array-like or int\n        If an ndarray, a random sample is generated from its elements.\n        If an int, the random sample is generated as if a were np.arange(a)\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  Default is None, in which case a\n        single value is returned.\n    replace : boolean, optional\n        Whether the sample is with or without replacement\n    p : 1-D array-like, optional\n        The probabilities associated with each entry in a.\n        If not given the sample assumes a uniform distribution over all\n        entries in a.\n    device : Device, optional\n        Device context of output. Default is current device.\n\n    Returns\n    --------\n    samples : ndarray\n        The generated random samples\n\n    Examples\n    ---------\n    Generate a uniform random sample from np.arange(5) of size 3:\n\n    >>> np.random.choice(5, 3)\n    array([0, 3, 4])\n    >>> #This is equivalent to np.random.randint(0,5,3)\n\n    Generate a non-uniform random sample from np.arange(5) of size 3:\n\n    >>> np.random.choice(5, 3, p=[0.1, 0, 0.3, 0.6, 0])\n    array([3, 3, 0])\n\n    Generate a uniform random sample from np.arange(5) of size 3 without\n    replacement:\n\n    >>> np.random.choice(5, 3, replace=False)\n    array([3,1,0])\n    >>> #This is equivalent to np.random.permutation(np.arange(5))[:3]\n\n    Generate a non-uniform random sample from np.arange(5) of size\n    3 without replacement:\n\n    >>> np.random.choice(5, 3, replace=False, p=[0.1, 0, 0.3, 0.6, 0])\n    array([2, 3, 0])\n    \"\"\"\n    from ...numpy import ndarray as np_ndarray\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if size == ():\n        size = None\n    if isinstance(a, np_ndarray):\n        indices = _api_internal.choice(a, size, replace, p, device, out)\n        return _api_internal.take(a, indices, 0, 'raise', out)\n    else:\n        return _api_internal.choice(a, size, replace, p, device, out)\n\n\n@wrap_ctx_to_device_func\ndef exponential(scale=1.0, size=None, device=None, out=None):\n    r\"\"\"Draw samples from an exponential distribution.\n\n    Parameters\n    ----------\n    scale : float or array_like of floats\n        The scale parameter, :math:`\\beta = 1/\\lambda`. Must be\n        non-negative.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``scale`` is a scalar.  Otherwise,\n        ``np.array(scale).size`` samples are drawn.\n    device : Device, optional\n        Device context of output. Default is current device.\n    out : ``ndarray``, optional\n        Store output to an existing ``ndarray``.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized exponential distribution.\n    \"\"\"\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if size == ():\n        size = None\n    return _api_internal.exponential(scale, size, device, out)\n\n\n@wrap_ctx_to_device_func\ndef weibull(a, size=None, device=None, out=None):\n    r\"\"\"Draw samples from a 1-parameter Weibull distribution with given\n    parameter a, via inversion.\n\n    Parameters\n    ----------\n    a : float or array_like of floats\n        Shape of the distribution. Must be non-negative.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``a`` is a scalar. Otherwise,\n        ``np.array(a).size`` samples are drawn.\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the 1-parameter Weibull distribution.\n    Examples\n    --------\n    >>> np.random.weibull(a=5)\n    array(0.9553641)\n\n    >>> np.random.weibull(a=5, size=[2,3])\n    array([[1.0466299 , 1.1320982 , 0.98415005],\n          [1.1430776 , 0.9532727 , 1.1344457 ]])\n\n    >>> np.random.weibull(a=np.array([2,3])\n    array([0.98843634, 1.0125613 ])\n\n    The Weibull distribution is one of a class of Generalized Extreme\n    Value (GEV) distributions. This class includes the Gumbel and Frechet\n    distributions.\n\n    The probability density for the Weibull distribution is\n    f(x) = \\frac{a}{\\lambda}(\\frac{x}{\\lambda})^{a-1}e^{-(x/\\lambda)^a},\n    where a is the shape and \\lambda the scale. The generated 1-parameter\n    Weibull sample has the scale parameter \\lambda = 1.\n\n    The Weibull distribution is commonly used in reliability engineering to\n    model time to failure, in modeling particle sizes, in information retrieval\n    to model dwell time on pages, in quantitative finance to model risk etc.\n    \"\"\"\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if size == ():\n        size = None\n    return _api_internal.weibull(a, size, device, out)\n\n\n@wrap_ctx_to_device_func\ndef pareto(a, size=None, device=None, out=None):\n    r\"\"\"Draw samples from a Pareto II or Lomax distribution with specified shape a.\n\n    Parameters\n    ----------\n    a : float or array_like of floats\n            Shape of the distribution. Must be > 0.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``a`` is a scalar. Otherwise,\n        ``np.array(a).size`` samples are drawn.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the Pareto distribution.\n\n    Examples\n    --------\n    >>> np.random.pareto(a=5)\n    array(0.12749612)\n    >>> mx.numpy.random.pareto(a=5, size=[2,3])\n    array([[0.06933999, 0.0344373 , 0.10654891],\n            [0.0311172 , 0.12911797, 0.03370714]])\n    >>> np.random.pareto(a=np.array([2,3])\n    array([0.26636696, 0.15685666])\n\n    The probability density for the Pareto distribution is f(x) = \\frac{am^a}{x^{a+1}}\n    where a is the shape and m the scale. Here m is assumed 1. The Pareto distribution\n    is a power law distribution. Pareto created it to describe the wealth in the economy.\n    \"\"\"\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if size == ():\n        size = None\n    return _api_internal.pareto(a, size, device, out)\n\n\n@wrap_ctx_to_device_func\ndef power(a, size=None, device=None, out=None):\n    r\"\"\"Draw samples in [0, 1] from a power distribution with given parameter a.\n\n    Parameters\n    ----------\n    a : float or array_like of floats\n        Shape of the distribution. Must be > 0.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``a`` is a scalar. Otherwise,\n        ``np.array(a).size`` samples are drawn.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the power distribution.\n\n    Examples\n    --------\n    >>> np.random.power(a=5)\n    array(0.8602478)\n    >>> np.random.power(a=5, size=[2,3])\n    array([[0.988391  , 0.5153122 , 0.9383134 ],\n           [0.9078098 , 0.87819266, 0.730635]])\n    >>> np.random.power(a=np.array([2,3])\n    array([0.7499419 , 0.88894516])\n\n    The probability density function is f(x; a) = ax^{a-1}, 0 \\le x \\le 1, a>0.\n    The power distribution is just the inverse of the Pareto distribution and\n    a special case of the Beta distribution.\n    \"\"\"\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if size == ():\n        size = None\n    return _api_internal.powerd(a, size, device, out)\n\n\n@wrap_ctx_to_device_func\ndef gamma(shape, scale=1.0, size=None, dtype=None, device=None, out=None):\n    \"\"\"Draw samples from a Gamma distribution.\n\n    Samples are drawn from a Gamma distribution with specified parameters,\n    `shape` (sometimes designated \"k\") and `scale` (sometimes designated\n    \"theta\"), where both parameters are > 0.\n\n    Parameters\n    ----------\n    shape : float or array_like of floats\n        The shape of the gamma distribution. Should be greater than zero.\n    scale : float or array_like of floats, optional\n        The scale of the gamma distribution. Should be greater than zero.\n        Default is equal to 1.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``shape`` and ``scale`` are both scalars.\n        Otherwise, ``np.broadcast(shape, scale).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n    device : Device, optional\n        Device context of output. Default is current device.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized gamma distribution.\n\n    The Gamma distribution is often used to model the times to failure of\n    electronic components, and arises naturally in processes for which the\n    waiting times between Poisson distributed events are relevant.\n    \"\"\"\n    if out is not None:\n        size = out.shape\n    if size == ():\n        size = None\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    return _api_internal.gamma(shape, scale, size, device, dtype, out)\n\n\n@wrap_ctx_to_device_func\ndef beta(a, b, size=None, dtype=None, device=None):\n    r\"\"\"Draw samples from a Beta distribution.\n\n    The Beta distribution is a special case of the Dirichlet distribution,\n    and is related to the Gamma distribution.  It has the probability\n    distribution function\n\n    .. math:: f(x; a,b) = \\frac{1}{B(\\alpha, \\beta)} x^{\\alpha - 1}\n                                                     (1 - x)^{\\beta - 1},\n\n    where the normalisation, B, is the beta function,\n\n    .. math:: B(\\alpha, \\beta) = \\int_0^1 t^{\\alpha - 1}\n                                 (1 - t)^{\\beta - 1} dt.\n\n    It is often seen in Bayesian inference and order statistics.\n\n    Parameters\n    ----------\n    a : float or array_like of floats\n        Alpha, positive (>0).\n    b : float or array_like of floats\n        Beta, positive (>0).\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``a`` and ``b`` are both scalars.\n        Otherwise, ``np.broadcast(a, b).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n    device : Device, optional\n        Device context of output. Default is current device.\n\n    Notes\n    -------\n    To use this  operator with scalars as input, please run ``npx.set_np()`` first.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized beta distribution.\n    \"\"\"\n    if dtype is None:\n        dtype = np.float64 if is_np_default_dtype() else np.float32\n    if device is None:\n        device = current_device()\n    if size == ():\n        size = None\n    # use fp64 to prevent precision loss\n    X = gamma(a, 1, size=size, dtype='float64', device=device)\n    Y = gamma(b, 1, size=size, dtype='float64', device=device)\n    out = X / (X + Y)\n    return out.astype(dtype)\n\n\n@wrap_ctx_to_device_func\ndef f(dfnum, dfden, size=None, device=None):\n    r\"\"\"Draw samples from an F distribution.\n\n    Samples are drawn from an F distribution with specified parameters,\n    `dfnum` (degrees of freedom in numerator) and `dfden` (degrees of\n    freedom in denominator), where both parameters must be greater than\n    zero.\n\n    The random variate of the F distribution (also known as the\n    Fisher distribution) is a continuous probability distribution\n    that arises in ANOVA tests, and is the ratio of two chi-square\n    variates.\n\n    Parameters\n    ----------\n    dfnum : float or ndarray of floats\n        Degrees of freedom in numerator, must be > 0.\n    dfden : float or ndarray of float\n        Degrees of freedom in denominator, must be > 0.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``dfnum`` and ``dfden`` are both scalars.\n        Otherwise, ``np.broadcast(dfnum, dfden).size`` samples are drawn.\n    device : Device, optional\n        Device context of output. Default is current device.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized Fisher distribution.\n\n    Examples\n    --------\n    An example from Glantz[1], pp 47-40:\n\n    Two groups, children of diabetics (25 people) and children from people\n    without diabetes (25 controls). Fasting blood glucose was measured,\n    case group had a mean value of 86.1, controls had a mean value of\n    82.2. Standard deviations were 2.09 and 2.49 respectively. Are these\n    data consistent with the null hypothesis that the parents diabetic\n    status does not affect their children's blood glucose levels?\n    Calculating the F statistic from the data gives a value of 36.01.\n\n    Draw samples from the distribution:\n\n    >>> dfnum = 1. # between group degrees of freedom\n    >>> dfden = 48. # within groups degrees of freedom\n    >>> s = np.random.f(dfnum, dfden, 1000)\n\n    The lower bound for the top 1% of the samples is :\n\n    >>> np.sort(s)[-10]\n    7.61988120985 # random\n\n    So there is about a 1% chance that the F statistic will exceed 7.62,\n    the measured value is 36, so the null hypothesis is rejected at the 1%\n    level.\n    \"\"\"\n    X = chisquare(df=dfnum, size=size, device=device)\n    Y = chisquare(df=dfden, size=size, device=device)\n    return (X * dfden) / (Y * dfnum)\n\n\n@wrap_ctx_to_device_func\ndef chisquare(df, size=None, dtype=None, device=None):\n    r\"\"\"\n    chisquare(df, size=None, dtype=None, device=None)\n\n    Draw samples from a chi-square distribution.\n\n    When `df` independent random variables, each with standard normal\n    distributions (mean 0, variance 1), are squared and summed, the\n    resulting distribution is chi-square (see Notes).  This distribution\n    is often used in hypothesis testing.\n\n    Parameters\n    ----------\n    df : float or ndarray of floats\n         Number of degrees of freedom, must be > 0.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``df`` is a scalar.  Otherwise,\n        ``np.array(df).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Dtype 'float32' or 'float64' is strongly recommended,\n        since lower precision might lead to out of range issue.\n    device : Device, optional\n        Device context of output. Default is current device.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized chi-square distribution.\n\n    Raises\n    ------\n    ValueError\n        When `df` <= 0 or when an inappropriate `size`\n        is given.\n\n    Notes\n    -----\n    The variable obtained by summing the squares of `df` independent,\n    standard normally distributed random variables:\n\n    .. math:: Q = \\sum_{i=0}^{\\mathtt{df}} X^2_i\n\n    is chi-square distributed, denoted\n\n    .. math:: Q \\sim \\chi^2_k.\n\n    The probability density function of the chi-squared distribution is\n\n    .. math:: p(x) = \\frac{(1/2)^{k/2}}{\\Gamma(k/2)}\n                     x^{k/2 - 1} e^{-x/2},\n\n    where :math:`\\Gamma` is the gamma function,\n\n    .. math:: \\Gamma(x) = \\int_0^{-\\infty} t^{x - 1} e^{-t} dt.\n\n    References\n    ----------\n    .. [1] NIST \"Engineering Statistics Handbook\"\n           https://www.itl.nist.gov/div898/handbook/eda/section3/eda3666.htm\n\n    Examples\n    --------\n    >>> np.random.chisquare(2,4)\n    array([ 1.89920014,  9.00867716,  3.13710533,  5.62318272]) # random\n    \"\"\"\n    if dtype is None:\n        dtype = np.float64 if is_np_default_dtype() else np.float32\n    if device is None:\n        device = current_device()\n    if size == ():\n        size = None\n    return gamma(df/2, 2, size=size, dtype=dtype, device=device)\n\n\n@wrap_ctx_to_device_func\ndef rand(*size, **kwargs):\n    r\"\"\"Random values in a given shape.\n\n    Create an array of the given shape and populate it with random\n    samples from a uniform distribution over [0, 1).\n    Parameters\n    ----------\n    d0, d1, ..., dn : int, optional\n        The dimensions of the returned array, should be all positive.\n        If no argument is given a single Python float is returned.\n    Returns\n    -------\n    out : ndarray\n       Random values.\n    Examples\n    --------\n    >>> np.random.rand(3,2)\n    array([[ 0.14022471,  0.96360618],  #random\n           [ 0.37601032,  0.25528411],  #random\n           [ 0.49313049,  0.94909878]]) #random\n    \"\"\"\n    output_shape = ()\n    for s in size:\n        output_shape += (s,)\n    return uniform(0, 1, size=output_shape, **kwargs)\n\n\ndef shuffle(x):\n    \"\"\"\n    Modify a sequence in-place by shuffling its contents.\n\n    This function only shuffles the array along the first axis of a\n    multi-dimensional array. The order of sub-arrays is changed but\n    their contents remain the same.\n\n    Parameters\n    ----------\n    x: ndarray\n        The array or list to be shuffled.\n\n    Returns\n    -------\n    None\n\n    Examples\n    --------\n    >>> arr = np.arange(10)\n    >>> np.random.shuffle(arr)\n    >>> arr\n    array([5., 1., 0., 6., 7., 3., 9., 8., 4., 2.])  # random\n\n    Multi-dimensional arrays are only shuffled along the first axis:\n\n    >>> arr = np.arange(9).reshape((3, 3))\n    >>> np.random.shuffle(arr)\n    >>> arr\n    array([[6., 7., 8.], # random\n           [3., 4., 5.],\n           [0., 1., 2.]])\n    \"\"\"\n    _api_internal.shuffle(x, x)\n\n\n@wrap_ctx_to_device_func\ndef laplace(loc=0.0, scale=1.0, size=None, dtype=None, device=None, out=None):\n    r\"\"\"Draw random samples from a Laplace distribution.\n\n    Samples are distributed according to a Laplace distribution parametrized\n    by *loc* (mean) and *scale* (the exponential decay).\n\n\n    Parameters\n    ----------\n    loc : float, The position of the distribution peak.\n\n    scale : float, the exponential decay.\n\n    size : int or tuple of ints, optional. Output shape.\n        If the given shape is, e.g., (m, n, k), then m * n * k samples are drawn.\n        Default is None, in which case a single value is returned.\n\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    device : Device, optional\n        Device context of output. Default is current device.\n    out : ``ndarray``, optional\n        Store output to an existing ``ndarray``.\n\n    Returns\n    -------\n    out : ndarray\n        Drawn samples from the parameterized Laplace distribution.\n    \"\"\"\n    if device is None:\n        device = str(current_device())\n    else:\n        device = str(device)\n    if dtype is not None and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    if size == ():\n        size = None\n    return _api_internal.laplace(loc, scale, size, dtype, device, out)\n"
  },
  {
    "path": "python/mxnet/ndarray/numpy_extension/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Module for the ops not belonging to the official numpy package.\"\"\"\n\nfrom . import _op\nfrom . import control_flow\nfrom . import image\nfrom . import random\nfrom . import _register\nfrom ._op import *  # pylint: disable=wildcard-import\nfrom .control_flow import *  # pylint: disable=wildcard-import\n\n__all__ = _op.__all__ + control_flow.__all__\n"
  },
  {
    "path": "python/mxnet/ndarray/numpy_extension/_api_internal.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for numpy_extension api.\"\"\"\n\nfrom ..._ffi.function import _init_api\n\n__all__ = []\n\n_init_api(\"_npx\", \"mxnet.ndarray.numpy_extension._api_internal\")\n"
  },
  {
    "path": "python/mxnet/ndarray/numpy_extension/_op.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for the operators not belonging to the official numpy package\nused in Gluon dispatched by F=ndarray module.\"\"\"\n\nimport numpy as _np\nfrom .._internal import NDArrayBase\nfrom . import _api_internal\nfrom ...util import set_module\nfrom ..ndarray import get_dtype_name\n\n\n__all__ = ['softmax', 'log_softmax', 'masked_softmax', 'masked_log_softmax',\n           'activation', 'batch_norm', 'fully_connected', 'pick', 'convolution',\n           'deconvolution', 'pooling', 'dropout', 'one_hot', 'rnn', 'embedding',\n           'topk', 'layer_norm', 'leaky_relu', 'batch_dot', 'broadcast_like',\n           'arange_like', 'group_norm']\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.ndarray.numpy_extension')\ndef softmax(data, axis=-1, length=None, temperature=None, use_length=False, dtype=None):\n    r\"\"\"Applies the softmax function.\n\n    The resulting array contains elements in the range (0,1) and the elements along the given axis sum up to 1.\n\n    .. math::\n       softmax(\\mathbf{z/t})_j = \\frac{e^{z_j/t}}{\\sum_{k=1}^K e^{z_k/t}}\n\n    for :math:`j = 1, ..., K`\n\n    t is the temperature parameter in softmax function. By default, t equals 1.0\n\n    Parameters\n    ----------\n    data : NDArray\n        The input array.\n    axis : int, optional, default='-1'\n        The axis along which to compute softmax.\n    length : NDArray\n        The length array.\n    temperature : double or None, optional, default=None\n        Temperature parameter in softmax\n    dtype : {None, 'float16', 'float32', 'float64'},optional, default='None'\n        DType of the output in case this can't be inferred. Defaults to\n        the same as input's dtype if not defined (dtype=None).\n    use_length : boolean or None, optional, default=0\n        Whether to use the length input as a mask over the data input.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Example\n    -------\n    >>> data = np.ones((2, 3))\n    >>> npx.softmax(data, axis=0)\n    array([[0.5, 0.5, 0.5],\n        [0.5, 0.5, 0.5]])\n    >>> npx.softmax(data, axis=1)\n    array([[0.33333334, 0.33333334, 0.33333334],\n        [0.33333334, 0.33333334, 0.33333334]])\n    \"\"\"\n    if dtype and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    if use_length:\n        assert length is not None, \"Missing length input\"\n        return _api_internal.softmax(data, length, axis, temperature, True, dtype)\n    else:\n        assert length is None, \"Length input is not used\"\n        return _api_internal.softmax(data, axis, temperature, False, dtype)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.ndarray.numpy_extension')\ndef log_softmax(data, axis=-1, length=None, temperature=None, use_length=False, dtype=None):\n    r\"\"\"Computes the log softmax of the input.\n    This is equivalent to computing softmax followed by log.\n\n    Parameters\n    ----------\n    data : NDArray\n        The input array.\n    axis : int, optional, default='-1'\n        The axis along which to compute softmax.\n    length : NDArray\n        The length array.\n    temperature : double or None, optional, default=None\n        Temperature parameter in softmax\n    dtype : {None, 'float16', 'float32', 'float64'},optional, default='None'\n        DType of the output in case this can't be inferred. Defaults to\n        the same as input's dtype if not defined (dtype=None).\n    use_length : boolean or None, optional, default=0\n        Whether to use the length input as a mask over the data input.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Examples\n    --------\n    >>> data = np.array([1, 2, .1])\n    >>> npx.log_softmax(data)\n    array([-1.4170278, -0.4170278, -2.3170278])\n    >>> data = np.array([[1, 2, .1],[.1, 2, 1]])\n    >>> npx.log_softmax(data, axis=0)\n    array([[-0.34115386, -0.6931472 , -1.2411538 ],\n        [-1.2411538 , -0.6931472 , -0.34115386]])\n    \"\"\"\n    if dtype and not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    if use_length:\n        assert length is not None, \"Missing length input\"\n        return _api_internal.log_softmax(data, length, axis, temperature, True, dtype)\n    else:\n        assert length is None, \"Length input is not used\"\n        return _api_internal.log_softmax(data, axis, temperature, False, dtype)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.ndarray.numpy_extension')\ndef masked_softmax(data, mask, axis=-1, temperature=1.0, normalize=True):\n    r\"\"\"Applies the softmax function masking elements according to the mask provided\n\n    Parameters\n    ----------\n    data : NDArray\n        The input array.\n    mask : NDArray\n        Mask to apply.\n    axis : int, optional, default='-1'\n        The axis along which to compute softmax.\n    temperature : double or None, optional, default=None\n        Temperature parameter in softmax\n    normalize : boolean or None, optional, default=1\n        Whether to normalize input data x: x = x - max(x)\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Examples\n    --------\n    >>> data = np.arange(5)\n    >>> mask = np.array([1, 0, 1, 0, 1])\n    >>> npx.masked_softmax(data, mask)\n    array([0.01587624, 0.        , 0.11731042, 0.        , 0.8668133 ])\n    >>> data = np.arange(10).reshape((2, 5))\n    >>> npx.masked_softmax(data, mask, axis=0)\n    array([[0.00669285, 0.        , 0.00669285, 0.        , 0.00669285],\n           [0.9933072 , 0.        , 0.9933072 , 0.        , 0.9933072 ]])\n    \"\"\"\n    assert data is not None and mask is not None, \"Missing input data and mask\"\n    return _api_internal.masked_softmax(data, mask, axis, temperature, normalize)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.ndarray.numpy_extension')\ndef masked_log_softmax(data, mask, axis=-1, temperature=1.0, normalize=True):\n    r\"\"\"Computes the masked log softmax of the input.\n    This is equivalent to computing masked softmax followed by log.\n\n    Parameters\n    ----------\n    data : NDArray\n        The input array.\n    mask : NDArray\n        Mask to apply.\n    axis : int, optional, default='-1'\n        The axis along which to compute softmax.\n    temperature : double or None, optional, default=None\n        Temperature parameter in softmax\n    normalize : boolean or None, optional, default=1\n        Whether to normalize input data x: x = x - max(x)\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Examples\n    --------\n    >>> data = np.arange(5)\n    >>> mask = np.array([1, 0, 1, 0, 1])\n    >>> npx.masked_log_softmax(data, mask)\n    array([-4.1429286 ,        -inf, -2.1429286 ,        -inf, -0.14292854])\n    >>> data = np.arange(10).reshape((2, 5))\n    >>> npx.masked_log_softmax(data, mask, axis=0)\n    array([[-5.0067153 ,        -inf, -5.0067153 ,        -inf, -5.0067153 ],\n           [-0.00671535,        -inf, -0.00671535,        -inf, -0.00671535]])\n    \"\"\"\n    assert data is not None and mask is not None, \"Missing input data and mask\"\n    return _api_internal.masked_log_softmax(data, mask, axis, temperature, normalize)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.ndarray.numpy_extension')\ndef activation(data, act_type='relu', **kwargs):\n    r\"\"\"Applies an activation function element-wise to the input.\n\n    The following activation functions are supported:\n\n    - `log_sigmoid`: :math:`y = log(\\frac{1}{1 + exp(-x)})`\n    - `mish`: :math:`y = x * tanh(log(1 + exp(x)))`\n    - `relu`: Rectified Linear Unit, :math:`y = max(x, 0)`\n    - `sigmoid`: :math:`y = \\frac{1}{1 + exp(-x)}`\n    - `tanh`: Hyperbolic tangent, :math:`y = \\frac{exp(x) - exp(-x)}{exp(x) + exp(-x)}`\n    - `softrelu`: Soft ReLU, or SoftPlus, :math:`y = log(1 + exp(x))`\n    - `softsign`: :math:`y = \\frac{x}{1 + abs(x)}`\n\n    Parameters\n    ----------\n    data : NDArray\n        The input array.\n    act_type : {'log_sigmoid', 'mish', 'relu', 'sigmoid', 'softrelu', 'softsign', 'tanh'}, required\n        Activation function to be applied.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    return _api_internal.activation(data, act_type)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.ndarray.numpy_extension')\ndef batch_norm(x, gamma, beta, running_mean, running_var, eps=1e-3, momentum=0.9,\n               fix_gamma=True, use_global_stats=False, output_mean_var=False, axis=1,\n               cudnn_off=False, min_calib_range=None, max_calib_range=None, **kwargs):\n    r\"\"\"Batch normalization.\n\n    Normalizes a data batch by mean and variance, and applies a scale ``gamma`` as\n    well as offset ``beta``.\n\n    Assume the input has more than one dimension and we normalize along axis 1.\n    We first compute the mean and variance along this axis:\n\n    .. math::\n\n      data\\_mean[i] = mean(data[:,i,:,...]) \\\\\n      data\\_var[i] = var(data[:,i,:,...])\n\n    Then compute the normalized output, which has the same shape as input, as following:\n\n    .. math::\n\n      out[:,i,:,...] = \\frac{data[:,i,:,...] - data\\_mean[i]}{\\sqrt{data\\_var[i]+\\epsilon}} * gamma[i] + beta[i]\n\n    Both *mean* and *var* returns a scalar by treating the input as a vector.\n\n    Assume the input has size *k* on axis 1, then both ``gamma`` and ``beta``\n    have shape *(k,)*. If ``output_mean_var`` is set to be true, then outputs both ``data_mean`` and\n    the inverse of ``data_var``, which are needed for the backward pass. Note that gradient of these\n    two outputs are blocked.\n\n    Besides the inputs and the outputs, this operator accepts two auxiliary\n    states, ``moving_mean`` and ``moving_var``, which are *k*-length\n    vectors. They are global statistics for the whole dataset, which are updated\n    by::\n\n      moving_mean = moving_mean * momentum + data_mean * (1 - momentum)\n      moving_var = moving_var * momentum + data_var * (1 - momentum)\n\n    If ``use_global_stats`` is set to be true, then ``moving_mean`` and\n    ``moving_var`` are used instead of ``data_mean`` and ``data_var`` to compute\n    the output. It is often used during inference.\n\n    The parameter ``axis`` specifies which axis of the input shape denotes\n    the 'channel' (separately normalized groups).  The default is 1.  Specifying -1 sets the channel\n    axis to be the last item in the input shape.\n\n    Both ``gamma`` and ``beta`` are learnable parameters. But if ``fix_gamma`` is true,\n    then set ``gamma`` to 1 and its gradient to 0.\n\n    .. Note::\n      When ``fix_gamma`` is set to True, no sparse support is provided. If ``fix_gamma is`` set to False,\n      the sparse tensors will fallback.\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data to batch normalization\n    gamma : NDArray\n        gamma array\n    beta : NDArray\n        beta array\n    moving_mean : NDArray\n        running mean of input\n    moving_var : NDArray\n        running variance of input\n    eps : double, optional, default=0.0010000000474974513\n        Epsilon to prevent div 0. Must be no less than CUDNN_BN_MIN_EPSILON\n        defined in cudnn.h when using cudnn (usually 1e-5)\n    momentum : float, optional, default=0.899999976\n        Momentum for moving average\n    fix_gamma : boolean, optional, default=1\n        Fix gamma while training\n    use_global_stats : boolean, optional, default=0\n        Whether use global moving statistics instead of local batch-norm.\n        This will force change batch-norm into a scale shift operator.\n    output_mean_var : boolean, optional, default=0\n        Output the mean and inverse std\n    axis : int, optional, default='1'\n        Specify which shape axis the channel is specified\n    cudnn_off : boolean, optional, default=0\n        Do not select CUDNN operator, if available\n    min_calib_range : float or None, optional, default=None\n        The minimum scalar value in the form of float32 obtained through calibration.\n        If present, it will be used to by quantized batch norm op to calculate primitive scale.\n        Note: this calib_range is to calib bn output.\n    max_calib_range : float or None, optional, default=None\n        The maximum scalar value in the form of float32 obtained through calibration.\n        If present, it will be used to by quantized batch norm op to calculate primitive scale.\n        Note: this calib_range is to calib bn output.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    out = _api_internal.batch_norm(x, gamma, beta, running_mean, running_var, eps, momentum,\n                                   fix_gamma, use_global_stats, output_mean_var, axis,\n                                   cudnn_off, min_calib_range, max_calib_range)\n    if isinstance(out, NDArrayBase):\n        return out\n    return list(out)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.ndarray.numpy_extension')\ndef fully_connected(x, weight, bias=None, num_hidden=None,\n                    no_bias=True, flatten=True, **kwargs):\n    r\"\"\"Applies a linear transformation: :math:`Y = XW^T + b`.\n\n    If ``flatten`` is set to be true, then the shapes are:\n\n    - **data**: `(batch_size, x1, x2, ..., xn)`\n    - **weight**: `(num_hidden, x1 * x2 * ... * xn)`\n    - **bias**: `(num_hidden,)`\n    - **out**: `(batch_size, num_hidden)`\n\n    If ``flatten`` is set to be false, then the shapes are:\n\n    - **data**: `(x1, x2, ..., xn, input_dim)`\n    - **weight**: `(num_hidden, input_dim)`\n    - **bias**: `(num_hidden,)`\n    - **out**: `(x1, x2, ..., xn, num_hidden)`\n\n    The learnable parameters include both ``weight`` and ``bias``.\n\n    If ``no_bias`` is set to be true, then the ``bias`` term is ignored.\n\n    .. Note::\n\n        The sparse support for FullyConnected is limited to forward evaluation with `row_sparse`\n        weight and bias, where the length of `weight.indices` and `bias.indices` must be equal\n        to `num_hidden`. This could be useful for model inference with `row_sparse` weights\n        trained with importance sampling or noise contrastive estimation.\n\n        To compute linear transformation with 'csr' sparse data, sparse.dot is recommended instead\n        of sparse.FullyConnected.\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data.\n    weight : NDArray\n        Weight matrix.\n    bias : NDArray\n        Bias parameter.\n    num_hidden : int, required\n        Number of hidden nodes of the output.\n    no_bias : boolean, optional, default=0\n        Whether to disable bias parameter.\n    flatten : boolean, optional, default=1\n        Whether to collapse all but the first axis of the input data tensor.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    assert num_hidden is not None, \"Please provide number of hidden nodes\"\n    if no_bias:\n        return _api_internal.fully_connected(x, weight, num_hidden, no_bias, flatten)\n    else:\n        assert bias is not None, \"Missing bias parameter\"\n        return _api_internal.fully_connected(x, weight, bias, num_hidden,\n                                             no_bias, flatten)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.ndarray.numpy_extension')\ndef pick(data, index, axis=-1, mode='clip', keepdims=False):\n    r\"\"\"Picks elements from an input array according to the input indices along the given axis.\n\n    Given an input array of shape ``(d0, d1)`` and indices of shape ``(i0,)``, the result will be\n    an output array of shape ``(i0,)`` with::\n\n      output[i] = input[i, indices[i]]\n\n    By default, if any index mentioned is too large, it is replaced by the index that addresses\n    the last element along an axis (the `clip` mode).\n\n    This function supports n-dimensional input and (n-1)-dimensional indices arrays.\n\n    Parameters\n    ----------\n    data : NDArray\n        The input array\n    index : NDArray\n        The index array\n    axis : int or None, optional, default='-1'\n        int or None. The axis to picking the elements.\n        Negative values means indexing from right to left.\n        If is `None`, the elements in the index w.r.t the flattened input will be picked.\n    keepdims : boolean, optional, default=0\n        If true, the axis where we pick the elements is\n        left in the result as dimension with size one.\n    mode : {'clip', 'wrap'},optional, default='clip'\n        Specify how out-of-bound indices behave. Default is \"clip\".\n        \"clip\" means clip to the range. So, if all indices mentioned are too large,\n        they are replaced by the index that addresses the last element along an axis.\n        \"wrap\" means to wrap around.\n\n    out : NDArray, optional\n        The output NDArray to hold the result.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Example\n    -------\n    >>> x = np.array([[1., 2.],[3., 4.],[5., 6.]])\n\n    picks elements with specified indices along axis 0\n\n    >>> npx.pick(x, np.array([0, 1]), 0)\n    array([1., 4.])\n\n    picks elements with specified indices along axis 1\n\n    >>> npx.pick(x, np.array([0, 1, 0]), 1)\n    array([1., 4., 5.])\n\n    picks elements with specified indices along axis 1 using 'wrap' mode\n    to place indicies that would normally be out of bounds\n\n    >>> npx.pick(x, np.array([2, -1, -2]), 1, mode='wrap')\n    array([1., 4., 5.])\n\n    picks elements with specified indices along axis 1 and dims are maintained\n\n    >>> npx.pick(x, np.array([[1.], [0.], [2.]]), 1, keepdims=True)\n    array([[2.],\n           [3.],\n           [6.]])\n    \"\"\"\n    return _api_internal.pick(data, index, axis, mode, keepdims)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.ndarray.numpy_extension')\ndef convolution(data=None, weight=None, bias=None, kernel=None, stride=None, dilate=None,\n                pad=None, num_filter=1, num_group=1, workspace=1024, no_bias=False,\n                cudnn_tune=None, cudnn_off=False, layout=None):\n    r\"\"\"Compute *N*-D convolution on *(N+2)*-D input.\n\n    In the 2-D convolution, given input data with shape *(batch_size,\n    channel, height, width)*, the output is computed by\n\n    .. math::\n\n       out[n,i,:,:] = bias[i] + \\sum_{j=0}^{channel} data[n,j,:,:] \\star\n       weight[i,j,:,:]\n\n    where :math:`\\star` is the 2-D cross-correlation operator.\n\n    For general 2-D convolution, the shapes are\n\n    - **data**: *(batch_size, channel, height, width)*\n    - **weight**: *(num_filter, channel, kernel[0], kernel[1])*\n    - **bias**: *(num_filter,)*\n    - **out**: *(batch_size, num_filter, out_height, out_width)*.\n\n    Define::\n\n      f(x,k,p,s,d) = floor((x+2*p-d*(k-1)-1)/s)+1\n\n    then we have::\n\n      out_height=f(height, kernel[0], pad[0], stride[0], dilate[0])\n      out_width=f(width, kernel[1], pad[1], stride[1], dilate[1])\n\n    If ``no_bias`` is set to be true, then the ``bias`` term is ignored.\n\n    The default data ``layout`` is *NCHW*, namely *(batch_size, channel, height,\n    width)*. We can choose other layouts such as *NWC*.\n\n    If ``num_group`` is larger than 1, denoted by *g*, then split the input ``data``\n    evenly into *g* parts along the channel axis, and also evenly split ``weight``\n    along the first dimension. Next compute the convolution on the *i*-th part of\n    the data with the *i*-th weight part. The output is obtained by concatenating all\n    the *g* results.\n\n    1-D convolution does not have *height* dimension but only *width* in space.\n\n    - **data**: *(batch_size, channel, width)*\n    - **weight**: *(num_filter, channel, kernel[0])*\n    - **bias**: *(num_filter,)*\n    - **out**: *(batch_size, num_filter, out_width)*.\n\n    3-D convolution adds an additional *depth* dimension besides *height* and\n    *width*. The shapes are\n\n    - **data**: *(batch_size, channel, depth, height, width)*\n    - **weight**: *(num_filter, channel, kernel[0], kernel[1], kernel[2])*\n    - **bias**: *(num_filter,)*\n    - **out**: *(batch_size, num_filter, out_depth, out_height, out_width)*.\n\n    Both ``weight`` and ``bias`` are learnable parameters.\n\n    There are other options to tune the performance.\n\n    - **cudnn_tune**: enable this option leads to higher startup time but may give\n      faster speed. Options are\n\n      - **off**: no tuning\n      - **limited_workspace**:run test and pick the fastest algorithm that doesn't\n        exceed workspace limit.\n      - **fastest**: pick the fastest algorithm and ignore workspace limit.\n      - **None** (default): the behavior is determined by environment variable\n        ``MXNET_CUDNN_AUTOTUNE_DEFAULT``. 0 for off, 1 for limited workspace\n        (default), 2 for fastest.\n\n    - **workspace**: A large number leads to more (GPU) memory usage but may improve\n      the performance.\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data to the ConvolutionOp.\n    weight : NDArray\n        Weight matrix.\n    bias : NDArray\n        Bias parameter.\n    kernel : Shape(tuple), required\n        Convolution kernel size: (w,), (h, w) or (d, h, w)\n    stride : Shape(tuple), optional, default=[]\n        Convolution stride: (w,), (h, w) or (d, h, w). Defaults to 1 for each dimension.\n    dilate : Shape(tuple), optional, default=[]\n        Convolution dilate: (w,), (h, w) or (d, h, w). Defaults to 1 for each dimension.\n    pad : Shape(tuple), optional, default=[]\n        Zero pad for convolution: (w,), (h, w) or (d, h, w). Defaults to no padding.\n    num_filter : int (non-negative), required\n        Convolution filter(channel) number\n    num_group : int (non-negative), optional, default=1\n        Number of group partitions.\n    workspace : long (non-negative), optional, default=1024\n        Maximum temporary workspace allowed (MB) in convolution.This parameter has two usages.\n        When CUDNN is not used, it determines the effective batch size of the convolution kernel.\n        When CUDNN is used, it controls the maximum temporary storage used for tuning the best\n        CUDNN kernel when `limited_workspace` strategy is used.\n    no_bias : boolean, optional, default=0\n        Whether to disable bias parameter.\n    cudnn_tune : {None, 'fastest', 'limited_workspace', 'off'},optional, default='None'\n        Whether to pick convolution algo by running performance test.\n    cudnn_off : boolean, optional, default=0\n        Turn off cudnn for this layer.\n    layout : {None, 'NCDHW', 'NCHW', 'NCW', 'NDHWC', 'NHWC'},optional, default='None'\n        Set layout for input, output and weight. Empty for\n        default layout: NCW for 1d, NCHW for 2d and NCDHW for 3d.\n        NHWC and NDHWC are only supported on GPU.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    assert data is not None and weight is not None and kernel is not None, \\\n           \"Missing input data, weight or kernel\"\n    assert num_filter >= 1, \"Number of output filters should be greater equal to 1.\"\n    assert workspace >= 0, \"Maximum temporary workspace should be greater equal to 0.\"\n    if no_bias:\n        assert bias is None, \"Using no bias\"\n        return _api_internal.convolution(data, weight, kernel, stride, dilate, pad,\n                                         num_filter, num_group, workspace, no_bias,\n                                         cudnn_tune, cudnn_off, layout)\n    else:\n        assert bias is not None, \"Using bias\"\n        return _api_internal.convolution(data, weight, bias, kernel, stride, dilate, pad,\n                                         num_filter, num_group, workspace, no_bias,\n                                         cudnn_tune, cudnn_off, layout)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.ndarray.numpy_extension')\ndef deconvolution(data=None, weight=None, bias=None, kernel=None, stride=None, dilate=None,\n                  pad=None, adj=None, target_shape=None, num_filter=1, num_group=1,\n                  workspace=1024, no_bias=False, cudnn_tune=None,\n                  cudnn_off=False, layout=None):\n    r\"\"\"Computes 1D, 2D or 3D transposed convolution (aka fractionally strided convolution) of\n    the input tensor. This operation can be seen as the gradient of Convolution operation\n    with respect to its input. Convolution usually reduces the size of the input.\n    Transposed convolution works the other way, going from a smaller input\n    to a larger output while preserving the connectivity pattern.\n\n    Parameters\n    ----------\n    data : NDArray\n        Input tensor to the deconvolution operation.\n    weight : NDArray\n        Weights representing the kernel.\n    bias : NDArray\n        Bias added to the result after the deconvolution operation.\n    kernel : Shape(tuple), required\n        Deconvolution kernel size: (w,), (h, w) or (d, h, w).\n        This is same as the kernel size used for the corresponding convolution\n    stride : Shape(tuple), optional, default=[]\n        The stride used for the corresponding convolution: (w,), (h, w) or (d, h, w).\n        Defaults to 1 for each dimension.\n    dilate : Shape(tuple), optional, default=[]\n        Dilation factor for each dimension of the input: (w,), (h, w) or (d, h, w).\n        Defaults to 1 for each dimension.\n    pad : Shape(tuple), optional, default=[]\n        The amount of implicit zero padding added during convolution for each dimension of\n        the input: (w,), (h, w) or (d, h, w). ``(kernel-1)/2`` is usually a good choice.\n        If `target_shape` is set, `pad` will be ignored and a padding that will generate\n        the target shape will be used. Defaults to no padding.\n    adj : Shape(tuple), optional, default=[]\n        Adjustment for output shape: (w,), (h, w) or (d, h, w).\n        If `target_shape` is set, `adj` will be ignored and computed accordingly.\n    target_shape : Shape(tuple), optional, default=[]\n        Shape of the output tensor: (w,), (h, w) or (d, h, w).\n    num_filter : int (non-negative), required\n        Number of output filters.\n    num_group : int (non-negative), optional, default=1\n        Number of groups partition.\n    workspace : long (non-negative), optional, default=512\n        Maximum temporary workspace allowed (MB) in deconvolution. This parameter has two usages.\n        When CUDNN is not used, it determines the effective batch size of the deconvolution kernel.\n        When CUDNN is used, it controls the maximum temporary storage used for tuning\n        the best CUDNN kernel when `limited_workspace` strategy is used.\n    no_bias : boolean, optional, default=1\n        Whether to disable bias parameter.\n    cudnn_tune : {None, 'fastest', 'limited_workspace', 'off'},optional, default='None'\n        Whether to pick convolution algorithm by running performance test.\n    cudnn_off : boolean, optional, default=0\n        Turn off cudnn for this layer.\n    layout : {None, 'NCDHW', 'NCHW', 'NCW', 'NDHWC', 'NHWC'},optional, default='None'\n        Set layout for input, output and weight. Empty for\n        default layout, NCW for 1d, NCHW for 2d and NCDHW for 3d.\n        NHWC and NDHWC are only supported on GPU.\n\n    out : NDArray, optional\n        The output NDArray to hold the result.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    assert data is not None and weight is not None and kernel is not None, \\\n           \"Missing input data, weight or kernel\"\n    assert num_filter >= 1, \"Number of output filters should be greater equal to 1.\"\n    assert workspace >= 0, \"Maximum temporary workspace should be greater equal to 0.\"\n    if no_bias:\n        assert bias is None, \"Using no bias\"\n        return _api_internal.deconvolution(data, weight, kernel, stride, dilate, pad,\n                                           adj, target_shape, num_filter, num_group,\n                                           workspace, no_bias, cudnn_tune, cudnn_off, layout)\n    else:\n        assert bias is not None, \"Using bias\"\n        return _api_internal.deconvolution(data, weight, bias, kernel, stride, dilate, pad,\n                                           adj, target_shape, num_filter, num_group,\n                                           workspace, no_bias, cudnn_tune, cudnn_off, layout)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.ndarray.numpy_extension')\ndef pooling(data=None, kernel=None, stride=None, pad=None, pool_type=\"max\",\n            pooling_convention=\"valid\", global_pool=False, cudnn_off=False,\n            p_value=None, count_include_pad=None, layout=None, **kwargs):\n    r\"\"\"Performs pooling on the input.\n\n    The shapes for 1-D pooling are\n\n    - **data** and **out**: *(batch_size, channel, width)* (NCW layout) or\n      *(batch_size, width, channel)* (NWC layout),\n\n    The shapes for 2-D pooling are\n\n    - **data** and **out**: *(batch_size, channel, height, width)* (NCHW layout) or\n      *(batch_size, height, width, channel)* (NHWC layout),\n\n        out_height = f(height, kernel[0], pad[0], stride[0])\n        out_width = f(width, kernel[1], pad[1], stride[1])\n\n    The definition of *f* depends on ``pooling_convention``, which has two options:\n\n    - **valid** (default)::\n\n        f(x, k, p, s) = floor((x+2*p-k)/s)+1\n\n    - **full**, which is compatible with Caffe::\n\n        f(x, k, p, s) = ceil((x+2*p-k)/s)+1\n\n    When ``global_pool`` is set to be true, then global pooling is performed. It will reset\n    ``kernel=(height, width)`` and set the appropiate padding to 0.\n\n    Three pooling options are supported by ``pool_type``:\n\n    - **avg**: average pooling\n    - **max**: max pooling\n    - **sum**: sum pooling\n    - **lp**: Lp pooling\n\n    For 3-D pooling, an additional *depth* dimension is added before\n    *height*. Namely the input data and output will have shape *(batch_size, channel, depth,\n    height, width)* (NCDHW layout) or *(batch_size, depth, height, width, channel)* (NDHWC layout).\n\n    Notes on Lp pooling:\n\n    Lp pooling was first introduced by this paper: https://arxiv.org/pdf/1204.3968.pdf.\n    L-1 pooling is simply sum pooling, while L-inf pooling is simply max pooling.\n    We can see that Lp pooling stands between those two, in practice the most common value for p is 2.\n\n    For each window ``X``, the mathematical expression for Lp pooling is:\n\n    :math:`f(X) = \\sqrt[p]{\\sum_{x}^{X} x^p}`\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data to the pooling operator.\n    kernel : Shape(tuple), optional, default=[]\n        Pooling kernel size: (y, x) or (d, y, x)\n    pool_type : {'avg', 'lp', 'max', 'sum'},optional, default='max'\n        Pooling type to be applied.\n    global_pool : boolean, optional, default=0\n        Ignore kernel size, do global pooling based on current input feature map.\n    cudnn_off : boolean, optional, default=0\n        Turn off cudnn pooling and use MXNet pooling operator.\n    pooling_convention : {'full', 'same', 'valid'},optional, default='valid'\n        Pooling convention to be applied.\n    stride : Shape(tuple), optional, default=[]\n        Stride: for pooling (y, x) or (d, y, x). Defaults to 1 for each dimension.\n    pad : Shape(tuple), optional, default=[]\n        Pad for pooling: (y, x) or (d, y, x). Defaults to no padding.\n    p_value : int or None, optional, default='None'\n        Value of p for Lp pooling, can be 1 or 2, required for Lp Pooling.\n    count_include_pad : boolean or None, optional, default=None\n        Only used for AvgPool, specify whether to count padding elements for averagecalculation.\n        For example, with a 5*5 kernel on a 3*3 corner of a image,the sum of the 9 valid elements will\n        be divided by 25 if this is set to true,or it will be divided by 9 if this is set to false.\n        Defaults to true.\n    layout : {None, 'NCDHW', 'NCHW', 'NCW', 'NDHWC', 'NHWC', 'NWC'},optional, default='None'\n        Set layout for input and output. Empty for\n        default layout: NCW for 1d, NCHW for 2d and NCDHW for 3d.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    assert data is not None and kernel is not None, \"Missing input data or kernel\"\n    out = _api_internal.pooling(data, kernel, stride, pad, pool_type, pooling_convention,\n                                global_pool, cudnn_off, p_value, count_include_pad, layout)\n    if isinstance(out, NDArrayBase):\n        return out\n    else:\n        return list(out)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.ndarray.numpy_extension')\ndef dropout(data, p=0.5, mode=\"training\", axes=None, cudnn_off=False, **kwargs):\n    r\"\"\"Applies dropout operation to input array.\n\n    - During training, each element of the input is set to zero with probability p.\n      The whole array is rescaled by :math:`1/(1-p)` to keep the expected\n      sum of the input unchanged.\n\n    - During testing, this operator does not change the input if mode is 'training'.\n      If mode is 'always', the same computaion as during training will be applied.\n\n    Parameters\n    ----------\n    data : NDArray\n        Input array to which dropout will be applied.\n    p : float, optional, default=0.5\n        Fraction of the input that gets dropped out during training time.\n    mode : {'always', 'training'},optional, default='training'\n        Whether to only turn on dropout during training or to also turn on for inference.\n    axes : Shape(tuple), optional, default=[]\n        Axes for variational dropout kernel.\n    cudnn_off : boolean or None, optional, default=0\n        Whether to turn off cudnn in dropout operator. This option is ignored if axes is specified.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    return _api_internal.dropout(data, p, mode, axes, cudnn_off)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.ndarray.numpy_extension')\ndef one_hot(data, depth=None, on_value=1.0, off_value=0.0, dtype=\"float32\"):\n    r\"\"\"Returns a one-hot array.\n\n    The locations represented by `indices` take value `on_value`, while all\n    other locations take value `off_value`.\n\n    `one_hot` operation with `indices` of shape ``(i0, i1)`` and `depth`  of ``d`` would result\n    in an output array of shape ``(i0, i1, d)`` with::\n\n      output[i,j,:] = off_value\n      output[i,j,indices[i,j]] = on_value\n\n    Parameters\n    ----------\n    indices : NDArray\n        array of locations where to set on_value\n    depth : long, required\n        Depth of the one hot dimension.\n    on_value : double, optional, default=1\n        The value assigned to the locations represented by indices.\n    off_value : double, optional, default=0\n        The value assigned to the locations not represented by indices.\n    dtype : {'bfloat16', 'float16', 'float32', 'float64', 'int32', 'int64', 'int8', 'uint8'},\n            optional, default='float32'\n        DType of the output\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Example\n    -------\n    >>> data = np.array([1,0,2,0])\n    >>> npx.one_hot(data, 3)\n    array([[0., 1., 0.],\n           [1., 0., 0.],\n           [0., 0., 1.],\n           [1., 0., 0.]], dtype=float64)\n    >>> npx.one_hot(data, 3, on_value=8, off_value=1, dtype='int32')\n    array([[1, 8, 1],\n           [8, 1, 1],\n           [1, 1, 8],\n           [8, 1, 1]], dtype=int32)\n    >>> data = np.array([[1,0],[1,0],[2,0]])\n    >>> npx.one_hot(data, 3)\n    array([[[0., 1., 0.],\n            [1., 0., 0.]],\n           [[0., 1., 0.],\n            [1., 0., 0.]],\n           [[0., 0., 1.],\n            [1., 0., 0.]]], dtype=float64)\n    \"\"\"\n    assert depth is not None, \"Please provide the depth of one hot dimension.\"\n    if not isinstance(dtype, str):\n        dtype = get_dtype_name(dtype)\n    return _api_internal.one_hot(data, depth, on_value, off_value, dtype)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.ndarray.numpy_extension')\ndef rnn(data=None, parameters=None, state=None, state_cell=None, sequence_length=None,\n        mode=None, state_size=None, num_layers=None, bidirectional=False,\n        state_outputs=False, p=0.0, use_sequence_length=False, projection_size=None,\n        lstm_state_clip_min=None, lstm_state_clip_max=None, lstm_state_clip_nan=None):\n    r\"\"\"Applies recurrent layers to input data. Currently, vanilla RNN, LSTM and GRU are\n    implemented, with both multi-layer and bidirectional support.\n\n    When the input data is of type float32 and the environment variables MXNET_CUDA_ALLOW_TENSOR_CORE\n    and MXNET_CUDA_TENSOR_OP_MATH_ALLOW_CONVERSION are set to 1, this operator will try to use\n    pseudo-float16 precision (float32 math with float16 I/O) precision in order to use\n    Tensor Cores on suitable NVIDIA GPUs. This can sometimes give significant speedups.\n\n    **Vanilla RNN**\n\n    Applies a single-gate recurrent layer to input X. Two kinds of activation function are supported:\n    ReLU and Tanh.\n\n    With ReLU activation function:\n\n    .. math::\n        h_t = relu(W_{ih} * x_t + b_{ih}  +  W_{hh} * h_{(t-1)} + b_{hh})\n\n    With Tanh activtion function:\n\n    .. math::\n        h_t = \\tanh(W_{ih} * x_t + b_{ih}  +  W_{hh} * h_{(t-1)} + b_{hh})\n\n    Reference paper: Finding structure in time - Elman, 1988.\n    https://axon.cs.byu.edu/~martinez/classes/678/Papers/Elman_time.pdf\n\n    **LSTM**\n\n    Long Short-Term Memory - Hochreiter, 1997. http://www.bioinf.jku.at/publications/older/2604.pdf\n\n    .. math::\n      \\begin{array}{ll}\n                i_t = \\mathrm{sigmoid}(W_{ii} x_t + b_{ii} + W_{hi} h_{(t-1)} + b_{hi}) \\\\\n                f_t = \\mathrm{sigmoid}(W_{if} x_t + b_{if} + W_{hf} h_{(t-1)} + b_{hf}) \\\\\n                g_t = \\tanh(W_{ig} x_t + b_{ig} + W_{hc} h_{(t-1)} + b_{hg}) \\\\\n                o_t = \\mathrm{sigmoid}(W_{io} x_t + b_{io} + W_{ho} h_{(t-1)} + b_{ho}) \\\\\n                c_t = f_t * c_{(t-1)} + i_t * g_t \\\\\n                h_t = o_t * \\tanh(c_t)\n                \\end{array}\n\n    With the projection size being set, LSTM could use the projection feature to reduce the parameters\n    size and give some speedups without significant damage to the accuracy.\n\n    Long Short-Term Memory Based Recurrent Neural Network Architectures for Large Vocabulary Speech\n    Recognition - Sak et al. 2014. https://arxiv.org/abs/1402.1128\n\n    .. math::\n      \\begin{array}{ll}\n                i_t = \\mathrm{sigmoid}(W_{ii} x_t + b_{ii} + W_{ri} r_{(t-1)} + b_{ri}) \\\\\n                f_t = \\mathrm{sigmoid}(W_{if} x_t + b_{if} + W_{rf} r_{(t-1)} + b_{rf}) \\\\\n                g_t = \\tanh(W_{ig} x_t + b_{ig} + W_{rc} r_{(t-1)} + b_{rg}) \\\\\n                o_t = \\mathrm{sigmoid}(W_{io} x_t + b_{o} + W_{ro} r_{(t-1)} + b_{ro}) \\\\\n                c_t = f_t * c_{(t-1)} + i_t * g_t \\\\\n                h_t = o_t * \\tanh(c_t)\n                r_t = W_{hr} h_t\n                \\end{array}\n\n    **GRU**\n\n    Gated Recurrent Unit - Cho et al. 2014. http://arxiv.org/abs/1406.1078\n\n    The definition of GRU here is slightly different from paper but compatible with CUDNN.\n\n    .. math::\n      \\begin{array}{ll}\n                r_t = \\mathrm{sigmoid}(W_{ir} x_t + b_{ir} + W_{hr} h_{(t-1)} + b_{hr}) \\\\\n                z_t = \\mathrm{sigmoid}(W_{iz} x_t + b_{iz} + W_{hz} h_{(t-1)} + b_{hz}) \\\\\n                n_t = \\tanh(W_{in} x_t + b_{in} + r_t * (W_{hn} h_{(t-1)}+ b_{hn})) \\\\\n                h_t = (1 - z_t) * n_t + z_t * h_{(t-1)} \\\\\n                \\end{array}\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data to RNN\n    parameters : NDArray\n        Vector of all RNN trainable parameters concatenated\n    state : NDArray\n        initial hidden state of the RNN\n    state_cell : NDArray\n        initial cell state for LSTM networks (only for LSTM)\n    sequence_length : NDArray\n        Vector of valid sequence lengths for each element in batch.\n        (Only used if use_sequence_length kwarg is True)\n    state_size : int (non-negative), required\n        size of the state for each layer\n    num_layers : int (non-negative), required\n        number of stacked layers\n    bidirectional : boolean, optional, default=0\n        whether to use bidirectional recurrent layers\n    mode : {'gru', 'lstm', 'rnn_relu', 'rnn_tanh'}, required\n        the type of RNN to compute\n    p : float, optional, default=0\n        drop rate of the dropout on the outputs of each RNN layer, except the last layer.\n    state_outputs : boolean, optional, default=0\n        Whether to have the states as symbol outputs.\n    projection_size : int or None, optional, default='None'\n        size of project size\n    lstm_state_clip_min : double or None, optional, default=None\n        Minimum clip value of LSTM states. This option must be used together with lstm_state_clip_max.\n    lstm_state_clip_max : double or None, optional, default=None\n        Maximum clip value of LSTM states. This option must be used together with lstm_state_clip_min.\n    lstm_state_clip_nan : boolean, optional, default=0\n        Whether to stop NaN from propagating in state by clipping it to min/max.\n        If clipping range is not specified, this option is ignored.\n    use_sequence_length : boolean, optional, default=0\n        If set to true, this layer takes in an extra input parameter `sequence_length`\n        to specify variable length sequence\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    assert mode is not None, \"Please provide rnn type to compute. e.g. rnn_relu, rnn_tanh, lstm, gru\"\n    assert data is not None and parameters is not None and state is not None, \\\n        \"Missing input data/parameters/state.\"\n    assert state_size is not None, \"Please provide state_size\"\n    assert num_layers is not None, \"Please provide num_layers\"\n    if use_sequence_length:\n        assert sequence_length is not None, \\\n            \"use_sequence_length is set True, but no sequence_length provided.\"\n        if mode == \"lstm\":\n            assert state_cell is not None, \\\n                \"RNN computing mode is lstm, but no state_cell is provided\"\n            return _api_internal.rnn(data, parameters, state, state_cell, sequence_length,\n                                     state_size, num_layers, bidirectional, state_outputs,\n                                     mode, p, use_sequence_length, projection_size,\n                                     lstm_state_clip_min, lstm_state_clip_max, lstm_state_clip_nan)\n        else:\n            return _api_internal.rnn(data, parameters, state, sequence_length,\n                                     state_size, num_layers, bidirectional, state_outputs,\n                                     mode, p, use_sequence_length, projection_size,\n                                     lstm_state_clip_min, lstm_state_clip_max, lstm_state_clip_nan)\n    else:\n        if mode == \"lstm\":\n            assert state_cell is not None, \\\n                \"RNN computing mode is lstm, but no state_cell is provided\"\n            return _api_internal.rnn(data, parameters, state, state_cell,\n                                     state_size, num_layers, bidirectional, state_outputs,\n                                     mode, p, use_sequence_length, projection_size,\n                                     lstm_state_clip_min, lstm_state_clip_max, lstm_state_clip_nan)\n        else:\n            return _api_internal.rnn(data, parameters, state,\n                                     state_size, num_layers, bidirectional, state_outputs,\n                                     mode, p, use_sequence_length, projection_size,\n                                     lstm_state_clip_min, lstm_state_clip_max, lstm_state_clip_nan)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.ndarray.numpy_extension')\ndef embedding(data, weight, input_dim=None, output_dim=None, dtype=\"float32\", sparse_grad=False,\n              **kwargs):\n    r\"\"\"Maps integer indices to vector representations (embeddings).\n\n    This operator maps words to real-valued vectors in a high-dimensional space,\n    called word embeddings. These embeddings can capture semantic and syntactic properties of the words.\n    For example, it has been noted that in the learned embedding spaces, similar words tend\n    to be close to each other and dissimilar words far apart.\n\n    For an input array of shape (d1, ..., dK),\n    the shape of an output array is (d1, ..., dK, output_dim).\n    All the input values should be integers in the range [0, input_dim).\n\n    If the input_dim is ip0 and output_dim is op0, then shape of the embedding weight matrix must be\n    (ip0, op0).\n\n    When \"sparse_grad\" is False, if any index mentioned is too large, it is replaced by the index that\n    addresses the last vector in an embedding matrix.\n    When \"sparse_grad\" is True, an error will be raised if invalid indices are found.\n\n    The storage type of weight can be either row_sparse or default.\n\n    .. Note::\n\n        If \"sparse_grad\" is set to True, the storage type of gradient w.r.t weights will be\n        \"row_sparse\". Only a subset of optimizers support sparse gradients, including SGD, AdaGrad\n        and Adam. Note that by default lazy updates is turned on, which may perform differently\n        from standard updates. For more details, please check the Optimization API at:\n        https://mxnet.apache.org/versions/master/api/python/docs/api/optimizer/index.html\n\n    Parameters\n    ----------\n    data : NDArray\n        The input array to the embedding operator.\n    weight : NDArray\n        The embedding weight matrix.\n    input_dim : long, required\n        Vocabulary size of the input indices.\n    output_dim : long, required\n        Dimension of the embedding vectors.\n    dtype : {'bfloat16', 'float16', 'float32', 'float64', 'int32', 'int64', 'int8', 'uint8'},\n            optional, default='float32'\n        Data type of weight.\n    sparse_grad : boolean, optional, default=0\n        Compute row sparse gradient in the backward calculation.\n        If set to True, the grad's storage type is row_sparse.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Example\n    -------\n    >>> input_dim = 4\n    >>> output_dim = 5\n\n    Each row in weight matrix y represents a word. So, y = (w0,w1,w2,w3)\n\n    >>> y = np.arange(input_dim * output_dim).reshape(input_dim, output_dim)\n    >>> y\n    array([[ 0.,  1.,  2.,  3.,  4.],\n           [ 5.,  6.,  7.,  8.,  9.],\n           [10., 11., 12., 13., 14.],\n           [15., 16., 17., 18., 19.]])\n\n    Input array x represents n-grams(2-gram). So, x = [(w1,w3), (w0,w2)]\n\n    >>> x = np.array([[1., 3.], [0., 2.]])\n    >>> x\n    array([[1., 3.],\n           [0., 2.]])\n\n    Mapped input x to its vector representation y.\n\n    >>> npx.embedding(x, y, input_dim, output_dim)\n    array([[[ 5.,  6.,  7.,  8.,  9.],\n            [15., 16., 17., 18., 19.]],\n\n           [[ 0.,  1.,  2.,  3.,  4.],\n            [10., 11., 12., 13., 14.]]])\n    \"\"\"\n    assert input_dim > 0, \"Vocabulary size of the input indices should be greater than 0.\"\n    assert output_dim > 0, \"Dimension of the embedding vectors should greater than 0.\"\n    assert not sparse_grad, \"Currently row sparse gradient is not supported in npx.embedding\"\n    return _api_internal.embedding(data, weight, input_dim, output_dim, dtype, sparse_grad)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.ndarray.numpy_extension')\ndef topk(data, axis=-1, k=1, ret_typ=\"indices\", is_ascend=False, dtype=\"float32\"):\n    r\"\"\"Returns the indices of the top *k* elements in an input array along the given\n     axis (by default).\n     If ret_type is set to 'value' returns the value of top *k* elements (instead of indices).\n     In case of ret_type = 'both', both value and index would be returned.\n     The returned elements will be sorted.\n\n    Parameters\n    ----------\n    data : NDArray\n        The input array\n    axis : int or None, optional, default='-1'\n        Axis along which to choose the top k indices.\n        If not given, the flattened array is used. Default is -1.\n    k : int, optional, default='1'\n        Number of top elements to select, should be always smaller than or equal to\n        the element number in the given axis. A global sort is performed if set k < 1.\n    ret_typ : {'both', 'indices', 'mask', 'value'},optional, default='indices'\n        The return type.\n     \"value\" means to return the top k values,\n     \"indices\" means to return the indices of the top k values,\n     \"mask\" means to return a mask array containing 0 and 1. 1 means the top k values.\n     \"both\" means to return a list of both values and indices of top k elements.\n    is_ascend : boolean, optional, default=0\n        Whether to choose k largest or k smallest elements.\n        Top K largest elements will be chosen if set to false.\n    dtype : {'float16', 'float32', 'float64', 'int32', 'int64', 'uint8'},\n            optional, default='float32'\n        DType of the output indices when ret_typ is \"indices\" or \"both\".\n        An error will be raised if the selected data type cannot precisely represent the indices.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Example\n    -------\n    >>> x = np.array([[0.3, 0.2, 0.4], [0.1, 0.3, 0.2]])\n\n    returns an index of the largest element on last axis\n\n    >>> npx.topk(x)\n    array([[2.],\n           [1.]])\n\n    returns the value of top-2 largest elements on last axis\n\n    >>> npx.topk(x, ret_typ='value', k=2)\n    array([[0.4, 0.3],\n           [0.3, 0.2]])\n\n    returns the value of top-2 smallest elements on last axis\n\n    >>> npx.topk(x, ret_typ='value', k=2, is_ascend=1)\n    array([[0.2, 0.3],\n           [0.1, 0.2]])\n\n    returns the value of top-2 largest elements on axis 0\n\n    >>> npx.topk(x, axis=0, ret_typ='value', k=2)\n    array([[0.3, 0.3, 0.4],\n           [0.1, 0.2, 0.2]])\n\n    flattens and then returns list of both values and indices\n\n    >>> npx.topk(x, ret_typ='both', k=2)\n    [array([[0.4, 0.3], [0.3, 0.2]]),\n     array([[2., 0.], [1., 2.]])]\n    \"\"\"\n    out = _api_internal.topk(data, axis, k, ret_typ, is_ascend, dtype)\n    if isinstance(out, NDArrayBase):\n        return out\n    return list(out)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.ndarray.numpy_extension')\ndef layer_norm(data=None, gamma=None, beta=None, axis=None, eps=None, output_mean_var=None):\n    r\"\"\"Layer normalization.\n\n    Normalizes the channels of the input tensor by mean and variance, and applies a scale ``gamma`` as\n    well as offset ``beta``.\n\n    Assume the input has more than one dimension and we normalize along axis 1.\n    We first compute the mean and variance along this axis and then\n    compute the normalized output, which has the same shape as input, as following:\n\n    .. math::\n\n      out = \\frac{data - mean(data, axis)}{\\sqrt{var(data, axis) + \\epsilon}} * gamma + beta\n\n    Both ``gamma`` and ``beta`` are learnable parameters.\n\n    Unlike BatchNorm and InstanceNorm,  the *mean* and *var* are computed along the channel dimension.\n\n    Assume the input has size *k* on axis 1, then both ``gamma`` and ``beta``\n    have shape *(k,)*. If ``output_mean_var`` is set to be true, then outputs both ``data_mean`` and\n    ``data_std``. Note that no gradient will be passed through these two outputs.\n\n    The parameter ``axis`` specifies which axis of the input shape denotes\n    the 'channel' (separately normalized groups).  The default is -1, which sets the channel\n    axis to be the last item in the input shape.\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data to layer normalization\n    gamma : NDArray\n        gamma array\n    beta : NDArray\n        beta array\n    axis : int, optional, default='-1'\n        The axis to perform layer normalization.\n        Usually, this should be be axis of the channel dimension.\n        Negative values means indexing from right to left.\n    eps : float, optional, default=9.99999975e-06\n        An `epsilon` parameter to prevent division by 0.\n    output_mean_var : boolean, optional, default=0\n        Output the mean and std calculated along the given axis.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    out = _api_internal.layer_norm(data, gamma, beta, axis, eps, output_mean_var)\n    if isinstance(out, NDArrayBase):\n        return out\n    return list(out)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.ndarray.numpy_extension')\ndef leaky_relu(data=None, gamma=None, act_type=\"leaky\", slope=0.25, lower_bound=0.125,\n               upper_bound=0.334, **kwargs):\n    r\"\"\"Applies Leaky rectified linear unit activation element-wise to the input.\n\n    Leaky ReLUs attempt to fix the \"dying ReLU\" problem by allowing a small `slope`\n    when the input is negative and has a slope of one when input is positive.\n\n    The following modified ReLU Activation functions are supported:\n\n    - *elu*: Exponential Linear Unit. `y = x > 0 ? x : slope * (exp(x)-1)`\n    - *gelu*: Gaussian Error Linear Unit. `y = 0.5 * x * (1 + erf(x / sqrt(2)))`\n    - *selu*: Scaled Exponential Linear Unit. `y = lambda * (x > 0 ? x : alpha * (exp(x) - 1))` where\n      *lambda = 1.0507009873554804934193349852946* and *alpha = 1.6732632423543772848170429916717*.\n    - *leaky*: Leaky ReLU. `y = x > 0 ? x : slope * x`\n    - *prelu*: Parametric ReLU. This is same as *leaky* except that `slope` is learnt during training.\n    - *rrelu*: Randomized ReLU. same as *leaky* but the `slope` is uniformly and randomly chosen from\n      *[lower_bound, upper_bound)* for training, while fixed to be\n      *(lower_bound+upper_bound)/2* for inference.\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data to activation function.\n    gamma : NDArray\n        Input data to activation function.\n    act_type : {'elu', 'gelu', 'leaky', 'prelu', 'rrelu', 'selu'},optional, default='leaky'\n        Activation function to be applied.\n    slope : float, optional, default=0.25\n        Init slope for the activation. (For leaky and elu only)\n    lower_bound : float, optional, default=0.125\n        Lower bound of random slope. (For rrelu only)\n    upper_bound : float, optional, default=0.333999991\n        Upper bound of random slope. (For rrelu only)\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    if act_type == \"prelu\":\n        assert gamma is not None, \"If activation function is prelu, please provide input gamma\"\n        out = _api_internal.leaky_relu(data, gamma, act_type, slope, lower_bound, upper_bound)\n        if isinstance(out, NDArrayBase):\n            return out\n        return list(out)\n    else:\n        return _api_internal.leaky_relu(data, act_type, slope, lower_bound, upper_bound)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.ndarray.numpy_extension')\ndef batch_dot(a, b, transpose_a=False, transpose_b=False, forward_stype=\"default\"):\n    r\"\"\"Batchwise dot product.\n\n    ``batch_dot`` is used to compute dot product of ``x`` and ``y`` when ``x`` and\n    ``y`` are data in batch, namely N-D (N >= 3) arrays in shape of `(B0, ..., B_i, :, :)`.\n\n    For example, given ``x`` with shape `(B_0, ..., B_i, N, M)` and ``y`` with shape\n    `(B_0, ..., B_i, M, K)`, the result array will have shape `(B_0, ..., B_i, N, K)`,\n    which is computed by::\n\n       batch_dot(x,y)[b_0, ..., b_i, :, :] = dot(x[b_0, ..., b_i, :, :], y[b_0, ..., b_i, :, :])\n\n    Parameters\n    ----------\n    lhs : NDArray\n        The first input\n    rhs : NDArray\n        The second input\n    transpose_a : boolean, optional, default=0\n        If true then transpose the first input before dot.\n    transpose_b : boolean, optional, default=0\n        If true then transpose the second input before dot.\n    forward_stype : {None, 'csr', 'default', 'row_sparse'},optional, default='None'\n        The desired storage type of the forward output given by user,\n        if thecombination of input storage types and this hint does not matchany implemented ones,\n        the dot operator will perform fallback operationand still produce\n        an output of the desired storage type.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    return _api_internal.batch_dot(a, b, transpose_a, transpose_b, forward_stype)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.ndarray.numpy_extension')\ndef broadcast_like(lhs, rhs, lhs_axes=None, rhs_axes=None):\n    r\"\"\"Broadcasts lhs to have the same shape as rhs.\n\n    Broadcasting is a mechanism that allows NDArrays to perform arithmetic operations\n    with arrays of different shapes efficiently without creating multiple copies of arrays.\n    Also see, `Broadcasting <https://docs.scipy.org/doc/numpy/user/basics.broadcasting.html>`_\n    for more explanation.\n\n    Broadcasting is allowed on axes with size 1, such as from `(2,1,3,1)` to\n    `(2,8,3,9)`. Elements will be duplicated on the broadcasted axes.\n\n    Parameters\n    ----------\n    lhs : NDArray\n        First input.\n    rhs : NDArray\n        Second input.\n    lhs_axes : Shape or None, optional, default=None\n        Axes to perform broadcast on in the first input array\n    rhs_axes : Shape or None, optional, default=None\n        Axes to copy from the second input array\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    example\n    -------\n    >>> a = np.array([[1,2,3]])\n    >>> b = np.array([[5,6,7],[7,8,9]])\n    >>> npx.broadcast_like(a, b)\n    array([[1., 2., 3.],\n           [1., 2., 3.]])\n    >>> a = np.array([9])\n    >>> b = np.array([1,2,3,4,5])\n    >>> npx.broadcast_like(a, b, lhs_axes=(0,), rhs_axes=(-1,))\n    array([9., 9., 9., 9., 9.])\n    \"\"\"\n    return _api_internal.broadcast_like(lhs, rhs, lhs_axes, rhs_axes)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.ndarray.numpy_extension')\ndef arange_like(data, start=0.0, step=1.0, repeat=1, ctx=None, axis=None):\n    r\"\"\"Return an array with evenly spaced values. If axis is not given, the output will\n    have the same shape as the input array. Otherwise, the output will be a 1-D array with size of\n    the specified axis in input shape.\n\n    Parameters\n    ----------\n    data : NDArray\n        The input\n    start : double, optional, default=0\n        Start of interval. The interval includes this value. The default start value is 0.\n    step : double, optional, default=1\n        Spacing between values.\n    repeat : int, optional, default='1'\n        The repeating time of all elements.\n        E.g repeat=3, the element a will be repeated three times --> a, a, a.\n    ctx : string, optional, default=''\n        Context of output, in format [cpu|gpu|cpu_pinned](n).Only used for imperative calls.\n    axis : int or None, optional, default='None'\n        Arange elements according to the size of a certain axis of input array.\n        The negative numbers are interpreted counting from the backward.\n        If not provided, will arange elements according to the input shape.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Example\n    -------\n    >>> x = np.random.uniform(0, 1, size=(3,4))\n    >>> x\n    array([[0.5488135 , 0.5928446 , 0.71518934, 0.84426576],\n           [0.60276335, 0.8579456 , 0.5448832 , 0.8472517 ],\n           [0.4236548 , 0.6235637 , 0.6458941 , 0.3843817 ]])\n    >>> npx.arange_like(x, start=0)\n    array([[ 0.,  1.,  2.,  3.],\n           [ 4.,  5.,  6.,  7.],\n           [ 8.,  9., 10., 11.]])\n    >>> npx.arange_like(x, start=0, axis=-1)\n    array([0., 1., 2., 3.])\n    \"\"\"\n    return _api_internal.arange_like(data, start, step, repeat, ctx, axis)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.ndarray.numpy_extension')\ndef group_norm(data, gamma, beta, num_groups=1, eps=1e-3, output_mean_var=False):\n    r\"\"\"Group normalization.\n\n    The input channels are separated into ``num_groups`` groups,\n    each containing ``num_channels / num_groups`` channels.\n    The mean and standard-deviation are calculated separately over the each group.\n\n    .. math::\n\n      data = data.reshape((N, num_groups, C // num_groups, ...))\n      out = \\frac{data - mean(data, axis)}{\\sqrt{var(data, axis) + \\epsilon}} * gamma + beta\n\n    Both ``gamma`` and ``beta`` are learnable parameters.\n\n\n\n    Defined in ../src/operator/nn/group_norm.cc:L78\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data\n    gamma : NDArray\n        gamma array\n    beta : NDArray\n        beta array\n    num_groups : int, optional, default='1'\n        Total number of groups.\n    eps : float, optional, default=9.99999975e-06\n        An `epsilon` parameter to prevent division by 0.\n    output_mean_var : boolean, optional, default=0\n        Output the mean and std calculated along the given axis.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    out = _api_internal.group_norm(data, gamma, beta, num_groups, eps, output_mean_var)\n    if isinstance(out, NDArrayBase):\n        return out\n    return list(out)\n"
  },
  {
    "path": "python/mxnet/ndarray/numpy_extension/_register.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Registering numpy_extension ops.\"\"\"\n\nfrom ...base import _init_np_op_module\nfrom ..register import _make_ndarray_function\n\n\n_init_np_op_module(root_module_name='mxnet', np_module_name='numpy_extension',\n                   mx_module_name='ndarray', make_op_func=_make_ndarray_function)\n"
  },
  {
    "path": "python/mxnet/ndarray/numpy_extension/control_flow.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for registering control flow ops for imperative programming.\"\"\"\n\nfrom . import _api_internal\nfrom .._internal import NDArrayBase\nfrom ...util import set_module\nfrom ...numpy import ndarray as np_ndarray\nfrom ...symbol import Symbol\nfrom ...base import _as_list\nfrom ... import symbol, _deferred_compute as dc, autograd as ag\nfrom ...attribute import AttrScope, current as current_attribute\n\n\n__all__ = [\"foreach\", \"while_loop\", \"cond\"]\n\n\ndef _flatten(args, inout_str):\n    \"\"\"Parse the arguments into a flattened list + an additional format array.\n    The format array stores the structure of the original arguments to help reconstruct the inputs.\n\n    Parameters\n    ----------\n    args : NDArray, Symbol, or (nested) list of Symbol or NDArray\n        We allow None inside the args.\n    inout_str : str\n        The name of the HybridBlock\n\n    Returns\n    -------\n    flat : list of Symbol or NDArray\n        The flatten version of the input args.\n    fmts : (nested) list of ints\n        Stores the format information of the original structured args.\n    \"\"\"\n    if isinstance(args, np_ndarray):\n        return [args], int(0)\n    if isinstance(args, Symbol):\n        length = len(args.list_outputs())\n        length = length if length > 1 else 0\n        return [args], int(length)\n    if args is None:\n        return [None], int(-1)\n\n    if not isinstance(args, (list, tuple)):\n        raise ValueError(\"When hybridized, the input of HybridBlock {}\"\n                         \" must be (nested) list of Symbol\"\n                         \" or NDArray, \"\n                         \"but got {} of type {}\".format(inout_str, str(args), str(type(args))))\n    flat = []\n    fmts = []\n    for i in args:\n        arg, fmt = _flatten(i, inout_str)\n        flat.extend(arg)\n        fmts.append(fmt)\n    return flat, fmts\n\n\ndef _regroup(args, fmt):\n    \"\"\"Reconstruct the structured arguments based on the flattened version.\n\n    Parameters\n    ----------\n    args : NDArray, Symbol, or (nested) list of Symbol or NDArray\n        We allow None inside the args.\n    fmt : (nested) list of ints\n        Stores the format information of the original structured args.\n\n    Returns\n    -------\n    ret : NDArray, Symbol, or (nested) list of Symbol or NDArray\n\n    \"\"\"\n    def _merger(args, fmt):\n        \"\"\"Recursive call to merge the arguments\"\"\"\n        if isinstance(fmt, int):\n            if fmt < -1:\n                raise ValueError(\"Unsupported encoded format {}.\".format(fmt))\n            if fmt == 0:\n                return args[0], args[1:]\n            if fmt == -1:\n                if args[0] is not None:\n                    raise ValueError('We do not support passing types that are not None'\n                                     ' when the initial HybridBlock has received NoneType and'\n                                     ' has been hybridized.'\n                                     ' Received arg = {}, fmt = {}.'.format(args[0], fmt))\n                return None, args[1:]\n            else:\n                return args[:fmt], args[fmt:]\n\n        if not isinstance(args, (list, tuple)):\n            raise ValueError(\"When hybridized, the output of HybridBlock must be (nested)\"\n                             \" list of Symbol or NDArray, \"\n                             \"but got {} of type {}\".format(args, type(args)))\n        ret = []\n        for i in fmt:\n            res, args = _merger(args, i)\n            ret.append(res)\n        return ret, args\n    return _merger(args, fmt)[0]\n\ndef _get_unique_subgraph_name(subgraph_name):\n    attrs = current_attribute()._attr\n    if attrs.get(\"__subgraph_name__\", \"\") != \"\":\n        subgraph_name = \"\".join([attrs[\"__subgraph_name__\"], \"$\", subgraph_name])\n    AttrScope._subgraph_names[subgraph_name] += 1\n    subgraph_name = subgraph_name + str(AttrScope._subgraph_names[subgraph_name] - 1)\n    return subgraph_name\n\n# This construct a subgraph for given output nodes.\n# If an output node is one of the input nodes, we call identity to make sure\n# that outputs nodes are different from input nodes.\ndef _construct_subgraph(sym_out, sym_states):\n    sym_out = _as_list(sym_out)\n    sym_states = _as_list(sym_states)\n    all_outputs = []\n    all_outputs.extend(sym_out)\n    all_outputs.extend(sym_states)\n    g = symbol.Group(all_outputs)\n\n    flat_out = []\n    all_input_names = g.list_inputs()\n    output_names = {o.name for o in sym_out}\n    for o in sym_out:\n        if o.name in all_input_names:\n            flat_out.append(symbol.op.identity(o))\n        else:\n            flat_out.append(o)\n\n    for s in sym_states:\n        if s.name in all_input_names or s.name in output_names:\n            flat_out.append(symbol.op.identity(s))\n        else:\n            flat_out.append(s)\n    return symbol.Group(flat_out)\n\n@set_module('mxnet.ndarray.numpy_extension')\ndef foreach(body, data, init_states, name=\"foreach\"):\n    \"\"\"Run a for loop with user-defined computation over NDArrays on dimension 0.\n\n    This operator simulates a for loop and body has the computation for an iteration\n    of the for loop. It runs the computation in body on each slice from the input\n    NDArrays.\n\n    body takes two arguments as input and outputs a tuple of two elements,\n    as illustrated below::\n\n        out, states = body(data1, states)\n\n    data1 can be either an NDArray or a list of NDArrays. If data is an NDArray,\n    data1 is an NDArray. Otherwise, data1 is a list of NDArrays and has the same\n    size as data. states is a list of NDArrays and have the same size as init_states.\n    Similarly, out can be either an NDArray or a list of NDArrays, which are concatenated\n    as the first output of foreach; states from the last execution of body\n    are the second output of foreach.\n\n    The computation done by this operator is equivalent to the pseudo code below\n    when the input data is NDArray::\n\n        states = init_states\n        outs = []\n        for i in data.shape[0]:\n            s = data[i]\n            out, states = body(s, states)\n            outs.append(out)\n        outs = stack(*outs)\n\n\n    Parameters\n    ----------\n    body : HybridBlock.\n        Define computation in an iteration.\n    data: an NDArray or a list of NDArrays.\n        The input data.\n    init_states: an NDArray or nested lists of NDArrays.\n        The initial values of the loop states.\n\n    Returns\n    -------\n    outputs: an NDArray or nested lists of NDArrays.\n        The output data concatenated from the output of all iterations.\n    states: an NDArray or nested lists of NDArrays.\n        The loop states in the last iteration.\n\n    Examples\n    --------\n    >>> step = lambda data, states: (data + states[0], [states[0] * 2])\n    >>> data = mx.np.random.uniform(size=(2, 10))\n    >>> states = [mx.np.random.uniform(size=(10))]\n    >>> outs, states = npx.control_flow.foreach(step, data, states)\n    \"\"\"\n\n    def check_input(inputs, in_type, msg):\n        is_NDArray_or_list = True\n        if isinstance(inputs, list):\n            for i in inputs:\n                if not isinstance(i, in_type):\n                    is_NDArray_or_list = False\n                    break\n        else:\n            is_NDArray_or_list = isinstance(inputs, in_type)\n        assert is_NDArray_or_list, msg\n\n    flatten_data, data_fmt = _flatten(data, \"foreach input\")\n    check_input(flatten_data, np_ndarray,\n                \"data should be an mxnet.numpy.ndarray or a nested list of mxnet.numpy.ndarray\")\n    flatten_state, state_fmt = _flatten(init_states, \"foreach states\")\n    check_input(flatten_state, np_ndarray,\n                \"init_states should be an mxnet.numpy.ndarray or a nested list of mxnet.numpy.ndarray\")\n\n    real_data = [ele[0].copy().detach() if ele is not None else None for ele in flatten_data]\n    real_state = [ele.copy().detach() if ele is not None else None for ele in flatten_state]\n\n    # If the input python function references to the symbols outside\n    # the python function, we need to prune the computation graph constructed from\n    # the function. One way of doing it is to mark the nodes in the computation graph\n    # with AttrScope and prune the nodes without the special attribute.\n    name = _get_unique_subgraph_name(name)\n    with AttrScope(__subgraph_name__=name):\n        data_names = ['data_subgraph{}'.format(i) for i, ele in enumerate(real_data)]\n        state_names = ['state_subgraph{}'.format(i) for i, ele in enumerate(real_state)]\n        symbol_data = [\n            symbol.var(name).as_np_ndarray()\n            for arg, name in zip(real_data, data_names)\n        ]\n        symbol_state = [\n            symbol.var(name).as_np_ndarray()\n            for arg, name in zip(real_state, state_names)\n        ]\n        dc.set_variable(real_data, symbol_data)\n        dc.set_variable(real_state, symbol_state)\n        in_eles = _regroup(real_data, data_fmt)\n        in_states = _regroup(real_state, state_fmt)\n        if dc.is_deferred_compute():\n            out, states = body(in_eles, in_states)\n        else:\n            with ag.pause(), dc.context():\n                out, states = body(in_eles, in_states)\n\n        flatten_out, out_fmt = _flatten(out, \"foreach output\")\n        flatten_out_state, state_fmt = _flatten(states, \"foreach loop_vars\")\n\n        num_out_data = len(flatten_out)\n        num_states = len(flatten_out_state)\n        num_outputs = num_out_data + num_states\n        sym_out = [dc.get_symbol(out_data) for out_data in flatten_out]\n        sym_states = [dc.get_symbol(out_state) for out_state in flatten_out_state]\n        dc.clear(flatten_out)\n        dc.clear(flatten_out_state)\n        g = _construct_subgraph(sym_out, sym_states)\n\n    params_names = []\n    params_data = []\n    if hasattr(body, \"collect_params\"):\n        for p in body.collect_params().values():\n            params_names.append(p.var().name)\n            params_data.append(p.data())\n\n    subg_input_names = g.list_inputs()\n\n    in_data, in_states, params = [], [], []\n    in_data_locs, in_state_locs, remain_locs, in_state_index = [], [], [], []\n    for i, sub_name in enumerate(subg_input_names):\n        if sub_name in data_names:\n            in_data_locs.append(i)\n            idx = data_names.index(sub_name)\n            in_data.append(flatten_data[idx])\n        elif sub_name in state_names:\n            in_state_locs.append(i)\n            idx = state_names.index(sub_name)\n            in_states.append(flatten_state[idx])\n            in_state_index.append(idx)\n        elif sub_name in params_names:\n            remain_locs.append(i)\n            idx = params_names.index(sub_name)\n            params.append(params_data[idx])\n        else:\n            raise AssertionError(\"the data arrays have to be used in the loop body\")\n\n    ordered_ins = in_data + in_states + params\n\n    ndoutput = _api_internal.foreach(g.handle, *ordered_ins, num_outputs, num_out_data, in_state_locs,\n                                     in_data_locs, remain_locs, in_state_index)\n    if isinstance(ndoutput, NDArrayBase):\n        ret = ndoutput\n    else:\n        ret = list(ndoutput)\n    outs = []\n    for i in range(num_outputs - num_states):\n        outs.append(ret[i])\n    outs = _regroup(outs, out_fmt)\n    states = []\n    for i in range(num_states):\n        states.append(ret[num_outputs - num_states + i])\n    states = _regroup(states, state_fmt)\n\n    return (outs, states)\n\n\n#pylint: disable=W0621\n@set_module('mxnet.ndarray.numpy_extension')\ndef while_loop(cond, func, loop_vars, max_iterations=None, name=\"while_loop\"):\n    \"\"\"Run a while loop with user-defined computation and loop condition.\n\n    This operator simulates a while loop which iterately does customized computation\n    as long as the condition is satisfied.\n\n    `loop_vars` is a list of NDArrays on which the computation uses.\n\n    `cond` is a user-defined function, used as the loop condition.\n    It consumes `loop_vars`, and produces a scalar MXNet NDArray,\n    indicating the termination of the loop.\n    The loop ends when `cond` returns false (zero).\n    The `cond` is variadic, and its signature should be\n    `cond(*loop_vars) => NDArray`.\n\n    `func` is a user-defined function, used as the loop body.\n    It also consumes `loop_vars`, and produces `step_output` and `new_loop_vars` at each step.\n    In each step, `step_output` should contain the same number elements.\n    Through all steps, the i-th element of `step_output` should have the same shape and dtype.\n    Also, `new_loop_vars` should contain the same number of elements as `loop_vars`,\n    and the corresponding element should have the same shape and dtype.\n    The `func` is variadic, and its signature should be\n    `func(*loop_vars) =>\n    (NDArray or nested List[NDArray] step_output, NDArray or nested List[NDArray] new_loop_vars)`.\n\n    `max_iterations` is a scalar that defines the maximum number of iterations allowed.\n\n    This function returns two lists.\n    The first list has the length of `|step_output|`,\n    in which the i-th element are all i-th elements of\n    `step_output` from all steps, stacked along axis 0.\n    The second list has the length of `|loop_vars|`,\n    which represents final states of loop variables.\n\n    .. warning::\n\n       For now, the axis 0 of all NDArrays in the first list are `max_iterations`,\n       due to lack of dynamic shape inference.\n\n    .. warning::\n\n       When `cond` is never satisfied, we assume `step_output` is empty,\n       because it cannot be inferred. This is different from the symbolic version.\n\n    Parameters\n    ----------\n    cond: a Python function.\n        The loop condition.\n    func: a Python function.\n        The loop body.\n    loop_vars: an NDArray or nested lists of NDArrays.\n        The initial values of the loop variables.\n    max_iterations: a python int.\n        Maximum number of iterations.\n\n    Returns\n    ------\n    outputs: an NDArray or nested lists of NDArrays\n        stacked output from each step\n    states: an NDArray or nested lists of NDArrays\n        final state\n\n    Examples\n    --------\n    >>> cond = lambda i, s: i <= 5\n    >>> func = lambda i, s: ([i + s], [i + 1, s + i])\n    >>> loop_vars = (mx.np.array([0], dtype=\"int64\"), mx.np.array([1], dtype=\"int64\"))\n    >>> outputs, states = mx.npx.while_loop(cond, func, loop_vars, max_iterations=10)\n    >>> outputs\n    [array([[ 1],\n           [ 2],\n           [ 4],\n           [ 7],\n           [11],\n           [16],\n           [ 0],\n           [ 0],\n           [ 0],\n           [ 0]], dtype=int64)]\n    >>> states\n    [array([6], dtype=int64), array([16], dtype=int64)]\n    \"\"\"\n    def _to_python_scalar(inputs, type_, name):\n        \"\"\"Converts \"inputs\", possibly typed mxnet NDArray, a numpy ndarray, other python types,\n        to the given type\n        \"\"\"\n        if isinstance(inputs, np_ndarray):\n            inputs = inputs.item()\n        try:\n            inputs = type_(inputs)\n        except:\n            raise ValueError(f\"Cannot convert {name} to python {type_.__name__}\")\n        return inputs\n\n    def _cond_wrapper(loop_vars):\n        if dc.is_deferred_compute():\n            result = cond(*loop_vars).astype(\"int\")\n        else:\n            with ag.pause(), dc.context():\n                result = cond(*loop_vars).astype(\"int\")\n        flatten_out, _ = _flatten(result, \"while_loop output\")\n        out = dc.get_symbol(flatten_out)\n        dc.clear(flatten_out)\n        return [], [out], [], []\n\n    def _func_wrapper(loop_vars):\n        \"\"\"This wrapper unifies\n             \"func: loop_vars -> new_loop_vars\"\n         and \"func: loop_vars -> (step_output, new_loop_vars)\"\n        into \"func: loop_vars -> (None or tuple of step_outputs, tuple of new_loop_vars)\n        \"\"\"\n        if dc.is_deferred_compute():\n            step_output, new_loop_vars = func(*loop_vars)\n        else:\n            with ag.pause(), dc.context():\n                step_output, new_loop_vars = func(*loop_vars)\n        if step_output is None:\n            step_output = []\n        if new_loop_vars is None:\n            new_loop_vars = []\n        if isinstance(step_output, tuple):\n            step_output = list(step_output)\n        if isinstance(new_loop_vars, tuple):\n            new_loop_vars = list(new_loop_vars)\n        new_loop_vars = _as_list(new_loop_vars)\n        if len(loop_vars) != len(new_loop_vars):\n            raise ValueError(\"The length of loop_vars should be consistent during the loop\")\n        step_output_flatten, out_fmt = _flatten(step_output, \"while output\")\n        new_loop_vars_flatten, var_fmt = _flatten(new_loop_vars, \"while loop_vars\")\n        if isinstance(step_output, list):\n            if len(step_output) == 0:\n                step_out = []\n            else:\n                step_out = [dc.get_symbol(out) for out in step_output_flatten]\n        else:\n            step_output_flatten, out_fmt = _flatten(step_output, \"while output\")\n            step_out = [dc.get_symbol(step_output_flatten)]\n        if len(new_loop_vars) == 0:\n            new_var = []\n        else:\n            new_var = [dc.get_symbol(var) for var in new_loop_vars_flatten]\n        return step_out, new_var, out_fmt, var_fmt\n\n    def _create_subgraph(graph_vars, graph_func, subgraph_name):\n        subgraph_name = _get_unique_subgraph_name(subgraph_name)\n        with AttrScope(__subgraph_name__=subgraph_name):\n            # create new variables with the same name,\n            # them feed them to the given func\n            flatten_data, data_fmt = _flatten(graph_vars, \"foreach input\")\n            real_data = [ele.copy().detach() if ele is not None else None for ele in flatten_data]\n            data_names = ['data_subgraph{}'.format(i) for i, ele in enumerate(real_data)]\n            symbol_data = [\n                symbol.var(name).as_np_ndarray()\n                for arg, name in zip(real_data, data_names)\n            ]\n            dc.set_variable(real_data, symbol_data)\n            new_graph_vars = _regroup(real_data, data_fmt)\n            outputs, final_state, out_fmt, var_fmt = graph_func(new_graph_vars)\n            # first `num_out_data` elements belong to `outputs`\n            # other elements belong to `final_state`\n            num_out_data = len(outputs)\n            num_outputs = len(outputs) + len(final_state)\n            # group all outputs of graph_func\n            graph = _construct_subgraph(outputs, final_state)\n        return graph, num_out_data, num_outputs, out_fmt, var_fmt\n\n    flatten_loop_vars, init_loop_var_fmt = _flatten(loop_vars, \"while loop_vars\")\n\n    def _union_inputs(*graphs):\n        # Given a list of graphs, each whose inputs are either from loop_vars or other variables.\n        # 1) calculate a list `inputs`, the union of their inputs.\n        # 2) for each graph, determine in which indices their inputs reside in `inputs`\n        # 3) for each variable in the input of `graph`, find which index it is\n        inputs = []             # List[Symbol], result of 1)\n        locs = []               # List[Tuple(List[Int], List[Int])], a list of tuples,\n                                # where tuples are results of 2) and 3)\n        input_id_to_loc = {}    # Dict[int, int], given id(sym), input_id_to_loc maps it\n                                # to a `loc`, where inputs[loc] = sym\n        for graph in graphs:\n            # some loop_vars are inputs to `graph`, some are not\n            name_to_loop_vars = {'data_subgraph{}'.format(i): ele for i, ele in enumerate(flatten_loop_vars)}\n            # also we collect the mapping from var's name to var's loc in loop_vars\n            name_to_var_locs = {'data_subgraph{}'.format(i): i for i, ele in enumerate(flatten_loop_vars)}\n            # collect arguments for each subgraph\n            input_locs = []                         # results from the second step\n            var_locs = [-1] * len(flatten_loop_vars)        # results from the third step\n            subg_input_names = graph.list_inputs()\n            for name in subg_input_names:\n                assert name in name_to_loop_vars   # it should obviously hold\n                array = name_to_loop_vars[name]\n                # do 2), and 1) is implicitly done\n                if id(array) in input_id_to_loc:\n                    loc = input_id_to_loc[id(array)]\n                else:\n                    loc = len(input_id_to_loc)\n                    inputs.append(array)\n                    input_id_to_loc[id(array)] = loc\n                input_locs.append(loc)\n                # do 3)\n                if name in name_to_var_locs:\n                    var_locs[name_to_var_locs[name]] = len(input_locs) - 1\n                    name_to_var_locs.pop(name, None)\n            locs.append((input_locs, var_locs))\n        return inputs, locs\n    if max_iterations is None:\n        raise ValueError(\"max_iterations should be specified\")\n    max_iterations = _to_python_scalar(max_iterations, int, \"max_iteration\")\n    # It should be work as fine if loop_vars are empty I guess,\n    # but it is semantically unnecessary to include this case.\n    if isinstance(loop_vars, (list, tuple)):\n        if len(loop_vars) == 0:\n            raise ValueError(\"loop_vars should contain at least one element\")\n    else:\n        assert isinstance(loop_vars, np_ndarray), (\"loop_vars should be either mxnet.numpy.ndarray\" \\\n            \" or list/tuple of mxnet.numpy.ndarray\")\n        loop_vars = [loop_vars]\n    # create graph for `cond'\n    cond_g, num_out_data, num_outputs, _, _ = \\\n        _create_subgraph(loop_vars, _cond_wrapper, name + \"_cond\")\n    assert num_out_data == 0\n    assert num_outputs == 1\n    # create graph for `func`\n    func_g, num_out_data, num_outputs, out_fmt, _ = \\\n        _create_subgraph(loop_vars, _func_wrapper, name + \"_func\")\n    # find symbols used in either cond_g or func_g\n    input_vars, ((cond_input_locs, _), (func_input_locs, func_var_locs)) = \\\n        _union_inputs(cond_g, func_g)\n    for i_th, loc in enumerate(func_var_locs, 1):\n        if loc == -1:\n            raise ValueError(f\"The {i_th}-th loop_var doesn't involve into the computation\")\n    result = _api_internal.while_loop(\n        cond_g.handle,\n        func_g.handle,\n        *input_vars,\n        max_iterations,\n        cond_input_locs,\n        func_input_locs,\n        func_var_locs,\n        num_out_data,\n        num_outputs\n    )\n    if isinstance(result, np_ndarray):\n        ret = [result]\n    else:\n        ret = list(result)\n    outputs = [ret[i] for i in range(num_out_data)]\n    outputs = _regroup(outputs, out_fmt)\n    final_loop_vars = [ret[i] for i in range(num_out_data, num_outputs)]\n    final_loop_vars = _regroup(final_loop_vars, init_loop_var_fmt)\n    return outputs, final_loop_vars\n\n\n@set_module('mxnet.ndarray.numpy_extension')\ndef cond(pred, then_func, else_func, inputs, name=\"cond\"):\n    \"\"\"Run an if-then-else using user-defined condition and computation\n\n    This operator simulates a if-like branch which chooses to do one of\n    the two customized computations according to the specified condition.\n\n    `pred` is a scalar MXNet NDArray,\n    indicating which branch of computation should be used.\n\n    `then_func` is a user-defined function, used as computation of the then branch.\n    It produces `outputs`, which is a list of NDArrays.\n    The signature of `then_func` should be\n    `then_func() => NDArray or nested List[NDArray]`.\n\n    `else_func` is a user-defined function, used as computation of the else branch.\n    It produces `outputs`, which is a list of NDArrays.\n    The signature of `else_func` should be\n    `else_func() => NDArray or nested List[NDArray]`.\n\n    The `outputs` produces by `then_func` and `else_func` should have the same number\n    of elements, all of which should be in the same shape, of the same dtype and stype.\n\n    This function returns a list of symbols, representing the computation result.\n\n    Parameters\n    ----------\n    pred: a Python function.\n        The branch condition.\n    then_func: a Python function.\n        The computation to be executed if `pred` is true.\n    else_func: a Python function.\n        The computation to be executed if `pred` is false.\n\n    Returns\n    -------\n    outputs: an NDArray or nested lists of NDArrays, representing the result of computation.\n\n    Examples\n    --------\n    >>> a, b = mx.np.array([1]), mx.np.array([2])\n    >>> pred = a * b < 5\n    >>> then_func = lambda: (a + 5) * (b + 5)\n    >>> else_func = lambda: (a - 5) * (b - 5)\n    >>> outputs = mx.npx.cond(pred, then_func, else_func)\n    >>> outputs[0]\n    42.0\n    \"\"\"\n\n    def _create_subgraph(graph_vars, graph_func, subgraph_name):\n        subgraph_name = _get_unique_subgraph_name(subgraph_name)\n        with AttrScope(__subgraph_name__=subgraph_name):\n            # create new variables with the same name,\n            # them feed them to the given func\n            flatten_data, data_fmt = _flatten(graph_vars, \"cond input\")\n            real_data = [ele.copy().detach() if ele is not None else None for ele in flatten_data]\n            data_names = ['data_subgraph{}'.format(i) for i, ele in enumerate(real_data)]\n            symbol_data = [\n                symbol.var(name).as_np_ndarray()\n                for arg, name in zip(real_data, data_names)\n            ]\n            dc.set_variable(real_data, symbol_data)\n            new_graph_vars = _regroup(real_data, data_fmt)\n            if dc.is_deferred_compute():\n                outputs = graph_func(*new_graph_vars)\n                if \"pred\" in subgraph_name:\n                    outputs = outputs.astype(\"int\")\n            else:\n                with ag.pause(), dc.context():\n                    outputs = graph_func(*new_graph_vars)\n                    if \"pred\" in subgraph_name:\n                        outputs = outputs.astype(\"int\")\n            outputs, out_fmt = _flatten(outputs, \"cond outputs\")\n            num_outputs = len(outputs)\n            sym_out = [dc.get_symbol(out_data) for out_data in outputs]\n            dc.clear(outputs)\n            graph = _construct_subgraph(sym_out, [])\n        return graph, num_outputs, out_fmt\n\n    flatten_inputs, _ = _flatten(inputs, \"while loop_vars\")\n\n    def _union_inputs(*graphs):\n        # Given a list of graphs, each whose inputs are either from input_vars or other variables.\n        # 1) calculate a list `inputs`, the union of their inputs.\n        # 2) for each graph, determine in which indices their inputs reside in `inputs`\n        # 3) for each variable in the input of `graph`, find which index it is\n        inputs = []             # List[Symbol], result of 1)\n        locs = []               # List[Tuple(List[Int], List[Int])], a list of tuples,\n                                # where tuples are results of 2) and 3)\n        input_id_to_loc = {}    # Dict[int, int], given id(sym), input_id_to_loc maps it\n                                # to a `loc`, where inputs[loc] = sym\n        for graph in graphs:\n            # some input_vars are inputs to `graph`, some are not\n            name_to_input_syms = {'data_subgraph{}'.format(i): ele for i, ele in enumerate(flatten_inputs)}\n            # collect arguments for each subgraph\n            input_locs = []                         # results from the second step\n            for name in graph.list_inputs():\n                assert name in name_to_input_syms   # it should obviously hold\n                array = name_to_input_syms[name]\n                # do 2), and 1) is implicitly done\n                if id(array) in input_id_to_loc:\n                    loc = input_id_to_loc[id(array)]\n                else:\n                    loc = len(input_id_to_loc)\n                    inputs.append(array)\n                    input_id_to_loc[id(array)] = loc\n                input_locs.append(loc)\n            locs.append(input_locs)\n        return inputs, locs\n    if isinstance(inputs, (list, tuple)):\n        if len(inputs) == 0:\n            raise ValueError(\"inputs should contain at least one element\")\n    else:\n        assert isinstance(inputs, np_ndarray), (\"inputs should be either mxnet.numpy.ndarray\" \\\n            \" or list/tuple of mxnet.numpy.ndarray\")\n        inputs = [inputs]\n    # create graph for `cond_func'\n    cond_g, cond_num_outputs, _ = _create_subgraph(inputs, pred, name + \"_pred\")\n    if cond_num_outputs != 1:\n        raise ValueError(\"pred should always be a single output\")\n    # create graph for `then`\n    then_g, then_num_outputs, then_fmt = _create_subgraph(inputs, then_func, name + \"_then\")\n    # create graph for `else`\n    else_g, else_num_outputs, _ = _create_subgraph(inputs, else_func, name + \"_else\")\n    if then_num_outputs != else_num_outputs:\n        raise ValueError(\"Number of outputs differs between then-branch and else-branch\")\n    # find symbols used in either cond_g or func_g\n    union_inputs, (cond_input_locs, then_input_locs, else_input_locs) = \\\n        _union_inputs(cond_g, then_g, else_g)\n    result = _api_internal.cond(\n        cond_g.handle,\n        then_g.handle,\n        else_g.handle,\n        *union_inputs,\n        cond_input_locs,\n        then_input_locs,\n        else_input_locs,\n        then_num_outputs\n    )\n    if isinstance(result, np_ndarray):\n        ret = [result]\n    else:\n        ret = list(result)\n    outputs = [ret[i] for i in range(then_num_outputs)]\n    outputs = _regroup(outputs, then_fmt)\n    return outputs\n"
  },
  {
    "path": "python/mxnet/ndarray/numpy_extension/image.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Image pre-processing operators.\"\"\"\n\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/ndarray/numpy_extension/random.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for operators used in Gluon dispatched by F=ndarray.\"\"\"\nfrom ...device import current_device\nfrom ..numpy import _internal as _npi\nfrom ...util import wrap_ctx_to_device_func\n\n\n__all__ = ['bernoulli', 'normal_n', 'uniform_n']\n\n\n@wrap_ctx_to_device_func\ndef bernoulli(prob=None, logit=None, size=None, dtype=None, device=None, out=None):\n    \"\"\"Creates a Bernoulli distribution parameterized by :attr:`prob`\n    or :attr:`logit` (but not both).\n\n    Samples are binary (0 or 1). They take the value `1` with probability `p`\n    and `0` with probability `1 - p`.\n\n    Parameters\n    ----------\n    prob : float, ndarray\n        The probability of sampling '1'.\n        Only one of prob or logit should be passed in.\n    logit : float, ndarray\n        The log-odds of sampling '1'.\n        Only one of prob or logit should be passed in.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  Default is None, in which case a\n        single value is returned.\n    dtype : dtype, optional\n        Desired dtype of the result. All dtypes are determined by their\n        name, i.e., 'int64', 'int', etc, so byteorder is not available\n        and a specific precision may have different C types depending\n        on the platform. The default value is 'np.float32'.\n    device : Device, optional\n        Device context of output. Default is current device.\n    out : symbol, optional\n        The output symbol (default is `None`).\n\n    Returns\n    -------\n    out : ndarray\n        Drawn samples from the parameterized bernoulli distribution.\n\n    Examples\n    --------\n    >>> prob = np.random.uniform(size=(4,4))\n    >>> logit = np.log(prob) - np.log(1 - prob)\n    >>> npx.random.bernoulli(logit=logit)\n    array([[0., 1., 1., 1.],\n        [0., 1., 1., 1.],\n        [0., 1., 0., 0.],\n        [1., 0., 1., 0.]])\n\n    >>> npx.random.bernoulli(prob=prob)\n    array([[0., 1., 0., 1.],\n        [1., 1., 1., 1.],\n        [1., 1., 1., 0.],\n        [1., 0., 1., 0.]])\n    \"\"\"\n    from ...numpy import ndarray as np_ndarray\n    tensor_type_name = np_ndarray\n    if (prob is None) == (logit is None):\n        raise ValueError(\n            \"Either `prob` or `logit` must be specified, but not both. \" +\n            \"Received prob={}, logit={}\".format(prob, logit))\n    if dtype is None:\n        dtype = 'float32'\n    if device is None:\n        device = current_device()\n    if size == ():\n        size = None\n    if prob is not None:\n        is_tensor = isinstance(prob, tensor_type_name)\n        if is_tensor:\n            return _npi.bernoulli(prob, prob=None, logit=None, is_logit=False,\n                                  size=size, ctx=device, dtype=dtype, out=out)\n        else:\n            return _npi.bernoulli(prob=prob, logit=None, is_logit=False,\n                                  size=size, ctx=device, dtype=dtype, out=out)\n    else:\n        is_tensor = isinstance(logit, tensor_type_name)\n        if is_tensor:\n            return _npi.bernoulli(logit, prob=None, logit=None, is_logit=True,\n                                  size=size, ctx=device, dtype=dtype, out=out)\n        else:\n            return _npi.bernoulli(prob=None, logit=logit, is_logit=True,\n                                  size=size, ctx=device, dtype=dtype, out=out)\n\n\n@wrap_ctx_to_device_func\ndef uniform_n(low=0.0, high=1.0, batch_shape=None, dtype=None, device=None):\n    r\"\"\"Draw samples from a uniform distribution.\n\n    Samples are uniformly distributed over the half-open interval\n    ``[low, high)`` (includes low, but excludes high).  In other words,\n    any value within the given interval is equally likely to be drawn\n    by `uniform`.\n\n    Parameters\n    ----------\n    low : float, ndarray, optional\n        Lower boundary of the output interval.  All values generated will be\n        greater than or equal to low.  The default value is 0.\n    high : float, ndarray, optional\n        Upper boundary of the output interval.  All values generated will be\n        less than high.  The default value is 1.0.\n    batch_shape : int or tuple of ints, optional\n        Batch shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k * broadcast(low, high).size`` samples are drawn.\n        If size is ``None`` (default),\n        a scalar tensor containing a single value is returned if\n        ``low`` and ``high`` are both scalars. Otherwise,\n        ``np.broadcast(low, high).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    device : Device, optional\n        Device context of output. Default is current device.\n\n    Returns\n    -------\n    out : ndarray\n        Drawn samples from the parameterized uniform distribution.\n\n    See Also\n    --------\n    randint : Discrete uniform distribution, yielding integers.\n    rand : Convenience function that accepts dimensions as input, e.g.,\n           ``rand(2,2)`` would generate a 2-by-2 array of floats,\n           uniformly distributed over ``[0, 1)``.\n\n    Notes\n    -----\n    The probability density function of the uniform distribution is\n\n    .. math:: p(x) = \\frac{1}{b - a}\n\n    anywhere within the interval ``[a, b)``, and zero elsewhere.\n\n    When ``high`` == ``low``, values of ``low`` will be returned.\n    If ``high`` < ``low``, the results are officially undefined\n    and may eventually raise an error, i.e. do not rely on this\n    function to behave when passed arguments satisfying that\n    inequality condition.\n    \"\"\"\n    from ...numpy import ndarray as np_ndarray\n    input_type = (isinstance(low, np_ndarray), isinstance(high, np_ndarray))\n    if dtype is None:\n        dtype = 'float32'\n    if device is None:\n        device = current_device()\n    if batch_shape == ():\n        batch_shape = None\n    else:\n        if isinstance(batch_shape, int):\n            batch_shape = (batch_shape,)\n        batch_shape = (-2,) + batch_shape\n    if input_type == (True, True):\n        return _npi.uniform(low, high, low=None, high=None, size=batch_shape,\n                            ctx=device, dtype=dtype)\n    elif input_type == (False, True):\n        return _npi.uniform(high, low=low, high=None, size=batch_shape,\n                            ctx=device, dtype=dtype)\n    elif input_type == (True, False):\n        return _npi.uniform(low, low=None, high=high, size=batch_shape,\n                            ctx=device, dtype=dtype)\n    else:\n        return _npi.uniform(low=low, high=high, size=batch_shape,\n                            ctx=device, dtype=dtype)\n\n\n@wrap_ctx_to_device_func\ndef normal_n(loc=0.0, scale=1.0, batch_shape=None, dtype=None, device=None):\n    r\"\"\"Draw random samples from a normal (Gaussian) distribution.\n\n    Samples are distributed according to a normal distribution parametrized\n    by *loc* (mean) and *scale* (standard deviation).\n\n\n    Parameters\n    ----------\n    loc : float, optional\n        Mean (centre) of the distribution.\n    scale : float, optional\n        Standard deviation (spread or \"width\") of the distribution.\n    batch_shape : int or tuple of ints, optional\n        Batch shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k * broadcast(low, high).size`` samples are drawn.\n        If size is ``None`` (default),\n        a scalar tensor containing a single value is returned if\n        ``low`` and ``high`` are both scalars. Otherwise,\n        ``np.broadcast(loc, scale).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    device : Device, optional\n        Device context of output, default is current device.\n\n    Returns\n    -------\n    out : ndarray\n        Drawn samples from the parameterized normal distribution.\n\n    Notes\n    -----\n    The probability density for the Gaussian distribution is\n\n    .. math:: p(x) = \\frac{1}{\\sqrt{ 2 \\pi \\sigma^2 }}\n                     e^{ - \\frac{ (x - \\mu)^2 } {2 \\sigma^2} },\n\n    where :math:`\\mu` is the mean and :math:`\\sigma` the standard\n    deviation. The square of the standard deviation, :math:`\\sigma^2`,\n    is called the variance.\n\n    The function has its peak at the mean, and its \"spread\" increases with\n    the standard deviation (the function reaches 0.607 times its maximum at\n    :math:`x + \\sigma` and :math:`x - \\sigma` [2]_).  This implies that\n    `numpy.random.normal` is more likely to return samples lying close to\n    the mean, rather than those far away.\n\n    References\n    ----------\n    .. [1] Wikipedia, \"Normal distribution\",\n           https://en.wikipedia.org/wiki/Normal_distribution\n    .. [2] P. R. Peebles Jr., \"Central Limit Theorem\" in \"Probability,\n           Random Variables and Random Signal Principles\", 4th ed., 2001,\n           pp. 51, 51, 125.\n\n    Examples\n    --------\n    >>> mu, sigma = 0, 0.1 # mean and standard deviation\n    >>> s = np.random.normal(mu, sigma, 1000)\n\n    Verify the mean and the variance:\n\n    >>> np.abs(mu - np.mean(s)) < 0.01\n    array(True)\n    \"\"\"\n    from ...numpy import ndarray as np_ndarray\n    input_type = (isinstance(loc, np_ndarray), isinstance(scale, np_ndarray))\n    if dtype is None:\n        dtype = 'float32'\n    if device is None:\n        device = current_device()\n    if batch_shape == ():\n        batch_shape = None\n    else:\n        if isinstance(batch_shape, int):\n            batch_shape = (batch_shape,)\n        batch_shape = (-2,) + batch_shape\n    if input_type == (True, True):\n        return _npi.normal(loc, scale, loc=None, scale=None, size=batch_shape,\n                           ctx=device, dtype=dtype)\n    elif input_type == (False, True):\n        return _npi.normal(scale, loc=loc, scale=None, size=batch_shape,\n                           ctx=device, dtype=dtype)\n    elif input_type == (True, False):\n        return _npi.normal(loc, loc=None, scale=scale, size=batch_shape,\n                           ctx=device, dtype=dtype)\n    else:\n        return _npi.normal(loc=loc, scale=scale, size=batch_shape,\n                           ctx=device, dtype=dtype)\n"
  },
  {
    "path": "python/mxnet/ndarray/op.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import, unused-wildcard-import, redefined-builtin\n\"\"\"Backend ops in mxnet.ndarray namespace\"\"\"\nfrom ._internal import CachedOp\ntry:\n    from .gen_op import * # pylint: disable=unused-wildcard-import\nexcept ImportError:\n    pass\n\n__all__ = ['CachedOp']\n"
  },
  {
    "path": "python/mxnet/ndarray/random.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Random distribution generator NDArray API of MXNet.\"\"\"\n\nfrom ..base import numeric_types, _Null\nfrom ..context import current_context\nfrom . import _internal\nfrom .ndarray import NDArray\n\n\n__all__ = ['uniform', 'normal', 'randn', 'poisson', 'exponential', 'gamma', 'binomial',\n           'categorical', 'multinomial', 'negative_binomial', 'generalized_negative_binomial',\n           'shuffle', 'randint']\n\n\ndef _random_helper(random, sampler, params, shape, dtype, ctx, out, kwargs):\n    \"\"\"Helper function for random generators.\"\"\"\n    if isinstance(params[0], NDArray):\n        for i in params[1:]:\n            assert isinstance(i, NDArray), \\\n                \"Distribution parameters must all have the same type, but got \" \\\n                f\"both {type(params[0])} and {type(i)}.\"\n        return sampler(*params, shape=shape, dtype=dtype, out=out, **kwargs)\n    elif isinstance(params[0], numeric_types):\n        if ctx is None:\n            ctx = current_context()\n        if shape is _Null and out is None:\n            shape = 1\n        for i in params[1:]:\n            assert isinstance(i, numeric_types), \\\n                \"Distribution parameters must all have the same type, but got \" \\\n                f\"both {type(params[0])} and {type(i)}.\"\n        return random(*params, shape=shape, dtype=dtype, ctx=ctx, out=out, **kwargs)\n\n    raise ValueError(\"Distribution parameters must be either NDArray or numbers, \"\n                     f\"but got {type(params[0])}.\")\n\n\ndef uniform(low=0, high=1, shape=_Null, dtype=_Null, ctx=None, out=None, **kwargs):\n    \"\"\"Draw random samples from a uniform distribution.\n\n    Samples are uniformly distributed over the half-open interval *[low, high)*\n    (includes *low*, but excludes *high*).\n\n    Parameters\n    ----------\n    low : float or NDArray, optional\n        Lower boundary of the output interval. All values generated will be\n        greater than or equal to low. The default value is 0.\n    high : float or NDArray, optional\n        Upper boundary of the output interval. All values generated will be\n        less than high. The default value is 1.0.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `low` and\n        `high` are scalars, output shape will be `(m, n)`. If `low` and `high`\n        are NDArrays with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each `[low, high)` pair.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    ctx : Context, optional\n        Device context of output. Default is current context. Overridden by\n        `low.context` when `low` is an NDArray.\n    out : NDArray, optional\n        Store output to an existing NDArray.\n\n    Returns\n    -------\n    NDArray\n        An NDArray of type `dtype`. If input `shape` has shape, e.g.,\n        `(m, n)` and `low` and `high` are scalars, output shape will be `(m, n)`.\n        If `low` and `high` are NDArrays with shape, e.g., `(x, y)`, then the\n        return NDArray will have shape `(x, y, m, n)`, where `m*n` uniformly distributed\n        samples are drawn for each `[low, high)` pair.\n\n    Examples\n    --------\n    >>> mx.nd.random.uniform(0, 1)\n    [ 0.54881352]\n    <NDArray 1 @cpu(0)\n    >>> mx.nd.random.uniform(0, 1, ctx=mx.gpu(0))\n    [ 0.92514056]\n    <NDArray 1 @gpu(0)>\n    >>> mx.nd.random.uniform(-1, 1, shape=(2,))\n    [ 0.71589124  0.08976638]\n    <NDArray 2 @cpu(0)>\n    >>> low = mx.nd.array([1,2,3])\n    >>> high = mx.nd.array([2,3,4])\n    >>> mx.nd.random.uniform(low, high, shape=2)\n    [[ 1.78653979  1.93707538]\n     [ 2.01311183  2.37081361]\n     [ 3.30491424  3.69977832]]\n    <NDArray 3x2 @cpu(0)>\n    \"\"\"\n    return _random_helper(_internal._random_uniform, _internal._sample_uniform,\n                          [low, high], shape, dtype, ctx, out, kwargs)\n\n\ndef normal(loc=0, scale=1, shape=_Null, dtype=_Null, ctx=None, out=None, **kwargs):\n    \"\"\"Draw random samples from a normal (Gaussian) distribution.\n\n    Samples are distributed according to a normal distribution parametrized\n    by *loc* (mean) and *scale* (standard deviation).\n\n\n    Parameters\n    ----------\n    loc : float or NDArray, optional\n        Mean (centre) of the distribution.\n    scale : float or NDArray, optional\n        Standard deviation (spread or width) of the distribution.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `loc` and\n        `scale` are scalars, output shape will be `(m, n)`. If `loc` and `scale`\n        are NDArrays with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each `[loc, scale)` pair.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    ctx : Context, optional\n        Device context of output. Default is current context. Overridden by\n        `loc.context` when `loc` is an NDArray.\n    out : NDArray, optional\n        Store output to an existing NDArray.\n\n    Returns\n    -------\n    NDArray\n        An NDArray of type `dtype`. If input `shape` has shape, e.g., `(m, n)` and\n        `loc` and `scale` are scalars, output shape will be `(m, n)`. If `loc` and\n        `scale` are NDArrays with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each `[loc, scale)` pair.\n\n    Examples\n    --------\n    >>> mx.nd.random.normal(0, 1)\n    [ 2.21220636]\n    <NDArray 1 @cpu(0)>\n    >>> mx.nd.random.normal(0, 1, ctx=mx.gpu(0))\n    [ 0.29253659]\n    <NDArray 1 @gpu(0)>\n    >>> mx.nd.random.normal(-1, 1, shape=(2,))\n    [-0.2259962  -0.51619542]\n    <NDArray 2 @cpu(0)>\n    >>> loc = mx.nd.array([1,2,3])\n    >>> scale = mx.nd.array([2,3,4])\n    >>> mx.nd.random.normal(loc, scale, shape=2)\n    [[ 0.55912292  3.19566321]\n     [ 1.91728961  2.47706747]\n     [ 2.79666662  5.44254589]]\n    <NDArray 3x2 @cpu(0)>\n    \"\"\"\n    return _random_helper(_internal._random_normal, _internal._sample_normal,\n                          [loc, scale], shape, dtype, ctx, out, kwargs)\n\n\ndef randn(*shape, **kwargs):\n    \"\"\"Draw random samples from a normal (Gaussian) distribution.\n\n    Samples are distributed according to a normal distribution parametrized\n    by *loc* (mean) and *scale* (standard deviation).\n\n\n    Parameters\n    ----------\n    loc : float or NDArray\n        Mean (centre) of the distribution.\n    scale : float or NDArray\n        Standard deviation (spread or width) of the distribution.\n    shape : int or tuple of ints\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `loc` and\n        `scale` are scalars, output shape will be `(m, n)`. If `loc` and `scale`\n        are NDArrays with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each `[loc, scale)` pair.\n    dtype : {'float16', 'float32', 'float64'}\n        Data type of output samples. Default is 'float32'\n    ctx : Context\n        Device context of output. Default is current context. Overridden by\n        `loc.context` when `loc` is an NDArray.\n    out : NDArray\n        Store output to an existing NDArray.\n\n    Returns\n    -------\n    NDArray\n        If input `shape` has shape, e.g., `(m, n)` and `loc` and `scale` are scalars, output\n        shape will be `(m, n)`. If `loc` and `scale` are NDArrays with shape, e.g., `(x, y)`,\n        then output will have shape `(x, y, m, n)`, where `m*n` samples are drawn for\n        each `[loc, scale)` pair.\n\n    Examples\n    --------\n    >>> mx.nd.random.randn()\n    2.21220636\n    <NDArray 1 @cpu(0)>\n    >>> mx.nd.random.randn(2, 2)\n    [[-1.856082   -1.9768796 ]\n    [-0.20801921  0.2444218 ]]\n    <NDArray 2x2 @cpu(0)>\n    >>> mx.nd.random.randn(2, 3, loc=5, scale=1)\n    [[4.19962   4.8311777 5.936328 ]\n    [5.357444  5.7793283 3.9896927]]\n    <NDArray 2x3 @cpu(0)>\n    \"\"\"\n    loc = kwargs.pop('loc', 0)\n    scale = kwargs.pop('scale', 1)\n    dtype = kwargs.pop('dtype', _Null)\n    ctx = kwargs.pop('ctx', None)\n    out = kwargs.pop('out', None)\n    assert isinstance(loc, (int, float, NDArray))\n    assert isinstance(scale, (int, float, NDArray))\n    return _random_helper(_internal._random_normal, _internal._sample_normal,\n                          [loc, scale], shape, dtype, ctx, out, kwargs)\n\n\ndef poisson(lam=1, shape=_Null, dtype=_Null, ctx=None, out=None, **kwargs):\n    \"\"\"Draw random samples from a Poisson distribution.\n\n    Samples are distributed according to a Poisson distribution parametrized\n    by *lambda* (rate). Samples will always be returned as a floating point data type.\n\n    Parameters\n    ----------\n    lam : float or NDArray, optional\n        Expectation of interval, should be >= 0.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `lam` is\n        a scalar, output shape will be `(m, n)`. If `lam`\n        is an NDArray with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each entry in `lam`.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    ctx : Context, optional\n        Device context of output. Default is current context. Overridden by\n        `lam.context` when `lam` is an NDArray.\n    out : NDArray, optional\n        Store output to an existing NDArray.\n\n    Returns\n    -------\n    NDArray\n        If input `shape` has shape, e.g., `(m, n)` and `lam` is\n        a scalar, output shape will be `(m, n)`. If `lam`\n        is an NDArray with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each entry in `lam`.\n\n    Examples\n    --------\n    >>> mx.nd.random.poisson(1)\n    [ 1.]\n    <NDArray 1 @cpu(0)>\n    >>> mx.nd.random.poisson(1, shape=(2,))\n    [ 0.  2.]\n    <NDArray 2 @cpu(0)>\n    >>> lam = mx.nd.array([1,2,3])\n    >>> mx.nd.random.poisson(lam, shape=2)\n    [[ 1.  3.]\n     [ 3.  2.]\n     [ 2.  3.]]\n    <NDArray 3x2 @cpu(0)>\n    \"\"\"\n    return _random_helper(_internal._random_poisson, _internal._sample_poisson,\n                          [lam], shape, dtype, ctx, out, kwargs)\n\n\ndef exponential(scale=1, shape=_Null, dtype=_Null, ctx=None, out=None, **kwargs):\n    r\"\"\"Draw samples from an exponential distribution.\n\n    Its probability density function is\n\n    .. math:: f(x; \\frac{1}{\\beta}) = \\frac{1}{\\beta} \\exp(-\\frac{x}{\\beta}),\n\n    for x > 0 and 0 elsewhere. \\beta is the scale parameter, which is the\n    inverse of the rate parameter \\lambda = 1/\\beta.\n\n    Parameters\n    ----------\n    scale : float or NDArray, optional\n        The scale parameter, \\beta = 1/\\lambda.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `scale` is\n        a scalar, output shape will be `(m, n)`. If `scale`\n        is an NDArray with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each entry in `scale`.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    ctx : Context, optional\n        Device context of output. Default is current context. Overridden by\n        `scale.context` when `scale` is an NDArray.\n    out : NDArray, optional\n        Store output to an existing NDArray.\n\n    Returns\n    -------\n    NDArray\n        If input `shape` has shape, e.g., `(m, n)` and `scale` is a scalar, output shape will\n        be `(m, n)`. If `scale` is an NDArray with shape, e.g., `(x, y)`, then `output`\n        will have shape `(x, y, m, n)`, where `m*n` samples are drawn for each entry in scale.\n\n    Examples\n    --------\n    >>> mx.nd.random.exponential(1)\n    [ 0.79587454]\n    <NDArray 1 @cpu(0)>\n    >>> mx.nd.random.exponential(1, shape=(2,))\n    [ 0.89856035  1.25593066]\n    <NDArray 2 @cpu(0)>\n    >>> scale = mx.nd.array([1,2,3])\n    >>> mx.nd.random.exponential(scale, shape=2)\n    [[  0.41063145   0.42140478]\n     [  2.59407091  10.12439728]\n     [  2.42544937   1.14260709]]\n    <NDArray 3x2 @cpu(0)>\n    \"\"\"\n    return _random_helper(_internal._random_exponential, _internal._sample_exponential,\n                          [1.0/scale], shape, dtype, ctx, out, kwargs)\n\n\ndef gamma(alpha=1, beta=1, shape=_Null, dtype=_Null, ctx=None, out=None, **kwargs):\n    \"\"\"Draw random samples from a gamma distribution.\n\n    Samples are distributed according to a gamma distribution parametrized\n    by *alpha* (shape) and *beta* (scale).\n\n    Parameters\n    ----------\n    alpha : float or NDArray, optional\n        The shape of the gamma distribution. Should be greater than zero.\n    beta : float or NDArray, optional\n        The scale of the gamma distribution. Should be greater than zero.\n        Default is equal to 1.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `alpha` and\n        `beta` are scalars, output shape will be `(m, n)`. If `alpha` and `beta`\n        are NDArrays with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each `[alpha, beta)` pair.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    ctx : Context, optional\n        Device context of output. Default is current context. Overridden by\n        `alpha.context` when `alpha` is an NDArray.\n    out : NDArray, optional\n        Store output to an existing NDArray.\n\n    Returns\n    -------\n    NDArray\n        If input `shape` has shape, e.g., `(m, n)` and `alpha` and `beta` are scalars, output\n        shape will be `(m, n)`. If `alpha` and `beta` are NDArrays with shape, e.g.,\n        `(x, y)`, then output will have shape `(x, y, m, n)`, where `m*n` samples are\n        drawn for each `[alpha, beta)` pair.\n\n    Examples\n    --------\n    >>> mx.nd.random.gamma(1, 1)\n    [ 1.93308783]\n    <NDArray 1 @cpu(0)>\n    >>> mx.nd.random.gamma(1, 1, shape=(2,))\n    [ 0.48216391  2.09890771]\n    <NDArray 2 @cpu(0)>\n    >>> alpha = mx.nd.array([1,2,3])\n    >>> beta = mx.nd.array([2,3,4])\n    >>> mx.nd.random.gamma(alpha, beta, shape=2)\n    [[  3.24343276   0.94137681]\n     [  3.52734375   0.45568955]\n     [ 14.26264095  14.0170126 ]]\n    <NDArray 3x2 @cpu(0)>\n    \"\"\"\n    return _random_helper(_internal._random_gamma, _internal._sample_gamma,\n                          [alpha, beta], shape, dtype, ctx, out, kwargs)\n\n\ndef binomial(n=1, p=0.5, shape=_Null, dtype=_Null, ctx=None, out=None, **kwargs):\n    \"\"\"Draw random samples from a binomial distribution.\n\n    Samples are distributed according to a binomial distribution parametrized\n    by *n* (number of trials) and *p* (success probability).\n\n    Parameters\n    ----------\n    n : float or NDArray, optional\n        Number of experiments, > 0.\n    p : float or NDArray, optional\n        Success probability in each experiment, >= 0 and <= 1.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `n` and\n        `p` are scalars, output shape will be `(m, n)`. If `n` and `p`\n        are NDArrays with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each `[n, p)` pair.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    ctx : Context, optional\n        Device context of output. Default is current context. Overridden by\n        `n.context` when `n` is an NDArray.\n    out : NDArray, optional\n        Store output to an existing NDArray.\n\n    Returns\n    -------\n    NDArray\n        If input `shape` has shape, e.g., `(m, n)` and `n` and `p` are scalars, output\n        shape will be `(m, n)`. If `n` and `p` are NDArrays with shape, e.g.,\n        `(x, y)`, then output will have shape `(x, y, m, n)`, where `m*n` samples are\n        drawn for each `[n, p)` pair.\n\n    Examples\n    --------\n    >>> mx.nd.random.binomial(10, 0.1)\n    [ 1.]\n    <NDArray 1 @cpu(0)>\n    >>> mx.nd.random.binomial(10, 0.6, shape=(2,))\n    [ 4. 6.]\n    <NDArray 2 @cpu(0)>\n    >>> n = mx.nd.array([10,2,3])\n    >>> p = mx.nd.array([0.2,0.3,0.4])\n    >>> mx.nd.random.binomial(n, p, shape=2)\n    [[  1. 4.]\n     [  0. 2.]\n     [  1. 1.]]\n    <NDArray 3x2 @cpu(0)>\n    \"\"\"\n    return _random_helper(_internal._random_binomial, _internal._sample_binomial,\n                          [n, p], shape, dtype, ctx, out, kwargs)\n\n\ndef negative_binomial(k=1, p=1, shape=_Null, dtype=_Null, ctx=None,\n                      out=None, **kwargs):\n    \"\"\"Draw random samples from a negative binomial distribution.\n\n    Samples are distributed according to a negative binomial distribution\n    parametrized by *k* (limit of unsuccessful experiments) and *p* (failure\n    probability in each experiment). Samples will always be returned as a\n    floating point data type.\n\n    Parameters\n    ----------\n    k : float or NDArray, optional\n        Limit of unsuccessful experiments, > 0.\n    p : float or NDArray, optional\n        Failure probability in each experiment, >= 0 and <=1.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `k` and\n        `p` are scalars, output shape will be `(m, n)`. If `k` and `p`\n        are NDArrays with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each `[k, p)` pair.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    ctx : Context, optional\n        Device context of output. Default is current context. Overridden by\n        `k.context` when `k` is an NDArray.\n    out : NDArray, optional\n        Store output to an existing NDArray.\n\n    Returns\n    -------\n    NDArray\n        If input `shape` has shape, e.g., `(m, n)` and `k` and `p` are scalars, output shape\n        will be `(m, n)`. If `k` and `p` are NDArrays with shape, e.g., `(x, y)`, then\n        output will have shape `(x, y, m, n)`, where `m*n` samples are drawn for each `[k, p)` pair.\n\n    Examples\n    --------\n    >>> mx.nd.random.negative_binomial(10, 0.5)\n    [ 4.]\n    <NDArray 1 @cpu(0)>\n    >>> mx.nd.random.negative_binomial(10, 0.5, shape=(2,))\n    [ 3.  4.]\n    <NDArray 2 @cpu(0)>\n    >>> k = mx.nd.array([1,2,3])\n    >>> p = mx.nd.array([0.2,0.4,0.6])\n    >>> mx.nd.random.negative_binomial(k, p, shape=2)\n    [[ 3.  2.]\n     [ 4.  4.]\n     [ 0.  5.]]\n    <NDArray 3x2 @cpu(0)>\n    \"\"\"\n    return _random_helper(_internal._random_negative_binomial,\n                          _internal._sample_negative_binomial,\n                          [k, p], shape, dtype, ctx, out, kwargs)\n\n\ndef generalized_negative_binomial(mu=1, alpha=1, shape=_Null, dtype=_Null, ctx=None,\n                                  out=None, **kwargs):\n    \"\"\"Draw random samples from a generalized negative binomial distribution.\n\n    Samples are distributed according to a generalized negative binomial\n    distribution parametrized by *mu* (mean) and *alpha* (dispersion).\n    *alpha* is defined as *1/k* where *k* is the failure limit of the\n    number of unsuccessful experiments (generalized to real numbers).\n    Samples will always be returned as a floating point data type.\n\n    Parameters\n    ----------\n    mu : float or NDArray, optional\n        Mean of the negative binomial distribution.\n    alpha : float or NDArray, optional\n        Alpha (dispersion) parameter of the negative binomial distribution.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `mu` and\n        `alpha` are scalars, output shape will be `(m, n)`. If `mu` and `alpha`\n        are NDArrays with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each `[mu, alpha)` pair.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    ctx : Context, optional\n        Device context of output. Default is current context. Overridden by\n        `mu.context` when `mu` is an NDArray.\n    out : NDArray, optional\n        Store output to an existing NDArray.\n\n    Returns\n    -------\n    NDArray\n        If input `shape` has shape, e.g., `(m, n)` and `mu` and `alpha` are scalars, output\n        shape will be `(m, n)`. If `mu` and `alpha` are NDArrays with shape, e.g., `(x, y)`,\n        then output will have shape `(x, y, m, n)`, where `m*n` samples are drawn for\n        each `[mu, alpha)` pair.\n\n    Examples\n    --------\n    >>> mx.nd.random.generalized_negative_binomial(10, 0.5)\n    [ 19.]\n    <NDArray 1 @cpu(0)>\n    >>> mx.nd.random.generalized_negative_binomial(10, 0.5, shape=(2,))\n    [ 30.  21.]\n    <NDArray 2 @cpu(0)>\n    >>> mu = mx.nd.array([1,2,3])\n    >>> alpha = mx.nd.array([0.2,0.4,0.6])\n    >>> mx.nd.random.generalized_negative_binomial(mu, alpha, shape=2)\n    [[ 4.  0.]\n     [ 3.  2.]\n     [ 6.  2.]]\n    <NDArray 3x2 @cpu(0)>\n    \"\"\"\n    return _random_helper(_internal._random_generalized_negative_binomial,\n                          _internal._sample_generalized_negative_binomial,\n                          [mu, alpha], shape, dtype, ctx, out, kwargs)\n\ndef categorical(data, shape=_Null, get_prob=False, out=None, dtype='int32', **kwargs):\n    \"\"\"Concurrent sampling from multiple categorical distributions.\n\n    .. note:: The input distribution must be normalized, i.e. `data` must sum to\n              1 along its last dimension.\n\n    Parameters\n    ----------\n    data : NDArray\n        An *n* dimensional array whose last dimension has length `k`, where\n        `k` is the number of possible outcomes of each categorical distribution.\n        For example, data with shape `(m, n, k)` specifies `m*n` categorical\n        distributions each with `k` possible outcomes.\n    shape : int or tuple of ints, optional\n        The number of samples to draw from each distribution. If shape is empty\n        one sample will be drawn from each distribution.\n    get_prob : bool, optional\n        If true, a second array containing log likelihood of the drawn\n        samples will also be returned.\n        This is usually used for reinforcement learning, where you can provide\n        reward as head gradient w.r.t. this array to estimate gradient.\n    out : NDArray, optional\n        Store output to an existing NDArray.\n    dtype : str or numpy.dtype, optional\n        Data type of the sample output array. The default is int32.\n        Note that the data type of the log likelihood array is the same with that of `data`.\n\n    Returns\n    -------\n    List, or NDArray\n        For input `data` with `n` dimensions and shape `(d1, d2, ..., dn-1, k)`, and input\n        `shape` with shape `(s1, s2, ..., sx)`, returns an NDArray with shape\n        `(d1, d2, ... dn-1, s1, s2, ..., sx)`. The `s1, s2, ... sx` dimensions of the\n        returned NDArray consist of 0-indexed values sampled from each respective categorical\n        distribution provided in the `k` dimension of `data`.\n\n        For the case `n`=1, and `x`=1 (one shape dimension), returned NDArray has shape `(s1,)`.\n\n        If `get_prob` is set to True, this function returns a list of format:\n        `[ndarray_output, log_likelihood_output]`, where `log_likelihood_output` is an NDArray of the\n        same shape as the sampled outputs.\n\n    Examples\n    --------\n    >>> probs = mx.nd.array([0, 0.1, 0.2, 0.3, 0.4])\n    >>> mx.nd.random.categorical(probs)\n    [3]\n    <NDArray 1 @cpu(0)>\n    >>> probs = mx.nd.array([[0, 0.1, 0.2, 0.3, 0.4], [0.4, 0.3, 0.2, 0.1, 0]])\n    >>> mx.nd.random.categorical(probs)\n    [3 1]\n    <NDArray 2 @cpu(0)>\n    >>> mx.nd.random.categorical(probs, shape=2)\n    [[4 4]\n     [1 2]]\n    <NDArray 2x2 @cpu(0)>\n    >>> mx.nd.random.categorical(probs, get_prob=True)\n    [3 2]\n    <NDArray 2 @cpu(0)>\n    [-1.20397282 -1.60943794]\n    <NDArray 2 @cpu(0)>\n    \"\"\"\n    return _internal._sample_categorical(data, shape, get_prob, out=out, dtype=dtype, **kwargs)\n\n\ndef multinomial(n=[1], p=[[1.0]], shape=_Null, dtype='float32', ctx=None, out=None, **kwargs):\n    \"\"\"Concurrent sampling from multiple multinomial distributions.\n\n    .. note:: The input distribution must be normalized, i.e. `p` must sum to\n              1 along its last dimension.\n\n    Parameters\n    ----------\n    n : NDArray\n        An *n* dimensional array containing the number of trials of each\n        multinomial distribution.\n    p : NDArray\n        An *n+1* dimensional array containing the probabilities of each multinomial\n        distribution. Its last dimension has length `k`, where `k` is the number\n        of possible outcomes of each multinomial distribution.\n        For example, p with shape `(m, n, k)` specifies `m*n` multinomial\n        distributions each with `k` possible outcomes.\n    shape : int or tuple of ints, optional\n        The number of samples to draw from each distribution. If shape is empty\n        one sample will be drawn from each distribution.\n    out : NDArray, optional\n        Store output to an existing NDArray.\n    ctx : Context, optional\n        Device context of output. Default is current context. Overridden by\n        `n.context` when `n` is an NDArray.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n\n    Returns\n    -------\n    NDArray\n        If input `shape` has shape, e.g., `(m, n)` and `n` and `p` are a scalar and an array of length k\n        respectively, output shape will be `(m, n, k)`. If `n` and `p` are NDArrays with shape, e.g.,\n        `(x, y)` and `(x, y, k)`, then output will have shape `(x, y, m, n, k)`, where `m*n`\n        samples are drawn for each `[n, p)` pair.\n\n    Examples\n    --------\n    >>> mx.nd.random.multinomial(mx.nd.array([10]), mx.nd.array([[0.1, 0.9]]))\n    [[ 1. 9.]]\n    <NDArray 1x2 @cpu(0)>\n    >>> mx.nd.random.multinomial(mx.nd.array([10]), mx.nd.array([[0.6, 0.4]]), shape=(2,))\n    [[[ 5. 5.]\n      [ 6. 4.]]]\n    <NDArray 1x2x2 @cpu(0)>\n    >>> n = mx.nd.array([10, 2, 3])\n    >>> p = mx.nd.array([[0.2, 0.8], [0.3, 0.7], [0.4, 0.6]])\n    >>> mx.nd.random.binomial(n, p)\n    [[  2. 8.]\n     [  1. 1.]\n     [  1. 2.]]\n    <NDArray 3x2 @cpu(0)>\n    \"\"\"\n    return _internal._sample_multinomial(n, p, shape=shape, out=out, ctx=ctx, dtype=dtype, **kwargs)\n\n\ndef shuffle(data, **kwargs):\n    \"\"\"Shuffle the elements randomly.\n\n    This shuffles the array along the first axis.\n    The order of the elements in each subarray does not change.\n    For example, if a 2D array is given, the order of the rows randomly changes,\n    but the order of the elements in each row does not change.\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data array.\n    out : NDArray, optional\n        Array to store the result.\n\n    Returns\n    -------\n    NDArray\n        A new NDArray with the same shape and type as input `data`, but\n        with items in the first axis of the returned NDArray shuffled randomly.\n        The original input `data` is not modified.\n\n    Examples\n    --------\n    >>> data = mx.nd.array([[0, 1, 2], [3, 4, 5], [6, 7, 8]])\n    >>> mx.nd.random.shuffle(data)\n    [[ 0.  1.  2.]\n     [ 6.  7.  8.]\n     [ 3.  4.  5.]]\n    <NDArray 2x3 @cpu(0)>\n    >>> mx.nd.random.shuffle(data)\n    [[ 3.  4.  5.]\n     [ 0.  1.  2.]\n     [ 6.  7.  8.]]\n    <NDArray 2x3 @cpu(0)>\n    \"\"\"\n    return _internal._shuffle(data, **kwargs)\n\n\ndef randint(low, high, shape=_Null, dtype=_Null, ctx=None, out=None, **kwargs):\n    \"\"\"Draw random samples from a discrete uniform distribution.\n\n    Samples are uniformly distributed over the half-open interval *[low, high)*\n    (includes *low*, but excludes *high*).\n\n    Parameters\n    ----------\n    low : int, required\n        Lower boundary of the output interval. All values generated will be\n        greater than or equal to low.\n    high : int, required\n        Upper boundary of the output interval. All values generated will be\n        less than high.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `low` and\n        `high` are scalars, output shape will be `(m, n)`.\n    dtype : {'int32', 'int64'}, optional\n        Data type of output samples. Default is 'int32'\n    ctx : Context, optional\n        Device context of output. Default is current context. Overridden by\n        `low.context` when `low` is an NDArray.\n    out : NDArray, optional\n        Store output to an existing NDArray.\n\n    Returns\n    -------\n    NDArray\n        An NDArray of type `dtype`. If input `shape` has shape, e.g.,\n        `(m, n)`, the returned NDArray will shape will be `(m, n)`. Contents\n        of the returned NDArray will be samples from the interval `[low, high)`.\n\n    Examples\n    --------\n    >>> mx.nd.random.randint(5, 100)\n    [ 90]\n    <NDArray 1 @cpu(0)\n    >>> mx.nd.random.randint(-10, 2, ctx=mx.gpu(0))\n    [ -8]\n    <NDArray 1 @gpu(0)>\n    >>> mx.nd.random.randint(-10, 10, shape=(2,))\n    [ -5  4]\n    <NDArray 2 @cpu(0)>\n    \"\"\"\n    return _random_helper(_internal._random_randint, None,\n                          [low, high], shape, dtype, ctx, out, kwargs)\n"
  },
  {
    "path": "python/mxnet/ndarray/register.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Register backend ops in mxnet.ndarray namespace\"\"\"\nimport os as _os\nimport ctypes\nimport numpy as _np  # pylint: disable=unused-import\n\nfrom .ndarray import get_dtype_name\nfrom ._internal import NDArrayBase, _imperative_invoke # pylint: disable=unused-import\nfrom ..ndarray_doc import _build_doc\n\nfrom ..base import mx_uint, check_call, _LIB, py_str, _init_op_module, _Null, _is_np_op, _output_is_list  # pylint: disable=unused-import\nfrom ..util import use_np_shape  # pylint: disable=unused-import\n\n\ndef _verify_all_np_ndarrays(op_name, func_name, args, out):\n    \"\"\"Verify if all the arrays are numpy ndarrays.\n\n    Parameters\n    ----------\n    op_name : str\n        Operator full name registered in backend.\n    func_name : str\n        Operator name exposed to users. This is usually the name by stripping off\n        the prefix of the full operator names registered in backend.\n    args : list of arrays\n        Input ndarray arguments to be checked.\n    out : ndarray or None or list of ndarrays\n        User-provided output ndarrays.\n    \"\"\"\n    from ..numpy import ndarray as np_ndarray\n    for arr in args:\n        if (arr is not None) and (not isinstance(arr, np_ndarray)):\n            raise TypeError('Operator `{}` registered in backend is known as `{}` in Python. '\n                            'This is a numpy operator which can only accept '\n                            'MXNet numpy ndarrays, while received a legacy ndarray. '\n                            'Please ensure that you have activated numpy semantics by calling '\n                            '`npx.set_np()` in your code. If you still see this error with numpy '\n                            'semantics activated, please call `as_np_ndarray()` upon the legacy '\n                            'ndarray to convert it to an MXNet numpy ndarray, and then feed the '\n                            'converted array to this operator.'\n                            .format(op_name, func_name))\n    if out is None:\n        return\n    if not isinstance(out, (list, tuple)):\n        out = [out]\n    for arr in out:\n        if (arr is not None) and (not isinstance(arr, np_ndarray)):\n            raise TypeError('Operator `{}` registered in backend is known as `{}` in Python. '\n                            'This is a numpy operator which can only accept '\n                            'MXNet numpy ndarrays, while received a legacy ndarray. '\n                            'Please ensure that you have activated numpy semantics by calling '\n                            '`npx.set_np()` in your code. If you still see this error with numpy '\n                            'semantics activated, please call `as_np_ndarray()` upon the legacy '\n                            'ndarray to convert it to an MXNet numpy ndarray, and then feed the '\n                            'converted array to this operator.'\n                            .format(op_name, func_name))\n\n\ndef _verify_all_legacy_ndarrays(op_name, func_name, args, out):\n    \"\"\"Verify if all the arrays are legacy ndarrays.\n\n    Parameters\n    ----------\n    op_name : str\n        Operator full name registered in backend.\n    func_name : str\n        Operator name exposed to users. This is usually the name by stripping off\n        the prefix of the full operator names registered in backend.\n    args : list of arrays\n        Input ndarray arguments to be checked.\n    out : ndarray or None or list of ndarrays\n        User-provided output ndarrays.\n    \"\"\"\n    from ..numpy import ndarray as np_ndarray\n    for arr in args:\n        if (arr is not None) and (isinstance(arr, np_ndarray)):\n            raise TypeError('Operator `{}` registered in backend is known as `{}` in Python. '\n                            'This is a legacy operator which can only accept '\n                            'legacy ndarrays, while received an MXNet numpy ndarray. '\n                            'Please call `as_nd_ndarray()` upon the numpy ndarray to '\n                            'convert it to a legacy ndarray, and then feed the converted '\n                            'array to this operator.'\n                            .format(op_name, func_name))\n    if out is None:\n        return\n    if not isinstance(out, (list, tuple)):\n        out = [out]\n    for arr in out:\n        if (arr is not None) and (isinstance(arr, np_ndarray)):\n            raise TypeError('Operator `{}` registered in backend is known as `{}` in Python. '\n                            'This is a legacy operator which can only write to '\n                            'legacy ndarrays, while received an MXNet numpy ndarray. '\n                            'Please call `as_nd_ndarray()` upon the numpy ndarray to '\n                            'convert it to a legacy ndarray, and then feed the converted '\n                            'array to this operator.'\n                            .format(op_name, func_name))\n\n\n# pylint: disable=too-many-locals\ndef _generate_ndarray_function_code(handle, op_name, func_name, signature_only=False):\n    \"\"\"Generate function for ndarray op by handle and function op_name.\"\"\"\n    real_name = ctypes.c_char_p()\n    desc = ctypes.c_char_p()\n    num_args = mx_uint()\n    arg_names = ctypes.POINTER(ctypes.c_char_p)()\n    arg_types = ctypes.POINTER(ctypes.c_char_p)()\n    arg_descs = ctypes.POINTER(ctypes.c_char_p)()\n    key_var_num_args = ctypes.c_char_p()\n    ret_type = ctypes.c_char_p()\n\n    check_call(_LIB.MXSymbolGetAtomicSymbolInfo(\n        handle, ctypes.byref(real_name), ctypes.byref(desc),\n        ctypes.byref(num_args),\n        ctypes.byref(arg_names),\n        ctypes.byref(arg_types),\n        ctypes.byref(arg_descs),\n        ctypes.byref(key_var_num_args),\n        ctypes.byref(ret_type)))\n    narg = int(num_args.value)\n    arg_names = [py_str(arg_names[i]) for i in range(narg)]\n    arg_types = [py_str(arg_types[i]) for i in range(narg)]\n    key_var_num_args = py_str(key_var_num_args.value)\n    ret_type = py_str(ret_type.value) if ret_type.value is not None else ''\n    doc_str = _build_doc(op_name,\n                         py_str(desc.value),\n                         arg_names,\n                         arg_types,\n                         [py_str(arg_descs[i]) for i in range(narg)],\n                         key_var_num_args,\n                         ret_type)\n\n    dtype_name = None\n    arr_name = None\n    ndsignature = []\n    signature = []\n    ndarg_names = []\n    kwarg_names = []\n    for i in range(narg):\n        name, atype = arg_names[i], arg_types[i]\n        if name == 'dtype':\n            dtype_name = name\n            signature.append(f'{name}=_Null')\n        elif atype.startswith('NDArray') or atype.startswith('Symbol'):\n            assert not arr_name, \\\n                \"Op can only have one argument with variable \" \\\n                \"size and it must be the last argument.\"\n            if atype.endswith('[]'):\n                ndsignature.append(f'*{name}')\n                arr_name = name\n            else:\n                ndsignature.append(f'{name}=None')\n                ndarg_names.append(name)\n        else:\n            signature.append(f'{name}=_Null')\n            kwarg_names.append(name)\n    signature.append('out=None')\n    signature.append('name=None')\n    signature.append('**kwargs')\n    signature = ndsignature + signature\n\n    code = []\n    is_np_op = _is_np_op(op_name)\n    output_is_list = _output_is_list(op_name)\n    doc_str_idx = 1\n    if is_np_op:\n        doc_str_idx = 2\n    if arr_name:\n        code.append(\"\"\"\ndef %s(*%s, **kwargs):\"\"\"%(func_name, arr_name))\n        if not signature_only:\n            code.append(\"\"\"\n    ndargs = []\n    for i in {}:\n        assert isinstance(i, NDArrayBase), \\\\\n            \"Positional arguments must have NDArray type, \" \\\\\n            \"but got %s\"%str(i)\n        ndargs.append(i)\"\"\".format(arr_name))\n            if dtype_name is not None:\n                code.append(\"\"\"\n    if '%s' in kwargs:\n        kwargs['%s'] = get_dtype_name(kwargs['%s'])\"\"\"%(dtype_name, dtype_name, dtype_name))\n            code.append(\"\"\"\n    _ = kwargs.pop('name', None)\n    out = kwargs.pop('out', None)\n    keys = list(kwargs.keys())\n    vals = list(kwargs.values())\"\"\")\n    else:\n        code.append(\"\"\"\ndef %s(%s):\"\"\"%(func_name, ', '.join(signature)))\n        if not signature_only:\n            code.append(\"\"\"\n    ndargs = []\n    keys = list(kwargs.keys())\n    vals = list(kwargs.values())\"\"\")\n            # NDArray args\n            for name in ndarg_names: # pylint: disable=redefined-argument-from-local\n                code.append(\"\"\"\n    if {name} is not None:\n        assert isinstance({name}, NDArrayBase), \\\\\n            \"Argument {name} must have NDArray type, but got %s\"%str({name})\n        ndargs.append({name})\"\"\".format(name=name))\n            # kwargs\n            for name in kwarg_names: # pylint: disable=redefined-argument-from-local\n                code.append(\"\"\"\n    if %s is not _Null:\n        keys.append('%s')\n        vals.append(%s)\"\"\"%(name, name, name))\n            # dtype\n            if dtype_name is not None:\n                if is_np_op:\n                    code.append(\"\"\"\n    if %s is not _Null and %s is not None:\n        keys.append('%s')\n        vals.append(get_dtype_name(%s))\"\"\"%(dtype_name, dtype_name, dtype_name, dtype_name))\n                else:\n                    code.append(\"\"\"\n    if %s is not _Null:\n        keys.append('%s')\n        vals.append(get_dtype_name(%s))\"\"\"%(dtype_name, dtype_name, dtype_name))\n\n    verify_ndarrays_fn =\\\n        _verify_all_np_ndarrays.__name__ if is_np_op else _verify_all_legacy_ndarrays.__name__\n    if not signature_only:\n        code.append(\"\"\"\n    {verify_fn}(\"{op_name}\", \"{func_name}\", ndargs, out)\n        \"\"\".format(verify_fn=verify_ndarrays_fn, op_name=op_name, func_name=func_name))\n        code.append(\"\"\"\n    return _imperative_invoke(%d, ndargs, keys, vals, out, %s, %s)\"\"\"%(\n        handle.value, str(is_np_op), str(output_is_list)))\n    else:\n        code.append(\"\"\"\n    return (0,)\"\"\")\n\n    doc_str_lines = _os.linesep+''.join(['    '+s if s.strip() else s\n                                         for s in 'r\"\"\"{doc_str}\"\"\"'.format(doc_str=doc_str)\n                                         .splitlines(True)])\n    code.insert(doc_str_idx, doc_str_lines)\n    return ''.join(code), doc_str\n\n\n# pylint: disable=too-many-locals, invalid-name\ndef _make_ndarray_function(handle, name, func_name):\n    \"\"\"Create a NDArray function from the FunctionHandle.\"\"\"\n    code, doc_str = _generate_ndarray_function_code(handle, name, func_name)\n\n    local = {}\n    exec(code, None, local)  # pylint: disable=exec-used\n    ndarray_function = local[func_name]\n    ndarray_function.__name__ = func_name\n    ndarray_function.__doc__ = doc_str\n    ndarray_function.__module__ = 'mxnet.ndarray'\n    return ndarray_function\n\n_init_op_module('mxnet', 'ndarray', _make_ndarray_function)\n\n# Update operator documentation with added float support\n# Note that we can only do this after the op module is initialized\n# Otherwise the backend operators cannot be found\n# pylint: disable=wrong-import-position\nfrom .contrib import adamw_update, mp_adamw_update\nfrom ._internal import _adamw_update, _mp_adamw_update\nadamw_update.__doc__ = _adamw_update.__doc__.replace(\"rescale_grad : NDArray\",\n                                                     \"rescale_grad : NDArray or float\")\nmp_adamw_update.__doc__ = _mp_adamw_update.__doc__.replace(\"rescale_grad : NDArray\",\n                                                           \"rescale_grad : NDArray or float\")\n"
  },
  {
    "path": "python/mxnet/ndarray/sparse.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import, unused-wildcard-import, too-many-lines\n\"\"\"Sparse NDArray API of MXNet.\"\"\"\n\ntry:\n    from __builtin__ import slice as py_slice\n    from __builtin__ import sum as py_sum\nexcept ImportError:\n    from builtins import slice as py_slice\n    from builtins import sum as py_sum\n\nimport ctypes\nimport warnings\nimport operator\nfrom array import array as native_array\n\n__all__ = [\"_ndarray_cls\", \"csr_matrix\", \"row_sparse_array\",\n           \"BaseSparseNDArray\", \"CSRNDArray\", \"RowSparseNDArray\",\n           \"add\", \"subtract\", \"multiply\", \"divide\"]\n\nimport numpy as np\nfrom ..base import NotSupportedForSparseNDArray\nfrom ..base import _LIB, numeric_types\nfrom ..base import c_array_buf, mx_real_t, integer_types\nfrom ..base import NDArrayHandle, check_call\nfrom ..device import Device, current_device\nfrom . import _internal\nfrom . import op\ntry:\n    from .gen_sparse import retain as gs_retain # pylint: disable=redefined-builtin\nexcept ImportError:\n    gs_retain = None\nfrom ._internal import _set_ndarray_class\nfrom .ndarray import NDArray, _storage_type, dtype_np_to_mx, dtype_mx_to_np\nfrom .ndarray import _STORAGE_TYPE_STR_TO_ID, _STORAGE_TYPE_ROW_SPARSE, _STORAGE_TYPE_CSR, _int64_enabled\nfrom .ndarray import _STORAGE_TYPE_UNDEFINED, _STORAGE_TYPE_DEFAULT\nfrom .ndarray import zeros as _zeros_ndarray\nfrom .ndarray import array as _array\nfrom .ndarray import _ufunc_helper\n\n\ntry:\n    import scipy.sparse as spsp\nexcept ImportError:\n    spsp = None\n\n_STORAGE_AUX_TYPES = {\n    'row_sparse': [np.int64],\n    'csr': [np.int64, np.int64]\n}\n\n\ndef _new_alloc_handle(stype, shape, ctx, delay_alloc, dtype, aux_types, aux_shapes=None):\n    \"\"\"Return a new handle with specified storage type, shape, dtype and context.\n\n    Empty handle is only used to hold results\n\n    Returns\n    -------\n    handle\n        A new empty ndarray handle\n    \"\"\"\n    hdl = NDArrayHandle()\n    for aux_t in aux_types:\n        if np.dtype(aux_t) != np.dtype(\"int64\"):\n            raise NotImplementedError(\"only int64 is supported for aux types\")\n    aux_type_ids = [int(dtype_np_to_mx(aux_t)) for aux_t in aux_types]\n    aux_shapes = [(0,) for aux_t in aux_types] if aux_shapes is None else aux_shapes\n    aux_shape_lens = [len(aux_shape) for aux_shape in aux_shapes]\n    aux_shapes = py_sum(aux_shapes, ())\n    num_aux = ctypes.c_uint(len(aux_types))\n    if _int64_enabled():\n        check_call(_LIB.MXNDArrayCreateSparseEx64(\n            ctypes.c_int(int(_STORAGE_TYPE_STR_TO_ID[stype])),\n            c_array_buf(ctypes.c_int64, native_array('q', shape)),\n            ctypes.c_int(len(shape)),\n            ctypes.c_int(ctx.device_typeid),\n            ctypes.c_int(ctx.device_id),\n            ctypes.c_int(int(delay_alloc)),\n            ctypes.c_int(int(dtype_np_to_mx(dtype))),\n            num_aux,\n            c_array_buf(ctypes.c_int, native_array('i', aux_type_ids)),\n            c_array_buf(ctypes.c_int, native_array('i', aux_shape_lens)),\n            c_array_buf(ctypes.c_int64, native_array('q', aux_shapes)),\n            ctypes.byref(hdl)))\n    else:\n        check_call(_LIB.MXNDArrayCreateSparseEx(\n            ctypes.c_int(int(_STORAGE_TYPE_STR_TO_ID[stype])),\n            c_array_buf(ctypes.c_uint, native_array('I', shape)),\n            ctypes.c_uint(len(shape)),\n            ctypes.c_int(ctx.device_typeid),\n            ctypes.c_int(ctx.device_id),\n            ctypes.c_int(int(delay_alloc)),\n            ctypes.c_int(int(dtype_np_to_mx(dtype))),\n            num_aux,\n            c_array_buf(ctypes.c_int, native_array('i', aux_type_ids)),\n            c_array_buf(ctypes.c_uint, native_array('I', aux_shape_lens)),\n            c_array_buf(ctypes.c_uint, native_array('I', aux_shapes)),\n            ctypes.byref(hdl)))\n    return hdl\n\n\nclass BaseSparseNDArray(NDArray):\n    \"\"\"The base class of an NDArray stored in a sparse storage format.\n\n    See CSRNDArray and RowSparseNDArray for more details.\n    \"\"\"\n\n    def __repr__(self):\n        \"\"\"Returns a string representation of the sparse array.\"\"\"\n        shape_info = 'x'.join([f'{x}' for x in self.shape])\n        # The data content is not displayed since the array usually has big shape\n        return f'\\n<{self.__class__.__name__} {shape_info} @{self.context}>'\n\n    def __add__(self, other):\n        return add(self, other)\n\n    def __sub__(self, other):\n        return subtract(self, other)\n\n    def __mul__(self, other):\n        return multiply(self, other)\n\n    def __div__(self, other):\n        return divide(self, other)\n\n    def __iadd__(self, other):\n        raise NotImplementedError()\n\n    def __isub__(self, other):\n        raise NotImplementedError()\n\n    def __imul__(self, other):\n        raise NotImplementedError()\n\n    def __idiv__(self, other):\n        raise NotImplementedError()\n\n    def __itruediv__(self, other):\n        raise NotImplementedError()\n\n    def _sync_copyfrom(self, source_array):\n        raise NotImplementedError()\n\n    def _at(self, idx):\n        raise NotSupportedForSparseNDArray(self._at, '[idx]', idx)\n\n    def _slice(self, start, stop):\n        raise NotSupportedForSparseNDArray(self._slice, None, start, stop)\n\n    def reshape(self, *shape, **kwargs):\n        raise NotSupportedForSparseNDArray(self.reshape, None, shape)\n\n    @property\n    def size(self):\n        # the `size` for a sparse ndarray is ambiguous, hence disabled.\n        raise NotImplementedError()\n\n    def _aux_type(self, i):\n        \"\"\"Data-type of the array's ith aux data.\n\n        Returns\n        -------\n        numpy.dtype\n            This BaseSparseNDArray's aux data type.\n        \"\"\"\n        aux_type = ctypes.c_int()\n        check_call(_LIB.MXNDArrayGetAuxType(self.handle, i, ctypes.byref(aux_type)))\n        return dtype_mx_to_np(aux_type.value)\n\n    @property\n    def _num_aux(self):\n        \"\"\"The number of aux data used to help store the sparse ndarray.\n        \"\"\"\n        return len(_STORAGE_AUX_TYPES[self.stype])\n\n    @property\n    def _aux_types(self):\n        \"\"\"The data types of the aux data for the BaseSparseNDArray.\n        \"\"\"\n        aux_types = []\n        num_aux = self._num_aux\n        for i in range(num_aux):\n            aux_types.append(self._aux_type(i))\n        return aux_types\n\n    def asnumpy(self):\n        \"\"\"Return a dense ``numpy.ndarray`` object with value copied from this array\n        \"\"\"\n        return self.tostype('default').asnumpy()\n\n    def astype(self, dtype, copy=True):\n        \"\"\"Return a copy of the array after casting to a specified type.\n\n        Parameters\n        ----------\n        dtype : numpy.dtype or str\n            The type of the returned array.\n        copy : bool\n            Default `True`. By default, astype always returns a newly\n            allocated ndarray on the same context. If this is set to\n            `False`, and the dtype requested is the same as the ndarray's\n            dtype, the ndarray is returned instead of a copy.\n\n        Examples\n        --------\n        >>> x = mx.nd.sparse.zeros('row_sparse', (2,3), dtype='float32')\n        >>> y = x.astype('int32')\n        >>> y.dtype\n        <type 'numpy.int32'>\n        \"\"\"\n        if not copy and np.dtype(dtype) == self.dtype:\n            return self\n\n        # Use copyto for casting, as op.cast(self, dtype=dtype) doesn't support sparse stype\n        res = zeros(shape=self.shape, ctx=self.context,\n                    dtype=dtype, stype=self.stype)\n        self.copyto(res)\n        return res\n\n    def copyto(self, other):\n        \"\"\"Copies the value of this array to another array.\n\n        Parameters\n        ----------\n        other : NDArray or CSRNDArray or RowSparseNDArray or Context\n            The destination array or context.\n\n        Returns\n        -------\n        NDArray or CSRNDArray or RowSparseNDArray\n            The copied array.\n        \"\"\"\n        # pylint: disable= no-member, protected-access\n        if isinstance(other, NDArray):\n            if other.handle is self.handle:\n                warnings.warn('You are attempting to copy an array to itself', RuntimeWarning)\n                return False\n            return _internal._copyto(self, out=other)\n        elif isinstance(other, Device):\n            hret = _ndarray_cls(_new_alloc_handle(self.stype, self.shape, other,\n                                                  True, self.dtype, self._aux_types))\n            return _internal._copyto(self, out=hret)\n        else:\n            raise TypeError('copyto does not support type ' + str(type(other)))\n        # pylint: enable= no-member, protected-access\n\n    def check_format(self, full_check=True):\n        \"\"\"Check whether the NDArray format is valid.\n\n        Parameters\n        ----------\n        full_check : bool, optional\n            If `True`, rigorous check, O(N) operations. Otherwise\n            basic check, O(1) operations (default True).\n        \"\"\"\n        check_call(_LIB.MXNDArraySyncCheckFormat(self.handle, ctypes.c_bool(full_check)))\n\n    def _data(self):\n        \"\"\"A deep copy NDArray of the data array associated with the BaseSparseNDArray.\n\n        This function blocks. Do not use it in performance critical code.\n        \"\"\"\n        self.wait_to_read()\n        hdl = NDArrayHandle()\n        check_call(_LIB.MXNDArrayGetDataNDArray(self.handle, ctypes.byref(hdl)))\n        return NDArray(hdl)\n\n\n    def _aux_data(self, i):\n        \"\"\" Get a deep copy NDArray of the i-th aux data array associated with the\n        BaseSparseNDArray.\n\n        This function blocks. Do not use it in performance critical code.\n        \"\"\"\n        self.wait_to_read()\n        hdl = NDArrayHandle()\n        check_call(_LIB.MXNDArrayGetAuxNDArray(self.handle, i, ctypes.byref(hdl)))\n        return NDArray(hdl)\n\n\n# pylint: disable=abstract-method\nclass CSRNDArray(BaseSparseNDArray):\n    \"\"\"A sparse representation of 2D NDArray in the Compressed Sparse Row format.\n\n    A CSRNDArray represents an NDArray as three separate arrays: `data`,\n    `indptr` and `indices`. It uses the CSR representation where the column indices for\n    row i are stored in ``indices[indptr[i]:indptr[i+1]]`` and their corresponding values are stored\n    in ``data[indptr[i]:indptr[i+1]]``.\n\n    The column indices for a given row are expected to be sorted in ascending order.\n    Duplicate column entries for the same row are not allowed.\n\n    Example\n    -------\n    >>> a = mx.nd.array([[0, 1, 0], [2, 0, 0], [0, 0, 0], [0, 0, 3]])\n    >>> a = a.tostype('csr')\n    >>> a.data.asnumpy()\n    array([ 1.,  2.,  3.], dtype=float32)\n    >>> a.indices.asnumpy()\n    array([1, 0, 2])\n    >>> a.indptr.asnumpy()\n    array([0, 1, 2, 2, 3])\n\n    See Also\n    --------\n    csr_matrix: Several ways to construct a CSRNDArray\n    \"\"\"\n\n    def __reduce__(self):\n        return CSRNDArray, (None,), super(CSRNDArray, self).__getstate__()\n\n    def __iadd__(self, other):\n        (self + other).copyto(self)\n        return self\n\n    def __isub__(self, other):\n        (self - other).copyto(self)\n        return self\n\n    def __imul__(self, other):\n        (self * other).copyto(self)\n        return self\n\n    def __idiv__(self, other):\n        (self / other).copyto(self)\n        return self\n\n    def __itruediv__(self, other):\n        (self / other).copyto(self)\n        return self\n\n    def __getitem__(self, key):\n        \"\"\"x.__getitem__(i) <=> x[i]\n\n        Returns a newly created NDArray based on the indexing key.\n\n        Parameters\n        ----------\n        key : int or mxnet.ndarray.NDArray.slice\n            Indexing key.\n\n        Examples\n        --------\n        >>> indptr = np.array([0, 2, 3, 6])\n        >>> indices = np.array([0, 2, 2, 0, 1, 2])\n        >>> data = np.array([1, 2, 3, 4, 5, 6])\n        >>> a = mx.nd.sparse.csr_matrix((data, indices, indptr), shape=(3, 3))\n        >>> a.asnumpy()\n        array([[ 1.,  0.,  2.],\n               [ 0.,  0.,  3.],\n               [ 4.,  5.,  6.]], dtype=float32)\n        >>> a[1:2].asnumpy()\n        array([[ 0.,  0.,  3.]], dtype=float32)\n        >>> a[1].asnumpy()\n        array([[ 0.,  0.,  3.]], dtype=float32)\n        >>> a[-1].asnumpy()\n        array([[ 4.,  5.,  6.]], dtype=float32)\n        \"\"\"\n        # pylint: disable= no-member, protected-access\n        if isinstance(key, int):\n            if key == -1:\n                begin = self.shape[0] - 1\n            else:\n                begin = key\n            return op.slice(self, begin=begin, end=begin+1)\n        if isinstance(key, py_slice):\n            if key.step is not None:\n                raise ValueError('CSRNDArray only supports continuous slicing on axis 0')\n            if key.start is not None or key.stop is not None:\n                begin = key.start if key.start else 0\n                end = key.stop if key.stop else self.shape[0]\n                return op.slice(self, begin=begin, end=end)\n            else:\n                return self\n        if isinstance(key, tuple):\n            raise ValueError('Multi-dimension indexing is not supported')\n        raise ValueError('Undefined behaviour for {}'.format(key))\n        # pylint: enable= no-member, protected-access\n\n    def __setitem__(self, key, value):\n        \"\"\"x.__setitem__(i, y) <=> x[i]=y\n\n        Set self[key] to value. Only slice key [:] is supported.\n\n        Parameters\n        ----------\n        key : mxnet.ndarray.NDArray.slice\n            The indexing key.\n        value : NDArray or CSRNDArray or numpy.ndarray\n            The value to set.\n\n        Examples\n        --------\n        >>> src = mx.nd.sparse.zeros('csr', (3,3))\n        >>> src.asnumpy()\n        array([[ 0.,  0.,  0.],\n               [ 0.,  0.,  0.],\n               [ 0.,  0.,  0.]], dtype=float32)\n        >>> # assign CSRNDArray with same storage type\n        >>> x = mx.nd.ones((3,3)).tostype('csr')\n        >>> x[:] = src\n        >>> x.asnumpy()\n        array([[ 1.,  1.,  1.],\n               [ 1.,  1.,  1.],\n               [ 1.,  1.,  1.]], dtype=float32)\n        >>> # assign NDArray to CSRNDArray\n        >>> x[:] = mx.nd.ones((3,3)) * 2\n        >>> x.asnumpy()\n        array([[ 2.,  2.,  2.],\n               [ 2.,  2.,  2.],\n               [ 2.,  2.,  2.]], dtype=float32)\n        \"\"\"\n        if not self.writable:\n            raise ValueError('Failed to assign to a readonly CSRNDArray')\n        if isinstance(key, py_slice):\n            if key.step is not None or key.start is not None or key.stop is not None:\n                raise ValueError('Assignment with slice for CSRNDArray is not ' \\\n                                 'implemented yet.')\n            if isinstance(value, NDArray):\n                # avoid copying to itself\n                if value.handle is not self.handle:\n                    value.copyto(self)\n            elif isinstance(value, numeric_types):\n                raise ValueError(\"Assigning numeric types to CSRNDArray is \" \\\n                                 \"not implemented yet.\")\n            elif isinstance(value, (np.ndarray, np.generic)):\n                # TODO(haibin/anisub) check scipy.sparse and use _sync_copy_from to\n                # avoid the temporary copy\n                warnings.warn('Assigning non-NDArray object to CSRNDArray is not efficient',\n                              RuntimeWarning)\n                tmp = _array(value)\n                tmp.copyto(self)\n            else:\n                raise TypeError(f'type {str(type(value))} not supported')\n        else:\n            assert(isinstance(key, (int, tuple)))\n            raise Exception('CSRNDArray only supports [:] for assignment')\n\n    @property\n    def indices(self):\n        \"\"\"A deep copy NDArray of the indices array of the CSRNDArray.\n        This generates a deep copy of the column indices of the current `csr` matrix.\n\n        Returns\n        -------\n        NDArray\n            This CSRNDArray's indices array.\n        \"\"\"\n        return self._aux_data(1)\n\n    @property\n    def indptr(self):\n        \"\"\"A deep copy NDArray of the indptr array of the CSRNDArray.\n        This generates a deep copy of the `indptr` of the current `csr` matrix.\n\n        Returns\n        -------\n        NDArray\n            This CSRNDArray's indptr array.\n        \"\"\"\n        return self._aux_data(0)\n\n    @property\n    def data(self):\n        \"\"\"A deep copy NDArray of the data array of the CSRNDArray.\n        This generates a deep copy of the `data` of the current `csr` matrix.\n\n        Returns\n        -------\n        NDArray\n            This CSRNDArray's data array.\n        \"\"\"\n        return self._data()\n\n    @indices.setter\n    def indices(self, indices):\n        raise NotImplementedError()\n\n    @indptr.setter\n    def indptr(self, indptr):\n        raise NotImplementedError()\n\n    @data.setter\n    def data(self, data):\n        raise NotImplementedError()\n\n\n    def tostype(self, stype):\n        \"\"\"Return a copy of the array with chosen storage type.\n\n        Returns\n        -------\n        NDArray or CSRNDArray\n            A copy of the array with the chosen storage stype\n        \"\"\"\n        # pylint: disable= no-member, protected-access\n        if stype == 'row_sparse':\n            raise ValueError(\"cast_storage from csr to row_sparse is not supported\")\n        return op.cast_storage(self, stype=stype)\n        # pylint: enable= no-member, protected-access\n\n    def copyto(self, other):\n        \"\"\"Copies the value of this array to another array.\n\n        If ``other`` is a ``NDArray`` or ``CSRNDArray`` object, then ``other.shape`` and\n        ``self.shape`` should be the same. This function copies the value from\n        ``self`` to ``other``.\n\n        If ``other`` is a context, a new ``CSRNDArray`` will be first created on\n        the target context, and the value of ``self`` is copied.\n\n        Parameters\n        ----------\n        other : NDArray or CSRNDArray or Context\n            The destination array or context.\n\n        Returns\n        -------\n        NDArray or CSRNDArray\n            The copied array. If ``other`` is an ``NDArray`` or ``CSRNDArray``, then the return\n            value and ``other`` will point to the same ``NDArray`` or ``CSRNDArray``.\n        \"\"\"\n        if isinstance(other, Device):\n            return super(CSRNDArray, self).copyto(other)\n        elif isinstance(other, NDArray):\n            stype = other.stype\n            if stype in ('default', 'csr'):\n                return super(CSRNDArray, self).copyto(other)\n            else:\n                raise TypeError('copyto does not support destination NDArray stype ' + str(stype))\n        else:\n            raise TypeError('copyto does not support type ' + str(type(other)))\n\n    def asscipy(self):\n        \"\"\"Returns a ``scipy.sparse.csr.csr_matrix`` object with value copied from this array\n\n        Examples\n        --------\n        >>> x = mx.nd.sparse.zeros('csr', (2,3))\n        >>> y = x.asscipy()\n        >>> type(y)\n        <type 'scipy.sparse.csr.csr_matrix'>\n        >>> y\n        <2x3 sparse matrix of type '<type 'numpy.float32'>'\n        with 0 stored elements in Compressed Sparse Row format>\n        \"\"\"\n        data = self.data.asnumpy()\n        indices = self.indices.asnumpy()\n        indptr = self.indptr.asnumpy()\n        if not spsp:\n            raise ImportError(\"scipy could not be imported. \"\n                              \"Please make sure that the scipy is installed.\")\n        return spsp.csr_matrix((data, indices, indptr), shape=self.shape, dtype=self.dtype)\n\n# pylint: disable=abstract-method\nclass RowSparseNDArray(BaseSparseNDArray):\n    \"\"\"A sparse representation of a set of NDArray row slices at given indices.\n\n    A RowSparseNDArray represents a multidimensional NDArray using two separate arrays: `data` and\n    `indices`. The number of dimensions has to be at least 2.\n\n    - data: an NDArray of any dtype with shape [D0, D1, ..., Dn].\n    - indices: a 1-D int64 NDArray with shape [D0] with values sorted in ascending order.\n\n    The `indices` stores the indices of the row slices with non-zeros,\n    while the values are stored in `data`. The corresponding NDArray ``dense``\n    represented by RowSparseNDArray ``rsp`` has\n\n    ``dense[rsp.indices[i], :, :, :, ...] = rsp.data[i, :, :, :, ...]``\n\n        >>> dense.asnumpy()\n        array([[ 1.,  2., 3.],\n               [ 0.,  0., 0.],\n               [ 4.,  0., 5.],\n               [ 0.,  0., 0.],\n               [ 0.,  0., 0.]], dtype=float32)\n        >>> rsp = dense.tostype('row_sparse')\n        >>> rsp.indices.asnumpy()\n        array([0, 2], dtype=int64)\n        >>> rsp.data.asnumpy()\n        array([[ 1.,  2., 3.],\n               [ 4.,  0., 5.]], dtype=float32)\n\n    A RowSparseNDArray is typically used to represent non-zero row slices of a large NDArray\n    of shape [LARGE0, D1, .. , Dn] where LARGE0 >> D0 and most row slices are zeros.\n\n    RowSparseNDArray is used principally in the definition of gradients for operations\n    that have sparse gradients (e.g. sparse dot and sparse embedding).\n\n    See Also\n    --------\n    row_sparse_array: Several ways to construct a RowSparseNDArray\n    \"\"\"\n    def __reduce__(self):\n        return RowSparseNDArray, (None,), super(RowSparseNDArray, self).__getstate__()\n\n    def __iadd__(self, other):\n        (self + other).copyto(self)\n        return self\n\n    def __isub__(self, other):\n        (self - other).copyto(self)\n        return self\n\n    def __imul__(self, other):\n        (self * other).copyto(self)\n        return self\n\n    def __idiv__(self, other):\n        (self / other).copyto(self)\n        return self\n\n    def __itruediv__(self, other):\n        (self / other).copyto(self)\n        return self\n\n    def __getitem__(self, key):\n        \"\"\"x.__getitem__(i) <=> x[i]\n\n        Returns a sliced view of this array.\n\n        Parameters\n        ----------\n        key : mxnet.ndarray.NDArray.slice\n            Indexing key.\n\n        Examples\n        --------\n        >>> x = mx.nd.sparse.zeros('row_sparse', (2, 3))\n        >>> x[:].asnumpy()\n        array([[ 0.,  0.,  0.],\n               [ 0.,  0.,  0.]], dtype=float32)\n        \"\"\"\n        if isinstance(key, int):\n            raise Exception(\"__getitem__ with int key is not implemented for RowSparseNDArray yet\")\n        if isinstance(key, py_slice):\n            if key.step is not None or key.start is not None or key.stop is not None:\n                raise Exception('RowSparseNDArray only supports [:] for __getitem__')\n\n            return self\n        if isinstance(key, tuple):\n            raise ValueError('Multi-dimension indexing is not supported')\n        raise ValueError('Undefined behaviour for {}'.format(key))\n\n    def __setitem__(self, key, value):\n        \"\"\"x.__setitem__(i, y) <=> x[i]=y\n\n        Set self[key] to value. Only slice key [:] is supported.\n\n        Parameters\n        ----------\n        key : mxnet.ndarray.NDArray.slice\n            The indexing key.\n        value : NDArray or numpy.ndarray\n            The value to set.\n\n        Examples\n        --------\n        >>> src = mx.nd.row_sparse([[1, 0, 2], [4, 5, 6]], [0, 2], (3,3))\n        >>> src.asnumpy()\n        array([[ 1.,  0.,  2.],\n               [ 0.,  0.,  0.],\n               [ 4.,  5.,  6.]], dtype=float32)\n        >>> # assign RowSparseNDArray with same storage type\n        >>> x = mx.nd.sparse.zeros('row_sparse', (3,3))\n        >>> x[:] = src\n        >>> x.asnumpy()\n        array([[ 1.,  0.,  2.],\n               [ 0.,  0.,  0.],\n               [ 4.,  5.,  6.]], dtype=float32)\n        >>> # assign NDArray to RowSparseNDArray\n        >>> x[:] = mx.nd.ones((3,3))\n        >>> x.asnumpy()\n        array([[ 1.,  1.,  1.],\n               [ 1.,  1.,  1.],\n               [ 1.,  1.,  1.]], dtype=float32)\n        \"\"\"\n        # pylint: disable= no-member, protected-access\n        if not self.writable:\n            raise ValueError('Failed to assign to a readonly RowSparseNDArray')\n        if isinstance(key, py_slice):\n            if key.step is not None or key.start is not None or key.stop is not None:\n                raise ValueError('Assignment with slice for RowSparseNDArray ' \\\n                                 'is not implmented yet.')\n            if isinstance(value, NDArray):\n                # avoid copying to itself\n                if value.handle is not self.handle:\n                    value.copyto(self)\n            elif isinstance(value, numeric_types):\n                _internal._set_value(float(value), out=self)\n            elif isinstance(value, (np.ndarray, np.generic)):\n                warnings.warn('Assigning non-NDArray object to RowSparseNDArray is not efficient',\n                              RuntimeWarning)\n                tmp = _array(value)\n                tmp.copyto(self)\n            else:\n                raise TypeError(f'type {str(type(value))} not supported')\n        else:\n            assert(isinstance(key, (int, tuple)))\n            raise TypeError('RowSparseNDArray only supports [:] for assignment')\n        # pylint: enable= no-member, protected-access\n\n    @property\n    def indices(self):\n        \"\"\"A deep copy NDArray of the indices array of the RowSparseNDArray.\n        This generates a deep copy of the row indices of the current `row_sparse` matrix.\n\n        Returns\n        -------\n        NDArray\n            This RowSparseNDArray's indices array.\n        \"\"\"\n        return self._aux_data(0)\n\n    @property\n    def data(self):\n        \"\"\"A deep copy NDArray of the data array of the RowSparseNDArray.\n        This generates a deep copy of the `data` of the current `row_sparse` matrix.\n\n        Returns\n        -------\n        NDArray\n            This RowSparseNDArray's data array.\n        \"\"\"\n        return self._data()\n\n    @indices.setter\n    def indices(self, indices):\n        raise NotImplementedError()\n\n    @data.setter\n    def data(self, data):\n        raise NotImplementedError()\n\n    def tostype(self, stype):\n        \"\"\"Return a copy of the array with chosen storage type.\n\n        Returns\n        -------\n        NDArray or RowSparseNDArray\n            A copy of the array with the chosen storage stype\n        \"\"\"\n        # pylint: disable= no-member, protected-access\n        if stype == 'csr':\n            raise ValueError(\"cast_storage from row_sparse to csr is not supported\")\n        return op.cast_storage(self, stype=stype)\n        # pylint: enable= no-member, protected-access\n\n    def copyto(self, other):\n        \"\"\"Copies the value of this array to another array.\n\n        If ``other`` is a ``NDArray`` or ``RowSparseNDArray`` object, then ``other.shape``\n        and ``self.shape`` should be the same. This function copies the value from\n        ``self`` to ``other``.\n\n        If ``other`` is a context, a new ``RowSparseNDArray`` will be first created on\n        the target context, and the value of ``self`` is copied.\n\n        Parameters\n        ----------\n        other : NDArray or RowSparseNDArray or Context\n            The destination array or context.\n\n        Returns\n        -------\n        NDArray or RowSparseNDArray\n            The copied array. If ``other`` is an ``NDArray`` or ``RowSparseNDArray``, then the\n            return value and ``other`` will point to the same ``NDArray`` or ``RowSparseNDArray``.\n        \"\"\"\n        if isinstance(other, Device):\n            return super(RowSparseNDArray, self).copyto(other)\n        elif isinstance(other, NDArray):\n            stype = other.stype\n            if stype in ('default', 'row_sparse'):\n                return super(RowSparseNDArray, self).copyto(other)\n            else:\n                raise TypeError('copyto does not support destination NDArray stype ' + str(stype))\n        else:\n            raise TypeError('copyto does not support type ' + str(type(other)))\n\n    def retain(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`retain`.\n\n        The arguments are the same as for :py:func:`retain`, with\n        this array as data.\n        \"\"\"\n        if not gs_retain:\n            raise ImportError(\"gen_sparse could not be imported\")\n        return gs_retain(*args, **kwargs)\n\ndef _prepare_src_array(source_array, dtype):\n    \"\"\"Prepare `source_array` so that it can be used to construct NDArray.\n    `source_array` is converted to a `np.ndarray` if it's neither an `NDArray` \\\n    nor an `np.ndarray`.\n    \"\"\"\n    if not isinstance(source_array, NDArray) and not isinstance(source_array, np.ndarray):\n        try:\n            source_array = np.array(source_array, dtype=dtype)\n        except:\n            raise TypeError('values must be array like object')\n    return source_array\n\ndef _prepare_default_dtype(src_array, dtype):\n    \"\"\"Prepare the value of dtype if `dtype` is None. If `src_array` is an NDArray, numpy.ndarray\n    or scipy.sparse.csr.csr_matrix, return src_array.dtype. float32 is returned otherwise.\"\"\"\n    if dtype is None:\n        if isinstance(src_array, (NDArray, np.ndarray)):\n            dtype = src_array.dtype\n        elif spsp and isinstance(src_array, spsp.csr.csr_matrix):\n            dtype = src_array.dtype\n        else:\n            dtype = mx_real_t\n    return dtype\n\ndef _check_shape(s1, s2):\n    \"\"\"check s1 == s2 if both are not None\"\"\"\n    if s1 and s2 and s1 != s2:\n        raise ValueError(\"Shape mismatch detected. \" + str(s1) + \" v.s. \" + str(s2))\n\ndef csr_matrix(arg1, shape=None, ctx=None, dtype=None):\n    \"\"\"Creates a `CSRNDArray`, an 2D array with compressed sparse row (CSR) format.\n\n    The CSRNDArray can be instantiated in several ways:\n\n    - csr_matrix(D):\n        to construct a CSRNDArray with a dense 2D array ``D``\n            -  **D** (*array_like*) - An object exposing the array interface, an object whose \\\n            `__array__` method returns an array, or any (nested) sequence.\n            - **ctx** (*Context, optional*) - Device context \\\n            (default is the current default context).\n            - **dtype** (*str or numpy.dtype, optional*) - The data type of the output array. \\\n            The default dtype is ``D.dtype`` if ``D`` is an NDArray or numpy.ndarray, \\\n            float32 otherwise.\n\n    - csr_matrix(S)\n        to construct a CSRNDArray with a sparse 2D array ``S``\n            -  **S** (*CSRNDArray or scipy.sparse.csr.csr_matrix*) - A sparse matrix.\n            - **ctx** (*Context, optional*) - Device context \\\n            (default is the current default context).\n            - **dtype** (*str or numpy.dtype, optional*) - The data type of the output array. \\\n            The default dtype is ``S.dtype``.\n\n    - csr_matrix((M, N))\n        to construct an empty CSRNDArray with shape ``(M, N)``\n            -  **M** (*int*) - Number of rows in the matrix\n            -  **N** (*int*) - Number of columns in the matrix\n            - **ctx** (*Context, optional*) - Device context \\\n            (default is the current default context).\n            - **dtype** (*str or numpy.dtype, optional*) - The data type of the output array. \\\n            The default dtype is float32.\n\n    - csr_matrix((data, indices, indptr))\n        to construct a CSRNDArray based on the definition of compressed sparse row format \\\n        using three separate arrays, \\\n        where the column indices for row i are stored in ``indices[indptr[i]:indptr[i+1]]`` \\\n        and their corresponding values are stored in ``data[indptr[i]:indptr[i+1]]``. \\\n        The column indices for a given row are expected to be **sorted in ascending order.** \\\n        Duplicate column entries for the same row are not allowed.\n            - **data** (*array_like*) - An object exposing the array interface, which \\\n            holds all the non-zero entries of the matrix in row-major order.\n            - **indices** (*array_like*) - An object exposing the array interface, which \\\n            stores the column index for each non-zero element in ``data``.\n            - **indptr** (*array_like*) - An object exposing the array interface, which \\\n            stores the offset into ``data`` of the first non-zero element number of each \\\n            row of the matrix.\n            - **shape** (*tuple of int, optional*) - The shape of the array. The default \\\n            shape is inferred from the indices and indptr arrays.\n            - **ctx** (*Context, optional*) - Device context \\\n            (default is the current default context).\n            - **dtype** (*str or numpy.dtype, optional*) - The data type of the output array. \\\n            The default dtype is ``data.dtype`` if ``data`` is an NDArray or numpy.ndarray, \\\n            float32 otherwise.\n\n    - csr_matrix((data, (row, col)))\n        to construct a CSRNDArray based on the COOrdinate format \\\n        using three seperate arrays, \\\n        where ``row[i]`` is the row index of the element, \\\n        ``col[i]`` is the column index of the element \\\n        and ``data[i]`` is the data corresponding to the element. All the missing \\\n        elements in the input are taken to be zeroes.\n            - **data** (*array_like*) - An object exposing the array interface, which \\\n            holds all the non-zero entries of the matrix in COO format.\n            - **row** (*array_like*) - An object exposing the array interface, which \\\n            stores the row index for each non zero element in ``data``.\n            - **col** (*array_like*) - An object exposing the array interface, which \\\n            stores the col index for each non zero element in ``data``.\n            - **shape** (*tuple of int, optional*) - The shape of the array. The default \\\n            shape is inferred from the ``row`` and ``col`` arrays.\n            - **ctx** (*Context, optional*) - Device context \\\n            (default is the current default context).\n            - **dtype** (*str or numpy.dtype, optional*) - The data type of the output array. \\\n            The default dtype is float32.\n\n    Parameters\n    ----------\n    arg1: tuple of int, tuple of array_like, array_like, CSRNDArray, scipy.sparse.csr_matrix, \\\n    scipy.sparse.coo_matrix, tuple of int or tuple of array_like\n        The argument to help instantiate the csr matrix. See above for further details.\n    shape : tuple of int, optional\n        The shape of the csr matrix.\n    ctx: Context, optional\n        Device context (default is the current default context).\n    dtype: str or numpy.dtype, optional\n        The data type of the output array.\n\n    Returns\n    -------\n    CSRNDArray\n        A `CSRNDArray` with the `csr` storage representation.\n\n    Example\n    -------\n    >>> a = mx.nd.sparse.csr_matrix(([1, 2, 3], [1, 0, 2], [0, 1, 2, 2, 3]), shape=(4, 3))\n    >>> a.asnumpy()\n    array([[ 0.,  1.,  0.],\n           [ 2.,  0.,  0.],\n           [ 0.,  0.,  0.],\n           [ 0.,  0.,  3.]], dtype=float32)\n\n    See Also\n    --------\n    CSRNDArray : MXNet NDArray in compressed sparse row format.\n    \"\"\"\n    # construct a csr matrix from (M, N) or (data, indices, indptr)\n    if isinstance(arg1, tuple):\n        arg_len = len(arg1)\n        if arg_len == 2:\n            # construct a sparse csr matrix from\n            # scipy coo matrix if input format is coo\n            if isinstance(arg1[1], tuple) and len(arg1[1]) == 2:\n                data, (row, col) = arg1\n                if isinstance(data, NDArray):\n                    data = data.asnumpy()\n                if isinstance(row, NDArray):\n                    row = row.asnumpy()\n                if isinstance(col, NDArray):\n                    col = col.asnumpy()\n                if not spsp:\n                    raise ImportError(\"scipy could not be imported. \"\n                                      \"Please make sure that the scipy is installed.\")\n                coo = spsp.coo_matrix((data, (row, col)), shape=shape)\n                _check_shape(coo.shape, shape)\n                csr = coo.tocsr()\n                return array(csr, ctx=ctx, dtype=dtype)\n            else:\n                # empty matrix with shape\n                _check_shape(arg1, shape)\n                return empty('csr', arg1, ctx=ctx, dtype=dtype)\n        elif arg_len == 3:\n            # data, indices, indptr\n            return _csr_matrix_from_definition(arg1[0], arg1[1], arg1[2], shape=shape,\n                                               ctx=ctx, dtype=dtype)\n        else:\n            raise ValueError(\"Unexpected length of input tuple: \" + str(arg_len))\n    else:\n        # construct a csr matrix from a sparse / dense one\n        if isinstance(arg1, CSRNDArray) or (spsp and isinstance(arg1, spsp.csr.csr_matrix)):\n            # construct a csr matrix from scipy or CSRNDArray\n            _check_shape(arg1.shape, shape)\n            return array(arg1, ctx=ctx, dtype=dtype)\n        elif isinstance(arg1, RowSparseNDArray):\n            raise ValueError(\"Unexpected input type: RowSparseNDArray\")\n        else:\n            # construct a csr matrix from a dense one\n            # prepare default ctx and dtype since mx.nd.array doesn't use default values\n            # based on source_array\n            dtype = _prepare_default_dtype(arg1, dtype)\n            # create dns array with provided dtype. ctx is not passed since copy across\n            # ctx requires dtype to be the same\n            dns = _array(arg1, dtype=dtype)\n            if ctx is not None and dns.context != ctx:\n                dns = dns.as_in_context(ctx)\n            _check_shape(dns.shape, shape)\n            return dns.tostype('csr')\n\ndef _csr_matrix_from_definition(data, indices, indptr, shape=None, ctx=None,\n                                dtype=None, indices_type=None, indptr_type=None):\n    \"\"\"Create a `CSRNDArray` based on data, indices and indptr\"\"\"\n    # pylint: disable= no-member, protected-access\n    storage_type = 'csr'\n    # context\n    ctx = current_device() if ctx is None else ctx\n    # types\n    dtype = _prepare_default_dtype(data, dtype)\n    indptr_type = _STORAGE_AUX_TYPES[storage_type][0] if indptr_type is None else indptr_type\n    indices_type = _STORAGE_AUX_TYPES[storage_type][1] if indices_type is None else indices_type\n    # prepare src array and types\n    data = _prepare_src_array(data, dtype)\n    indptr = _prepare_src_array(indptr, indptr_type)\n    indices = _prepare_src_array(indices, indices_type)\n\n    # TODO(junwu): Convert data, indptr, and indices to mxnet NDArrays\n    # if they are not for now. In the future, we should provide a c-api\n    # to accept np.ndarray types to copy from to result.data and aux_data\n    if not isinstance(data, NDArray):\n        data = _array(data, ctx, dtype)\n    if not isinstance(indptr, NDArray):\n        indptr = _array(indptr, ctx, indptr_type)\n    if not isinstance(indices, NDArray):\n        indices = _array(indices, ctx, indices_type)\n    if shape is None:\n        if indices.shape[0] == 0:\n            raise ValueError('invalid shape')\n        shape = (len(indptr) - 1, op.max(indices).asscalar() + 1)\n    # verify shapes\n    aux_shapes = [indptr.shape, indices.shape]\n    if data.ndim != 1 or indptr.ndim != 1 or indices.ndim != 1 or \\\n        indptr.shape[0] == 0 or len(shape) != 2:\n        raise ValueError('invalid shape')\n    result = CSRNDArray(_new_alloc_handle(storage_type, shape, ctx, False, dtype,\n                                          [indptr_type, indices_type], aux_shapes))\n    check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, data.handle, ctypes.c_int(-1)))\n    check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, indptr.handle, ctypes.c_int(0)))\n    check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, indices.handle, ctypes.c_int(1)))\n    return result\n    # pylint: enable= no-member, protected-access\n\ndef row_sparse_array(arg1, shape=None, ctx=None, dtype=None):\n    \"\"\"Creates a `RowSparseNDArray`, a multidimensional row sparse array with a set of \\\n    tensor slices at given indices.\n\n    The RowSparseNDArray can be instantiated in several ways:\n\n    - row_sparse_array(D):\n        to construct a RowSparseNDArray with a dense ndarray ``D``\n        -  **D** (*array_like*) - An object exposing the array interface, an object whose \\\n        `__array__` method returns an array, or any (nested) sequence.\n        - **ctx** (*Context, optional*) - Device context \\\n        (default is the current default context).\n        - **dtype** (*str or numpy.dtype, optional*) - The data type of the output array. \\\n        The default dtype is ``D.dtype`` if ``D`` is an NDArray or numpy.ndarray, \\\n        float32 otherwise.\n\n    - row_sparse_array(S)\n        to construct a RowSparseNDArray with a sparse ndarray ``S``\n        -  **S** (*RowSparseNDArray*) - A sparse ndarray.\n        - **ctx** (*Context, optional*) - Device context \\\n        (default is the current default context).\n        - **dtype** (*str or numpy.dtype, optional*) - The data type of the output array. \\\n        The default dtype is ``S.dtype``.\n\n    - row_sparse_array((D0, D1 .. Dn))\n        to construct an empty RowSparseNDArray with shape ``(D0, D1, ... Dn)``\n        -  **D0, D1 .. Dn** (*int*) - The shape of the ndarray\n        - **ctx** (*Context, optional*) - Device context \\\n        (default is the current default context).\n        - **dtype** (*str or numpy.dtype, optional*) - The data type of the output array. \\\n            The default dtype is float32.\n\n    - row_sparse_array((data, indices))\n        to construct a RowSparseNDArray based on the definition of row sparse format \\\n        using two separate arrays, \\\n        where the `indices` stores the indices of the row slices with non-zeros,\n        while the values are stored in `data`. The corresponding NDArray ``dense``\n        represented by RowSparseNDArray ``rsp`` has \\\n        ``dense[rsp.indices[i], :, :, :, ...] = rsp.data[i, :, :, :, ...]``\n        The row indices for are expected to be **sorted in ascending order.** \\\n        - **data** (*array_like*) - An object exposing the array interface, which \\\n        holds all the non-zero row slices of the array.\n        - **indices** (*array_like*) - An object exposing the array interface, which \\\n        stores the row index for each row slice with non-zero elements.\n        - **shape** (*tuple of int, optional*) - The shape of the array. The default \\\n        shape is inferred from the indices and indptr arrays.\n        - **ctx** (*Context, optional*) - Device context \\\n        (default is the current default context).\n        - **dtype** (*str or numpy.dtype, optional*) - The data type of the output array. \\\n        The default dtype is float32.\n\n    Parameters\n    ----------\n    arg1 : NDArray, numpy.ndarray, RowSparseNDArray, tuple of int or tuple of array_like\n        The argument to help instantiate the row sparse ndarray. See above for further details.\n    shape : tuple of int, optional\n        The shape of the row sparse ndarray. (Default value = None)\n    ctx : Context, optional\n        Device context (default is the current default context).\n    dtype : str or numpy.dtype, optional\n        The data type of the output array. (Default value = None)\n\n    Returns\n    -------\n    RowSparseNDArray\n        An `RowSparseNDArray` with the `row_sparse` storage representation.\n\n    Examples\n    --------\n    >>> a = mx.nd.sparse.row_sparse_array(([[1, 2], [3, 4]], [1, 4]), shape=(6, 2))\n    >>> a.asnumpy()\n    array([[ 0.,  0.],\n           [ 1.,  2.],\n           [ 0.,  0.],\n           [ 0.,  0.],\n           [ 3.,  4.],\n           [ 0.,  0.]], dtype=float32)\n\n    See Also\n    --------\n    RowSparseNDArray : MXNet NDArray in row sparse format.\n    \"\"\"\n    # construct a row sparse array from (D0, D1 ..) or (data, indices)\n    if isinstance(arg1, tuple):\n        arg_len = len(arg1)\n        if arg_len < 2:\n            raise ValueError(\"Unexpected length of input tuple: \" + str(arg_len))\n        if arg_len > 2:\n            # empty ndarray with shape\n            _check_shape(arg1, shape)\n            return empty('row_sparse', arg1, ctx=ctx, dtype=dtype)\n        else:\n            # len(arg1) = 2, is either shape or (data, indices)\n            if isinstance(arg1[0], integer_types) and isinstance(arg1[1], integer_types):\n                # empty ndarray with shape\n                _check_shape(arg1, shape)\n                return empty('row_sparse', arg1, ctx=ctx, dtype=dtype)\n            else:\n                # data, indices, indptr\n                return _row_sparse_ndarray_from_definition(arg1[0], arg1[1], shape=shape,\n                                                           ctx=ctx, dtype=dtype)\n    else:\n        # construct a row sparse ndarray from a dense / sparse array\n        if isinstance(arg1, RowSparseNDArray):\n            # construct a row sparse ndarray from RowSparseNDArray\n            _check_shape(arg1.shape, shape)\n            return array(arg1, ctx=ctx, dtype=dtype)\n        elif isinstance(arg1, CSRNDArray):\n            raise ValueError(\"Unexpected input type: CSRNDArray\")\n        else:\n            # construct a csr matrix from a dense one\n            # prepare default dtype since mx.nd.array doesn't use default values\n            # based on source_array\n            dtype = _prepare_default_dtype(arg1, dtype)\n            # create dns array with provided dtype. ctx is not passed since copy across\n            # ctx requires dtype to be the same\n            dns = _array(arg1, dtype=dtype)\n            if ctx is not None and dns.context != ctx:\n                dns = dns.as_in_context(ctx)\n            _check_shape(dns.shape, shape)\n            return dns.tostype('row_sparse')\n\ndef _row_sparse_ndarray_from_definition(data, indices, shape=None, ctx=None,\n                                        dtype=None, indices_type=None):\n    \"\"\"Create a `RowSparseNDArray` based on data and indices\"\"\"\n    storage_type = 'row_sparse'\n    # context\n    ctx = current_device() if ctx is None else ctx\n    # types\n    dtype = _prepare_default_dtype(data, dtype)\n    indices_type = _STORAGE_AUX_TYPES[storage_type][0] if indices_type is None else indices_type\n    # prepare src array and types\n    data = _prepare_src_array(data, dtype)\n    indices = _prepare_src_array(indices, indices_type)\n\n    # TODO(junwu): Convert data, indptr, and indices to mxnet NDArrays\n    # if they are not for now. In the future, we should provide a c-api\n    # to accept np.ndarray types to copy from to result.data and aux_data\n    if not isinstance(data, NDArray):\n        data = _array(data, ctx, dtype)\n    if not isinstance(indices, NDArray):\n        indices = _array(indices, ctx, indices_type)\n    if shape is None:\n        num_indices = indices.shape[0]\n        if num_indices == 0:\n            raise ValueError('invalid shape')\n        dim0 = indices[num_indices - 1].asscalar() + 1\n        shape = (dim0, ) + data.shape[1:]\n    # verify shapes\n    if data.ndim != len(shape) or indices.ndim != 1 or np.prod(shape[1:]) == 0:\n        raise ValueError(\"invalid shape\")\n    result = RowSparseNDArray(_new_alloc_handle(storage_type, shape, ctx, False, dtype,\n                                                [indices_type], [indices.shape]))\n    check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, data.handle, ctypes.c_int(-1)))\n    check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, indices.handle, ctypes.c_int(0)))\n    return result\n\ndef _ndarray_cls(handle, writable=True, stype=_STORAGE_TYPE_UNDEFINED):\n    if stype == _STORAGE_TYPE_UNDEFINED:\n        stype = _storage_type(handle)\n    if stype == _STORAGE_TYPE_DEFAULT:\n        return NDArray(handle, writable=writable)\n    elif stype == _STORAGE_TYPE_CSR:\n        return CSRNDArray(handle, writable=writable)\n    elif stype == _STORAGE_TYPE_ROW_SPARSE:\n        return RowSparseNDArray(handle, writable=writable)\n    else:\n        raise Exception(f\"unknown storage type: {stype}\")\n\n\n_set_ndarray_class(_ndarray_cls)\n\n\ndef add(lhs, rhs):\n    \"\"\"Returns element-wise sum of the input arrays with broadcasting.\n\n    Equivalent to ``lhs + rhs``, ``mx.nd.broadcast_add(lhs, rhs)`` and\n    ``mx.nd.broadcast_plus(lhs, rhs)`` when shapes of lhs and rhs do not\n    match. If lhs.shape == rhs.shape, this is equivalent to\n    ``mx.nd.elemwise_add(lhs, rhs)``\n\n    .. note::\n\n        If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n        then the arrays are broadcastable to a common shape.abs\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.sparse.array\n        First array to be added.\n    rhs : scalar or mxnet.ndarray.sparse.array\n         Second array to be added.\n        If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        The element-wise sum of the input arrays.\n\n    Examples\n    --------\n    >>> a = mx.nd.ones((2,3)).tostype('csr')\n    >>> b = mx.nd.ones((2,3)).tostype('csr')\n    >>> a.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> b.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> (a+b).asnumpy()\n    array([[ 2.,  2.,  2.],\n           [ 2.,  2.,  2.]], dtype=float32)\n    >>> c = mx.nd.ones((2,3)).tostype('row_sparse')\n    >>> d = mx.nd.ones((2,3)).tostype('row_sparse')\n    >>> c.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> d.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> (c+d).asnumpy()\n    array([[ 2.,  2.,  2.],\n           [ 2.,  2.,  2.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    if isinstance(lhs, NDArray) and isinstance(rhs, NDArray) and lhs.shape == rhs.shape:\n        return _ufunc_helper(\n            lhs,\n            rhs,\n            op.elemwise_add,\n            operator.add,\n            _internal._plus_scalar,\n            None)\n\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_add,\n        operator.add,\n        _internal._plus_scalar,\n        None)\n    # pylint: enable= no-member, protected-access\n\n\ndef subtract(lhs, rhs):\n    \"\"\"Returns element-wise difference of the input arrays with broadcasting.\n\n    Equivalent to ``lhs - rhs``, ``mx.nd.broadcast_sub(lhs, rhs)`` and\n    ``mx.nd.broadcast_minus(lhs, rhs)`` when shapes of lhs and rhs do not\n    match. If lhs.shape == rhs.shape, this is equivalent to\n    ``mx.nd.elemwise_sub(lhs, rhs)``\n\n    .. note::\n\n        If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n        then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.sparse.array\n        First array to be subtracted.\n    rhs : scalar or mxnet.ndarray.sparse.array\n         Second array to be subtracted.\n        If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.__spec__\n\n    Returns\n    -------\n    NDArray\n        The element-wise difference of the input arrays.\n\n    Examples\n    --------\n    >>> a = mx.nd.ones((2,3)).tostype('csr')\n    >>> b = mx.nd.ones((2,3)).tostype('csr')\n    >>> a.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> b.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> (a-b).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 0.,  0.,  0.]], dtype=float32)\n    >>> c = mx.nd.ones((2,3)).tostype('row_sparse')\n    >>> d = mx.nd.ones((2,3)).tostype('row_sparse')\n    >>> c.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> d.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> (c-d).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 0.,  0.,  0.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    if isinstance(lhs, NDArray) and isinstance(rhs, NDArray) and lhs.shape == rhs.shape:\n        return _ufunc_helper(\n            lhs,\n            rhs,\n            op.elemwise_sub,\n            operator.sub,\n            _internal._minus_scalar,\n            None)\n\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_sub,\n        operator.sub,\n        _internal._minus_scalar,\n        None)\n    # pylint: enable= no-member, protected-access\n\n\ndef multiply(lhs, rhs):\n    \"\"\"Returns element-wise product of the input arrays with broadcasting.\n\n        Equivalent to ``lhs * rhs`` and ``mx.nd.broadcast_mul(lhs, rhs)``\n        when shapes of lhs and rhs do not match. If lhs.shape == rhs.shape,\n        this is equivalent to ``mx.nd.elemwise_mul(lhs, rhs)``\n\n    .. note::\n\n        If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n        then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.sparse.array\n        First array to be multiplied.\n    rhs : scalar or mxnet.ndarray.sparse.array\n         Second array to be multiplied.\n        If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        The element-wise multiplication of the input arrays.\n\n    Examples\n    --------\n    >>> x = mx.nd.ones((2,3)).tostype('csr')\n    >>> y = mx.nd.arange(2).reshape((2,1))\n    >>> z = mx.nd.arange(3)\n    >>> x.asnumpy()\n    array([[ 1.,  1.,  1.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 0.],\n           [ 1.]], dtype=float32)\n    >>> z.asnumpy()\n    array([ 0.,  1.,  2.], dtype=float32)\n    >>> (x*2).asnumpy()\n    array([[ 2.,  2.,  2.],\n           [ 2.,  2.,  2.]], dtype=float32)\n    >>> (x*y).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> mx.nd.sparse.multiply(x, y).asnumpy()\n    array([[ 0.,  0.,  0.],\n           [ 1.,  1.,  1.]], dtype=float32)\n    >>> (x*z).asnumpy()\n    array([[ 0.,  1.,  2.],\n           [ 0.,  1.,  2.]], dtype=float32)\n    >>> mx.nd.sparse.multiply(x, z).asnumpy()\n    array([[ 0.,  1.,  2.],\n           [ 0.,  1.,  2.]], dtype=float32)\n    >>> z = z.reshape((1, 3))\n    >>> z.asnumpy()\n    array([[ 0.,  1.,  2.]], dtype=float32)\n    >>> (x*z).asnumpy()\n    array([[ 0.,  1.,  2.],\n           [ 0.,  1.,  2.]], dtype=float32)\n    >>> mx.nd.sparse.multiply(x, z).asnumpy()\n    array([[ 0.,  1.,  2.],\n           [ 0.,  1.,  2.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    if isinstance(lhs, NDArray) and isinstance(rhs, NDArray) and lhs.shape == rhs.shape:\n        return _ufunc_helper(\n            lhs,\n            rhs,\n            op.elemwise_mul,\n            operator.mul,\n            _internal._mul_scalar,\n            None)\n\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_mul,\n        operator.mul,\n        _internal._mul_scalar,\n        None)\n    # pylint: enable= no-member, protected-access\n\n\ndef divide(lhs, rhs):\n    \"\"\"Returns element-wise division of the input arrays with broadcasting.\n\n    Equivalent to ``lhs / rhs`` and ``mx.nd.broadcast_div(lhs, rhs)``\n    when shapes of lhs and rhs do not match. If lhs.shape == rhs.shape,\n    this is equivalent to ``mx.nd.elemwise_div(lhs, rhs)``\n\n    .. note::\n\n        If the corresponding dimensions of two arrays have the same size or one of them has size 1,\n        then the arrays are broadcastable to a common shape.\n\n    Parameters\n    ----------\n    lhs : scalar or mxnet.ndarray.sparse.array\n        First array in division.\n    rhs : scalar or mxnet.ndarray.sparse.array\n         Second array in division.\n        The arrays to be divided. If ``lhs.shape != rhs.shape``, they must be\n        broadcastable to a common shape.\n\n    Returns\n    -------\n    NDArray\n        The element-wise division of the input arrays.\n\n    Examples\n    --------\n    >>> x = (mx.nd.ones((2,3))*6).tostype('csr')\n    >>> y = mx.nd.arange(2).reshape((2,1)) + 1\n    >>> z = mx.nd.arange(3) + 1\n    >>> x.asnumpy()\n    array([[ 6.,  6.,  6.],\n           [ 6.,  6.,  6.]], dtype=float32)\n    >>> y.asnumpy()\n    array([[ 1.],\n           [ 2.]], dtype=float32)\n    >>> z.asnumpy()\n    array([ 1.,  2.,  3.], dtype=float32)\n    >>> x/2\n    <NDArray 2x3 @cpu(0)>\n    >>> (x/3).asnumpy()\n    array([[ 2.,  2.,  2.],\n           [ 2.,  2.,  2.]], dtype=float32)\n    >>> (x/y).asnumpy()\n    array([[ 6.,  6.,  6.],\n           [ 3.,  3.,  3.]], dtype=float32)\n    >>> mx.nd.sparse.divide(x,y).asnumpy()\n    array([[ 6.,  6.,  6.],\n           [ 3.,  3.,  3.]], dtype=float32)\n    >>> (x/z).asnumpy()\n    array([[ 6.,  3.,  2.],\n           [ 6.,  3.,  2.]], dtype=float32)\n    >>> mx.nd.sprase.divide(x,z).asnumpy()\n    array([[ 6.,  3.,  2.],\n           [ 6.,  3.,  2.]], dtype=float32)\n    >>> z = z.reshape((1,3))\n    >>> z.asnumpy()\n    array([[ 1.,  2.,  3.]], dtype=float32)\n    >>> (x/z).asnumpy()\n    array([[ 6.,  3.,  2.],\n           [ 6.,  3.,  2.]], dtype=float32)\n    >>> mx.nd.sparse.divide(x,z).asnumpy()\n    array([[ 6.,  3.,  2.],\n           [ 6.,  3.,  2.]], dtype=float32)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    if isinstance(lhs, NDArray) and isinstance(rhs, NDArray) and lhs.shape == rhs.shape:\n        return _ufunc_helper(\n            lhs,\n            rhs,\n            op.elemwise_div,\n            operator.truediv,\n            _internal._div_scalar,\n            None)\n\n    return _ufunc_helper(\n        lhs,\n        rhs,\n        op.broadcast_div,\n        operator.truediv,\n        _internal._div_scalar,\n        None)\n    # pylint: enable= no-member, protected-access\n\n\ndef zeros(stype, shape, ctx=None, dtype=None, **kwargs):\n    \"\"\"Return a new array of given shape and type, filled with zeros.\n\n    Parameters\n    ----------\n    stype: string\n        The storage type of the empty array, such as 'row_sparse', 'csr', etc\n    shape : int or tuple of int\n        The shape of the empty array\n    ctx : Context, optional\n        An optional device context (default is the current default context)\n    dtype : str or numpy.dtype, optional\n        An optional value type (default is `float32`)\n\n    Returns\n    -------\n    RowSparseNDArray or CSRNDArray\n        A created array\n    Examples\n    --------\n    >>> mx.nd.sparse.zeros('csr', (1,2))\n    <CSRNDArray 1x2 @cpu(0)>\n    >>> mx.nd.sparse.zeros('row_sparse', (1,2), ctx=mx.cpu(), dtype='float16').asnumpy()\n    array([[ 0.,  0.]], dtype=float16)\n    \"\"\"\n    # pylint: disable= no-member, protected-access\n    if stype == 'default':\n        return _zeros_ndarray(shape, ctx=ctx, dtype=dtype, **kwargs)\n    if ctx is None:\n        ctx = current_device()\n    dtype = mx_real_t if dtype is None else dtype\n    if stype in ('row_sparse', 'csr'):\n        aux_types = _STORAGE_AUX_TYPES[stype]\n    else:\n        raise ValueError(\"unknown storage type: \" + stype)\n    out = _ndarray_cls(_new_alloc_handle(stype, shape, ctx, True, dtype, aux_types))\n    return _internal._zeros(shape=shape, ctx=ctx, dtype=dtype, out=out, **kwargs)\n    # pylint: enable= no-member, protected-access\n\n\ndef empty(stype, shape, ctx=None, dtype=None):\n    \"\"\"Returns a new array of given shape and type, without initializing entries.\n\n    Parameters\n    ----------\n    stype: string\n        The storage type of the empty array, such as 'row_sparse', 'csr', etc\n    shape : int or tuple of int\n        The shape of the empty array.\n    ctx : Context, optional\n        An optional device context (default is the current default context).\n    dtype : str or numpy.dtype, optional\n        An optional value type (default is `float32`).\n\n    Returns\n    -------\n    CSRNDArray or RowSparseNDArray\n        A created array.\n    \"\"\"\n    if isinstance(shape, int):\n        shape = (shape, )\n    if ctx is None:\n        ctx = current_device()\n    if dtype is None:\n        dtype = mx_real_t\n    assert(stype is not None)\n    if stype in ('csr', 'row_sparse'):\n        return zeros(stype, shape, ctx=ctx, dtype=dtype)\n    else:\n        raise Exception(\"unknown stype : \" + str(stype))\n\n\ndef array(source_array, ctx=None, dtype=None):\n    \"\"\"Creates a sparse array from any object exposing the array interface.\n\n    Parameters\n    ----------\n    source_array : RowSparseNDArray, CSRNDArray or scipy.sparse.csr.csr_matrix\n        The source sparse array\n    ctx : Context, optional\n        The default context is ``source_array.context`` if ``source_array`` is an NDArray. \\\n        The current default context otherwise.\n    dtype : str or numpy.dtype, optional\n        The data type of the output array. The default dtype is ``source_array.dtype``\n        if `source_array` is an `NDArray`, `numpy.ndarray` or `scipy.sparse.csr.csr_matrix`, \\\n        `float32` otherwise.\n\n    Returns\n    -------\n    RowSparseNDArray or CSRNDArray\n        An array with the same contents as the `source_array`.\n\n    Examples\n    --------\n    >>> import scipy.sparse as spsp\n    >>> csr = spsp.csr_matrix((2, 100))\n    >>> mx.nd.sparse.array(csr)\n    <CSRNDArray 2x100 @cpu(0)>\n    >>> mx.nd.sparse.array(mx.nd.sparse.zeros('csr', (3, 2)))\n    <CSRNDArray 3x2 @cpu(0)>\n    >>> mx.nd.sparse.array(mx.nd.sparse.zeros('row_sparse', (3, 2)))\n    <RowSparseNDArray 3x2 @cpu(0)>\n    \"\"\"\n    ctx = current_device() if ctx is None else ctx\n    if isinstance(source_array, NDArray):\n        assert(source_array.stype != 'default'), \\\n               \"Please use `tostype` to create RowSparseNDArray or CSRNDArray from an NDArray\"\n        # prepare dtype and ctx based on source_array, if not provided\n        dtype = _prepare_default_dtype(source_array, dtype)\n        # if both dtype and ctx are different from source_array, we cannot copy directly\n        if source_array.dtype != dtype and source_array.context != ctx:\n            arr = empty(source_array.stype, source_array.shape, dtype=dtype)\n            arr[:] = source_array\n            arr = arr.as_in_context(ctx)\n        else:\n            arr = empty(source_array.stype, source_array.shape, dtype=dtype, ctx=ctx)\n            arr[:] = source_array\n        return arr\n    elif spsp and isinstance(source_array, spsp.csr.csr_matrix):\n        # TODO(haibin) implement `_sync_copy_from` with scipy csr object to reduce a copy\n        # preprocess scipy csr to canonical form\n        csr = source_array.sorted_indices()\n        csr.sum_duplicates()\n        dtype = _prepare_default_dtype(source_array, dtype)\n        return csr_matrix((csr.data, csr.indices, csr.indptr), shape=csr.shape, \\\n                          dtype=dtype, ctx=ctx)\n    elif isinstance(source_array, (np.ndarray, np.generic)):\n        raise ValueError(\"Please use mx.nd.array to create an NDArray with source_array of type \",\n                         type(source_array))\n    else:\n        raise ValueError(\"Unexpected source_array type: \", type(source_array))\n"
  },
  {
    "path": "python/mxnet/ndarray/utils.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Utility functions for NDArray and BaseSparseNDArray.\"\"\"\nimport ctypes\n\nfrom ..base import _LIB, check_call, py_str, c_str, string_types, mx_uint, NDArrayHandle\nfrom ..base import c_array, c_handle_array, c_str_array\nfrom .ndarray import NDArray\nfrom .ndarray import array as _array\nfrom .ndarray import empty as _empty_ndarray\nfrom .ndarray import zeros as _zeros_ndarray\nfrom .sparse import zeros as _zeros_sparse_ndarray\nfrom .sparse import empty as _empty_sparse_ndarray\nfrom .sparse import array as _sparse_array\nfrom .sparse import _ndarray_cls\ntry:\n    import scipy.sparse as spsp\nexcept ImportError:\n    spsp = None\n\n__all__ = ['zeros', 'empty', 'array', 'load', 'load_frombuffer', 'save']\n\n\ndef zeros(shape, ctx=None, dtype=None, stype=None, **kwargs):\n    \"\"\"Return a new array of given shape and type, filled with zeros.\n\n    Parameters\n    ----------\n    shape : int or tuple of int\n        The shape of the empty array\n    ctx : Context, optional\n        An optional device context (default is the current default context)\n    dtype : str or numpy.dtype, optional\n        An optional value type (default is `float32`)\n    stype: string, optional\n        The storage type of the empty array, such as 'row_sparse', 'csr', etc.\n\n    Returns\n    -------\n    NDArray, CSRNDArray or RowSparseNDArray\n        A created array\n    Examples\n    --------\n    >>> mx.nd.zeros((1,2), mx.cpu(), stype='csr')\n    <CSRNDArray 1x2 @cpu(0)>\n    >>> mx.nd.zeros((1,2), mx.cpu(), 'float16', stype='row_sparse').asnumpy()\n    array([[ 0.,  0.]], dtype=float16)\n    \"\"\"\n\n    if stype is None or stype == 'default':\n        return _zeros_ndarray(shape, ctx, dtype, **kwargs)\n    else:\n        return _zeros_sparse_ndarray(stype, shape, ctx, dtype, **kwargs)\n\n\ndef empty(shape, ctx=None, dtype=None, stype=None):\n    \"\"\"Returns a new array of given shape and type, without initializing entries.\n\n    Parameters\n    ----------\n    shape : int or tuple of int\n        The shape of the empty array.\n    ctx : Context, optional\n        An optional device context (default is the current default context).\n    dtype : str or numpy.dtype, optional\n        An optional value type (default is `float32`).\n    stype : str, optional\n        An optional storage type (default is `default`).\n\n    Returns\n    -------\n    NDArray, CSRNDArray or RowSparseNDArray\n        A created array.\n\n    Examples\n    --------\n    >>> mx.nd.empty(1)\n    <NDArray 1 @cpu(0)>\n    >>> mx.nd.empty((1,2), mx.gpu(0))\n    <NDArray 1x2 @gpu(0)>\n    >>> mx.nd.empty((1,2), mx.gpu(0), 'float16')\n    <NDArray 1x2 @gpu(0)>\n    >>> mx.nd.empty((1,2), stype='csr')\n    <CSRNDArray 1x2 @cpu(0)>\n    \"\"\"\n    if stype is None or stype == 'default':\n        return _empty_ndarray(shape, ctx, dtype)\n    else:\n        return _empty_sparse_ndarray(stype, shape, ctx, dtype)\n\n\ndef array(source_array, ctx=None, dtype=None):\n    \"\"\"Creates an array from any object exposing the array interface.\n\n    Parameters\n    ----------\n    source_array : array_like\n        An object exposing the array interface, an object whose `__array__`\n        method returns an array, or any (nested) sequence.\n    ctx : Context, optional\n        Device context (default is the current default context).\n    dtype : str or numpy.dtype, optional\n        The data type of the output array. The default dtype is ``source_array.dtype``\n        if `source_array` is an `NDArray`, `float32` otherwise.\n\n    Returns\n    -------\n    NDArray, RowSparseNDArray or CSRNDArray\n        An array with the same contents as the `source_array`.\n\n    Examples\n    --------\n    >>> import numpy as np\n    >>> mx.nd.array([1, 2, 3])\n    <NDArray 3 @cpu(0)>\n    >>> mx.nd.array([[1, 2], [3, 4]])\n    <NDArray 2x2 @cpu(0)>\n    >>> mx.nd.array(np.zeros((3, 2)))\n    <NDArray 3x2 @cpu(0)>\n    >>> mx.nd.array(np.zeros((3, 2)), mx.gpu(0))\n    <NDArray 3x2 @gpu(0)>\n    >>> mx.nd.array(mx.nd.zeros((3, 2), stype='row_sparse'))\n    <RowSparseNDArray 3x2 @cpu(0)>\n    \"\"\"\n    if spsp is not None and isinstance(source_array, spsp.csr.csr_matrix):\n        return _sparse_array(source_array, ctx=ctx, dtype=dtype)\n    elif isinstance(source_array, NDArray) and source_array.stype != 'default':\n        return _sparse_array(source_array, ctx=ctx, dtype=dtype)\n    else:\n        return _array(source_array, ctx=ctx, dtype=dtype)\n\n\ndef load(fname):\n    \"\"\"Loads an array from file.\n\n    See more details in ``save``.\n\n    Parameters\n    ----------\n    fname : str\n        The filename.\n\n    Returns\n    -------\n    list of NDArray, RowSparseNDArray or CSRNDArray, or \\\n    dict of str to NDArray, RowSparseNDArray or CSRNDArray\n        Loaded data.\n    \"\"\"\n    if not isinstance(fname, string_types):\n        raise TypeError('fname required to be a string')\n    out_size = mx_uint()\n    out_name_size = mx_uint()\n    handles = ctypes.POINTER(NDArrayHandle)()\n    names = ctypes.POINTER(ctypes.c_char_p)()\n    check_call(_LIB.MXNDArrayLoad(c_str(fname),\n                                  ctypes.byref(out_size),\n                                  ctypes.byref(handles),\n                                  ctypes.byref(out_name_size),\n                                  ctypes.byref(names)))\n    if out_name_size.value == 0:\n        return [_ndarray_cls(NDArrayHandle(handles[i])) for i in range(out_size.value)]\n    else:\n        assert out_name_size.value == out_size.value\n        return dict(\n            (py_str(names[i]), _ndarray_cls(NDArrayHandle(handles[i])))\n            for i in range(out_size.value))\n\n\ndef load_frombuffer(buf):\n    \"\"\"Loads an array dictionary or list from a buffer\n\n    See more details in ``save``.\n\n    Parameters\n    ----------\n    buf : str\n        Buffer containing contents of a file as a string or bytes.\n\n    Returns\n    -------\n    list of NDArray, RowSparseNDArray or CSRNDArray, or \\\n    dict of str to NDArray, RowSparseNDArray or CSRNDArray\n        Loaded data.\n    \"\"\"\n    if not isinstance(buf, string_types + tuple([bytes])):\n        raise TypeError('buf required to be a string or bytes')\n    out_size = mx_uint()\n    out_name_size = mx_uint()\n    handles = ctypes.POINTER(NDArrayHandle)()\n    names = ctypes.POINTER(ctypes.c_char_p)()\n    check_call(_LIB.MXNDArrayLoadFromBuffer(buf,\n                                            mx_uint(len(buf)),\n                                            ctypes.byref(out_size),\n                                            ctypes.byref(handles),\n                                            ctypes.byref(out_name_size),\n                                            ctypes.byref(names)))\n    if out_name_size.value == 0:\n        return [_ndarray_cls(NDArrayHandle(handles[i])) for i in range(out_size.value)]\n    else:\n        assert out_name_size.value == out_size.value\n        return dict(\n            (py_str(names[i]), _ndarray_cls(NDArrayHandle(handles[i])))\n            for i in range(out_size.value))\n\n\ndef save(fname, data):\n    \"\"\"Saves a list of arrays or a dict of str->array to file.\n\n    Parameters\n    ----------\n    fname : str\n        The filename.\n    data : NDArray, RowSparseNDArray or CSRNDArray, \\\n           or list of NDArray, RowSparseNDArray or CSRNDArray, \\\n           or dict of str to NDArray, RowSparseNDArray or CSRNDArray\n        The data to save.\n\n    Examples\n    --------\n    >>> x = mx.nd.zeros((2,3))\n    >>> y = mx.nd.ones((1,4))\n    >>> mx.nd.save('my_list', [x,y])\n    >>> mx.nd.save('my_dict', {'x':x, 'y':y})\n    >>> mx.nd.load('my_list')\n    [<NDArray 2x3 @cpu(0)>, <NDArray 1x4 @cpu(0)>]\n    >>> mx.nd.load('my_dict')\n    {'y': <NDArray 1x4 @cpu(0)>, 'x': <NDArray 2x3 @cpu(0)>}\n    \"\"\"\n    from ..numpy import ndarray as np_ndarray\n    if isinstance(data, NDArray):\n        data = [data]\n        handles = c_array(NDArrayHandle, [])\n    if isinstance(data, dict):\n        str_keys = data.keys()\n        nd_vals = data.values()\n        if any(not isinstance(k, string_types) for k in str_keys) or \\\n           any(not isinstance(v, NDArray) for v in nd_vals):\n            raise TypeError('save only accept dict str->NDArray or list of NDArray')\n        if any(isinstance(v, np_ndarray) for v in nd_vals):\n            raise TypeError('cannot save mxnet.numpy.ndarray using mxnet.ndarray.save;'\n                            ' use mxnet.numpy.save instead.')\n        keys = c_str_array(str_keys)\n        handles = c_handle_array(nd_vals)\n    elif isinstance(data, list):\n        if any(not isinstance(v, NDArray) for v in data):\n            raise TypeError('save only accept dict str->NDArray or list of NDArray')\n        if any(isinstance(v, np_ndarray) for v in data):\n            raise TypeError('cannot save mxnet.numpy.ndarray using mxnet.ndarray.save;'\n                            ' use mxnet.numpy.save instead.')\n        keys = None\n        handles = c_handle_array(data)\n    else:\n        raise ValueError(\"data needs to either be a NDArray, dict of str, NDArray pairs \"\n                         \"or a list of NDarrays.\")\n    check_call(_LIB.MXNDArrayLegacySave(c_str(fname), mx_uint(len(handles)), handles, keys))\n"
  },
  {
    "path": "python/mxnet/ndarray_doc.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=unused-argument, too-many-arguments, unnecessary-pass\n\"\"\"Extra symbol documents\"\"\"\nfrom __future__ import absolute_import as _abs\nimport re as _re\n\nfrom .base import build_param_doc as _build_param_doc\n\nclass NDArrayDoc(object):\n    \"\"\"The basic class\"\"\"\n    pass\n\ndef _build_doc(func_name,\n               desc,\n               arg_names,\n               arg_types,\n               arg_desc,\n               key_var_num_args=None,\n               ret_type=None):\n    \"\"\"Build docstring for imperative functions.\"\"\"\n    param_str = _build_param_doc(arg_names, arg_types, arg_desc)\n    # if key_var_num_args:\n    #     desc += '\\nThis function support variable length of positional input.'\n    doc_str = (f'{desc}\\n\\n' +\n               f'{param_str}\\n' +\n               'out : NDArray, optional\\n' +\n               '    The output NDArray to hold the result.\\n\\n'+\n               'Returns\\n' +\n               '-------\\n' +\n               'out : NDArray or list of NDArrays\\n' +\n               '    The output of this function.')\n    extra_doc = \"\\n\" + '\\n'.join([x.__doc__ for x in type.__subclasses__(NDArrayDoc)\n                                  if x.__name__ == f'{func_name}Doc'])\n    doc_str += _re.sub(_re.compile(\"    \"), \"\", extra_doc)\n    doc_str = _re.sub('NDArray-or-Symbol', 'NDArray', doc_str)\n\n    return doc_str\n"
  },
  {
    "path": "python/mxnet/notebook/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=invalid-name, missing-docstring, no-init, old-style-class, multiple-statements\n\n\"\"\"MXNet notebook: an easy to use visualization platform\"\"\"\n\ntry:\n    import bokeh\nexcept ImportError:\n    class Bokeh_Failed_To_Import: pass\n    bokeh = Bokeh_Failed_To_Import\n\ntry:\n    import boken.io\nexcept ImportError:\n    pass\n"
  },
  {
    "path": "python/mxnet/notebook/callback.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=fixme, invalid-name, missing-docstring, no-init, old-style-class, multiple-statements\n# pylint: disable=arguments-differ, too-many-arguments, no-member\n\"\"\"Visualization callback function\n\"\"\"\ntry:\n    import datetime\nexcept ImportError:\n    class Datetime_Failed_To_Import: pass\n    datetime = Datetime_Failed_To_Import\n\ntry:\n    import bokeh.plotting\nexcept ImportError:\n    pass\n\ntry:\n    from collections import defaultdict\nexcept ImportError:\n    class Defaultdict_Failed_To_Import: pass\n    defaultdict = Defaultdict_Failed_To_Import\n\ntry:\n    import pandas as pd\nexcept ImportError:\n    class Pandas_Failed_To_Import: pass\n    pd = Pandas_Failed_To_Import\n\nimport time\n# pylint: enable=missing-docstring, no-init, old-style-class, multiple-statements\n\n\ndef _add_new_columns(dataframe, metrics):\n    \"\"\"Add new metrics as new columns to selected pandas dataframe.\n\n    Parameters\n    ----------\n    dataframe : pandas.DataFrame\n        Selected dataframe needs to be modified.\n    metrics : metric.EvalMetric\n        New metrics to be added.\n    \"\"\"\n    #TODO(leodirac): we don't really need to do this on every update.  Optimize\n    new_columns = set(metrics.keys()) - set(dataframe.columns)\n    for col in new_columns:\n        dataframe[col] = None\n\n\ndef _extend(baseData, newData):\n    \"\"\"Assuming a is shorter than b, copy the end of b onto a\n    \"\"\"\n    baseData.extend(newData[len(baseData):])\n\n\nclass PandasLogger(object):\n    \"\"\"Logs statistics about training run into Pandas dataframes.\n    Records three separate dataframes: train, eval, epoch.\n\n    Parameters\n    ----------\n    batch_size: int\n        batch_size of data\n    frequent: int\n        How many training mini-batches between calculations.\n        Defaults to calculating every 50 batches.\n        (Eval data is stored once per epoch over the entire\n        eval data set.)\n    \"\"\"\n    def __init__(self, batch_size, frequent=50):\n        self.batch_size = batch_size\n        self.frequent = frequent\n        self._dataframes = {\n            'train': pd.DataFrame(),\n            'eval': pd.DataFrame(),\n            'epoch': pd.DataFrame(),\n        }\n        self.last_time = time.time()\n        self.start_time = datetime.datetime.now()\n        self.last_epoch_time = datetime.datetime.now()\n\n    @property\n    def train_df(self):\n        \"\"\"The dataframe with training data.\n        This has metrics for training minibatches, logged every\n        \"frequent\" batches.  (frequent is a constructor param)\n        \"\"\"\n        return self._dataframes['train']\n\n    @property\n    def eval_df(self):\n        \"\"\"The dataframe with evaluation data.\n        This has validation scores calculated at the end of each epoch.\n        \"\"\"\n        return self._dataframes['eval']\n\n    @property\n    def epoch_df(self):\n        \"\"\"The dataframe with epoch data.\n        This has timing information.\n        \"\"\"\n        return self._dataframes['epoch']\n\n    @property\n    def all_dataframes(self):\n        \"\"\"Return a dict of dataframes\n        \"\"\"\n        return self._dataframes\n\n    def elapsed(self):\n        \"\"\"Calcaulate the elapsed time from training starting.\n        \"\"\"\n        return datetime.datetime.now() - self.start_time\n\n    def append_metrics(self, metrics, df_name):\n        \"\"\"Append new metrics to selected dataframes.\n\n        Parameters\n        ----------\n        metrics : metric.EvalMetric\n            New metrics to be added.\n        df_name : str\n            Name of the dataframe to be modified.\n        \"\"\"\n        dataframe = self._dataframes[df_name]\n        _add_new_columns(dataframe, metrics)\n        dataframe.loc[len(dataframe)] = metrics\n\n    def train_cb(self, param):\n        \"\"\"Callback funtion for training.\n        \"\"\"\n        if param.nbatch % self.frequent == 0:\n            self._process_batch(param, 'train')\n\n    def eval_cb(self, param):\n        \"\"\"Callback function for evaluation\n        \"\"\"\n        self._process_batch(param, 'eval')\n\n    def _process_batch(self, param, dataframe):\n        \"\"\"Update parameters for selected dataframe after a completed batch\n        Parameters\n        ----------\n        dataframe : pandas.DataFrame\n            Selected dataframe needs to be modified.\n        \"\"\"\n        now = time.time()\n        if param.eval_metric is not None:\n            metrics = dict(param.eval_metric.get_name_value())\n            param.eval_metric.reset()\n        else:\n            metrics = {}\n        # #11504\n        try:\n            speed = self.frequent / (now - self.last_time)\n        except ZeroDivisionError:\n            speed = float('inf')\n        metrics['batches_per_sec'] = speed * self.batch_size\n        metrics['records_per_sec'] = speed\n        metrics['elapsed'] = self.elapsed()\n        metrics['minibatch_count'] = param.nbatch\n        metrics['epoch'] = param.epoch\n        self.append_metrics(metrics, dataframe)\n        self.last_time = now\n\n    def epoch_cb(self):\n        \"\"\"Callback function after each epoch. Now it records each epoch time\n        and append it to epoch dataframe.\n        \"\"\"\n        metrics = {}\n        metrics['elapsed'] = self.elapsed()\n        now = datetime.datetime.now()\n        metrics['epoch_time'] = now - self.last_epoch_time\n        self.append_metrics(metrics, 'epoch')\n        self.last_epoch_time = now\n\n    def callback_args(self):\n        \"\"\"returns **kwargs parameters for model.fit()\n        to enable all callbacks.  e.g.\n        model.fit(X=train, eval_data=test, **pdlogger.callback_args())\n        \"\"\"\n        return {\n            'batch_end_callback': self.train_cb,\n            'eval_end_callback': self.eval_cb,\n            'epoch_end_callback': self.epoch_cb,\n        }\n\n\nclass LiveBokehChart(object):\n    \"\"\"Callback object that renders a bokeh chart in a jupyter notebook\n    that gets updated as the training run proceeds.\n\n    Requires a PandasLogger to collect the data it will render.\n\n    This is an abstract base-class.  Sub-classes define the specific chart.\n    \"\"\"\n    def __init__(self, pandas_logger, metric_name, display_freq=10,\n                 batch_size=None, frequent=50):\n        if pandas_logger:\n            self.pandas_logger = pandas_logger\n        else:\n            self.pandas_logger = PandasLogger(batch_size=batch_size, frequent=frequent)\n        self.display_freq = display_freq\n        self.last_update = time.time()\n        #NOTE: would be nice to auto-detect the metric_name if there's only one.\n        self.metric_name = metric_name\n        bokeh.io.output_notebook()\n        self.handle = self.setup_chart()\n\n    def setup_chart(self):\n        \"\"\"Render a bokeh object and return a handle to it.\n        \"\"\"\n        raise NotImplementedError(\"Incomplete base class: LiveBokehChart must be sub-classed\")\n\n    def update_chart_data(self):\n        \"\"\"Update the bokeh object with new data.\n        \"\"\"\n        raise NotImplementedError(\"Incomplete base class: LiveBokehChart must be sub-classed\")\n\n    def interval_elapsed(self):\n        \"\"\"Check whether it is time to update plot.\n        Returns\n        -------\n        Boolean value of whethe to update now\n        \"\"\"\n        return time.time() - self.last_update > self.display_freq\n\n    def _push_render(self):\n        \"\"\"Render the plot with bokeh.io and push to notebook.\n        \"\"\"\n        bokeh.io.push_notebook(handle=self.handle)\n        self.last_update = time.time()\n\n    def _do_update(self):\n        \"\"\"Update the plot chart data and render the updates.\n        \"\"\"\n        self.update_chart_data()\n        self._push_render()\n\n    def batch_cb(self, param):\n        \"\"\"Callback function after a completed batch.\n        \"\"\"\n        if self.interval_elapsed():\n            self._do_update()\n\n    def eval_cb(self, param):\n        \"\"\"Callback function after an evaluation.\n        \"\"\"\n        # After eval results, force an update.\n        self._do_update()\n\n    def callback_args(self):\n        \"\"\"returns **kwargs parameters for model.fit()\n        to enable all callbacks.  e.g.\n        model.fit(X=train, eval_data=test, **pdlogger.callback_args())\n        \"\"\"\n        return {\n            'batch_end_callback': self.batch_cb,\n            'eval_end_callback': self.eval_cb,\n        }\n\n\nclass LiveTimeSeries(LiveBokehChart):\n    \"\"\"Plot the elasped time during live learning.\n    \"\"\"\n    def __init__(self, **fig_params):\n        self.fig = bokeh.plotting.Figure(x_axis_type='datetime',\n                                         x_axis_label='Elapsed time', **fig_params)\n        super(LiveTimeSeries, self).__init__(None, None)  # TODO: clean up this class hierarchy\n\n    def setup_chart(self):\n        self.start_time = datetime.datetime.now()\n        self.x_axis_val = []\n        self.y_axis_val = []\n        self.fig.line(self.x_axis_val, self.y_axis_val)\n        return bokeh.plotting.show(self.fig, notebook_handle=True)\n\n    def elapsed(self):\n        \"\"\"Calculate elasped time from starting\n        \"\"\"\n        return datetime.datetime.now() - self.start_time\n\n    def update_chart_data(self, value):\n        self.x_axis_val.append(self.elapsed())\n        self.y_axis_val.append(value)\n        self._push_render()\n\n\nclass LiveLearningCurve(LiveBokehChart):\n    \"\"\"Draws a learning curve with training & validation metrics\n    over time as the network trains.\n    \"\"\"\n    def __init__(self, metric_name, display_freq=10, frequent=50):\n        self.frequent = frequent\n        self.start_time = datetime.datetime.now()\n        self._data = {\n            'train': {'elapsed': [],},\n            'eval': {'elapsed': [],},\n        }\n        super(LiveLearningCurve, self).__init__(None, metric_name, display_freq, frequent)\n\n    def setup_chart(self):\n        self.fig = bokeh.plotting.Figure(x_axis_type='datetime',\n                                         x_axis_label='Training time')\n        #TODO(leodirac): There's got to be a better way to\n        # get a bokeh plot to dynamically update as a pandas dataframe changes,\n        # instead of copying into a list.\n        # I can't figure it out though.  Ask a pyData expert.\n        self.x_axis_val1 = []\n        self.y_axis_val1 = []\n        self.train1 = self.fig.line(self.x_axis_val1, self.y_axis_val1, line_dash='dotted',\n                                    alpha=0.3, legend=\"train\")\n        self.train2 = self.fig.circle(self.x_axis_val1, self.y_axis_val1, size=1.5,\n                                      line_alpha=0.3, fill_alpha=0.3, legend=\"train\")\n        self.train2.visible = False  # Turn this on later.\n        self.x_axis_val2 = []\n        self.y_axis_val2 = []\n        self.valid1 = self.fig.line(self.x_axis_val2, self.y_axis_val2,\n                                    line_color='green',\n                                    line_width=2,\n                                    legend=\"validation\")\n        self.valid2 = self.fig.circle(self.x_axis_val2,\n                                      self.y_axis_val2,\n                                      line_color='green',\n                                      line_width=2, legend=None)\n        self.fig.legend.location = \"bottom_right\"\n        self.fig.yaxis.axis_label = self.metric_name\n        return bokeh.plotting.show(self.fig, notebook_handle=True)\n\n    def _do_update(self):\n        self.update_chart_data()\n        self._push_render()\n\n    def batch_cb(self, param):\n        if param.nbatch % self.frequent == 0:\n            self._process_batch(param, 'train')\n        if self.interval_elapsed():\n            self._do_update()\n\n    def eval_cb(self, param):\n        # After eval results, force an update.\n        self._process_batch(param, 'eval')\n        self._do_update()\n\n    def _process_batch(self, param, df_name):\n        \"\"\"Update selected dataframe after a completed batch\n        Parameters\n        ----------\n        df_name : str\n            Selected dataframe name needs to be modified.\n        \"\"\"\n        if param.eval_metric is not None:\n            metrics = dict(param.eval_metric.get_name_value())\n            param.eval_metric.reset()\n        else:\n            metrics = {}\n        metrics['elapsed'] = datetime.datetime.now() - self.start_time\n        for key, value in metrics.items():\n            if key not in self._data[df_name]:\n                self._data[df_name][key] = []\n            self._data[df_name][key].append(value)\n\n    def update_chart_data(self):\n        dataframe = self._data['train']\n        if len(dataframe['elapsed']):\n            _extend(self.x_axis_val1, dataframe['elapsed'])\n            _extend(self.y_axis_val1, dataframe[self.metric_name])\n        dataframe = self._data['eval']\n        if len(dataframe['elapsed']):\n            _extend(self.x_axis_val2, dataframe['elapsed'])\n            _extend(self.y_axis_val2, dataframe[self.metric_name])\n        if len(dataframe) > 10:\n            self.train1.visible = False\n            self.train2.visible = True\n\n\ndef args_wrapper(*args):\n    \"\"\"Generates callback arguments for model.fit()\n    for a set of callback objects.\n    Callback objects like PandasLogger(), LiveLearningCurve()\n    get passed in.  This assembles all their callback arguments.\n    \"\"\"\n    out = defaultdict(list)\n    for callback in args:\n        callback_args = callback.callback_args()\n        for k, v in callback_args.items():\n            out[k].append(v)\n    return dict(out)\n"
  },
  {
    "path": "python/mxnet/numpy/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"MXNet NumPy module.\"\"\"\n\n\nfrom . import random\nfrom . import linalg\nfrom .multiarray import *  # pylint: disable=wildcard-import\nfrom . import _op\nfrom . import _register\nfrom ._op import *  # pylint: disable=wildcard-import\nfrom .utils import *  # pylint: disable=wildcard-import\nfrom .function_base import *  # pylint: disable=wildcard-import\nfrom .stride_tricks import *  # pylint: disable=wildcard-import\nfrom .set_functions import *  # pylint: disable=wildcard-import\nfrom .type_functions import * # pylint: disable=wildcard-import\nfrom .io import *  # pylint: disable=wildcard-import\nfrom .arrayprint import *  # pylint: disable=wildcard-import\n\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/numpy/_op.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for registering numpy ops for imperative programming.\"\"\"\n\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/numpy/_register.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Registering ops in mxnet.numpy for imperative programming.\"\"\"\n\n\nfrom ..base import _init_np_op_module\nfrom ..ndarray.register import _make_ndarray_function\n\n_init_np_op_module(root_module_name='mxnet', np_module_name='numpy',\n                   mx_module_name=None, make_op_func=_make_ndarray_function)\n"
  },
  {
    "path": "python/mxnet/numpy/arrayprint.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"ndarray print format controller.\"\"\"\n\n\nimport numpy as onp\nfrom ..util import set_module\n\n__all__ = ['set_printoptions']\n\n\n@set_module('mxnet.numpy')\ndef set_printoptions(precision=None, threshold=None, **kwarg):\n    \"\"\"\n    Set printing options.\n\n    These options determine the way floating point numbers and arrays are displayed.\n\n    Parameters\n    ----------\n    precision : int or None, optional\n        Number of digits of precision for floating point output (default 8).\n        May be `None` if `floatmode` is not `fixed`, to print as many digits as\n        necessary to uniquely specify the value.\n    threshold : int, optional\n        Total number of array elements which trigger summarization\n        rather than full repr (default 1000).\n\n    Examples\n    --------\n    Floating point precision can be set:\n\n    >>> np.set_printoptions(precision=4)\n    >>> print(np.array([1.123456789]))\n    [ 1.1235]\n\n    Long arrays can be summarised:\n\n    >>> np.set_printoptions(threshold=5)\n    >>> print(np.arange(10))\n    [0. 1. 2. ... 7. 8. 9.]\n    \"\"\"\n    if kwarg:\n        raise NotImplementedError('mxnet.numpy.set_printoptions only supports parameters'\n                                  ' precision and threshold for now.')\n    onp.set_printoptions(precision=precision, threshold=threshold, **kwarg)\n"
  },
  {
    "path": "python/mxnet/numpy/fallback.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=undefined-all-variable, not-callable, cell-var-from-loop\n\"\"\"Operators that fallback to official NumPy implementation.\"\"\"\n\nimport sys\nfrom functools import wraps\nimport numpy as onp\n\nfallbacks = [\n    '__version__',\n    '_NoValue',\n    'allclose',\n    'alltrue',\n    'apply_along_axis',\n    'apply_over_axes',\n    'argpartition',\n    'argwhere',\n    'array_equal',\n    'array_equiv',\n    'choose',\n    'compress',\n    'corrcoef',\n    'correlate',\n    'count_nonzero',\n    'cov',\n    'cumprod',\n    'digitize',\n    'divmod',\n    'dtype',\n    'extract',\n    'float_power',\n    'frexp',\n    'heaviside',\n    'histogram2d',\n    'histogram_bin_edges',\n    'histogramdd',\n    'i0',\n    'in1d',\n    'intersect1d',\n    'isclose',\n    'isin',\n    'ix_',\n    'lexsort',\n    'min_scalar_type',\n    'mirr',\n    'modf',\n    'msort',\n    'nanargmax',\n    'nanargmin',\n    'nancumprod',\n    'nancumsum',\n    'nanmax',\n    'nanmedian',\n    'nanmin',\n    'nanpercentile',\n    'nanprod',\n    'nanquantile',\n    'nanstd',\n    'nansum',\n    'nanvar',\n    'ndim',\n    'npv',\n    'packbits',\n    'partition',\n    'piecewise',\n    'pmt',\n    'poly',\n    'polyadd',\n    'polydiv',\n    'polyfit',\n    'polyint',\n    'polymul',\n    'polysub',\n    'positive',\n    'ppmt',\n    'promote_types',\n    'ptp',\n    'pv',\n    'rate',\n    'real',\n    'roots',\n    'searchsorted',\n    'select',\n    'setdiff1d',\n    'setxor1d',\n    'signbit',\n    'size',\n    'spacing',\n    'take_along_axis',\n    'trapz',\n    'tril_indices_from',\n    'trim_zeros',\n    'union1d',\n    'unpackbits',\n    'unwrap',\n    'vander',\n]\n\nfallback_mod = sys.modules[__name__]\n\ndef get_func(obj, doc):\n    \"\"\"Get new numpy function with object and doc\"\"\"\n    @wraps(obj)\n    def wrapper(*args, **kwargs):\n        return obj(*args, **kwargs)\n    wrapper.__doc__ = doc\n    return wrapper\n\nfor obj_name in fallbacks:\n    onp_obj = getattr(onp, obj_name)\n    if callable(onp_obj):\n        new_fn_doc = onp_obj.__doc__\n        if obj_name in {'divmod', 'float_power', 'frexp', 'heaviside', 'modf', 'signbit', 'spacing'}:\n            # remove reference of kwargs doc and the reference to ufuncs\n            new_fn_doc = new_fn_doc.replace(\"**kwargs\\n    For other keyword-only arguments, see the\"\n                                            + \"\\n    :ref:`ufunc docs <ufuncs.kwargs>`.\", '')\n        elif obj_name == 'trapz':\n            # remove unused reference\n            new_fn_doc = new_fn_doc.replace(\n                '.. [1] Wikipedia page: https://en.wikipedia.org/wiki/Trapezoidal_rule', '')\n        elif obj_name == \"i0\":\n            # replace broken link\n            new_fn_doc = new_fn_doc.replace(\n                '.. [3] http://kobesearch.cpan.org/htdocs/Math-Cephes/Math/Cephes.html',\n                '.. [3] https://metacpan.org/pod/distribution/Math-Cephes/lib/Math/Cephes.pod \\\n                    #i0:-Modified-Bessel-function-of-order-zero')\n        setattr(fallback_mod, obj_name, get_func(onp_obj, new_fn_doc))\n    else:\n        setattr(fallback_mod, obj_name, onp_obj)\n\n__all__ = fallbacks\n"
  },
  {
    "path": "python/mxnet/numpy/fallback_linalg.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Operators that fallback to official NumPy implementation for np.linalg.\"\"\"\n\n\nimport numpy as onp\n\n\n__all__ = [\n    'cond',\n    'matrix_power',\n    'multi_dot'\n]\n\ncond = onp.linalg.cond\nmatrix_power = onp.linalg.matrix_power\nmulti_dot = onp.linalg.multi_dot\n"
  },
  {
    "path": "python/mxnet/numpy/function_base.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Numpy basic functions.\"\"\"\n\nfrom .stride_tricks import broadcast_arrays\n\n__all__ = ['meshgrid']\n\n\ndef meshgrid(*xi, **kwargs):\n    \"\"\"\n    Return coordinate matrices from coordinate vectors.\n\n    Make N-D coordinate arrays for vectorized evaluations of\n    N-D scalar/vector fields over N-D grids, given\n    one-dimensional coordinate arrays x1, x2,..., xn.\n\n    Parameters\n    ----------\n    x1, x2,..., xn : ndarrays\n        1-D arrays representing the coordinates of a grid.\n    indexing : {'xy', 'ij'}, optional\n        Cartesian ('xy', default) or matrix ('ij') indexing of output.\n        See Notes for more details.\n\n    sparse : bool, optional\n        If True a sparse grid is returned in order to conserve memory.\n        Default is False. Please note that `sparse=True` is currently\n        not supported.\n\n    copy : bool, optional\n        If False, a view into the original arrays are returned in order to\n        conserve memory.  Default is True. Please note that `copy=False`\n        is currently not supported.\n\n    Returns\n    -------\n    X1, X2,..., XN : ndarray\n        For vectors `x1`, `x2`,..., 'xn' with lengths ``Ni=len(xi)`` ,\n        return ``(N1, N2, N3,...Nn)`` shaped arrays if indexing='ij'\n        or ``(N2, N1, N3,...Nn)`` shaped arrays if indexing='xy'\n        with the elements of `xi` repeated to fill the matrix along\n        the first dimension for `x1`, the second for `x2` and so on.\n\n    Notes\n    -----\n    This function supports both indexing conventions through the indexing\n    keyword argument.  Giving the string 'ij' returns a meshgrid with\n    matrix indexing, while 'xy' returns a meshgrid with Cartesian indexing.\n    In the 2-D case with inputs of length M and N, the outputs are of shape\n    (N, M) for 'xy' indexing and (M, N) for 'ij' indexing.  In the 3-D case\n    with inputs of length M, N and P, outputs are of shape (N, M, P) for\n    'xy' indexing and (M, N, P) for 'ij' indexing.  The difference is\n    illustrated by the following code snippet::\n\n        xv, yv = np.meshgrid(x, y, sparse=False, indexing='ij')\n        for i in range(nx):\n            for j in range(ny):\n                # treat xv[i,j], yv[i,j]\n\n        xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy')\n        for i in range(nx):\n            for j in range(ny):\n                # treat xv[j,i], yv[j,i]\n\n    In the 1-D and 0-D case, the indexing and sparse keywords have no effect.\n    \"\"\"\n    ndim = len(xi)\n\n    copy_ = kwargs.pop('copy', True)\n    if not copy_:\n        raise NotImplementedError('copy=False is not implemented')\n    sparse = kwargs.pop('sparse', False)\n    if sparse:\n        raise NotImplementedError('sparse=False is not implemented')\n    indexing = kwargs.pop('indexing', 'xy')\n\n    if kwargs:\n        raise TypeError(f\"meshgrid() got an unexpected keyword argument '{list(kwargs)[0]}'\")\n\n    if indexing not in ['xy', 'ij']:\n        raise ValueError(\n            \"Valid values for `indexing` are 'xy' and 'ij'.\")\n\n    s0 = (1,) * ndim\n    output = [x.reshape(s0[:i] + (-1,) + s0[i + 1:])\n              for i, x in enumerate(xi)]\n\n    if indexing == 'xy' and ndim > 1:\n        # switch first and second axis\n        output[0] = output[0].reshape(1, -1, *s0[2:])\n        output[1] = output[1].reshape(-1, 1, *s0[2:])\n\n    if not sparse:\n        # Return the full N-D matrix (not only the 1-D vector)\n        output = broadcast_arrays(*output)\n\n    return output\n"
  },
  {
    "path": "python/mxnet/numpy/io.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\n\"\"\"I/O functions for ndarrays.\"\"\"\nimport numpy as onp\nfrom ..device import current_device\nfrom .multiarray import array\n\n__all__ = ['genfromtxt']\n\n\n# TODO(junwu): Add doc\ndef genfromtxt(*args, **kwargs):\n    \"\"\"This is a wrapper of the official NumPy's `genfromtxt` function.\n    Please refer to the documentation here\n    https://docs.scipy.org/doc/numpy/reference/generated/numpy.genfromtxt.html.\n\n    Notes\n    -----\n    This function has added an additional parameter `device` which allows to create\n    ndarrays on the user-specified device.\n    \"\"\"\n    device = kwargs.pop('device', current_device())\n    if device is None:\n        device = current_device()\n    ret = onp.genfromtxt(*args, **kwargs)\n    return array(ret, dtype=ret.dtype, device=device)\n"
  },
  {
    "path": "python/mxnet/numpy/linalg.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for ops used in imperative programming.\"\"\"\n\nfrom functools import reduce\n\nfrom ..ndarray import numpy as _mx_nd_np\nfrom ..util import wrap_data_api_linalg_func\nfrom .fallback_linalg import *  # pylint: disable=wildcard-import,unused-wildcard-import\nfrom . import fallback_linalg\n\n__all__ = ['norm', 'svd', 'cholesky', 'qr', 'inv', 'det', 'slogdet', 'solve', 'tensorinv', 'tensorsolve',\n           'pinv', 'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq', 'matrix_rank', 'cross', 'diagonal', 'outer',\n           'tensordot', 'trace', 'matrix_transpose', 'vecdot', 'svdvals', 'vector_norm', 'matrix_norm']\n\n__all__ += fallback_linalg.__all__\n\n\n@wrap_data_api_linalg_func\ndef matrix_rank(M, rtol=None, hermitian=False):\n    r\"\"\"\n    Return matrix rank of array using SVD method\n\n    Rank of the array is the number of singular values of the array that are\n    greater than `rtol`.\n\n    Notes\n    -----\n    `rtol` param is requested in array-api-standard in\n    https://data-apis.org/array-api/latest/extensions/generated/signatures.linalg.matrix_rank.html\n    instead of a parameter in official NumPy operator.\n\n    Parameters\n    ----------\n    M : {(M,), (..., M, N)} ndarray\n        Input vector or stack of matrices.\n    rtol : (...) ndarray, float, optional\n        Threshold below which SVD values are considered zero. If `rtol` is\n        None, and ``S`` is an array with singular values for `M`, and\n        ``eps`` is the epsilon value for datatype of ``S``, then `rtol` is\n        set to ``S.max() * max(M.shape) * eps``.\n    hermitian : bool, optional\n        If True, `M` is assumed to be Hermitian (symmetric if real-valued),\n        enabling a more efficient method for finding singular values.\n        Default: False.\n\n    Returns\n    -------\n    rank : (...) ndarray\n        Rank of M.\n\n    Examples\n    --------\n    >>> from mxnet import np\n    >>> np.linalg.matrix_rank(np.eye(4)) # Full rank matrix\n    4\n    >>> I=np.eye(4); I[-1,-1] = 0. # rank deficient matrix\n    >>> np.linalg.matrix_rank(I)\n    3\n    >>> np.linalg.matrix_rank(np.ones((4,))) # 1 dimension - rank 1 unless all 0\n    1\n    >>> np.linalg.matrix_rank(np.zeros((4,)))\n    0\n    \"\"\"\n    return _mx_nd_np.linalg.matrix_rank(M, rtol, hermitian)\n\n\ndef matrix_transpose(a):\n    r\"\"\"\n    Transposes a matrix (or a stack of matrices) `a`.\n\n    Notes\n    -----\n    `matrix_transpose` is new in array API spec:\n    https://data-apis.org/array-api/latest/extensions/linear_algebra_functions.html#linalg-matrix-transpose-x\n    instead of an official NumPy operator. Unlike transpose, it only transposes the last two axes.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array having shape (..., M, N) and whose innermost two dimensions form MxN matrices.\n\n    Returns\n    ----------\n    out : ndarray\n        An array containing the transpose for each matrix and having shape (..., N, M).\n        The returned array must have the same data type as `a`.\n\n    Examples\n    --------\n    >>> x = np.arange(4).reshape((2,2))\n    >>> x\n    array([[0., 1.],\n           [2., 3.]])\n    >>> np.linalg.matrix_transpose(x)\n    array([[0., 2.],\n           [1., 3.]])\n    >>> x = np.ones((1, 2, 3))\n    >>> np.linalg.matrix_transpose(x)\n    array([[[1., 1.],\n            [1., 1.],\n            [1., 1.]]])\n    \"\"\"\n    if a.ndim < 2:\n        raise ValueError(\"x must be at least 2-dimensional for matrix_transpose\")\n    return _mx_nd_np.swapaxes(a, -1, -2)\n\n\ndef trace(a, offset=0):\n    r\"\"\"\n    Returns a tensor contraction of `a` and `b` over specific axes.\n\n    Notes\n    -----\n    `trace` is an alias for `trace`. It is a standard API in\n    https://data-apis.org/array-api/latest/extensions/linear_algebra_functions.html#linalg-trace-x-offset-0\n    instead of an official NumPy operator.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array having shape (..., M, N) and whose innermost two dimensions form MxN matrices.\n        Should have a numeric data type.\n    offset : int\n        Offset specifying the off-diagonal relative to the main diagonal.\n\n        offset = 0 : the main diagonal.\n        offset > 0 : off-diagonal above the main diagonal.\n        offset < 0 : off-diagonal below the main diagonal.\n\n        Default: 0.\n\n    Returns\n    ----------\n    out : ndarray\n        An array containing the traces and whose shape is determined by removing the last two dimensions and storing\n        the traces in the last array dimension. For example, if `a` has rank `k` and shape `(I, J, K, ..., L, M, N)`,\n        then an output array has rank `k-2` and shape `(I, J, K, ..., L)`\n        where: `out[i, j, k, ..., l] = trace(a[i, j, k, ..., l, :, :])`\n        The returned array must have the same data type as `a`.\n\n    Examples\n    --------\n    >>> x = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]])\n    >>> np.linalg.trace(x)\n    array(3.)\n    >>> x = np.arange(8).reshape((2, 2, 2))\n    >>> np.linalg.trace(x)\n    array([6., 8.])\n    >>> x = np.arange(24).reshape((2, 2, 2, 3))\n    >>> np.linalg.trace(x).shape\n    (2, 3)\n    >>> np.linalg.trace(x)\n    array([[18., 20., 22.],\n        [24., 26., 28.]])\n    \"\"\"\n    # axis1, axis2: defaults are the first two axes of `a`.\n    return _mx_nd_np.trace(a, offset=offset, axis1=0, axis2=1, out=None)\n\n\ndef tensordot(a, b, axes=2):\n    r\"\"\"\n    Returns a tensor contraction of `a` and `b` over specific axes.\n\n    Notes\n    -----\n    `tensordot` is an alias for `tensordot`. It is a standard API in\n    https://data-apis.org/array-api/latest/extensions/linear_algebra_functions.html#linalg-tensordot-x1-x2-axes-2\n    instead of an official NumPy operator.\n\n    Parameters\n    ----------\n    a : ndarray\n        First input array. Should have a numeric data type.\n    b : ndarray\n        Second input array. Must be compatible with `a` (see Broadcasting). Should have a numeric data type.\n    axes : int, tuple\n        Number of axes to contract or explicit sequences of axes for `a` and `b`, respectively.\n        If axes is an int equal to `N` , then contraction must be performed over the last `N` axes of `a`\n        and the first `N` axes of `b` in order.\n        The size of each corresponding axis (dimension) must match. Must be nonnegative.\n\n        If N equals 0 , the result is the tensor (outer) product.\n        If N equals 1 , the result is the tensor dot product.\n        If N equals 2 , the result is the tensor double contraction (default).\n\n        Default: 2.\n\n    Returns\n    ----------\n    out : ndarray\n        An array containing the tensor contraction whose shape consists of the non-contracted axes (dimensions) of the\n        first array `a`, followed by the non-contracted axes (dimensions) of the second array `b`.\n\n    Examples\n    --------\n    >>> x = np.arange(60.).reshape(3,4,5)\n    >>> y = np.arange(24.).reshape(4,3,2)\n    >>> z = np.linalg.tensordot(x, y, axes=([1,0],[0,1]))\n    >>> z.shape\n    (5, 2)\n    >>> z\n    array([[ 4400.,  4730.],\n           [ 4532.,  4874.],\n           [ 4664.,  5018.],\n           [ 4796.,  5162.],\n           [ 4928.,  5306.]])\n    \"\"\"\n    return _mx_nd_np.tensordot(a, b, axes)\n\n\ndef diagonal(a, offset=0):\n    r\"\"\"\n    Returns the specified diagonals of a matrix (or a stack of matrices) `a`.\n\n    Notes\n    -----\n    `diagonal` is an alias for `diagonal`. It is a standard API in\n    https://data-apis.org/array-api/latest/extensions/linear_algebra_functions.html#linalg-diagonal-x-offset-0\n    instead of an official NumPy operator.\n\n    Parameters\n    ----------\n    a : ndarray\n        The array to apply diag method.\n    offset : int\n        Extracts or constructs kth diagonal given input array.\n        Offset specifying the off-diagonal relative to the main diagonal.\n\n        offset = 0 : the main diagonal.\n        offset > 0 : off-diagonal above the main diagonal.\n        offset < 0 : off-diagonal below the main diagonal.\n\n        Default: 0.\n\n    Returns\n    ----------\n    out : ndarray\n        An array containing the diagonals and whose shape is determined by removing the last two dimensions and\n        appending a dimension equal to the size of the resulting diagonals.\n        The returned array must have the same data type as a.\n\n    Examples\n    --------\n    >>> x = np.arange(9).reshape((3,3))\n    >>> x\n    array([[0., 1., 2.],\n           [3., 4., 5.],\n           [6., 7., 8.]])\n    >>> np.linalg.diagonal(x)\n    array([0., 4., 8.])\n    >>> np.linalg.diagonal(x, offset=1)\n    array([1., 5.])\n    >>> np.linalg.diagonal(x, offset=-1)\n    array([3., 7.])\n    \"\"\"\n    return _mx_nd_np.diag(a, k=offset)\n\n\ndef cross(a, b, axis=-1):\n    r\"\"\"\n    Returns the cross product of 3-element vectors.\n\n    If `a` and `b` are multi-dimensional arrays (i.e., both have a rank greater than 1),\n    then the cross-product of each pair of corresponding 3-element vectors is independently computed.\n\n    Notes\n    -----\n    `cross` is an alias for `cross`. It is a standard API in\n    https://data-apis.org/array-api/latest/extensions/linear_algebra_functions.html#linalg-cross-x1-x2-axis-1\n    instead of an official NumPy operator.\n\n    Parameters\n    ----------\n    a : ndarray\n        First input array. Should have a numeric data type.\n    b : ndarray\n        Second input array. Must have the same shape as a. Should have a numeric data type.\n    axis : int\n        If defined, the axis of `a` and `b` that defines the vector(s) and cross product(s).\n\n        Default: -1.\n\n    Returns\n    -------\n    out : (...) ndarray\n        An array containing the cross products.\n\n    Examples\n    --------\n    Vector cross-product.\n\n    >>> x = np.array([1., 2., 3.])\n    >>> y = np.array([4., 5., 6.])\n    >>> np.linalg.cross(x, y)\n    array([-3.,  6., -3.])\n\n    One vector with dimension 2.\n\n    >>> x = np.array([1., 2.])\n    >>> y = np.array([4., 5., 6.])\n    >>> np.linalg.cross(x, y)\n    array([12., -6., -3.])\n\n    Equivalently:\n\n    >>> x = np.array([1., 2., 0.])\n    >>> y = np.array([4., 5., 6.])\n    >>>np.linalg.cross(x, y)\n    array([12., -6., -3.])\n\n    Both vectors with dimension 2.\n\n    >>> x = np.array([1., 2.])\n    >>> y = np.array([4., 5.])\n    >>> np.linalg.cross(x, y)\n    array(-3.)\n\n    Multiple vector cross-products. Note that the direction of the cross\n    product vector is defined by the `right-hand rule`.\n\n    >>> x = np.array([[1., 2., 3.], [4., 5., 6.]])\n    >>> y = np.array([[4., 5., 6.], [1., 2., 3.]])\n    >>> np.linalg.cross(x, y)\n    array([[-3.,  6., -3.],\n           [ 3., -6.,  3.]])\n    \"\"\"\n    # For a given API standard, the axis of axisa, axisb, axisc are equal to the axis\n    return _mx_nd_np.cross(a, b, axisa=axis, axisb=axis, axisc=axis, axis=axis)\n\n\ndef outer(a, b):\n    r\"\"\"\n    Computes the outer product of two vectors `a` and `b`.\n\n    Notes\n    -----\n    `outer` is an alias for `outer`. It is a standard API in\n    https://data-apis.org/array-api/latest/extensions/linear_algebra_functions.html#linalg-outer-x1-x2\n    instead of an official NumPy operator.\n\n    Parameters\n    ----------\n    a : ndarray\n        One-dimensional input array of size `N` . Should have a numeric data type.\n    b : ndarray\n        One-dimensional input array of size `M` . Should have a numeric data type.\n\n    Returns\n    -------\n    out : ndarray\n        A two-dimensional array containing the outer product and whose shape is `(N, M)`.\n        The returned array must have a data type determined by Type Promotion Rules.\n\n    Examples\n    --------\n    Make a (*very* coarse) grid for computing a Mandelbrot set:\n\n    >>> x = np.linalg.outer(np.ones((5,)), np.linspace(-2, 2, 5))\n    >>> x\n    array([[-2., -1.,  0.,  1.,  2.],\n           [-2., -1.,  0.,  1.,  2.],\n           [-2., -1.,  0.,  1.,  2.],\n           [-2., -1.,  0.,  1.,  2.],\n           [-2., -1.,  0.,  1.,  2.]])\n    \"\"\"\n    return _mx_nd_np.tensordot(a.flatten(), b.flatten(), 0)\n\n\ndef vecdot(a, b, axis=None):\n    r\"\"\"\n    Return the dot product of two vectors.\n    Note that `vecdot` handles multidimensional arrays differently than `dot`:\n    it does *not* perform a matrix product, but flattens input arguments\n    to 1-D vectors first. Consequently, it should only be used for vectors.\n\n    Notes\n    ----------\n    `vecdot` is a alias for `vdot`. It is a standard API in\n    https://data-apis.org/array-api/latest/API_specification/linear_algebra_functions.html#vecdot-x1-x2-axis-1\n    instead of an official NumPy operator.\n\n    Parameters\n    ----------\n    a : ndarray\n        First argument to the dot product.\n    b : ndarray\n        Second argument to the dot product.\n    axis : axis over which to compute the dot product. Must be an integer on\n        the interval [-N, N) , where N is the rank (number of dimensions) of\n        the shape determined according to Broadcasting . If specified as a\n        negative integer, the function must determine the axis along which\n        to compute the dot product by counting backward from the last dimension\n        (where -1 refers to the last dimension). If None , the function must\n        compute the dot product over the last axis. Default: None .\n\n    Returns\n    -------\n    output : ndarray\n        Dot product of `a` and `b`.\n\n    See Also\n    --------\n    dot : Return the dot product without using the complex conjugate of the\n        first argument.\n\n    Examples\n    --------\n    Note that higher-dimensional arrays are flattened!\n\n    >>> a = np.array([[1, 4], [5, 6]])\n    >>> b = np.array([[4, 1], [2, 2]])\n    >>> np.linalg.vecdot(a, b)\n    array(30.)\n    >>> np.linalg.vecdot(b, a)\n    array(30.)\n    >>> 1*4 + 4*1 + 5*2 + 6*2\n    30\n    \"\"\"\n    return _mx_nd_np.tensordot(a.flatten(), b.flatten(), axis)\n\n\ndef lstsq(a, b, rcond='warn'):\n    r\"\"\"\n    Return the least-squares solution to a linear matrix equation.\n\n    Solves the equation :math:`a x = b` by computing a vector `x` that\n    minimizes the squared Euclidean 2-norm :math:`\\| b - a x \\|^2_2`.\n    The equation may be under-, well-, or over-determined (i.e., the\n    number of linearly independent rows of `a` can be less than, equal\n    to, or greater than its number of linearly independent columns).\n    If `a` is square and of full rank, then `x` (but for round-off error)\n    is the \"exact\" solution of the equation.\n\n    Parameters\n    ----------\n    a : (M, N) ndarray\n        \"Coefficient\" matrix.\n    b : {(M,), (M, K)} ndarray\n        Ordinate or \"dependent variable\" values. If `b` is two-dimensional,\n        the least-squares solution is calculated for each of the `K` columns\n        of `b`.\n    rcond : float, optional\n        Cut-off ratio for small singular values of `a`.\n        For the purposes of rank determination, singular values are treated\n        as zero if they are smaller than `rcond` times the largest singular\n        value of `a`\n        The default of ``warn`` or ``-1`` will use the machine precision as\n        `rcond` parameter. The default of ``None`` will use the machine\n        precision times `max(M, N)`.\n\n    Returns\n    -------\n    x : {(N,), (N, K)} ndarray\n        Least-squares solution. If `b` is two-dimensional,\n        the solutions are in the `K` columns of `x`.\n    residuals : {(1,), (K,), (0,)} ndarray\n        Sums of residuals.\n        Squared Euclidean 2-norm for each column in ``b - a*x``.\n        If the rank of `a` is < N or M <= N, this is an empty array.\n        If `b` is 1-dimensional, this is a (1,) shape array.\n        Otherwise the shape is (K,).\n    rank : int\n        Rank of matrix `a`.\n    s : (min(M, N),) ndarray\n        Singular values of `a`.\n\n    Raises\n    ------\n    MXNetError\n        If computation does not converge.\n\n    Notes\n    -----\n    If `b` is a matrix, then all array results are returned as matrices.\n\n    Examples\n    --------\n    >>> x = np.array([0, 1, 2, 3])\n    >>> y = np.array([-1, 0.2, 0.9, 2.1])\n    >>> A = np.vstack([x, np.ones(len(x))]).T\n    >>> A\n    array([[ 0.,  1.],\n           [ 1.,  1.],\n           [ 2.,  1.],\n           [ 3.,  1.]])\n    >>> m, c = np.linalg.lstsq(A, y, rcond=None)[0]\n    >>> m, c\n    (1.0 -0.95) # may vary\n    \"\"\"\n    return _mx_nd_np.linalg.lstsq(a, b, rcond)\n\n\n@wrap_data_api_linalg_func\ndef pinv(a, rtol=None, hermitian=False):\n    r\"\"\"\n    Compute the (Moore-Penrose) pseudo-inverse of a matrix.\n\n    Calculate the generalized inverse of a matrix using its\n    singular-value decomposition (SVD) and including all\n    *large* singular values.\n\n    Notes\n    -----\n    `rtol` param is requested in array-api-standard in\n    https://data-apis.org/array-api/latest/extensions/generated/signatures.linalg.pinv.html\n    instead of a parameter in official NumPy operator.\n\n    Parameters\n    ----------\n    a : (..., M, N) ndarray\n        Matrix or stack of matrices to be pseudo-inverted.\n    rtol : (...) {float or ndarray of float}, optional\n        Cutoff for small singular values.\n        Singular values less than or equal to\n        ``rtol * largest_singular_value`` are set to zero.\n        Broadcasts against the stack of matrices.\n    hermitian : bool, optional\n        If True, `a` is assumed to be Hermitian (symmetric if real-valued),\n        enabling a more efficient method for finding singular values.\n        Defaults to False.\n\n    Returns\n    -------\n    B : (..., N, M) ndarray\n        The pseudo-inverse of `a`. If `a` is a `matrix` instance, then so\n        is `B`.\n\n    Raises\n    ------\n    MXNetError\n        If the SVD computation does not converge.\n\n    Notes\n    -----\n    The pseudo-inverse of a matrix A, denoted :math:`A^+`, is\n    defined as: \"the matrix that 'solves' [the least-squares problem]\n    :math:`Ax = b`,\" i.e., if :math:`\\\\bar{x}` is said solution, then\n    :math:`A^+` is that matrix such that :math:`\\\\bar{x} = A^+b`.\n\n    It can be shown that if :math:`Q_1 \\\\Sigma Q_2^T = A` is the singular\n    value decomposition of A, then\n    :math:`A^+ = Q_2 \\\\Sigma^+ Q_1^T`, where :math:`Q_{1,2}` are\n    orthogonal matrices, :math:`\\\\Sigma` is a diagonal matrix consisting\n    of A's so-called singular values, (followed, typically, by\n    zeros), and then :math:`\\\\Sigma^+` is simply the diagonal matrix\n    consisting of the reciprocals of A's singular values\n    (again, followed by zeros). [1]_\n\n    References\n    ----------\n    .. [1] G. Strang, *Linear Algebra and Its Applications*, 2nd Ed., Orlando,\n           FL, Academic Press, Inc., 1980, pp. 139-142.\n\n    Examples\n    --------\n    The following example checks that ``a * a+ * a == a`` and\n    ``a+ * a * a+ == a+``:\n    >>> a = np.random.randn(2, 3)\n    >>> pinv_a = np.linalg.pinv(a)\n    >>> (a - np.dot(a, np.dot(pinv_a, a))).sum()\n    array(0.)\n    >>> (pinv_a - np.dot(pinv_a, np.dot(a, pinv_a))).sum()\n    array(0.)\n    \"\"\"\n    return _mx_nd_np.linalg.pinv(a, rtol, hermitian)\n\n\ndef norm(x, ord=None, axis=None, keepdims=False):\n    r\"\"\"\n    Matrix or vector norm.\n\n    This function can only support Frobenius norm for now.\n    The Frobenius norm is given by [1]_:\n\n        :math:`||A||_F = [\\sum_{i,j} abs(a_{i,j})^2]^{1/2}`\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array.\n    ord : {'fro'}, optional\n        Order of the norm.\n    axis : {int, 2-tuple of ints, None}, optional\n        If `axis` is an integer, it specifies the axis of `x` along which to\n        compute the vector norms.  If `axis` is a 2-tuple, it specifies the\n        axes that hold 2-D matrices, and the matrix norms of these matrices\n        are computed.  If `axis` is None, the norm of the whole ndarray is\n        returned.\n\n    keepdims : bool, optional\n        If this is set to True, the axes which are normed over are left in the\n        result as dimensions with size one.  With this option the result will\n        broadcast correctly against the original `x`.\n\n    Returns\n    -------\n    n : float or ndarray\n        Norm of the matrix or vector(s).\n\n    Notes\n    -----\n    This operator differs from NumPy in the aspect that it always returns a\n    zero-dim tensor for the cases where Python float values are expected\n    in NumPy.\n\n    References\n    ----------\n    .. [1] G. H. Golub and C. F. Van Loan, *Matrix Computations*,\n           Baltimore, MD, Johns Hopkins University Press, 1985, pg. 15\n\n    Examples\n    --------\n    >>> from numpy import linalg as LA\n    >>> a = np.arange(9) - 4\n    >>> a\n    array([-4., -3., -2., -1.,  0.,  1.,  2.,  3.,  4.])\n    >>> b = a.reshape((3, 3))\n    >>> b\n    array([[-4., -3., -2.],\n           [-1.,  0.,  1.],\n           [ 2.,  3.,  4.]])\n    >>> LA.norm(a)\n    array(7.745967)\n    >>>\n    >>> LA.norm(b)\n    array(7.745967)\n    >>> LA.norm(b, 'fro')\n    array(7.745967)\n    \"\"\"\n    return _mx_nd_np.linalg.norm(x, ord, axis, keepdims)\n\n\ndef vector_norm(x, ord=None, axis=None, keepdims=False):\n    r\"\"\"\n    Computes the vector norm of a vector (or batch of vectors) `x`.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array. Should have a floating-point data type.\n    ord : {non-zero int, inf, -inf}, optional\n        Order of the norm.\n    axis : {int, n-tuple of ints, None}, optional\n        If `axis` is an integer, it specifies the axis of `x` along which to\n        compute the vector norms.  If `axis` is a n-tuple, it specifies the\n        axes along which to compute batched vector norms. If `axis` is None,\n        the norm of the whole ndarray is returned.\n    keepdims : bool, optional\n        If this is set to True, the axes which are normed over are left in the\n        result as dimensions with size one.  With this option the result will\n        broadcast correctly against the original `x`.\n\n    Returns\n    -------\n    n : float or ndarray\n        Norm of the vector(s).\n\n    Notes\n    -----\n    `vector_norm` is a standard API in\n    https://data-apis.org/array-api/latest/extensions/linear_algebra_functions.html#linalg-vector-norm-x-axis-none-keepdims-false-ord-2\n    instead of an official NumPy operator.\n\n    \"\"\"\n    if axis is None:\n        x = x.flatten()\n        axis = 0\n    elif isinstance(axis, tuple):\n        rest = tuple(i for i in range(x.ndim) if i not in axis)\n        newshape = axis + rest\n        x = _mx_nd_np.transpose(x, newshape).\\\n            reshape((reduce(lambda a, b: a * b, [x.shape[a] for a in axis]),\\\n                     *[x.shape[i] for i in rest]))\n        axis = 0\n    return _mx_nd_np.linalg.norm(x, axis=axis, keepdims=keepdims, ord=ord)\n\n\ndef matrix_norm(x, ord='fro', axis=(-2, -1), keepdims=False):\n    r\"\"\"\n    Computes the matrix norm of a matrix (or a stack of matrices) `x`.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array. Should have a floating-point data type.\n    ord : {non-zero int, inf, -inf, ‘fro’, ‘nuc’}, optional\n        Order of the norm.\n    axis : {2-tuple of ints}\n        a 2-tuple which specifies the axes (dimensions) defining two-dimensional\n        matrices for which to compute matrix norms.\n    keepdims : bool, optional\n        If this is set to True, the axes which are normed over are left in the\n        result as dimensions with size one.  With this option the result will\n        broadcast correctly against the original `x`.\n\n    Returns\n    -------\n    n : float or ndarray\n        Norm of the matrix.\n\n    Notes\n    -----\n    `matrix_norm` is a standard API in\n    https://data-apis.org/array-api/latest/extensions/linear_algebra_functions.html#linalg-matrix-norm-x-axis-2-1-keepdims-false-ord-fro\n    instead of an official NumPy operator.\n\n    \"\"\"\n    if isinstance(axis, tuple) and len(axis) == 2:\n        return _mx_nd_np.linalg.norm(x, axis=axis, keepdims=keepdims, ord=ord)\n    raise ValueError(\"The axis of matrix_norm must be a 2-tuple of ints\")\n\n\ndef svd(a):\n    r\"\"\"\n    Singular Value Decomposition.\n\n    When `a` is a 2D array, it is factorized as ``ut @ np.diag(s) @ v``,\n    where `ut` and `v` are 2D orthonormal arrays and `s` is a 1D\n    array of `a`'s singular values. When `a` is higher-dimensional, SVD is\n    applied in stacked mode as explained below.\n\n    Parameters\n    ----------\n    a : (..., M, N) ndarray\n        A real array with ``a.ndim >= 2`` and ``M <= N``.\n\n    Returns\n    -------\n    ut: (..., M, M) ndarray\n        Orthonormal array(s). The first ``a.ndim - 2`` dimensions have the same\n        size as those of the input `a`.\n    s : (..., M) ndarray\n        Vector(s) with the singular values, within each vector sorted in\n        descending order. The first ``a.ndim - 2`` dimensions have the same\n        size as those of the input `a`.\n    v : (..., M, N) ndarray\n        Orthonormal array(s). The first ``a.ndim - 2`` dimensions have the same\n        size as those of the input `a`.\n\n    .. note::\n       The decomposition is performed using LAPACK routine ``_gesvd``.\n\n       SVD is usually described for the factorization of a 2D matrix :math:`A`.\n       The higher-dimensional case will be discussed below. In the 2D case, SVD is\n       written as :math:`A = U^T S V`, where :math:`A = a`, :math:`U^T = ut`,\n       :math:`S= \\mathtt{np.diag}(s)` and :math:`V = v`. The 1D array `s`\n       contains the singular values of `a` and `ut` and `v` are orthonormal. The rows\n       of `v` are the eigenvectors of :math:`A^T A` and the columns of `ut` are\n       the eigenvectors of :math:`A A^T`. In both cases the corresponding\n       (possibly non-zero) eigenvalues are given by ``s**2``.\n\n       The sign of rows of `u` and `v` are determined as described in\n       `Auto-Differentiating Linear Algebra <https://arxiv.org/pdf/1710.08717.pdf>`_.\n\n       If `a` has more than two dimensions, then broadcasting rules apply.\n       This means that SVD is working in \"stacked\" mode: it iterates over\n       all indices of the first ``a.ndim - 2`` dimensions and for each\n       combination SVD is applied to the last two indices. The matrix `a`\n       can be reconstructed from the decomposition with either\n       ``(ut * s[..., None, :]) @ v`` or\n       ``ut @ (s[..., None] * v)``. (The ``@`` operator denotes batch matrix multiplication)\n\n       This function differs from the original `numpy.linalg.svd\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.svd.html>`_ in\n       the following way(s):\n       * The sign of rows of `u` and `v` may differ.\n       * Does not support complex input.\n\n    Examples\n    --------\n    >>> a = np.arange(54).reshape(6, 9)\n    >>> ut, s, v = np.linalg.svd(a)\n    >>> ut.shape, s.shape, v.shape\n    ((6, 6), (6,), (6, 9))\n    >>> s = s.reshape(6, 1)\n    >>> ret = np.dot(ut, s * v)\n    >>> (ret - a > 1e-3).sum()\n    array(0.)\n    >>> (ret - a < -1e-3).sum()\n    array(0.)\n    \"\"\"\n    return _mx_nd_np.linalg.svd(a)\n\n\ndef svdvals(a):\n    r\"\"\"\n    Computes the singular values of a matrix (or a stack of matrices) `x`.\n\n    Parameters\n    ----------\n    a : (..., M, N) ndarray\n        A real array with ``a.ndim >= 2`` and ``M <= N``.\n\n    Returns\n    -------\n    out : (..., M) ndarray\n        Vector(s) with the singular values, within each vector sorted in\n        descending order. The first ``a.ndim - 2`` dimensions have the same\n        size as those of the input `a`.\n\n    .. note::\n       `svdvals` is a standard api in\n       https://data-apis.org/array-api/latest/extensions/linear_algebra_functions.html#linalg-svdvals-x\n       instead of an official NumPy operator.\n    \"\"\"\n    _, s, _ = _mx_nd_np.linalg.svd(a)\n    return s\n\n\ndef cholesky(a, upper=False):\n    r\"\"\"\n    Cholesky decomposition.\n\n    Notes\n    -----\n    `upper` param is requested by API standardization in\n    https://data-apis.org/array-api/latest/extensions/generated/signatures.linalg.cholesky.html\n    instead of parameter in official NumPy operator.\n\n    Return the Cholesky decomposition, `L * L.T`, of the square matrix `a`,\n    where `L` is lower-triangular and .T is the transpose operator. `a` must be\n    symmetric and positive-definite. Only `L` is actually returned. Complex-valued\n    input is currently not supported.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Symmetric, positive-definite input matrix.\n    upper : bool\n        If `True`, the result must be the upper-triangular Cholesky factor.\n        If `False`, the result must be the lower-triangular Cholesky factor.\n        Default: `False`.\n\n    Returns\n    -------\n    L : (..., M, M) ndarray\n        Lower-triangular Cholesky factor of `a`.\n\n    Raises\n    ------\n    MXNetError\n        If the decomposition fails, for example, if `a` is not positive-definite.\n\n    Notes\n    -----\n    Broadcasting rules apply.\n\n    The Cholesky decomposition is often used as a fast way of solving\n\n    .. math:: A \\mathbf{x} = \\mathbf{b}\n\n    (when `A` is both symmetric and positive-definite).\n\n    First, we solve for :math:`\\mathbf{y}` in\n\n    .. math:: L \\mathbf{y} = \\mathbf{b},\n\n    and then for :math:`\\mathbf{x}` in\n\n    .. math:: L.T \\mathbf{x} = \\mathbf{y}.\n\n    Examples\n    --------\n    >>> A = np.array([[16, 4], [4, 10]])\n    >>> A\n    array([[16.,  4.],\n           [ 4., 10.]])\n    >>> L = np.linalg.cholesky(A)\n    >>> L\n    array([[4., 0.],\n           [1., 3.]])\n    >>> np.dot(L, L.T)\n    array([[16.,  4.],\n           [ 4., 10.]])\n    \"\"\"\n    return _mx_nd_np.linalg.cholesky(a, upper)\n\n\ndef qr(a, mode='reduced'):\n    r\"\"\"\n    Compute the qr factorization of a matrix a.\n    Factor the matrix a as qr, where q is orthonormal and r is upper-triangular.\n\n    Parameters\n    ----------\n    a : (..., M, N) ndarray\n        Matrix or stack of matrices to be qr factored.\n    mode: {‘reduced’, ‘complete’, ‘r’, ‘raw’, ‘full’, ‘economic’}, optional\n        Only default mode, 'reduced', is implemented. If K = min(M, N), then\n        * 'reduced’ : returns q, r with dimensions (M, K), (K, N) (default)\n\n    Returns\n    -------\n    q : (..., M, K) ndarray\n        A matrix or stack of matrices with K orthonormal columns, with K = min(M, N).\n    r : (..., K, N) ndarray\n        A matrix or stack of upper triangular matrices.\n\n    Raises\n    ------\n    MXNetError\n        If factoring fails.\n\n    Notes\n    -----\n    Currently, the gradient for the QR factorization is well-defined\n    only when the first K columns of the input matrix are linearly independent.\n\n    Examples\n    --------\n    >>> from mxnet import np\n    >>> a = np.random.uniform(-10, 10, (2, 2))\n    >>> q, r = np.linalg.qr(a)\n    >>> q\n    array([[-0.22121978, -0.97522414],\n           [-0.97522414,  0.22121954]])\n    >>> r\n    array([[-4.4131265 , -7.1255064 ],\n           [ 0.        , -0.28771925]])\n    >>> a = np.random.uniform(-10, 10, (2, 3))\n    >>> q, r = np.linalg.qr(a)\n    >>> q\n    array([[-0.28376842, -0.9588929 ],\n           [-0.9588929 ,  0.28376836]])\n    >>> r\n    array([[-7.242763  , -0.5673361 , -2.624416  ],\n           [ 0.        , -7.297918  , -0.15949416]])\n    >>> a = np.random.uniform(-10, 10, (3, 2))\n    >>> q, r = np.linalg.qr(a)\n    >>> q\n    array([[-0.34515655,  0.10919492],\n           [ 0.14765628, -0.97452265],\n           [-0.92685735, -0.19591334]])\n    >>> r\n    array([[-8.453794,  8.4175  ],\n           [ 0.      ,  5.430561]])\n    \"\"\"\n    return _mx_nd_np.linalg.qr(a, mode)\n\n\ndef inv(a):\n    r\"\"\"\n    Compute the (multiplicative) inverse of a matrix.\n\n    Given a square matrix `a`, return the matrix `ainv` satisfying\n    ``dot(a, ainv) = dot(ainv, a) = eye(a.shape[0])``.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Matrix to be inverted.\n\n    Returns\n    -------\n    ainv : (..., M, M) ndarray\n        (Multiplicative) inverse of the matrix `a`.\n\n    Raises\n    ------\n    MXNetError\n        If `a` is not square or inversion fails.\n\n    Examples\n    --------\n    >>> from mxnet import np\n    >>> a = np.array([[1., 2.], [3., 4.]])\n    array([[-2. ,  1. ],\n           [ 1.5, -0.5]])\n\n    Inverses of several matrices can be computed at once:\n\n    >>> a = np.array([[[1., 2.], [3., 4.]], [[1, 3], [3, 5]]])\n    >>> np.linalg.inv(a)\n    array([[[-2.        ,  1.        ],\n            [ 1.5       , -0.5       ]],\n\n           [[-1.2500001 ,  0.75000006],\n            [ 0.75000006, -0.25000003]]])\n    \"\"\"\n    return _mx_nd_np.linalg.inv(a)\n\n\ndef det(a):\n    r\"\"\"\n    Compute the determinant of an array.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Input array to compute determinants for.\n\n    Returns\n    -------\n    det : (...) ndarray\n        Determinant of `a`.\n\n    See Also\n    --------\n    slogdet : Another way to represent the determinant, more suitable\n    for large matrices where underflow/overflow may occur.\n\n    Notes\n    -----\n    Broadcasting rules apply, see the `numpy.linalg` documentation for\n    details.\n    The determinant is computed via LU factorization using the LAPACK\n    routine z/dgetrf.\n\n    Examples\n    --------\n    The determinant of a 2-D array [[a, b], [c, d]] is ad - bc:\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> np.linalg.det(a)\n    -2.0\n\n    Computing determinants for a stack of matrices:\n    >>> a = np.array([ [[1, 2], [3, 4]], [[1, 2], [2, 1]], [[1, 3], [3, 1]] ])\n    >>> a.shape\n    (3, 2, 2)\n\n    >>> np.linalg.det(a)\n    array([-2., -3., -8.])\n    \"\"\"\n    return _mx_nd_np.linalg.det(a)\n\n\ndef slogdet(a):\n    r\"\"\"\n    Compute the sign and (natural) logarithm of the determinant of an array.\n    If an array has a very small or very large determinant, then a call to\n    `det` may overflow or underflow. This routine is more robust against such\n    issues, because it computes the logarithm of the determinant rather than\n    the determinant itself.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Input array, has to be a square 2-D array.\n\n    Returns\n    -------\n    sign : (...) ndarray\n        A number representing the sign of the determinant. For a real matrix,\n        this is 1, 0, or -1.\n    logdet : (...) array_like\n        The natural log of the absolute value of the determinant.\n    If the determinant is zero, then `sign` will be 0 and `logdet` will be\n    -Inf. In all cases, the determinant is equal to ``sign * np.exp(logdet)``.\n\n    See Also\n    --------\n    det\n\n    Notes\n    -----\n    Broadcasting rules apply, see the `numpy.linalg` documentation for\n    details.\n    The determinant is computed via LU factorization using the LAPACK\n    routine z/dgetrf.\n\n    Examples\n    --------\n    The determinant of a 2-D array ``[[a, b], [c, d]]`` is ``ad - bc``:\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> (sign, logdet) = np.linalg.slogdet(a)\n    >>> (sign, logdet)\n    (-1., 0.69314718055994529)\n\n    >>> sign * np.exp(logdet)\n    -2.0\n\n    Computing log-determinants for a stack of matrices:\n    >>> a = np.array([ [[1, 2], [3, 4]], [[1, 2], [2, 1]], [[1, 3], [3, 1]] ])\n    >>> a.shape\n    (3, 2, 2)\n\n    >>> sign, logdet = np.linalg.slogdet(a)\n    >>> (sign, logdet)\n    (array([-1., -1., -1.]), array([ 0.69314718,  1.09861229,  2.07944154]))\n\n    >>> sign * np.exp(logdet)\n    array([-2., -3., -8.])\n\n    This routine succeeds where ordinary `det` does not:\n    >>> np.linalg.det(np.eye(500) * 0.1)\n    0.0\n    >>> np.linalg.slogdet(np.eye(500) * 0.1)\n    (1., -1151.2925464970228)\n    \"\"\"\n    return _mx_nd_np.linalg.slogdet(a)\n\n\ndef solve(a, b):\n    r\"\"\"\n    Solve a linear matrix equation, or system of linear scalar equations.\n\n    Computes the \"exact\" solution, `x`, of the well-determined, i.e., full\n    rank, linear matrix equation `ax = b`.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Coefficient matrix.\n    b : {(..., M,), (..., M, K)}, ndarray\n        Ordinate or \"dependent variable\" values.\n\n    Returns\n    -------\n    x : {(..., M,), (..., M, K)} ndarray\n        Solution to the system a x = b.  Returned shape is identical to `b`.\n\n    Raises\n    ------\n    MXNetError\n        If `a` is singular or not square.\n\n    Notes\n    -----\n    Broadcasting rules apply, see the `numpy.linalg` documentation for\n    details.\n\n    The solutions are computed using LAPACK routine ``_gesv``.\n\n    `a` must be square and of full-rank, i.e., all rows (or, equivalently,\n    columns) must be linearly independent; if either is not true, use\n    `lstsq` for the least-squares best \"solution\" of the\n    system/equation.\n\n    Examples\n    --------\n    Solve the system of equations ``3 * x0 + x1 = 9`` and ``x0 + 2 * x1 = 8``:\n\n    >>> a = np.array([[3,1], [1,2]])\n    >>> b = np.array([9,8])\n    >>> x = np.linalg.solve(a, b)\n    >>> x\n    array([2.,  3.])\n\n    Check that the solution is correct:\n\n    >>> np.allclose(np.dot(a, x), b)\n    True\n    \"\"\"\n    return _mx_nd_np.linalg.solve(a, b)\n\n\ndef tensorinv(a, ind=2):\n    r\"\"\"\n    Compute the 'inverse' of an N-dimensional array.\n\n    The result is an inverse for `a` relative to the tensordot operation\n    ``tensordot(a, b, ind)``, i. e., up to floating-point accuracy,\n    ``tensordot(tensorinv(a), a, ind)`` is the \"identity\" tensor for the\n    tensordot operation.\n\n    Parameters\n    ----------\n    a : array_like\n        Tensor to 'invert'. Its shape must be 'square', i. e.,\n        ``prod(a.shape[:ind]) == prod(a.shape[ind:])``.\n    ind : int, optional\n        Number of first indices that are involved in the inverse sum.\n        Must be a positive integer, default is 2.\n\n    Returns\n    -------\n    b : ndarray\n        `a`'s tensordot inverse, shape ``a.shape[ind:] + a.shape[:ind]``.\n\n    Raises\n    ------\n    MXNetError\n        If `a` is singular or not 'square' (in the above sense).\n\n    See Also\n    --------\n    tensordot, tensorsolve\n\n    Examples\n    --------\n    >>> a = np.eye(4*6)\n    >>> a.shape = (4, 6, 8, 3)\n    >>> ainv = np.linalg.tensorinv(a, ind=2)\n    >>> ainv.shape\n    (8, 3, 4, 6)\n    >>> b = np.random.randn(4, 6)\n    >>> np.allclose(np.tensordot(ainv, b), np.linalg.tensorsolve(a, b))\n    True\n\n    >>> a = np.eye(4*6)\n    >>> a.shape = (24, 8, 3)\n    >>> ainv = np.linalg.tensorinv(a, ind=1)\n    >>> ainv.shape\n    (8, 3, 24)\n    >>> b = np.random.randn(24)\n    >>> np.allclose(np.tensordot(ainv, b, 1), np.linalg.tensorsolve(a, b))\n    True\n    \"\"\"\n    return _mx_nd_np.linalg.tensorinv(a, ind)\n\n\ndef tensorsolve(a, b, axes=None):\n    r\"\"\"\n    Solve the tensor equation ``a x = b`` for x.\n    It is assumed that all indices of `x` are summed over in the product,\n    together with the rightmost indices of `a`, as is done in, for example,\n    ``tensordot(a, x, axes=b.ndim)``.\n\n    Parameters\n    ----------\n    a : ndarray\n        Coefficient tensor, of shape ``b.shape + Q``. `Q`, a tuple, equals\n        the shape of that sub-tensor of `a` consisting of the appropriate\n        number of its rightmost indices, and must be such that\n        ``prod(Q) == prod(b.shape)`` (in which sense `a` is said to be\n        'square').\n    b : ndarray\n        Right-hand tensor, which can be of any shape.\n    axes : tuple of ints, optional\n        Axes in `a` to reorder to the right, before inversion.\n        If None (default), no reordering is done.\n\n    Returns\n    -------\n    x : ndarray, shape Q\n\n    Raises\n    ------\n    MXNetError\n        If `a` is singular or not 'square' (in the above sense).\n\n    See Also\n    --------\n    numpy.tensordot, tensorinv, numpy.einsum\n\n    Examples\n    --------\n    >>> a = np.eye(2*3*4)\n    >>> a.shape = (2*3, 4, 2, 3, 4)\n    >>> b = np.random.randn(2*3, 4)\n    >>> x = np.linalg.tensorsolve(a, b)\n    >>> x.shape\n    (2, 3, 4)\n    >>> np.allclose(np.tensordot(a, x, axes=3), b)\n    True\n    \"\"\"\n    return _mx_nd_np.linalg.tensorsolve(a, b, axes)\n\n\ndef eigvals(a):\n    r\"\"\"\n    Compute the eigenvalues of a general matrix.\n\n    Main difference between `eigvals` and `eig`: the eigenvectors aren't\n    returned.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        A real-valued matrix whose eigenvalues will be computed.\n\n    Returns\n    -------\n    w : (..., M,) ndarray\n        The eigenvalues, each repeated according to its multiplicity.\n        They are not necessarily ordered.\n\n    Raises\n    ------\n    MXNetError\n        If the eigenvalue computation does not converge.\n\n    See Also\n    --------\n    eig : eigenvalues and right eigenvectors of general arrays\n    eigh : eigenvalues and eigenvectors of a real symmetric array.\n    eigvalsh : eigenvalues of a real symmetric.\n\n    .. note::\n       Broadcasting rules apply, see the `numpy.linalg` documentation for\n       details.\n\n       This is implemented using the ``_geev`` LAPACK routines which compute\n       the eigenvalues and eigenvectors of general square arrays.\n\n       This function differs from the original `numpy.linalg.eigvals\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.eigvals.html>`_ in\n       the following way(s):\n       * Does not support complex input and output.\n\n    Examples\n    --------\n    Illustration, using the fact that the eigenvalues of a diagonal matrix\n    are its diagonal elements, that multiplying a matrix on the left\n    by an orthogonal matrix, `Q`, and on the right by `Q.T` (the transpose\n    of `Q`), preserves the eigenvalues of the \"middle\" matrix.  In other words,\n    if `Q` is orthogonal, then ``Q * A * Q.T`` has the same eigenvalues as\n    ``A``:\n\n    >>> from numpy import linalg as LA\n    >>> x = np.random.random()\n    >>> Q = np.array([[np.cos(x), -np.sin(x)], [np.sin(x), np.cos(x)]])\n    >>> LA.norm(Q[0, :]), LA.norm(Q[1, :]), np.dot(Q[0, :],Q[1, :])\n    (1.0, 1.0, 0.0)\n\n    Now multiply a diagonal matrix by ``Q`` on one side and by ``Q.T`` on the other:\n\n    >>> D = np.diag((-1,1))\n    >>> LA.eigvals(D)\n    array([-1.,  1.])\n    >>> A = np.dot(Q, D)\n    >>> A = np.dot(A, Q.T)\n    >>> LA.eigvals(A)\n    array([ 1., -1.]) # random\n    \"\"\"\n    return _mx_nd_np.linalg.eigvals(a)\n\n\n@wrap_data_api_linalg_func\ndef eigvalsh(a, upper=False):\n    r\"\"\"\n    Compute the eigenvalues real symmetric matrix.\n\n    Main difference from eigh: the eigenvectors are not computed.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        A real-valued matrix whose eigenvalues are to be computed.\n    UPLO : {'L', 'U'}, optional\n        Specifies whether the calculation is done with the lower triangular\n        part of `a` ('L', default) or the upper triangular part ('U').\n        Irrespective of this value only the real parts of the diagonal will\n        be considered in the computation to preserve the notion of a Hermitian\n        matrix. It therefore follows that the imaginary part of the diagonal\n        will always be treated as zero.\n\n    Returns\n    -------\n    w : (..., M,) ndarray\n        The eigenvalues in ascending order, each repeated according to\n        its multiplicity.\n\n    Raises\n    ------\n    MXNetError\n        If the eigenvalue computation does not converge.\n\n    See Also\n    --------\n    eig : eigenvalues and right eigenvectors of general arrays\n    eigvals : eigenvalues of a non-symmetric array.\n    eigh : eigenvalues and eigenvectors of a real symmetric array.\n\n    .. note::\n       Broadcasting rules apply, see the `numpy.linalg` documentation for\n       details.\n\n       The eigenvalues are computed using LAPACK routines ``_syevd``.\n\n       This function differs from the original `numpy.linalg.eigvalsh\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.eigvalsh.html>`_ in\n       the following way(s):\n       * Does not support complex input and output.\n\n    Examples\n    --------\n    >>> from numpy import linalg as LA\n    >>> a = np.array([[ 5.4119368 ,  8.996273  , -5.086096  ],\n    ...               [ 0.8866155 ,  1.7490431 , -4.6107802 ],\n    ...               [-0.08034172,  4.4172044 ,  1.4528792 ]])\n    >>> LA.eigvalsh(a, UPLO='L')\n    array([-2.87381886,  5.10144682,  6.38623114]) # in ascending order\n    \"\"\"\n    if not upper:\n        UPLO = 'L'\n    else:\n        UPLO = 'U'\n    return _mx_nd_np.linalg.eigvalsh(a, UPLO)\n\n\ndef eig(a):\n    r\"\"\"\n    Compute the eigenvalues and right eigenvectors of a square array.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Matrices for which the eigenvalues and right eigenvectors will\n        be computed\n\n    Returns\n    -------\n    w : (..., M) ndarray\n        The eigenvalues, each repeated according to its multiplicity.\n        The eigenvalues are not necessarily ordered.\n    v : (..., M, M) ndarray\n        The normalized (unit \"length\") eigenvectors, such that the\n        column ``v[:,i]`` is the eigenvector corresponding to the\n        eigenvalue ``w[i]``.\n\n    Raises\n    ------\n    MXNetError\n        If the eigenvalue computation does not converge.\n\n    See Also\n    --------\n    eigvals : eigenvalues of a non-symmetric array.\n    eigh : eigenvalues and eigenvectors of a real symmetric array.\n    eigvalsh : eigenvalues of a real symmetric.\n\n    .. note::\n       This is implemented using the ``_geev`` LAPACK routines which compute\n       the eigenvalues and eigenvectors of general square arrays.\n\n       The number `w` is an eigenvalue of `a` if there exists a vector\n       `v` such that ``dot(a,v) = w * v``. Thus, the arrays `a`, `w`, and\n       `v` satisfy the equations ``dot(a[:,:], v[:,i]) = w[i] * v[:,i]``\n       for :math:`i \\\\in \\\\{0,...,M-1\\\\}`.\n\n       The array `v` of eigenvectors may not be of maximum rank, that is, some\n       of the columns may be linearly dependent, although round-off error may\n       obscure that fact. If the eigenvalues are all different, then theoretically\n       the eigenvectors are linearly independent.\n\n       This function differs from the original `numpy.linalg.eig\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.eig.html>`_ in\n       the following way(s):\n       * Does not support complex input and output.\n\n    Examples\n    --------\n    >>> from numpy import linalg as LA\n    >>> a = np.array([[-1.9147992 ,  6.054115  , 18.046988  ],\n    ...               [ 0.77563655, -4.860152  ,  2.1012988 ],\n    ...               [ 2.6083658 ,  2.3705218 ,  0.3192524 ]])\n    >>> w, v = LA.eig(a)\n    >>> w\n    array([ 6.9683027, -7.768063 , -5.655937 ])\n    >>> v\n    array([[ 0.90617794,  0.9543622 ,  0.2492316 ],\n           [ 0.13086087, -0.04077047, -0.9325615 ],\n           [ 0.4021404 , -0.29585576,  0.26117516]])\n    \"\"\"\n    return _mx_nd_np.linalg.eig(a)\n\n\n@wrap_data_api_linalg_func\ndef eigh(a, upper=False):\n    r\"\"\"\n    Return the eigenvalues and eigenvectors real symmetric matrix.\n\n    Returns two objects, a 1-D array containing the eigenvalues of `a`, and\n    a 2-D square array or matrix (depending on the input type) of the\n    corresponding eigenvectors (in columns).\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        real symmetric matrices whose eigenvalues and eigenvectors are to be computed.\n    UPLO : {'L', 'U'}, optional\n        Specifies whether the calculation is done with the lower triangular\n        part of `a` ('L', default) or the upper triangular part ('U').\n        Irrespective of this value only the real parts of the diagonal will\n        be considered in the computation to preserve the notion of a Hermitian\n        matrix. It therefore follows that the imaginary part of the diagonal\n        will always be treated as zero.\n\n    Returns\n    -------\n    w : (..., M) ndarray\n        The eigenvalues in ascending order, each repeated according to\n        its multiplicity.\n    v : {(..., M, M) ndarray, (..., M, M) matrix}\n        The column ``v[:, i]`` is the normalized eigenvector corresponding\n        to the eigenvalue ``w[i]``.  Will return a matrix object if `a` is\n        a matrix object.\n\n    Raises\n    ------\n    MXNetError\n        If the eigenvalue computation does not converge.\n\n    See Also\n    --------\n    eig : eigenvalues and right eigenvectors of general arrays\n    eigvals : eigenvalues of a non-symmetric array.\n    eigvalsh : eigenvalues of a real symmetric.\n\n    .. note::\n\n       The eigenvalues/eigenvectors are computed using LAPACK routines ``_syevd``.\n\n       This function differs from the original `numpy.linalg.eigh\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.eigh.html>`_ in\n       the following way(s):\n       * Does not support complex input and output.\n\n    Examples\n    --------\n    >>> from numpy import linalg as LA\n    >>> a = np.array([[ 6.8189726 , -3.926585  ,  4.3990498 ],\n    ...               [-0.59656644, -1.9166266 ,  9.54532   ],\n    ...               [ 2.1093285 ,  0.19688708, -1.1634291 ]])\n    >>> w, v = LA.eigh(a, upper=False)\n    >>> w\n    array([-2.175445 , -1.4581827,  7.3725457])\n    >>> v\n    array([[ 0.1805163 , -0.16569263,  0.9695154 ],\n           [ 0.8242942 ,  0.56326365, -0.05721384],\n           [-0.53661287,  0.80949366,  0.23825769]])\n    \"\"\"\n    if not upper:\n        UPLO = 'L'\n    else:\n        UPLO = 'U'\n    return _mx_nd_np.linalg.eigh(a, UPLO)\n"
  },
  {
    "path": "python/mxnet/numpy/multiarray.py",
    "content": "#!/usr/bin/env python\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=too-many-lines, unused-argument\n\"\"\"numpy ndarray and util functions.\"\"\"\n\n\ntry:\n    from __builtin__ import all as py_all\n    from __builtin__ import slice as py_slice\nexcept ImportError:\n    from builtins import all as py_all\n    from builtins import slice as py_slice\n\nfrom array import array as native_array\nimport functools\nimport ctypes\nimport sys\nimport datetime\nimport warnings\nimport numpy as _np\nfrom .. import _deferred_compute as dc\nfrom ..autograd import is_recording\nfrom ..ndarray import NDArray, dtype_np_to_mx, _GRAD_REQ_MAP\nfrom ..ndarray import indexing_key_expand_implicit_axes, get_indexing_dispatch_code,\\\n                      get_oshape_of_gather_nd_op\nfrom ..ndarray._internal import _set_np_ndarray_class\nfrom . import _op as _mx_np_op\nfrom ..base import check_call, _LIB, NDArrayHandle, c_array, mx_int, mx_int64\nfrom ..base import mx_real_t, c_array_buf, mx_uint, numeric_types, integer_types\nfrom ..runtime import Features\nfrom ..device import Device\nfrom ..util import set_module, wrap_np_unary_func, wrap_np_binary_func,\\\n                   is_np_default_dtype, wrap_ctx_to_device_func,\\\n                   dtype_from_number, wrap_data_api_statical_func,\\\n                   wrap_sort_functions\nfrom ..device import current_device\nfrom ..ndarray import numpy as _mx_nd_np\nfrom ..ndarray.numpy import _internal as _npi\nfrom ..ndarray.ndarray import _storage_type\nfrom ..dlpack import ndarray_from_numpy, ndarray_to_dlpack_for_write, DLDeviceType,\\\n                     ndarray_from_dlpack\nfrom .utils import _get_np_op\nfrom .fallback import *  # pylint: disable=wildcard-import,unused-wildcard-import\nfrom . import fallback\n\n\n__all__ = ['ndarray', 'empty', 'empty_like', 'array', 'shape', 'median',\n           'zeros', 'zeros_like', 'ones', 'ones_like', 'full', 'full_like', 'all', 'any', 'broadcast_to',\n           'add', 'subtract', 'multiply', 'divide', 'mod', 'remainder', 'fmod', 'pow', 'power', 'bitwise_not',\n           'delete', 'trace', 'transpose', 'copy', 'moveaxis', 'reshape', 'dot',\n           'arctan2', 'atan2', 'sin', 'cos', 'tan', 'sinh', 'cosh', 'tanh', 'log10', 'bitwise_invert', 'invert',\n           'sqrt', 'cbrt', 'abs', 'absolute', 'fabs', 'exp', 'expm1', 'arcsin', 'asin', 'arccos', 'acos', 'arctan',\n           'atan', 'sign', 'log', 'degrees', 'log2', 'log1p', 'rint', 'radians', 'reciprocal', 'square',\n           'negative', 'histogram', 'fix', 'ceil', 'floor', 'trunc', 'logical_not', 'arcsinh', 'asinh',\n           'arccosh', 'acosh', 'arctanh', 'atanh', 'append', 'argsort', 'sort', 'tensordot', 'eye', 'linspace',\n           'logspace', 'expand_dims', 'tile', 'arange', 'array_split', 'split', 'hsplit', 'vsplit',\n           'dsplit', 'flatnonzero', 'tril_indices', 'concatenate', 'concat', 'stack', 'vstack', 'row_stack',\n           'column_stack', 'hstack', 'dstack', 'average', 'mean', 'maximum', 'fmax', 'minimum', 'fmin',\n           'amax', 'amin', 'max', 'min', 'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'insert',\n           'indices', 'copysign', 'ravel', 'unravel_index', 'diag_indices_from', 'hanning', 'hamming', 'blackman',\n           'logical_and', 'logical_or', 'logical_xor',\n           'flip', 'flipud', 'fliplr', 'around', 'round', 'round_', 'arctan2', 'hypot',\n           'triu_indices_from', 'triu_indices', 'tri',\n           'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad',\n           'unique', 'lcm', 'gcd', 'tril', 'triu', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer',\n           'cross', 'kron', 'equal', 'not_equal', 'interp',\n           'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum', 'true_divide', 'nonzero',\n           'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d', 'resize', 'matmul',\n           'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite', 'polyval', 'where', 'bincount',\n           'atleast_1d', 'atleast_2d', 'atleast_3d', 'fill_diagonal', 'squeeze',\n           'diagflat', 'repeat', 'prod', 'pad', 'cumsum', 'sum', 'rollaxis', 'diag', 'diagonal',\n           'positive', 'logaddexp', 'floor_divide', 'permute_dims', 'bitwise_left_shift', 'bitwise_right_shift',\n           'asarray', 'from_dlpack']\n\n__all__ += fallback.__all__\n\n# Return code for dispatching indexing function call\n_NDARRAY_UNSUPPORTED_INDEXING = -1\n_NDARRAY_BASIC_INDEXING = 0\n_NDARRAY_ADVANCED_INDEXING = 1\n_NDARRAY_EMPTY_TUPLE_INDEXING = 2\n\n# Return code for 0-d boolean array handler\n_NDARRAY_NO_ZERO_DIM_BOOL_ARRAY = -1\n_NDARRAY_ZERO_DIM_BOOL_ARRAY_FALSE = 0\n_NDARRAY_ZERO_DIM_BOOL_ARRAY_TRUE = 1\n_SIGNED_INT32_UPPER_LIMIT = (2**31 - 1)\n\n# Caching whether MXNet was built with INT64 support or not\n_INT64_TENSOR_SIZE_ENABLED = None\n\ndef _int64_enabled():\n    global _INT64_TENSOR_SIZE_ENABLED\n    if _INT64_TENSOR_SIZE_ENABLED is None:\n        _INT64_TENSOR_SIZE_ENABLED = Features().is_enabled('INT64_TENSOR_SIZE')\n    return _INT64_TENSOR_SIZE_ENABLED\n\n# This function is copied from ndarray.py since pylint\n# keeps giving false alarm error of undefined-all-variable\ndef _new_alloc_handle(shape, device, delay_alloc, dtype=mx_real_t):  # pylint: disable=redefined-outer-name\n    \"\"\"Return a new handle with specified shape and device.\n\n    Empty handle is only used to hold results.\n\n    Returns\n    -------\n    handle\n        A new empty `ndarray` handle.\n    \"\"\"\n    hdl = NDArrayHandle()\n    if _int64_enabled():\n        check_call(_LIB.MXNDArrayCreate64(\n            c_array_buf(mx_int64, native_array('q', shape)),\n            ctypes.c_int(len(shape)),\n            ctypes.c_int(device.device_typeid),\n            ctypes.c_int(device.device_id),\n            ctypes.c_int(int(delay_alloc)),\n            ctypes.c_int(int(dtype_np_to_mx(dtype))),\n            ctypes.byref(hdl)))\n    else:\n        # When shape is larger than uint32 then there is an overflow error at python end itself.\n        # It needs to be caught here since the call doesn't even reach backend.\n        array_size = 1\n        for idx in shape:\n            array_size = array_size * idx\n        if array_size > _SIGNED_INT32_UPPER_LIMIT:\n            raise Exception(\"[_new_alloc_handle] Size of tensor you are trying to allocate is \" +\n                            \"larger than 2^31 elements. Please build with flag \" +\n                            \"USE_INT64_TENSOR_SIZE=1\")\n        check_call(_LIB.MXNDArrayCreate(\n            c_array_buf(mx_uint, native_array('I', shape)),\n            mx_uint(len(shape)),\n            ctypes.c_int(device.device_typeid),\n            ctypes.c_int(device.device_id),\n            ctypes.c_int(int(delay_alloc)),\n            ctypes.c_int(int(dtype_np_to_mx(dtype))),\n            ctypes.byref(hdl)))\n    return hdl\n\n\ndef _reshape_view(a, *shape):  # pylint: disable=redefined-outer-name\n    \"\"\"Returns a **view** of this array with a new shape without altering any data.\n\n    Parameters\n    ----------\n    shape : tuple of int, or n ints\n        The new shape should not change the array size, namely\n        ``np.prod(new_shape)`` should be equal to ``np.prod(a.shape)``.\n        Some dimensions of the shape can take special value -1, which\n        infers the dimension of the output shape by using the remainder of the\n        input dimensions keeping the size of the new array same as that of the input array.\n        At most one dimension of shape can be -1.\n\n    Returns\n    -------\n    ndarray\n        An array with desired shape that shares data with this array.\n    \"\"\"\n    if len(shape) == 1 and isinstance(shape[0], (list, tuple)):\n        shape = shape[0]\n    handle = NDArrayHandle()\n    check_call(_LIB.MXNDArrayReshape64(a.handle,\n                                       len(shape),\n                                       c_array(ctypes.c_int64, shape),\n                                       False,\n                                       ctypes.byref(handle)))\n    return ndarray(handle=handle, writable=a.writable)\n\ndef _as_mx_np_array(object, device=None, zero_copy=False):\n    \"\"\"Convert arrays or any array member of container to mxnet.numpy.ndarray on device.\"\"\"\n    if object is None or isinstance(object, ndarray):\n        return object\n    elif isinstance(object, _np.ndarray):\n        from_numpy = ndarray_from_numpy(ndarray, array)\n        return from_numpy(object, zero_copy and object.flags['C_CONTIGUOUS'])\n    elif isinstance(object, (integer_types, numeric_types)):\n        return object\n    elif isinstance(object, (_np.bool_, _np.bool)):\n        return array(object, dtype=_np.bool_, device=device)\n    elif isinstance(object, (list, tuple)):\n        tmp = [_as_mx_np_array(arr, device=device, zero_copy=zero_copy) for arr in object]\n        return object.__class__(tmp)\n    else:\n        raise TypeError('Does not support converting {} to mx.np.ndarray.'.format(str(type(object))))\n\n\ndef _as_onp_array(object, cur_device=None):\n    \"\"\"Convert object to numpy.ndarray.\"\"\"\n    def _update_device(cur_device, tmp_device):\n        if cur_device is None:\n            cur_device = tmp_device\n        elif tmp_device is not None and cur_device != tmp_device:\n            raise ValueError('Ambiguous to set the device for the output ndarray since'  # pylint: disable=too-few-format-args\n                             ' input ndarrays are allocated on different devices: {} and {}'\n                             .format(str(cur_device, tmp_device)))\n        return cur_device\n\n    if isinstance(object, ndarray):\n        return object.asnumpy(), object.device\n    elif isinstance(object, (list, tuple)):\n        tmp = []\n        for arr in object:\n            arr, tmp_device = _as_onp_array(arr, cur_device)\n            tmp.append(arr)\n            cur_device = _update_device(cur_device, tmp_device)\n        return object.__class__(tmp), cur_device\n    elif isinstance(object, dict):\n        tmp = dict()\n        for key, value in object.items():\n            value, tmp_device = _as_onp_array(value, cur_device)\n            tmp[key] = value\n            cur_device = _update_device(cur_device, tmp_device)\n        return object.__class__(tmp), cur_device\n    else:\n        return object, cur_device\n\n\n# Have to use 0 as default value for stype since pylint does not allow\n# importing _STORAGE_TYPE_DEFAULT from ndarray.py.\ndef _np_ndarray_cls(handle, writable=True, stype=0):\n    if stype == -1:\n        stype = _storage_type(handle)\n    if stype != 0:\n        raise ValueError('_np_ndarray_cls currently only supports default storage '\n                         'type, while received stype = {}'.format(stype))\n    return ndarray(handle, writable=writable)\n\n\n_set_np_ndarray_class(_np_ndarray_cls)\n\n_NUMPY_ARRAY_FUNCTION_DICT = {}\n_NUMPY_ARRAY_UFUNC_DICT = {}\n_FALLBACK_ARRAY_FUNCTION_WARNED_RECORD = {}\n_FALLBACK_ARRAY_UFUNC_WARNED_RECORD = {}\n\ndef wrap_mxnp_np_ufunc(func):\n    \"\"\"\n    A convenience decorator for wrapping for python overload-able ops to provide type\n    casting for mixed use of mx_np and onp inputs.\n\n    Parameters\n    ----------\n    func : a python overload-able binary function to be wrapped for type casting.\n\n    Returns\n    -------\n    Function\n        A function wrapped with type casted.\n    \"\"\"\n    @functools.wraps(func)\n    def _wrap_mxnp_np_ufunc(x1, x2):\n        if isinstance(x2, _np.ndarray):\n            x2 = _as_mx_np_array(x2, device=x1.device)\n        return func(x1, x2)\n    return _wrap_mxnp_np_ufunc\n\n@set_module('mxnet.numpy')\nclass ndarray(NDArray):  # pylint: disable=invalid-name\n    \"\"\"\n    ndarray(handle, writable=True):\n\n    An array object represents a multidimensional, homogeneous array of fixed-size items.\n    An associated data-type object describes the format of each element in the array\n    (its byte-order, how many bytes it occupies in memory, whether it is an integer, a\n    floating point number, or something else, etc.). Arrays should be constructed using\n    `array`, `zeros` or `empty`. Currently, only c-contiguous arrays are supported.\n\n    Arrays should be constructed using `array`, `zeros` or `empty` (refer\n    to the See Also section below).  The parameters given here refer to\n    a low-level method (`ndarray(...)`) for instantiating an array.\n\n    For more information, refer to the `mxnet.numpy` module and examine the\n    methods and attributes of an array.\n\n    Parameters\n    ----------\n    handle: int\n        The ndarray handle in backend (C++).\n    writable: bool\n        Indicates whether inplace-assignment is allowed for the array.\n\n    Attributes\n    ----------\n    T : ndarray\n        Transpose of the array.\n    dtype : dtype object\n        Describes the format of the elements in the array.\n    size : int\n        Number of elements in the array.\n    ndim : int\n        The array's number of dimensions.\n    shape : tuple of ints\n        Shape of the array.\n\n    See Also\n    --------\n    array : Construct an array.\n    zeros : Create an array, each element of which is zero.\n    empty : Create an array, but leave its allocated memory unchanged (i.e.,\n            it contains \"garbage\").\n    \"\"\"\n\n    @staticmethod\n    def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):  # pylint: disable=bad-staticmethod-argument\n        \"\"\"\n        Dispatch official NumPy unary/binary operator calls on mxnet.numpy.ndarray\n        to this function. The operators must comply with the ufunc definition in NumPy.\n        The following code is adapted from CuPy.\n        Casting rules for operator with mx_np and onp (inplace op will keep its type)\n        | Expression | a type | b type | out type|\n        | --- | --- | --- | --- |\n        | `a += b` | onp | mx_np | onp |\n        | `a += b` | mx_np | onp | mx_np |\n        | `c = a + b` | onp | mx_np | mx_np |\n        | `c = a + b` | mx_np | onp | mx_np |\n        \"\"\"\n        ufunc_list = [\"add\", \"subtract\", \"multiply\", \"divide\", \"true_divide\", \"floor_divide\", \"power\",\n                      \"remainder\", \"bitwise_and\", \"bitwise_or\", \"bitwise_xor\", \"left_shift\", \"right_shift\",\n                      \"greater\", \"greater_equal\", \"less\", \"less_equal\", \"not_equal\", \"equal\", \"matmul\"]\n        if 'out' in kwargs:\n            # need to unfold tuple argument in kwargs\n            out = kwargs['out']\n            if len(out) != 1:\n                raise ValueError('The `out` parameter must have exactly one ndarray')\n            kwargs['out'] = out[0]\n\n        if method == '__call__':\n            name = ufunc.__name__\n            mx_ufunc = _NUMPY_ARRAY_UFUNC_DICT.get(name, None)\n            onp_op = _get_np_op(name)\n            if mx_ufunc is None:\n                # try to fallback to official NumPy op\n                if is_recording():\n                    raise ValueError(\"Falling back to NumPy operator {} with autograd active is not supported.\"\n                                     \"Please consider moving the operator to the outside of the autograd scope.\")\\\n                                     .format(name)\n                new_inputs = [arg.asnumpy() if isinstance(arg, ndarray) else arg for arg in inputs]\n                if onp_op not in _FALLBACK_ARRAY_UFUNC_WARNED_RECORD:\n                    import logging\n                    logging.warning(\"np.%s is a fallback operator, \"\n                                    \"which is actually using official numpy's implementation\", name)\n                    _FALLBACK_ARRAY_UFUNC_WARNED_RECORD[onp_op] = True\n                out = onp_op(*new_inputs, **kwargs)\n                return _as_mx_np_array(out, device=inputs[0].device)\n            # ops with np mx_np\n            elif name in ufunc_list and isinstance(inputs[0], _np.ndarray):\n                # inplace\n                if 'out' in kwargs:\n                    new_inputs = [arg.asnumpy() if isinstance(arg, ndarray) else arg for arg in inputs]\n                    return onp_op(*new_inputs, **kwargs)\n                else:\n                    new_inputs = [_as_mx_np_array(arg, device=inputs[1].device)\n                                  if isinstance(arg, _np.ndarray) else arg for arg in inputs]\n                    return mx_ufunc(*new_inputs, **kwargs)\n            else:\n                return mx_ufunc(*inputs, **kwargs)\n        else:\n            return NotImplemented\n\n    @staticmethod\n    def __array_function__(self, func, types, args, kwargs):  # pylint: disable=bad-staticmethod-argument\n        \"\"\"\n        Dispatch official NumPy operators that comply with the array function protocol to\n        this function.\n        \"\"\"\n        mx_np_func = _NUMPY_ARRAY_FUNCTION_DICT.get(func, None)\n        func_name = func.__name__\n        if mx_np_func is None:\n            # try to fallback to official NumPy op\n            if is_recording():\n                raise ValueError(\"Falling back to NumPy operator {} with autograd active is not supported.\"\n                                 \"Please consider moving the operator to the outside of the autograd scope.\")\\\n                                 .format(func)\n            cur_device = None\n            new_args, cur_device = _as_onp_array(args, cur_device)\n            new_kwargs, cur_device = _as_onp_array(kwargs, cur_device)\n            if cur_device is None:\n                raise ValueError('Unknown device for the input ndarrays. It is probably a bug. Please'\n                                 ' create an issue on GitHub.')\n            if func not in _FALLBACK_ARRAY_FUNCTION_WARNED_RECORD:\n                import logging\n                logging.warning(\"np.%s is a fallback operator, \"\n                                \"which is actually using official numpy's implementation.\", func_name)\n                _FALLBACK_ARRAY_FUNCTION_WARNED_RECORD[func] = True\n            out = func(*new_args, **new_kwargs)\n            return _as_mx_np_array(out, device=cur_device)\n        else:\n            if py_all(issubclass(t, ndarray) for t in types):\n                return mx_np_func(*args, **kwargs)\n            else:\n                try:\n                    cur_device = next(a.device for a in args if hasattr(a, 'device'))\n                except StopIteration:\n                    cur_device = next(a.device for a in kwargs.values() if hasattr(a, 'device'))\n                new_args = _as_mx_np_array(args, device=cur_device,\n                                           zero_copy=func_name in {'may_share_memory', 'shares_memory'})\n                new_kwargs = {k: _as_mx_np_array(v, cur_device) for k, v in kwargs.items()}\n                return mx_np_func(*new_args, **new_kwargs)\n\n\n    def __array_namespace__(self, api_version=None):\n        \"\"\"\n        Returns an object that has all the array API functions on it.\n\n        Notes\n        -----\n        This is a standard API in\n        https://data-apis.org/array-api/latest/API_specification/array_object.html#array-namespace-self-api-version-none.\n\n        Parameters\n        ----------\n        self : ndarray\n            The indexing key.\n        api_version : Optional, string\n            string representing the version of the array API specification to be returned, in `YYYY.MM` form.\n            If it is None, it should return the namespace corresponding to latest version of the array API\n            specification.\n        \"\"\"\n        if api_version is not None:\n            try:\n                date = datetime.datetime.strptime(api_version, '%Y.%m')\n                if date.year != 2021:\n                    raise ValueError\n            except ValueError:\n                raise ValueError(f\"Unrecognized array API version: {api_version!r}\")\n        return sys.modules[self.__module__]\n\n\n    def __dlpack__(self, stream=None):\n        \"\"\"Exports the array for consumption by from_dlpack() as a DLPack capsule.\n\n        Parameters\n        ----------\n        stream : int, optional\n            A Python integer representing a pointer to a stream (CUDA or ROCm).\n            Stream is provided by the consumer to the producer to instruct the producer\n            to ensure that operations can safely be performed on the array. The pointer must\n            be positive integer or -1. If stream is -1, the value must be used by the consumer\n            to signal \"producer must not perform any synchronization\". \n\n        Returns\n        -------\n        capsule : PyCapsule\n            A DLPack capsule for the array, containing a DLPackManagedTensor.\n        \"\"\"\n        if stream is not None:\n            if type(stream) is not int:\n                raise TypeError('The input stream must be int or None')\n            if self.device.device_type != \"gpu\":\n                raise ValueError('Stream {} is not supported in current device {}'\\\n                    .format(stream, self.device.device_type))\n            if stream != -1:\n                check_call(_LIB.MXPushStreamDep(self.handle, ctypes.c_int64(stream)))\n        to_dlpack_write = ndarray_to_dlpack_for_write()\n        return to_dlpack_write(self)\n\n\n    def __dlpack_device__(self):\n        \"\"\"Returns device type and device ID in DLPack format\"\"\"\n        devtype_map = {'cpu': DLDeviceType.DLCPU,\n                       'gpu': DLDeviceType.DLGPU,\n                       'cpu_pinned': DLDeviceType.DLCPUPINNED}\n        if self.device.device_type not in devtype_map:\n            raise ValueError('Unkown device type {} for DLPack'.format(self.device.device_type))\n        return (devtype_map[self.device.device_type], self.device.device_id)\n\n\n    def _get_np_basic_indexing(self, key):\n        \"\"\"\n        This function indexes ``self`` with a tuple of `slice` objects only.\n        \"\"\"\n        key_nd = tuple(idx for idx in key if idx is not None)\n        if len(key_nd) < self.ndim:\n            raise RuntimeError(\n                'too few indices after normalization: expected `ndim` ({}) '\n                'but got {}. This is a bug, please report it!'\n                ''.format(self.ndim, len(key_nd))\n            )\n        if len(key_nd) > self.ndim:\n            raise IndexError(\n                'too many indices ({}) for array with {} dimensions'\n                ''.format(len(key_nd), self.ndim)\n            )\n\n        none_axes = [ax for ax in range(len(key)) if key[ax] is None]  # pylint: disable=invalid-name\n        slc_key, int_axes = self._basic_indexing_key_int_to_slice(key_nd)\n        new_axes = self._new_axes_after_basic_indexing(none_axes, key)\n\n        # Check bounds for integer axes\n        for ax in int_axes:  # pylint: disable=invalid-name\n            if not -self.shape[ax] <= key_nd[ax] < self.shape[ax]:\n                raise IndexError(\n                    'index {} is out of bounds for axis {} with size {}'\n                    ''.format(key_nd[ax], ax, self.shape[ax]))\n\n        if self._basic_indexing_slice_is_contiguous(slc_key, self.shape):\n            # Create a shared-memory view by using low-level flat slicing\n            flat_begin, flat_end = self._basic_indexing_contiguous_flat_begin_end(\n                slc_key, self.shape\n            )\n            handle = NDArrayHandle()\n            flat_self = self.reshape_view(-1)\n            if _int64_enabled():\n                check_call(\n                    _LIB.MXNDArraySlice64(\n                        flat_self.handle,\n                        ctypes.c_int64(flat_begin),\n                        ctypes.c_int64(flat_end),\n                        ctypes.byref(handle),\n                    )\n                )\n            else:\n                check_call(\n                    _LIB.MXNDArraySlice(\n                        flat_self.handle,\n                        ctypes.c_uint32(flat_begin),\n                        ctypes.c_uint32(flat_end),\n                        ctypes.byref(handle),\n                    )\n                )\n            sliced_shape = self._basic_indexing_sliced_shape(slc_key, self.shape)\n            sliced = self.__class__(handle=handle, writable=self.writable)\n            if 0 in sliced_shape:\n                sliced = sliced.reshape(sliced_shape)\n            else:\n                sliced = sliced.reshape_view(sliced_shape)\n\n        else:\n            begin, end, step = self._basic_indexing_key_to_begin_end_step(\n                slc_key, self.shape, keep_none=True\n            )\n            sliced = _npi.slice(self, begin, end, step)\n\n        # Reshape to final shape due to integer and `None` entries in `key`.\n        final_shape = [sliced.shape[i] for i in range(sliced.ndim) if i not in int_axes]\n        for ax in new_axes:  # pylint: disable=invalid-name\n            final_shape.insert(ax, 1)\n\n        if sliced.size == 0:\n            return sliced.reshape(tuple(final_shape))\n        else:\n            return sliced.reshape_view(tuple(final_shape))\n\n    def _get_np_empty_tuple_indexing(self, key):\n        new_shape = []\n        num_none = 0\n        for i, idx in enumerate(key):\n            if idx is None:\n                new_shape.append(1) # expand dimension\n                num_none += 1\n            elif idx == ():\n                new_shape.append(0) # 0 shape\n            elif idx == slice(None, None, None):\n                new_shape.append(self.shape[i - num_none])\n        return empty(new_shape, dtype=self.dtype)\n\n    def _get_np_advanced_indexing(self, key):\n        idcs, new_axes = self._get_index_nd(key)\n        if type(idcs) == NDArray:  # pylint: disable=unidiomatic-typecheck\n            idcs = idcs.as_np_ndarray()\n        else:\n            idcs = _mx_nd_np.stack([i if isinstance(i, self.__class__) else i.as_np_ndarray() for i in idcs])\n        sliced = _npi.gather_nd(self, idcs)\n        # Reshape due to `None` entries in `key`.\n        if new_axes:\n            final_shape = [sliced.shape[i] for i in range(sliced.ndim)]\n            for ax in new_axes:  # pylint: disable=invalid-name\n                final_shape.insert(ax, 1)\n            return sliced.reshape(tuple(final_shape))\n        else:\n            return sliced\n\n    def _set_np_advanced_indexing(self, key, value):\n        \"\"\"This function is called by __setitem__ when key is an advanced index.\"\"\"\n        idcs, new_axes = self._get_index_nd(key)\n        if type(idcs) == NDArray:  # pylint: disable=unidiomatic-typecheck\n            idcs = idcs.as_np_ndarray()\n        else:\n            idcs = _mx_nd_np.stack([i if isinstance(i, self.__class__) else i.as_np_ndarray() for i in idcs])\n        vshape = get_oshape_of_gather_nd_op(self.shape, idcs.shape)\n        value_nd = self._prepare_value_nd(value, bcast_shape=vshape, squeeze_axes=new_axes)\n        self._scatter_set_nd(value_nd, idcs)\n\n    # pylint: disable=redefined-outer-name\n    def _get_np_boolean_indexing(self, key, ndim, shape):\n        \"\"\"\n        There are two types of boolean indices (which are equivalent,\n        for the most part though). This function will handle single\n        boolean indexing for higher speed.\n        If this is not the case, it is instead expanded into (multiple)\n        integer array indices and will be handled by advanced indexing.\n        \"\"\"\n        key_shape = key.shape\n        key_ndim = len(key_shape)\n        if ndim < key_ndim:\n            raise IndexError('too many indices, whose ndim = {}, for array with ndim = {}'\n                             .format(key_ndim, ndim))\n        for i in range(key_ndim):\n            if key_shape[i] != shape[i]:\n                raise IndexError('boolean index did not match indexed array along dimension {};'\n                                 ' dimension is {} but corresponding boolean dimension is {}'\n                                 .format(i, shape[i], key_shape[i]))\n        remaining_dims = shape[key_ndim:]\n        data = _reshape_view(self, -1, *remaining_dims)\n        key = _reshape_view(key, -1)\n        if data.size == 0 and key.size == 0:\n            return data\n        return _reshape_view(_npi.boolean_mask(data, key), -1, *remaining_dims)\n\n    def _set_np_boolean_indexing(self, key, value):\n        \"\"\"\n        There are two types of boolean indices (which are equivalent,\n        for the most part though). This function will handle single boolean assign for higher speed.\n        If this is not the case, it is instead expanded into (multiple)\n        integer array indices and will be handled by advanced assign.\n        \"\"\"\n        if isinstance(value, numeric_types):\n            _npi.boolean_mask_assign_scalar(data=self, mask=key,\n                                            value=int(value) if isinstance(value, bool) else value,\n                                            start_axis=0, out=self)\n        elif isinstance(value, ndarray):\n            _npi.boolean_mask_assign_tensor(data=self, mask=key, value=value, start_axis=0, out=self)\n        else:\n            raise NotImplementedError(f'type {type(value)} is not supported.')\n\n    # pylint: disable=too-many-return-statements\n    def __getitem__(self, key):\n        \"\"\"Return self[key].\n\n        Returns a sliced view of this array if the elements fetched are contiguous in memory;\n        otherwise, returns a newly created NDArray.\n        This functions supports advanced indexing defined in the following reference with\n        some restrictions. Boolean indexing is supported only for a single boolean ndarray\n        as a key. Mixing boolean ndarray with other index types is not supported in ``advanced``\n        indexing.\n\n        For basic indexing, i.e., if ``key`` consists only of integers,\n        ``slice``, ``Ellipsis`` (``...``) and ``None``, a mutable view is\n        returned that shares memory with this array if the accessed portion is\n        contiguous in memory.\n        Otherwise, a newly created ``ndarray`` is returned.\n\n        This functions supports advanced indexing as defined in `the NumPy\n        advanced indexing documentation\n        <https://docs.scipy.org/doc/numpy/reference/arrays.indexing.html#advanced-indexing>`_.\n\n        Parameters\n        ----------\n        key : int, slice, list, np.ndarray, mx.np.ndarray, or tuple of all previous types\n            Indexing key.\n\n        Examples\n        --------\n        The default is to give explicit indices for all axes:\n\n        >>> x = np.arange(6).reshape(2, 3)\n        >>> x\n        array([[0., 1., 2.],\n               [3., 4., 5.]])\n        >>> x[0, :2]\n        array([0., 1.])\n        >>> x[:, :-1]\n        array([[0., 1.],\n               [3., 4.]])\n\n        If fewer indices are given, they are automatically supplemented by an\n        appropriate number of ``slice(None)`` (\"``:``\") to the right. For\n        instance, a single integer indexes along the first axis:\n\n        >>> x[0]\n        array([0., 1., 2.])\n        >>> x[1:]\n        array([[3., 4., 5.]])\n\n        To omit a range of axes that should be kept as-is, an `Ellipsis`\n        (\"``...``\") can be used:\n\n        >>> x = np.arange(16).reshape(2, 2, 2, 2)\n        >>> x[0, ..., 1]\n        array([[1., 3.],\n               [5., 7.]])\n        >>> x[0, :, :, 1]  # equivalent\n        array([[1., 3.],\n               [5., 7.]])\n\n        New axes of length 1 can be created by inserting ``None``\n        (`numpy.newaxis`) in the index:\n\n        >>> x = np.arange(6).reshape(2, 3)\n        >>> x[None, :, :]\n        array([[[0., 1., 2.],\n                [3., 4., 5.]]])\n        >>> x[None, :, :].shape\n        (1, 2, 3)\n\n        If the indexed portion of the array is contiguous in memory, no data\n        is copied. Instead, a shared-memory view of the original array is\n        returned, and changes to that view affect the original array:\n\n        >>> x = np.arange(8).reshape(2, 2, 2)\n        >>> y = x[0]  # contiguous\n        >>> y\n        array([[0., 1.],\n               [2., 3.]])\n        >>> y[:] = -1\n        >>> x\n        array([[[-1., -1.],\n                [-1., -1.]],\n               [[ 4.,  5.],\n                [ 6.,  7.]]])\n        >>> x = np.arange(8).reshape(2, 2, 2)\n        >>> y = x[1, :1, :]  # contiguous\n        >>> y\n        array([[4., 5.]])\n        >>> y[:] = -1\n        >>> x\n        array([[[ 0.,  1.],\n                [ 2.,  3.]],\n               [[-1., -1.],\n                [ 6.,  7.]]])\n        >>> x = np.arange(0, 8).reshape(2, 2, 2)\n        >>> y = x[:, :, 1]  # not contiguous\n        >>> y\n        array([[1., 3.],\n               [5., 7.]])\n        >>> y[:] = -1\n        >>> x\n        array([[[0., 1.],\n                [2., 3.]],\n               [[4., 5.],\n                [6., 7.]]])\n\n        If the indexing key contains `list`, `numpy.ndarray` or `NDArray`\n        objects, advanced indexing is triggered, which always returns a\n        copy:\n\n        >>> x = np.arange(8).reshape(2, 2, 2)\n        >>> x[[0, 1]]\n        array([[[0., 1.],\n                [2., 3.]],\n               [[4., 5.],\n                [6., 7.]]])\n        >>> x[[0, 1], :]  # equivalent\n        array([[[0., 1.],\n                [2., 3.]],\n               [[4., 5.],\n                [6., 7.]]])\n        >>> y = np.array([0, 1], dtype='int32')\n        >>> x[1:, y]\n        array([[[4., 5.],\n                [6., 7.]]])\n        >>> y = np.array([0, 1], dtype='int32')\n        >>> x[1:, y]\n        array([[[4., 5.],\n                [6., 7.]]])\n\n        Get negative elements in an ndarray through boolean array indexing\n        >>> x = np.array([1., -1., -2., 3])\n        >>> x[x < 0]\n        array([-1., -2.])\n\n        For more imformation related to boolean indexing, please refer to\n        https://docs.scipy.org/doc/numpy-1.17.0/reference/arrays.indexing.html.\n        \"\"\"\n        ndim = self.ndim  # pylint: disable=redefined-outer-name\n        shape = self.shape  # pylint: disable=redefined-outer-name\n        if isinstance(key, bool): # otherwise will be treated as 0 and 1\n            key = array(key, dtype=_np.bool, device=self.device)\n        if isinstance(key, list):\n            try:\n                new_key = _np.array(key)\n                if new_key.dtype == _np.bool_:\n                    key = new_key\n            except Exception as err:\n                raise TypeError('{}'.format(str(err)))\n        if isinstance(key, _np.ndarray):\n            if dc.is_deferred_compute():\n                raise TypeError('Indexing with a numpy array is not supported in HybridBlock.')\n            if key.dtype == _np.bool_:\n                key = array(key, dtype='bool', device=self.device)\n\n        # Handle single boolean index of matching dimensionality and size first for higher speed\n        # If the boolean array is mixed with other idices, it is instead expanded into (multiple)\n        # integer array indices and will be handled by advanced indexing.\n        # Come before the check self.dim == 0 as it also handle the 0-dim case.\n        if isinstance(key, ndarray) and key.dtype == _np.bool_:\n            return self._get_np_boolean_indexing(key, ndim, shape)\n\n        all = __builtins__['all']  # `def all` below shadows the all builtin\n        if ndim == 0 and key != ():\n            raise IndexError('scalar tensor can only accept `()` as index')\n        # Handle simple cases for higher speed\n        if isinstance(key, tuple) and len(key) == 0:\n            return self\n        if isinstance(key, tuple) and len(key) == ndim\\\n                and py_all(isinstance(idx, integer_types) for idx in key):\n            out = self\n            for idx in key:\n                out = out[idx]\n            return out\n        if isinstance(key, integer_types):\n            # Equivalent to isinstance(key, integer_types) case in numpy/_symbol.py\n            if key > shape[0] - 1:\n                raise IndexError(\n                    'index {} is out of bounds for axis 0 with size {}'.format(\n                        key, shape[0]))\n            return self._at(key)\n        elif isinstance(key, py_slice):\n            # Unlike numpy/_symbol.py, calls MXNDArraySlice64 writable memory\n            # sharing if key.step not in [None, 1]. Equivalent otherwise to\n            # isinstance(key, py_slice) case in _symbol.py otherwise.\n            if key.step is None or key.step == 1:\n                if key.start is not None or key.stop is not None:\n                    return self._slice(key.start, key.stop)\n                else:\n                    return self\n            elif key.step != 0:\n                start = [None] if key.start is None else key.start\n                stop = [None] if key.stop is None else key.stop\n                return _npi.slice(self, start, stop, key.step)\n            else:\n                raise ValueError(\"slice step cannot be zero\")\n        elif isinstance(key, tuple) and \\\n           all((isinstance(arr, NDArray) and _np.issubdtype(arr.dtype, _np.integer) and \\\n                arr.ndim > 0) for arr in key):\n            # Equivalent case in numpy/_symbol.py\n            return _npi.advanced_indexing_multiple(self, _mx_nd_np.stack(key))\n        elif isinstance(key, tuple) and dc.is_deferred_compute():\n            # Equivalent to isinstance(key, tuple) case in numpy/_symbol.py\n            # Only enabled in deferred compute mode, as this codepath prevents\n            # memory sharing which may be desired in non-deferred compute\n            # imperative mode.\n            begin = []\n            end = []\n            step = []\n            new_shape = ()\n            assert len(key)  # len(key) == 0 is handled a above\n            unsupported = False\n            for index in key:\n                if isinstance(index, py_slice):\n                    if index.step is not None and index.step == 0:\n                        raise ValueError(\"slice step cannot be zero\")\n                    begin.append(index.start)\n                    end.append(index.stop)\n                    step.append(index.step)\n                    new_shape += (-2,)\n                elif isinstance(index, integer_types):\n                    if index >= 0:\n                        begin.append(index)\n                        end.append(index+1)\n                        step.append(1)\n                    else:\n                        begin.append(index)\n                        end.append(index - 1)\n                        step.append(-1)\n                    new_shape += (-3,)\n                else:\n                    unsupported = True\n                    break\n            if not unsupported:\n                new_shape += (-4,)\n                sliced = _npi.slice(self, begin, end, step)\n                return _mx_nd_np.reshape(sliced, new_shape)\n\n        # Special handling for cases only supported in imperative mode\n        if dc.is_deferred_compute():\n            raise TypeError('The type of indexing used is not supported in HybridBlock.')\n        # For 0-d boolean indices: A new axis is added,\n        # but at the same time no axis is \"used\". So if we have True,\n        # we add a new axis (a bit like with np.newaxis). If it is\n        # False, we add a new axis, but this axis has 0 entries.\n        # prepend is defined to handle this case.\n        # prepend = _NDARRAY_NO_ZERO_DIM_BOOL_ARRAY/-1 means there is no 0-d boolean scalar\n        # prepend = _NDARRAY_ZERO_DIM_BOOL_ARRAY_FALSE/0 means an zero dim must be expanded\n        # prepend = _NDARRAY_ZERO_DIM_BOOL_ARRAY_TRUE/1 means a new axis must be prepended\n        key, prepend = indexing_key_expand_implicit_axes(key, self.shape)\n        indexing_dispatch_code = get_indexing_dispatch_code(key)\n        if indexing_dispatch_code == _NDARRAY_EMPTY_TUPLE_INDEXING:\n            # won't be affected by zero-dim boolean indices\n            return self._get_np_empty_tuple_indexing(key)\n        elif indexing_dispatch_code == _NDARRAY_BASIC_INDEXING:\n            if prepend == _NDARRAY_ZERO_DIM_BOOL_ARRAY_FALSE:\n                return empty((0,) + self._get_np_basic_indexing(key).shape,\n                             dtype=self.dtype, device=self.device)\n            if prepend == _NDARRAY_ZERO_DIM_BOOL_ARRAY_TRUE:\n                key = (_np.newaxis,) + key\n            return self._get_np_basic_indexing(key)\n        elif indexing_dispatch_code == _NDARRAY_ADVANCED_INDEXING:\n            if prepend == _NDARRAY_ZERO_DIM_BOOL_ARRAY_FALSE:\n                return empty((0,) + self._get_np_adanced_indexing(key).shape,\n                             dtype=self.dtype, device=self.device)\n            if prepend == _NDARRAY_ZERO_DIM_BOOL_ARRAY_TRUE:\n                key = (_np.newaxis,) + key\n            return self._get_np_advanced_indexing(key)\n        else:\n            raise RuntimeError\n\n    # pylint: disable=inconsistent-return-statements\n    def __setitem__(self, key, value):\n        \"\"\"Sets ``self[key]`` to ``value``.\n\n        This functions supports advanced indexing as defined in `the NumPy\n        advanced indexing documentation\n        <https://docs.scipy.org/doc/numpy/reference/arrays.indexing.html#advanced-indexing>`_,\n        with the restriction that boolean array indexing is not supported.\n\n        Parameters\n        ----------\n        key : int, slice, list, np.ndarray, mx.np.ndarray, or tuple of all previous types\n            The indexing key.\n        value : scalar or array-like object that can be broadcast to the shape of self[key]\n            The value to set.\n\n        Examples\n        --------\n        >>> x = np.zeros((2, 3))\n        >>> x[:] = 1\n        >>> x\n        array([[ 1.,  1.,  1.],\n               [ 1.,  1.,  1.]])\n        >>> x[:, 1:2] = 2\n        >>> x\n        array([[ 1.,  2.,  1.],\n               [ 1.,  2.,  1.]])\n        >>> x[1:2, 1:] = 3\n        >>> x\n        array([[ 1.,  2.,  1.],\n               [ 1.,  3.,  3.]])\n        >>> x[1:, 0:2] = np.zeros((1, 2))\n        >>> x\n        array([[ 1.,  2.,  1.],\n               [ 0.,  0.,  3.]])\n        >>> x[1, 2] = 4\n        >>> x\n        array([[ 1.,  2.,  1.],\n               [ 0.,  0.,  4.]])\n        >>> x[[0], [1, 2]] = 5\n        >>> x\n        array([[ 1.,  5.,  5.],\n               [ 0.,  0.,  4.]])\n        >>> x[::-1, 0:2:2] = [6]\n        >>> x\n        array([[ 6.,  5.,  5.],\n               [ 6.,  0.,  4.]])\n\n        For imformation related to boolean indexing, please refer to\n        https://docs.scipy.org/doc/numpy-1.17.0/reference/arrays.indexing.html.\n        \"\"\"\n        if isinstance(value, NDArray) and not isinstance(value, ndarray):\n            raise TypeError('Cannot assign mx.nd.NDArray to mxnet.numpy.ndarray')\n        if isinstance(key, bool): # otherwise will be treated as 0 and 1\n            key = array(key, dtype=_np.bool)\n\n        # Handle single boolean assign of matching dimensionality and size first for higher speed\n        # If the boolean array is mixed with other idices, it is instead expanded into (multiple)\n        # integer array indices and will be handled by advanced assign.\n        # Come before the check self.dim == 0 as it also handle the 0-dim case.\n        if isinstance(key, ndarray) and key.dtype == _np.bool:\n            return self._set_np_boolean_indexing(key, value)\n\n        # handle basic and advanced indexing\n        if self.ndim == 0:\n            if not isinstance(key, tuple) or len(key) != 0:\n                raise IndexError('scalar tensor can only accept `()` as index')\n            if isinstance(value, numeric_types):\n                self._full(value)\n            elif isinstance(value, ndarray) and value.size == 1:\n                if value.shape != self.shape:\n                    value = value.reshape(self.shape)\n                value.copyto(self)\n            elif isinstance(value, (_np.ndarray, _np.generic)) and value.size == 1:\n                if isinstance(value, _np.generic) or value.shape != self.shape:\n                    value = value.reshape(self.shape)\n                self._sync_copyfrom(value)\n            else:\n                raise ValueError('setting an array element with a sequence.')\n        else:\n            # For 0-d boolean indices: A new axis is added,\n            # but at the same time no axis is \"used\". So if we have True,\n            # we add a new axis (a bit like with np.newaxis). If it is\n            # False, we add a new axis, but this axis has 0 entries.\n            # prepend is defined to handle this case.\n            # prepend == _NDARRAY_NO_ZERO_DIM_BOOL_ARRAY/-1 means there is no 0-d boolean scalar\n            # prepend == _NDARRAY_ZERO_DIM_BOOL_ARRAY_FALSE/0 means an zero dim must be expanded\n            # prepend == _NDARRAY_ZERO_DIM_BOOL_ARRAY_TRUE/1 means a new axis must be expanded\n            # prepend actually has no influence on __setitem__\n            key, prepend = indexing_key_expand_implicit_axes(key, self.shape)\n            if prepend == _NDARRAY_ZERO_DIM_BOOL_ARRAY_FALSE:\n                return # no action is needed\n            slc_key = tuple(idx for idx in key if idx is not None)\n            if len(slc_key) < self.ndim:\n                raise RuntimeError(\n                    'too few indices after normalization: expected `ndim` ({}) '\n                    'but got {}. This is a bug, please report it!'\n                    ''.format(self.ndim, len(slc_key))\n                )\n            if len(slc_key) > self.ndim and self.ndim != 0:\n                raise IndexError(\n                    'too many indices ({}) for array with {} dimensions'\n                    ''.format(len(slc_key), self.ndim)\n                )\n            indexing_dispatch_code = get_indexing_dispatch_code(slc_key)\n            if indexing_dispatch_code == _NDARRAY_BASIC_INDEXING:\n                self._set_nd_basic_indexing(key, value)  # function is inheritated from NDArray class\n            elif indexing_dispatch_code == _NDARRAY_EMPTY_TUPLE_INDEXING:\n                pass # no action needed\n            elif indexing_dispatch_code == _NDARRAY_ADVANCED_INDEXING:\n                self._set_np_advanced_indexing(key, value)\n            else:\n                raise ValueError(\n                    'Indexing NDArray with index {} of type {} is not supported'\n                    ''.format(key, type(key))\n                )\n\n    def _prepare_value_nd(self, value, bcast_shape, squeeze_axes=None):\n        \"\"\"Return a broadcast `ndarray` with same device and dtype as ``self``.\n        For setting item, The returned `ndarray` is squeezed according to squeeze_axes since the\n        value_nd is assigned to not yet expanded space in original array.\n        `value`: numeric types or array like.\n        `bcast_shape`: a shape tuple.\n        `squeeze_axes`: a sequence of axes to squeeze in the value array.\n        Note: mxnet.numpy.ndarray not support NDArray as assigned value.\n        \"\"\"\n        if isinstance(value, numeric_types):\n            value_nd = full(bcast_shape, value, device=self.device, dtype=self.dtype)\n        elif isinstance(value, self.__class__):\n            value_nd = value.to_device(self.device)\n            if value_nd.dtype != self.dtype:\n                value_nd = value_nd.astype(self.dtype)\n        else:\n            try:\n                value_nd = array(value, device=self.device, dtype=self.dtype)\n            except:\n                raise TypeError('mxnet.np.ndarray does not support assignment with non-array-like '\n                                'object {} of type {}'.format(value, type(value)))\n\n        # For advanced indexing setitem, if there is None in indices, we need to squeeze the\n        # assigned value_nd since None is also ignored in slicing the original array.\n        if squeeze_axes and value_nd.ndim > len(bcast_shape):\n            squeeze_axes = tuple([ax for ax in squeeze_axes if ax < len(value_nd.shape)])\n            value_nd = value_nd.squeeze(axis=tuple(squeeze_axes))\n\n        # handle the cases like the following\n        # a = np.zeros((3, 3)), b = np.ones((1, 1, 1, 1, 3)), a[0] = b\n        # b cannot broadcast directly to a[0].shape unless its leading 1-size axes are trimmed\n        if value_nd.ndim > len(bcast_shape):\n            squeeze_axes = []\n            for i in range(value_nd.ndim - len(bcast_shape)):\n                if value_nd.shape[i] == 1:\n                    squeeze_axes.append(i)\n                else:\n                    break\n            if squeeze_axes:\n                value_nd = value_nd.squeeze(squeeze_axes)\n\n        if value_nd.shape != bcast_shape:\n            if value_nd.size == 0:\n                value_nd = value_nd.reshape(bcast_shape)\n            else:\n                value_nd = value_nd.broadcast_to(bcast_shape)\n        return value_nd\n\n    @wrap_mxnp_np_ufunc\n    def __add__(self, other):\n        \"\"\"x.__add__(y) <=> x + y\"\"\"\n        return add(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __iadd__(self, other):\n        \"\"\"x.__iadd__(y) <=> x += y\"\"\"\n        if not self.writable:\n            raise ValueError('trying to add to a readonly ndarray')\n        return add(self, other, out=self)\n\n    @wrap_mxnp_np_ufunc\n    def __radd__(self, other):\n        \"\"\"x.__radd__(y) <=> y + x\"\"\"\n        return add(other, self)\n\n    def __invert__(self):\n        \"\"\"x.__invert__() <=> ~x\"\"\"\n        return invert(self)\n\n    @wrap_mxnp_np_ufunc\n    def __and__(self, other):\n        \"\"\"x.__and__(y) <=> x & y\"\"\"\n        return bitwise_and(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __rand__(self, other):\n        \"\"\"x.__rand__(y) <=> y & x\"\"\"\n        return bitwise_and(other, self)\n\n    @wrap_mxnp_np_ufunc\n    def __or__(self, other):\n        \"\"\"x.__or__(y) <=> x | y\"\"\"\n        return bitwise_or(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __ror__(self, other):\n        \"\"\"x.__ror__(y) <=> y | x\"\"\"\n        return bitwise_or(other, self)\n\n    @wrap_mxnp_np_ufunc\n    def __xor__(self, other):\n        \"\"\"x.__xor__(y) <=> x ^ y\"\"\"\n        return bitwise_xor(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __rxor__(self, other):\n        \"\"\"x.__rxor__(y) <=> y ^ x\"\"\"\n        return bitwise_xor(other, self)\n\n    @wrap_mxnp_np_ufunc\n    def __lshift__(self, other):\n        \"\"\"x.__lshift__(y) <=> x << y\"\"\"\n        return bitwise_left_shift(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __rshift__(self, other):\n        \"\"\"x.__rshift__(y) <=> x >> y\"\"\"\n        return bitwise_right_shift(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __iand__(self, other):\n        \"\"\"x.__iand__(y) <=> x &= y\"\"\"\n        return bitwise_and(self, other, out=self)\n\n    @wrap_mxnp_np_ufunc\n    def __ior__(self, other):\n        r\"\"\"x.__ior__(y) <=> x \\|= y\"\"\"\n        return bitwise_or(self, other, out=self)\n\n    @wrap_mxnp_np_ufunc\n    def __ixor__(self, other):\n        \"\"\"x.__ixor__(y) <=> x ^= y\"\"\"\n        return bitwise_xor(self, other, out=self)\n\n    @wrap_mxnp_np_ufunc\n    def __ilshift__(self, other):\n        \"\"\"x.__ilshift__(y) <=> x <<= y\"\"\"\n        return bitwise_left_shift(self, other, out=self)\n\n    @wrap_mxnp_np_ufunc\n    def __irshift__(self, other):\n        \"\"\"x.__irshift__(y) <=> x >>= y\"\"\"\n        return bitwise_right_shift(self, other, out=self)\n\n    @wrap_mxnp_np_ufunc\n    def __rlshift__(self, other):\n        \"\"\"x.__rlshift__(y) <=> y << x\"\"\"\n        return bitwise_left_shift(other, self)\n\n    @wrap_mxnp_np_ufunc\n    def __rrshift__(self, other):\n        \"\"\"x.__rrshift__(y) <=> y >> x\"\"\"\n        return bitwise_right_shift(other, self)\n\n    def __round__(self, n=0):\n        \"\"\"x.__round__(n)\"\"\"\n        return round(self, decimals=n)\n\n    def __abs__(self):\n        \"\"\"x.__abs__()\"\"\"\n        return absolute(self)\n\n    def __ceil__(self):\n        \"\"\"x.__ceil__()\"\"\"\n        return ceil(self)\n\n    def __floor__(self):\n        \"\"\"x.__floor__()\"\"\"\n        return floor(self)\n\n    def __trunc__(self):\n        \"\"\"x.__trunc__()\"\"\"\n        return trunc(self)\n\n    @wrap_mxnp_np_ufunc\n    def __sub__(self, other):\n        \"\"\"x.__sub__(y) <=> x - y\"\"\"\n        return subtract(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __isub__(self, other):\n        \"\"\"x.__isub__(y) <=> x -= y\"\"\"\n        if not self.writable:\n            raise ValueError('trying to subtract from a readonly ndarray')\n        return subtract(self, other, out=self)\n\n    @wrap_mxnp_np_ufunc\n    def __rsub__(self, other):\n        \"\"\"x.__rsub__(y) <=> y - x\"\"\"\n        return subtract(other, self)\n\n    @wrap_mxnp_np_ufunc\n    def __mul__(self, other):\n        \"\"\"x.__mul__(y) <=> x * y\"\"\"\n        return multiply(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __floordiv__(self, other):\n        \"\"\"x.__floordiv__(y) <=> x // y\"\"\"\n        return floor_divide(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __ifloordiv__(self, other):\n        \"\"\"x.__ifloordiv__(y) <=> x //= y\"\"\"\n        if not self.writable:\n            raise ValueError('trying to divide from a readonly ndarray')\n        return floor_divide(self, other, out=self)\n\n    @wrap_mxnp_np_ufunc\n    def __rfloordiv__(self, other):\n        \"\"\"x.__rfloordiv__(y) <=> y // x\"\"\"\n        return floor_divide(other, self)\n\n    def __neg__(self):\n        \"\"\"x.__neg__() <=> -x\"\"\"\n        return negative(self)\n\n    def __pos__(self):\n        \"\"\"x.__pos__() <=> +x\"\"\"\n        return positive(self)\n\n    @wrap_mxnp_np_ufunc\n    def __imul__(self, other):\n        r\"\"\"x.__imul__(y) <=> x \\*= y\"\"\"\n        if not self.writable:\n            raise ValueError('trying to add to a readonly ndarray')\n        return multiply(self, other, out=self)\n\n    @wrap_mxnp_np_ufunc\n    def __rmul__(self, other):\n        \"\"\"x.__rmul__(y) <=> y * x\"\"\"\n        return self.__mul__(other)\n\n    @wrap_mxnp_np_ufunc\n    def __div__(self, other):\n        \"\"\"x.__div__(y) <=> x / y\"\"\"\n        return divide(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __rdiv__(self, other):\n        \"\"\"x.__rdiv__(y) <=> y / x\"\"\"\n        return divide(other, self)\n\n    @wrap_mxnp_np_ufunc\n    def __idiv__(self, other):\n        \"\"\"x.__idiv__(y) <=> x /= y\"\"\"\n        return divide(self, other, out=self)\n\n    @wrap_mxnp_np_ufunc\n    def __truediv__(self, other):\n        \"\"\"x.__truediv__(y) <=> x / y\"\"\"\n        return divide(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __rtruediv__(self, other):\n        \"\"\"x.__rtruediv__(y) <=> y / x\"\"\"\n        return divide(other, self)\n\n    @wrap_mxnp_np_ufunc\n    def __itruediv__(self, other):\n        \"\"\"x.__itruediv__(y) <=> x /= y\"\"\"\n        return divide(self, other, out=self)\n\n    @wrap_mxnp_np_ufunc\n    def __mod__(self, other):\n        \"\"\"x.__mod__(y) <=> x % y\"\"\"\n        return mod(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __rmod__(self, other):\n        \"\"\"x.__rmod__(y) <=> y % x\"\"\"\n        return mod(other, self)\n\n    @wrap_mxnp_np_ufunc\n    def __imod__(self, other):\n        \"\"\"x.__imod__(y) <=> x %= y\"\"\"\n        return mod(self, other, out=self)\n\n    @wrap_mxnp_np_ufunc\n    def __pow__(self, other):\n        \"\"\"x.__pow__(y) <=> x ** y\"\"\"\n        return power(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __rpow__(self, other):\n        \"\"\"x.__rpow__(y) <=> y ** x\"\"\"\n        return power(other, self)\n\n    @wrap_mxnp_np_ufunc\n    def __ipow__(self, other):\n        \"\"\"x.__ipow__(y) <=> x **= y\"\"\"\n        return power(self, other, out=self)\n\n    @wrap_mxnp_np_ufunc\n    def __eq__(self, other):\n        \"\"\"x.__eq__(y) <=> x == y\"\"\"\n        return equal(self, other)\n\n    def __hash__(self):\n        raise NotImplementedError\n\n    @wrap_mxnp_np_ufunc\n    def __ne__(self, other):\n        \"\"\"x.__ne__(y) <=> x != y\"\"\"\n        return not_equal(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __gt__(self, other):\n        \"\"\"x.__gt__(y) <=> x > y\"\"\"\n        return greater(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __ge__(self, other):\n        \"\"\"x.__ge__(y) <=> x >= y\"\"\"\n        return greater_equal(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __lt__(self, other):\n        \"\"\"x.__lt__(y) <=> x < y\"\"\"\n        return less(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __le__(self, other):\n        \"\"\"x.__le__(y) <=> x <= y\"\"\"\n        return less_equal(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __matmul__(self, other):\n        \"\"\"x.__matmul__(y) <=> x @ y\"\"\"\n        return matmul(self, other)\n\n    @wrap_mxnp_np_ufunc\n    def __rmatmul__(self, other):\n        \"\"\"x.__rmatmul__(y) <=> y @ x\"\"\"\n        return matmul(other, self)\n\n    @wrap_mxnp_np_ufunc\n    def __imatmul__(self, other):\n        \"\"\"x.__imatmul__(y) <=> x @= y\"\"\"\n        return matmul(self, other, out=self)\n\n    def __bool__(self):\n        num_elements = self.size\n        if num_elements == 0:\n            warnings.simplefilter('default')\n            warnings.warn('The truth value of an empty array is ambiguous. Returning False, but in'\n                          ' future this will result in an error.', DeprecationWarning)\n            return False\n        elif num_elements == 1:\n            return bool(self.item())\n        else:\n            raise ValueError(\"The truth value of an ndarray with multiple elements is ambiguous.\")\n\n    __nonzero__ = __bool__\n\n    def __index__(self):\n        if self.ndim == 0 and _np.issubdtype(self.dtype, _np.integer):\n            return self.item()\n        raise TypeError('only integer scalar arrays can be converted to a scalar index')\n\n    def __float__(self):\n        num_elements = self.size\n        if num_elements != 1:\n            raise TypeError('only size-1 arrays can be converted to Python scalars')\n        return float(self.item())\n\n    def __int__(self):\n        num_elements = self.size\n        if num_elements != 1:\n            raise TypeError('only size-1 arrays can be converted to Python scalars')\n        return int(self.item())\n\n    def __len__(self):\n        \"\"\"Number of elements along the first axis.\"\"\"\n        shape = self.shape  # pylint: disable=redefined-outer-name\n        if len(shape) == 0:\n            raise TypeError('len() of unsized object')\n        return self.shape[0]\n\n    def __reduce__(self):\n        return ndarray, (None,), self.__getstate__()\n\n    def item(self, *args):\n        \"\"\"Copy an element of an array to a standard Python scalar and return it.\n\n        Parameters\n        ----------\n        *args : Arguments (variable number and type)\n            none: in this case, the method only works for arrays with one element (a.size == 1),\n            which element is copied into a standard Python scalar object and returned.\n\n            int_type: this argument is interpreted as a flat index into the array, specifying which\n            element to copy and return.\n\n            tuple of int_types: functions as does a single int_type argument, except that the\n            argument is interpreted as an nd-index into the array.\n\n        Returns\n        -------\n        z : Standard Python scalar object\n            A copy of the specified element of the array as a suitable Python scalar.\n        \"\"\"\n        # TODO(junwu): no need to call asnumpy() on the whole array.\n        return self.asnumpy().item(*args)\n\n    def nonzero(self):\n        \"\"\"Return the indices of the elements that are non-zero.\n\n        Refer to `numpy.nonzero` for full documentation.\n\n        See Also\n        --------\n        numpy.nonzero : equivalent function\n        \"\"\"\n        return nonzero(self)\n\n    @property\n    # pylint: disable= invalid-name, undefined-variable\n    def T(self):\n        \"\"\"Same as self.transpose(). This always returns a copy of self.\"\"\"\n        if self.ndim != 2:\n            warnings.warn('x.T requires x to have 2 dimensions. '\n                          'Use x.mT to transpose stacks of matrices and '\n                          'permute_dims() to permute dimensions.')\n        return self.transpose()\n    # pylint: enable= invalid-name, undefined-variable\n\n    @property\n    # pylint: disable= invalid-name, undefined-variable\n    def mT(self):\n        \"\"\"Same as self.transpose(). This always returns a copy of self.\"\"\"\n        if self.ndim < 2:\n            raise ValueError(\"x must be at least 2-dimensional for matrix_transpose\")\n        return _mx_nd_np.swapaxes(self, -1, -2)\n    # pylint: enable= invalid-name, undefined-variable\n\n    def all(self, axis=None, out=None, keepdims=False):\n        return _mx_nd_np.all(self, axis=axis, out=out, keepdims=keepdims)\n\n    def any(self, axis=None, out=None, keepdims=False):\n        return _mx_nd_np.any(self, axis=axis, out=out, keepdims=keepdims)\n\n    def as_nd_ndarray(self):\n        \"\"\"Convert mxnet.numpy.ndarray to mxnet.ndarray.NDArray to use its fluent methods.\"\"\"\n        hdl = NDArrayHandle()\n        check_call(_LIB.MXShallowCopyNDArray(self.handle, ctypes.byref(hdl)))\n        return NDArray(handle=hdl, writable=self.writable)\n\n    def as_np_ndarray(self):\n        \"\"\"A convenience function for creating a numpy ndarray from the current ndarray\n        with zero copy. For this class, it just returns itself since it's already a\n        numpy ndarray.\"\"\"\n        return self\n\n    def __repr__(self):\n        \"\"\"\n        Returns a string representation of the array.\n        The dtype of the ndarray will be appended if it's inconsistent with current dtype.\n        The device of the ndarray will be appended for devices other than CPU.\n\n        Examples\n        --------\n        >>> from mxnet import np, npx\n        >>> a = np.random.uniform(size=(2, 3))\n        >>> a\n        array([[0.5488135 , 0.5928446 , 0.71518934],\n               [0.84426576, 0.60276335, 0.8579456 ]])\n        >>> print(a)\n        [[0.5488135  0.5928446  0.71518934]\n         [0.84426576 0.60276335 0.8579456 ]]\n        >>> a.dtype\n        dtype('float32')\n        >>> npx.set_np_float64()\n        >>> a\n        array([[0.5488135 , 0.5928446 , 0.71518934],\n               [0.84426576, 0.60276335, 0.8579456 ]], dtype=float32)\n        >>> npx.set_np_float64(default_float64=False)\n        >>> a\n        array([[0.5488135 , 0.5928446 , 0.71518934],\n               [0.84426576, 0.60276335, 0.8579456 ]])\n        >>> b = a.astype(np.float64)\n        >>> b\n        array([[0.54881352, 0.59284461, 0.71518934],\n               [0.84426576, 0.60276335, 0.85794562]], dtype=float64)\n        >>> print(b)\n        [[0.54881352 0.59284461 0.71518934]\n         [0.84426576 0.60276335 0.85794562]]\n        >>> b.dtype\n        dtype('float64')\n        >>> c = a.copyto(npx.gpu(0))\n        >>> c\n        array([[0.5488135 , 0.5928446 , 0.71518934],\n               [0.84426576, 0.60276335, 0.8579456 ]], device=gpu(0))\n        >>> print(c)\n        [[0.5488135  0.5928446  0.71518934]\n         [0.84426576 0.60276335 0.8579456 ]] @gpu(0)\n        >>> d = b.copyto(npx.gpu(0))\n        >>> d\n        array([[0.54881352, 0.59284461, 0.71518934],\n               [0.84426576, 0.60276335, 0.85794562]], dtype=float64, device=gpu(0))\n        >>> print(d)\n        [[0.54881352 0.59284461 0.71518934]\n         [0.84426576 0.60276335 0.85794562]] @gpu(0)\n\n        \"\"\"\n        if self._alive:\n            array_str = self.asnumpy().__repr__()\n            dtype = self.dtype\n            default_dtype = _np.float64 if is_np_default_dtype() else _np.float32\n            if 'dtype=' in array_str:\n                if dtype == default_dtype:\n                    array_str = array_str[:array_str.rindex(',')] + ')'\n            elif dtype not in (default_dtype, _np.bool_):\n                array_str = array_str[:-1] + ', dtype={})'.format(dtype)\n\n            device = self.device\n            if device.device_type == 'cpu':\n                return array_str\n            return array_str[:-1] + ', device={})'.format(str(device))\n        else:\n            return '<FREED {}>'.format(self.__class__.__name__)\n\n    def __str__(self):\n        \"\"\"Returns a string representation of the array.\"\"\"\n        array_str = self.asnumpy().__str__()\n        device = self.device\n        if device.device_type == 'cpu' or self.ndim == 0:\n            return array_str\n        return '{array} @{device}'.format(array=array_str, device=device)\n\n    def __format__(self, fmt):\n        \"\"\"Return value.__format__(format_spec). Overwrite to include 0-d array\"\"\"\n        if self.ndim == 0:\n            return self.item().__format__(fmt)\n        elif len(fmt) == 0:\n            return self.__str__().__format__(fmt)\n        else:\n            raise TypeError(\"Cannot format mxnet.numpy.ndarray with format_spec\")\n\n    def attach_grad(self, grad_req='write'):  # pylint: disable=arguments-differ\n        \"\"\"Attach a gradient buffer to this ndarray, so that `backward`\n        can compute gradient with respect to it.\n\n        Parameters\n        ----------\n        grad_req : {'write', 'add', 'null'}\n            How gradient will be accumulated.\n            * 'write': gradient will be overwritten on every backward.\n            * 'add': gradient will be added to existing value on every backward.\n            * 'null': do not compute gradient for this NDArray.\n        \"\"\"\n        grad = _mx_nd_np.zeros_like(self)  # pylint: disable=undefined-variable\n        grad_req = _GRAD_REQ_MAP[grad_req]\n        check_call(_LIB.MXAutogradMarkVariables(\n            1, ctypes.pointer(self.handle),\n            ctypes.pointer(mx_uint(grad_req)),\n            ctypes.pointer(grad.handle)))\n\n    def drop_grad(self):\n        \"\"\"Free the memory of the marked ndarray.\"\"\"\n        check_call(_LIB.MXAutogradDropGrads(\n            1, ctypes.pointer(self.handle)))\n\n    @property\n    def grad(self):\n        \"\"\"Returns gradient buffer attached to this ndarray.\"\"\"\n        hdl = NDArrayHandle()\n        check_call(_LIB.MXNDArrayGetGrad(self.handle, ctypes.byref(hdl)))\n        if hdl.value is None:\n            return None\n        return _np_ndarray_cls(hdl)\n\n    def detach(self):\n        \"\"\"Returns a new ndarray, detached from the current graph.\"\"\"\n        hdl = NDArrayHandle()\n        check_call(_LIB.MXNDArrayDetach(self.handle, ctypes.byref(hdl)))\n        return _np_ndarray_cls(hdl)\n\n    def astype(self, dtype, order='K', casting='unsafe', subok=True, copy=True):  # pylint: disable=arguments-differ,unused-argument, too-many-arguments\n        \"\"\"\n        Copy of the array, cast to a specified type.\n\n        Parameters\n        ----------\n        dtype : str or dtype\n            Typecode or data-type to which the array is cast.\n        order : {'C', 'F', 'A', 'K'}, optional\n            Controls the memory layout order of the result.\n            'C' means C order, 'F' means Fortran order, 'A'\n            means 'F' order if all the arrays are Fortran contiguous,\n            'C' order otherwise, and 'K' means as close to the\n            order the array elements appear in memory as possible.\n            Default is 'K'.\n        casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional\n            Controls what kind of data casting may occur. Defaults to 'unsafe'\n            for backwards compatibility.\n\n              * 'no' means the data types should not be cast at all.\n              * 'equiv' means only byte-order changes are allowed.\n              * 'safe' means only casts which can preserve values are allowed.\n              * 'same_kind' means only safe casts or casts within a kind,\n                like float64 to float32, are allowed.\n              * 'unsafe' means any data conversions may be done.\n        subok : bool, optional\n            If True, then sub-classes will be passed-through (default), otherwise\n            the returned array will be forced to be a base-class array.\n        copy : bool, optional\n            Default `True`. By default, astype always returns a newly\n            allocated ndarray on the same device. If this is set to\n            `False`, and the dtype requested is the same as the ndarray's\n            dtype, the ndarray is returned instead of a copy.\n\n        Returns\n        -------\n        arr_t : ndarray\n            Unless `copy` is False and the other conditions for returning the input\n            array are satisfied (see description for `copy` input parameter), `arr_t`\n            is a new array of the same shape as the input array with `dtype`.\n\n        Notes\n        -----\n        This function differs from the official `ndarray`'s ``astype`` function in the following\n        aspects:\n            * `order` only supports 'C' and 'K'.\n            * `casting` only supports 'unsafe'.\n            * `subok` only supports ``True``.\n        \"\"\"\n        if order is not None and order != 'K' and order != 'C':\n            raise ValueError('order must be either \\'K\\' or \\'C\\'')\n        if casting != 'unsafe':\n            raise ValueError('casting must be equal to \\'unsafe\\'')\n        if not subok:\n            raise ValueError('subok must be equal to True')\n        if dtype is None:\n            dtype = _np.float32\n        if not copy and _np.dtype(dtype) == self.dtype:\n            return self\n\n        return _npi.cast(self, dtype=dtype)\n\n    def copyto(self, other):\n        \"\"\"Copies the value of this array to another array.\n\n        If ``other`` is a ``ndarray`` object, then ``other.shape`` and\n        ``self.shape`` should be the same. This function copies the value from\n        ``self`` to ``other``.\n\n        If ``other`` is a device, a new ``np.ndarray`` will be first created on\n        the target device, and the value of ``self`` is copied.\n\n        Parameters\n        ----------\n        other : ndarray or Device\n            The destination array or device.\n\n        Returns\n        -------\n        out: ndarray\n            The copied array. If ``other`` is an ``ndarray``, then the return value\n            and ``other`` will point to the same ``ndarray``.\n\n        Examples\n        --------\n        >>> x = np.ones((2, 3))\n        >>> y = np.zeros((2, 3), device=npx.gpu(0))\n        >>> z = x.copyto(y)\n        >>> z is y\n        True\n        >>> y\n        array([[ 1.,  1.,  1.],\n               [ 1.,  1.,  1.]])\n        \"\"\"\n        if isinstance(other, ndarray):\n            if other.handle is self.handle:\n                warnings.warn('You are attempting to copy an array to itself', RuntimeWarning)\n                return False\n            return _npi.copyto(self, out=other)\n        elif isinstance(other, Device):\n            hret = ndarray(_new_alloc_handle(self.shape, other, True, self.dtype))\n            return _npi.copyto(self, out=hret)\n        else:\n            raise TypeError('copyto does not support type ' + str(type(other)))\n\n    def asscalar(self):\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute asscalar')\n\n    def argmax(self, axis=None, out=None, keepdims=False):  # pylint: disable=arguments-differ\n        \"\"\"Return indices of the maximum values along the given axis.\n        Refer to `mxnet.numpy.argmax` for full documentation.\"\"\"\n        return argmax(self, axis, out, keepdims)\n\n    def as_in_context(self, context):\n        \"\"\"This function has been deprecated. Please refer to ``ndarray.to_device``.\"\"\"\n        warnings.warn('ndarray.as_in_context has been renamed to'\n                      ' ndarray.to_device', DeprecationWarning)\n        return self.as_nd_ndarray().as_in_context(context).as_np_ndarray()\n\n    def as_in_ctx(self, ctx):\n        \"\"\"This function has been deprecated. Please refer to ``ndarray.to_device``.\"\"\"\n        warnings.warn('ndarray.to_device has been renamed to'\n                      ' ndarray.to_device', DeprecationWarning)\n        return self.to_device(ctx)\n\n    @property\n    def ctx(self):\n        \"\"\"This property has been deprecated. Please refer to ``ndarray.device``.\"\"\"\n        warnings.warn('ndarray.ctx has been renamed to ndarray.device', DeprecationWarning)\n        return self.device\n\n\n    def to_device(self, device):\n        \"\"\"Returns an array on the target device with the same value as this array.\n\n        If the target device is the same as ``self.device``, then ``self`` is\n        returned.  Otherwise, a copy is made.\n\n        Parameters\n        ----------\n        device : Device\n            The target device.\n\n        Returns\n        -------\n        ndarray\n            The target array.\n        \"\"\"\n        if self.device == device:\n            return self\n        return self.copyto(device)\n\n    @property\n    def device(self):\n        \"\"\"Hardware device the array data resides on.\n\n        Examples\n        --------\n        >>> x = np.array([1, 2, 3, 4])\n        >>> x.device\n        cpu(0)\n        >>> type(x.device)\n        <class 'mxnet.device.Device'>\n        >>> y = np.zeros((2, 3), npx.gpu(0))\n        >>> y.device\n        gpu(0)\n        \"\"\"\n        dev_typeid = ctypes.c_int()\n        dev_id = ctypes.c_int()\n        check_call(_LIB.MXNDArrayGetContext(\n            self.handle, ctypes.byref(dev_typeid), ctypes.byref(dev_id)))\n        return Device(Device.devtype2str[dev_typeid.value], dev_id.value)\n\n\n    @property\n    def context(self):\n        \"\"\"This function has been deprecated. Please refer to ``ndarray.ctx``.\"\"\"\n        warnings.warn('ndarray.context has been renamed to ndarray.ctx', DeprecationWarning)\n        return self.as_nd_ndarray().context\n\n    def copy(self, order='C'):  # pylint: disable=arguments-differ\n        \"\"\"Return a coyp of the array, keeping the same device.\n\n        Parameters\n        ----------\n        order : str\n            The memory layout of the copy. Currently, only c-contiguous memory\n            layout is supported.\n\n        Examples\n        --------\n        >>> x = np.ones((2, 3))\n        >>> y = x.copy()\n        >>> y\n        array([[ 1.,  1.,  1.],\n               [ 1.,  1.,  1.]])\n        \"\"\"\n        if order != 'C':\n            raise NotImplementedError('ndarray.copy only supports order=\\'C\\', while '\n                                      'received {}'.format(str(order)))\n        return self.copyto(self.device)\n\n    def dot(self, b, out=None):\n        \"\"\"Dot product of two arrays.\n        Refer to ``numpy.dot`` for full documentation.\"\"\"\n        return dot(self, b, out=out)\n\n    def reshape(self, *args, **kwargs):  # pylint: disable=arguments-differ\n        \"\"\"Returns a copy of the array with a new shape.\n\n        Notes\n        -----\n        Unlike the free function `numpy.reshape`, this method on `ndarray` allows\n        the elements of the shape parameter to be passed in as separate arguments.\n        For example, ``a.reshape(10, 11)`` is equivalent to\n        ``a.reshape((10, 11))``.\n        \"\"\"\n        order = 'C'\n        if len(kwargs) > 1:\n            raise TypeError('function takes at most 1 keyword argument')\n        if len(kwargs) == 1:\n            if 'order' not in kwargs:\n                raise TypeError(\"'{}' is an invalid keyword argument for this function\"\n                                .format(list(kwargs.keys())[0]))\n            order = kwargs.pop('order', 'C')\n            if order != 'C':\n                raise NotImplementedError('only supports C-order,'\n                                          ' while received {}'.format(order))\n        if len(args) == 0:\n            raise TypeError('reshape() takes exactly 1 argument (0 given)')\n        if len(args) == 1 and isinstance(args[0], tuple):\n            return _mx_nd_np.reshape(self, newshape=args[0], order=order)\n        else:\n            return _mx_nd_np.reshape(self, newshape=args, order=order)\n\n    def reshape_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`reshape_like`.\n\n        The arguments are the same as for :py:func:`reshape_like`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute reshape_like')\n\n    def reshape_view(self, *shape, **kwargs):  # pylint: disable=redefined-outer-name\n        \"\"\"Returns a **view** of this array with a new shape without altering any data.\n        Inheritated from NDArray.reshape.\n        \"\"\"\n        return super(ndarray, self).reshape(*shape, **kwargs)\n\n    def zeros_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`zeros_like`.\n\n        The arguments are the same as for :py:func:`zeros_like`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute zeros_like')\n\n    def ones_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`ones_like`.\n\n        The arguments are the same as for :py:func:`ones_like`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute ones_like')\n\n    def broadcast_axes(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`broadcast_axes`.\n\n        The arguments are the same as for :py:func:`broadcast_axes`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute broadcast_like')\n\n    def repeat(self, repeats, axis=None):  # pylint: disable=arguments-differ\n        \"\"\"Repeat elements of an array.\"\"\"\n        return repeat(self, repeats=repeats, axis=axis)\n\n    def pad(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`pad`.\n\n        The arguments are the same as for :py:func:`pad`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute pad')\n\n    def swapaxes(self, axis1, axis2):  # pylint: disable=arguments-differ\n        \"\"\"Return a copy of the array with axis1 and axis2 interchanged.\n        Refer to `mxnet.numpy.swapaxes` for full documentation.\n        \"\"\"\n        return swapaxes(self, axis1, axis2)\n\n    def split(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`split`.\n\n        The arguments are the same as for :py:func:`split`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute split')\n\n    def split_v2(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`split_v2`.\n\n        The arguments are the same as for :py:func:`split_v2`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute split_v2')\n\n    def slice(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`slice`.\n\n        The arguments are the same as for :py:func:`slice`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute slice')\n\n    def slice_axis(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`slice_axis`.\n\n        The arguments are the same as for :py:func:`slice_axis`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute slice_axis')\n\n    def slice_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`slice_like`.\n\n        The arguments are the same as for :py:func:`slice_like`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute slice_like')\n\n    def slice_assign_scalar(self, value, begin, end, step):\n        \"\"\"\n        Assign the scalar to a cropped subset of this ndarray. Value will broadcast to the shape of the cropped shape\n        and will be cast to the same dtype of the ndarray.\n\n        Parameters\n        ----------\n        value: numeric value\n            Value and this ndarray should be of the same data type.\n            The shape of rhs should be the same as the cropped shape of this ndarray.\n        begin: tuple of begin indices\n        end: tuple of end indices\n        step: tuple of step lenghths\n\n        Returns\n        -------\n        This ndarray.\n\n        Examples\n        --------\n        >>> x = np.ones((2, 2, 2))\n        >>> y = x.slice_assign_scalar(0, (0, 0, None), (1, 1, None), (None, None, None))\n        >>> y\n        array([[[0., 0.],\n                [1., 1.]],\n\n               [[1., 1.],\n                [1., 1.]]])\n        >>> x\n        array([[[0., 0.],\n                [1., 1.]],\n\n               [[1., 1.],\n                [1., 1.]]])\n        \"\"\"\n        return _npi.slice_assign_scalar(self, value, begin=begin, end=end, step=step, out=self)\n\n    def slice_assign(self, rhs, begin, end, step):\n        \"\"\"\n        Assign the rhs to a cropped subset of this ndarray in place.\n        Returns the view of this ndarray.\n\n        Parameters\n        ----------\n        rhs: ndarray.\n            rhs and this NDArray should be of the same data type, and on the same device.\n            The shape of rhs should be the same as the cropped shape of this ndarray.\n        begin: tuple of begin indices\n        end: tuple of end indices\n        step: tuple of step lenghths\n\n        Returns\n        -------\n        out : ndarray\n            This ndarray.\n\n        Examples\n        --------\n        >>> x = np.ones((2, 2, 2))\n        >>> assigned = np.zeros((1, 1, 2))\n        >>> y = x.slice_assign(assigned, (0, 0, None), (1, 1, None), (None, None, None))\n        >>> y\n        array([[[0., 0.],\n                [1., 1.]],\n\n               [[1., 1.],\n                [1., 1.]]])\n        >>> x\n        array([[[0., 0.],\n                [1., 1.]],\n\n               [[1., 1.],\n                [1., 1.]]])\n        \"\"\"\n        return _npi.slice_assign(self, rhs, begin=begin, end=end, step=step, out=self)\n\n    def take(self, indices, axis=None, mode='raise'):  # pylint: disable=arguments-differ, redefined-outer-name\n        \"\"\"Convenience fluent method for :py:func:`take`.\n\n        The arguments are the same as for :py:func:`take`, with\n        this array as data.\n        \"\"\"\n        return take(self, indices, axis, mode=mode)\n\n    def one_hot(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`one_hot`.\n\n        The arguments are the same as for :py:func:`one_hot`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute one_hot')\n\n    def pick(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`pick`.\n\n        The arguments are the same as for :py:func:`pick`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute pick')\n\n    def sort(self, axis=-1, descending=False, stable=True):  # pylint: disable=arguments-differ\n        \"\"\"Convenience fluent method for :py:func:`sort`.\n\n        The arguments are the same as for :py:func:`sort`, with\n        this array as data.\n        \"\"\"\n        return sort(self, axis=axis, descending=descending, stable=stable)\n\n    def topk(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`topk`.\n\n        The arguments are the same as for :py:func:`topk`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute topk')\n\n    def argsort(self, axis=-1, descending=False, stable=True):  # pylint: disable=arguments-differ\n        \"\"\"Convenience fluent method for :py:func:`argsort`.\n\n        The arguments are the same as for :py:func:`argsort`, with\n        this array as data.\n        \"\"\"\n        return argsort(self, axis=axis, descending=descending, stable=stable)\n\n    def argmax_channel(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`argmax_channel`.\n\n        The arguments are the same as for :py:func:`argmax_channel`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute argmax_channel')\n\n    def argmin(self, axis=None, out=None, keepdims=False):  # pylint: disable=arguments-differ\n        \"\"\"Return indices of the minium values along the given axis.\n        Refer to `mxnet.numpy.argmin` for full documentation.\"\"\"\n        return argmin(self, axis, out, keepdims)\n\n    def clip(self, min=None, max=None, out=None):  # pylint: disable=arguments-differ\n        \"\"\"Return an array whose values are limited to [min, max].\n        One of max or min must be given.\n        \"\"\"\n        return clip(self, min, max, out=out)\n\n    def abs(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`abs`.\n\n        The arguments are the same as for :py:func:`abs`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute abs')\n\n    def sign(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sign`.\n\n        The arguments are the same as for :py:func:`sign`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute sign')\n\n    def flatten(self, order='C'):  # pylint: disable=arguments-differ\n        \"\"\"Return a copy of the array collapsed into one dimension.\"\"\"\n        return self.reshape(-1, order=order)\n\n    def shape_array(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`shape_array`.\n\n        The arguments are the same as for :py:func:`shape_array`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute shape_array')\n\n    def size_array(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`size_array`.\n\n        The arguments are the same as for :py:func:`size_array`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute size_array')\n\n    def expand_dims(self, *args, **kwargs):  # pylint: disable=arguments-differ,unused-argument\n        \"\"\"Convenience fluent method for :py:func:`expand_dims`.\n\n        The arguments are the same as for :py:func:`expand_dims`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute expand_dims')\n\n    def tile(self, reps):  # pylint: disable=arguments-differ\n        \"\"\"Construct an array by repeating A the number of times given by reps.\n        Refer to `mxnet.numpy.tile` for full documentation.\"\"\"\n        return tile(self, reps=reps)\n\n    def transpose(self, *axes):  # pylint: disable=arguments-differ\n        \"\"\"Permute the dimensions of an array.\"\"\"\n        if len(axes) == 0:\n            axes = None\n        elif len(axes) == 1:\n            if isinstance(axes[0], (tuple, list)):\n                axes = axes[0]\n            elif axes[0] is None:\n                axes = None\n        return transpose(self, axes=axes)\n\n    def flip(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`flip`.\n\n        The arguments are the same as for :py:func:`flip`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute flip')\n\n    def depth_to_space(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`depth_to_space`.\n\n        The arguments are the same as for :py:func:`depth_to_space`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute depth_to_space')\n\n    def space_to_depth(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`space_to_depth`.\n\n        The arguments are the same as for :py:func:`space_to_depth`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute space_to_depth')\n\n    def diag(self, k=0, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`diag`.\n\n        The arguments are the same as for :py:func:`diag`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute diag')\n\n    def diagonal(self, offset=0, axis1=0, axis2=1):  # pylint: disable=arguments-differ\n        \"\"\"Return the diagonal with the given offset.\n\n        If array has more than two dimensions, then the axes specified by axis1 and\n        axis2 are used to determine the 2-D sub-array whose diagonal is returned.\n\n        Refer to `mxnet.numpy.diagonal` for full documents.\n        \"\"\"\n        return diagonal(self, offset=offset, axis1=axis1, axis2=axis2)\n\n    def sum(self, axis=None, dtype=None, out=None, keepdims=False):  # pylint: disable=arguments-differ\n        \"\"\"Return the sum of the array elements over the given axis.\"\"\"\n        return sum(self, axis=axis, dtype=dtype, out=out, keepdims=keepdims)\n\n    def nansum(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`nansum`.\n\n        The arguments are the same as for :py:func:`nansum`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute nansum')\n\n    def prod(self, axis=None, dtype=None, out=None, keepdims=False):  # pylint: disable=arguments-differ\n        \"\"\"Return the product of the array elements over the given axis.\"\"\"\n        return _mx_np_op.prod(self, axis=axis, dtype=dtype, keepdims=keepdims, out=out)\n\n    def nanprod(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`nanprod`.\n\n        The arguments are the same as for :py:func:`nanprod`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute nanprod')\n\n    def mean(self, axis=None, dtype=None, out=None, keepdims=False):  # pylint: disable=arguments-differ\n        \"\"\"Returns the average of the array elements along given axis.\"\"\"\n        return mean(self, axis=axis, dtype=dtype, out=out, keepdims=keepdims)\n\n    # pylint: disable=too-many-arguments, arguments-differ\n\n    @wrap_data_api_statical_func\n    def std(self, axis=None, dtype=None, out=None, correction=0, keepdims=False):\n        \"\"\"Returns the standard deviation of the array elements along given axis.\"\"\"\n        return std(self, axis=axis, dtype=dtype, correction=correction, keepdims=keepdims, out=out)\n\n    @wrap_data_api_statical_func\n    def var(self, axis=None, dtype=None, out=None, correction=0, keepdims=False):\n        \"\"\"Returns the variance of the array elements, along given axis.\"\"\"\n        return var(self, axis=axis, dtype=dtype, out=out, correction=correction, keepdims=keepdims)\n    # pylint: enable=too-many-arguments, arguments-differ\n\n    def cumsum(self, axis=None, dtype=None, out=None):\n        \"\"\"Return the cumulative sum of the elements along the given axis.\"\"\"\n        return _mx_nd_np.cumsum(self, axis=axis, dtype=dtype, out=out)\n\n    def tolist(self):\n        return self.asnumpy().tolist()\n\n    def max(self, axis=None, out=None, keepdims=False):  # pylint: disable=arguments-differ\n        \"\"\"Return the maximum along a given axis.\"\"\"\n        return _mx_nd_np.max(self, axis=axis, out=out, keepdims=keepdims)\n\n    def min(self, axis=None, out=None, keepdims=False):  # pylint: disable=arguments-differ\n        \"\"\"Convenience fluent method for :py:func:`min`.\n\n        The arguments are the same as for :py:func:`min`, with\n        this array as data.\n        \"\"\"\n        return _mx_nd_np.min(self, axis=axis, out=out, keepdims=keepdims)\n\n    def norm(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`norm`.\n\n        The arguments are the same as for :py:func:`norm`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute norm')\n\n    def round(self, decimals=0, out=None, **kwargs): # pylint: disable=arguments-differ\n        \"\"\"Convenience fluent method for :py:func:`round`.\n\n        The arguments are the same as for :py:func:`round`, with\n        this array as data.\n        \"\"\"\n        return round(self, decimals=decimals, out=out, **kwargs)\n\n    def rint(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`rint`.\n\n        The arguments are the same as for :py:func:`rint`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute rint')\n\n    def fix(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`fix`.\n\n        The arguments are the same as for :py:func:`fix`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute fix')\n\n    def floor(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`floor`.\n\n        The arguments are the same as for :py:func:`floor`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute floor')\n\n    def ceil(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`ceil`.\n\n        The arguments are the same as for :py:func:`ceil`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute ceil')\n\n    def trunc(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`trunc`.\n\n        The arguments are the same as for :py:func:`trunc`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute trunc')\n\n    def sin(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sin`.\n\n        The arguments are the same as for :py:func:`sin`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute sin')\n\n    def cos(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`cos`.\n\n        The arguments are the same as for :py:func:`cos`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute cos')\n\n    def tan(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`tan`.\n\n        The arguments are the same as for :py:func:`tan`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute tan')\n\n    def arcsin(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arcsin`.\n\n        The arguments are the same as for :py:func:`arcsin`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute arcsin')\n\n    def arccos(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arccos`.\n\n        The arguments are the same as for :py:func:`arccos`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute arccos')\n\n    def arctan(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arctan`.\n\n        The arguments are the same as for :py:func:`arctan`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute arctan')\n\n    def degrees(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`degrees`.\n\n        The arguments are the same as for :py:func:`degrees`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute degrees')\n\n    def radians(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`radians`.\n\n        The arguments are the same as for :py:func:`radians`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute radians')\n\n    def sinh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sinh`.\n\n        The arguments are the same as for :py:func:`sinh`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute sinh')\n\n    def cosh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`cosh`.\n\n        The arguments are the same as for :py:func:`cosh`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute cosh')\n\n    def tanh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`tanh`.\n\n        The arguments are the same as for :py:func:`tanh`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute tanh')\n\n    def arcsinh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arcsinh`.\n\n        The arguments are the same as for :py:func:`arcsinh`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute arcsinh')\n\n    def arccosh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arccosh`.\n\n        The arguments are the same as for :py:func:`arccosh`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute arccosh')\n\n    def arctanh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arctanh`.\n\n        The arguments are the same as for :py:func:`arctanh`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute arctanh')\n\n    def exp(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`exp`.\n\n        The arguments are the same as for :py:func:`exp`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute exp')\n\n    def expm1(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`expm1`.\n\n        The arguments are the same as for :py:func:`expm1`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute expm1')\n\n    def log(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log`.\n\n        The arguments are the same as for :py:func:`log`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute log')\n\n    def log10(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log10`.\n\n        The arguments are the same as for :py:func:`log10`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute log10')\n\n    def log2(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log2`.\n\n        The arguments are the same as for :py:func:`log2`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute log2')\n\n    def log1p(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log1p`.\n\n        The arguments are the same as for :py:func:`log1p`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute log1p')\n\n    def log_sigmoid(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log_sigmoid`.\n\n        The arguments are the same as for :py:func:`log_sigmoid`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute log_sigmoid')\n\n    def sqrt(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sqrt`.\n\n        The arguments are the same as for :py:func:`sqrt`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute sqrt')\n\n    def rsqrt(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`rsqrt`.\n\n        The arguments are the same as for :py:func:`rsqrt`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute rsqrt')\n\n    def cbrt(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`cbrt`.\n\n        The arguments are the same as for :py:func:`cbrt`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute cqrt')\n\n    def rcbrt(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`rcbrt`.\n\n        The arguments are the same as for :py:func:`rcbrt`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute rcqrt')\n\n    def square(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`square`.\n\n        The arguments are the same as for :py:func:`square`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute square')\n\n    def reciprocal(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`reciprocal`.\n\n        The arguments are the same as for :py:func:`reciprocal`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute reciprocal')\n\n    def relu(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`relu`.\n\n        The arguments are the same as for :py:func:`relu`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute relu')\n\n    def sigmoid(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sigmoid`.\n\n        The arguments are the same as for :py:func:`sigmoid`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute sigmoid')\n\n    def softmax(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`softmax`.\n\n        The arguments are the same as for :py:func:`softmax`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute softmax')\n\n    def log_softmax(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log_softmax`.\n\n        The arguments are the same as for :py:func:`log_softmax`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute log_softmax')\n\n    def softmin(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`softmin`.\n\n        The arguments are the same as for :py:func:`softmin`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute softmin')\n\n    def mish(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`mish`.\n\n        The arguments are the same as for :py:func:`mish`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute mish')\n\n    def squeeze(self, axis=None):  # pylint: disable=arguments-differ\n        \"\"\"Remove single-dimensional entries from the shape of a.\"\"\"\n        return squeeze(self, axis=axis)\n\n    def broadcast_to(self, shape):  # pylint: disable=redefined-outer-name\n        return _mx_nd_np.broadcast_to(self, shape)\n\n    def broadcast_like(self, other):\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute broadcast_like')\n\n    def _full(self, value):\n        \"\"\"\n        Currently for internal use only. Implemented for __setitem__.\n        Assign to self an array of self's same shape and type, filled with value.\n        \"\"\"\n        return _mx_nd_np.full(self.shape, value, device=self.device, dtype=self.dtype, out=self)\n\n    # pylint: disable=redefined-outer-name\n    def _scatter_set_nd(self, value_nd, indices):\n        \"\"\"\n        This is added as an ndarray class method in order to support polymorphism in NDArray and numpy.ndarray indexing\n        \"\"\"\n        return _npi.scatter_set_nd(\n            lhs=self, rhs=value_nd, indices=indices, shape=self.shape, out=self\n        )\n    # pylint: enable=redefined-outer-name\n\n    @property\n    def shape(self):\n        \"\"\"Tuple of array dimensions.\n\n        Examples\n        --------\n        >>> x = mx.np.array([1, 2, 3, 4])\n        >>> x.shape\n        (4L,)\n        >>> y = mx.np.zeros((2, 3, 4))\n        >>> y.shape\n        (2L, 3L, 4L)\n        >>> z = mx.np.array(3)\n        >>> z.shape\n        ()\n        \"\"\"\n        num_dim = mx_int()\n        if _int64_enabled():\n            pdata = ctypes.POINTER(mx_int64)()\n            check_call(_LIB.MXNDArrayGetShape64(\n                self.handle, ctypes.byref(num_dim), ctypes.byref(pdata)))\n        else:\n            pdata = ctypes.POINTER(mx_int)()\n            check_call(_LIB.MXNDArrayGetShape(\n                self.handle, ctypes.byref(num_dim), ctypes.byref(pdata)))\n        if num_dim.value == -1:\n            return None\n        else:\n            return tuple(pdata[:num_dim.value])  # pylint: disable=invalid-slice-index\n\n    @property\n    def ndim(self):\n        \"\"\"Number of array dimensions.\"\"\"\n        return len(self.shape)\n\n    @property\n    def size(self):\n        \"\"\"Number of elements in the array.\"\"\"\n        return super(ndarray, self).size\n\n    @property\n    def dtype(self):\n        \"\"\"Data-type of the array's elements.\n\n        Returns\n        -------\n        numpy.dtype\n            This NDArray's data type.\n\n        Examples\n        --------\n        >>> x = np.zeros((2,3))\n        >>> x.dtype\n        dtype('float32')\n        >>> y = np.zeros((2,3), dtype='int32')\n        >>> y.dtype\n        dtype('int32')\n        \"\"\"\n        return _np.dtype(super(ndarray, self).dtype)\n\n    def tostype(self, stype):\n        raise AttributeError('mxnet.numpy.ndarray object has no attribute tostype')\n\n\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef empty(shape, dtype=None, order='C', device=None):  # pylint: disable=redefined-outer-name\n    \"\"\"Return a new array of given shape and type, without initializing entries.\n\n    Parameters\n    ----------\n    shape : int or tuple of int Shape of the empty array, e.g., ``(2, 3)`` or ``2``.\n    dtype : data-type, optional\n        Desired output data-type for the array, e.g, `numpy.int8`.\n        Note that this behavior is different from NumPy's `empty` function where `float64`\n        is the default value, here you can set your default dtype as 'float32' or 'float64'\n        because `float32` is considered as the default data type in deep learning.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n    order : {'C'}, optional, default: 'C'\n        How to store multi-dimensional data in memory, currently only row-major\n        (C-style) is supported.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    out : ndarray\n        Array of uninitialized (arbitrary) data of the given shape, dtype, and order.\n\n    Examples\n    --------\n    >>> np.empty([2, 2])\n    array([[ 0.000000e+00, -2.524355e-29],\n           [          nan, -8.592023e+09]])  # uninitialized\n\n    >>> np.empty([2, 2], dtype=int)\n    array([[8751743591039004782, 3196766424264760104],\n           [7583328881310196768,     562950123910254]], dtype=int64)  # uninitialized\n    \"\"\"\n    if order != 'C':\n        raise NotImplementedError('`empty` only supports order equal to `C`, while received {}'\n                                  .format(str(order)))\n    if device is None:\n        device = current_device()\n    if dtype is None or dtype is float:\n        dtype = _np.float64 if is_np_default_dtype() else _np.float32\n    if isinstance(shape, int):\n        shape = (shape,)\n    return ndarray(handle=_new_alloc_handle(shape, device, False, dtype))\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef array(object, dtype=None, device=None):\n    \"\"\"\n    Create an array.\n\n    Parameters\n    ----------\n    object : array_like or `numpy.ndarray` or `mxnet.numpy.ndarray`\n        An array, any object exposing the array interface, an object whose\n        __array__ method returns an array, or any (nested) sequence.\n    dtype : data-type, optional\n        The desired data-type for the array.\n        The default dtype is ``object.dtype`` if `object` is an `ndarray`, `float32` otherwise.\n        Default dtype can be set to be consistent with offical numpy by `npx.set_np(dtype=True)`.\n\n        * When npx.is_np_default_dtype() returns False, default dtype is float32;\n        * When npx.is_np_default_dtype() returns True, default dtype is float64.\n\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    out : ndarray\n        An array object satisfying the specified requirements.\n\n    Examples\n    --------\n    >>> np.array([1, 2, 3])\n    array([1., 2., 3.])\n\n    >>> np.array([[1, 2], [3, 4]])\n    array([[1., 2.],\n           [3., 4.]])\n\n    >>> np.array([[1, 0], [0, 1]], dtype=bool)\n    array([[ True, False],\n           [False,  True]])\n\n    >>> np.array([1, 2, 3]).dtype\n    dtype('float32')\n\n    >>> npx.set_np(dtype=True)\n    >>> np.array([1, 2, 3]).dtype\n    dtype('float64')\n    \"\"\"\n    if device is None:\n        device = current_device()\n    if isinstance(object, _np.ndarray):\n        if is_np_default_dtype():\n            dtype = object.dtype if dtype is None else dtype\n        else:\n            dtype = _np.float32 if dtype is None or object.dtype is _np.float64 else dtype\n    if isinstance(object, ndarray):\n        dtype = object.dtype if dtype is None else dtype\n    elif isinstance(object, NDArray):\n        raise ValueError(\"If you're trying to create a mxnet.numpy.ndarray \"\n                         \"from mx.nd.NDArray, please use the zero-copy as_np_ndarray function.\")\n    else:\n        if dtype is None:\n            default_dtype = _np.float64 if is_np_default_dtype() else _np.float32\n            dtype = object.dtype if hasattr(object, \"dtype\") else default_dtype\n        try:\n            object = _np.array(object, dtype=dtype)\n        except Exception as e:\n            # printing out the error raised by official NumPy's array function\n            # for transparency on users' side\n            raise TypeError('{}'.format(str(e)))\n    ret = empty(object.shape, dtype=dtype, device=device)\n    if len(object.shape) == 0:\n        ret[()] = object\n    else:\n        ret[:] = object\n    return ret\n# pylint: enable=redefined-outer-name\n\n\n@set_module('mxnet.numpy')\ndef shape(a):\n    \"\"\"\n    Return the shape of an array.\n\n    Parameters\n    ----------\n    a : array_like\n        Input array.\n\n    Returns\n    -------\n    shape : tuple of ints\n        The elements of the shape tuple give the lengths of the\n        corresponding array dimensions.\n\n    See Also\n    --------\n    ndarray.shape : Equivalent array method.\n\n    Examples\n    --------\n    >>> np.shape(np.eye(3))\n    (3, 3)\n    >>> np.shape([[1, 2]])\n    (1, 2)\n    >>> np.shape([0])\n    (1,)\n    >>> np.shape(0)\n    ()\n    \"\"\"\n    return _mx_nd_np.shape(a)\n\n\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef zeros(shape, dtype=None, order='C', device=None):  # pylint: disable=redefined-outer-name\n    \"\"\"Return a new array of given shape and type, filled with zeros.\n    This function currently only supports storing multi-dimensional data\n    in row-major (C-style).\n\n    Parameters\n    ----------\n    shape : int or tuple of int\n        The shape of the empty array.\n    dtype : str or numpy.dtype, optional\n        An optional value type,\n        When npx.is_np_default_dtype() returns False, default dtype is float32,\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Note that this behavior is different from NumPy's `zeros` function where `float64`\n        is the default value, here we can set 'float32' or 'float64' as your default dtype,\n        because `float32` is considered as the default data type in deep learning.\n    order : {'C'}, optional, default: 'C'\n        How to store multi-dimensional data in memory, currently only row-major\n        (C-style) is supported.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    out : ndarray\n        Array of zeros with the given shape, dtype, and device.\n\n    Examples\n    --------\n    >>> np.zeros(5)\n    array([0., 0., 0., 0., 0.])\n\n    >>> np.zeros((5,), dtype=int)\n    array([0, 0, 0, 0, 0], dtype=int64)\n\n    >>> np.zeros((2, 1))\n    array([[0.],\n           [0.]])\n    \"\"\"\n    return _mx_nd_np.zeros(shape, dtype, order, device)\n\n\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef ones(shape, dtype=None, order='C', device=None):  # pylint: disable=redefined-outer-name\n    \"\"\"Return a new array of given shape and type, filled with ones.\n    This function currently only supports storing multi-dimensional data\n    in row-major (C-style).\n\n    Parameters\n    ----------\n    shape : int or tuple of int\n        The shape of the empty array.\n    dtype : str or numpy.dtype, optional\n        An optional value type. Default is depend on your current default dtype.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Note that this behavior is different from NumPy's `ones` function where\n        `float64` is the default value.\n    order : {'C'}, optional, default: 'C'\n        How to store multi-dimensional data in memory, currently only row-major\n        (C-style) is supported.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    out : ndarray\n        Array of ones with the given shape, dtype, and device.\n\n    Examples\n    --------\n    >>> np.ones(5)\n    array([1., 1., 1., 1., 1.])\n\n    >>> np.ones((5,), dtype=int)\n    array([1, 1, 1, 1, 1], dtype=int64)\n\n    >>> np.ones((2, 1))\n    array([[1.],\n           [1.]])\n\n    >>> s = (2,2)\n    >>> np.ones(s)\n    array([[1., 1.],\n           [1., 1.]])\n    \"\"\"\n    return _mx_nd_np.ones(shape, dtype, order, device)\n\n\n@set_module('mxnet.numpy')\ndef broadcast_to(array, shape):  # pylint: disable=redefined-outer-name\n    \"\"\"\n    Broadcast an array to a new shape.\n\n    Parameters\n    ----------\n    array : ndarray or scalar\n        The array to broadcast.\n    shape : tuple\n        The shape of the desired array.\n\n    Returns\n    -------\n    broadcast : array\n        A readonly view on the original array with the given shape. It is\n        typically not contiguous. Furthermore, more than one element of a\n        broadcasted array may refer to a single memory location.\n\n    Raises\n    ------\n    MXNetError\n        If the array is not compatible with the new shape according to NumPy's\n        broadcasting rules.\n    \"\"\"\n    return _mx_nd_np.broadcast_to(array, shape)\n\n\n# pylint: disable=too-many-arguments, redefined-outer-name\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef full(shape, fill_value, dtype=None, order='C', device=None, out=None):\n    r\"\"\"Return a new array of given shape and type, filled with `fill_value`.\n\n    Parameters\n    ----------\n    shape : int or sequence of ints\n        Shape of the new array, e.g., ``(2, 3)`` or ``2``.\n    fill_value : scalar or ndarray\n        Fill value.\n    dtype : data-type, optional\n        If dtype is None, the output array data type must be inferred from fill_value.\n        If it’s an int, the output array dtype must be the default integer dtype;\n        If it’s a float, then the output array dtype must be the default floating-point data type;\n        If it’s a bool then the output array must have boolean dtype. Default: None.\n    order : {'C'}, optional\n        Whether to store multidimensional data in C- or Fortran-contiguous\n        (row- or column-wise) order in memory. Currently only supports C order.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n        Array of `fill_value` with the given shape, dtype, and order.\n        If `fill_value` is an ndarray, out will have the same device as `fill_value`\n        regardless of the provided `device`.\n\n    .. note::\n       This function differs from the original numpy.full in the following way(s):\n\n       * Has an additional `device` argument to specify the device\n       * Has an additional `out` argument\n       * Currently does not support `order` selection\n\n    See Also\n    --------\n    empty : Return a new uninitialized array.\n    ones : Return a new array setting values to one.\n    zeros : Return a new array setting values to zero.\n\n    Examples\n    --------\n    >>> np.full((2, 2), 10)\n    array([[10., 10.],\n           [10., 10.]])\n    >>> np.full((2, 2), 2, dtype=np.int32, device=mx.cpu(0))\n    array([[2, 2],\n           [2, 2]], dtype=int32)\n    \"\"\"\n    return _mx_nd_np.full(shape, fill_value, order=order, device=device, dtype=dtype, out=out)\n# pylint: enable=too-many-arguments, redefined-outer-name\n\n\n# pylint: disable=redefined-outer-name, too-many-arguments\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef empty_like(prototype, dtype=None, device=None, order='C', subok=False, shape=None): # pylint: disable=W0621\n    \"\"\"\n    Return a new array with the same shape and type as a given array.\n\n    Parameters\n    ----------\n    prototype : ndarray\n        The shape and data-type of `prototype` define these same attributes\n        of the returned array.\n    dtype : data-type, optional\n        Overrides the data type of the result.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n    order : {'C'}, optional\n        Whether to store multidimensional data in C- or Fortran-contiguous\n        (row- or column-wise) order in memory. Currently only supports C order.\n    subok : {False}, optional\n        If True, then the newly created array will use the sub-class\n        type of 'a', otherwise it will be a base-class array. Defaults\n        to False.\n        (Only support False at this moment)\n    shape : int or sequence of ints, optional.\n        Overrides the shape of the result. If order='K' and the number of\n        dimensions is unchanged, will try to keep order, otherwise,\n        order='C' is implied.\n        (Not supported at this moment)\n\n    Returns\n    -------\n    out : ndarray\n        Array of uninitialized (arbitrary) data with the same\n        shape and type as `prototype`.\n\n    See Also\n    --------\n    ones_like : Return an array of ones with shape and type of input.\n    zeros_like : Return an array of zeros with shape and type of input.\n    full_like : Return a new array with shape of input filled with value.\n    empty : Return a new uninitialized array.\n\n    Notes\n    -----\n    This function does *not* initialize the returned array; to do that use\n    `zeros_like` or `ones_like` instead.  It may be marginally faster than\n    the functions that do set the array values.\n\n    Examples\n    --------\n    >>> a = np.array([[1,2,3], [4,5,6]])\n    >>> np.empty_like(a)\n    array([[-5764607523034234880, -2305834244544065442,           4563075075], # uninitialized\n           [          4567052944, -5764607523034234880,      844424930131968]])\n    >>> a = np.array([[1., 2., 3.],[4.,5.,6.]])\n    >>> np.empty_like(a)\n    array([[4.9e-324, 9.9e-324, 1.5e-323], # uninitialized\n           [2.0e-323, 2.5e-323, 3.0e-323]])\n    \"\"\"\n    ret = _mx_nd_np.empty_like(prototype, dtype=dtype, order=order, subok=subok, shape=shape)\n    if device is not None:\n        ret.to_device(device)\n    return ret\n# pylint: enable=redefined-outer-name\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\ndef all(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Test whether all array elements along a given axis evaluate to True.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array or object that can be converted to an array.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which a logical AND reduction is performed.\n        The default (axis = None) is to perform a logical AND over\n        all the dimensions of the input array.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in\n        the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the input array.\n    out : ndarray, optional\n        Alternate output array in which to place the result. It must have\n        the same shape as the expected output and its type is preserved\n\n    Returns\n    --------\n    all : ndarray, bool\n        A new boolean or array is returned unless out is specified,\n        in which case a reference to out is returned.\n\n    Examples:\n    ---------\n    >>> np.all([[True,False],[True,True]])\n    False\n\n    >>> np.all([[True,False],[True,True]], axis=0)\n    array([ True, False])\n\n    >>> np.all([-1, 4, 5])\n    True\n\n    >>> np.all([1.0, np.nan])\n    True\n\n    >>> o=np.array(False)\n    >>> z=np.all([-1, 4, 5], out=o)\n    >>> id(z), id(o), z\n    (28293632, 28293632, array(True)) # may vary\n    \"\"\"\n    return _mx_nd_np.all(a, axis=axis, out=out, keepdims=keepdims)\n\n\n@set_module('mxnet.numpy')\ndef any(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Test whether any array element along a given axis evaluates to True.\n    Returns single boolean unless axis is not None\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array or object that can be converted to an array.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which a logical AND reduction is performed.\n        The default (axis = None) is to perform a logical AND over\n        all the dimensions of the input array.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in\n        the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the input array.\n    out : ndarray, optional\n        Alternate output array in which to place the result. It must have\n        the same shape as the expected output and its type is preserved\n\n    Returns\n    --------\n    any : bool or ndarray\n        A new boolean or ndarray is returned unless out is specified,\n        in which case a reference to out is returned.\n\n    Examples:\n    ---------\n    >>> np.any([[True, False], [True, True]])\n    True\n\n    >>> np.any([[True, False], [False, False]], axis=0)\n    array([ True, False])\n\n    >>> np.any([-1, 0, 5])\n    True\n\n    >>> np.any(np.nan)\n    True\n\n    >>> o=np.array(False)\n    >>> z=np.any([-1, 4, 5], out=o)\n    >>> z, o\n    (array(True), array(True))\n    >>> # Check now that z is a reference to o\n    >>> z is o\n    True\n    >>> id(z), id(o) # identity of z and o              # doctest: +SKIP\n    (191614240, 191614240)\n    \"\"\"\n    return _mx_nd_np.any(a, axis=axis, out=out, keepdims=keepdims)\n\n\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef identity(n, dtype=None, device=None):\n    \"\"\"\n    Return the identity array.\n\n    The identity array is a square array with ones on\n    the main diagonal.\n\n    Parameters\n    ----------\n    n : int\n        Number of rows (and columns) in `n` x `n` output.\n    dtype : data-type, optional\n        Data-type of the output.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    out : ndarray\n        `n` x `n` array with its main diagonal set to one,\n        and all other elements 0.\n\n    Examples\n    --------\n    >>> np.identity(3)\n    >>> np.identity(3)\n    array([[1., 0., 0.],\n           [0., 1., 0.],\n           [0., 0., 1.]])\n    \"\"\"\n    return _mx_nd_np.identity(n, dtype, device)\n# pylint: enable=redefined-outer-name\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\ndef take(a, indices, axis=None, mode='raise', out=None):\n    r\"\"\"\n    Take elements from an array along an axis.\n\n    When axis is not None, this function does the same thing as \"fancy\"\n    indexing (indexing arrays using arrays); however, it can be easier to use\n    if you need elements along a given axis. A call such as\n    ``np.take(arr, indices, axis=3)`` is equivalent to\n    ``arr[:,:,:,indices,...]``.\n\n    Explained without fancy indexing, this is equivalent to the following use\n    of `ndindex`, which sets each of ``ii``, ``jj``, and ``kk`` to a tuple of\n    indices::\n\n        Ni, Nk = a.shape[:axis], a.shape[axis+1:]\n        Nj = indices.shape\n        for ii in ndindex(Ni):\n            for jj in ndindex(Nj):\n                for kk in ndindex(Nk):\n                    out[ii + jj + kk] = a[ii + (indices[jj],) + kk]\n\n    Parameters\n    ----------\n    a : ndarray\n        The source array.\n    indices : ndarray\n        The indices of the values to extract. Also allow scalars for indices.\n    axis : int, optional\n        The axis over which to select values. By default, the flattened\n        input array is used.\n    out : ndarray, optional\n        If provided, the result will be placed in this array. It should\n        be of the appropriate shape and dtype.\n    mode : {'clip', 'wrap'}, optional\n        Specifies how out-of-bounds indices will behave.\n\n        * 'clip' -- clip to the range (default)\n        * 'wrap' -- wrap around\n\n        'clip' mode means that all indices that are too large are replaced\n        by the index that addresses the last element along that axis. Note\n        that this disables indexing with negative numbers.\n\n    Returns\n    -------\n    out : ndarray\n        The returned array has the same type as `a`.\n\n    .. note::\n\n       This function differs from the original `numpy.take\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.take.html>`_ in\n       the following way(s):\n\n       * Only ndarray or scalar ndarray is accepted as valid input.\n\n    Examples\n    --------\n    >>> a = np.array([4, 3, 5, 7, 6, 8])\n    >>> indices = np.array([0, 1, 4])\n    >>> np.take(a, indices)\n    array([4., 3., 6.])\n\n    In this example for `a` is an ndarray, \"fancy\" indexing can be used.\n\n    >>> a[indices]\n    array([4., 3., 6.])\n\n    If `indices` is not one dimensional, the output also has these dimensions.\n\n    >>> np.take(a, np.array([[0, 1], [2, 3]]))\n    array([[4., 3.],\n           [5., 7.]])\n    \"\"\"\n    return _mx_nd_np.take(a, indices, axis, mode, out)\n# pylint: enable=redefined-outer-name\n\n\n@set_module('mxnet.numpy')\ndef unique(ar, return_index=False, return_inverse=False, return_counts=False, axis=None):\n    \"\"\"\n    Find the unique elements of an array.\n\n    Returns the sorted unique elements of an array. There are three optional\n    outputs in addition to the unique elements:\n\n    * the indices of the input array that give the unique values\n    * the indices of the unique array that reconstruct the input array\n    * the number of times each unique value comes up in the input array\n\n    Parameters\n    ----------\n    ar : ndarray\n        Input array. Unless `axis` is specified, this will be flattened if it\n        is not already 1-D.\n    return_index : bool, optional\n        If True, also return the indices of `ar` (along the specified axis,\n        if provided, or in the flattened array) that result in the unique array.\n    return_inverse : bool, optional\n        If True, also return the indices of the unique array (for the specified\n        axis, if provided) that can be used to reconstruct `ar`.\n    return_counts : bool, optional\n        If True, also return the number of times each unique item appears\n        in `ar`.\n    axis : int or None, optional\n        The axis to operate on. If None, `ar` will be flattened. If an integer,\n        the subarrays indexed by the given axis will be flattened and treated\n        as the elements of a 1-D array with the dimension of the given axis,\n        see the notes for more details. The default is None.\n\n    Returns\n    -------\n    unique : ndarray\n        The sorted unique values.\n    unique_indices : ndarray, optional\n        The indices of the first occurrences of the unique values in the\n        original array. Only provided if `return_index` is True.\n    unique_inverse : ndarray, optional\n        The indices to reconstruct the original array from the\n        unique array. Only provided if `return_inverse` is True.\n    unique_counts : ndarray, optional\n        The number of times each of the unique values comes up in the\n        original array. Only provided if `return_counts` is True.\n\n    .. note::\n\n       When an axis is specified the subarrays indexed by the axis are sorted.\n       This is done by making the specified axis the first dimension of the array\n       and then flattening the subarrays in C order. The flattened subarrays are\n       then viewed as a structured type with each element given a label, with the\n       effect that we end up with a 1-D array of structured types that can be\n       treated in the same way as any other 1-D array. The result is that the\n       flattened subarrays are sorted in lexicographic order starting with the\n       first element.\n\n       This function differs from the original `numpy.unique\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.unique.html>`_ in\n       the following aspects:\n\n       * Only support ndarray as input.\n       * Object arrays or structured arrays are not supported.\n\n    Examples\n    --------\n    >>> np.unique(np.array([1, 1, 2, 2, 3, 3]))\n    array([1., 2., 3.])\n    >>> a = np.array([[1, 1], [2, 3]])\n    >>> np.unique(a)\n    array([1., 2., 3.])\n\n    Return the unique rows of a 2D array\n\n    >>> a = np.array([[1, 0, 0], [1, 0, 0], [2, 3, 4]])\n    >>> np.unique(a, axis=0)\n    array([[1., 0., 0.],\n           [2., 3., 4.]])\n\n    Return the indices of the original array that give the unique values:\n\n    >>> a = np.array([1, 2, 6, 4, 2, 3, 2])\n    >>> u, indices = np.unique(a, return_index=True)\n    >>> u\n    array([1., 2., 3., 4., 6.])\n    >>> indices\n    array([0, 1, 5, 3, 2], dtype=int64)\n    >>> a[indices]\n    array([1., 2., 3., 4., 6.])\n\n    Reconstruct the input array from the unique values:\n\n    >>> a = np.array([1, 2, 6, 4, 2, 3, 2])\n    >>> u, indices = np.unique(a, return_inverse=True)\n    >>> u\n    array([1., 2., 3., 4., 6.])\n    >>> indices\n    array([0, 1, 4, 3, 1, 2, 1], dtype=int64)\n    >>> u[indices]\n    array([1., 2., 6., 4., 2., 3., 2.])\n    \"\"\"\n    return _mx_nd_np.unique(ar, return_index, return_inverse, return_counts, axis)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef add(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Add arguments element-wise.\n\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalar values\n        The arrays to be added. If x1.shape != x2.shape, they must be broadcastable to\n        a common shape (which may be the shape of one or the other).\n\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    The sum of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars.\n\n    .. note::\n\n       This operator now supports automatic type promotion. The resulting type will be determined\n       according to the following rules:\n       * If both inputs are of floating number types, the output is the more precise type.\n       * If only one of the inputs is floating number type, the result is that type.\n       * If both inputs are of integer types (including boolean), not supported yet.\n\n    Examples\n    --------\n    >>> np.add(1.0, 4.0)\n    5.0\n    >>>\n    >>> x1 = np.arange(9.0).reshape((3, 3))\n    >>> x2 = np.arange(3.0)\n    >>> np.add(x1, x2)\n    array([[ 0.,  2.,  4.],\n           [ 3.,  5.,  7.],\n           [ 6.,  8., 10.]])\n    \"\"\"\n    return _mx_nd_np.add(x1, x2, out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef subtract(x1, x2, out=None, **kwargs):\n    r\"\"\"Subtract arguments element-wise.\n\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalar values\n        The arrays to be subtracted from each other. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which may be the shape\n        of one or the other).\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    subtract : ndarray or scalar\n        The difference of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars.\n\n    .. note::\n       This operator now supports automatic type promotion. The resulting type will be determined\n       according to the following rules:\n       * If both inputs are of floating number types, the output is the more precise type.\n       * If only one of the inputs is floating number type, the result is that type.\n       * If both inputs are of integer types (including boolean), not supported yet.\n\n    Examples\n    --------\n    >>> np.subtract(1.0, 4.0)\n    -3.0\n    >>> x1 = np.arange(9.0).reshape((3, 3))\n    >>> x2 = np.arange(3.0)\n    >>> np.subtract(x1, x2)\n    array([[0., 0., 0.],\n           [3., 3., 3.],\n           [6., 6., 6.]])\n    \"\"\"\n    return _mx_nd_np.subtract(x1, x2, out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef multiply(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Multiply arguments element-wise.\n\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalar values\n        The arrays to be multiplied. If x1.shape != x2.shape, they must be broadcastable to\n        a common shape (which may be the shape of one or the other).\n\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        The difference of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars.\n\n    .. note::\n       This operator now supports automatic type promotion. The resulting type will be determined\n       according to the following rules:\n\n       * If both inputs are of floating number types, the output is the more precise type.\n       * If only one of the inputs is floating number type, the result is that type.\n       * If both inputs are of integer types (including boolean), not supported yet.\n\n    Examples\n    --------\n    >>> np.multiply(2.0, 4.0)\n    8.0\n    >>> x1 = np.arange(9.0).reshape((3, 3))\n    >>> x2 = np.arange(3.0)\n    >>> np.multiply(x1, x2)\n    array([[ 0.,  1.,  4.],\n           [ 0.,  4., 10.],\n           [ 0.,  7., 16.]])\n    \"\"\"\n    return _mx_nd_np.multiply(x1, x2, out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef divide(x1, x2, out=None, **kwargs):\n    \"\"\"Returns a true division of the inputs, element-wise.\n\n    .. note::\n       This operator now supports automatic type promotion. The resulting type will be determined\n       according to the following rules:\n\n       * If both inputs are of floating number types, the output is the more precise type.\n       * If only one of the inputs is floating number type, the result is that type.\n       * If both inputs are of integer types including boolean, the output is of float32 or\n         float64 type, which depends on your current default dtype:\n\n         * When ``npx.is_np_default_dtype()`` returns False, default dtype is float32.\n         * When ``npx.is_np_default_dtype()`` returns True, default dtype is float64.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Dividend array.\n    x2 : ndarray or scalar\n        Divisor array.\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        This is a scalar if both x1 and x2 are scalars.\n\n    Examples\n    --------\n    >>> np.true_divide(x, 4)\n    array([0.  , 0.25, 0.5 , 0.75, 1.  ])\n    \"\"\"\n    return _mx_nd_np.divide(x1, x2, out=out)\n\n\n@set_module('mxnet.numpy')\ndef true_divide(x1, x2, out=None):\n    \"\"\"Returns a true division of the inputs, element-wise.\n\n    Instead of the Python traditional 'floor division', this returns a true\n    division.  True division adjusts the output type to present the best\n    answer, regardless of input types.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Dividend array.\n    x2 : ndarray or scalar\n        Divisor array.\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        This is a scalar if both x1 and x2 are scalars.\n\n    .. note::\n\n       This operator now supports automatic type promotion. The resulting type will be determined\n       according to the following rules:\n\n       * If both inputs are of floating number types, the output is the more precise type.\n       * If only one of the inputs is floating number type, the result is that type.\n       * If both inputs are of integer types (including boolean), the output is of float32 or\n         float64 type, which depends on your current default dtype.\n         When npx.is_np_default_dtype() returns False, default dtype is float32;\n         When npx.is_np_default_dtype() returns True, default dtype is float64.\n\n    Examples\n    --------\n    >>> x = np.arange(5)\n    >>> np.true_divide(x, 4)\n    array([0.  , 0.25, 0.5 , 0.75, 1.  ])\n    \"\"\"\n    return _mx_nd_np.true_divide(x1, x2, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef floor_divide(x1, x2, out=None):\n    \"\"\"Return the largest integer smaller or equal to the division of the inputs.\n\n    It is equivalent to the Python // operator and pairs with the Python % (remainder),\n    function so that a = a % b + b * (a // b) up to roundoff.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Dividend array.\n    x2 : ndarray or scalar\n        Divisor array.\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        This is a scalar if both x1 and x2 are scalars.\n\n    .. note::\n\n       This operator now supports automatic type promotion. The resulting type will be determined\n       according to the following rules:\n\n       * If both inputs are of floating number types, the output is the more precise type.\n       * If only one of the inputs is floating number type, the result is that type.\n       * If both inputs are of integer types (including boolean), the output is the more\n         precise type\n\n    Examples\n    --------\n    >>> np.floor_divide(7,3)\n    2\n    >>> np.floor_divide([1., 2., 3., 4.], 2.5)\n    array([ 0.,  0.,  1.,  1.])\n    \"\"\"\n    return _mx_nd_np.floor_divide(x1, x2, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef mod(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Return element-wise remainder of division.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Dividend array.\n\n    x2 : ndarray or scalar\n        Divisor array.\n\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        This is a scalar if both x1 and x2 are scalars.\n\n    Examples\n    --------\n    >>> np.mod(np.arange(7), 5)\n    array([0., 1., 2., 3., 4., 0., 1.])\n    \"\"\"\n    return _mx_nd_np.mod(x1, x2, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef fmod(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Return element-wise remainder of division.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Dividend array.\n\n    x2 : ndarray or scalar\n        Divisor array.\n\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        This is a scalar if both x1 and x2 are scalars.\n\n    Examples\n    --------\n    >>> np.fmod(np.arange(7), 5)\n    array([0., 1., 2., 3., 4., 0., 1.])\n    \"\"\"\n    return _mx_nd_np.fmod(x1, x2, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef matmul(a, b, out=None, **kwargs):\n    r\"\"\"Matrix product of two arrays.\n\n    Parameters\n    ----------\n    a, b : ndarray\n        Input arrays, scalars not allowed.\n    out : ndarray, optional\n        A location into which the result is stored.\n        If provided, it must have a shape that matches the signature (n,k),(k,m)->(n,m).\n        If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray\n        The matrix product of the inputs.\n        This is a scalar only when both x1, x2 are 1-d vectors.\n\n    Raises\n    ------\n    MXNetError\n        If the last dimension of a is not the same size as the second-to-last dimension of b.\n        If a scalar value is passed in.\n\n    See Also\n    --------\n    tensordot : Sum products over arbitrary axes.\n    dot : alternative matrix product with different broadcasting rules.\n    einsum : Einstein summation convention.\n\n    .. note::\n\n       The behavior depends on the arguments in the following way.\n\n       * If both arguments are ``2-D`` they are multiplied like conventional matrices.\n       * If either argument is ``N-D``, ``N > 2``, it is treated as a stack of matrices\n         residing in the last two indexes and broadcast accordingly.\n       * If the first argument is ``1-D``, it is promoted to a matrix by prepending\n         a 1 to its dimensions. After matrix multiplication the prepended 1 is removed.\n       * If the second argument is ``1-D``, it is promoted to a matrix by appending a 1\n         to its dimensions. After matrix multiplication the appended 1 is removed.\n\n       matmul differs from dot in two important ways:\n\n       * Multiplication by scalars is not allowed, use multiply instead.\n       * Stacks of matrices are broadcast together as if the matrices were elements,\n         respecting the signature ``(n,k),(k,m)->(n,m)``:\n\n       >>> a = np.ones([9, 5, 7, 4])\n       >>> c = np.ones([9, 5, 4, 3])\n       >>> np.dot(a, c).shape\n       (9, 5, 7, 9, 5, 3)\n       >>> np.matmul(a, c).shape\n       (9, 5, 7, 3)\n       >>> # n is 7, k is 4, m is 3\n\n    Examples\n    --------\n    For 2-D arrays it is the matrix product:\n\n    >>> a = np.array([[1, 0],\n    ...               [0, 1]])\n    >>> b = np.array([[4, 1],\n    ...               [2, 2]])\n    >>> np.matmul(a, b)\n    array([[4., 1.],\n           [2., 2.]])\n\n    For 2-D mixed with 1-D, the result is the usual.\n\n    >>> a = np.array([[1, 0],\n    ...               [0, 1]])\n    >>> b = np.array([1, 2])\n    >>> np.matmul(a, b)\n    array([1., 2.])\n    >>> np.matmul(b, a)\n    array([1., 2.])\n\n    Broadcasting is conventional for stacks of arrays\n\n    >>> a = np.arange(2 * 2 * 4).reshape((2, 2, 4))\n    >>> b = np.arange(2 * 2 * 4).reshape((2, 4, 2))\n    >>> np.matmul(a, b).shape\n    (2, 2, 2)\n    >>> np.matmul(a, b)[0, 1, 1]\n    array(98.)\n    >>> sum(a[0, 1, :] * b[0, :, 1])\n    array(98.)\n\n    Scalar multiplication raises an error.\n\n    >>> np.matmul([1, 2], 3)\n    Traceback (most recent call last):\n    ...\n    mxnet.base.MXNetError: ... : Multiplication by scalars is not allowed.\n\n    \"\"\"\n    return _mx_nd_np.matmul(a, b, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef remainder(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Return element-wise remainder of division.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Dividend array.\n\n    x2 : ndarray or scalar\n        Divisor array.\n\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        This is a scalar if both x1 and x2 are scalars.\n\n    Examples\n    --------\n    >>> np.remainder(np.arange(7), 5)\n    array([0., 1., 2., 3., 4., 0., 1.])\n    \"\"\"\n    return _mx_nd_np.remainder(x1, x2, out=out)\n\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef power(x1, x2, out=None, **kwargs):\n    \"\"\"\n    First array elements raised to powers from second array, element-wise.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        The bases.\n\n    x2 : ndarray or scalar\n        The exponent.\n\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        The bases in x1 raised to the exponents in x2.\n        This is a scalar if both x1 and x2 are scalars.\n\n    Examples\n    --------\n    >>> x1 = np.arange(6)\n    >>> np.power(x1, 3)\n    array([  0.,   1.,   8.,  27.,  64., 125.])\n\n    Raise the bases to different exponents.\n\n    >>> x2 = np.array([1.0, 2.0, 3.0, 3.0, 2.0, 1.0])\n    >>> np.power(x1, x2)\n    array([ 0.,  1.,  8., 27., 16.,  5.])\n\n    The effect of broadcasting.\n\n    >>> x2 = np.array([[1, 2, 3, 3, 2, 1], [1, 2, 3, 3, 2, 1]])\n    >>> x2\n    array([[1., 2., 3., 3., 2., 1.],\n           [1., 2., 3., 3., 2., 1.]])\n\n    >>> np.power(x1, x2)\n    array([[ 0.,  1.,  8., 27., 16.,  5.],\n           [ 0.,  1.,  8., 27., 16.,  5.]])\n    \"\"\"\n    return _mx_nd_np.power(x1, x2, out=out)\n\npow = power\npow.__doc_ = \"\"\"\n    First array elements raised to powers from second array, element-wise.\n    \n    Notes \n    ----- \n    `pow` is an alias for `power`. It is a standard API in \n    https://data-apis.org/array-api/latest/API_specification/elementwise_functions.html#pow-x1-x2 \n    instead of an official NumPy operator. \n    \n    >>> np.pow is np.power \n    True \n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        The bases.\n\n    x2 : ndarray or scalar\n        The exponent.\n\n    out : ndarray\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        The bases in x1 raised to the exponents in x2.\n        This is a scalar if both x1 and x2 are scalars.\n\n    Examples\n    --------\n    >>> x1 = np.arange(6)\n    >>> np.pow(x1, 3)\n    array([  0.,   1.,   8.,  27.,  64., 125.])\n\n    Raise the bases to different exponents.\n\n    >>> x2 = np.array([1.0, 2.0, 3.0, 3.0, 2.0, 1.0])\n    >>> np.pow(x1, x2)\n    array([ 0.,  1.,  8., 27., 16.,  5.])\n\n    The effect of broadcasting.\n\n    >>> x2 = np.array([[1, 2, 3, 3, 2, 1], [1, 2, 3, 3, 2, 1]])\n    >>> x2\n    array([[1., 2., 3., 3., 2., 1.],\n           [1., 2., 3., 3., 2., 1.]])\n\n    >>> np.pow(x1, x2)\n    array([[ 0.,  1.,  8., 27., 16.,  5.],\n           [ 0.,  1.,  8., 27., 16.,  5.]])\n    \"\"\"\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef gcd(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns the greatest common divisor of ``|x1|`` and ``|x2|``\n\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalar values\n        The arrays for computing greatest common divisor. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which may be the shape of\n        one or the other).\n\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The greatest common divisor of the absolute value of the inputs\n        This is a scalar if both `x1` and `x2` are scalars.\n\n    See Also\n    --------\n    gcd : The lowest common multiple\n\n    Examples\n    --------\n    >>> np.gcd(12, 20)\n    4\n    >>> np.gcd(np.arange(6, dtype=int), 20)\n    array([20,  1,  2,  1,  4,  5], dtype=int64)\n    \"\"\"\n    return _mx_nd_np.gcd(x1, x2, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef lcm(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns the lowest common multiple of ``|x1|`` and ``|x2|``\n\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalar values\n        The arrays for computing lowest common multiple. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which may be the shape of\n        one or the other).\n\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The lowest common multiple of the absolute value of the inputs\n        This is a scalar if both `x1` and `x2` are scalars.\n\n    See Also\n    --------\n    gcd : The greatest common divisor\n\n    Examples\n    --------\n    >>> np.lcm(12, 20)\n    60\n    >>> np.lcm(np.arange(6, dtype=int), 20)\n    array([ 0, 20, 20, 60, 20, 20], dtype=int64)\n    \"\"\"\n    return _mx_nd_np.lcm(x1, x2, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef sin(x, out=None, **kwargs):\n    r\"\"\"\n    Trigonometric sine, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Angle, in radians (:math:`2 \\pi` rad equals 360 degrees).\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a shape that the inputs broadcast to. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output is the same as that of the input if the input is an ndarray.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The sine of each element of x. This is a scalar if `x` is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n\n    Examples\n    --------\n    >>> np.sin(np.pi/2.)\n    1.0\n    >>> np.sin(np.array((0., 30., 45., 60., 90.)) * np.pi / 180.)\n    array([0.        , 0.5       , 0.70710677, 0.86602545, 1.        ])\n    \"\"\"\n    return _mx_nd_np.sin(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef cos(x, out=None, **kwargs):\n    r\"\"\"\n    Cosine, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Angle, in radians (:math:`2 \\pi` rad equals 360 degrees).\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a shape that the inputs broadcast to. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output is the same as that of the input if the input is an ndarray.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The corresponding cosine values. This is a scalar if x is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n\n    Examples\n    --------\n    >>> np.cos(np.array([0, np.pi/2, np.pi]))\n    array([ 1.000000e+00, -4.371139e-08, -1.000000e+00])\n    >>> # Example of providing the optional output parameter\n    >>> out1 = np.array([0], dtype='f')\n    >>> out2 = np.cos(np.array([0.1]), out1)\n    >>> out2 is out1\n    True\n    \"\"\"\n    return _mx_nd_np.cos(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef sinh(x, out=None, **kwargs):\n    \"\"\"\n    Hyperbolic sine, element-wise.\n    Equivalent to ``1/2 * (np.exp(x) - np.exp(-x))`` or ``-1j * np.sin(1j*x)``.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array or scalar.\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a shape that the inputs broadcast to. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output is the same as that of the input if the input is an ndarray.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The corresponding hyperbolic sine values. This is a scalar if `x` is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n\n    Examples\n    --------\n    >>> np.sinh(0)\n    0.0\n    >>> # Example of providing the optional output parameter\n    >>> out1 = np.array([0], dtype='f')\n    >>> out2 = np.sinh(np.array([0.1]), out1)\n    >>> out2 is out1\n    True\n    \"\"\"\n    return _mx_nd_np.sinh(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef cosh(x, out=None, **kwargs):\n    \"\"\"\n    Hyperbolic cosine, element-wise.\n    Equivalent to ``1/2 * (np.exp(x) + np.exp(-x))`` and ``np.cos(1j*x)``.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array or scalar.\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a shape that the inputs broadcast to. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output is the same as that of the input if the input is an ndarray.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The corresponding hyperbolic cosine values. This is a scalar if `x` is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n\n    Examples\n    --------\n    >>> np.cosh(0)\n    1.0\n    \"\"\"\n    return _mx_nd_np.cosh(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef tanh(x, out=None, **kwargs):\n    \"\"\"\n    Compute hyperbolic tangent element-wise.\n    Equivalent to ``np.sinh(x)/np.cosh(x)``.\n\n    Parameters\n    ----------\n    x : ndarray or scalar.\n        Input array.\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a shape that the inputs fill into. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output and input must be the same.\n\n    Returns\n    ----------\n    y : ndarray or scalar\n       The corresponding hyperbolic tangent values.\n\n    .. note::\n       If `out` is provided, the function writes the result into it,\n       and returns a reference to `out`.  (See Examples)\n\n       * input x does not support complex computation (like imaginary number)\n\n       >>> np.tanh(np.pi*1j)\n       TypeError: type <type 'complex'> not supported\n\n    Examples\n    --------\n    >>> np.tanh(np.array[0, np.pi]))\n    array([0.       , 0.9962721])\n    >>> np.tanh(np.pi)\n    0.99627207622075\n    >>> # Example of providing the optional output parameter illustrating\n    >>> # that what is returned is a reference to said parameter\n    >>> out1 = np.array(1)\n    >>> out2 = np.tanh(np.array(0.1), out1)\n    >>> out2 is out1\n    True\n    \"\"\"\n    return _mx_nd_np.tanh(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef log10(x, out=None, **kwargs):\n    \"\"\"\n    Return the base 10 logarithm of the input array, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array or scalar.\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a shape that the inputs broadcast to. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output is the same as that of the input if the input is an ndarray.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The logarithm to the base 10 of `x`, element-wise. NaNs are\n        returned where x is negative. This is a scalar if `x` is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n\n    Examples\n    --------\n    >>> np.log10(np.array([1e-15, -3.]))\n    array([-15.,  nan])\n    \"\"\"\n    return _mx_nd_np.log10(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef sqrt(x, out=None, **kwargs):\n    \"\"\"\n    Return the non-negative square-root of an array, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        The values whose square-roots are required.\n    out : ndarray, or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        An array of the same shape as `x`, containing the positive\n        square-root of each element in `x`. This is a scalar if `x` is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n\n    Examples\n    --------\n    >>> np.sqrt(np.array([1,4,9]))\n    array([1., 2., 3.])\n    >>> np.sqrt(np.array([4, -1, _np.inf]))\n    array([ 2., nan, inf])\n    \"\"\"\n    return _mx_nd_np.sqrt(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef cbrt(x, out=None, **kwargs):\n    \"\"\"\n    Return the cube-root of an array, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray\n        The values whose cube-roots are required.\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape that the\n        inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n        A tuple (possible only as a keyword argument) must have length equal to the number of outputs.\n\n    Returns\n    ----------\n    y : ndarray\n        An array of the same shape as x, containing the cube cube-root of each element in x.\n        If out was provided, y is a reference to it. This is a scalar if x is a scalar.\n\n    Examples\n    ----------\n    >>> np.cbrt([1,8,27])\n    array([ 1.,  2.,  3.])\n    \"\"\"\n    return _mx_nd_np.cbrt(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef abs(x, out=None, **kwargs):\n    r\"\"\"\n    Calculate the absolute value element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    absolute : ndarray\n        An ndarray containing the absolute value of\n        each element in `x`. This is a scalar if `x` is a scalar.\n\n    Examples\n    --------\n    >>> x = np.array([-1.2, 1.2])\n    >>> np.abs(x)\n    array([1.2, 1.2])\n    \"\"\"\n    return _mx_nd_np.abs(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef fabs(x, out=None, **kwargs):\n    r\"\"\"\n    Calculate the absolute value element-wise.\n\n    This function returns the absolute values (positive magnitude) of the\n    data in `x`. Complex values are not handled, use `absolute` to find the\n    absolute values of complex data.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    absolute : ndarray\n        An ndarray containing the absolute value of\n        each element in `x`. This is a scalar if `x` is a scalar.\n\n    Examples\n    --------\n    >>> np.fabs(-1)\n    1.0\n    >>> np.fabs(np.array([-1.2, 1.2]))s\n    array([ 1.2,  1.2])\n    \"\"\"\n    return _mx_nd_np.fabs(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef absolute(x, out=None, **kwargs):\n    \"\"\"\n    Calculate the absolute value element-wise.\n    np.abs is a shorthand for this function.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array.\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n        A tuple (possible only as a keyword argument) must have length equal to the number of outputs.\n\n    Returns\n    ----------\n    absolute : ndarray\n        An ndarray containing the absolute value of each element in x.\n\n    Examples\n    ----------\n    >>> x = np.array([-1.2, 1.2])\n    >>> np.absolute(x)\n    array([ 1.2,  1.2])\n    \"\"\"\n    return _mx_nd_np.absolute(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef exp(x, out=None, **kwargs):\n    r\"\"\"\n    Calculate the exponential of all elements in the input array.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input values.\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array, element-wise exponential of `x`.\n        This is a scalar if `x` is a scalar.\n\n    Examples\n    --------\n    >>> np.exp(1)\n    2.718281828459045\n    >>> x = np.array([-1, 1, -2, 2])\n    >>> np.exp(x)\n    array([0.36787945, 2.7182817 , 0.13533528, 7.389056  ])\n    \"\"\"\n    return _mx_nd_np.exp(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef expm1(x, out=None, **kwargs):\n    r\"\"\"\n    Calculate `exp(x) - 1` for all elements in the array.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input values.\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array, element-wise exponential minus one: `out = exp(x) - 1`.\n        This is a scalar if `x` is a scalar.\n\n    Examples\n    --------\n    >>> np.expm1(1)\n    1.718281828459045\n    >>> x = np.array([-1, 1, -2, 2])\n    >>> np.exp(x)\n    array([-0.63212056,  1.71828183, -0.86466472,  6.3890561])\n    \"\"\"\n    return _mx_nd_np.expm1(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef arcsin(x, out=None, **kwargs):\n    r\"\"\"\n    Inverse sine, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        `y`-coordinate on the unit circle.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape as the input.\n        If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    angle : ndarray or scalar\n        Output array is same shape and type as x. This is a scalar if x is a scalar.\n        The inverse sine of each element in `x`, in radians and in the\n        closed interval ``[-pi/2, pi/2]``.\n\n    Examples\n    --------\n    >>> np.arcsin(1)     # pi/2\n    1.5707963267948966\n    >>> np.arcsin(-1)    # -pi/2\n    -1.5707963267948966\n    >>> np.arcsin(0)\n    0.0\n\n    .. note::\n       `arcsin` is a multivalued function: for each `x` there are infinitely\n       many numbers `z` such that :math:`sin(z) = x`.  The convention is to\n       return the angle `z` whose real part lies in [-pi/2, pi/2].\n       For real-valued input data types, *arcsin* always returns real output.\n       For each value that cannot be expressed as a real number or infinity,\n       it yields ``nan`` and sets the `invalid` floating point error flag.\n       The inverse sine is also known as `asin` or sin^{-1}.\n       The output `ndarray` has the same `device` as the input `ndarray`.\n       This function differs from the original `numpy.arcsin\n       <https://numpy.org/doc/stable/reference/generated/numpy.arcsin.html>`_ in\n       the following aspects:\n\n       * Only support ndarray or scalar now.\n       * `where` argument is not supported.\n       * Complex input is not supported.\n\n    References\n    ----------\n    Abramowitz, M. and Stegun, I. A., *Handbook of Mathematical Functions*,\n    10th printing, New York: Dover, 1964, pp. 79ff.\n    http://www.math.sfu.ca/~cbm/aands/\n    \"\"\"\n    return _mx_nd_np.arcsin(x, out=out, **kwargs)\n\nasin = arcsin\nasin.__doc__ = \"\"\"\n    Inverse sine, element-wise.\n    \n    >>>np.asin is np.asin\n    True\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        `y`-coordinate on the unit circle.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape as the input.\n        If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    angle : ndarray or scalar\n        Output array is same shape and type as x. This is a scalar if x is a scalar.\n        The inverse sine of each element in `x`, in radians and in the\n        closed interval ``[-pi/2, pi/2]``.\n\n    Examples\n    --------\n    >>> np.asin(1)     # pi/2\n    1.5707963267948966\n    >>> np.asin(-1)    # -pi/2\n    -1.5707963267948966\n    >>> np.asin(0)\n    0.0\n\n    .. note::\n       `asin` is a alias for `arcsin`. It is a standard API in\n       https://data-apis.org/array-api/latest/API_specification/generated/signatures.elementwise_functions.asin.html\n       instead of an official NumPy operator.\n       \n       `asin` is a multivalued function: for each `x` there are infinitely\n       many numbers `z` such that :math:`sin(z) = x`.  The convention is to\n       return the angle `z` whose real part lies in [-pi/2, pi/2].\n       For real-valued input data types, *asin* always returns real output.\n       For each value that cannot be expressed as a real number or infinity,\n       it yields ``nan`` and sets the `invalid` floating point error flag.\n       The inverse sine is also known as `asin` or sin^{-1}.\n       The output `ndarray` has the same `ctx` as the input `ndarray`.\n       This function differs from the original `numpy.arcsin\n       <https://numpy.org/doc/stable/reference/generated/numpy.arcsin.html>`_ in\n       the following aspects:\n\n       * Only support ndarray or scalar now.\n       * `where` argument is not supported.\n       * Complex input is not supported.\n\n    References\n    ----------\n    Abramowitz, M. and Stegun, I. A., *Handbook of Mathematical Functions*,\n    10th printing, New York: Dover, 1964, pp. 79ff.\n    http://www.math.sfu.ca/~cbm/aands/\n    \"\"\"\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef arccos(x, out=None, **kwargs):\n    \"\"\"\n    Trigonometric inverse cosine, element-wise.\n    The inverse of cos so that, if y = cos(x), then x = arccos(y).\n\n    Parameters\n    ----------\n    x : ndarray\n        x-coordinate on the unit circle. For real arguments, the domain is [-1, 1].\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape that\n        the inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n        A tuple (possible only as a keyword argument) must have length equal to the number of outputs.\n\n    Returns\n    ----------\n    angle : ndarray\n        The angle of the ray intersecting the unit circle at the given x-coordinate in radians [0, pi].\n        This is a scalar if x is a scalar.\n\n    Notes\n    ----------\n    arccos is a multivalued function: for each x there are infinitely many numbers z such that\n    cos(z) = x. The convention is to return the angle z whose real part lies in [0, pi].\n    For real-valued input data types, arccos always returns real output.\n    For each value that cannot be expressed as a real number or infinity, it yields nan and sets\n    the invalid floating point error flag.\n    The inverse cos is also known as acos or cos^-1.\n\n    Examples\n    ----------\n    >>> np.arccos([1, -1])\n    array([ 0.        ,  3.14159265])\n    \"\"\"\n    return _mx_nd_np.arccos(x, out=out, **kwargs)\n\nacos = arccos\nacos.__doc__ = \"\"\"\n    Trigonometric inverse cosine, element-wise.\n    The inverse of cos so that, if y = cos(x), then x = acos(y).\n    \n    >>>np.acos is np.arccos\n    True\n\n    Parameters\n    ----------\n    x : ndarray\n        x-coordinate on the unit circle. For real arguments, the domain is [-1, 1].\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape that\n        the inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n        A tuple (possible only as a keyword argument) must have length equal to the number of outputs.\n\n    Returns\n    ----------\n    angle : ndarray\n        The angle of the ray intersecting the unit circle at the given x-coordinate in radians [0, pi].\n        This is a scalar if x is a scalar.\n\n    Notes\n    ----------\n    `acos` is a alias for `arccos`. It is a standard API in\n    https://data-apis.org/array-api/latest/API_specification/generated/signatures.elementwise_functions.acos.html\n    instead of an official NumPy operator.\n    \n    acos is a multivalued function: for each x there are infinitely many numbers z such that\n    cos(z) = x. The convention is to return the angle z whose real part lies in [0, pi].\n    For real-valued input data types, acos always returns real output.\n    For each value that cannot be expressed as a real number or infinity, it yields nan and sets\n    the invalid floating point error flag.\n    The inverse cos is also known as acos or cos^-1.\n\n    Examples\n    ----------\n    >>> np.acos([1, -1])\n    array([ 0.        ,  3.14159265])\n    \"\"\"\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef arctan(x, out=None, **kwargs):\n    r\"\"\"\n    Trigonometric inverse tangent, element-wise.\n    The inverse of tan, so that if ``y = tan(x)`` then ``x = arctan(y)``.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input values.\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Out has the same shape as `x`. It lies is in\n        ``[-pi/2, pi/2]`` (``arctan(+/-inf)`` returns ``+/-pi/2``).\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    `arctan` is a multi-valued function: for each `x` there are infinitely\n    many numbers `z` such that tan(`z`) = `x`.  The convention is to return\n    the angle `z` whose real part lies in [-pi/2, pi/2].\n    For real-valued input data types, `arctan` always returns real output.\n    For each value that cannot be expressed as a real number or infinity,\n    it yields ``nan`` and sets the `invalid` floating point error flag.\n    For complex-valued input, we do not have support for them yet.\n    The inverse tangent is also known as `atan` or tan^{-1}.\n\n    Examples\n    --------\n    >>> x = np.array([0, 1])\n    >>> np.arctan(x)\n    array([0.       , 0.7853982])\n    >>> np.pi/4\n    0.7853981633974483\n    \"\"\"\n    return _mx_nd_np.arctan(x, out=out, **kwargs)\n\natan = arctan\natan.__doc__ = \"\"\"\n    Trigonometric inverse tangent, element-wise.\n    The inverse of tan, so that if ``y = tan(x)`` then ``x = atan(y)``.\n    \n    >>>np.atan is np.arctan\n    True\n    \n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input values.\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Out has the same shape as `x`. It lies is in\n        ``[-pi/2, pi/2]`` (``atan(+/-inf)`` returns ``+/-pi/2``).\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    `atan` is a alias for `arctan`. It is a standard API in\n    https://data-apis.org/array-api/latest/API_specification/generated/signatures.elementwise_functions.atan.html\n    instead of an official NumPy operator.\n    \n    `atan` is a multi-valued function: for each `x` there are infinitely\n    many numbers `z` such that tan(`z`) = `x`.  The convention is to return\n    the angle `z` whose real part lies in [-pi/2, pi/2].\n    For real-valued input data types, `atan` always returns real output.\n    For each value that cannot be expressed as a real number or infinity,\n    it yields ``nan`` and sets the `invalid` floating point error flag.\n    For complex-valued input, we do not have support for them yet.\n    The inverse tangent is also known as `atan` or tan^{-1}.\n\n    Examples\n    --------\n    >>> x = np.array([0, 1])\n    >>> np.atan(x)\n    array([0.       , 0.7853982])\n    >>> np.pi/4\n    0.7853981633974483\n    \"\"\"\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef sign(x, out=None, **kwargs):\n    \"\"\"\n    Returns an element-wise indication of the sign of a number.\n    The `sign` function returns ``-1 if x < 0, 0 if x==0, 1 if x > 0``. Only supports real number.\n\n    Parameters\n    ----------\n    x : ndarray or a scalar\n        Input values.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray\n        The sign of `x`.\n        This is a scalar if `x` is a scalar.\n\n    .. note::\n       * Only supports real number as input elements.\n       * Input type does not support Python native iterables(list, tuple, ...).\n       * ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be\n         the same as the expected output.\n       * ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the\n         same as the expected output.\n       * ``out`` param does not support scalar input case.\n\n    Examples\n    --------\n    >>> a = np.array([-5., 4.5])\n    >>> np.sign(a)\n    array([-1.,  1.])\n    Scalars as input:\n    >>> np.sign(4.0)\n    1.0\n    >>> np.sign(0)\n    0\n    Use ``out`` parameter:\n    >>> b = np.zeros((2, ))\n    >>> np.sign(a, out=b)\n    array([-1.,  1.])\n    >>> b\n    array([-1.,  1.])\n    \"\"\"\n    return _mx_nd_np.sign(x, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef log(x, out=None, **kwargs):\n    \"\"\"\n    Natural logarithm, element-wise.\n    The natural logarithm `log` is the inverse of the exponential function,\n    so that `log(exp(x)) = x`. The natural logarithm is logarithm in base\n    `e`.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input value. Elements must be of real value.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray\n        The natural logarithm of `x`, element-wise.\n        This is a scalar if `x` is a scalar.\n\n    .. note::\n       Currently only supports data of real values and ``inf`` as input. Returns data of\n       real value, ``inf``, ``-inf`` and ``nan`` according to the input.\n       This function differs from the original `numpy.log\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.log.html>`_ in\n       the following aspects:\n\n       * Does not support complex number for now\n       * Input type does not support Python native iterables(list, tuple, ...).\n       * ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be\n         the same as the expected output.\n       * ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the\n         same as the expected output.\n       * ``out`` param does not support scalar input case.\n\n    Examples\n    --------\n    >>> a = np.array([1, np.exp(1), np.exp(2), 0], dtype=np.float64)\n    >>> np.log(a)\n    array([  0.,   1.,   2., -inf], dtype=float64)\n    >>> # Using the default float32 dtype leads to slightly different behavior\n    >>> a = np.array([1, np.exp(1), np.exp(2), 0])\n    >>> np.log(a)\n    array([  0.,  0.99999994,   2., -inf])\n    >>> np.log(1)\n    0.0\n    \"\"\"\n    return _mx_nd_np.log(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef rint(x, out=None, **kwargs):\n    \"\"\"\n    Round elements of the array to the nearest integer.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None\n        A location into which the result is stored.\n        If provided, it must have the same shape and type as the input.\n        If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array is same shape and type as x. This is a scalar if x is a scalar.\n\n    .. note::\n       This function differs from the original `numpy.rint\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.rint.html>`_ in\n       the following way(s):\n\n       * only ndarray or scalar is accpted as valid input, tuple of ndarray is not supported\n       * broadcasting to `out` of different shape is currently not supported\n       * when input is plain python numerics, the result will not be stored in the `out` param\n\n    Examples\n    --------\n    >>> a = np.array([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0])\n    >>> np.rint(a)\n    array([-2., -2., -0.,  0.,  1.,  2.,  2.])\n    \"\"\"\n    return _mx_nd_np.rint(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef log2(x, out=None, **kwargs):\n    \"\"\"\n    Base-2 logarithm of x.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input values.\n    out : ndarray or None\n        A location into which the result is stored.\n        If provided, it must have the same shape and type as the input.\n        If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray\n        The logarithm base two of `x`, element-wise.\n        This is a scalar if `x` is a scalar.\n\n    .. note::\n       This function differs from the original `numpy.log2\n       <https://www.google.com/search?q=numpy+log2>`_ in\n       the following way(s):\n\n       * only ndarray or scalar is accpted as valid input, tuple of ndarray is not supported\n       * broadcasting to `out` of different shape is currently not supported\n       * when input is plain python numerics, the result will not be stored in the `out` param\n\n    Examples\n    --------\n    >>> x = np.array([0, 1, 2, 2**4])\n    >>> np.log2(x)\n    array([-inf,   0.,   1.,   4.])\n    \"\"\"\n    return _mx_nd_np.log2(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef log1p(x, out=None, **kwargs):\n    \"\"\"\n    Return the natural logarithm of one plus the input array, element-wise.\n    Calculates ``log(1 + x)``.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a shape that the inputs fill into. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output and input must be the same.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        Natural logarithm of 1 + x, element-wise. This is a scalar\n        if x is a scalar.\n\n    Notes\n    -----\n    For real-valued input, `log1p` is accurate also for `x` so small\n    that `1 + x == 1` in floating-point accuracy.\n    Logarithm is a multivalued function: for each `x` there is an infinite\n    number of `z` such that `exp(z) = 1 + x`. The convention is to return\n    the `z` whose imaginary part lies in `[-pi, pi]`.\n    For real-valued input data types, `log1p` always returns real output.\n    For each value that cannot be expressed as a real number or infinity,\n    it yields ``nan`` and sets the `invalid` floating point error flag.\n    cannot support complex-valued input.\n\n    Examples\n    --------\n    >>> np.log1p(1e-99)\n    1e-99\n    >>> a = np.array([3, 4, 5])\n    >>> np.log1p(a)\n    array([1.3862944, 1.609438 , 1.7917595])\n    \"\"\"\n    return _mx_nd_np.log1p(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef degrees(x, out=None, **kwargs):\n    \"\"\"\n    Convert angles from radians to degrees.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input value. Elements must be of real value.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray\n        The corresponding degree values; if `out` was supplied this is a\n        reference to it.\n        This is a scalar if `x` is a scalar.\n\n    .. note::\n       This function differs from the original `numpy.degrees\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.degrees.html>`_ in\n       the following aspects:\n\n       * Input type does not support Python native iterables(list, tuple, ...).\n         Only ndarray is supported.\n       * ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be\n         the same as the expected output.\n       * ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the\n         same as the expected output.\n       * ``out`` param does not support scalar input case.\n\n    Examples\n    --------\n    >>> rad = np.arange(12.) * np.pi / 6\n    >>> np.degrees(rad)\n    array([  0.,  30.,  60.,  90., 120., 150., 180., 210., 240., 270., 300., 330.])\n    >>> # Use specified ``out`` ndarray:\n    >>> out = np.zeros((rad.shape))\n    >>> np.degrees(rad, out)\n    array([  0.,  30.,  60.,  90., 120., 150., 180., 210., 240., 270., 300., 330.])\n    >>> out\n    array([  0.,  30.,  60.,  90., 120., 150., 180., 210., 240., 270., 300., 330.])\n    \"\"\"\n    return _mx_nd_np.degrees(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef rad2deg(x, out=None, **kwargs):\n    r\"\"\"Convert angles from radians to degrees.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Angles in degrees.\n    out : ndarray or None, optional\n        A location into which the result is stored. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The corresponding angle in radians.\n        This is a scalar if `x` is a scalar.\n\n    .. note::\n\n       \"rad2deg(x)\" is \"x * 180 / pi\".\n\n       This function differs from the original numpy.arange in the following aspects:\n\n       * Only support float32 and float64.\n       * `out` must be in the same size of input.\n\n    Examples\n    --------\n    >>> np.rad2deg(np.pi/2)\n    90.0\n    \"\"\"\n    return _mx_nd_np.rad2deg(x, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef radians(x, out=None, **kwargs):\n    \"\"\"\n    Convert angles from degrees to radians.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array in degrees.\n    out : ndarray or None\n        A location into which the result is stored.\n        If provided, it must have the same shape and type as the input.\n        If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray\n        The corresponding radian values. This is a scalar if x is a scalar.\n\n    .. note::\n       This function differs from the original `numpy.radians\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.radians.html>`_ in\n       the following way(s):\n\n       * only ndarray or scalar is accpted as valid input, tuple of ndarray is not supported\n       * broadcasting to `out` of different shape is currently not supported\n       * when input is plain python numerics, the result will not be stored in the `out` param\n\n    Examples\n    --------\n    >>> deg = np.arange(12.) * 30.\n    >>> np.radians(deg)\n    array([0.       , 0.5235988, 1.0471976, 1.5707964, 2.0943952, 2.6179938,\n           3.1415927, 3.6651914, 4.1887903, 4.712389 , 5.2359877, 5.7595863],\n           dtype=float32)\n    \"\"\"\n    return _mx_nd_np.radians(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef deg2rad(x, out=None, **kwargs):\n    r\"\"\"\n    Convert angles from degrees to radians.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Angles in degrees.\n    out : ndarray or None, optional\n        A location into which the result is stored. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The corresponding angle in radians.\n        This is a scalar if `x` is a scalar.\n\n    .. note::\n       \"deg2rad(x)\" is \"x * pi / 180\".\n\n       This function differs from the original numpy.arange in the following aspects:\n\n       * Only support float32 and float64.\n       * `out` must be in the same size of input.\n\n    Examples\n    --------\n    >>> np.deg2rad(180)\n    3.1415927\n    \"\"\"\n    return _mx_nd_np.deg2rad(x, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef reciprocal(x, out=None, **kwargs):\n    r\"\"\"Return the reciprocal of the argument, element-wise.\n    Calculates ``1/x``.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        The values whose reciprocals are required.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape as the input.\n        If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        Output array is same shape and type as x. This is a scalar if x is a scalar.\n\n    Examples\n    --------\n    >>> np.reciprocal(2.)\n    0.5\n    >>> x = np.array([1, 2., 3.33])\n    >>> np.reciprocal(x)\n    array([1.       , 0.5      , 0.3003003])\n\n    .. note::\n\n       This function is not designed to work with integers.\n       For integer arguments with absolute value larger than 1 the result is\n       always zero because of the way Python handles integer division.  For\n       integer zero the result is an overflow.\n       The output `ndarray` has the same `device` as the input `ndarray`.\n       This function differs from the original `numpy.reciprocal\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.reciprocal.html>`_ in\n       the following aspects:\n\n       * Only support ndarray and scalar now.\n       * `where` argument is not supported.\n\n    \"\"\"\n    return _mx_nd_np.reciprocal(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef square(x, out=None, **kwargs):\n    r\"\"\"\n    Return the element-wise square of the input.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        The values whose squares are required.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape as the input.\n        If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        Output array is same shape and type as x. This is a scalar if x is a scalar.\n\n    Examples\n    --------\n    >>> np.square(2.)\n    4.0\n    >>> x = np.array([1, 2., -1])\n    >>> np.square(x)\n    array([1., 4., 1.])\n\n    .. note::\n       The output `ndarray` has the same `device` as the input `ndarray`.\n       This function differs from the original `numpy.square\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.square.html>`_ in\n       the following aspects:\n\n       * Only support ndarray and scalar now.\n       * `where` argument is not supported.\n       * Complex input is not supported.\n\n    \"\"\"\n    return _mx_nd_np.square(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef negative(x, out=None, **kwargs):\n    r\"\"\"\n    Numerical negative, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray, None, or tuple of ndarray and None, optional\n          A location into which the result is stored.\n          If provided, it must have a shape that the inputs broadcast to.\n          If not provided or None, a freshly-allocated array is returned.\n          A tuple (possible only as a keyword argument) must have length\n          equal to the number of outputs.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        Returned array or scalar: y = -x. This is a scalar if x is a scalar.\n\n    Examples\n    --------\n    >>> np.negative(1)\n    -1\n    \"\"\"\n    return _mx_nd_np.negative(x, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef positive(x, out=None, **kwargs):\n    r\"\"\"\n    Computes the numerical positive of each element `x_i` (i.e.,`y_i = +x_i`)\n    of the input array x .\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        Returned array or scalar: y = +x. This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    Equivalent to `x.copy()`, but only defined for types that support arithmetic.\n\n    Examples\n    --------\n    >>> x1 = np.array(([1., -1.]))\n    >>> np.positive(x1)\n    array([ 1., -1.])\n    >>> +x1\n    array([ 1., -1.])\n    \"\"\"\n    return _mx_nd_np.positive(x, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef fix(x, out=None, **kwargs):\n    \"\"\"\n    Round an array of floats element-wise to nearest integer towards zero.\n    The rounded values are returned as floats.\n\n    Parameters\n    ----------\n    x : ndarray\n        An array of floats to be rounded\n    out : ndarray, optional\n        Output array\n\n    Returns\n    -------\n    y : ndarray or scalar\n    Returned array or scalar: y = -x. This is a scalar if x is a scalar.ndarray of floats\n\n    Examples\n    ---------\n    >>> np.fix(3.14)\n    3\n    \"\"\"\n    return _mx_nd_np.fix(x, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef tan(x, out=None, **kwargs):\n    r\"\"\"\n    Compute tangent element-wise.\n    Equivalent to np.sin(x)/np.cos(x) element-wise.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array.\n    out : ndarray or none, optional\n          A location into which the result is stored. If provided,\n          it must have a shape that the inputs broadcast to. If not provided or None,\n          a freshly-allocated array is returned. A tuple (possible only as a keyword argument)\n          must have length equal to the number of outputs.\n\n    Returns\n    -------\n    y : ndarray\n    The corresponding tangent values. This is a scalar if x is a scalar.\n\n    Examples\n    ---------\n    >>> np.tan(np.array([-np.pi, np.pi/2, np.pi]))\n    array([-8.7422777e-08, -2.2877332e+07,  8.7422777e-08])\n    \"\"\"\n\n    return _mx_nd_np.tan(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef ceil(x, out=None, **kwargs):\n    r\"\"\"\n    Return the ceiling of the input, element-wise.\n    The ceil of the ndarray `x` is the smallest integer `i`, such that\n    `i >= x`.  It is often denoted as :math:`\\lceil x \\rceil`.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a shape that the inputs fill into. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output and input must be the same.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The ceiling of each element in `x`, with `float` dtype.\n        This is a scalar if `x` is a scalar.\n\n    Examples\n    --------\n    >>> a = np.array([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0])\n    >>> np.ceil(a)\n    array([-1., -1., -0.,  1.,  2.,  2.,  2.])\n    >>> # if you use parameter out, x and out must be ndarray.\n    >>> a = np.array(1)\n    >>> np.ceil(np.array(3.5), a)\n    array(4.)\n    >>> a\n    array(4.)\n    \"\"\"\n    return _mx_nd_np.ceil(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef floor(x, out=None, **kwargs):\n    r\"\"\"\n    Return the floor of the input, element-wise.\n    The ceil of the ndarray `x` is the largest integer `i`, such that\n    `i <= x`.  It is often denoted as :math:`\\lfloor x \\rfloor`.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None\n        A location into which the result is stored. If provided, it\n        must have a shape that the inputs fill into. If not provided\n        or None, a freshly-allocated array is returned. The dtype of the\n        output and input must be the same.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The floor of each element in `x`, with `float` dtype.\n        This is a scalar if `x` is a scalar.\n\n    Examples\n    --------\n    >>> a = np.array([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0])\n    >>> np.floor(a)\n    array([-2., -2., -1.,  0.,  1.,  1.,  2.])\n    >>> # if you use parameter out, x and out must be ndarray.\n    >>> a = np.array(1)\n    >>> np.floor(np.array(3.5), a)\n    array(3.)\n    >>> a\n    array(3.)\n    \"\"\"\n    return _mx_nd_np.floor(x, out=out, **kwargs)\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef bitwise_invert(x, out=None, **kwargs):\n    r\"\"\"\n    Compute bit-wise inversion, or bit-wise NOT, element-wise.\n    Computes the bit-wise NOT of the underlying binary representation of\n    the integers in the input arrays. This ufunc implements the C/Python\n    operator ``~``.\n\n    Parameters\n    ----------\n    x : array_like\n        Only integer and boolean types are handled.\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Result.\n        This is a scalar if `x` is a scalar.\n\n    See Also\n    --------\n    bitwise_and, bitwise_or, bitwise_xor\n    logical_not\n    binary_repr :\n        Return the binary representation of the input number as a string.\n\n    Examples\n    --------\n    We've seen that 13 is represented by ``00001101``.\n    The invert or bit-wise NOT of 13 is then:\n\n    >>> x = np.bitwise_invert(np.array(13, dtype=np.uint8))\n    >>> x\n    242\n    >>> np.binary_repr(x, width=8)\n    '11110010'\n\n    Notes\n    -----\n    `bitwise_not` is an alias for `invert`:\n\n    >>> np.bitwise_not is np.invert\n    True\n    \"\"\"\n    return _mx_nd_np.bitwise_not(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef invert(x, out=None, **kwargs):\n    r\"\"\"\n    Compute bit-wise inversion, or bit-wise NOT, element-wise.\n    Computes the bit-wise NOT of the underlying binary representation of\n    the integers in the input arrays. This ufunc implements the C/Python\n    operator ``~``.\n\n    Parameters\n    ----------\n    x : array_like\n        Only integer and boolean types are handled.\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Result.\n        This is a scalar if `x` is a scalar.\n\n    See Also\n    --------\n    bitwise_and, bitwise_or, bitwise_xor\n    logical_not\n    binary_repr :\n        Return the binary representation of the input number as a string.\n\n    Examples\n    --------\n    We've seen that 13 is represented by ``00001101``.\n    The invert or bit-wise NOT of 13 is then:\n\n    >>> x = np.invert(np.array(13, dtype=np.uint8))\n    >>> x\n    242\n    >>> np.binary_repr(x, width=8)\n    '11110010'\n\n    Notes\n    -----\n    `bitwise_not` is an alias for `invert`:\n\n    >>> np.bitwise_not is np.invert\n    True\n    \"\"\"\n    return _mx_nd_np.bitwise_not(x, out=out, **kwargs)\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef bitwise_not(x, out=None, **kwargs):\n    r\"\"\"\n    Compute bit-wise inversion, or bit-wise NOT, element-wise.\n    Computes the bit-wise NOT of the underlying binary representation of\n    the integers in the input arrays. This ufunc implements the C/Python\n    operator ``~``.\n\n    Parameters\n    ----------\n    x : array_like\n        Only integer and boolean types are handled.\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Result.\n        This is a scalar if `x` is a scalar.\n\n    See Also\n    --------\n    bitwise_and, bitwise_or, bitwise_xor\n    logical_not\n    binary_repr :\n        Return the binary representation of the input number as a string.\n\n    Examples\n    --------\n    We've seen that 13 is represented by ``00001101``.\n    The invert or bit-wise NOT of 13 is then:\n\n    >>> x = np.invert(np.array(13, dtype=np.uint8))\n    >>> x\n    242\n    >>> np.binary_repr(x, width=8)\n    '11110010'\n\n    Notes\n    -----\n    `bitwise_not` is an alias for `invert`:\n\n    >>> np.bitwise_not is np.invert\n    True\n    \"\"\"\n    return _mx_nd_np.bitwise_not(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef trunc(x, out=None, **kwargs):\n    r\"\"\"\n    Return the truncated value of the input, element-wise.\n    The truncated value of the scalar `x` is the nearest integer `i` which\n    is closer to zero than `x` is. In short, the fractional part of the\n    signed number `x` is discarded.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input data.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The truncated value of each element in `x`.\n        This is a scalar if `x` is a scalar.\n\n    .. note::\n       This function differs from the original numpy.trunc in the following aspects:\n\n       * Do not support `where`, a parameter in numpy which indicates where to calculate.\n       * Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n       * Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n       * If `x` is plain python numeric, the result won't be stored in out.\n\n    Examples\n    --------\n    >>> a = np.array([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0])\n    >>> np.trunc(a)\n    array([-1., -1., -0.,  0.,  1.,  1.,  2.])\n    \"\"\"\n    return _mx_nd_np.trunc(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef logical_not(x, out=None, **kwargs):\n    r\"\"\"\n    Compute the truth value of NOT x element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Logical NOT is applied to the elements of `x`.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n\n    Returns\n    -------\n    y : bool or ndarray of bool\n        Boolean result with the same shape as `x` of the NOT operation\n        on elements of `x`.\n        This is a scalar if `x` is a scalar.\n\n    .. note::\n       This function differs from the original numpy.logical_not in the following aspects:\n       * Do not support `where`, a parameter in numpy which indicates where to calculate.\n       * Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n       * Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n       * If `x` is plain python numeric, the result won't be stored in out.\n\n    Examples\n    --------\n    >>> x= np.array([True, False, 0, 1])\n    >>> np.logical_not(x)\n    array([False,  True,  True, False])\n\n    >>> x = np.arange(5)\n    >>> np.logical_not(x<3)\n    array([False, False, False,  True,  True])\n    \"\"\"\n    return _mx_nd_np.logical_not(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef arcsinh(x, out=None, **kwargs):\n    r\"\"\"\n    Inverse hyperbolic cosine, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n\n    Returns\n    -------\n    arcsinh : ndarray\n        Array of the same shape as `x`.\n        This is a scalar if `x` is a scalar.\n\n    .. note::\n       `arcsinh` is a multivalued function: for each `x` there are infinitely\n       many numbers `z` such that `sinh(z) = x`.\n\n       For real-valued input data types, `arcsinh` always returns real output.\n       For each value that cannot be expressed as a real number or infinity, it\n       yields ``nan`` and sets the `invalid` floating point error flag.\n\n       This function differs from the original numpy.arcsinh in the following aspects:\n\n       * Do not support `where`, a parameter in numpy which indicates where to calculate.\n       * Do not support complex-valued input.\n       * Cannot cast type automatically. DType of `out` must be same as the expected one.\n       * Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n       * If `x` is plain python numeric, the result won't be stored in out.\n\n    Examples\n    --------\n    >>> a = np.array([3.2, 5.0])\n    >>> np.arcsinh(a)\n    array([1.8309381, 2.2924316])\n\n    >>> np.arcsinh(1)\n    0.0\n    \"\"\"\n    return _mx_nd_np.arcsinh(x, out=out, **kwargs)\n\nasinh = arcsinh\nasinh.__doc__ = \"\"\"\n    Inverse hyperbolic cosine, element-wise.\n    \n    >>>np.asinh is np.arcsinh\n    True\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n\n    Returns\n    -------\n    asinh : ndarray\n        Array of the same shape as `x`.\n        This is a scalar if `x` is a scalar.\n\n    .. note::\n       `asinh` is a alias for `arcsinh`. It is a standard API in\n       https://data-apis.org/array-api/latest/API_specification/generated/signatures.elementwise_functions.asinh.html\n       instead of an official NumPy operator.\n       \n       `asinh` is a multivalued function: for each `x` there are infinitely\n       many numbers `z` such that `sinh(z) = x`.\n\n       For real-valued input data types, `asinh` always returns real output.\n       For each value that cannot be expressed as a real number or infinity, it\n       yields ``nan`` and sets the `invalid` floating point error flag.\n\n       This function differs from the original numpy.arcsinh in the following aspects:\n\n       * Do not support `where`, a parameter in numpy which indicates where to calculate.\n       * Do not support complex-valued input.\n       * Cannot cast type automatically. DType of `out` must be same as the expected one.\n       * Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n       * If `x` is plain python numeric, the result won't be stored in out.\n\n    Examples\n    --------\n    >>> a = np.array([3.2, 5.0])\n    >>> np.asinh(a)\n    array([1.8309381, 2.2924316])\n\n    >>> np.asinh(1)\n    0.0\n    \"\"\"\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef arccosh(x, out=None, **kwargs):\n    r\"\"\"\n    Inverse hyperbolic cosine, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n\n    Returns\n    -------\n    arccosh : ndarray\n        Array of the same shape as `x`.\n        This is a scalar if `x` is a scalar.\n\n    .. note::\n       `arccosh` is a multivalued function: for each `x` there are infinitely\n       many numbers `z` such that `cosh(z) = x`.\n\n       For real-valued input data types, `arccosh` always returns real output.\n       For each value that cannot be expressed as a real number or infinity, it\n       yields ``nan`` and sets the `invalid` floating point error flag.\n\n       This function differs from the original numpy.arccosh in the following aspects:\n\n       * Do not support `where`, a parameter in numpy which indicates where to calculate.\n       * Do not support complex-valued input.\n       * Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n       * Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n       * If `x` is plain python numeric, the result won't be stored in out.\n\n    Examples\n    --------\n    >>> a = np.array([3.2, 5.0])\n    >>> np.arccosh(a)\n    array([1.8309381, 2.2924316])\n\n    >>> np.arccosh(1)\n    0.0\n    \"\"\"\n    return _mx_nd_np.arccosh(x, out=out, **kwargs)\n\nacosh = arccosh\nacosh.__doc__ = \"\"\"\n    Inverse hyperbolic cosine, element-wise.\n    \n    >>>np.acosh is np.arccosh\n    True\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n\n    Returns\n    -------\n    acosh : ndarray\n        Array of the same shape as `x`.\n        This is a scalar if `x` is a scalar.\n\n    .. note::\n       `acosh` is a alias for `arccosh`. It is a standard API in\n       https://data-apis.org/array-api/latest/API_specification/generated/signatures.elementwise_functions.acosh.html\n       instead of an official NumPy operator.\n       \n       `acosh` is a multivalued function: for each `x` there are infinitely\n       many numbers `z` such that `cosh(z) = x`.\n\n       For real-valued input data types, `acosh` always returns real output.\n       For each value that cannot be expressed as a real number or infinity, it\n       yields ``nan`` and sets the `invalid` floating point error flag.\n\n       This function differs from the original numpy.arccosh in the following aspects:\n\n       * Do not support `where`, a parameter in numpy which indicates where to calculate.\n       * Do not support complex-valued input.\n       * Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n       * Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n       * If `x` is plain python numeric, the result won't be stored in out.\n\n    Examples\n    --------\n    >>> a = np.array([3.2, 5.0])\n    >>> np.acosh(a)\n    array([1.8309381, 2.2924316])\n\n    >>> np.acosh(1)\n    0.0\n    \"\"\"\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef arctanh(x, out=None, **kwargs):\n    r\"\"\"\n    Inverse hyperbolic tangent, element-wise.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n\n    Returns\n    -------\n    arctanh : ndarray\n        Array of the same shape as `x`.\n        This is a scalar if `x` is a scalar.\n\n    .. note::\n       `arctanh` is a multivalued function: for each `x` there are infinitely\n       many numbers `z` such that `tanh(z) = x`.\n\n       For real-valued input data types, `arctanh` always returns real output.\n       For each value that cannot be expressed as a real number or infinity, it\n       yields ``nan`` and sets the `invalid` floating point error flag.\n\n       This function differs from the original numpy.arctanh in the following aspects:\n\n       * Do not support `where`, a parameter in numpy which indicates where to calculate.\n       * Do not support complex-valued input.\n       * Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n       * Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n       * If `x` is plain python numeric, the result won't be stored in out.\n\n    Examples\n    --------\n    >>> a = np.array([0.0, -0.5])\n    >>> np.arctanh(a)\n    array([0., -0.54930615])\n\n    >>> np.arctanh(1)\n    0.0\n    \"\"\"\n    return _mx_nd_np.arctanh(x, out=out, **kwargs)\n\natanh = arctanh\natanh.__doc__ = \"\"\"\n    Inverse hyperbolic tangent, element-wise.\n\n    >>>np.atanh is np.arctanh\n    True\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n\n    Returns\n    -------\n    atanh : ndarray\n        Array of the same shape as `x`.\n        This is a scalar if `x` is a scalar.\n\n    .. note::\n       `atanh` is a alias for `arctanh`. It is a standard API in\n       https://data-apis.org/array-api/latest/API_specification/generated/signatures.elementwise_functions.atanh.html\n       instead of an official NumPy operator.\n    \n       `atanh` is a multivalued function: for each `x` there are infinitely\n       many numbers `z` such that `tanh(z) = x`.\n\n       For real-valued input data types, `atanh` always returns real output.\n       For each value that cannot be expressed as a real number or infinity, it\n       yields ``nan`` and sets the `invalid` floating point error flag.\n\n       This function differs from the original numpy.arctanh in the following aspects:\n\n       * Do not support `where`, a parameter in numpy which indicates where to calculate.\n       * Do not support complex-valued input.\n       * Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n       * Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n       * If `x` is plain python numeric, the result won't be stored in out.\n\n    Examples\n    --------\n    >>> a = np.array([0.0, -0.5])\n    >>> np.atanh(a)\n    array([0., -0.54930615])\n\n    >>> np.atanh(1)\n    0.0\n    \"\"\"\n\n\n@set_module('mxnet.numpy')\n@wrap_sort_functions\ndef argsort(a, axis=-1, descending=False, stable=True):\n    \"\"\"\n    Returns the indices that sort an array `x` along a specified axis.\n\n    Notes\n    -----\n    `argsort` is a standard API in\n    https://data-apis.org/array-api/latest/API_specification/generated/signatures.sorting_functions.argsort.html\n    instead of an official NumPy operator.\n\n    Parameters\n    ----------\n    a : ndarray\n        Array to sort.\n    axis : int or None, optional\n        Axis along which to sort.  The default is -1 (the last axis). If None,\n        the flattened array is used.\n    descending : bool, optional\n        sort order. If `True`, the returned indices sort x in descending order (by value).\n        If `False`, the returned indices sort x in ascending order (by value).Default: False.\n    stable : bool, optional\n        sort stability. If `True`, the returned indices must maintain the relative order\n        of x values which compare as equal. If `False`, the returned indices may or may not\n        maintain the relative order of x values which compare as equal. Default: True.\n\n    Returns\n    -------\n    index_array : ndarray, int\n        Array of indices that sort `a` along the specified `axis`.\n        If `a` is one-dimensional, ``a[index_array]`` yields a sorted `a`.\n        More generally, ``np.take_along_axis(a, index_array, axis=axis)``\n        always yields the sorted `a`, irrespective of dimensionality.\n\n    Notes\n    -----\n    This operator does not support different sorting algorithms.\n\n    Examples\n    --------\n    One dimensional array:\n\n    >>> x = np.array([3, 1, 2])\n    >>> np.argsort(x)\n    array([1, 2, 0])\n\n    Two-dimensional array:\n\n    >>> x = np.array([[0, 3], [2, 2]])\n    >>> x\n    array([[0, 3],\n           [2, 2]])\n    >>> ind = np.argsort(x, axis=0)  # sorts along first axis (down)\n    >>> ind\n    array([[0, 1],\n           [1, 0]])\n    >>> np.take_along_axis(x, ind, axis=0)  # same as np.sort(x, axis=0)\n    array([[0, 2],\n           [2, 3]])\n    >>> ind = np.argsort(x, axis=1)  # sorts along last axis (across)\n    >>> ind\n    array([[0, 1],\n           [0, 1]])\n    >>> np.take_along_axis(x, ind, axis=1)  # same as np.sort(x, axis=1)\n    array([[0, 3],\n           [2, 2]])\n\n    Indices of the sorted elements of a N-dimensional array:\n\n    >>> ind = np.unravel_index(np.argsort(x, axis=None), x.shape)\n    >>> ind\n    (array([0, 1, 1, 0]), array([0, 0, 1, 1]))\n    >>> x[ind]  # same as np.sort(x, axis=None)\n    array([0, 2, 2, 3])\n    \"\"\"\n    if stable:\n        warnings.warn(\"Currently, MXNet only support quicksort in backend, which is not stable\")\n    return _mx_nd_np.argsort(a, axis=axis, descending=descending)\n\n\n@set_module('mxnet.numpy')\n@wrap_sort_functions\ndef sort(a, axis=-1, descending=False, stable=True):\n    \"\"\"\n    Return a sorted copy of an array.\n\n    Notes\n    -----\n    `sort` is a standard API in\n    https://data-apis.org/array-api/latest/API_specification/generated/signatures.sorting_functions.sort.html\n    instead of an official NumPy operator.\n\n    Parameters\n    ----------\n    a : ndarray\n        Array to sort.\n    axis : int or None, optional\n        Axis along which to sort.  The default is -1 (the last axis). If None,\n        the flattened array is used.\n    descending : bool, optional\n        sort order. If `True`, the returned indices sort x in descending order (by value).\n        If `False`, the returned indices sort x in ascending order (by value).Default: False.\n    stable : bool, optional\n        sort stability. If `True`, the returned indices must maintain the relative order\n        of x values which compare as equal. If `False`, the returned indices may or may not\n        maintain the relative order of x values which compare as equal. Default: True.\n\n    Returns\n    -------\n    sorted_array : ndarray\n        Array of the same type and shape as `a`.\n\n    Notes\n    -----\n    This operator does not support different sorting algorithms.\n\n    Examples\n    --------\n    >>> a = np.array([[1,4],[3,1]])\n    >>> np.sort(a)                # sort along the last axis\n    array([[1, 4],\n           [1, 3]])\n    >>> np.sort(a, axis=None)     # sort the flattened array\n    array([1, 1, 3, 4])\n    >>> np.sort(a, axis=0)        # sort along the first axis\n    array([[1, 1],\n           [3, 4]])\n    \"\"\"\n    return _mx_nd_np.sort(a, axis=axis, descending=descending)\n\n\n@set_module('mxnet.numpy')\ndef tensordot(a, b, axes=2):\n    r\"\"\"Compute tensor dot product along specified axes for arrays >= 1-D.\n    Given two tensors (arrays of dimension greater than or equal to one),\n    ``a`` and ``b``, and an ndarray object containing two ndarray\n    objects, ``(a_axes, b_axes)``, sum the products of ``a``'s and ``b``'s\n    elements (components) over the axes specified by ``a_axes`` and\n    ``b_axes``. The third argument can be a single non-negative\n    integer_like scalar, ``N``; if it is such, then the last ``N``\n    dimensions of ``a`` and the first ``N`` dimensions of ``b`` are summed\n    over.\n\n    Parameters\n    ----------\n    a, b : ndarray, len(shape) >= 1\n        Tensors to \"dot\".\n    axes : int or (2,) ndarray\n\n        * integer_like\n          If an int N, sum over the last N axes of `a` and the first N axes\n          of `b` in order. The sizes of the corresponding axes must match.\n        * (2,) ndarray\n          Or, a list of axes to be summed over, first sequence applying to `a`,\n          second to `b`. Both elements ndarray must be of the same length.\n\n    See Also\n    --------\n    dot, einsum\n\n    .. note::\n\n       Three common use cases are:\n\n           * ``axes = 0`` : tensor product :math:`a\\otimes b`\n           * ``axes = 1`` : tensor dot product :math:`a\\cdot b`\n           * ``axes = 2`` : (default) tensor double contraction :math:`a:b`\n       When `axes` is integer_like, the sequence for evaluation will be: first\n       the -Nth axis in `a` and 0th axis in `b`, and the -1th axis in `a` and\n       Nth axis in `b` last.\n       When there is more than one axis to sum over - and they are not the last\n       (first) axes of `a` (`b`) - the argument `axes` should consist of\n       two sequences of the same length, with the first axis to sum over given\n       first in both sequences, the second axis second, and so forth.\n\n    Examples\n    --------\n    >>> a = np.arange(60.).reshape(3,4,5)\n    >>> b = np.arange(24.).reshape(4,3,2)\n    >>> c = np.tensordot(a,b, axes=([1,0],[0,1]))\n    >>> c.shape\n    (5, 2)\n    >>> c\n    array([[ 4400.,  4730.],\n           [ 4532.,  4874.],\n           [ 4664.,  5018.],\n           [ 4796.,  5162.],\n           [ 4928.,  5306.]])\n    \"\"\"\n    return _mx_nd_np.tensordot(a, b, axes)\n\n\n@set_module('mxnet.numpy')\ndef histogram(a, bins=10, range=None, normed=None, weights=None, density=None):  # pylint: disable=too-many-arguments\n    \"\"\"\n    Compute the histogram of a set of data.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data. The histogram is computed over the flattened array.\n    bins : int or ndarray\n        If `bins` is an int, it defines the number of equal-width\n        bins in the given range (10, by default). If `bins` is a\n        sequence, it defines a monotonically increasing array of bin edges,\n        including the rightmost edge, allowing for non-uniform bin widths.\n        .. versionadded:: 1.11.0\n        If `bins` is a string, it defines the method used to calculate the\n        optimal bin width, as defined by `histogram_bin_edges`.\n    range : (float, float)\n        The lower and upper range of the bins. Required when `bins` is an integer.\n        Values outside the range are ignored. The first element of the range must\n        be less than or equal to the second.\n    normed : bool, optional\n        Not supported yet, coming soon.\n    weights : array_like, optional\n        Not supported yet, coming soon.\n    density : bool, optional\n        Not supported yet, coming soon.\n\n    Examples\n    --------\n    >>> np.histogram(np.arange(4), bins=np.arange(5))\n    [array([1, 1, 1, 1], dtype=int64), array([0., 1., 2., 3., 4.])]\n    \"\"\"\n    return _mx_nd_np.histogram(a, bins=bins, range=range, normed=normed, weights=weights, density=density)\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef eye(N, M=None, k=0, dtype=None, device=None, **kwargs):\n    \"\"\"\n    Return a 2-D array with ones on the diagonal and zeros elsewhere.\n\n    Parameters\n    ----------\n    N : int\n        Number of rows in the output.\n    M : int, optional\n        Number of columns in the output. If None, defaults to N.\n    k : int, optional\n        Index of the diagonal: 0 (the default) refers to the main diagonal,\n        a positive value refers to an upper diagonal,\n        and a negative value to a lower diagonal.\n    dtype : data-type, optional\n        Data-type of the returned array.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    I : ndarray of shape (N,M)\n        An array where all elements are equal to zero,\n        except for the k-th diagonal, whose values are equal to one.\n\n    Examples\n    --------\n    >>> np.eye(2, dtype=int)\n    array([[1, 0],\n           [0, 1]], dtype=int64)\n    >>> np.eye(3, k=1)\n    array([[0., 1., 0.],\n           [0., 0., 1.],\n           [0., 0., 0.]])\n    \"\"\"\n    return _mx_nd_np.eye(N, M, k, dtype, device=device, **kwargs)\n# pylint: enable=redefined-outer-name\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis=0, device=None):  # pylint: disable=too-many-arguments\n    r\"\"\"\n    Return evenly spaced numbers over a specified interval.\n\n    Returns num evenly spaced samples, calculated over the interval [start, stop].\n    The endpoint of the interval can optionally be excluded.\n\n    Parameters\n    ----------\n    start : int or float\n        The starting value of the sequence.\n    stop : int or float\n        The end value of the sequence, unless endpoint is set to False. In\n        that case, the sequence consists of all but the last of num + 1\n        evenly spaced samples, so that stop is excluded. Note that the step\n        size changes when endpoint is False.\n    num : int, optional\n        Number of samples to generate. Default is 50. Must be non-negative.\n    endpoint : bool, optional\n        If True, stop is the last sample. Otherwise, it is not included.\n        Default is True.\n    retstep : bool, optional\n        If True, return (samples, step), where step is the spacing between samples.\n    dtype : dtype, optional\n        The type of the output array. If dtype is not given, infer the data\n        type from the other input arguments.\n    axis : int, optional\n        The axis in the result to store the samples. Relevant only if start or\n        stop are array-like. By default (0), the samples will be along a new\n        axis inserted at the beginning. Use -1 to get an axis at the end.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    samples : ndarray\n        There are num equally spaced samples in the closed interval\n        `[start, stop]` or the half-open interval `[start, stop)`\n        (depending on whether endpoint is True or False).\n    step : float, optional\n        Only returned if retstep is True\n        Size of spacing between samples.\n\n\n    See Also\n    --------\n    arange : Similar to `linspace`, but uses a step size (instead of the\n             number of samples).\n\n    Examples\n    --------\n    >>> np.linspace(2.0, 3.0, num=5)\n    array([2.  , 2.25, 2.5 , 2.75, 3.  ])\n    >>> np.linspace(2.0, 3.0, num=5, endpoint=False)\n    array([2. , 2.2, 2.4, 2.6, 2.8])\n    >>> np.linspace(2.0, 3.0, num=5, retstep=True)\n    (array([2.  , 2.25, 2.5 , 2.75, 3.  ]), 0.25)\n\n    Graphical illustration:\n\n    >>> import matplotlib.pyplot as plt\n    >>> N = 8\n    >>> y = np.zeros(N)\n    >>> x1 = np.linspace(0, 10, N, endpoint=True)\n    >>> x2 = np.linspace(0, 10, N, endpoint=False)\n    >>> plt.plot(x1.asnumpy(), y.asnumpy(), 'o')\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.plot(x2.asnumpy(), (y + 0.5).asnumpy(), 'o')\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.ylim([-0.5, 1])\n    (-0.5, 1)\n    >>> plt.show()\n\n    .. note::\n\n       This function differs from the original `numpy.linspace\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linspace.html>`_ in\n       the following aspects:\n\n       * `start` and `stop` do not support list, numpy ndarray and mxnet ndarray\n       * axis could only be 0\n       * There could be an additional `device` argument to specify the device, e.g. the i-th\n         GPU.\n    \"\"\"\n    return _mx_nd_np.linspace(start, stop, num, endpoint, retstep, dtype, axis, device)\n# pylint: enable=redefined-outer-name\n\n\n# pylint: disable=too-many-arguments, redefined-outer-name\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None, axis=0, device=None):\n    r\"\"\"Return numbers spaced evenly on a log scale.\n\n    In linear space, the sequence starts at ``base ** start``\n    (`base` to the power of `start`) and ends with ``base ** stop``\n    (see `endpoint` below).\n\n        Non-scalar `start` and `stop` are now supported.\n\n    Parameters\n    ----------\n    start : int or float\n        ``base ** start`` is the starting value of the sequence.\n    stop : int or float\n        ``base ** stop`` is the final value of the sequence, unless `endpoint`\n        is False.  In that case, ``num + 1`` values are spaced over the\n        interval in log-space, of which all but the last (a sequence of\n        length `num`) are returned.\n    num : integer, optional\n        Number of samples to generate.  Default is 50.\n    endpoint : boolean, optional\n        If true, `stop` is the last sample. Otherwise, it is not included.\n        Default is True.\n    base : float, optional\n        The base of the log space. The step size between the elements in\n        ``ln(samples) / ln(base)`` (or ``log_base(samples)``) is uniform.\n        Default is 10.0.\n    dtype : dtype\n        The type of the output array.  If `dtype` is not given, infer the data\n        type from the other input arguments.\n    axis : int, optional\n        The axis in the result to store the samples.  Relevant only if start\n        or stop are array-like.  By default (0), the samples will be along a\n        new axis inserted at the beginning. Now, axis only support axis = 0.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    samples : ndarray\n        `num` samples, equally spaced on a log scale.\n\n    See Also\n    --------\n    arange : Similar to linspace, with the step size specified instead of the\n             number of samples. Note that, when used with a float endpoint, the\n             endpoint may or may not be included.\n    linspace : Similar to logspace, but with the samples uniformly distributed\n               in linear space, instead of log space.\n\n    Notes\n    -----\n    Logspace is equivalent to the code\n\n    >>> y = np.linspace(start, stop, num=num, endpoint=endpoint)\n    ...\n    >>> power(base, y).astype(dtype)\n    ...\n\n    Examples\n    --------\n    >>> np.logspace(2.0, 3.0, num=4)\n    array([ 100.     ,  215.44347,  464.15887, 1000.     ])\n    >>> np.logspace(2.0, 3.0, num=4, endpoint=False)\n    array([100.     , 177.82794, 316.22775, 562.3413 ])\n    >>> np.logspace(2.0, 3.0, num=4, base=2.0)\n    array([4.       , 5.0396843, 6.349604 , 8.       ])\n    >>> np.logspace(2.0, 3.0, num=4, base=2.0, dtype=np.int32)\n    array([4, 5, 6, 8], dtype=int32)\n    >>> np.logspace(2.0, 3.0, num=4, device=npx.gpu(0))\n    array([ 100.     ,  215.44347,  464.15887, 1000.     ], device=gpu(0))\n    \"\"\"\n    return _mx_nd_np.logspace(start, stop, num, endpoint, base, dtype, axis, device=device)\n# pylint: enable=too-many-arguments, redefined-outer-name\n\n\n@set_module('mxnet.numpy')\ndef expand_dims(a, axis):\n    \"\"\"Expand the shape of an array.\n\n    Insert a new axis that will appear at the `axis` position in the expanded array shape.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array.\n    axis : int\n        Position in the expanded axes where the new axis is placed.\n\n    Returns\n    -------\n    res : ndarray\n        Output array. The number of dimensions is one greater than that of\n        the input array.\n\n    See Also\n    --------\n    squeeze : The inverse operation, removing singleton dimensions\n    reshape : Insert, remove, and combine dimensions, and resize existing ones\n\n    Examples\n    --------\n    >>> x = np.array([1,2])\n    >>> x.shape\n    (2,)\n\n    >>> y = np.expand_dims(x, axis=0)\n    >>> y\n    array([[1., 2.]])\n\n    >>> y.shape\n    (1, 2)\n\n    >>> y = np.expand_dims(x, axis=1)  # Equivalent to x[:,np.newaxis]\n    >>> y\n    array([[1.],\n           [2.]])\n\n    >>> y.shape\n    (2, 1)\n\n    Note that some examples may use None instead of np.newaxis. These are the same objects:\n\n    >>> np.newaxis is None\n    True\n    \"\"\"\n    return _mx_nd_np.expand_dims(a, axis)\n\n\n@set_module('mxnet.numpy')\ndef tile(A, reps):\n    r\"\"\"\n    Construct an array by repeating A the number of times given by reps.\n\n    If `reps` has length ``d``, the result will have dimension of\n    ``max(d, A.ndim)``.\n\n    If ``A.ndim < d``, `A` is promoted to be d-dimensional by prepending new\n    axes. So a shape (3,) array is promoted to (1, 3) for 2-D replication,\n    or shape (1, 1, 3) for 3-D replication. If this is not the desired\n    behavior, promote `A` to d-dimensions manually before calling this\n    function.\n\n    If ``A.ndim > d``, `reps` is promoted to `A`.ndim by pre-pending 1's to it.\n    Thus for an `A` of shape (2, 3, 4, 5), a `reps` of (2, 2) is treated as\n    (1, 1, 2, 2).\n\n    Parameters\n    ----------\n    A : ndarray or scalar\n        An input array or a scalar to repeat.\n    reps : a single integer or tuple of integers\n        The number of repetitions of `A` along each axis.\n\n    Returns\n    -------\n    c : ndarray\n        The tiled output array.\n\n    Examples\n    --------\n    >>> a = np.array([0, 1, 2])\n    >>> np.tile(a, 2)\n    array([0., 1., 2., 0., 1., 2.])\n    >>> np.tile(a, (2, 2))\n    array([[0., 1., 2., 0., 1., 2.],\n           [0., 1., 2., 0., 1., 2.]])\n    >>> np.tile(a, (2, 1, 2))\n    array([[[0., 1., 2., 0., 1., 2.]],\n           [[0., 1., 2., 0., 1., 2.]]])\n\n    >>> b = np.array([[1, 2], [3, 4]])\n    >>> np.tile(b, 2)\n    array([[1., 2., 1., 2.],\n           [3., 4., 3., 4.]])\n    >>> np.tile(b, (2, 1))\n    array([[1., 2.],\n           [3., 4.],\n           [1., 2.],\n           [3., 4.]])\n\n    >>> c = np.array([1,2,3,4])\n    >>> np.tile(c,(4,1))\n    array([[1., 2., 3., 4.],\n           [1., 2., 3., 4.],\n           [1., 2., 3., 4.],\n           [1., 2., 3., 4.]])\n\n    Scalar as input:\n\n    >>> np.tile(2, 3)\n    array([2, 2, 2]) # repeating integer `2`\n\n    \"\"\"\n    return _mx_nd_np.tile(A, reps)\n\n\n@set_module('mxnet.numpy')\ndef trace(a, offset=0, axis1=0, axis2=1, out=None):\n    \"\"\"\n    Return the sum along diagonals of the array.\n    If `a` is 2-D, the sum along its diagonal with the given offset\n    is returned, i.e., the sum of elements ``a[i,i+offset]`` for all i.\n    If `a` has more than two dimensions, then the axes specified by axis1 and\n    axis2 are used to determine the 2-D sub-arrays whose traces are returned.\n    The shape of the resulting array is the same as that of `a` with `axis1`\n    and `axis2` removed.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array, from which the diagonals are taken.\n    offset : int, optional\n        Offset of the diagonal from the main diagonal. Can be both positive\n        and negative. Defaults to 0.\n    axis1, axis2 : int, optional\n        Axes to be used as the first and second axis of the 2-D sub-arrays\n        from which the diagonals should be taken. Defaults are the first two\n        axes of `a`.\n    out : ndarray, optional\n        Array into which the output is placed. It must be of the right shape\n        and right type to hold the output.\n\n    Returns\n    -------\n    sum_along_diagonals : ndarray\n        If `a` is 2-D, the sum along the diagonal is returned.  If `a` has\n        larger dimensions, then an array of sums along diagonals is returned.\n\n    Examples\n    --------\n    >>> a = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]])\n    >>> np.trace(a)\n    array(3.)\n    >>> a = np.arange(8).reshape((2, 2, 2))\n    >>> np.trace(a)\n    array([6., 8.])\n    >>> a = np.arange(24).reshape((2, 2, 2, 3))\n    >>> np.trace(a).shape\n    (2, 3)\n    \"\"\"\n    return _mx_nd_np.trace(a, offset, axis1, axis2, out)\n\n\n@set_module('mxnet.numpy')\ndef transpose(a, axes=None):\n    \"\"\"\n    Permute the dimensions of an array.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array.\n    axes : list of ints, optional\n        By default, reverse the dimensions,\n        otherwise permute the axes according to the values given.\n\n    Returns\n    -------\n    p : ndarray\n        a with its axes permuted.\n\n    .. note::\n\n       This function differs from the original `numpy.transpose\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.transpose.html>`_ in\n       the following way(s):\n\n       * only ndarray is accepted as valid input, python iterables are not supported\n       * the operator always returns an `ndarray` that does not share the memory with the input\n\n    Examples\n    --------\n    >>> x = np.arange(4).reshape((2,2))\n    >>> x\n    array([[0., 1.],\n           [2., 3.]])\n    >>> np.transpose(x)\n    array([[0., 2.],\n           [1., 3.]])\n    >>> x = np.ones((1, 2, 3))\n    >>> np.transpose(x, (1, 0, 2)).shape\n    (2, 1, 3)\n    \"\"\"\n    return _mx_nd_np.transpose(a, axes)\n\n\n@set_module('mxnet.numpy')\ndef permute_dims(a, axes=None):\n    \"\"\"\n    Permute the dimensions of an array.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array.\n    axes : list of ints, optional\n        By default, reverse the dimensions,\n        otherwise permute the axes according to the values given.\n\n    Returns\n    -------\n    p : ndarray\n        a with its axes permuted.\n\n    Note\n    --------\n    `permute_dims` is a alias for `transpose`. It is a standard API in\n    https://data-apis.org/array-api/latest/API_specification/manipulation_functions.html#permute-dims-x-axes\n    instead of an official NumPy operator.\n\n    Examples\n    --------\n    >>> x = np.arange(4).reshape((2,2))\n    >>> x\n    array([[0., 1.],\n           [2., 3.]])\n    >>> np.permute_dims(x)\n    array([[0., 2.],\n           [1., 3.]])\n    >>> x = np.ones((1, 2, 3))\n    >>> np.permute_dims(x, (1, 0, 2)).shape\n    (2, 1, 3)\n    \"\"\"\n    return _mx_nd_np.transpose(a, axes)\n\n\n@set_module('mxnet.numpy')\ndef repeat(a, repeats, axis=None):\n    \"\"\"\n    Repeat elements of an array.\n\n    Parameters\n    ----------\n    a : array_like\n        Input array.\n    repeats : int\n        The number of repetitions for each element.\n    axis : int, optional\n        The axis along which to repeat values.  By default, use the\n        flattened input array, and return a flat output array.\n\n    Returns\n    -------\n    repeated_array : ndarray\n        Output array which has the same shape as `a`, except along\n        the given axis.\n\n    See Also\n    --------\n    tile : Tile an array.\n\n    Examples\n    --------\n    >>> np.repeat(3, 4)\n    array([3, 3, 3, 3])\n    >>> x = np.array([[1,2],[3,4]])\n    >>> np.repeat(x, 2)\n    array([1, 1, 2, 2, 3, 3, 4, 4])\n    >>> np.repeat(x, 3, axis=1)\n    array([[1, 1, 1, 2, 2, 2],\n           [3, 3, 3, 4, 4, 4]])\n    >>> np.repeat(x, [1, 2], axis=0)\n    array([[1, 2],\n           [3, 4],\n           [3, 4]])\n    \"\"\"\n    return _mx_nd_np.repeat(a, repeats, axis)\n\n\n@set_module('mxnet.numpy')\ndef tril(m, k=0):\n    r\"\"\"\n    Lower triangle of an array.\n\n    Return a copy of an array with elements above the `k`-th diagonal zeroed.\n\n    Parameters\n    ----------\n    m : ndarray, shape (M, N)\n        Input array.\n    k : int, optional\n        Diagonal above which to zero elements.  `k = 0` (the default) is the\n        main diagonal, `k < 0` is below it and `k > 0` is above.\n\n    Returns\n    -------\n    tril : ndarray, shape (M, N)\n        Lower triangle of `m`, of same shape and data-type as `m`.\n\n    See Also\n    --------\n    triu : same thing, only for the upper triangle\n\n    Examples\n    --------\n    >>> a = np.array([[1,2,3],[4,5,6],[7,8,9],[10,11,12]])\n    >>> np.tril(a, -1)\n    array([[ 0.,  0.,  0.],\n           [ 4.,  0.,  0.],\n           [ 7.,  8.,  0.],\n           [10., 11., 12.]])\n    \"\"\"\n    return _mx_nd_np.tril(m, k)\n\n\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef tri(N, M=None, k=0, dtype=None, device=None):    # pylint: disable=redefined-outer-name\n    r\"\"\"\n    An array with ones at and below the given diagonal and zeros elsewhere.\n    Parameters\n    ----------\n    N : int\n        Number of rows in the array.\n    M : int, optional\n        Number of columns in the array.\n        By default, `M` is taken equal to `N`.\n    k : int, optional\n        The sub-diagonal at and below which the array is filled.\n        `k` = 0 is the main diagonal, while `k` < 0 is below it,\n        and `k` > 0 is above.  The default is 0.\n    dtype : dtype, optional\n        Data type of the returned array.  The default is float.\n    Returns\n    -------\n    tri : ndarray of shape (N, M)\n        Array with its lower triangle filled with ones and zero elsewhere;\n        in other words ``T[i,j] == 1`` for ``i <= j + k``, 0 otherwise.\n    Examples\n    --------\n    >>> np.tri(3, 5, 2, dtype=int)\n    array([[1, 1, 1, 0, 0],\n           [1, 1, 1, 1, 0],\n           [1, 1, 1, 1, 1]])\n    >>> np.tri(3, 5, -1)\n    array([[0.,  0.,  0.,  0.,  0.],\n           [1.,  0.,  0.,  0.,  0.],\n           [1.,  1.,  0.,  0.,  0.]])\n    \"\"\"\n    return _mx_nd_np.tri(N, M, k, dtype, device)\n\n\n@set_module('mxnet.numpy')\ndef triu_indices(n, k=0, m=None, device=None):    # pylint: disable=redefined-outer-name\n    r\"\"\"\n    Return the indices for the upper-triangle of an (n, m) array.\n    Parameters\n    ----------\n    n : int\n        The size of the arrays for which the returned indices will\n        be valid.\n    k : int, optional\n        Diagonal offset (see `triu` for details).\n    m : int, optional\n        .. versionadded:: 1.9.0\n        The column dimension of the arrays for which the returned\n        arrays will be valid.\n        By default `m` is taken equal to `n`.\n    Returns\n    -------\n    inds : tuple, shape(2) of ndarrays, shape(`n`)\n        The indices for the triangle. The returned tuple contains two arrays,\n        each with the indices along one dimension of the array.  Can be used\n        to slice a ndarray of shape(`n`, `n`).\n    See also\n    --------\n    tril_indices : similar function, for lower-triangular.\n    mask_indices : generic function accepting an arbitrary mask function.\n    triu, tril\n    Examples\n    --------\n    Compute two different sets of indices to access 4x4 arrays, one for the\n    upper triangular part starting at the main diagonal, and one starting two\n    diagonals further right:\n    >>> iu1 = np.triu_indices(4)\n    >>> iu2 = np.triu_indices(4, 2)\n    Here is how they can be used with a sample array:\n    >>> a = np.arange(16).reshape(4, 4)\n    >>> a\n    array([[ 0,  1,  2,  3],\n           [ 4,  5,  6,  7],\n           [ 8,  9, 10, 11],\n           [12, 13, 14, 15]])\n    Both for indexing:\n    >>> a[iu1]\n    array([ 0,  1,  2, ..., 10, 11, 15])\n    And for assigning values:\n    >>> a[iu1] = -1\n    >>> a\n    array([[-1, -1, -1, -1],\n           [ 4, -1, -1, -1],\n           [ 8,  9, -1, -1],\n           [12, 13, 14, -1]])\n    These cover only a small part of the whole array (two diagonals right\n    of the main one):\n    >>> a[iu2] = -10\n    >>> a\n    array([[ -1,  -1, -10, -10],\n           [  4,  -1,  -1, -10],\n           [  8,   9,  -1,  -1],\n           [ 12,  13,  14,  -1]])\n        \"\"\"\n    return _mx_nd_np.triu_indices(n, k, m, device)\n\n\n@set_module('mxnet.numpy')\ndef triu_indices_from(arr, k=0):\n    \"\"\"\n    Return the indices for the upper-triangle of arr.\n    See `triu_indices` for full details.\n    Parameters\n    ----------\n    arr : ndarray, shape(N, N)\n        The indices will be valid for square arrays.\n    k : int, optional\n        Diagonal offset (see `triu` for details).\n    Returns\n    -------\n    triu_indices_from : tuple, shape(2) of ndarray, shape(N)\n        Indices for the upper-triangle of `arr`.\n    See Also\n    --------\n    triu_indices, triu\n    \"\"\"\n    return _mx_nd_np.triu_indices_from(arr, k)\n\n\n@set_module('mxnet.numpy')\ndef tril_indices(n, k=0, m=None):\n    \"\"\"\n    Return the indices for the lower-triangle of an (n, m) array.\n\n    Parameters\n    ----------\n    n : int\n        The row dimension of the arrays for which the returned\n        indices will be valid.\n    k : int, optional\n        Diagonal offset (see `tril` for details).\n    m : int, optional\n        .. versionadded:: 1.9.0\n\n        The column dimension of the arrays for which the returned\n        arrays will be valid.\n        By default `m` is taken equal to `n`.\n\n    Returns\n    -------\n    inds : tuple of arrays\n        The indices for the triangle. The returned tuple contains two arrays,\n        each with the indices along one dimension of the array.\n\n    See also\n    --------\n    triu_indices : similar function, for upper-triangular.\n    mask_indices : generic function accepting an arbitrary mask function.\n    tril, triu\n\n    Examples\n    --------\n    Compute two different sets of indices to access 4x4 arrays, one for the\n    lower triangular part starting at the main diagonal, and one starting two\n    diagonals further right:\n\n    >>> il1 = np.tril_indices(4)\n    >>> il2 = np.tril_indices(4, 2)\n\n    Here is how they can be used with a sample array:\n\n    >>> a = np.arange(16).reshape(4, 4)\n    >>> a\n    array([[ 0,  1,  2,  3],\n           [ 4,  5,  6,  7],\n           [ 8,  9, 10, 11],\n           [12, 13, 14, 15]])\n\n    Both for indexing:\n\n    >>> a[il1]\n    array([ 0,  4,  5,  8,  9, 10, 12, 13, 14, 15])\n\n    And for assigning values:\n\n    >>> a[il1] = -1\n    >>> a\n    array([[-1,  1,  2,  3],\n           [-1, -1,  6,  7],\n           [-1, -1, -1, 11],\n           [-1, -1, -1, -1]])\n\n    These cover almost the whole array (two diagonals right of the main one):\n\n    >>> a[il2] = -10\n    >>> a\n    array([[-10, -10, -10,   3],\n           [-10, -10, -10, -10],\n           [-10, -10, -10, -10],\n           [-10, -10, -10, -10]])\n\n    \"\"\"\n    if m is None:\n        m = n\n    return _mx_nd_np.tril_indices(n, k, m)\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\ndef triu(m, k=0):\n    r\"\"\"\n    Upper triangle of an array.\n\n    Return a copy of a matrix with the elements below the `k`-th diagonal\n    zeroed.\n\n    Please refer to the documentation for `tril` for further details.\n\n    See Also\n    --------\n    tril : lower triangle of an array\n\n    Examples\n    --------\n    >>> np.triu(np.array([[1,2,3],[4,5,6],[7,8,9],[10,11,12]]), -1)\n    array([[ 1,  2,  3],\n           [ 4,  5,  6],\n           [ 0,  8,  9],\n           [ 0,  0, 12]])\n    \"\"\"\n    return _mx_nd_np.triu(m, k)\n\n\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef arange(start, stop=None, step=1, dtype=None, device=None):\n    \"\"\"Return evenly spaced values within a given interval.\n\n    Values are generated within the half-open interval ``[start, stop)``\n    (in other words, the interval including `start` but excluding `stop`).\n    For integer arguments the function is equivalent to the Python built-in\n    `range` function, but returns an ndarray rather than a list.\n\n    Parameters\n    ----------\n    start : number, optional\n        Start of interval. The interval includes this value.  The default\n        start value is 0.\n    stop : number\n        End of interval. The interval does not include this value, except\n        in some cases where `step` is not an integer and floating point\n        round-off affects the length of `out`.\n    step : number, optional\n        Spacing between values. For any output `out`, this is the distance\n        between two adjacent values, ``out[i+1] - out[i]``.  The default\n        step size is 1.  If `step` is specified as a position argument,\n        `start` must also be given.\n    dtype : dtype\n        The type of the output array.\n        Default dtype can be set to be consistent with offical numpy by `npx.set_np(dtype=True)`.\n        * When npx.is_np_default_dtype() returns False, default dtype is float32;\n        * When npx.is_np_default_dtype() returns True, default dtype is int64.\n    device : device context, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    arange : ndarray\n        Array of evenly spaced values.\n\n        For floating point arguments, the length of the result is\n        ``ceil((stop - start)/step)``.  Because of floating point overflow,\n        this rule may result in the last element of `out` being greater\n        than `stop`.\n\n    Examples\n    --------\n    >>> np.arange(3)\n    array([0., 1., 2.])\n\n    >>> np.arange(3.0)\n    array([0., 1., 2.])\n\n    >>> np.arange(3,7)\n    array([3., 4., 5., 6.])\n\n    >>> np.arange(3,7,2)\n    array([3., 5.])\n\n    >>> np.arange(3).dtype\n    dtype('float32')\n    >>> npx.set_np(dtype=True)\n    >>> np.arange(3).dtype\n    dtype('int64')\n    \"\"\"\n    return _mx_nd_np.arange(start, stop, step, dtype, device)\n# pylint: enable=redefined-outer-name\n\n\n@set_module('mxnet.numpy')\ndef split(ary, indices_or_sections, axis=0):\n    \"\"\"Split an array into multiple sub-arrays.\n\n    Parameters\n    ----------\n    ary : ndarray\n        Array to be divided into sub-arrays.\n    indices_or_sections : int or 1-D Python tuple, list or set.\n        If `indices_or_sections` is an integer, N, the array will be divided\n        into N equal arrays along `axis`.  If such a split is not possible,\n        an error is raised.\n        If `indices_or_sections` is a 1-D array of sorted integers, the entries\n        indicate where along `axis` the array is split.  For example,\n        ``[2, 3]`` would, for ``axis=0``, result in\n\n        * ary[:2]\n        * ary[2:3]\n        * ary[3:]\n\n        If an index exceeds the dimension of the array along `axis`,\n        an empty sub-array is returned correspondingly.\n    axis : int, optional\n        The axis along which to split, default is 0.\n\n    Returns\n    -------\n    sub-arrays : list of ndarrays\n        A list of sub-arrays.\n\n    Raises\n    ------\n    ValueError\n        If `indices_or_sections` is given as an integer, but\n        a split does not result in equal division.\n\n    See Also\n    --------\n    hsplit : Split array into multiple sub-arrays horizontally (column-wise).\n    vsplit : Split array into multiple sub-arrays vertically (row wise).\n    dsplit : Split array into multiple sub-arrays along the 3rd axis (depth).\n    concatenate : Join a sequence of arrays along an existing axis.\n    stack : Join a sequence of arrays along a new axis.\n    hstack : Stack arrays in sequence horizontally (column wise).\n    vstack : Stack arrays in sequence vertically (row wise).\n    dstack : Stack arrays in sequence depth wise (along third dimension).\n\n    Examples\n    --------\n    >>> x = np.arange(9.0)\n    >>> np.split(x, 3)\n    [array([0., 1., 2.]), array([3., 4., 5.]), array([6., 7., 8.])]\n\n    >>> np.split(x, [3, 5, 6, 8])\n    [array([0., 1., 2.]), array([3., 4.]), array([5.]), array([6., 7.]), array([])]\n    \"\"\"\n    return _mx_nd_np.split(ary, indices_or_sections, axis=axis)\n\n\n@set_module('mxnet.numpy')\ndef array_split(ary, indices_or_sections, axis=0):\n    \"\"\"Split an array into multiple sub-arrays.\n\n    If `indices_or_sections` is an integer, N, the array will be divided\n    into N equal arrays along `axis`.  If such a split is not possible,\n    an array of length l that should be split into n sections, it returns\n    l % n sub-arrays of size l//n + 1 and the rest of size l//n.\n\n    If `indices_or_sections` is a 1-D array of sorted integers, the entries\n    indicate where along `axis` the array is split.  For example, ``[2, 3]``\n    would, for ``axis=0``, result in\n    * ary[:2]\n    * ary[2:3]\n    * ary[3:]\n\n    If an index exceeds the dimension of the array along `axis`,\n    an empty sub-array is returned correspondingly.\n\n    Parameters\n    ----------\n    ary : ndarray\n        Array to be divided into sub-arrays.\n    indices_or_sections : int or 1-D Python tuple, list or set.\n        Param used to determine the number and size of the subarray.\n    axis : int, optional\n        The axis along which to split, default is 0.\n\n    Returns\n    -------\n    sub-arrays : list of ndarrays\n        A list of sub-arrays.\n\n    Examples\n    --------\n    >>> x = np.arange(9.0)\n    >>> np.array_split(x, 3)\n    [array([0., 1., 2.]), array([3., 4., 5.]), array([6., 7., 8.])]\n\n    >>> np.array_split(x, [3, 5, 6, 8])\n    [array([0., 1., 2.]), array([3., 4.]), array([5.]), array([6., 7.]), array([])]\n\n    >>> x = np.arange(8.0)\n    >>> np.array_split(x, 3)\n    [array([0.,  1.,  2.]), array([3.,  4.,  5.]), array([6.,  7.])]\n\n    >>> x = np.arange(7.0)\n    >>> np.array_split(x, 3)\n    [array([0.,  1.,  2.]), array([3.,  4.]), array([5.,  6.])]\n    \"\"\"\n    return _mx_nd_np.array_split(ary, indices_or_sections, axis=axis)\n\n\n@set_module('mxnet.numpy')\ndef vsplit(ary, indices_or_sections):\n    r\"\"\"Split an array into multiple sub-arrays vertically (row-wise).\n\n    ``vsplit`` is equivalent to ``split`` with `axis=0` (default): the array is always split\n    along the first axis regardless of the array dimension.\n\n    Parameters\n    ----------\n    ary : ndarray\n        Array to be divided into sub-arrays.\n    indices_or_sections : int or 1 - D Python tuple, list or set.\n        If `indices_or_sections` is an integer, N, the array will be divided into N equal arrays\n        along axis 0.  If such a split is not possible, an error is raised.\n\n        If `indices_or_sections` is a 1-D array of sorted integers, the entries indicate where\n        along axis 0 the array is split.  For example, ``[2, 3]`` would result in\n\n        * ary[:2]\n        * ary[2:3]\n        * ary[3:]\n\n        If an index exceeds the dimension of the array along axis 0, an error will be thrown.\n\n    Returns\n    -------\n    sub-arrays : list of ndarrays\n        A list of sub-arrays.\n\n    See Also\n    --------\n    split : Split an array into multiple sub-arrays of equal size.\n\n    .. note::\n       This function differs from the original `numpy.vsplit\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.vsplit.html>`_ in\n       the following aspects:\n\n       * Currently parameter ``indices_or_sections`` does not support ndarray, but supports scalar,\n         tuple and list.\n       * In ``indices_or_sections``, if an index exceeds the dimension of the array along axis 0,\n         an error will be thrown.\n\n\n    Examples\n    --------\n    >>> x = np.arange(16.0).reshape(4, 4)\n    >>> x\n    array([[  0.,   1.,   2.,   3.],\n           [  4.,   5.,   6.,   7.],\n           [  8.,   9.,  10.,  11.],\n           [ 12.,  13.,  14.,  15.]])\n    >>> np.vsplit(x, 2)\n    [array([[0., 1., 2., 3.],\n            [4., 5., 6., 7.]]), array([[ 8.,  9., 10., 11.],\n            [12., 13., 14., 15.]])]\n\n    >>> # With a higher dimensional array the split is still along the first axis.\n    >>> x = np.arange(8.0).reshape(2, 2, 2)\n    >>> x\n    array([[[ 0.,  1.],\n            [ 2.,  3.]],\n           [[ 4.,  5.],\n            [ 6.,  7.]]])\n    >>> np.vsplit(x, 2)\n    [array([[[0., 1.],\n            [2., 3.]]]), array([[[4., 5.],\n            [6., 7.]]])]\n\n    \"\"\"\n    return _mx_nd_np.vsplit(ary, indices_or_sections)\n\n\n@set_module('mxnet.numpy')\ndef dsplit(ary, indices_or_sections):\n    r\"\"\"\n    Split array into multiple sub-arrays along the 3rd axis (depth).\n    Please refer to the `split` documentation.  `dsplit` is equivalent\n    to `split` with ``axis=2``, the array is always split along the third\n    axis provided the array dimension is greater than or equal to 3.\n\n    Parameters\n    ----------\n    ary : ndarray\n        Array to be divided into sub-arrays.\n    indices_or_sections : int or 1 - D Python tuple, list or set.\n        If `indices_or_sections` is an integer, N, the array will be divided into N equal arrays\n        along axis 2.  If such a split is not possible, an error is raised.\n\n        If `indices_or_sections` is a 1-D array of sorted integers, the entries indicate where\n        along axis 2 the array is split.  For example, ``[2, 3]`` would result in\n\n        * ary[:, :, :2]\n        * ary[:, :, 2:3]\n        * ary[:, :, 3:]\n\n        If an index exceeds the dimension of the array along axis 2, an error will be thrown.\n\n    Returns\n    -------\n    sub-arrays : list of ndarrays\n        A list of sub-arrays.\n\n    See Also\n    --------\n    split : Split an array into multiple sub-arrays of equal size.\n\n    .. note::\n       This function differs from the original `numpy.dsplit\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.dsplit.html>`_ in\n       the following aspects:\n       * Currently parameter ``indices_or_sections`` does not support ndarray, but supports scalar,\n       tuple and list.\n       * In ``indices_or_sections``, if an index exceeds the dimension of the array along axis 2,\n       an error will be thrown.\n\n    Examples\n    --------\n    >>> x = np.arange(16.0).reshape(2, 2, 4)\n    >>> x\n    array([[[ 0.,   1.,   2.,   3.],\n            [ 4.,   5.,   6.,   7.]],\n           [[ 8.,   9.,  10.,  11.],\n            [12.,  13.,  14.,  15.]]])\n    >>> np.dsplit(x, 2)\n    [array([[[ 0.,  1.],\n            [ 4.,  5.]],\n           [[ 8.,  9.],\n            [12., 13.]]]), array([[[ 2.,  3.],\n            [ 6.,  7.]],\n           [[10., 11.],\n            [14., 15.]]])]\n    >>> np.dsplit(x, np.array([3, 6]))\n    [array([[[ 0.,   1.,   2.],\n            [ 4.,   5.,   6.]],\n           [[ 8.,   9.,  10.],\n            [12.,  13.,  14.]]]),\n     array([[[ 3.],\n            [ 7.]],\n           [[11.],\n            [15.]]]),\n    array([], shape=(2, 2, 0), dtype=float64)]\n\n    \"\"\"\n    return _mx_nd_np.dsplit(ary, indices_or_sections)\n\n@set_module('mxnet.numpy')\ndef concat(seq, axis=0, out=None):\n    \"\"\"Join a sequence of arrays along an existing axis.\n\n    Parameters\n    ----------\n    a1, a2, ... : sequence of array_like\n        The arrays must have the same shape, except in the dimension\n        corresponding to `axis` (the first, by default).\n    axis : int, optional\n        The axis along which the arrays will be joined.  If axis is None,\n        arrays are flattened before use.  Default is 0.\n    out : ndarray, optional\n        If provided, the destination to place the result. The shape must be\n        correct, matching that of what concatenate would have returned if no\n        out argument were specified.\n\n    Returns\n    -------\n    res : ndarray\n        The concatenated array.\n\n    Note\n    --------\n    `concate` is a alias for `concatante`. It is a standard API in\n    https://data-apis.org/array-api/latest/API_specification/manipulation_functions.html#concat-arrays-axis-0\n    instead of an official NumPy operator.\n\n    See Also\n    --------\n    split : Split array into a list of multiple sub-arrays of equal size.\n    hsplit : Split array into multiple sub-arrays horizontally (column wise)\n    vsplit : Split array into multiple sub-arrays vertically (row wise)\n    dsplit : Split array into multiple sub-arrays along the 3rd axis (depth).\n    stack : Stack a sequence of arrays along a new axis.\n    hstack : Stack arrays in sequence horizontally (column wise)\n    vstack : Stack arrays in sequence vertically (row wise)\n    dstack : Stack arrays in sequence depth wise (along third dimension)\n\n    Examples\n    --------\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> b = np.array([[5, 6]])\n    >>> np.concat((a, b), axis=0)\n    array([[1., 2.],\n           [3., 4.],\n           [5., 6.]])\n\n    >>> np.concat((a, b.T), axis=1)\n    array([[1., 2., 5.],\n           [3., 4., 6.]])\n\n    >>> np.concat((a, b), axis=None)\n    array([1., 2., 3., 4., 5., 6.])\n    \"\"\"\n    return _mx_nd_np.concatenate(seq, axis=axis, out=out)\n\n@set_module('mxnet.numpy')\ndef concatenate(seq, axis=0, out=None):\n    \"\"\"Join a sequence of arrays along an existing axis.\n\n    Parameters\n    ----------\n    a1, a2, ... : sequence of array_like\n        The arrays must have the same shape, except in the dimension\n        corresponding to `axis` (the first, by default).\n    axis : int, optional\n        The axis along which the arrays will be joined.  If axis is None,\n        arrays are flattened before use.  Default is 0.\n    out : ndarray, optional\n        If provided, the destination to place the result. The shape must be\n        correct, matching that of what concatenate would have returned if no\n        out argument were specified.\n\n    Returns\n    -------\n    res : ndarray\n        The concatenated array.\n\n    See Also\n    --------\n    split : Split array into a list of multiple sub-arrays of equal size.\n    hsplit : Split array into multiple sub-arrays horizontally (column wise)\n    vsplit : Split array into multiple sub-arrays vertically (row wise)\n    dsplit : Split array into multiple sub-arrays along the 3rd axis (depth).\n    stack : Stack a sequence of arrays along a new axis.\n    hstack : Stack arrays in sequence horizontally (column wise)\n    vstack : Stack arrays in sequence vertically (row wise)\n    dstack : Stack arrays in sequence depth wise (along third dimension)\n\n    Examples\n    --------\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> b = np.array([[5, 6]])\n    >>> np.concatenate((a, b), axis=0)\n    array([[1., 2.],\n           [3., 4.],\n           [5., 6.]])\n\n    >>> np.concatenate((a, b.T), axis=1)\n    array([[1., 2., 5.],\n           [3., 4., 6.]])\n\n    >>> np.concatenate((a, b), axis=None)\n    array([1., 2., 3., 4., 5., 6.])\n    \"\"\"\n    return _mx_nd_np.concatenate(seq, axis=axis, out=out)\n\n\n@set_module('mxnet.numpy')\ndef append(arr, values, axis=None):  # pylint: disable=redefined-outer-name\n    \"\"\"\n    Append values to the end of an array.\n\n    Parameters\n    ----------\n    arr : ndarray\n        Values are appended to a copy of this array.\n    values : ndarray\n        These values are appended to a copy of `arr`.  It must be of the\n        correct shape (the same shape as `arr`, excluding `axis`).  If\n        `axis` is not specified, `values` can be any shape and will be\n        flattened before use.\n    axis : int, optional\n        The axis along which `values` are appended.  If `axis` is not\n        given, both `arr` and `values` are flattened before use.\n\n    Returns\n    -------\n    append : ndarray\n        A copy of `arr` with `values` appended to `axis`.  Note that\n        `append` does not occur in-place: a new array is allocated and\n        filled.  If `axis` is None, `out` is a flattened array.\n\n    Examples\n    --------\n    >>> np.append(np.array([1, 2, 3]), np.array([[4, 5, 6],[7, 8, 9]]))\n    array([1., 2., 3., 4., 5., 6., 7., 8., 9.])\n\n    When `axis` is specified, `values` must have the correct shape.\n\n    >>> np.append(np.array([[1, 2, 3], [4, 5, 6]]), np.array([[7, 8, 9]]), axis=0)\n    array([[1., 2., 3.],\n           [4., 5., 6.],\n           [7., 8., 9.]])\n    \"\"\"\n    return _mx_nd_np.append(arr, values, axis=axis)\n\n\n@set_module('mxnet.numpy')\ndef stack(arrays, axis=0, out=None):\n    \"\"\"Join a sequence of arrays along a new axis.\n        The axis parameter specifies the index of the new axis in the dimensions of the result.\n        For example, if `axis=0` it will be the first dimension and if `axis=-1` it will be the last dimension.\n\n    Parameters\n    ----------\n    arrays : sequence of array_like\n        Each array must have the same shape.\n    axis : int, optional\n        The axis in the result array along which the input arrays are stacked.\n    out : ndarray, optional\n        If provided, the destination to place the result. The shape must be correct,\n        matching that of what stack would have returned if no out argument were specified.\n\n    Returns\n    -------\n    stacked : ndarray\n        The stacked array has one more dimension than the input arrays.\n\n    See Also\n    --------\n    concatenate : Join a sequence of arrays along an existing axis.\n    split : Split array into a list of multiple sub-arrays of equal size.\n\n    Examples\n    --------\n    >>> arrays = [np.random.rand(3, 4) for _ in range(10)]\n    >>> np.stack(arrays, axis=0).shape\n    (10, 3, 4)\n\n    >>> np.stack(arrays, axis=1).shape\n    (3, 10, 4)\n\n    >>> np.stack(arrays, axis=2).shape\n    (3, 4, 10)\n\n    >>> a = np.array([1, 2, 3])\n    >>> b = np.array([2, 3, 4])\n    >>> np.stack((a, b))\n    array([[1., 2., 3.],\n           [2., 3., 4.]])\n\n    >>> np.stack((a, b), axis=-1)\n    array([[1., 2.],\n           [2., 3.],\n           [3., 4.]])\n    \"\"\"\n    return _mx_nd_np.stack(arrays, axis=axis, out=out)\n\n\n@set_module('mxnet.numpy')\ndef vstack(arrays, out=None):\n    r\"\"\"Stack arrays in sequence vertically (row wise).\n\n    This is equivalent to concatenation along the first axis after 1-D arrays\n    of shape `(N,)` have been reshaped to `(1,N)`. Rebuilds arrays divided by\n    `vsplit`.\n\n    This function makes most sense for arrays with up to 3 dimensions. For\n    instance, for pixel-data with a height (first axis), width (second axis),\n    and r/g/b channels (third axis). The functions `concatenate` and `stack`\n    provide more general stacking and concatenation operations.\n\n    Parameters\n    ----------\n    tup : sequence of ndarrays\n        The arrays must have the same shape along all but the first axis.\n        1-D arrays must have the same length.\n\n    Returns\n    -------\n    stacked : ndarray\n        The array formed by stacking the given arrays, will be at least 2-D.\n\n    Examples\n    --------\n    >>> a = np.array([1, 2, 3])\n    >>> b = np.array([2, 3, 4])\n    >>> np.vstack((a, b))\n    array([[1., 2., 3.],\n           [2., 3., 4.]])\n\n    >>> a = np.array([[1], [2], [3]])\n    >>> b = np.array([[2], [3], [4]])\n    >>> np.vstack((a, b))\n    array([[1.],\n           [2.],\n           [3.],\n           [2.],\n           [3.],\n           [4.]])\n    \"\"\"\n    return _mx_nd_np.vstack(arrays)\n\n\n@set_module('mxnet.numpy')\ndef row_stack(arrays):\n    r\"\"\"Stack arrays in sequence vertically (row wise).\n    This is equivalent to concatenation along the first axis after 1-D arrays\n    of shape `(N,)` have been reshaped to `(1,N)`. Rebuilds arrays divided by\n    `vsplit`.\n    This function makes most sense for arrays with up to 3 dimensions. For\n    instance, for pixel-data with a height (first axis), width (second axis),\n    and r/g/b channels (third axis). The functions `concatenate` and `stack`\n    provide more general stacking and concatenation operations.\n    Parameters\n    ----------\n    tup : sequence of ndarrays\n        The arrays must have the same shape along all but the first axis.\n        1-D arrays must have the same length.\n    Returns\n    -------\n    stacked : ndarray\n        The array formed by stacking the given arrays, will be at least 2-D.\n    Examples\n    --------\n    >>> a = np.array([1, 2, 3])\n    >>> b = np.array([2, 3, 4])\n    >>> np.vstack((a, b))\n    array([[1., 2., 3.],\n           [2., 3., 4.]])\n    >>> a = np.array([[1], [2], [3]])\n    >>> b = np.array([[2], [3], [4]])\n    >>> np.vstack((a, b))\n    array([[1.],\n           [2.],\n           [3.],\n           [2.],\n           [3.],\n           [4.]])\n    \"\"\"\n    return _mx_nd_np.row_stack(arrays)\n\n\n@set_module('mxnet.numpy')\ndef column_stack(tup):\n    \"\"\"\n    Stack 1-D arrays as columns into a 2-D array.\n\n    Take a sequence of 1-D arrays and stack them as columns\n    to make a single 2-D array. 2-D arrays are stacked as-is,\n    just like with `hstack`.  1-D arrays are turned into 2-D columns\n    first.\n\n    Parameters\n    ----------\n    tup : sequence of 1-D or 2-D arrays.\n        Arrays to stack. All of them must have the same first dimension.\n\n    Returns\n    --------\n    stacked : 2-D array\n        The array formed by stacking the given arrays.\n\n    See Also\n    --------\n    stack, hstack, vstack, concatenate\n\n    Examples\n    --------\n    >>> a = np.array((1,2,3))\n    >>> b = np.array((2,3,4))\n    >>> np.column_stack((a,b))\n    array([[1., 2.],\n           [2., 3.],\n           [3., 4.]])\n    \"\"\"\n    return _mx_nd_np.column_stack(tup)\n\n\n@set_module('mxnet.numpy')\ndef hstack(arrays):\n    \"\"\"\n    Stack arrays in sequence horizontally (column wise).\n    This is equivalent to concatenation along the second axis,\n    except for 1-D arrays where it concatenates along the first axis.\n    Rebuilds arrays divided by hsplit.\n    This function makes most sense for arrays with up to 3 dimensions.\n    For instance, for pixel-data with a height (first axis), width (second axis),\n    and r/g/b channels (third axis). The functions concatenate,\n    stack and block provide more general stacking and concatenation operations.\n\n    Parameters\n    ----------\n    tup : sequence of ndarrays\n        The arrays must have the same shape along all but the second axis, except 1-D arrays which can be any length.\n\n    Returns\n    -------\n    stacked : ndarray\n        The array formed by stacking the given arrays.\n\n    Examples\n    --------\n    >>> from mxnet import np,npx\n    >>> a = np.array((1,2,3))\n    >>> b = np.array((2,3,4))\n    >>> np.hstack((a,b))\n    array([1., 2., 3., 2., 3., 4.])\n    >>> a = np.array([[1],[2],[3]])\n    >>> b = np.array([[2],[3],[4]])\n    >>> np.hstack((a,b))\n    array([[1., 2.],\n           [2., 3.],\n           [3., 4.]])\n    \"\"\"\n    return _mx_nd_np.hstack(arrays)\n\n\n@set_module('mxnet.numpy')\ndef dstack(arrays):\n    \"\"\"\n    Stack arrays in sequence depth wise (along third axis).\n\n    This is equivalent to concatenation along the third axis after 2-D arrays\n    of shape `(M,N)` have been reshaped to `(M,N,1)` and 1-D arrays of shape\n    `(N,)` have been reshaped to `(1,N,1)`. Rebuilds arrays divided by\n    `dsplit`.\n\n    This function makes most sense for arrays with up to 3 dimensions. For\n    instance, for pixel-data with a height (first axis), width (second axis),\n    and r/g/b channels (third axis). The functions `concatenate`, `stack` and\n    `block` provide more general stacking and concatenation operations.\n\n    Parameters\n    ----------\n    tup : sequence of arrays\n        The arrays must have the same shape along all but the third axis.\n        1-D or 2-D arrays must have the same shape.\n\n    Returns\n    -------\n    stacked : ndarray\n        The array formed by stacking the given arrays, will be at least 3-D.\n\n    Examples\n    --------\n    >>> a = np.array((1,2,3))\n    >>> b = np.array((2,3,4))\n    >>> np.dstack((a,b))\n    array([[[1, 2],\n            [2, 3],\n            [3, 4]]])\n    >>> a = np.array([[1],[2],[3]])\n    >>> b = np.array([[2],[3],[4]])\n    >>> np.dstack((a,b))\n    array([[[1, 2]],\n           [[2, 3]],\n           [[3, 4]]])\n    \"\"\"\n    return _npi.dstack(*arrays)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef maximum(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns element-wise maximum of the input arrays with broadcasting.\n\n    Parameters\n    ----------\n    x1, x2 : scalar or mxnet.numpy.ndarray\n        The arrays holding the elements to be compared. They must have the same shape,\n        or shapes that can be broadcast to a single shape.\n\n    Returns\n    -------\n    out : mxnet.numpy.ndarray or scalar\n        The maximum of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars.\n\n    Examples\n    --------\n    >>> np.maximum(np.array([2, 3, 4]), np.array([1, 5, 2]))\n    array([2., 5., 4.])\n\n    >>> np.maximum(np.eye(2), np.array([0.5, 2])) # broadcasting\n    array([[1. , 2. ],\n           [0.5, 2. ]])\n    \"\"\"\n    return _mx_nd_np.maximum(x1, x2, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef fmax(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns element-wise maximum of the input arrays with broadcasting. (Ignores NaNs)\n\n    Parameters\n    ----------\n    x1, x2 : scalar or mxnet.numpy.ndarray\n        The arrays holding the elements to be compared. They must have the same shape,\n        or shapes that can be broadcast to a single shape.\n\n    Returns\n    -------\n    out : mxnet.numpy.ndarray or scalar\n        The maximum of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars.\n\n    Examples\n    --------\n    >>> np.fmax(np.array([2, 3, 4]), np.array([1, 5, 2]))\n    array([2., 5., 4.])\n\n    >>> np.fmax(np.eye(2), np.array([0.5, 2])) # broadcasting\n    array([[1. , 2. ],\n           [0.5, 2. ]])\n    \"\"\"\n    return _mx_nd_np.fmax(x1, x2, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef minimum(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns element-wise minimum of the input arrays with broadcasting.\n\n    Parameters\n    ----------\n    x1, x2 : scalar or mxnet.numpy.ndarray\n        The arrays holding the elements to be compared. They must have the same shape,\n        or shapes that can be broadcast to a single shape.\n\n    Returns\n    -------\n    out : mxnet.numpy.ndarray or scalar\n        The minimum of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars.\n\n    Examples\n    --------\n    >>> np.minimum(np.array([2, 3, 4]), np.array([1, 5, 2]))\n    array([1., 3., 2.])\n\n    >>> np.minimum(np.eye(2), np.array([0.5, 2])) # broadcasting\n    array([[0.5, 0. ],\n           [0. , 1. ]])\n    \"\"\"\n    return _mx_nd_np.minimum(x1, x2, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef fmin(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns element-wise minimum of the input arrays with broadcasting. (Ignores NaNs)\n\n    Parameters\n    ----------\n    x1, x2 : scalar or mxnet.numpy.ndarray\n        The arrays holding the elements to be compared. They must have the same shape,\n        or shapes that can be broadcast to a single shape.\n\n    Returns\n    -------\n    out : mxnet.numpy.ndarray or scalar\n        The fmin of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars.\n\n    Examples\n    --------\n    >>> np.fmin(np.array([2, 3, 4]), np.array([1, 5, 2]))\n    array([1., 3., 2.])\n\n    >>> np.fmin(np.eye(2), np.array([0.5, 2])) # broadcasting\n    array([[0.5, 0. ],\n           [0. , 1. ]])\n    \"\"\"\n    return _mx_nd_np.fmin(x1, x2, out=out)\n\n\n@set_module('mxnet.numpy')\ndef max(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Return the maximum of an array or maximum along an axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data.\n    axis : int, optional\n        Axis along which to operate.  By default, flattened input is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result.  Must\n        be of the same shape and buffer length as the expected output.\n        See `doc.ufuncs` (Section \"Output arguments\") for more details.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the original `arr`.\n\n    Returns\n    -------\n    max : ndarray\n        Maximum of `a`. If `axis` is None, the result is an array of dimension 1.\n        If `axis` is given, the result is an array of dimension\n        ``a.ndim - 1``.\n\n    See Also\n    --------\n    min :\n        The minimum value of an array along a given axis, ignoring any nan.\n    maximum :\n        Element-wise maximum of two arrays, ignoring any nan.\n    argmax :\n        Return the indices of the maximum values.\n\n    Notes\n    -----\n    NaN in the orginal `numpy` is denoted as nan and will be ignored.\n\n    Don't use `max` for element-wise comparison of 2 arrays; when\n    ``a.shape[0]`` is 2, ``maximum(a[0], a[1])`` is faster than\n    ``max(a, axis=0)``.\n\n    Examples\n    --------\n    >>> a = np.arange(4).reshape((2,2))\n    >>> a\n    array([[0., 1.],\n        [2., 3.]])\n    >>> np.max(a)            # Maximum of the flattened array\n    array(3.)\n    >>> np.max(a, axis=0)    # Maxima along the first axis\n    array([2., 3.])\n    >>> np.max(a, axis=1)    # Maxima along the second axis\n    array([1., 3.])\n\n    >>> b = np.arange(5, dtype=np.float32)\n    >>> b[2] = np.nan\n    >>> np.max(b)\n    array(4.)\n    \"\"\"\n    return _mx_nd_np.max(a, axis=axis, out=out, keepdims=keepdims)\n\n\n@set_module('mxnet.numpy')\ndef min(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Return the minimum of an array or minimum along an axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data.\n    axis : int, optional\n        Axis along which to operate.  By default, flattened input is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result.  Must\n        be of the same shape and buffer length as the expected output.\n        See `doc.ufuncs` (Section \"Output arguments\") for more details.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the original `arr`.\n\n    Returns\n    -------\n    min : ndarray\n        Minimum of `a`. If `axis` is None, the result is an array of dimension 1.\n        If `axis` is given, the result is an array of dimension\n        ``a.ndim - 1``.\n\n    See Also\n    --------\n    max :\n        The maximum value of an array along a given axis, ignoring any nan.\n    minimum :\n        Element-wise minimum of two arrays, ignoring any nan.\n\n    Notes\n    -----\n    NaN in the orginal `numpy` is denoted as nan and will be ignored.\n\n    Don't use `min` for element-wise comparison of 2 arrays; when\n    ``a.shape[0]`` is 2, ``minimum(a[0], a[1])`` is faster than\n    ``min(a, axis=0)``.\n\n    Examples\n    --------\n    >>> a = np.arange(4).reshape((2,2))\n    >>> a\n    array([[0., 1.],\n        [2., 3.]])\n    >>> np.min(a)           # Minimum of the flattened array\n    array(0.)\n    >>> np.min(a, axis=0)   # Minima along the first axis\n    array([0., 1.])\n    >>> np.min(a, axis=1)   # Minima along the second axis\n    array([0., 2.])\n    >>> b = np.arange(5, dtype=np.float32)\n    >>> b[2] = np.nan\n    >>> np.min(b)\n    array(0.) # nan will be ignored\n    \"\"\"\n    return _mx_nd_np.min(a, axis=axis, out=out, keepdims=keepdims)\n\n\n@set_module('mxnet.numpy')\ndef swapaxes(a, axis1, axis2):\n    \"\"\"Interchange two axes of an array.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array.\n    axis1 : int\n        First axis.\n    axis2 : int\n        Second axis.\n\n    Returns\n    -------\n    a_swapped : ndarray\n        Swapped array. This is always a copy of the input array.\n\n    Examples\n    --------\n    >>> x = np.array([[1,2,3]])\n    >>> np.swapaxes(x,0,1)\n    array([[1.],\n           [2.],\n           [3.]])\n\n    >>> x = np.array([[[0,1],[2,3]],[[4,5],[6,7]]])\n    >>> x\n    array([[[0., 1.],\n            [2., 3.]],\n\n           [[4., 5.],\n            [6., 7.]]])\n\n    >>> np.swapaxes(x,0,2)\n    array([[[0., 4.],\n            [2., 6.]],\n\n           [[1., 5.],\n            [3., 7.]]])\n    \"\"\"\n    return _npi.swapaxes(a, dim1=axis1, dim2=axis2)\n\n\n@set_module('mxnet.numpy')\ndef clip(a, a_min, a_max, out=None):\n    \"\"\"clip(a, a_min, a_max, out=None)\n\n    Clip (limit) the values in an array.\n    Given an interval, values outside the interval are clipped to\n    the interval edges.  For example, if an interval of ``[0, 1]``\n    is specified, values smaller than 0 become 0, and values larger\n    than 1 become 1.\n\n    Parameters\n    ----------\n    a : ndarray\n        Array containing elements to clip.\n    a_min : scalar or `None`\n        Minimum value. If `None`, clipping is not performed on lower\n        interval edge. Not more than one of `a_min` and `a_max` may be\n        `None`.\n    a_max : scalar or `None`\n        Maximum value. If `None`, clipping is not performed on upper\n        interval edge. Not more than one of `a_min` and `a_max` may be\n        `None`.\n    out : ndarray, optional\n        The results will be placed in this array. It may be the input\n        array for in-place clipping.  `out` must be of the right shape\n        to hold the output.  Its type is preserved.\n\n    Returns\n    -------\n    clipped_array : ndarray\n        An array with the elements of `a`, but where values\n        < `a_min` are replaced with `a_min`, and those > `a_max`\n        with `a_max`.\n\n    Notes\n    -----\n    array_like `a_min` and `a_max` are not supported.\n\n    Examples\n    --------\n    >>> a = np.arange(10)\n    >>> np.clip(a, 1, 8)\n    array([1., 1., 2., 3., 4., 5., 6., 7., 8., 8.])\n    >>> a\n    array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.])\n    >>> np.clip(a, 3, 6, out=a)\n    array([3., 3., 3., 3., 4., 5., 6., 6., 6., 6.])\n    \"\"\"\n    from numbers import Number\n    if isinstance(a, Number):\n        # In case input is a scalar, the computation would fall back to native numpy.\n        # The value returned would be a python scalar.\n        return _np.clip(a, a_min, a_max, out=None)\n    return _mx_nd_np.clip(a, a_min, a_max, out=out)\n\n\n@set_module('mxnet.numpy')\ndef argmax(a, axis=None, out=None, keepdims=False):\n    r\"\"\"\n    Returns the indices of the maximum values along an axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array. Only support ndarrays of dtype `float16`, `float32`, and `float64`.\n    axis : int, optional\n        By default, the index is into the flattened array, otherwise\n        along the specified axis.\n    out : ndarray or None, optional\n        If provided, the result will be inserted into this array. It should\n        be of the appropriate shape and dtype.\n    keepdims : bool\n        If True, the reduced axes (dimensions) must be included in the result as\n        singleton dimensions, and, accordingly, the result must be compatible with\n        the input array. Otherwise, if False, the reduced axes (dimensions) must\n        not be included in the result. Default: False .\n\n    Returns\n    -------\n    index_array : ndarray of indices whose dtype is same as the input ndarray.\n        Array of indices into the array. It has the same shape as `a.shape`\n        with the dimension along `axis` removed.\n\n    .. note::\n       ``keepdims`` param is part of request in data-api-standard\n       <https://data-apis.org/array-api/latest/API_specification/generated/signatures.searching_functions.argmax.html>`_,\n       which is not the parameter in official NumPy\n\n       In case of multiple occurrences of the maximum values, the indices\n       corresponding to the first occurrence are returned.\n\n       This function differs from the original `numpy.argmax\n       <https://numpy.org/doc/stable/reference/generated/numpy.argmax.html>`_ in\n       the following aspects:\n\n       * Input type does not support Python native iterables(list, tuple, ...).\n       * ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be\n         the same as the expected output.\n       * ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the\n         same as the expected output.\n       * ``out`` param does not support scalar input case.\n\n    Examples\n    --------\n    >>> a = np.arange(6).reshape(2,3) + 10\n    >>> a\n    array([[10., 11., 12.],\n           [13., 14., 15.]])\n    >>> np.argmax(a)\n    array(5.)\n    >>> np.argmax(a, axis=0)\n    array([1., 1., 1.])\n    >>> np.argmax(a, axis=1)\n    array([2., 2.])\n\n    >>> b = np.arange(6)\n    >>> b[1] = 5\n    >>> b\n    array([0., 5., 2., 3., 4., 5.])\n    >>> np.argmax(b)  # Only the first occurrence is returned.\n    array(1.)\n\n    Specify ``out`` ndarray:\n\n    >>> a = np.arange(6).reshape(2,3) + 10\n    >>> b = np.zeros((2,))\n    >>> np.argmax(a, axis=1, out=b)\n    array([2., 2.])\n    >>> b\n    array([2., 2.])\n    \"\"\"\n    return _mx_nd_np.argmax(a, axis, out, keepdims)\n\n\n@set_module('mxnet.numpy')\ndef argmin(a, axis=None, out=None, keepdims=False):\n    r\"\"\"\n    Returns the indices of the minimum values along an axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array. Only support ndarrays of dtype `float16`, `float32`, and `float64`.\n    axis : int, optional\n        By default, the index is into the flattened array, otherwise\n        along the specified axis.\n    out : ndarray or None, optional\n        If provided, the result will be inserted into this array. It should\n        be of the appropriate shape and dtype.\n    keepdims : bool\n        If True, the reduced axes (dimensions) must be included in the result as\n        singleton dimensions, and, accordingly, the result must be compatible with\n        the input array. Otherwise, if False, the reduced axes (dimensions) must\n        not be included in the result. Default: False .\n\n    Returns\n    -------\n    index_array : ndarray of indices whose dtype is same as the input ndarray.\n        Array of indices into the array. It has the same shape as `a.shape`\n        with the dimension along `axis` removed.\n\n    .. note::\n       ``keepdims`` param is part of request in data-api-standard\n       <https://data-apis.org/array-api/latest/API_specification/generated/signatures.searching_functions.argmin.html>`_,\n       which is not the parameter in official NumPy\n\n       In case of multiple occurrences of the minimum values, the indices\n       corresponding to the first occurrence are returned.\n\n       This function differs from the original `numpy.argmin\n       <https://numpy.org/doc/stable/reference/generated/numpy.argmin.html>`_ in\n       the following aspects:\n\n       * Input type does not support Python native iterables(list, tuple, ...).\n       * ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be\n         the same as the expected output.\n       * ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the\n         same as the expected output.\n       * ``out`` param does not support scalar input case.\n\n    Examples\n    --------\n    >>> a = np.arange(6).reshape(2,3) + 10\n    >>> a\n    array([[10., 11., 12.],\n           [13., 14., 15.]])\n    >>> np.argmin(a)\n    array(0.)\n    >>> np.argmin(a, axis=0)\n    array([0., 0., 0.])\n    >>> np.argmin(a, axis=1)\n    array([0., 0.])\n\n    >>> b = np.arange(6)\n    >>> b[2] = 0\n    >>> b\n    array([0., 1., 0., 3., 4., 5.])\n    >>> np.argmax(b)  # Only the first occurrence is returned.\n    array(0.)\n\n    Specify ``out`` ndarray:\n\n    >>> a = np.arange(6).reshape(2,3) + 10\n    >>> b = np.zeros((2,))\n    >>> np.argmin(a, axis=1, out=b)\n    array([0., 0.])\n    >>> b\n    array([0., 0.])\n    \"\"\"\n    return _mx_nd_np.argmin(a, axis, out, keepdims)\n\n\n@set_module('mxnet.numpy')\ndef amax(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Return the maximum of an array or maximum along an axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data.\n    axis : int, optional\n        Axis along which to operate.  By default, flattened input is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result.  Must\n        be of the same shape and buffer length as the expected output.\n        See `doc.ufuncs` (Section \"Output arguments\") for more details.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the original `arr`.\n\n    Returns\n    -------\n    max : ndarray\n        Maximum of `a`. If `axis` is None, the result is an array of dimension 1.\n        If `axis` is given, the result is an array of dimension\n        ``a.ndim - 1``.\n\n    See Also\n    --------\n    min :\n        The minimum value of an array along a given axis, ignoring any nan.\n    maximum :\n        Element-wise maximum of two arrays, ignoring any nan.\n    argmax :\n        Return the indices of the maximum values.\n\n    Notes\n    -----\n    NaN in the orginal `numpy` is denoted as nan and will be ignored.\n\n    Don't use `max` for element-wise comparison of 2 arrays; when\n    ``a.shape[0]`` is 2, ``maximum(a[0], a[1])`` is faster than\n    ``max(a, axis=0)``.\n\n    Examples\n    --------\n    >>> a = np.arange(4).reshape((2,2))\n    >>> a\n    array([[0., 1.],\n        [2., 3.]])\n    >>> np.max(a)            # Maximum of the flattened array\n    array(3.)\n    >>> np.max(a, axis=0)    # Maxima along the first axis\n    array([2., 3.])\n    >>> np.max(a, axis=1)    # Maxima along the second axis\n    array([1., 3.])\n\n    >>> b = np.arange(5, dtype=np.float32)\n    >>> b[2] = np.nan\n    >>> np.max(b)\n    array(4.)\n    \"\"\"\n    return _mx_nd_np.amax(a, axis=axis, out=out, keepdims=keepdims)\n\n\n@set_module('mxnet.numpy')\ndef amin(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Return the minimum of an array or minimum along an axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data.\n    axis : int, optional\n        Axis along which to operate.  By default, flattened input is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result.  Must\n        be of the same shape and buffer length as the expected output.\n        See `doc.ufuncs` (Section \"Output arguments\") for more details.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the original `arr`.\n\n    Returns\n    -------\n    min : ndarray\n        Minimum of `a`. If `axis` is None, the result is an array of dimension 1.\n        If `axis` is given, the result is an array of dimension\n        ``a.ndim - 1``.\n\n    See Also\n    --------\n    max :\n        The maximum value of an array along a given axis, ignoring any nan.\n    minimum :\n        Element-wise minimum of two arrays, ignoring any nan.\n\n    Notes\n    -----\n    NaN in the orginal `numpy` is denoted as nan and will be ignored.\n\n    Don't use `min` for element-wise comparison of 2 arrays; when\n    ``a.shape[0]`` is 2, ``minimum(a[0], a[1])`` is faster than\n    ``min(a, axis=0)``.\n\n    Examples\n    --------\n    >>> a = np.arange(4).reshape((2,2))\n    >>> a\n    array([[0., 1.],\n        [2., 3.]])\n    >>> np.min(a)           # Minimum of the flattened array\n    array(0.)\n    >>> np.min(a, axis=0)   # Minima along the first axis\n    array([0., 1.])\n    >>> np.min(a, axis=1)   # Minima along the second axis\n    array([0., 2.])\n    >>> b = np.arange(5, dtype=np.float32)\n    >>> b[2] = np.nan\n    >>> np.min(b)\n    array(0.) # nan will be ignored\n    \"\"\"\n    return _mx_nd_np.amin(a, axis=axis, out=out, keepdims=keepdims)\n\n\n@set_module('mxnet.numpy')\ndef average(a, axis=None, weights=None, returned=False, out=None):\n    \"\"\"\n    Compute the weighted average along the specified axis.\n\n    Parameters\n    --------\n    a : ndarray\n        Array containing data to be averaged.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which to average a.\n        The default, axis=None, will average over\n        all of the elements of the input array.\n        If axis is negative it counts from the last to the first axis.\n        New in version 1.7.0.\n        If axis is a tuple of ints, averaging is\n        performed on all of the axes specified in the tuple\n        instead of a single axis or all the axes as before.\n    weights : ndarray, optional\n        An array of weights associated with the values in a, must be the same dtype with a.\n        Each value in a contributes to the average according to its associated weight.\n        The weights array can either be 1-D (in which case its length must be\n        the size of a along the given axis) or of the same shape as a.\n        If weights=None, then all data in a are assumed to have a weight equal to one.\n        The 1-D calculation is: avg = sum(a * weights) / sum(weights)\n        The only constraint on weights is that sum(weights) must not be 0.\n    returned : bool, optional\n        Default is False.\n        If True, the tuple (average, sum_of_weights) is returned,\n        otherwise only the average is returned.\n        If weights=None, sum_of_weights is equivalent to\n        the number of elements over which the average is taken.\n    out : ndarray, optional\n        If provided, the calculation is done into this array.\n\n    Returns\n    --------\n    retval, [sum_of_weights] : ndarray\n        Return the average along the specified axis.\n        When returned is True, return a tuple with the average as the first element\n        and the sum of the weights as the second element. sum_of_weights is of the same type as retval.\n        If a is integral, the result dtype will be current default dtype,\n        When npx.is_np_default_dtype() returns False, default dtype is float32,\n        When npx.is_np_default_dtype() returns True, default dtype is float64;\n        otherwise it will be the same as dtype of a.\n\n    Raises\n    --------\n        MXNetError\n        * When all weights along axis sum to zero.\n        * When the length of 1D weights is not the same as the shape of a along axis.\n        * When given 1D weights, the axis is not specified or is not int.\n        * When the shape of weights and a differ, but weights are not 1D.\n\n    See also\n    --------\n        mean\n\n    .. note::\n       This function differs from the original `numpy.average`\n       <https://numpy.org/devdocs/reference/generated/numpy.average.html>`_ in\n       the following way(s):\n\n       * Does not guarantee the same behavior with numpy when given float16 dtype and overflow happens\n       * Does not support complex dtype\n       * The dtypes of a and weights must be the same\n       * Integral a results in float32 or float64 returned dtype:\n\n         * When npx.is_np_default_dtype() returns False, default dtype is float32,\n         * When npx.is_np_default_dtype() returns True, default dtype is float64;\n\n    Examples\n    --------\n    >>> data = np.arange(1, 5)\n    >>> data\n    array([1., 2., 3., 4.])\n    >>> np.average(data)\n    array(2.5)\n    >>> np.average(np.arange(1, 11), weights=np.arange(10, 0, -1))\n    array(4.)\n    >>> data = np.arange(6).reshape((3,2))\n    >>> data\n    array([[0., 1.],\n           [2., 3.],\n           [4., 5.]])\n    >>> weights = np.array([0.25, 0.75])\n    array([0.25, 0.75])\n    >>> np.average(data, axis=1, weights=weights)\n    array([0.75, 2.75, 4.75])\n    \"\"\"\n    return _mx_nd_np.average(a, axis=axis, weights=weights, returned=returned, out=out)\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\ndef mean(a, axis=None, dtype=None, out=None, keepdims=False):  # pylint: disable=arguments-differ\n    \"\"\"\n    Compute the arithmetic mean along the specified axis.\n    Returns the average of the array elements.\n    The average is taken over the flattened array by default, otherwise over the specified axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        ndarray containing numbers whose mean is desired.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which the means are computed. The default is to compute the mean of the flattened array.\n        If this is a tuple of ints, a mean is performed over multiple axes,\n        instead of a single axis or all the axes as before.\n    dtype : data-type, optional\n        Type to use in computing the mean.\n        For integer inputs, the default is of your current default dtype,\n        When npx.is_np_default_dtype() returns False, default dtype is float32,\n        When npx.is_np_default_dtype() returns True, default dtype is float64;\n        For floating point inputs, it is the same as the input dtype.\n    out : ndarray, optional\n        Alternate output array in which to place the result. The default is None; if provided,\n        it must have the same shape and type as the expected output.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in the result\n        as dimensions with size one. With this option, the result will broadcast correctly\n        against the input array.\n        If the default value is passed, then keepdims will not be passed through to the mean\n        method of sub-classes of ndarray, however any non-default value will be. If the sub-class\n        method does not implement keepdims any exceptions will be raised.\n\n    Returns\n    -------\n    m : ndarray, see dtype parameter above\n        If out=None, returns a new array containing the mean values,\n        otherwise a reference to the output array is returned.\n\n    .. note::\n\n       This function differs from the original `numpy.mean\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.mean.html>`_ in\n       the following way(s):\n\n       * only ndarray is accepted as valid input, python iterables or scalar is not supported\n       * default data type for integer input is float32 or float64, which depends on your current default dtype\n\n    Examples\n    --------\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> np.mean(a)\n    array(2.5)\n    >>> a = np.zeros((2, 512*512), dtype=np.float32)\n    >>> a[0,:] = 1.0\n    >>> a[1,:] = 0.1\n    >>> np.mean(a)\n    array(0.55)\n    >>> np.mean(a, dtype=np.float64)\n    array(0.55, dtype=float64)\n    \"\"\"\n    return _mx_nd_np.mean(a, axis=axis, dtype=dtype, keepdims=keepdims, out=out)\n# pylint: enable=redefined-outer-name\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\n@wrap_data_api_statical_func\ndef std(a, axis=None, dtype=None, out=None, correction=0, keepdims=False):  # pylint: disable=too-many-arguments\n    \"\"\"\n    Compute the standard deviation along the specified axis.\n    Returns the standard deviation, a measure of the spread of a distribution,\n    of the array elements. The standard deviation is computed for the\n    flattened array by default, otherwise over the specified axis.\n\n    Parameters\n    ----------\n    a : array_like\n        Calculate the standard deviation of these values.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which the standard deviation is computed. The\n        default is to compute the standard deviation of the flattened array.\n        .. versionadded:: 1.7.0\n        If this is a tuple of ints, a standard deviation is performed over\n        multiple axes, instead of a single axis or all the axes as before.\n    dtype : dtype, optional\n        Type to use in computing the standard deviation. For arrays of\n        integer type the default is float64, for arrays of float types it is\n        the same as the array type.\n    out : ndarray, optional\n        Alternative output array in which to place the result. It must have\n        the same shape as the expected output but the type (of the calculated\n        values) will be cast if necessary.\n    correction : int, optional\n        Means Delta Degrees of Freedom.  The divisor used in calculations\n        is ``N - correction``, where ``N`` represents the number of elements.\n        By default `correction` is zero.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the input array.\n        If the default value is passed, then `keepdims` will not be\n        passed through to the `std` method of sub-classes of\n        `ndarray`, however any non-default value will be.  If the\n        sub-class' method does not implement `keepdims` any\n        exceptions will be raised.\n\n    Returns\n    -------\n    standard_deviation : ndarray, see dtype parameter above.\n        If `out` is None, return a new array containing the standard deviation,\n        otherwise return a reference to the output array.\n\n    Examples\n    --------\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> np.std(a)\n    1.1180339887498949 # may vary\n    >>> np.std(a, axis=0)\n    array([1.,  1.])\n    >>> np.std(a, axis=1)\n    array([0.5,  0.5])\n    In single precision, std() can be inaccurate:\n    >>> a = np.zeros((2, 512*512), dtype=np.float32)\n    >>> a[0, :] = 1.0\n    >>> a[1, :] = 0.1\n    >>> np.std(a)\n    array(0.45)\n    >>> np.std(a, dtype=np.float64)\n    array(0.45, dtype=float64)\n    \"\"\"\n    return _mx_nd_np.std(a, axis=axis, dtype=dtype, ddof=correction, keepdims=keepdims, out=out)\n# pylint: enable=redefined-outer-name\n\n\n@set_module('mxnet.numpy')\ndef delete(arr, obj, axis=None):\n    \"\"\"\n    Return a new array with sub-arrays along an axis deleted. For a one\n    dimensional array, this returns those entries not returned by\n    `arr[obj]`.\n\n    Parameters\n    ----------\n    arr : ndarray\n      Input array.\n    obj : slice, int or ndarray of ints\n      Indicate indices of sub-arrays to remove along the specified axis.\n    axis : int, optional\n      The axis along which to delete the subarray defined by `obj`.\n      If `axis` is None, `obj` is applied to the flattened array.\n\n    Returns\n    -------\n    out : ndarray\n        A copy of `arr` with the elements specified by `obj` removed. Note\n        that `delete` does not occur in-place. If `axis` is None, `out` is\n        a flattened array.\n\n    Examples\n    --------\n    >>> arr = np.array([[1,2,3,4], [5,6,7,8], [9,10,11,12]])\n    >>> arr\n    array([[ 1.,  2.,  3.,  4.],\n           [ 5.,  6.,  7.,  8.],\n           [ 9., 10., 11., 12.]])\n\n    >>> np.delete(arr, 1, 0)\n    array([[ 1.,  2.,  3.,  4.],\n           [ 9., 10., 11., 12.]])\n\n    >>> np.delete(arr, slice(None, None, 2), 1)\n    array([[ 2.,  4.],\n           [ 6.,  8.],\n           [10., 12.]])\n\n    >>> np.delete(arr, np.array([1,3,5]), None)\n    array([ 1.,  3.,  5.,  7.,  8.,  9., 10., 11., 12.])\n    >>> np.delete(arr, np.array([1,1,5]), None)\n    array([ 1.,  3.,  4.,  5.,  7.,  8.,  9., 10., 11., 12.])\n    \"\"\"\n    return _mx_nd_np.delete(arr, obj, axis=axis)\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\n@wrap_data_api_statical_func\ndef var(a, axis=None, dtype=None, out=None, correction=0, keepdims=False):  # pylint: disable=too-many-arguments\n    \"\"\"\n    Compute the variance along the specified axis.\n    Returns the variance of the array elements, a measure of the spread of a\n    distribution.  The variance is computed for the flattened array by\n    default, otherwise over the specified axis.\n\n    Parameters\n    ----------\n    a : array_like\n        Array containing numbers whose variance is desired.  If `a` is not an\n        array, a conversion is attempted.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which the variance is computed.  The default is to\n        compute the variance of the flattened array.\n        .. versionadded:: 1.7.0\n        If this is a tuple of ints, a variance is performed over multiple axes,\n        instead of a single axis or all the axes as before.\n    dtype : data-type, optional\n        Type to use in computing the variance.\n        For arrays of integer type, the default is of your current default dtype,\n        When npx.is_np_default_dtype() returns False, default dtype is float32,\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        For arrays of float types it is the same as the array type.\n    out : ndarray, optional\n        Alternate output array in which to place the result.  It must have\n        the same shape as the expected output, but the type is cast if\n        necessary.\n    correction : int, optional\n        \"Delta Degrees of Freedom\": the divisor used in the calculation is\n        ``N - correction``, where ``N`` represents the number of elements. By\n        default `correction` is zero.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the input array.\n        If the default value is passed, then `keepdims` will not be\n        passed through to the `var` method of sub-classes of\n        `ndarray`, however any non-default value will be.  If the\n        sub-class' method does not implement `keepdims` any\n        exceptions will be raised.\n\n    Returns\n    -------\n    variance : ndarray, see dtype parameter above\n        If ``out=None``, returns a new array containing the variance;\n        otherwise, a reference to the output array is returned.\n\n    Examples\n    --------\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> np.var(a)\n    array(1.25)\n    >>> np.var(a, axis=0)\n    array([1.,  1.])\n    >>> np.var(a, axis=1)\n    array([0.25,  0.25])\n\n    >>> a = np.zeros((2, 512*512), dtype=np.float32)\n    >>> a[0, :] = 1.0\n    >>> a[1, :] = 0.1\n    >>> np.var(a)\n    array(0.2025)\n    >>> np.var(a, dtype=np.float64)\n    array(0.2025, dtype=float64)\n    >>> ((1-0.55)**2 + (0.1-0.55)**2)/2\n    0.2025\n    \"\"\"\n    return _mx_nd_np.var(a, axis=axis, dtype=dtype, ddof=correction, keepdims=keepdims, out=out)\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef indices(dimensions, dtype=None, device=None):\n    \"\"\"Return an array representing the indices of a grid.\n\n    Compute an array where the subarrays contain index values 0,1,...\n    varying only along the corresponding axis.\n\n    Parameters\n    ----------\n    dimensions : sequence of ints\n        The shape of the grid.\n    dtype : data-type, optional\n        The desired data-type for the array. Default is `int64`.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    grid : ndarray\n        The array of grid indices,\n        ``grid.shape = (len(dimensions),) + tuple(dimensions)``.\n\n    Notes\n    -----\n    The output shape is obtained by prepending the number of dimensions\n    in front of the tuple of dimensions, i.e. if `dimensions` is a tuple\n    ``(r0, ..., rN-1)`` of length ``N``, the output shape is\n    ``(N,r0,...,rN-1)``.\n\n    The subarrays ``grid[k]`` contains the N-D array of indices along the\n    ``k-th`` axis. Explicitly::\n\n        grid[k,i0,i1,...,iN-1] = ik\n\n    Examples\n    --------\n    >>> grid = np.indices((2, 3))\n    >>> grid.shape\n    (2, 2, 3)\n    >>> grid[0]        # row indices\n    array([[0, 0, 0],\n           [1, 1, 1]], dtype=int64)\n    >>> grid[1]        # column indices\n    array([[0, 0, 0],\n           [1, 1, 1]], dtype=int64)\n\n    The indices can be used as an index into an array.\n\n    >>> x = np.arange(20).reshape(5, 4)\n    >>> row, col = np.indices((2, 3))\n    >>> x[row, col]\n    array([[0., 1., 2.],\n           [4., 5., 6.]])\n\n    Note that it would be more straightforward in the above example to\n    extract the required elements directly with ``x[:2, :3]``.\n    \"\"\"\n    return _mx_nd_np.indices(dimensions=dimensions, dtype=dtype, device=device)\n# pylint: enable=redefined-outer-name\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef copysign(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Change the sign of x1 to that of x2, element-wise.\n\n    If `x2` is a scalar, its sign will be copied to all elements of `x1`.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Values to change the sign of.\n    x2 : ndarray or scalar\n        The sign of `x2` is copied to `x1`.\n    out : ndarray or None, optional\n        A location into which the result is stored. It must be of the\n        right shape and right type to hold the output. If not provided\n        or `None`,a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        The values of `x1` with the sign of `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n\n    .. note::\n       This function differs from the original `numpy.copysign\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.copysign.html>`_ in\n       the following aspects:\n\n       * ``where`` param is not supported.\n\n    Examples\n    --------\n    >>> np.copysign(1.3, -1)\n    -1.3\n    >>> 1/np.copysign(0, 1)\n    inf\n    >>> 1/np.copysign(0, -1)\n    -inf\n\n    >>> a = np.array([-1, 0, 1])\n    >>> np.copysign(a, -1.1)\n    array([-1., -0., -1.])\n    >>> np.copysign(a, np.arange(3)-1)\n    array([-1.,  0.,  1.])\n    \"\"\"\n    return _mx_nd_np.copysign(x1, x2, out=out)\n\n\n@set_module('mxnet.numpy')\ndef ravel(x, order='C'):\n    r\"\"\"\n    ravel(x)\n\n    Return a contiguous flattened array.\n    A 1-D array, containing the elements of the input, is returned.  A copy is\n    made only if needed.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array.  The elements in `x` are read in row-major, C-style order and\n        packed as a 1-D array.\n    order : `C`, optional\n        Only support row-major, C-style order.\n\n    Returns\n    -------\n    y : ndarray\n        y is an array of the same subtype as `x`, with shape ``(x.size,)``.\n        Note that matrices are special cased for backward compatibility, if `x`\n        is a matrix, then y is a 1-D ndarray.\n\n    .. note::\n       This function differs from the original numpy.arange in the following aspects:\n\n       * Only support row-major, C-style order.\n\n    Examples\n    --------\n    It is equivalent to ``reshape(x, -1)``.\n\n    >>> x = np.array([[1, 2, 3], [4, 5, 6]])\n    >>> print(np.ravel(x))\n    [1. 2. 3. 4. 5. 6.]\n\n    >>> print(x.reshape(-1))\n    [1. 2. 3. 4. 5. 6.]\n\n    >>> print(np.ravel(x.T))\n    [1. 4. 2. 5. 3. 6.]\n    \"\"\"\n    return _mx_nd_np.ravel(x, order)\n\n\n@set_module('mxnet.numpy')\ndef unravel_index(indices, shape, order='C'): # pylint: disable=redefined-outer-name\n    \"\"\"\n    Converts a flat index or array of flat indices into a tuple of coordinate arrays.\n\n    Parameters\n    ----------\n    indices : array_like\n            An integer array whose elements are indices into the flattened version of an array of dimensions shape.\n            Before version 1.6.0, this function accepted just one index value.\n    shape : tuple of ints\n            The shape of the array to use for unraveling indices.\n    order : Only row-major is supported currently.\n\n    Returns\n    -------\n    unraveled_coords : ndarray\n            Each row in the ndarray has the same shape as the indices array.\n            Each column in the ndarray represents the unravelled index\n\n    Examples:\n    -------------\n    >>> np.unravel_index([22, 41, 37], (7,6))\n    [[3. 6. 6.]\n      [4. 5. 1.]]\n    >>> np.unravel_index(1621, (6,7,8,9))\n    [3, 1, 4, 1]\n    \"\"\"\n    return _mx_nd_np.unravel_index(indices, shape, order=order)\n\n\n@set_module('mxnet.numpy')\ndef flatnonzero(a):\n    r\"\"\"\n    Return indices that are non-zero in the flattened version of a.\n\n    This is equivalent to np.nonzero(np.ravel(a))[0].\n\n    Parameters\n    ----------\n    a : array_like\n        Input data.\n\n    Returns\n    -------\n    res : ndarray\n        Output array, containing the indices of the elements of `a.ravel()`\n        that are non-zero.\n\n    See Also\n    --------\n    nonzero : Return the indices of the non-zero elements of the input array.\n    ravel : Return a 1-D array containing the elements of the input array.\n\n    Examples\n    --------\n    >>> x = np.arange(-2, 3)\n    >>> x\n    array([-2, -1,  0,  1,  2])\n    >>> np.flatnonzero(x)\n    array([0, 1, 3, 4])\n\n    Use the indices of the non-zero elements as an index array to extract\n    these elements:\n\n    >>> x.ravel()[np.flatnonzero(x)]\n    array([-2, -1,  1,  2])\n    \"\"\"\n    return _mx_nd_np.flatnonzero(a)\n\n\n@set_module('mxnet.numpy')\ndef diag_indices_from(arr):\n    \"\"\"\n    This returns a tuple of indices that can be used to access the main diagonal of an array\n    a with a.ndim >= 2 dimensions and shape (n, n, ..., n). For a.ndim = 2 this is\n    the usual diagonal, for a.ndim > 2 this is the set of indices to access\n    a[i, i, ..., i] for i = [0..n-1].\n\n    Parameters\n    ----------\n    arr : ndarray\n        Input array for acessing the main diagonal. All dimensions\n        should have equal length.\n\n    Return:\n    -------------\n    diag: tuple of ndarray\n        indices of the main diagonal.\n\n    Examples:\n    -------------\n    >>> a = np.arange(16).reshape(4, 4)\n    >>> a\n    array([[ 0,  1,  2,  3],\n        [ 4,  5,  6,  7],\n        [ 8,  9, 10, 11],\n        [12, 13, 14, 15]])\n    >>> idx = np.diag_indices_from(a)\n    >>> idx\n    (array([0, 1, 2, 3]), array([0, 1, 2, 3]))\n    >>> a[idx] = 100\n    >>> a\n    array([[100,   1,   2,   3],\n        [  4, 100,   6,   7],\n        [  8,   9, 100,  11],\n        [ 12,  13,  14, 100]])\n    \"\"\"\n    return _mx_nd_np.diag_indices_from(arr)\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef hanning(M, dtype=None, device=None):\n    r\"\"\"Return the Hanning window.\n\n    The Hanning window is a taper formed by using a weighted cosine.\n\n    Parameters\n    ----------\n    M : int\n        Number of points in the output window. If zero or less, an\n        empty array is returned.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    out : ndarray, shape(M,)\n        The window, with the maximum value normalized to one (the value\n        one appears only if `M` is odd).\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Note that you need select numpy.float32 or float64 in this operator.\n\n    See Also\n    --------\n    blackman, hamming\n\n    Notes\n    -----\n    The Hanning window is defined as\n\n    .. math::  w(n) = 0.5 - 0.5cos\\left(\\frac{2\\pi{n}}{M-1}\\right)\n               \\qquad 0 \\leq n \\leq M-1\n\n    The Hanning was named for Julius von Hann, an Austrian meteorologist.\n    It is also known as the Cosine Bell. Some authors prefer that it be\n    called a Hann window, to help avoid confusion with the very similar\n    Hamming window.\n\n    Most references to the Hanning window come from the signal processing\n    literature, where it is used as one of many windowing functions for\n    smoothing values.  It is also known as an apodization (which means\n    \"removing the foot\", i.e. smoothing discontinuities at the beginning\n    and end of the sampled signal) or tapering function.\n\n    References\n    ----------\n    .. [1] Blackman, R.B. and Tukey, J.W., (1958) The measurement of power\n           spectra, Dover Publications, New York.\n    .. [2] E.R. Kanasewich, \"Time Sequence Analysis in Geophysics\",\n           The University of Alberta Press, 1975, pp. 106-108.\n    .. [3] Wikipedia, \"Window function\",\n           http://en.wikipedia.org/wiki/Window_function\n    .. [4] W.H. Press,  B.P. Flannery, S.A. Teukolsky, and W.T. Vetterling,\n           \"Numerical Recipes\", Cambridge University Press, 1986, page 425.\n\n    Examples\n    --------\n    >>> np.hanning(12)\n    array([0.        , 0.07937324, 0.29229254, 0.5711574 , 0.8274304 ,\n           0.9797465 , 0.97974646, 0.82743025, 0.5711573 , 0.29229245,\n           0.07937312, 0.        ])\n\n    Plot the window and its frequency response:\n\n    >>> import matplotlib.pyplot as plt\n    >>> window = np.hanning(51)\n    >>> plt.plot(window.asnumpy())\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.title(\"Hann window\")\n    Text(0.5, 1.0, 'Hann window')\n    >>> plt.ylabel(\"Amplitude\")\n    Text(0, 0.5, 'Amplitude')\n    >>> plt.xlabel(\"Sample\")\n    Text(0.5, 0, 'Sample')\n    >>> plt.show()\n    \"\"\"\n    return _mx_nd_np.hanning(M, dtype=dtype, device=device)\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef hamming(M, dtype=None, device=None):\n    r\"\"\"Return the hamming window.\n\n    The hamming window is a taper formed by using a weighted cosine.\n\n    Parameters\n    ----------\n    M : int\n        Number of points in the output window. If zero or less, an\n        empty array is returned.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    out : ndarray, shape(M,)\n        The window, with the maximum value normalized to one (the value\n        one appears only if `M` is odd).\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Note that you need select numpy.float32 or float64 in this operator.\n\n    See Also\n    --------\n    blackman, hanning\n\n    Notes\n    -----\n    The Hamming window is defined as\n\n    .. math::  w(n) = 0.54 - 0.46cos\\left(\\frac{2\\pi{n}}{M-1}\\right)\n               \\qquad 0 \\leq n \\leq M-1\n\n    The Hamming was named for R. W. Hamming, an associate of J. W. Tukey\n    and is described in Blackman and Tukey. It was recommended for\n    smoothing the truncated autocovariance function in the time domain.\n    Most references to the Hamming window come from the signal processing\n    literature, where it is used as one of many windowing functions for\n    smoothing values.  It is also known as an apodization (which means\n    \"removing the foot\", i.e. smoothing discontinuities at the beginning\n    and end of the sampled signal) or tapering function.\n\n    References\n    ----------\n    .. [1] Blackman, R.B. and Tukey, J.W., (1958) The measurement of power\n           spectra, Dover Publications, New York.\n    .. [2] E.R. Kanasewich, \"Time Sequence Analysis in Geophysics\", The\n           University of Alberta Press, 1975, pp. 109-110.\n    .. [3] Wikipedia, \"Window function\",\n           https://en.wikipedia.org/wiki/Window_function\n    .. [4] W.H. Press,  B.P. Flannery, S.A. Teukolsky, and W.T. Vetterling,\n           \"Numerical Recipes\", Cambridge University Press, 1986, page 425.\n\n    Examples\n    --------\n    >>> np.hamming(12)\n    array([0.08000001, 0.15302339, 0.34890914, 0.6054648 , 0.841236  ,\n           0.9813669 , 0.9813668 , 0.8412359 , 0.6054647 , 0.34890908,\n           0.15302327, 0.08000001])\n\n    Plot the window and its frequency response:\n\n    >>> import matplotlib.pyplot as plt\n    >>> window = np.hamming(51)\n    >>> plt.plot(window.asnumpy())\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.title(\"hamming window\")\n    Text(0.5, 1.0, 'hamming window')\n    >>> plt.ylabel(\"Amplitude\")\n    Text(0, 0.5, 'Amplitude')\n    >>> plt.xlabel(\"Sample\")\n    Text(0.5, 0, 'Sample')\n    >>> plt.show()\n    \"\"\"\n    return _mx_nd_np.hamming(M, dtype=dtype, device=device)\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef blackman(M, dtype=None, device=None):\n    r\"\"\"Return the Blackman window.\n\n    The Blackman window is a taper formed by using the first three\n    terms of a summation of cosines. It was designed to have close to the\n    minimal leakage possible.  It is close to optimal, only slightly worse\n    than a Kaiser window.\n\n    Parameters\n    ----------\n    M : int\n        Number of points in the output window. If zero or less, an\n        empty array is returned.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n\n    Returns\n    -------\n    out : ndarray\n        The window, with the maximum value normalized to one (the value one\n        appears only if the number of samples is odd).\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Note that you need select numpy.float32 or float64 in this operator.\n\n    See Also\n    --------\n    hamming, hanning\n\n    Notes\n    -----\n    The Blackman window is defined as\n\n    .. math::  w(n) = 0.42 - 0.5 \\cos(2\\pi n/{M-1}) + 0.08 \\cos(4\\pi n/{M-1})\n\n    Most references to the Blackman window come from the signal processing\n    literature, where it is used as one of many windowing functions for\n    smoothing values.  It is also known as an apodization (which means\n    \"removing the foot\", i.e. smoothing discontinuities at the beginning\n    and end of the sampled signal) or tapering function. It is known as a\n    \"near optimal\" tapering function, almost as good (by some measures)\n    as the kaiser window.\n\n    References\n    ----------\n    Blackman, R.B. and Tukey, J.W., (1958) The measurement of power spectra,\n    Dover Publications, New York.\n\n    Oppenheim, A.V., and R.W. Schafer. Discrete-Time Signal Processing.\n    Upper Saddle River, NJ: Prentice-Hall, 1999, pp. 468-471.\n\n    Examples\n    --------\n    >>> np.blackman(12)\n    array([-1.4901161e-08,  3.2606423e-02,  1.5990365e-01,  4.1439798e-01,\n            7.3604530e-01,  9.6704686e-01,  9.6704674e-01,  7.3604506e-01,\n            4.1439781e-01,  1.5990359e-01,  3.2606363e-02, -1.4901161e-08])\n\n    Plot the window and its frequency response:\n\n    >>> import matplotlib.pyplot as plt\n    >>> window = np.blackman(51)\n    >>> plt.plot(window.asnumpy())\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.title(\"blackman window\")\n    Text(0.5, 1.0, 'blackman window')\n    >>> plt.ylabel(\"Amplitude\")\n    Text(0, 0.5, 'Amplitude')\n    >>> plt.xlabel(\"Sample\")\n    Text(0.5, 0, 'Sample')\n    >>> plt.show()\n    \"\"\"\n    return _mx_nd_np.blackman(M, dtype=dtype, device=device)\n\n\n@set_module('mxnet.numpy')\ndef flip(m, axis=None, out=None):\n    r\"\"\"\n    flip(m, axis=None, out=None)\n\n    Reverse the order of elements in an array along the given axis.\n\n    The shape of the array is preserved, but the elements are reordered.\n\n    Parameters\n    ----------\n    m : ndarray or scalar\n        Input array.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which to flip over. The default,\n        axis=None, will flip over all of the axes of the input array.\n        If axis is negative it counts from the last to the first axis.\n\n        If axis is a tuple of ints, flipping is performed on all of the axes\n        specified in the tuple.\n    out : ndarray or scalar, optional\n        Alternative output array in which to place the result. It must have\n        the same shape and type as the expected output.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        A view of `m` with the entries of axis reversed.  Since a view is\n        returned, this operation is done in constant time.\n\n    Examples\n    --------\n    >>> A = np.arange(8).reshape((2,2,2))\n    >>> A\n    array([[[0, 1],\n            [2, 3]],\n           [[4, 5],\n            [6, 7]]])\n    >>> np.flip(A, 0)\n    array([[[4, 5],\n            [6, 7]],\n           [[0, 1],\n            [2, 3]]])\n    >>> np.flip(A, 1)\n    array([[[2, 3],\n            [0, 1]],\n           [[6, 7],\n            [4, 5]]])\n    >>> np.flip(A)\n    array([[[7, 6],\n            [5, 4]],\n           [[3, 2],\n            [1, 0]]])\n    >>> np.flip(A, (0, 2))\n    array([[[5, 4],\n            [7, 6]],\n           [[1, 0],\n            [3, 2]]])\n    \"\"\"\n    return _mx_nd_np.flip(m, axis, out=out)\n\n\n@set_module('mxnet.numpy')\ndef flipud(m):\n    r\"\"\"\n    flipud(*args, **kwargs)\n\n    Flip array in the up/down direction.\n\n    Flip the entries in each column in the up/down direction.\n    Rows are preserved, but appear in a different order than before.\n\n    Parameters\n    ----------\n    m : array_like\n        Input array.\n\n    Returns\n    -------\n    out : array_like\n        A view of `m` with the rows reversed.  Since a view is\n        returned, this operation is :math:`\\mathcal O(1)`.\n\n    See Also\n    --------\n    fliplr : Flip array in the left/right direction.\n    rot90 : Rotate array counterclockwise.\n\n    Notes\n    -----\n    Equivalent to ``m[::-1,...]``.\n    Does not require the array to be two-dimensional.\n\n    Examples\n    --------\n    >>> A = np.diag(np.array([1.0, 2, 3]))\n    >>> A\n    array([[1.,  0.,  0.],\n           [0.,  2.,  0.],\n           [0.,  0.,  3.]])\n    >>> np.flipud(A)\n    array([[0.,  0.,  3.],\n           [0.,  2.,  0.],\n           [1.,  0.,  0.]])\n\n    >>> A = np.random.randn(2,3,5)\n    >>> np.all(np.flipud(A) == A[::-1,...])\n    array(True)\n\n    >>> np.flipud(np.array([1,2]))\n    array([2., 1.])\n    \"\"\"\n    return flip(m, 0)\n\n\n@set_module('mxnet.numpy')\ndef fliplr(m):\n    r\"\"\"\n    fliplr(*args, **kwargs)\n\n    Flip array in the left/right direction.\n\n    Flip the entries in each row in the left/right direction.\n    Columns are preserved, but appear in a different order than before.\n\n    Parameters\n    ----------\n    m : array_like\n        Input array, must be at least 2-D.\n\n    Returns\n    -------\n    f : ndarray\n        A view of `m` with the columns reversed.  Since a view\n        is returned, this operation is :math:`\\mathcal O(1)`.\n\n    See Also\n    --------\n    flipud : Flip array in the up/down direction.\n    rot90 : Rotate array counterclockwise.\n\n    Notes\n    -----\n    Equivalent to m[:,::-1]. Requires the array to be at least 2-D.\n\n    Examples\n    --------\n    >>> A = np.diag([1.,2.,3.])\n    >>> A\n    array([[1.,  0.,  0.],\n        [0.,  2.,  0.],\n        [0.,  0.,  3.]])\n    >>> np.fliplr(A)\n    array([[0.,  0.,  1.],\n        [0.,  2.,  0.],\n        [3.,  0.,  0.]])\n\n    >>> A = np.random.randn(2,3,5)\n    >>> np.all(np.fliplr(A) == A[:,::-1,...])\n    array(True)\n    \"\"\"\n    return flip(m, 1)\n\n\n@set_module('mxnet.numpy')\ndef around(x, decimals=0, out=None, **kwargs):\n    r\"\"\"\n    around(x, decimals=0, out=None)\n\n    Evenly round to the given number of decimals.\n\n    Parameters\n    ----------\n    x : ndarray or scalar\n        Input data.\n    decimals : int, optional\n        Number of decimal places to round to (default: 0).  If\n        decimals is negative, it specifies the number of positions to\n        the left of the decimal point.\n    out : ndarray, optional\n        Alternative output array in which to place the result. It must have\n        the same shape and type as the expected output.\n\n    Returns\n    -------\n    rounded_array : ndarray or scalar\n        An array of the same type as `x`, containing the rounded values.\n        A reference to the result is returned.\n\n    .. note::\n       For values exactly halfway between rounded decimal values, NumPy\n       rounds to the nearest even value. Thus 1.5 and 2.5 round to 2.0,\n       -0.5 and 0.5 round to 0.0, etc.\n\n       This function differs from the original numpy.prod in the following aspects:\n\n       * Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n       * Cannot support complex-valued number.\n\n    Examples\n    --------\n    >>> np.around([0.37, 1.64])\n    array([ 0.,  2.])\n    >>> np.around([0.37, 1.64], decimals=1)\n    array([ 0.4,  1.6])\n    >>> np.around([.5, 1.5, 2.5, 3.5, 4.5]) # rounds to nearest even value\n    array([ 0.,  2.,  2.,  4.,  4.])\n    >>> np.around([1, 2, 3, 11], decimals=1) # ndarray of ints is returned\n    array([ 1,  2,  3, 11])\n    >>> np.around([1, 2, 3, 11], decimals=-1)\n    array([ 0,  0,  0, 10])\n    \"\"\"\n    return _mx_nd_np.around(x, decimals, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\ndef round(x, decimals=0, out=None, **kwargs):\n    r\"\"\"\n    round(a, decimals=0, out=None)\n    Round an array to the given number of decimals.\n\n    See Also\n    --------\n    around : equivalent function; see for details.\n    \"\"\"\n    return _mx_nd_np.round(x, decimals, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\ndef round_(x, decimals=0, out=None, **kwargs):\n    r\"\"\"\n    round_(a, decimals=0, out=None)\n    Round an array to the given number of decimals.\n\n    See Also\n    --------\n    around : equivalent function; see for details.\n    \"\"\"\n    return _mx_nd_np.round_(x, decimals, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef arctan2(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Element-wise arc tangent of ``x1/x2`` choosing the quadrant correctly.\n\n    The quadrant (i.e., branch) is chosen so that ``arctan2(x1, x2)`` is\n    the signed angle in radians between the ray ending at the origin and\n    passing through the point (1,0), and the ray ending at the origin and\n    passing through the point (`x2`, `x1`).  (Note the role reversal: the\n    \"`y`-coordinate\" is the first function parameter, the \"`x`-coordinate\"\n    is the second.)  By IEEE convention, this function is defined for\n    `x2` = +/-0 and for either or both of `x1` and `x2` = +/-inf (see\n    Notes for specific values).\n\n    This function is not defined for complex-valued arguments; for the\n    so-called argument of complex values, use `angle`.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        `y`-coordinates.\n    x2 : ndarray or scalar\n        `x`-coordinates. `x2` must be broadcastable to match the shape of\n        `x1` or vice versa.\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Array of angles in radians, in the range ``[-pi, pi]``. This is a scalar if\n        `x1` and `x2` are scalars.\n\n    .. notes::\n       *arctan2* is identical to the ``atan2`` function of the underlying\n       C library.  The following special values are defined in the C\n       standard: [1]_\n\n       +========+========+==================+\n       | `x1`   | `x2`   | `arctan2(x1,x2)` |\n       +========+========+==================+\n       | +/- 0  | +0     | +/- 0            |\n       +========+========+==================+\n       | +/- 0  | -0     | +/- pi           |\n       +========+========+==================+\n       | > 0    | +/-inf | +0 / +pi         |\n       +========+========+==================+\n       | < 0    | +/-inf | -0 / -pi         |\n       +========+========+==================+\n       | +/-inf | +inf   | +/- (pi/4)       |\n       +========+========+==================+\n       | +/-inf | -inf   | +/- (3*pi/4)     |\n       +========+========+==================+\n\n       Note that +0 and -0 are distinct floating point numbers, as are +inf\n       and -inf.\n\n       This function differs from the original numpy.arange in the following aspects:\n\n       * Only support float16, float32 and float64.\n\n    References\n    ----------\n    .. [1] ISO/IEC standard 9899:1999, \"Programming language C.\"\n\n    Examples\n    --------\n    Consider four points in different quadrants:\n\n    >>> x = np.array([-1, +1, +1, -1])\n    >>> y = np.array([-1, -1, +1, +1])\n    >>> np.arctan2(y, x) * 180 / np.pi\n    array([-135.,  -45.,   45.,  135.])\n\n    Note the order of the parameters. `arctan2` is defined also when `x2` = 0\n    and at several other special points, obtaining values in\n    the range ``[-pi, pi]``:\n\n    >>> x = np.array([1, -1])\n    >>> y = np.array([0, 0])\n    >>> np.arctan2(x, y)\n    array([ 1.5707964, -1.5707964])\n    \"\"\"\n    return _mx_nd_np.arctan2(x1, x2, out=out)\n\natan2 = arctan2\natan2.__doc__ = \"\"\"\n    Element-wise arc tangent of ``x1/x2`` choosing the quadrant correctly.\n\n    The quadrant (i.e., branch) is chosen so that ``atan2(x1, x2)`` is\n    the signed angle in radians between the ray ending at the origin and\n    passing through the point (1,0), and the ray ending at the origin and\n    passing through the point (`x2`, `x1`).  (Note the role reversal: the\n    \"`y`-coordinate\" is the first function parameter, the \"`x`-coordinate\"\n    is the second.)  By IEEE convention, this function is defined for\n    `x2` = +/-0 and for either or both of `x1` and `x2` = +/-inf (see\n    Notes for specific values).\n\n    This function is not defined for complex-valued arguments; for the\n    so-called argument of complex values, use `angle`.\n    \n    >>>np.atan2 is np.arctan2\n    True\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        `y`-coordinates.\n    x2 : ndarray or scalar\n        `x`-coordinates. `x2` must be broadcastable to match the shape of\n        `x1` or vice versa.\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Array of angles in radians, in the range ``[-pi, pi]``. This is a scalar if\n        `x1` and `x2` are scalars.\n\n    .. notes::\n       `atan2` is a alias for `arctan2`. It is a standard API in\n       https://data-apis.org/array-api/latest/API_specification/generated/signatures.elementwise_functions.atan2.html\n       instead of an official NumPy operator.\n       \n       *atan2* is identical to the ``atan2`` function of the underlying\n       C library.  The following special values are defined in the C\n       standard: [1]_\n\n       +========+========+==================+\n       | `x1`   | `x2`   | `atan2(x1,x2)` |\n       +========+========+==================+\n       | +/- 0  | +0     | +/- 0            |\n       +========+========+==================+\n       | +/- 0  | -0     | +/- pi           |\n       +========+========+==================+\n       | > 0    | +/-inf | +0 / +pi         |\n       +========+========+==================+\n       | < 0    | +/-inf | -0 / -pi         |\n       +========+========+==================+\n       | +/-inf | +inf   | +/- (pi/4)       |\n       +========+========+==================+\n       | +/-inf | -inf   | +/- (3*pi/4)     |\n       +========+========+==================+\n\n       Note that +0 and -0 are distinct floating point numbers, as are +inf\n       and -inf.\n\n       This function differs from the original numpy.arange in the following aspects:\n\n       * Only support float16, float32 and float64.\n\n    References\n    ----------\n    .. [1] ISO/IEC standard 9899:1999, \"Programming language C.\"\n\n    Examples\n    --------\n    Consider four points in different quadrants:\n\n    >>> x = np.array([-1, +1, +1, -1])\n    >>> y = np.array([-1, -1, +1, +1])\n    >>> np.atan2(y, x) * 180 / np.pi\n    array([-135.,  -45.,   45.,  135.])\n\n    Note the order of the parameters. `atan2` is defined also when `x2` = 0\n    and at several other special points, obtaining values in\n    the range ``[-pi, pi]``:\n\n    >>> x = np.array([1, -1])\n    >>> y = np.array([0, 0])\n    >>> np.atan2(x, y)\n    array([ 1.5707964, -1.5707964])\n    \"\"\"\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef hypot(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Given the \"legs\" of a right triangle, return its hypotenuse.\n\n    Equivalent to ``sqrt(x1**2 + x2**2)``, element-wise.  If `x1` or\n    `x2` is scalar_like (i.e., unambiguously cast-able to a scalar type),\n    it is broadcast for use with each element of the other argument.\n\n    Parameters\n    ----------\n    x1, x2 : array_like\n        Leg of the triangle(s).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n\n    Returns\n    -------\n    z : ndarray\n        The hypotenuse of the triangle(s).\n        This is a scalar if both `x1` and `x2` are scalars.\n\n    .. note::\n       This function differs from the original numpy.arange in the following aspects:\n\n       * Only support float16, float32 and float64.\n\n    Examples\n    --------\n    >>> np.hypot(3*np.ones((3, 3)), 4*np.ones((3, 3)))\n    array([[ 5.,  5.,  5.],\n           [ 5.,  5.,  5.],\n           [ 5.,  5.,  5.]])\n\n    Example showing broadcast of scalar_like argument:\n\n    >>> np.hypot(3*np.ones((3, 3)), [4])\n    array([[ 5.,  5.,  5.],\n           [ 5.,  5.,  5.],\n           [ 5.,  5.,  5.]])\n    \"\"\"\n    return _mx_nd_np.hypot(x1, x2, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef bitwise_and(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Compute the bit-wise XOR of two arrays element-wise.\n\n    Parameters\n    ----------\n    x1, x2 : ndarray or scalar\n        Only integer and boolean types are handled. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which becomes the shape of the output).\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape that the\n        inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n        Result.\n\n    Examples\n    --------\n    >>> np.bitwise_and(13, 17)\n    1\n\n    >>> np.bitwise_and(14, 13)\n    12\n    >>> np.bitwise_and(np.array([14,3], dtype='int32'), 13)\n    array([26,  5], dtype=int32)\n\n    >>> np.bitwise_and(np.array([11,7], dtype='int32'), np.array([4,25], dtype='int32'))\n    array([0, 1], dtype=int32)\n    >>> np.bitwise_and(np.array([2,5,255], dtype='int32'), np.array([3,14,16], dtype='int32'))\n    array([ 2,  4, 16], dtype=int32)\n    >>> np.bitwise_and(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))\n    array([False,  True])\n    \"\"\"\n    return _mx_nd_np.bitwise_and(x1, x2, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef bitwise_xor(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Compute the bit-wise XOR of two arrays element-wise.\n\n    Parameters\n    ----------\n    x1, x2 : ndarray or scalar\n        Only integer and boolean types are handled. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which becomes the shape of the output).\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape that the\n        inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n        Result.\n\n    Examples\n    --------\n    >>> np.bitwise_xor(13, 17)\n    28\n\n    >>> np.bitwise_xor(31, 5)\n    26\n    >>> np.bitwise_xor(np.array([31,3], dtype=np.int32), 5)\n    array([26,  6], dtype=int32)\n\n    >>> np.bitwise_xor(np.array([31,3], dtype='int32'), np.array([5,6], dtype='int32'))\n    array([26,  5], dtype=int32)\n    >>> np.bitwise_xor(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))\n    array([ True, False])\n    \"\"\"\n    return _mx_nd_np.bitwise_xor(x1, x2, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef bitwise_or(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Compute the bit-wise OR of two arrays element-wise.\n\n    Parameters\n    ----------\n    x1, x2 : ndarray or scalar\n        Only integer and boolean types are handled. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which becomes the shape of the output).\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape that the\n        inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n        Result.\n\n    Examples\n    --------\n    >>> np.bitwise_or(13, 17)\n    29\n\n    >>> np.bitwise_or(31, 5)\n    31\n    >>> np.bitwise_or(np.array([31,3], dtype=np.int32), 5)\n    array([31,  7])\n\n    >>> np.bitwise_or(np.array([31,3], dtype='int32'), np.array([5,6], dtype='int32'))\n    array([31,  7])\n    >>> np.bitwise_or(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))\n    array([ True, True])\n    \"\"\"\n    return _mx_nd_np.bitwise_or(x1, x2, out=out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef ldexp(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns x1 * 2**x2, element-wise.\n    The mantissas `x1` and twos exponents `x2` are used to construct\n    floating point numbers ``x1 * 2**x2``.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Array of multipliers.\n    x2 : ndarray or scalar, int\n        Array of twos exponents.\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The result of ``x1 * 2**x2``.\n        This is a scalar if both `x1` and `x2` are scalars.\n\n    Notes\n    -----\n    Complex dtypes are not supported, they will raise a TypeError.\n    Different from numpy, we allow x2 to be float besides int.\n    `ldexp` is useful as the inverse of `frexp`, if used by itself it is\n    more clear to simply use the expression ``x1 * 2**x2``.\n\n    Examples\n    --------\n    >>> np.ldexp(5, np.arange(4))\n    array([  5.,  10.,  20.,  40.])\n    \"\"\"\n    return _mx_nd_np.ldexp(x1, x2, out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef logaddexp(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Logarithm of the sum of exponentiations of the inputs.\n\n    Calculates log(exp(x1) + exp(x2)). This function is useful in statistics where\n    the calculated probabilities of events may be so small as to exceed the range of\n    normal floating point numbers. In such cases the logarithm of the calculate\n    probability is stored. This function allows adding probabilities stored\n    in such a fashion.\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Array of multipliers.\n    x2 : ndarray or scalar, int\n        Array of twos exponents.\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        Logarithm of exp(x1) + exp(x2). This is a scalar if both x1 and x2 are scalars.\n\n    Examples\n    --------\n    >>> prob1 = np.log(1e-50)\n    >>> prob2 = np.log(2.5e-50)\n    >>> prob12 = np.logaddexp(prob1, prob2)\n    >>> prob12\n    -113.87649168120691\n    >>> np.exp(prob12)\n    3.5000000000000057e-50\n    \"\"\"\n    return _mx_nd_np.logaddexp(x1, x2, out)\n\n\n@set_module('mxnet.numpy')\ndef vdot(a, b):\n    r\"\"\"\n    Return the dot product of two vectors.\n    Note that `vdot` handles multidimensional arrays differently than `dot`:\n    it does *not* perform a matrix product, but flattens input arguments\n    to 1-D vectors first. Consequently, it should only be used for vectors.\n\n    Parameters\n    ----------\n    a : ndarray\n        First argument to the dot product.\n    b : ndarray\n        Second argument to the dot product.\n\n    Returns\n    -------\n    output : ndarray\n        Dot product of `a` and `b`.\n\n    See Also\n    --------\n    dot : Return the dot product without using the complex conjugate of the\n        first argument.\n\n    Examples\n    --------\n    Note that higher-dimensional arrays are flattened!\n\n    >>> a = np.array([[1, 4], [5, 6]])\n    >>> b = np.array([[4, 1], [2, 2]])\n    >>> np.vdot(a, b)\n    array(30.)\n    >>> np.vdot(b, a)\n    array(30.)\n    >>> 1*4 + 4*1 + 5*2 + 6*2\n    30\n    \"\"\"\n    return tensordot(a.flatten(), b.flatten(), 1)\n\n\n@set_module('mxnet.numpy')\ndef inner(a, b):\n    r\"\"\"Inner product of two arrays.\n    Ordinary inner product of vectors for 1-D arrays (without complex\n    conjugation), in higher dimensions a sum product over the last axes.\n\n    Parameters\n    ----------\n    a, b : ndarray\n        If `a` and `b` are nonscalar, their last dimensions must match.\n\n    Returns\n    -------\n    out : ndarray\n        `out.shape = a.shape[:-1] + b.shape[:-1]`\n\n    Raises\n    ------\n    ValueError\n        If the last dimension of `a` and `b` has different size.\n\n    See Also\n    --------\n    tensordot : Sum products over arbitrary axes.\n    dot : Generalised matrix product, using second last dimension of `b`.\n    einsum : Einstein summation convention.\n\n    .. note::\n\n       For vectors (1-D arrays) it computes the ordinary inner-product::\n\n           np.inner(a, b) = sum(a[:]*b[:])\n\n       More generally, if `ndim(a) = r > 0` and `ndim(b) = s > 0`::\n\n           np.inner(a, b) = np.tensordot(a, b, axes=(-1,-1))\n\n       or explicitly::\n\n           np.inner(a, b)[i0,...,ir-1,j0,...,js-1]\n               = sum(a[i0,...,ir-1,:]*b[j0,...,js-1,:])\n\n       In addition `a` or `b` may be scalars, in which case::\n\n           np.inner(a,b) = a*b\n\n    Examples\n    --------\n    Ordinary inner product for vectors:\n\n    >>> a = np.array([1,2,3])\n    >>> b = np.array([0,1,0])\n    >>> np.inner(a, b)\n    array(2.)\n\n    A multidimensional example:\n\n    >>> a = np.arange(24).reshape((2,3,4))\n    >>> b = np.arange(4)\n    >>> np.inner(a, b)\n    array([[ 14.,  38.,  62.],\n           [ 86., 110., 134.]])\n    \"\"\"\n    return tensordot(a, b, [-1, -1])\n\n\n@set_module('mxnet.numpy')\ndef outer(a, b):\n    r\"\"\"Compute the outer product of two vectors.\n    Given two vectors, ``a = [a0, a1, ..., aM]`` and\n    ``b = [b0, b1, ..., bN]``,\n    the outer product [1]_ is::\n    [[a0*b0  a0*b1 ... a0*bN ]\n    [a1*b0    .\n    [ ...          .\n    [aM*b0            aM*bN ]]\n\n    Parameters\n    ----------\n    a : (M,) ndarray\n        First input vector.  Input is flattened if\n        not already 1-dimensional.\n    b : (N,) ndarray\n        Second input vector.  Input is flattened if\n        not already 1-dimensional.\n\n    Returns\n    -------\n    out : (M, N) ndarray\n        ``out[i, j] = a[i] * b[j]``\n\n    See also\n    --------\n    inner\n    einsum : ``einsum('i,j->ij', a.ravel(), b.ravel())`` is the equivalent.\n    ufunc.outer : A generalization to N dimensions and other operations.\n                ``np.multiply.outer(a.ravel(), b.ravel())`` is the equivalent.\n\n    References\n    ----------\n    .. [1] : G. H. Golub and C. F. Van Loan, *Matrix Computations*, 3rd\n            ed., Baltimore, MD, Johns Hopkins University Press, 1996,\n            pg. 8.\n\n    Examples\n    --------\n    Make a (*very* coarse) grid for computing a Mandelbrot set:\n\n    >>> rl = np.outer(np.ones((5,)), np.linspace(-2, 2, 5))\n    >>> rl\n    array([[-2., -1.,  0.,  1.,  2.],\n           [-2., -1.,  0.,  1.,  2.],\n           [-2., -1.,  0.,  1.,  2.],\n           [-2., -1.,  0.,  1.,  2.],\n           [-2., -1.,  0.,  1.,  2.]])\n    \"\"\"\n    return tensordot(a.flatten(), b.flatten(), 0)\n\n\n@set_module('mxnet.numpy')\ndef cross(a, b, axisa=-1, axisb=-1, axisc=-1, axis=None): # pylint: disable=too-many-arguments\n    \"\"\"\n    Return the cross product of two (arrays of) vectors.\n\n    The cross product of `a` and `b` in :math:`R^3` is a vector perpendicular\n    to both `a` and `b`.  If `a` and `b` are arrays of vectors, the vectors\n    are defined by the last axis of `a` and `b` by default, and these axes\n    can have dimensions 2 or 3.  Where the dimension of either `a` or `b` is\n    2, the third component of the input vector is assumed to be zero and the\n    cross product calculated accordingly.  In cases where both input vectors\n    have dimension 2, the z-component of the cross product is returned.\n\n    Parameters\n    ----------\n    a : ndarray\n        Components of the first vector(s).\n    b : ndarray\n        Components of the second vector(s).\n    axisa : int, optional\n        Axis of `a` that defines the vector(s).  By default, the last axis.\n    axisb : int, optional\n        Axis of `b` that defines the vector(s).  By default, the last axis.\n    axisc : int, optional\n        Axis of `c` containing the cross product vector(s).  Ignored if\n        both input vectors have dimension 2, as the return is scalar.\n        By default, the last axis.\n    axis : int, optional\n        If defined, the axis of `a`, `b` and `c` that defines the vector(s)\n        and cross product(s).  Overrides `axisa`, `axisb` and `axisc`.\n\n    Returns\n    -------\n    c : ndarray\n        Vector cross product(s).\n\n    Raises\n    ------\n    ValueError\n        When the dimension of the vector(s) in `a` and/or `b` does not\n        equal 2 or 3.\n\n    Notes\n    -----\n    Supports full broadcasting of the inputs.\n\n    Examples\n    --------\n    Vector cross-product.\n\n    >>> x = np.array([1., 2., 3.])\n    >>> y = np.array([4., 5., 6.])\n    >>> np.cross(x, y)\n    array([-3.,  6., -3.])\n\n    One vector with dimension 2.\n\n    >>> x = np.array([1., 2.])\n    >>> y = np.array([4., 5., 6.])\n    >>> np.cross(x, y)\n    array([12., -6., -3.])\n\n    Equivalently:\n\n    >>> x = np.array([1., 2., 0.])\n    >>> y = np.array([4., 5., 6.])\n    >>> np.cross(x, y)\n    array([12., -6., -3.])\n\n    Both vectors with dimension 2.\n\n    >>> x = np.array([1., 2.])\n    >>> y = np.array([4., 5.])\n    >>> np.cross(x, y)\n    array(-3.)\n\n    Multiple vector cross-products. Note that the direction of the cross\n    product vector is defined by the `right-hand rule`.\n\n    >>> x = np.array([[1., 2., 3.], [4., 5., 6.]])\n    >>> y = np.array([[4., 5., 6.], [1., 2., 3.]])\n    >>> np.cross(x, y)\n    array([[-3.,  6., -3.],\n           [ 3., -6.,  3.]])\n\n    The orientation of `c` can be changed using the `axisc` keyword.\n\n    >>> np.cross(x, y, axisc=0)\n    array([[-3.,  3.],\n           [ 6., -6.],\n           [-3.,  3.]])\n\n    Change the vector definition of `x` and `y` using `axisa` and `axisb`.\n\n    >>> x = np.array([[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]])\n    >>> y = np.array([[7., 8., 9.], [4., 5., 6.], [1., 2., 3.]])\n    >>> np.cross(x, y)\n    array([[ -6.,  12.,  -6.],\n           [  0.,   0.,   0.],\n           [  6., -12.,   6.]])\n    >>> np.cross(x, y, axisa=0, axisb=0)\n    array([[-24.,  48., -24.],\n           [-30.,  60., -30.],\n           [-36.,  72., -36.]])\n    \"\"\"\n    return _mx_nd_np.cross(a, b, axisa=axisa, axisb=axisb, axisc=axisc, axis=axis)\n\n\n@set_module('mxnet.numpy')\ndef kron(a, b):\n    r\"\"\"Kronecker product of two arrays.\n\n    Computes the Kronecker product, a composite array made of blocks of the\n    second array scaled by the first.\n\n    Parameters\n    ----------\n    a, b : ndarray\n\n    Returns\n    -------\n    out : ndarray\n\n    See Also\n    --------\n    outer : The outer product\n\n    .. note::\n       The function assumes that the number of dimensions of `a` and `b`\n       are the same, if necessary prepending the smallest with ones.\n       If `a.shape = (r0,r1,..,rN)` and `b.shape = (s0,s1,...,sN)`,\n       the Kronecker product has shape `(r0*s0, r1*s1, ..., rN*SN)`.\n       The elements are products of elements from `a` and `b`, organized\n       explicitly by::\n\n           kron(a,b)[k0,k1,...,kN] = a[i0,i1,...,iN] * b[j0,j1,...,jN]\n\n       where::\n\n           kt = it * st + jt,  t = 0,...,N\n\n       In the common 2-D case (N=1), the block structure can be visualized::\n\n           [[ a[0,0]*b,   a[0,1]*b,  ... , a[0,-1]*b  ],\n           [  ...                              ...   ],\n           [ a[-1,0]*b,  a[-1,1]*b, ... , a[-1,-1]*b ]]\n\n\n    Examples\n    --------\n    >>> np.kron([1,10,100], [5,6,7])\n    array([  5,   6,   7,  50,  60,  70, 500, 600, 700])\n    >>> np.kron([5,6,7], [1,10,100])\n    array([  5,  50, 500,   6,  60, 600,   7,  70, 700])\n    \"\"\"\n    return _mx_nd_np.kron(a, b)\n\n\n@set_module('mxnet.numpy')\ndef equal(x1, x2, out=None):\n    \"\"\"\n    Return (x1 == x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    not_equal, greater_equal, less_equal, greater, less\n    Examples\n    --------\n    >>> np.equal(np.ones(2, 1)), np.zeros(1, 3))\n    array([[False, False, False],\n           [False, False, False]])\n    >>> np.equal(1, np.ones(1))\n    array([ True])\n    \"\"\"\n    return _mx_nd_np.equal(x1, x2, out)\n\n\n@set_module('mxnet.numpy')\ndef not_equal(x1, x2, out=None):\n    \"\"\"\n    Return (x1 != x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    equal, greater, greater_equal, less, less_equal\n    Examples\n    --------\n    >>> np.not_equal(np.ones(2, 1)), np.zeros(1, 3))\n    array([[ True,  True,  True],\n           [ True,  True,  True]])\n    >>> np.not_equal(1, np.ones(1))\n    array([False])\n    \"\"\"\n    return _mx_nd_np.not_equal(x1, x2, out)\n\n\n@set_module('mxnet.numpy')\ndef greater(x1, x2, out=None):\n    \"\"\"\n    Return the truth value of (x1 > x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    equal, greater, greater_equal, less, less_equal\n    Examples\n    --------\n    >>> np.greater(np.ones(2, 1)), np.zeros(1, 3))\n    array([[ True,  True,  True],\n           [ True,  True,  True]])\n    >>> np.greater(1, np.ones(1))\n    array([False])\n    \"\"\"\n    return _mx_nd_np.greater(x1, x2, out)\n\n\n@set_module('mxnet.numpy')\ndef less(x1, x2, out=None):\n    \"\"\"\n    Return the truth value of (x1 < x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    equal, greater, greater_equal, less, less_equal\n    Examples\n    --------\n    >>> np.less(np.ones(2, 1)), np.zeros(1, 3))\n    array([[ True,  True,  True],\n           [ True,  True,  True]])\n    >>> np.less(1, np.ones(1))\n    array([False])\n    \"\"\"\n    return _mx_nd_np.less(x1, x2, out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef logical_and(x1, x2, out=None):\n    r\"\"\"\n    Compute the truth value of x1 AND x2 element-wise.\n    Parameters\n    ----------\n    x1, x2 : array_like\n        Logical AND is applied to the elements of `x1` and `x2`.\n        If ``x1.shape != x2.shape``, they must be broadcastable to a common\n        shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n    Returns\n    -------\n    y : ndarray or bool\n        Boolean result of the logical AND operation applied to the elements\n        of `x1` and `x2`; the shape is determined by broadcasting.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    logical_or, logical_not, logical_xor, bitwise_or\n    Examples\n    --------\n    >>> np.logical_and(True, False)\n    False\n    >>> np.logical_and(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))\n    array([False,  True])\n    \"\"\"\n    return _mx_nd_np.logical_and(x1, x2, out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef logical_or(x1, x2, out=None):\n    r\"\"\"\n    Compute the truth value of x1 OR x2 element-wise.\n    Parameters\n    ----------\n    x1, x2 : array_like\n        Logical OR is applied to the elements of `x1` and `x2`.\n        If ``x1.shape != x2.shape``, they must be broadcastable to a common\n        shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n    Returns\n    -------\n    y : ndarray or bool\n        Boolean result of the logical OR operation applied to the elements\n        of `x1` and `x2`; the shape is determined by broadcasting.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    logical_and, logical_not, logical_xor, bitwise_or\n    Examples\n    --------\n    >>> np.logical_or(True, False)\n    True\n    >>> np.logical_or(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))\n    array([True,  True])\n    \"\"\"\n    return _mx_nd_np.logical_or(x1, x2, out)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_binary_func\ndef logical_xor(x1, x2, out=None):\n    r\"\"\"\n    Compute the truth value of x1 XOR x2 element-wise.\n    Parameters\n    ----------\n    x1, x2 : array_like\n        Logical XOR is applied to the elements of `x1` and `x2`.\n        If ``x1.shape != x2.shape``, they must be broadcastable to a common\n        shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n    Returns\n    -------\n    y : ndarray or bool\n        Boolean result of the logical XOR operation applied to the elements\n        of `x1` and `x2`; the shape is determined by broadcasting.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    logical_and, logical_not, logical_or, bitwise_or\n    Examples\n    --------\n    >>> np.logical_xor(True, False)\n    True\n    >>> np.logical_xor(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))\n    array([ True, False])\n    \"\"\"\n    return _mx_nd_np.logical_xor(x1, x2, out)\n\n\n@set_module('mxnet.numpy')\ndef greater_equal(x1, x2, out=None):\n    \"\"\"\n    Return the truth value of (x1 >= x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    equal, greater, greater_equal, less, less_equal\n    Examples\n    --------\n    >>> np.greater_equal(np.ones(2, 1)), np.zeros(1, 3))\n    array([[ True,  True,  True],\n           [ True,  True,  True]])\n    >>> np.greater_equal(1, np.ones(1))\n    array([True])\n    \"\"\"\n    return _mx_nd_np.greater_equal(x1, x2, out)\n\n\n@set_module('mxnet.numpy')\ndef less_equal(x1, x2, out=None):\n    \"\"\"\n    Return the truth value of (x1 <= x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : ndarray or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    equal, greater, greater_equal, less, less_equal\n    Examples\n    --------\n    >>> np.less_equal(np.ones(2, 1)), np.zeros(1, 3))\n    array([[False, False, False],\n           [False, False, False]])\n    >>> np.less_equal(1, np.ones(1))\n    array([True])\n    \"\"\"\n    return _mx_nd_np.less_equal(x1, x2, out)\n\n\n@set_module('mxnet.numpy')\ndef roll(a, shift, axis=None):\n    \"\"\"\n    Roll array elements along a given axis.\n\n    Elements that roll beyond the last position are re-introduced at\n    the first.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array.\n    shift : int or tuple of ints\n        The number of places by which elements are shifted.  If a tuple,\n        then `axis` must be a tuple of the same size, and each of the\n        given axes is shifted by the corresponding number.  If an int\n        while `axis` is a tuple of ints, then the same value is used for\n        all given axes.\n    axis : int or tuple of ints, optional\n        Axis or axes along which elements are shifted.  By default, the\n        array is flattened before shifting, after which the original\n        shape is restored.\n\n    Returns\n    -------\n    res : ndarray\n        Output array, with the same shape as `a`.\n\n    Notes\n    -----\n    Supports rolling over multiple dimensions simultaneously.\n\n    Examples\n    --------\n    >>> x = np.arange(10)\n    >>> np.roll(x, 2)\n    array([8., 9., 0., 1., 2., 3., 4., 5., 6., 7.])\n    >>> np.roll(x, -2)\n    array([2., 3., 4., 5., 6., 7., 8., 9., 0., 1.])\n\n    >>> x2 = np.reshape(x, (2,5))\n    >>> x2\n    array([[0., 1., 2., 3., 4.],\n           [5., 6., 7., 8., 9.]])\n    >>> np.roll(x2, 1)\n    array([[9., 0., 1., 2., 3.],\n           [4., 5., 6., 7., 8.]])\n    >>> np.roll(x2, -1)\n    array([[1., 2., 3., 4., 5.],\n           [6., 7., 8., 9., 0.]])\n    >>> np.roll(x2, 1, axis=0)\n    array([[5., 6., 7., 8., 9.],\n           [0., 1., 2., 3., 4.]])\n    >>> np.roll(x2, -1, axis=0)\n    array([[5., 6., 7., 8., 9.],\n           [0., 1., 2., 3., 4.]])\n    >>> np.roll(x2, 1, axis=1)\n    array([[4., 0., 1., 2., 3.],\n           [9., 5., 6., 7., 8.]])\n    >>> np.roll(x2, -1, axis=1)\n    array([[1., 2., 3., 4., 0.],\n           [6., 7., 8., 9., 5.]])\n   \"\"\"\n    return _mx_nd_np.roll(a, shift, axis=axis)\n\n\n@set_module('mxnet.numpy')\ndef rot90(m, k=1, axes=(0, 1)):\n    \"\"\"\n    Rotate an array by 90 degrees in the plane specified by axes.\n    Rotation direction is from the first towards the second axis.\n\n    Parameters\n    ----------\n    m : ndarray\n        Array of two or more dimensions.\n    k : integer\n        Number of times the array is rotated by 90 degrees.\n    axes: (2,) array_like\n        The array is rotated in the plane defined by the axes.\n        Axes must be different.\n\n    Returns\n    -------\n    y : ndarray\n        A rotated view of `m`.\n\n    Notes\n    -----\n    rot90(m, k=1, axes=(1,0)) is the reverse of rot90(m, k=1, axes=(0,1))\n    rot90(m, k=1, axes=(1,0)) is equivalent to rot90(m, k=-1, axes=(0,1))\n\n    Examples\n    --------\n    >>> m = np.array([[1,2],[3,4]], 'int')\n    >>> m\n    array([[1, 2],\n           [3, 4]], dtype=int64)\n    >>> np.rot90(m)\n    array([[2, 4],\n           [1, 3]], dtype=int64)\n    >>> np.rot90(m, 2)\n    array([[4, 3],\n           [2, 1]], dtype=int64)\n    >>> m = np.arange(8).reshape((2,2,2))\n    >>> np.rot90(m, 1, (1,2))\n    array([[[1., 3.],\n            [0., 2.]],\n\n           [[5., 7.],\n            [4., 6.]]])\n    \"\"\"\n    return _mx_nd_np.rot90(m, k=k, axes=axes)\n\n\n@set_module('mxnet.numpy')\ndef hsplit(ary, indices_or_sections):\n    \"\"\"Split an array into multiple sub-arrays horizontally (column-wise).\n    This is equivalent to ``split`` with ``axis=0`` if ``ary`` has one\n    dimension, and otherwise that with ``axis=1``.\n\n    Parameters\n    ----------\n    ary : ndarray\n        Array to be divided into sub-arrays.\n    indices_or_sections : int, list of ints or tuple of ints.\n        If `indices_or_sections` is an integer, N, the array will be divided\n        into N equal arrays along `axis`.  If such a split is not possible,\n        an error is raised.\n        If `indices_or_sections` is a list of sorted integers, the entries\n        indicate where along `axis` the array is split.\n        If an index exceeds the dimension of the array along `axis`,\n        it will raises errors. so index must less than or euqal to\n        the dimension of the array along axis.\n\n    Returns\n    -------\n    sub-arrays : list of ndarrays\n        A list of sub-arrays.\n\n    .. note::\n       * If `indices_or_sections` is given as an integer, but a split\n         does not result in equal division.It will raises ValueErrors.\n       * If indices_or_sections is an integer, and the number is 1, it will\n         raises an error. Because single output from split is not supported yet...\n\n    See Also\n    --------\n    split : Split an array into multiple sub-arrays of equal size.\n\n    Examples\n    --------\n    >>> x = np.arange(16.0).reshape(4, 4)\n    >>> x\n    array([[ 0.,  1.,  2.,  3.],\n           [ 4.,  5.,  6.,  7.],\n           [ 8.,  9., 10., 11.],\n           [12., 13., 14., 15.]])\n    >>> np.hsplit(x, 2)\n    [array([[ 0.,  1.],\n           [ 4.,  5.],\n           [ 8.,  9.],\n           [12., 13.]]),\n    array([[ 2.,  3.],\n           [ 6.,  7.],\n           [10., 11.],\n           [14., 15.]])]\n    >>> np.hsplit(x, [3, 6])\n    [array([[ 0.,  1.,  2.],\n           [ 4.,  5.,  6.],\n           [ 8.,  9., 10.],\n           [12., 13., 14.]]),\n    array([[ 3.],\n           [ 7.],\n           [11.],\n           [15.]]),\n    array([], shape=(4, 0), dtype=float32)]\n    With a higher dimensional array the split is still along the second axis.\n    >>> x = np.arange(8.0).reshape(2, 2, 2)\n    >>> x\n    array([[[ 0.,  1.],\n            [ 2.,  3.]],\n           [[ 4.,  5.],\n            [ 6.,  7.]]])\n    >>> np.hsplit(x, 2)\n    [array([[[ 0.,  1.]],\n            [[ 4.,  5.]]]),\n     array([[[ 2.,  3.]],\n            [[ 6.,  7.]]])]\n    If ``ary`` has one dimension, 'axis' = 0.\n    >>> x = np.arange(4)\n    array([0., 1., 2., 3.])\n    >>> np.hsplit(x, 2)\n    [array([0., 1.]), array([2., 3.])]\n    If you want to produce an empty sub-array, you can see an example.\n    >>> np.hsplit(x, [2, 2])\n    [array([0., 1.]), array([], dtype=float32), array([2., 3.])]\n    \"\"\"\n    return _mx_nd_np.hsplit(ary, indices_or_sections)\n\n\n@set_module('mxnet.numpy')\ndef einsum(*operands, **kwargs):\n    r\"\"\"\n    einsum(subscripts, *operands, out=None, optimize=False)\n\n    Evaluates the Einstein summation convention on the operands.\n\n    Using the Einstein summation convention, many common multi-dimensional,\n    linear algebraic array operations can be represented in a simple fashion.\n    In *implicit* mode `einsum` computes these values.\n\n    In *explicit* mode, `einsum` provides further flexibility to compute\n    other array operations that might not be considered classical Einstein\n    summation operations, by disabling, or forcing summation over specified\n    subscript labels.\n\n    See the notes and examples for clarification.\n\n    Parameters\n    ----------\n    subscripts : str\n        Specifies the subscripts for summation as comma separated list of\n        subscript labels. An implicit (classical Einstein summation)\n        calculation is performed unless the explicit indicator '->' is\n        included as well as subscript labels of the precise output form.\n    operands : list of ndarray\n        These are the arrays for the operation.\n    out : ndarray, optional\n        If provided, the calculation is done into this array.\n    optimize : {False, True}, optional\n        Controls if intermediate optimization should occur. No optimization\n        will occur if False. Defaults to False.\n\n    Returns\n    -------\n    output : ndarray\n        The calculation based on the Einstein summation convention.\n\n    Notes\n    -----\n    The Einstein summation convention can be used to compute\n    many multi-dimensional, linear algebraic array operations. `einsum`\n    provides a succinct way of representing these.\n\n    A non-exhaustive list of these operations,\n    which can be computed by `einsum`, is shown below along with examples:\n\n    * Trace of an array, :py:func:`np.trace`.\n    * Return a diagonal, :py:func:`np.diag`.\n    * Array axis summations, :py:func:`np.sum`.\n    * Transpositions and permutations, :py:func:`np.transpose`.\n    * Matrix multiplication and dot product, :py:func:`np.matmul` :py:func:`np.dot`.\n    * Vector inner and outer products, :py:func:`np.inner` :py:func:`np.outer`.\n    * Broadcasting, element-wise and scalar multiplication, :py:func:`np.multiply`.\n    * Tensor contractions, :py:func:`np.tensordot`.\n\n    The subscripts string is a comma-separated list of subscript labels,\n    where each label refers to a dimension of the corresponding operand.\n    Whenever a label is repeated it is summed, so ``np.einsum('i,i', a, b)``\n    is equivalent to :py:func:`np.inner(a,b) <np.inner>`. If a label\n    appears only once, it is not summed, so ``np.einsum('i', a)`` produces a\n    view of ``a`` with no changes. A further example ``np.einsum('ij,jk', a, b)``\n    describes traditional matrix multiplication and is equivalent to\n    :py:func:`np.matmul(a,b) <np.matmul>`. Repeated subscript labels in one\n    operand take the diagonal. For example, ``np.einsum('ii', a)`` is equivalent\n    to :py:func:`np.trace(a) <np.trace>`.\n\n    In *implicit mode*, the chosen subscripts are important\n    since the axes of the output are reordered alphabetically.  This\n    means that ``np.einsum('ij', a)`` doesn't affect a 2D array, while\n    ``np.einsum('ji', a)`` takes its transpose. Additionally,\n    ``np.einsum('ij,jk', a, b)`` returns a matrix multiplication, while,\n    ``np.einsum('ij,jh', a, b)`` returns the transpose of the\n    multiplication since subscript 'h' precedes subscript 'i'.\n\n    In *explicit mode* the output can be directly controlled by\n    specifying output subscript labels.  This requires the\n    identifier '->' as well as the list of output subscript labels.\n    This feature increases the flexibility of the function since\n    summing can be disabled or forced when required. The call\n    ``np.einsum('i->', a)`` is like :py:func:`np.sum(a, axis=-1) <np.sum>`,\n    and ``np.einsum('ii->i', a)`` is like :py:func:`np.diag(a) <np.diag>`.\n    The difference is that `einsum` does not allow broadcasting by default.\n    Additionally ``np.einsum('ij,jh->ih', a, b)`` directly specifies the\n    order of the output subscript labels and therefore returns matrix\n    multiplication, unlike the example above in implicit mode.\n\n    To enable and control broadcasting, use an ellipsis.  Default\n    NumPy-style broadcasting is done by adding an ellipsis\n    to the left of each term, like ``np.einsum('...ii->...i', a)``.\n    To take the trace along the first and last axes,\n    you can do ``np.einsum('i...i', a)``, or to do a matrix-matrix\n    product with the left-most indices instead of rightmost, one can do\n    ``np.einsum('ij...,jk...->ik...', a, b)``.\n\n    When there is only one operand, no axes are summed, and no output\n    parameter is provided, a view into the operand is returned instead\n    of a new array.  Thus, taking the diagonal as ``np.einsum('ii->i', a)``\n    produces a view.\n\n    The ``optimize`` argument which will optimize the contraction order\n    of an einsum expression. For a contraction with three or more operands this\n    can greatly increase the computational efficiency at the cost of a larger\n    memory footprint during computation.\n\n    Typically a 'greedy' algorithm is applied which empirical tests have shown\n    returns the optimal path in the majority of cases. 'optimal' is not supported\n    for now.\n\n    .. note::\n       This function differs from the original `numpy.einsum\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.einsum.html>`_ in\n       the following way(s):\n\n       * Does not support 'optimal' strategy\n       * Does not support the alternative subscript like\n           `einsum(op0, sublist0, op1, sublist1, ..., [sublistout])`\n       * Does not produce view in any cases\n\n    Examples\n    --------\n    >>> a = np.arange(25).reshape(5,5)\n    >>> b = np.arange(5)\n    >>> c = np.arange(6).reshape(2,3)\n\n    Trace of a matrix:\n\n    >>> np.einsum('ii', a)\n    array(60.)\n\n    Extract the diagonal (requires explicit form):\n\n    >>> np.einsum('ii->i', a)\n    array([ 0.,  6., 12., 18., 24.])\n\n    Sum over an axis (requires explicit form):\n\n    >>> np.einsum('ij->i', a)\n    array([ 10.,  35.,  60.,  85., 110.])\n    >>> np.sum(a, axis=1)\n    array([ 10.,  35.,  60.,  85., 110.])\n\n    For higher dimensional arrays summing a single axis can be done with ellipsis:\n\n    >>> np.einsum('...j->...', a)\n    array([ 10.,  35.,  60.,  85., 110.])\n\n    Compute a matrix transpose, or reorder any number of axes:\n\n    >>> np.einsum('ji', c)\n    array([[0., 3.],\n           [1., 4.],\n           [2., 5.]])\n    >>> np.einsum('ij->ji', c)\n    array([[0., 3.],\n           [1., 4.],\n           [2., 5.]])\n    >>> np.transpose(c)\n    array([[0., 3.],\n           [1., 4.],\n           [2., 5.]])\n\n    Vector inner products:\n\n    >>> np.einsum('i,i', b, b)\n    array(30.)\n\n    Matrix vector multiplication:\n\n    >>> np.einsum('ij,j', a, b)\n    array([ 30.,  80., 130., 180., 230.])\n    >>> np.dot(a, b)\n    array([ 30.,  80., 130., 180., 230.])\n    >>> np.einsum('...j,j', a, b)\n    array([ 30.,  80., 130., 180., 230.])\n\n    Broadcasting and scalar multiplication:\n\n    >>> np.einsum('..., ...', np.array(3), c)\n    array([[ 0.,  3.,  6.],\n           [ 9., 12., 15.]])\n    >>> np.einsum(',ij', np.array(3), c)\n    array([[ 0.,  3.,  6.],\n           [ 9., 12., 15.]])\n    >>> np.multiply(3, c)\n    array([[ 0.,  3.,  6.],\n           [ 9., 12., 15.]])\n\n    Vector outer product:\n\n    >>> np.einsum('i,j', np.arange(2)+1, b)\n    array([[0., 1., 2., 3., 4.],\n           [0., 2., 4., 6., 8.]])\n\n    Tensor contraction:\n\n    >>> a = np.arange(60.).reshape(3,4,5)\n    >>> b = np.arange(24.).reshape(4,3,2)\n    >>> np.einsum('ijk,jil->kl', a, b)\n    array([[4400., 4730.],\n           [4532., 4874.],\n           [4664., 5018.],\n           [4796., 5162.],\n           [4928., 5306.]])\n\n    Example of ellipsis use:\n\n    >>> a = np.arange(6).reshape((3,2))\n    >>> b = np.arange(12).reshape((4,3))\n    >>> np.einsum('ki,jk->ij', a, b)\n    array([[10., 28., 46., 64.],\n           [13., 40., 67., 94.]])\n    >>> np.einsum('ki,...k->i...', a, b)\n    array([[10., 28., 46., 64.],\n           [13., 40., 67., 94.]])\n    >>> np.einsum('k...,jk', a, b)\n    array([[10., 28., 46., 64.],\n           [13., 40., 67., 94.]])\n\n    Chained array operations. For more complicated contractions, speed ups\n    might be achieved by repeatedly computing a 'greedy' path. Performance\n    improvements can be particularly significant with larger arrays:\n\n    >>> a = np.ones(64).reshape(2,4,8)\n    # Basic `einsum`: ~42.22ms  (benchmarked on 3.4GHz Intel Xeon.)\n    >>> for iteration in range(500):\n    ...     np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a)\n    # Greedy `einsum` (faster optimal path approximation): ~0.117ms\n    >>> for iteration in range(500):\n    ...     np.einsum('ijk,ilm,njm,nlk,abc->',a,a,a,a,a, optimize=True)\n    \"\"\"\n    return _mx_nd_np.einsum(*operands, **kwargs)\n\n\n@set_module('mxnet.numpy')\ndef insert(arr, obj, values, axis=None):\n    r\"\"\"Insert values along the given axis before the given indices.\n\n    Parameters\n    ----------\n    arr : ndarray\n        Input array.\n    obj : int, slice or ndarray of int64\n        Object that defines the index or indices before which `values` is\n        inserted.\n        Support for multiple insertions when `obj` is a single scalar or a\n        sequence with one element (only support int32 and int64 element).\n    values : ndarray\n        Values to insert into `arr`.\n        If the type of values is different from that of arr, values is converted\n        to the type of arr.\n    axis : int, optional\n        Axis along which to insert `values`.  If `axis` is None then `arr`\n        is flattened first.\n\n    Returns\n    -------\n    out : ndarray\n        A copy of `arr` with `values` inserted.  Note that `insert`\n        does not occur in-place: a new array is returned. If\n        `axis` is None, `out` is a flattened array.\n\n    .. note::\n       * Note that for higher dimensional inserts `obj=0` behaves very different\n         from `obj=[0]` just like `arr[:,0,:] = values` is different from\n         `arr[:,[0],:] = values`.\n       * If obj is a ndarray, it's dtype only supports int64\n\n    Examples\n    --------\n    >>> a = np.array([[1, 1], [2, 2], [3, 3]])\n    >>> a\n    array([[1., 1.],\n           [2., 2.],\n           [3., 3.]])\n    >>> np.insert(a, 1, np.array(5))\n    array([1., 5., 1., 2., 2., 3., 3.])\n    >>> np.insert(a, 1, np.array(5), axis=1)\n    array([[1., 5., 1.],\n           [2., 5., 2.],\n           [3., 5., 3.]])\n\n    Difference between sequence and scalars:\n\n    >>> np.insert(a, np.array([1], dtype=np.int64), np.array([[1],[2],[3]]), axis=1)\n    array([[1., 1., 1.],\n           [2., 2., 2.],\n           [3., 3., 3.]])\n    >>> np.insert(a, 1, np.array([1, 2, 3]), axis=1)\n    array([[1., 1., 1.],\n           [2., 2., 2.],\n           [3., 3., 3.]])\n\n    >>> b = a.flatten()\n    >>> b\n    array([1., 1., 2., 2., 3., 3.])\n    >>> np.insert(b, np.array([2, 2], dtype=np.int64), np.array([5, 6]))\n    array([1., 1., 5., 6., 2., 2., 3., 3.])\n\n    >>> np.insert(b, slice(2, 4), np.array([5, 6]))\n    array([1., 1., 5., 2., 6., 2., 3., 3.])\n\n    # type casting\n    >>> np.insert(b.astype(np.int32), np.array([2, 2],dtype='int64'), np.array([7.13, False]))\n    array([1, 1, 7, 0, 2, 2, 3, 3], dtype=int32)\n\n    >>> x = np.arange(8).reshape(2, 4)\n    >>> idx = np.array([1, 3], dtype=np.int64)\n    >>> np.insert(x, idx, np.array([999]), axis=1)\n    array([[  0., 999.,   1.,   2., 999.,   3.],\n           [  4., 999.,   5.,   6., 999.,   7.]])\n    \"\"\"\n    return _mx_nd_np.insert(arr, obj, values, axis=axis)\n\n\n@set_module('mxnet.numpy')\ndef nonzero(a):\n    \"\"\"\n    Return the indices of the elements that are non-zero.\n\n    Returns a tuple of arrays, one for each dimension of `a`,\n    containing the indices of the non-zero elements in that\n    dimension. The values in `a` are always returned in\n    row-major, C-style order.\n\n    To group the indices by element, rather than dimension, use `argwhere`,\n    which returns a row for each non-zero element.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array.\n\n    Returns\n    -------\n    tuple_of_arrays : tuple\n        Indices of elements that are non-zero.\n\n    See Also\n    --------\n    ndarray.nonzero :\n        Equivalent ndarray method.\n\n    Notes\n    -----\n    While the nonzero values can be obtained with ``a[nonzero(a)]``, it is\n    recommended to use ``x[x.astype(bool)]`` or ``x[x != 0]`` instead, which\n    will correctly handle 0-d arrays.\n\n    Examples\n    --------\n    >>> x = np.array([[3, 0, 0], [0, 4, 0], [5, 6, 0]])\n    >>> x\n    array([[3, 0, 0],\n           [0, 4, 0],\n           [5, 6, 0]], dtype=int32)\n    >>> np.nonzero(x)\n    (array([0, 1, 2, 2], dtype=int64), array([0, 1, 0, 1], dtype=int64))\n\n    >>> x[np.nonzero(x)]\n    array([3, 4, 5, 6])\n    >>> np.transpose(np.stack(np.nonzero(x)))\n    array([[0, 0],\n           [1, 1],\n           [2, 0],\n           [2, 1]], dtype=int64)\n\n    A common use for ``nonzero`` is to find the indices of an array, where\n    a condition is True.  Given an array `a`, the condition `a` > 3 is a\n    boolean array and since False is interpreted as 0, np.nonzero(a > 3)\n    yields the indices of the `a` where the condition is true.\n\n    >>> a = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.int32)\n    >>> a > 3\n    array([[False, False, False],\n           [ True,  True,  True],\n           [ True,  True,  True]])\n    >>> np.nonzero(a > 3)\n    (array([1, 1, 1, 2, 2, 2], dtype=int64), array([0, 1, 2, 0, 1, 2], dtype=int64))\n\n    Using this result to index `a` is equivalent to using the mask directly:\n\n    >>> a[np.nonzero(a > 3)]\n    array([4, 5, 6, 7, 8, 9], dtype=int32)\n    >>> a[a > 3]\n    array([4, 5, 6, 7, 8, 9], dtype=int32)\n\n    ``nonzero`` can also be called as a method of the array.\n\n    >>> (a > 3).nonzero()\n    (array([1, 1, 1, 2, 2, 2], dtype=int64), array([0, 1, 2, 0, 1, 2], dtype=int64))\n    \"\"\"\n    return _mx_nd_np.nonzero(a)\n\n\n@set_module('mxnet.numpy')\ndef percentile(a, q, axis=None, out=None, overwrite_input=None, interpolation='linear', keepdims=False): # pylint: disable=too-many-arguments\n    \"\"\"\n    Compute the q-th percentile of the data along the specified axis.\n    Returns the q-th percentile(s) of the array elements.\n\n    Parameters\n    ----------\n    a : array_like\n        Input array\n    q : array_like\n        Percentile or sequence of percentiles to compute.\n    axis : {int, tuple of int, None}, optional\n        Axis or axes along which the percentiles are computed. The default is to\n        compute the percentile(s) along a flattened version of the array.\n    out : ndarray, optional\n        Alternative output array in which to place the result. It must have the same\n        shape and buffer length as the expected output, but the type (of the output)\n        will be cast if necessary.\n    overwrite_input : bool, optional (Not supported yet)\n        If True, then allow the input array a to be modified by intermediate calculations,\n        to save memory. In this case, the contents of the input a after this function\n        completes is undefined.\n    interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}\n        This optional parameter specifies the interpolation method to use when the\n        desired percentile lies between two data points i < j:\n        'linear': i + (j - i) * fraction, where fraction is the fractional part of the\n        index surrounded by i and j.\n        'lower': i.\n        'higher': j.\n        'nearest': i or j, whichever is nearest.\n        'midpoint': (i + j) / 2.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in the result as\n        dimensions with size one. With this option, the result will broadcast\n        correctly against the original array a.\n\n    Returns\n    -------\n    percentile : scalar or ndarray\n        Output array.\n\n    Examples\n    --------\n    >>> a = np.array([[10, 7, 4], [3, 2, 1]])\n    >>> a\n    array([[10,  7,  4],\n        [ 3,  2,  1]])\n    >>> np.percentile(a, np.array(50))\n    array(3.5)\n    >>> np.percentile(a, np.array(50), axis=0)\n    array([6.5, 4.5, 2.5])\n    >>> np.percentile(a, np.array(50), axis=1)\n    array([7.,  2.])\n    >>> np.percentile(a, np.array(50), axis=1, keepdims=True)\n    array([[7.],\n        [2.]])\n\n    >>> m = np.percentile(a, np.array(50), axis=0)\n    >>> out = np.zeros_like(m)\n    >>> np.percentile(a, np.array(50), axis=0, out=out)\n    array([6.5, 4.5, 2.5])\n    >>> m\n    array([6.5, 4.5, 2.5])\n    \"\"\"\n    return _mx_nd_np.percentile(a, q, axis=axis, out=out, overwrite_input=overwrite_input,\n                                interpolation=interpolation, keepdims=keepdims)\n\n\n@set_module('mxnet.numpy')\ndef median(a, axis=None, out=None, overwrite_input=None, keepdims=False):\n    r\"\"\"Compute the median along the specified axis.\n    Returns the median of the array elements.\n\n    Parameters\n    ----------\n    a : array_like\n        Input array or object that can be converted to an array.\n    axis : {int, sequence of int, None}, optional\n        Axis or axes along which the medians are computed. The default\n        is to compute the median along a flattened version of the array.\n        A sequence of axes is supported since version 1.9.0.\n    out : ndarray, optional\n        Alternative output array in which to place the result. It must\n        have the same shape and buffer length as the expected output,\n        but the type (of the output) will be cast if necessary.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the original `arr`.\n\n    Returns\n    -------\n    median : ndarray\n        A new array holding the result. If the input contains integers\n        or floats smaller than ``float32``, then the output data-type is\n        ``np.float32``.  Otherwise, the data-type of the output is the\n        same as that of the input. If `out` is specified, that array is\n        returned instead.\n\n    See Also\n    --------\n    mean, percentile\n\n    Examples\n    --------\n    >>> a = np.array([[10, 7, 4], [3, 2, 1]])\n    >>> a\n    array([[10,  7,  4],\n        [ 3,  2,  1]])\n    >>> np.median(a)\n    3.5\n    >>> np.median(a, axis=0)\n    array([6.5, 4.5, 2.5])\n    >>> np.median(a, axis=1)\n    array([7.,  2.])\n    \"\"\"\n    return _mx_nd_np.median(a, axis=axis, overwrite_input=overwrite_input,\n                            keepdims=keepdims, out=out)\n\n\n@set_module('mxnet.numpy')\ndef quantile(a, q, axis=None, out=None, overwrite_input=None, interpolation='linear', keepdims=False): # pylint: disable=too-many-arguments\n    \"\"\"Compute the q-th quantile of the data along the specified axis.\n    New in version 1.15.0.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array or object that can be converted to an array.\n    q : ndarray\n        Quantile or sequence of quantiles to compute, which must be between 0 and 1 inclusive.\n    axis : {int, tuple of int, None}, optional\n        Axis or axes along which the quantiles are computed.\n        The default is to compute the quantile(s) along a flattened version of the array.\n    out : ndarray, optional\n        Alternative output array in which to place the result.\n        It must have the same shape and buffer length as the expected output,\n        but the type (of the output) will be cast if necessary.\n    interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}\n        This optional parameter specifies the interpolation method to use\n        when the desired quantile lies between two data points i < j:\n\n        * linear: i + (j - i) * fraction, where fraction is the fractional part of the index surrounded by i and j.\n        * lower: i.\n        * higher: j.\n        * nearest: i or j, whichever is nearest.\n        * midpoint: (i + j) / 2.\n\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in the result as dimensions with size one.\n        With this option, the result will broadcast correctly against the original array a.\n\n    Returns\n    -------\n    quantile : ndarray\n        If q is a single quantile and axis=None, then the result is a scalar.\n        If multiple quantiles are given, first axis of the result corresponds to the quantiles.\n        The other axes are the axes that remain after the reduction of a.\n        If out is specified, that array is returned instead.\n\n    See also\n    --------\n    mean\n\n    .. note::\n       Given a vector V of length N, the q-th quantile of V is the value q of the way from the minimum\n       to the maximum in a sorted copy of V. The values and distances of the two nearest neighbors\n       as well as the interpolation parameter will determine the quantile if the normalized ranking\n       does not match the location of q exactly. This function is the same as the median if q=0.5,\n       the same as the minimum if q=0.0 and the same as the maximum if q=1.0.\n       This function differs from the original `numpy.quantile\n       <https://numpy.org/devdocs/reference/generated/numpy.quantile.html>`_ in\n       the following aspects:\n\n       * q must be ndarray type even if it is a scalar\n       * do not support overwrite_input\n\n    Examples\n    --------\n    >>> a = np.array([[10, 7, 4], [3, 2, 1]])\n    >>> a\n    array([[10., 7., 4.],\n           [3., 2., 1.]])\n    >>> q = np.array(0.5)\n    >>> q\n    array(0.5)\n    >>> np.quantile(a, q)\n    array(3.5)\n    >>> np.quantile(a, q, axis=0)\n    array([6.5, 4.5, 2.5])\n    >>> np.quantile(a, q, axis=1)\n    array([7., 2.])\n    >>> np.quantile(a, q, axis=1, keepdims=True)\n    array([[7.],\n           [2.]])\n    >>> m = np.quantile(a, q, axis=0)\n    >>> out = np.zeros_like(m)\n    >>> np.quantile(a, q, axis=0, out=out)\n    array([6.5, 4.5, 2.5])\n    >>> out\n    array([6.5, 4.5, 2.5])\n    \"\"\"\n    return _mx_nd_np.quantile(a, q, axis=axis, out=out, overwrite_input=overwrite_input,\n                              interpolation=interpolation, keepdims=keepdims)\n\n\n@set_module('mxnet.numpy')\ndef shares_memory(a, b, max_work=None):\n    \"\"\"\n    Determine if two arrays share memory\n\n    Parameters\n    ----------\n    a, b : ndarray\n        Input arrays\n\n    Returns\n    -------\n    out : bool\n\n    See Also\n    --------\n    may_share_memory\n\n    Examples\n    --------\n    >>> np.may_share_memory(np.array([1,2]), np.array([5,8,9]))\n    False\n\n    .. note::\n       This function differs from the original `numpy.shares_memory\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.shares_memory.html>`_ in\n       the following way(s):\n\n       * Does not support `max_work`, it is a dummy argument\n       * Actually it is same as `may_share_memory` in MXNet np\n    \"\"\"\n    return _mx_nd_np.shares_memory(a, b, max_work)\n\n\n@set_module('mxnet.numpy')\ndef may_share_memory(a, b, max_work=None):\n    \"\"\"\n    Determine if two arrays might share memory\n\n    A return of True does not necessarily mean that the two arrays\n    share any element.  It just means that they *might*.\n\n    Only the memory bounds of a and b are checked by default.\n\n    Parameters\n    ----------\n    a, b : ndarray\n        Input arrays\n\n    Returns\n    -------\n    out : bool\n\n    See Also\n    --------\n    shares_memory\n\n    Examples\n    --------\n    >>> np.may_share_memory(np.array([1,2]), np.array([5,8,9]))\n    False\n    >>> x = np.zeros([3, 4])\n    >>> np.may_share_memory(x[:,0], x[:,1])\n    True\n\n    .. note::\n       This function differs from the original `numpy.may_share_memory\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.may_share_memory.html>`_ in\n       the following way(s):\n\n       * Does not support `max_work`, it is a dummy argument\n       * Actually it is same as `shares_memory` in MXNet np\n    \"\"\"\n    return _mx_nd_np.may_share_memory(a, b, max_work)\n\n\n@set_module('mxnet.numpy')\ndef diff(a, n=1, axis=-1, prepend=None, append=None):  # pylint: disable=redefined-outer-name\n    r\"\"\"\n    Calculate the n-th discrete difference along the given axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array\n    n : int, optional\n        The number of times values are differenced. If zero, the input is returned as-is.\n    axis : int, optional\n        The axis along which the difference is taken, default is the last axis.\n    prepend, append : ndarray, optional\n        Not supported yet\n\n    Returns\n    -------\n    diff : ndarray\n        The n-th differences.\n        The shape of the output is the same as a except along axis where the dimension is smaller by n.\n        The type of the output is the same as the type of the difference between any two elements of a.\n        This is the same as the type of a in most cases.\n\n    Examples\n    --------\n    >>> x = np.array([1, 2, 4, 7, 0])\n    >>> np.diff(x)\n    array([ 1,  2,  3, -7])\n    >>> np.diff(x, n=2)\n    array([  1,   1, -10])\n\n    >>> x = np.array([[1, 3, 6, 10], [0, 5, 6, 8]])\n    >>> np.diff(x)\n    array([[2, 3, 4],\n        [5, 1, 2]])\n    >>> np.diff(x, axis=0)\n    array([[-1,  2,  0, -2]])\n\n    Notes\n    -----\n    Optional inputs `prepend` and `append` are not supported yet\n    \"\"\"\n    if (prepend or append):\n        raise NotImplementedError('prepend and append options are not supported yet')\n    return _mx_nd_np.diff(a, n=n, axis=axis)\n\n\n@set_module('mxnet.numpy')\ndef ediff1d(ary, to_end=None, to_begin=None):\n    \"\"\"\n    The differences between consecutive elements of an array.\n\n    Parameters\n    ----------\n    ary : ndarray\n        If necessary, will be flattened before the differences are taken.\n    to_end : ndarray or scalar, optional\n        Number(s) to append at the end of the returned differences.\n    to_begin : ndarray or scalar, optional\n        Number(s) to prepend at the beginning of the returned differences.\n\n    Returns\n    -------\n    ediff1d : ndarray\n        The differences. Loosely, this is ``ary.flat[1:] - ary.flat[:-1]``.\n\n    Examples\n    --------\n    >>> x = np.array([1, 2, 4, 7, 0])\n    >>> np.ediff1d(x)\n    array([ 1.,  2.,  3., -7.])\n\n    >>> np.ediff1d(x, to_begin=-99, to_end=np.array([88, 99]))\n    rray([-99.,   1.,   2.,   3.,  -7.,  88.,  99.])\n\n    The returned array is always 1D.\n\n    >>> y = np.array([[1, 2, 4], [1, 6, 24]])\n    >>> np.ediff1d(y)\n    array([ 1.,  2., -3.,  5., 18.])\n\n    >>> np.ediff1d(x, to_begin=y)\n    array([ 1.,  2.,  4.,  1.,  6., 24.,  1.,  2.,  3., -7.])\n    \"\"\"\n    return _mx_nd_np.ediff1d(ary, to_end=to_end, to_begin=to_begin)\n\n\n@set_module('mxnet.numpy')\ndef resize(a, new_shape):\n    \"\"\"\n    Return a new array with the specified shape.\n    If the new array is larger than the original array, then the new\n    array is filled with repeated copies of `a`.  Note that this behavior\n    is different from a.resize(new_shape) which fills with zeros instead\n    of repeated copies of `a`.\n\n    Parameters\n    ----------\n    a : ndarray\n        Array to be resized.\n    new_shape : int or tuple of int\n        Shape of resized array.\n\n    Returns\n    -------\n    reshaped_array : ndarray\n        The new array is formed from the data in the old array, repeated\n        if necessary to fill out the required number of elements.  The\n        data are repeated in the order that they are stored in memory.\n\n    See Also\n    --------\n    ndarray.resize : resize an array in-place.\n\n    Notes\n    -----\n    Warning: This functionality does **not** consider axes separately,\n    i.e. it does not apply interpolation/extrapolation.\n    It fills the return array with the required number of elements, taken\n    from `a` as they are laid out in memory, disregarding strides and axes.\n    (This is in case the new shape is smaller. For larger, see above.)\n    This functionality is therefore not suitable to resize images,\n    or data where each axis represents a separate and distinct entity.\n\n    Examples\n    --------\n    >>> a = np.array([[0, 1], [2, 3]])\n    >>> np.resize(a, (2, 3))\n    array([[0., 1., 2.],\n           [3., 0., 1.]])\n    >>> np.resize(a, (1, 4))\n    array([[0., 1., 2., 3.]])\n    >>> np.resize(a,(2, 4))\n    array([[0., 1., 2., 3.],\n           [0., 1., 2., 3.]])\n    \"\"\"\n    return _mx_nd_np.resize(a, new_shape)\n\n\n@set_module('mxnet.numpy')\ndef interp(x, xp, fp, left=None, right=None, period=None):  # pylint: disable=too-many-arguments\n    r\"\"\"One-dimensional linear interpolation.\n\n    Returns the one-dimensional piecewise linear interpolant to a function\n    with given values at discrete data-points.\n\n    Parameters\n    ----------\n    x : ndarray\n        The x-coordinates of the interpolated values.\n    xp : 1-D array of floats\n        The x-coordinates of the data points, must be increasing if argument\n        `period` is not specified. Otherwise, `xp` is internally sorted after\n        normalizing the periodic boundaries with ``xp = xp % period``.\n    fp : 1-D array of floats\n        The y-coordinates of the data points, same length as `xp`.\n    left : optional float corresponding to fp\n        Value to return for `x < xp[0]`, default is `fp[0]`.\n    right : optional float corresponding to fp\n        Value to return for `x > xp[-1]`, default is `fp[-1]`.\n    period : None or float, optional\n        A period for the x-coordinates. This parameter allows the proper\n        interpolation of angular x-coordinates. Parameters `left` and `right`\n        are ignored if `period` is specified.\n\n    Returns\n    -------\n    y : float (corresponding to fp) or ndarray\n        The interpolated values, same shape as `x`.\n\n    Raises\n    ------\n    ValueError\n        If `xp` and `fp` have different length\n        If `xp` or `fp` are not 1-D sequences\n        If `period == 0`\n\n    .. note::\n       Does not check that the x-coordinate sequence `xp` is increasing.\n       If `xp` is not increasing, the results are nonsense.\n       A simple check for increasing is::\n\n           np.all(np.diff(xp) > 0)\n\n\n    Examples\n    --------\n    >>> xp = [1, 2, 3]\n    >>> fp = [3, 2, 0]\n    >>> np.interp(2.5, xp, fp)\n    1.0\n    >>> np.interp([0, 1, 1.5, 2.72, 3.14], xp, fp)\n    array([ 3. ,  3. ,  2.5 ,  0.56,  0. ])\n    >>> UNDEF = -99.0\n    >>> np.interp(3.14, xp, fp, right=UNDEF)\n    -99.0\n    Plot an interpolant to the sine function:\n    >>> x = np.linspace(0, 2*np.pi, 10)\n    >>> y = np.sin(x)\n    >>> xvals = np.linspace(0, 2*np.pi, 50)\n    >>> yinterp = np.interp(xvals, x, y)\n    >>> import matplotlib.pyplot as plt\n    >>> plt.plot(x, y, 'o')\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.plot(xvals, yinterp, '-x')\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.show()\n    Interpolation with periodic x-coordinates:\n    >>> x = [-180, -170, -185, 185, -10, -5, 0, 365]\n    >>> xp = [190, -190, 350, -350]\n    >>> fp = [5, 10, 3, 4]\n    >>> np.interp(x, xp, fp, period=360)\n    array([7.5, 5., 8.75, 6.25, 3., 3.25, 3.5, 3.75])\n    \"\"\"\n    return _mx_nd_np.interp(x, xp, fp, left=left, right=right, period=period)\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef full_like(a, fill_value, dtype=None, order='C', device=None, out=None): # pylint: disable=too-many-arguments\n    \"\"\"\n    Return a full array with the same shape and type as a given array.\n\n    Parameters\n    ----------\n    a : ndarray\n        The shape and data-type of `a` define these same attributes of\n        the returned array.\n    fill_value : scalar\n        Fill value.\n    dtype : data-type, optional\n        Overrides the data type of the result.\n        Temporarily do not support boolean type.\n    order : {'C'}, optional\n        Whether to store multidimensional data in C- or Fortran-contiguous\n        (row- or column-wise) order in memory. Currently only supports C order.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n        Array of `fill_value` with the same shape and type as `a`.\n\n    See Also\n    --------\n    empty_like : Return an empty array with shape and type of input.\n    ones_like : Return an array of ones with shape and type of input.\n    zeros_like : Return an array of zeros with shape and type of input.\n    full : Return a new array of given shape filled with value.\n\n    Examples\n    --------\n    >>> x = np.arange(6, dtype=int)\n    >>> np.full_like(x, 1)\n    array([1, 1, 1, 1, 1, 1], dtype=int64)\n    >>> np.full_like(x, 0.1)\n    array([0, 0, 0, 0, 0, 0], dtype=int64)\n    >>> np.full_like(x, 0.1, dtype=np.float64)\n    array([0.1, 0.1, 0.1, 0.1, 0.1, 0.1], dtype=float64)\n    >>> np.full_like(x, np.nan, dtype=np.float64)\n    array([nan, nan, nan, nan, nan, nan], dtype=float64)\n    >>> y = np.arange(6, dtype=np.float32)\n    >>> np.full_like(y, 0.1)\n    array([0.1, 0.1, 0.1, 0.1, 0.1, 0.1])\n    \"\"\"\n    return _mx_nd_np.full_like(a, fill_value=fill_value, dtype=dtype, order=order, device=device, out=out)\n# pylint: enable=redefined-outer-name\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef zeros_like(a, dtype=None, order='C', device=None, out=None):\n    \"\"\"\n    Return an array of zeros with the same shape and type as a given array.\n\n    Parameters\n    ----------\n    a : ndarray\n        The shape and data-type of `a` define these same attributes of\n        the returned array.\n    dtype : data-type, optional\n        Overrides the data type of the result.\n        Temporarily do not support boolean type.\n    order : {'C'}, optional\n        Whether to store multidimensional data in C- or Fortran-contiguous\n        (row- or column-wise) order in memory. Currently only supports C order.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n          Array of zeros with the same shape and type as a.\n\n    See Also\n    --------\n    empty_like : Return an empty array with shape and type of input.\n    ones_like : Return an array of ones with shape and type of input.\n    zeros_like : Return an array of zeros with shape and type of input.\n    full : Return a new array of given shape filled with value.\n\n    Examples\n    --------\n    >>> x = np.arange(6)\n    >>> x = x.reshape((2, 3))\n    >>> x\n    array([[0., 1., 2.],\n           [3., 4., 5.]])\n    >>> np.zeros_like(x)\n    array([[0., 0., 0.],\n           [0., 0., 0.]])\n    >>> np.zeros_like(x, int)\n    array([[0, 0, 0],\n           [0, 0, 0]], dtype=int64)\n    >>> y = np.arange(3, dtype=float)\n    >>> y\n    array([0., 1., 2.], dtype=float64)\n    >>> np.zeros_like(y)\n    array([0., 0., 0.], dtype=float64)\n    \"\"\"\n    return _mx_nd_np.full_like(a, fill_value=0, dtype=dtype, order=order, device=device, out=out)\n# pylint: enable=redefined-outer-name\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef ones_like(a, dtype=None, order='C', device=None, out=None):\n    \"\"\"\n    Return an array of ones with the same shape and type as a given array.\n\n    Parameters\n    ----------\n    a : ndarray\n        The shape and data-type of `a` define these same attributes of\n        the returned array.\n    dtype : data-type, optional\n        Overrides the data type of the result.\n        Temporarily do not support boolean type.\n    order : {'C'}, optional\n        Whether to store multidimensional data in C- or Fortran-contiguous\n        (row- or column-wise) order in memory. Currently only supports C order.\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n        Array of ones with the same shape and type as a.\n\n    See Also\n    --------\n    empty_like : Return an empty array with shape and type of input.\n    zeros_like : Return an array of zeros with shape and type of input.\n    full_like : Return a new array with shape of input filled with value.\n    ones : Return a new array setting values to one.\n\n    Examples\n    --------\n    >>> x = np.arange(6)\n    >>> x = x.reshape((2, 3))\n    >>> x\n    array([[0., 1., 2.],\n           [3., 4., 5.]])\n    >>> np.ones_like(x)\n    array([[1., 1., 1.],\n           [1., 1., 1.]])\n    >>> np.ones_like(x, int)\n    array([[1, 1, 1],\n           [1, 1, 1]], dtype=int64)\n    >>> y = np.arange(3, dtype=float)\n    >>> y\n    array([0., 1., 2.], dtype=float64)\n    >>> np.ones_like(y)\n    array([1., 1., 1.], dtype=float64)\n    \"\"\"\n    return _mx_nd_np.full_like(a, fill_value=1, dtype=dtype, order=order, device=device, out=out)\n# pylint: enable=redefined-outer-name\n\n\n@set_module('mxnet.numpy')\ndef fill_diagonal(a, val, wrap=False):\n    \"\"\"\n    Fill the main diagonal of the given array of any dimensionality.\n    For an array `a` with ``a.ndim >= 2``, the diagonal is the list of\n    locations with indices ``a[i, ..., i]`` all identical. This function\n    modifies the input array in-place, it does not return a value.\n    Parameters\n    ----------\n    a : array, at least 2-D.\n      Array whose diagonal is to be filled, it gets modified in-place.\n    val : scalar\n      Value to be written on the diagonal, its type must be compatible with\n      that of the array a.\n    wrap : bool\n      For tall matrices in NumPy version up to 1.6.2, the\n      diagonal \"wrapped\" after N columns. You can have this behavior\n      with this option. This affects only tall matrices.\n\n    Examples\n    --------\n    >>> a = np.zeros((3, 3), int)\n    >>> np.fill_diagonal(a, 5)\n    >>> a\n    array([[5, 0, 0],\n           [0, 5, 0],\n           [0, 0, 5]])\n    The same function can operate on a 4-D array:\n    >>> a = np.zeros((3, 3, 3, 3), int)\n    >>> np.fill_diagonal(a, 4)\n    We only show a few blocks for clarity:\n    >>> a[0, 0]\n    array([[4, 0, 0],\n           [0, 0, 0],\n           [0, 0, 0]])\n    >>> a[1, 1]\n    array([[0, 0, 0],\n           [0, 4, 0],\n           [0, 0, 0]])\n    >>> a[2, 2]\n    array([[0, 0, 0],\n           [0, 0, 0],\n           [0, 0, 4]])\n    The wrap option affects only tall matrices:\n    >>> # tall matrices no wrap\n    >>> a = np.zeros((5, 3), int)\n    >>> np.fill_diagonal(a, 4)\n    >>> a\n    array([[4, 0, 0],\n           [0, 4, 0],\n           [0, 0, 4],\n           [0, 0, 0],\n           [0, 0, 0]])\n    >>> # tall matrices wrap\n    >>> a = np.zeros((5, 3), int)\n    >>> np.fill_diagonal(a, 4, wrap=True)\n    >>> a\n    array([[4, 0, 0],\n           [0, 4, 0],\n           [0, 0, 4],\n           [0, 0, 0],\n           [4, 0, 0]])\n    >>> # wide matrices\n    >>> a = np.zeros((3, 5), int)\n    >>> np.fill_diagonal(a, 4, wrap=True)\n    >>> a\n    array([[4, 0, 0, 0, 0],\n           [0, 4, 0, 0, 0],\n           [0, 0, 4, 0, 0]])\n    The anti-diagonal can be filled by reversing the order of elements\n    using either `numpy.flipud` or `numpy.fliplr`.\n    >>> a = np.zeros((3, 3), int);\n    >>> np.fill_diagonal(np.fliplr(a), [1,2,3])  # Horizontal flip\n    >>> a\n    array([[0, 0, 1],\n           [0, 2, 0],\n           [3, 0, 0]])\n    >>> np.fill_diagonal(np.flipud(a), [1,2,3])  # Vertical flip\n    >>> a\n    array([[0, 0, 3],\n           [0, 2, 0],\n           [1, 0, 0]])\n    Note that the order in which the diagonal is filled varies depending\n    on the flip function.\n    \"\"\"\n    _mx_nd_np.fill_diagonal(a, val=val, wrap=wrap)\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\ndef nan_to_num(x, copy=True, nan=0.0, posinf=None, neginf=None, **kwargs):\n    \"\"\"\n    Replace NaN with zero and infinity with large finite numbers (default\n    behaviour) or with the numbers defined by the user using the `nan`,\n    `posinf` and/or `neginf` keywords.\n\n    If `x` is inexact, NaN is replaced by zero or by the user defined value in\n    `nan` keyword, infinity is replaced by the largest finite floating point\n    values representable by ``x.dtype`` or by the user defined value in\n    `posinf` keyword and -infinity is replaced by the most negative finite\n    floating point values representable by ``x.dtype`` or by the user defined\n    value in `neginf` keyword.\n\n    For complex dtypes, the above is applied to each of the real and\n    imaginary components of `x` separately.\n\n    If `x` is not inexact, then no replacements are made.\n\n    Parameters\n    ----------\n    x : scalar\n        ndarray\n        Input data.\n    copy : bool, optional\n        Whether to create a copy of `x` (True) or to replace values\n        in-place (False). The in-place operation only occurs if\n        casting to an array does not require a copy.\n        Default is True.\n        Gluon does not support copy = False.\n    nan : int, float, optional\n        Value to be used to fill NaN values. If no value is passed\n        then NaN values will be replaced with 0.0.\n    posinf : int, float, optional\n        Value to be used to fill positive infinity values. If no value is\n        passed then positive infinity values will be replaced with a very\n        large number.\n    neginf : int, float, optional\n        Value to be used to fill negative infinity values. If no value is\n        passed then negative infinity values will be replaced with a very\n        small (or negative) number.\n\n        .. versionadded:: 1.13\n\n    Returns\n    -------\n    out : ndarray\n        `x`, with the non-finite values replaced. If `copy` is False, this may\n        be `x` itself.\n\n    Notes\n    -----\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic\n    (IEEE 754). This means that Not a Number is not equivalent to infinity.\n\n    Examples\n    --------\n    >>> np.nan_to_num(np.inf)\n    1.7976931348623157e+308\n    >>> np.nan_to_num(-np.inf)\n    -1.7976931348623157e+308\n    >>> np.nan_to_num(np.nan)\n    0.0\n    >>> x = np.array([np.inf, -np.inf, np.nan, -128, 128])\n    >>> np.nan_to_num(x)\n    array([ 3.4028235e+38, -3.4028235e+38,  0.0000000e+00, -1.2800000e+02,\n            1.2800000e+02])\n    >>> np.nan_to_num(x, nan=-9999, posinf=33333333, neginf=33333333)\n    array([ 3.3333332e+07,  3.3333332e+07, -9.9990000e+03, -1.2800000e+02,\n            1.2800000e+02])\n    >>> y = np.array([[-1, 0, 1],[9999,234,-14222]],dtype=\"float64\")/0\n    array([[-inf,  nan,  inf],\n        [ inf,  inf, -inf]], dtype=float64)\n    >>> np.nan_to_num(y)\n    array([[-1.79769313e+308,  0.00000000e+000,  1.79769313e+308],\n        [ 1.79769313e+308,  1.79769313e+308, -1.79769313e+308]], dtype=float64)\n    >>> np.nan_to_num(y, nan=111111, posinf=222222)\n    array([[-1.79769313e+308,  1.11111000e+005,  2.22222000e+005],\n        [ 2.22222000e+005,  2.22222000e+005, -1.79769313e+308]], dtype=float64)\n    >>> y\n    array([[-inf,  nan,  inf],\n       [ inf,  inf, -inf]], dtype=float64)\n    >>> np.nan_to_num(y, copy=False, nan=111111, posinf=222222)\n    array([[-1.79769313e+308,  1.11111000e+005,  2.22222000e+005],\n       [ 2.22222000e+005,  2.22222000e+005, -1.79769313e+308]], dtype=float64)\n    >>> y\n    array([[-1.79769313e+308,  1.11111000e+005,  2.22222000e+005],\n       [ 2.22222000e+005,  2.22222000e+005, -1.79769313e+308]], dtype=float64)\n    \"\"\"\n    return _mx_nd_np.nan_to_num(x, copy=copy, nan=nan, posinf=posinf, neginf=neginf)\n\n\n@set_module('mxnet.numpy')\ndef squeeze(x, axis=None):\n    r\"\"\"Remove single-dimensional entries from the shape of an array.\n\n    Parameters\n    ----------\n    a : array_like\n        Input data.\n    axis : None or int or tuple of ints, optional\n        Selects a subset of the single-dimensional entries in the\n        shape. If an axis is selected with shape entry greater than\n        one, an error is raised.\n\n    Returns\n    -------\n    squeezed : ndarray\n        The input array, but with all or a subset of the\n        dimensions of length 1 removed. This is always `a` itself\n        or a view into `a`.\n\n    Raises\n    ------\n    ValueError\n        If `axis` is not `None`, and an axis being squeezed is not of length 1\n\n    See Also\n    --------\n    expand_dims : The inverse operation, adding singleton dimensions\n    reshape : Insert, remove, and combine dimensions, and resize existing ones\n\n    Examples\n    --------\n    >>> x = np.array([[[0], [1], [2]]])\n    >>> x.shape\n    (1, 3, 1)\n    >>> np.squeeze(x).shape\n    (3,)\n    >>> np.squeeze(x, axis=0).shape\n    (3, 1)\n    >>> np.squeeze(x, axis=1).shape\n    Traceback (most recent call last):\n    ...\n    ValueError: cannot select an axis to squeeze out which has size not equal to one\n    >>> np.squeeze(x, axis=2).shape\n    (1, 3)\n    \"\"\"\n    return _mx_nd_np.squeeze(x, axis=axis)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef isnan(x, out=None, **kwargs):\n    \"\"\"\n    Test element-wise for NaN and return result as a boolean array.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or bool\n        True where x is NaN, false otherwise.\n        This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic (IEEE 754).\n\n    .. note::\n\n       This function differs from the original `numpy.isinf\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.isnan.html>`_ in\n       the following aspects:\n\n       * Does not support complex number for now\n       * Input type does not support Python native iterables(list, tuple, ...).\n       * ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be\n         the same as the expected output.\n       * ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the\n         same as the expected output.\n       * ``out`` param does not support scalar input case.\n\n    Examples\n    --------\n    >>> np.isnan(np.nan)\n    True\n    >>> np.isnan(np.inf)\n    False\n    >>> np.isnan(np.array([np.log(-1.),1.,np.log(0)]))\n    array([ True, False, False])\n    \"\"\"\n    return _mx_nd_np.isnan(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef isinf(x, out=None, **kwargs):\n    \"\"\"\n    Test element-wise for positive or negative infinity.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or bool\n        True where x is positive or negative infinity, false otherwise.\n        This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic (IEEE 754).\n    This means that Not a Number is not equivalent to infinity.\n\n    .. note::\n\n       This function differs from the original `numpy.isnan\n       <https://docs.scipy.org/doc/numpy/reference/generated/numpy.isnan.html>`_ in\n       the following aspects:\n\n       * Does not support complex number for now\n       * Input type does not support Python native iterables(list, tuple, ...).\n       * ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be\n         the same as the expected output.\n       * ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the\n         same as the expected output.\n       * ``out`` param does not support scalar input case.\n\n    Examples\n    --------\n    >>> np.isinf(np.inf)\n    True\n    >>> np.isinf(np.nan)\n    False\n    >>> np.isinf(np.array([np.inf, -np.inf, 1.0, np.nan]))\n    array([ True,  True, False, False])\n    >>> x = np.array([-np.inf, 0., np.inf])\n    >>> y = np.array([True, True, True], dtype=np.bool_)\n    >>> np.isinf(x, y)\n    array([ True, False,  True])\n    >>> y\n    array([ True, False,  True])\n    \"\"\"\n    return _mx_nd_np.isinf(x, out=out, **kwargs)\n\n\n@set_module('mxnet.ndarray.numpy')\n@wrap_np_unary_func\ndef isposinf(x, out=None, **kwargs):\n    \"\"\"\n    Test element-wise for positive infinity, return result as bool array.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or bool\n        True where x is positive infinity, false otherwise.\n        This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic (IEEE 754).\n    This means that Not a Number is not equivalent to infinity.\n\n    Examples\n    --------\n    >>> np.isposinf(np.inf)\n    True\n    >>> np.isposinf(-np.inf)\n    False\n    >>> np.isposinf(np.nan)\n    False\n    >>> np.isposinf(np.array([-np.inf, 0., np.inf]))\n    array([False, False,  True])\n    >>> x = np.array([-np.inf, 0., np.inf])\n    >>> y = np.array([True, True, True], dtype=np.bool)\n    >>> np.isposinf(x, y)\n    array([False, False,  True])\n    >>> y\n    array([False, False,  True])\n    \"\"\"\n    return _mx_nd_np.isposinf(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef isneginf(x, out=None, **kwargs):\n    \"\"\"\n    Test element-wise for negative infinity, return result as bool array.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or bool\n        True where x is negative infinity, false otherwise.\n        This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic (IEEE 754).\n    This means that Not a Number is not equivalent to infinity.\n\n    Examples\n    --------\n    >>> np.isneginf(-np.inf)\n    True\n    >>> np.isneginf(np.inf)\n    False\n    >>> np.isneginf(float('-inf'))\n    True\n    >>> np.isneginf(np.array([-np.inf, 0., np.inf]))\n    array([ True, False, False])\n    >>> x = np.array([-np.inf, 0., np.inf])\n    >>> y = np.array([True, True, True], dtype=np.bool)\n    >>> np.isneginf(x, y)\n    array([ True, False, False])\n    >>> y\n    array([ True, False, False])\n    \"\"\"\n    return _mx_nd_np.isneginf(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\n@wrap_np_unary_func\ndef isfinite(x, out=None, **kwargs):\n    \"\"\"\n    Test element-wise for finiteness (not infinity or not Not a Number).\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : ndarray or bool\n        True where x is negative infinity, false otherwise.\n        This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    Not a Number, positive infinity and negative infinity are considered to be non-finite.\n\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic (IEEE 754).\n    This means that Not a Number is not equivalent to infinity.\n    Also that positive infinity is not equivalent to negative infinity.\n    But infinity is equivalent to positive infinity. Errors result if the second argument\n    is also supplied when x is a scalar input, or if first and second arguments have different shapes.\n\n    Examples\n    --------\n    >>> np.isfinite(1)\n    True\n    >>> np.isfinite(0)\n    True\n    >>> np.isfinite(np.nan)\n    False\n    >>> np.isfinite(np.inf)\n    False\n    >>> np.isfinite(-np.inf)\n    False\n    >>> np.isfinite(np.array([np.log(-1.),1.,np.log(0)]))\n    array([False,  True, False])\n    >>> x = np.array([-np.inf, 0., np.inf])\n    >>> y = np.array([True, True, True], dtype=np.bool)\n    >>> np.isfinite(x, y)\n    array([False,  True, False])\n    >>> y\n    array([False,  True, False])\n    \"\"\"\n    return _mx_nd_np.isfinite(x, out=out, **kwargs)\n\n\n@set_module('mxnet.numpy')\ndef where(condition, x=None, y=None):\n    \"\"\"where(condition, [x, y])\n    Return elements chosen from `x` or `y` depending on `condition`.\n\n    .. note::\n        When only `condition` is provided, this function is a shorthand for\n        ``np.asarray(condition).nonzero()``. The rest of this documentation\n        covers only the case where all three arguments are provided.\n\n    Parameters\n    ----------\n    condition : ndarray\n        Where True, yield `x`, otherwise yield `y`.\n    x, y : ndarray\n        Values from which to choose. `x`, `y` and `condition` need to be\n        broadcastable to some shape. `x` and `y` must have the same dtype.\n\n    Returns\n    -------\n    out : ndarray\n        An array with elements from `x` where `condition` is True, and elements\n        from `y` elsewhere.\n\n    Notes\n    -----\n    If all the arrays are 1-D, `where` is equivalent to::\n\n        [xv if c else yv\n        for c, xv, yv in zip(condition, x, y)]\n\n    Examples\n    --------\n    >>> a = np.arange(10)\n    >>> a\n    array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.])\n    >>> np.where(a < 5, a, 10*a)\n    array([ 0.,  1.,  2.,  3.,  4., 50., 60., 70., 80., 90.])\n\n    This can be used on multidimensional arrays too:\n\n    >>> cond = np.array([[True, False], [True, True]])\n    >>> x = np.array([[1, 2], [3, 4]])\n    >>> y = np.array([[9, 8], [7, 6]])\n    >>> np.where(cond, x, y)\n    array([[1., 8.],\n           [3., 4.]])\n\n    The shapes of x, y, and the condition are broadcast together:\n\n    >>> x, y = onp.ogrid[:3, :4]\n    >>> x = np.array(x)\n    >>> y = np.array(y)\n    >>> np.where(x < y, x, 10 + y)  # both x and 10+y are broadcast\n    array([[10,  0,  0,  0],\n           [10, 11,  1,  1],\n           [10, 11, 12,  2]], dtype=int64)\n\n    >>> a = np.array([[0, 1, 2],\n    ...               [0, 2, 4],\n    ...               [0, 3, 6]])\n    >>> np.where(a < 4, a, -1)  # -1 is broadcast\n    array([[ 0.,  1.,  2.],\n           [ 0.,  2., -1.],\n           [ 0.,  3., -1.]])\n    \"\"\"\n    return _mx_nd_np.where(condition, x, y)\n\n\n@set_module('mxnet.numpy')\ndef polyval(p, x):\n    \"\"\"\n    Evaluate a polynomial at specific values.\n    If p is of length N, this function returns the value:\n    p[0]*x**(N-1) + p[1]*x**(N-2) + ... + p[N-2]*x + p[N-1]\n    If x is a sequence, then p(x) is returned for each element of x.\n    If x is another polynomial then the composite polynomial p(x(t)) is returned.\n\n    Parameters\n    ----------\n    p : ndarray\n        1D array of polynomial coefficients (including coefficients equal to zero)\n        from highest degree to the constant term.\n    x : ndarray\n        An array of numbers, at which to evaluate p.\n\n    Returns\n    -------\n    values : ndarray\n        Result array of polynomials\n\n    .. note::\n       This function differs from the original `numpy.polyval\n       <https://numpy.org/devdocs/reference/generated/numpy.polyval.html>`_ in\n       the following way(s):\n\n       * Does not support poly1d.\n       * X should be ndarray type even if it contains only one element.\n\n    Examples\n    --------\n    >>> p = np.array([3, 0, 1])\n    array([3., 0., 1.])\n    >>> x = np.array([5])\n    array([5.])\n    >>> np.polyval(p, x)  # 3 * 5**2 + 0 * 5**1 + 1\n    array([76.])\n    >>> x = np.array([5, 4])\n    array([5., 4.])\n    >>> np.polyval(p, x)\n    array([76., 49.])\n    \"\"\"\n    return _mx_nd_np.polyval(p, x)\n\n\n@set_module('mxnet.numpy')\ndef bincount(x, weights=None, minlength=0):\n    \"\"\"\n    Count number of occurrences of each value in array of non-negative ints.\n\n    Parameters\n    ----------\n    x : ndarray\n        input array, 1 dimension, nonnegative ints.\n    weights: ndarray\n        input weigths same shape as x. (Optional)\n    minlength: int\n        A minimum number of bins for the output. (Optional)\n\n    Returns\n    --------\n    out : ndarray\n        the result of binning the input array. The length of out is equal to amax(x)+1.\n\n    Raises\n    --------\n    Value Error\n        If the input is not 1-dimensional, or contains elements with negative values,\n        or if minlength is negative\n    TypeError\n        If the type of the input is float or complex.\n\n    Examples\n    --------\n    >>> np.bincount(np.arange(5))\n    array([1, 1, 1, 1, 1])\n    >>> np.bincount(np.array([0, 1, 1, 3, 2, 1, 7]))\n    array([1, 3, 1, 1, 0, 0, 0, 1])\n\n    >>> x = np.array([0, 1, 1, 3, 2, 1, 7, 23])\n    >>> np.bincount(x).size == np.amax(x)+1\n    True\n\n    >>> np.bincount(np.arange(5, dtype=float))\n    Traceback (most recent call last):\n    File \"<stdin>\", line 1, in <module>\n    TypeError: array cannot be safely cast to required type\n\n    >>> w = np.array([0.3, 0.5, 0.2, 0.7, 1., -0.6]) # weights\n    >>> x = np.array([0, 1, 1, 2, 2, 2])\n    >>> np.bincount(x,  weights=w)\n    array([ 0.3,  0.7,  1.1])\n    \"\"\"\n    return _mx_nd_np.bincount(x, weights=weights, minlength=minlength)\n\n\n@set_module('mxnet.numpy')\ndef atleast_1d(*arys):\n    \"\"\"\n    Convert inputs to arrays with at least one dimension.\n\n    Scalar inputs are converted to 1-dimensional arrays, whilst higher-dimensional inputs are preserved.\n\n    Parameters\n    ----------\n    arys1, arys2, ... : ndarray\n        One or more input arrays.\n\n    Returns\n    -------\n    ret : ndarray\n        An array, or list of arrays, each with a.ndim >= 1. Copies are made only if necessary.\n\n    See also\n    --------\n    atleast_2d, atleast_3d\n\n    Examples\n    --------\n    >>> np.atleast_1d(1.0)\n    array([1.])\n    >>> x = np.arange(9.0).reshape(3,3)\n    >>> np.atleast_1d(x)\n    array([[0., 1., 2.],\n           [3., 4., 5.],\n           [6., 7., 8.]])\n    >>> np.atleast_1d(np.array(1), np.array([3, 4]))\n    [array([1.]), array([3., 4.])]\n    \"\"\"\n    res = []\n    for ary in arys:\n        if not isinstance(ary, NDArray):\n            ary = array(ary)\n        res.append(ary)\n    return _mx_nd_np.atleast_1d(*res)\n\n\n@set_module('mxnet.numpy')\ndef atleast_2d(*arys):\n    \"\"\"\n    Convert inputs to arrays with at least two dimensions.\n\n    Parameters\n    ----------\n    arys1, arys2, ... : ndarray\n        One or more input arrays.\n\n    Returns\n    -------\n    ret : ndarray\n        An array, or list of arrays, each with a.ndim >= 2. Copies are made only if necessary.\n\n    See also\n    --------\n    atleast_1d, atleast_3d\n\n    Examples\n    --------\n    >>> np.atleast_2d(3.0)\n    array([[3.]])\n    >>> x = np.arange(3.0)\n    >>> np.atleast_2d(x)\n    array([[0., 1., 2.]])\n    >>> np.atleast_2d(np.array(1), np.array([1, 2]), np.array([[1, 2]]))\n    [array([[1.]]), array([[1., 2.]]), array([[1., 2.]])]\n    \"\"\"\n    res = []\n    for ary in arys:\n        if not isinstance(ary, NDArray):\n            ary = array(ary)\n        res.append(ary)\n    return _mx_nd_np.atleast_2d(*res)\n\n\n@set_module('mxnet.numpy')\ndef atleast_3d(*arys):\n    \"\"\"\n    Convert inputs to arrays with at least three dimension.\n\n    Parameters\n    ----------\n    arys1, arys2, ... : ndarray\n        One or more input arrays.\n\n    Returns\n    -------\n    ret : ndarray\n        An array, or list of arrays, each with a.ndim >= 3.\n        For example, a 1-D array of shape (N,) becomes a view of shape (1, N, 1),\n        and a 2-D array of shape (M, N) becomes a view of shape (M, N, 1).\n\n    See also\n    --------\n    atleast_1d, atleast_2d\n\n    Examples\n    --------\n    >>> np.atleast_3d(3.0)\n    array([[[3.]]])\n    >>> x = np.arange(3.0)\n    >>> np.atleast_3d(x).shape\n    (1, 3, 1)\n    >>> x = np.arange(12.0).reshape(4,3)\n    >>> np.atleast_3d(x).shape\n    (4, 3, 1)\n    >>> for arr in np.atleast_3d(np.array([1, 2]), np.array([[1, 2]]), np.array([[[1, 2]]])):\n    ...     print(arr, arr.shape)\n    ...\n    [[[1.]\n      [2.]]] (1, 2, 1)\n    [[[1.]\n      [2.]]] (1, 2, 1)\n    [[[1. 2.]]] (1, 1, 2)\n    \"\"\"\n    res = []\n    for ary in arys:\n        if not isinstance(ary, NDArray):\n            ary = array(ary)\n        res.append(ary)\n    return _mx_nd_np.atleast_3d(*res)\n\n\n@set_module('mxnet.numpy')\ndef pad(x, pad_width=None, mode=\"constant\", **kwargs): # pylint: disable=too-many-arguments\n    # pylint: disable=too-many-return-statements\n    \"\"\"\n    Pad an array.\n\n    Parameters\n    ----------\n    array : array_like of rank N\n        The array to pad.\n    pad_width : {sequence, array_like, int}\n        Number of values padded to the edges of each axis.\n        ((before_1, after_1), ... (before_N, after_N)) unique pad widths\n        for each axis.\n        ((before, after),) yields same before and after pad for each axis.\n        (pad,) or int is a shortcut for before = after = pad width for all\n        axes.\n    mode : str or function, optional\n        One of the following string values or a user supplied function.\n        'constant' (default)\n            Pads with a constant value.\n        'edge'\n            Pads with the edge values of array.\n        'linear_ramp'\n            not supported yet\n        'maximum'\n            Pads with the maximum value of all of the\n            vector along each axis.\n        'mean'\n            not supported yet\n        'median'\n            not supported yet\n        'minimum'\n            Pads with the minimum value of all of the\n            vector along each axis.\n        'reflect'\n            Pads with the reflection of the vector mirrored on\n            the first and last values of the vector along each\n            axis.\n        'symmetric'\n            Pads with the reflection of the vector mirrored\n            along the edge of the array.\n        'wrap'\n            not supported yet.\n        'empty'\n            not supported yet.\n        <function>\n            not supported yet.\n    stat_length : not supported yet\n    constant_values : scalar, optional\n        Used in 'constant'.  The values to set the padded values for each\n        axis.\n        Default is 0.\n\n    end_values : not supported yet\n    reflect_type : {'even', 'odd'}, optional\n        only support even now\n\n    Returns\n    -------\n    pad : ndarray\n        Padded array of rank equal to `array` with shape increased\n        according to `pad_width`.\n\n    Examples\n    --------\n    >>> a = [1, 2, 3, 4, 5]\n    >>> np.pad(a, (2, 3), 'edge')\n    array([1, 1, 1, ..., 5, 5, 5])\n    >>> np.pad(a, (2, 2), 'maximum')\n    array([5, 5, 1, 2, 3, 4, 5, 5, 5])\n    >>> np.pad(a, (2, 2), 'mean')\n    array([3, 3, 1, 2, 3, 4, 5, 3, 3])\n    >>> a = [[1, 2], [3, 4]]\n    >>> np.pad(a, ((3, 2), (2, 3)), 'minimum')\n    array([[1, 1, 1, 2, 1, 1, 1],\n           [1, 1, 1, 2, 1, 1, 1],\n           [1, 1, 1, 2, 1, 1, 1],\n           [1, 1, 1, 2, 1, 1, 1],\n           [3, 3, 3, 4, 3, 3, 3],\n           [1, 1, 1, 2, 1, 1, 1],\n           [1, 1, 1, 2, 1, 1, 1]])\n    >>> a = [1, 2, 3, 4, 5]\n    >>> np.pad(a, (2, 3), 'reflect')\n    array([3, 2, 1, 2, 3, 4, 5, 4, 3, 2])\n    >>> np.pad(a, (2, 3), 'symmetric')\n    array([2, 1, 1, 2, 3, 4, 5, 5, 4, 3])\n    >>> a = np.arange(6)\n    >>> a = a.reshape((2, 3))\n    >>> np.pad(a, ((2, 2), (2, 2)), pad_with)\n    array([[10, 10, 10, 10, 10, 10, 10],\n           [10, 10, 10, 10, 10, 10, 10],\n           [10, 10,  0,  1,  2, 10, 10],\n           [10, 10,  3,  4,  5, 10, 10],\n           [10, 10, 10, 10, 10, 10, 10],\n           [10, 10, 10, 10, 10, 10, 10]])\n    \"\"\"\n    return _mx_nd_np.pad(x, pad_width=pad_width, mode=mode, **kwargs)\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\ndef prod(a, axis=None, dtype=None, out=None, keepdims=False, initial=None): # pylint: disable=too-many-arguments\n    \"\"\"\n    Return the product of array elements over a given axis.\n\n    Parameters\n    ----------\n    a : array_like\n        Input data.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which a product is performed.  The default,\n        axis=None, will calculate the product of all the elements in the\n        input array. If axis is negative it counts from the last to the\n        first axis.\n        .. versionadded:: 1.7.0\n        If axis is a tuple of ints, a product is performed on all of the\n        axes specified in the tuple instead of a single axis or all the\n        axes as before.\n    dtype : dtype, optional\n        The type of the returned array, as well as of the accumulator in\n        which the elements are multiplied.  The dtype of `a` is used by\n        default unless `a` has an integer dtype of less precision than the\n        default platform integer.  In that case, if `a` is signed then the\n        platform integer is used while if `a` is unsigned then an unsigned\n        integer of the same precision as the platform integer is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result. It must have\n        the same shape as the expected output, but the type of the output\n        values will be cast if necessary.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in the\n        result as dimensions with size one. With this option, the result\n        will broadcast correctly against the input array.\n        If the default value is passed, then `keepdims` will not be\n        passed through to the `prod` method of sub-classes of\n        `ndarray`, however any non-default value will be.  If the\n        sub-class' method does not implement `keepdims` any\n        exceptions will be raised.\n    initial : scalar, optional\n        The starting value for this product. See `~numpy.ufunc.reduce` for details.\n    where : not supported\n\n    Returns\n    -------\n    product_along_axis : ndarray, see `dtype` parameter above.\n        An array shaped as `a` but with the specified axis removed.\n        Returns a reference to `out` if specified.\n\n    Examples\n    --------\n    By default, calculate the product of all elements:\n    >>> np.prod([1.,2.])\n    2.0\n    Even when the input array is two-dimensional:\n    >>> np.prod([[1.,2.],[3.,4.]])\n    24.0\n    But we can also specify the axis over which to multiply:\n    >>> np.prod([[1.,2.],[3.,4.]], axis=1)\n    array([  2.,  12.])\n    Or select specific elements to include:\n    >>> np.prod([1., np.nan, 3.], where=[True, False, True])\n    3.0\n    If the type of `x` is unsigned, then the output type is\n    the unsigned platform integer:\n    >>> x = np.array([1, 2, 3], dtype=np.uint8)\n    >>> np.prod(x).dtype == np.uint\n    True\n    If `x` is of a signed integer type, then the output type\n    is the default platform integer:\n    >>> x = np.array([1, 2, 3], dtype=np.int8)\n    >>> np.prod(x).dtype == int\n    True\n    You can also start the product with a value other than one:\n    >>> np.prod([1, 2], initial=5)\n    10\n    \"\"\"\n    return _mx_nd_np.prod(a, axis=axis, dtype=dtype, keepdims=keepdims, initial=initial, out=out)\n\n@set_module('mxnet.numpy')\ndef dot(a, b, out=None):\n    \"\"\"\n    Dot product of two arrays. Specifically,\n\n    * If both `a` and `b` are 1-D arrays, it is inner product of vectors\n\n    * If both `a` and `b` are 2-D arrays, it is matrix multiplication,\n\n    * If either `a` or `b` is 0-D (scalar), it is equivalent to :func:`multiply`\n      and using ``np.multiply(a, b)`` or ``a * b`` is preferred.\n\n    * If `a` is an N-D array and `b` is a 1-D array, it is a sum product over\n      the last axis of `a` and `b`.\n\n    * If `a` is an N-D array and `b` is a 2-D array, it is a\n      sum product over the last axis of `a` and the second-to-last axis of `b`::\n\n        dot(a, b)[i,j,k] = sum(a[i,j,:] * b[:,k])\n\n    Parameters\n    ----------\n    a : ndarray\n        First argument.\n    b : ndarray\n        Second argument.\n\n    out : ndarray, optional\n        Output argument. It must have the same shape and type as the expected output.\n\n    Returns\n    -------\n    output : ndarray\n        Returns the dot product of `a` and `b`.  If `a` and `b` are both\n        scalars or both 1-D arrays then a scalar is returned; otherwise\n        an array is returned.\n        If `out` is given, then it is returned\n\n    Examples\n    --------\n    >>> a = np.array(3)\n    >>> b = np.array(4)\n    >>> np.dot(a, b)\n    array(12.)\n\n    For 2-D arrays it is the matrix product:\n\n    >>> a = np.array([[1, 0], [0, 1]])\n    >>> b = np.array([[4, 1], [2, 2]])\n    >>> np.dot(a, b)\n    array([[4., 1.],\n           [2., 2.]])\n\n    >>> a = np.arange(3*4*5*6).reshape((3,4,5,6))\n    >>> b = np.arange(5*6)[::-1].reshape((6,5))\n    >>> np.dot(a, b)[2,3,2,2]\n    array(29884.)\n    >>> np.sum(a[2,3,2,:] * b[:,2])\n    array(29884.)\n    \"\"\"\n    return _mx_nd_np.dot(a, b, out=out)\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\ndef cumsum(a, axis=None, dtype=None, out=None):\n    \"\"\"\n    Return the cumulative sum of the elements along a given axis.\n\n    Parameters\n    ----------\n    a : array_like\n        Input array.\n    axis : int, optional\n        Axis along which the cumulative sum is computed. The default\n        (None) is to compute the cumsum over the flattened array.\n    dtype : dtype, optional\n        Type of the returned array and of the accumulator in which the\n        elements are summed.  If `dtype` is not specified, it defaults\n        to the dtype of `a`, unless `a` has an integer dtype with a\n        precision less than that of the default platform integer.  In\n        that case, the default platform integer is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result. It must\n        have the same shape and buffer length as the expected output\n        but the type will be cast if necessary. See `doc.ufuncs`\n        (Section \"Output arguments\") for more details.\n\n    Returns\n    -------\n    cumsum_along_axis : ndarray.\n        A new array holding the result is returned unless `out` is\n        specified, in which case a reference to `out` is returned. The\n        result has the same size as `a`, and the same shape as `a` if\n        `axis` is not None or `a` is a 1-d array.\n\n    Examples\n    --------\n    >>> a = np.array([[1,2,3], [4,5,6]])\n    >>> a\n    array([[1, 2, 3],\n           [4, 5, 6]])\n    >>> np.cumsum(a)\n    array([ 1,  3,  6, 10, 15, 21])\n    >>> np.cumsum(a, dtype=float)     # specifies type of output value(s)\n    array([  1.,   3.,   6.,  10.,  15.,  21.])\n    >>> np.cumsum(a,axis=0)      # sum over rows for each of the 3 columns\n    array([[1, 2, 3],\n           [5, 7, 9]])\n    >>> np.cumsum(a,axis=1)      # sum over columns for each of the 2 rows\n    array([[ 1,  3,  6],\n           [ 4,  9, 15]])\n    \"\"\"\n    return _mx_nd_np.cumsum(a, axis=axis, dtype=dtype, out=out)\n\n@set_module('mxnet.numpy')\ndef reshape(a, newshape, order='C'):\n    \"\"\"\n    Gives a new shape to an array without changing its data.\n    This function always returns a copy of the input array if\n    ``out`` is not provided.\n\n    Parameters\n    ----------\n    a : ndarray\n        Array to be reshaped.\n\n    newshape : int or tuple of ints\n        The new shape should be compatible with the original shape. If\n        an integer, then the result will be a 1-D array of that length.\n        One shape dimension can be -1. In this case, the value is\n        inferred from the length of the array and remaining dimensions.\n\n    order : {'C'}, optional\n        Read the elements of `a` using this index order, and place the\n        elements into the reshaped array using this index order.  'C'\n        means to read / write the elements using C-like index order,\n        with the last axis index changing fastest, back to the first\n        axis index changing slowest. Other order types such as 'F'/'A'\n        may be added in the future.\n\n    Returns\n    -------\n    reshaped_array : ndarray\n        It will be always a copy of the original array. This behavior is different\n        from the official NumPy ``reshape`` operator where views of the original array may be\n        generated.\n\n    See Also\n    --------\n    ndarray.reshape : Equivalent method.\n\n    Examples\n    --------\n    >>> a = np.arange(6).reshape((3, 2))\n    >>> a\n    array([[0., 1.],\n           [2., 3.],\n           [4., 5.]])\n\n    >>> np.reshape(a, (2, 3)) # C-like index ordering\n    array([[0., 1., 2.],\n           [3., 4., 5.]])\n\n    >>> np.reshape(np.ravel(a), (2, 3)) # equivalent to C ravel then C reshape\n    array([[0., 1., 2.],\n           [3., 4., 5.]])\n\n    >>> a = np.array([[1,2,3], [4,5,6]])\n    >>> np.reshape(a, 6)\n    array([1., 2., 3., 4., 5., 6.])\n\n    >>> np.reshape(a, (3,-1))       # the unspecified value is inferred to be 2\n    array([[1., 2.],\n           [3., 4.],\n           [5., 6.]])\n    \"\"\"\n    return _mx_nd_np.reshape(a, newshape, order)\n\n@set_module('mxnet.numpy')\ndef moveaxis(a, source, destination):\n    \"\"\"Move axes of an array to new positions.\n    Other axes remain in their original order.\n\n    Parameters\n    ----------\n    a : ndarray\n        The array whose axes should be reordered.\n        source : int or sequence of int\n        Original positions of the axes to move. These must be unique.\n        destination : int or sequence of int\n        Destination positions for each of the original axes. These must also be\n        unique.\n\n    Returns\n    -------\n    result : ndarray\n        Array with moved axes. This array is a view of the input array.\n\n    See Also\n    --------\n        transpose: Permute the dimensions of an array.\n        swapaxes: Interchange two axes of an array.\n\n    Examples\n    --------\n    >>> x = np.zeros((3, 4, 5))\n    >>> np.moveaxis(x, 0, -1).shape\n    (4, 5, 3)\n    >>> np.moveaxis(x, -1, 0).shape\n    (5, 3, 4)\n    These all achieve the same result:\n    >>> np.transpose(x).shape\n    (5, 4, 3)\n    >>> np.swapaxes(x, 0, -1).shape\n    (5, 4, 3)\n    >>> np.moveaxis(x, [0, 1], [-1, -2]).shape\n    (5, 4, 3)\n    >>> np.moveaxis(x, [0, 1, 2], [-1, -2, -3]).shape\n    (5, 4, 3)\n    \"\"\"\n    return _mx_nd_np.moveaxis(a, source, destination)\n\n@set_module('mxnet.numpy')\ndef copy(a): # pylint: disable=redefined-outer-name\n    \"\"\"\n    Return an array copy of the given object.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input array.\n\n    Returns\n    -------\n    arr : _Symbol\n        Array interpretation of a.\n\n    -----\n    Examples\n    --------\n    >>> x = np.array([1, 2, 3])\n    >>> y = x\n    >>> z = np.copy(x)\n    >>> x[0] = 10\n    >>> x[0] == y[0]\n        True\n    >>> x[0] == z[0]\n        False\n    \"\"\"\n    return _mx_nd_np.copy(a)\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\ndef rollaxis(a, axis, start=0):\n    \"\"\"\n    Roll the specified axis backwards, until it lies in a given position.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input array.\n    axis : integer\n        The axis to roll backwards. The positions of the other axes do not\n        change relative to one another.\n    start: int, optional\n        The axis is rolled until it lies before this position.\n        The default, 0, results in a “complete” roll.\n\n    Returns\n    -------\n    res : ndarray\n        A view after applying rollaxis to `a` is returned.\n\n    -----\n    Examples\n    --------\n    >>> a = np.ones((3,4,5,6))\n    >>> np.rollaxis(a, 3, 1).shape\n    (3, 6, 4, 5)\n    >>> np.rollaxis(a, 2).shape\n    (5, 3, 4, 6)\n    >>> np.rollaxis(a, 1, 4).shape\n    (3, 5, 6, 4)\n    \"\"\"\n    return _mx_nd_np.rollaxis(a, axis, start)\n\n\n@set_module('mxnet.numpy')\ndef diag(v, k=0):\n    \"\"\"\n    Extracts a diagonal or constructs a diagonal array.\n    * 1-D arrays: constructs a 2-D array with the input as its diagonal, all other elements are zero.\n    * 2-D arrays: extracts the k-th Diagonal\n\n    Parameters\n    ----------\n    array : ndarray\n        The array to apply diag method.\n    k : offset\n        extracts or constructs kth diagonal given input array\n\n    Returns\n    ----------\n    out : ndarray\n    The extracted diagonal or constructed diagonal array.\n\n    Examples\n    --------\n    >>> x = np.arange(9).reshape((3,3))\n    >>> x\n    array([[0, 1, 2],\n           [3, 4, 5],\n           [6, 7, 8]])\n    >>> np.diag(x)\n    array([0, 4, 8])\n    >>> np.diag(x, k=1)\n    array([1, 5])\n    >>> np.diag(x, k=-1)\n    array([3, 7])\n\n    >>> np.diag(np.diag(x))\n    array([[0, 0, 0],\n           [0, 4, 0],\n           [0, 0, 8]])\n    \"\"\"\n    return _mx_nd_np.diag(v, k=k)\n\n\n@set_module('mxnet.numpy')\ndef diagflat(v, k=0):\n    \"\"\"\n    Create a two-dimensional array with the flattened input as a diagonal.\n\n    Parameters\n    ----------\n    v : array_like\n        Input data, which is flattened and set as the `k`-th\n        diagonal of the output.\n    k : int, optional\n        Diagonal to set; 0, the default, corresponds to the \"main\" diagonal,\n        a positive (negative) `k` giving the number of the diagonal above\n        (below) the main.\n\n    Returns\n    -------\n    out : ndarray\n        The 2-D output array.\n\n    See Also\n    --------\n    diag : MATLAB work-alike for 1-D and 2-D arrays.\n    diagonal : Return specified diagonals.\n    trace : Sum along diagonals.\n\n    Examples\n    --------\n    >>> np.diagflat([[1,2], [3,4]])\n    array([[1, 0, 0, 0],\n           [0, 2, 0, 0],\n           [0, 0, 3, 0],\n           [0, 0, 0, 4]])\n    >>> np.diagflat([1,2], 1)\n    array([[0, 1, 0],\n           [0, 0, 2],\n           [0, 0, 0]])\n    \"\"\"\n    return _mx_nd_np.diagflat(v, k=k)\n\n\n@set_module('mxnet.numpy')\ndef diagonal(a, offset=0, axis1=0, axis2=1):\n    \"\"\"\n    If a is 2-D, returns the diagonal of a with the given offset, i.e., the collection of elements of\n    the form a[i, i+offset]. If a has more than two dimensions, then the axes specified by axis1 and\n    axis2 are used to determine the 2-D sub-array whose diagonal is returned. The shape of the\n    resulting array can be determined by removing axis1 and axis2 and appending an index to the\n    right equal to the size of the resulting diagonals.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data from which diagonal are taken.\n    offset: int, Optional\n        Offset of the diagonal from the main diagonal\n    axis1: int, Optional\n        Axis to be used as the first axis of the 2-D sub-arrays\n    axis2: int, Optional\n        Axis to be used as the second axis of the 2-D sub-arrays\n\n    Returns\n    -------\n    out : ndarray\n        Output result\n\n    Raises\n    -------\n    ValueError:  If the dimension of a is less than 2.\n\n    Examples\n    --------\n    >>> a = np.arange(4).reshape(2,2)\n    >>> a\n    array([[0, 1],\n        [2, 3]])\n    >>> np.diagonal(a)\n    array([0, 3])\n    >>> np.diagonal(a, 1)\n    array([1])\n\n    >>> a = np.arange(8).reshape(2,2,2)\n    >>>a\n    array([[[0, 1],\n            [2, 3]],\n            [[4, 5],\n            [6, 7]]])\n    >>> np.diagonal(a, 0, 0, 1)\n    array([[0, 6],\n            [1, 7]])\n    \"\"\"\n    return _mx_nd_np.diagonal(a, offset=offset, axis1=axis1, axis2=axis2)\n\n\n# pylint: disable=redefined-outer-name, too-many-arguments\n@set_module('mxnet.numpy')\ndef sum(a, axis=None, dtype=None, out=None, keepdims=None, initial=None, where=None):\n    r\"\"\"\n    Sum of array elements over a given axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data.\n    axis : None or int, optional\n        Axis or axes along which a sum is performed.  The default,\n        axis=None, will sum all of the elements of the input array.  If\n        axis is negative it counts from the last to the first axis.\n    dtype : dtype, optional\n        The type of the returned array and of the accumulator in which the\n        elements are summed. The default type is float32.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the input array.\n\n        If the default value is passed, then `keepdims` will not be\n        passed through to the `sum` method of sub-classes of\n        `ndarray`, however any non-default value will be.  If the\n        sub-classes `sum` method does not implement `keepdims` any\n        exceptions will be raised.\n    initial: Currently only supports None as input, optional\n        Starting value for the sum.\n        Currently not implemented. Please use ``None`` as input or skip this argument.\n    out : ndarray or None, optional\n        Alternative output array in which to place the result. It must have\n        the same shape and dtype as the expected output.\n\n    Returns\n    -------\n    sum_along_axis : ndarray\n        An ndarray with the same shape as `a`, with the specified\n        axis removed. If an output array is specified, a reference to\n        `out` is returned.\n\n    Notes\n    -----\n    * Input type does not support Python native iterables.\n    * \"out\" param: cannot perform auto type change. out ndarray's dtype must be the same as the expected output.\n    * \"initial\" param is not supported yet. Please use None as input.\n    * Arithmetic is modular when using integer types, and no error is raised on overflow.\n    * The sum of an empty array is the neutral element 0:\n\n    >>> a = np.empty(1)\n    >>> np.sum(a)\n    array(0.)\n\n    This function differs from the original `numpy.sum\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.sum.html>`_ in\n    the following aspects:\n\n    * Input type does not support Python native iterables(list, tuple, ...).\n    * \"out\" param: cannot perform auto type cast. out ndarray's dtype must be the same as the expected output.\n    * \"initial\" param is not supported yet. Please use ``None`` as input or skip it.\n    * The default type is float32.\n\n    Examples\n    --------\n    >>> a = np.array([0.5, 1.5])\n    >>> np.sum(a)\n    array(2.)\n    >>> a = np.array([0.5, 0.7, 0.2, 1.5])\n    >>> np.sum(a, dtype=np.int32)\n    array(2, dtype=int32)\n    >>> a = np.array([[0, 1], [0, 5]])\n    >>> np.sum(a)\n    array(6.)\n    >>> np.sum(a, axis=0)\n    array([0., 6.])\n    >>> np.sum(a, axis=1)\n    array([1., 5.])\n\n    With output ndarray:\n\n    >>> a = np.array([[0, 1], [0, 5]])\n    >>> b = np.ones((2,), dtype=np.float32)\n    >>> np.sum(a, axis = 0, out=b)\n    array([0., 6.])\n    >>> b\n    array([0., 6.])\n\n    If the accumulator is too small, overflow occurs:\n\n    >>> np.ones(128, dtype=np.int8).sum(dtype=np.int8)\n    array(-128, dtype=int8)\n    \"\"\"\n    return _mx_nd_np.sum(a, axis=axis, dtype=dtype, out=out, keepdims=keepdims, initial=initial, where=where)\n\n\n@set_module('mxnet.numpy')\ndef bitwise_left_shift(x1, x2, out=None):\n    r\"\"\"\n    Shift the bits of and integer to the left. Bits are shifted to the left by\n    appending x2 0s at the right of x1. Since the internal representation of numbers\n    is in binary format, this operation is equivalent to ``x1 * 2**x2``\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Input values.\n    x2 : ndarray or scalar\n        Number of zeros to append to x1. Has to be non-negative. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which becomes the shape of the output).\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape that the\n        inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n        Result.\n\n    Examples\n    --------\n    >>> np.binary_repr(5)\n    '101'\n    >>> np.left_shift(5, 2)\n    20\n    >>> np.binary_repr(20)\n    '10100'\n    \"\"\"\n    return _mx_nd_np.bitwise_left_shift(x1, x2, out)\n\n\n@set_module('mxnet.numpy')\ndef bitwise_right_shift(x1, x2, out=None):\n    r\"\"\"\n    Shift the bits of and integer to the right. Bits are shifted to the right by\n    x2. Because the internal representation of numbers is in binary format,\n    this operation is equivalent to ``x1 / 2**x2``\n\n    Parameters\n    ----------\n    x1 : ndarray or scalar\n        Input values.\n    x1 : ndarray or scalar\n        Number of bits to remove at the right of x1. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which becomes the shape of the output).\n    out : ndarray, optional\n        A location into which the result is stored. If provided, it must have a shape that the\n        inputs broadcast to. If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : ndarray\n        Result.\n\n    Examples\n    --------\n    >>> np.binary_repr(10)\n    '1010'\n    >>> np.right_shift(10, 1)\n    5\n    >>> np.binary_repr(5)\n    '101'\n    >>> np.right_shift(10, np.array([1,2,3]))\n    array([5, 2, 1])\n    \"\"\"\n    return _mx_nd_np.bitwise_right_shift(x1, x2, out)\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\n@wrap_ctx_to_device_func\ndef asarray(obj, dtype=None, device=None, copy=None):\n    \"\"\"\n    Convert the input to an array.\n\n    Parameters\n    ----------\n    obj : <array>, bool, int, float, NestedSequence[ bool | int | float ]\n        Object to be converted to an array. Can be a Python scalar,\n        a (possibly nested) sequence of Python scalars,\n        or an object supporting DLPack or the Python buffer protocol.\n    dtype : dtype, Optional\n        output array data type. Default: None .\n    device : Device, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.device.current_device()`.\n    copy : bool, Optional\n        Whether or not to make a copy of the input.\n        If True, always copies.\n        If False, never copies for input which supports DLPack or the buffer protocol,\n        and raises ValueError in case that would be necessary.\n        If None, reuses existing memory buffer if possible, copies otherwise. Default: None .\n\n        An array containing the data from obj.\n\n    Examples\n    --------\n    >>> np.asarray([1, 2, 3])\n    array([1., 2., 3.])\n\n    >>> np.asarray([[1, 2], [3, 4]], dtype=np.int32)\n    array([[1, 2],\n           [3, 4]], dtype=int32)\n\n    >>> np.asarray([1.2], device=mx.gpu())\n    array([1.2], device=gpu(0))\n    \"\"\"\n    if isinstance(obj, numeric_types):\n        dtype = dtype_from_number(obj) if dtype is None else dtype\n        obj = _np.asarray(obj, dtype=dtype)\n    elif isinstance(obj, _np.ndarray):\n        if is_np_default_dtype():\n            dtype = obj.dtype if dtype is None else dtype\n        else:\n            dtype = _np.float32 if dtype is None or obj.dtype is _np.float64 else dtype\n    elif isinstance(obj, ndarray):\n        if dtype is not None:\n            obj = obj.astype(dtype, copy=copy)\n        if device is not None:\n            obj = obj.to_device(device)\n        return obj\n    elif hasattr(obj, '__dlpack__'):\n        return from_dlpack(obj)\n    else:\n        if dtype is None:\n            default_dtype = _np.float64 if is_np_default_dtype() else _np.float32\n            dtype = obj.dtype if hasattr(obj, \"dtype\") else default_dtype\n        try:\n            obj = _np.array(obj, dtype=dtype)\n        except Exception as e:\n            # printing out the error raised by official NumPy's array function\n            # for transparency on users' side\n            raise TypeError('{}'.format(str(e)))\n    if device is None:\n        device = current_device()\n    ret = empty(obj.shape, dtype=dtype, device=device)\n    if len(obj.shape) == 0:\n        ret[()] = obj\n    else:\n        ret[:] = obj\n    return ret\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.numpy')\ndef from_dlpack(x):\n    \"\"\"\n    Returns a np.ndarray backed by a dlpack tensor.\n\n    Parameters\n    ----------\n    dlpack : an object with __dlpack__ method or PyCapsule (the pointer of DLManagedTensor)\n        input data\n\n    Returns\n    -------\n    out : np.ndarray\n        an ndarray backed by a dlpack tensor\n\n    Examples\n    --------\n    >>> x = mx.np.ones((2,3))\n    >>> y = mx.np.from_dlpack(x)\n    >>> y\n    array([[1., 1., 1.],\n           [1., 1., 1.]])\n    >>> y += 1\n    >>> x\n    array([[2., 2., 2.],\n           [2., 2., 2.]])\n    \"\"\"\n    from_dlpack = ndarray_from_dlpack(ndarray)\n    return from_dlpack(x)\n"
  },
  {
    "path": "python/mxnet/numpy/random.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for ops used in imperative programming.\"\"\"\n\nfrom ..ndarray import numpy as _mx_nd_np\nfrom ..random import seed\nfrom ..util import wrap_ctx_to_device_func\n\n\n__all__ = [\"randint\", \"uniform\", \"normal\", \"choice\", \"rand\", \"multinomial\", \"multivariate_normal\",\n           \"logistic\", \"gumbel\", \"f\",\n           \"laplace\",\n           \"shuffle\", \"randn\", \"gamma\", \"beta\", \"chisquare\", \"exponential\", \"lognormal\",\n           \"weibull\", \"pareto\", \"power\", \"rayleigh\",\n           \"seed\"]\n\n\n@wrap_ctx_to_device_func\ndef randint(low, high=None, size=None, dtype=None, device=None, out=None):\n    r\"\"\"Return random integers from `low` (inclusive) to `high` (exclusive).\n\n    Return random integers from the \"discrete uniform\" distribution of\n    the specified dtype in the \"half-open\" interval [`low`, `high`). If\n    `high` is None (the default), then results are from [0, `low`).\n\n    Parameters\n    ----------\n    low : int\n        Lowest (signed) integer to be drawn from the distribution (unless\n        ``high=None``, in which case this parameter is one above the\n        *highest* such integer).\n    high : int, optional\n        If provided, one above the largest (signed) integer to be drawn\n        from the distribution (see above for behavior if ``high=None``).\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  Default is None, in which case a\n        single value is returned.\n    dtype : dtype, optional\n        Desired dtype of the result. All dtypes are determined by their\n        name, i.e., 'int64', 'int', etc, so byteorder is not available\n        and a specific precision may have different C types depending\n        on the platform. The default value is 'np.int'.\n    device : Device, optional\n        Device context of output. Default is current device.\n    out : ndarray, optional\n        The output ndarray (default is `None`).\n\n    Returns\n    -------\n    out : ndarray of ints\n        `size`-shaped array of random integers from the appropriate\n        distribution, or a single such random int if `size` not provided.\n\n    Examples\n    --------\n    >>> np.random.randint(2, size=10)\n    array([1, 0, 0, 0, 1, 1, 0, 0, 1, 0])\n    >>> np.random.randint(1, size=10)\n    array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n\n    Generate a 2 x 4 array of ints between 0 and 4, inclusive:\n\n    >>> np.random.randint(5, size=(2, 4))\n    array([[4, 0, 2, 1],\n        [3, 2, 2, 0]])\n    \"\"\"\n    return _mx_nd_np.random.randint(low, high, size, dtype, device, out)\n\n\n@wrap_ctx_to_device_func\ndef uniform(low=0.0, high=1.0, size=None, dtype=None, device=None, out=None):\n    r\"\"\"Draw samples from a uniform distribution.\n\n    Samples are uniformly distributed over the half-open interval\n    ``[low, high)`` (includes low, but excludes high).  In other words,\n    any value within the given interval is equally likely to be drawn\n    by `uniform`.\n\n    Parameters\n    ----------\n    low : float, ndarray, optional\n        Lower boundary of the output interval.  All values generated will be\n        greater than or equal to low.  The default value is 0.\n    high : float, ndarray, optional\n        Upper boundary of the output interval.  All values generated will be\n        less than high.  The default value is 1.0.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a scalar tensor containing a single value is returned if\n        ``low`` and ``high`` are both scalars. Otherwise,\n        ``np.broadcast(low, high).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n    device : Device, optional\n        Device context of output. Default is current device.\n\n    Returns\n    -------\n    out : ndarray\n        Drawn samples from the parameterized uniform distribution.\n\n    See Also\n    --------\n    randint : Discrete uniform distribution, yielding integers.\n    rand : Convenience function that accepts dimensions as input, e.g.,\n           ``rand(2,2)`` would generate a 2-by-2 array of floats,\n           uniformly distributed over ``[0, 1)``.\n\n    Notes\n    -----\n    The probability density function of the uniform distribution is\n\n    .. math:: p(x) = \\frac{1}{b - a}\n\n    anywhere within the interval ``[a, b)``, and zero elsewhere.\n\n    When ``high`` == ``low``, values of ``low`` will be returned.\n    If ``high`` < ``low``, the results are officially undefined\n    and may eventually raise an error, i.e. do not rely on this\n    function to behave when passed arguments satisfying that\n    inequality condition.\n    \"\"\"\n    return _mx_nd_np.random.uniform(low, high, size=size, device=device, dtype=dtype, out=out)\n\n\n@wrap_ctx_to_device_func\ndef normal(loc=0.0, scale=1.0, size=None, dtype=None, device=None, out=None):\n    r\"\"\"Draw random samples from a normal (Gaussian) distribution.\n\n    Samples are distributed according to a normal distribution parametrized\n    by *loc* (mean) and *scale* (standard deviation).\n\n    Parameters\n    ----------\n    loc : float, optional\n        Mean (centre) of the distribution.\n    scale : float, optional\n        Standard deviation (spread or \"width\") of the distribution.\n    size : int or tuple of ints, optional\n        Output shape. If the given shape is, e.g., `(m, n, k)`, then `m * n * k`\n        samples are drawn. If size is `None` (default), a scalar tensor containing\n        a single value is returned if loc and scale are both scalars. Otherwise,\n        ``np.broadcast(low, high).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n    device : Device, optional\n        Device context of output, default is current device.\n    out : ``ndarray``, optional\n        Store output to an existing ``ndarray``.\n\n    Returns\n    -------\n    out : ndarray\n        Drawn samples from the parameterized `normal distribution` [1]_.\n\n    Notes\n    -----\n    The probability density for the Gaussian distribution is\n\n    .. math:: p(x) = \\frac{1}{\\sqrt{ 2 \\pi \\sigma^2 }}\n                     e^{ - \\frac{ (x - \\mu)^2 } {2 \\sigma^2} },\n\n    where :math:`\\mu` is the mean and :math:`\\sigma` the standard\n    deviation. The square of the standard deviation, :math:`\\sigma^2`,\n    is called the variance.\n\n    The function has its peak at the mean, and its \"spread\" increases with\n    the standard deviation (the function reaches 0.607 times its maximum at\n    :math:`x + \\sigma` and :math:`x - \\sigma` [2]_).  This implies that\n    `numpy.random.normal` is more likely to return samples lying close to\n    the mean, rather than those far away.\n\n    References\n    ----------\n    .. [1] Wikipedia, \"Normal distribution\",\n           https://en.wikipedia.org/wiki/Normal_distribution\n    .. [2] P. R. Peebles Jr., \"Central Limit Theorem\" in \"Probability,\n           Random Variables and Random Signal Principles\", 4th ed., 2001,\n           pp. 51, 51, 125.\n\n    Examples\n    --------\n    >>> mu, sigma = 0, 0.1 # mean and standard deviation\n    >>> s = np.random.normal(mu, sigma, 1000)\n\n    Verify the mean and the variance:\n\n    >>> np.abs(mu - np.mean(s)) < 0.01\n    array(True)\n    \"\"\"\n    return _mx_nd_np.random.normal(loc, scale, size, dtype, device, out)\n\n\n@wrap_ctx_to_device_func\ndef lognormal(mean=0.0, sigma=1.0, size=None, dtype=None, device=None, out=None):\n    r\"\"\"Draw samples from a log-normal distribution.\n\n    Draw samples from a `log-normal distribution` [1]_ with specified mean,\n    standard deviation, and array shape. Note that the mean and standard\n    deviation are not the values for the distribution itself, but of the\n    underlying normal distribution it is derived from.\n\n    Parameters\n    ----------\n    mean : float or array_like of floats, optional\n        Mean value of the underlying normal distribution. Default is 0.\n    sigma : float or array_like of floats, optional\n        Standard deviation of the underlying normal distribution. Must be\n        non-negative. Default is 1.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``mean`` and ``sigma`` are both scalars.\n        Otherwise, ``np.broadcast(mean, sigma).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    device : Device, optional\n        Device context of output. Default is current device.\n    out : ``ndarray``, optional\n        Store output to an existing ``ndarray``.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized log-normal distribution.\n\n    Notes\n    -----\n    A variable `x` has a log-normal distribution if `log(x)` is normally\n    distributed.  The `probability density function for the log-normal\n    distribution` [2]_ is:\n\n    .. math:: p(x) = \\frac{1}{\\sigma x \\sqrt{2\\pi}}\n                    e^{(-\\frac{(ln(x)-\\mu)^2}{2\\sigma^2})}\n\n    where :math:`\\mu` is the mean and :math:`\\sigma` is the standard\n    deviation of the normally distributed logarithm of the variable.\n    A log-normal distribution results if a random variable is the *product*\n    of a large number of independent, identically-distributed variables in\n    the same way that a normal distribution results if the variable is the\n    *sum* of a large number of independent, identically-distributed\n    variables.\n\n    References\n    ----------\n    .. [1] Limpert, E., Stahel, W. A., and Abbt, M., \"Log-normal\n           Distributions across the Sciences: Keys and Clues,\"\n           BioScience, Vol. 51, No. 5, May, 2001.\n           http://www.statlit.org/pdf/2001-Limpert-Bioscience2.pdf\n    .. [2] Reiss, R.D. and Thomas, M., \"Statistical Analysis of Extreme\n           Values,\" Basel: Birkhauser Verlag, 2001, pp. 31-32.\n\n    Examples\n    --------\n    Draw samples from the distribution:\n    >>> mu, sigma = 3., 1. # mean and standard deviation\n    >>> s = np.random.lognormal(mu, sigma, 1000)\n    \"\"\"\n    return _mx_nd_np.random.lognormal(mean, sigma, size, dtype, device, out)\n\n\n@wrap_ctx_to_device_func\ndef logistic(loc=0.0, scale=1.0, size=None, device=None, out=None):\n    r\"\"\"Draw samples from a logistic distribution.\n\n    Samples are drawn from a logistic distribution with specified\n    parameters, loc (location or mean, also median), and scale (>0).\n\n    Parameters\n    ----------\n    loc : float or array_like of floats, optional\n        Parameter of the distribution. Default is 0.\n    scale : float or array_like of floats, optional\n        Parameter of the distribution. Must be non-negative.\n        Default is 1.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``loc`` and ``scale`` are both scalars.\n        Otherwise, ``np.broadcast(loc, scale).size`` samples are drawn.\n    device : Device, optional\n        Device context of output, default is current device.\n    out : ``ndarray``, optional\n        Store output to an existing ``ndarray``.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized logistic distribution.\n\n    Examples\n    --------\n    Draw samples from the distribution:\n    >>> loc, scale = 10, 1\n    >>> s = np.random.logistic(loc, scale, 10000)\n    >>> import matplotlib.pyplot as plt\n    >>> count, bins, ignored = plt.hist(s, bins=50)\n    #   plot against distribution\n    >>> def logist(x, loc, scale):\n    ...     return np.exp((loc-x)/scale)/(scale*(1+np.exp((loc-x)/scale))**2)\n    >>> lgst_val = logist(bins, loc, scale)\n    >>> plt.plot(bins, lgst_val * count.max() / lgst_val.max())\n    >>> plt.show()\n    \"\"\"\n    return _mx_nd_np.random.logistic(loc, scale, size, device, out)\n\n\n@wrap_ctx_to_device_func\ndef gumbel(loc=0.0, scale=1.0, size=None, device=None, out=None):\n    r\"\"\"Draw samples from a Gumbel distribution.\n\n    Draw samples from a Gumbel distribution with specified location and\n    scale.\n\n    Parameters\n    ----------\n    loc : float or array_like of floats, optional\n        The location of the mode of the distribution. Default is 0.\n    scale : float or array_like of floats, optional\n        The scale parameter of the distribution. Default is 1. Must be non-\n        negative.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``loc`` and ``scale`` are both scalars.\n        Otherwise, ``np.broadcast(loc, scale).size`` samples are drawn.\n    device : Device, optional\n        Device context of output, default is current device.\n    out : ``ndarray``, optional\n        Store output to an existing ``ndarray``.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized Gumbel distribution.\n\n    Examples\n    --------\n    Draw samples from the distribution:\n    >>> mu, beta = 0, 0.1 # location and scale\n    >>> s = np.random.gumbel(mu, beta, 1000)\n    Display the histogram of the samples, along with\n    the probability density function:\n    >>> import matplotlib.pyplot as plt\n    >>> count, bins, ignored = plt.hist(s, 30, density=True)\n    >>> plt.plot(bins, (1/beta)*np.exp(-(bins - mu)/beta)\n    ...          * np.exp( -np.exp( -(bins - mu) /beta) ),\n    ...          linewidth=2, color='r')\n    >>> plt.show()\n    Show how an extreme value distribution can arise from a Gaussian process\n    and compare to a Gaussian:\n    >>> means = []\n    >>> maxima = []\n    >>> for i in range(0,1000) :\n    ...    a = np.random.normal(mu, beta, 1000)\n    ...    means.append(a.mean())\n    ...    maxima.append(a.max())\n    >>> count, bins, ignored = plt.hist(maxima, 30, density=True)\n    >>> beta = np.std(maxima) * np.sqrt(6) / np.pi\n    >>> mu = np.mean(maxima) - 0.57721*beta\n    >>> plt.plot(bins, (1/beta)*np.exp(-(bins - mu)/beta)\n    ...          * np.exp(-np.exp(-(bins - mu)/beta)),\n    ...          linewidth=2, color='r')\n    >>> plt.plot(bins, 1/(beta * np.sqrt(2 * np.pi))\n    ...          * np.exp(-(bins - mu)**2 / (2 * beta**2)),\n    ...          linewidth=2, color='g')\n    >>> plt.show()\n    \"\"\"\n    return _mx_nd_np.random.gumbel(loc, scale, size, device, out)\n\n\ndef multinomial(n, pvals, size=None, **kwargs):\n    r\"\"\"\n    Draw samples from a multinomial distribution.\n    The multinomial distribution is a multivariate generalisation of the binomial distribution.\n    Take an experiment with one of ``p`` possible outcomes. An example of such an experiment is throwing a dice,\n    where the outcome can be 1 through 6. Each sample drawn from the distribution represents n such experiments.\n    Its values, ``X_i = [X_0, X_1, ..., X_p]``, represent the number of times the outcome was ``i``.\n\n    Parameters\n    ----------\n    n : int\n        Number of experiments.\n    pvals : sequence of floats, length p\n        Probabilities of each of the p different outcomes. These should sum to 1.\n    size : int or tuple of ints, optional\n        Output shape. If the given shape is, e.g., ``(m, n, k)``, then ``m * n * k`` samples\n        are drawn. Default is None, in which case a single value is returned.\n\n    Returns\n    -------\n    out : ndarray\n        The drawn samples, of shape size, if that was provided. If not, the shape is ``(N,)``.\n        In other words, each entry ``out[i,j,...,:]`` is an N-dimensional value drawn from the distribution.\n\n    Examples\n    --------\n    Throw a dice 1000 times, and 1000 times again:\n\n    >>> np.random.multinomial(1000, [1/6.]*6, size=2)\n    array([[164, 161, 179, 158, 150, 188],\n           [178, 162, 177, 143, 163, 177]])\n\n    A loaded die is more likely to land on number 6:\n\n    >>> np.random.multinomial(100, [1/7.]*5 + [2/7.])\n    array([19, 14, 12, 11, 21, 23])\n    >>> np.random.multinomial(100, [1.0 / 3, 2.0 / 3])\n    array([32, 68])\n    \"\"\"\n    return _mx_nd_np.random.multinomial(n, pvals, size, **kwargs)\n\n\n# pylint: disable=unused-argument\ndef multivariate_normal(mean, cov, size=None, check_valid=None, tol=None):\n    \"\"\"\n    multivariate_normal(mean, cov, size=None, check_valid=None, tol=None)\n\n    Draw random samples from a multivariate normal distribution.\n\n    The multivariate normal, multinormal or Gaussian distribution is a\n    generalization of the one-dimensional normal distribution to higher\n    dimensions.  Such a distribution is specified by its mean and\n    covariance matrix.  These parameters are analogous to the mean\n    (average or \"center\") and variance (standard deviation, or \"width,\"\n    squared) of the one-dimensional normal distribution.\n\n    This operator is a little different from the one in official NumPy.\n    The official NumPy operator only accepts 1-D ndarray as mean and 2-D ndarray as cov,\n    whereas the operator in MXNet np supports batch operation and auto-broadcasting.\n\n    Both `mean` and `cov` may have any number of leading dimensions, which correspond\n    to a batch shape. They are not necessarily assumed to have the same batch shape,\n    just ones which can be broadcasted.\n\n    Parameters\n    ----------\n    mean : K-D ndarray, of shape (..., N)\n        Mean of the N-dimensional distribution.\n    cov : (K+1)-D ndarray, of shape (..., N, N)\n        Covariance matrix of the distribution. The last two dimensions must be symmetric and\n        positive-semidefinite for proper sampling.\n    size : int or tuple of ints, optional\n        Given a shape of, for example, ``(m,n,k)``,\n        ``m*n*k`` identically distributed batchs of samples are\n        generated, and packed in an `m`-by-`n`-by-`k` arrangement.\n        If no shape is specified, a batch of (`N`-D) sample is returned.\n    check_valid : { 'warn', 'raise', 'ignore' }, optional\n        Behavior when the covariance matrix is not positive semidefinite.\n        (Not supported)\n    tol : float, optional\n        Tolerance when checking the singular values in covariance matrix.\n        cov is cast to double before the check.\n        (Not supported)\n\n    Returns\n    -------\n    out : ndarray\n        The input shape of `mean` and `cov` should satisfy the requirements of broadcasting.\n        If the parameter `size` is not provided,\n        the output shape is ``np.broadcast(mean.shape, cov.shape[:-1])``.\n        Otherwise, the output shape is ``size + np.broadcast(mean.shape, cov.shape[:-1])``\n\n    Examples\n    --------\n    >>> mean = np.array([1, 2])\n    >>> cov = np.array([[1, 0], [0, 1]])\n    >>> x = np.random.multivariate_normal(mean, cov, (3, 3))\n    >>> x.shape\n    (3, 3, 2)\n\n    The following is probably true, given that 0.6 is roughly twice the\n    standard deviation:\n\n    >>> list((x[0,0,:] - mean) < 0.6)\n    [True, True] # random\n\n    # Performs autobroadcasting when the batch shape of\n    # `mean` and `cov` is different but compatible.\n\n    >>> mean = np.zeros((3,2)) # shape (3, 2)\n    >>> cov = np.array([[1, 0], [0, 100]]) # shape (2, 2)\n    >>> x = np.random.multivariate_normal(mean, cov)\n    >>> x\n    array([[-1.6115597 , -8.726251  ],\n           [ 2.2425299 ,  2.8104177 ],\n           [ 0.36229908, -8.386591  ]])\n    \"\"\"\n    return _mx_nd_np.random.multivariate_normal(mean, cov, size=size, check_valid=None, tol=None)\n\n\n@wrap_ctx_to_device_func\ndef choice(a, size=None, replace=True, p=None, device=None, out=None):\n    r\"\"\"Generates a random sample from a given 1-D array\n\n    Parameters\n    -----------\n    a : 1-D array-like or int\n        If an ndarray, a random sample is generated from its elements.\n        If an int, the random sample is generated as if a were np.arange(a)\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  Default is None, in which case a\n        single value is returned.\n    replace : boolean, optional\n        Whether the sample is with or without replacement\n    p : 1-D array-like, optional\n        The probabilities associated with each entry in a.\n        If not given the sample assumes a uniform distribution over all\n        entries in a.\n    device : Device, optional\n        Device context of output. Default is current device.\n\n    Returns\n    --------\n    samples : ndarray\n        The generated random samples\n\n    Examples\n    ---------\n    Generate a uniform random sample from np.arange(5) of size 3:\n\n    >>> np.random.choice(5, 3)\n    array([0, 3, 4])\n    >>> #This is equivalent to np.random.randint(0,5,3)\n\n    Generate a non-uniform random sample from np.arange(5) of size 3:\n\n    >>> np.random.choice(5, 3, p=[0.1, 0, 0.3, 0.6, 0])\n    array([3, 3, 0])\n\n    Generate a uniform random sample from np.arange(5) of size 3 without\n    replacement:\n\n    >>> np.random.choice(5, 3, replace=False)\n    array([3,1,0])\n    >>> #This is equivalent to np.random.permutation(np.arange(5))[:3]\n\n    Generate a non-uniform random sample from np.arange(5) of size\n    3 without replacement:\n\n    >>> np.random.choice(5, 3, replace=False, p=[0.1, 0, 0.3, 0.6, 0])\n    array([2, 3, 0])\n    \"\"\"\n    return _mx_nd_np.random.choice(a, size, replace, p, device, out)\n\n\n@wrap_ctx_to_device_func\ndef rayleigh(scale=1.0, size=None, device=None, out=None):\n    r\"\"\"Draw samples from a Rayleigh distribution.\n\n    The :math:`\\chi` and Weibull distributions are generalizations of the\n    Rayleigh.\n\n    Parameters\n    ----------\n    scale : float, optional\n        Scale, also equals the mode. Must be non-negative. Default is 1.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``scale`` is a scalar.  Otherwise,\n        ``np.array(scale).size`` samples are drawn.\n    device : Device, optional\n        Device context of output, default is current device.\n    out : ``ndarray``, optional\n        Store output to an existing ``ndarray``.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized Rayleigh distribution.\n    \"\"\"\n    return _mx_nd_np.random.rayleigh(scale, size, device, out)\n\n\ndef rand(*size, **kwargs):\n    r\"\"\"Random values in a given shape.\n\n    Create an array of the given shape and populate it with random\n    samples from a uniform distribution over [0, 1).\n\n    Parameters\n    ----------\n    d0, d1, ..., dn : int, optional\n        The dimensions of the returned array, should be all positive.\n        If no argument is given a single Python float is returned.\n\n    Returns\n    -------\n    out : ndarray\n       Random values.\n\n    Examples\n    --------\n    >>> np.random.rand(3,2)\n    array([[ 0.14022471,  0.96360618],  #random\n           [ 0.37601032,  0.25528411],  #random\n           [ 0.49313049,  0.94909878]]) #random\n    \"\"\"\n    output_shape = ()\n    for s in size:\n        output_shape += (s,)\n    return _mx_nd_np.random.uniform(0, 1, size=output_shape, **kwargs)\n\n\n@wrap_ctx_to_device_func\ndef exponential(scale=1.0, size=None, device=None, out=None):\n    r\"\"\"Draw samples from an exponential distribution.\n\n    Parameters\n    ----------\n    scale : float or array_like of floats\n        The scale parameter, :math:`\\beta = 1/\\lambda`. Must be\n        non-negative.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``scale`` is a scalar.  Otherwise,\n        ``np.array(scale).size`` samples are drawn.\n    device : Device, optional\n        Device context of output, default is current device.\n    out : ``ndarray``, optional\n        Store output to an existing ``ndarray``.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized exponential distribution.\n    \"\"\"\n    return _mx_nd_np.random.exponential(scale, size=size, device=device, out=out)\n\n\n@wrap_ctx_to_device_func\ndef weibull(a, size=None, device=None, out=None):\n    r\"\"\"Draw samples from a 1-parameter Weibull distribution with given parameter a\n    via inversion.\n\n    Parameters\n    ----------\n    a : float or array_like of floats\n        Shape of the distribution. Must be non-negative.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``a`` is a scalar. Otherwise,\n        ``np.array(a).size`` samples are drawn.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the 1-parameter Weibull distribution.\n\n    Examples\n    --------\n    >>> np.random.weibull(a=5)\n    array(0.9553641)\n    >>> np.random.weibull(a=5, size=[2,3])\n    array([[1.0466299 , 1.1320982 , 0.98415005],\n          [1.1430776 , 0.9532727 , 1.1344457 ]])\n    >>> np.random.weibull(a=np.array([2,3])\n    array([0.98843634, 1.0125613 ])\n    The Weibull distribution is one of a class of Generalized Extreme\n    Value (GEV) distributions. This class includes the Gumbel and Frechet\n    distributions.\n    The probability density for the Weibull distribution is\n    f(x) = \\frac{a}{\\lambda}(\\frac{x}{\\lambda})^{a-1}e^{-(x/\\lambda)^a},\n    where a is the shape and \\lambda the scale. The generated 1-parameter Weibull\n    sample has the scale parameter \\lambda = 1.\n    The Weibull distribution is commonly used in reliability engineering to\n    model time to failure, in modeling particle sizes, in information retrieval\n    to model dwell time on pages, in quantitative finance to model risk etc.\n    \"\"\"\n    return _mx_nd_np.random.weibull(a, size=size, device=device, out=out)\n\n\n@wrap_ctx_to_device_func\ndef pareto(a, size=None, device=None, out=None):\n    r\"\"\"Draw samples from a Pareto II or Lomax distribution with specified shape a.\n\n    Parameters\n    ----------\n    a : float or array_like of floats\n            Shape of the distribution. Must be > 0.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``a`` is a scalar. Otherwise,\n        ``np.array(a).size`` samples are drawn.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the Pareto distribution.\n\n    Examples\n    --------\n    >>> np.random.pareto(a=5)\n    array(0.12749612)\n    >>> mx.numpy.random.pareto(a=5, size=[2,3])\n    array([[0.06933999, 0.0344373 , 0.10654891],\n            [0.0311172 , 0.12911797, 0.03370714]])\n    >>> np.random.pareto(a=np.array([2,3])\n    array([0.26636696, 0.15685666])\n    The probability density for the Pareto distribution is f(x) = \\frac{am^a}{x^{a+1}}\n    where a is the shape and m the scale. Here m is assumed 1. The Pareto distribution\n    is a power law distribution. Pareto created it to describe the wealth in the economy.\n    \"\"\"\n    return _mx_nd_np.random.pareto(a, size=size, device=device, out=out)\n\n\n@wrap_ctx_to_device_func\ndef power(a, size=None, device=None, out=None):\n    r\"\"\"Draw samples in [0, 1] from a power distribution with given parameter a.\n\n    Parameters\n    ----------\n    a : float or array_like of floats\n        Shape of the distribution. Must be > 0.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``a`` is a scalar. Otherwise,\n        ``np.array(a).size`` samples are drawn.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the power distribution.\n\n    Examples\n    --------\n    >>> np.random.power(a=5)\n    array(0.8602478)\n    >>> np.random.power(a=5, size=[2,3])\n    array([[0.988391  , 0.5153122 , 0.9383134 ],\n           [0.9078098 , 0.87819266, 0.730635]])\n    >>> np.random.power(a=np.array([2,3])\n    array([0.7499419 , 0.88894516])\n    The probability density function is f(x; a) = ax^{a-1}, 0 \\le x \\le 1, a>0.\n    The power distribution is just the inverse of the Pareto distribution and\n    a special case of the Beta distribution.\n    \"\"\"\n    return _mx_nd_np.random.power(a, size=size, device=device, out=out)\n\n\ndef shuffle(x):\n    \"\"\"\n    Modify a sequence in-place by shuffling its contents.\n\n    This function only shuffles the array along the first axis of a\n    multi-dimensional array. The order of sub-arrays is changed but\n    their contents remain the same.\n\n    Parameters\n    ----------\n    x: ndarray\n        The array or list to be shuffled.\n\n    Examples\n    --------\n    >>> arr = np.arange(10)\n    >>> np.random.shuffle(arr)\n    >>> arr\n    array([5., 1., 0., 6., 7., 3., 9., 8., 4., 2.])  # random\n\n    Multi-dimensional arrays are only shuffled along the first axis:\n\n    >>> arr = np.arange(9).reshape((3, 3))\n    >>> np.random.shuffle(arr)\n    >>> arr\n    array([[6., 7., 8.], # random\n           [3., 4., 5.],\n           [0., 1., 2.]])\n    \"\"\"\n    _mx_nd_np.random.shuffle(x)\n\n\n@wrap_ctx_to_device_func\ndef gamma(shape, scale=1.0, size=None, dtype=None, device=None, out=None):\n    \"\"\"Draw samples from a Gamma distribution.\n\n    Samples are drawn from a Gamma distribution with specified parameters,\n    `shape` (sometimes designated \"k\") and `scale` (sometimes designated\n    \"theta\"), where both parameters are > 0.\n\n    The Gamma distribution is often used to model the times to failure of\n    electronic components, and arises naturally in processes for which the\n    waiting times between Poisson distributed events are relevant.\n\n    Parameters\n    ----------\n    shape : float or array_like of floats\n        The shape of the gamma distribution. Should be greater than zero.\n    scale : float or array_like of floats, optional\n        The scale of the gamma distribution. Should be greater than zero.\n        Default is equal to 1.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``shape`` and ``scale`` are both scalars.\n        Otherwise, ``np.broadcast(shape, scale).size`` samples are drawn.\n    device : Device, optional\n        Device context of output. Default is current device.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized gamma distribution.\n    \"\"\"\n    return _mx_nd_np.random.gamma(shape, scale, size, dtype, device, out)\n\n\n@wrap_ctx_to_device_func\ndef beta(a, b, size=None, dtype=None, device=None):\n    r\"\"\"Draw samples from a Beta distribution.\n\n    The Beta distribution is a special case of the Dirichlet distribution,\n    and is related to the Gamma distribution.  It has the probability\n    distribution function\n\n    .. math:: f(x; a,b) = \\frac{1}{B(\\alpha, \\beta)} x^{\\alpha - 1}\n                                                     (1 - x)^{\\beta - 1},\n\n    where the normalisation, B, is the beta function,\n\n    .. math:: B(\\alpha, \\beta) = \\int_0^1 t^{\\alpha - 1}\n                                 (1 - t)^{\\beta - 1} dt.\n\n    It is often seen in Bayesian inference and order statistics.\n\n    Parameters\n    ----------\n    a : float or array_like of floats\n        Alpha, positive (>0).\n    b : float or array_like of floats\n        Beta, positive (>0).\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``a`` and ``b`` are both scalars.\n        Otherwise, ``np.broadcast(a, b).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'.\n        Dtype 'float32' or 'float64' is strongly recommended,\n        since lower precision might lead to out of range issue.\n    device : Device, optional\n        Device context of output. Default is current device.\n\n    Notes\n    -----\n    To use this operator with scalars as input, please run\n    ``npx.set_np()`` first.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized beta distribution.\n    \"\"\"\n    return _mx_nd_np.random.beta(a, b, size=size, dtype=dtype, device=device)\n\n\n@wrap_ctx_to_device_func\ndef f(dfnum, dfden, size=None, device=None):\n    r\"\"\"Draw samples from an F distribution.\n\n    Samples are drawn from an F distribution with specified parameters,\n    `dfnum` (degrees of freedom in numerator) and `dfden` (degrees of\n    freedom in denominator), where both parameters must be greater than\n    zero.\n\n    The random variate of the F distribution (also known as the\n    Fisher distribution) is a continuous probability distribution\n    that arises in ANOVA tests, and is the ratio of two chi-square\n    variates.\n\n    Parameters\n    ----------\n    dfnum : float or ndarray of floats\n        Degrees of freedom in numerator, must be > 0.\n    dfden : float or ndarray of float\n        Degrees of freedom in denominator, must be > 0.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``dfnum`` and ``dfden`` are both scalars.\n        Otherwise, ``np.broadcast(dfnum, dfden).size`` samples are drawn.\n    device : Device, optional\n        Device context of output. Default is current device.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized Fisher distribution.\n\n    Examples\n    --------\n    An example from Glantz[1], pp 47-40:\n\n    Two groups, children of diabetics (25 people) and children from people\n    without diabetes (25 controls). Fasting blood glucose was measured,\n    case group had a mean value of 86.1, controls had a mean value of\n    82.2. Standard deviations were 2.09 and 2.49 respectively. Are these\n    data consistent with the null hypothesis that the parents diabetic\n    status does not affect their children's blood glucose levels?\n    Calculating the F statistic from the data gives a value of 36.01.\n\n    Draw samples from the distribution:\n\n    >>> dfnum = 1. # between group degrees of freedom\n    >>> dfden = 48. # within groups degrees of freedom\n    >>> s = np.random.f(dfnum, dfden, 1000)\n\n    The lower bound for the top 1% of the samples is :\n\n    >>> np.sort(s)[-10]\n    7.61988120985 # random\n\n    So there is about a 1% chance that the F statistic will exceed 7.62,\n    the measured value is 36, so the null hypothesis is rejected at the 1%\n    level.\n    \"\"\"\n    return _mx_nd_np.random.f(dfnum, dfden, size=size, device=device)\n\n\n@wrap_ctx_to_device_func\ndef chisquare(df, size=None, dtype=None, device=None):\n    r\"\"\"Draw samples from a chi-square distribution.\n\n    When `df` independent random variables, each with standard normal\n    distributions (mean 0, variance 1), are squared and summed, the\n    resulting distribution is chi-square (see Notes).  This distribution\n    is often used in hypothesis testing.\n\n    Parameters\n    ----------\n    df : float or ndarray of floats\n         Number of degrees of freedom, must be > 0.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``df`` is a scalar.  Otherwise,\n        ``np.array(df).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'.\n    device : Device, optional\n        Device context of output. Default is current device.\n\n    Returns\n    -------\n    out : ndarray or scalar\n        Drawn samples from the parameterized `chi-square distribution` [1]_.\n\n    Raises\n    ------\n    ValueError\n        When `df` <= 0 or when an inappropriate `size`\n        is given.\n\n    Notes\n    -----\n    The variable obtained by summing the squares of `df` independent,\n    standard normally distributed random variables:\n\n    .. math:: Q = \\sum_{i=0}^{\\mathtt{df}} X^2_i\n\n    is chi-square distributed, denoted\n\n    .. math:: Q \\sim \\chi^2_k.\n\n    The probability density function of the chi-squared distribution is\n\n    .. math:: p(x) = \\frac{(1/2)^{k/2}}{\\Gamma(k/2)}\n                     x^{k/2 - 1} e^{-x/2},\n\n    where :math:`\\Gamma` is the gamma function,\n\n    .. math:: \\Gamma(x) = \\int_0^{-\\infty} t^{x - 1} e^{-t} dt.\n\n    References\n    ----------\n    .. [1] NIST \"Engineering Statistics Handbook\"\n           https://www.itl.nist.gov/div898/handbook/eda/section3/eda3666.htm\n\n    Examples\n    --------\n    >>> np.random.chisquare(2,4)\n    array([ 1.89920014,  9.00867716,  3.13710533,  5.62318272]) # random\n    \"\"\"\n    return _mx_nd_np.random.chisquare(df, size=size, dtype=dtype, device=device)\n\n\ndef randn(*size, **kwargs):\n    r\"\"\"Return a sample (or samples) from the \"standard normal\" distribution.\n    If positive, int_like or int-convertible arguments are provided,\n    `randn` generates an array of shape ``(d0, d1, ..., dn)``, filled\n    with random floats sampled from a univariate \"normal\" (Gaussian)\n    distribution of mean 0 and variance 1 (if any of the :math:`d_i` are\n    floats, they are first converted to integers by truncation). A single\n    float randomly sampled from the distribution is returned if no\n    argument is provided.\n    This is a convenience function.  If you want an interface that takes a\n    tuple as the first argument, use `numpy.random.standard_normal` instead.\n    Parameters\n    ----------\n    d0, d1, ..., dn : int, optional\n        The dimensions of the returned array, should be all positive.\n        If no argument is given a single Python float is returned.\n    Returns\n    -------\n    Z : ndarray\n        A ``(d0, d1, ..., dn)``-shaped array of floating-point samples from\n        the standard normal distribution, or a single such float if\n        no parameters were supplied.\n    Notes\n    -----\n    For random samples from :math:`N(\\mu, \\sigma^2)`, use:\n    ``sigma * np.random.randn(...) + mu``\n    Examples\n    --------\n    >>> np.random.randn()\n    2.1923875335537315 #random\n    Two-by-four array of samples from N(3, 6.25):\n    >>> 2.5 * np.random.randn(2, 4) + 3\n    array([[-4.49401501,  4.00950034, -1.81814867,  7.29718677],  #random\n        [ 0.39924804,  4.68456316,  4.99394529,  4.84057254]]) #random\n    \"\"\"\n    output_shape = ()\n    for s in size:\n        output_shape += (s,)\n    return _mx_nd_np.random.normal(0, 1, size=output_shape, **kwargs)\n\n\n@wrap_ctx_to_device_func\ndef laplace(loc=0.0, scale=1.0, size=None, dtype=None, device=None, out=None):\n    r\"\"\"Draw random samples from a Laplace distribution.\n\n    Samples are distributed according to a Laplace distribution parametrized\n    by *loc* (mean) and *scale* (the exponential decay).\n\n    Parameters\n    ----------\n    loc : float, The position of the distribution peak.\n\n    scale : float, the exponential decay.\n\n    size : int or tuple of ints, optional. Output shape.\n        If the given shape is, e.g., (m, n, k), then m * n * k samples are drawn.\n        Default is None, in which case a single value is returned.\n\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    device : Device, optional\n        Device context of output. Default is current device.\n    out : ``ndarray``, optional\n        Store output to an existing ``ndarray``.\n\n    Returns\n    -------\n    out : ndarray\n        Drawn samples from the parameterized Laplace distribution.\n    \"\"\"\n    return _mx_nd_np.random.laplace(loc, scale, size, dtype, device, out)\n"
  },
  {
    "path": "python/mxnet/numpy/set_functions.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Standard Array API for creating and operating on sets.\"\"\"\n\nfrom collections import namedtuple\n\nfrom ..ndarray import numpy as _mx_nd_np\n\n\n__all__ = ['unique_all', 'unique_inverse', 'unique_values']\n\n\ndef unique_all(x):\n    \"\"\"\n    Returns the unique elements of an input array `x`\n\n    Notes\n    -----\n    `unique_all` is a standard API in\n    https://data-apis.org/array-api/latest/API_specification/set_functions.html#unique-all-x\n    instead of an official NumPy operator.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array. This will be flattened if it is not already 1-D.\n\n    Returns\n    -------\n    out : Tuple[ndarray, ndarray, ndarray, ndarray]\n        a namedtuple (values, indices, inverse_indices, counts):\n        values : ndarray\n            The sorted unique values.\n        indices : ndarray, optional\n            The indices of the first occurrences of the unique values in the\n            original array.\n        inverse_indices : ndarray\n            The indices to reconstruct the original array from the\n            unique array.\n        counts : ndarray\n            The number of times each of the unique values comes up in the\n            original array.\n    \"\"\"\n    UniqueAll = namedtuple('UniqueAll', ['values', 'indices', 'inverse_indices', 'counts'])\n    return UniqueAll(*_mx_nd_np.unique(x, True, True, True))\n\n\ndef unique_inverse(x):\n    \"\"\"\n    Returns the unique elements of an input array `x` and the indices\n    from the set of unique elements that reconstruct `x`.\n\n    Notes\n    -----\n    `unique_inverse` is a standard API in\n    https://data-apis.org/array-api/latest/API_specification/set_functions.html#unique-inverse-x\n    instead of an official NumPy operator.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array. This will be flattened if it is not already 1-D.\n\n    Returns\n    -------\n    out : Tuple[ndarray, ndarray]\n        a namedtuple (values, inverse_indices):\n        values : ndarray\n            The sorted unique values.\n        inverse_indices : ndarray\n            The indices to reconstruct the original array from the\n            unique array.\n    \"\"\"\n    UniqueInverse = namedtuple('UniqueInverse', ['values', 'inverse_indices'])\n    return UniqueInverse(*_mx_nd_np.unique(x, False, True, False))\n\n\ndef unique_values(x):\n    \"\"\"\n    Returns the unique elements of an input array `x`.\n\n    Notes\n    -----\n    `unique_values` is a standard API in\n    https://data-apis.org/array-api/latest/API_specification/set_functions.html#unique-values-x\n    instead of an official NumPy operator.\n\n    Parameters\n    ----------\n    x : ndarray\n        Input array. This will be flattened if it is not already 1-D.\n\n    Returns\n    -------\n    out : ndarray\n        The sorted unique values.\n    \"\"\"\n    return _mx_nd_np.unique(x, False, False, False)\n"
  },
  {
    "path": "python/mxnet/numpy/stride_tricks.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Util functions with broadcast.\"\"\"\n\nfrom ..ndarray.ndarray import _get_broadcast_shape\nfrom ..ndarray import numpy as _mx_nd_np\n\n\n__all__ = ['broadcast_arrays']\n\n\ndef _broadcast_shape(*args):\n    shape = ()\n    for arr in args:\n        shape = _get_broadcast_shape(shape, arr.shape)\n    return shape\n\n\ndef broadcast_arrays(*args):\n    \"\"\"\n    Broadcast any number of arrays against each other.\n\n    Parameters\n    ----------\n    `*args` : a list of ndarrays\n        The arrays to broadcast.\n\n    Returns\n    -------\n    broadcasted : list of arrays\n        These arrays are copies of the original arrays unless that all the input\n        arrays have the same shape, the input list of arrays are returned\n        instead of a list of copies.\n\n    Examples\n    --------\n    >>> x = np.array([[1,2,3]])\n    >>> y = np.array([[4],[5]])\n    >>> np.broadcast_arrays(x, y)\n    [array([[1., 2., 3.],\n           [1., 2., 3.]]), array([[4., 4., 4.],\n           [5., 5., 5.]])]\n    \"\"\"\n    shape = _broadcast_shape(*args)\n\n    if all(array.shape == shape for array in args):\n        # Common case where nothing needs to be broadcasted.\n        return list(args)\n\n    return [_mx_nd_np.broadcast_to(array, shape) for array in args]\n"
  },
  {
    "path": "python/mxnet/numpy/type_functions.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Type functions for the numpy module.\"\"\"\n\nfrom typing import NamedTuple\n\nimport numpy as onp\nfrom .multiarray import ndarray\nfrom .utils import _type_promotion_table\n\n\n__all__ = ['can_cast', 'finfo', 'iinfo', 'result_type']\n\nclass finfo_obj(NamedTuple):\n    bits: int\n    eps: float\n    max: float\n    min: float\n    smallest_normal: float\n\n\nclass iinfo_obj(NamedTuple):\n    bits: int\n    max: int\n    min: int\n\n\ndef can_cast(from_, to):\n    \"\"\"\n    Returns True if cast between data types can occur according to\n    the casting rule. If from is a scalar or array scalar,\n    also returns True if the scalar value can be cast without\n    overflow or truncation to an integer.\n    Parameters\n    ----------\n    from_ : dtype, ndarray or scalar\n        Data type, scalar, or array to cast from.\n    to : dtype\n        Data type to cast to.\n    Returns\n    -------\n    out : bool\n        True if cast can occur according to the casting rule.\n    \"\"\"\n    if isinstance(from_, ndarray):\n        from_ = from_.asnumpy()\n    return onp.can_cast(from_, to)\n\n\ndef finfo(dtype):\n    \"\"\"\n    Machine limits for floating-point data types.\n    Notes\n    -----\n    `finfo` is a standard API in\n    https://data-apis.org/array-api/latest/API_specification/data_type_functions.html#finfo-type\n    instead of an official NumPy operator.\n    Parameters\n    ----------\n    dtype : ndarray, float or dtype\n        Kind of floating point data-type about which to get information.\n    Returns\n    -------\n    out : finfo object\n        an object having the following attributes:\n            - bits : int\n                number of bits occupied by the floating-point data type.\n            - eps : float\n                difference between 1.0 and the next smallest representable floating-point\n                number larger than 1.0 according to the IEEE-754 standard.\n            - max : float\n                largest representable number.\n            - min : float\n                smallest representable number.\n            - smallest_normal : float\n                smallest positive floating-point number with full precision.\n    \"\"\"\n    f_info = onp.finfo(dtype)\n    return finfo_obj(f_info.bits, float(f_info.eps),\n                     float(f_info.max), float(f_info.min), float(f_info.tiny))\n\n\ndef iinfo(dtype):\n    \"\"\"\n    Machine limits for floating-point data types.\n    Notes\n    -----\n    `iinfo` is a standard API in\n    https://data-apis.org/array-api/latest/API_specification/data_type_functions.html#iinfo-type\n    instead of an official NumPy operator.\n    Parameters\n    ----------\n    dtype : ndarray, integer or dtype\n        The kind of integer data type to get information about.\n    Returns\n    -------\n    out : iinfo object\n        an object having the following attributes:\n            - bits : int\n                number of bits occupied by the type\n            - max : int\n                largest representable number.\n            - min : int\n                smallest representable number.\n    \"\"\"\n    i_info = onp.iinfo(dtype)\n    return iinfo_obj(i_info.bits, i_info.max, i_info.min)\n\n\ndef _get_dtype(array_or_dtype):\n    \"\"\"Utility function for result_type\"\"\"\n    if isinstance(array_or_dtype, (ndarray, onp.ndarray)):\n        return array_or_dtype.dtype\n    elif isinstance(array_or_dtype, onp.dtype):\n        return array_or_dtype\n    else:\n        raise ValueError(\"Inputs of result_type must be ndarrays or dtypes\")\n\n\ndef result_type(*arrays_and_dtypes):\n    \"\"\"\n    Returns the dtype that results from applying the type promotion rules to the arguments.\n    Notes\n    -----\n    `result_type` is a standard API in\n    https://data-apis.org/array-api/latest/API_specification/data_type_functions.html#result-type-arrays-and-dtypes\n    instead of an official NumPy operator.\n    Parameters\n    ----------\n    arrays_and_dtypes : mixed ndarrays and dtypes\n        an arbitrary number of input arrays and/or dtypes.\n    Returns\n    -------\n    out : dtype\n        the dtype resulting from an operation involving the input arrays and dtypes.\n    \"\"\"\n    if len(arrays_and_dtypes) > 0:\n        ret = _get_dtype(arrays_and_dtypes[0])\n        for d in arrays_and_dtypes[1:]:\n            dd = _get_dtype(d)\n            if (ret, dd) in _type_promotion_table:\n                ret = _type_promotion_table[ret, dd]\n            elif (dd, ret) in _type_promotion_table:\n                ret = _type_promotion_table[dd, ret]\n            else:\n                raise TypeError(\"Unknown type promotion between {} and {}\".format(ret, dd))\n        return ret\n    raise ValueError(\"at least one array or dtype is required\")\n"
  },
  {
    "path": "python/mxnet/numpy/utils.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Util functions for the numpy module.\"\"\"\n\n\n\nimport numpy as onp\n\n__all__ = ['float16', 'float32', 'float64', 'uint8', 'int32', 'int8', 'int64',\n           'int16', 'uint16', 'uint32', 'uint64',\n           'bool', 'bool_', 'pi', 'inf', 'nan', 'PZERO', 'NZERO', 'newaxis',\n           'e', 'NINF', 'PINF', 'NAN', 'NaN',\n           '_STR_2_DTYPE_', '_DTYPE_2_STR_', '_type_promotion_table',\n           'integer_dtypes', 'floating_dtypes', 'boolean_dtypes', 'numeric_dtypes']\n\npy_bool = bool\n\nfloat16 = onp.dtype(onp.float16)\nfloat32 = onp.dtype(onp.float32)\nfloat64 = onp.dtype(onp.float64)\nuint8 = onp.dtype(onp.uint8)\nint32 = onp.dtype(onp.int32)\nint8 = onp.dtype(onp.int8)\nint64 = onp.dtype(onp.int64)\nbool_ = onp.dtype(onp.bool_)\nbool = onp.dtype(onp.bool)\nint16 = onp.dtype(onp.int16)\nuint16 = onp.dtype(onp.uint16)\nuint32 = onp.dtype(onp.uint32)\nuint64 = onp.dtype(onp.uint64)\n\npi = onp.pi\ninf = onp.inf\nnan = onp.nan\nPZERO = onp.PZERO\nNZERO = onp.NZERO\nNINF = onp.NINF\nPINF = onp.PINF\ne = onp.e\nNAN = onp.NAN\nNaN = onp.NaN\n\nnewaxis = None\n\n_STR_2_DTYPE_ = {'float16': float16, 'float32': float32, 'float64': float64, 'float': float64,\n                 'int8': int8, 'int16': int16, 'int32': int32, 'int64': int64, 'int': int64,\n                 'uint8': uint8, 'uint16': uint16, 'uint32': uint32, 'uint64': uint64,\n                 'bool': bool, 'bool_': bool_, 'None': None}\n\n_DTYPE_2_STR_ = {float16: 'float16', float32: 'float32', float64: 'float64', float: 'float64',\n                 int8: 'int8', int16: 'int16', int32: 'int32', int64: 'int64', int:'int64',\n                 uint8: 'uint8', uint16: 'uint16', uint32: 'uint32', uint64: 'uint64',\n                 bool: 'bool', bool_: 'bool_', py_bool: 'bool', None: 'None'}\n\n_ONP_OP_MODULES = [onp, onp.linalg, onp.random, onp.fft]\n\n\ndef _get_np_op(name):\n    \"\"\"Get official NumPy operator with `name`. If not found, raise ValueError.\"\"\"\n    for mod in _ONP_OP_MODULES:\n        op = getattr(mod, name, None)\n        if op is not None:\n            return op\n    raise ValueError('Operator `{}` is not supported by `mxnet.numpy`.'.format(name))\n\n\n_type_promotion_table = {\n    # signed integer type promotion\n    (int8, int8): int8,\n    (int8, int16): int16,\n    (int8, int32): int32,\n    (int8, int64): int64,\n    (int16, int16): int16,\n    (int16, int32): int32,\n    (int16, int64): int64,\n    (int32, int32): int32,\n    (int32, int64): int64,\n    (int64, int64): int64,\n    # unsigned integer type promotion\n    (uint8, uint8): uint8,\n    (uint8, uint16): uint16,\n    (uint8, uint32): uint32,\n    (uint8, uint64): uint64,\n    (uint16, uint16): uint16,\n    (uint16, uint32): uint32,\n    (uint16, uint64): uint64,\n    (uint32, uint32): uint32,\n    (uint32, uint64): uint64,\n    (uint64, uint64): uint64,\n    # mixed signed and unsigned integer type promotion\n    (int8, uint8): int16,\n    (int8, uint16): int32,\n    (int8, uint32): int64,\n    (int16, uint8): int16,\n    (int16, uint16): int32,\n    (int16, uint32): int64,\n    (int32, uint8): int32,\n    (int32, uint16): int32,\n    (int32, uint32): int64,\n    (int64, uint8): int64,\n    (int64, uint16): int64,\n    (int64, uint32): int64,\n    # float type promotion\n    (float16, float16): float16,\n    (float16, float32): float32,\n    (float16, float64): float64,\n    (float32, float32): float32,\n    (float32, float64): float64,\n    (float64, float64): float64,\n    # bool type promotion\n    (bool, bool): bool,\n    # mixed integer and float16 type promotion\n    (int8, float16): float16,\n    (int16, float16): float16,\n    (int32, float16): float16,\n    (int64, float16): float16,\n    (uint8, float16): float16,\n    (uint16, float16): float16,\n    (uint32, float16): float16,\n    (uint64, float16): float16,\n    # mixed integer and float16 type promotion\n    (int8, float32): float32,\n    (int16, float32): float32,\n    (int32, float32): float32,\n    (int64, float32): float32,\n    (uint8, float32): float32,\n    (uint16, float32): float32,\n    (uint32, float32): float32,\n    (uint64, float32): float32,\n    # mixed integer and float32 type promotion\n    (int8, float32): float32,\n    (int16, float32): float32,\n    (int32, float32): float32,\n    (int64, float32): float32,\n    (uint8, float32): float32,\n    (uint16, float32): float32,\n    (uint32, float32): float32,\n    (uint64, float32): float32,\n    # mixed integer and float64 type promotion\n    (int8, float64): float64,\n    (int16, float64): float64,\n    (int32, float64): float64,\n    (int64, float64): float64,\n    (uint8, float64): float64,\n    (uint16, float64): float64,\n    (uint32, float64): float64,\n    (uint64, float64): float64,\n    # mixed bool and other type promotion\n    (bool, int8): int8,\n    (bool, int16): int16,\n    (bool, int32): int32,\n    (bool, int64): int64,\n    (bool, uint8): uint8,\n    (bool, uint16): uint16,\n    (bool, uint32): uint32,\n    (bool, uint64): uint64,\n    (bool, float16): float16,\n    (bool, float32): float32,\n    (bool, float64): float64,\n}\n\ninteger_dtypes = [\n    int8,\n    int16,\n    int32,\n    int64,\n    uint8,\n    uint16,\n    uint32,\n    uint64,\n]\n\nfloating_dtypes = [\n    float16,\n    float32,\n    float64,\n]\n\nnumeric_dtypes = [\n    *integer_dtypes,\n    *floating_dtypes,\n]\n\nboolean_dtypes = [\n    bool_,\n]\n"
  },
  {
    "path": "python/mxnet/numpy_dispatch_protocol.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Utils for registering NumPy array function protocol for mxnet.numpy ops.\"\"\"\n\nimport functools\nimport numpy as _np\nfrom . import numpy as mx_np  # pylint: disable=reimported\nfrom .numpy.multiarray import _NUMPY_ARRAY_FUNCTION_DICT, _NUMPY_ARRAY_UFUNC_DICT\n\n\ndef _find_duplicate(strs):\n    str_set = set()\n    for s in strs:\n        if s in str_set:\n            return s\n        else:\n            str_set.add(s)\n    return None\n\n\ndef _implements(numpy_function):\n    \"\"\"Register an __array_function__ implementation for MyArray objects.\"\"\"\n    def decorator(func):\n        _NUMPY_ARRAY_FUNCTION_DICT[numpy_function] = func\n        return func\n    return decorator\n\n\ndef with_array_function_protocol(func):\n    \"\"\"A decorator for functions that expect array function protocol.\n    The decorated function only runs when NumPy version >= 1.17.\"\"\"\n    from distutils.version import LooseVersion\n    cur_np_ver = LooseVersion(_np.__version__)\n    np_1_17_ver = LooseVersion('1.17')\n\n    @functools.wraps(func)\n    def _run_with_array_func_proto(*args, **kwargs):\n        if cur_np_ver >= np_1_17_ver:\n            try:\n                func(*args, **kwargs)\n            except Exception as e:\n                raise RuntimeError('Running function {} with NumPy array function protocol failed'\n                                   ' with exception {}'\n                                   .format(func.__name__, str(e)))\n\n    return _run_with_array_func_proto\n\n\ndef with_array_ufunc_protocol(func):\n    \"\"\"A decorator for functions that expect array ufunc protocol.\n    The decorated function only runs when NumPy version >= 1.15.\"\"\"\n    from distutils.version import LooseVersion\n    cur_np_ver = LooseVersion(_np.__version__)\n    np_1_15_ver = LooseVersion('1.15')\n\n    @functools.wraps(func)\n    def _run_with_array_ufunc_proto(*args, **kwargs):\n        if cur_np_ver >= np_1_15_ver:\n            try:\n                func(*args, **kwargs)\n            except Exception as e:\n                raise RuntimeError('Running function {} with NumPy array ufunc protocol failed'\n                                   ' with exception {}'\n                                   .format(func.__name__, str(e)))\n\n    return _run_with_array_ufunc_proto\n\n\n_NUMPY_ARRAY_FUNCTION_LIST = [\n    'all',\n    'any',\n    'sometrue',\n    'argmin',\n    'argmax',\n    'around',\n    'round',\n    'round_',\n    'argsort',\n    'sort',\n    'append',\n    'broadcast_arrays',\n    'broadcast_to',\n    'clip',\n    'concatenate',\n    'copy',\n    'cumsum',\n    'diag',\n    'diagonal',\n    'diagflat',\n    'dot',\n    'expand_dims',\n    'fix',\n    'flip',\n    'flipud',\n    'fliplr',\n    'inner',\n    'insert',\n    'interp',\n    'max',\n    'amax',\n    'mean',\n    'min',\n    'amin',\n    'nonzero',\n    'ones_like',\n    'atleast_1d',\n    'atleast_2d',\n    'atleast_3d',\n    'prod',\n    'product',\n    'ravel',\n    'repeat',\n    'reshape',\n    'roll',\n    'split',\n    'array_split',\n    'hsplit',\n    'vsplit',\n    'dsplit',\n    'squeeze',\n    'stack',\n    'std',\n    'sum',\n    'swapaxes',\n    'take',\n    'tensordot',\n    'tile',\n    'transpose',\n    'unique',\n    'unravel_index',\n    'flatnonzero',\n    'diag_indices_from',\n    'delete',\n    'var',\n    'vdot',\n    'vstack',\n    'column_stack',\n    'hstack',\n    'dstack',\n    'zeros_like',\n    'linalg.norm',\n    'linalg.cholesky',\n    'linalg.inv',\n    'linalg.solve',\n    'linalg.tensorinv',\n    'linalg.tensorsolve',\n    'linalg.lstsq',\n    'linalg.pinv',\n    'linalg.eigvals',\n    'linalg.eig',\n    'linalg.eigvalsh',\n    'linalg.eigh',\n    'linalg.qr',\n    'linalg.matrix_rank',\n    'shape',\n    'trace',\n    'tril',\n    'triu',\n    'meshgrid',\n    'outer',\n    'kron',\n    'einsum',\n    'polyval',\n    'shares_memory',\n    'may_share_memory',\n    'quantile',\n    'median',\n    'percentile',\n    'diff',\n    'ediff1d',\n    'resize',\n    'where',\n    'full_like',\n    'bincount',\n    'empty_like',\n    'nan_to_num',\n    'isnan',\n    'isfinite',\n    'isposinf',\n    'isneginf',\n    'isinf',\n    'pad',\n    'cross',\n]\n\n\n@with_array_function_protocol\ndef _register_array_function():\n    \"\"\"Register __array_function__ protocol for mxnet.numpy operators so that\n    ``mxnet.numpy.ndarray`` can be fed into the official NumPy operators and\n    dispatched to MXNet implementation.\n\n    Notes\n    -----\n    According the __array_function__ protocol (see the following reference),\n    there are three kinds of operators that cannot be dispatched using this\n    protocol:\n    1. Universal functions, which already have their own protocol in the official\n    NumPy package.\n    2. Array creation functions.\n    3. Dispatch for methods of any kind, e.g., methods on np.random.RandomState objects.\n\n    References\n    ----------\n    https://numpy.org/neps/nep-0018-array-function-protocol.html\n    \"\"\"\n    dup = _find_duplicate(_NUMPY_ARRAY_FUNCTION_LIST)\n    if dup is not None:\n        raise ValueError('Duplicate operator name {} in _NUMPY_ARRAY_FUNCTION_LIST'.format(dup))\n    for op_name in _NUMPY_ARRAY_FUNCTION_LIST:\n        strs = op_name.split('.')\n        if len(strs) == 1:\n            mx_np_op = getattr(mx_np, op_name)\n            onp_op = getattr(_np, op_name)\n            setattr(mx_np, op_name, _implements(onp_op)(mx_np_op))\n        elif len(strs) == 2:\n            mx_np_submodule = getattr(mx_np, strs[0])\n            mx_np_op = getattr(mx_np_submodule, strs[1])\n            onp_submodule = getattr(_np, strs[0])\n            onp_op = getattr(onp_submodule, strs[1])\n            setattr(mx_np_submodule, strs[1], _implements(onp_op)(mx_np_op))\n        else:\n            raise ValueError('Does not support registering __array_function__ protocol '\n                             'for operator {}'.format(op_name))\n\n\n# https://docs.scipy.org/doc/numpy/reference/ufuncs.html#available-ufuncs\n_NUMPY_ARRAY_UFUNC_LIST = [\n    'abs',\n    'fabs',\n    'add',\n    'arctan2',\n    'copysign',\n    'degrees',\n    'hypot',\n    'lcm',\n    'gcd',\n    # 'ldexp',\n    'logaddexp',\n    'subtract',\n    'multiply',\n    'floor_divide',\n    'true_divide',\n    'negative',\n    'power',\n    'mod',\n    'fmod',\n    'matmul',\n    'absolute',\n    'rint',\n    'sign',\n    'exp',\n    'log',\n    'log2',\n    'log10',\n    'expm1',\n    'sqrt',\n    'square',\n    'cbrt',\n    'reciprocal',\n    'invert',\n    'bitwise_not',\n    'remainder',\n    'sin',\n    'cos',\n    'tan',\n    'sinh',\n    'cosh',\n    'tanh',\n    'arcsin',\n    'arccos',\n    'arctan',\n    'arcsinh',\n    'arccosh',\n    'arctanh',\n    'maximum',\n    'fmax',\n    'minimum',\n    'fmin',\n    'ceil',\n    'trunc',\n    'floor',\n    'bitwise_and',\n    'bitwise_xor',\n    'bitwise_or',\n    'logical_and',\n    'logical_or',\n    'logical_xor',\n    'logical_not',\n    'equal',\n    'not_equal',\n    'less',\n    'less_equal',\n    'greater',\n    'greater_equal',\n]\n\n\n@with_array_ufunc_protocol\ndef _register_array_ufunc():\n    \"\"\"Register NumPy array ufunc protocol.\n\n    References\n    ----------\n    https://numpy.org/neps/nep-0013-ufunc-overrides.html\n    \"\"\"\n    dup = _find_duplicate(_NUMPY_ARRAY_UFUNC_LIST)\n    if dup is not None:\n        raise ValueError('Duplicate operator name {} in _NUMPY_ARRAY_UFUNC_LIST'.format(dup))\n    for op_name in _NUMPY_ARRAY_UFUNC_LIST:\n        try:\n            mx_np_op = getattr(mx_np, op_name)\n            _NUMPY_ARRAY_UFUNC_DICT[op_name] = mx_np_op\n        except AttributeError:\n            raise AttributeError('mxnet.numpy does not have operator named {}'.format(op_name))\n\n\n_register_array_function()\n_register_array_ufunc()\n"
  },
  {
    "path": "python/mxnet/numpy_extension/__init__.py",
    "content": "#!/usr/bin/env python\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Module for ops not belonging to the official numpy package for imperative programming.\"\"\"\n\nfrom . import _op\nfrom . import image\nfrom . import random  # pylint: disable=wildcard-import\nfrom . import _register\nfrom . import control_flow\nfrom ._op import *  # pylint: disable=wildcard-import\nfrom .control_flow import *  # pylint: disable=wildcard-import\nfrom ..device import *  # pylint: disable=wildcard-import\nfrom ..util import is_np_shape, is_np_array, set_np, reset_np, get_cuda_compute_capability,\\\n                   is_np_default_dtype, set_np_default_dtype\nfrom ..ndarray import waitall\nfrom .utils import *  # pylint: disable=wildcard-import\n\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/numpy_extension/_op.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for registering numpy_extension ops for imperative programming.\"\"\"\n\nfrom ..ndarray import numpy_extension as _mx_nd_npx\nfrom ..util import set_module\n\n\n__all__ = ['softmax', 'log_softmax', 'masked_softmax', 'masked_log_softmax',\n           'activation', 'batch_norm', 'fully_connected', 'pick', 'convolution',\n           'deconvolution', 'pooling', 'dropout', 'one_hot', 'rnn', 'embedding',\n           'topk', 'layer_norm', 'leaky_relu', 'batch_dot', 'broadcast_like',\n           'arange_like', 'group_norm']\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.numpy_extension')\ndef softmax(data, length=None, axis=-1, temperature=None, use_length=False, dtype=None):\n    r\"\"\"Applies the softmax function.\n\n    The resulting array contains elements in the range (0,1) and the elements along the given axis sum up to 1.\n\n    .. math::\n       softmax(\\mathbf{z/t})_j = \\frac{e^{z_j/t}}{\\sum_{k=1}^K e^{z_k/t}}\n\n    for :math:`j = 1, ..., K`\n\n    t is the temperature parameter in softmax function. By default, t equals 1.0\n\n    Parameters\n    ----------\n    data : NDArray\n        The input array.\n    axis : int, optional, default='-1'\n        The axis along which to compute softmax.\n    length : NDArray\n        The length array.\n    temperature : double or None, optional, default=None\n        Temperature parameter in softmax\n    dtype : {None, 'float16', 'float32', 'float64'},optional, default='None'\n        DType of the output in case this can't be inferred. Defaults to\n        the same as input's dtype if not defined (dtype=None).\n    use_length : boolean or None, optional, default=0\n        Whether to use the length input as a mask over the data input.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Example\n    -------\n    >>> data = np.ones((2, 3))\n    >>> npx.softmax(data, axis=0)\n    array([[0.5, 0.5, 0.5],\n        [0.5, 0.5, 0.5]])\n    >>> npx.softmax(data, axis=1)\n    array([[0.33333334, 0.33333334, 0.33333334],\n        [0.33333334, 0.33333334, 0.33333334]])\n    \"\"\"\n    return _mx_nd_npx.softmax(data, axis=axis, length=length, temperature=temperature,\n                              use_length=use_length, dtype=dtype)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.numpy_extension')\ndef log_softmax(data, axis=-1, length=None, temperature=None, use_length=False, dtype=None):\n    r\"\"\"Computes the log softmax of the input.\n    This is equivalent to computing softmax followed by log.\n\n    Parameters\n    ----------\n    data : NDArray\n        The input array.\n    axis : int, optional, default='-1'\n        The axis along which to compute softmax.\n    length : NDArray\n        The length array.\n    temperature : double or None, optional, default=None\n        Temperature parameter in softmax\n    dtype : {None, 'float16', 'float32', 'float64'},optional, default='None'\n        DType of the output in case this can't be inferred. Defaults to\n        the same as input's dtype if not defined (dtype=None).\n    use_length : boolean or None, optional, default=0\n        Whether to use the length input as a mask over the data input.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Examples\n    --------\n    >>> data = np.array([1, 2, .1])\n    >>> npx.log_softmax(data)\n    array([-1.4170278, -0.4170278, -2.3170278])\n    >>> data = np.array([[1, 2, .1],[.1, 2, 1]])\n    >>> npx.log_softmax(data, axis=0)\n    array([[-0.34115386, -0.6931472 , -1.2411538 ],\n        [-1.2411538 , -0.6931472 , -0.34115386]])\n    \"\"\"\n    return _mx_nd_npx.log_softmax(data, axis=axis, length=length, temperature=temperature,\n                                  use_length=use_length, dtype=dtype)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.numpy_extension')\ndef masked_softmax(data, mask, axis=-1, temperature=1.0, normalize=True):\n    r\"\"\"Applies the softmax function masking elements according to the mask provided\n\n    Parameters\n    ----------\n    data : NDArray\n        The input array.\n    mask : NDArray\n        Mask to apply.\n    axis : int, optional, default='-1'\n        The axis along which to compute softmax.\n    temperature : double or None, optional, default=None\n        Temperature parameter in softmax\n    normalize : boolean or None, optional, default=1\n        Whether to normalize input data x: x = x - max(x)\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Examples\n    --------\n    >>> data = np.arange(5)\n    >>> mask = np.array([1, 0, 1, 0, 1])\n    >>> npx.masked_softmax(data, mask)\n    array([0.01587624, 0.        , 0.11731042, 0.        , 0.8668133 ])\n    >>> data = np.arange(10).reshape((2, 5))\n    >>> npx.masked_softmax(data, mask, axis=0)\n    array([[0.00669285, 0.        , 0.00669285, 0.        , 0.00669285],\n           [0.9933072 , 0.        , 0.9933072 , 0.        , 0.9933072 ]])\n    \"\"\"\n    return _mx_nd_npx.masked_softmax(data, mask, axis=axis, temperature=temperature,\n                                     normalize=normalize)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.numpy_extension')\ndef masked_log_softmax(data, mask, axis=-1, temperature=1.0, normalize=True):\n    r\"\"\"Computes the masked log softmax of the input.\n    This is equivalent to computing masked softmax followed by log.\n\n    Parameters\n    ----------\n    data : NDArray\n        The input array.\n    mask : NDArray\n        Mask to apply.\n    axis : int, optional, default='-1'\n        The axis along which to compute softmax.\n    temperature : double or None, optional, default=None\n        Temperature parameter in softmax\n    normalize : boolean or None, optional, default=1\n        Whether to normalize input data x: x = x - max(x)\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Examples\n    --------\n    >>> data = np.arange(5)\n    >>> mask = np.array([1, 0, 1, 0, 1])\n    >>> npx.masked_log_softmax(data, mask)\n    array([-4.1429286 ,        -inf, -2.1429286 ,        -inf, -0.14292854])\n    >>> data = np.arange(10).reshape((2, 5))\n    >>> npx.masked_log_softmax(data, mask, axis=0)\n    array([[-5.0067153 ,        -inf, -5.0067153 ,        -inf, -5.0067153 ],\n           [-0.00671535,        -inf, -0.00671535,        -inf, -0.00671535]])\n    \"\"\"\n    return _mx_nd_npx.masked_log_softmax(data, mask, axis=axis, temperature=temperature,\n                                         normalize=normalize)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.numpy_extension')\ndef activation(data, act_type='relu', **kwargs):\n    r\"\"\"Applies an activation function element-wise to the input.\n\n    The following activation functions are supported:\n\n    - `log_sigmoid`: :math:`y = log(\\frac{1}{1 + exp(-x)})`\n    - `mish`: :math:`y = x * tanh(log(1 + exp(x)))`\n    - `relu`: Rectified Linear Unit, :math:`y = max(x, 0)`\n    - `sigmoid`: :math:`y = \\frac{1}{1 + exp(-x)}`\n    - `tanh`: Hyperbolic tangent, :math:`y = \\frac{exp(x) - exp(-x)}{exp(x) + exp(-x)}`\n    - `softrelu`: Soft ReLU, or SoftPlus, :math:`y = log(1 + exp(x))`\n    - `softsign`: :math:`y = \\frac{x}{1 + abs(x)}`\n\n    Parameters\n    ----------\n    data : NDArray\n        The input array.\n    act_type : {'log_sigmoid', 'mish', 'relu', 'sigmoid', 'softrelu', 'softsign', 'tanh'}, required\n        Activation function to be applied.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    return _mx_nd_npx.activation(data, act_type=act_type)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.numpy_extension')\ndef batch_norm(x, gamma, beta, running_mean, running_var, eps=1e-3, momentum=0.9,\n               fix_gamma=True, use_global_stats=False, output_mean_var=False, axis=1,\n               cudnn_off=False, min_calib_range=None, max_calib_range=None, **kwargs):\n    r\"\"\"Batch normalization.\n\n    Normalizes a data batch by mean and variance, and applies a scale ``gamma`` as\n    well as offset ``beta``.\n\n    Assume the input has more than one dimension and we normalize along axis 1.\n    We first compute the mean and variance along this axis:\n\n    .. math::\n\n      data\\_mean[i] = mean(data[:,i,:,...]) \\\\\n      data\\_var[i] = var(data[:,i,:,...])\n\n    Then compute the normalized output, which has the same shape as input, as following:\n\n    .. math::\n\n      out[:,i,:,...] = \\frac{data[:,i,:,...] - data\\_mean[i]}{\\sqrt{data\\_var[i]+\\epsilon}} * gamma[i] + beta[i]\n\n    Both *mean* and *var* returns a scalar by treating the input as a vector.\n\n    Assume the input has size *k* on axis 1, then both ``gamma`` and ``beta``\n    have shape *(k,)*. If ``output_mean_var`` is set to be true, then outputs both ``data_mean`` and\n    the inverse of ``data_var``, which are needed for the backward pass. Note that gradient of these\n    two outputs are blocked.\n\n    Besides the inputs and the outputs, this operator accepts two auxiliary\n    states, ``moving_mean`` and ``moving_var``, which are *k*-length\n    vectors. They are global statistics for the whole dataset, which are updated\n    by::\n\n      moving_mean = moving_mean * momentum + data_mean * (1 - momentum)\n      moving_var = moving_var * momentum + data_var * (1 - momentum)\n\n    If ``use_global_stats`` is set to be true, then ``moving_mean`` and\n    ``moving_var`` are used instead of ``data_mean`` and ``data_var`` to compute\n    the output. It is often used during inference.\n\n    The parameter ``axis`` specifies which axis of the input shape denotes\n    the 'channel' (separately normalized groups).  The default is 1.  Specifying -1 sets the channel\n    axis to be the last item in the input shape.\n\n    Both ``gamma`` and ``beta`` are learnable parameters. But if ``fix_gamma`` is true,\n    then set ``gamma`` to 1 and its gradient to 0.\n\n    .. Note::\n      When ``fix_gamma`` is set to True, no sparse support is provided. If ``fix_gamma is`` set to False,\n      the sparse tensors will fallback.\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data to batch normalization\n    gamma : NDArray\n        gamma array\n    beta : NDArray\n        beta array\n    moving_mean : NDArray\n        running mean of input\n    moving_var : NDArray\n        running variance of input\n    eps : double, optional, default=0.0010000000474974513\n        Epsilon to prevent div 0. Must be no less than CUDNN_BN_MIN_EPSILON\n        defined in cudnn.h when using cudnn (usually 1e-5)\n    momentum : float, optional, default=0.899999976\n        Momentum for moving average\n    fix_gamma : boolean, optional, default=1\n        Fix gamma while training\n    use_global_stats : boolean, optional, default=0\n        Whether use global moving statistics instead of local batch-norm.\n        This will force change batch-norm into a scale shift operator.\n    output_mean_var : boolean, optional, default=0\n        Output the mean and inverse std\n    axis : int, optional, default='1'\n        Specify which shape axis the channel is specified\n    cudnn_off : boolean, optional, default=0\n        Do not select CUDNN operator, if available\n    min_calib_range : float or None, optional, default=None\n        The minimum scalar value in the form of float32 obtained through calibration.\n        If present, it will be used to by quantized batch norm op to calculate primitive scale.\n        Note: this calib_range is to calib bn output.\n    max_calib_range : float or None, optional, default=None\n        The maximum scalar value in the form of float32 obtained through calibration.\n        If present, it will be used to by quantized batch norm op to calculate primitive scale.\n        Note: this calib_range is to calib bn output.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    return _mx_nd_npx.batch_norm(x, gamma, beta, running_mean, running_var, eps=eps,\n                                 momentum=momentum, fix_gamma=fix_gamma,\n                                 use_global_stats=use_global_stats,\n                                 output_mean_var=output_mean_var, axis=axis, cudnn_off=cudnn_off,\n                                 min_calib_range=min_calib_range, max_calib_range=max_calib_range)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.numpy_extension')\ndef fully_connected(x, weight, bias=None, num_hidden=None,\n                    no_bias=True, flatten=True, **kwargs):\n    r\"\"\"Applies a linear transformation: :math:`Y = XW^T + b`.\n\n    If ``flatten`` is set to be true, then the shapes are:\n\n    - **data**: `(batch_size, x1, x2, ..., xn)`\n    - **weight**: `(num_hidden, x1 * x2 * ... * xn)`\n    - **bias**: `(num_hidden,)`\n    - **out**: `(batch_size, num_hidden)`\n\n    If ``flatten`` is set to be false, then the shapes are:\n\n    - **data**: `(x1, x2, ..., xn, input_dim)`\n    - **weight**: `(num_hidden, input_dim)`\n    - **bias**: `(num_hidden,)`\n    - **out**: `(x1, x2, ..., xn, num_hidden)`\n\n    The learnable parameters include both ``weight`` and ``bias``.\n\n    If ``no_bias`` is set to be true, then the ``bias`` term is ignored.\n\n    .. Note::\n\n        The sparse support for FullyConnected is limited to forward evaluation with `row_sparse`\n        weight and bias, where the length of `weight.indices` and `bias.indices` must be equal\n        to `num_hidden`. This could be useful for model inference with `row_sparse` weights\n        trained with importance sampling or noise contrastive estimation.\n\n        To compute linear transformation with 'csr' sparse data, sparse.dot is recommended instead\n        of sparse.FullyConnected.\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data.\n    weight : NDArray\n        Weight matrix.\n    bias : NDArray\n        Bias parameter.\n    num_hidden : int, required\n        Number of hidden nodes of the output.\n    no_bias : boolean, optional, default=0\n        Whether to disable bias parameter.\n    flatten : boolean, optional, default=1\n        Whether to collapse all but the first axis of the input data tensor.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    return _mx_nd_npx.fully_connected(x, weight, bias, num_hidden=num_hidden,\n                                      no_bias=no_bias, flatten=flatten)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.numpy_extension')\ndef pick(data, index, axis=-1, mode='clip', keepdims=False):\n    r\"\"\"Picks elements from an input array according to the input indices along the given axis.\n\n    Given an input array of shape ``(d0, d1)`` and indices of shape ``(i0,)``, the result will be\n    an output array of shape ``(i0,)`` with::\n\n      output[i] = input[i, indices[i]]\n\n    By default, if any index mentioned is too large, it is replaced by the index that addresses\n    the last element along an axis (the `clip` mode).\n\n    This function supports n-dimensional input and (n-1)-dimensional indices arrays.\n\n    Parameters\n    ----------\n    data : NDArray\n        The input array\n    index : NDArray\n        The index array\n    axis : int or None, optional, default='-1'\n        int or None. The axis to picking the elements.\n        Negative values means indexing from right to left.\n        If is `None`, the elements in the index w.r.t the flattened input will be picked.\n    keepdims : boolean, optional, default=0\n        If true, the axis where we pick the elements is\n        left in the result as dimension with size one.\n    mode : {'clip', 'wrap'},optional, default='clip'\n        Specify how out-of-bound indices behave. Default is \"clip\".\n        \"clip\" means clip to the range. So, if all indices mentioned are too large,\n        they are replaced by the index that addresses the last element along an axis.\n        \"wrap\" means to wrap around.\n\n    out : NDArray, optional\n        The output NDArray to hold the result.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Example\n    -------\n    >>> x = np.array([[1., 2.],[3., 4.],[5., 6.]])\n\n    picks elements with specified indices along axis 0\n\n    >>> npx.pick(x, np.array([0, 1]), 0)\n    array([1., 4.])\n\n    picks elements with specified indices along axis 1\n\n    >>> npx.pick(x, np.array([0, 1, 0]), 1)\n    array([1., 4., 5.])\n\n    picks elements with specified indices along axis 1 using 'wrap' mode\n    to place indicies that would normally be out of bounds\n\n    >>> npx.pick(x, np.array([2, -1, -2]), 1, mode='wrap')\n    array([1., 4., 5.])\n\n    picks elements with specified indices along axis 1 and dims are maintained\n\n    >>> npx.pick(x, np.array([[1.], [0.], [2.]]), 1, keepdims=True)\n    array([[2.],\n           [3.],\n           [6.]])\n    \"\"\"\n    return _mx_nd_npx.pick(data, index, axis, mode, keepdims)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.numpy_extension')\ndef convolution(data=None, weight=None, bias=None, kernel=None, stride=None, dilate=None,\n                pad=None, num_filter=1, num_group=1, workspace=1024, no_bias=False,\n                cudnn_tune=None, cudnn_off=False, layout=None):\n    r\"\"\"Compute *N*-D convolution on *(N+2)*-D input.\n\n    In the 2-D convolution, given input data with shape *(batch_size,\n    channel, height, width)*, the output is computed by\n\n    .. math::\n\n       out[n,i,:,:] = bias[i] + \\sum_{j=0}^{channel} data[n,j,:,:] \\star\n       weight[i,j,:,:]\n\n    where :math:`\\star` is the 2-D cross-correlation operator.\n\n    For general 2-D convolution, the shapes are\n\n    - **data**: *(batch_size, channel, height, width)*\n    - **weight**: *(num_filter, channel, kernel[0], kernel[1])*\n    - **bias**: *(num_filter,)*\n    - **out**: *(batch_size, num_filter, out_height, out_width)*.\n\n    Define::\n\n      f(x,k,p,s,d) = floor((x+2*p-d*(k-1)-1)/s)+1\n\n    then we have::\n\n      out_height=f(height, kernel[0], pad[0], stride[0], dilate[0])\n      out_width=f(width, kernel[1], pad[1], stride[1], dilate[1])\n\n    If ``no_bias`` is set to be true, then the ``bias`` term is ignored.\n\n    The default data ``layout`` is *NCHW*, namely *(batch_size, channel, height,\n    width)*. We can choose other layouts such as *NWC*.\n\n    If ``num_group`` is larger than 1, denoted by *g*, then split the input ``data``\n    evenly into *g* parts along the channel axis, and also evenly split ``weight``\n    along the first dimension. Next compute the convolution on the *i*-th part of\n    the data with the *i*-th weight part. The output is obtained by concatenating all\n    the *g* results.\n\n    1-D convolution does not have *height* dimension but only *width* in space.\n\n    - **data**: *(batch_size, channel, width)*\n    - **weight**: *(num_filter, channel, kernel[0])*\n    - **bias**: *(num_filter,)*\n    - **out**: *(batch_size, num_filter, out_width)*.\n\n    3-D convolution adds an additional *depth* dimension besides *height* and\n    *width*. The shapes are\n\n    - **data**: *(batch_size, channel, depth, height, width)*\n    - **weight**: *(num_filter, channel, kernel[0], kernel[1], kernel[2])*\n    - **bias**: *(num_filter,)*\n    - **out**: *(batch_size, num_filter, out_depth, out_height, out_width)*.\n\n    Both ``weight`` and ``bias`` are learnable parameters.\n\n    There are other options to tune the performance.\n\n    - **cudnn_tune**: enable this option leads to higher startup time but may give\n      faster speed. Options are\n\n      - **off**: no tuning\n      - **limited_workspace**:run test and pick the fastest algorithm that doesn't\n        exceed workspace limit.\n      - **fastest**: pick the fastest algorithm and ignore workspace limit.\n      - **None** (default): the behavior is determined by environment variable\n        ``MXNET_CUDNN_AUTOTUNE_DEFAULT``. 0 for off, 1 for limited workspace\n        (default), 2 for fastest.\n\n    - **workspace**: A large number leads to more (GPU) memory usage but may improve\n      the performance.\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data to the ConvolutionOp.\n    weight : NDArray\n        Weight matrix.\n    bias : NDArray\n        Bias parameter.\n    kernel : Shape(tuple), required\n        Convolution kernel size: (w,), (h, w) or (d, h, w)\n    stride : Shape(tuple), optional, default=[]\n        Convolution stride: (w,), (h, w) or (d, h, w). Defaults to 1 for each dimension.\n    dilate : Shape(tuple), optional, default=[]\n        Convolution dilate: (w,), (h, w) or (d, h, w). Defaults to 1 for each dimension.\n    pad : Shape(tuple), optional, default=[]\n        Zero pad for convolution: (w,), (h, w) or (d, h, w). Defaults to no padding.\n    num_filter : int (non-negative), required\n        Convolution filter(channel) number\n    num_group : int (non-negative), optional, default=1\n        Number of group partitions.\n    workspace : long (non-negative), optional, default=1024\n        Maximum temporary workspace allowed (MB) in convolution.This parameter has two usages.\n        When CUDNN is not used, it determines the effective batch size of the convolution kernel.\n        When CUDNN is used, it controls the maximum temporary storage used for tuning the best\n        CUDNN kernel when `limited_workspace` strategy is used.\n    no_bias : boolean, optional, default=0\n        Whether to disable bias parameter.\n    cudnn_tune : {None, 'fastest', 'limited_workspace', 'off'},optional, default='None'\n        Whether to pick convolution algo by running performance test.\n    cudnn_off : boolean, optional, default=0\n        Turn off cudnn for this layer.\n    layout : {None, 'NCDHW', 'NCHW', 'NCW', 'NDHWC', 'NHWC'},optional, default='None'\n        Set layout for input, output and weight. Empty for\n        default layout: NCW for 1d, NCHW for 2d and NCDHW for 3d.\n        NHWC and NDHWC are only supported on GPU.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    return _mx_nd_npx.convolution(data=data, weight=weight, bias=bias, kernel=kernel,\n                                  stride=stride, dilate=dilate, pad=pad, num_filter=num_filter,\n                                  num_group=num_group, workspace=workspace, no_bias=no_bias,\n                                  cudnn_tune=cudnn_tune, cudnn_off=cudnn_off, layout=layout)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.numpy_extension')\ndef deconvolution(data=None, weight=None, bias=None, kernel=None, stride=None, dilate=None,\n                  pad=None, adj=None, target_shape=None, num_filter=1, num_group=1,\n                  workspace=1024, no_bias=False, cudnn_tune=None,\n                  cudnn_off=False, layout=None):\n    r\"\"\"Computes 1D, 2D or 3D transposed convolution (aka fractionally strided convolution) of\n    the input tensor. This operation can be seen as the gradient of Convolution operation\n    with respect to its input. Convolution usually reduces the size of the input.\n    Transposed convolution works the other way, going from a smaller input\n    to a larger output while preserving the connectivity pattern.\n\n    Parameters\n    ----------\n    data : NDArray\n        Input tensor to the deconvolution operation.\n    weight : NDArray\n        Weights representing the kernel.\n    bias : NDArray\n        Bias added to the result after the deconvolution operation.\n    kernel : Shape(tuple), required\n        Deconvolution kernel size: (w,), (h, w) or (d, h, w).\n        This is same as the kernel size used for the corresponding convolution\n    stride : Shape(tuple), optional, default=[]\n        The stride used for the corresponding convolution: (w,), (h, w) or (d, h, w).\n        Defaults to 1 for each dimension.\n    dilate : Shape(tuple), optional, default=[]\n        Dilation factor for each dimension of the input: (w,), (h, w) or (d, h, w).\n        Defaults to 1 for each dimension.\n    pad : Shape(tuple), optional, default=[]\n        The amount of implicit zero padding added during convolution for each dimension of\n        the input: (w,), (h, w) or (d, h, w). ``(kernel-1)/2`` is usually a good choice.\n        If `target_shape` is set, `pad` will be ignored and a padding that will generate\n        the target shape will be used. Defaults to no padding.\n    adj : Shape(tuple), optional, default=[]\n        Adjustment for output shape: (w,), (h, w) or (d, h, w).\n        If `target_shape` is set, `adj` will be ignored and computed accordingly.\n    target_shape : Shape(tuple), optional, default=[]\n        Shape of the output tensor: (w,), (h, w) or (d, h, w).\n    num_filter : int (non-negative), required\n        Number of output filters.\n    num_group : int (non-negative), optional, default=1\n        Number of groups partition.\n    workspace : long (non-negative), optional, default=512\n        Maximum temporary workspace allowed (MB) in deconvolution. This parameter has two usages.\n        When CUDNN is not used, it determines the effective batch size of the deconvolution kernel.\n        When CUDNN is used, it controls the maximum temporary storage used for tuning\n        the best CUDNN kernel when `limited_workspace` strategy is used.\n    no_bias : boolean, optional, default=1\n        Whether to disable bias parameter.\n    cudnn_tune : {None, 'fastest', 'limited_workspace', 'off'},optional, default='None'\n        Whether to pick convolution algorithm by running performance test.\n    cudnn_off : boolean, optional, default=0\n        Turn off cudnn for this layer.\n    layout : {None, 'NCDHW', 'NCHW', 'NCW', 'NDHWC', 'NHWC'},optional, default='None'\n        Set layout for input, output and weight. Empty for\n        default layout, NCW for 1d, NCHW for 2d and NCDHW for 3d.\n        NHWC and NDHWC are only supported on GPU.\n\n    out : NDArray, optional\n        The output NDArray to hold the result.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    return _mx_nd_npx.deconvolution(data=data, weight=weight, bias=bias, kernel=kernel,\n                                    stride=stride, dilate=dilate, pad=pad, adj=adj,\n                                    target_shape=target_shape, num_filter=num_filter,\n                                    num_group=num_group, workspace=workspace, no_bias=no_bias,\n                                    cudnn_tune=cudnn_tune, cudnn_off=cudnn_off, layout=layout)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.numpy_extension')\ndef pooling(data=None, kernel=None, stride=None, pad=None, pool_type=\"max\",\n            pooling_convention=\"valid\", global_pool=False, cudnn_off=False,\n            p_value=None, count_include_pad=None, layout=None, **kwargs):\n    r\"\"\"Performs pooling on the input.\n\n    The shapes for 1-D pooling are\n\n    - **data** and **out**: *(batch_size, channel, width)* (NCW layout) or\n      *(batch_size, width, channel)* (NWC layout),\n\n    The shapes for 2-D pooling are\n\n    - **data** and **out**: *(batch_size, channel, height, width)* (NCHW layout) or\n      *(batch_size, height, width, channel)* (NHWC layout),\n\n        out_height = f(height, kernel[0], pad[0], stride[0])\n        out_width = f(width, kernel[1], pad[1], stride[1])\n\n    The definition of *f* depends on ``pooling_convention``, which has two options:\n\n    - **valid** (default)::\n\n        f(x, k, p, s) = floor((x+2*p-k)/s)+1\n\n    - **full**, which is compatible with Caffe::\n\n        f(x, k, p, s) = ceil((x+2*p-k)/s)+1\n\n    When ``global_pool`` is set to be true, then global pooling is performed. It will reset\n    ``kernel=(height, width)`` and set the appropiate padding to 0.\n\n    Three pooling options are supported by ``pool_type``:\n\n    - **avg**: average pooling\n    - **max**: max pooling\n    - **sum**: sum pooling\n    - **lp**: Lp pooling\n\n    For 3-D pooling, an additional *depth* dimension is added before\n    *height*. Namely the input data and output will have shape *(batch_size, channel, depth,\n    height, width)* (NCDHW layout) or *(batch_size, depth, height, width, channel)* (NDHWC layout).\n\n    Notes on Lp pooling:\n\n    Lp pooling was first introduced by this paper: https://arxiv.org/pdf/1204.3968.pdf.\n    L-1 pooling is simply sum pooling, while L-inf pooling is simply max pooling.\n    We can see that Lp pooling stands between those two, in practice the most common value for p is 2.\n\n    For each window ``X``, the mathematical expression for Lp pooling is:\n\n    :math:`f(X) = \\sqrt[p]{\\sum_{x}^{X} x^p}`\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data to the pooling operator.\n    kernel : Shape(tuple), optional, default=[]\n        Pooling kernel size: (y, x) or (d, y, x)\n    pool_type : {'avg', 'lp', 'max', 'sum'},optional, default='max'\n        Pooling type to be applied.\n    global_pool : boolean, optional, default=0\n        Ignore kernel size, do global pooling based on current input feature map.\n    cudnn_off : boolean, optional, default=0\n        Turn off cudnn pooling and use MXNet pooling operator.\n    pooling_convention : {'full', 'same', 'valid'},optional, default='valid'\n        Pooling convention to be applied.\n    stride : Shape(tuple), optional, default=[]\n        Stride: for pooling (y, x) or (d, y, x). Defaults to 1 for each dimension.\n    pad : Shape(tuple), optional, default=[]\n        Pad for pooling: (y, x) or (d, y, x). Defaults to no padding.\n    p_value : int or None, optional, default='None'\n        Value of p for Lp pooling, can be 1 or 2, required for Lp Pooling.\n    count_include_pad : boolean or None, optional, default=None\n        Only used for AvgPool, specify whether to count padding elements for averagecalculation.\n        For example, with a 5*5 kernel on a 3*3 corner of a image,the sum of the 9 valid elements will\n        be divided by 25 if this is set to true,or it will be divided by 9 if this is set to false.\n        Defaults to true.\n    layout : {None, 'NCDHW', 'NCHW', 'NCW', 'NDHWC', 'NHWC', 'NWC'},optional, default='None'\n        Set layout for input and output. Empty for\n        default layout: NCW for 1d, NCHW for 2d and NCDHW for 3d.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    return _mx_nd_npx.pooling(data=data, kernel=kernel, stride=stride, pad=pad,\n                              pool_type=pool_type, pooling_convention=pooling_convention,\n                              global_pool=global_pool, cudnn_off=cudnn_off, p_value=p_value,\n                              count_include_pad=count_include_pad, layout=layout)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.numpy_extension')\ndef dropout(data, p=0.5, mode=\"training\", axes=None, cudnn_off=False, **kwargs):\n    r\"\"\"Applies dropout operation to input array.\n\n    - During training, each element of the input is set to zero with probability p.\n      The whole array is rescaled by :math:`1/(1-p)` to keep the expected\n      sum of the input unchanged.\n\n    - During testing, this operator does not change the input if mode is 'training'.\n      If mode is 'always', the same computaion as during training will be applied.\n\n    Parameters\n    ----------\n    data : NDArray\n        Input array to which dropout will be applied.\n    p : float, optional, default=0.5\n        Fraction of the input that gets dropped out during training time.\n    mode : {'always', 'training'},optional, default='training'\n        Whether to only turn on dropout during training or to also turn on for inference.\n    axes : Shape(tuple), optional, default=[]\n        Axes for variational dropout kernel.\n    cudnn_off : boolean or None, optional, default=0\n        Whether to turn off cudnn in dropout operator. This option is ignored if axes is specified.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    return _mx_nd_npx.dropout(data=data, p=p, mode=mode, axes=axes, cudnn_off=cudnn_off)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.numpy_extension')\ndef one_hot(data, depth=None, on_value=1.0, off_value=0.0, dtype=\"float32\"):\n    r\"\"\"Returns a one-hot array.\n\n    The locations represented by `indices` take value `on_value`, while all\n    other locations take value `off_value`.\n\n    `one_hot` operation with `indices` of shape ``(i0, i1)`` and `depth`  of ``d`` would result\n    in an output array of shape ``(i0, i1, d)`` with::\n\n      output[i,j,:] = off_value\n      output[i,j,indices[i,j]] = on_value\n\n    Parameters\n    ----------\n    indices : NDArray\n        array of locations where to set on_value\n    depth : long, required\n        Depth of the one hot dimension.\n    on_value : double, optional, default=1\n        The value assigned to the locations represented by indices.\n    off_value : double, optional, default=0\n        The value assigned to the locations not represented by indices.\n    dtype : {'bfloat16', 'float16', 'float32', 'float64', 'int32', 'int64', 'int8', 'uint8'},\n            optional, default='float32'\n        DType of the output\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Example\n    -------\n    >>> data = np.array([1,0,2,0])\n    >>> npx.one_hot(data, 3)\n    array([[0., 1., 0.],\n           [1., 0., 0.],\n           [0., 0., 1.],\n           [1., 0., 0.]], dtype=float64)\n    >>> npx.one_hot(data, 3, on_value=8, off_value=1, dtype='int32')\n    array([[1, 8, 1],\n           [8, 1, 1],\n           [1, 1, 8],\n           [8, 1, 1]], dtype=int32)\n    >>> data = np.array([[1,0],[1,0],[2,0]])\n    >>> npx.one_hot(data, 3)\n    array([[[0., 1., 0.],\n            [1., 0., 0.]],\n           [[0., 1., 0.],\n            [1., 0., 0.]],\n           [[0., 0., 1.],\n            [1., 0., 0.]]], dtype=float64)\n    \"\"\"\n    return _mx_nd_npx.one_hot(data=data, depth=depth, on_value=on_value, off_value=off_value,\n                              dtype=dtype)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.numpy_extension')\ndef rnn(data=None, parameters=None, state=None, state_cell=None, sequence_length=None,\n        mode=None, state_size=None, num_layers=None, bidirectional=False,\n        state_outputs=False, p=0.0, use_sequence_length=False, projection_size=None,\n        lstm_state_clip_min=None, lstm_state_clip_max=None, lstm_state_clip_nan=None):\n    r\"\"\"Applies recurrent layers to input data. Currently, vanilla RNN, LSTM and GRU are\n    implemented, with both multi-layer and bidirectional support.\n\n    When the input data is of type float32 and the environment variables MXNET_CUDA_ALLOW_TENSOR_CORE\n    and MXNET_CUDA_TENSOR_OP_MATH_ALLOW_CONVERSION are set to 1, this operator will try to use\n    pseudo-float16 precision (float32 math with float16 I/O) precision in order to use\n    Tensor Cores on suitable NVIDIA GPUs. This can sometimes give significant speedups.\n\n    **Vanilla RNN**\n\n    Applies a single-gate recurrent layer to input X. Two kinds of activation function are supported:\n    ReLU and Tanh.\n\n    With ReLU activation function:\n\n    .. math::\n        h_t = relu(W_{ih} * x_t + b_{ih}  +  W_{hh} * h_{(t-1)} + b_{hh})\n\n    With Tanh activtion function:\n\n    .. math::\n        h_t = \\tanh(W_{ih} * x_t + b_{ih}  +  W_{hh} * h_{(t-1)} + b_{hh})\n\n    Reference paper: Finding structure in time - Elman, 1988.\n    https://axon.cs.byu.edu/~martinez/classes/678/Papers/Elman_time.pdf\n\n    **LSTM**\n\n    Long Short-Term Memory - Hochreiter, 1997. http://www.bioinf.jku.at/publications/older/2604.pdf\n\n    .. math::\n      \\begin{array}{ll}\n                i_t = \\mathrm{sigmoid}(W_{ii} x_t + b_{ii} + W_{hi} h_{(t-1)} + b_{hi}) \\\\\n                f_t = \\mathrm{sigmoid}(W_{if} x_t + b_{if} + W_{hf} h_{(t-1)} + b_{hf}) \\\\\n                g_t = \\tanh(W_{ig} x_t + b_{ig} + W_{hc} h_{(t-1)} + b_{hg}) \\\\\n                o_t = \\mathrm{sigmoid}(W_{io} x_t + b_{io} + W_{ho} h_{(t-1)} + b_{ho}) \\\\\n                c_t = f_t * c_{(t-1)} + i_t * g_t \\\\\n                h_t = o_t * \\tanh(c_t)\n                \\end{array}\n\n    With the projection size being set, LSTM could use the projection feature to reduce the parameters\n    size and give some speedups without significant damage to the accuracy.\n\n    Long Short-Term Memory Based Recurrent Neural Network Architectures for Large Vocabulary Speech\n    Recognition - Sak et al. 2014. https://arxiv.org/abs/1402.1128\n\n    .. math::\n      \\begin{array}{ll}\n                i_t = \\mathrm{sigmoid}(W_{ii} x_t + b_{ii} + W_{ri} r_{(t-1)} + b_{ri}) \\\\\n                f_t = \\mathrm{sigmoid}(W_{if} x_t + b_{if} + W_{rf} r_{(t-1)} + b_{rf}) \\\\\n                g_t = \\tanh(W_{ig} x_t + b_{ig} + W_{rc} r_{(t-1)} + b_{rg}) \\\\\n                o_t = \\mathrm{sigmoid}(W_{io} x_t + b_{o} + W_{ro} r_{(t-1)} + b_{ro}) \\\\\n                c_t = f_t * c_{(t-1)} + i_t * g_t \\\\\n                h_t = o_t * \\tanh(c_t)\n                r_t = W_{hr} h_t\n                \\end{array}\n\n    **GRU**\n\n    Gated Recurrent Unit - Cho et al. 2014. http://arxiv.org/abs/1406.1078\n\n    The definition of GRU here is slightly different from paper but compatible with CUDNN.\n\n    .. math::\n      \\begin{array}{ll}\n                r_t = \\mathrm{sigmoid}(W_{ir} x_t + b_{ir} + W_{hr} h_{(t-1)} + b_{hr}) \\\\\n                z_t = \\mathrm{sigmoid}(W_{iz} x_t + b_{iz} + W_{hz} h_{(t-1)} + b_{hz}) \\\\\n                n_t = \\tanh(W_{in} x_t + b_{in} + r_t * (W_{hn} h_{(t-1)}+ b_{hn})) \\\\\n                h_t = (1 - z_t) * n_t + z_t * h_{(t-1)} \\\\\n                \\end{array}\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data to RNN\n    parameters : NDArray\n        Vector of all RNN trainable parameters concatenated\n    state : NDArray\n        initial hidden state of the RNN\n    state_cell : NDArray\n        initial cell state for LSTM networks (only for LSTM)\n    sequence_length : NDArray\n        Vector of valid sequence lengths for each element in batch.\n        (Only used if use_sequence_length kwarg is True)\n    state_size : int (non-negative), required\n        size of the state for each layer\n    num_layers : int (non-negative), required\n        number of stacked layers\n    bidirectional : boolean, optional, default=0\n        whether to use bidirectional recurrent layers\n    mode : {'gru', 'lstm', 'rnn_relu', 'rnn_tanh'}, required\n        the type of RNN to compute\n    p : float, optional, default=0\n        drop rate of the dropout on the outputs of each RNN layer, except the last layer.\n    state_outputs : boolean, optional, default=0\n        Whether to have the states as symbol outputs.\n    projection_size : int or None, optional, default='None'\n        size of project size\n    lstm_state_clip_min : double or None, optional, default=None\n        Minimum clip value of LSTM states. This option must be used together with lstm_state_clip_max.\n    lstm_state_clip_max : double or None, optional, default=None\n        Maximum clip value of LSTM states. This option must be used together with lstm_state_clip_min.\n    lstm_state_clip_nan : boolean, optional, default=0\n        Whether to stop NaN from propagating in state by clipping it to min/max.\n        If clipping range is not specified, this option is ignored.\n    use_sequence_length : boolean, optional, default=0\n        If set to true, this layer takes in an extra input parameter `sequence_length`\n        to specify variable length sequence\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    return _mx_nd_npx.rnn(data=data, parameters=parameters, state=state, state_cell=state_cell,\n                          sequence_length=sequence_length, mode=mode, state_size=state_size,\n                          num_layers=num_layers, bidirectional=bidirectional,\n                          state_outputs=state_outputs, p=p, use_sequence_length=use_sequence_length,\n                          projection_size=projection_size, lstm_state_clip_min=lstm_state_clip_min,\n                          lstm_state_clip_max=lstm_state_clip_max,\n                          lstm_state_clip_nan=lstm_state_clip_nan)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.numpy_extension')\ndef embedding(data, weight, input_dim=None, output_dim=None, dtype=\"float32\", sparse_grad=False,\n              **kwargs):\n    r\"\"\"Maps integer indices to vector representations (embeddings).\n\n    This operator maps words to real-valued vectors in a high-dimensional space,\n    called word embeddings. These embeddings can capture semantic and syntactic properties of the words.\n    For example, it has been noted that in the learned embedding spaces, similar words tend\n    to be close to each other and dissimilar words far apart.\n\n    For an input array of shape (d1, ..., dK),\n    the shape of an output array is (d1, ..., dK, output_dim).\n    All the input values should be integers in the range [0, input_dim).\n\n    If the input_dim is ip0 and output_dim is op0, then shape of the embedding weight matrix must be\n    (ip0, op0).\n\n    When \"sparse_grad\" is False, if any index mentioned is too large, it is replaced by the index that\n    addresses the last vector in an embedding matrix.\n    When \"sparse_grad\" is True, an error will be raised if invalid indices are found.\n\n    The storage type of weight can be either row_sparse or default.\n\n    .. Note::\n\n        If \"sparse_grad\" is set to True, the storage type of gradient w.r.t weights will be\n        \"row_sparse\". Only a subset of optimizers support sparse gradients, including SGD, AdaGrad\n        and Adam. Note that by default lazy updates is turned on, which may perform differently\n        from standard updates. For more details, please check the Optimization API at:\n        https://mxnet.apache.org/versions/master/api/python/docs/api/optimizer/index.html\n\n    Parameters\n    ----------\n    data : NDArray\n        The input array to the embedding operator.\n    weight : NDArray\n        The embedding weight matrix.\n    input_dim : long, required\n        Vocabulary size of the input indices.\n    output_dim : long, required\n        Dimension of the embedding vectors.\n    dtype : {'bfloat16', 'float16', 'float32', 'float64', 'int32', 'int64', 'int8', 'uint8'},\n            optional, default='float32'\n        Data type of weight.\n    sparse_grad : boolean, optional, default=0\n        Compute row sparse gradient in the backward calculation.\n        If set to True, the grad's storage type is row_sparse.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Example\n    -------\n    >>> input_dim = 4\n    >>> output_dim = 5\n\n    Each row in weight matrix y represents a word. So, y = (w0,w1,w2,w3)\n\n    >>> y = np.arange(input_dim * output_dim).reshape(input_dim, output_dim)\n    >>> y\n    array([[ 0.,  1.,  2.,  3.,  4.],\n           [ 5.,  6.,  7.,  8.,  9.],\n           [10., 11., 12., 13., 14.],\n           [15., 16., 17., 18., 19.]])\n\n    Input array x represents n-grams(2-gram). So, x = [(w1,w3), (w0,w2)]\n\n    >>> x = np.array([[1., 3.], [0., 2.]])\n    >>> x\n    array([[1., 3.],\n           [0., 2.]])\n\n    Mapped input x to its vector representation y.\n\n    >>> npx.embedding(x, y, input_dim, output_dim)\n    array([[[ 5.,  6.,  7.,  8.,  9.],\n            [15., 16., 17., 18., 19.]],\n\n           [[ 0.,  1.,  2.,  3.,  4.],\n            [10., 11., 12., 13., 14.]]])\n    \"\"\"\n    return _mx_nd_npx.embedding(data=data, weight=weight, input_dim=input_dim, output_dim=output_dim,\n                                dtype=dtype, sparse_grad=sparse_grad)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.numpy_extension')\ndef topk(data, axis=-1, k=1, ret_typ=\"indices\", is_ascend=False, dtype=\"float32\"):\n    r\"\"\"Returns the indices of the top *k* elements in an input array along the given\n     axis (by default).\n     If ret_type is set to 'value' returns the value of top *k* elements (instead of indices).\n     In case of ret_type = 'both', both value and index would be returned.\n     The returned elements will be sorted.\n\n    Parameters\n    ----------\n    data : NDArray\n        The input array\n    axis : int or None, optional, default='-1'\n        Axis along which to choose the top k indices.\n        If not given, the flattened array is used. Default is -1.\n    k : int, optional, default='1'\n        Number of top elements to select, should be always smaller than or equal to\n        the element number in the given axis. A global sort is performed if set k < 1.\n    ret_typ : {'both', 'indices', 'mask', 'value'},optional, default='indices'\n        The return type.\n     \"value\" means to return the top k values,\n     \"indices\" means to return the indices of the top k values,\n     \"mask\" means to return a mask array containing 0 and 1. 1 means the top k values.\n     \"both\" means to return a list of both values and indices of top k elements.\n    is_ascend : boolean, optional, default=0\n        Whether to choose k largest or k smallest elements.\n        Top K largest elements will be chosen if set to false.\n    dtype : {'float16', 'float32', 'float64', 'int32', 'int64', 'uint8'},\n            optional, default='float32'\n        DType of the output indices when ret_typ is \"indices\" or \"both\".\n        An error will be raised if the selected data type cannot precisely represent the indices.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Example\n    -------\n    >>> x = np.array([[0.3, 0.2, 0.4], [0.1, 0.3, 0.2]])\n\n    returns an index of the largest element on last axis\n\n    >>> npx.topk(x)\n    array([[2.],\n           [1.]])\n\n    returns the value of top-2 largest elements on last axis\n\n    >>> npx.topk(x, ret_typ='value', k=2)\n    array([[0.4, 0.3],\n           [0.3, 0.2]])\n\n    returns the value of top-2 smallest elements on last axis\n\n    >>> npx.topk(x, ret_typ='value', k=2, is_ascend=1)\n    array([[0.2, 0.3],\n           [0.1, 0.2]])\n\n    returns the value of top-2 largest elements on axis 0\n\n    >>> npx.topk(x, axis=0, ret_typ='value', k=2)\n    array([[0.3, 0.3, 0.4],\n           [0.1, 0.2, 0.2]])\n\n    flattens and then returns list of both values and indices\n\n    >>> npx.topk(x, ret_typ='both', k=2)\n    [array([[0.4, 0.3], [0.3, 0.2]]),\n     array([[2., 0.], [1., 2.]])]\n    \"\"\"\n    return _mx_nd_npx.topk(data=data, axis=axis, k=k, ret_typ=ret_typ, is_ascend=is_ascend, dtype=dtype)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.numpy_extension')\ndef layer_norm(data=None, gamma=None, beta=None, axis=None, eps=None, output_mean_var=None):\n    r\"\"\"Layer normalization.\n\n    Normalizes the channels of the input tensor by mean and variance, and applies a scale ``gamma`` as\n    well as offset ``beta``.\n\n    Assume the input has more than one dimension and we normalize along axis 1.\n    We first compute the mean and variance along this axis and then\n    compute the normalized output, which has the same shape as input, as following:\n\n    .. math::\n\n      out = \\frac{data - mean(data, axis)}{\\sqrt{var(data, axis) + \\epsilon}} * gamma + beta\n\n    Both ``gamma`` and ``beta`` are learnable parameters.\n\n    Unlike BatchNorm and InstanceNorm,  the *mean* and *var* are computed along the channel dimension.\n\n    Assume the input has size *k* on axis 1, then both ``gamma`` and ``beta``\n    have shape *(k,)*. If ``output_mean_var`` is set to be true, then outputs both ``data_mean`` and\n    ``data_std``. Note that no gradient will be passed through these two outputs.\n\n    The parameter ``axis`` specifies which axis of the input shape denotes\n    the 'channel' (separately normalized groups).  The default is -1, which sets the channel\n    axis to be the last item in the input shape.\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data to layer normalization\n    gamma : NDArray\n        gamma array\n    beta : NDArray\n        beta array\n    axis : int, optional, default='-1'\n        The axis to perform layer normalization.\n        Usually, this should be be axis of the channel dimension.\n        Negative values means indexing from right to left.\n    eps : float, optional, default=9.99999975e-06\n        An `epsilon` parameter to prevent division by 0.\n    output_mean_var : boolean, optional, default=0\n        Output the mean and std calculated along the given axis.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    return _mx_nd_npx.layer_norm(data=data, gamma=gamma, beta=beta, axis=axis, eps=eps,\n                                 output_mean_var=output_mean_var)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.numpy_extension')\ndef leaky_relu(data=None, gamma=None, act_type=\"leaky\", slope=0.25, lower_bound=0.125,\n               upper_bound=0.334, **kwargs):\n    r\"\"\"Applies Leaky rectified linear unit activation element-wise to the input.\n\n    Leaky ReLUs attempt to fix the \"dying ReLU\" problem by allowing a small `slope`\n    when the input is negative and has a slope of one when input is positive.\n\n    The following modified ReLU Activation functions are supported:\n\n    - *elu*: Exponential Linear Unit. `y = x > 0 ? x : slope * (exp(x)-1)`\n    - *gelu*: Gaussian Error Linear Unit. `y = 0.5 * x * (1 + erf(x / sqrt(2)))`\n    - *selu*: Scaled Exponential Linear Unit. `y = lambda * (x > 0 ? x : alpha * (exp(x) - 1))` where\n      *lambda = 1.0507009873554804934193349852946* and *alpha = 1.6732632423543772848170429916717*.\n    - *leaky*: Leaky ReLU. `y = x > 0 ? x : slope * x`\n    - *prelu*: Parametric ReLU. This is same as *leaky* except that `slope` is learnt during training.\n    - *rrelu*: Randomized ReLU. same as *leaky* but the `slope` is uniformly and randomly chosen from\n      *[lower_bound, upper_bound)* for training, while fixed to be\n      *(lower_bound+upper_bound)/2* for inference.\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data to activation function.\n    gamma : NDArray\n        Input data to activation function.\n    act_type : {'elu', 'gelu', 'leaky', 'prelu', 'rrelu', 'selu'},optional, default='leaky'\n        Activation function to be applied.\n    slope : float, optional, default=0.25\n        Init slope for the activation. (For leaky and elu only)\n    lower_bound : float, optional, default=0.125\n        Lower bound of random slope. (For rrelu only)\n    upper_bound : float, optional, default=0.333999991\n        Upper bound of random slope. (For rrelu only)\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    return _mx_nd_npx.leaky_relu(data=data, gamma=gamma, act_type=act_type, slope=slope,\n                                 lower_bound=lower_bound, upper_bound=upper_bound)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.numpy_extension')\ndef batch_dot(a, b, transpose_a=False, transpose_b=False, forward_stype=\"default\"):\n    r\"\"\"Batchwise dot product.\n\n    ``batch_dot`` is used to compute dot product of ``x`` and ``y`` when ``x`` and\n    ``y`` are data in batch, namely N-D (N >= 3) arrays in shape of `(B0, ..., B_i, :, :)`.\n\n    For example, given ``x`` with shape `(B_0, ..., B_i, N, M)` and ``y`` with shape\n    `(B_0, ..., B_i, M, K)`, the result array will have shape `(B_0, ..., B_i, N, K)`,\n    which is computed by::\n\n       batch_dot(x,y)[b_0, ..., b_i, :, :] = dot(x[b_0, ..., b_i, :, :], y[b_0, ..., b_i, :, :])\n\n    Parameters\n    ----------\n    lhs : NDArray\n        The first input\n    rhs : NDArray\n        The second input\n    transpose_a : boolean, optional, default=0\n        If true then transpose the first input before dot.\n    transpose_b : boolean, optional, default=0\n        If true then transpose the second input before dot.\n    forward_stype : {None, 'csr', 'default', 'row_sparse'},optional, default='None'\n        The desired storage type of the forward output given by user,\n        if thecombination of input storage types and this hint does not matchany implemented ones,\n        the dot operator will perform fallback operationand still produce\n        an output of the desired storage type.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    return _mx_nd_npx.batch_dot(a=a, b=b, transpose_a=transpose_a,\n                                transpose_b=transpose_b, forward_stype=forward_stype)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.numpy_extension')\ndef broadcast_like(lhs, rhs, lhs_axes=None, rhs_axes=None):\n    r\"\"\"Broadcasts lhs to have the same shape as rhs.\n\n    Broadcasting is a mechanism that allows NDArrays to perform arithmetic operations\n    with arrays of different shapes efficiently without creating multiple copies of arrays.\n    Also see, `Broadcasting <https://docs.scipy.org/doc/numpy/user/basics.broadcasting.html>`_\n    for more explanation.\n\n    Broadcasting is allowed on axes with size 1, such as from `(2,1,3,1)` to\n    `(2,8,3,9)`. Elements will be duplicated on the broadcasted axes.\n\n    Parameters\n    ----------\n    lhs : NDArray\n        First input.\n    rhs : NDArray\n        Second input.\n    lhs_axes : Shape or None, optional, default=None\n        Axes to perform broadcast on in the first input array\n    rhs_axes : Shape or None, optional, default=None\n        Axes to copy from the second input array\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    example\n    -------\n    >>> a = np.array([[1,2,3]])\n    >>> b = np.array([[5,6,7],[7,8,9]])\n    >>> npx.broadcast_like(a, b)\n    array([[1., 2., 3.],\n           [1., 2., 3.]])\n    >>> a = np.array([9])\n    >>> b = np.array([1,2,3,4,5])\n    >>> npx.broadcast_like(a, b, lhs_axes=(0,), rhs_axes=(-1,))\n    array([9., 9., 9., 9., 9.])\n    \"\"\"\n    return _mx_nd_npx.broadcast_like(lhs=lhs, rhs=rhs, lhs_axes=lhs_axes, rhs_axes=rhs_axes)\n\n\n# pylint: disable=too-many-arguments, unused-argument\n@set_module('mxnet.numpy_extension')\ndef arange_like(data, start=0.0, step=1.0, repeat=1, ctx=None, axis=None):\n    r\"\"\"Return an array with evenly spaced values. If axis is not given, the output will\n    have the same shape as the input array. Otherwise, the output will be a 1-D array with size of\n    the specified axis in input shape.\n\n    Parameters\n    ----------\n    data : NDArray\n        The input\n    start : double, optional, default=0\n        Start of interval. The interval includes this value. The default start value is 0.\n    step : double, optional, default=1\n        Spacing between values.\n    repeat : int, optional, default='1'\n        The repeating time of all elements.\n        E.g repeat=3, the element a will be repeated three times --> a, a, a.\n    ctx : string, optional, default=''\n        Context of output, in format [cpu|gpu|cpu_pinned](n).Only used for imperative calls.\n    axis : int or None, optional, default='None'\n        Arange elements according to the size of a certain axis of input array.\n        The negative numbers are interpreted counting from the backward.\n        If not provided, will arange elements according to the input shape.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n\n    Example\n    -------\n    >>> x = np.random.uniform(0, 1, size=(3,4))\n    >>> x\n    array([[0.5488135 , 0.5928446 , 0.71518934, 0.84426576],\n           [0.60276335, 0.8579456 , 0.5448832 , 0.8472517 ],\n           [0.4236548 , 0.6235637 , 0.6458941 , 0.3843817 ]])\n    >>> npx.arange_like(x, start=0)\n    array([[ 0.,  1.,  2.,  3.],\n           [ 4.,  5.,  6.,  7.],\n           [ 8.,  9., 10., 11.]])\n    >>> npx.arange_like(x, start=0, axis=-1)\n    array([0., 1., 2., 3.])\n    \"\"\"\n    return _mx_nd_npx.arange_like(data=data, start=start, step=step, repeat=repeat,\n                                  ctx=ctx, axis=axis)\n\n\n# pylint: disable=too-many-arguments\n@set_module('mxnet.numpy_extension')\ndef group_norm(data, gamma, beta, num_groups=1, eps=1e-3, output_mean_var=False):\n    r\"\"\"Group normalization.\n\n    The input channels are separated into ``num_groups`` groups,\n    each containing ``num_channels / num_groups`` channels.\n    The mean and standard-deviation are calculated separately over the each group.\n\n    .. math::\n\n      data = data.reshape((N, num_groups, C // num_groups, ...))\n      out = \\frac{data - mean(data, axis)}{\\sqrt{var(data, axis) + \\epsilon}} * gamma + beta\n\n    Both ``gamma`` and ``beta`` are learnable parameters.\n\n\n\n    Defined in ../src/operator/nn/group_norm.cc:L78\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data\n    gamma : NDArray\n        gamma array\n    beta : NDArray\n        beta array\n    num_groups : int, optional, default='1'\n        Total number of groups.\n    eps : float, optional, default=9.99999975e-06\n        An `epsilon` parameter to prevent division by 0.\n    output_mean_var : boolean, optional, default=0\n        Output the mean and std calculated along the given axis.\n\n    Returns\n    -------\n    out : NDArray or list of NDArrays\n        The output of this function.\n    \"\"\"\n    return _mx_nd_npx.group_norm(data=data, gamma=gamma, beta=beta, num_groups=num_groups,\n                                 eps=eps, output_mean_var=output_mean_var)\n"
  },
  {
    "path": "python/mxnet/numpy_extension/_register.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Registering ops in mxnet.numpy_extension for imperative programming.\"\"\"\n\n\nfrom ..base import _init_np_op_module\nfrom ..ndarray.register import _make_ndarray_function\n\n\n_init_np_op_module(root_module_name='mxnet', np_module_name='numpy_extension',\n                   mx_module_name=None, make_op_func=_make_ndarray_function)\n"
  },
  {
    "path": "python/mxnet/numpy_extension/control_flow.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for registering control flow ops for imperative programming.\"\"\"\n\nfrom ..ndarray import numpy_extension as _mx_nd_npx\nfrom ..util import set_module\n\n\n__all__ = [\"foreach\", \"while_loop\", \"cond\"]\n\n\n@set_module('mxnet.numpy_extension')\ndef foreach(body, data, init_states):\n    \"\"\"Run a for loop with user-defined computation over NDArrays on dimension 0.\n\n    This operator simulates a for loop and body has the computation for an iteration\n    of the for loop. It runs the computation in body on each slice from the input\n    NDArrays.\n\n    body takes two arguments as input and outputs a tuple of two elements,\n    as illustrated below::\n\n        out, states = body(data1, states)\n\n    data1 can be either an NDArray or a list of NDArrays. If data is an NDArray,\n    data1 is an NDArray. Otherwise, data1 is a list of NDArrays and has the same\n    size as data. states is a list of NDArrays and have the same size as init_states.\n    Similarly, out can be either an NDArray or a list of NDArrays, which are concatenated\n    as the first output of foreach; states from the last execution of body\n    are the second output of foreach.\n\n    The computation done by this operator is equivalent to the pseudo code below\n    when the input data is NDArray::\n\n        states = init_states\n        outs = []\n        for i in data.shape[0]:\n            s = data[i]\n            out, states = body(s, states)\n            outs.append(out)\n        outs = stack(*outs)\n\n\n    Parameters\n    ----------\n    body : HybridBlock.\n        Define computation in an iteration.\n    data: an NDArray or a list of NDArrays.\n        The input data.\n    init_states: an NDArray or nested lists of NDArrays.\n        The initial values of the loop states.\n\n    Returns\n    -------\n    outputs: an NDArray or nested lists of NDArrays.\n        The output data concatenated from the output of all iterations.\n    states: an NDArray or nested lists of NDArrays.\n        The loop states in the last iteration.\n\n    Examples\n    --------\n    >>> step = lambda data, states: (data + states[0], [states[0] * 2])\n    >>> data = mx.np.random.uniform(size=(2, 10))\n    >>> states = [mx.np.random.uniform(size=(10))]\n    >>> outs, states = npx.control_flow.foreach(step, data, states)\n    \"\"\"\n    return _mx_nd_npx.foreach(body, data, init_states)\n\n\n#pylint: disable=W0621\n@set_module('mxnet.numpy_extension')\ndef while_loop(cond, func, loop_vars, max_iterations=None):\n    \"\"\"Run a while loop with user-defined computation and loop condition.\n\n    This operator simulates a while loop which iterately does customized computation\n    as long as the condition is satisfied.\n\n    `loop_vars` is a list of NDArrays on which the computation uses.\n\n    `cond` is a user-defined function, used as the loop condition.\n    It consumes `loop_vars`, and produces a scalar MXNet NDArray,\n    indicating the termination of the loop.\n    The loop ends when `cond` returns false (zero).\n    The `cond` is variadic, and its signature should be\n    `cond(*loop_vars) => NDArray`.\n\n    `func` is a user-defined function, used as the loop body.\n    It also consumes `loop_vars`, and produces `step_output` and `new_loop_vars` at each step.\n    In each step, `step_output` should contain the same number elements.\n    Through all steps, the i-th element of `step_output` should have the same shape and dtype.\n    Also, `new_loop_vars` should contain the same number of elements as `loop_vars`,\n    and the corresponding element should have the same shape and dtype.\n    The `func` is variadic, and its signature should be\n    `func(*loop_vars) =>\n    (NDArray or nested List[NDArray] step_output, NDArray or nested List[NDArray] new_loop_vars)`.\n\n    `max_iterations` is a scalar that defines the maximum number of iterations allowed.\n\n    This function returns two lists.\n    The first list has the length of `|step_output|`,\n    in which the i-th element are all i-th elements of\n    `step_output` from all steps, stacked along axis 0.\n    The second list has the length of `|loop_vars|`,\n    which represents final states of loop variables.\n\n    .. warning::\n\n       For now, the axis 0 of all NDArrays in the first list are `max_iterations`,\n       due to lack of dynamic shape inference.\n\n    .. warning::\n\n       When `cond` is never satisfied, we assume `step_output` is empty,\n       because it cannot be inferred. This is different from the symbolic version.\n\n    Parameters\n    ----------\n    cond: a Python function.\n        The loop condition.\n    func: a Python function.\n        The loop body.\n    loop_vars: an NDArray or nested lists of NDArrays.\n        The initial values of the loop variables.\n    max_iterations: a python int.\n        Maximum number of iterations.\n\n    Returns\n    ------\n    outputs: an NDArray or nested lists of NDArrays\n        stacked output from each step\n    states: an NDArray or nested lists of NDArrays\n        final state\n\n    Examples\n    --------\n    >>> cond = lambda i, s: i <= 5\n    >>> func = lambda i, s: ([i + s], [i + 1, s + i])\n    >>> loop_vars = (mx.np.array([0], dtype=\"int64\"), mx.np.array([1], dtype=\"int64\"))\n    >>> outputs, states = mx.npx.while_loop(cond, func, loop_vars, max_iterations=10)\n    >>> outputs\n    [array([[ 1],\n           [ 2],\n           [ 4],\n           [ 7],\n           [11],\n           [16],\n           [ 0],\n           [ 0],\n           [ 0],\n           [ 0]], dtype=int64)]\n    >>> states\n    [array([6], dtype=int64), array([16], dtype=int64)]\n    \"\"\"\n    return _mx_nd_npx.while_loop(cond, func, loop_vars, max_iterations=max_iterations)\n\n\n@set_module('mxnet.numpy_extension')\ndef cond(pred, then_func, else_func, inputs, name=\"cond\"):\n    \"\"\"Run an if-then-else using user-defined condition and computation\n\n    This operator simulates a if-like branch which chooses to do one of\n    the two customized computations according to the specified condition.\n\n    `pred` is a scalar MXNet NDArray,\n    indicating which branch of computation should be used.\n\n    `then_func` is a user-defined function, used as computation of the then branch.\n    It produces `outputs`, which is a list of NDArrays.\n    The signature of `then_func` should be\n    `then_func() => NDArray or nested List[NDArray]`.\n\n    `else_func` is a user-defined function, used as computation of the else branch.\n    It produces `outputs`, which is a list of NDArrays.\n    The signature of `else_func` should be\n    `else_func() => NDArray or nested List[NDArray]`.\n\n    The `outputs` produces by `then_func` and `else_func` should have the same number\n    of elements, all of which should be in the same shape, of the same dtype and stype.\n\n    This function returns a list of symbols, representing the computation result.\n\n    Parameters\n    ----------\n    pred: a Python function.\n        The branch condition.\n    then_func: a Python function.\n        The computation to be executed if `pred` is true.\n    else_func: a Python function.\n        The computation to be executed if `pred` is false.\n\n    Returns\n    -------\n    outputs: an NDArray or nested lists of NDArrays, representing the result of computation.\n\n    Examples\n    --------\n    >>> a, b = mx.np.array([1]), mx.np.array([2])\n    >>> pred = a * b < 5\n    >>> then_func = lambda: (a + 5) * (b + 5)\n    >>> else_func = lambda: (a - 5) * (b - 5)\n    >>> outputs = mx.npx.cond(pred, then_func, else_func)\n    >>> outputs[0]\n    42.0\n    \"\"\"\n    return _mx_nd_npx.cond(pred, then_func, else_func, inputs, name=name)\n"
  },
  {
    "path": "python/mxnet/numpy_extension/image.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Image pre-processing operators.\"\"\"\n\nfrom ..image import *  # pylint: disable=wildcard-import, unused-wildcard-import\n\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/numpy_extension/random.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for ops used in imperative programming.\"\"\"\n\nfrom .. import random as _mx_rand\nfrom ..ndarray import numpy_extension as _mx_nd_npx\nfrom ..util import wrap_ctx_to_device_func\n\n\n__all__ = ['seed', 'bernoulli', 'normal_n', 'uniform_n']\n\n\n@wrap_ctx_to_device_func\ndef seed(seed, device='all'):  # pylint: disable=redefined-outer-name\n    r\"\"\"Seeds the random number generators in MXNet.\n\n    This affects the behavior of modules in MXNet that uses random number generators,\n    like the dropout operator and `ndarray`'s random sampling operators.\n\n    Parameters\n    ----------\n    seed : int\n        The random number seed.\n\n    device : Device\n        The device context of the generator. The default is \"all\" which means seeding random\n        number generators of all devices.\n\n    Notes\n    -----\n    Random number generators in MXNet are device specific.\n    `npx.random.seed(seed)` sets the state of each generator using `seed` and the\n    device id. Therefore, random numbers generated from different devices can be different\n    even if they are seeded using the same seed.\n\n    To produce identical random number sequences independent of the device id,\n    set optional `device` argument. This produces the same sequence of random numbers independent\n    of the device id, but the sequence can be different on different kind of devices as MXNet's\n    random number generators for CPU and GPU use different algorithms.\n\n    Example\n    -------\n    >>> from mxnet import np, npx\n    >>> npx.set_np()\n    >>> npx.random.seed(0)\n    >>> np.random.uniform()\n    array(0.5488135)\n    >>> npx.random.seed(128)\n    >>> np.random.uniform()\n    array(0.03812965)\n    >>> npx.random.seed(128)\n    >>> np.random.uniform()\n    array(0.03812965)\n    >>> npx.random.seed(128)\n    >>> np.random.uniform(device=npx.gpu(0))\n    array(0.9894903, device=gpu(0))\n    >>> npx.random.seed(128)\n    >>> np.random.uniform(device=npx.gpu(0))\n    array(0.9894903, device=gpu(0))\n    \"\"\"\n    _mx_rand.seed(seed_state=seed, device=device)\n\n\n@wrap_ctx_to_device_func\ndef bernoulli(prob=None, logit=None, size=None, dtype=None, device=None, out=None):\n    \"\"\"Creates a Bernoulli distribution parameterized by :attr:`prob`\n    or :attr:`logit` (but not both).\n\n    Samples are binary (0 or 1). They take the value `1` with probability `p`\n    and `0` with probability `1 - p`.\n\n    Parameters\n    ----------\n    prob : float, ndarray\n        The probability of sampling '1'.\n        Only one of prob or logit should be passed in.\n    logit : float, ndarray\n        The log-odds of sampling '1'.\n        Only one of prob or logit should be passed in.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  Default is None, in which case a\n        single value is returned.\n    dtype : dtype, optional\n        Desired dtype of the result. All dtypes are determined by their\n        name, i.e., 'int64', 'int', etc, so byteorder is not available\n        and a specific precision may have different C types depending\n        on the platform. The default value is 'np.float32'.\n    device : Device, optional\n        Device context of output. Default is current device.\n    out : symbol, optional\n        The output symbol (default is `None`).\n\n    Returns\n    -------\n    out : ndarray\n        Drawn samples from the parameterized bernoulli distribution.\n\n    Examples\n    --------\n    >>> prob = np.random.uniform(size=(4,4))\n    >>> logit = np.log(prob) - np.log(1 - prob)\n    >>> npx.random.bernoulli(logit=logit)\n    array([[0., 1., 1., 1.],\n        [0., 1., 1., 1.],\n        [0., 1., 0., 0.],\n        [1., 0., 1., 0.]])\n\n    >>> npx.random.bernoulli(prob=prob)\n    array([[0., 1., 0., 1.],\n        [1., 1., 1., 1.],\n        [1., 1., 1., 0.],\n        [1., 0., 1., 0.]])\n    \"\"\"\n    return _mx_nd_npx.random.bernoulli(prob, logit, size, dtype, device, out)\n\n\n@wrap_ctx_to_device_func\ndef uniform_n(low=0.0, high=1.0, batch_shape=None, dtype=None, device=None):\n    r\"\"\"Draw samples from a uniform distribution.\n\n    Samples are uniformly distributed over the half-open interval\n    ``[low, high)`` (includes low, but excludes high).  In other words,\n    any value within the given interval is equally likely to be drawn\n    by `uniform`.\n\n    Parameters\n    ----------\n    low : float, ndarray, optional\n        Lower boundary of the output interval.  All values generated will be\n        greater than or equal to low.  The default value is 0.\n    high : float, ndarray, optional\n        Upper boundary of the output interval.  All values generated will be\n        less than high.  The default value is 1.0.\n    batch_shape : int or tuple of ints, optional\n        Batch shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k * broadcast(low, high).size`` samples are drawn.\n        If size is ``None`` (default),\n        a scalar tensor containing a single value is returned if\n        ``low`` and ``high`` are both scalars. Otherwise,\n        ``np.broadcast(low, high).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    device : Device, optional\n        Device context of output. Default is current device.\n\n    Returns\n    -------\n    out : ndarray\n        Drawn samples from the parameterized uniform distribution.\n\n    See Also\n    --------\n    randint : Discrete uniform distribution, yielding integers.\n    rand : Convenience function that accepts dimensions as input, e.g.,\n           ``rand(2,2)`` would generate a 2-by-2 array of floats,\n           uniformly distributed over ``[0, 1)``.\n\n    Notes\n    -----\n    The probability density function of the uniform distribution is\n\n    .. math:: p(x) = \\frac{1}{b - a}\n\n    anywhere within the interval ``[a, b)``, and zero elsewhere.\n\n    When ``high`` == ``low``, values of ``low`` will be returned.\n    If ``high`` < ``low``, the results are officially undefined\n    and may eventually raise an error, i.e. do not rely on this\n    function to behave when passed arguments satisfying that\n    inequality condition.\n    \"\"\"\n    return _mx_nd_npx.random.uniform_n(low, high, batch_shape=batch_shape, device=device, dtype=dtype)\n\n\n@wrap_ctx_to_device_func\ndef normal_n(loc=0.0, scale=1.0, batch_shape=None, dtype=None, device=None):\n    r\"\"\"Draw random samples from a normal (Gaussian) distribution.\n\n    Samples are distributed according to a normal distribution parametrized\n    by *loc* (mean) and *scale* (standard deviation).\n\n\n    Parameters\n    ----------\n    loc : float, optional\n        Mean (centre) of the distribution.\n    scale : float, optional\n        Standard deviation (spread or \"width\") of the distribution.\n    batch_shape : int or tuple of ints, optional\n        Batch shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k * broadcast(low, high).size`` samples are drawn.\n        If size is ``None`` (default),\n        a scalar tensor containing a single value is returned if\n        ``low`` and ``high`` are both scalars. Otherwise,\n        ``np.broadcast(loc, scale).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    device : Device, optional\n        Device context of output, default is current device.\n\n    Returns\n    -------\n    out : ndarray\n        Drawn samples from the parameterized normal distribution.\n\n    Notes\n    -----\n    The probability density for the Gaussian distribution is\n\n    .. math:: p(x) = \\frac{1}{\\sqrt{ 2 \\pi \\sigma^2 }}\n                     e^{ - \\frac{ (x - \\mu)^2 } {2 \\sigma^2} },\n\n    where :math:`\\mu` is the mean and :math:`\\sigma` the standard\n    deviation. The square of the standard deviation, :math:`\\sigma^2`,\n    is called the variance.\n\n    The function has its peak at the mean, and its \"spread\" increases with\n    the standard deviation (the function reaches 0.607 times its maximum at\n    :math:`x + \\sigma` and :math:`x - \\sigma` [2]_).  This implies that\n    `numpy.random.normal` is more likely to return samples lying close to\n    the mean, rather than those far away.\n\n    References\n    ----------\n    .. [1] Wikipedia, \"Normal distribution\",\n           https://en.wikipedia.org/wiki/Normal_distribution\n    .. [2] P. R. Peebles Jr., \"Central Limit Theorem\" in \"Probability,\n           Random Variables and Random Signal Principles\", 4th ed., 2001,\n           pp. 51, 51, 125.\n\n    Examples\n    --------\n    >>> mu, sigma = 0, 0.1 # mean and standard deviation\n    >>> s = np.random.normal(mu, sigma, 1000)\n\n    Verify the mean and the variance:\n\n    >>> np.abs(mu - np.mean(s)) < 0.01\n    array(True)\n    \"\"\"\n    return _mx_nd_npx.random.normal_n(loc, scale, batch_shape, dtype, device)\n"
  },
  {
    "path": "python/mxnet/numpy_extension/utils.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Util functions for the numpy module.\"\"\"\n\n\n\nimport ctypes\nfrom ..util import is_np_array, is_np_shape\nfrom ..base import _LIB, check_call, string_types, c_str_array\nfrom ..base import c_handle_array, c_str, mx_uint, NDArrayHandle, py_str\nfrom ..dlpack import ndarray_to_dlpack_for_read, ndarray_to_dlpack_for_write\nfrom ..dlpack import ndarray_from_dlpack, ndarray_from_numpy\nfrom ..numpy import ndarray, array\nfrom ..ndarray import NDArray\n\n__all__ = ['save', 'savez', 'load', 'to_dlpack_for_read', 'to_dlpack_for_write',\n           'from_dlpack', 'from_numpy']\n\ndef save(file, arr):\n    \"\"\"Save an array to a binary file in NumPy ``.npy`` format.\n\n    Parameters\n    ----------\n    file : str\n        File or filename to which the data is saved.  If file is a file-object,\n        then the filename is unchanged.\n    arr : ndarray\n        Array data to be saved. Sparse formats are not supported. Please use\n        savez function to save sparse arrays.\n\n    See Also\n    --------\n    savez : Save several arrays into a ``.npz`` archive\n\n    Notes\n    -----\n    For a description of the ``.npy`` format, see :py:mod:`numpy.lib.format`.\n    \"\"\"\n    if not isinstance(arr, NDArray):\n        raise ValueError(\"data needs to either be a MXNet ndarray\")\n    arr = [arr]\n    keys = None\n    handles = c_handle_array(arr)\n    check_call(_LIB.MXNDArraySave(c_str(file), mx_uint(len(handles)), handles, keys))\n\n\ndef savez(file, *args, **kwds):\n    \"\"\"Save several arrays into a single file in uncompressed ``.npz`` format.\n\n    If arguments are passed in with no keywords, the corresponding variable\n    names, in the ``.npz`` file, are 'arr_0', 'arr_1', etc. If keyword\n    arguments are given, the corresponding variable names, in the ``.npz``\n    file will match the keyword names.\n\n    Parameters\n    ----------\n    file : str\n        Either the filename (string) or an open file (file-like object)\n        where the data will be saved.\n    args : Arguments, optional\n        Arrays to save to the file. Since it is not possible for Python to\n        know the names of the arrays outside `savez`, the arrays will be saved\n        with names \"arr_0\", \"arr_1\", and so on. These arguments can be any\n        expression.\n    kwds : Keyword arguments, optional\n        Arrays to save to the file. Arrays will be saved in the file with the\n        keyword names.\n\n    Returns\n    -------\n    None\n\n    See Also\n    --------\n    save : Save a single array to a binary file in NumPy format.\n\n    Notes\n    -----\n    The ``.npz`` file format is a zipped archive of files named after the\n    variables they contain.  The archive is not compressed and each file\n    in the archive contains one variable in ``.npy`` format. For a\n    description of the ``.npy`` format, see :py:mod:`numpy.lib.format`.\n\n    When opening the saved ``.npz`` file with `load` a dictionary object\n    mapping file-names to the arrays themselves.\n\n    When saving dictionaries, the dictionary keys become filenames\n    inside the ZIP archive. Therefore, keys should be valid filenames.\n    E.g., avoid keys that begin with ``/`` or contain ``.``.\n    \"\"\"\n\n    if len(args):\n        for i, arg in enumerate(args):\n            name = 'arr_{}'.format(str(i))\n            assert name not in kwds, 'Naming conflict between arg {} and kwargs.'.format(str(i))\n            kwds[name] = arg\n\n    str_keys = kwds.keys()\n    nd_vals = kwds.values()\n    if any(not isinstance(k, string_types) for k in str_keys) or \\\n            any(not isinstance(v, NDArray) for v in nd_vals):\n        raise TypeError('Only accepts dict str->ndarray or list of ndarrays')\n\n    keys = c_str_array(str_keys)\n    handles = c_handle_array(nd_vals)\n    check_call(_LIB.MXNDArraySave(c_str(file), mx_uint(len(handles)), handles, keys))\n\n\ndef load(file):\n    \"\"\"Load arrays from ``.npy``, ``.npz`` or legacy MXNet file format.\n\n    See more details in ``save``.\n\n    Parameters\n    ----------\n    file : str\n        The filename.\n\n    Returns\n    -------\n    result : list of ndarrays or dict of str -> ndarray\n        Data stored in the file.\n\n    Notes\n    -----\n    This function can only be called within numpy semantics, i.e., `npx.is_np_shape()`\n    and `npx.is_np_array()` must both return true.\n    \"\"\"\n    if not (is_np_shape() and is_np_array()):\n        raise ValueError('Cannot load `mxnet.numpy.ndarray` in legacy mode. Please activate'\n                         ' numpy semantics by calling `npx.set_np()` in the global scope'\n                         ' before calling this function.')\n    if not isinstance(file, string_types):\n        raise TypeError('file required to be a string')\n    out_size = mx_uint()\n    out_name_size = mx_uint()\n    handles = ctypes.POINTER(NDArrayHandle)()\n    names = ctypes.POINTER(ctypes.c_char_p)()\n    check_call(_LIB.MXNDArrayLoad(c_str(file),\n                                  ctypes.byref(out_size),\n                                  ctypes.byref(handles),\n                                  ctypes.byref(out_name_size),\n                                  ctypes.byref(names)))\n    if out_name_size.value == 0:\n        if out_size.value != 1:\n            return [ndarray(NDArrayHandle(handles[i])) for i in range(out_size.value)]\n        return ndarray(NDArrayHandle(handles[0]))\n    else:\n        assert out_name_size.value == out_size.value\n        return dict(\n            (py_str(names[i]), ndarray(NDArrayHandle(handles[i])))\n            for i in range(out_size.value))\n\nfrom_dlpack = ndarray_from_dlpack(ndarray)\nfrom_dlpack_doc = \"\"\"Returns a np.ndarray backed by a dlpack tensor.\n\n    Parameters\n    ----------\n    dlpack: PyCapsule (the pointer of DLManagedTensor)\n        input data\n\n    Returns\n    -------\n    np.ndarray\n        an ndarray backed by a dlpack tensor\n\n    Examples\n    --------\n    >>> x = mx.np.ones((2,3))\n    >>> y = mx.npx.to_dlpack_for_read(x)\n    >>> type(y)\n    <class 'PyCapsule'>\n    >>> z = mx.npx.from_dlpack(y)\n    >>> type(z)\n    <class 'mxnet.numpy.ndarray'>\n    >>> z\n    array([[1., 1., 1.],\n           [1., 1., 1.]])\n\n    >>> w = mx.npx.to_dlpack_for_write(x)\n    >>> type(w)\n    <class 'PyCapsule'>\n    >>> u = mx.npx.from_dlpack(w)\n    >>> u += 1\n    >>> x\n    array([[2., 2., 2.],\n           [2., 2., 2.]])\n    \"\"\"\nfrom_dlpack.__doc__ = from_dlpack_doc\n\n\nfrom_numpy = ndarray_from_numpy(ndarray, array)\nfrom_numpy_doc = \"\"\"Returns an MXNet's np.ndarray backed by numpy's ndarray.\n    When `zero_copy` is set to be true,\n    this API consumes numpy's ndarray and produces MXNet's np.ndarray\n    without having to copy the content. In this case, we disallow\n    users to modify the given numpy ndarray, and it is suggested\n    not to read the numpy ndarray as well for internal correctness.\n\n    Parameters\n    ----------\n    ndarray: np.ndarray\n        input data\n    zero_copy: bool\n        Whether we use DLPack's zero-copy conversion to convert to MXNet's\n        np.ndarray.\n        This is only available for c-contiguous arrays, i.e. array.flags[C_CONTIGUOUS] == True.\n\n    Returns\n    -------\n    np.ndarray\n        a np.ndarray backed by a dlpack tensor\n    \"\"\"\nfrom_numpy.__doc__ = from_numpy_doc\n\nto_dlpack_for_read = ndarray_to_dlpack_for_read()\nto_dlpack_for_read_doc = \"\"\"Returns a reference view of np.ndarray that represents\nas DLManagedTensor until all previous write operations on the current array are finished.\n\n    Parameters\n    ----------\n    data: np.ndarray\n        input data.\n\n    Returns\n    -------\n    PyCapsule (the pointer of DLManagedTensor)\n        a reference view of ndarray that represents as DLManagedTensor.\n\n    Examples\n    --------\n    >>> x = mx.np.ones((2,3))\n    >>> y = mx.npx.to_dlpack_for_read(x)\n    >>> type(y)\n    <class 'PyCapsule'>\n    >>> z = mx.npx.from_dlpack(y)\n    >>> z\n    array([[1., 1., 1.],\n           [1., 1., 1.]])\n    \"\"\"\nto_dlpack_for_read.__doc__ = to_dlpack_for_read_doc\n\nto_dlpack_for_write = ndarray_to_dlpack_for_write()\nto_dlpack_for_write_doc = \"\"\"Returns a reference view of ndarray that represents\nas DLManagedTensor until all previous read/write operations on the current array are finished.\n\n    Parameters\n    ----------\n    data: np.ndarray\n        input data.\n\n    Returns\n    -------\n    PyCapsule (the pointer of DLManagedTensor)\n        a reference view of np.ndarray that represents as DLManagedTensor.\n\n    Examples\n    --------\n    >>> x = mx.np.ones((2,3))\n    >>> w = mx.npx.to_dlpack_for_write(x)\n    >>> type(w)\n    <class 'PyCapsule'>\n    >>> u = mx.npx.from_dlpack(w)\n    >>> u += 1\n    >>> x\n    array([[2., 2., 2.],\n           [2., 2., 2.]])\n    \"\"\"\nto_dlpack_for_write.__doc__ = to_dlpack_for_write_doc\n"
  },
  {
    "path": "python/mxnet/numpy_op_fallback.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Fallback-to-NumPy operator implementation.\"\"\"\n\nfrom distutils.version import StrictVersion\nimport functools\nimport ast\nimport numpy as np\nfrom . import operator\nfrom . import numpy as _mx_np  # pylint: disable=reimported\nfrom .util import np_array, use_np\nfrom .numpy.utils import _STR_2_DTYPE_\nfrom .ndarray.numpy import _internal as _nd_npi\nfrom .symbol.numpy import _internal as _sym_npi\n\n\ndef register(op_name, imperative=True, symbolic=True):\n    \"\"\"Register operators that fallback to NumPy in modules\n    ``mxnet.ndarray.numpy._internal`` and ``mxnet.symbol.numpy._internal``.\"\"\"\n    def _save_op(mod):\n        if hasattr(mod, op_name):\n            raise ValueError('Duplicate name {} found in module {}'.format(op_name, str(mod)))\n        op = functools.partial(mod.Custom, op_type=op_name)\n        setattr(mod, op_name, op)\n\n    def _register_helper(prop_cls):\n        with np_array():\n            prop_cls = operator.register(op_name)(prop_cls)\n        if imperative:\n            _save_op(_nd_npi)\n        if symbolic:\n            _save_op(_sym_npi)\n        return prop_cls\n\n    return _register_helper\n\n\n@use_np  # enforce np shape and array semantics for all the methods in this class\nclass EmptyLike(operator.CustomOp):\n    \"\"\"Fallback to NumPy empty_like operator.\"\"\"\n    def __init__(self, dtype, order, subok, shape):\n        super(EmptyLike, self).__init__()\n        self._dtype = dtype\n        self._order = order\n        self._subok = subok\n        self._shape = shape\n\n    def forward(self, is_train, req, in_data, out_data, aux):\n        np_version = np.version.version\n        if StrictVersion(np_version) >= StrictVersion('1.6.0'):\n            out = np.empty_like(in_data[0].asnumpy(), dtype=self._dtype, order=self._order,\n                                subok=self._subok)\n        else:\n            out = np.empty_like(in_data[0].asnumpy())\n        self.assign(out_data[0], req[0], _mx_np.array(out, device=in_data[0].device))\n\n    def backward(self, req, out_grad, in_data, out_data, in_grad, aux):\n        raise NotImplementedError('Operator empty_like does not support gradient computation')\n\n\n@register('empty_like_fallback')\nclass EmptyLikeProp(operator.CustomOpProp):\n    \"\"\"Fallback empty_like operator properties.\"\"\"\n    def __init__(self, dtype, order, subok, shape):\n        super(EmptyLikeProp, self).__init__(need_top_grad=True)\n        self._dtype = None if dtype == 'None' else dtype\n        self._order = order\n        self._subok = ast.literal_eval(subok)\n        self._shape = ast.literal_eval(shape)\n\n    def list_arguments(self):\n        return ['prototype']\n\n    def infer_shape(self, in_shape):\n        return (in_shape[0],), (in_shape[0],), ()\n\n    def infer_type(self, in_type):\n        if self._dtype is None:\n            return (in_type[0],), (in_type[0],), ()\n        else:\n            return (in_type[0],), (_STR_2_DTYPE_[self._dtype],), ()\n\n    def create_operator(self, ctx, in_shapes, in_dtypes):\n        return EmptyLike(self._dtype, self._order, self._subok, self._shape)\n\n\n@use_np  # enforce np shape and array semantics for all the methods in this class\nclass Resize(operator.CustomOp):\n    \"\"\"Fallback to NumPy resize operator.\"\"\"\n    def __init__(self, new_shape):\n        super(Resize, self).__init__()\n        self._new_shape = new_shape\n\n    def forward(self, is_train, req, in_data, out_data, aux):\n        out = np.resize(in_data[0].asnumpy(), self._new_shape)\n        self.assign(out_data[0], req[0], _mx_np.array(out, dtype=out.dtype, device=out_data[0].device))\n\n    def backward(self, req, out_grad, in_data, out_data, in_grad, aux):\n        raise NotImplementedError('Operator resize does not support gradient computation')\n\n\n@register('resize_fallback')\nclass ResizeProp(operator.CustomOpProp):\n    \"\"\"Fallback resize operator properties.\"\"\"\n    def __init__(self, new_shape):\n        super(ResizeProp, self).__init__(need_top_grad=True)\n        self._new_shape = ast.literal_eval(new_shape)\n\n    def list_arguments(self):\n        return ['a']\n\n    def infer_shape(self, in_shape):\n        out_shape = (self._new_shape,) if np.isscalar(self._new_shape) else self._new_shape\n        return (in_shape[0],), (out_shape,), ()\n\n    def create_operator(self, ctx, in_shapes, in_dtypes):\n        return Resize(self._new_shape)\n\n\n@use_np\nclass Unravel_index(operator.CustomOp):\n    \"\"\"Fallback to NumPy Unravel_index operator.\"\"\"\n    def __init__(self, shape):\n        super(Unravel_index, self).__init__()\n        self._shape = shape\n\n    def forward(self, is_train, req, in_data, out_data, aux):\n        out = np.unravel_index(in_data[0].asnumpy(), self._shape)\n        self.assign(out_data[0], req[0], _mx_np.array(out, dtype=out[0].dtype, device=out_data[0].device))\n\n    def backward(self, req, out_grad, in_data, out_data, in_grad, aux):\n        raise NotImplementedError('Operator Unravel_index does not support gradient computation')\n\n\n@register('unravel_index_fallback')\nclass Unravel_indexProp(operator.CustomOpProp):\n    \"\"\"Fallback unravel_index operator properties.\"\"\"\n    def __init__(self, shape):\n        super(Unravel_indexProp, self).__init__(need_top_grad=True)\n        self._shape = ast.literal_eval(shape)\n\n    def list_arguments(self):\n        return ['indices']\n\n    def infer_shape(self, in_shape):\n        dim_list = (1,) if np.isscalar(self._shape) else (len(self._shape),)\n        out_shape = dim_list + tuple(in_shape[0])\n        return (in_shape[0],), (out_shape,), ()\n\n    def create_operator(self, ctx, in_shapes, in_dtypes):\n        return Unravel_index(self._shape)\n\n\n@use_np\nclass MultivariateNormal(operator.CustomOp):\n    \"\"\"Fallback to the front-end implementation of random.multivariate_normal.\"\"\"\n    def __init__(self, size=None):\n        super(MultivariateNormal, self).__init__()\n        self._size = size\n\n    def forward(self, is_train, req, in_data, out_data, aux):\n        loc = in_data[0]\n        cov = in_data[1]\n        if cov.dtype == np.float16:\n            scale = _mx_np.linalg.cholesky(cov.astype(np.float32)).astype(np.float16)\n        else:\n            scale = _mx_np.linalg.cholesky(cov)\n        #set context\n        noise = _mx_np.random.normal(size=out_data[0].shape, dtype=loc.dtype, device=loc.device)\n        out = loc + _mx_np.einsum('...jk,...k->...j', scale, noise)\n        self.assign(out_data[0], req[0], out)\n\n    def backward(self, req, out_grad, in_data, out_data, in_grad, aux):\n        raise NotImplementedError('Operator random.multivariate_normal'\n                                  ' does not support gradient computation')\n\n\n@register('mvn_fallback')\nclass MultivariateNormalProp(operator.CustomOpProp):\n    \"\"\"Fallback np.random.multivariate_normal operator properties.\"\"\"\n\n    def __init__(self, size=None):\n        super(MultivariateNormalProp, self).__init__(need_top_grad=True)\n        self._size = ast.literal_eval(\n            size) if size is not None else None\n\n    def list_arguments(self):\n        return ['mean', 'cov']\n\n    def infer_shape(self, in_shape):\n        loc_shape = in_shape[0]\n        cov_shape = in_shape[1]\n        if len(loc_shape) < 1:\n            raise ValueError(\"mean must be at least 1 dimensional\")\n        if len(cov_shape) < 2:\n            raise ValueError(\"cov must be at least 2 dimensional\")\n        if cov_shape[-1] != cov_shape[-2]:\n            raise ValueError(\"the last two dimentions of the parameter cov have to be the same,\"\n                             \" whereas the shape of cov is {}\".format(cov_shape))\n        if cov_shape[-1] != loc_shape[-1]:\n            raise ValueError(\"mean and cov must have same length.\"\n                             \"The shape of mean is {} but the shape of cov is {}\"\n                             .format(loc_shape[-1:], cov_shape[-2:]))\n        # handle shape mismatch here\n        out_shape = np.broadcast(np.empty(loc_shape), np.empty(cov_shape[:-1])).shape\n        if self._size is not None:\n            self._size = (self._size,) if np.isscalar(\n                self._size) else self._size\n            out_shape = self._size + out_shape\n\n        return in_shape, (out_shape,), ()\n\n    def create_operator(self, ctx, in_shapes, in_dtypes):\n        return MultivariateNormal(self._size)\n"
  },
  {
    "path": "python/mxnet/numpy_op_signature.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Make builtin ops' signatures compatible with NumPy.\"\"\"\n\nimport inspect\nfrom . import _numpy_op_doc\nfrom . import numpy as mx_np\nfrom . import numpy_extension as mx_npx\nfrom .base import _NP_OP_SUBMODULE_LIST, _NP_EXT_OP_SUBMODULE_LIST, _get_op_submodule_name\n\n\ndef _get_builtin_op(op_name):\n    if op_name.startswith('_np_'):\n        root_module = mx_np\n        op_name_prefix = '_np_'\n        submodule_name_list = _NP_OP_SUBMODULE_LIST\n    elif op_name.startswith('_npx_'):\n        root_module = mx_npx\n        op_name_prefix = '_npx_'\n        submodule_name_list = _NP_EXT_OP_SUBMODULE_LIST\n    else:\n        return None\n\n    submodule_name = _get_op_submodule_name(op_name, op_name_prefix, submodule_name_list)\n    op_module = root_module\n    if len(submodule_name) > 0:\n        op_module = getattr(root_module, submodule_name[1:-1], None)\n        if op_module is None:\n            raise ValueError('Cannot find submodule {} in module {}'\n                             .format(submodule_name[1:-1], root_module.__name__))\n\n    op = getattr(op_module, op_name[(len(op_name_prefix)+len(submodule_name)):], None)\n    if op is None:\n        raise ValueError('Cannot find operator {} in module {}'\n                         .format(op_name[len(op_name_prefix):], root_module.__name__))\n    return op\n\n\ndef _register_op_signatures():\n    for op_name in dir(_numpy_op_doc):\n        op = _get_builtin_op(op_name)\n        if op is not None:\n            op.__signature__ = inspect.signature(getattr(_numpy_op_doc, op_name))\n\n\n_register_op_signatures()\n"
  },
  {
    "path": "python/mxnet/onnx/README.md",
    "content": "<!--\n  ~ Licensed to the Apache Software Foundation (ASF) under one\n  ~ or more contributor license agreements.  See the NOTICE file\n  ~ distributed with this work for additional information\n  ~ regarding copyright ownership.  The ASF licenses this file\n  ~ to you under the Apache License, Version 2.0 (the\n  ~ \"License\"); you may not use this file except in compliance\n  ~ with the License.  You may obtain a copy of the License at\n  ~\n  ~   http://www.apache.org/licenses/LICENSE-2.0\n  ~\n  ~ Unless required by applicable law or agreed to in writing,\n  ~ software distributed under the License is distributed on an\n  ~ \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n  ~ KIND, either express or implied.  See the License for the\n  ~ specific language governing permissions and limitations\n  ~ under the License.\n  ~\n-->\n\n# ONNX Export Support for MXNet\n\n### Overview\n[ONNX](https://onnx.ai/), or Open Neural Network Exchange, is an open source deep learning model format that acts as a framework neutral graph representation between DL frameworks or between training and inference. With the ability to export models to the ONNX format, MXNet users can enjoy faster inference and a wider range of deployment device choices, including edge and mobile devices where MXNet installation may be constrained. Popular hardware-accelerated and/or cross-platform ONNX runtime frameworks include Nvidia [TensorRT](https://github.com/onnx/onnx-tensorrt), Microsoft [ONNXRuntime](https://github.com/microsoft/onnxruntime), Apple [CoreML](https://github.com/onnx/onnx-coreml), etc.\n\n### ONNX Versions Supported\nONNX 1.7 & 1.8\n\n### Installation\nFrom MXNet 1.9 release and on, the ONNX export module has become an offical, built-in feature in MXNet. You can access the module at `mxnet.onnx`.\n\nIf you are a user of earlier MXNet versions and do not want to upgrade MXNet, you can still enjoy the latest ONNX support by pulling the MXNet source code and building the wheel for only the mx2onnx module. Just do `cd python/mxnet/onnx` and then build the wheel with `python3 -m build`. You should be able to find the wheel under `python/mxnet/onnx/dist/mx2onnx-0.0.0-py3-none-any.whl` and install it with `pip install mx2onnx-0.0.0-py3-none-any.whl`. You can then access the module with `import mx2onnx`. The `mx2onnx` namespace is equivalent to `mxnet.onnx`.\n\n### APIs\nThe main API is `export_model`, which, as the name suggests, exports an MXNet model to the ONNX format.\n\n```python\nmxnet.onnx.export_model(sym, params, in_shapes=None, in_types=np.float32,\n                 onnx_file_path='model.onnx', verbose=False, dynamic=False,\n                 dynamic_input_shapes=None, run_shape_inference=False, input_type=None,\n                 input_shape=None)\n```\n\nParameters:\n\n    sym : str or symbol object\n        Path to the MXNet json file or Symbol object\n    params : str or dict or list of dict\n        str - Path to the MXNet params file\n        dict - MXNet params dictionary (Including both arg_params and aux_params)\n        list - list of length 2 that contains MXNet arg_params and aux_params\n    in_shapes : List of tuple\n        Input shape of the model e.g [(1,3,224,224)]\n    in_types : data type or list of data types\n        Input data type e.g. np.float32, or [np.float32, np.int32]\n    onnx_file_path : str\n        Path where to save the generated onnx file\n    verbose : Boolean\n        If True will print logs of the model conversion\n    dynamic: Boolean\n        If True will allow for dynamic input shapes to the model\n    dynamic_input_shapes: list of tuple\n        Specifies the dynamic input_shapes. If None then all dimensions are set to None\n    run_shape_inference : Boolean\n        If True will run shape inference on the model\n    input_type : data type or list of data types\n        This is the old name of in_types. We keep this parameter name for backward compatibility\n    input_shape : List of tuple\n        This is the old name of in_shapes. We keep this parameter name for backward compatibility\n    large_model : Boolean\n        Whether to export a model that is larger than 2 GB. If true will save param tensors in separate\n        files along with .onnx model file. This feature is supported since onnx 1.8.0\n\nReturns:\n\n    onnx_file_path : str\n        Onnx file path\n\n#### Model with Multiple Input\nWhen the model has multiple inputs, all the input shapes and dtypes must be provided with `in_shapes` and `in_dtypes`. Note that the shape/dtype in `in_shapes`/`in_dtypes` must follow the same order as in the MXNet model symbol file. If `in_dtypes` is provided as a single data type, then that type will be applied to all input nodes.\n\n#### Dynamic Shape Input\nWe can set `dynamic=True` to turn on support for dynamic input shapes. Note that even with dynamic shapes, a set of static input shapes still need to be specified in `in_shapes`; on top of that, we'll also need to specify which dimensions of the input shapes are dynamic in `dynamic_input_shapes`. We can simply set the dynamic dimensions as `None`, e.g. `(1, 3, None, None)`, or use strings in place of the `None`'s for better understandability in the exported onnx graph, e.g. `(1, 3, 'Height', 'Width')`\n\n```python\n# The batch dimension will be dynamic in this case\nin_shapes = [(1, 3, 224, 224)]\ndynamic_input_shapes = [(None, 3, 224, 224)]\nmx.onnx.export_model(mx_sym, mx_params, in_shapes, in_types, onnx_file,\n                     dynamic=True, dynamic_input_shapes=dynamic_input_shapes)\n```\n\n#### Export Large Model\nUsers can set `large_model=True` to export models that are larger than 2GB. In this case, all parameter tensors will be saved into separate files along with the .onnx model file.\n\n### Operator Support Matrix\nWe have implemented export logics for a wide range of MXNet operators, and thus supported most CV and NLP use cases. Below is our most up-to-date operator support matrix.\n\n|MXNet Op|ONNX Version|\n|:-|:-:|\n|TODO|TODO|\n"
  },
  {
    "path": "python/mxnet/onnx/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"ONNX Export module\"\"\"\n\nfrom .mx2onnx import export_model, get_operator_support\n"
  },
  {
    "path": "python/mxnet/onnx/mx2onnx/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"ONNX Export module\"\"\"\n\nfrom ._export_model import export_model, get_operator_support\nfrom ._op_translations import _op_translations_opset12\nfrom ._op_translations import _op_translations_opset13\n"
  },
  {
    "path": "python/mxnet/onnx/mx2onnx/_export_helper.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"export helper functions\"\"\"\n# coding: utf-8\nimport os\nimport logging\nimport mxnet as mx\n\n\ndef load_module(sym_filepath, params_filepath):\n    \"\"\"Loads the MXNet model file and\n    returns MXNet symbol and params (weights).\n\n    Parameters\n    ----------\n    json_path : str\n        Path to the json file\n    params_path : str\n        Path to the params file\n\n    Returns\n    -------\n    sym : MXNet symbol\n        Model symbol object\n\n    params : params object\n        Model weights including both arg and aux params.\n    \"\"\"\n    if not (os.path.isfile(sym_filepath) and os.path.isfile(params_filepath)):\n        raise ValueError(\"Symbol and params files provided are invalid\")\n\n    try:\n        # reads symbol.json file from given path and\n        # retrieves model prefix and number of epochs\n        model_name = sym_filepath.rsplit('.', 1)[0].rsplit('-', 1)[0]\n        params_file_list = params_filepath.rsplit('.', 1)[0].rsplit('-', 1)\n        # Setting num_epochs to 0 if not present in filename\n        num_epochs = 0 if len(params_file_list) == 1 else int(params_file_list[1])\n    except IndexError:\n        logging.info(\"Model and params name should be in format: \"\n                     \"prefix-symbol.json, prefix-epoch.params\")\n        raise\n\n    sym, arg_params, aux_params = mx.model.load_checkpoint(model_name, num_epochs)\n\n    # Merging arg and aux parameters\n    params = {}\n    params.update(arg_params)\n    params.update(aux_params)\n\n    return sym, params\n"
  },
  {
    "path": "python/mxnet/onnx/mx2onnx/_export_model.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n#pylint: disable-msg=too-many-arguments\n\n\"\"\"Exports an MXNet model to the ONNX model format\"\"\"\nimport logging\nimport numpy as np\n\nfrom mxnet.base import string_types\nfrom mxnet import symbol\nfrom ._export_onnx import MXNetGraph\nfrom ._export_helper import load_module\n\n\ndef get_operator_support(opset_version=None):\n    \"\"\"Return a list of MXNet operators supported by the current/specified opset\n    \"\"\"\n    try:\n        from onnx.defs import onnx_opset_version\n    except ImportError:\n        raise ImportError(\"Onnx and protobuf need to be installed. \"\n                          + \"Instructions to install - https://github.com/onnx/onnx\")\n    if opset_version is None:\n        opset_version = onnx_opset_version()\n    all_versions = range(opset_version, 11, -1)\n    ops = set()\n    for ver in all_versions:\n        if ver in MXNetGraph.registry_:\n            ops.update(MXNetGraph.registry_[ver].keys())\n    ops = list(ops)\n    ops.sort()\n    return ops\n\n\ndef export_model(sym, params, in_shapes=None, in_types=np.float32,\n                 onnx_file_path='model.onnx', verbose=False, dynamic=False,\n                 dynamic_input_shapes=None, run_shape_inference=False, input_type=None,\n                 input_shape=None, large_model=False):\n    \"\"\"Exports the MXNet model file, passed as a parameter, into ONNX model.\n    Accepts both symbol,parameter objects as well as json and params filepaths as input.\n    Operator support and coverage -\n    https://github.com/apache/mxnet/tree/v1.x/python/mxnet/onnx#user-content-operator-support-matrix\n\n    Parameters\n    ----------\n    sym : str or symbol object\n        Path to the json file or Symbol object\n    params : str or dict or list of dict\n        str - Path to the params file\n        dict - params dictionary (Including both arg_params and aux_params)\n        list - list of length 2 that contains arg_params and aux_params\n    in_shapes : List of tuple\n        Input shape of the model e.g [(1,3,224,224)]\n    in_types : data type or list of data types\n        Input data type e.g. np.float32, or [np.float32, np.int32]\n    onnx_file_path : str\n        Path where to save the generated onnx file\n    verbose : Boolean\n        If True will print logs of the model conversion\n    dynamic: Boolean\n        If True will allow for dynamic input shapes to the model\n    dynamic_input_shapes: list of tuple\n        Specifies the dynamic input_shapes. If None then all dimensions are set to None\n    run_shape_inference : Boolean\n        If True will run shape inference on the model\n    input_type : data type or list of data types\n        This is the old name of in_types. We keep this parameter name for backward compatibility\n    input_shape : List of tuple\n        This is the old name of in_shapes. We keep this parameter name for backward compatibility\n    large_model : Boolean\n        Whether to export a model that is larger than 2 GB. If true will save param tensors in separate\n        files along with .onnx model file. This feature is supported since onnx 1.8.0\n\n    Returns\n    -------\n    onnx_file_path : str\n        Onnx file path\n\n    Notes\n    -----\n    This method is available when you ``import mxnet.onnx``\n\n    \"\"\"\n\n    try:\n        import onnx\n        from onnx import helper, mapping, shape_inference\n        from onnx.defs import onnx_opset_version\n    except ImportError:\n        raise ImportError(\"Onnx and protobuf need to be installed. \"\n                          + \"Instructions to install - https://github.com/onnx/onnx\")\n\n    if input_type is not None:\n        in_types = input_type\n\n    if input_shape is not None:\n        in_shapes = input_shape\n\n    converter = MXNetGraph()\n    opset_version = onnx_opset_version()\n\n    if not isinstance(in_types, list):\n        in_types = [in_types for _ in range(len(in_shapes))]\n    in_types_t = [mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(i_t)] for i_t in in_types]\n    assert len(in_types) == len(in_shapes), \"The lengths of in_types and in_shapes must equal\"\n    # if input parameters are strings(file paths), load files and create symbol parameter objects\n    if isinstance(sym, string_types) and isinstance(params, string_types):\n        logging.info(\"Converting json and weight file to sym and params\")\n        sym_obj, params_obj = load_module(sym, params)\n        onnx_graph = converter.create_onnx_graph_proto(sym_obj, params_obj, in_shapes,\n                                                       in_types_t,\n                                                       verbose=verbose, opset_version=opset_version,\n                                                       dynamic=dynamic, dynamic_input_shapes=dynamic_input_shapes)\n    elif isinstance(sym, symbol.Symbol) and isinstance(params, dict):\n        onnx_graph = converter.create_onnx_graph_proto(sym, params, in_shapes,\n                                                       in_types_t,\n                                                       verbose=verbose, opset_version=opset_version,\n                                                       dynamic=dynamic, dynamic_input_shapes=dynamic_input_shapes)\n    elif isinstance(sym, symbol.Symbol) and isinstance(params, list) and len(params) == 2:\n        # when params contains arg_params and aux_params\n        p = {}\n        p.update(params[0])\n        p.update(params[1])\n        onnx_graph = converter.create_onnx_graph_proto(sym, p, in_shapes,\n                                                       in_types_t,\n                                                       verbose=verbose, opset_version=opset_version,\n                                                       dynamic=dynamic, dynamic_input_shapes=dynamic_input_shapes)\n    else:\n        raise ValueError(\"Input sym and params should either be files or objects\")\n\n    # Create the model (ModelProto)\n    onnx_model = helper.make_model(onnx_graph)\n\n    # Run shape inference on the model. Due to ONNX bug/incompatibility this may or may not crash\n    if run_shape_inference:\n        try:\n            onnx_model = shape_inference.infer_shapes(onnx_model)\n        except: # pylint: disable=bare-except\n            logging.info(\"Shape inference failed, original export is kept.\")\n\n    if large_model:\n        from onnx.external_data_helper import convert_model_to_external_data\n        convert_model_to_external_data(onnx_model, all_tensors_to_one_file=False, location=onnx_file_path+'.data')\n\n    onnx.save_model(onnx_model, onnx_file_path)\n    onnx.checker.check_model(onnx_file_path)\n    return onnx_file_path\n"
  },
  {
    "path": "python/mxnet/onnx/mx2onnx/_export_onnx.py",
    "content": "#  Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.\n#\n#  Redistribution and use in source and binary forms, with or without\n#  modification, are permitted provided that the following conditions\n#  are met:\n#  * Redistributions of source code must retain the above copyright\n#    notice, this list of conditions and the following disclaimer.\n#  * Redistributions in binary form must reproduce the above copyright\n#    notice, this list of conditions and the following disclaimer in the\n#    documentation and/or other materials provided with the distribution.\n#  * Neither the name of NVIDIA CORPORATION nor the names of its\n#    contributors may be used to endorse or promote products derived\n#    from this software without specific prior written permission.\n#\n#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n#  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n#  PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n#  OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# Based on\n# https://github.com/NVIDIA/mxnet_to_onnx/blob/master/mx2onnx_converter/mx2onnx_converter.py\n\n# coding: utf-8\n# pylint: disable=invalid-name,too-many-locals,no-self-use,too-many-arguments,\n# pylint: disable=maybe-no-member,too-many-nested-blocks,logging-not-lazy\n# pylint: disable=cell-var-from-loop\n\"\"\"MXNet to ONNX graph converter functions\"\"\"\nimport logging\nimport json\n\nimport numpy as np\nfrom mxnet import ndarray as nd\n\n\nclass MXNetGraph(object):\n    \"\"\"Class to convert MXNet to ONNX graph\"\"\"\n    registry_ = {}\n    input_output_maps_ = {}\n\n    def __init__(self):\n        # topologically sorted nodes\n        self.nodes = []\n        self.input_tensors = []\n        self.output_tensors = []\n\n    @staticmethod\n    def register(op_name, opset_version=12):\n        \"\"\"Register operators\"\"\"\n        def wrapper(func):\n            \"\"\"Helper function to map functions\"\"\"\n            try:\n                import onnx as _\n                op_map = MXNetGraph.registry_.setdefault(opset_version, {})\n                op_map[op_name] = func\n            except ImportError:\n                pass\n            return func\n\n        return wrapper\n\n    @staticmethod\n    def convert_layer(node, **kwargs):\n        \"\"\"Convert MXNet layer to ONNX\"\"\"\n        try:\n            from onnx.defs import onnx_opset_version\n        except ImportError:\n            raise ImportError(\"Onnx and protobuf need to be installed. \"\n                              + \"Instructions to install - https://github.com/onnx/onnx\")\n\n        op = str(node[\"op\"])\n        opset_version = kwargs.get(\"opset_version\", onnx_opset_version())\n        if opset_version < 12:\n            logging.warning('Your ONNX op set version is {}, '\n                            'which is lower than then lowest tested op set (12), please consider '\n                            'updating ONNX'.format(str(opset_version)))\n            opset_version = 12\n        # Fallback to older opset versions if op is not registered in current version\n        convert_func = None\n        for op_version in range(opset_version, 11, -1):\n            if op_version not in MXNetGraph.registry_ or op not in MXNetGraph.registry_[op_version]:\n                continue\n            convert_func = MXNetGraph.registry_[op_version][op]\n            break\n\n        # The conversion logic is not implemented\n        if convert_func is None:\n            raise AttributeError(f\"No conversion function registered for op type {op} yet.\")\n\n        ret = convert_func(node, **kwargs)\n        # in case the conversion function does not specify the returned dtype, we just return None\n        # as the second value\n        if isinstance(ret, list):\n            return ret, None\n        else:\n            return ret\n\n    @staticmethod\n    def split_params(sym, params):\n        \"\"\"Helper function to split params dictionary into args and aux params\n\n        Parameters\n        ----------\n        sym : :class:`~mxnet.symbol.Symbol`\n            MXNet symbol object\n        params : dict of ``str`` to :class:`~mxnet.ndarray.NDArray`\n            Dict of converted parameters stored in ``mxnet.ndarray.NDArray`` format\n\n        Returns\n        -------\n        arg_params : dict of ``str`` to :class:`~mxnet.ndarray.NDArray`\n            Dict of converted parameters stored in ``mxnet.ndarray.NDArray`` format\n        aux_params : dict of ``str`` to :class:`~mxnet.ndarray.NDArray`\n            Dict of converted parameters stored in ``mxnet.ndarray.NDArray`` format\n        \"\"\"\n        arg_params = {}\n        aux_params = {}\n        for args in sym.list_arguments():\n            if args in params:\n                arg_params.update({args: nd.array(params[args])})\n        for aux in sym.list_auxiliary_states():\n            if aux in params:\n                aux_params.update({aux: nd.array(params[aux])})\n        return arg_params, aux_params\n\n    @staticmethod\n    def get_outputs(sym, params, in_shapes, output_label, in_types, dynamic=False,\n                    dynamic_input_shapes=None):\n        \"\"\"Helper function to collect the output names, types, and shapes\n\n        Parameters\n        ----------\n        sym : :class:`~mxnet.symbol.Symbol`\n            MXNet symbol object\n        params : dict of ``str`` to :class:`~mxnet.ndarray.NDArray`\n            Dict of converted parameters stored in ``mxnet.ndarray.NDArray`` format\n        in_shapes : list of tuple\n            Input shapes\n        out_label : ``str``\n            Name of label typically used in loss that may be left in graph. This name is\n            removed from list of inputs required by symbol\n        in_types : list of Int\n            Input ONNX data types\n        dynamic : Boolean\n            If True will allow for dynamic input shapes to the model\n        dynamic_input_shapes: list of tuple\n            Specifies the dynamic input_shapes. If None then all dimensions are set to None\n\n        Returns\n        in_shapes : list of tuple\n            Updated input shapes\n        graph_outputs : dict ``str`` to dict\n            This maps output name to {'shape':tuple, 'dtype':Int}\n        -------\n        \"\"\"\n        from onnx import mapping\n        import re\n\n        # Collect graph output names\n        out_names = list()\n        for name in sym.list_outputs():\n            if name.endswith('_state_output'): # handel special cases for RNN operator\n                out_names.append(name[:-len('_state_output')]+'1')\n            elif name.endswith('_statecell_output'): # handel special cases for RNN operator\n                out_names.append(name[:-len('_statecell_output')]+'2')\n            elif name.endswith('_output'):\n                out_names.append(name[:-len('_output')])\n            elif name.endswith('_out'):\n                out_names.append(name[:-len('_out')])\n            elif re.search('.*_output[0-9]$', name):\n                out_names.append(name[:-len('_output0')]+name[-1])\n            else:\n                logging.info(\"output '%s' does not end with '_output'\", name)\n                out_names.append(name)\n\n        # Collect graph output shapes\n        # Remove any input listed in params from sym.list_inputs() and bind them to the input shapes provided\n        # by user. Also remove output_label, which is the name of the label symbol that may have been used\n        # as the label for loss during training.\n        inputs = {n: tuple(s) for n, s in\n                  zip([n for n in sym.list_inputs() if n not in params and n != output_label],\n                      in_shapes)}\n        # Add params and their shape to list of inputs\n        inputs.update({n: v.shape for n, v in params.items() if n in sym.list_inputs()})\n        # Provide input data as well as input params to infer_shape()\n        _, out_shapes, _ = sym.infer_shape(**inputs)\n        if dynamic:\n            # Keep the dimensionality of the output shapes but change the values to None\n            out_shapes = [tuple(None for _ in i_s) for i_s in out_shapes]\n\n            if dynamic_input_shapes is None:\n                # Set all dimensions to None\n                in_shapes = [tuple(None for _ in i_s) for i_s in in_shapes]\n            else:\n                assert len(in_shapes) == len(dynamic_input_shapes), \"The length of \" \\\n                    \"dynamic_input_shapes must equal to the length of in_shapes.\"\n                for i_s, d_i_s in zip(in_shapes, dynamic_input_shapes):\n                    assert len(i_s) == len(d_i_s), \"The dimensionality \" \\\n                        \"of each shape must match.\"\n                in_shapes = dynamic_input_shapes\n        else:\n            assert dynamic_input_shapes is None, \"dynamic_input_shapes is specified. Please \" \\\n                \"set dynamic_input_shapes=True to enable dynamic input shapes\"\n\n        # Collect graph output types\n        # Remove any input listed in params from sym.list_inputs() and bind them to the input types provided\n        # by user. Also remove output_label\n        in_dtypes = {n: mapping.TENSOR_TYPE_TO_NP_TYPE[t] for n, t in\n                     zip([n for n in sym.list_inputs() if n not in params and n != output_label],\n                         in_types)}\n        # Add params and their types to list of inputs\n        in_dtypes.update({n: v.dtype for n, v in params.items() if n in sym.list_inputs()})\n        _, out_type, _ = sym.infer_type(**in_dtypes)\n        out_types = [mapping.NP_TYPE_TO_TENSOR_TYPE[o(0).dtype] for o in out_type]\n\n        # Make sure the types, names, and shapes all align up\n        assert len(out_types) == len(out_names) == len(out_shapes)\n\n        # Bind output shapes/types with output names\n        graph_outputs = {n: {'shape': s, 'dtype': d} for n, s, d in zip(out_names, out_shapes, out_types)}\n\n        return in_shapes, graph_outputs\n\n    @staticmethod\n    def convert_weights_to_numpy(weights_dict):\n        \"\"\"Convert weights to numpy\"\"\"\n        return dict([(k.replace(\"arg:\", \"\").replace(\"aux:\", \"\"), v.asnumpy())\n                     for k, v in weights_dict.items()])\n\n    def create_onnx_graph_proto(self, sym, params, in_shapes, in_types, verbose=False, opset_version=None,\n                                dynamic=True, dynamic_input_shapes=None):\n        \"\"\"Convert MXNet graph to ONNX graph\n\n        Parameters\n        ----------\n        sym : :class:`~mxnet.symbol.Symbol`\n            MXNet symbol object\n        params : dict of ``str`` to :class:`~mxnet.ndarray.NDArray`\n            Dict of converted parameters stored in ``mxnet.ndarray.NDArray`` format\n        in_shapes : List of tuple\n            Input shape of the model e.g [(1,3,224,224)]\n        in_types : List of Int\n            Input ONNX data types\n        verbose : Boolean\n            If true will print logs of the model conversion\n        opset_version : Int\n            ONNX opset version to use for export, defaults to latest supported by onnx package\n        dynamic: Boolean\n            If True will allow for dynamic input shapes to the model\n        dynamic_input_shapes: list of tuple\n            Specifies the dynamic input_shapes. If None then all dimensions are set to None\n\n        Returns\n        -------\n        graph : GraphProto\n            ONNX graph\n        \"\"\"\n        try:\n            from onnx import (helper, NodeProto, ValueInfoProto, TensorProto)\n            from onnx.helper import make_tensor_value_info\n            from onnx.defs import onnx_opset_version\n        except ImportError:\n            raise ImportError(\"Onnx and protobuf need to be installed. \"\n                              + \"Instructions to install - https://github.com/onnx/onnx\")\n\n        if opset_version is None:\n            opset_version = onnx_opset_version()\n\n        # When MXNet model is saved to json file , MXNet adds a node for label.\n        # The name of this node is, name of the last node + \"_label\" ( i.e if last node\n        # name is \"Softmax\", this node will have a name \"Softmax_label\". Also, the new node\n        # will always be second last node in the json graph.\n        # Deriving the output_label name.\n        output_label = sym.get_internals()[len(sym.get_internals()) - 1].name + \"_label\"\n\n        weights = MXNetGraph.convert_weights_to_numpy(params)\n\n        mx_graph = json.loads(sym.tojson())[\"nodes\"]\n\n        class NodeOutput:\n            def __init__(self, name, dtype):\n                self.name = name\n                self.dtype = np.dtype(dtype)\n\n        initializer = []\n        all_processed_nodes = []\n        onnx_processed_nodes = []\n        onnx_processed_inputs = []\n        onnx_processed_outputs = []\n        outputs_lookup = []\n\n        # Determine graph output names, shapes, and dtypes. Also update in_shapes\n        in_shapes, graph_outputs = MXNetGraph.get_outputs(sym, params, in_shapes, output_label,\n                                                          in_types, dynamic, dynamic_input_shapes)\n        appeared_names = set()\n        graph_input_idx = 0\n        for idx, node in enumerate(mx_graph):\n            op = node[\"op\"]\n            # check if the current node has the same name as nodes before\n            if node[\"name\"] in appeared_names:\n                node[\"name\"] = 'idx_' + str(idx) + '_' + node[\"name\"]\n            else:\n                appeared_names.add(node[\"name\"])\n            name = node[\"name\"]\n            if verbose:\n                logging.info(\"Converting idx: %d, op: %s, name: %s\", idx, op, name)\n\n            # A node is an input node if its op_name is \"null\" and is not\n            # in params dict\n            if op == \"null\" and name not in params:\n                # Handle graph input\n\n                # Skip output_label node, as this node is not part of graph\n                # Refer to \"output_label\" assignment above for more details.\n                if name == output_label:\n                    continue\n\n                converted, dtypes = MXNetGraph.convert_layer(\n                    node,\n                    is_input=True,\n                    mx_graph=mx_graph,\n                    weights=weights,\n                    in_shape=in_shapes[graph_input_idx],\n                    in_type=in_types[graph_input_idx],\n                    proc_nodes=all_processed_nodes,\n                    initializer=initializer,\n                    outputs_lookup=outputs_lookup)\n                graph_input_idx += 1\n            else:\n                # Handle graph layers\n                converted, dtypes = MXNetGraph.convert_layer(\n                    node,\n                    is_input=False,\n                    mx_graph=mx_graph,\n                    weights=weights,\n                    proc_nodes=all_processed_nodes,\n                    initializer=initializer,\n                    outputs_lookup=outputs_lookup,\n                    idx=idx,\n                    opset_version=opset_version\n                )\n            if isinstance(converted, list):\n                # Collect all the node's output names\n                node_possible_names = [name] + [name + str(i) for i in range(100)]\n                node_output_names = []\n                # Collect all the graph's output names\n                graph_output_names = []\n                # Iterate for all converted nodes\n                for converted_node in converted:\n                    # If converted node is ValueInfoProto, add it in inputs\n                    if isinstance(converted_node, ValueInfoProto):\n                        onnx_processed_inputs.append(converted_node)\n                    # If converted node is NodeProto, add it in processed nodes list\n                    elif isinstance(converted_node, NodeProto):\n                        onnx_processed_nodes.append(converted_node)\n                        # some operators have multiple outputs,\n                        # therefore, check all output node names\n                        node_names = list(converted_node.output)\n                        for nodename in node_names:\n                            if nodename in node_possible_names:\n                                node_output_names.append(nodename)\n                            if nodename in graph_outputs:\n                                graph_output_names.append(nodename)\n                                if verbose:\n                                    logging.info(\"Output node is: {}\".format(nodename))\n                    elif isinstance(converted_node, TensorProto):\n                        raise ValueError(\"Did not expect TensorProto\")\n                    else:\n                        raise ValueError(f\"node is of an unrecognized type: {type(node)}\")\n\n                    all_processed_nodes.append(converted_node)\n\n                # if node_output_names is empty then we use the last returned node as output\n                if not node_output_names:\n                    node_output_names = [converted[-1].name]\n                # process node outputs (sort by output index)\n                def str2int(s, l):\n                    if len(s) == l:\n                        return -1\n                    else:\n                        return int(s[l:])\n\n                node_output_names = sorted(node_output_names, key=lambda x: str2int(x, len(name)))\n\n                # match the output names to output dtypes\n                if dtypes is not None:\n                    assert len(node_output_names) == len(dtypes)\n                    node_outputs = [NodeOutput(node_output_names[i], dtypes[i])\n                                    for i in range(len(dtypes))]\n                else:\n                    # in case dtypes is None, we just default to the dtype of the first input\n                    assert len(node[\"inputs\"]) > 0\n                    first_input = node[\"inputs\"][0]\n                    first_input_dtype = outputs_lookup[first_input[0]][first_input[1]].dtype\n                    node_outputs = [NodeOutput(n, first_input_dtype)\n                                    for n in node_output_names]\n                outputs_lookup.append(node_outputs)\n\n                # process graph outputs (sort by alphabetical order)\n                graph_output_names.sort()\n                for nodename in graph_output_names:\n                    onnx_processed_outputs.append(\n                        make_tensor_value_info(\n                            name=nodename,\n                            elem_type=graph_outputs[nodename]['dtype'],\n                            shape=graph_outputs[nodename]['shape']\n                        )\n                    )\n\n            else:\n                logging.info(\"Operator converter function should always return a list\")\n\n        # sometimes the graph output can also be in the intializer\n        for i in initializer:\n            if i.name in graph_outputs:\n                onnx_processed_outputs.append(\n                    make_tensor_value_info(\n                        name=i.name,\n                        elem_type=graph_outputs[i.name]['dtype'],\n                        shape=graph_outputs[i.name]['shape']\n                    )\n                )\n\n        graph = helper.make_graph(\n            onnx_processed_nodes,\n            \"mxnet_converted_model\",\n            onnx_processed_inputs,\n            onnx_processed_outputs\n        )\n\n        graph.initializer.extend(initializer)\n\n        return graph\n"
  },
  {
    "path": "python/mxnet/onnx/mx2onnx/_op_translations/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"ONNX export op translation\"\"\"\n\nfrom . import _op_translations_opset12\nfrom . import _op_translations_opset13\n"
  },
  {
    "path": "python/mxnet/onnx/mx2onnx/_op_translations/_op_translations_opset12.py",
    "content": "#  Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.\n#\n#  Redistribution and use in source and binary forms, with or without\n#  modification, are permitted provided that the following conditions\n#  are met:\n#  * Redistributions of source code must retain the above copyright\n#    notice, this list of conditions and the following disclaimer.\n#  * Redistributions in binary form must reproduce the above copyright\n#    notice, this list of conditions and the following disclaimer in the\n#    documentation and/or other materials provided with the distribution.\n#  * Neither the name of NVIDIA CORPORATION nor the names of its\n#    contributors may be used to endorse or promote products derived\n#    from this software without specific prior written permission.\n#\n#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n#  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n#  PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n#  OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# Based on\n#  https://github.com/NVIDIA/mxnet_to_onnx/blob/master/mx2onnx_converter/\n# mx2onnx_converter_functions.py\n\n# coding: utf-8\n# pylint: disable=too-many-locals,no-else-return,too-many-lines\n# pylint: disable=anomalous-backslash-in-string,eval-used\n# pylint: disable=too-many-function-args\n\"\"\"\nConversion Functions for common layers.\nAdd new functions here with a decorator.\n\"\"\"\n\nimport re\nimport logging\nimport numpy as np\nfrom .._export_onnx import MXNetGraph as mx_op\ntry:\n    import onnx\nexcept ImportError:\n    onnx = None\n\n\ndef parse_helper(attrs, attrs_name, alt_value=None):\n    \"\"\"Helper function to parse operator attributes in required format.\"\"\"\n    tuple_re = re.compile(r'\\([0-9L|,| ]+\\)')\n    if not attrs:\n        return alt_value\n    attrs_str = None if attrs.get(attrs_name) is None else str(attrs.get(attrs_name))\n    if attrs_str is None:\n        return alt_value\n    attrs_match = tuple_re.search(attrs_str)\n    if attrs_match is not None:\n        if attrs_match.span() == (0, len(attrs_str)):\n            dims = eval(attrs_str)\n            return dims\n        else:\n            raise AttributeError(f\"Malformed {attrs_name} dimensions: {str(attrs_str)}\")\n    return alt_value\n\ndef transform_padding(pad_width):\n    \"\"\"Helper function to convert padding format for pad operator.\n    \"\"\"\n    num_pad_values = len(pad_width)\n    onnx_pad_width = [0]*num_pad_values\n\n    start_index = 0\n    # num_pad_values will always be multiple of 2\n    end_index = int(num_pad_values/2)\n    for idx in range(0, num_pad_values):\n        if idx % 2 == 0:\n            onnx_pad_width[start_index] = pad_width[idx]\n            start_index += 1\n        else:\n            onnx_pad_width[end_index] = pad_width[idx]\n            end_index += 1\n\n    return onnx_pad_width\n\n\ndef convert_string_to_list(string_val):\n    \"\"\"Helper function to convert string to list.\n     Used to convert shape attribute string to list format.\n    \"\"\"\n    result_list = []\n\n    list_string = string_val.split(',')\n    for val in list_string:\n        val = str(val.strip())\n        val = val.replace(\"(\", \"\")\n        val = val.replace(\")\", \"\")\n        val = val.replace(\"L\", \"\")\n        val = val.replace(\"[\", \"\")\n        val = val.replace(\"]\", \"\")\n        if val == \"None\":\n            result_list.append(None)\n        elif val != \"\":\n            result_list.append(int(val))\n\n    return result_list\n\ndef get_boolean_attribute_value(attrs, attr_name):\n    \"\"\" Helper function to convert a string version\n    of Boolean attributes to integer for ONNX.\n    Takes attribute dictionary and attr_name as\n    parameters.\n    \"\"\"\n    return 1 if attrs.get(attr_name, 0) in [\"True\", \"1\"] else 0\n\ndef get_inputs(node, kwargs):\n    \"\"\"Helper function to get inputs\"\"\"\n    name = node[\"name\"]\n    outputs_lookup = kwargs[\"outputs_lookup\"]\n    inputs = node[\"inputs\"]\n    attrs = node.get(\"attrs\", {})\n\n    input_nodes = []\n    for ip in inputs:\n        input_node_name = outputs_lookup[ip[0]][ip[1]].name\n        input_nodes.append(input_node_name)\n\n    return name, input_nodes, attrs\n\ndef get_input_dtypes(node, kwargs):\n    outputs_lookup = kwargs['outputs_lookup']\n    inputs = node['inputs']\n    input_dtypes = []\n    for ip in inputs:\n        input_node_dtype = outputs_lookup[ip[0]][ip[1]].dtype\n        input_dtypes.append(input_node_dtype)\n    return input_dtypes\n\ndef create_basic_op_node(op_name, node, kwargs):\n    \"\"\"Helper function to create a basic operator\n    node that doesn't contain op specific attrs\"\"\"\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    node = onnx.helper.make_node(\n        op_name,\n        input_nodes,\n        [name],\n        name=name\n    )\n    return [node]\n\ndef create_const_scalar_node(input_name, value, kwargs):\n    \"\"\"Helper function to create a tensor value node and a\n    initializer tensor node with constant value.\"\"\"\n    from onnx.helper import make_tensor\n    initializer = kwargs[\"initializer\"]\n    dtype = value.dtype\n    if dtype == 'float16':\n        # when using float16, we must convert it to np.uint16 view first\n        value = np.float16(value).view(np.uint16)\n    input_type = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n    tensor_node = make_tensor(input_name, input_type, (), ([value]))\n    initializer.append(tensor_node)\n\ndef create_const_node(input_name, value, kwargs):\n    \"\"\"Helper function to create a tensor value node and a\n    initializer tensor node with constant value.\"\"\"\n    from onnx.helper import make_tensor\n    initializer = kwargs[\"initializer\"]\n    dtype = value.dtype\n    if dtype == 'float16':\n        # when using float16, we must convert it to np.uint16 view first\n        value = np.float16(value).view(np.uint16)\n    input_type = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n    input_shape = value.shape\n    tensor_node = make_tensor(input_name, input_type, input_shape, value)\n    initializer.append(tensor_node)\n\ndef create_tensor(tensor_list, tensor_name, initializer, dtype='int64'):\n    \"\"\"Helper function to create a tensor value node and a\n    initializer tensor node with constant value.\"\"\"\n    tensor_np = np.array(tensor_list, dtype=dtype)\n    data_type = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[tensor_np.dtype]\n    dims = np.shape(tensor_np)\n    if dtype == np.float16:\n        tensor_np = tensor_np.view(dtype=np.uint16)\n    tensor = onnx.helper.make_tensor(\n        name=tensor_name,\n        data_type=data_type,\n        dims=dims,\n        vals=tensor_np.flatten().tolist(),\n        raw=False\n    )\n    initializer.append(tensor)\n\n\n@mx_op.register(\"null\")\ndef convert_weights_and_inputs(node, **kwargs):\n    \"\"\"Helper function to convert weights and inputs.\n    \"\"\"\n    name, _, _ = get_inputs(node, kwargs)\n    if kwargs[\"is_input\"] is False:\n        weights = kwargs[\"weights\"]\n        initializer = kwargs[\"initializer\"]\n        np_arr = weights[name]\n        data_type = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[np_arr.dtype]\n        dims = np.shape(np_arr)\n\n        tensor_node = onnx.helper.make_tensor_value_info(name, data_type, dims)\n\n        from onnx import numpy_helper\n        tensor = numpy_helper.from_array(np_arr, name=name)\n        initializer.append(tensor)\n\n        return [tensor_node], (np_arr.dtype,)\n    else:\n        dtype_t = kwargs[\"in_type\"]\n        dtype = onnx.mapping.TENSOR_TYPE_TO_NP_TYPE[dtype_t]\n        tval_node = onnx.helper.make_tensor_value_info(name, dtype_t, kwargs[\"in_shape\"])\n        return [tval_node], (dtype,)\n\n\n@mx_op.register('Convolution')\ndef convert_convolution(node, **kwargs):\n    \"\"\"Map MXNet's convolution operator attributes to onnx's Conv operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    kernel = convert_string_to_list(attrs.get('kernel', '()'))\n    stride = convert_string_to_list(attrs.get('stride', '()'))\n    dilate = convert_string_to_list(attrs.get('dilate', '()'))\n    pad = convert_string_to_list(attrs.get('pad', '()'))\n    num_group = int(attrs.get('num_group', 1))\n    no_bias = attrs.get('no_bias', 'False')\n    layout = attrs.get('layout', 'NCHW')\n\n    if layout not in ['NCHW', 'NCDHW']:\n        raise NotImplementedError('Convolution currently does not support layout not in '\n                                  '[\\'NCHW\\', \\'NCDHW\\']')\n\n    if no_bias in ['True', '1']:\n        assert len(input_nodes) == 2, 'Convolution takes 2 input if no_bias==True'\n    else:\n        assert len(input_nodes) == 3, 'Convolution takes 3 input if no_bias==False'\n\n    kwargs_ = {}\n    if kernel:\n        kwargs_['kernel_shape'] = tuple(kernel)\n    if pad:\n        kwargs_['pads'] = tuple(pad) + tuple(pad)\n    if stride:\n        kwargs_['strides'] = stride\n    if dilate:\n        kwargs_['dilations'] = dilate\n\n    nodes = [\n        make_node('Conv', input_nodes, [name], group=num_group, **kwargs_)\n    ]\n\n    return nodes\n\n\n@mx_op.register('Deconvolution')\ndef convert_deconvolution(node, **kwargs):\n    \"\"\"Map MXNet's deconvolution operator attributes to onnx's ConvTranspose operator\n    and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    kernel_shape = convert_string_to_list(attrs.get('kernel', '()'))\n    strides = convert_string_to_list(attrs.get('stride', '()'))\n    pads = convert_string_to_list(attrs.get('pad', '()'))\n    group = int(attrs.get(\"num_group\", 1))\n    dilations = convert_string_to_list(attrs.get('dilate', '()'))\n    output_padding = convert_string_to_list(attrs.get('adj', '()'))\n    layout = attrs.get('layout', 'NCHW')\n    target_shape = attrs.get('target_shape', '')\n    no_bias = attrs.get('no_bias', 'False')\n\n    pads = pads + pads\n\n    if target_shape not in ['', 'None']:\n        raise NotImplementedError('Deconvolution currently does not support target_shape')\n\n    if layout not in ['NCHW', 'NCDHW', 'NCW']:\n        raise NotImplementedError('Deconvolution currently does not support layout not in '\n                                  '[\\'NCHW\\', \\'NCDHW\\', \\'NCW\\']')\n\n    if no_bias in ['1', 'True']:\n        assert len(input_nodes) == 2, 'Deconvolution takes 2 input if no_bias==True'\n    else:\n        assert len(input_nodes) == 3, 'Deconvolution takes 3 input if no_bias==False'\n\n    kwargs_ = {}\n    if kernel_shape:\n        kwargs_['kernel_shape'] = kernel_shape\n    if pads:\n        kwargs_['pads'] = pads\n    if strides:\n        kwargs_['strides'] = strides\n    if dilations:\n        kwargs_['dilations'] = dilations\n    if output_padding:\n        kwargs_['output_padding'] = output_padding\n\n    deconv_node = onnx.helper.make_node(\n        \"ConvTranspose\",\n        inputs=input_nodes,\n        outputs=[name],\n        group=group,\n        **kwargs_\n    )\n\n    return [deconv_node]\n\n\n@mx_op.register('Crop')\ndef convert_crop(node, **kwargs):\n    \"\"\"Map MXNet's crop operator attributes to onnx's Slice operator\n    \"\"\"\n    from onnx.helper import make_node\n    name, inputs, attrs = get_inputs(node, kwargs)\n\n    num_inputs = len(inputs)\n    y, x = convert_string_to_list(attrs.get('offset', '(0, 0)')) # pylint: disable=unbalanced-tuple-unpacking\n    h, w = convert_string_to_list(attrs.get('h_w', '(0, 0)')) # pylint: disable=unbalanced-tuple-unpacking\n    center_crop = attrs.get('center_crop', 'False')\n\n    if center_crop in ['True', '1']:\n        raise NotImplementedError('Crop does not currently support center_crop==True')\n\n    nodes = []\n    create_tensor([y, x], name+'_starts', kwargs['initializer'])\n    create_tensor([2, 3], name+'_axes', kwargs['initializer'])\n    if num_inputs == 1:\n        create_tensor([y + h, x + w], name+'_ends', kwargs['initializer'])\n    else:\n        create_tensor([0], name+'_0', kwargs['initializer'])\n        create_tensor([2], name+'_2', kwargs['initializer'])\n        create_tensor([4], name+'_4', kwargs['initializer'])\n        nodes += [\n            make_node('Shape', [inputs[1]], [name+'_shape']),\n            make_node('Slice', [name+'_shape', name+'_2', name+'_4', name+'_0'], [name+'_h_w']),\n            make_node('Add', [name+'_starts', name+'_h_w'], [name+'_ends'])\n\n        ]\n    nodes += [\n        make_node('Slice', [inputs[0], name+'_starts', name+'_ends', name+'_axes'], [name])\n    ]\n\n    return nodes\n\n@mx_op.register(\"FullyConnected\")\ndef convert_fully_connected(node, **kwargs):\n    \"\"\"Map MXNet's FullyConnected operator attributes to onnx's Gemm operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n    flatten = get_boolean_attribute_value(attrs, 'flatten')\n    no_bias = get_boolean_attribute_value(attrs, 'no_bias')\n    num_hidden = int(attrs.get('num_hidden'))\n\n    nodes = []\n    if flatten:\n        nodes += [\n            make_node('Flatten', [input_nodes[0]], [name+'_data_flattened'])\n        ]\n    else:\n        nodes += [\n            make_node('Shape', [input_nodes[0]], [name+'_orig_shape']),\n            make_node('Shape', [name+'_orig_shape'], [name+'_dim']),\n            make_node('Flatten', [input_nodes[0]], [name+'_data_flattened'], axis=-1),\n        ]\n\n    in_nodes = [name+'_data_flattened', input_nodes[1]]\n\n    if no_bias:\n        create_const_scalar_node(name+'_bias', np.int32(0).astype(dtype), kwargs)\n        in_nodes.append(name+'_bias')\n    else:\n        in_nodes.append(input_nodes[2])\n\n    if flatten:\n        nodes += [\n            make_node('Gemm', in_nodes, [name], alpha=1.0, beta=1.0, transA=0, transB=1, name=name)\n        ]\n    else:\n        create_tensor([0], name+'_0', kwargs['initializer'])\n        create_tensor([1], name+'_1', kwargs['initializer'])\n        create_tensor([num_hidden], name+'_num_hidden', kwargs['initializer'])\n        nodes += [\n            make_node('Gemm', in_nodes, [name+'_gemm'], alpha=1.0, beta=1.0, transA=0, transB=1),\n            make_node('Sub', [name+'_dim', name+'_1'], [name+'dim_minus_1']),\n            make_node('Slice', [name+'_orig_shape', name+'_0', name+'dim_minus_1'],\n                      [name+'_shape_sliced']),\n            make_node('Concat', [name+'_shape_sliced', name+'_num_hidden'],\n                      [name+'_shape_new'], axis=0),\n            make_node('Reshape', [name+'_gemm', name+'_shape_new'], [name], name=name)\n        ]\n\n    return nodes\n\n\n@mx_op.register(\"BatchNorm\")\ndef convert_batchnorm(node, **kwargs):\n    \"\"\"Map MXNet's BatchNorm operator attributes to onnx's BatchNormalization operator\n    and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    momentum = float(attrs.get(\"momentum\", 0.9))\n    eps = float(attrs.get(\"eps\", 0.001))\n    axis = int(attrs.get(\"axis\", 1))\n\n    if axis != 1:\n        raise NotImplementedError(\"batchnorm axis != 1 is currently not supported.\")\n\n    bn_node = onnx.helper.make_node(\n        \"BatchNormalization\",\n        input_nodes,\n        [name],\n        name=name,\n        epsilon=eps,\n        momentum=momentum\n        # MXNet computes mean and variance per channel for batchnorm.\n        # Default for onnx is across all spatial features. Relying on default\n        # ONNX behavior of spatial=1 for ONNX opset 8 and below. As the spatial\n        # attribute is deprecated in opset 9 and above, not explicitly encoding it.\n    )\n    return [bn_node]\n\n\n@mx_op.register(\"tanh\")\n@mx_op.register(\"_npi_tanh\")\ndef convert_tanh(node, **kwargs):\n    \"\"\"Map MXNet's tanh operator attributes to onnx's Tanh operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Tanh', node, kwargs)\n\n@mx_op.register(\"cos\")\n@mx_op.register(\"_npi_cos\")\ndef convert_cos(node, **kwargs):\n    \"\"\"Map MXNet's cos operator attributes to onnx's Cos operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Cos', node, kwargs)\n\n@mx_op.register(\"sin\")\n@mx_op.register(\"_npi_sin\")\ndef convert_sin(node, **kwargs):\n    \"\"\"Map MXNet's sin operator attributes to onnx's Sin operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Sin', node, kwargs)\n\n@mx_op.register(\"tan\")\n@mx_op.register(\"_npi_tan\")\ndef convert_tan(node, **kwargs):\n    \"\"\"Map MXNet's tan operator attributes to onnx's tan operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Tan', node, kwargs)\n\n@mx_op.register(\"arccos\")\n@mx_op.register(\"_npi_arccos\")\ndef convert_acos(node, **kwargs):\n    \"\"\"Map MXNet's acos operator attributes to onnx's acos operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Acos', node, kwargs)\n\n@mx_op.register(\"arcsin\")\n@mx_op.register(\"_npi_arcsin\")\ndef convert_asin(node, **kwargs):\n    \"\"\"Map MXNet's asin operator attributes to onnx's asin operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Asin', node, kwargs)\n\n@mx_op.register(\"arctan\")\n@mx_op.register(\"_npi_arctan\")\ndef convert_atan(node, **kwargs):\n    \"\"\"Map MXNet's atan operator attributes to onnx's atan operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Atan', node, kwargs)\n\n#Basic neural network functions\n@mx_op.register(\"sigmoid\")\n@mx_op.register(\"_npx_sigmoid\")\ndef convert_sigmoid(node, **kwargs):\n    \"\"\"Map MXNet's sigmoid operator attributes to onnx's Sigmoid operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Sigmoid', node, kwargs)\n\n@mx_op.register(\"relu\")\n@mx_op.register(\"_npx_relu\")\ndef convert_relu(node, **kwargs):\n    \"\"\"Map MXNet's relu operator attributes to onnx's Relu operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Relu', node, kwargs)\n\n@mx_op.register(\"Activation\")\ndef convert_activation(node, **kwargs):\n    \"\"\"Map MXNet's Activation operator attributes to onnx's Tanh/Relu operator\n    and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    act_type = attrs[\"act_type\"]\n\n    # Creating a dictionary here, but if this titlecase pattern\n    # mxnet_name.title()\n    act_types = {\n        \"tanh\": \"Tanh\",\n        \"relu\": \"Relu\",\n        \"sigmoid\": \"Sigmoid\",\n        \"softrelu\": \"Softplus\",\n        \"softsign\": \"Softsign\"\n    }\n\n    act_name = act_types.get(act_type)\n    if act_name:\n        node = onnx.helper.make_node(\n            act_name,\n            input_nodes,\n            [name],\n            name=name\n        )\n    else:\n        raise AttributeError(\n            f\"Activation {act_type} not implemented or recognized in the converter\"\n        )\n\n    return [node]\n\n\n@mx_op.register(\"Pad\")\ndef convert_pad(node, **kwargs):\n    \"\"\"Map MXNet's pad operator attributes to onnx's Pad operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    opset_version = kwargs[\"opset_version\"]\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n\n    mxnet_pad_width = convert_string_to_list(attrs.get(\"pad_width\"))\n    onnx_pad_width = transform_padding(mxnet_pad_width)\n\n    pad_mode = attrs.get(\"mode\")\n    pad_value = float(attrs.get(\"constant_value\", 0.0))\n    pad_value = dtype.type(pad_value)\n\n    if opset_version >= 11:\n        # starting with opset 11, pads and constant_value are inputs instead of attributes\n        create_const_node(name+\"_pads\", np.array(onnx_pad_width, dtype='int64'), kwargs)\n        nodes = []\n        if pad_mode == \"constant\":\n            create_const_scalar_node(name+\"_const\", pad_value, kwargs)\n            nodes += [\n                make_node(\"Pad\", [input_nodes[0], name+\"_pads\", name+\"_const\"], [name], mode=pad_mode, name=name)\n            ]\n        else:\n            nodes += [\n                make_node(\"Pad\", [input_nodes[0], name+\"_pads\"], [name], mode=pad_mode, name=name)\n            ]\n        return nodes\n    else:\n        if pad_mode == \"constant\":\n            node = onnx.helper.make_node(\n                'Pad',\n                inputs=input_nodes,\n                outputs=[name],\n                mode='constant',\n                value=pad_value,\n                pads=onnx_pad_width,\n                name=name\n            )\n        else:\n            node = onnx.helper.make_node(\n                'Pad',\n                inputs=input_nodes,\n                outputs=[name],\n                mode=pad_mode,\n                pads=onnx_pad_width,\n                name=name\n            )\n        return [node]\n\n\ndef create_helper_trans_node(node_name, input_node):\n    \"\"\"create extra transpose node for dot operator\"\"\"\n    trans_node = onnx.helper.make_node(\n        'Transpose',\n        inputs=[input_node],\n        outputs=[node_name],\n        name=node_name\n    )\n    return trans_node\n\n\n# Note that due to ONNX limitation, the behavior for when inputs > 2-D is different from that of\n# MXNet\n@mx_op.register(\"dot\")\ndef convert_dot(node, **kwargs):\n    \"\"\"Map MXNet's dot operator attributes to onnx's\n    MatMul and Transpose operators based on the values set for\n    transpose_a, transpose_b attributes.\"\"\"\n    logging.warning('Converting dot operator... Please note that due to ONNX limitation, the '\n                    'behavior for when inputs > 2-D is different from that of MXNet dot.')\n\n    name, inputs, attrs = get_inputs(node, kwargs)\n    trans_a = get_boolean_attribute_value(attrs, \"transpose_a\")\n    trans_b = get_boolean_attribute_value(attrs, \"transpose_b\")\n\n    nodes = []\n    input_nodes = []\n    if trans_a:\n        nodes.append(create_helper_trans_node(name+\"_a\", inputs[0]))\n        input_nodes.append(name+\"_a\")\n    else:\n        input_nodes.append(inputs[0])\n\n    if trans_b:\n        nodes.append(create_helper_trans_node(name+\"_b\", inputs[1]))\n        input_nodes.append(name+\"_b\")\n    else:\n        input_nodes.append(inputs[1])\n\n    nodes.append(onnx.helper.make_node('MatMul', input_nodes, [name], name=name))\n    return nodes\n\n\ndef transpose_last_two_dim(name, kwargs):\n    \"\"\"Helper function to transpose the last two dims of the input tensor\n    \"\"\"\n    from onnx.helper import make_node\n    create_tensor([0], name+'_0', kwargs['initializer'])\n    create_tensor([1], name+'_1', kwargs['initializer'])\n    create_tensor([8], name+'_8', kwargs['initializer'])\n    perm = [i for i in range(8)]\n    perm[6], perm[7] = 7, 6\n    nodes = [\n        make_node('Shape', [name], [name+'_shape']),\n        make_node('Shape', [name+'_shape'], [name+'_dim']),\n        make_node('Sub', [name+'_8', name+'_dim'], [name+'_sub']),\n        make_node('Concat', [name+'_sub', name+'_0'], [name+'_concat'], axis=0),\n        make_node('Pad', [name+'_shape', name+'_concat', name+'_1'], [name+'_shape_8_dim']),\n        make_node('Reshape', [name, name+'_shape_8_dim'], [name+'_data_8_dim']),\n        make_node('Transpose', [name+'_data_8_dim'], [name+'_data_t'], perm=perm),\n        make_node('Shape', [name+'_data_t'], [name+'_new_shape_']),\n        make_node('Slice', [name+'_new_shape_', name+'_sub', name+'_8', name+'_0'],\n                  [name+'_new_shape']),\n        make_node('Reshape', [name+'_data_t', name+'_new_shape'], [name+'_transposed']),\n    ]\n\n    return nodes\n\n\n@mx_op.register(\"_linalg_gemm2\")\ndef convert_linalg_gemm2(node, **kwargs):\n    \"\"\"Map MXNet's _linalg_gemm2 operator attributes to onnx's\n    MatMul and Transpose operators based on the values set for\n    transpose_a, transpose_b attributes.\n    Return multiple nodes created.\n    \"\"\"\n    from onnx.helper import make_node\n    name, inputs, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n\n    # Getting the attributes and assigning default values.\n    alpha = float(attrs.get('alpha', 1.0))\n    axis = attrs.get('axis', 'None')\n    trans_a = get_boolean_attribute_value(attrs, 'transpose_a')\n    trans_b = get_boolean_attribute_value(attrs, 'transpose_b')\n\n    if axis != 'None':\n        raise NotImplementedError('_linalg_gemm2 does not currently support axis!=None')\n\n    nodes = []\n    input_nodes = []\n    if trans_a:\n        nodes += transpose_last_two_dim(inputs[0], kwargs)\n        input_nodes.append(inputs[0]+'_transposed')\n    else:\n        input_nodes.append(inputs[0])\n\n    if trans_b:\n        nodes += transpose_last_two_dim(inputs[1], kwargs)\n        input_nodes.append(inputs[1]+'_transposed')\n    else:\n        input_nodes.append(inputs[1])\n\n    if alpha == 1:\n        nodes += [\n            make_node('MatMul', input_nodes, [name])\n        ]\n        return nodes\n\n    create_const_scalar_node(name+\"_alpha\", dtype.type(alpha), kwargs)\n    nodes += [\n        make_node('MatMul', input_nodes, [name+'_matmul']),\n        make_node('Mul', [name+'_matmul', name+'_alpha'], [name])\n    ]\n    return nodes\n\n@mx_op.register('Pooling')\ndef convert_pooling(node, **kwargs):\n    \"\"\"Map MXNet's Pooling operator attributes to onnx's\n    MaxPool/AveragePool/GlobalMaxPool/GlobalAveragePool operators\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    kernel = convert_string_to_list(attrs.get('kernel', '()'))\n    pool_type = attrs.get('pool_type', 'max')\n    global_pool = attrs.get('global_pool', 'False')\n    global_pool = global_pool in ['True', '1']\n    _ = attrs.get('cudnn_off', 'False')\n    pooling_convention = attrs.get('pooling_convention', 'valid')\n    stride = convert_string_to_list(attrs.get('stride', '()'))\n    pad = convert_string_to_list(attrs.get('pad', '()'))\n    p_value = attrs.get('p_value', '0')\n    if p_value != 'None':\n        p_value = int(p_value)\n    count_include_pad = attrs.get('count_include_pad', 'True')\n    layout = attrs.get('layout', 'NCHW')\n\n    if pooling_convention == 'same':\n        raise NotImplementedError('Pooling currently does not support '\n                                  'pooling_convention==\\'same\\'')\n    if pool_type == 'sum':\n        raise NotImplementedError('Pooling currently does not support pool_type==\\'sum\\'')\n    if pool_type == 'lp' and not global_pool and pooling_convention != 'valid':\n        raise NotImplementedError('Pooling currently does not support '\n                                  'pooling_convention!=\\'valid\\' when pool_type==\\'lp\\' and global_pool==False')\n\n    if layout not in ['NCHW', 'NCDHW']:\n        raise NotImplementedError('Pooling currently does not support layout not in '\n                                  '[\\'NCHW\\', \\'NCDHW\\']')\n\n    kwargs_ = {}\n    if kernel:\n        kwargs_['kernel_shape'] = tuple(kernel)\n    if pad:\n        kwargs_['pads'] = tuple(pad) + tuple(pad)\n    if stride:\n        kwargs_['strides'] = stride\n\n    ceil_mode = 1 if pooling_convention == 'full' else 0\n    count_include_pad = 1 if count_include_pad == 'True' else 0\n\n    nodes = []\n    if pool_type == 'avg' and not global_pool:\n        nodes += [\n            make_node('AveragePool', [input_nodes[0]], [name], ceil_mode=ceil_mode,\n                      count_include_pad=count_include_pad, **kwargs_)\n        ]\n    elif pool_type == 'max' and not global_pool:\n        nodes += [\n            make_node('MaxPool', [input_nodes[0]], [name], ceil_mode=ceil_mode, **kwargs_)\n        ]\n    elif pool_type == 'lp' and not global_pool:\n        nodes += [\n            make_node('LpPool', [input_nodes[0]], [name], p=p_value, **kwargs_)\n        ]\n    elif pool_type == 'avg' and global_pool:\n        nodes += [\n            make_node('GlobalAveragePool', [input_nodes[0]], [name])\n        ]\n    elif pool_type == 'max' and global_pool:\n        nodes += [\n            make_node('GlobalMaxPool', [input_nodes[0]], [name])\n        ]\n    elif pool_type == 'lp' and global_pool:\n        nodes += [\n            make_node('GlobalLpPool', [input_nodes[0]], [name], p=p_value)\n        ]\n    else:\n        raise NotImplementedError('Unknown pool_type in Pooling')\n\n    return nodes\n\n\n@mx_op.register(\"exp\")\n@mx_op.register(\"_npi_exp\")\ndef convert_exp(node, **kwargs):\n    \"\"\"Map MXNet's exp operator attributes to onnx's Exp operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Exp', node, kwargs)\n\n@mx_op.register(\"_copy\")\ndef convert_copy(node, **kwargs):\n    \"\"\"Map MXNet's _copy operator attributes to onnx's Identity operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Identity', node, kwargs)\n\n@mx_op.register(\"identity\")\ndef convert_identity(node, **kwargs):\n    \"\"\"Map MXNet's identity operator attributes to onnx's Identity operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Identity', node, kwargs)\n\n@mx_op.register(\"InstanceNorm\")\ndef convert_instancenorm(node, **kwargs):\n    \"\"\"Map MXNet's InstanceNorm operator attributes to onnx's InstanceNormalization operator\n    based on the input node's attributes and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    eps = float(attrs.get(\"eps\", 0.001))\n\n    node = onnx.helper.make_node(\n        'InstanceNormalization',\n        inputs=input_nodes,\n        outputs=[name],\n        name=name,\n        epsilon=eps)\n\n    return [node]\n\n@mx_op.register(\"LeakyReLU\")\ndef convert_leakyrelu(node, **kwargs):\n    \"\"\"Map MXNet's LeakyReLU operator attributes to onnx's Elu/LeakyRelu/PRelu operators\n    based on the input node's attributes and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    act_type = attrs.get(\"act_type\", \"leaky\")\n    alpha = float(attrs.get(\"slope\", 0.25))\n\n    act_name = {\"elu\": \"Elu\", \"leaky\": \"LeakyRelu\", \"prelu\": \"PRelu\",\n                \"selu\": \"Selu\"}\n\n    if act_type in (\"prelu\", \"selu\"):\n        node = onnx.helper.make_node(\n            act_name[act_type],\n            inputs=input_nodes,\n            outputs=[name],\n            name=name)\n    elif act_type in ('gelu', 'gelu_erf'):\n        sqrt2 = np.float32(1.4142135623730951)\n        create_const_scalar_node(name+\"_sqrt2\", sqrt2, kwargs)\n        create_const_scalar_node(name+\"_one\", np.float32(1.0), kwargs)\n        create_const_scalar_node(name+\"_half\", np.float32(0.5), kwargs)\n        nodes = [\n            make_node(\"Div\", [input_nodes[0], name+\"_sqrt2\"], [name+\"_div0_out\"]),\n            make_node(\"Erf\", [name+\"_div0_out\"], [name+\"_erf0_out\"]),\n            make_node(\"Add\", [name+\"_erf0_out\", name+\"_one\"], [name+\"_add0_out\"]),\n            make_node(\"Mul\", [input_nodes[0], name+\"_add0_out\"], [name+\"_mul0_out\"]),\n            make_node(\"Mul\", [name+\"_mul0_out\", name+\"_half\"], [name], name=name)\n        ]\n        return nodes\n    else:\n        node = onnx.helper.make_node(\n            act_name[act_type],\n            inputs=input_nodes,\n            outputs=[name],\n            name=name,\n            alpha=alpha)\n\n    return [node]\n\n\n@mx_op.register(\"softmax\")\ndef convert_softmax(node, **kwargs):\n    \"\"\"Map MXNet's softmax operator attributes to onnx's Softmax operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    axis = int(attrs.get(\"axis\", -1))\n    temperature = str(attrs.get(\"temperature\", 'None'))\n    if temperature == 'None':\n        temperature = 1.\n    else:\n        temperature = float(temperature)\n\n    use_length = str(attrs.get(\"use_length\", 'None'))\n    use_length = use_length in ['1', 'True']\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n    data = input_nodes[0]\n\n    # use op set 11 ONNX Softmax\n    if axis == -1 and temperature == 1.:\n        nodes = []\n        if use_length:\n            # magic number, this is fp16 min\n            create_tensor([-65500.0], name+\"_mask_val\", kwargs[\"initializer\"], dtype=dtype)\n            create_tensor([1], name+\"_1\", kwargs[\"initializer\"])\n            create_const_scalar_node(name+\"_0_s\", np.int64(0), kwargs)\n            create_const_scalar_node(name+\"_1_s\", np.int64(1), kwargs)\n            nodes += [\n                make_node(\"Shape\", [data], [name+\"_shape\"]),\n                make_node(\"Shape\", [name+\"_shape\"], [name+\"_dim\"]),\n                make_node(\"Sub\", [name+\"_dim\", name+\"_1\"], [name+\"_dim_m1\"]),\n                make_node(\"Slice\", [name+\"_shape\", name+\"_dim_m1\", name+\"_dim\"],\n                          [name+\"_dim_last_\"]),\n                make_node(\"Squeeze\", [name+\"_dim_last_\"], [name+\"_dim_last\"], axes=[0]),\n                make_node(\"Range\", [name+\"_0_s\", name+\"_dim_last\", name+\"_1_s\"], [name+\"_range\"]),\n                make_node(\"Cast\", [input_nodes[1]], [name+\"_len\"], to=int(TensorProto.INT64)),\n                make_node(\"Unsqueeze\", [name+\"_len\"], [name+\"_len_unsqueezed\"], axes=[-1]),\n                make_node(\"Less\", [name+\"_range\", name+\"_len_unsqueezed\"], [name+\"_less\"]),\n                make_node(\"Where\", [name+'_less', data, name+\"_mask_val\"], [name+\"_data_masked\"])\n            ]\n            data = name+\"_data_masked\"\n\n        nodes += [\n            make_node(\"Softmax\", [data], [name], axis=-1)\n        ]\n\n        return nodes\n\n    create_tensor([temperature], name+\"_tmp\", kwargs[\"initializer\"], dtype=dtype)\n    nodes = [\n        make_node(\"Div\", [data, name+\"_tmp\"], [name+'_data']),\n        make_node(\"Exp\", [name+'_data'], [name+\"_exp_out\"]),\n        make_node(\"ReduceSum\", [name+\"_exp_out\"], [name+\"_rsum_out\"], axes=[axis], keepdims=1),\n    ]\n    if len(input_nodes) == 1:\n        nodes += [\n            make_node(\"Div\", [name+\"_exp_out\", name+\"_rsum_out\"], [name], name=name),\n        ]\n        return nodes\n    elif use_length:\n        length = input_nodes[1]\n\n        create_tensor([axis], name+\"_axis\", kwargs[\"initializer\"])\n        create_tensor([0], name+\"_0\", kwargs[\"initializer\"])\n        create_tensor([1], name+\"_1\", kwargs[\"initializer\"])\n        create_const_scalar_node(name+'_-1_s', np.int64(-1), kwargs)\n        create_const_scalar_node(name+'_0_s', np.int64(0), kwargs)\n        create_const_scalar_node(name+'_1_s', np.int64(1), kwargs)\n        nodes += [\n            # cast data type\n            make_node(\"Cast\", [length], [name+\"_length\"], to=int(TensorProto.INT64)),\n            make_node(\"Cast\", [name+\"_0\"], [name+\"_0_itype\"], to=dtype_t),\n            make_node(\"Cast\", [name+\"_1\"], [name+\"_1_itype\"], to=dtype_t),\n            # softmax output\n            make_node(\"Div\", [name+\"_exp_out\", name+\"_rsum_out\"], [name+\"_div1_out\"]),\n            # update axis\n            make_node(\"Shape\", [data], [name+\"_shape0_out\"]),\n            make_node(\"Shape\", [name+\"_shape0_out\"], [name+\"_in_dim\"]),\n            make_node(\"Add\", [name+\"_in_dim\", name+\"_axis\"], [name+\"_dim+axis\"]),\n            make_node(\"Less\", [name+\"_axis\", name+\"_0_s\"], [name+\"_less0_out\"]),\n            make_node(\"Where\", [name+\"_less0_out\", name+\"_dim+axis\", name+\"_axis\"], [name+\"_final_axis\"]),\n            # data mask\n            make_node(\"Add\", [name+\"_final_axis\", name+\"_1_s\"], [name+\"_final_axis+1\"]),\n            make_node(\"Slice\", [name+\"_shape0_out\", name+\"_final_axis\", name+\"_final_axis+1\"], [name+\"_axis_dim\"]),\n            make_node(\"Squeeze\", [name+\"_axis_dim\"], [name+\"_axis_dim_s\"], axes=[0]),\n            make_node(\"Range\", [name+\"_0_s\", name+\"_axis_dim_s\", name+\"_1_s\"], [name+\"_range0_out\"]),\n            # one hot for axis\n            make_node(\"Squeeze\", [name+\"_in_dim\"], [name+\"_in_dim_s\"], axes=[0]),\n            make_node(\"Range\", [name+\"_0_s\", name+\"_in_dim_s\", name+\"_1_s\"], [name+\"_range1_out\"]),\n            make_node(\"Equal\", [name+\"_range1_out\", name+\"_final_axis\"], [name+\"_equal_out\"]),\n            make_node(\"Cast\", [name+\"_equal_out\"], [name+\"_one_hot\"], to=int(TensorProto.INT64)),\n            # reshape data mask for less\n            make_node(\"Sub\", [name+\"_axis_dim_s\", name+\"_1_s\"], [name+\"_sub0_out\"]),\n            make_node(\"Mul\", [name+\"_one_hot\", name+\"_sub0_out\"], [name+\"_mul0_out\"]),\n            make_node(\"Add\", [name+\"_mul0_out\", name+\"_1_s\"], [name+\"_add0_out\"]),\n            make_node('Reshape', [name+\"_range0_out\", name+\"_add0_out\"], [name+\"_reshape0_out\"]),\n            # reshape length for less\n            make_node(\"Mul\", [name+\"_one_hot\", name+\"_-1_s\"], [name+\"_mul1_out\"]),\n            make_node(\"Add\", [name+\"_mul1_out\", name+\"_1_s\"], [name+\"_add1_out\"]),\n            make_node(\"Sub\", [name+\"_shape0_out\", name+\"_1_s\"], [name+\"_sub1_out\"]),\n            make_node(\"Mul\", [name+\"_add1_out\", name+\"_sub1_out\"], [name+\"_mul2_out\"]),\n            make_node(\"Add\", [name+\"_mul2_out\", name+\"_1_s\"], [name+\"_add2_out\"]),\n            make_node('Reshape', [name+\"_length\", name+\"_add2_out\"], [name+\"_reshape1_out\"]),\n            # mask output\n            make_node(\"Less\", [name+\"_reshape0_out\", name+\"_reshape1_out\"], [name+\"_less_out\"]),\n            make_node(\"Cast\", [name+\"_less_out\"], [name+\"_mask\"], to=dtype_t),\n            make_node(\"Mul\", [name+\"_div1_out\", name+\"_mask\"], [name+\"_mul3_out\"]),\n            make_node(\"ReduceSum\", [name+\"_mul3_out\"], [name+\"_rsum1_out\"], axes=[axis], keepdims=1),\n            make_node(\"Equal\", [name+\"_rsum1_out\", name+\"_0_itype\"], [name+\"_equal1_out\"]),\n            make_node(\"Where\", [name+\"_equal1_out\", name+\"_1_itype\", name+\"_rsum1_out\"], [name+\"_where_out\"]),\n            make_node(\"Div\", [name+\"_mul3_out\", name+\"_where_out\"], [name], name=name)\n        ]\n        return nodes\n\n    else:\n        raise NotImplementedError(\"use_length must be true when both data and length are paased in.\")\n\n# There's also mx.sym.softmax(), which doesn't do cross-entropy loss,\n# just softmax for inference - hence the name convert_softmax_output.\n@mx_op.register(\"SoftmaxOutput\")\ndef convert_softmax_output(node, **kwargs):\n    \"\"\"Map MXNet's SoftmaxOutput operator attributes to onnx's Softmax operator\n    and return the created node.\n    \"\"\"\n    name = node[\"name\"]\n\n    input1 = kwargs[\"outputs_lookup\"][node[\"inputs\"][0][0]][node[\"inputs\"][0][1]].name\n\n    softmax_node = onnx.helper.make_node(\n        \"Softmax\",\n        [input1],\n        [name],\n        axis=1,\n        name=name\n    )\n\n    return [softmax_node]\n\n@mx_op.register(\"LogisticRegressionOutput\")\ndef convert_logistic_regression_output(node, **kwargs):\n    \"\"\"Map MXNet's SoftmaxOutput operator attributes to onnx's Softmax operator\n    and return the created node.\n    \"\"\"\n    name = node[\"name\"]\n    input1 = kwargs[\"outputs_lookup\"][node[\"inputs\"][0][0]][node[\"inputs\"][0][1]].name\n\n    sigmoid_node = onnx.helper.make_node(\n        \"Sigmoid\",\n        [input1],\n        [name],\n        name=name\n    )\n    return [sigmoid_node]\n\n@mx_op.register(\"BlockGrad\")\ndef convert_blockgrad(node, **kwargs):\n    \"\"\" Skip operator  \"\"\"\n    return create_basic_op_node('Identity', node, kwargs)\n\n@mx_op.register(\"MakeLoss\")\ndef convert_makeloss(node, **kwargs):\n    \"\"\" Skip operator  \"\"\"\n    return create_basic_op_node('Identity', node, kwargs)\n\n@mx_op.register('Concat')\n@mx_op.register('_npi_concatenate')\ndef convert_concat(node, **kwargs):\n    \"\"\"Map MXNet's Concat operator attributes to onnx's Concat operator\n    and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    if 'dim' in attrs:\n        axis = int(attrs.get('dim', 1))\n    else:\n        axis = int(attrs.get('axis', 1))\n    concat_node = onnx.helper.make_node(\n        'Concat',\n        input_nodes,\n        [name],\n        axis=axis,\n        name=name\n    )\n    return [concat_node]\n\n\n@mx_op.register(\"transpose\")\n@mx_op.register('_npi_transpose')\ndef convert_transpose(node, **kwargs):\n    \"\"\"Map MXNet's transpose operator attributes to onnx's Transpose operator\n    and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    axes = attrs.get(\"axes\", ())\n    if axes == 'None':\n        axes = ()\n    if axes:\n        axes = tuple(map(int, re.findall(r'\\d+', axes)))\n\n        transpose_node = onnx.helper.make_node(\n            \"Transpose\",\n            input_nodes,\n            [name],\n            perm=axes,\n            name=name\n        )\n    else:\n        transpose_node = onnx.helper.make_node(\n            \"Transpose\",\n            input_nodes,\n            [name],\n            name=name\n        )\n\n    return [transpose_node]\n\n\n@mx_op.register(\"LRN\")\ndef convert_lrn(node, **kwargs):\n    \"\"\"Map MXNet's LRN operator attributes to onnx's LRN operator\n    and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    alpha = float(attrs.get(\"alpha\", 0.0001))\n    beta = float(attrs.get(\"beta\", 0.75))\n    bias = float(attrs.get(\"knorm\", 1.0))\n    size = int(attrs.get(\"nsize\"))\n\n    lrn_node = onnx.helper.make_node(\n        \"LRN\",\n        inputs=input_nodes,\n        outputs=[name],\n        name=name,\n        alpha=alpha,\n        beta=beta,\n        bias=bias,\n        size=size\n    )\n\n    return [lrn_node]\n\n\n@mx_op.register(\"L2Normalization\")\ndef convert_l2normalization(node, **kwargs):\n    \"\"\"Map MXNet's L2Normalization operator attributes to onnx's LpNormalization operator\n    and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mode = attrs.get(\"mode\", \"instance\")\n\n    if mode != \"channel\":\n        raise AttributeError(\"L2Normalization: ONNX currently supports channel mode only\")\n\n    l2norm_node = onnx.helper.make_node(\n        \"LpNormalization\",\n        input_nodes,\n        [name],\n        axis=1,  # channel only\n        name=name\n    )\n    return [l2norm_node]\n\n\n@mx_op.register(\"Dropout\")\ndef convert_dropout(node, **kwargs):\n    \"\"\"Map MXNet's Dropout operator attributes to onnx's Dropout operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    _ = float(attrs.get(\"p\", 0.5))\n    _ = convert_string_to_list(attrs.get(\"axes\", \"None\"))\n    mode = attrs.get('mode', 'training')\n\n    if mode != 'training':\n        raise NotImplementedError(\"Dropout does not currently support mode!=\\'training\\'\")\n\n    nodes = [\n        make_node('Identity', [input_nodes[0]], [name])\n    ]\n\n    return nodes\n\n\n@mx_op.register(\"Flatten\")\ndef convert_flatten(node, **kwargs):\n    \"\"\"Map MXNet's Flatten operator attributes to onnx's Flatten operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Flatten', node, kwargs)\n\n@mx_op.register(\"clip\")\ndef convert_clip(node, **kwargs):\n    \"\"\"Map MXNet's Clip operator attributes to onnx's Clip operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    opset_version = kwargs[\"opset_version\"]\n\n    a_min = float(attrs.get('a_min', -np.inf))\n    a_max = float(attrs.get('a_max', np.inf))\n\n    if opset_version >= 11:\n        # opset >= 11 requires min/max to be inputs\n        input_dtype = get_input_dtypes(node, kwargs)[0]\n        create_const_scalar_node(name+\"_min\", np.float32(a_min).astype(input_dtype), kwargs)\n        create_const_scalar_node(name+\"_max\", np.float32(a_max).astype(input_dtype), kwargs)\n        nodes = [\n            make_node(\"Clip\", [input_nodes[0], name+\"_min\", name+\"_max\"], [name], name=name)\n        ]\n    else:\n        nodes = [\n            make_node(\"Clip\", input_nodes, [name], name=name, min=a_min, max=a_max)\n        ]\n    return nodes\n\n\ndef scalar_op_helper(node, op_name, reverse=False, **kwargs):\n    \"\"\"Helper function for scalar arithmetic operations\"\"\"\n    from onnx import numpy_helper\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n    scalar_value = float(attrs.get('scalar', '1'))\n    if str(dtype).startswith('int'):\n        # This irregular dtype inference is made to be consistent with MXNet 2.0 behavior\n        is_int = attrs.get('is_int', '1')\n        if is_int in ['0', 'False']:\n            if op_name == 'Div':\n                dtype = np.dtype('float32')\n            else:\n                dtype = np.dtype('float64')\n            dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n        else:\n            scalar_value = int(scalar_value)\n    else:\n        if dtype == 'float16':\n            # when using float16, we must convert it to np.uint16 view first\n            scalar_value = np.float16(scalar_value).view(np.uint16)\n    scalar_value = [scalar_value]\n\n    initializer = kwargs[\"initializer\"]\n    flag = True\n    # If the input value is in initializer, just multiply with scalar input\n    # and create a new initializer\n    for i in initializer:\n        if i.name == input_nodes[0]:\n            if op_name == 'Mul':\n                new_initializer = numpy_helper.to_array(i) * scalar_value[0]\n            elif op_name == 'Sub':\n                if reverse:\n                    new_initializer = scalar_value[0] - numpy_helper.to_array(i)\n                else:\n                    new_initializer = numpy_helper.to_array(i) - scalar_value[0]\n            elif op_name == 'Add':\n                new_initializer = numpy_helper.to_array(i) + scalar_value[0]\n            elif op_name == 'Div':\n                if reverse:\n                    new_initializer = scalar_value[0] / numpy_helper.to_array(i)\n                else:\n                    new_initializer = numpy_helper.to_array(i) / scalar_value[0]\n            elif op_name == 'Pow':\n                new_initializer = numpy_helper.to_array(i) ** scalar_value[0]\n            flag = False\n            break\n\n    # else create a new tensor of the scalar value, add it in initializer\n    if flag is True:\n        nodes = []\n        if input_dtypes[0] != dtype:\n            nodes += [\n                make_node('Cast', [input_nodes[0]], [name+'_cast'], to=dtype_t)\n            ]\n            input_nodes[0] = name+'_cast'\n\n        dims = np.shape(scalar_value)\n        scalar_op_name = \"scalar_op\" + str(kwargs[\"idx\"])\n        tensor_node = onnx.helper.make_tensor_value_info(scalar_op_name, dtype_t, dims)\n        print('in op trans', scalar_value)\n        initializer.append(\n            onnx.helper.make_tensor(\n                name=scalar_op_name,\n                data_type=dtype_t,\n                dims=dims,\n                vals=scalar_value,\n                raw=False,\n            )\n        )\n        # reverse op\n        if reverse:\n            nodes += [\n                make_node(op_name, [scalar_op_name, input_nodes[0]], [name])\n            ]\n        else:\n            nodes += [\n                make_node(op_name, [input_nodes[0], scalar_op_name], [name])\n            ]\n        return nodes, (dtype,)\n    else:\n        dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[new_initializer.dtype]\n        dims = np.shape(new_initializer)\n\n        tensor_node = onnx.helper.make_tensor_value_info(name, dtype_t, dims)\n\n        initializer.append(\n            onnx.helper.make_tensor(\n                name=name,\n                data_type=dtype_t,\n                dims=dims,\n                vals=new_initializer.flatten(),\n                raw=False,\n            )\n        )\n        return [tensor_node], (dtype,)\n\n\n# Convert scalar value into node and pass it as input to mul_node\n@mx_op.register(\"_mul_scalar\")\n@mx_op.register(\"_npi_multiply_scalar\")\ndef convert_mul_scalar(node, **kwargs):\n    \"\"\"Map MXNet's _mul_scalar operator attributes to onnx's Mul operator.\n    Creates a new node for the input scalar value, adds it to the initializer\n    and return multiple created nodes.\n    \"\"\"\n    return scalar_op_helper(node, 'Mul', **kwargs)\n\n\n# Convert scalar value into node and pass it as input to mul_node\n@mx_op.register(\"_minus_scalar\")\n@mx_op.register(\"_npi_subtract_scalar\")\ndef convert_minus_scalar(node, **kwargs):\n    \"\"\"Map MXNet's _minus_scalar operator attributes to onnx's Minus operator.\n    Creates a new node for the input scalar value, adds it to the initializer\n    and return multiple created nodes.\n    \"\"\"\n    return scalar_op_helper(node, 'Sub', **kwargs)\n\n@mx_op.register(\"_rminus_scalar\")\n@mx_op.register(\"_npi_rsubtract_scalar\")\ndef convert_rminus_scalar(node, **kwargs):\n    \"\"\"Map MXNet's _rminus_scalar operator attributes to onnx's Sub operator.\n    Creates a new node for the input scalar value, adds it to the initializer\n    and return multiple created nodes.\n    \"\"\"\n    return scalar_op_helper(node, 'Sub', reverse=True, **kwargs)\n\n# Convert scalar value into node and pass it as input to mul_node\n@mx_op.register(\"_plus_scalar\")\n@mx_op.register(\"_npi_add_scalar\")\ndef convert_add_scalar(node, **kwargs):\n    \"\"\"Map MXNet's _plus_scalar operator attributes to onnx's Add operator.\n    Creates a new node for the input scalar value, adds it to the initializer\n    and return multiple created nodes.\n    \"\"\"\n    return scalar_op_helper(node, 'Add', **kwargs)\n\n# Convert scalar value into node and pass it as input to mul_node\n@mx_op.register(\"_div_scalar\")\n@mx_op.register(\"_npi_true_divide_scalar\")\ndef convert_div_scalar(node, **kwargs):\n    \"\"\"Map MXNet's _div_scalar operator attributes to onnx's Div operator.\n    Creates a new node for the input scalar value, adds it to the initializer\n    and return multiple created nodes.\n    \"\"\"\n    return scalar_op_helper(node, 'Div', **kwargs)\n\n@mx_op.register(\"_rdiv_scalar\")\n@mx_op.register(\"_npi_rtrue_divide_scalar\")\ndef convert_rdiv_scalar(node, **kwargs):\n    \"\"\"Map MXNet's _rdiv_scalar operator attributes to onnx's Div operator.\n    Creates a new node for the input scalar value, adds it to the initializer\n    and return multiple created nodes.\n    \"\"\"\n    return scalar_op_helper(node, 'Div', reverse=True, **kwargs)\n\n@mx_op.register(\"_power_scalar\")\n@mx_op.register(\"_npi_power_scalar\")\ndef convert_pow_scalar(node, **kwargs):\n    \"\"\"Map MXNet's _pow_scalar operator attributes to onnx's Pow operator.\n    Creates a new node for the input scalar value, adds it to the initializer\n    and return multiple created nodes.\n    \"\"\"\n    return scalar_op_helper(node, 'Pow', **kwargs)\n\n# Sorting and Searching\n@mx_op.register(\"argmax\")\ndef convert_argmax(node, **kwargs):\n    \"\"\"Map MXNet's argmax operator attributes to onnx's ArgMax operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    axis = str(attrs.get('axis', 'None'))\n    keepdims = get_boolean_attribute_value(attrs, 'keepdims')\n\n    input_dtype = get_input_dtypes(node, kwargs)[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[input_dtype]\n\n    if axis == 'None':\n        create_tensor([-1], name+'_-1', kwargs['initializer'])\n        if keepdims:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('Shape', [input_nodes[0]], [name+'_shape']),\n                make_node('Shape', [name+'_shape'], [name+'_dim']),\n                make_node('Tile', [name+'_1', name+'_dim'], [name+'_tile']),\n                make_node('Reshape', [input_nodes[0], name+'_-1'], [name+'_reshape']),\n                make_node('ArgMax', [name+'_reshape'], [name+'_argmax'], axis=0, keepdims=True,),\n                make_node('Reshape', [name+'_argmax', name+'_tile'], [name+'_ret']),\n                make_node('Cast', [name+'_ret'], [name], to=dtype_t, name=name)\n            ]\n        else:\n            nodes = [\n                make_node('Reshape', [input_nodes[0], name+'_-1'], [name+'_reshape']),\n                make_node('ArgMax', [name+'_reshape'], [name+'_argmax'], axis=0, keepdims=True,),\n                make_node('Cast', [name+'_argmax'], [name], to=dtype_t, name=name)\n            ]\n    else:\n        axis = int(axis)\n        nodes = [\n            make_node('ArgMax', [input_nodes[0]], [name+'_argmax'], axis=axis, keepdims=keepdims,),\n            make_node('Cast', [name+'_argmax'], [name], to=dtype_t, name=name)\n        ]\n    return nodes\n\n\n@mx_op.register(\"argmin\")\ndef convert_argmin(node, **kwargs):\n    \"\"\"Map MXNet's argmin operator attributes to onnx's ArgMin operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    axis = str(attrs.get('axis', 'None'))\n    keepdims = get_boolean_attribute_value(attrs, 'keepdims')\n\n    input_dtype = get_input_dtypes(node, kwargs)[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[input_dtype]\n\n    if axis == 'None':\n        create_tensor([-1], name+'_-1', kwargs['initializer'])\n        if keepdims:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('Shape', [input_nodes[0]], [name+'_shape']),\n                make_node('Shape', [name+'_shape'], [name+'_dim']),\n                make_node('Tile', [name+'_1', name+'_dim'], [name+'_tile']),\n                make_node('Reshape', [input_nodes[0], name+'_-1'], [name+'_reshape']),\n                make_node('ArgMin', [name+'_reshape'], [name+'_argmin'], axis=0, keepdims=True,),\n                make_node('Reshape', [name+'_argmin', name+'_tile'], [name+'_ret']),\n                make_node('Cast', [name+'_ret'], [name], to=dtype_t, name=name)\n            ]\n        else:\n            nodes = [\n                make_node('Reshape', [input_nodes[0], name+'_-1'], [name+'_reshape']),\n                make_node('ArgMin', [name+'_reshape'], [name+'_argmin'], axis=0, keepdims=True,),\n                make_node('Cast', [name+'_argmin'], [name], to=dtype_t, name=name)\n            ]\n    else:\n        axis = int(axis)\n        nodes = [\n            make_node('ArgMin', [input_nodes[0]], [name+'_argmin'], axis=axis, keepdims=keepdims,),\n            make_node('Cast', [name+'_argmin'], [name], to=dtype_t, name=name)\n        ]\n    return nodes\n\n@mx_op.register(\"_maximum\")\ndef convert_maximum(node, **kwargs):\n    \"\"\"Map MXNet's _maximum operator attributes to onnx's Max operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Max', node, kwargs)\n\n\n@mx_op.register(\"_minimum\")\ndef convert_minimum(node, **kwargs):\n    \"\"\"Map MXNet's _minimum operator attributes to onnx's Min operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Min', node, kwargs)\n\n@mx_op.register(\"min\")\n@mx_op.register(\"_npi_min\")\ndef convert_min(node, **kwargs):\n    \"\"\"Map MXNet's min operator attributes to onnx's ReduceMin operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = str(attrs.get(\"axis\", 'None'))\n    axes = convert_string_to_list(mx_axis) if mx_axis != 'None' else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n\n    if axes is not None:\n        if keepdims:\n            node = make_node('ReduceMin', input_nodes, [name], axes=axes, keepdims=keepdims)\n            return [node]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceMin', input_nodes, [name+'_reduce'], axes=axes, keepdims=keepdims),\n                make_node('Shape', [name+'_reduce'], [name+'_reduce_shape']),\n                make_node('Concat', [name+'_1', name+'_reduce_shape'], [name+'_concat'], axis=0),\n                make_node('Reshape', [name+'_reduce', name+'_concat'], [name+'_reshape']),\n                make_node('Squeeze', [name+'_reshape'], [name], axes=[0]),\n            ]\n            return nodes\n    else:\n        if keepdims:\n            node = make_node('ReduceMin', input_nodes, [name], keepdims=keepdims)\n            return [node]\n\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceMin', input_nodes, [name+'_rmin'], keepdims=keepdims),\n                make_node('Reshape', [name+'_rmin', name+'_1'], [name])\n            ]\n            return nodes\n\n\n@mx_op.register(\"max\")\n@mx_op.register(\"_npi_max\")\ndef convert_max(node, **kwargs):\n    \"\"\"Map MXNet's max operator attributes to onnx's ReduceMax operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = str(attrs.get(\"axis\", 'None'))\n    axes = convert_string_to_list(mx_axis) if mx_axis != 'None' else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n\n    if axes is not None:\n        if keepdims:\n            node = make_node('ReduceMax', input_nodes, [name], axes=axes, keepdims=keepdims)\n            return [node]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceMax', input_nodes, [name+'_reduce'], axes=axes, keepdims=keepdims),\n                make_node('Shape', [name+'_reduce'], [name+'_reduce_shape']),\n                make_node('Concat', [name+'_1', name+'_reduce_shape'], [name+'_concat'], axis=0),\n                make_node('Reshape', [name+'_reduce', name+'_concat'], [name+'_reshape']),\n                make_node('Squeeze', [name+'_reshape'], [name], axes=[0]),\n            ]\n            return nodes\n    else:\n        if keepdims:\n            node = make_node('ReduceMax', input_nodes, [name], keepdims=keepdims)\n            return [node]\n\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceMax', input_nodes, [name+'_rmax'], keepdims=keepdims),\n                make_node('Reshape', [name+'_rmax', name+'_1'], [name])\n            ]\n            return nodes\n\n\n@mx_op.register(\"mean\")\ndef convert_mean(node, **kwargs):\n    \"\"\"Map MXNet's mean operator attributes to onnx's ReduceMean operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = str(attrs.get(\"axis\", 'None'))\n    axes = convert_string_to_list(mx_axis) if mx_axis != 'None' else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n\n    if axes is not None:\n        if keepdims:\n            node = make_node('ReduceMean', input_nodes, [name], axes=axes, keepdims=keepdims)\n            return [node]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceMean', input_nodes, [name+'_reduce'], axes=axes, keepdims=keepdims),\n                make_node('Shape', [name+'_reduce'], [name+'_reduce_shape']),\n                make_node('Concat', [name+'_1', name+'_reduce_shape'], [name+'_concat'], axis=0),\n                make_node('Reshape', [name+'_reduce', name+'_concat'], [name+'_reshape']),\n                make_node('Squeeze', [name+'_reshape'], [name], axes=[0]),\n            ]\n            return nodes\n    else:\n        if keepdims:\n            node = make_node('ReduceMean', input_nodes, [name], keepdims=keepdims)\n            return [node]\n\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceMean', input_nodes, [name+'_reduce'], keepdims=keepdims),\n                make_node('Reshape', [name+'_reduce', name+'_1'], [name])\n            ]\n            return nodes\n\n\n@mx_op.register(\"prod\")\n@mx_op.register(\"_npi_prod\")\ndef convert_prod(node, **kwargs):\n    \"\"\"Map MXNet's prod operator attributes to onnx's ReduceProd operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = str(attrs.get(\"axis\", 'None'))\n    axes = convert_string_to_list(mx_axis) if mx_axis != 'None' else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n\n    if axes is not None:\n        if keepdims:\n            node = make_node('ReduceProd', input_nodes, [name], axes=axes, keepdims=keepdims)\n            return [node]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceProd', input_nodes, [name+'_reduce'], axes=axes, keepdims=keepdims),\n                make_node('Shape', [name+'_reduce'], [name+'_reduce_shape']),\n                make_node('Concat', [name+'_1', name+'_reduce_shape'], [name+'_concat'], axis=0),\n                make_node('Reshape', [name+'_reduce', name+'_concat'], [name+'_reshape']),\n                make_node('Squeeze', [name+'_reshape'], [name], axes=[0]),\n            ]\n            return nodes\n    else:\n        if keepdims:\n            node = make_node('ReduceProd', input_nodes, [name], keepdims=keepdims)\n            return [node]\n\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceProd', input_nodes, [name+'_reduce'], keepdims=keepdims),\n                make_node('Reshape', [name+'_reduce', name+'_1'], [name])\n            ]\n            return nodes\n\n\n# Arithmetic Operations\n@mx_op.register(\"elemwise_add\")\ndef convert_elementwise_add(node, **kwargs):\n    \"\"\"Map MXNet's elemwise_add operator attributes to onnx's Add operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Add', node, kwargs)\n\n\n@mx_op.register(\"broadcast_add\")\n@mx_op.register(\"_npi_add\")\ndef covert_broadcast_add(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_add operator attributes to onnx's Add operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Add', node, kwargs)\n\n\n@mx_op.register(\"elemwise_sub\")\n@mx_op.register(\"_npi_subtract\")\ndef convert_elementwise_sub(node, **kwargs):\n    \"\"\"Map MXNet's elemwise_sub operator attributes to onnx's Sub operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Sub', node, kwargs)\n\n@mx_op.register(\"broadcast_sub\")\ndef covert_broadcast_sub(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_sub operator attributes to onnx's Sub operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Sub', node, kwargs)\n\n@mx_op.register(\"elemwise_mul\")\n@mx_op.register(\"_npi_multiply\")\ndef convert_elemwise_mul(node, **kwargs):\n    \"\"\"Map MXNet's elemwise_mul operator attributes to onnx's Mul operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Mul', node, kwargs)\n\n@mx_op.register(\"broadcast_mul\")\ndef convert_broadcast_mul(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_mul operator attributes to onnx's Mul operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Mul', node, kwargs)\n\n@mx_op.register(\"broadcast_minimum\")\ndef convert_broadcast_min(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_minimum operator attributes to onnx's Min operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Min', node, kwargs)\n\n\n@mx_op.register(\"broadcast_maximum\")\ndef convert_broadcast_max(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_maximum operator attributes to onnx's Min operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Max', node, kwargs)\n\n\n@mx_op.register(\"elemwise_div\")\ndef convert_elemwise_div(node, **kwargs):\n    \"\"\"Map MXNet's elemwise_div operator attributes to onnx's Div operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Div', node, kwargs)\n\n@mx_op.register(\"broadcast_div\")\ndef convert_broadcast_div(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_div operator attributes to onnx's Div operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Div', node, kwargs)\n\n@mx_op.register(\"negative\")\n@mx_op.register(\"_npi_negative\")\ndef convert_negative(node, **kwargs):\n    \"\"\"Map MXNet's negative operator attributes to onnx's Neg operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Neg', node, kwargs)\n\n@mx_op.register(\"abs\")\n@mx_op.register(\"_npi_absolute\")\ndef convert_abs(node, **kwargs):\n    \"\"\"Map MXNet's abs operator attributes to onnx's Abs operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Abs', node, kwargs)\n\n@mx_op.register(\"add_n\")\ndef convert_addn(node, **kwargs):\n    \"\"\"Map MXNet's add_n operator attributes to onnx's Sum operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Sum', node, kwargs)\n\n # Rounding\n@mx_op.register(\"ceil\")\n@mx_op.register(\"_npi_ceil\")\ndef convert_ceil(node, **kwargs):\n    \"\"\"Map MXNet's ceil operator attributes to onnx's Ceil operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Ceil', node, kwargs)\n\n@mx_op.register(\"floor\")\n@mx_op.register(\"_npi_floor\")\ndef convert_floor(node, **kwargs):\n    \"\"\"Map MXNet's floor operator attributes to onnx's Floor operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Floor', node, kwargs)\n\n\n@mx_op.register(\"_npx_reshape\")\ndef convert_npx_reshape(node, **kwargs):\n    \"\"\" reshape\n    \"\"\"\n    from onnx.helper import make_node\n\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    reverse = attrs.get('reverse', 'False')\n    targ_shape = convert_string_to_list(attrs['newshape'])\n\n    if reverse in ['True', '1']:\n        raise NotImplementedError('conversion of _npx_reshape with reverse==True is not '\\\n                                  'implemented yet')\n\n    if [x for x in targ_shape if x in [0, -2, -3, -4, -5, -6]] != []:\n        raise NotImplementedError('conversion of _npx_reshape with 0, -2, -3, -4, -5, -6 is not '\\\n                                  'implemented yet')\n\n    create_tensor(targ_shape, name+'_targ_shape', kwargs['initializer'])\n\n    nodes = []\n    nodes += [\n        make_node('Reshape', [input_nodes[0], name+'_targ_shape'], [name])\n    ]\n\n    return nodes\n\n\n# Legacy Reshape\n@mx_op.register(\"Reshape\")\ndef convert_reshape(node, **kwargs):\n    \"\"\"Map MXNet's Reshape operator attributes to onnx's Reshape operator.\n    Converts output shape attribute to output shape tensor\n    and return multiple created nodes.\n    \"\"\"\n    from onnx.helper import make_node\n\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    reverse = attrs.get('reverse', 'False')\n    targ_shape = convert_string_to_list(attrs[\"shape\"])\n    # In general -2, -3, -4 in the target shape are not supoorted, but there are\n    # a few special cases that we can convert to supported scenarios\n\n    # If -2 and -3 are not used and there is no 0 to the right of -4, then we can just remove -4\n    if -4 in targ_shape and -3 not in targ_shape and -2 not in targ_shape and reverse != 'True':\n        if 0 not in targ_shape:\n            targ_shape = [i for i in targ_shape if i != -4]\n        else:\n            # index of first -4\n            ind_4 = targ_shape.index(-4)\n            # index of last 0\n            ind0 = len(targ_shape) - 1 - targ_shape[::-1].index(0)\n            if ind_4 > ind0:\n                targ_shape = [i for i in targ_shape if i != -4]\n\n    if targ_shape == [-3, 0] and reverse != 'True':\n        targ_shape = [-1, 0]\n        reverse = 'True'\n\n    special_case = False\n    if targ_shape == [0, 0, -3, -3] and reverse != 'True':\n        special_case = True\n        nodes = [\n            make_node('Shape', [input_nodes[0]], [name+'_shape']),\n            make_node('Split', [name+'_shape'], [name+'_dim0', name+'_dim1', name+'_dim2',\n                                                 name+'_dim3', name+'_dim4', name+'_dim5'],\n                      axis=0),\n            make_node('Mul', [name+'_dim2', name+'_dim3'], [name+'_mul_1']),\n            make_node('Mul', [name+'_dim4', name+'_dim5'], [name+'_mul_2']),\n            make_node('Concat', [name+'_dim0', name+'_dim1', name+'_mul_1', name+'_mul_2'],\n                      [name+'_shape_new'], axis=0),\n            make_node('Reshape', [input_nodes[0], name+'_shape_new'], [name], name=name)\n        ]\n\n    if targ_shape == [0, -4, -1, 4, 0, 0] and reverse != 'True':\n        special_case = True\n        create_tensor([4], name+'_4', kwargs['initializer'])\n        nodes = [\n            make_node('Shape', [input_nodes[0]], [name+'_shape']),\n            make_node('Split', [name+'_shape'], [name+'_dim0', name+'_dim1', name+'_dim2',\n                                                 name+'_dim3'], axis=0),\n            make_node('Div', [name+'_dim1', name+'_4'], [name+'_div']),\n            make_node('Concat', [name+'_dim0', name+'_div', name+'_4', name+'_dim2', name+'_dim3'],\n                      [name+'_shape_new'], axis=0),\n            make_node('Reshape', [input_nodes[0], name+'_shape_new'], [name], name=name)\n        ]\n\n    if targ_shape == [0, 0, -4, 2, 2, 0, 0] and reverse != 'True':\n        special_case = True\n        create_tensor([2], name+'_2', kwargs['initializer'])\n        nodes = [\n            make_node('Shape', [input_nodes[0]], [name+'_shape']),\n            make_node('Split', [name+'_shape'], [name+'_dim0', name+'_dim1', name+'_dim2',\n                                                 name+'_dim3', name+'_dim4'], axis=0),\n            make_node('Concat', [name+'_dim0', name+'_dim1', name+'_2', name+'_2',\n                                 name+'_dim3', name+'_dim4'], [name+'_shape_new'], axis=0),\n            make_node('Reshape', [input_nodes[0], name+'_shape_new'], [name], name=name)\n        ]\n\n    if targ_shape == [-4, 1, -1, 0, 0, 0] and reverse != 'True':\n        special_case = True\n        create_tensor([1], name+'_1', kwargs['initializer'])\n        create_tensor([-1], name+'_m1', kwargs['initializer'])\n        nodes = [\n            make_node('Shape', [input_nodes[0]], [name+'_shape']),\n            make_node('Split', [name+'_shape'], [name+'_dim0', name+'_dim1', name+'_dim2',\n                                                 name+'_dim3'], axis=0),\n            make_node('Concat', [name+'_1', name+'_m1', name+'_dim1', name+'_dim2', name+'_dim3'],\n                      [name+'_shape_new'], axis=0),\n            make_node('Reshape', [input_nodes[0], name+'_shape_new'], [name], name=name)\n        ]\n\n    if targ_shape == [-4, 1, 1000, 0, 0] and reverse != 'True':\n        special_case = True\n        create_tensor([1], name+'_1', kwargs['initializer'])\n        create_tensor([1000], name+'_1000', kwargs['initializer'])\n        nodes = [\n            make_node('Shape', [input_nodes[0]], [name+'_shape']),\n            make_node('Split', [name+'_shape'], [name+'_dim0', name+'_dim1', name+'_dim2'], axis=0),\n            make_node('Concat', [name+'_1', name+'_1000', name+'_dim1', name+'_dim2'],\n                      [name+'_shape_new'], axis=0),\n            make_node('Reshape', [input_nodes[0], name+'_shape_new'], [name], name=name)\n        ]\n\n    if targ_shape == [0, -4, 12, -1, 0] and reverse != 'True':\n        special_case = True\n        create_tensor([-1], name+'_m1', kwargs['initializer'])\n        create_tensor([12], name+'_12', kwargs['initializer'])\n        nodes = [\n            make_node('Shape', [input_nodes[0]], [name+'_shape']),\n            make_node('Split', [name+'_shape'], [name+'_dim0', name+'_dim1', name+'_dim2'], axis=0),\n            make_node('Concat', [name+'_dim0', name+'_12', name+'_m1', name+'_dim2'],\n                      [name+'_shape_new'], axis=0),\n            make_node('Reshape', [input_nodes[0], name+'_shape_new'], [name], name=name)\n        ]\n\n    if targ_shape == [0, -4, 16, -1, 0] and reverse != 'True':\n        special_case = True\n        create_tensor([-1], name+'_m1', kwargs['initializer'])\n        create_tensor([16], name+'_16', kwargs['initializer'])\n        nodes = [\n            make_node('Shape', [input_nodes[0]], [name+'_shape']),\n            make_node('Split', [name+'_shape'], [name+'_dim0', name+'_dim1', name+'_dim2'], axis=0),\n            make_node('Concat', [name+'_dim0', name+'_16', name+'_m1', name+'_dim2'],\n                      [name+'_shape_new'], axis=0),\n            make_node('Reshape', [input_nodes[0], name+'_shape_new'], [name], name=name)\n        ]\n\n    if targ_shape == [-3, -1] and reverse != 'True':\n        special_case = True\n        create_tensor([0], name+'_0', kwargs['initializer'])\n        create_tensor([1], name+'_1', kwargs['initializer'])\n        create_tensor([2], name+'_2', kwargs['initializer'])\n        create_tensor([-1], name+'_-1', kwargs['initializer'])\n        nodes = [\n            make_node('Shape', [input_nodes[0]], [name+'_shape']),\n            make_node('Slice', [name+'_shape', name+'_0',\n                                name+'_1'], [name+'_1st_dim']),\n            make_node('Slice', [name+'_shape', name+'_1',\n                                name+'_2'], [name+'_2nd_dim']),\n            make_node('Mul', [name+'_1st_dim', name+'_2nd_dim'], [name+'_mul']),\n            make_node('Concat', [name+'_mul', name+'_-1'], [name+'_shape_new'], axis=0),\n            make_node('Reshape', [input_nodes[0], name+'_shape_new'], [name], name=name),\n        ]\n\n    if special_case:\n        return nodes\n\n    not_supported_shape = [-2, -3, -4]\n    for val in targ_shape:\n        if val in not_supported_shape:\n            raise AttributeError(\"Reshape: Shape value not supported in ONNX\", val)\n\n    create_tensor(targ_shape, name+'_targ_shape', kwargs['initializer'])\n\n    nodes = []\n    if reverse == 'False':\n        nodes += [\n            make_node('Reshape', [input_nodes[0], name+'_targ_shape'], [name], name=name)\n            ]\n    else:\n        create_tensor([0], name+'_0', kwargs['initializer'])\n        create_tensor([1], name+'_1', kwargs['initializer'])\n        nodes += [\n            make_node('Shape', [name+'_targ_shape'], [name+'_targ_dim']),\n            make_node('Shape', [input_nodes[0]], [name+'_orig_shape']),\n            make_node('Shape', [name+'_orig_shape'], [name+'_orig_dim']),\n            make_node('Sub', [name+'_targ_dim', name+'_orig_dim'], [name+'_dim_diff']),\n            make_node('Abs', [name+'_dim_diff'], [name+'_pad_len']),\n            make_node('Less', [name+'_targ_dim', name+'_orig_dim'], [name+'_targ_less_orig']),\n            make_node('Less', [name+'_orig_dim', name+'_targ_dim'], [name+'_orig_less_targ']),\n            make_node('Where', [name+'_targ_less_orig', name+'_pad_len', name+'_0'],\n                      [name+'_targ_pad_len']),\n            make_node('Where', [name+'_orig_less_targ', name+'_pad_len', name+'_0'],\n                      [name+'_orig_pad_len']),\n            make_node('Concat', [name+'_targ_pad_len', name+'_0'], [name+'_targ_pads'], axis=0),\n            make_node('Concat', [name+'_orig_pad_len', name+'_0'], [name+'_orig_pads'], axis=0),\n            make_node('Pad', [name+'_targ_shape', name+'_targ_pads', name+'_1'],\n                      [name+'_targ_shape_padded'], mode='constant'),\n            make_node('Pad', [name+'_orig_shape', name+'_orig_pads', name+'_1'],\n                      [name+'_orig_shape_padded'], mode='constant'),\n            make_node('Equal', [name+'_targ_shape_padded', name+'_0'],\n                      [name+'_targ_shape_0_mask']),\n            make_node('Where', [name+'_targ_shape_0_mask', name+'_orig_shape_padded',\n                                name+'_targ_shape_padded'], [name+'_targ_shape_new']),\n            make_node('Shape', [name+'_targ_shape_new'], [name+'_targ_new_dim']),\n            make_node('Slice', [name+'_targ_shape_new', name+'_targ_pad_len',\n                                name+'_targ_new_dim'], [name+'_targ_shape_final']),\n            make_node('Reshape', [input_nodes[0], name+'_targ_shape_final'], [name], name=name)\n            ]\n\n    return nodes\n\n@mx_op.register(\"Cast\")\ndef convert_cast(node, **kwargs):\n    \"\"\"Map MXNet's Cast operator attributes to onnx's Cast operator\n    and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    dtype = np.dtype(attrs.get('dtype'))\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n    nodes = [\n        onnx.helper.make_node(\"Cast\", input_nodes, [name], to=dtype_t, name=name)\n    ]\n    return nodes, (dtype,)\n\n\n@mx_op.register(\"slice_axis\")\ndef convert_slice_axis(node, **kwargs):\n    \"\"\"Map MXNet's slice_axis operator attributes to onnx's Slice operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    axis = int(attrs.get(\"axis\"))\n    begin = int(attrs.get(\"begin\"))\n    end = attrs.get(\"end\", None)\n\n    nodes = []\n    create_tensor([axis], name+'_axis', kwargs[\"initializer\"])\n    create_tensor([begin], name+'_begin', kwargs[\"initializer\"])\n    if not end or end == 'None':\n        # ONNX doesn't support None for ends. Since ends=None depicts\n        # length of dimension, passing dimension in this case.\n        nodes += [\n            make_node('Shape', [input_nodes[0]], [name+\"_data_shape\"])\n        ]\n        # corner case when end = None and axis = -1\n        if axis == -1:\n            create_tensor([-1], name+'_-1', kwargs[\"initializer\"])\n            nodes += [\n                make_node('Shape', [name+'_data_shape'], [name+'_data_dim']),\n                make_node('Add', [name+'_data_dim', name+'_-1'], [name+'_axis_max']),\n                make_node('Slice', [name+'_data_shape', name+'_axis_max', name+'_data_dim'], [name+'_end']),\n            ]\n        else:\n            create_tensor([axis+1], name+\"_axis_plus_1\", kwargs[\"initializer\"])\n            nodes += [\n                make_node('Slice', [name+'_data_shape', name+'_axis', name+'_axis_plus_1'],\n                          [name+\"_end\"])\n            ]\n    else:\n        create_tensor([int(end)], name+'_end', kwargs[\"initializer\"])\n\n    nodes += [\n        make_node('Slice', [input_nodes[0], name+'_begin', name+'_end', name+'_axis'],\n                  [name], name=name)\n        ]\n\n    return nodes\n\n\n@mx_op.register('SliceChannel')\ndef convert_slice_channel(node, **kwargs):\n    \"\"\"Map MXNet's SliceChannel operator attributes to onnx's Squeeze or Split\n    operator based on squeeze_axis attribute\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    num_outputs = int(attrs.get('num_outputs'))\n    axis = int(attrs.get('axis', 1))\n    squeeze_axis = attrs.get('squeeze_axis', 'False')\n\n    nodes = []\n    if squeeze_axis in ['True', '1']:\n        nodes += [\n            make_node('Split', [input_nodes[0]], [name+str(i)+'_' for i in range(num_outputs)],\n                      axis=axis)\n        ]\n        for i in range(num_outputs):\n            nodes += [\n                make_node('Squeeze', [name+str(i)+'_'], [name+str(i)], axes=[axis])\n            ]\n    else:\n        nodes += [\n            make_node('Split', [input_nodes[0]], [name+str(i) for i in range(num_outputs)],\n                      axis=axis)\n        ]\n\n    return nodes\n\n@mx_op.register(\"expand_dims\")\ndef convert_expand_dims(node, **kwargs):\n    \"\"\"Map MXNet's expand_dims operator attributes to onnx's Unsqueeze operator\n    and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    axis = int(attrs.get(\"axis\"))\n\n    node = onnx.helper.make_node(\n        \"Unsqueeze\",\n        input_nodes,\n        [name],\n        axes=[axis],\n        name=name,\n    )\n    return [node]\n\n@mx_op.register(\"squeeze\")\n@mx_op.register(\"_npi_squeeze\")\ndef convert_squeeze(node, **kwargs):\n    \"\"\"Map MXNet's squeeze operator attributes to onnx's squeeze operator\n    and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = str(attrs.get(\"axis\", 'None'))\n    axes = convert_string_to_list(mx_axis) if mx_axis != 'None' else None\n\n    if not axes:\n        node = onnx.helper.make_node(\n            \"Squeeze\",\n            input_nodes,\n            [name],\n            name=name\n        )\n    else:\n        node = onnx.helper.make_node(\n            \"Squeeze\",\n            input_nodes,\n            [name],\n            axes=axes,\n            name=name,\n        )\n    return [node]\n\n\n@mx_op.register(\"log\")\n@mx_op.register(\"_npi_log\")\ndef convert_log(node, **kwargs):\n    \"\"\"Map MXNet's log operator attributes to onnx's Log operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Log', node, kwargs)\n\n@mx_op.register(\"reciprocal\")\n@mx_op.register(\"_npi_reciprocal\")\ndef convert_reciprocal(node, **kwargs):\n    \"\"\"Map MXNet's reciprocal operator attributes to onnx's Reciprocal operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Reciprocal', node, kwargs)\n\n@mx_op.register(\"_power\")\n@mx_op.register(\"_npi_power\")\ndef convert_power(node, **kwargs):\n    \"\"\"Map MXNet's _power operator attributes to onnx's Pow operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Pow', node, kwargs)\n\n@mx_op.register(\"broadcast_power\")\ndef convert_broadcast_power(node, **kwargs):\n    \"\"\"Map MXNet's _power operator attributes to onnx's Pow operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Pow', node, kwargs)\n\n@mx_op.register(\"sqrt\")\n@mx_op.register(\"_npi_sqrt\")\ndef convert_sqrt(node, **kwargs):\n    \"\"\"Map MXNet's sqrt operator attributes to onnx's Sqrt operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Sqrt', node, kwargs)\n\n@mx_op.register(\"depth_to_space\")\ndef convert_depthtospace(node, **kwargs):\n    \"\"\"Map MXNet's depth_to_space operator attributes to onnx's\n    DepthToSpace operator and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    blksize = int(attrs.get(\"block_size\", 0))\n\n    node = onnx.helper.make_node(\n        \"DepthToSpace\",\n        input_nodes,\n        [name],\n        blocksize=blksize,\n        name=name,\n    )\n    return [node]\n\n@mx_op.register(\"space_to_depth\")\ndef convert_spacetodepth(node, **kwargs):\n    \"\"\"Map MXNet's space_to_depth operator attributes to onnx's\n    SpaceToDepth operator and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    blksize = int(attrs.get(\"block_size\", 0))\n\n    node = onnx.helper.make_node(\n        \"SpaceToDepth\",\n        input_nodes,\n        [name],\n        blocksize=blksize,\n        name=name,\n    )\n    return [node]\n\n@mx_op.register(\"square\")\n@mx_op.register(\"_npi_square\")\ndef convert_square(node, **kwargs):\n    \"\"\"Map MXNet's square operator attributes to onnx's Pow operator\n    and return the created node.\n    \"\"\"\n    name, input_nodes, _ = get_inputs(node, kwargs)\n\n    initializer = kwargs[\"initializer\"]\n    data_type = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype('int64')]\n\n    power2_name = \"square_tensor\" + str(kwargs[\"idx\"])\n    tensor_node = onnx.helper.make_tensor_value_info(power2_name, data_type, (1,))\n    initializer.append(\n        onnx.helper.make_tensor(\n            name=power2_name,\n            data_type=data_type,\n            dims=(1,),\n            vals=[2],\n            raw=False,\n        )\n    )\n\n    input_nodes.append(power2_name)\n\n    node = onnx.helper.make_node(\n        \"Pow\",\n        input_nodes,\n        [name],\n        name=name\n    )\n    return [tensor_node, node]\n\n# sum_axis is equivalent to sum in MXNet\n@mx_op.register(\"sum\")\n@mx_op.register(\"sum_axis\")\n@mx_op.register(\"_npi_sum\")\ndef convert_sum(node, **kwargs):\n    \"\"\"Map MXNet's sum operator attributes to onnx's ReduceSum operator\n    and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = attrs.get(\"axis\", None)\n    axes = convert_string_to_list(str(mx_axis)) if mx_axis is not None else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n    print(axes)\n    if axes != [None]:\n        node = onnx.helper.make_node(\n            'ReduceSum',\n            inputs=input_nodes,\n            outputs=[name],\n            axes=axes,\n            keepdims=keepdims,\n            name=name\n        )\n    else:\n        node = onnx.helper.make_node(\n            'ReduceSum',\n            inputs=input_nodes,\n            outputs=[name],\n            keepdims=keepdims,\n            name=name\n        )\n    return [node]\n\n\n@mx_op.register(\"shape_array\")\ndef convert_shape(node, **kwargs):\n    \"\"\"Map MXNet's shape_array operator attributes to onnx's Shape operator\n    and return the created node.\n    \"\"\"\n    return create_basic_op_node('Shape', node, kwargs)\n\n\n@mx_op.register(\"hard_sigmoid\")\ndef convert_hardsigmoid(node, **kwargs):\n    \"\"\"Map MXNet's hard_sigmoid operator attributes to onnx's HardSigmoid operator\n    and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    # Converting to float32\n    alpha = float(attrs.get(\"alpha\", 0.2))\n    beta = float(attrs.get(\"beta\", 0.5))\n\n    node = onnx.helper.make_node(\n        'HardSigmoid',\n        input_nodes,\n        [name],\n        alpha=alpha,\n        beta=beta,\n        name=name\n    )\n    return [node]\n\n@mx_op.register(\"broadcast_lesser\")\ndef convert_broadcast_lesser(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_lesser operator attributes to onnx's Less operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    nodes = [\n        make_node('Less', [input_nodes[0], input_nodes[1]], [name+'_lt']),\n        make_node('Cast', [name+'_lt'], [name], to=dtype_t)\n    ]\n\n    return nodes\n\n\n@mx_op.register(\"broadcast_lesser_equal\")\ndef convert_broadcast_lesser_equal(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_lesser_equal operator\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    nodes = [\n        make_node('LessOrEqual', [input_nodes[0], input_nodes[1]], [name+'_lt']),\n        make_node('Cast', [name+'_lt'], [name], to=dtype_t)\n    ]\n\n    return nodes\n\n\n@mx_op.register(\"broadcast_greater_equal\")\ndef convert_broadcast_greater_equal(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_greater_equal operator\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    nodes = [\n        make_node('GreaterOrEqual', [input_nodes[0], input_nodes[1]], [name+'_gt']),\n        make_node('Cast', [name+'_gt'], [name], to=dtype_t)\n    ]\n\n    return nodes\n\n\n@mx_op.register(\"broadcast_greater\")\ndef convert_broadcast_greater(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_greater operator attributes to onnx's Greater operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    nodes = [\n        make_node('Greater', [input_nodes[0], input_nodes[1]], [name+'_gt']),\n        make_node('Cast', [name+'_gt'], [name], to=dtype_t)\n    ]\n\n    return nodes\n\n\n@mx_op.register(\"broadcast_equal\")\ndef convert_broadcast_equal(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_equal operator attributes to onnx's Equal operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    nodes = [\n        make_node(\"Equal\", input_nodes, [name+\"_equal\"]),\n        make_node(\"Cast\", [name+\"_equal\"], [name], name=name, to=int(dtype_t))\n    ]\n    return nodes\n\n\n@mx_op.register(\"broadcast_not_equal\")\ndef convert_broadcast_not_equal(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_not_equal operator attributes to onnx's Equal operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    nodes = [\n        make_node(\"Equal\", input_nodes, [name+\"_equal\"]),\n        make_node(\"Not\", [name+\"_equal\"], [name+\"_not\"]),\n        make_node(\"Cast\", [name+\"_not\"], [name], name=name, to=int(dtype_t))\n    ]\n    return nodes\n\n\n@mx_op.register(\"broadcast_logical_and\")\ndef convert_broadcast_logical_and(node, **kwargs):\n    \"\"\"Map MXNet's broadcast logical and operator attributes to onnx's And operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n    nodes = [\n        make_node(\"Cast\", [input_nodes[0]], [name+\"_cast0\"], to=int(TensorProto.BOOL)),\n        make_node(\"Cast\", [input_nodes[1]], [name+\"_cast1\"], to=int(TensorProto.BOOL)),\n        make_node(\"And\", [name+\"_cast0\", name+\"_cast1\"], [name+\"_and\"]),\n        make_node(\"Cast\", [name+\"_and\"], [name], name=name, to=int(dtype_t))\n    ]\n    return nodes\n\n\n@mx_op.register(\"broadcast_logical_or\")\ndef convert_broadcast_logical_or(node, **kwargs):\n    \"\"\"Map MXNet's broadcast logical or operator attributes to onnx's Or operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n    nodes = [\n        make_node(\"Cast\", [input_nodes[0]], [name+\"_cast0\"], to=int(TensorProto.BOOL)),\n        make_node(\"Cast\", [input_nodes[1]], [name+\"_cast1\"], to=int(TensorProto.BOOL)),\n        make_node(\"Or\", [name+\"_cast0\", name+\"_cast1\"], [name+\"_or\"]),\n        make_node(\"Cast\", [name+\"_or\"], [name], name=name, to=int(dtype_t))\n    ]\n    return nodes\n\n\n@mx_op.register(\"broadcast_logical_xor\")\ndef convert_broadcast_logical_xor(node, **kwargs):\n    \"\"\"Map MXNet's broadcast logical xor operator attributes to onnx's Xor operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n    nodes = [\n        make_node(\"Cast\", [input_nodes[0]], [name+\"_cast0\"], to=int(TensorProto.BOOL)),\n        make_node(\"Cast\", [input_nodes[1]], [name+\"_cast1\"], to=int(TensorProto.BOOL)),\n        make_node(\"Xor\", [name+\"_cast0\", name+\"_cast1\"], [name+\"_xor\"]),\n        make_node(\"Cast\", [name+\"_xor\"], [name], name=name, to=int(dtype_t))\n    ]\n    return nodes\n\n\n@mx_op.register(\"logical_not\")\ndef convert_logical_not(node, **kwargs):\n    \"\"\"Map MXNet's logical not operator attributes to onnx's Not operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n    nodes = [\n        make_node(\"Cast\", [input_nodes[0]], [name+\"_cast\"], to=int(TensorProto.BOOL)),\n        make_node(\"Not\", [name+\"_cast\"], [name+\"_not\"]),\n        make_node(\"Cast\", [name+\"_not\"], [name], name=name, to=int(dtype_t))\n    ]\n    return nodes\n\n\n@mx_op.register(\"size_array\")\ndef convert_size(node, **kwargs):\n    \"\"\"Map MXNet's size_array operator attributes to onnx's Size operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, _ = get_inputs(node, kwargs)\n\n    create_tensor([1], name+'_1', kwargs['initializer'])\n    nodes = [\n        make_node('Size', [input_nodes[0]], [name+'_size']),\n        make_node('Reshape', [name+'_size', name+'_1'], [name], name=name)\n    ]\n    return nodes\n\n\n@mx_op.register(\"log_softmax\")\ndef convert_logsoftmax(node, **kwargs):\n    \"\"\"Map MXNet's log_softmax operator attributes to onnx's LogSoftMax operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    # Converting to int\n    axis = int(attrs.get(\"axis\", -1))\n    temp = attrs.get('temperature', 'None')\n    use_length = attrs.get('use_length', 'False')\n\n    if temp != 'None':\n        raise AttributeError('LogSoftMax currently does not support temperature!=None')\n\n    if use_length in ['1', 'True']:\n        raise AttributeError('LogSoftMax currently does not support use_length==True')\n\n    nodes = [\n        make_node('Exp', [input_nodes[0]], [name+'_exp']),\n        make_node('ReduceSum', [name+'_exp'], [name+'_rsum'], axes=[axis], keepdims=1),\n        make_node('Div', [name+'_exp', name+'_rsum'], [name+'_div']),\n        make_node('Log', [name+'_div'], [name])\n    ]\n\n    return nodes\n\n@mx_op.register(\"norm\")\ndef convert_norm(node, **kwargs):\n    \"\"\"Map MXNet's norm operator attributes to onnx's ReduceL1 and ReduceL2 operators\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = attrs.get(\"axis\", None)\n    axes = convert_string_to_list(str(mx_axis)) if mx_axis else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n    ord = int(attrs.get(\"ord\", 2))\n\n    if ord not in [1, 2]:\n        raise AttributeError(\"norm export operator only supports ord=1 or ord=2.\")\n\n    onnx_op_name = \"ReduceL1\" if ord == 1 else \"ReduceL2\"\n\n    if axes:\n        if keepdims:\n            reduce_node = make_node(onnx_op_name, input_nodes, [name], axes=axes, keepdims=keepdims)\n            return [reduce_node]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node(onnx_op_name, input_nodes, [name+'_norm'], axes=axes, keepdims=keepdims),\n                make_node('Shape', [name+'_norm'], [name+'_norm_shape']),\n                make_node('Concat', [name+'_1', name+'_norm_shape'], [name+'_concat'], axis=0),\n                make_node('Reshape', [name+'_norm', name+'_concat'], [name+'_reshape']),\n                make_node('Squeeze', [name+'_reshape'], [name], axes=[0]),\n            ]\n            return nodes\n    else:\n\n        if keepdims:\n            reduce_node = make_node(onnx_op_name, input_nodes, [name], keepdims=keepdims)\n            return [reduce_node]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node(onnx_op_name, input_nodes, [name+'_norm'], keepdims=keepdims),\n                make_node('Reshape', [name+'_norm', name+'_1'], [name])\n            ]\n            return nodes\n\n\n@mx_op.register(\"_sample_multinomial\")\ndef convert_multinomial(node, **kwargs):\n    \"\"\"Map MXNet's multinomial operator attributes to onnx's\n    Multinomial operator and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    dtype = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(attrs.get(\"dtype\", 'int32'))]\n    sample_size = convert_string_to_list(attrs.get(\"shape\", '1'))\n    if len(sample_size) < 2:\n        sample_size = sample_size[-1]\n    else:\n        raise AttributeError(\"ONNX currently supports integer sample_size only\")\n    node = onnx.helper.make_node(\n        \"Multinomial\",\n        input_nodes,\n        [name],\n        dtype=dtype,\n        sample_size=sample_size,\n        name=name,\n    )\n    return [node]\n\n\n@mx_op.register(\"_random_uniform\")\ndef convert_random_uniform(node, **kwargs):\n    \"\"\"Map MXNet's random_uniform operator attributes to onnx's RandomUniform\n    operator and return the created node.\n    \"\"\"\n    name, _, attrs = get_inputs(node, kwargs)\n\n    # Converting to float32\n    low = float(attrs.get(\"low\", 0))\n    high = float(attrs.get(\"high\", 1.0))\n    shape = convert_string_to_list(attrs.get('shape', '[]'))\n    dtype = np.dtype(attrs.get('dtype', 'float32'))\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    node = onnx.helper.make_node(\n        'RandomUniform',\n        [],\n        [name],\n        low=low,\n        high=high,\n        dtype=dtype_t,\n        shape=shape,\n        name=name\n    )\n    return [node], (dtype,)\n\n\n@mx_op.register(\"_random_normal\")\ndef convert_random_normal(node, **kwargs):\n    \"\"\"Map MXNet's random_normal operator attributes to onnx's RandomNormal\n    operator and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    # Converting to float32\n    mean = float(attrs.get(\"loc\", 0))\n    scale = float(attrs.get(\"scale\", 1.0))\n    shape = convert_string_to_list(attrs.get('shape', '[]'))\n    dtype = np.dtype(attrs.get('dtype', 'float32'))\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    node = onnx.helper.make_node(\n        'RandomNormal',\n        input_nodes,\n        [name],\n        mean=mean,\n        scale=scale,\n        dtype=dtype_t,\n        shape=shape,\n        name=name\n    )\n    return [node], (dtype,)\n\n\n@mx_op.register(\"ROIPooling\")\ndef convert_roipooling(node, **kwargs):\n    \"\"\"Map MXNet's ROIPooling operator attributes to onnx's MaxRoiPool\n    operator and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    pooled_shape = convert_string_to_list(attrs.get('pooled_size'))\n    scale = float(attrs.get(\"spatial_scale\"))\n\n    node = onnx.helper.make_node(\n        'MaxRoiPool',\n        input_nodes,\n        [name],\n        pooled_shape=pooled_shape,\n        spatial_scale=scale,\n        name=name\n    )\n    return [node]\n\n\n@mx_op.register(\"tile\")\ndef convert_tile(node, **kwargs):\n    \"\"\"Map MXNet's Tile operator attributes to onnx's Tile\n    operator and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    data = input_nodes[0]\n    reps = convert_string_to_list(attrs[\"reps\"])\n\n    create_tensor([0], name+'_0', kwargs['initializer'])\n    create_tensor([1], name+'_1', kwargs['initializer'])\n    create_tensor(reps, name+'_reps', kwargs['initializer'], dtype='int64')\n    create_tensor([len(reps)], name+'_reps_len', kwargs['initializer'])\n\n    nodes = [\n        make_node('Shape', [data], [name+'_data_shape']),\n        make_node('Shape', [name+'_data_shape'], [name+'_data_dim']),\n        make_node('Max', [name+'_data_dim', name+'_reps_len'], [name+'_max']),\n        make_node('Sub', [name+'_max', name+'_data_dim'], [name+'_data_diff']),\n        make_node('Concat', [name+'_data_diff', name+'_0'], [name+'_concat0_out'], axis=0),\n        make_node('Pad', [name+'_data_shape', name+'_concat0_out', name+'_1'], [name+'_data_shape_pad']),\n        make_node('Reshape', [data, name+'_data_shape_pad'], [name+'_data']),\n        make_node('Sub', [name+'_max', name+'_reps_len'], [name+'_reps_diff']),\n        make_node('Concat', [name+'_reps_diff', name+'_0'], [name+'_concat1_out'], axis=0),\n        make_node('Pad', [name+'_reps', name+'_concat1_out', name+'_1'], [name+'_reps_pad']),\n        make_node('Tile', [name+'_data', name+'_reps_pad'], [name], name=name),\n    ]\n\n    return nodes\n\n\n@mx_op.register(\"broadcast_to\")\n@mx_op.register(\"_npi_broadcast_to\")\ndef convert_broadcast_to(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_to operator attributes to onnx's Expand\n    operator and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    shape_list = convert_string_to_list(attrs[\"shape\"])\n\n    initializer = kwargs[\"initializer\"]\n    output_shape_np = np.array(shape_list, dtype='int64')\n    data_type = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[output_shape_np.dtype]\n    dims = np.shape(output_shape_np)\n\n    output_shape_name = \"expand_attr_tensor\" + str(kwargs[\"idx\"])\n    tensor_node = onnx.helper.make_tensor_value_info(output_shape_name, data_type, dims)\n\n    initializer.append(\n        onnx.helper.make_tensor(\n            name=output_shape_name,\n            data_type=data_type,\n            dims=dims,\n            vals=shape_list,\n            raw=False,\n        )\n    )\n\n    input_nodes.append(output_shape_name)\n    expand_node = onnx.helper.make_node(\n        \"Expand\",\n        input_nodes,\n        [name],\n        name=name\n    )\n\n    return [tensor_node, expand_node]\n\n\n@mx_op.register('topk')\ndef convert_topk(node, **kwargs):\n    \"\"\"Map MXNet's topk operator attributes to onnx's TopK operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    opset_version = kwargs['opset_version']\n    if opset_version < 11:\n        raise AttributeError('ONNX opset 11 or greater is required to export this operator')\n\n    axis = int(attrs.get('axis', '-1'))\n    k = int(attrs.get('k', '1'))\n    ret_type = attrs.get('ret_typ', 'indices')\n    is_ascend = attrs.get('is_ascend', 'False')\n    is_ascend = is_ascend in ['1', 'True']\n    dtype = attrs.get('dtype', 'float32')\n\n    if ret_type == 'mask':\n        raise NotImplementedError('topk does not currently support ret_type==\\'mask\\'')\n\n    create_tensor([k], name+'_k', kwargs['initializer'])\n\n    nodes = []\n\n    if ret_type == 'both':\n        if dtype == 'int64':\n            nodes += [\n                make_node('TopK', [input_nodes[0], name+'_k'], [name+'0', name+'1'], axis=axis,\n                          largest=(not is_ascend), sorted=1),\n            ]\n        else:\n            nodes += [\n                make_node('TopK', [input_nodes[0], name+'_k'], [name+'0', name+'_1_i'], axis=axis,\n                          largest=(not is_ascend), sorted=1),\n                make_node('Cast', [name+'_1_i'], [name+'1'],\n                          to=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(dtype)])\n            ]\n    elif ret_type == 'value':\n        nodes += [\n            make_node('TopK', [input_nodes[0], name+'_k'], [name+'0', name+'_'], axis=axis,\n                      largest=(not is_ascend), sorted=1),\n        ]\n    else:\n        if dtype == 'int64':\n            nodes += [\n                make_node('TopK', [input_nodes[0], name+'_k'], [name+'_', name], axis=axis,\n                          largest=(not is_ascend), sorted=1),\n            ]\n        else:\n            nodes += [\n                make_node('TopK', [input_nodes[0], name+'_k'], [name+'__', name+'_tmp'], axis=axis,\n                          largest=(not is_ascend), sorted=1),\n                make_node('Cast', [name+'_tmp'], [name],\n                          to=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(dtype)])\n            ]\n\n    return nodes\n\n\n@mx_op.register(\"take\")\ndef convert_take(node, **kwargs):\n    \"\"\"Map MXNet's Take operator attributes to onnx's Gather operator.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    axis = int(attrs.get('axis', 0))\n    mode = str(attrs.get('mode', 'clip'))\n\n    data = input_nodes[0]\n    indices = input_nodes[1]\n\n    nodes = [\n        make_node('Cast', [indices], [name+'_indices'], to=int(TensorProto.INT64)),\n    ]\n\n    if mode == 'raise':\n        nodes += [\n            make_node('Gather', [data, name+'_indices'], [name], axis=axis, name=name)\n        ]\n\n        return nodes\n\n    create_tensor([-1], name+'_-1', kwargs[\"initializer\"])\n    nodes += [\n        make_node('Shape', [data], [name+'_data_shape']),\n    ]\n\n    # corner case\n    if axis == -1:\n        nodes += [\n            make_node('Shape', [name+'_data_shape'], [name+'_data_dim']),\n            make_node('Add', [name+'_data_dim', name+'_-1'], [name+'_axis_max']),\n            make_node('Slice', [name+'_data_shape', name+'_axis_max', name+'_data_dim'], [name+'_slice0_out']),\n        ]\n\n    else:\n        create_tensor([axis], name+'_axis', kwargs[\"initializer\"])\n        create_tensor([axis+1], name+'_axis+1', kwargs[\"initializer\"])\n        nodes += [\n            make_node('Slice', [name+'_data_shape', name+'_axis', name+'_axis+1'], [name+'_slice0_out']),\n        ]\n\n    if mode == 'clip':\n        create_tensor([0], name+'_0', kwargs[\"initializer\"])\n        nodes += [\n            make_node('Add', [name+'_slice0_out', name+'_-1'], [name+'_max']),\n            make_node('Greater', [name+'_indices', name+'_max'], [name+'_max_mask']),\n            make_node('Where', [name+'_max_mask', name+'_max', name+'_indices'], [name+'_where0_out']),\n            make_node('Less', [name+'_indices', name+'_0'], [name+'_min_mask']),\n            make_node('Where', [name+'_min_mask', name+'_0', name+'_where0_out'], [name+'_where1_out']),\n            make_node('Gather', [data, name+'_where1_out'], [name], axis=axis, name=name)\n        ]\n\n    elif mode == 'wrap':\n        nodes += [\n            make_node('Mod', [name+'_indices', name+'_slice0_out'], [name+'_mod0_out']),\n            make_node('Gather', [data, name+'_mod0_out'], [name], axis=axis, name=name)\n        ]\n\n    else:\n        raise NotImplementedError(\"mode must be clip, wrap or raise.\")\n\n    return nodes\n\n\n@mx_op.register(\"LayerNorm\")\ndef convert_layer_norm(node, **kwargs):\n    \"\"\"Map MXNet's LayerNorm operator attributes to onnx operators.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n\n    axes = int(attrs.get('axis', -1))\n    eps = attrs.get('eps', 9.99999975e-06)\n\n    create_tensor([axes], name+\"_axes\", kwargs[\"initializer\"])\n    create_tensor([axes+1], name+\"_axes+1\", kwargs[\"initializer\"])\n    create_const_scalar_node(name+'_0_s', np.int64(0), kwargs)\n    create_const_scalar_node(name+'_1_s', np.int64(1), kwargs)\n    create_const_scalar_node(name+\"_2_s\", np.int64(2).astype(dtype), kwargs)\n    create_const_scalar_node(name+\"_eps\", np.float32(eps), kwargs)\n\n    nodes = [\n        make_node(\"ReduceMean\", [input_nodes[0]], [name+\"_rm0_out\"], axes=[axes]),\n        make_node(\"Sub\", [input_nodes[0], name+\"_rm0_out\"], [name+\"_sub0_out\"]),\n        make_node(\"Pow\", [name+\"_sub0_out\", name+\"_2_s\"], [name+\"_pow0_out\"]),\n        make_node(\"ReduceMean\", [name+\"_pow0_out\"], [name+\"_rm1_out\"], axes=[axes]),\n        make_node(\"Add\", [name+\"_rm1_out\", name+\"_eps\"], [name+\"_add0_out\"]),\n        make_node(\"Sqrt\", [name+\"_add0_out\"], [name+\"_sqrt0_out\"]),\n        make_node(\"Div\", [name+\"_sub0_out\", name+\"_sqrt0_out\"], [name+\"_div0_out\"]),\n    ]\n\n    if axes == -1:\n        nodes += [\n            make_node(\"Mul\", [name+\"_div0_out\", input_nodes[1]], [name+\"_mul0_out\"]),\n            # make_node(\"Add\", [name+\"_mul0_out\", input_nodes[2]], [name])\n            # the Add operator triggers a weird NaN issue in onnxruntime\n            # a workaround is to use Neg + Sub\n            make_node('Neg', [input_nodes[2]], [name+'_neg']),\n            make_node(\"Sub\", [name+\"_mul0_out\", name+'_neg'], [name])\n        ]\n    else:\n        nodes += [\n            make_node(\"Shape\", [input_nodes[0]], [name+\"_shape0_out\"]),\n            make_node(\"Shape\", [name+\"_shape0_out\"], [name+\"_in_dim\"]),\n            make_node(\"Squeeze\", [name+\"_in_dim\"], [name+\"_in_dim_s\"], axes=[0]),\n            make_node(\"Range\", [name+\"_0_s\", name+\"_in_dim_s\", name+\"_1_s\"], [name+\"_range\"]),\n            make_node(\"Equal\", [name+\"_range\", name+\"_axes\"], [name+\"_equal\"]),\n            make_node(\"Cast\", [name+\"_equal\"], [name+\"_one_hot\"], to=int(TensorProto.INT64)),\n            make_node(\"Slice\", [name+\"_shape0_out\", name+\"_axes\", name+\"_axes+1\"], [name+\"_slice_out\"]),\n            make_node(\"Squeeze\", [name+\"_slice_out\"], [name+\"_slice_out_s\"], axes=[0]),\n            make_node(\"Sub\", [name+\"_slice_out_s\", name+\"_1_s\"], [name+\"_sub1_out\"]),\n            make_node(\"Mul\", [name+\"_one_hot\", name+\"_sub1_out\"], [name+\"_mul0_out\"]),\n            make_node(\"Add\", [name+\"_mul0_out\", name+\"_1_s\"], [name+\"_add1_out\"]),\n            make_node('Reshape', [input_nodes[1], name+\"_add1_out\"], [name+\"gamma_exp\"]),\n            make_node('Reshape', [input_nodes[2], name+\"_add1_out\"], [name+\"beta_exp\"]),\n            make_node('Expand', [name+\"gamma_exp\", name+\"_shape0_out\"], [name+\"gamma_exp1\"]),\n            make_node('Expand', [name+\"beta_exp\", name+\"_shape0_out\"], [name+\"beta_exp1\"]),\n            make_node(\"Mul\", [name+\"_div0_out\", name+\"gamma_exp1\"], [name+\"_mul1_out\"]),\n            make_node(\"Add\", [name+\"_mul1_out\", name+\"beta_exp1\"], [name], name=name)\n        ]\n\n    return nodes\n\n\n@mx_op.register(\"_contrib_interleaved_matmul_selfatt_qk\")\ndef convert_matmul_selfatt_qk(node, **kwargs):\n    \"\"\"Map MXNet's _contrib_interleaved_matmul_selfatt_qk operator\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    heads = int(attrs.get('heads'))\n\n    # a, b, c, d, e are seq_len, batch_size, num_heads, 3, head_dim respectively\n    create_tensor([0], name+\"_0\", kwargs[\"initializer\"])\n    create_tensor([1], name+\"_1\", kwargs[\"initializer\"])\n    create_tensor([1], name+\"_1_f\", kwargs[\"initializer\"], dtype='float32')\n    create_tensor([2], name+\"_2\", kwargs[\"initializer\"])\n    create_tensor([3], name+\"_3\", kwargs[\"initializer\"])\n    create_tensor([heads], name+\"_c\", kwargs[\"initializer\"])\n    create_tensor([3], name+\"_d\", kwargs[\"initializer\"])\n    nodes = [\n        make_node('Shape', [input_nodes[0]], [name+\"_data_shape\"]),\n        make_node('Slice', [name+'_data_shape', name+'_0', name+'_1'], [name+\"_a\"]),\n        make_node('Slice', [name+'_data_shape', name+'_1', name+'_2'], [name+\"_b\"]),\n        make_node('Slice', [name+'_data_shape', name+'_2', name+'_3'], [name+\"_cde\"]),\n        make_node('Div', [name+'_cde', name+'_c'], [name+'_de']),\n        make_node('Div', [name+'_de', name+'_d'], [name+'_e']),\n        make_node('Cast', [name+'_e'], [name+'_e_f'], to=int(TensorProto.FLOAT)),\n        make_node('Sqrt', [name+'_e_f'], [name+'_sqrt_e']),\n        make_node('Div', [name+'_1_f', name+'_sqrt_e'], [name+'_1_over_sqrt_e']),\n        make_node('Mul', [name+'_b', name+'_c'], [name+'_bc']),\n\n        make_node(\"Concat\", [name+'_a', name+'_b', name+'_c', name+'_d', name+'_e'], \\\n            [name+'_shape0'], axis=0),\n        make_node(\"Concat\", [name+'_0', name+'_0', name+'_0', name+'_0', name+'_0'], \\\n            [name+'_slice_start0'], axis=0),\n        make_node(\"Concat\", [name+'_a', name+'_b', name+'_c', name+'_1', name+'_e'], \\\n            [name+'_slice_end0'], axis=0),\n        make_node(\"Concat\", [name+'_a', name+'_b', name+'_c', name+'_e'], \\\n            [name+'_shape1'], axis=0),\n        make_node(\"Concat\", [name+'_bc', name+'_a', name+'_e'], \\\n            [name+'_shape2'], axis=0),\n        make_node(\"Concat\", [name+'_0', name+'_0', name+'_0', name+'_1', name+'_0'], \\\n            [name+'_slice_start1'], axis=0),\n        make_node(\"Concat\", [name+'_a', name+'_b', name+'_c', name+'_2', name+'_e'], \\\n            [name+'_slice_end1'], axis=0),\n\n        make_node('Reshape', [input_nodes[0], name+'_shape0'], [name+'_reshape0_out']),\n        make_node('Slice', [name+'_reshape0_out', name+'_slice_start0', name+'_slice_end0'], \\\n            [name+'_slice0_out']),\n        make_node('Reshape', [name+'_slice0_out', name+'_shape1'], [name+'_reshape1_out']),\n        make_node('Transpose', [name+'_reshape1_out'], [name+'_transpose0_out'], \\\n            perm=(1, 2, 0, 3)),\n        make_node('Reshape', [name+'_transpose0_out', name+'_shape2'], [name+'_reshape2_out']),\n        make_node('Mul', [name+'_reshape2_out', name+'_1_over_sqrt_e'], [name+'_mul0_out']),\n        make_node('Slice', [name+'_reshape0_out', name+'_slice_start1', name+'_slice_end1'], \\\n            [name+'_slice1_out']),\n        make_node('Reshape', [name+'_slice1_out', name+'_shape1'], [name+'_reshape3_out']),\n        make_node('Transpose', [name+'_reshape3_out'], [name+'_transpose1_out'], \\\n            perm=(1, 2, 0, 3)),\n        make_node('Reshape', [name+'_transpose1_out', name+'_shape2'], [name+'_reshape4_out']),\n        make_node('Transpose', [name+'_reshape4_out'], [name+'_transpose2_out'], \\\n            perm=(0, 2, 1)),\n        make_node('MatMul', [name+'_mul0_out', name+'_transpose2_out'], [name], name=name)\n    ]\n\n    return nodes\n\n@mx_op.register(\"_contrib_interleaved_matmul_selfatt_valatt\")\ndef convert_contrib_interleaved_matmul_selfatt_valatt(node, **kwargs):\n    \"\"\"Map MXNet's _contrib_interleaved_matmul_selfatt_valatt operator attributes to onnx's operator.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    qkv = input_nodes[0]\n    att = input_nodes[1]\n    num_heads = int(attrs.get('heads'))\n\n    create_tensor([num_heads], name+\"_const_num_heads\", kwargs[\"initializer\"])\n    create_tensor([0], name+\"_const_0\", kwargs[\"initializer\"])\n    create_tensor([1], name+\"_const_1\", kwargs[\"initializer\"])\n    create_tensor([2], name+\"_const_2\", kwargs[\"initializer\"])\n    create_tensor([3], name+\"_const_3\", kwargs[\"initializer\"])\n    create_tensor([4], name+\"_const_4\", kwargs[\"initializer\"])\n    create_tensor([5], name+\"_const_5\", kwargs[\"initializer\"])\n    create_tensor([0, 0, num_heads, 3, -1], name+\"_reshape0_shape\", kwargs[\"initializer\"])\n    create_tensor([0, 0, 0, 2, 0], name+\"_slice_start\", kwargs[\"initializer\"])\n    create_tensor([0, 0, 0, -1], name+\"_reshape1_shape\", kwargs[\"initializer\"])\n    create_tensor([0, 0, -1], name+\"_reshape4_shape\", kwargs[\"initializer\"])\n\n    nodes = [\n        make_node(\"Shape\", [qkv], [name+\"_shape_qkv\"]),\n        make_node(\"Slice\", [name+\"_shape_qkv\", name+\"_const_0\", name+\"_const_1\"], [name+\"_qkv_d0\"]),\n        make_node(\"Slice\", [name+\"_shape_qkv\", name+\"_const_1\", name+\"_const_2\"], [name+\"_qkv_d1\"]),\n        make_node(\"Slice\", [name+\"_shape_qkv\", name+\"_const_2\", name+\"_const_3\"], [name+\"_qkv_d2\"]),\n        make_node('Mul', [name+\"_qkv_d1\", name+'_const_num_heads'], [name+'_mul_out']),\n        make_node(\"Reshape\", [qkv, name+\"_reshape0_shape\"], [name+\"_reshape0_output\"]),\n        make_node(\"Shape\", [name+\"_reshape0_output\"], [name+\"_shape_reshape0\"]),\n        make_node(\"Slice\", [name+\"_shape_reshape0\", name+\"_const_4\", name+\"_const_5\"], [name+\"_d4\"]),\n        make_node(\"Concat\", [name+\"_mul_out\", name+\"_qkv_d0\", name+\"_d4\"], [name+\"_reshape2_shape\"], axis=0),\n        make_node(\"Concat\", [name+\"_qkv_d1\", name+\"_const_num_heads\", name+\"_qkv_d0\", name+\"_d4\"], \\\n            [name+\"_reshape3_shape\"], axis=0),\n        make_node(\"Concat\", [name+\"_qkv_d0\", name+\"_qkv_d1\", name+\"_qkv_d2\", name+\"_const_3\", name+\"_d4\"], \\\n            [name+\"_slice_end\"], axis=0),\n        make_node(\"Slice\", [name+\"_reshape0_output\", name+\"_slice_start\", name+\"_slice_end\"], [name+\"_slice_output\"]),\n        make_node(\"Reshape\", [name+\"_slice_output\", name+\"_reshape1_shape\"], [name+\"_reshape1_output\"]),\n        make_node(\"Transpose\", [name+\"_reshape1_output\"], [name+\"_transpose0_output\"], perm=[1, 2, 0, 3]),\n        make_node(\"Reshape\", [name+\"_transpose0_output\", name+\"_reshape2_shape\"], [name+\"_reshape2_output\"]),\n        make_node(\"MatMul\", [att, name+\"_reshape2_output\"], [name+\"_matmul_output\"]),\n        make_node(\"Reshape\", [name+\"_matmul_output\", name+\"_reshape3_shape\"], [name+\"_reshape3_output\"]),\n        make_node(\"Transpose\", [name+\"_reshape3_output\"], [name+\"_transpose2_output\"], perm=[2, 0, 1, 3]),\n        make_node(\"Reshape\", [name+\"_transpose2_output\", name+\"_reshape4_shape\"], [name], name=name)\n    ]\n    return nodes\n\n\n@mx_op.register(\"broadcast_axis\")\ndef convert_broadcast_axis(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_axis\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    axis = convert_string_to_list(attrs.get('axis', '()'))\n    size = convert_string_to_list(attrs.get('size', '()'))\n    assert len(axis) == len(size)\n\n    shape_name = name+'_shape_0'\n\n    create_tensor([0], name+'_0', kwargs[\"initializer\"])\n    create_tensor([1], name+'_1', kwargs[\"initializer\"])\n    create_const_scalar_node(name+'_0_s', np.int64(0), kwargs)\n    create_const_scalar_node(name+'_1_s', np.int64(1), kwargs)\n\n    nodes = [\n        make_node('Shape', [input_nodes[0]], [shape_name]),\n        make_node('Shape', [shape_name], [name+'_in_dim']),\n        make_node('Squeeze', [name+'_in_dim'], [name+'_in_dim_s'], axes=[0]),\n        make_node('Range', [name+'_0_s', name+'_in_dim_s', name+'_1_s'], [name+'_range']),\n    ]\n\n    for i, axis in enumerate(axis):\n        if axis not in (0, 1):\n            create_tensor([axis], name+'_'+str(axis), kwargs[\"initializer\"])\n        create_tensor([size[i]-1], name+'_size_'+str(i), kwargs[\"initializer\"])\n        nodes += [\n            make_node('Equal', [name+'_range', name+'_'+str(axis)], [name+'_equal_'+str(i)]),\n            make_node('Cast', [name+'_equal_'+str(i)], [name+'_cast_'+str(i)], to=int(TensorProto.INT64)),\n            make_node('Mul', [name+'_size_'+str(i), name+'_cast_'+str(i)], [name+'_mul_'+str(i)]),\n            make_node('Add', [name+'_mul_'+str(i), name+'_1'], [name+'_add_'+str(i)]),\n            make_node('Mul', [name+'_add_'+str(i), shape_name], [name+'_shape_'+str(i+1)])\n        ]\n        shape_name = name+'_shape_'+str(i+1)\n\n    nodes += [\n        make_node('Expand', [input_nodes[0], shape_name], [name], name=name)\n    ]\n\n    return nodes\n\n\n@mx_op.register(\"SequenceMask\")\ndef convert_sequencemask(node, **kwargs):\n    \"\"\"Map MXNet's SequenceMask operator\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    use_sequence_length = attrs.get('use_sequence_length', 'False')\n    mask_val = float(attrs.get('value', '0'))\n    axis = int(attrs.get('axis', '0'))\n\n    if(use_sequence_length == 'False'):\n        return [make_node('Identity', [input_nodes[0]], [name], name=name)]\n\n    create_tensor([0], name+'_0', kwargs[\"initializer\"])\n    create_tensor([1], name+'_1', kwargs[\"initializer\"])\n    create_tensor([2], name+'_2', kwargs[\"initializer\"])\n    create_const_scalar_node(name+'_0_s', np.int64(0), kwargs)\n    create_const_scalar_node(name+'_1_s', np.int64(1), kwargs)\n    create_const_scalar_node(name+'_2_s', np.int64(2), kwargs)\n    create_tensor([mask_val], name+'_mask_val', kwargs[\"initializer\"], dtype='float32')\n\n    nodes = [\n        make_node('Shape', [input_nodes[0]], [name+'_in_shape']),\n        make_node('Slice', [name+'_in_shape', name+'_0', name+'_1'], [name+'_slice_0']),\n        make_node('Slice', [name+'_in_shape', name+'_1', name+'_2'], [name+'_slice_1']),\n        make_node('Concat', [name+'_slice_0', name+'_1'], [name+'_shape_0'], axis=0),\n        make_node('Shape', [name+'_in_shape'], [name+'_in_dim']),\n        make_node('Squeeze', [name+'_in_dim'], [name+'_in_dim_s'], axes=[0]),\n        make_node('Range', [name+'_0_s', name+'_in_dim_s', name+'_1_s'], [name+'_range_0']),\n        make_node('Less', [name+'_range_0', name+'_2'], [name+'_less_0']),\n        make_node('Where', [name+'_less_0', name+'_in_shape', name+'_1'], [name+'_shape_1'])\n    ]\n\n    if(axis == 0):\n        nodes += [\n            make_node('Squeeze', [name+'_slice_0'], [name+'_max_len'], axes=[0]),\n            make_node('Range', [name+'_0_s', name+'_max_len', name+'_1_s'], [name+'_range_1']),\n            make_node('Reshape', [name+'_range_1', name+'_shape_0'], [name+\"_reshape_0\"]),\n            make_node('Cast', [input_nodes[1]], [name+'_cast'], to=int(TensorProto.INT64)),\n            make_node('Less', [name+'_reshape_0', name+'_cast'], [name+'_less_1']),\n            make_node('Reshape', [name+'_less_1', name+'_shape_1'], [name+\"_reshape_1\"]),\n            make_node('Where', [name+'_reshape_1', input_nodes[0], name+'_mask_val'], [name], name=name),\n        ]\n    else:\n        nodes += [\n            make_node('Squeeze', [name+'_slice_1'], [name+'_max_len'], axes=[0]),\n            make_node('Range', [name+'_0_s', name+'_max_len', name+'_1_s'], [name+'_range_1']),\n            make_node('Reshape', [input_nodes[1], name+'_shape_0'], [name+\"_reshape_0\"]),\n            make_node('Cast', [name+\"_reshape_0\"], [name+'_cast'], to=int(TensorProto.INT64)),\n            make_node('Less', [name+'_range_1', name+'_cast'], [name+'_less_1']),\n            make_node('Reshape', [name+'_less_1', name+'_shape_1'], [name+\"_reshape_1\"]),\n            make_node('Where', [name+'_reshape_1', input_nodes[0], name+'_mask_val'], [name], name=name),\n        ]\n    return nodes\n\n\n@mx_op.register(\"Embedding\")\ndef convert_embedding(node, **kwargs):\n    \"\"\"Map MXNet's Embedding operator attributes to onnx's\n    Gather operator.\"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    axis = int(attrs.get('axis', 0))\n    dtype = str(attrs.get('dtype', 'float32'))\n\n    nodes = [\n        make_node('Cast', [input_nodes[0]], [name+'_indices_casted'], to=int(TensorProto.INT64)),\n        make_node('Gather', [input_nodes[1], name+'_indices_casted'], [name], axis=axis, name=name)\n    ]\n\n    return nodes, (dtype, )\n\n\n@mx_op.register(\"stack\")\ndef convert_stack(node, **kwargs):\n    \"\"\"Map MXNet's stack operator to onnx operators.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    axis = int(attrs.get('axis', 0))\n    idx = 0\n    nodes = []\n    for input_node in input_nodes:\n        nodes.append(onnx.helper.make_node(\n            \"Unsqueeze\",\n            inputs=[input_node],\n            outputs=[name+\"_unsqueeze\"+str(idx)],\n            axes=[axis]\n        ))\n        idx += 1\n\n    nodes.append(onnx.helper.make_node(\n        \"Concat\",\n        inputs=[name+\"_unsqueeze\"+str(i) for i in range(len(nodes))],\n        outputs=[name],\n        name=name,\n        axis=axis\n    ))\n    return nodes\n\n\n@mx_op.register(\"slice\")\ndef convert_slice(node, **kwargs):\n    \"\"\"Map MXNet's slice operator to onnx Slice operator.\"\"\"\n    from onnx.helper import make_node\n\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    starts = convert_string_to_list(attrs.get('begin'))\n    ends = convert_string_to_list(attrs.get('end'))\n    steps = convert_string_to_list(attrs.get('step', '[]'))\n\n    assert len(starts) == len(ends)\n    if len(steps) == 0 or (len(steps) == 1 and steps[0] is None):\n        steps = [1 for x in starts]\n    else:\n        assert len(steps) == len(starts)\n    steps = [1 if x is None else x for x in steps]\n    for i, s in enumerate(steps):\n        if s < 0:\n            raise NotImplementedError('slice operator does not support negative steps yet')\n        if starts[i] is None:\n            starts[i] = 0\n        if ends[i] is None:\n            ends[i] = 2**63-1\n\n    axes = [i for i in range(len(starts))]\n\n    create_tensor(axes, name+'_axes', kwargs['initializer'])\n    create_tensor(starts, name+'_starts', kwargs['initializer'])\n    create_tensor(ends, name+'_ends', kwargs['initializer'])\n    create_tensor(steps, name+'_steps', kwargs['initializer'])\n\n    nodes = [\n        make_node(\"Slice\", [input_nodes[0], name+'_starts', name+'_ends', name+'_axes',\n                            name+'_steps'], [name], name=name)\n    ]\n\n    return nodes\n\n\n@mx_op.register(\"_zeros\")\n@mx_op.register(\"_npi_zeros\")\ndef convert_zeros(node, **kwargs):\n    \"\"\"Map MXNet's zeros operator attributes to onnx's ConstantOfShape operator.\n    \"\"\"\n    from onnx.helper import make_node, make_tensor\n    name, _, attrs = get_inputs(node, kwargs)\n    dtype = attrs.get('dtype')\n    data_type = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(dtype)]\n    shape = convert_string_to_list(attrs.get('shape'))\n    # replace 0 with 1\n    shape = [x if x else 1 for x in shape]\n    create_tensor(shape, name+'_shape', kwargs['initializer'])\n    tensor_value = make_tensor(name+'_zero', data_type, [1], [0])\n    nodes = [\n        make_node('ConstantOfShape', [name+'_shape'], [name], name=name, value=tensor_value)\n    ]\n    return nodes, (dtype,)\n\n\n@mx_op.register(\"_ones\")\n@mx_op.register(\"_npi_ones\")\ndef convert_ones(node, **kwargs):\n    \"\"\"Map MXNet's ones operator attributes to onnx's ConstantOfShape operator.\n    \"\"\"\n    from onnx.helper import make_node, make_tensor\n    name, _, attrs = get_inputs(node, kwargs)\n    dtype = attrs.get('dtype')\n    data_type = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(dtype)]\n    shape = convert_string_to_list(attrs.get('shape'))\n    # replace 0 with 1\n    shape = [x if x else 1 for x in shape]\n    create_tensor(shape, name+'_shape', kwargs['initializer'])\n    tensor_value = make_tensor(name+'_one', data_type, [1], [1])\n    nodes = [\n        make_node('ConstantOfShape', [name+'_shape'], [name], name=name, value=tensor_value)\n    ]\n    return nodes, (dtype,)\n\n\n@mx_op.register(\"zeros_like\")\ndef convert_zeros_like(node, **kwargs):\n    \"\"\"Map MXNet's zeros_like operator attributes to onnx's ConstantOfShape operator.\n    \"\"\"\n    from onnx.helper import make_node, make_tensor\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n    dtype = np.dtype(input_dtypes[0])\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    # create tensor with shape of input\n    tensor_value = make_tensor(name+\"_zero\", dtype_t, [1], [0])\n    nodes = [\n        make_node(\"Shape\", [input_nodes[0]], [name+\"_shape\"]),\n        make_node(\"ConstantOfShape\", [name+\"_shape\"], [name], name=name, value=tensor_value)\n    ]\n    return nodes\n\n\n@mx_op.register(\"ones_like\")\ndef convert_ones_like(node, **kwargs):\n    \"\"\"Map MXNet's ones_like operator attributes to onnx's ConstantOfShape operator.\n    \"\"\"\n    from onnx.helper import make_node, make_tensor\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n    dtype = np.dtype(input_dtypes[0])\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    # create tensor with shape of input\n    tensor_value = make_tensor(name+\"_one\", dtype_t, [1], [1])\n    nodes = [\n        make_node(\"Shape\", [input_nodes[0]], [name+\"_shape\"]),\n        make_node(\"ConstantOfShape\", [name+\"_shape\"], [name], name=name, value=tensor_value)\n    ]\n    return nodes\n\n\n@mx_op.register(\"_contrib_arange_like\")\ndef convert_arange_like(node, **kwargs):\n    \"\"\"Map MXNet's arange_like operator attributes to onnx's Range and Reshape operators.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    opset_version = kwargs['opset_version']\n    if opset_version < 11:\n        raise AttributeError(\"ONNX opset 11 or greater is required to export this operator\")\n\n    # use the same dtype as the that of the input node\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n    axis = attrs.get('axis', 'None')\n    start = attrs.get('start', 0.)\n    step = attrs.get('step', 1.)\n    repeat = int(attrs.get('repeat', 1))\n    if repeat != 1:\n        raise NotImplementedError(\"arange_like operator with repeat != 1 not yet implemented.\")\n\n    create_const_scalar_node(name+\"_start\", np.dtype(dtype).type(start), kwargs)\n    create_const_scalar_node(name+\"_step\", np.dtype(dtype).type(step), kwargs)\n    create_const_scalar_node(name+\"_half_step\", np.dtype(dtype).type(float(step)*0.5), kwargs)\n\n    nodes = []\n    if axis == 'None':\n        # output will be same shape as input\n        nodes += [\n            make_node('Shape', [input_nodes[0]], [name+\"_shape0_out\"]),\n            make_node(\"ReduceProd\", [name+\"_shape0_out\"], [name+\"_redprod0_out\"]),\n            make_node('Squeeze', [name+'_redprod0_out'], [name+'_reshape0_out'], axes=[0]),\n            make_node(\"Cast\", [name+\"_reshape0_out\"], [name+\"_cast0_out\"], to=dtype_t),\n            make_node(\"Mul\", [name+\"_cast0_out\", name+\"_step\"], [name+\"_mul0_out\"]),\n            make_node(\"Add\", [name+\"_mul0_out\", name+\"_start\"], [name+\"_add1_out\"]),\n            make_node(\"Sub\", [name+\"_add1_out\", name+\"_half_step\"], [name+\"_sub0_out\"]),\n            make_node(\"Range\", [name+\"_start\", name+\"_sub0_out\", name+\"_step\"], [name+\"_range0_out\"]),\n            make_node(\"Reshape\", [name+\"_range0_out\", name+\"_shape0_out\"], [name], name=name)\n        ]\n    else:\n        # determine shape of axis\n        create_tensor([int(axis)], name+\"_axis_start\", kwargs[\"initializer\"], dtype='int64')\n        create_tensor([int(axis)+1], name+\"_axis_end\", kwargs[\"initializer\"], dtype='int64')\n        nodes += [\n            make_node('Shape', [input_nodes[0]], [name+\"_shape0_out\"]),\n            make_node('Slice', [name+\"_shape0_out\", name+\"_axis_start\", name+\"_axis_end\"], [name+\"_slice0_out\"]),\n            make_node(\"ReduceProd\", [name+\"_slice0_out\"], [name+\"_reprod0_out\"]),\n            make_node('Squeeze', [name+'_reprod0_out'], [name+'_reshape0_out'], axes=[0]),\n            make_node(\"Cast\", [name+\"_reshape0_out\"], [name+\"_cast0_out\"], to=dtype_t),\n            make_node(\"Mul\", [name+\"_cast0_out\", name+\"_step\"], [name+\"_mul0_out\"]),\n            make_node(\"Add\", [name+\"_mul0_out\", name+\"_start\"], [name+\"_add1_out\"]),\n            make_node(\"Sub\", [name+\"_add1_out\", name+\"_half_step\"], [name+\"_sub0_out\"]),\n            make_node(\"Range\", [name+\"_start\", name+\"_sub0_out\", name+\"_step\"], [name], name=name)\n        ]\n\n    return nodes\n\n\n@mx_op.register(\"_contrib_BilinearResize2D\")\ndef convert_contrib_BilinearResize2D(node, **kwargs):\n    \"\"\"Map MXNet's contrib_BilinearResize2D operator attributes to onnx.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    opset_version = kwargs['opset_version']\n    if opset_version < 11:\n        raise AttributeError(\"ONNX opset 11 or greater is required to export this operator\")\n\n    height = int(attrs.get('height', 0))\n    width = int(attrs.get('width', 0))\n\n    scale_height = float(attrs.get('scale_height', 0))\n    scale_width = float(attrs.get('scale_width', 0))\n\n    if height * width == 0 and scale_height * scale_width == 0:\n        raise AttributeError('height, width or scale_height, scale_width cannot be 0')\n\n    mode = attrs.get('mode', 'size')\n    if mode != 'size':\n        raise NotImplementedError('contrib_BilinearResize2D with mode other than \"size\" is \\\n                                   not supported')\n\n    create_tensor([], name+'_roi', kwargs['initializer'], dtype='float32')\n    create_tensor([], name+'_scales_empty', kwargs['initializer'],\n                  dtype='float32')\n\n    nodes = []\n    if scale_height == 0:\n        create_tensor([0], name+'_0', kwargs['initializer'])\n        create_tensor([2], name+'_2', kwargs['initializer'])\n        create_tensor([height, width], name+'_h_w', kwargs['initializer'], dtype='int64')\n        nodes += [\n            make_node('Shape', [input_nodes[0]], [name+'_shape']),\n            make_node('Slice', [name+'_shape', name+'_0', name+'_2'], [name+'_shape_01']),\n            make_node('Concat', [name+'_shape_01', name+'_h_w'], [name+'_sizes'], axis=0),\n        ]\n    else:\n        create_tensor([1, 1, scale_height, scale_width], name+'_scales', kwargs['initializer'],\n                      dtype='float32')\n        nodes += [\n            make_node('Shape', [input_nodes[0]], [name+'_shape']),\n            make_node('Cast', [name+'_shape'], [name+'_shape_f'], to=int(TensorProto.FLOAT)),\n            make_node('Mul', [name+'_shape_f', name+'_scales'], [name+'_sizes_']),\n            make_node('Cast', [name+'_sizes_'], [name+'_sizes'], to=int(TensorProto.INT64)),\n        ]\n    nodes += [\n        make_node('Resize', [input_nodes[0], name+'_roi', name+'_scales_empty', name+'_sizes'], [name],\n                  mode='linear', coordinate_transformation_mode='align_corners', name=name)\n    ]\n\n    return nodes\n\n\n@mx_op.register(\"_arange\")\n@mx_op.register(\"_npi_arange\")\ndef convert_arange(node, **kwargs):\n    \"\"\"Map MXNet's arange operator attributes to onnx's Range operator.\n    \"\"\"\n    from onnx.helper import make_node\n    name, _, attrs = get_inputs(node, kwargs)\n\n    opset_version = kwargs['opset_version']\n    if opset_version < 11:\n        raise AttributeError(\"ONNX opset 11 or greater is required to export this operator\")\n\n    start = attrs.get('start', 0.)\n    stop = attrs.get('stop')\n    step = attrs.get('step', 1.)\n    dtype = attrs.get('dtype', 'float32')\n    repeat = int(attrs.get('repeat', 1))\n\n    if stop == 'None':\n        stop = start\n        start = 0\n\n    if repeat != 1:\n        raise NotImplementedError(\"arange operator with repeat != 1 not yet implemented.\")\n\n    create_const_scalar_node(name+\"_start\", np.dtype(dtype).type(start), kwargs)\n    create_const_scalar_node(name+\"_stop\", np.dtype(dtype).type(stop), kwargs)\n    create_const_scalar_node(name+\"_step\", np.dtype(dtype).type(step), kwargs)\n\n    nodes = [\n        make_node(\"Range\", [name+\"_start\", name+\"_stop\", name+\"_step\"], [name], name=name)\n    ]\n\n    return nodes, (dtype,)\n\n\n@mx_op.register(\"reverse\")\ndef convert_reverse(node, **kwargs):\n    \"\"\"Map MXNet's reverse operator attributes to ONNX\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    axis = int(attrs.get('axis', 0))\n\n    # Transpose takes perm as a parameter, so we must 'pad' the input to a known dim (8 here)\n    perm = [i for i in range(8)]\n    perm[0], perm[axis] = axis, 0\n\n    create_tensor([8], name+'_8', kwargs['initializer'])\n    create_tensor([0], name+'_0', kwargs['initializer'])\n    create_tensor([1], name+'_1', kwargs['initializer'])\n    create_tensor([-1], name+'_m1', kwargs['initializer'])\n    create_tensor([axis], name+'_axis', kwargs['initializer'])\n    create_tensor([axis+1], name+'_axis_p1', kwargs['initializer'])\n    create_const_scalar_node(name+'_m1_s', np.int64(-1), kwargs)\n\n    nodes = [\n        make_node('Shape', [input_nodes[0]], [name+'_shape']),\n        make_node('Shape', [name+'_shape'], [name+'_dim']),\n        make_node('Sub', [name+'_8', name+'_dim'], [name+'_sub']),\n        make_node('Concat', [name+'_0', name+'_sub'], [name+'_concat'], axis=0),\n        make_node('Pad', [name+'_shape', name+'_concat', name+'_1'], [name+'_shape_8_dim']),\n        make_node('Reshape', [input_nodes[0], name+'_shape_8_dim'], [name+'_data_8_dim']),\n        make_node('Transpose', [name+'_data_8_dim'], [name+'_data_t'], perm=perm),\n        make_node('Slice', [name+'_shape', name+'_axis', name+'_axis_p1'], [name+'_axis_len']),\n        make_node('Sub', [name+'_axis_len', name+'_1'], [name+'_axis_len_m1']),\n        make_node('Squeeze', [name+'_axis_len_m1'], [name+'_axis_len_m1_s'], axes=[0]),\n        make_node('Range', [name+'_axis_len_m1_s', name+'_m1_s', name+'_m1_s'], [name+'_indices']),\n        make_node('Gather', [name+'_data_t', name+'_indices'], [name+'_gather']),\n        make_node('Transpose', [name+'_gather'], [name+'_data_reversed'], perm=perm),\n        make_node('Reshape', [name+'_data_reversed', name+'_shape'], [name], name=name)\n    ]\n\n    return nodes\n\n\n@mx_op.register('repeat')\ndef convert_repeat(node, **kwargs):\n    \"\"\"Map MXNet's repeat operator attributes to onnx's Tile operator.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    opset_version = kwargs['opset_version']\n    if opset_version < 11:\n        raise AttributeError('ONNX opset 11 or greater is required to export this operator')\n\n    repeats = int(attrs.get('repeats', 1))\n    axis = attrs.get('axis', 'None')\n\n    if repeats <= 0:\n        raise NotImplementedError('repeat operator does not support parameter repeats==0')\n\n    nodes = []\n    if axis == 'None':\n        create_tensor([repeats], name+'_rep', kwargs['initializer'])\n        create_tensor([1, repeats], name+'_repeats', kwargs['initializer'])\n        nodes += [\n            make_node('Shape', [input_nodes[0]], [name+'_shape']),\n            make_node('ReduceProd', [name+'_shape'], [name+'_size']),\n            make_node('Reshape', [input_nodes[0], name+'_size'], [name+'_flat']),\n            make_node('Unsqueeze', [name+'_flat'], [name+'_unsqueeze'], axes=[-1]),\n            make_node('Tile', [name+'_unsqueeze', name+'_repeats'], [name+'_tile']),\n            make_node('Mul', [name+'_size', name+'_rep'], [name+'_new_size']),\n            make_node('Reshape', [name+'_tile', name+'_new_size'], [name], name=name)\n        ]\n    else:\n        axis = int(axis)\n        repeats -= 1\n        create_tensor([repeats], name+'_repeats', kwargs['initializer'])\n        create_tensor([1], name+'_1', kwargs['initializer'])\n        create_tensor([0], name+'_0', kwargs['initializer'])\n        create_tensor([axis], name+'_axis', kwargs['initializer'])\n        create_const_scalar_node(name+\"_0_s\", np.int64(0), kwargs)\n        create_const_scalar_node(name+\"_1_s\", np.int64(1), kwargs)\n        nodes += [\n            make_node('Shape', [input_nodes[0]], [name+'_shape']),\n            make_node('Shape', [name+'_shape'], [name+'_dim']),\n            make_node('Squeeze', [name+'_dim'], [name+'_dim_s'], axes=[0]),\n            make_node('Range', [name+'_0_s', name+'_dim_s', name+'_1_s'], [name+'_range'])\n        ]\n        if axis < 0:\n            nodes += [\n                make_node('Add', [name+'_axis', name+'_dim'], [name+'_true_axis']),\n                make_node('Equal', [name+'_range', name+'_true_axis'], [name+'_one_hot'])\n                ]\n        else:\n            nodes += [\n                make_node('Equal', [name+'_range', name+'_axis'], [name+'_one_hot'])\n                ]\n        nodes += [\n            make_node('Cast', [name+'_one_hot'], [name+'_one_hot_int'], to=int(TensorProto.INT64)),\n            make_node('Mul', [name+'_repeats', name+'_one_hot_int'], [name+'_mul']),\n            make_node('Add', [name+'_mul', name+'_1'], [name+'_add']),\n            make_node('Concat', [name+'_1', name+'_add'], [name+'_repeats_tensor'], axis=0)\n            ]\n        if axis == -1:\n            nodes += [\n                make_node('Concat', [name+'_shape', name+'_1'], [name+'_unsqueeze_shape'], axis=0),\n                make_node('Reshape', [input_nodes[0], name+'_unsqueeze_shape'],\n                          [name+'_unsqueeze'])\n                ]\n        else:\n            nodes += [\n                make_node('Unsqueeze', [input_nodes[0]], [name+'_unsqueeze'], axes=[axis+1])\n                ]\n        nodes += [\n            make_node('Tile', [name+'_unsqueeze', name+'_repeats_tensor'], [name+'_tile']),\n            make_node('Mul', [name+'_shape', name+'_add'], [name+'_new_shape']),\n            make_node('Reshape', [name+'_tile', name+'_new_shape'], [name], name=name)\n            ]\n\n    return nodes\n\n\n@mx_op.register('_contrib_box_nms')\ndef convert_contrib_box_nms(node, **kwargs):\n    \"\"\"Map MXNet's _contrib_box_nms operator to ONNX\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n    #dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    opset_version = kwargs['opset_version']\n    if opset_version < 11:\n        raise AttributeError('ONNX opset 11 or greater is required to export this operator')\n\n    overlap_thresh = float(attrs.get('overlap_thresh', '0.5'))\n    valid_thresh = float(attrs.get('valid_thresh', '0'))\n    topk = int(attrs.get('topk', '-1'))\n    coord_start = int(attrs.get('coord_start', '2'))\n    score_index = int(attrs.get('score_index', '1'))\n    id_index = int(attrs.get('id_index', '-1'))\n    force_suppress = attrs.get('force_suppress', 'True')\n    background_id = int(attrs.get('background_id', '-1'))\n    in_format = attrs.get('in_format', 'corner')\n    out_format = attrs.get('out_format', 'corner')\n\n    center_point_box = 0 if in_format == 'corner' else 1\n\n    if topk == -1:\n        topk = 2**31-1\n\n    if in_format != out_format:\n        raise NotImplementedError('box_nms does not currently support in_fomat != out_format')\n\n    if background_id != -1:\n        raise NotImplementedError('box_nms does not currently support background_id != -1')\n\n    if id_index != -1 or force_suppress == 'False':\n        logging.warning('box_nms: id_idex != -1 or/and force_suppress == False detected. '\n                        'However, due to ONNX limitations, boxes of different categories will NOT '\n                        'be exempted from suppression. This might lead to different behavior than '\n                        'native MXNet')\n\n    create_tensor([coord_start], name+'_cs', kwargs['initializer'])\n    create_tensor([coord_start+4], name+'_cs_p4', kwargs['initializer'])\n    create_tensor([score_index], name+'_si', kwargs['initializer'])\n    create_tensor([score_index+1], name+'_si_p1', kwargs['initializer'])\n    create_tensor([topk], name+'_topk', kwargs['initializer'])\n    create_tensor([overlap_thresh], name+'_ot', kwargs['initializer'], dtype=np.float32)\n    create_tensor([valid_thresh], name+'_vt', kwargs['initializer'], dtype=np.float32)\n    create_tensor([-1], name+'_m1', kwargs['initializer'])\n    create_tensor([-1], name+'_m1_f', kwargs['initializer'], dtype=dtype)\n    create_tensor([0], name+'_0', kwargs['initializer'])\n    create_tensor([1], name+'_1', kwargs['initializer'])\n    create_tensor([2], name+'_2', kwargs['initializer'])\n    create_tensor([3], name+'_3', kwargs['initializer'])\n    create_tensor([0, 1, -1], name+'_scores_shape', kwargs['initializer'])\n    create_tensor([0, 0, 1, 0], name+'_pad', kwargs['initializer'])\n    create_tensor([0, -1], name+'_bat_spat_helper', kwargs['initializer'])\n    create_const_scalar_node(name+\"_0_s\", np.int64(0), kwargs)\n    create_const_scalar_node(name+\"_1_s\", np.int64(1), kwargs)\n\n    nodes = [\n        make_node('Shape', [input_nodes[0]], [name+'_shape']),\n        make_node('Shape', [name+'_shape'], [name+'_dim']),\n        make_node('Sub', [name+'_dim', name+'_2'], [name+'_dim_m2']),\n        make_node('Slice', [name+'_shape', name+'_dim_m2', name+'_dim'], [name+'_shape_last2']),\n        make_node('Concat', [name+'_m1', name+'_shape_last2'], [name+'_shape_3d'], axis=0),\n        make_node('Reshape', [input_nodes[0], name+'_shape_3d'], [name+'_data_3d']),\n        make_node('Slice', [name+'_data_3d', name+'_cs', name+'_cs_p4', name+'_m1'],\n                  [name+'_boxes']),\n        make_node('Slice', [name+'_data_3d', name+'_si', name+'_si_p1', name+'_m1'],\n                  [name+'_scores_raw']),\n        make_node('Reshape', [name+'_scores_raw', name+'_scores_shape'], [name+'_scores']),\n        make_node('Shape', [name+'_scores'], [name+'_scores_shape_actual']),\n        make_node('NonMaxSuppression',\n                  [name+'_boxes', name+'_scores', name+'_topk', name+'_ot', name+'_vt'],\n                  [name+'_nms'], center_point_box=center_point_box),\n        make_node('Slice', [name+'_nms', name+'_0', name+'_3', name+'_m1', name+'_2'],\n                  [name+'_nms_sliced']),\n        make_node('GatherND', [name+'_data_3d', name+'_nms_sliced'], [name+'_candidates']),\n        make_node('Pad', [name+'_candidates', name+'_pad', name+'_m1_f'], [name+'_cand_padded']),\n        make_node('Shape', [name+'_nms'], [name+'_nms_shape']),\n        make_node('Slice', [name+'_nms_shape', name+'_0', name+'_1'], [name+'_cand_cnt']),\n        make_node('Squeeze', [name+'_cand_cnt'], [name+'_cc_s'], axes=[0]),\n        make_node('Range', [name+'_0_s', name+'_cc_s', name+'_1_s'], [name+'_cand_indices']),\n        make_node('Slice', [name+'_scores_shape_actual', name+'_0', name+'_3', name+'_m1',\n                            name+'_2'], [name+'_shape_bat_spat']),\n        make_node('Slice', [name+'_shape_bat_spat', name+'_1', name+'_2'], [name+'_spat_dim']),\n        make_node('Expand', [name+'_cand_cnt', name+'_shape_bat_spat'], [name+'_base_indices']),\n        make_node('ScatterND', [name+'_base_indices', name+'_nms_sliced', name+'_cand_indices'],\n                  [name+'_indices']),\n        make_node('TopK', [name+'_indices', name+'_spat_dim'], [name+'_indices_sorted', name+'__'],\n                  largest=0, axis=-1, sorted=1),\n        make_node('Gather', [name+'_cand_padded', name+'_indices_sorted'], [name+'_gather']),\n        make_node('Reshape', [name+'_gather', name+'_shape'], [name+'0'])\n    ]\n\n    return nodes\n\n\n@mx_op.register(\"_greater_scalar\")\ndef convert_greater_scalar(node, **kwargs):\n    \"\"\"Map MXNet's greater_scalar operator attributes to onnx's Greater\n    operator and return the created node.\n    \"\"\"\n    from onnx.helper import make_node, make_tensor\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    scalar = float(attrs.get('scalar'))\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    if str(dtype).startswith('int'):\n        scalar = int(scalar)\n    else:\n        if dtype == 'float16':\n            # when using float16, we must convert it to np.uint16 view first\n            scalar = np.float16(scalar).view(np.uint16)\n    tensor_value = make_tensor(name+\"_scalar\", dtype_t, [1], [scalar])\n    nodes = [\n        make_node(\"Constant\", [], [name+\"_rhs\"], value=tensor_value),\n        make_node(\"Greater\", [input_nodes[0], name+\"_rhs\"], [name+\"_gt\"]),\n        make_node(\"Cast\", [name+\"_gt\"], [name], to=dtype_t, name=name)\n    ]\n    return nodes\n\n\n@mx_op.register(\"_lesser_scalar\")\ndef convert_lesser_scalar(node, **kwargs):\n    \"\"\"Map MXNet's lesser_scalar operator attributes to onnx's Less\n    operator and return the created node.\n    \"\"\"\n    from onnx.helper import make_node, make_tensor\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    scalar = float(attrs.get('scalar'))\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    if str(dtype).startswith('int'):\n        scalar = int(scalar)\n    else:\n        if dtype == 'float16':\n            # when using float16, we must convert it to np.uint16 view first\n            scalar = np.float16(scalar).view(np.uint16)\n\n    tensor_value = make_tensor(name+\"_scalar\", dtype_t, [1], [scalar])\n    nodes = [\n        make_node(\"Constant\", [], [name+\"_rhs\"], value=tensor_value),\n        make_node(\"Less\", [input_nodes[0], name+\"_rhs\"], [name+\"_lt\"]),\n        make_node(\"Cast\", [name+\"_lt\"], [name], to=dtype_t, name=name)\n    ]\n    return nodes\n\n\n@mx_op.register(\"_equal_scalar\")\ndef convert_equal_scalar(node, **kwargs):\n    \"\"\"Map MXNet's equal_scalar operator attributes to onnx.\n    \"\"\"\n    from onnx.helper import make_node, make_tensor\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    scalar = float(attrs.get('scalar'))\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    if str(dtype).startswith('int'):\n        scalar = int(scalar)\n    else:\n        if dtype == 'float16':\n            # when using float16, we must convert it to np.uint16 view first\n            scalar = np.float16(scalar).view(np.uint16)\n\n    tensor_value = make_tensor(name+\"_scalar\", dtype_t, [1], [scalar])\n    nodes = [\n        make_node(\"Constant\", [], [name+\"_rhs\"], value=tensor_value),\n        make_node(\"Equal\", [input_nodes[0], name+\"_rhs\"], [name+\"_eq\"]),\n        make_node(\"Cast\", [name+\"_eq\"], [name], to=dtype_t, name=name)\n    ]\n    return nodes\n\n\n@mx_op.register('where')\n@mx_op.register('_npi_where')\ndef convert_where(node, **kwargs):\n    \"\"\"Map MXNet's where operator attributes to onnx's Where\n    operator and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    # note that in mxnet the condition tensor can either have the same shape as x and y OR\n    # have shape (first dim of x,)\n    create_tensor([0], name+'_0', kwargs['initializer'])\n    create_tensor([1], name+'_1', kwargs['initializer'])\n    nodes = [\n        make_node('Shape', [input_nodes[0]], [name+'_cond_shape']),\n        make_node('Shape', [name+'_cond_shape'], [name+'_cond_dim']),\n        make_node('Shape', [input_nodes[1]], [name+'_x_shape']),\n        make_node('Shape', [name+'_x_shape'], [name+'_x_dim']),\n        make_node('Sub', [name+'_x_dim', name+'_cond_dim'], [name+'_sub']),\n        make_node('Concat', [name+'_0', name+'_sub'], [name+'_concat'], axis=0),\n        make_node('Pad', [name+'_cond_shape', name+'_concat', name+'_1'], [name+'_cond_new_shape']),\n        make_node('Reshape', [input_nodes[0], name+'_cond_new_shape'], [name+'_cond']),\n        make_node('Cast', [name+'_cond'], [name+'_bool'], to=int(TensorProto.BOOL)),\n        make_node('Where', [name+'_bool', input_nodes[1], input_nodes[2]], [name], name=name)\n    ]\n    return nodes\n\n\n@mx_op.register('_maximum_scalar')\ndef convert_maximum_scalar(node, **kwargs):\n    \"\"\"Map MXNet's _maximum_scalar\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    input_dtypes = get_input_dtypes(node, kwargs)\n    dtype = input_dtypes[0]\n\n    scalar = None\n    if 'float' in str(dtype):\n        scalar = float(attrs.get('scalar', '0'))\n    else:\n        scalar = int(attrs.get('scalar', '0'))\n\n    create_tensor([scalar], name+'_scalar', kwargs['initializer'], dtype=dtype)\n    nodes = [\n        make_node('Max', [input_nodes[0], name+'_scalar'], [name], name=name)\n    ]\n\n    return nodes\n\n@mx_op.register('_minimum_scalar')\ndef convert_minimum_scalar(node, **kwargs):\n    \"\"\"Map MXNet's _minimum_scalar\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    input_dtypes = get_input_dtypes(node, kwargs)\n    dtype = input_dtypes[0]\n\n    scalar = None\n    if 'float' in str(dtype):\n        scalar = float(attrs.get('scalar', '0'))\n    else:\n        scalar = int(attrs.get('scalar', '0'))\n\n    create_tensor([scalar], name+'_scalar', kwargs['initializer'], dtype=dtype)\n    nodes = [\n        make_node('Min', [input_nodes[0], name+'_scalar'], [name], name=name)\n    ]\n\n    return nodes\n\n@mx_op.register(\"_contrib_box_decode\")\ndef convert_contrib_box_decode(node, **kwargs):\n    \"\"\"Map MXNet's _contrib_box_decode operator attributes to onnx's operator.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    data = input_nodes[0]\n    anchors = input_nodes[1]\n    fmt = attrs.get('format', 'center')\n    std0 = float(attrs.get('std0', '1.'))\n    std1 = float(attrs.get('std1', '1.'))\n    std2 = float(attrs.get('std2', '1.'))\n    std3 = float(attrs.get('std3', '1.'))\n    clip = float(attrs.get('clip', '-1.'))\n\n    if fmt not in ['center', 'corner']:\n        raise NotImplementedError(\"format must be either corner or center.\")\n\n    create_tensor([0], name+'_0', kwargs[\"initializer\"])\n    create_tensor([2], name+'_2', kwargs[\"initializer\"])\n    create_tensor([4], name+'_4', kwargs[\"initializer\"])\n    create_tensor([2], name+'_2f', kwargs[\"initializer\"], dtype='float32')\n    create_tensor([clip], name+'_clip', kwargs[\"initializer\"], dtype='float32')\n    create_tensor([std0, std1, std2, std3], name+'_std_1d', kwargs[\"initializer\"], dtype='float32')\n    create_tensor([1, 4], name+'_std_shape', kwargs[\"initializer\"])\n\n    nodes = [\n        make_node(\"Cast\", [data], [name+'_data'], to=int(onnx.TensorProto.FLOAT)),\n        make_node(\"Cast\", [anchors], [name+'_anchors'], to=int(onnx.TensorProto.FLOAT)),\n        make_node('Reshape', [name+'_std_1d', name+'_std_shape'], [name+'_std']),\n        make_node(\"Mul\", [name+'_data', name+'_std'], [name+'_mul0_out']),\n        make_node('Slice', [name+'_mul0_out', name+'_0', name+'_2', name+'_2'], [name+'_data_xy']),\n        make_node('Slice', [name+'_mul0_out', name+'_2', name+'_4', name+'_2'], [name+'_data_wh']),\n    ]\n\n    if fmt == 'corner':\n        nodes += [\n            make_node('Slice', [name+'_anchors', name+'_0', name+'_2', name+'_2'], [name+'_slice0_out']),\n            make_node('Slice', [name+'_anchors', name+'_2', name+'_4', name+'_2'], [name+'_slice1_out']),\n            make_node('Sub', [name+'_slice1_out', name+'_slice0_out'], [name+'_anchor_wh']),\n            make_node('Div', [name+'_anchor_wh', name+'_2f'], [name+'_div0_out']),\n            make_node(\"Add\", [name+'_slice0_out', name+'_div0_out'], [name+'_anchor_xy']),\n        ]\n    else:\n        nodes += [\n            make_node('Slice', [name+'_anchors', name+'_0', name+'_2', name+'_2'], [name+'_anchor_xy']),\n            make_node('Slice', [name+'_anchors', name+'_2', name+'_4', name+'_2'], [name+'_anchor_wh']),\n        ]\n\n    nodes += [\n        make_node(\"Mul\", [name+'_data_xy', name+'_anchor_wh'], [name+'_mul1_out']),\n        make_node(\"Add\", [name+'_mul1_out', name+'_anchor_xy'], [name+'_add0_out']),\n    ]\n\n    if clip > 0.:\n        nodes += [\n            make_node(\"Less\", [name+\"_data_wh\", name+\"_clip\"], [name+\"_less0_out\"]),\n            make_node('Where', [name+'_less0_out', name+'_data_wh', name+'_clip'], [name+'_where0_out']),\n            make_node(\"Exp\", [name+'_where0_out'], [name+'_exp0_out']),\n        ]\n    else:\n        nodes += [\n            make_node(\"Exp\", [name+'_data_wh'], [name+'_exp0_out']),\n        ]\n\n    nodes += [\n        make_node(\"Mul\", [name+'_exp0_out', name+'_anchor_wh'], [name+'_mul2_out']),\n        make_node('Div', [name+'_mul2_out', name+'_2f'], [name+'_div1_out']),\n        make_node('Sub', [name+'_add0_out', name+'_div1_out'], [name+'_sub0_out']),\n        make_node('Add', [name+'_add0_out', name+'_div1_out'], [name+'_add1_out']),\n        make_node('Concat', [name+'_sub0_out', name+'_add1_out'], [name+'concat0_out'], axis=2),\n        make_node(\"Cast\", [name+'concat0_out'], [name], to=dtype_t, name=name)\n    ]\n\n    return nodes\n\n@mx_op.register(\"_contrib_AdaptiveAvgPooling2D\")\ndef convert_contrib_AdaptiveAvgPooling2D(node, **kwargs):\n    \"\"\"Map MXNet's _contrib_AdaptiveAvgPooling2D operator\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    output_size = attrs.get('output_size', '1')\n    output_size = convert_string_to_list(output_size)\n\n    if len(output_size) <= 2:\n        if output_size[0] != 1 or (len(output_size) == 2 and output_size[1] != 1):\n            raise NotImplementedError(\"_contrib_AdaptiveAvgPooling2D operator with output_size != 1 \\\n                                not yet implemented.\")\n    nodes = [\n        make_node(\"GlobalAveragePool\", [input_nodes[0]], [name], name=name)\n    ]\n\n    return nodes\n\n\n@mx_op.register('broadcast_mod')\n@mx_op.register('_npi_mod')\ndef convert_broadcast_mod(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_mod operator\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, _ = get_inputs(node, kwargs)\n\n    # The behavior of MXNet mod is a mixture of np.mod and np.fmod\n    # note: the behavior when divison by 0 is supposed to be platform dependent\n    #       but here we set the result to 0 to be consistent with MXNet\n    nodes = [\n        make_node('Sub', [input_nodes[1], input_nodes[1]], [name+'_zero']),\n        make_node('Mod', [input_nodes[0], input_nodes[1]], [name+'_mod'], fmod=1),\n        make_node('Less', [input_nodes[0], name+'_zero'], [name+'_mask_0']),\n        make_node('Less', [input_nodes[1], name+'_zero'], [name+'_mask_1']),\n        make_node('Equal', [name+'_mod', name+'_zero'], [name+'_mask_2_']),\n        make_node('Not', [name+'_mask_2_'], [name+'_mask_2']),\n        make_node('Xor', [name+'_mask_0', name+'_mask_1'], [name+'_mask_']),\n        make_node('And', [name+'_mask_', name+'_mask_2'], [name+'_mask']),\n        make_node('Where', [name+'_mask', input_nodes[1], name+'_zero'], [name+'_adjustment']),\n        make_node('Add', [name+'_mod', name+'_adjustment'], [name+'_adjusted']),\n        make_node('Equal', [input_nodes[1], name+'_zero'], [name+'_mask_div_0']),\n        make_node('Where', [name+'_mask_div_0', name+'_zero', name+'_adjusted'], [name], name=name)\n        ]\n\n    return nodes\n\n\n@mx_op.register(\"reshape_like\")\ndef convert_reshape_like(node, **kwargs):\n    \"\"\"Map MXNet's reshape_like operator attributes to onnx's operator.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    lhs = input_nodes[0]\n    rhs = input_nodes[1]\n\n    lhs_begin = str(attrs.get('lhs_begin', '0'))\n    rhs_begin = str(attrs.get('rhs_begin', '0'))\n    lhs_end = str(attrs.get('lhs_end', 'None'))\n    rhs_end = str(attrs.get('rhs_end', 'None'))\n\n    if lhs_begin == 'None' or rhs_begin == 'None':\n        raise NotImplementedError(\"lhs_begin and rhs_begin should not be None.\")\n\n    lhs_begin = int(lhs_begin)\n    rhs_begin = int(rhs_begin)\n\n    # basic case\n    if lhs_begin == 0 and lhs_end == 'None' and rhs_begin == 0 and rhs_end == 'None':\n        nodes = [\n            make_node('Shape', [rhs], [name+'_shape_rhs']),\n            make_node('Reshape', [lhs, name+'_shape_rhs'], [name], name=name)\n        ]\n        return nodes\n\n    create_tensor([0], name+'_0', kwargs[\"initializer\"])\n    nodes = [\n        make_node('Shape', [lhs], [name+'_lhs_shape']),\n        make_node('Shape', [name+'_lhs_shape'], [name+'_lhs_dim']),\n        make_node('Shape', [rhs], [name+'_rhs_shape']),\n        make_node('Shape', [name+'_rhs_shape'], [name+'_rhs_dim']),\n    ]\n\n    if lhs_begin >= 0:\n        create_tensor([lhs_begin], name+'_lhs_begin', kwargs[\"initializer\"])\n    else:\n        create_tensor([lhs_begin], name+'_lhs_begin_neg', kwargs[\"initializer\"])\n        nodes += [\n            make_node('Add', [name+'_lhs_dim', name+'_lhs_begin_neg'], [name+'_lhs_begin']),\n        ]\n\n    if rhs_begin >= 0:\n        create_tensor([rhs_begin], name+'_rhs_begin', kwargs[\"initializer\"])\n    else:\n        create_tensor([rhs_begin], name+'_rhs_begin_neg', kwargs[\"initializer\"])\n        nodes += [\n            make_node('Add', [name+'_rhs_dim', name+'_rhs_begin_neg'], [name+'_rhs_begin']),\n        ]\n\n    if lhs_end == 'None':\n        nodes += [\n            make_node('Add', [name+'_lhs_dim', name+'_0'], [name+'_lhs_end']),\n        ]\n    else:\n        lhs_end = int(lhs_end)\n        if lhs_end >= 0:\n            create_tensor([lhs_end], name+'_lhs_end', kwargs[\"initializer\"])\n        else:\n            create_tensor([lhs_end], name+'_lhs_end_neg', kwargs[\"initializer\"])\n            nodes += [\n                make_node('Add', [name+'_lhs_dim', name+'_lhs_end_neg'], [name+'_lhs_end']),\n            ]\n\n    if rhs_end == 'None':\n        nodes += [\n            make_node('Add', [name+'_rhs_dim', name+'_0'], [name+'_rhs_end']),\n        ]\n    else:\n        rhs_end = int(rhs_end)\n        if rhs_end >= 0:\n            create_tensor([rhs_end], name+'_rhs_end', kwargs[\"initializer\"])\n        else:\n            create_tensor([rhs_end], name+'_rhs_end_neg', kwargs[\"initializer\"])\n            nodes += [\n                make_node('Add', [name+'_rhs_dim', name+'_rhs_end_neg'], [name+'_rhs_end']),\n            ]\n\n    nodes += [\n        make_node('Slice', [name+'_lhs_shape', name+'_0', name+'_lhs_begin'], [name+'_slice0_out']),\n        make_node('Slice', [name+'_rhs_shape', name+'_rhs_begin', name+'_rhs_end'], [name+'_slice1_out']),\n        make_node('Concat', [name+'_slice0_out', name+'_slice1_out'], [name+'_concat0_out'], axis=0),\n        make_node('Slice', [name+'_lhs_shape', name+'_lhs_end', name+'_lhs_dim'], [name+'_slice2_out']),\n        make_node('Concat', [name+'_concat0_out', name+'_slice2_out'], [name+'_concat1_out'], axis=0),\n        make_node('Reshape', [lhs, name+'_concat1_out'], [name], name=name)\n    ]\n\n    return nodes\n\n\n@mx_op.register(\"gather_nd\")\ndef convert_gather_nd(node, **kwargs):\n    \"\"\"Map MXNet's gather_ND operator attributes to onnx's operator.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, _ = get_inputs(node, kwargs)\n\n    data = input_nodes[0]\n    indices = input_nodes[1]\n\n    # Onnx Transpose operator takes perm as a parameter, so we need to 'pad'\n    # the input to a known dim (8 here)\n    perm = [7] + [i for i in range(1, 7)] + [0]\n\n    create_tensor([0], name+'_0', kwargs['initializer'])\n    create_tensor([1], name+'_1', kwargs['initializer'])\n    create_tensor([8], name+'_8', kwargs['initializer'])\n    nodes = [\n        # Generate 8-d filter\n        make_node('Shape', [indices], [name+'_indices_shape']),\n        make_node('Shape', [name+'_indices_shape'], [name+'_indices_dim']),\n        make_node('Sub', [name+'_8', name+'_indices_dim'], [name+'_sub0_out']),\n        make_node('Concat', [name+'_0', name+'_sub0_out'], [name+'_concat0_out'], axis=0),\n        make_node('Pad', [name+'_indices_shape', name+'_concat0_out', name+'_1'], [name+'_shape_8_dim']),\n        make_node('Reshape', [indices, name+'_shape_8_dim'], [name+'_indices_8_dim']),\n        make_node('Transpose', [name+'_indices_8_dim'], [name+'_transpose0_output'], perm=perm),\n        # Reshape filter to acutall dim for GatherND computation\n        make_node('Slice', [name+'_indices_shape', name+'_0', name+'_1'],\n                  [name+'_slice0_out']),\n        make_node('Slice', [name+'_indices_shape', name+'_1', name+'_indices_dim'],\n                  [name+'_slice1_out']),\n        make_node('Concat', [name+'_slice1_out', name+'_slice0_out'], [name+'_concat1_out'], axis=0),\n        make_node('Reshape', [name+'_transpose0_output', name+'_concat1_out'], [name+'_reshape0_out']),\n        # Cast data type for indicies\n        make_node('Cast', [name+'_reshape0_out'], [name+'_cast0_out'], to=int(onnx.TensorProto.INT64)),\n        make_node('GatherND', [data, name+'_cast0_out'], [name], name=name)\n    ]\n\n    return nodes\n\n\n@mx_op.register('UpSampling')\ndef convert_upsampling(node, **kwargs):\n    \"\"\"Map MXNet's UpSampling operator to onnx.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    scale = int(attrs.get('scale', '1'))\n    sample_type = attrs.get('sample_type')\n    num_args = int(attrs.get('num_args', '1'))\n\n    if num_args > 1:\n        raise NotImplementedError('Upsampling conversion does not currently support num_args > 1')\n\n    if sample_type != 'nearest':\n        raise NotImplementedError('Upsampling conversion does not currently support \\\n                                   sample_type != nearest')\n\n    create_tensor([], name+'_roi', kwargs['initializer'], dtype='float32')\n    create_tensor([1, 1, scale, scale], name+'_scales', kwargs['initializer'],\n                  dtype='float32')\n    nodes = [\n        make_node('Resize', [input_nodes[0], name+'_roi', name+'_scales'], [name], mode='nearest',\n                  coordinate_transformation_mode='half_pixel')\n    ]\n\n    return nodes\n\n\n@mx_op.register('SwapAxis')\ndef convert_swapaxis(node, **kwargs):\n    \"\"\"Map MXNet's SwapAxis operator\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    dim1 = int(attrs.get('dim1', '0'))\n    dim2 = int(attrs.get('dim2', '0'))\n\n    if dim1 < 0 or dim2 < 0:\n        raise NotImplementedError('SwapAxis conversion does not support dim1 < 0\\\n                                   or dim2 < 0')\n\n    indices = [[dim1], [dim2]]\n    vals = [dim2, dim1]\n    perm = [i for i in range(8)]\n    perm[dim1], perm[dim2] = dim2, dim1\n\n    create_tensor(indices, name+'_ind', kwargs['initializer'])\n    create_tensor(indices[::-1], name+'_ind_rev', kwargs['initializer'])\n    create_tensor(vals, name+'_vals', kwargs['initializer'])\n    create_tensor(perm, name+'_perm', kwargs['initializer'])\n    create_tensor([0], name+'_0', kwargs['initializer'])\n    create_tensor([1], name+'_1', kwargs['initializer'])\n    create_tensor([8], name+'_8', kwargs['initializer'])\n\n    nodes = [\n        make_node('Shape', [input_nodes[0]], [name+'_shape']),\n        make_node('Shape', [name+'_shape'], [name+'_dim']),\n        make_node('Sub', [name+'_8', name+'_dim'], [name+'_sub']),\n        make_node('ScatterND', [name+'_perm', name+'_ind', name+'_vals'],\n                  [name+'_perm_new']),\n        make_node('GatherND', [name+'_shape', name+'_ind'], [name+'_gather']),\n        make_node('ScatterND', [name+'_shape', name+'_ind_rev', name+'_gather'],\n                  [name+'_shape_new']),\n        make_node('Concat', [name+'_0', name+'_sub'], [name+'_pad'], axis=0),\n        make_node('Pad', [name+'_shape', name+'_pad', name+'_1'], [name+'_shape_padded']),\n        make_node('Reshape', [input_nodes[0], name+'_shape_padded'], [name+'_data_padded']),\n        make_node('Transpose', [name+'_data_padded'], [name+'_trans'], perm=perm),\n        make_node('Reshape', [name+'_trans', name+'_shape_new'], [name])\n    ]\n\n    return nodes\n\n\n@mx_op.register('slice_like')\ndef convert_slice_like(node, **kwargs):\n    \"\"\"Map MXNet's slice_like operator to onnx Slice operator.\"\"\"\n    from onnx.helper import make_node, make_tensor\n    from onnx import TensorProto\n\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    axes = convert_string_to_list(attrs.get('axes', 'None'))\n    zero = make_tensor(name+'_zero', TensorProto.INT64, [1], [0])\n\n    nodes = []\n    if axes == [None]:\n        nodes += [\n            make_node('Shape', [input_nodes[1]], [name+'_shape_1']),\n            make_node('Shape', [name+'_shape_1'], [name+'_dim_1']),\n            make_node('ConstantOfShape', [name+'_dim_1'], [name+'_starts'], value=zero),\n            make_node('Slice', [input_nodes[0], name+'_starts', name+'_shape_1'], [name])\n        ]\n    else:\n        axes = [[i] for i in axes]\n        create_tensor([0], name+'_0', kwargs['initializer'])\n        create_tensor(axes, name+'_axes_', kwargs['initializer'])\n        nodes += [\n            make_node('Shape', [input_nodes[0]], [name+'_shape_0']),\n            make_node('Shape', [input_nodes[1]], [name+'_shape_1']),\n            make_node('Shape', [name+'_shape_0'], [name+'_dim_0']),\n            make_node('Less', [name+'_axes_', name+'_0'], [name+'_less']),\n            make_node('Cast', [name+'_less'], [name+'_mask'], to=int(TensorProto.INT64)),\n            make_node('Mul', [name+'_mask', name+'_dim_0'], [name+'_mul']),\n            make_node('Add', [name+'_axes_', name+'_mul'], [name+'_axes']),\n            make_node('ConstantOfShape', [name+'_dim_0'], [name+'_starts'], value=zero),\n            make_node('GatherND', [name+'_shape_1', name+'_axes'], [name+'_gather']),\n            make_node('ScatterND', [name+'_shape_0', name+'_axes', name+'_gather'],\n                      [name+'_ends']),\n            make_node('Slice', [input_nodes[0], name+'_starts', name+'_ends'], [name])\n            ]\n\n    return nodes\n\n\n@mx_op.register(\"broadcast_like\")\ndef convert_broadcast_like(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_like operator attributes to onnx's operator.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    lhs = input_nodes[0]\n    rhs = input_nodes[1]\n    lhs_axes = convert_string_to_list(str(attrs.get('lhs_axes', 'None')))\n    rhs_axes = convert_string_to_list(str(attrs.get('rhs_axes', 'None')))\n\n    if lhs_axes[0] is None or rhs_axes[0] is None:\n        nodes = [\n            make_node('Shape', [rhs], [name+'_rhs_shape']),\n            make_node('Expand', [lhs, name+'_rhs_shape'], [name], name=name)\n        ]\n        return nodes\n\n    lhs_axes = [[i] for i in lhs_axes]\n    rhs_axes = [[i] for i in rhs_axes]\n\n    create_tensor([0], name+'_0', kwargs['initializer'])\n    create_tensor(lhs_axes, name+'_lhs_axes', kwargs['initializer'])\n    create_tensor(rhs_axes, name+'_rhs_axes', kwargs['initializer'])\n\n    nodes = [\n        make_node('Shape', [lhs], [name+'_lhs_shape']),\n        make_node('Shape', [rhs], [name+'_rhs_shape']),\n        make_node('Shape', [name+'_lhs_shape'], [name+'_lhs_dim']),\n        make_node('Less', [name+'_lhs_axes', name+'_0'], [name+'_less']),\n        make_node('Cast', [name+'_less'], [name+'_mask'], to=int(onnx.TensorProto.INT64)),\n        make_node('Mul', [name+'_mask', name+'_lhs_dim'], [name+'_mul']),\n        make_node('Add', [name+'_lhs_axes', name+'_mul'], [name+'_lhs_axes_positive']),\n        make_node('GatherND', [name+'_rhs_shape', name+'_rhs_axes'], [name+'_gather']),\n        make_node('ScatterND', [name+'_lhs_shape', name+'_lhs_axes_positive', name+'_gather'],\n                  [name+'_scatter']),\n        make_node('Expand', [lhs, name+'_scatter'], [name], name=name)\n    ]\n\n    return nodes\n\n\n@mx_op.register('_contrib_ROIAlign')\ndef convert_contrib_roialign(node, **kwargs):\n    \"\"\"Map MXNet's _contrib_ROIAlign\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    pooled_size = convert_string_to_list(str(attrs.get('pooled_size')))\n    spatial_scale = float(attrs.get('spatial_scale'))\n    sample_ratio = int(attrs.get('sample_ratio', '0'))\n    position_sensitive = attrs.get('position_sensitive', 'False')\n    aligned = attrs.get('aligned', 'False')\n\n    if position_sensitive != 'False':\n        raise NotImplementedError('_contrib_ROIAlign does not currently support \\\n                                   position_sensitive!=False')\n    if aligned != 'False':\n        raise NotImplementedError('_contrib_ROIAlign does not currently support \\\n                                   aligned!=False')\n\n    create_tensor([0], name+'_0', kwargs['initializer'])\n    create_tensor([0], name+'_0_s', kwargs['initializer'], dtype='float32')\n    create_tensor([1], name+'_1', kwargs['initializer'])\n    create_tensor([5], name+'_5', kwargs['initializer'])\n\n    nodes = [\n        make_node('Slice', [input_nodes[1], name+'_1', name+'_5', name+'_1'], [name+'_rois']),\n        make_node('Slice', [input_nodes[1], name+'_0', name+'_1', name+'_1'], [name+'_inds___']),\n        make_node('Squeeze', [name+'_inds___'], [name+'_inds__'], axes=[1]),\n        make_node('Relu', [name+'_inds__'], [name+'_inds_']),\n        make_node('Cast', [name+'_inds_'], [name+'_inds'], to=int(TensorProto.INT64)),\n        make_node('RoiAlign', [input_nodes[0], name+'_rois', name+'_inds'], [name+'_roi'],\n                  mode='avg', output_height=pooled_size[0], output_width=pooled_size[1],\n                  sampling_ratio=sample_ratio, spatial_scale=spatial_scale),\n        make_node('Unsqueeze', [name+'_inds___'], [name+'_unsq'], axes=(2, 3)),\n        make_node('Less', [name+'_unsq', name+'_0_s'], [name+'_less']),\n        make_node('Where', [name+'_less', name+'_0_s', name+'_roi'], [name])\n    ]\n\n    return nodes\n\n\n@mx_op.register(\"batch_dot\")\ndef convert_batch_dot(node, **kwargs):\n    \"\"\"Map MXNet's batch_dot operator attributes to onnx's operator.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    lhs = input_nodes[0]\n    rhs = input_nodes[1]\n    transpose_a = str(attrs.get('transpose_a', 'False'))\n    transpose_b = str(attrs.get('transpose_b', 'False'))\n    perm = [0, 2, 1]\n\n    if transpose_a == 'False' and transpose_b == 'False':\n        nodes = [\n            make_node('MatMul', [lhs, rhs], [name]),\n        ]\n        return nodes\n\n    create_tensor([-2], name+'_-2', kwargs['initializer'])\n    create_tensor([-1], name+'_-1', kwargs['initializer'])\n    create_tensor([0], name+'_0', kwargs['initializer'])\n    create_tensor([100], name+'_100', kwargs['initializer'])\n\n    nodes = []\n    if transpose_a != 'False' and transpose_b == 'False':\n        nodes += [\n            make_node('Shape', [lhs], [name+'_lhs_shape']),\n            make_node('Shape', [name+'_lhs_shape'], [name+'_lhs_dim']),\n            make_node('Slice', [name+'_lhs_shape', name+'_0', name+'_-2'],\n                      [name+'_lhs_slice0']),\n            make_node('Slice', [name+'_lhs_shape', name+'_-2', name+'_100'],\n                      [name+'_lhs_slice1']),\n            make_node('Concat', [name+'_-1', name+'_lhs_slice1'], [name+'_lhs_concat1'], axis=0),\n            make_node('Reshape', [lhs, name+'_lhs_concat1'], [name+'_lhs_3d']),\n            make_node('Transpose', [name+'_lhs_3d'], [name+'_lhs_3d_transpose'], perm=perm),\n            make_node('Shape', [name+'_lhs_3d_transpose'], [name+'_lhs_shape_3d']),\n            make_node('Slice', [name+'_lhs_shape_3d', name+'_-2', name+'_100'],\n                      [name+'_lhs_slice2']),\n            make_node('Concat', [name+'_lhs_slice0', name+'_lhs_slice2'], [name+'_lhs_concat2'], axis=0),\n            make_node('Reshape', [name+'_lhs_3d_transpose', name+'_lhs_concat2'], [name+'_lhs']),\n            make_node('MatMul', [name+'_lhs', rhs], [name]),\n        ]\n\n    elif transpose_a == 'False' and transpose_b != 'False':\n        nodes += [\n            make_node('Shape', [rhs], [name+'_rhs_shape']),\n            make_node('Shape', [name+'_rhs_shape'], [name+'_rhs_dim']),\n            make_node('Slice', [name+'_rhs_shape', name+'_0', name+'_-2'],\n                      [name+'_rhs_slice0']),\n            make_node('Slice', [name+'_rhs_shape', name+'_-2', name+'_100'],\n                      [name+'_rhs_slice1']),\n            make_node('Concat', [name+'_-1', name+'_rhs_slice1'], [name+'_rhs_concat1'], axis=0),\n            make_node('Reshape', [rhs, name+'_rhs_concat1'], [name+'_rhs_3d']),\n            make_node('Transpose', [name+'_rhs_3d'], [name+'_rhs_3d_transpose'], perm=perm),\n            make_node('Shape', [name+'_rhs_3d_transpose'], [name+'_rhs_shape_3d']),\n            make_node('Slice', [name+'_rhs_shape_3d', name+'_-2', name+'_100'],\n                      [name+'_rhs_slice2']),\n            make_node('Concat', [name+'_rhs_slice0', name+'_rhs_slice2'], [name+'_rhs_concat2'], axis=0),\n            make_node('Reshape', [name+'_rhs_3d_transpose', name+'_rhs_concat2'], [name+'_rhs']),\n            make_node('MatMul', [lhs, name+'_rhs'], [name]),\n        ]\n\n    else:\n        nodes += [\n            make_node('Shape', [lhs], [name+'_lhs_shape']),\n            make_node('Shape', [name+'_lhs_shape'], [name+'_lhs_dim']),\n            make_node('Slice', [name+'_lhs_shape', name+'_0', name+'_-2'],\n                      [name+'_lhs_slice0']),\n            make_node('Slice', [name+'_lhs_shape', name+'_-2', name+'_100'],\n                      [name+'_lhs_slice1']),\n            make_node('Concat', [name+'_-1', name+'_lhs_slice1'], [name+'_lhs_concat1'], axis=0),\n            make_node('Reshape', [lhs, name+'_lhs_concat1'], [name+'_lhs_3d']),\n            make_node('Transpose', [name+'_lhs_3d'], [name+'_lhs_3d_transpose'], perm=perm),\n            make_node('Shape', [name+'_lhs_3d_transpose'], [name+'_lhs_shape_3d']),\n            make_node('Slice', [name+'_lhs_shape_3d', name+'_-2', name+'_100'],\n                      [name+'_lhs_slice2']),\n            make_node('Concat', [name+'_lhs_slice0', name+'_lhs_slice2'], [name+'_lhs_concat2'], axis=0),\n            make_node('Reshape', [name+'_lhs_3d_transpose', name+'_lhs_concat2'], [name+'_lhs']),\n\n            make_node('Shape', [rhs], [name+'_rhs_shape']),\n            make_node('Shape', [name+'_rhs_shape'], [name+'_rhs_dim']),\n            make_node('Slice', [name+'_rhs_shape', name+'_0', name+'_-2'],\n                      [name+'_rhs_slice0']),\n            make_node('Slice', [name+'_rhs_shape', name+'_-2', name+'_100'],\n                      [name+'_rhs_slice1']),\n            make_node('Concat', [name+'_-1', name+'_rhs_slice1'], [name+'_rhs_concat1'], axis=0),\n            make_node('Reshape', [rhs, name+'_rhs_concat1'], [name+'_rhs_3d']),\n            make_node('Transpose', [name+'_rhs_3d'], [name+'_rhs_3d_transpose'], perm=perm),\n            make_node('Shape', [name+'_rhs_3d_transpose'], [name+'_rhs_shape_3d']),\n            make_node('Slice', [name+'_rhs_shape_3d', name+'_-2', name+'_100'],\n                      [name+'_rhs_slice2']),\n            make_node('Concat', [name+'_rhs_slice0', name+'_rhs_slice2'], [name+'_rhs_concat2'], axis=0),\n            make_node('Reshape', [name+'_rhs_3d_transpose', name+'_rhs_concat2'], [name+'_rhs']),\n            make_node('MatMul', [name+'_lhs', name+'_rhs'], [name]),\n        ]\n\n    return nodes\n\n\n@mx_op.register('log2')\n@mx_op.register('_npi_log2')\ndef convert_log2(node, **kwargs):\n    \"\"\"Map MXNet's log2 operator attributes to onnx's operator.\n    \"\"\"\n    from onnx.helper import make_node, make_tensor\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    ln2 = np.array([0.693147180559945309], dtype=dtype)\n    if dtype == 'float16':\n        ln2 = ln2.view(dtype=np.uint16)\n    ln2v = make_tensor(name+'_ln2', dtype_t, [1], ln2)\n\n    nodes = [\n        make_node('Log', [input_nodes[0]], [name+'_log']),\n        make_node('Constant', [], [name+'_ln2'], value=ln2v),\n        make_node('Div', [name+'_log', name+'_ln2'], [name], name=name)\n    ]\n\n    return nodes\n\n\n@mx_op.register('argsort')\ndef convert_argsort(node, **kwargs):\n    \"\"\"Map MXNet's argsort operator attributes to onnx's TopK operator\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    opset_version = kwargs['opset_version']\n    if opset_version < 11:\n        raise AttributeError('ONNX opset 11 or greater is required to export this operator')\n\n    axis = int(attrs.get('axis', '-1'))\n    is_ascend = attrs.get('is_ascend', 'True')\n    is_ascend = is_ascend in ['True', '1']\n    dtype = attrs.get('dtype', 'float32')\n\n    create_tensor([axis], name+'_axis', kwargs['initializer'])\n    nodes = [\n        make_node('Shape', [input_nodes[0]], [name+'_shape']),\n        make_node('Gather', [name+'_shape', name+'_axis'], [name+'_k'])\n    ]\n    if dtype == 'int64':\n        nodes += [\n            make_node('TopK', [input_nodes[0], name+'_k'], [name+'_', name], axis=axis,\n                      largest=(not is_ascend), sorted=1),\n        ]\n    else:\n        nodes += [\n            make_node('TopK', [input_nodes[0], name+'_k'], [name+'_', name+'_temp'], axis=axis,\n                      largest=(not is_ascend), sorted=1),\n            make_node('Cast', [name+'_temp'], [name],\n                      to=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(dtype)])\n        ]\n\n    return nodes\n\n\n@mx_op.register('one_hot')\ndef convert_one_hot(node, **kwargs):\n    \"\"\"Map MXNet's one_hot operator attributes to onnx's OneHot operator\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    depth = int(attrs.get('depth'))\n    on_value = float(attrs.get('on_value', 1.))\n    off_value = float(attrs.get('off_value', 0.))\n    dtype = attrs.get('dtype', 'float32')\n\n    create_tensor([off_value, on_value], name+'_values', kwargs['initializer'], dtype=np.dtype(dtype))\n    create_tensor([depth], name+'_depth', kwargs['initializer'])\n    nodes = [\n        make_node('Cast', [input_nodes[0]], [name+'_cast'], to=int(TensorProto.INT64)),\n        make_node('OneHot', [name+'_cast', name+'_depth', name+'_values'], [name], name=name)\n    ]\n\n    return nodes\n\n\n@mx_op.register('_random_uniform_like')\ndef convert_random_uniform_like(node, **kwargs):\n    \"\"\"Map MXNet's random_uniform_like operator attributes to onnx's RandomUniformLike operator\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    low = float(attrs.get('low', 0.))\n    high = float(attrs.get('high', 1.))\n\n    nodes = [\n        make_node('RandomUniformLike', [input_nodes[0]], [name], name=name,\n                  dtype=dtype_t, low=low, high=high)\n    ]\n\n    return nodes\n\n\n@mx_op.register('SequenceReverse')\ndef convert_sequence_reverse(node, **kwargs):\n    \"\"\"Map MXNet's SequenceReverse op\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    batch_axis = 1\n    time_axis = 0\n    use_sequence_length = attrs.get('use_sequence_length', 'False')\n\n    nodes = []\n    if use_sequence_length == 'False':\n        nodes += [\n            make_node('Shape', [input_nodes[0]], [name+'_shape']),\n            make_node('Split', [name+'_shape'], [name+'_dim0', name+'_dim1', name+'_dim2']),\n            make_node('Expand', [name+'_dim0', name+'_dim1'], [name+'_seq_len']),\n            make_node('ReverseSequence', [input_nodes[0], name+'_seq_len'], [name],\n                      batch_axis=batch_axis, time_axis=time_axis)\n        ]\n    else:\n        nodes += [\n            make_node('Cast', [input_nodes[1]], [name+'_seq_len'], to=int(TensorProto.INT64)),\n            make_node('ReverseSequence', [input_nodes[0], name+'_seq_len'], [name],\n                      batch_axis=batch_axis, time_axis=time_axis)\n        ]\n\n    return nodes\n\n\n@mx_op.register(\"RNN\")\ndef convert_RNN(node, **kwargs):\n    \"\"\"Map MXNet's RNN operator attributes to onnx's operators\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node, make_tensor\n    from onnx import TensorProto\n\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mode = str(attrs.get('mode'))\n    bidirectional = str(attrs.get('bidirectional', 'False'))\n    if bidirectional != 'False' and mode not in ['lstm']:\n        raise NotImplementedError('Currently RNN onnx export only supports bidirectional is False')\n\n    num_layers = int(attrs.get('num_layers', '1'))\n\n    use_sequence_length = str(attrs.get('use_sequence_length', 'False'))\n    if use_sequence_length != 'False':\n        raise NotImplementedError('Currently RNN onnx export only supports use_sequence_length equals to False')\n\n    projection_size = str(attrs.get('projection_size', 'None'))\n    if projection_size != 'None':\n        raise NotImplementedError('Currently RNN onnx export only supports projection_size equals to None')\n\n    state_outputs = str(attrs.get('state_outputs', 'False'))\n    if state_outputs != 'True':\n        raise NotImplementedError('Currently RNN onnx export only supports state_outputs equals to True')\n\n    state_size = int(attrs.get('state_size'))\n\n    direction = 1\n    if bidirectional != 'False':\n        direction = 2\n\n    data = input_nodes[0]\n    param = input_nodes[1]\n    dtype = get_input_dtypes(node, kwargs)[2]\n\n    create_tensor([0], name+'_0', kwargs['initializer'])\n    create_tensor([1], name+'_1', kwargs['initializer'])\n    create_tensor([state_size], name+'_state_size', kwargs['initializer'])\n    create_tensor([direction], name+'_direction', kwargs['initializer'])\n\n    tensor_1 = make_tensor(name+'_1_f', onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype], [1], [1])\n\n    nodes = [\n        make_node('Shape', [data], [name+'_data_shape']),\n        make_node('Split', [name+'_data_shape'], [name+'_seq_length', name+'_batch_size', name+'_input_size']),\n        make_node('Concat', [name+'_direction', name+'_batch_size', name+'_state_size'], [name+'_concat'], axis=0),\n        make_node('ConstantOfShape', [name+'_concat'], [name+'_COS'], value=tensor_1),\n        make_node('Mul', [input_nodes[2], name+'_COS'], [name+'initial_h']),\n\n    ]\n\n    if mode == 'lstm':\n        nodes += [\n            make_node('Mul', [input_nodes[3], name+'_COS'], [name+'initial_c']),\n        ]\n\n        if num_layers == 2:\n            if bidirectional != 'False':\n                raise NotImplementedError('Currently lstm onnx export only supports bidirectional when num_layers = 1')\n            create_tensor([8*state_size], name+'_8*state_size', kwargs['initializer'])\n            create_tensor([4*state_size*state_size], name+'_4*state_size^2', kwargs['initializer'])\n            create_tensor([1, 4*state_size, state_size], name+'_WR_shape', kwargs['initializer'])\n            create_tensor([1, 8*state_size], name+'_B_shape', kwargs['initializer'])\n            create_tensor([4*4*state_size*state_size], name+'_WR_offset', kwargs['initializer'])\n\n            nodes += [\n                # Layer 0\n                # get W\n                make_node('Slice', [param, name+'_0', name+'_4*state_size^2'], [name+'_W0_1d']),\n                make_node('Split', [name+'_W0_1d'], [name+'_W00', name+'_W01', name+'_W02', name+'_W03']),\n                make_node('Concat', [name+'_W00', name+'_W03', name+'_W01', name+'_W02'], [name+'_W0_'], axis=0),\n                make_node('Reshape', [name+'_W0_', name+'_WR_shape'], [name+'_W0']),\n                # get R\n                make_node('Add', [name+'_4*state_size^2', name+'_4*state_size^2'], [name+'_R0_offset']),\n                make_node('Slice', [param, name+'_4*state_size^2', name+'_R0_offset'], [name+'_R0_1d']),\n                make_node('Split', [name+'_R0_1d'], [name+'_R00', name+'_R01', name+'_R02', name+'_R03']),\n                make_node('Concat', [name+'_R00', name+'_R03', name+'_R01', name+'_R02'], [name+'_R0_'], axis=0),\n                make_node('Reshape', [name+'_R0_', name+'_WR_shape'], [name+'_R0']),\n                # get B\n                make_node('Add', [name+'_WR_offset', name+'_8*state_size'], [name+'_B0_offset']),\n                make_node('Slice', [param, name+'_WR_offset', name+'_B0_offset'], [name+'_B0_1d']),\n                make_node('Split', [name+'_B0_1d'], [name+'_B00', name+'_B01', name+'_B02', name+'_B03',\n                                                     name+'_B04', name+'_B05', name+'_B06', name+'_B07']),\n                make_node('Concat', [name+'_B00', name+'_B03', name+'_B01', name+'_B02',\n                                     name+'_B04', name+'_B07', name+'_B05', name+'_B06'], [name+'_B0_'], axis=0),\n                make_node('Reshape', [name+'_B0_', name+'_B_shape'], [name+'_B0']),\n                # get initial states\n                make_node('Split', [name+'initial_h'], [name+'_initial_h0', name+'_initial_h1'], axis=0),\n                make_node('Split', [name+'initial_c'], [name+'_initial_c0', name+'_initial_c1'], axis=0),\n                # get seq_len\n                make_node('Tile', [name+'_seq_length', name+'_batch_size'], [name+'_seq_len_']),\n                make_node(\"Cast\", [name+'_seq_len_'], [name+\"_seq_len\"], to=int(TensorProto.INT32)),\n                # Layer 0 LSTM\n                make_node('LSTM', [data, name+'_W0', name+'_R0', name+'_B0', name+'_seq_len',\n                                   name+'_initial_h0', name+'_initial_c0'],\n                          [name+'_lstm0_out_', name+'_lstm0_h', name+'_lstm0_c'], hidden_size=state_size),\n                make_node('Squeeze', [name+'_lstm0_out_'], [name+'_lstm0_out'], axes=[1]),\n\n                # Layer 1\n                # get W\n                make_node('Add', [name+'_R0_offset', name+'_4*state_size^2'], [name+'_W1_offset']),\n                make_node('Slice', [param, name+'_R0_offset', name+'_W1_offset'], [name+'_W1_1d']),\n                make_node('Split', [name+'_W1_1d'], [name+'_W10', name+'_W11', name+'_W12', name+'_W13']),\n                make_node('Concat', [name+'_W10', name+'_W13', name+'_W11', name+'_W12'], [name+'_W1_'], axis=0),\n                make_node('Reshape', [name+'_W1_', name+'_WR_shape'], [name+'_W1']),\n                # get R\n                make_node('Slice', [param, name+'_W1_offset', name+'_WR_offset'], [name+'_R1_1d']),\n                make_node('Split', [name+'_R1_1d'], [name+'_R10', name+'_R11', name+'_R12', name+'_R13']),\n                make_node('Concat', [name+'_R10', name+'_R13', name+'_R11', name+'_R12'], [name+'_R1_'], axis=0),\n                make_node('Reshape', [name+'_R1_', name+'_WR_shape'], [name+'_R1']),\n                # get B\n                make_node('Add', [name+'_B0_offset', name+'_8*state_size'], [name+'_B1_offset']),\n                make_node('Slice', [param, name+'_B0_offset', name+'_B1_offset'], [name+'_B1_1d']),\n                make_node('Split', [name+'_B1_1d'], [name+'_B10', name+'_B11', name+'_B12', name+'_B13',\n                                                     name+'_B14', name+'_B15', name+'_B16', name+'_B17']),\n                make_node('Concat', [name+'_B10', name+'_B13', name+'_B11', name+'_B12',\n                                     name+'_B14', name+'_B17', name+'_B15', name+'_B16'], [name+'_B1_'], axis=0),\n                make_node('Reshape', [name+'_B1_', name+'_B_shape'], [name+'_B1']),\n                # Layer 1 LSTM\n                make_node('LSTM', [name+'_lstm0_out', name+'_W1', name+'_R1', name+'_B1', name+'_seq_len',\n                                   name+'_initial_h1', name+'_initial_c1'],\n                          [name+'_lstm1_out_', name+'_lstm1_h', name+'_lstm1_c'], hidden_size=state_size),\n                make_node('Squeeze', [name+'_lstm1_out_'], [name], axes=[1]),\n                make_node('Concat', [name+'_lstm0_h', name+'_lstm1_h'], [name+'1'], axis=0),\n                make_node('Concat', [name+'_lstm0_c', name+'_lstm1_c'], [name+'2'], axis=0),\n            ]\n        elif num_layers == 1:\n            if bidirectional == 'False':\n                create_tensor([4*state_size], name+'_4*state_size', kwargs['initializer'])\n                create_tensor([8*state_size], name+'_8*state_size', kwargs['initializer'])\n                create_tensor([4*state_size*state_size], name+'_4*state_size^2', kwargs['initializer'])\n                create_tensor([1, 4*state_size, state_size], name+'_R_shape', kwargs['initializer'])\n                create_tensor([1, 8*state_size], name+'_B_shape', kwargs['initializer'])\n\n                nodes += [\n                    # get W\n                    make_node('Mul', [name+'_4*state_size', name+'_input_size'], [name+'_mul0']),\n                    make_node('Slice', [param, name+'_0', name+'_mul0'], [name+'_W_1d']),\n                    make_node('Split', [name+'_W_1d'], [name+'_W0', name+'_W1', name+'_W2', name+'_W3']),\n                    make_node('Concat', [name+'_W0', name+'_W3', name+'_W1', name+'_W2'], [name+'_W_'], axis=0),\n                    make_node('Concat', [name+'_1', name+'_4*state_size', name+'_input_size'],\n                              [name+'_W_shape'], axis=0),\n                    make_node('Reshape', [name+'_W_', name+'_W_shape'], [name+'_W']),\n                    # get R\n                    make_node('Add', [name+'_mul0', name+'_4*state_size^2'], [name+'_add0']),\n                    make_node('Slice', [param, name+'_mul0', name+'_add0'], [name+'_R_1d']),\n                    make_node('Split', [name+'_R_1d'], [name+'_R0', name+'_R1', name+'_R2', name+'_R3']),\n                    make_node('Concat', [name+'_R0', name+'_R3', name+'_R1', name+'_R2'], [name+'_R_'], axis=0),\n                    make_node('Reshape', [name+'_R_', name+'_R_shape'], [name+'_R']),\n                    # get B\n                    make_node('Add', [name+'_add0', name+'_8*state_size'], [name+'_add1']),\n                    make_node('Slice', [param, name+'_add0', name+'_add1'], [name+'_B_1d']),\n                    make_node('Split', [name+'_B_1d'], [name+'_B0', name+'_B1', name+'_B2', name+'_B3',\n                                                        name+'_B4', name+'_B5', name+'_B6', name+'_B7']),\n                    make_node('Concat', [name+'_B0', name+'_B3', name+'_B1', name+'_B2',\n                                         name+'_B4', name+'_B7', name+'_B5', name+'_B6'], [name+'_B_'], axis=0),\n                    make_node('Reshape', [name+'_B_', name+'_B_shape'], [name+'_B']),\n                    # get seq_len\n                    make_node('Tile', [name+'_seq_length', name+'_batch_size'], [name+'_seq_len_']),\n                    make_node(\"Cast\", [name+'_seq_len_'], [name+\"_seq_len\"], to=int(TensorProto.INT32)),\n                    # compute LSTM\n                    make_node('LSTM', [data, name+'_W', name+'_R', name+'_B',\n                                       name+'_seq_len', name+'initial_h', name+'initial_c'],\n                              [name+'0_', name+'1', name+'2'], hidden_size=state_size),\n                    make_node('Squeeze', [name+'0_'], [name], axes=[1]),\n                ]\n            else:\n                create_tensor([-1], name+'_-1', kwargs['initializer'])\n                create_tensor([4*state_size], name+'_4*state_size', kwargs['initializer'])\n                create_tensor([8*state_size], name+'_8*state_size', kwargs['initializer'])\n                create_tensor([4*state_size*state_size], name+'_4*state_size^2', kwargs['initializer'])\n                create_tensor([1, 4*state_size, state_size], name+'_R_shape', kwargs['initializer'])\n                create_tensor([1, 8*state_size], name+'_B_shape', kwargs['initializer'])\n\n                nodes += [\n                    # get W_fwd\n                    make_node('Mul', [name+'_4*state_size', name+'_input_size'], [name+'_mul0']),\n                    make_node('Slice', [param, name+'_0', name+'_mul0'], [name+'_W_1d']),\n                    make_node('Split', [name+'_W_1d'], [name+'_W0', name+'_W1', name+'_W2', name+'_W3']),\n                    make_node('Concat', [name+'_W0', name+'_W3', name+'_W1', name+'_W2'],\n                              [name+'_W_'], axis=0),\n                    make_node('Concat', [name+'_1', name+'_4*state_size', name+'_input_size'],\n                              [name+'_W_shape'], axis=0),\n                    make_node('Reshape', [name+'_W_', name+'_W_shape'], [name+'_W_fwd']),\n                    # get R_fwd\n                    make_node('Add', [name+'_mul0', name+'_4*state_size^2'], [name+'_add0']),\n                    make_node('Slice', [param, name+'_mul0', name+'_add0'], [name+'_R_1d']),\n                    make_node('Split', [name+'_R_1d'], [name+'_R0', name+'_R1', name+'_R2', name+'_R3']),\n                    make_node('Concat', [name+'_R0', name+'_R3', name+'_R1', name+'_R2'], [name+'_R_'], axis=0),\n                    make_node('Reshape', [name+'_R_', name+'_R_shape'], [name+'_R_fwd']),\n                    # get W_bwd\n                    make_node('Add', [name+'_add0', name+'_mul0'], [name+'_add1']),\n                    make_node('Slice', [param, name+'_add0', name+'_add1'], [name+'_W_1d_bwd']),\n                    make_node('Split', [name+'_W_1d_bwd'],\n                              [name+'_W0_bwd', name+'_W1_bwd', name+'_W2_bwd', name+'_W3_bwd']),\n                    make_node('Concat', [name+'_W0_bwd', name+'_W3_bwd', name+'_W1_bwd', name+'_W2_bwd'],\n                              [name+'_W_bwd_'], axis=0),\n                    make_node('Reshape', [name+'_W_bwd_', name+'_W_shape'], [name+'_W_bwd']),\n                    # get R_bwd\n                    make_node('Add', [name+'_add1', name+'_4*state_size^2'], [name+'_add2']),\n                    make_node('Slice', [param, name+'_add1', name+'_add2'], [name+'_R_1d_bwd']),\n                    make_node('Split', [name+'_R_1d_bwd'],\n                              [name+'_R0_bwd', name+'_R1_bwd', name+'_R2_bwd', name+'_R3_bwd']),\n                    make_node('Concat', [name+'_R0_bwd', name+'_R3_bwd', name+'_R1_bwd', name+'_R2_bwd'],\n                              [name+'_R_bwd_'], axis=0),\n                    make_node('Reshape', [name+'_R_bwd_', name+'_R_shape'], [name+'_R_bwd']),\n                    # get B_fwd\n                    make_node('Add', [name+'_add2', name+'_8*state_size'], [name+'_add3']),\n                    make_node('Slice', [param, name+'_add2', name+'_add3'], [name+'_B_1d']),\n                    make_node('Split', [name+'_B_1d'], [name+'_B0', name+'_B1', name+'_B2', name+'_B3',\n                                                        name+'_B4', name+'_B5', name+'_B6', name+'_B7']),\n                    make_node('Concat', [name+'_B0', name+'_B3', name+'_B1', name+'_B2',\n                                         name+'_B4', name+'_B7', name+'_B5', name+'_B6'], [name+'_B_'], axis=0),\n                    make_node('Reshape', [name+'_B_', name+'_B_shape'], [name+'_B_fwd']),\n                    # get B_bwd\n                    make_node('Add', [name+'_add3', name+'_8*state_size'], [name+'_add4']),\n                    make_node('Slice', [param, name+'_add3', name+'_add4'], [name+'_B_1d_bwd']),\n                    make_node('Split', [name+'_B_1d_bwd'],\n                              [name+'_B0_bwd', name+'_B1_bwd', name+'_B2_bwd', name+'_B3_bwd',\n                               name+'_B4_bwd', name+'_B5_bwd', name+'_B6_bwd', name+'_B7_bwd']),\n                    make_node('Concat', [name+'_B0_bwd', name+'_B3_bwd', name+'_B1_bwd', name+'_B2_bwd',\n                                         name+'_B4_bwd', name+'_B7_bwd', name+'_B5_bwd', name+'_B6_bwd'],\n                              [name+'_B_bwd_'], axis=0),\n                    make_node('Reshape', [name+'_B_bwd_', name+'_B_shape'], [name+'_B_bwd']),\n                    # get seq_len\n                    make_node('Tile', [name+'_seq_length', name+'_batch_size'], [name+'_seq_len_']),\n                    make_node(\"Cast\", [name+'_seq_len_'], [name+\"_seq_len\"], to=int(TensorProto.INT32)),\n                    # compute LSTM\n                    make_node('Concat', [name+'_W_fwd', name+'_W_bwd'], [name+'_W'], axis=0),\n                    make_node('Concat', [name+'_R_fwd', name+'_R_bwd'], [name+'_R'], axis=0),\n                    make_node('Concat', [name+'_B_fwd', name+'_B_bwd'], [name+'_B'], axis=0),\n                    make_node('LSTM', [data, name+'_W', name+'_R', name+'_B',\n                                       name+'_seq_len', name+'initial_h', name+'initial_c'],\n                              [name+'0_', name+'1', name+'2'], hidden_size=state_size, direction='bidirectional'),\n                    make_node('Transpose', [name+'0_'], [name+'0_t'], perm=[0, 2, 1, 3]),\n                    make_node('Concat', [name+'_seq_length', name+'_batch_size', name+'_-1'],\n                              [name+'_shape_out'], axis=0),\n                    make_node('Reshape', [name+'0_t', name+'_shape_out'], [name]),\n                ]\n        else:\n            raise NotImplementedError('Currently RNN onnx export only supports num_layers equals to 1 or 2')\n\n    elif mode == 'gru':\n        if num_layers == 2:\n            create_tensor([6*state_size], name+'_6*state_size', kwargs['initializer'])\n            create_tensor([3*state_size*state_size], name+'_3*state_size^2', kwargs['initializer'])\n            create_tensor([1, 3*state_size, state_size], name+'_WR_shape', kwargs['initializer'])\n            create_tensor([1, 6*state_size], name+'_B_shape', kwargs['initializer'])\n            create_tensor([4*3*state_size*state_size], name+'_WR_offset', kwargs['initializer'])\n\n            nodes += [\n                # Layer 0\n                # get W\n                make_node('Slice', [param, name+'_0', name+'_3*state_size^2'], [name+'_W0_1d']),\n                make_node('Split', [name+'_W0_1d'], [name+'_W00', name+'_W01', name+'_W02']),\n                make_node('Concat', [name+'_W01', name+'_W00', name+'_W02'], [name+'_W0_'], axis=0),\n                make_node('Reshape', [name+'_W0_', name+'_WR_shape'], [name+'_W0']),\n                # get R\n                make_node('Add', [name+'_3*state_size^2', name+'_3*state_size^2'], [name+'_R0_offset']),\n                make_node('Slice', [param, name+'_3*state_size^2', name+'_R0_offset'], [name+'_R0_1d']),\n                make_node('Split', [name+'_R0_1d'], [name+'_R00', name+'_R01', name+'_R02']),\n                make_node('Concat', [name+'_R01', name+'_R00', name+'_R02'], [name+'_R0_'], axis=0),\n                make_node('Reshape', [name+'_R0_', name+'_WR_shape'], [name+'_R0']),\n                # get B\n                make_node('Add', [name+'_WR_offset', name+'_6*state_size'], [name+'_B0_offset']),\n                make_node('Slice', [param, name+'_WR_offset', name+'_B0_offset'], [name+'_B0_1d']),\n                make_node('Split', [name+'_B0_1d'], [name+'_B00', name+'_B01', name+'_B02',\n                                                     name+'_B03', name+'_B04', name+'_B05']),\n                make_node('Concat', [name+'_B01', name+'_B00', name+'_B02',\n                                     name+'_B04', name+'_B03', name+'_B05'], [name+'_B0_'], axis=0),\n                make_node('Reshape', [name+'_B0_', name+'_B_shape'], [name+'_B0']),\n                # get initial states\n                make_node('Split', [name+'initial_h'], [name+'_initial_h0', name+'_initial_h1'], axis=0),\n                # get seq_len\n                make_node('Tile', [name+'_seq_length', name+'_batch_size'], [name+'_seq_len_']),\n                make_node(\"Cast\", [name+'_seq_len_'], [name+\"_seq_len\"], to=int(TensorProto.INT32)),\n                # Layer 0 GRU\n                make_node('GRU', [data, name+'_W0', name+'_R0', name+'_B0', name+'_seq_len',\n                                  name+'_initial_h0'],\n                          [name+'_gru0_out_', name+'_gru0_h'], hidden_size=state_size, linear_before_reset=1),\n                make_node('Squeeze', [name+'_gru0_out_'], [name+'_gru0_out'], axes=[1]),\n\n                # Layer 1\n                # get W\n                make_node('Add', [name+'_R0_offset', name+'_3*state_size^2'], [name+'_W1_offset']),\n                make_node('Slice', [param, name+'_R0_offset', name+'_W1_offset'], [name+'_W1_1d']),\n                make_node('Split', [name+'_W1_1d'], [name+'_W10', name+'_W11', name+'_W12']),\n                make_node('Concat', [name+'_W11', name+'_W10', name+'_W12'], [name+'_W1_'], axis=0),\n                make_node('Reshape', [name+'_W1_', name+'_WR_shape'], [name+'_W1']),\n                # get R\n                make_node('Slice', [param, name+'_W1_offset', name+'_WR_offset'], [name+'_R1_1d']),\n                make_node('Split', [name+'_R1_1d'], [name+'_R10', name+'_R11', name+'_R12']),\n                make_node('Concat', [name+'_R11', name+'_R10', name+'_R12'], [name+'_R1_'], axis=0),\n                make_node('Reshape', [name+'_R1_', name+'_WR_shape'], [name+'_R1']),\n                # get B\n                make_node('Add', [name+'_B0_offset', name+'_6*state_size'], [name+'_B1_offset']),\n                make_node('Slice', [param, name+'_B0_offset', name+'_B1_offset'], [name+'_B1_1d']),\n                make_node('Split', [name+'_B1_1d'], [name+'_B10', name+'_B11', name+'_B12',\n                                                     name+'_B13', name+'_B14', name+'_B15']),\n                make_node('Concat', [name+'_B11', name+'_B10', name+'_B12',\n                                     name+'_B14', name+'_B13', name+'_B15'], [name+'_B1_'], axis=0),\n                make_node('Reshape', [name+'_B1_', name+'_B_shape'], [name+'_B1']),\n                # Layer 1 GRU\n                make_node('GRU', [name+'_gru0_out', name+'_W1', name+'_R1', name+'_B1', name+'_seq_len',\n                                  name+'_initial_h1'],\n                          [name+'_gru1_out_', name+'_gru1_h'], hidden_size=state_size, linear_before_reset=1),\n                make_node('Squeeze', [name+'_gru1_out_'], [name], axes=[1]),\n                make_node('Concat', [name+'_gru0_h', name+'_gru1_h'], [name+'1'], axis=0)\n            ]\n\n        elif num_layers == 1:\n            create_tensor([3*state_size], name+'_3*state_size', kwargs['initializer'])\n            create_tensor([6*state_size], name+'_6*state_size', kwargs['initializer'])\n            create_tensor([3*state_size*state_size], name+'_3*state_size^2', kwargs['initializer'])\n            create_tensor([1, 3*state_size, state_size], name+'_R_shape', kwargs['initializer'])\n            create_tensor([1, 6*state_size], name+'_B_shape', kwargs['initializer'])\n\n            nodes += [\n                # get W\n                make_node('Mul', [name+'_3*state_size', name+'_input_size'], [name+'_mul0']),\n                make_node('Slice', [param, name+'_0', name+'_mul0'], [name+'_W_1d']),\n                make_node('Split', [name+'_W_1d'], [name+'_W0', name+'_W1', name+'_W2']),\n                make_node('Concat', [name+'_W1', name+'_W0', name+'_W2'], [name+'_W_'], axis=0),\n                make_node('Concat', [name+'_1', name+'_3*state_size', name+'_input_size'], [name+'_W_shape'], axis=0),\n                make_node('Reshape', [name+'_W_', name+'_W_shape'], [name+'_W']),\n                # get R\n                make_node('Add', [name+'_mul0', name+'_3*state_size^2'], [name+'_add0']),\n                make_node('Slice', [param, name+'_mul0', name+'_add0'], [name+'_R_1d']),\n                make_node('Split', [name+'_R_1d'], [name+'_R0', name+'_R1', name+'_R2']),\n                make_node('Concat', [name+'_R1', name+'_R0', name+'_R2'], [name+'_R_'], axis=0),\n                make_node('Reshape', [name+'_R_', name+'_R_shape'], [name+'_R']),\n                # get B\n                make_node('Add', [name+'_add0', name+'_6*state_size'], [name+'_add1']),\n                make_node('Slice', [param, name+'_add0', name+'_add1'], [name+'_B_1d']),\n                make_node('Split', [name+'_B_1d'], [name+'_B0', name+'_B1', name+'_B2',\n                                                    name+'_B3', name+'_B4', name+'_B5']),\n                make_node('Concat', [name+'_B1', name+'_B0', name+'_B2',\n                                     name+'_B4', name+'_B3', name+'_B5'], [name+'_B_'], axis=0),\n                make_node('Reshape', [name+'_B_', name+'_B_shape'], [name+'_B']),\n                # get seq_len\n                make_node('Tile', [name+'_seq_length', name+'_batch_size'], [name+'_seq_len_']),\n                make_node(\"Cast\", [name+'_seq_len_'], [name+\"_seq_len\"], to=int(TensorProto.INT32)),\n                # compute GRU\n                make_node('GRU', [data, name+'_W', name+'_R', name+'_B', name+'_seq_len', name+'initial_h'],\n                          [name+'0_', name+'1'], hidden_size=state_size, linear_before_reset=1),\n                make_node('Squeeze', [name+'0_'], [name], axes=[1]),\n            ]\n        else:\n            raise NotImplementedError('Currently RNN onnx export only supports num_layers equals to 1 or 2')\n\n    elif mode in ['rnn_tanh', 'rnn_relu']:\n        activations = ['Tanh']\n        if mode == 'rnn_relu':\n            activations = ['Relu']\n        if num_layers == 2:\n            create_tensor([2*state_size], name+'_2*state_size', kwargs['initializer'])\n            create_tensor([state_size*state_size], name+'_state_size^2', kwargs['initializer'])\n            create_tensor([1, state_size, state_size], name+'_WR_shape', kwargs['initializer'])\n            create_tensor([1, 2*state_size], name+'_B_shape', kwargs['initializer'])\n            create_tensor([4*state_size*state_size], name+'_WR_offset', kwargs['initializer'])\n\n            nodes += [\n                # Layer 0\n                # get W\n                make_node('Slice', [param, name+'_0', name+'_state_size^2'], [name+'_W0_1d']),\n                make_node('Reshape', [name+'_W0_1d', name+'_WR_shape'], [name+'_W0']),\n                # get R\n                make_node('Add', [name+'_state_size^2', name+'_state_size^2'], [name+'_R0_offset']),\n                make_node('Slice', [param, name+'_state_size^2', name+'_R0_offset'], [name+'_R0_1d']),\n                make_node('Reshape', [name+'_R0_1d', name+'_WR_shape'], [name+'_R0']),\n                # get B\n                make_node('Add', [name+'_WR_offset', name+'_2*state_size'], [name+'_B0_offset']),\n                make_node('Slice', [param, name+'_WR_offset', name+'_B0_offset'], [name+'_B0_1d']),\n                make_node('Reshape', [name+'_B0_1d', name+'_B_shape'], [name+'_B0']),\n                # get initial states\n                make_node('Split', [name+'initial_h'], [name+'_initial_h0', name+'_initial_h1'], axis=0),\n                # get seq_len\n                make_node('Tile', [name+'_seq_length', name+'_batch_size'], [name+'_seq_len_']),\n                make_node(\"Cast\", [name+'_seq_len_'], [name+\"_seq_len\"], to=int(TensorProto.INT32)),\n                # Layer 0 RNN\n                make_node('RNN', [data, name+'_W0', name+'_R0', name+'_B0', name+'_seq_len',\n                                  name+'_initial_h0'], [name+'_rnn0_out_', name+'_rnn0_h'],\n                          hidden_size=state_size, activations=activations),\n                make_node('Squeeze', [name+'_rnn0_out_'], [name+'_rnn0_out'], axes=[1]),\n\n                # Layer 1\n                # get W\n                make_node('Add', [name+'_R0_offset', name+'_state_size^2'], [name+'_W1_offset']),\n                make_node('Slice', [param, name+'_R0_offset', name+'_W1_offset'], [name+'_W1_1d']),\n                make_node('Reshape', [name+'_W1_1d', name+'_WR_shape'], [name+'_W1']),\n                # get R\n                make_node('Slice', [param, name+'_W1_offset', name+'_WR_offset'], [name+'_R1_1d']),\n                make_node('Reshape', [name+'_R1_1d', name+'_WR_shape'], [name+'_R1']),\n                # get B\n                make_node('Add', [name+'_B0_offset', name+'_2*state_size'], [name+'_B1_offset']),\n                make_node('Slice', [param, name+'_B0_offset', name+'_B1_offset'], [name+'_B1_1d']),\n                make_node('Reshape', [name+'_B1_1d', name+'_B_shape'], [name+'_B1']),\n                # Layer 1 RNN\n                make_node('RNN', [name+'_rnn0_out', name+'_W1', name+'_R1', name+'_B1', name+'_seq_len',\n                                  name+'_initial_h1'], [name+'_rnn1_out_', name+'_rnn1_h'],\n                          hidden_size=state_size, activations=activations),\n                make_node('Squeeze', [name+'_rnn1_out_'], [name], axes=[1]),\n                make_node('Concat', [name+'_rnn0_h', name+'_rnn1_h'], [name+'1'], axis=0)\n            ]\n\n        elif num_layers == 1:\n            create_tensor([2*state_size], name+'_2*state_size', kwargs['initializer'])\n            create_tensor([state_size*state_size], name+'_state_size^2', kwargs['initializer'])\n            create_tensor([1, state_size, state_size], name+'_R_shape', kwargs['initializer'])\n            create_tensor([1, 2*state_size], name+'_B_shape', kwargs['initializer'])\n\n            nodes += [\n                # get W\n                make_node('Mul', [name+'_state_size', name+'_input_size'], [name+'_mul0']),\n                make_node('Slice', [param, name+'_0', name+'_mul0'], [name+'_W_1d']),\n                make_node('Concat', [name+'_1', name+'_state_size', name+'_input_size'], [name+'_W_shape'], axis=0),\n                make_node('Reshape', [name+'_W_1d', name+'_W_shape'], [name+'_W']),\n                # get R\n                make_node('Add', [name+'_mul0', name+'_state_size^2'], [name+'_add0']),\n                make_node('Slice', [param, name+'_mul0', name+'_add0'], [name+'_R_1d']),\n                make_node('Reshape', [name+'_R_1d', name+'_R_shape'], [name+'_R']),\n                # get B\n                make_node('Add', [name+'_add0', name+'_2*state_size'], [name+'_add1']),\n                make_node('Slice', [param, name+'_add0', name+'_add1'], [name+'_B_1d']),\n                make_node('Reshape', [name+'_B_1d', name+'_B_shape'], [name+'_B']),\n                # get seq_len\n                make_node('Tile', [name+'_seq_length', name+'_batch_size'], [name+'_seq_len_']),\n                make_node(\"Cast\", [name+'_seq_len_'], [name+\"_seq_len\"], to=int(TensorProto.INT32)),\n                # compute RNN\n                make_node('RNN', [data, name+'_W', name+'_R', name+'_B', name+'_seq_len', name+'initial_h'],\n                          [name+'0_', name+'1'], hidden_size=state_size, activations=activations),\n                make_node('Squeeze', [name+'0_'], [name], axes=[1]),\n            ]\n        else:\n            raise NotImplementedError('Currently RNN onnx export only supports num_layers equals to 1 or 2')\n    else:\n        raise NotImplementedError(f\"Currently RNN onnx export does not support {mode} mode\")\n    return nodes\n\n\n@mx_op.register('_rnn_param_concat')\ndef convert_rnn_param_concat(node, **kwargs):\n    \"\"\"Map MXNet's _rnn_param_concat operator\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    axis = int(attrs.get('dim', 1))\n\n    nodes = [\n        make_node('Concat', input_nodes, [name], axis=axis)\n    ]\n\n    return nodes\n\n\n@mx_op.register('_contrib_div_sqrt_dim')\ndef convert_contrib_div_sqrt_dim(node, **kwargs):\n    \"\"\"Map MXNet's _contrib_div_sqrt_dim operator\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    create_tensor([0], name+'_0', kwargs['initializer'])\n    create_tensor([1], name+'_1', kwargs['initializer'])\n    create_tensor([1], name+'_1_f', kwargs['initializer'], dtype=dtype)\n    nodes = [\n        make_node('Shape', [input_nodes[0]], [name+'_shape']),\n        make_node('Shape', [name+'_shape'], [name+'_dim']),\n        make_node('Sub', [name+'_dim', name+'_1'], [name+'_dim_m1']),\n        make_node('Slice', [name+'_shape', name+'_dim_m1', name+'_dim', name+'_0'], [name+'_c_']),\n        make_node('Cast', [name+'_c_'], [name+'_c'], to=dtype_t),\n        make_node('Sqrt', [name+'_c'], [name+'_c_sqrt']),\n        make_node('Div', [name+'_1_f', name+'_c_sqrt'], [name+'_1_over_c_sqrt']),\n        make_node('Mul', [input_nodes[0], name+'_1_over_c_sqrt'], [name])\n    ]\n\n    return nodes\n\n\n@mx_op.register('_split_v2')\ndef convert_contrib_split_v2(node, **kwargs):\n    \"\"\"Map MXNet's _split_v2 operator\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    axis = int(attrs.get('axis', 0))\n    squeeze_axis = attrs.get('squeeze_axis', 'False')\n    sections = int(attrs.get('sections', 0))\n    indices = convert_string_to_list(attrs.get('indices', '[]'))\n    if sections <= 0 and len(indices) == 0:\n        raise NotImplementedError('section or indices must be set')\n    if sections > 0:\n        output_nodes = [name+str(i) for i in range(sections)]\n        if squeeze_axis == 'False':\n            nodes = [\n                make_node('Split', input_nodes, output_nodes, axis=axis),\n            ]\n        else:\n            output_nodes_ = [name+str(i)+'_' for i in range(sections)]\n            nodes = [\n                make_node('Split', input_nodes, output_nodes_, axis=axis),\n            ]\n            for i in range(sections):\n                nodes += [\n                    make_node(\"Squeeze\", [output_nodes_[i]], [output_nodes[i]], axes=[axis]),\n                ]\n    else:\n        raise NotImplementedError('indices is supported since ONNX 1.8.0 (opset13), please upgrade ONNX version')\n\n    return nodes\n\n\n@mx_op.register('_npi_full_like')\ndef convert_full_like(node, **kwargs):\n    \"\"\"Map MXNet's npi_full_like operator attributes to onnx's ConstantOfShape operator.\n    \"\"\"\n    from onnx.helper import make_node, make_tensor\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    dtype = attrs.get('dtype', 'float32')\n    if dtype == 'None':\n        dtype = 'float32'\n    dtype = np.dtype(dtype)\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    fill_value = dtype.type(float(attrs.get('fill_value', 0)))\n\n    # create tensor with shape of input\n    tensor_value = make_tensor(name+'_fill_value', dtype_t, [1], [fill_value])\n    nodes = [\n        make_node('Shape', [input_nodes[0]], [name+'_shape']),\n        make_node('ConstantOfShape', [name+'_shape'], [name], name=name, value=tensor_value)\n    ]\n    return nodes\n\n\n@mx_op.register('_npi_equal')\ndef covert_np_equal(node, **kwargs):\n    \"\"\" npi_equal\n    \"\"\"\n    return create_basic_op_node('Equal', node, kwargs)\n\n\n@mx_op.register('_npi_not_equal')\ndef convert_not_equal(node, **kwargs):\n    \"\"\" npi_not_equal\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, _ = get_inputs(node, kwargs)\n\n    nodes = [\n        make_node('Equal', input_nodes, [name+'_equal']),\n        make_node('Not', [name+'_equal'], [name]),\n    ]\n    return nodes\n\n\n@mx_op.register('_npi_greater')\ndef convert_broadcast_npi_greater(node, **kwargs):\n    \"\"\" npi_greater\n    \"\"\"\n    return create_basic_op_node('Greater', node, kwargs)\n\n\n@mx_op.register('_npi_less')\ndef convert_broadcast_npi_less(node, **kwargs):\n    \"\"\" npi_less\n    \"\"\"\n    return create_basic_op_node('Less', node, kwargs)\n\n\n@mx_op.register('_npi_greater_equal')\ndef convert_broadcast_npi_greater_equal(node, **kwargs):\n    \"\"\" npi_greater_equal\n    \"\"\"\n    return create_basic_op_node('GreaterOrEqual', node, kwargs)\n\n\n@mx_op.register('_npi_less_equal')\ndef convert_broadcast_npi_less_equal(node, **kwargs):\n    \"\"\" npi_less_equal\n    \"\"\"\n    return create_basic_op_node('LessOrEqual', node, kwargs)\n\n\n@mx_op.register('_npi_argmin')\ndef convert_np_argmin(node, **kwargs):\n    \"\"\" _npi_argmin\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    axis = str(attrs.get('axis', 'None'))\n\n    dtype = np.dtype('int64')\n\n    if axis == 'None':\n        create_tensor([-1], name+'_-1', kwargs['initializer'])\n        nodes = [\n            make_node('Reshape', [input_nodes[0], name+'_-1'], [name+'_reshape']),\n            make_node('ArgMin', [name+'_reshape'], [name], axis=0, keepdims=False),\n        ]\n    else:\n        axis = int(axis)\n        nodes = [\n            make_node('ArgMin', [input_nodes[0]], [name], axis=axis, keepdims=False),\n        ]\n    return nodes, (dtype,)\n\n\n@mx_op.register('_npi_argmax')\ndef convert_np_argmax(node, **kwargs):\n    \"\"\" _npi_argmax\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    axis = str(attrs.get('axis', 'None'))\n\n    dtype = np.dtype('int64')\n\n    if axis == 'None':\n        create_tensor([-1], name+'_-1', kwargs['initializer'])\n        nodes = [\n            make_node('Reshape', [input_nodes[0], name+'_-1'], [name+'_reshape']),\n            make_node('ArgMax', [name+'_reshape'], [name], axis=0, keepdims=False),\n        ]\n    else:\n        axis = int(axis)\n        nodes = [\n            make_node('ArgMax', [input_nodes[0]], [name], axis=axis, keepdims=False),\n        ]\n    return nodes, (dtype,)\n\n\n@mx_op.register(\"_npi_mean\")\ndef convert_npi_mean(node, **kwargs):\n    \"\"\"Map MXNet's mean operator attributes to onnx's ReduceMean operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    dtype = np.dtype('float32')\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    mx_axis = str(attrs.get(\"axis\", 'None'))\n    axes = convert_string_to_list(mx_axis) if mx_axis != 'None' else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n\n    if axes is not None:\n        if keepdims:\n            nodes = [\n                make_node('Cast', input_nodes, [name+'_cast'], to=dtype_t),\n                make_node('ReduceMean', [name+'_cast'], [name], axes=axes, keepdims=keepdims),\n            ]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('Cast', input_nodes, [name+'_cast'], to=dtype_t),\n                make_node('ReduceMean', [name+'_cast'], [name+'_reduce'], axes=axes, keepdims=keepdims),\n                make_node('Shape', [name+'_reduce'], [name+'_reduce_shape']),\n                make_node('Concat', [name+'_1', name+'_reduce_shape'], [name+'_concat'], axis=0),\n                make_node('Reshape', [name+'_reduce', name+'_concat'], [name+'_reshape']),\n                make_node('Squeeze', [name+'_reshape'], [name], axes=[0]),\n            ]\n    else:\n        if keepdims:\n            nodes = [\n                make_node('Cast', input_nodes, [name+'_cast'], to=dtype_t),\n                make_node('ReduceMean', [name+'_cast'], [name], keepdims=keepdims),\n            ]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('Cast', input_nodes, [name+'_cast'], to=dtype_t),\n                make_node('ReduceMean', [name+'_cast'], [name+'_reduce'], keepdims=keepdims),\n                make_node('Reshape', [name+'_reduce', name+'_1'], [name]),\n            ]\n    return nodes, (dtype,)\n\n\n@mx_op.register(\"_npi_logical_and\")\ndef convert_np_logical_and(node, **kwargs):\n    \"\"\"Map MXNet's broadcast logical and operator attributes to onnx's And operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    nodes = [\n        make_node(\"Cast\", [input_nodes[0]], [name+\"_cast0\"], to=int(TensorProto.BOOL)),\n        make_node(\"Cast\", [input_nodes[1]], [name+\"_cast1\"], to=int(TensorProto.BOOL)),\n        make_node(\"And\", [name+\"_cast0\", name+\"_cast1\"], [name]),\n    ]\n    return nodes, (np.dtype('bool'),)\n\n\n@mx_op.register(\"_npi_logical_xor\")\ndef convert_np_logical_xor(node, **kwargs):\n    \"\"\"Map MXNet's broadcast logical xor operator attributes to onnx's XOR operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    nodes = [\n        make_node(\"Cast\", [input_nodes[0]], [name+\"_cast0\"], to=int(TensorProto.BOOL)),\n        make_node(\"Cast\", [input_nodes[1]], [name+\"_cast1\"], to=int(TensorProto.BOOL)),\n        make_node(\"Xor\", [name+\"_cast0\", name+\"_cast1\"], [name]),\n    ]\n    return nodes, (np.dtype('bool'),)\n\n\n@mx_op.register(\"_npi_logical_or\")\ndef convert_np_logical_or(node, **kwargs):\n    \"\"\"Map MXNet's broadcast logical or operator attributes to onnx's OR operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    nodes = [\n        make_node(\"Cast\", [input_nodes[0]], [name+\"_cast0\"], to=int(TensorProto.BOOL)),\n        make_node(\"Cast\", [input_nodes[1]], [name+\"_cast1\"], to=int(TensorProto.BOOL)),\n        make_node(\"Or\", [name+\"_cast0\", name+\"_cast1\"], [name]),\n    ]\n    return nodes, (np.dtype('bool'),)\n\n\n@mx_op.register(\"_npi_logical_not\")\ndef convert_np_logical_not(node, **kwargs):\n    \"\"\"Map MXNet's logical not operator attributes to onnx's Not operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    nodes = [\n        make_node(\"Cast\", [input_nodes[0]], [name+\"_cast\"], to=int(TensorProto.BOOL)),\n        make_node(\"Not\", [name+\"_cast\"], [name]),\n    ]\n    return nodes, (np.dtype('bool'),)\n\n\n@mx_op.register(\"_npi_true_divide\")\ndef convert_np_divide(node, **kwargs):\n    \"\"\"np.divide\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, _ = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n    print(input_dtypes[0])\n    if np.issubdtype(input_dtypes[0], np.integer):\n        nodes = [\n            make_node(\"Cast\", [input_nodes[0]], [name+\"_cast0\"], to=int(TensorProto.FLOAT)),\n            make_node(\"Cast\", [input_nodes[1]], [name+\"_cast1\"], to=int(TensorProto.FLOAT)),\n            make_node(\"Div\", [name+\"_cast0\", name+\"_cast1\"], [name]),\n        ]\n        return nodes, (np.dtype('float32'),)\n    return create_basic_op_node('Div', node, kwargs)\n"
  },
  {
    "path": "python/mxnet/onnx/mx2onnx/_op_translations/_op_translations_opset13.py",
    "content": "#  Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.\n#\n#  Redistribution and use in source and binary forms, with or without\n#  modification, are permitted provided that the following conditions\n#  are met:\n#  * Redistributions of source code must retain the above copyright\n#    notice, this list of conditions and the following disclaimer.\n#  * Redistributions in binary form must reproduce the above copyright\n#    notice, this list of conditions and the following disclaimer in the\n#    documentation and/or other materials provided with the distribution.\n#  * Neither the name of NVIDIA CORPORATION nor the names of its\n#    contributors may be used to endorse or promote products derived\n#    from this software without specific prior written permission.\n#\n#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY\n#  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n#  PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR\n#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,\n#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,\n#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY\n#  OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n# Based on\n#  https://github.com/NVIDIA/mxnet_to_onnx/blob/master/mx2onnx_converter/\n# mx2onnx_converter_functions.py\n\n# coding: utf-8\n# pylint: disable=too-many-locals,no-else-return,too-many-lines\n# pylint: disable=anomalous-backslash-in-string,eval-used\n# pylint: disable=too-many-function-args\n\"\"\"\nConversion Functions for common layers.\nAdd new functions here with a decorator.\n\"\"\"\n\nimport re\nimport logging\nimport numpy as np\nfrom .._export_onnx import MXNetGraph as mx_op\ntry:\n    import onnx\nexcept ImportError:\n    onnx = None\n\nOPSET_VERSION = 13\n\ndef parse_helper(attrs, attrs_name, alt_value=None):\n    \"\"\"Helper function to parse operator attributes in required format.\"\"\"\n    tuple_re = re.compile(r'\\([0-9L|,| ]+\\)')\n    if not attrs:\n        return alt_value\n    attrs_str = None if attrs.get(attrs_name) is None else str(attrs.get(attrs_name))\n    if attrs_str is None:\n        return alt_value\n    attrs_match = tuple_re.search(attrs_str)\n    if attrs_match is not None:\n        if attrs_match.span() == (0, len(attrs_str)):\n            dims = eval(attrs_str)\n            return dims\n        else:\n            raise AttributeError(f\"Malformed {attrs_name} dimensions: {str(attrs_str)}\")\n    return alt_value\n\ndef transform_padding(pad_width):\n    \"\"\"Helper function to convert padding format for pad operator.\n    \"\"\"\n    num_pad_values = len(pad_width)\n    onnx_pad_width = [0]*num_pad_values\n\n    start_index = 0\n    # num_pad_values will always be multiple of 2\n    end_index = int(num_pad_values/2)\n    for idx in range(0, num_pad_values):\n        if idx % 2 == 0:\n            onnx_pad_width[start_index] = pad_width[idx]\n            start_index += 1\n        else:\n            onnx_pad_width[end_index] = pad_width[idx]\n            end_index += 1\n\n    return onnx_pad_width\n\n\ndef convert_string_to_list(string_val):\n    \"\"\"Helper function to convert string to list.\n     Used to convert shape attribute string to list format.\n    \"\"\"\n    result_list = []\n\n    list_string = string_val.split(',')\n    for val in list_string:\n        val = str(val.strip())\n        val = val.replace(\"(\", \"\")\n        val = val.replace(\")\", \"\")\n        val = val.replace(\"L\", \"\")\n        val = val.replace(\"[\", \"\")\n        val = val.replace(\"]\", \"\")\n        if val == \"None\":\n            result_list.append(None)\n        elif val != \"\":\n            result_list.append(int(val))\n\n    return result_list\n\ndef get_boolean_attribute_value(attrs, attr_name):\n    \"\"\" Helper function to convert a string version\n    of Boolean attributes to integer for ONNX.\n    Takes attribute dictionary and attr_name as\n    parameters.\n    \"\"\"\n    return 1 if attrs.get(attr_name, 0) in [\"True\", \"1\"] else 0\n\ndef get_inputs(node, kwargs):\n    \"\"\"Helper function to get inputs\"\"\"\n    name = node[\"name\"]\n    outputs_lookup = kwargs[\"outputs_lookup\"]\n    inputs = node[\"inputs\"]\n    attrs = node.get(\"attrs\", {})\n    input_nodes = []\n    for ip in inputs:\n        input_node_name = outputs_lookup[ip[0]][ip[1]].name\n        input_nodes.append(input_node_name)\n\n    return name, input_nodes, attrs\n\ndef get_input_dtypes(node, kwargs):\n    outputs_lookup = kwargs['outputs_lookup']\n    inputs = node['inputs']\n    input_dtypes = []\n    for ip in inputs:\n        input_node_dtype = outputs_lookup[ip[0]][ip[1]].dtype\n        input_dtypes.append(input_node_dtype)\n    return input_dtypes\n\ndef create_basic_op_node(op_name, node, kwargs):\n    \"\"\"Helper function to create a basic operator\n    node that doesn't contain op specific attrs\"\"\"\n    name, input_nodes, _ = get_inputs(node, kwargs)\n\n    node = onnx.helper.make_node(\n        op_name,\n        input_nodes,\n        [name],\n        name=name\n    )\n    return [node]\n\ndef create_const_scalar_node(input_name, value, kwargs):\n    \"\"\"Helper function to create a tensor value node and a\n    initializer tensor node with constant value.\"\"\"\n    from onnx.helper import make_tensor\n    initializer = kwargs[\"initializer\"]\n    dtype = value.dtype\n    if dtype == 'float16':\n        # when using float16, we must convert it to np.uint16 view first\n        value = np.float16(value).view(np.uint16)\n    input_type = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n    tensor_node = make_tensor(input_name, input_type, (), ([value]))\n    initializer.append(tensor_node)\n\ndef create_const_node(input_name, value, kwargs):\n    \"\"\"Helper function to create a tensor value node and a\n    initializer tensor node with constant value.\"\"\"\n    from onnx.helper import make_tensor\n    initializer = kwargs[\"initializer\"]\n    dtype = value.dtype\n    if dtype == 'float16':\n        # when using float16, we must convert it to np.uint16 view first\n        value = np.float16(value).view(np.uint16)\n    input_type = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n    input_shape = value.shape\n    tensor_node = make_tensor(input_name, input_type, input_shape, value)\n    initializer.append(tensor_node)\n\ndef create_tensor(tensor_list, tensor_name, initializer, dtype='int64'):\n    \"\"\"Helper function to create a tensor value node and a\n    initializer tensor node with constant value.\"\"\"\n    tensor_np = np.array(tensor_list, dtype=dtype)\n    data_type = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[tensor_np.dtype]\n    dims = np.shape(tensor_np)\n    if dtype == np.float16:\n        tensor_np = tensor_np.view(dtype=np.uint16)\n    tensor = onnx.helper.make_tensor(\n        name=tensor_name,\n        data_type=data_type,\n        dims=dims,\n        vals=tensor_np.flatten().tolist(),\n        raw=False\n    )\n    initializer.append(tensor)\n\n\ndef create_helper_trans_node(node_name, input_node):\n    \"\"\"create extra transpose node for dot operator\"\"\"\n    trans_node = onnx.helper.make_node(\n        'Transpose',\n        inputs=[input_node],\n        outputs=[node_name],\n        name=node_name\n    )\n    return trans_node\n\n\ndef scalar_op_helper(node, op_name, **kwargs):\n    \"\"\"Helper function for scalar arithmetic operations\"\"\"\n    from onnx import numpy_helper\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    scalar_value = np.array([attrs.get(\"scalar\", 1)],\n                            dtype=dtype)\n    initializer = kwargs[\"initializer\"]\n    flag = True\n    # If the input value is in initializer, just multiply with scalar input\n    # and create a new initializer\n    for i in initializer:\n        if i.name == input_nodes[0]:\n            if op_name == 'Mul':\n                new_initializer = numpy_helper.to_array(i) * scalar_value[0]\n            elif op_name == 'Sub':\n                if name.startswith(\"_rminusscalar\"):\n                    new_initializer = scalar_value[0] - numpy_helper.to_array(i)\n                else:\n                    new_initializer = numpy_helper.to_array(i) - scalar_value[0]\n            elif op_name == 'Add':\n                new_initializer = numpy_helper.to_array(i) + scalar_value[0]\n            elif op_name == 'Div':\n                if name.startswith(\"_rdivscalar\"):\n                    new_initializer = scalar_value[0] / numpy_helper.to_array(i)\n                else:\n                    new_initializer = numpy_helper.to_array(i) / scalar_value[0]\n            elif op_name == 'Pow':\n                new_initializer = numpy_helper.to_array(i) ** scalar_value[0]\n            flag = False\n            break\n\n    # else create a new tensor of the scalar value, add it in initializer\n    if flag is True:\n        dims = np.shape(scalar_value)\n\n        scalar_op_name = \"scalar_op\" + str(kwargs[\"idx\"])\n        tensor_node = onnx.helper.make_tensor_value_info(scalar_op_name, dtype_t, dims)\n\n        initializer.append(\n            onnx.helper.make_tensor(\n                name=scalar_op_name,\n                data_type=dtype_t,\n                dims=dims,\n                vals=scalar_value,\n                raw=False,\n            )\n        )\n\n        mul_node = onnx.helper.make_node(\n            op_name,\n            [input_nodes[0], scalar_op_name],\n            [name],\n            name=name\n        )\n\n        return [tensor_node, mul_node]\n    else:\n        dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[new_initializer.dtype]\n        dims = np.shape(new_initializer)\n\n        tensor_node = onnx.helper.make_tensor_value_info(name, dtype_t, dims)\n\n        initializer.append(\n            onnx.helper.make_tensor(\n                name=name,\n                data_type=dtype_t,\n                dims=dims,\n                vals=new_initializer.flatten(),\n                raw=False,\n            )\n        )\n        return [tensor_node]\n\n\n    return create_basic_op_node('Shape', node, kwargs)\n\n\n@mx_op.register(\"_contrib_arange_like\", OPSET_VERSION)\ndef convert_arange_like(node, **kwargs):\n    \"\"\"Map MXNet's arange_like operator attributes to onnx's Range and Reshape operators.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    opset_version = kwargs['opset_version']\n    if opset_version < 11:\n        raise AttributeError(\"ONNX opset 11 or greater is required to export this operator\")\n    # use the same dtype as the that of the input node\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n    axis = attrs.get('axis', 'None')\n    start = attrs.get('start', 0.)\n    step = attrs.get('step', 1.)\n    repeat = int(attrs.get('repeat', 1))\n    if repeat != 1:\n        raise NotImplementedError(\"arange_like operator with repeat != 1 not yet implemented.\")\n\n    create_const_scalar_node(name+\"_start\", np.dtype(dtype).type(start), kwargs)\n    create_const_scalar_node(name+\"_step\", np.dtype(dtype).type(step), kwargs)\n    create_const_scalar_node(name+\"_half_step\", np.dtype(dtype).type(float(step)*0.5), kwargs)\n    create_tensor([0], name+\"_0\", kwargs[\"initializer\"], dtype='int64')\n    nodes = []\n    if axis == 'None':\n        # output will be same shape as input\n        nodes += [\n            make_node(\"Shape\", [input_nodes[0]], [name+\"_shape0_out\"]),\n            make_node(\"ReduceProd\", [name+\"_shape0_out\"], [name+\"_redprod0_out\"]),\n            make_node(\"Squeeze\", [name+\"_redprod0_out\", name+\"_0\"], [name+'_reshape0_out']),\n            make_node(\"Cast\", [name+\"_reshape0_out\"], [name+\"_cast0_out\"], to=dtype_t),\n            make_node(\"Mul\", [name+\"_cast0_out\", name+\"_step\"], [name+\"_mul0_out\"]),\n            make_node(\"Add\", [name+\"_mul0_out\", name+\"_start\"], [name+\"_add1_out\"]),\n            make_node(\"Sub\", [name+\"_add1_out\", name+\"_half_step\"], [name+\"_sub0_out\"]),\n            make_node(\"Range\", [name+\"_start\", name+\"_sub0_out\", name+\"_step\"], [name+\"_range0_out\"]),\n            make_node(\"Reshape\", [name+\"_range0_out\", name+\"_shape0_out\"], [name], name=name)\n        ]\n    else:\n        # determine shape of axis\n        create_tensor([int(axis)], name+\"_axis_start\", kwargs[\"initializer\"], dtype='int64')\n        create_tensor([int(axis)+1], name+\"_axis_end\", kwargs[\"initializer\"], dtype='int64')\n        nodes += [\n            make_node(\"Shape\", [input_nodes[0]], [name+\"_shape0_out\"]),\n            make_node(\"Slice\", [name+\"_shape0_out\", name+\"_axis_start\", name+\"_axis_end\"], [name+\"_slice0_out\"]),\n            make_node(\"ReduceProd\", [name+\"_slice0_out\"], [name+\"_reprod0_out\"]),\n            make_node(\"Squeeze\", [name+\"_reprod0_out\", name+\"_0\"], [name+\"_reshape0_out\"]),\n            make_node(\"Cast\", [name+\"_reshape0_out\"], [name+\"_cast0_out\"], to=dtype_t),\n            make_node(\"Mul\", [name+\"_cast0_out\", name+\"_step\"], [name+\"_mul0_out\"]),\n            make_node(\"Add\", [name+\"_mul0_out\", name+\"_start\"], [name+\"_add1_out\"]),\n            make_node(\"Sub\", [name+\"_add1_out\", name+\"_half_step\"], [name+\"_sub0_out\"]),\n            make_node(\"Range\", [name+\"_start\", name+\"_sub0_out\", name+\"_step\"], [name], name=name)\n        ]\n\n    return nodes\n\n\n@mx_op.register(\"LayerNorm\", OPSET_VERSION)\ndef convert_layer_norm(node, **kwargs):\n    \"\"\"Map MXNet's LayerNorm operator attributes to onnx operators.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n\n    axes = int(attrs.get('axis', -1))\n    eps = attrs.get('eps', 9.99999975e-06)\n\n    create_tensor([axes], name+\"_axes\", kwargs[\"initializer\"])\n    create_tensor([axes+1], name+\"_axes+1\", kwargs[\"initializer\"])\n    create_tensor([0], name+\"_0\", kwargs[\"initializer\"], dtype='int64')\n    create_const_scalar_node(name+'_0_s', np.int64(0), kwargs)\n    create_const_scalar_node(name+'_1_s', np.int64(1), kwargs)\n    create_const_scalar_node(name+\"_2_s\", np.int64(2).astype(dtype), kwargs)\n    create_const_scalar_node(name+\"_eps\", np.float32(eps), kwargs)\n\n    nodes = [\n        make_node(\"ReduceMean\", [input_nodes[0]], [name+\"_rm0_out\"], axes=[axes]),\n        make_node(\"Sub\", [input_nodes[0], name+\"_rm0_out\"], [name+\"_sub0_out\"]),\n        make_node(\"Pow\", [name+\"_sub0_out\", name+\"_2_s\"], [name+\"_pow0_out\"]),\n        make_node(\"ReduceMean\", [name+\"_pow0_out\"], [name+\"_rm1_out\"], axes=[axes]),\n        make_node(\"Add\", [name+\"_rm1_out\", name+\"_eps\"], [name+\"_add0_out\"]),\n        make_node(\"Sqrt\", [name+\"_add0_out\"], [name+\"_sqrt0_out\"]),\n        make_node(\"Div\", [name+\"_sub0_out\", name+\"_sqrt0_out\"], [name+\"_div0_out\"]),\n    ]\n\n    if axes == -1:\n        nodes += [\n            make_node(\"Mul\", [name+\"_div0_out\", input_nodes[1]], [name+\"_mul0_out\"]),\n            # make_node(\"Add\", [name+\"_mul0_out\", input_nodes[2]], [name])\n            # the Add operator triggers a weird NaN issue in onnxruntime\n            # a workaround is to use Neg + Sub\n            make_node('Neg', [input_nodes[2]], [name+'_neg']),\n            make_node(\"Sub\", [name+\"_mul0_out\", name+'_neg'], [name])\n        ]\n    else:\n        nodes += [\n            make_node(\"Shape\", [input_nodes[0]], [name+\"_shape0_out\"]),\n            make_node(\"Shape\", [name+\"_shape0_out\"], [name+\"_in_dim\"]),\n            make_node(\"Squeeze\", [name+\"_in_dim\", name+\"_0\"], [name+\"_in_dim_s\"]),\n            make_node(\"Range\", [name+\"_0_s\", name+\"_in_dim_s\", name+\"_1_s\"], [name+\"_range\"]),\n            make_node(\"Equal\", [name+\"_range\", name+\"_axes\"], [name+\"_equal\"]),\n            make_node(\"Cast\", [name+\"_equal\"], [name+\"_one_hot\"], to=int(TensorProto.INT64)),\n            make_node(\"Slice\", [name+\"_shape0_out\", name+\"_axes\", name+\"_axes+1\"], [name+\"_slice_out\"]),\n            make_node(\"Squeeze\", [name+\"_slice_out\", name+\"_0\"], [name+\"_slice_out_s\"]),\n            make_node(\"Sub\", [name+\"_slice_out_s\", name+\"_1_s\"], [name+\"_sub1_out\"]),\n            make_node(\"Mul\", [name+\"_one_hot\", name+\"_sub1_out\"], [name+\"_mul0_out\"]),\n            make_node(\"Add\", [name+\"_mul0_out\", name+\"_1_s\"], [name+\"_add1_out\"]),\n            make_node('Reshape', [input_nodes[1], name+\"_add1_out\"], [name+\"gamma_exp\"]),\n            make_node('Reshape', [input_nodes[2], name+\"_add1_out\"], [name+\"beta_exp\"]),\n            make_node('Expand', [name+\"gamma_exp\", name+\"_shape0_out\"], [name+\"gamma_exp1\"]),\n            make_node('Expand', [name+\"beta_exp\", name+\"_shape0_out\"], [name+\"beta_exp1\"]),\n            make_node(\"Mul\", [name+\"_div0_out\", name+\"gamma_exp1\"], [name+\"_mul1_out\"]),\n            make_node(\"Add\", [name+\"_mul1_out\", name+\"beta_exp1\"], [name], name=name)\n        ]\n\n    return nodes\n\n\n@mx_op.register(\"broadcast_axis\", OPSET_VERSION)\ndef convert_broadcast_axis(node, **kwargs):\n    \"\"\"Map MXNet's broadcast_axis\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    axis = convert_string_to_list(attrs.get('axis', '()'))\n    size = convert_string_to_list(attrs.get('size', '()'))\n    assert len(axis) == len(size)\n\n    shape_name = name+'_shape_0'\n\n    create_tensor([0], name+'_0', kwargs[\"initializer\"])\n    create_tensor([1], name+'_1', kwargs[\"initializer\"])\n    create_const_scalar_node(name+'_0_s', np.int64(0), kwargs)\n    create_const_scalar_node(name+'_1_s', np.int64(1), kwargs)\n\n    nodes = [\n        make_node('Shape', [input_nodes[0]], [shape_name]),\n        make_node('Shape', [shape_name], [name+'_in_dim']),\n        make_node('Squeeze', [name+'_in_dim', name+'_0'], [name+'_in_dim_s']),\n        make_node('Range', [name+'_0_s', name+'_in_dim_s', name+'_1_s'], [name+'_range']),\n    ]\n\n    for i, axis in enumerate(axis):\n        if axis not in (0, 1):\n            create_tensor([axis], name+'_'+str(axis), kwargs[\"initializer\"])\n        create_tensor([size[i]-1], name+'_size_'+str(i), kwargs[\"initializer\"])\n        nodes += [\n            make_node('Equal', [name+'_range', name+'_'+str(axis)], [name+'_equal_'+str(i)]),\n            make_node('Cast', [name+'_equal_'+str(i)], [name+'_cast_'+str(i)], to=int(TensorProto.INT64)),\n            make_node('Mul', [name+'_size_'+str(i), name+'_cast_'+str(i)], [name+'_mul_'+str(i)]),\n            make_node('Add', [name+'_mul_'+str(i), name+'_1'], [name+'_add_'+str(i)]),\n            make_node('Mul', [name+'_add_'+str(i), shape_name], [name+'_shape_'+str(i+1)])\n        ]\n        shape_name = name+'_shape_'+str(i+1)\n\n    nodes += [\n        make_node('Expand', [input_nodes[0], shape_name], [name], name=name)\n    ]\n\n    return nodes\n\n\n@mx_op.register(\"SequenceMask\", OPSET_VERSION)\ndef convert_sequencemask(node, **kwargs):\n    \"\"\"Map MXNet's SequenceMask operator\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    use_sequence_length = attrs.get('use_sequence_length', 'False')\n    mask_val = float(attrs.get('value', '0'))\n    axis = int(attrs.get('axis', '0'))\n\n    if(use_sequence_length == 'False'):\n        return [make_node('Identity', [input_nodes[0]], [name], name=name)]\n\n    create_tensor([0], name+'_0', kwargs[\"initializer\"])\n    create_tensor([1], name+'_1', kwargs[\"initializer\"])\n    create_tensor([2], name+'_2', kwargs[\"initializer\"])\n    create_const_scalar_node(name+'_0_s', np.int64(0), kwargs)\n    create_const_scalar_node(name+'_1_s', np.int64(1), kwargs)\n    create_const_scalar_node(name+'_2_s', np.int64(2), kwargs)\n    create_tensor([mask_val], name+'_mask_val', kwargs[\"initializer\"], dtype='float32')\n\n    nodes = [\n        make_node('Shape', [input_nodes[0]], [name+'_in_shape']),\n        make_node('Slice', [name+'_in_shape', name+'_0', name+'_1'], [name+'_slice_0']),\n        make_node('Slice', [name+'_in_shape', name+'_1', name+'_2'], [name+'_slice_1']),\n        make_node('Concat', [name+'_slice_0', name+'_1'], [name+'_shape_0'], axis=0),\n        make_node('Shape', [name+'_in_shape'], [name+'_in_dim']),\n        make_node('Squeeze', [name+'_in_dim', name+'_0'], [name+'_in_dim_s']),\n        make_node('Range', [name+'_0_s', name+'_in_dim_s', name+'_1_s'], [name+'_range_0']),\n        make_node('Less', [name+'_range_0', name+'_2'], [name+'_less_0']),\n        make_node('Where', [name+'_less_0', name+'_in_shape', name+'_1'], [name+'_shape_1'])\n    ]\n\n    if(axis == 0):\n        nodes += [\n            make_node('Squeeze', [name+'_slice_0', name+'_0'], [name+'_max_len']),\n            make_node('Range', [name+'_0_s', name+'_max_len', name+'_1_s'], [name+'_range_1']),\n            make_node('Reshape', [name+'_range_1', name+'_shape_0'], [name+\"_reshape_0\"]),\n            make_node('Cast', [input_nodes[1]], [name+'_cast'], to=int(TensorProto.INT64)),\n            make_node('Less', [name+'_reshape_0', name+'_cast'], [name+'_less_1']),\n            make_node('Reshape', [name+'_less_1', name+'_shape_1'], [name+\"_reshape_1\"]),\n            make_node('Where', [name+'_reshape_1', input_nodes[0], name+'_mask_val'], [name], name=name),\n        ]\n    else:\n        nodes += [\n            make_node('Squeeze', [name+'_slice_1', name+'_0'], [name+'_max_len']),\n            make_node('Range', [name+'_0_s', name+'_max_len', name+'_1_s'], [name+'_range_1']),\n            make_node('Reshape', [input_nodes[1], name+'_shape_0'], [name+\"_reshape_0\"]),\n            make_node('Cast', [name+\"_reshape_0\"], [name+'_cast'], to=int(TensorProto.INT64)),\n            make_node('Less', [name+'_range_1', name+'_cast'], [name+'_less_1']),\n            make_node('Reshape', [name+'_less_1', name+'_shape_1'], [name+\"_reshape_1\"]),\n            make_node('Where', [name+'_reshape_1', input_nodes[0], name+'_mask_val'], [name], name=name),\n        ]\n    return nodes\n\n\n@mx_op.register(\"expand_dims\", OPSET_VERSION)\ndef convert_expand_dims(node, **kwargs):\n    \"\"\"Map MXNet's expand_dims operator attributes to onnx's Unsqueeze operator\n    and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    axis = int(attrs.get(\"axis\"))\n    create_tensor([axis], name+\"_axis\", kwargs[\"initializer\"])\n    input_nodes.append(name+\"_axis\")\n    node = onnx.helper.make_node(\n        \"Unsqueeze\",\n        input_nodes,\n        [name],\n        name=name,\n    )\n    return [node]\n\n\n@mx_op.register(\"stack\", OPSET_VERSION)\ndef convert_stack(node, **kwargs):\n    \"\"\"Map MXNet's stack operator to onnx operators.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    axis = int(attrs.get(\"axis\", 0))\n    create_tensor([axis], name+\"_axis\", kwargs[\"initializer\"])\n    idx = 0\n    nodes = []\n    for input_node in input_nodes:\n        nodes.append(onnx.helper.make_node(\n            \"Unsqueeze\",\n            inputs=[input_node, name+\"_axis\"],\n            outputs=[name+\"_unsqueeze\"+str(idx)]\n        ))\n        idx += 1\n\n    nodes.append(onnx.helper.make_node(\n        \"Concat\",\n        inputs=[name+\"_unsqueeze\"+str(i) for i in range(len(nodes))],\n        outputs=[name],\n        name=name,\n        axis=axis\n    ))\n    return nodes\n\n\n@mx_op.register(\"softmax\", OPSET_VERSION)\ndef convert_softmax(node, **kwargs):\n    \"\"\"Map MXNet's softmax operator attributes to onnx's Softmax operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    axis = int(attrs.get(\"axis\", -1))\n    temperature = str(attrs.get(\"temperature\", 'None'))\n    if temperature == 'None':\n        temperature = 1.\n    else:\n        temperature = float(temperature)\n\n    use_length = str(attrs.get(\"use_length\", 'None'))\n    use_length = use_length in ['1', 'True']\n    dtype = input_dtypes[0]\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n    data = input_nodes[0]\n\n    create_tensor([0], name+\"_0\", kwargs[\"initializer\"])\n    if axis == -1 and temperature == 1.:\n        nodes = []\n        if use_length:\n            # magic number, this is fp16 min\n            create_tensor([-65500.0], name+\"_mask_val\", kwargs[\"initializer\"], dtype=dtype)\n            create_tensor([1], name+\"_1\", kwargs[\"initializer\"])\n            create_tensor([-1], name+\"_-1\", kwargs[\"initializer\"])\n            create_const_scalar_node(name+\"_0_s\", np.int64(0), kwargs)\n            create_const_scalar_node(name+\"_1_s\", np.int64(1), kwargs)\n            nodes += [\n                make_node(\"Shape\", [data], [name+\"_shape\"]),\n                make_node(\"Shape\", [name+\"_shape\"], [name+\"_dim\"]),\n                make_node(\"Sub\", [name+\"_dim\", name+\"_1\"], [name+\"_dim_m1\"]),\n                make_node(\"Slice\", [name+\"_shape\", name+\"_dim_m1\", name+\"_dim\"],\n                          [name+\"_dim_last_\"]),\n                make_node(\"Squeeze\", [name+\"_dim_last_\", name+\"_0\"], [name+\"_dim_last\"]),\n                make_node(\"Range\", [name+\"_0_s\", name+\"_dim_last\", name+\"_1_s\"], [name+\"_range\"]),\n                make_node(\"Cast\", [input_nodes[1]], [name+\"_len\"], to=int(TensorProto.INT64)),\n                make_node(\"Unsqueeze\", [name+\"_len\", name+\"_-1\"], [name+\"_len_unsqueezed\"]),\n                make_node(\"Less\", [name+\"_range\", name+\"_len_unsqueezed\"], [name+\"_less\"]),\n                make_node(\"Where\", [name+'_less', data, name+\"_mask_val\"], [name+\"_data_masked\"])\n            ]\n            data = name+\"_data_masked\"\n\n        nodes += [\n            make_node(\"Softmax\", [data], [name], axis=-1)\n        ]\n\n        return nodes\n\n    create_tensor([axis], name+\"_axes\", kwargs[\"initializer\"])\n    create_tensor([temperature], name+\"_tmp\", kwargs[\"initializer\"], dtype=dtype)\n    nodes = [\n        make_node(\"Div\", [data, name+\"_tmp\"], [name+'_data']),\n    ]\n    if len(input_nodes) == 1:\n        nodes += [\n            make_node(\"Softmax\", [name+'_data'], [name], axis=axis)\n        ]\n        return nodes\n    elif use_length:\n        length = input_nodes[1]\n        create_tensor([1], name+\"_1\", kwargs[\"initializer\"])\n        create_const_scalar_node(name+'_-1_s', np.int64(-1), kwargs)\n        create_const_scalar_node(name+'_0_s', np.int64(0), kwargs)\n        create_const_scalar_node(name+'_1_s', np.int64(1), kwargs)\n        nodes += [\n            # cast data type\n            make_node(\"Cast\", [length], [name+\"_length\"], to=int(TensorProto.INT64)),\n            make_node(\"Cast\", [name+\"_0\"], [name+\"_0_itype\"], to=dtype_t),\n            make_node(\"Cast\", [name+\"_1\"], [name+\"_1_itype\"], to=dtype_t),\n            # softmax output\n            make_node(\"Softmax\", [name+'_data'], [name+\"_softmax_out\"], axis=axis),\n            # update axis\n            make_node(\"Shape\", [data], [name+\"_shape0_out\"]),\n            make_node(\"Shape\", [name+\"_shape0_out\"], [name+\"_in_dim\"]),\n            make_node(\"Add\", [name+\"_in_dim\", name+\"_axes\"], [name+\"_dim+axis\"]),\n            make_node(\"Less\", [name+\"_axes\", name+\"_0_s\"], [name+\"_less0_out\"]),\n            make_node(\"Where\", [name+\"_less0_out\", name+\"_dim+axis\", name+\"_axes\"], [name+\"_final_axis\"]),\n            # data mask\n            make_node(\"Add\", [name+\"_final_axis\", name+\"_1_s\"], [name+\"_final_axis+1\"]),\n            make_node(\"Slice\", [name+\"_shape0_out\", name+\"_final_axis\", name+\"_final_axis+1\"], [name+\"_axis_dim\"]),\n            make_node(\"Squeeze\", [name+\"_axis_dim\", name+\"_0\"], [name+\"_axis_dim_s\"]),\n            make_node(\"Range\", [name+\"_0_s\", name+\"_axis_dim_s\", name+\"_1_s\"], [name+\"_range0_out\"]),\n            # one hot for axis\n            make_node(\"Squeeze\", [name+\"_in_dim\", name+\"_0\"], [name+\"_in_dim_s\"]),\n            make_node(\"Range\", [name+\"_0_s\", name+\"_in_dim_s\", name+\"_1_s\"], [name+\"_range1_out\"]),\n            make_node(\"Equal\", [name+\"_range1_out\", name+\"_final_axis\"], [name+\"_equal_out\"]),\n            make_node(\"Cast\", [name+\"_equal_out\"], [name+\"_one_hot\"], to=int(TensorProto.INT64)),\n            # reshape data mask for less\n            make_node(\"Sub\", [name+\"_axis_dim_s\", name+\"_1_s\"], [name+\"_sub0_out\"]),\n            make_node(\"Mul\", [name+\"_one_hot\", name+\"_sub0_out\"], [name+\"_mul0_out\"]),\n            make_node(\"Add\", [name+\"_mul0_out\", name+\"_1_s\"], [name+\"_add0_out\"]),\n            make_node('Reshape', [name+\"_range0_out\", name+\"_add0_out\"], [name+\"_reshape0_out\"]),\n            # reshape length for less\n            make_node(\"Mul\", [name+\"_one_hot\", name+\"_-1_s\"], [name+\"_mul1_out\"]),\n            make_node(\"Add\", [name+\"_mul1_out\", name+\"_1_s\"], [name+\"_add1_out\"]),\n            make_node(\"Sub\", [name+\"_shape0_out\", name+\"_1_s\"], [name+\"_sub1_out\"]),\n            make_node(\"Mul\", [name+\"_add1_out\", name+\"_sub1_out\"], [name+\"_mul2_out\"]),\n            make_node(\"Add\", [name+\"_mul2_out\", name+\"_1_s\"], [name+\"_add2_out\"]),\n            make_node('Reshape', [name+\"_length\", name+\"_add2_out\"], [name+\"_reshape1_out\"]),\n            # mask output\n            make_node(\"Less\", [name+\"_reshape0_out\", name+\"_reshape1_out\"], [name+\"_less_out\"]),\n            make_node(\"Cast\", [name+\"_less_out\"], [name+\"_mask\"], to=dtype_t),\n            make_node(\"Mul\", [name+\"_softmax_out\", name+\"_mask\"], [name+\"_mul3_out\"]),\n            make_node(\"ReduceSum\", [name+\"_mul3_out\", name+\"_axes\"], [name+\"_rsum1_out\"], keepdims=1),\n            make_node(\"Equal\", [name+\"_rsum1_out\", name+\"_0_itype\"], [name+\"_equal1_out\"]),\n            make_node(\"Where\", [name+\"_equal1_out\", name+\"_1_itype\", name+\"_rsum1_out\"], [name+\"_where_out\"]),\n            make_node(\"Div\", [name+\"_mul3_out\", name+\"_where_out\"], [name], name=name)\n        ]\n        return nodes\n\n    else:\n        raise NotImplementedError(\"use_length must be true when both data and length are paased in.\")\n\n\n@mx_op.register(\"reverse\", OPSET_VERSION)\ndef convert_reverse(node, **kwargs):\n    \"\"\"Map MXNet's reverse operator attributes to ONNX\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    axis = int(attrs.get('axis', 0))\n\n    # Transpose takes perm as a parameter, so we must 'pad' the input to a known dim (8 here)\n    perm = [i for i in range(8)]\n    perm[0], perm[axis] = axis, 0\n\n    create_tensor([8], name+'_8', kwargs['initializer'])\n    create_tensor([0], name+'_0', kwargs['initializer'])\n    create_tensor([1], name+'_1', kwargs['initializer'])\n    create_tensor([-1], name+'_m1', kwargs['initializer'])\n    create_tensor([axis], name+'_axis', kwargs['initializer'])\n    create_tensor([axis+1], name+'_axis_p1', kwargs['initializer'])\n    create_const_scalar_node(name+'_m1_s', np.int64(-1), kwargs)\n\n    nodes = [\n        make_node('Shape', [input_nodes[0]], [name+'_shape']),\n        make_node('Shape', [name+'_shape'], [name+'_dim']),\n        make_node('Sub', [name+'_8', name+'_dim'], [name+'_sub']),\n        make_node('Concat', [name+'_0', name+'_sub'], [name+'_concat'], axis=0),\n        make_node('Pad', [name+'_shape', name+'_concat', name+'_1'], [name+'_shape_8_dim']),\n        make_node('Reshape', [input_nodes[0], name+'_shape_8_dim'], [name+'_data_8_dim']),\n        make_node('Transpose', [name+'_data_8_dim'], [name+'_data_t'], perm=perm),\n        make_node('Slice', [name+'_shape', name+'_axis', name+'_axis_p1'], [name+'_axis_len']),\n        make_node('Sub', [name+'_axis_len', name+'_1'], [name+'_axis_len_m1']),\n        make_node('Squeeze', [name+'_axis_len_m1', name+'_0'], [name+'_axis_len_m1_s']),\n        make_node('Range', [name+'_axis_len_m1_s', name+'_m1_s', name+'_m1_s'], [name+'_indices']),\n        make_node('Gather', [name+'_data_t', name+'_indices'], [name+'_gather']),\n        make_node('Transpose', [name+'_gather'], [name+'_data_reversed'], perm=perm),\n        make_node('Reshape', [name+'_data_reversed', name+'_shape'], [name], name=name)\n    ]\n\n    return nodes\n\n\n@mx_op.register('repeat', OPSET_VERSION)\ndef convert_repeat(node, **kwargs):\n    \"\"\"Map MXNet's repeat operator attributes to onnx's Tile operator.\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    opset_version = kwargs['opset_version']\n    if opset_version < 11:\n        raise AttributeError('ONNX opset 11 or greater is required to export this operator')\n\n    repeats = int(attrs.get('repeats', 1))\n    axis = attrs.get('axis', 'None')\n\n    if repeats <= 0:\n        raise NotImplementedError('repeat operator does not support parameter repeats==0')\n\n    nodes = []\n    if axis == 'None':\n        create_tensor([-1], name+'_-1', kwargs['initializer'])\n        create_tensor([repeats], name+'_rep', kwargs['initializer'])\n        create_tensor([1, repeats], name+'_repeats', kwargs['initializer'])\n        nodes += [\n            make_node('Shape', [input_nodes[0]], [name+'_shape']),\n            make_node('ReduceProd', [name+'_shape'], [name+'_size']),\n            make_node('Reshape', [input_nodes[0], name+'_size'], [name+'_flat']),\n            make_node('Unsqueeze', [name+'_flat', name+'_-1'], [name+'_unsqueeze']),\n            make_node('Tile', [name+'_unsqueeze', name+'_repeats'], [name+'_tile']),\n            make_node('Mul', [name+'_size', name+'_rep'], [name+'_new_size']),\n            make_node('Reshape', [name+'_tile', name+'_new_size'], [name], name=name)\n        ]\n    else:\n        axis = int(axis)\n        repeats -= 1\n        create_tensor([repeats], name+'_repeats', kwargs['initializer'])\n        create_tensor([1], name+'_1', kwargs['initializer'])\n        create_tensor([0], name+'_0', kwargs['initializer'])\n        create_tensor([axis], name+'_axis', kwargs['initializer'])\n        create_const_scalar_node(name+\"_0_s\", np.int64(0), kwargs)\n        create_const_scalar_node(name+\"_1_s\", np.int64(1), kwargs)\n        nodes += [\n            make_node('Shape', [input_nodes[0]], [name+'_shape']),\n            make_node('Shape', [name+'_shape'], [name+'_dim']),\n            make_node('Squeeze', [name+'_dim', name+'_0'], [name+'_dim_s']),\n            make_node('Range', [name+'_0_s', name+'_dim_s', name+'_1_s'], [name+'_range'])\n        ]\n        if axis < 0:\n            nodes += [\n                make_node('Add', [name+'_axis', name+'_dim'], [name+'_true_axis']),\n                make_node('Equal', [name+'_range', name+'_true_axis'], [name+'_one_hot'])\n                ]\n        else:\n            nodes += [\n                make_node('Equal', [name+'_range', name+'_axis'], [name+'_one_hot'])\n                ]\n        nodes += [\n            make_node('Cast', [name+'_one_hot'], [name+'_one_hot_int'], to=int(TensorProto.INT64)),\n            make_node('Mul', [name+'_repeats', name+'_one_hot_int'], [name+'_mul']),\n            make_node('Add', [name+'_mul', name+'_1'], [name+'_add']),\n            make_node('Concat', [name+'_1', name+'_add'], [name+'_repeats_tensor'], axis=0)\n            ]\n        if axis == -1:\n            nodes += [\n                make_node('Concat', [name+'_shape', name+'_1'], [name+'_unsqueeze_shape'], axis=0),\n                make_node('Reshape', [input_nodes[0], name+'_unsqueeze_shape'],\n                          [name+'_unsqueeze'])\n                ]\n        else:\n            create_tensor([axis+1], name+'_axis+1', kwargs['initializer'])\n            nodes += [\n                make_node('Unsqueeze', [input_nodes[0], name+'_axis+1'], [name+'_unsqueeze'])\n                ]\n        nodes += [\n            make_node('Tile', [name+'_unsqueeze', name+'_repeats_tensor'], [name+'_tile']),\n            make_node('Mul', [name+'_shape', name+'_add'], [name+'_new_shape']),\n            make_node('Reshape', [name+'_tile', name+'_new_shape'], [name], name=name)\n            ]\n\n    return nodes\n\n\n@mx_op.register('_contrib_box_nms', OPSET_VERSION)\ndef convert_contrib_box_nms(node, **kwargs):\n    \"\"\"Map MXNet's _contrib_box_nms operator to ONNX\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    input_dtypes = get_input_dtypes(node, kwargs)\n\n    dtype = input_dtypes[0]\n    #dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    opset_version = kwargs['opset_version']\n    if opset_version < 11:\n        raise AttributeError('ONNX opset 11 or greater is required to export this operator')\n\n    overlap_thresh = float(attrs.get('overlap_thresh', '0.5'))\n    valid_thresh = float(attrs.get('valid_thresh', '0'))\n    topk = int(attrs.get('topk', '-1'))\n    coord_start = int(attrs.get('coord_start', '2'))\n    score_index = int(attrs.get('score_index', '1'))\n    id_index = int(attrs.get('id_index', '-1'))\n    force_suppress = attrs.get('force_suppress', 'True')\n    background_id = int(attrs.get('background_id', '-1'))\n    in_format = attrs.get('in_format', 'corner')\n    out_format = attrs.get('out_format', 'corner')\n\n    center_point_box = 0 if in_format == 'corner' else 1\n\n    if topk == -1:\n        topk = 2**31-1\n\n    if in_format != out_format:\n        raise NotImplementedError('box_nms does not currently support in_fomat != out_format')\n\n    if background_id != -1:\n        raise NotImplementedError('box_nms does not currently support background_id != -1')\n\n    if id_index != -1 or force_suppress == 'False':\n        logging.warning('box_nms: id_idex != -1 or/and force_suppress == False detected. '\n                        'However, due to ONNX limitations, boxes of different categories will NOT '\n                        'be exempted from suppression. This might lead to different behavior than '\n                        'native MXNet')\n\n    create_tensor([coord_start], name+'_cs', kwargs['initializer'])\n    create_tensor([coord_start+4], name+'_cs_p4', kwargs['initializer'])\n    create_tensor([score_index], name+'_si', kwargs['initializer'])\n    create_tensor([score_index+1], name+'_si_p1', kwargs['initializer'])\n    create_tensor([topk], name+'_topk', kwargs['initializer'])\n    create_tensor([overlap_thresh], name+'_ot', kwargs['initializer'], dtype=np.float32)\n    create_tensor([valid_thresh], name+'_vt', kwargs['initializer'], dtype=np.float32)\n    create_tensor([-1], name+'_m1', kwargs['initializer'])\n    create_tensor([-1], name+'_m1_f', kwargs['initializer'], dtype=dtype)\n    create_tensor([0], name+'_0', kwargs['initializer'])\n    create_tensor([1], name+'_1', kwargs['initializer'])\n    create_tensor([2], name+'_2', kwargs['initializer'])\n    create_tensor([3], name+'_3', kwargs['initializer'])\n    create_tensor([0, 1, -1], name+'_scores_shape', kwargs['initializer'])\n    create_tensor([0, 0, 1, 0], name+'_pad', kwargs['initializer'])\n    create_tensor([0, -1], name+'_bat_spat_helper', kwargs['initializer'])\n    create_const_scalar_node(name+\"_0_s\", np.int64(0), kwargs)\n    create_const_scalar_node(name+\"_1_s\", np.int64(1), kwargs)\n\n    nodes = [\n        make_node('Shape', [input_nodes[0]], [name+'_shape']),\n        make_node('Shape', [name+'_shape'], [name+'_dim']),\n        make_node('Sub', [name+'_dim', name+'_2'], [name+'_dim_m2']),\n        make_node('Slice', [name+'_shape', name+'_dim_m2', name+'_dim'], [name+'_shape_last2']),\n        make_node('Concat', [name+'_m1', name+'_shape_last2'], [name+'_shape_3d'], axis=0),\n        make_node('Reshape', [input_nodes[0], name+'_shape_3d'], [name+'_data_3d']),\n        make_node('Slice', [name+'_data_3d', name+'_cs', name+'_cs_p4', name+'_m1'],\n                  [name+'_boxes']),\n        make_node('Slice', [name+'_data_3d', name+'_si', name+'_si_p1', name+'_m1'],\n                  [name+'_scores_raw']),\n        make_node('Reshape', [name+'_scores_raw', name+'_scores_shape'], [name+'_scores']),\n        make_node('Shape', [name+'_scores'], [name+'_scores_shape_actual']),\n        make_node('NonMaxSuppression',\n                  [name+'_boxes', name+'_scores', name+'_topk', name+'_ot', name+'_vt'],\n                  [name+'_nms'], center_point_box=center_point_box),\n        make_node('Slice', [name+'_nms', name+'_0', name+'_3', name+'_m1', name+'_2'],\n                  [name+'_nms_sliced']),\n        make_node('GatherND', [name+'_data_3d', name+'_nms_sliced'], [name+'_candidates']),\n        make_node('Pad', [name+'_candidates', name+'_pad', name+'_m1_f'], [name+'_cand_padded']),\n        make_node('Shape', [name+'_nms'], [name+'_nms_shape']),\n        make_node('Slice', [name+'_nms_shape', name+'_0', name+'_1'], [name+'_cand_cnt']),\n        make_node('Squeeze', [name+'_cand_cnt', name+'_0'], [name+'_cc_s']),\n        make_node('Range', [name+'_0_s', name+'_cc_s', name+'_1_s'], [name+'_cand_indices']),\n        make_node('Slice', [name+'_scores_shape_actual', name+'_0', name+'_3', name+'_m1',\n                            name+'_2'], [name+'_shape_bat_spat']),\n        make_node('Slice', [name+'_shape_bat_spat', name+'_1', name+'_2'], [name+'_spat_dim']),\n        make_node('Expand', [name+'_cand_cnt', name+'_shape_bat_spat'], [name+'_base_indices']),\n        make_node('ScatterND', [name+'_base_indices', name+'_nms_sliced', name+'_cand_indices'],\n                  [name+'_indices']),\n        make_node('TopK', [name+'_indices', name+'_spat_dim'], [name+'_indices_sorted', name+'__'],\n                  largest=0, axis=-1, sorted=1),\n        make_node('Gather', [name+'_cand_padded', name+'_indices_sorted'], [name+'_gather']),\n        make_node('Reshape', [name+'_gather', name+'_shape'], [name+'0'])\n    ]\n\n    return nodes\n\n\n@mx_op.register('_contrib_ROIAlign', OPSET_VERSION)\ndef convert_contrib_roialign(node, **kwargs):\n    \"\"\"Map MXNet's _contrib_ROIAlign\n    \"\"\"\n    from onnx.helper import make_node\n    from onnx import TensorProto\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    pooled_size = convert_string_to_list(str(attrs.get('pooled_size')))\n    spatial_scale = float(attrs.get('spatial_scale'))\n    sample_ratio = int(attrs.get('sample_ratio', '0'))\n    position_sensitive = attrs.get('position_sensitive', 'False')\n    aligned = attrs.get('aligned', 'False')\n\n    if position_sensitive != 'False':\n        raise NotImplementedError('_contrib_ROIAlign does not currently support \\\n                                   position_sensitive!=False')\n    if aligned != 'False':\n        raise NotImplementedError('_contrib_ROIAlign does not currently support \\\n                                   aligned!=False')\n\n    create_tensor([0], name+'_0', kwargs['initializer'])\n    create_tensor([0], name+'_0_s', kwargs['initializer'], dtype='float32')\n    create_tensor([1], name+'_1', kwargs['initializer'])\n    create_tensor([5], name+'_5', kwargs['initializer'])\n    create_tensor([2, 3], name+'_2_3', kwargs['initializer'])\n\n    nodes = [\n        make_node('Slice', [input_nodes[1], name+'_1', name+'_5', name+'_1'], [name+'_rois']),\n        make_node('Slice', [input_nodes[1], name+'_0', name+'_1', name+'_1'], [name+'_inds___']),\n        make_node('Squeeze', [name+'_inds___', name+'_1'], [name+'_inds__']),\n        make_node('Relu', [name+'_inds__'], [name+'_inds_']),\n        make_node('Cast', [name+'_inds_'], [name+'_inds'], to=int(TensorProto.INT64)),\n        make_node('RoiAlign', [input_nodes[0], name+'_rois', name+'_inds'], [name+'_roi'],\n                  mode='avg', output_height=pooled_size[0], output_width=pooled_size[1],\n                  sampling_ratio=sample_ratio, spatial_scale=spatial_scale),\n        make_node('Unsqueeze', [name+'_inds___', name+'_2_3'], [name+'_unsq']),\n        make_node('Less', [name+'_unsq', name+'_0_s'], [name+'_less']),\n        make_node('Where', [name+'_less', name+'_0_s', name+'_roi'], [name])\n    ]\n\n    return nodes\n\n\n@mx_op.register(\"sum\", OPSET_VERSION)\n@mx_op.register(\"_npi_sum\", OPSET_VERSION)\ndef convert_sum(node, **kwargs):\n    \"\"\"Map MXNet's sum operator attributes to onnx's ReduceSum operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = attrs.get(\"axis\", None)\n    axes = convert_string_to_list(str(mx_axis)) if mx_axis not in [None, 'None'] else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n\n    if axes:\n        create_tensor(axes, name+'_axes', kwargs['initializer'])\n        input_nodes.append(name+'_axes')\n        node = make_node(\n            'ReduceSum',\n            inputs=input_nodes,\n            outputs=[name],\n            keepdims=keepdims,\n            name=name\n        )\n        return [node]\n    else:\n        create_tensor([1], name+'_1', kwargs['initializer'])\n        nodes = [\n            onnx.helper.make_node(\n                'ReduceSum',\n                inputs=input_nodes,\n                outputs=[name],\n                keepdims=keepdims,\n            )\n        ]\n    return nodes\n\n\n@mx_op.register(\"RNN\", OPSET_VERSION)\ndef convert_RNN(node, **kwargs):\n    \"\"\"Map MXNet's RNN operator attributes to onnx's operators\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node, make_tensor\n    from onnx import TensorProto\n\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mode = str(attrs.get('mode'))\n    bidirectional = str(attrs.get('bidirectional', 'False'))\n    if bidirectional != 'False' and mode not in ['lstm']:\n        raise NotImplementedError('Currently RNN onnx export only supports bidirectional is False')\n\n    num_layers = int(attrs.get('num_layers', '1'))\n\n    use_sequence_length = str(attrs.get('use_sequence_length', 'False'))\n    if use_sequence_length != 'False':\n        raise NotImplementedError('Currently RNN onnx export only supports use_sequence_length equals to False')\n\n    projection_size = str(attrs.get('projection_size', 'None'))\n    if projection_size != 'None':\n        raise NotImplementedError('Currently RNN onnx export only supports projection_size equals to None')\n\n    state_outputs = str(attrs.get('state_outputs', 'False'))\n    if state_outputs != 'True':\n        raise NotImplementedError('Currently RNN onnx export only supports state_outputs equals to True')\n\n    state_size = int(attrs.get('state_size'))\n\n    direction = 1\n    if bidirectional != 'False':\n        direction = 2\n\n    data = input_nodes[0]\n    param = input_nodes[1]\n    dtype = get_input_dtypes(node, kwargs)[2]\n\n    create_tensor([0], name+'_0', kwargs['initializer'])\n    create_tensor([1], name+'_1', kwargs['initializer'])\n    create_tensor([state_size], name+'_state_size', kwargs['initializer'])\n    create_tensor([direction], name+'_direction', kwargs['initializer'])\n\n    tensor_1 = make_tensor(name+'_1_f', onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype], [1], [1])\n\n    nodes = [\n        make_node('Shape', [data], [name+'_data_shape']),\n        make_node('Split', [name+'_data_shape'], [name+'_seq_length', name+'_batch_size', name+'_input_size']),\n        make_node('Concat', [name+'_direction', name+'_batch_size', name+'_state_size'], [name+'_concat'], axis=0),\n        make_node('ConstantOfShape', [name+'_concat'], [name+'_COS'], value=tensor_1),\n        make_node('Mul', [input_nodes[2], name+'_COS'], [name+'initial_h']),\n\n    ]\n\n    if mode == 'lstm':\n        nodes += [\n            make_node('Mul', [input_nodes[3], name+'_COS'], [name+'initial_c']),\n        ]\n\n        if num_layers == 2:\n            if bidirectional != 'False':\n                raise NotImplementedError('Currently lstm onnx export only supports bidirectional when num_layers = 1')\n            create_tensor([8*state_size], name+'_8*state_size', kwargs['initializer'])\n            create_tensor([4*state_size*state_size], name+'_4*state_size^2', kwargs['initializer'])\n            create_tensor([1, 4*state_size, state_size], name+'_WR_shape', kwargs['initializer'])\n            create_tensor([1, 8*state_size], name+'_B_shape', kwargs['initializer'])\n            create_tensor([4*4*state_size*state_size], name+'_WR_offset', kwargs['initializer'])\n\n            nodes += [\n                # Layer 0\n                # get W\n                make_node('Slice', [param, name+'_0', name+'_4*state_size^2'], [name+'_W0_1d']),\n                make_node('Split', [name+'_W0_1d'], [name+'_W00', name+'_W01', name+'_W02', name+'_W03']),\n                make_node('Concat', [name+'_W00', name+'_W03', name+'_W01', name+'_W02'], [name+'_W0_'], axis=0),\n                make_node('Reshape', [name+'_W0_', name+'_WR_shape'], [name+'_W0']),\n                # get R\n                make_node('Add', [name+'_4*state_size^2', name+'_4*state_size^2'], [name+'_R0_offset']),\n                make_node('Slice', [param, name+'_4*state_size^2', name+'_R0_offset'], [name+'_R0_1d']),\n                make_node('Split', [name+'_R0_1d'], [name+'_R00', name+'_R01', name+'_R02', name+'_R03']),\n                make_node('Concat', [name+'_R00', name+'_R03', name+'_R01', name+'_R02'], [name+'_R0_'], axis=0),\n                make_node('Reshape', [name+'_R0_', name+'_WR_shape'], [name+'_R0']),\n                # get B\n                make_node('Add', [name+'_WR_offset', name+'_8*state_size'], [name+'_B0_offset']),\n                make_node('Slice', [param, name+'_WR_offset', name+'_B0_offset'], [name+'_B0_1d']),\n                make_node('Split', [name+'_B0_1d'], [name+'_B00', name+'_B01', name+'_B02', name+'_B03',\n                                                     name+'_B04', name+'_B05', name+'_B06', name+'_B07']),\n                make_node('Concat', [name+'_B00', name+'_B03', name+'_B01', name+'_B02',\n                                     name+'_B04', name+'_B07', name+'_B05', name+'_B06'], [name+'_B0_'], axis=0),\n                make_node('Reshape', [name+'_B0_', name+'_B_shape'], [name+'_B0']),\n                # get initial states\n                make_node('Split', [name+'initial_h'], [name+'_initial_h0', name+'_initial_h1'], axis=0),\n                make_node('Split', [name+'initial_c'], [name+'_initial_c0', name+'_initial_c1'], axis=0),\n                # get seq_len\n                make_node('Tile', [name+'_seq_length', name+'_batch_size'], [name+'_seq_len_']),\n                make_node(\"Cast\", [name+'_seq_len_'], [name+\"_seq_len\"], to=int(TensorProto.INT32)),\n                # Layer 0 LSTM\n                make_node('LSTM', [data, name+'_W0', name+'_R0', name+'_B0', name+'_seq_len',\n                                   name+'_initial_h0', name+'_initial_c0'],\n                          [name+'_lstm0_out_', name+'_lstm0_h', name+'_lstm0_c'], hidden_size=state_size),\n                make_node('Squeeze', [name+'_lstm0_out_', name+'_1'], [name+'_lstm0_out']),\n\n                # Layer 1\n                # get W\n                make_node('Add', [name+'_R0_offset', name+'_4*state_size^2'], [name+'_W1_offset']),\n                make_node('Slice', [param, name+'_R0_offset', name+'_W1_offset'], [name+'_W1_1d']),\n                make_node('Split', [name+'_W1_1d'], [name+'_W10', name+'_W11', name+'_W12', name+'_W13']),\n                make_node('Concat', [name+'_W10', name+'_W13', name+'_W11', name+'_W12'], [name+'_W1_'], axis=0),\n                make_node('Reshape', [name+'_W1_', name+'_WR_shape'], [name+'_W1']),\n                # get R\n                make_node('Slice', [param, name+'_W1_offset', name+'_WR_offset'], [name+'_R1_1d']),\n                make_node('Split', [name+'_R1_1d'], [name+'_R10', name+'_R11', name+'_R12', name+'_R13']),\n                make_node('Concat', [name+'_R10', name+'_R13', name+'_R11', name+'_R12'], [name+'_R1_'], axis=0),\n                make_node('Reshape', [name+'_R1_', name+'_WR_shape'], [name+'_R1']),\n                # get B\n                make_node('Add', [name+'_B0_offset', name+'_8*state_size'], [name+'_B1_offset']),\n                make_node('Slice', [param, name+'_B0_offset', name+'_B1_offset'], [name+'_B1_1d']),\n                make_node('Split', [name+'_B1_1d'], [name+'_B10', name+'_B11', name+'_B12', name+'_B13',\n                                                     name+'_B14', name+'_B15', name+'_B16', name+'_B17']),\n                make_node('Concat', [name+'_B10', name+'_B13', name+'_B11', name+'_B12',\n                                     name+'_B14', name+'_B17', name+'_B15', name+'_B16'], [name+'_B1_'], axis=0),\n                make_node('Reshape', [name+'_B1_', name+'_B_shape'], [name+'_B1']),\n                # Layer 1 LSTM\n                make_node('LSTM', [name+'_lstm0_out', name+'_W1', name+'_R1', name+'_B1', name+'_seq_len',\n                                   name+'_initial_h1', name+'_initial_c1'],\n                          [name+'_lstm1_out_', name+'_lstm1_h', name+'_lstm1_c'], hidden_size=state_size),\n                make_node('Squeeze', [name+'_lstm1_out_', name+'_1'], [name]),\n                make_node('Concat', [name+'_lstm0_h', name+'_lstm1_h'], [name+'1'], axis=0),\n                make_node('Concat', [name+'_lstm0_c', name+'_lstm1_c'], [name+'2'], axis=0),\n            ]\n        elif num_layers == 1:\n            if bidirectional == 'False':\n                create_tensor([4*state_size], name+'_4*state_size', kwargs['initializer'])\n                create_tensor([8*state_size], name+'_8*state_size', kwargs['initializer'])\n                create_tensor([4*state_size*state_size], name+'_4*state_size^2', kwargs['initializer'])\n                create_tensor([1, 4*state_size, state_size], name+'_R_shape', kwargs['initializer'])\n                create_tensor([1, 8*state_size], name+'_B_shape', kwargs['initializer'])\n\n                nodes += [\n                    # get W\n                    make_node('Mul', [name+'_4*state_size', name+'_input_size'], [name+'_mul0']),\n                    make_node('Slice', [param, name+'_0', name+'_mul0'], [name+'_W_1d']),\n                    make_node('Split', [name+'_W_1d'], [name+'_W0', name+'_W1', name+'_W2', name+'_W3']),\n                    make_node('Concat', [name+'_W0', name+'_W3', name+'_W1', name+'_W2'], [name+'_W_'], axis=0),\n                    make_node('Concat', [name+'_1', name+'_4*state_size', name+'_input_size'],\n                              [name+'_W_shape'], axis=0),\n                    make_node('Reshape', [name+'_W_', name+'_W_shape'], [name+'_W']),\n                    # get R\n                    make_node('Add', [name+'_mul0', name+'_4*state_size^2'], [name+'_add0']),\n                    make_node('Slice', [param, name+'_mul0', name+'_add0'], [name+'_R_1d']),\n                    make_node('Split', [name+'_R_1d'], [name+'_R0', name+'_R1', name+'_R2', name+'_R3']),\n                    make_node('Concat', [name+'_R0', name+'_R3', name+'_R1', name+'_R2'], [name+'_R_'], axis=0),\n                    make_node('Reshape', [name+'_R_', name+'_R_shape'], [name+'_R']),\n                    # get B\n                    make_node('Add', [name+'_add0', name+'_8*state_size'], [name+'_add1']),\n                    make_node('Slice', [param, name+'_add0', name+'_add1'], [name+'_B_1d']),\n                    make_node('Split', [name+'_B_1d'], [name+'_B0', name+'_B1', name+'_B2', name+'_B3',\n                                                        name+'_B4', name+'_B5', name+'_B6', name+'_B7']),\n                    make_node('Concat', [name+'_B0', name+'_B3', name+'_B1', name+'_B2',\n                                         name+'_B4', name+'_B7', name+'_B5', name+'_B6'], [name+'_B_'], axis=0),\n                    make_node('Reshape', [name+'_B_', name+'_B_shape'], [name+'_B']),\n                    # get seq_len\n                    make_node('Tile', [name+'_seq_length', name+'_batch_size'], [name+'_seq_len_']),\n                    make_node(\"Cast\", [name+'_seq_len_'], [name+\"_seq_len\"], to=int(TensorProto.INT32)),\n                    # compute LSTM\n                    make_node('LSTM', [data, name+'_W', name+'_R', name+'_B',\n                                       name+'_seq_len', name+'initial_h', name+'initial_c'],\n                              [name+'0_', name+'1', name+'2'], hidden_size=state_size),\n                    make_node('Squeeze', [name+'0_', name+'_1'], [name]),\n                ]\n            else:\n                create_tensor([-1], name+'_-1', kwargs['initializer'])\n                create_tensor([4*state_size], name+'_4*state_size', kwargs['initializer'])\n                create_tensor([8*state_size], name+'_8*state_size', kwargs['initializer'])\n                create_tensor([4*state_size*state_size], name+'_4*state_size^2', kwargs['initializer'])\n                create_tensor([1, 4*state_size, state_size], name+'_R_shape', kwargs['initializer'])\n                create_tensor([1, 8*state_size], name+'_B_shape', kwargs['initializer'])\n\n                nodes += [\n                    # get W_fwd\n                    make_node('Mul', [name+'_4*state_size', name+'_input_size'], [name+'_mul0']),\n                    make_node('Slice', [param, name+'_0', name+'_mul0'], [name+'_W_1d']),\n                    make_node('Split', [name+'_W_1d'], [name+'_W0', name+'_W1', name+'_W2', name+'_W3']),\n                    make_node('Concat', [name+'_W0', name+'_W3', name+'_W1', name+'_W2'],\n                              [name+'_W_'], axis=0),\n                    make_node('Concat', [name+'_1', name+'_4*state_size', name+'_input_size'],\n                              [name+'_W_shape'], axis=0),\n                    make_node('Reshape', [name+'_W_', name+'_W_shape'], [name+'_W_fwd']),\n                    # get R_fwd\n                    make_node('Add', [name+'_mul0', name+'_4*state_size^2'], [name+'_add0']),\n                    make_node('Slice', [param, name+'_mul0', name+'_add0'], [name+'_R_1d']),\n                    make_node('Split', [name+'_R_1d'], [name+'_R0', name+'_R1', name+'_R2', name+'_R3']),\n                    make_node('Concat', [name+'_R0', name+'_R3', name+'_R1', name+'_R2'], [name+'_R_'], axis=0),\n                    make_node('Reshape', [name+'_R_', name+'_R_shape'], [name+'_R_fwd']),\n                    # get W_bwd\n                    make_node('Add', [name+'_add0', name+'_mul0'], [name+'_add1']),\n                    make_node('Slice', [param, name+'_add0', name+'_add1'], [name+'_W_1d_bwd']),\n                    make_node('Split', [name+'_W_1d_bwd'],\n                              [name+'_W0_bwd', name+'_W1_bwd', name+'_W2_bwd', name+'_W3_bwd']),\n                    make_node('Concat', [name+'_W0_bwd', name+'_W3_bwd', name+'_W1_bwd', name+'_W2_bwd'],\n                              [name+'_W_bwd_'], axis=0),\n                    make_node('Reshape', [name+'_W_bwd_', name+'_W_shape'], [name+'_W_bwd']),\n                    # get R_bwd\n                    make_node('Add', [name+'_add1', name+'_4*state_size^2'], [name+'_add2']),\n                    make_node('Slice', [param, name+'_add1', name+'_add2'], [name+'_R_1d_bwd']),\n                    make_node('Split', [name+'_R_1d_bwd'],\n                              [name+'_R0_bwd', name+'_R1_bwd', name+'_R2_bwd', name+'_R3_bwd']),\n                    make_node('Concat', [name+'_R0_bwd', name+'_R3_bwd', name+'_R1_bwd', name+'_R2_bwd'],\n                              [name+'_R_bwd_'], axis=0),\n                    make_node('Reshape', [name+'_R_bwd_', name+'_R_shape'], [name+'_R_bwd']),\n                    # get B_fwd\n                    make_node('Add', [name+'_add2', name+'_8*state_size'], [name+'_add3']),\n                    make_node('Slice', [param, name+'_add2', name+'_add3'], [name+'_B_1d']),\n                    make_node('Split', [name+'_B_1d'], [name+'_B0', name+'_B1', name+'_B2', name+'_B3',\n                                                        name+'_B4', name+'_B5', name+'_B6', name+'_B7']),\n                    make_node('Concat', [name+'_B0', name+'_B3', name+'_B1', name+'_B2',\n                                         name+'_B4', name+'_B7', name+'_B5', name+'_B6'], [name+'_B_'], axis=0),\n                    make_node('Reshape', [name+'_B_', name+'_B_shape'], [name+'_B_fwd']),\n                    # get B_bwd\n                    make_node('Add', [name+'_add3', name+'_8*state_size'], [name+'_add4']),\n                    make_node('Slice', [param, name+'_add3', name+'_add4'], [name+'_B_1d_bwd']),\n                    make_node('Split', [name+'_B_1d_bwd'],\n                              [name+'_B0_bwd', name+'_B1_bwd', name+'_B2_bwd', name+'_B3_bwd',\n                               name+'_B4_bwd', name+'_B5_bwd', name+'_B6_bwd', name+'_B7_bwd']),\n                    make_node('Concat', [name+'_B0_bwd', name+'_B3_bwd', name+'_B1_bwd', name+'_B2_bwd',\n                                         name+'_B4_bwd', name+'_B7_bwd', name+'_B5_bwd', name+'_B6_bwd'],\n                              [name+'_B_bwd_'], axis=0),\n                    make_node('Reshape', [name+'_B_bwd_', name+'_B_shape'], [name+'_B_bwd']),\n                    # get seq_len\n                    make_node('Tile', [name+'_seq_length', name+'_batch_size'], [name+'_seq_len_']),\n                    make_node(\"Cast\", [name+'_seq_len_'], [name+\"_seq_len\"], to=int(TensorProto.INT32)),\n                    # compute LSTM\n                    make_node('Concat', [name+'_W_fwd', name+'_W_bwd'], [name+'_W'], axis=0),\n                    make_node('Concat', [name+'_R_fwd', name+'_R_bwd'], [name+'_R'], axis=0),\n                    make_node('Concat', [name+'_B_fwd', name+'_B_bwd'], [name+'_B'], axis=0),\n                    make_node('LSTM', [data, name+'_W', name+'_R', name+'_B',\n                                       name+'_seq_len', name+'initial_h', name+'initial_c'],\n                              [name+'0_', name+'1', name+'2'], hidden_size=state_size, direction='bidirectional'),\n                    make_node('Transpose', [name+'0_'], [name+'0_t'], perm=[0, 2, 1, 3]),\n                    make_node('Concat', [name+'_seq_length', name+'_batch_size', name+'_-1'],\n                              [name+'_shape_out'], axis=0),\n                    make_node('Reshape', [name+'0_t', name+'_shape_out'], [name]),\n                ]\n        else:\n            raise NotImplementedError('Currently RNN onnx export only supports num_layers equals to 1 or 2')\n\n    elif mode == 'gru':\n        if num_layers == 2:\n            create_tensor([6*state_size], name+'_6*state_size', kwargs['initializer'])\n            create_tensor([3*state_size*state_size], name+'_3*state_size^2', kwargs['initializer'])\n            create_tensor([1, 3*state_size, state_size], name+'_WR_shape', kwargs['initializer'])\n            create_tensor([1, 6*state_size], name+'_B_shape', kwargs['initializer'])\n            create_tensor([4*3*state_size*state_size], name+'_WR_offset', kwargs['initializer'])\n\n            nodes += [\n                # Layer 0\n                # get W\n                make_node('Slice', [param, name+'_0', name+'_3*state_size^2'], [name+'_W0_1d']),\n                make_node('Split', [name+'_W0_1d'], [name+'_W00', name+'_W01', name+'_W02']),\n                make_node('Concat', [name+'_W01', name+'_W00', name+'_W02'], [name+'_W0_'], axis=0),\n                make_node('Reshape', [name+'_W0_', name+'_WR_shape'], [name+'_W0']),\n                # get R\n                make_node('Add', [name+'_3*state_size^2', name+'_3*state_size^2'], [name+'_R0_offset']),\n                make_node('Slice', [param, name+'_3*state_size^2', name+'_R0_offset'], [name+'_R0_1d']),\n                make_node('Split', [name+'_R0_1d'], [name+'_R00', name+'_R01', name+'_R02']),\n                make_node('Concat', [name+'_R01', name+'_R00', name+'_R02'], [name+'_R0_'], axis=0),\n                make_node('Reshape', [name+'_R0_', name+'_WR_shape'], [name+'_R0']),\n                # get B\n                make_node('Add', [name+'_WR_offset', name+'_6*state_size'], [name+'_B0_offset']),\n                make_node('Slice', [param, name+'_WR_offset', name+'_B0_offset'], [name+'_B0_1d']),\n                make_node('Split', [name+'_B0_1d'], [name+'_B00', name+'_B01', name+'_B02',\n                                                     name+'_B03', name+'_B04', name+'_B05']),\n                make_node('Concat', [name+'_B01', name+'_B00', name+'_B02',\n                                     name+'_B04', name+'_B03', name+'_B05'], [name+'_B0_'], axis=0),\n                make_node('Reshape', [name+'_B0_', name+'_B_shape'], [name+'_B0']),\n                # get initial states\n                make_node('Split', [name+'initial_h'], [name+'_initial_h0', name+'_initial_h1'], axis=0),\n                # get seq_len\n                make_node('Tile', [name+'_seq_length', name+'_batch_size'], [name+'_seq_len_']),\n                make_node(\"Cast\", [name+'_seq_len_'], [name+\"_seq_len\"], to=int(TensorProto.INT32)),\n                # Layer 0 GRU\n                make_node('GRU', [data, name+'_W0', name+'_R0', name+'_B0', name+'_seq_len',\n                                  name+'_initial_h0'],\n                          [name+'_gru0_out_', name+'_gru0_h'], hidden_size=state_size, linear_before_reset=1),\n                make_node('Squeeze', [name+'_gru0_out_', name+'_1'], [name+'_gru0_out']),\n\n                # Layer 1\n                # get W\n                make_node('Add', [name+'_R0_offset', name+'_3*state_size^2'], [name+'_W1_offset']),\n                make_node('Slice', [param, name+'_R0_offset', name+'_W1_offset'], [name+'_W1_1d']),\n                make_node('Split', [name+'_W1_1d'], [name+'_W10', name+'_W11', name+'_W12']),\n                make_node('Concat', [name+'_W11', name+'_W10', name+'_W12'], [name+'_W1_'], axis=0),\n                make_node('Reshape', [name+'_W1_', name+'_WR_shape'], [name+'_W1']),\n                # get R\n                make_node('Slice', [param, name+'_W1_offset', name+'_WR_offset'], [name+'_R1_1d']),\n                make_node('Split', [name+'_R1_1d'], [name+'_R10', name+'_R11', name+'_R12']),\n                make_node('Concat', [name+'_R11', name+'_R10', name+'_R12'], [name+'_R1_'], axis=0),\n                make_node('Reshape', [name+'_R1_', name+'_WR_shape'], [name+'_R1']),\n                # get B\n                make_node('Add', [name+'_B0_offset', name+'_6*state_size'], [name+'_B1_offset']),\n                make_node('Slice', [param, name+'_B0_offset', name+'_B1_offset'], [name+'_B1_1d']),\n                make_node('Split', [name+'_B1_1d'], [name+'_B10', name+'_B11', name+'_B12',\n                                                     name+'_B13', name+'_B14', name+'_B15']),\n                make_node('Concat', [name+'_B11', name+'_B10', name+'_B12',\n                                     name+'_B14', name+'_B13', name+'_B15'], [name+'_B1_'], axis=0),\n                make_node('Reshape', [name+'_B1_', name+'_B_shape'], [name+'_B1']),\n                # Layer 1 GRU\n                make_node('GRU', [name+'_gru0_out', name+'_W1', name+'_R1', name+'_B1', name+'_seq_len',\n                                  name+'_initial_h1'],\n                          [name+'_gru1_out_', name+'_gru1_h'], hidden_size=state_size, linear_before_reset=1),\n                make_node('Squeeze', [name+'_gru1_out_', name+'_1'], [name]),\n                make_node('Concat', [name+'_gru0_h', name+'_gru1_h'], [name+'1'], axis=0)\n            ]\n\n        elif num_layers == 1:\n            create_tensor([3*state_size], name+'_3*state_size', kwargs['initializer'])\n            create_tensor([6*state_size], name+'_6*state_size', kwargs['initializer'])\n            create_tensor([3*state_size*state_size], name+'_3*state_size^2', kwargs['initializer'])\n            create_tensor([1, 3*state_size, state_size], name+'_R_shape', kwargs['initializer'])\n            create_tensor([1, 6*state_size], name+'_B_shape', kwargs['initializer'])\n\n            nodes += [\n                # get W\n                make_node('Mul', [name+'_3*state_size', name+'_input_size'], [name+'_mul0']),\n                make_node('Slice', [param, name+'_0', name+'_mul0'], [name+'_W_1d']),\n                make_node('Split', [name+'_W_1d'], [name+'_W0', name+'_W1', name+'_W2']),\n                make_node('Concat', [name+'_W1', name+'_W0', name+'_W2'], [name+'_W_'], axis=0),\n                make_node('Concat', [name+'_1', name+'_3*state_size', name+'_input_size'], [name+'_W_shape'], axis=0),\n                make_node('Reshape', [name+'_W_', name+'_W_shape'], [name+'_W']),\n                # get R\n                make_node('Add', [name+'_mul0', name+'_3*state_size^2'], [name+'_add0']),\n                make_node('Slice', [param, name+'_mul0', name+'_add0'], [name+'_R_1d']),\n                make_node('Split', [name+'_R_1d'], [name+'_R0', name+'_R1', name+'_R2']),\n                make_node('Concat', [name+'_R1', name+'_R0', name+'_R2'], [name+'_R_'], axis=0),\n                make_node('Reshape', [name+'_R_', name+'_R_shape'], [name+'_R']),\n                # get B\n                make_node('Add', [name+'_add0', name+'_6*state_size'], [name+'_add1']),\n                make_node('Slice', [param, name+'_add0', name+'_add1'], [name+'_B_1d']),\n                make_node('Split', [name+'_B_1d'], [name+'_B0', name+'_B1', name+'_B2',\n                                                    name+'_B3', name+'_B4', name+'_B5']),\n                make_node('Concat', [name+'_B1', name+'_B0', name+'_B2',\n                                     name+'_B4', name+'_B3', name+'_B5'], [name+'_B_'], axis=0),\n                make_node('Reshape', [name+'_B_', name+'_B_shape'], [name+'_B']),\n                # get seq_len\n                make_node('Tile', [name+'_seq_length', name+'_batch_size'], [name+'_seq_len_']),\n                make_node(\"Cast\", [name+'_seq_len_'], [name+\"_seq_len\"], to=int(TensorProto.INT32)),\n                # compute GRU\n                make_node('GRU', [data, name+'_W', name+'_R', name+'_B', name+'_seq_len', name+'initial_h'],\n                          [name+'0_', name+'1'], hidden_size=state_size, linear_before_reset=1),\n                make_node('Squeeze', [name+'0_', name+'_1'], [name]),\n            ]\n        else:\n            raise NotImplementedError('Currently RNN onnx export only supports num_layers equals to 1 or 2')\n\n    elif mode in ['rnn_tanh', 'rnn_relu']:\n        activations = ['Tanh']\n        if mode == 'rnn_relu':\n            activations = ['Relu']\n        if num_layers == 2:\n            create_tensor([2*state_size], name+'_2*state_size', kwargs['initializer'])\n            create_tensor([state_size*state_size], name+'_state_size^2', kwargs['initializer'])\n            create_tensor([1, state_size, state_size], name+'_WR_shape', kwargs['initializer'])\n            create_tensor([1, 2*state_size], name+'_B_shape', kwargs['initializer'])\n            create_tensor([4*state_size*state_size], name+'_WR_offset', kwargs['initializer'])\n\n            nodes += [\n                # Layer 0\n                # get W\n                make_node('Slice', [param, name+'_0', name+'_state_size^2'], [name+'_W0_1d']),\n                make_node('Reshape', [name+'_W0_1d', name+'_WR_shape'], [name+'_W0']),\n                # get R\n                make_node('Add', [name+'_state_size^2', name+'_state_size^2'], [name+'_R0_offset']),\n                make_node('Slice', [param, name+'_state_size^2', name+'_R0_offset'], [name+'_R0_1d']),\n                make_node('Reshape', [name+'_R0_1d', name+'_WR_shape'], [name+'_R0']),\n                # get B\n                make_node('Add', [name+'_WR_offset', name+'_2*state_size'], [name+'_B0_offset']),\n                make_node('Slice', [param, name+'_WR_offset', name+'_B0_offset'], [name+'_B0_1d']),\n                make_node('Reshape', [name+'_B0_1d', name+'_B_shape'], [name+'_B0']),\n                # get initial states\n                make_node('Split', [name+'initial_h'], [name+'_initial_h0', name+'_initial_h1'], axis=0),\n                # get seq_len\n                make_node('Tile', [name+'_seq_length', name+'_batch_size'], [name+'_seq_len_']),\n                make_node(\"Cast\", [name+'_seq_len_'], [name+\"_seq_len\"], to=int(TensorProto.INT32)),\n                # Layer 0 RNN\n                make_node('RNN', [data, name+'_W0', name+'_R0', name+'_B0', name+'_seq_len',\n                                  name+'_initial_h0'], [name+'_rnn0_out_', name+'_rnn0_h'],\n                          hidden_size=state_size, activations=activations),\n                make_node('Squeeze', [name+'_rnn0_out_', name+'_1'], [name+'_rnn0_out']),\n\n                # Layer 1\n                # get W\n                make_node('Add', [name+'_R0_offset', name+'_state_size^2'], [name+'_W1_offset']),\n                make_node('Slice', [param, name+'_R0_offset', name+'_W1_offset'], [name+'_W1_1d']),\n                make_node('Reshape', [name+'_W1_1d', name+'_WR_shape'], [name+'_W1']),\n                # get R\n                make_node('Slice', [param, name+'_W1_offset', name+'_WR_offset'], [name+'_R1_1d']),\n                make_node('Reshape', [name+'_R1_1d', name+'_WR_shape'], [name+'_R1']),\n                # get B\n                make_node('Add', [name+'_B0_offset', name+'_2*state_size'], [name+'_B1_offset']),\n                make_node('Slice', [param, name+'_B0_offset', name+'_B1_offset'], [name+'_B1_1d']),\n                make_node('Reshape', [name+'_B1_1d', name+'_B_shape'], [name+'_B1']),\n                # Layer 1 RNN\n                make_node('RNN', [name+'_rnn0_out', name+'_W1', name+'_R1', name+'_B1', name+'_seq_len',\n                                  name+'_initial_h1'], [name+'_rnn1_out_', name+'_rnn1_h'],\n                          hidden_size=state_size, activations=activations),\n                make_node('Squeeze', [name+'_rnn1_out_', name+'_1'], [name]),\n                make_node('Concat', [name+'_rnn0_h', name+'_rnn1_h'], [name+'1'], axis=0)\n            ]\n\n        elif num_layers == 1:\n            create_tensor([2*state_size], name+'_2*state_size', kwargs['initializer'])\n            create_tensor([state_size*state_size], name+'_state_size^2', kwargs['initializer'])\n            create_tensor([1, state_size, state_size], name+'_R_shape', kwargs['initializer'])\n            create_tensor([1, 2*state_size], name+'_B_shape', kwargs['initializer'])\n\n            nodes += [\n                # get W\n                make_node('Mul', [name+'_state_size', name+'_input_size'], [name+'_mul0']),\n                make_node('Slice', [param, name+'_0', name+'_mul0'], [name+'_W_1d']),\n                make_node('Concat', [name+'_1', name+'_state_size', name+'_input_size'], [name+'_W_shape'], axis=0),\n                make_node('Reshape', [name+'_W_1d', name+'_W_shape'], [name+'_W']),\n                # get R\n                make_node('Add', [name+'_mul0', name+'_state_size^2'], [name+'_add0']),\n                make_node('Slice', [param, name+'_mul0', name+'_add0'], [name+'_R_1d']),\n                make_node('Reshape', [name+'_R_1d', name+'_R_shape'], [name+'_R']),\n                # get B\n                make_node('Add', [name+'_add0', name+'_2*state_size'], [name+'_add1']),\n                make_node('Slice', [param, name+'_add0', name+'_add1'], [name+'_B_1d']),\n                make_node('Reshape', [name+'_B_1d', name+'_B_shape'], [name+'_B']),\n                # get seq_len\n                make_node('Tile', [name+'_seq_length', name+'_batch_size'], [name+'_seq_len_']),\n                make_node(\"Cast\", [name+'_seq_len_'], [name+\"_seq_len\"], to=int(TensorProto.INT32)),\n                # compute RNN\n                make_node('RNN', [data, name+'_W', name+'_R', name+'_B', name+'_seq_len', name+'initial_h'],\n                          [name+'0_', name+'1'], hidden_size=state_size, activations=activations),\n                make_node('Squeeze', [name+'0_', name+'_1'], [name]),\n            ]\n        else:\n            raise NotImplementedError('Currently RNN onnx export only supports num_layers equals to 1 or 2')\n    else:\n        raise NotImplementedError(f\"Currently RNN onnx export does not support {mode} mode\")\n    return nodes\n\n\n@mx_op.register('SliceChannel', OPSET_VERSION)\ndef convert_slice_channel(node, **kwargs):\n    \"\"\"Map MXNet's SliceChannel operator attributes to onnx's Squeeze or Split\n    operator based on squeeze_axis attribute\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    num_outputs = int(attrs.get('num_outputs'))\n    axis = int(attrs.get('axis', 1))\n    squeeze_axis = attrs.get('squeeze_axis', 'False')\n\n    create_tensor([axis], name+'_axis', kwargs['initializer'])\n\n    nodes = []\n    if squeeze_axis in ['True', '1']:\n        nodes += [\n            make_node('Split', [input_nodes[0]], [name+str(i)+'_' for i in range(num_outputs)],\n                      axis=axis)\n        ]\n        for i in range(num_outputs):\n            nodes += [\n                make_node('Squeeze', [name+str(i)+'_', name+'_axis'], [name+str(i)])\n            ]\n    else:\n        nodes += [\n            make_node('Split', [input_nodes[0]], [name+str(i) for i in range(num_outputs)],\n                      axis=axis)\n        ]\n\n    return nodes\n\n\n@mx_op.register(\"max\", OPSET_VERSION)\ndef convert_max(node, **kwargs):\n    \"\"\"Map MXNet's max operator attributes to onnx's ReduceMax operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = str(attrs.get(\"axis\", 'None'))\n    axes = convert_string_to_list(mx_axis) if mx_axis != 'None' else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n\n    if axes is not None:\n        if keepdims:\n            node = make_node('ReduceMax', input_nodes, [name], axes=axes, keepdims=keepdims)\n            return [node]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            create_tensor([0], name+'_0', kwargs['initializer'])\n            create_tensor([len(axes)], name+'_axes_dim', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceMax', input_nodes, [name+'_rmax'], axes=axes, keepdims=keepdims),\n                make_node('Shape', [name+'_rmax'], [name+'_rmax_shape']),\n                make_node('Shape', [name+'_rmax_shape'], [name+'_rmax_dim']),\n                make_node('Shape', [input_nodes[0]], [name+'_in_shape']),\n                make_node('Shape', [name+'_in_shape'], [name+'_in_dim']),\n                make_node('Equal', [name+'_axes_dim', name+'_in_dim'], [name+'_equal']),\n                make_node('Where', [name+'_equal', name+'_1', name+'_rmax_dim'], [name+'_where0']),\n                make_node('Tile', [name+'_0', name+'_where0'], [name+'_tile']),\n                make_node('Unsqueeze', [name+'_0', name+'_0'], [name+'_unsqueeze']),\n                make_node('Where', [name+'_equal', name+'_1', name+'_0'], [name+'_where1']),\n                make_node('ScatterND', [name+'_tile', name+'_unsqueeze', name+'_where1'], [name+'_SND']),\n                make_node('Reshape', [name+'_rmax', name+'_SND'], [name]),\n            ]\n            return nodes\n    else:\n        if keepdims:\n            node = make_node('ReduceMax', input_nodes, [name], keepdims=keepdims)\n            return [node]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceMax', input_nodes, [name+'_rmax'], keepdims=keepdims),\n                make_node('Reshape', [name+'_rmax', name+'_1'], [name])\n            ]\n            return nodes\n\n\n@mx_op.register(\"min\", OPSET_VERSION)\ndef convert_min(node, **kwargs):\n    \"\"\"Map MXNet's min operator attributes to onnx's ReduceMin operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = str(attrs.get(\"axis\", 'None'))\n    axes = convert_string_to_list(mx_axis) if mx_axis != 'None' else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n\n    if axes is not None:\n        if keepdims:\n            node = make_node('ReduceMin', input_nodes, [name], axes=axes, keepdims=keepdims)\n            return [node]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            create_tensor([0], name+'_0', kwargs['initializer'])\n            create_tensor([len(axes)], name+'_axes_dim', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceMin', input_nodes, [name+'_rmin'], axes=axes, keepdims=keepdims),\n                make_node('Shape', [name+'_rmin'], [name+'_rmin_shape']),\n                make_node('Shape', [name+'_rmin_shape'], [name+'_rmin_dim']),\n                make_node('Shape', [input_nodes[0]], [name+'_in_shape']),\n                make_node('Shape', [name+'_in_shape'], [name+'_in_dim']),\n                make_node('Equal', [name+'_axes_dim', name+'_in_dim'], [name+'_equal']),\n                make_node('Where', [name+'_equal', name+'_1', name+'_rmin_dim'], [name+'_where0']),\n                make_node('Tile', [name+'_0', name+'_where0'], [name+'_tile']),\n                make_node('Unsqueeze', [name+'_0', name+'_0'], [name+'_unsqueeze']),\n                make_node('Where', [name+'_equal', name+'_1', name+'_0'], [name+'_where1']),\n                make_node('ScatterND', [name+'_tile', name+'_unsqueeze', name+'_where1'], [name+'_SND']),\n                make_node('Reshape', [name+'_rmin', name+'_SND'], [name]),\n            ]\n            return nodes\n    else:\n        if keepdims:\n            node = make_node('ReduceMin', input_nodes, [name], keepdims=keepdims)\n            return [node]\n\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceMin', input_nodes, [name+'_rmin'], keepdims=keepdims),\n                make_node('Reshape', [name+'_rmin', name+'_1'], [name])\n            ]\n            return nodes\n\n\n@mx_op.register(\"mean\", OPSET_VERSION)\ndef convert_mean(node, **kwargs):\n    \"\"\"Map MXNet's mean operator attributes to onnx's ReduceMean operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = str(attrs.get(\"axis\", 'None'))\n    axes = convert_string_to_list(mx_axis) if mx_axis != 'None' else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n\n    if axes is not None:\n        if keepdims:\n            node = make_node('ReduceMean', input_nodes, [name], axes=axes, keepdims=keepdims)\n            return [node]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            create_tensor([0], name+'_0', kwargs['initializer'])\n            create_tensor([len(axes)], name+'_axes_dim', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceMean', input_nodes, [name+'_reduce'], axes=axes, keepdims=keepdims),\n                make_node('Shape', [name+'_reduce'], [name+'_reduce_shape']),\n                make_node('Shape', [name+'_reduce_shape'], [name+'_reduce_dim']),\n                make_node('Shape', [input_nodes[0]], [name+'_in_shape']),\n                make_node('Shape', [name+'_in_shape'], [name+'_in_dim']),\n                make_node('Equal', [name+'_axes_dim', name+'_in_dim'], [name+'_equal']),\n                make_node('Where', [name+'_equal', name+'_1', name+'_reduce_dim'], [name+'_where0']),\n                make_node('Tile', [name+'_0', name+'_where0'], [name+'_tile']),\n                make_node('Unsqueeze', [name+'_0', name+'_0'], [name+'_unsqueeze']),\n                make_node('Where', [name+'_equal', name+'_1', name+'_0'], [name+'_where1']),\n                make_node('ScatterND', [name+'_tile', name+'_unsqueeze', name+'_where1'], [name+'_SND']),\n                make_node('Reshape', [name+'_reduce', name+'_SND'], [name]),\n            ]\n            return nodes\n    else:\n        if keepdims:\n            node = make_node('ReduceMean', input_nodes, [name], keepdims=keepdims)\n            return [node]\n\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceMean', input_nodes, [name+'_reduce'], keepdims=keepdims),\n                make_node('Reshape', [name+'_reduce', name+'_1'], [name])\n            ]\n            return nodes\n\n\n@mx_op.register(\"prod\", OPSET_VERSION)\ndef convert_prod(node, **kwargs):\n    \"\"\"Map MXNet's prod operator attributes to onnx's ReduceProd operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = str(attrs.get(\"axis\", 'None'))\n    axes = convert_string_to_list(mx_axis) if mx_axis != 'None' else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n\n    if axes is not None:\n        if keepdims:\n            node = make_node('ReduceProd', input_nodes, [name], axes=axes, keepdims=keepdims)\n            return [node]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            create_tensor([0], name+'_0', kwargs['initializer'])\n            create_tensor([len(axes)], name+'_axes_dim', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceProd', input_nodes, [name+'_reduce'], axes=axes, keepdims=keepdims),\n                make_node('Shape', [name+'_reduce'], [name+'_reduce_shape']),\n                make_node('Shape', [name+'_reduce_shape'], [name+'_reduce_dim']),\n                make_node('Shape', [input_nodes[0]], [name+'_in_shape']),\n                make_node('Shape', [name+'_in_shape'], [name+'_in_dim']),\n                make_node('Equal', [name+'_axes_dim', name+'_in_dim'], [name+'_equal']),\n                make_node('Where', [name+'_equal', name+'_1', name+'_reduce_dim'], [name+'_where0']),\n                make_node('Tile', [name+'_0', name+'_where0'], [name+'_tile']),\n                make_node('Unsqueeze', [name+'_0', name+'_0'], [name+'_unsqueeze']),\n                make_node('Where', [name+'_equal', name+'_1', name+'_0'], [name+'_where1']),\n                make_node('ScatterND', [name+'_tile', name+'_unsqueeze', name+'_where1'], [name+'_SND']),\n                make_node('Reshape', [name+'_reduce', name+'_SND'], [name]),\n            ]\n            return nodes\n    else:\n        if keepdims:\n            node = make_node('ReduceProd', input_nodes, [name], keepdims=keepdims)\n            return [node]\n\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceProd', input_nodes, [name+'_reduce'], keepdims=keepdims),\n                make_node('Reshape', [name+'_reduce', name+'_1'], [name])\n            ]\n            return nodes\n\n\n@mx_op.register(\"squeeze\", OPSET_VERSION)\n@mx_op.register(\"_npi_squeeze\", OPSET_VERSION)\ndef convert_squeeze(node, **kwargs):\n    \"\"\"Map MXNet's squeeze operator attributes to onnx's squeeze operator\n    and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = str(attrs.get(\"axis\", 'None'))\n    axes = convert_string_to_list(mx_axis) if mx_axis != 'None' else None\n\n    if not axes:\n        node = onnx.helper.make_node(\n            \"Squeeze\",\n            input_nodes,\n            [name],\n            name=name\n        )\n    else:\n        create_tensor(axes, name+'_axes', kwargs['initializer'])\n        node = onnx.helper.make_node(\n            \"Squeeze\",\n            [input_nodes[0], name+'_axes'],\n            [name],\n            name=name,\n        )\n    return [node]\n\n\n@mx_op.register(\"SoftmaxOutput\", OPSET_VERSION)\ndef convert_softmax_output(node, **kwargs):\n    \"\"\"Map MXNet's SoftmaxOutput operator attributes to onnx's Softmax operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, _ = get_inputs(node, kwargs)\n\n    nodes = [\n        make_node('Shape', [input_nodes[0]], [name+'_shape']),\n        make_node('Flatten', [input_nodes[0]], [name+'_flat'], axis=1),\n        make_node('Softmax', [name+'_flat'], [name+'_sm'], axis=1),\n        make_node('Reshape', [name+'_sm', name+'_shape'], [name])\n    ]\n\n    return nodes\n\n\n@mx_op.register(\"norm\", OPSET_VERSION)\ndef convert_norm(node, **kwargs):\n    \"\"\"Map MXNet's norm operator attributes to onnx's ReduceL1 and ReduceL2 operators\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = attrs.get(\"axis\", None)\n    axes = convert_string_to_list(str(mx_axis)) if mx_axis else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n    ord = int(attrs.get(\"ord\", 2))\n\n    onnx_op_name = \"ReduceL1\" if ord == 1 else \"ReduceL2\"\n\n    if axes:\n        if keepdims:\n            reduce_node = make_node(onnx_op_name, input_nodes, [name], axes=axes, keepdims=keepdims)\n            return [reduce_node]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            create_tensor([0], name+'_0', kwargs['initializer'])\n            create_tensor([len(axes)], name+'_axes_dim', kwargs['initializer'])\n            nodes = [\n                make_node(onnx_op_name, input_nodes, [name+'_reduce'], axes=axes, keepdims=keepdims),\n                make_node('Shape', [name+'_reduce'], [name+'_reduce_shape']),\n                make_node('Shape', [name+'_reduce_shape'], [name+'_reduce_dim']),\n                make_node('Shape', [input_nodes[0]], [name+'_in_shape']),\n                make_node('Shape', [name+'_in_shape'], [name+'_in_dim']),\n                make_node('Equal', [name+'_axes_dim', name+'_in_dim'], [name+'_equal']),\n                make_node('Where', [name+'_equal', name+'_1', name+'_reduce_dim'], [name+'_where0']),\n                make_node('Tile', [name+'_0', name+'_where0'], [name+'_tile']),\n                make_node('Unsqueeze', [name+'_0', name+'_0'], [name+'_unsqueeze']),\n                make_node('Where', [name+'_equal', name+'_1', name+'_0'], [name+'_where1']),\n                make_node('ScatterND', [name+'_tile', name+'_unsqueeze', name+'_where1'], [name+'_SND']),\n                make_node('Reshape', [name+'_reduce', name+'_SND'], [name]),\n            ]\n            return nodes\n    else:\n\n        if keepdims:\n            reduce_node = make_node(onnx_op_name, input_nodes, [name], keepdims=keepdims)\n            return [reduce_node]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node(onnx_op_name, input_nodes, [name+'_norm'], keepdims=keepdims),\n                make_node('Reshape', [name+'_norm', name+'_1'], [name])\n            ]\n            return nodes\n\n\n@mx_op.register(\"log_softmax\", OPSET_VERSION)\ndef convert_logsoftmax(node, **kwargs):\n    \"\"\"Map MXNet's log_softmax operator attributes to onnx's LogSoftMax operator\n    and return the created node.\n    \"\"\"\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    # Converting to int\n    axis = int(attrs.get(\"axis\", -1))\n    temp = attrs.get('temperature', 'None')\n    use_length = attrs.get('use_length', 'False')\n\n    if temp != 'None':\n        raise AttributeError('LogSoftMax currently does not support temperature!=None')\n\n    if use_length in ['1', 'True']:\n        raise AttributeError('LogSoftMax currently does not support use_length==True')\n\n    node = onnx.helper.make_node(\n        'LogSoftmax',\n        input_nodes,\n        [name],\n        axis=axis,\n        name=name\n    )\n\n    return [node]\n\n\n@mx_op.register('_split_v2', OPSET_VERSION)\ndef convert_contrib_split_v2(node, **kwargs):\n    \"\"\"Map MXNet's _split_v2 operator\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n    axis = int(attrs.get('axis', 0))\n    squeeze_axis = attrs.get('squeeze_axis', 'False')\n    sections = int(attrs.get('sections', 0))\n    indices = convert_string_to_list(attrs.get('indices', '[]'))\n    if sections <= 0 and len(indices) == 0:\n        raise NotImplementedError('section or indices must be set')\n    if sections > 0:\n        output_nodes = [name+str(i) for i in range(sections)]\n        if squeeze_axis == 'False':\n            nodes = [\n                make_node('Split', input_nodes, output_nodes, axis=axis),\n            ]\n        else:\n            output_nodes_ = [name+str(i)+'_' for i in range(sections)]\n            create_tensor([axis], name+'_axis', kwargs['initializer'])\n            nodes = [\n                make_node('Split', input_nodes, output_nodes_, axis=axis),\n            ]\n            for i in range(sections):\n                nodes += [\n                    make_node(\"Squeeze\", [output_nodes_[i], name+'_axis'], [output_nodes[i]]),\n                ]\n    else:\n        indices.sort()\n        split = []\n        for i in range(1, len(indices)):\n            if indices[i] >= indices[i-1]:\n                split.append(indices[i] - indices[i-1])\n\n        output_nodes = [name+str(i) for i in range(len(split)+1)]\n        create_tensor([0], name+'_0', kwargs['initializer'])\n        create_tensor([axis], name+'_axis', kwargs['initializer'])\n        create_tensor([axis+1], name+'_axis+1', kwargs['initializer'])\n        create_tensor(split, name+'_split_', kwargs['initializer'])\n        create_tensor([sum(split)], name+'_sum', kwargs['initializer'])\n        nodes = [\n            make_node('Shape', input_nodes, [name+'_shape']),\n            make_node('Slice', [name+'_shape', name+'_axis', name+'_axis+1', name+'_0'], [name+'_dim']),\n            make_node('Sub', [name+'_dim', name+'_sum'], [name+'_sub']),\n            make_node('Concat', [name+'_split_', name+'_sub'], [name+'_concat'], axis=0),\n            make_node('Less', [name+'_concat', name+'_0'], [name+'_less']),\n            make_node('Where', [name+'_less', name+'_0', name+'_concat'], [name+'_split']),\n            ]\n        if squeeze_axis == 'False':\n            nodes += [\n                make_node('Split', [input_nodes[0], name+'_split'], output_nodes, axis=axis),\n            ]\n        else:\n            output_nodes_ = [name+str(i)+'_' for i in range(len(split)+1)]\n            nodes += [\n                make_node('Split', [input_nodes[0], name+'_split'], output_nodes_, axis=axis),\n            ]\n            for i, output_node in enumerate(output_nodes):\n                nodes += [\n                    make_node(\"Squeeze\", [output_nodes_[i], name+'_axis'], [output_node]),\n                ]\n\n    return nodes\n\n\n@mx_op.register(\"_npi_mean\", OPSET_VERSION)\ndef convert_npi_mean(node, **kwargs):\n    \"\"\"Map MXNet's mean operator attributes to onnx's ReduceMean operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    dtype = np.dtype('float32')\n    dtype_t = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[dtype]\n\n    mx_axis = str(attrs.get(\"axis\", 'None'))\n    axes = convert_string_to_list(mx_axis) if mx_axis != 'None' else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n\n    if axes is not None:\n        create_tensor(axes, name+'_axes', kwargs['initializer'])\n        if keepdims:\n            nodes = [\n                make_node('Cast', input_nodes, [name+'_cast'], to=dtype_t),\n                make_node('ReduceMean', [name+'_cast'], [name], axes=axes,\n                          keepdims=keepdims),\n            ]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            create_tensor([0], name+'_0', kwargs['initializer'])\n            nodes = [\n                make_node('Cast', input_nodes, [name+'_cast'], to=dtype_t),\n                make_node('ReduceMean', [name+'_cast'], [name+'_reduce'], axes=axes,\n                          keepdims=keepdims),\n                make_node('Shape', [name+'_reduce'], [name+'_reduce_shape']),\n                make_node('Concat', [name+'_1', name+'_reduce_shape'], [name+'_concat'], axis=0),\n                make_node('Reshape', [name+'_reduce', name+'_concat'], [name+'_reshape']),\n                make_node('Squeeze', [name+'_reshape', name+'_0'], [name]),\n            ]\n    else:\n        if keepdims:\n            nodes = [\n                make_node('Cast', input_nodes, [name+'_cast'], to=dtype_t),\n                make_node('ReduceMean', [name+'_cast'], [name], keepdims=keepdims),\n            ]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('Cast', input_nodes, [name+'_cast'], to=dtype_t),\n                make_node('ReduceMean', [name+'_cast'], [name], keepdims=keepdims),\n            ]\n    return nodes, (dtype,)\n\n\n@mx_op.register(\"_npi_prod\", OPSET_VERSION)\ndef convert_npi_prod(node, **kwargs):\n    \"\"\"Map MXNet's prod operator attributes to onnx's ReduceProd operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = str(attrs.get(\"axis\", 'None'))\n    axes = convert_string_to_list(mx_axis) if mx_axis != 'None' else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n\n    if axes is not None:\n        create_tensor(axes, name+'_axes', kwargs['initializer'])\n        if keepdims:\n            nodes = [\n                make_node('ReduceProd', [input_nodes[0]], [name], axes=axes,\n                          keepdims=keepdims),\n            ]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            create_tensor([0], name+'_0', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceProd', [input_nodes[0]], [name+'_reduce'], axes=axes,\n                          keepdims=keepdims),\n                make_node('Shape', [name+'_reduce'], [name+'_reduce_shape']),\n                make_node('Concat', [name+'_1', name+'_reduce_shape'], [name+'_concat'], axis=0),\n                make_node('Reshape', [name+'_reduce', name+'_concat'], [name+'_reshape']),\n                make_node('Squeeze', [name+'_reshape', name+'_0'], [name]),\n            ]\n    else:\n        if keepdims:\n            nodes = [\n                make_node('ReduceProd', [input_nodes[0]], [name], keepdims=keepdims),\n            ]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceProd', [input_nodes[0]], [name], keepdims=keepdims),\n            ]\n    return nodes\n\n\n@mx_op.register(\"_npi_min\", OPSET_VERSION)\ndef convert_npi_min(node, **kwargs):\n    \"\"\"Map MXNet's min operator attributes to onnx's ReduceMin operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = str(attrs.get(\"axis\", 'None'))\n    axes = convert_string_to_list(mx_axis) if mx_axis != 'None' else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n\n    if axes is not None:\n        create_tensor(axes, name+'_axes', kwargs['initializer'])\n        if keepdims:\n            nodes = [\n                make_node('ReduceMin', [input_nodes[0]], [name], axes=axes,\n                          keepdims=keepdims),\n            ]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            create_tensor([0], name+'_0', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceMin', [input_nodes[0]], [name+'_reduce'], axes=axes,\n                          keepdims=keepdims),\n                make_node('Shape', [name+'_reduce'], [name+'_reduce_shape']),\n                make_node('Concat', [name+'_1', name+'_reduce_shape'], [name+'_concat'], axis=0),\n                make_node('Reshape', [name+'_reduce', name+'_concat'], [name+'_reshape']),\n                make_node('Squeeze', [name+'_reshape', name+'_0'], [name]),\n            ]\n    else:\n        if keepdims:\n            nodes = [\n                make_node('ReduceMin', [input_nodes[0]], [name], keepdims=keepdims),\n            ]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceMin', [input_nodes[0]], [name], keepdims=keepdims),\n            ]\n    return nodes\n\n\n@mx_op.register(\"_npi_max\", OPSET_VERSION)\ndef convert_npi_max(node, **kwargs):\n    \"\"\"Map MXNet's min operator attributes to onnx's ReduceMin operator\n    and return the created node.\n    \"\"\"\n    from onnx.helper import make_node\n    name, input_nodes, attrs = get_inputs(node, kwargs)\n\n    mx_axis = str(attrs.get(\"axis\", 'None'))\n    axes = convert_string_to_list(mx_axis) if mx_axis != 'None' else None\n\n    keepdims = get_boolean_attribute_value(attrs, \"keepdims\")\n\n    if axes is not None:\n        create_tensor(axes, name+'_axes', kwargs['initializer'])\n        if keepdims:\n            nodes = [\n                make_node('ReduceMax', [input_nodes[0]], [name], axes=axes,\n                          keepdims=keepdims),\n            ]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            create_tensor([0], name+'_0', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceMax', [input_nodes[0]], [name+'_reduce'], axes=axes,\n                          keepdims=keepdims),\n                make_node('Shape', [name+'_reduce'], [name+'_reduce_shape']),\n                make_node('Concat', [name+'_1', name+'_reduce_shape'], [name+'_concat'], axis=0),\n                make_node('Reshape', [name+'_reduce', name+'_concat'], [name+'_reshape']),\n                make_node('Squeeze', [name+'_reshape', name+'_0'], [name]),\n            ]\n    else:\n        if keepdims:\n            nodes = [\n                make_node('ReduceMax', [input_nodes[0]], [name], keepdims=keepdims),\n            ]\n        else:\n            create_tensor([1], name+'_1', kwargs['initializer'])\n            nodes = [\n                make_node('ReduceMax', [input_nodes[0]], [name], keepdims=keepdims),\n            ]\n    return nodes\n"
  },
  {
    "path": "python/mxnet/onnx/setup.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"\nsetup.py for mx2onnx\n\"\"\"\n\nfrom setuptools import setup, find_packages\n\nsetup(\n    name='mx2onnx',\n    version='0.0.0',\n    description='Module to convert MXNet models to the ONNX format',\n    author='',\n    author_email='',\n    url='https://github.com/apache/mxnet/tree/v1.x/python/mxnet/onnx',\n    install_requires=[\n        'onnx >= 1.7.0',\n    ],\n    classifiers=[\n        'Intended Audience :: Developers',\n        'License :: OSI Approved :: Apache Software License',\n        'Programming Language :: Python :: 3 :: Only',\n    ],\n    packages=find_packages(),\n    python_requires='>=3.6'\n)\n"
  },
  {
    "path": "python/mxnet/operator.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=invalid-name, protected-access, too-many-arguments, no-self-use, too-many-locals, broad-except, too-many-lines, unnecessary-pass\n\"\"\"numpy interface for operators.\"\"\"\n\nimport traceback\nimport warnings\nimport collections\n\nfrom array import array\nfrom threading import Lock\nimport ctypes\nfrom ctypes import CFUNCTYPE, POINTER, Structure, pointer\nfrom ctypes import c_void_p, c_int, c_char, c_char_p, cast, c_bool\n\nfrom .base import _LIB, check_call, MXCallbackList, c_array, c_array_buf, mx_int, OpHandle\nfrom .base import c_str, mx_uint, mx_float, ctypes2numpy_shared, NDArrayHandle, py_str\nfrom . import symbol, context\nfrom .ndarray import NDArray, dtype_np_to_mx, dtype_mx_to_np\nfrom .ndarray.ndarray import _STORAGE_TYPE_STR_TO_ID, _STORAGE_TYPE_ID_TO_STR\nfrom .ndarray.ndarray import _STORAGE_TYPE_UNDEFINED, _STORAGE_TYPE_DEFAULT\nfrom .ndarray.ndarray import _STORAGE_TYPE_CSR, _STORAGE_TYPE_ROW_SPARSE\nfrom .ndarray import _ndarray_cls\nfrom .numpy.multiarray import _np_ndarray_cls\nfrom .util import is_np_array\n\nc_int_p = POINTER(c_int)\n\n\nclass PythonOp(object):\n    \"\"\"Base class for operators implemented in Python.\n\n    Parameters\n    ----------\n    need_top_grad : bool\n        the default need_top_grad() function returns this value.\n    \"\"\"\n    _ref_holder = []\n\n    def __init__(self, need_top_grad=True):\n        self.info_ = None\n        self.need_top_grad_ = need_top_grad\n        warnings.warn('PythonOp has been deprecated. Please use CustomOp')\n\n    def __call__(self, *args, **kwargs):\n        return self.get_symbol(*args, **kwargs)\n\n    def get_symbol(self, *args, **kwargs):\n        \"\"\"Create a symbol from numpy operator.\n        This should only be called once per instance if the operator contains\n        internal states.\n\n        Parameters\n        ----------\n        args : list\n            a list of input arguments (symbols).\n\n        Returns\n        -------\n        sym : mxnet.symbol.Symbol\n        \"\"\"\n        raise NotImplementedError(\"Must override this\")\n\n    def forward(self, in_data, out_data):\n        \"\"\"Forward interface. Override to create new operators.\n\n        Parameters\n        ----------\n        in_data, out_data: list\n            input and output for forward. See document for\n            corresponding arguments of Operator::Forward\n        \"\"\"\n        out_data[0][:] = in_data[0]\n\n    def backward(self, out_grad, in_data, out_data, in_grad):\n        \"\"\"Backward interface. Can override when creating new operators.\n\n        Parameters\n        ----------\n        out_grad, in_data, out_data, in_grad : list\n            input and output for backward. See document for\n            corresponding arguments of Operator::Backward\n        \"\"\"\n        # pylint: disable=W0613\n        in_grad[0][:] = 1.0\n\n    def infer_shape(self, in_shape):\n        \"\"\"Interface for ``infer_shape``. Can override when creating new operators.\n\n        Parameters\n        ----------\n        in_shape : list\n            List of argument shapes in the same order as\n            declared in list_arguments.\n\n        Returns\n        -------\n        in_shape : list\n            List of argument shapes. Can be modified from in_shape.\n        out_shape : list\n            List of output shapes calculated from in_shape,\n            in the same order as declared in list_arguments.\n        \"\"\"\n        return in_shape, [in_shape[0]]\n\n    def list_outputs(self):\n        \"\"\"Interface for ``list_outputs``. Can override when creating new operators.\n\n        Returns\n        -------\n        outputs : list\n            List of output blob names.\n        \"\"\"\n        return ['output']\n\n    def list_arguments(self):\n        \"\"\"Interface for ``list_arguments``. Can override when creating new operators.\n\n        Returns\n        -------\n        in_shape : list\n            list of argument shapes in the same order as\n            declared in list_arguments.\n        \"\"\"\n        return ['data']\n\n    def need_top_grad(self):\n        \"\"\"Whether this operator needs out_grad for backward.\n\n        Returns\n        -------\n        need_top_grad : bool\n            Whether this operator needs out_grad for backward.\n            Should be set to False for loss layers.\n        \"\"\"\n        return self.need_top_grad_\n\n\nclass NumpyOp(PythonOp):\n    \"\"\"Base class for numpy operators. numpy operators allow parts\n    of computation in symbolic graph to be writen in numpy. This feature\n    is intended for quickly hacking out a solution for non performance\n    critical parts. Please consider write a c++ implementation if it becomes\n    a bottleneck.\n    Note that if your operator contains internal states (like arrays),\n    it cannot be used for multi-gpu training.\n    \"\"\"\n    def __init__(self, need_top_grad=True):\n        super(NumpyOp, self).__init__(need_top_grad)\n        warnings.warn('NumpyOp has been deprecated. Please use CustomOp')\n\n    def get_symbol(self, *args, **kwargs):\n        fb_functype = CFUNCTYPE(None, c_int, POINTER(POINTER(mx_float)), POINTER(c_int),\n                                POINTER(POINTER(mx_uint)), POINTER(c_int), c_void_p)\n        infer_functype = CFUNCTYPE(None, c_int, POINTER(c_int),\n                                   POINTER(POINTER(mx_int)), c_void_p)\n        list_functype = CFUNCTYPE(None, POINTER(POINTER(POINTER(c_char))), c_void_p)\n\n        class NumpyOpInfo(Structure):\n            \"\"\"Structure that holds Callback information. Passed to NumpyOpProp\"\"\"\n            _fields_ = [\n                ('forward', fb_functype),\n                ('backward', fb_functype),\n                ('infer_shape', infer_functype),\n                ('list_outputs', list_functype),\n                ('list_arguments', list_functype),\n                ('p_forward', c_void_p),\n                ('p_backward', c_void_p),\n                ('p_infer_shape', c_void_p),\n                ('p_list_outputs', c_void_p),\n                ('p_list_arguments', c_void_p),\n                ]\n\n        def forward_entry(num_tensor, tensor_ptrs, tensor_dims,\n                          tensor_shapes, tensor_tags, _):\n            \"\"\"C Callback for NumpyOp::Forward\"\"\"\n            tensors = [[] for i in range(4)]\n            for i in range(num_tensor):\n                shape = [tensor_shapes[i][j] for j in range(tensor_dims[i])]\n                buff = ctypes2numpy_shared(tensor_ptrs[i], shape)\n                tensors[tensor_tags[i]].append(buff)\n            self.forward(in_data=tensors[0], out_data=tensors[1])\n\n        def backward_entry(num_tensor, tensor_ptrs, tensor_dims,\n                           tensor_shapes, tensor_tags, _):\n            \"\"\"C Callback for NumpyOp::Backward\"\"\"\n            tensors = [[] for i in range(4)]\n            for i in range(num_tensor):\n                shape = [tensor_shapes[i][j] for j in range(tensor_dims[i])]\n                buff = ctypes2numpy_shared(tensor_ptrs[i], shape)\n                tensors[tensor_tags[i]].append(buff)\n            self.backward(in_data=tensors[0], out_data=tensors[1],\n                          in_grad=tensors[2], out_grad=tensors[3])\n\n        def infer_shape_entry(num_tensor, tensor_dims,\n                              tensor_shapes, _):\n            \"\"\"C Callback for NumpyOpProp::InferShape\"\"\"\n            n_in = len(self.list_arguments())\n            n_out = len(self.list_outputs())\n            assert num_tensor == n_in + n_out\n\n            shapes = [[tensor_shapes[i][j] for j in range(tensor_dims[i])] for i in range(n_in)]\n            ishape, oshape = self.infer_shape(shapes)\n            assert len(oshape) == n_out\n            assert len(ishape) == n_in\n            rshape = list(ishape) + list(oshape)\n            for i in range(n_in+n_out):\n                tensor_shapes[i] = cast(c_array_buf(mx_int,\n                                                    array('i', rshape[i])),\n                                        POINTER(mx_int))\n                tensor_dims[i] = len(rshape[i])\n\n        def list_outputs_entry(out, _):\n            \"\"\"C Callback for NumpyOpProp::ListOutputs\"\"\"\n            ret = self.list_outputs()\n            ret = [c_str(i) for i in ret] + [c_char_p(0)]\n            ret = c_array(c_char_p, ret)\n            out[0] = cast(ret, POINTER(POINTER(c_char)))\n\n        def list_arguments_entry(out, _):\n            \"\"\"C Callback for NumpyOpProp::ListArguments\"\"\"\n            ret = self.list_arguments()\n            ret = [c_str(i) for i in ret] + [c_char_p(0)]\n            ret = c_array(c_char_p, ret)\n            out[0] = cast(ret, POINTER(POINTER(c_char)))\n\n        self.info_ = NumpyOpInfo(fb_functype(forward_entry),\n                                 fb_functype(backward_entry),\n                                 infer_functype(infer_shape_entry),\n                                 list_functype(list_outputs_entry),\n                                 list_functype(list_arguments_entry),\n                                 None, None, None, None, None)\n        cb_ptr = format(cast(pointer(self.info_), c_void_p).value, 'x')\n        # pylint: disable=E1101\n        sym = symbol._internal._Native(*args,\n                                       info=cb_ptr,\n                                       need_top_grad=self.need_top_grad(),\n                                       **kwargs)\n        # keep a reference of ourself in PythonOp so we don't get garbage collected.\n        PythonOp._ref_holder.append(self)\n        return sym\n\n\nclass NDArrayOp(PythonOp):\n    \"\"\"Base class for numpy operators. numpy operators allow parts\n    of computation in symbolic graph to be writen in numpy. This feature\n    is intended for quickly hacking out a solution for non performance\n    critical parts. Please consider write a c++ implementation if it becomes\n    a bottleneck.\n    Note that if your operator contains internal states (like arrays),\n    it cannot be used for multi-gpu training.\n    \"\"\"\n    def __init__(self, need_top_grad=True):\n        super(NDArrayOp, self).__init__(need_top_grad)\n        warnings.warn('NDArrayOp has been deprecated. Please use CustomOp')\n\n    def get_symbol(self, *args, **kwargs):\n        fb_functype = CFUNCTYPE(c_bool, c_int, POINTER(c_void_p), POINTER(c_int), c_void_p)\n        infer_functype = CFUNCTYPE(c_bool, c_int, POINTER(c_int),\n                                   POINTER(POINTER(mx_int)), c_void_p)\n        list_functype = CFUNCTYPE(c_bool, POINTER(POINTER(POINTER(c_char))), c_void_p)\n        deps_functype = CFUNCTYPE(c_bool, c_int_p, c_int_p, c_int_p,\n                                  c_int_p, POINTER(c_int_p), c_void_p)\n        class NDArrayOpInfo(Structure):\n            \"\"\"Structure that holds Callback information. Passed to NDArrayOpProp\"\"\"\n            _fields_ = [\n                ('forward', fb_functype),\n                ('backward', fb_functype),\n                ('infer_shape', infer_functype),\n                ('list_outputs', list_functype),\n                ('list_arguments', list_functype),\n                ('declare_backward_dependency', deps_functype),\n                ('p_forward', c_void_p),\n                ('p_backward', c_void_p),\n                ('p_infer_shape', c_void_p),\n                ('p_list_outputs', c_void_p),\n                ('p_list_arguments', c_void_p),\n                ('p_declare_backward_dependency', c_void_p)\n                ]\n        def forward_entry(num_ndarray, ndarraies, tags, _):\n            \"\"\"C Callback for NDArrayOp::Forward\"\"\"\n            try:\n                tensors = [[] for i in range(4)]\n                for i in range(num_ndarray):\n                    if tags[i] == 1:\n                        tensors[tags[i]].append(NDArray(cast(ndarraies[i], NDArrayHandle),\n                                                        writable=True))\n                    else:\n                        tensors[tags[i]].append(NDArray(cast(ndarraies[i], NDArrayHandle),\n                                                        writable=False))\n                self.forward(in_data=tensors[0], out_data=tensors[1])\n            except Exception:\n                print(f'Error in NDArrayOp.forward: {traceback.format_exc()}')\n                return False\n            return True\n\n        def backward_entry(num_ndarray, ndarraies, tags, _):\n            \"\"\"C Callback for NDArrayOp::Backward\"\"\"\n            try:\n                tensors = [[] for i in range(4)]\n                for i in range(num_ndarray):\n                    if tags[i] == 2:\n                        tensors[tags[i]].append(NDArray(cast(ndarraies[i], NDArrayHandle),\n                                                        writable=True))\n                    else:\n                        tensors[tags[i]].append(NDArray(cast(ndarraies[i], NDArrayHandle),\n                                                        writable=False))\n                self.backward(in_data=tensors[0], out_data=tensors[1],\n                              in_grad=tensors[2], out_grad=tensors[3])\n            except Exception:\n                print(f'Error in NDArrayOp.backward: {traceback.format_exc()}')\n                return False\n            return True\n\n        def infer_shape_entry(num_tensor, tensor_dims,\n                              tensor_shapes, _):\n            \"\"\"C Callback for NDArrayOpProp::InferShape\"\"\"\n            try:\n                n_in = len(self.list_arguments())\n                n_out = len(self.list_outputs())\n                assert num_tensor == n_in + n_out\n\n                shapes = [[tensor_shapes[i][j] for j in range(tensor_dims[i])] for i in range(n_in)]\n                ishape, oshape = self.infer_shape(shapes)\n                assert len(oshape) == n_out\n                assert len(ishape) == n_in\n                rshape = list(ishape) + list(oshape)\n                for i in range(n_in+n_out):\n                    tensor_shapes[i] = cast(c_array_buf(mx_int,\n                                                        array('i', rshape[i])),\n                                            POINTER(mx_int))\n                    tensor_dims[i] = len(rshape[i])\n            except Exception:\n                print(f'Error in NDArrayOp.infer_shape: {traceback.format_exc()}')\n                return False\n            return True\n\n        def list_outputs_entry(out, _):\n            \"\"\"C Callback for NDArrayOpProp::ListOutputs\"\"\"\n            try:\n                ret = self.list_outputs()\n                ret = [c_str(i) for i in ret] + [c_char_p(0)]\n                ret = c_array(c_char_p, ret)\n                out[0] = cast(ret, POINTER(POINTER(c_char)))\n            except Exception:\n                print(f'Error in NDArrayOp.list_outputs: {traceback.format_exc()}')\n                return False\n            return True\n\n        def list_arguments_entry(out, _):\n            \"\"\"C Callback for NDArrayOpProp::ListArguments\"\"\"\n            try:\n                ret = self.list_arguments()\n                ret = [c_str(i) for i in ret] + [c_char_p(0)]\n                ret = c_array(c_char_p, ret)\n                out[0] = cast(ret, POINTER(POINTER(c_char)))\n            except Exception:\n                print(f'Error in NDArrayOp.list_arguments: {traceback.format_exc()}')\n                return False\n            return True\n\n        def declare_backward_dependency(out_grad, in_data, out_data, num_dep, deps, _):\n            \"\"\"C Callback for NDArrayOpProp::DeclareBacwardDependency\"\"\"\n            try:\n                out_grad = [out_grad[i] for i in range(len(self.list_outputs()))]\n                in_data = [in_data[i] for i in range(len(self.list_arguments()))]\n                out_data = [out_data[i] for i in range(len(self.list_outputs()))]\n                rdeps = self.declare_backward_dependency(out_grad, in_data, out_data)\n                num_dep[0] = len(rdeps)\n                rdeps = cast(c_array_buf(c_int, array('i', rdeps)), c_int_p)\n                deps[0] = rdeps\n            except Exception:\n                print(f'Error in NDArrayOp.declare_backward_dependency: {traceback.format_exc()}')\n                return False\n            return True\n\n        self.info_ = NDArrayOpInfo(fb_functype(forward_entry),\n                                   fb_functype(backward_entry),\n                                   infer_functype(infer_shape_entry),\n                                   list_functype(list_outputs_entry),\n                                   list_functype(list_arguments_entry),\n                                   deps_functype(declare_backward_dependency),\n                                   None, None, None, None, None, None)\n        cb_ptr = format(cast(pointer(self.info_), c_void_p).value, 'x')\n        # pylint: disable=E1101\n        sym = symbol._internal._NDArray(*args,\n                                        info=cb_ptr,\n                                        **kwargs)\n        # keep a reference of ourself in PythonOp so we don't get garbage collected.\n        PythonOp._ref_holder.append(self)\n        return sym\n\n    def declare_backward_dependency(self, out_grad, in_data, out_data):\n        \"\"\"Declare dependencies of this operator for backward pass.\n\n        Parameters\n        ----------\n        out_grad : list of int\n            ids of out_grad blobs.\n        in_data : list of int\n            ids of in_data blobs.\n        out_data: list of int\n            ids of out_data blobs.\n\n        Returns\n        -------\n        deps : list of int\n            ids of the needed blobs.\n        \"\"\"\n        deps = []\n        if self.need_top_grad():\n            deps.extend(out_grad)\n        deps.extend(in_data)\n        deps.extend(out_data)\n        return deps\n\n\nclass CustomOp(object):\n    \"\"\"Base class for operators implemented in python\"\"\"\n    def __init__(self):\n        pass\n\n    def forward(self, is_train, req, in_data, out_data, aux):\n        \"\"\"Forward interface. Can override when creating new operators.\n\n        Parameters\n        ----------\n        is_train : bool\n            whether this is for training\n        req : list of str\n            how to assign to out_data. can be 'null', 'write', or 'add'.\n            You can optionally use self.assign(dst, req, src) to handle this.\n        in_data, out_data, aux: list of NDArrays\n            input, output, and auxiliary states for forward. See document for\n            corresponding arguments of Operator::Forward\n        \"\"\"\n        # pylint: disable=W0613\n        pass\n\n    def backward(self, req, out_grad, in_data, out_data, in_grad, aux):\n        \"\"\"Backward interface. Can override when creating new operators.\n\n        Parameters\n        ----------\n        req : list of str\n            how to assign to in_grad. can be 'null', 'write', or 'add'.\n            You can optionally use self.assign(dst, req, src) to handle this.\n        out_grad, in_data, out_data, in_grad, aux : list of NDArrays\n            input and output for backward. See document for\n            corresponding arguments of Operator::Backward\n        \"\"\"\n        # pylint: disable=W0613\n        pass\n\n    def assign(self, dst, req, src):\n        \"\"\"Helper function for assigning into dst depending on requirements.\"\"\"\n        if req == 'null':\n            return\n        elif req in ('write', 'inplace'):\n            if is_np_array():\n                dst[()] = src\n            else:\n                dst[:] = src\n        elif req == 'add':\n            if is_np_array():\n                dst[()] += src\n            else:\n                dst[:] += src\n\n\nclass CustomOpProp(object):\n    \"\"\"Base class for operator property class implemented in python.\n\n    Parameters\n    ----------\n    need_top_grad : bool\n        The default declare_backward_dependency function. Use this value\n        to determine whether this operator needs gradient input.\n    \"\"\"\n    def __init__(self, need_top_grad=True):\n        self.need_top_grad_ = need_top_grad\n\n    def infer_shape(self, in_shape):\n        \"\"\"infer_shape interface. Can override when creating new operators.\n\n        Parameters\n        ----------\n        in_shape : list\n            List of argument shapes in the same order as\n            declared in list_arguments.\n\n        Returns\n        -------\n        in_shape : list\n            List of argument shapes. Can be modified from in_shape.\n        out_shape : list\n            List of output shapes calculated from in_shape,\n            in the same order as declared in list_outputs.\n        aux_shape : Optional, list\n            List of aux shapes calculated from in_shape,\n            in the same order as declared in list_auxiliary_states.\n        \"\"\"\n        return in_shape, (in_shape[0],)*len(self.list_outputs()), ()\n\n    def infer_type(self, in_type):\n        \"\"\"infer_type interface. override to create new operators\n\n        Parameters\n        ----------\n        in_type : list of np.dtype\n            list of argument types in the same order as\n            declared in list_arguments.\n\n        Returns\n        -------\n        in_type : list\n            list of argument types. Can be modified from in_type.\n        out_type : list\n            list of output types calculated from in_type,\n            in the same order as declared in list_outputs.\n        aux_type : Optional, list\n            list of aux types calculated from in_type,\n            in the same order as declared in list_auxiliary_states.\n        \"\"\"\n        return in_type, [in_type[0]]*len(self.list_outputs()), \\\n            [in_type[0]]*len(self.list_auxiliary_states())\n\n    def infer_storage_type(self, in_stype):\n        \"\"\"infer_storage_type interface. Used to infer storage type of\n        inputs and outputs in the forward pass. When this interface is not implemented,\n        all stypes will be inferred as default.\n\n        Parameters\n        ----------\n        in_stype : list of stypes, valid stypes are default, row_sparse and\n            csr\n\n        Returns\n        -------\n        in_stype : list\n            list of argument stypes.\n        out_stype : list\n            list of output types calculated from in_stype,\n            in the same order as declared in list_outputs.\n        aux_type : Optional, list\n            list of aux types calculated from in_stype,\n            in the same order as declared in list_auxiliary_states.\n        \"\"\"\n        for i, stype in enumerate(in_stype):\n            assert stype == _STORAGE_TYPE_ID_TO_STR[_STORAGE_TYPE_DEFAULT], \\\n            \"Default infer_storage_type implementation doesnt allow non default stypes: \" \\\n            f\"found non default stype '{stype}' for in_stype[{i}]. Please implement \" \\\n            \"infer_storage_type and infer_storage_type_backward interface \" \\\n            \"in your custom operator if you have non-default input/output stypes\"\n        return in_stype, \\\n               [_STORAGE_TYPE_ID_TO_STR[_STORAGE_TYPE_DEFAULT]]*len(self.list_outputs()), \\\n               [_STORAGE_TYPE_ID_TO_STR[_STORAGE_TYPE_DEFAULT]]*len(self.list_auxiliary_states())\n\n    def infer_storage_type_backward(self, ograd_stype, in_stype, out_stype, igrad_stype, aux_stype):\n        \"\"\"infer_storage_type_backward interface. Used to infer storage\n        type of inputs and outputs in the backward pass.\n\n        Will raise an error if undefined storage type is returned.\n        Returned lists have to be the same size as the input lists to infer_storage_type_backward,\n        otherwise an exception will be thrown. When this interface is not implemented,\n        all stypes will be inferred as default.\n\n        Parameters\n        ----------\n        ograd_stype : list\n            list of output gradient storage types\n        in_stype : list\n            list of input storage types\n        out_stype : list\n            list of output storage types\n        igrad_stype : list\n            list of input gradient storage types\n        aux_stype : list\n            list of auxiliary storage types\n\n        Returns\n        -------\n        ograd_stype : list\n            list of inferred output gradient storage types\n        in_stype : list\n            list of inferred input storage types\n        out_stype : list\n            list of inferred output storage types\n        igrad_stype : list\n            list of inferred input gradient storage types\n        aux_stype : list\n            list of inferred storage types for auxiliary states\n        \"\"\"\n        for i, stype in enumerate(ograd_stype):\n            assert stype == _STORAGE_TYPE_ID_TO_STR[_STORAGE_TYPE_DEFAULT], \\\n            \"Default infer_storage_type_backward implementation doesnt allow non default stypes: \" \\\n             f\"found non default stype '{stype}' for ograd_stype[{i}]. Please implement \" \\\n             \"infer_storage_type and infer_storage_type_backward interface \" \\\n             \"in your custom operator if you have non-default output gradient stypes\"\n        for i, stype in enumerate(igrad_stype):\n            if stype == _STORAGE_TYPE_ID_TO_STR[_STORAGE_TYPE_UNDEFINED]:\n                stype = _STORAGE_TYPE_ID_TO_STR[_STORAGE_TYPE_DEFAULT]\n            assert stype == _STORAGE_TYPE_ID_TO_STR[_STORAGE_TYPE_DEFAULT], \\\n            \"Default infer_storage_type_backward implementation doesnt allow non default stypes: \" \\\n            f\"found non default stype '{stype}' for igrad_stype[{i}]. Please implement \" \\\n            \"infer_storage_type and infer_storage_type_backward interface \" \\\n            \"in your custom operator if you have non-default input gradient stypes\"\n        stype_lists = [ograd_stype, in_stype, out_stype, igrad_stype, aux_stype]\n        for stype_list in stype_lists:\n            stype_list[:] = len(stype_list) * [_STORAGE_TYPE_ID_TO_STR[_STORAGE_TYPE_DEFAULT]]\n        return stype_lists[0], stype_lists[1], stype_lists[2], stype_lists[3], stype_lists[4]\n\n    def list_outputs(self):\n        \"\"\"list_outputs interface. Can override when creating new operators.\n\n        Returns\n        -------\n        outputs : list\n            List of output blob names.\n        \"\"\"\n        return ['output']\n\n    def list_arguments(self):\n        \"\"\"list_arguments interface. Can override when creating new operators.\n\n        Returns\n        -------\n        arguments : list\n            List of argument blob names.\n        \"\"\"\n        return ['data']\n\n    def list_auxiliary_states(self):\n        \"\"\"list_auxiliary_states interface. Can override when creating new operators.\n\n        Returns\n        -------\n        auxs : list\n            list of auxiliary state blob names.\n        \"\"\"\n        return []\n\n    def declare_backward_dependency(self, out_grad, in_data, out_data):\n        \"\"\"Declare dependencies of this operator for backward pass.\n\n        Parameters\n        ----------\n        out_grad : list of int\n            ids of out_grad blobs.\n        in_data : list of int\n            ids of in_data blobs.\n        out_data: list of int\n            ids of out_data blobs.\n\n        Returns\n        -------\n        deps : list of int\n            ids of the needed blobs.\n        \"\"\"\n        deps = []\n        if self.need_top_grad_:\n            deps.extend(out_grad)\n        deps.extend(in_data)\n        deps.extend(out_data)\n        return deps\n\n    def create_operator(self, ctx, in_shapes, in_dtypes):\n        \"\"\"Create an operator that carries out the real computation\n        given the context, input shapes, and input data types.\"\"\"\n        # pylint: disable=W0613\n        return CustomOp()\n\n\nclass _Registry(object):\n    \"\"\"CustomOp registry.\"\"\"\n    def __init__(self):\n        self.ref_holder = {}\n        self.counter = 0\n        self.result_deps = set()\n        self.lock = Lock()\n\n    def inc(self):\n        \"\"\"Get index for new entry.\"\"\"\n        self.lock.acquire()\n        cur = self.counter\n        self.counter += 1\n        self.lock.release()\n        return cur\n\n\n_registry = _Registry()\n\n\ndef register(reg_name):\n    \"\"\"Register a subclass of CustomOpProp to the registry with name reg_name.\"\"\"\n    def do_register(prop_cls):\n        \"\"\"Register a subclass of CustomOpProp to the registry.\"\"\"\n        fb_functype = CFUNCTYPE(c_int, c_int, POINTER(c_void_p), POINTER(c_int),\n                                POINTER(c_int), c_int, c_void_p)\n        del_functype = CFUNCTYPE(c_int, c_void_p)\n\n        infershape_functype = CFUNCTYPE(c_int, c_int, POINTER(c_int),\n                                        POINTER(POINTER(mx_int)), c_void_p)\n        infertype_functype = CFUNCTYPE(c_int, c_int, POINTER(c_int), c_void_p)\n        inferstorage_functype = CFUNCTYPE(c_int, c_int, POINTER(c_int), c_void_p)\n        inferstorage_backward_functype = CFUNCTYPE(c_int, c_int, POINTER(c_int), \\\n                                                   POINTER(c_int), c_void_p)\n        list_functype = CFUNCTYPE(c_int, POINTER(POINTER(POINTER(c_char))), c_void_p)\n        deps_functype = CFUNCTYPE(c_int, c_int_p, c_int_p, c_int_p,\n                                  c_int_p, POINTER(c_int_p), c_void_p)\n        createop_functype = CFUNCTYPE(c_int, c_char_p, c_int, POINTER(POINTER(mx_uint)),\n                                      POINTER(c_int), POINTER(c_int),\n                                      POINTER(MXCallbackList), c_void_p)\n        req_enum = ('null', 'write', 'inplace', 'add')\n        create_ndarray_fn = _np_ndarray_cls if is_np_array() else _ndarray_cls\n\n        def creator(op_type, argc, keys, vals, ret):\n            \"\"\"internal function\"\"\"\n            assert py_str(op_type) == reg_name\n            kwargs = {}\n            for i in range(argc):\n                key = py_str(keys[i])\n                if key not in ['__ctx_group__', '__lr_mult__', '__wd_mult__',\n                               '__force_mirroring__',\n                               '__mirror_stage__', '__profiler_scope__']:\n                    kwargs[key] = py_str(vals[i])\n            op_prop = prop_cls(**kwargs)\n\n            def infer_shape_entry(num_tensor, tensor_dims,\n                                  tensor_shapes, _):\n                \"\"\"C Callback for ``CustomOpProp::InferShape``.\"\"\"\n                try:\n                    n_in = len(op_prop.list_arguments())\n                    n_out = len(op_prop.list_outputs())\n                    n_aux = len(op_prop.list_auxiliary_states())\n                    assert num_tensor == n_in + n_out + n_aux\n\n                    shapes = [[tensor_shapes[i][j] for j in range(tensor_dims[i])]\n                              for i in range(n_in)]\n                    ret = op_prop.infer_shape(shapes)\n                    if len(ret) == 2:\n                        ishape, oshape = ret\n                        ashape = []\n                    elif len(ret) == 3:\n                        ishape, oshape, ashape = ret\n                    else:\n                        raise AssertionError(\"infer_shape must return 2 or 3 lists\")\n                    assert len(oshape) == n_out, \\\n                        f\"InferShape Error: expecting {n_out} entries in returned output \" \\\n                        f\"shapes, got {len(oshape)}.\"\n                    assert len(ishape) == n_in, \\\n                        f\"InferShape Error: expecting {n_in} entries in returned input \" \\\n                        f\"shapes, got {len(ishape)}.\"\n                    assert len(ashape) == n_aux, \\\n                        f\"InferShape Error: expecting {n_aux} entries in returned aux state \" \\\n                        f\"shapes, got {len(ashape)}.\"\n                    rshape = list(ishape) + list(oshape) + list(ashape)\n                    for i in range(n_in+n_out+n_aux):\n                        tensor_shapes[i] = cast(c_array_buf(mx_int,\n                                                            array('i', rshape[i])),\n                                                POINTER(mx_int))\n                        tensor_dims[i] = len(rshape[i])\n\n                    infer_shape_entry._ref_holder = [tensor_shapes]\n                except Exception:\n                    print(f'Error in {reg_name}.infer_shape: {traceback.format_exc()}')\n                    return False\n                return True\n\n            def infer_storage_type_backward_entry(num_tensor, tensor_stypes, tags, _):\n                # pylint: disable=C0301\n                \"\"\"C Callback for CustomOpProp::InferStorageTypeBackward\"\"\"\n                try:\n                    tensors = [[] for i in range(5)]\n                    for i in range(num_tensor):\n                        tensors[tags[i]].append(_STORAGE_TYPE_ID_TO_STR[tensor_stypes[i]])\n                    # Ordering of stypes: ograd, input, output, igrad, aux\n                    tensors = [tensors[3], tensors[0], tensors[1], tensors[2], tensors[4]]\n                    ret = op_prop.infer_storage_type_backward(tensors[0],\n                                                              tensors[1],\n                                                              tensors[2],\n                                                              tensors[3],\n                                                              tensors[4])\n                    if len(ret) == 4:\n                        ret += []\n                    elif len(ret) == 5:\n                        pass\n                    else:\n                        raise AssertionError(\"infer_storage_type_backward must return 4 or 5 lists\")\n                    assert len(ret[0]) == len(tensors[0]), \\\n                        f\"InferStorageTypeBackward Error: expecting == {len(tensors[0])} \" \\\n                        \"entries in returned output gradient \" \\\n                        f\"stypes, got {len(ret[0])}.\"\n                    assert len(ret[1]) == len(tensors[1]), \\\n                        f\"InferStorageTypeBackward Error: expecting == {len(tensors[1])} \" \\\n                        \"entries in returned input stypes, \" \\\n                        f\"got {len(ret[1])}.\"\n                    assert len(ret[2]) == len(tensors[2]), \\\n                        f\"InferStorageTypeBackward Error: expecting == {len(tensors[2])} \" \\\n                        \"entries in returned output stypes, \" \\\n                        f\"got {len(ret[2])}.\"\n                    assert len(ret[3]) == len(tensors[3]), \\\n                        f\"InferStorageTypeBackward Error: expecting == {len(tensors[3])} \" \\\n                        \"entries in returned input gradient stypes, \" \\\n                        f\"got {len(ret[3])}.\"\n                    assert len(ret[4]) == len(tensors[4]), \\\n                        f\"InferStorageTypeBackward Error: expecting == {len(tensors[4])} \" \\\n                        \"entries in returned aux stypes, \" \\\n                        f\"got {len(ret[4])}.\"\n                    rstype = []\n                    for ret_list in ret:\n                        rstype.extend(ret_list)\n\n                    for i, stype in enumerate(rstype):\n                        assert stype != _STORAGE_TYPE_ID_TO_STR[_STORAGE_TYPE_UNDEFINED], \\\n                            \"stype should not be undefined\"\n                        assert stype in _STORAGE_TYPE_STR_TO_ID, \\\n                            f\"Provided stype: {stype} is not valid \" \\\n                            \"valid stypes are {}, {}, {}\".format(_STORAGE_TYPE_ID_TO_STR[_STORAGE_TYPE_DEFAULT],\n                                                                 _STORAGE_TYPE_ID_TO_STR[_STORAGE_TYPE_ROW_SPARSE],\n                                                                 _STORAGE_TYPE_ID_TO_STR[_STORAGE_TYPE_CSR])\n                        tensor_stypes[i] = _STORAGE_TYPE_STR_TO_ID[stype]\n\n                    infer_storage_type_backward_entry._ref_holder = [tensor_stypes]\n                except Exception:\n                    print(f'Error in {reg_name}.infer_type: {traceback.format_exc()}')\n                    return False\n                return True\n\n            def infer_storage_type_entry(num_tensor, tensor_stypes, _):\n                \"\"\"C Callback for CustomOpProp::InferStorageType\"\"\"\n                try:\n                    n_in = len(op_prop.list_arguments())\n                    n_out = len(op_prop.list_outputs())\n                    n_aux = len(op_prop.list_auxiliary_states())\n                    assert num_tensor == n_in + n_out + n_aux\n\n                    stypes = [_STORAGE_TYPE_ID_TO_STR[tensor_stypes[i]] for i in range(n_in)]\n                    ret = op_prop.infer_storage_type(stypes)\n                    if len(ret) == 2:\n                        istype, ostype = ret\n                        astype = []\n                    elif len(ret) == 3:\n                        istype, ostype, astype = ret\n                    else:\n                        raise AssertionError(\"infer_storage_type must return 2 or 3 lists\")\n\n                    assert len(ostype) == n_out, \\\n                        f\"InferStorageType Error: expecting {n_out} entries in returned output \" \\\n                        f\"stypes, got {len(ostype)}.\"\n                    assert len(istype) == n_in, \\\n                        f\"InferStorageType Error: expecting {n_in} entries in returned input \" \\\n                        f\"stypes, got {len(istype)}.\"\n                    assert len(astype) == n_aux, \\\n                        f\"InferStorageType Error: expecting {n_aux} entries in returned aux state \" \\\n                        f\"stypes, got {len(astype)}.\"\n                    rtype = list(istype) + list(ostype) + list(astype)\n                    for i, dtype in enumerate(rtype):\n                        tensor_stypes[i] = _STORAGE_TYPE_STR_TO_ID[dtype]\n                    infer_storage_type_entry._ref_holder = [tensor_stypes]\n                except Exception:\n                    print(f'Error in {reg_name}.infer_type: {traceback.format_exc()}')\n                    return False\n                return True\n\n            def infer_type_entry(num_tensor, tensor_types, _):\n                \"\"\"C Callback for CustomOpProp::InferType\"\"\"\n                try:\n                    n_in = len(op_prop.list_arguments())\n                    n_out = len(op_prop.list_outputs())\n                    n_aux = len(op_prop.list_auxiliary_states())\n                    assert num_tensor == n_in + n_out + n_aux\n\n                    types = [dtype_mx_to_np(tensor_types[i]) for i in range(n_in)]\n                    ret = op_prop.infer_type(types)\n                    if len(ret) == 2:\n                        itype, otype = ret\n                        atype = []\n                    elif len(ret) == 3:\n                        itype, otype, atype = ret\n                    else:\n                        raise AssertionError(\"infer_type must return 2 or 3 lists\")\n                    assert len(otype) == n_out, \\\n                        f\"InferType Error: expecting {n_out} entries in returned output \" \\\n                        f\"types, got {len(otype)}.\"\n                    assert len(itype) == n_in, \\\n                        f\"InferType Error: expecting {n_in} entries in returned input \" \\\n                        f\"types, got {len(itype)}.\"\n                    assert len(atype) == n_aux, \\\n                        f\"InferType Error: expecting {n_aux} entries in returned aux state \" \\\n                        f\"types, got {len(atype)}.\"\n                    rtype = list(itype) + list(otype) + list(atype)\n                    for i, dtype in enumerate(rtype):\n                        tensor_types[i] = dtype_np_to_mx(dtype)\n\n                    infer_type_entry._ref_holder = [tensor_types]\n                except Exception:\n                    print(f'Error in {reg_name}.infer_type: {traceback.format_exc()}')\n                    return False\n                return True\n\n            def list_outputs_entry(out, _):\n                \"\"\"C Callback for CustomOpProp::ListOutputs\"\"\"\n                try:\n                    ret = op_prop.list_outputs()\n                    ret = [c_str(i) for i in ret] + [c_char_p(0)]\n                    ret = c_array(c_char_p, ret)\n                    out[0] = cast(ret, POINTER(POINTER(c_char)))\n\n                    list_outputs_entry._ref_holder = [out]\n                except Exception:\n                    print(f'Error in {reg_name}.list_outputs: {traceback.format_exc()}')\n                    return False\n                return True\n\n            def list_arguments_entry(out, _):\n                \"\"\"C Callback for CustomOpProp::ListArguments\"\"\"\n                try:\n                    ret = op_prop.list_arguments()\n                    ret = [c_str(i) for i in ret] + [c_char_p(0)]\n                    ret = c_array(c_char_p, ret)\n                    out[0] = cast(ret, POINTER(POINTER(c_char)))\n\n                    list_arguments_entry._ref_holder = [out]\n                except Exception:\n                    print(f'Error in {reg_name}.list_arguments: {traceback.format_exc()}')\n                    return False\n                return True\n\n            def list_auxiliary_states_entry(out, _):\n                \"\"\"C Callback for CustomOpProp::ListAuxiliaryStates\"\"\"\n                try:\n                    ret = op_prop.list_auxiliary_states()\n                    ret = [c_str(i) for i in ret] + [c_char_p(0)]\n                    ret = c_array(c_char_p, ret)\n                    out[0] = cast(ret, POINTER(POINTER(c_char)))\n\n                    list_auxiliary_states_entry._ref_holder = [out]\n                except Exception:\n                    tb = traceback.format_exc()\n                    print(f'Error in {reg_name}.list_auxiliary_states: {tb}')\n                    return False\n                return True\n\n            def declare_backward_dependency_entry(out_grad, in_data, out_data, num_dep, deps, _):\n                \"\"\"C Callback for CustomOpProp::DeclareBacwardDependency\"\"\"\n                try:\n                    out_grad = [out_grad[i] for i in range(len(op_prop.list_outputs()))]\n                    in_data = [in_data[i] for i in range(len(op_prop.list_arguments()))]\n                    out_data = [out_data[i] for i in range(len(op_prop.list_outputs()))]\n                    rdeps = op_prop.declare_backward_dependency(out_grad, in_data, out_data)\n                    num_dep[0] = len(rdeps)\n                    _registry.result_deps = set()\n                    for dep in rdeps:\n                        _registry.result_deps.add(dep)\n                    rdeps = cast(c_array_buf(c_int, array('i', rdeps)), c_int_p)\n                    deps[0] = rdeps\n\n                    declare_backward_dependency_entry._ref_holder = [deps]\n                except Exception:\n                    tb = traceback.format_exc()\n                    print(f'Error in {reg_name}.declare_backward_dependency: {tb}')\n                    return False\n                return True\n\n            def create_operator_entry(ctx, num_inputs, shapes, ndims, dtypes, ret, _):\n                \"\"\"C Callback for CustomOpProp::CreateOperator\"\"\"\n                try:\n                    ctx = py_str(ctx)\n                    sep = ctx.find('(')\n                    ctx = context.Context(ctx[:sep], int(ctx[sep+1:-1]))\n                    ndims = [ndims[i] for i in range(num_inputs)]\n                    shapes = [[shapes[i][j] for j in range(ndims[i])] for i in range(num_inputs)]\n                    dtypes = [dtypes[i] for i in range(num_inputs)]\n                    op = op_prop.create_operator(ctx, shapes, dtypes)\n\n                    def forward_entry(num_ndarray, ndarraies, tags, reqs, is_train, _):\n                        \"\"\"C Callback for CustomOp::Forward\"\"\"\n                        try:\n                            tensors = [[] for i in range(5)]\n                            for i in range(num_ndarray):\n                                if tags[i] == 1 or tags[i] == 4:\n                                    tensors[tags[i]].append(\n                                        create_ndarray_fn(cast(ndarraies[i], NDArrayHandle), writable=True)\n                                    )\n                                else:\n                                    tensors[tags[i]].append(\n                                        create_ndarray_fn(cast(ndarraies[i], NDArrayHandle), writable=False)\n                                    )\n                            reqs = [req_enum[reqs[i]] for i in range(len(tensors[1]))]\n                            with ctx:\n                                op.forward(is_train=is_train, req=reqs,\n                                           in_data=tensors[0], out_data=tensors[1],\n                                           aux=tensors[4])\n                        except Exception:\n                            print(f'Error in CustomOp.forward: {traceback.format_exc()}')\n                            return False\n                        return True\n\n                    def backward_entry(num_ndarray, ndarraies, tags, reqs, is_train, _):\n                        \"\"\"C Callback for CustomOp::Backward\"\"\"\n                        # pylint: disable=W0613\n                        try:\n                            tensors = [[] for i in range(5)]\n                            num_outputs = len(op_prop.list_outputs())\n                            num_args = len(op_prop.list_arguments())\n                            for i in range(num_ndarray):\n                                if i in _registry.result_deps or i >= (num_outputs * 2 + num_args):\n                                    # If it is a backward dependency or output or aux:\n                                    # Set stype as undefined so that it returns\n                                    # ndarray based on existing stype\n                                    stype = _STORAGE_TYPE_UNDEFINED\n                                else:\n                                    # If it is some input, output or out grad ndarray not part of\n                                    # backward dependency it is empty and thus the ndarray should\n                                    # be set to default\n                                    stype = _STORAGE_TYPE_DEFAULT\n                                if tags[i] == 2 or tags[i] == 4:\n                                    tensors[tags[i]].append(\n                                        create_ndarray_fn(cast(ndarraies[i], NDArrayHandle),\n                                                          writable=True, stype=stype)\n                                    )\n                                else:\n                                    tensors[tags[i]].append(\n                                        create_ndarray_fn(cast(ndarraies[i], NDArrayHandle),\n                                                          writable=False, stype=stype)\n                                    )\n                            reqs = [req_enum[reqs[i]] for i in range(len(tensors[2]))]\n                            with ctx:\n                                op.backward(req=reqs,\n                                            in_data=tensors[0], out_data=tensors[1],\n                                            in_grad=tensors[2], out_grad=tensors[3],\n                                            aux=tensors[4])\n                        except Exception:\n                            print(f'Error in CustomOp.backward: {traceback.format_exc()}')\n                            return False\n                        return True\n\n                    cur = _registry.inc()\n\n                    def delete_entry(_):\n                        \"\"\"C Callback for CustomOp::del\"\"\"\n                        try:\n                            del _registry.ref_holder[cur]\n                        except Exception:\n                            print(f'Error in CustomOp.delete: {traceback.format_exc()}')\n                            return False\n                        return True\n\n                    callbacks = [del_functype(delete_entry),\n                                 fb_functype(forward_entry),\n                                 fb_functype(backward_entry)]\n                    callbacks = [cast(i, CFUNCTYPE(c_int)) for i in callbacks]\n                    contexts = [None, None, None]\n                    ret[0] = MXCallbackList(c_int(len(callbacks)),\n                                            cast(c_array(CFUNCTYPE(c_int), callbacks),\n                                                 POINTER(CFUNCTYPE(c_int))),\n                                            cast(c_array(c_void_p, contexts),\n                                                 POINTER(c_void_p)))\n                    op._ref_holder = [ret]\n                    _registry.ref_holder[cur] = op\n                except Exception:\n                    print(f'Error in {reg_name}.create_operator: {traceback.format_exc()}')\n                    return False\n                return True\n\n            cur = _registry.inc()\n\n            def delete_entry(_):\n                \"\"\"C Callback for CustomOpProp::del\"\"\"\n                try:\n                    del _registry.ref_holder[cur]\n                except Exception:\n                    print(f'Error in CustomOpProp.delete: {traceback.format_exc()}')\n                    return False\n                return True\n\n            callbacks = [del_functype(delete_entry),\n                         list_functype(list_arguments_entry),\n                         list_functype(list_outputs_entry),\n                         list_functype(list_auxiliary_states_entry),\n                         infershape_functype(infer_shape_entry),\n                         deps_functype(declare_backward_dependency_entry),\n                         createop_functype(create_operator_entry),\n                         infertype_functype(infer_type_entry),\n                         inferstorage_functype(infer_storage_type_entry),\n                         inferstorage_backward_functype(infer_storage_type_backward_entry)]\n            callbacks = [cast(i, CFUNCTYPE(c_int)) for i in callbacks]\n            contexts = [None]*len(callbacks)\n            ret[0] = MXCallbackList(c_int(len(callbacks)),\n                                    cast(c_array(CFUNCTYPE(c_int), callbacks),\n                                         POINTER(CFUNCTYPE(c_int))),\n                                    cast(c_array(c_void_p, contexts),\n                                         POINTER(c_void_p)))\n            op_prop._ref_holder = [ret]\n            _registry.ref_holder[cur] = op_prop\n            return True\n\n        creator_functype = CFUNCTYPE(c_int, c_char_p, c_int, POINTER(c_char_p),\n                                     POINTER(c_char_p), POINTER(MXCallbackList))\n        creator_func = creator_functype(creator)\n        check_call(_LIB.MXCustomOpRegister(c_str(reg_name), creator_func))\n        cur = _registry.inc()\n        _registry.ref_holder[cur] = creator_func\n        return prop_cls\n    return do_register\n\n\nregister(\"custom_op\")(CustomOpProp)\n\n\ndef get_all_registered_operators():\n    \"\"\"Get all registered MXNet operator names.\n\n    Returns\n    -------\n    operator_names : list of string\n    \"\"\"\n    plist = ctypes.POINTER(ctypes.c_char_p)()\n    size = ctypes.c_uint()\n\n    check_call(_LIB.MXListAllOpNames(ctypes.byref(size),\n                                     ctypes.byref(plist)))\n\n    mx_registered_operator_names = [py_str(plist[i]) for i in range(size.value)]\n    return mx_registered_operator_names\n\n\ndef get_all_registered_operators_grouped():\n    \"\"\"Get all registered MXNet operator names, grouped by 'original' operator.\n\n    Returns\n    -------\n    names : a dictionary, mapping op name to the list of all its aliases (including the original).\n    \"\"\"\n    ret = {}\n    for aname in get_all_registered_operators():\n        op_handle = OpHandle()\n        check_call(_LIB.NNGetOpHandle(c_str(aname), ctypes.byref(op_handle)))\n        name = ctypes.c_char_p()\n        desc = ctypes.c_char_p()\n        num_args = mx_uint()\n        arg_names = ctypes.POINTER(ctypes.c_char_p)()\n        arg_types = ctypes.POINTER(ctypes.c_char_p)()\n        arg_descs = ctypes.POINTER(ctypes.c_char_p)()\n        ret_types = ctypes.POINTER(ctypes.c_char_p)()\n        check_call(_LIB.NNGetOpInfo(op_handle, ctypes.byref(name), ctypes.byref(desc),\n                                    ctypes.byref(num_args), ctypes.byref(arg_names),\n                                    ctypes.byref(arg_types), ctypes.byref(arg_descs),\n                                    ctypes.byref(ret_types)))\n        ret.setdefault(py_str(name.value), []).append(aname)\n    return ret\n\n\nOperatorArguments = collections.namedtuple('OperatorArguments', ['narg', 'names', 'types'])\n\n\ndef get_operator_arguments(op_name):\n    \"\"\"Given operator name, fetch operator arguments - number of arguments,\n    argument names, argument types.\n\n    Parameters\n    ----------\n    op_name: str\n        Handle for the operator\n\n    Returns\n    -------\n    operator_arguments : OperatorArguments, namedtuple with number of arguments, names and types\n    \"\"\"\n    op_handle = OpHandle()\n    check_call(_LIB.NNGetOpHandle(c_str(op_name), ctypes.byref(op_handle)))\n    real_name = ctypes.c_char_p()\n    desc = ctypes.c_char_p()\n    num_args = mx_uint()\n    arg_names = ctypes.POINTER(ctypes.c_char_p)()\n    arg_types = ctypes.POINTER(ctypes.c_char_p)()\n    arg_descs = ctypes.POINTER(ctypes.c_char_p)()\n    key_var_num_args = ctypes.c_char_p()\n    ret_type = ctypes.c_char_p()\n\n    check_call(_LIB.MXSymbolGetAtomicSymbolInfo(\n        op_handle, ctypes.byref(real_name), ctypes.byref(desc),\n        ctypes.byref(num_args),\n        ctypes.byref(arg_names),\n        ctypes.byref(arg_types),\n        ctypes.byref(arg_descs),\n        ctypes.byref(key_var_num_args),\n        ctypes.byref(ret_type)))\n\n    narg = int(num_args.value)\n    arg_names = [py_str(arg_names[i]) for i in range(narg)]\n    arg_types = [py_str(arg_types[i]) for i in range(narg)]\n    return OperatorArguments(narg, arg_names, arg_types)\n"
  },
  {
    "path": "python/mxnet/optimizer/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Optimizer API of MXNet.\"\"\"\n\nfrom . import (optimizer, contrib, updater, utils, sgd,\n               sgld, signum, dcasgd, nag, adagrad,\n               adadelta, adam, adamax, nadam, ftrl,\n               ftml, lars, lamb, rmsprop, lans, adamW,\n               adabelief)\n# pylint: disable=wildcard-import\nfrom .adabelief import *\n\nfrom .adamW import *\n\nfrom .optimizer import *\n\nfrom .updater import *\n\nfrom .utils import *\n\nfrom .sgd import *\n\nfrom .sgld import *\n\nfrom .signum import *\n\nfrom .dcasgd import *\n\nfrom .nag import *\n\nfrom .adagrad import *\n\nfrom .adadelta import *\n\nfrom .adam import *\n\nfrom .adamax import *\n\nfrom .nadam import *\n\nfrom .ftrl import *\n\nfrom .ftml import *\n\nfrom .lars import *\n\nfrom .lamb import *\n\nfrom .rmsprop import *\n\nfrom .lans import *\n\n__all__ = optimizer.__all__ + updater.__all__ + ['contrib'] + sgd.__all__ + sgld.__all__ \\\n          + signum.__all__ + dcasgd.__all__ + nag.__all__ + adabelief.__all__ \\\n          + adagrad.__all__ + adadelta.__all__ + adam.__all__ + adamax.__all__ \\\n          + nadam.__all__ + ftrl.__all__ + ftml.__all__ + lars.__all__ \\\n          + lamb.__all__ + rmsprop.__all__ + lans.__all__\n"
  },
  {
    "path": "python/mxnet/optimizer/adabelief.py",
    "content": "# coding: utf-8\r\n# Licensed to the Apache Software Foundation (ASF) under one\r\n# or more contributor license agreements.  See the NOTICE file\r\n# distributed with this work for additional information\r\n# regarding copyright ownership.  The ASF licenses this file\r\n# to you under the Apache License, Version 2.0 (the\r\n# \"License\"); you may not use this file except in compliance\r\n# with the License.  You may obtain a copy of the License at\r\n#\r\n#   http://www.apache.org/licenses/LICENSE-2.0\r\n#\r\n# Unless required by applicable law or agreed to in writing,\r\n# software distributed under the License is distributed on an\r\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\r\n# KIND, either express or implied.  See the License for the\r\n# specific language governing permissions and limitations\r\n# under the License.\r\n\"\"\"AdaBelief optimizer.\"\"\"\r\nimport math\r\nimport os\r\nimport numpy as np\r\nfrom .optimizer import Optimizer, register\r\nfrom ..ndarray import (zeros, clip, sqrt, square, full, NDArray)\r\nfrom ..ndarray.contrib import mp_adabelief_update, adabelief_update,\\\r\n    multi_mp_adabelief_update, multi_adabelief_update\r\n\r\n\r\n__all__ = ['AdaBelief']\r\n\r\n\r\n@register\r\nclass AdaBelief(Optimizer):\r\n    \"\"\"The AdaBelief optimizer.\r\n\r\n    This class implements the optimizer described in *Adapting Stepsizes by the Belief in Observed Gradients*,\r\n     available at https://arxiv.org/pdf/2010.07468.pdf.\r\n\r\n    Updates are applied by::\r\n\r\n        grad = clip(grad * rescale_grad, clip_gradient) + wd * w\r\n        m = beta1 * m + (1 - beta1) * grad\r\n        s = beta2 * s + (1 - beta2) * ((grad - m)**2) + epsilon\r\n        lr = learning_rate * sqrt(1 - beta2**t) / (1 - beta1**t)\r\n        w = w - lr * (m / (sqrt(s) + epsilon))\r\n\r\n\r\n    Also, we can turn off the bias correction term and the updates are as follows::\r\n\r\n        grad = clip(grad * rescale_grad, clip_gradient) + wd * w\r\n        m = beta1 * m + (1 - beta1) * grad\r\n        s = beta2 * s + (1 - beta2) * ((grad - m)**2) + epsilon\r\n        lr = learning_rate\r\n        w = w - lr * (m / (sqrt(s) + epsilon))\r\n\r\n    This optimizer accepts the following parameters in addition to those accepted\r\n    by :class:`.Optimizer`.\r\n\r\n    Parameters\r\n    ----------\r\n    learning_rate : float, default 0.001\r\n        The initial learning rate. If None, the optimization will use the\r\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\r\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\r\n        is also None, then it will be set to 0.01 by default.\r\n    beta1 : float, default 0.9\r\n        Exponential decay rate for the first moment estimates.\r\n    beta2 : float, default 0.999\r\n        Exponential decay rate for the second moment estimates.\r\n    epsilon : float, default 1e-6\r\n        Small value to avoid division by 0.\r\n    correct_bias : bool, default True\r\n       Can be set to False to avoid correcting bias in Adam (e.g. like in Bert TF repository).\r\n       Default True.\r\n    use_fused_step : bool, default True\r\n        Whether or not to use fused kernels for optimizer.\r\n        When use_fused_step=False, step is called,\r\n        otherwise, fused_step is called.\r\n    \"\"\"\r\n    def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-6,\r\n                 correct_bias=True, use_fused_step=True, **kwargs):\r\n        super().__init__(use_fused_step=use_fused_step,\r\n                         learning_rate=learning_rate,\r\n                         **kwargs)\r\n        self.beta1 = beta1\r\n        self.beta2 = beta2\r\n        self.epsilon = epsilon\r\n        self.correct_bias = correct_bias\r\n        self.aggregate_num = max(1, min(50,\r\n                                        int(os.getenv('MXNET_OPTIMIZER_AGGREGATION_SIZE', '4'))))\r\n\r\n    def create_state(self, index, weight):\r\n        \"\"\"state creation function.\"\"\"\r\n        return (zeros(weight.shape, weight.context, dtype=weight.dtype),  # mean\r\n                zeros(weight.shape, weight.context, dtype=weight.dtype))  # variance\r\n\r\n    def step(self, indices, weights, grads, states):\r\n        \"\"\"Perform an optimization step using gradients and states.\r\n\r\n        Parameters\r\n        ----------\r\n        indices : list of int\r\n            List of unique indices of the parameters into the individual learning rates\r\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\r\n            and `set_wd_mult()`, respectively.\r\n        weights : list of NDArray\r\n            List of parameters to be updated.\r\n        grads : list of NDArray\r\n            List of gradients of the objective with respect to this parameter.\r\n        states : List of any obj\r\n            List of state returned by `create_state()`.\r\n        \"\"\"\r\n        for index, weight, grad, state in zip(indices, weights, grads, states):\r\n            self._update_count(index)\r\n            lr = self._get_lr(index)\r\n            wd = self._get_wd(index)\r\n            eps = self.epsilon\r\n            t = self._index_update_count[index]\r\n\r\n            # preprocess grad\r\n            grad *= self.rescale_grad\r\n            grad += wd * weight\r\n            if self.clip_gradient is not None:\r\n                grad = clip(grad, -self.clip_gradient, self.clip_gradient)\r\n            if self.correct_bias:\r\n                coef1 = 1. - self.beta1**t\r\n                coef2 = 1. - self.beta2**t\r\n                lr *= math.sqrt(coef2) / coef1\r\n\r\n            # update mean and var\r\n            mean, var = state\r\n            mean[:] *= self.beta1\r\n            mean[:] += (1. - self.beta1) * grad\r\n            var[:] *= self.beta2\r\n            var[:] += (1. - self.beta2) * square(grad - mean)\r\n            var[:] += eps\r\n\r\n            # update weight\r\n            d = mean / (sqrt(var) + eps)\r\n            weight[:] -= lr * d\r\n\r\n    def fused_step(self, indices, weights, grads, states):\r\n        \"\"\"Perform a fused optimization step using gradients and states.\r\n        Fused kernel is used for update.\r\n\r\n        Parameters\r\n        ----------\r\n        indices : list of int\r\n            List of unique indices of the parameters into the individual learning rates\r\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\r\n            and `set_wd_mult()`, respectively.\r\n        weights : list of NDArray\r\n            List of parameters to be updated.\r\n        grads : list of NDArray\r\n            List of gradients of the objective with respect to this parameter.\r\n        states : List of any obj\r\n            List of state returned by `create_state()`.\r\n        \"\"\"\r\n        multi_precision = self.multi_precision and weights[0].dtype == np.float16\r\n        aggregate = self.aggregate_num > 1\r\n        if not isinstance(indices, (tuple, list)):\r\n            indices = [indices]\r\n            weights = [weights]\r\n            grads = [grads]\r\n            states = [states]\r\n        for w_i, g_i in zip(weights, grads):\r\n            assert(isinstance(w_i, NDArray))\r\n            assert(isinstance(g_i, NDArray))\r\n            aggregate = (aggregate and\r\n                         w_i.stype == 'default' and\r\n                         g_i.stype == 'default')\r\n        self._update_count(indices)\r\n        lrs = self._get_lrs(indices)\r\n        wds = self._get_wds(indices)\r\n        if self.correct_bias:\r\n            new_lrs = []\r\n            for idx, lr in zip(indices, lrs):\r\n                t = self._index_update_count[idx]\r\n                coef1 = 1. - self.beta1 ** t\r\n                coef2 = 1. - self.beta2 ** t\r\n                new_lrs.append(lr * math.sqrt(coef2) / coef1)\r\n            lrs = new_lrs\r\n        if not isinstance(self.rescale_grad, NDArray):\r\n            self.rescale_grad = full(shape=(1,), val=self.rescale_grad, ctx=weights[0].context)\r\n        else:\r\n            self.rescale_grad = self.rescale_grad.as_in_context(weights[0].context)\r\n        kwargs = {'beta1': self.beta1, 'beta2': self.beta2, 'epsilon': self.epsilon,\r\n                  'rescale_grad': self.rescale_grad}\r\n        if self.clip_gradient:\r\n            kwargs['clip_gradient'] = self.clip_gradient\r\n\r\n        if aggregate:\r\n            current_index = 0\r\n            while current_index < len(indices):\r\n                sidx = current_index\r\n                eidx = min(current_index + self.aggregate_num, len(indices))\r\n                if not multi_precision:\r\n                    mean, var = list(zip(*states[sidx:eidx]))\r\n                    multi_adabelief_update(weights[sidx:eidx], grads[sidx:eidx],\r\n                                           mean, var,\r\n                                           out=weights[sidx:eidx],\r\n                                           size=len(weights[sidx:eidx]),\r\n                                           lrs=list(np.ones(len(weights[sidx:eidx]))),\r\n                                           wds=wds[sidx:eidx],\r\n                                           etas=lrs[sidx:eidx],\r\n                                           **kwargs)\r\n                else:\r\n                    mean_var = list(zip(*states[sidx:eidx]))[0]\r\n                    tmean_var = list(zip(*mean_var))\r\n                    mean = tmean_var[0]\r\n                    var = tmean_var[1]\r\n                    multi_mp_adabelief_update(weights[sidx:eidx],\r\n                                              grads[sidx:eidx],\r\n                                              mean, var,\r\n                                              list(zip(*states[sidx:eidx]))[1],\r\n                                              out=weights[sidx:eidx],\r\n                                              size=len(weights[sidx:eidx]),\r\n                                              lrs=list(np.ones(len(weights[sidx:eidx]))),\r\n                                              wds=wds[sidx:eidx],\r\n                                              etas=lrs[sidx:eidx],\r\n                                              **kwargs)\r\n                current_index += self.aggregate_num\r\n        else:\r\n            for w_i, g_i, s_i, lr, wd in zip(weights, grads, states, lrs, wds):\r\n                if not multi_precision:\r\n                    mean, var = s_i\r\n                    adabelief_update(w_i, g_i, mean, var, out=w_i,\r\n                                     lr=1, wd=wd, eta=lr, **kwargs)\r\n                else:\r\n                    mean, var = s_i[0]\r\n                    mp_adabelief_update(w_i, g_i, mean, var, s_i[1], out=w_i,\r\n                                        lr=1, wd=wd, eta=lr, **kwargs)\r\n"
  },
  {
    "path": "python/mxnet/optimizer/adadelta.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=W0223\n\"\"\"AdaDelta optimizer.\"\"\"\nfrom __future__ import absolute_import\nfrom ..ndarray import (zeros, clip, sqrt, square)\nfrom .optimizer import Optimizer, register\n\n__all__ = ['AdaDelta']\n\n\n@register\nclass AdaDelta(Optimizer):\n    \"\"\"The AdaDelta optimizer.\n\n    This class implements AdaDelta, an optimizer described in  *ADADELTA: An adaptive\n    learning rate method*, available at https://arxiv.org/abs/1212.5701.\n\n    This optimizer updates each weight by::\n\n        grad = clip(grad * rescale_grad, clip_gradient) + wd * weight\n        acc_grad = rho * acc_grad + (1. - rho) * grad * grad\n        delta = sqrt(acc_delta + epsilon) / sqrt(acc_grad + epsilon) * grad\n        acc_delta = rho * acc_delta + (1. - rho) * delta * delta\n        weight -= learning_rate * delta\n\n    This optimizer accepts the following parameters in addition to those accepted\n    by :class:`.Optimizer`.\n\n    Parameters\n    ----------\n    learning_rate : float, default 1.0\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    rho: float, default 0.9\n        Decay rate for both squared gradients and delta.\n    epsilon : float, default 1e-6\n        Small value to avoid division by 0.\n    use_fused_step : bool, default False\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False, step is called,\n        otherwise, fused_step is called.\n    \"\"\"\n    def __init__(self, learning_rate=1.0, rho=0.9, epsilon=1e-6, use_fused_step=False, **kwargs):\n        super(AdaDelta, self).__init__(learning_rate=learning_rate,\n                                       use_fused_step=use_fused_step,\n                                       **kwargs)\n        self.rho = rho\n        self.epsilon = epsilon\n\n    def create_state(self, index, weight):\n        return (zeros(weight.shape, weight.context),  # accumulated g\n                zeros(weight.shape, weight.context))  # accumulated delta\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform an optimization step using gradients and states.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n\n            # preprocess grad\n            grad *= self.rescale_grad\n            if self.clip_gradient is not None:\n                grad = clip(grad, - self.clip_gradient, self.clip_gradient)\n            grad += wd * weight\n\n            acc_g, acc_delta = state\n\n            # update g, delta\n            acc_g[:] *= self.rho\n            acc_g[:] += (1. - self.rho) * square(grad)\n            current_delta = sqrt(acc_delta + self.epsilon)\n            current_delta /= sqrt(acc_g + self.epsilon)\n            current_delta *= grad\n            acc_delta[:] *= self.rho\n            acc_delta[:] += (1. - self.rho) * square(current_delta)\n\n            # update weight\n            weight[:] -= lr * current_delta\n"
  },
  {
    "path": "python/mxnet/optimizer/adagrad.py",
    "content": "# coding: utf-8\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"AdaGrad optimizer\"\"\"\nfrom __future__ import absolute_import\nfrom ..ndarray import (zeros, clip, sqrt, square)\nfrom ..ndarray import sparse\nfrom .optimizer import Optimizer, register\n\n__all__ = ['AdaGrad']\n\n\n@register\nclass AdaGrad(Optimizer):\n    \"\"\"AdaGrad optimizer.\n\n    This class implements the AdaGrad optimizer described in *Adaptive Subgradient\n    Methods for Online Learning and Stochastic Optimization*, and available at\n    http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf.\n\n    This optimizer updates each weight by::\n\n        grad = clip(grad * rescale_grad, clip_gradient) + wd * weight\n        history += square(grad)\n        weight -= learning_rate * grad / (sqrt(history) + epsilon)\n\n    This optimizer accepts the following parameters in addition to those accepted\n    by :class:`.Optimizer`.\n\n    See Also\n    ----------\n    :meth:`mxnet.ndarray.sparse.adagrad_update`.\n\n    Parameters\n    ----------\n    learning_rate : float, default 0.01\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    epsilon : float, default 1e-6\n        Small value to avoid division by 0.\n    use_fused_step : bool, default True\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False or grad is not sparse, step is called,\n        otherwise, fused_step is called.\n\n    \"\"\"\n    def __init__(self, learning_rate=0.01, epsilon=1e-6, use_fused_step=True, **kwargs):\n        if kwargs.get(\"eps\") is not None:\n            raise DeprecationWarning(\n                'parameter \\'eps\\' is deprecated. Please use \\'epsilon\\' instead...')\n        super(AdaGrad, self).__init__(learning_rate=learning_rate,\n                                      use_fused_step=use_fused_step,\n                                      **kwargs)\n        self.epsilon = epsilon\n\n    def create_state(self, index, weight):\n        return zeros(weight.shape, weight.context, stype=weight.stype)  # history\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform an optimization step using gradients and states.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n\n            # preprocess grad\n            grad *= self.rescale_grad\n            if self.clip_gradient is not None:\n                grad = clip(grad, - self.clip_gradient, self.clip_gradient)\n            grad += wd * weight\n\n            # update history\n            history = state\n            history[:] += square(grad)\n            d = grad / (sqrt(history) + self.epsilon)\n\n            # update weight\n            weight[:] -= lr * d\n\n    def fused_step(self, indices, weights, grads, states):\n        \"\"\"Perform a fused optimization step using gradients and states.\n        Fused kernel is used for update.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            is_sparse = grad.stype == 'row_sparse'\n\n            if is_sparse:\n                self._update_count(index)\n                lr = self._get_lr(index)\n                wd = self._get_wd(index)\n                kwargs = {'epsilon': self.epsilon, 'rescale_grad': self.rescale_grad}\n                if self.clip_gradient:\n                    kwargs['clip_gradient'] = self.clip_gradient\n\n                history = state\n\n                # When grad is sparse, update weight with fused kernel\n                sparse.adagrad_update(weight, grad, history, out=weight, lr=lr, wd=wd, **kwargs)\n            else:\n                # When the grad is not sparse, the func step is called to update weight and state\n                self.step([index], [weight], [grad], [state])\n"
  },
  {
    "path": "python/mxnet/optimizer/adam.py",
    "content": "# coding: utf-8\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Adam optimizer.\"\"\"\nfrom __future__ import absolute_import\nimport math\nfrom ..ndarray import (zeros, clip, sqrt, square)\nfrom ..ndarray import adam_update\nfrom .optimizer import Optimizer, register\n\n__all__ = ['Adam']\n\n\n@register\nclass Adam(Optimizer):\n    \"\"\"The Adam optimizer.\n\n    This class implements the optimizer described in *Adam: A Method for\n    Stochastic Optimization*, available at http://arxiv.org/abs/1412.6980.\n\n    If the storage types of grad is ``row_sparse``, and ``lazy_update`` is True, \\\n    **lazy updates** at step t are applied by::\n\n        for row in grad.indices:\n            rescaled_grad[row] = clip(grad[row] * rescale_grad, clip_gradient) + wd * weight[row]\n            m[row] = beta1 * m[row] + (1 - beta1) * rescaled_grad[row]\n            v[row] = beta2 * v[row] + (1 - beta2) * (rescaled_grad[row]**2)\n            lr = learning_rate * sqrt(1 - beta2**t) / (1 - beta1**t)\n            w[row] = w[row] - lr * m[row] / (sqrt(v[row]) + epsilon)\n\n    The lazy update only updates the mean and var for the weights whose row_sparse\n    gradient indices appear in the current batch, rather than updating it for all indices.\n    Compared with the original update, it can provide large improvements in model training\n    throughput for some applications. However, it provides slightly different semantics than\n    the original update, and may lead to different empirical results.\n\n    Otherwise, **standard updates** at step t are applied by::\n\n        rescaled_grad = clip(grad * rescale_grad, clip_gradient) + wd * weight\n        m = beta1 * m + (1 - beta1) * rescaled_grad\n        v = beta2 * v + (1 - beta2) * (rescaled_grad**2)\n        lr = learning_rate * sqrt(1 - beta2**t) / (1 - beta1**t)\n        w = w - lr * m / (sqrt(v) + epsilon)\n\n    This optimizer accepts the following parameters in addition to those accepted\n    by :class:`.Optimizer`.\n\n    For details of the update algorithm, see :class:`~mxnet.ndarray.adam_update`.\n\n    Parameters\n    ----------\n    learning_rate : float, default 0.001\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    beta1 : float, default 0.9\n        Exponential decay rate for the first moment estimates.\n    beta2 : float, default 0.999\n        Exponential decay rate for the second moment estimates.\n    epsilon : float, default 1e-8\n        Small value to avoid division by 0.\n    lazy_update : bool, default False\n       Default is False. If True, lazy updates are applied \\\n       if the storage types of weight and grad are both ``row_sparse``.\n    use_fused_step : bool, default True\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False, step is called,\n        otherwise, fused_step is called.\n    \"\"\"\n    def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8,\n                 lazy_update=False, use_fused_step=True, **kwargs):\n        super(Adam, self).__init__(use_fused_step=use_fused_step,\n                                   learning_rate=learning_rate,\n                                   **kwargs)\n        if not self.use_fused_step:\n            assert not lazy_update,\\\n                'When use_fused_step is set to False, lazy_update has to be turned off.'\n        self.lazy_update = lazy_update\n        self.beta1 = beta1\n        self.beta2 = beta2\n        self.epsilon = epsilon\n        self.lazy_update = lazy_update\n\n    def create_state(self, index, weight):\n        stype = weight.stype if self.lazy_update else 'default'\n        return (zeros(weight.shape, weight.context, dtype=weight.dtype,\n                      stype=stype),  # mean\n                zeros(weight.shape, weight.context, dtype=weight.dtype,\n                      stype=stype))  # variance\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform an optimization step using gradients and states.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n            t = self._index_update_count[index]\n\n            # preprocess grad\n            grad *= self.rescale_grad\n            if self.clip_gradient is not None:\n                grad = clip(grad, - self.clip_gradient, self.clip_gradient)\n            grad += wd * weight\n\n            coef1 = 1. - self.beta1**t\n            coef2 = 1. - self.beta2**t\n            lr *= math.sqrt(coef2) / coef1\n\n            # update mean and var\n            mean, var = state\n            mean[:] *= self.beta1\n            mean[:] += (1. - self.beta1) * grad\n            var[:] *= self.beta2\n            var[:] += (1. - self.beta2) * square(grad)\n\n            # update weight\n            d = mean / (sqrt(var) + self.epsilon)\n            weight[:] -= lr * d\n\n    def fused_step(self, indices, weights, grads, states):\n        \"\"\"Perform a fused optimization step using gradients and states.\n        Fused kernel is used for update.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n            t = self._index_update_count[index]\n\n            coef1 = 1. - self.beta1**t\n            coef2 = 1. - self.beta2**t\n\n            lr *= math.sqrt(coef2)/coef1\n\n            kwargs = {'beta1': self.beta1, 'beta2': self.beta2, 'epsilon': self.epsilon,\n                      'rescale_grad': self.rescale_grad}\n            if self.clip_gradient:\n                kwargs['clip_gradient'] = self.clip_gradient\n\n            mean, var = state\n\n            # update weight with fused kernel\n            adam_update(weight, grad, mean, var, out=weight,\n                        lazy_update=self.lazy_update, lr=lr, wd=wd, **kwargs)\n"
  },
  {
    "path": "python/mxnet/optimizer/adamW.py",
    "content": "# coding: utf-8\r\n# Licensed to the Apache Software Foundation (ASF) under one\r\n# or more contributor license agreements.  See the NOTICE file\r\n# distributed with this work for additional information\r\n# regarding copyright ownership.  The ASF licenses this file\r\n# to you under the Apache License, Version 2.0 (the\r\n# \"License\"); you may not use this file except in compliance\r\n# with the License.  You may obtain a copy of the License at\r\n#\r\n#   http://www.apache.org/licenses/LICENSE-2.0\r\n#\r\n# Unless required by applicable law or agreed to in writing,\r\n# software distributed under the License is distributed on an\r\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\r\n# KIND, either express or implied.  See the License for the\r\n# specific language governing permissions and limitations\r\n# under the License.\r\n\"\"\"AdamW optimizer.\"\"\"\r\nimport math\r\nimport os\r\nimport numpy as np\r\nfrom .optimizer import Optimizer, register\r\nfrom ..ndarray import (zeros, clip, sqrt, square, full, NDArray)\r\nfrom ..ndarray.contrib import mp_adamw_update, adamw_update,\\\r\n    multi_mp_adamw_update, multi_adamw_update\r\n\r\n\r\n__all__ = ['AdamW']\r\n\r\n\r\n@register\r\nclass AdamW(Optimizer):\r\n    \"\"\"The AdamW optimizer.\r\n\r\n    This class implements the optimizer described in *Decoupled Weight Decay Regularization*,\r\n     available at https://arxiv.org/pdf/1711.05101.pdf.\r\n\r\n    Updates are applied by::\r\n\r\n        grad = clip(grad * rescale_grad, clip_gradient)\r\n        m = beta1 * m + (1 - beta1) * grad\r\n        v = beta2 * v + (1 - beta2) * (grad**2)\r\n        lr = learning_rate * sqrt(1 - beta2**t) / (1 - beta1**t)\r\n        w = w - lr * (m / (sqrt(v) + epsilon) + wd * w)\r\n\r\n\r\n    Also, we can turn off the bias correction term and the updates are as follows::\r\n\r\n        grad = clip(grad * rescale_grad, clip_gradient) + wd * weight\r\n        m = beta1 * m + (1 - beta1) * grad\r\n        v = beta2 * v + (1 - beta2) * (grad**2)\r\n        lr = learning_rate\r\n        w = w - lr * (m / (sqrt(v) + epsilon) + wd * w)\r\n\r\n    This optimizer accepts the following parameters in addition to those accepted\r\n    by :class:`.Optimizer`.\r\n\r\n\r\n    Parameters\r\n    ----------\r\n    learning_rate : float, default 0.001\r\n        The initial learning rate. If None, the optimization will use the\r\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\r\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\r\n        is also None, then it will be set to 0.01 by default.\r\n    beta1 : float, default 0.9\r\n        Exponential decay rate for the first moment estimates.\r\n    beta2 : float, default 0.999\r\n        Exponential decay rate for the second moment estimates.\r\n    epsilon : float, default 1e-6\r\n        Small value to avoid division by 0.\r\n    correct_bias : bool, default True\r\n       Can be set to False to avoid correcting bias in Adam (e.g. like in Bert TF repository).\r\n       Default True.\r\n    use_fused_step : bool, default True\r\n        Whether or not to use fused kernels for optimizer.\r\n        When use_fused_step=False, step is called,\r\n        otherwise, fused_step is called.\r\n    \"\"\"\r\n    def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-6,\r\n                 correct_bias=True, use_fused_step=True, **kwargs):\r\n        super().__init__(use_fused_step=use_fused_step,\r\n                         learning_rate=learning_rate,\r\n                         **kwargs)\r\n        self.beta1 = beta1\r\n        self.beta2 = beta2\r\n        self.epsilon = epsilon\r\n        self.correct_bias = correct_bias\r\n        self.aggregate_num = max(1, min(50,\r\n                                        int(os.getenv('MXNET_OPTIMIZER_AGGREGATION_SIZE', '4'))))\r\n\r\n    def create_state(self, index, weight):\r\n        \"\"\"state creation function.\"\"\"\r\n        return (zeros(weight.shape, weight.context, dtype=weight.dtype),  # mean\r\n                zeros(weight.shape, weight.context, dtype=weight.dtype))  # variance\r\n\r\n    def step(self, indices, weights, grads, states):\r\n        \"\"\"Perform an optimization step using gradients and states.\r\n\r\n        Parameters\r\n        ----------\r\n        indices : list of int\r\n            List of unique indices of the parameters into the individual learning rates\r\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\r\n            and `set_wd_mult()`, respectively.\r\n        weights : list of NDArray\r\n            List of parameters to be updated.\r\n        grads : list of NDArray\r\n            List of gradients of the objective with respect to this parameter.\r\n        states : List of any obj\r\n            List of state returned by `create_state()`.\r\n        \"\"\"\r\n        for index, weight, grad, state in zip(indices, weights, grads, states):\r\n            self._update_count(index)\r\n            lr = self._get_lr(index)\r\n            wd = self._get_wd(index)\r\n            t = self._index_update_count[index]\r\n\r\n            # preprocess grad\r\n            grad *= self.rescale_grad\r\n            if self.clip_gradient is not None:\r\n                grad = clip(grad, - self.clip_gradient, self.clip_gradient)\r\n            if self.correct_bias:\r\n                coef1 = 1. - self.beta1**t\r\n                coef2 = 1. - self.beta2**t\r\n                lr *= math.sqrt(coef2) / coef1\r\n\r\n            # update mean and var\r\n            mean, var = state\r\n            mean[:] *= self.beta1\r\n            mean[:] += (1. - self.beta1) * grad\r\n            var[:] *= self.beta2\r\n            var[:] += (1. - self.beta2) * square(grad)\r\n\r\n            # update weight\r\n            d = mean / (sqrt(var) + self.epsilon)\r\n            weight[:] -= lr * d\r\n            # add wd\r\n            if wd > 0:\r\n                weight[:] -= lr * wd * weight\r\n\r\n    def fused_step(self, indices, weights, grads, states):\r\n        \"\"\"Perform a fused optimization step using gradients and states.\r\n        Fused kernel is used for update.\r\n\r\n        Parameters\r\n        ----------\r\n        indices : list of int\r\n            List of unique indices of the parameters into the individual learning rates\r\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\r\n            and `set_wd_mult()`, respectively.\r\n        weights : list of NDArray\r\n            List of parameters to be updated.\r\n        grads : list of NDArray\r\n            List of gradients of the objective with respect to this parameter.\r\n        states : List of any obj\r\n            List of state returned by `create_state()`.\r\n        \"\"\"\r\n        multi_precision = self.multi_precision and weights[0].dtype == np.float16\r\n        aggregate = self.aggregate_num > 1\r\n        if not isinstance(indices, (tuple, list)):\r\n            indices = [indices]\r\n            weights = [weights]\r\n            grads = [grads]\r\n            states = [states]\r\n        for w_i, g_i in zip(weights, grads):\r\n            assert(isinstance(w_i, NDArray))\r\n            assert(isinstance(g_i, NDArray))\r\n            aggregate = (aggregate and\r\n                         w_i.stype == 'default' and\r\n                         g_i.stype == 'default')\r\n        self._update_count(indices)\r\n        lrs = self._get_lrs(indices)\r\n        wds = self._get_wds(indices)\r\n        if self.correct_bias:\r\n            new_lrs = []\r\n            for idx, lr in zip(indices, lrs):\r\n                t = self._index_update_count[idx]\r\n                coef1 = 1. - self.beta1 ** t\r\n                coef2 = 1. - self.beta2 ** t\r\n                new_lrs.append(lr * math.sqrt(coef2) / coef1)\r\n            lrs = new_lrs\r\n        if not isinstance(self.rescale_grad, NDArray):\r\n            self.rescale_grad = full(shape=(1,), val=self.rescale_grad, ctx=weights[0].context)\r\n        else:\r\n            self.rescale_grad = self.rescale_grad.as_in_context(weights[0].context)\r\n        kwargs = {'beta1': self.beta1, 'beta2': self.beta2, 'epsilon': self.epsilon,\r\n                  'rescale_grad': self.rescale_grad}\r\n        if self.clip_gradient:\r\n            kwargs['clip_gradient'] = self.clip_gradient\r\n\r\n        if aggregate:\r\n            current_index = 0\r\n            while current_index < len(indices):\r\n                sidx = current_index\r\n                eidx = min(current_index + self.aggregate_num, len(indices))\r\n                if not multi_precision:\r\n                    mean, var = list(zip(*states[sidx:eidx]))\r\n                    multi_adamw_update(weights[sidx:eidx],\r\n                                       grads[sidx:eidx],\r\n                                       mean, var,\r\n                                       out=weights[sidx:eidx],\r\n                                       size=len(weights[sidx:eidx]),\r\n                                       lrs=list(np.ones(len(weights[sidx:eidx]))),\r\n                                       wds=wds[sidx:eidx],\r\n                                       etas=lrs[sidx:eidx],\r\n                                       **kwargs)\r\n                else:\r\n                    mean_var = list(zip(*states[sidx:eidx]))[0]\r\n                    tmean_var = list(zip(*mean_var))\r\n                    mean = tmean_var[0]\r\n                    var = tmean_var[1]\r\n                    multi_mp_adamw_update(weights[sidx:eidx],\r\n                                          grads[sidx:eidx],\r\n                                          mean, var,\r\n                                          list(zip(*states[sidx:eidx]))[1],\r\n                                          out=weights[sidx:eidx],\r\n                                          size=len(weights[sidx:eidx]),\r\n                                          lrs=list(np.ones(len(weights[sidx:eidx]))),\r\n                                          wds=wds[sidx:eidx],\r\n                                          etas=lrs[sidx:eidx],\r\n                                          **kwargs)\r\n                current_index += self.aggregate_num\r\n        else:\r\n            for w_i, g_i, s_i, lr, wd in zip(weights, grads, states, lrs, wds):\r\n                if not multi_precision:\r\n                    mean, var = s_i\r\n                    adamw_update(w_i, g_i, mean, var, out=w_i,\r\n                                 lr=1, wd=wd, eta=lr, **kwargs)\r\n                else:\r\n                    mean, var = s_i[0]\r\n                    mp_adamw_update(w_i, g_i, mean, var, s_i[1], out=w_i,\r\n                                    lr=1, wd=wd, eta=lr, **kwargs)\r\n"
  },
  {
    "path": "python/mxnet/optimizer/adamax.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=W0223\n\"\"\"Adamax optimizer.\"\"\"\nfrom __future__ import absolute_import\nfrom ..ndarray import (zeros, clip, maximum, abs as NDabs)\nfrom .optimizer import Optimizer, register\n\n__all__ = ['Adamax']\n\n\n# pylint: enable=line-too-long\n@register\nclass Adamax(Optimizer):\n    \"\"\"The AdaMax optimizer.\n\n    It is a variant of Adam based on the infinity norm\n    available at http://arxiv.org/abs/1412.6980 Section 7.\n\n    The optimizer updates the weight by::\n\n        grad = clip(grad * rescale_grad, clip_gradient) + wd * weight\n        m = beta1 * m_t + (1 - beta1) * grad\n        u = maximum(beta2 * u, abs(grad))\n        weight -= lr / (1 - beta1**t) * m / (u + epsilon)\n\n    This optimizer accepts the following parameters in addition to those accepted\n    by :class:`.Optimizer`.\n\n    Parameters\n    ----------\n    learning_rate : float, default 0.002\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    beta1 : float, default 0.9\n        Exponential decay rate for the first moment estimates.\n    beta2 : float, default 0.999\n        Exponential decay rate for the second moment estimates.\n    use_fused_step : bool, default False\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False, step is called,\n        otherwise, fused_step is called.\n    \"\"\"\n    def __init__(self, learning_rate=0.002, beta1=0.9, beta2=0.999, epsilon=1e-8,\n                 use_fused_step=False, **kwargs):\n        super(Adamax, self).__init__(learning_rate=learning_rate,\n                                     use_fused_step=use_fused_step,\n                                     **kwargs)\n        self.beta1 = beta1\n        self.beta2 = beta2\n        self.epsilon = epsilon\n\n    def create_state(self, index, weight):\n        return (zeros(weight.shape, weight.context, dtype=weight.dtype),  # mean\n                zeros(weight.shape, weight.context, dtype=weight.dtype))  # variance\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform an optimization step using gradients and states.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n            t = self._index_update_count[index]\n\n            lr /= (1. - self.beta1**t)\n\n            # preprocess grad\n            grad *= self.rescale_grad\n            if self.clip_gradient is not None:\n                grad = clip(grad, -self.clip_gradient, self.clip_gradient)\n            grad += wd * weight\n\n            # update mean and var\n            mean, var = state\n            mean[:] *= self.beta1\n            mean[:] += (1. - self.beta1) * grad\n            var[:] = maximum(self.beta2 * var, NDabs(grad))\n\n            # update weight\n            d = mean / (var + self.epsilon)\n            weight[:] -= lr * d\n"
  },
  {
    "path": "python/mxnet/optimizer/contrib.py",
    "content": "# coding: utf-8\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Contrib optimizers.\"\"\"\nfrom ..ndarray import (clip, contrib, mean, sqrt, square, zeros)\nfrom .optimizer import Optimizer, register\n\n__all__ = ['GroupAdaGrad']\n\n\n@register\nclass GroupAdaGrad(Optimizer):\n    \"\"\"Adagrad optimizer with row-wise learning rates.\n\n    This class implements the AdaGrad optimizer described in *Adaptive\n    Subgradient Methods for Online Learning and Stochastic Optimization*, and\n    available at http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf but\n    uses only a single learning rate for every row of the parameter array.\n\n    This optimizer updates each weight by::\n\n        grad = clip(grad * rescale_grad, clip_gradient)\n        history += mean(square(grad), axis=1, keepdims=True)\n        weight -= lr * grad / (sqrt(history) + epsilon)\n\n    Weights are updated lazily if the gradient is sparse.\n\n    For details of the update algorithm see\n    :class:`~mxnet.ndarray.contrib.group_adagrad_update`.\n\n    This optimizer accepts the following parameters in addition to those\n    accepted by :class:`.Optimizer`. Weight decay is not supported.\n\n    Parameters\n    ----------\n    learning_rate : float, default 0.01\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    epsilon : float, default 1e-6\n        Small value to avoid division by 0.\n    use_fused_step : bool, default True\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False or grad is not sparse, step is called,\n        otherwise, fused_step is called.\n    \"\"\"\n\n    def __init__(self, learning_rate=0.01, epsilon=1e-6, use_fused_step=True, **kwargs):\n        super(GroupAdaGrad, self).__init__(learning_rate=learning_rate,\n                                           use_fused_step=use_fused_step,\n                                           **kwargs)\n        self.epsilon = epsilon\n\n    def create_state(self, index, weight):\n        assert len(weight.shape) == 2\n        history = zeros(\n            (weight.shape[0], 1), weight.context, stype=weight.stype)\n        return history\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform an optimization step using gradients and states.\n\n         Parameters\n         ----------\n         indices : list of int\n             List of unique indices of the parameters into the individual learning rates\n             and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n             and `set_wd_mult()`, respectively.\n         weights : list of NDArray\n             List of parameters to be updated.\n         grads : list of NDArray\n             List of gradients of the objective with respect to this parameter.\n         states : List of any obj\n             List of state returned by `create_state()`.\n         \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n            assert wd == 0, 'Weight decay is not supported for GroupAdaGrad'\n\n            # preprocess grad\n            grad = grad * self.rescale_grad\n            if self.clip_gradient is not None:\n                grad = clip(grad, -self.clip_gradient, self.clip_gradient)\n\n            # update history\n            history = state\n            history[:] += mean(square(grad), axis=1, keepdims=True)\n\n            # update weight\n            d = grad / (sqrt(history) + self.epsilon)\n            weight[:] -= lr * d\n\n    def fused_step(self, indices, weights, grads, states):\n        \"\"\"Perform a fused optimization step using gradients and states.\n        Fused kernel is used for update.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            is_sparse = grad.stype == 'row_sparse'\n\n            if is_sparse:\n                self._update_count(index)\n                lr = self._get_lr(index)\n                wd = self._get_wd(index)\n                assert wd == 0, 'Weight decay is not supported for GroupAdaGrad'\n\n                kwargs = {'epsilon': self.epsilon, 'rescale_grad': self.rescale_grad}\n                if self.clip_gradient:\n                    kwargs['clip_gradient'] = self.clip_gradient\n\n                history = state\n\n                # When grad is sparse, update weight with fused kernel\n                contrib.group_adagrad_update(\n                    weight,\n                    grad,\n                    history,\n                    out=weight,\n                    lr=lr,\n                    **kwargs)\n            else:\n                # When the grad is not sparse, the func step is called to update weight and state\n                self.step([index], [weight], [grad], [state])\n"
  },
  {
    "path": "python/mxnet/optimizer/dcasgd.py",
    "content": "# coding: utf-8\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=W0223\n\"\"\"DCASGD optimizer.\"\"\"\nfrom __future__ import absolute_import\nfrom ..ndarray import (zeros, clip, square)\nfrom .optimizer import Optimizer, register\n\n__all__ = ['DCASGD']\n\n\n@register\nclass DCASGD(Optimizer):\n    \"\"\"The DCASGD optimizer.\n\n    This class implements the optimizer described in *Asynchronous Stochastic Gradient Descent\n    with Delay Compensation for Distributed Deep Learning*,\n    available at https://arxiv.org/abs/1609.08326.\n\n    This optimizer accepts the following parameters in addition to those accepted\n    by :class:`.Optimizer`.\n\n    Parameters\n    ----------\n    learning_rate : float, default 0.1\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    momentum : float, optional\n       The momentum value.\n    lamda : float, optional\n       Scale DC value.\n    use_fused_step : bool, default False\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False, step is called,\n        otherwise, fused_step is called.\n    \"\"\"\n    def __init__(self, learning_rate=0.1, momentum=0.0, lamda=0.04,\n                 use_fused_step=False, **kwargs):\n        super(DCASGD, self).__init__(learning_rate=learning_rate,\n                                     use_fused_step=use_fused_step,\n                                     **kwargs)\n        self.momentum = momentum\n        self.weight_previous = {}\n        self.lamda = lamda\n\n    def create_state(self, index, weight):\n        if self.momentum == 0.0:\n            return None, weight.copy()  # previous weight\n        else:\n            return (zeros(weight.shape, weight.context, dtype=weight.dtype),  # momentum\n                    weight.copy())  # previous weight\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform an optimization step using gradients and states.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n\n            # preprocess grad\n            grad *= self.rescale_grad\n            if self.clip_gradient is not None:\n                grad = clip(grad, -self.clip_gradient, self.clip_gradient)\n            grad += wd * weight\n\n            # update mom, previous_weight\n            mom, previous_weight = state\n\n            d = square(grad)\n            d *= weight - previous_weight\n            d *= self.lamda\n            d += grad\n\n            if mom is not None:\n                mom[:] *= self.momentum\n                mom[:] -= lr * d\n            else:\n                assert (self.momentum == 0.0)\n                mom = d\n                mom *= -lr\n            previous_weight[:] = weight\n\n            # update weight\n            weight[:] += mom\n"
  },
  {
    "path": "python/mxnet/optimizer/ftml.py",
    "content": "# coding: utf-8\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"FTML optimizer.\"\"\"\nfrom __future__ import absolute_import\nfrom ..ndarray import (zeros, clip, sqrt, square)\nfrom ..ndarray import ftml_update\nfrom .optimizer import Optimizer, register\n\n__all__ = ['FTML']\n\n\n@register\nclass FTML(Optimizer):\n    \"\"\"The FTML optimizer.\n\n    This class implements the optimizer described in\n    *FTML - Follow the Moving Leader in Deep Learning*,\n    available at http://proceedings.mlr.press/v70/zheng17a/zheng17a.pdf.\n\n    Denote time step by t. The optimizer updates the weight by::\n\n        rescaled_grad = clip(grad * rescale_grad, clip_gradient) + wd * weight\n        v = beta2 * v + (1 - beta2) * square(rescaled_grad)\n        d_t = (1 - power(beta1, t)) / lr * (square_root(v / (1 - power(beta2, t))) + epsilon)\n        z = beta1 * z + (1 - beta1) * rescaled_grad - (d_t - beta1 * d_(t-1)) * weight\n        weight = - z / d_t\n\n    For details of the update algorithm, see :class:`~mxnet.ndarray.ftml_update`.\n\n    This optimizer accepts the following parameters in addition to those accepted\n    by :class:`.Optimizer`.\n\n    Parameters\n    ----------\n    learning_rate : float, default 0.0025\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    beta1 : float, default 0.6\n        0 < beta1 < 1. Generally close to 0.5.\n    beta2 : float, default 0.999\n        0 < beta2 < 1. Generally close to 1.\n    epsilon : float, default 1e-8\n        Small value to avoid division by 0.\n    use_fused_step : bool, default True\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False, step is called,\n        otherwise, fused_step is called.\n    \"\"\"\n    def __init__(self, learning_rate=0.0025, beta1=0.6, beta2=0.999, epsilon=1e-8,\n                 use_fused_step=True, **kwargs):\n        super(FTML, self).__init__(learning_rate=learning_rate,\n                                   use_fused_step=use_fused_step,\n                                   **kwargs)\n        self.beta1 = beta1\n        self.beta2 = beta2\n        self.epsilon = epsilon\n\n    def create_state(self, index, weight):\n        return (zeros(weight.shape, weight.context, dtype=weight.dtype), # d_0\n                zeros(weight.shape, weight.context, dtype=weight.dtype), # v_0\n                zeros(weight.shape, weight.context, dtype=weight.dtype)) # z_0\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform an optimization step using gradients and states.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n            t = self._index_update_count[index]\n\n            # preprocess grad\n            grad *= self.rescale_grad\n            if self.clip_gradient is not None:\n                grad = clip(grad, - self.clip_gradient, self.clip_gradient)\n            grad += wd * weight\n\n            coef1 = 1. - self.beta1**t\n            coef2 = 1. - self.beta2**t\n\n            # update d, v, z\n            d, v, z = state\n\n            v[:] *= self.beta2\n            v[:] += (1. - self.beta2) * square(grad)\n            sigma = - self.beta1 * d\n            d[:] = sqrt(v / coef2) + self.epsilon\n            d[:] *= coef1 / lr\n            sigma += d\n            z[:] *= self.beta1\n            z[:] += (1. - self.beta1) * grad\n            z[:] -= sigma * weight\n\n            # update weight\n            weight[:] = - z / d\n\n    def fused_step(self, indices, weights, grads, states):\n        \"\"\"Perform a fused optimization step using gradients and states.\n        Fused kernel is used for update.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n            t = self._index_update_count[index]\n\n            kwargs = {'beta1': self.beta1, 'beta2': self.beta2, 'epsilon': self.epsilon,\n                      'rescale_grad': self.rescale_grad, 't': t}\n            if self.clip_gradient:\n                kwargs['clip_grad'] = self.clip_gradient\n\n            d, v, z = state\n\n            # update weight with fused kernel\n            ftml_update(weight, grad, d, v, z, out=weight, lr=lr, wd=wd, **kwargs)\n"
  },
  {
    "path": "python/mxnet/optimizer/ftrl.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"FTRL optimizer.\"\"\"\nfrom __future__ import absolute_import\nfrom ..ndarray import (zeros, clip, sqrt, square, sign, maximum, abs as NDabs)\nfrom ..ndarray import ftrl_update\nfrom .optimizer import Optimizer, register\n\n__all__ = ['Ftrl']\n\n\n#pylint: disable=invalid-name\n#pylint: disable=line-too-long\n@register\nclass Ftrl(Optimizer):\n    \"\"\"The Ftrl optimizer.\n\n    Referenced from *Ad Click Prediction: a View from the Trenches*, available at\n    http://dl.acm.org/citation.cfm?id=2488200.\n\n    eta :\n        .. math::\n           \\\\eta_{t,i} = \\\\frac{learningrate}{\\\\beta+\\\\sqrt{\\\\sum_{s=1}^tg_{s,i}^2}}\n\n    The optimizer updates the weight by::\n\n        rescaled_grad = clip(grad * rescale_grad, clip_gradient)\n        z += rescaled_grad - (sqrt(n + rescaled_grad**2) - sqrt(n)) * weight / learning_rate\n        n += rescaled_grad**2\n        w = (sign(z) * lamda1 - z) / ((beta + sqrt(n)) / learning_rate + wd) * (abs(z) > lamda1)\n\n    If the storage types of weight, state and grad are all ``row_sparse``, \\\n    **sparse updates** are applied by::\n\n        for row in grad.indices:\n            rescaled_grad[row] = clip(grad[row] * rescale_grad, clip_gradient)\n            z[row] += rescaled_grad[row] - (sqrt(n[row] + rescaled_grad[row]**2) - sqrt(n[row])) * weight[row] / learning_rate\n            n[row] += rescaled_grad[row]**2\n            w[row] = (sign(z[row]) * lamda1 - z[row]) / ((beta + sqrt(n[row])) / learning_rate + wd) * (abs(z[row]) > lamda1)\n\n    The sparse update only updates the z and n for the weights whose row_sparse\n    gradient indices appear in the current batch, rather than updating it for all\n    indices. Compared with the original update, it can provide large\n    improvements in model training throughput for some applications. However, it\n    provides slightly different semantics than the original update, and\n    may lead to different empirical results.\n\n    For details of the update algorithm, see :class:`~mxnet.ndarray.ftrl_update`.\n\n    This optimizer accepts the following parameters in addition to those accepted\n    by :class:`.Optimizer`.\n\n    Parameters\n    ----------\n    learning_rate : float, default 0.1\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    lamda1 : float, default 0.01\n        L1 regularization coefficient.\n    beta : float, default 1.0\n        Per-coordinate learning rate correlation parameter.\n    use_fused_step : bool, default True\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False, step is called,\n        otherwise, fused_step is called.\n    \"\"\"\n\n    def __init__(self, learning_rate=0.1, lamda1=0.01, beta=1.,\n                 use_fused_step=True, **kwargs):\n        super(Ftrl, self).__init__(learning_rate=learning_rate,\n                                   use_fused_step=use_fused_step,\n                                   **kwargs)\n        self.lamda1 = lamda1\n        self.beta = beta\n\n    def create_state(self, index, weight):\n        return (zeros(weight.shape, weight.context, stype=weight.stype),  # z\n                zeros(weight.shape, weight.context, stype=weight.stype))  # n\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform an optimization step using gradients and states.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n\n            # preprocess grad\n            grad *= self.rescale_grad\n            if self.clip_gradient is not None:\n                grad = clip(grad, - self.clip_gradient, self.clip_gradient)\n\n            # update z, n\n            z, n = state\n\n            sigma = - sqrt(n)\n            n[:] += square(grad)\n            denom = sqrt(n)\n            sigma += denom\n            sigma /= lr\n            z[:] += grad - sigma * weight\n\n            # update weight\n            denom += self.beta\n            denom /= lr\n            denom += wd\n            d = sign(z) * maximum(NDabs(z) - self.lamda1, 0)\n            weight[:] = - d / denom\n\n    def fused_step(self, indices, weights, grads, states):\n        \"\"\"Perform a fused optimization step using gradients and states.\n        Fused kernel is used for update.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n\n            kwargs = {'lamda1': self.lamda1, 'beta': self.beta, 'rescale_grad': self.rescale_grad}\n            if self.clip_gradient:\n                kwargs['clip_gradient'] = self.clip_gradient\n\n            # update weight with fused kernel\n            z, n = state\n            ftrl_update(weight, grad, z, n, out=weight, lr=lr, wd=wd, **kwargs)\n"
  },
  {
    "path": "python/mxnet/optimizer/lamb.py",
    "content": "# coding: utf-8\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Lamb optimizer.\"\"\"\nfrom __future__ import absolute_import\nimport numpy\nfrom ..ndarray import (zeros, clip, sqrt, where, square, ones_like,\n                       maximum, minimum)\nfrom ..ndarray import (lamb_update_phase1, lamb_update_phase2,\n                       mp_lamb_update_phase1, mp_lamb_update_phase2)\nfrom ..ndarray.contrib import (multi_lamb_update, multi_mp_lamb_update)\nfrom .optimizer import Optimizer, register\n\n__all__ = ['LAMB']\n\n\n@register\nclass LAMB(Optimizer):\n    \"\"\"LAMB Optimizer.\n\n    Referenced from 'Large Batch Optimization for Deep Learning: Training BERT in 76 minutes'\n    (https://arxiv.org/pdf/1904.00962.pdf)\n\n    Parameters\n    ----------\n    learning_rate : float, default 0.001\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    beta1 : float, default 0.9\n        Exponential decay rate for the first moment estimates.\n    beta2 : float, default 0.999\n        Exponential decay rate for the second moment estimates.\n    epsilon : float, default 1e-6\n        Small value to avoid division by 0.\n    lower_bound : float, default None\n        Lower limit of norm of weight\n    upper_bound : float, default None\n        Upper limit of norm of weight\n    bias_correction : bool, default True\n        Whether or not to apply bias correction\n    aggregate_num : int, default 4\n        Number of weights to be aggregated in a list.\n        They are passed to the optimizer for a single optimization step.\n        In default, all the weights are aggregated.\n    use_fused_step : bool, default True\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False, step is called,\n        otherwise, fused_step is called.\n    \"\"\"\n    def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-6,\n                 lower_bound=None, upper_bound=None, bias_correction=True,\n                 aggregate_num=4, use_fused_step=True, **kwargs):\n        assert aggregate_num <= 45,\\\n            'When use_fused_step is True, LAMB only supports aggregate_num <= 45,' \\\n            ' and receives {}'.format(aggregate_num)\n        super(LAMB, self).__init__(learning_rate=learning_rate,\n                                   aggregate_num=aggregate_num,\n                                   use_fused_step=use_fused_step,\n                                   **kwargs)\n        self.beta1 = beta1\n        self.beta2 = beta2\n        self.epsilon = epsilon\n        self.lower_bound = lower_bound\n        self.upper_bound = upper_bound\n        self.bias_correction = bias_correction\n\n    def create_state(self, index, weight):\n        stype = weight.stype\n        return (zeros(weight.shape, weight.context, dtype=numpy.float32, stype=stype),  # mean\n                zeros(weight.shape, weight.context, dtype=numpy.float32, stype=stype))  # var\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform a fused optimization step using gradients and states.\n        Fused kernel is used for update.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n            t = self._index_update_count[index]\n\n            # preprocess grad\n            grad *= self.rescale_grad\n            if self.clip_gradient is not None:\n                grad = clip(grad, -self.clip_gradient, self.clip_gradient)\n\n            # update mean, var\n            mean, var = state\n            mean[:] *= self.beta1\n            mean[:] += (1. - self.beta1) * grad\n            var[:] *= self.beta2\n            var[:] += (1. - self.beta2) * square(grad)\n\n            r1 = weight.norm()\n            if self.lower_bound is not None:\n                r1 = maximum(r1, self.lower_bound)\n            if self.upper_bound is not None:\n                r1 = minimum(r1, self.upper_bound)\n\n            if self.bias_correction:\n                # apply bias correction\n                coef1 = 1. - self.beta1**t\n                coef2 = 1. - self.beta2**t\n                mean_hat = mean / coef1\n                var_hat = var / coef2\n                sqrt(var_hat, out=var_hat)\n                var_hat += self.epsilon\n                mean_hat /= var_hat\n                mean_hat += wd * weight\n            else:\n                mean_hat = sqrt(var)\n                mean_hat += self.epsilon\n                mean_hat[:] = mean / mean_hat\n                mean_hat += wd * weight\n\n            g = mean_hat\n            r2 = g.norm()\n\n            # calculate lamb_trust_ratio\n            ratio = r1 / r2\n            # becomes NaN if ratio == NaN or 0, otherwise 0\n            nan_or_zero = 1 - ratio / ratio\n            r = where(nan_or_zero, ones_like(ratio), ratio)\n            lr *= r\n\n            # update weight\n            g *= lr\n            weight[:] -= g\n\n    def fused_step(self, indices, weights, grads, states):\n        \"\"\"Perform a fused optimization step using gradients and states.\n        Fused kernel is used for update.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        aggregate = self.aggregate_num > 1\n        for weight, grad in zip(weights, grads):\n            aggregate = (aggregate and\n                         weight.stype == 'default' and\n                         grad.stype == 'default')\n        self._update_count(indices)\n        lrs = self._get_lrs(indices)\n        wds = self._get_wds(indices)\n\n        if aggregate:\n            kwargs = {'beta1': self.beta1, 'beta2': self.beta2, 'epsilon': self.epsilon,\n                      'bias_correction': self.bias_correction,\n                      'rescale_grad': self.rescale_grad}\n            if self.clip_gradient:\n                kwargs['clip_gradient'] = self.clip_gradient\n            if self.lower_bound:\n                kwargs['lower_bound'] = self.lower_bound\n            if self.upper_bound:\n                kwargs['upper_bound'] = self.upper_bound\n\n            step_counts = []\n            for index in indices:\n                step_counts.append(self._index_update_count[index])\n\n            multi_precision = self.multi_precision and weights[0].dtype == numpy.float16\n\n            if not multi_precision:\n                mean, var = list(zip(*states))\n                multi_lamb_update(weights, grads, mean, var,\n                                  out=weights, step_count=step_counts,\n                                  lrs=lrs, wds=wds, **kwargs)\n            else:\n                weights32, mean_var = list(zip(*states))\n                mean, var = list(zip(*mean_var))\n                multi_mp_lamb_update(weights, grads,\n                                     mean, var, weights32,\n                                     out=weights, step_count=step_counts,\n                                     lrs=lrs, wds=wds, **kwargs)\n        else:\n            for index, weight, grad, state in zip(indices, weights, grads, states):\n                self._update_count(index)\n                lr = self._get_lr(index)\n                wd = self._get_wd(index)\n                t = self._index_update_count[index]\n                kwargs = {'beta1': self.beta1, 'beta2': self.beta2, 'epsilon': self.epsilon,\n                          'bias_correction': self.bias_correction,\n                          'rescale_grad': self.rescale_grad, 't': t}\n                if self.clip_gradient:\n                    kwargs['clip_gradient'] = self.clip_gradient\n\n                multi_precision = self.multi_precision and weight.dtype == numpy.float16\n\n                if multi_precision:\n                    weight32 = state[0]\n                    mean, var = state[1]\n                    g = mp_lamb_update_phase1(weight, grad, mean, var, weight32, wd=wd, **kwargs)\n\n                    kwargs = {}\n                    if self.lower_bound:\n                        kwargs['lower_bound'] = self.lower_bound\n                    if self.upper_bound:\n                        kwargs['upper_bound'] = self.upper_bound\n                    r_1 = weight32.norm()\n                    r_2 = g.norm()\n                    mp_lamb_update_phase2(weight, g, r_1, r_2, weight32, lr=lr,\n                                          out=weight, **kwargs)\n                else:\n                    mean, var = state\n                    g = lamb_update_phase1(weight, grad, mean, var, wd=wd, **kwargs)\n\n                    kwargs = {}\n                    if self.lower_bound:\n                        kwargs['lower_bound'] = self.lower_bound\n                    if self.upper_bound:\n                        kwargs['upper_bound'] = self.upper_bound\n                    r_1 = weight.norm()\n                    r_2 = g.norm()\n                    lamb_update_phase2(weight, g, r_1, r_2, lr=lr, out=weight, **kwargs)\n\n    def update_multi_precision(self, indices, weights, grads, states):\n        \"\"\"Override update_multi_precision.\n        \"\"\"\n        if self.use_fused_step:\n            self.update(indices, weights, grads, states)\n        else:\n            super(LAMB, self).update_multi_precision(indices, weights, grads, states)\n"
  },
  {
    "path": "python/mxnet/optimizer/lans.py",
    "content": "# coding: utf-8\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"LANS optimizer.\"\"\"\nfrom __future__ import absolute_import\nimport numpy\nfrom ..ndarray import (zeros, clip, sqrt, where, square, ones_like,\n                       maximum, minimum)\nfrom ..ndarray.contrib import (multi_lans_update, multi_mp_lans_update)\nfrom .optimizer import Optimizer, register\n\n__all__ = ['LANS']\n\n\n@register\nclass LANS(Optimizer):\n    \"\"\"LANS Optimizer.\n\n    Referenced from 'Accelerated Large Batch Optimization of BERT Pretraining in 54 minutes'\n    (http://arxiv.org/abs/2006.13484)\n\n    Parameters\n    ----------\n    learning_rate : float, default 0.001\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    beta1 : float, default 0.9\n        Exponential decay rate for the first moment estimates.\n    beta2 : float, default 0.999\n        Exponential decay rate for the second moment estimates.\n    epsilon : float, default 1e-6\n        Small value to avoid division by 0.\n    lower_bound : float, default None\n        Lower limit of norm of weight\n    upper_bound : float, default None\n        Upper limit of norm of weight\n    aggregate_num : int, default 4\n        Number of weights to be aggregated in a list.\n        They are passed to the optimizer for a single optimization step.\n        In default, all the weights are aggregated.\n    use_fused_step : bool, default True\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False, step is called,\n        otherwise, fused_step is called.\n    \"\"\"\n    def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-6,\n                 lower_bound=None, upper_bound=None, aggregate_num=4, use_fused_step=True,\n                 **kwargs):\n        assert aggregate_num <= 45,\\\n            'When use_fused_step is True, LAMB only supports aggregate_num <= 45,' \\\n            ' and receives {}'.format(aggregate_num)\n        super(LANS, self).__init__(learning_rate=learning_rate,\n                                   aggregate_num=aggregate_num,\n                                   use_fused_step=use_fused_step,\n                                   **kwargs)\n        self.beta1 = beta1\n        self.beta2 = beta2\n        self.epsilon = epsilon\n        self.lower_bound = lower_bound\n        self.upper_bound = upper_bound\n\n    def create_state(self, index, weight):\n        stype = weight.stype\n        return (zeros(weight.shape, weight.context, dtype=numpy.float32, stype=stype),  # mean\n                zeros(weight.shape, weight.context, dtype=numpy.float32, stype=stype))  # var\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform a fused optimization step using gradients and states.\n        Fused kernel is used for update.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n            t = self._index_update_count[index]\n\n            # preprocess grad\n            grad *= self.rescale_grad\n            grad /= grad.norm()\n            if self.clip_gradient is not None:\n                grad = clip(grad, -self.clip_gradient, self.clip_gradient)\n\n            # update mean, var\n            mean, var = state\n            mean[:] *= self.beta1\n            mean[:] += (1. - self.beta1) * grad\n            var[:] *= self.beta2\n            var[:] += (1. - self.beta2) * square(grad)\n\n            r1 = weight.norm()\n            if self.lower_bound is not None:\n                r1 = maximum(r1, self.lower_bound)\n            if self.upper_bound is not None:\n                r1 = minimum(r1, self.upper_bound)\n\n            # apply bias correction\n            coef1 = 1. - self.beta1 ** t\n            coef2 = 1. - self.beta2 ** t\n            mean_hat = mean / coef1\n            var_hat = var / coef2\n            sqrt(var_hat, out=var_hat)\n            var_hat += self.epsilon\n            mean_hat /= var_hat\n            mean_hat += wd * weight\n\n            g = mean_hat\n            r2 = g.norm()\n\n            # calculate lans_trust_ratio for first part\n            ratio_m = r1 / r2\n            # becomes NaN if ratio == NaN or 0, otherwise 0\n            nan_or_zero = 1 - ratio_m / ratio_m\n            r_m = where(nan_or_zero, ones_like(ratio_m), ratio_m)\n\n            # update weight using first part of the estimator\n            g *= lr * r_m * self.beta1\n            weight[:] -= g\n\n            # calculate the second part of the estimator\n            mean_hat = grad / var_hat\n            mean_hat += wd * weight\n\n            g = mean_hat\n            r2 = g.norm()\n\n            # calculate lans_trust_ratio for second part\n            ratio_g = r1 / r2\n            # becomes NaN if ratio == NaN or 0, otherwise 0\n            nan_or_zero = 1 - ratio_g / ratio_g\n            r_g = where(nan_or_zero, ones_like(ratio_g), ratio_g)\n\n            # update weight using second part of the estimator\n            g *= lr * r_g * (1 - self.beta1)\n            weight[:] -= g\n\n    def fused_step(self, indices, weights, grads, states):\n        \"\"\"Perform a fused optimization step using gradients and states.\n        Fused kernel is used for update.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        self._update_count(indices)\n        lrs = self._get_lrs(indices)\n        wds = self._get_wds(indices)\n\n        kwargs = {'beta1': self.beta1, 'beta2': self.beta2, 'epsilon': self.epsilon,\n                  'rescale_grad': self.rescale_grad}\n        if self.clip_gradient:\n            kwargs['clip_gradient'] = self.clip_gradient\n        if self.lower_bound:\n            kwargs['lower_bound'] = self.lower_bound\n        if self.upper_bound:\n            kwargs['upper_bound'] = self.upper_bound\n\n        step_counts = []\n        for index in indices:\n            step_counts.append(self._index_update_count[index])\n\n        multi_precision = self.multi_precision and weights[0].dtype == numpy.float16\n\n        if not multi_precision:\n            mean, var = list(zip(*states))\n            multi_lans_update(weights, grads, mean, var,\n                              out=weights, step_count=step_counts,\n                              lrs=lrs, wds=wds, **kwargs)\n        else:\n            weights32, mean_var = list(zip(*states))\n            mean, var = list(zip(*mean_var))\n            multi_mp_lans_update(weights, grads,\n                                 mean, var, weights32,\n                                 out=weights, step_count=step_counts,\n                                 lrs=lrs, wds=wds, **kwargs)\n\n    def update_multi_precision(self, indices, weights, grads, states):\n        \"\"\"Override update_multi_precision.\n        \"\"\"\n        if self.use_fused_step:\n            self.update(indices, weights, grads, states)\n        else:\n            super(LANS, self).update_multi_precision(indices, weights, grads, states)\n"
  },
  {
    "path": "python/mxnet/optimizer/lars.py",
    "content": "# coding: utf-8\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"LARS optimizer.\"\"\"\nfrom __future__ import absolute_import\nimport numpy\nfrom ..ndarray import (zeros, clip, array,\n                       multi_sum_sq, multi_lars,\n                       norm as NDnorm,\n                       where, ones_like)\nfrom ..ndarray import (sgd_update, sgd_mom_update,\n                       mp_sgd_update, mp_sgd_mom_update,\n                       preloaded_multi_sgd_update, preloaded_multi_sgd_mom_update,\n                       preloaded_multi_mp_sgd_update, preloaded_multi_mp_sgd_mom_update)\nfrom .optimizer import Optimizer, register\nfrom .utils import _flatten_list\n\n__all__ = ['LARS']\n\n\n@register\nclass LARS(Optimizer):\n    \"\"\"the LARS optimizer from 'Large Batch Training of Convolution Networks' \\\n    (https://arxiv.org/abs/1708.03888)\n\n    Behave mostly like SGD with momentum and weight decay but is scaling \\\n    adaptively the learning for each layer:\n\n    .. code-block::\n\n       w_norm = L2norm(weights)\n       g_norm = L2norm(gradients)\n       if w_norm > 0 and g_norm > 0:\n           lr_layer = lr * w_norm / (g_norm + weight_decay * w_norm + epsilon)\n       else:\n           lr_layer = lr\n\n\n    Parameters\n    ----------\n    learning_rate : float, default 0.1\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    momentum : float, default 0.\n        The momentum value.\n    eta : float, default 0.001\n        LARS coefficient used to scale the learning rate.\n    epsilon : float, default 1e-8\n        Small value to avoid division by 0.\n    lazy_update : bool, default False\n        Default is False. If True, lazy updates are applied \\\n        if the storage types of weight and grad are both ``row_sparse``.\n    aggregate_num : int, default 1\n        Number of weights to be aggregated in a list.\n        They are passed to the optimizer for a single optimization step.\n    use_fused_step : bool, default True\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False, step is called,\n        otherwise, fused_step is called.\n    \"\"\"\n    def __init__(self, learning_rate=0.1, momentum=0.0, eta=0.001,\n                 epsilon=1e-8, lazy_update=False, use_fused_step=True,\n                 aggregate_num=1, **kwargs):\n        super(LARS, self).__init__(learning_rate=learning_rate,\n                                   use_fused_step=use_fused_step,\n                                   aggregate_num=aggregate_num,\n                                   **kwargs)\n        if not self.use_fused_step:\n            assert not lazy_update,\\\n                'When use_fused_step is set to False, lazy_update has to be turned off.'\n        if lazy_update:\n            assert not self.multi_precision, \\\n                'When lazy_update is set to True, multi_precision has be turned off.'\n        self.lazy_update = lazy_update\n        self.momentum = momentum\n        self.eta = eta\n        self.epsilon = epsilon\n        self.lazy_update = lazy_update\n\n    def create_state(self, index, weight):\n        momentum = None\n        if self.momentum != 0.0:\n            stype = weight.stype if self.lazy_update else 'default'\n            momentum = zeros(weight.shape, weight.context, dtype=weight.dtype, stype=stype)\n        return momentum\n\n    def _l2norm(self, v, rescale=False):\n        \"\"\"L2 Norm implementation\"\"\"\n        v = v.astype('float32')\n        if rescale:\n            v *= self.rescale_grad\n        norm = NDnorm(v)\n        return norm\n\n    def _get_lars(self, index, weight, grad, wd):\n        \"\"\"Returns a scaling factor for the learning rate for this layer\"\"\"\n        lars = 1.0\n        name = self.idx2name[index] if index in self.idx2name else str(index)\n        if name.endswith('gamma') or name.endswith('beta') or name.endswith('bias'):\n            return lars\n\n        w_norm = self._l2norm(weight)\n        g_norm = self._l2norm(grad, rescale=True)\n\n        # calculate lars_trust_ratio\n        ratio = w_norm / g_norm\n        # becomes NaN if ratio == NaN or 0, otherwise 0\n        nan_or_zero = 1 - ratio / ratio\n        lars = self.eta * w_norm / (g_norm + wd * w_norm + self.epsilon)\n        lars = where(nan_or_zero, ones_like(lars), lars)\n\n        return lars.asscalar()\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform an optimization step using gradients and states.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n\n            # compute lars\n            # clip grad + wd * weight is performed after computing lars\n            lars = self._get_lars(index, weight, grad, wd)\n            lr *= lars\n\n            # preprocess grad\n            grad *= self.rescale_grad\n            if self.clip_gradient is not None:\n                grad = clip(grad, -self.clip_gradient, self.clip_gradient)\n            grad += wd * weight\n\n            # update mom\n            mom = state\n            if mom is not None:\n                mom[:] *= self.momentum\n                mom[:] -= lr * grad\n            else:\n                mom = -lr * grad\n\n            # update weight\n            weight[:] += mom\n\n    def fused_step(self, indices, weights, grads, states):\n        \"\"\"Perform a fused optimization step using gradients and states.\n        Fused kernel is used for update.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        aggregate = self.aggregate_num > 1\n        for weight, grad in zip(weights, grads):\n            aggregate = (aggregate and\n                         weight.stype == 'default' and\n                         grad.stype == 'default')\n        self._update_count(indices)\n        lrs = self._get_lrs(indices)\n        wds = self._get_wds(indices)\n\n        kwargs = {'rescale_grad': self.rescale_grad}\n        if self.momentum > 0:\n            kwargs['momentum'] = self.momentum\n        if self.clip_gradient is not None:\n            kwargs['clip_gradient'] = self.clip_gradient\n\n        if aggregate:\n            nb_params = len(indices)\n            names = [self.idx2name[i] if i in self.idx2name else str(i) for i in indices]\n            lars_idx = [i for i in range(nb_params) if\n                        not(names[i].endswith('gamma') or names[i].endswith('beta') or\n                            names[i].endswith('bias'))]\n            nb_lars = len(lars_idx)\n            no_lars_idx = [i for i in range(nb_params) if\n                           (names[i].endswith('gamma') or names[i].endswith('beta') or\n                            names[i].endswith('bias'))]\n            cur_ctx = weights[0].context\n            full_idx = lars_idx + no_lars_idx\n            new_lrs = array([lrs[i] for i in full_idx], ctx=cur_ctx, dtype='float32')\n            new_wds = array([wds[i] for i in full_idx], ctx=cur_ctx, dtype='float32')\n            new_weights = [weights[i] for i in full_idx]\n            new_grads = [grads[i] for i in full_idx]\n            new_states = [states[i] for i in full_idx]\n            if nb_lars > 0:\n                w_sum_sq = multi_sum_sq(*new_weights[:nb_lars], num_arrays=nb_lars)\n                g_sum_sq = multi_sum_sq(*new_grads[:nb_lars], num_arrays=nb_lars)\n                multi_lars(new_lrs[:nb_lars], w_sum_sq, g_sum_sq, new_wds[:nb_lars],\n                           eta=self.eta, eps=self.epsilon, rescale_grad=self.rescale_grad,\n                           out=new_lrs[:nb_lars])\n            # Same than usual using preloaded sgd functions\n            multi_precision = self.multi_precision and weights[0].dtype == numpy.float16\n            if not multi_precision:\n                if self.momentum > 0:\n                    preloaded_multi_sgd_mom_update(\n                        *(_flatten_list(zip(new_weights, new_grads, new_states)) +\n                          [new_lrs, new_wds]), out=new_weights, num_weights=len(new_weights),\n                        **kwargs)\n                else:\n                    preloaded_multi_sgd_update(\n                        *(_flatten_list(zip(new_weights, new_grads)) +\n                          [new_lrs, new_wds]), out=new_weights, num_weights=len(new_weights),\n                        **kwargs)\n            else:\n                states = list(zip(*states))\n                weights32, moms = states\n                if self.momentum > 0:\n                    preloaded_multi_mp_sgd_mom_update(\n                        *(_flatten_list(zip(new_weights, new_grads, moms, weights32)) +\n                          [new_lrs, new_wds]), out=new_weights, num_weights=len(new_weights),\n                        **kwargs)\n                else:\n                    preloaded_multi_mp_sgd_update(\n                        *(_flatten_list(zip(new_weights, new_grads, weights32)) +\n                          [new_lrs, new_wds]), out=new_weights, num_weights=len(new_weights),\n                        **kwargs)\n        else:\n            for i, (index, weight, grad, state) in enumerate(zip(indices, weights, grads, states)):\n                wd = wds[i]\n                lr = lrs[i]\n                lr *= self._get_lars(index, weight, grad, wd)\n                multi_precision = self.multi_precision and weights[0].dtype == numpy.float16\n                if not multi_precision:\n                    mom = state\n                    if state is not None:\n                        sgd_mom_update(weight, grad, mom, out=weight,\n                                       lazy_update=self.lazy_update, lr=lr, wd=wd, **kwargs)\n                    else:\n                        sgd_update(weight, grad, out=weight, lazy_update=self.lazy_update,\n                                   lr=lr, wd=wd, **kwargs)\n                else:\n                    weight32, mom = state\n                    if mom is not None:\n                        mp_sgd_mom_update(weight, grad, mom, weight32, out=weight,\n                                          lr=lr, wd=wd, **kwargs)\n                    else:\n                        mp_sgd_update(weight, grad, weight32, out=weight,\n                                      lr=lr, wd=wd, **kwargs)\n\n    def update_multi_precision(self, indices, weights, grads, states):\n        \"\"\"Override update_multi_precision.\n        \"\"\"\n        if self.use_fused_step:\n            self.update(indices, weights, grads, states)\n        else:\n            super(LARS, self).update_multi_precision(indices, weights, grads, states)\n"
  },
  {
    "path": "python/mxnet/optimizer/nadam.py",
    "content": "# coding: utf-8\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=W0223\n\"\"\"Nadam optimizer.\"\"\"\nfrom __future__ import absolute_import\nfrom ..ndarray import (zeros, clip, sqrt, square)\nfrom .optimizer import Optimizer, register\n\n__all__ = ['Nadam']\n\n\n@register\nclass Nadam(Optimizer):\n    \"\"\"The Nesterov Adam optimizer.\n\n    Much like Adam is essentially RMSprop with momentum,\n    Nadam is Adam RMSprop with Nesterov momentum available\n    at http://cs229.stanford.edu/proj2015/054_report.pdf.\n\n    This optimizer accepts the following parameters in addition to those accepted\n    by :class:`.Optimizer`.\n\n    Parameters\n    ----------\n    learning_rate : float, default 0.001\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    beta1 : float, default 0.9\n        Exponential decay rate for the first moment estimates.\n    beta2 : float, default 0.999\n        Exponential decay rate for the second moment estimates.\n    epsilon : float, default 1e-8\n        Small value to avoid division by 0.\n    schedule_decay : float, default 0.004\n        Exponential decay rate for the momentum schedule\n    use_fused_step : bool, default False\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False, step is called,\n        otherwise, fused_step is called.\n    \"\"\"\n    def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8,\n                 schedule_decay=0.004, use_fused_step=False, **kwargs):\n        super(Nadam, self).__init__(learning_rate=learning_rate,\n                                    use_fused_step=use_fused_step,\n                                    **kwargs)\n        self.beta1 = beta1\n        self.beta2 = beta2\n        self.epsilon = epsilon\n        self.schedule_decay = schedule_decay\n        self.m_schedule = 1.\n\n    def create_state(self, index, weight):\n        return (zeros(weight.shape, weight.context, dtype=weight.dtype),  # mean\n                zeros(weight.shape, weight.context, dtype=weight.dtype))  # variance\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform an optimization step using gradients and states.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n            t = self._index_update_count[index]\n\n            # preprocess grad\n            grad *= self.rescale_grad\n            if self.clip_gradient is not None:\n                grad = clip(grad, -self.clip_gradient, self.clip_gradient)\n            grad += wd * weight\n\n            coef2 = 1. - self.beta2**t\n\n            # warming momentum schedule\n            momentum_t = self.beta1 * (1. - 0.5 * (pow(0.96, t * self.schedule_decay)))\n            momentum_t_1 = self.beta1 * (1. - 0.5 * (pow(0.96, (t + 1) * self.schedule_decay)))\n            self.m_schedule = self.m_schedule * momentum_t\n            m_schedule_next = self.m_schedule * momentum_t_1\n\n            # update mean and var\n            mean, var = state\n            mean[:] *= self.beta1\n            mean[:] += (1. - self.beta1) * grad\n            var[:] *= self.beta2\n            var[:] += (1. - self.beta2) * square(grad)\n\n            grad_prime = grad / (1. - self.m_schedule)\n            mean_prime = mean / (1. - m_schedule_next)\n            var_prime = var / coef2\n            mean_bar = momentum_t_1 * mean_prime + (1. - momentum_t) * grad_prime\n\n            # update weight\n            d = mean_bar / (sqrt(var_prime) + self.epsilon)\n            weight[:] -= lr * d\n"
  },
  {
    "path": "python/mxnet/optimizer/nag.py",
    "content": "# coding: utf-8\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"NAG optimizer.\"\"\"\nfrom __future__ import absolute_import\nimport numpy\nfrom ..ndarray import (zeros, clip)\nfrom ..ndarray import (sgd_update, mp_sgd_update, nag_mom_update, mp_nag_mom_update)\nfrom .optimizer import Optimizer, register\n\n__all__ = ['NAG']\n\n\n@register\nclass NAG(Optimizer):\n    \"\"\"Nesterov accelerated gradient.\n\n    This optimizer updates each weight by::\n\n        grad = clip(grad * rescale_grad, clip_gradient) + wd * weight\n        state = momentum * state + lr * grad\n        weight = weight - (momentum * state + lr * grad)\n\n    Parameters\n    ----------\n    learning_rate : float, default 0.1\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    momentum : float, default 0.9\n       The momentum value.\n    multi_precision: bool, default False\n        Flag to control the internal precision of the optimizer.\n        False: results in using the same precision as the weights (default),\n        True: makes internal 32-bit copy of the weights and applies gradients\n        in 32-bit precision even if actual weights used in the model have lower precision.\n        Turning this on can improve convergence and accuracy when training with float16.\n    use_fused_step : bool, default True\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False, step is called,\n        otherwise, fused_step is called.\n    \"\"\"\n    def __init__(self, learning_rate=0.1, momentum=0.9, multi_precision=False,\n                 use_fused_step=True, **kwargs):\n        super(NAG, self).__init__(learning_rate=learning_rate,\n                                  multi_precision=multi_precision,\n                                  use_fused_step=use_fused_step,\n                                  **kwargs)\n        self.momentum = momentum\n\n    def create_state(self, index, weight):\n        momentum = None\n        if self.momentum != 0.0:\n            momentum = zeros(weight.shape, weight.context, dtype=weight.dtype)\n        return momentum\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform an optimization step using gradients and states.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n\n            # preprocess grad\n            grad *= self.rescale_grad\n            if self.clip_gradient is not None:\n                grad = clip(grad, -self.clip_gradient, self.clip_gradient)\n            grad += wd * weight\n\n            # update mom\n            mom = state\n            if mom is not None:\n                mom[:] *= self.momentum\n                mom[:] -= lr * grad\n                d = self.momentum * mom - lr * grad\n            else:\n                d = -lr * grad\n\n            # update weight\n            weight[:] += d\n\n    def fused_step(self, indices, weights, grads, states):\n        \"\"\"Perform a fused optimization step using gradients and states.\n        Fused kernel is used for update.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n\n            kwargs = {'rescale_grad': self.rescale_grad}\n            if self.momentum > 0:\n                kwargs['momentum'] = self.momentum\n            if self.clip_gradient:\n                kwargs['clip_gradient'] = self.clip_gradient\n\n            multi_precision = self.multi_precision and weight.dtype == numpy.float16\n\n            if not multi_precision:\n                mom = state\n                if mom is not None:\n                    nag_mom_update(weight, grad, mom, out=weight, lr=lr, wd=wd, **kwargs)\n                else:\n                    sgd_update(weight, grad, out=weight, lr=lr, wd=wd, **kwargs)\n            else:\n                weight32, mom = state\n                if mom is not None:\n                    mp_nag_mom_update(weight, grad, mom, weight32, out=weight,\n                                      lr=lr, wd=wd, **kwargs)\n                else:\n                    mp_sgd_update(weight, grad, weight32, out=weight,\n                                  lr=lr, wd=wd, **kwargs)\n\n    def update_multi_precision(self, indices, weights, grads, states):\n        \"\"\"Override update_multi_precision.\n        \"\"\"\n        if self.use_fused_step:\n            self.update(indices, weights, grads, states)\n        else:\n            super(NAG, self).update_multi_precision(indices, weights, grads, states)\n"
  },
  {
    "path": "python/mxnet/optimizer/optimizer.py",
    "content": "# coding: utf-8\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=too-many-lines\n\"\"\"Base Optimizer class.\"\"\"\nimport warnings\nimport numpy\nfrom ..ndarray import (NDArray, zeros, cast)\nfrom ..util import is_np_array\n\n__all__ = ['Optimizer', 'Test', 'create', 'register']\n\n\nclass Optimizer(object):\n    \"\"\"The base class inherited by all optimizers.\n\n    Parameters\n    ----------\n    rescale_grad : float, optional, default 1.0\n        Multiply the gradient with `rescale_grad` before updating. Often\n        choose to be ``1.0/batch_size``.\n\n    param_idx2name : dict from int to string, optional, default None\n        A dictionary that maps int index to string name.\n\n    clip_gradient : float, optional, default None\n        Clip the gradient by projecting onto the box ``[-clip_gradient, clip_gradient]``.\n\n    learning_rate : float, optional, default None\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n\n    lr_scheduler : LRScheduler, optional, default None\n        The learning rate scheduler.\n\n    wd : float, optional, default 0.0\n        The weight decay (or L2 regularization) coefficient. Modifies objective\n        by adding a penalty for having large weights.\n\n    sym: Symbol, optional, default None\n        The Symbol this optimizer is applying to.\n\n    begin_num_update : int, optional, default 0\n        The initial number of updates.\n\n    multi_precision : bool, optional, default False\n       Flag to control the internal precision of the optimizer.\n       False: results in using the same precision as the weights (default),\n       True: makes internal 32-bit copy of the weights and applies gradients\n       in 32-bit precision even if actual weights used in the model have lower precision.\n       Turning this on can improve convergence and accuracy when training with float16.\n\n    param_dict : dict of int -> gluon.Parameter, default None\n        Dictionary of parameter index to gluon.Parameter, used to lookup parameter attributes\n        such as lr_mult, wd_mult, etc. param_dict shall not be deep copied.\n\n    aggregate_num : int, optional, default None\n        Number of weights to be aggregated in a list.\n        They are passed to the optimizer for a single optimization step.\n        In default, only one weight is aggregated.\n        When `aggregate_num` is set to numpy.inf, all the weights are aggregated.\n\n    use_fused_step : bool, optional, default None\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False, step is called,\n        otherwise, fused_step is called.\n\n    Properties\n    ----------\n    learning_rate : float\n        The current learning rate of the optimizer. Given an Optimizer object\n        optimizer, its learning rate can be accessed as optimizer.learning_rate.\n    \"\"\"\n    def __init__(self, rescale_grad=1., param_idx2name=None, wd=0.,\n                 clip_gradient=None, learning_rate=None,\n                 lr_scheduler=None, sym=None, begin_num_update=0,\n                 multi_precision=False, param_dict=None, aggregate_num=None,\n                 use_fused_step=None, **kwargs):\n        super(Optimizer, self).__init__(**kwargs)\n        self.rescale_grad = rescale_grad\n        self.lr_scheduler = lr_scheduler\n        if self.lr_scheduler is None and learning_rate is None:\n            learning_rate = 0.01\n        self.lr = learning_rate\n        if self.lr_scheduler is not None and learning_rate is not None:\n            if self.lr_scheduler.base_lr != learning_rate:\n                print(UserWarning(\"learning rate from ``lr_scheduler`` has been \"\n                                  \"overwritten by ``learning_rate`` in optimizer.\"))\n                self.lr_scheduler.base_lr = learning_rate\n\n        self.wd = wd\n        self.lr_mult = {}\n        self.wd_mult = {}\n        self.begin_num_update = begin_num_update\n        self.num_update = begin_num_update\n        self._all_index_update_counts = {0 : {}}\n        self._index_update_count = self._all_index_update_counts[0]\n        self.clip_gradient = clip_gradient\n        self.multi_precision = multi_precision\n\n        if aggregate_num is None:\n            self.aggregate_num = 1\n        else:\n            self.aggregate_num = aggregate_num\n\n        if param_idx2name is None:\n            param_idx2name = {}\n        assert isinstance(param_idx2name, dict), \\\n            'param_idx2name should be a dict of param indexes to names.'\n        self.idx2name = param_idx2name.copy()\n        self.sym_info = (sym.attr_dict(), sym.list_arguments()) if sym is not None else ()\n        self.param_dict = param_dict if param_dict else {}\n        self.allow_np_array = is_np_array()\n        self.use_fused_step = use_fused_step \\\n            if use_fused_step is not None else False\n\n        self.set_lr_mult({})\n        self.set_wd_mult({})\n\n    opt_registry = {}\n\n    @staticmethod\n    def register(klass):\n        \"\"\"Registers a new optimizer.\n\n        Once an optimizer is registered, we can create an instance of this\n        optimizer with `create_optimizer` later.\n\n        Examples\n        --------\n\n        >>> @mx.optimizer.Optimizer.register\n        ... class MyOptimizer(mx.optimizer.Optimizer):\n        ...     pass\n        >>> optim = mx.optimizer.Optimizer.create_optimizer('MyOptimizer')\n        >>> print(type(optim))\n        <class '__main__.MyOptimizer'>\n        \"\"\"\n        assert(isinstance(klass, type))\n        name = klass.__name__.lower()\n        if name in Optimizer.opt_registry:\n            warnings.warn(f'WARNING: New optimizer {klass.__module__}.{klass.__name__} is overriding '\n                          f'existing optimizer {Optimizer.opt_registry[name].__module__}.{Optimizer.opt_registry[name].__name__}')\n        Optimizer.opt_registry[name] = klass\n        return klass\n\n    @staticmethod\n    def create_optimizer(name, **kwargs):\n        \"\"\"Instantiates an optimizer with a given name and kwargs.\n\n        .. note:: We can use the alias `create` for ``Optimizer.create_optimizer``.\n\n        Parameters\n        ----------\n        name: str\n            Name of the optimizer. Should be the name\n            of a subclass of Optimizer. Case insensitive.\n\n        kwargs: dict\n            Parameters for the optimizer.\n\n        Returns\n        -------\n        Optimizer\n            An instantiated optimizer.\n\n        Examples\n        --------\n        >>> sgd = mx.optimizer.Optimizer.create_optimizer('sgd')\n        >>> type(sgd)\n        <class 'mxnet.optimizer.SGD'>\n        >>> adam = mx.optimizer.create('adam', learning_rate=.1)\n        >>> type(adam)\n        <class 'mxnet.optimizer.Adam'>\n        \"\"\"\n        if name.lower() in Optimizer.opt_registry:\n            return Optimizer.opt_registry[name.lower()](**kwargs)\n        else:\n            raise ValueError(f'Cannot find optimizer {name}')\n\n    @property\n    def learning_rate(self):\n        if self.lr_scheduler is not None:\n            return self.lr_scheduler(self.num_update)\n        else:\n            return self.lr\n\n    def create_state(self, index, weight):\n        \"\"\"Creates auxiliary state for a given weight.\n\n        Some optimizers require additional states, e.g. as momentum, in addition\n        to gradients in order to update weights. This function creates state\n        for a given weight which will be used in `update`. This function is\n        called only once for each weight.\n\n        Parameters\n        ----------\n        index : int\n            An unique index to identify the weight.\n        weight : NDArray\n            The weight.\n\n        Returns\n        -------\n        state : any obj\n            The state associated with the weight.\n        \"\"\"\n\n    def create_state_multi_precision(self, index, weight):\n        \"\"\"Creates auxiliary state for a given weight, including FP32 high\n        precision copy if original weight is FP16.\n\n        This method is provided to perform automatic mixed precision training\n        for optimizers that do not support it themselves.\n\n        Parameters\n        ----------\n        index : int\n            An unique index to identify the weight.\n        weight : NDArray\n            The weight.\n\n        Returns\n        -------\n        state : any obj\n            The state associated with the weight.\n        \"\"\"\n        if self.multi_precision and weight.dtype == numpy.float16:\n            weight_master_copy = weight.astype(numpy.float32)\n            return (weight_master_copy,) + (self.create_state(index, weight_master_copy),)\n        if weight.dtype == numpy.float16 and not self.multi_precision:\n            warnings.warn(\"Accumulating with float16 in optimizer can lead to \"\n                          \"poor accuracy or slow convergence. \"\n                          \"Consider using multi_precision=True option of the \"\n                          \"optimizer\")\n        return self.create_state(index, weight)\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform an optimization step using gradients and states.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        raise NotImplementedError\n\n    def fused_step(self, indices, weights, grads, states):\n        \"\"\"Perform a fused optimization step using gradients and states.\n        New operators that fuses optimizer's update should be put in this function.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        raise NotImplementedError\n\n    def update(self, indices, weights, grads, states):\n        \"\"\"Call step to perform a single optimization update if use_fused_step is False,\n         otherwise fused_step is called.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for weight, grad in zip(weights, grads):\n            assert(isinstance(weight, NDArray))\n            assert(isinstance(grad, NDArray))\n        if not self.use_fused_step:\n            self.step(indices, weights, grads, states)\n        else:\n            self.fused_step(indices, weights, grads, states)\n\n    def update_multi_precision(self, indices, weights, grads, states):\n        \"\"\"Call step to perform a single optimization update if use_fused_step is False,\n         otherwise fused_step is called. Mixed precision version.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        weights_master_copy = []\n        original_states = []\n        grads32 = []\n        for weight, grad, state in zip(weights, grads, states):\n            if self.multi_precision and weight.dtype == numpy.float16:\n                weights_master_copy.append(state[0])\n                original_states.append(state[1])\n                grads32.append(grad.astype(numpy.float32))\n            else:\n                weights_master_copy.append(weight)\n                original_states.append(state)\n                grads32.append(grad)\n        self.update(indices, weights_master_copy, grads32, original_states)\n        for weight_master_copy, weight in zip(weights_master_copy, weights):\n            if self.multi_precision and weight.dtype == numpy.float16:\n                cast(weight_master_copy, dtype=weight.dtype, out=weight)\n\n    def set_learning_rate(self, lr):\n        \"\"\"Sets a new learning rate of the optimizer.\n\n        Parameters\n        ----------\n        lr : float\n            The new learning rate of the optimizer.\n        \"\"\"\n        if self.lr_scheduler is not None: # pylint: disable=no-else-raise\n            raise UserWarning(\"LRScheduler of the optimizer has already been \"\n                              \"defined. Note that set_learning_rate can mutate \"\n                              \"the value of the learning rate of the optimizer \"\n                              \"only when the LRScheduler of the optimizer is \"\n                              \"undefined.\")\n        else:\n            self.lr = lr\n\n    def set_lr_mult(self, args_lr_mult):\n        \"\"\"Sets an individual learning rate multiplier for each parameter.\n\n        If you specify a learning rate multiplier for a parameter, then\n        the learning rate for the parameter will be set as the product of\n        the global learning rate `self.lr` and its multiplier.\n\n        .. note:: The default learning rate multiplier of a `Variable`\n            can be set with `lr_mult` argument in the constructor.\n\n        Parameters\n        ----------\n        args_lr_mult : dict of str/int to float\n            For each of its key-value entries, the learning rate multipler for the\n            parameter specified in the key will be set as the given value.\n\n            You can specify the parameter with either its name or its index.\n            If you use the name, you should pass `sym` in the constructor,\n            and the name you specified in the key of `args_lr_mult` should match\n            the name of the parameter in `sym`. If you use the index, it should\n            correspond to the index of the parameter used in the `update` method.\n\n            Specifying a parameter by its index is only supported for backward\n            compatibility, and we recommend to use the name instead.\n        \"\"\"\n        self.lr_mult = {}\n        if self.sym_info:\n            attr, arg_names = self.sym_info\n            for name in arg_names:\n                if name in attr and '__lr_mult__' in attr[name]:\n                    self.lr_mult[name] = float(attr[name]['__lr_mult__'])\n        self.lr_mult.update(args_lr_mult)\n\n    def set_wd_mult(self, args_wd_mult):\n        \"\"\"Sets an individual weight decay multiplier for each parameter.\n\n        .. note:: The default weight decay multiplier for a `Variable`\n            can be set with its `wd_mult` argument in the constructor.\n\n        Parameters\n        ----------\n        args_wd_mult : dict of string/int to float\n            For each of its key-value entries, the weight decay multipler for the\n            parameter specified in the key will be set as the given value.\n\n            You can specify the parameter with either its name or its index.\n            If you use the name, you should pass `sym` in the constructor,\n            and the name you specified in the key of `args_lr_mult` should match\n            the name of the parameter in `sym`. If you use the index, it should\n            correspond to the index of the parameter used in the `update` method.\n\n            Specifying a parameter by its index is only supported for backward\n            compatibility, and we recommend to use the name instead.\n        \"\"\"\n        self.wd_mult = {}\n        if self.sym_info:\n            attr, arg_names = self.sym_info\n            for name in arg_names:\n                if name in attr and '__wd_mult__' in attr[name]:\n                    self.wd_mult[name] = float(attr[name]['__wd_mult__'])\n        self.wd_mult.update(args_wd_mult)\n\n    def _set_current_context(self, device_id):\n        \"\"\"This function has been deprecated. Please refer to ``Optimizer._set_current_context``.\"\"\"\n        warnings.warn('Optimizer._set_current_context has been renamed to'\n                      ' Optimizer._set_current_device', DeprecationWarning)\n        return self._set_current_device(device_id)\n\n    def _set_current_device(self, device_id):\n        \"\"\"Sets the number of the currently handled device.\n\n        Parameters\n        ----------\n        device_id : int\n            The number of current device.\n        \"\"\"\n        if device_id not in self._all_index_update_counts:\n            self._all_index_update_counts[device_id] = {}\n        self._index_update_count = self._all_index_update_counts[device_id]\n\n    def _update_count(self, index):\n        \"\"\"Updates num_update.\n\n        Parameters\n        ----------\n        index : int or list of int\n            The index to be updated.\n        \"\"\"\n        if not isinstance(index, (list, tuple)):\n            index = [index]\n        for idx in index:\n            if idx not in self._index_update_count:\n                self._index_update_count[idx] = self.begin_num_update\n            self._index_update_count[idx] += 1\n            self.num_update = max(self._index_update_count[idx], self.num_update)\n\n    def _get_lrs(self, indices):\n        \"\"\"Gets the learning rates given the indices of the weights.\n\n        Parameters\n        ----------\n        indices : list of int\n            Indices corresponding to weights.\n\n        Returns\n        -------\n        lrs : list of float\n            Learning rates for those indices.\n        \"\"\"\n        if self.lr_scheduler is not None:\n            lr = self.lr_scheduler(self.num_update)\n        else:\n            lr = self.lr\n\n        lrs = [lr for _ in indices]\n        for i, index in enumerate(indices):\n            if index in self.param_dict:\n                lrs[i] *= self.param_dict[index].lr_mult\n            elif index in self.lr_mult:\n                lrs[i] *= self.lr_mult[index]\n            elif index in self.idx2name:\n                lrs[i] *= self.lr_mult.get(self.idx2name[index], 1.0)\n        return lrs\n\n    def _get_lr(self, index):\n        \"\"\"Gets the learning rate given the index of the weight.\n\n        Parameters\n        ----------\n        index : int\n            The index corresponding to the weight.\n\n        Returns\n        -------\n        lr : float\n            Learning rate for this index.\n        \"\"\"\n        return self._get_lrs([index])[0]\n\n    def _get_wds(self, indices):\n        \"\"\"Gets weight decays for indices.\n        Returns 0 for non-weights if the name of weights are provided for `__init__`.\n\n        Parameters\n        ----------\n        indices : list of int\n            Indices of weights.\n\n        Returns\n        -------\n        wds : list of float\n            Weight decays for those indices.\n        \"\"\"\n        wds = [self.wd for _ in indices]\n        for i, index in enumerate(indices):\n            if index in self.param_dict:\n                wds[i] *= self.param_dict[index].wd_mult\n            elif index in self.wd_mult:\n                wds[i] *= self.wd_mult[index]\n            elif index in self.idx2name:\n                wds[i] *= self.wd_mult.get(self.idx2name[index], 1.0)\n        return wds\n\n    def _get_wd(self, index):\n        \"\"\"Gets weight decay for index.\n        Returns 0 for non-weights if the name of weights are provided for `__init__`.\n\n        Parameters\n        ----------\n        index : int\n            The index of weight.\n\n        Returns\n        -------\n        wd : float\n            Weight decay for this index.\n        \"\"\"\n        return self._get_wds([index])[0]\n\n    def __getstate__(self):\n        ret = self.__dict__.copy()\n        # do not include param_dict in the state\n        del ret['param_dict']\n        return ret\n\n    def __setstate__(self, state):\n        self.__dict__ = state\n        # param_dict needs to be explicitly set by the trainer\n        self.param_dict = {}\n\n\n# convenience wrapper for Optimizer.Register\nregister = Optimizer.register   # pylint: disable=invalid-name\n\n# pylint: disable=W0223\n@register\nclass Test(Optimizer):\n    \"\"\"The Test optimizer\"\"\"\n    def __init__(self, **kwargs):\n        super(Test, self).__init__(**kwargs)\n\n    def create_state(self, index, weight):\n        \"\"\"Creates a state to duplicate weight.\"\"\"\n        return zeros(weight.shape, weight.context)\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Performs w += rescale_grad * grad.\"\"\"\n        for index, weight, grad in zip(indices, weights, grads):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n            grad = self.rescale_grad * grad\n            weight[:] -= lr * (grad + wd * weight)\n\n\ncreate = Optimizer.create_optimizer  # pylint: disable=invalid-name\n"
  },
  {
    "path": "python/mxnet/optimizer/rmsprop.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"RMSProp optimizer.\"\"\"\nfrom __future__ import absolute_import\nfrom ..ndarray import (zeros, clip, sqrt, square)\nfrom ..ndarray import (rmsprop_update, rmspropalex_update)\nfrom .optimizer import Optimizer, register\n\n__all__ = ['RMSProp']\n\n\n@register\nclass RMSProp(Optimizer):\n    \"\"\"The RMSProp optimizer.\n\n    Two versions of RMSProp are implemented:\n\n    If ``centered=False``, we follow\n    http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf by\n    Tieleman & Hinton, 2012.\n    For details of the update algorithm see :class:`~mxnet.ndarray.rmsprop_update`.\n\n    If ``centered=True``, we follow http://arxiv.org/pdf/1308.0850v5.pdf (38)-(45)\n    by Alex Graves, 2013.\n    For details of the update algorithm see :class:`~mxnet.ndarray.rmspropalex_update`.\n\n    This optimizer accepts the following parameters in addition to those accepted\n    by :class:`.Optimizer`.\n\n    Parameters\n    ----------\n    learning_rate : float, default 0.001\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    rho: float, default 0.9\n        A decay factor of moving average over past squared gradient.\n    momentum: float, default 0.9\n        Heavy ball momentum factor. Only used if `centered`=``True``.\n    epsilon : float, default 1e-8\n        Small value to avoid division by 0.\n    centered : bool, default False\n        Flag to control which version of RMSProp to use.::\n\n            True: will use Graves's version of `RMSProp`,\n            False: will use Tieleman & Hinton's version of `RMSProp`.\n\n    clip_weights : float, optional\n        Clips weights into range ``[-clip_weights, clip_weights]``.\n    use_fused_step : bool, default True\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False, step is called,\n        otherwise, fused_step is called.\n    \"\"\"\n    def __init__(self, learning_rate=0.001, rho=0.9, momentum=0.9,\n                 epsilon=1e-8, centered=False, clip_weights=None,\n                 use_fused_step=True, **kwargs):\n        if kwargs.get(\"gamma1\") is not None:\n            raise DeprecationWarning(\n                'parameter \\'gamma1\\' is deprecated. Please use \\'rho\\' instead...')\n        if kwargs.get(\"gamma2\") is not None:\n            raise DeprecationWarning(\n                'parameter \\'gamma2\\' is deprecated. Please use \\'momentum\\' instead...')\n        super(RMSProp, self).__init__(learning_rate=learning_rate,\n                                      use_fused_step=use_fused_step,\n                                      **kwargs)\n        self.rho = rho\n        self.momentum = momentum\n        self.centered = centered\n        self.epsilon = epsilon\n        self.clip_weights = clip_weights\n\n    def create_state(self, index, weight):\n        if self.centered:\n            return (\n                zeros(weight.shape, weight.context, stype=weight.stype),  # mean\n                zeros(weight.shape, weight.context, stype=weight.stype),  # var\n                zeros(weight.shape, weight.context, stype=weight.stype))  # mom\n        else:\n            return zeros(weight.shape, weight.context, stype=weight.stype)  # var\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform an optimization step using gradients and states.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n\n            # preprocess grad\n            grad *= self.rescale_grad\n            if self.clip_gradient is not None:\n                grad = clip(grad, - self.clip_gradient, self.clip_gradient)\n            grad += wd * weight\n\n            if not self.centered:\n                # update var\n                var = state\n                var[:] *= self.rho\n                var[:] += (1 - self.rho) * square(grad)\n\n                # update weight\n                d = grad / (sqrt(var) + self.epsilon)\n                weight[:] -= lr * d\n            else:\n                # update mean, var, mom\n                mean, var, mom = state\n                mean[:] *= self.rho\n                mean[:] += (1 - self.rho) * grad\n                var[:] *= self.rho\n                var[:] += (1 - self.rho) * square(grad)\n                mom[:] *= self.momentum\n                mom[:] -= lr * grad / sqrt(var - square(mean) + self.epsilon)\n\n                # update weight\n                weight[:] += mom\n\n            if self.clip_weights:\n                clip(weight, -self.clip_weights, self.clip_weights, out=weight)\n\n    def fused_step(self, indices, weights, grads, states):\n        \"\"\"Perform a fused optimization step using gradients and states.\n        Fused kernel is used for update.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n\n            kwargs = {'rho': self.rho, 'epsilon': self.epsilon,\n                      'rescale_grad': self.rescale_grad}\n            if self.centered:\n                kwargs['momentum'] = self.momentum\n            if self.clip_gradient:\n                kwargs['clip_gradient'] = self.clip_gradient\n            if self.clip_weights:\n                kwargs['clip_weights'] = self.clip_weights\n\n            # update weight with fused kernel\n            if not self.centered:\n                var = state\n                rmsprop_update(weight, grad, var, out=weight, lr=lr, wd=wd, **kwargs)\n            else:\n                mean, var, mom = state\n                rmspropalex_update(weight, grad, mean, var, mom, out=weight,\n                                   lr=lr, wd=wd, **kwargs)\n"
  },
  {
    "path": "python/mxnet/optimizer/sgd.py",
    "content": "# coding: utf-8\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"SGD optimizer\"\"\"\nfrom __future__ import absolute_import\nimport numpy\nfrom ..ndarray import (zeros, clip)\nfrom ..ndarray import (sgd_update, sgd_mom_update,\n                       mp_sgd_update, mp_sgd_mom_update,\n                       multi_sgd_update, multi_sgd_mom_update,\n                       multi_mp_sgd_update, multi_mp_sgd_mom_update)\nfrom .optimizer import Optimizer, register\nfrom .utils import _flatten_list\n\n__all__ = ['SGD']\n\n\n@register\nclass SGD(Optimizer):\n    \"\"\"The SGD optimizer with momentum and weight decay.\n\n    If the storage types of grad is ``row_sparse`` and ``lazy_update`` is True, \\\n    **lazy updates** are applied by::\n\n        for row in grad.indices:\n            rescaled_grad[row] = clip(rescale_grad * grad[row] + wd * weight[row], clip_gradient)\n            state[row] = momentum[row] * state[row] + lr * rescaled_grad[row]\n            weight[row] = weight[row] - state[row]\n\n    The sparse update only updates the momentum for the weights whose row_sparse\n    gradient indices appear in the current batch, rather than updating it for all\n    indices. Compared with the original update, it can provide large\n    improvements in model training throughput for some applications. However, it\n    provides slightly different semantics than the original update, and\n    may lead to different empirical results.\n\n    In the case when ``update_on_kvstore`` is set to False (either globally via\n    MXNET_UPDATE_ON_KVSTORE=0 environment variable or as a parameter in\n    :class:`~mxnet.gluon.Trainer`) SGD optimizer can perform aggregated update\n    of parameters, which may lead to improved performance. The aggregation size\n    is controlled by ``aggregate_num`` and defaults to 4.\n\n    Otherwise, **standard updates** are applied by::\n\n        rescaled_grad = clip(rescale_grad * grad, clip_gradient)) + wd * weight\n        state = momentum * state + lr * rescaled_grad\n        weight = weight - state\n\n    For details of the update algorithm see\n    :class:`~mxnet.ndarray.sgd_update` and :class:`~mxnet.ndarray.sgd_mom_update`.\n\n    This optimizer accepts the following parameters in addition to those accepted\n    by :class:`.Optimizer`.\n\n    Parameters\n    ----------\n    learning_rate : float, default 0.1\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    momentum : float, default 0.\n        The momentum value.\n    lazy_update : bool, default False\n        Default is False. If True, lazy updates are applied \\\n        if the storage types of weight and grad are both ``row_sparse``.\n    multi_precision: bool, default False\n        Flag to control the internal precision of the optimizer.\n        False: results in using the same precision as the weights (default),\n        True: makes internal 32-bit copy of the weights and applies gradients\n        in 32-bit precision even if actual weights used in the model have lower precision.\n        Turning this on can improve convergence and accuracy when training with float16.\n    aggregate_num : int, default 1\n        Number of weights to be aggregated in a list.\n        They are passed to the optimizer for a single optimization step.\n    use_fused_step : bool, default True\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False, step is called,\n        otherwise, fused_step is called.\n    \"\"\"\n    def __init__(self, learning_rate=0.1, momentum=0.0, lazy_update=False,\n                 multi_precision=False, use_fused_step=True, aggregate_num=1, **kwargs):\n        super(SGD, self).__init__(learning_rate=learning_rate,\n                                  multi_precision=multi_precision,\n                                  aggregate_num=aggregate_num,\n                                  use_fused_step=use_fused_step,\n                                  **kwargs)\n        if not self.use_fused_step:\n            assert not lazy_update, \\\n                'When use_fused_step is set to False, lazy_update has to be turned off.'\n        if lazy_update:\n            assert not multi_precision, \\\n                'When lazy_update is set to True, multi_precision has be turned off.'\n        self.momentum = momentum\n        self.lazy_update = lazy_update\n\n    def create_state(self, index, weight):\n        momentum = None\n        if self.momentum != 0.0:\n            stype = weight.stype if self.lazy_update else 'default'\n            momentum = zeros(weight.shape, weight.context, dtype=weight.dtype, stype=stype)\n        return momentum\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform an optimization step using gradients and states.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n\n            # preprocess grad\n            grad *= self.rescale_grad\n            if self.clip_gradient is not None:\n                grad = clip(grad, -self.clip_gradient, self.clip_gradient)\n            grad += wd * weight\n\n            # update mom\n            mom = state\n            if mom is not None:\n                mom[:] *= self.momentum\n                mom[:] -= lr * grad\n            else:\n                mom = -lr * grad\n\n            # update weight\n            weight[:] += mom\n\n    def fused_step(self, indices, weights, grads, states):\n        \"\"\"Perform a fused optimization step using gradients and states.\n        Fused kernel is used for update.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        # When either weight or gradient is sparse, aggregate is False.\n        aggregate = self.aggregate_num > 1\n        for weight, grad in zip(weights, grads):\n            aggregate = (aggregate and\n                         weight.stype == 'default' and\n                         grad.stype == 'default')\n        self._update_count(indices)\n        lrs = self._get_lrs(indices)\n        wds = self._get_wds(indices)\n\n        kwargs = {'rescale_grad': self.rescale_grad}\n        if self.momentum > 0:\n            kwargs['momentum'] = self.momentum\n        if self.clip_gradient:\n            kwargs['clip_gradient'] = self.clip_gradient\n\n        if aggregate:\n            # update `aggregate_num` number of weights in a single kernel.\n            # this does not support sparse weight or gradient.\n            multi_precision = self.multi_precision and weights[0].dtype == numpy.float16\n            if not multi_precision:\n                if self.momentum > 0:\n                    multi_sgd_mom_update(*_flatten_list(zip(weights, grads, states)), out=weights,\n                                         num_weights=len(weights), lrs=lrs, wds=wds, **kwargs)\n                else:\n                    multi_sgd_update(*_flatten_list(zip(weights, grads)), out=weights,\n                                     num_weights=len(weights), lrs=lrs, wds=wds, **kwargs)\n            else:\n                states = list(zip(*states))\n                weights32, moms = states\n                if self.momentum > 0:\n                    multi_mp_sgd_mom_update(*_flatten_list(zip(weights, grads,\n                                                               moms, weights32)),\n                                            out=weights, num_weights=len(weights),\n                                            lrs=lrs, wds=wds, **kwargs)\n                else:\n                    multi_mp_sgd_update(*_flatten_list(zip(weights, grads,\n                                                           weights32)),\n                                        out=weights, num_weights=len(weights),\n                                        lrs=lrs, wds=wds, **kwargs)\n        else:\n            for weight, grad, state, lr, wd in zip(weights, grads, states, lrs, wds):\n                multi_precision = self.multi_precision and weight.dtype == numpy.float16\n                if not multi_precision:\n                    mom = state\n                    if mom is not None:\n                        sgd_mom_update(weight, grad, mom, out=weight,\n                                       lazy_update=self.lazy_update, lr=lr, wd=wd, **kwargs)\n                    else:\n                        sgd_update(weight, grad, out=weight, lazy_update=self.lazy_update,\n                                   lr=lr, wd=wd, **kwargs)\n                else:\n                    # weight32 is a float32 copy of weight.\n                    # in the kernel, we firstly update weight32,\n                    # and then cast the result to float16 and save it to weight.\n                    weight32, mom = state\n                    if mom is not None:\n                        mp_sgd_mom_update(weight, grad, mom, weight32, out=weight,\n                                          lr=lr, wd=wd, **kwargs)\n                    else:\n                        mp_sgd_update(weight, grad, weight32, out=weight,\n                                      lr=lr, wd=wd, **kwargs)\n\n    def update_multi_precision(self, indices, weights, grads, states):\n        \"\"\"Override update_multi_precision.\n        \"\"\"\n        if self.use_fused_step:\n            self.update(indices, weights, grads, states)\n        else:\n            super(SGD, self).update_multi_precision(indices, weights, grads, states)\n"
  },
  {
    "path": "python/mxnet/optimizer/sgld.py",
    "content": "# coding: utf-8\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=W0223\n\"\"\"SGLD optimizer.\"\"\"\nfrom __future__ import absolute_import\nimport math\nfrom ..ndarray import clip\nfrom ..random import normal\nfrom .optimizer import Optimizer, register\n\n__all__ = ['SGLD']\n\n\n@register\nclass SGLD(Optimizer):\n    \"\"\"Stochastic Gradient Riemannian Langevin Dynamics.\n\n    This class implements the optimizer described in the paper *Stochastic Gradient\n    Riemannian Langevin Dynamics on the Probability Simplex*, available at\n    https://papers.nips.cc/paper/4883-stochastic-gradient-riemannian-langevin-dynamics-on-the-probability-simplex.pdf.\n\n    Parameters\n    ----------\n    learning_rate : float, default 0.001\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    use_fused_step : bool, default False\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False, step is called,\n        otherwise, fused_step is called.\n    \"\"\"\n    def __init__(self, learning_rate=0.1, use_fused_step=False, **kwargs):\n        super(SGLD, self).__init__(learning_rate=learning_rate,\n                                   use_fused_step=use_fused_step,\n                                   **kwargs)\n\n    def create_state(self, index, weight):\n        return None\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform an optimization step using gradients and states.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad in zip(indices, weights, grads):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n\n            # preprocess grad\n            grad *= self.rescale_grad\n            if self.clip_gradient is not None:\n                grad = clip(grad, - self.clip_gradient, self.clip_gradient)\n            grad += wd * weight\n\n            # update weight\n            weight[:] -= lr / 2 * grad\n            weight[:] += normal(0, math.sqrt(lr), shape=weight.shape,\n                                dtype=weight.dtype, ctx=weight.context)\n"
  },
  {
    "path": "python/mxnet/optimizer/signum.py",
    "content": "# coding: utf-8\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Signum optimizer.\"\"\"\nfrom __future__ import absolute_import\nfrom ..ndarray import (zeros, clip)\nfrom ..ndarray import (signsgd_update, signum_update)\nfrom .optimizer import Optimizer, register\n\n__all__ = ['Signum']\n\n\n@register\nclass Signum(Optimizer):\n    r\"\"\"The Signum optimizer that takes the sign of gradient or momentum.\n\n    The optimizer updates the weight by::\n\n        rescaled_grad = rescale_grad * clip(grad, clip_gradient) + wd * weight\n        state = momentum * state + (1-momentum)*rescaled_grad\n        weight = (1 - lr * wd_lh) * weight - lr * sign(state)\n\n    References\n    ----------\n    Jeremy Bernstein, Yu-Xiang Wang, Kamyar Azizzadenesheli & Anima Anandkumar. (2018).\n    signSGD: Compressed Optimisation for Non-Convex Problems. In ICML'18.\n\n    See: https://arxiv.org/abs/1802.04434\n\n    For details of the update algorithm see\n    :class:`~mxnet.ndarray.signsgd_update` and :class:`~mxnet.ndarray.signum_update`.\n\n    This optimizer accepts the following parameters in addition to those accepted\n    by :class:`.Optimizer`.\n\n    Parameters\n    ----------\n    learning_rate : float, default 0.01\n        The initial learning rate. If None, the optimization will use the\n        learning rate from ``lr_scheduler``. If not None, it will overwrite\n        the learning rate in ``lr_scheduler``. If None and ``lr_scheduler``\n        is also None, then it will be set to 0.01 by default.\n    momentum : float, optional\n       The momentum value.\n    wd_lh : float, optional\n       The amount of decoupled weight decay regularization, see details in the original paper at:\\\n       https://arxiv.org/abs/1711.05101\n    use_fused_step : bool, default True\n        Whether or not to use fused kernels for optimizer.\n        When use_fused_step=False, step is called,\n        otherwise, fused_step is called.\n    \"\"\"\n    def __init__(self, learning_rate=0.01, momentum=0.9, wd_lh=0.0, use_fused_step=True, **kwargs):\n        super(Signum, self).__init__(learning_rate=learning_rate,\n                                     use_fused_step=use_fused_step,\n                                     **kwargs)\n        self.momentum = momentum\n        self.wd_lh = wd_lh\n\n    def create_state(self, index, weight):\n        momentum = None\n        if self.momentum != 0.0:\n            momentum = zeros(weight.shape, weight.context, dtype=weight.dtype, stype=weight.stype)\n        return momentum\n\n    def step(self, indices, weights, grads, states):\n        \"\"\"Perform an optimization step using gradients and states.\n\n         Parameters\n         ----------\n         indices : list of int\n             List of unique indices of the parameters into the individual learning rates\n             and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n             and `set_wd_mult()`, respectively.\n         weights : list of NDArray\n             List of parameters to be updated.\n         grads : list of NDArray\n             List of gradients of the objective with respect to this parameter.\n         states : List of any obj\n             List of state returned by `create_state()`.\n         \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n\n            if state is not None:\n                # preprocess grad\n                grad *= self.rescale_grad\n                if self.clip_gradient is not None:\n                    grad = clip(grad, - self.clip_gradient, self.clip_gradient)\n                grad += wd * weight\n\n                # update mom\n                mom = state\n                mom[:] *= self.momentum\n                mom[:] -= (1 - self.momentum) * grad\n\n                # update weight\n                weight[:] *= 1 - lr * self.wd_lh\n                weight[:] += lr * ((mom > 0) - (mom < 0))\n            else:\n                # update weight\n                weight[:] *= 1 - lr * (wd + self.wd_lh)\n                weight[:] -= lr * ((grad > 0) - (grad < 0))\n\n    def fused_step(self, indices, weights, grads, states):\n        \"\"\"Perform a fused optimization step using gradients and states.\n        Fused kernel is used for update.\n\n        Parameters\n        ----------\n        indices : list of int\n            List of unique indices of the parameters into the individual learning rates\n            and weight decays. Learning rates and weight decay may be set via `set_lr_mult()`\n            and `set_wd_mult()`, respectively.\n        weights : list of NDArray\n            List of parameters to be updated.\n        grads : list of NDArray\n            List of gradients of the objective with respect to this parameter.\n        states : List of any obj\n            List of state returned by `create_state()`.\n        \"\"\"\n        for index, weight, grad, state in zip(indices, weights, grads, states):\n            self._update_count(index)\n            lr = self._get_lr(index)\n            wd = self._get_wd(index)\n\n            kwargs = {'rescale_grad': self.rescale_grad}\n            if self.momentum > 0:\n                kwargs['momentum'] = self.momentum\n            if self.clip_gradient:\n                kwargs['clip_gradient'] = self.clip_gradient\n\n            # update weight with fused kernel\n            if state is not None:\n                if self.wd_lh:\n                    kwargs['wd_lh'] = self.wd_lh\n                signum_update(weight, grad, state, out=weight,\n                              lr=lr, wd=wd, **kwargs)\n            else:\n                wd += self.wd_lh\n                signsgd_update(weight, grad, out=weight,\n                               lr=lr, wd=wd, **kwargs)\n"
  },
  {
    "path": "python/mxnet/optimizer/updater.py",
    "content": "# coding: utf-8\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Updater class.\"\"\"\nfrom __future__ import absolute_import\nimport pickle\nimport numpy\nfrom ..base import py_str\nfrom ..ndarray import NDArray\nfrom ..profiler import scope as profiler_scope\nfrom ..util import is_np_array\nfrom .utils import _as_classic\n\n__all__ = ['Updater', 'get_updater']\n\n\nclass Updater(object):\n    \"\"\"Updater for kvstore.\"\"\"\n    def __init__(self, optimizer):\n        self.optimizer = optimizer\n        self.states = {}\n        self.states_synced = {}\n        self.aggregate_updates = optimizer.aggregate_num > 1\n\n    def __call__(self, index, grad, weight):\n        \"\"\"Updates weight given gradient and index.\"\"\"\n        allow_np = self.optimizer.allow_np_array if hasattr(self.optimizer, \"allow_np_array\") else is_np_array()\n        if not isinstance(index, (list, tuple)):\n            indices = [index]\n            grads = [_as_classic(grad, allow_np)]\n            weights = [_as_classic(weight, allow_np)]\n        else:\n            indices = index\n            grads = _as_classic(grad, allow_np)\n            weights = _as_classic(weight, allow_np)\n        if weights:\n            self.optimizer._set_current_device(weights[0].context.device_id)\n        for i, idx in enumerate(indices):\n            # convert ctypes.char_p.value back to python str if needed\n            if isinstance(idx, bytes):\n                indices[i] = py_str(idx)\n                idx = indices[i]\n            if idx not in self.states:\n                with profiler_scope(\"updater:optimizer_state\"):\n                    self.states[idx] = self.optimizer.create_state_multi_precision(idx, weights[i])\n                self.states_synced[idx] = True\n            elif not self.states_synced[idx]:\n                self.states[idx] = \\\n                    self.sync_state_context(self.states[idx], weights[i].context)\n                self.states_synced[idx] = True\n        if self.aggregate_updates:\n            # segregate values based on type\n            if self.optimizer.aggregate_num is not numpy.inf:\n                type_map = {}\n                for i, w, g in zip(indices, weights, grads):\n                    if w.dtype in type_map:\n                        type_map[w.dtype].append((i, w, g))\n                    else:\n                        type_map[w.dtype] = [(i, w, g)]\n                for idx in type_map:\n                    current_index = 0\n                    indices, weights, grads = zip(*type_map[idx])\n                    while current_index < len(indices):\n                        states = []\n                        step = min(self.optimizer.aggregate_num, len(indices) - current_index)\n                        for j in range(step):\n                            states.append(self.states[indices[current_index + j]])\n                        self.optimizer.update_multi_precision(\n                            indices[current_index:current_index + self.optimizer.aggregate_num],\n                            weights[current_index:current_index + self.optimizer.aggregate_num],\n                            grads[current_index:current_index + self.optimizer.aggregate_num],\n                            states)\n                        current_index += self.optimizer.aggregate_num\n            else:\n                states = [self.states[i] for i in indices]\n                self.optimizer.update_multi_precision(indices, weights, grads, states)\n        else:\n            for i, w, g in zip(indices, weights, grads):\n                self.optimizer.update_multi_precision([i], [w], [g], [self.states[i]])\n\n    def sync_state_context(self, state, context):\n        \"\"\"sync state context.\"\"\"\n        if isinstance(state, NDArray):\n            return state.as_in_context(context)\n        elif isinstance(state, (tuple, list)):\n            synced_state = (self.sync_state_context(i, context) for i in state)\n            if isinstance(state, tuple):\n                return tuple(synced_state)\n            else:\n                return list(synced_state)\n        else:\n            return state\n\n    def set_states(self, states):\n        \"\"\"Sets updater states.\"\"\"\n        states = pickle.loads(states)\n        if isinstance(states, tuple) and len(states) == 2:\n            self.states, self.optimizer = states\n        else:\n            self.states = states\n        self.states_synced = dict.fromkeys(self.states.keys(), False)\n\n    def get_states(self, dump_optimizer=False):\n        \"\"\"Gets updater states.\n\n        Parameters\n        ----------\n        dump_optimizer : bool, default False\n            Whether to also save the optimizer itself. This would also save optimizer\n            information such as learning rate and weight decay schedules.\n        \"\"\"\n        return pickle.dumps((self.states, self.optimizer) if dump_optimizer else self.states)\n\n\ndef get_updater(optimizer):\n    \"\"\"Returns a closure of the updater needed for kvstore.\n\n    Parameters\n    ----------\n    optimizer: Optimizer\n         The optimizer.\n\n    Returns\n    -------\n    updater: function\n         The closure of the updater.\n    \"\"\"\n    return Updater(optimizer)\n"
  },
  {
    "path": "python/mxnet/optimizer/utils.py",
    "content": "# coding: utf-8\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"Optimizer utility functions.\"\"\"\nfrom __future__ import absolute_import\n\n\ndef _flatten_list(nested_list):\n    return [item for sublist in nested_list for item in sublist]\n\n\ndef _as_classic(a, allow_np):\n    # TODO(junwu): This is a temp solution for allowing converting\n    # np.ndarray to mx.nd.NDArray to be fed into the optimizer since\n    # users may have custom optimizers implemented using mx.nd.NDArray ops.\n    from ..numpy import ndarray as np_ndarray\n    if isinstance(a, (tuple, list)):\n        if any(isinstance(x, np_ndarray) for x in a):\n            if allow_np:\n                return [x.as_nd_ndarray() for x in a]\n            else:\n                raise ValueError('Converting np.ndarray to mx.nd.NDArray is not allowed')\n    else:\n        if isinstance(a, np_ndarray):\n            if allow_np:\n                return a.as_nd_ndarray()\n            else:\n                raise ValueError('Converting np.ndarray to mx.nd.NDArray is not allowed')\n    return a\n"
  },
  {
    "path": "python/mxnet/profiler.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=fixme, invalid-name, too-many-arguments, too-many-locals, too-many-lines\n# pylint: disable=too-many-branches, too-many-statements\n\"\"\"Profiler setting methods.\"\"\"\nimport ctypes\nimport contextlib\nimport contextvars\nimport warnings\nfrom .base import _LIB, check_call, c_str, ProfileHandle, c_str_array, py_str, KVStoreHandle\n\nprofiler_kvstore_handle = KVStoreHandle()\n\ndef set_kvstore_handle(handle):\n    global profiler_kvstore_handle\n    profiler_kvstore_handle = handle\n\ndef set_config(**kwargs):\n    \"\"\"Set up the configure of profiler (only accepts keyword arguments).\n\n    Parameters\n    ----------\n    filename : string,\n        output file for profile data\n    gpu_memory_profile_filename_prefix : string\n        filename prefix for the GPU memory profile\n    profile_all : boolean,\n        all profile types enabled\n    profile_symbolic : boolean,\n        whether to profile symbolic operators\n    profile_imperative : boolean,\n        whether to profile imperative operators\n    profile_memory : boolean,\n        whether to profile memory usage\n    profile_api : boolean,\n        whether to profile the C API\n    continuous_dump : boolean,\n        whether to periodically dump profiling data to file\n    dump_period : float,\n        seconds between profile data dumps\n    aggregate_stats : boolean,\n        whether to maintain aggregate stats in memory for console\n        dump.  Has some negative performance impact.\n    profile_process : string\n        whether to profile kvstore `server` or `worker`.\n        server can only be profiled when kvstore is of type dist.\n        if this is not passed, defaults to `worker`\n    \"\"\"\n    kk = kwargs.keys()\n    vv = kwargs.values()\n    check_call(_LIB.MXSetProcessProfilerConfig(len(kwargs),\n                                               c_str_array([key for key in kk]),\n                                               c_str_array([str(val) for val in vv]),\n                                               profiler_kvstore_handle))\n\n\ndef profiler_set_config(mode='symbolic', filename='profile.json'):\n    \"\"\"Set up the configure of profiler (Deprecated).\n\n    Parameters\n    ----------\n    mode : string, optional\n        Indicates whether to enable the profiler, can\n        be 'symbolic', or 'all'. Defaults to `symbolic`.\n    filename : string, optional\n        The name of output trace file. Defaults to 'profile.json'.\n    \"\"\"\n    warnings.warn('profiler.profiler_set_config() is deprecated. '\n                  'Please use profiler.set_config() instead')\n    keys = c_str_array([key for key in [\"profile_\" + mode, \"filename\"]])\n    values = c_str_array([str(val) for val in [True, filename]])\n    assert len(keys) == len(values)\n    check_call(_LIB.MXSetProcessProfilerConfig(len(keys), keys, values, profiler_kvstore_handle))\n\n\ndef set_state(state='stop', profile_process='worker'):\n    \"\"\"Set up the profiler state to 'run' or 'stop'.\n\n    Parameters\n    ----------\n    state : string, optional\n        Indicates whether to run the profiler, can\n        be 'stop' or 'run'. Default is `stop`.\n    profile_process : string\n        whether to profile kvstore `server` or `worker`.\n        server can only be profiled when kvstore is of type dist.\n        if this is not passed, defaults to `worker`\n    \"\"\"\n    state2int = {'stop': 0, 'run': 1}\n    profile_process2int = {'worker': 0, 'server': 1}\n    check_call(_LIB.MXSetProcessProfilerState(ctypes.c_int(state2int[state]),\n                                              profile_process2int[profile_process],\n                                              profiler_kvstore_handle))\n\n\ndef profiler_set_state(state='stop'):\n    \"\"\"Set up the profiler state to 'run' or 'stop' (Deprecated).\n\n    Parameters\n    ----------\n    state : string, optional\n        Indicates whether to run the profiler, can\n        be 'stop' or 'run'. Default is `stop`.\n    \"\"\"\n    warnings.warn('profiler.profiler_set_state() is deprecated. '\n                  'Please use profiler.set_state() instead')\n    set_state(state)\n\ndef dump(finished=True, profile_process='worker'):\n    \"\"\"Dump profile and stop profiler. Use this to save profile\n    in advance in case your program cannot exit normally.\n\n    Parameters\n    ----------\n    finished : boolean\n        Indicates whether to stop statistic output (dumping) after this dump.\n        Default is True\n    profile_process : string\n        whether to profile kvstore `server` or `worker`.\n        server can only be profiled when kvstore is of type dist.\n        if this is not passed, defaults to `worker`\n    \"\"\"\n    fin = 1 if finished is True else 0\n    profile_process2int = {'worker': 0, 'server': 1}\n    check_call(_LIB.MXDumpProcessProfile(fin,\n                                         profile_process2int[profile_process],\n                                         profiler_kvstore_handle))\n\n\ndef dump_profile():\n    \"\"\"Dump profile and stop profiler. Use this to save profile\n    in advance in case your program cannot exit normally.\"\"\"\n    warnings.warn('profiler.dump_profile() is deprecated. '\n                  'Please use profiler.dump() instead')\n    dump(True)\n\n\ndef dumps(reset=False, format='table', sort_by='total', ascending=False):\n    \"\"\"Return a printable string of aggregate profile stats.\n\n    Parameters\n    ----------\n    reset: boolean\n        indicates whether to clean aggeregate statistical data collected up to this point\n    format: string\n        whether to return the aggregate stats in table of json format\n        can take 'table' or 'json'\n        defaults to 'table'\n    sort_by: string\n        can take 'total', 'avg', 'min', 'max', or 'count'\n        by which stat to sort the entries in each category\n        defaults to 'total'\n    ascending: boolean\n        whether to sort ascendingly\n        defaults to False\n    \"\"\"\n    debug_str = ctypes.c_char_p()\n    reset_to_int = {False: 0, True: 1}\n    format_to_int = {'table': 0, 'json': 1}\n    sort_by_to_int = {'total': 0, 'avg': 1, 'min': 2, 'max': 3, 'count': 4}\n    asc_to_int = {False: 0, True: 1}\n    assert format in format_to_int.keys(),\\\n            \"Invalid value provided for format: {0}. Support: 'table', 'json'\".format(format)\n    assert sort_by in sort_by_to_int.keys(),\\\n            \"Invalid value provided for sort_by: {0}.\\\n             Support: 'total', 'avg', 'min', 'max', 'count'\"\\\n            .format(sort_by)\n    assert  ascending in asc_to_int.keys(),\\\n            \"Invalid value provided for ascending: {0}. Support: False, True\".format(ascending)\n    assert  reset in reset_to_int.keys(),\\\n            \"Invalid value provided for reset: {0}. Support: False, True\".format(reset)\n    check_call(_LIB.MXAggregateProfileStatsPrint(ctypes.byref(debug_str),\n                                                 reset_to_int[reset],\n                                                 format_to_int[format],\n                                                 sort_by_to_int[sort_by],\n                                                 asc_to_int[ascending]))\n    return py_str(debug_str.value)\n\n\ndef pause(profile_process='worker'):\n    \"\"\"Pause profiling.\n\n    Parameters\n    ----------\n    profile_process : string\n        whether to profile kvstore `server` or `worker`.\n        server can only be profiled when kvstore is of type dist.\n        if this is not passed, defaults to `worker`\n    \"\"\"\n    profile_process2int = {'worker': 0, 'server': 1}\n    check_call(_LIB.MXProcessProfilePause(int(1),\n                                          profile_process2int[profile_process],\n                                          profiler_kvstore_handle))\n\n\ndef resume(profile_process='worker'):\n    \"\"\"Resume paused profiling.\n\n    Parameters\n    ----------\n    profile_process : string\n        whether to profile kvstore `server` or `worker`.\n        server can only be profiled when kvstore is of type dist.\n        if this is not passed, defaults to `worker`\n    \"\"\"\n    profile_process2int = {'worker': 0, 'server': 1}\n    check_call(_LIB.MXProcessProfilePause(int(0),\n                                          profile_process2int[profile_process],\n                                          profiler_kvstore_handle))\n\n\nclass Domain(object):\n    \"\"\"Profiling domain, used to group sub-objects like tasks, counters, etc into categories\n    Serves as part of 'categories' for chrome://tracing\n\n    Note: Domain handles are never destroyed.\n\n    Parameters\n    ----------\n    name : string\n        Name of the domain\n    \"\"\"\n    def __init__(self, name):\n        self.name = name\n        self.handle = ProfileHandle()\n        check_call(_LIB.MXProfileCreateDomain(c_str(self.name), ctypes.byref(self.handle)))\n\n    def __str__(self):\n        return self.name\n\n    def new_task(self, name):\n        \"\"\"Create new Task object owned by this domain\n\n        Parameters\n        ----------\n        name : string\n            Name of the task\n        \"\"\"\n        return Task(self, name)\n\n    def new_frame(self, name):\n        \"\"\"Create new Frame object owned by this domain\n\n        Parameters\n        ----------\n        name : string\n            Name of the frame\n        \"\"\"\n        return Frame(self, name)\n\n    def new_counter(self, name, value=None):\n        \"\"\"Create new Counter object owned by this domain\n\n        Parameters\n        ----------\n        name : string\n            Name of the counter\n        \"\"\"\n        return Counter(self, name, value)\n\n    def new_marker(self, name):\n        \"\"\"Create new Marker object owned by this domain\n\n        Parameters\n        ----------\n        name : string\n            Name of the marker\n        \"\"\"\n        return Marker(self, name)\n\nclass Task(object):\n    \"\"\"Profiling Task class.\n\n    A task is a logical unit of work performed by a particular thread.\n    Tasks can nest; thus, tasks typically correspond to functions, scopes, or a case block\n    in a switch statement.\n    You can use the Task API to assign tasks to threads.\n\n    This is different from Frame in that all profiling statistics for passes\n    through the task's begin and endpoints are accumulated together into a single statistical\n    analysys, rather than a separate analysis for each pass (as with a Frame)\n\n    Parameters\n    ----------\n    domain : Domain object\n        Domain to which this object belongs\n    name : string\n        Name of the task\n    \"\"\"\n    def __init__(self, domain, name):\n        self.name = name\n        self.handle = ProfileHandle()\n        check_call(_LIB.MXProfileCreateTask(domain.handle,\n                                            c_str(self.name),\n                                            ctypes.byref(self.handle)))\n\n    def __del__(self):\n        if self.handle is not None:\n            check_call(_LIB.MXProfileDestroyHandle(self.handle))\n\n    def start(self):\n        \"\"\"Start timing scope for this object\"\"\"\n        check_call(_LIB.MXProfileDurationStart(self.handle))\n\n    def stop(self):\n        \"\"\"Stop timing scope for this object\"\"\"\n        check_call(_LIB.MXProfileDurationStop(self.handle))\n\n    def __str__(self):\n        return self.name\n\n\nclass Frame(object):\n    \"\"\"Profiling Frame class.\n\n    Use the frame API to insert calls to the desired places in your code and analyze\n    performance per frame, where frame is the time period between frame begin and end points.\n    When frames are displayed in Intel VTune Amplifier, they are displayed in a\n    separate track, so they provide a way to visually separate this data from normal task data.\n\n    This is different from Task in that each 'Frame' duration will be a discretely-numbered\n    event in the VTune output, as well as its rate (frame-rate) shown.  This is analogous to\n    profiling each frame of some visual output, such as rendering a video game frame.\n\n    Parameters\n    ----------\n    domain : Domain object\n        Domain to which this object belongs\n    name : string\n        Name of the frame\n    \"\"\"\n    def __init__(self, domain, name):\n        self.name = name\n        self.handle = ProfileHandle()\n        check_call(_LIB.MXProfileCreateFrame(domain.handle,\n                                             c_str(self.name),\n                                             ctypes.byref(self.handle)))\n\n    def __del__(self):\n        if self.handle is not None:\n            check_call(_LIB.MXProfileDestroyHandle(self.handle))\n\n    def start(self):\n        \"\"\"Start timing scope for this object\"\"\"\n        check_call(_LIB.MXProfileDurationStart(self.handle))\n\n    def stop(self):\n        \"\"\"Stop timing scope for this object\"\"\"\n        check_call(_LIB.MXProfileDurationStop(self.handle))\n\n    def __str__(self):\n        return self.name\n\n\nclass Event(object):\n    \"\"\"Profiling Event class.\n\n    The event API is used to observe when demarcated events occur in your application, or to\n    identify how long it takes to execute demarcated regions of code. Set annotations in the\n    application to demarcate areas where events of interest occur.\n    After running analysis, you can see the events marked in the Timeline pane.\n    Event API is a per-thread function that works in resumed state.\n    This function does not work in paused state.\n\n    Parameters\n    ----------\n    name : string\n        Name of the event\n    \"\"\"\n    def __init__(self, name):\n        self.name = name\n        self.handle = ProfileHandle()\n        check_call(_LIB.MXProfileCreateEvent(c_str(self.name), ctypes.byref(self.handle)))\n\n    def __del__(self):\n        if self.handle is not None:\n            check_call(_LIB.MXProfileDestroyHandle(self.handle))\n\n    def start(self):\n        \"\"\"Start timing scope for this object\"\"\"\n        check_call(_LIB.MXProfileDurationStart(self.handle))\n\n    def stop(self):\n        \"\"\"Stop timing scope for this object\"\"\"\n        check_call(_LIB.MXProfileDurationStop(self.handle))\n\n    def __str__(self):\n        return self.name\n\n\nclass Counter(object):\n    \"\"\"Profiling Counter class.\n\n    The counter event can track a value as it changes over time.\n\n    Parameters\n    ----------\n    domain : Domain object\n        Domain to which this object belongs\n    name : string\n        Name of the counter\n    value: integer, optional\n        Initial value of the counter\n    \"\"\"\n    def __init__(self, domain, name, value=None):\n        self.name = name\n        self.handle = ProfileHandle()\n        check_call(_LIB.MXProfileCreateCounter(domain.handle,\n                                               c_str(name),\n                                               ctypes.byref(self.handle)))\n        if value is not None:\n            self.set_value(value)\n\n    def __del__(self):\n        if self.handle is not None:\n            check_call(_LIB.MXProfileDestroyHandle(self.handle))\n\n\n    def set_value(self, value):\n        \"\"\"Set counter value.\n\n        Parameters\n        ----------\n        value : int\n            Value for the counter\n        \"\"\"\n        check_call(_LIB.MXProfileSetCounter(self.handle, int(value)))\n\n    def increment(self, delta=1):\n        \"\"\"Increment counter value.\n\n        Parameters\n        ----------\n        value_change : int\n            Amount by which to add to the counter\n        \"\"\"\n        check_call(_LIB.MXProfileAdjustCounter(self.handle, int(delta)))\n\n    def decrement(self, delta=1):\n        \"\"\"Decrement counter value.\n\n        Parameters\n        ----------\n        value_change : int\n            Amount by which to subtract from the counter\n        \"\"\"\n        check_call(_LIB.MXProfileAdjustCounter(self.handle, -int(delta)))\n\n    def __iadd__(self, delta):\n        self.increment(delta)\n        return self\n\n    def __isub__(self, delta):\n        self.decrement(delta)\n        return self\n\n    def __str__(self):\n        return self.name\n\n\nclass Marker(object):\n    \"\"\"Set marker for an instant in time.\n\n    The marker event marks a particular instant in time across some scope boundaries.\n\n    Parameters\n    ----------\n    domain : Domain object\n        Domain to which this object belongs\n    name : string\n        Name of the marker\n    \"\"\"\n    def __init__(self, domain, name):\n        self.name = name\n        self.domain = domain\n\n    def mark(self, scope='process'):  # pylint: disable=redefined-outer-name\n        \"\"\"Set up the profiler state to record operator.\n\n        Parameters\n        ----------\n        scope : string, optional\n            Indicates what scope the marker should refer to.\n            Can be 'global', 'process', thread', task', and 'marker'\n            Default is `process`.\n        \"\"\"\n        check_call(_LIB.MXProfileSetMarker(self.domain.handle, c_str(self.name), c_str(scope)))\n\n\n@contextlib.contextmanager\ndef scope(name='<unk>:', append_mode=True):\n    \"\"\"Assign the profiler scope for the GPU memory profiler.\n\n    It is implicitly invoked when the Gluon API is used.\n\n    Parameters\n    ==========\n    name : Name of the Profiler Scope\n    append_mode : Whether to append the old profiler scope at the front.\n\n    \"\"\"\n    name = name + \":\" if not name.endswith(\":\") else name\n    if append_mode and _current_scope.get() != \"<unk>:\":\n        name = _current_scope.get() + name\n    token = _current_scope.set(name)\n    # Invoke the C API to propagate the profiler scope information to the\n    # C++ backend.\n    check_call(_LIB.MXSetProfilerScope(c_str(name)))\n    yield name\n    _current_scope.reset(token)\n    # Invoke the C API once again to recover the previous scope information.\n    check_call(_LIB.MXSetProfilerScope(c_str(_current_scope.get())))\n\n# initialize the default profiler scope\n_current_scope = contextvars.ContextVar('profilerscope', default='<unk>:')\n"
  },
  {
    "path": "python/mxnet/random.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=no-member, protected-access, unused-import, no-name-in-module\n# pylint: disable=wildcard-import, unused-wildcard-import\n\"\"\"Random number interface of MXNet.\"\"\"\n\nimport ctypes\nfrom .base import _LIB, check_call, integer_types\nfrom .ndarray.random import *\nfrom .device import Device\nfrom .util import wrap_ctx_to_device_func\n\n\n@wrap_ctx_to_device_func\ndef seed(seed_state, device=\"all\"):\n    \"\"\"Seeds the random number generators in MXNet.\n\n    This affects the behavior of modules in MXNet that uses random number generators,\n    like the dropout operator and `NDArray`'s random sampling operators.\n\n    Parameters\n    ----------\n    seed_state : int\n        The random number seed.\n\n    device : Device\n        The device context of the generator. The default is \"all\" which means seeding random\n        number generators of all devices.\n\n    Notes\n    -----\n    Random number generators in MXNet are device specific.\n    `mx.random.seed(seed_state)` sets the state of each generator using `seed_state` and the\n    device id. Therefore, random numbers generated from different devices can be different\n    even if they are seeded using the same seed.\n\n    To produce identical random number sequences independent of the device id,\n    set optional `device` argument. This produces the same sequence of random numbers independent\n    of the device id, but the sequence can be different on different kind of devices as MXNet's\n    random number generators for CPU and GPU use different algorithms.\n\n    Example\n    -------\n    >>> print(mx.np.random.normal(shape=(2,2)).asnumpy())\n    [[ 1.36481571 -0.62203991]\n     [-1.4962182  -0.08511394]]\n    >>> print(mx.np.random.normal(shape=(2,2)).asnumpy())\n    [[ 1.09544981 -0.20014545]\n     [-0.20808885  0.2527658 ]]\n    # Same results on the same device with the same seed\n    >>> mx.np.random.seed(128)\n    >>> print(mx.np.random.normal(shape=(2,2)).asnumpy())\n    [[ 0.47400656 -0.75213492]\n     [ 0.20251541  0.95352972]]\n    >>> mx.np.random.seed(128)\n    >>> print(mx.np.random.normal(shape=(2,2)).asnumpy())\n    [[ 0.47400656 -0.75213492]\n     [ 0.20251541  0.95352972]]\n    # Different results on gpu(0) and gpu(1) with the same seed\n    >>> mx.np.random.seed(128)\n    >>> print(mx.np.random.normal(shape=(2,2), device=mx.gpu(0)).asnumpy())\n    [[ 2.5020072 -1.6884501]\n     [-0.7931333 -1.4218881]]\n    >>> mx.np.random.seed(128)\n    >>> print(mx.np.random.normal(shape=(2,2), device=mx.gpu(1)).asnumpy())\n    [[ 0.24336822 -1.664805  ]\n     [-1.0223296   1.253198  ]]\n    # Seeding with `device` argument produces identical results on gpu(0) and gpu(1)\n    >>> mx.np.random.seed(128, device=mx.gpu(0))\n    >>> print(mx.np.random.normal(shape=(2,2), device=mx.gpu(0)).asnumpy())\n    [[ 2.5020072 -1.6884501]\n     [-0.7931333 -1.4218881]]\n    >>> mx.np.random.seed(128, device=mx.gpu(1))\n    >>> print(mx.np.random.normal(shape=(2,2), device=mx.gpu(1)).asnumpy())\n    [[ 2.5020072 -1.6884501]\n     [-0.7931333 -1.4218881]]\n    \"\"\"\n    if not isinstance(seed_state, integer_types):\n        raise ValueError('seed_state must be int')\n    seed_state = ctypes.c_int(int(seed_state))\n    if device == \"all\":\n        check_call(_LIB.MXRandomSeed(seed_state))\n    else:\n        device = Device(device)\n        check_call(_LIB.MXRandomSeedContext(seed_state, device.device_typeid, device.device_id))\n"
  },
  {
    "path": "python/mxnet/recordio.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Read and write for the RecordIO data format.\"\"\"\nfrom collections import namedtuple\nfrom multiprocessing import current_process\n\nimport ctypes\nimport struct\nimport numbers\nimport numpy as np\n\nfrom .base import _LIB\nfrom .base import RecordIOHandle\nfrom .base import check_call\nfrom .base import c_str\ntry:\n    import cv2\nexcept ImportError:\n    cv2 = None\n\nclass MXRecordIO(object):\n    \"\"\"Reads/writes `RecordIO` data format, supporting sequential read and write.\n\n    Examples\n    ---------\n    >>> record = mx.recordio.MXRecordIO('tmp.rec', 'w')\n    <mxnet.recordio.MXRecordIO object at 0x10ef40ed0>\n    >>> for i in range(5):\n    ...    record.write('record_%d'%i)\n    >>> record.close()\n    >>> record = mx.recordio.MXRecordIO('tmp.rec', 'r')\n    >>> for i in range(5):\n    ...    item = record.read()\n    ...    print(item)\n    record_0\n    record_1\n    record_2\n    record_3\n    record_4\n    >>> record.close()\n\n    Parameters\n    ----------\n    uri : string\n        Path to the record file.\n    flag : string\n        'w' for write or 'r' for read.\n    \"\"\"\n    def __init__(self, uri, flag):\n        self.uri = c_str(uri)\n        self.handle = RecordIOHandle()\n        self.flag = flag\n        self.pid = None\n        self.is_open = False\n        self.open()\n\n    def open(self):\n        \"\"\"Opens the record file.\"\"\"\n        if self.flag == \"w\":\n            check_call(_LIB.MXRecordIOWriterCreate(self.uri, ctypes.byref(self.handle)))\n            self.writable = True\n        elif self.flag == \"r\":\n            check_call(_LIB.MXRecordIOReaderCreate(self.uri, ctypes.byref(self.handle)))\n            self.writable = False\n        else:\n            raise ValueError(f\"Invalid flag {self.flag}\")\n        # pylint: disable=not-callable\n        # It's bug from pylint(astroid). See https://github.com/PyCQA/pylint/issues/1699\n        self.pid = current_process().pid\n        self.is_open = True\n\n    def __del__(self):\n        self.close()\n\n    def __getstate__(self):\n        \"\"\"Override pickling behavior.\"\"\"\n        # pickling pointer is not allowed\n        is_open = self.is_open\n        self.close()\n        d = dict(self.__dict__)\n        d['is_open'] = is_open\n        uri = self.uri.value\n        try:\n            uri = uri.decode('utf-8')\n        except AttributeError:\n            pass\n        del d['handle']\n        d['uri'] = uri\n        return d\n\n    def __setstate__(self, d):\n        \"\"\"Restore from pickled.\"\"\"\n        self.__dict__ = d\n        is_open = d['is_open']\n        self.is_open = False\n        self.handle = RecordIOHandle()\n        self.uri = c_str(self.uri)\n        if is_open:\n            self.open()\n\n    def _check_pid(self, allow_reset=False):\n        \"\"\"Check process id to ensure integrity, reset if in new process.\"\"\"\n        # pylint: disable=not-callable\n        # It's bug from pylint(astroid). See https://github.com/PyCQA/pylint/issues/1699\n        if not self.pid == current_process().pid:\n            if allow_reset:\n                self.reset()\n            else:\n                raise RuntimeError(\"Forbidden operation in multiple processes\")\n\n    def close(self):\n        \"\"\"Closes the record file.\"\"\"\n        if not self.is_open:\n            return\n        if self.writable:\n            check_call(_LIB.MXRecordIOWriterFree(self.handle))\n        else:\n            check_call(_LIB.MXRecordIOReaderFree(self.handle))\n        self.is_open = False\n        self.pid = None\n\n    def reset(self):\n        \"\"\"Resets the pointer to first item.\n\n        If the record is opened with 'w', this function will truncate the file to empty.\n\n        Examples\n        ---------\n        >>> record = mx.recordio.MXRecordIO('tmp.rec', 'r')\n        >>> for i in range(2):\n        ...    item = record.read()\n        ...    print(item)\n        record_0\n        record_1\n        >>> record.reset()  # Pointer is reset.\n        >>> print(record.read()) # Started reading from start again.\n        record_0\n        >>> record.close()\n        \"\"\"\n        self.close()\n        self.open()\n\n    def write(self, buf):\n        \"\"\"Inserts a string buffer as a record.\n\n        Examples\n        ---------\n        >>> record = mx.recordio.MXRecordIO('tmp.rec', 'w')\n        >>> for i in range(5):\n        ...    record.write('record_%d'%i)\n        >>> record.close()\n\n        Parameters\n        ----------\n        buf : string (python2), bytes (python3)\n            Buffer to write.\n        \"\"\"\n        assert self.writable\n        self._check_pid(allow_reset=False)\n        check_call(_LIB.MXRecordIOWriterWriteRecord(self.handle,\n                                                    ctypes.c_char_p(buf),\n                                                    ctypes.c_size_t(len(buf))))\n\n    def read(self):\n        \"\"\"Returns record as a string.\n\n        Examples\n        ---------\n        >>> record = mx.recordio.MXRecordIO('tmp.rec', 'r')\n        >>> for i in range(5):\n        ...    item = record.read()\n        ...    print(item)\n        record_0\n        record_1\n        record_2\n        record_3\n        record_4\n        >>> record.close()\n\n        Returns\n        ----------\n        buf : string\n            Buffer read.\n        \"\"\"\n        assert not self.writable\n        # trying to implicitly read from multiple processes is forbidden,\n        # there's no elegant way to handle unless lock is introduced\n        self._check_pid(allow_reset=False)\n        buf = ctypes.c_char_p()\n        size = ctypes.c_size_t()\n        check_call(_LIB.MXRecordIOReaderReadRecord(self.handle,\n                                                   ctypes.byref(buf),\n                                                   ctypes.byref(size)))\n        if buf:\n            buf = ctypes.cast(buf, ctypes.POINTER(ctypes.c_char*size.value))\n            return buf.contents.raw\n        else:\n            return None\n\nclass MXIndexedRecordIO(MXRecordIO):\n    \"\"\"Reads/writes `RecordIO` data format, supporting random access.\n\n    Examples\n    ---------\n    >>> for i in range(5):\n    ...     record.write_idx(i, 'record_%d'%i)\n    >>> record.close()\n    >>> record = mx.recordio.MXIndexedRecordIO('tmp.idx', 'tmp.rec', 'r')\n    >>> record.read_idx(3)\n    record_3\n\n    Parameters\n    ----------\n    idx_path : str\n        Path to the index file.\n    uri : str\n        Path to the record file. Only supports seekable file types.\n    flag : str\n        'w' for write or 'r' for read.\n    key_type : type\n        Data type for keys.\n    \"\"\"\n    def __init__(self, idx_path, uri, flag, key_type=int):\n        self.idx_path = idx_path\n        self.idx = {}\n        self.keys = []\n        self.key_type = key_type\n        self.fidx = None\n        super(MXIndexedRecordIO, self).__init__(uri, flag)\n\n    def open(self):\n        super(MXIndexedRecordIO, self).open()\n        self.fidx = open(self.idx_path, self.flag)\n        if self.writable:\n            self.idx = {}\n            self.keys = []\n        elif not self.idx:\n            for line in iter(self.fidx.readline, ''):\n                line = line.strip().split('\\t')\n                key = self.key_type(line[0])\n                self.idx[key] = int(line[1])\n                self.keys.append(key)\n\n    def close(self):\n        \"\"\"Closes the record file.\"\"\"\n        if not self.is_open:\n            return\n        super(MXIndexedRecordIO, self).close()\n        self.fidx.close()\n\n    def __getstate__(self):\n        \"\"\"Override pickling behavior.\"\"\"\n        d = super(MXIndexedRecordIO, self).__getstate__()\n        d['fidx'] = None\n        return d\n\n    def seek(self, idx):\n        \"\"\"Sets the current read pointer position.\n\n        This function is internally called by `read_idx(idx)` to find the current\n        reader pointer position. It doesn't return anything.\"\"\"\n        assert not self.writable\n        self._check_pid(allow_reset=True)\n        pos = ctypes.c_size_t(self.idx[idx])\n        check_call(_LIB.MXRecordIOReaderSeek(self.handle, pos))\n\n    def tell(self):\n        \"\"\"Returns the current position of write head.\n\n        Examples\n        ---------\n        >>> record = mx.recordio.MXIndexedRecordIO('tmp.idx', 'tmp.rec', 'w')\n        >>> print(record.tell())\n        0\n        >>> for i in range(5):\n        ...     record.write_idx(i, 'record_%d'%i)\n        ...     print(record.tell())\n        16\n        32\n        48\n        64\n        80\n        \"\"\"\n        assert self.writable\n        pos = ctypes.c_size_t()\n        check_call(_LIB.MXRecordIOWriterTell(self.handle, ctypes.byref(pos)))\n        return pos.value\n\n    def read_idx(self, idx):\n        \"\"\"Returns the record at given index.\n\n        Examples\n        ---------\n        >>> record = mx.recordio.MXIndexedRecordIO('tmp.idx', 'tmp.rec', 'w')\n        >>> for i in range(5):\n        ...     record.write_idx(i, 'record_%d'%i)\n        >>> record.close()\n        >>> record = mx.recordio.MXIndexedRecordIO('tmp.idx', 'tmp.rec', 'r')\n        >>> record.read_idx(3)\n        record_3\n        \"\"\"\n        self.seek(idx)\n        return self.read()\n\n    def write_idx(self, idx, buf):\n        \"\"\"Inserts input record at given index.\n\n        Examples\n        ---------\n        >>> for i in range(5):\n        ...     record.write_idx(i, 'record_%d'%i)\n        >>> record.close()\n\n        Parameters\n        ----------\n        idx : int\n            Index of a file.\n        buf :\n            Record to write.\n        \"\"\"\n        key = self.key_type(idx)\n        pos = self.tell()\n        self.write(buf)\n        self.fidx.write(f'{str(key)}\\t{pos}\\n')\n        self.idx[key] = pos\n        self.keys.append(key)\n\n\nIRHeader = namedtuple('HEADER', ['flag', 'label', 'id', 'id2'])\n\"\"\"An alias for HEADER. Used to store metadata (e.g. labels) accompanying a record.\nSee mxnet.recordio.pack and mxnet.recordio.pack_img for example uses.\n\nParameters\n----------\n    flag : int\n        Available for convenience, can be set arbitrarily.\n    label : float or an array of float\n        Typically used to store label(s) for a record.\n    id: int\n        Usually a unique id representing record.\n    id2: int\n        Higher order bits of the unique id, should be set to 0 (in most cases).\n\"\"\"\n_IR_FORMAT = 'IfQQ'\n_IR_SIZE = struct.calcsize(_IR_FORMAT)\n\ndef pack(header, s):\n    \"\"\"Pack a string into MXImageRecord.\n\n    Parameters\n    ----------\n    header : IRHeader\n        Header of the image record.\n        ``header.label`` can be a number or an array. See more detail in ``IRHeader``.\n    s : str\n        Raw image string to be packed.\n\n    Returns\n    -------\n    s : str\n        The packed string.\n\n    Examples\n    --------\n    >>> label = 4 # label can also be a 1-D array, for example: label = [1,2,3]\n    >>> id = 2574\n    >>> header = mx.recordio.IRHeader(0, label, id, 0)\n    >>> with open(path, 'r') as file:\n    ...     s = file.read()\n    >>> packed_s = mx.recordio.pack(header, s)\n    \"\"\"\n    header = IRHeader(*header)\n    if isinstance(header.label, numbers.Number):\n        header = header._replace(flag=0)\n    else:\n        label = np.asarray(header.label, dtype=np.float32)\n        header = header._replace(flag=label.size, label=0)\n        s = label.tostring() + s\n    s = struct.pack(_IR_FORMAT, *header) + s\n    return s\n\ndef unpack(s):\n    \"\"\"Unpack a MXImageRecord to string.\n\n    Parameters\n    ----------\n    s : str\n        String buffer from ``MXRecordIO.read``.\n\n    Returns\n    -------\n    header : IRHeader\n        Header of the image record.\n    s : str\n        Unpacked string.\n\n    Examples\n    --------\n    >>> record = mx.recordio.MXRecordIO('test.rec', 'r')\n    >>> item = record.read()\n    >>> header, s = mx.recordio.unpack(item)\n    >>> header\n    HEADER(flag=0, label=14.0, id=20129312, id2=0)\n    \"\"\"\n    header = IRHeader(*struct.unpack(_IR_FORMAT, s[:_IR_SIZE]))\n    s = s[_IR_SIZE:]\n    if header.flag > 0:\n        header = header._replace(label=np.frombuffer(s, np.float32, header.flag))\n        s = s[header.flag*4:]\n    return header, s\n\ndef unpack_img(s, iscolor=-1):\n    \"\"\"Unpack a MXImageRecord to image.\n\n    Parameters\n    ----------\n    s : str\n        String buffer from ``MXRecordIO.read``.\n    iscolor : int\n        Image format option for ``cv2.imdecode``.\n\n    Returns\n    -------\n    header : IRHeader\n        Header of the image record.\n    img : numpy.ndarray\n        Unpacked image.\n\n    Examples\n    --------\n    >>> record = mx.recordio.MXRecordIO('test.rec', 'r')\n    >>> item = record.read()\n    >>> header, img = mx.recordio.unpack_img(item)\n    >>> header\n    HEADER(flag=0, label=14.0, id=20129312, id2=0)\n    >>> img\n    array([[[ 23,  27,  45],\n            [ 28,  32,  50],\n            ...,\n            [ 36,  40,  59],\n            [ 35,  39,  58]],\n           ...,\n           [[ 91,  92, 113],\n            [ 97,  98, 119],\n            ...,\n            [168, 169, 167],\n            [166, 167, 165]]], dtype=uint8)\n    \"\"\"\n    header, s = unpack(s)\n    img = np.frombuffer(s, dtype=np.uint8)\n    assert cv2 is not None\n    img = cv2.imdecode(img, iscolor)\n    return header, img\n\ndef pack_img(header, img, quality=95, img_fmt='.jpg'):\n    \"\"\"Pack an image into ``MXImageRecord``.\n\n    Parameters\n    ----------\n    header : IRHeader\n        Header of the image record.\n        ``header.label`` can be a number or an array. See more detail in ``IRHeader``.\n    img : numpy.ndarray\n        Image to be packed.\n    quality : int\n        Quality for JPEG encoding in range 1-100, or compression for PNG encoding in range 1-9.\n    img_fmt : str\n        Encoding of the image (.jpg for JPEG, .png for PNG).\n\n    Returns\n    -------\n    s : str\n        The packed string.\n\n    Examples\n    --------\n    >>> label = 4 # label can also be a 1-D array, for example: label = [1,2,3]\n    >>> id = 2574\n    >>> header = mx.recordio.IRHeader(0, label, id, 0)\n    >>> img = cv2.imread('test.jpg')\n    >>> packed_s = mx.recordio.pack_img(header, img)\n    \"\"\"\n    assert cv2 is not None\n    jpg_formats = ['.JPG', '.JPEG']\n    png_formats = ['.PNG']\n    encode_params = None\n    if img_fmt.upper() in jpg_formats:\n        encode_params = [cv2.IMWRITE_JPEG_QUALITY, quality]\n    elif img_fmt.upper() in png_formats:\n        encode_params = [cv2.IMWRITE_PNG_COMPRESSION, quality]\n\n    ret, buf = cv2.imencode(img_fmt, img, encode_params)\n    assert ret, 'failed to encode image'\n    return pack(header, buf.tostring())\n"
  },
  {
    "path": "python/mxnet/registry.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=no-member\n\n\"\"\"Registry for serializable objects.\"\"\"\n\nimport json\nimport warnings\n\nfrom .base import string_types\n\n_REGISTRY = {}\n\n\ndef get_registry(base_class):\n    \"\"\"Get a copy of the registry.\n\n    Parameters\n    ----------\n    base_class : type\n        base class for classes that will be registered.\n\n    Returns\n    -------\n    a registrator\n    \"\"\"\n    if base_class not in _REGISTRY:\n        _REGISTRY[base_class] = {}\n    return _REGISTRY[base_class].copy()\n\n\ndef get_register_func(base_class, nickname):\n    \"\"\"Get registrator function.\n\n    Parameters\n    ----------\n    base_class : type\n        base class for classes that will be reigstered\n    nickname : str\n        nickname of base_class for logging\n\n    Returns\n    -------\n    a registrator function\n    \"\"\"\n    if base_class not in _REGISTRY:\n        _REGISTRY[base_class] = {}\n    registry = _REGISTRY[base_class]\n\n    def register(klass, name=None):\n        \"\"\"Register functions\"\"\"\n        assert issubclass(klass, base_class), \\\n             f\"Can only register subclass of {base_class.__name__}\"\n        if name is None:\n            name = klass.__name__\n        name = name.lower()\n        if name in registry:\n            warnings.warn(\n                f\"\\033[91mNew {nickname} {klass.__module__}.{klass.__name__} registered with name {name} is\"\n                f\"overriding existing {nickname} {registry[name].__module__}.{registry[name].__name__}\\033[0m\",\n                UserWarning, stacklevel=2)\n        registry[name] = klass\n        return klass\n\n    register.__doc__ = f\"Register {nickname} to the {nickname} factory\"\n    return register\n\n\ndef get_alias_func(base_class, nickname):\n    \"\"\"Get registrator function that allow aliases.\n\n    Parameters\n    ----------\n    base_class : type\n        base class for classes that will be reigstered\n    nickname : str\n        nickname of base_class for logging\n\n    Returns\n    -------\n    a registrator function\n    \"\"\"\n    register = get_register_func(base_class, nickname)\n\n    def alias(*aliases):\n        \"\"\"alias registrator\"\"\"\n        def reg(klass):\n            \"\"\"registrator function\"\"\"\n            for name in aliases:\n                register(klass, name)\n            return klass\n        return reg\n    return alias\n\n\ndef get_create_func(base_class, nickname):\n    \"\"\"Get creator function\n\n    Parameters\n    ----------\n    base_class : type\n        base class for classes that will be reigstered\n    nickname : str\n        nickname of base_class for logging\n\n    Returns\n    -------\n    a creator function\n    \"\"\"\n    if base_class not in _REGISTRY:\n        _REGISTRY[base_class] = {}\n    registry = _REGISTRY[base_class]\n\n    def create(*args, **kwargs):\n        \"\"\"Create instance from config\"\"\"\n        if len(args):\n            name = args[0]\n            args = args[1:]\n        else:\n            name = kwargs.pop(nickname)\n\n        if isinstance(name, base_class):\n            assert len(args) == 0 and len(kwargs) == 0, \\\n                f\"{nickname} is already an instance. Additional arguments are invalid\"\n            return name\n\n        if isinstance(name, dict):\n            return create(**name)\n\n        assert isinstance(name, string_types), f\"{nickname} must be of string type\"\n\n        if name.startswith('['):\n            assert not args and not kwargs\n            name, kwargs = json.loads(name)\n            return create(name, **kwargs)\n        elif name.startswith('{'):\n            assert not args and not kwargs\n            kwargs = json.loads(name)\n            return create(**kwargs)\n\n        name = name.lower()\n        assert name in registry, \\\n            f\"{str(name)} is not registered. Please register with {nickname}.register first\"\n        return registry[name](*args, **kwargs)\n\n    create.__doc__ = f\"\"\"Create a {nickname} instance from config.\n\nParameters\n----------\n{nickname} : str or {base_class.__name__} instance\n    class name of desired instance. If is a instance,\n    it will be returned directly.\n**kwargs : dict\n    arguments to be passed to constructor\"\"\"\n\n    return create\n"
  },
  {
    "path": "python/mxnet/rtc.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Interface to runtime cuda kernel compile module.\"\"\"\n\nfrom array import array\nimport re\nimport ctypes\nimport numpy as np\n\nfrom .base import _LIB, mx_uint, c_array, c_array_buf, c_str_array, check_call\nfrom .base import c_str, CudaModuleHandle, CudaKernelHandle, numeric_types, string_types\nfrom .ndarray import dtype_np_to_mx, dtype_mx_to_np, NDArray\n\n_DTYPE_CPP_TO_NP = {\n    'float': np.float32,\n    'double': np.float64,\n    '__half': np.float16,\n    'uint8_t': np.uint8,\n    'int': np.int32,\n    'int32_t': np.int32,\n    'int8_t': np.int8,\n    'char': np.int8,\n    'int64_t': np.int64,\n}\n\nclass CudaModule(object):\n    r\"\"\"Compile and run CUDA code from Python.\n\n    In CUDA 7.5, you need to prepend your kernel definitions\n    with 'extern \"C\"' to avoid name mangling::\n\n        source = r'''\n        extern \"C\" __global__ void axpy(const float *x, float *y, float alpha) {\n            int i = threadIdx.x + blockIdx.x * blockDim.x;\n            y[i] += alpha * x[i];\n        }\n        '''\n        module = mx.rtc.CudaModule(source)\n        func = module.get_kernel(\"axpy\", \"const float *x, float *y, float alpha\")\n        x = mx.nd.ones((10,), ctx=mx.gpu(0))\n        y = mx.nd.zeros((10,), ctx=mx.gpu(0))\n        func.launch([x, y, 3.0], mx.gpu(0), (1, 1, 1), (10, 1, 1))\n        print(y)\n\n    Starting from CUDA 8.0, you can instead export functions by name.\n    This also allows you to use templates::\n\n        source = r'''\n        template<typename DType>\n        __global__ void axpy(const DType *x, DType *y, DType alpha) {\n            int i = threadIdx.x + blockIdx.x * blockDim.x;\n            y[i] += alpha * x[i];\n        }\n        '''\n        module = mx.rtc.CudaModule(source, exports=['axpy<float>', 'axpy<double>'])\n        func32 = module.get_kernel(\"axpy<float>\", \"const float *x, float *y, float alpha\")\n        x = mx.nd.ones((10,), dtype='float32', ctx=mx.gpu(0))\n        y = mx.nd.zeros((10,), dtype='float32', ctx=mx.gpu(0))\n        func32.launch([x, y, 3.0], mx.gpu(0), (1, 1, 1), (10, 1, 1))\n        print(y)\n\n        func64 = module.get_kernel(\"axpy<double>\", \"const double *x, double *y, double alpha\")\n        x = mx.nd.ones((10,), dtype='float64', ctx=mx.gpu(0))\n        y = mx.nd.zeros((10,), dtype='float64', ctx=mx.gpu(0))\n        func32.launch([x, y, 3.0], mx.gpu(0), (1, 1, 1), (10, 1, 1))\n        print(y)\n\n\n    Parameters\n    ----------\n    source : str\n        Complete source code.\n    options : tuple of str\n        Compiler flags. For example, use \"-I/usr/local/cuda/include\" to\n        add cuda headers to include path.\n    exports : tuple of str\n        Export kernel names.\n    \"\"\"\n    def __init__(self, source, options=(), exports=()):\n        if isinstance(options, string_types):\n            options = (options,)\n        if isinstance(exports, string_types):\n            exports = (exports,)\n        self.handle = CudaModuleHandle()\n        check_call(_LIB.MXRtcCudaModuleCreate(\n            c_str(source),\n            len(options),\n            c_str_array(options),\n            len(exports),\n            c_str_array(exports),\n            ctypes.byref(self.handle)))\n\n    def __del__(self):\n        check_call(_LIB.MXRtcCudaModuleFree(self.handle))\n\n    def get_kernel(self, name, signature):\n        r\"\"\"Get CUDA kernel from compiled module.\n\n        Parameters\n        ----------\n        name : str\n            String name of the kernel.\n        signature : str\n            Function signature for the kernel. For example, if a kernel is\n            declared as::\n\n                extern \"C\" __global__ void axpy(const float *x, double *y, int alpha)\n\n            Then its signature should be::\n\n                const float *x, double *y, int alpha\n\n            or::\n\n                const float *, double *, int\n\n            Note that `*` in signature marks an argument as array and\n            `const` marks an argument as constant (input) array.\n\n        Returns\n        -------\n        CudaKernel\n            CUDA kernels that can be launched on GPUs.\n        \"\"\"\n        hdl = CudaKernelHandle()\n        is_ndarray = []\n        is_const = []\n        dtypes = []\n        pattern = re.compile(r\"\"\"^(const)?\\s?([\\w_]+)\\s?(\\*)?\\s?([\\w_]+)?$\"\"\")\n        args = re.sub(r\"\\s+\", \" \", signature).split(\",\")\n        for arg in args:\n            sanitized_arg = \" \".join(arg.split())\n            match = pattern.match(sanitized_arg)\n            if not match or match.groups()[1] == 'const':\n                raise ValueError(\n                    f'Invalid function prototype \"{sanitized_arg}\". Must be in the '\n                    'form of \"(const) type (*) (name)\"')\n            is_const.append(bool(match.groups()[0]))\n            dtype = match.groups()[1]\n            is_ndarray.append(bool(match.groups()[2]))\n            if dtype not in _DTYPE_CPP_TO_NP:\n                raise TypeError(\n                    \"Unsupported kernel argument type {}. Supported types are: {}.\".format(\n                        sanitized_arg, ','.join(_DTYPE_CPP_TO_NP.keys())))\n            dtypes.append(dtype_np_to_mx(_DTYPE_CPP_TO_NP[dtype]))\n\n        check_call(_LIB.MXRtcCudaKernelCreate(\n            self.handle,\n            c_str(name),\n            len(dtypes),\n            c_array_buf(ctypes.c_int, array('i', is_ndarray)),\n            c_array_buf(ctypes.c_int, array('i', is_const)),\n            c_array_buf(ctypes.c_int, array('i', dtypes)),\n            ctypes.byref(hdl)))\n\n        return CudaKernel(hdl, name, is_ndarray, dtypes)\n\nclass CudaKernel(object):\n    \"\"\"Constructs CUDA kernel. Should be created by `CudaModule.get_kernel`,\n    not intended to be used by users.\n    \"\"\"\n    def __init__(self, handle, name, is_ndarray, dtypes):\n        self.handle = handle\n        self._name = name\n        self._is_ndarray = is_ndarray\n        self._dtypes = [dtype_mx_to_np(i) for i in dtypes]\n\n    def __del__(self):\n        check_call(_LIB.MXRtcCudaKernelFree(self.handle))\n\n    def launch(self, args, ctx, grid_dims, block_dims, shared_mem=0):\n        \"\"\"Launch cuda kernel.\n\n        Parameters\n        ----------\n        args : tuple of NDArray or numbers\n            List of arguments for kernel. NDArrays are expected for pointer\n            types (e.g. `float*`, `double*`) while numbers are expected for\n            non-pointer types (e.g. `int`, `float`).\n        ctx : Context\n            The context to launch kernel on. Must be GPU context.\n        grid_dims : tuple of 3 integers\n            Grid dimensions for CUDA kernel.\n        block_dims : tuple of 3 integers\n            Block dimensions for CUDA kernel.\n        shared_mem : integer, optional\n            Size of dynamically allocated shared memory. Defaults to 0.\n        \"\"\"\n        assert ctx.device_type == 'gpu', \"Cuda kernel can only be launched on GPU\"\n        assert len(grid_dims) == 3, \"grid_dims must be a tuple of 3 integers\"\n        assert len(block_dims) == 3, \"grid_dims must be a tuple of 3 integers\"\n        assert len(args) == len(self._dtypes), \\\n            f\"CudaKernel({self._name}) expects {len(self._dtypes)} arguments but got {len(args)}\"\n        void_args = []\n        ref_holder = []\n        for i, (arg, is_nd, dtype) in enumerate(zip(args, self._is_ndarray, self._dtypes)):\n            if is_nd:\n                assert isinstance(arg, NDArray), \\\n                    f\"The {i}-th argument is expected to be a NDArray but got {type(arg)}\"\n                void_args.append(arg.handle)\n            else:\n                assert isinstance(arg, numeric_types), \\\n                    f\"The {i}-th argument is expected to be a number, but got {type(arg)}\"\n                ref_holder.append(np.array(arg, dtype=dtype))\n                void_args.append(ref_holder[-1].ctypes.data_as(ctypes.c_void_p))\n\n        check_call(_LIB.MXRtcCudaKernelCall(\n            self.handle,\n            ctx.device_id,\n            c_array(ctypes.c_void_p, void_args),\n            mx_uint(grid_dims[0]), mx_uint(grid_dims[1]), mx_uint(grid_dims[2]),\n            mx_uint(block_dims[0]), mx_uint(block_dims[1]), mx_uint(block_dims[2]),\n            mx_uint(shared_mem)))\n"
  },
  {
    "path": "python/mxnet/runtime.py",
    "content": "# coding: utf-8\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=not-an-iterable\n\n\"\"\"Runtime querying of compile time features in the native library.\n\nWith this module you can check at runtime which libraries and features were compiled in the library.\n\nExample usage:\n\n.. code-block:: python\n\n    import mxnet\n    features=mxnet.runtime.Features()\n\n    features.is_enabled(\"CUDNN\")\n    False\n\n    features.is_enabled(\"CPU_SSE\")\n    True\n\n    print(features)\n    [✖ CUDA, ✖ CUDNN, ✖ NCCL, ✖ TENSORRT, ✔ CPU_SSE, ✔ CPU_SSE2, ✔ CPU_SSE3,\n    ✔ CPU_SSE4_1, ✔ CPU_SSE4_2, ✖ CPU_SSE4A, ✔ CPU_AVX, ✖ CPU_AVX2, ✔ OPENMP, ✖ SSE,\n    ✔ F16C, ✔ JEMALLOC, ✔ BLAS_OPEN, ✖ BLAS_ATLAS, ✖ BLAS_MKL, ✖ BLAS_APPLE, ✔ LAPACK,\n    ✖ ONEDNN, ✔ OPENCV, ✖ DIST_KVSTORE, ✖ INT64_TENSOR_SIZE, ✔ SIGNAL_HANDLER, ✔ DEBUG, ✖ TVM_OP]\n\n\n\"\"\"\n\nimport ctypes\nimport collections\nfrom .base import _LIB, check_call\n\nclass Feature(ctypes.Structure):\n    \"\"\"Compile time feature description, member fields: `name` and `enabled`.\"\"\"\n    _fields_ = [\n        (\"_name\", ctypes.c_char_p),\n        (\"_enabled\", ctypes.c_bool)\n    ]\n\n    @property\n    def name(self):\n        \"\"\"Feature name.\"\"\"\n        return self._name.decode()\n\n    @property\n    def enabled(self):\n        \"\"\"True if MXNet was compiled with the given compile-time feature.\"\"\"\n        return self._enabled\n\n    def __repr__(self):\n        if self.enabled:\n            return \"✔ {}\".format(self.name)\n        else:\n            return \"✖ {}\".format(self.name)\n\ndef feature_list():\n    \"\"\"Check the library for compile-time features. The list of features are maintained in libinfo.h and libinfo.cc\n\n    Returns\n    -------\n    list\n        List of :class:`.Feature` objects\n    \"\"\"\n    lib_features_c_array = ctypes.POINTER(Feature)()\n    lib_features_size = ctypes.c_size_t()\n    check_call(_LIB.MXLibInfoFeatures(ctypes.byref(lib_features_c_array), ctypes.byref(lib_features_size)))\n    features = [lib_features_c_array[i] for i in range(lib_features_size.value)]\n    return features\n\nclass Features(collections.OrderedDict):\n    \"\"\"OrderedDict of name to Feature\"\"\"\n    instance = None\n    def __new__(cls):\n        if cls.instance is None:\n            cls.instance = super(Features, cls).__new__(cls)\n            super(Features, cls.instance).__init__([(f.name, f) for f in feature_list()])\n        return cls.instance\n\n    def __repr__(self):\n        return str(list(self.values()))\n\n    def is_enabled(self, feature_name):\n        \"\"\"Check for a particular feature by name\n\n        Parameters\n        ----------\n        feature_name: str\n            The name of a valid feature as string for example 'CUDA'\n\n        Returns\n        -------\n        Boolean\n            True if it's enabled, False if it's disabled, RuntimeError if the feature is not known\n        \"\"\"\n        feature_name = feature_name.upper()\n        if feature_name not in self:\n            raise RuntimeError(\"Feature '{}' is unknown, known features are: {}\".format(\n                feature_name, list(self.keys())))\n        return self[feature_name].enabled\n\ndef get_branch():\n    out = ctypes.c_char_p()\n    check_call(_LIB.MXGetBranch(ctypes.byref(out)))\n    return out.value.decode('utf-8')\n\ndef get_commit_hash():\n    out = ctypes.c_char_p()\n    check_call(_LIB.MXGetCommitHash(ctypes.byref(out)))\n    return out.value.decode('utf-8')\n"
  },
  {
    "path": "python/mxnet/symbol/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Symbol API of MXNet.\"\"\"\n\nfrom . import _internal, contrib, linalg, op, random, sparse, image, symbol, numpy\n# pylint: disable=wildcard-import, redefined-builtin\ntry:\n    from .gen_op import * # pylint: disable=unused-wildcard-import\nexcept ImportError:\n    pass\nfrom . import register\nfrom .op import *\nfrom .symbol import *\n# pylint: enable=wildcard-import\nfrom . import numpy as np\nfrom . import numpy_extension as npx\n\n__all__ = op.__all__ + symbol.__all__\\\n          + ['contrib', 'linalg', 'random', 'sparse', 'image', 'numpy', 'numpy_extension']\n"
  },
  {
    "path": "python/mxnet/symbol/_internal.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=wildcard-import, unused-import\n\"\"\"Symbol namespace used to register internal functions.\"\"\"\n# Use different version of SymbolBase\n# When possible, use cython to speedup part of computation.\nimport sys as _sys\nimport os as _os\n\ntry:\n    if int(_os.environ.get(\"MXNET_ENABLE_CYTHON\", True)) == 0:\n        from .._ctypes.symbol import SymbolBase, _set_symbol_class, _set_np_symbol_class\n        from .._ctypes.symbol import _symbol_creator\n    else:\n        from .._cy3.symbol import SymbolBase, _set_symbol_class, _set_np_symbol_class\n        from .._cy3.symbol import _symbol_creator\nexcept ImportError:\n    if int(_os.environ.get(\"MXNET_ENFORCE_CYTHON\", False)) != 0:\n        raise ImportError(\"Cython Module cannot be loaded but MXNET_ENFORCE_CYTHON=1\")\n    from .._ctypes.symbol import SymbolBase, _set_symbol_class, _set_np_symbol_class\n    from .._ctypes.symbol import _symbol_creator\nfrom ..attribute import AttrScope\nfrom ..base import _Null\nfrom ..name import NameManager\ntry:\n    from .gen__internal import * # pylint: disable=unused-wildcard-import\nexcept ImportError:\n    pass\n\n__all__ = ['SymbolBase', '_set_symbol_class', '_symbol_creator', '_set_np_symbol_class']\n"
  },
  {
    "path": "python/mxnet/symbol/contrib.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import, unused-wildcard-import,redefined-outer-name\n\"\"\"Contrib Symbol API of MXNet.\"\"\"\nimport math\nimport ctypes\nimport copy\n\nfrom .random import uniform\nfrom .symbol import Symbol\ntry:\n    from .gen_contrib import *\nexcept ImportError:\n    pass\n\nfrom . import symbol\nfrom ..base import _LIB, check_call\nfrom ..base import SymbolHandle, _as_list\nfrom ..attribute import AttrScope, current as current_attribute\n\n__all__ = [\"rand_zipfian\", \"foreach\", \"while_loop\", \"cond\"]\n\ndef rand_zipfian(true_classes, num_sampled, range_max):\n    \"\"\"Draw random samples from an approximately log-uniform or Zipfian distribution.\n\n    This operation randomly samples *num_sampled* candidates the range of integers [0, range_max).\n    The elements of sampled_candidates are drawn with replacement from the base distribution.\n\n    The base distribution for this operator is an approximately log-uniform or Zipfian distribution:\n\n    P(class) = (log(class + 2) - log(class + 1)) / log(range_max + 1)\n\n    This sampler is useful when the true classes approximately follow such a distribution.\n    For example, if the classes represent words in a lexicon sorted in decreasing order of \\\n    frequency. If your classes are not ordered by decreasing frequency, do not use this op.\n\n    Additionaly, it also returns the number of times each of the \\\n    true classes and the sampled classes is expected to occur.\n\n    Parameters\n    ----------\n    true_classes : Symbol\n        The target classes in 1-D.\n    num_sampled: int\n        The number of classes to randomly sample.\n    range_max: int\n        The number of possible classes.\n\n    Returns\n    -------\n    samples: Symbol\n        The sampled candidate classes in 1-D `int64` dtype.\n    expected_count_true: Symbol\n        The expected count for true classes in 1-D `float64` dtype.\n    expected_count_sample: Symbol\n        The expected count for sampled candidates in 1-D `float64` dtype.\n\n    Examples\n    --------\n    >>> true_cls = mx.sym.Variable('true_cls')\n    >>> samples, exp_count_true, exp_count_sample = mx.sym.contrib.rand_zipfian(true_cls, 4, 5)\n    >>> samples.eval(true_cls=mx.nd.array([3]))[0].asnumpy()\n    array([1, 3, 3, 3])\n    >>> exp_count_true.eval(true_cls=mx.nd.array([3]))[0].asnumpy()\n    array([0.12453879])\n    >>> exp_count_sample.eval(true_cls=mx.nd.array([3]))[0].asnumpy()\n    array([0.22629439, 0.12453879, 0.12453879, 0.12453879])\n    \"\"\"\n    assert(isinstance(true_classes, Symbol)), f\"unexpected type {type(true_classes)}\"\n    log_range = math.log(range_max + 1)\n    rand = uniform(0, log_range, shape=(num_sampled,), dtype='float64')\n    # make sure sampled_classes are in the range of [0, range_max)\n    sampled_classes = (rand.exp() - 1).astype('int64') % range_max\n\n    true_classes = true_classes.astype('float64')\n    expected_prob_true = ((true_classes + 2.0) / (true_classes + 1.0)).log() / log_range\n    expected_count_true = expected_prob_true * num_sampled\n    # cast sampled classes to fp64 to avoid interget division\n    sampled_cls_fp64 = sampled_classes.astype('float64')\n    expected_prob_sampled = ((sampled_cls_fp64 + 2.0) / (sampled_cls_fp64 + 1.0)).log() / log_range\n    expected_count_sampled = expected_prob_sampled * num_sampled\n    return sampled_classes, expected_count_true, expected_count_sampled\n\n\ndef _flatten(args, inout_str):\n    if isinstance(args, symbol.Symbol):\n        length = len(args.list_outputs())\n        length = length if length > 1 else 0\n        return [args], int(length)\n\n    assert isinstance(args, (list, tuple)), \\\n        f\"{inout_str} must be (nested) list of Symbol, \" \\\n        f\"but got {str(args)} of type {str(type(args))}\"\n    flat = []\n    fmts = []\n    for i in args:\n        arg, fmt = _flatten(i, inout_str)\n        flat.extend(arg)\n        fmts.append(fmt)\n    return flat, fmts\n\n\ndef _regroup(args, fmt):\n    if isinstance(fmt, int):\n        if fmt == 0:\n            return args[0], args[1:]\n        return args[:fmt], args[fmt:]\n\n    assert isinstance(args, (list, tuple)), \\\n        \"output must be (nested) list of Symbol, \" \\\n        f\"but got {str(args)} of type {str(type(args))}\"\n    ret = []\n    for i in fmt:\n        res, args = _regroup(args, i)\n        ret.append(res)\n    return ret, args\n\n\n# We want to generate a unique name for input symbols to a control flow\n# operator. The names are generated on purpose differently from the symbols\n# cut from the graph.\ndef _get_sym_uniq_name(sym):\n    return '{}-{}'.format(sym.name, sym.attr('_value_index'))\n\ndef _get_graph_inputs(subg):\n    num_handles = ctypes.c_int(0)\n    handles = ctypes.POINTER(SymbolHandle)()\n    check_call(_LIB.MXSymbolGetInputSymbols(subg.handle, ctypes.byref(handles),\n                                            ctypes.byref(num_handles)))\n\n    syms = []\n    for i in range(num_handles.value):\n        s = Symbol(ctypes.cast(handles[i], SymbolHandle))\n        syms.append(s)\n    return syms\n\ndef _cut_subgraph(subg):\n    num_handles = ctypes.c_int(0)\n    handles = ctypes.POINTER(SymbolHandle)()\n    check_call(_LIB.MXSymbolCutSubgraph(subg.handle, ctypes.byref(handles),\n                                        ctypes.byref(num_handles)))\n\n    syms = []\n    for i in range(num_handles.value):\n        s = Symbol(ctypes.cast(handles[i], SymbolHandle))\n        syms.append(s)\n    return syms\n\ndef _get_unique_subgraph_name(subgraph_name):\n    attrs = current_attribute()._attr\n    if attrs.get(\"__subgraph_name__\", \"\") != \"\":\n        subgraph_name = \"\".join([attrs[\"__subgraph_name__\"], \"$\", subgraph_name])\n    AttrScope._subgraph_names[subgraph_name] += 1\n    subgraph_name = subgraph_name + str(AttrScope._subgraph_names[subgraph_name] - 1)\n    return subgraph_name\n\n# This construct a subgraph for given output nodes.\n# If an output node is one of the input nodes, we call identity to make sure\n# that outputs nodes are different from input nodes.\ndef _construct_subgraph(sym_out, sym_states, name):\n    sym_out = _as_list(sym_out)\n    sym_states = _as_list(sym_states)\n    all_outputs = []\n    all_outputs.extend(sym_out)\n    all_outputs.extend(sym_states)\n    g = symbol.Group(all_outputs)\n\n    flat_out = []\n    all_input_names = g.list_inputs()\n    output_names = {o.name for o in sym_out}\n    for o in sym_out:\n        if o.name in all_input_names or o.list_attr().get(\"__subgraph_name__\", \"\") != name:\n            flat_out.append(symbol.op.identity(o))\n        else:\n            flat_out.append(o)\n\n    for s in sym_states:\n        if s.name in all_input_names or s.name in output_names or \\\n           s.list_attr().get(\"__subgraph_name__\", \"\") != name:\n            flat_out.append(symbol.op.identity(s))\n        else:\n            flat_out.append(s)\n    return symbol.Group(flat_out)\n\ndef _check_data(inputs, in_type, msg):\n    is_NDArray_or_list = True\n    if isinstance(inputs, list):\n        for i in inputs:\n            if not isinstance(i, in_type):\n                is_NDArray_or_list = False\n                break\n    else:\n        is_NDArray_or_list = isinstance(inputs, in_type)\n    assert is_NDArray_or_list, msg\n\ndef foreach(body, data, init_states, name=\"foreach\"):\n    \"\"\"Run a for loop with user-defined computation over Symbols on dimension 0.\n\n    This operator simulates a for loop and body has the computation for an iteration\n    of the for loop. It runs the computation in body on each slice from the input\n    NDArrays.\n\n    body takes two arguments as input and outputs a tuple of two elements,\n    as illustrated below:\n\n    out, states = body(data1, states)\n\n    data1 can be either a symbol or a list of symbols. If data is a symbol,\n    data1 is a symbol. Otherwise, data1 is a list of symbols and has the same\n    size as data. states is a list of symbols and have the same size as init_states.\n    Similarly, out can be either a symbol or a list of symbols, which are concatenated\n    as the first output of foreach; states from the last execution of body\n    are the second output of foreach.\n\n    foreach can output only output data or states. If a user only wants states,\n    the body function can return ([], states). Similarly, if a user only wants\n    output data, the body function can return (out, []).\n\n    The computation done by this operator is equivalent to the pseudo code below\n    when the input data is NDArray::\n\n        states = init_states\n        outs = []\n        for i in data.shape[0]:\n            s = data[i]\n            out, states = body(s, states)\n            outs.append(out)\n        outs = stack(*outs)\n\n\n    Parameters\n    ----------\n    body : a Python function.\n        Define computation in an iteration.\n    data: a symbol or a list of symbols.\n        The input data.\n    init_states: a Symbol or nested lists of symbols.\n        The initial values of the loop states.\n    name: string.\n        The name of the operator.\n\n    Returns\n    -------\n    outputs: a Symbol or nested lists of Symbols.\n        The output data concatenated from the output of all iterations.\n    states: a Symbol or nested lists of Symbols.\n        The loop states in the last iteration.\n\n    Examples\n    --------\n    >>> step = lambda data, states: (data + states[0], [states[0] * 2])\n    >>> data = mx.sym.var('data')\n    >>> states = [mx.sym.var('state')]\n    >>> outs, states = mx.sym.contrib.foreach(step, data, states)\n    \"\"\"\n\n    flatten_data, data_fmt = _flatten(data, \"foreach input\")\n    _check_data(flatten_data, symbol.Symbol,\n                \"data should be a symbol or a nested list of symbols\")\n    init_flatten_states, init_state_fmt = _flatten(init_states, \"foreach states\")\n    _check_data(init_flatten_states, symbol.Symbol,\n                \"init_states should be a symbol or a nested list of symbols\")\n\n    # If the input python function references to the symbols outside\n    # the python function, we need to prune the computation graph constructed from\n    # the function. One way of doing it is to mark the nodes in the computation graph\n    # with AttrScope and prune the nodes without the special attribute.\n    name = _get_unique_subgraph_name(name)\n    with AttrScope(__subgraph_name__=name):\n        in_eles = [symbol.var(_get_sym_uniq_name(sym)) for sym in flatten_data]\n        in_eles, _ = _regroup(in_eles, data_fmt)\n        states = [symbol.var(_get_sym_uniq_name(s)) for s in init_flatten_states]\n        states, _ = _regroup(states, copy.deepcopy(init_state_fmt))\n        sym_out, sym_states = body(in_eles, states)\n\n        sym_out, out_fmt = _flatten(sym_out, \"foreach output\")\n        sym_states, state_fmt = _flatten(sym_states, \"foreach loop_vars\")\n        assert init_state_fmt == state_fmt, \"The input and output loop_vars have different format\"\n        _check_data(sym_out, symbol.Symbol,\n                    \"the output should be an NDArray or a nested list of NDArrays\")\n        _check_data(sym_states, symbol.Symbol,\n                    \"the output states should be an NDArray or a nested list of NDArrays\")\n        num_out_data = len(sym_out)\n        num_states = len(sym_states)\n        num_outputs = num_out_data + num_states\n        g = _construct_subgraph(sym_out, sym_states, name)\n\n    input_syms = _get_graph_inputs(g)\n    cut_syms = _cut_subgraph(g)\n    input_syms = _get_graph_inputs(g)\n\n    # Here we need to find out how the input symbols are ordered as well as\n    # where the loop states are located in the list of inputs.\n\n    # This dict contains the symbols of the subgraph.\n    input_syms = {sym.name:sym for sym in input_syms}\n    gin_names = input_syms.keys()\n    # This array contains the symbols for the inputs of foreach.\n    # They are ordered according to the inputs of the subgraph.\n    state_names = [_get_sym_uniq_name(sym) for sym in init_flatten_states]\n    data_names = [_get_sym_uniq_name(sym) for sym in flatten_data]\n    cut_var_map = {sym.list_outputs()[0]:sym for sym in cut_syms}\n    cut_var_names = cut_var_map.keys()\n\n    subg_input_names = g.list_inputs()\n    assert len(set(subg_input_names)) == len(subg_input_names), \\\n            \"The inputs of the subgraph don't have unique names: \" + str(subg_input_names)\n    # ordered_ins contains input symbols in the following order:\n    # data_syms, state_syms, followed by cut_vars and vars in the closure.\n    ordered_ins = [x for x in flatten_data]\n    # this defines the location of data_syms in the list of subgraph inputs\n    in_data_locs = []\n    for dname in data_names:\n        # Some data may not be used.\n        if dname in subg_input_names:\n            in_data_locs.append(subg_input_names.index(dname))\n        else:\n            raise AssertionError(\"the data arrays have to be used in the loop body\")\n\n    ordered_ins.extend(init_flatten_states)\n    # this defines the location of state_syms in the list of subgraph inputs.\n    in_state_locs = []\n    for sname in state_names:\n        # Some state may not be used.\n        if sname in subg_input_names:\n            in_state_locs.append(subg_input_names.index(sname))\n        else:\n            raise AssertionError(\"the state arrays have to be used in the loop body\")\n\n    remain_locs = []\n    for in_name in subg_input_names:\n        assert in_name in gin_names, f\"The input variable {in_name} can't be found in graph inputs: {str(gin_names)}\"\n        if in_name in cut_var_names:\n            ordered_ins.append(cut_var_map[in_name])\n            remain_locs.append(subg_input_names.index(in_name))\n        elif in_name not in data_names and in_name not in state_names:\n            # The remaining inputs are the variable nodes created inside the UDF.\n            # The subgraph can't have nodes shared with the main graph. As such,\n            # we need to make a copy of these variable nodes.\n            assert in_name in gin_names\n            ordered_ins.append(copy.deepcopy(input_syms[in_name]))\n            remain_locs.append(subg_input_names.index(in_name))\n\n    ret = symbol._internal._foreach(g, *ordered_ins, num_outputs=num_outputs,\n                                    num_out_data=num_out_data, in_state_locs=in_state_locs,\n                                    in_data_locs=in_data_locs, remain_locs=remain_locs)\n    outs = []\n    for i in range(num_outputs - num_states):\n        outs.append(ret[i])\n    outs, _ = _regroup(outs, out_fmt)\n    states = []\n    for i in range(num_states):\n        states.append(ret[num_outputs - num_states + i])\n    states, _ = _regroup(states, state_fmt)\n\n    return (outs, states)\n\ndef while_loop(cond, func, loop_vars, max_iterations=None, name=\"while_loop\"):\n    \"\"\"Run a while loop with user-defined computation and loop condition.\n\n    This operator simulates a while loop which iterately does customized computation\n    as long as the condition is satisfied.\n\n    `loop_vars` is a Symbol or nested lists of Symbols on which the computation uses.\n\n    `cond` is a user-defined function, used as the loop condition.\n    It consumes `loop_vars`, and produces a scalar MXNet symbol,\n    indicating the termination of the loop.\n    The loop ends when `cond` returns false (zero).\n    The `cond` is variadic, and its signature should be\n    `cond(*loop_vars) => Symbol`.\n\n    `func` is a user-defined function, used as the loop body.\n    It also consumes `loop_vars`, and produces `step_output` and `new_loop_vars` at each step.\n    In each step, `step_output` should contain the same number elements.\n    Through all steps, the i-th element of `step_output` should have the same shape and dtype.\n    Also, `new_loop_vars` should contain the same number of elements as `loop_vars`,\n    and the corresponding element should have the same shape and dtype.\n    The `func` is variadic, and its signature should be\n    `func(*loop_vars) =>\n    (Symbol or nested List[Symbol] step_output, Symbol or nested List[Symbol] new_loop_vars)`.\n\n    `max_iterations` is a scalar that defines the maximum number of iterations allowed.\n\n    This function returns two lists.\n    The first list has the length of `|step_output|`,\n    in which the i-th element are all i-th elements of\n    `step_output` from all steps, stacked along axis 0.\n    The second list has the length of `|loop_vars|`,\n    which represents final states of loop variables.\n\n    .. warning::\n\n       For now, the axis 0 of all Symbols in the first list are `max_iterations`,\n       due to lack of dynamic shape inference.\n\n    .. warning::\n\n       Even if `cond` is never satisfied,\n       while_loop returns a list of outputs with inferred dtype and shape.\n       This is different from the Symbol version,\n       where in this case `step_outputs` are assumed as an empty list.\n\n    Parameters\n    ----------\n    cond: a Python function.\n        The loop condition.\n    func: a Python function.\n        The loop body.\n    loop_vars: a Symbol or nested lists of Symbol.\n        The initial values of the loop variables.\n    max_iterations: a python int.\n        Maximum number of iterations.\n\n    Returns\n    ------\n    outputs: a Symbol or nested lists of Symbols\n        stacked output from each step\n    states: a Symbol or nested lists of Symbols\n        final state\n\n    Examples\n    --------\n    >>> cond = lambda i, s: i <= 5\n    >>> func = lambda i, s: ([i + s], [i + 1, s + i])\n    >>> loop_vars = (mx.sym.var('i'), mx.sym.var('s'))\n    >>> outputs, states = mx.sym.contrib.while_loop(cond, func, loop_vars, max_iterations=10)\n    \"\"\"\n    def _to_python_scalar(inputs, type_, name):\n        \"\"\"Converts \"inputs\", possibly typed mxnet NDArray, a numpy ndarray, other python types,\n        to the given type\n        \"\"\"\n        if hasattr(inputs, \"asscalar\"):\n            inputs = inputs.asscalar()\n        try:\n            inputs = type_(inputs)\n        except:\n            raise ValueError(f\"Cannot convert {name} to python {type_.__name__}\")\n        return inputs\n\n    def _cond_wrapper(loop_vars):\n        result = cond(*loop_vars)\n        if not isinstance(result, Symbol):\n            raise ValueError(\"Return of cond must be a Symbol\")\n        return [], [result], [], []\n\n    def _func_wrapper(loop_vars):\n        \"\"\"This wrapper unifies\n             \"func: loop_vars -> new_loop_vars\"\n         and \"func: loop_vars -> (step_output, new_loop_vars)\"\n        into \"func: loop_vars -> (list of step_outputs, tuple of new_loop_vars)\n        \"\"\"\n        step_output, new_loop_vars = func(*loop_vars)\n        if step_output is None:\n            step_output = []\n        if new_loop_vars is None:\n            new_loop_vars = []\n        if isinstance(step_output, tuple):\n            step_output = list(step_output)\n        if isinstance(new_loop_vars, tuple):\n            new_loop_vars = list(new_loop_vars)\n        step_output, out_fmt = _flatten(step_output, \"while output\")\n        new_loop_vars, var_fmt = _flatten(new_loop_vars, \"while loop_vars\")\n        if len(loop_vars) != len(new_loop_vars):\n            raise ValueError(\"The number of loop_vars should be consistent during the loop\")\n        return step_output, new_loop_vars, out_fmt, var_fmt\n\n    def _create_subgraph(graph_vars, graph_func, subgraph_name):\n        subgraph_name = _get_unique_subgraph_name(subgraph_name)\n        with AttrScope(__subgraph_name__=subgraph_name):\n            # create new variables with the same name,\n            # them feed them to the given func\n            graph_vars, var_fmt = _flatten(graph_vars, \"while loop_vars\")\n            new_graph_vars = [symbol.var(_get_sym_uniq_name(sym)) for sym in graph_vars]\n            new_graph_vars, _ = _regroup(new_graph_vars, var_fmt)\n            outputs, final_state, out_fmt, var_fmt = graph_func(new_graph_vars)\n            # first `num_out_data` elements belong to `outputs`\n            # other elements belong to `final_state`\n            num_out_data = len(outputs)\n            num_outputs = len(outputs) + len(final_state)\n            # nnvm cut-graph does not allow inputs and outputs overlap\n            # so we calculate the name of inputs, and copy outputs once it overlaps with inputs\n            # group all outputs of graph_func\n            all_input_names = symbol.Group(outputs + final_state).list_inputs()\n            in_input = lambda x: x.name in all_input_names\n            in_graph = lambda x: x.list_attr().get(\"__subgraph_name__\", \"\") == subgraph_name\n            make_identity = lambda x: symbol.op.identity(x) if in_input(x) or not in_graph(x) \\\n                                      else x\n            graph = symbol.Group(list(map(make_identity, outputs + final_state)))\n        return graph, num_out_data, num_outputs, out_fmt, var_fmt\n\n    flatten_loop_vars, init_loop_var_fmt = _flatten(loop_vars, \"while loop_vars\")\n    _check_data(flatten_loop_vars, symbol.Symbol,\n                \"loop_vars should be a symbol or a nested list of symbols\")\n\n    def _union_inputs(*graphs):\n        # Given a list of graphs, each whose inputs are either from loop_vars or other variables.\n        # 1) calculate a list `inputs`, the union of their inputs.\n        # 2) for each graph, determine in which indices their inputs reside in `inputs`\n        # 3) for each variable in the input of `graph`, find which index it is\n        inputs = []             # List[Symbol], result of 1)\n        locs = []               # List[Tuple(List[Int], List[Int])], a list of tuples,\n                                # where tuples are results of 2) and 3)\n        input_id_to_loc = {}    # Dict[int, int], given id(sym), input_id_to_loc maps it\n                                # to a `loc`, where inputs[loc] = sym\n        for graph in graphs:\n            # some loop_vars are inputs to `graph`, some are not\n            name_to_loop_vars = {_get_sym_uniq_name(sym): sym for sym in flatten_loop_vars}\n            # other inputs to `graph` created by cut_graph\n            name_to_cut_g_syms = {sym.list_outputs()[0]: sym for sym in _cut_subgraph(graph)}\n            # input_syms: all inputs to the `graph`\n            name_to_input_syms = {sym.name: sym for sym in _get_graph_inputs(graph)}\n            # also we collect the mapping from var's name to var's loc in loop_vars\n            name_to_var_locs = {_get_sym_uniq_name(sym): i for i, sym in enumerate(flatten_loop_vars)}\n            # collect arguments for each subgraph\n            input_locs = []                         # results from the second step\n            var_locs = [-1] * len(flatten_loop_vars)        # results from the third step\n            subg_input_names = graph.list_inputs()\n            assert len(set(subg_input_names)) == len(subg_input_names), \\\n                    \"The inputs of the subgraph don't have unique names: \" + str(subg_input_names)\n            for name in subg_input_names:\n                assert name in name_to_input_syms   # it should obviously hold\n                # name -> sym\n                if name in name_to_loop_vars:\n                    sym = name_to_loop_vars[name]\n                elif name in name_to_cut_g_syms:\n                    sym = name_to_cut_g_syms[name]\n                else:\n                    sym = copy.deepcopy(name_to_input_syms[name])\n                # do 2), and 1) is implicitly done\n                if id(sym) in input_id_to_loc:\n                    loc = input_id_to_loc[id(sym)]\n                else:\n                    loc = len(input_id_to_loc)\n                    inputs.append(sym)\n                    input_id_to_loc[id(sym)] = loc\n                input_locs.append(loc)\n                # do 3)\n                if name in name_to_var_locs:\n                    var_locs[name_to_var_locs[name]] = len(input_locs) - 1\n            locs.append((input_locs, var_locs))\n        return inputs, locs\n    if max_iterations is None:\n        raise ValueError(\"max_iterations should be specified\")\n    max_iterations = _to_python_scalar(max_iterations, int, \"max_iteration\")\n    # It should be work as fine if loop_vars are empty I guess,\n    # but it is semantically unnecessary to include this case.\n    if len(loop_vars) == 0:\n        raise ValueError(\"loop_vars should contain at least one element\")\n    # create graph for `cond'\n    cond_g, num_out_data, num_outputs, _, _ = \\\n        _create_subgraph(loop_vars, _cond_wrapper, name + \"_cond\")\n    assert num_out_data == 0\n    assert num_outputs == 1\n    # create graph for `func`\n    func_g, num_out_data, num_outputs, out_fmt, _ = \\\n        _create_subgraph(loop_vars, _func_wrapper, name + \"_func\")\n    # find symbols used in either cond_g or func_g\n    input_syms, ((cond_input_locs, _), (func_input_locs, func_var_locs)) = \\\n        _union_inputs(cond_g, func_g)\n    for i_th, loc in enumerate(func_var_locs, 1):\n        if loc == -1:\n            raise ValueError(f\"The {i_th}-th loop_var doesn't involve into the computation\")\n    result = symbol._internal._while_loop(\n        cond_g,\n        func_g,\n        *input_syms,\n        max_iterations=max_iterations,\n        cond_input_locs=cond_input_locs,\n        func_input_locs=func_input_locs,\n        func_var_locs=func_var_locs,\n        num_out_data=num_out_data,\n        num_outputs=num_outputs\n    )\n    outputs = [result[i] for i in range(num_out_data)]\n    outputs, _ = _regroup(outputs, out_fmt)\n    final_loop_vars = [result[i] for i in range(num_out_data, num_outputs)]\n    final_loop_vars, _ = _regroup(final_loop_vars, init_loop_var_fmt)\n    return outputs, final_loop_vars\n\ndef cond(pred, then_func, else_func, name=\"cond\"):\n    \"\"\"Run an if-then-else using user-defined condition and computation\n\n    This operator simulates a if-like branch which chooses to do one of\n    the two customized computations according to the specified condition.\n\n    `pred` is a scalar MXNet Symbol,\n    indicating which branch of computation should be used.\n\n    `then_func` is a user-defined function, used as computation of the then branch.\n    It produces `outputs`, which is a list of Symbols.\n    The signature of `then_func` should be\n    `then_func() => nested List[Symbol]`.\n\n    `else_func` is a user-defined function, used as computation of the else branch.\n    It produces `outputs`, which is a list of Symbols.\n    The signature of `else_func` should be\n    `else_func() => nested List[Symbol]`.\n\n    The `outputs` produces by `then_func` and `else_func` should have the same number\n    of elements, all of which should be in the same shape, of the same dtype and stype.\n\n    This function returns a list of symbols, representing the computation result.\n\n    Parameters\n    ----------\n    pred: a MXNet Symbol representing a scalar.\n        The branch condition.\n    then_func: a Python function.\n        The computation to be executed if `pred` is true.\n    else_func: a Python function.\n        The computation to be executed if `pred` is false.\n\n    Returns\n    -------\n    outputs: a Symbol or nested lists of Symbols, representing the result of computation.\n\n    Examples\n    --------\n    >>> a, b = mx.sym.var('a'), mx.sym.var('b')\n    >>> pred = a * b < 5\n    >>> then_func = lambda: (a + 5) * (b + 5)\n    >>> else_func = lambda: (a - 5) * (b - 5)\n    >>> outputs = mx.sym.contrib.cond(pred, then_func, else_func)\n    \"\"\"\n\n    def _create_subgraph(graph_vars, graph_func, subgraph_name):\n        subgraph_name = _get_unique_subgraph_name(subgraph_name)\n        with AttrScope(__subgraph_name__=subgraph_name):\n            # create new variables with the same name,\n            # them feed them to the given func\n            new_graph_vars = [symbol.var(sym.name) for sym in graph_vars]\n            outputs = graph_func(*new_graph_vars)\n            outputs, out_fmt = _flatten(outputs, \"cond outputs\")\n            num_outputs = len(outputs)\n            # nnvm cut-graph does not allow inputs and outputs overlap\n            # so we calculate the name of inputs, and copy outputs once it overlaps with inputs\n            # group all outputs of graph_func\n            all_input_names = symbol.Group(outputs).list_inputs()\n            in_input = lambda x: x.name in all_input_names\n            in_graph = lambda x: x.list_attr().get(\"__subgraph_name__\", \"\") == subgraph_name\n            make_identity = lambda x: symbol.op.identity(x) if in_input(x) or not in_graph(x) \\\n                                      else x\n            graph = symbol.Group(list(map(make_identity, outputs)))\n        return graph, num_outputs, out_fmt\n\n    def _union_inputs(*graphs):\n        # Given a list of graphs, each whose inputs are either from input_vars or other variables.\n        # 1) calculate a list `inputs`, the union of their inputs.\n        # 2) for each graph, determine in which indices their inputs reside in `inputs`\n        # 3) for each variable in the input of `graph`, find which index it is\n        inputs = []             # List[Symbol], result of 1)\n        locs = []               # List[Tuple(List[Int], List[Int])], a list of tuples,\n                                # where tuples are results of 2) and 3)\n        input_id_to_loc = {}    # Dict[int, int], given id(sym), input_id_to_loc maps it\n                                # to a `loc`, where inputs[loc] = sym\n        for graph in graphs:\n            # some input_vars are inputs to `graph`, some are not\n            name_to_input_vars = {sym.name: sym for sym in inputs}\n            # other inputs to `graph` created by cut_graph\n            name_to_cut_g_syms = {sym.list_outputs()[0]: sym for sym in _cut_subgraph(graph)}\n            # input_syms: all inputs to the `graph`\n            name_to_input_syms = {sym.name: sym for sym in _get_graph_inputs(graph)}\n            # collect arguments for each subgraph\n            input_locs = []                         # results from the second step\n            for name in graph.list_inputs():\n                assert name in name_to_input_syms   # it should obviously hold\n                # name -> sym\n                if name in name_to_input_vars:\n                    sym = name_to_input_vars[name]\n                elif name in name_to_cut_g_syms:\n                    sym = name_to_cut_g_syms[name]\n                else:\n                    sym = copy.deepcopy(name_to_input_syms[name])\n                # do 2), and 1) is implicitly done\n                if id(sym) in input_id_to_loc:\n                    loc = input_id_to_loc[id(sym)]\n                else:\n                    loc = len(input_id_to_loc)\n                    inputs.append(sym)\n                    input_id_to_loc[id(sym)] = loc\n                input_locs.append(loc)\n            locs.append(input_locs)\n        return inputs, locs\n    inputs = []\n    # create graph for `cond_func'\n    cond_g, cond_num_outputs, _ = _create_subgraph(inputs, lambda: pred, name + \"_pred\")\n    if cond_num_outputs != 1:\n        raise ValueError(\"pred should always be a single output\")\n    # create graph for `then`\n    then_g, then_num_outputs, then_fmt = _create_subgraph(inputs, then_func, name + \"_then\")\n    # create graph for `else`\n    else_g, else_num_outputs, _ = _create_subgraph(inputs, else_func, name + \"_else\")\n    if then_num_outputs != else_num_outputs:\n        raise ValueError(\"Number of outputs differs between then-branch and else-branch\")\n    # find symbols used in either cond_g or func_g\n    input_syms, (cond_input_locs, then_input_locs, else_input_locs) = \\\n        _union_inputs(cond_g, then_g, else_g)\n    result = symbol._internal._cond(\n        # [cond, then_g, else_g, *input_syms]\n        cond_g,\n        then_g,\n        else_g,\n        *input_syms,\n        cond_input_locs=cond_input_locs,\n        then_input_locs=then_input_locs,\n        else_input_locs=else_input_locs,\n        num_outputs=then_num_outputs\n    )\n    outputs = [result[i] for i in range(then_num_outputs)]\n    outputs, _ = _regroup(outputs, then_fmt)\n    return outputs\n\ndef adamw_update(weight, grad, mean, var, rescale_grad, lr, eta, beta1=0.9, beta2=0.999,\n                 epsilon=1e-8, wd=0, clip_gradient=-1, out=None, name=None, **kwargs):\n    if not isinstance(rescale_grad, Symbol):\n        rescale_grad = symbol.full(shape=(1,), val=rescale_grad)\n    return symbol._internal._adamw_update(weight=weight, grad=grad, mean=mean, var=var,\n                                          rescale_grad=rescale_grad, lr=lr, eta=eta,\n                                          beta1=beta1, beta2=beta2, epsilon=epsilon,\n                                          wd=wd, clip_gradient=clip_gradient, out=out,\n                                          name=name, **kwargs)\n\ndef mp_adamw_update(weight, grad, mean, var, weight32, rescale_grad, lr, eta, beta1=0.9,\n                    beta2=0.999, epsilon=1e-8, wd=0, clip_gradient=-1, out=None,\n                    name=None, **kwargs):\n    if not isinstance(rescale_grad, Symbol):\n        rescale_grad = symbol.full(shape=(1,), val=rescale_grad)\n    return symbol._internal._mp_adamw_update(weight=weight, grad=grad, mean=mean, var=var,\n                                             weight32=weight32,\n                                             rescale_grad=rescale_grad, lr=lr, eta=eta,\n                                             beta1=beta1, beta2=beta2, epsilon=epsilon,\n                                             wd=wd, clip_gradient=clip_gradient, out=out,\n                                             name=name, **kwargs)\n"
  },
  {
    "path": "python/mxnet/symbol/image.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import, unused-wildcard-import\n\"\"\"Image Symbol API of MXNet.\"\"\"\ntry:\n    from .gen_image import *\nexcept ImportError:\n    pass\n\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/symbol/linalg.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import, unused-wildcard-import\n\"\"\"Linear Algebra Symbol API of MXNet.\"\"\"\ntry:\n    from .gen_linalg import *\nexcept ImportError:\n    pass\n\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/symbol/numpy/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Module for numpy ops under mxnet.symbol.\"\"\"\n\nfrom . import random\nfrom . import linalg\nfrom . import _op, _symbol, _internal\nfrom ._symbol import _Symbol\nfrom . import _register\nfrom ._op import *  # pylint: disable=wildcard-import\nfrom ._symbol import *  # pylint: disable=wildcard-import\n\n__all__ = _op.__all__ + _symbol.__all__\n"
  },
  {
    "path": "python/mxnet/symbol/numpy/_internal.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for numpy internal ops.\"\"\"\n\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/symbol/numpy/_op.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for operators used in Gluon dispatched by F=symbol module.\"\"\"\n\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/symbol/numpy/_register.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Registering numpy ops.\"\"\"\n\nfrom ...base import _init_np_op_module\nfrom ..register import _make_symbol_function\n\n_init_np_op_module(root_module_name='mxnet', np_module_name='numpy',\n                   mx_module_name='symbol', make_op_func=_make_symbol_function)\n\n\n_init_np_op_module(root_module_name='mxnet', np_module_name='numpy._internal',\n                   mx_module_name='symbol', make_op_func=_make_symbol_function)\n"
  },
  {
    "path": "python/mxnet/symbol/numpy/_symbol.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=too-many-lines, unused-argument\n\"\"\"numpy namespace for operators used in Gluon APIs dispatched by F=symbol module.\"\"\"\n\nimport ctypes\nimport numpy as _np\nfrom . import _op as _mx_np_op\nfrom ...base import _LIB, SymbolHandle, numeric_types, mx_uint, integer_types, string_types\nfrom ...base import c_str\nfrom ...base import py_str\nfrom ...util import check_call, set_module, _sanity_check_params\nfrom ...util import wrap_np_unary_func, wrap_np_binary_func\nfrom ...util import is_np_default_dtype\nfrom ...context import current_context\nfrom ..symbol import Symbol, Group\nfrom .._internal import _set_np_symbol_class\nfrom . import _internal as _npi\ntry:\n    from __builtin__ import slice as py_slice\nexcept ImportError:\n    from builtins import slice as py_slice\n\n__all__ = ['zeros', 'zeros_like', 'ones', 'ones_like', 'full', 'full_like', 'empty_like', 'bitwise_not', 'invert',\n           'delete', 'add', 'broadcast_to', 'subtract', 'multiply', 'divide', 'mod', 'remainder', 'fmod',\n           'power', 'arctan2', 'trace', 'transpose', 'copy', 'moveaxis', 'reshape', 'dot',\n           'sin', 'cos', 'tan', 'sinh', 'cosh', 'tanh', 'log10', 'sqrt', 'cbrt', 'abs', 'absolute', 'fabs', 'exp',\n           'expm1', 'arcsin', 'arccos', 'arctan', 'sign', 'log', 'degrees', 'log2', 'log1p', 'matmul', 'median',\n           'rint', 'radians', 'reciprocal', 'square', 'negative', 'fix', 'ceil', 'floor', 'histogram', 'insert',\n           'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'argsort', 'sort', 'tensordot', 'eye', 'linspace',\n           'logspace', 'expand_dims', 'tile', 'arange', 'array_split', 'split', 'hsplit', 'vsplit', 'dsplit',\n           'concatenate', 'append', 'stack', 'vstack', 'row_stack', 'column_stack', 'hstack', 'dstack',\n           'average', 'mean', 'maximum', 'fmax', 'minimum', 'fmin', 'any', 'all', 'around', 'round', 'round_',\n           'flatnonzero', 'tril_indices', 'amax', 'amin', 'max', 'min', 'logical_and', 'logical_or', 'logical_xor',\n           'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'indices', 'copysign', 'ravel', 'unravel_index',\n           'diag_indices_from', 'hanning', 'hamming', 'blackman', 'flip', 'flipud', 'fliplr',\n           'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm', 'gcd', 'interp',\n           'tril', 'triu', 'tri', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'cross', 'kron',\n           'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum',\n           'true_divide', 'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d',\n           'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',\n           'atleast_1d', 'atleast_2d', 'atleast_3d', 'squeeze',\n           'where', 'bincount', 'rollaxis', 'diagflat', 'repeat', 'prod', 'pad', 'cumsum', 'sum', 'diag', 'diagonal']\n\n\n@set_module('mxnet.symbol.numpy')\nclass _Symbol(Symbol):\n    def __getitem__(self, key): # pylint: disable = too-many-return-statements, inconsistent-return-statements\n        \"\"\"Return self[key].\n\n        If the symbol is a symbol list, it returns the i-th symbol or a list of symbols\n        selected by key.\n\n        Otherwise, it outputs a symbol that slice the input by the given key. Currently, this\n        function supports the following types of key:\n\n        - integer types, e.g., int, long, np.int32, np.int64\n        - slice containing integer constants, e.g., slice(0, None, None)\n        - tuple contaning the above elements, which is used for multidimensional indexing\n\n        Parameters\n        ----------\n        key : int, slice, or tuple of all previous types\n            Indexing key.\n\n        \"\"\"\n        num_outputs = self.num_outputs\n        if num_outputs > 1:\n            num_outputs = self.num_outputs\n            if isinstance(key, integer_types):\n                key = int(key)\n                if key < -num_outputs or key >= num_outputs:\n                    raise IndexError('list index out of range')\n                if key < 0:\n                    key += num_outputs\n                ret_handle = SymbolHandle()\n                check_call(_LIB.MXSymbolGetOutput(self.handle, mx_uint(key),\n                                                  ctypes.byref(ret_handle)))\n                return _Symbol(handle=ret_handle)\n            elif isinstance(key, py_slice):\n                start, stop, step = key.indices(num_outputs)\n                return Group([self[i] for i in range(start, stop, step)], _Symbol)\n            else:\n                raise TypeError('indices of symbol group must be integers or slices, not {}'\n                                .format(type(key)))\n        else:\n            all = __builtins__['all']  # pylint: disable=redefined-outer-name\n            if isinstance(key, integer_types):\n                if key == -1:\n                    sliced = _npi.slice(self, [key], [None])\n                else:\n                    sliced = _npi.slice(self, [key], [key+1])\n                return _npi.reshape(sliced, (-3, -4))\n            elif isinstance(key, py_slice):\n                if key.step is None or key.step != 0:\n                    start = [None] if key.start is None else key.start\n                    stop = [None] if key.stop is None else key.stop\n                    return _npi.slice(self, start, stop, key.step)\n                else:\n                    raise ValueError(\"slice step cannot be zero\")\n            elif isinstance(key, Symbol):\n                return _npi.advanced_indexing(self, key)\n            elif isinstance(key, tuple) and len(key) == 0:\n                return self\n            elif isinstance(key, tuple) and all(isinstance(k, Symbol) for k in key):\n                key = _npi.stack(*[i for i in key])\n                sliced = _npi.advanced_indexing_multiple(self, key)\n                return sliced\n            elif isinstance(key, tuple):\n                begin = []\n                end = []\n                step = []\n                new_shape = ()\n                assert len(key)  # len(key) == 0 handled above\n                for index in key:\n                    if isinstance(index, py_slice):\n                        if index.step is not None and index.step == 0:\n                            raise ValueError(\"slice step cannot be zero\")\n                        begin.append(index.start)\n                        end.append(index.stop)\n                        step.append(index.step)\n                        new_shape += (-2,)\n                    elif isinstance(index, integer_types):\n                        if index >= 0:\n                            begin.append(index)\n                            end.append(index+1)\n                            step.append(1)\n                        else:\n                            begin.append(index)\n                            end.append(index - 1)\n                            step.append(-1)\n                        new_shape += (-3,)\n                    else:\n                        raise IndexError('Only integer, slice, symbol or tuple of these types'\n                                         ' are supported! Received key={}'.format(key))\n                new_shape += (-4,)\n                sliced = _npi.slice(self, begin, end, step)\n                return _npi.reshape(sliced, new_shape)\n            else:\n                raise IndexError('Only integer, slice, tuple or Symbol of these types are supported! '\n                                 'Received key={}'.format(key))\n\n    def __setitem__(self, key, value):\n        raise NotImplementedError\n\n    def __repr__(self):\n        \"\"\"Gets a string representation of the symbol.\"\"\"\n        if self._alive:\n            if self.num_outputs > 1:\n                name = ', '.join([str(ele_sym) for ele_sym in self])\n                return f'<{self.__class__.__name__} group [{name}]>'\n            else:\n                return f'<{self.__class__.__name__} {self.name}>'\n        else:\n            return '<FREED {}>'.format(self.__class__.__name__)\n\n    @property\n    def name(self):\n        \"\"\"Gets name string from the symbol, this function only works for symbols\n         that are not a list (grouped symbols).\n\n        Returns\n        -------\n        value : str\n            The name of this symbol, returns ``None`` for list symbol.\n        \"\"\"\n        if self.num_outputs > 1:\n            raise AttributeError('This is a Group Symbol that contains {} elements and'\n                                 ' does not have a name. Use str(sym) to print the name of '\n                                 'all the elements instead.'.format(self.num_outputs))\n        ret = ctypes.c_char_p()\n        success = ctypes.c_int()\n        check_call(_LIB.MXSymbolGetName(\n            self.handle, ctypes.byref(ret), ctypes.byref(success)))\n        assert success.value != 0,\\\n            'Fail to infer the name of a symbol that is not a list!'\n        return py_str(ret.value)\n\n    def __iter__(self):\n        if self.num_outputs == 1:\n            raise TypeError(\"'{}' is not iterable.\".format(self))\n        return iter((self[i] for i in range(self.num_outputs)))\n\n    def __add__(self, other):\n        \"\"\"x.__add__(y) <=> x + y\"\"\"\n        return add(self, other)\n\n    def __invert__(self):\n        \"\"\"x.__invert__() <=> ~x\"\"\"\n        return invert(self)\n\n    def __and__(self, other):\n        \"\"\"x.__and__(y) <=> x & y\"\"\"\n        return bitwise_and(self, other)\n\n    def __or__(self, other):\n        \"\"\"x.__or__(y) <=> x | y\"\"\"\n        return bitwise_or(self, other)\n\n    def __xor__(self, other):\n        \"\"\"x.__xor__(y) <=> x ^ y\"\"\"\n        return bitwise_xor(self, other)\n\n    def __round__(self, n=0):\n        \"\"\"x.__round__(n)\"\"\"\n        return round(self, decimals=n)\n\n    def __abs__(self):\n        \"\"\"x.__abs__()\"\"\"\n        return absolute(self)\n\n    def __ceil__(self):\n        \"\"\"x.__ceil__()\"\"\"\n        return ceil(self)\n\n    def __floor__(self):\n        \"\"\"x.__floor__()\"\"\"\n        return floor(self)\n\n    def __trunc__(self):\n        \"\"\"x.__trunc__()\"\"\"\n        return trunc(self)\n\n    def __sub__(self, other):\n        \"\"\"x.__sub__(y) <=> x - y\"\"\"\n        return subtract(self, other)\n\n    def __rsub__(self, other):\n        \"\"\"x.__rsub__(y) <=> y - x\"\"\"\n        return subtract(other, self)\n\n    def __mul__(self, other):\n        \"\"\"x.__mul__(y) <=> x * y\"\"\"\n        return multiply(self, other)\n\n    def __rmul__(self, other):\n        \"\"\"x.__rmul__(y) <=> y * x\"\"\"\n        return multiply(other, self)\n\n    def __div__(self, other):\n        \"\"\"x.__truediv__(y) <=> x / y\"\"\"\n        return divide(self, other)\n\n    def __rdiv__(self, other):\n        \"\"\"x.__rdiv__(y) <=> y / x\"\"\"\n        return divide(other, self)\n\n    def __mod__(self, other):\n        \"\"\"x.__mod__(y) <=> x % y\"\"\"\n        return mod(self, other)\n\n    def __rmod__(self, other):\n        \"\"\"x.__rmod__(y) <=> y % x\"\"\"\n        return mod(other, self)\n\n    def __idiv__(self, other):\n        raise NotImplementedError\n\n    def __truediv__(self, other):\n        \"\"\"x.__truediv__(y) <=> x / y\"\"\"\n        return divide(self, other)\n\n    def __rtruediv__(self, other):\n        \"\"\"x.__rtruediv__(y) <=> y / x\"\"\"\n        return divide(other, self)\n\n    def __itruediv__(self, other):\n        raise NotImplementedError\n\n    def __pow__(self, other):\n        \"\"\"x.__pow__(y) <=> x ** y\"\"\"\n        return power(self, other)\n\n    def __rpow__(self, other):\n        return power(other, self)\n\n    def __neg__(self):\n        \"\"\"x.__neg__() <=> - x\"\"\"\n        return negative(self)\n\n    def __deepcopy__(self, _):\n        return super().__deepcopy__(_).as_np_ndarray()\n\n    def __eq__(self, other):\n        \"\"\"x.__eq__(y) <=> x == y\"\"\"\n        return equal(self, other)\n\n    def __ne__(self, other):\n        \"\"\"x.__ne__(y) <=> x != y\"\"\"\n        return not_equal(self, other)\n\n    def __gt__(self, other):\n        \"\"\"x.__gt__(y) <=> x > y\"\"\"\n        return greater(self, other)\n\n    def __ge__(self, other):\n        \"\"\"x.__ge__(y) <=> x >= y\"\"\"\n        return greater_equal(self, other)\n\n    def __lt__(self, other):\n        \"\"\"x.__lt__(y) <=> x < y\"\"\"\n        return less(self, other)\n\n    def __le__(self, other):\n        \"\"\"x.__le__(y) <=> x <= y\"\"\"\n        return less_equal(self, other)\n\n    def __len__(self):\n        if self.num_outputs == 1:\n            raise TypeError('{} is not a list and does not support len().'.format(self))\n        return self.num_outputs\n\n    @property\n    def num_outputs(self):\n        \"\"\"The number of outputs of a symbol. If the symbol is not a symbollist, it returns 1.\n        Otherwise, it returns the number of elements of the list.\"\"\"\n        output_count = mx_uint()\n        check_call(_LIB.MXSymbolGetNumOutputs(self.handle, ctypes.byref(output_count)))\n        return output_count.value\n\n    def as_nd_ndarray(self):\n        \"\"\"Convert _Symbol to mxnet.symbol.Symbol to use its convenience fluent methods.\"\"\"\n        hdl = SymbolHandle()\n        check_call(_LIB.MXShallowCopySymbol(self.handle, ctypes.byref(hdl)))\n        return Symbol(handle=hdl)\n\n    def as_np_ndarray(self):\n        \"\"\"For the convenience of conversion between legacy and np symbols.\"\"\"\n        return self\n\n    @property\n    # pylint: disable= invalid-name, undefined-variable\n    def T(self):\n        \"\"\"Same as self.transpose().\"\"\"\n        return self.transpose()\n    # pylint: enable= invalid-name, undefined-variable\n\n    def astype(self, dtype, order='K', casting='unsafe', subok=True, copy=True):  # pylint: disable=arguments-differ,unused-argument,too-many-arguments,redefined-outer-name\n        \"\"\"\n        Copy of the array, cast to a specified type.\n\n        Parameters\n        ----------\n        dtype : str or dtype\n            Typecode or data-type to which the array is cast.\n        order : {'C', 'F', 'A', 'K'}, optional\n            Controls the memory layout order of the result.\n            'C' means C order, 'F' means Fortran order, 'A'\n            means 'F' order if all the arrays are Fortran contiguous,\n            'C' order otherwise, and 'K' means as close to the\n            order the array elements appear in memory as possible.\n            Default is 'K'.\n        casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional\n            Controls what kind of data casting may occur. Defaults to 'unsafe'\n            for backwards compatibility.\n\n              * 'no' means the data types should not be cast at all.\n              * 'equiv' means only byte-order changes are allowed.\n              * 'safe' means only casts which can preserve values are allowed.\n              * 'same_kind' means only safe casts or casts within a kind,\n                like float64 to float32, are allowed.\n              * 'unsafe' means any data conversions may be done.\n        subok : bool, optional\n            If True, then sub-classes will be passed-through (default), otherwise\n            the returned array will be forced to be a base-class array.\n        copy : bool, optional\n            Default `True`. By default, astype always returns a newly\n            allocated ndarray on the same context. If this is set to\n            `False`, and the dtype requested is the same as the ndarray's\n            dtype, the ndarray is returned instead of a copy.\n\n        Returns\n        -------\n        arr_t : ndarray\n            Unless `copy` is False and the other conditions for returning the input\n            array are satisfied (see description for `copy` input parameter), `arr_t`\n            is a new array of the same shape as the input array with `dtype`.\n\n        Notes\n        -----\n        This function differs from the official `ndarray`'s ``astype`` function in the following\n        aspects:\n            - `order` only supports 'C' and 'K'.\n            - `casting` only supports 'unsafe'.\n            - `subok` only supports ``True``.\n        \"\"\"\n        if order is not None and order != 'K' and order != 'C':\n            raise ValueError('order must be either \\'K\\' or \\'C\\'')\n        if casting != 'unsafe':\n            raise ValueError('casting must be equal to \\'unsafe\\'')\n        if not subok:\n            raise ValueError('subok must be equal to True')\n        return _npi.cast(self, dtype=dtype)\n\n    def dot(self, b, out=None):\n        \"\"\"Dot product of two arrays.\n        Refer to ``numpy.dot`` for full documentation.\"\"\"\n        return _npi.dot(self, b, out=out)\n\n    def reshape(self, *args, **kwargs):  # pylint: disable=arguments-differ\n        \"\"\"Returns a copy of the array with a new shape.\n\n        Notes\n        -----\n        Unlike the free function `mxnet.numpy.reshape`, this method on `ndarray` allows\n        the elements of the shape parameter to be passed in as separate arguments.\n        For example, ``a.reshape(10, 11)`` is equivalent to\n        ``a.reshape((10, 11))``.\n        \"\"\"\n        order = 'C'\n        if len(kwargs) > 1:\n            raise TypeError('function takes at most 1 keyword argument')\n        if len(kwargs) == 1:\n            if 'order' not in kwargs:\n                raise TypeError('{} is an invalid keyword argument for this function'\n                                .format(kwargs.keys()[0]))\n            order = kwargs.pop('order', 'C')\n            if order != 'C':\n                raise NotImplementedError('only supports C-order,'\n                                          ' while received {}'.format(order))\n        if len(args) == 0:\n            raise TypeError('reshape() takes exactly 1 argument (0 given)')\n        if len(args) == 1 and isinstance(args[0], tuple):\n            return _mx_np_op.reshape(self, newshape=args[0], order=order)\n        else:\n            return _mx_np_op.reshape(self, newshape=args, order=order)\n\n    def argmax(self, axis=None, out=None):  # pylint: disable=arguments-differ\n        \"\"\"Return indices of the maximum values along the given axis.\n        Refer to `mxnet.numpy.argmax` for full documentation.\"\"\"\n        return argmax(self, axis, out)\n\n    def reshape_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`reshape_like`.\n\n        The arguments are the same as for :py:func:`reshape_like`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute reshape_like')\n\n    def zeros_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`zeros_like`.\n\n        The arguments are the same as for :py:func:`zeros_like`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute zeros_like')\n\n    def ones_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`ones_like`.\n\n        The arguments are the same as for :py:func:`ones_like`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute ones_like')\n\n    def broadcast_axes(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`broadcast_axes`.\n\n        The arguments are the same as for :py:func:`broadcast_axes`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute broadcast_like')\n\n    def repeat(self, repeats, axis=None):  # pylint: disable=arguments-differ\n        \"\"\"Repeat elements of an array.\"\"\"\n        return repeat(self, repeats=repeats, axis=axis)\n\n    def pad(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`pad`.\n\n        The arguments are the same as for :py:func:`pad`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute pad')\n\n    def swapaxes(self, axis1, axis2):  # pylint: disable=arguments-differ\n        \"\"\"Return a copy of the array with axis1 and axis2 interchanged.\n        Refer to `mxnet.numpy.swapaxes` for full documentation.\n        \"\"\"\n        return swapaxes(self, axis1, axis2)\n\n    def split(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`split`.\n\n        The arguments are the same as for :py:func:`split`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute split')\n\n    def split_v2(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`split_v2`.\n\n        The arguments are the same as for :py:func:`split_v2`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute split_v2')\n\n    def slice(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`slice`.\n\n        The arguments are the same as for :py:func:`slice`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute slice')\n\n    def slice_axis(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`slice_axis`.\n\n        The arguments are the same as for :py:func:`slice_axis`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute slice_axis')\n\n    def slice_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`slice_like`.\n\n        The arguments are the same as for :py:func:`slice_like`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute slice_like')\n\n    def take(self, indices, axis=None, mode='raise'):  # pylint: disable=arguments-differ, redefined-outer-name\n        \"\"\"Convenience fluent method for :py:func:`take`.\n\n        The arguments are the same as for :py:func:`take`, with\n        this array as data.\n        \"\"\"\n        return take(self, indices, axis, mode=mode)\n\n    def one_hot(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`one_hot`.\n\n        The arguments are the same as for :py:func:`one_hot`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute one_hot')\n\n    def pick(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`pick`.\n\n        The arguments are the same as for :py:func:`pick`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute pick')\n\n    def sort(self, axis=-1, kind=None, order=None):  # pylint: disable=arguments-differ\n        \"\"\"Convenience fluent method for :py:func:`sort`.\n\n        The arguments are the same as for :py:func:`sort`, with\n        this array as data.\n        \"\"\"\n        raise sort(self, axis=axis, kind=kind, order=order)\n\n    def topk(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`topk`.\n\n        The arguments are the same as for :py:func:`topk`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute topk')\n\n    def argsort(self, axis=-1, kind=None, order=None):  # pylint: disable=arguments-differ\n        \"\"\"Convenience fluent method for :py:func:`argsort`.\n\n        The arguments are the same as for :py:func:`argsort`, with\n        this array as data.\n        \"\"\"\n        return argsort(self, axis=axis, kind=kind, order=order)\n\n    def argmax_channel(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`argmax_channel`.\n\n        The arguments are the same as for :py:func:`argmax_channel`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute argmax_channel')\n\n    def argmin(self, axis=None, out=None):  # pylint: disable=arguments-differ\n        \"\"\"Return indices of the minimum values along the given axis.\n        Refer to `mxnet.numpy.argmax` for full documentation.\"\"\"\n        return argmin(self, axis, out)\n\n    def clip(self, min=None, max=None, out=None):  # pylint: disable=arguments-differ, redefined-outer-name\n        \"\"\"Return an array whose values are limited to [min, max].\n        One of max or min must be given.\n        \"\"\"\n        return clip(self, min, max, out=out)\n\n    def abs(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`abs`.\n\n        The arguments are the same as for :py:func:`abs`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute abs')\n\n    def sign(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sign`.\n\n        The arguments are the same as for :py:func:`sign`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute abs')\n\n    def flatten(self, order='C'):  # pylint: disable=arguments-differ\n        \"\"\"Return a copy of the array collapsed into one dimension.\"\"\"\n        return self.reshape(-1, order=order)\n\n    def shape_array(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`shape_array`.\n\n        The arguments are the same as for :py:func:`shape_array`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute shape_array')\n\n    def size_array(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`size_array`.\n\n        The arguments are the same as for :py:func:`size_array`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute size_array')\n\n    def expand_dims(self, *args, **kwargs):  # pylint: disable=arguments-differ,unused-argument\n        \"\"\"Convenience fluent method for :py:func:`expand_dims`.\n\n        The arguments are the same as for :py:func:`expand_dims`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute expand_dims')\n\n    def tile(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`tile`.\n\n        The arguments are the same as for :py:func:`tile`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute tile')\n\n    def transpose(self, *axes):  # pylint: disable=arguments-differ\n        \"\"\"The arguments are the same as for :py:func:`transpose`, with\n        this array as data.\n        \"\"\"\n        if len(axes) == 0:\n            axes = None\n        elif len(axes) == 1:\n            if isinstance(axes[0], (tuple, list)):\n                axes = axes[0]\n            elif axes[0] is None:\n                axes = None\n        return transpose(self, axes=axes)\n\n    def flip(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`flip`.\n\n        The arguments are the same as for :py:func:`flip`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute flip')\n\n    def depth_to_space(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`depth_to_space`.\n\n        The arguments are the same as for :py:func:`depth_to_space`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute depth_to_space')\n\n    def space_to_depth(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`space_to_depth`.\n\n        The arguments are the same as for :py:func:`space_to_depth`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute space_to_depth')\n\n    def diag(self, k=0, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`diag`.\n\n        The arguments are the same as for :py:func:`diag`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute diag')\n\n    def diagonal(self, offset=0, axis1=0, axis2=1):  # pylint: disable=arguments-differ\n        \"\"\"Return the diagonal with the given offset.\n\n        If array has more than two dimensions, then the axes specified by axis1 and\n        axis2 are used to determine the 2-D sub-array whose diagonal is returned.\n\n        Refer to `mxnet.symbol.numpy.diagonal` for full documents.\n        \"\"\"\n        return diagonal(self, offset=offset, axis1=axis1, axis2=axis2)\n\n    def sum(self, axis=None, dtype=None, out=None, keepdims=False):  # pylint: disable=arguments-differ\n        \"\"\"Return the sum of the array elements over the given axis.\"\"\"\n        return _npi.sum(self, axis=axis, dtype=dtype, out=out, keepdims=keepdims)\n\n    def nansum(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`nansum`.\n\n        The arguments are the same as for :py:func:`nansum`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute nansum')\n\n    def prod(self, axis=None, dtype=None, out=None, keepdims=False):  # pylint: disable=arguments-differ\n        \"\"\"Return the product of the array elements over the given axis.\"\"\"\n        return _mx_np_op.prod(self, axis=axis, dtype=dtype, keepdims=keepdims, out=out)\n\n    def nanprod(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`nanprod`.\n\n        The arguments are the same as for :py:func:`nanprod`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute nanprod')\n\n    def mean(self, axis=None, dtype=None, out=None, keepdims=False):  # pylint: disable=arguments-differ\n        \"\"\"Returns the average of the array elements along given axis.\"\"\"\n        return mean(self, axis=axis, dtype=dtype, out=out, keepdims=keepdims)\n\n    def std(self, axis=None, dtype=None, out=None, ddof=0, keepdims=False):  # pylint: disable=arguments-differ,too-many-arguments\n        \"\"\"Returns the standard deviation of the array elements along given axis.\"\"\"\n        return std(self, axis=axis, dtype=dtype, ddof=ddof, keepdims=keepdims, out=out)\n\n    def var(self, axis=None, dtype=None, out=None, ddof=0, keepdims=False):  # pylint: disable=arguments-differ,too-many-arguments\n        \"\"\"Returns the variance of the array elements, along given axis.\"\"\"\n        return var(self, axis=axis, dtype=dtype, out=out, ddof=ddof, keepdims=keepdims)\n\n    def cumsum(self, axis=None, dtype=None, out=None):\n        \"\"\"Return the cumulative sum of the elements along the given axis.\"\"\"\n        return _npi.cumsum(self, axis=axis, dtype=dtype, out=out)\n\n    def max(self, axis=None, out=None, keepdims=False):  # pylint: disable=arguments-differ\n        \"\"\"Return the maximum along a given axis.\"\"\"\n        return _npi.max(self, axis=axis, keepdims=keepdims, out=out)\n\n    def min(self, axis=None, out=None, keepdims=False):  # pylint: disable=arguments-differ\n        \"\"\"Return the minimum along a given axis.\"\"\"\n        return _npi.min(self, axis=axis, keepdims=keepdims, out=out)\n\n    def norm(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`norm`.\n\n        The arguments are the same as for :py:func:`norm`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute norm')\n\n    def round(self, decimals=0, out=None, **kwargs): # pylint: disable=arguments-differ\n        \"\"\"Convenience fluent method for :py:func:`round`.\n\n        The arguments are the same as for :py:func:`round`, with\n        this array as data.\n        \"\"\"\n        return round(self, decimals=decimals, out=out, **kwargs)\n\n    def rint(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`rint`.\n\n        The arguments are the same as for :py:func:`rint`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute rint')\n\n    def fix(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`fix`.\n\n        The arguments are the same as for :py:func:`fix`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute fix')\n\n    def floor(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`floor`.\n\n        The arguments are the same as for :py:func:`floor`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute floor')\n\n    def ceil(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`ceil`.\n\n        The arguments are the same as for :py:func:`ceil`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute ceil')\n\n    def trunc(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`trunc`.\n\n        The arguments are the same as for :py:func:`trunc`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute trunc')\n\n    def sin(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sin`.\n\n        The arguments are the same as for :py:func:`sin`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute sin')\n\n    def cos(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`cos`.\n\n        The arguments are the same as for :py:func:`cos`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute cos')\n\n    def tan(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`tan`.\n\n        The arguments are the same as for :py:func:`tan`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute tan')\n\n    def arcsin(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arcsin`.\n\n        The arguments are the same as for :py:func:`arcsin`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute arcsin')\n\n    def arccos(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arccos`.\n\n        The arguments are the same as for :py:func:`arccos`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute arccos')\n\n    def arctan(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arctan`.\n\n        The arguments are the same as for :py:func:`arctan`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute arctan')\n\n    def degrees(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`degrees`.\n\n        The arguments are the same as for :py:func:`degrees`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute degrees')\n\n    def radians(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`radians`.\n\n        The arguments are the same as for :py:func:`radians`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute radians')\n\n    def sinh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sinh`.\n\n        The arguments are the same as for :py:func:`sinh`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute sinh')\n\n    def cosh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`cosh`.\n\n        The arguments are the same as for :py:func:`cosh`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute cosh')\n\n    def tanh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`tanh`.\n\n        The arguments are the same as for :py:func:`tanh`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute tanh')\n\n    def arcsinh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arcsinh`.\n\n        The arguments are the same as for :py:func:`arcsinh`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute arcsinh')\n\n    def arccosh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arccosh`.\n\n        The arguments are the same as for :py:func:`arccosh`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute arccosh')\n\n    def arctanh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arctanh`.\n\n        The arguments are the same as for :py:func:`arctanh`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute arctanh')\n\n    def exp(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`exp`.\n\n        The arguments are the same as for :py:func:`exp`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute exp')\n\n    def expm1(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`expm1`.\n\n        The arguments are the same as for :py:func:`expm1`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute expm1')\n\n    def log(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log`.\n\n        The arguments are the same as for :py:func:`log`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute log')\n\n    def log10(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log10`.\n\n        The arguments are the same as for :py:func:`log10`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute log10')\n\n    def log2(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log2`.\n\n        The arguments are the same as for :py:func:`log2`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute log2')\n\n    def log1p(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log1p`.\n\n        The arguments are the same as for :py:func:`log1p`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute log1p')\n\n    def sqrt(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sqrt`.\n\n        The arguments are the same as for :py:func:`sqrt`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute sqrt')\n\n    def rsqrt(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`rsqrt`.\n\n        The arguments are the same as for :py:func:`rsqrt`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute rsqrt')\n\n    def cbrt(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`cbrt`.\n\n        The arguments are the same as for :py:func:`cbrt`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute cqrt')\n\n    def rcbrt(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`rcbrt`.\n\n        The arguments are the same as for :py:func:`rcbrt`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute rcqrt')\n\n    def square(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`square`.\n\n        The arguments are the same as for :py:func:`square`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute square')\n\n    def reciprocal(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`reciprocal`.\n\n        The arguments are the same as for :py:func:`reciprocal`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute reciprocal')\n\n    def relu(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`relu`.\n\n        The arguments are the same as for :py:func:`relu`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute relu')\n\n    def sigmoid(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sigmoid`.\n\n        The arguments are the same as for :py:func:`sigmoid`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute sigmoid')\n\n    def softmax(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`softmax`.\n\n        The arguments are the same as for :py:func:`softmax`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute softmax')\n\n    def log_softmax(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log_softmax`.\n\n        The arguments are the same as for :py:func:`log_softmax`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute log_softmax')\n\n    def softmin(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`softmin`.\n\n        The arguments are the same as for :py:func:`softmin`, with\n        this array as data.\n        \"\"\"\n        raise AttributeError('_Symbol object has no attribute softmin')\n\n    def squeeze(self, axis=None):  # pylint: disable=arguments-differ\n        \"\"\"Remove single-dimensional entries from the shape of a.\"\"\"\n        return squeeze(self, axis=axis)\n\n    def broadcast_to(self, *args, **kwargs):\n        raise AttributeError('_Symbol object has no attribute broadcast_to')\n\n    def broadcast_like(self, *args, **kwargs):\n        raise AttributeError('_Symbol object has no attribute broadcast_like')\n\n    # pylint: disable=too-many-arguments\n    def optimize_for(self, backend, args=None, aux=None, ctx=None,\n                     shape_dict=None, type_dict=None, stype_dict=None, skip_infer=False, **kwargs):\n        \"\"\"Partitions current symbol and optimizes it for a given backend.\"\"\"\n        new_sym = super().optimize_for(backend, args, aux, ctx, shape_dict, type_dict,\n                                       stype_dict, skip_infer, **kwargs)\n        new_sym = new_sym.as_np_ndarray()\n        return new_sym\n\n@set_module('mxnet.symbol.numpy')\ndef zeros(shape, dtype=float, order='C', ctx=None):\n    \"\"\"Return a new array of given shape and type, filled with zeros.\n    This function currently only supports storing multi-dimensional data\n    in row-major (C-style).\n\n    Parameters\n    ----------\n    shape : int or tuple of int\n        The shape of the empty array.\n    dtype : str or numpy.dtype, optional\n        An optional value type .\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Note that this behavior is different from NumPy's `zeros` function where `float64`\n        is the default value, here we can set 'float32' or 'float64' as your default dtype,\n        because `float32` is considered as the default data type in deep learning.\n    order : {'C'}, optional, default: 'C'\n        How to store multi-dimensional data in memory, currently only row-major\n        (C-style) is supported.\n    ctx : Context, optional\n        An optional device context (default is the current default context).\n\n    Returns\n    -------\n    out : Symbol\n        Array of zeros with the given shape, dtype, and ctx.\n    \"\"\"\n    if order != 'C':\n        raise NotImplementedError\n    if ctx is None:\n        ctx = current_context()\n    if dtype is None or dtype is float:\n        dtype = _np.float64 if is_np_default_dtype() else _np.float32\n    return _npi.zeros(shape=shape, ctx=ctx, dtype=dtype)\n\n\n@set_module('mxnet.symbol.numpy')\ndef ones(shape, dtype=None, order='C', ctx=None):\n    \"\"\"Return a new array of given shape and type, filled with ones.\n    This function currently only supports storing multi-dimensional data\n    in row-major (C-style).\n\n    Parameters\n    ----------\n    shape : int or tuple of int\n        The shape of the empty array.\n    dtype : str or numpy.dtype, optional\n        An optional value type.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Note that this behavior is different from NumPy's `ones` function where\n        `float64` is the default value.\n    order : {'C'}, optional, default: 'C'\n        How to store multi-dimensional data in memory, currently only row-major\n        (C-style) is supported.\n    ctx : Context, optional\n        An optional device context (default is the current default context).\n\n    Returns\n    -------\n    out : _Symbol\n        Array of ones with the given shape, dtype, and ctx.\n    \"\"\"\n    if order != 'C':\n        raise NotImplementedError\n    if ctx is None:\n        ctx = current_context()\n    return _npi.ones(shape=shape, ctx=ctx, dtype=dtype)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef invert(x, out=None, **kwargs):\n    r\"\"\"\n    Compute bit-wise inversion, or bit-wise NOT, element-wise.\n    Computes the bit-wise NOT of the underlying binary representation of\n    the integers in the input arrays. This ufunc implements the C/Python\n    operator ``~``.\n    Parameters\n    ----------\n    x : array_like\n        Only integer and boolean types are handled.\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n    Returns\n    -------\n    out : ndarray or scalar\n        Result.\n        This is a scalar if `x` is a scalar.\n    See Also\n    --------\n    bitwise_and, bitwise_or, bitwise_xor\n    logical_not\n    binary_repr :\n        Return the binary representation of the input number as a string.\n    Examples\n    --------\n    We've seen that 13 is represented by ``00001101``.\n    The invert or bit-wise NOT of 13 is then:\n    >>> x = np.invert(np.array(13, dtype=np.uint8))\n    >>> x\n    242\n    >>> np.binary_repr(x, width=8)\n    '11110010'\n    Notes\n    -----\n    `bitwise_not` is an alias for `invert`:\n    >>> np.bitwise_not is np.invert\n    True\n    \"\"\"\n    return _unary_func_helper(x, _npi.bitwise_not, _np.bitwise_not, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef bitwise_not(x, out=None, **kwargs):\n    r\"\"\"\n    Compute bit-wise inversion, or bit-wise NOT, element-wise.\n    Computes the bit-wise NOT of the underlying binary representation of\n    the integers in the input arrays. This ufunc implements the C/Python\n    operator ``~``.\n    Parameters\n    ----------\n    x : array_like\n        Only integer and boolean types are handled.\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n    Returns\n    -------\n    out : ndarray or scalar\n        Result.\n        This is a scalar if `x` is a scalar.\n    See Also\n    --------\n    bitwise_and, bitwise_or, bitwise_xor\n    logical_not\n    binary_repr :\n        Return the binary representation of the input number as a string.\n    Examples\n    --------\n    We've seen that 13 is represented by ``00001101``.\n    The invert or bit-wise NOT of 13 is then:\n    >>> x = np.invert(np.array(13, dtype=np.uint8))\n    >>> x\n    242\n    >>> np.binary_repr(x, width=8)\n    '11110010'\n    Notes\n    -----\n    `bitwise_not` is an alias for `invert`:\n    >>> np.bitwise_not is np.invert\n    True\n    \"\"\"\n    return _unary_func_helper(x, _npi.bitwise_not, _np.bitwise_not, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\ndef broadcast_to(array, shape):\n    \"\"\"\n    Broadcast an array to a new shape.\n\n    Parameters\n    ----------\n    array : _Symbol or scalar\n        The array to broadcast.\n    shape : tuple\n        The shape of the desired array.\n\n    Returns\n    -------\n    broadcast : array\n        A readonly view on the original array with the given shape. It is\n        typically not contiguous. Furthermore, more than one element of a\n        broadcasted array may refer to a single memory location.\n\n    Raises\n    ------\n    MXNetError\n        If the array is not compatible with the new shape according to NumPy's\n        broadcasting rules.\n    \"\"\"\n    if _np.isscalar(array):\n        return full(shape, array)\n    return _npi.broadcast_to(array, shape)\n\n\n@set_module('mxnet.symbol.numpy')\ndef full(shape, fill_value, dtype=None, order='C', ctx=None, out=None):  # pylint: disable=too-many-arguments\n    \"\"\"\n    Return a new array of given shape and type, filled with `fill_value`.\n    Parameters\n    ----------\n    shape : int or sequence of ints\n        Shape of the new array, e.g., ``(2, 3)`` or ``2``.\n    fill_value : scalar or _Symbol\n        Fill value.\n    dtype : data-type, optional\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        The desired data-type for the array. The default, `None`, means\n        `np.array(fill_value).dtype`.\n    order : {'C'}, optional\n        Whether to store multidimensional data in C- or Fortran-contiguous\n        (row- or column-wise) order in memory. Currently only supports C order.\n    ctx: to specify the device, e.g. the i-th GPU.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n    Returns\n    -------\n    out : ndarray\n        Array of `fill_value` with the given shape, dtype, and order.\n    Notes\n    -----\n    This function differs from the original `numpy.full\n    https://docs.scipy.org/doc/numpy/reference/generated/numpy.full.html`_ in\n    the following way(s):\n    - Have an additional `ctx` argument to specify the device\n    - Have an additional `out` argument\n    - Currently does not support `order` selection\n    See Also\n    --------\n    empty : Return a new uninitialized array.\n    ones : Return a new array setting values to one.\n    zeros : Return a new array setting values to zero.\n    Examples\n    --------\n    >>> np.full((2, 2), 10)\n    array([[10., 10.],\n           [10., 10.]])\n    >>> np.full((2, 2), 2, dtype=np.int32, ctx=mx.cpu(0))\n    array([[2, 2],\n           [2, 2]], dtype=int32)\n    \"\"\"\n    if order != 'C':\n        raise NotImplementedError\n    if ctx is None:\n        ctx = current_context()\n    if isinstance(fill_value, Symbol):\n        if dtype is None:\n            ret = broadcast_to(fill_value, shape)\n        else:\n            ret = broadcast_to(fill_value, shape).astype(dtype)\n        return ret\n    if isinstance(fill_value, bool):\n        fill_value = int(fill_value)\n        dtype = _np.bool if dtype is None else dtype\n    return _npi.full(shape=shape, value=fill_value, ctx=ctx, dtype=dtype, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef full_like(a, fill_value, dtype=None, order='C', ctx=None, out=None):  # pylint: disable=too-many-arguments\n    \"\"\"\n    Return a full array with the same shape and type as a given array.\n\n    Parameters\n    ----------\n    a : _Symbol\n        The shape and data-type of `a` define these same attributes of\n        the returned array.\n    fill_value : scalar\n        Fill value.\n    dtype : data-type, optional\n        Overrides the data type of the result.\n        Temporarily do not support boolean type.\n    order : {'C'}, optional\n        Whether to store multidimensional data in C- or Fortran-contiguous\n        (row- or column-wise) order in memory. Currently only supports C order.\n    ctx: to specify the device, e.g. the i-th GPU.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : _Symbol\n          Array `fill_value` with the same shape and type as `a`.\n\n    See Also\n    --------\n    empty_like : Return an empty array with shape and type of input.\n    ones_like : Return an array of ones with shape and type of input.\n    zeros_like : Return an array of zeros with shape and type of input.\n    full : Return a new array of given shape filled with value.\n    \"\"\"\n    if order != 'C':\n        raise NotImplementedError\n    if ctx is None:\n        ctx = current_context()\n    if isinstance(fill_value, bool):\n        fill_value = int(fill_value)\n    return _npi.full_like(a, fill_value=fill_value, ctx=ctx, dtype=dtype, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef zeros_like(a, dtype=None, order='C', ctx=None, out=None):  # pylint: disable=too-many-arguments\n    \"\"\"\n    Return an array of zeros with the same shape and type as a given array.\n\n    Parameters\n    ----------\n    a : _Symbol\n        The shape and data-type of `a` define these same attributes of\n        the returned array.\n    fill_value : scalar\n        Fill value.\n    dtype : data-type, optional\n        Overrides the data type of the result.\n        Temporarily do not support boolean type.\n    order : {'C'}, optional\n        Whether to store multidimensional data in C- or Fortran-contiguous\n        (row- or column-wise) order in memory. Currently only supports C order.\n    ctx: to specify the device, e.g. the i-th GPU.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : _Symbol\n        Array of zeros with the same shape and type as `a`.\n\n    See Also\n    --------\n    empty_like : Return an empty array with shape and type of input.\n    ones_like : Return an array of ones with shape and type of input.\n    zeros_like : Return an array of zeros with shape and type of input.\n    zeros : Return a new array of given shape filled with zeros.\n    \"\"\"\n    if order != 'C':\n        raise NotImplementedError\n    if ctx is None:\n        ctx = current_context()\n    return _npi.full_like(a, fill_value=0, ctx=ctx, dtype=dtype, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef ones_like(a, dtype=None, order='C', ctx=None, out=None):  # pylint: disable=too-many-arguments\n    \"\"\"\n    Return an array of ones with the same shape and type as a given array.\n\n    Parameters\n    ----------\n    a : _Symbol\n        The shape and data-type of `a` define these same attributes of\n        the returned array.\n    fill_value : scalar\n        Fill value.\n    dtype : data-type, optional\n        Overrides the data type of the result.\n        Temporarily do not support boolean type.\n    order : {'C'}, optional\n        Whether to store multidimensional data in C- or Fortran-contiguous\n        (row- or column-wise) order in memory. Currently only supports C order.\n    ctx: to specify the device, e.g. the i-th GPU.\n    out : ndarray or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : _Symbol\n          Array of ones with the same shape and type as `a`.\n\n    See Also\n    --------\n    empty_like : Return an empty array with shape and type of input.\n    ones_like : Return an array of ones with shape and type of input.\n    zeros_like : Return an array of zeros with shape and type of input.\n    zeros : Return a new array of given shape filled with zeros.\n    \"\"\"\n    if order != 'C':\n        raise NotImplementedError\n    if ctx is None:\n        ctx = current_context()\n    return _npi.full_like(a, fill_value=1, ctx=ctx, dtype=dtype, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef identity(n, dtype=None, ctx=None):\n    \"\"\"\n    Return the identity array.\n\n    The identity array is a square array with ones on\n    the main diagonal.\n\n    Parameters\n    ----------\n    n : int\n        Number of rows (and columns) in `n` x `n` output.\n    dtype : data-type, optional\n        Data-type of the output.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n    ctx : Context, optional\n        An optional device context (default is the current default context).\n\n    Returns\n    -------\n    out : _Symbol\n        `n` x `n` array with its main diagonal set to one,\n        and all other elements 0.\n    \"\"\"\n    if not isinstance(n, int):\n        raise TypeError(\"Input 'n' should be an integer\")\n    if n < 0:\n        raise ValueError(\"Input 'n' cannot be negative\")\n    if ctx is None:\n        ctx = current_context()\n    return _npi.identity(shape=(n, n), ctx=ctx, dtype=dtype)\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.symbol.numpy')\ndef take(a, indices, axis=None, mode='raise', out=None):\n    r\"\"\"\n    Take elements from an array along an axis.\n\n    When axis is not None, this function does the same thing as \"fancy\"\n    indexing (indexing arrays using arrays); however, it can be easier to use\n    if you need elements along a given axis. A call such as\n    ``np.take(arr, indices, axis=3)`` is equivalent to\n    ``arr[:,:,:,indices,...]``.\n\n    Explained without fancy indexing, this is equivalent to the following use\n    of `ndindex`, which sets each of ``ii``, ``jj``, and ``kk`` to a tuple of\n    indices::\n\n        Ni, Nk = a.shape[:axis], a.shape[axis+1:]\n        Nj = indices.shape\n        for ii in ndindex(Ni):\n            for jj in ndindex(Nj):\n                for kk in ndindex(Nk):\n                    out[ii + jj + kk] = a[ii + (indices[jj],) + kk]\n\n    Parameters\n    ----------\n    a : _Symbol\n        The source array.\n    indices : _Symbol\n        The indices of the values to extract. Also allow scalars for indices.\n    axis : int, optional\n        The axis over which to select values. By default, the flattened\n        input array is used.\n    out : _Symbol or None, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n    mode : {'clip', 'wrap'}, optional\n        Specifies how out-of-bounds indices will behave.\n\n        * 'clip' -- clip to the range (default)\n        * 'wrap' -- wrap around\n\n        'clip' mode means that all indices that are too large are replaced\n        by the index that addresses the last element along that axis. Note\n        that this disables indexing with negative numbers.\n\n    Returns\n    -------\n    out : _Symbol\n        The returned array has the same type as `a`.\n\n    Notes\n    -----\n\n    This function differs from the original `numpy.take\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.take.html>`_ in\n    the following way(s):\n\n    - Only ndarray or scalar ndarray is accepted as valid input.\n    \"\"\"\n    if mode not in ('wrap', 'clip', 'raise'):\n        raise NotImplementedError(\n            \"function take does not support mode '{}'\".format(mode))\n    if axis is None:\n        return _npi.take(_npi.reshape(a, -1), indices, 0, mode, out)\n    else:\n        return _npi.take(a, indices, axis, mode, out)\n# pylint: enable=redefined-outer-name\n\n\n#pylint: disable= too-many-arguments, no-member, protected-access\ndef _ufunc_helper(lhs, rhs, fn_array, fn_scalar, lfn_scalar, rfn_scalar=None, out=None):\n    \"\"\" Helper function for element-wise operation.\n    The function will perform numpy-like broadcasting if needed and call different functions.\n\n    Parameters\n    --------\n    lhs : Symbol or numeric value\n        Left-hand side operand.\n\n    rhs : Symbol or numeric value\n        Right-hand operand,\n\n    fn_array : function\n        Function to be called if both lhs and rhs are of ``Symbol`` type.\n\n    fn_scalar : function\n        Function to be called if both lhs and rhs are numeric values.\n\n    lfn_scalar : function\n        Function to be called if lhs is ``Symbol`` while rhs is numeric value\n\n    rfn_scalar : function\n        Function to be called if lhs is numeric value while rhs is ``Symbol``;\n        if none is provided, then the function is commutative, so rfn_scalar is equal to lfn_scalar\n\n    Returns\n    --------\n    mxnet.numpy.ndarray\n        result array\n    \"\"\"\n    if isinstance(lhs, numeric_types):\n        if isinstance(rhs, numeric_types):\n            return fn_scalar(lhs, rhs, out=out)\n        else:\n            is_int = isinstance(rhs, integer_types)\n            if rfn_scalar is None:\n                # commutative function\n                return lfn_scalar(rhs, scalar=float(lhs), is_int=is_int, out=out)\n            else:\n                return rfn_scalar(rhs, scalar=float(lhs), is_int=is_int, out=out)\n    elif isinstance(rhs, numeric_types):\n        is_int = isinstance(rhs, integer_types)\n        return lfn_scalar(lhs, scalar=float(rhs), is_int=is_int, out=out)\n    elif isinstance(rhs, Symbol):\n        return fn_array(lhs, rhs, out=out)\n    else:\n        raise TypeError(f'type {str(type(rhs))} not supported')\n#pylint: enable= too-many-arguments, no-member, protected-access\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef add(x1, x2, out=None, **kwargs):\n    return _ufunc_helper(x1, x2, _npi.add, _np.add, _npi.add_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef subtract(x1, x2, out=None, **kwargs):\n    return _ufunc_helper(x1, x2, _npi.subtract, _np.subtract, _npi.subtract_scalar,\n                         _npi.rsubtract_scalar, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef multiply(x1, x2, out=None, **kwargs):\n    return _ufunc_helper(x1, x2, _npi.multiply, _np.multiply, _npi.multiply_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef divide(x1, x2, out=None, **kwargs):\n    return _ufunc_helper(x1, x2, _npi.true_divide, _np.divide, _npi.true_divide_scalar,\n                         _npi.rtrue_divide_scalar, out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef true_divide(x1, x2, out=None):\n    return _ufunc_helper(x1, x2, _npi.true_divide, _np.divide, _npi.true_divide_scalar,\n                         _npi.rtrue_divide_scalar, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef mod(x1, x2, out=None, **kwargs):\n    return _ufunc_helper(x1, x2, _npi.mod, _np.mod, _npi.mod_scalar, _npi.rmod_scalar, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef fmod(x1, x2, out=None, **kwargs):\n    return _ufunc_helper(x1, x2, _npi.fmod, _np.fmod, _npi.fmod_scalar, _npi.rfmod_scalar, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef remainder(x1, x2, out=None, **kwargs):\n    return _ufunc_helper(x1, x2, _npi.mod, _np.mod, _npi.mod_scalar, _npi.rmod_scalar, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef power(x1, x2, out=None, **kwargs):\n    return _ufunc_helper(x1, x2, _npi.power, _np.power, _npi.power_scalar, _npi.rpower_scalar, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef gcd(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns the greatest common divisor of ``|x1|`` and ``|x2|``\n\n    Parameters\n    ----------\n    x1, x2 : ndarrays or scalar values\n        The arrays for computing greatest common divisor. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which may be the shape of\n        one or the other).\n\n    out : ndarray or None, optional\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    y : ndarray or scalar\n        The greatest common divisor of the absolute value of the inputs\n        This is a scalar if both `x1` and `x2` are scalars.\n\n    See Also\n    --------\n    lcm : The lowest common multiple\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.gcd, _np.gcd, _npi.gcd_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef matmul(a, b, out=None, **kwargs):\n    \"\"\"\n    Matrix product of two arrays.\n\n    Parameters\n    ----------\n    a, b : _Symbol.\n    out : _Symbol, optional\n        A location into which the result is stored.\n        If provided, it must have a shape that matches the signature (n,k),(k,m)->(n,m).\n        If not provided or None, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : _Symbol\n        The matrix product of the inputs.\n        This is a scalar only when both x1, x2 are 1-d vectors.\n\n    Raises\n    ------\n    MXNetError\n        If the last dimension of a is not the same size as the second-to-last dimension of b.\n        If a scalar value is passed in.\n\n    See Also\n    --------\n    tensordot :\n        Sum products over arbitrary axes.\n    dot :\n        alternative matrix product with different broadcasting rules.\n    einsum :\n        Einstein summation convention.\n\n    Notes\n    -----\n    The behavior depends on the arguments in the following way.\n\n    - If both arguments are 2-D they are multiplied like conventional matrices.\n    - If either argument is N-D, N > 2, it is treated as a stack of matrices\n      residing in the last two indexes and broadcast accordingly.\n    - If the first argument is 1-D, it is promoted to a matrix by prepending\n      a 1 to its dimensions. After matrix multiplication the prepended 1 is removed.\n    - If the second argument is 1-D, it is promoted to a matrix by appending a 1\n      to its dimensions. After matrix multiplication the appended 1 is removed.\n\n    matmul differs from dot in two important ways:\n\n    - Multiplication by scalars is not allowed, use multiply instead.\n    - Stacks of matrices are broadcast together as if the matrices were elements,\n      respecting the signature (n,k),(k,m)->(n,m).\n    \"\"\"\n    return _npi.matmul(a, b, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef lcm(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns the lowest common multiple of ``|x1|`` and ``|x2|``\n\n    Parameters\n    ----------\n    x1, x2 : _Symbols or scalar values\n        The arrays for computing lowest common multiple. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which may be the shape of\n        one or the other).\n\n    out : _Symbol or None, optional\n        A location into which the result is stored. If provided, it must have a shape\n        that the inputs broadcast to. If not provided or None, a freshly-allocated array\n        is returned.\n\n    Returns\n    -------\n    y : _Symbol or scalar\n        The lowest common multiple of the absolute value of the inputs\n        This is a scalar if both `x1` and `x2` are scalars.\n\n    See Also\n    --------\n    gcd : The greatest common divisor\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.lcm, _np.lcm, _npi.lcm_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef argsort(a, axis=-1, kind=None, order=None):\n    \"\"\"\n    Returns the indices that would sort an array.\n    Perform an indirect sort along the given axis using the algorithm specified\n    by the `kind` keyword. It returns an array of indices of the same shape as\n    `a` that index data along the given axis in sorted order.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Array to sort.\n    axis : int or None, optional\n        Axis along which to sort.  The default is -1 (the last axis). If None,\n        the flattened array is used.\n    kind : string, optional\n        This argument can take any string, but it does not have any effect on the\n        final result.\n    order : str or list of str, optional\n        Not supported yet, will raise NotImplementedError if not None.\n\n    Returns\n    -------\n    index_array : _Symbol, int\n        Array of indices that sort `a` along the specified `axis`.\n        If `a` is one-dimensional, ``a[index_array]`` yields a sorted `a`.\n        More generally, ``np.take_along_axis(a, index_array, axis=axis)``\n        always yields the sorted `a`, irrespective of dimensionality.\n\n    Notes\n    -----\n    This operator does not support different sorting algorithms.\n    \"\"\"\n    if order is not None:\n        raise NotImplementedError(\"order is not supported yet...\")\n\n    return _npi.argsort(data=a, axis=axis, is_ascend=True, dtype='int64')\n\n\n@set_module('mxnet.symbol.numpy')\ndef sort(a, axis=-1, kind=None, order=None):\n    \"\"\"\n    Return a sorted copy of an array.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Array to be sorted.\n    axis : int or None, optional\n        Axis along which to sort.  The default is -1 (the last axis). If None,\n        the flattened array is used.\n    kind : string, optional\n        This argument can take any string, but it does not have any effect on the\n        final result.\n    order : str or list of str, optional\n        Not supported yet, will raise NotImplementedError if not None.\n\n    Returns\n    -------\n    sorted_array : ndarray\n        Array of the same type and shape as `a`.\n\n    Notes\n    -----\n    This operator does not support different sorting algorithms.\n    \"\"\"\n    if order is not None:\n        raise NotImplementedError(\"order is not supported yet...\")\n\n    return _npi.sort(data=a, axis=axis, is_ascend=True)\n\n@set_module('mxnet.symbol.numpy')\ndef dot(a, b, out=None):\n    \"\"\"\n    Dot product of two arrays. Specifically,\n\n    - If both `a` and `b` are 1-D arrays, it is inner product of vectors\n\n    - If both `a` and `b` are 2-D arrays, it is matrix multiplication,\n\n    - If either `a` or `b` is 0-D (scalar), it is equivalent to :func:`multiply`\n      and using ``np.multiply(a, b)`` or ``a * b`` is preferred.\n\n    - If `a` is an N-D array and `b` is a 1-D array, it is a sum product over\n      the last axis of `a` and `b`.\n\n    - If `a` is an N-D array and `b` is a 2-D array, it is a\n      sum product over the last axis of `a` and the second-to-last axis of `b`::\n\n        dot(a, b)[i,j,k] = sum(a[i,j,:] * b[:,k])\n\n    Parameters\n    ----------\n    a : _Symbol\n        First argument.\n    b : _Symbol\n        Second argument.\n\n    out : _Symbol, optional\n        Output argument. It must have the same shape and type as the expected output.\n\n    Returns\n    -------\n    output : _Symbol\n        Returns the dot product of `a` and `b`.  If `a` and `b` are both\n        scalars or both 1-D arrays then a scalar is returned; otherwise\n        an array is returned.\n        If `out` is given, then it is returned\n\n    Examples\n    --------\n    >>> a = np.array(3)\n    >>> b = np.array(4)\n    >>> np.dot(a, b)\n    array(12.)\n\n    For 2-D arrays it is the matrix product:\n\n    >>> a = np.array([[1, 0], [0, 1]])\n    >>> b = np.array([[4, 1], [2, 2]])\n    >>> np.dot(a, b)\n    array([[4., 1.],\n           [2., 2.]])\n\n    >>> a = np.arange(3*4*5*6).reshape((3,4,5,6))\n    >>> b = np.arange(5*6)[::-1].reshape((6,5))\n    >>> np.dot(a, b)[2,3,2,2]\n    array(29884.)\n    >>> np.sum(a[2,3,2,:] * b[:,2])\n    array(29884.)\n    \"\"\"\n    return _npi.dot(a, b, out=out)\n\n@set_module('mxnet.symbol.numpy')\ndef tensordot(a, b, axes=2):\n    r\"\"\"\n    tensordot(a, b, axes=2)\n    Compute tensor dot product along specified axes for arrays >= 1-D.\n    Given two tensors (arrays of dimension greater than or equal to one),\n    `a` and `b`, and an ndarray object containing two ndarray\n    objects, ``(a_axes, b_axes)``, sum the products of `a`'s and `b`'s\n    elements (components) over the axes specified by ``a_axes`` and\n    ``b_axes``. The third argument can be a single non-negative\n    integer_like scalar, ``N``; if it is such, then the last ``N``\n    dimensions of `a` and the first ``N`` dimensions of `b` are summed\n    over.\n    Parameters\n    ----------\n    a, b : _Symbol\n        Tensors to \"dot\".\n    axes : int or (2,) ndarray\n        * integer_like\n        If an int N, sum over the last N axes of `a` and the first N axes\n        of `b` in order. The sizes of the corresponding axes must match.\n        * (2,) array_like\n        Or, a list of axes to be summed over, first sequence applying to `a`,\n        second to `b`. Both elements array_like must be of the same length.\n    Notes\n    -----\n    Three common use cases are:\n        * ``axes = 0`` : tensor product :math:`a\\otimes b`\n        * ``axes = 1`` : tensor dot product :math:`a\\cdot b`\n        * ``axes = 2`` : (default) tensor double contraction :math:`a:b`\n    When `axes` is integer_like, the sequence for evaluation will be: first\n    the -Nth axis in `a` and 0th axis in `b`, and the -1th axis in `a` and\n    Nth axis in `b` last.\n    When there is more than one axis to sum over - and they are not the last\n    (first) axes of `a` (`b`) - the argument `axes` should consist of\n    two sequences of the same length, with the first axis to sum over given\n    first in both sequences, the second axis second, and so forth.\n    \"\"\"\n    if _np.isscalar(axes):\n        return _npi.tensordot_int_axes(a, b, axes)\n\n    if len(axes) != 2:\n        raise ValueError('Axes must consist of two arrays.')\n    a_axes_summed, b_axes_summed = axes\n    if _np.isscalar(a_axes_summed):\n        a_axes_summed = (a_axes_summed,)\n    if _np.isscalar(b_axes_summed):\n        b_axes_summed = (b_axes_summed,)\n\n    if len(a_axes_summed) != len(b_axes_summed):\n        raise ValueError('Axes length mismatch')\n\n    return _npi.tensordot(a, b, a_axes_summed, b_axes_summed)\n\n\n@set_module('mxnet.symbol.numpy')\ndef histogram(a, bins=10, range=None, normed=None, weights=None, density=None):  # pylint: disable= too-many-arguments\n    \"\"\"\n    Compute the histogram of a set of data.\n\n    Parameters\n    ----------\n    a : Symbol\n        Input data. The histogram is computed over the flattened array.\n    bins : int or Symbol\n        If `bins` is an int, it defines the number of equal-width\n        bins in the given range (10, by default). If `bins` is a\n        sequence, it defines a monotonically increasing array of bin edges,\n        including the rightmost edge, allowing for non-uniform bin widths.\n        .. versionadded:: 1.11.0\n        If `bins` is a string, it defines the method used to calculate the\n        optimal bin width, as defined by `histogram_bin_edges`.\n    range : (float, float)\n        The lower and upper range of the bins. Required when `bins` is an integer.\n        Values outside the range are ignored. The first element of the range must\n        be less than or equal to the second.\n    normed : bool, optional\n        Not supported yet, coming soon.\n    weights : array_like, optional\n        Not supported yet, coming soon.\n    density : bool, optional\n        Not supported yet, coming soon.\n    \"\"\"\n    if normed is True:\n        raise NotImplementedError(\"normed is not supported yet...\")\n    if weights is not None:\n        raise NotImplementedError(\"weights is not supported yet...\")\n    if density is True:\n        raise NotImplementedError(\"density is not supported yet...\")\n    if isinstance(bins, numeric_types):\n        if range is None:\n            raise NotImplementedError(\"automatic range is not avaialble yet...\")\n        return _npi.histogram(a, bin_cnt=bins, range=range)\n    if isinstance(bins, (list, tuple)):\n        raise NotImplementedError(\"array_like bins is not supported yet...\")\n    if isinstance(bins, str):\n        raise NotImplementedError(\"string bins is not supported yet...\")\n    if isinstance(bins, Symbol):\n        return _npi.histogram(a, bins)\n    raise ValueError(\"histogram fails with\", locals())\n\n\n@set_module('mxnet.symbol.numpy')\ndef eye(N, M=None, k=0, dtype=float, **kwargs):\n    \"\"\"\n    Return a 2-D array with ones on the diagonal and zeros elsewhere.\n\n    Parameters\n    ----------\n    N : int\n        Number of rows in the output.\n    M : int, optional\n        Number of columns in the output. If None, defaults to N.\n    k : int, optional\n        Index of the diagonal: 0 (the default) refers to the main diagonal,\n        a positive value refers to an upper diagonal,\n        and a negative value to a lower diagonal.\n    dtype : data-type, optional\n        Data-type of the returned array.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n\n    Returns\n    -------\n    I : _Symbol of shape (N,M)\n        An array where all elements are equal to zero,\n        except for the k-th diagonal, whose values are equal to one.\n    \"\"\"\n    _sanity_check_params('eye', ['order'], kwargs)\n    ctx = kwargs.pop('ctx', current_context())\n    if ctx is None:\n        ctx = current_context()\n    if dtype is None or dtype is float:\n        dtype = _np.float64 if is_np_default_dtype() else _np.float32\n    return _npi.eye(N, M, k, ctx, dtype)\n\n\n@set_module('mxnet.symbol.numpy')\ndef empty_like(prototype, dtype=None, order='C', subok=False, shape=None): # pylint: disable=W0621\n    \"\"\"\n    Return a new array with the same shape and type as a given array.\n\n    Parameters\n    ----------\n    prototype : _Symbol\n        The shape and data-type of `prototype` define these same attributes\n        of the returned array.\n    dtype : data-type, optional\n        Overrides the data type of the result.\n    order : {'C'}, optional\n        Whether to store multidimensional data in C- or Fortran-contiguous\n        (row- or column-wise) order in memory. Currently only supports C order.\n    subok : bool, optional.\n        If True, then the newly created array will use the sub-class\n        type of 'a', otherwise it will be a base-class array. Defaults\n        to False.\n        (Only support False at this moment)\n    shape : int or sequence of ints, optional.\n        Overrides the shape of the result. If order='K' and the number of\n        dimensions is unchanged, will try to keep order, otherwise,\n        order='C' is implied.\n        (This parameter is not supported at this moment)\n\n    Returns\n    -------\n    out : _Symbol\n        Array of uninitialized (arbitrary) data with the same\n        shape and type as `prototype`.\n\n    See Also\n    --------\n    ones_like : Return an array of ones with shape and type of input.\n    zeros_like : Return an array of zeros with shape and type of input.\n    full_like : Return a new array with shape of input filled with value.\n    empty : Return a new uninitialized array.\n\n    Notes\n    -----\n    This function does *not* initialize the returned array; to do that use\n    `zeros_like` or `ones_like` instead.  It may be marginally faster than\n    the functions that do set the array values.\n    \"\"\"\n    dtype_list = {None:'None', _np.int8:'int8', _np.uint8:'uint8', _np.int32:'int32',\n                  _np.int64:'int64', _np.float16:'float16', _np.float32:'float32',\n                  _np.float64:'float64', _np.bool_:'bool_', bool:'bool', int:'int64', float:'float64'}\n    if order != 'C':\n        raise NotImplementedError(\"Only support C order at this moment\")\n    if subok:\n        raise NotImplementedError(\"Creating array by using sub-class is not supported at this moment\")\n    if shape is not None:\n        raise NotImplementedError(\"Parameter 'shape' is not supported at this moment\")\n    try:\n        dtype = dtype if isinstance(dtype, str) else dtype_list[dtype]\n    except:\n        raise NotImplementedError(\"Do not support this dtype at this moment\")\n    return _npi.empty_like_fallback(prototype, dtype=dtype, order=order, subok=subok, shape=shape)\n\n\n@set_module('mxnet.symbol.numpy')\ndef linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis=0, ctx=None): # pylint: disable=too-many-arguments\n    r\"\"\"\n    Return evenly spaced numbers over a specified interval.\n\n    Returns num evenly spaced samples, calculated over the interval [start, stop].\n    The endpoint of the interval can optionally be excluded.\n\n    Parameters\n    ----------\n    start : real number\n        The starting value of the sequence.\n    stop : real number\n        The end value of the sequence, unless endpoint is set to False. In\n        that case, the sequence consists of all but the last of num + 1\n        evenly spaced samples, so that stop is excluded. Note that the step\n        size changes when endpoint is False.\n    num : int, optional\n        Number of samples to generate. Default is 50. Must be non-negative.\n    endpoint : bool, optional\n        If True, stop is the last sample. Otherwise, it is not included.\n        Default is True.\n    retstep : bool, optional\n        If True, return (samples, step), where step is the spacing between samples.\n    dtype : dtype, optional\n        The type of the output array. If dtype is not given, infer the data\n        type from the other input arguments.\n    axis : int, optional\n        The axis in the result to store the samples. Relevant only if start or\n        stop are array-like. By default (0), the samples will be along a new\n        axis inserted at the beginning. Use -1 to get an axis at the end.\n\n    Returns\n    -------\n    samples : _Symbol\n        There are num equally spaced samples in the closed interval\n        `[start, stop]` or the half-open interval `[start, stop)`\n        (depending on whether endpoint is True or False).\n    step : float, optional\n        Only returned if retstep is True\n        Size of spacing between samples.\n\n\n    See Also\n    --------\n    arange : Similar to `linspace`, but uses a step size (instead of the\n             number of samples).\n\n    Notes\n    -----\n\n    This function differs from the original `numpy.linspace\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linspace.html>`_ in\n    the following aspects:\n\n    - `start` and `stop` do not support list, numpy ndarray and mxnet ndarray\n    - axis could only be 0\n    - There could be an additional `ctx` argument to specify the device, e.g. the i-th\n      GPU.\n    \"\"\"\n    if isinstance(start, (list, _np.ndarray)) or isinstance(stop, (list, _np.ndarray)):\n        raise NotImplementedError('start and stop only support int')\n    if axis != 0:\n        raise NotImplementedError(\"the function only support axis 0\")\n    if ctx is None:\n        ctx = current_context()\n    if retstep:\n        step = (stop - start) / (num - 1)\n        return _npi.linspace(start=start, stop=stop, num=num, endpoint=endpoint, ctx=ctx, dtype=dtype), step\n    else:\n        return _npi.linspace(start=start, stop=stop, num=num, endpoint=endpoint, ctx=ctx, dtype=dtype)\n\n\n@set_module('mxnet.symbol.numpy')\ndef logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None, axis=0, ctx=None): # pylint: disable=too-many-arguments\n    r\"\"\"Return numbers spaced evenly on a log scale.\n\n    In linear space, the sequence starts at ``base ** start``\n    (`base` to the power of `start`) and ends with ``base ** stop``\n    (see `endpoint` below).\n\n        Non-scalar `start` and `stop` are now supported.\n\n    Parameters\n    ----------\n    start : scalar\n        ``base ** start`` is the starting value of the sequence.\n    stop : scalar\n        ``base ** stop`` is the final value of the sequence, unless `endpoint`\n        is False.  In that case, ``num + 1`` values are spaced over the\n        interval in log-space, of which all but the last (a sequence of\n        length `num`) are returned.\n    num : scalar, optional\n        Number of samples to generate.  Default is 50.\n    endpoint : boolean, optional\n        If true, `stop` is the last sample. Otherwise, it is not included.\n        Default is True.\n    base : scalar, optional\n        The base of the log space. The step size between the elements in\n        ``ln(samples) / ln(base)`` (or ``log_base(samples)``) is uniform.\n        Default is 10.0.\n    dtype : dtype\n        The type of the output array.  If `dtype` is not given, infer the data\n        type from the other input arguments.\n    axis : scalar, optional\n        The axis in the result to store the samples.  Relevant only if start\n        or stop are array-like.  By default (0), the samples will be along a\n        new axis inserted at the beginning. Now, axis only support axis = 0.\n    ctx : Context, optional\n        An optional device context (default is the current default context).\n\n    Returns\n    -------\n    samples : _Symbol\n        `num` samples, equally spaced on a log scale.\n\n    See Also\n    --------\n    arange : Similar to linspace, with the step size specified instead of the\n             number of samples. Note that, when used with a float endpoint, the\n             endpoint may or may not be included.\n    linspace : Similar to logspace, but with the samples uniformly distributed\n               in linear space, instead of log space.\n\n    Notes\n    -----\n    Logspace is equivalent to the code\n\n    >>> y = np.linspace(start, stop, num=num, endpoint=endpoint)\n    ...\n    >>> power(base, y).astype(dtype)\n    ...\n\n    Examples\n    --------\n    >>> np.logspace(2.0, 3.0, num=4)\n    array([ 100.     ,  215.44347,  464.15887, 1000.     ])\n    >>> np.logspace(2.0, 3.0, num=4, endpoint=False)\n    array([100.     , 177.82794, 316.22775, 562.3413 ])\n    >>> np.logspace(2.0, 3.0, num=4, base=2.0)\n    array([4.       , 5.0396843, 6.349604 , 8.       ])\n    >>> np.logspace(2.0, 3.0, num=4, base=2.0, dtype=np.int32)\n    array([4, 5, 6, 8], dtype=int32)\n    >>> np.logspace(2.0, 3.0, num=4, ctx=npx.gpu(0))\n    array([ 100.     ,  215.44347,  464.15887, 1000.     ], ctx=gpu(0))\n    \"\"\"\n    if isinstance(start, (list, _np.ndarray)) or \\\n       isinstance(stop, (list, _np.ndarray)):\n        raise NotImplementedError('start and stop only support int')\n    if axis != 0:\n        raise NotImplementedError(\"the function only support axis 0\")\n    if ctx is None:\n        ctx = current_context()\n    return _npi.logspace(start=start, stop=stop, num=num, endpoint=endpoint, base=base, ctx=ctx, dtype=dtype)\n\n\n@set_module('mxnet.symbol.numpy')\ndef expand_dims(a, axis):\n    \"\"\"Expand the shape of an array.\n\n    Insert a new axis that will appear at the `axis` position in the expanded\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input array.\n    axis : int\n        Position in the expanded axes where the new axis is placed.\n\n    Returns\n    -------\n    res : _Symbol\n        Output array. The number of dimensions is one greater than that of\n        the input array.\n    \"\"\"\n    return _npi.expand_dims(a, axis)\n\n\n@set_module('mxnet.symbol.numpy')\ndef tril(m, k=0):\n    r\"\"\"\n    Lower triangle of an array.\n\n    Return a copy of an array with elements above the `k`-th diagonal zeroed.\n\n    Parameters\n    ----------\n    m : _Symbol, shape (M, N)\n        Input array.\n    k : int, optional\n        Diagonal above which to zero elements.  `k = 0` (the default) is the\n        main diagonal, `k < 0` is below it and `k > 0` is above.\n\n    Returns\n    -------\n    tril : _Symbol, shape (M, N)\n        Lower triangle of `m`, of same shape and data-type as `m`.\n\n    See Also\n    --------\n    triu : same thing, only for the upper triangle\n    \"\"\"\n    return _npi.tril(m, k)\n\n\n@set_module('mxnet.symbol.numpy')\ndef triu(m, k=0):\n    r\"\"\"\n    Upper triangle of an array.\n\n    Return a copy of an array with elements under the `k`-th diagonal zeroed.\n\n    Parameters\n    ----------\n    m : _Symbol, shape (M, N)\n        Input array.\n    k : int, optional\n        Diagonal under which to zero elements.  `k = 0` (the default) is the\n        main diagonal, `k < 0` is below it and `k > 0` is under.\n\n    Returns\n    -------\n    triu : _Symbol, shape (M, N)\n        Upper triangle of `m`, of same shape and data-type as `m`.\n\n    See Also\n    --------\n    tril : same thing, only for the lower triangle\n    \"\"\"\n    return _npi.triu(m, k)\n\n\ndef tril_indices(n, k=0, m=None):\n    \"\"\"\n    Return the indices for the lower-triangle of an (n, m) array.\n\n    Parameters\n    ----------\n    n : int\n        The row dimension of the arrays for which the returned\n        indices will be valid.\n    k : int, optional\n        Diagonal offset (see `tril` for details).\n    m : int, optional\n        .. versionadded:: 1.9.0\n\n        The column dimension of the arrays for which the returned\n        arrays will be valid.\n        By default `m` is taken equal to `n`.\n\n\n    Returns\n    -------\n    inds : tuple of _Symbol\n        The indices for the triangle. The returned tuple contains two arrays,\n        each with the indices along one dimension of the array.\n\n    See also\n    --------\n    triu_indices : similar function, for upper-triangular.\n    mask_indices : generic function accepting an arbitrary mask function.\n    tril, triu\n\n    Notes\n    -----\n    .. versionadded:: 1.4.0\n\n    Examples\n    --------\n    Compute two different sets of indices to access 4x4 arrays, one for the\n    lower triangular part starting at the main diagonal, and one starting two\n    diagonals further right:\n\n    >>> il1 = np.tril_indices(4)\n    >>> il2 = np.tril_indices(4, 2)\n\n    Here is how they can be used with a sample array:\n\n    >>> a = np.arange(16).reshape(4, 4)\n    >>> a\n    array([[ 0,  1,  2,  3],\n           [ 4,  5,  6,  7],\n           [ 8,  9, 10, 11],\n           [12, 13, 14, 15]])\n\n    Both for indexing:\n\n    >>> a[il1]\n    array([ 0,  4,  5,  8,  9, 10, 12, 13, 14, 15])\n\n    And for assigning values:\n\n    >>> a[il1] = -1\n    >>> a\n    array([[-1,  1,  2,  3],\n           [-1, -1,  6,  7],\n           [-1, -1, -1, 11],\n           [-1, -1, -1, -1]])\n\n    These cover almost the whole array (two diagonals right of the main one):\n\n    >>> a[il2] = -10\n    >>> a\n    array([[-10, -10, -10,   3],\n           [-10, -10, -10, -10],\n           [-10, -10, -10, -10],\n           [-10, -10, -10, -10]])\n\n    \"\"\"\n    if m is None:\n        m = n\n    return _npi.tril_indices(n, k, m)\n\n\n@set_module('mxnet.symbol.numpy')\ndef trace(a, offset=0, axis1=0, axis2=1, out=None):\n    \"\"\"\n    Return the sum along diagonals of the array.\n    If `a` is 2-D, the sum along its diagonal with the given offset\n    is returned, i.e., the sum of elements ``a[i,i+offset]`` for all i.\n    If `a` has more than two dimensions, then the axes specified by axis1 and\n    axis2 are used to determine the 2-D sub-arrays whose traces are returned.\n    The shape of the resulting array is the same as that of `a` with `axis1`\n    and `axis2` removed.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input array, from which the diagonals are taken.\n    offset : int, optional\n        Offset of the diagonal from the main diagonal. Can be both positive\n        and negative. Defaults to 0.\n    axis1, axis2 : int, optional\n        Axes to be used as the first and second axis of the 2-D sub-arrays\n        from which the diagonals should be taken. Defaults are the first two\n        axes of `a`.\n    out : _Symbol\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    sum_along_diagonals : _Symbol\n        If `a` is 2-D, the sum along the diagonal is returned.  If `a` has\n        larger dimensions, then an array of sums along diagonals is returned.\n    \"\"\"\n    return _npi.trace(a, offset=offset, axis1=axis1, axis2=axis2, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef transpose(a, axes=None):\n    \"\"\"\n    Permute the dimensions of an array.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input array.\n    axes : list of ints, optional\n        By default, reverse the dimensions,\n        otherwise permute the axes according to the values given.\n\n    Returns\n    -------\n    p : _Symbol\n        a with its axes permuted.\n    \"\"\"\n    return _npi.transpose(a, axes=axes)\n\n\n@set_module('mxnet.symbol.numpy')\ndef tri(N, M=None, k=0, dtype=None, ctx=None):\n    r\"\"\"\n    An array with ones at and below the given diagonal and zeros elsewhere.\n\n    Parameters\n    ----------\n    N : int\n        Number of rows in the array.\n    M : int, optional\n        Number of columns in the array.\n        By default, `M` is taken equal to `N`.\n    k : int, optional\n        The sub-diagonal at and below which the array is filled.\n        `k` = 0 is the main diagonal, while `k` < 0 is below it,\n        and `k` > 0 is above.  The default is 0.\n    dtype : dtype, optional\n        Data type of the returned array.  The default is float.\n\n    Returns\n    -------\n    tri : Symbol of shape (N, M)\n        Array with its lower triangle filled with ones and zero elsewhere;\n        in other words ``T[i,j] == 1`` for ``i <= j + k``, 0 otherwise.\n    \"\"\"\n    if dtype is None:\n        dtype = 'float32'\n    if M is None:\n        M = N\n    if ctx is None:\n        ctx = current_context()\n    return _npi.tri(N, M, k, dtype, ctx)\n\n\ndef repeat(a, repeats, axis=None):\n    \"\"\"\n    Repeat elements of an array.\n\n    Parameters\n    ----------\n    a : array_like\n        Input array.\n    repeats : int\n        The number of repetitions for each element.\n    axis : int, optional\n        The axis along which to repeat values.  By default, use the\n        flattened input array, and return a flat output array.\n\n    Returns\n    -------\n    repeated_array : ndarray\n        Output array which has the same shape as `a`, except along\n        the given axis.\n\n    See Also\n    --------\n    tile : Tile an array.\n\n    Examples\n    --------\n    >>> np.repeat(3, 4)\n    array([3, 3, 3, 3])\n    >>> x = np.array([[1,2],[3,4]])\n    >>> np.repeat(x, 2)\n    array([1, 1, 2, 2, 3, 3, 4, 4])\n    >>> np.repeat(x, 3, axis=1)\n    array([[1, 1, 1, 2, 2, 2],\n           [3, 3, 3, 4, 4, 4]])\n    >>> np.repeat(x, [1, 2], axis=0)\n    array([[1, 2],\n           [3, 4],\n           [3, 4]])\n    \"\"\"\n    if isinstance(repeats, numeric_types):\n        repeats = [repeats]\n    if axis is not None:\n        tmp = swapaxes(a, 0, axis)\n        res = _npi.repeats(tmp, repeats=repeats, axis=0)\n        return swapaxes(res, 0, axis)\n    return _npi.repeats(a, repeats=repeats, axis=axis)\n\n\ndef _unary_func_helper(x, fn_array, fn_scalar, out=None, **kwargs):\n    \"\"\"Helper function for unary operators.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input of the unary operator.\n    fn_array : function\n        Function to be called if x is of ``_Symbol`` type.\n    fn_scalar : function\n        Function to be called if x is a Python scalar.\n    out : _Symbol\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    out : _Symbol or scalar\n        Result _Symbol or scalar.\n    \"\"\"\n    if isinstance(x, numeric_types):\n        return fn_scalar(x, **kwargs)\n    elif isinstance(x, _Symbol):\n        return fn_array(x, out=out, **kwargs)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(x))))\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef sin(x, out=None, **kwargs):\n    r\"\"\"\n    Trigonometric sine, element-wise.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Angle, in radians (:math:`2 \\pi` rad equals 360 degrees).\n    out : _Symbol or None\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : _Symbol\n        The sine of each element of x.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n    \"\"\"\n    return _unary_func_helper(x, _npi.sin, _np.sin, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef cos(x, out=None, **kwargs):\n    r\"\"\"\n    Cosine, element-wise.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Angle, in radians (:math:`2 \\pi` rad equals 360 degrees).\n    out : _Symbol or None\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : _Symbol\n        The corresponding cosine values. This is a scalar if x is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n    \"\"\"\n    return _unary_func_helper(x, _npi.cos, _np.cos, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef sinh(x, out=None, **kwargs):\n    \"\"\"\n    Hyperbolic sine, element-wise.\n    Equivalent to ``1/2 * (np.exp(x) - np.exp(-x))`` or ``-1j * np.sin(1j*x)``.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array or scalar.\n    out : _Symbol or None\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : _Symbol or scalar\n        The corresponding hyperbolic sine values. This is a scalar if `x` is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n    \"\"\"\n    return _unary_func_helper(x, _npi.sinh, _np.sinh, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef cosh(x, out=None, **kwargs):\n    \"\"\"\n    Hyperbolic cosine, element-wise.\n    Equivalent to ``1/2 * (np.exp(x) + np.exp(-x))`` and ``np.cos(1j*x)``.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array or scalar.\n    out : _Symbol or None\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : _Symbol or scalar\n        The corresponding hyperbolic cosine values. This is a scalar if `x` is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n    \"\"\"\n    return _unary_func_helper(x, _npi.cosh, _np.cosh, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef tanh(x, out=None, **kwargs):\n    \"\"\"\n    Compute hyperbolic tangent element-wise.\n    Equivalent to ``np.sinh(x)/np.cosh(x)``.\n\n    Parameters\n    ----------\n    x : _Symbol\n        Input array.\n    out : _Symbol or None\n          Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : _Symbol\n        The corresponding hyperbolic tangent values.\n\n    Notes\n    -----\n    If `out` is provided, the function writes the result into it,\n    and returns a reference to `out`.  (See Examples)\n    - input x does not support complex computation (like imaginary number)\n    >>> np.tanh(np.pi*1j)\n    TypeError: type <type 'complex'> not supported\n\n    Examples\n    --------\n    >>> np.tanh(np.array[0, np.pi]))\n    array([0.       , 0.9962721])\n    >>> np.tanh(np.pi)\n    0.99627207622075\n    >>> # Example of providing the optional output parameter illustrating\n    >>> # that what is returned is a reference to said parameter\n    >>> out1 = np.array(1)\n    >>> out2 = np.tanh(np.array(0.1), out1)\n    >>> out2 is out1\n    True\n    >>> # Example of ValueError due to provision of shape mis-matched `out`\n    >>> np.tanh(np.zeros((3,3)),np.zeros((2,2)))\n    mxnet.base.MXNetError:\n    [07:17:36] ../src/ndarray/./../operator/tensor/../elemwise_op_common.h:135:\n    Check failed: assign(&dattr, vec.at(i)): Incompatible attr in node\n    at 0-th output: expected [3,3], got [2,2]\n    \"\"\"\n    return _unary_func_helper(x, _npi.tanh, _np.tanh, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef log10(x, out=None, **kwargs):\n    \"\"\"\n    Return the base 10 logarithm of the input array, element-wise.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array or scalar.\n    out : _Symbol or None\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : _Symbol or scalar\n        The logarithm to the base 10 of `x`, element-wise. NaNs are\n        returned where x is negative. This is a scalar if `x` is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n    \"\"\"\n    return _unary_func_helper(x, _npi.log10, _np.log10, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef sqrt(x, out=None, **kwargs):\n    \"\"\"\n    Return the non-negative square-root of an array, element-wise.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        The values whose square-roots are required.\n    out : _Symbol, or None, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : _Symbol or scalar\n        An array of the same shape as `x`, containing the positive\n        square-root of each element in `x`. This is a scalar if `x` is a scalar.\n\n    Notes\n    ----\n    This function only supports input type of float.\n    \"\"\"\n    return _unary_func_helper(x, _npi.sqrt, _np.sqrt, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef cbrt(x, out=None, **kwargs):\n    r\"\"\"\n    Return the cube-root of an array, element-wise.\n\n    Parameters\n    ----------\n    x : _Symbol\n        The values whose cube-roots are required.\n    out : _Symbol or None\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    ----------\n    y : _Symbol\n        An array of the same shape as x, containing the cube cube-root of each element in x.\n        If out was provided, y is a reference to it. This is a scalar if x is a scalar.\n    \"\"\"\n    return _unary_func_helper(x, _npi.cbrt, _np.cbrt, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef abs(x, out=None, **kwargs):\n    r\"\"\"\n    Calculate the absolute value element-wise.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array.\n    out : _Symbol or None\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    absolute : _Symbol\n        An ndarray containing the absolute value of\n        each element in `x`. This is a scalar if `x` is a scalar.\n    \"\"\"\n    return _unary_func_helper(x, _npi.abs, _np.abs, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef fabs(x, out=None, **kwargs):\n    r\"\"\"\n    Calculate the absolute value element-wise.\n\n    This function returns the absolute values (positive magnitude) of the\n    data in `x`. Complex values are not handled, use `absolute` to find the\n    absolute values of complex data.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array.\n    out : _Symbol or None\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    absolute : _Symbol\n        An ndarray containing the absolute value of\n        each element in `x`. This is a scalar if `x` is a scalar.\n    \"\"\"\n    return _unary_func_helper(x, _npi.abs, _np.abs, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef absolute(x, out=None, **kwargs):\n    r\"\"\"\n    Calculate the absolute value element-wise.\n    np.abs is a shorthand for this function.\n\n    Parameters\n    ----------\n    x : _Symbol\n        Input array.\n    out : _Symbol or None\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    ----------\n    absolute : _Symbol\n        An ndarray containing the absolute value of each element in x.\n    \"\"\"\n    return _unary_func_helper(x, _npi.absolute, _np.absolute, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef sign(x, out=None, **kwargs):\n    r\"\"\"\n    Returns an element-wise indication of the sign of a number.\n    The `sign` function returns ``-1 if x < 0, 0 if x==0, 1 if x > 0``. Only supports real number.\n\n    Parameters\n    ----------\n    x : _Symbol or a scalar\n        Input values.\n    out : _Symbol or None, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : _Symbol\n        The sign of `x`.\n        This is a scalar if `x` is a scalar.\n\n    Note\n    -------\n    - Only supports real number as input elements.\n    - Input type does not support Python native iterables(list, tuple, ...)\n    - ``out`` param: cannot perform auto broadcasting. ``out`` symbol's shape must be the same as the expected output.\n    - ``out`` param: cannot perform auto type cast. ``out`` symbol's dtype must be the same as the expected output.\n    - ``out`` param does not support scalar input case.\n    \"\"\"\n    return _unary_func_helper(x, _npi.sign, _np.sign, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef exp(x, out=None, **kwargs):\n    r\"\"\"\n    Calculate the exponential of all elements in the input array.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input values.\n    out : _Symbol or None\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    out : _Symbol\n        Output array, element-wise exponential of `x`.\n        This is a scalar if `x` is a scalar.\n    \"\"\"\n    return _unary_func_helper(x, _npi.exp, _np.exp, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef expm1(x, out=None, **kwargs):\n    r\"\"\"\n    Calculate `exp(x) - 1` for all elements in the array.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input values.\n    out : _Symbol or None\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    out : _Symbol\n        Output array, .\n        This is a scalar if `x` is a scalar.\n    \"\"\"\n    return _unary_func_helper(x, _npi.expm1, _np.expm1, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef arcsin(x, out=None, **kwargs):\n    r\"\"\"\n    Inverse sine, element-wise.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        The values whose reciprocals are required.\n    out : _Symbol, or None, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    angle : _Symbol or scalar\n        Output array is same shape and type as x. This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    `arcsin` is a multivalued function: for each `x` there are infinitely\n    many numbers `z` such that :math:`sin(z) = x`.  The convention is to\n    return the angle `z` whose real part lies in [-pi/2, pi/2].\n    For real-valued input data types, *arcsin* always returns real output.\n    For each value that cannot be expressed as a real number or infinity,\n    it yields ``nan`` and sets the `invalid` floating point error flag.\n    The inverse sine is also known as `asin` or sin^{-1}.\n    The output `symbol` has the same `ctx` as the input `symbol`.\n    This function differs from the original `numpy.arcsin\n    <https://numpy.org/doc/stable/reference/generated/numpy.arcsin.html>`_ in\n    the following aspects:\n    - Only support _Symbol or scalar now.\n    - `where` argument is not supported.\n    - Complex input is not supported.\n\n    References\n    ----------\n    Abramowitz, M. and Stegun, I. A., *Handbook of Mathematical Functions*,\n    10th printing, New York: Dover, 1964, pp. 79ff.\n    http://www.math.sfu.ca/~cbm/aands/\n    \"\"\"\n    return _unary_func_helper(x, _npi.arcsin, _np.arcsin, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef arccos(x, out=None, **kwargs):\n    r\"\"\"\n    Trigonometric inverse cosine, element-wise.\n    The inverse of cos so that, if y = cos(x), then x = arccos(y).\n\n    Parameters\n    ----------\n    x : _Symbol\n        x-coordinate on the unit circle. For real arguments, the domain is [-1, 1].\n    out : _Symbol or None\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    ----------\n    angle : _Symbol\n        The angle of the ray intersecting the unit circle at the given x-coordinate in radians [0, pi].\n        This is a scalar if x is a scalar.\n\n    See also\n    ----------\n    cos, arctan, arcsin\n\n    Notes\n    ----------\n    arccos is a multivalued function: for each x there are infinitely many numbers z such that\n    cos(z) = x. The convention is to return the angle z whose real part lies in [0, pi].\n    For real-valued input data types, arccos always returns real output.\n    For each value that cannot be expressed as a real number or infinity, it yields nan and sets\n    the invalid floating point error flag.\n    The inverse cos is also known as acos or cos^-1.\n    \"\"\"\n    return _unary_func_helper(x, _npi.arccos, _np.arccos, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef arctan(x, out=None, **kwargs):\n    r\"\"\"\n    Trigonometric inverse tangent, element-wise.\n    The inverse of tan, so that if ``y = tan(x)`` then ``x = arctan(y)``.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input values.\n    out : _Symbol or None\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    out : _Symbol\n        Out has the same shape as `x`. It lies is in\n        ``[-pi/2, pi/2]`` (``arctan(+/-inf)`` returns ``+/-pi/2``).\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    `arctan` is a multi-valued function: for each `x` there are infinitely\n    many numbers `z` such that tan(`z`) = `x`.  The convention is to return\n    the angle `z` whose real part lies in [-pi/2, pi/2].\n    For real-valued input data types, `arctan` always returns real output.\n    For each value that cannot be expressed as a real number or infinity,\n    it yields ``nan`` and sets the `invalid` floating point error flag.\n    For complex-valued input, we do not have support for them yet.\n    The inverse tangent is also known as `atan` or tan^{-1}.\n    \"\"\"\n    return _unary_func_helper(x, _npi.arctan, _np.arctan, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef log(x, out=None, **kwargs):\n    \"\"\"\n    Natural logarithm, element-wise.\n    The natural logarithm `log` is the inverse of the exponential function,\n    so that `log(exp(x)) = x`. The natural logarithm is logarithm in base\n    `e`.\n\n    Parameters\n    ----------\n    x : _Symbol\n        Input value. Elements must be of real value.\n    out : _Symbol or None, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : _Symbol\n        The natural logarithm of `x`, element-wise.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n     Currently only supports data of real values and ``inf`` as input. Returns data of real value, ``inf``, ``-inf`` and\n    ``nan`` according to the input.\n    This function differs from the original `numpy.log\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.log.html>`_ in\n    the following aspects:\n    - Does not support complex number for now\n    - Input type does not support Python native iterables(list, tuple, ...). Only ndarray is supported.\n    - ``out`` param: cannot perform auto braodcasting. ``out`` symbol's shape must be the same as the expected output.\n    - ``out`` param: cannot perform auto type cast. ``out`` symbol's dtype must be the same as the expected output.\n    - ``out`` param does not support scalar input case.\n    \"\"\"\n    return _unary_func_helper(x, _npi.log, _np.log, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef degrees(x, out=None, **kwargs):\n    \"\"\"\n    Convert angles from radians to degrees.\n\n    Parameters\n    ----------\n    x : _Symbol\n        Input value. Elements must be of real value.\n    out : _Symbol or None, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : _Symbol of floats\n        The corresponding degree values; if `out` was supplied this is a\n        reference to it.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -------\n    This function differs from the original `numpy.degrees\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.degrees.html>`_ in\n    the following aspects:\n    - Input type does not support Python native iterables(list, tuple, ...). Only ndarray is supported.\n    - ``out`` param: cannot perform auto broadcasting. ``out`` symbol's shape must be the same as the expected output.\n    - ``out`` param: cannot perform auto type cast. ``out`` symbol's dtype must be the same as the expected output.\n    - ``out`` param does not support scalar input case.\n    \"\"\"\n    return _unary_func_helper(x, _npi.degrees, _np.degrees, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef rad2deg(x, out=None, **kwargs):\n    r\"\"\"\n    Convert angles from radians to degrees.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Angles in degrees.\n    out : _Symbol or None, optional\n        A location into which the result is stored.\n\n    Returns\n    -------\n    y : _Symbol or scalar\n        The corresponding angle in radians.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    \"rad2deg(x)\" is \"x * 180 / pi\".\n\n    This function differs from the original numpy.arange in the following aspects:\n        - Only support float32 and float64.\n        - `out` must be in the same size of input.\n    \"\"\"\n    return _unary_func_helper(x, _npi.rad2deg, _np.rad2deg, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef rint(x, out=None, **kwargs):\n    \"\"\"\n    Round elements of the array to the nearest integer.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array.\n    out : _Symbol or None\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    out : _Symbol or scalar\n        Output array is same shape and type as x. This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    This function differs from the original `numpy.rint\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.rint.html>`_ in\n    the following way(s):\n    - only _Symbol or scalar is accpted as valid input, tuple of _Symbol is not supported\n     - broadcasting to `out` of different shape is currently not supported\n    - when input is plain python numerics, the result will not be stored in the `out` param\n    \"\"\"\n    return _unary_func_helper(x, _npi.rint, _np.rint, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef log2(x, out=None, **kwargs):\n    \"\"\"\n    Base-2 logarithm of x.\n    Parameters\n    ----------\n    x : _Symbol\n        Input values.\n    out : _Symbol or None\n        A location into which the result is stored.\n        If provided, it must have the same shape and type as the input.\n        If not provided or None, a freshly-allocated array is returned.\n    Returns\n    -------\n    y : _Symbol\n        The logarithm base two of `x`, element-wise.\n        This is a scalar if `x` is a scalar.\n    Notes\n    -----\n    This function differs from the original `numpy.log2\n    <https://www.google.com/search?q=numpy+log2>`_ in\n    the following way(s):\n    - only ndarray or scalar is accpted as valid input, tuple of ndarray is not supported\n    - broadcasting to `out` of different shape is currently not supported\n    - when input is plain python numerics, the result will not be stored in the `out` param\n    \"\"\"\n    return _unary_func_helper(x, _npi.log2, _np.log2, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef log1p(x, out=None, **kwargs):\n    \"\"\"\n    Return the natural logarithm of one plus the input array, element-wise.\n    Calculates ``log(1 + x)``.\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array.\n    out : _Symbol or None\n          Dummy parameter to keep the consistency with the ndarray counterpart.\n    Returns\n    -------\n    y : _Symbol or scalar\n        Natural logarithm of 1 + x, element-wise. This is a scalar\n        if x is a scalar.\n    Notes\n    -----\n    For real-valued input, `log1p` is accurate also for `x` so small\n    that `1 + x == 1` in floating-point accuracy.\n    Logarithm is a multivalued function: for each `x` there is an infinite\n    number of `z` such that `exp(z) = 1 + x`. The convention is to return\n    the `z` whose imaginary part lies in `[-pi, pi]`.\n    For real-valued input data types, `log1p` always returns real output.\n    For each value that cannot be expressed as a real number or infinity,\n    it yields ``nan`` and sets the `invalid` floating point error flag.\n    cannot support complex-valued input.\n    Examples\n    --------\n    >>> np.log1p(1e-99)\n    1e-99\n    >>> a = np.array([3, 4, 5])\n    >>> np.log1p(a)\n    array([1.3862944, 1.609438 , 1.7917595])\n    \"\"\"\n    return _unary_func_helper(x, _npi.log1p, _np.log1p, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef radians(x, out=None, **kwargs):\n    \"\"\"\n    Convert angles from degrees to radians.\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array in degrees.\n    out : _Symbol or None\n       Dummy parameter to keep the consistency with the ndarray counterpart.\n    Returns\n    -------\n    y : _Symbol\n        The corresponding radian values. This is a scalar if x is a scalar.\n    Notes\n    -----\n    This function differs from the original `numpy.radians\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.radians.html>`_ in\n    the following way(s):\n    - only _Symbol or scalar is accpted as valid input, tuple of _Symbol is not supported\n    - broadcasting to `out` of different shape is currently not supported\n    - when input is plain python numerics, the result will not be stored in the `out` param\n    Examples\n    --------\n    >>> deg = np.arange(12.) * 30.\n    >>> np.radians(deg)\n    array([0.       , 0.5235988, 1.0471976, 1.5707964, 2.0943952, 2.6179938,\n           3.1415927, 3.6651914, 4.1887903, 4.712389 , 5.2359877, 5.7595863],\n           dtype=float32)\n    \"\"\"\n    return _unary_func_helper(x, _npi.radians, _np.radians, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef deg2rad(x, out=None, **kwargs):\n    r\"\"\"\n    deg2rad(x, out=None)\n\n    Convert angles from degrees to radians.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Angles in degrees.\n    out : _Symbol or None, optional\n        A location into which the result is stored.\n\n    Returns\n    -------\n    y : _Symbol or scalar\n        The corresponding angle in radians.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    \"deg2rad(x)\" is \"x * pi / 180\".\n\n    This function differs from the original numpy.arange in the following aspects:\n        - Only support float32 and float64.\n        - `out` must be in the same size of input.\n    \"\"\"\n    return _unary_func_helper(x, _npi.deg2rad, _np.deg2rad, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef reciprocal(x, out=None, **kwargs):\n    r\"\"\"\n    Return the reciprocal of the argument, element-wise.\n    Calculates ``1/x``.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        The values whose reciprocals are required.\n    out : _Symbol, or None, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : _Symbol or scalar\n        Output array is same shape and type as x. This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    .. note::\n        This function is not designed to work with integers.\n    For integer arguments with absolute value larger than 1 the result is\n    always zero because of the way Python handles integer division.  For\n    integer zero the result is an overflow.\n    The output `symbol` has the same `ctx` as the input `symbol`.\n    This function differs from the original `numpy.reciprocal\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.reciprocal.html>`_ in\n    the following aspects:\n    - Only support _Symbol and scalar now.\n    - `where` argument is not supported.\n    \"\"\"\n    return _unary_func_helper(x, _npi.reciprocal, _np.reciprocal, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef square(x, out=None, **kwargs):\n    r\"\"\"\n    Return the element-wise square of the input.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        The values whose reciprocals are required.\n    out : _Symbol, or None, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : _Symbol or scalar\n        Output array is same shape and type as x. This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    The output `symbol` has the same `ctx` as the input `symbol`.\n    This function differs from the original `numpy.square\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.square.html>`_ in\n    the following aspects:\n    - Only support _Symbol and scalar now.\n    - `where` argument is not supported.\n    \"\"\"\n    return _unary_func_helper(x, _npi.square, _np.square, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef negative(x, out=None, **kwargs):\n    r\"\"\"\n    Numerical negative, element-wise.\n\n    Parameters:\n    ------------\n    x : _Symbol or scalar\n        Input array.\n    out : _Symbol or None, optional\n          A location into which the result is stored.\n          If provided, it must have a shape that the inputs broadcast to.\n          If not provided or None, a freshly-allocated array is returned.\n          A tuple (possible only as a keyword argument) must have length\n          equal to the number of outputs.\n\n    Returns:\n    -------\n    y : _Symbol or scalar\n        Returned array or scalar: y = -x. This is a scalar if x is a scalar.\n\n    Examples:\n    ---------\n    >>> np.negative(1)\n    -1\n    \"\"\"\n    return _unary_func_helper(x, _npi.negative, _np.negative, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef fix(x, out=None, **kwargs):\n    \"\"\"\n    Round to nearest integer towards zero.\n\n    Round an array of floats element-wise to nearest integer towards zero. The rounded values are returned as floats.\n\n    Parameters:\n    ----------\n    x : _Symbol or scalar\n        An array of floats to be rounded\n    out : _Symbol or scalar, optional\n          Output array\n\n    Returns:\n    ---------\n    y : _Symbol or scalar\n\n    Examples:\n    ----------\n    >>> np.fix(3.14)\n    3\n    \"\"\"\n    return _unary_func_helper(x, _npi.fix, _np.fix, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef tan(x, out=None, **kwargs):\n    r\"\"\"\n    Compute tangent element-wise.\n    Equivalent to np.sin(x)/np.cos(x) element-wise.\n\n    Parameters:\n    ----------\n    x : _Symbol or scalar\n        Input array.\n    out : _Symbol or scalar or None.\n        A location into which the result is stored. If provided,\n        it must have a shape that the inputs broadcast to. If not provided or None,\n        a freshly-allocated array is returned. A tuple (possible only as a keyword argument)\n        must have length equal to the number of outputs.\n\n    Returns:\n    -------\n    y : _Symbol or scalar\n        The corresponding tangent values. This is a scalar if x is a scalar.\n    \"\"\"\n\n    return _unary_func_helper(x, _npi.tan, _np.tan, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef ceil(x, out=None, **kwargs):\n    r\"\"\"\n    Return the ceiling of the input, element-wise.\n    The ceil of the ndarray `x` is the smallest integer `i`, such that\n    `i >= x`.  It is often denoted as :math:`\\lceil x \\rceil`.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array.\n    out : _Symbol or None\n          Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : _Symbol or scalar\n        The ceiling of each element in `x`, with `float` dtype.\n        This is a scalar if `x` is a scalar.\n\n    Examples\n    --------\n    >>> a = np.array([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0])\n    >>> np.ceil(a)\n    array([-1., -1., -0.,  1.,  2.,  2.,  2.])\n    >>> #if you use parameter out, x and out must be ndarray. if not, you will get an error!\n    >>> a = np.array(1)\n    >>> np.ceil(np.array(3.5), a)\n    array(4.)\n    >>> a\n    array(4.)\n    \"\"\"\n    return _unary_func_helper(x, _npi.ceil, _np.ceil, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\ndef insert(arr, obj, values, axis=None):\n    \"\"\"\n    Insert values along the given axis before the given indices.\n\n    Parameters\n    ----------\n    arr : _Symbol\n        Input array.\n    obj : int, slice or ndarray of int64\n        Object that defines the index or indices before which `values` is\n        inserted.\n        Support for multiple insertions when `obj` is a single scalar or a\n        sequence with one element (only support int32 and int64 element).\n    values : _Symbol\n        Values to insert into `arr`.\n        If the type of values is different from that of arr, values is converted\n        to the type of arr.\n    axis : int, optional\n        Axis along which to insert `values`.  If `axis` is None then `arr`\n        is flattened first.\n\n    Returns\n    -------\n    out : _Symbol\n        A copy of `arr` with `values` inserted.  Note that `insert`\n        does not occur in-place: a new array is returned. If\n        `axis` is None, `out` is a flattened array.\n\n    Notes\n    -----\n    - Note that for higher dimensional inserts `obj=0` behaves very different\n    from `obj=[0]` just like `arr[:,0,:] = values` is different from\n    `arr[:,[0],:] = values`.\n    - If obj is a ndarray, it's dtype only supports int64\n    \"\"\"\n    if isinstance(values, numeric_types):\n        if isinstance(obj, slice):\n            start = obj.start\n            stop = obj.stop\n            step = 1 if obj.step is None else obj.step\n            return _npi.insert_slice(arr, val=values, start=start, stop=stop, step=step, axis=axis)\n        elif isinstance(obj, integer_types):\n            return _npi.insert_scalar(arr, val=values, int_ind=obj, axis=axis)\n        elif isinstance(obj, Symbol):\n            return _npi.insert_tensor(arr, obj, val=values, axis=axis)\n    if not isinstance(arr, Symbol): # pylint: disable= undefined-variable\n        raise TypeError(\"'arr' can not support type {}\".format(str(type(arr))))\n    if not isinstance(values, Symbol): # pylint: disable= undefined-variable\n        raise TypeError(\"'values' can not support type {}\".format(str(type(values))))\n    if isinstance(obj, slice):\n        start = obj.start\n        stop = obj.stop\n        step = 1 if obj.step is None else obj.step\n        return _npi.insert_slice(arr, values, start=start, stop=stop, step=step, axis=axis)\n    elif isinstance(obj, integer_types):\n        return _npi.insert_scalar(arr, values, int_ind=obj, axis=axis)\n    elif isinstance(obj, Symbol):\n        return _npi.insert_tensor(arr, values, obj, axis=axis)\n    else:\n        raise TypeError(\"'obj' can not support type {}\".format(str(type(obj))))\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef floor(x, out=None, **kwargs):\n    r\"\"\"\n    Return the floor of the input, element-wise.\n    The floor of the ndarray `x` is the largest integer `i`, such that\n    `i <= x`.  It is often denoted as :math:`\\lfloor x \\rfloor`.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array.\n    out : _Symbol or None\n          Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : _Symbol or scalar\n        The floor of each element in `x`, with `float` dtype.\n        This is a scalar if `x` is a scalar.\n\n    Examples\n    --------\n    >>> a = np.array([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0])\n    >>> np.floor(a)\n    array([-2., -2., -1.,  0.,  1.,  1.,  2.])\n    >>> # if you use parameter out, x and out must be ndarray. if not, you will get an error!\n    >>> a = np.array(1)\n    >>> np.floor(np.array(3.5), a)\n    array(3.)\n    >>> a\n    array(3.)\n    \"\"\"\n    return _unary_func_helper(x, _npi.floor, _np.floor, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef trunc(x, out=None, **kwargs):\n    r\"\"\"\n    Return the truncated value of the input, element-wise.\n    The truncated value of the scalar `x` is the nearest integer `i` which\n    is closer to zero than `x` is. In short, the fractional part of the\n    signed number `x` is discarded.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input data.\n    out : _Symbol or None, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : _Symbol or scalar\n        The truncated value of each element in `x`.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    This function differs from the original numpy.trunc in the following aspects:\n        - Do not support `where`, a parameter in numpy which indicates where to calculate.\n        - Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n        - Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n        - If `x` is plain python numeric, the result won't be stored in out.\n    \"\"\"\n    return _unary_func_helper(x, _npi.trunc, _np.trunc, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef logical_not(x, out=None, **kwargs):\n    r\"\"\"\n    Compute the truth value of NOT x element-wise.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Logical NOT is applied to the elements of `x`.\n    out : _Symbol or None, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : bool or _Symbol\n        Boolean result with the same shape as `x` of the NOT operation\n        on elements of `x`.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    This function differs from the original numpy.logical_not in the following aspects:\n        - Do not support `where`, a parameter in numpy which indicates where to calculate.\n        - Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n        - Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n        - If `x` is plain python numeric, the result won't be stored in out.\n    \"\"\"\n    return _unary_func_helper(x, _npi.logical_not, _np.logical_not, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef arcsinh(x, out=None, **kwargs):\n    r\"\"\"\n    Inverse hyperbolic sine, element-wise.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array.\n    out : _Symbol or None, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    arcsinh : _Symbol\n        Array of the same shape as `x`.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    `arcsinh` is a multivalued function: for each `x` there are infinitely\n    many numbers `z` such that `sinh(z) = x`.\n\n    For real-valued input data types, `arcsinh` always returns real output.\n    For each value that cannot be expressed as a real number or infinity, it\n    yields ``nan`` and sets the `invalid` floating point error flag.\n\n    This function differs from the original numpy.arcsinh in the following aspects:\n        - Do not support `where`, a parameter in numpy which indicates where to calculate.\n        - Do not support complex-valued input.\n        - Cannot cast type automatically. DType of `out` must be same as the expected one.\n        - Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n        - If `x` is plain python numeric, the result won't be stored in out.\n    \"\"\"\n    return _unary_func_helper(x, _npi.arcsinh, _np.arcsinh, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef arccosh(x, out=None, **kwargs):\n    r\"\"\"\n    Inverse hyperbolic cosine, element-wise.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array.\n    out : _Symbol or None, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    arccosh : _Symbol\n        Array of the same shape as `x`.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    `arccosh` is a multivalued function: for each `x` there are infinitely\n    many numbers `z` such that `cosh(z) = x`.\n\n    For real-valued input data types, `arccosh` always returns real output.\n    For each value that cannot be expressed as a real number or infinity, it\n    yields ``nan`` and sets the `invalid` floating point error flag.\n\n    This function differs from the original numpy.arccosh in the following aspects:\n        - Do not support `where`, a parameter in numpy which indicates where to calculate.\n        - Do not support complex-valued input.\n        - Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n        - Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n        - If `x` is plain python numeric, the result won't be stored in out.\n    \"\"\"\n    return _unary_func_helper(x, _npi.arccosh, _np.arccosh, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef arctanh(x, out=None, **kwargs):\n    r\"\"\"\n    Inverse hyperbolic tangent, element-wise.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array.\n    out : _Symbol or None, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    arctanh : _Symbol\n        Array of the same shape as `x`.\n        This is a scalar if `x` is a scalar.\n\n    Notes\n    -----\n    `arctanh` is a multivalued function: for each `x` there are infinitely\n    many numbers `z` such that `tanh(z) = x`.\n\n    For real-valued input data types, `arctanh` always returns real output.\n    For each value that cannot be expressed as a real number or infinity, it\n    yields ``nan`` and sets the `invalid` floating point error flag.\n\n    This function differs from the original numpy.arctanh in the following aspects:\n        - Do not support `where`, a parameter in numpy which indicates where to calculate.\n        - Do not support complex-valued input.\n        - Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n        - Cannot broadcast automatically. Shape of `out` must be same as the expected one.\n        - If `x` is plain python numeric, the result won't be stored in out.\n    \"\"\"\n    return _unary_func_helper(x, _npi.arctanh, _np.arctanh, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\ndef tile(A, reps):\n    r\"\"\"\n    Construct an array by repeating A the number of times given by reps.\n\n    If `reps` has length ``d``, the result will have dimension of\n    ``max(d, A.ndim)``.\n\n    If ``A.ndim < d``, `A` is promoted to be d-dimensional by prepending new\n    axes. So a shape (3,) array is promoted to (1, 3) for 2-D replication,\n    or shape (1, 1, 3) for 3-D replication. If this is not the desired\n    behavior, promote `A` to d-dimensions manually before calling this\n    function.\n\n    If ``A.ndim > d``, `reps` is promoted to `A`.ndim by pre-pending 1's to it.\n    Thus for an `A` of shape (2, 3, 4, 5), a `reps` of (2, 2) is treated as\n    (1, 1, 2, 2).\n\n    Parameters\n    ----------\n    A : _Symbol or scalar\n        An input array or a scalar to repeat.\n    reps : a single integer or tuple of integers\n        The number of repetitions of `x` along each axis.\n\n    Returns\n    -------\n    c : _Symbol\n        The tiled output array.\n    \"\"\"\n    return _unary_func_helper(A, _npi.tile, _np.tile, reps=reps)\n\n\n@set_module('mxnet.symbol.numpy')\ndef arange(start, stop=None, step=1, dtype=None, ctx=None):\n    \"\"\"Return evenly spaced values within a given interval.\n\n    Values are generated within the half-open interval ``[start, stop)``\n    (in other words, the interval including `start` but excluding `stop`).\n    For integer arguments the function is equivalent to the Python built-in\n    `range` function, but returns an ndarray rather than a list.\n\n    Parameters\n    ----------\n    start : number, optional\n        Start of interval. The interval includes this value.  The default\n        start value is 0.\n    stop : number\n        End of interval. The interval does not include this value, except\n        in some cases where `step` is not an integer and floating point\n        round-off affects the length of `out`.\n    step : number, optional\n        Spacing between values. For any output `out`, this is the distance\n        between two adjacent values, ``out[i+1] - out[i]``.  The default\n        step size is 1.  If `step` is specified as a position argument,\n        `start` must also be given.\n    dtype : dtype\n        The type of the output array.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n\n    Returns\n    -------\n    arange : _Symbol\n        Array of evenly spaced values.\n\n        For floating point arguments, the length of the result is\n        ``ceil((stop - start)/step)``.  Because of floating point overflow,\n        this rule may result in the last element of `out` being greater\n        than `stop`.\n    \"\"\"\n    if ctx is None:\n        ctx = current_context()\n    if stop is None:\n        stop = start\n        start = 0\n    if step is None:\n        step = 1\n    if start is None and stop is None:\n        raise ValueError('start and stop cannot be both None')\n    if step == 0:\n        raise ZeroDivisionError('step cannot be 0')\n    return _npi.arange(start=start, stop=stop, step=step, dtype=dtype, ctx=ctx)\n\n\n@set_module('mxnet.symbol.numpy')\ndef delete(arr, obj, axis=None):\n    \"\"\"\n    Return a new array with sub-arrays along an axis deleted. For a one\n    dimensional array, this returns those entries not returned by\n    `arr[obj]`.\n\n    Parameters\n    ----------\n    arr : _Symbol\n      Input array.\n    obj : slice, scaler or _Symbol of ints\n      Indicate indices of sub-arrays to remove along the specified axis.\n    axis : scaler, optional\n      The axis along which to delete the subarray defined by `obj`.\n      If `axis` is None, `obj` is applied to the flattened array.\n\n    Returns\n    -------\n    out : _Symbol\n        A copy of `arr` with the elements specified by `obj` removed. Note\n        that `delete` does not occur in-place. If `axis` is None, `out` is\n        a flattened array.\n    \"\"\"\n    if not isinstance(arr, Symbol):\n        raise TypeError(\"'arr' can not support type {}\".format(str(type(arr))))\n    if isinstance(obj, slice):\n        start = obj.start\n        stop = obj.stop\n        step = 1 if obj.step is None else obj.step\n        return _npi.delete(arr, start=start, stop=stop, step=step, axis=axis)\n    elif isinstance(obj, integer_types):\n        return _npi.delete(arr, int_ind=obj, axis=axis)\n    elif isinstance(obj, Symbol):\n        return _npi.delete(arr, obj, axis=axis)\n    else:\n        raise TypeError(\"'obj' can not support type {}\".format(str(type(obj))))\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.symbol.numpy')\ndef split(ary, indices_or_sections, axis=0):\n    \"\"\"Split an array into multiple sub-arrays.\n\n    Parameters\n    ----------\n    ary : _Symbol\n        Array to be divided into sub-arrays.\n    indices_or_sections : int or 1-D python tuple, list or set.\n        If `indices_or_sections` is an integer, N, the array will be divided\n        into N equal arrays along `axis`.  If such a split is not possible,\n        an error is raised.\n        If `indices_or_sections` is a 1-D array of sorted integers, the entries\n        indicate where along `axis` the array is split.  For example,\n        ``[2, 3]`` would, for ``axis=0``, result in\n          - ary[:2]\n          - ary[2:3]\n          - ary[3:]\n        If an index exceeds the dimension of the array along `axis`,\n        an empty sub-array is returned correspondingly.\n    axis : int, optional\n        The axis along which to split, default is 0.\n\n    Returns\n    -------\n    sub-arrays : _Symbol\n        A list of sub-arrays.\n\n    Raises\n    ------\n    ValueError\n        If `indices_or_sections` is given as an integer, but\n        a split does not result in equal division.\"\"\"\n    indices = []\n    sections = 0\n    if isinstance(indices_or_sections, int):\n        sections = indices_or_sections\n    elif isinstance(indices_or_sections, (list, set, tuple)):\n        indices = [0] + list(indices_or_sections)\n    else:\n        raise ValueError('indices_or_sections must either int or tuple / list / set of ints')\n    return _npi.split(ary, indices, axis, False, sections)\n# pylint: enable=redefined-outer-name\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.symbol.numpy')\ndef array_split(ary, indices_or_sections, axis=0):\n    \"\"\"Split an array into multiple sub-arrays.\n\n    If `indices_or_sections` is an integer, N, the array will be divided\n    into N equal arrays along `axis`.  If such a split is not possible,\n    an array of length l that should be split into n sections, it returns\n    l % n sub-arrays of size l//n + 1 and the rest of size l//n.\n\n    If `indices_or_sections` is a 1-D array of sorted integers, the entries\n        indicate where along `axis` the array is split.  For example,\n        ``[2, 3]`` would, for ``axis=0``, result in\n          - ary[:2]\n          - ary[2:3]\n          - ary[3:]\n    If an index exceeds the dimension of the array along `axis`,\n    an empty sub-array is returned correspondingly.\n\n    Parameters\n    ----------\n    ary : _Symbol\n        Array to be divided into sub-arrays.\n    indices_or_sections : int or 1-D Python tuple, list or set.\n        Param used to determine the number and size of the subarray.\n    axis : int, optional\n        The axis along which to split, default is 0.\n\n    Returns\n    -------\n    sub-arrays : list of ndarrays\n        A list of sub-arrays.\n    \"\"\"\n    indices = []\n    sections = 0\n    if isinstance(indices_or_sections, int):\n        sections = indices_or_sections\n    elif isinstance(indices_or_sections, (list, set, tuple)):\n        indices = [0] + list(indices_or_sections)\n    else:\n        raise ValueError('indices_or_sections must either int or tuple / list / set of ints')\n    ret = _npi.array_split(ary, indices, axis, False, sections)\n    if not isinstance(ret, list):\n        return [ret]\n    return ret\n# pylint: enable=redefined-outer-name\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.symbol.numpy')\ndef hsplit(ary, indices_or_sections):\n    \"\"\"Split an array into multiple sub-arrays horizontally (column-wise).\n\n    This is equivalent to ``split`` with ``axis=0`` if ``ary`` has one\n    dimension, and otherwise that with ``axis=1``.\n\n    Parameters\n    ----------\n    ary : _Symbol\n        Array to be divided into sub-arrays.\n    indices_or_sections : int, list of ints or tuple of ints.\n        If `indices_or_sections` is an integer, N, the array will be divided\n        into N equal arrays along `axis`.  If such a split is not possible,\n        an error is raised.\n\n        If `indices_or_sections` is a list of sorted integers, the entries\n        indicate where along `axis` the array is split.\n\n        If an index exceeds the dimension of the array along `axis`,\n        it will raises errors. so index must less than or euqal to\n        the dimension of the array along axis.\n\n    Returns\n    -------\n    sub-arrays : _Symbol\n        A list of sub-arrays.\n\n    Notes\n    ------\n    - If `indices_or_sections` is given as an integer, but a split\n      does not result in equal division.It will raises ValueErrors.\n\n    - If indices_or_sections is an integer, and the number is 1, it will\n      raises an error. Because single output from split is not supported yet...\n\n    See Also\n    --------\n    split : Split an array into multiple sub-arrays of equal size.\n\n    Examples\n    --------\n    >>> x = np.arange(16.0).reshape(4, 4)\n    >>> x\n    array([[ 0.,  1.,  2.,  3.],\n           [ 4.,  5.,  6.,  7.],\n           [ 8.,  9., 10., 11.],\n           [12., 13., 14., 15.]])\n    >>> np.hsplit(x, 2)\n    [array([[ 0.,  1.],\n           [ 4.,  5.],\n           [ 8.,  9.],\n           [12., 13.]]),\n    array([[ 2.,  3.],\n           [ 6.,  7.],\n           [10., 11.],\n           [14., 15.]])]\n    >>> np.hsplit(x, [3, 6])\n    [array([[ 0.,  1.,  2.],\n           [ 4.,  5.,  6.],\n           [ 8.,  9., 10.],\n           [12., 13., 14.]]),\n    array([[ 3.],\n           [ 7.],\n           [11.],\n           [15.]]),\n    array([], shape=(4, 0), dtype=float32)]\n\n    With a higher dimensional array the split is still along the second axis.\n\n    >>> x = np.arange(8.0).reshape(2, 2, 2)\n    >>> x\n    array([[[ 0.,  1.],\n            [ 2.,  3.]],\n           [[ 4.,  5.],\n            [ 6.,  7.]]])\n    >>> np.hsplit(x, 2)\n    [array([[[ 0.,  1.]],\n            [[ 4.,  5.]]]),\n     array([[[ 2.,  3.]],\n            [[ 6.,  7.]]])]\n\n    If ``ary`` has one dimension, 'axis' = 0.\n    >>> x = np.arange(4)\n    array([0., 1., 2., 3.])\n    >>> np.hsplit(x, 2)\n    [array([0., 1.]), array([2., 3.])]\n\n    If you want to produce an empty sub-array, you can see an example.\n    >>> np.hsplit(x, [2, 2])\n    [array([0., 1.]), array([], dtype=float32), array([2., 3.])]\n    \"\"\"\n    indices = []\n    sections = 0\n    if isinstance(indices_or_sections, int):\n        sections = indices_or_sections\n    elif isinstance(indices_or_sections, (list, set, tuple)):\n        indices = [0] + list(indices_or_sections)\n    else:\n        raise ValueError('indices_or_sections must either int or tuple of ints')\n    return _npi.hsplit(ary, indices, 1, False, sections)\n# pylint: enable=redefined-outer-name\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.symbol.numpy')\ndef vsplit(ary, indices_or_sections):\n    r\"\"\"\n    vsplit(ary, indices_or_sections)\n\n    Split an array into multiple sub-arrays vertically (row-wise).\n\n    ``vsplit`` is equivalent to ``split`` with `axis=0` (default): the array is always split\n    along the first axis regardless of the array dimension.\n\n    Parameters\n    ----------\n    ary : _Symbol\n        Array to be divided into sub-arrays.\n    indices_or_sections : int or 1 - D Python tuple, list or set.\n        If `indices_or_sections` is an integer, N, the array will be divided into N equal arrays\n        along axis 0.  If such a split is not possible, an error is raised.\n\n        If `indices_or_sections` is a 1-D array of sorted integers, the entries indicate where\n        along axis 0 the array is split.  For example, ``[2, 3]`` would result in\n\n          - ary[:2]\n          - ary[2:3]\n          - ary[3:]\n\n        If an index exceeds the dimension of the array along axis 0, an error will be thrown.\n\n    Returns\n    -------\n    sub-arrays : list of _Symbols\n        A list of sub-arrays.\n\n    See Also\n    --------\n    split : Split an array into multiple sub-arrays of equal size.\n\n    Notes\n    -------\n    This function differs from the original `numpy.degrees\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.degrees.html>`_ in\n    the following aspects:\n\n    - Currently parameter ``indices_or_sections`` does not support ndarray, but supports scalar,\n    tuple and list\n    - In ``indices_or_sections``, if an index exceeds the dimension of the array along axis 0,\n    an error will be thrown.\n\n    \"\"\"\n    indices = []\n    sections = 0\n    if isinstance(indices_or_sections, int):\n        sections = indices_or_sections\n    elif isinstance(indices_or_sections, (list, set, tuple)):\n        indices = [0] + list(indices_or_sections)\n    else:\n        raise ValueError('indices_or_sections must either int or tuple of ints')\n    return _npi.split(ary, indices, 0, False, sections)\n# pylint: enable=redefined-outer-name\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.symbol.numpy')\ndef dsplit(ary, indices_or_sections):\n    \"\"\"\n    Split array into multiple sub-arrays along the 3rd axis (depth).\n\n    Please refer to the `split` documentation.  `dsplit` is equivalent\n    to `split` with ``axis=2``, the array is always split along the third\n    axis provided the array dimension is greater than or equal to 3.\n\n    Parameters\n    ----------\n    ary : _Symbol\n        Array to be divided into sub-arrays.\n    indices_or_sections : int or 1-D Python tuple, list or set.\n        If `indices_or_sections` is an integer, N, the array will be divided into N equal arrays\n        along axis 2.  If such a split is not possible, an error is raised.\n\n        If `indices_or_sections` is a 1-D array of sorted integers, the entries indicate where\n        along axis 2 the array is split.  For example, ``[2, 3]`` would result in\n\n          - ary[:, :, :2]\n          - ary[:, :, 2:3]\n          - ary[:, :, 3:]\n\n        If an index exceeds the dimension of the array along axis 2, an error will be thrown.\n    \"\"\"\n    indices = []\n    sections = 0\n    if isinstance(indices_or_sections, int):\n        sections = indices_or_sections\n    elif isinstance(indices_or_sections, (list, set, tuple)):\n        indices = [0] + list(indices_or_sections)\n    else:\n        raise ValueError('indices_or_sections must either int or tuple of ints')\n    return _npi.dsplit(ary, indices, 2, False, sections)\n# pylint: enable=redefined-outer-name\n\n\n@set_module('mxnet.symbol.numpy')\ndef concatenate(seq, axis=0, out=None):\n    \"\"\"Join a sequence of arrays along an existing axis.\n\n    Parameters\n    ----------\n    a1, a2, ... : sequence of _Symbols\n        The arrays must have the same shape, except in the dimension\n        corresponding to `axis` (the first, by default).\n    axis : int, optional\n        The axis along which the arrays will be joined.  If axis is None,\n        arrays are flattened before use.  Default is 0.\n    out : ndarray, optional\n        If provided, the destination to place the result. The shape must be\n        correct, matching that of what concatenate would have returned if no\n        out argument were specified.\n\n    Returns\n    -------\n    res : _Symbol\n        The concatenated array.\n\n    Examples\n    --------\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> b = np.array([[5, 6]])\n    >>> np.concatenate((a, b), axis=0)\n    array([[1., 2.],\n           [3., 4.],\n           [5., 6.]])\n\n    >>> np.concatenate((a, b), axis=None)\n    array([1., 2., 3., 4., 5., 6.])\n\n    >>> np.concatenate((a, b.T), axis=1)\n    array([[1., 2., 5.],\n           [3., 4., 6.]])\n    \"\"\"\n    return _npi.concatenate(*seq, axis=axis, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef append(arr, values, axis=None):  # pylint: disable=redefined-outer-name\n    \"\"\"\n    Append values to the end of an array.\n\n    Parameters\n    ----------\n    arr : _Symbol\n        Values are appended to a copy of this array.\n    values : _Symbol\n        These values are appended to a copy of `arr`.  It must be of the\n        correct shape (the same shape as `arr`, excluding `axis`).  If\n        `axis` is not specified, `values` can be any shape and will be\n        flattened before use.\n    axis : int, optional\n        The axis along which `values` are appended.  If `axis` is not\n        given, both `arr` and `values` are flattened before use.\n\n    Returns\n    -------\n    append : _Symbol\n        A copy of `arr` with `values` appended to `axis`.  Note that\n        `append` does not occur in-place: a new array is allocated and\n        filled.  If `axis` is None, `out` is a flattened array.\n\n    Examples\n    --------\n    >>> np.append(np.array([1, 2, 3]), np.array([[4, 5, 6],[7, 8, 9]]))\n    array([1., 2., 3., 4., 5., 6., 7., 8., 9.])\n\n    When `axis` is specified, `values` must have the correct shape.\n\n    >>> np.append(np.array([[1, 2, 3], [4, 5, 6]]), np.array([[7, 8, 9]]), axis=0)\n    array([[1., 2., 3.],\n           [4., 5., 6.],\n           [7., 8., 9.]])\n    \"\"\"\n    return _npi.concatenate(arr, values, axis=axis, out=None)\n\n\n@set_module('mxnet.symbol.numpy')\ndef stack(arrays, axis=0, out=None):\n    \"\"\"Join a sequence of arrays along a new axis.\n        The axis parameter specifies the index of the new axis in the dimensions of the result.\n        For example, if `axis=0` it will be the first dimension and if `axis=-1` it will be the last dimension.\n    Parameters\n    ----------\n    arrays : sequence of _Symbols\n        Each array must have the same shape.\n    axis : int, optional\n        The axis in the result array along which the input arrays are stacked.\n    out : _Symbol, optional\n        If provided, the destination to place the result. The shape must be correct,\n        matching that of what stack would have returned if no out argument were specified.\n    Returns\n    -------\n    stacked : _Symbol\n        The stacked array has one more dimension than the input arrays.\"\"\"\n    def get_list(arrays):\n        if not hasattr(arrays, '__getitem__') and hasattr(arrays, '__iter__'):\n            raise ValueError(\"expected iterable for arrays but got {}\".format(type(arrays)))\n        return [arr for arr in arrays]\n    arrays = get_list(arrays)\n    return _npi.stack(*arrays, axis=axis, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef vstack(arrays, out=None):\n    r\"\"\"Stack arrays in sequence vertically (row wise).\n\n    This is equivalent to concatenation along the first axis after 1-D arrays\n    of shape `(N,)` have been reshaped to `(1,N)`. Rebuilds arrays divided by\n    `vsplit`.\n\n    This function makes most sense for arrays with up to 3 dimensions. For\n    instance, for pixel-data with a height (first axis), width (second axis),\n    and r/g/b channels (third axis). The functions `concatenate` and `stack`\n    provide more general stacking and concatenation operations.\n\n    Parameters\n    ----------\n    tup : sequence of _Symbol\n        The arrays must have the same shape along all but the first axis.\n        1-D arrays must have the same length.\n\n    Returns\n    -------\n    stacked : _Symbol\n        The array formed by stacking the given arrays, will be at least 2-D.\n    \"\"\"\n    def get_list(arrays):\n        if not hasattr(arrays, '__getitem__') and hasattr(arrays, '__iter__'):\n            raise ValueError(\"expected iterable for arrays but got {}\".format(type(arrays)))\n        return [arr for arr in arrays]\n    arrays = get_list(arrays)\n    return _npi.vstack(*arrays)\n\n\n@set_module('mxnet.symbol.numpy')\ndef row_stack(arrays):\n    r\"\"\"Stack arrays in sequence vertically (row wise).\n    This is equivalent to concatenation along the first axis after 1-D arrays\n    of shape `(N,)` have been reshaped to `(1,N)`. Rebuilds arrays divided by\n    `vsplit`.\n    This function makes most sense for arrays with up to 3 dimensions. For\n    instance, for pixel-data with a height (first axis), width (second axis),\n    and r/g/b channels (third axis). The functions `concatenate` and `stack`\n    provide more general stacking and concatenation operations.\n    Parameters\n    ----------\n    tup : sequence of _Symbol\n        The arrays must have the same shape along all but the first axis.\n        1-D arrays must have the same length.\n    Returns\n    -------\n    stacked : _Symbol\n        The array formed by stacking the given arrays, will be at least 2-D.\n    \"\"\"\n    def get_list(arrays):\n        if not hasattr(arrays, '__getitem__') and hasattr(arrays, '__iter__'):\n            raise ValueError(\"expected iterable for arrays but got {}\".format(type(arrays)))\n        return [arr for arr in arrays]\n\n    arrays = get_list(arrays)\n    return _npi.vstack(*arrays)\n\n\n@set_module('mxnet.symbol.numpy')\ndef column_stack(tup):\n    \"\"\"\n    Stack 1-D arrays as columns into a 2-D array.\n\n    Take a sequence of 1-D arrays and stack them as columns\n    to make a single 2-D array. 2-D arrays are stacked as-is,\n    just like with `hstack`.  1-D arrays are turned into 2-D columns\n    first.\n\n    Parameters\n    ----------\n    tup : sequence of 1-D or 2-D arrays.\n        Arrays to stack. All of them must have the same first dimension.\n\n    Returns\n    -------\n    stacked : 2-D array\n        The array formed by stacking the given arrays.\n\n    See Also\n    --------\n    stack, hstack, vstack, concatenate\n\n    Examples\n    --------\n    >>> a = np.array((1,2,3))\n    >>> b = np.array((2,3,4))\n    >>> np.column_stack((a,b))\n    array([[1., 2.],\n           [2., 3.],\n           [3., 4.]])\n    \"\"\"\n    return _npi.column_stack(*tup)\n\n\n@set_module('mxnet.symbol.numpy')\ndef hstack(arrays):\n    \"\"\"\n    Stack arrays in sequence horizontally (column wise).\n    This is equivalent to concatenation along the second axis,\n    except for 1-D arrays where it concatenates along the first axis.\n    Rebuilds arrays divided by hsplit.\n    This function makes most sense for arrays with up to 3 dimensions.\n    For instance, for pixel-data with a height (first axis), width (second axis),\n    and r/g/b channels (third axis). The functions concatenate,\n    stack and block provide more general stacking and concatenation operations.\n\n    Parameters\n    ----------\n    tup : _Symbol\n        The arrays must have the same shape along all but the second axis, except 1-D arrays which can be any length.\n\n    Returns\n    -------\n    stacked : _Symbol\n        The array formed by stacking the given arrays.\n\n    Examples\n    --------\n    >>> from mxnet import np,npx\n    >>> a = np.array((1,2,3))\n    >>> b = np.array((2,3,4))\n    >>> np.hstack((a,b))\n    array([1., 2., 3., 2., 3., 4.])\n    >>> a = np.array([[1],[2],[3]])\n    >>> b = np.array([[2],[3],[4]])\n    >>> np.hstack((a,b))\n    array([[1., 2.],\n           [2., 3.],\n           [3., 4.]])\n    \"\"\"\n    return _npi.hstack(*arrays)\n\n\n@set_module('mxnet.symbol.numpy')\ndef dstack(arrays):\n    \"\"\"\n    Stack arrays in sequence depth wise (along third axis).\n\n    This is equivalent to concatenation along the third axis after 2-D arrays\n    of shape `(M,N)` have been reshaped to `(M,N,1)` and 1-D arrays of shape\n    `(N,)` have been reshaped to `(1,N,1)`. Rebuilds arrays divided by\n    `dsplit`.\n\n    This function makes most sense for arrays with up to 3 dimensions. For\n    instance, for pixel-data with a height (first axis), width (second axis),\n    and r/g/b channels (third axis). The functions `concatenate`, `stack` and\n    `block` provide more general stacking and concatenation operations.\n\n    Parameters\n    ----------\n    tup : sequence of _Symbol\n        The arrays must have the same shape along all but the first axis.\n        1-D arrays must have the same length.\n\n    Returns\n    -------\n    stacked : _Symbol\n        The array formed by stacking the given arrays, will be at least 2-D.\n    \"\"\"\n    return _npi.dstack(*arrays)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef maximum(x1, x2, out=None, **kwargs):\n    return _ufunc_helper(x1, x2, _npi.maximum, _np.maximum, _npi.maximum_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef fmax(x1, x2, out=None, **kwargs):\n    return _ufunc_helper(x1, x2, _npi.fmax, _np.fmax, _npi.fmax_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef minimum(x1, x2, out=None, **kwargs):\n    return _ufunc_helper(x1, x2, _npi.minimum, _np.minimum, _npi.minimum_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef fmin(x1, x2, out=None, **kwargs):\n    return _ufunc_helper(x1, x2, _npi.fmin, _np.fmin, _npi.fmin_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef max(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Return the maximum of an array or maximum along an axis.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input data.\n    axis : int, optional\n        Axis along which to operate.  By default, flattened input is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result.  Must\n        be of the same shape and buffer length as the expected output.\n        See `doc.ufuncs` (Section \"Output arguments\") for more details.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the original `arr`.\n\n    Returns\n    -------\n    max : _Symbol\n        Maximum of `a`. If `axis` is None, the result is an array of dimension 1.\n        If `axis` is given, the result is an array of dimension\n        ``a.ndim - 1``.\n\n    See Also\n    --------\n    min :\n        The minimum value of an array along a given axis, ignoring any nan.\n    maximum :\n        Element-wise maximum of two arrays, ignoring any nan.\n    argmax :\n        Return the indices of the maximum values.\n\n    Notes\n    -----\n    NaN in the orginal `numpy` is denoted as nan and will be ignored.\n\n    Don't use `max` for element-wise comparison of 2 arrays; when\n    ``a.shape[0]`` is 2, ``maximum(a[0], a[1])`` is faster than\n    ``max(a, axis=0)``.\n    \"\"\"\n    return _npi.max(a, axis=axis, keepdims=keepdims, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef min(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Return the minimum of an array or minimum along an axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data.\n    axis : int, optional\n        Axis along which to operate.  By default, flattened input is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result.  Must\n        be of the same shape and buffer length as the expected output.\n        See `doc.ufuncs` (Section \"Output arguments\") for more details.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the original `arr`.\n\n    Returns\n    -------\n    min : ndarray\n        Minimum of `a`. If `axis` is None, the result is an array of dimension 1.\n        If `axis` is given, the result is an array of dimension\n        ``a.ndim - 1``.\n\n    See Also\n    --------\n    max :\n        The maximum value of an array along a given axis, ignoring any nan.\n    minimum :\n        Element-wise minimum of two arrays, ignoring any nan.\n\n    Notes\n    -----\n    NaN in the orginal `numpy` is denoted as nan and will be ignored.\n\n    Don't use `min` for element-wise comparison of 2 arrays; when\n    ``a.shape[0]`` is 2, ``minimum(a[0], a[1])`` is faster than\n    ``min(a, axis=0)``.\n    \"\"\"\n    return _npi.min(a, axis=axis, keepdims=keepdims, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef amax(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Return the maximum of an array or maximum along an axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data.\n    axis : int, optional\n        Axis along which to operate.  By default, flattened input is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result.  Must\n        be of the same shape and buffer length as the expected output.\n        See `doc.ufuncs` (Section \"Output arguments\") for more details.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the original `arr`.\n\n    Returns\n    -------\n    max : ndarray\n        Maximum of `a`. If `axis` is None, the result is an array of dimension 1.\n        If `axis` is given, the result is an array of dimension\n        ``a.ndim - 1``.\n\n    See Also\n    --------\n    min :\n        The minimum value of an array along a given axis, ignoring any nan.\n    maximum :\n        Element-wise maximum of two arrays, ignoring any nan.\n    argmax :\n        Return the indices of the maximum values.\n\n    Notes\n    -----\n    NaN in the orginal `numpy` is denoted as nan and will be ignored.\n\n    Don't use `max` for element-wise comparison of 2 arrays; when\n    ``a.shape[0]`` is 2, ``maximum(a[0], a[1])`` is faster than\n    ``max(a, axis=0)``.\n    \"\"\"\n    return _npi.amax(a, axis=axis, keepdims=keepdims, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef amin(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Return the minimum of an array or minimum along an axis.\n\n    Parameters\n    ----------\n    a : ndarray\n        Input data.\n    axis : int, optional\n        Axis along which to operate.  By default, flattened input is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result.  Must\n        be of the same shape and buffer length as the expected output.\n        See `doc.ufuncs` (Section \"Output arguments\") for more details.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the original `arr`.\n\n    Returns\n    -------\n    min : ndarray\n        Minimum of `a`. If `axis` is None, the result is an array of dimension 1.\n        If `axis` is given, the result is an array of dimension\n        ``a.ndim - 1``.\n\n    See Also\n    --------\n    max :\n        The maximum value of an array along a given axis, ignoring any nan.\n    minimum :\n        Element-wise minimum of two arrays, ignoring any nan.\n\n    Notes\n    -----\n    NaN in the orginal `numpy` is denoted as nan and will be ignored.\n\n    Don't use `min` for element-wise comparison of 2 arrays; when\n    ``a.shape[0]`` is 2, ``minimum(a[0], a[1])`` is faster than\n    ``min(a, axis=0)``.\n    \"\"\"\n    return _npi.amin(a, axis=axis, keepdims=keepdims, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef all(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Test whether all array elements along a given axis evaluate to True.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input array or object that can be converted to an array.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which a logical AND reduction is performed.\n        The default (axis = None) is to perform a logical AND over\n        all the dimensions of the input array.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in\n        the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the input array.\n    out : ndarray, optional\n        Alternate output array in which to place the result. It must have\n        the same shape as the expected output and its type is preserved\n\n    Returns\n    --------\n    all : _Symbol, bool\n        A new boolean or array is returned unless out is specified,\n        in which case a reference to out is returned.\n    \"\"\"\n    return _npi.all(a, axis=axis, keepdims=keepdims, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef any(a, axis=None, out=None, keepdims=False):\n    \"\"\"\n    Test whether any array element along a given axis evaluates to True.\n    Returns single boolean unless axis is not None\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input array or object that can be converted to an array.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which a logical AND reduction is performed.\n        The default (axis = None) is to perform a logical AND over\n        all the dimensions of the input array.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in\n        the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the input array.\n    out : ndarray, optional\n        Alternate output array in which to place the result. It must have\n        the same shape as the expected output and its type is preserved\n\n    Returns\n    --------\n    any : bool or _Symbol\n        A new boolean or ndarray is returned unless out is specified,\n        in which case a reference to out is returned.\n    \"\"\"\n    return _npi.any(a, axis=axis, keepdims=keepdims, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef clip(a, a_min, a_max, out=None):\n    \"\"\"clip(a, a_min, a_max, out=None)\n\n    Clip (limit) the values in an array.\n    Given an interval, values outside the interval are clipped to\n    the interval edges.  For example, if an interval of ``[0, 1]``\n    is specified, values smaller than 0 become 0, and values larger\n    than 1 become 1.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Array containing elements to clip.\n    a_min : scalar or `None`\n        Minimum value. If `None`, clipping is not performed on lower\n        interval edge. Not more than one of `a_min` and `a_max` may be\n        `None`.\n    a_max : scalar or `None`\n        Maximum value. If `None`, clipping is not performed on upper\n        interval edge. Not more than one of `a_min` and `a_max` may be\n        `None`.\n    out : _Symbol or `None`\n        The results will be placed in this array. It may be the input\n        array for in-place clipping.  `out` must be of the right shape\n        to hold the output.  Its type is preserved.\n\n    Returns\n    -------\n    clipped_array : _Symbol\n        An array with the elements of `a`, but where values\n        < `a_min` are replaced with `a_min`, and those > `a_max`\n        with `a_max`.\n\n    Notes\n    -----\n    array_like `a_min` and `a_max` are not supported.\n    \"\"\"\n    if a_min is None and a_max is None:\n        raise ValueError('array_clip: must set either max or min')\n    if a_min is None:\n        a_min = float('-inf')\n    if a_max is None:\n        a_max = float('inf')\n    return _npi.clip(a, a_min, a_max, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef swapaxes(a, axis1, axis2):\n    \"\"\"Interchange two axes of an array.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input array.\n    axis1 : int\n        First axis.\n    axis2 : int\n        Second axis.\n\n    Returns\n    -------\n    a_swapped : _Symbol\n        Swapped array symbol.\n    \"\"\"\n    return _npi.swapaxes(a, dim1=axis1, dim2=axis2)\n\n\n@set_module('mxnet.symbol.numpy')\ndef argmax(a, axis=None, out=None):\n    r\"\"\"\n    Returns the indices of the maximum values along an axis.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input array. Only support dtype `float16`, `float32`, and `float64`.\n    axis : int, optional\n        By default, the index is into the flattened array, otherwise\n        along the specified axis.\n    out : _Symbol or None, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    index_array : _Symbol of indices whose dtype is same as the input ndarray.\n        Array of indices into the array. It has the same shape as `a.shape`\n        with the dimension along `axis` removed.\n\n    Notes\n    -----\n    In case of multiple occurrences of the maximum values, the indices\n    corresponding to the first occurrence are returned.\n\n    This function differs from the original `numpy.argmax\n    <https://numpy.org/doc/stable/reference/generated/numpy.argmax.html>`_ in\n    the following aspects:\n\n    - Input type does not support Python native iterables(list, tuple, ...).\n    - ``out`` param: cannot perform auto broadcasting. ``out`` symbol's shape must be the same as the expected output.\n    - ``out`` param: cannot perform auto type cast. ``out`` symnbol's dtype must be the same as the expected output.\n    - ``out`` param does not support scalar input case.\n\n    \"\"\"\n    return _npi.argmax(a, axis=axis, keepdims=False, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef argmin(a, axis=None, out=None):\n    r\"\"\"\n    Returns the indices of the minimum values along an axis.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input array. Only support dtype `float16`, `float32`, and `float64`.\n    axis : int, optional\n        By default, the index is into the flattened array, otherwise\n        along the specified axis.\n    out : _Symbol or None, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    index_array : _Symbol of indices whose dtype is same as the input ndarray.\n        Array of indices into the array. It has the same shape as `a.shape`\n        with the dimension along `axis` removed.\n\n    Notes\n    -----\n    In case of multiple occurrences of the minimum values, the indices\n    corresponding to the first occurrence are returned.\n\n    This function differs from the original `numpy.argmin\n    <https://numpy.org/doc/stable/reference/generated/numpy.argmin.html>`_ in\n    the following aspects:\n\n    - Input type does not support Python native iterables(list, tuple, ...).\n    - ``out`` param: cannot perform auto broadcasting. ``out`` symbol's shape must be the same as the expected output.\n    - ``out`` param: cannot perform auto type cast. ``out`` symnbol's dtype must be the same as the expected output.\n    - ``out`` param does not support scalar input case.\n\n    \"\"\"\n    return _npi.argmin(a, axis=axis, keepdims=False, out=out)\n\n\ndef average(a, axis=None, weights=None, returned=False, out=None):\n    \"\"\"\n    Compute the weighted average along the specified axis.\n\n    Parameters\n    --------\n    a : _Symbol\n        Array containing data to be averaged.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which to average a.\n        The default, axis=None, will average over\n        all of the elements of the input array.\n        If axis is negative it counts from the last to the first axis.\n        New in version 1.7.0.\n        If axis is a tuple of ints, averaging is\n        performed on all of the axes specified in the tuple\n        instead of a single axis or all the axes as before.\n    weights : _Symbol, optional\n        An array of weights associated with the values in a, must be the same dtype with a.\n        Each value in a contributes to the average according to its associated weight.\n        The weights array can either be 1-D (in which case its length must be\n        the size of a along the given axis) or of the same shape as a.\n        If weights=None, then all data in a are assumed to have a weight equal to one.\n        The 1-D calculation is: avg = sum(a * weights) / sum(weights)\n        The only constraint on weights is that sum(weights) must not be 0.\n    returned : bool, optional\n        Default is False.\n        If True, the tuple (average, sum_of_weights) is returned,\n        otherwise only the average is returned.\n        If weights=None, sum_of_weights is equivalent to\n        the number of elements over which the average is taken.\n    out : _Symbol, optional\n        If provided, the calculation is done into this array.\n\n    Returns\n    --------\n    retval, [sum_of_weights] : _Symbol\n        Return the average along the specified axis.\n        When returned is True, return a tuple with the average as the first element\n        and the sum of the weights as the second element. sum_of_weights is of the same type as retval.\n        If a is integral, the result dtype will beyour current default dtype,\n        When npx.is_np_default_dtype() returns False, default dtype is float32,\n        When npx.is_np_default_dtype() returns True, default dtype is float64;\n        otherwise it will be the same as dtype of a.\n\n    Raises\n    --------\n        MXNetError\n        - When all weights along axis sum to zero.\n        - When the length of 1D weights is not the same as the shape of a along axis.\n        - When given 1D weights, the axis is not specified or is not int.\n        - When the shape of weights and a differ, but weights are not 1D.\n\n    See also\n    --------\n        mean\n\n    Notes\n    --------\n    This function differs from the original `numpy.average`\n    <https://numpy.org/devdocs/reference/generated/numpy.average.html>`_ in\n    the following way(s):\n\n    - Does not guarantee the same behavior with numpy when given float16 dtype and overflow happens\n    - Does not support complex dtype\n    - The dtypes of a and weights must be the same\n    - Integral a results in float32 or float64 returned dtype, which depends on your current default dtype\n\n\n    Examples\n    --------\n    >>> data = np.arange(1, 5)\n    >>> data\n    array([1., 2., 3., 4.])\n    >>> np.average(data)\n    array(2.5)\n    >>> np.average(np.arange(1, 11), weights=np.arange(10, 0, -1))\n    array(4.)\n    >>> data = np.arange(6).reshape((3,2))\n    >>> data\n    array([[0., 1.],\n           [2., 3.],\n           [4., 5.]])\n    >>> weights = np.array([0.25, 0.75])\n    array([0.25, 0.75])\n    >>> np.average(data, axis=1, weights=weights)\n    array([0.75, 2.75, 4.75])\n    \"\"\"\n    if weights is None:\n        return _npi.average(a, axis=axis, weights=None, returned=returned, weighted=False, out=out)\n    else:\n        return _npi.average(a, axis=axis, weights=weights, returned=returned, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef mean(a, axis=None, dtype=None, out=None, keepdims=False):  # pylint: disable=arguments-differ\n    \"\"\"\n    mean(a, axis=None, dtype=None, out=None, keepdims=None)\n\n    Compute the arithmetic mean along the specified axis.\n    Returns the average of the array elements.\n    The average is taken over the flattened array by default, otherwise over the specified axis.\n\n    Parameters\n    ----------\n    a : `_Symbol`\n        _Symbol containing numbers whose mean is desired.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which the means are computed. The default is to compute the mean of the flattened array.\n        If this is a tuple of ints, a mean is performed over multiple axes,\n        instead of a single axis or all the axes as before.\n    dtype : data-type, optional\n        Type to use in computing the mean.\n        For integer inputs, When npx.is_np_default_dtype() returns False, default dtype is float32,\n        When npx.is_np_default_dtype() returns True, default dtype is float64;\n        for floating point inputs, it is the same as the input dtype.\n    out : _Symbol, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in the result\n        as dimensions with size one. With this option, the result will broadcast correctly\n        against the input array.\n        If the default value is passed, then keepdims will not be passed through to the mean\n        method of sub-classes of _Symbol, however any non-default value will be. If the sub-class\n        method does not implement keepdims any exceptions will be raised.\n\n    Returns\n    -------\n    m : _Symbol, see dtype parameter above\n        If out=None, returns a new array containing the mean values,\n        otherwise a reference to the output array is returned.\n\n    Notes\n    -----\n    This function differs from the original `numpy.mean\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.mean.html>`_ in\n    the following way(s):\n\n    - only _Symbol is accepted as valid input, python iterables or scalar is not supported\n    - default data type for integer input is float32 or float64, which depends on your current default dtype\n\n    Examples\n    --------\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> np.mean(a)\n    array(2.5)\n    >>> a = np.zeros((2, 512*512), dtype=np.float32)\n    >>> a[0,:] = 1.0\n    >>> a[1,:] = 0.1\n    >>> np.mean(a)\n    array(0.55)\n    >>> np.mean(a, dtype=np.float64)\n    array(0.55)\n    \"\"\"\n    return _npi.mean(a, axis=axis, dtype=dtype, keepdims=keepdims, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):  # pylint: disable=too-many-arguments\n    \"\"\"\n    Compute the standard deviation along the specified axis.\n\n    Returns the standard deviation, a measure of the spread of a distribution,\n    of the array elements. The standard deviation is computed for the\n    flattened array by default, otherwise over the specified axis.\n\n    Parameters\n    ----------\n    a : `_Symbol`\n        _Symbol containing numbers whose standard deviation is desired.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which the standard deviations are computed.\n        The default is to compute the standard deviation of the flattened array.\n        If this is a tuple of ints, computation is performed over multiple axes,\n        instead of a single axis or all the axes as before.\n    dtype : data-type, optional\n        Type to use in computing the standard deviation. For integer inputs, the default is float32;\n        for floating point inputs, it is the same as the input dtype.\n    out : _Symbol, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in the result\n        as dimensions with size one. With this option, the result will broadcast correctly\n        against the input array.\n        If the default value is passed, then keepdims will not be passed through to the mean\n        method of sub-classes of _Symbol, however any non-default value will be. If the sub-class\n        method does not implement keepdims any exceptions will be raised.\n\n    Returns\n    -------\n    m : _Symbol, see dtype parameter above\n        If out=None, returns a new array containing the standard deviation values,\n        otherwise a reference to the output array is returned.\n\n    Notes\n    -----\n    This function differs from the original `numpy.std\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.mean.html>`_ in\n    the following way(s):\n\n    - only _Symbol is accepted as valid input, python iterables or scalar is not supported\n    - default output data type for integer input is float32\n\n    \"\"\"\n    return _npi.std(a, axis=axis, dtype=dtype, ddof=ddof, keepdims=keepdims, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):  # pylint: disable=too-many-arguments\n    \"\"\"\n    Compute the variance along the specified axis.\n\n    Returns the variance of the array elements, a measure of the spread of a\n    distribution.  The variance is computed for the flattened array by\n    default, otherwise over the specified axis.\n\n    Parameters\n    ----------\n    a : `_Symbol`\n        _Symbol containing numbers whose variance is desired.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which the variance is computed.\n        The default is to compute the variance of the flattened array.\n        If this is a tuple of ints, computation is performed over multiple axes,\n        instead of a single axis or all the axes as before.\n    dtype : data-type, optional\n        Type to use in computing the variance.\n        For arrays of integer type,\n        When npx.is_np_default_dtype() returns False, default dtype is float32,\n        When npx.is_np_default_dtype() returns True, default dtype is float64;\n        For arrays of float types it is the same as the array type.\n    out : _Symbol, optional\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in the result\n        as dimensions with size one. With this option, the result will broadcast correctly\n        against the input array.\n        If the default value is passed, then keepdims will not be passed through to the mean\n        method of sub-classes of _Symbol, however any non-default value will be. If the sub-class\n        method does not implement keepdims any exceptions will be raised.\n\n    Returns\n    -------\n    m : _Symbol, see dtype parameter above\n        If out=None, returns a new array containing the variance values,\n        otherwise a reference to the output array is returned.\n\n    Notes\n    -----\n    This function differs from the original `numpy.var\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.mean.html>`_ in\n    the following way(s):\n\n    - only _Symbol is accepted as valid input, python iterables or scalar is not supported\n    - default output data type for integer input is float32\n\n    \"\"\"\n    return _npi.var(a, axis=axis, dtype=dtype, ddof=ddof, keepdims=keepdims, out=out)\n\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.symbol.numpy')\ndef indices(dimensions, dtype=None, ctx=None):\n    \"\"\"Return an array representing the indices of a grid.\n\n    Compute an array where the subarrays contain index values 0,1,...\n    varying only along the corresponding axis.\n\n    Parameters\n    ----------\n    dimensions : sequence of ints\n        The shape of the grid.\n    dtype : data-type, optional\n        The desired data-type for the array. Default is `int64`.\n    ctx : device context, optional\n        Device context on which the memory is allocated. Default is\n        `mxnet.context.current_context()`.\n\n    Returns\n    -------\n    grid : _Symbol\n        The array of grid indices,\n        ``grid.shape = (len(dimensions),) + tuple(dimensions)``.\n\n    Notes\n    -----\n    The output shape is obtained by prepending the number of dimensions\n    in front of the tuple of dimensions, i.e. if `dimensions` is a tuple\n    ``(r0, ..., rN-1)`` of length ``N``, the output shape is\n    ``(N,r0,...,rN-1)``.\n\n    The subarrays ``grid[k]`` contains the N-D array of indices along the\n    ``k-th`` axis. Explicitly::\n\n        grid[k,i0,i1,...,iN-1] = ik\n\n    Examples\n    --------\n    >>> grid = np.indices((2, 3))\n    >>> grid.shape\n    (2, 2, 3)\n    >>> grid[0]        # row indices\n    array([[0, 0, 0],\n           [1, 1, 1]], dtype=int64)\n    >>> grid[1]        # column indices\n    array([[0, 0, 0],\n           [1, 1, 1]], dtype=int64)\n\n    The indices can be used as an index into an array.\n\n    >>> x = np.arange(20).reshape(5, 4)\n    >>> row, col = np.indices((2, 3))\n    >>> x[row, col]\n    array([[0., 1., 2.],\n           [4., 5., 6.]])\n\n    Note that it would be more straightforward in the above example to\n    extract the required elements directly with ``x[:2, :3]``.\n    \"\"\"\n    if isinstance(dimensions, (tuple, list)):\n        if ctx is None:\n            ctx = current_context()\n        return _npi.indices(dimensions=dimensions, dtype=dtype, ctx=ctx)\n    else:\n        raise ValueError(\"The dimensions must be sequence of ints\")\n# pylint: enable=redefined-outer-name\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef copysign(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Change the sign of x1 to that of x2, element-wise.\n\n    If `x2` is a scalar, its sign will be copied to all elements of `x1`.\n\n    Parameters\n    ----------\n    x1 : _Symbol or scalar\n        Values to change the sign of.\n    x2 : _Symbol or scalar\n        The sign of `x2` is copied to `x1`.\n    out : _Symbol or None\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    out : _Symbol\n        The values of `x1` with the sign of `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n\n    Notes\n    -------\n    This function differs from the original `numpy.copysign\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.copysign.html>`_ in\n    the following aspects:\n\n    - ``where`` param is not supported.\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.copysign, _np.copysign, _npi.copysign_scalar, _npi.rcopysign_scalar, out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef ravel(x, order='C'):\n    r\"\"\"\n    ravel(x)\n\n    Return a contiguous flattened array.\n    A 1-D array, containing the elements of the input, is returned.  A copy is\n    made only if needed.\n\n    Parameters\n    ----------\n    x : _Symbol\n        Input array.  The elements in `x` are read in row-major, C-style order and\n        packed as a 1-D array.\n    order : `C`, optional\n        Only support row-major, C-style order.\n\n    Returns\n    -------\n    y : _Symbol\n        y is an array of the same subtype as `x`, with shape ``(x.size,)``.\n        Note that matrices are special cased for backward compatibility, if `x`\n        is a matrix, then y is a 1-D ndarray.\n\n    Notes\n    -----\n    This function differs from the original numpy.arange in the following aspects:\n        - Only support row-major, C-style order.\n    \"\"\"\n    if order == 'F':\n        raise NotImplementedError('order {} is not supported'.format(order))\n    if isinstance(x, numeric_types):\n        return _np.reshape(x, -1)\n    elif isinstance(x, _Symbol):\n        return reshape(x, -1)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(x))))\n\n\ndef unravel_index(indices, shape, order='C'): # pylint: disable=redefined-outer-name\n    \"\"\"\n    Converts a flat index or array of flat indices into a tuple of coordinate arrays.\n\n    Parameters:\n    -------------\n    indices : _Symbol\n            An integer array whose elements are indices into the flattened version of an array of dimensions shape.\n            Before version 1.6.0, this function accepted just one index value.\n    shape : tuple of ints\n            The shape of the array to use for unraveling indices.\n\n    Returns:\n    -------------\n    unraveled_coords : _Symbol\n            Each row in the ndarray has the same shape as the indices array.\n            Each column in the ndarray represents the unravelled index\n\n    Examples:\n    -------------\n    >>> np.unravel_index([22, 41, 37], (7,6))\n    ([3. 6. 6.]\n      [4. 5. 1.])\n    >>> np.unravel_index(1621, (6,7,8,9))\n    (3, 1, 4, 1)\n    \"\"\"\n    if order == 'C':\n        return _npi.unravel_index_fallback(indices, shape=shape)\n    else:\n        raise NotImplementedError('Don not support column-major (Fortran-style) order at this moment')\n\n\ndef flatnonzero(a):\n    r\"\"\"\n    Return indices that are non-zero in the flattened version of a.\n\n    This is equivalent to np.nonzero(np.ravel(a))[0].\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input data.\n\n    Returns\n    -------\n    res : _Symbol\n        Output array, containing the indices of the elements of `a.ravel()`\n        that are non-zero.\n\n    See Also\n    --------\n    nonzero : Return the indices of the non-zero elements of the input array.\n    ravel : Return a 1-D array containing the elements of the input array.\n    \"\"\"\n    out = _npi.nonzero(ravel(a))\n    return out.reshape(-1,)\n\n\ndef diag_indices_from(arr):\n    \"\"\"\n    This returns a tuple of indices that can be used to access the main diagonal of an array\n    a with a.ndim >= 2 dimensions and shape (n, n, ..., n). For a.ndim = 2 this is\n    the usual diagonal, for a.ndim > 2 this is the set of indices to access\n    a[i, i, ..., i] for i = [0..n-1].\n\n    Parameters:\n    -------------\n    arr : _Symbol\n        Input array for acessing the main diagonal. All dimensions\n        should have equal length.\n\n    Return:\n    -------------\n    diag: _Symbol\n        indices of the main diagonal.\n\n    Examples:\n    -------------\n    >>> a = np.arange(16).reshape(4, 4)\n    >>> a\n    array([[ 0,  1,  2,  3],\n        [ 4,  5,  6,  7],\n        [ 8,  9, 10, 11],\n        [12, 13, 14, 15]])\n    >>> idx = np.diag_indices_from(a)\n    >>> idx\n    (array([0, 1, 2, 3]), array([0, 1, 2, 3]))\n    >>> a[idx] = 100\n    >>> a\n    array([[100,   1,   2,   3],\n        [  4, 100,   6,   7],\n        [  8,   9, 100,  11],\n        [ 12,  13,  14, 100]])\n    \"\"\"\n    return _npi.diag_indices_from(arr)\n\n\n@set_module('mxnet.symbol.numpy')\ndef hanning(M, dtype=None, ctx=None):\n    r\"\"\"Return the Hanning window.\n\n    The Hanning window is a taper formed by using a weighted cosine.\n\n    Parameters\n    ----------\n    M : int\n        Number of points in the output window. If zero or less, an\n        empty array is returned.\n    ctx : Context, optional\n        An optional device context (default is the current default context).\n\n    Returns\n    -------\n    out : _Symbol, shape(M,)\n        The window, with the maximum value normalized to one (the value\n        one appears only if `M` is odd).\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Note that you need select numpy.float32 or float64 in this operator.\n\n    See Also\n    --------\n    blackman, hamming\n\n    Notes\n    -----\n    The Hanning window is defined as\n\n    .. math::  w(n) = 0.5 - 0.5cos\\left(\\frac{2\\pi{n}}{M-1}\\right)\n               \\qquad 0 \\leq n \\leq M-1\n\n    The Hanning was named for Julius von Hann, an Austrian meteorologist.\n    It is also known as the Cosine Bell. Some authors prefer that it be\n    called a Hann window, to help avoid confusion with the very similar\n    Hamming window.\n\n    Most references to the Hanning window come from the signal processing\n    literature, where it is used as one of many windowing functions for\n    smoothing values.  It is also known as an apodization (which means\n    \"removing the foot\", i.e. smoothing discontinuities at the beginning\n    and end of the sampled signal) or tapering function.\n\n    References\n    ----------\n    .. [1] Blackman, R.B. and Tukey, J.W., (1958) The measurement of power\n           spectra, Dover Publications, New York.\n    .. [2] E.R. Kanasewich, \"Time Sequence Analysis in Geophysics\",\n           The University of Alberta Press, 1975, pp. 106-108.\n    .. [3] Wikipedia, \"Window function\",\n           http://en.wikipedia.org/wiki/Window_function\n    .. [4] W.H. Press,  B.P. Flannery, S.A. Teukolsky, and W.T. Vetterling,\n           \"Numerical Recipes\", Cambridge University Press, 1986, page 425.\n\n    Examples\n    --------\n    >>> np.hanning(12)\n    array([0.        , 0.07937324, 0.29229254, 0.5711574 , 0.8274304 ,\n           0.9797465 , 0.97974646, 0.82743025, 0.5711573 , 0.29229245,\n           0.07937312, 0.        ])\n\n    Plot the window and its frequency response:\n\n    >>> import matplotlib.pyplot as plt\n    >>> window = np.hanning(51)\n    >>> plt.plot(window.asnumpy())\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.title(\"Hann window\")\n    Text(0.5, 1.0, 'Hann window')\n    >>> plt.ylabel(\"Amplitude\")\n    Text(0, 0.5, 'Amplitude')\n    >>> plt.xlabel(\"Sample\")\n    Text(0.5, 0, 'Sample')\n    >>> plt.show()\n    \"\"\"\n    if ctx is None:\n        ctx = current_context()\n    return _npi.hanning(M, dtype=dtype, ctx=ctx)\n\n\n@set_module('mxnet.symbol.numpy')\ndef hamming(M, dtype=None, ctx=None):\n    r\"\"\"Return the hamming window.\n\n    The hamming window is a taper formed by using a weighted cosine.\n\n    Parameters\n    ----------\n    M : int\n        Number of points in the output window. If zero or less, an\n        empty array is returned.\n    ctx : Context, optional\n        An optional device context (default is the current default context).\n\n    Returns\n    -------\n    out : _Symbol, shape(M,)\n        The window, with the maximum value normalized to one (the value\n        one appears only if `M` is odd).\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Note that you need select numpy.float32 or float64 in this operator.\n\n    See Also\n    --------\n    blackman, hanning\n\n    Notes\n    -----\n    The Hamming window is defined as\n\n    .. math::  w(n) = 0.54 - 0.46cos\\left(\\frac{2\\pi{n}}{M-1}\\right)\n               \\qquad 0 \\leq n \\leq M-1\n\n    The Hamming was named for R. W. Hamming, an associate of J. W. Tukey\n    and is described in Blackman and Tukey. It was recommended for\n    smoothing the truncated autocovariance function in the time domain.\n    Most references to the Hamming window come from the signal processing\n    literature, where it is used as one of many windowing functions for\n    smoothing values.  It is also known as an apodization (which means\n    \"removing the foot\", i.e. smoothing discontinuities at the beginning\n    and end of the sampled signal) or tapering function.\n\n    References\n    ----------\n    .. [1] Blackman, R.B. and Tukey, J.W., (1958) The measurement of power\n           spectra, Dover Publications, New York.\n    .. [2] E.R. Kanasewich, \"Time Sequence Analysis in Geophysics\", The\n           University of Alberta Press, 1975, pp. 109-110.\n    .. [3] Wikipedia, \"Window function\",\n           https://en.wikipedia.org/wiki/Window_function\n    .. [4] W.H. Press,  B.P. Flannery, S.A. Teukolsky, and W.T. Vetterling,\n           \"Numerical Recipes\", Cambridge University Press, 1986, page 425.\n\n    Examples\n    --------\n    >>> np.hamming(12)\n    array([0.08000001, 0.15302339, 0.34890914, 0.6054648 , 0.841236  ,\n           0.9813669 , 0.9813668 , 0.8412359 , 0.6054647 , 0.34890908,\n           0.15302327, 0.08000001])\n\n    Plot the window and its frequency response:\n\n    >>> import matplotlib.pyplot as plt\n    >>> window = np.hamming(51)\n    >>> plt.plot(window.asnumpy())\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.title(\"hamming window\")\n    Text(0.5, 1.0, 'hamming window')\n    >>> plt.ylabel(\"Amplitude\")\n    Text(0, 0.5, 'Amplitude')\n    >>> plt.xlabel(\"Sample\")\n    Text(0.5, 0, 'Sample')\n    >>> plt.show()\n    \"\"\"\n    if ctx is None:\n        ctx = current_context()\n    return _npi.hamming(M, dtype=dtype, ctx=ctx)\n\n\n@set_module('mxnet.symbol.numpy')\ndef blackman(M, dtype=None, ctx=None):\n    r\"\"\"Return the Blackman window.\n\n    The Blackman window is a taper formed by using the first three\n    terms of a summation of cosines. It was designed to have close to the\n    minimal leakage possible.  It is close to optimal, only slightly worse\n    than a Kaiser window.\n\n    Parameters\n    ----------\n    M : int\n        Number of points in the output window. If zero or less, an\n        empty array is returned.\n    ctx : Context, optional\n        An optional device context (default is the current default context).\n\n    Returns\n    -------\n    out : _Symbol\n        The window, with the maximum value normalized to one (the value one\n        appears only if the number of samples is odd).\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Note that you need select numpy.float32 or float64 in this operator.\n\n    See Also\n    --------\n    hamming, hanning\n\n    Notes\n    -----\n    The Blackman window is defined as\n\n    .. math::  w(n) = 0.42 - 0.5 \\cos(2\\pi n/{M-1}) + 0.08 \\cos(4\\pi n/{M-1})\n\n    Most references to the Blackman window come from the signal processing\n    literature, where it is used as one of many windowing functions for\n    smoothing values.  It is also known as an apodization (which means\n    \"removing the foot\", i.e. smoothing discontinuities at the beginning\n    and end of the sampled signal) or tapering function. It is known as a\n    \"near optimal\" tapering function, almost as good (by some measures)\n    as the kaiser window.\n\n    References\n    ----------\n    Blackman, R.B. and Tukey, J.W., (1958) The measurement of power spectra,\n    Dover Publications, New York.\n\n    Oppenheim, A.V., and R.W. Schafer. Discrete-Time Signal Processing.\n    Upper Saddle River, NJ: Prentice-Hall, 1999, pp. 468-471.\n\n    Examples\n    --------\n    >>> np.blackman(12)\n    array([-1.4901161e-08,  3.2606423e-02,  1.5990365e-01,  4.1439798e-01,\n            7.3604530e-01,  9.6704686e-01,  9.6704674e-01,  7.3604506e-01,\n            4.1439781e-01,  1.5990359e-01,  3.2606363e-02, -1.4901161e-08])\n\n    Plot the window and its frequency response:\n\n    >>> import matplotlib.pyplot as plt\n    >>> window = np.blackman(51)\n    >>> plt.plot(window.asnumpy())\n    [<matplotlib.lines.Line2D object at 0x...>]\n    >>> plt.title(\"blackman window\")\n    Text(0.5, 1.0, 'blackman window')\n    >>> plt.ylabel(\"Amplitude\")\n    Text(0, 0.5, 'Amplitude')\n    >>> plt.xlabel(\"Sample\")\n    Text(0.5, 0, 'Sample')\n    >>> plt.show()\n    \"\"\"\n    if ctx is None:\n        ctx = current_context()\n    return _npi.blackman(M, dtype=dtype, ctx=ctx)\n\n\n@set_module('mxnet.symbol.numpy')\ndef flip(m, axis=None, out=None):\n    r\"\"\"\n    flip(m, axis=None, out=None)\n\n    Reverse the order of elements in an array along the given axis.\n\n    The shape of the array is preserved, but the elements are reordered.\n\n    Parameters\n    ----------\n    m : _Symbol or scalar\n        Input array.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which to flip over. The default,\n        axis=None, will flip over all of the axes of the input array.\n        If axis is negative it counts from the last to the first axis.\n\n        If axis is a tuple of ints, flipping is performed on all of the axes\n        specified in the tuple.\n    out : _Symbol or scalar, optional\n        Alternative output array in which to place the result. It must have\n        the same shape and type as the expected output.\n\n    Returns\n    -------\n    out : _Symbol or scalar\n        A view of `m` with the entries of axis reversed.  Since a view is\n        returned, this operation is done in constant time.\n    \"\"\"\n    if isinstance(m, numeric_types):\n        return _np.flip(m, axis)\n    elif isinstance(m, _Symbol):\n        return _npi.flip(m, axis, out=out)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(m))))\n\n\n@set_module('mxnet.symbol.numpy')\ndef flipud(m):\n    r\"\"\"\n    flipud(*args, **kwargs)\n\n    Flip array in the up/down direction.\n\n    Flip the entries in each column in the up/down direction.\n    Rows are preserved, but appear in a different order than before.\n\n    Parameters\n    ----------\n    m : array_like\n        Input array.\n\n    Returns\n    -------\n    out : array_like\n        A view of `m` with the rows reversed.  Since a view is\n        returned, this operation is :math:`\\mathcal O(1)`.\n    \"\"\"\n    return flip(m, 0)\n\n\n@set_module('mxnet.symbol.numpy')\ndef fliplr(m):\n    r\"\"\"\n    fliplr(*args, **kwargs)\n\n    Flip array in the left/right direction.\n\n    Flip the entries in each row in the left/right direction.\n    Columns are preserved, but appear in a different order than before.\n\n    Parameters\n    ----------\n    m : array_like\n        Input array, must be at least 2-D.\n\n    Returns\n    -------\n    f : ndarray\n        A view of `m` with the columns reversed.  Since a view\n        is returned, this operation is :math:`\\mathcal O(1)`.\n    \"\"\"\n    return flip(m, 1)\n\n\n@set_module('mxnet.symbol.numpy')\ndef around(x, decimals=0, out=None, **kwargs):\n    r\"\"\"\n    around(x, decimals=0, out=None)\n\n    Evenly round to the given number of decimals.\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input data.\n    decimals : int, optional\n        Number of decimal places to round to (default: 0).  If\n        decimals is negative, it specifies the number of positions to\n        the left of the decimal point.\n    out : _Symbol, optional\n        Alternative output array in which to place the result. It must have\n        the same shape and type as the expected output.\n\n    Returns\n    -------\n    rounded_array : _Symbol or scalar\n        An array of the same type as `x`, containing the rounded values.\n        A reference to the result is returned.\n\n    Notes\n    -----\n    For values exactly halfway between rounded decimal values, NumPy\n    rounds to the nearest even value. Thus 1.5 and 2.5 round to 2.0,\n    -0.5 and 0.5 round to 0.0, etc.\n\n    This function differs from the original numpy.prod in the following aspects:\n\n        - Cannot cast type automatically. Dtype of `out` must be same as the expected one.\n        - Cannot support complex-valued number.\n    \"\"\"\n    if isinstance(x, numeric_types):\n        return _np.around(x, decimals, **kwargs)\n    elif isinstance(x, _Symbol):\n        return _npi.around(x, decimals, out=out, **kwargs)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(x))))\n\n\n@set_module('mxnet.symbol.numpy')\ndef round(x, decimals=0, out=None, **kwargs):\n    r\"\"\"\n    round(a, decimals=0, out=None)\n    Round an array to the given number of decimals.\n\n    See Also\n    --------\n    around : equivalent function; see for details.\n    \"\"\"\n    if isinstance(x, numeric_types):\n        return _np.around(x, decimals, **kwargs)\n    elif isinstance(x, _Symbol):\n        return _npi.around(x, decimals, out=out, **kwargs)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(x))))\n\n\n@set_module('mxnet.symbol.numpy')\ndef round_(x, decimals=0, out=None, **kwargs):\n    r\"\"\"\n    round_(a, decimals=0, out=None)\n    Round an array to the given number of decimals.\n\n    See Also\n    --------\n    around : equivalent function; see for details.\n    \"\"\"\n    if isinstance(x, numeric_types):\n        return _np.around(x, decimals, **kwargs)\n    elif isinstance(x, _Symbol):\n        return _npi.around(x, decimals, out=out, **kwargs)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(x))))\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef arctan2(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Element-wise arc tangent of ``x1/x2`` choosing the quadrant correctly.\n\n    The quadrant (i.e., branch) is chosen so that ``arctan2(x1, x2)`` is\n    the signed angle in radians between the ray ending at the origin and\n    passing through the point (1,0), and the ray ending at the origin and\n    passing through the point (`x2`, `x1`).  (Note the role reversal: the\n    \"`y`-coordinate\" is the first function parameter, the \"`x`-coordinate\"\n    is the second.)  By IEEE convention, this function is defined for\n    `x2` = +/-0 and for either or both of `x1` and `x2` = +/-inf (see\n    Notes for specific values).\n\n    This function is not defined for complex-valued arguments; for the\n    so-called argument of complex values, use `angle`.\n\n    Parameters\n    ----------\n    x1 : _Symbol or scalar\n        `y`-coordinates.\n    x2 : _Symbol or scalar\n        `x`-coordinates. `x2` must be broadcastable to match the shape of\n        `x1` or vice versa.\n    out : _Symbol or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : _Symbol or scalar\n        Array of angles in radians, in the range ``[-pi, pi]``. This is a scalar if\n        `x1` and `x2` are scalars.\n\n    Notes\n    -----\n    *arctan2* is identical to the `atan2` function of the underlying\n    C library.  The following special values are defined in the C\n    standard: [1]_\n\n    ====== ====== ================\n    `x1`   `x2`   `arctan2(x1,x2)`\n    ====== ====== ================\n    +/- 0  +0     +/- 0\n    +/- 0  -0     +/- pi\n        > 0   +/-inf +0 / +pi\n        < 0   +/-inf -0 / -pi\n    +/-inf +inf   +/- (pi/4)\n    +/-inf -inf   +/- (3*pi/4)\n    ====== ====== ================\n\n    Note that +0 and -0 are distinct floating point numbers, as are +inf\n    and -inf.\n\n    This function differs from the original numpy.arange in the following aspects:\n        - Only support float16, float32 and float64.\n\n    References\n    ----------\n    .. [1] ISO/IEC standard 9899:1999, \"Programming language C.\"\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.arctan2, _np.arctan2,\n                         _npi.arctan2_scalar, _npi.rarctan2_scalar, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef hypot(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Given the \"legs\" of a right triangle, return its hypotenuse.\n\n    Equivalent to ``sqrt(x1**2 + x2**2)``, element-wise.  If `x1` or\n    `x2` is scalar_like (i.e., unambiguously cast-able to a scalar type),\n    it is broadcast for use with each element of the other argument.\n\n    Parameters\n    ----------\n    x1, x2 : _Symbol or scalar\n        Leg of the triangle(s).\n    out : _Symbol or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    z : _Symbol or scalar\n        The hypotenuse of the triangle(s).\n        This is a scalar if both `x1` and `x2` are scalars.\n\n    Notes\n    -----\n    This function differs from the original numpy.arange in the following aspects:\n        - Only support float16, float32 and float64.\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.hypot, _np.hypot, _npi.hypot_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef bitwise_and(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Compute the bit-wise XOR of two arrays element-wise.\n\n    Parameters\n    ----------\n    x1, x2 : _Symbol or scalar\n        Only integer and boolean types are handled. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which becomes the shape of the output).\n    out : _Symbol or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : _Symbol or scalar\n        Result.\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.bitwise_and, _np.bitwise_and, _npi.bitwise_and_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef bitwise_xor(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Compute the bit-wise XOR of two arrays element-wise.\n\n    Parameters\n    ----------\n    x1, x2 : _Symbol or scalar\n        Only integer and boolean types are handled. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which becomes the shape of the output).\n    out : _Symbol or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : _Symbol or scalar\n        Result.\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.bitwise_xor, _np.bitwise_xor, _npi.bitwise_xor_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef bitwise_or(x1, x2, out=None, **kwargs):\n    r\"\"\"\n    Compute the bit-wise OR of two arrays element-wise.\n\n    Parameters\n    ----------\n    x1, x2 : _Symbol or scalar\n        Only integer and boolean types are handled. If x1.shape != x2.shape,\n        they must be broadcastable to a common shape (which becomes the shape of the output).\n    out : _Symbol or None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n\n    Returns\n    -------\n    out : _Symbol or scalar\n        Result.\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.bitwise_or, _np.bitwise_or, _npi.bitwise_or_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef unique(ar, return_index=False, return_inverse=False, return_counts=False, axis=None):\n    \"\"\"\n    Find the unique elements of an array.\n\n    Returns the sorted unique elements of an array. There are three optional\n    outputs in addition to the unique elements:\n\n    * the indices of the input array that give the unique values\n    * the indices of the unique array that reconstruct the input array\n    * the number of times each unique value comes up in the input array\n\n    Parameters\n    ----------\n    ar : _Symbol\n        Input array. Unless `axis` is specified, this will be flattened if it\n        is not already 1-D.\n    return_index : bool, optional\n        If True, also return the indices of `ar` (along the specified axis,\n        if provided, or in the flattened array) that result in the unique array.\n    return_inverse : bool, optional\n        If True, also return the indices of the unique array (for the specified\n        axis, if provided) that can be used to reconstruct `ar`.\n    return_counts : bool, optional\n        If True, also return the number of times each unique item appears\n        in `ar`.\n    axis : int or None, optional\n        The axis to operate on. If None, `ar` will be flattened. If an integer,\n        the subarrays indexed by the given axis will be flattened and treated\n        as the elements of a 1-D array with the dimension of the given axis,\n        see the notes for more details. The default is None.\n\n    Returns\n    -------\n    unique : _Symbol\n        The sorted unique values.\n    unique_indices : _Symbol, optional\n        The indices of the first occurrences of the unique values in the\n        original array. Only provided if `return_index` is True.\n    unique_inverse : _Symbol, optional\n        The indices to reconstruct the original array from the\n        unique array. Only provided if `return_inverse` is True.\n    unique_counts : _Symbol, optional\n        The number of times each of the unique values comes up in the\n        original array. Only provided if `return_counts` is True.\n\n    Notes\n    -----\n    When an axis is specified the subarrays indexed by the axis are sorted.\n    This is done by making the specified axis the first dimension of the array\n    and then flattening the subarrays in C order. The flattened subarrays are\n    then viewed as a structured type with each element given a label, with the\n    effect that we end up with a 1-D array of structured types that can be\n    treated in the same way as any other 1-D array. The result is that the\n    flattened subarrays are sorted in lexicographic order starting with the\n    first element.\n    \"\"\"\n    return _npi.unique(ar, return_index, return_inverse, return_counts, axis)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef ldexp(x1, x2, out=None, **kwargs):\n    \"\"\"\n    Returns x1 * 2**x2, element-wise.\n    The mantissas `x1` and twos exponents `x2` are used to construct\n    floating point numbers ``x1 * 2**x2``.\n\n    Parameters\n    ----------\n    x1 : _Symbol\n        Array of multipliers.\n    x2 : _Symbol\n        Array of twos exponents.\n    out : _Symbol or None\n        Dummy parameter to keep the consistency with the ndarray counterpart.\n\n    Returns\n    -------\n    y : _Symbol\n        The result of ``x1 * 2**x2``.\n\n    Notes\n    -----\n    Complex dtypes are not supported, they will raise a TypeError.\n    Different from numpy, we allow x2 to be float besides int.\n    `ldexp` is useful as the inverse of `frexp`, if used by itself it is\n    more clear to simply use the expression ``x1 * 2**x2``.\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.ldexp, _np.ldexp, _npi.ldexp_scalar, _npi.rldexp_scalar, out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef vdot(a, b):\n    r\"\"\"\n    Return the dot product of two vectors.\n    Note that `vdot` handles multidimensional arrays differently than `dot`:\n    it does *not* perform a matrix product, but flattens input arguments\n    to 1-D vectors first. Consequently, it should only be used for vectors.\n\n    Parameters\n    ----------\n    a : _Symbol\n        First argument to the dot product.\n    b : _Symbol\n        Second argument to the dot product.\n\n    Returns\n    -------\n    output : _Symbol\n        Dot product of `a` and `b`.\n\n    See Also\n    --------\n    dot : Return the dot product without using the complex conjugate of the\n        first argument.\n\n    Examples\n    --------\n    Note that higher-dimensional arrays are flattened!\n    >>> a = np.array([[1, 4], [5, 6]])\n    >>> b = np.array([[4, 1], [2, 2]])\n    >>> np.vdot(a, b)\n    30\n    >>> np.vdot(b, a)\n    30\n    >>> 1*4 + 4*1 + 5*2 + 6*2\n    30\n    \"\"\"\n    return tensordot(a.flatten(), b.flatten(), 1)\n\n\n@set_module('mxnet.symbol.numpy')\ndef inner(a, b):\n    r\"\"\"Inner product of two arrays.\n    Ordinary inner product of vectors for 1-D arrays (without complex\n    conjugation), in higher dimensions a sum product over the last axes.\n\n    Parameters\n    ----------\n    a, b : _Symbol\n        If `a` and `b` are nonscalar, their last dimensions must match.\n\n    Returns\n    -------\n    out : _Symbol\n        `out.shape = a.shape[:-1] + b.shape[:-1]`\n\n    Raises\n    ------\n    ValueError\n        If the last dimension of `a` and `b` has different size.\n\n    See Also\n    --------\n    tensordot : Sum products over arbitrary axes.\n    dot : Generalised matrix product, using second last dimension of `b`.\n    einsum : Einstein summation convention.\n\n    Notes\n    -----\n    For vectors (1-D arrays) it computes the ordinary inner-product::\n        np.inner(a, b) = sum(a[:]*b[:])\n    More generally, if `ndim(a) = r > 0` and `ndim(b) = s > 0`::\n        np.inner(a, b) = np.tensordot(a, b, axes=(-1,-1))\n    or explicitly::\n        np.inner(a, b)[i0,...,ir-1,j0,...,js-1]\n            = sum(a[i0,...,ir-1,:]*b[j0,...,js-1,:])\n    In addition `a` or `b` may be scalars, in which case::\n    np.inner(a,b) = a*b\n\n    Examples\n    --------\n    Ordinary inner product for vectors:\n    >>> a = np.array([1,2,3])\n    >>> b = np.array([0,1,0])\n    >>> np.inner(a, b)\n    2\n    A multidimensional example:\n    >>> a = np.arange(24).reshape((2,3,4))\n    >>> b = np.arange(4)\n    >>> np.inner(a, b)\n    array([[ 14,  38,  62],\n           [ 86, 110, 134]])\n    \"\"\"\n    return tensordot(a, b, [-1, -1])\n\n\n@set_module('mxnet.symbol.numpy')\ndef outer(a, b):\n    r\"\"\"Compute the outer product of two vectors.\n    Given two vectors, ``a = [a0, a1, ..., aM]`` and\n    ``b = [b0, b1, ..., bN]``,\n    the outer product [1]_ is::\n    [[a0*b0  a0*b1 ... a0*bN ]\n    [a1*b0    .\n    [ ...          .\n    [aM*b0            aM*bN ]]\n\n    Parameters\n    ----------\n    a : (M,) _Symbol\n        First input vector.  Input is flattened if\n        not already 1-dimensional.\n    b : (N,) _Symbol\n        Second input vector.  Input is flattened if\n        not already 1-dimensional.\n\n    Returns\n    -------\n    out : (M, N) _Symbol\n        ``out[i, j] = a[i] * b[j]``\n\n    See also\n    --------\n    inner\n    einsum : ``einsum('i,j->ij', a.ravel(), b.ravel())`` is the equivalent.\n    ufunc.outer : A generalization to N dimensions and other operations.\n                ``np.multiply.outer(a.ravel(), b.ravel())`` is the equivalent.\n\n    References\n    ----------\n    .. [1] : G. H. Golub and C. F. Van Loan, *Matrix Computations*, 3rd\n            ed., Baltimore, MD, Johns Hopkins University Press, 1996,\n            pg. 8.\n\n    Examples\n    --------\n    Make a (*very* coarse) grid for computing a Mandelbrot set:\n    >>> rl = np.outer(np.ones((5,)), np.linspace(-2, 2, 5))\n    >>> rl\n    array([[-2., -1.,  0.,  1.,  2.],\n        [-2., -1.,  0.,  1.,  2.],\n        [-2., -1.,  0.,  1.,  2.],\n        [-2., -1.,  0.,  1.,  2.],\n        [-2., -1.,  0.,  1.,  2.]])\n    \"\"\"\n    return tensordot(a.flatten(), b.flatten(), 0)\n\n\n@set_module('mxnet.symbol.numpy')\ndef cross(a, b, axisa=-1, axisb=-1, axisc=-1, axis=None): # pylint: disable=too-many-arguments\n    \"\"\"\n    Return the cross product of two (arrays of) vectors.\n\n    The cross product of `a` and `b` in :math:`R^3` is a vector perpendicular\n    to both `a` and `b`.  If `a` and `b` are arrays of vectors, the vectors\n    are defined by the last axis of `a` and `b` by default, and these axes\n    can have dimensions 2 or 3.  Where the dimension of either `a` or `b` is\n    2, the third component of the input vector is assumed to be zero and the\n    cross product calculated accordingly.  In cases where both input vectors\n    have dimension 2, the z-component of the cross product is returned.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Components of the first vector(s).\n    b : _Symbol\n        Components of the second vector(s).\n    axisa : int, optional\n        Axis of `a` that defines the vector(s).  By default, the last axis.\n    axisb : int, optional\n        Axis of `b` that defines the vector(s).  By default, the last axis.\n    axisc : int, optional\n        Axis of `c` containing the cross product vector(s).  Ignored if\n        both input vectors have dimension 2, as the return is scalar.\n        By default, the last axis.\n    axis : int, optional\n        If defined, the axis of `a`, `b` and `c` that defines the vector(s)\n        and cross product(s).  Overrides `axisa`, `axisb` and `axisc`.\n\n    Returns\n    -------\n    c : _Symbol\n        Vector cross product(s).\n\n    Raises\n    ------\n    ValueError\n        When the dimension of the vector(s) in `a` and/or `b` does not\n        equal 2 or 3.\n\n    Notes\n    -----\n    Supports full broadcasting of the inputs.\n    \"\"\"\n    if axis is not None:\n        axisa, axisb, axisc = (axis,) * 3\n\n    return _npi.cross(a, b, axisa, axisb, axisc)\n\n\n@set_module('mxnet.symbol.numpy')\ndef kron(a, b):\n    r\"\"\"\n    kron(a, b)\n    Kronecker product of two arrays.\n    Computes the Kronecker product, a composite array made of blocks of the\n    second array scaled by the first.\n    Parameters\n    ----------\n    a, b : ndarray\n    Returns\n    -------\n    out : ndarray\n    See Also\n    --------\n    outer : The outer product\n    Notes\n    -----\n    The function assumes that the number of dimensions of `a` and `b`\n    are the same, if necessary prepending the smallest with ones.\n    If `a.shape = (r0,r1,..,rN)` and `b.shape = (s0,s1,...,sN)`,\n    the Kronecker product has shape `(r0*s0, r1*s1, ..., rN*SN)`.\n    The elements are products of elements from `a` and `b`, organized\n    explicitly by::\n        kron(a,b)[k0,k1,...,kN] = a[i0,i1,...,iN] * b[j0,j1,...,jN]\n    where::\n        kt = it * st + jt,  t = 0,...,N\n    In the common 2-D case (N=1), the block structure can be visualized::\n        [[ a[0,0]*b,   a[0,1]*b,  ... , a[0,-1]*b  ],\n        [  ...                              ...   ],\n        [ a[-1,0]*b,  a[-1,1]*b, ... , a[-1,-1]*b ]]\n    Examples\n    --------\n    >>> np.kron([1,10,100], [5,6,7])\n    array([  5,   6,   7,  50,  60,  70, 500, 600, 700])\n    >>> np.kron([5,6,7], [1,10,100])\n    array([  5,  50, 500,   6,  60, 600,   7,  70, 700])\n    \"\"\"\n    return _npi.kron(a, b)\n\n\n@set_module('mxnet.symbol.numpy')\ndef equal(x1, x2, out=None):\n    \"\"\"\n    Return (x1 == x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : _Symbol or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : Dummy parameter, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : _Symbol or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    not_equal, greater_equal, less_equal, greater, less\n    Examples\n    --------\n    >>> np.equal(np.ones(2, 1)), np.zeros(1, 3))\n    array([[False, False, False],\n           [False, False, False]])\n    >>> np.equal(1, np.ones(1))\n    array([ True])\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.equal, _np.equal, _npi.equal_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef not_equal(x1, x2, out=None):\n    \"\"\"\n    Return (x1 != x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : _Symbol or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : Dummy parameter, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : _Symbol or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    equal, greater, greater_equal, less, less_equal\n    Examples\n    --------\n    >>> np.not_equal(np.ones(2, 1)), np.zeros(1, 3))\n    array([[ True,  True,  True],\n           [ True,  True,  True]])\n    >>> np.not_equal(1, np.ones(1))\n    array([False])\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.not_equal, _np.not_equal, _npi.not_equal_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef greater(x1, x2, out=None):\n    \"\"\"\n    Return the truth value of (x1 > x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : _Symbol or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : Dummy parameter, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : _Symbol or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    equal, greater, greater_equal, less, less_equal\n    Examples\n    --------\n    >>> np.greater(np.ones(2, 1)), np.zeros(1, 3))\n    array([[ True,  True,  True],\n           [ True,  True,  True]])\n    >>> np.greater(1, np.ones(1))\n    array([False])\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.greater, _np.greater, _npi.greater_scalar,\n                         _npi.less_scalar, out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef less(x1, x2, out=None):\n    \"\"\"\n    Return the truth value of (x1 < x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : _Symbol or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : Dummy parameter, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : _Symbol or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    equal, greater, greater_equal, less, less_equal\n    Examples\n    --------\n    >>> np.less(np.ones(2, 1)), np.zeros(1, 3))\n    array([[ True,  True,  True],\n           [ True,  True,  True]])\n    >>> np.less(1, np.ones(1))\n    array([False])\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.less, _np.less, _npi.less_scalar, _npi.greater_scalar, out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef greater_equal(x1, x2, out=None):\n    \"\"\"\n    Return the truth value of (x1 >= x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : _Symbol or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : Dummy parameter, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : _Symbol or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    equal, greater, greater_equal, less, less_equal\n    Examples\n    --------\n    >>> np.greater_equal(np.ones(2, 1)), np.zeros(1, 3))\n    array([[ True,  True,  True],\n           [ True,  True,  True]])\n    >>> np.greater_equal(1, np.ones(1))\n    array([True])\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.greater_equal, _np.greater_equal, _npi.greater_equal_scalar,\n                         _npi.less_equal_scalar, out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef less_equal(x1, x2, out=None):\n    \"\"\"\n    Return the truth value of (x1 <= x2) element-wise.\n    Parameters\n    ----------\n    x1, x2 : _Symbol or scalars\n        Input arrays. If ``x1.shape != x2.shape``, they must be broadcastable to\n        a common shape (which becomes the shape of the output).\n    out : Dummy parameter, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned.\n    Returns\n    -------\n    out : _Symbol or scalar\n        Output array of type bool, element-wise comparison of `x1` and `x2`.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    equal, greater, greater_equal, less, less_equal\n    Examples\n    --------\n    >>> np.less_equal(np.ones(2, 1)), np.zeros(1, 3))\n    array([[False, False, False],\n           [False, False, False]])\n    >>> np.less_equal(1, np.ones(1))\n    array([True])\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.less_equal, _np.less_equal, _npi.less_equal_scalar,\n                         _npi.greater_equal_scalar, out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef roll(a, shift, axis=None):\n    \"\"\"\n    Roll array elements along a given axis.\n\n    Elements that roll beyond the last position are re-introduced at\n    the first.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input array.\n    shift : int or tuple of ints\n        The number of places by which elements are shifted.  If a tuple,\n        then `axis` must be a tuple of the same size, and each of the\n        given axes is shifted by the corresponding number.  If an int\n        while `axis` is a tuple of ints, then the same value is used for\n        all given axes.\n    axis : int or tuple of ints, optional\n        Axis or axes along which elements are shifted.  By default, the\n        array is flattened before shifting, after which the original\n        shape is restored.\n\n    Returns\n    -------\n    res : _Symbol\n        Output array, with the same shape as `a`.\n\n    Notes\n    -----\n    Supports rolling over multiple dimensions simultaneously.\n    \"\"\"\n    return _npi.roll(a, shift, axis=axis)\n\n\n@wrap_np_binary_func\ndef logical_and(x1, x2, out=None):\n    r\"\"\"\n    Compute the truth value of x1 AND x2 element-wise.\n    Parameters\n    ----------\n    x1, x2 : array_like\n        Logical AND is applied to the elements of `x1` and `x2`.\n        If ``x1.shape != x2.shape``, they must be broadcastable to a common\n        shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n    Returns\n    -------\n    y : ndarray or bool\n        Boolean result of the logical AND operation applied to the elements\n        of `x1` and `x2`; the shape is determined by broadcasting.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    logical_or, logical_not, logical_xor, bitwise_or\n    Examples\n    --------\n    >>> np.logical_and(True, False)\n    False\n    >>> np.logical_and(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))\n    array([False,  True])\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.logical_and, _np.logical_and, _npi.logical_and_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef logical_or(x1, x2, out=None):\n    r\"\"\"\n    Compute the truth value of x1 OR x2 element-wise.\n    Parameters\n    ----------\n    x1, x2 : array_like\n        Logical OR is applied to the elements of `x1` and `x2`.\n        If ``x1.shape != x2.shape``, they must be broadcastable to a common\n        shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n    Returns\n    -------\n    y : ndarray or bool\n        Boolean result of the logical OR operation applied to the elements\n        of `x1` and `x2`; the shape is determined by broadcasting.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    logical_and, logical_not, logical_xor, bitwise_or\n    Examples\n    --------\n    >>> np.logical_or(True, False)\n    True\n    >>> np.logical_or(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))\n    array([True,  True])\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.logical_or, _np.logical_or, _npi.logical_or_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_binary_func\ndef logical_xor(x1, x2, out=None):\n    r\"\"\"\n    Compute the truth value of x1 XOR x2 element-wise.\n    Parameters\n    ----------\n    x1, x2 : array_like\n        Logical XOR is applied to the elements of `x1` and `x2`.\n        If ``x1.shape != x2.shape``, they must be broadcastable to a common\n        shape (which becomes the shape of the output).\n    out : ndarray, None, or tuple of ndarray and None, optional\n        A location into which the result is stored. If provided, it must have\n        a shape that the inputs broadcast to. If not provided or `None`,\n        a freshly-allocated array is returned. A tuple (possible only as a\n        keyword argument) must have length equal to the number of outputs.\n    Returns\n    -------\n    y : ndarray or bool\n        Boolean result of the logical XOR operation applied to the elements\n        of `x1` and `x2`; the shape is determined by broadcasting.\n        This is a scalar if both `x1` and `x2` are scalars.\n    See Also\n    --------\n    logical_and, logical_not, logical_or, bitwise_or\n    Examples\n    --------\n    >>> np.logical_xor(True, False)\n    True\n    >>> np.logical_xor(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))\n    array([ True, False])\n    \"\"\"\n    return _ufunc_helper(x1, x2, _npi.logical_xor, _np.logical_xor, _npi.logical_xor_scalar, None, out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef rot90(m, k=1, axes=(0, 1)):\n    \"\"\"\n    Rotate an array by 90 degrees in the plane specified by axes.\n    Rotation direction is from the first towards the second axis.\n    Parameters\n    ----------\n    m : _Symbol\n        Array of two or more dimensions.\n    k : integer\n        Number of times the array is rotated by 90 degrees.\n    axes: (2,) array_like\n        The array is rotated in the plane defined by the axes.\n        Axes must be different.\n    Returns\n    -------\n    y : _Symbol\n        A rotated view of `m`.\n    -----\n    rot90(m, k=1, axes=(1,0)) is the reverse of rot90(m, k=1, axes=(0,1))\n    rot90(m, k=1, axes=(1,0)) is equivalent to rot90(m, k=-1, axes=(0,1))\n    Examples\n    --------\n    >>> m = np.array([[1,2],[3,4]], 'int')\n    >>> m\n    array([[1, 2],\n           [3, 4]], dtype=int64)\n    >>> np.rot90(m)\n    array([[2, 4],\n           [1, 3]], dtype=int64)\n    >>> np.rot90(m, 2)\n    array([[4, 3],\n           [2, 1]], dtype=int64)\n    >>> m = np.arange(8).reshape((2,2,2))\n    >>> np.rot90(m, 1, (1,2))\n    array([[[1., 3.],\n            [0., 2.]],\n           [[5., 7.],\n            [4., 6.]]])\n    \"\"\"\n    return _npi.rot90(m, k=k, axes=axes)\n\n\n@set_module('mxnet.symbol.numpy')\ndef einsum(*operands, **kwargs):\n    r\"\"\"\n    einsum(subscripts, *operands, out=None, optimize=False)\n\n    Evaluates the Einstein summation convention on the operands.\n\n    Using the Einstein summation convention, many common multi-dimensional,\n    linear algebraic array operations can be represented in a simple fashion.\n    In *implicit* mode `einsum` computes these values.\n\n    In *explicit* mode, `einsum` provides further flexibility to compute\n    other array operations that might not be considered classical Einstein\n    summation operations, by disabling, or forcing summation over specified\n    subscript labels.\n\n    See the notes and examples for clarification.\n\n    Parameters\n    ----------\n    subscripts : str\n        Specifies the subscripts for summation as comma separated list of\n        subscript labels. An implicit (classical Einstein summation)\n        calculation is performed unless the explicit indicator '->' is\n        included as well as subscript labels of the precise output form.\n    operands : list of _Symbol\n        These are the arrays for the operation.\n    out : _Symbol, optional\n        If provided, the calculation is done into this array.\n    optimize : {False, True}, optional\n        Controls if intermediate optimization should occur. No optimization\n        will occur if False. Defaults to False.\n\n    Returns\n    -------\n    output : _Symbol\n        The calculation based on the Einstein summation convention.\n\n    Notes\n    -----\n    The Einstein summation convention can be used to compute\n    many multi-dimensional, linear algebraic array operations. `einsum`\n    provides a succinct way of representing these.\n\n    A non-exhaustive list of these operations,\n    which can be computed by `einsum`, is shown below along with examples:\n\n    * Trace of an array, :py:func:`np.trace`.\n    * Return a diagonal, :py:func:`np.diag`.\n    * Array axis summations, :py:func:`np.sum`.\n    * Transpositions and permutations, :py:func:`np.transpose`.\n    * Matrix multiplication and dot product, :py:func:`np.matmul` :py:func:`np.dot`.\n    * Vector inner and outer products, :py:func:`np.inner` :py:func:`np.outer`.\n    * Broadcasting, element-wise and scalar multiplication, :py:func:`np.multiply`.\n    * Tensor contractions, :py:func:`np.tensordot`.\n\n    The subscripts string is a comma-separated list of subscript labels,\n    where each label refers to a dimension of the corresponding operand.\n    Whenever a label is repeated it is summed, so ``np.einsum('i,i', a, b)``\n    is equivalent to :py:func:`np.inner(a,b) <np.inner>`. If a label\n    appears only once, it is not summed, so ``np.einsum('i', a)`` produces a\n    view of ``a`` with no changes. A further example ``np.einsum('ij,jk', a, b)``\n    describes traditional matrix multiplication and is equivalent to\n    :py:func:`np.matmul(a,b) <np.matmul>`. Repeated subscript labels in one\n    operand take the diagonal. For example, ``np.einsum('ii', a)`` is equivalent\n    to :py:func:`np.trace(a) <np.trace>`.\n\n    In *implicit mode*, the chosen subscripts are important\n    since the axes of the output are reordered alphabetically.  This\n    means that ``np.einsum('ij', a)`` doesn't affect a 2D array, while\n    ``np.einsum('ji', a)`` takes its transpose. Additionally,\n    ``np.einsum('ij,jk', a, b)`` returns a matrix multiplication, while,\n    ``np.einsum('ij,jh', a, b)`` returns the transpose of the\n    multiplication since subscript 'h' precedes subscript 'i'.\n\n    In *explicit mode* the output can be directly controlled by\n    specifying output subscript labels.  This requires the\n    identifier '->' as well as the list of output subscript labels.\n    This feature increases the flexibility of the function since\n    summing can be disabled or forced when required. The call\n    ``np.einsum('i->', a)`` is like :py:func:`np.sum(a, axis=-1) <np.sum>`,\n    and ``np.einsum('ii->i', a)`` is like :py:func:`np.diag(a) <np.diag>`.\n    The difference is that `einsum` does not allow broadcasting by default.\n    Additionally ``np.einsum('ij,jh->ih', a, b)`` directly specifies the\n    order of the output subscript labels and therefore returns matrix\n    multiplication, unlike the example above in implicit mode.\n\n    To enable and control broadcasting, use an ellipsis.  Default\n    NumPy-style broadcasting is done by adding an ellipsis\n    to the left of each term, like ``np.einsum('...ii->...i', a)``.\n    To take the trace along the first and last axes,\n    you can do ``np.einsum('i...i', a)``, or to do a matrix-matrix\n    product with the left-most indices instead of rightmost, one can do\n    ``np.einsum('ij...,jk...->ik...', a, b)``.\n\n    When there is only one operand, no axes are summed, and no output\n    parameter is provided, a view into the operand is returned instead\n    of a new array.  Thus, taking the diagonal as ``np.einsum('ii->i', a)``\n    produces a view.\n\n    The ``optimize`` argument which will optimize the contraction order\n    of an einsum expression. For a contraction with three or more operands this\n    can greatly increase the computational efficiency at the cost of a larger\n    memory footprint during computation.\n\n    Typically a 'greedy' algorithm is applied which empirical tests have shown\n    returns the optimal path in the majority of cases. 'optimal' is not supported\n    for now.\n\n    This function differs from the original `numpy.einsum\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.einsum.html>`_ in\n    the following way(s):\n\n    - Does not support 'optimal' strategy\n    - Does not support the alternative subscript like\n        `einsum(op0, sublist0, op1, sublist1, ..., [sublistout])`\n    - Does not produce view in any cases\n    \"\"\"\n    # Grab non-einsum kwargs; do not optimize by default.\n    optimize_arg = kwargs.pop('optimize', False)\n    out = kwargs.pop('out', None)\n\n    subscripts = operands[0]\n    operands = operands[1:]\n    return _npi.einsum(*operands, subscripts=subscripts, out=out, optimize=int(optimize_arg))\n\n\n@set_module('mxnet.symbol.numpy')\ndef percentile(a, q, axis=None, out=None, overwrite_input=None, interpolation='linear', keepdims=False): # pylint: disable=too-many-arguments\n    \"\"\"\n    Compute the q-th percentile of the data along the specified axis.\n    Returns the q-th percentile(s) of the array elements.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input array\n    q : _Symbol\n        Percentile or sequence of percentiles to compute.\n    axis : {int, tuple of int, None}, optional\n        Axis or axes along which the percentiles are computed. The default is to\n        compute the percentile(s) along a flattened version of the array.\n    out : ndarray, optional\n        Alternative output array in which to place the result. It must have the same\n        shape and buffer length as the expected output, but the type (of the output)\n        will be cast if necessary.\n    overwrite_input : bool, optional (Not supported yet)\n        If True, then allow the input array a to be modified by intermediate calculations,\n        to save memory. In this case, the contents of the input a after this function\n        completes is undefined.\n    interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}\n        This optional parameter specifies the interpolation method to use when the\n        desired percentile lies between two data points i < j:\n        'linear': i + (j - i) * fraction, where fraction is the fractional part of the\n        index surrounded by i and j.\n        'lower': i.\n        'higher': j.\n        'nearest': i or j, whichever is nearest.\n        'midpoint': (i + j) / 2.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in the result as\n        dimensions with size one. With this option, the result will broadcast\n        correctly against the original array a.\n\n    Returns\n    -------\n    percentile : _Symbol\n        Output array.\n    \"\"\"\n    if overwrite_input is not None:\n        raise NotImplementedError('overwrite_input is not supported yet')\n    if isinstance(q, numeric_types):\n        return _npi.percentile(a, axis=axis, interpolation=interpolation,\n                               keepdims=keepdims, q_scalar=q, out=out)\n    return _npi.percentile(a, q, axis=axis, interpolation=interpolation,\n                           keepdims=keepdims, q_scalar=None, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef median(a, axis=None, out=None, overwrite_input=None, keepdims=False):\n    r\"\"\"\n    Compute the median along the specified axis.\n    Returns the median of the array elements.\n    Parameters\n    ----------\n    a : _Symbol\n        Input array or object that can be converted to an array.\n    axis : {int, sequence of int, None}, optional\n        Axis or axes along which the medians are computed. The default\n        is to compute the median along a flattened version of the array.\n        A sequence of axes is supported since version 1.9.0.\n    out :  _Symbol, optional\n        Alternative output array in which to place the result. It must\n        have the same shape and buffer length as the expected output,\n        but the type (of the output) will be cast if necessary.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the original `arr`.\n    Returns\n    -------\n    median :  _Symbol\n        A new array holding the result. If the input contains integers\n        or floats smaller than ``float32``, then the output data-type is\n        ``np.float32``.  Otherwise, the data-type of the output is the\n        same as that of the input. If `out` is specified, that array is\n        returned instead.\n    See Also\n    --------\n    mean, percentile\n    \"\"\"\n    return quantile(a=a, q=0.5, axis=axis, out=out, overwrite_input=overwrite_input,\n                    interpolation='midpoint', keepdims=keepdims)\n\n\n@set_module('mxnet.symbol.numpy')\ndef quantile(a, q, axis=None, out=None, overwrite_input=None, interpolation='linear', keepdims=False): # pylint: disable=too-many-arguments\n    \"\"\"\n    Compute the q-th quantile of the data along the specified axis.\n    New in version 1.15.0.\n    Parameters\n    ----------\n    a : _Symbol\n        Input array or object that can be converted to an array.\n    q : _Symbol\n        Quantile or sequence of quantiles to compute, which must be between 0 and 1 inclusive.\n    axis : {int, tuple of int, None}, optional\n        Axis or axes along which the quantiles are computed.\n        The default is to compute the quantile(s) along a flattened version of the array.\n    out : ndarray, optional\n        Alternative output array in which to place the result.\n        It must have the same shape and buffer length as the expected output,\n        but the type (of the output) will be cast if necessary.\n    interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}\n        This optional parameter specifies the interpolation method to use\n        when the desired quantile lies between two data points i < j:\n            linear: i + (j - i) * fraction, where fraction is the fractional part of the index surrounded by i and j.\n            lower: i.\n            higher: j.\n            nearest: i or j, whichever is nearest.\n            midpoint: (i + j) / 2.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in the result as dimensions with size one.\n        With this option, the result will broadcast correctly against the original array a.\n    Returns\n    -------\n    quantile : _Symbol\n        If q is a single quantile and axis=None, then the result is a scalar.\n        If multiple quantiles are given, first axis of the result corresponds to the quantiles.\n        The other axes are the axes that remain after the reduction of a.\n        If out is specified, that array is returned instead.\n    See also\n    --------\n    mean\n    Notes\n    -----\n    Given a vector V of length N, the q-th quantile of V is the value q of the way from the minimum\n    to the maximum in a sorted copy of V. The values and distances of the two nearest neighbors\n    as well as the interpolation parameter will determine the quantile if the normalized ranking\n    does not match the location of q exactly. This function is the same as the median if q=0.5,\n    the same as the minimum if q=0.0 and the same as the maximum if q=1.0.\n    This function differs from the original `numpy.quantile\n    <https://numpy.org/devdocs/reference/generated/numpy.quantile.html>`_ in\n    the following aspects:\n    - q must be _Symbol type even if it is a scalar\n    - do not support overwrite_input\n    \"\"\"\n    if overwrite_input is not None:\n        raise NotImplementedError('overwrite_input is not supported yet')\n    if isinstance(q, numeric_types):\n        return _npi.percentile(a, axis=axis, interpolation=interpolation,\n                               keepdims=keepdims, q_scalar=q * 100, out=out)\n    return _npi.percentile(a, q * 100, axis=axis, interpolation=interpolation,\n                           keepdims=keepdims, q_scalar=None, out=out)\n\n\n@set_module('mxnet.symbol.numpy')\ndef shares_memory(a, b, max_work=None):\n    \"\"\"\n    Determine if two arrays share memory\n\n    Parameters\n    ----------\n    a, b : _Symbol\n        Input arrays\n\n    Returns\n    -------\n    out : _Symbol\n    \"\"\"\n    return _npi.share_memory(a, b)\n\n\n@set_module('mxnet.symbol.numpy')\ndef may_share_memory(a, b, max_work=None):\n    \"\"\"\n    Determine if two arrays might share memory\n\n    A return of True does not necessarily mean that the two arrays\n    share any element.  It just means that they *might*.\n\n    Only the memory bounds of a and b are checked by default.\n\n    Parameters\n    ----------\n    a, b : _Symbol\n        Input arrays\n\n    Returns\n    -------\n    out : _Symbol\n    \"\"\"\n    return _npi.share_memory(a, b)\n\n\n@set_module('mxnet.symbol.numpy')\ndef diff(a, n=1, axis=-1, prepend=None, append=None):  # pylint: disable=redefined-outer-name\n    r\"\"\"\n    Calculate the n-th discrete difference along the given axis.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input array\n    n : int, optional\n        The number of times values are differenced. If zero, the input is returned as-is.\n    axis : int, optional\n        The axis along which the difference is taken, default is the last axis.\n    prepend, append : _Symbol, optional\n        Not supported yet\n\n    Returns\n    -------\n    diff : _Symbol\n        The n-th differences.\n        The shape of the output is the same as a except along axis where the dimension is smaller by n.\n        The type of the output is the same as the type of the difference between any two elements of a.\n        This is the same as the type of a in most cases.\n\n    Examples\n    --------\n    >>> x = np.array([1, 2, 4, 7, 0])\n    >>> np.diff(x)\n    array([ 1,  2,  3, -7])\n    >>> np.diff(x, n=2)\n    array([  1,   1, -10])\n\n    >>> x = np.array([[1, 3, 6, 10], [0, 5, 6, 8]])\n    >>> np.diff(x)\n    array([[2, 3, 4],\n        [5, 1, 2]])\n    >>> np.diff(x, axis=0)\n    array([[-1,  2,  0, -2]])\n\n    Notes\n    -----\n    Optional inputs `prepend` and `append` are not supported yet\n    \"\"\"\n    if (prepend or append):\n        raise NotImplementedError('prepend and append options are not supported yet')\n    return _npi.diff(a, n=n, axis=axis)\n\n\n@set_module('mxnet.symbol.numpy')\ndef ediff1d(ary, to_end=None, to_begin=None):\n    \"\"\"\n    The differences between consecutive elements of an array.\n\n    Parameters\n    ----------\n    ary : _Symbol\n        If necessary, will be flattened before the differences are taken.\n    to_end : _Symbol or scalar, optional\n        Number(s) to append at the end of the returned differences.\n    to_begin : _Symbol or scalar, optional\n        Number(s) to prepend at the beginning of the returned differences.\n\n    Returns\n    -------\n    ediff1d : _Symbol\n        The differences. Loosely, this is ``ary.flat[1:] - ary.flat[:-1]``.\n    \"\"\"\n    input_type = (isinstance(to_begin, _Symbol), isinstance(to_end, _Symbol))\n    # case 1: when both `to_begin` and `to_end` are arrays\n    if input_type == (True, True):\n        return _npi.ediff1d(ary, to_begin, to_end, to_begin_arr_given=True, to_end_arr_given=True,\n                            to_begin_scalar=None, to_end_scalar=None)\n    # case 2: only `to_end` is array but `to_begin` is scalar/None\n    elif input_type == (False, True):\n        return _npi.ediff1d(ary, to_end, to_begin_arr_given=False, to_end_arr_given=True,\n                            to_begin_scalar=to_begin, to_end_scalar=None)\n    # case 3: only `to_begin` is array but `to_end` is scalar/None\n    elif input_type == (True, False):\n        return _npi.ediff1d(ary, to_begin, to_begin_arr_given=True, to_end_arr_given=False,\n                            to_begin_scalar=None, to_end_scalar=to_end)\n    # case 4: both `to_begin` and `to_end` are scalar/None\n    else:\n        return _npi.ediff1d(ary, to_begin_arr_given=False, to_end_arr_given=False,\n                            to_begin_scalar=to_begin, to_end_scalar=to_end)\n\n\n@set_module('mxnet.symbol.numpy')\ndef interp(x, xp, fp, left=None, right=None, period=None):  # pylint: disable=too-many-arguments\n    \"\"\"\n    One-dimensional linear interpolation.\n    Returns the one-dimensional piecewise linear interpolant to a function\n    with given values at discrete data-points.\n\n    Parameters\n    ----------\n    x : _Symbol\n        The x-coordinates of the interpolated values.\n    xp : _Symbol\n        The x-coordinates of the data points, must be increasing if argument\n        `period` is not specified. Otherwise, `xp` is internally sorted after\n        normalizing the periodic boundaries with ``xp = xp % period``.\n    fp : _Symbol\n        The y-coordinates of the data points, same length as `xp`.\n    left : optional float corresponding to fp\n        Value to return for `x < xp[0]`, default is `fp[0]`.\n    right : optional float corresponding to fp\n        Value to return for `x > xp[-1]`, default is `fp[-1]`.\n    period : None or float, optional\n        A period for the x-coordinates. This parameter allows the proper\n        interpolation of angular x-coordinates. Parameters `left` and `right`\n        are ignored if `period` is specified.\n        .. versionadded:: 1.10.0\n\n    Returns\n    -------\n    y : _Symbol\n        The interpolated values, same shape as `x`.\n\n    Raises\n    ------\n    ValueError\n        If `xp` and `fp` have different length\n        If `xp` or `fp` are not 1-D sequences\n        If `period == 0`\n\n    Notes\n    -----\n    Does not check that the x-coordinate sequence `xp` is increasing.\n    If `xp` is not increasing, the results are nonsense.\n    A simple check for increasing is::\n        np.all(np.diff(xp) > 0)\n    \"\"\"\n    if isinstance(x, numeric_types):\n        return _npi.interp(xp.astype(float), fp.astype(float), left=left,\n                           right=right, period=period, x_scalar=x, x_is_scalar=True)\n    return _npi.interp(xp.astype(float), fp.astype(float), x.astype(float), left=left,\n                       right=right, period=period, x_scalar=0.0, x_is_scalar=False)\n\n\n@set_module('mxnet.symbol.numpy')\ndef resize(a, new_shape):\n    \"\"\"\n    Return a new array with the specified shape.\n    If the new array is larger than the original array, then the new\n    array is filled with repeated copies of `a`.  Note that this behavior\n    is different from a.resize(new_shape) which fills with zeros instead\n    of repeated copies of `a`.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Array to be resized.\n    new_shape : int or tuple of int\n        Shape of resized array.\n\n    Returns\n    -------\n    reshaped_array : _Symbol\n        The new array is formed from the data in the old array, repeated\n        if necessary to fill out the required number of elements.  The\n        data are repeated in the order that they are stored in memory.\n\n    See Also\n    --------\n    ndarray.resize : resize an array in-place.\n\n    Notes\n    -----\n    Warning: This functionality does **not** consider axes separately,\n    i.e. it does not apply interpolation/extrapolation.\n    It fills the return array with the required number of elements, taken\n    from `a` as they are laid out in memory, disregarding strides and axes.\n    (This is in case the new shape is smaller. For larger, see above.)\n    This functionality is therefore not suitable to resize images,\n    or data where each axis represents a separate and distinct entity.\n\n    Examples\n    --------\n    >>> a = np.array([[0, 1], [2, 3]])\n    >>> np.resize(a, (2, 3))\n    array([[0., 1., 2.],\n           [3., 0., 1.]])\n    >>> np.resize(a, (1, 4))\n    array([[0., 1., 2., 3.]])\n    >>> np.resize(a,(2, 4))\n    array([[0., 1., 2., 3.],\n           [0., 1., 2., 3.]])\n    \"\"\"\n    return _npi.resize_fallback(a, new_shape=new_shape)\n\n# pylint: disable=redefined-outer-name\n@set_module('mxnet.symbol.numpy')\ndef nan_to_num(x, copy=True, nan=0.0, posinf=None, neginf=None, **kwargs):\n    \"\"\"\n    Replace NaN with zero and infinity with large finite numbers (default\n    behaviour) or with the numbers defined by the user using the `nan`,\n    `posinf` and/or `neginf` keywords.\n\n    If `x` is inexact, NaN is replaced by zero or by the user defined value in\n    `nan` keyword, infinity is replaced by the largest finite floating point\n    values representable by ``x.dtype`` or by the user defined value in\n    `posinf` keyword and -infinity is replaced by the most negative finite\n    floating point values representable by ``x.dtype`` or by the user defined\n    value in `neginf` keyword.\n\n    For complex dtypes, the above is applied to each of the real and\n    imaginary components of `x` separately.\n\n    If `x` is not inexact, then no replacements are made.\n\n    Parameters\n    ----------\n    x : _Symbol\n        Input data.\n    copy : bool, optional\n        Whether to create a copy of `x` (True) or to replace values\n        in-place (False). The in-place operation only occurs if\n        casting to an array does not require a copy.\n        Default is True.\n    nan : int, float, optional\n        Value to be used to fill NaN values. If no value is passed\n        then NaN values will be replaced with 0.0.\n    posinf : int, float, optional\n        Value to be used to fill positive infinity values. If no value is\n        passed then positive infinity values will be replaced with a very\n        large number.\n    neginf : int, float, optional\n        Value to be used to fill negative infinity values. If no value is\n        passed then negative infinity values will be replaced with a very\n        small (or negative) number.\n\n        .. versionadded:: 1.13\n\n    Returns\n    -------\n    out : _Symbol\n        `x`, with the non-finite values replaced. If `copy` is False, this may\n        be `x` itself.\n\n    Notes\n    -----\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic\n    (IEEE 754). This means that Not a Number is not equivalent to infinity.\n\n    \"\"\"\n    if isinstance(x, numeric_types):\n        return _np.nan_to_num(x, copy, nan, posinf, neginf)\n    elif isinstance(x, _Symbol):\n        if not copy:\n            return _npi.nan_to_num(x, copy=copy, nan=nan, posinf=posinf, neginf=neginf, out=x)\n        return _npi.nan_to_num(x, copy=copy, nan=nan, posinf=posinf, neginf=neginf, out=None)\n    else:\n        raise TypeError('type {} not supported'.format(str(type(x))))\n\n\n@set_module('mxnet.symbol.numpy')\ndef squeeze(x, axis=None):\n    \"\"\"\n    Remove single-dimensional entries from the shape of an array.\n\n    Parameters\n    ----------\n    a : array_like\n        Input data.\n    axis : None or int or tuple of ints, optional\n        .. versionadded:: 1.7.0\n        Selects a subset of the single-dimensional entries in the\n        shape. If an axis is selected with shape entry greater than\n        one, an error is raised.\n\n    Returns\n    -------\n    squeezed : ndarray\n        The input array, but with all or a subset of the\n        dimensions of length 1 removed. This is always `a` itself\n        or a view into `a`.\n\n    Raises\n    ------\n    ValueError\n        If `axis` is not `None`, and an axis being squeezed is not of length 1\n\n    See Also\n    --------\n    expand_dims : The inverse operation, adding singleton dimensions\n    reshape : Insert, remove, and combine dimensions, and resize existing ones\n\n    Examples\n    --------\n    >>> x = np.array([[[0], [1], [2]]])\n    >>> x.shape\n    (1, 3, 1)\n    >>> np.squeeze(x).shape\n    (3,)\n    >>> np.squeeze(x, axis=0).shape\n    (3, 1)\n    >>> np.squeeze(x, axis=1).shape\n    Traceback (most recent call last):\n    ...\n    ValueError: cannot select an axis to squeeze out which has size not equal to one\n    >>> np.squeeze(x, axis=2).shape\n    (1, 3)\n    \"\"\"\n    return _npi.squeeze(x, axis=axis)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef isnan(x, out=None, **kwargs):\n    \"\"\"\n    Test element-wise for NaN and return result as a boolean array.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array.\n    out : _Symbol or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : _Symbol or bool\n        True where x is NaN, false otherwise.\n        This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic (IEEE 754).\n    This means that Not a Number is not equivalent to infinity.\n\n    This function differs from the original `numpy.isnan\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.isnan.html>`_ in\n    the following aspects:\n    - Does not support complex number for now\n    - Input type does not support Python native iterables(list, tuple, ...).\n    - ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be the same as the expected output.\n    - ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the same as the expected output.\n    - ``out`` param does not support scalar input case.\n    \"\"\"\n    return _unary_func_helper(x, _npi.isnan, _np.isnan, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef isinf(x, out=None, **kwargs):\n    \"\"\"\n    Test element-wise for positive or negative infinity.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array.\n    out : _Symbol or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : _Symbol or bool\n        True where x is positive or negative infinity, false otherwise.\n        This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic (IEEE 754).\n\n    This function differs from the original `numpy.isinf\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.isnan.html>`_ in\n    the following aspects:\n    - Does not support complex number for now\n    - Input type does not support Python native iterables(list, tuple, ...).\n    - ``out`` param: cannot perform auto broadcasting. ``out`` ndarray's shape must be the same as the expected output.\n    - ``out`` param: cannot perform auto type cast. ``out`` ndarray's dtype must be the same as the expected output.\n    - ``out`` param does not support scalar input case.\n    \"\"\"\n    return _unary_func_helper(x, _npi.isinf, _np.isinf, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef isposinf(x, out=None, **kwargs):\n    \"\"\"\n    Test element-wise for positive infinity, return result as bool array.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array.\n    out : _Symbol or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : _Symbol or bool\n        True where x is positive infinity, false otherwise.\n        This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic (IEEE 754).\n    This means that Not a Number is not equivalent to infinity.\n    \"\"\"\n    return _unary_func_helper(x, _npi.isposinf, _np.isposinf, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef isneginf(x, out=None, **kwargs):\n    \"\"\"\n    Test element-wise for negative infinity, return result as bool array.\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array.\n    out : _Symbol or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : _Symbol or bool\n        True where x is negative infinity, false otherwise.\n        This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic (IEEE 754).\n    This means that Not a Number is not equivalent to infinity.\n    \"\"\"\n    return _unary_func_helper(x, _npi.isneginf, _np.isneginf, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\n@wrap_np_unary_func\ndef isfinite(x, out=None, **kwargs):\n    \"\"\"\n    Test element-wise for finiteness (not infinity or not Not a Number).\n\n    Parameters\n    ----------\n    x : _Symbol or scalar\n        Input array.\n    out : _Symbol or None, optional\n        A location into which the result is stored.\n        If provided, it must have the same shape and dtype as input ndarray.\n        If not provided or `None`, a freshly-allocated array is returned.\n\n    Returns\n    -------\n    y : _Symbol or bool\n        True where x is negative infinity, false otherwise.\n        This is a scalar if x is a scalar.\n\n    Notes\n    -----\n    Not a Number, positive infinity and negative infinity are considered to be non-finite.\n\n    NumPy uses the IEEE Standard for Binary Floating-Point for Arithmetic (IEEE 754).\n    This means that Not a Number is not equivalent to infinity.\n    Also that positive infinity is not equivalent to negative infinity.\n    But infinity is equivalent to positive infinity. Errors result if the second argument\n    is also supplied when x is a scalar input, or if first and second arguments have different shapes.\n    \"\"\"\n    return _unary_func_helper(x, _npi.isfinite, _np.isfinite, out=out, **kwargs)\n\n\n@set_module('mxnet.symbol.numpy')\ndef atleast_1d(*arys):\n    \"\"\"\n    Convert inputs to arrays with at least one dimension.\n\n    Scalar inputs are converted to 1-dimensional arrays, whilst higher-dimensional inputs are preserved.\n\n    Parameters\n    ----------\n    arys1, arys2, ... : _Symbol\n        One or more input arrays.\n\n    Returns\n    -------\n    ret : _Symbol\n        An array, or list of arrays, each with a.ndim >= 1. Copies are made only if necessary.\n\n    See also\n    --------\n    atleast_2d, atleast_3d\n    \"\"\"\n    return _npi.atleast_1d(*arys)\n\n\n@set_module('mxnet.symbol.numpy')\ndef atleast_2d(*arys):\n    \"\"\"\n    Convert inputs to arrays with at least two dimensions.\n\n    Parameters\n    ----------\n    arys1, arys2, ... : _Symbol\n        One or more input arrays.\n\n    Returns\n    -------\n    ret : _Symbol\n        An array, or list of arrays, each with a.ndim >= 2. Copies are made only if necessary.\n\n    See also\n    --------\n    atleast_1d, atleast_3d\n    \"\"\"\n    return _npi.atleast_2d(*arys)\n\n\n@set_module('mxnet.symbol.numpy')\ndef atleast_3d(*arys):\n    \"\"\"\n    Convert inputs to arrays with at least three dimension.\n\n    Parameters\n    ----------\n    arys1, arys2, ... : _Symbol\n        One or more input arrays.\n\n    Returns\n    -------\n    ret : _Symbol\n        An array, or list of arrays, each with a.ndim >= 3.\n        For example, a 1-D array of shape (N,) becomes a view of shape (1, N, 1),\n        and a 2-D array of shape (M, N) becomes a view of shape (M, N, 1).\n\n    See also\n    --------\n    atleast_1d, atleast_2d\n    \"\"\"\n    return _npi.atleast_3d(*arys)\n\n\n@set_module('mxnet.symbol.numpy')\ndef where(condition, x, y):\n    \"\"\"\n    Return elements chosen from `x` or `y` depending on `condition`.\n\n    Parameters\n    ----------\n    condition : _Symbol\n        Where True, yield `x`, otherwise yield `y`.\n    x, y : _Symbol\n        Values from which to choose. `x`, `y` and `condition` need to be\n        broadcastable to some shape. `x` and `y` must have the same dtype.\n\n    Returns\n    -------\n    out : _Symbol\n        An array with elements from `x` where `condition` is True, and elements\n        from `y` elsewhere.\n\n    \"\"\"\n    if isinstance(condition, numeric_types):\n        if condition != 0:\n            return x\n        else:\n            return y\n    else:\n        if isinstance(x, numeric_types) and isinstance(y, numeric_types):\n            return _npi.where_scalar2(condition, float(x), float(y), out=None)\n        elif isinstance(x, Symbol) and isinstance(y, Symbol):\n            return _npi.where(condition, x, y, out=None)\n        elif isinstance(y, Symbol):\n            return _npi.where_lscalar(condition, y, float(x), out=None)\n        elif isinstance(x, Symbol):\n            return _npi.where_rscalar(condition, x, float(y), out=None)\n        else:\n            raise TypeError('type {0} and {1} not supported'.format(str(type(x)), str(type(y))))\n\n\n@set_module('mxnet.symbol.numpy')\ndef load(fname):\n    \"\"\"Loads symbol from a JSON file.\n    You can also use pickle to do the job if you only work on python.\n    The advantage of load/save is the file is language agnostic.\n    This means the file saved using save can be loaded by other language binding of mxnet.\n    You also get the benefit being able to directly load/save from cloud storage(S3, HDFS).\n\n    Parameters\n    ----------\n    fname : str\n        The name of the file, examples:\n        - `s3://my-bucket/path/my-s3-symbol`\n        - `hdfs://my-bucket/path/my-hdfs-symbol`\n        - `/path-to/my-local-symbol`\n\n    Returns\n    -------\n    sym : _Symbol\n        The loaded symbol.\n\n    See Also\n    --------\n    _Symbol.save : Used to save symbol into file.\n    \"\"\"\n    if not isinstance(fname, string_types):\n        raise TypeError('fname needs to be string')\n    handle = SymbolHandle()\n    check_call(_LIB.MXSymbolCreateFromFile(c_str(fname), ctypes.byref(handle)))\n    return _Symbol(handle)\n\n\n@set_module('mxnet.symbol.numpy')\ndef load_json(json_str):\n    \"\"\"Loads symbol from json string.\n\n    Parameters\n    ----------\n    json_str : str\n        A JSON string.\n\n    Returns\n    -------\n    sym : Symbol\n        The loaded symbol.\n\n    See Also\n    --------\n    _Symbol.tojson : Used to save symbol into json string.\n    \"\"\"\n    if not isinstance(json_str, string_types):\n        raise TypeError('json_str needs to be string')\n    handle = SymbolHandle()\n    check_call(_LIB.MXSymbolCreateFromJSON(c_str(json_str), ctypes.byref(handle)))\n    return _Symbol(handle)\n\n\n@set_module('mxnet.symbol.numpy')\ndef polyval(p, x):\n    \"\"\"\n    Evaluate a polynomial at specific values.\n    If p is of length N, this function returns the value:\n    p[0]*x**(N-1) + p[1]*x**(N-2) + ... + p[N-2]*x + p[N-1]\n    If x is a sequence, then p(x) is returned for each element of x.\n    If x is another polynomial then the composite polynomial p(x(t)) is returned.\n\n    Parameters\n    ----------\n    p : _Symbol\n        1D array of polynomial coefficients (including coefficients equal to zero)\n        from highest degree to the constant term.\n    x : _Symbol\n        An array of numbers, at which to evaluate p.\n\n    Returns\n    -------\n    values : _Symbol\n        Result array of polynomials\n\n    Notes\n    -----\n    This function differs from the original `numpy.polyval\n    <https://numpy.org/devdocs/reference/generated/numpy.polyval.html>`_ in\n    the following way(s):\n    - Does not support poly1d.\n    - X should be ndarray type even if it contains only one element.\n    \"\"\"\n    if isinstance(p, Symbol) and isinstance(x, Symbol):\n        return _npi.polyval(p, x)\n    elif not isinstance(p, Symbol) and not isinstance(x, Symbol):\n        return _np.polyval(p, x)\n    else:\n        raise TypeError('type not supported')\n\n\n@set_module('mxnet.symbol.numpy')\ndef bincount(x, weights=None, minlength=0):\n    \"\"\"\n    Count number of occurrences of each value in array of non-negative ints.\n\n    Parameters\n    ----------\n    x : _Symbol\n        input data\n    weights: _Symbol\n        input weigths same shape as x. (Optional)\n    minlength: int\n        A minimum number of bins for the output. (Optional)\n\n    Returns\n    --------\n    out : _Symbol\n        the result of binning the input data. The length of out is equal to amax(x)+1.\n\n    Raises:\n    --------\n    Value Error\n        If the input is not 1-dimensional, or contains elements with negative values,\n        or if minlength is negative\n    TypeError\n        If the type of the input is float or complex.\n    \"\"\"\n    if minlength < 0:\n        raise ValueError(\"Minlength value should greater than 0\")\n    if weights is None:\n        return _npi.bincount(x, minlength=minlength, has_weights=False)\n    return _npi.bincount(x, weights=weights, minlength=minlength, has_weights=True)\n\n\n@set_module('mxnet.symbol.numpy')\ndef pad(x, pad_width, mode='constant', **kwargs): # pylint: disable=too-many-arguments\n    \"\"\"\n    Pad an array.\n\n    Parameters\n    ----------\n    array : array_like of rank N\n        The array to pad.\n    pad_width : {sequence, array_like, int}\n        Number of values padded to the edges of each axis.\n        ((before_1, after_1), ... (before_N, after_N)) unique pad widths\n        for each axis.\n        ((before, after),) yields same before and after pad for each axis.\n        (pad,) or int is a shortcut for before = after = pad width for all\n        axes.\n    mode : str or function, optional\n        One of the following string values or a user supplied function.\n        'constant' (default)\n            Pads with a constant value.\n        'edge'\n            Pads with the edge values of array.\n        'linear_ramp'\n            not supported yet\n        'maximum'\n            Pads with the maximum value of all of the\n            vector along each axis.\n        'mean'\n            not supported yet\n        'median'\n            not supported yet\n        'minimum'\n            Pads with the minimum value of all of the\n            vector along each axis.\n        'reflect'\n            Pads with the reflection of the vector mirrored on\n            the first and last values of the vector along each\n            axis.\n        'symmetric'\n            Pads with the reflection of the vector mirrored\n            along the edge of the array.\n        'wrap'\n            not supported yet.\n        'empty'\n            not supported yet.\n        <function>\n            not supported yet.\n    stat_length : not supported yet\n    constant_values : scalar, optional\n        Used in 'constant'.  The values to set the padded values for each\n        axis.\n        Default is 0.\n\n    end_values : not supported yet\n    reflect_type : {'even', 'odd'}, optional\n        only support even now\n\n    Returns\n    -------\n    pad : ndarray\n        Padded array of rank equal to `array` with shape increased\n        according to `pad_width`.\n    \"\"\"\n    # pylint: disable = too-many-return-statements, inconsistent-return-statements\n    if not _np.asarray(pad_width).dtype.kind == 'i':\n        raise TypeError('`pad_width` must be of integral type.')\n    if not isinstance(pad_width, tuple):\n        raise TypeError(\"`pad_width` must be tuple.\")\n    if mode == \"linear_ramp\":\n        raise ValueError(\"mode {'linear_ramp'} is not supported.\")\n    if mode == \"wrap\":\n        raise ValueError(\"mode {'wrap'} is not supported.\")\n    if mode == \"median\":\n        raise ValueError(\"mode {'median'} is not supported.\")\n    if mode == \"mean\":\n        raise ValueError(\"mode {'mean'} is not supported.\")\n    if mode == \"empty\":\n        raise ValueError(\"mode {'empty'} is not supported.\")\n    if callable(mode):\n        raise ValueError(\"mode {'<function>'} is not supported.\")\n\n    allowedkwargs = {\n        'constant': ['constant_values'],\n        'edge': [],\n        'linear_ramp': ['end_values'],\n        'maximum': ['stat_length'],\n        'mean': ['stat_length'],\n        'median': ['stat_length'],\n        'minimum': ['stat_length'],\n        'reflect': ['reflect_type'],\n        'symmetric': ['reflect_type'],\n        'wrap': [],\n        }\n\n    if isinstance(mode, _np.compat.basestring):\n        # Make sure have allowed kwargs appropriate for mode\n        for key in kwargs:\n            if key not in allowedkwargs[mode]:\n                raise ValueError(f'{key} keyword not in allowed keywords {allowedkwargs[mode]}')\n\n    unsupported_kwargs = set(kwargs) - set(allowedkwargs[mode])\n    if unsupported_kwargs:\n        raise ValueError(\"unsupported keyword arguments for mode '{}': {}\"\n                         .format(mode, unsupported_kwargs))\n    if mode == \"constant\":\n        values = kwargs.get(\"constant_values\", 0)\n        if isinstance(values, tuple):\n            raise TypeError(\"unsupported constant_values type: {'tuple'}.\")\n        return _npi.pad(x, pad_width, mode='constant', constant_values=values)\n    elif mode == \"symmetric\":\n        values = kwargs.get(\"reflect_type\", \"even\")\n        if values != \"even\" and values is not None:\n            raise ValueError(\"unsupported reflect_type '{}'\".format(values))\n        return _npi.pad(x, pad_width, mode='symmetric', reflect_type=\"even\")\n    elif mode == \"edge\":\n        return _npi.pad(x, pad_width, mode='edge')\n    elif mode == \"reflect\":\n        values = kwargs.get(\"reflect_type\", \"even\")\n        if values != \"even\" and values is not None:\n            raise ValueError(\"unsupported reflect_type '{}'\".format(values))\n        return _npi.pad(x, pad_width, mode='reflect', reflect_type=\"even\")\n    elif mode == \"maximum\":\n        values = kwargs.get(\"stat_length\", None)\n        if values is not None:\n            raise ValueError(\"unsupported stat_length '{}'\".format(values))\n        return _npi.pad(x, pad_width, mode='maximum')\n    elif mode == \"minimum\":\n        values = kwargs.get(\"stat_length\", None)\n        if values is not None:\n            raise ValueError(\"unsupported stat_length '{}'\".format(values))\n        return _npi.pad(x, pad_width, mode='minimum')\n    return _npi.pad(x, pad_width, mode='constant', constant_values=0)\n\n\n@set_module('mxnet.symbol.numpy')\ndef prod(a, axis=None, dtype=None, keepdims=False, initial=None, output=None): # pylint: disable=too-many-arguments\n    \"\"\"\n    Return the product of array elements over a given axis.\n\n    Parameters\n    ----------\n    a : array_like\n        Input data.\n    axis : None or int or tuple of ints, optional\n        Axis or axes along which a product is performed.  The default,\n        axis=None, will calculate the product of all the elements in the\n        input array. If axis is negative it counts from the last to the\n        first axis.\n        .. versionadded:: 1.7.0\n        If axis is a tuple of ints, a product is performed on all of the\n        axes specified in the tuple instead of a single axis or all the\n        axes as before.\n    dtype : dtype, optional\n        The type of the returned array, as well as of the accumulator in\n        which the elements are multiplied.  The dtype of `a` is used by\n        default unless `a` has an integer dtype of less precision than the\n        default platform integer.  In that case, if `a` is signed then the\n        platform integer is used while if `a` is unsigned then an unsigned\n        integer of the same precision as the platform integer is used.\n    out : ndarray, optional\n        Alternative output array in which to place the result. It must have\n        the same shape as the expected output, but the type of the output\n        values will be cast if necessary.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left in the\n        result as dimensions with size one. With this option, the result\n        will broadcast correctly against the input array.\n        If the default value is passed, then `keepdims` will not be\n        passed through to the `prod` method of sub-classes of\n        `ndarray`, however any non-default value will be.  If the\n        sub-class' method does not implement `keepdims` any\n        exceptions will be raised.\n    initial : scalar, optional\n        The starting value for this product. See `~numpy.ufunc.reduce` for details.\n    where : not supported\n\n    Returns\n    -------\n    product_along_axis : ndarray, see `dtype` parameter above.\n        An array shaped as `a` but with the specified axis removed.\n        Returns a reference to `out` if specified.\n\n    Examples\n    --------\n    By default, calculate the product of all elements:\n    >>> np.prod([1.,2.])\n    2.0\n    Even when the input array is two-dimensional:\n    >>> np.prod([[1.,2.],[3.,4.]])\n    24.0\n    But we can also specify the axis over which to multiply:\n    >>> np.prod([[1.,2.],[3.,4.]], axis=1)\n    array([  2.,  12.])\n    Or select specific elements to include:\n    >>> np.prod([1., np.nan, 3.], where=[True, False, True])\n    3.0\n    If the type of `x` is unsigned, then the output type is\n    the unsigned platform integer:\n    >>> x = np.array([1, 2, 3], dtype=np.uint8)\n    >>> np.prod(x).dtype == np.uint\n    True\n    If `x` is of a signed integer type, then the output type\n    is the default platform integer:\n    >>> x = np.array([1, 2, 3], dtype=np.int8)\n    >>> np.prod(x).dtype == int\n    True\n    You can also start the product with a value other than one:\n    >>> np.prod([1, 2], initial=5)\n    10\n    \"\"\"\n    return _npi.prod(a, axis=axis, dtype=dtype, keepdims=keepdims, initial=initial)\n\n@set_module('mxnet.symbol.numpy')\ndef cumsum(a, axis=None, dtype=None, out=None):\n    \"\"\"\n    Return the cumulative sum of the elements along a given axis.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input array.\n    axis : int, optional\n        Axis along which the cumulative sum is computed. The default\n        (None) is to compute the cumsum over the flattened array.\n    dtype : dtype, optional\n        Type of the returned array and of the accumulator in which the\n        elements are summed.  If `dtype` is not specified, it defaults\n        to the dtype of `a`, unless `a` has an integer dtype with a\n        precision less than that of the default platform integer.  In\n        that case, the default platform integer is used.\n    out : _Symbol, optional\n        Alternative output array in which to place the result. It must\n        have the same shape and buffer length as the expected output\n        but the type will be cast if necessary. See `doc.ufuncs`\n        (Section \"Output arguments\") for more details.\n\n    Returns\n    -------\n    cumsum_along_axis : _Symbol.\n        A new array holding the result is returned unless `out` is\n        specified, in which case a reference to `out` is returned. The\n        result has the same size as `a`, and the same shape as `a` if\n        `axis` is not None or `a` is a 1-d array.\n    \"\"\"\n    return _npi.cumsum(a, axis=axis, dtype=dtype, out=out)\n\n@set_module('mxnet.symbol.numpy')\ndef reshape(a, newshape, reverse=False, order='C'):\n    \"\"\"\n    Gives a new shape to an array without changing its data.\n    This function always returns a copy of the input array if\n    ``out`` is not provided.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Array to be reshaped.\n\n    newshape : int or tuple of ints\n        The new shape should be compatible with the original shape. If\n        an integer, then the result will be a 1-D array of that length.\n        One shape dimension can be -1. In this case, the value is\n        inferred from the length of the array and remaining dimensions.\n\n    order : {'C'}, optional\n        Read the elements of `a` using this index order, and place the\n        elements into the reshaped array using this index order.  'C'\n        means to read / write the elements using C-like index order,\n        with the last axis index changing fastest, back to the first\n        axis index changing slowest. Other order types such as 'F'/'A'\n        may be added in the future.\n\n    Returns\n    -------\n    reshaped_array : _Symbol\n        It will be always a copy of the original array. This behavior is different\n        from the official NumPy ``reshape`` operator where views of the original array may be\n        generated.\n\n    See Also\n    --------\n    ndarray.reshape : Equivalent method.\n\n    Examples\n    --------\n    >>> a = np.arange(6).reshape((3, 2))\n    >>> a\n    array([[0., 1.],\n           [2., 3.],\n           [4., 5.]])\n\n    >>> np.reshape(a, (2, 3)) # C-like index ordering\n    array([[0., 1., 2.],\n           [3., 4., 5.]])\n\n    >>> np.reshape(np.ravel(a), (2, 3)) # equivalent to C ravel then C reshape\n    array([[0., 1., 2.],\n           [3., 4., 5.]])\n\n    >>> a = np.array([[1,2,3], [4,5,6]])\n    >>> np.reshape(a, 6)\n    array([1., 2., 3., 4., 5., 6.])\n\n    >>> np.reshape(a, (3,-1))       # the unspecified value is inferred to be 2\n    array([[1., 2.],\n           [3., 4.],\n           [5., 6.]])\n    \"\"\"\n    return _npi.reshape(a, newshape, reverse, order)\n\n@set_module('mxnet.symbol.numpy')\ndef moveaxis(a, source, destination):\n    \"\"\"Move axes of an array to new positions.\n    Other axes remain in their original order.\n\n    Parameters\n    ----------\n    a : _Symbol\n        The array whose axes should be reordered.\n        source : int or sequence of int\n        Original positions of the axes to move. These must be unique.\n        destination : int or sequence of int\n        Destination positions for each of the original axes. These must also be\n        unique.\n\n    Returns\n    -------\n    result : _Symbol\n        Array with moved axes. This array is a view of the input array.\n\n    See Also\n    --------\n        transpose: Permute the dimensions of an array.\n        swapaxes: Interchange two axes of an array.\n\n    Examples\n    --------\n    >>> x = np.zeros((3, 4, 5))\n    >>> np.moveaxis(x, 0, -1).shape\n    (4, 5, 3)\n    >>> np.moveaxis(x, -1, 0).shape\n    (5, 3, 4)\n    These all achieve the same result:\n    >>> np.transpose(x).shape\n    (5, 4, 3)\n    >>> np.swapaxes(x, 0, -1).shape\n    (5, 4, 3)\n    >>> np.moveaxis(x, [0, 1], [-1, -2]).shape\n    (5, 4, 3)\n    >>> np.moveaxis(x, [0, 1, 2], [-1, -2, -3]).shape\n    (5, 4, 3)\n    \"\"\"\n    return _npi.moveaxis(a, source, destination)\n\n@set_module('mxnet.symbol.numpy')\ndef copy(a):  # pylint: disable=redefined-outer-name\n    \"\"\"\n    Return an array copy of the given object.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input array.\n\n    Returns\n    -------\n    arr : _Symbol\n        Array interpretation of a.\n\n    -----\n    Examples\n    --------\n    >>> x = np.array([1, 2, 3])\n    >>> y = x\n    >>> z = np.copy(x)\n    >>> x[0] = 10\n    >>> x[0] == y[0]\n        True\n    >>> x[0] == z[0]\n        False\n    \"\"\"\n    return _npi.copy(a)\n\n@set_module('mxnet.symbol.numpy')\ndef rollaxis(a, axis, start=0):\n    \"\"\"\n    Roll the specified axis backwards, until it lies in a given position.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input array.\n    axis : integer\n        The axis to roll backwards. The positions of the other axes do not\n        change relative to one another.\n    start: int, optional\n        The axis is rolled until it lies before this position.\n        The default, 0, results in a “complete” roll.\n\n    Returns\n    -------\n    res : _Symbol\n        A view after applying rollaxis to `a` is returned.\n\n    -----\n    Examples\n    --------\n    >>> a = np.ones((3,4,5,6))\n    >>> np.rollaxis(a, 3, 1).shape\n    (3, 6, 4, 5)\n    >>> np.rollaxis(a, 2).shape\n    (5, 3, 4, 6)\n    >>> np.rollaxis(a, 1, 4).shape\n    (3, 5, 6, 4)\n    \"\"\"\n    return _npi.rollaxis(a, axis, start)\n\n\n@set_module('mxnet.symbol.numpy')\ndef diag(v, k=0):\n    \"\"\"\n    Extracts a diagonal or constructs a diagonal array.\n    - 1-D arrays: constructs a 2-D array with the input as its diagonal, all other elements are zero.\n    - 2-D arrays: extracts the k-th Diagonal\n\n    Parameters\n    ----------\n    array : _Symbol\n        The array to apply diag method.\n    k : offset\n        extracts or constructs kth diagonal given input array\n\n    Returns\n    ----------\n    out : _Symbol\n    The extracted diagonal or constructed diagonal array.\n    \"\"\"\n    return _npi.diag(v, k=k)\n\n\n@set_module('mxnet.symbol.numpy')\ndef diagflat(v, k=0):\n    \"\"\"\n    Create a two-dimensional array with the flattened input as a diagonal.\n\n    Parameters\n    ----------\n    v : array_like\n        Input data, which is flattened and set as the `k`-th\n        diagonal of the output.\n    k : int, optional\n        Diagonal to set; 0, the default, corresponds to the \"main\" diagonal,\n        a positive (negative) `k` giving the number of the diagonal above\n        (below) the main.\n\n    Returns\n    -------\n    out : ndarray\n        The 2-D output array.\n\n    See Also\n    --------\n    diag : MATLAB work-alike for 1-D and 2-D arrays.\n    diagonal : Return specified diagonals.\n    trace : Sum along diagonals.\n\n    Examples\n    --------\n    >>> np.diagflat([[1,2], [3,4]])\n    array([[1, 0, 0, 0],\n           [0, 2, 0, 0],\n           [0, 0, 3, 0],\n           [0, 0, 0, 4]])\n    >>> np.diagflat([1,2], 1)\n    array([[0, 1, 0],\n           [0, 0, 2],\n           [0, 0, 0]])\n    \"\"\"\n    return _npi.diagflat(v, k=k)\n\n\n@set_module('mxnet.symbol.numpy')\ndef diagonal(a, offset=0, axis1=0, axis2=1):\n    \"\"\"\n    If a is 2-D, returns the diagonal of a with the given offset, i.e., the collection of elements of\n    the form a[i, i+offset]. If a has more than two dimensions, then the axes specified by axis1 and\n    axis2 are used to determine the 2-D sub-array whose diagonal is returned. The shape of the\n    resulting array can be determined by removing axis1 and axis2 and appending an index to the\n    right equal to the size of the resulting diagonals.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input data from which diagonal are taken.\n    offset: int, Optional\n        Offset of the diagonal from the main diagonal\n    axis1: int, Optional\n        Axis to be used as the first axis of the 2-D sub-arrays\n    axis2: int, Optional\n        Axis to be used as the second axis of the 2-D sub-arrays\n\n    Returns\n    -------\n    out : _Symbol\n        Output result\n\n    Raises\n    -------\n    ValueError:  If the dimension of a is less than 2.\n    \"\"\"\n    return _npi.diagonal(a, offset=offset, axis1=axis1, axis2=axis2)\n\n\n# pylint:disable=redefined-outer-name, too-many-arguments\n@set_module('mxnet.symbol.numpy')\ndef sum(a, axis=None, dtype=None, out=None, keepdims=False, initial=None, where=None):\n    r\"\"\"\n    Sum of array elements over a given axis.\n\n    Parameters\n    ----------\n    a : _Symbol\n        Input data.\n    axis : None or int, optional\n        Axis or axes along which a sum is performed.  The default,\n        axis=None, will sum all of the elements of the input array.  If\n        axis is negative it counts from the last to the first axis.\n    dtype : dtype, optional\n        The type of the returned array and of the accumulator in which the\n        elements are summed. The default type is float32.\n    keepdims : bool, optional\n        If this is set to True, the axes which are reduced are left\n        in the result as dimensions with size one. With this option,\n        the result will broadcast correctly against the input array.\n\n        If the default value is passed, then `keepdims` will not be\n        passed through to the `sum` method of sub-classes of\n        `ndarray`, however any non-default value will be.  If the\n        sub-classes `sum` method does not implement `keepdims` any\n        exceptions will be raised.\n    initial: Currently only supports None as input, optional\n        Starting value for the sum.\n        Currently not implemented. Please use ``None`` as input or skip this argument.\n    out : ndarray or None, optional\n        Alternative output array in which to place the result. It must have\n        the same shape and dtype as the expected output.\n\n    Returns\n    -------\n    sum_along_axis : _Symbol\n        An ndarray with the same shape as `a`, with the specified\n        axis removed. If an output array is specified, a reference to\n        `out` is returned.\n    \"\"\"\n    if where is not None and where is not True:\n        raise ValueError(\"only where=None or where=True cases are supported for now\")\n    return _npi.sum(a, axis=axis, dtype=dtype, keepdims=keepdims, initial=initial, out=out)\n# pylint:enable=redefined-outer-name, too-many-arguments\n\n\n_set_np_symbol_class(_Symbol)\n"
  },
  {
    "path": "python/mxnet/symbol/numpy/linalg.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for operators used in Gluon dispatched by F=symbol.\"\"\"\n\nimport numpy as _np\nfrom . import _symbol\nfrom . import _op as _mx_sym_np  # pylint: disable=unused-import\nfrom . import _internal as _npi\n\n__all__ = ['norm', 'svd', 'cholesky', 'qr', 'inv', 'det', 'slogdet', 'solve', 'tensorinv', 'tensorsolve',\n           'pinv', 'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq', 'matrix_rank']\n\n\ndef matrix_rank(M, tol=None, hermitian=False):\n    \"\"\"\n    Return matrix rank of array using SVD method\n\n    Rank of the array is the number of singular values of the array that are\n    greater than `tol`.\n\n    Parameters\n    M : {(M,), (..., M, N)} _Symbol\n        Input vector or stack of matrices.\n    tol : (...) _Symbol, float, optional\n        Threshold below which SVD values are considered zero. If `tol` is\n        None, and ``S`` is an array with singular values for `M`, and\n        ``eps`` is the epsilon value for datatype of ``S``, then `tol` is\n        set to ``S.max() * max(M.shape) * eps``.\n    hermitian : bool, optional\n        If True, `M` is assumed to be Hermitian (symmetric if real-valued),\n        enabling a more efficient method for finding singular values.\n        Defaults to False.\n\n    Returns\n    -------\n    rank : (...) _Symbol\n        Rank of M.\n    \"\"\"\n    finfo_eps_32 = _np.finfo(_np.float32).eps\n    finfo_eps_64 = _np.finfo(_np.float64).eps\n    if tol is None:\n        return _npi.matrix_rank_none_tol(M, finfo_eps_32, finfo_eps_64, hermitian)\n    else:\n        return _npi.matrix_rank(M, tol, hermitian)\n\n\ndef lstsq(a, b, rcond='warn'):\n    r\"\"\"\n    Return the least-squares solution to a linear matrix equation.\n\n    Solves the equation :math:`a x = b` by computing a vector `x` that\n    minimizes the squared Euclidean 2-norm :math:`\\| b - a x \\|^2_2`.\n    The equation may be under-, well-, or over-determined (i.e., the\n    number of linearly independent rows of `a` can be less than, equal\n    to, or greater than its number of linearly independent columns).\n    If `a` is square and of full rank, then `x` (but for round-off error)\n    is the \"exact\" solution of the equation.\n\n    Parameters\n    ----------\n    a : (M, N) _Symbol\n        \"Coefficient\" matrix.\n    b : {(M,), (M, K)} _Symbol\n        Ordinate or \"dependent variable\" values. If `b` is two-dimensional,\n        the least-squares solution is calculated for each of the `K` columns\n        of `b`.\n    rcond : float, optional\n        Cut-off ratio for small singular values of `a`.\n        For the purposes of rank determination, singular values are treated\n        as zero if they are smaller than `rcond` times the largest singular\n        value of `a`\n        The default of ``warn`` or ``-1`` will use the machine precision as\n        `rcond` parameter. The default of ``None`` will use the machine\n        precision times `max(M, N)` as `rcond` parameter.\n\n    Returns\n    -------\n    x : {(N,), (N, K)} _Symbol\n        Least-squares solution. If `b` is two-dimensional,\n        the solutions are in the `K` columns of `x`.\n    residuals : {(1,), (K,), (0,)} _Symbol\n        Sums of residuals.\n        Squared Euclidean 2-norm for each column in ``b - a*x``.\n        If the rank of `a` is < N or M <= N, this is an empty array.\n        If `b` is 1-dimensional, this is a (1,) shape array.\n        Otherwise the shape is (K,).\n    rank : int\n        Rank of matrix `a`.\n    s : (min(M, N),) _Symbol\n        Singular values of `a`.\n\n    Raises\n    ------\n    MXNetError\n        If computation does not converge.\n\n    Notes\n    -----\n    If `b` is a matrix, then all array results are returned as matrices.\n    \"\"\"\n    new_default = False\n    finfo_eps_32 = _np.finfo(_np.float32).eps\n    finfo_eps_64 = _np.finfo(_np.float64).eps\n    if rcond is None:\n        rcond = 1\n        new_default = True\n    if rcond == \"warn\":\n        rcond = -1\n    x, residuals, rank, s = _npi.lstsq(a, b, rcond=rcond, finfoEps32=finfo_eps_32, finfoEps64=finfo_eps_64, new_default=new_default)  # pylint: disable=line-too-long\n    return (x, residuals, rank, s)\n\n\ndef pinv(a, rcond=1e-15, hermitian=False):\n    r\"\"\"\n    Compute the (Moore-Penrose) pseudo-inverse of a matrix.\n\n    Calculate the generalized inverse of a matrix using its\n    singular-value decomposition (SVD) and including all\n    *large* singular values.\n\n    Parameters\n    ----------\n    a : (..., M, N) ndarray\n        Matrix or stack of matrices to be pseudo-inverted.\n    rcond : (...) {float or ndarray of float}, optional\n        Cutoff for small singular values.\n        Singular values less than or equal to\n        ``rcond * largest_singular_value`` are set to zero.\n        Broadcasts against the stack of matrices.\n    hermitian : bool, optional\n        If True, `a` is assumed to be Hermitian (symmetric if real-valued),\n        enabling a more efficient method for finding singular values.\n        Defaults to False.\n\n    Returns\n    -------\n    B : (..., N, M) ndarray\n        The pseudo-inverse of `a`. If `a` is a `matrix` instance, then so\n        is `B`.\n\n    Raises\n    ------\n    MXNetError\n        If the SVD computation does not converge.\n\n    Notes\n    -----\n    The pseudo-inverse of a matrix A, denoted :math:`A^+`, is\n    defined as: \"the matrix that 'solves' [the least-squares problem]\n    :math:`Ax = b`,\" i.e., if :math:`\\\\bar{x}` is said solution, then\n    :math:`A^+` is that matrix such that :math:`\\\\bar{x} = A^+b`.\n\n    It can be shown that if :math:`Q_1 \\\\Sigma Q_2^T = A` is the singular\n    value decomposition of A, then\n    :math:`A^+ = Q_2 \\\\Sigma^+ Q_1^T`, where :math:`Q_{1,2}` are\n    orthogonal matrices, :math:`\\\\Sigma` is a diagonal matrix consisting\n    of A's so-called singular values, (followed, typically, by\n    zeros), and then :math:`\\\\Sigma^+` is simply the diagonal matrix\n    consisting of the reciprocals of A's singular values\n    (again, followed by zeros). [1]_\n\n    References\n    ----------\n    .. [1] G. Strang, *Linear Algebra and Its Applications*, 2nd Ed., Orlando,\n           FL, Academic Press, Inc., 1980, pp. 139-142.\n\n    Examples\n    --------\n    The following example checks that ``a * a+ * a == a`` and\n    ``a+ * a * a+ == a+``:\n    >>> a = np.random.randn(2, 3)\n    >>> pinv_a = np.linalg.pinv(a)\n    >>> (a - np.dot(a, np.dot(pinv_a, a))).sum()\n    array(0.)\n    >>> (pinv_a - np.dot(pinv_a, np.dot(a, pinv_a))).sum()\n    array(0.)\n    \"\"\"\n    if hermitian is True:\n        raise NotImplementedError(\"hermitian is not supported yet...\")\n    if _symbol._np.isscalar(rcond):\n        return _npi.pinv_scalar_rcond(a, rcond, hermitian)\n    return _npi.pinv(a, rcond, hermitian)\n\n\n# pylint: disable=too-many-return-statements\ndef norm(x, ord=None, axis=None, keepdims=False):\n    r\"\"\"Matrix or vector norm.\n    This function is able to return one of eight different matrix norms,\n    or one of an infinite number of vector norms (described below), depending\n    on the value of the ``ord`` parameter.\n    Parameters\n    ----------\n    x : _Symbol\n        Input array.  If `axis` is None, `x` must be 1-D or 2-D.\n    ord : {non-zero int, inf, -inf, 'fro', 'nuc'}, optional\n        Order of the norm (see table under ``Notes``). inf means numpy's\n        `inf` object.\n    axis : {int, 2-tuple of ints, None}, optional\n        If `axis` is an integer, it specifies the axis of `x` along which to\n        compute the vector norms.  If `axis` is a 2-tuple, it specifies the\n        axes that hold 2-D matrices, and the matrix norms of these matrices\n        are computed.  If `axis` is None then either a vector norm (when `x`\n        is 1-D) or a matrix norm (when `x` is 2-D) is returned.\n    keepdims : bool, optional\n        If this is set to True, the axes which are normed over are left in the\n        result as dimensions with size one.  With this option the result will\n        broadcast correctly against the original `x`.\n    Returns\n    -------\n    n : _Symbol\n        Norm of the matrix or vector(s).\n    Notes\n    -----\n    For values of ``ord <= 0``, the result is, strictly speaking, not a\n    mathematical 'norm', but it may still be useful for various numerical\n    purposes.\n    The following norms can be calculated:\n    =====  ============================  ==========================\n    ord    norm for matrices             norm for vectors\n    =====  ============================  ==========================\n    None   Frobenius norm                2-norm\n    'fro'  Frobenius norm                --\n    'nuc'  --                            --\n    inf    max(sum(abs(x), axis=1))      max(abs(x))\n    -inf   min(sum(abs(x), axis=1))      min(abs(x))\n    0      --                            sum(x != 0)\n    1      max(sum(abs(x), axis=0))      as below\n    -1     min(sum(abs(x), axis=0))      as below\n    2      --                            as below\n    -2     --                            as below\n    other  --                            sum(abs(x)**ord)**(1./ord)\n    =====  ============================  ==========================\n    The Frobenius norm is given by [1]_:\n        :math:`||A||_F = [\\sum_{i,j} abs(a_{i,j})^2]^{1/2}`\n    The nuclear norm is the sum of the singular values.\n    When you want to operate norm for matrices,if you ord is (-1, 1, inf, -inf),\n    you must give you axis, it is not support default axis.\n    References\n    ----------\n    .. [1] G. H. Golub and C. F. Van Loan, *Matrix Computations*,\n           Baltimore, MD, Johns Hopkins University Press, 1985, pg. 15\n    Examples\n    --------\n    >>> from mxnet import np\n    >>> a = np.arange(9) - 4\n    >>> a\n    array([-4., -3., -2., -1.,  0.,  1.,  2.,  3.,  4.])\n    >>> b = a.reshape((3, 3))\n    >>> b\n    array([[-4., -3., -2.],\n           [-1.,  0.,  1.],\n           [ 2.,  3.,  4.]])\n    >>> np.linalg.norm(a)\n    array(7.745967)\n    >>> np.linalg.norm(b)\n    array(7.745967)\n    >>> np.linalg.norm(b, 'fro')\n    array(7.745967)\n    >>> np.linalg.norm(a, 'inf')\n    array(4.)\n    >>> np.linalg.norm(b, 'inf', axis=(0, 1))\n    array(9.)\n    >>> np.linalg.norm(a, '-inf')\n    array(0.)\n    >>> np.linalg.norm(b, '-inf', axis=(0, 1))\n    array(2.)\n    >>> np.linalg.norm(a, 1)\n    array(20.)\n    >>> np.linalg.norm(b, 1, axis=(0, 1))\n    array(7.)\n    >>> np.linalg.norm(a, -1)\n    array(0.)\n    >>> np.linalg.norm(b, -1, axis=(0, 1))\n    array(6.)\n    >>> np.linalg.norm(a, 2)\n    array(7.745967)\n    >>> np.linalg.norm(a, -2)\n    array(0.)\n    >>> np.linalg.norm(a, 3)\n    array(5.8480353)\n    >>> np.linalg.norm(a, -3)\n    array(0.)\n    Using the `axis` argument to compute vector norms:\n    >>> c = np.array([[ 1, 2, 3],\n    ...               [-1, 1, 4]])\n    >>> np.linalg.norm(c, axis=0)\n    array([1.4142135, 2.236068 , 5.       ])\n    >>> np.linalg.norm(c, axis=1)\n    array([3.7416573, 4.2426405])\n    >>> np.linalg.norm(c, ord=1, axis=1)\n    array([6., 6.])\n    Using the `axis` argument to compute matrix norms:\n    >>> m = np.arange(8).reshape(2,2,2)\n    >>> np.linalg.norm(m, axis=(1,2))\n    array([ 3.7416573, 11.224973 ])\n    >>> np.linalg.norm(m[0, :, :]), np.linalg.norm(m[1, :, :])\n    (array(3.7416573), array(11.224973))\n    \"\"\"\n    if axis is None and ord is None:\n        return _npi.norm(x, ord=2, axis=None, keepdims=keepdims, flag=-2)\n    if axis is None or isinstance(axis, (int, tuple)):  # pylint: disable=too-many-nested-blocks\n        if axis is not None:\n            if isinstance(axis, int):\n                axis = (axis, )\n            if len(axis) == 2:\n                if ord in ['inf', '-inf']:\n                    row_axis, col_axis = axis\n                    if not keepdims:\n                        if row_axis > col_axis:\n                            row_axis -= 1\n                    if ord == 'inf':\n                        return _npi.sum(_symbol.abs(x), axis=col_axis, keepdims=keepdims).max(axis=row_axis, keepdims=keepdims)  # pylint: disable=line-too-long\n                    else:\n                        return _npi.sum(_symbol.abs(x), axis=col_axis, keepdims=keepdims).min(axis=row_axis, keepdims=keepdims)  # pylint: disable=line-too-long\n                if ord in [1, -1]:\n                    row_axis, col_axis = axis\n                    if not keepdims:\n                        if row_axis < col_axis:\n                            col_axis -= 1\n                    if ord == 1:\n                        return _npi.sum(_symbol.abs(x), axis=row_axis, keepdims=keepdims).max(axis=col_axis, keepdims=keepdims)  # pylint: disable=line-too-long\n                    elif ord == -1:\n                        return _npi.sum(_symbol.abs(x), axis=row_axis, keepdims=keepdims).min(axis=col_axis, keepdims=keepdims)  # pylint: disable=line-too-long\n                if ord in [2, -2]:\n                    return _npi.norm(x, ord=ord, axis=axis, keepdims=keepdims, flag=0)\n                if ord is None:\n                    return _npi.norm(x, ord=2, axis=axis, keepdims=keepdims, flag=1)\n        if ord == 'inf':\n            return _npi.max(_symbol.abs(x), axis=axis, keepdims=keepdims)\n            #return _npi.norm(x, ord=float('inf'), axis=axis, keepdims=keepdims, flag=3)\n        elif ord == '-inf':\n            return _npi.min(_symbol.abs(x), axis=axis, keepdims=keepdims)\n            #return _npi.norm(x, ord=-float('inf'), axis=axis, keepdims=keepdims, flag=4)\n        elif ord is None:\n            return _npi.norm(x, ord=2, axis=axis, keepdims=keepdims, flag=1)\n        elif ord == 2:\n            return _npi.norm(x, ord=2, axis=axis, keepdims=keepdims, flag=-1)\n        elif ord == 'nuc':\n            return _npi.norm(x, ord=2, axis=axis, keepdims=keepdims, flag=2)\n        elif ord in ['fro', 'f']:\n            return _npi.norm(x, ord=2, axis=axis, keepdims=keepdims, flag=1)\n        else:\n            return _npi.norm(x, ord=ord, axis=axis, keepdims=keepdims, flag=-1)\n    else:\n        raise TypeError(\"'axis' must be None, an integer or a tuple of integers.\")\n# pylint: enable=too-many-return-statements\n\n\ndef svd(a):\n    r\"\"\"\n    Singular Value Decomposition.\n\n    When `a` is a 2D array, it is factorized as ``ut @ np.diag(s) @ v``,\n    where `ut` and `v` are 2D orthonormal arrays and `s` is a 1D\n    array of `a`'s singular values. When `a` is higher-dimensional, SVD is\n    applied in stacked mode as explained below.\n\n    Parameters\n    ----------\n    a : (..., M, N) _Symbol\n        A real array with ``a.ndim >= 2`` and ``M <= N``.\n\n    Returns\n    -------\n    ut: (..., M, M) _Symbol\n        Orthonormal array(s). The first ``a.ndim - 2`` dimensions have the same\n        size as those of the input `a`.\n    s : (..., M) _Symbol\n        Vector(s) with the singular values, within each vector sorted in\n        descending order. The first ``a.ndim - 2`` dimensions have the same\n        size as those of the input `a`.\n    v : (..., M, N) _Symbol\n        Orthonormal array(s). The first ``a.ndim - 2`` dimensions have the same\n        size as those of the input `a`.\n\n    Notes\n    -----\n\n    The decomposition is performed using LAPACK routine ``_gesvd``.\n\n    SVD is usually described for the factorization of a 2D matrix :math:`A`.\n    The higher-dimensional case will be discussed below. In the 2D case, SVD is\n    written as :math:`A = U^T S V`, where :math:`A = a`, :math:`U^T = ut`,\n    :math:`S= \\mathtt{np.diag}(s)` and :math:`V = v`. The 1D array `s`\n    contains the singular values of `a` and `ut` and `v` are orthonormal. The rows\n    of `v` are the eigenvectors of :math:`A^T A` and the columns of `ut` are\n    the eigenvectors of :math:`A A^T`. In both cases the corresponding\n    (possibly non-zero) eigenvalues are given by ``s**2``.\n\n    The sign of rows of `u` and `v` are determined as described in\n    `Auto-Differentiating Linear Algebra <https://arxiv.org/pdf/1710.08717.pdf>`_.\n\n    If `a` has more than two dimensions, then broadcasting rules apply.\n    This means that SVD is working in \"stacked\" mode: it iterates over\n    all indices of the first ``a.ndim - 2`` dimensions and for each\n    combination SVD is applied to the last two indices. The matrix `a`\n    can be reconstructed from the decomposition with either\n    ``(ut * s[..., None, :]) @ v`` or\n    ``ut @ (s[..., None] * v)``. (The ``@`` operator denotes batch matrix multiplication)\n\n    This function differs from the original `numpy.linalg.svd\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.svd.html>`_ in\n    the following way(s):\n     - The sign of rows of `u` and `v` may differ.\n     - Does not support complex input.\n    \"\"\"\n    return _npi.svd(a)\n\n\ndef cholesky(a):\n    r\"\"\"\n    Cholesky decomposition.\n\n    Return the Cholesky decomposition, `L * L.T`, of the square matrix `a`,\n    where `L` is lower-triangular and .T is the transpose operator. `a` must be\n    symmetric and positive-definite. Only `L` is actually returned. Complex-valued\n    input is currently not supported.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Symmetric, positive-definite input matrix.\n\n    Returns\n    -------\n    L : (..., M, M) ndarray\n        Lower-triangular Cholesky factor of `a`.\n\n    Raises\n    ------\n    MXNetError\n        If the decomposition fails, for example, if `a` is not positive-definite.\n\n    Notes\n    -----\n    Broadcasting rules apply.\n\n    The Cholesky decomposition is often used as a fast way of solving\n\n    .. math:: A \\mathbf{x} = \\mathbf{b}\n\n    (when `A` is both symmetric and positive-definite).\n\n    First, we solve for :math:`\\mathbf{y}` in\n\n    .. math:: L \\mathbf{y} = \\mathbf{b},\n\n    and then for :math:`\\mathbf{x}` in\n\n    .. math:: L.T \\mathbf{x} = \\mathbf{y}.\n\n    Examples\n    --------\n    >>> A = np.array([[16, 4], [4, 10]])\n    >>> A\n    array([[16.,  4.],\n           [ 4., 10.]])\n    >>> L = np.linalg.cholesky(A)\n    >>> L\n    array([[4., 0.],\n           [1., 3.]])\n    >>> np.dot(L, L.T)\n    array([[16.,  4.],\n           [ 4., 10.]])\n    \"\"\"\n    return _npi.cholesky(a, True)\n\n\ndef qr(a, mode='reduced'):\n    r\"\"\"\n    Compute the qr factorization of a matrix a.\n    Factor the matrix a as qr, where q is orthonormal and r is upper-triangular.\n\n    Parameters\n    ----------\n    a : (..., M, N) _Symbol\n        Matrix or stack of matrices to be qr factored.\n    mode: {‘reduced’, ‘complete’, ‘r’, ‘raw’, ‘full’, ‘economic’}, optional\n        Only default mode, 'reduced', is implemented. If K = min(M, N), then\n        * 'reduced’ : returns q, r with dimensions (M, K), (K, N) (default)\n\n    Returns\n    -------\n    q : (..., M, K) _Symbol\n        A matrix or stack of matrices with K orthonormal columns, with K = min(M, N).\n    r : (..., K, N) _Symbol\n        A matrix or stack of upper triangular matrices.\n\n    Raises\n    ------\n    MXNetError\n        If factoring fails.\n\n    Examples\n    --------\n    >>> from mxnet import np\n    >>> a = np.random.uniform(-10, 10, (2, 2))\n    >>> q, r = np.linalg.qr(a)\n    >>> q\n    array([[-0.22121978, -0.97522414],\n           [-0.97522414,  0.22121954]])\n    >>> r\n    array([[-4.4131265 , -7.1255064 ],\n           [ 0.        , -0.28771925]])\n    >>> a = np.random.uniform(-10, 10, (2, 3))\n    >>> q, r = np.linalg.qr(a)\n    >>> q\n    array([[-0.28376842, -0.9588929 ],\n           [-0.9588929 ,  0.28376836]])\n    >>> r\n    array([[-7.242763  , -0.5673361 , -2.624416  ],\n           [ 0.        , -7.297918  , -0.15949416]])\n    >>> a = np.random.uniform(-10, 10, (3, 2))\n    >>> q, r = np.linalg.qr(a)\n    >>> q\n    array([[-0.34515655,  0.10919492],\n           [ 0.14765628, -0.97452265],\n           [-0.92685735, -0.19591334]])\n    >>> r\n    array([[-8.453794,  8.4175  ],\n           [ 0.      ,  5.430561]])\n    \"\"\"\n    if mode is not None and mode != 'reduced':\n        raise NotImplementedError(\"Only default mode='reduced' is implemented.\")\n    return _npi.qr(a)\n\n\ndef inv(a):\n    r\"\"\"\n    Compute the (multiplicative) inverse of a matrix.\n\n    Given a square matrix `a`, return the matrix `ainv` satisfying\n    ``dot(a, ainv) = dot(ainv, a) = eye(a.shape[0])``.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Matrix to be inverted.\n\n    Returns\n    -------\n    ainv : (..., M, M) ndarray\n        (Multiplicative) inverse of the matrix `a`.\n\n    Raises\n    ------\n    MXNetError\n        If `a` is not square or inversion fails.\n\n    Examples\n    --------\n    >>> from mxnet import np\n    >>> a = np.array([[1., 2.], [3., 4.]])\n    array([[-2. ,  1. ],\n           [ 1.5, -0.5]])\n\n    Inverses of several matrices can be computed at once:\n\n    >>> a = np.array([[[1., 2.], [3., 4.]], [[1, 3], [3, 5]]])\n    >>> np.linalg.inv(a)\n    array([[[-2.        ,  1.        ],\n            [ 1.5       , -0.5       ]],\n\n           [[-1.2500001 ,  0.75000006],\n            [ 0.75000006, -0.25000003]]])\n    \"\"\"\n    return _npi.inv(a)\n\n\ndef det(a):\n    r\"\"\"\n    Compute the determinant of an array.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Input array to compute determinants for.\n\n    Returns\n    -------\n    det : (...) ndarray\n        Determinant of `a`.\n\n    See Also\n    --------\n    slogdet : Another way to represent the determinant, more suitable\n    for large matrices where underflow/overflow may occur.\n\n    Notes\n    -----\n    Broadcasting rules apply, see the `numpy.linalg` documentation for\n    details.\n    The determinant is computed via LU factorization using the LAPACK\n    routine z/dgetrf.\n\n    Examples\n    --------\n    The determinant of a 2-D array [[a, b], [c, d]] is ad - bc:\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> np.linalg.det(a)\n    -2.0\n\n    Computing determinants for a stack of matrices:\n    >>> a = np.array([ [[1, 2], [3, 4]], [[1, 2], [2, 1]], [[1, 3], [3, 1]] ])\n    >>> a.shape\n    (3, 2, 2)\n\n    >>> np.linalg.det(a)\n    array([-2., -3., -8.])\n    \"\"\"\n    return _npi.det(a)\n\n\ndef slogdet(a):\n    r\"\"\"\n    Compute the sign and (natural) logarithm of the determinant of an array.\n    If an array has a very small or very large determinant, then a call to\n    `det` may overflow or underflow. This routine is more robust against such\n    issues, because it computes the logarithm of the determinant rather than\n    the determinant itself.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Input array, has to be a square 2-D array.\n\n    Returns\n    -------\n    sign : (...) ndarray\n        A number representing the sign of the determinant. For a real matrix,\n        this is 1, 0, or -1.\n    logdet : (...) array_like\n        The natural log of the absolute value of the determinant.\n    If the determinant is zero, then `sign` will be 0 and `logdet` will be\n    -Inf. In all cases, the determinant is equal to ``sign * np.exp(logdet)``.\n\n    See Also\n    --------\n    det\n\n    Notes\n    -----\n    Broadcasting rules apply, see the `numpy.linalg` documentation for\n    details.\n    The determinant is computed via LU factorization using the LAPACK\n    routine z/dgetrf.\n\n    Examples\n    --------\n    The determinant of a 2-D array ``[[a, b], [c, d]]`` is ``ad - bc``:\n    >>> a = np.array([[1, 2], [3, 4]])\n    >>> (sign, logdet) = np.linalg.slogdet(a)\n    >>> (sign, logdet)\n    (-1., 0.69314718055994529)\n\n    >>> sign * np.exp(logdet)\n    -2.0\n\n    Computing log-determinants for a stack of matrices:\n    >>> a = np.array([ [[1, 2], [3, 4]], [[1, 2], [2, 1]], [[1, 3], [3, 1]] ])\n    >>> a.shape\n    (3, 2, 2)\n\n    >>> sign, logdet = np.linalg.slogdet(a)\n    >>> (sign, logdet)\n    (array([-1., -1., -1.]), array([ 0.69314718,  1.09861229,  2.07944154]))\n\n    >>> sign * np.exp(logdet)\n    array([-2., -3., -8.])\n\n    This routine succeeds where ordinary `det` does not:\n    >>> np.linalg.det(np.eye(500) * 0.1)\n    0.0\n    >>> np.linalg.slogdet(np.eye(500) * 0.1)\n    (1., -1151.2925464970228)\n    \"\"\"\n    return _npi.slogdet(a)\n\n\ndef solve(a, b):\n    r\"\"\"\n    Solve a linear matrix equation, or system of linear scalar equations.\n\n    Computes the \"exact\" solution, `x`, of the well-determined, i.e., full\n    rank, linear matrix equation `ax = b`.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Coefficient matrix.\n    b : {(..., M,), (..., M, K)}, ndarray\n        Ordinate or \"dependent variable\" values.\n\n    Returns\n    -------\n    x : {(..., M,), (..., M, K)} ndarray\n        Solution to the system a x = b.  Returned shape is identical to `b`.\n\n    Raises\n    ------\n    MXNetError\n        If `a` is singular or not square.\n\n    Notes\n    -----\n    Broadcasting rules apply, see the `numpy.linalg` documentation for\n    details.\n\n    The solutions are computed using LAPACK routine ``_gesv``.\n\n    `a` must be square and of full-rank, i.e., all rows (or, equivalently,\n    columns) must be linearly independent; if either is not true, use\n    `lstsq` for the least-squares best \"solution\" of the\n    system/equation.\n\n    Examples\n    --------\n    Solve the system of equations ``3 * x0 + x1 = 9`` and ``x0 + 2 * x1 = 8``:\n\n    >>> a = np.array([[3,1], [1,2]])\n    >>> b = np.array([9,8])\n    >>> x = np.linalg.solve(a, b)\n    >>> x\n    array([2.,  3.])\n\n    Check that the solution is correct:\n\n    >>> np.allclose(np.dot(a, x), b)\n    True\n    \"\"\"\n    return _npi.solve(a, b)\n\n\ndef tensorinv(a, ind=2):\n    r\"\"\"\n    Compute the 'inverse' of an N-dimensional array.\n\n    The result is an inverse for `a` relative to the tensordot operation\n    ``tensordot(a, b, ind)``, i. e., up to floating-point accuracy,\n    ``tensordot(tensorinv(a), a, ind)`` is the \"identity\" tensor for the\n    tensordot operation.\n\n    Parameters\n    ----------\n    a : array_like\n        Tensor to 'invert'. Its shape must be 'square', i. e.,\n        ``prod(a.shape[:ind]) == prod(a.shape[ind:])``.\n    ind : int, optional\n        Number of first indices that are involved in the inverse sum.\n        Must be a positive integer, default is 2.\n\n    Returns\n    -------\n    b : ndarray\n        `a`'s tensordot inverse, shape ``a.shape[ind:] + a.shape[:ind]``.\n\n    Raises\n    ------\n    MXNetError\n        If `a` is singular or not 'square' (in the above sense).\n\n    See Also\n    --------\n    tensordot, tensorsolve\n\n    Examples\n    --------\n    >>> a = np.eye(4*6)\n    >>> a.shape = (4, 6, 8, 3)\n    >>> ainv = np.linalg.tensorinv(a, ind=2)\n    >>> ainv.shape\n    (8, 3, 4, 6)\n    >>> b = np.random.randn(4, 6)\n    >>> np.allclose(np.tensordot(ainv, b), np.linalg.tensorsolve(a, b))\n    True\n\n    >>> a = np.eye(4*6)\n    >>> a.shape = (24, 8, 3)\n    >>> ainv = np.linalg.tensorinv(a, ind=1)\n    >>> ainv.shape\n    (8, 3, 24)\n    >>> b = np.random.randn(24)\n    >>> np.allclose(np.tensordot(ainv, b, 1), np.linalg.tensorsolve(a, b))\n    True\n    \"\"\"\n    return _npi.tensorinv(a, ind)\n\n\ndef tensorsolve(a, b, axes=None):\n    r\"\"\"\n    Solve the tensor equation ``a x = b`` for x.\n    It is assumed that all indices of `x` are summed over in the product,\n    together with the rightmost indices of `a`, as is done in, for example,\n    ``tensordot(a, x, axes=b.ndim)``.\n\n    Parameters\n    ----------\n    a : ndarray\n        Coefficient tensor, of shape ``b.shape + Q``. `Q`, a tuple, equals\n        the shape of that sub-tensor of `a` consisting of the appropriate\n        number of its rightmost indices, and must be such that\n        ``prod(Q) == prod(b.shape)`` (in which sense `a` is said to be\n        'square').\n    b : ndarray\n        Right-hand tensor, which can be of any shape.\n    axes : tuple of ints, optional\n        Axes in `a` to reorder to the right, before inversion.\n        If None (default), no reordering is done.\n\n    Returns\n    -------\n    x : ndarray, shape Q\n\n    Raises\n    ------\n    MXNetError\n        If `a` is singular or not 'square' (in the above sense).\n\n    See Also\n    --------\n    numpy.tensordot, tensorinv, numpy.einsum\n\n    Examples\n    --------\n    >>> a = np.eye(2*3*4)\n    >>> a.shape = (2*3, 4, 2, 3, 4)\n    >>> b = np.random.randn(2*3, 4)\n    >>> x = np.linalg.tensorsolve(a, b)\n    >>> x.shape\n    (2, 3, 4)\n    >>> np.allclose(np.tensordot(a, x, axes=3), b)\n    True\n    \"\"\"\n    return _npi.tensorsolve(a, b, axes)\n\n\ndef eigvals(a):\n    r\"\"\"\n    Compute the eigenvalues of a general matrix.\n\n    Main difference between `eigvals` and `eig`: the eigenvectors aren't\n    returned.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        A real-valued matrix whose eigenvalues will be computed.\n\n    Returns\n    -------\n    w : (..., M,) ndarray\n        The eigenvalues, each repeated according to its multiplicity.\n        They are not necessarily ordered.\n\n    Raises\n    ------\n    MXNetError\n        If the eigenvalue computation does not converge.\n\n    See Also\n    --------\n    eig : eigenvalues and right eigenvectors of general arrays\n    eigh : eigenvalues and eigenvectors of a real symmetric array.\n    eigvalsh : eigenvalues of a real symmetric.\n\n    Notes\n    -----\n    Broadcasting rules apply, see the `numpy.linalg` documentation for\n    details.\n\n    This is implemented using the ``_geev`` LAPACK routines which compute\n    the eigenvalues and eigenvectors of general square arrays.\n\n    This function differs from the original `numpy.linalg.eigvals\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.eigvals.html>`_ in\n    the following way(s):\n     - Does not support complex input and output.\n    \"\"\"\n    return _npi.eigvals(a)\n\n\ndef eigvalsh(a, UPLO='L'):\n    r\"\"\"\n    Compute the eigenvalues real symmetric matrix.\n\n    Main difference from eigh: the eigenvectors are not computed.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        A real-valued matrix whose eigenvalues are to be computed.\n    UPLO : {'L', 'U'}, optional\n        Specifies whether the calculation is done with the lower triangular\n        part of `a` ('L', default) or the upper triangular part ('U').\n        Irrespective of this value only the real parts of the diagonal will\n        be considered in the computation to preserve the notion of a Hermitian\n        matrix. It therefore follows that the imaginary part of the diagonal\n        will always be treated as zero.\n\n    Returns\n    -------\n    w : (..., M,) ndarray\n        The eigenvalues in ascending order, each repeated according to\n        its multiplicity.\n\n    Raises\n    ------\n    MXNetError\n        If the eigenvalue computation does not converge.\n\n    See Also\n    --------\n    eig : eigenvalues and right eigenvectors of general arrays\n    eigvals : eigenvalues of a non-symmetric array.\n    eigh : eigenvalues and eigenvectors of a real symmetric array.\n\n    Notes\n    -----\n    Broadcasting rules apply, see the `numpy.linalg` documentation for\n    details.\n\n    The eigenvalues are computed using LAPACK routines ``_syevd``.\n\n    This function differs from the original `numpy.linalg.eigvalsh\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.eigvalsh.html>`_ in\n    the following way(s):\n     - Does not support complex input and output.\n    \"\"\"\n    return _npi.eigvalsh(a, UPLO)\n\n\ndef eig(a):\n    r\"\"\"\n    Compute the eigenvalues and right eigenvectors of a square array.\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        Matrices for which the eigenvalues and right eigenvectors will\n        be computed\n\n    Returns\n    -------\n    w : (..., M) ndarray\n        The eigenvalues, each repeated according to its multiplicity.\n        The eigenvalues are not necessarily ordered.\n    v : (..., M, M) ndarray\n        The normalized (unit \"length\") eigenvectors, such that the\n        column ``v[:,i]`` is the eigenvector corresponding to the\n        eigenvalue ``w[i]``.\n\n    Raises\n    ------\n    MXNetError\n        If the eigenvalue computation does not converge.\n\n    See Also\n    --------\n    eigvals : eigenvalues of a non-symmetric array.\n    eigh : eigenvalues and eigenvectors of a real symmetric array.\n    eigvalsh : eigenvalues of a real symmetric.\n\n    Notes\n    -----\n    This is implemented using the ``_geev`` LAPACK routines which compute\n    the eigenvalues and eigenvectors of general square arrays.\n\n    The number `w` is an eigenvalue of `a` if there exists a vector\n    `v` such that ``dot(a,v) = w * v``. Thus, the arrays `a`, `w`, and\n    `v` satisfy the equations ``dot(a[:,:], v[:,i]) = w[i] * v[:,i]``\n    for :math:`i \\\\in \\\\{0,...,M-1\\\\}`.\n\n    The array `v` of eigenvectors may not be of maximum rank, that is, some\n    of the columns may be linearly dependent, although round-off error may\n    obscure that fact. If the eigenvalues are all different, then theoretically\n    the eigenvectors are linearly independent.\n\n    This function differs from the original `numpy.linalg.eig\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.eig.html>`_ in\n    the following way(s):\n     - Does not support complex input and output.\n    \"\"\"\n    return _npi.eig(a)\n\n\ndef eigh(a, UPLO='L'):\n    r\"\"\"\n    Return the eigenvalues and eigenvectors real symmetric matrix.\n\n    Returns two objects, a 1-D array containing the eigenvalues of `a`, and\n    a 2-D square array or matrix (depending on the input type) of the\n    corresponding eigenvectors (in columns).\n\n    Parameters\n    ----------\n    a : (..., M, M) ndarray\n        real symmetric matrices whose eigenvalues and eigenvectors are to be computed.\n    UPLO : {'L', 'U'}, optional\n        Specifies whether the calculation is done with the lower triangular\n        part of `a` ('L', default) or the upper triangular part ('U').\n        Irrespective of this value only the real parts of the diagonal will\n        be considered in the computation to preserve the notion of a Hermitian\n        matrix. It therefore follows that the imaginary part of the diagonal\n        will always be treated as zero.\n\n    Returns\n    -------\n    w : (..., M) ndarray\n        The eigenvalues in ascending order, each repeated according to\n        its multiplicity.\n    v : {(..., M, M) ndarray, (..., M, M) matrix}\n        The column ``v[:, i]`` is the normalized eigenvector corresponding\n        to the eigenvalue ``w[i]``.  Will return a matrix object if `a` is\n        a matrix object.\n\n    Raises\n    ------\n    MXNetError\n        If the eigenvalue computation does not converge.\n\n    See Also\n    --------\n    eig : eigenvalues and right eigenvectors of general arrays\n    eigvals : eigenvalues of a non-symmetric array.\n    eigvalsh : eigenvalues of a real symmetric.\n\n    Notes\n    -----\n    The eigenvalues/eigenvectors are computed using LAPACK routines ``_syevd``.\n\n    This function differs from the original `numpy.linalg.eigh\n    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.eigh.html>`_ in\n    the following way(s):\n     - Does not support complex input and output.\n    \"\"\"\n    return _npi.eigh(a, UPLO)\n"
  },
  {
    "path": "python/mxnet/symbol/numpy/random.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for operators used in Gluon dispatched by F=symbol.\"\"\"\n\nimport numpy as np\nfrom ...context import current_context\nfrom ...util import is_np_default_dtype\nfrom . import _internal as _npi\n\n\n__all__ = ['randint', 'uniform', 'normal', 'multivariate_normal',\n           'logistic', 'gumbel', 'rayleigh', 'f',\n           'rand', 'shuffle', 'gamma', 'beta', 'chisquare', 'exponential', 'lognormal',\n           'weibull', 'pareto', 'power', 'laplace']\n\n\ndef randint(low, high=None, size=None, dtype=None, ctx=None, out=None):\n    r\"\"\"Return random integers from `low` (inclusive) to `high` (exclusive).\n\n    Return random integers from the \"discrete uniform\" distribution of\n    the specified dtype in the \"half-open\" interval [`low`, `high`). If\n    `high` is None (the default), then results are from [0, `low`).\n\n    Parameters\n    ----------\n    low : int\n        Lowest (signed) integer to be drawn from the distribution (unless\n        ``high=None``, in which case this parameter is one above the\n        *highest* such integer).\n    high : int, optional\n        If provided, one above the largest (signed) integer to be drawn\n        from the distribution (see above for behavior if ``high=None``).\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  Default is None, in which case a\n        single value is returned.\n    dtype : dtype, optional\n        Desired dtype of the result. All dtypes are determined by their\n        name, i.e., 'int64', 'int', etc, so byteorder is not available\n        and a specific precision may have different C types depending\n        on the platform. The default value is 'np.int'.\n    ctx : Context, optional\n        Device context of output. Default is current context.\n    out : _Symbol, optional\n        The output symbol (default is `None`).\n\n    Returns\n    -------\n    out : _Symbol\n        `size`-shaped array of random integers from the appropriate\n        distribution, or a single such random int if `size` not provided.\n\n    Examples\n    --------\n    >>> np.random.randint(2, size=10)\n    array([1, 0, 0, 0, 1, 1, 0, 0, 1, 0])\n    >>> np.random.randint(1, size=10)\n    array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n\n    Generate a 2 x 4 array of ints between 0 and 4, inclusive:\n\n    >>> np.random.randint(5, size=(2, 4))\n    array([[4, 0, 2, 1],\n        [3, 2, 2, 0]])\n    \"\"\"\n    if dtype is None:\n        dtype = 'int'\n    if ctx is None:\n        ctx = current_context()\n    if size is None:\n        size = ()\n    if high is None:\n        high = low\n        low = 0\n    return _npi.random_randint(low, high, shape=size, dtype=dtype, ctx=ctx, out=out)\n\n\ndef rand(*size, **kwargs):\n    r\"\"\"Random values in a given shape.\n\n    Create an array of the given shape and populate it with random\n    samples from a uniform distribution over [0, 1).\n    Parameters\n    ----------\n    d0, d1, ..., dn : int, optional\n        The dimensions of the returned array, should be all positive.\n        If no argument is given a single Python float is returned.\n    Returns\n    -------\n    out : _Symbol\n       Random values.\n    Examples\n    --------\n    >>> np.random.rand(3,2)\n    array([[ 0.14022471,  0.96360618],  #random\n           [ 0.37601032,  0.25528411],  #random\n           [ 0.49313049,  0.94909878]]) #random\n    \"\"\"\n    output_shape = ()\n    for s in size:\n        output_shape += (s,)\n    return uniform(0, 1, size=output_shape, **kwargs)\n\n\ndef uniform(low=0.0, high=1.0, size=None, dtype=None, ctx=None, out=None):\n    r\"\"\"Draw samples from a uniform distribution.\n\n    Samples are uniformly distributed over the half-open interval\n    ``[low, high)`` (includes low, but excludes high).  In other words,\n    any value within the given interval is equally likely to be drawn\n    by `uniform`.\n\n    Parameters\n    ----------\n    low : float, _Symbol, optional\n        Lower boundary of the output interval.  All values generated will be\n        greater than or equal to low.  The default value is 0.\n    high : float, _Symbol, optional\n        Upper boundary of the output interval.  All values generated will be\n        less than high.  The default value is 1.0.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a scalar tensor containing a single value is returned if\n        ``low`` and ``high`` are both scalars.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n    ctx : Context, optional\n        Device context of output. Default is current context.\n\n    Returns\n    -------\n    out : _Symbol\n        Drawn samples from the parameterized uniform distribution.\n    \"\"\"\n    from ._symbol import _Symbol as np_symbol\n    input_type = (isinstance(low, np_symbol), isinstance(high, np_symbol))\n    if ctx is None:\n        ctx = current_context()\n    if out is not None:\n        size = out.shape\n    if size == ():\n        size = None\n    if input_type == (True, True):\n        return _npi.uniform(low, high, low=None, high=None, size=size,\n                            ctx=ctx, dtype=dtype, out=out)\n    elif input_type == (False, True):\n        return _npi.uniform(high, low=low, high=None, size=size,\n                            ctx=ctx, dtype=dtype, out=out)\n    elif input_type == (True, False):\n        return _npi.uniform(low, low=None, high=high, size=size,\n                            ctx=ctx, dtype=dtype, out=out)\n    else:\n        return _npi.uniform(low=low, high=high, size=size,\n                            ctx=ctx, dtype=dtype, out=out)\n\n\ndef normal(loc=0.0, scale=1.0, size=None, dtype=None, ctx=None, out=None):\n    r\"\"\"Draw random samples from a normal (Gaussian) distribution.\n\n    Samples are distributed according to a normal distribution parametrized\n    by *loc* (mean) and *scale* (standard deviation).\n\n\n    Parameters\n    ----------\n    loc : float, optional\n        Mean (centre) of the distribution.\n    scale : float, optional\n        Standard deviation (spread or \"width\") of the distribution.\n    size : int or tuple of ints, optional\n        Output shape. If the given shape is, e.g., `(m, n, k)`, then `m * n * k`\n        samples are drawn. If size is `None` (default), a scalar tensor containing\n        a single value is returned if loc and scale are both scalars.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n    ctx : Context, optional\n        Device context of output. Default is current context.\n\n    Returns\n    -------\n    out : _Symbol (symbol representing `mxnet.numpy.ndarray` in computational graphs)\n        Drawn samples from the parameterized normal distribution.\n    \"\"\"\n    from ._symbol import _Symbol as np_symbol\n    input_type = (isinstance(loc, np_symbol), isinstance(scale, np_symbol))\n    if ctx is None:\n        ctx = current_context()\n    if size == ():\n        size = None\n    if input_type == (True, True):\n        return _npi.normal(loc, scale, loc=None, scale=None, size=size,\n                           ctx=ctx, dtype=dtype, out=out)\n    elif input_type == (False, True):\n        return _npi.normal(scale, loc=loc, scale=None, size=size,\n                           ctx=ctx, dtype=dtype, out=out)\n    elif input_type == (True, False):\n        return _npi.normal(loc, loc=None, scale=scale, size=size,\n                           ctx=ctx, dtype=dtype, out=out)\n    else:\n        return _npi.normal(loc=loc, scale=scale, size=size,\n                           ctx=ctx, dtype=dtype, out=out)\n\n\ndef lognormal(mean=0.0, sigma=1.0, size=None, dtype=None, ctx=None, out=None):\n    r\"\"\"Draw samples from a log-normal distribution.\n\n    Draw samples from a log-normal distribution with specified mean,\n    standard deviation, and array shape.  Note that the mean and standard\n    deviation are not the values for the distribution itself, but of the\n    underlying normal distribution it is derived from.\n\n    Parameters\n    ----------\n    mean : float, optional\n        Mean value of the underlying normal distribution. Default is 0.\n    sigma : float, optional\n        Standard deviation of the underlying normal distribution. Must be\n        non-negative. Default is 1.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``mean`` and ``sigma`` are both scalars.\n        Otherwise, ``np.broadcast(mean, sigma).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    ctx : Context, optional\n        Device context of output. Default is current context.\n\n    Returns\n    -------\n    out : _Symbol (symbol representing `mxnet.numpy.ndarray` in computational graphs)\n        Drawn samples from the parameterized lognormal distribution.\n    \"\"\"\n    from . import _symbol as _mx_np_symbol\n    return _mx_np_symbol.exp(normal(loc=mean, scale=sigma, size=size, dtype=dtype, ctx=ctx, out=out))\n\n\ndef logistic(loc=0.0, scale=1.0, size=None, ctx=None, out=None):\n    r\"\"\"Draw samples from a logistic distribution.\n\n    Samples are drawn from a logistic distribution with specified\n    parameters, loc (location or mean, also median), and scale (>0).\n\n    Parameters\n    ----------\n    loc : float, optional\n        Parameter of the distribution. Default is 0.\n    scale : float, optional\n        Parameter of the distribution. Must be non-negative.\n        Default is 1.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``loc`` and ``scale`` are both scalars.\n        Otherwise, ``np.broadcast(loc, scale).size`` samples are drawn.\n    ctx : Context, optional\n        Device context of output. Default is current context.\n\n    Returns\n    -------\n    out : _Symbol (symbol representing `mxnet.numpy.ndarray` in computational graphs)\n        Drawn samples from the parameterized logistic distribution.\n    \"\"\"\n    from ._symbol import _Symbol as np_symbol\n    input_type = (isinstance(loc, np_symbol), isinstance(scale, np_symbol))\n    if ctx is None:\n        ctx = current_context()\n    if size == ():\n        size = None\n    if input_type == (True, True):\n        return _npi.logistic(loc, scale, loc=None, scale=None, size=size,\n                             ctx=ctx, out=out)\n    elif input_type == (False, True):\n        return _npi.logistic(scale, loc=loc, scale=None, size=size,\n                             ctx=ctx, out=out)\n    elif input_type == (True, False):\n        return _npi.logistic(loc, loc=None, scale=scale, size=size,\n                             ctx=ctx, out=out)\n    else:\n        return _npi.logistic(loc=loc, scale=scale, size=size,\n                             ctx=ctx, out=out)\n\n\ndef gumbel(loc=0.0, scale=1.0, size=None, ctx=None, out=None):\n    r\"\"\"Draw samples from a Gumbel distribution.\n\n    Parameters\n    ----------\n    loc : float or array_like of floats, optional\n        The location of the mode of the distribution. Default is 0.\n    scale : float or array_like of floats, optional\n        The scale parameter of the distribution. Default is 1. Must be non-\n        negative.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``loc`` and ``scale`` are both scalars.\n        Otherwise, ``np.broadcast(loc, scale).size`` samples are drawn.\n    ctx : Context, optional\n        Device context of output. Default is current context.\n\n    Returns\n    -------\n    out : _Symbol (symbol representing `mxnet.numpy.ndarray` in computational graphs)\n        Drawn samples from the parameterized gumbel distribution.\n    \"\"\"\n    from ._symbol import _Symbol as np_symbol\n    input_type = (isinstance(loc, np_symbol), isinstance(scale, np_symbol))\n    if ctx is None:\n        ctx = current_context()\n    if size == ():\n        size = None\n    if input_type == (True, True):\n        return _npi.gumbel(loc, scale, loc=None, scale=None, size=size,\n                           ctx=ctx, out=out)\n    elif input_type == (False, True):\n        return _npi.gumbel(scale, loc=loc, scale=None, size=size,\n                           ctx=ctx, out=out)\n    elif input_type == (True, False):\n        return _npi.gumbel(loc, loc=None, scale=scale, size=size,\n                           ctx=ctx, out=out)\n    else:\n        return _npi.gumbel(loc=loc, scale=scale, size=size,\n                           ctx=ctx, out=out)\n\n\ndef choice(a, size=None, replace=True, p=None, ctx=None, out=None):\n    r\"\"\"Generates a random sample from a given 1-D array\n\n    Parameters\n    -----------\n    a : 1-D array-like or int\n        If an ndarray, a random sample is generated from its elements.\n        If an int, the random sample is generated as if a were np.arange(a)\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  Default is None, in which case a\n        single value is returned.\n    replace : boolean, optional\n        Whether the sample is with or without replacement\n    p : 1-D array-like, optional\n        The probabilities associated with each entry in a.\n        If not given the sample assumes a uniform distribution over all\n        entries in a.\n    ctx : Context, optional\n        Device context of output. Default is current context.\n\n    Returns\n    --------\n    samples : _Symbol\n        The generated random samples\n\n    Examples\n    ---------\n    Generate a uniform random sample from np.arange(5) of size 3:\n\n    >>> np.random.choice(5, 3)\n    array([0, 3, 4])\n    >>> #This is equivalent to np.random.randint(0,5,3)\n\n    Generate a non-uniform random sample from np.arange(5) of size 3:\n\n    >>> np.random.choice(5, 3, p=[0.1, 0, 0.3, 0.6, 0])\n    array([3, 3, 0])\n\n    Generate a uniform random sample from np.arange(5) of size 3 without\n    replacement:\n\n    >>> np.random.choice(5, 3, replace=False)\n    array([3,1,0])\n    >>> #This is equivalent to np.random.permutation(np.arange(5))[:3]\n\n    Generate a non-uniform random sample from np.arange(5) of size\n    3 without replacement:\n\n    >>> np.random.choice(5, 3, replace=False, p=[0.1, 0, 0.3, 0.6, 0])\n    array([2, 3, 0])\n    \"\"\"\n    from ._symbol import _Symbol as np_symbol\n    if ctx is None:\n        ctx = current_context()\n    if size == ():\n        size = None\n    if isinstance(a, np_symbol):\n        ctx = None\n        if p is None:\n            indices = _npi.choice(a, a=None, size=size,\n                                  replace=replace, ctx=ctx, weighted=False)\n            return _npi.take(a, indices)\n        else:\n            indices = _npi.choice(a, p, a=None, size=size,\n                                  replace=replace, ctx=ctx, weighted=True)\n            return _npi.take(a, indices)\n    else:\n        if p is None:\n            return _npi.choice(a=a, size=size, replace=replace, ctx=ctx, weighted=False, out=out)\n        else:\n            return _npi.choice(p, a=a, size=size, replace=replace, ctx=ctx, weighted=True, out=out)\n\n\ndef laplace(loc=0.0, scale=1.0, size=None, dtype=None, ctx=None, out=None):\n    r\"\"\"Draw random samples from a Laplace distribution.\n\n    Samples are distributed according to a Laplace distribution parametrized\n    by *loc* (mean) and *scale* (the exponential decay).\n\n    Parameters\n    ----------\n    loc : float, The position of the distribution peak.\n\n    scale : float, the exponential decay.\n\n    size : int or tuple of ints, optional. Output shape.\n        If the given shape is, e.g., (m, n, k), then m * n * k samples are drawn.\n        Default is None, in which case a single value is returned.\n\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    ctx : Context, optional\n        Device context of output. Default is current context.\n    out : ``ndarray``, optional\n        Store output to an existing ``ndarray``.\n\n    Returns\n    -------\n    out : _Symbol (symbol representing `mxnet.numpy.ndarray` in computational graphs)\n        Drawn samples from the parameterized Laplace distribution.\n    \"\"\"\n    from ._symbol import _Symbol as np_symbol\n    input_type = (isinstance(loc, np_symbol), isinstance(scale, np_symbol))\n    if dtype is None:\n        dtype = 'float32'\n    if ctx is None:\n        ctx = current_context()\n    if size == ():\n        size = None\n    if input_type == (True, True):\n        return _npi.laplace(loc, scale, loc=None, scale=None, size=size,\n                            ctx=ctx, dtype=dtype, out=out)\n    elif input_type == (False, True):\n        return _npi.laplace(scale, loc=loc, scale=None, size=size,\n                            ctx=ctx, dtype=dtype, out=out)\n    elif input_type == (True, False):\n        return _npi.laplace(loc, loc=None, scale=scale, size=size,\n                            ctx=ctx, dtype=dtype, out=out)\n    else:\n        return _npi.laplace(loc=loc, scale=scale, size=size,\n                            ctx=ctx, dtype=dtype, out=out)\n\n\ndef gamma(shape, scale=1.0, size=None, dtype=None, ctx=None, out=None):\n    \"\"\"Draw samples from a Gamma distribution.\n\n    Samples are drawn from a Gamma distribution with specified parameters,\n    `shape` (sometimes designated \"k\") and `scale` (sometimes designated\n    \"theta\"), where both parameters are > 0.\n\n    Parameters\n    ----------\n    shape : float or array_like of floats\n        The shape of the gamma distribution. Should be greater than zero.\n    scale : float or array_like of floats, optional\n        The scale of the gamma distribution. Should be greater than zero.\n        Default is equal to 1.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``shape`` and ``scale`` are both scalars.\n        Otherwise, ``np.broadcast(shape, scale).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n    ctx : Context, optional\n        Device context of output. Default is current context.\n\n    Returns\n    -------\n    out : _Symbol\n        Drawn samples from the parameterized gamma distribution.\n\n    The Gamma distribution is often used to model the times to failure of\n    electronic components, and arises naturally in processes for which the\n    waiting times between Poisson distributed events are relevant.\n    \"\"\"\n    from ._symbol import _Symbol as np_symbol\n    input_type = (isinstance(shape, np_symbol), isinstance(scale, np_symbol))\n    if ctx is None:\n        ctx = current_context()\n    if out is not None:\n        size = out.shape\n    if size == ():\n        size = None\n    if input_type == (True, True):\n        return _npi.gamma(shape, scale, shape=None, scale=None, size=size,\n                          ctx=ctx, dtype=dtype, out=out)\n    elif input_type == (False, True):\n        return _npi.gamma(scale, shape=shape, scale=None, size=size,\n                          ctx=ctx, dtype=dtype, out=out)\n    elif input_type == (True, False):\n        return _npi.gamma(shape, shape=None, scale=scale, size=size,\n                          ctx=ctx, dtype=dtype, out=out)\n    else:\n        return _npi.gamma(shape=shape, scale=scale, size=size,\n                          ctx=ctx, dtype=dtype, out=out)\n\n    raise ValueError(\"Distribution parameters must be either _Symbol or numbers\")\n\n\ndef rayleigh(scale=0.0, size=None, ctx=None, out=None):\n    r\"\"\"Draw samples from a Rayleigh distribution.\n    The :math:`\\chi` and Weibull distributions are generalizations of the\n    Rayleigh.\n    Parameters\n    ----------\n    scale : float or _Symbol\n        Scale, also equals the mode. Must be non-negative. Default is 1.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``scale`` is a scalar.  Otherwise,\n        ``np.array(scale).size`` samples are drawn.\n    ctx : Context, optional\n        Device context of output. Default is current context.\n    Returns\n    -------\n    out : _Symbol\n        Drawn samples from the parameterized Rayleigh distribution.\n    \"\"\"\n    from ..numpy import _Symbol as np_symbol\n    tensor_type_name = np_symbol\n    if ctx is None:\n        ctx = current_context()\n    if size == ():\n        size = None\n    is_tensor = isinstance(scale, tensor_type_name)\n    if is_tensor:\n        return _npi.rayleigh(scale, scale=None, size=size, ctx=ctx, out=out)\n    else:\n        return _npi.rayleigh(scale=scale, size=size, ctx=ctx, out=out)\n\n\ndef beta(a, b, size=None, dtype=None, ctx=None):\n    r\"\"\"Draw samples from a Beta distribution.\n\n    The Beta distribution is a special case of the Dirichlet distribution,\n    and is related to the Gamma distribution.  It has the probability\n    distribution function\n\n    .. math:: f(x; a,b) = \\frac{1}{B(\\alpha, \\beta)} x^{\\alpha - 1}\n                                                     (1 - x)^{\\beta - 1},\n\n    where the normalisation, B, is the beta function,\n\n    .. math:: B(\\alpha, \\beta) = \\int_0^1 t^{\\alpha - 1}\n                                 (1 - t)^{\\beta - 1} dt.\n\n    It is often seen in Bayesian inference and order statistics.\n\n    Parameters\n    ----------\n    a : float or _Symbol of floats\n        Alpha, positive (>0).\n    b : float or _Symbol of floats\n        Beta, positive (>0).\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``a`` and ``b`` are both scalars.\n        Otherwise, ``np.broadcast(a, b).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n        Dtype 'float32' or 'float64' is strongly recommended,\n        since lower precision might lead to out of range issue.\n    ctx : Context, optional\n        Device context of output. Default is current context.\n\n    Notes\n    -------\n    To use this  operator with scalars as input, please run ``npx.set_np()`` first.\n\n    Returns\n    -------\n    out : _Symbol\n        Drawn samples from the parameterized beta distribution.\n    \"\"\"\n    if dtype is None:\n        dtype = np.float64 if is_np_default_dtype() else np.float32\n    if ctx is None:\n        ctx = current_context()\n    if size == ():\n        size = None\n    # use fp64 to prevent precision loss\n    X = gamma(a, 1, size=size, dtype='float64', ctx=ctx)\n    Y = gamma(b, 1, size=size, dtype='float64', ctx=ctx)\n    out = X/(X + Y)\n    return out.astype(dtype)\n\n\ndef f(dfnum, dfden, size=None, ctx=None):\n    r\"\"\"Draw samples from an F distribution.\n\n    Samples are drawn from an F distribution with specified parameters,\n    `dfnum` (degrees of freedom in numerator) and `dfden` (degrees of\n    freedom in denominator), where both parameters must be greater than\n    zero.\n\n    The random variate of the F distribution (also known as the\n    Fisher distribution) is a continuous probability distribution\n    that arises in ANOVA tests, and is the ratio of two chi-square\n    variates.\n\n    Parameters\n    ----------\n    dfnum : float or _Symbol of floats\n        Degrees of freedom in numerator, must be > 0.\n    dfden : float or _Symbol of float\n        Degrees of freedom in denominator, must be > 0.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``dfnum`` and ``dfden`` are both scalars.\n        Otherwise, ``np.broadcast(dfnum, dfden).size`` samples are drawn.\n    ctx : Context, optional\n        Device context of output. Default is current context.\n\n    Returns\n    -------\n    out : _Symbol\n        Drawn samples from the parameterized Fisher distribution.\n    \"\"\"\n    X = chisquare(df=dfnum, size=size, ctx=ctx)\n    Y = chisquare(df=dfden, size=size, ctx=ctx)\n    return (X * dfden) / (Y * dfnum)\n\n\ndef chisquare(df, size=None, dtype=None, ctx=None):\n    r\"\"\"\n    chisquare(df, size=None, dtype=None, ctx=None)\n\n    Draw samples from a chi-square distribution.\n\n    When `df` independent random variables, each with standard normal\n    distributions (mean 0, variance 1), are squared and summed, the\n    resulting distribution is chi-square (see Notes).  This distribution\n    is often used in hypothesis testing.\n\n    Parameters\n    ----------\n    df : float or _Symbol of floats\n         Number of degrees of freedom, must be > 0.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``df`` is a scalar.  Otherwise,\n        ``np.array(df).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples.\n        When npx.is_np_default_dtype() returns False, default dtype is float32;\n        When npx.is_np_default_dtype() returns True, default dtype is float64.\n    ctx : Context, optional\n        Device context of output. Default is current context.\n\n    Returns\n    -------\n    out : _Symbol\n        Drawn samples from the parameterized chi-square distribution.\n\n    Raises\n    ------\n    ValueError\n        When `df` <= 0 or when an inappropriate `size`\n        is given.\n\n    Notes\n    -----\n    The variable obtained by summing the squares of `df` independent,\n    standard normally distributed random variables:\n\n    .. math:: Q = \\sum_{i=0}^{\\mathtt{df}} X^2_i\n\n    is chi-square distributed, denoted\n\n    .. math:: Q \\sim \\chi^2_k.\n\n    The probability density function of the chi-squared distribution is\n\n    .. math:: p(x) = \\frac{(1/2)^{k/2}}{\\Gamma(k/2)}\n                     x^{k/2 - 1} e^{-x/2},\n\n    where :math:`\\Gamma` is the gamma function,\n\n    .. math:: \\Gamma(x) = \\int_0^{-\\infty} t^{x - 1} e^{-t} dt.\n\n    References\n    ----------\n    .. [1] NIST \"Engineering Statistics Handbook\"\n           https://www.itl.nist.gov/div898/handbook/eda/section3/eda3666.htm\n\n    \"\"\"\n    if dtype is None:\n        dtype = np.float64 if is_np_default_dtype() else np.float32\n    if ctx is None:\n        ctx = current_context()\n    if size == ():\n        size = None\n    return gamma(df/2, 2, size=size, dtype=dtype, ctx=ctx)\n\n\ndef exponential(scale=1.0, size=None, ctx=None, out=None):\n    r\"\"\"Draw samples from an exponential distribution.\n\n    Parameters\n    ----------\n    scale : float or array_like of floats\n        The scale parameter, :math:`\\beta = 1/\\lambda`. Must be\n        non-negative.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``scale`` is a scalar.  Otherwise,\n        ``np.array(scale).size`` samples are drawn.\n    ctx : Context, optional\n        Device context of output. Default is current context.\n\n    Returns\n    -------\n    out : _Symbol (symbol representing `mxnet.numpy.ndarray` in computational graphs)\n        Drawn samples from the parameterized exponential distribution.\n    \"\"\"\n    from ..numpy import _Symbol as np_symbol\n    tensor_type_name = np_symbol\n    if ctx is None:\n        ctx = current_context()\n    if size == ():\n        size = None\n    is_tensor = isinstance(scale, tensor_type_name)\n    if is_tensor:\n        return _npi.exponential(scale, scale=None, size=size,\n                                ctx=ctx, out=out)\n    else:\n        return _npi.exponential(scale=scale, size=size, ctx=ctx, out=out)\n\n\ndef weibull(a, size=None, ctx=None, out=None):\n    r\"\"\"Draw samples from a 1-parameter Weibull distribution with given parameter a\n    via inversion.\n\n    Parameters\n    ----------\n    a : float or array_like of floats\n        Shape of the distribution. Must be non-negative.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``a`` is a scalar. Otherwise,\n        ``np.array(a).size`` samples are drawn.\n\n    Returns\n    -------\n    out : _Symbol\n        Drawn samples from the 1-parameter Weibull distribution.\n\n    Examples\n    --------\n    >>> np.random.weibull(a=5)\n    array(0.9553641)\n\n    >>> np.random.weibull(a=5, size=[2,3])\n    array([[1.0466299 , 1.1320982 , 0.98415005],\n          [1.1430776 , 0.9532727 , 1.1344457 ]])\n\n    >>> np.random.weibull(a=np.array([2,3])\n    array([0.98843634, 1.0125613 ])\n\n    The Weibull distribution is one of a class of Generalized Extreme\n    Value (GEV) distributions. This class includes the Gumbel and Frechet\n    distributions.\n\n    The probability density for the Weibull distribution is\n    f(x) = \\frac{a}{\\lambda}(\\frac{x}{\\lambda})^{a-1}e^{-(x/\\lambda)^a},\n    where a is the shape and \\lambda the scale. The generated 1-parameter Weibull\n    sample has the scale parameter \\lambda = 1.\n\n    The Weibull distribution is commonly used in reliability engineering to\n    model time to failure, in modeling particle sizes, in information retrieval\n    to model dwell time on pages, in quantitative finance to model risk etc.\n    \"\"\"\n    from ..numpy import _Symbol as np_symbol\n    tensor_type_name = np_symbol\n    if ctx is None:\n        ctx = current_context()\n    if size == ():\n        size = None\n    is_tensor = isinstance(a, tensor_type_name)\n    if is_tensor:\n        return _npi.weibull(a, a=None, size=size, ctx=ctx, out=out)\n    else:\n        return _npi.weibull(a=a, size=size, ctx=ctx, out=out)\n\n\ndef pareto(a, size=None, ctx=None, out=None):\n    r\"\"\"Draw samples from a Pareto II or Lomax distribution with specified shape a.\n\n    Parameters\n    ----------\n    a : float or array_like of floats\n            Shape of the distribution. Must be > 0.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``a`` is a scalar. Otherwise,\n        ``np.array(a).size`` samples are drawn.\n\n    Returns\n    -------\n    out : _Symbol\n        Drawn samples from the Pareto distribution.\n\n    Examples\n    --------\n    >>> np.random.pareto(a=5)\n    array(0.12749612)\n    >>> mx.numpy.random.pareto(a=5, size=[2,3])\n    array([[0.06933999, 0.0344373 , 0.10654891],\n            [0.0311172 , 0.12911797, 0.03370714]])\n    >>> np.random.pareto(a=np.array([2,3])\n    array([0.26636696, 0.15685666])\n\n    The probability density for the Pareto distribution is f(x) = \\frac{am^a}{x^{a+1}}\n    where a is the shape and m the scale. Here m is assumed 1. The Pareto distribution\n    is a power law distribution. Pareto created it to describe the wealth in the economy.\n    \"\"\"\n    from ..numpy import _Symbol as np_symbol\n    tensor_type_name = np_symbol\n    if ctx is None:\n        ctx = current_context()\n    if size == ():\n        size = None\n    is_tensor = isinstance(a, tensor_type_name)\n    if is_tensor:\n        return _npi.pareto(a, a=None, size=size, ctx=ctx, out=out)\n    else:\n        return _npi.pareto(a=a, size=size, ctx=ctx, out=out)\n\n\ndef power(a, size=None, ctx=None, out=None):\n    r\"\"\"Draw samples in [0, 1] from a power distribution with given parameter a.\n\n    Parameters\n    ----------\n    a : float or array_like of floats\n        Shape of the distribution. Must be > 0.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  If size is ``None`` (default),\n        a single value is returned if ``a`` is a scalar. Otherwise,\n        ``np.array(a).size`` samples are drawn.\n\n    Returns\n    -------\n    out : _Symbol\n        Drawn samples from the power distribution.\n\n    Examples\n    --------\n    >>> np.random.power(a=5)\n    array(0.8602478)\n    >>> np.random.power(a=5, size=[2,3])\n    array([[0.988391  , 0.5153122 , 0.9383134 ],\n           [0.9078098 , 0.87819266, 0.730635]])\n    >>> np.random.power(a=np.array([2,3])\n    array([0.7499419 , 0.88894516])\n\n    The probability density function is f(x; a) = ax^{a-1}, 0 \\le x \\le 1, a>0.\n    The power distribution is just the inverse of the Pareto distribution and\n    a special case of the Beta distribution.\n    \"\"\"\n    from ..numpy import _Symbol as np_symbol\n    tensor_type_name = np_symbol\n    if ctx is None:\n        ctx = current_context()\n    if size == ():\n        size = None\n    is_tensor = isinstance(a, tensor_type_name)\n    if is_tensor:\n        return _npi.powerd(a, a=None, size=size, ctx=ctx, out=out)\n    else:\n        return _npi.powerd(a=a, size=size, ctx=ctx, out=out)\n\n\ndef multivariate_normal(mean, cov, size=None, check_valid=None, tol=None):\n    \"\"\"\n    multivariate_normal(mean, cov, size=None, check_valid=None, tol=None)\n\n    Draw random samples from a multivariate normal distribution.\n\n    The multivariate normal, multinormal or Gaussian distribution is a\n    generalization of the one-dimensional normal distribution to higher\n    dimensions.  Such a distribution is specified by its mean and\n    covariance matrix.  These parameters are analogous to the mean\n    (average or \"center\") and variance (standard deviation, or \"width,\"\n    squared) of the one-dimensional normal distribution.\n\n    This operator is a little different from the one in official NumPy.\n    The official NumPy operator only accepts 1-D ndarray as mean and 2-D ndarray as cov,\n    whereas the operator in MXNet np supports batch operation and auto-broadcasting.\n\n    Both `mean` and `cov` may have any number of leading dimensions, which correspond\n    to a batch shape. They are not necessarily assumed to have the same batch shape,\n    just ones which can be broadcasted.\n\n    Parameters\n    ----------\n    mean : K-D _Symbol, of shape (..., N)\n        Mean of the N-dimensional distribution.\n    cov : (K+1)-D _Symbol, of shape (..., N, N)\n        Covariance matrix of the distribution. The last two dimensions must be symmetric and\n        positive-semidefinite for proper sampling.\n    size : int or tuple of ints, optional\n        Given a shape of, for example, ``(m,n,k)``,\n        ``m*n*k`` identically distributed batchs of samples are\n        generated, and packed in an `m`-by-`n`-by-`k` arrangement.\n        If no shape is specified, a batch of (`N`-D) sample is returned.\n    check_valid : { 'warn', 'raise', 'ignore' }, optional\n        Behavior when the covariance matrix is not positive semidefinite.\n        (Not supported)\n    tol : float, optional\n        Tolerance when checking the singular values in covariance matrix.\n        cov is cast to double before the check.\n        (Not supported)\n\n    Returns\n    -------\n    out : _Symbol\n        The input shape of `mean` and `cov` should satisfy the requirements of broadcasting.\n        If the parameter `size` is not provided,\n        the output shape is ``np.broadcast(mean.shape, cov.shape[:-1])``.\n        Otherwise, the output shape is ``size + np.broadcast(mean.shape, cov.shape[:-1])``\n\n    Examples\n    --------\n    >>> mean = np.array([1, 2])\n    >>> cov = np.array([[1, 0], [0, 1]])\n    >>> x = np.random.multivariate_normal(mean, cov, (3, 3))\n    >>> x.shape\n    (3, 3, 2)\n\n    The following is probably true, given that 0.6 is roughly twice the\n    standard deviation:\n\n    >>> list((x[0,0,:] - mean) < 0.6)\n    [True, True] # random\n\n    # Performs autobroadcasting when the batch shape of\n    # `mean` and `cov` is different but compatible.\n\n    >>> mean = np.zeros((3,2)) # shape (3, 2)\n    >>> cov = np.array([[1, 0], [0, 100]]) # shape (2, 2)\n    >>> x = np.random.multivariate_normal(mean, cov)\n    >>> x\n    array([[-1.6115597 , -8.726251  ],\n           [ 2.2425299 ,  2.8104177 ],\n           [ 0.36229908, -8.386591  ]])\n    \"\"\"\n    if check_valid is not None:\n        raise NotImplementedError('Parameter `check_valid` is not supported')\n    if tol is not None:\n        raise NotImplementedError('Parameter `tol` is not supported')\n    return _npi.mvn_fallback(mean, cov, size=size)\n\n\ndef shuffle(x):\n    \"\"\"\n    Modify a sequence in-place by shuffling its contents.\n\n    This function only shuffles the array along the first axis of a\n    multi-dimensional array. The order of sub-arrays is changed but\n    their contents remain the same.\n\n    Parameters\n    ----------\n    x: _Symbol\n        The array or list to be shuffled.\n\n    Returns\n    -------\n    None\n\n    Examples\n    --------\n    >>> arr = np.arange(10)\n    >>> np.random.shuffle(arr)\n    >>> arr\n    array([5., 1., 0., 6., 7., 3., 9., 8., 4., 2.])  # random\n\n    Multi-dimensional arrays are only shuffled along the first axis:\n\n    >>> arr = np.arange(9).reshape((3, 3))\n    >>> np.random.shuffle(arr)\n    >>> arr\n    array([[6., 7., 8.], # random\n           [3., 4., 5.],\n           [0., 1., 2.]])\n    \"\"\"\n    _npi.shuffle(x, out=x)\n"
  },
  {
    "path": "python/mxnet/symbol/numpy_extension/__init__.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Module for the ops not belonging to the official numpy package.\"\"\"\n\nfrom . import _op\nfrom . import image\nfrom . import random\nfrom . import _register\nfrom ._op import *  # pylint: disable=wildcard-import\n\n__all__ = _op.__all__\n"
  },
  {
    "path": "python/mxnet/symbol/numpy_extension/_op.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for operators not belonging to the official numpy package\nused in Gluon APIs dispatched by F=symbol module.\"\"\"\n\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/symbol/numpy_extension/_register.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Registering numpy_extension ops.\"\"\"\n\nfrom ...base import _init_np_op_module\nfrom ..register import _make_symbol_function\n\n_init_np_op_module(root_module_name='mxnet', np_module_name='numpy_extension',\n                   mx_module_name='symbol', make_op_func=_make_symbol_function)\n"
  },
  {
    "path": "python/mxnet/symbol/numpy_extension/image.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Image pre-processing operators.\"\"\"\n\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/symbol/numpy_extension/random.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Namespace for operators used in Gluon dispatched by F=symbol.\"\"\"\n\nfrom ...context import current_context\nfrom ..numpy import _internal as _npi\n\n__all__ = ['bernoulli', 'normal_n', 'uniform_n']\n\n\ndef bernoulli(prob=None, logit=None, size=None, dtype=None, ctx=None, out=None):\n    \"\"\"Creates a Bernoulli distribution parameterized by :attr:`prob`\n    or :attr:`logit` (but not both).\n\n    Samples are binary (0 or 1). They take the value `1` with probability `p`\n    and `0` with probability `1 - p`.\n\n    Parameters\n    ----------\n    prob : float, ndarray\n        The probability of sampling '1'.\n        Only one of prob or logit should be passed in.\n    logit : float, ndarray\n        The log-odds of sampling '1'.\n        Only one of prob or logit should be passed in.\n    size : int or tuple of ints, optional\n        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k`` samples are drawn.  Default is None, in which case a\n        single value is returned.\n    dtype : dtype, optional\n        Desired dtype of the result. All dtypes are determined by their\n        name, i.e., 'int64', 'int', etc, so byteorder is not available\n        and a specific precision may have different C types depending\n        on the platform. The default value is 'np.float32'.\n    ctx : Context, optional\n        Device context of output. Default is current context.\n    out : symbol, optional\n        The output symbol (default is `None`).\n\n    Returns\n    -------\n    out : _Symbol\n        Drawn samples from the parameterized bernoulli distribution.\n\n    Examples\n    --------\n    >>> prob = np.random.uniform(size=(4,4))\n    >>> logit = np.log(prob) - np.log(1 - prob)\n    >>> npx.random.bernoulli(logit=logit)\n    array([[0., 1., 1., 1.],\n        [0., 1., 1., 1.],\n        [0., 1., 0., 0.],\n        [1., 0., 1., 0.]])\n\n    >>> npx.random.bernoulli(prob=prob)\n    array([[0., 1., 0., 1.],\n        [1., 1., 1., 1.],\n        [1., 1., 1., 0.],\n        [1., 0., 1., 0.]])\n    \"\"\"\n    from ..numpy import _Symbol as np_symbol\n    tensor_type_name = np_symbol\n    if (prob is None) == (logit is None):\n        raise ValueError(\n            \"Either `prob` or `logit` must be specified, but not both. \" +\n            \"Received prob={}, logit={}\".format(prob, logit))\n    if ctx is None:\n        ctx = current_context()\n    if size == ():\n        size = None\n    if prob is not None:\n        is_tensor = isinstance(prob, tensor_type_name)\n        if is_tensor:\n            return _npi.bernoulli(prob, prob=None, logit=None, is_logit=False,\n                                  size=size, ctx=ctx, dtype=dtype, out=out)\n        else:\n            return _npi.bernoulli(prob=prob, logit=None, is_logit=False,\n                                  size=size, ctx=ctx, dtype=dtype, out=out)\n    else:\n        is_tensor = isinstance(logit, tensor_type_name)\n        if is_tensor:\n            return _npi.bernoulli(logit, prob=None, logit=None, is_logit=True,\n                                  size=size, ctx=ctx, dtype=dtype, out=out)\n        else:\n            return _npi.bernoulli(prob=None, logit=logit, is_logit=True,\n                                  size=size, ctx=ctx, dtype=dtype, out=out)\n\n\ndef uniform_n(low=0.0, high=1.0, batch_shape=None, dtype=None, ctx=None):\n    r\"\"\"Draw samples from a uniform distribution.\n\n    Samples are uniformly distributed over the half-open interval\n    ``[low, high)`` (includes low, but excludes high).  In other words,\n    any value within the given interval is equally likely to be drawn\n    by `uniform`.\n\n    Parameters\n    ----------\n    low : float, ndarray, optional\n        Lower boundary of the output interval.  All values generated will be\n        greater than or equal to low.  The default value is 0.\n    high : float, ndarray, optional\n        Upper boundary of the output interval.  All values generated will be\n        less than high.  The default value is 1.0.\n    shape : int or tuple of ints, optional\n        Batch shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k * broadcast(low, high).size`` samples are drawn.\n        If size is ``None`` (default),\n        a scalar tensor containing a single value is returned if\n        ``low`` and ``high`` are both scalars. Otherwise,\n        ``np.broadcast(low, high).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    ctx : Context, optional\n        Device context of output. Default is current context.\n\n    Returns\n    -------\n    out : ndarray\n        Drawn samples from the parameterized uniform distribution.\n\n    See Also\n    --------\n    randint : Discrete uniform distribution, yielding integers.\n    rand : Convenience function that accepts dimensions as input, e.g.,\n           ``rand(2,2)`` would generate a 2-by-2 array of floats,\n           uniformly distributed over ``[0, 1)``.\n\n    Notes\n    -----\n    The probability density function of the uniform distribution is\n\n    .. math:: p(x) = \\frac{1}{b - a}\n\n    anywhere within the interval ``[a, b)``, and zero elsewhere.\n\n    When ``high`` == ``low``, values of ``low`` will be returned.\n    If ``high`` < ``low``, the results are officially undefined\n    and may eventually raise an error, i.e. do not rely on this\n    function to behave when passed arguments satisfying that\n    inequality condition.\n    \"\"\"\n    from ..numpy import _Symbol as np_symbol\n    input_type = (isinstance(low, np_symbol), isinstance(high, np_symbol))\n    if dtype is None:\n        dtype = 'float32'\n    if ctx is None:\n        ctx = current_context()\n    if batch_shape == ():\n        batch_shape = None\n    else:\n        if isinstance(batch_shape, int):\n            batch_shape = (batch_shape,)\n        batch_shape = (-2,) + batch_shape\n    if input_type == (True, True):\n        return _npi.uniform(low, high, low=None, high=None, size=batch_shape,\n                            ctx=ctx, dtype=dtype)\n    elif input_type == (False, True):\n        return _npi.uniform(high, low=low, high=None, size=batch_shape,\n                            ctx=ctx, dtype=dtype)\n    elif input_type == (True, False):\n        return _npi.uniform(low, low=None, high=high, size=batch_shape,\n                            ctx=ctx, dtype=dtype)\n    else:\n        return _npi.uniform(low=low, high=high, size=batch_shape,\n                            ctx=ctx, dtype=dtype)\n\n\ndef normal_n(loc=0.0, scale=1.0, batch_shape=None, dtype=None, ctx=None):\n    r\"\"\"Draw random samples from a normal (Gaussian) distribution.\n\n    Samples are distributed according to a normal distribution parametrized\n    by *loc* (mean) and *scale* (standard deviation).\n\n\n    Parameters\n    ----------\n    loc : float, optional\n        Mean (centre) of the distribution.\n    scale : float, optional\n        Standard deviation (spread or \"width\") of the distribution.\n    shape : int or tuple of ints, optional\n        Batch shape.  If the given shape is, e.g., ``(m, n, k)``, then\n        ``m * n * k * broadcast(low, high).size`` samples are drawn.\n        If size is ``None`` (default),\n        a scalar tensor containing a single value is returned if\n        ``low`` and ``high`` are both scalars. Otherwise,\n        ``np.broadcast(loc, scale).size`` samples are drawn.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    ctx : Context, optional\n        Device context of output, default is current context.\n\n    Returns\n    -------\n    out : _Symbol\n        Drawn samples from the parameterized normal distribution.\n\n    Notes\n    -----\n    The probability density for the Gaussian distribution is\n\n    .. math:: p(x) = \\frac{1}{\\sqrt{ 2 \\pi \\sigma^2 }}\n                     e^{ - \\frac{ (x - \\mu)^2 } {2 \\sigma^2} },\n\n    where :math:`\\mu` is the mean and :math:`\\sigma` the standard\n    deviation. The square of the standard deviation, :math:`\\sigma^2`,\n    is called the variance.\n\n    The function has its peak at the mean, and its \"spread\" increases with\n    the standard deviation (the function reaches 0.607 times its maximum at\n    :math:`x + \\sigma` and :math:`x - \\sigma` [2]_).  This implies that\n    `numpy.random.normal` is more likely to return samples lying close to\n    the mean, rather than those far away.\n\n    References\n    ----------\n    .. [1] Wikipedia, \"Normal distribution\",\n           https://en.wikipedia.org/wiki/Normal_distribution\n    .. [2] P. R. Peebles Jr., \"Central Limit Theorem\" in \"Probability,\n           Random Variables and Random Signal Principles\", 4th ed., 2001,\n           pp. 51, 51, 125.\n\n    Examples\n    --------\n    >>> mu, sigma = 0, 0.1 # mean and standard deviation\n    >>> s = np.random.normal(mu, sigma, 1000)\n\n    Verify the mean and the variance:\n\n    >>> np.abs(mu - np.mean(s)) < 0.01\n    array(True)\n    \"\"\"\n    from ..numpy import _Symbol as np_symbol\n    input_type = (isinstance(loc, np_symbol), isinstance(scale, np_symbol))\n    if dtype is None:\n        dtype = 'float32'\n    if ctx is None:\n        ctx = current_context()\n    if batch_shape == ():\n        batch_shape = None\n    else:\n        if isinstance(batch_shape, int):\n            batch_shape = (batch_shape,)\n        batch_shape = (-2,) + batch_shape\n    if input_type == (True, True):\n        return _npi.normal(loc, scale, loc=None, scale=None, size=batch_shape,\n                           ctx=ctx, dtype=dtype)\n    elif input_type == (False, True):\n        return _npi.normal(scale, loc=loc, scale=None, size=batch_shape,\n                           ctx=ctx, dtype=dtype)\n    elif input_type == (True, False):\n        return _npi.normal(loc, loc=None, scale=scale, size=batch_shape,\n                           ctx=ctx, dtype=dtype)\n    else:\n        return _npi.normal(loc=loc, scale=scale, size=batch_shape,\n                           ctx=ctx, dtype=dtype)\n"
  },
  {
    "path": "python/mxnet/symbol/op.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import, unused-wildcard-import, redefined-builtin\n\"\"\"Backend ops in mxnet.symbol namespace.\"\"\"\ntry:\n    from .gen_op import *\nexcept ImportError:\n    pass\n\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/symbol/random.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Random distribution generator Symbol API of MXNet.\"\"\"\n\nfrom ..base import numeric_types, _Null\nfrom . import _internal\nfrom .symbol import Symbol\n\n\n__all__ = ['uniform', 'normal', 'randn', 'poisson', 'exponential', 'gamma', 'categorical', 'multinomial',\n           'binomial', 'negative_binomial', 'generalized_negative_binomial', 'shuffle', 'randint']\n\n\ndef _random_helper(random, sampler, params, shape, dtype, kwargs):\n    \"\"\"Helper function for random generators.\"\"\"\n    if isinstance(params[0], Symbol):\n        for i in params[1:]:\n            assert isinstance(i, Symbol), \\\n                \"Distribution parameters must all have the same type, but got \" \\\n                f\"both {type(params[0])} and {type(i)}.\"\n        return sampler(*params, shape=shape, dtype=dtype, **kwargs)\n    elif isinstance(params[0], numeric_types):\n        for i in params[1:]:\n            assert isinstance(i, numeric_types), \\\n                \"Distribution parameters must all have the same type, but got \" \\\n                f\"both {type(params[0])} and {type(i)}.\"\n        return random(*params, shape=shape, dtype=dtype, **kwargs)\n\n    raise ValueError(\"Distribution parameters must be either Symbol or numbers, \"\n                     f\"but got {type(params[0])}.\")\n\n\ndef uniform(low=0, high=1, shape=_Null, dtype=_Null, **kwargs):\n    \"\"\"Draw random samples from a uniform distribution.\n\n    Samples are uniformly distributed over the half-open interval *[low, high)*\n    (includes *low*, but excludes *high*).\n\n    Parameters\n    ----------\n    low : float or Symbol, optional\n        Lower boundary of the output interval. All values generated will be\n        greater than or equal to low. The default value is 0.\n    high : float or Symbol, optional\n        Upper boundary of the output interval. All values generated will be\n        less than high. The default value is 1.0.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `low` and\n        `high` are scalars, output shape will be `(m, n)`. If `low` and `high`\n        are Symbols with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each `[low, high)` pair.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n\n    Returns\n    -------\n    Symbol\n        If input `shape` has dimensions, e.g., `(m, n)`, and `low` and `high` are\n        scalars, returned Symbol will resolve to shape `(m, n)`. If `low` and `high`\n        are Symbols with shape, e.g., `(x, y)`, returned Symbol will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each `[low, high)` pair.\n    \"\"\"\n    return _random_helper(_internal._random_uniform, _internal._sample_uniform,\n                          [low, high], shape, dtype, kwargs)\n\n\ndef normal(loc=0, scale=1, shape=_Null, dtype=_Null, **kwargs):\n    \"\"\"Draw random samples from a normal (Gaussian) distribution.\n\n    Samples are distributed according to a normal distribution parametrized\n    by *loc* (mean) and *scale* (standard deviation).\n\n\n    Parameters\n    ----------\n    loc : float or Symbol, optional\n        Mean (centre) of the distribution.\n    scale : float or Symbol, optional\n        Standard deviation (spread or width) of the distribution.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `loc` and\n        `scale` are scalars, output shape will be `(m, n)`. If `loc` and `scale`\n        are Symbols with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each `[loc, scale)` pair.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n\n    Returns\n    -------\n    Symbol\n        If input `shape` has dimensions, e.g., `(m, n)`, and `loc` and\n        `scale` are scalars, returned Symbol will resolve to shape `(m, n)`.\n        If `loc` and `scale` are Symbols with shape, e.g., `(x, y)`, returned\n        Symbol will resolve to shape `(x, y, m, n)`, where `m*n` samples are drawn\n        for each `[loc, scale)` pair.\n    \"\"\"\n    return _random_helper(_internal._random_normal, _internal._sample_normal,\n                          [loc, scale], shape, dtype, kwargs)\n\n\ndef randn(*shape, **kwargs):\n    \"\"\"Draw random samples from a normal (Gaussian) distribution.\n\n    Samples are distributed according to a normal distribution parametrized\n    by *loc* (mean) and *scale* (standard deviation).\n\n\n    Parameters\n    ----------\n    loc : float or Symbol, optional\n        Mean (centre) of the distribution.\n    scale : float or Symbol, optional\n        Standard deviation (spread or width) of the distribution.\n    shape : int or tuple of ints\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `loc` and\n        `scale` are scalars, output shape will be `(m, n)`. If `loc` and `scale`\n        are NDArrays with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each `[loc, scale)` pair.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n    \"\"\"\n    loc = kwargs.pop('loc', 0)\n    scale = kwargs.pop('scale', 1)\n    dtype = kwargs.pop('dtype', _Null)\n    assert isinstance(loc, (int, float, Symbol))\n    assert isinstance(scale, (int, float, Symbol))\n    return _random_helper(_internal._random_normal, _internal._sample_normal,\n                          [loc, scale], shape, dtype, kwargs)\n\n\ndef poisson(lam=1, shape=_Null, dtype=_Null, **kwargs):\n    \"\"\"Draw random samples from a Poisson distribution.\n\n    Samples are distributed according to a Poisson distribution parametrized\n    by *lambda* (rate). Samples will always be returned as a floating point data type.\n\n    Parameters\n    ----------\n    lam : float or Symbol, optional\n        Expectation of interval, should be >= 0.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `lam` is\n        a scalar, output shape will be `(m, n)`. If `lam`\n        is an Symbol with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each entry in `lam`.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n\n    Returns\n    -------\n    Symbol\n        If input `shape` has dimensions, e.g., `(m, n)`, and `lam` is\n        a scalar, output shape will be `(m, n)`. If `lam`\n        is an Symbol with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each entry in `lam`.\n    \"\"\"\n    return _random_helper(_internal._random_poisson, _internal._sample_poisson,\n                          [lam], shape, dtype, kwargs)\n\n\ndef exponential(scale=1, shape=_Null, dtype=_Null, **kwargs):\n    r\"\"\"Draw samples from an exponential distribution.\n\n    Its probability density function is\n\n    .. math:: f(x; \\frac{1}{\\beta}) = \\frac{1}{\\beta} \\exp(-\\frac{x}{\\beta}),\n\n    for x > 0 and 0 elsewhere. \\beta is the scale parameter, which is the\n    inverse of the rate parameter \\lambda = 1/\\beta.\n\n    Parameters\n    ----------\n    scale : float or Symbol, optional\n        The scale parameter, \\beta = 1/\\lambda.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `scale` is\n        a scalar, output shape will be `(m, n)`. If `scale`\n        is an Symbol with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each entry in `scale`.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n\n    Returns\n    -------\n    Symbol\n        If input `shape` has dimensions, e.g., `(m, n)`, and `scale` is\n        a scalar, returned Symbol will have shape `(m, n)`. If `scale`\n        is a Symbol with shape, e.g., `(x, y)`, returned Symbol will resolve to\n        shape `(x, y, m, n)`, where `m*n` samples are drawn for each entry in `scale`.\n    \"\"\"\n    return _random_helper(_internal._random_exponential, _internal._sample_exponential,\n                          [1.0/scale], shape, dtype, kwargs)\n\n\ndef gamma(alpha=1, beta=1, shape=_Null, dtype=_Null, **kwargs):\n    \"\"\"Draw random samples from a gamma distribution.\n\n    Samples are distributed according to a gamma distribution parametrized\n    by *alpha* (shape) and *beta* (scale).\n\n    Parameters\n    ----------\n    alpha : float or Symbol, optional\n        The shape of the gamma distribution. Should be greater than zero.\n    beta : float or Symbol, optional\n        The scale of the gamma distribution. Should be greater than zero.\n        Default is equal to 1.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `alpha` and\n        `beta` are scalars, output shape will be `(m, n)`. If `alpha` and `beta`\n        are Symbols with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each `[alpha, beta)` pair.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n\n    Returns\n    -------\n    Symbol\n        If input `shape` has dimensions, e.g., `(m, n)` and `alpha` and\n        `beta` are scalars, returned Symbol will resolve to shape `(m, n)`. If `alpha`\n        and `beta` are Symbols with shape, e.g., `(x, y)`, returned Symbol will resolve\n        to shape `(x, y, m, n)`, where `m*n` samples are drawn for each `[alpha, beta)` pair.\n    \"\"\"\n    return _random_helper(_internal._random_gamma, _internal._sample_gamma,\n                          [alpha, beta], shape, dtype, kwargs)\n\n\ndef binomial(n=1, p=0.5, shape=_Null, dtype=_Null, **kwargs):\n    \"\"\"Draw random samples from a binomial distribution.\n\n    Samples are distributed according to a binomial distribution parametrized\n    by *n* (number of trials) and *p* (success probability).\n\n    Parameters\n    ----------\n    n : float or Symbol, optional\n        Number of experiments, > 0.\n    p : float or Symbol, optional\n        Success probability in each experiment, >= 0 and <= 1.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `n` and\n        `p` are scalars, output shape will be `(m, n)`. If `n` and `p`\n        are NDArrays with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each `[n, p)` pair.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n\n    Returns\n    -------\n    Symbol\n        If input `shape` has shape, e.g., `(m, n)` and `n` and `p` are scalars, output\n        shape will be `(m, n)`. If `n` and `p` are NDArrays with shape, e.g.,\n        `(x, y)`, then output will have shape `(x, y, m, n)`, where `m*n` samples are\n        drawn for each `[n, p)` pair.\n    \"\"\"\n    return _random_helper(_internal._random_binomial, _internal._sample_binomial,\n                          [n, p], shape, dtype, kwargs)\n\n\ndef negative_binomial(k=1, p=1, shape=_Null, dtype=_Null, **kwargs):\n    \"\"\"Draw random samples from a negative binomial distribution.\n\n    Samples are distributed according to a negative binomial distribution\n    parametrized by *k* (limit of unsuccessful experiments) and *p* (failure\n    probability in each experiment). Samples will always be returned as a\n    floating point data type.\n\n    Parameters\n    ----------\n    k : float or Symbol, optional\n        Limit of unsuccessful experiments, > 0.\n    p : float or Symbol, optional\n        Failure probability in each experiment, >= 0 and <=1.\n    shape : int or tuple of ints\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `k` and\n        `p` are scalars, output shape will be `(m, n)`. If `k` and `p`\n        are Symbols with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each `[k, p)` pair.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n\n    Returns\n    -------\n    Symbol\n        If input `shape` has dimensions, e.g., `(m, n)`, and `k` and\n        `p` are scalars, returned Symbol will resolve to shape `(m, n)`. If `k`\n        and `p` are Symbols with shape, e.g., `(x, y)`, returned Symbol will resolve\n        to shape `(x, y, m, n)`, where `m*n` samples are drawn for each `[k, p)` pair.\n    \"\"\"\n    return _random_helper(_internal._random_negative_binomial,\n                          _internal._sample_negative_binomial,\n                          [k, p], shape, dtype, kwargs)\n\n\ndef generalized_negative_binomial(mu=1, alpha=1, shape=_Null, dtype=_Null, **kwargs):\n    \"\"\"Draw random samples from a generalized negative binomial distribution.\n\n    Samples are distributed according to a generalized negative binomial\n    distribution parametrized by *mu* (mean) and *alpha* (dispersion).\n    *alpha* is defined as *1/k* where *k* is the failure limit of the\n    number of unsuccessful experiments (generalized to real numbers).\n    Samples will always be returned as a floating point data type.\n\n    Parameters\n    ----------\n    mu : float or Symbol, optional\n        Mean of the negative binomial distribution.\n    alpha : float or Symbol, optional\n        Alpha (dispersion) parameter of the negative binomial distribution.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `mu` and\n        `alpha` are scalars, output shape will be `(m, n)`. If `mu` and `alpha`\n        are Symbols with shape, e.g., `(x, y)`, then output will have shape\n        `(x, y, m, n)`, where `m*n` samples are drawn for each `[mu, alpha)` pair.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n\n    Returns\n    -------\n    Symbol\n        If input `shape` has dimensions, e.g., `(m, n)`, and `mu` and\n        `alpha` are scalars, returned Symbol will resolve to shape `(m, n)`. If `mu`\n        and `alpha` are Symbols with shape, e.g., `(x, y)`, returned Symbol will resolve\n        to shape `(x, y, m, n)`, where `m*n` samples are drawn for each `[mu, alpha)` pair.\n    \"\"\"\n    return _random_helper(_internal._random_generalized_negative_binomial,\n                          _internal._sample_generalized_negative_binomial,\n                          [mu, alpha], shape, dtype, kwargs)\n\n\ndef categorical(data, shape=_Null, get_prob=True, dtype='int32', **kwargs):\n    \"\"\"Concurrent sampling from multiple categorical distributions.\n\n    .. note:: The input distribution must be normalized, i.e. `data` must sum to\n              1 along its last dimension.\n\n    Parameters\n    ----------\n    data : Symbol\n        An *n* dimensional array whose last dimension has length `k`, where\n        `k` is the number of possible outcomes of each categorical distribution.\n        For example, data with shape `(m, n, k)` specifies `m*n` categorical\n        distributions each with `k` possible outcomes.\n    shape : int or tuple of ints, optional\n        The number of samples to draw from each distribution. If shape is empty\n        one sample will be drawn from each distribution.\n    get_prob : bool, optional\n        If true, a second array containing log likelihood of the drawn\n        samples will also be returned.\n        This is usually used for reinforcement learning, where you can provide\n        reward as head gradient w.r.t. this array to estimate gradient.\n    dtype : str or numpy.dtype, optional\n        Data type of the sample output array. The default is int32.\n        Note that the data type of the log likelihood array is the same with that of `data`.\n\n    Returns\n    -------\n    Symbol\n        For input `data` with `n` dimensions and shape `(d1, d2, ..., dn-1, k)`, and input\n        `shape` with shape `(s1, s2, ..., sx)`, returns a Symbol that resovles to shape\n        `(d1, d2, ... dn-1, s1, s2, ..., sx)`. The `s1, s2, ... sx` dimensions of the\n        returned Symbol's resolved value will consist of 0-indexed values sampled from each\n        respective categorical distribution provided in the `k` dimension of `data`.\n\n        For the case `n`=1, and `x`=1 (one shape dimension), returned Symbol will resolve to\n        shape `(s1,)`.\n\n        If `get_prob` is set to True, this function returns a Symbol that will resolve to a list of\n        outputs: `[ndarray_output, log_likelihood_output]`, where `log_likelihood_output` will resolve\n        to the same shape as the sampled outputs in ndarray_output.\n    \"\"\"\n    return _internal._sample_categorical(data, shape, get_prob, dtype=dtype, **kwargs)\n\n\ndef multinomial(n=[1], p=[[1.0]], shape=_Null, dtype='float32', **kwargs):\n    \"\"\"Concurrent sampling from multiple multinomial distributions.\n\n    .. note:: The input distribution must be normalized, i.e. `p` must sum to\n              1 along its last dimension.\n\n    Parameters\n    ----------\n    n : Symbol\n        An *n* dimensional array containing the number of trials of each\n        multinomial distribution.\n    p : Symbol\n        An *n+1* dimensional array containing the probabilities of each multinomial\n        distribution. Its last dimension has length `k`, where `k` is the number\n        of possible outcomes of each multinomial distribution.\n        For example, p with shape `(m, n, k)` specifies `m*n` multinomial\n        distributions each with `k` possible outcomes.\n    shape : int or tuple of ints, optional\n        The number of samples to draw from each distribution. If shape is empty\n        one sample will be drawn from each distribution.\n    dtype : {'float16', 'float32', 'float64'}, optional\n        Data type of output samples. Default is 'float32'\n\n    Returns\n    -------\n    Symbol\n        If input `shape` has shape, e.g., `(m, n)` and `n` and `p` are a scalar and an array of length k\n        respectively, output shape will be `(m, n, k)`. If `n` and `p` are NDArrays with shape, e.g.,\n        `(x, y)` and `(x, y, k)`, then output will have shape `(x, y, m, n, k)`, where `m*n`\n        samples are drawn for each `[n, p)` pair.\n    \"\"\"\n    return _internal._sample_multinomial(n, p, shape, dtype=dtype, **kwargs)\n\n\ndef shuffle(data, **kwargs):\n    \"\"\"Shuffle the elements randomly.\n\n    This shuffles the array along the first axis.\n    The order of the elements in each subarray does not change.\n    For example, if a 2D array is given, the order of the rows randomly changes,\n    but the order of the elements in each row does not change.\n\n    Parameters\n    ----------\n    data : NDArray\n        Input data array.\n\n    Returns\n    -------\n    Symbol\n        A new symbol representing the shuffled version of input `data`.\n\n    Examples\n    --------\n    >>> data = mx.nd.array([[0, 1, 2], [3, 4, 5], [6, 7, 8]])\n    >>> a = mx.sym.Variable('a')\n    >>> b = mx.sym.random.shuffle(a)\n    >>> b.eval(a=data)\n    [[ 0.  1.  2.]\n     [ 6.  7.  8.]\n     [ 3.  4.  5.]]\n    <NDArray 2x3 @cpu(0)>\n    >>> b.eval(a=data)\n    [[ 3.  4.  5.]\n     [ 0.  1.  2.]\n     [ 6.  7.  8.]]\n    <NDArray 2x3 @cpu(0)>\n    \"\"\"\n    return _internal._shuffle(data, **kwargs)\n\n\ndef randint(low, high, shape=_Null, dtype=_Null, **kwargs):\n    \"\"\"Draw random samples from a discrete uniform distribution.\n\n    Samples are uniformly distributed over the half-open interval *[low, high)*\n    (includes *low*, but excludes *high*).\n\n    Parameters\n    ----------\n    low : int, required\n        Lower boundary of the output interval. All values generated will be\n        greater than or equal to low.\n    high : int, required\n        Upper boundary of the output interval. All values generated will be\n        less than high.\n    shape : int or tuple of ints, optional\n        The number of samples to draw. If shape is, e.g., `(m, n)` and `low` and\n        `high` are scalars, output shape will be `(m, n)`.\n    dtype : {'int32', 'int64'}, optional\n        Data type of output samples. Default is 'int32'\n\n    Returns\n    -------\n    Symbol\n        If input `shape` has dimensions, e.g., `(m, n)`, and `low` and\n        `high` are scalars, returned Symbol will resolve to shape `(m, n)`.\n    \"\"\"\n    return _random_helper(_internal._random_randint, None,\n                          [low, high], shape, dtype, kwargs)\n"
  },
  {
    "path": "python/mxnet/symbol/register.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=unused-import\n\"\"\"Register backend ops in mxnet.symbol namespace.\"\"\"\nimport os as _os\nimport ctypes\nimport numpy as _np\n\nfrom . import _internal\nfrom .. import name as _name, attribute\nfrom ._internal import SymbolBase, _symbol_creator\nfrom ..base import mx_uint, check_call, _LIB, py_str\nfrom ..symbol_doc import _build_doc\nfrom ..base import _Null, _init_op_module, _is_np_op, _output_is_list\nfrom ..name import NameManager\nfrom ..profiler import _current_scope as _profiler_scope\nfrom ..ndarray import get_dtype_name\n# pylint: enable=unused-import\n\n\ndef _verify_np_symbol(op_name, func_name, sym):\n    \"\"\"Verify if the sym is a numpy symbol.\n\n    Parameters\n    ----------\n    op_name : str\n        Operator full name registered in backend.\n    func_name : str\n        Operator name exposed to users. This is usually the name by stripping off\n        the prefix of the full operator names registered in backend.\n    sym : symbol to be verified\n    \"\"\"\n    from .numpy._symbol import _Symbol as np_symbol\n    if not isinstance(sym, np_symbol):\n        raise TypeError('Operator `{}` registered in backend is known as `{}` in Python. '\n                        'This is a numpy operator which can only accept '\n                        'MXNet numpy ndarrays, while received a legacy ndarray. '\n                        'Please ensure that you have activated numpy semantics by calling '\n                        '`npx.set_np()` in your code. If you still see this error with numpy '\n                        'semantics activated, please call `as_np_ndarray()` upon the legacy '\n                        'ndarray to convert it to an MXNet numpy ndarray, and then feed the '\n                        'converted array to this operator.'\n                        .format(op_name, func_name))\n\n\ndef _verify_legacy_symbol(op_name, func_name, sym):\n    \"\"\"Verify if the sym is a legacy symbol.\n\n    Parameters\n    ----------\n    op_name : str\n        Operator full name registered in backend.\n    func_name : str\n        Operator name exposed to users. This is usually the name by stripping off\n        the prefix of the full operator names registered in backend.\n    sym : symbol to be verified\n    \"\"\"\n    from .numpy._symbol import _Symbol as np_symbol\n    if isinstance(sym, np_symbol):\n        raise TypeError('Operator `{}` registered in backend is known as `{}` in Python. '\n                        'This is a legacy operator which can only accept '\n                        'legacy ndarrays, while received an MXNet numpy ndarray. '\n                        'Please call `as_nd_ndarray()` upon the numpy ndarray to '\n                        'convert it to a legacy ndarray, and then feed the converted '\n                        'array to this operator.'\n                        .format(op_name, func_name))\n\n\ndef _generate_symbol_function_code(handle, op_name, func_name, signature_only=False):\n    \"\"\"Generate function for symbol op by handle and function name.\"\"\"\n    real_name = ctypes.c_char_p()\n    desc = ctypes.c_char_p()\n    num_args = mx_uint()\n    arg_names = ctypes.POINTER(ctypes.c_char_p)()\n    arg_types = ctypes.POINTER(ctypes.c_char_p)()\n    arg_descs = ctypes.POINTER(ctypes.c_char_p)()\n    key_var_num_args = ctypes.c_char_p()\n    ret_type = ctypes.c_char_p()\n\n    check_call(_LIB.MXSymbolGetAtomicSymbolInfo(\n        handle, ctypes.byref(real_name), ctypes.byref(desc),\n        ctypes.byref(num_args),\n        ctypes.byref(arg_names),\n        ctypes.byref(arg_types),\n        ctypes.byref(arg_descs),\n        ctypes.byref(key_var_num_args),\n        ctypes.byref(ret_type)))\n    narg = int(num_args.value)\n    arg_names = [py_str(arg_names[i]) for i in range(narg)]\n    arg_types = [py_str(arg_types[i]) for i in range(narg)]\n    key_var_num_args = py_str(key_var_num_args.value)\n    ret_type = py_str(ret_type.value) if ret_type.value is not None else ''\n    doc_str = _build_doc(op_name,\n                         py_str(desc.value),\n                         arg_names,\n                         arg_types,\n                         [py_str(arg_descs[i]) for i in range(narg)],\n                         key_var_num_args,\n                         ret_type)\n\n    dtype_name = None\n    arr_name = None\n    ndsignature = []\n    signature = []\n    ndarg_names = []\n    kwarg_names = []\n    for i in range(narg):\n        name, atype = arg_names[i], arg_types[i]\n        if name == 'dtype':\n            dtype_name = name\n            signature.append(f'{name}=_Null')\n        elif atype.startswith('NDArray') or atype.startswith('Symbol'):\n            assert not arr_name, \\\n                \"Op can only have one argument with variable \" \\\n                \"size and it must be the last argument.\"\n            if atype.endswith('[]'):\n                ndsignature.append(f'*{name}')\n                arr_name = name\n            else:\n                ndsignature.append(f'{name}=None')\n                ndarg_names.append(name)\n        else:\n            signature.append(f'{name}=_Null')\n            kwarg_names.append(name)\n    #signature.append('is_train=False')\n    signature.append('name=None')\n    signature.append('attr=None')\n    signature.append('out=None')\n    signature.append('**kwargs')\n    signature = ndsignature + signature\n\n    is_np_op = _is_np_op(op_name)\n    output_is_list = _output_is_list(op_name)\n    verify_symbol_fn = _verify_np_symbol.__name__ if is_np_op else _verify_legacy_symbol.__name__\n    code = []\n    if arr_name:\n        code.append(\"\"\"\ndef %s(*%s, **kwargs):\"\"\"%(func_name, arr_name))\n        if not signature_only:\n            code.append(\"\"\"\n    sym_args = []\n    for i in {}:\n        assert isinstance(i, SymbolBase), \\\\\n            \"Positional arguments must be Symbol instances, \" \\\\\n            \"but got %s\"%str(i)\n        {}('{}', '{}', i)\n        sym_args.append(i)\"\"\".format(arr_name, verify_symbol_fn, op_name, func_name))\n            if dtype_name is not None:\n                code.append(\"\"\"\n    if '%s' in kwargs:\n        kwargs['%s'] = get_dtype_name(kwargs['%s'])\"\"\"%(dtype_name, dtype_name, dtype_name))\n            code.append(\"\"\"\n    attr = kwargs.pop('attr', None)\n    kwargs.update(attribute.current().get(attr))\n    name = kwargs.pop('name', None)\n    name = _name.current().get(name, '%s')\n    _ = kwargs.pop('out', None)\n    keys = []\n    vals = []\n    sym_kwargs = dict()\n    for k, v in kwargs.items():\n        if isinstance(v, SymbolBase):\n            sym_kwargs[k] = v\n            %s('%s', '%s', v)\n        else:\n            keys.append(k)\n            vals.append(v)\"\"\"%(func_name.lower(), verify_symbol_fn, op_name, func_name))\n            if key_var_num_args: # pylint: disable=using-constant-test\n                code.append(\"\"\"\n    if '%s' not in kwargs:\n        keys.append('%s')\n        vals.append(len(sym_args) + len(sym_kwargs))\"\"\"%(\n            key_var_num_args, key_var_num_args))\n\n            code.append(\"\"\"\n    if 'profiler_scope' not in keys:\n        keys.append('profiler_scope')\n        vals.append(_profiler_scope.get())\n    return _symbol_creator(%d, sym_args, sym_kwargs, keys, vals, name, %s, %s)\"\"\"%(\n                handle.value, str(is_np_op), str(output_is_list)))\n    else:\n        code.append(\"\"\"\ndef %s(%s):\"\"\"%(func_name, ', '.join(signature)))\n        if not signature_only:\n            code.append(\"\"\"\n    kwargs.update(attribute.current().get(attr))\n    sym_kwargs = dict()\n    _keys = []\n    _vals = []\n    for _k, _v in kwargs.items():\n        if isinstance(_v, SymbolBase):\n            sym_kwargs[_k] = _v\n            {}('{}', '{}', _v)\n        else:\n            _keys.append(_k)\n            _vals.append(_v)\"\"\".format(verify_symbol_fn, op_name, func_name))\n            # NDArray args\n            for name in ndarg_names: # pylint: disable=redefined-argument-from-local\n                code.append(\"\"\"\n    if {name} is not None:\n        assert isinstance({name}, SymbolBase), \\\\\n            \"Argument {name} must be Symbol instances, but got %s\"%str({name})\n        sym_kwargs['{name}'] = {name}\"\"\".format(name=name))\n                code.append(\"\"\"\n        {}('{}', '{}', {name})\n                \"\"\".format(verify_symbol_fn, op_name, func_name, name=name))\n            # kwargs\n            for name in kwarg_names: # pylint: disable=redefined-argument-from-local\n                code.append(\"\"\"\n    if %s is not _Null:\n        _keys.append('%s')\n        _vals.append(%s)\"\"\"%(name, name, name))\n            # dtype\n            if dtype_name is not None:\n                if is_np_op:\n                    code.append(\"\"\"\n    if %s is not _Null and %s is not None:\n        _keys.append('%s')\n        _vals.append(get_dtype_name(%s))\"\"\"%(dtype_name, dtype_name, dtype_name, dtype_name))\n                else:\n                    code.append(\"\"\"\n    if %s is not _Null:\n        _keys.append('%s')\n        _vals.append(get_dtype_name(%s))\"\"\"%(dtype_name, dtype_name, dtype_name))\n\n            code.append(\"\"\"\n    name = _name.current().get(name, '%s')\n    if 'profiler_scope' not in _keys:\n        _keys.append('profiler_scope')\n        _vals.append(_profiler_scope.get())\n    return _symbol_creator(%d, None, sym_kwargs, _keys, _vals, name, %s, %s)\"\"\"%(\n        func_name.lower(), handle.value, str(is_np_op), str(output_is_list)))\n\n    if signature_only:\n        code.append(\"\"\"\n    return (0,)\"\"\")\n\n    doc_str_lines = _os.linesep+''.join(['    '+s if s.strip() else s\n                                         for s in 'r\"\"\"{doc_str}\"\"\"'.format(doc_str=doc_str)\n                                         .splitlines(True)])\n    code.insert(1, doc_str_lines)\n    return ''.join(code), doc_str\n\n\ndef _make_symbol_function(handle, name, func_name):\n    \"\"\"Create a symbol function by handle and function name.\"\"\"\n    code, doc_str = _generate_symbol_function_code(handle, name, func_name)\n\n    local = {}\n    exec(code, None, local)  # pylint: disable=exec-used\n    symbol_function = local[func_name]\n    symbol_function.__name__ = func_name\n    symbol_function.__doc__ = doc_str\n    symbol_function.__module__ = 'mxnet.symbol'\n    return symbol_function\n\n_init_op_module('mxnet', 'symbol', _make_symbol_function)\n\n# Update operator documentation with added float support\n# Note that we can only do this after the op module is initialized\n# Otherwise the backend operators cannot be found\n# pylint: disable=wrong-import-position\nfrom .contrib import adamw_update, mp_adamw_update\nfrom ._internal import _adamw_update, _mp_adamw_update\nadamw_update.__doc__ = _adamw_update.__doc__.replace(\"rescale_grad : Symbol\",\n                                                     \"rescale_grad : Symbol or float\")\nmp_adamw_update.__doc__ = _mp_adamw_update.__doc__.replace(\"rescale_grad : Symbol\",\n                                                           \"rescale_grad : Symbol or float\")\n"
  },
  {
    "path": "python/mxnet/symbol/sparse.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=wildcard-import, unused-wildcard-import\n\"\"\"Sparse Symbol API of MXNet.\"\"\"\ntry:\n    from .gen_sparse import * # pylint: disable=redefined-builtin\nexcept ImportError:\n    pass\n\n__all__ = []\n"
  },
  {
    "path": "python/mxnet/symbol/symbol.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=invalid-name, protected-access, too-many-arguments, too-many-lines\n# pylint: disable=import-error, no-name-in-module\n\"\"\"Symbolic configuration API of MXNet.\"\"\"\ntry:\n    from __builtin__ import slice as py_slice\nexcept ImportError:\n    from builtins import slice as py_slice\n\nfrom array import array\nimport ctypes\nimport warnings\nfrom numbers import Number\nimport numpy as _numpy  # pylint: disable=relative-import\n\nfrom .. import attribute\nfrom ..base import _LIB, numeric_types, c_array, c_array_buf, c_str, c_str_array, c_handle_array\nfrom ..base import mx_uint, py_str, string_types, integer_types, mx_int, mx_int64\nfrom ..base import NDArrayHandle, SymbolHandle\nfrom ..base import check_call, MXNetError, NotImplementedForSymbol\nfrom ..device import Device, current_device\nfrom ..ndarray import NDArray, dtype_np_to_mx, dtype_mx_to_np, is_mx_dtype\nfrom ..ndarray.ndarray import _STORAGE_TYPE_STR_TO_ID, _int64_enabled, _SIGNED_INT32_UPPER_LIMIT\nfrom ..executor import Executor\nfrom . import _internal\nfrom . import op\nfrom ._internal import SymbolBase, _set_symbol_class\nfrom ..util import is_np_shape\nfrom ..profiler import scope as _profiler_scope\nfrom ..profiler import _current_scope as _current_profiler_scope\n\n__all__ = [\"Symbol\", \"var\", \"Variable\", \"Group\", \"load\", \"fromjson\",\n           \"pow\", \"power\", \"maximum\", \"minimum\", \"hypot\", \"eye\", \"zeros\",\n           \"ones\", \"full\", \"arange\", \"linspace\", \"histogram\", \"split_v2\"]\n\n\nclass Symbol(SymbolBase):\n    \"\"\"Symbol is symbolic graph of the mxnet.\"\"\"\n    # disable dictionary storage, also do not have parent type.\n    # pylint: disable=no-member\n    __slots__ = []\n\n    # Make numpy functions return Symbol instead of numpy object array\n    __array_priority__ = 1000.0\n\n    def as_np_ndarray(self):\n        \"\"\"Convert mx.sym.Symbol to mx.sym.np._Symbol.\"\"\"\n        from .numpy import _Symbol\n        hdl = SymbolHandle()\n        check_call(_LIB.MXShallowCopySymbol(self.handle, ctypes.byref(hdl)))\n        return _Symbol(hdl)\n\n    def as_nd_ndarray(self):\n        \"\"\"Returns self. For the convenience of conversion between legacy and np symbols.\"\"\"\n        return self\n\n    def __repr__(self):\n        \"\"\"Gets a string representation of the symbol.\"\"\"\n        if self._alive:\n            name = self.name\n            if name is None:\n                name = ', '.join([i.name for i in self])\n                return f'<{self.__class__.__name__} group [{name}]>'\n            else:\n                return f'<{self.__class__.__name__} {name}>'\n        else:\n            return '<FREED {}>'.format(self.__class__.__name__)\n\n    def __iter__(self):\n        \"\"\"Returns a generator object of symbol.\n\n        One can loop through the returned object list to get outputs.\n\n        Example\n        -------\n        >>> a = mx.sym.Variable('a')\n        >>> b = mx.sym.Variable('b')\n        >>> c = a+b\n        >>> d = mx.sym.Variable('d')\n        >>> e = d+c\n        >>> out = e.get_children()\n        >>> out\n        <Symbol Grouped>\n        >>> for i in out:\n        ...     print(i)\n        ...\n        <Symbol d>\n        <Symbol _plus0>\n        \"\"\"\n        return (self[i] for i in range(len(self)))\n\n    def __abs__(self):\n        \"\"\"x.__abs__() <=> abs(x) <=> x.abs() <=> mx.symbol.abs(x, y)\"\"\"\n        return self.abs()\n\n    def __add__(self, other):\n        \"\"\"x.__add__(y) <=> x+y\n\n        Scalar input is supported.\n        Broadcasting is not supported. Use `broadcast_add` instead. \"\"\"\n        if isinstance(other, Symbol):\n            return _internal._Plus(self, other)\n        if isinstance(other, Number):\n            return _internal._PlusScalar(self, scalar=other)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __bool__(self):\n        raise NotImplementedForSymbol(self.__bool__, 'bool')\n\n    __nonzero__ = __bool__\n\n    def __iadd__(self, other):\n        raise NotImplementedForSymbol(self.__iadd__, '+=', other, 1)\n\n    def __radd__(self, other):\n        return self.__add__(other)\n\n    def __sub__(self, other):\n        \"\"\"x.__sub__(y) <=> x-y\n\n        Scalar input is supported.\n        Broadcasting is not supported. Use `broadcast_sub` instead. \"\"\"\n        if isinstance(other, Symbol):\n            return _internal._Minus(self, other)\n        if isinstance(other, Number):\n            return _internal._MinusScalar(self, scalar=other)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __isub__(self, other):\n        raise NotImplementedForSymbol(self.__isub__, '-=', other)\n\n    def __rsub__(self, other):\n        \"\"\"x.__rsub__(y) <=> y-x\n\n        Only `NDArray` is supported for now.\n\n        Example\n        -------\n        >>> x = mx.nd.ones((2,3))*3\n        >>> y = mx.nd.ones((2,3))\n        >>> x.__rsub__(y).asnumpy()\n        array([[-2., -2., -2.],\n               [-2., -2., -2.]], dtype=float32)\n        \"\"\"\n        if isinstance(other, Symbol):\n            return other.__sub__(self)\n        if isinstance(other, Number):\n            return _internal._RMinusScalar(self, scalar=other)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __mul__(self, other):\n        \"\"\"x.__mul__(y) <=> x*y\n\n        Scalar input is supported.\n        Broadcasting is not supported. Use `broadcast_mul` instead. \"\"\"\n        if isinstance(other, Symbol):\n            return _internal._Mul(self, other)\n        if isinstance(other, Number):\n            return _internal._MulScalar(self, scalar=other)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __imul__(self, other):\n        raise NotImplementedForSymbol(self.__imul__, '*=', other)\n\n    def __rmul__(self, other):\n        return self.__mul__(other)\n\n    def __div__(self, other):\n        \"\"\"x.__div__(y) <=> x/y\n\n        Scalar input is supported.\n        Broadcasting is not supported. Use `broadcast_div` instead. \"\"\"\n        if isinstance(other, Symbol):\n            return _internal._Div(self, other)\n        if isinstance(other, Number):\n            return _internal._DivScalar(self, scalar=other)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __rdiv__(self, other):\n        \"\"\"x.__rdiv__(y) <=> y/x\n\n        Only `NDArray` is supported for now.\n\n        Example\n        -------\n        >>> x = mx.nd.ones((2,3))*3\n        >>> y = mx.nd.ones((2,3))\n        >>> x.__rdiv__(y).asnumpy()\n        array([[ 0.33333334,  0.33333334,  0.33333334],\n               [ 0.33333334,  0.33333334,  0.33333334]], dtype=float32)\n        \"\"\"\n        if isinstance(other, Symbol):\n            return other.__truediv__(self)\n        if isinstance(other, Number):\n            return _internal._RDivScalar(self, scalar=other)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __mod__(self, other):\n        \"\"\"x.__mod__(y) <=> x%y\n\n        Scalar input is supported.\n        Broadcasting is not supported. Use `broadcast_mod` instead. \"\"\"\n        if isinstance(other, Symbol):\n            return _internal._Mod(self, other)\n        if isinstance(other, Number):\n            return _internal._ModScalar(self, scalar=other)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __rmod__(self, other):\n        \"\"\"x.__rmod__(y) <=> y%x\n\n        Only `NDArray` is supported for now.\n\n        Example\n        -------\n        >>> x = mx.nd.ones((2,3))*3\n        >>> y = mx.nd.ones((2,3))\n        >>> x.__rmod__(y).asnumpy()\n        array([[ 1.,  1.,  1.,\n               [ 1.,  1.,  1., dtype=float32)\n        \"\"\"\n        if isinstance(other, Symbol):\n            return other.__mod__(self)\n        if isinstance(other, Number):\n            return _internal._RModScalar(self, scalar=other)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __idiv__(self, other):\n        raise NotImplementedForSymbol(self.__idiv__, '/=', other)\n\n    def __truediv__(self, other):\n        return self.__div__(other)\n\n    def __rtruediv__(self, other):\n        return self.__rdiv__(other)\n\n    def __itruediv__(self, other):\n        raise NotImplementedForSymbol(self.__itruediv__, '/=', other)\n\n    def __pow__(self, other):\n        \"\"\"x.__pow__(y) <=> x**y\n\n        Scalar input is supported.\n        Broadcasting is not supported. Use `broadcast_pow` instead. \"\"\"\n        if isinstance(other, Symbol):\n            return _internal._Power(self, other)\n        if isinstance(other, Number):\n            return _internal._PowerScalar(self, scalar=other)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __rpow__(self, other):\n        \"\"\"x.__rpow__(y) <=> y ** x\"\"\"\n        if isinstance(other, Symbol):\n            return other.__pow__(self)\n        elif isinstance(other, Number):\n            return _internal._rpower_scalar(self, scalar=other)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __neg__(self):\n        \"\"\"x.__neg__() <=> -x\n\n        Numerical negative, element-wise.\n\n        Example\n        -------\n        >>> a = mx.sym.Variable('a')\n        >>> a\n        <Symbol a>\n        >>> -a\n        <Symbol _mulscalar0>\n        >>> a_neg = a.__neg__()\n        >>> c = a_neg*b\n        >>> ex = c.eval(ctx=mx.cpu(), a=mx.nd.ones([2,3]), b=mx.nd.ones([2,3]))\n        >>> ex[0].asnumpy()\n        array([[-1., -1., -1.],\n               [-1., -1., -1.]], dtype=float32)\n        \"\"\"\n        return self.__mul__(-1.0)\n\n    def __copy__(self):\n        return self.__deepcopy__(None)\n\n    def __deepcopy__(self, _):\n        \"\"\"Returns a deep copy of the input object.\n\n        This function returns a deep copy of the input object including the current state\n        of all its parameters such as weights, biases, etc.\n\n        Any changes made to the deep copy do not reflect in the original object.\n\n        Example\n        -------\n        >>> import copy\n        >>> data = mx.sym.Variable('data')\n        >>> data_1 = copy.deepcopy(data)\n        >>> data_1 = 2*data\n        >>> data_1.tojson()\n        >>> data_1 is data    # Data got modified\n        False\n        \"\"\"\n        handle = SymbolHandle()\n        check_call(_LIB.MXSymbolCopy(self.handle,\n                                     ctypes.byref(handle)))\n        return Symbol(handle)\n\n    def __eq__(self, other):\n        \"\"\"x.__eq__(y) <=> x==y\n\n        Scalar input is supported.\n        Broadcasting is not supported. Use `broadcast_equal` instead. \"\"\"\n        if isinstance(other, Symbol):\n            return _internal._equal(self, other)\n        if isinstance(other, numeric_types):\n            return _internal._equal_scalar(self, scalar=other)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __ne__(self, other):\n        \"\"\"x.__ne__(y) <=> x!=y\n\n        Scalar input is supported.\n        Broadcasting is not supported. Use `broadcast_not_equal` instead. \"\"\"\n        if isinstance(other, Symbol):\n            return _internal._not_equal(self, other)\n        if isinstance(other, numeric_types):\n            return _internal._not_equal_scalar(self, scalar=other)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __gt__(self, other):\n        \"\"\"x.__gt__(y) <=> x>y\n\n        Scalar input is supported.\n        Broadcasting is not supported. Use `broadcast_greater` instead. \"\"\"\n        if isinstance(other, Symbol):\n            return _internal._greater(self, other)\n        if isinstance(other, numeric_types):\n            return _internal._greater_scalar(self, scalar=other)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __ge__(self, other):\n        \"\"\"x.__ge__(y) <=> x>=y\n\n        Scalar input is supported.\n        Broadcasting is not supported. Use `broadcast_greater_equal` instead. \"\"\"\n        if isinstance(other, Symbol):\n            return _internal._greater_equal(self, other)\n        if isinstance(other, numeric_types):\n            return _internal._greater_equal_scalar(self, scalar=other)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __lt__(self, other):\n        \"\"\"x.__lt__(y) <=> x<y\n\n        Scalar input is supported.\n        Broadcasting is not supported. Use `broadcast_lesser` instead. \"\"\"\n        if isinstance(other, Symbol):\n            return _internal._lesser(self, other)\n        if isinstance(other, numeric_types):\n            return _internal._lesser_scalar(self, scalar=other)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __le__(self, other):\n        \"\"\"x.__le__(y) <=> x<=y\n\n        Scalar input is supported.\n        Broadcasting is not supported. Use `broadcast_lesser_equal` instead. \"\"\"\n        if isinstance(other, Symbol):\n            return _internal._lesser_equal(self, other)\n        if isinstance(other, numeric_types):\n            return _internal._lesser_equal_scalar(self, scalar=other)\n        else:\n            raise TypeError(f'type {str(type(other))} not supported')\n\n    def __getstate__(self):\n        handle = self.handle\n        if handle is not None:\n            return {'handle': self.tojson()}\n        else:\n            return {'handle': None}\n\n    def __setstate__(self, state):\n        # pylint: disable=assigning-non-slot\n        handle = state['handle']\n        if handle is not None:\n            json_str = handle\n            handle = SymbolHandle()\n            check_call(_LIB.MXSymbolCreateFromJSON(c_str(json_str), ctypes.byref(handle)))\n            self.handle = handle\n        else:\n            self.handle = None\n\n    def __call__(self, *args, **kwargs):\n        \"\"\"Composes symbol using inputs.\n\n        x.__call__(y, z) <=> x(y,z)\n\n        This function internally calls `_compose` to compose the symbol and\n        returns the composed symbol.\n\n        Example\n        -------\n        >>> data = mx.symbol.Variable('data')\n        >>> net1 = mx.symbol.FullyConnected(data=data, name='fc1', num_hidden=10)\n        >>> net2 = mx.symbol.FullyConnected(name='fc3', num_hidden=10)\n        >>> composed = net2(fc3_data=net1, name='composed')\n        >>> composed\n        <Symbol composed>\n        >>> called = net2.__call__(fc3_data=net1, name='composed')\n        >>> called\n        <Symbol composed>\n\n        Parameters\n        ----------\n        args:\n            Positional arguments.\n\n        kwargs:\n            Keyword arguments.\n\n        Returns\n        -------\n            The resulting symbol.\n        \"\"\"\n        s = self.__copy__()\n        s._compose(*args, **kwargs)\n        return s\n\n    def _compose(self, *args, **kwargs):\n        \"\"\"Composes symbol using inputs.\n\n        x._compose(y, z) <=> x(y,z)\n\n        This function mutates the current symbol.\n\n        Example\n        -------\n        >>> data = mx.symbol.Variable('data')\n        >>> net1 = mx.symbol.FullyConnected(data=data, name='fc1', num_hidden=10)\n        >>> net2 = mx.symbol.FullyConnected(name='fc3', num_hidden=10)\n        >>> net2\n        <Symbol fc3>\n        >>> net2._compose(fc3_data=net1, name='composed')\n        >>> net2\n        <Symbol composed>\n\n        Parameters\n        ----------\n        args:\n            Positional arguments.\n\n        kwargs:\n            Keyword arguments.\n\n        Returns\n        -------\n            The resulting symbol.\n        \"\"\"\n        name = kwargs.pop('name', None)\n\n        if name:\n            name = c_str(name)\n        if len(args) != 0 and len(kwargs) != 0:\n            raise TypeError('compose only accept input Symbols \\\n                either as positional or keyword arguments, not both')\n\n        for arg in args:\n            if not isinstance(arg, Symbol):\n                raise TypeError('Compose expect `Symbol` as arguments')\n        for val in kwargs.values():\n            if not isinstance(val, Symbol):\n                raise TypeError('Compose expect `Symbol` as arguments')\n\n        num_args = len(args) + len(kwargs)\n        if len(kwargs) != 0:\n            keys = c_str_array(kwargs.keys())\n            args = c_handle_array(kwargs.values())\n        else:\n            keys = None\n            args = c_handle_array(args)\n        check_call(_LIB.MXSymbolCompose(\n            self.handle, name, num_args, keys, args))\n\n    def __getitem__(self, index):\n        \"\"\"x.__getitem__(i) <=> x[i]\n\n        Returns a sliced view of the input symbol.\n\n        Example\n        -------\n        >>> a = mx.sym.var('a')\n        >>> a.__getitem__(0)\n        <Symbol a>\n        >>> a[0]\n        <Symbol a>\n\n        Parameters\n        ----------\n        index : int or str\n            Indexing key\n\n        \"\"\"\n        output_count = len(self)\n        if isinstance(index, py_slice):\n            start = 0 if index.start is None else index.start\n            stop = output_count if index.stop is None else index.stop\n            step = 1 if index.step is None else index.step\n            return Group([self[i] for i in range(start, stop, step)])\n\n        if isinstance(index, string_types):\n            # Returning this list of names is expensive. Some symbols may have hundreds of outputs\n            output_names = self.list_outputs()\n            idx = None\n            for i, name in enumerate(output_names):\n                if name == index:\n                    if idx is not None:\n                        raise ValueError(f'There are multiple outputs with name \\\"{index}\\\"')\n                    idx = i\n            if idx is None:\n                raise ValueError(f'Cannot find output that matches name \\\"{index}\\\"')\n            index = idx\n\n        if not isinstance(index, int):\n            raise TypeError('Symbol only support integer index to fetch i-th output')\n        if index >= output_count:\n            # Important, python determines the end by this exception\n            raise IndexError\n        handle = SymbolHandle()\n        check_call(_LIB.MXSymbolGetOutput(\n            self.handle, mx_uint(index), ctypes.byref(handle)))\n        return Symbol(handle=handle)\n\n    @property\n    def name(self):\n        \"\"\"Gets name string from the symbol, this function only works for non-grouped symbol.\n\n        Returns\n        -------\n        value : str\n            The name of this symbol, returns ``None`` for grouped symbol.\n        \"\"\"\n        ret = ctypes.c_char_p()\n        success = ctypes.c_int()\n        check_call(_LIB.MXSymbolGetName(\n            self.handle, ctypes.byref(ret), ctypes.byref(success)))\n        if success.value != 0:\n            return py_str(ret.value)\n        else:\n            return None\n\n    def attr(self, key):\n        \"\"\"Returns the attribute string for corresponding input key from the symbol.\n\n        This function only works for non-grouped symbols.\n\n        Example\n        -------\n        >>> data = mx.sym.Variable('data', attr={'mood': 'angry'})\n        >>> data.attr('mood')\n        'angry'\n\n        Parameters\n        ----------\n        key : str\n            The key corresponding to the desired attribute.\n\n        Returns\n        -------\n        value : str\n            The desired attribute value, returns ``None`` if the attribute does not exist.\n        \"\"\"\n        ret = ctypes.c_char_p()\n        success = ctypes.c_int()\n        check_call(_LIB.MXSymbolGetAttr(\n            self.handle, c_str(key), ctypes.byref(ret), ctypes.byref(success)))\n        if success.value != 0:\n            return py_str(ret.value)\n        else:\n            return None\n\n    def list_attr(self, recursive=False):\n        \"\"\"Gets all attributes from the symbol.\n\n        Example\n        -------\n        >>> data = mx.sym.Variable('data', attr={'mood': 'angry'})\n        >>> data.list_attr()\n        {'mood': 'angry'}\n\n        Returns\n        -------\n        ret : Dict of str to str\n            A dictionary mapping attribute keys to values.\n        \"\"\"\n        if recursive:\n            raise DeprecationWarning(\"Symbol.list_attr with recursive=True has been deprecated. \"\n                                     \"Please use attr_dict instead.\")\n        size = mx_uint()\n        pairs = ctypes.POINTER(ctypes.c_char_p)()\n        f_handle = _LIB.MXSymbolListAttrShallow\n        check_call(f_handle(self.handle, ctypes.byref(size), ctypes.byref(pairs)))\n        return {py_str(pairs[i * 2]): py_str(pairs[i * 2 + 1]) for i in range(size.value)}\n\n    def attr_dict(self):\n        \"\"\"Recursively gets all attributes from the symbol and its children.\n\n        Example\n        -------\n        >>> a = mx.sym.Variable('a', attr={'a1':'a2'})\n        >>> b = mx.sym.Variable('b', attr={'b1':'b2'})\n        >>> c = a+b\n        >>> c.attr_dict()\n        {'a': {'a1': 'a2'}, 'b': {'b1': 'b2'}}\n\n        Returns\n        -------\n        ret : Dict of str to dict\n            There is a key in the returned dict for every child with non-empty attribute set.\n            For each symbol, the name of the symbol is its key in the dict\n            and the correspond value is that symbol's attribute list (itself a dictionary).\n        \"\"\"\n        size = mx_uint()\n        pairs = ctypes.POINTER(ctypes.c_char_p)()\n        f_handle = _LIB.MXSymbolListAttr\n        check_call(f_handle(self.handle, ctypes.byref(size), ctypes.byref(pairs)))\n        ret = {}\n        for i in range(size.value):\n            name, key = py_str(pairs[i * 2]).split('$')\n            val = py_str(pairs[i * 2 + 1])\n            if name not in ret:\n                ret[name] = {}\n            ret[name][key] = val\n        return ret\n\n    def _set_attr(self, **kwargs):\n        \"\"\"Sets an attribute of the symbol.\n\n        For example. A._set_attr(foo=\"bar\") adds the mapping ``\"{foo: bar}\"``\n        to the symbol's attribute dictionary.\n\n        Parameters\n        ----------\n        **kwargs\n            The attributes to set\n        \"\"\"\n        for key, value in kwargs.items():\n            if not isinstance(value, string_types):\n                raise ValueError(\"Set Attr only accepts string values\")\n            check_call(_LIB.MXSymbolSetAttr(\n                self.handle, c_str(key), c_str(str(value))))\n\n    def get_inputs(self):\n        \"\"\"Gets a new grouped symbol `sgroup`. The output of `sgroup` is a list of inputs to this symbol.\n\n        Consider the following code:\n\n        Example\n        -------\n        >>> a = mx.sym.var('a')\n        >>> b = mx.sym.var('b')\n        >>> c = a + b\n        >>> d = c.get_inputs()\n        >>> d\n        <Symbol Grouped>\n        >>> d.list_outputs()\n        ['a', 'b']\n\n        Returns\n        -------\n        sgroup : Symbol\n            A symbol group containing all input nodes of the computation graph\n            used to compute the symbol.\n        \"\"\"\n        handle = SymbolHandle()\n        check_call(_LIB.MXSymbolGetInputs(\n            self.handle, ctypes.byref(handle)))\n        return Symbol(handle=handle)\n\n    def get_internals(self):\n        \"\"\"Gets a new grouped symbol `sgroup`. The output of `sgroup` is a list of\n        outputs of all of the internal nodes.\n\n        Consider the following code:\n\n        Example\n        -------\n        >>> a = mx.sym.var('a')\n        >>> b = mx.sym.var('b')\n        >>> c = a + b\n        >>> d = c.get_internals()\n        >>> d\n        <Symbol Grouped>\n        >>> d.list_outputs()\n        ['a', 'b', '_plus4_output']\n\n        Returns\n        -------\n        sgroup : Symbol\n            A symbol group containing all internal and leaf nodes of the computation graph\n            used to compute the symbol.\n        \"\"\"\n        handle = SymbolHandle()\n        check_call(_LIB.MXSymbolGetInternals(\n            self.handle, ctypes.byref(handle)))\n        return Symbol(handle=handle)\n\n    def get_children(self):\n        \"\"\"Gets a new grouped symbol whose output contains\n        inputs to output nodes of the original symbol.\n\n        Example\n        -------\n        >>> x = mx.sym.Variable('x')\n        >>> y = mx.sym.Variable('y')\n        >>> z = mx.sym.Variable('z')\n        >>> a = y+z\n        >>> b = x+a\n        >>> b.get_children()\n        <Symbol Grouped>\n        >>> b.get_children().list_outputs()\n        ['x', '_plus10_output']\n        >>> b.get_children().get_children().list_outputs()\n        ['y', 'z']\n\n        Returns\n        -------\n        sgroup : Symbol or None\n            The children of the head node. If the symbol has no\n            inputs then ``None`` will be returned.\n        \"\"\"\n        handle = SymbolHandle()\n        check_call(_LIB.MXSymbolGetChildren(\n            self.handle, ctypes.byref(handle)))\n        ret = Symbol(handle=handle)\n        if len(ret.list_outputs()) == 0:\n            return None\n        return ret\n\n    def list_arguments(self):\n        \"\"\"Lists all the arguments in the symbol.\n\n        Example\n        -------\n        >>> a = mx.sym.var('a')\n        >>> b = mx.sym.var('b')\n        >>> c = a + b\n        >>> c.list_arguments\n        ['a', 'b']\n\n        Returns\n        -------\n        args : list of string\n            List containing the names of all the arguments required to compute the symbol.\n        \"\"\"\n        size = ctypes.c_uint()\n        sarr = ctypes.POINTER(ctypes.c_char_p)()\n        check_call(_LIB.MXSymbolListArguments(\n            self.handle, ctypes.byref(size), ctypes.byref(sarr)))\n        return [py_str(sarr[i]) for i in range(size.value)]\n\n    def list_outputs(self):\n        \"\"\"Lists all the outputs in the symbol.\n\n        Example\n        -------\n        >>> a = mx.sym.var('a')\n        >>> b = mx.sym.var('b')\n        >>> c = a + b\n        >>> c.list_outputs()\n        ['_plus12_output']\n\n        Returns\n        -------\n        list of str\n            List of all the outputs.\n            For most symbols, this list contains only the name of this symbol.\n            For symbol groups, this is a list with the names of all symbols\n            in the group.\n        \"\"\"\n        size = ctypes.c_uint()\n        sarr = ctypes.POINTER(ctypes.c_char_p)()\n        check_call(_LIB.MXSymbolListOutputs(\n            self.handle, ctypes.byref(size), ctypes.byref(sarr)))\n        return [py_str(sarr[i]) for i in range(size.value)]\n\n    # pylint: disable=invalid-length-returned\n    def __len__(self):\n        \"\"\"Get number of outputs for the symbol.\n\n        Example\n        -------\n        >>> a = mx.sym.var('a')\n        >>> b = mx.sym.var('b')\n        >>> c = a + b\n        >>> len(c)\n\n        Returns\n        -------\n        len(self): Number of outputs\n            Number of outputs\n        \"\"\"\n        output_count = mx_uint()\n        check_call(_LIB.MXSymbolGetNumOutputs(self.handle, ctypes.byref(output_count)))\n        return output_count.value\n\n    def list_auxiliary_states(self):\n        \"\"\"Lists all the auxiliary states in the symbol.\n\n        Example\n        -------\n        >>> a = mx.sym.var('a')\n        >>> b = mx.sym.var('b')\n        >>> c = a + b\n        >>> c.list_auxiliary_states()\n        []\n\n        Example of auxiliary states in `BatchNorm`.\n\n        >>> data = mx.symbol.Variable('data')\n        >>> weight = mx.sym.Variable(name='fc1_weight')\n        >>> fc1  = mx.symbol.FullyConnected(data = data, weight=weight, name='fc1', num_hidden=128)\n        >>> fc2 = mx.symbol.BatchNorm(fc1, name='batchnorm0')\n        >>> fc2.list_auxiliary_states()\n        ['batchnorm0_moving_mean', 'batchnorm0_moving_var']\n\n        Returns\n        -------\n        aux_states : list of str\n            List of the auxiliary states in input symbol.\n\n        Notes\n        -----\n        Auxiliary states are special states of symbols that do not correspond to an argument,\n        and are not updated by gradient descent. Common examples of auxiliary states\n        include the `moving_mean` and `moving_variance` in `BatchNorm`.\n        Most operators do not have auxiliary states.\n        \"\"\"\n        size = ctypes.c_uint()\n        sarr = ctypes.POINTER(ctypes.c_char_p)()\n        check_call(_LIB.MXSymbolListAuxiliaryStates(\n            self.handle, ctypes.byref(size), ctypes.byref(sarr)))\n        return [py_str(sarr[i]) for i in range(size.value)]\n\n    def list_inputs(self):\n        \"\"\"Lists all arguments and auxiliary states of this Symbol.\n\n        Returns\n        -------\n        inputs : list of str\n            List of all inputs.\n\n        Examples\n        --------\n        >>> bn = mx.sym.BatchNorm(name='bn')\n        >>> bn.list_arguments()\n        ['bn_data', 'bn_gamma', 'bn_beta']\n        >>> bn.list_auxiliary_states()\n        ['bn_moving_mean', 'bn_moving_var']\n        >>> bn.list_inputs()\n        ['bn_data', 'bn_gamma', 'bn_beta', 'bn_moving_mean', 'bn_moving_var']\n        \"\"\"\n        size = ctypes.c_uint()\n        sarr = ctypes.POINTER(ctypes.c_char_p)()\n        check_call(_LIB.NNSymbolListInputNames(\n            self.handle, 0, ctypes.byref(size), ctypes.byref(sarr)))\n        return [py_str(sarr[i]) for i in range(size.value)]\n\n    def infer_type(self, *args, **kwargs):\n        \"\"\"Infers the type of all arguments and all outputs, given the known types\n        for some arguments.\n\n        This function takes the known types of some arguments in either positional way\n        or keyword argument way as input. It returns a tuple of `None` values\n        if there is not enough information to deduce the missing types.\n\n        Inconsistencies in the known types will cause an error to be raised.\n\n        Example\n        -------\n        >>> a = mx.sym.var('a')\n        >>> b = mx.sym.var('b')\n        >>> c = a + b\n        >>> arg_types, out_types, aux_types = c.infer_type(a='float32')\n        >>> arg_types\n        [<type 'numpy.float32'>, <type 'numpy.float32'>]\n        >>> out_types\n        [<type 'numpy.float32'>]\n        >>> aux_types\n        []\n\n        Parameters\n        ----------\n        *args :\n            Type of known arguments in a positional way.\n            Unknown type can be marked as None.\n\n        **kwargs :\n            Keyword arguments of known types.\n\n        Returns\n        -------\n        arg_types : list of numpy.dtype or None\n            List of argument types.\n            The order is same as the order of list_arguments().\n        out_types : list of numpy.dtype or None\n            List of output types.\n            The order is same as the order of list_outputs().\n        aux_types : list of numpy.dtype or None\n            List of auxiliary state types.\n            The order is same as the order of list_auxiliary_states().\n        \"\"\"\n        try:\n            res = self._infer_type_impl(False, *args, **kwargs)\n            if res[1] is None:\n                arg_shapes, _, _ = self._infer_type_impl(True, *args, **kwargs)\n                arg_names = self.list_arguments()\n                unknowns = []\n                for name, dtype in zip(arg_names, arg_shapes):\n                    if not dtype:\n                        if len(unknowns) >= 10:\n                            unknowns.append('...')\n                            break\n                        unknowns.append(f'{name}: {str(dtype)}')\n                warnings.warn(\n                    \"Cannot decide type for the following arguments. \" +\n                    \"Consider providing them as input:\\n\\t\" +\n                    \"\\n\\t\".join(unknowns), stacklevel=2)\n            return res\n        except MXNetError:\n            print(\"infer_type error. Arguments:\")\n            for i, arg in enumerate(args):\n                print(f\"  #{i}: {arg}\")\n            for k, v in kwargs.items():\n                print(f\"  {k}: {v}\")\n            raise\n\n    def infer_type_partial(self, *args, **kwargs):\n        \"\"\"Infers the type partially.\n\n        This functions works the same way as `infer_type`,\n        except that this function can return partial results.\n\n        In the following example, information about fc2 is not available. So, `infer_shape`\n        will return a tuple of `None` values but `infer_shape_partial` will return partial values.\n\n        Example\n        -------\n        >>> data = mx.sym.Variable('data')\n        >>> prev = mx.sym.Variable('prev')\n        >>> casted_prev  = mx.sym.cast(prev, dtype='float32')\n        >>> out  = mx.sym.Activation(data=mx.sym.elemwise_add(data, casted_prev), act_type='relu')\n        >>> out.list_arguments()\n        ['data', 'prev']\n        >>> out.infer_type(data='float32')\n        (None, None, None)\n        >>> out.infer_type_partial(data='float32')\n        ([numpy.float32, None], [numpy.float32], [])\n        >>> # infers type if you give information about prev\n        >>> out.infer_type(data='float32', prev='float16')\n        ([numpy.float32, numpy.float16], [numpy.float32], [])\n\n        Parameters\n        ----------\n        *args :\n            Type of known arguments in a positional way.\n            Unknown type can be marked as None.\n\n        **kwargs :\n            Keyword arguments of known types.\n\n        Returns\n        -------\n        arg_types : list of numpy.dtype or None\n            List of argument types.\n            The order is same as the order of list_arguments().\n        out_types : list of numpy.dtype or None\n            List of output types.\n            The order is same as the order of list_outputs().\n        aux_types : list of numpy.dtype or None\n            List of auxiliary state types.\n            The order is same as the order of list_auxiliary_states().\n        \"\"\"\n        return self._infer_type_impl(True, *args, **kwargs)\n\n    def _infer_type_impl(self, partial, *args, **kwargs):\n        \"\"\"The actual implementation for calling type inference API.\"\"\"\n        # pylint: disable=too-many-locals\n        if len(args) != 0 and len(kwargs) != 0:\n            raise ValueError('Can only specify known argument \\\n                    types either by positional or kwargs way.')\n        sdata = []\n        if len(args) != 0:\n            keys = c_array(ctypes.c_char_p, [])\n            for s in args:\n                if s is not None:\n                    sdata.append(dtype_np_to_mx(s))\n                else:\n                    sdata.append(-1)\n        else:\n            str_keys = []\n            for k, v in kwargs.items():\n                if is_mx_dtype(v):\n                    v = dtype_np_to_mx(v)\n                    str_keys.append(k)\n                    sdata.append(v)\n            keys = c_str_array(str_keys)\n        arg_type_size = mx_uint()\n        arg_type_data = ctypes.POINTER(ctypes.c_int)()\n        out_type_size = mx_uint()\n        out_type_data = ctypes.POINTER(ctypes.c_int)()\n        aux_type_size = mx_uint()\n        aux_type_data = ctypes.POINTER(ctypes.c_int)()\n        complete = ctypes.c_int()\n        if partial:\n            infer_func = _LIB.MXSymbolInferTypePartial\n        else:\n            infer_func = _LIB.MXSymbolInferType\n        check_call(infer_func(\n            self.handle,\n            mx_uint(len(sdata)),\n            keys,\n            c_array_buf(ctypes.c_int, array('i', sdata)),\n            ctypes.byref(arg_type_size),\n            ctypes.byref(arg_type_data),\n            ctypes.byref(out_type_size),\n            ctypes.byref(out_type_data),\n            ctypes.byref(aux_type_size),\n            ctypes.byref(aux_type_data),\n            ctypes.byref(complete)))\n        if complete.value != 0:\n            arg_types = [dtype_mx_to_np(arg_type_data[i]) for i in range(arg_type_size.value)]\n            out_types = [dtype_mx_to_np(out_type_data[i]) for i in range(out_type_size.value)]\n            aux_types = [dtype_mx_to_np(aux_type_data[i]) for i in range(aux_type_size.value)]\n            return (arg_types, out_types, aux_types)\n        else:\n            return (None, None, None)\n\n    def infer_shape(self, *args, **kwargs):\n        \"\"\"Infers the shapes of all arguments and all outputs given the known shapes of\n        some arguments.\n\n        This function takes the known shapes of some arguments in either positional way\n        or keyword argument way as input. It returns a tuple of `None` values\n        if there is not enough information to deduce the missing shapes.\n\n        Example\n        -------\n        >>> a = mx.sym.var('a')\n        >>> b = mx.sym.var('b')\n        >>> c = a + b\n        >>> arg_shapes, out_shapes, aux_shapes = c.infer_shape(a=(3,3))\n        >>> arg_shapes\n        [(3L, 3L), (3L, 3L)]\n        >>> out_shapes\n        [(3L, 3L)]\n        >>> aux_shapes\n        []\n        >>> c.infer_shape(a=(0,3)) # 0s in shape means unknown dimensions. So, returns None.\n        (None, None, None)\n\n        Inconsistencies in the known shapes will cause an error to be raised.\n        See the following example:\n\n        >>> data = mx.sym.Variable('data')\n        >>> out = mx.sym.FullyConnected(data=data, name='fc1', num_hidden=1000)\n        >>> out = mx.sym.Activation(data=out, act_type='relu')\n        >>> out = mx.sym.FullyConnected(data=out, name='fc2', num_hidden=10)\n        >>> weight_shape= (1, 100)\n        >>> data_shape = (100, 100)\n        >>> out.infer_shape(data=data_shape, fc1_weight=weight_shape)\n        Error in operator fc1: Shape inconsistent, Provided=(1,100), inferred shape=(1000,100)\n\n        Parameters\n        ----------\n        *args :\n            Shape of arguments in a positional way.\n            Unknown shape can be marked as None.\n\n        **kwargs :\n            Keyword arguments of the known shapes.\n\n        Returns\n        -------\n        arg_shapes : list of tuple or None\n            List of argument shapes.\n            The order is same as the order of list_arguments().\n        out_shapes : list of tuple or None\n            List of output shapes.\n            The order is same as the order of list_outputs().\n        aux_shapes : list of tuple or None\n            List of auxiliary state shapes.\n            The order is same as the order of list_auxiliary_states().\n        \"\"\"\n        # pylint: disable=too-many-locals\n        try:\n            res = self._infer_shape_impl(False, *args, **kwargs)\n            if res[1] is None:\n                arg_shapes, _, _ = self._infer_shape_impl(True, *args, **kwargs)\n                arg_names = self.list_arguments()\n                unknowns = []\n                for name, shape in zip(arg_names, arg_shapes):\n                    if is_np_shape():\n                        shape_is_none = not shape or -1 in shape\n                    else:\n                        shape_is_none = not shape or 0 in shape\n                    if shape_is_none:\n                        if len(unknowns) >= 10:\n                            unknowns.append('...')\n                            break\n                        unknowns.append(f'{name}: {str(shape)}')\n                warnings.warn(\n                    \"Cannot decide shape for the following arguments \" +\n                    \"(0s in shape means unknown dimensions). \" +\n                    \"Consider providing them as input:\\n\\t\" +\n                    \"\\n\\t\".join(unknowns), stacklevel=2)\n            return res\n        except MXNetError:\n            print(\"infer_shape error. Arguments:\")\n            for i, arg in enumerate(args):\n                print(f\"  #{i}: {arg}\")\n            for k, v in kwargs.items():\n                print(f\"  {k}: {v}\")\n            raise\n\n    def infer_shape_partial(self, *args, **kwargs):\n        \"\"\"Infers the shape partially.\n\n        This functions works the same way as `infer_shape`,\n        except that this function can return partial results.\n\n        In the following example, information about fc2 is not available. So, `infer_shape`\n        will return a tuple of `None` values but `infer_shape_partial` will return partial values.\n\n        Example\n        -------\n        >>> data = mx.sym.Variable('data')\n        >>> prev = mx.sym.Variable('prev')\n        >>> fc1  = mx.sym.FullyConnected(data=data, name='fc1', num_hidden=128)\n        >>> fc2  = mx.sym.FullyConnected(data=prev, name='fc2', num_hidden=128)\n        >>> out  = mx.sym.Activation(data=mx.sym.elemwise_add(fc1, fc2), act_type='relu')\n        >>> out.list_arguments()\n        ['data', 'fc1_weight', 'fc1_bias', 'prev', 'fc2_weight', 'fc2_bias']\n        >>> out.infer_shape(data=(10,64))\n        (None, None, None)\n        >>> out.infer_shape_partial(data=(10,64))\n        ([(10L, 64L), (128L, 64L), (128L,), (), (), ()], [(10L, 128L)], [])\n        >>> # infers shape if you give information about fc2\n        >>> out.infer_shape(data=(10,64), prev=(10,128))\n        ([(10L, 64L), (128L, 64L), (128L,), (10L, 128L), (128L, 128L), (128L,)], [(10L, 128L)], [])\n\n        Parameters\n        ----------\n        *args :\n            Shape of arguments in a positional way.\n            Unknown shape can be marked as None\n\n        **kwargs :\n            Keyword arguments of known shapes.\n\n        Returns\n        -------\n        arg_shapes : list of tuple or None\n            List of argument shapes.\n            The order is same as the order of list_arguments().\n        out_shapes : list of tuple or None\n            List of output shapes.\n            The order is same as the order of list_outputs().\n        aux_shapes : list of tuple or None\n            List of auxiliary state shapes.\n            The order is same as the order of list_auxiliary_states().\n        \"\"\"\n        return self._infer_shape_impl(True, *args, **kwargs)\n\n    def _infer_shape_impl(self, partial, *args, **kwargs):\n        \"\"\"The actual implementation for calling shape inference API.\"\"\"\n        # pylint: disable=too-many-locals\n        if len(args) != 0 and len(kwargs) != 0:\n            raise ValueError('Can only specify known argument \\\n                    shapes either by positional or kwargs way.')\n        sdata = []\n        indptr = [0]\n        if len(args) != 0:\n            keys = c_array(ctypes.c_char_p, [])\n            for i, s in enumerate(args):\n                if s is not None:\n                    if not isinstance(s, tuple):\n                        raise TypeError(\"Arguments need to be shapes (tuple), \"\n                                        f\"but argument {i} is {type(s)}.\")\n                    sdata.extend(s)\n                indptr.append(len(sdata))\n        else:\n            str_keys = []\n            for k, v in kwargs.items():\n                if not isinstance(v, tuple):\n                    raise TypeError(\"Arguments need to be shapes (tuple), \"\n                                    f\"but '{k}' is {type(v)}.\")\n                str_keys.append(k)\n                sdata.extend(v)\n                indptr.append(len(sdata))\n            keys = c_str_array(str_keys)\n        arg_shape_size = mx_uint()\n        arg_shape_ndim = ctypes.POINTER(mx_int)()\n        out_shape_size = mx_uint()\n        out_shape_ndim = ctypes.POINTER(mx_int)()\n        aux_shape_size = mx_uint()\n        aux_shape_ndim = ctypes.POINTER(mx_int)()\n        complete = ctypes.c_int()\n        if _int64_enabled():\n            arg_shape_data = ctypes.POINTER(ctypes.POINTER(mx_int64))()\n            out_shape_data = ctypes.POINTER(ctypes.POINTER(mx_int64))()\n            aux_shape_data = ctypes.POINTER(ctypes.POINTER(mx_int64))()\n            if partial:\n                infer_func = _LIB.MXSymbolInferShapePartial64\n            else:\n                infer_func = _LIB.MXSymbolInferShape64\n            check_call(infer_func(\n                self.handle,\n                mx_uint(len(indptr) - 1),\n                keys,\n                c_array_buf(mx_int64, array('q', indptr)),\n                c_array_buf(mx_int64, array('q', sdata)),\n                ctypes.byref(arg_shape_size),\n                ctypes.byref(arg_shape_ndim),\n                ctypes.byref(arg_shape_data),\n                ctypes.byref(out_shape_size),\n                ctypes.byref(out_shape_ndim),\n                ctypes.byref(out_shape_data),\n                ctypes.byref(aux_shape_size),\n                ctypes.byref(aux_shape_ndim),\n                ctypes.byref(aux_shape_data),\n                ctypes.byref(complete)))\n        else:\n            for size in sdata:\n                if size > _SIGNED_INT32_UPPER_LIMIT:\n                    raise Exception(\"[_infer_shape_impl] Size of tensor you are trying to \" +\n                                    \"allocate is larger than 2^31 elements. Please build \" +\n                                    \"with flag USE_INT64_TENSOR_SIZE=1\")\n            arg_shape_data = ctypes.POINTER(ctypes.POINTER(mx_int))()\n            out_shape_data = ctypes.POINTER(ctypes.POINTER(mx_int))()\n            aux_shape_data = ctypes.POINTER(ctypes.POINTER(mx_int))()\n            if partial:\n                infer_func = _LIB.MXSymbolInferShapePartial\n            else:\n                infer_func = _LIB.MXSymbolInferShape\n            check_call(infer_func(\n                self.handle,\n                mx_uint(len(indptr) - 1),\n                keys,\n                c_array_buf(mx_uint, array('I', indptr)),\n                c_array_buf(mx_int, array('i', sdata)),\n                ctypes.byref(arg_shape_size),\n                ctypes.byref(arg_shape_ndim),\n                ctypes.byref(arg_shape_data),\n                ctypes.byref(out_shape_size),\n                ctypes.byref(out_shape_ndim),\n                ctypes.byref(out_shape_data),\n                ctypes.byref(aux_shape_size),\n                ctypes.byref(aux_shape_ndim),\n                ctypes.byref(aux_shape_data),\n                ctypes.byref(complete)))\n        if complete.value != 0:\n            arg_shapes = [tuple(arg_shape_data[i][:arg_shape_ndim[i]])\n                          if arg_shape_ndim[i] >= 0 else None\n                          for i in range(arg_shape_size.value)]\n            out_shapes = [tuple(out_shape_data[i][:out_shape_ndim[i]])\n                          if out_shape_ndim[i] >= 0 else None\n                          for i in range(out_shape_size.value)]\n            aux_shapes = [tuple(aux_shape_data[i][:aux_shape_ndim[i]])\n                          if aux_shape_ndim[i] >= 0 else None\n                          for i in range(aux_shape_size.value)]\n            return (arg_shapes, out_shapes, aux_shapes)\n        else:\n            return (None, None, None)\n        # pylint: enable=too-many-locals\n\n    def debug_str(self):\n        \"\"\"Gets a debug string of symbol.\n\n        It contains Symbol output, variables and operators in the computation graph\n        with their inputs, variables and attributes.\n\n        Returns\n        -------\n        string\n            Debug string of the symbol.\n\n        Examples\n        --------\n        >>> a = mx.sym.Variable('a')\n        >>> b = mx.sym.sin(a)\n        >>> c = 2 * a + b\n        >>> d = mx.sym.FullyConnected(data=c, num_hidden=10)\n        >>> d.debug_str()\n        >>> print d.debug_str()\n        Symbol Outputs:\n\t        output[0]=fullyconnected0(0)\n        Variable:a\n        --------------------\n        Op:_mul_scalar, Name=_mulscalar0\n        Inputs:\n        \targ[0]=a(0) version=0\n        Attrs:\n        \tscalar=2\n        --------------------\n        Op:sin, Name=sin0\n        Inputs:\n        \targ[0]=a(0) version=0\n        --------------------\n        Op:elemwise_add, Name=_plus0\n        Inputs:\n        \targ[0]=_mulscalar0(0)\n        \targ[1]=sin0(0)\n        Variable:fullyconnected0_weight\n        Variable:fullyconnected0_bias\n        --------------------\n        Op:FullyConnected, Name=fullyconnected0\n        Inputs:\n        \targ[0]=_plus0(0)\n        \targ[1]=fullyconnected0_weight(0) version=0\n        \targ[2]=fullyconnected0_bias(0) version=0\n        Attrs:\n        \tnum_hidden=10\n        \"\"\"\n        debug_str = ctypes.c_char_p()\n        check_call(_LIB.MXSymbolPrint(\n            self.handle, ctypes.byref(debug_str)))\n        return py_str(debug_str.value)\n\n    def save(self, fname, remove_amp_cast=True):\n        \"\"\"Saves symbol to a file.\n\n        You can also use pickle to do the job if you only work on python.\n        The advantage of `load`/`save` functions is that the file contents are language agnostic.\n        This means the model saved by one language binding can be loaded by a different\n        language binding of `MXNet`.\n        You also get the benefit of being able to directly load/save from cloud storage(S3, HDFS).\n\n        Parameters\n        ----------\n        fname : str\n            The name of the file.\n\n            - \"s3://my-bucket/path/my-s3-symbol\"\n            - \"hdfs://my-bucket/path/my-hdfs-symbol\"\n            - \"/path-to/my-local-symbol\"\n        remove_amp_cast : bool, optional\n            Whether to remove the amp_cast and amp_multicast operators, before saving the model.\n\n        See Also\n        --------\n        symbol.load : Used to load symbol from file.\n        \"\"\"\n        if not isinstance(fname, string_types):\n            raise TypeError('fname need to be string')\n        if remove_amp_cast:\n            handle = SymbolHandle()\n            check_call(_LIB.MXSymbolRemoveAmpCast(self.handle, ctypes.byref(handle)))\n            check_call(_LIB.MXSymbolSaveToFile(handle, c_str(fname)))\n        else:\n            check_call(_LIB.MXSymbolSaveToFile(self.handle, c_str(fname)))\n\n    def tojson(self, remove_amp_cast=True):\n        \"\"\"Saves symbol to a JSON string.\n\n        See Also\n        --------\n        symbol.fromjson : Used to load symbol from JSON string.\n        \"\"\"\n        json_str = ctypes.c_char_p()\n        if remove_amp_cast:\n            handle = SymbolHandle()\n            check_call(_LIB.MXSymbolRemoveAmpCast(self.handle, ctypes.byref(handle)))\n            check_call(_LIB.MXSymbolSaveToJSON(handle, ctypes.byref(json_str)))\n        else:\n            check_call(_LIB.MXSymbolSaveToJSON(self.handle, ctypes.byref(json_str)))\n        return py_str(json_str.value)\n\n    @staticmethod\n    def _get_ndarray_inputs(arg_key, args, arg_names, allow_missing):\n        \"\"\"Helper function to get NDArray lists handles from various inputs.\n\n        Parameters\n        ----------\n        arg_key : str\n            The name of argument, used for error message.\n\n        args : list of NDArray or dict of str to NDArray\n            Input arguments to the symbols.\n            If type is list of NDArray, the position is in the same order of arg_names.\n            If type is dict of str to NDArray, then it maps the name of arguments\n            to the corresponding NDArray,\n\n        args_names : list of string\n            List of argument names.\n\n        allow_missing : boolean\n            Whether missing argument is allowed.\n            When allowed, the missing handle will be set to None(null)\n\n        Returns\n        -------\n        handles : list of NDArrayHandle\n            The positional list of NDArrayHandles generated from input.\n        \"\"\"\n        # setup args\n        arg_handles = []\n        arg_arrays = []\n        if isinstance(args, list):\n            if len(args) != len(arg_names):\n                raise ValueError(f'Length of {arg_key} does not match the number of arguments')\n            for narr in args:\n                if narr is None and allow_missing:\n                    arg_handles.append(None)\n                elif not isinstance(narr, NDArray):\n                    raise TypeError('Only accept list of NDArrays or dict of str to NDArray')\n                else:\n                    arg_handles.append(narr.handle)\n            arg_arrays = args\n        elif isinstance(args, dict):\n            for name in arg_names:\n                if name in args:\n                    narr = args[name]\n                    if not isinstance(narr, NDArray):\n                        raise TypeError('Only accept list of NDArrays or dict of str to NDArray')\n                    arg_handles.append(narr.handle)\n                    arg_arrays.append(narr)\n                else:\n                    if allow_missing:\n                        arg_handles.append(None)\n                        arg_arrays.append(None)\n                    else:\n                        raise ValueError(f'key `{name}` is missing in `{arg_key}`')\n        else:\n            raise TypeError('Only accept list of NDArrays or dict of str to NDArray')\n        return c_array(NDArrayHandle, arg_handles), arg_arrays\n\n    def _gen_atomic_symbol(self):\n        handle = SymbolHandle()\n        check_call(_LIB.MXGenAtomicSymbolFromSymbol(self.handle, ctypes.byref(handle)))\n        return Symbol(handle)\n\n\n    # pylint: disable=too-many-locals\n    def optimize_for(self, backend, args=None, aux=None, ctx=None,\n                     shape_dict=None, type_dict=None, stype_dict=None, skip_infer=False, **kwargs):\n        r\"\"\"Partitions current symbol and optimizes it for a given backend.\n\n        The backend must have registered the partitioning graph pass in\n        ``SubgraphBackendRegistry``.\n\n        Parameters\n        ----------\n        backend : str\n            The name of backend, as registered in ``SubgraphBackendRegistry``\n        args : dict of str to NDArray, optional\n            Input arguments to the symbol, required to infer shapes/types before partitioning\n            If type is a dict of str to NDArray, then it maps the names of arguments\n            to the corresponding NDArray. Undefined arguments' NDArrays\n            don't have to be specified in the dict.\n        aux : dict of str to NDArray, optional\n            Input auxiliary arguments to the symbol\n            If type is a dict of str to :class:`NDArray`, then it maps the name of arguments\n            to the corresponding :class:`NDArray`.\n        ctx : Context, optional\n            Device context, used to infer stypes\n        shape_dict : Dict of str->tuple, optional\n            Input shape dictionary.\n            Used iff input :class:`NDArray` is not in ``args``.\n        type_dict : Dict of str->numpy.dtype, optional\n            Input type dictionary.\n            Used iff input :class:`NDArray` is not in ``args``.\n        stype_dict  : Dict of str->str, optional\n            Input storage type dictionary.\n            Used iff input :class:`NDArray` is not in ``args``.\n        skip_infer : bool, optional\n            If True, the optimization skips the shape, type and storage type inference pass.\n        kwargs : optional arguments\n            Passed on to ``PrePartition`` and ``PostPartition`` functions of ``SubgraphProperty``\n\n        Returns\n        -------\n        out : SymbolHandle\n            A symbol with the partitioned graph for target backend.\n        \"\"\"\n        out = SymbolHandle()\n        assert isinstance(backend, str)\n        assert isinstance(args, dict) or args is None\n        assert isinstance(aux, dict) or aux is None\n\n        if args is None or len(args) == 0:\n            args_ = []\n            args_handle = c_array(NDArrayHandle, [])\n        else:\n            args_handle, args_ = self._get_ndarray_inputs('args', args,\n                                                          self.list_arguments(), True)\n\n        if aux is None or len(aux) == 0:\n            aux_ = []\n            aux_handle = c_array(NDArrayHandle, [])\n        else:\n            aux_handle, aux_ = self._get_ndarray_inputs('aux_states', aux,\n                                                        self.list_auxiliary_states(), True)\n        if ctx is None:\n            ctx = current_device()\n        assert isinstance(ctx, Device)\n\n\n        # parse input data shape dict\n        num_input_shapes = 0\n        input_shape_names = ctypes.POINTER(ctypes.c_char_p)()\n        input_shape_data = ctypes.POINTER(mx_int64)()\n        input_shape_idx = ctypes.POINTER(mx_uint)()\n        if shape_dict is not None:\n            input_shape_names = []\n            input_shape_data = []\n            input_shape_idx = [0]\n            for k, v in shape_dict.items():\n                if isinstance(v, (tuple, list)):\n                    input_shape_names.append(k)\n                    input_shape_data.extend(v)\n                    input_shape_idx.append(len(input_shape_data))\n                else:\n                    raise ValueError(str(v) + \" has to be a tuple or list.\")\n            num_input_shapes = mx_uint(len(input_shape_names))\n            input_shape_names = c_str_array(input_shape_names)\n            input_shape_data = c_array_buf(mx_int64, array('q', input_shape_data))\n            input_shape_idx = c_array_buf(mx_uint, array('i', input_shape_idx))\n\n        # parse input data types dict\n        num_input_types = 0\n        input_type_names = ctypes.POINTER(ctypes.c_char_p)()  # provided type argument names\n        input_type_data = ctypes.POINTER(mx_uint)()  # provided types\n        if type_dict is not None:\n            input_type_names = []\n            input_type_data = []\n            for k, v in type_dict.items():\n                v = dtype_np_to_mx(v)\n                input_type_names.append(k)\n                input_type_data.append(v)\n\n            num_input_types = mx_uint(len(input_type_names))\n            input_type_names = c_str_array(input_type_names)\n            input_type_data = c_array_buf(ctypes.c_int, array('i', input_type_data))\n\n        # parse input data storage types dict\n        num_input_stypes = 0\n        # provided storage type argument names\n        input_stype_names = ctypes.POINTER(ctypes.c_char_p)()\n        input_stype_data = ctypes.POINTER(mx_uint)()  # provided storage types\n        if stype_dict is not None:\n            input_stype_names = []\n            input_stype_data = []\n            for k, v in stype_dict.items():\n                if v in _STORAGE_TYPE_STR_TO_ID:\n                    input_stype_names.append(k)\n                    input_stype_data.append(_STORAGE_TYPE_STR_TO_ID[v])\n                else:\n                    raise ValueError(str(v) + \" is not a MXNet storage type.\")\n\n            num_input_stypes = mx_uint(len(input_stype_names))\n            input_stype_names = c_str_array(input_stype_names)\n            input_stype_data = c_array_buf(ctypes.c_int, array('i', input_stype_data))\n\n        new_args_size = ctypes.c_uint()\n        new_arg_names = ctypes.POINTER(ctypes.c_char_p)()\n        new_args_handle = ctypes.POINTER(NDArrayHandle)()\n        new_aux_size = ctypes.c_uint()\n        new_aux_names = ctypes.POINTER(ctypes.c_char_p)()\n        new_aux_handle = ctypes.POINTER(NDArrayHandle)()\n\n        key_list = []\n        val_list = []\n        for key, val in kwargs.items():\n            key_list.append(key)\n            val_list.append(str(val))\n        check_call(_LIB.MXOptimizeForBackend(self.handle,\n                                             c_str(backend),\n                                             ctypes.c_int(ctx.device_typeid),\n                                             ctypes.byref(out),\n                                             mx_uint(len(args_)),\n                                             args_handle,\n                                             mx_uint(len(aux_)),\n                                             aux_handle,\n                                             mx_uint(len(key_list)),\n                                             c_str_array(key_list),\n                                             c_str_array(val_list),\n                                             num_input_shapes,\n                                             input_shape_names,\n                                             input_shape_data,\n                                             input_shape_idx,\n                                             num_input_types,\n                                             input_type_names,\n                                             input_type_data,\n                                             num_input_stypes,\n                                             input_stype_names,\n                                             input_stype_data,\n                                             ctypes.c_bool(skip_infer),\n                                             ctypes.byref(new_args_size),\n                                             ctypes.byref(new_args_handle),\n                                             ctypes.byref(new_arg_names),\n                                             ctypes.byref(new_aux_size),\n                                             ctypes.byref(new_aux_handle),\n                                             ctypes.byref(new_aux_names)))\n        # add new args/aux\n        if not args is None:\n            for i in range(new_args_size.value):\n                args[py_str(new_arg_names[i])] = NDArray(NDArrayHandle(new_args_handle[i]))\n        elif new_args_size.value > 0:\n            raise RuntimeError('Cannot add new args in optimize_for since args is None\\n' +\n                               'Provide a dictionary to the args argument to optimize_for')\n\n        if not aux is None:\n            for i in range(new_aux_size.value):\n                aux[py_str(new_aux_names[i])] = NDArray(NDArrayHandle(new_aux_handle[i]))\n        elif new_aux_size.value > 0:\n            raise RuntimeError('Cannot add new aux in optimize_for since aux is None\\n' +\n                               'Provide a dictionary to the aux argument to optimize_for')\n\n        new_sym = Symbol(out)\n\n        arg_names = self.list_arguments()\n        new_arg_names = new_sym.list_arguments()\n        deleted_arg_names = set([item for item in arg_names\n                                 if item not in set(new_arg_names)])\n\n        if len(deleted_arg_names) > 0:\n            if args is not None:\n                for a_n in deleted_arg_names:\n                    if a_n in args:\n                        args.pop(a_n)\n            else:\n                warnings.warn('A param was deleted during optimization, but no args dictionary was provided.\\n' +\n                              'Please ensure that your model weights match the newly optimized model.')\n\n        aux_names = self.list_auxiliary_states()\n        new_aux_names = new_sym.list_auxiliary_states()\n        deleted_aux_names = set([item for item in aux_names\n                                 if item not in set(new_aux_names)])\n        if len(deleted_aux_names) > 0:\n            if aux is not None:\n                for a_n in deleted_aux_names:\n                    if a_n in aux:\n                        aux.pop(a_n)\n            else:\n                warnings.warn('A param was deleted during optimization, but no args dictionary was provided.\\n' +\n                              'Please ensure that your model weights match the newly optimized model.')\n\n        return new_sym\n\n    # pylint: disable=too-many-locals\n    def _simple_bind(self, ctx, grad_req='write', type_dict=None, stype_dict=None,\n                     **kwargs):\n        \"\"\"Bind current symbol to get an executor, allocate all the arguments needed.\n        Allows specifying data types.\n\n        This function simplifies the binding procedure. You need to specify only input data shapes.\n        Before binding the executor, the function allocates arguments and auxiliary states\n        that were not explicitly specified. Allows specifying data types.\n\n        Example\n        -------\n        >>> x = mx.sym.Variable('x')\n        >>> y = mx.sym.FullyConnected(x, num_hidden=4)\n        >>> exe = y.simple_bind(mx.cpu(), x=(5,4), grad_req='null')\n        >>> exe.forward()\n        [<NDArray 5x4 @cpu(0)>]\n        >>> exe.outputs[0].asnumpy()\n        array([[ 0.,  0.,  0.,  0.],\n               [ 0.,  0.,  0.,  0.],\n               [ 0.,  0.,  0.,  0.],\n               [ 0.,  0.,  0.,  0.],\n               [ 0.,  0.,  0.,  0.]], dtype=float32)\n        >>> exe.arg_arrays\n        [<NDArray 5x4 @cpu(0)>, <NDArray 4x4 @cpu(0)>, <NDArray 4 @cpu(0)>]\n        >>> exe.grad_arrays\n        [<NDArray 5x4 @cpu(0)>, <NDArray 4x4 @cpu(0)>, <NDArray 4 @cpu(0)>]\n\n        Parameters\n        ----------\n        ctx : Context\n            The device context the generated executor to run on.\n\n        grad_req: string\n            {'write', 'add', 'null'}, or list of str or dict of str to str, optional\n            To specify how we should update the gradient to the `args_grad`.\n\n            - 'write' means every time gradient is written to specified `args_grad` NDArray.\n            - 'add' means every time gradient is added to the specified NDArray.\n            - 'null' means no action is taken, the gradient may not be calculated.\n\n        type_dict  : Dict of str->numpy.dtype\n            Input type dictionary, name->dtype\n\n        stype_dict  : Dict of str->str\n            Input storage type dictionary, name->storage_type\n\n        kwargs : Dict of str->shape\n            Input shape dictionary, name->shape\n\n        Returns\n        -------\n        executor : mxnet.Executor\n            The generated executor\n        \"\"\"\n        assert isinstance(grad_req, (str, dict))\n        # infer shape\n        arg_shapes, _, aux_shapes = self.infer_shape(**kwargs)\n        type_dict = {} if type_dict is None else type_dict\n        arg_dtypes, _, _ = None, None, None\n        try:\n            arg_dtypes, _, aux_dtypes = self.infer_type(**type_dict)\n        except Exception: # pylint: disable=broad-except\n            pass\n        args = [None] * len(arg_shapes) if arg_shapes else []\n        aux_states = [None] * len(aux_shapes) if aux_shapes else []\n\n        arg_names = self.list_arguments()\n        aux_names = self.list_auxiliary_states()\n\n        from ..ndarray import zeros as nd_zeros\n        if arg_shapes:\n            for i, shape in enumerate(arg_shapes):\n                if arg_dtypes:\n                    args[i] = nd_zeros(shape, dtype=arg_dtypes[i])\n                else:\n                    args[i] = nd_zeros(shape)\n        if aux_shapes:\n            for i, shape in enumerate(aux_shapes):\n                if aux_dtypes:\n                    aux_states[i] = nd_zeros(shape, dtype=aux_dtypes[i])\n                else:\n                    aux_states[i] = nd_zeros(shape)\n\n        if stype_dict:\n            for name, stype in stype_dict.items():\n                if name in arg_names:\n                    index = arg_names.index(name)\n                    args[index] = args[index].tostype(stype)\n                else:\n                    assert name in aux_names\n                    index = aux_names.index(name)\n                    aux_states[index] = aux_states[index].totype(stype)\n\n        with _profiler_scope(\"symbol:arg_grad:\"):\n            if grad_req == 'null':\n                args_grad = None\n            elif isinstance(grad_req, dict):\n                args_grad = {}\n                for i, name in enumerate(arg_names):\n                    if grad_req[name] != 'null':\n                        args_grad[name] = args[i].copy()\n            else:\n                args_grad = [x.copy() for x in args]\n        return Executor(self, ctx, args, args_grad, grad_req, aux_states)\n\n    def _bind(self, ctx, args, args_grad=None, grad_req='write',\n              aux_states=None, static_alloc=False):\n        \"\"\"Binds the current symbol to an executor and returns it.\n\n        We first declare the computation and then bind to the data to run.\n        This function returns an executor which provides method `forward()` method for evaluation\n        and a `outputs()` method to get all the results.\n\n        Example\n        -------\n        >>> a = mx.sym.Variable('a')\n        >>> b = mx.sym.Variable('b')\n        >>> c = a + b\n        <Symbol _plus1>\n        >>> ex = c._bind(ctx=mx.cpu(), args={'a' : mx.nd.ones([2,3]), 'b' : mx.nd.ones([2,3])})\n        >>> ex.forward()\n        [<NDArray 2x3 @cpu(0)>]\n        >>> ex.outputs[0].asnumpy()\n        [[ 2.  2.  2.]\n        [ 2.  2.  2.]]\n\n        Parameters\n        ----------\n        ctx : Context\n            The device context the generated executor to run on.\n\n        args : list of NDArray or dict of str to NDArray\n            Input arguments to the symbol.\n\n            - If the input type is a list of `NDArray`, the order should be same as the order\n              of `list_arguments()`.\n            - If the input type is a dict of str to `NDArray`, then it maps the name of arguments\n              to the corresponding `NDArray`.\n            - In either case, all the arguments must be provided.\n\n        args_grad : list of NDArray or dict of str to `NDArray`, optional\n            When specified, `args_grad` provides NDArrays to hold\n            the result of gradient value in backward.\n\n            - If the input type is a list of `NDArray`, the order should be same as the order\n              of `list_arguments()`.\n            - If the input type is a dict of str to `NDArray`, then it maps the name of arguments\n              to the corresponding NDArray.\n            - When the type is a dict of str to `NDArray`, one only need to provide the dict\n              for required argument gradient.\n              Only the specified argument gradient will be calculated.\n\n        grad_req : {'write', 'add', 'null'}, or list of str or dict of str to str, optional\n            To specify how we should update the gradient to the `args_grad`.\n\n            - 'write' means everytime gradient is write to specified `args_grad` `NDArray`.\n            - 'add' means everytime gradient is add to the specified NDArray.\n            - 'null' means no action is taken, the gradient may not be calculated.\n\n        aux_states : list of `NDArray`, or dict of str to `NDArray`, optional\n            Input auxiliary states to the symbol, only needed when the output of\n            `list_auxiliary_states()` is not empty.\n\n            - If the input type is a list of `NDArray`, the order should be same as the order\n              of `list_auxiliary_states()`.\n            - If the input type is a dict of str to `NDArray`, then it maps the name of\n              `auxiliary_states` to the corresponding `NDArray`,\n            - In either case, all the auxiliary states need to be provided.\n\n        static_alloc : bool, default False\n            Statically allocate memory to improve speed. Memory usage may increase.\n\n        Returns\n        -------\n        executor : Executor\n            The generated executor\n\n        Notes\n        -----\n        Auxiliary states are the special states of symbols that do not correspond\n        to an argument, and do not have gradient but are still useful\n        for the specific operations. Common examples of auxiliary states include\n        the `moving_mean` and `moving_variance` states in `BatchNorm`.\n        Most operators do not have auxiliary states and in those cases,\n        this parameter can be safely ignored.\n\n        One can give up gradient by using a dict in `args_grad` and only specify\n        gradient they interested in.\n        \"\"\"\n        assert isinstance(grad_req, (str, dict))\n        return Executor(self, ctx, args, args_grad, grad_req, aux_states, static_alloc)\n\n    def gradient(self, wrt):\n        \"\"\"Gets the autodiff of current symbol.\n\n        This function can only be used if current symbol is a loss function.\n\n        .. note:: This function is currently not implemented.\n\n        Parameters\n        ----------\n        wrt : Array of String\n            keyword arguments of the symbol that the gradients are taken.\n\n        Returns\n        -------\n        grad : Symbol\n            A gradient Symbol with returns to be the corresponding gradients.\n        \"\"\"\n        handle = SymbolHandle()\n        c_wrt = c_str_array(wrt)\n        check_call(_LIB.MXSymbolGrad(self.handle,\n                                     mx_uint(len(wrt)),\n                                     c_wrt,\n                                     ctypes.byref(handle)))\n        return Symbol(handle)\n\n    # pylint: enable= no-member\n\n    def eval(self, ctx=None, **kwargs):\n        \"\"\"Evaluates a symbol given arguments.\n\n        The `eval` method combines a call to `bind` (which returns an executor)\n        with a call to `forward` (executor method).\n        For the common use case, where you might repeatedly evaluate with same arguments,\n        eval is slow.\n        In that case, you should call `bind` once and then repeatedly call forward.\n        This function allows simpler syntax for less cumbersome introspection.\n\n        Example\n        -------\n        >>> a = mx.sym.Variable('a')\n        >>> b = mx.sym.Variable('b')\n        >>> c = a + b\n        >>> ex = c.eval(ctx = mx.cpu(), a = mx.nd.ones([2,3]), b = mx.nd.ones([2,3]))\n        >>> ex\n        [<NDArray 2x3 @cpu(0)>]\n        >>> ex[0].asnumpy()\n        array([[ 2.,  2.,  2.],\n               [ 2.,  2.,  2.]], dtype=float32)\n\n        Parameters\n        ----------\n        ctx : Context\n            The device context the generated executor to run on.\n\n        kwargs : Keyword arguments of type `NDArray`\n            Input arguments to the symbol. All the arguments must be provided.\n\n        Returns\n        ----------\n        result :  a list of NDArrays corresponding to the values taken by each symbol when\n        evaluated on given args. When called on a single symbol (not a group),\n        the result will be a list with one element.\n        \"\"\"\n        if ctx is None:\n            ctx = current_device()\n        return self._bind(ctx, kwargs).forward()\n\n    def reshape(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`reshape`.\n\n        The arguments are the same as for :py:func:`reshape`, with\n        this array as data.\n        \"\"\"\n        return op.reshape(self, *args, **kwargs)\n\n    def reshape_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`reshape_like`.\n\n        The arguments are the same as for :py:func:`reshape_like`, with\n        this array as data.\n        \"\"\"\n        return op.reshape_like(self, *args, **kwargs)\n\n    def astype(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`cast`.\n\n        The arguments are the same as for :py:func:`cast`, with\n        this array as data.\n        \"\"\"\n        return op.cast(self, *args, **kwargs)\n\n    def zeros_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`zeros_like`.\n\n        The arguments are the same as for :py:func:`zeros_like`, with\n        this array as data.\n        \"\"\"\n        return op.zeros_like(self, *args, **kwargs)\n\n    def ones_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`ones_like`.\n\n        The arguments are the same as for :py:func:`ones_like`, with\n        this array as data.\n        \"\"\"\n        return op.ones_like(self, *args, **kwargs)\n\n    def broadcast_axes(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`broadcast_axes`.\n\n        The arguments are the same as for :py:func:`broadcast_axes`, with\n        this array as data.\n        \"\"\"\n        return op.broadcast_axes(self, *args, **kwargs)\n\n    def repeat(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`repeat`.\n\n        The arguments are the same as for :py:func:`repeat`, with\n        this array as data.\n        \"\"\"\n        return op.repeat(self, *args, **kwargs)\n\n    def pad(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`pad`.\n\n        The arguments are the same as for :py:func:`pad`, with\n        this array as data.\n        \"\"\"\n        return op.pad(self, *args, **kwargs)\n\n    def swapaxes(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`swapaxes`.\n\n        The arguments are the same as for :py:func:`swapaxes`, with\n        this array as data.\n        \"\"\"\n        return op.swapaxes(self, *args, **kwargs)\n\n    def split(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`split`.\n\n        The arguments are the same as for :py:func:`split`, with\n        this array as data.\n        \"\"\"\n        return op.split(self, *args, **kwargs)\n\n    def split_v2(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`split_v2`.\n\n        The arguments are the same as for :py:func:`split_v2`, with\n        this array as data.\n        \"\"\"\n        return split_v2(self, *args, **kwargs)\n\n    def slice(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`slice`.\n\n        The arguments are the same as for :py:func:`slice`, with\n        this array as data.\n        \"\"\"\n        return op.slice(self, *args, **kwargs)\n\n    def slice_axis(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`slice_axis`.\n\n        The arguments are the same as for :py:func:`slice_axis`, with\n        this array as data.\n        \"\"\"\n        return op.slice_axis(self, *args, **kwargs)\n\n    def slice_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`slice_like`.\n\n        The arguments are the same as for :py:func:`slice_like`, with\n        this array as data.\n        \"\"\"\n        return op.slice_like(self, *args, **kwargs)\n\n    def take(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`take`.\n\n        The arguments are the same as for :py:func:`take`, with\n        this array as data.\n        \"\"\"\n        return op.take(self, *args, **kwargs)\n\n    def one_hot(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`one_hot`.\n\n        The arguments are the same as for :py:func:`one_hot`, with\n        this array as data.\n        \"\"\"\n        return op.one_hot(self, *args, **kwargs)\n\n    def pick(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`pick`.\n\n        The arguments are the same as for :py:func:`pick`, with\n        this array as data.\n        \"\"\"\n        return op.pick(self, *args, **kwargs)\n\n    def sort(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sort`.\n\n        The arguments are the same as for :py:func:`sort`, with\n        this array as data.\n        \"\"\"\n        return op.sort(self, *args, **kwargs)\n\n    def topk(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`topk`.\n\n        The arguments are the same as for :py:func:`topk`, with\n        this array as data.\n        \"\"\"\n        return op.topk(self, *args, **kwargs)\n\n    def argsort(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`argsort`.\n\n        The arguments are the same as for :py:func:`argsort`, with\n        this array as data.\n        \"\"\"\n        return op.argsort(self, *args, **kwargs)\n\n    def argmax(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`argmax`.\n\n        The arguments are the same as for :py:func:`argmax`, with\n        this array as data.\n        \"\"\"\n        return op.argmax(self, *args, **kwargs)\n\n    def argmax_channel(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`argmax_channel`.\n\n        The arguments are the same as for :py:func:`argmax_channel`, with\n        this array as data.\n        \"\"\"\n        return op.argmax_channel(self, *args, **kwargs)\n\n    def argmin(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`argmin`.\n\n        The arguments are the same as for :py:func:`argmin`, with\n        this array as data.\n        \"\"\"\n        return op.argmin(self, *args, **kwargs)\n\n    def clip(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`clip`.\n\n        The arguments are the same as for :py:func:`clip`, with\n        this array as data.\n        \"\"\"\n        return op.clip(self, *args, **kwargs)\n\n    def abs(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`abs`.\n\n        The arguments are the same as for :py:func:`abs`, with\n        this array as data.\n        \"\"\"\n        return op.abs(self, *args, **kwargs)\n\n    def sign(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sign`.\n\n        The arguments are the same as for :py:func:`sign`, with\n        this array as data.\n        \"\"\"\n        return op.sign(self, *args, **kwargs)\n\n    def flatten(self, inplace=False, **kwargs): # pylint: disable=unused-argument\n        \"\"\"Convenience fluent method for :py:func:`flatten`.\n\n        The arguments are the same as for :py:func:`flatten`, with\n        this array as data.\n        \"\"\"\n        return op.flatten(self, **kwargs)\n\n    def shape_array(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`shape_array`.\n\n        The arguments are the same as for :py:func:`shape_op`, with\n        this array as data.\n        \"\"\"\n        return op.shape_array(self, *args, **kwargs)\n\n    def size_array(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`size_array`.\n\n        The arguments are the same as for :py:func:`size_array`, with\n        this array as data.\n        \"\"\"\n        return op.size_array(self, *args, **kwargs)\n\n    def expand_dims(self, axis, inplace=False, **kwargs): # pylint: disable=unused-argument\n        \"\"\"Convenience fluent method for :py:func:`expand_dims`.\n\n        The arguments are the same as for :py:func:`expand_dims`, with\n        this array as data.\n        \"\"\"\n        return op.expand_dims(self, axis=axis, **kwargs)\n\n    def broadcast_to(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`broadcast_to`.\n\n        The arguments are the same as for :py:func:`broadcast_to`, with\n        this array as data.\n        \"\"\"\n        return op.broadcast_to(self, *args, **kwargs)\n\n    def broadcast_like(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`broadcast_like`.\n\n        The arguments are the same as for :py:func:`broadcast_like`, with\n        this array as data.\n        \"\"\"\n        return op.broadcast_like(self, *args, **kwargs)\n\n    def tile(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`tile`.\n\n        The arguments are the same as for :py:func:`tile`, with\n        this array as data.\n        \"\"\"\n        return op.tile(self, *args, **kwargs)\n\n    def transpose(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`transpose`.\n\n        The arguments are the same as for :py:func:`transpose`, with\n        this array as data.\n        \"\"\"\n        return op.transpose(self, *args, **kwargs)\n\n    def flip(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`flip`.\n\n        The arguments are the same as for :py:func:`flip`, with\n        this array as data.\n        \"\"\"\n        return op.flip(self, *args, **kwargs)\n\n    def depth_to_space(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`depth_to_space`.\n\n        The arguments are the same as for :py:func:`depth_to_space`, with\n        this array as data.\n        \"\"\"\n        return op.depth_to_space(self, *args, **kwargs)\n\n    def space_to_depth(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`space_to_depth`.\n\n        The arguments are the same as for :py:func:`space_to_depth`, with\n        this array as data.\n        \"\"\"\n        return op.space_to_depth(self, *args, **kwargs)\n\n    def diag(self, k=0, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`diag`.\n\n        The arguments are the same as for :py:func:`diag`, with\n        this array as data.\n        \"\"\"\n        return op.diag(self, k, **kwargs)\n\n    def sum(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sum`.\n\n        The arguments are the same as for :py:func:`sum`, with\n        this array as data.\n        \"\"\"\n        return op.sum(self, *args, **kwargs)\n\n    def nansum(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`nansum`.\n\n        The arguments are the same as for :py:func:`nansum`, with\n        this array as data.\n        \"\"\"\n        return op.nansum(self, *args, **kwargs)\n\n    def prod(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`prod`.\n\n        The arguments are the same as for :py:func:`prod`, with\n        this array as data.\n        \"\"\"\n        return op.prod(self, *args, **kwargs)\n\n    def nanprod(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`nanprod`.\n\n        The arguments are the same as for :py:func:`nanprod`, with\n        this array as data.\n        \"\"\"\n        return op.nanprod(self, *args, **kwargs)\n\n    def mean(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`mean`.\n\n        The arguments are the same as for :py:func:`mean`, with\n        this array as data.\n        \"\"\"\n        return op.mean(self, *args, **kwargs)\n\n    def max(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`max`.\n\n        The arguments are the same as for :py:func:`max`, with\n        this array as data.\n        \"\"\"\n        return op.max(self, *args, **kwargs)\n\n    def min(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`min`.\n\n        The arguments are the same as for :py:func:`min`, with\n        this array as data.\n        \"\"\"\n        return op.min(self, *args, **kwargs)\n\n    def norm(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`norm`.\n\n        The arguments are the same as for :py:func:`norm`, with\n        this array as data.\n        \"\"\"\n        return op.norm(self, *args, **kwargs)\n\n    def round(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`round`.\n\n        The arguments are the same as for :py:func:`round`, with\n        this array as data.\n        \"\"\"\n        return op.round(self, *args, **kwargs)\n\n    def rint(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`rint`.\n\n        The arguments are the same as for :py:func:`rint`, with\n        this array as data.\n        \"\"\"\n        return op.rint(self, *args, **kwargs)\n\n    def fix(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`fix`.\n\n        The arguments are the same as for :py:func:`fix`, with\n        this array as data.\n        \"\"\"\n        return op.fix(self, *args, **kwargs)\n\n    def floor(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`floor`.\n\n        The arguments are the same as for :py:func:`floor`, with\n        this array as data.\n        \"\"\"\n        return op.floor(self, *args, **kwargs)\n\n    def ceil(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`ceil`.\n\n        The arguments are the same as for :py:func:`ceil`, with\n        this array as data.\n        \"\"\"\n        return op.ceil(self, *args, **kwargs)\n\n    def trunc(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`trunc`.\n\n        The arguments are the same as for :py:func:`trunc`, with\n        this array as data.\n        \"\"\"\n        return op.trunc(self, *args, **kwargs)\n\n    def sin(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sin`.\n\n        The arguments are the same as for :py:func:`sin`, with\n        this array as data.\n        \"\"\"\n        return op.sin(self, *args, **kwargs)\n\n    def cos(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`cos`.\n\n        The arguments are the same as for :py:func:`cos`, with\n        this array as data.\n        \"\"\"\n        return op.cos(self, *args, **kwargs)\n\n    def tan(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`tan`.\n\n        The arguments are the same as for :py:func:`tan`, with\n        this array as data.\n        \"\"\"\n        return op.tan(self, *args, **kwargs)\n\n    def arcsin(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arcsin`.\n\n        The arguments are the same as for :py:func:`arcsin`, with\n        this array as data.\n        \"\"\"\n        return op.arcsin(self, *args, **kwargs)\n\n    def arccos(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arccos`.\n\n        The arguments are the same as for :py:func:`arccos`, with\n        this array as data.\n        \"\"\"\n        return op.arccos(self, *args, **kwargs)\n\n    def arctan(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arctan`.\n\n        The arguments are the same as for :py:func:`arctan`, with\n        this array as data.\n        \"\"\"\n        return op.arctan(self, *args, **kwargs)\n\n    def degrees(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`degrees`.\n\n        The arguments are the same as for :py:func:`degrees`, with\n        this array as data.\n        \"\"\"\n        return op.degrees(self, *args, **kwargs)\n\n    def radians(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`radians`.\n\n        The arguments are the same as for :py:func:`radians`, with\n        this array as data.\n        \"\"\"\n        return op.radians(self, *args, **kwargs)\n\n    def sinh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sinh`.\n\n        The arguments are the same as for :py:func:`sinh`, with\n        this array as data.\n        \"\"\"\n        return op.sinh(self, *args, **kwargs)\n\n    def cosh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`cosh`.\n\n        The arguments are the same as for :py:func:`cosh`, with\n        this array as data.\n        \"\"\"\n        return op.cosh(self, *args, **kwargs)\n\n    def tanh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`tanh`.\n\n        The arguments are the same as for :py:func:`tanh`, with\n        this array as data.\n        \"\"\"\n        return op.tanh(self, *args, **kwargs)\n\n    def arcsinh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arcsinh`.\n\n        The arguments are the same as for :py:func:`arcsinh`, with\n        this array as data.\n        \"\"\"\n        return op.arcsinh(self, *args, **kwargs)\n\n    def arccosh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arccosh`.\n\n        The arguments are the same as for :py:func:`arccosh`, with\n        this array as data.\n        \"\"\"\n        return op.arccosh(self, *args, **kwargs)\n\n    def arctanh(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`arctanh`.\n\n        The arguments are the same as for :py:func:`arctanh`, with\n        this array as data.\n        \"\"\"\n        return op.arctanh(self, *args, **kwargs)\n\n    def exp(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`exp`.\n\n        The arguments are the same as for :py:func:`exp`, with\n        this array as data.\n        \"\"\"\n        return op.exp(self, *args, **kwargs)\n\n    def expm1(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`expm1`.\n\n        The arguments are the same as for :py:func:`expm1`, with\n        this array as data.\n        \"\"\"\n        return op.expm1(self, *args, **kwargs)\n\n    def log(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log`.\n\n        The arguments are the same as for :py:func:`log`, with\n        this array as data.\n        \"\"\"\n        return op.log(self, *args, **kwargs)\n\n    def log10(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log10`.\n\n        The arguments are the same as for :py:func:`log10`, with\n        this array as data.\n        \"\"\"\n        return op.log10(self, *args, **kwargs)\n\n    def log2(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log2`.\n\n        The arguments are the same as for :py:func:`log2`, with\n        this array as data.\n        \"\"\"\n        return op.log2(self, *args, **kwargs)\n\n    def log1p(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log1p`.\n\n        The arguments are the same as for :py:func:`log1p`, with\n        this array as data.\n        \"\"\"\n        return op.log1p(self, *args, **kwargs)\n\n    def log_sigmoid(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log_sigmoid`.\n\n        The arguments are the same as for :py:func:`log_sigmoid`, with\n        this array as data.\n        \"\"\"\n        return op.log_sigmoid(self, *args, **kwargs)\n\n    def mish(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`mish`.\n\n        The arguments are the same as for :py:func:`mish`, with\n        this array as data.\n        \"\"\"\n        return op.mish(self, *args, **kwargs)\n\n    def sqrt(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sqrt`.\n\n        The arguments are the same as for :py:func:`sqrt`, with\n        this array as data.\n        \"\"\"\n        return op.sqrt(self, *args, **kwargs)\n\n    def rsqrt(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`rsqrt`.\n\n        The arguments are the same as for :py:func:`rsqrt`, with\n        this array as data.\n        \"\"\"\n        return op.rsqrt(self, *args, **kwargs)\n\n    def cbrt(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`cbrt`.\n\n        The arguments are the same as for :py:func:`cbrt`, with\n        this array as data.\n        \"\"\"\n        return op.cbrt(self, *args, **kwargs)\n\n    def rcbrt(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`rcbrt`.\n\n        The arguments are the same as for :py:func:`rcbrt`, with\n        this array as data.\n        \"\"\"\n        return op.rcbrt(self, *args, **kwargs)\n\n    def square(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`square`.\n\n        The arguments are the same as for :py:func:`square`, with\n        this array as data.\n        \"\"\"\n        return op.square(self, *args, **kwargs)\n\n    def reciprocal(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`reciprocal`.\n\n        The arguments are the same as for :py:func:`reciprocal`, with\n        this array as data.\n        \"\"\"\n        return op.reciprocal(self, *args, **kwargs)\n\n    def relu(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`relu`.\n\n        The arguments are the same as for :py:func:`relu`, with\n        this array as data.\n        \"\"\"\n        return op.relu(self, *args, **kwargs)\n\n    def sigmoid(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`sigmoid`.\n\n        The arguments are the same as for :py:func:`sigmoid`, with\n        this array as data.\n        \"\"\"\n        return op.sigmoid(self, *args, **kwargs)\n\n    def softmax(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`softmax`.\n\n        The arguments are the same as for :py:func:`softmax`, with\n        this array as data.\n        \"\"\"\n        return op.softmax(self, *args, **kwargs)\n\n    def log_softmax(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`log_softmax`.\n\n        The arguments are the same as for :py:func:`log_softmax`, with\n        this array as data.\n        \"\"\"\n        return op.log_softmax(self, *args, **kwargs)\n\n    def softmin(self, *args, **kwargs):\n        \"\"\"Convenience fluent method for :py:func:`softmin`.\n\n        The arguments are the same as for :py:func:`softmin`, with\n        this array as data.\n        \"\"\"\n        return op.softmin(self, *args, **kwargs)\n\n    def squeeze(self, axis=None, inplace=False, **kwargs): # pylint: disable=unused-argument\n        \"\"\"Convenience fluent method for :py:func:`squeeze`.\n\n        The arguments are the same as for :py:func:`squeeze`, with\n        this array as data.\n        \"\"\"\n        return op.squeeze(self, axis=axis, **kwargs)\n\n    def get_backend_symbol(self, backend):\n        \"\"\"Return symbol for target backend.\n\n        Parameters\n        ----------\n        backend : str\n            The backend names.\n\n        Returns\n        -------\n        out : Symbol\n            The created Symbol for target backend.\n        \"\"\"\n        out = SymbolHandle()\n        check_call(_LIB.MXGenBackendSubgraph(self.handle, c_str(backend), ctypes.byref(out)))\n        return Symbol(out)\n\n    def wait_to_read(self):\n        raise NotImplementedForSymbol(self.wait_to_read, None)\n\n    def asnumpy(self):\n        raise NotImplementedForSymbol(self.asnumpy, None)\n\n    def asscalar(self):\n        raise NotImplementedForSymbol(self.asscalar, None)\n\n    def copy(self):\n        raise NotImplementedForSymbol(self.copy, None)\n\n    def as_in_context(self):\n        raise NotImplementedForSymbol(self.as_in_context, None)\n\n    def detach(self):\n        raise NotImplementedForSymbol(self.detach, None)\n\n    def backward(self):\n        raise NotImplementedForSymbol(self.backward, None)\n\n\n    def has_dynamic_shape_op(self):\n        \"\"\"Check if any dynamic shape op is present in the symbol.\n        \"\"\"\n        has_dynamic_shape = ctypes.c_bool(False)\n        check_call(_LIB.MXCheckDynamicShapeOp(self.handle,\n                                              ctypes.byref(has_dynamic_shape)))\n        return has_dynamic_shape.value\n\ndef var(name, attr=None, shape=None, lr_mult=None, wd_mult=None, dtype=None,\n        init=None, stype=None, profiler_scope=None, **kwargs):\n    \"\"\"Creates a symbolic variable with specified name.\n\n    Example\n    -------\n    >>> data = mx.sym.Variable('data', attr={'a': 'b'})\n    >>> data\n    <Symbol data>\n    >>> csr_data = mx.sym.Variable('csr_data', stype='csr')\n    >>> csr_data\n    <Symbol csr_data>\n    >>> row_sparse_weight = mx.sym.Variable('weight', stype='row_sparse')\n    >>> row_sparse_weight\n    <Symbol weight>\n\n    Parameters\n    ----------\n    name : str\n        Variable name.\n    attr : Dict of strings\n        Additional attributes to set on the variable. Format {string : string}.\n    shape : tuple\n        The shape of a variable. If specified, this will be used during the shape inference.\n        If one has specified a different shape for this variable using\n        a keyword argument when calling shape inference, this shape information will be ignored.\n    lr_mult : float\n        The learning rate multiplier for input variable.\n    wd_mult : float\n        Weight decay multiplier for input variable.\n    dtype : str or numpy.dtype\n        The dtype for input variable. If not specified, this value will be inferred.\n    init : initializer (mxnet.init.*)\n        Initializer for this variable to (optionally) override the default initializer.\n    stype : str\n        The storage type of the variable, such as 'row_sparse', 'csr', 'default', etc\n    profiler_scope : str\n        The profiler scope for input variable.\n    kwargs : Additional attribute variables\n        Additional attributes must start and end with double underscores.\n\n    Returns\n    -------\n    variable : Symbol\n        A symbol corresponding to an input to the computation graph.\n    \"\"\"\n    if not isinstance(name, string_types):\n        raise TypeError('Expect a string for variable `name`')\n    handle = SymbolHandle()\n    check_call(_LIB.MXSymbolCreateVariable(c_str(name), ctypes.byref(handle)))\n    ret = Symbol(handle)\n    attr = attribute.current().get(attr)\n    attr = {} if attr is None else attr\n    if shape is not None:\n        attr['__shape__'] = str(shape)\n    if lr_mult is not None:\n        attr['__lr_mult__'] = str(lr_mult)\n    if wd_mult is not None:\n        attr['__wd_mult__'] = str(wd_mult)\n    if dtype is not None:\n        attr['__dtype__'] = str(dtype_np_to_mx(dtype))\n    if init is not None:\n        if not isinstance(init, string_types):\n            init = init.dumps()\n        attr['__init__'] = init\n    if stype is not None:\n        attr['__storage_type__'] = str(_STORAGE_TYPE_STR_TO_ID[stype])\n    if profiler_scope is not None:\n        attr['__profiler_scope__'] = profiler_scope\n    else:\n        attr['__profiler_scope__'] = _current_profiler_scope.get()\n    for k, v in kwargs.items():\n        if k.startswith('__') and k.endswith('__'):\n            attr[k] = str(v)\n        else:\n            raise ValueError(f'Attribute name={k} is not supported.'\n                             ' Additional attributes must start and end with double underscores,'\n                             ' e.g, __yourattr__')\n    ret._set_attr(**attr)\n    return ret\n\n\n# for back compatibility\nVariable = var\n\n\ndef Group(symbols, create_fn=Symbol):\n    \"\"\"Creates a symbol that contains a collection of other symbols, grouped together.\n    A classic symbol (`mx.sym.Symbol`) will be returned if all the symbols in the list\n    are of that type; a numpy symbol (`mx.sym.np._Symbol`) will be returned if all the\n    symbols in the list are of that type. A type error will be raised if a list of mixed\n    classic and numpy symbols are provided.\n\n    Example\n    -------\n    >>> a = mx.sym.Variable('a')\n    >>> b = mx.sym.Variable('b')\n    >>> mx.sym.Group([a,b])\n    <Symbol Grouped>\n\n    Parameters\n    ----------\n    symbols : list\n        List of symbols to be grouped.\n\n    create_fn : mx.sym.Symbol or mx.sym.np._Symbol\n        Symbol class for creating the grouped symbol.\n\n    Returns\n    -------\n    sym : Symbol\n        A group symbol.\n     \"\"\"\n    if not symbols or any(not isinstance(sym, Symbol) for sym in symbols):\n        raise TypeError('Expected a list of symbols as input')\n    handle = SymbolHandle()\n    check_call(_LIB.MXSymbolCreateGroup(\n        mx_uint(len(symbols)),\n        c_handle_array(symbols), ctypes.byref(handle)))\n    return create_fn(handle)\n\n\ndef load(fname):\n    \"\"\"Loads symbol from a JSON file.\n\n    You can also use pickle to do the job if you only work on python.\n    The advantage of load/save is the file is language agnostic.\n    This means the file saved using save can be loaded by other language binding of mxnet.\n    You also get the benefit being able to directly load/save from cloud storage(S3, HDFS).\n\n    Parameters\n    ----------\n    fname : str\n        The name of the file, examples:\n\n        - `s3://my-bucket/path/my-s3-symbol`\n        - `hdfs://my-bucket/path/my-hdfs-symbol`\n        - `/path-to/my-local-symbol`\n\n    Returns\n    -------\n    sym : Symbol\n        The loaded symbol.\n\n    See Also\n    --------\n    Symbol.save : Used to save symbol into file.\n    \"\"\"\n    if not isinstance(fname, string_types):\n        raise TypeError('fname need to be string')\n    handle = SymbolHandle()\n    check_call(_LIB.MXSymbolCreateFromFile(c_str(fname), ctypes.byref(handle)))\n    return Symbol(handle)\n\n\ndef fromjson(json_str):\n    \"\"\"Loads symbol from json string.\n\n    Parameters\n    ----------\n    json_str : str\n        A JSON string.\n\n    Returns\n    -------\n    sym : Symbol\n        The loaded symbol.\n\n    See Also\n    --------\n    Symbol.tojson : Used to save symbol into json string.\n    \"\"\"\n    if not isinstance(json_str, string_types):\n        raise TypeError('fname required to be string')\n    handle = SymbolHandle()\n    check_call(_LIB.MXSymbolCreateFromJSON(c_str(json_str), ctypes.byref(handle)))\n    return Symbol(handle)\n\n\n# pylint: disable=no-member\n# pylint: disable=redefined-builtin\ndef pow(base, exp):\n    \"\"\"Returns element-wise result of base element raised to powers from exp element.\n\n    Both inputs can be Symbol or scalar number.\n    Broadcasting is not supported. Use `broadcast_pow` instead.\n\n    `sym.pow` is being deprecated, please use `sym.power` instead.\n\n    Parameters\n    ---------\n    base : Symbol or scalar\n        The base symbol\n    exp : Symbol or scalar\n        The exponent symbol\n\n    Returns\n    -------\n    Symbol or scalar\n        The bases in x raised to the exponents in y.\n\n    Examples\n    --------\n    >>> mx.sym.pow(2, 3)\n    8\n    >>> x = mx.sym.Variable('x')\n    >>> y = mx.sym.Variable('y')\n    >>> z = mx.sym.pow(x, 2)\n    >>> z.eval(x=mx.nd.array([1,2]))[0].asnumpy()\n    array([ 1.,  4.], dtype=float32)\n    >>> z = mx.sym.pow(3, y)\n    >>> z.eval(y=mx.nd.array([2,3]))[0].asnumpy()\n    array([  9.,  27.], dtype=float32)\n    >>> z = mx.sym.pow(x, y)\n    >>> z.eval(x=mx.nd.array([3,4]), y=mx.nd.array([2,3]))[0].asnumpy()\n    array([  9.,  64.], dtype=float32)\n    \"\"\"\n    if isinstance(base, Symbol) and isinstance(exp, Symbol):\n        return _internal._Power(base, exp)\n    if isinstance(base, Symbol) and isinstance(exp, Number):\n        return _internal._PowerScalar(base, scalar=exp)\n    if isinstance(base, Number) and isinstance(exp, Symbol):\n        return _internal._RPowerScalar(exp, scalar=base)\n    if isinstance(base, Number) and isinstance(exp, Number):\n        return base**exp\n    else:\n        raise TypeError(f'types ({str(type(base))}, {str(type(exp))}) not supported')\n\n\ndef power(base, exp):\n    \"\"\"Returns element-wise result of base element raised to powers from exp element.\n\n    Both inputs can be Symbol or scalar number.\n    Broadcasting is not supported. Use `broadcast_pow` instead.\n\n    Parameters\n    ---------\n    base : Symbol or scalar\n        The base symbol\n    exp : Symbol or scalar\n        The exponent symbol\n\n    Returns\n    -------\n    Symbol or scalar\n        The bases in x raised to the exponents in y.\n\n    Examples\n    --------\n    >>> mx.sym.power(2, 3)\n    8\n    >>> x = mx.sym.Variable('x')\n    >>> y = mx.sym.Variable('y')\n    >>> z = mx.sym.power(x, 2)\n    >>> z.eval(x=mx.nd.array([1,2]))[0].asnumpy()\n    array([ 1.,  4.], dtype=float32)\n    >>> z = mx.sym.power(3, y)\n    >>> z.eval(y=mx.nd.array([2,3]))[0].asnumpy()\n    array([  9.,  27.], dtype=float32)\n    >>> z = mx.sym.power(x, y)\n    >>> z.eval(x=mx.nd.array([3,4]), y=mx.nd.array([2,3]))[0].asnumpy()\n    array([  9.,  64.], dtype=float32)\n    \"\"\"\n    return pow(base, exp)\n\n\n# pylint: disable=no-member\n# pylint: disable=redefined-builtin\ndef maximum(left, right):\n    \"\"\"Returns element-wise maximum of the input elements.\n\n    Both inputs can be Symbol or scalar number. Broadcasting is not supported.\n\n    Parameters\n    ---------\n    left : Symbol or scalar\n        First symbol to be compared.\n    right : Symbol or scalar\n        Second symbol to be compared.\n\n    Returns\n    -------\n    Symbol or scalar\n        The element-wise maximum of the input symbols.\n\n    Examples\n    --------\n    >>> mx.sym.maximum(2, 3.5)\n    3.5\n    >>> x = mx.sym.Variable('x')\n    >>> y = mx.sym.Variable('y')\n    >>> z = mx.sym.maximum(x, 4)\n    >>> z.eval(x=mx.nd.array([3,5,2,10]))[0].asnumpy()\n    array([  4.,   5.,   4.,  10.], dtype=float32)\n    >>> z = mx.sym.maximum(x, y)\n    >>> z.eval(x=mx.nd.array([3,4]), y=mx.nd.array([10,2]))[0].asnumpy()\n    array([ 10.,   4.], dtype=float32)\n    \"\"\"\n    if isinstance(left, Symbol) and isinstance(right, Symbol):\n        return _internal._Maximum(left, right)\n    if isinstance(left, Symbol) and isinstance(right, Number):\n        return _internal._MaximumScalar(left, scalar=right)\n    if isinstance(left, Number) and isinstance(right, Symbol):\n        return _internal._MaximumScalar(right, scalar=left)\n    if isinstance(left, Number) and isinstance(right, Number):\n        return left if left > right else right\n    else:\n        raise TypeError(f'types ({str(type(left))}, {str(type(right))}) not supported')\n\n\n# pylint: disable=no-member\n# pylint: disable=redefined-builtin\ndef minimum(left, right):\n    \"\"\"Returns element-wise minimum of the input elements.\n\n    Both inputs can be Symbol or scalar number. Broadcasting is not supported.\n\n    Parameters\n    ---------\n    left : Symbol or scalar\n        First symbol to be compared.\n    right : Symbol or scalar\n        Second symbol to be compared.\n\n    Returns\n    -------\n    Symbol or scalar\n        The element-wise minimum of the input symbols.\n\n    Examples\n    --------\n    >>> mx.sym.minimum(2, 3.5)\n    2\n    >>> x = mx.sym.Variable('x')\n    >>> y = mx.sym.Variable('y')\n    >>> z = mx.sym.minimum(x, 4)\n    >>> z.eval(x=mx.nd.array([3,5,2,10]))[0].asnumpy()\n    array([ 3.,  4.,  2.,  4.], dtype=float32)\n    >>> z = mx.sym.minimum(x, y)\n    >>> z.eval(x=mx.nd.array([3,4]), y=mx.nd.array([10,2]))[0].asnumpy()\n    array([ 3.,  2.], dtype=float32)\n    \"\"\"\n    if isinstance(left, Symbol) and isinstance(right, Symbol):\n        return _internal._Minimum(left, right)\n    if isinstance(left, Symbol) and isinstance(right, Number):\n        return _internal._MinimumScalar(left, scalar=right)\n    if isinstance(left, Number) and isinstance(right, Symbol):\n        return _internal._MinimumScalar(right, scalar=left)\n    if isinstance(left, Number) and isinstance(right, Number):\n        return left if left < right else right\n    else:\n        raise TypeError(f'types ({str(type(left))}, {str(type(right))}) not supported')\n\n\n# pylint: disable=no-member\n# pylint: disable=redefined-builtin\ndef hypot(left, right):\n    \"\"\"Given the \"legs\" of a right triangle, returns its hypotenuse.\n\n    Equivalent to :math:`\\\\sqrt(left^2 + right^2)`, element-wise.\n    Both inputs can be Symbol or scalar number. Broadcasting is not supported.\n\n    Parameters\n    ---------\n    left : Symbol or scalar\n        First leg of the triangle(s).\n    right : Symbol or scalar\n        Second leg of the triangle(s).\n\n    Returns\n    -------\n    Symbol or scalar\n        The hypotenuse of the triangle(s)\n\n    Examples\n    --------\n    >>> mx.sym.hypot(3, 4)\n    5.0\n    >>> x = mx.sym.Variable('x')\n    >>> y = mx.sym.Variable('y')\n    >>> z = mx.sym.hypot(x, 4)\n    >>> z.eval(x=mx.nd.array([3,5,2]))[0].asnumpy()\n    array([ 5.,  6.40312433,  4.47213602], dtype=float32)\n    >>> z = mx.sym.hypot(x, y)\n    >>> z.eval(x=mx.nd.array([3,4]), y=mx.nd.array([10,2]))[0].asnumpy()\n    array([ 10.44030666,   4.47213602], dtype=float32)\n    \"\"\"\n    if isinstance(left, Symbol) and isinstance(right, Symbol):\n        return _internal._Hypot(left, right)\n    if isinstance(left, Symbol) and isinstance(right, Number):\n        return _internal._HypotScalar(left, scalar=right)\n    if isinstance(left, Number) and isinstance(right, Symbol):\n        return _internal._HypotScalar(right, scalar=left)\n    if isinstance(left, Number) and isinstance(right, Number):\n        return _numpy.hypot(left, right)\n    else:\n        raise TypeError(f'types ({str(type(left))}, {str(type(right))}) not supported')\n\n\ndef eye(N, M=0, k=0, dtype=None, **kwargs):\n    \"\"\"Returns a new symbol of 2-D shpae, filled with ones on the diagonal and zeros elsewhere.\n\n    Parameters\n    ----------\n    N: int\n        Number of rows in the output.\n    M: int, optional\n        Number of columns in the output. If 0, defaults to N.\n    k: int, optional\n        Index of the diagonal: 0 (the default) refers to the main diagonal,\n        a positive value refers to an upper diagonal,\n        and a negative value to a lower diagonal.\n    dtype : str or numpy.dtype, optional\n        The value type of the inner value, default to ``np.float32``.\n\n    Returns\n    -------\n    out : Symbol\n        The created Symbol.\n    \"\"\"\n    if dtype is None:\n        dtype = _numpy.float32\n    return _internal._eye(N, M, k, dtype=dtype, **kwargs)\n\ndef zeros(shape, dtype=None, **kwargs):\n    \"\"\"Returns a new symbol of given shape and type, filled with zeros.\n\n    Parameters\n    ----------\n    shape :  int or sequence of ints\n        Shape of the new array.\n    dtype : str or numpy.dtype, optional\n        The value type of the inner value, default to ``np.float32``.\n\n    Returns\n    -------\n    out : Symbol\n        The created Symbol.\n    \"\"\"\n    if dtype is None:\n        dtype = _numpy.float32\n    return _internal._zeros(shape=shape, dtype=dtype, **kwargs)\n\n\ndef ones(shape, dtype=None, **kwargs):\n    \"\"\"Returns a new symbol of given shape and type, filled with ones.\n\n    Parameters\n    ----------\n    shape :  int or sequence of ints\n        Shape of the new array.\n    dtype : str or numpy.dtype, optional\n        The value type of the inner value, default to ``np.float32``.\n\n    Returns\n    -------\n    out : Symbol\n        The created Symbol\n    \"\"\"\n    if dtype is None:\n        dtype = _numpy.float32\n    return _internal._ones(shape=shape, dtype=dtype, **kwargs)\n\n\ndef full(shape, val, dtype=None, **kwargs):\n    \"\"\"Returns a new array of given shape and type, filled with the given value `val`.\n\n    Parameters\n    ----------\n    shape :  int or sequence of ints\n        Shape of the new array.\n    val : scalar\n        Fill value.\n    dtype : str or numpy.dtype, optional\n        The value type of the inner value, default to ``np.float32``.\n\n    Returns\n    -------\n    out : Symbol\n        The created Symbol\n    \"\"\"\n    if dtype is None:\n        dtype = _numpy.float32\n    return _internal._full(shape=shape, dtype=dtype, value=float(val), **kwargs)\n\n# pylint: disable=redefined-outer-name\ndef arange(start, stop=None, step=1.0, repeat=1, infer_range=False, name=None, dtype=None):\n    \"\"\"Returns evenly spaced values within a given interval.\n\n    Values are generated within the half-open interval [`start`, `stop`). In other\n    words, the interval includes `start` but excludes `stop`. The function is\n    similar to the built-in Python function `range` and to `numpy.arange`,\n    but returns a `Symbol`.\n\n    Parameters\n    ----------\n    start : number, optional\n        Start of interval. The interval includes this value. The default start value is 0.\n    stop : number\n        End of interval. The interval does not include this value.\n    step : number, optional\n        Spacing between values.\n    repeat : int, optional\n        \"The repeating time of all elements.\n        E.g repeat=3, the element a will be repeated three times --> a, a, a.\n    infer_range : boolean, optional\n        When set to True, infer the stop position from the start, step,\n        repeat, and output tensor size.\n    dtype : str or numpy.dtype, optional\n        The value type of the inner value, default to ``np.float32``.\n\n    Returns\n    -------\n    out : Symbol\n        The created Symbol\n    \"\"\"\n    if dtype is None:\n        dtype = _numpy.float32\n    return _internal._arange(start=start, stop=stop, step=step, repeat=repeat,\n                             infer_range=infer_range, name=name, dtype=dtype)\n\ndef linspace(start, stop, num, endpoint=True, name=None, dtype=None):\n    \"\"\"Return evenly spaced numbers within a specified interval.\n\n    Values are generated within the half-open interval [`start`, `stop`) or\n    closed interval [start, stop] depending on whether `endpoint` is True or\n    False. The function is similar to `numpy.linspace`, but returns a `Symbol`.\n\n    Parameters\n    ----------\n    start : number\n        Start of interval.\n    stop : number\n        End of interval, unless endpoint is set to False.  In that case,\n        the sequence consists of all but the last of `num + 1` evenly spaced\n        samples, so that stop is excluded. Note that the step size changes\n        when endpoint is False.\n    num : number\n        Number of samples to generate. Must be non-negative.\n    endpoint : bool\n        If True, stop is the last sample. Otherwise, it is not included.\n        The default is True.\n    dtype : str or numpy.dtype, optional\n        The data type of the `NDArray`. The default datatype is `np.float32`.\n\n    Returns\n    -------\n    out : Symbol\n        The created Symbol\n    \"\"\"\n    if dtype is None:\n        dtype = _numpy.float32\n    return _internal._linspace(start=start, stop=stop, num=num, endpoint=endpoint,\n                               name=name, dtype=dtype)\n\ndef histogram(a, bins=10, range=None, **kwargs):\n    \"\"\"Compute the histogram of the input data.\n\n    Parameters\n    ----------\n    a : NDArray\n        Input data. The histogram is computed over the flattened array.\n    bins : int or sequence of scalars\n        If bins is an int, it defines the number of equal-width bins in the\n        given range (10, by default). If bins is a sequence, it defines the bin edges,\n        including the rightmost edge, allowing for non-uniform bin widths.\n    range : (float, float), required if bins is an integer\n        The lower and upper range of the bins. If not provided, range is simply (a.min(), a.max()).\n        Values outside the range are ignored. The first element of the range must be less than or\n        equal to the second. range affects the automatic bin computation as well, the range will\n        be equally divided by the number of bins.\n\n    Returns\n    -------\n    out : Symbol\n        The created Symbol\n    \"\"\"\n    if isinstance(bins, Symbol):\n        return _internal._histogram(data=a, bins=bins, **kwargs)\n    elif isinstance(bins, integer_types):\n        if range is None:\n            raise ValueError(\"null range is not supported in symbol mode\")\n        return _internal._histogram(data=a, bin_cnt=bins, range=range, **kwargs)\n    raise ValueError(\"bins argument should be either an integer or an NDArray\")\n\ndef split_v2(ary, indices_or_sections, axis=0, squeeze_axis=False):\n    \"\"\"Split an array into multiple sub-arrays.\n\n    Parameters\n    ----------\n    ary : NDArray\n        Array to be divided into sub-arrays.\n    indices_or_sections : int or tuple of ints\n        If `indices_or_sections` is an integer, N, the array will be divided\n        into N equal arrays along `axis`.  If such a split is not possible,\n        an error is raised.\n        If `indices_or_sections` is a 1-D array of sorted integers, the entries\n        indicate where along `axis` the array is split.  For example,\n        ``[2, 3]`` would, for ``axis=0``, result in\n        - ary[:2]\n        - ary[2:3]\n        - ary[3:]\n        If an index exceeds the dimension of the array along `axis`,\n        an empty sub-array is returned correspondingly.\n    axis : int, optional\n        The axis along which to split, default is 0.\n    squeeze_axis: boolean, optional\n        Whether to squeeze the axis of sub-arrays or not, only useful when size\n        of the sub-arrays are 1 on the `axis`. Default is False.\n\n    Returns\n    -------\n    out : Symbol\n        The created Symbol\n    \"\"\"\n    indices = []\n    sections = 0\n    if isinstance(indices_or_sections, int):\n        sections = indices_or_sections\n    elif isinstance(indices_or_sections, tuple):\n        indices = [0] + list(indices_or_sections)\n    else:\n        raise ValueError('indices_or_sections must either int or tuple of ints')\n    return _internal._split_v2(ary, indices, axis, squeeze_axis, sections)\n\n_set_symbol_class(Symbol)\n"
  },
  {
    "path": "python/mxnet/symbol_doc.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=unused-argument, too-many-arguments\n\"\"\"Extra symbol documents\n\nGuidelines\n----------\n\nTo add extra doc to the operator `XXX`, write a class `XXXDoc`, deriving\nfrom the base class `SymbolDoc`, and put the extra doc as the docstring\nof `XXXDoc`.\n\nThe document added here should be Python-specific. Documents that are useful\nfor all language bindings should be added to the C++ side where the operator\nis defined / registered.\n\nThe code snippet in the docstring will be run using `doctest`. During running,\nthe environment will have access to\n\n- all the global names in this file (e.g. `SymbolDoc`)\n- all the operators (e.g. `FullyConnected`)\n- the name `test_utils` for `mx.test_utils` (e.g. `test_utils.reldiff`)\n- the name `mx` (e.g. `mx.nd.zeros`)\n- the name `np`\n\nThe following documents are recommended:\n\n- *Examples*: simple and short code snippet showing how to use this operator.\n  It should show typical calling examples and behaviors (e.g. maps an input\n  of what shape to an output of what shape).\n\"\"\"\nfrom __future__ import absolute_import as _abs\nimport re as _re\nfrom .base import build_param_doc as _build_param_doc\n\nclass SymbolDoc(object):\n    \"\"\"The base class for attaching doc to operators.\"\"\"\n\n    @staticmethod\n    def get_output_shape(sym, **input_shapes):\n        \"\"\"Get user friendly information of the output shapes.\"\"\"\n        _, s_outputs, _ = sym.infer_shape(**input_shapes)\n        return dict(zip(sym.list_outputs(), s_outputs))\n\ndef _build_doc(func_name,\n               desc,\n               arg_names,\n               arg_types,\n               arg_desc,\n               key_var_num_args=None,\n               ret_type=None):\n    \"\"\"Build docstring for symbolic functions.\"\"\"\n    param_str = _build_param_doc(arg_names, arg_types, arg_desc)\n    if key_var_num_args:\n        desc += '\\nThis function support variable length of positional input.'\n    doc_str = (f'{desc}\\n\\n' +\n               f'{param_str}\\n' +\n               'name : string, optional.\\n' +\n               '    Name of the resulting symbol.\\n\\n' +\n               'Returns\\n' +\n               '-------\\n' +\n               'Symbol\\n' +\n               '    The result symbol.')\n    extra_doc = \"\\n\" + '\\n'.join([x.__doc__ for x in type.__subclasses__(SymbolDoc)\n                                  if x.__name__ == f'{func_name}Doc'])\n    doc_str += _re.sub(_re.compile(\"    \"), \"\", extra_doc)\n    doc_str = _re.sub('NDArray-or-Symbol', 'Symbol', doc_str)\n    return doc_str\n"
  },
  {
    "path": "python/mxnet/test_utils.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n\"\"\"Tools for testing.\"\"\"\n# pylint: disable=too-many-lines\nimport time\nimport gzip\nimport struct\nimport traceback\nimport numbers\nimport sys\nimport os\nimport platform\nimport errno\nimport logging\nimport bz2\nimport zipfile\nimport json\nfrom contextlib import contextmanager\nfrom collections import OrderedDict\nimport numpy as np\nimport numpy.testing as npt\nimport numpy.random as rnd\ntry:\n    import scipy.stats as ss\nexcept ImportError:\n    ss = None\ntry:\n    import requests\nexcept ImportError:\n    # in rare cases requests may be not installed\n    pass\nimport mxnet as mx\nfrom .device import current_device\nfrom .ndarray.ndarray import _STORAGE_TYPE_STR_TO_ID, get_dtype_name\nfrom .symbol import Symbol\nfrom .symbol.numpy import _Symbol as np_symbol\nfrom .util import use_np, use_np_default_dtype, getenv, setenv  # pylint: disable=unused-import\nfrom .util import get_max_supported_compute_capability, get_rtc_compile_opts # pylint: disable=unused-import\nfrom .runtime import Features\nfrom .numpy_extension import get_cuda_compute_capability\n\n\ndef default_device():\n    \"\"\"Get default device for regression test.\"\"\"\n    # _TODO: get device from environment variable to support\n    # testing with GPUs\n    return current_device()\n\n\ndef set_default_device(device):\n    \"\"\"Set default device.\"\"\"\n    mx.device._current.set(device)\n\n\ndef default_dtype():\n    \"\"\"Get default data type for regression test.\"\"\"\n    # _TODO: get default dtype from environment variable\n    return np.float32\n\ndef default_rtols():\n    \"\"\"Get default relative tolerances for data comparisons involving each data type.\"\"\"\n    return {np.dtype(np.float16): 1e-2,\n            np.dtype(np.float32): 1e-4,\n            np.dtype(np.float64): 1e-5,\n            np.dtype(np.bool): 0,\n            np.dtype(np.int8): 0,\n            np.dtype(np.uint8): 0,\n            np.dtype(np.int32): 0,\n            np.dtype(np.uint32): 0,\n            np.dtype(np.int64): 0,\n            np.dtype(np.uint64): 0}\n\ndef default_atols():\n    \"\"\"Get default absolute tolerances for data comparisons involving each data type.\"\"\"\n    return {np.dtype(np.float16): 1e-1,\n            np.dtype(np.float32): 1e-3,\n            np.dtype(np.float64): 1e-20,\n            np.dtype(np.bool): 0,\n            np.dtype(np.int8): 0,\n            np.dtype(np.uint8): 0,\n            np.dtype(np.int32): 0,\n            np.dtype(np.uint32): 0,\n            np.dtype(np.int64): 0,\n            np.dtype(np.uint64): 0}\n\ndef default_numeric_eps():\n    \"\"\"Get default epsilon for finite difference gradient calculations with data type.\"\"\"\n    # prefer a power-of-two eps, since no bits are dropped when serving as an input delta\n    return {np.dtype(np.float16): 1.0 / 2**6,\n            np.dtype(np.float32): 1.0 / 2**9,\n            np.dtype(np.float64): 1.0 / 2**14}\n\n\ndef effective_dtype(dat):\n    \"\"\" Return the most appropriate dtype for determining the tolerance used in dat comparisons\n    Parameters\n    ----------\n    dat : np.ndarray or mx.nd.array or mx.np.ndarray\n    \"\"\"\n    # On arch 80 gpus or later, a float32-io gemm or conv op will trim the mantissa of\n    # data inputs to be of comparable precision to a float16, so float16 becomes the\n    # 'effective dtype' for tolerance tests involving such op outputs.\n\n    # Is TF32 enabled in the device (the default on arch 80 GPUs)\n    def is_TF32_enabled(device):\n        try:\n            return (device.device_type == 'gpu' and\n                    get_cuda_compute_capability(device) >= 80 and\n                    os.environ.get('NVIDIA_TF32_OVERRIDE') != '0')\n        except:  # pylint: disable=bare-except\n            return False\n\n    device = dat.device if hasattr(dat, 'device') else None\n    dtype = np.dtype(dat.dtype)\n    if dtype == np.dtype(np.float32) and is_TF32_enabled(device):\n        return np.dtype(np.float16)\n    else:\n        return dtype\n\n\ndef get_tolerance(dat, tol, default_tol):\n    \"\"\" Return the tolerance to be used for dat comparisons based on the given tol, datatype and device.\n    Parameters\n    ----------\n    dat : np.ndarray or mx.nd.array or mx.np.ndarray\n    tol : float, or a dict of dtype->float\n    default_tol : default dict of dtype->float for all types\n    \"\"\"\n\n    if isinstance(tol, numbers.Number):\n        return tol\n\n    # If the caller has supplied a tol dict, use that if it has an entry for dtype,\n    # else use the supplied default tol dict.\n    dtype = effective_dtype(dat)\n    tol = {} if tol is None else tol\n    return tol.get(dtype, default_tol[dtype])\n\n\ndef get_tols(x, y, rtol, atol):\n    \"\"\"For comparing two datasets 'x' and 'y', what relative and absolute tolerances should be used.\"\"\"\n    # Tolerance analysis needs 'dtype' of 'x' and 'y', so convert numbers to numpy scalars as needed\n    if isinstance(x, numbers.Number):\n        x = np.array(x)\n    if isinstance(y, numbers.Number):\n        y = np.array(y)\n\n    # If tols are not specified, use the largest default tol for 'x' and 'y' based on their ctx and dtype.\n    rtol = max(get_tolerance(x, rtol, default_rtols()),\n               get_tolerance(y, rtol, default_rtols()))\n    atol = max(get_tolerance(x, atol, default_atols()),\n               get_tolerance(y, atol, default_atols()))\n\n    return rtol, atol\n\n\ndef get_atol(atol=None, dtype=np.dtype(np.float64)):\n    \"\"\"Get default numerical threshold for regression test.\"\"\"\n    return default_atols()[dtype] if atol is None else atol\n\ndef get_rtol(rtol=None, dtype=np.dtype(np.float64)):\n    \"\"\"Get default numerical threshold for regression test.\"\"\"\n    return default_rtols()[dtype] if rtol is None else rtol\n\ndef get_etol(etol=None):\n    \"\"\"Get default numerical threshold for regression test.\"\"\"\n    # _TODO: get from env variable, different threshold might\n    # be needed for different device and dtype\n    return 0 if etol is None else etol\n\ndef random_arrays(*shapes):\n    \"\"\"Generate some random numpy arrays.\"\"\"\n    arrays = [np.array(np.random.randn(), dtype=default_dtype())\n              if len(s) == 0 else np.random.randn(*s).astype(default_dtype())\n              for s in shapes]\n    if len(arrays) == 1:\n        return arrays[0]\n    return arrays\n\n\ndef random_uniform_arrays(*shapes, **kwargs):\n    \"\"\"Generate some random numpy arrays.\"\"\"\n    low = kwargs.pop('low', 0.0)\n    high = kwargs.pop('high', 1.0)\n    dtype = kwargs.pop('dtype', default_dtype())\n    if len(kwargs) > 0:\n        raise TypeError('Got unexpected argument/s : ' + str(kwargs.keys()))\n    arrays = [np.random.uniform(low, high, size=s).astype(dtype)\n              for s in shapes]\n    return arrays\n\n\ndef random_sample(population, k):\n    \"\"\"Return a k length list of the elements chosen from the population sequence.\"\"\"\n    assert 0 <= k <= len(population)\n    population_copy = population[:]\n    np.random.shuffle(population_copy)\n    return population_copy[0:k]\n\n\ndef _sorted_items(d):\n    \"\"\"Return (key, value) pairs of dict 'd' in a deterministic order (sorted by key).\"\"\"\n    return sorted(d.items(), key=lambda t: t[0])\n\n\ndef _sorted_dict(d):\n    \"\"\"Return ordered dictionary containing items ordered by their keys.\"\"\"\n    return OrderedDict(_sorted_items(d))\n\n\ndef _validate_csr_generation_inputs(num_rows, num_cols, density,\n                                    distribution=\"uniform\"):\n    \"\"\"Validates inputs for csr generation helper functions\n    \"\"\"\n    total_nnz = int(num_rows * num_cols * density)\n    if density < 0 or density > 1:\n        raise ValueError(\"density has to be between 0 and 1\")\n\n    if num_rows <= 0 or num_cols <= 0:\n        raise ValueError(\"num_rows or num_cols should be greater than 0\")\n\n    if distribution == \"powerlaw\":\n        if total_nnz < 2 * num_rows:\n            raise ValueError(f\"not supported for this density: {density}\"\n                             f\" for this shape ({num_rows}, {num_cols})\"\n                             \" Please keep :\"\n                             \" num_rows * num_cols * density >= 2 * num_rows\")\n\n\ndef shuffle_csr_column_indices(csr):\n    \"\"\"Shuffle CSR column indices per row\n    This allows validation of unordered column indices, which is not a requirement\n    for a valid CSR matrix\n    \"\"\"\n    row_count = len(csr.indptr) - 1\n    for i in range(row_count):\n        start_index = csr.indptr[i]\n        end_index = csr.indptr[i + 1]\n        sublist = np.array(csr.indices[start_index : end_index])\n        np.random.shuffle(sublist)\n        csr.indices[start_index : end_index] = sublist\n\n\ndef _get_uniform_dataset_csr(num_rows, num_cols, density=0.1, dtype=None,\n                             data_init=None, shuffle_csr_indices=False):\n    \"\"\"Returns CSRNDArray with uniform distribution\n    This generates a csr matrix with totalnnz unique randomly chosen numbers\n    from num_rows*num_cols and arranges them in the 2d array in the\n    following way:\n    row_index = (random_number_generated / num_rows)\n    col_index = random_number_generated - row_index * num_cols\n    \"\"\"\n    _validate_csr_generation_inputs(num_rows, num_cols, density,\n                                    distribution=\"uniform\")\n    try:\n        from scipy import sparse as spsp\n        csr = spsp.rand(num_rows, num_cols, density, dtype=dtype, format=\"csr\")\n        if data_init is not None:\n            csr.data.fill(data_init)\n        if shuffle_csr_indices is True:\n            shuffle_csr_column_indices(csr)\n        result = mx.nd.sparse.csr_matrix((csr.data, csr.indices, csr.indptr),\n                                         shape=(num_rows, num_cols), dtype=dtype)\n    except ImportError:\n        assert(data_init is None), \\\n               \"data_init option is not supported when scipy is absent\"\n        assert(not shuffle_csr_indices), \\\n               \"shuffle_csr_indices option is not supported when scipy is absent\"\n        # scipy not available. try to generate one from a dense array\n        dns = mx.nd.random.uniform(shape=(num_rows, num_cols), dtype=dtype)\n        masked_dns = dns * (dns < density)\n        result = masked_dns.tostype('csr')\n    return result\n\ndef _get_powerlaw_dataset_csr(num_rows, num_cols, density=0.1, dtype=None):\n    \"\"\"Returns CSRNDArray with powerlaw distribution\n    with exponentially increasing number of non zeros in each row.\n    Not supported for cases where total_nnz < 2*num_rows. This is because\n    the algorithm first tries to ensure that there are rows with no zeros by\n    putting non zeros at beginning of each row.\n    \"\"\"\n\n    _validate_csr_generation_inputs(num_rows, num_cols, density,\n                                    distribution=\"powerlaw\")\n\n    total_nnz = int(num_rows * num_cols * density)\n\n    unused_nnz = total_nnz\n    output_arr = np.zeros((num_rows, num_cols), dtype=dtype)\n    # Start with ones on each row so that no row is empty\n    for row in range(num_rows):\n        output_arr[row][0] = 1 + rnd.uniform(0.001, 2)\n        unused_nnz = unused_nnz - 1\n        if unused_nnz <= 0:\n            return mx.nd.array(output_arr).tostype(\"csr\")\n\n    # Populate rest of matrix with 2^i items in ith row.\n    # if we have used all total nnz return the sparse matrix\n    # else if we reached max column size then fill up full columns until we use all nnz\n    col_max = 2\n    for row in range(num_rows):\n        col_limit = min(num_cols, col_max)\n        # In case col_limit reached assign same value to all elements, which is much faster\n        if col_limit == num_cols and unused_nnz > col_limit:\n            output_arr[row] = 1 + rnd.uniform(0.001, 2)\n            unused_nnz = unused_nnz - col_limit + 1\n            if unused_nnz <= 0:\n                return mx.nd.array(output_arr).tostype(\"csr\")\n            else:\n                continue\n        for col_index in range(1, col_limit):\n            output_arr[row][col_index] = 1 + rnd.uniform(0.001, 2)\n            unused_nnz = unused_nnz - 1\n            if unused_nnz <= 0:\n                return mx.nd.array(output_arr).tostype(\"csr\")\n        col_max = col_max * 2\n\n    if unused_nnz > 0:\n        raise ValueError(f\"not supported for this density: {density}\"\n                         f\" for this shape ({num_rows},{num_cols})\")\n\n    return mx.nd.array(output_arr).tostype(\"csr\")\n\n\ndef assign_each(the_input, function):\n    \"\"\"Return ndarray composed of passing each array value through some function\"\"\"\n    if function is None:\n        output = np.array(the_input)\n    else:\n        it_input = np.nditer(the_input, flags=['f_index'])\n\n        output = np.zeros(the_input.shape)\n        it_out = np.nditer(output, flags=['f_index'], op_flags=['writeonly'])\n\n        while not it_input.finished:\n            val_input = it_input[0]\n            it_out[0] = function(val_input)\n            it_input.iternext()\n            it_out.iternext()\n\n    return output\n\ndef assign_each2(input1, input2, function):\n    \"\"\"Return ndarray composed of passing two array values through some function\"\"\"\n    if function is None:\n        output = np.array(input1)\n    else:\n        assert input1.shape == input2.shape\n        it_input1 = np.nditer(input1, flags=['f_index'])\n        it_input2 = np.nditer(input2, flags=['f_index'])\n\n        output = np.zeros(input1.shape)\n        it_out = np.nditer(output, flags=['f_index'], op_flags=['writeonly'])\n\n        while not it_input1.finished:\n            val_input1 = it_input1[0]\n            val_input2 = it_input2[0]\n            it_out[0] = function(val_input1, val_input2)\n            it_input1.iternext()\n            it_input2.iternext()\n            it_out.iternext()\n\n    return output\n\ndef create_2d_np_tensor(rows, columns, dtype=np.int64):\n    inp = mx.np.arange(0, rows, dtype=dtype).reshape(rows, 1)\n    inp = mx.np.broadcast_to(inp, shape=(inp.shape[0], columns))\n    return inp\n\n# For testing Large Tensors having total size > 2^32 elements\ndef create_2d_tensor(rows, columns, dtype=np.int64):\n    a = mx.nd.arange(0, rows, dtype=dtype).reshape(rows, 1)\n    b = mx.nd.broadcast_to(a, shape=(a.shape[0], columns))\n    return b\n\n# For testing Large Vectors having total size > 2^32 elements\ndef create_vector(size, dtype=np.int64):\n    a = mx.nd.arange(0, size, dtype=dtype)\n    return a\n\ndef rand_sparse_ndarray(shape, stype, density=None, dtype=None, distribution=None,\n                        data_init=None, rsp_indices=None, modifier_func=None,\n                        shuffle_csr_indices=False, ctx=None):\n    \"\"\"Generate a random sparse ndarray. Returns the ndarray, value(np) and indices(np)\n\n    Parameters\n    ----------\n    shape: list or tuple\n    stype: str\n        valid values: \"csr\" or \"row_sparse\"\n    density: float, optional\n        should be between 0 and 1\n    distribution: str, optional\n        valid values: \"uniform\" or \"powerlaw\"\n    dtype: numpy.dtype, optional\n        default value is None\n\n    Returns\n    -------\n    Result of type CSRNDArray or RowSparseNDArray\n\n    Examples\n    --------\n    Below is an example of the powerlaw distribution with csr as the stype.\n    It calculates the nnz using the shape and density.\n    It fills up the ndarray with exponentially increasing number of elements.\n    If there are enough unused_nnzs, n+1th row will have twice more nnzs compared to nth row.\n    else, remaining unused_nnzs will be used in n+1th row\n    If number of cols is too small and we have already reached column size it will fill up\n    all following columns in all followings rows until we reach the required density.\n\n    >>> csr_arr, _ = rand_sparse_ndarray(shape=(5, 16), stype=\"csr\",\n                                         density=0.50, distribution=\"powerlaw\")\n    >>> indptr = csr_arr.indptr.asnumpy()\n    >>> indices = csr_arr.indices.asnumpy()\n    >>> data = csr_arr.data.asnumpy()\n    >>> row2nnz = len(data[indptr[1]:indptr[2]])\n    >>> row3nnz = len(data[indptr[2]:indptr[3]])\n    >>> assert(row3nnz == 2*row2nnz)\n    >>> row4nnz = len(data[indptr[3]:indptr[4]])\n    >>> assert(row4nnz == 2*row3nnz)\n\n    \"\"\"\n    ctx = ctx if ctx else default_device()\n    density = rnd.rand() if density is None else density\n    dtype = default_dtype() if dtype is None else dtype\n    distribution = \"uniform\" if distribution is None else distribution\n    if stype == 'row_sparse':\n        assert (distribution == \"uniform\"), \\\n               f\"Distribution {distribution} not supported for row_sparse\"\n        # sample index\n        if rsp_indices is not None:\n            indices = rsp_indices\n            assert(len(indices) <= shape[0])\n        else:\n            idx_sample = rnd.rand(shape[0])\n            indices = np.argwhere(idx_sample < density).flatten()\n        if indices.shape[0] == 0:\n            result = mx.nd.zeros(shape, stype='row_sparse', dtype=dtype, ctx=ctx)\n            return result, (np.array([], dtype=dtype), np.array([]))\n        # generate random values\n        val = rnd.rand(indices.shape[0], *shape[1:]).astype(dtype)\n\n        # Allow caller to override or adjust random values\n        if data_init is not None:\n            val.fill(data_init)\n        if modifier_func is not None:\n            val = assign_each(val, modifier_func)\n\n        arr = mx.nd.sparse.row_sparse_array((val, indices), shape=shape, dtype=dtype, ctx=ctx)\n        return arr, (val, indices)\n    elif stype == 'csr':\n        assert len(shape) == 2\n        if distribution == \"uniform\":\n            csr = _get_uniform_dataset_csr(shape[0], shape[1], density,\n                                           data_init=data_init,\n                                           shuffle_csr_indices=shuffle_csr_indices, dtype=dtype).as_in_context(ctx)\n            return csr, (csr.indptr, csr.indices, csr.data)\n        elif distribution == \"powerlaw\":\n            csr = _get_powerlaw_dataset_csr(shape[0], shape[1], density=density, dtype=dtype).as_in_context(ctx)\n            return csr, (csr.indptr, csr.indices, csr.data)\n        else:\n            assert(False), f\"Distribution not supported: {distribution}\"\n            return False\n    else:\n        assert(False), \"unknown storage type\"\n        return False\n\ndef rand_ndarray(shape, stype='default', density=None, dtype=None, modifier_func=None,\n                 shuffle_csr_indices=False, distribution=None, ctx=None):\n    \"\"\"Generate a random sparse ndarray. Returns the generated ndarray.\"\"\"\n    ctx = ctx if ctx else default_device()\n    if stype == 'default':\n        arr = mx.nd.array(random_arrays(shape), dtype=dtype, ctx=ctx)\n    else:\n        arr, _ = rand_sparse_ndarray(shape, stype, density=density,\n                                     modifier_func=modifier_func, dtype=dtype,\n                                     shuffle_csr_indices=shuffle_csr_indices,\n                                     distribution=distribution, ctx=ctx)\n    return arr\n\n\ndef create_sparse_array(shape, stype, data_init=None, rsp_indices=None,\n                        dtype=None, modifier_func=None, density=.5,\n                        shuffle_csr_indices=False):\n    \"\"\"Create a sparse array, For Rsp, assure indices are in a canonical format\"\"\"\n    if stype == 'row_sparse':\n        if rsp_indices is not None:\n            arr_indices = np.asarray(rsp_indices)\n            arr_indices.sort()\n        else:\n            arr_indices = None\n        arr_data, (_, _) = rand_sparse_ndarray(shape, stype,\n                                               density=density,\n                                               data_init=data_init,\n                                               rsp_indices=arr_indices,\n                                               dtype=dtype,\n                                               modifier_func=modifier_func)\n    elif stype == 'csr':\n        arr_data, (_, _, _) = rand_sparse_ndarray(shape,\n                                                  stype,\n                                                  density=density,\n                                                  data_init=data_init,\n                                                  dtype=dtype,\n                                                  modifier_func=modifier_func,\n                                                  shuffle_csr_indices=shuffle_csr_indices)\n    else:\n        msg = \"Unknown storage type: \" + stype\n        raise AssertionError(msg)\n\n    return arr_data\n\n\ndef create_sparse_array_zd(shape, stype, density, data_init=None,\n                           rsp_indices=None, dtype=None, modifier_func=None,\n                           shuffle_csr_indices=False):\n    \"\"\"Create sparse array, using only rsp_indices to determine density\"\"\"\n    if stype == 'row_sparse':\n        density = 0.0\n        if rsp_indices is not None:\n            assert len(rsp_indices) <= shape[0]\n    return create_sparse_array(shape, stype,\n                               data_init=data_init,\n                               rsp_indices=rsp_indices,\n                               dtype=dtype,\n                               modifier_func=modifier_func,\n                               density=density,\n                               shuffle_csr_indices=shuffle_csr_indices)\n\n\ndef rand_shape_2d(dim0=10, dim1=10, allow_zero_size=False):\n    low = 0 if allow_zero_size else 1\n    return rnd.randint(low, dim0 + 1), rnd.randint(low, dim1 + 1)\n\n\ndef rand_shape_3d(dim0=10, dim1=10, dim2=10, allow_zero_size=False):\n    low = 0 if allow_zero_size else 1\n    return rnd.randint(low, dim0 + 1), rnd.randint(low, dim1 + 1), rnd.randint(low, dim2 + 1)\n\n\ndef rand_shape_nd(num_dim, dim=10, allow_zero_size=False):\n    low = 0 if allow_zero_size else 1\n    return tuple(rnd.randint(low, dim+1, size=num_dim))\n\n\ndef rand_coord_2d(x_low, x_high, y_low, y_high):\n    x = np.random.randint(x_low, x_high, dtype=np.int64)\n    y = np.random.randint(y_low, y_high, dtype=np.int64)\n    return x, y\n\n\ndef np_reduce(dat, axis, keepdims, numpy_reduce_func):\n    \"\"\"Compatible reduce for old version of NumPy.\n\n    Parameters\n    ----------\n    dat : np.ndarray\n        Same as NumPy.\n\n    axis : None or int or list-like\n        Same as NumPy.\n\n    keepdims : bool\n        Same as NumPy.\n\n    numpy_reduce_func : function\n        A NumPy reducing function like ``np.sum`` or ``np.max``.\n    \"\"\"\n    if isinstance(axis, int):\n        axis = [axis]\n    else:\n        axis = list(axis) if axis is not None else range(len(dat.shape))\n    ret = dat\n    for i in reversed(sorted(axis)):\n        ret = numpy_reduce_func(ret, axis=i)\n    if keepdims:\n        keepdims_shape = list(dat.shape)\n        for i in axis:\n            keepdims_shape[i] = 1\n        ret = ret.reshape(tuple(keepdims_shape))\n    return ret\n\n\ndef _find_max_violation(a, b, rtol, atol):\n    \"\"\"Finds and returns the location of maximum violation.\"\"\"\n    # 'smart' absdiff that considers inf's as equals (to match np.allclose)\n    absdiff = np.where(np.equal(a, b), 0, np.abs(a-b))\n    tol = atol + rtol*np.abs(b)\n    violation = absdiff/(tol+1e-20)\n    loc = np.argmax(violation)\n    idx = np.unravel_index(loc, violation.shape)\n    return idx, np.max(violation)\n\n\ndef same(a, b):\n    \"\"\"Test if two NumPy arrays are the same.\n\n    Parameters\n    ----------\n    a : np.ndarray\n    b : np.ndarray\n    \"\"\"\n    return np.array_equal(a, b)\n\n\ndef checkShapes(a, b):\n    if a.shape != b.shape:\n        msg = npt.build_err_msg([a, b],\n                                err_msg=\"a.shape = {} and b.shape = {} are not equal\"\n                                .format(str(a.shape), str(b.shape)))\n        raise AssertionError(msg)\n\n\ndef almost_equal(a, b, rtol=None, atol=None, equal_nan=False, use_broadcast=True):\n    \"\"\"Test if two numpy arrays are almost equal.\"\"\"\n    # pylint: disable=unexpected-keyword-arg\n    if not use_broadcast:\n        checkShapes(a, b)\n\n    return np.allclose(a, b, rtol=get_rtol(rtol), atol=get_atol(atol), equal_nan=equal_nan)\n    # pylint: enable=unexpected-keyword-arg\n\ndef locationError(a, b, index, names, maxError=False):\n    \"\"\"Create element mismatch comment\n\n    Parameters\n    ----------\n    a, b : compared np.ndarray's\n    index : tuple of coordinate arrays\n        Location of violation\n    names : tuple of names\n        The names of compared arrays.\n    maxError: boolean, optional\n        Flag indicating that maximum error is reporting.\n    \"\"\"\n    maximum = \"maximum \" if maxError else \"\"\n    return f\"Location of {maximum} error: {str(index)}, {names[0]}={a[index]:.8f}, {names[1]}={b[index]:.8f}\"\ndef assert_almost_equal(a, b, rtol=None, atol=None, names=('a', 'b'), equal_nan=False,\n                        use_broadcast=True, mismatches=(10, 10)):\n    \"\"\"Test that two numpy arrays are almost equal. Raise exception message if not.\n\n    Parameters\n    ----------\n    a : np.ndarray or mx.nd.array\n    b : np.ndarray or mx.nd.array\n    rtol : None or float or dict of dtype -> float\n        The relative threshold. Default threshold will be used if set to ``None``.\n    atol : None or float or dict of dtype -> float\n        The absolute threshold. Default threshold will be used if set to ``None``.\n    names : tuple of names, optional\n        The names used in error message when an exception occurs\n    equal_nan : boolean, optional\n        The flag determining how to treat NAN values in comparison\n    mismatches : tuple of mismatches\n        Maximum number of mismatches to be printed (mismatches[0]) and determine (mismatches[1])\n    \"\"\"\n    if not use_broadcast:\n        checkShapes(a, b)\n\n    rtol, atol = get_tols(a, b, rtol, atol)\n\n    if isinstance(a, mx.numpy.ndarray):\n        a = a.asnumpy()\n    if isinstance(b, mx.numpy.ndarray):\n        b = b.asnumpy()\n    use_np_allclose = isinstance(a, np.ndarray) and isinstance(b, np.ndarray)\n    if not use_np_allclose:\n        if not (hasattr(a, 'ctx') and hasattr(b, 'ctx') and a.device == b.device and a.dtype == b.dtype):\n            use_np_allclose = True\n            if isinstance(a, mx.nd.NDArray):\n                a = a.asnumpy()\n            if isinstance(b, mx.nd.NDArray):\n                b = b.asnumpy()\n\n    if use_np_allclose:\n        if hasattr(a, 'dtype') and a.dtype == np.bool_ and hasattr(b, 'dtype') and b.dtype == np.bool_:\n            np.testing.assert_equal(a, b)\n            return\n        if almost_equal(a, b, rtol, atol, equal_nan=equal_nan):\n            return\n    else:\n        output = mx.nd.contrib.allclose(a, b, rtol, atol, equal_nan)\n        if output.asnumpy() == 1:\n            return\n\n        a = a.asnumpy()\n        b = b.asnumpy()\n\n    index, rel = _find_max_violation(a, b, rtol, atol)\n    if index != ():\n        # a, b are the numpy arrays\n        indexErr = index\n        relErr = rel\n\n        print('\\n*** Maximum errors for vector of size {}:  rtol={}, atol={}\\n'.format(a.size, rtol, atol))\n        aTmp = a.copy()\n        bTmp = b.copy()\n        i = 1\n        while i <= a.size:\n            if i <= mismatches[0]:\n                print(f\"{i:3d}: Error {rel}  {locationError(a, b, index, names)}\")\n\n            aTmp[index] = bTmp[index] = 0\n            if almost_equal(aTmp, bTmp, rtol, atol, equal_nan=equal_nan):\n                break\n\n            i += 1\n            if i <= mismatches[1] or mismatches[1] <= 0:\n                index, rel = _find_max_violation(aTmp, bTmp, rtol, atol)\n            else:\n                break\n\n        mismatchDegree = \"at least \" if mismatches[1] > 0 and i > mismatches[1] else \"\"\n        errMsg = f\"Error {relErr} exceeds tolerance rtol={rtol:e}, atol={atol:e} \" \\\n                 f\"(mismatch {mismatchDegree}{100*i/a.size}%).\\n\" \\\n                 f\"{locationError(a, b, indexErr, names, maxError=True)}\"\n    else:\n        errMsg = f\"Error {rel} exceeds tolerance rtol={rtol:e}, atol={atol:e}.\\n\"\n\n    np.set_printoptions(threshold=4, suppress=True)\n    msg = npt.build_err_msg([a, b], err_msg=errMsg)\n\n    raise AssertionError(msg)\n\n\ndef assert_allclose(a, b, rtol=1e-07, atol=0, equal_nan=True):\n    assert_almost_equal(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan)\n\n\ndef assert_almost_equal_with_err(a, b, rtol=None, atol=None, etol=None,\n                                 names=('a', 'b'), equal_nan=False, mismatches=(10, 10)):\n    \"\"\"Test that two numpy arrays are almost equal within given error rate. Raise exception message if not.\n\n    Parameters\n    ----------\n    a : np.ndarray\n    b : np.ndarray\n    rtol : None or float or dict of dtype -> float\n        The relative threshold. Default threshold will be used if set to ``None``.\n    atol : None or float or dict of dtype -> float\n        The absolute threshold. Default threshold will be used if set to ``None``.\n    etol : None or float\n        The error rate threshold. If etol is float, return true if error_rate < etol even if\n        any error is found.\n    names : tuple of names, optional\n        The names used in error message when an exception occurs\n    equal_nan : boolean, optional\n        The flag determining how to treat NAN values in comparison\n    mismatches : tuple of mismatches\n        Maximum number of mismatches to be printed (mismatches[0]) and determine (mismatches[1])\n    \"\"\"\n    etol = get_etol(etol)\n    if etol > 0:\n        rtol, atol = get_tols(a, b, rtol, atol)\n        if isinstance(a, mx.nd.NDArray):\n            a = a.asnumpy()\n        if isinstance(b, mx.nd.NDArray):\n            b = b.asnumpy()\n        equals = np.isclose(a, b, rtol=rtol, atol=atol)\n        err = 1 - np.count_nonzero(equals) / equals.size\n        if err > etol:\n            index, rel = _find_max_violation(a, b, rtol, atol)\n            indexErr = index\n            relErr = rel\n\n            print('\\n*** Maximum errors for vector of size {}:  rtol={}, atol={}\\n'.format(a.size, rtol, atol))\n            aTmp = a.copy()\n            bTmp = b.copy()\n            i = 1\n            while i <= a.size:\n                if i <= mismatches[0]:\n                    print(f\"{i:3d}: Error {rel}  {locationError(a, b, index, names)}\")\n\n                aTmp[index] = bTmp[index] = 0\n                if almost_equal(aTmp, bTmp, rtol, atol, equal_nan=equal_nan):\n                    break\n\n                i += 1\n                if i <= mismatches[1] or mismatches[1] <= 0:\n                    index, rel = _find_max_violation(aTmp, bTmp, rtol, atol)\n                else:\n                    break\n\n            mismatchDegree = \"at least \" if mismatches[1] > 0 and i > mismatches[1] else \"\"\n            errMsg = f\"Error {relErr} exceeds tolerance rtol={rtol:e}, atol={atol:e} \" \\\n                     f\"(mismatch {mismatchDegree}{100*i/a.size}%).\\n\" \\\n                     f\"{locationError(a, b, indexErr, names, maxError=True)}\"\n            np.set_printoptions(threshold=4, suppress=True)\n            msg = npt.build_err_msg([a, b], err_msg=errMsg)\n            raise AssertionError(msg)\n    else:\n        assert_almost_equal(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan)\n\n\ndef assert_almost_equal_ignore_nan(a, b, rtol=None, atol=None, names=('a', 'b')):\n    \"\"\"Test that two NumPy arrays are almost equal (ignoring NaN in either array).\n    Combines a relative and absolute measure of approximate eqality.\n    If either the relative or absolute check passes, the arrays are considered equal.\n    Including an absolute check resolves issues with the relative check where all\n    array values are close to zero.\n\n    Parameters\n    ----------\n    a : np.ndarray\n    b : np.ndarray\n    rtol : None or float\n        The relative threshold. Default threshold will be used if set to ``None``.\n    atol : None or float\n        The absolute threshold. Default threshold will be used if set to ``None``.\n    \"\"\"\n    a = np.copy(a)\n    b = np.copy(b)\n    nan_mask = np.logical_or(np.isnan(a), np.isnan(b))\n    a[nan_mask] = 0\n    b[nan_mask] = 0\n\n    assert_almost_equal(a, b, rtol, atol, names)\n\ndef assert_exception(f, exception_type, *args, **kwargs):\n    \"\"\"Test that function f will throw an exception of type given by `exception_type`\"\"\"\n    try:\n        f(*args, **kwargs)\n        assert(False)\n    except exception_type:\n        return\n\n\ndef _parse_location(sym, location, ctx, dtype=default_dtype()):\n    \"\"\"Parses the given location to a ordered dictionary.\n\n    Arguments of the provided op `sym` are used as dictionary keys\n    and elements of `location` are used as values.\n\n    Parameters\n    ----------\n    sym : Symbol\n        Symbol containing op\n    location : list or tuple or dict\n        Argument values location\n\n        - if type is list or tuple of `np.ndarray`\n            inner elements are arrays correspoding to\n            ``sym.list_arguments()``.\n        - if type is dict of str -> `np.ndarray`\n            maps the name of arguments to the corresponding `np.ndarray`.\n        *In either case, value of all the arguments must be provided.*\n    ctx : Device\n        Device context.\n    dtype: \"asnumpy\" or np.float16 or np.float32 or np.float64\n        If dtype is \"asnumpy\" then the mx.nd.array created will have the same\n        type as th numpy array from which it is copied.\n        Otherwise, dtype is the explicit datatype for all mx.nd.array objects\n        created in this function.\n\n    Returns\n    -------\n    dict\n        Dictionary with `sym` arguments as keys and `location` elements as\n        values.\n\n    Examples\n    -------\n    >>> a = mx.symbol.Variable('a')\n    >>> b = mx.symbol.Variable('b')\n    >>> l1 = np.ndarray([2,3])\n    >>> l2 = np.ndarray([3,4])\n    >>> _parse_location(a * b, [l1, l2], None)\n    {'a': <NDArray 2x3 @cpu(0)>, 'b': <NDArray 3x4 @cpu(0)>}\n    >>> _parse_location(a * b, {'a': l1, 'b': l2}, None)\n    {'a': <NDArray 2x3 @cpu(0)>, 'b': <NDArray 3x4 @cpu(0)>}\n    >>> _parse_location(a * b, {'a': l1}, None)\n    ValueError: Symbol arguments and keys of the given location do not match.\n    \"\"\"\n    assert isinstance(location, (dict, list, tuple))\n    assert dtype == \"asnumpy\" or dtype in (np.float16, np.float32, np.float64)\n    if isinstance(location, dict):\n        if set(location.keys()) != set(sym.list_arguments()):\n            raise ValueError(\"Symbol arguments and keys of the given location do not match.\"\n                             f\"symbol args:{str(set(sym.list_arguments()))}, location.keys():{str(set(location.keys()))}\")\n    else:\n        location = {k: v for k, v in zip(sym.list_arguments(), location)}\n    location = {k: mx.nd.array(v, ctx=ctx, dtype=v.dtype if dtype == \"asnumpy\" else dtype) \\\n               if isinstance(v, np.ndarray) else v for k, v in location.items()}\n    return _sorted_dict(location)\n\n\ndef _parse_aux_states(sym, aux_states, ctx, dtype=default_dtype()):\n    \"\"\"Parses the given auxiliary states to a dictionary.\n\n    Auxiliary states of the provided op `sym` are used as dictionary\n    keys and elements of `aux_states` are used as values.\n\n    Parameters\n    ----------\n    sym : Symbol\n        Symbol containing op\n    aux_states : None or list or dict\n        Aux states\n\n        - if type is list or tuple of `np.ndarray`\n            inner elements are arrays correspoding to\n            ``sym.list_auxiliary_states()``.\n        - if type is dict of str -> `np.ndarray`\n            maps the name of arguments to the corresponding `np.ndarray`.\n        *In either case, all aux states of `sym` must be provided.*\n    ctx : Device\n        Device context.\n    dtype: \"asnumpy\" or np.float16 or np.float32 or np.float64\n        If dtype is \"asnumpy\" then the mx.nd.array created will have the same\n        type as th numpy array from which it is copied.\n        Otherwise, dtype is the explicit datatype for all mx.nd.array objects\n        created in this function.\n\n    Returns\n    -------\n    dict\n        Dictionary with `sym` aux states as keys and `aux_states` elements\n        as values.\n\n    Examples\n    -------\n    >>> data = mx.symbol.Variable('data')\n    >>> weight = mx.sym.Variable(name='fc1_weight')\n    >>> fc1 = mx.symbol.FullyConnected(data = data, weight=weight, name='fc1', num_hidden=128)\n    >>> fc2 = mx.symbol.BatchNorm(fc1, name='batchnorm0')\n    >>> mean_states = np.ones(3)\n    >>> var_states = np.ones(3)\n    >>> _parse_aux_states(fc2, [mean_states, var_states], None)\n    {'batchnorm0_moving_var': <NDArray 3 @cpu(0)>, 'batchnorm0_moving_mean': <NDArray 3 @cpu(0)>}\n    >>> _parse_aux_states(fc2, {'batchnorm0_moving_var': mean_states,\n    ...                         'batchnorm0_moving_mean': var_states}, None)\n    {'batchnorm0_moving_var': <NDArray 3 @cpu(0)>, 'batchnorm0_moving_mean': <NDArray 3 @cpu(0)>}\n    >>> _parse_aux_states(fc2, {'batchnorm0_moving_var': mean_states}, None)\n    ValueError: Symbol aux_states names and given aux_states do not match.\n    \"\"\"\n    assert dtype == \"asnumpy\" or dtype in (np.float16, np.float32, np.float64)\n    if aux_states is not None:\n        if isinstance(aux_states, dict):\n            if set(aux_states.keys()) != set(sym.list_auxiliary_states()):\n                raise ValueError(\"Symbol aux_states names and given aux_states do not match.\"\n                                 f\"symbol aux_names:{str(set(sym.list_auxiliary_states()))}, aux_states.keys:{str(set(aux_states.keys()))}\")\n        elif isinstance(aux_states, (list, tuple)):\n            aux_names = sym.list_auxiliary_states()\n            aux_states = {k:v for k, v in zip(aux_names, aux_states)}\n        aux_states = {k: mx.nd.array(v, ctx=ctx, dtype=v.dtype if dtype == \"asnumpy\" else dtype) \\\n                      for k, v in aux_states.items()}\n    return aux_states\n\n\ndef numeric_grad(executor, location, aux_states=None, eps=1e-4,\n                 use_forward_train=True, dtype=default_dtype()):\n    \"\"\"Calculates a numeric gradient via finite difference method.\n\n    Class based on Theano's `theano.gradient.numeric_grad` [1]\n\n    Parameters\n    ----------\n    executor : Executor\n        Executor that computes the forward pass.\n    location : list of numpy.ndarray or dict of str to numpy.ndarray\n        Argument values used as location to compute gradient\n        Maps the name of arguments to the corresponding numpy.ndarray.\n        Value of all the arguments must be provided.\n    aux_states : None or list of numpy.ndarray or dict of str to numpy.ndarray, optional\n        Auxiliary states values used as location to compute gradient\n        Maps the name of aux_states to the corresponding numpy.ndarray.\n        Value of all the auxiliary arguments must be provided.\n    eps : float, optional\n        Epsilon for the finite-difference method.\n    use_forward_train : bool, optional\n        Whether to use `is_train=True` in testing.\n    dtype: np.float16 or np.float32 or np.float64\n        Datatype for mx.nd.array.\n\n    References\n    ---------\n    ..[1] https://github.com/Theano/Theano/blob/master/theano/gradient.py\n    \"\"\"\n    def as_stype(var, stype, dtype):\n        return mx.nd.cast_storage(mx.nd.array(var, dtype=dtype), stype=stype)\n\n    assert dtype in (np.float16, np.float32, np.float64)\n    approx_grads = {k: np.zeros(v.shape, dtype=dtype)\n                    for k, v in location.items()}\n    for k, v in location.items():\n        stype = executor.arg_dict[k].stype\n        if stype == 'default':\n            executor.arg_dict[k][:] = as_stype(v, stype, dtype=dtype)\n    for k in location:\n        location[k] = np.asarray(location[k], order='C')\n    for k, v in location.items():\n        if v.dtype.kind != 'f':\n            continue\n        stype = executor.arg_dict[k].stype\n        old_value = v.copy()\n        for i in range(int(np.prod(v.shape))):\n            # inplace update\n            v.ravel()[i] += eps/2.0\n            executor.arg_dict[k][:] = as_stype(v, stype, dtype=dtype)\n            if aux_states is not None:\n                for key, val in aux_states.items():\n                    executor.aux_dict[key][:] = val\n            executor.forward(is_train=use_forward_train)\n            f_peps = executor.outputs[0].asnumpy()\n\n            v.ravel()[i] -= eps\n            executor.arg_dict[k][:] = as_stype(v, stype, dtype=dtype)\n            if aux_states is not None:\n                for key, val in aux_states.items():\n                    adstype = executor.aux_dict[key].stype\n                    executor.aux_dict[key][:] = as_stype(val, adstype, dtype=dtype)\n            executor.forward(is_train=use_forward_train)\n            f_neps = executor.outputs[0].asnumpy()\n\n            approx_grad = (f_peps - f_neps).sum() / eps\n            approx_grads[k].ravel()[i] = approx_grad\n            v.ravel()[i] = old_value.ravel()[i]\n        # copy back the original value\n        executor.arg_dict[k][:] = as_stype(old_value, stype, dtype=dtype)\n\n    return approx_grads\n\ndef check_numeric_gradient(sym, location, aux_states=None, numeric_eps=None, rtol=None,\n                           atol=None, grad_nodes=None, use_forward_train=True, ctx=None,\n                           grad_stype_dict=None, dtype=default_dtype()):\n    \"\"\"Verify an operation by checking backward pass via finite difference method.\n\n    Based on Theano's `theano.gradient.verify_grad` [1]\n\n    Parameters\n    ----------\n    sym : Symbol\n        Symbol containing op to test\n    location : list or tuple or dict\n        Argument values used as location to compute gradient\n\n        - if type is list of numpy.ndarray, \\\n            inner elements should have the same order as mxnet.sym.list_arguments().\n\n        - if type is dict of str -> numpy.ndarray, \\\n            maps the name of arguments to the corresponding numpy.ndarray.\n\n        *In either case, value of all the arguments must be provided.*\n    aux_states : list or tuple or dict, optional\n        The auxiliary states required when generating the executor for the symbol.\n    numeric_eps : float, optional\n        Delta for the finite difference method that approximates the gradient.\n    rtol : None or float\n        The relative threshold. Default threshold will be used if set to ``None``.\n    atol : None or float\n        The absolute threshold. Default threshold will be used if set to ``None``.\n    grad_nodes : None or list or tuple or dict, optional\n        Names of the nodes to check gradient on\n    use_forward_train : bool\n        Whether to use is_train=True when computing the finite-difference.\n    ctx : Context, optional\n        Check the gradient computation on the specified device.\n    grad_stype_dict : dict of str->str, optional\n        Storage type dictionary for gradient ndarrays.\n    dtype: np.float16 or np.float32 or np.float64\n        Datatype for mx.nd.array.\n\n    References\n    ---------\n    [1] https://github.com/Theano/Theano/blob/master/theano/gradient.py\n    \"\"\"\n    assert dtype in (np.float16, np.float32, np.float64)\n    if ctx is None:\n        ctx = default_device()\n\n    def random_projection(shape):\n        \"\"\"Get a random weight matrix with not too small elements\n\n        Parameters\n        ----------\n        shape : list or tuple\n        \"\"\"\n        # random_projection should not have elements too small,\n        # otherwise too much precision is lost in numerical gradient\n        plain = np.random.rand(*shape) + 0.1\n        return plain\n\n    location = _parse_location(sym=sym, location=location, ctx=ctx, dtype=dtype)\n    location_npy = {k:v.asnumpy() for k, v in location.items()}\n    aux_states = _parse_aux_states(sym=sym, aux_states=aux_states, ctx=ctx,\n                                   dtype=dtype)\n    if aux_states is not None:\n        aux_states_npy = {k: v.asnumpy() for k, v in aux_states.items()}\n    else:\n        aux_states_npy = None\n    if grad_nodes is None:\n        grad_nodes = sym.list_arguments()\n        grad_req = {k: 'write' for k in grad_nodes}\n    elif isinstance(grad_nodes, (list, tuple)):\n        grad_nodes = list(grad_nodes)\n        grad_req = {k: 'write' for k in grad_nodes}\n    elif isinstance(grad_nodes, dict):\n        grad_req = grad_nodes.copy()\n        grad_nodes = grad_nodes.keys()\n    else:\n        raise ValueError\n\n    input_shape = {k: v.shape for k, v in location.items()}\n    _, out_shape, _ = sym.infer_shape(**input_shape)\n    proj = mx.sym.Variable(\"__random_proj\")\n    is_np_sym = bool(isinstance(sym, np_symbol))\n    if is_np_sym:  # convert to np symbol for using element-wise multiplication\n        proj = proj.as_np_ndarray()\n    out = sym * proj\n    if is_np_sym:  # convert to classic symbol so that make_loss can be used\n        out = out.as_nd_ndarray()\n    out = mx.sym.make_loss(out)\n\n    location = dict(list(location.items()) +\n                    [(\"__random_proj\", mx.nd.array(random_projection(out_shape[0]),\n                                                   ctx=ctx, dtype=dtype))])\n    args_grad_npy = dict([(k, np.random.normal(0, 0.01, size=location[k].shape))\n                          for k in grad_nodes]\n                         + [(\"__random_proj\", np.random.normal(0, 0.01, size=out_shape[0]))])\n\n    args_grad = {k: mx.nd.array(v, ctx=ctx, dtype=dtype) for k, v in args_grad_npy.items()}\n    if grad_stype_dict is not None:\n        assert isinstance(grad_stype_dict, dict), \"grad_stype_dict must be a dict\"\n        for k, v in grad_stype_dict.items():\n            if k in args_grad and v in _STORAGE_TYPE_STR_TO_ID and v != 'default':\n                # create an uninitialized sparse ndarray for executor\n                # if the symbolic grad is expected to be zero, it should not be initialized at all\n                args_grad[k] = mx.nd.zeros(args_grad[k].shape, args_grad[k].context,\n                                           args_grad[k].dtype, v)\n\n    grad_req[\"__random_proj\"] = 'write'\n    executor = out._bind(ctx, grad_req=grad_req,\n                         args=location, args_grad=args_grad, aux_states=aux_states)\n\n    inps = executor.arg_arrays\n    if len(inps) != len(location):\n        raise ValueError(\"Executor arg_arrays and and location len do not match.\"\n                         f\"Got {len(inps)} inputs and {len(location)} locations\")\n\n    executor.forward(is_train=True)\n    assert len(executor.outputs) == 1\n\n    eps = get_tolerance(executor.outputs[0], numeric_eps, default_numeric_eps())\n    # cannot use finite differences with small eps without high precision\n    if dtype in (np.float32, np.float16):\n        assert eps >= 1e-5\n\n    executor.backward()\n    symbolic_grads = executor.grad_dict\n\n    numeric_gradients = numeric_grad(\n        executor, location_npy, aux_states_npy,\n        eps=eps, use_forward_train=use_forward_train, dtype=dtype)\n\n    for name in grad_nodes:\n        fd_grad = numeric_gradients[name]\n        orig_grad = args_grad_npy[name]\n        sym_grad = symbolic_grads[name]\n        if grad_req[name] == 'write':\n            assert_almost_equal(fd_grad, sym_grad, rtol, atol,\n                                (f\"NUMERICAL_{name}\", f\"BACKWARD_{name}\"))\n        elif grad_req[name] == 'add':\n            if isinstance(sym_grad, mx.nd.NDArray):\n                sym_grad = sym_grad.asnumpy()\n            assert_almost_equal(fd_grad, sym_grad - orig_grad, rtol, atol,\n                                (f\"NUMERICAL_{name}\", f\"BACKWARD_{name}\"))\n        elif grad_req[name] == 'null':\n            assert sym_grad is None\n        else:\n            raise ValueError(f\"Invalid grad_req {grad_req[name]} for argument {name}\")\n\n\ndef check_symbolic_forward(sym, location, expected, rtol=None, atol=None,\n                           aux_states=None, ctx=None, equal_nan=False,\n                           dtype=default_dtype()):\n    \"\"\"Compares a symbol's forward results with the expected ones.\n    Prints error messages if the forward results are not the same as the expected ones.\n\n    Parameters\n    ---------\n    sym : Symbol\n        output symbol\n    location : list of np.ndarray or dict of str to np.ndarray\n        The evaluation point\n\n        - if type is list of np.ndarray\n            Contains all the numpy arrays corresponding to `sym.list_arguments()`.\n        - if type is dict of str to np.ndarray\n            Contains the mapping between argument names and their values.\n    expected : list of np.ndarray or dict of str to np.ndarray\n        The expected output value\n\n        - if type is list of np.ndarray\n            Contains arrays corresponding to exe.outputs.\n        - if type is dict of str to np.ndarray\n            Contains mapping between sym.list_output() and exe.outputs.\n    rtol : None or float\n        The relative threshold. Default threshold will be used if set to ``None``.\n    atol : None or float\n        The absolute threshold. Default threshold will be used if set to ``None``.\n    aux_states : list of np.ndarray of dict, optional\n        - if type is list of np.ndarray\n            Contains all the NumPy arrays corresponding to sym.list_auxiliary_states\n        - if type is dict of str to np.ndarray\n            Contains the mapping between names of auxiliary states and their values.\n    device : Device, optional\n        running context\n    dtype: \"asnumpy\" or np.float16 or np.float32 or np.float64\n        If dtype is \"asnumpy\" then the mx.nd.array created will have the same\n        type as th numpy array from which it is copied.\n        Otherwise, dtype is the explicit datatype for all mx.nd.array objects\n        created in this function.\n\n    equal_nan: Boolean\n        if True, `nan` is a valid value for checking equivalency (ie `nan` == `nan`)\n\n    Example\n    -------\n    >>> shape = (2, 2)\n    >>> lhs = mx.symbol.Variable('lhs')\n    >>> rhs = mx.symbol.Variable('rhs')\n    >>> sym_dot = mx.symbol.dot(lhs, rhs)\n    >>> mat1 = np.array([[1, 2], [3, 4]])\n    >>> mat2 = np.array([[5, 6], [7, 8]])\n    >>> ret_expected = np.array([[19, 22], [43, 50]])\n    >>> check_symbolic_forward(sym_dot, [mat1, mat2], [ret_expected])\n    \"\"\"\n    assert dtype == \"asnumpy\" or dtype in (np.float16, np.float32, np.float64)\n    if ctx is None:\n        ctx = default_device()\n\n    location = _parse_location(sym=sym, location=location, ctx=ctx, dtype=dtype)\n    aux_states = _parse_aux_states(sym=sym, aux_states=aux_states, ctx=ctx,\n                                   dtype=dtype)\n    if isinstance(expected, dict):\n        expected = [expected[k] for k in sym.list_outputs()]\n    args_grad_data = {k:mx.nd.empty(v.shape, ctx=ctx, dtype=v.dtype if dtype == \"asnumpy\" else dtype) \\\n                      for k, v in location.items()}\n\n    executor = sym._bind(ctx=ctx, args=location, args_grad=args_grad_data, aux_states=aux_states)\n    for g in executor.grad_arrays:\n        if g.ndim == 0:\n            g[()] = 0\n        else:\n            g[:] = 0\n\n    executor.forward(is_train=False)\n\n    outputs = executor.outputs\n    for output_name, expect, output in zip(sym.list_outputs(), expected, outputs):\n        assert_almost_equal(expect, output, rtol, atol,\n                            (f\"EXPECTED_{output_name}\", f\"FORWARD_{output_name}\"),\n                            equal_nan=equal_nan)\n    return executor.outputs\n\ndef check_symbolic_backward(sym, location, out_grads, expected, rtol=None, atol=None,\n                            aux_states=None, grad_req='write', ctx=None, grad_stypes=None,\n                            equal_nan=False, dtype=default_dtype()):\n    \"\"\"Compares a symbol's backward results with the expected ones.\n    Prints error messages if the backward results are not the same as the expected results.\n\n    Parameters\n    ---------\n    sym : Symbol\n        output symbol\n    location : list of np.ndarray or dict of str to np.ndarray\n        The evaluation point\n\n        - if type is list of np.ndarray\n            Contains all the NumPy arrays corresponding to ``mx.sym.list_arguments``.\n        - if type is dict of str to np.ndarray\n            Contains the mapping between argument names and their values.\n    out_grads : None or list of np.ndarray or dict of str to np.ndarray\n        NumPys arrays corresponding to sym.outputs for incomming gradient.\n\n        - if type is list of np.ndarray\n            Contains arrays corresponding to ``exe.outputs``.\n        - if type is dict of str to np.ndarray\n            contains mapping between mxnet.sym.list_output() and Executor.outputs\n    expected : list of np.ndarray or dict of str to np.ndarray\n        expected gradient values\n\n        - if type is list of np.ndarray\n            Contains arrays corresponding to exe.grad_arrays\n        - if type is dict of str to np.ndarray\n            Contains mapping between ``sym.list_arguments()`` and exe.outputs.\n    rtol : None or float\n        The relative threshold. Default threshold will be used if set to ``None``.\n    atol : None or float\n        The absolute threshold. Default threshold will be used if set to ``None``.\n    aux_states : list of np.ndarray or dict of str to np.ndarray\n    grad_req : str or list of str or dict of str to str, optional\n        Gradient requirements. 'write', 'add' or 'null'.\n    ctx : Context, optional\n        Running context.\n    grad_stypes: dict of str->str\n        dictionary of mapping argument name to stype for the gradient\n    equal_nan: Boolean\n        if True, `nan` is a valid value for checking equivalency (ie `nan` == `nan`)\n    dtype: np.float16 or np.float32 or np.float64\n        Datatype for mx.nd.array.\n\n    Example\n    -------\n    >>> lhs = mx.symbol.Variable('lhs')\n    >>> rhs = mx.symbol.Variable('rhs')\n    >>> sym_add = mx.symbol.elemwise_add(lhs, rhs)\n    >>> mat1 = np.array([[1, 2], [3, 4]])\n    >>> mat2 = np.array([[5, 6], [7, 8]])\n    >>> grad1 = mx.nd.zeros(shape)\n    >>> grad2 = mx.nd.zeros(shape)\n    >>> exec_add = sym_add._bind(default_device(), args={'lhs': mat1, 'rhs': mat2},\n    ... args_grad={'lhs': grad1, 'rhs': grad2}, grad_req={'lhs': 'write', 'rhs': 'write'})\n    >>> exec_add.forward(is_train=True)\n    >>> ograd = mx.nd.ones(shape)\n    >>> grad_expected = ograd.copy().asnumpy()\n    >>> check_symbolic_backward(sym_add, [mat1, mat2], [ograd], [grad_expected, grad_expected])\n    \"\"\"\n    assert dtype == 'asnumpy' or dtype in (np.float16, np.float32, np.float64)\n    if ctx is None:\n        ctx = default_device()\n\n    location = _parse_location(sym=sym, location=location, ctx=ctx, dtype=dtype)\n    aux_states = _parse_aux_states(sym=sym, aux_states=aux_states, ctx=ctx,\n                                   dtype=dtype)\n    if isinstance(expected, (list, tuple)):\n        expected = {k:v for k, v in zip(sym.list_arguments(), expected)}\n\n    # Dirty the output buffer deterministically, for reproducibility.\n    args_grad_npy = {k:np.random.normal(size=v.shape) for k, v in _sorted_items(expected)}\n    args_grad_data = {}\n    for k, v in args_grad_npy.items():\n        nd = mx.nd.array(v, ctx=ctx, dtype=expected[k].dtype if dtype == \"asnumpy\" else dtype)\n        if grad_stypes is not None and k in grad_stypes:\n            stype = grad_stypes[k]\n            if stype is not None and stype != 'default':\n                out = create_sparse_array(v.shape, stype, density=0.0)\n            else:\n                out = nd\n            args_grad_data[k] = out\n        else:\n            args_grad_data[k] = nd\n\n    if isinstance(grad_req, str):\n        grad_req = {k:grad_req for k in sym.list_arguments()}\n    elif isinstance(grad_req, (list, tuple)):\n        grad_req = {k:v for k, v in zip(sym.list_arguments(), grad_req)}\n\n    executor = sym._bind(ctx=ctx, args=location, args_grad=args_grad_data,\n                         aux_states=aux_states, grad_req=grad_req)\n    outputs = executor.forward(is_train=True)\n\n    if isinstance(out_grads, (tuple, list)):\n        outg = list()\n        for i, arr in enumerate(out_grads):\n            stype = outputs[i].stype\n            if isinstance(arr, np.ndarray):\n                dtype = arr.dtype if dtype == \"asnumpy\" else dtype\n                outg.append(mx.nd.array(arr, ctx=ctx, dtype=dtype).tostype(stype))\n            else:\n                outg.append(arr.tostype(stype))\n        out_grads = outg\n    elif isinstance(out_grads, dict):\n        outg = dict()\n        for k, v in out_grads.items():\n            if isinstance(v, np.ndarray):\n                dtype = v.dtype if dtype == \"asnumpy\" else dtype\n                outg[k] = mx.nd.array(v, ctx=ctx, dtype=dtype)\n            else:\n                outg[k] = v\n        out_grads = outg\n    else:\n        assert out_grads is None\n    executor.backward(out_grads)\n\n    grads = args_grad_data\n\n    for name in expected:\n        if grad_req[name] == 'write':\n            assert_almost_equal(expected[name], grads[name], rtol, atol,\n                                (f\"EXPECTED_{name}\", f\"BACKWARD_{name}\"),\n                                equal_nan=equal_nan)\n        elif grad_req[name] == 'add':\n            grad = grads[name].asnumpy() if isinstance(grads[name], mx.nd.NDArray) else grads[name]\n            assert_almost_equal(expected[name], grad - args_grad_npy[name],\n                                rtol, atol, (f\"EXPECTED_{name}\", f\"BACKWARD_{name}\"),\n                                equal_nan=equal_nan)\n        elif grad_req[name] == 'null':\n            assert_almost_equal(args_grad_npy[name], grads[name],\n                                rtol, atol, (f\"EXPECTED_{name}\", f\"BACKWARD_{name}\"),\n                                equal_nan=equal_nan)\n        else:\n            raise ValueError(f\"Invalid grad_req {grad_req[name]} for argument {name}\")\n    return args_grad_data\n\ndef check_speed(sym, location=None, ctx=None, N=20, grad_req=None, typ=\"whole\",\n                **kwargs):\n    \"\"\"Check the running speed of a symbol.\n\n    Parameters\n    ----------\n    sym : Symbol\n        Symbol to run the speed test.\n    location : none or dict of str to np.ndarray\n        Location to evaluate the inner executor.\n    ctx : Context\n        Running context.\n    N : int, optional\n        Repeat times.\n    grad_req : None or str or list of str or dict of str to str, optional\n        Gradient requirements.\n    typ : str, optional\n        \"whole\" or \"forward\"\n\n        - \"whole\"\n            Test the forward_backward speed.\n        - \"forward\"\n            Only test the forward speed.\n    \"\"\"\n    if ctx is None:\n        ctx = default_device()\n\n    if grad_req is None:\n        grad_req = 'write'\n    if location is None:\n        exe = sym._simple_bind(grad_req=grad_req, ctx=ctx, **kwargs)\n        location = {k: np.random.normal(size=arr.shape, scale=1.0) for k, arr in\n                    exe.arg_dict.items()}\n    else:\n        assert isinstance(location, dict), f'Expect dict, get \"location\"={str(location)}'\n        exe = sym._simple_bind(grad_req=grad_req, ctx=ctx,\n                               **{k: v.shape for k, v in location.items()})\n\n    for name, iarr in location.items():\n        exe.arg_dict[name][:] = iarr.astype(exe.arg_dict[name].dtype)\n\n    if typ == \"whole\":\n        # Warm up\n        exe.forward(is_train=True)\n        exe.backward(out_grads=exe.outputs)\n        for output in exe.outputs:\n            output.wait_to_read()\n        # Test forward + backward\n        tic = time.time()\n        for _ in range(N):\n            exe.forward(is_train=True)\n            exe.backward(out_grads=exe.outputs)\n        mx.nd.waitall()\n        toc = time.time()\n        forward_backward_time = (toc - tic) * 1.0 / N\n        return forward_backward_time\n    elif typ == \"forward\":\n        # Warm up\n        exe.forward(is_train=False)\n        for output in exe.outputs:\n            output.wait_to_read()\n\n        # Test forward only\n        tic = time.time()\n        for _ in range(N):\n            exe.forward(is_train=False)\n        mx.nd.waitall()\n        toc = time.time()\n        forward_time = (toc - tic) * 1.0 / N\n        return forward_time\n    else:\n        raise ValueError('typ can only be \"whole\" or \"forward\".')\n\n\ndef check_consistency(sym, ctx_list, scale=1.0, grad_req='write',\n                      arg_params=None, aux_params=None, rtol=None, atol=None,\n                      raise_on_err=True, ground_truth=None, equal_nan=False,\n                      use_uniform=False, rand_type=np.float64):\n    \"\"\"Check symbol gives the same output for different running context\n\n    Parameters\n    ----------\n    sym : Symbol or list of Symbols\n        Symbol(s) to run the consistency test.\n    ctx_list : list\n        Running context. See example for more detail.\n    scale : float, optional\n        Standard deviation of the inner normal distribution. Used in initialization.\n    grad_req : str or list of str or dict of str to str\n        Gradient requirement.\n    arg_params : dict of input name -> input data\n        data to use for non-aux inputs\n    aux_params : dict of input name -> input data\n        data to use for aux inputs\n    rtol : float or dictionary dtype->float, optional\n        The relative error tolerance.\n    atol : float or dictionary dtype->float, optional\n        The absolute error tolerance.\n    raise_on_err : bool, optional, defaults to True\n        Should an error raise an exception (or just output exception message)\n    ground_truth : dict of output name -> data, optional\n        Provided ideal result to be compared against\n    equal_nan : bool, optional, defaults to False\n        Should nans be treated as equal in the comparison\n    use_uniform: bool\n        Optional, When flag set to true,\n        random input data generated follows uniform distribution,\n        not normal distribution\n    rand_type: np.dtype\n        casts the randomly generated data to this type\n        Optional, when input data is passed via arg_params,\n        defaults to np.float64 (numpy float default)\n\n    Examples\n    --------\n    >>> # create the symbol\n    >>> sym = mx.sym.Convolution(num_filter=3, kernel=(3,3), name='conv')\n    >>> # initialize the running context\n    >>> ctx_list =\\\n[{'ctx': mx.gpu(0), 'conv_data': (2, 2, 10, 10), 'type_dict': {'conv_data': np.float64}},\\\n {'ctx': mx.gpu(0), 'conv_data': (2, 2, 10, 10), 'type_dict': {'conv_data': np.float32}},\\\n {'ctx': mx.gpu(0), 'conv_data': (2, 2, 10, 10), 'type_dict': {'conv_data': np.float16}},\\\n {'ctx': mx.cpu(0), 'conv_data': (2, 2, 10, 10), 'type_dict': {'conv_data': np.float64}},\\\n {'ctx': mx.cpu(0), 'conv_data': (2, 2, 10, 10), 'type_dict': {'conv_data': np.float32}}]\n    >>> check_consistency(sym, ctx_list)\n    >>> sym = mx.sym.Concat(name='concat', num_args=2)\n    >>> ctx_list = \\\n[{'ctx': mx.gpu(0), 'concat_arg1': (2, 10), 'concat_arg0': (2, 10),\\\n  'type_dict': {'concat_arg0': np.float64, 'concat_arg1': np.float64}},\\\n {'ctx': mx.gpu(0), 'concat_arg1': (2, 10), 'concat_arg0': (2, 10),\\\n  'type_dict': {'concat_arg0': np.float32, 'concat_arg1': np.float32}},\\\n {'ctx': mx.gpu(0), 'concat_arg1': (2, 10), 'concat_arg0': (2, 10),\\\n  'type_dict': {'concat_arg0': np.float16, 'concat_arg1': np.float16}},\\\n {'ctx': mx.cpu(0), 'concat_arg1': (2, 10), 'concat_arg0': (2, 10),\\\n  'type_dict': {'concat_arg0': np.float64, 'concat_arg1': np.float64}},\\\n {'ctx': mx.cpu(0), 'concat_arg1': (2, 10), 'concat_arg0': (2, 10),\\\n  'type_dict': {'concat_arg0': np.float32, 'concat_arg1': np.float32}}]\n    >>> check_consistency(sym, ctx_list)\n    \"\"\"\n\n    assert len(ctx_list) > 1\n    if isinstance(sym, Symbol):\n        sym = [sym]*len(ctx_list)\n    else:\n        assert len(sym) == len(ctx_list)\n\n    output_names = sym[0].list_outputs()\n    arg_names = sym[0].list_arguments()\n    exe_list = []\n    for s, ctx in zip(sym, ctx_list):\n        assert s.list_arguments() == arg_names\n        assert s.list_outputs() == output_names\n        exe_list.append(s._simple_bind(grad_req=grad_req, **ctx))\n\n    arg_params = {} if arg_params is None else arg_params\n    aux_params = {} if aux_params is None else aux_params\n\n    # returns the least precise of two dtypes\n    def smaller_dtype(dt1, dt2):\n        return dt1 if dt2 is None or np.dtype(dt1).itemsize < np.dtype(dt2).itemsize else dt2\n\n    # It's important to assign random inputs in a deterministic order, for reproducibility.\n    for n, arr in _sorted_items(exe_list[0].arg_dict):\n        if n not in arg_params:\n            if use_uniform:\n                arg_params[n] = np.random.uniform(low=-0.92 * scale, high=0.92 * scale,\n                                                  size=arr.shape).astype(rand_type)\n            else:\n                arg_params[n] = np.random.normal(size=arr.shape,\n                                                 scale=scale).astype(rand_type)\n    for n in exe_list[0].aux_dict:\n        if n not in aux_params:\n            aux_params[n] = 0\n    for exe in exe_list:\n        for name, arr in exe.arg_dict.items():\n            arr[:] = arg_params[name]\n        for name, arr in exe.aux_dict.items():\n            arr[:] = aux_params[name]\n        # We need to initialize the gradient arrays if it's add.\n        if (grad_req == \"add\"):\n            for arr in exe.grad_arrays:\n                arr[:] = np.zeros(arr.shape, dtype=arr.dtype)\n\n    # test\n    for exe in exe_list:\n        exe.forward(is_train=False)\n\n    dtypes = [np.dtype(exe.outputs[0].dtype) for exe in exe_list]\n    # Select the ground truth as the first model having the highest precision output[0]\n    gt_idx = np.argmax(dtypes)\n    gt = ground_truth\n    if gt is None:\n        gt = exe_list[gt_idx].output_dict.copy()\n\n    for i, exe in enumerate(exe_list):\n        if i == gt_idx:\n            continue\n\n        for name, arr in zip(output_names, exe.outputs):\n            gtarr = gt[name]\n            try:\n                assert_almost_equal(arr, gtarr, rtol=rtol, atol=atol, equal_nan=equal_nan)\n            except AssertionError as e:\n                print(f'Predict Err: ctx {i} vs ctx {gt_idx} at {name}')\n                traceback.print_exc()\n                if raise_on_err:\n                    raise e\n\n                print(str(e))\n\n    # train\n    if grad_req != 'null':\n        # Perform forward()\n        for exe in exe_list:\n            exe.forward(is_train=True)\n        # Use the first executor's output data, cast to the least precise dtype,\n        # as the gradient data to pass to all executor's backward() call.\n        least_precise_dtype = [out.dtype for out in exe_list[0].outputs]\n        for exe in exe_list:\n            least_precise_dtype = [smaller_dtype(out1.dtype, dt) \\\n                                    for (out1, dt) in zip(exe.outputs, least_precise_dtype)]\n        golden_data_np = [out.astype(dt).asnumpy() \\\n                          for (out, dt) in zip(exe_list[0].outputs, least_precise_dtype)]\n        # Perform backward()\n        for exe in exe_list:\n            out_grads = [mx.nd.array(golden_np, ctx=exe._device,\n                                     dtype=out.dtype).tostype(out.stype)\n                         for (golden_np, out) in zip(golden_data_np, exe.outputs)]\n            exe.backward(out_grads)\n\n        gt = ground_truth\n        if gt is None:\n            gt = exe_list[gt_idx].output_dict.copy()\n            if grad_req != 'null':\n                gt.update(exe_list[gt_idx].grad_dict)\n        for i, exe in enumerate(exe_list):\n            if i == gt_idx:\n                continue\n\n            curr = zip(output_names + arg_names, exe.outputs + exe.grad_arrays)\n            for name, arr in curr:\n                if gt[name] is None:\n                    assert arr is None, name\n                    continue\n\n                gtarr = gt[name]\n                try:\n                    rt, at = rtol, atol\n                    # If the primary data i/o type is float16, then the tolerance used when\n                    # comparing a float32 input gradient (e.g. batchnorm gamma) should be float16.\n                    smaller_arr_dtype = smaller_dtype(arr.dtype, dtypes[i])\n                    smaller_gt_dtype = smaller_dtype(gtarr.dtype, dtypes[gt_idx])\n                    if smaller_arr_dtype != arr.dtype or \\\n                       smaller_gt_dtype != gtarr.dtype:\n                        rt, at = get_tols(arr.astype(smaller_arr_dtype),\n                                          gtarr.astype(smaller_gt_dtype), rtol, atol)\n                    assert_almost_equal(arr, gtarr, rtol=rt, atol=at, equal_nan=equal_nan)\n                except AssertionError as e:\n                    print('Train Err: {} {} ctx {} vs {} {} ctx {} at {}'.format(\n                        get_dtype_name(arr.dtype), arr.device, i,\n                        get_dtype_name(gtarr.dtype), gtarr.device, gt_idx, name))\n                    traceback.print_exc()\n                    if raise_on_err:\n                        raise e\n\n                    print(str(e))\n\n    return gt\n\ndef list_gpus():\n    \"\"\"Return a list of GPUs\n\n    Returns\n    -------\n    list of int:\n        If there are n GPUs, then return a list [0,1,...,n-1]. Otherwise returns\n        [].\n    \"\"\"\n    return range(mx.util.get_gpu_count())\n\ndef download(url, fname=None, dirname=None, overwrite=False, retries=5):\n    \"\"\"Download an given URL\n\n    Parameters\n    ----------\n\n    url : str\n        URL to download\n    fname : str, optional\n        filename of the downloaded file. If None, then will guess a filename\n        from url.\n    dirname : str, optional\n        output directory name. If None, then guess from fname or use the current\n        directory\n    overwrite : bool, optional\n        Default is false, which means skipping download if the local file\n        exists. If true, then download the url to overwrite the local file if\n        exists.\n    retries : integer, default 5\n        The number of times to attempt the download in case of failure or non 200 return codes\n\n    Returns\n    -------\n    str\n        The filename of the downloaded file\n    \"\"\"\n\n    assert retries >= 0, \"Number of retries should be at least 0\"\n\n    if fname is None:\n        fname = url.split('/')[-1]\n\n    if dirname is None:\n        dirname = os.path.dirname(fname)\n    else:\n        fname = os.path.join(dirname, fname)\n    if dirname != \"\":\n        if not os.path.exists(dirname):\n            try:\n                logging.info('create directory %s', dirname)\n                os.makedirs(dirname)\n            except OSError as exc:\n                if exc.errno != errno.EEXIST:\n                    raise OSError('failed to create ' + dirname)\n\n    if not overwrite and os.path.exists(fname):\n        logging.info(\"%s exists, skipping download\", fname)\n        return fname\n\n    while retries+1 > 0:\n        # Disable pyling too broad Exception\n        # pylint: disable=W0703\n        try:\n            r = requests.get(url, stream=True)\n            assert r.status_code == 200, f\"failed to open {url}\"\n            with open(fname, 'wb') as f:\n                for chunk in r.iter_content(chunk_size=1024):\n                    if chunk: # filter out keep-alive new chunks\n                        f.write(chunk)\n                break\n        except Exception as e:\n            retries -= 1\n            if retries <= 0:\n                raise e\n\n            print(\"download failed, retrying, {} attempt{} left\"\n                  .format(retries, 's' if retries > 1 else ''))\n    logging.info(\"downloaded %s into %s successfully\", url, fname)\n    return fname\n\n\ndef get_mnist(path='data'):\n    \"\"\"Download and load the MNIST dataset\n\n    Parameters\n    ----------\n    path : str\n        Path in which to save the files.\n\n    Returns\n    -------\n    dict\n        A dict containing the data.\n    \"\"\"\n    def read_data(label_url, image_url):\n        if not os.path.isdir(path):\n            os.makedirs(path)\n        with gzip.open(mx.gluon.utils.download(label_url, path=path)) as flbl:\n            struct.unpack(\">II\", flbl.read(8))\n            label = np.frombuffer(flbl.read(), dtype=np.int8)\n        with gzip.open(mx.gluon.utils.download(image_url, path=path), 'rb') as fimg:\n            _, _, rows, cols = struct.unpack(\">IIII\", fimg.read(16))\n            image = np.frombuffer(fimg.read(), dtype=np.uint8).reshape(len(label), rows, cols)\n            image = image.reshape(image.shape[0], 1, 28, 28).astype(np.float32)/255\n        return (label, image)\n\n    # changed to mxnet.io for more stable hosting\n    url_path = 'https://repo.mxnet.io/gluon/dataset/mnist/'\n    (train_lbl, train_img) = read_data(\n        url_path+'train-labels-idx1-ubyte.gz', url_path+'train-images-idx3-ubyte.gz')\n    (test_lbl, test_img) = read_data(\n        url_path+'t10k-labels-idx1-ubyte.gz', url_path+'t10k-images-idx3-ubyte.gz')\n    return {'train_data':train_img, 'train_label':train_lbl,\n            'test_data':test_img, 'test_label':test_lbl}\n\ndef get_mnist_ubyte(path='data'):\n    \"\"\"Downloads ubyte version of the MNIST dataset into a directory in the current directory\n    with the name `data` and extracts all files in the zip archive to this directory.\n    \"\"\"\n    if not os.path.isdir(path):\n        os.makedirs(path)\n    files = ['train-images-idx3-ubyte', 'train-labels-idx1-ubyte',\n             't10k-images-idx3-ubyte', 't10k-labels-idx1-ubyte']\n    if not all(os.path.exists(os.path.join(path, f)) for f in files):\n        get_mnist(path)\n        for f in files:\n            ubyte_file_path = os.path.join(path, f)\n            zip_file_path = ubyte_file_path + '.gz'\n            with gzip.GzipFile(zip_file_path) as zf:\n                with open(ubyte_file_path, 'wb') as ubyte_file:\n                    ubyte_file.write(zf.read())\n\ndef get_cifar10(path='data'):\n    \"\"\"Downloads CIFAR10 dataset into a directory in the current directory with the name `data`,\n    and then extracts all files into the directory `data/cifar`.\n    \"\"\"\n    if not os.path.isdir(path):\n        os.makedirs(path)\n    if (not os.path.exists(os.path.join(path, 'cifar', 'train.rec'))) or \\\n            (not os.path.exists(os.path.join(path, 'cifar', 'test.rec'))) or \\\n            (not os.path.exists(os.path.join(path, 'cifar', 'train.lst'))) or \\\n            (not os.path.exists(os.path.join(path, 'cifar', 'test.lst'))):\n        url = 'https://repo.mxnet.io/gluon/dataset/cifar10/cifar10-b9ac2870.zip'\n        sha1 = 'b9ac287012f2dad9dfb49d8271c39ecdd7db376c'\n        zip_file_path = mx.gluon.utils.download(url, path=path, sha1_hash=sha1,\n                                                verify_ssl=False)\n        with zipfile.ZipFile(zip_file_path) as zf:\n            zf.extractall(path)\n\ndef get_mnist_iterator(batch_size, input_shape, num_parts=1, part_index=0, path='data'):\n    \"\"\"Returns training and validation iterators for MNIST dataset\n    \"\"\"\n\n    get_mnist_ubyte(path)\n    flat = len(input_shape) != 3\n\n    train_dataiter = mx.io.MNISTIter(\n        image=os.path.join(path, \"train-images-idx3-ubyte\"),\n        label=os.path.join(path, \"train-labels-idx1-ubyte\"),\n        input_shape=input_shape,\n        batch_size=batch_size,\n        shuffle=True,\n        flat=flat,\n        num_parts=num_parts,\n        part_index=part_index)\n\n    val_dataiter = mx.io.MNISTIter(\n        image=os.path.join(path, \"t10k-images-idx3-ubyte\"),\n        label=os.path.join(path, \"t10k-labels-idx1-ubyte\"),\n        input_shape=input_shape,\n        batch_size=batch_size,\n        flat=flat,\n        num_parts=num_parts,\n        part_index=part_index)\n\n    return (train_dataiter, val_dataiter)\n\ndef get_bz2_data(data_dir, data_name, url, data_origin_name):\n    \"\"\"Download and extract bz2 data.\n\n    Parameters\n    ----------\n\n    data_dir : str\n        Absolute or relative path of the directory name to store bz2 files\n    data_name : str\n        Name of the output file in which bz2 contents will be extracted\n    url : str\n        URL to download data from\n    data_origin_name : str\n        Name of the downloaded b2 file\n\n    Examples\n    --------\n    >>> get_bz2_data(\"data_dir\", \"kdda.t\",\n                     \"https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/kdda.t.bz2\",\n                     \"kdda.t.bz2\")\n    \"\"\"\n\n    data_name = os.path.join(data_dir, data_name)\n    data_origin_name = os.path.join(data_dir, data_origin_name)\n    if not os.path.exists(data_name):\n        download(url, fname=data_origin_name, dirname=data_dir, overwrite=False)\n        bz_file = bz2.BZ2File(data_origin_name, 'rb')\n        with open(data_name, 'wb') as fout:\n            for line in bz_file:\n                fout.write(line)\n            bz_file.close()\n        os.remove(data_origin_name)\n\n\ndef same_array(array1, array2):\n    \"\"\"Check whether two NDArrays sharing the same memory block\n\n    Parameters\n    ----------\n\n    array1 : NDArray\n        First NDArray to be checked\n    array2 : NDArray\n        Second NDArray to be checked\n\n    Returns\n    -------\n    bool\n        Whether two NDArrays share the same memory\n    \"\"\"\n    array1[:] += 1\n    if not same(array1.asnumpy(), array2.asnumpy()):\n        array1[:] -= 1\n        return False\n    array1[:] -= 1\n    return same(array1.asnumpy(), array2.asnumpy())\n\n\n@contextmanager\ndef discard_stderr():\n    \"\"\"\n    Discards error output of a routine if invoked as:\n\n    with discard_stderr():\n        ...\n    \"\"\"\n    with open(os.devnull, 'w') as bit_bucket:\n        try:\n            stderr_fileno = sys.stderr.fileno()\n            old_stderr = os.dup(stderr_fileno)\n            try:\n                os.dup2(bit_bucket.fileno(), stderr_fileno)\n                yield\n            finally:\n                os.dup2(old_stderr, stderr_fileno)\n        except AttributeError:\n            # On some systems is stderr not a file descriptor but actually a virtual pipeline\n            # that can not be copied\n            yield\n\n\nclass DummyIter(mx.io.DataIter):\n    \"\"\"A dummy iterator that always returns the same batch of data\n    (the first data batch of the real data iter). This is usually used for speed testing.\n\n    Parameters\n    ----------\n    real_iter: mx.io.DataIter\n        The real data iterator where the first batch of data comes from\n    \"\"\"\n    def __init__(self, real_iter):\n        super(DummyIter, self).__init__()\n        self.real_iter = real_iter\n        self.provide_data = real_iter.provide_data\n        self.provide_label = real_iter.provide_label\n        self.batch_size = real_iter.batch_size\n        self.the_batch = next(real_iter)\n\n    def __iter__(self):\n        return self\n\n    def next(self):\n        \"\"\"Get a data batch from iterator. The first data batch of real iter is always returned.\n        StopIteration will never be raised.\n\n        Returns\n        -------\n        DataBatch\n            The data of next batch.\n        \"\"\"\n        return self.the_batch\n\ndef gen_buckets_probs_with_ppf(ppf, nbuckets):\n    \"\"\"Generate the buckets and probabilities for chi_square test when the ppf (Quantile function)\n     is specified.\n\n    Parameters\n    ----------\n    ppf : function\n        The Quantile function that takes a probability and maps it back to a value.\n        It's the inverse of the cdf function\n    nbuckets : int\n        size of the buckets\n\n    Returns\n    -------\n    buckets : list of tuple\n        The generated buckets\n    probs : list\n        The generate probabilities\n    \"\"\"\n    assert nbuckets > 0\n    probs = [1.0 / nbuckets for _ in range(nbuckets)]\n    buckets = [(ppf(i / float(nbuckets)), ppf((i + 1) / float(nbuckets))) for i in range(nbuckets)]\n    return buckets, probs\n\ndef mean_check(generator, mu, sigma, nsamples=1000000):\n    \"\"\"Test the generator by matching the mean.\n\n    We test the sample mean by checking if it falls inside the range\n        (mu - 3 * sigma / sqrt(n), mu + 3 * sigma / sqrt(n))\n\n    References::\n\n        @incollection{goucher2009beautiful,\n              title={Beautiful Testing: Leading Professionals Reveal How They Improve Software},\n              author={Goucher, Adam and Riley, Tim},\n              year={2009},\n              chapter=10\n        }\n\n    Examples::\n\n        generator = lambda x: np.random.normal(0, 1.0, size=x)\n        mean_check_ret = mean_check(generator, 0, 1.0)\n\n    Parameters\n    ----------\n    generator : function\n        The generator function. It's expected to generate N i.i.d samples by calling generator(N).\n    mu : float\n    sigma : float\n    nsamples : int\n\n    Returns\n    -------\n    ret : bool\n        Whether the mean test succeeds\n    \"\"\"\n    samples = np.array(generator(nsamples))\n    sample_mean = samples.mean()\n    ret = (sample_mean > mu - 3 * sigma / np.sqrt(nsamples)) and\\\n          (sample_mean < mu + 3 * sigma / np.sqrt(nsamples))\n    return ret\n\ndef get_im2rec_path(home_env=\"MXNET_HOME\"):\n    \"\"\"Get path to the im2rec.py tool\n\n    Parameters\n    ----------\n\n    home_env : str\n        Env variable that holds the path to the MXNET folder\n\n    Returns\n    -------\n    str\n        The path to im2rec.py\n    \"\"\"\n    # Check first if the path to MXNET is passed as an env variable\n    if home_env in os.environ:\n        mxnet_path = os.environ[home_env]\n    else:\n        # Else use currently imported mxnet as reference\n        mxnet_path = os.path.dirname(mx.__file__)\n    # If MXNet was installed through pip, the location of im2rec.py\n    im2rec_path = os.path.join(mxnet_path, 'tools', 'im2rec.py')\n    if os.path.isfile(im2rec_path):\n        return im2rec_path\n    # If MXNet has been built locally\n    im2rec_path = os.path.join(mxnet_path, '..', '..', 'tools', 'im2rec.py')\n    if os.path.isfile(im2rec_path):\n        return im2rec_path\n    raise IOError('Could not find path to tools/im2rec.py')\n\ndef var_check(generator, sigma, nsamples=1000000):\n    \"\"\"Test the generator by matching the variance.\n    It will need a large number of samples and is not recommended to use\n\n    We test the sample variance by checking if it falls inside the range\n        (sigma^2 - 3 * sqrt(2 * sigma^4 / (n-1)), sigma^2 + 3 * sqrt(2 * sigma^4 / (n-1)))\n\n    References::\n\n        @incollection{goucher2009beautiful,\n              title={Beautiful Testing: Leading Professionals Reveal How They Improve Software},\n              author={Goucher, Adam and Riley, Tim},\n              year={2009},\n              chapter=10\n        }\n\n    Examples::\n\n        generator = lambda x: np.random.normal(0, 1.0, size=x)\n        var_check_ret = var_check(generator, 0, 1.0)\n\n    Parameters\n    ----------\n    generator : function\n        The generator function. It's expected to generate N i.i.d samples by calling generator(N).\n    sigma : float\n    nsamples : int\n\n    Returns\n    -------\n    ret : bool\n        Whether the variance test succeeds\n    \"\"\"\n    samples = np.array(generator(nsamples))\n    sample_var = samples.var(ddof=1)\n    ret = (sample_var > sigma ** 2 - 3 * np.sqrt(2 * sigma ** 4 / (nsamples - 1))) and\\\n          (sample_var < sigma ** 2 + 3 * np.sqrt(2 * sigma ** 4 / (nsamples - 1)))\n    return ret\n\ndef chi_square_check(generator, buckets, probs, nsamples=1000000):\n    \"\"\"Run the chi-square test for the generator. The generator can be both continuous and discrete.\n\n    If the generator is continuous, the buckets should contain tuples of (range_min, range_max) \\\n    and the probs should be the corresponding ideal probability within the specific ranges. \\\n    Otherwise, the buckets should contain all the possible values generated over the discrete distribution and the \\\n    probs should be groud-truth probability.\n\n    Usually the user is required to specify the probs parameter.\n\n    After obtaining the p value, we could further use the standard p > 0.05 (alpha) threshold to get \\\n    the final result.\n\n    Examples::\n\n      buckets, probs = gen_buckets_probs_with_ppf(lambda x: ss.norm.ppf(x, 0, 1), 5)\n      generator = lambda x: np.random.normal(0, 1.0, size=x)\n      p = chi_square_check(generator=generator, buckets=buckets, probs=probs)\n      assert(p > 0.05)\n\n    Parameters\n    ----------\n    generator: function\n        A function that is assumed to generate i.i.d samples from a specific distribution.\n        generator(N) should generate N random samples.\n    buckets: list of tuple or list of number\n        The buckets to run the chi-square the test. Make sure that the buckets cover\n        the whole range of the distribution. Also, the buckets must be in ascending order and have\n        no intersection\n    probs: list or tuple\n        The ground-truth probability of the random value fall in a specific bucket.\n    nsamples:int\n        The number of samples to generate for the testing\n\n    Returns\n    -------\n    p : float\n        p value that the generator has the expected distribution.\n        A higher value indicates a larger confidence\n    obs_freq : list\n        Observed frequency of buckets\n    expected_freq : list\n        The expected (ground-truth) frequency of the buckets\n    \"\"\"\n    if not ss:\n        raise ImportError(\"scipy is not available.\"\n                          \" Please check if the scipy python bindings are installed.\")\n    assert isinstance(buckets, list)\n    samples = generator(nsamples)\n    assert len(probs) == len(buckets)\n    if isinstance(buckets[0], (list, tuple)):\n        # Check whether the buckets are valid and fill them into a npy array\n        continuous_dist = True\n        buckets_npy = np.zeros((len(buckets) * 2, ), dtype=np.float32)\n        for i, _ in enumerate(buckets):\n            assert(buckets[i][0] <= buckets[i][1])\n            if i < len(buckets) - 1:\n                assert(buckets[i][1] <= buckets[i + 1][0])\n            buckets_npy[i * 2] = buckets[i][0]\n            buckets_npy[i * 2 + 1] = buckets[i][1]\n    else:\n        continuous_dist = False\n    expected_freq = (nsamples * np.array(probs, dtype=np.float32)).astype(np.int32)\n    if continuous_dist:\n        sample_bucket_ids = np.searchsorted(buckets_npy, samples, side='right')\n    else:\n        sample_bucket_ids = np.array(samples)\n    if continuous_dist:\n        sample_bucket_ids = sample_bucket_ids // 2\n    obs_freq = np.zeros(shape=len(buckets), dtype=np.int)\n    for i, _ in enumerate(buckets):\n        if continuous_dist:\n            obs_freq[i] = (sample_bucket_ids == i).sum()\n        else:\n            obs_freq[i] = (sample_bucket_ids == buckets[i]).sum()\n    _, p = ss.chisquare(f_obs=obs_freq, f_exp=expected_freq)\n    return p, obs_freq, expected_freq\n\ndef verify_generator(generator, buckets, probs, nsamples=1000000, nrepeat=5, success_rate=0.2, alpha=0.05):\n    \"\"\"Verify whether the generator is correct using chi-square testing.\n\n    The test is repeated for \"nrepeat\" times and we check if the success rate is\n     above the threshold (25% by default).\n\n    Parameters\n    ----------\n    generator: function\n        A function that is assumed to generate i.i.d samples from a specific distribution.\n            generator(N) should generate N random samples.\n    buckets: list of tuple or list of number\n        The buckets to run the chi-square the test. Make sure that the buckets cover\n         the whole range of the distribution. Also, the buckets must be in ascending order and\n         have no intersection\n    probs: list or tuple\n        The ground-truth probability of the random value fall in a specific bucket.\n    nsamples: int\n        The number of samples to generate for the testing\n    nrepeat: int\n        The times to repeat the test\n    success_rate: float\n        The desired success rate\n    alpha: float\n        The desired threshold for type-I error i.e. when a true null hypothesis is rejected\n\n    Returns\n    -------\n    cs_ret_l: list\n        The p values of the chi-square test.\n    \"\"\"\n    cs_ret_l = []\n    obs_freq_l = []\n    expected_freq_l = []\n    for _ in range(nrepeat):\n        cs_ret, obs_freq, expected_freq = chi_square_check(generator=generator, buckets=buckets,\n                                                           probs=probs, nsamples=nsamples)\n        cs_ret_l.append(cs_ret)\n        obs_freq_l.append(obs_freq)\n        expected_freq_l.append(expected_freq)\n    success_num = (np.array(cs_ret_l) > alpha).sum()\n    if success_num < nrepeat * success_rate:\n        raise AssertionError(f\"Generator test fails, Chi-square p={str(cs_ret_l)}, \"\n                             f\"obs_freq={str(obs_freq_l)}, expected_freq={str(expected_freq_l)}.\"\n                             f\"\\nbuckets={str(buckets)}, probs={str(probs)}\")\n    return cs_ret_l\n\n\ndef compare_ndarray_tuple(t1, t2, rtol=None, atol=None):\n    \"\"\"Compare ndarray tuple.\"\"\"\n    if t1 is None or t2 is None:\n        return\n\n    if isinstance(t1, tuple):\n        for s1, s2 in zip(t1, t2):\n            compare_ndarray_tuple(s1, s2, rtol, atol)\n    else:\n        assert_almost_equal(t1, t2, rtol=rtol, atol=atol)\n\n\ndef compare_optimizer(opt1, opt2, shapes, dtype, w_stype='default', g_stype='default',\n                      rtol=1e-4, atol=1e-5, compare_states=True):\n    \"\"\"Compare opt1 and opt2.\"\"\"\n\n    w1_list, w2_list = [], []\n    g1_list, g2_list = [], []\n    s1_list, s2_list = [], []\n    for i, shape in enumerate(shapes):\n        if w_stype == 'default':\n            w2 = mx.random.uniform(shape=shape, ctx=default_device(), dtype=dtype)\n            w1 = w2.copyto(default_device())\n        elif w_stype in ('row_sparse', 'csr'):\n            w2 = rand_ndarray(shape, w_stype, density=1, dtype=dtype)\n            w1 = w2.copyto(default_device()).tostype('default')\n        else:\n            raise Exception(\"type not supported yet\")\n        if g_stype == 'default':\n            g2 = mx.random.uniform(shape=shape, ctx=default_device(), dtype=dtype)\n            g1 = g2.copyto(default_device())\n        elif g_stype in ('row_sparse', 'csr'):\n            g2 = rand_ndarray(shape, g_stype, dtype=dtype)\n            g1 = g2.copyto(default_device()).tostype('default')\n        else:\n            raise Exception(\"type not supported yet\")\n        s1 = opt1.create_state_multi_precision(i, w1)\n        s2 = opt2.create_state_multi_precision(i, w2)\n\n        if compare_states:\n            compare_ndarray_tuple(s1, s2)\n\n        w1_list.append(w1)\n        w2_list.append(w2)\n        g1_list.append(g1)\n        g2_list.append(g2)\n        s1_list.append(s1)\n        s2_list.append(s2)\n\n    indices = list(range(len(shapes)))\n    opt1.update_multi_precision(indices, w1_list, g1_list, s1_list)\n    opt2.update_multi_precision(indices, w2_list, g2_list, s2_list)\n    if compare_states:\n        compare_ndarray_tuple(tuple(s1_list), tuple(s2_list), rtol=rtol, atol=atol)\n    compare_ndarray_tuple(tuple(w1_list), tuple(w2_list), rtol=rtol, atol=atol)\n\n\ndef compare_optimizer_noise_seeded(opt1, opt2, shapes, dtype, noise_seed,\n                                   w_stype='default', g_stype='default',\n                                   rtol=1e-4, atol=1e-5, compare_states=True):\n    \"\"\"Compare opt1 and opt2 with the added functionality that the seed for generating random noise\n    in the SGLD optimizer update is set so that the same noise is used in opt1 and opt2.\n\n    \"\"\"\n    w1_list, w2_list = [], []\n    g1_list, g2_list = [], []\n    s1_list, s2_list = [], []\n    for i, shape in enumerate(shapes):\n        if w_stype == 'default':\n            w2 = mx.random.uniform(shape=shape, ctx=default_device(), dtype=dtype)\n            w1 = w2.copyto(default_device())\n        elif w_stype in ('row_sparse', 'csr'):\n            w2 = rand_ndarray(shape, w_stype, density=1, dtype=dtype)\n            w1 = w2.copyto(default_device()).tostype('default')\n        else:\n            raise Exception(\"type not supported yet\")\n        if g_stype == 'default':\n            g2 = mx.random.uniform(shape=shape, ctx=default_device(), dtype=dtype)\n            g1 = g2.copyto(default_device())\n        elif g_stype in ('row_sparse', 'csr'):\n            g2 = rand_ndarray(shape, g_stype, dtype=dtype)\n            g1 = g2.copyto(default_device()).tostype('default')\n        else:\n            raise Exception(\"type not supported yet\")\n        s1 = opt1.create_state_multi_precision(i, w1)\n        s2 = opt2.create_state_multi_precision(i, w2)\n\n        if compare_states:\n            compare_ndarray_tuple(s1, s2)\n\n        w1_list.append(w1)\n        w2_list.append(w2)\n        g1_list.append(g1)\n        g2_list.append(g2)\n        s1_list.append(s1)\n        s2_list.append(s2)\n\n    indices = list(range(len(shapes)))\n    # set seed for Gaussian noise replication\n    mx.random.seed(noise_seed)\n    opt1.update_multi_precision(indices, w1_list, g1_list, s1_list)\n    mx.random.seed(noise_seed)\n    opt2.update_multi_precision(indices, w2_list, g2_list, s2_list)\n    if compare_states:\n        compare_ndarray_tuple(tuple(s1_list), tuple(s2_list), rtol=rtol, atol=atol)\n    compare_ndarray_tuple(tuple(w1_list), tuple(w2_list), rtol=rtol, atol=atol)\n\n\ndef same_symbol_structure(sym1, sym2):\n    \"\"\"Compare two symbols to check if they have the same computation graph structure.\n    Returns true if operator corresponding to a particular node id is same in both\n    symbols for all nodes\n    \"\"\"\n    conf = json.loads(sym1.tojson())\n    nodes = conf[\"nodes\"]\n    conf2 = json.loads(sym2.tojson())\n    nodes2 = conf2[\"nodes\"]\n    for node1, node2 in zip(nodes, nodes2):\n        if node1[\"op\"] != node2[\"op\"]:\n            return False\n    return True\n\n\n@contextmanager\ndef environment(*args):\n    \"\"\"\n    Environment variable setter and unsetter via `with` idiom.\n\n    Takes a specification of env var names and desired values and adds those\n    settings to the environment in advance of running the body of the `with`\n    statement.  The original environment state is restored afterwards, even\n    if exceptions are raised in the `with` body.\n\n    Parameters\n    ----------\n    args:\n        if 2 args are passed:\n            name, desired_value strings of the single env var to update, or\n        if 1 arg is passed:\n            a dict of name:desired_value for env var's to update\n\n    \"\"\"\n\n    # On Linux, env var changes made through python's os.environ are seen\n    # by the backend.  On Windows though, the C runtime gets a snapshot\n    # of the environment that cannot be altered by os.environ.  Here we\n    # check, using a wrapped version of the backend's getenv(), that\n    # the desired env var value is seen by the backend, and otherwise use\n    # a wrapped setenv() to establish that value in the backend.\n\n    # Also on Windows, a set env var can never have the value '', since\n    # the command 'set FOO= ' is used to unset the variable.  Perhaps\n    # as a result, the wrapped dmlc::GetEnv() routine returns the same\n    # value for unset variables and those set to ''.  As a result, we\n    # ignore discrepancy.\n    def validate_backend_setting(name, value, can_use_setenv=True):\n        backend_value = getenv(name)\n        if value == backend_value or \\\n           value == '' and backend_value is None and platform.system() == 'Windows':\n            return\n        if not can_use_setenv:\n            raise RuntimeError('Could not set env var {}={} within C Runtime'.format(name, value))\n        setenv(name, value)\n        validate_backend_setting(name, value, can_use_setenv=False)\n\n    # Core routine to alter environment from a dict of env_var_name, env_var_value pairs\n    def set_environ(env_var_dict):\n        for env_var_name, env_var_value in env_var_dict.items():\n            if env_var_value is None:\n                os.environ.pop(env_var_name, None)\n            else:\n                os.environ[env_var_name] = env_var_value\n            validate_backend_setting(env_var_name, env_var_value)\n\n    # Create env_var name:value dict from the two calling methods of this routine\n    if len(args) == 1 and isinstance(args[0], dict):\n        env_vars = args[0]\n    else:\n        assert len(args) == 2, 'Expecting one dict arg or two args: env var name and value'\n        env_vars = {args[0]: args[1]}\n\n    # Take a snapshot of the existing environment variable state\n    # for those variables to be changed.  get() return None for unset keys.\n    snapshot = {x: os.environ.get(x) for x in env_vars.keys()}\n\n    # Alter the environment per the env_vars dict\n    set_environ(env_vars)\n\n    # Now run the wrapped code\n    try:\n        yield\n    finally:\n        # the backend engines may still be referencing the changed env var state\n        mx.nd.waitall()\n        # reinstate original env_var state per the snapshot taken earlier\n        set_environ(snapshot)\n\n\ndef collapse_sum_like(a, shape):\n    \"\"\"Given `a` as a numpy ndarray, perform reduce_sum on `a` over the axes that do not\n    exist in `shape`. Note that an ndarray with `shape` must be broadcastable to `a`.\n    \"\"\"\n    assert len(a.shape) >= len(shape)\n    if np.prod(shape) == 0 or a.size == 0:\n        return np.zeros(shape, dtype=a.dtype)\n    axes = []\n    ndim_diff = len(a.shape) - len(shape)\n    for i in range(ndim_diff):\n        axes.append(i)\n    for i, s in enumerate(shape):\n        if s != a.shape[i+ndim_diff]:\n            assert s == 1\n            axes.append(i+ndim_diff)\n    return np.sum(a, axis=tuple(axes)).reshape(shape)\n\n\ndef is_cd_run():\n    \"\"\"Checks if the test is running as part of a Continuous Delivery run\"\"\"\n    return os.environ.get(\"CD_JOB\", 0) == \"1\"\n\n\n_features = Features()\n\n\ndef has_tvm_ops():\n    \"\"\"Returns True if MXNet is compiled with TVM generated operators. If current ctx\n    is GPU, it only returns True for CUDA compute capability > 52 where FP16 is supported.\n    \"\"\"\n    built_with_tvm_op = _features.is_enabled(\"TVM_OP\")\n    device = current_device()\n    if device.device_type == 'gpu':\n        try:\n            cc = get_cuda_compute_capability(device)\n        except:  # pylint: disable=bare-except\n            print('Failed to get CUDA compute capability for context {}. The operators '\n                  'built with USE_TVM_OP=1 will not be run in unit tests.'.format(device))\n            return False\n        print('Cuda arch compute capability: sm_{}'.format(str(cc)))\n        return built_with_tvm_op and cc >= 53\n    return built_with_tvm_op\n\n\ndef is_op_runnable():\n    \"\"\"Returns True for all CPU tests. Returns True for GPU tests that are either of the following.\n    1. Built with USE_TVM_OP=0.\n    2. Built with USE_TVM_OP=1, but with compute capability >= 53.\n    \"\"\"\n    device = current_device()\n    if device.device_type == 'gpu':\n        if not _features.is_enabled(\"TVM_OP\"):\n            return True\n        else:\n            try:\n                cc = get_cuda_compute_capability(device)\n            except:  # pylint: disable=bare-except\n                print('Failed to get CUDA compute capability for context {}. The operators '\n                      'built with USE_TVM_OP=1 will not be run in unit tests.'.format(device))\n                return False\n            print('Cuda arch compute capability: sm_{}'.format(str(cc)))\n            return cc >= 53\n    return True\n\n\n@use_np\ndef check_gluon_hybridize_consistency(net_builder, data_l, numpy_func=None, test_grad=True,\n                                      rtol=1E-4, atol=1E-4):\n    \"\"\"Check whether a HybridBlock has consistent output when hybridized or not hybridized\n\n    The network should not contain any random number generators.\n\n    Parameters\n    ----------\n    net_builder : function\n        The builder of the HybridBlock that we are going to check the consistency.\n        Inside the implementation, we will call net_builder() to construct the hybrid block.\n        Also, the net_builder will need to support specifying the params\n    data_l : list of mx.np.ndarray\n        List of input ndarrays.\n    numpy_func : function, optional\n        The ground truth numpy function that has the same functionality as net_builder().\n        Default None.\n    test_grad : bool, optional\n        Whether to test the consistency of the gradient. Default True.\n    rtol : float, optional\n        The relative error tolerance, default 1E-4. Default 1E-4.\n    atol : float, optional\n        The absolute error tolerance, default 1E-4. Default 1E-4.\n    \"\"\"\n    saved_out_np = None\n    saved_grad_np_l = None\n    params_init = None\n    use_autograd_flags = [False, True] if test_grad else [False]\n    for hybridize in [False, True]:\n        for use_autograd in use_autograd_flags:\n            net = net_builder()\n            if params_init is None:\n                net.initialize()\n            else:\n                net.load_dict(params_init)\n            if hybridize:\n                net.hybridize()\n            in_data_l = [ele.copy() for ele in data_l]\n            if use_autograd:\n                for ele in in_data_l:\n                    ele.attach_grad()\n                with mx.autograd.record():\n                    out = net(*in_data_l)\n                out.backward(out)\n            else:\n                out = net(*in_data_l)\n            if params_init is None:  # Deferred initialization finished\n                params_init = {k: v.data().asnumpy() for k, v in net.collect_params().items()}\n            if saved_out_np is None:\n                saved_out_np = out.asnumpy()\n            else:\n                # Check for correctness\n                assert_almost_equal(out.asnumpy(), saved_out_np, rtol=rtol, atol=atol)\n            if use_autograd:\n                if saved_grad_np_l is None:\n                    saved_grad_np_l = [ele.grad.asnumpy() for ele in in_data_l]\n                else:\n                    # Check for correctness\n                    for data, saved_grad_np in zip(in_data_l, saved_grad_np_l):\n                        assert_almost_equal(data.grad.asnumpy(), saved_grad_np,\n                                            rtol=rtol, atol=atol)\n    if numpy_func is not None:\n        numpy_out = numpy_func(*[ele.asnumpy() for ele in data_l])\n        assert_almost_equal(saved_out_np, numpy_out, rtol=rtol, atol=atol)\n\n\ndef new_matrix_with_real_eigvals_2d(n):\n    \"\"\"Generate a well-conditioned matrix with small real eigenvalues.\"\"\"\n    shape = (n, n)\n    q = np.ones(shape)\n    while 1:\n        D = np.diag(np.random.uniform(-1.0, 1.0, shape[-1]))\n        I = np.eye(shape[-1]).reshape(shape)\n        v = np.random.uniform(-1., 1., shape[-1]).reshape(shape[:-1] + (1,))\n        v = v / np.linalg.norm(v, axis=-2, keepdims=True)\n        v_T = np.swapaxes(v, -1, -2)\n        U = I - 2 * np.matmul(v, v_T)\n        q = np.matmul(U, D)\n        if (np.linalg.cond(q, 2) < 3):\n            break\n    D = np.diag(np.random.uniform(-10.0, 10.0, n))\n    q_inv = np.linalg.inv(q)\n    return np.matmul(np.matmul(q_inv, D), q)\n\n\ndef new_matrix_with_real_eigvals_nd(shape):\n    \"\"\"Generate well-conditioned matrices with small real eigenvalues.\"\"\"\n    n = int(np.prod(shape[:-2])) if len(shape) > 2 else 1\n    return np.array([new_matrix_with_real_eigvals_2d(shape[-1]) for i in range(n)]).reshape(shape)\n\n\ndef new_orthonormal_matrix_2d(n):\n    \"\"\"Generate a orthonormal matrix.\"\"\"\n    x = np.random.randn(n, n)\n    x_trans = x.T\n    sym_mat = np.matmul(x_trans, x)\n    return np.linalg.qr(sym_mat)[0]\n\n\ndef new_sym_matrix_with_real_eigvals_2d(n):\n    \"\"\"Generate a sym matrix with real eigenvalues.\"\"\"\n    q = new_orthonormal_matrix_2d(n)\n    D = np.diag(np.random.uniform(-10.0, 10.0, n))\n    return np.matmul(np.matmul(q.T, D), q)\n\n\ndef new_sym_matrix_with_real_eigvals_nd(shape):\n    \"\"\"Generate sym matrices with real eigenvalues.\"\"\"\n    n = int(np.prod(shape[:-2])) if len(shape) > 2 else 1\n    return np.array([new_sym_matrix_with_real_eigvals_2d(shape[-1]) for i in range(n)]).reshape(shape)\n"
  },
  {
    "path": "python/mxnet/tvmop.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n\"\"\"Init tvm ops.\"\"\"\nfrom .runtime import Features\n\nif Features().is_enabled(\"TVM_OP\"):\n    import json\n    import logging\n\n    from ._ctypes.space import _set_tvm_op_config\n    from .base import check_call, _LIB, c_str\n    from .space import ConfigSpaces\n    from .libinfo import find_lib_path, find_conf_path\n\n    _LIB_TVM_OP = find_lib_path(\"libtvmop\")\n    check_call(_LIB.MXLoadTVMOp(c_str(_LIB_TVM_OP[0])))\n\n    # op sch config\n    try:\n        _CONF_TVM_OP = find_conf_path(\"tvmop\")\n    except RuntimeError as e:\n        logging.warning(\"TVM config file missing, falling back to default schedule\", exc_info=True)\n    else:\n        logging.info(\"TVM op config has been loaded\")\n        with open(_CONF_TVM_OP[0], \"r\") as f:\n            ret = ConfigSpaces.from_json_dict(json.load(f))\n        _set_tvm_op_config(ret)\n"
  },
  {
    "path": "python/mxnet/util.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\"\"\"general utility functions\"\"\"\n\nimport ctypes\nimport functools\nimport inspect\nimport threading\nimport tempfile\nimport platform\nfrom contextlib import contextmanager\n\nfrom struct import calcsize\nfrom .base import (_LIB, check_call, c_str, py_str,\n                   numeric_types, integer_types, long,\n                   _MAX_VALUE_64_BIT_UNSIGNED_,\n                   _MAX_VALUE_64_BIT_SIGNED_,\n                   _MAX_VALUE_FLOAT32_REPRESENT_)\n\n\n_np_ufunc_default_kwargs = {\n    'where': True,\n    'casting': 'same_kind',\n    'order': 'K',\n    'dtype': None,\n    'subok': True,\n}\n\n_set_np_shape_logged = False\n_set_np_array_logged = False\n_set_np_default_dtype_logged = False\n\n\ndef get_gpu_count():\n    size = ctypes.c_int()\n    check_call(_LIB.MXGetGPUCount(ctypes.byref(size)))\n    return size.value\n\n\ndef get_gpu_memory(gpu_dev_id):\n    free_mem = ctypes.c_uint64(0)\n    total_mem = ctypes.c_uint64(0)\n    check_call(_LIB.MXGetGPUMemoryInformation64(gpu_dev_id, ctypes.byref(free_mem), ctypes.byref(total_mem)))\n    return free_mem.value, total_mem.value\n\n\ndef set_np_shape(active):\n    \"\"\"Turns on/off NumPy shape semantics, in which `()` represents the shape of scalar tensors,\n    and tuples with `0` elements, for example, `(0,)`, `(1, 0, 2)`, represent the shapes\n    of zero-size tensors. This is turned off by default for keeping backward compatibility.\n\n    Please note that this is designed as an infrastructure for the incoming\n    MXNet-NumPy operators. Legacy operators registered in the modules\n    `mx.nd` and `mx.sym` are not guaranteed to behave like their counterparts\n    in NumPy within this semantics.\n\n    Parameters\n    ----------\n    active : bool\n        Indicates whether to turn on/off NumPy shape semantics.\n\n    Returns\n    -------\n        A bool value indicating the previous state of NumPy shape semantics.\n\n    Example\n    -------\n    >>> import mxnet as mx\n    >>> prev_state = mx.set_np_shape(True)\n    >>> print(prev_state)\n    False\n    >>> print(mx.is_np_shape())\n    True\n    \"\"\"\n    global _set_np_shape_logged\n    if active:\n        if not _set_np_shape_logged:\n            import logging\n            logging.info('NumPy-shape semantics has been activated in your code. '\n                         'This is required for creating and manipulating scalar and zero-size '\n                         'tensors, which were not supported in MXNet before, as in the official '\n                         'NumPy library. Please DO NOT manually deactivate this semantics while '\n                         'using `mxnet.numpy` and `mxnet.numpy_extension` modules.')\n            _set_np_shape_logged = True\n    elif is_np_array():\n        raise ValueError('Deactivating NumPy shape semantics while NumPy array semantics is still'\n                         ' active is not allowed. Please consider calling `npx.reset_np()` to'\n                         ' deactivate both of them.')\n    prev = ctypes.c_int()\n    check_call(_LIB.MXSetIsNumpyShape(ctypes.c_int(active), ctypes.byref(prev)))\n    return bool(prev.value)\n\n\ndef is_np_shape():\n    \"\"\"Checks whether the NumPy shape semantics is currently turned on.\n    In NumPy shape semantics, `()` represents the shape of scalar tensors,\n    and tuples with `0` elements, for example, `(0,)`, `(1, 0, 2)`, represent\n    the shapes of zero-size tensors. This is turned off by default for keeping\n    backward compatibility.\n\n    In the NumPy shape semantics, `-1` indicates an unknown size. For example,\n    `(-1, 2, 2)` means that the size of the first dimension is unknown. Its size\n    may be inferred during shape inference.\n\n    Please note that this is designed as an infrastructure for the incoming\n    MXNet-NumPy operators. Legacy operators registered in the modules\n    `mx.nd` and `mx.sym` are not guaranteed to behave like their counterparts\n    in NumPy within this semantics.\n\n    Returns\n    -------\n        A bool value indicating whether the NumPy shape semantics is currently on.\n\n    Example\n    -------\n    >>> import mxnet as mx\n    >>> prev_state = mx.set_np_shape(True)\n    >>> print(prev_state)\n    False\n    >>> print(mx.is_np_shape())\n    True\n    \"\"\"\n    curr = ctypes.c_bool()\n    check_call(_LIB.MXIsNumpyShape(ctypes.byref(curr)))\n    return curr.value\n\n\nclass _NumpyShapeScope(object):\n    \"\"\"Scope for managing NumPy shape semantics.\n    In NumPy shape semantics, `()` represents the shape of scalar tensors,\n    and tuples with `0` elements, for example, `(0,)`, `(1, 0, 2)`, represent\n    the shapes of zero-size tensors.\n\n    Do not use this class directly. Use `np_shape(active)` instead.\n\n    Example::\n\n        with _NumpyShapeScope(True):\n            y = model(x)\n            backward([y])\n\n    \"\"\"\n    def __init__(self, is_np_shape):  #pylint: disable=redefined-outer-name\n        self._enter_is_np_shape = is_np_shape\n        self._prev_is_np_shape = None\n\n    def __enter__(self):\n        if self._enter_is_np_shape is not None:\n            self._prev_is_np_shape = set_np_shape(self._enter_is_np_shape)\n\n    def __exit__(self, ptype, value, trace):\n        if self._enter_is_np_shape is not None and self._prev_is_np_shape != self._enter_is_np_shape:\n            set_np_shape(self._prev_is_np_shape)\n\n\ndef np_shape(active=True):\n    \"\"\"Returns an activated/deactivated NumPy shape scope to be used in 'with' statement\n    and captures code that needs the NumPy shape semantics, i.e. support of scalar and\n    zero-size tensors.\n\n    Please note that this is designed as an infrastructure for the incoming\n    MXNet-NumPy operators. Legacy operators registered in the modules\n    `mx.nd` and `mx.sym` are not guaranteed to behave like their counterparts\n    in NumPy even within this scope.\n\n    Parameters\n    ----------\n    active : bool\n        Indicates whether to activate NumPy-shape semantics.\n\n    Returns\n    -------\n    _NumpyShapeScope\n        A scope object for wrapping the code w/ or w/o NumPy-shape semantics.\n\n    Example::\n\n        with mx.np_shape(active=True):\n            # A scalar tensor's shape is `()`, whose `ndim` is `0`.\n            scalar = mx.nd.ones(shape=())\n            assert scalar.shape == ()\n\n            # If NumPy shape semantics is enabled, 0 in a shape means that\n            # dimension contains zero elements.\n            data = mx.sym.var(\"data\", shape=(0, 2, 3))\n            ret = mx.sym.sin(data)\n            arg_shapes, out_shapes, _ = ret.infer_shape()\n            assert arg_shapes[0] == (0, 2, 3)\n            assert out_shapes[0] == (0, 2, 3)\n\n            # -1 means unknown shape dimension size in the new NumPy shape definition\n            data = mx.sym.var(\"data\", shape=(-1, 2, 3))\n            ret = mx.sym.sin(data)\n            arg_shapes, out_shapes, _ = ret.infer_shape_partial()\n            assert arg_shapes[0] == (-1, 2, 3)\n            assert out_shapes[0] == (-1, 2, 3)\n\n            # When a shape is completely unknown when NumPy shape semantics is on, it is\n            # represented as `None` in Python.\n            data = mx.sym.var(\"data\")\n            ret = mx.sym.sin(data)\n            arg_shapes, out_shapes, _ = ret.infer_shape_partial()\n            assert arg_shapes[0] is None\n            assert out_shapes[0] is None\n\n        with mx.np_shape(active=False):\n            # 0 means unknown shape dimension size in the legacy shape definition.\n            data = mx.sym.var(\"data\", shape=(0, 2, 3))\n            ret = mx.sym.sin(data)\n            arg_shapes, out_shapes, _ = ret.infer_shape_partial()\n            assert arg_shapes[0] == (0, 2, 3)\n            assert out_shapes[0] == (0, 2, 3)\n\n            # When a shape is completely unknown in the legacy mode (default), its ndim is\n            # equal to 0 and it is represented as `()` in Python.\n            data = mx.sym.var(\"data\")\n            ret = mx.sym.sin(data)\n            arg_shapes, out_shapes, _ = ret.infer_shape_partial()\n            assert arg_shapes[0] == ()\n            assert out_shapes[0] == ()\n    \"\"\"\n    return _NumpyShapeScope(active)\n\n\ndef use_np_shape(func):\n    \"\"\"A decorator wrapping a function or class with activated NumPy-shape semantics.\n    When `func` is a function, this ensures that the execution of the function is scoped with NumPy\n    shape semantics, such as the support for zero-dim and zero size tensors. When\n    `func` is a class, it ensures that all the methods, static functions, and properties\n    of the class are executed with the NumPy shape semantics.\n\n    .. code-block:: python\n\n        import mxnet as mx\n        @mx.use_np_shape\n        def scalar_one():\n            return mx.nd.ones(())\n        print(scalar_one())\n\n        @np.use_np_shape\n        class ScalarTensor(object):\n            def __init__(self, val=None):\n                if val is None:\n                    val = ScalarTensor.random().value\n                self._scalar = mx.nd.ones(()) * val\n\n            def __repr__(self):\n                print(\"Is __repr__ in np_shape semantics? {}!\".format(str(np.is_np_shape())))\n                return str(self._scalar.asnumpy())\n\n            @staticmethod\n            def random():\n                val = mx.nd.random.uniform().asnumpy().item()\n                return ScalarTensor(val)\n\n            @property\n            def value(self):\n                print(\"Is value property in np_shape semantics? {}!\".format(str(np.is_np_shape())))\n                return self._scalar.asnumpy().item()\n\n        print(\"Is global scope of np_shape activated? {}!\".format(str(np.is_np_shape())))\n        scalar_tensor = ScalarTensor()\n        print(scalar_tensor)\n\n\n    Parameters\n    ----------\n    func : a user-provided callable function or class to be scoped by the NumPy-shape semantics.\n\n    Returns\n    -------\n    Function or class\n        A function or class wrapped in the NumPy-shape scope.\n    \"\"\"\n\n    if inspect.isclass(func):\n        for name, method in inspect.getmembers(\n                func,\n                predicate=\n                lambda f: inspect.isfunction(f) or inspect.ismethod(f) or isinstance(f, property)):\n            if isinstance(method, property):\n                setattr(func, name, property(use_np_shape(method.__get__),\n                                             method.__set__,\n                                             method.__delattr__,\n                                             method.__doc__))\n            else:\n                setattr(func, name, use_np_shape(method))\n        return func\n    elif callable(func):\n        @functools.wraps(func)\n        def _with_np_shape(*args, **kwargs):\n            with np_shape(active=True):\n                return func(*args, **kwargs)\n        return _with_np_shape\n    else:\n        raise TypeError('use_np_shape can only decorate classes and callable objects, '\n                        'while received a {}'.format(str(type(func))))\n\n\ndef _sanity_check_params(func_name, unsupported_params, param_dict):\n    for param_name in unsupported_params:\n        if param_name in param_dict:\n            raise NotImplementedError(\"function {} does not support parameter {}\"\n                                      .format(func_name, param_name))\n\n\ndef set_module(module):\n    \"\"\"Decorator for overriding __module__ on a function or class.\n\n    Example usage::\n\n        @set_module('mxnet.numpy')\n        def example():\n            pass\n\n        assert example.__module__ == 'numpy'\n    \"\"\"\n    def decorator(func):\n        if module is not None:\n            func.__module__ = module\n        return func\n    return decorator\n\n\nclass _NumpyArrayScope(object):\n    \"\"\"Scope for managing NumPy array creation. This is often used\n    with `is_np_array=True` in initializer to enforce array creation\n    as type `mxnet.numpy.ndarray`, instead of `mx.nd.NDArray` in Gluon.\n\n    Do not use this class directly. Use `np_array(active)` instead.\n    \"\"\"\n    _current = threading.local()\n\n    def __init__(self, is_np_array):  # pylint: disable=redefined-outer-name\n        self._old_scope = None\n        self._is_np_array = is_np_array\n\n    def __enter__(self):\n        if not hasattr(_NumpyArrayScope._current, \"value\"):\n            _NumpyArrayScope._current.value = _NumpyArrayScope(False)\n        self._old_scope = _NumpyArrayScope._current.value\n        _NumpyArrayScope._current.value = self\n        return self\n\n    def __exit__(self, ptype, value, trace):\n        assert self._old_scope\n        _NumpyArrayScope._current.value = self._old_scope\n\n\ndef np_array(active=True):\n    \"\"\"Returns an activated/deactivated NumPy-array scope to be used in 'with' statement\n    and captures code that needs the NumPy-array semantics.\n\n    Currently, this is used in Gluon to enforce array creation in `Block`s as type\n    `mxnet.numpy.ndarray`, instead of `mx.nd.NDArray`.\n\n    It is recommended to use the decorator `use_np_array` to decorate the classes\n    that need this semantics, instead of using this function in a `with` statement\n    unless you know exactly what has been scoped by this semantics.\n\n    Please note that this is designed as an infrastructure for the incoming\n    MXNet-NumPy operators. Legacy operators registered in the modules\n    `mx.nd` and `mx.sym` are not guaranteed to behave like their counterparts\n    in NumPy even within this scope.\n\n    Parameters\n    ----------\n    active : bool\n        Indicates whether to activate NumPy-array semantics.\n\n    Returns\n    -------\n    _NumpyShapeScope\n        A scope object for wrapping the code w/ or w/o NumPy-shape semantics.\n    \"\"\"\n    return _NumpyArrayScope(active)\n\n\ndef is_np_array():\n    \"\"\"Checks whether the NumPy-array semantics is currently turned on.\n    This is currently used in Gluon for checking whether an array of type `mxnet.numpy.ndarray`\n    or `mx.nd.NDArray` should be created. For example, at the time when a parameter\n    is created in a `Block`, an `mxnet.numpy.ndarray` is created if this returns true; else\n    an `mx.nd.NDArray` is created.\n\n    Normally, users are not recommended to use this API directly unless you known exactly\n    what is going on under the hood.\n\n    Please note that this is designed as an infrastructure for the incoming\n    MXNet-NumPy operators. Legacy operators registered in the modules\n    `mx.nd` and `mx.sym` are not guaranteed to behave like their counterparts\n    in NumPy within this semantics.\n\n    Returns\n    -------\n        A bool value indicating whether the NumPy-array semantics is currently on.\n    \"\"\"\n    return _NumpyArrayScope._current.value._is_np_array if hasattr(\n        _NumpyArrayScope._current, \"value\") else False\n\n\ndef use_np_array(func):\n    \"\"\"A decorator wrapping Gluon `Block`s and all its methods, properties, and static functions\n    with the semantics of NumPy-array, which means that where ndarrays are created,\n    `mxnet.numpy.ndarray`s should be created, instead of legacy ndarrays of type `mx.nd.NDArray`.\n    For example, at the time when a parameter is created in a `Block`, an `mxnet.numpy.ndarray`\n    is created if it's decorated with this decorator.\n\n    .. code-block:: python\n\n        import mxnet as mx\n        from mxnet import gluon, nd, np\n        from mxnet.gluon import Parameter\n\n        class TestHybridBlock1(gluon.HybridBlock):\n            def __init__(self):\n                super(TestHybridBlock1, self).__init__()\n                self.w = Parameter('w', shape=(2, 2))\n\n            def forward(self, x):\n                return nd.dot(x, self.w.data())\n\n        x = mx.nd.ones((2, 2))\n        net1 = TestHybridBlock1()\n        net1.initialize()\n        out = net1.forward(x)\n        for _, v in net1.collect_params().items():\n            assert type(v.data()) is mx.nd.NDArray\n        assert type(out) is mx.nd.NDArray\n\n        @mx.util.use_np_array\n        class TestHybridBlock2(gluon.HybridBlock):\n            def __init__(self):\n                super(TestHybridBlock2, self).__init__()\n                self.w = Parameter('w', shape=(2, 2))\n\n            def forward(self, x):\n                return np.dot(x, self.w.data())\n\n        x = np.ones((2, 2))\n        net2 = TestHybridBlock2()\n        net2.initialize()\n        out = net2.forward(x)\n        for _, v in net2.collect_params().items():\n            print(type(v.data()))\n            assert type(v.data()) is np.ndarray\n        assert type(out) is np.ndarray\n\n    Parameters\n    ----------\n    func : a user-provided callable function or class to be scoped by the NumPy-array semantics.\n\n    Returns\n    -------\n    Function or class\n        A function or class wrapped in the NumPy-array scope.\n    \"\"\"\n    if inspect.isclass(func):\n        for name, method in inspect.getmembers(\n                func,\n                predicate=\n                lambda f: inspect.isfunction(f) or inspect.ismethod(f) or isinstance(f, property)):\n            if isinstance(method, property):\n                setattr(func, name, property(use_np_array(method.__get__),\n                                             method.__set__,\n                                             method.__delattr__,\n                                             method.__doc__))\n            else:\n                setattr(func, name, use_np_array(method))\n        return func\n    elif callable(func):\n        @functools.wraps(func)\n        def _with_np_array(*args, **kwargs):\n            with np_array(active=True):\n                return func(*args, **kwargs)\n        return _with_np_array\n    else:\n        raise TypeError('use_np_array can only decorate classes and callable objects, '\n                        'while received a {}'.format(str(type(func))))\n\n\ndef use_np(func):\n    \"\"\"A convenience decorator for wrapping user provided functions and classes in the scope of\n    both NumPy-shape and NumPy-array semantics, which means that ``(1)`` empty tuples ``()`` and\n    tuples with zeros, such as ``(0, 1)``, ``(1, 0, 2)``, will be treated as scalar tensors' shapes and\n    zero-size tensors' shapes in shape inference functions of operators, instead of as unknown\n    in legacy mode; (2) ndarrays of type :class:`mxnet.numpy.ndarray` should be created instead of\n    :class:`mx.nd.NDArray`.\n\n    .. code-block:: python\n\n        import mxnet as mx\n        from mxnet import gluon, nd, np\n        from mxnet.gluon import Parameter\n\n        class TestHybridBlock1(gluon.HybridBlock):\n            def __init__(self):\n                super(TestHybridBlock1, self).__init__()\n                self.w = Parameter('w', shape=(2, 2))\n\n            def forward(self, x):\n                return nd.dot(x, self.w.data()) + nd.ones((1,))\n\n        x = mx.nd.ones((2, 2))\n        net1 = TestHybridBlock1()\n        net1.initialize()\n        out = net1.forward(x)\n        for _, v in net1.collect_params().items():\n            assert type(v.data()) is mx.nd.NDArray\n        assert type(out) is mx.nd.NDArray\n\n        @mx.util.use_np\n        class TestHybridBlock2(gluon.HybridBlock):\n            def __init__(self):\n                super(TestHybridBlock2, self).__init__()\n                self.w = Parameter('w', shape=(2, 2))\n\n            def forward(self, x):\n                return np.dot(x, self.w.data()) + np.ones(())\n\n        x = np.ones((2, 2))\n        net2 = TestHybridBlock2()\n        net2.initialize()\n        out = net2.forward(x)\n        for _, v in net2.collect_params().items():\n            print(type(v.data()))\n            assert type(v.data()) is np.ndarray\n        assert type(out) is np.ndarray\n\n\n    Parameters\n    ----------\n    func : a user-provided callable function or class to be scoped by the\n        NumPy-shape and NumPy-array semantics.\n\n    Returns\n    -------\n    Function or class\n        A function or class wrapped in the Numpy-shape and NumPy-array scope.\n    \"\"\"\n    return use_np_shape(use_np_array(func))\n\n\ndef np_ufunc_legal_option(key, value):\n    \"\"\"Checking if ufunc arguments are legal inputs\n\n    Parameters\n    ----------\n    key : string\n        the key of the ufunc argument.\n    value : string\n        the value of the ufunc argument.\n\n    Returns\n    -------\n    legal : boolean\n        Whether or not the argument is a legal one. True when the key is one of the ufunc\n        arguments and value is an allowed value. False when the key is not one of the ufunc\n        arugments or the value is not an allowed value even when the key is a legal one.\n    \"\"\"\n    if key == 'where':\n        return True\n    elif key == 'casting':\n        return (value in set(['no', 'equiv', 'safe', 'same_kind', 'unsafe']))\n    elif key == 'order':\n        if isinstance(value, str):\n            return True\n    elif key == 'dtype':\n        import numpy as _np\n        return (value in set([_np.int8, _np.uint8, _np.int32, _np.int64,\n                              _np.float16, _np.float32, _np.float64,\n                              'int8', 'uint8', 'int32', 'int64',\n                              'float16', 'float32', 'float64']))\n    elif key == 'subok':\n        return isinstance(value, bool)\n    return False\n\n\ndef wrap_np_unary_func(func):\n    \"\"\"A convenience decorator for wrapping numpy-compatible unary ufuncs to provide uniform\n    error handling.\n\n    Parameters\n    ----------\n    func : a numpy-compatible unary function to be wrapped for better error handling.\n\n    Returns\n    -------\n    Function\n        A function wrapped with proper error handling.\n    \"\"\"\n    @functools.wraps(func)\n    def _wrap_np_unary_func(x, out=None, **kwargs):\n        if len(kwargs) != 0:\n            for key, value in kwargs.items():\n                # if argument is not in the set of ufunc arguments\n                if key not in _np_ufunc_default_kwargs:\n                    raise TypeError(\"{} is an invalid keyword to function \\'{}\\'\".format(key, func.__name__))\n                # if argument is one of the ufunc arguments, but not with the default value\n                if value != _np_ufunc_default_kwargs[key]:\n                    # if the provided value of the argument is a legal option, raise NotImplementedError\n                    if np_ufunc_legal_option(key, value):\n                        raise NotImplementedError(\"{}={} is not implemented yet for operator {}\"\n                                                  .format(key, str(value), func.__name__))\n                    # otherwise raise TypeError with not understood error message\n                    raise TypeError(\"{}={} not understood for operator {}\"\n                                    .format(key, value, func.__name__))\n        return func(x, out=out)\n    return _wrap_np_unary_func\n\n\ndef wrap_np_binary_func(func):\n    \"\"\"A convenience decorator for wrapping numpy-compatible binary ufuncs to provide uniform\n    error handling.\n\n    Parameters\n    ----------\n    func : a numpy-compatible binary function to be wrapped for better error handling.\n\n    Returns\n    -------\n    Function\n        A function wrapped with proper error handling.\n    \"\"\"\n    @functools.wraps(func)\n    def _wrap_np_binary_func(x1, x2, out=None, **kwargs):\n        if len(kwargs) != 0:\n            for key, value in kwargs.items():\n                # if argument is not in the set of ufunc arguments\n                if key not in _np_ufunc_default_kwargs:\n                    raise TypeError(\"{} is an invalid keyword to function \\'{}\\'\".format(key, func.__name__))\n                # if argument is one of the ufunc arguments, but not with the default value\n                if value != _np_ufunc_default_kwargs[key]:\n                    # if the provided value of the argument is a legal option, raise NotImplementedError\n                    if np_ufunc_legal_option(key, value):\n                        raise NotImplementedError(\"{}={} is not implemented yet\".format(key, str(value)))\n                    # otherwise raise TypeError with not understood error message\n                    raise TypeError(\"{} {} not understood\".format(key, value))\n        return func(x1, x2, out=out)\n    return _wrap_np_binary_func\n\ndef wrap_data_api_statical_func(func):\n    \"\"\"\n    A convenience decorator for wrapping data apis standardized statical functions to provide\n    context keyward backward compatibility\n    Parameters\n    ----------\n    func : a numpy-compatible array statical function to be wrapped for context keyward change.\n    Returns\n    -------\n    Function\n    A function wrapped with context keyward changes.\n    \"\"\"\n\n    @functools.wraps(func)\n    def _wrap_api_creation_func(*args, **kwargs):\n        if len(kwargs) != 0:\n            correction = kwargs.pop('ddof', None)\n            if correction is not None:\n                kwargs['correction'] = correction\n        return func(*args, **kwargs)\n\n    return _wrap_api_creation_func\n\ndef wrap_data_api_linalg_func(func):\n    \"\"\"\n    A convenience decorator for wrapping data apis standardized linalg functions to provide\n    context keyward backward compatibility\n    Parameters\n    ----------\n    func : a numpy-compatible array linalg function to be wrapped for context keyward change.\n    Returns\n    -------\n    Function\n    A function wrapped with context keyward changes.\n    \"\"\"\n\n    @functools.wraps(func)\n    def _wrap_linalg_func(*args, **kwargs):\n        if len(kwargs) != 0:\n            upper = kwargs.pop('UPLO', None)\n            rcond = kwargs.pop('rcond', None)\n            tol = kwargs.pop('tol', None)\n            if upper is not None:\n                if upper == 'U':\n                    kwargs['upper'] = True\n                else:\n                    kwargs['upper'] = False\n            if rcond is not None:\n                kwargs['rtol'] = rcond\n            if tol is not None:\n                kwargs['rtol'] = tol\n        return func(*args, **kwargs)\n\n    return _wrap_linalg_func\n\n\ndef wrap_sort_functions(func):\n    \"\"\"A convenience decorator for wrapping sort functions\n\n    Parameters\n    ----------\n    func : a numpy-compatible array creation function to be wrapped for parameter keyword change.\n\n    Returns\n    -------\n    Function\n        A function wrapped with changed keywords.\n    \"\"\"\n    @functools.wraps(func)\n    def _wrap_sort_func(*args, **kwargs):\n        if len(kwargs) != 0:\n            kind = kwargs.pop('kind', None)\n            order = kwargs.pop('order', None)\n            if kind is not None:\n                kwargs['stable'] = kind == 'stable'\n            if order is not None:\n                raise NotImplementedError(\"order not supported here\")\n        return func(*args, **kwargs)\n    return _wrap_sort_func\n\n\ndef wrap_ctx_to_device_func(func):\n    \"\"\"A convenience decorator for converting ctx to device keyward backward compatibility\n\n    Parameters\n    ----------\n    func : a function to be wrapped for context keyward change.\n\n    Returns\n    -------\n    Function\n        A function wrapped with context keyward changes.\n    \"\"\"\n    @functools.wraps(func)\n    def _wrap_func_with_ctx(*args, **kwargs):\n        if len(kwargs) != 0:\n            device = kwargs.pop('ctx', None)\n            if device is not None:\n                kwargs['device'] = device\n        return func(*args, **kwargs)\n    return _wrap_func_with_ctx\n\n\n# pylint: disable=exec-used\ndef numpy_fallback(func):\n    \"\"\"decorator for falling back to offical numpy for a specific function\"\"\"\n    def get_device(device, new_device):\n        if device is None:\n            return new_device\n        else:\n            if new_device is None:\n                new_device = device\n            assert device == new_device, f\"inconsistent device {str(device)} and {str(new_device)}\"\n            return device\n\n    def _as_official_np_array(object):\n        device = None\n        if hasattr(object, 'asnumpy'):\n            return object.asnumpy(), object.device\n        elif isinstance(object, (list, tuple)):\n            tmp = []\n            for arr in object:\n                new_arr, new_device = _as_official_np_array(arr)\n                device = get_device(device, new_device)\n                tmp.append(new_arr)\n            return object.__class__(tmp), device\n        elif isinstance(object, dict):\n            tmp = {}\n            for k, v in object.items():\n                new_v, new_device = _as_official_np_array(v)\n                device = get_device(device, new_device)\n                tmp[k] = new_v\n            return tmp, device\n        else:\n            return object, None\n\n    from .ndarray import from_numpy\n    from .numpy import array\n    from .device import current_device\n    def _as_mx_np_array(object, device=current_device()):\n        import numpy as _np\n        if isinstance(object, _np.ndarray):\n            try:\n                ret = from_numpy(object).as_np_ndarray()\n            except ValueError:\n                ret = array(object, dtype=object.dtype, device=device)\n            return (ret if ('cpu' in str(device)) else ret.to_device(device))\n        elif isinstance(object, (list, tuple)):\n            tmp = [_as_mx_np_array(arr, device) for arr in object]\n            return object.__class__(tmp)\n        elif isinstance(object, dict):\n            return {k:_as_mx_np_array(v, device) for k, v in object}\n        else:\n            return object\n\n    import re\n    func_name = func.__name__\n    func_doc = func.__doc__\n    func_source = inspect.getsource(func)\n    func_source = re.sub(r'np\\.', 'onp.', func_source)\n    func_source = func_source.split('\\n')[1:]\n    indentation = func_source[0].find('def')\n    if indentation == -1:\n        raise ValueError(\"should wrap a function\")\n    stripped = []\n    for line in func_source:\n        stripped.append(line[indentation:])\n    stripped.insert(1, '    import numpy as onp')\n    func_source = '\\n'.join(stripped)\n    local = {}\n    exec(func_source, None, local)\n    func = local[func_name]\n    func.__doc__ = func_doc\n\n    @functools.wraps(func)\n    def _fallback_to_official_np(*args, **kwargs):\n        # for every ndarray input, fallback\n        new_args, device0 = _as_official_np_array(args)\n        new_kwargs, device1 = _as_official_np_array(kwargs)\n        device = get_device(device0, device1)\n        ret = func(*new_args, **new_kwargs)\n        if ret is None:\n            raise ValueError(\"Only functions with return values are allowed to use this decorator\")\n        ret = _as_mx_np_array(ret, device=device)\n        return ret\n\n    return _fallback_to_official_np\n# pylint: enable=exec-used\n\n\ndef _set_np_array(active):\n    \"\"\"Turns on/off NumPy array semantics for the current thread in which `mxnet.numpy.ndarray`\n    is expected to be created, instead of the legacy `mx.nd.NDArray`.\n\n    Parameters\n    ---------\n    active : bool\n        A boolean value indicating whether the NumPy-array semantics should be turned on or off.\n\n    Returns\n    -------\n        A bool value indicating the previous state of NumPy array semantics.\n    \"\"\"\n    global _set_np_array_logged\n    if active:\n        if not _set_np_array_logged:\n            import logging\n            logging.info('NumPy array semantics has been activated in your code. This allows you'\n                         ' to use operators from MXNet NumPy and NumPy Extension modules as well'\n                         ' as MXNet NumPy `ndarray`s.')\n            _set_np_array_logged = True\n    cur_state = is_np_array()\n    _NumpyArrayScope._current.value = _NumpyArrayScope(active)\n    return cur_state\n\n\ndef set_np(shape=True, array=True, dtype=False):\n    \"\"\"Setting NumPy shape and array semantics at the same time.\n    It is required to keep NumPy shape semantics active while activating NumPy array semantics.\n    Deactivating NumPy shape semantics while NumPy array semantics is still active is not allowed.\n    It is highly recommended to set these two flags to `True` at the same time to fully enable\n    NumPy-like behaviors. Please refer to the Examples section for a better understanding.\n\n    Parameters\n    ----------\n    shape : bool\n        A boolean value indicating whether the NumPy-shape semantics should be turned on or off.\n        When this flag is set to `True`, zero-size and zero-dim shapes are all valid shapes in\n        shape inference process, instead of treated as unknown shapes in legacy mode.\n    array : bool\n        A boolean value indicating whether the NumPy-array semantics should be turned on or off.\n        When this flag is set to `True`, it enables Gluon code flow to use or generate `mxnet.numpy.ndarray`s\n        instead of `mxnet.ndarray.NDArray`. For example, a `Block` would create parameters of type\n        `mxnet.numpy.ndarray`.\n    dtype : bool\n         A boolean value indicating whether the NumPy-dtype semantics should be turned on or off.\n         When this flag is set to `True`, default dtype is float64.\n         When this flag is set to `False`, default dtype is float32.\n    Examples\n    --------\n    >>> import mxnet as mx\n\n    Creating zero-dim ndarray in legacy mode would fail at shape inference.\n\n    >>> mx.nd.ones(shape=())\n    mxnet.base.MXNetError: Operator _ones inferring shapes failed.\n\n    >>> mx.nd.ones(shape=(2, 0, 3))\n    mxnet.base.MXNetError: Operator _ones inferring shapes failed.\n\n    In legacy mode, Gluon layers would create parameters and outputs of type `mx.nd.NDArray`.\n\n    >>> from mxnet.gluon import nn\n    >>> dense = nn.Dense(2)\n    >>> dense.initialize()\n    >>> dense(mx.nd.ones(shape=(3, 2)))\n    [[0.01983214 0.07832371]\n     [0.01983214 0.07832371]\n     [0.01983214 0.07832371]]\n    <NDArray 3x2 @cpu(0)>\n\n    >>> [p.data() for p in dense.collect_params().values()]\n    [\n    [[0.0068339  0.01299825]\n     [0.0301265  0.04819721]]\n    <NDArray 2x2 @cpu(0)>,\n    [0. 0.]\n    <NDArray 2 @cpu(0)>]\n\n    When the `shape` flag is `True`, both shape inferences are successful.\n\n    >>> from mxnet import np, npx\n    >>> npx.set_np()  # this is required to activate NumPy-like behaviors\n\n    >>> np.ones(shape=())\n    array(1.)\n    >>> np.ones(shape=(2, 0, 3))\n    array([], shape=(2, 0, 3))\n\n    When the `array` flag is `True`, Gluon layers would create parameters and outputs of type `mx.np.ndarray`.\n\n    >>> dense = nn.Dense(2)\n    >>> dense.initialize()\n    >>> dense(np.ones(shape=(3, 2)))\n    array([[0.01983214, 0.07832371],\n           [0.01983214, 0.07832371],\n           [0.01983214, 0.07832371]])\n\n    >>> [p.data() for p in dense.collect_params().values()]\n    [array([[0.0068339 , 0.01299825],\n           [0.0301265 , 0.04819721]]), array([0., 0.])]\n\n    >>> npx.set_np(dtype=True)\n    >>> np.ones(shape=()).dtype\n    dtype('float64')\n    \"\"\"\n    if not shape and array:\n        raise ValueError('NumPy Shape semantics is required in using NumPy array semantics.')\n    _set_np_array(array)\n    set_np_shape(shape)\n    set_np_default_dtype(dtype)\n\n\ndef reset_np():\n    \"\"\"Deactivate NumPy shape and array and deafult dtype semantics at the same time.\"\"\"\n    set_np(shape=False, array=False, dtype=False)\n\n\n_CUDA_SUCCESS = 0\n\n\ndef get_cuda_compute_capability(device):\n    \"\"\"Returns the cuda compute capability of the input `device`.\n\n    Parameters\n    ----------\n    device : Device\n        GPU context whose corresponding cuda compute capability is to be retrieved.\n\n    Returns\n    -------\n    cuda_compute_capability : int\n        CUDA compute capability. For example, it returns 70 for CUDA arch equal to `sm_70`.\n\n    References\n    ----------\n    https://gist.github.com/f0k/63a664160d016a491b2cbea15913d549#file-cuda_check-py\n    \"\"\"\n    if device.device_type != 'gpu':\n        raise ValueError('Expecting a gpu context to get cuda compute capability, '\n                         'while received device {}'.format(str(device)))\n\n    libnames = ('libcuda.so', 'libcuda.dylib', 'nvcuda.dll', 'cuda.dll')\n    for libname in libnames:\n        try:\n            cuda = ctypes.CDLL(libname)\n        except OSError:\n            continue\n        else:\n            break\n    else:\n        raise OSError(\"could not load any of: \" + ' '.join(libnames))\n\n    # Some constants taken from cuda.h\n\n    cc_major = ctypes.c_int()\n    cc_minor = ctypes.c_int()\n    cuda_device = ctypes.c_int()\n    error_str = ctypes.c_char_p()\n\n    ret = cuda.cuInit(0)\n    if ret != _CUDA_SUCCESS:\n        cuda.cuGetErrorString(ret, ctypes.byref(error_str))\n        raise RuntimeError('cuInit failed with erro code {}: {}'\n                           .format(ret, error_str.value.decode()))\n\n    ret = cuda.cuDeviceGet(ctypes.byref(cuda_device), device.device_id)\n    if ret != _CUDA_SUCCESS:\n        cuda.cuGetErrorString(ret, ctypes.byref(error_str))\n        raise RuntimeError('cuDeviceGet failed with error code {}: {}'\n                           .format(ret, error_str.value.decode()))\n    ret = cuda.cuDeviceComputeCapability(ctypes.byref(cc_major), ctypes.byref(cc_minor), cuda_device)\n    if ret != _CUDA_SUCCESS:\n        cuda.cuGetErrorString(ret, ctypes.byref(error_str))\n        raise RuntimeError('cuDeviceComputeCapability failed with error code {}: {}'\n                           .format(ret, error_str.value.decode()))\n    return cc_major.value * 10 + cc_minor.value\n\n\ndef default_array(source_array, device=None, dtype=None):\n    \"\"\"Creates an array from any object exposing the default(nd or np) array interface.\n\n    Parameters\n    ----------\n    source_array : array_like\n        An object exposing the array interface, an object whose `__array__`\n        method returns an array, or any (nested) sequence.\n    device : Device, optional\n        Device context (default is the current default context).\n    dtype : str or numpy.dtype, optional\n        The data type of the output array. The default dtype is ``source_array.dtype``\n        if `source_array` is an `NDArray`, `float32` otherwise.\n\n    Returns\n    -------\n    NDArray\n        An `NDArray`(nd or np) with the same contents as the `source_array`.\n    \"\"\"\n    from . import nd as _mx_nd\n    from . import np as _mx_np\n    if is_np_array():\n        return _mx_np.array(source_array, device=device, dtype=dtype)\n    else:\n        return _mx_nd.array(source_array, ctx=device, dtype=dtype)\n\nclass _NumpyDefaultDtypeScope(object):\n    \"\"\"Scope for managing NumPy default dtype semantics.\n    In NumPy default dtype semantics, default dtype is 'float64',\n    i.e. np.array([1, 2, 3]).dtype = np.float64\n    Original default dtype without this semantic is 'float32'.\n\n    Do not use this class directly. Use `np_shape(active)` instead.\n\n    Example::\n\n        with _NumpyDefaultDtypeScope(True):\n            y = model(x)\n            backward([y])\n\n    \"\"\"\n    def __init__(self, is_np_default_dtype):  #pylint: disable=redefined-outer-name\n        self._enter_is_np_default_dtype = is_np_default_dtype\n        self._prev_is_np_default_dtype = None\n\n    def __enter__(self):\n        if self._enter_is_np_default_dtype is not None:\n            self._prev_is_np_default_dtype = set_np_default_dtype(self._enter_is_np_default_dtype)\n\n    def __exit__(self, ptype, value, trace):\n        if self._enter_is_np_default_dtype is not None and\\\n           self._prev_is_np_default_dtype != self._enter_is_np_default_dtype:\n            set_np_default_dtype(self._prev_is_np_default_dtype)\n\ndef np_default_dtype(active=True):\n    \"\"\"Returns an activated/deactivated NumPy-default_dtype scope to be used in 'with' statement\n    and captures code that needs the NumPy default dtype semantics. i.e. default dtype is float64.\n\n    Please note that this is designed as an infrastructure for the incoming\n    MXNet-NumPy operators. Legacy operators registered in the modules\n    `mx.nd` and `mx.sym` are not guaranteed to behave like their counterparts\n    in NumPy even within this scope.\n\n    Parameters\n    ----------\n    active : bool\n        Indicates whether to activate NumPy default dtype semantics.\n\n    Returns\n    -------\n    _NumpyDefaultDtypeScope\n        A scope object for wrapping the code w/ or w/o NumPy-default_dtype semantics.\n\n    Example::\n\n        with mx.np_default_dtype(active=True):\n            # Default Dtype is 'float64', consistent with offical NumPy behavior.\n            arr = mx.np.array([1, 2, 3])\n            assert arr.dtype == 'float64'\n\n        with mx.np_default_dtype(active=False):\n            # Default Dtype is 'float32' in the legacy default dtype definition.\n            arr = mx.np.array([1, 2, 3])\n            assert arr.dtype == 'float32'\n\n    \"\"\"\n    return _NumpyDefaultDtypeScope(active)\n\ndef use_np_default_dtype(func):\n    \"\"\"A decorator wrapping a function or class with activated NumPy-default_dtype semantics.\n    When `func` is a function, this ensures that the execution of the function is scoped with NumPy\n    default dtype semantics, with the support for float64 as default dtype.\n    When`func` is a class, it ensures that all the methods, static functions, and properties\n    of the class are executed with the NumPy-default_dtype semantics.\n\n    .. code-block:: python\n\n        import mxnet as mx\n        @mx.use_np_default_dtype\n        def float64_one():\n            return mx.nd.ones(()).dtype\n        print(float64_one())\n\n        @np.use_np_default_dtype\n        class Float64Tensor(object):\n            def __init__(self, data=None):\n                if data is None:\n                    data = Float64Tensor.random().data\n                self._data = data\n\n            def __repr__(self):\n                print(\"Is __repr__ in np_default_dtype semantics? {}!\".format(str(np.is_np_deafult_dtype())))\n                return str(self._data.asnumpy())\n\n            @staticmethod\n            def random():\n                data = mx.nd.random.uniform(shape=(2,2))\n                return ScalarTensor(data)\n\n            @property\n            def value(self):\n                print(\"Is value property in np_dafault_dtype semantics? {}!\".format(str(np.is_np_default_dtype())))\n                return self._data.asnumpy()\n\n        print(\"Is global scope of np_default_dtype activated? {}!\".format(str(np.is_np_default_dtype())))\n        float64_tensor = Float64Tensor()\n        print(float64_tensor)\n\n\n    Parameters\n    ----------\n    func : a user-provided callable function or class to be scoped by the NumPy-default_dtype semantics.\n\n    Returns\n    -------\n    Function or class\n        A function or class wrapped in the NumPy-default_dtype scope.\n    \"\"\"\n    if inspect.isclass(func):\n        for name, method in inspect.getmembers(\n                func,\n                predicate=\n                lambda f: inspect.isfunction(f) or inspect.ismethod(f) or isinstance(f, property)):\n            if isinstance(method, property):\n                setattr(func, name, property(use_np_default_dtype(method.__get__),\n                                             method.__set__,\n                                             method.__delattr__,\n                                             method.__doc__))\n            else:\n                setattr(func, name, use_np_default_dtype(method))\n        return func\n    elif callable(func):\n        @functools.wraps(func)\n        def _with_np_default_dtype(*args, **kwargs):\n            with np_default_dtype(active=True):\n                return func(*args, **kwargs)\n        return _with_np_default_dtype\n    else:\n        raise TypeError('use_np_default_dtype can only decorate classes and callable objects, '\n                        'while received a {}'.format(str(type(func))))\n\ndef is_np_default_dtype():\n    \"\"\"Checks whether the NumPy default dtype semantics is currently turned on.\n    In NumPy default dtype semantics, default dtype is float64.\n\n    Please note that this is designed as an infrastructure for the incoming\n    MXNet-NumPy operators. Legacy operators registered in the modules\n    `mx.nd` and `mx.sym` are not guaranteed to behave like their counterparts\n    in NumPy even within this scope.\n\n    Returns\n    -------\n        A bool value indicating whether the NumPy default dtype semantics is currently on.\n\n    See Also\n    --------\n    set_np_default_dtype : Set default dtype equals to offical numpy\n    set_np : npx.set_np(dtype=True) has equal performance to npx.set_np_default_dtype(True)\n\n    Example\n    -------\n    >>> import mxnet as mx\n    >>> from mxnet import npx\n    >>> prev_state = npx.set_np_default_dtype(True)\n    >>> print(prev_state)\n    False\n    >>> print(npx.is_np_default_dtype())\n    True\n    \"\"\"\n    curr = ctypes.c_bool()\n    check_call(_LIB.MXIsNumpyDefaultDtype(ctypes.byref(curr)))\n    return curr.value\n\ndef set_np_default_dtype(is_np_default_dtype=True):  # pylint: disable=redefined-outer-name\n    \"\"\"Turns on/off NumPy default dtype semantics, because mxnet.numpy.ndarray use\n    32 bit data storage as default (e.g. float32 and int 32) while offical NumPy use\n    64 bit data storage as default (e.g. float64 and int64).\n    This is turned off by default for keeping backward compatibility.\n\n    Please note that this is designed as an infrastructure for the incoming\n    MXNet-NumPy operators. Legacy operators registered in the modules\n    `mx.nd` and `mx.sym` are not guaranteed to behave like their counterparts\n    in NumPy within this semantics.\n\n    Parameters\n    ----------\n    active : bool\n        Indicates whether to turn on/off NumPy default dtype semantics.\n\n    Returns\n    -------\n        A bool value indicating the previous state of NumPy default dtype semantics.\n\n    Example\n    -------\n    >>> import mxnet as mx\n    >>> from mxnet import npx\n    >>> prev_state = npx.set_np_default_dtype(True)\n    >>> print(prev_state)\n    False\n    >>> print(npx.is_np_default_dtype())\n    True\n    \"\"\"\n    global _set_np_default_dtype_logged\n    if is_np_default_dtype:\n        if not _set_np_default_dtype_logged:\n            import logging\n            logging.info('NumPy array default dtype has been changed from flaot32 to float64 in your code.')\n            _set_np_default_dtype_logged = True\n    prev = ctypes.c_bool()\n    check_call(_LIB.MXSetIsNumpyDefaultDtype(ctypes.c_bool(is_np_default_dtype), ctypes.byref(prev)))\n    return prev.value\n\n\ndef getenv(name):\n    \"\"\"Get the setting of an environment variable from the C Runtime.\n\n    Parameters\n    ----------\n    name : string type\n        The environment variable name\n\n    Returns\n    -------\n    value : string\n        The value of the environment variable, or None if not set\n    \"\"\"\n    ret = ctypes.c_char_p()\n    check_call(_LIB.MXGetEnv(c_str(name), ctypes.byref(ret)))\n    return None if ret.value is None else py_str(ret.value)\n\n\ndef setenv(name, value):\n    \"\"\"Set an environment variable in the C Runtime.\n\n    Parameters\n    ----------\n    name : string type\n        The environment variable name\n    value : string type\n        The desired value to set the environment value to\n    \"\"\"\n    passed_value = None if value is None else c_str(value)\n    check_call(_LIB.MXSetEnv(c_str(name), passed_value))\n\n\ndef get_max_supported_compute_capability():\n    \"\"\"Get the maximum compute capability (SM arch) supported by the nvrtc compiler\n    \"\"\"\n    max_supported_cc = ctypes.c_int()\n    check_call(_LIB.MXGetMaxSupportedArch(ctypes.byref(max_supported_cc)))\n    return max_supported_cc.value\n\n\ndef get_rtc_compile_opts(device):\n    \"\"\"Get the compile ops suitable for the context, given the toolkit/driver config\n    \"\"\"\n    device_cc = get_cuda_compute_capability(device)\n    max_supported_cc = get_max_supported_compute_capability()\n\n    # CUDA toolkits starting with 11.1 (first to support arch 86) can compile directly to SASS\n    can_compile_to_SASS = max_supported_cc >= 86\n    should_compile_to_SASS = can_compile_to_SASS and \\\n                             device_cc <= max_supported_cc\n    device_cc_as_used = min(device_cc, max_supported_cc)\n    arch_opt = \"--gpu-architecture={}_{}\".format(\"sm\" if should_compile_to_SASS else \"compute\",\n                                                 device_cc_as_used)\n    return [arch_opt]\n\ndef set_flush_denorms(value):\n    \"\"\"Change floating-point calculations on CPU when dealing with denormalized values.\n       This is only applicable to architectures which supports flush-to-zero.\n       Denormalized values are positive and negative values that are very close to 0\n       (exponent is the smallest possible value).\n       Flushing denormalized values to 0 can speedup calculations if such values occurs,\n       but if fulfilling whole IEEE 754 standard is required this option should be disabled.\n       Flushing denormalized values is enabled in MXNet by default.\n\n    Parameters\n    ----------\n    value : bool\n        State of flush-to-zero and denormals-are-zero in MXCSR register\n\n    Returns\n    -------\n    prev_state : bool\n        Previous state of flush-to-zero in MXCSR register\n    \"\"\"\n    ret = ctypes.c_bool()\n    passed_value = ctypes.c_bool(value)\n    check_call(_LIB.MXSetFlushDenorms(passed_value, ctypes.byref(ret)))\n    return ret.value\n\n\ndef dtype_from_number(number):\n    \"\"\"Get the data type from the given int or float number\n    \"\"\"\n    assert isinstance(number, numeric_types),\\\n        \"The input number should be either int for float types\"\n    import numpy as _np\n    if isinstance(number, (int, long)):\n        if number > _MAX_VALUE_64_BIT_UNSIGNED_:\n            raise OverflowError(\"Integer out of bounds\")\n        if number > _MAX_VALUE_64_BIT_SIGNED_:\n            return _np.uint64\n        elif calcsize(\"P\") == 8:\n            return _np.int64\n        else:\n            return _np.int32\n    elif isinstance(number, float):\n        if abs(number) > _MAX_VALUE_FLOAT32_REPRESENT_ or \\\n            ((not _np.isnan(number)) and \\\n                (_np.float32(number) == int(number)) and \\\n                    (number != int(number))):\n            return _np.float64\n        else:\n            return _np.float64 if is_np_default_dtype() else _np.float32\n    elif isinstance(number, _np.generic):\n        return number.dtype\n    raise TypeError('type {} not supported'.format(str(type(number))))\n\n# This is a wrapping of tempfile.TemporaryDirectory(), known to have cleanup issues on Windows.\n# The problem is partially handled as of Python 3.10 by the adding of a 'ignore_cleanup_errors'\n# parameter.  Once MXNet's Python version is forced to be >= 3.10, a simplification of this\n# function to use 'ignore_cleanup_errors' would be possible.  Until the fundamental Windows\n# issues are resolved, best to use this routine instead of tempfile.TemporaryDirectory().\n@contextmanager\ndef TemporaryDirectory(*args, **kwargs):\n    \"\"\"A context wrapper of tempfile.TemporaryDirectory() that ignores cleanup errors on Windows.\n    \"\"\"\n    dir = tempfile.TemporaryDirectory(*args, **kwargs)\n    try:\n        yield dir.name\n    finally:\n        try:\n            dir.cleanup()\n        except PermissionError:\n            if platform.system() != 'Windows':\n                raise\n"
  },
  {
    "path": "python/mxnet/visualization.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# coding: utf-8\n# pylint: disable=invalid-name, too-many-locals, fixme\n# pylint: disable=too-many-branches, too-many-statements\n# pylint: disable=too-many-arguments\n# pylint: disable=dangerous-default-value\n\"\"\"Visualization module\"\"\"\n\nimport re\nimport copy\nimport json\nimport warnings\nfrom .symbol import Symbol\n\ndef _str2tuple(string):\n    \"\"\"Convert shape string to list, internal use only.\n\n    Parameters\n    ----------\n    string: str\n        Shape string.\n\n    Returns\n    -------\n    list of str\n        Represents shape.\n    \"\"\"\n    return re.findall(r\"\\d+\", string)\n\ndef print_summary(symbol, shape=None, line_length=120, positions=[.44, .64, .74, 1.]):\n    \"\"\"Convert symbol for detail information.\n\n    Parameters\n    ----------\n    symbol: Symbol\n        Symbol to be visualized.\n    shape: dict\n        A dict of shapes, str->shape (tuple), given input shapes.\n    line_length: int\n        Rotal length of printed lines\n    positions: list\n        Relative or absolute positions of log elements in each line.\n\n    Returns\n    ------\n    None\n\n    Notes\n    -----\n    If ``mxnet`` is imported, the visualization module can be used in its short-form.\n    For example, if we ``import mxnet`` as follows::\n\n        import mxnet\n\n    this method in visualization module can be used in its short-form as::\n\n        mxnet.viz.print_summary(...)\n\n    \"\"\"\n    if not isinstance(symbol, Symbol):\n        raise TypeError(\"symbol must be Symbol\")\n    show_shape = False\n    if shape is not None:\n        show_shape = True\n        interals = symbol.get_internals()\n        _, out_shapes, _ = interals.infer_shape(**shape)\n        if out_shapes is None:\n            raise ValueError(\"Input shape is incomplete\")\n        shape_dict = dict(zip(interals.list_outputs(), out_shapes))\n    conf = json.loads(symbol.tojson())\n    nodes = conf[\"nodes\"]\n    heads = set(conf[\"heads\"][0])\n    if positions[-1] <= 1:\n        positions = [int(line_length * p) for p in positions]\n    # header names for the different log elements\n    to_display = ['Layer (type)', 'Output Shape', 'Param #', 'Previous Layer']\n    def print_row(fields, positions):\n        \"\"\"Print format row.\n\n        Parameters\n        ----------\n        fields: list\n            Information field.\n        positions: list\n            Field length ratio.\n        Returns\n        ------\n        None\n        \"\"\"\n        line = ''\n        for i, field in enumerate(fields):\n            line += str(field)\n            line = line[:positions[i]]\n            line += ' ' * (positions[i] - len(line))\n        print(line)\n    print('_' * line_length)\n    print_row(to_display, positions)\n    print('=' * line_length)\n    def print_layer_summary(node, out_shape):\n        \"\"\"print layer information\n\n        Parameters\n        ----------\n        node: dict\n            Node information.\n        out_shape: dict\n            Node shape information.\n        Returns\n        ------\n            Node total parameters.\n        \"\"\"\n        op = node[\"op\"]\n        pre_node = []\n        pre_filter = 0\n        if op != \"null\":\n            inputs = node[\"inputs\"]\n            for item in inputs:\n                input_node = nodes[item[0]]\n                input_name = input_node[\"name\"]\n                if input_node[\"op\"] != \"null\" or item[0] in heads:\n                    # add precede\n                    pre_node.append(input_name)\n                    if show_shape:\n                        if input_node[\"op\"] != \"null\":\n                            key = input_name + \"_output\"\n                        else:\n                            key = input_name\n                        if key in shape_dict:\n                            shape = shape_dict[key][1:]\n                            pre_filter = pre_filter + int(shape[0])\n        cur_param = 0\n        if op == 'Convolution':\n            if \"no_bias\" in node[\"attrs\"] and node[\"attrs\"][\"no_bias\"] == 'True':\n                num_group = int(node['attrs'].get('num_group', '1'))\n                cur_param = pre_filter * int(node[\"attrs\"][\"num_filter\"]) \\\n                   // num_group\n                for k in _str2tuple(node[\"attrs\"][\"kernel\"]):\n                    cur_param *= int(k)\n            else:\n                num_group = int(node['attrs'].get('num_group', '1'))\n                cur_param = pre_filter * int(node[\"attrs\"][\"num_filter\"]) \\\n                   // num_group\n                for k in _str2tuple(node[\"attrs\"][\"kernel\"]):\n                    cur_param *= int(k)\n                cur_param += int(node[\"attrs\"][\"num_filter\"])\n        elif op == 'FullyConnected':\n            if \"no_bias\" in node[\"attrs\"] and node[\"attrs\"][\"no_bias\"] == 'True':\n                cur_param = pre_filter * int(node[\"attrs\"][\"num_hidden\"])\n            else:\n                cur_param = (pre_filter+1) * int(node[\"attrs\"][\"num_hidden\"])\n        elif op == 'BatchNorm':\n            key = node[\"name\"] + \"_output\"\n            if show_shape:\n                num_filter = shape_dict[key][1]\n                cur_param = int(num_filter) * 2\n        elif op == 'Embedding':\n            cur_param = int(node[\"attrs\"]['input_dim']) * int(node[\"attrs\"]['output_dim'])\n        if not pre_node:\n            first_connection = ''\n        else:\n            first_connection = pre_node[0]\n        fields = [node['name'] + '(' + op + ')',\n                  \"x\".join([str(x) for x in out_shape]),\n                  cur_param,\n                  first_connection]\n        print_row(fields, positions)\n        if len(pre_node) > 1:\n            for i in range(1, len(pre_node)):\n                fields = ['', '', '', pre_node[i]]\n                print_row(fields, positions)\n        return cur_param\n    total_params = 0\n    for i, node in enumerate(nodes):\n        out_shape = []\n        op = node[\"op\"]\n        if op == \"null\" and i > 0:\n            continue\n        if op != \"null\" or i in heads:\n            if show_shape:\n                if op != \"null\":\n                    key = node[\"name\"] + \"_output\"\n                else:\n                    key = node[\"name\"]\n                if key in shape_dict:\n                    out_shape = shape_dict[key][1:]\n        total_params += print_layer_summary(nodes[i], out_shape)\n        if i == len(nodes) - 1:\n            print('=' * line_length)\n        else:\n            print('_' * line_length)\n    print(\"Total params: {params}\".format(params=total_params))\n    print('_' * line_length)\n\ndef plot_network(symbol, title=\"plot\", save_format='pdf', shape=None, dtype=None, node_attrs={},\n                 hide_weights=True):\n    \"\"\"Creates a visualization (Graphviz digraph object) of the given computation graph.\n    Graphviz must be installed for this function to work.\n\n    Parameters\n    ----------\n    title: str, optional\n        Title of the generated visualization.\n    symbol: Symbol\n        A symbol from the computation graph. The generated digraph will visualize the part\n        of the computation graph required to compute `symbol`.\n    shape: dict, optional\n        Specifies the shape of the input tensors. If specified, the visualization will include\n        the shape of the tensors between the nodes. `shape` is a dictionary mapping\n        input symbol names (str) to the corresponding tensor shape (tuple).\n    dtype: dict, optional\n        Specifies the type of the input tensors. If specified, the visualization will include\n        the type of the tensors between the nodes. `dtype` is a dictionary mapping\n        input symbol names (str) to the corresponding tensor type (e.g. `numpy.float32`).\n    node_attrs: dict, optional\n        Specifies the attributes for nodes in the generated visualization. `node_attrs` is\n        a dictionary of Graphviz attribute names and values. For example::\n\n            node_attrs={\"shape\":\"oval\",\"fixedsize\":\"false\"}\n\n        will use oval shape for nodes and allow variable sized nodes in the visualization.\n    hide_weights: bool, optional\n        If True (default), then inputs with names of form *_weight* (corresponding to weight\n        tensors) or *_bias* (corresponding to bias vectors) will be hidden for a cleaner\n        visualization.\n\n    Returns\n    -------\n    dot: Digraph\n        A Graphviz digraph object visualizing the computation graph to compute `symbol`.\n\n    Example\n    -------\n    >>> net = mx.sym.Variable('data')\n    >>> net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=128)\n    >>> net = mx.sym.Activation(data=net, name='relu1', act_type=\"relu\")\n    >>> net = mx.sym.FullyConnected(data=net, name='fc2', num_hidden=10)\n    >>> digraph = mx.viz.plot_network(net, shape={'data':(100,200)},\n    ... node_attrs={\"fixedsize\":\"false\"})\n    >>> digraph.view()\n\n    Notes\n    -----\n    If ``mxnet`` is imported, the visualization module can be used in its short-form.\n    For example, if we ``import mxnet`` as follows::\n\n        import mxnet\n\n    this method in visualization module can be used in its short-form as::\n\n        mxnet.viz.plot_network(...)\n\n    \"\"\"\n    # todo add shape support\n    try:\n        from graphviz import Digraph\n    except:\n        raise ImportError(\"Draw network requires graphviz library\")\n    if not isinstance(symbol, Symbol):\n        raise TypeError(\"symbol must be a Symbol\")\n    internals = symbol.get_internals()\n    draw_shape = shape is not None\n    if draw_shape:\n        _, out_shapes, _ = internals.infer_shape(**shape)\n        if out_shapes is None:\n            raise ValueError(\"Input shape is incomplete\")\n        shape_dict = dict(zip(internals.list_outputs(), out_shapes))\n    draw_type = dtype is not None\n    if draw_type:\n        _, out_types, _ = internals.infer_type(**dtype)\n        if out_types is None:\n            raise ValueError(\"Input type is incomplete\")\n        type_dict = dict(zip(internals.list_outputs(), out_types))\n    conf = json.loads(symbol.tojson())\n    nodes = conf[\"nodes\"]\n    # check if multiple nodes have the same name\n    if len(nodes) != len(set([node[\"name\"] for node in nodes])):\n        seen_nodes = set()\n        # find all repeated names\n        repeated = set(node['name'] for node in nodes if node['name'] in seen_nodes\n                       or seen_nodes.add(node['name']))\n        warning_message = \"There are multiple variables with the same name in your graph, \" \\\n                          \"this may result in cyclic graph. Repeated names: \" + ','.join(repeated)\n        warnings.warn(warning_message, RuntimeWarning)\n    # default attributes of node\n    node_attr = {\"shape\": \"box\", \"fixedsize\": \"true\",\n                 \"width\": \"1.3\", \"height\": \"0.8034\", \"style\": \"filled\"}\n    # merge the dict provided by user and the default one\n    node_attr.update(node_attrs)\n    dot = Digraph(name=title, format=save_format)\n    # color map\n    cm = (\"#8dd3c7\", \"#fb8072\", \"#ffffb3\", \"#bebada\", \"#80b1d3\",\n          \"#fdb462\", \"#b3de69\", \"#fccde5\")\n\n    def looks_like_weight(name):\n        \"\"\"Internal helper to figure out if node should be hidden with `hide_weights`.\n        \"\"\"\n        weight_like = ('_weight', '_bias', '_beta', '_gamma',\n                       '_moving_var', '_moving_mean', '_running_var', '_running_mean')\n        return name.endswith(weight_like)\n\n    # make nodes\n    hidden_nodes = set()\n    for node in nodes:\n        op = node[\"op\"]\n        name = node[\"name\"]\n        # input data\n        attr = copy.deepcopy(node_attr)\n        label = name\n\n        if op == \"null\":\n            if looks_like_weight(node[\"name\"]):\n                if hide_weights:\n                    hidden_nodes.add(node[\"name\"])\n                # else we don't render a node, but\n                # don't add it to the hidden_nodes set\n                # so it gets rendered as an empty oval\n                continue\n            attr[\"shape\"] = \"oval\" # inputs get their own shape\n            label = node[\"name\"]\n            attr[\"fillcolor\"] = cm[0]\n        elif op == \"Convolution\":\n            label = \"Convolution\\n{kernel}/{stride}, {filter}\".format(\n                kernel=\"x\".join(_str2tuple(node[\"attrs\"][\"kernel\"])),\n                stride=\"x\".join(_str2tuple(node[\"attrs\"][\"stride\"]))\n                if \"stride\" in node[\"attrs\"] else \"1\",\n                filter=node[\"attrs\"][\"num_filter\"]\n            )\n            attr[\"fillcolor\"] = cm[1]\n        elif op == \"FullyConnected\":\n            label = \"FullyConnected\\n{hidden}\".format(hidden=node[\"attrs\"][\"num_hidden\"])\n            attr[\"fillcolor\"] = cm[1]\n        elif op == \"BatchNorm\":\n            attr[\"fillcolor\"] = cm[3]\n        elif op == 'Activation':\n            act_type = node[\"attrs\"][\"act_type\"]\n            label = 'Activation\\n{activation}'.format(activation=act_type)\n            attr[\"fillcolor\"] = cm[2]\n        elif op == 'LeakyReLU':\n            attrs = node.get(\"attrs\")\n            act_type = attrs.get(\"act_type\", \"Leaky\") if attrs else \"Leaky\"\n            label = 'LeakyReLU\\n{activation}'.format(activation=act_type)\n            attr[\"fillcolor\"] = cm[2]\n        elif op == \"Pooling\":\n            label = \"Pooling\\n{pooltype}, {kernel}/{stride}\".format(pooltype=node[\"attrs\"][\"pool_type\"],\n                                                                    kernel=\"x\".join(_str2tuple(node[\"attrs\"][\"kernel\"]))\n                                                                    if \"kernel\" in node[\"attrs\"] else \"[]\",\n                                                                    stride=\"x\".join(_str2tuple(node[\"attrs\"][\"stride\"]))\n                                                                    if \"stride\" in node[\"attrs\"] else \"1\")\n            attr[\"fillcolor\"] = cm[4]\n        elif op in (\"Concat\", \"Flatten\", \"Reshape\"):\n            attr[\"fillcolor\"] = cm[5]\n        elif op == \"Softmax\":\n            attr[\"fillcolor\"] = cm[6]\n        else:\n            attr[\"fillcolor\"] = cm[7]\n            if op == \"Custom\":\n                label = node[\"attrs\"][\"op_type\"]\n\n        dot.node(name=name, label=label, **attr)\n\n    # add edges\n    for node in nodes:          # pylint: disable=too-many-nested-blocks\n        op = node[\"op\"]\n        name = node[\"name\"]\n        if op == \"null\":\n            continue\n        else:\n            inputs = node[\"inputs\"]\n\n            if node['op'] == '_contrib_BilinearResize2D':\n                inputs = [inputs[0]]\n\n            for item in inputs:\n                input_node = nodes[item[0]]\n                input_name = input_node[\"name\"]\n                if input_name not in hidden_nodes:\n                    attr = {\"dir\": \"back\", 'arrowtail':'open', 'label': ''}\n                    # add shapes\n                    if draw_shape:\n                        if input_node[\"op\"] != \"null\":\n                            key = input_name + \"_output\"\n                            if \"attrs\" in input_node:\n                                params = input_node[\"attrs\"]\n                                if \"num_outputs\" in params:\n                                    key += str(int(params[\"num_outputs\"]) - 1)\n                            shape = shape_dict[key][1:]\n                            label = \"x\".join([str(x) for x in shape])\n                            attr[\"label\"] = label\n                        else:\n                            key = input_name\n                            shape = shape_dict[key][1:]\n                            label = \"x\".join([str(x) for x in shape])\n                            attr[\"label\"] = label\n                    if draw_type:\n                        if input_node[\"op\"] != \"null\":\n                            key = input_name + \"_output\"\n                            if \"attrs\" in input_node:\n                                params = input_node[\"attrs\"]\n                                if \"num_outputs\" in params:\n                                    key += str(int(params[\"num_outputs\"]) - 1)\n                            dtype = type_dict[key]\n                            attr[\"label\"] += '(' + dtype.__name__ + ')'\n                        else:\n                            key = input_name\n                            dtype = type_dict[key]\n                            attr[\"label\"] += '(' + dtype.__name__ + ')'\n                    dot.edge(tail_name=name, head_name=input_name, **attr)\n\n    return dot\n"
  },
  {
    "path": "python/setup.py",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# pylint: disable=invalid-name, exec-used\n\"\"\"Setup mxnet package.\"\"\"\nfrom __future__ import absolute_import\nimport os\nimport sys\nfrom setuptools import find_packages # This must precede distutils\n\n# need to use distutils.core for correct placement of cython dll\nkwargs = {}\nif \"--inplace\" in sys.argv:\n    from distutils.core import setup\n    from distutils.extension import Extension\nelse:\n    from setuptools import setup\n    from setuptools.extension import Extension\n    kwargs = {'install_requires': ['numpy>=1.17', 'requests>=2.20.0,<3', 'graphviz<0.9.0,>=0.8.1', 'contextvars;python_version<\"3.7\"'], 'zip_safe': False}\n\nwith_cython = False\nif '--with-cython' in sys.argv:\n    with_cython = True\n    sys.argv.remove('--with-cython')\n\n# We can not import `mxnet.info.py` in setup.py directly since mxnet/__init__.py\n# Will be invoked which introduces dependences\nCURRENT_DIR = os.path.dirname(__file__)\nlibinfo_py = os.path.join(CURRENT_DIR, 'mxnet/libinfo.py')\nlibinfo = {'__file__': libinfo_py}\nexec(compile(open(libinfo_py, \"rb\").read(), libinfo_py, 'exec'), libinfo, libinfo)\n\nLIB_PATH = libinfo['find_lib_path']()\n__version__ = libinfo['__version__']\n\nsys.path.insert(0, CURRENT_DIR)\n\n# Try to generate auto-complete code\ntry:\n    from mxnet.base import _generate_op_module_signature\n    from mxnet.ndarray.register import _generate_ndarray_function_code\n    from mxnet.symbol.register import _generate_symbol_function_code\n    _generate_op_module_signature('mxnet', 'symbol', _generate_symbol_function_code)\n    _generate_op_module_signature('mxnet', 'ndarray', _generate_ndarray_function_code)\nexcept: # pylint: disable=bare-except\n    pass\n\ndef config_cython():\n    \"\"\"Try to configure cython and return cython configuration\"\"\"\n    if not with_cython:\n        return []\n    # pylint: disable=unreachable\n    if os.name == 'nt':\n        print(\"WARNING: Cython is not supported on Windows, will compile without cython module\")\n        return []\n\n    try:\n        from Cython.Build import cythonize\n        subdir = \"_cy3\"\n        ret = []\n        path = \"mxnet/cython\"\n        if os.name == 'nt':\n            library_dirs = ['mxnet', '../build/Release', '../build']\n            libraries = ['libmxnet']\n        else:\n            library_dirs = [os.path.dirname(p) for p in LIB_PATH]\n            libraries = ['mxnet']\n            # Default paths to libmxnet.so relative to the shared library file generated by cython.\n            # These precede LD_LIBRARY_PATH.\n            extra_link_args = [\"-Wl,-rpath,$ORIGIN/..:$ORIGIN/../../../lib:$ORIGIN/../../../build\"]\n\n        for fn in os.listdir(path):\n            if not fn.endswith(\".pyx\"):\n                continue\n            ret.append(Extension(\n                f\"mxnet.{subdir}.{fn[:-4]}\",\n                [f\"mxnet/cython/{fn}\"],\n                include_dirs=[\"../include/\", \"../3rdparty/tvm/nnvm/include\"],\n                library_dirs=library_dirs,\n                libraries=libraries,\n                extra_link_args=extra_link_args,\n                language=\"c++\"))\n\n        path = \"mxnet/_ffi/_cython\"\n        for fn in os.listdir(path):\n            if not fn.endswith(\".pyx\"):\n                continue\n            ret.append(Extension(\n                f\"mxnet._ffi.{subdir}.{fn[:-4]}\",\n                [f\"mxnet/_ffi/_cython/{fn}\"],\n                include_dirs=[\"../include/\", \"../3rdparty/tvm/nnvm/include\"],\n                library_dirs=library_dirs,\n                libraries=libraries,\n                extra_compile_args=[\"-std=c++17\"],\n                extra_link_args=extra_link_args,\n                language=\"c++\"))\n\n        # If `force=True` is not used and you cythonize the modules for python2 and python3\n        # successively, you need to delete `mxnet/cython/ndarray.cpp` after the first cythonize.\n        return cythonize(ret, force=True)\n    except ImportError:\n        print(\"WARNING: Cython is not installed, will compile without cython module\")\n        return []\n\n\nsetup(name='mxnet',\n      version=__version__,\n      description=open(os.path.join(CURRENT_DIR, 'README.md')).read(),\n      packages=find_packages(),\n      data_files=[('mxnet', [LIB_PATH[0]])],\n      url='https://github.com/apache/mxnet',\n      ext_modules=config_cython(),\n      classifiers=[\n          # https://pypi.org/pypi?%3Aaction=list_classifiers\n          'Development Status :: 5 - Production/Stable',\n          'Intended Audience :: Developers',\n          'Intended Audience :: Education',\n          'Intended Audience :: Science/Research',\n          'License :: OSI Approved :: Apache Software License',\n          'Programming Language :: Cython',\n          'Programming Language :: Python',\n          'Programming Language :: Python :: 3.6',\n          'Programming Language :: Python :: 3.7',\n          'Programming Language :: Python :: 3.8',\n          'Programming Language :: Python :: Implementation :: CPython',\n          'Topic :: Scientific/Engineering',\n          'Topic :: Scientific/Engineering :: Artificial Intelligence',\n          'Topic :: Scientific/Engineering :: Mathematics',\n          'Topic :: Software Development',\n          'Topic :: Software Development :: Libraries',\n          'Topic :: Software Development :: Libraries :: Python Modules',\n      ],\n      **kwargs)\n"
  },
  {
    "path": "rat-excludes",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n\n#   http://www.apache.org/licenses/LICENSE-2.0\n\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# This file lists all the files, directories and file formats that are excluded from license checks for various reasons.\n\n## How to run the RAT check locally\n# The following commands can be used to run a Apache RAT check locally - \n\n# Docker based 1-click-method:\n# ci/build.py -p ubuntu_cpu /work/runtime_functions.sh test_rat_check\n\n# You can also manually download Apache RAT. For installation on Ubuntu:\n# sudo apt-get install maven -y #>/dev/null\n# sudo apt-get install subversion -y #>/dev/null\n# svn co http://svn.apache.org/repos/asf/creadur/rat/tags/apache-rat-project-0.13/ #>/dev/null\n# cd trunk\n# mvn install #>/dev/null\n# cd apache-rat/target\n\n# Once installed, you can run Apache RAT check on the src\n# java -jar apache-rat-0.13.jar -E <path-to-this-file> -d <path-to-mxnet-source>\n\n# Apache RAT License Checker Allowlist\n#\n# Version control\n.github\n.gitignore\n.gitattributes\n.gitmodules\n\n# The Apache MXNET (incubating) project contains subcomponents with separate\n# copyright notices and license terms. Your use of the source code for the\n# these subcomponents is subject to the terms and conditions of the following\n# licenses. If not stated otherwise, their copyright notices and license terms\n# are available at the path of the subcomponent.\n3rdparty\nclipboard.js\n\n# Licenses\nlicenses/*\nLICENSE.binary.dependencies\n.licenserc.yaml\n\n# Generated files during build\n.buildinfo\nGemfile.lock\n_build/*\n_static/*\n_site/*\n_api/*\n_includes/*\nbuild/*\nlatex/*\ntarget/*\nsite/*\nxml/*\nDartConfiguration.tcl\n.*\\.egg-info\n.*\\.t\n\n# SPDX-License-Identifier: git-clang-format-13\ngit-clang-format-13\n\n# Files generated by Cython\ncore.cpp\nsymbol.cpp\nndarray.cpp\n\n# Sphinx themes\nthemes/*\n\n# Binary or data files\n.*\\.ipynb\n.*\\.pyc\n.*\\.so\n.*\\.json\n.*\\.txt\n.*\\.svg\n.*\\.lst\n.*\\.lds\n.*\\.in\n.*\\.diff\n.*\\.edl\n.*\\.md5\n.*\\.csv\n.*\\.log\n.*\\.interp\n.*\\.tokens\n.*\\.cPickle\n\n# Modules that are deleted prior to distribution\nR-package/*\n\n\n# Specific files\n# Files that don't support comment\nMANIFEST\n.codecov.yml\n\n# GitHub files\nCODEOWNERS\n.asf.yaml\n\n# Incorporated third-party source files that carry its own license, captured in licenses/\n_export_onnx.py\n_op_translations_opset12.py\n_op_translations_opset13.py\npool.h\npool.cuh\nerfinv-inl.h\nim2col.cuh\nim2col.h\ndeformable_im2col.cuh\ndeformable_im2col.h\nmodulated_deformable_convolution-inl.h\nmodulated_deformable_convolution.cc\nmodulated_deformable_convolution.cu\nmodulated_deformable_im2col.cuh\nmodulated_deformable_im2col.h\nFindCUDAToolkit.cmake\nFindBLAS.cmake\nFindJeMalloc.cmake\nselect_compute_arch.cmake\nnp_einsum_op-inl.h\nnp_einsum_op.cc\nnp_einsum_path_op-inl.h\n\n# Incorporated third-party source files from Microsoft that carry Apache 2.0 license, captured in licenses/\ndeformable_psroi_pooling.cu\ndeformable_convolution.cu\ndeformable_convolution-inl.h\npsroi_pooling.cc\nmulti_proposal.cu\ndeformable_psroi_pooling-inl.h\ndeformable_psroi_pooling.cc\ndeformable_convolution.cc\npsroi_pooling.cu\nmulti_proposal.cc\nmulti_proposal-inl.h\n\n# Incorporated third-party source files from Microsoft that carry MIT license, captured in licenses/\nmodulated_deformable_convolution-inl.h\nmodulated_deformable_convolution.cc\nmodulated_deformable_convolution.cu\n\n# AL2 License header not at the beginning of the file\ndoap.rdf\n\n# Header symlinks\ninclude\n"
  },
  {
    "path": "readthedocs.yml",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nformats:\n        - none\nrequirements_file: docs/requirements.txt\n"
  },
  {
    "path": "snap.python",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n\n#   http://www.apache.org/licenses/LICENSE-2.0\n\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nexport MXNET_HOME=$SNAP\nexport LD_LIBRARY_PATH=${SNAP}/lib:${SNAP}/usr/lib:$LD_LIBRARY_PATH \nexport PYTHONPATH=$MXNET_HOME:${SNAP}/lib/python2.7/site-packages/:${SNAP}/usr/lib/python2.7/dist-packages/:$PYTHONPATH \n\nexec ${SNAP}/usr/bin/python $@\n"
  },
  {
    "path": "src/api/_api_internal/_api_internal.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file _api_internal.cc\n * \\brief Internal functions exposed to python for FFI use only\n */\n// Acknowledgement: This file originates from incubator-tvm\n#include <mxnet/api_registry.h>\n#include <mxnet/base.h>\n#include <mxnet/expr_operator.h>\n#include <mxnet/runtime/packed_func.h>\n#include <mxnet/ir/expr.h>\n#include <mxnet/runtime/container.h>\n#include <mxnet/runtime/container_ext.h>\n#include <mxnet/runtime/ffi_helper.h>\n#include <nnvm/c_api.h>\n\nnamespace mxnet {\n\nMXNET_REGISTER_GLOBAL(\"_Integer\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      if (args[0].type_code() == kDLInt) {\n        *ret = Integer(args[0].operator int64_t());\n      } else {\n        LOG(FATAL) << \"only accept int\";\n      }\n    });\n\nMXNET_REGISTER_GLOBAL(\"_Float\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  if (args[0].type_code() == kDLFloat) {\n    *ret = Float(args[0].operator double());\n  } else {\n    LOG(FATAL) << \"only accept float\";\n  }\n});\n\nMXNET_REGISTER_GLOBAL(\"_ADT\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  std::vector<ObjectRef> data;\n  for (int i = 0; i < args.size(); ++i) {\n    if (args[i].type_code() == kNDArrayHandle) {\n      mxnet::NDArray* array = args[i].operator mxnet::NDArray*();\n      ObjectRef input       = NDArrayHandle(array);\n      data.push_back(input);\n    } else if (args[i].type_code() != kNull) {\n      ObjectRef input = String::CanConvertFrom(args[i]) ? args[i].operator String() :\n                                                          args[i].operator ObjectRef();\n      data.push_back(input);\n    } else {\n      data.emplace_back(nullptr);\n    }\n  }\n  *ret = ADT(0, data.begin(), data.end());\n});\n\nMXNET_REGISTER_GLOBAL(\"_Map\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  CHECK_EQ(args.size() % 2, 0);\n  std::unordered_map<ObjectRef, ObjectRef, ObjectHash, ObjectEqual> data;\n  for (int i = 0; i < args.num_args; i += 2) {\n    ObjectRef k =\n        String::CanConvertFrom(args[i]) ? args[i].operator String() : args[i].operator ObjectRef();\n    ObjectRef v;\n    if (args[i + 1].type_code() == kNDArrayHandle) {\n      mxnet::NDArray* array = args[i + 1].operator mxnet::NDArray*();\n      v                     = NDArrayHandle(array);\n    } else {\n      v = args[i + 1];\n    }\n    data.emplace(std::move(k), std::move(v));\n  }\n  *ret = Map<ObjectRef, ObjectRef>(data);\n});\n\nMXNET_REGISTER_GLOBAL(\"_String\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  std::string str = args[0].operator std::string();\n  *ret            = String(std::move(str));\n});\n\nMXNET_REGISTER_GLOBAL(\"_echo\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  *ret = args[0];\n});\n\nMXNET_REGISTER_API(\"_nop\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/cached_op_api.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file cached_op_api.cc\n * \\brief The API of function to invoke CachedOp in src/imperative/cached_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include <mxnet/runtime/container_ext.h>\n#include \"../imperative/cached_op.h\"\n#include \"../imperative/cached_op_threadsafe.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_GLOBAL(\"cached_op.invoke\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      CachedOpPtr op_shared = *static_cast<CachedOpPtr*>(args[0].value().v_handle);\n      // CachedOp* points to CachedOpThreadSafe object if CreateCachedOpEX\n      // was called with thread_safe=true\n      CachedOp* op = dynamic_cast<CachedOp*>(op_shared.get());\n\n      int num_inputs = args[1];\n      int args_size  = args.size();\n      std::vector<NDArray*> ndinputs;\n      ndinputs.reserve(num_inputs);\n      for (int i = 2; i < num_inputs + 2; ++i) {\n        ndinputs.push_back(static_cast<mxnet::NDArray*>(args[i]));\n      }\n\n      std::vector<NDArray*> ndoutputs;\n      ndoutputs.reserve(op->num_outputs());\n      if (args[num_inputs + 4].type_code() == kNull) {\n        for (int i = 0; i < op->num_outputs(); ++i)\n          ndoutputs.push_back(new NDArray());\n      } else {\n        int array_size = args_size - num_inputs - 4;\n        CHECK_EQ(array_size, op->num_outputs()) << \"CachedOp expects \" << op->num_outputs()\n                                                << \" outputs, but \" << array_size << \" was given.\";\n        for (int i = num_inputs + 4; i < array_size; ++i) {\n          ndoutputs.push_back(args[i].operator mxnet::NDArray*());\n        }\n      }\n\n      int default_dev_type;\n      int default_dev_id;\n      if (args[num_inputs + 2].type_code() != kNull) {\n        default_dev_type = args[num_inputs + 2];\n        default_dev_id   = args[num_inputs + 3];\n      } else {\n        const Context& ctx = ndinputs[0]->ctx();\n        default_dev_type   = ctx.dev_type;\n        default_dev_id     = ctx.dev_id;\n      }\n\n      // construct default context\n      Context ctx =\n          Context::Create(static_cast<Context::DeviceType>(default_dev_type), default_dev_id);\n      op->Forward(op_shared, ndinputs, ndoutputs, ctx);\n\n      if (op->num_outputs() == 1) {\n        *ret = ndoutputs[0];\n      } else {\n        std::vector<ObjectRef> outputs;\n        outputs.reserve(op->num_outputs());\n        for (int i = 0; i < op->num_outputs(); ++i) {\n          ObjectRef out = NDArrayHandle(ndoutputs[i]);\n          outputs.push_back(out);\n          delete ndoutputs[i];\n        }\n        *ret = runtime::ADT(0, outputs.begin(), outputs.end());\n      }\n    });\n\nMXNET_REGISTER_GLOBAL(\"cached_op.create\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      nnvm::Symbol* sym = static_cast<nnvm::Symbol*>(args[0].value().v_handle);\n      Object* flags_ptr = static_cast<Object*>(args[1].value().v_handle);\n      auto* n           = static_cast<const runtime::MapObj*>(flags_ptr);\n      int num_flags     = static_cast<int>(n->size());\n      bool thread_safe  = args[2];\n      std::vector<std::pair<std::string, std::string> > flags;\n      flags.reserve(num_flags);\n      for (const auto& kv : *n) {\n        flags.emplace_back(std::string(runtime::Downcast<runtime::String>(kv.first)),\n                           std::string(runtime::Downcast<runtime::String>(kv.second)));\n      }\n      mxnet::CachedOpPtr* out;\n      if (!thread_safe) {\n        out = new CachedOpPtr(new CachedOp(*sym, flags));\n      } else {\n        out = new CachedOpPtr(new CachedOpThreadSafe(*sym, flags));\n      }\n      *ret = static_cast<void*>(out);\n    });\n\nMXNET_REGISTER_GLOBAL(\"cached_op.free\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      CachedOpPtr* g = static_cast<CachedOpPtr*>(args[0].value().v_handle);\n      delete g;\n    });\n\nMXNET_REGISTER_GLOBAL(\"cached_op.get_optimized_symbol\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      auto s         = new nnvm::Symbol();\n      CachedOpPtr op = *static_cast<CachedOpPtr*>(args[0].value().v_handle);\n      *s             = op->GetOptimizedSymbol();\n      *ret           = static_cast<void*>(static_cast<SymbolHandle>(s));\n    });\n\nMXNET_REGISTER_GLOBAL(\"cached_op.register_op_hook\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      CachedOpHandle handle            = static_cast<CachedOpHandle>(args[0].value().v_handle);\n      CachedOpMonitorCallback callback = reinterpret_cast<CachedOpMonitorCallback>(\n          reinterpret_cast<void (*)(const char*, const char*, void*)>(args[1].value().v_handle));\n      bool monitor_all                      = args[2];\n      CachedOpMonitorCallback callback_temp = nullptr;\n      std::function<void(const char*, const char*, void*)> clbk;\n      if (callback) {\n        callback_temp = callback;\n        clbk          = [callback_temp](const char* name, const char* opr_name, void* handle) {\n          callback_temp(name, opr_name, handle);\n        };\n      } else {\n        clbk = nullptr;\n      }\n      CachedOpPtr op = *static_cast<CachedOpPtr*>(handle);\n      op->RegisterOpHook(clbk, monitor_all);\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/linalg/np_det.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_det.cc\n * \\brief Implementation of the API of functions in src/operator/tensor/la_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.det\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_det\");\n  nnvm::NodeAttrs attrs;\n  attrs.op          = op;\n  int num_inputs    = 1;\n  int num_outputs   = 0;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = NDArrayHandle(ndoutputs[0]);\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/linalg/np_eig.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_eig.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/linalg/np_eig.cc\n */\n\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/linalg/np_eig-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.eig\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_eig\");\n  nnvm::NodeAttrs attrs;\n  attrs.op          = op;\n  int num_inputs    = 1;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  int num_outputs   = 0;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = ADT(0, {NDArrayHandle(ndoutputs[0]), NDArrayHandle(ndoutputs[1])});\n});\n\nMXNET_REGISTER_API(\"_npi.eigh\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_eigh\");\n  nnvm::NodeAttrs attrs;\n  op::EighParam param = {};\n  param.UPLO   = *((args[1].operator std::string()).c_str());\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::EighParam>(&attrs);\n  int num_inputs    = 1;\n  int num_outputs   = 0;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = ADT(0, {NDArrayHandle(ndoutputs[0]), NDArrayHandle(ndoutputs[1])});\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/linalg/np_eigvals.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_eigvals.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/linalg/np_eigvals.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/linalg/np_eigvals-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.eigvals\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_eigvals\");\n      nnvm::NodeAttrs attrs;\n      attrs.op          = op;\n      int num_inputs    = 1;\n      int num_outputs   = 0;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n    });\n\nMXNET_REGISTER_API(\"_npi.eigvalsh\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_eigvalsh\");\n      nnvm::NodeAttrs attrs;\n      op::EigvalshParam param = {};\n      param.UPLO   = *((args[1].operator std::string()).c_str());\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::EigvalshParam>(&attrs);\n      int num_inputs    = 1;\n      int num_outputs   = 0;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/linalg/np_gesvd.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_gesvd.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/linalg/np_gesvd.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.svd\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  nnvm::NodeAttrs attrs;\n  const nnvm::Op* op = Op::Get(\"_npi_svd\");\n  attrs.op           = op;\n  // inputs\n  NDArray* inputs[] = {args[0].operator NDArray*()};\n  int num_inputs    = 1;\n  // outputs\n  int num_outputs = 0;\n  auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret            = ADT(\n      0, {NDArrayHandle(ndoutputs[0]), NDArrayHandle(ndoutputs[1]), NDArrayHandle(ndoutputs[2])});\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/linalg/np_inv.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_inv.cc\n * \\brief Implementation of the API of functions in src/operator/tensor/la_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.inv\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_inv\");\n  nnvm::NodeAttrs attrs;\n  attrs.op          = op;\n  int num_inputs    = 1;\n  int num_outputs   = 0;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/linalg/np_lstsq.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_lstsq.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/linalg/np_lstsq.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/linalg/np_lstsq-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.lstsq\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_lstsq\");\n  nnvm::NodeAttrs attrs;\n  op::LstsqParam param = {};\n  if (args[2].type_code() == kNull) {\n    param.rcond = static_cast<double>(1);\n  } else if (args[2].type_code() == kStr) {\n    const std::string rcond_str = args[2].operator std::string();\n    if (rcond_str == \"warn\") {\n      param.rcond = static_cast<double>(-1);\n    } else {\n      CHECK(false) << \"ValueError: wrong parameter rcond = \" << rcond_str;\n    }\n  } else {\n    param.rcond = args[2].operator double();\n  }\n  param.finfoEps32  = args[3].operator double();\n  param.finfoEps64  = args[4].operator double();\n  param.new_default = args[2].type_code() == kNull ? true : false;\n  attrs.parsed      = param;\n  attrs.op          = op;\n  SetAttrDict<op::LstsqParam>(&attrs);\n  int num_inputs    = 2;\n  int num_outputs   = 0;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = ADT(0,\n             {NDArrayHandle(ndoutputs[0]),\n              NDArrayHandle(ndoutputs[1]),\n              NDArrayHandle(ndoutputs[2]),\n              NDArrayHandle(ndoutputs[3])});\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/linalg/np_matrix_rank.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_pinv.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/linalg/np_matrix_rank.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/linalg/np_matrix_rank-inl.h\"\n\nnamespace mxnet {\n\ninline static void _npi_matrix_rank_none_tol(runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_matrix_rank_none_tol\");\n  op::MatrixRankNoneTolParam param = {};\n  nnvm::NodeAttrs attrs;\n  param.hermitian  = args[2].operator bool();\n  param.finfoEps32 = args[3].operator double();\n  param.finfoEps64 = args[4].operator double();\n  attrs.parsed     = param;\n  attrs.op         = op;\n  SetAttrDict<op::MatrixRankNoneTolParam>(&attrs);\n  int num_inputs    = 1;\n  int num_outputs   = 0;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n}\n\ninline static void _npi_matrix_rank(runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_matrix_rank\");\n  op::MatrixRankParam param = {};\n  nnvm::NodeAttrs attrs;\n  param.hermitian = args[2].operator bool();\n  attrs.parsed    = param;\n  attrs.op        = op;\n  SetAttrDict<op::MatrixRankParam>(&attrs);\n  int num_inputs    = 2;\n  int num_outputs   = 0;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n}\n\nMXNET_REGISTER_API(\"_npi.matrix_rank\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      if (args[1].type_code() == kNull) {\n        _npi_matrix_rank_none_tol(args, ret);\n      } else {\n        _npi_matrix_rank(args, ret);\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/linalg/np_norm.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_norm.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/linalg/np_norm_forward.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/linalg/np_norm-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.norm\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  nnvm::NodeAttrs attrs;\n  const nnvm::Op* op = Op::Get(\"_npi_norm\");\n  op::NumpyNormParam param = {};\n  param.ord = args[1].operator double();\n  if (args[2].type_code() == kNull) {\n    param.axis = dmlc::optional<mxnet::TShape>();\n  } else {\n    param.axis = mxnet::TShape(args[2].operator ObjectRef());\n  }\n  param.keepdims = args[3].operator bool();\n  param.flag     = args[4].operator int();\n\n  attrs.op     = op;\n  attrs.parsed = param;\n  SetAttrDict<op::NumpyNormParam>(&attrs);\n\n  // inputs\n  NDArray* inputs[] = {args[0].operator NDArray*()};\n  int num_inputs    = 1;\n  // outputs\n  int num_outputs = 0;\n  auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret            = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/linalg/np_pinv.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_pinv.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/linalg/np_pinv.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/linalg/np_pinv-inl.h\"\n\nnamespace mxnet {\n\ninline static void _npi_pinv(runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_pinv\");\n  op::PinvParam param = {};\n  nnvm::NodeAttrs attrs;\n  param.hermitian = args[2].operator bool();\n  attrs.parsed    = param;\n  attrs.op        = op;\n  SetAttrDict<op::PinvParam>(&attrs);\n  int num_inputs    = 2;\n  int num_outputs   = 0;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n}\n\ninline static void _npi_pinv_scalar_rcond(runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_pinv_scalar_rcond\");\n  op::PinvScalarRcondParam param = {};\n  nnvm::NodeAttrs attrs;\n  param.rcond     = args[1].operator double();\n  param.hermitian = args[2].operator bool();\n  attrs.parsed    = param;\n  attrs.op        = op;\n  SetAttrDict<op::PinvScalarRcondParam>(&attrs);\n  int num_inputs    = 1;\n  int num_outputs   = 0;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n}\n\nMXNET_REGISTER_API(\"_npi.pinv\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt) {\n    _npi_pinv_scalar_rcond(args, ret);\n  } else {\n    _npi_pinv(args, ret);\n  }\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/linalg/np_potrf.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_potrf.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/linalg/np_potrf.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/linalg/np_potrf-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.cholesky\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_cholesky\");\n      nnvm::NodeAttrs attrs;\n      op::LaCholeskyParam param = {};\n      param.lower  = args[1].operator bool();\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::LaCholeskyParam>(&attrs);\n      int num_inputs    = 1;\n      int num_outputs   = 0;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/linalg/np_qr.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_qr.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/linalg/np_qr.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.qr\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_qr\");\n  nnvm::NodeAttrs attrs;\n  attrs.op          = op;\n  int num_inputs    = 1;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  int num_outputs   = 0;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = ADT(0, {NDArrayHandle(ndoutputs[0]), NDArrayHandle(ndoutputs[1])});\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/linalg/np_slogdet.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_slogdet.cc\n * \\brief Implementation of the API of functions in src/operator/tensor/la_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.slogdet\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_slogdet\");\n      nnvm::NodeAttrs attrs;\n      attrs.op          = op;\n      int num_inputs    = 1;\n      int num_outputs   = 0;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = ADT(0, {NDArrayHandle(ndoutputs[0]), NDArrayHandle(ndoutputs[1])});\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/linalg/np_solve.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_solve.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/linalg/np_solve.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.solve\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_solve\");\n  nnvm::NodeAttrs attrs;\n  attrs.op          = op;\n  int num_inputs    = 2;\n  int num_outputs   = 0;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/linalg/np_tensorinv.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_tensorinv.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/linalg/np_tensorinv.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/linalg/np_tensorinv-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.tensorinv\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_tensorinv\");\n      nnvm::NodeAttrs attrs;\n      op::TensorinvParam param = {};\n      param.ind    = args[1].operator int();\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::TensorinvParam>(&attrs);\n      int num_inputs    = 1;\n      int num_outputs   = 0;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/linalg/np_tensorsolve.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_tensorsolve.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/linalg/np_tensorsolve.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/linalg/np_tensorsolve-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.tensorsolve\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_tensorsolve\");\n      nnvm::NodeAttrs attrs;\n      op::TensorsolveParam param = {};\n      if (args[2].type_code() == kNull) {\n        param.a_axes = Tuple<int>();\n      } else {\n        if (args[2].type_code() == kDLInt) {\n          param.a_axes = Tuple<int>(1, args[2].operator int64_t());\n        } else {\n          param.a_axes = Tuple<int>(args[2].operator ObjectRef());\n        }\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::TensorsolveParam>(&attrs);\n      int num_inputs    = 2;\n      int num_outputs   = 0;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_bincount_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_bincount_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_bincount_op.cc\n */\n#include <mxnet/api_registry.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_bincount_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.bincount\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_bincount\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyBincountParam param = {};\n\n      int num_outputs = 0;\n      if (args[1].type_code() == kNull) {\n        param.minlength   = args[2].operator int64_t();\n        param.has_weights = false;\n        NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n        int num_inputs    = 1;\n        attrs.parsed      = param;\n        attrs.op          = op;\n        SetAttrDict<op::NumpyBincountParam>(&attrs);\n        auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n        *ret           = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n      } else {\n        param.minlength   = args[2].operator int64_t();\n        param.has_weights = true;\n        NDArray* inputs[] = {args[0].operator mxnet::NDArray*(),\n                             args[1].operator mxnet::NDArray*()};\n        int num_inputs    = 2;\n        attrs.parsed      = param;\n        attrs.op          = op;\n        SetAttrDict<op::NumpyBincountParam>(&attrs);\n        auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n        *ret           = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_broadcast_reduce_op_boolean.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_broadcast_reduce_op_boolean.cc\n * \\brief Implementation of the API of functions in\n * src/operator/numpy/np_broadcast_reduce_op_boolean.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_broadcast_reduce_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.all\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_all\");\n  nnvm::NodeAttrs attrs;\n  op::NumpyReduceAxesBoolParam param = {};\n\n  NDArray* out      = args[3].operator mxnet::NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  if (args[1].type_code() == kNull) {\n    param.axis = dmlc::nullopt;\n  } else if (args[1].type_code() == kDLInt) {\n    param.axis = Tuple<int>(1, args[1].operator int64_t());\n  } else {\n    param.axis = Tuple<int>(args[1].operator ObjectRef());\n  }\n  param.keepdims    = args[2].operator bool();\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  int num_inputs    = 1;\n  attrs.parsed      = param;\n  attrs.op          = op;\n  SetAttrDict<op::NumpyReduceAxesBoolParam>(&attrs);\n  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (out) {\n    *ret = PythonArg(3);\n  } else {\n    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n  }\n});\n\nMXNET_REGISTER_API(\"_npi.any\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_any\");\n  nnvm::NodeAttrs attrs;\n  op::NumpyReduceAxesBoolParam param;\n\n  NDArray* out      = args[3].operator mxnet::NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  if (args[1].type_code() == kNull) {\n    param.axis = dmlc::nullopt;\n  } else if (args[1].type_code() == kDLInt) {\n    param.axis = Tuple<int>(1, args[1].operator int64_t());\n  } else {\n    param.axis = Tuple<int>(args[1].operator ObjectRef());\n  }\n  param.keepdims    = args[2].operator bool();\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  int num_inputs    = 1;\n  attrs.parsed      = param;\n  attrs.op          = op;\n  SetAttrDict<op::NumpyReduceAxesBoolParam>(&attrs);\n  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (out) {\n    *ret = PythonArg(3);\n  } else {\n    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n  }\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_broadcast_reduce_op_index.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_broadcast_reduce_op_index.cc\n * \\brief Implementation of the API of functions in\n          src/operator/numpy/np_broadcast_reduce_op_index.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/tensor/broadcast_reduce_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.argmax\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_argmax\");\n      nnvm::NodeAttrs attrs;\n      op::ReduceAxisParam param = {};\n      // param.axis\n      if (args[1].type_code() == kNull) {\n        param.axis = dmlc::nullopt;\n      } else {\n        param.axis = args[1].operator int();\n      }\n      // param.keepdims\n      param.keepdims = args[2].operator bool();\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::ReduceAxisParam>(&attrs);\n      // inputs\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      int num_inputs    = 1;\n      // outputs\n      NDArray* out      = args[3].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(3);\n      } else {\n        *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n      }\n    });\n\nMXNET_REGISTER_API(\"_npi.argmin\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_argmin\");\n      nnvm::NodeAttrs attrs;\n      op::ReduceAxisParam param;\n      // param.axis\n      if (args[1].type_code() == kNull) {\n        param.axis = dmlc::nullopt;\n      } else {\n        param.axis = args[1].operator int();\n      }\n      // param.keepdims\n      param.keepdims = args[2].operator bool();\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::ReduceAxisParam>(&attrs);\n      // inputs\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      int num_inputs    = 1;\n      // outputs\n      NDArray* out      = args[3].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(3);\n      } else {\n        *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_broadcast_reduce_op_value.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_broadcast_reduce_op_value.cc\n * \\brief Implementation of the API of functions in\n * src/operator/tensor/np_broadcast_reduce_op_value.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_broadcast_reduce_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.broadcast_to\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_broadcast_to\");\n      nnvm::NodeAttrs attrs;\n      op::BroadcastToParam param = {};\n      if (args[1].type_code() == kDLInt) {\n        param.shape = TShape(1, args[1].operator int64_t());\n      } else {\n        param.shape = TShape(args[1].operator ObjectRef());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::BroadcastToParam>(&attrs);\n\n      int num_outputs   = 0;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      int num_inputs    = 1;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.sum\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_sum\");\n  op::NumpyReduceAxesParam param;\n  nnvm::NodeAttrs attrs;\n  attrs.op = op;\n\n  // parse axis\n  if (args[1].type_code() == kNull) {\n    param.axis = dmlc::nullopt;\n  } else {\n    if (args[1].type_code() == kDLInt) {\n      param.axis = Tuple<int>(1, args[1].operator int64_t());\n    } else {\n      param.axis = Tuple<int>(args[1].operator ObjectRef());\n    }\n  }\n\n  // parse dtype\n  if (args[2].type_code() == kNull) {\n    param.dtype = dmlc::nullopt;\n  } else {\n    param.dtype = String2MXNetTypeWithBool(args[2].operator std::string());\n  }\n\n  // parse keepdims\n  if (args[3].type_code() == kNull) {\n    param.keepdims = false;\n  } else {\n    param.keepdims = args[3].operator bool();\n  }\n\n  // parse initial\n  if (args[4].type_code() == kNull) {\n    param.initial = dmlc::nullopt;\n  } else {\n    param.initial = args[4].operator double();\n  }\n\n  attrs.parsed = param;\n\n  SetAttrDict<op::NumpyReduceAxesParam>(&attrs);\n\n  NDArray* inputs[] = {args[0].operator NDArray*()};\n  int num_inputs    = 1;\n\n  NDArray* outputs[] = {args[5].operator NDArray*()};\n  NDArray** out      = (outputs[0] == nullptr) ? nullptr : outputs;\n  int num_outputs    = (outputs[0] != nullptr);\n  auto ndoutputs     = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, out);\n\n  if (out) {\n    *ret = PythonArg(5);\n  } else {\n    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n  }\n});\n\nMXNET_REGISTER_API(\"_npi.mean\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_mean\");\n  nnvm::NodeAttrs attrs;\n  op::NumpyReduceAxesParam param;\n  if (args[1].type_code() == kNull) {\n    param.axis = dmlc::optional<mxnet::Tuple<int>>();\n  } else if (args[1].type_code() == kDLInt) {\n    param.axis = Tuple<int>(1, args[1].operator int64_t());\n  } else {\n    param.axis = mxnet::Tuple<int>(args[1].operator ObjectRef());\n  }\n  if (args[2].type_code() == kNull) {\n    param.dtype = mxnet::common::GetDefaultDtype();\n  } else {\n    param.dtype = String2MXNetTypeWithBool(args[2].operator std::string());\n  }\n\n  if (args[3].type_code() == kNull) {\n    param.keepdims = false;\n  } else {\n    param.keepdims = args[3].operator bool();\n  }\n  param.initial = dmlc::optional<double>();\n  attrs.parsed  = param;\n  attrs.op      = op;\n  SetAttrDict<op::NumpyReduceAxesParam>(&attrs);\n  int num_inputs    = 1;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  NDArray* out      = args[4].operator mxnet::NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (out) {\n    *ret = PythonArg(4);\n  } else {\n    *ret = ndoutputs[0];\n  }\n});\n\nMXNET_REGISTER_API(\"_npi.prod\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_prod\");\n  nnvm::NodeAttrs attrs;\n  op::NumpyReduceAxesParam param;\n  if (args[1].type_code() == kNull) {\n    param.axis = dmlc::optional<mxnet::Tuple<int>>();\n  } else if (args[1].type_code() == kDLInt) {\n    param.axis = Tuple<int>(1, args[1].operator int64_t());\n  } else {\n    param.axis = Tuple<int>(args[1].operator ObjectRef());\n  }\n  if (args[2].type_code() == kNull) {\n    param.dtype = dmlc::optional<int>();\n  } else {\n    param.dtype = String2MXNetTypeWithBool(args[2].operator std::string());\n  }\n  if (args[3].type_code() == kNull) {\n    param.keepdims = false;\n  } else {\n    param.keepdims = args[3].operator bool();\n  }\n  if (args[4].type_code() == kNull) {\n    param.initial = dmlc::optional<double>();\n  } else {\n    param.initial = args[4].operator double();\n  }\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::NumpyReduceAxesParam>(&attrs);\n  int num_inputs    = 1;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  NDArray* out      = args[5].operator mxnet::NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (out) {\n    *ret = PythonArg(5);\n  } else {\n    *ret = ndoutputs[0];\n  }\n});\n\nMXNET_REGISTER_API(\"_npi.max\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  static const nnvm::Op* op = Op::Get(\"_npi_max\");\n  nnvm::NodeAttrs attrs;\n  op::NumpyReduceAxesNoDTypeParam param;\n\n  NDArray* out      = args[3].operator mxnet::NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  if (args[1].type_code() == kNull) {\n    param.axis = dmlc::nullopt;\n  } else if (args[1].type_code() == kDLInt) {\n    param.axis = Tuple<int>(1, args[1].operator int64_t());\n  } else {\n    param.axis = Tuple<int>(args[1].operator ObjectRef());\n  }\n  param.keepdims    = args[2].operator bool();\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  int num_inputs    = 1;\n  attrs.parsed      = param;\n  attrs.op          = op;\n  SetAttrDict<op::NumpyReduceAxesNoDTypeParam>(&attrs);\n  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (out) {\n    *ret = PythonArg(3);\n  } else {\n    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n  }\n});\n\nMXNET_REGISTER_API(\"_npi.min\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  static const nnvm::Op* op = Op::Get(\"_npi_min\");\n  nnvm::NodeAttrs attrs;\n  op::NumpyReduceAxesNoDTypeParam param;\n\n  NDArray* out      = args[3].operator mxnet::NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  if (args[1].type_code() == kNull) {\n    param.axis = dmlc::nullopt;\n  } else if (args[1].type_code() == kDLInt) {\n    param.axis = Tuple<int>(1, args[1].operator int64_t());\n  } else {\n    param.axis = Tuple<int>(args[1].operator ObjectRef());\n  }\n  param.keepdims    = args[2].operator bool();\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  int num_inputs    = 1;\n  attrs.parsed      = param;\n  attrs.op          = op;\n  SetAttrDict<op::NumpyReduceAxesNoDTypeParam>(&attrs);\n  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (out) {\n    *ret = PythonArg(3);\n  } else {\n    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n  }\n});\n\nMXNET_REGISTER_API(\"_npi.amax\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  static const nnvm::Op* op = Op::Get(\"_npi_amax\");\n  nnvm::NodeAttrs attrs;\n  op::NumpyReduceAxesNoDTypeParam param;\n\n  NDArray* out      = args[3].operator mxnet::NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  if (args[1].type_code() == kNull) {\n    param.axis = dmlc::nullopt;\n  } else if (args[1].type_code() == kDLInt) {\n    param.axis = Tuple<int>(1, args[1].operator int64_t());\n  } else {\n    param.axis = Tuple<int>(args[1].operator ObjectRef());\n  }\n  param.keepdims    = args[2].operator bool();\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  int num_inputs    = 1;\n  attrs.parsed      = param;\n  attrs.op          = op;\n  SetAttrDict<op::NumpyReduceAxesNoDTypeParam>(&attrs);\n  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (out) {\n    *ret = PythonArg(3);\n  } else {\n    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n  }\n});\n\nMXNET_REGISTER_API(\"_npi.amin\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  static const nnvm::Op* op = Op::Get(\"_npi_amin\");\n  nnvm::NodeAttrs attrs;\n  op::NumpyReduceAxesNoDTypeParam param;\n\n  NDArray* out      = args[3].operator mxnet::NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  if (args[1].type_code() == kNull) {\n    param.axis = dmlc::nullopt;\n  } else if (args[1].type_code() == kDLInt) {\n    param.axis = Tuple<int>(1, args[1].operator int64_t());\n  } else {\n    param.axis = Tuple<int>(args[1].operator ObjectRef());\n  }\n  param.keepdims    = args[2].operator bool();\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  int num_inputs    = 1;\n  attrs.parsed      = param;\n  attrs.op          = op;\n  SetAttrDict<op::NumpyReduceAxesNoDTypeParam>(&attrs);\n  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (out) {\n    *ret = PythonArg(3);\n  } else {\n    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n  }\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_cross.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_cross.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_cross.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_cross-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.cross\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  nnvm::NodeAttrs attrs;\n  const nnvm::Op* op = Op::Get(\"_npi_cross\");\n  op::NumpyCrossParam param = {};\n  param.axisa  = args[2].operator int();\n  param.axisb  = args[3].operator int();\n  param.axisc  = args[4].operator int();\n  attrs.op     = op;\n  attrs.parsed = param;\n  SetAttrDict<op::NumpyCrossParam>(&attrs);\n  int num_inputs    = 2;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};\n  int num_outputs   = 0;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_cumsum.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_cumsum.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_cumsum.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_cumsum-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.cumsum\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npi_cumsum\");\n      op::CumsumParam param = {};\n      // axis\n      if (args[1].type_code() == kNull) {\n        param.axis = dmlc::nullopt;\n      } else {\n        param.axis = args[1].operator int();\n      }\n      // dtype\n      if (args[2].type_code() == kNull) {\n        param.dtype = dmlc::nullopt;\n      } else {\n        param.dtype = String2MXNetTypeWithBool(args[2].operator std::string());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::CumsumParam>(&attrs);\n      // inputs\n      NDArray* inputs[] = {args[0].operator NDArray*()};\n      int num_inputs    = 1;\n      // outputs\n      NDArray* outputs[] = {args[3].operator NDArray*()};\n      NDArray** out      = outputs[0] == nullptr ? nullptr : outputs;\n      int num_outputs    = outputs[0] != nullptr;\n      auto ndoutputs     = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, out);\n      if (out) {\n        *ret = PythonArg(3);\n      } else {\n        *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_delete_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_delete_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_delete_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include <vector>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_delete_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.delete\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      static const nnvm::Op* op = Op::Get(\"_npi_delete\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyDeleteParam param = {};\n      int num_inputs = 0;\n      param.start    = dmlc::nullopt;\n      param.step     = dmlc::nullopt;\n      param.stop     = dmlc::nullopt;\n      param.int_ind  = dmlc::nullopt;\n      param.axis     = dmlc::nullopt;\n      if (args.num_args == 3) {\n        if (args[1].type_code() == kDLInt || args[1].type_code() == kDLFloat) {\n          if (args[1].type_code() == kDLInt) {\n            param.int_ind = args[1].operator int64_t();\n          } else if (args[1].type_code() == kDLFloat) {\n            param.int_ind = static_cast<int64_t>(args[1].operator double());\n          }\n          if (args[2].type_code() == kDLInt) {\n            param.axis = args[2].operator int();\n          } else if (args[2].type_code() == kDLFloat) {\n            param.axis = static_cast<int>(args[2].operator double());\n          }\n          num_inputs = 1;\n        } else {\n          if (args[2].type_code() == kDLInt) {\n            param.axis = args[2].operator int();\n          } else if (args[2].type_code() == kDLFloat) {\n            param.axis = static_cast<int>(args[2].operator double());\n          }\n          num_inputs = 2;\n        }\n      } else {\n        num_inputs = 1;\n        if (args[1].type_code() == kDLInt) {\n          param.start = args[1].operator int64_t();\n        } else if (args[1].type_code() == kDLFloat) {\n          param.start = static_cast<int64_t>(args[1].operator double());\n        }\n        if (args[2].type_code() == kDLInt) {\n          param.stop = args[2].operator int64_t();\n        } else if (args[2].type_code() == kDLFloat) {\n          param.stop = static_cast<int64_t>(args[2].operator double());\n        }\n        if (args[3].type_code() == kDLInt) {\n          param.step = args[3].operator int64_t();\n        } else if (args[3].type_code() == kDLFloat) {\n          param.step = static_cast<int64_t>(args[3].operator double());\n        }\n        if (args[4].type_code() == kDLInt) {\n          param.axis = args[4].operator int();\n        } else if (args[4].type_code() == kDLFloat) {\n          param.axis = static_cast<int>(args[4].operator double());\n        }\n      }\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyDeleteParam>(&attrs);\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_diff_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_diff_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_diff.cc\n */\n#include <mxnet/api_registry.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_diff-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.diff\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_diff\");\n  nnvm::NodeAttrs attrs;\n  op::DiffParam param = {};\n  param.n    = args[1].operator int();\n  param.axis = args[2].operator int();\n\n  // we directly copy DiffParam, which is trivially-copyable\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::DiffParam>(&attrs);\n\n  int num_outputs   = 0;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  int num_inputs    = 1;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = ndoutputs[0];\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_dot_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_dot_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_dot.cc\n */\n#include <mxnet/api_registry.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_dot-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.dot\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_dot\");\n  nnvm::NodeAttrs attrs;\n  attrs.op          = op;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};\n  NDArray* out      = args[2].operator NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_inputs    = 2;\n  int num_outputs   = out != nullptr;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (outputs) {\n    *ret = PythonArg(2);\n  } else {\n    *ret = ndoutputs[0];\n  }\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_ediff1d_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_ediff1d_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_ediff1d_op.cc\n */\n#include <mxnet/api_registry.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_ediff1d_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.ediff1d\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_ediff1d\");\n      nnvm::NodeAttrs attrs;\n      op::EDiff1DParam param = {};\n      int num_inputs = 1;\n      NDArray* inputs[3];\n      inputs[0] = args[0].operator mxnet::NDArray*();\n      // the order of `to_end` and `to_begin` array in the backend is different from the front-end\n      if (args[2].type_code() == kDLFloat || args[2].type_code() == kDLInt) {\n        param.to_begin_scalar    = args[2].operator double();\n        param.to_begin_arr_given = false;\n      } else if (args[2].type_code() == kNull) {\n        param.to_begin_scalar    = dmlc::nullopt;\n        param.to_begin_arr_given = false;\n      } else {\n        param.to_begin_scalar    = dmlc::nullopt;\n        param.to_begin_arr_given = true;\n        inputs[num_inputs]       = args[2].operator mxnet::NDArray*();\n        num_inputs++;\n      }\n\n      if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt) {\n        param.to_end_scalar    = args[1].operator double();\n        param.to_end_arr_given = false;\n      } else if (args[1].type_code() == kNull) {\n        param.to_end_scalar    = dmlc::nullopt;\n        param.to_end_arr_given = false;\n      } else {\n        param.to_end_scalar    = dmlc::nullopt;\n        param.to_end_arr_given = true;\n        inputs[num_inputs]     = args[1].operator mxnet::NDArray*();\n        num_inputs++;\n      }\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::EDiff1DParam>(&attrs);\n\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_einsum_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_einsum_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_einsum_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include <vector>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_einsum_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.einsum\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_einsum\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyEinsumParam param = {};\n      int args_size = args.size();\n      // param.num_args\n      param.num_args = args_size - 3;\n      // param.subscripts\n      param.subscripts = args[args_size - 3].operator std::string();\n      // param.optimize\n      param.optimize = args[args_size - 1].operator int();\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyEinsumParam>(&attrs);\n\n      // inputs\n      int num_inputs = param.num_args;\n      std::vector<NDArray*> inputs_vec(num_inputs, nullptr);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs_vec[i] = args[i].operator mxnet::NDArray*();\n      }\n      NDArray** inputs = inputs_vec.data();\n\n      // outputs\n      NDArray* out      = args[args_size - 2].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n\n      auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(args_size - 2);\n      } else {\n        *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_elemwise_broadcast_logic_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_elemwise_broadcast_logic_op.cc\n * \\brief Implementation of the API of functions in\n * src/operator/numpy/np_elemwise_broadcast_logic_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../ufunc_helper.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.equal\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op        = Op::Get(\"_npi_equal\");\n  const nnvm::Op* op_scalar = Op::Get(\"_npi_equal_scalar\");\n  UFuncHelper(args, ret, op, op_scalar, nullptr);\n});\n\nMXNET_REGISTER_API(\"_npi.not_equal\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op        = Op::Get(\"_npi_not_equal\");\n      const nnvm::Op* op_scalar = Op::Get(\"_npi_not_equal_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, nullptr);\n    });\n\nvoid SetUFuncHelper(runtime::MXNetArgs args,\n                    runtime::MXNetRetValue* ret,\n                    const nnvm::Op* op,\n                    const nnvm::Op* op_scalar,\n                    const nnvm::Op* op_rscalar) {\n  if (args[0].type_code() == kNDArrayHandle && args[1].type_code() == kNDArrayHandle) {\n    UFuncHelper(args, ret, op, nullptr, nullptr);\n  } else if (args[0].type_code() == kNDArrayHandle) {\n    UFuncHelper(args, ret, nullptr, op_scalar, nullptr);\n  } else {\n    UFuncHelper(args, ret, nullptr, nullptr, op_rscalar);\n  }\n}\n\nMXNET_REGISTER_API(\"_npi.greater\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op         = Op::Get(\"_npi_greater\");\n      const nnvm::Op* op_scalar  = Op::Get(\"_npi_greater_scalar\");\n      const nnvm::Op* op_rscalar = Op::Get(\"_npi_less_scalar\");\n      SetUFuncHelper(args, ret, op, op_scalar, op_rscalar);\n    });\n\nMXNET_REGISTER_API(\"_npi.less\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op         = Op::Get(\"_npi_less\");\n  const nnvm::Op* op_scalar  = Op::Get(\"_npi_less_scalar\");\n  const nnvm::Op* op_rscalar = Op::Get(\"_npi_greater_scalar\");\n  SetUFuncHelper(args, ret, op, op_scalar, op_rscalar);\n});\n\nMXNET_REGISTER_API(\"_npi.greater_equal\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op         = Op::Get(\"_npi_greater_equal\");\n      const nnvm::Op* op_scalar  = Op::Get(\"_npi_greater_equal_scalar\");\n      const nnvm::Op* op_rscalar = Op::Get(\"_npi_less_equal_scalar\");\n      SetUFuncHelper(args, ret, op, op_scalar, op_rscalar);\n    });\n\nMXNET_REGISTER_API(\"_npi.less_equal\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op         = Op::Get(\"_npi_less_equal\");\n      const nnvm::Op* op_scalar  = Op::Get(\"_npi_less_equal_scalar\");\n      const nnvm::Op* op_rscalar = Op::Get(\"_npi_greater_equal_scalar\");\n      SetUFuncHelper(args, ret, op, op_scalar, op_rscalar);\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_elemwise_broadcast_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_elemwise_broadcast_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_elemwise_broadcast_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../ufunc_helper.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.add\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op        = Op::Get(\"_npi_add\");\n  const nnvm::Op* op_scalar = Op::Get(\"_npi_add_scalar\");\n  UFuncHelper(args, ret, op, op_scalar, nullptr);\n});\n\nMXNET_REGISTER_API(\"_npi.subtract\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op         = Op::Get(\"_npi_subtract\");\n      const nnvm::Op* op_scalar  = Op::Get(\"_npi_subtract_scalar\");\n      const nnvm::Op* op_rscalar = Op::Get(\"_npi_rsubtract_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, op_rscalar);\n    });\n\nMXNET_REGISTER_API(\"_npi.multiply\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op        = Op::Get(\"_npi_multiply\");\n      const nnvm::Op* op_scalar = Op::Get(\"_npi_multiply_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, nullptr);\n    });\n\nMXNET_REGISTER_API(\"_npi.true_divide\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op         = Op::Get(\"_npi_true_divide\");\n      const nnvm::Op* op_scalar  = Op::Get(\"_npi_true_divide_scalar\");\n      const nnvm::Op* op_rscalar = Op::Get(\"_npi_rtrue_divide_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, op_rscalar);\n    });\n\nMXNET_REGISTER_API(\"_npi.floor_divide\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op         = Op::Get(\"_npi_floor_divide\");\n      const nnvm::Op* op_scalar  = Op::Get(\"_npi_floor_divide_scalar\");\n      const nnvm::Op* op_rscalar = Op::Get(\"_npi_rfloor_divide_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, op_rscalar);\n    });\n\nMXNET_REGISTER_API(\"_npi.mod\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op         = Op::Get(\"_npi_mod\");\n  const nnvm::Op* op_scalar  = Op::Get(\"_npi_mod_scalar\");\n  const nnvm::Op* op_rscalar = Op::Get(\"_npi_rmod_scalar\");\n  UFuncHelper(args, ret, op, op_scalar, op_rscalar);\n});\n\nMXNET_REGISTER_API(\"_npi.power\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op         = Op::Get(\"_npi_power\");\n  const nnvm::Op* op_scalar  = Op::Get(\"_npi_power_scalar\");\n  const nnvm::Op* op_rscalar = Op::Get(\"_npi_rpower_scalar\");\n  UFuncHelper(args, ret, op, op_scalar, op_rscalar);\n});\n\nMXNET_REGISTER_API(\"_npi.lcm\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op        = Op::Get(\"_npi_lcm\");\n  const nnvm::Op* op_scalar = Op::Get(\"_npi_lcm_scalar\");\n  UFuncHelper(args, ret, op, op_scalar, nullptr);\n});\n\nMXNET_REGISTER_API(\"_npi.gcd\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op        = Op::Get(\"_npi_gcd\");\n  const nnvm::Op* op_scalar = Op::Get(\"_npi_gcd_scalar\");\n  UFuncHelper(args, ret, op, op_scalar, nullptr);\n});\n\nMXNET_REGISTER_API(\"_npi.logical_and\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op        = Op::Get(\"_npi_logical_and\");\n      const nnvm::Op* op_scalar = Op::Get(\"_npi_logical_and_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, nullptr);\n    });\n\nMXNET_REGISTER_API(\"_npi.logical_or\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op        = Op::Get(\"_npi_logical_or\");\n      const nnvm::Op* op_scalar = Op::Get(\"_npi_logical_or_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, nullptr);\n    });\n\nMXNET_REGISTER_API(\"_npi.logical_xor\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op        = Op::Get(\"_npi_logical_xor\");\n      const nnvm::Op* op_scalar = Op::Get(\"_npi_logical_xor_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, nullptr);\n    });\n\nMXNET_REGISTER_API(\"_npi.bitwise_or\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op        = Op::Get(\"_npi_bitwise_or\");\n      const nnvm::Op* op_scalar = Op::Get(\"_npi_bitwise_or_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, nullptr);\n    });\n\nMXNET_REGISTER_API(\"_npi.bitwise_xor\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op        = Op::Get(\"_npi_bitwise_xor\");\n      const nnvm::Op* op_scalar = Op::Get(\"_npi_bitwise_xor_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, nullptr);\n    });\n\nMXNET_REGISTER_API(\"_npi.bitwise_and\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op        = Op::Get(\"_npi_bitwise_and\");\n      const nnvm::Op* op_scalar = Op::Get(\"_npi_bitwise_and_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, nullptr);\n    });\n\nMXNET_REGISTER_API(\"_npi.logaddexp\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op        = Op::Get(\"_npi_logaddexp\");\n      const nnvm::Op* op_scalar = Op::Get(\"_npi_logaddexp_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, nullptr);\n    });\n\nMXNET_REGISTER_API(\"_npi.copysign\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op         = Op::Get(\"_npi_copysign\");\n      const nnvm::Op* op_scalar  = Op::Get(\"_npi_copysign_scalar\");\n      const nnvm::Op* op_rscalar = Op::Get(\"_npi_rcopysign_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, op_rscalar);\n    });\n\nMXNET_REGISTER_API(\"_npi.arctan2\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op         = Op::Get(\"_npi_arctan2\");\n      const nnvm::Op* op_scalar  = Op::Get(\"_npi_arctan2_scalar\");\n      const nnvm::Op* op_rscalar = Op::Get(\"_npi_rarctan2_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, op_rscalar);\n    });\n\nMXNET_REGISTER_API(\"_npi.hypot\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op        = Op::Get(\"_npi_hypot\");\n  const nnvm::Op* op_scalar = Op::Get(\"_npi_hypot_scalar\");\n  UFuncHelper(args, ret, op, op_scalar, nullptr);\n});\n\nMXNET_REGISTER_API(\"_npi.ldexp\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op         = Op::Get(\"_npi_ldexp\");\n  const nnvm::Op* op_scalar  = Op::Get(\"_npi_ldexp_scalar\");\n  const nnvm::Op* op_rscalar = Op::Get(\"_npi_rldexp_scalar\");\n  UFuncHelper(args, ret, op, op_scalar, op_rscalar);\n});\n\nMXNET_REGISTER_API(\"_npi.bitwise_left_shift\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op         = Op::Get(\"_npi_bitwise_left_shift\");\n      const nnvm::Op* op_scalar  = Op::Get(\"_npi_bitwise_left_shift_scalar\");\n      const nnvm::Op* op_rscalar = Op::Get(\"_npi_rbitwise_left_shift_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, op_rscalar);\n    });\n\nMXNET_REGISTER_API(\"_npi.bitwise_right_shift\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op         = Op::Get(\"_npi_bitwise_right_shift\");\n      const nnvm::Op* op_scalar  = Op::Get(\"_npi_bitwise_right_shift_scalar\");\n      const nnvm::Op* op_rscalar = Op::Get(\"_npi_rbitwise_right_shift_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, op_rscalar);\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_elemwise_broadcast_op_extended_sec.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_elemwise_broadcast_op_extended_sec.cc\n * \\brief Implementation of the API of functions in\n * src/operator/numpy/np_elemwise_broadcast_op_extended_sec.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../ufunc_helper.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.fmax\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op        = Op::Get(\"_npi_fmax\");\n  const nnvm::Op* op_scalar = Op::Get(\"_npi_fmax_scalar\");\n  UFuncHelper(args, ret, op, op_scalar, nullptr);\n});\n\nMXNET_REGISTER_API(\"_npi.fmin\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op        = Op::Get(\"_npi_fmin\");\n  const nnvm::Op* op_scalar = Op::Get(\"_npi_fmin_scalar\");\n  UFuncHelper(args, ret, op, op_scalar, nullptr);\n});\n\nMXNET_REGISTER_API(\"_npi.fmod\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op         = Op::Get(\"_npi_fmod\");\n  const nnvm::Op* op_scalar  = Op::Get(\"_npi_fmod_scalar\");\n  const nnvm::Op* op_rscalar = Op::Get(\"_npi_rfmod_scalar\");\n  UFuncHelper(args, ret, op, op_scalar, op_rscalar);\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_elemwise_unary_op_basic.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_elemwise_broadcast_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_elemwise_unary_op_basic.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../ufunc_helper.h\"\n#include \"../../../operator/tensor/elemwise_unary_op.h\"\n\nnamespace mxnet {\n\n#define MXNET_REGISTER_UNARY_API(op_name)                                  \\\n  MXNET_REGISTER_API(\"_npi.\" #op_name)                                     \\\n      .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) { \\\n        const nnvm::Op* op = Op::Get(\"_npi_\" #op_name);                    \\\n        UFuncHelper(args, ret, op);                                        \\\n      })\n\nMXNET_REGISTER_UNARY_API(negative);\nMXNET_REGISTER_UNARY_API(reciprocal);\nMXNET_REGISTER_UNARY_API(abs);\nMXNET_REGISTER_UNARY_API(sign);\nMXNET_REGISTER_UNARY_API(rint);\nMXNET_REGISTER_UNARY_API(ceil);\nMXNET_REGISTER_UNARY_API(floor);\nMXNET_REGISTER_UNARY_API(bitwise_not);\nMXNET_REGISTER_UNARY_API(trunc);\nMXNET_REGISTER_UNARY_API(fix);\nMXNET_REGISTER_UNARY_API(square);\nMXNET_REGISTER_UNARY_API(sqrt);\nMXNET_REGISTER_UNARY_API(cbrt);\nMXNET_REGISTER_UNARY_API(exp);\nMXNET_REGISTER_UNARY_API(log);\nMXNET_REGISTER_UNARY_API(log10);\nMXNET_REGISTER_UNARY_API(log2);\nMXNET_REGISTER_UNARY_API(log1p);\nMXNET_REGISTER_UNARY_API(expm1);\nMXNET_REGISTER_UNARY_API(logical_not);\nMXNET_REGISTER_UNARY_API(isnan);\nMXNET_REGISTER_UNARY_API(isinf);\nMXNET_REGISTER_UNARY_API(isposinf);\nMXNET_REGISTER_UNARY_API(isneginf);\nMXNET_REGISTER_UNARY_API(isfinite);\nMXNET_REGISTER_UNARY_API(sin);\nMXNET_REGISTER_UNARY_API(cos);\nMXNET_REGISTER_UNARY_API(tan);\nMXNET_REGISTER_UNARY_API(arcsin);\nMXNET_REGISTER_UNARY_API(arccos);\nMXNET_REGISTER_UNARY_API(arctan);\nMXNET_REGISTER_UNARY_API(degrees);\nMXNET_REGISTER_UNARY_API(radians);\n#if MXNET_USE_TVM_OP\nMXNET_REGISTER_UNARY_API(rad2deg);  // from src/operator/contrib/tvmop/ufunc.cc\nMXNET_REGISTER_UNARY_API(deg2rad);  // from src/operator/contrib/tvmop/ufunc.cc\n#else                               // MXNET_USE_TVM_OP\nMXNET_REGISTER_API(\"_npi.rad2deg\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      const nnvm::Op* op = Op::Get(\"_npi_degrees\");\n      UFuncHelper(args, ret, op);\n    });\nMXNET_REGISTER_API(\"_npi.deg2rad\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      const nnvm::Op* op = Op::Get(\"_npi_radians\");\n      UFuncHelper(args, ret, op);\n    });\n#endif                              // MXNET_USE_TVM_OP\nMXNET_REGISTER_UNARY_API(sinh);\nMXNET_REGISTER_UNARY_API(cosh);\nMXNET_REGISTER_UNARY_API(tanh);\nMXNET_REGISTER_UNARY_API(arcsinh);\nMXNET_REGISTER_UNARY_API(arccosh);\nMXNET_REGISTER_UNARY_API(arctanh);\n\nMXNET_REGISTER_API(\"_npi.around\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_around\");\n      nnvm::NodeAttrs attrs;\n      op::AroundParam param = {};\n      param.decimals = args[1].operator int64_t();\n      attrs.parsed   = param;\n      attrs.op       = op;\n      SetAttrDict<op::AroundParam>(&attrs);\n      int num_inputs    = 1;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      NDArray* out      = args[2].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(2);\n      } else {\n        *ret = ndoutputs[0];\n      }\n    });\n\nMXNET_REGISTER_API(\"_npi.copy\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_copy\");\n  nnvm::NodeAttrs attrs;\n  attrs.op          = op;\n  NDArray* inputs[] = {args[0].operator NDArray*()};\n  int num_inputs    = 1;\n  int num_outputs   = 0;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = ndoutputs[0];\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_fill_diagonal_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_fill_diagonal_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_fill_diagonal.cc */\n#include <mxnet/api_registry.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_fill_diagonal_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.fill_diagonal\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_fill_diagonal\");\n      nnvm::NodeAttrs attrs;\n\n      op::NumpyFillDiagonalParam param = {};\n      int num_inputs    = 1;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n\n      if (args[1].type_code() == kDLInt || args[1].type_code() == kDLUInt ||\n          args[1].type_code() == kDLFloat || args[1].type_code() == kDLBfloat) {\n        param.val = Tuple<double>(1, args[1].operator double());\n      } else {\n        param.val = Obj2Tuple<double, Float>(args[1].operator ObjectRef());\n      }\n      param.wrap = args[2].operator bool();\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyFillDiagonalParam>(&attrs);\n\n      NDArray* out      = args[3].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      // set the number of outputs provided by the `out` arugment\n      int num_outputs = out != nullptr;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(3);\n      } else {\n        *ret = ndoutputs[0];\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_histogram_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_histogram_op.cc\n * \\brief Implementation of the API of functions in src/operator/tensor/histogram.cc\n */\n\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/tensor/histogram-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.histogram\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npi_histogram\");\n      op::HistogramParam param = {};\n      // parse bin_cnt\n      if (args[2].type_code() == kNull) {\n        param.bin_cnt = dmlc::nullopt;\n      } else {\n        param.bin_cnt = args[2].operator int();\n      }\n\n      // parse range\n      if (args[3].type_code() == kNull) {\n        param.range = dmlc::nullopt;\n      } else {\n        param.range = Obj2Tuple<double, Float>(args[3].operator ObjectRef());\n      }\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::HistogramParam>(&attrs);\n\n      std::vector<NDArray*> inputs_vec;\n      int num_inputs = 0;\n\n      if (args[2].type_code() != kNull) {\n        CHECK_EQ(args[1].type_code(), kNull) << \"bins should be None when bin_cnt is provided\";\n        inputs_vec.push_back((args[0].operator NDArray*()));\n        num_inputs = 1;\n      } else {\n        CHECK_NE(args[1].type_code(), kNull)\n            << \"bins should not be None when bin_cnt is not provided\";\n        // inputs\n        inputs_vec.push_back((args[0].operator NDArray*()));\n        inputs_vec.push_back((args[1].operator NDArray*()));\n        num_inputs = 2;\n      }\n\n      // outputs\n      NDArray** out   = nullptr;\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs_vec.data(), &num_outputs, out);\n      *ret            = ADT(0, {NDArrayHandle(ndoutputs[0]), NDArrayHandle(ndoutputs[1])});\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_init_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_init_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_init_op.cc\n */\n#include <dmlc/optional.h>\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/tensor/init_op.h\"\n#include \"../../../operator/numpy/np_init_op.h\"\n#include \"../../../common/utils.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.zeros\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_zeros\");\n  nnvm::NodeAttrs attrs;\n  op::InitOpParam param = {};\n  if (args[0].type_code() == kDLInt) {\n    param.shape = TShape(1, args[0].operator int64_t());\n  } else {\n    param.shape = TShape(args[0].operator ObjectRef());\n  }\n  if (args[1].type_code() == kNull) {\n    param.dtype = mxnet::common::GetDefaultDtype();\n  } else {\n    param.dtype = String2MXNetTypeWithBool(args[1].operator std::string());\n  }\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::InitOpParam>(&attrs);\n  if (args[2].type_code() != kNull) {\n    attrs.dict[\"ctx\"] = args[2].operator std::string();\n  }\n  int num_outputs = 0;\n  auto ndoutputs  = Invoke(op, &attrs, 0, nullptr, &num_outputs, nullptr);\n  *ret            = ndoutputs[0];\n});\n\nMXNET_REGISTER_API(\"_npi.full_like\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_full_like\");\n      nnvm::NodeAttrs attrs;\n      op::FullLikeOpParam param = {};\n      param.fill_value = args[1].operator double();\n      if (args[2].type_code() == kNull) {\n        param.dtype = dmlc::nullopt;\n      } else {\n        param.dtype = String2MXNetTypeWithBool(args[2].operator std::string());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      if (args[3].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[3].operator std::string();\n      }\n      SetAttrDict<op::FullLikeOpParam>(&attrs);\n      NDArray* out      = args[4].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      int num_inputs    = 1;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(4);\n      } else {\n        *ret = ndoutputs[0];\n      }\n    });\n\nMXNET_REGISTER_API(\"_npi.indices\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_indices\");\n      nnvm::NodeAttrs attrs;\n      op::IndicesOpParam param = {};\n      // param.dimensions\n      if (args[0].type_code() == kDLInt) {\n        param.dimensions = TShape(1, args[0].operator int64_t());\n      } else {\n        param.dimensions = TShape(args[0].operator ObjectRef());\n      }\n      // param.dtype\n      if (args[1].type_code() == kNull) {\n        param.dtype = -1;\n      } else {\n        param.dtype = String2MXNetTypeWithBool(args[1].operator std::string());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::IndicesOpParam>(&attrs);\n      // param.ctx\n      if (args[2].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[2].operator std::string();\n      }\n      int num_inputs  = 0;\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, nullptr, &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.atleast_1d\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_atleast_1d\");\n      nnvm::NodeAttrs attrs;\n      op::AtleastNDParam param = {};\n      int args_size  = args.size();\n      param.num_args = args_size;\n      attrs.parsed   = param;\n      attrs.op       = op;\n      SetAttrDict<op::AtleastNDParam>(&attrs);\n      int num_inputs = args_size;\n      std::vector<NDArray*> inputs_vec(args_size, nullptr);\n      for (int i = 0; i < args_size; ++i) {\n        inputs_vec[i] = args[i].operator mxnet::NDArray*();\n      }\n      NDArray** inputs = inputs_vec.data();\n      int num_outputs  = 0;\n      auto ndoutputs   = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      std::vector<NDArrayHandle> ndarray_handles;\n      ndarray_handles.reserve(num_outputs);\n      for (int i = 0; i < num_outputs; ++i) {\n        ndarray_handles.emplace_back(ndoutputs[i]);\n      }\n      *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n    });\n\nMXNET_REGISTER_API(\"_npi.atleast_2d\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_atleast_2d\");\n      nnvm::NodeAttrs attrs;\n      op::AtleastNDParam param = {};\n      int args_size  = args.size();\n      param.num_args = args_size;\n      attrs.parsed   = param;\n      attrs.op       = op;\n      SetAttrDict<op::AtleastNDParam>(&attrs);\n      int num_inputs = args_size;\n      std::vector<NDArray*> inputs_vec(args_size, nullptr);\n      for (int i = 0; i < args_size; ++i) {\n        inputs_vec[i] = args[i].operator mxnet::NDArray*();\n      }\n      NDArray** inputs = inputs_vec.data();\n      int num_outputs  = 0;\n      auto ndoutputs   = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      std::vector<NDArrayHandle> ndarray_handles;\n      ndarray_handles.reserve(num_outputs);\n      for (int i = 0; i < num_outputs; ++i) {\n        ndarray_handles.emplace_back(ndoutputs[i]);\n      }\n      *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n    });\n\nMXNET_REGISTER_API(\"_npi.atleast_3d\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_atleast_3d\");\n      nnvm::NodeAttrs attrs;\n      op::AtleastNDParam param = {};\n      int args_size  = args.size();\n      param.num_args = args_size;\n      attrs.parsed   = param;\n      attrs.op       = op;\n      SetAttrDict<op::AtleastNDParam>(&attrs);\n      int num_inputs = args_size;\n      std::vector<NDArray*> inputs_vec(args_size, nullptr);\n      for (int i = 0; i < args_size; ++i) {\n        inputs_vec[i] = args[i].operator mxnet::NDArray*();\n      }\n      NDArray** inputs = inputs_vec.data();\n      int num_outputs  = 0;\n      auto ndoutputs   = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      std::vector<NDArrayHandle> ndarray_handles;\n      ndarray_handles.reserve(num_outputs);\n      for (int i = 0; i < num_outputs; ++i) {\n        ndarray_handles.emplace_back(ndoutputs[i]);\n      }\n      *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n    });\n\nMXNET_REGISTER_API(\"_npi.arange\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_arange\");\n      nnvm::NodeAttrs attrs;\n      op::RangeParam param = {};\n      param.start = args[0].operator double();\n      if (args[1].type_code() == kNull) {\n        param.stop = dmlc::nullopt;\n      } else {\n        param.stop = args[1].operator double();\n      }\n      param.step        = args[2].operator double();\n      param.repeat      = 1;\n      param.infer_range = false;\n      if (args[3].type_code() == kNull) {\n        param.dtype =\n            Imperative::Get()->is_np_default_dtype() ? mshadow::kInt64 : mshadow::kFloat32;\n      } else {\n        param.dtype = String2MXNetTypeWithBool(args[3].operator std::string());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::RangeParam>(&attrs);\n      if (args[4].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[4].operator std::string();\n      }\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, 0, nullptr, &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.eye\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_eye\");\n  nnvm::NodeAttrs attrs;\n  op::NumpyEyeParam param = {};\n  param.N = args[0].operator nnvm::dim_t();\n  if (args[1].type_code() == kNull) {\n    param.M = dmlc::nullopt;\n  } else {\n    param.M = args[1].operator nnvm::dim_t();\n  }\n  param.k = args[2].operator nnvm::dim_t();\n  if (args[4].type_code() == kNull) {\n    param.dtype = mxnet::common::GetDefaultDtype();\n  } else {\n    param.dtype = String2MXNetTypeWithBool(args[4].operator std::string());\n  }\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::NumpyEyeParam>(&attrs);\n  if (args[3].type_code() != kNull) {\n    attrs.dict[\"ctx\"] = args[3].operator std::string();\n  }\n  int num_outputs = 0;\n  auto ndoutputs  = Invoke(op, &attrs, 0, nullptr, &num_outputs, nullptr);\n  *ret            = ndoutputs[0];\n});\n\nMXNET_REGISTER_API(\"_npi.linspace\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_linspace\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyLinspaceParam param;\n      if (args[0].type_code() == kDLFloat || args[1].type_code() == kDLFloat) {\n        param.start_double = args[0].operator double();\n        param.stop_double  = args[1].operator double();\n        param.value_type   = 2;\n      } else if (args[0].type_code() == kDLUInt || args[1].type_code() == kDLUInt) {\n        if (args[0].type_code() == kDLUInt) {\n          param.start_uint = args[0].operator uint64_t();\n        } else {\n          param.start_uint = args[0].operator int64_t();\n        }\n        if (args[1].type_code() == kDLUInt) {\n          param.stop_uint = args[1].operator uint64_t();\n        } else {\n          param.stop_uint = args[1].operator int64_t();\n        }\n        param.value_type = 1;\n      } else {\n        param.start_int  = args[0].operator int64_t();\n        param.stop_int   = args[1].operator int64_t();\n        param.value_type = 0;\n      }\n      if (features::is_enabled(features::INT64_TENSOR_SIZE))\n        param.num = args[2].operator int64_t();\n      else\n        param.num = args[2].operator int();\n      if (args[3].type_code() == kNull) {\n        param.endpoint = true;\n      } else {\n        param.endpoint = args[3].operator bool();\n      }\n      if (args[5].type_code() == kNull) {\n        param.dtype = mxnet::common::GetDefaultDtype();\n      } else {\n        param.dtype = String2MXNetTypeWithBool(args[5].operator std::string());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyLinspaceParam>(&attrs);\n      if (args[4].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[4].operator std::string();\n      }\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, 0, nullptr, &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.logspace\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_logspace\");\n      nnvm::NodeAttrs attrs;\n      op::LogspaceParam param = {};\n      param.start = args[0].operator double();\n      param.stop  = args[1].operator double();\n      if (features::is_enabled(features::INT64_TENSOR_SIZE))\n        param.num = args[2].operator int64_t();\n      else\n        param.num = args[2].operator int();\n      if (args[3].type_code() == kNull) {\n        param.endpoint = true;\n      } else {\n        param.endpoint = args[3].operator bool();\n      }\n      if (args[4].type_code() == kNull) {\n        param.base = 10.0;\n      } else {\n        param.base = args[4].operator double();\n      }\n      if (args[6].type_code() == kNull) {\n        param.dtype = mxnet::common::GetDefaultDtype();\n      } else {\n        param.dtype = String2MXNetTypeWithBool(args[6].operator std::string());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::LogspaceParam>(&attrs);\n      if (args[5].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[5].operator std::string();\n      }\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, 0, nullptr, &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.ones\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_ones\");\n  nnvm::NodeAttrs attrs;\n  op::InitOpParam param = {};\n  if (args[0].type_code() == kDLInt) {\n    param.shape = TShape(1, args[0].operator int64_t());\n  } else {\n    param.shape = TShape(args[0].operator ObjectRef());\n  }\n  if (args[1].type_code() == kNull) {\n    param.dtype = mxnet::common::GetDefaultDtype();\n  } else {\n    param.dtype = String2MXNetTypeWithBool(args[1].operator std::string());\n  }\n  attrs.parsed = param;\n  attrs.op     = op;\n  if (args[2].type_code() != kNull) {\n    attrs.dict[\"ctx\"] = args[2].operator std::string();\n  }\n  int num_outputs = 0;\n  SetAttrDict<op::InitOpParam>(&attrs);\n  auto ndoutputs = Invoke(op, &attrs, 0, nullptr, &num_outputs, nullptr);\n  *ret           = ndoutputs[0];\n});\n\nMXNET_REGISTER_API(\"_npi.full\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_full\");\n  nnvm::NodeAttrs attrs;\n  op::NumpyInitOpWithScalarParam param = {};\n  if (args[0].type_code() == kDLInt) {\n    param.shape = TShape(1, args[0].operator int64_t());\n  } else {\n    param.shape = TShape(args[0].operator ObjectRef());\n  }\n  if (args[1].type_code() == kNull) {\n    param.dtype = mxnet::common::GetDefaultDtype();\n  } else {\n    param.dtype = String2MXNetTypeWithBool(args[1].operator std::string());\n  }\n  if (args[2].type_code() == kDLInt) {\n    param.value_type = 0;\n    param.int_value  = args[2].operator int64_t();\n  } else if (args[2].type_code() == kDLUInt) {\n    param.value_type = 1;\n    param.uint_value = args[2].operator uint64_t();\n  } else {\n    param.value_type   = 2;\n    param.double_value = args[2].operator double();\n  }\n  attrs.parsed = param;\n  attrs.op     = op;\n  if (args[3].type_code() != kNull) {\n    attrs.dict[\"ctx\"] = args[3].operator std::string();\n  }\n  SetAttrDict<op::NumpyInitOpWithScalarParam>(&attrs);\n  NDArray* out      = args[4].operator mxnet::NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  auto ndoutputs    = Invoke(op, &attrs, 0, nullptr, &num_outputs, outputs);\n  if (out) {\n    *ret = PythonArg(4);\n  } else {\n    *ret = ndoutputs[0];\n  }\n});\n\nMXNET_REGISTER_API(\"_npi.identity\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_identity\");\n      nnvm::NodeAttrs attrs;\n      op::InitOpParam param = {};\n      param.shape = TShape(args[0].operator ObjectRef());\n      if (args[1].type_code() == kNull) {\n        param.dtype = mxnet::common::GetDefaultDtype();\n      } else {\n        param.dtype = String2MXNetTypeWithBool(args[1].operator std::string());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      if (args[2].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[2].operator std::string();\n      }\n      int num_outputs = 0;\n      SetAttrDict<op::InitOpParam>(&attrs);\n      auto ndoutputs = Invoke(op, &attrs, 0, nullptr, &num_outputs, nullptr);\n      *ret           = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_insert_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_insert_op.cc\n * \\brief Implementation of the API of functions in the following file:\n          src/operator/numpy/np_insert_op_scalar.cc\n          src/operator/numpy/np_insert_op_slice.cc\n          src/operator/numpy/np_insert_op_tensor.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include <vector>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_insert_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.insert_scalar\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      static const nnvm::Op* op = Op::Get(\"_npi_insert_scalar\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyInsertParam param = {};\n      int num_inputs = 0;\n      param.start    = dmlc::nullopt;\n      param.step     = dmlc::nullopt;\n      param.stop     = dmlc::nullopt;\n      if (args[1].type_code() == kDLInt || args[1].type_code() == kDLUInt ||\n          args[1].type_code() == kDLFloat) {\n        param.val  = args[1].operator double();\n        num_inputs = 1;\n      } else {\n        param.val  = dmlc::nullopt;\n        num_inputs = 2;\n      }\n      if (features::is_enabled(features::INT64_TENSOR_SIZE)) {\n        param.int_ind = args[2].operator int64_t();\n      } else {\n        param.int_ind = args[2].operator int();\n      }\n      if (args[3].type_code() == kNull) {\n        param.axis = dmlc::nullopt;\n      } else {\n        param.axis = args[3].operator int();\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyInsertParam>(&attrs);\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.insert_slice\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      static const nnvm::Op* op = Op::Get(\"_npi_insert_slice\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyInsertParam param = {};\n      int num_inputs = 0;\n      if (args[1].type_code() == kDLInt || args[1].type_code() == kDLUInt ||\n          args[1].type_code() == kDLFloat) {\n        param.val  = args[1].operator double();\n        num_inputs = 1;\n      } else {\n        param.val  = dmlc::nullopt;\n        num_inputs = 2;\n      }\n      if (args[2].type_code() == kNull) {\n        param.start = dmlc::nullopt;\n      } else {\n        param.start = args[2].operator int64_t();\n      }\n      if (args[3].type_code() == kNull) {\n        param.stop = dmlc::nullopt;\n      } else {\n        param.stop = args[3].operator int64_t();\n      }\n      if (args[4].type_code() == kNull) {\n        param.step = dmlc::nullopt;\n      } else {\n        param.step = args[4].operator int64_t();\n      }\n      if (args[5].type_code() == kNull) {\n        param.axis = dmlc::nullopt;\n      } else {\n        param.axis = args[5].operator int();\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyInsertParam>(&attrs);\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.insert_tensor\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      static const nnvm::Op* op = Op::Get(\"_npi_insert_tensor\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyInsertParam param = {};\n      param.start    = dmlc::nullopt;\n      param.step     = dmlc::nullopt;\n      param.stop     = dmlc::nullopt;\n      int num_inputs = 0;\n      if (args[2].type_code() == kDLInt || args[2].type_code() == kDLUInt ||\n          args[2].type_code() == kDLFloat) {\n        param.val  = args[2].operator double();\n        num_inputs = 2;\n      } else {\n        param.val  = dmlc::nullopt;\n        num_inputs = 3;\n      }\n      if (args[3].type_code() == kNull) {\n        param.axis = dmlc::nullopt;\n      } else {\n        param.axis = args[3].operator int();\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyInsertParam>(&attrs);\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_interp_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_interp_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_interp_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_interp_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.interp\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      static const nnvm::Op* op = Op::Get(\"_npi_interp\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyInterpParam param = {};\n      if (args[3].type_code() == kNull) {\n        param.left = dmlc::nullopt;\n      } else {\n        param.left = args[3].operator double();\n      }\n      if (args[4].type_code() == kNull) {\n        param.right = dmlc::nullopt;\n      } else {\n        param.right = args[4].operator double();\n      }\n      if (args[5].type_code() == kNull) {\n        param.period = dmlc::nullopt;\n      } else {\n        param.period = args[5].operator double();\n      }\n      if (args[2].type_code() == kDLInt || args[2].type_code() == kDLFloat) {\n        param.x_scalar    = args[2].operator double();\n        param.x_is_scalar = true;\n        attrs.op          = op;\n        attrs.parsed      = param;\n        SetAttrDict<op::NumpyInterpParam>(&attrs);\n        NDArray* inputs[] = {args[0].operator mxnet::NDArray*(),\n                             args[1].operator mxnet::NDArray*()};\n        int num_inputs    = 2;\n        int num_outputs   = 0;\n        auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n        *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n      } else {\n        param.x_scalar    = 0.0;\n        param.x_is_scalar = false;\n        attrs.op          = op;\n        attrs.parsed      = param;\n        SetAttrDict<op::NumpyInterpParam>(&attrs);\n        NDArray* inputs[] = {args[0].operator mxnet::NDArray*(),\n                             args[1].operator mxnet::NDArray*(),\n                             args[2].operator mxnet::NDArray*()};\n        int num_inputs    = 3;\n        int num_outputs   = 0;\n        auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n        *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_kron.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_kron.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_kron.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_kron-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.kron\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  nnvm::NodeAttrs attrs;\n  const nnvm::Op* op = Op::Get(\"_npi_kron\");\n  attrs.op           = op;\n  NDArray* inputs[]  = {args[0].operator NDArray*(), args[1].operator NDArray*()};\n  int num_inputs     = 2;\n  int num_outputs    = 0;\n  auto ndoutputs     = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret               = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_matmul_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_matmul_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_matmul_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_matmul_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.matmul\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      static const nnvm::Op* op = Op::Get(\"_npi_matmul\");\n      nnvm::NodeAttrs attrs;\n      int num_inputs     = 2;\n      NDArray* inputs[2] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};\n      attrs.op           = op;\n      NDArray* out       = args[2].operator mxnet::NDArray*();\n      NDArray** outputs  = out == nullptr ? nullptr : &out;\n      int num_outputs    = out != nullptr;\n      auto ndoutputs     = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(2);\n      } else {\n        *ret = ndoutputs[0];\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_matrix_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_matrix_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_matrix_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include <vector>\n#include \"../utils.h\"\n#include \"../../../operator/nn/concat-inl.h\"\n#include \"../../../operator/tensor/matrix_op-inl.h\"\n#include \"../../../operator/numpy/np_matrix_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.transpose\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      static const nnvm::Op* op = Op::Get(\"_npi_transpose\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyTransposeParam param = {};\n      if (args[1].type_code() == kNull) {\n        param.axes = TShape(-1, 0);\n      } else if (args[1].type_code() == kDLInt) {\n        param.axes = TShape(1, args[1].operator int64_t());\n      } else {\n        param.axes = TShape(args[1].operator ObjectRef());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyTransposeParam>(&attrs);\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      int num_inputs    = 1;\n      int num_outputs   = 0;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.expand_dims\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_expand_dims\");\n      nnvm::NodeAttrs attrs;\n      op::ExpandDimParam param = {};\n      param.axis = args[1].operator int();\n\n      // we directly copy ExpandDimParam, which is trivially-copyable\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::ExpandDimParam>(&attrs);\n\n      int num_outputs   = 0;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      int num_inputs    = 1;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.stack\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_stack\");\n  nnvm::NodeAttrs attrs;\n  op::StackParam param = {};\n\n  int i          = 0;\n  int num_inputs = 0;\n  std::vector<NDArray*> inputs;\n  while (args[i].type_code() != kDLInt) {\n    inputs.push_back(args[i].operator mxnet::NDArray*());\n    i++;\n    num_inputs++;\n  }\n\n  param.num_args = i;\n  param.axis     = args[i].operator int64_t();\n  attrs.parsed   = param;\n  attrs.op       = op;\n  SetAttrDict<op::StackParam>(&attrs);\n  NDArray* out      = args[i + 1].operator mxnet::NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, outputs);\n  if (out) {\n    *ret = PythonArg(i + 1);\n  } else {\n    *ret = ndoutputs[0];\n  }\n});\n\nMXNET_REGISTER_API(\"_npi.flip\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_flip\");\n  nnvm::NodeAttrs attrs;\n  op::FlipParam param = {};\n\n  NDArray* out      = args[2].operator mxnet::NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  if (args[1].type_code() == kNull) {\n    param.axis = mxnet::Tuple<int>(-1, dim_t(0));\n  } else if (args[1].type_code() == kDLInt) {\n    param.axis = Tuple<int>(1, args[1].operator int64_t());\n  } else {\n    param.axis = Tuple<int>(args[1].operator ObjectRef());\n  }\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  int num_inputs    = 1;\n  attrs.parsed      = param;\n  attrs.op          = op;\n  SetAttrDict<op::FlipParam>(&attrs);\n  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (out) {\n    *ret = PythonArg(2);\n  } else {\n    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n  }\n});\n\nMXNET_REGISTER_API(\"_npi.concatenate\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_concatenate\");\n      nnvm::NodeAttrs attrs;\n      op::ConcatParam param = {};\n      int arg_size   = args.num_args;\n      param.num_args = arg_size - 2;\n      if (args[arg_size - 2].type_code() == kNull) {\n        param.dim = dmlc::nullopt;\n      } else {\n        param.dim = args[arg_size - 2].operator int();\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::ConcatParam>(&attrs);\n      int num_inputs = arg_size - 2;\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      NDArray* out      = args[arg_size - 1].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(arg_size - 1);\n      } else {\n        *ret = ndoutputs[0];\n      }\n    });\n\nMXNET_REGISTER_API(\"_npi.dstack\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_dstack\");\n      nnvm::NodeAttrs attrs;\n      op::ConcatParam param = {};\n      int args_size = args.size();\n      // param.num_args\n      param.num_args = args_size;\n      attrs.parsed   = param;\n      attrs.op       = op;\n      SetAttrDict<op::ConcatParam>(&attrs);\n      // inputs\n      int num_inputs = args_size;\n      std::vector<NDArray*> inputs_vec(args_size, nullptr);\n      for (int i = 0; i < args_size; ++i) {\n        inputs_vec[i] = args[i].operator mxnet::NDArray*();\n      }\n      NDArray** inputs = inputs_vec.data();\n      // outputs\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.split\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_split\");\n  int num_inputs     = 1;\n  NDArray* inputs[]  = {args[0].operator mxnet::NDArray*()};\n  nnvm::NodeAttrs attrs;\n  op::SplitParam param = {};\n  param.axis         = args[2].operator int();\n  param.squeeze_axis = false;\n  if (args[1].type_code() == kDLInt) {\n    param.indices  = TShape(0, 0);\n    param.sections = args[1].operator int();\n    int index      = param.axis >= 0 ? param.axis : param.axis + inputs[0]->shape().ndim();\n    CHECK_GE(index, 0) << \"IndexError: tuple index out of range\";\n    CHECK_GT(param.sections, 0) << \"ValueError: number sections must be larger than 0\";\n    CHECK_EQ(inputs[0]->shape()[index] % param.sections, 0)\n        << \"ValueError: array split does not result in an equal division\";\n  } else {\n    TShape t      = TShape(args[1].operator ObjectRef());\n    param.indices = TShape(t.ndim() + 1, 0);\n    for (int i = 0; i < t.ndim(); ++i) {\n      param.indices[i + 1] = t[i];\n    }\n    param.sections = 0;\n  }\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::SplitParam>(&attrs);\n\n  int num_outputs = 0;\n  auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  std::vector<NDArrayHandle> ndarray_handles;\n  ndarray_handles.reserve(num_outputs);\n  for (int i = 0; i < num_outputs; ++i) {\n    ndarray_handles.emplace_back(ndoutputs[i]);\n  }\n  *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n});\n\nMXNET_REGISTER_API(\"_npi.roll\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  static const nnvm::Op* op = Op::Get(\"_npi_roll\");\n  nnvm::NodeAttrs attrs;\n  op::NumpyRollParam param = {};\n  if (args[1].type_code() == kNull) {\n    param.shift = dmlc::nullopt;\n  } else if (args[1].type_code() == kDLInt) {\n    param.shift = TShape(1, args[1].operator int64_t());\n  } else {\n    param.shift = TShape(args[1].operator ObjectRef());\n  }\n  if (args[2].type_code() == kNull) {\n    param.axis = dmlc::nullopt;\n  } else if (args[2].type_code() == kDLInt) {\n    param.axis = TShape(1, args[2].operator int64_t());\n  } else {\n    param.axis = TShape(args[2].operator ObjectRef());\n  }\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::NumpyRollParam>(&attrs);\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  int num_inputs    = 1;\n  int num_outputs   = 0;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = ndoutputs[0];\n});\n\nMXNET_REGISTER_API(\"_npi.rot90\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  static const nnvm::Op* op = Op::Get(\"_npi_rot90\");\n  nnvm::NodeAttrs attrs;\n  op::NumpyRot90Param param = {};\n  param.k = args[1].operator int();\n  if (args[2].type_code() == kNull) {\n    param.axes = dmlc::nullopt;\n  } else if (args[2].type_code() == kDLInt) {\n    param.axes = TShape(1, args[2].operator int64_t());\n  } else {\n    param.axes = TShape(args[2].operator ObjectRef());\n  }\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::NumpyRot90Param>(&attrs);\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  int num_inputs    = 1;\n  int num_outputs   = 0;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = ndoutputs[0];\n});\n\nMXNET_REGISTER_API(\"_npi.column_stack\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_column_stack\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyColumnStackParam param = {};\n      param.num_args = args.size();\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyColumnStackParam>(&attrs);\n      int num_outputs = 0;\n      std::vector<NDArray*> inputs;\n      inputs.reserve(param.num_args);\n      for (int i = 0; i < param.num_args; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      auto ndoutputs = Invoke(op, &attrs, param.num_args, &inputs[0], &num_outputs, nullptr);\n      *ret           = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.hstack\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_hstack\");\n      nnvm::NodeAttrs attrs;\n      op::ConcatParam param = {};\n      param.num_args = args.size();\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::ConcatParam>(&attrs);\n      int num_outputs = 0;\n      std::vector<NDArray*> inputs;\n      inputs.reserve(param.num_args);\n      for (int i = 0; i < param.num_args; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      auto ndoutputs = Invoke(op, &attrs, param.num_args, &inputs[0], &num_outputs, nullptr);\n      *ret           = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.array_split\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      static const nnvm::Op* op = Op::Get(\"_npi_array_split\");\n      nnvm::NodeAttrs attrs;\n      op::SplitParam param = {};\n      param.axis         = args[2].operator int();\n      param.squeeze_axis = false;\n      if (args[1].type_code() == kDLInt) {\n        param.indices  = TShape(0, 0);\n        param.sections = args[1].operator int();\n        CHECK_GT(param.sections, 0) << \"ValueError: number sections must be larger than 0\";\n      } else {\n        TShape t      = TShape(args[1].operator ObjectRef());\n        param.indices = TShape(t.ndim() + 1, 0);\n        for (int i = 0; i < t.ndim(); ++i) {\n          param.indices[i + 1] = t[i];\n        }\n        param.sections = 0;\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::SplitParam>(&attrs);\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      int num_inputs    = 1;\n      int num_outputs   = 0;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      std::vector<NDArrayHandle> ndarray_handles;\n      ndarray_handles.reserve(num_outputs);\n      for (int i = 0; i < num_outputs; ++i) {\n        ndarray_handles.emplace_back(ndoutputs[i]);\n      }\n      *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n    });\n\nMXNET_REGISTER_API(\"_npi.dsplit\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      static const nnvm::Op* op = Op::Get(\"_npi_split\");\n      int num_inputs            = 1;\n      NDArray* inputs[]         = {args[0].operator mxnet::NDArray*()};\n      CHECK_GE(inputs[0]->shape().ndim(), 3)\n          << \"ValueError: dsplit only works on arrays of 3 or more dimensions\";\n      nnvm::NodeAttrs attrs;\n      op::SplitParam param = {};\n      param.axis         = 2;\n      param.squeeze_axis = false;\n      if (args[1].type_code() == kDLInt) {\n        param.indices  = TShape(0, 0);\n        param.sections = args[1].operator int();\n        CHECK_EQ(inputs[0]->shape()[2] % param.sections, 0)\n            << \"ValueError: array split does not result in an equal division\";\n        CHECK_GT(param.sections, 0) << \"ValueError: number sections must be larger than 0\";\n      } else {\n        TShape t      = TShape(args[1].operator ObjectRef());\n        param.indices = TShape(t.ndim() + 1, 0);\n        for (int i = 0; i < t.ndim(); ++i) {\n          param.indices[i + 1] = t[i];\n        }\n        param.sections = 0;\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::SplitParam>(&attrs);\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      std::vector<NDArrayHandle> ndarray_handles;\n      ndarray_handles.reserve(num_outputs);\n      for (int i = 0; i < num_outputs; ++i) {\n        ndarray_handles.emplace_back(ndoutputs[i]);\n      }\n      *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n    });\n\nMXNET_REGISTER_API(\"_npi.hsplit\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      static const nnvm::Op* op = Op::Get(\"_npi_hsplit\");\n      int num_inputs            = 1;\n      NDArray* inputs[]         = {args[0].operator mxnet::NDArray*()};\n      CHECK_GE(inputs[0]->shape().ndim(), 1)\n          << \"ValueError: hsplit only works on arrays of 1 or more dimensions\";\n      nnvm::NodeAttrs attrs;\n      op::SplitParam param = {};\n      param.axis         = 0;\n      param.squeeze_axis = false;\n      if (args[1].type_code() == kDLInt) {\n        param.indices  = TShape(0, 0);\n        param.sections = args[1].operator int();\n        CHECK_GT(param.sections, 0) << \"ValueError: number sections must be larger than 0\";\n      } else {\n        TShape t      = TShape(args[1].operator ObjectRef());\n        param.indices = TShape(t.ndim() + 1, 0);\n        for (int i = 0; i < t.ndim(); ++i) {\n          param.indices[i + 1] = t[i];\n        }\n        param.sections = 0;\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::SplitParam>(&attrs);\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      std::vector<NDArrayHandle> ndarray_handles;\n      ndarray_handles.reserve(num_outputs);\n      for (int i = 0; i < num_outputs; ++i) {\n        ndarray_handles.emplace_back(ndoutputs[i]);\n      }\n      *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n    });\n\nMXNET_REGISTER_API(\"_npi.vsplit\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      static const nnvm::Op* op = Op::Get(\"_npi_split\");\n      int num_inputs            = 1;\n      NDArray* inputs[]         = {args[0].operator mxnet::NDArray*()};\n      CHECK_GE(inputs[0]->shape().ndim(), 2)\n          << \"ValueError: vsplit only works on arrays of 2 or more dimensions\";\n      nnvm::NodeAttrs attrs;\n      op::SplitParam param = {};\n      param.axis         = 0;\n      param.squeeze_axis = false;\n      if (args[1].type_code() == kDLInt) {\n        param.indices  = TShape(0, 0);\n        param.sections = args[1].operator int();\n        CHECK_EQ(inputs[0]->shape()[0] % param.sections, 0)\n            << \"ValueError: array split does not result in an equal division\";\n        CHECK_GT(param.sections, 0) << \"ValueError: number sections must be larger than 0\";\n      } else {\n        TShape t      = TShape(args[1].operator ObjectRef());\n        param.indices = TShape(t.ndim() + 1, 0);\n        for (int i = 0; i < t.ndim(); ++i) {\n          param.indices[i + 1] = t[i];\n        }\n        param.sections = 0;\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::SplitParam>(&attrs);\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      std::vector<NDArrayHandle> ndarray_handles;\n      ndarray_handles.reserve(num_outputs);\n      for (int i = 0; i < num_outputs; ++i) {\n        ndarray_handles.emplace_back(ndoutputs[i]);\n      }\n      *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n    });\n\nMXNET_REGISTER_API(\"_npi.diag\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_diag\");\n  nnvm::NodeAttrs attrs;\n  op::NumpyDiagParam param = {};\n  if (features::is_enabled(features::INT64_TENSOR_SIZE))\n    param.k = args[1].operator int64_t();\n  else\n    param.k = args[1].operator int();\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::NumpyDiagParam>(&attrs);\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  int num_inputs    = 1;\n  int num_outputs   = 0;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = ndoutputs[0];\n});\n\nMXNET_REGISTER_API(\"_npi.rollaxis\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_rollaxis\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyRollaxisParam param = {};\n      param.axis   = args[1].operator int();\n      param.start  = args[2].operator int();\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyRollaxisParam>(&attrs);\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      int num_inputs    = 1;\n      int num_outputs   = 0;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.reshape\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_reshape\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyXReshapeParam param = {};\n      if (args[1].type_code() == kNull) {\n        param.newshape = TShape(-1, 0);\n      } else if (args[1].type_code() == kDLInt) {\n        param.newshape = TShape(1, args[1].operator int64_t());\n      } else {\n        param.newshape = TShape(args[1].operator ObjectRef());\n      }\n      param.reverse = args[2].operator bool();\n      param.order   = args[3].operator std::string();\n      attrs.parsed  = param;\n      attrs.op      = op;\n      SetAttrDict<op::NumpyXReshapeParam>(&attrs);\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      int num_inputs    = 1;\n      int num_outputs   = 0;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.moveaxis\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_moveaxis\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyMoveaxisParam param = {};\n      if (args[1].type_code() == kNull) {\n        param.source = TShape(-1, 0);\n      } else if (args[1].type_code() == kDLInt) {\n        param.source = TShape(1, args[1].operator int64_t());\n      } else {\n        param.source = TShape(args[1].operator ObjectRef());\n      }\n      if (args[2].type_code() == kNull) {\n        param.destination = TShape(-1, 0);\n      } else if (args[2].type_code() == kDLInt) {\n        param.destination = TShape(1, args[2].operator int64_t());\n      } else {\n        param.destination = TShape(args[2].operator ObjectRef());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyMoveaxisParam>(&attrs);\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      int num_inputs    = 1;\n      int num_outputs   = 0;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.diagonal\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_diagonal\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyDiagonalParam param = {};\n      if (features::is_enabled(features::INT64_TENSOR_SIZE))\n        param.offset = args[1].operator int64_t();\n      else\n        param.offset = args[1].operator int();\n      param.axis1  = args[2].operator int();\n      param.axis2  = args[3].operator int();\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyDiagonalParam>(&attrs);\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      int num_inputs    = 1;\n      int num_outputs   = 0;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.diag_indices_from\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_diag_indices_from\");\n      nnvm::NodeAttrs attrs;\n      attrs.op          = op;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      int num_inputs    = 1;\n      int num_outputs   = 0;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.diagflat\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_diagflat\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyDiagflatParam param = {};\n      param.k         = args[1].operator int();\n      int num_inputs  = 1;\n      int num_outputs = 0;\n      attrs.parsed    = param;\n      attrs.op        = op;\n      SetAttrDict<op::NumpyDiagflatParam>(&attrs);\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.squeeze\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_squeeze\");\n      nnvm::NodeAttrs attrs;\n      op::SqueezeParam param = {};\n      if (args[1].type_code() == kNull) {\n        param.axis = dmlc::optional<mxnet::Tuple<int>>();\n      } else if (args[1].type_code() == kDLInt) {\n        param.axis = Tuple<int>(1, args[1].operator int64_t());\n      } else {\n        param.axis = Tuple<int>(args[1].operator ObjectRef());\n      }\n      int num_inputs  = 1;\n      int num_outputs = 0;\n      attrs.parsed    = param;\n      attrs.op        = op;\n      SetAttrDict<op::SqueezeParam>(&attrs);\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npi.tril_indices\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_tril_indices\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyTrilindicesParam param = {};\n      if (features::is_enabled(features::INT64_TENSOR_SIZE)) {\n        param.n = args[0].operator int64_t();\n        param.k = args[1].operator int64_t();\n        param.m = args[2].operator int64_t();\n      } else {\n        param.n = args[0].operator int();\n        param.k = args[1].operator int();\n        param.m = args[2].operator int();\n      }\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyTrilindicesParam>(&attrs);\n\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, 0, nullptr, &num_outputs, nullptr);\n      std::vector<NDArrayHandle> ndarray_handles;\n      ndarray_handles.reserve(num_outputs);\n      for (int i = 0; i < num_outputs; ++i) {\n        ndarray_handles.emplace_back(ndoutputs[i]);\n      }\n      *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n    });\n\nMXNET_REGISTER_API(\"_npi.vstack\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_vstack\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyVstackParam param = {};\n      param.num_args = args.size();\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyVstackParam>(&attrs);\n      int num_outputs = 0;\n      std::vector<NDArray*> inputs_vec(args.size(), nullptr);\n      for (int i = 0; i < args.size(); ++i) {\n        inputs_vec[i] = args[i].operator mxnet::NDArray*();\n      }\n      NDArray** inputs = inputs_vec.data();\n      auto ndoutputs   = Invoke(op, &attrs, param.num_args, inputs, &num_outputs, nullptr);\n      *ret             = ndoutputs[0];\n    });\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_memory_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_memory_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_memory_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.share_memory\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_share_memory\");\n      nnvm::NodeAttrs attrs;\n      attrs.op          = op;\n      int num_inputs    = 2;\n      int num_outputs   = 0;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_moments_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_moments_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_moments_op.cc\n */\n\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_broadcast_reduce_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.std\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_std\");\n  op::NumpyMomentsParam param = {};\n  nnvm::NodeAttrs attrs;\n  attrs.op = op;\n\n  // parse axis\n  if (args[1].type_code() == kNull) {\n    param.axis = dmlc::nullopt;\n  } else {\n    if (args[1].type_code() == kDLInt) {\n      param.axis = Tuple<int>(1, args[1].operator int64_t());\n    } else {\n      param.axis = Tuple<int>(args[1].operator ObjectRef());\n    }\n  }\n\n  // parse dtype\n  if (args[2].type_code() == kNull) {\n    param.dtype = dmlc::nullopt;\n  } else {\n    param.dtype = String2MXNetTypeWithBool(args[2].operator std::string());\n  }\n\n  // parse ddof\n  param.ddof = args[3].operator int();\n\n  // parse keepdims\n  if (args[4].type_code() == kNull) {\n    param.keepdims = false;\n  } else {\n    param.keepdims = args[4].operator bool();\n  }\n\n  attrs.parsed = param;\n\n  SetAttrDict<op::NumpyMomentsParam>(&attrs);\n\n  NDArray* inputs[] = {args[0].operator NDArray*()};\n  int num_inputs    = 1;\n\n  NDArray* outputs[] = {args[5].operator NDArray*()};\n  NDArray** out      = (outputs[0] == nullptr) ? nullptr : outputs;\n  int num_outputs    = (outputs[0] != nullptr);\n  auto ndoutputs     = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, out);\n\n  if (out) {\n    *ret = PythonArg(5);\n  } else {\n    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n  }\n});\n\nMXNET_REGISTER_API(\"_npi.var\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_var\");\n  op::NumpyMomentsParam param = {};\n  nnvm::NodeAttrs attrs;\n  attrs.op = op;\n\n  // parse axis\n  if (args[1].type_code() == kNull) {\n    param.axis = dmlc::nullopt;\n  } else {\n    if (args[1].type_code() == kDLInt) {\n      param.axis = Tuple<int>(1, args[1].operator int64_t());\n    } else {\n      param.axis = Tuple<int>(args[1].operator ObjectRef());\n    }\n  }\n\n  // parse dtype\n  if (args[2].type_code() == kNull) {\n    param.dtype = dmlc::nullopt;\n  } else {\n    param.dtype = String2MXNetTypeWithBool(args[2].operator std::string());\n  }\n\n  // parse ddof\n  param.ddof = args[3].operator int();\n\n  // parse keepdims\n  if (args[4].type_code() == kNull) {\n    param.keepdims = false;\n  } else {\n    param.keepdims = args[4].operator bool();\n  }\n\n  attrs.parsed = param;\n\n  SetAttrDict<op::NumpyMomentsParam>(&attrs);\n\n  NDArray* inputs[] = {args[0].operator NDArray*()};\n  int num_inputs    = 1;\n\n  NDArray* outputs[] = {args[5].operator NDArray*()};\n  NDArray** out      = (outputs[0] == nullptr) ? nullptr : outputs;\n  int num_outputs    = (outputs[0] != nullptr);\n  auto ndoutputs     = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, out);\n\n  if (out) {\n    *ret = PythonArg(5);\n  } else {\n    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n  }\n});\n\nMXNET_REGISTER_API(\"_npi.average\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_average\");\n      op::NumpyWeightedAverageParam param = {};\n      nnvm::NodeAttrs attrs;\n      attrs.op = op;\n\n      // parse axis\n      if (args[2].type_code() == kNull) {\n        param.axis = dmlc::nullopt;\n      } else {\n        if (args[2].type_code() == kDLInt) {\n          param.axis = Tuple<int>(1, args[2].operator int64_t());\n        } else {\n          param.axis = Tuple<int>(args[2].operator ObjectRef());\n        }\n      }\n\n      // parse returned\n      CHECK_NE(args[3].type_code(), kNull) << \"returned cannot be None\";\n      param.returned = args[3].operator bool();\n\n      // parse weighted\n      CHECK_NE(args[4].type_code(), kNull) << \"weighted cannot be None\";\n      param.weighted = args[4].operator bool();\n\n      attrs.parsed = param;\n\n      SetAttrDict<op::NumpyWeightedAverageParam>(&attrs);\n\n      int num_inputs     = param.weighted ? 2 : 1;\n      NDArray* outputs[] = {args[5].operator NDArray*()};\n      NDArray** out      = (outputs[0] == nullptr) ? nullptr : outputs;\n      int num_outputs    = (outputs[0] != nullptr);\n\n      if (param.weighted) {\n        NDArray* inputs[] = {args[0].operator NDArray*(), args[1].operator NDArray*()};\n        auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, out);\n        if (out) {\n          *ret = PythonArg(5);\n        } else {\n          if (param.returned) {\n            *ret = ADT(0, {NDArrayHandle(ndoutputs[0]), NDArrayHandle(ndoutputs[1])});\n          } else {\n            *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n          }\n        }\n      } else {\n        NDArray* inputs[] = {args[0].operator NDArray*()};\n        auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, out);\n        if (out) {\n          *ret = PythonArg(5);\n        } else {\n          if (param.returned) {\n            *ret = ADT(0, {NDArrayHandle(ndoutputs[0]), NDArrayHandle(ndoutputs[1])});\n          } else {\n            *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n          }\n        }\n      }\n    });\n\n};  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_nan_to_num_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_nan_to_num_op.cc\n * \\brief Implementation of the API of nan_to_num function in\n *        src/operator/tensor/np_elemwise_unary_op_basic.cc\n */\n#include <mxnet/api_registry.h>\n#include \"../utils.h\"\n#include \"../../../operator/tensor/elemwise_unary_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.nan_to_num\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_nan_to_num\");\n      nnvm::NodeAttrs attrs;\n\n      op::NumpyNanToNumParam param = {};\n      int num_inputs    = 1;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n\n      param.copy = args[1].operator bool();\n      param.nan  = args[2].operator double();\n\n      if (args[3].type_code() == kNull) {\n        param.posinf = dmlc::nullopt;\n      } else {\n        param.posinf = args[3].operator double();\n      }\n\n      if (args[4].type_code() == kNull) {\n        param.neginf = dmlc::nullopt;\n      } else {\n        param.neginf = args[4].operator double();\n      }\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyNanToNumParam>(&attrs);\n\n      NDArray* out      = args[5].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      // set the number of outputs provided by the `out` arugment\n      int num_outputs = out != nullptr;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(5);\n      } else {\n        *ret = ndoutputs[0];\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_nonzero_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_nonzero_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_nonzero_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.nonzero\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_nonzero\");\n      nnvm::NodeAttrs attrs;\n\n      attrs.op = op;\n\n      int num_inputs    = 1;\n      int num_outputs   = 0;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_ordering_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_ordering_op.cc\n * \\brief Implementation of the API of functions in src/operator/tensor/ordering_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/tensor/ordering_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.sort\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_sort\");\n  nnvm::NodeAttrs attrs;\n  op::SortParam param = {};\n\n  if (args[1].type_code() == kNull) {\n    param.axis = dmlc::nullopt;\n  } else {\n    param.axis = args[1].operator int();\n  }\n  param.is_ascend = args[2].operator bool();\n\n  attrs.parsed = std::move(param);\n  attrs.op     = op;\n\n  int num_inputs    = 1;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n\n  int num_outputs = 0;\n  SetAttrDict<op::SortParam>(&attrs);\n  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret           = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n});\n\nMXNET_REGISTER_API(\"_npi.argsort\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_argsort\");\n      nnvm::NodeAttrs attrs;\n      op::ArgSortParam param = {};\n\n      if (args[1].type_code() == kNull) {\n        param.axis = dmlc::nullopt;\n      } else {\n        param.axis = args[1].operator int();\n      }\n      param.is_ascend = args[2].operator bool();\n      if (args[3].type_code() == kNull) {\n        param.dtype = mshadow::kFloat32;\n      } else {\n        param.dtype = String2MXNetTypeWithBool(args[3].operator std::string());\n      }\n\n      attrs.parsed = std::move(param);\n      attrs.op     = op;\n\n      int num_inputs    = 1;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n\n      int num_outputs = 0;\n      SetAttrDict<op::ArgSortParam>(&attrs);\n      auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret           = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_pad_op.cc",
    "content": "\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_pad_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_pad_op.cc\n */\n#include <dmlc/optional.h>\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_pad_op-inl.h\"\n\nnamespace mxnet {\n\ninline int String2MXNetPadType(const std::string& s) {\n  using namespace op;\n  if (s == \"constant\") {\n    return pad_enum::kConstant;\n  } else if (s == \"edge\") {\n    return pad_enum::kEdge;\n  } else if (s == \"reflect\") {\n    return pad_enum::kReflect;\n  } else if (s == \"symmetric\") {\n    return pad_enum::kSymmetric;\n  } else if (s == \"maximum\") {\n    return pad_enum::kMaximum;\n  } else if (s == \"minimum\") {\n    return pad_enum::kMinimum;\n  } else {\n    LOG(FATAL) << \"unknown type \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\ninline Tuple<Tuple<int>> BroadcastPadWidth(int ndim, runtime::ADT adt) {\n  std::vector<mxnet::Tuple<int>> temp;\n  int adt_size = adt.size();\n  if (const runtime::IntegerObj* pad = adt[0].as<runtime::IntegerObj>()) {\n    if (adt_size == 1) {\n      int pad_width = static_cast<int>(pad->value);\n      if (ndim == 1) {\n        temp.emplace_back(mxnet::Tuple<int>({pad_width}));\n        temp.emplace_back(mxnet::Tuple<int>({pad_width}));\n      } else {\n        for (int dim = 0; dim < ndim; dim++) {\n          temp.emplace_back(mxnet::Tuple<int>({pad_width, pad_width}));\n        }\n      }\n    } else {\n      CHECK_EQ(adt_size, 2) << \"Invalid Input pad_width\";\n      int pad_before = static_cast<int>(pad->value);\n      int pad_after  = static_cast<int>(Downcast<runtime::Integer, ObjectRef>(adt[1])->value);\n      if (ndim == 1) {\n        temp.emplace_back(mxnet::Tuple<int>({pad_before}));\n        temp.emplace_back(mxnet::Tuple<int>({pad_after}));\n      } else {\n        for (int dim = 0; dim < ndim; dim++) {\n          temp.emplace_back(mxnet::Tuple<int>({pad_before, pad_after}));\n        }\n      }\n    }\n  } else {\n    if (adt_size == 1) {\n      if (ndim == 1) {\n        runtime::ADT pad_adt = Downcast<runtime::ADT, ObjectRef>(adt[0]);\n        int pad_before = static_cast<int>(Downcast<runtime::Integer, ObjectRef>(pad_adt[0])->value);\n        int pad_after  = static_cast<int>(Downcast<runtime::Integer, ObjectRef>(pad_adt[1])->value);\n        temp.emplace_back(mxnet::Tuple<int>({pad_before}));\n        temp.emplace_back(mxnet::Tuple<int>({pad_after}));\n      } else {\n        for (int dim = 0; dim < ndim; dim++) {\n          temp.emplace_back(mxnet::Tuple<int>(adt[0]));\n        }\n      }\n    } else {\n      CHECK_EQ(adt_size, ndim) << \"Invalid Input pad_width\";\n      for (int dim = 0; dim < ndim; dim++) {\n        temp.emplace_back(mxnet::Tuple<int>(adt[dim]));\n      }\n    }\n  }\n  return Tuple<Tuple<int>>(temp.begin(), temp.end());\n}\n\nMXNET_REGISTER_API(\"_npi.pad\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_pad\");\n  nnvm::NodeAttrs attrs;\n  op::NumpyPadParam param = {};\n  NDArray* inputs[]    = {args[0].operator mxnet::NDArray*()};\n  mxnet::TShape ashape = inputs[0]->shape();\n  int ndim             = ashape.ndim();\n  ADT adt              = Downcast<ADT, ObjectRef>(args[1].operator ObjectRef());\n  // broadcast pad_width to (ndim, 2)\n  param.pad_width = BroadcastPadWidth(ndim, adt);\n  param.mode      = String2MXNetPadType(args[2].operator std::string());\n  if (args[3].type_code() != kNull) {\n    param.constant_values = args[3].operator double();\n  }\n  if (args[4].type_code() != kNull) {\n    param.reflect_type = args[4].operator std::string();\n  }\n  attrs.op     = op;\n  attrs.parsed = param;\n  SetAttrDict<op::NumpyPadParam>(&attrs);\n  int num_inputs  = 1;\n  int num_outputs = 0;\n  auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret            = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_percentile_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_percentile_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_percentile_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_percentile_op-inl.h\"\n\nnamespace mxnet {\n\ninline int String2MXNetPercentileType(const std::string& s) {\n  using namespace op;\n  if (s == \"linear\") {\n    return percentile_enum::kLinear;\n  } else if (s == \"lower\") {\n    return percentile_enum::kLower;\n  } else if (s == \"higher\") {\n    return percentile_enum::kHigher;\n  } else if (s == \"midpoint\") {\n    return percentile_enum::kMidpoint;\n  } else if (s == \"nearest\") {\n    return percentile_enum::kNearest;\n  } else {\n    LOG(FATAL) << \"unknown type \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\nMXNET_REGISTER_API(\"_npi.percentile\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_percentile\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyPercentileParam param = {};\n\n      NDArray* out      = args[5].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      if (args[2].type_code() == kNull) {\n        param.axis = dmlc::nullopt;\n      } else if (args[2].type_code() == kDLInt) {\n        param.axis = Tuple<int>(1, args[2].operator int64_t());\n      } else {\n        param.axis = Tuple<int>(args[2].operator ObjectRef());\n      }\n      param.interpolation = String2MXNetPercentileType(args[3].operator std::string());\n      param.keepdims      = args[4].operator bool();\n      if (args[1].type_code() == kDLInt || args[1].type_code() == kDLFloat) {\n        param.q_scalar    = args[1].operator double();\n        NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n        int num_inputs    = 1;\n        attrs.parsed      = param;\n        attrs.op          = op;\n        SetAttrDict<op::NumpyPercentileParam>(&attrs);\n        auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n        if (out) {\n          *ret = PythonArg(5);\n        } else {\n          *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n        }\n      } else {\n        param.q_scalar    = dmlc::nullopt;\n        NDArray* inputs[] = {args[0].operator mxnet::NDArray*(),\n                             args[1].operator mxnet::NDArray*()};\n        int num_inputs    = 2;\n        attrs.parsed      = param;\n        attrs.op          = op;\n        SetAttrDict<op::NumpyPercentileParam>(&attrs);\n        auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n        if (out) {\n          *ret = PythonArg(5);\n        } else {\n          *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n        }\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_polynomial_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_polynomial_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_polynomial_op.cc\n */\n#include <mxnet/api_registry.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_polynomial_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.polyval\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_polyval\");\n      nnvm::NodeAttrs attrs;\n      attrs.op = op;\n\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};\n      int num_inputs    = 2;\n      int num_outputs   = 0;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_repeat_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_repeat_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_repeat_op.cc\n */\n#include <mxnet/api_registry.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_repeat_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.repeats\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_repeats\");\n      nnvm::NodeAttrs attrs;\n      op::RepeatsParam param = {};\n      param.repeats = Tuple<int>(args[1].operator ObjectRef());\n      if (args[2].type_code() == kNull) {\n        param.axis = dmlc::optional<int>();\n      } else {\n        param.axis = args[2].operator int64_t();\n      }\n      int num_inputs  = 1;\n      int num_outputs = 0;\n      attrs.parsed    = param;\n      attrs.op        = op;\n      SetAttrDict<op::RepeatsParam>(&attrs);\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_tensordot_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_tensordot_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_tensordot_op.cc\n */\n#include <mxnet/api_registry.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_tensordot_op-inl.h\"\n\nnamespace mxnet {\n\ninline static void _npi_tensordot_int_axes(runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_tensordot_int_axes\");\n  op::TensordotIntAxesParam param = {};\n  nnvm::NodeAttrs attrs;\n  param.axes = args[2].operator int();\n  attrs.op   = op;\n  // we directly copy TensordotIntAxesParam, which is trivially-copyable\n  attrs.parsed = param;\n  SetAttrDict<op::TensordotIntAxesParam>(&attrs);\n  int num_outputs   = 0;\n  int num_inputs    = 2;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n}\n\ninline static void _npi_tensordot(runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_tensordot\");\n  op::TensordotParam param = {};\n  nnvm::NodeAttrs attrs;\n  ADT adt = Downcast<ADT, ObjectRef>(args[2].operator ObjectRef());\n  if (const IntegerObj* lop = adt[0].as<IntegerObj>()) {\n    // axes is a tuple of int, like axes=(0, 1)\n    param.a_axes_summed = Tuple<int>(1, lop->value);\n    param.b_axes_summed = Tuple<int>(1, Downcast<Integer, ObjectRef>(adt[1])->value);\n  } else {\n    // axes is a tuple of tuples of int, like axes=((0, 1), (1, 0))\n    param.a_axes_summed = Tuple<int>(adt[0]);\n    param.b_axes_summed = Tuple<int>(adt[1]);\n  }\n  attrs.op     = op;\n  attrs.parsed = param;\n  SetAttrDict<op::TensordotParam>(&attrs);\n  int num_outputs   = 0;\n  int num_inputs    = 2;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n}\n\nMXNET_REGISTER_API(\"_npi.tensordot\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      if (args[2].type_code() == kDLInt) {\n        _npi_tensordot_int_axes(args, ret);\n      } else {\n        _npi_tensordot(args, ret);\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_trace_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_trace_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_trace_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_trace_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.trace\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_trace\");\n  nnvm::NodeAttrs attrs;\n  op::NumpyTraceParam param = {};\n  param.offset = args[1].operator int64_t();\n  param.axis1  = args[2].operator int64_t();\n  param.axis2  = args[3].operator int64_t();\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::NumpyTraceParam>(&attrs);\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  int num_inputs    = 1;\n  NDArray* out      = args[4].operator mxnet::NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (out) {\n    *ret = PythonArg(4);\n  } else {\n    *ret = ndoutputs[0];\n  }\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_tri_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_tri_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_diff.cc\n */\n#include <mxnet/api_registry.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_tri_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.tri\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_tri\");\n  nnvm::NodeAttrs attrs;\n  op::TriParam param = {};\n  param.N = args[0].operator nnvm::dim_t();\n  if (args[1].type_code() == kNull) {\n    param.M = dmlc::nullopt;\n  } else {\n    param.M = args[1].operator nnvm::dim_t();\n  }\n  param.k     = args[2].operator int();\n  param.dtype = args[3].type_code() == kNull ?\n                    mshadow::kFloat32 :\n                    String2MXNetTypeWithBool(args[3].operator std::string());\n  if (args[4].type_code() != kNull) {\n    attrs.dict[\"ctx\"] = args[4].operator std::string();\n  }\n\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::TriParam>(&attrs);\n\n  int num_outputs = 0;\n  auto ndoutputs  = Invoke(op, &attrs, 0, nullptr, &num_outputs, nullptr);\n  *ret            = ndoutputs[0];\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_tril_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_tril_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_diff.cc\n */\n#include <mxnet/api_registry.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_tril_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.tril\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_tril\");\n  nnvm::NodeAttrs attrs;\n  op::TrilParam param = {};\n  param.k = args[1].operator int();\n\n  // we directly copy TrilParam, which is trivially-copyable\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::TrilParam>(&attrs);\n\n  int num_outputs   = 0;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  int num_inputs    = 1;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = ndoutputs[0];\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_triu_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_cumsum.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_triu_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_triu_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.triu\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  op::TriuParam param = {};\n  nnvm::NodeAttrs attrs;\n  const nnvm::Op* op = Op::Get(\"_npi_triu\");\n  // inputs\n  param.k           = args[1].operator int();\n  NDArray* inputs[] = {args[0].operator NDArray*()};\n\n  attrs.op     = op;\n  attrs.parsed = param;\n  SetAttrDict<op::TriuParam>(&attrs);\n\n  int num_outputs = 0;\n  auto ndoutputs  = Invoke(op, &attrs, 1, inputs, &num_outputs, nullptr);\n  *ret            = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_unique_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_unique_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_unique_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include <vector>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_unique_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.unique\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_unique\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyUniqueParam param = {};\n      // param\n      param.return_index   = args[1].operator bool();\n      param.return_inverse = args[2].operator bool();\n      param.return_counts  = args[3].operator bool();\n      if (args[4].type_code() == kNull) {\n        param.axis = dmlc::nullopt;\n      } else {\n        param.axis = args[4].operator int();\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyUniqueParam>(&attrs);\n      // inputs\n      int num_inputs    = 1;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      // outputs\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      std::vector<NDArrayHandle> ndarray_handles;\n      ndarray_handles.reserve(num_outputs);\n      for (int i = 0; i < num_outputs; ++i) {\n        ndarray_handles.emplace_back(ndoutputs[i]);\n      }\n      *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_where_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_where_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_where_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_where_op-inl.h\"\n\nnamespace mxnet {\n\ninline static bool isScalar(const runtime::MXNetArgValue& arg) {\n  return arg.type_code() == kDLInt || arg.type_code() == kDLUInt || arg.type_code() == kDLFloat;\n}\n\ninline static void _npi_where(runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_where\");\n  nnvm::NodeAttrs attrs;\n  attrs.op          = op;\n  int num_inputs    = 3;\n  int num_outputs   = 0;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(),\n                       args[1].operator mxnet::NDArray*(),\n                       args[2].operator mxnet::NDArray*()};\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n}\n\ninline static void _npi_where_scalar1(runtime::MXNetArgs args,\n                                      runtime::MXNetRetValue* ret,\n                                      bool isl) {\n  using namespace runtime;\n  nnvm::NodeAttrs attrs;\n  const nnvm::Op* op = isl ? Op::Get(\"_npi_where_lscalar\") : Op::Get(\"_npi_where_rscalar\");\n  op::NumpyWhereScalarParam param = {};\n  param.scalar = isl ? args[1].operator double() : args[2].operator double();\n  attrs.op     = op;\n  attrs.parsed = param;\n  SetAttrDict<op::NumpyWhereScalarParam>(&attrs);\n  int num_inputs    = 2;\n  int num_outputs   = 0;\n  NDArray* inputs[] = {\n      args[0].operator mxnet::NDArray*(),\n      isl ? args[2].operator mxnet::NDArray*() : args[1].operator mxnet::NDArray*()};\n  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret           = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n}\n\ninline static void _npi_where_scalar2(runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_where_scalar2\");\n  op::NumpyWhereScalar2Param param = {};\n  nnvm::NodeAttrs attrs;\n  param.x      = args[1].operator double();\n  param.y      = args[2].operator double();\n  attrs.op     = op;\n  attrs.parsed = param;\n  SetAttrDict<op::NumpyWhereScalar2Param>(&attrs);\n  int num_inputs    = 1;\n  int num_outputs   = 0;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n}\n\nMXNET_REGISTER_API(\"_npi.where\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  if (isScalar(args[1]) && isScalar(args[2])) {\n    _npi_where_scalar2(args, ret);\n  } else if (!isScalar(args[1]) && !isScalar(args[2])) {\n    _npi_where(args, ret);\n  } else {\n    _npi_where_scalar1(args, ret, isScalar(args[1]));\n  }\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/np_window_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_window_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_window_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/np_window_op.h\"\n#include \"../../../common/utils.h\"\n\nnamespace mxnet {\n\ninline static void SetNumpyWindowsParam(runtime::MXNetArgs args,\n                                        runtime::MXNetRetValue* ret,\n                                        const nnvm::Op* op) {\n  using namespace runtime;\n  nnvm::NodeAttrs attrs;\n  op::NumpyWindowsParam param = {};\n  if (args[0].type_code() == kNull) {\n    param.M = dmlc::nullopt;\n  } else {\n    param.M = args[0].operator nnvm::dim_t();\n  }\n  if (args[1].type_code() == kNull) {\n    param.dtype = mxnet::common::GetDefaultDtype();\n  } else {\n    param.dtype = String2MXNetTypeWithBool(args[1].operator std::string());\n  }\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::NumpyWindowsParam>(&attrs);\n  if (args[2].type_code() != kNull) {\n    attrs.dict[\"ctx\"] = args[2].operator std::string();\n  }\n  int num_outputs = 0;\n  auto ndoutputs  = Invoke(op, &attrs, 0, nullptr, &num_outputs, nullptr);\n  *ret            = ndoutputs[0];\n}\n\nMXNET_REGISTER_API(\"_npi.blackman\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_blackman\");\n      SetNumpyWindowsParam(args, ret, op);\n    });\n\nMXNET_REGISTER_API(\"_npi.hamming\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_hamming\");\n      SetNumpyWindowsParam(args, ret, op);\n    });\n\nMXNET_REGISTER_API(\"_npi.hanning\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_hanning\");\n      SetNumpyWindowsParam(args, ret, op);\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/random/np_choice_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_choice_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/np_choice_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/random/np_choice_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.choice\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_choice\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyChoiceParam param = {};\n\n      NDArray* inputs[2];\n      int num_inputs = 0;\n\n      if (args[0].type_code() == kDLInt) {\n        param.a = args[0].operator int();\n      } else if (args[0].type_code() == kNDArrayHandle) {\n        param.a            = dmlc::nullopt;\n        inputs[num_inputs] = args[0].operator mxnet::NDArray*();\n        num_inputs++;\n      }\n\n      if (args[1].type_code() == kNull) {\n        param.size = dmlc::nullopt;\n      } else {\n        if (args[1].type_code() == kDLInt) {\n          param.size = mxnet::Tuple<int64_t>(1, args[1].operator int64_t());\n        } else {\n          param.size = mxnet::Tuple<int64_t>(args[1].operator ObjectRef());\n        }\n      }\n\n      if (args[2].type_code() == kNull) {\n        param.replace = true;\n      } else {\n        param.replace = args[2].operator bool();\n      }\n\n      if (args[3].type_code() == kNull) {\n        param.weighted = false;\n      } else if (args[0].type_code() == kNDArrayHandle) {\n        param.weighted     = true;\n        inputs[num_inputs] = args[3].operator mxnet::NDArray*();\n        num_inputs++;\n      }\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      if (args[4].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[4].operator std::string();\n      }\n      NDArray* out      = args[5].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(5);\n      } else {\n        *ret = ndoutputs[0];\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/random/np_exponential_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_exponential_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/random/np_exponential_op.h\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/random/np_exponential_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.exponential\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_exponential\");\n      op::NumpyExponentialParam param = {};\n      nnvm::NodeAttrs attrs;\n      attrs.op = op;\n      if (args[1].type_code() == kDLInt) {\n        param.size = Tuple<index_t>(1, args[1].operator int64_t());\n      } else if (args[1].type_code() == kNull) {\n        param.size = dmlc::nullopt;\n      } else {\n        param.size = Tuple<index_t>(args[1].operator ObjectRef());\n      }\n      if (args[2].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[2].operator std::string();\n      }\n      NDArray* out      = args[3].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      NDArray* inputs[1];\n      int num_inputs = 0;\n      if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {\n        param.scale = args[0].operator double();\n        num_inputs  = 0;\n      } else {\n        param.scale = dmlc::nullopt;\n        inputs[0]   = args[0].operator mxnet::NDArray*();\n        num_inputs  = 1;\n      }\n      attrs.parsed = param;\n      SetAttrDict<op::NumpyExponentialParam>(&attrs);\n      auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(3);\n      } else {\n        *ret = ndoutputs[0];\n      }\n    });\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/random/np_laplace_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_laplace_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/random/np_laplace_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/random/np_laplace_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.laplace\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_laplace\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyLaplaceParam param = {};\n\n      NDArray** inputs = new NDArray*[2]();\n      int num_inputs   = 0;\n\n      if (args[0].type_code() == kNull) {\n        param.loc = dmlc::nullopt;\n      } else if (args[0].type_code() == kNDArrayHandle) {\n        param.loc          = dmlc::nullopt;\n        inputs[num_inputs] = args[0].operator mxnet::NDArray*();\n        num_inputs++;\n      } else {\n        param.loc = args[0].operator double();  // convert arg to T\n      }\n\n      if (args[1].type_code() == kNull) {\n        param.scale = dmlc::nullopt;\n      } else if (args[1].type_code() == kNDArrayHandle) {\n        param.scale        = dmlc::nullopt;\n        inputs[num_inputs] = args[1].operator mxnet::NDArray*();\n        num_inputs++;\n      } else {\n        param.scale = args[1].operator double();  // convert arg to T\n      }\n\n      if (args[2].type_code() == kNull) {\n        param.size = dmlc::nullopt;\n      } else {\n        if (args[2].type_code() == kDLInt) {\n          param.size = mxnet::Tuple<index_t>(1, args[2].operator int64_t());\n        } else {\n          param.size = mxnet::Tuple<index_t>(args[2].operator ObjectRef());\n        }\n      }\n\n      if (args[3].type_code() == kNull) {\n        param.dtype = mshadow::kFloat32;\n      } else {\n        param.dtype = String2MXNetTypeWithBool(args[3].operator std::string());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::NumpyLaplaceParam>(&attrs);\n      if (args[4].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[4].operator std::string();\n      }\n\n      inputs = inputs == nullptr ? nullptr : inputs;\n\n      NDArray* out      = args[5].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(5);\n      } else {\n        *ret = ndoutputs[0];\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/random/np_location_scale_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_location_scale_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/random/np_location_scale_op.h\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include <vector>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/random/np_location_scale_op.h\"\n\nnamespace mxnet {\n\nint scalar_number(const runtime::MXNetArgs& args) {\n  int result = 0;\n  if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt)\n    result++;\n  if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt)\n    result++;\n  return result;\n}\n\nMXNET_REGISTER_API(\"_npi.gumbel\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_gumbel\");\n      op::NumpyLocationScaleParam param = {};\n      nnvm::NodeAttrs attrs;\n      attrs.op = op;\n      if (args[2].type_code() == kDLInt) {\n        param.size = Tuple<index_t>(1, args[2].operator int64_t());\n      } else if (args[2].type_code() == kNull) {\n        param.size = dmlc::optional<mxnet::Tuple<index_t>>();\n      } else {\n        param.size = Tuple<index_t>(args[2].operator ObjectRef());\n      }\n      if (args[3].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[3].operator std::string();\n      }\n      NDArray* out      = args[4].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      int scalar        = scalar_number(args);\n      std::vector<NDArray*> inputs;\n      int num_inputs = 0;\n      if (scalar == 2) {\n        param.loc   = args[0].operator double();\n        param.scale = args[1].operator double();\n      } else if (scalar == 0) {\n        param.loc   = dmlc::nullopt;\n        param.scale = dmlc::nullopt;\n        inputs.push_back(args[0].operator mxnet::NDArray*());\n        inputs.push_back(args[1].operator mxnet::NDArray*());\n        num_inputs = 2;\n      } else {\n        if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {\n          param.loc   = args[0].operator double();\n          param.scale = dmlc::nullopt;\n          inputs.push_back(args[1].operator mxnet::NDArray*());\n        } else {\n          param.loc   = dmlc::nullopt;\n          param.scale = args[1].operator double();\n          inputs.push_back(args[0].operator mxnet::NDArray*());\n        }\n        num_inputs = 1;\n      }\n      attrs.parsed = param;\n      SetAttrDict<op::NumpyLocationScaleParam>(&attrs);\n      auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(4);\n      } else {\n        *ret = ndoutputs[0];\n      }\n    });\n\nMXNET_REGISTER_API(\"_npi.logistic\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_logistic\");\n      op::NumpyLocationScaleParam param = {};\n      nnvm::NodeAttrs attrs;\n      attrs.op = op;\n      if (args[2].type_code() == kDLInt) {\n        param.size = Tuple<index_t>(1, args[2].operator int64_t());\n      } else if (args[2].type_code() == kNull) {\n        param.size = dmlc::nullopt;\n      } else {\n        param.size = Tuple<index_t>(args[2].operator ObjectRef());\n      }\n      if (args[3].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[3].operator std::string();\n      }\n      NDArray* out      = args[4].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      int scalar        = scalar_number(args);\n      std::vector<NDArray*> inputs;\n      int num_inputs = 0;\n      if (scalar == 2) {\n        param.loc   = args[0].operator double();\n        param.scale = args[1].operator double();\n      } else if (scalar == 0) {\n        param.loc   = dmlc::nullopt;\n        param.scale = dmlc::nullopt;\n        inputs.push_back(args[0].operator mxnet::NDArray*());\n        inputs.push_back(args[1].operator mxnet::NDArray*());\n        num_inputs = 2;\n      } else {\n        if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {\n          param.loc   = args[0].operator double();\n          param.scale = dmlc::nullopt;\n          inputs.push_back(args[1].operator mxnet::NDArray*());\n        } else {\n          param.loc   = dmlc::nullopt;\n          param.scale = args[1].operator double();\n          inputs.push_back(args[0].operator mxnet::NDArray*());\n        }\n        num_inputs = 1;\n      }\n      attrs.parsed = param;\n      SetAttrDict<op::NumpyLocationScaleParam>(&attrs);\n      auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(4);\n      } else {\n        *ret = ndoutputs[0];\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/random/np_multinomial_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_multinomial_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/random/np_multinomial_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include <vector>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/random/np_multinomial_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.multinomial\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_multinomial\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyMultinomialParam param = {};\n      NDArray** inputs = new NDArray*[1]();\n      int num_inputs   = 0;\n\n      // parse int\n      param.n = args[0].operator int();\n\n      // parse pvals\n      if (args[1].type_code() == kNull) {\n        param.pvals = dmlc::nullopt;\n      } else if (args[1].type_code() == kNDArrayHandle) {\n        param.pvals = dmlc::nullopt;\n        inputs[0]   = args[1].operator mxnet::NDArray*();\n        num_inputs  = 1;\n      } else {\n        param.pvals = Obj2Tuple<double, Float>(args[1].operator ObjectRef());\n      }\n\n      // parse size\n      if (args[2].type_code() == kNull) {\n        param.size = dmlc::nullopt;\n      } else {\n        if (args[2].type_code() == kDLInt) {\n          param.size = mxnet::Tuple<index_t>(1, args[2].operator int64_t());\n        } else {\n          param.size = mxnet::Tuple<index_t>(args[2].operator ObjectRef());\n        }\n      }\n\n      attrs.parsed = std::move(param);\n      attrs.op     = op;\n      SetAttrDict<op::NumpyMultinomialParam>(&attrs);\n      inputs          = num_inputs == 0 ? nullptr : inputs;\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/random/np_pareto_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_pareto_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/random/np_pareto_op.h\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/random/np_pareto_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.pareto\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_pareto\");\n      op::NumpyParetoParam param = {};\n      nnvm::NodeAttrs attrs;\n      attrs.op = op;\n      if (args[1].type_code() == kDLInt) {\n        param.size = Tuple<index_t>(1, args[1].operator int64_t());\n      } else if (args[1].type_code() == kNull) {\n        param.size = dmlc::nullopt;\n      } else {\n        param.size = Tuple<index_t>(args[1].operator ObjectRef());\n      }\n      if (args[2].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[2].operator std::string();\n      }\n      NDArray* out      = args[3].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      NDArray* inputs[1];\n      int num_inputs = 0;\n      if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {\n        param.a    = args[0].operator double();\n        num_inputs = 0;\n      } else {\n        param.a    = dmlc::nullopt;\n        inputs[0]  = args[0].operator mxnet::NDArray*();\n        num_inputs = 1;\n      }\n      attrs.parsed = param;\n      SetAttrDict<op::NumpyParetoParam>(&attrs);\n      auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(3);\n      } else {\n        *ret = ndoutputs[0];\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/random/np_power_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_power_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/random/np_power_op.h\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/random/np_power_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.powerd\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_powerd\");\n      op::NumpyPowerParam param = {};\n      nnvm::NodeAttrs attrs;\n      attrs.op = op;\n      if (args[1].type_code() == kDLInt) {\n        param.size = Tuple<index_t>(1, args[1].operator int64_t());\n      } else if (args[1].type_code() == kNull) {\n        param.size = dmlc::nullopt;\n      } else {\n        param.size = Tuple<index_t>(args[1].operator ObjectRef());\n      }\n      if (args[2].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[2].operator std::string();\n      }\n      NDArray* out      = args[3].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      NDArray* inputs[1];\n      int num_inputs = 0;\n      if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {\n        param.a    = args[0].operator double();\n        num_inputs = 0;\n      } else {\n        param.a    = dmlc::nullopt;\n        inputs[0]  = args[0].operator mxnet::NDArray*();\n        num_inputs = 1;\n      }\n      attrs.parsed = param;\n      SetAttrDict<op::NumpyPowerParam>(&attrs);\n      auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(3);\n      } else {\n        *ret = ndoutputs[0];\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/random/np_rayleigh_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_rayleigh_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/random/np_rayleigh_op.h\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/random/np_rayleigh_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.rayleigh\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_rayleigh\");\n      op::NumpyRayleighParam param = {};\n      nnvm::NodeAttrs attrs;\n      attrs.op = op;\n      if (args[1].type_code() == kDLInt) {\n        param.size = Tuple<index_t>(1, args[1].operator int64_t());\n      } else if (args[1].type_code() == kNull) {\n        param.size = dmlc::nullopt;\n      } else {\n        param.size = Tuple<index_t>(args[1].operator ObjectRef());\n      }\n      if (args[2].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[2].operator std::string();\n      }\n      NDArray* out      = args[3].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      NDArray* inputs[1];\n      int num_inputs = 0;\n      if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {\n        param.scale = args[0].operator double();\n        num_inputs  = 0;\n      } else {\n        param.scale = dmlc::nullopt;\n        inputs[0]   = args[0].operator mxnet::NDArray*();\n        num_inputs  = 1;\n      }\n      attrs.parsed = param;\n      SetAttrDict<op::NumpyRayleighParam>(&attrs);\n      auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(3);\n      } else {\n        *ret = ndoutputs[0];\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy/random/np_weibull_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_weibull_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/random/np_weibull_op.h\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../../utils.h\"\n#include \"../../../../operator/numpy/random/np_weibull_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.weibull\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_weibull\");\n      op::NumpyWeibullParam param = {};\n      nnvm::NodeAttrs attrs;\n      attrs.op = op;\n      if (args[1].type_code() == kDLInt) {\n        param.size = Tuple<index_t>(1, args[1].operator int64_t());\n      } else if (args[1].type_code() == kNull) {\n        param.size = dmlc::nullopt;\n      } else {\n        param.size = Tuple<index_t>(args[1].operator ObjectRef());\n      }\n      if (args[2].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[2].operator std::string();\n      }\n      NDArray* out      = args[3].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      NDArray* inputs[1];\n      int num_inputs = 0;\n      if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {\n        param.a    = args[0].operator double();\n        num_inputs = 0;\n      } else {\n        param.a    = dmlc::nullopt;\n        inputs[0]  = args[0].operator mxnet::NDArray*();\n        num_inputs = 1;\n      }\n      attrs.parsed = param;\n      SetAttrDict<op::NumpyWeibullParam>(&attrs);\n      auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(3);\n      } else {\n        *ret = ndoutputs[0];\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_activation_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_activation_op.cc\n * \\brief Implementation of the API of functions in\n * src/operator/numpy_extension/npx_activation_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/nn/activation-inl.h\"\n\nnamespace mxnet {\n\ninline int String2MXNetActType(const std::string& s) {\n  using namespace op;\n  if (s == \"relu\") {\n    return activation::kReLU;\n  } else if (s == \"sigmoid\") {\n    return activation::kSigmoid;\n  } else if (s == \"log_sigmoid\") {\n    return activation::kLogSigmoid;\n  } else if (s == \"mish\") {\n    return activation::kMish;\n  } else if (s == \"tanh\") {\n    return activation::kTanh;\n  } else if (s == \"softrelu\") {\n    return activation::kSoftReLU;\n  } else if (s == \"softsign\") {\n    return activation::kSoftSign;\n  } else {\n    LOG(FATAL) << \"unknown activation type \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\nMXNET_REGISTER_API(\"_npx.activation\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_activation\");\n      op::ActivationParam param = {};\n      // act_type\n      param.act_type = String2MXNetActType(args[1].operator std::string());\n      attrs.parsed   = param;\n      attrs.op       = op;\n      SetAttrDict<op::ActivationParam>(&attrs);\n      // inputs\n      NDArray* inputs[] = {args[0].operator NDArray*()};\n      int num_inputs    = 1;\n      int num_outputs   = 0;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret              = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_arange_like_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_arange_like_op.cc\n * \\brief Implementation of the API of functions in\n * src/operator/numpy_extension/npx_arange_like_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/tensor/init_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npx.arange_like\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_arange_like\");\n      op::RangeLikeParam param = {};\n      // inputs\n      int num_inputs    = 1;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      // start\n      if (args[1].type_code() == kNull) {\n        param.start = 0.0;\n      } else {\n        param.start = args[1].operator double();\n      }\n      // step\n      if (args[2].type_code() == kNull) {\n        param.step = 1.0;\n      } else {\n        param.step = args[2].operator double();\n      }\n      // repeat\n      if (args[3].type_code() == kNull) {\n        param.repeat = 1;\n      } else {\n        param.repeat = args[3].operator int();\n      }\n      // ctx\n      if (args[4].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[4].operator std::string();\n      }\n      // axis\n      if (args[5].type_code() == kNull) {\n        param.axis = dmlc::nullopt;\n      } else {\n        param.axis = args[5].operator int();\n      }\n      attrs.op     = op;\n      attrs.parsed = param;\n      SetAttrDict<op::RangeLikeParam>(&attrs);\n\n      // outputs\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret            = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_batch_dot_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_batch_dot_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy_extension/npx_batch_dot_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/tensor/dot-inl.h\"\n\nnamespace mxnet {\n\ninline int String2ForwardStype(const std::string& s) {\n  using namespace op;\n  if (s == \"default\") {\n    return kDefaultStorage;\n  } else if (s == \"row_sparse\") {\n    return kRowSparseStorage;\n  } else if (s == \"csr\") {\n    return kCSRStorage;\n  } else {\n    LOG(FATAL) << \"unknown forward storage type \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\nMXNET_REGISTER_API(\"_npx.batch_dot\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_batch_dot\");\n      op::DotParam param = {};\n      // inputs\n      int num_inputs = 2;\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      // transpose_a\n      if (args[2].type_code() == kNull) {\n        param.transpose_a = false;\n      } else {\n        param.transpose_a = args[2].operator bool();\n      }\n      // transpose_b\n      if (args[3].type_code() == kNull) {\n        param.transpose_b = false;\n      } else {\n        param.transpose_b = args[3].operator bool();\n      }\n      // forward_stype\n      if (args[4].type_code() == kNull) {\n        param.forward_stype = dmlc::nullopt;\n      } else {\n        param.forward_stype = String2ForwardStype(args[4].operator std::string());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::DotParam>(&attrs);\n      int num_outputs = 1;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_batch_norm_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_batch_norm_op.cc\n * \\brief Implementation of the API of functions in\n * src/operator/numpy_extension/npx_batch_norm_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/nn/batch_norm-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npx.batch_norm\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_batch_norm\");\n      op::BatchNormParam param = {};\n      // eps\n      param.eps = args[5].operator double();\n      // momentum\n      param.momentum = args[6].operator double();\n      // fix_gamma\n      param.fix_gamma = args[7].operator bool();\n      // use_global_stats\n      param.use_global_stats = args[8].operator bool();\n      // output_mean_var\n      param.output_mean_var = args[9].operator bool();\n      // axis\n      param.axis = args[10].operator int();\n      // cudnn_off\n      param.cudnn_off = args[11].operator bool();\n      // min_calib_range\n      if (args[12].type_code() == kDLFloat || args[12].type_code() == kDLInt) {\n        param.min_calib_range = args[12].operator double();\n      } else {\n        param.min_calib_range = dmlc::nullopt;\n      }\n      // max_calib_range\n      if (args[13].type_code() == kDLFloat || args[13].type_code() == kDLInt) {\n        param.max_calib_range = args[13].operator double();\n      } else {\n        param.max_calib_range = dmlc::nullopt;\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::BatchNormParam>(&attrs);\n      // inputs\n      int num_inputs = 5;\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      if (num_outputs == 1) {\n        *ret = ndoutputs[0];\n      } else {\n        std::vector<NDArrayHandle> ndarray_handles;\n        ndarray_handles.reserve(num_outputs);\n        for (int i = 0; i < num_outputs; ++i) {\n          ndarray_handles.emplace_back(ndoutputs[i]);\n        }\n        *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_broadcast_like_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_broadcast_like_op.cc\n * \\brief Implementation of the API of functions in\n * src/operator/numpy_extension/npx_broadcast_like_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/tensor/broadcast_reduce_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npx.broadcast_like\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_broadcast_like\");\n      op::BroadcastLikeParam param = {};\n      // inputs\n      int num_inputs = 2;\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      // lhs_axes\n      if (args[2].type_code() == kNull) {\n        param.lhs_axes = dmlc::optional<mxnet::TShape>();\n      } else if (args[2].type_code() == kDLInt) {\n        param.lhs_axes = TShape(1, args[2].operator int64_t());\n      } else {\n        param.lhs_axes = mxnet::TShape(args[2].operator ObjectRef());\n      }\n      // rhs_axes\n      if (args[3].type_code() == kNull) {\n        param.rhs_axes = dmlc::optional<mxnet::TShape>();\n      } else if (args[3].type_code() == kDLInt) {\n        param.rhs_axes = TShape(1, args[3].operator int64_t());\n      } else {\n        param.rhs_axes = mxnet::TShape(args[3].operator ObjectRef());\n      }\n\n      attrs.op     = op;\n      attrs.parsed = param;\n      SetAttrDict<op::BroadcastLikeParam>(&attrs);\n\n      // outputs\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      *ret            = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_control_flow_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_control_flow_op.cc\n * \\brief Implementation of the API of functions in\n * src/operator/numpy_extension/npx_control_flow_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include <mxnet/operator.h>\n#include \"../utils.h\"\n#include \"../../../operator/npx_control_flow.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npx.foreach\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_foreach\");\n      op::NPXForeachParam param = {};\n      int args_size  = args.size();\n      int num_inputs = args_size - 7;\n      // inputs\n      nnvm::Symbol* sym = static_cast<nnvm::Symbol*>(args[0].value().v_handle);\n      std::vector<std::shared_ptr<nnvm::Symbol> > subgraphs;\n      subgraphs.push_back(std::make_shared<nnvm::Symbol>(*sym));\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 1; i < num_inputs + 1; ++i) {\n        inputs.push_back(static_cast<mxnet::NDArray*>(args[i]));\n      }\n\n      param.num_args     = num_inputs;\n      param.num_outputs  = args[1 + num_inputs].operator int();\n      param.num_out_data = args[2 + num_inputs].operator int();\n      if (args[3 + num_inputs].type_code() == kDLInt) {\n        param.in_state_locs = mxnet::Tuple<int64_t>(1, args[3 + num_inputs].operator int64_t());\n      } else {\n        param.in_state_locs = mxnet::Tuple<int64_t>(args[3 + num_inputs].operator ObjectRef());\n      }\n      if (args[4 + num_inputs].type_code() == kDLInt) {\n        param.in_data_locs = mxnet::Tuple<int64_t>(1, args[4 + num_inputs].operator int64_t());\n      } else {\n        param.in_data_locs = mxnet::Tuple<int64_t>(args[4 + num_inputs].operator ObjectRef());\n      }\n      if (args[5 + num_inputs].type_code() == kDLInt) {\n        param.remain_locs = mxnet::Tuple<int64_t>(1, args[5 + num_inputs].operator int64_t());\n      } else {\n        param.remain_locs = mxnet::Tuple<int64_t>(args[5 + num_inputs].operator ObjectRef());\n      }\n      if (args[6 + num_inputs].type_code() == kDLInt) {\n        param.in_state_index = mxnet::Tuple<int64_t>(1, args[6 + num_inputs].operator int64_t());\n      } else {\n        param.in_state_index = mxnet::Tuple<int64_t>(args[6 + num_inputs].operator ObjectRef());\n      }\n      attrs.parsed    = param;\n      attrs.op        = op;\n      attrs.subgraphs = subgraphs;\n      SetAttrDict<op::NPXForeachParam>(&attrs);\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      if (num_outputs == 1) {\n        *ret = ndoutputs[0];\n      } else {\n        std::vector<NDArrayHandle> ndarray_handles;\n        ndarray_handles.reserve(num_outputs);\n        for (int i = 0; i < num_outputs; ++i) {\n          ndarray_handles.emplace_back(ndoutputs[i]);\n        }\n        *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n      }\n    });\n\nMXNET_REGISTER_API(\"_npx.while_loop\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_while_loop\");\n      op::NPXWhileLoopParam param = {};\n      int args_size  = args.size();\n      int num_inputs = args_size - 8;\n      // inputs\n      std::vector<std::shared_ptr<nnvm::Symbol> > subgraphs;\n      subgraphs.reserve(2);\n      for (int i = 0; i < 2; i++) {\n        nnvm::Symbol* sym = static_cast<nnvm::Symbol*>(args[i].value().v_handle);\n        subgraphs.push_back(std::make_shared<nnvm::Symbol>(*sym));\n      }\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 2; i < num_inputs + 2; ++i) {\n        inputs.push_back(static_cast<mxnet::NDArray*>(args[i]));\n      }\n\n      param.num_args       = num_inputs;\n      param.max_iterations = args[2 + num_inputs].operator int();\n      if (args[3 + num_inputs].type_code() == kDLInt) {\n        param.cond_input_locs = mxnet::Tuple<int64_t>(1, args[3 + num_inputs].operator int64_t());\n      } else {\n        param.cond_input_locs = mxnet::Tuple<int64_t>(args[3 + num_inputs].operator ObjectRef());\n      }\n      if (args[4 + num_inputs].type_code() == kDLInt) {\n        param.func_input_locs = mxnet::Tuple<int64_t>(1, args[4 + num_inputs].operator int64_t());\n      } else {\n        param.func_input_locs = mxnet::Tuple<int64_t>(args[4 + num_inputs].operator ObjectRef());\n      }\n      if (args[5 + num_inputs].type_code() == kDLInt) {\n        param.func_var_locs = mxnet::Tuple<int64_t>(1, args[5 + num_inputs].operator int64_t());\n      } else {\n        param.func_var_locs = mxnet::Tuple<int64_t>(args[5 + num_inputs].operator ObjectRef());\n      }\n      param.num_out_data = args[6 + num_inputs].operator int();\n      param.num_outputs  = args[7 + num_inputs].operator int();\n      attrs.parsed       = param;\n      attrs.op           = op;\n      attrs.subgraphs    = subgraphs;\n      SetAttrDict<op::NPXWhileLoopParam>(&attrs);\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      if (num_outputs == 1) {\n        *ret = ndoutputs[0];\n      } else {\n        std::vector<NDArrayHandle> ndarray_handles;\n        ndarray_handles.reserve(num_outputs);\n        for (int i = 0; i < num_outputs; ++i) {\n          ndarray_handles.emplace_back(ndoutputs[i]);\n        }\n        *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n      }\n    });\n\nMXNET_REGISTER_API(\"_npx.cond\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  nnvm::NodeAttrs attrs;\n  const nnvm::Op* op = Op::Get(\"_npx_cond\");\n  op::NPXCondParam param = {};\n  int args_size  = args.size();\n  int num_inputs = args_size - 7;\n  // inputs\n  std::vector<std::shared_ptr<nnvm::Symbol> > subgraphs;\n  subgraphs.reserve(3);\n  for (int i = 0; i < 3; i++) {\n    nnvm::Symbol* sym = static_cast<nnvm::Symbol*>(args[i].value().v_handle);\n    subgraphs.push_back(std::make_shared<nnvm::Symbol>(*sym));\n  }\n  std::vector<NDArray*> inputs;\n  inputs.reserve(num_inputs);\n  for (int i = 3; i < num_inputs + 3; ++i) {\n    inputs.push_back(static_cast<mxnet::NDArray*>(args[i]));\n  }\n\n  param.num_args = num_inputs;\n  if (args[3 + num_inputs].type_code() == kDLInt) {\n    param.cond_input_locs = mxnet::Tuple<int64_t>(1, args[3 + num_inputs].operator int64_t());\n  } else {\n    param.cond_input_locs = mxnet::Tuple<int64_t>(args[3 + num_inputs].operator ObjectRef());\n  }\n  if (args[4 + num_inputs].type_code() == kDLInt) {\n    param.then_input_locs = mxnet::Tuple<int64_t>(1, args[4 + num_inputs].operator int64_t());\n  } else {\n    param.then_input_locs = mxnet::Tuple<int64_t>(args[4 + num_inputs].operator ObjectRef());\n  }\n  if (args[5 + num_inputs].type_code() == kDLInt) {\n    param.else_input_locs = mxnet::Tuple<int64_t>(1, args[5 + num_inputs].operator int64_t());\n  } else {\n    param.else_input_locs = mxnet::Tuple<int64_t>(args[5 + num_inputs].operator ObjectRef());\n  }\n  param.num_outputs = args[6 + num_inputs].operator int();\n  attrs.parsed      = param;\n  attrs.op          = op;\n  attrs.subgraphs   = subgraphs;\n  SetAttrDict<op::NPXCondParam>(&attrs);\n  int num_outputs = 0;\n  auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n  if (num_outputs == 1) {\n    *ret = ndoutputs[0];\n  } else {\n    std::vector<NDArrayHandle> ndarray_handles;\n    ndarray_handles.reserve(num_outputs);\n    for (int i = 0; i < num_outputs; ++i) {\n      ndarray_handles.emplace_back(ndoutputs[i]);\n    }\n    *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n  }\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_convolution_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_convolution_op.cc\n * \\brief Implementation of the API of functions in\n * src/operator/numpy_extension/npx_convolution_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/nn/convolution-inl.h\"\n\nnamespace mxnet {\n\ninline int String2Layout(const std::string& s) {\n  using namespace op;\n  if (s == \"NCW\") {\n    return mshadow::kNCW;\n  } else if (s == \"NCHW\") {\n    return mshadow::kNCHW;\n  } else if (s == \"NCDHW\") {\n    return mshadow::kNCDHW;\n  } else if (s == \"NHWC\") {\n    return mshadow::kNHWC;\n  } else if (s == \"NDHWC\") {\n    return mshadow::kNDHWC;\n  } else {\n    LOG(FATAL) << \"unknown layout type \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\ninline int String2CudnnTune(const std::string& s) {\n  using namespace op;\n  if (s == \"off\") {\n    return conv::kOff;\n  } else if (s == \"limited_workspace\") {\n    return conv::kLimited;\n  } else if (s == \"fastest\") {\n    return conv::kFastest;\n  } else {\n    LOG(FATAL) << \"unknown cudnn tune type \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\nMXNET_REGISTER_API(\"_npx.convolution\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_convolution\");\n      op::ConvolutionParam param = {};\n      int args_size = args.size();\n      // no_bias\n      if (args[args_size - 4].type_code() == kNull) {\n        param.no_bias = false;\n      } else {\n        param.no_bias = args[args_size - 4].operator bool();\n      }\n      // inputs\n      int num_inputs = param.no_bias ? 2 : 3;\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      // kernel\n      if (args[num_inputs].type_code() == kDLInt) {\n        param.kernel = TShape(1, args[num_inputs].operator int64_t());\n      } else {\n        param.kernel = TShape(args[num_inputs].operator ObjectRef());\n      }\n      // layout\n      if (args[num_inputs + 10].type_code() == kNull) {\n        param.layout = dmlc::nullopt;\n      } else {\n        param.layout = String2Layout(args[num_inputs + 10]);\n      }\n      // Check\n      if (param.kernel.ndim() == 1) {\n        param.layout = param.layout ? param.layout.value() : mshadow::kNCW;\n      } else if (param.kernel.ndim() == 2) {\n        param.layout = param.layout ? param.layout.value() : mshadow::kNCHW;\n      } else {\n        CHECK_EQ(param.kernel.ndim(), 3U) << param.kernel.ndim() << \"D convolution not supported\";\n        param.layout = param.layout ? param.layout.value() : mshadow::kNCDHW;\n      }\n      // stride\n      if (args[num_inputs + 1].type_code() == kNull) {\n        if (param.kernel.ndim() == 1) {\n          param.stride = Shape1(1);\n        } else if (param.kernel.ndim() == 2) {\n          param.stride = Shape2(1, 1);\n        } else {\n          param.stride = Shape3(1, 1, 1);\n        }\n      } else if (args[num_inputs + 1].type_code() == kDLInt) {\n        param.stride = TShape(1, args[num_inputs + 1].operator int64_t());\n      } else {\n        param.stride = TShape(args[num_inputs + 1].operator ObjectRef());\n      }\n      // dilate\n      if (args[num_inputs + 2].type_code() == kNull) {\n        if (param.kernel.ndim() == 1) {\n          param.dilate = Shape1(1);\n        } else if (param.kernel.ndim() == 2) {\n          param.dilate = Shape2(1, 1);\n        } else {\n          param.dilate = Shape3(1, 1, 1);\n        }\n      } else if (args[num_inputs + 2].type_code() == kDLInt) {\n        param.dilate = TShape(1, args[num_inputs + 2].operator int64_t());\n      } else {\n        param.dilate = TShape(args[num_inputs + 2].operator ObjectRef());\n      }\n      // pad\n      if (args[num_inputs + 3].type_code() == kNull) {\n        if (param.kernel.ndim() == 1) {\n          param.pad = Shape1(0);\n        } else if (param.kernel.ndim() == 2) {\n          param.pad = Shape2(0, 0);\n        } else {\n          param.pad = Shape3(0, 0, 0);\n        }\n      } else if (args[num_inputs + 3].type_code() == kDLInt) {\n        param.pad = TShape(1, args[num_inputs + 3].operator int64_t());\n      } else {\n        param.pad = TShape(args[num_inputs + 3].operator ObjectRef());\n      }\n      // num_filter\n      param.num_filter = (uint32_t)(args[num_inputs + 4].operator int());\n      // num_group\n      param.num_group = (uint32_t)(args[num_inputs + 5].operator int());\n      // workspace\n      param.workspace = args[num_inputs + 6].operator int64_t();\n      // cudnn_tune\n      if (args[num_inputs + 8].type_code() == kNull) {\n        param.cudnn_tune = dmlc::nullopt;\n      } else {\n        param.cudnn_tune = String2CudnnTune(args[num_inputs + 8]);\n      }\n      // cudnn_off\n      if (args[num_inputs + 9].type_code() == kNull) {\n        param.cudnn_off = false;\n      } else {\n        param.cudnn_off = args[num_inputs + 9].operator bool();\n      }\n\n      CHECK_EQ(param.kernel.ndim(), param.stride.ndim())\n          << \"Stride must have the same number of dimensions with kernel_size,\"\n          << \"but kernel_size is set to \" << param.kernel << \" while stride is \" << param.stride;\n      CHECK_EQ(param.kernel.ndim(), param.dilate.ndim())\n          << \"Dilate must have the same number of dimensions with kernel_size,\"\n          << \"but kernel_size is set to \" << param.kernel << \" while dilate is \" << param.dilate;\n      CHECK_EQ(param.kernel.ndim(), param.pad.ndim())\n          << \"Padding must have the same number of dimensions with kernel_size,\"\n          << \"but kernel_size is set to \" << param.kernel << \" while padding is \" << param.pad;\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::ConvolutionParam>(&attrs);\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_deconvolution_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_deconvolution_op.cc\n * \\brief Implementation of the API of functions in\n * src/operator/numpy_extension/npx_deconvolution_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/nn/deconvolution-inl.h\"\n\nnamespace mxnet {\n\ninline int String2Layout(const std::string& s) {\n  using namespace op;\n  if (s == \"NCW\") {\n    return mshadow::kNCW;\n  } else if (s == \"NCHW\") {\n    return mshadow::kNCHW;\n  } else if (s == \"NCDHW\") {\n    return mshadow::kNCDHW;\n  } else if (s == \"NHWC\") {\n    return mshadow::kNHWC;\n  } else if (s == \"NDHWC\") {\n    return mshadow::kNDHWC;\n  } else {\n    LOG(FATAL) << \"unknown layout type \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\ninline int String2CudnnTune(const std::string& s) {\n  using namespace op;\n  if (s == \"off\") {\n    return deconv::kOff;\n  } else if (s == \"limited_workspace\") {\n    return deconv::kLimited;\n  } else if (s == \"fastest\") {\n    return deconv::kFastest;\n  } else {\n    LOG(FATAL) << \"unknown cudnn tune type \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\nMXNET_REGISTER_API(\"_npx.deconvolution\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_deconvolution\");\n      op::DeconvolutionParam param = {};\n      int args_size = args.size();\n      // no_bias\n      if (args[args_size - 4].type_code() == kNull) {\n        param.no_bias = false;\n      } else {\n        param.no_bias = args[args_size - 4].operator bool();\n      }\n      // inputs\n      int num_inputs = param.no_bias ? 2 : 3;\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      // kernel\n      if (args[num_inputs].type_code() == kDLInt) {\n        param.kernel = TShape(1, args[num_inputs].operator int64_t());\n      } else {\n        param.kernel = TShape(args[num_inputs].operator ObjectRef());\n      }\n      // layout\n      if (args[num_inputs + 12].type_code() == kNull) {\n        param.layout = dmlc::nullopt;\n      } else {\n        param.layout = String2Layout(args[num_inputs + 12]);\n      }\n      // Check\n      if (param.kernel.ndim() == 1) {\n        param.layout = param.layout ? param.layout.value() : mshadow::kNCW;\n      } else if (param.kernel.ndim() == 2) {\n        param.layout = param.layout ? param.layout.value() : mshadow::kNCHW;\n      } else {\n        CHECK_EQ(param.kernel.ndim(), 3U) << param.kernel.ndim() << \"D convolution not supported\";\n        param.layout = param.layout ? param.layout.value() : mshadow::kNCDHW;\n      }\n      // stride\n      if (args[num_inputs + 1].type_code() == kNull) {\n        if (param.kernel.ndim() == 1) {\n          param.stride = Shape1(1);\n        } else if (param.kernel.ndim() == 2) {\n          param.stride = Shape2(1, 1);\n        } else {\n          param.stride = Shape3(1, 1, 1);\n        }\n      } else if (args[num_inputs + 1].type_code() == kDLInt) {\n        param.stride = TShape(1, args[num_inputs + 1].operator int64_t());\n      } else {\n        param.stride = TShape(args[num_inputs + 1].operator ObjectRef());\n      }\n      // dilate\n      if (args[num_inputs + 2].type_code() == kNull) {\n        if (param.kernel.ndim() == 1) {\n          param.dilate = Shape1(1);\n        } else if (param.kernel.ndim() == 2) {\n          param.dilate = Shape2(1, 1);\n        } else {\n          param.dilate = Shape3(1, 1, 1);\n        }\n      } else if (args[num_inputs + 2].type_code() == kDLInt) {\n        param.dilate = TShape(1, args[num_inputs + 2].operator int64_t());\n      } else {\n        param.dilate = TShape(args[num_inputs + 2].operator ObjectRef());\n      }\n      // pad\n      if (args[num_inputs + 3].type_code() == kNull) {\n        if (param.kernel.ndim() == 1) {\n          param.pad = Shape1(0);\n        } else if (param.kernel.ndim() == 2) {\n          param.pad = Shape2(0, 0);\n        } else {\n          param.pad = Shape3(0, 0, 0);\n        }\n      } else if (args[num_inputs + 3].type_code() == kDLInt) {\n        param.pad = TShape(1, args[num_inputs + 3].operator int64_t());\n      } else {\n        param.pad = TShape(args[num_inputs + 3].operator ObjectRef());\n      }\n      // adj\n      if (args[num_inputs + 4].type_code() == kNull) {\n        if (param.kernel.ndim() == 1) {\n          param.adj = Shape1(0);\n        } else if (param.kernel.ndim() == 2) {\n          param.adj = Shape2(0, 0);\n        } else {\n          param.adj = Shape3(0, 0, 0);\n        }\n      } else if (args[num_inputs + 4].type_code() == kDLInt) {\n        param.adj = TShape(1, args[num_inputs + 4].operator int64_t());\n      } else {\n        param.adj = TShape(args[num_inputs + 4].operator ObjectRef());\n      }\n      // target_shape\n      if (args[num_inputs + 5].type_code() != kNull) {\n        if (args[num_inputs + 5].type_code() == kDLInt) {\n          param.target_shape = TShape(1, args[num_inputs + 5].operator int64_t());\n        } else {\n          param.target_shape = TShape(args[num_inputs + 5].operator ObjectRef());\n        }\n      }\n      // num_filter\n      param.num_filter = (uint32_t)(args[num_inputs + 6].operator int());\n      // num_group\n      param.num_group = (uint32_t)(args[num_inputs + 7].operator int());\n      // workspace\n      param.workspace = args[num_inputs + 8].operator int64_t();\n      // cudnn_tune\n      if (args[num_inputs + 10].type_code() == kNull) {\n        param.cudnn_tune = dmlc::nullopt;\n      } else {\n        param.cudnn_tune = String2CudnnTune(args[num_inputs + 10]);\n      }\n      // cudnn_off\n      if (args[num_inputs + 11].type_code() == kNull) {\n        param.cudnn_off = false;\n      } else {\n        param.cudnn_off = args[num_inputs + 11].operator bool();\n      }\n\n      CHECK_EQ(param.kernel.ndim(), param.stride.ndim())\n          << \"Stride must have the same number of dimensions with kernel_size,\"\n          << \"but kernel_size is set to \" << param.kernel << \" while stride is \" << param.stride;\n      CHECK_EQ(param.kernel.ndim(), param.dilate.ndim())\n          << \"Dilate must have the same number of dimensions with kernel_size,\"\n          << \"but kernel_size is set to \" << param.kernel << \" while dilate is \" << param.dilate;\n      CHECK_EQ(param.kernel.ndim(), param.pad.ndim())\n          << \"Padding must have the same number of dimensions with kernel_size,\"\n          << \"but kernel_size is set to \" << param.kernel << \" while padding is \" << param.pad;\n      CHECK_EQ(param.kernel.ndim(), param.adj.ndim())\n          << \"Adjustment must have the same number of dimensions with kernel_size,\"\n          << \"but kernel_size is set to \" << param.kernel << \" while adjustment is \" << param.adj;\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::DeconvolutionParam>(&attrs);\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_dropout_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_dropout_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy_extension/npx_dropout_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/nn/dropout-inl.h\"\n\nnamespace mxnet {\n\ninline int String2Mode(const std::string& s) {\n  using namespace op;\n  if (s == \"training\") {\n    return dropout::kTraining;\n  } else if (s == \"always\") {\n    return dropout::kAlways;\n  } else {\n    LOG(FATAL) << \"unknown dropout mode \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\nMXNET_REGISTER_API(\"_npx.dropout\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_dropout\");\n      op::DropoutParam param = {};\n      // inputs\n      int num_inputs    = 1;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      // p\n      param.p = args[1].operator double();\n      // mode\n      param.mode = String2Mode(args[2].operator std::string());\n      // axes\n      if (args[3].type_code() == kNull) {\n        param.axes = TShape(0, 0);\n      } else if (args[3].type_code() == kDLInt) {\n        param.axes = TShape(1, args[3].operator int64_t());\n      } else {\n        param.axes = TShape(args[3].operator ObjectRef());\n      }\n      // cudnn_off\n      if (args[4].type_code() == kNull) {\n        param.cudnn_off = false;\n      } else {\n        param.cudnn_off = args[4].operator bool();\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::DropoutParam>(&attrs);\n      int num_outputs = 1;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_embedding_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_embedding_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy_extension/npx_embedding_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/tensor/indexing_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npx.embedding\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_embedding\");\n      op::EmbeddingParam param = {};\n      // inputs\n      int num_inputs = 2;\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      // input_dim\n      param.input_dim = args[2].operator int64_t();\n      // output_dim\n      param.output_dim = args[3].operator int64_t();\n      // dtype\n      param.dtype = String2MXNetTypeWithBool(args[4].operator std::string());\n      // sparse_grad;\n      if (args[5].type_code() == kNull) {\n        param.sparse_grad = false;\n      } else {\n        param.sparse_grad = args[5].operator bool();\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::EmbeddingParam>(&attrs);\n      int num_outputs = 1;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_fully_connected_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_fully_connected_op.cc\n * \\brief Implementation of the API of functions in\n * src/operator/numpy_extension/npx_fully_connected_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/nn/fully_connected-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npx.fully_connected\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      int args_size = args.size();\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_fully_connected\");\n      op::FullyConnectedParam param = {};\n      // no_bias\n      param.no_bias = args[args_size - 2].operator bool();\n      // inputs\n      int num_inputs = 2;\n      if (param.no_bias) {\n        num_inputs = 2;\n      } else {\n        num_inputs = 3;\n      }\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      // num_hidden\n      param.num_hidden = args[args_size - 3].operator int();\n      // flatten\n      param.flatten = args[args_size - 1].operator bool();\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::FullyConnectedParam>(&attrs);\n\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_group_norm_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_group_norm_op.cc\n * \\brief Implementation of the API of functions in\n * src/operator/numpy_extension/npx_group_norm_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/nn/group_norm-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npx.group_norm\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_group_norm\");\n      op::GroupNormParam param = {};\n      // num_groups\n      param.num_groups = args[3];\n      // eps\n      param.eps = args[4].operator double();\n      // output_mean_var\n      param.output_mean_var = args[5].operator bool();\n      attrs.parsed          = param;\n      attrs.op              = op;\n      SetAttrDict<op::GroupNormParam>(&attrs);\n      // inputs\n      int num_inputs = 3;\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      if (num_outputs == 1) {\n        *ret = ndoutputs[0];\n      } else {\n        std::vector<NDArrayHandle> ndarray_handles;\n        ndarray_handles.reserve(num_outputs);\n        for (int i = 0; i < num_outputs; ++i) {\n          ndarray_handles.emplace_back(ndoutputs[i]);\n        }\n        *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_layer_norm_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_layer_norm_op.cc\n * \\brief Implementation of the API of functions in\n * src/operator/numpy_extension/npx_layer_norm_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/nn/layer_norm-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npx.layer_norm\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_layer_norm\");\n      op::LayerNormParam param = {};\n      // inputs\n      int num_inputs = 3;\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      // axis\n      if (args[3].type_code() == kNull) {\n        param.axis = -1;\n      } else {\n        param.axis = args[3].operator int();\n      }\n      // eps\n      if (args[4].type_code() == kNull) {\n        param.eps = 1e-5f;\n      } else {\n        param.eps = args[4].operator double();\n      }\n      // output_mean_var\n      if (args[5].type_code() == kNull) {\n        param.output_mean_var = false;\n      } else {\n        param.output_mean_var = args[5].operator bool();\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::LayerNormParam>(&attrs);\n      int num_outputs = 3;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      if (num_outputs == 1) {\n        *ret = ndoutputs[0];\n      } else {\n        std::vector<NDArrayHandle> ndarray_handles;\n        ndarray_handles.reserve(num_outputs);\n        for (int i = 0; i < num_outputs; ++i) {\n          ndarray_handles.emplace_back(ndoutputs[i]);\n        }\n        *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_leaky_relu_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_leaky_relu_op.cc\n * \\brief Implementation of the API of functions in\n * src/operator/numpy_extension/npx_leaky_relu_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/leaky_relu-inl.h\"\n\nnamespace mxnet {\n\ninline int String2ActType(const std::string& s) {\n  using namespace op;\n  if (s == \"rrelu\") {\n    return leakyrelu::kRReLU;\n  } else if (s == \"leaky\") {\n    return leakyrelu::kLeakyReLU;\n  } else if (s == \"prelu\") {\n    return leakyrelu::kPReLU;\n  } else if (s == \"elu\") {\n    return leakyrelu::kELU;\n  } else if (s == \"selu\") {\n    return leakyrelu::kSELU;\n  } else if (s == \"gelu\" || s == \"gelu_erf\") {\n    return leakyrelu::kGELU_ERF;\n  } else if (s == \"gelu_tanh\") {\n    return leakyrelu::kGELU_TANH;\n  } else {\n    LOG(FATAL) << \"unknown activation type \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\nMXNET_REGISTER_API(\"_npx.leaky_relu\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_leaky_relu\");\n      op::LeakyReLUParam param = {};\n      int args_size = args.size();\n      // act_type\n      param.act_type = String2ActType(args[args_size - 4].operator std::string());\n      // inputs\n      int num_inputs  = param.act_type == op::leakyrelu::kPReLU ? 2 : 1;\n      int num_outputs = param.act_type == op::leakyrelu::kPReLU ? 2 : 1;\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      // slope\n      if (args[args_size - 3].type_code() == kNull) {\n        param.slope = 0.25f;\n      } else {\n        param.slope = args[args_size - 3].operator double();\n      }\n      // lower_bound\n      if (args[args_size - 2].type_code() == kNull) {\n        param.lower_bound = 0.125f;\n      } else {\n        param.lower_bound = args[args_size - 2].operator double();\n      }\n      // upper_bound\n      if (args[args_size - 1].type_code() == kNull) {\n        param.upper_bound = 0.334f;\n      } else {\n        param.upper_bound = args[args_size - 1].operator double();\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::LeakyReLUParam>(&attrs);\n\n      auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      if (num_outputs == 1) {\n        *ret = ndoutputs[0];\n      } else {\n        std::vector<NDArrayHandle> ndarray_handles;\n        ndarray_handles.reserve(num_outputs);\n        for (int i = 0; i < num_outputs; ++i) {\n          ndarray_handles.emplace_back(ndoutputs[i]);\n        }\n        *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_one_hot_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_one_hot_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy_extension/npx_one_hot_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/tensor/indexing_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npx.one_hot\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_one_hot\");\n      op::OneHotParam param = {};\n      // inputs\n      int num_inputs    = 1;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n      // depth\n      param.depth = args[1].operator int64_t();\n      // on_value\n      if (args[2].type_code() == kNull) {\n        param.on_value = 1.0;\n      } else {\n        param.on_value = args[2].operator double();\n      }\n      // off_value\n      if (args[3].type_code() == kNull) {\n        param.off_value = 0.0;\n      } else {\n        param.off_value = args[3].operator double();\n      }\n      // dtype\n      if (args[4].type_code() != kNull) {\n        param.dtype = String2MXNetTypeWithBool(args[4].operator std::string());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::OneHotParam>(&attrs);\n      int num_outputs = 1;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_pick_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_pick_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy_extension/npx_pick_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/tensor/broadcast_reduce_op.h\"\n\nnamespace mxnet {\n\ninline int String2PickMode(const std::string& s) {\n  using namespace op;\n  if (s == \"wrap\") {\n    return kWrap;\n  } else if (s == \"clip\") {\n    return kClip;\n  } else {\n    LOG(FATAL) << \"unknown mode type \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\nMXNET_REGISTER_API(\"_npx.pick\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  nnvm::NodeAttrs attrs;\n  const nnvm::Op* op = Op::Get(\"_npx_pick\");\n  op::PickParam param = {};\n  // axis\n  if (args[2].type_code() == kNull) {\n    param.axis = dmlc::nullopt;\n  } else {\n    param.axis = args[2].operator int();\n  }\n  // mode\n  param.mode = String2PickMode(args[3].operator std::string());\n  // keepdims\n  if (args[4].type_code() == kNull) {\n    param.keepdims = false;\n  } else {\n    param.keepdims = args[4].operator bool();\n  }\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::PickParam>(&attrs);\n  // inputs\n  int num_inputs = 2;\n  std::vector<NDArray*> inputs;\n  inputs.reserve(num_inputs);\n  for (int i = 0; i < 2; ++i) {\n    inputs.push_back(args[i].operator mxnet::NDArray*());\n  }\n  int num_outputs = 0;\n  auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n  *ret            = ndoutputs[0];\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_pooling_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_pooling_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy_extension/npx_pooling_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/nn/pooling-inl.h\"\n\nnamespace mxnet {\n\ninline int String2PoolingLayout(const std::string& s) {\n  using namespace op;\n  if (s == \"NCW\") {\n    return mshadow::kNCW;\n  } else if (s == \"NCHW\") {\n    return mshadow::kNCHW;\n  } else if (s == \"NCDHW\") {\n    return mshadow::kNCDHW;\n  } else if (s == \"NWC\") {\n    return mshadow::kNWC;\n  } else if (s == \"NHWC\") {\n    return mshadow::kNHWC;\n  } else if (s == \"NDHWC\") {\n    return mshadow::kNDHWC;\n  } else {\n    LOG(FATAL) << \"unknown layout type \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\ninline int String2PoolType(const std::string& s) {\n  using namespace op;\n  if (s == \"max\") {\n    return pool_enum::kMaxPooling;\n  } else if (s == \"avg\") {\n    return pool_enum::kAvgPooling;\n  } else if (s == \"sum\") {\n    return pool_enum::kSumPooling;\n  } else if (s == \"lp\") {\n    return pool_enum::kLpPooling;\n  } else {\n    LOG(FATAL) << \"unknown pooling type type \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\ninline int String2Convention(const std::string& s) {\n  using namespace op;\n  if (s == \"full\") {\n    return pool_enum::kFull;\n  } else if (s == \"valid\") {\n    return pool_enum::kValid;\n  } else if (s == \"same\") {\n    return pool_enum::kSame;\n  } else {\n    LOG(FATAL) << \"unknown pooling convention type \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\nMXNET_REGISTER_API(\"_npx.pooling\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      const nnvm::Op* op = Op::Get(\"_npx_pooling\");\n      op::PoolingParam param = {};\n      // inputs\n      int num_inputs    = 1;\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n\n      // kernel\n      if (args[1].type_code() == kDLInt) {\n        param.kernel = TShape(1, args[1].operator int64_t());\n      } else {\n        param.kernel = TShape(args[1].operator ObjectRef());\n      }\n      // global pool\n      param.global_pool = args[6].operator bool();\n      // stride\n      if (args[2].type_code() == kNull) {\n        if (param.kernel.ndim() == 1) {\n          param.stride = mshadow::Shape1(1);\n        } else if (param.kernel.ndim() == 2) {\n          param.stride = mshadow::Shape2(1, 1);\n        } else {\n          if (param.global_pool == false) {\n            CHECK_EQ(param.kernel.ndim(), 3U)\n                << param.kernel.ndim()\n                << \"D pooling not supported. Only 1D, 2D, and 3D pooling are supported.\";\n          }\n          param.stride = mshadow::Shape3(1, 1, 1);\n        }\n      } else if (args[2].type_code() == kDLInt) {\n        param.stride = TShape(1, args[2].operator int64_t());\n      } else {\n        param.stride = TShape(args[2].operator ObjectRef());\n      }\n      // pad\n      if (args[3].type_code() == kNull) {\n        if (param.kernel.ndim() == 1) {\n          param.pad = mshadow::Shape1(0);\n        } else if (param.kernel.ndim() == 2) {\n          param.pad = mshadow::Shape2(0, 0);\n        } else {\n          param.pad = mshadow::Shape3(0, 0, 0);\n        }\n      } else if (args[3].type_code() == kDLInt) {\n        param.pad = TShape(1, args[3].operator int64_t());\n      } else {\n        param.pad = TShape(args[3].operator ObjectRef());\n      }\n      // pool type\n      param.pool_type = String2PoolType(args[4].operator std::string());\n      // pooling convention\n      param.pooling_convention = String2Convention(args[5].operator std::string());\n      // cudnn_off\n      if (args[7].type_code() == kNull) {\n        param.cudnn_off = false;\n      } else {\n        param.cudnn_off = args[7].operator bool();\n      }\n      // p_value\n      if (args[8].type_code() == kNull) {\n        param.p_value = dmlc::nullopt;\n      } else {\n        param.p_value = args[8].operator int();\n      }\n      // count_include_pad\n      if (args[9].type_code() == kNull) {\n        param.count_include_pad = dmlc::nullopt;\n      } else {\n        param.count_include_pad = args[9].operator bool();\n      }\n      // layout\n      if (args[10].type_code() == kNull) {\n        param.layout = dmlc::nullopt;\n      } else {\n        param.layout = String2PoolingLayout(args[10]);\n      }\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::PoolingParam>(&attrs);\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      if (num_outputs == 1) {\n        *ret = ndoutputs[0];\n      } else {\n        std::vector<NDArrayHandle> ndarray_handles;\n        ndarray_handles.reserve(num_outputs);\n        for (int i = 0; i < num_outputs; ++i) {\n          ndarray_handles.emplace_back(ndoutputs[i]);\n        }\n        *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_rnn_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_rnn_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy_extension/npx_rnn_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/rnn-inl.h\"\n\nnamespace mxnet {\n\ninline int String2ComputeMode(const std::string& s) {\n  using namespace op;\n  if (s == \"rnn_relu\") {\n    return rnn_enum::kRnnRelu;\n  } else if (s == \"rnn_tanh\") {\n    return rnn_enum::kRnnTanh;\n  } else if (s == \"lstm\") {\n    return rnn_enum::kLstm;\n  } else if (s == \"gru\") {\n    return rnn_enum::kGru;\n  } else {\n    LOG(FATAL) << \"unknown compute mode \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\nMXNET_REGISTER_API(\"_npx.rnn\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  nnvm::NodeAttrs attrs;\n  const nnvm::Op* op = Op::Get(\"_npx_rnn\");\n  op::RNNParam param = {};\n  int args_size  = args.size();\n  int num_inputs = 0;\n\n  // mode\n  param.mode = String2ComputeMode(args[args_size - 7].operator std::string());\n  num_inputs = (param.mode == op::rnn_enum::kLstm) ? 4 : 3;\n  // use_sequence_length\n  if (args[args_size - 5].type_code() == kNull) {\n    param.use_sequence_length = false;\n  } else {\n    param.use_sequence_length = args[args_size - 5].operator bool();\n  }\n  if (param.use_sequence_length)\n    num_inputs += 1;\n  // inputs\n  std::vector<NDArray*> inputs;\n  inputs.reserve(num_inputs);\n  for (int i = 0; i < num_inputs; ++i) {\n    inputs.push_back(args[i].operator mxnet::NDArray*());\n  }\n  // state_size\n  param.state_size = (uint32_t)(args[args_size - 11].operator int());\n  // num_layers\n  param.num_layers = (uint32_t)(args[args_size - 10].operator int());\n  // bidirectional\n  if (args[args_size - 9].type_code() == kNull) {\n    param.bidirectional = false;\n  } else {\n    param.bidirectional = args[args_size - 9].operator bool();\n  }\n  // state_outputs\n  if (args[args_size - 8].type_code() == kNull) {\n    param.state_outputs = false;\n  } else {\n    param.state_outputs = args[args_size - 8].operator bool();\n  }\n  // p\n  if (args[args_size - 6].type_code() == kNull) {\n    param.p = 0.0;\n  } else {\n    param.p = args[args_size - 6].operator double();\n  }\n  // projection_size\n  if (args[args_size - 4].type_code() == kNull) {\n    param.projection_size = dmlc::nullopt;\n  } else {\n    param.projection_size = args[args_size - 4].operator int();\n  }\n  // lstm_state_clip_min\n  if (args[args_size - 3].type_code() == kNull) {\n    param.lstm_state_clip_min = dmlc::nullopt;\n  } else {\n    param.lstm_state_clip_min = args[args_size - 3].operator double();\n  }\n  // lstm_state_clip_max\n  if (args[args_size - 2].type_code() == kNull) {\n    param.lstm_state_clip_max = dmlc::nullopt;\n  } else {\n    param.lstm_state_clip_max = args[args_size - 2].operator double();\n  }\n  // lstm_state_clip_nan\n  if (args[args_size - 1].type_code() == kNull) {\n    param.lstm_state_clip_nan = false;\n  } else {\n    param.lstm_state_clip_nan = args[args_size - 1].operator bool();\n  }\n  // initialize\n  param.seq_length_ = 0;\n  param.batch_size_ = 0;\n  param.input_size_ = 0;\n  attrs.parsed      = param;\n  attrs.op          = op;\n  SetAttrDict<op::RNNParam>(&attrs);\n  int num_outputs = 0;\n  auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n  if (num_outputs == 1) {\n    *ret = ndoutputs[0];\n  } else {\n    std::vector<NDArrayHandle> ndarray_handles;\n    ndarray_handles.reserve(num_outputs);\n    for (int i = 0; i < num_outputs; ++i) {\n      ndarray_handles.emplace_back(ndoutputs[i]);\n    }\n    *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n  }\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_softmax_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_softmax_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy_extension/npx_softmax_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/nn/softmax-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npx.softmax\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      static const nnvm::Op* op = Op::Get(\"_npx_softmax\");\n      op::SoftmaxParam param    = {};\n      int args_size             = args.size();\n      // inputs\n      int num_inputs = args_size - 4;\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n\n      // parse use_length\n      if (args[args_size - 2].type_code() == kNull) {\n        param.use_length = false;\n      } else {\n        param.use_length = args[args_size - 2].operator bool();\n      }\n\n      // parse axis\n      if (args[args_size - 4].type_code() == kDLInt) {\n        param.axis = args[args_size - 4].operator int();\n      } else if (args[args_size - 4].type_code() == kDLFloat) {\n        param.axis = static_cast<int>(args[args_size - 4].operator double());\n      } else {\n        param.axis = -1;\n      }\n\n      // parse temperature\n      if (args[args_size - 3].type_code() == kNull) {\n        param.temperature = dmlc::nullopt;\n      } else {\n        param.temperature = args[args_size - 3].operator double();\n      }\n\n      // parse dtype\n      if (args[args_size - 1].type_code() == kNull) {\n        param.dtype = dmlc::nullopt;\n      } else {\n        param.dtype = String2MXNetTypeWithBool(args[args_size - 1].operator std::string());\n      }\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::SoftmaxParam>(&attrs);\n\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npx.log_softmax\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      static const nnvm::Op* op = Op::Get(\"_npx_log_softmax\");\n      op::SoftmaxParam param    = {};\n\n      int args_size = args.size();\n      // inputs\n      int num_inputs = args_size - 4;\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n\n      // parse use_length\n      if (args[args_size - 2].type_code() == kNull) {\n        param.use_length = false;\n      } else {\n        param.use_length = args[args_size - 2].operator bool();\n      }\n\n      // parse axis\n      if (args[args_size - 4].type_code() == kDLInt) {\n        param.axis = args[args_size - 4].operator int();\n      } else if (args[args_size - 4].type_code() == kDLFloat) {\n        param.axis = static_cast<int>(args[args_size - 4].operator double());\n      } else {\n        param.axis = -1;\n      }\n\n      // parse temperature\n      if (args[args_size - 3].type_code() == kNull) {\n        param.temperature = dmlc::nullopt;\n      } else {\n        param.temperature = args[args_size - 3].operator double();\n      }\n\n      // parse dtype\n      if (args[args_size - 1].type_code() == kNull) {\n        param.dtype = dmlc::nullopt;\n      } else {\n        param.dtype = String2MXNetTypeWithBool(args[args_size - 1].operator std::string());\n      }\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::SoftmaxParam>(&attrs);\n\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npx.masked_softmax\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      static const nnvm::Op* op    = Op::Get(\"_npx_masked_softmax\");\n      op::MaskedSoftmaxParam param = {};\n\n      // inputs\n      int num_inputs = 2;\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      // parse axis\n      if (args[2].type_code() == kDLInt) {\n        param.axis = args[2].operator int();\n      } else if (args[2].type_code() == kDLFloat) {\n        param.axis = static_cast<int>(args[2].operator double());\n      } else {\n        param.axis = -1;\n      }\n      // parse temperature\n      if (args[3].type_code() == kNull) {\n        param.temperature = dmlc::nullopt;\n      } else {\n        param.temperature = args[3].operator double();\n      }\n      // parse normalize\n      if (args[4].type_code() == kNull) {\n        param.normalize = true;\n      } else {\n        param.normalize = args[4].operator bool();\n      }\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::MaskedSoftmaxParam>(&attrs);\n\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\nMXNET_REGISTER_API(\"_npx.masked_log_softmax\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      nnvm::NodeAttrs attrs;\n      static const nnvm::Op* op    = Op::Get(\"_npx_masked_log_softmax\");\n      op::MaskedSoftmaxParam param = {};\n\n      // inputs\n      int num_inputs = 2;\n      std::vector<NDArray*> inputs;\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      // parse axis\n      if (args[2].type_code() == kDLInt) {\n        param.axis = args[2].operator int();\n      } else if (args[2].type_code() == kDLFloat) {\n        param.axis = static_cast<int>(args[2].operator double());\n      } else {\n        param.axis = -1;\n      }\n      // parse temperature\n      if (args[3].type_code() == kNull) {\n        param.temperature = dmlc::nullopt;\n      } else {\n        param.temperature = args[3].operator double();\n      }\n      // parse normalize\n      if (args[4].type_code() == kNull) {\n        param.normalize = true;\n      } else {\n        param.normalize = args[4].operator bool();\n      }\n\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::MaskedSoftmaxParam>(&attrs);\n\n      int num_outputs = 0;\n      auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);\n      *ret            = ndoutputs[0];\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/numpy_extension/npx_topk_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file npx_topk_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy_extension/npx_topk_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/tensor/ordering_op-inl.h\"\n\nnamespace mxnet {\n\ninline int String2ReturnType(const std::string& s) {\n  using namespace op;\n  if (s == \"value\") {\n    return topk_enum::kReturnValue;\n  } else if (s == \"indices\") {\n    return topk_enum::kReturnIndices;\n  } else if (s == \"mask\") {\n    return topk_enum::kReturnMask;\n  } else if (s == \"both\") {\n    return topk_enum::kReturnBoth;\n  } else {\n    LOG(FATAL) << \"unknown return type \" << s;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return 0;\n}\n\nMXNET_REGISTER_API(\"_npx.topk\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  nnvm::NodeAttrs attrs;\n  const nnvm::Op* op = Op::Get(\"_npx_topk\");\n  op::TopKParam param = {};\n  // inputs\n  int num_inputs    = 1;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  // axis\n  if (args[1].type_code() == kNull) {\n    param.axis = dmlc::nullopt;\n  } else {\n    param.axis = args[1].operator int();\n  }\n  // k\n  if (args[2].type_code() == kNull) {\n    param.k = 1;\n  } else {\n    param.k = args[2].operator int();\n  }\n  // ret_typ\n  param.ret_typ = String2ReturnType(args[3].operator std::string());\n  // is_ascend\n  if (args[4].type_code() == kNull) {\n    param.is_ascend = false;\n  } else {\n    param.is_ascend = args[4].operator bool();\n  }\n  // dtype\n  param.dtype  = String2MXNetTypeWithBool(args[5].operator std::string());\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::TopKParam>(&attrs);\n  int num_outputs = 1;\n  auto ndoutputs  = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  if (num_outputs == 1) {\n    *ret = ndoutputs[0];\n  } else {\n    std::vector<NDArrayHandle> ndarray_handles;\n    ndarray_handles.reserve(num_outputs);\n    for (int i = 0; i < num_outputs; ++i) {\n      ndarray_handles.emplace_back(ndoutputs[i]);\n    }\n    *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n  }\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/op_utils.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file op_utils.cc\n * \\brief Utility functions for modification in src/operator\n */\n\n#include \"op_utils.h\"\n#include <mxnet/base.h>\n#include \"../../operator/numpy/np_percentile_op-inl.h\"\n\nnamespace mxnet {\n\nstd::string MXNetTypeWithBool2String(int dtype) {\n  switch (dtype) {\n    case mshadow::kFloat32:\n      return \"float32\";\n    case mshadow::kFloat64:\n      return \"float64\";\n    case mshadow::kFloat16:\n      return \"float16\";\n    case mshadow::kUint8:\n      return \"uint8\";\n    case mshadow::kInt8:\n      return \"int8\";\n    case mshadow::kInt32:\n      return \"int32\";\n    case mshadow::kInt64:\n      return \"int64\";\n    case mshadow::kBool:\n      return \"bool\";\n    default:\n      LOG(FATAL) << \"Unknown type enum \" << dtype;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return \"\";\n}\n\nstd::string MXNetPercentileType2String(int interpolation) {\n  using namespace op;\n  switch (interpolation) {\n    case percentile_enum::kLinear:\n      return \"linear\";\n    case percentile_enum::kLower:\n      return \"lower\";\n    case percentile_enum::kHigher:\n      return \"higher\";\n    case percentile_enum::kMidpoint:\n      return \"midpoint\";\n    case percentile_enum::kNearest:\n      return \"nearest\";\n    default:\n      LOG(FATAL) << \"Unknown type enum \" << interpolation;\n  }\n  LOG(FATAL) << \"should not reach here \";\n  return \"\";\n}\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/op_utils.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file op_utils.h\n * \\brief Utility functions for modification in src/operator\n */\n#ifndef MXNET_API_OPERATOR_OP_UTILS_H_\n#define MXNET_API_OPERATOR_OP_UTILS_H_\n\n#include <string>\n\nnamespace mxnet {\n\nstd::string MXNetTypeWithBool2String(int dtype);\nstd::string MXNetPercentileType2String(int interpolation);\n\n}  // namespace mxnet\n\n#endif  // MXNET_API_OPERATOR_OP_UTILS_H_\n"
  },
  {
    "path": "src/api/operator/random/np_gamma_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_gamma_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/random/np_gamma_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include <vector>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/random/np_gamma_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.gamma\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_gamma\");\n  nnvm::NodeAttrs attrs;\n  op::NumpyGammaParam param = {};\n  int num_inputs = 0;\n  std::vector<NDArray*> inputs;\n  if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {\n    if (args[0].type_code() == kNull) {\n      param.shape = dmlc::nullopt;\n    } else {\n      param.shape = args[0].operator double();\n    }\n    if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt) {\n      // both 'shape' and 'scale' are numeric types\n      num_inputs = 0;\n      if (args[1].type_code() == kNull) {\n        param.scale = dmlc::nullopt;\n      } else {\n        param.scale = args[1].operator double();\n      }\n    } else {\n      // 'shape' is numeric types but 'scale' is not\n      num_inputs  = 1;\n      param.scale = dmlc::nullopt;\n      inputs.push_back(args[1].operator mxnet::NDArray*());\n    }\n  } else {\n    param.shape = dmlc::nullopt;\n    inputs.push_back(args[0].operator mxnet::NDArray*());\n    if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt) {\n      // 'shape' is not numeric types but 'scale' is numeric types\n      num_inputs = 1;\n      if (args[1].type_code() == kNull) {\n        param.scale = dmlc::nullopt;\n      } else {\n        param.scale = args[1].operator double();\n      }\n    } else {\n      // nither 'shape' or 'scale' is numeric types\n      num_inputs  = 2;\n      param.scale = dmlc::nullopt;\n      inputs.push_back(args[1].operator mxnet::NDArray*());\n    }\n  }\n  if (args[2].type_code() == kNull) {\n    param.size = dmlc::optional<mxnet::Tuple<index_t>>();\n  } else if (args[2].type_code() == kDLInt || args[2].type_code() == kDLFloat) {\n    param.size = Tuple<index_t>(1, args[2].operator int64_t());\n  } else {\n    param.size = Tuple<index_t>(args[2].operator ObjectRef());\n  }\n  if (args[4].type_code() == kNull) {\n    param.dtype = mxnet::common::GetDefaultDtype();\n  } else {\n    param.dtype = String2MXNetTypeWithBool(args[4].operator std::string());\n  }\n  NDArray* out      = args[5].operator mxnet::NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  attrs.parsed      = param;\n  attrs.op          = op;\n  if (args[3].type_code() != kNull) {\n    attrs.dict[\"ctx\"] = args[3].operator std::string();\n  }\n  SetAttrDict<op::NumpyGammaParam>(&attrs);\n  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, outputs);\n  if (out) {\n    *ret = PythonArg(5);\n  } else {\n    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n  }\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/random/np_normal_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_normal_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/random/np_normal_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include <vector>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/random/np_normal_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.normal\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_normal\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyNormalParam param = {};\n      int num_inputs = 0;\n      std::vector<NDArray*> inputs;\n      if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {\n        if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt) {\n          // 'loc' and 'scale' are both numeric types\n          num_inputs  = 0;\n          param.loc   = args[0].operator double();\n          param.scale = args[1].operator double();\n        } else {\n          // 'loc' is numeric types but 'scale' is not numeric types\n          num_inputs  = 1;\n          param.loc   = args[0].operator double();\n          param.scale = dmlc::nullopt;\n          inputs.push_back(args[1].operator mxnet::NDArray*());\n        }\n      } else {\n        if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt) {\n          // 'loc' is not numeric types but 'scale' is numeric types\n          num_inputs  = 1;\n          param.loc   = dmlc::nullopt;\n          param.scale = args[1].operator double();\n          inputs.push_back(args[0].operator mxnet::NDArray*());\n        } else {\n          // nither 'loc' or 'scale' is numeric types\n          num_inputs = 2;\n          inputs.push_back(args[0].operator mxnet::NDArray*());\n          inputs.push_back(args[1].operator mxnet::NDArray*());\n        }\n      }\n      if (args[2].type_code() == kNull) {\n        param.size = dmlc::optional<mxnet::Tuple<index_t>>();\n      } else if (args[2].type_code() == kDLInt || args[2].type_code() == kDLFloat) {\n        param.size = Tuple<index_t>(1, args[2].operator int64_t());\n      } else {\n        param.size = Tuple<index_t>(args[2].operator ObjectRef());\n      }\n      if (args[4].type_code() == kNull) {\n        param.dtype = mxnet::common::GetDefaultDtype();\n      } else {\n        param.dtype = String2MXNetTypeWithBool(args[4].operator std::string());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      if (args[3].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[3].operator std::string();\n      }\n      NDArray* out      = args[5].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      SetAttrDict<op::NumpyNormalParam>(&attrs);\n      auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(5);\n      } else {\n        *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/random/np_randint_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_randint_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/random/np_randint_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include <vector>\n#include \"../utils.h\"\n#include \"../../../operator/random/sample_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.randint\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_random_randint\");\n      nnvm::NodeAttrs attrs;\n      op::SampleRandIntParam param = {};\n      int num_inputs = 0;\n      param.low      = args[0].operator int();\n      param.high     = args[1].operator int();\n      if (args[2].type_code() == kDLInt) {\n        param.shape = TShape(1, args[2].operator int64_t());\n      } else {\n        param.shape = TShape(args[2].operator ObjectRef());\n      }\n      if (args[3].type_code() == kNull) {\n        param.dtype = mxnet::common::GetDefaultDtype();\n      } else {\n        param.dtype = String2MXNetTypeWithBool(args[3].operator std::string());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      if (args[4].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[4].operator std::string();\n      }\n      NDArray* out      = args[5].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      SetAttrDict<op::SampleRandIntParam>(&attrs);\n      auto ndoutputs = Invoke(op, &attrs, num_inputs, nullptr, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(5);\n      } else {\n        *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/random/np_uniform_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file np_uniform_op.cc\n * \\brief Implementation of the API of functions in src/operator/numpy/random/np_uniform_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include <vector>\n#include \"../utils.h\"\n#include \"../../../operator/numpy/random/np_uniform_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.uniform\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_uniform\");\n      nnvm::NodeAttrs attrs;\n      op::NumpyUniformParam param = {};\n      int num_inputs = 0;\n      std::vector<NDArray*> inputs;\n      if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {\n        if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt) {\n          // 'low' and 'high' are both numeric types\n          num_inputs = 0;\n          param.low  = args[0].operator double();\n          param.high = args[1].operator double();\n        } else {\n          // 'low' is numeric types but 'high' is not numeric types\n          num_inputs = 1;\n          param.low  = args[0].operator double();\n          param.high = dmlc::nullopt;\n        }\n      } else {\n        if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt) {\n          // 'low' is not numeric types but 'high' is numeric types\n          num_inputs = 1;\n          param.low  = dmlc::nullopt;\n          param.high = args[1].operator double();\n        } else {\n          // nither 'low' or 'high' is numeric types\n          num_inputs = 2;\n        }\n      }\n      inputs.reserve(num_inputs);\n      for (int i = 0; i < num_inputs; ++i) {\n        inputs.push_back(args[i].operator mxnet::NDArray*());\n      }\n      if (args[2].type_code() == kNull) {\n        param.size = dmlc::optional<mxnet::Tuple<index_t>>();\n      } else if (args[2].type_code() == kDLInt || args[2].type_code() == kDLFloat) {\n        param.size = Tuple<index_t>(1, args[2].operator int64_t());\n      } else {\n        param.size = Tuple<index_t>(args[2].operator ObjectRef());\n      }\n      if (args[4].type_code() == kNull) {\n        param.dtype = mxnet::common::GetDefaultDtype();\n      } else {\n        param.dtype = String2MXNetTypeWithBool(args[4].operator std::string());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      if (args[3].type_code() != kNull) {\n        attrs.dict[\"ctx\"] = args[3].operator std::string();\n      }\n      NDArray* out      = args[5].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      SetAttrDict<op::NumpyUniformParam>(&attrs);\n      auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(5);\n      } else {\n        *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/random/shuffle_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file shuffle_op.cc\n * \\brief Implementation of the API of functions in src/operator/random/shuffle_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/elemwise_op_common.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.shuffle\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_shuffle\");\n      nnvm::NodeAttrs attrs;\n\n      NDArray* inputs[1];\n      int num_inputs = 1;\n\n      if (args[0].type_code() != kNull) {\n        inputs[0] = args[0].operator mxnet::NDArray*();\n      }\n\n      attrs.op = op;\n\n      NDArray* out      = args[1].operator mxnet::NDArray*();\n      NDArray** outputs = out == nullptr ? nullptr : &out;\n      int num_outputs   = out != nullptr;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n      if (out) {\n        *ret = PythonArg(1);\n      } else {\n        *ret = ndoutputs[0];\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/tensor/elemwise_binary_broadcast_op_extended.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file elemwise_binary_broadcast_op_extended.cc\n * \\brief Implementation of the API of functions in\n * src/operator/tensor/elemwise_binary_broadcast_op_extended.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../ufunc_helper.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.maximum\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op        = Op::Get(\"_npi_maximum\");\n      const nnvm::Op* op_scalar = Op::Get(\"_npi_maximum_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, nullptr);\n    });\n\nMXNET_REGISTER_API(\"_npi.minimum\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op        = Op::Get(\"_npi_minimum\");\n      const nnvm::Op* op_scalar = Op::Get(\"_npi_minimum_scalar\");\n      UFuncHelper(args, ret, op, op_scalar, nullptr);\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/tensor/indexing_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file indexing_op.cc\n * \\brief Implementation of the API of functions in src/operator/tensor/indexing_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/tensor/indexing_op.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.take\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_take\");\n  nnvm::NodeAttrs attrs;\n  op::TakeParam param = {};\n  NDArray* inputs[2];\n\n  if (args[0].type_code() != kNull) {\n    inputs[0] = args[0].operator mxnet::NDArray*();\n  }\n\n  if (args[1].type_code() != kNull) {\n    inputs[1] = args[1].operator mxnet::NDArray*();\n  }\n\n  if (args[2].type_code() == kDLInt) {\n    param.axis = args[2].operator int();\n  }\n\n  if (args[3].type_code() != kNull) {\n    std::string mode = args[3].operator std::string();\n    if (mode == \"raise\") {\n      param.mode = op::take_::kRaise;\n    } else if (mode == \"clip\") {\n      param.mode = op::take_::kClip;\n    } else if (mode == \"wrap\") {\n      param.mode = op::take_::kWrap;\n    }\n  }\n\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::TakeParam>(&attrs);\n\n  NDArray* out      = args[4].operator mxnet::NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  // set the number of outputs provided by the `out` arugment\n  int num_outputs = out != nullptr;\n  auto ndoutputs  = Invoke(op, &attrs, 2, inputs, &num_outputs, outputs);\n  if (out) {\n    *ret = PythonArg(4);\n  } else {\n    *ret = ndoutputs[0];\n  }\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/tensor/matrix_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file matrix_op.cc\n * \\brief Implementation of the API of functions in src/operator/tensor/matrix_op.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/tensor/matrix_op-inl.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.clip\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_clip\");\n  nnvm::NodeAttrs attrs;\n  op::ClipParam param = {};\n  NDArray* inputs[1];\n\n  if (args[0].type_code() != kNull) {\n    inputs[0] = args[0].operator mxnet::NDArray*();\n  }\n\n  if (args[1].type_code() != kNull) {\n    param.a_min = args[1].operator double();\n  } else {\n    param.a_min = -INFINITY;\n  }\n\n  if (args[2].type_code() != kNull) {\n    param.a_max = args[2].operator double();\n  } else {\n    param.a_max = INFINITY;\n  }\n\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::ClipParam>(&attrs);\n\n  NDArray* out      = args[3].operator mxnet::NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  // set the number of outputs provided by the `out` arugment\n  int num_outputs = out != nullptr;\n  auto ndoutputs  = Invoke(op, &attrs, 1, inputs, &num_outputs, outputs);\n  if (out) {\n    *ret = PythonArg(3);\n  } else {\n    *ret = ndoutputs[0];\n  }\n});\n\nMXNET_REGISTER_API(\"_npi.tile\").set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n  using namespace runtime;\n  const nnvm::Op* op = Op::Get(\"_npi_tile\");\n  nnvm::NodeAttrs attrs;\n  op::TileParam param;\n  if (args[1].type_code() == kDLInt) {\n    param.reps = Tuple<int>(1, args[1].operator int64_t());\n  } else {\n    param.reps = Tuple<int>(args[1].operator ObjectRef());\n  }\n  attrs.parsed = param;\n  attrs.op     = op;\n  SetAttrDict<op::TileParam>(&attrs);\n  int num_outputs   = 0;\n  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};\n  int num_inputs    = 1;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n  *ret              = ndoutputs[0];\n});\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/tensor/unravel.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file unravel.cc\n * \\brief Implementation of the API of functions in src/operator/tensor/ravel.cc\n */\n#include <mxnet/api_registry.h>\n#include <mxnet/runtime/packed_func.h>\n#include \"../utils.h\"\n#include \"../../../operator/tensor/ravel.h\"\n\nnamespace mxnet {\n\nMXNET_REGISTER_API(\"_npi.unravel_index\")\n    .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {\n      using namespace runtime;\n      const nnvm::Op* op = Op::Get(\"_npi_unravel_index\");\n      nnvm::NodeAttrs attrs;\n      op::RavelParam param;\n      if (args[1].type_code() == kNull) {\n        param.shape = TShape(-1, 0);\n      } else if (args[1].type_code() == kDLInt) {\n        param.shape = TShape(1, args[1].operator int64_t());\n      } else {\n        param.shape = TShape(args[1].operator ObjectRef());\n      }\n      attrs.parsed = param;\n      attrs.op     = op;\n      SetAttrDict<op::RavelParam>(&attrs);\n      NDArray* inputs[] = {args[0].operator mxnet::NDArray *()};\n      int num_inputs    = 1;\n      int num_outputs   = 0;\n      auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);\n      if (num_outputs == 1) {\n        *ret = ndoutputs[0];\n      } else {\n        std::vector<NDArrayHandle> ndarray_handles;\n        ndarray_handles.reserve(num_outputs);\n        for (int i = 0; i < num_outputs; ++i) {\n          ndarray_handles.emplace_back(ndoutputs[i]);\n        }\n        *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());\n      }\n    });\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/ufunc_helper.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file ufunc_helper.cc\n * \\brief ufunc helper\n */\n#include \"ufunc_helper.h\"\n#include \"utils.h\"\n#include \"../../imperative/imperative_utils.h\"\n#include \"../../operator/tensor/elemwise_binary_scalar_op.h\"\n\nnamespace mxnet {\n\ntemplate <>\nvoid SetAttrDict<double>(nnvm::NodeAttrs* attrs) {\n  if (Imperative::Get()->is_recording()) {\n    attrs->dict[\"scalar\"] = std::to_string(::dmlc::get<double>(attrs->parsed));\n  }\n}\n\nvoid UFuncHelper(NDArray* lhs,\n                 NDArray* rhs,\n                 NDArray* out,\n                 runtime::MXNetRetValue* ret,\n                 const nnvm::Op* op) {\n  using namespace runtime;\n  nnvm::NodeAttrs attrs;\n  attrs.op          = op;\n  NDArray* inputs[] = {lhs, rhs};\n  int num_inputs    = 2;\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (outputs) {\n    *ret = PythonArg(2);\n  } else {\n    *ret = reinterpret_cast<NDArray*>(ndoutputs[0]);\n  }\n}\n\nvoid UFuncHelper(NDArray* lhs,\n                 int64_t rhs,\n                 NDArray* out,\n                 runtime::MXNetRetValue* ret,\n                 const nnvm::Op* op) {\n  using namespace runtime;\n  nnvm::NodeAttrs attrs;\n  op::NumpyBinaryScalarParam param = {};\n  param.scalar = rhs;\n  param.is_int = true;\n  attrs.op     = op;\n  attrs.parsed = param;\n  SetAttrDict<op::NumpyBinaryScalarParam>(&attrs);\n  NDArray** inputs  = &lhs;\n  int num_inputs    = 1;\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (outputs) {\n    *ret = PythonArg(2);\n  } else {\n    *ret = reinterpret_cast<NDArray*>(ndoutputs[0]);\n  }\n}\n\nvoid UFuncHelper(NDArray* lhs,\n                 double rhs,\n                 NDArray* out,\n                 runtime::MXNetRetValue* ret,\n                 const nnvm::Op* op) {\n  using namespace runtime;\n  nnvm::NodeAttrs attrs;\n  op::NumpyBinaryScalarParam param = {};\n  param.scalar = rhs;\n  param.is_int = false;\n  attrs.op     = op;\n  attrs.parsed = param;\n  SetAttrDict<op::NumpyBinaryScalarParam>(&attrs);\n  NDArray** inputs  = &lhs;\n  int num_inputs    = 1;\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (outputs) {\n    *ret = PythonArg(2);\n  } else {\n    *ret = reinterpret_cast<NDArray*>(ndoutputs[0]);\n  }\n}\n\nvoid UFuncHelper(int64_t lhs,\n                 NDArray* rhs,\n                 NDArray* out,\n                 runtime::MXNetRetValue* ret,\n                 const nnvm::Op* op) {\n  using namespace runtime;\n  nnvm::NodeAttrs attrs;\n  op::NumpyBinaryScalarParam param = {};\n  param.scalar = lhs;\n  param.is_int = true;\n  attrs.op     = op;\n  attrs.parsed = param;\n  SetAttrDict<op::NumpyBinaryScalarParam>(&attrs);\n  NDArray** inputs  = &rhs;\n  int num_inputs    = 1;\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (outputs) {\n    *ret = PythonArg(2);\n  } else {\n    *ret = reinterpret_cast<NDArray*>(ndoutputs[0]);\n  }\n}\n\nvoid UFuncHelper(double lhs,\n                 NDArray* rhs,\n                 NDArray* out,\n                 runtime::MXNetRetValue* ret,\n                 const nnvm::Op* op) {\n  using namespace runtime;\n  nnvm::NodeAttrs attrs;\n  op::NumpyBinaryScalarParam param = {};\n  param.scalar = lhs;\n  param.is_int = false;\n  attrs.op     = op;\n  attrs.parsed = param;\n  SetAttrDict<op::NumpyBinaryScalarParam>(&attrs);\n  NDArray** inputs  = &rhs;\n  int num_inputs    = 1;\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_outputs   = out != nullptr;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (outputs) {\n    *ret = PythonArg(2);\n  } else {\n    *ret = reinterpret_cast<NDArray*>(ndoutputs[0]);\n  }\n}\n\nvoid UFuncHelper(runtime::MXNetArgs args,\n                 runtime::MXNetRetValue* ret,\n                 const nnvm::Op* fn_array,\n                 const nnvm::Op* lfn_scalar,\n                 const nnvm::Op* rfn_scalar) {\n  using namespace runtime;\n  NDArray* out = args[2].operator NDArray*();\n  if (args[0].type_code() == kNDArrayHandle) {\n    if (args[1].type_code() == kNDArrayHandle) {\n      UFuncHelper(args[0].operator NDArray*(), args[1].operator NDArray*(), out, ret, fn_array);\n    } else if (args[1].type_code() == kDLInt) {\n      UFuncHelper(args[0].operator NDArray*(), args[1].operator int64_t(), out, ret, lfn_scalar);\n    } else {\n      UFuncHelper(args[0].operator NDArray*(), args[1].operator double(), out, ret, lfn_scalar);\n    }\n  } else if (args[0].type_code() == kDLInt) {\n    UFuncHelper(args[0].operator int64_t(),\n                args[1].operator NDArray*(),\n                out,\n                ret,\n                rfn_scalar ? rfn_scalar : lfn_scalar);\n  } else {\n    UFuncHelper(args[0].operator double(),\n                args[1].operator NDArray*(),\n                out,\n                ret,\n                rfn_scalar ? rfn_scalar : lfn_scalar);\n  }\n}\n\nvoid UFuncHelper(runtime::MXNetArgs args, runtime::MXNetRetValue* ret, const nnvm::Op* op) {\n  using namespace runtime;\n  nnvm::NodeAttrs attrs;\n  attrs.op          = op;\n  NDArray* inputs[] = {args[0].operator NDArray*()};\n  NDArray* out      = args[1].operator NDArray*();\n  NDArray** outputs = out == nullptr ? nullptr : &out;\n  int num_inputs    = 1;\n  int num_outputs   = out != nullptr;\n  auto ndoutputs    = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);\n  if (outputs) {\n    *ret = PythonArg(1);\n  } else {\n    *ret = ndoutputs[0];\n  }\n}\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/ufunc_helper.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file ufunc_helper.h\n * \\brief ufunc helper\n */\n#ifndef MXNET_API_OPERATOR_UFUNC_HELPER_H_\n#define MXNET_API_OPERATOR_UFUNC_HELPER_H_\n#include <mxnet/runtime/packed_func.h>\nnamespace mxnet {\n\n/*\n * Ufunc helper for unary operators\n */\nvoid UFuncHelper(runtime::MXNetArgs args, runtime::MXNetRetValue* ret, const nnvm::Op* fn_array);\n\n/*\n * Ufunc helper for binary operators\n */\nvoid UFuncHelper(runtime::MXNetArgs args,\n                 runtime::MXNetRetValue* ret,\n                 const nnvm::Op* fn_array,\n                 const nnvm::Op* lfn_scalar,\n                 const nnvm::Op* rfn_scalar);\n\n}  // namespace mxnet\n\n#endif  // MXNET_API_OPERATOR_UFUNC_HELPER_H_\n"
  },
  {
    "path": "src/api/operator/utils.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file utils.cc\n * \\brief Utility functions for operator invoke\n */\n#include \"utils.h\"\n#include \"../../imperative/imperative_utils.h\"\n\nnamespace mxnet {\n\nbool is_recording() {\n  return Imperative::Get()->is_recording();\n}\n\nbool is_deferred_compute() {\n  return Imperative::Get()->is_deferred_compute();\n}\n\nvoid SetInOut(std::vector<NDArray*>* ndinputs,\n              std::vector<NDArray*>* ndoutputs,\n              int num_inputs,\n              NDArray** inputs,\n              int* num_outputs,\n              int infered_num_outputs,\n              int num_visible_outputs,\n              NDArray** out_array) {\n  ndinputs->clear();\n  ndinputs->reserve(num_inputs);\n  for (int i = 0; i < num_inputs; ++i) {\n    NDArray* inp = reinterpret_cast<NDArray*>(inputs[i]);\n    if (!features::is_enabled(features::INT64_TENSOR_SIZE)) {\n      if (shape_is_known(inp->shape())) {  // Shape may be unknown after dynamic shape operators\n        CHECK_LT(inp->shape().Size(), (int64_t{1} << 31) - 1)\n            << \"[SetInOut] Size of tensor you are trying to allocate is larger than \"\n               \"2^31 elements. Please build with flag USE_INT64_TENSOR_SIZE=1\";\n      }\n    }\n    ndinputs->emplace_back(inp);\n  }\n\n  ndoutputs->clear();\n  ndoutputs->reserve(infered_num_outputs);\n  if (out_array == nullptr) {\n    for (int i = 0; i < infered_num_outputs; ++i) {\n      ndoutputs->emplace_back(new NDArray());\n    }\n    *num_outputs = num_visible_outputs;\n  } else {\n    CHECK(*num_outputs == infered_num_outputs || *num_outputs == num_visible_outputs)\n        << \"Operator expects \" << infered_num_outputs << \" (all) or \" << num_visible_outputs\n        << \" (visible only) outputs, but got \" << *num_outputs << \" instead.\";\n    for (int i = 0; i < *num_outputs; ++i) {\n      ndoutputs->emplace_back(out_array[i]);\n    }\n    for (int i = *num_outputs; i < infered_num_outputs; ++i) {\n      ndoutputs->emplace_back(new NDArray());\n    }\n  }\n}\n\nstd::vector<NDArray*> Invoke(const nnvm::Op* op,\n                             nnvm::NodeAttrs* attrs,\n                             int num_inputs,\n                             NDArray** inputs,\n                             int* num_outputs,\n                             NDArray** outputs) {\n  int infered_num_outputs;\n  int num_visible_outputs;\n  imperative::SetNumOutputs(op, *attrs, num_inputs, &infered_num_outputs, &num_visible_outputs);\n\n  std::vector<NDArray*> ndinputs, ndoutputs;\n  SetInOut(&ndinputs,\n           &ndoutputs,\n           num_inputs,\n           inputs,\n           num_outputs,\n           infered_num_outputs,\n           num_visible_outputs,\n           outputs);\n\n  if (Imperative::Get()->is_deferred_compute()) {\n    Imperative::Get()->RecordDeferredCompute(std::move(*attrs), ndinputs, ndoutputs);\n  } else {\n    for (NDArray* input : ndinputs) {\n      Imperative::DCInfo::Compute(*input);\n    }\n    auto state = Imperative::Get()->Invoke(Context::CPU(), *attrs, ndinputs, ndoutputs);\n    if (is_recording()) {\n      Imperative::Get()->RecordOp(std::move(*attrs), ndinputs, ndoutputs, state);\n    }\n  }\n  for (int i = *num_outputs; i < infered_num_outputs; ++i)\n    delete ndoutputs[i];\n  return ndoutputs;\n}\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/api/operator/utils.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file utils.h\n * \\brief Utility functions for operator invoke\n */\n#ifndef MXNET_API_OPERATOR_UTILS_H_\n#define MXNET_API_OPERATOR_UTILS_H_\n\n#include <mxnet/base.h>\n#include <nnvm/c_api.h>\n#include <vector>\n#include <string>\n\nnamespace mxnet {\n\nvoid SetInOut(std::vector<NDArray*>* ndinputs,\n              std::vector<NDArray*>* ndoutputs,\n              int num_inputs,\n              NDArray** inputs,\n              int* num_outputs,\n              int infered_num_outputs,\n              int num_visible_outputs,\n              NDArray** out_array);\n\nstd::vector<NDArray*> Invoke(const nnvm::Op* op,\n                             nnvm::NodeAttrs* attrs,\n                             int num_inputs,\n                             NDArray** inputs,\n                             int* num_outputs,\n                             NDArray** outputs);\n\nbool is_recording();\nbool is_deferred_compute();\n\ntemplate <typename T>\nvoid SetAttrDict(nnvm::NodeAttrs* attrs) {\n  if (is_recording() || is_deferred_compute()) {\n    ::dmlc::get<T>(attrs->parsed).SetAttrDict(&(attrs->dict));\n  }\n}\n\ntemplate <typename ValueType, typename T>\nTuple<ValueType> Obj2Tuple(const runtime::ObjectRef& src) {\n  runtime::ADT adt = Downcast<runtime::ADT, runtime::ObjectRef>(src);\n  Tuple<ValueType> ret(adt.size(), 0);\n  for (size_t i = 0; i < adt.size(); ++i) {\n    ret[i] = Downcast<T, runtime::ObjectRef>(adt[i])->value;\n  }\n  return ret;\n}\n\n}  // namespace mxnet\n\n#endif  // MXNET_API_OPERATOR_UTILS_H_\n"
  },
  {
    "path": "src/base.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file base.cc\n * \\brief Implementation of base declarations, e.g. context\n */\n#include <mxnet/base.h>\n\nnamespace mxnet {\n\n#define UNUSED(x) (void)(x)\n\n#if MXNET_USE_CUDA == 1\n// The oldest version of cuda used in upstream MXNet CI testing, both for unix and windows.\n// Users that have rebuilt MXNet against older versions will we advised with a warning to upgrade\n// their systems to match the CI level.  Minimally, users should rerun the CI locally.\n#if defined(_MSC_VER)\n#define MXNET_CI_OLDEST_CUDA_VERSION 9020\n#else\n#define MXNET_CI_OLDEST_CUDA_VERSION 10000\n#endif\n\nvoid Context::CudaLibChecks() {\n  // One-time init here will emit a warning if no gpus or gpu driver is seen.\n  // Also if the user has recompiled their source to a version no longer tested by upstream CI.\n  static bool cuda_lib_checks_performed = []() {\n    if (dmlc::GetEnv(\"MXNET_CUDA_LIB_CHECKING\", true)) {\n      if (!GPUDriverPresent())\n        LOG(WARNING) << \"Please install cuda driver for GPU use.  No cuda driver detected.\";\n      else if (GetGPUCount() == 0)\n        LOG(WARNING) << \"GPU context requested, but no GPUs found.\";\n      else if (CUDA_VERSION < MXNET_CI_OLDEST_CUDA_VERSION)\n        LOG(WARNING) << \"Upgrade advisory: this mxnet has been built against cuda library version \"\n                     << CUDA_VERSION << \", which is older than the oldest version tested by CI (\"\n                     << MXNET_CI_OLDEST_CUDA_VERSION << \").  \"\n                     << \"Set MXNET_CUDA_LIB_CHECKING=0 to quiet this warning.\";\n    }\n    return true;\n  }();\n  UNUSED(cuda_lib_checks_performed);\n}\n#endif  // MXNET_USE_CUDA\n\n#if MXNET_USE_CUDNN == 1\n// The oldest version of CUDNN used in upstream MXNet CI testing, both for unix and windows.\n// Users that have rebuilt MXNet against older versions will we advised with a warning to upgrade\n// their systems to match the CI level.  Minimally, users should rerun the CI locally.\n#if defined(_MSC_VER)\n#define MXNET_CI_OLDEST_CUDNN_VERSION 7600\n#else\n#define MXNET_CI_OLDEST_CUDNN_VERSION 7600\n#endif\n\nvoid Context::CuDNNLibChecks() {\n  // One-time init here will emit a warning if runtime and compile-time cudnn lib versions mismatch.\n  // Also if the user has recompiled their source to a version no longer tested by upstream CI.\n  static bool cudnn_lib_checks_performed = []() {\n    // Don't bother with checks if there are no GPUs visible (e.g. with CUDA_VISIBLE_DEVICES=\"\")\n    if (dmlc::GetEnv(\"MXNET_CUDNN_LIB_CHECKING\", true) && GetGPUCount() > 0) {\n      size_t linkedAgainstCudnnVersion = cudnnGetVersion();\n      if (linkedAgainstCudnnVersion != CUDNN_VERSION)\n        LOG(WARNING) << \"cuDNN lib mismatch: linked-against version \" << linkedAgainstCudnnVersion\n                     << \" != compiled-against version \" << CUDNN_VERSION << \".  \"\n                     << \"Set MXNET_CUDNN_LIB_CHECKING=0 to quiet this warning.\";\n      if (CUDNN_VERSION < MXNET_CI_OLDEST_CUDNN_VERSION)\n        LOG(WARNING) << \"Upgrade advisory: this mxnet has been built against cuDNN lib version \"\n                     << CUDNN_VERSION << \", which is older than the oldest version tested by CI (\"\n                     << MXNET_CI_OLDEST_CUDNN_VERSION << \").  \"\n                     << \"Set MXNET_CUDNN_LIB_CHECKING=0 to quiet this warning.\";\n    }\n    return true;\n  }();\n  UNUSED(cudnn_lib_checks_performed);\n}\n#endif  // MXNET_USE_CUDNN\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/c_api/.clang-tidy",
    "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Disable most clang-tidy checks in the c_api folder.\nChecks: -*,readability-non-const-parameter\n"
  },
  {
    "path": "src/c_api/c_api.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file c_api.cc\n * \\brief C API of mxnet\n */\n#include <vector>\n#include <sstream>\n#include <string>\n#include <mutex>\n#include <memory>\n#include <functional>\n#include <unordered_map>\n#include <utility>\n#include \"dmlc/base.h\"\n#include \"dmlc/logging.h\"\n#include \"dmlc/io.h\"\n#include \"dmlc/memory_io.h\"\n#include \"dmlc/recordio.h\"\n#include \"dmlc/omp.h\"\n#include \"mxnet/base.h\"\n#include \"mxnet/ndarray.h\"\n#include \"mxnet/operator.h\"\n#include \"mxnet/io.h\"\n#include \"mxnet/c_api.h\"\n#include \"mxnet/kvstore.h\"\n#include \"mxnet/rtc.h\"\n#include \"mxnet/storage.h\"\n#include \"mxnet/libinfo.h\"\n#include \"mxnet/imperative.h\"\n#include \"mxnet/lib_api.h\"\n#include \"../initialize.h\"\n#include \"./c_api_common.h\"\n#include \"../operator/custom/custom-inl.h\"\n#include \"../operator/operator_common.h\"\n#include \"../operator/subgraph/common.h\"\n#include \"../operator/tensor/matrix_op-inl.h\"\n#include \"../operator/tvmop/op_module.h\"\n#include \"../operator/subgraph/partitioner/custom_subgraph_property.h\"\n#include \"../operator/subgraph/subgraph_property.h\"\n#include \"../common/alm.h\"\n#include \"../common/utils.h\"\n#include \"../profiler/profiler.h\"\n#include \"../serialization/cnpy.h\"\n#include \"miniz.h\"\n#include \"nnvm/pass_functions.h\"\n\n// FTZ only applies to SSE and AVX instructions.\n#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || \\\n    (defined(_M_IX86_FP) && _M_IX86_FP >= 1)\n#define SUPPORT_FTZ_DMZ 1\n#else\n#define SUPPORT_FTZ_DMZ 0\n#endif\n\n#if SUPPORT_FTZ_DMZ\n#include <immintrin.h>\n#include <xmmintrin.h>\n#endif\n#if SUPPORT_FTZ_DMZ && !defined(_MSC_VER)\n#include <x86intrin.h>\n#endif\n\n#if MXNET_USE_CUDA\n#include <cuda_profiler_api.h>\n#endif\n#include \"../common/cuda/nvtx.h\"\n\nusing namespace mxnet;\n\n// Internal function to get the information\n// from function registry\n// Used to implement MXSymbolGetAtomicSymbolInfo and MXFuncGetInfo\ntemplate <typename FunRegType>\ninline int MXAPIGetFunctionRegInfo(const FunRegType* e,\n                                   const char** name,\n                                   const char** description,\n                                   uint32_t* num_args,\n                                   const char*** arg_names,\n                                   const char*** arg_type_infos,\n                                   const char*** arg_descriptions,\n                                   const char** return_type) {\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n\n  API_BEGIN();\n  *name        = e->name.c_str();\n  *description = e->description.c_str();\n  *num_args    = static_cast<uint32_t>(e->arguments.size());\n  if (return_type)\n    *return_type = e->return_type.c_str();\n  ret->ret_vec_charp.clear();\n  for (size_t i = 0; i < e->arguments.size(); ++i) {\n    ret->ret_vec_charp.push_back(e->arguments[i].name.c_str());\n  }\n  for (size_t i = 0; i < e->arguments.size(); ++i) {\n    ret->ret_vec_charp.push_back(e->arguments[i].type_info_str.c_str());\n  }\n  for (size_t i = 0; i < e->arguments.size(); ++i) {\n    ret->ret_vec_charp.push_back(e->arguments[i].description.c_str());\n  }\n  *arg_names        = dmlc::BeginPtr(ret->ret_vec_charp);\n  *arg_type_infos   = dmlc::BeginPtr(ret->ret_vec_charp) + e->arguments.size();\n  *arg_descriptions = dmlc::BeginPtr(ret->ret_vec_charp) + (e->arguments.size() * 2);\n  API_END();\n}\n\n// NOTE: return value is added in API_END\n\nstd::string getExtensionMsgs(mxnet::ext::msgSize_t msgSize, mxnet::ext::msgGet_t msgGet) {\n  std::string str;\n  if (msgSize() > 0) {\n    str = \"\\nExtension Traceback:\\n\";\n    for (int i = 0; i < msgSize(); i++) {\n      const char* tmp;\n      msgGet(i, &tmp);\n      // format: [i] message\n      str += std::string(\"\\t[\") + std::to_string(i) + std::string(\"] \") + std::string(tmp) +\n             std::string(\"\\n\");\n    }\n  }\n  return str;\n}\n\n/*!\n * \\brief Common compute function dispatcher for forward/backward and stateful forward/backward\n * state_ptr will be nullptr for regular ops; fcomp_fp is nullptr for stateful ops\n */\nvoid CustomFComputeDispatcher(const std::string op_name,\n                              const mxnet::ext::opCallFComp_t callFComp,\n                              const mxnet::ext::fcomp_t fcomp_fp,\n                              const nnvm::NodeAttrs* attrs,\n                              const mxnet::ext::opCallFStatefulComp_t callFStatefulComp,\n                              int stateful_forward_flag,\n                              const OpStatePtr* state_ptr,\n                              const OpContext& ctx,\n                              const std::vector<NDArray>& inputs,\n                              const std::vector<OpReqType>& req,\n                              const std::vector<NDArray>& outputs,\n                              mxnet::ext::msgSize_t msgSize,\n                              mxnet::ext::msgGet_t msgGet) {\n  using namespace mxnet::ext;\n\n  std::vector<void*> in_data, out_data;\n  std::vector<const int64_t*> in_shapes, out_shapes;\n  std::vector<int> in_dims, out_dims;\n  std::vector<int> in_types, out_types;\n  std::vector<size_t> in_verIDs, out_verIDs;\n  std::vector<const char*> in_dev_type, out_dev_type;\n  std::vector<int> in_dev_id, out_dev_id;\n  std::vector<NDArray> conv_dnnl;  // converted NDArrays from DNNL format\n\n  // Extra data for sparse inputs and outputs.\n  std::vector<int> in_stypes(inputs.size(), 0), out_stypes(outputs.size(), 0);\n  std::vector<void*> in_indices(inputs.size(), nullptr), out_indices(outputs.size(), nullptr);\n  std::vector<void*> in_indptr(inputs.size(), nullptr), out_indptr(outputs.size(), nullptr);\n  std::vector<int64_t> in_indices_shapes(inputs.size(), 0), out_indices_shapes(outputs.size(), 0);\n  std::vector<int64_t> in_indptr_shapes(inputs.size(), 0), out_indptr_shapes(outputs.size(), 0);\n\n  // convert inputs/outpus NDArray to C types to be passed to lib_api.h\n  for (size_t i = 0; i < inputs.size(); i++) {\n    NDArray const* in_nd = &(inputs[i]);\n#if MXNET_USE_ONEDNN == 1\n    // reorder data if in DNNL format\n    if (in_nd->IsDNNLData()) {\n      // convert from DNNL\n      conv_dnnl.push_back(in_nd->Reorder2Default());\n      in_nd = &(conv_dnnl.back());\n    }\n#endif\n    // pull out parts to pass over to library\n    in_data.push_back(in_nd->data().dptr_);\n    in_shapes.push_back(in_nd->shape().data());\n    in_dims.push_back(in_nd->shape().ndim());\n    in_types.push_back(in_nd->dtype());\n    in_verIDs.push_back(in_nd->version());\n    // string repr of supported context for custom library, currently only \"cpu\" and \"gpu\"\n    const char* ctx_str = in_nd->ctx().dev_mask() == Context::kCPU ? \"cpu\" : \"gpu\";\n    in_dev_type.push_back(ctx_str);\n\n    in_dev_id.push_back(in_nd->ctx().real_dev_id());\n    if (inputs[i].storage_type() == mxnet::kRowSparseStorage) {\n      in_stypes[i]         = 1;\n      in_indices[i]        = inputs[i].aux_data(rowsparse::kIdx).dptr_;\n      in_indices_shapes[i] = inputs[i].aux_shape(rowsparse::kIdx).Size();\n    } else if (inputs[i].storage_type() == mxnet::kCSRStorage) {\n      in_stypes[i]         = 2;\n      in_indices[i]        = inputs[i].aux_data(csr::kIdx).dptr_;\n      in_indptr[i]         = inputs[i].aux_data(csr::kIndPtr).dptr_;\n      in_indices_shapes[i] = inputs[i].aux_shape(csr::kIdx).Size();\n      in_indptr_shapes[i]  = inputs[i].aux_shape(csr::kIndPtr).Size();\n    }\n  }\n\n  for (size_t i = 0; i < outputs.size(); i++) {\n    out_data.push_back(outputs[i].data().dptr_);\n    out_shapes.push_back(outputs[i].shape().data());\n    out_dims.push_back(outputs[i].shape().ndim());\n    out_types.push_back(outputs[i].dtype());\n    out_verIDs.push_back(outputs[i].version());\n    const char* ctx_str = outputs[i].ctx().dev_mask() == Context::kCPU ? \"cpu\" : \"gpu\";\n    out_dev_type.push_back(ctx_str);\n    out_dev_id.push_back(outputs[i].ctx().real_dev_id());\n\n    if (outputs[i].storage_type() == mxnet::kRowSparseStorage) {\n      out_stypes[i]         = 1;\n      out_indices[i]        = outputs[i].aux_data(rowsparse::kIdx).dptr_;\n      out_indices_shapes[i] = outputs[i].aux_shape(rowsparse::kIdx).Size();\n    } else if (outputs[i].storage_type() == mxnet::kCSRStorage) {\n      out_stypes[i]         = 2;\n      out_indices[i]        = outputs[i].aux_data(csr::kIdx).dptr_;\n      out_indptr[i]         = outputs[i].aux_data(csr::kIndPtr).dptr_;\n      out_indices_shapes[i] = outputs[i].aux_shape(csr::kIdx).Size();\n      out_indptr_shapes[i]  = outputs[i].aux_shape(csr::kIndPtr).Size();\n    }\n  }\n\n  // get memory resource and mxnet backend streams\n  CHECK(ctx.requested.size() >= 2)\n      << \"Custom operator should register at least memory resource and parallel random resource\";\n  const Resource& resource                = ctx.requested.at(0);\n  mshadow::Stream<mxnet::cpu>* cpu_stream = ctx.get_stream<mxnet::cpu>();\n  mshadow::Stream<mxnet::gpu>* gpu_stream = ctx.get_stream<mxnet::gpu>();\n\n  // create lambda that captures stream & resource objects\n  // this temp workspace holds memory allocated by custom library via OpResource\n  auto cpu_alloc = [&](int size) {\n    mshadow::Tensor<mxnet::cpu, 1, char> workspace =\n        resource.get_space_typed<mxnet::cpu, 1, char>(mshadow::Shape1(size), cpu_stream);\n    return workspace.dptr_;\n  };\n  auto gpu_alloc = [&](int size) {\n    mshadow::Tensor<mxnet::gpu, 1, char> workspace =\n        resource.get_space_typed<mxnet::gpu, 1, char>(mshadow::Shape1(size), gpu_stream);\n    return workspace.dptr_;\n  };\n\n  // create lambda that allocates memory for sparse and\n  // returns allocated arrays for data, indices and indptr.\n  auto sparse_alloc = [&](int index,\n                          int indices_len,\n                          int idxptr_len,\n                          void** data,\n                          int64_t** indices,\n                          int64_t** indptr) {\n    if (idxptr_len == 0) {\n      // Row Sparse\n      outputs[index].CheckAndAlloc({mshadow::Shape1(indices_len)});\n      *data    = outputs[index].data().dptr_;\n      *indices = reinterpret_cast<int64_t*>(outputs[index].aux_data(rowsparse::kIdx).dptr_);\n    } else {\n      // CSR\n      outputs[index].CheckAndAlloc({mshadow::Shape1(idxptr_len), mshadow::Shape1(indices_len)});\n      *data    = outputs[index].data().dptr_;\n      *indices = reinterpret_cast<int64_t*>(outputs[index].aux_data(csr::kIdx).dptr_);\n      *indptr  = reinterpret_cast<int64_t*>(outputs[index].aux_data(csr::kIndPtr).dptr_);\n    }\n  };\n\n  // create no-capture lambda so that we can cast it to function pointer\n  // lambda with captures cannot be cast to function pointer and pass to lib_api.h\n  // this needs to be a lambda function so that we can do the decltype cast\n  typedef decltype(cpu_alloc) alloc_type_cpu;\n  auto cpu_malloc = [](void* _cpu_alloc, int size) {\n    // cast the void* argument to the type for the cpu_alloc lambda function\n    alloc_type_cpu* cpualloc = static_cast<alloc_type_cpu*>(_cpu_alloc);\n    // call cpu_alloc to actually allocate memory and return the pointer\n    return static_cast<void*>((*cpualloc)(size));\n  };\n\n  using alloc_type_gpu = decltype(gpu_alloc);\n  auto gpu_malloc      = [](void* _gpu_alloc, int size) {\n    alloc_type_gpu* gpualloc = static_cast<alloc_type_gpu*>(_gpu_alloc);\n    return static_cast<void*>((*gpualloc)(size));\n  };\n\n  using alloc_type_sparse = decltype(sparse_alloc);\n  auto sparse_malloc      = [](void* _sparse_alloc,\n                          int index,\n                          int indices_len,\n                          int idxptr_len,\n                          void** data,\n                          int64_t** indices,\n                          int64_t** indptr) {\n    alloc_type_sparse* sparsealloc = static_cast<alloc_type_sparse*>(_sparse_alloc);\n    (*sparsealloc)(index, indices_len, idxptr_len, data, indices, indptr);\n  };\n\n  // get actual cudaStream_t out of mxnet gpu stream and pass to lib_api.h\n  void* cuda_stream = nullptr;\n#if MXNET_USE_CUDA\n  if ((inputs.size() > 0 && inputs[0].ctx().dev_mask() == Context::kGPU) ||\n      (outputs.size() > 0 && outputs[0].ctx().dev_mask() == Context::kGPU)) {\n    cuda_stream = static_cast<void*>(gpu_stream->stream_);\n  }\n#endif\n\n  // get mxnet initialized and seeded RNG states and pass to lib_api.h\n  void *rng_cpu_states = nullptr, *rng_gpu_states = nullptr;\n  using mxnet::common::random::RandGenerator;\n  RandGenerator<cpu, float>* pgen_cpu = ctx.requested.at(1).get_parallel_random<cpu, float>();\n  rng_cpu_states                      = pgen_cpu->GetStates();\n#if MXNET_USE_CUDA\n  RandGenerator<gpu, float>* pgen_gpu = ctx.requested.at(1).get_parallel_random<gpu, float>();\n  rng_gpu_states                      = pgen_gpu->GetStates();\n#endif\n\n  CHECK((fcomp_fp != nullptr && state_ptr == nullptr) ||\n        (fcomp_fp == nullptr && state_ptr != nullptr))\n      << \"Can only register either regular op or stateful op for '\" << op_name << \"'\";\n\n  if (fcomp_fp != nullptr) {\n    // convert attributes to vector of char*\n    std::vector<const char*> attr_keys, attr_vals;\n    for (auto& kv : attrs->dict) {\n      attr_keys.push_back(kv.first.c_str());\n      attr_vals.push_back(kv.second.c_str());\n    }\n\n    // call fcompute function\n    int retval       = callFComp(fcomp_fp,\n                           attr_keys.data(),\n                           attr_vals.data(),\n                           attr_keys.size(),\n                           in_shapes.data(),\n                           in_dims.data(),\n                           in_data.data(),\n                           in_types.data(),\n                           in_verIDs.data(),\n                           in_dev_type.data(),\n                           in_dev_id.data(),\n                           in_data.size(),\n                           out_shapes.data(),\n                           out_dims.data(),\n                           out_data.data(),\n                           out_types.data(),\n                           out_verIDs.data(),\n                           out_dev_type.data(),\n                           out_dev_id.data(),\n                           out_data.size(),\n                           cpu_malloc,\n                           &cpu_alloc,\n                           gpu_malloc,\n                           &gpu_alloc,\n                           cuda_stream,\n                           sparse_malloc,\n                           &sparse_alloc,\n                           in_stypes.data(),\n                           out_stypes.data(),\n                           in_indices.data(),\n                           out_indices.data(),\n                           in_indptr.data(),\n                           out_indptr.data(),\n                           in_indices_shapes.data(),\n                           out_indices_shapes.data(),\n                           in_indptr_shapes.data(),\n                           out_indptr_shapes.data(),\n                           rng_cpu_states,\n                           rng_gpu_states);\n    std::string msgs = getExtensionMsgs(msgSize, msgGet);\n    CHECK(retval) << \"Error calling FCompute for custom operator '\" << op_name << \"'\" << msgs;\n  }\n\n  if (state_ptr != nullptr) {\n    // retrieve op state object created from CreateOpState\n    CustomStatefulOpWrapper& op     = state_ptr->get_state<CustomStatefulOpWrapper>();\n    CustomStatefulOp* state_op_inst = op.get_instance();\n    std::string msgs                = getExtensionMsgs(msgSize, msgGet);\n    CHECK(state_op_inst != nullptr)\n        << \"Error custom stateful operator is null for operator '\" << op_name << \"'\" << msgs;\n\n    // call fcompute function\n    int retval = callFStatefulComp(stateful_forward_flag,\n                                   state_op_inst,\n                                   in_shapes.data(),\n                                   in_dims.data(),\n                                   in_data.data(),\n                                   in_types.data(),\n                                   in_verIDs.data(),\n                                   in_dev_type.data(),\n                                   in_dev_id.data(),\n                                   in_data.size(),\n                                   out_shapes.data(),\n                                   out_dims.data(),\n                                   out_data.data(),\n                                   out_types.data(),\n                                   out_verIDs.data(),\n                                   out_dev_type.data(),\n                                   out_dev_id.data(),\n                                   out_data.size(),\n                                   cpu_malloc,\n                                   &cpu_alloc,\n                                   gpu_malloc,\n                                   &gpu_alloc,\n                                   cuda_stream,\n                                   sparse_malloc,\n                                   &sparse_alloc,\n                                   in_stypes.data(),\n                                   out_stypes.data(),\n                                   in_indices.data(),\n                                   out_indices.data(),\n                                   in_indptr.data(),\n                                   out_indptr.data(),\n                                   in_indices_shapes.data(),\n                                   out_indices_shapes.data(),\n                                   in_indptr_shapes.data(),\n                                   out_indptr_shapes.data(),\n                                   rng_cpu_states,\n                                   rng_gpu_states);\n    msgs       = getExtensionMsgs(msgSize, msgGet);\n    CHECK(retval) << \"Error calling FStatefulCompute for custom operator '\" << op_name << \"'\"\n                  << msgs;\n  }\n}\n\ntemplate <typename RescReq,\n          typename AttrParser,\n          typename NumInputs,\n          typename NumOutputs,\n          typename NumInOuts,\n          typename InferType,\n          typename InferShape,\n          typename InferSType,\n          typename MutateInputs,\n          typename SubgraphNumInputs,\n          typename SubgraphInferType,\n          typename SubgraphInferShape,\n          typename SubgraphInferSType,\n          typename CreateOpState,\n          typename GradReg>\nvoid registerOp(const char* name,\n                const std::string& name_str,\n                bool isSubgraphOp,\n                RescReq resc_req,\n                AttrParser attr_parser,\n                NumInputs num_inputs,\n                NumOutputs num_outputs,\n                NumInOuts num_inouts,\n                InferType infer_type,\n                InferShape infer_shape,\n                InferSType infer_storage_type,\n                MutateInputs mutate_inputs,\n                SubgraphNumInputs num_subgraph_inputs,\n                SubgraphInferType infer_subgraph_type,\n                SubgraphInferShape infer_subgraph_shape,\n                SubgraphInferSType infer_subgraph_storage_type,\n                CreateOpState create_opstate,\n                GradReg grad_reg,\n                mxnet::ext::mutateInputs_t mutate_fp,\n                const std::unordered_map<std::string, mxnet::ext::createOpState_t>& createop_map,\n                const std::unordered_map<std::string, mxnet::ext::fcomp_t>& forward_ctx_map,\n                const std::unordered_map<std::string, mxnet::ext::fcomp_t>& backward_ctx_map,\n                mxnet::ext::opCallFComp_t callFComp,\n                mxnet::ext::opCallFStatefulComp_t callFStatefulComp,\n                mxnet::ext::msgSize_t msgSize,\n                mxnet::ext::msgGet_t msgGet) {\n  using namespace mxnet::ext;\n\n  // check if operator is already registered\n  const nnvm::Op* regOpPtr = dmlc::Registry<nnvm::Op>::Get()->Find(name);\n  nnvm::Op& regOp          = dmlc::Registry<nnvm::Op>::Get()->__REGISTER_OR_GET__(name);\n  int plevel               = 10;\n  if (regOpPtr != nullptr) {\n    // overwrite registration of existing op with custom op\n    regOp.arguments.clear();\n    // set attribute with higher plevel (11) to allow re-registering once\n    // TODO(samskalicky): enable constant overwriting of registertion multiple times\n    plevel++;\n  }\n  // define supported resources for both subgraph ops and regular ops\n  regOp.set_attr<FResourceRequest>(\"FResourceRequest\", resc_req, plevel);\n  if (!isSubgraphOp) {\n    regOp.set_attr_parser(attr_parser);\n    regOp.set_num_inputs(num_inputs);\n    regOp.set_num_outputs(num_outputs);\n    regOp.set_attr<nnvm::FInferType>(\"FInferType\", infer_type, plevel);\n    regOp.set_attr<FInferStorageType>(\"FInferStorageType\", infer_storage_type, plevel);\n    regOp.set_attr<mxnet::FInferShape>(\"FInferShape\", infer_shape, plevel);\n    // optionally add fmutate inputs if user specified a function\n    if (mutate_fp != nullptr)\n      regOp.set_attr<nnvm::FMutateInputs>(\"FMutateInputs\", mutate_inputs, plevel);\n  } else {\n    using namespace mxnet::op;\n    regOp.set_num_inputs(num_subgraph_inputs);\n    regOp.set_num_outputs(DefaultSubgraphOpNumOutputs);\n    regOp.set_attr<nnvm::FInferType>(\"FInferType\", infer_subgraph_type, plevel);\n    regOp.set_attr<mxnet::FInferShape>(\"FInferShape\", infer_subgraph_shape, plevel);\n    regOp.set_attr<FInferStorageType>(\"FInferStorageType\", infer_subgraph_storage_type, plevel);\n    regOp.set_attr<nnvm::FMutateInputs>(\"FMutateInputs\", DefaultSubgraphOpMutableInputs, plevel);\n  }\n  // optionally add stateful forward\n  if (createop_map.size() != 0) {\n    regOp.set_attr<FCreateOpState>(\"FCreateOpState\", create_opstate, plevel);\n    auto fstate_forward = [=](const OpStatePtr& state_ptr,\n                              const OpContext& ctx,\n                              const std::vector<NDArray>& inputs,\n                              const std::vector<OpReqType>& req,\n                              const std::vector<NDArray>& outputs) {\n      CustomFComputeDispatcher(name_str,\n                               nullptr,\n                               nullptr,\n                               nullptr,\n                               callFStatefulComp,\n                               1,\n                               &state_ptr,\n                               ctx,\n                               inputs,\n                               req,\n                               outputs,\n                               msgSize,\n                               msgGet);\n    };\n    if (createop_map.count(\"cpu\") > 0)\n      regOp.set_attr<FStatefulComputeEx>(\"FStatefulComputeEx<cpu>\", fstate_forward, plevel);\n    if (createop_map.count(\"gpu\") > 0)\n      regOp.set_attr<FStatefulComputeEx>(\"FStatefulComputeEx<gpu>\", fstate_forward, plevel);\n  } else {\n    auto forward_lambda = [=](const nnvm::NodeAttrs& attrs,\n                              const OpContext& ctx,\n                              const std::vector<NDArray>& inputs,\n                              const std::vector<OpReqType>& req,\n                              const std::vector<NDArray>& outputs) {\n      if (ctx.run_ctx.ctx.dev_mask() == Context::kCPU) {\n        CHECK_GT(forward_ctx_map.count(\"cpu\"), 0);\n        fcomp_t fcomp = forward_ctx_map.at(\"cpu\");\n        CustomFComputeDispatcher(name_str,\n                                 callFComp,\n                                 fcomp,\n                                 &attrs,\n                                 nullptr,\n                                 0,\n                                 nullptr,\n                                 ctx,\n                                 inputs,\n                                 req,\n                                 outputs,\n                                 msgSize,\n                                 msgGet);\n      } else if (ctx.run_ctx.ctx.dev_mask() == Context::kGPU) {\n        CHECK_GT(forward_ctx_map.count(\"gpu\"), 0);\n        fcomp_t fcomp = forward_ctx_map.at(\"gpu\");\n        CustomFComputeDispatcher(name_str,\n                                 callFComp,\n                                 fcomp,\n                                 &attrs,\n                                 nullptr,\n                                 0,\n                                 nullptr,\n                                 ctx,\n                                 inputs,\n                                 req,\n                                 outputs,\n                                 msgSize,\n                                 msgGet);\n      }\n    };\n    if (forward_ctx_map.count(\"cpu\") > 0)\n      regOp.set_attr<FComputeEx>(\"FComputeEx<cpu>\", forward_lambda, plevel);\n    if (forward_ctx_map.count(\"gpu\") > 0)\n      regOp.set_attr<FComputeEx>(\"FComputeEx<gpu>\", forward_lambda, plevel);\n  }\n  // optionally add fgradient if user specified a function, or for stateful ops\n  if (backward_ctx_map.size() != 0 || createop_map.size() != 0) {\n    std::string grad_name = \"_backward_\" + name_str;\n    nnvm::Op& gradOp      = dmlc::Registry<nnvm::Op>::Get()->__REGISTER_OR_GET__(grad_name);\n    regOp.set_attr<nnvm::FGradient>(\"FGradient\", grad_reg, plevel);\n    gradOp.set_attr<nnvm::TIsBackward>(\"TIsBackward\", true, plevel);\n    gradOp.set_attr<FInferStorageType>(\"FInferStorageType\", infer_storage_type, plevel);\n    gradOp.set_attr<FResourceRequest>(\"FResourceRequest\", resc_req, plevel);\n\n    if (!isSubgraphOp) {\n      // register attr parser and standard functions for non-subgraph ops\n      gradOp.set_attr_parser(attr_parser);\n      gradOp.set_num_inputs(num_inouts);\n      gradOp.set_num_outputs(num_inputs);\n    } else {\n      // for subgraph ops use special functions that do not invoke attr_parser\n      using namespace mxnet::op;\n      auto grad_inouts = [=](const nnvm::NodeAttrs& attrs) {\n        // for backward passes, inputs + outputs + input gradients (one for each output)\n        uint32_t cnt = num_subgraph_inputs(attrs);\n        cnt += 2 * DefaultSubgraphOpNumOutputs(attrs);\n        return cnt;\n      };\n      gradOp.set_num_inputs(grad_inouts);\n      gradOp.set_num_outputs(num_subgraph_inputs);\n    }\n\n    if (createop_map.size() != 0) {\n      // for stateful operators\n      gradOp.set_attr<bool>(\"TIsLayerOpBackward\", true, plevel);\n      auto fstate_backward = [=](const OpStatePtr& state_ptr,\n                                 const OpContext& ctx,\n                                 const std::vector<NDArray>& inputs,\n                                 const std::vector<OpReqType>& req,\n                                 const std::vector<NDArray>& outputs) {\n        CustomFComputeDispatcher(name_str,\n                                 nullptr,\n                                 nullptr,\n                                 nullptr,\n                                 callFStatefulComp,\n                                 0,\n                                 &state_ptr,\n                                 ctx,\n                                 inputs,\n                                 req,\n                                 outputs,\n                                 msgSize,\n                                 msgGet);\n      };\n      gradOp.set_attr<FStatefulComputeEx>(\"FStatefulComputeEx<cpu>\", fstate_backward, plevel);\n      gradOp.set_attr<FStatefulComputeEx>(\"FStatefulComputeEx<gpu>\", fstate_backward, plevel);\n    } else {\n      // for stateless operators\n      if (backward_ctx_map.count(\"cpu\") > 0) {\n        fcomp_t fcomp_back_cpu   = backward_ctx_map.at(\"cpu\");\n        auto backward_cpu_lambda = [=](const nnvm::NodeAttrs& attrs,\n                                       const OpContext& ctx,\n                                       const std::vector<NDArray>& inputs,\n                                       const std::vector<OpReqType>& req,\n                                       const std::vector<NDArray>& outputs) {\n          CustomFComputeDispatcher(name_str,\n                                   callFComp,\n                                   fcomp_back_cpu,\n                                   &attrs,\n                                   nullptr,\n                                   0,\n                                   nullptr,\n                                   ctx,\n                                   inputs,\n                                   req,\n                                   outputs,\n                                   msgSize,\n                                   msgGet);\n        };\n        gradOp.set_attr<FComputeEx>(\"FComputeEx<cpu>\", backward_cpu_lambda, plevel);\n      }\n      if (backward_ctx_map.count(\"gpu\") > 0) {\n        fcomp_t fcomp_back_gpu   = backward_ctx_map.at(\"gpu\");\n        auto backward_gpu_lambda = [=](const nnvm::NodeAttrs& attrs,\n                                       const OpContext& ctx,\n                                       const std::vector<NDArray>& inputs,\n                                       const std::vector<OpReqType>& req,\n                                       const std::vector<NDArray>& outputs) {\n          CustomFComputeDispatcher(name_str,\n                                   callFComp,\n                                   fcomp_back_gpu,\n                                   &attrs,\n                                   nullptr,\n                                   0,\n                                   nullptr,\n                                   ctx,\n                                   inputs,\n                                   req,\n                                   outputs,\n                                   msgSize,\n                                   msgGet);\n        };\n        gradOp.set_attr<FComputeEx>(\"FComputeEx<gpu>\", backward_gpu_lambda, plevel);\n      }\n    }\n  }\n  regOp.add_argument(\"data\", \"NDArray[]\", \"Source inputs\");\n}\n\nvoid registerOperators(void* lib,\n                       int verbose,\n                       mxnet::ext::msgSize_t msgSize,\n                       mxnet::ext::msgGet_t msgGet) {\n  using namespace mxnet::ext;\n\n  // get C type interface functions\n  opCallFree_t callFree = get_func<opCallFree_t>(lib, const_cast<char*>(MXLIB_OPCALLFREE_STR));\n\n  opCallParseAttrs_t callParseAttrs =\n      get_func<opCallParseAttrs_t>(lib, const_cast<char*>(MXLIB_OPCALLPARSEATTRS_STR));\n\n  opCallInferShape_t callInferShape =\n      get_func<opCallInferShape_t>(lib, const_cast<char*>(MXLIB_OPCALLINFERSHAPE_STR));\n\n  opCallInferType_t callInferType =\n      get_func<opCallInferType_t>(lib, const_cast<char*>(MXLIB_OPCALLINFERTYPE_STR));\n\n  opCallInferSType_t callInferSType =\n      get_func<opCallInferSType_t>(lib, const_cast<char*>(MXLIB_OPCALLINFERSTYPE_STR));\n\n  opCallFComp_t callFComp = get_func<opCallFComp_t>(lib, const_cast<char*>(MXLIB_OPCALLFCOMP_STR));\n\n  opCallMutateInputs_t callMutateInputs =\n      get_func<opCallMutateInputs_t>(lib, const_cast<char*>(MXLIB_OPCALLMUTATEINPUTS_STR));\n\n  opCallCreateOpState_t callCreateOpState =\n      get_func<opCallCreateOpState_t>(lib, const_cast<char*>(MXLIB_OPCALLCREATEOPSTATE_STR));\n\n  opCallDestroyOpState_t callDestroyOpState =\n      get_func<opCallDestroyOpState_t>(lib, const_cast<char*>(MXLIB_OPCALLDESTROYOPSTATE_STR));\n\n  opCallFStatefulComp_t callFStatefulComp =\n      get_func<opCallFStatefulComp_t>(lib, const_cast<char*>(MXLIB_OPCALLFSTATEFULCOMP_STR));\n\n  // get number of operators registered in the library\n  opRegSize_t opRegSize = get_func<opRegSize_t>(lib, const_cast<char*>(MXLIB_OPREGSIZE_STR));\n  int numOps            = opRegSize();\n  if (verbose)\n    LOG(INFO) << \"Found \" << numOps << \" operators in library\";\n\n  /*\n   * Get all custom operators implementation from custom library\n   * loop and register each operator in the library to NNVM\n   */\n  opRegGet_t opRegGet = get_func<opRegGet_t>(lib, const_cast<char*>(MXLIB_OPREGGET_STR));\n  for (int i = 0; i < numOps; i++) {\n    const char* name;\n    // function pointers holding implementation from custom library\n    parseAttrs_t parse_fp = nullptr;\n    inferType_t type_fp   = nullptr;\n    inferSType_t stype_fp = nullptr;\n    inferShape_t shape_fp = nullptr;\n    // optional attributes\n    mutateInputs_t mutate_fp = nullptr;\n    bool isSubgraphOp        = false;\n    int _isSubgraphOp        = 0;\n    // lists of forward and backward function associated with each context\n    const char **forward_ctx, **backward_ctx, **createop_ctx;\n    fcomp_t *forward_fcomp, *backward_fcomp;\n    createOpState_t* createop_fp;\n    int forward_count, backward_count, createop_count;\n\n    // main function to get custom operator implemenation from the custom library\n    opRegGet(i,\n             &name,\n             &_isSubgraphOp,\n             &forward_ctx,\n             &forward_fcomp,\n             &forward_count,\n             &backward_ctx,\n             &backward_fcomp,\n             &backward_count,\n             &createop_ctx,\n             &createop_fp,\n             &createop_count,\n             &parse_fp,\n             &type_fp,\n             &stype_fp,\n             &shape_fp,\n             &mutate_fp);\n\n    // construct maps of context to forward/backward custom library function\n    std::unordered_map<std::string, fcomp_t> forward_ctx_map;\n    std::unordered_map<std::string, fcomp_t> backward_ctx_map;\n    std::unordered_map<std::string, createOpState_t> createop_map;\n    for (int i = 0; i < forward_count; i++) {\n      std::string ctx_str(forward_ctx[i]);\n      forward_ctx_map[ctx_str] = forward_fcomp[i];\n    }\n    for (int i = 0; i < backward_count; i++) {\n      std::string ctx_str(backward_ctx[i]);\n      backward_ctx_map[ctx_str] = backward_fcomp[i];\n    }\n    for (int i = 0; i < createop_count; i++) {\n      std::string ctx_str(createop_ctx[i]);\n      createop_map[ctx_str] = createop_fp[i];\n    }\n    // set bool, dont pass bool across ABI boundary\n    isSubgraphOp = _isSubgraphOp;\n\n    // validate custom operator functions from the dynamic library\n    if (!isSubgraphOp) {\n      CHECK(parse_fp != nullptr) << \"Error loading '\" << name\n                                 << \"' custom op, ParseAttrs function was not set.\";\n      CHECK(forward_ctx_map.size() != 0 || createop_map.size() != 0)\n          << \"Error loading '\" << name\n          << \"' custom op, Forward or CreateOpState function was not set.\";\n      CHECK(type_fp != nullptr) << \"Error loading '\" << name\n                                << \"' custom op, InferType function was not set.\";\n      CHECK(shape_fp != nullptr) << \"Error loading '\" << name\n                                 << \"' custom op, InferShape function was not set.\";\n    } else {\n      CHECK(createop_map.size() != 0)\n          << \"Error loading '\" << name\n          << \"' custom subgraph op, CreateOpState function was not set.\";\n    }\n    if (verbose)\n      LOG(INFO) << \"\\tOp[\" << i << \"] \" << name;\n    if (verbose && isSubgraphOp)\n      LOG(INFO) << \"\\t\\tisSubgraphOp\";\n    std::string name_str(name);\n\n    /*\n     * Below are a series of lambda functions that will be registered in the NNVM op registration\n     * Each one has the standard MXNet signature and converts to types supported by externally\n     * registered operators.\n     */\n\n    // lambda function to call parse attributes\n    auto attr_parser = [=](const NodeAttrs* attrs) {\n      // convert attributes to vector of char\n      std::vector<const char*> attr_keys, attr_vals;\n      for (auto& kv : attrs->dict) {\n        attr_keys.push_back(kv.first.c_str());\n        attr_vals.push_back(kv.second.c_str());\n      }\n      // convert subgraph symbol from node attributes to char*\n      std::string subgraph_json;\n      if (!attrs->subgraphs.empty()) {\n        nnvm::Graph g;\n        g.outputs     = attrs->subgraphs[0].get()->outputs;\n        subgraph_json = nnvm::pass::SaveJSON(g);\n        attr_keys.push_back(MX_STR_SUBGRAPH_SYM_JSON);\n        attr_vals.push_back(subgraph_json.c_str());\n      }\n\n      int num_in  = -1;\n      int num_out = -1;\n      int retval  = callParseAttrs(\n          parse_fp, attr_keys.data(), attr_vals.data(), attr_keys.size(), &num_in, &num_out);\n      std::string msgs = getExtensionMsgs(msgSize, msgGet);\n      CHECK(retval) << \"Error calling ParseAttrs for custom operator '\" << name_str << \"'\" << msgs;\n\n      // return type void\n    };\n\n    // lambda function to call parse attributes and return the number of inputs\n    auto num_inputs = [=](const NodeAttrs& attrs) {\n      // convert attributes to vector of char\n      std::vector<const char*> attr_keys, attr_vals;\n      for (auto& kv : attrs.dict) {\n        attr_keys.push_back(kv.first.c_str());\n        attr_vals.push_back(kv.second.c_str());\n      }\n\n      int num_in  = -1;\n      int num_out = -1;\n      int retval  = callParseAttrs(\n          parse_fp, attr_keys.data(), attr_vals.data(), attr_keys.size(), &num_in, &num_out);\n      std::string msgs = getExtensionMsgs(msgSize, msgGet);\n      CHECK(retval) << \"Error calling ParseAttrs::num_inputs for custom operator '\" << name_str\n                    << \"'\" << msgs;\n\n      // get extra inputs, if exists\n      int extra_inputs = 0;\n      if (attrs.dict.count(MX_STR_EXTRA_INPUTS) > 0)\n        extra_inputs = std::stoi(attrs.dict.at(MX_STR_EXTRA_INPUTS));\n\n      return num_in + extra_inputs;\n    };\n\n    // lambda function to call parse attributes and return the number of inputs for subgraph ops\n    auto num_subgraph_inputs = [=](const NodeAttrs& attrs) {\n      // get number of inputs for subgraph\n      int num_in = mxnet::op::DefaultSubgraphOpNumInputs(attrs);\n\n      // get extra inputs, if exists\n      int extra_inputs = 0;\n      if (attrs.dict.count(MX_STR_EXTRA_INPUTS) > 0)\n        extra_inputs = std::stoi(attrs.dict.at(MX_STR_EXTRA_INPUTS));\n\n      return num_in + extra_inputs;\n    };\n\n    // lambda function to call parse attributes and return the number of outputs\n    auto num_outputs = [=](const NodeAttrs& attrs) {\n      // convert attributes to vector of char*\n      std::vector<const char*> attr_keys, attr_vals;\n      for (auto& kv : attrs.dict) {\n        attr_keys.push_back(kv.first.c_str());\n        attr_vals.push_back(kv.second.c_str());\n      }\n\n      int num_in  = -1;\n      int num_out = -1;\n      int retval  = callParseAttrs(\n          parse_fp, attr_keys.data(), attr_vals.data(), attr_keys.size(), &num_in, &num_out);\n      std::string msgs = getExtensionMsgs(msgSize, msgGet);\n      CHECK(retval) << \"Error calling ParseAttrs::num_outputs for custom operator '\" << name_str\n                    << \"'\" << msgs;\n\n      return num_out;\n    };\n\n    // lambda function to call parse attributes and return the number of inputs and outputs\n    // for backward computation\n    auto num_inouts = [=](const NodeAttrs& attrs) {\n      // convert attributes to vector of char*\n      std::vector<const char*> attr_keys, attr_vals;\n      for (auto& kv : attrs.dict) {\n        attr_keys.push_back(kv.first.c_str());\n        attr_vals.push_back(kv.second.c_str());\n      }\n\n      int num_in  = -1;\n      int num_out = -1;\n      int retval  = callParseAttrs(\n          parse_fp, attr_keys.data(), attr_vals.data(), attr_keys.size(), &num_in, &num_out);\n      std::string msgs = getExtensionMsgs(msgSize, msgGet);\n      CHECK(retval) << \"Error calling ParseAttrs::num_outputs for custom operator '\" << name_str\n                    << \"'\" << msgs;\n      // for backward passes, inputs + outputs + input gradients (one for each output)\n\n      // get extra inputs, if exists\n      int extra_inputs = 0;\n      if (attrs.dict.count(MX_STR_EXTRA_INPUTS) > 0)\n        extra_inputs = std::stoi(attrs.dict.at(MX_STR_EXTRA_INPUTS));\n\n      return num_in + extra_inputs + 2 * num_out;\n    };\n\n    // lambda function to call infer shape\n    auto infer_shape = [=](const nnvm::NodeAttrs& attrs,\n                           mxnet::ShapeVector* in_shape,\n                           mxnet::ShapeVector* out_shape) {\n      // convert attributes to vector of char*\n      std::vector<const char*> attr_keys, attr_vals;\n      for (auto& kv : attrs.dict) {\n        attr_keys.push_back(kv.first.c_str());\n        attr_vals.push_back(kv.second.c_str());\n      }\n\n      // get extra inputs, if exists\n      int extra_inputs = 0;\n      if (attrs.dict.count(MX_STR_EXTRA_INPUTS) > 0)\n        extra_inputs = std::stoi(attrs.dict.at(MX_STR_EXTRA_INPUTS));\n      int num_inputs = in_shape->size() - extra_inputs;\n\n      std::vector<uint32_t*> inshapes(num_inputs);\n      std::vector<int> indims(num_inputs);\n\n      // determine amount of memory needed to store all the input shapes\n      size_t buff_size = 0;\n      for (size_t i = 0; i < num_inputs; ++i)\n        buff_size += (*in_shape)[i].ndim();\n\n      // copy input shapes from ShapeVector to raw memory layout\n      std::vector<uint32_t> inbuff(buff_size);\n      uint32_t* ptr = inbuff.data();\n      for (size_t i = 0; i < num_inputs; ++i) {\n        inshapes[i] = ptr;\n        indims[i]   = (*in_shape)[i].ndim();\n        for (int j = 0; j < (*in_shape)[i].ndim(); ++j, ++ptr) {\n          *ptr = static_cast<uint32_t>((*in_shape)[i][j]);\n        }\n      }\n\n      // modified input shapes will be allocated by infer shape function\n      uint32_t** mod_inshapes = nullptr;\n      int* mod_indims         = nullptr;\n      // output shapes will be allocated by infer shape function\n      uint32_t** outshapes = nullptr;\n      int* outdims         = nullptr;\n\n      int retval       = callInferShape(shape_fp,\n                                  attr_keys.data(),\n                                  attr_vals.data(),\n                                  attr_keys.size(),\n                                  inshapes.data(),\n                                  indims.data(),\n                                  num_inputs,\n                                  &mod_inshapes,\n                                  &mod_indims,\n                                  &outshapes,\n                                  &outdims,\n                                  out_shape->size());\n      std::string msgs = getExtensionMsgs(msgSize, msgGet);\n      CHECK(retval) << \"Error calling InferShape for custom operator '\" << name_str << \"'\" << msgs;\n\n      std::vector<uint32_t*> in_shapes(num_inputs);\n      // determine amount of memory needed to store all the modified input shapes\n      buff_size = 0;\n      for (unsigned i = 0; i < num_inputs; i++) {\n        buff_size += mod_indims[i];\n      }\n\n      // copy modified input shapes from custom op memory to MXNet memory\n      std::vector<uint32_t> mod_inbuff(buff_size);\n      ptr = mod_inbuff.data();\n      for (unsigned i = 0; i < num_inputs; ++i) {\n        in_shapes[i] = ptr;\n        for (int j = 0; j < mod_indims[i]; ++j, ++ptr) {\n          *ptr = static_cast<uint32_t>(mod_inshapes[i][j]);\n        }\n      }\n\n      // assign modified input shapes to ShapeVector\n      for (unsigned i = 0; i < num_inputs; ++i) {\n        SHAPE_ASSIGN_CHECK(*in_shape, i, mxnet::TShape(in_shapes[i], in_shapes[i] + mod_indims[i]));\n      }\n\n      std::vector<uint32_t*> out_shapes(out_shape->size());\n      // determine amount of memory needed to store all the output shapes\n      buff_size = 0;\n      for (unsigned i = 0; i < out_shape->size(); i++) {\n        buff_size += outdims[i];\n      }\n\n      // copy output shapes from custom op memory to MXNet memory\n      std::vector<uint32_t> outbuff(buff_size);\n      ptr = outbuff.data();\n      for (unsigned i = 0; i < out_shape->size(); ++i) {\n        out_shapes[i] = ptr;\n        for (int j = 0; j < outdims[i]; ++j, ++ptr) {\n          *ptr = static_cast<uint32_t>(outshapes[i][j]);\n        }\n      }\n\n      // assign output shapes to ShapeVector\n      for (unsigned i = 0; i < out_shape->size(); ++i) {\n        SHAPE_ASSIGN_CHECK(*out_shape, i, mxnet::TShape(out_shapes[i], out_shapes[i] + outdims[i]));\n      }\n\n      // free memory used by custom op to allocate shapes/dims\n      callFree(mod_indims);\n      for (unsigned i = 0; i < num_inputs; i++) {\n        callFree(mod_inshapes[i]);\n      }\n      callFree(mod_inshapes);\n\n      callFree(outdims);\n      for (unsigned i = 0; i < out_shape->size(); i++) {\n        callFree(outshapes[i]);\n      }\n      callFree(outshapes);\n\n      return true;\n    };\n\n    // lambda function to call infer shape for subgraph ops\n    auto infer_subgraph_shape = [=](const nnvm::NodeAttrs& attrs,\n                                    mxnet::ShapeVector* in_shape,\n                                    mxnet::ShapeVector* out_shape) {\n      // convert attributes to vector of char*\n      std::vector<const char*> attr_keys, attr_vals;\n      for (auto& kv : attrs.dict) {\n        attr_keys.push_back(kv.first.c_str());\n        attr_vals.push_back(kv.second.c_str());\n      }\n\n      // get extra inputs, if exists\n      int extra_inputs = 0;\n      if (attrs.dict.count(MX_STR_EXTRA_INPUTS) > 0)\n        extra_inputs = std::stoi(attrs.dict.at(MX_STR_EXTRA_INPUTS));\n\n      auto in_first                    = in_shape->begin();\n      auto in_last                     = in_first + in_shape->size() - extra_inputs;\n      mxnet::ShapeVector* sg_in_shapes = new mxnet::ShapeVector(in_first, in_last);\n      bool res = mxnet::op::DefaultSubgraphOpShape(attrs, sg_in_shapes, out_shape);\n\n      // assign modified input shapes to ShapeVector\n      for (unsigned i = 0; i < sg_in_shapes->size(); ++i) {\n        SHAPE_ASSIGN_CHECK(*in_shape, i, sg_in_shapes->at(i));\n      }\n      return res;\n    };\n\n    // lambda function to call infer type\n    auto infer_type = [=](const nnvm::NodeAttrs& attrs,\n                          std::vector<int>* in_type,\n                          std::vector<int>* out_type) {\n      // convert attributes to vector of char*\n      std::vector<const char*> attr_keys, attr_vals;\n      for (auto& kv : attrs.dict) {\n        attr_keys.push_back(kv.first.c_str());\n        attr_vals.push_back(kv.second.c_str());\n      }\n\n      // get extra inputs, if exists\n      int extra_inputs = 0;\n      if (attrs.dict.count(MX_STR_EXTRA_INPUTS) > 0)\n        extra_inputs = std::stoi(attrs.dict.at(MX_STR_EXTRA_INPUTS));\n      int num_inputs = in_type->size() - extra_inputs;\n\n      // copy input types from in_type\n      std::vector<int> intypes(*in_type);\n\n      // output types will be populated by inferType function\n      std::vector<int> outtypes(out_type->size());\n\n      int retval       = callInferType(type_fp,\n                                 attr_keys.data(),\n                                 attr_vals.data(),\n                                 attr_keys.size(),\n                                 intypes.data(),\n                                 num_inputs,\n                                 outtypes.data(),\n                                 out_type->size());\n      std::string msgs = getExtensionMsgs(msgSize, msgGet);\n      CHECK(retval) << \"Error calling InferType for custom operator '\" << name_str << \"'\" << msgs;\n\n      // copy and assign modified input types from custom op to MXNet memory\n      for (size_t i = 0; i < num_inputs; i++) {\n        TYPE_ASSIGN_CHECK(*in_type, i, intypes[i]);\n      }\n      // copy and assign output types from custom op to MXNet memory\n      for (size_t i = 0; i < out_type->size(); i++) {\n        TYPE_ASSIGN_CHECK(*out_type, i, outtypes[i]);\n      }\n\n      return true;\n    };\n\n    // lambda function to call infer type for subgraph ops\n    auto infer_subgraph_type =\n        [=](const nnvm::NodeAttrs& attrs, std::vector<int>* in_type, std::vector<int>* out_type) {\n          // convert attributes to vector of char*\n          std::vector<const char*> attr_keys, attr_vals;\n          for (auto& kv : attrs.dict) {\n            attr_keys.push_back(kv.first.c_str());\n            attr_vals.push_back(kv.second.c_str());\n          }\n\n          // get extra inputs, if exists\n          int extra_inputs = 0;\n          if (attrs.dict.count(MX_STR_EXTRA_INPUTS) > 0)\n            extra_inputs = std::stoi(attrs.dict.at(MX_STR_EXTRA_INPUTS));\n\n          auto in_first                 = in_type->begin();\n          auto in_last                  = in_first + in_type->size() - extra_inputs;\n          std::vector<int>* sg_in_types = new std::vector<int>(in_first, in_last);\n\n          bool res = mxnet::op::DefaultSubgraphOpType(attrs, sg_in_types, out_type);\n          // copy and assign modified input types\n          for (size_t i = 0; i < sg_in_types->size(); i++) {\n            TYPE_ASSIGN_CHECK(*in_type, i, sg_in_types->at(i));\n          }\n          return res;\n        };\n\n    // lambda function to convert from external mutate_inputs to internal MXNet types\n    auto mutate_inputs = [=](const nnvm::NodeAttrs& attrs) {\n      // convert attributes to vector of char*\n      std::vector<const char*> attr_keys, attr_vals;\n      for (auto& kv : attrs.dict) {\n        attr_keys.push_back(kv.first.c_str());\n        attr_vals.push_back(kv.second.c_str());\n      }\n\n      // C type placeholder for mutate input indices vector\n      int* mutate_indices = nullptr;\n      int indices_size    = 0;\n\n      // call mutate inputs function\n      int retval       = callMutateInputs(mutate_fp,\n                                    attr_keys.data(),\n                                    attr_vals.data(),\n                                    attr_keys.size(),\n                                    &mutate_indices,\n                                    &indices_size);\n      std::string msgs = getExtensionMsgs(msgSize, msgGet);\n      CHECK(retval) << \"Error calling MutateInputs for custom operator '\" << name_str << \"'\"\n                    << msgs;\n\n      std::vector<uint32_t> mutate_indices_list(indices_size);\n      for (int i = 0; i < indices_size; i++) {\n        mutate_indices_list[i] = static_cast<uint32_t>(mutate_indices[i]);\n      }\n\n      return mutate_indices_list;\n    };\n\n    // lambda function to set storage types\n    auto infer_storage_type = [=](const nnvm::NodeAttrs& attrs,\n                                  const int dev_mask,\n                                  DispatchMode* dispatch_mode,\n                                  std::vector<int>* in_stypes,\n                                  std::vector<int>* out_stypes) {\n      if (stype_fp == nullptr) {\n        // InferSType is not defined in customized lib.\n        CHECK(mxnet::common::ContainsOnlyStorage(*in_stypes, mxnet::kDefaultStorage))\n            << \"Error input tensors are not dense for custom operator '\" << name_str << \"'\";\n        // set outputs as dense\n        return op::storage_type_assign(\n            out_stypes, mxnet::kDefaultStorage, dispatch_mode, DispatchMode::kFComputeEx);\n      } else {\n        // InferSType is defined in customized lib.\n        // convert attributes to vector of char*\n        std::vector<const char*> attr_keys, attr_vals;\n        for (const auto& kv : attrs.dict) {\n          attr_keys.push_back(kv.first.c_str());\n          attr_vals.push_back(kv.second.c_str());\n        }\n\n        // get extra inputs, if exists\n        int extra_inputs = 0;\n        if (attrs.dict.count(MX_STR_EXTRA_INPUTS) > 0)\n          extra_inputs = std::stoi(attrs.dict.at(MX_STR_EXTRA_INPUTS));\n        int num_inputs = in_stypes->size() - extra_inputs;\n\n        // copy input types from in_stype\n        std::vector<int> instypes(*in_stypes);\n\n        // output types will be populated by inferType function\n        std::vector<int> outstypes(out_stypes->size());\n        int retval       = callInferSType(stype_fp,\n                                    attr_keys.data(),\n                                    attr_vals.data(),\n                                    attr_keys.size(),\n                                    instypes.data(),\n                                    num_inputs,\n                                    outstypes.data(),\n                                    out_stypes->size());\n        std::string msgs = getExtensionMsgs(msgSize, msgGet);\n        CHECK(retval) << \"Error calling InferSType for custom operator '\" << name_str << \"'\"\n                      << msgs;\n\n        // copy and assign modified input storage types from custom op to MXNet memory.\n        for (size_t i = 0; i < num_inputs; i++) {\n          STORAGE_TYPE_ASSIGN_CHECK(*in_stypes, i, instypes[i]);\n        }\n        // copy and assign output storage types from custom op to MXNet memory.\n        for (size_t i = 0; i < out_stypes->size(); i++) {\n          STORAGE_TYPE_ASSIGN_CHECK(*out_stypes, i, outstypes[i]);\n        }\n        // assign dispatch mode\n        DISPATCH_MODE_ASSIGN_CHECK(dispatch_mode, 0, DispatchMode::kFComputeEx);\n        return true;\n      }\n    };\n\n    // lambda function to set storage types for subgraph ops\n    auto infer_subgraph_storage_type = [=](const nnvm::NodeAttrs& attrs,\n                                           const int dev_mask,\n                                           DispatchMode* dispatch_mode,\n                                           std::vector<int>* in_stypes,\n                                           std::vector<int>* out_stypes) {\n      // get extra inputs, if exists\n      int extra_inputs = 0;\n      if (attrs.dict.count(MX_STR_EXTRA_INPUTS) > 0)\n        extra_inputs = std::stoi(attrs.dict.at(MX_STR_EXTRA_INPUTS));\n\n      auto in_first                  = in_stypes->begin();\n      auto in_last                   = in_first + in_stypes->size() - extra_inputs;\n      std::vector<int>* sg_in_stypes = new std::vector<int>(in_first, in_last);\n\n      bool res = mxnet::op::DefaultSubgraphOpStorageType(\n          attrs, dev_mask, dispatch_mode, sg_in_stypes, out_stypes);\n      // copy and assign modified input storage types\n      for (size_t i = 0; i < sg_in_stypes->size(); i++) {\n        STORAGE_TYPE_ASSIGN_CHECK(*in_stypes, i, sg_in_stypes->at(i));\n      }\n      return res;\n    };\n\n    // FGradient register lambda\n    auto grad_reg = [=](const nnvm::ObjectPtr& n, const std::vector<nnvm::NodeEntry>& ograds) {\n      // create node for gradient\n      auto p                = nnvm::Node::Create();\n      std::string grad_name = \"_backward_\" + name_str;\n      p->attrs.op           = nnvm::Op::Get(grad_name.c_str());\n      p->attrs.name         = n->attrs.name + \"_backward\";\n      // copy attributes and subgraphs\n      p->attrs.dict = n->attrs.dict;\n      for (const auto& s : n->attrs.subgraphs)\n        p->attrs.subgraphs.push_back(s);\n      // set control dependency and attr parser\n      p->control_deps.emplace_back(n);\n      if (p->op()->attr_parser != nullptr) {\n        p->op()->attr_parser(&(p->attrs));\n      }\n      // gradient inputs: copy gradients first\n      std::vector<nnvm::NodeEntry> heads(ograds.begin(), ograds.end());\n      // copy inputs second\n      for (auto& h : n->inputs) {\n        heads.push_back(h);\n      }\n      // gradient inputs: copy outputs last\n      uint32_t n_out = n->num_outputs();\n      for (uint32_t i = 0; i < n_out; ++i) {\n        heads.emplace_back(n, i, 0);\n      }\n      // set inputs to gradient node\n      p->inputs = heads;\n      CHECK_EQ(p->num_inputs(), p->inputs.size())\n          << \"Number of inputs to operator \" << grad_name << \" (\" << p->num_inputs()\n          << \") does not match the actual number of inputs provided to operator \" << p->attrs.name\n          << \" (\" << p->inputs.size() << \").\";\n      // create output node entries\n      return mxnet::op::CreateNodeEntries(p);\n    };\n\n    auto resc_req = [=](const NodeAttrs& attrs) {\n      return std::vector<ResourceRequest>{ResourceRequest::kTempSpace,\n                                          ResourceRequest::kParallelRandom};\n    };\n\n    // library author should implement and return a 'state' which points to an instance\n    // in lambda we create OpStatePtr using the returned 'state'\n    auto create_opstate = [=](const NodeAttrs& attrs,\n                              Context ctx,\n                              const std::vector<TShape>& in_shapes,\n                              const std::vector<int>& in_types) {\n      // convert attributes to vector of char*\n      std::vector<const char*> attr_keys, attr_vals;\n      for (auto& kv : attrs.dict) {\n        attr_keys.push_back(kv.first.c_str());\n        attr_vals.push_back(kv.second.c_str());\n      }\n\n      // string repr of supported context for custom library, currently only \"cpu\" and \"gpu\"\n      const char* ctx_str = ctx.dev_mask() == Context::kCPU ? \"cpu\" : \"gpu\";\n\n      std::vector<uint32_t*> inshapes(in_shapes.size());\n      std::vector<int> indims(in_shapes.size());\n\n      // determine amount of memory needed to store all the input shapes\n      size_t buff_size = 0;\n      for (const auto& in_shape : in_shapes)\n        buff_size += in_shape.ndim();\n\n      // copy input shapes to raw memory layout\n      std::vector<uint32_t> inbuff(buff_size);\n      uint32_t* ptr = inbuff.data();\n      for (size_t i = 0; i < in_shapes.size(); ++i) {\n        inshapes[i] = ptr;\n        indims[i]   = in_shapes[i].ndim();\n        for (int j = 0; j < in_shapes[i].ndim(); ++j, ++ptr) {\n          *ptr = static_cast<uint32_t>(in_shapes[i][j]);\n        }\n      }\n\n      // convert subgraph symbol from node attributes to char*\n      std::string subgraph_json;\n      if (!attrs.subgraphs.empty()) {\n        nnvm::Graph g;\n        g.outputs     = attrs.subgraphs[0].get()->outputs;\n        subgraph_json = nnvm::pass::SaveJSON(g);\n        attr_keys.push_back(MX_STR_SUBGRAPH_SYM_JSON);\n        attr_vals.push_back(subgraph_json.c_str());\n      }\n\n      // create a pointer to hold custom op state object\n      // only create one stateful op depending on passing context\n      // user can add new supported context and call to custom library\n      void* state_op_inst = nullptr;\n      if (ctx.dev_mask() == Context::kCPU) {\n        CHECK(createop_map.count(\"cpu\") > 0)\n            << \"CPU CreateOpState not implemented for '\" << name_str << \"'\";\n        int retval       = callCreateOpState(createop_map.at(\"cpu\"),\n                                       attr_keys.data(),\n                                       attr_vals.data(),\n                                       attr_keys.size(),\n                                       ctx_str,\n                                       ctx.real_dev_id(),\n                                       inshapes.data(),\n                                       indims.data(),\n                                       in_shapes.size(),\n                                       in_types.data(),\n                                       &state_op_inst);\n        std::string msgs = getExtensionMsgs(msgSize, msgGet);\n        CHECK(retval) << \"Error calling CreateOpState CPU for custom operator '\" << name_str << \"'\"\n                      << msgs;\n      } else if (ctx.dev_mask() == Context::kGPU) {\n        CHECK(createop_map.count(\"gpu\") > 0)\n            << \"GPU CreateOpState not implemented for '\" << name_str << \"'\";\n        int retval       = callCreateOpState(createop_map.at(\"gpu\"),\n                                       attr_keys.data(),\n                                       attr_vals.data(),\n                                       attr_keys.size(),\n                                       ctx_str,\n                                       ctx.real_dev_id(),\n                                       inshapes.data(),\n                                       indims.data(),\n                                       in_shapes.size(),\n                                       in_types.data(),\n                                       &state_op_inst);\n        std::string msgs = getExtensionMsgs(msgSize, msgGet);\n        CHECK(retval) << \"Error calling CreateOpState GPU for custom operator '\" << name_str << \"'\"\n                      << msgs;\n      }\n\n      std::string msgs = getExtensionMsgs(msgSize, msgGet);\n      CHECK(state_op_inst != nullptr)\n          << \"Error custom library failed to create stateful operator '\" << name_str << \"'\" << msgs;\n\n      CustomStatefulOp* state_op = reinterpret_cast<CustomStatefulOp*>(state_op_inst);\n      if (!state_op->wasCreated() && !state_op->ignore_warn)\n        LOG(INFO) << \"WARNING! Custom stateful op \" << state_op_inst << \" was created without \"\n                  << \"calling CustomStatefulOp::create(). Please ensure this object was \"\n                  << \"allocated with 'new' since it will be destructed with 'delete'. \"\n                  << \"To suppress this message without calling CustomStatefulOp::create() \"\n                  << \"set ignore_warn to 'true' on custom stateful op instance.\";\n      return OpStatePtr::Create<CustomStatefulOpWrapper>(state_op, callDestroyOpState);\n    };\n\n    /* -------------- BELOW IS THE REGISTRATION FOR CUSTOM OPERATORS --------------- */\n\n    registerOp(name,\n               name_str,\n               isSubgraphOp,\n               resc_req,\n               attr_parser,\n               num_inputs,\n               num_outputs,\n               num_inouts,\n               infer_type,\n               infer_shape,\n               infer_storage_type,\n               mutate_inputs,\n               num_subgraph_inputs,\n               infer_subgraph_type,\n               infer_subgraph_shape,\n               infer_subgraph_storage_type,\n               create_opstate,\n               grad_reg,\n               mutate_fp,\n               createop_map,\n               forward_ctx_map,\n               backward_ctx_map,\n               callFComp,\n               callFStatefulComp,\n               msgSize,\n               msgGet);\n  }\n}  // NOLINT\n\nvoid registerPartitioners(void* lib,\n                          int verbose,\n                          mxnet::ext::msgSize_t msgSize,\n                          mxnet::ext::msgGet_t msgGet) {\n  using namespace mxnet::ext;\n\n  // get C type interface functions\n  opCallFree_t callFree = get_func<opCallFree_t>(lib, const_cast<char*>(MXLIB_OPCALLFREE_STR));\n\n  partCallSupportedOps_t callSupportedOps =\n      get_func<partCallSupportedOps_t>(lib, const_cast<char*>(MXLIB_PARTCALLSUPPORTEDOPS_STR));\n\n  partCallCreateSelector_t callCreateSelector =\n      get_func<partCallCreateSelector_t>(lib, const_cast<char*>(MXLIB_PARTCALLCREATESELECTOR_STR));\n\n  partCallSelect_t callSelect =\n      get_func<partCallSelect_t>(lib, const_cast<char*>(MXLIB_PARTCALLSELECT_STR));\n\n  partCallSelectInput_t callSelectInput =\n      get_func<partCallSelectInput_t>(lib, const_cast<char*>(MXLIB_PARTCALLSELECTINPUT_STR));\n\n  partCallSelectOutput_t callSelectOutput =\n      get_func<partCallSelectOutput_t>(lib, const_cast<char*>(MXLIB_PARTCALLSELECTOUTPUT_STR));\n\n  partCallFilter_t callFilter =\n      get_func<partCallFilter_t>(lib, const_cast<char*>(MXLIB_PARTCALLFILTER_STR));\n\n  partCallReset_t callReset =\n      get_func<partCallReset_t>(lib, const_cast<char*>(MXLIB_PARTCALLRESET_STR));\n\n  partCallReviewSubgraph_t callReviewSubgraph =\n      get_func<partCallReviewSubgraph_t>(lib, const_cast<char*>(MXLIB_PARTCALLREVIEWSUBGRAPH_STR));\n\n  // get number of partitioners registered in the library\n  partRegSize_t partRegSize =\n      get_func<partRegSize_t>(lib, const_cast<char*>(MXLIB_PARTREGSIZE_STR));\n  int numParts = partRegSize();\n  if (verbose)\n    LOG(INFO) << \"Found \" << numParts << \" partitioners in library\";\n\n  /*\n   * Get all custom partitioners implementation from custom library\n   * loop and register each partitioner in the library to NNVM\n   */\n  partRegGetCount_t partRegGetCount =\n      get_func<partRegGetCount_t>(lib, const_cast<char*>(MXLIB_PARTREGGETCOUNT_STR));\n  partRegGet_t partRegGet = get_func<partRegGet_t>(lib, const_cast<char*>(MXLIB_PARTREGGET_STR));\n  for (int i = 0; i < numParts; i++) {\n    const char* name;\n    // get custom partitioner strategy count from the dynamic library\n    int count = partRegGetCount(i, &name);\n    CHECK(count > 0) << \"Error loading '\" << name << \"' custom partitioner, no strategies defined\";\n    std::string name_str(name);\n    if (verbose)\n      LOG(INFO) << \"\\tPartitioner[\" << i << \"] \" << name;\n\n    mxnet::op::SubgraphBackendRegistry::Get()->__REGISTER_BACKEND__(name);\n\n    for (int j = 0; j < count; j++) {\n      const char* strategy;\n      // function pointers holding implementation from custom library\n      supportedOps_t supportedOps_fp     = nullptr;\n      createSelector_t createSelector_fp = nullptr;\n      reviewSubgraph_t reviewSubgraph_fp = nullptr;\n      // name of subgraph op\n      const char* op_name = nullptr;\n\n      // get custom partitioner strategy from the dynamic library\n      partRegGet(\n          i, j, &strategy, &supportedOps_fp, &createSelector_fp, &reviewSubgraph_fp, &op_name);\n      // validate custom partitioner functions from the dynamic library\n      if (supportedOps_fp == nullptr && createSelector_fp == nullptr)\n        LOG(ERROR) << \"Error loading '\" << name << \"' custom partitioner strategy '\" << strategy\n                   << \"', must implement supportedOps or createSelector\";\n      std::string strategy_str(strategy);\n      std::string op_name_str(op_name);\n      if (verbose)\n        LOG(INFO) << \"\\t\\tStrategy[\" << j << \"] \" << strategy_str << \" subgraphOp: '\" << op_name_str\n                  << \"'\";\n      mxnet::op::SubgraphBackendRegistry::Get()->__REGISTER_CUSTOM_PROPERTY__(\n          name_str,\n          std::make_shared<mxnet::op::CustomSubgraphProperty>(strategy_str,\n                                                              callSupportedOps,\n                                                              supportedOps_fp,\n                                                              callCreateSelector,\n                                                              createSelector_fp,\n                                                              callSelect,\n                                                              callSelectInput,\n                                                              callSelectOutput,\n                                                              callFilter,\n                                                              callReset,\n                                                              callReviewSubgraph,\n                                                              reviewSubgraph_fp,\n                                                              callFree,\n                                                              op_name_str));\n    }\n  }\n}\n\nvoid registerPasses(void* lib,\n                    int verbose,\n                    mxnet::ext::msgSize_t msgSize,\n                    mxnet::ext::msgGet_t msgGet) {\n  using namespace mxnet::ext;\n\n  // get C type interface functions\n  opCallFree_t callFree = get_func<opCallFree_t>(lib, const_cast<char*>(MXLIB_OPCALLFREE_STR));\n\n  passCallGraphPass_t callGraphPass =\n      get_func<passCallGraphPass_t>(lib, const_cast<char*>(MXLIB_PASSCALLGRAPHPASS_STR));\n\n  // get number of passes registered in the library\n  partRegSize_t passRegSize =\n      get_func<passRegSize_t>(lib, const_cast<char*>(MXLIB_PASSREGSIZE_STR));\n  int numPasses = passRegSize();\n  if (verbose)\n    LOG(INFO) << \"Found \" << numPasses << \" graph passes in library\";\n\n  /*\n   * Get all custom pass implementation from custom library\n   * loop and register each pass in the library to NNVM\n   */\n  passRegGet_t passRegGet = get_func<passRegGet_t>(lib, const_cast<char*>(MXLIB_PASSREGGET_STR));\n  for (int i = 0; i < numPasses; i++) {\n    const char* name;\n    // function pointers holding implementation from custom library\n    graphPass_t pass_fp = nullptr;\n\n    // main function to get custom pass implemenation from the custom library\n    passRegGet(i, &pass_fp, &name);\n\n    if (verbose)\n      LOG(INFO) << \"\\tGraph Pass [\" << i << \"] \" << name;\n\n    auto pass_lambda = [=](nnvm::Graph&& g) {\n      // get pass name\n      const char* pass_name = g.GetAttr<const char*>(\"pass_name\");\n      // get options\n      const std::unordered_map<std::string, std::string>& options_map =\n          g.GetAttr<const std::unordered_map<std::string, std::string>>(\"options_map\");\n      // convert options_map_ to char* to pass to backend library\n      std::vector<const char*> opt_keys, opt_vals;\n      for (auto& kv : options_map) {\n        opt_keys.push_back(kv.first.c_str());\n        opt_vals.push_back(kv.second.c_str());\n      }\n\n      // get input args and arg names\n      std::vector<std::string> in_arg_names = g.GetAttr<std::vector<std::string>>(\"in_arg_names\");\n      std::vector<std::string> in_aux_names = g.GetAttr<std::vector<std::string>>(\"in_aux_names\");\n      NDArray** in_args_ptr                 = g.GetAttr<NDArray**>(\"in_args\");\n      NDArray** in_aux_ptr                  = g.GetAttr<NDArray**>(\"in_aux\");\n\n      // get shapes/types\n      mxnet::ShapeVector shapes;\n      if (g.HasAttr(\"shape\"))\n        shapes = g.GetAttr<mxnet::ShapeVector>(\"shape\");\n      std::vector<int> dtypes;\n      if (g.HasAttr(\"dtype\"))\n        dtypes = g.GetAttr<std::vector<int>>(\"dtype\");\n      g.attrs.clear();\n      const nnvm::IndexedGraph& indexed_graph = g.indexed_graph();\n\n      // set shape attrs for each node in the graph\n      if (shapes.size() > 0) {\n        for (unsigned nid = 0; nid < indexed_graph.num_nodes(); nid++) {\n          nnvm::Node* node = const_cast<nnvm::Node*>(indexed_graph[nid].source);\n          std::stringstream ss;\n          ss << \"[\";\n          // set the output shapes for this node\n          for (unsigned oid = 0; oid < node->num_outputs(); oid++) {\n            const uint32_t out_entry_id = indexed_graph.entry_id(nid, oid);\n            mxnet::TShape& shape        = shapes[out_entry_id];\n            ss << shape;\n            if (oid < node->num_outputs() - 1)\n              ss << \",\";\n          }\n          ss << \"]\";\n          node->attrs.dict[MX_STR_SHAPE] = ss.str();\n        }\n      }\n      // set dtype attrs for each node in the graph\n      if (dtypes.size() > 0) {\n        for (unsigned nid = 0; nid < indexed_graph.num_nodes(); nid++) {\n          nnvm::Node* node = const_cast<nnvm::Node*>(indexed_graph[nid].source);\n          std::stringstream ss;\n          ss << \"[\";\n          // set the output dtypes for this node\n          for (unsigned oid = 0; oid < node->num_outputs(); oid++) {\n            const uint32_t out_entry_id = indexed_graph.entry_id(nid, oid);\n            int dtype                   = dtypes[out_entry_id];\n            ss << dtype;\n            if (oid < node->num_outputs() - 1)\n              ss << \",\";\n          }\n          ss << \"]\";\n          node->attrs.dict[MX_STR_DTYPE] = ss.str();\n        }\n      }\n\n      std::vector<const char*> arg_names, aux_names;\n      std::vector<void*> arg_data, aux_data;\n      std::vector<const int64_t*> arg_shapes, aux_shapes;\n      std::vector<int> arg_dims, aux_dims;\n      std::vector<int> arg_types, aux_types;\n      std::vector<size_t> arg_verIDs, aux_verIDs;\n      std::vector<const char*> arg_dev_type, aux_dev_type;\n      std::vector<int> arg_dev_id, aux_dev_id;\n\n      // convert input args\n      for (size_t i = 0; i < in_arg_names.size(); i++) {\n        if (in_args_ptr[i] != nullptr) {\n          arg_names.push_back(in_arg_names[i].c_str());\n          const NDArray& in_arg = *(in_args_ptr[i]);\n\n#if MXNET_USE_ONEDNN == 1\n          // reorder data if in DNNL format\n          if (in_arg.IsDNNLData()) {\n            in_arg.Reorder2DefaultAsync();\n            in_arg.WaitToRead();\n          }\n#endif\n\n          // pull out parts of NDArray to send to backend\n          arg_data.push_back(in_arg.data().dptr_);\n          arg_shapes.push_back(in_arg.shape().data());\n          arg_dims.push_back(in_arg.shape().ndim());\n          arg_types.push_back(in_arg.dtype());\n          arg_verIDs.push_back(in_arg.version());\n          const char* arg_ctx_str = in_arg.ctx().dev_mask() == Context::kCPU ? \"cpu\" : \"gpu\";\n          arg_dev_type.push_back(arg_ctx_str);\n          arg_dev_id.push_back(in_arg.ctx().real_dev_id());\n        }\n      }\n\n      // convert input aux\n      for (size_t i = 0; i < in_aux_names.size(); i++) {\n        if (in_aux_ptr[i] != nullptr) {\n          aux_names.push_back(in_aux_names[i].c_str());\n          const auto& in_aux = *(in_aux_ptr[i]);\n\n#if MXNET_USE_ONEDNN == 1\n          // reorder data if in DNNL format\n          if (in_aux.IsDNNLData()) {\n            in_aux.Reorder2DefaultAsync();\n            in_aux.WaitToRead();\n          }\n#endif\n\n          // pull out parts of NDArray to send to backend\n          aux_data.push_back(in_aux.data().dptr_);\n          aux_shapes.push_back(in_aux.shape().data());\n          aux_dims.push_back(in_aux.shape().ndim());\n          aux_types.push_back(in_aux.dtype());\n          aux_verIDs.push_back(in_aux.version());\n          const char* aux_ctx_str = in_aux.ctx().dev_mask() == Context::kCPU ? \"cpu\" : \"gpu\";\n          aux_dev_type.push_back(aux_ctx_str);\n          aux_dev_id.push_back(in_aux.ctx().real_dev_id());\n        }\n      }\n\n      // convert graph to string\n      std::string in_json = nnvm::pass::SaveJSON(g);\n\n      std::vector<std::string> new_arg_names, new_aux_names;\n      std::vector<NDArray*> new_args, new_aux;\n\n      // create lambda that captures stream & resource objects\n      // this temp workspace holds memory allocated by custom library via OpResource\n      auto ndarray_alloc =\n          [&](const mxnet::TShape& shape, Context ctx, int dtype, std::string name, bool isArg) {\n            NDArray* arr = new NDArray(shape, ctx, false, dtype);\n            if (isArg) {\n              new_args.push_back(arr);\n              new_arg_names.push_back(name);\n            } else {\n              new_aux.push_back(arr);\n              new_aux_names.push_back(name);\n            }\n            return arr;\n          };\n\n      // create no-capture lambda so that we can cast it to function pointer\n      // lambda with captures cannot be cast to function pointer and pass to lib_api.h\n      // this needs to be a lambda function so that we can do the decltype cast\n      using alloc_type_ndarray = decltype(ndarray_alloc);\n      auto ndarray_malloc      = [](const void* _ndarray_alloc,\n                               const int64_t* shapes,\n                               int num_shapes,\n                               const char* dev_str,\n                               int dev_id,\n                               int dtype,\n                               const char* name,\n                               int isArg,\n                               void** data) {\n        mxnet::TShape shape(num_shapes, 0);\n        for (int i = 0; i < num_shapes; i++)\n          shape[i] = shapes[i];\n        int dev_type = -1;\n        if (strcmp(dev_str, \"cpu\") == 0)\n          dev_type = kCPU;\n        else\n          dev_type = kGPU;\n        Context ctx = Context::Create(static_cast<Context::DeviceType>(dev_type), dev_id);\n\n        // cast the void* argument to the type for the cpu_alloc lambda function\n        const alloc_type_ndarray* ndalloc = static_cast<const alloc_type_ndarray*>(_ndarray_alloc);\n        // call cpu_alloc to actually allocate memory and return the pointer\n        NDArray* arr = (*ndalloc)(shape, ctx, dtype, name, isArg);\n        *data        = arr->data().dptr_;\n      };\n\n      char* out_json;\n      int retval       = callGraphPass(pass_fp,\n                                 in_json.c_str(),\n                                 &out_json,\n                                 opt_keys.data(),\n                                 opt_vals.data(),\n                                 opt_keys.size(),\n                                 pass_name,\n                                 arg_names.data(),\n                                 arg_names.size(),\n                                 arg_data.data(),\n                                 arg_shapes.data(),\n                                 arg_dims.data(),\n                                 arg_types.data(),\n                                 arg_verIDs.data(),\n                                 arg_dev_type.data(),\n                                 arg_dev_id.data(),\n                                 aux_names.data(),\n                                 aux_names.size(),\n                                 aux_data.data(),\n                                 aux_shapes.data(),\n                                 aux_dims.data(),\n                                 aux_types.data(),\n                                 aux_verIDs.data(),\n                                 aux_dev_type.data(),\n                                 aux_dev_id.data(),\n                                 ndarray_malloc,\n                                 &ndarray_alloc);\n      std::string msgs = getExtensionMsgs(msgSize, msgGet);\n      CHECK(retval) << \"Error calling graph pass for '\" << pass_name << \"'\" << msgs;\n\n      std::string out_string(out_json);\n      nnvm::Graph out_graph = nnvm::pass::LoadJSON(out_string);\n\n      out_graph.attrs[\"new_args\"]      = std::make_shared<nnvm::any>(new_args);\n      out_graph.attrs[\"new_arg_names\"] = std::make_shared<nnvm::any>(new_arg_names);\n      out_graph.attrs[\"new_aux\"]       = std::make_shared<nnvm::any>(new_aux);\n      out_graph.attrs[\"new_aux_names\"] = std::make_shared<nnvm::any>(new_aux_names);\n\n      callFree(out_json);\n      return out_graph;\n    };\n\n    nnvm::PassFunctionReg& pass = dmlc::Registry<nnvm::PassFunctionReg>::Get()->__REGISTER__(name);\n    pass.set_body(pass_lambda);\n    pass.set_change_graph(true);\n  }\n}\n\n/*!\n * \\brief Loads dynamic custom library and initializes it\n * \\param path library path\n */\nint MXLoadLib(const char* path, unsigned verbose, void** lib) {\n  API_BEGIN();\n  *lib = LibraryInitializer::Get()->lib_load(path);\n  if (!*lib)\n    LOG(FATAL) << \"Unable to load library\";\n\n  // check that library and MXNet use same version of library API\n  mxnet::ext::opVersion_t opVersion =\n      get_func<mxnet::ext::opVersion_t>(*lib, const_cast<char*>(MXLIB_OPVERSION_STR));\n  int libVersion = opVersion();\n  if (MX_LIBRARY_VERSION != libVersion)\n    LOG(FATAL) << \"Library version (\" << libVersion << \") does not match MXNet version (\"\n               << MX_LIBRARY_VERSION << \")\";\n\n  // get error messaging APIs\n  mxnet::ext::msgSize_t msgSize =\n      get_func<mxnet::ext::msgSize_t>(*lib, const_cast<char*>(MXLIB_MSGSIZE_STR));\n  mxnet::ext::msgGet_t msgGet =\n      get_func<mxnet::ext::msgGet_t>(*lib, const_cast<char*>(MXLIB_MSGGET_STR));\n\n  // initialize library by passing MXNet version\n  mxnet::ext::initialize_t initialize =\n      get_func<mxnet::ext::initialize_t>(*lib, const_cast<char*>(MXLIB_INITIALIZE_STR));\n  if (!initialize(static_cast<int>(MXNET_VERSION))) {\n    std::string msgs = getExtensionMsgs(msgSize, msgGet);\n    LOG(FATAL) << \"Library failed to initialize\" << msgs;\n  }\n\n  // find ops, partitioners, and passes in library\n  registerOperators(*lib, verbose, msgSize, msgGet);\n  registerPartitioners(*lib, verbose, msgSize, msgGet);\n  registerPasses(*lib, verbose, msgSize, msgGet);\n  API_END();\n}\n\nint MXLibInfoFeatures(const struct LibFeature** lib_features, size_t* size) {\n  using namespace features;\n  API_BEGIN();\n  LibInfo* lib_info = LibInfo::getInstance();\n  *lib_features     = lib_info->getFeatures().data();\n  *size             = lib_info->getFeatures().size();\n  API_END();\n}\n\nint MXLibInfoCompiledWithCXX11ABI(int* result) {\n  API_BEGIN();\n#ifdef _GLIBCXX_USE_CXX11_ABI\n  *result = _GLIBCXX_USE_CXX11_ABI;\n#else\n  *result = -1;\n#endif\n  API_END();\n}\n\nint MXRandomSeed(int seed) {\n  API_BEGIN();\n  mxnet::RandomSeed(seed);\n  API_END();\n}\n\nint MXRandomSeedContext(int seed, int dev_type, int dev_id) {\n  API_BEGIN();\n  Context ctx = Context::Create(static_cast<Context::DeviceType>(dev_type), dev_id);\n  mxnet::RandomSeed(ctx, seed);\n  API_END();\n}\n\nint MXSetFlushDenorms(bool value, bool* prev_state) {\n  API_BEGIN();\n  *prev_state = false;\n\n#if SUPPORT_FTZ_DMZ\n  std::function<bool()> is_dmz_flag_available = []() {\n    // Intel 64 and IA-32 Architectures Software Developer’s Manual: Vol. 1\n    // \"Checking for the DAZ Flag in the MXCSR Register\"\n    constexpr unsigned int mxcsr_mask_offset = 28;\n    constexpr unsigned int dmz_flag_offset   = 5;\n    constexpr unsigned int fxsave_req_bytes  = 512;\n\n    char* fxsave_area_ptr = reinterpret_cast<char*>(malloc(fxsave_req_bytes));\n    memset(fxsave_area_ptr, 0, fxsave_req_bytes);  // fill memory with 0\n    _fxsave(fxsave_area_ptr);\n\n    char* mxcsr_mask_ptr = fxsave_area_ptr + mxcsr_mask_offset;\n    uint32_t mxcsr_mask  = *(reinterpret_cast<uint32_t*>((mxcsr_mask_ptr)));\n    // DMZ flag is supported if sixth bit of MXCSR_MASK is hot\n    bool dmz_flag = (mxcsr_mask >> dmz_flag_offset) & 0x1;\n    free(fxsave_area_ptr);\n    return dmz_flag;\n  };\n\n  Engine::Get()->PushSync(\n      [value, prev_state, is_dmz_flag_available](RunContext rctx) {\n        const unsigned int DMZ_STATE = value ? _MM_DENORMALS_ZERO_ON : _MM_DENORMALS_ZERO_OFF;\n        const unsigned int FTZ_STATE = value ? _MM_FLUSH_ZERO_ON : _MM_FLUSH_ZERO_OFF;\n        *prev_state                  = _MM_GET_FLUSH_ZERO_MODE();\n        _MM_SET_FLUSH_ZERO_MODE(FTZ_STATE);\n\n        // If the DAZ flag is not supported, then it is a reserved bit and attempting to write a 1\n        // to it will cause a general-protection exception (#GP)\n        if (is_dmz_flag_available()) {\n          _MM_SET_DENORMALS_ZERO_MODE(DMZ_STATE);\n        }\n      },\n      Context::CPU(),\n      {},\n      {},\n      FnProperty::kNormal,\n      0,\n      \"SetFlushDenorms\");\n\n  Engine::Get()->WaitForAll();\n\n#endif\n\n  API_END();\n}\n\nint MXNotifyShutdown() {\n  API_BEGIN();\n  mxnet::op::custom::CustomOperator::Get()->Stop();\n  Engine::Get()->NotifyShutdown();\n  Engine::Get()->WaitForAll();\n  API_END();\n}\n\nint MXSetNumOMPThreads(int thread_num) {\n  API_BEGIN();\n  omp_set_num_threads(thread_num);\n  API_END();\n}\n\nint MXEngineSetBulkSize(int bulk_size, int* prev_bulk_size) {\n  API_BEGIN();\n  *prev_bulk_size = Engine::Get()->set_bulk_size(bulk_size);\n  API_END();\n}\n\nint MXGetGPUCount(int* out) {\n  API_BEGIN();\n  *out = Context::GetGPUCount();\n  API_END();\n}\n\n// Deprecated: use MXGetGPUMemoryInformation64() instead.\nint MXGetGPUMemoryInformation(int dev, int* free_mem, int* total_mem) {\n  API_BEGIN();\n  uint64_t free_mem64  = 0UL;\n  uint64_t total_mem64 = 0UL;\n  Context::GetGPUMemoryInformation(dev, &free_mem64, &total_mem64);\n  *free_mem  = static_cast<int>(free_mem64);\n  *total_mem = static_cast<int>(total_mem64);\n  API_END();\n}\n\nint MXGetGPUMemoryInformation64(int dev, uint64_t* free_mem, uint64_t* total_mem) {\n  API_BEGIN();\n  Context::GetGPUMemoryInformation(dev, free_mem, total_mem);\n  API_END();\n}\n\nint MXGetVersion(int* out) {\n  API_BEGIN();\n  *out = static_cast<int>(MXNET_VERSION);\n  API_END();\n}\n\nint MXGetBranch(const char** out) {\n  API_BEGIN();\n  *out = MXNET_BRANCH;\n  API_END();\n}\n\nint MXGetCommitHash(const char** out) {\n  API_BEGIN();\n  *out = MXNET_COMMIT_HASH;\n  API_END();\n}\n\n#if MXNET_USE_TVM_OP\nint MXLoadTVMOp(const char* libpath) {\n  API_BEGIN();\n  tvm::runtime::TVMOpModule::Get()->Load(libpath);\n  tvm::runtime::TVMOpModule* global_module = tvm::runtime::TVMOpModule::Get();\n  global_module->Load(libpath);\n#if MXNET_USE_CUDA\n  std::string libpathstr(libpath);\n  std::string cubinpath = libpathstr.substr(0, libpathstr.size() - 11) + \"libtvmop.cubin\";\n  tvm::runtime::TVMOpModule cubin_module;\n  cubin_module.Load(cubinpath);\n  global_module->Import(cubin_module);\n#endif\n  API_END();\n}\n\nint MXLoadTVMConfig(ConfigSpaces config) {\n  API_BEGIN();\n  for (int k = 0; k < config.spaces_size; ++k) {\n    tvm::runtime::TVMOpConfig& entry =\n        ::dmlc::Registry<tvm::runtime::TVMOpConfig>::Get()->__REGISTER_OR_GET__(\n            std::string(config.spaces_key[k]));\n    const ConfigSpace& c = config.spaces_val[k];\n    for (int i = 0; i < c.entity_map_size; ++i) {\n      entry.add_entity(std::string(c.entity_map_key[i]), c.entity_map_val[i].val);\n    }\n    for (int i = 0; i < c.space_map_size; ++i) {\n      std::string name = std::string(c.space_map_key[i]);\n      std::vector<int> entities;\n      for (int j = 0; j < c.space_map_val[i].entities_size; ++j) {\n        int val = c.space_map_val[i].entities[j].val;\n        entities.push_back(val);\n      }\n      entry.add_space(name, entities);\n    }\n  }\n  API_END();\n}\n\n#endif  // MXNET_USE_TVM_OP\n\nint MXNDArrayCreateNone(NDArrayHandle* out) {\n  API_BEGIN();\n  *out = new NDArray();\n  API_END();\n}\n\ntemplate <typename DataType>\nvoid CreateNDArray(const DataType* shape,\n                   int ndim,\n                   int dev_type,\n                   int dev_id,\n                   int delay_alloc,\n                   int dtype,\n                   NDArrayHandle* out) {\n  mxnet::TShape requested_shape = mxnet::TShape(shape, shape + ndim);\n  if (!features::is_enabled(features::INT64_TENSOR_SIZE)) {\n    CHECK_LT(requested_shape.Size(), (int64_t{1} << 31) - 1)\n        << \"[CreateNDArray] Size of tensor you are trying to allocate is larger than \"\n           \"2^31 elements. Please build with flag USE_INT64_TENSOR_SIZE=1\";\n  }\n  NDArray* nd = new NDArray(requested_shape,\n                            Context::Create(static_cast<Context::DeviceType>(dev_type), dev_id),\n                            delay_alloc != 0,\n                            dtype);\n  nd->AssignStorageInfo(profiler::ProfilerScope::Get()->GetCurrentProfilerScope(),\n                        MXNET_STORAGE_DEFAULT_NAME_CSTR);\n  *out = nd;\n}\n\nint MXNDArrayCreate64(const int64_t* shape,\n                      int ndim,\n                      int dev_type,\n                      int dev_id,\n                      int delay_alloc,\n                      int dtype,\n                      NDArrayHandle* out) {\n  API_BEGIN();\n  CreateNDArray<int64_t>(shape, ndim, dev_type, dev_id, delay_alloc, dtype, out);\n  API_END();\n}\n\nint MXNDArrayCreate(const uint32_t* shape,\n                    uint32_t ndim,\n                    int dev_type,\n                    int dev_id,\n                    int delay_alloc,\n                    int dtype,\n                    NDArrayHandle* out) {\n  API_BEGIN();\n  CreateNDArray<uint32_t>(shape, static_cast<int>(ndim), dev_type, dev_id, delay_alloc, dtype, out);\n  API_END();\n}\n\ntemplate <typename DType>\nvoid CreateSparseNDArray(int storage_type,\n                         const DType* shape,\n                         int ndim,\n                         int dev_type,\n                         int dev_id,\n                         int delay_alloc,\n                         int dtype,\n                         uint32_t num_aux,\n                         int* aux_type,\n                         int* aux_ndims,\n                         const DType* aux_shape,\n                         NDArrayHandle* out) {\n  std::vector<int> aux_types;\n  mxnet::ShapeVector aux_shapes;\n  auto shape_start = aux_shape;\n  for (size_t i = 0; i < num_aux; i++) {\n    // types\n    aux_types.push_back(aux_type[i]);\n    // shapes\n    aux_shapes.emplace_back(shape_start, shape_start + aux_ndims[i]);\n    shape_start += aux_ndims[i];\n  }\n  NDArray* nd = new NDArray(NDArrayStorageType(storage_type),\n                            mxnet::TShape(shape, shape + ndim),\n                            Context::Create(static_cast<Context::DeviceType>(dev_type), dev_id),\n                            delay_alloc != 0,\n                            dtype,\n                            aux_types,\n                            aux_shapes);\n  nd->AssignStorageInfo(profiler::ProfilerScope::Get()->GetCurrentProfilerScope(),\n                        MXNET_STORAGE_DEFAULT_NAME_CSTR);\n  *out = nd;\n}\n\nint MXNDArrayCreateSparseEx(int storage_type,\n                            const uint32_t* shape,\n                            uint32_t ndim,\n                            int dev_type,\n                            int dev_id,\n                            int delay_alloc,\n                            int dtype,\n                            uint32_t num_aux,\n                            int* aux_type,\n                            uint32_t* aux_ndims,\n                            const uint32_t* aux_shape,\n                            NDArrayHandle* out) {\n  API_BEGIN();\n  CreateSparseNDArray<uint32_t>(storage_type,\n                                shape,\n                                static_cast<int>(ndim),\n                                dev_type,\n                                dev_id,\n                                delay_alloc,\n                                dtype,\n                                num_aux,\n                                aux_type,\n                                reinterpret_cast<int*>(aux_ndims),\n                                aux_shape,\n                                out);\n  API_END();\n}\n\nint MXNDArrayCreateSparseEx64(int storage_type,\n                              const int64_t* shape,\n                              int ndim,\n                              int dev_type,\n                              int dev_id,\n                              int delay_alloc,\n                              int dtype,\n                              uint32_t num_aux,\n                              int* aux_type,\n                              int* aux_ndims,\n                              const int64_t* aux_shape,\n                              NDArrayHandle* out) {\n  API_BEGIN();\n  CreateSparseNDArray<int64_t>(storage_type,\n                               shape,\n                               static_cast<int>(ndim),\n                               dev_type,\n                               dev_id,\n                               delay_alloc,\n                               dtype,\n                               num_aux,\n                               aux_type,\n                               reinterpret_cast<int*>(aux_ndims),\n                               aux_shape,\n                               out);\n  API_END();\n}\n\nint MXNDArrayLoadFromRawBytes(const void* buf, size_t size, NDArrayHandle* out) {\n  NDArray* ptr = nullptr;\n  API_BEGIN();\n  dmlc::MemoryFixedSizeStream strm((void*)buf, size);  // NOLINT(*)\n  ptr = new NDArray();\n  if (!ptr->Load(&strm)) {\n    throw dmlc::Error(\"Invalid NDArray serialization format\");\n  }\n  *out = ptr;\n  API_END_HANDLE_ERROR(delete ptr);\n}\n\nint MXNDArraySaveRawBytes(NDArrayHandle handle, size_t* out_size, const char** out_buf) {\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  API_BEGIN();\n  ret->ret_str.resize(0);\n  dmlc::MemoryStringStream strm(&ret->ret_str);\n  static_cast<NDArray*>(handle)->Save(&strm);\n  *out_size = ret->ret_str.length();\n  *out_buf  = ret->ret_str.c_str();\n  API_END();\n}\n\nint MXNDArraySyncCopyFromCPU(NDArrayHandle handle, const void* data, size_t size) {\n  API_BEGIN();\n  static_cast<NDArray*>(handle)->SyncCopyFromCPU(data, size);\n  API_END();\n}\n\nint MXNDArraySyncCopyToCPU(NDArrayHandle handle, void* data, size_t size) {\n  API_BEGIN();\n  static_cast<NDArray*>(handle)->SyncCopyToCPU(data, size);\n  API_END();\n}\n\n/*!\n * \\brief Copy src.data() to dst.data() if i = -1, else dst.aux_data(i) if i >= 0\n * This function blocks. Do not use it in performance critical code.\n * \\param handle_dst handle of a dst ndarray whose data/aux_data has been allocated\n * \\param handle_src handle of a src ndarray which has default storage type\n * \\param i dst data blob indicator\n */\nint MXNDArraySyncCopyFromNDArray(NDArrayHandle handle_dst,\n                                 const NDArrayHandle handle_src,\n                                 const int i) {\n  API_BEGIN();\n  NDArray* dst = static_cast<NDArray*>(handle_dst);\n  NDArray* src = static_cast<NDArray*>(handle_src);\n  dst->SyncCopyFromNDArray(*src, -1, i);\n  API_END();\n}\n\nint MXNDArraySyncCheckFormat(NDArrayHandle handle, const bool full_check) {\n  API_BEGIN();\n  NDArray* arr = static_cast<NDArray*>(handle);\n  arr->SyncCheckFormat(full_check);\n  API_END();\n}\n\nint MXNDArrayWaitToRead(NDArrayHandle handle) {\n  API_BEGIN();\n  static_cast<NDArray*>(handle)->WaitToRead();\n  API_END();\n}\n\nint MXNDArrayWaitToWrite(NDArrayHandle handle) {\n  API_BEGIN();\n  static_cast<NDArray*>(handle)->WaitToWrite();\n  API_END();\n}\n\nint MXNDArrayWaitAll() {\n  API_BEGIN();\n  Engine::Get()->WaitForAll();\n  API_END();\n}\n\nint MXNDArrayLegacySave(const char* fname,\n                        uint32_t num_args,\n                        NDArrayHandle* args,\n                        const char** keys) {\n  API_BEGIN();\n  std::vector<NDArray> data(num_args);\n  std::vector<std::string> names;\n  for (uint32_t i = 0; i < num_args; ++i) {\n    data[i] = *static_cast<NDArray*>(args[i]);\n  }\n  if (keys != nullptr) {\n    names.resize(num_args);\n    for (uint32_t i = 0; i < num_args; ++i) {\n      names[i] = keys[i];\n    }\n  }\n  {\n    std::unique_ptr<dmlc::Stream> fo(dmlc::Stream::Create(fname, \"w\"));\n    mxnet::NDArray::Save(fo.get(), data, names);\n  }\n  API_END();\n}\n\nint MXNDArraySave(const char* fname, uint32_t num_args, NDArrayHandle* args, const char** keys) {\n  API_BEGIN();\n\n  CHECK_NOTNULL(fname);\n\n  // We may use mz_zip_writer_init_v2 later instead of mz_zip_writer_init_file\n  // and write an adapter for DMLC stream based on pZip->m_pWrite (and\n  // pZip->m_pIO_opaque)\n  if (num_args == 1 && keys == nullptr) {\n    NDArray* array = static_cast<NDArray*>(args[0]);\n    if (array->storage_type() == kDefaultStorage) {\n      npy::save_array(fname, *array);\n    } else {\n      mz_zip_archive archive{};\n      CHECK(mz_zip_writer_init_file(&archive, fname, 0))\n          << \"Failed to open archive \" << fname << \": \"\n          << mz_zip_get_error_string(mz_zip_get_last_error(&archive));\n      npz::save_array(&archive, \"\", *array);\n      CHECK(mz_zip_writer_finalize_archive(&archive))\n          << \"Failed to finalize archive \" << fname\n          << mz_zip_get_error_string(mz_zip_get_last_error(&archive));\n      CHECK(mz_zip_writer_end(&archive))\n          << \"Failed to end archive \" << fname\n          << mz_zip_get_error_string(mz_zip_get_last_error(&archive));\n    }\n  } else {\n    mz_zip_archive archive{};\n    CHECK(mz_zip_writer_init_file(&archive, fname, 0))\n        << \"Failed to open archive \" << fname << \": \"\n        << mz_zip_get_error_string(mz_zip_get_last_error(&archive));\n    for (uint32_t i = 0; i < num_args; ++i) {\n      NDArray* array              = static_cast<NDArray*>(args[i]);\n      const std::string array_key = keys == nullptr ? \"arr_\" + std::to_string(i) : keys[i];\n      npz::save_array(&archive, array_key, *array);\n    }\n    CHECK(mz_zip_writer_finalize_archive(&archive))\n        << \"Failed to finalize archive \" << fname\n        << mz_zip_get_error_string(mz_zip_get_last_error(&archive));\n    CHECK(mz_zip_writer_end(&archive)) << \"Failed to end archive \" << fname\n                                       << mz_zip_get_error_string(mz_zip_get_last_error(&archive));\n  }\n  API_END();\n}\n\nint MXNDArrayLoad(const char* fname,\n                  uint32_t* out_size,\n                  NDArrayHandle** out_arr,\n                  uint32_t* out_name_size,\n                  const char*** out_names) {\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  ret->ret_vec_str.clear();\n  API_BEGIN();\n\n  uint32_t magic;\n  {\n    std::unique_ptr<dmlc::Stream> strm(dmlc::Stream::Create(fname, \"r\"));\n    CHECK_EQ(strm->Read(&magic, sizeof(uint32_t)), sizeof(uint32_t))\n        << \"Failed to read 32 bits from file.\";\n  }\n\n  if (magic == 0x04034b50 || magic == 0x504b0304 || magic == 0x06054b50 ||\n      magic == 0x504b0506) {                       // zip file format; assumed to be npz\n    auto [data, names] = npz::load_arrays(fname);  // NOLINT\n    ret->ret_handles.resize(data.size());\n    for (size_t i = 0; i < data.size(); ++i) {\n      NDArray* ptr        = new NDArray();\n      *ptr                = data[i];\n      ret->ret_handles[i] = ptr;\n    }\n    ret->ret_vec_str.resize(names.size());\n    for (size_t i = 0; i < names.size(); ++i) {\n      ret->ret_vec_str[i] = names[i];\n    }\n    ret->ret_vec_charp.resize(names.size());\n    for (size_t i = 0; i < names.size(); ++i) {\n      ret->ret_vec_charp[i] = ret->ret_vec_str[i].c_str();\n    }\n    *out_size      = static_cast<uint32_t>(data.size());\n    *out_arr       = dmlc::BeginPtr(ret->ret_handles);\n    *out_name_size = static_cast<uint32_t>(names.size());\n    *out_names     = dmlc::BeginPtr(ret->ret_vec_charp);\n  } else if (magic == 0x4d554e93 || magic == 0x934e554d) {  // first bytes of npy format\n    *out_size = 1;\n    ret->ret_handles.resize(1);\n    NDArray* ptr = new NDArray();\n    *ptr         = npy::load_array(fname);  // Only supports local filesystem at this point in time\n    ret->ret_handles[0] = ptr;\n    *out_arr            = dmlc::BeginPtr(ret->ret_handles);\n  } else {\n    std::vector<NDArray> data;\n    std::vector<std::string>& names = ret->ret_vec_str;\n    {\n      std::unique_ptr<dmlc::Stream> fi(dmlc::Stream::Create(fname, \"r\"));\n      mxnet::NDArray::Load(fi.get(), &data, &names);\n    }\n    ret->ret_handles.resize(data.size());\n    for (size_t i = 0; i < data.size(); ++i) {\n      NDArray* ptr        = new NDArray();\n      *ptr                = data[i];\n      ret->ret_handles[i] = ptr;\n    }\n    ret->ret_vec_charp.resize(names.size());\n    for (size_t i = 0; i < names.size(); ++i) {\n      ret->ret_vec_charp[i] = names[i].c_str();\n    }\n    *out_size      = static_cast<uint32_t>(data.size());\n    *out_arr       = dmlc::BeginPtr(ret->ret_handles);\n    *out_name_size = static_cast<uint32_t>(names.size());\n    *out_names     = dmlc::BeginPtr(ret->ret_vec_charp);\n  }\n  API_END();\n}\n\nint MXNDArrayLoadFromBuffer(const void* ndarray_buffer,\n                            size_t size,\n                            uint32_t* out_size,\n                            NDArrayHandle** out_arr,\n                            uint32_t* out_name_size,\n                            const char*** out_names) {\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  ret->ret_vec_str.clear();\n  API_BEGIN();\n  CHECK_NOTNULL(ndarray_buffer);\n  std::vector<NDArray> data;\n  std::vector<std::string>& names = ret->ret_vec_str;\n  {\n    std::unique_ptr<dmlc::MemoryFixedSizeStream> fi(\n        new dmlc::MemoryFixedSizeStream(const_cast<void*>(ndarray_buffer), size));\n    mxnet::NDArray::Load(fi.get(), &data, &names);\n  }\n  ret->ret_handles.resize(data.size());\n  for (size_t i = 0; i < data.size(); ++i) {\n    NDArray* ptr        = new NDArray();\n    *ptr                = data[i];\n    ret->ret_handles[i] = ptr;\n  }\n  ret->ret_vec_charp.resize(names.size());\n  for (size_t i = 0; i < names.size(); ++i) {\n    ret->ret_vec_charp[i] = names[i].c_str();\n  }\n  *out_size      = static_cast<uint32_t>(data.size());\n  *out_arr       = dmlc::BeginPtr(ret->ret_handles);\n  *out_name_size = static_cast<uint32_t>(names.size());\n  *out_names     = dmlc::BeginPtr(ret->ret_vec_charp);\n  API_END();\n}\n\nint MXNDArrayFree(NDArrayHandle handle) {\n  API_BEGIN();\n  delete static_cast<NDArray*>(handle);\n  API_END();\n}\n\ntemplate <typename dtype>\nvoid SliceArray(NDArrayHandle handle,\n                dtype slice_begin,\n                dtype slice_end,\n                NDArray* ptr,\n                NDArrayHandle* out) {\n  *ptr = static_cast<NDArray*>(handle)->SliceWithRecord(slice_begin, slice_end);\n  *out = ptr;\n}\n\nint MXNDArraySlice(NDArrayHandle handle,\n                   uint32_t slice_begin,\n                   uint32_t slice_end,\n                   NDArrayHandle* out) {\n  NDArray* ptr = new NDArray();\n  API_BEGIN();\n  SliceArray<uint32_t>(handle, slice_begin, slice_end, ptr, out);\n  API_END_HANDLE_ERROR(delete ptr);\n}\n\nint MXNDArraySlice64(NDArrayHandle handle,\n                     int64_t slice_begin,\n                     int64_t slice_end,\n                     NDArrayHandle* out) {\n  NDArray* ptr = new NDArray();\n  API_BEGIN();\n  SliceArray<int64_t>(handle, slice_begin, slice_end, ptr, out);\n  API_END_HANDLE_ERROR(delete ptr);\n}\n\nint MXNDArrayAt(NDArrayHandle handle, uint32_t idx, NDArrayHandle* out) {\n  NDArray* ptr = new NDArray();\n  API_BEGIN();\n  *ptr = static_cast<NDArray*>(handle)->AtWithRecord(idx);\n  *out = ptr;\n  API_END_HANDLE_ERROR(delete ptr);\n}\n\nint MXNDArrayAt64(NDArrayHandle handle, int64_t idx, NDArrayHandle* out) {\n  NDArray* ptr = new NDArray();\n  API_BEGIN();\n  *ptr = static_cast<NDArray*>(handle)->AtWithRecord(idx);\n  *out = ptr;\n  API_END_HANDLE_ERROR(delete ptr);\n}\n\nint MXNDArrayReshape(NDArrayHandle handle, int ndim, int* dims, NDArrayHandle* out) {\n  NDArray* ptr = new NDArray();\n  API_BEGIN();\n  NDArray* arr = static_cast<NDArray*>(handle);\n  mxnet::TShape new_shape(dims, dims + ndim);\n  int size = 1;\n  int pos  = -1;\n  for (int i = 0; i < ndim; ++i) {\n    int dim = dims[i];\n    if (dim == -1) {\n      CHECK_EQ(pos, -1) << \"Invalid new shape \" << new_shape << \": more than one dimensions are -1\";\n      pos = i;\n    } else {\n      if (dim == 0) {\n        CHECK_LT(i, arr->shape().ndim()) << \"Invalid new shape \" << new_shape\n                                         << \": 0 dimension exceeds original shape \" << arr->shape();\n        dim = arr->shape()[i];\n      }\n      size *= dim;\n      new_shape[i] = dim;\n    }\n  }\n  if (pos >= 0) {\n    new_shape[pos] = arr->shape().Size() / size;\n  }\n  *ptr = arr->ReshapeWithRecord(new_shape);\n  *out = ptr;\n  API_END_HANDLE_ERROR(delete ptr);\n}\n\nint MXNDArrayReshape64(NDArrayHandle handle,\n                       int ndim,\n                       dim_t* dims,\n                       bool reverse,\n                       NDArrayHandle* out) {\n  NDArray* ptr = new NDArray();\n  API_BEGIN();\n  NDArray* arr = static_cast<NDArray*>(handle);\n  mxnet::Tuple<dim_t> shape(dims, dims + ndim);\n  mxnet::TShape new_shape = mxnet::op::InferReshapeShape(shape, arr->shape(), reverse);\n  *ptr                    = arr->ReshapeWithRecord(new_shape);\n  *out                    = ptr;\n  API_END_HANDLE_ERROR(delete ptr);\n}\n\nint MXNDArrayGetStorageType(NDArrayHandle handle, int* out_storage_type) {\n  API_BEGIN();\n  NDArray* arr = static_cast<NDArray*>(handle);\n  if (!arr->is_none()) {\n    *out_storage_type = arr->storage_type();\n  } else {\n    *out_storage_type = kUndefinedStorage;\n  }\n  API_END();\n}\n\ntemplate <typename dtype>\ninline void GetShape(NDArrayHandle handle,\n                     const dtype** out_pdata,\n                     int* out_dim,\n                     MXAPIThreadLocalEntry<dtype>* ret) {\n  NDArray* arr = static_cast<NDArray*>(handle);\n  if (!arr->is_none()) {\n    mxnet::TShape s = arr->shape();\n    // Handle dynamic shape in deferred compute mode\n    if (!Imperative::DCInfo::IsNone(*arr)) {\n      if (!shape_is_known(s) && !Imperative::DCInfo::IsComputed(*arr)) {\n        Imperative::DCInfo::Compute(*arr);\n        s = arr->shape();\n      }\n    }\n\n    if (!features::is_enabled(features::INT64_TENSOR_SIZE)) {\n      CHECK_LT(s.Size(), (int64_t{1} << 31) - 1)\n          << \"[Get Shape] Size of tensor you are trying to allocate is larger than \"\n             \"2^31 elements. Please build with flag USE_INT64_TENSOR_SIZE=1\";\n    }\n\n    if (!Imperative::Get()->is_np_shape()) {\n      common::ConvertToLegacyShape(&s);\n    }\n    *out_dim = s.ndim();\n    if (s.ndim() >= 0) {\n      std::vector<dtype>& buffer = ret->arg_shape_buffer_ex;\n      buffer.resize(s.ndim());\n      mxnet::ShapeTypeCast(s.begin(), s.end(), buffer.data());\n      *out_pdata = buffer.data();\n    }\n  } else {\n    if (Imperative::Get()->is_np_shape()) {\n      *out_dim = -1;\n    } else {\n      *out_dim = 0;\n    }\n  }\n}\n\nint MXNDArrayGetShape(NDArrayHandle handle, int* out_dim, const int** out_pdata) {\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  API_BEGIN();\n  GetShape<int>(handle, out_pdata, out_dim, ret);\n  API_END();\n}\n\nint MXNDArrayGetShape64(NDArrayHandle handle, int* out_dim, const int64_t** out_pdata) {\n  MXAPIThreadLocalEntry<int64_t>* ret = MXAPIThreadLocalStore<int64_t>::Get();\n  API_BEGIN();\n  GetShape<int64_t>(handle, out_pdata, out_dim, ret);\n  API_END();\n}\n\nint MXNDArrayGetData(NDArrayHandle handle, void** out_pdata) {\n  API_BEGIN();\n  NDArray* arr = static_cast<NDArray*>(handle);\n#if MXNET_USE_ONEDNN == 1\n  if (arr->IsDNNLData()) {\n    arr->Reorder2DefaultAsync();\n    arr->WaitToRead();\n  }\n#endif\n  if (!arr->is_none()) {\n    *out_pdata = arr->data().dptr_;\n  } else {\n    *out_pdata = nullptr;\n  }\n  API_END();\n}\n\nint MXNDArrayToDLPack(NDArrayHandle handle, DLManagedTensorHandle* out_dlpack) {\n  API_BEGIN();\n  NDArray* arr = static_cast<NDArray*>(handle);\n  *out_dlpack  = arr->ToDLPack();\n  API_END();\n}\n\nint MXNDArrayFromDLPack(DLManagedTensorHandle dlpack,\n                        const bool transient_handle,\n                        NDArrayHandle* out_handle) {\n  API_BEGIN();\n  *out_handle =\n      new NDArray(NDArray::FromDLPack(static_cast<DLManagedTensor*>(dlpack), transient_handle));\n  API_END();\n}\n\nint MXNDArrayCallDLPackDeleter(DLManagedTensorHandle dlpack) {\n  API_BEGIN();\n  if (dlpack != nullptr) {\n    DLManagedTensor* p_dlpack = static_cast<DLManagedTensor*>(dlpack);\n    p_dlpack->deleter(p_dlpack);\n  }\n  API_END();\n}\n\nint MXNDArrayGetDType(NDArrayHandle handle, int* out_dtype) {\n  API_BEGIN();\n  NDArray* arr = static_cast<NDArray*>(handle);\n  if (!arr->is_none()) {\n    *out_dtype = arr->dtype();\n  } else {\n    *out_dtype = -1;\n  }\n  API_END();\n}\n\nint MXNDArrayGetAuxType(NDArrayHandle handle, uint32_t i, int* out_type) {\n  API_BEGIN();\n  NDArray* arr = static_cast<NDArray*>(handle);\n  *out_type    = arr->aux_type(i);\n  API_END();\n}\n\n/*!\n * \\brief Get a deep copy of the ith aux data blob\n * in the form of an NDArray of default storage type.\n * This function blocks. Do not use it in performance critical code.\n */\nint MXNDArrayGetAuxNDArray(NDArrayHandle handle, uint32_t i, NDArrayHandle* out) {\n  API_BEGIN();\n  NDArray* arr = static_cast<NDArray*>(handle);\n  *out         = new NDArray(arr->aux_ndarray(i));\n  API_END();\n}\n\n/*!\n * \\brief Get a deep copy of the data blob\n * in the form of an NDArray of default storage type.\n * This function blocks. Do not use it in performance critical code.\n */\nint MXNDArrayGetDataNDArray(NDArrayHandle handle, NDArrayHandle* out) {\n  API_BEGIN();\n  NDArray* arr = static_cast<NDArray*>(handle);\n  *out         = new NDArray(arr->data_ndarray());\n  API_END();\n}\n\nint MXNDArrayGetContext(NDArrayHandle handle, int* out_dev_type, int* out_dev_id) {\n  API_BEGIN();\n  NDArray* arr = static_cast<NDArray*>(handle);\n  if (!arr->is_none()) {\n    const Context& ctx = arr->ctx();\n    *out_dev_type      = ctx.dev_type;\n    *out_dev_id        = ctx.dev_id;\n  } else {\n    *out_dev_type = 0;\n    *out_dev_id   = 0;\n  }\n  API_END();\n}\n\nint MXNDArrayGetGrad(NDArrayHandle handle, NDArrayHandle* out) {\n  API_BEGIN();\n  NDArray* arr = static_cast<NDArray*>(handle);\n  NDArray ret  = arr->grad();\n  if (ret.is_none()) {\n    *out = nullptr;\n  } else {\n    *out = new NDArray(ret);\n  }\n  API_END();\n}\n\nint MXNDArrayDetach(NDArrayHandle handle, NDArrayHandle* out) {\n  API_BEGIN();\n  NDArray* arr = static_cast<NDArray*>(handle);\n  *out         = new NDArray(arr->Detach());\n  API_END();\n}\n\nint MXNDArraySetGradState(NDArrayHandle handle, int state) {\n  API_BEGIN();\n  NDArray* arr = static_cast<NDArray*>(handle);\n  arr->set_fresh_out_grad(static_cast<bool>(state));\n  API_END();\n}\n\nint MXNDArrayGetGradState(NDArrayHandle handle, int* out) {\n  API_BEGIN();\n  NDArray* arr = static_cast<NDArray*>(handle);\n  *out         = arr->fresh_out_grad();\n  API_END();\n}\n\nint MXListFunctions(uint32_t* out_size, FunctionHandle** out_array) {\n  API_BEGIN();\n  auto& vec  = dmlc::Registry<NDArrayFunctionReg>::List();\n  *out_size  = static_cast<uint32_t>(vec.size());\n  *out_array = (FunctionHandle*)(dmlc::BeginPtr(vec));  //  NOLINT(*)\n  API_END();\n}\n\nint MXGetFunction(const char* name, FunctionHandle* out) {\n  API_BEGIN();\n  *out = dmlc::Registry<NDArrayFunctionReg>::Find(name);\n  API_END();\n}\n\nint MXFuncGetInfo(FunctionHandle fun,\n                  const char** name,\n                  const char** description,\n                  uint32_t* num_args,\n                  const char*** arg_names,\n                  const char*** arg_type_infos,\n                  const char*** arg_descriptions,\n                  const char** return_type) {\n  return MXAPIGetFunctionRegInfo(static_cast<const NDArrayFunctionReg*>(fun),\n                                 name,\n                                 description,\n                                 num_args,\n                                 arg_names,\n                                 arg_type_infos,\n                                 arg_descriptions,\n                                 return_type);\n}\n\nint MXFuncDescribe(FunctionHandle fun,\n                   uint32_t* num_use_vars,\n                   uint32_t* num_scalars,\n                   uint32_t* num_mutate_vars,\n                   int* type_mask) {\n  API_BEGIN();\n  auto* f          = static_cast<const NDArrayFunctionReg*>(fun);\n  *num_use_vars    = f->num_use_vars;\n  *num_scalars     = f->num_scalars;\n  *num_mutate_vars = f->num_mutate_vars;\n  *type_mask       = f->type_mask;\n  API_END();\n}\n\nint MXFuncInvoke(FunctionHandle fun,\n                 NDArrayHandle* use_vars,\n                 float* scalar_args,\n                 NDArrayHandle* mutate_vars,\n                 int num_params,\n                 char** param_keys,\n                 char** param_vals) {\n  API_BEGIN();\n  auto* f = static_cast<const NDArrayFunctionReg*>(fun);\n  f->body((NDArray**)(use_vars),  //  NOLINT(*)\n          scalar_args,\n          (NDArray**)(mutate_vars),  //  NOLINT(*)\n          num_params,\n          param_keys,\n          param_vals);\n  API_END();\n}\n\n//--------------------------------------------\n// Part 5: IO Interface\n//--------------------------------------------\nint MXListDataIters(uint32_t* out_size, DataIterCreator** out_array) {\n  API_BEGIN();\n  auto& vec  = dmlc::Registry<DataIteratorReg>::List();\n  *out_size  = static_cast<uint32_t>(vec.size());\n  *out_array = (DataIterCreator*)(dmlc::BeginPtr(vec));  //  NOLINT(*)\n  API_END();\n}\n\nint MXDataIterGetIterInfo(DataIterCreator creator,\n                          const char** name,\n                          const char** description,\n                          uint32_t* num_args,\n                          const char*** arg_names,\n                          const char*** arg_type_infos,\n                          const char*** arg_descriptions) {\n  DataIteratorReg* e = static_cast<DataIteratorReg*>(creator);\n  return MXAPIGetFunctionRegInfo(\n      e, name, description, num_args, arg_names, arg_type_infos, arg_descriptions, nullptr);\n}\n\nint MXDataIterCreateIter(DataIterCreator creator,\n                         uint32_t num_param,\n                         const char** keys,\n                         const char** vals,\n                         DataIterHandle* out) {\n  IIterator<DataBatch>* iter = nullptr;\n  API_BEGIN();\n  DataIteratorReg* e = static_cast<DataIteratorReg*>(creator);\n  iter               = e->body();\n  std::vector<std::pair<std::string, std::string>> kwargs;\n  for (uint32_t i = 0; i < num_param; ++i) {\n    kwargs.emplace_back(std::string(keys[i]), std::string(vals[i]));\n  }\n  iter->Init(kwargs);\n  *out = iter;\n  API_END_HANDLE_ERROR(delete iter);\n}\n\nint MXDataIterFree(DataIterHandle handle) {\n  API_BEGIN();\n  delete static_cast<IIterator<DataBatch>*>(handle);\n  API_END();\n}\n\nint MXDataIterBeforeFirst(DataIterHandle handle) {\n  API_BEGIN();\n  static_cast<IIterator<DataBatch>*>(handle)->BeforeFirst();\n  API_END();\n}\n\nint MXDataIterGetLenHint(DataIterHandle handle, int64_t* len) {\n  API_BEGIN();\n  *len = static_cast<IIterator<DataBatch>*>(handle)->GetLenHint();\n  API_END();\n}\n\nint MXDataIterNext(DataIterHandle handle, int* out) {\n  API_BEGIN();\n  *out = static_cast<IIterator<DataBatch>*>(handle)->Next();\n  API_END();\n}\n\nint MXDataIterGetLabel(DataIterHandle handle, NDArrayHandle* out) {\n  API_BEGIN();\n  const DataBatch& db = static_cast<IIterator<DataBatch>*>(handle)->Value();\n  bool no_label       = db.data.size() < 2U;\n  NDArray* pndarray   = new NDArray();\n  // temp hack to make label 1D\n  // TODO(tianjun) make label 1D when label_width=0\n  mxnet::TShape shape = no_label ? TShape({\n                                       1,\n                                   }) :\n                                   db.data[1].shape();\n  if (no_label || shape.Size() < 1) {\n    // it's possible that label is not available and not required\n    // but we need to bypass the invalid copy\n    *pndarray = NDArray(TShape({1}), mxnet::Context::CPU(0));\n  } else if (shape.ndim() > 1 && shape[1] == 1) {\n    *pndarray = db.data[1].Reshape(mshadow::Shape1(shape[0]));\n  } else {\n    *pndarray = db.data[1];\n  }\n  *out = pndarray;\n  API_END();\n}\n\nint MXDataIterGetItems(DataIterHandle handle, int* num_outputs, NDArrayHandle** outputs) {\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  API_BEGIN();\n  const DataBatch& db = static_cast<IIterator<DataBatch>*>(handle)->Value();\n  std::vector<NDArray*> ndoutputs;\n  ndoutputs.reserve(db.data.size());\n  if (*outputs == nullptr) {\n    *num_outputs = db.data.size();\n    for (int i = 0; i < *num_outputs; ++i)\n      ndoutputs.push_back(new NDArray());\n  } else {\n    CHECK_EQ(*num_outputs, db.data.size()) << \"MXDataIterGetItems expects \" << db.data.size()\n                                           << \" outputs, but \" << *num_outputs << \" was given.\";\n    for (int i = 0; i < *num_outputs; ++i) {\n      ndoutputs.push_back(reinterpret_cast<NDArray*>((*outputs)[i]));\n    }\n  }\n\n  // copy outputs\n  for (int i = 0; i < *num_outputs; ++i)\n    *ndoutputs[i] = db.data[i];\n\n  if (*outputs == nullptr) {\n    ret->ret_handles.clear();\n    ret->ret_handles.reserve(*num_outputs);\n    for (int i = 0; i < *num_outputs; ++i) {\n      ret->ret_handles.push_back(ndoutputs[i]);\n    }\n    *outputs = dmlc::BeginPtr(ret->ret_handles);\n  }\n  API_END();\n}\n\nint MXDataIterGetIndex(DataIterHandle handle, uint64_t** out_index, uint64_t* out_size) {\n  API_BEGIN();\n  const DataBatch& db = static_cast<IIterator<DataBatch>*>(handle)->Value();\n  *out_size           = db.index.size();\n  *out_index          = const_cast<uint64_t*>(db.index.data());\n  API_END();\n}\n\nint MXDataIterGetData(DataIterHandle handle, NDArrayHandle* out) {\n  API_BEGIN();\n  const DataBatch& db = static_cast<IIterator<DataBatch>*>(handle)->Value();\n  NDArray* pndarray   = new NDArray();\n  *pndarray           = db.data[0];\n  *out                = pndarray;\n  API_END();\n}\n\nint MXDataIterGetPadNum(DataIterHandle handle, int* pad) {\n  API_BEGIN();\n  const DataBatch& db = static_cast<IIterator<DataBatch>*>(handle)->Value();\n  *pad                = db.num_batch_padd;\n  API_END();\n}\n\nint MXListDatasets(uint32_t* out_size, DatasetCreator** out_array) {\n  API_BEGIN();\n  auto& vec  = dmlc::Registry<DatasetReg>::List();\n  *out_size  = static_cast<uint32_t>(vec.size());\n  *out_array = (DatasetCreator*)(dmlc::BeginPtr(vec));  //  NOLINT(*)\n  API_END();\n}\n\nint MXDatasetCreateDataset(DatasetCreator handle,\n                           uint32_t num_param,\n                           const char** keys,\n                           const char** vals,\n                           DatasetHandle* out) {\n  Dataset* dataset = nullptr;\n  API_BEGIN();\n  DatasetReg* e = static_cast<DatasetReg*>(handle);\n  std::vector<std::pair<std::string, std::string>> kwargs;\n  for (uint32_t i = 0; i < num_param; ++i) {\n    kwargs.emplace_back(std::string(keys[i]), std::string(vals[i]));\n  }\n  dataset = e->body(kwargs);\n  *out    = new std::shared_ptr<Dataset>(dataset);\n  API_END_HANDLE_ERROR(delete dataset);\n}\n\nint MXDatasetGetDatasetInfo(DatasetCreator creator,\n                            const char** name,\n                            const char** description,\n                            uint32_t* num_args,\n                            const char*** arg_names,\n                            const char*** arg_type_infos,\n                            const char*** arg_descriptions) {\n  DatasetReg* e = static_cast<DatasetReg*>(creator);\n  return MXAPIGetFunctionRegInfo(\n      e, name, description, num_args, arg_names, arg_type_infos, arg_descriptions, nullptr);\n}\n\nint MXDatasetFree(DatasetHandle handle) {\n  API_BEGIN();\n  delete static_cast<std::shared_ptr<Dataset>*>(handle);\n  API_END();\n}\n\nint MXDatasetGetLen(DatasetHandle handle, uint64_t* out) {\n  API_BEGIN();\n  uint64_t len = (*static_cast<std::shared_ptr<Dataset>*>(handle))->GetLen();\n  *out         = len;\n  API_END();\n}\n\nint MXDatasetGetItems(DatasetHandle handle,\n                      uint64_t index,\n                      int* num_outputs,\n                      NDArrayHandle** outputs) {\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  API_BEGIN();\n  std::vector<NDArray> res;\n  CHECK((*static_cast<std::shared_ptr<Dataset>*>(handle))->GetItem(index, &res))\n      << \"Error getting item at index: \" << index;\n  std::vector<NDArray*> ndoutputs;\n  ndoutputs.reserve(res.size());\n  if (*outputs == nullptr) {\n    *num_outputs = res.size();\n    for (int i = 0; i < *num_outputs; ++i)\n      ndoutputs.push_back(new NDArray());\n  } else {\n    CHECK_EQ(*num_outputs, res.size()) << \"MXDatasetGetItems expects \" << res.size()\n                                       << \" outputs, but \" << *num_outputs << \" was given.\";\n    for (int i = 0; i < *num_outputs; ++i) {\n      ndoutputs.push_back(reinterpret_cast<NDArray*>((*outputs)[i]));\n    }\n  }\n  // copy ndarrays\n  for (int i = 0; i < *num_outputs; ++i)\n    *(ndoutputs[i]) = res[i];\n\n  if (*outputs == nullptr) {\n    ret->ret_handles.clear();\n    ret->ret_handles.reserve(*num_outputs);\n    for (int i = 0; i < *num_outputs; ++i) {\n      ret->ret_handles.push_back(ndoutputs[i]);\n    }\n    *outputs = dmlc::BeginPtr(ret->ret_handles);\n  }\n  API_END();\n}\n\nint MXListBatchifyFunctions(uint32_t* out_size, BatchifyFunctionCreator** out_array) {\n  API_BEGIN();\n  auto& vec  = dmlc::Registry<BatchifyFunctionReg>::List();\n  *out_size  = static_cast<uint32_t>(vec.size());\n  *out_array = (BatchifyFunctionCreator*)(dmlc::BeginPtr(vec));  //  NOLINT(*)\n  API_END();\n}\n\nint MXBatchifyFunctionCreateFunction(BatchifyFunctionCreator handle,\n                                     uint32_t num_param,\n                                     const char** keys,\n                                     const char** vals,\n                                     BatchifyFunctionHandle* out) {\n  BatchifyFunction* bf = nullptr;\n  API_BEGIN();\n  BatchifyFunctionReg* e = static_cast<BatchifyFunctionReg*>(handle);\n  std::vector<std::pair<std::string, std::string>> kwargs;\n  for (uint32_t i = 0; i < num_param; ++i) {\n    kwargs.emplace_back(std::string(keys[i]), std::string(vals[i]));\n  }\n  bf   = e->body(kwargs);\n  *out = new BatchifyFunctionPtr(bf);\n  API_END_HANDLE_ERROR(delete bf);\n}\n\nint MXBatchifyFunctionGetFunctionInfo(BatchifyFunctionCreator creator,\n                                      const char** name,\n                                      const char** description,\n                                      uint32_t* num_args,\n                                      const char*** arg_names,\n                                      const char*** arg_type_infos,\n                                      const char*** arg_descriptions) {\n  BatchifyFunctionReg* e = static_cast<BatchifyFunctionReg*>(creator);\n  return MXAPIGetFunctionRegInfo(\n      e, name, description, num_args, arg_names, arg_type_infos, arg_descriptions, nullptr);\n}\nint MXBatchifyFunctionInvoke(BatchifyFunctionHandle handle,\n                             int batch_size,\n                             int num_output,\n                             NDArrayHandle* inputs,\n                             NDArrayHandle** outputs) {\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  API_BEGIN();\n  CHECK_GT(batch_size, 0);\n  CHECK_GT(num_output, 0);\n  std::vector<std::vector<NDArray>> ndinputs;\n  ndinputs.reserve(batch_size);\n  int pos = 0;\n  for (int i = 0; i < batch_size; ++i) {\n    std::vector<NDArray> tmp;\n    tmp.reserve(num_output);\n    for (int j = 0; j < num_output; ++j) {\n      tmp.emplace_back(*reinterpret_cast<NDArray*>(inputs[pos++]));\n      tmp.back().WaitToRead();\n    }\n    ndinputs.emplace_back(tmp);\n  }\n  std::vector<NDArray> res;\n  CHECK((*static_cast<BatchifyFunctionPtr*>(handle))->Batchify(ndinputs, &res))\n      << \"Error call batchify with \" << ndinputs.size() << \" inputs\";\n  std::vector<NDArray*> ndoutputs;\n  ndoutputs.reserve(res.size());\n  if (*outputs == nullptr) {\n    for (int i = 0; i < num_output; ++i)\n      ndoutputs.push_back(new NDArray());\n  } else {\n    CHECK_EQ(num_output, res.size()) << \"MXBatchifyFunctionInvoke expects \" << res.size()\n                                     << \" outputs, but \" << num_output << \" was given.\";\n    for (int i = 0; i < num_output; ++i) {\n      ndoutputs.push_back(reinterpret_cast<NDArray*>((*outputs)[i]));\n    }\n  }\n\n  // copy ndarrays\n  for (int i = 0; i < num_output; ++i)\n    *(ndoutputs[i]) = res[i];\n\n  if (*outputs == nullptr) {\n    ret->ret_handles.clear();\n    ret->ret_handles.reserve(num_output);\n    for (int i = 0; i < num_output; ++i) {\n      ret->ret_handles.push_back(ndoutputs[i]);\n    }\n    *outputs = dmlc::BeginPtr(ret->ret_handles);\n  }\n  API_END();\n}\n\nint MXBatchifyFunctionFree(BatchifyFunctionHandle handle) {\n  API_BEGIN();\n  delete static_cast<BatchifyFunctionPtr*>(handle);\n  API_END();\n}\n//--------------------------------------------\n// Part 6: basic KVStore interface\n//--------------------------------------------\n\nint MXKVStoreCreate(const char* type, KVStoreHandle* out) {\n  API_BEGIN();\n  *out = KVStore::Create(type);\n  API_END();\n}\n\nint MXKVStoreSetGradientCompression(KVStoreHandle handle,\n                                    uint32_t num_params,\n                                    const char** keys,\n                                    const char** vals) {\n  API_BEGIN();\n  std::vector<std::pair<std::string, std::string>> params;\n  for (uint32_t i = 0; i < num_params; ++i) {\n    std::pair<std::string, std::string> p;\n    p.first  = keys[i];\n    p.second = vals[i];\n    params.push_back(p);\n  }\n  static_cast<KVStore*>(handle)->SetGradientCompression(params);\n  API_END();\n}\n\nint MXKVStoreFree(KVStoreHandle handle) {\n  API_BEGIN();\n  delete static_cast<KVStore*>(handle);\n  API_END();\n}\n\nint MXKVStoreInit(KVStoreHandle handle, uint32_t num, const int* keys, NDArrayHandle* vals) {\n  API_BEGIN();\n  std::vector<int> v_keys(num);\n  std::vector<NDArray> v_vals(num);\n  for (uint32_t i = 0; i < num; ++i) {\n    v_keys[i] = keys[i];\n    v_vals[i] = *static_cast<NDArray*>(vals[i]);\n  }\n  static_cast<KVStore*>(handle)->Init(v_keys, v_vals);\n  API_END();\n}\n\nint MXKVStoreInitEx(KVStoreHandle handle, uint32_t num, const char** keys, NDArrayHandle* vals) {\n  API_BEGIN();\n  std::vector<std::string> v_keys(num);\n  std::vector<NDArray> v_vals(num);\n  for (uint32_t i = 0; i < num; ++i) {\n    v_keys[i] = keys[i];\n    v_vals[i] = *static_cast<NDArray*>(vals[i]);\n  }\n  static_cast<KVStore*>(handle)->Init(v_keys, v_vals);\n  API_END();\n}\n\nint MXKVStorePush(KVStoreHandle handle,\n                  uint32_t num,\n                  const int* keys,\n                  NDArrayHandle* vals,\n                  int priority) {\n  API_BEGIN();\n  std::vector<int> v_keys(num);\n  std::vector<NDArray> v_vals(num);\n  for (uint32_t i = 0; i < num; ++i) {\n    v_keys[i] = keys[i];\n    v_vals[i] = *static_cast<NDArray*>(vals[i]);\n  }\n  static_cast<KVStore*>(handle)->Push(v_keys, v_vals, priority);\n  API_END();\n}\n\nint MXKVStorePushEx(KVStoreHandle handle,\n                    uint32_t num,\n                    const char** keys,\n                    NDArrayHandle* vals,\n                    int priority) {\n  API_BEGIN();\n  std::vector<std::string> v_keys(num);\n  std::vector<NDArray> v_vals(num);\n  for (uint32_t i = 0; i < num; ++i) {\n    v_keys[i] = keys[i];\n    v_vals[i] = *static_cast<NDArray*>(vals[i]);\n  }\n  static_cast<KVStore*>(handle)->Push(v_keys, v_vals, priority);\n  API_END();\n}\n\nint MXKVStorePull(KVStoreHandle handle,\n                  uint32_t num,\n                  const int* keys,\n                  NDArrayHandle* vals,\n                  int priority) {\n  API_BEGIN();\n  std::vector<int> v_keys(num);\n  std::vector<NDArray*> v_vals(num);\n  for (uint32_t i = 0; i < num; ++i) {\n    v_keys[i] = keys[i];\n    v_vals[i] = static_cast<NDArray*>(vals[i]);\n  }\n  static_cast<KVStore*>(handle)->Pull(v_keys, v_vals, priority, true);\n  API_END();\n}\n\nint MXKVStorePullEx(KVStoreHandle handle,\n                    uint32_t num,\n                    const char** keys,\n                    NDArrayHandle* vals,\n                    int priority) {\n  API_BEGIN();\n  std::vector<std::string> v_keys(num);\n  std::vector<NDArray*> v_vals(num);\n  for (uint32_t i = 0; i < num; ++i) {\n    v_keys[i] = keys[i];\n    v_vals[i] = static_cast<NDArray*>(vals[i]);\n  }\n  static_cast<KVStore*>(handle)->Pull(v_keys, v_vals, priority, true);\n  API_END();\n}\n\nint MXKVStoreBroadcast(KVStoreHandle handle,\n                       mx_uint vnum,\n                       const int* vkeys,\n                       mx_uint onum,\n                       const int* okeys,\n                       NDArrayHandle* vals,\n                       NDArrayHandle* outs,\n                       int priority) {\n  API_BEGIN();\n  std::vector<int> v_vkeys(vnum);\n  std::vector<int> v_okeys(onum);\n  std::vector<NDArray> v_vals(vnum);\n  std::vector<NDArray*> v_outs(onum);\n  for (mx_uint i = 0; i < vnum; ++i) {\n    v_vkeys[i] = vkeys[i];\n    v_vals[i]  = *static_cast<NDArray*>(vals[i]);\n  }\n  for (mx_uint i = 0; i < onum; ++i) {\n    v_okeys[i] = okeys[i];\n    v_outs[i]  = static_cast<NDArray*>(outs[i]);\n  }\n  static_cast<KVStore*>(handle)->Broadcast(v_vkeys, v_okeys, v_vals, v_outs, priority);\n  API_END();\n}\n\nint MXKVStoreBroadcastEx(KVStoreHandle handle,\n                         mx_uint vnum,\n                         const char** vkeys,\n                         mx_uint onum,\n                         const char** okeys,\n                         NDArrayHandle* vals,\n                         NDArrayHandle* outs,\n                         int priority) {\n  API_BEGIN();\n  std::vector<std::string> v_vkeys(vnum);\n  std::vector<std::string> v_okeys(onum);\n  std::vector<NDArray> v_vals(vnum);\n  std::vector<NDArray*> v_outs(onum);\n  for (mx_uint i = 0; i < vnum; ++i) {\n    v_vkeys[i] = vkeys[i];\n    v_vals[i]  = *static_cast<NDArray*>(vals[i]);\n  }\n  for (mx_uint i = 0; i < onum; ++i) {\n    v_okeys[i] = okeys[i];\n    v_outs[i]  = static_cast<NDArray*>(outs[i]);\n  }\n  static_cast<KVStore*>(handle)->Broadcast(v_vkeys, v_okeys, v_vals, v_outs, priority);\n  API_END();\n}\n\nint MXKVStorePushPull(KVStoreHandle handle,\n                      mx_uint vnum,\n                      const int* vkeys,\n                      mx_uint onum,\n                      const int* okeys,\n                      NDArrayHandle* vals,\n                      NDArrayHandle* outs,\n                      int priority) {\n  API_BEGIN();\n  std::vector<int> v_vkeys(vnum);\n  std::vector<int> v_okeys(onum);\n  std::vector<NDArray> v_vals(vnum);\n  std::vector<NDArray*> v_outs(onum);\n  for (mx_uint i = 0; i < vnum; ++i) {\n    v_vkeys[i] = vkeys[i];\n    v_vals[i]  = *static_cast<NDArray*>(vals[i]);\n  }\n  for (mx_uint i = 0; i < onum; ++i) {\n    v_okeys[i] = okeys[i];\n    v_outs[i]  = static_cast<NDArray*>(outs[i]);\n  }\n  static_cast<KVStore*>(handle)->PushPull(v_vkeys, v_okeys, v_vals, v_outs, priority);\n  API_END();\n}\n\nint MXKVStorePushPullEx(KVStoreHandle handle,\n                        mx_uint vnum,\n                        const char** vkeys,\n                        mx_uint onum,\n                        const char** okeys,\n                        NDArrayHandle* vals,\n                        NDArrayHandle* outs,\n                        int priority) {\n  API_BEGIN();\n  std::vector<std::string> v_vkeys(vnum);\n  std::vector<std::string> v_okeys(onum);\n  std::vector<NDArray> v_vals(vnum);\n  std::vector<NDArray*> v_outs(onum);\n  for (mx_uint i = 0; i < vnum; ++i) {\n    v_vkeys[i] = vkeys[i];\n    v_vals[i]  = *static_cast<NDArray*>(vals[i]);\n  }\n  for (mx_uint i = 0; i < onum; ++i) {\n    v_okeys[i] = okeys[i];\n    v_outs[i]  = static_cast<NDArray*>(outs[i]);\n  }\n  static_cast<KVStore*>(handle)->PushPull(v_vkeys, v_okeys, v_vals, v_outs, priority);\n  API_END();\n}\n\nint MXKVStorePullWithSparse(KVStoreHandle handle,\n                            uint32_t num,\n                            const int* keys,\n                            NDArrayHandle* vals,\n                            int priority,\n                            bool ignore_sparse) {\n  API_BEGIN();\n  std::vector<int> v_keys(num);\n  std::vector<NDArray*> v_vals(num);\n  for (uint32_t i = 0; i < num; ++i) {\n    v_keys[i] = keys[i];\n    v_vals[i] = static_cast<NDArray*>(vals[i]);\n  }\n  static_cast<KVStore*>(handle)->Pull(v_keys, v_vals, priority, ignore_sparse);\n  API_END();\n}\n\nint MXKVStorePullWithSparseEx(KVStoreHandle handle,\n                              uint32_t num,\n                              const char** keys,\n                              NDArrayHandle* vals,\n                              int priority,\n                              bool ignore_sparse) {\n  API_BEGIN();\n  std::vector<std::string> v_keys(num);\n  std::vector<NDArray*> v_vals(num);\n  for (uint32_t i = 0; i < num; ++i) {\n    v_keys[i] = keys[i];\n    v_vals[i] = static_cast<NDArray*>(vals[i]);\n  }\n  static_cast<KVStore*>(handle)->Pull(v_keys, v_vals, priority, ignore_sparse);\n  API_END();\n}\n\nint MXKVStorePullRowSparse(KVStoreHandle handle,\n                           uint32_t num,\n                           const int* keys,\n                           NDArrayHandle* vals,\n                           const NDArrayHandle* row_ids,\n                           int priority) {\n  API_BEGIN();\n  std::vector<int> v_keys(num);\n  std::vector<std::pair<NDArray*, NDArray>> v_val_rowids(num);\n  for (uint32_t i = 0; i < num; ++i) {\n    v_keys[i] = keys[i];\n    v_val_rowids[i] =\n        std::make_pair(static_cast<NDArray*>(vals[i]), *static_cast<NDArray*>(row_ids[i]));\n  }\n  static_cast<KVStore*>(handle)->PullRowSparse(v_keys, v_val_rowids, priority);\n  API_END();\n}\n\nint MXKVStorePullRowSparseEx(KVStoreHandle handle,\n                             uint32_t num,\n                             const char** keys,\n                             NDArrayHandle* vals,\n                             const NDArrayHandle* row_ids,\n                             int priority) {\n  API_BEGIN();\n  std::vector<std::string> v_keys(num);\n  std::vector<std::pair<NDArray*, NDArray>> v_val_rowids(num);\n  for (uint32_t i = 0; i < num; ++i) {\n    v_keys[i] = keys[i];\n    v_val_rowids[i] =\n        std::make_pair(static_cast<NDArray*>(vals[i]), *static_cast<NDArray*>(row_ids[i]));\n  }\n  static_cast<KVStore*>(handle)->PullRowSparse(v_keys, v_val_rowids, priority);\n  API_END();\n}\n\nvoid MXKVStoreSetUpdaterImpl(KVStoreHandle handle, MXKVStoreUpdater updater, void* updater_handle) {\n  MXKVStoreUpdater* updater_temp = updater;\n  void* updater_handle_temp      = updater_handle;\n  std::function<void(int, const NDArray&, NDArray*)> updt =\n      [updater_temp, updater_handle_temp](int key, const NDArray& recv, NDArray* local) {\n        NDArray* recv_copy  = new NDArray();\n        *recv_copy          = recv;\n        NDArray* local_copy = new NDArray();\n        *local_copy         = *local;\n        updater_temp(key, recv_copy, local_copy, updater_handle_temp);\n      };\n  static_cast<KVStore*>(handle)->set_updater(updt);\n}\n\nint MXKVStoreSetUpdater(KVStoreHandle handle, MXKVStoreUpdater updater, void* updater_handle) {\n  API_BEGIN();\n  MXKVStoreSetUpdaterImpl(handle, updater, updater_handle);\n  API_END();\n}\n\nint MXKVStoreSetUpdaterEx(KVStoreHandle handle,\n                          MXKVStoreUpdater updater,\n                          MXKVStoreStrUpdater str_updater,\n                          void* updater_handle) {\n  API_BEGIN();\n  // set updater with int keys\n  MXKVStoreSetUpdaterImpl(handle, updater, updater_handle);\n  // set updater with string keys\n  MXKVStoreStrUpdater* updater_temp = str_updater;\n  void* updater_handle_temp         = updater_handle;\n  std::function<void(const std::string&, const NDArray&, NDArray*)> updt =\n      [updater_temp, updater_handle_temp](\n          const std::string& key, const NDArray& recv, NDArray* local) {\n        NDArray* recv_copy  = new NDArray();\n        *recv_copy          = recv;\n        NDArray* local_copy = new NDArray();\n        *local_copy         = *local;\n        updater_temp(key.c_str(), recv_copy, local_copy, updater_handle_temp);\n      };\n  static_cast<KVStore*>(handle)->set_updater(updt);\n  API_END();\n}\n\nint MXKVStoreGetRank(KVStoreHandle handle, int* rank) {\n  API_BEGIN();\n  *rank = static_cast<KVStore*>(handle)->get_rank();\n  API_END();\n}\n\nint MXKVStoreGetGroupSize(KVStoreHandle handle, int* size) {\n  API_BEGIN();\n  *size = static_cast<KVStore*>(handle)->get_group_size();\n  API_END();\n}\n\nint MXKVStoreBarrier(KVStoreHandle handle) {\n  API_BEGIN();\n  static_cast<KVStore*>(handle)->Barrier();\n  API_END();\n}\n\nint MXKVStoreSetBarrierBeforeExit(KVStoreHandle handle, const int barrier_before_exit) {\n  API_BEGIN();\n  static_cast<KVStore*>(handle)->set_barrier_before_exit(barrier_before_exit);\n  API_END();\n}\n\nint MXInitPSEnv(uint32_t num_vars, const char** keys, const char** vals) {\n  API_BEGIN();\n  std::unordered_map<std::string, std::string> kwargs;\n  for (uint32_t i = 0; i < num_vars; ++i) {\n    kwargs[std::string(keys[i])] = std::string(vals[i]);\n  }\n  KVStore::InitPSEnv(kwargs);\n  API_END();\n}\n\nint MXKVStoreIsWorkerNode(int* ret) {\n  API_BEGIN();\n  *ret = KVStore::IsWorkerNode();\n  API_END();\n}\n\nint MXKVStoreIsServerNode(int* ret) {\n  API_BEGIN();\n  *ret = KVStore::IsServerNode();\n  API_END();\n}\n\nint MXKVStoreIsSchedulerNode(int* ret) {\n  API_BEGIN();\n  *ret = KVStore::IsSchedulerNode();\n  API_END();\n}\n\nint MXKVStoreRunServer(KVStoreHandle handle,\n                       MXKVStoreServerController controller,\n                       void* controller_handle) {\n  API_BEGIN();\n  MXKVStoreServerController* controller_temp = controller;\n  void* controller_handle_temp               = controller_handle;\n  auto ctrl = [controller_temp, controller_handle_temp](int head, const std::string& body) {\n    controller_temp(head, body.c_str(), controller_handle_temp);\n  };\n  static_cast<KVStore*>(handle)->RunServer(ctrl);\n  API_END();\n}\n\nint MXKVStoreSendCommmandToServers(KVStoreHandle handle, int cmd_id, const char* cmd_body) {\n  API_BEGIN();\n  static_cast<KVStore*>(handle)->SendCommandToServers(cmd_id, std::string(cmd_body));\n  API_END();\n}\n\nint MXKVStoreGetType(KVStoreHandle handle, const char** type) {\n  API_BEGIN();\n  *CHECK_NOTNULL(type) = static_cast<KVStore*>(handle)->type().c_str();\n  API_END();\n}\n\nint MXKVStoreGetNumDeadNode(KVStoreHandle handle,\n                            const int node_id,\n                            int* number,\n                            const int timeout_sec) {\n  API_BEGIN();\n  *number = static_cast<KVStore*>(handle)->get_num_dead_node(node_id, timeout_sec);\n  API_END();\n}\n\nstruct MXRecordIOContext {\n  dmlc::RecordIOWriter* writer;\n  dmlc::RecordIOReader* reader;\n  dmlc::Stream* stream;\n  std::string* read_buff;\n};\n\nint MXRecordIOWriterCreate(const char* uri, RecordIOHandle* out) {\n  API_BEGIN();\n  dmlc::Stream* stream       = dmlc::Stream::Create(uri, \"w\");\n  MXRecordIOContext* context = new MXRecordIOContext;\n  context->writer            = new dmlc::RecordIOWriter(stream);\n  context->reader            = nullptr;\n  context->stream            = stream;\n  context->read_buff         = nullptr;\n  *out                       = reinterpret_cast<RecordIOHandle>(context);\n  API_END();\n}\n\nint MXRecordIOWriterFree(RecordIOHandle handle) {\n  API_BEGIN();\n  MXRecordIOContext* context = reinterpret_cast<MXRecordIOContext*>(handle);\n  delete context->writer;\n  delete context->stream;\n  delete context;\n  API_END();\n}\n\nint MXRecordIOWriterWriteRecord(RecordIOHandle handle, const char* buf, size_t size) {\n  API_BEGIN();\n  MXRecordIOContext* context = reinterpret_cast<MXRecordIOContext*>(handle);\n  context->writer->WriteRecord(reinterpret_cast<const void*>(buf), size);\n  API_END();\n}\n\nint MXRecordIOWriterTell(RecordIOHandle handle, size_t* pos) {\n  API_BEGIN();\n  MXRecordIOContext* context = reinterpret_cast<MXRecordIOContext*>(handle);\n  *pos                       = context->writer->Tell();\n  API_END();\n}\n\nint MXRecordIOReaderCreate(const char* uri, RecordIOHandle* out) {\n  API_BEGIN();\n  dmlc::Stream* stream       = dmlc::Stream::Create(uri, \"r\");\n  MXRecordIOContext* context = new MXRecordIOContext;\n  context->reader            = new dmlc::RecordIOReader(stream);\n  context->writer            = nullptr;\n  context->stream            = stream;\n  context->read_buff         = new std::string();\n  *out                       = reinterpret_cast<RecordIOHandle>(context);\n  API_END();\n}\n\nint MXRecordIOReaderFree(RecordIOHandle handle) {\n  API_BEGIN();\n  MXRecordIOContext* context = reinterpret_cast<MXRecordIOContext*>(handle);\n  delete context->reader;\n  delete context->stream;\n  delete context->read_buff;\n  delete context;\n  API_END();\n}\n\nint MXRecordIOReaderReadRecord(RecordIOHandle handle, char const** buf, size_t* size) {\n  API_BEGIN();\n  MXRecordIOContext* context = reinterpret_cast<MXRecordIOContext*>(handle);\n  if (context->reader->NextRecord(context->read_buff)) {\n    *buf  = context->read_buff->c_str();\n    *size = context->read_buff->size();\n  } else {\n    *buf  = nullptr;\n    *size = 0;\n  }\n  API_END();\n}\n\nint MXRecordIOReaderSeek(RecordIOHandle handle, size_t pos) {\n  API_BEGIN();\n  MXRecordIOContext* context = reinterpret_cast<MXRecordIOContext*>(handle);\n  context->reader->Seek(pos);\n  API_END();\n}\n\nint MXRecordIOReaderTell(RecordIOHandle handle, size_t* pos) {\n  API_BEGIN();\n  MXRecordIOContext* context = reinterpret_cast<MXRecordIOContext*>(handle);\n  *pos                       = context->reader->Tell();\n  API_END();\n}\n\nint MXRtcCreate(char* name,\n                uint32_t num_input,\n                uint32_t num_output,\n                char** input_names,\n                char** output_names,\n                NDArrayHandle* inputs,\n                NDArrayHandle* outputs,\n                char* kernel,\n                RtcHandle* out) {\n  API_BEGIN();\n  LOG(FATAL) << \"Old rtc API is deprecated. Please use CudaModule\";\n  API_END();\n}\n\nint MXRtcPush(RtcHandle handle,\n              uint32_t num_input,\n              uint32_t num_output,\n              NDArrayHandle* inputs,\n              NDArrayHandle* outputs,\n              uint32_t gridDimX,\n              uint32_t gridDimY,\n              uint32_t gridDimZ,\n              uint32_t blockDimX,\n              uint32_t blockDimY,\n              uint32_t blockDimZ) {\n  API_BEGIN();\n  LOG(FATAL) << \"Old rtc API is deprecated. Please use CudaModule\";\n  API_END();\n}\n\nint MXRtcFree(RtcHandle handle) {\n  API_BEGIN();\n  LOG(FATAL) << \"Old rtc API is deprecated. Please use CudaModule\";\n  API_END();\n}\n\nint MXCustomOpRegister(const char* op_type, CustomOpPropCreator creator) {\n  API_BEGIN();\n  mxnet::op::custom::CustomOperator::Get()->Register(op_type, creator);\n  API_END();\n}\n\nint MXRtcCudaModuleCreate(const char* source,\n                          int num_options,\n                          const char** options,\n                          int num_exports,\n                          const char** exports,\n                          CudaModuleHandle* out) {\n  API_BEGIN();\n#if MXNET_USE_CUDA\n  std::vector<std::string> str_opts;\n  for (int i = 0; i < num_options; ++i)\n    str_opts.emplace_back(options[i]);\n  std::vector<std::string> str_exports;\n  for (int i = 0; i < num_exports; ++i)\n    str_exports.emplace_back(exports[i]);\n  *out = new rtc::CudaModule(source, str_opts, str_exports);\n#else\n  LOG(FATAL) << \"Compile with USE_CUDA=1 to have CUDA runtime compilation.\";\n#endif\n  API_END();\n}\n\nint MXRtcCudaModuleFree(CudaModuleHandle handle) {\n  API_BEGIN();\n#if MXNET_USE_CUDA\n  delete reinterpret_cast<rtc::CudaModule*>(handle);\n#else\n  LOG(FATAL) << \"Compile with USE_CUDA=1 to have CUDA runtime compilation.\";\n#endif\n  API_END();\n}\n\nint MXRtcCudaKernelCreate(CudaModuleHandle handle,\n                          const char* name,\n                          int num_args,\n                          int* is_ndarray,\n                          int* is_const,\n                          int* arg_types,\n                          CudaKernelHandle* out) {\n  API_BEGIN();\n#if MXNET_USE_CUDA\n  auto module = reinterpret_cast<rtc::CudaModule*>(handle);\n  std::vector<rtc::CudaModule::ArgType> signature;\n  for (int i = 0; i < num_args; ++i) {\n    signature.push_back(rtc::CudaModule::ArgType{static_cast<bool>(is_ndarray[i]),\n                                                 static_cast<bool>(is_const[i]),\n                                                 static_cast<mshadow::TypeFlag>(arg_types[i])});\n  }\n  auto kernel = module->GetKernel(name, signature);\n  *out        = new std::shared_ptr<rtc::CudaModule::Kernel>(kernel);\n#else\n  LOG(FATAL) << \"Compile with USE_CUDA=1 to have CUDA runtime compilation.\";\n#endif\n  API_END();\n}\n\nint MXRtcCudaKernelFree(CudaKernelHandle handle) {\n  API_BEGIN();\n#if MXNET_USE_CUDA\n  delete reinterpret_cast<std::shared_ptr<rtc::CudaModule::Kernel>*>(handle);\n#else\n  LOG(FATAL) << \"Compile with USE_CUDA=1 to have CUDA runtime compilation.\";\n#endif\n  API_END();\n}\n\nint MXRtcCudaKernelCall(CudaKernelHandle handle,\n                        int dev_id,\n                        void** args,\n                        uint32_t grid_dim_x,\n                        uint32_t grid_dim_y,\n                        uint32_t grid_dim_z,\n                        uint32_t block_dim_x,\n                        uint32_t block_dim_y,\n                        uint32_t block_dim_z,\n                        uint32_t shared_mem) {\n  API_BEGIN();\n#if MXNET_USE_CUDA\n  auto kernel           = reinterpret_cast<std::shared_ptr<rtc::CudaModule::Kernel>*>(handle);\n  const auto& signature = (*kernel)->signature();\n  std::vector<dmlc::any> any_args;\n  for (size_t i = 0; i < signature.size(); ++i) {\n    if (signature[i].is_ndarray) {\n      any_args.emplace_back(*static_cast<NDArray*>(args[i]));\n    } else {\n      MSHADOW_TYPE_SWITCH(\n          signature[i].dtype, DType, { any_args.emplace_back(*static_cast<DType*>(args[i])); });\n    }\n  }\n  (*kernel)->Launch(Context::GPU(dev_id),\n                    any_args,\n                    grid_dim_x,\n                    grid_dim_y,\n                    grid_dim_z,\n                    block_dim_x,\n                    block_dim_y,\n                    block_dim_z,\n                    shared_mem);\n#else\n  LOG(FATAL) << \"Compile with USE_CUDA=1 to have CUDA runtime compilation.\";\n#endif\n  API_END();\n}\n\nint MXNDArrayGetSharedMemHandle(NDArrayHandle handle, int* shared_pid, int* shared_id) {\n  API_BEGIN();\n  NDArray* arr = reinterpret_cast<NDArray*>(handle);\n  Storage::Handle shandle;\n  if (arr->ctx().dev_type == Context::kCPUShared) {\n    arr->WaitToRead();\n    shandle = arr->storage_handle();\n    Storage::Get()->SharedIncrementRefCount(shandle);\n  } else {\n    NDArray new_arr(arr->shape(), Context::CPUShared(0), false, arr->dtype());\n    CopyFromTo(*arr, new_arr);\n    new_arr.WaitToRead();\n    shandle = new_arr.storage_handle();\n    Storage::Get()->SharedIncrementRefCount(shandle);\n  }\n  *shared_pid = shandle.shared_pid;\n  *shared_id  = shandle.shared_id;\n  API_END();\n}\n\nint MXNDArrayCreateFromSharedMem(int shared_pid,\n                                 int shared_id,\n                                 const int* shape,\n                                 int ndim,\n                                 int dtype,\n                                 NDArrayHandle* out) {\n  API_BEGIN();\n  NDArray* nd = new NDArray(shared_pid, shared_id, mxnet::TShape(shape, shape + ndim), dtype);\n  nd->AssignStorageInfo(profiler::ProfilerScope::Get()->GetCurrentProfilerScope(),\n                        MXNET_STORAGE_DEFAULT_NAME_CSTR);\n  *out = nd;\n  API_END();\n}\n\nusing VarHandle          = Engine::VarHandle;\nusing CallbackOnStart    = Engine::CallbackOnStart;\nusing CallbackOnComplete = Engine::CallbackOnComplete;\n\nvoid AssertValidNumberVars(int num_const_vars, int num_mutable_vars) {\n  CHECK_GE(num_const_vars, 0) << \"Non-negative number of const vars expected.\";\n  CHECK_GE(num_mutable_vars, 0) << \"Non-negative number of mutable vars expected.\";\n}\n\nint MXEnginePushAsync(EngineAsyncFunc async_func,\n                      void* func_param,\n                      EngineFuncParamDeleter deleter,\n                      ContextHandle ctx_handle,\n                      EngineVarHandle const_vars_handle,\n                      int num_const_vars,\n                      EngineVarHandle mutable_vars_handle,\n                      int num_mutable_vars,\n                      EngineFnPropertyHandle prop_handle,\n                      int priority,\n                      const char* opr_name,\n                      bool wait) {\n  API_BEGIN();\n\n  auto exec_ctx     = *static_cast<const Context*>(ctx_handle);\n  auto const_vars   = static_cast<VarHandle*>(const_vars_handle);\n  auto mutable_vars = static_cast<VarHandle*>(mutable_vars_handle);\n  auto prop         = FnProperty::kNormal;\n  if (prop_handle) {\n    prop = *static_cast<const FnProperty*>(prop_handle);\n  }\n\n  Engine::AsyncFn exec_fn;\n  if (deleter == nullptr) {\n    exec_fn = [async_func, func_param](\n                  RunContext rctx, CallbackOnStart on_start, CallbackOnComplete on_complete) {\n      async_func(&rctx, &on_start, &on_complete, func_param);\n    };\n  } else {\n    // Wrap func_param in a shared_ptr with deleter such that deleter\n    // will be called when the lambda goes out of scope.\n    std::shared_ptr<void> shared_func_param(func_param, deleter);\n    exec_fn = [async_func, shared_func_param](\n                  RunContext rctx, CallbackOnStart on_start, CallbackOnComplete on_complete) {\n      async_func(&rctx, &on_start, &on_complete, shared_func_param.get());\n    };\n  }\n\n  AssertValidNumberVars(num_const_vars, num_mutable_vars);\n  std::vector<VarHandle> const_var_vec(const_vars, const_vars + num_const_vars);\n  std::vector<VarHandle> mutable_var_vec(mutable_vars, mutable_vars + num_mutable_vars);\n  Engine::Get()->PushAsync(\n      exec_fn, exec_ctx, const_var_vec, mutable_var_vec, prop, priority, opr_name, wait);\n\n  API_END();\n}\n\nint MXEnginePushSync(EngineSyncFunc sync_func,\n                     void* func_param,\n                     EngineFuncParamDeleter deleter,\n                     ContextHandle ctx_handle,\n                     EngineVarHandle const_vars_handle,\n                     int num_const_vars,\n                     EngineVarHandle mutable_vars_handle,\n                     int num_mutable_vars,\n                     EngineFnPropertyHandle prop_handle,\n                     int priority,\n                     const char* opr_name) {\n  API_BEGIN();\n\n  auto exec_ctx     = *static_cast<const Context*>(ctx_handle);\n  auto const_vars   = static_cast<VarHandle*>(const_vars_handle);\n  auto mutable_vars = static_cast<VarHandle*>(mutable_vars_handle);\n  auto prop         = FnProperty::kNormal;\n  if (prop_handle) {\n    prop = *static_cast<const FnProperty*>(prop_handle);\n  }\n\n  Engine::SyncFn exec_fn;\n  if (deleter == nullptr) {\n    exec_fn = [sync_func, func_param](RunContext rctx) { sync_func(&rctx, func_param); };\n  } else {\n    // Wrap func_param in a shared_ptr with deleter such that deleter\n    // will be called when the lambda goes out of scope.\n    std::shared_ptr<void> shared_func_param(func_param, deleter);\n    exec_fn = [sync_func, shared_func_param](RunContext rctx) {\n      sync_func(&rctx, shared_func_param.get());\n    };\n  }\n\n  AssertValidNumberVars(num_const_vars, num_mutable_vars);\n  std::vector<VarHandle> const_var_vec(const_vars, const_vars + num_const_vars);\n  std::vector<VarHandle> mutable_var_vec(mutable_vars, mutable_vars + num_mutable_vars);\n  Engine::Get()->PushSync(\n      exec_fn, exec_ctx, const_var_vec, mutable_var_vec, prop, priority, opr_name);\n\n  API_END();\n}\n\nint MXEnginePushAsyncND(EngineAsyncFunc async_func,\n                        void* func_param,\n                        EngineFuncParamDeleter deleter,\n                        ContextHandle ctx_handle,\n                        NDArrayHandle* const_nds_handle,\n                        int num_const_nds,\n                        NDArrayHandle* mutable_nds_handle,\n                        int num_mutable_nds,\n                        EngineFnPropertyHandle prop_handle,\n                        int priority,\n                        const char* opr_name,\n                        bool wait) {\n  API_BEGIN();\n  NDArray** const_nds   = reinterpret_cast<NDArray**>(const_nds_handle);\n  NDArray** mutable_nds = reinterpret_cast<NDArray**>(mutable_nds_handle);\n  std::vector<VarHandle> const_var_vec(num_const_nds);\n  for (int i = 0; i < num_const_nds; ++i)\n    const_var_vec[i] = const_nds[i]->var();\n  std::vector<VarHandle> mutable_var_vec(num_mutable_nds);\n  for (int i = 0; i < num_mutable_nds; ++i)\n    mutable_var_vec[i] = mutable_nds[i]->var();\n  return MXEnginePushAsync(async_func,\n                           func_param,\n                           deleter,\n                           ctx_handle,\n                           const_var_vec.data(),\n                           num_const_nds,\n                           mutable_var_vec.data(),\n                           num_mutable_nds,\n                           prop_handle,\n                           priority,\n                           opr_name,\n                           wait);\n  API_END();\n}\n\nint MXEnginePushSyncND(EngineSyncFunc sync_func,\n                       void* func_param,\n                       EngineFuncParamDeleter deleter,\n                       ContextHandle ctx_handle,\n                       NDArrayHandle* const_nds_handle,\n                       int num_const_nds,\n                       NDArrayHandle* mutable_nds_handle,\n                       int num_mutable_nds,\n                       EngineFnPropertyHandle prop_handle,\n                       int priority,\n                       const char* opr_name) {\n  API_BEGIN();\n  NDArray** const_nds   = reinterpret_cast<NDArray**>(const_nds_handle);\n  NDArray** mutable_nds = reinterpret_cast<NDArray**>(mutable_nds_handle);\n  std::vector<VarHandle> const_var_vec(num_const_nds);\n  for (int i = 0; i < num_const_nds; ++i)\n    const_var_vec[i] = const_nds[i]->var();\n  std::vector<VarHandle> mutable_var_vec(num_mutable_nds);\n  for (int i = 0; i < num_mutable_nds; ++i)\n    mutable_var_vec[i] = mutable_nds[i]->var();\n  return MXEnginePushSync(sync_func,\n                          func_param,\n                          deleter,\n                          ctx_handle,\n                          const_var_vec.data(),\n                          num_const_nds,\n                          mutable_var_vec.data(),\n                          num_mutable_nds,\n                          prop_handle,\n                          priority,\n                          opr_name);\n  API_END();\n}\n\nint MXStorageEmptyCache(int dev_type, int dev_id) {\n  API_BEGIN();\n  Context ctx = Context::Create(static_cast<Context::DeviceType>(dev_type), dev_id);\n  Storage::Get()->ReleaseAll(ctx);\n  API_END();\n}\n\nint MXShallowCopyNDArray(NDArrayHandle src_handle, NDArrayHandle* out) {\n  NDArray* ret = nullptr;\n  API_BEGIN();\n  NDArray* src_array = static_cast<NDArray*>(src_handle);\n  ret                = new NDArray(*src_array);\n  *out               = ret;\n  API_END_HANDLE_ERROR(delete ret);\n}\n\nint MXPushStreamDep(NDArrayHandle handle, int stream) {\n  API_BEGIN();\n  static_cast<NDArray*>(handle)->StreamSync(stream);\n  API_END();\n}\n\nint MXGetCurrentStream(int device_id, int* stream) {\n  API_BEGIN();\n#if MXNET_USE_CUDA\n  RunContext rctx{Context::GPU(device_id), new mshadow::Stream<gpu>(), nullptr};\n  mshadow::Stream<gpu>* cur_stream = rctx.get_stream<gpu>();\n  *stream = reinterpret_cast<int64_t>(mshadow::Stream<gpu>::GetStream(cur_stream));\n#else\n  LOG(FATAL) << \"GPU is not enabled.\";\n#endif\n  API_END();\n}\n\nint MXNVTXRangePush(const char* name, mx_uint color) {\n  API_BEGIN();\n#if MXNET_USE_CUDA && MXNET_USE_NVTX\n  mxnet::common::cuda::nvtx::gpuRangeStart(color, name);\n#else\n  LOG(FATAL) << \"Compile with USE_CUDA=1 and USE_NVTX=1 to have NVTX support.\";\n#endif\n  API_END();\n}\n\nint MXNVTXRangePop() {\n  API_BEGIN();\n#if MXNET_USE_CUDA && MXNET_USE_NVTX\n  mxnet::common::cuda::nvtx::gpuRangeStop();\n#else\n  LOG(FATAL) << \"Compile with USE_CUDA=1 and USE_NVTX=1 to have NVTX support.\";\n#endif\n  API_END();\n}\n\nint MXCUDAProfilerStart() {\n  API_BEGIN();\n#if MXNET_USE_CUDA && MXNET_USE_NVTX\n  cudaProfilerStart();\n#else\n  LOG(FATAL) << \"Compile with USE_CUDA=1 and USE_NVTX=1 to have CUDA profiler support.\";\n#endif\n  API_END();\n}\n\nint MXCUDAProfilerStop() {\n  API_BEGIN();\n#if MXNET_USE_CUDA && MXNET_USE_NVTX\n  cudaProfilerStop();\n#else\n  LOG(FATAL) << \"Compile with USE_CUDA=1 and USE_NVTX=1 to have CUDA Profiler support.\";\n#endif\n  API_END();\n}\n\nint MXSetOptimizeLayout(bool val) {\n  API_BEGIN();\n  mxnet::alm::ALMParams::get().optimize = val;\n  API_END();\n}\n\nint MXGetOptimizeLayout(bool* val) {\n  API_BEGIN();\n  *val = mxnet::alm::ALMParams::get().optimize;\n  API_END();\n}\n"
  },
  {
    "path": "src/c_api/c_api_common.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file c_api_common.h\n * \\brief Common C API utils\n */\n#ifndef MXNET_C_API_C_API_COMMON_H_\n#define MXNET_C_API_C_API_COMMON_H_\n\n#include <dmlc/base.h>\n#include <dmlc/logging.h>\n#include <dmlc/thread_local.h>\n#include <mxnet/c_api.h>\n#include <mxnet/c_api_error.h>\n#include <mxnet/base.h>\n#include <mxnet/op_attr_types.h>\n#include <nnvm/graph.h>\n#include <vector>\n#include <string>\n\n/*!\n * \\brief Macros to guard beginning and end section of all functions\n * every function starts with API_BEGIN()\n * and finishes with API_END() or API_END_HANDLE_ERROR()\n * The finally clause contains procedure to cleanup states when an error happens.\n */\n#ifndef API_BEGIN\n#define API_BEGIN MX_API_BEGIN\n#endif\n\n#ifndef API_END\n#define API_END MX_API_END\n#endif\n\n#ifndef API_END_HANDLE_ERROR\n#define API_END_HANDLE_ERROR MX_API_END_HANDLE_ERROR\n#endif\n\nusing namespace mxnet;\n\n/*! \\brief entry to to easily hold returning information */\ntemplate <typename dtype = int>\nstruct MXAPIThreadLocalEntry {\n  /*! \\brief result holder for returning string */\n  std::string ret_str;\n  /*! \\brief result holder for returning strings */\n  std::vector<std::string> ret_vec_str;\n  /*! \\brief result holder for returning string pointers */\n  std::vector<const char*> ret_vec_charp;\n  /*! \\brief result holder for returning handles */\n  std::vector<void*> ret_handles;\n  /*! \\brief holder for NDArray handles */\n  std::vector<NDArray*> ndinputs, ndoutputs;\n  /*! \\brief result holder for returning shapes */\n  mxnet::ShapeVector arg_shapes, out_shapes, aux_shapes;\n  /*! \\brief result holder for returning type flags */\n  std::vector<int> arg_types, out_types, aux_types;\n  /*! \\brief result holder for returning storage types */\n  std::vector<int> arg_storage_types, out_storage_types, aux_storage_types;\n  /*! \\brief result holder for returning shape dimensions */\n  std::vector<uint32_t> arg_shape_ndim, out_shape_ndim, aux_shape_ndim;\n  /*! \\brief result holder for returning shape dimensions */\n  std::vector<int> arg_shape_ndim_ex, out_shape_ndim_ex, aux_shape_ndim_ex;\n  /*! \\brief result holder for returning shape pointer */\n  std::vector<const uint32_t*> arg_shape_data, out_shape_data, aux_shape_data;\n  /*! \\brief result holder for returning shape pointer */\n  std::vector<const dtype*> arg_shape_data_ex, out_shape_data_ex, aux_shape_data_ex;\n  /*! \\brief uint32_t buffer for returning shape pointer */\n  std::vector<uint32_t> arg_shape_buffer, out_shape_buffer, aux_shape_buffer;\n  /*! \\brief uint32_t buffer for returning shape pointer */\n  std::vector<dtype> arg_shape_buffer_ex, out_shape_buffer_ex, aux_shape_buffer_ex;\n  /*! \\brief bool buffer */\n  std::vector<bool> save_inputs, save_outputs;\n  // DEPRECATED. Use SetupShapeArrayReturnWithBufferEx instead.\n  // helper function to setup return value of shape array\n  inline static void SetupShapeArrayReturnWithBuffer(const mxnet::ShapeVector& shapes,\n                                                     std::vector<uint32_t>* ndim,\n                                                     std::vector<const uint32_t*>* data,\n                                                     std::vector<uint32_t>* buffer) {\n    ndim->resize(shapes.size());\n    data->resize(shapes.size());\n    size_t size = 0;\n    for (const auto& s : shapes)\n      size += s.ndim();\n    buffer->resize(size);\n    uint32_t* ptr = buffer->data();\n    for (size_t i = 0; i < shapes.size(); ++i) {\n      ndim->at(i) = shapes[i].ndim();\n      data->at(i) = ptr;\n      ptr         = nnvm::ShapeTypeCast(shapes[i].begin(), shapes[i].end(), ptr);\n    }\n  }\n  // helper function to setup return value of shape array\n  inline static void SetupShapeArrayReturnWithBufferEx(const mxnet::ShapeVector& shapes,\n                                                       std::vector<int>* ndim,\n                                                       std::vector<const dtype*>* data,\n                                                       std::vector<dtype>* buffer) {\n    ndim->resize(shapes.size());\n    data->resize(shapes.size());\n    size_t size = 0;\n    for (const auto& s : shapes) {\n      if (s.ndim() > 0) {\n        size += s.ndim();\n      }\n    }\n    buffer->resize(size);\n    dtype* ptr = buffer->data();\n    for (size_t i = 0; i < shapes.size(); ++i) {\n      ndim->at(i) = shapes[i].ndim();\n      data->at(i) = ptr;\n      if (shapes[i].ndim() > 0) {\n        ptr = mxnet::ShapeTypeCast(shapes[i].begin(), shapes[i].end(), ptr);\n      }\n    }\n  }\n};\n\n// define the threadlocal store.\ntemplate <typename dtype = int>\nusing MXAPIThreadLocalStore = dmlc::ThreadLocalStore<MXAPIThreadLocalEntry<dtype>>;\n\nnamespace mxnet {\n// copy attributes from inferred vector back to the vector of each type.\ntemplate <typename AttrType>\ninline void CopyAttr(const nnvm::IndexedGraph& idx,\n                     const std::vector<AttrType>& attr_vec,\n                     std::vector<AttrType>* in_attr,\n                     std::vector<AttrType>* out_attr,\n                     std::vector<AttrType>* aux_attr) {\n  in_attr->clear();\n  out_attr->clear();\n  aux_attr->clear();\n  for (uint32_t nid : idx.input_nodes()) {\n    if (idx.mutable_input_nodes().count(nid) == 0) {\n      in_attr->push_back(attr_vec[idx.entry_id(nid, 0)]);\n    } else {\n      aux_attr->push_back(attr_vec[idx.entry_id(nid, 0)]);\n    }\n  }\n  for (auto& e : idx.outputs()) {\n    out_attr->push_back(attr_vec[idx.entry_id(e)]);\n  }\n}\n\n// stores keys that will be converted to __key__\nextern const std::vector<std::string> kHiddenKeys;\n}  // namespace mxnet\n\n#endif  // MXNET_C_API_C_API_COMMON_H_\n"
  },
  {
    "path": "src/c_api/c_api_function.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file custom.cc\n * \\brief\n * \\author Junyuan Xie\n */\n#include <mxnet/c_api.h>\n#include <mxnet/base.h>\n#include <mxnet/ndarray.h>\n#include <mxnet/imperative.h>\n\n#include \"./c_api_common.h\"\n#include \"../operator/operator_common.h\"\n#include \"../operator/custom/custom-inl.h\"\n\nnamespace mxnet {\nnamespace custom_function {\n\nstruct CustomFunctionParam {\n  size_t num_args, num_outs;\n  std::shared_ptr<MXCallbackList> info;\n  std::vector<mxnet::TShape> out_shapes;\n  std::vector<int> out_dtypes;\n};\n\nstd::vector<nnvm::NodeEntry> Gradient(const nnvm::ObjectPtr& n,\n                                      const std::vector<nnvm::NodeEntry>& out_grads) {\n  const CustomFunctionParam& params = nnvm::get<CustomFunctionParam>(n->attrs.parsed);\n\n  nnvm::ObjectPtr g = nnvm::Node::Create();\n  g->attrs.op       = nnvm::Op::Get(\"_backward_CustomFunction\");\n  g->attrs.name     = n->attrs.name + \"_backward\";\n  g->attrs.parsed   = params;\n  g->control_deps.emplace_back(n);\n\n  g->inputs = out_grads;\n\n  std::vector<nnvm::NodeEntry> ret;\n  for (uint32_t i = 0; i < g->num_outputs(); ++i) {\n    ret.emplace_back(g, i, 0);\n  }\n\n  return ret;\n}\n\nOpStatePtr CreateState(const nnvm::NodeAttrs& attrs,\n                       Context ctx,\n                       const mxnet::ShapeVector& ishape,\n                       const std::vector<int>& itype) {\n  LOG(FATAL) << \"Not reached\";\n  return OpStatePtr::Create<void*>(nullptr);\n}\n\nvoid Forward(const OpStatePtr& state,\n             const OpContext& ctx,\n             const std::vector<NDArray>& inputs,\n             const std::vector<OpReqType>& req,\n             const std::vector<NDArray>& outputs) {\n  LOG(FATAL) << \"Not reached\";\n}\n\nvoid Backward(const OpStatePtr& state,\n              const OpContext& ctx,\n              const std::vector<NDArray>& inputs,\n              const std::vector<OpReqType>& req,\n              const std::vector<NDArray>& outputs) {\n  const CustomFunctionParam& params = state.get_state<CustomFunctionParam>();\n\n  std::vector<NDArrayHandle> ptrs;\n  std::vector<NDArray> cpys;\n  std::vector<int> tags;\n  std::unordered_set<int> input_tags({0});\n  std::unordered_set<int> output_tags({1});\n\n  auto dev_id = ctx.run_ctx.ctx.dev_id;\n\n  for (const auto& i : inputs) {\n    NDArray* nd = new NDArray(i.data(), dev_id);\n    ptrs.push_back(reinterpret_cast<NDArrayHandle>(nd));\n    cpys.push_back(*nd);\n    tags.push_back(0);\n  }\n  for (const auto& i : outputs) {\n    NDArray* nd = new NDArray(i.data(), dev_id);\n    ptrs.push_back(reinterpret_cast<NDArrayHandle>(nd));\n    cpys.push_back(*nd);\n    tags.push_back(1);\n  }\n\n  op::custom::CustomOperator::Get()->Push(\n      [=]() {\n        CHECK(reinterpret_cast<CustomFunctionBwdFunc>(\n            params.info->callbacks[kCustomFunctionBackward])(\n            inputs.size(),\n            outputs.size(),\n            const_cast<NDArrayHandle*>(ptrs.data()),\n            reinterpret_cast<const int*>(req.data()),\n            ctx.is_train,\n            params.info->contexts[kCustomFunctionBackward]));\n      },\n      ctx,\n      false,\n      ctx.is_train,\n      cpys,\n      tags,\n      output_tags,\n      outputs);\n}\n\ninline bool InferStorageType(const nnvm::NodeAttrs& attrs,\n                             const int dev_mask,\n                             DispatchMode* dispatch_mode,\n                             std::vector<int>* iattr,\n                             std::vector<int>* oattr) {\n  using namespace op;\n\n  for (size_t i = 0; i < iattr->size(); ++i) {\n    STORAGE_TYPE_ASSIGN_CHECK(*iattr, i, kDefaultStorage);\n  }\n  for (size_t i = 0; i < oattr->size(); ++i) {\n    STORAGE_TYPE_ASSIGN_CHECK(*oattr, i, kDefaultStorage);\n  }\n  DISPATCH_MODE_ASSIGN_CHECK(dispatch_mode, 0, DispatchMode::kFComputeEx);\n  return true;\n}\n\nNNVM_REGISTER_OP(_CustomFunction)\n    .set_num_inputs([](const NodeAttrs& attrs) {\n      const CustomFunctionParam& params = nnvm::get<CustomFunctionParam>(attrs.parsed);\n      return params.num_args;\n    })\n    .set_num_outputs([](const NodeAttrs& attrs) {\n      const CustomFunctionParam& params = nnvm::get<CustomFunctionParam>(attrs.parsed);\n      return params.num_outs;\n    })\n    .set_attr<mxnet::FInferShape>(\n        \"FInferShape\",\n        [](const NodeAttrs& attrs, mxnet::ShapeVector* in_shape, mxnet::ShapeVector* out_shape) {\n          const CustomFunctionParam& params = nnvm::get<CustomFunctionParam>(attrs.parsed);\n          *out_shape                        = params.out_shapes;\n          return true;\n        })\n    .set_attr<nnvm::FInferType>(\n        \"FInferType\",\n        [](const NodeAttrs& attrs, std::vector<int>* in_type, std::vector<int>* out_type) {\n          const CustomFunctionParam& params = nnvm::get<CustomFunctionParam>(attrs.parsed);\n          *out_type                         = params.out_dtypes;\n          return true;\n        })\n    .set_attr<FCreateOpState>(\"FCreateOpState\", CreateState)\n    .set_attr<nnvm::FGradient>(\"FGradient\", Gradient)\n    .set_attr<FStatefulComputeEx>(\"FStatefulComputeEx<cpu>\", Forward)\n    .set_attr<FStatefulComputeEx>(\"FStatefulComputeEx<gpu>\", Forward)\n    .set_attr<FInferStorageType>(\"FInferStorageType\", InferStorageType);\n\nNNVM_REGISTER_OP(_backward_CustomFunction)\n    .set_num_inputs([](const NodeAttrs& attrs) {\n      const CustomFunctionParam& params = nnvm::get<CustomFunctionParam>(attrs.parsed);\n      return params.num_outs;\n    })\n    .set_num_outputs([](const NodeAttrs& attrs) {\n      const CustomFunctionParam& params = nnvm::get<CustomFunctionParam>(attrs.parsed);\n      return params.num_args;\n    })\n    .set_attr<bool>(\"TIsBackward\", true)\n    .set_attr<bool>(\"TIsLayerOpBackward\", true)\n    .set_attr<FExecType>(\"FExecType\", [](const NodeAttrs& attrs) { return ExecType::kAsync; })\n    .set_attr<FStatefulComputeEx>(\"FStatefulComputeEx<cpu>\", Backward)\n    .set_attr<FStatefulComputeEx>(\"FStatefulComputeEx<gpu>\", Backward)\n    .set_attr<FInferStorageType>(\"FInferStorageType\", InferStorageType);\n\n}  // namespace custom_function\n}  // namespace mxnet\n\nint MXCustomFunctionRecord(int num_inputs,\n                           NDArrayHandle* inputs,\n                           int num_outputs,\n                           NDArrayHandle* outputs,\n                           MXCallbackList* callbacks) {\n  using namespace mxnet;\n  using namespace mxnet::custom_function;\n  API_BEGIN();\n  CHECK(Imperative::Get()->is_recording());\n  auto state                  = OpStatePtr::Create<CustomFunctionParam>();\n  CustomFunctionParam& params = state.get_state<CustomFunctionParam>();\n  params.num_args             = num_inputs;\n  params.num_outs             = num_outputs;\n  params.info.reset(callbacks, [](MXCallbackList* ptr) {\n    reinterpret_cast<CustomFunctionDelFunc>(ptr->callbacks[kCustomFunctionDelete])(\n        ptr->contexts[kCustomFunctionDelete]);\n  });\n  std::vector<NDArray*> ndinputs, ndoutputs;\n  ndinputs.reserve(num_inputs);\n  ndoutputs.reserve(num_outputs);\n  params.out_shapes.reserve(num_outputs);\n  params.out_dtypes.reserve(num_outputs);\n  for (int i = 0; i < num_inputs; ++i) {\n    ndinputs.emplace_back(reinterpret_cast<NDArray*>(inputs[i]));\n  }\n  for (int i = 0; i < num_outputs; ++i) {\n    NDArray* arr = reinterpret_cast<NDArray*>(outputs[i]);\n    ndoutputs.emplace_back(arr);\n    params.out_shapes.emplace_back(arr->shape());\n    params.out_dtypes.emplace_back(arr->dtype());\n  }\n  nnvm::NodeAttrs attrs;\n  attrs.op     = nnvm::Op::Get(\"_CustomFunction\");\n  attrs.parsed = params;\n  Imperative::Get()->RecordOp(std::move(attrs), ndinputs, ndoutputs, state);\n\n  API_END();\n}\n"
  },
  {
    "path": "src/c_api/c_api_ndarray.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file c_api_ndarray.cc\n * \\brief C API of mxnet\n */\n\n#include <mxnet/base.h>\n#include <mxnet/c_api.h>\n#include <mxnet/operator.h>\n#include <mxnet/operator_util.h>\n#include <mxnet/op_attr_types.h>\n#include <mxnet/imperative.h>\n#include <nnvm/node.h>\n#include <nnvm/op_attr_types.h>\n#include <string>\n#include \"./c_api_common.h\"\n#include \"../common/utils.h\"\n#include \"../common/exec_utils.h\"\n#include \"../imperative/imperative_utils.h\"\n#include \"../imperative/cached_op.h\"\n#include \"../imperative/cached_op_threadsafe.h\"\n#include \"../profiler/profiler.h\"\n\nusing namespace mxnet;\n\nvoid SetNDInputsOutputs(const nnvm::Op* op,\n                        std::vector<NDArray*>* ndinputs,\n                        std::vector<NDArray*>* ndoutputs,\n                        int num_inputs,\n                        const NDArrayHandle* inputs,\n                        int* num_outputs,\n                        int infered_num_outputs,\n                        int num_visible_outputs,\n                        NDArrayHandle** outputs) {\n  NDArray** out_array = *reinterpret_cast<NDArray***>(outputs);\n\n  ndinputs->clear();\n  ndinputs->reserve(num_inputs);\n  for (int i = 0; i < num_inputs; ++i) {\n    NDArray* inp = reinterpret_cast<NDArray*>(inputs[i]);\n    if (!features::is_enabled(features::INT64_TENSOR_SIZE)) {\n      if (shape_is_known(inp->shape())) {  // Shape may be unknown after dynamic shape operators\n        CHECK_LT(inp->shape().Size(), (int64_t{1} << 31) - 1)\n            << \"[SetNDInputsOutputs] Size of tensor you are trying to allocate is larger than \"\n               \"2^31 elements. Please build with flag USE_INT64_TENSOR_SIZE=1\";\n      }\n    }\n    ndinputs->emplace_back(inp);\n  }\n\n  ndoutputs->clear();\n  ndoutputs->reserve(infered_num_outputs);\n  if (out_array == nullptr) {\n    for (int i = 0; i < infered_num_outputs; ++i) {\n      ndoutputs->emplace_back(new NDArray());\n    }\n    *num_outputs = num_visible_outputs;\n  } else {\n    CHECK(*num_outputs == infered_num_outputs || *num_outputs == num_visible_outputs)\n        << \"Operator expects \" << infered_num_outputs << \" (all) or \" << num_visible_outputs\n        << \" (visible only) outputs, but got \" << *num_outputs << \" instead.\";\n    for (int i = 0; i < *num_outputs; ++i) {\n      ndoutputs->emplace_back(out_array[i]);\n    }\n    for (int i = *num_outputs; i < infered_num_outputs; ++i) {\n      ndoutputs->emplace_back(new NDArray());\n    }\n  }\n}\n\nvoid MXImperativeInvokeImpl(AtomicSymbolCreator creator,\n                            int num_inputs,\n                            NDArrayHandle* inputs,\n                            int* num_outputs,\n                            NDArrayHandle** outputs,\n                            int num_params,\n                            const char** param_keys,\n                            const char** param_vals) {\n  const nnvm::Op* op           = static_cast<nnvm::Op*>(creator);\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n\n  nnvm::NodeAttrs attrs =\n      imperative::ParseAttrs(op, num_inputs, num_params, param_keys, param_vals);\n  attrs.dict[\"__profiler_scope__\"] = profiler::ProfilerScope::Get()->GetCurrentProfilerScope();\n  if (attrs.op) {\n    attrs.name = attrs.op->name;\n  }\n\n  int infered_num_outputs;\n  int num_visible_outputs;\n  imperative::SetNumOutputs(op, attrs, num_inputs, &infered_num_outputs, &num_visible_outputs);\n\n  std::vector<NDArray*> ndinputs, ndoutputs;\n  SetNDInputsOutputs(op,\n                     &ndinputs,\n                     &ndoutputs,\n                     num_inputs,\n                     inputs,\n                     num_outputs,\n                     infered_num_outputs,\n                     num_visible_outputs,\n                     outputs);\n\n  if (Imperative::Get()->is_deferred_compute()) {\n    Imperative::Get()->RecordDeferredCompute(std::move(attrs), ndinputs, ndoutputs);\n  } else {\n    for (NDArray* input : ndinputs) {\n      Imperative::DCInfo::Compute(*input);\n    }\n    auto state = Imperative::Get()->Invoke(Context::CPU(), attrs, ndinputs, ndoutputs);\n    if (Imperative::Get()->is_recording()) {\n      Imperative::Get()->RecordOp(std::move(attrs), ndinputs, ndoutputs, state);\n    }\n  }\n\n  for (int i = *num_outputs; i < infered_num_outputs; ++i)\n    delete ndoutputs[i];\n\n  if (*outputs == nullptr) {\n    ret->ret_handles.clear();\n    ret->ret_handles.reserve(*num_outputs);\n    for (int i = 0; i < *num_outputs; ++i)\n      ret->ret_handles.push_back(ndoutputs[i]);\n    *outputs = reinterpret_cast<NDArrayHandle*>(dmlc::BeginPtr(ret->ret_handles));\n  }\n}\n\nint MXImperativeInvoke(AtomicSymbolCreator creator,\n                       int num_inputs,\n                       NDArrayHandle* inputs,\n                       int* num_outputs,\n                       NDArrayHandle** outputs,\n                       int num_params,\n                       const char** param_keys,\n                       const char** param_vals,\n                       const int** out_stypes) {  // outputs storage types\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  API_BEGIN();\n  MXImperativeInvokeImpl(\n      creator, num_inputs, inputs, num_outputs, outputs, num_params, param_keys, param_vals);\n  if (out_stypes != nullptr) {\n    NDArray** out_array = *reinterpret_cast<NDArray***>(outputs);\n    ret->out_types.clear();\n    ret->out_types.reserve(*num_outputs);\n    for (int i = 0; i < *num_outputs; ++i) {\n      ret->out_types.emplace_back(out_array[i]->storage_type());\n    }\n    *out_stypes = dmlc::BeginPtr(ret->out_types);\n  }\n  API_END();\n}\n\nint MXCreateCachedOp(SymbolHandle handle,\n                     int num_flags,\n                     const char** keys,\n                     const char** vals,\n                     CachedOpHandle* out,\n                     bool thread_safe) {\n  nnvm::Symbol* sym = static_cast<nnvm::Symbol*>(handle);\n  API_BEGIN();\n  std::vector<std::pair<std::string, std::string> > flags;\n  flags.reserve(num_flags);\n  for (int i = 0; i < num_flags; ++i) {\n    flags.emplace_back(keys[i], vals[i]);\n  }\n  if (!thread_safe) {\n    *out = new CachedOpPtr(new CachedOp(*sym, flags));\n  } else {\n    *out = new CachedOpPtr(new CachedOpThreadSafe(*sym, flags));\n  }\n  API_END();\n}\n\nint MXFreeCachedOp(CachedOpHandle handle) {\n  CachedOpPtr* g = static_cast<CachedOpPtr*>(handle);\n  API_BEGIN();\n  delete g;\n  API_END();\n}\n\n/*!\n * \\brief get optimized graph from the cached op\n */\nint MXCachedOpGetOptimizedSymbol(CachedOpHandle handle, SymbolHandle* out) {\n  auto s = new nnvm::Symbol();\n  API_BEGIN();\n  CachedOpPtr op = *static_cast<CachedOpPtr*>(handle);\n  *s             = op->GetOptimizedSymbol();\n  *out           = s;\n  API_END_HANDLE_ERROR(delete s);\n}\n\nint MXInvokeCachedOp(CachedOpHandle handle,\n                     int num_inputs,\n                     NDArrayHandle* inputs,\n                     int default_dev_type,\n                     int default_dev_id,\n                     int* num_outputs,\n                     NDArrayHandle** outputs,\n                     const int** out_stypes) {  // outputs storage types\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n\n  API_BEGIN();\n  CachedOpPtr op_shared = *static_cast<CachedOpPtr*>(handle);\n  // CachedOp* points to CachedOpThreadSafe object if CreateCachedOpEX\n  // was called with thread_safe=true\n  CachedOp* op = dynamic_cast<CachedOp*>(op_shared.get());\n  std::vector<NDArray*> ndinputs;\n  ndinputs.reserve(num_inputs);\n  for (int i = 0; i < num_inputs; ++i) {\n    ndinputs.push_back(reinterpret_cast<NDArray*>(inputs[i]));\n  }\n\n  std::vector<NDArray*> ndoutputs;\n  ndoutputs.reserve(op->num_outputs());\n  if (*outputs == nullptr) {\n    *num_outputs = op->num_outputs();\n    for (int i = 0; i < *num_outputs; ++i)\n      ndoutputs.push_back(new NDArray());\n  } else {\n    CHECK_EQ(*num_outputs, op->num_outputs()) << \"CachedOp expects \" << op->num_outputs()\n                                              << \" outputs, but \" << *num_outputs << \" was given.\";\n    for (int i = 0; i < *num_outputs; ++i) {\n      ndoutputs.push_back(reinterpret_cast<NDArray*>((*outputs)[i]));\n    }\n  }\n  // construct default context\n  Context ctx = Context::Create(static_cast<Context::DeviceType>(default_dev_type), default_dev_id);\n  op->Forward(op_shared, ndinputs, ndoutputs, ctx);\n\n  if (*outputs == nullptr) {\n    ret->ret_handles.clear();\n    ret->ret_handles.reserve(*num_outputs);\n    for (int i = 0; i < *num_outputs; ++i) {\n      ret->ret_handles.push_back(ndoutputs[i]);\n    }\n    *outputs = dmlc::BeginPtr(ret->ret_handles);\n  }\n  if (out_stypes != nullptr) {\n    NDArray** out_array = reinterpret_cast<NDArray**>(*outputs);\n    ret->out_types.clear();\n    ret->out_types.reserve(*num_outputs);\n    for (int i = 0; i < *num_outputs; ++i) {\n      ret->out_types.emplace_back(out_array[i]->storage_type());\n    }\n    *out_stypes = dmlc::BeginPtr(ret->out_types);\n  }\n\n  API_END();\n}\n\nint MXAutogradIsTraining(bool* curr) {\n  API_BEGIN();\n  *curr = Imperative::Get()->is_training();\n  API_END();\n}\n\nint MXAutogradSetIsTraining(int is_training, int* prev) {\n  API_BEGIN();\n  *prev = Imperative::Get()->set_is_training(static_cast<bool>(is_training));\n  API_END();\n}\n\nint MXAutogradIsRecording(bool* curr) {\n  API_BEGIN();\n  *curr = Imperative::Get()->is_recording();\n  API_END();\n}\n\nint MXAutogradSetIsRecording(int is_recording, int* prev) {\n  API_BEGIN();\n  *prev = Imperative::Get()->set_is_recording(static_cast<bool>(is_recording));\n  API_END();\n}\n\nint MXSetOptimizationConstraints(unsigned int constraints, unsigned int* prev) {\n  API_BEGIN();\n  *prev =\n      static_cast<unsigned int>(Imperative::Get()->set_opt_constraints(OptConstraint(constraints)));\n  API_END();\n}\n\nint MXGetOptimizationConstraints(unsigned int* curr) {\n  API_BEGIN();\n  *curr = static_cast<unsigned int>(Imperative::Get()->get_opt_constraints());\n  API_END();\n}\n\nint MXIsNumpyShape(int* curr) {\n  API_BEGIN();\n  *curr = Imperative::Get()->is_np_shape();\n  API_END();\n}\n\nint MXSetIsNumpyShape(int is_np_shape, int* prev) {\n  API_BEGIN();\n  *prev = Imperative::Get()->set_is_np_shape(is_np_shape);\n  API_END();\n}\n\nint MXIsNumpyDefaultDtype(bool* curr) {\n  API_BEGIN();\n  *curr = Imperative::Get()->is_np_default_dtype();\n  API_END();\n}\n\nint MXSetIsNumpyDefaultDtype(bool default_dtype, bool* prev) {\n  API_BEGIN();\n  *prev = Imperative::Get()->set_is_np_default_dtype(default_dtype);\n  API_END();\n}\n\nint MXAutogradMarkVariables(uint32_t num_var,\n                            NDArrayHandle* var_handles,\n                            uint32_t* reqs_array,\n                            NDArrayHandle* grad_handles) {\n  API_BEGIN();\n  std::vector<NDArray*> variables, gradients;\n  std::vector<uint32_t> grad_reqs;\n  variables.reserve(num_var);\n  gradients.reserve(num_var);\n  grad_reqs.reserve(num_var);\n  for (uint32_t i = 0; i < num_var; ++i) {\n    variables.emplace_back(static_cast<NDArray*>(var_handles[i]));\n    gradients.emplace_back(static_cast<NDArray*>(grad_handles[i]));\n    grad_reqs.emplace_back(reqs_array[i]);\n  }\n  Imperative::Get()->MarkVariables(variables, grad_reqs, gradients);\n  API_END();\n}\n\nint MXAutogradDropGrads(uint32_t num_var, NDArrayHandle* var_handles) {\n  API_BEGIN();\n  std::vector<NDArray*> variables;\n  variables.reserve(num_var);\n  for (uint32_t i = 0; i < num_var; ++i) {\n    variables.emplace_back(static_cast<NDArray*>(var_handles[i]));\n  }\n  Imperative::Get()->DropGrads(variables);\n  API_END();\n}\n\nint MXAutogradComputeGradient(uint32_t num_output, NDArrayHandle* output_handles) {\n  return MXAutogradBackward(num_output, output_handles, nullptr, 0);\n}\n\nint MXAutogradBackward(uint32_t num_output,\n                       NDArrayHandle* output_handles,\n                       NDArrayHandle* ograd_handles,\n                       int retain_graph) {\n  return MXAutogradBackwardEx(num_output,\n                              output_handles,\n                              ograd_handles,\n                              0,\n                              nullptr,\n                              retain_graph,\n                              false,\n                              true,\n                              nullptr,\n                              nullptr);\n}\n\nint MXAutogradBackwardEx(uint32_t num_output,\n                         NDArrayHandle* output_handles,\n                         NDArrayHandle* ograd_handles,\n                         uint32_t num_variables,\n                         NDArrayHandle* var_handles,\n                         int retain_graph,\n                         int create_graph,\n                         int is_train,\n                         NDArrayHandle** grad_handles,\n                         int** grad_stypes) {\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  API_BEGIN();\n\n  std::vector<NDArray*> outputs, ograds, variables;\n  outputs.reserve(num_output);\n  for (uint32_t i = 0; i < num_output; ++i) {\n    outputs.emplace_back(reinterpret_cast<NDArray*>(output_handles[i]));\n  }\n\n  ograds.reserve(num_output);\n  for (uint32_t i = 0; i < num_output; ++i) {\n    if (ograd_handles != nullptr) {\n      ograds.emplace_back(reinterpret_cast<NDArray*>(ograd_handles[i]));\n    } else {\n      ograds.emplace_back(nullptr);\n    }\n  }\n\n  variables.reserve(num_variables);\n  for (uint32_t i = 0; i < num_variables; ++i) {\n    variables.emplace_back(reinterpret_cast<NDArray*>(var_handles[i]));\n  }\n\n  auto grads =\n      Imperative::Get()->Backward(outputs, ograds, variables, is_train, retain_graph, create_graph);\n  if (num_variables != 0) {\n    ret->ret_handles.clear();\n    ret->out_types.clear();\n    ret->ret_handles.reserve(grads.size());\n    ret->out_types.reserve(grads.size());\n    for (const auto& i : grads) {\n      ret->ret_handles.push_back(i);\n      ret->out_types.push_back(i->storage_type());\n    }\n    *grad_handles = dmlc::BeginPtr(ret->ret_handles);\n    *grad_stypes  = dmlc::BeginPtr(ret->out_types);\n  }\n  API_END();\n}\n\nint MXAutogradGetSymbol(NDArrayHandle handle, SymbolHandle* out) {\n  API_BEGIN();\n  NDArray* head = reinterpret_cast<NDArray*>(handle);\n  auto sym      = new nnvm::Symbol(head->get_autograd_symbol());\n  *out          = reinterpret_cast<SymbolHandle>(sym);\n  API_END();\n}\n\nint MXCachedOpRegisterOpHook(CachedOpHandle handle,\n                             CachedOpMonitorCallback callback,\n                             bool monitor_all) {\n  API_BEGIN();\n  CachedOpMonitorCallback callback_temp = nullptr;\n  std::function<void(const char*, const char*, void*)> clbk;\n  if (callback) {\n    callback_temp = callback;\n    clbk          = [callback_temp](const char* name, const char* opr_name, void* handle) {\n      callback_temp(name, opr_name, handle);\n    };\n  } else {\n    clbk = nullptr;\n  }\n  CachedOpPtr op = *static_cast<CachedOpPtr*>(handle);\n  op->RegisterOpHook(clbk, monitor_all);\n  API_END();\n}\n\nint MXNDArrayIsDeferredCompute(int* curr) {\n  API_BEGIN();\n  *curr = Imperative::Get()->is_deferred_compute();\n  API_END();\n}\n\nint MXNDArraySetIsDeferredCompute(int deferred_compute, int* prev) {\n  API_BEGIN();\n  *prev = Imperative::Get()->set_is_deferred_compute(static_cast<bool>(deferred_compute));\n  API_END();\n}\n\nint MXNDArraySetDeferredComputeVariable(NDArrayHandle* arrays, SymbolHandle* variables, int num) {\n  API_BEGIN();\n  Imperative::Get()->SetDeferredComputeVariable(arrays, variables, num);\n  API_END();\n}\n\nint MXNDArrayClearDeferredCompute(NDArrayHandle* arrays, int num) {\n  API_BEGIN();\n  Imperative::Get()->DeferredComputeClear(arrays, num);\n  API_END();\n}\n\nint MXNDArrayGetDeferredComputeSymbol(NDArrayHandle* output_handles,\n                                      int num_outputs,\n                                      SymbolHandle* out) {\n  nnvm::Symbol* s = new nnvm::Symbol();\n  API_BEGIN();\n  std::vector<NDArray*> outputs;\n  outputs.reserve(num_outputs);\n  for (int i = 0; i < num_outputs; ++i) {\n    NDArray* array = reinterpret_cast<NDArray*>(output_handles[i]);\n    outputs.emplace_back(array);\n  }\n  // Obtain Symbol\n  *s   = Imperative::Get()->GetDeferredComputeSymbol(outputs);\n  *out = s;\n  API_END_HANDLE_ERROR(delete s;);\n}\n"
  },
  {
    "path": "src/c_api/c_api_profile.cc",
    "content": "//\n// Created by coolivie on 11/25/17.\n//\n\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file c_api_profile.cc\n * \\brief C API of mxnet profiler and support functions\n */\n#include <dmlc/base.h>\n#include <dmlc/logging.h>\n#include <dmlc/thread_group.h>\n#include <mxnet/kvstore.h>\n#include <stack>\n#include \"./c_api_common.h\"\n#include \"../profiler/storage_profiler.h\"\n#include \"../profiler/profiler.h\"\n\nnamespace mxnet {\n\nstatic profiler::ProfileDomain api_domain(\"MXNET_C_API\");\nstatic profiler::ProfileCounter api_call_counter(\"MXNet C API Calls\", &api_domain);\nstatic profiler::ProfileCounter api_concurrency_counter(\"MXNet C API Concurrency\", &api_domain);\n\n/*! \\brief Per-API-call timing data */\nstruct APICallTimingData {\n  const char* name_;\n  profiler::ProfileTask* task_;\n};\n\n/*!\n * \\brief Per-thread profiling data\n */\nclass ProfilingThreadData {\n public:\n  /*!\n   * \\brief Constructor, nothrow\n   */\n  inline ProfilingThreadData() = default;\n\n  /*!\n   * \\brief Retreive ProfileTask object of the given name, or create if it doesn't exist\n   * \\param name Name of the task\n   * \\param domain Domain of the task\n   * \\return Pointer to the stored or created ProfileTask object\n   */\n  profiler::ProfileTask* profile_task(const char* name, profiler::ProfileDomain* domain) {\n    // Per-thread so no lock necessary\n    auto iter = tasks_.find(name);\n    if (iter == tasks_.end()) {\n      iter =\n          tasks_\n              .emplace(std::make_pair(name, std::make_unique<profiler::ProfileTask>(name, domain)))\n              .first;\n    }\n    return iter->second.get();\n  }\n\n  /*! \\brief nestable call stack */\n  std::stack<APICallTimingData> calls_;\n  /*! \\brief Whether profiling actions should be ignored/excluded */\n  volatile bool ignore_call_ = false;  // same-thread only, so not atomic\n\n private:\n  /*! \\brief tasks */\n  std::unordered_map<std::string, std::unique_ptr<profiler::ProfileTask>> tasks_;\n};\n\n#if DMLC_CXX11_THREAD_LOCAL\nstatic thread_local ProfilingThreadData thread_profiling_data;\n#else\nstatic MX_THREAD_LOCAL ProfilingThreadData thread_profiling_data;\n#endif\n\nextern void on_enter_api(const char* function) {\n  if (profiler::Profiler::Get()->IsProfiling(profiler::Profiler::kAPI)) {\n    if (!thread_profiling_data.ignore_call_) {\n      ++api_call_counter;\n      ++api_concurrency_counter;\n      APICallTimingData data = {function,\n                                thread_profiling_data.profile_task(function, &api_domain)};\n      thread_profiling_data.calls_.push(data);\n      data.task_->start();\n    }\n  }\n}\nextern void on_exit_api() {\n  if (profiler::Profiler::Get()->IsProfiling(profiler::Profiler::kAPI)) {\n    if (!thread_profiling_data.ignore_call_) {\n      CHECK(!thread_profiling_data.calls_.empty());\n      APICallTimingData data = thread_profiling_data.calls_.top();\n      data.task_->stop();\n      thread_profiling_data.calls_.pop();\n      --api_concurrency_counter;\n    }\n  }\n}\n\n/*!\n * \\brief Don't profile calls in this scope using RAII\n */\nstruct IgnoreProfileCallScope {\n  IgnoreProfileCallScope() {\n    DCHECK_EQ(thread_profiling_data.ignore_call_, false);\n    thread_profiling_data.ignore_call_ = true;\n  }\n  ~IgnoreProfileCallScope() {\n    DCHECK_EQ(thread_profiling_data.ignore_call_, true);\n    thread_profiling_data.ignore_call_ = false;\n  }\n};\n\n}  // namespace mxnet\n\n/*!\n * \\brief Simple global profile objects created from Python\n * \\note These mutexes will almost never have a collision, so internal futexes will be able\n *       to lock in user mode (good performance)\n *       I would use dmlc::SpinLock, except that I am concerned that if conditions change and\n *       there are frequent collisions (ie multithreaded inference), then the spin locks may\n *       start burning CPU unnoticed\n */\nstruct PythonProfileObjects {\n  // These will almost never collide, so locking will happen in user-space (at least on Linux)\n  // since pthreads uses futexes.\n  std::mutex cs_domains_;\n  std::mutex cs_counters_;\n  std::mutex cs_tasks_;\n  std::mutex cs_frames_;\n  std::mutex cs_events_;\n  std::list<std::shared_ptr<profiler::ProfileDomain>> domains_;\n  std::unordered_map<profiler::ProfileCounter*, std::shared_ptr<profiler::ProfileCounter>>\n      counters_;\n  std::unordered_map<profiler::ProfileDuration*, std::shared_ptr<profiler::ProfileDuration>> tasks_;\n  std::unordered_map<profiler::ProfileDuration*, std::shared_ptr<profiler::ProfileDuration>>\n      frames_;\n  std::unordered_map<profiler::ProfileDuration*, std::shared_ptr<profiler::ProfileDuration>>\n      events_;\n};\nstatic PythonProfileObjects python_profile_objects;\n\nenum class ProfileProcess { kWorker, kServer };\n\nenum class PrintFormat { table, json };\n\nstruct ProfileConfigParam : public dmlc::Parameter<ProfileConfigParam> {\n  bool profile_all;\n  bool profile_symbolic;\n  bool profile_imperative;\n  bool profile_memory;\n  bool profile_api;\n  std::string filename;\n  std::string gpu_memory_profile_filename_prefix;\n  bool continuous_dump;\n  float dump_period;\n  bool aggregate_stats;\n  int profile_process;\n  DMLC_DECLARE_PARAMETER(ProfileConfigParam) {\n    DMLC_DECLARE_FIELD(profile_all).set_default(false).describe(\"Profile all. Default is False.\");\n    DMLC_DECLARE_FIELD(profile_symbolic)\n        .set_default(true)\n        .describe(\"Profile symbolic operators.  Default is True.\");\n    DMLC_DECLARE_FIELD(profile_imperative)\n        .set_default(true)\n        .describe(\"Profile imperative operators.  Default is True.\");\n    DMLC_DECLARE_FIELD(profile_memory)\n        .set_default(true)\n        .describe(\"Profile memory.  Default is True.\");\n    DMLC_DECLARE_FIELD(profile_api).set_default(true).describe(\"Profile C API.  Default is True.\");\n    DMLC_DECLARE_FIELD(filename)\n        .set_default(\"profile.json\")\n        .describe(\"File name to write profiling info.\");\n#if MXNET_USE_CUDA\n    DMLC_DECLARE_FIELD(gpu_memory_profile_filename_prefix)\n        .set_default(\"gpu_memory_profile\")\n        .describe(\"File name prefix to write GPU memory profile info.\");\n#endif  // MXNET_USE_CUDA\n    DMLC_DECLARE_FIELD(continuous_dump)\n        .set_default(true)\n        .describe(\n            \"Periodically dump (and append) profiling data to file while running. \"\n            \"Default is True.\");\n    DMLC_DECLARE_FIELD(dump_period)\n        .set_default(1.0f)\n        .describe(\n            \"When continuous dump is enabled, the period between subsequent \"\n            \"profile info dumping.\");\n    DMLC_DECLARE_FIELD(aggregate_stats)\n        .set_default(false)\n        .describe(\n            \"Maintain aggregate stats, required for MXDumpAggregateStats.  Note that \"\n            \"this can have a negative performance impact. Default is False.\");\n    DMLC_DECLARE_FIELD(profile_process)\n        .add_enum(\"worker\", static_cast<int>(ProfileProcess::kWorker))\n        .add_enum(\"server\", static_cast<int>(ProfileProcess::kServer))\n        .set_default(static_cast<int>(ProfileProcess::kWorker))\n        .describe(\n            \"Specifies which process to profile: \"\n            \"worker: this is default. for single node training it should always be worker.\"\n            \"server: for distributed training, this profiles server process\");\n  }\n};\n\nDMLC_REGISTER_PARAMETER(ProfileConfigParam);\n\nstruct ProfileMarkerScopeParam : public dmlc::Parameter<ProfileMarkerScopeParam> {\n  int scope;\n  DMLC_DECLARE_PARAMETER(ProfileMarkerScopeParam) {\n    DMLC_DECLARE_FIELD(scope)\n        .set_default(profiler::ProfileMarker::kProcess)\n        .add_enum(\"global\", profiler::ProfileMarker::kGlobal)\n        .add_enum(\"process\", profiler::ProfileMarker::kProcess)\n        .add_enum(\"thread\", profiler::ProfileMarker::kThread)\n        .add_enum(\"task\", profiler::ProfileMarker::kTask)\n        .add_enum(\"marker\", profiler::ProfileMarker::kMarker)\n        .describe(\"Profile Instant-Marker scope.\");\n  }\n};\n\nDMLC_REGISTER_PARAMETER(ProfileMarkerScopeParam);\n\nint MXSetProcessProfilerConfig(int num_params,\n                               const char* const* keys,\n                               const char* const* vals,\n                               KVStoreHandle kvstoreHandle) {\n  mxnet::IgnoreProfileCallScope ignore;\n  API_BEGIN();\n  std::vector<std::pair<std::string, std::string>> kwargs;\n  kwargs.reserve(num_params);\n  for (int i = 0; i < num_params; ++i) {\n    CHECK_NOTNULL(keys[i]);\n    CHECK_NOTNULL(vals[i]);\n    kwargs.emplace_back(std::make_pair(keys[i], vals[i]));\n  }\n  ProfileConfigParam param = {};\n  param.Init(kwargs);\n  if (static_cast<ProfileProcess>(param.profile_process) == ProfileProcess::kServer) {\n    std::ostringstream os;\n    for (int i = 0; i < num_params; ++i) {\n      // this will be sent to the server now, those configs shouldn't have profile server again\n      if (strcmp(keys[i], \"profile_process\") == 0)\n        continue;\n      os << keys[i] << \":\" << vals[i];\n      if (i != num_params - 1)\n        os << \",\";\n    }\n    CHECK(kvstoreHandle) << \"KVStoreHandle passed to profiler is null\";\n    static_cast<KVStore*>(kvstoreHandle)\n        ->SetServerProfilerCommand(mxnet::KVStoreServerProfilerCommand::kSetConfig, os.str());\n  } else {\n    int mode = 0;\n    if (param.profile_api || param.profile_all) {\n      mode |= profiler::Profiler::kAPI;\n    }\n    if (param.profile_symbolic || param.profile_all) {\n      mode |= profiler::Profiler::kSymbolic;\n    }\n    if (param.profile_imperative || param.profile_all) {\n      mode |= profiler::Profiler::kImperative;\n    }\n    if (param.profile_memory || param.profile_all) {\n      mode |= profiler::Profiler::kMemory;\n    }\n    profiler::Profiler::Get()->SetConfig(profiler::Profiler::ProfilerMode(mode),\n                                         std::string(param.filename),\n                                         param.continuous_dump,\n                                         param.dump_period,\n                                         param.aggregate_stats);\n#if MXNET_USE_CUDA\n    profiler::GpuDeviceStorageProfiler::Get()->SetConfig(param.gpu_memory_profile_filename_prefix);\n#endif  // MXNET_USE_CUDA\n  }\n  API_END();\n}\n\nint MXSetProfilerConfig(int num_params, const char* const* keys, const char* const* vals) {\n  return MXSetProcessProfilerConfig(num_params, keys, vals, nullptr);\n}\n\nint MXAggregateProfileStatsPrint(const char** out_str,\n                                 int reset,\n                                 int format,\n                                 int sort_by,\n                                 int ascending) {\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  API_BEGIN();\n  CHECK_NOTNULL(out_str);\n  profiler::Profiler* profiler = profiler::Profiler::Get();\n  if (profiler->IsEnableOutput()) {\n    // Register stats up until now\n    profiler->DumpProfile(false);\n  }\n  std::shared_ptr<profiler::AggregateStats> stats = profiler->GetAggregateStats();\n  std::ostringstream os;\n  if (stats) {\n    if (static_cast<PrintFormat>(format) == PrintFormat::table)\n      stats->DumpTable(os, sort_by, ascending);\n    else if (static_cast<PrintFormat>(format) == PrintFormat::json)\n      stats->DumpJson(os, sort_by, ascending);\n    else\n      LOG(FATAL) << \"Invalid value for parameter format\";\n  }\n  if (reset != 0)\n    stats->clear();\n  ret->ret_str = os.str();\n  *out_str     = (ret->ret_str).c_str();\n  API_END();\n}\n\nint MXDumpProfile(int finished) {\n  return MXDumpProcessProfile(finished, static_cast<int>(ProfileProcess::kWorker), nullptr);\n}\n\nint MXDumpProcessProfile(int finished, int profile_process, KVStoreHandle kvStoreHandle) {\n  mxnet::IgnoreProfileCallScope ignore;\n  API_BEGIN();\n  if (static_cast<ProfileProcess>(profile_process) == ProfileProcess::kServer) {\n    CHECK(kvStoreHandle) << \"Kvstore Handle passed to profiler is null\";\n    static_cast<KVStore*>(kvStoreHandle)\n        ->SetServerProfilerCommand(mxnet::KVStoreServerProfilerCommand::kDump,\n                                   std::to_string(finished));\n  } else {\n    profiler::Profiler* profiler = profiler::Profiler::Get();\n    CHECK(profiler->IsEnableOutput())\n        << \"Profiler hasn't been run. Config and start profiler first\";\n    profiler->DumpProfile(finished != 0);\n#if MXNET_USE_CUDA\n    profiler::GpuDeviceStorageProfiler::Get()->DumpProfile();\n#endif  // MXNET_USE_CUDA\n  }\n  API_END()\n}\n\nint MXSetProfilerState(int state) {\n  return MXSetProcessProfilerState(state, static_cast<int>(ProfileProcess::kWorker), nullptr);\n}\n\nint MXSetProfilerScope(const char* const scope) {\n  API_BEGIN();\n  profiler::ProfilerScope::Get()->SetCurrentProfilerScope(scope);\n  API_END();\n}\n\nint MXSetProcessProfilerState(int state, int profile_process, KVStoreHandle kvStoreHandle) {\n  mxnet::IgnoreProfileCallScope ignore;\n  // state, kNotRunning: 0, kRunning: 1\n  API_BEGIN();\n  if (static_cast<ProfileProcess>(profile_process) == ProfileProcess::kServer) {\n    CHECK(kvStoreHandle) << \"Kvstore Handle passed to profiler is null\";\n    static_cast<KVStore*>(kvStoreHandle)\n        ->SetServerProfilerCommand(mxnet::KVStoreServerProfilerCommand::kState,\n                                   std::to_string(state));\n  } else {\n    switch (state) {\n      case profiler::Profiler::kNotRunning:\n        profiler::vtune::vtune_pause();\n        break;\n      case profiler::Profiler::kRunning:\n        profiler::vtune::vtune_resume();\n        break;\n    }\n    profiler::Profiler::Get()->SetState(profiler::Profiler::ProfilerState(state));\n  }\n  API_END();\n}\n\nint MXProfileCreateDomain(const char* domain, ProfileHandle* out) {\n  mxnet::IgnoreProfileCallScope ignore;\n  API_BEGIN();\n  auto dom = std::make_shared<profiler::ProfileDomain>(domain);\n  {\n    std::unique_lock<std::mutex> lock(python_profile_objects.cs_domains_);\n    python_profile_objects.domains_.push_back(dom);\n  }\n  *out = dom.get();\n  API_END();\n}\n\nint MXProfileCreateTask(ProfileHandle domain, const char* task_name, ProfileHandle* out) {\n  mxnet::IgnoreProfileCallScope ignore;\n  API_BEGIN();\n  auto ctr = std::make_shared<profiler::ProfileTask>(task_name,\n                                                     static_cast<profiler::ProfileDomain*>(domain));\n  {\n    std::unique_lock<std::mutex> lock(python_profile_objects.cs_tasks_);\n    python_profile_objects.tasks_.emplace(std::make_pair(ctr.get(), ctr));\n  }\n  *out = ctr.get();\n  API_END();\n}\n\nint MXProfileCreateFrame(ProfileHandle domain, const char* frame_name, ProfileHandle* out) {\n  mxnet::IgnoreProfileCallScope ignore;\n  API_BEGIN();\n  auto ctr = std::make_shared<profiler::ProfileFrame>(\n      frame_name, static_cast<profiler::ProfileDomain*>(domain));\n  {\n    std::unique_lock<std::mutex> lock(python_profile_objects.cs_frames_);\n    python_profile_objects.frames_.emplace(std::make_pair(ctr.get(), ctr));\n  }\n  *out = ctr.get();\n  API_END();\n}\n\nint MXProfileCreateEvent(const char* event_name, ProfileHandle* out) {\n  mxnet::IgnoreProfileCallScope ignore;\n  API_BEGIN();\n  auto ctr = std::make_shared<profiler::ProfileEvent>(event_name);\n  {\n    std::unique_lock<std::mutex> lock(python_profile_objects.cs_events_);\n    python_profile_objects.events_.emplace(std::make_pair(ctr.get(), ctr));\n  }\n  *out = ctr.get();\n  API_END();\n}\n\nint MXProfileDestroyHandle(ProfileHandle object_handle) {\n  mxnet::IgnoreProfileCallScope ignore;\n  API_BEGIN();\n  CHECK_NE(object_handle, static_cast<ProfileHandle>(nullptr))\n      << \"Invalid NULL handle passed to MXProfileDestroyHandle\";\n  std::shared_ptr<profiler::ProfileObject> shared_object_ptr(nullptr);\n  {\n    auto object = static_cast<profiler::ProfileObject*>(object_handle);\n    switch (object->type()) {\n      case profiler::kTask: {\n        auto p = static_cast<profiler::ProfileDuration*>(object_handle);\n        std::unique_lock<std::mutex> lock(python_profile_objects.cs_tasks_);\n        auto iter = python_profile_objects.tasks_.find(p);\n        if (iter != python_profile_objects.tasks_.end()) {\n          shared_object_ptr = iter->second;\n          python_profile_objects.tasks_.erase(iter);\n        }\n        break;\n      }\n      case profiler::kEvent: {\n        auto p = static_cast<profiler::ProfileDuration*>(object_handle);\n        std::unique_lock<std::mutex> lock(python_profile_objects.cs_events_);\n        auto iter = python_profile_objects.events_.find(p);\n        if (iter != python_profile_objects.events_.end()) {\n          shared_object_ptr = iter->second;\n          python_profile_objects.events_.erase(iter);\n        }\n        break;\n      }\n      case profiler::kFrame: {\n        auto p = static_cast<profiler::ProfileDuration*>(object_handle);\n        std::unique_lock<std::mutex> lock(python_profile_objects.cs_frames_);\n        auto iter = python_profile_objects.frames_.find(p);\n        if (iter != python_profile_objects.frames_.end()) {\n          shared_object_ptr = iter->second;\n          python_profile_objects.frames_.erase(iter);\n        }\n        break;\n      }\n      case profiler::kCounter: {\n        auto p = static_cast<profiler::ProfileCounter*>(object_handle);\n        std::unique_lock<std::mutex> lock(python_profile_objects.cs_counters_);\n        auto iter = python_profile_objects.counters_.find(p);\n        if (iter != python_profile_objects.counters_.end()) {\n          shared_object_ptr = iter->second;\n          python_profile_objects.counters_.erase(iter);\n        }\n        break;\n      }\n      case profiler::kDomain:\n        // Not destroyed\n        break;\n    }\n  }\n  shared_object_ptr.reset();  // Destroy out of lock scope\n  API_END();\n}\n\nint MXProfileDurationStart(ProfileHandle duration_handle) {\n  mxnet::IgnoreProfileCallScope ignore;\n  API_BEGIN();\n  CHECK_NOTNULL(duration_handle);\n  static_cast<profiler::ProfileDuration*>(duration_handle)->start();\n  API_END();\n}\n\nint MXProfileDurationStop(ProfileHandle duration_handle) {\n  mxnet::IgnoreProfileCallScope ignore;\n  API_BEGIN();\n  CHECK_NOTNULL(duration_handle);\n  static_cast<profiler::ProfileDuration*>(duration_handle)->stop();\n  API_END();\n}\n\nint MXProfilePause(int paused) {\n  return MXProcessProfilePause(paused, static_cast<int>(ProfileProcess::kWorker), nullptr);\n}\n\nint MXProcessProfilePause(int paused, int profile_process, KVStoreHandle kvStoreHandle) {\n  mxnet::IgnoreProfileCallScope ignore;\n  API_BEGIN();\n  if (static_cast<ProfileProcess>(profile_process) == ProfileProcess::kServer) {\n    CHECK(kvStoreHandle) << \"Kvstore Handle passed to profiler is null\";\n    static_cast<KVStore*>(kvStoreHandle)\n        ->SetServerProfilerCommand(mxnet::KVStoreServerProfilerCommand::kPause,\n                                   std::to_string(paused));\n  } else {\n    if (paused) {\n      profiler::vtune::vtune_pause();\n      profiler::Profiler::Get()->set_paused(true);\n    } else {\n      profiler::Profiler::Get()->set_paused(false);\n      profiler::vtune::vtune_resume();\n    }\n  }\n  API_END();\n}\n\nint MXProfileCreateCounter(ProfileHandle domain, const char* counter_name, ProfileHandle* out) {\n  mxnet::IgnoreProfileCallScope ignore;\n  API_BEGIN();\n  auto ctr = std::make_shared<profiler::ProfileCounter>(\n      counter_name, static_cast<profiler::ProfileDomain*>(domain));\n  {\n    std::unique_lock<std::mutex> lock(python_profile_objects.cs_counters_);\n    python_profile_objects.counters_.emplace(std::make_pair(ctr.get(), ctr));\n  }\n  *out = ctr.get();\n  API_END();\n}\n\nint MXProfileSetCounter(ProfileHandle counter_handle, uint64_t value) {\n  mxnet::IgnoreProfileCallScope ignore;\n  API_BEGIN();\n  static_cast<profiler::ProfileCounter*>(counter_handle)->operator=(value);\n  API_END();\n}\n\nint MXProfileAdjustCounter(ProfileHandle counter_handle, int64_t by_value) {\n  mxnet::IgnoreProfileCallScope ignore;\n  API_BEGIN();\n  static_cast<profiler::ProfileCounter*>(counter_handle)->operator+=(by_value);\n  API_END();\n}\n\nint MXProfileSetMarker(ProfileHandle domain, const char* instant_marker_name, const char* scope) {\n  mxnet::IgnoreProfileCallScope ignore;\n  API_BEGIN();\n  ProfileMarkerScopeParam param;\n  std::vector<std::pair<std::string, std::string>> kwargs = {{\"scope\", scope}};\n  param.Init(kwargs);\n  profiler::ProfileMarker marker(instant_marker_name,\n                                 static_cast<profiler::ProfileDomain*>(domain),\n                                 static_cast<profiler::ProfileMarker::MarkerScope>(param.scope));\n  marker.mark();\n  API_END();\n}\n"
  },
  {
    "path": "src/c_api/c_api_symbolic.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file c_api_symbolic.cc\n * \\brief C API of mxnet\n */\n#include \"mxnet/base.h\"\n#include \"mxnet/c_api.h\"\n#include \"mxnet/imperative.h\"\n#include \"nnvm/c_api.h\"\n#include \"nnvm/pass.h\"\n#include \"nnvm/pass_functions.h\"\n#include \"nnvm/symbolic.h\"\n#include \"./c_api_common.h\"\n#include \"../common/exec_utils.h\"\n#include \"../operator/operator_common.h\"\n#include \"../imperative/exec_pass.h\"\n#include \"../operator/subgraph/subgraph_property.h\"\n\nnamespace mxnet {\nnamespace op {\nvoid RegisterLegacyOpProp();\nvoid RegisterLegacyNDFunc();\n}  // namespace op\nconst std::vector<std::string> kHiddenKeys =\n    {\"ctx_group\", \"lr_mult\", \"wd_mult\", \"force_mirroring\", \"mirror_stage\", \"profiler_scope\"};\nconst std::vector<std::string> kReplacedHiddenKeys = {\"__ctx_group__\",\n                                                      \"__lr_mult__\",\n                                                      \"__wd_mult__\",\n                                                      \"__force_mirroring__\",\n                                                      \"__mirror_stage__\",\n                                                      \"__profiler_scope__\"};\nconst char* kNamespaceSeparator                    = \"$\";\n\nDMLC_JSON_ENABLE_ANY(int, int);\n\n// convert nnvm symbol to a nnvm graph.\nnnvm::Graph Symbol2Graph(const nnvm::Symbol& s) {\n  nnvm::Graph g;\n  g.outputs                = s.outputs;\n  g.attrs[\"mxnet_version\"] = std::make_shared<nnvm::any>(static_cast<int>(MXNET_VERSION));\n  if (Imperative::Get()->is_np_shape()) {\n    g.attrs[\"is_np_shape\"] =\n        std::make_shared<nnvm::any>(static_cast<int>(Imperative::Get()->is_np_shape()));\n  }\n  return g;\n}\n\nstd::vector<uint32_t> ReadOnlyArgIndices(const nnvm::IndexedGraph& idx) {\n  std::vector<uint32_t> ret;\n  auto& arg_nodes = idx.input_nodes();\n  for (uint32_t i = 0; i < arg_nodes.size(); ++i) {\n    if (idx.mutable_input_nodes().count(arg_nodes[i]) == 0) {\n      ret.push_back(i);\n    }\n  }\n  return ret;\n}\n\n}  // namespace mxnet\n\n// symbolic configuration generation API.\n// Redirect to NNVM's C API\nint MXListAllOpNames(nn_uint* out_size, const char*** out_array) {\n  mxnet::op::RegisterLegacyOpProp();\n  mxnet::op::RegisterLegacyNDFunc();\n  return NNListAllOpNames(out_size, out_array);\n}\n\nint MXSymbolListAtomicSymbolCreators(uint32_t* out_size, AtomicSymbolCreator** out_array) {\n  mxnet::op::RegisterLegacyOpProp();\n  mxnet::op::RegisterLegacyNDFunc();\n  return NNListUniqueOps(out_size, out_array);\n}\n\nint MXSymbolGetAtomicSymbolInfo(AtomicSymbolCreator creator,\n                                const char** name,\n                                const char** description,\n                                uint32_t* num_args,\n                                const char*** arg_names,\n                                const char*** arg_type_infos,\n                                const char*** arg_descriptions,\n                                const char** key_var_num_args,\n                                const char** return_type) {\n  static auto& map_key_var_args = nnvm::Op::GetAttr<std::string>(\"key_var_num_args\");\n  const Op* op                  = static_cast<Op*>(creator);\n  MXAPIThreadLocalEntry<>* ret  = MXAPIThreadLocalStore<>::Get();\n  ret->ret_str.resize(0);\n\n  if (map_key_var_args.count(op) != 0) {\n    *key_var_num_args = map_key_var_args[op].c_str();\n  } else {\n    *key_var_num_args = ret->ret_str.c_str();\n  }\n  return NNGetOpInfo(creator,\n                     name,\n                     description,\n                     num_args,\n                     arg_names,\n                     arg_type_infos,\n                     arg_descriptions,\n                     return_type);\n}\n\nint MXSymbolCreateAtomicSymbol(AtomicSymbolCreator creator,\n                               uint32_t num_param,\n                               const char** keys,\n                               const char** vals,\n                               SymbolHandle* out) {\n  nnvm::Symbol* s = new nnvm::Symbol();\n  API_BEGIN();\n  const nnvm::Op* op = static_cast<const nnvm::Op*>(creator);\n  std::unordered_map<std::string, std::string> kwargs;\n  for (nn_uint i = 0; i < num_param; ++i) {\n    bool flag = false;\n    for (const auto& k : kHiddenKeys) {\n      std::string tmp(keys[i]);\n      size_t pos = tmp.rfind(k);\n      if (pos == 0) {\n        kwargs.insert({\"__\" + tmp + \"__\", std::string(vals[i])});\n        flag = true;\n        break;\n      } else if (pos != std::string::npos && pos == tmp.length() - k.length()) {\n        std::ostringstream os;\n        os << \"setting variable attributes with \" << keys[i] << \" is deprecated. \"\n           << \"please instead use\\nw = Variable(\" << k << \"=\" << vals[i] << \")\\n\"\n           << \"sym = YourSymbolName(\" << tmp.substr(0, pos - 1) << \"=w)\";\n        throw dmlc::Error(os.str());\n      }\n    }\n    if (!flag)\n      kwargs.insert({std::string(keys[i]), std::string(vals[i])});\n  }\n  *s   = nnvm::Symbol::CreateFunctor(op, std::move(kwargs));\n  *out = s;\n  API_END_HANDLE_ERROR(delete s;);\n}\n\nint MXSymbolCreateVariable(const char* name, SymbolHandle* out) {\n  return NNSymbolCreateVariable(name, out);\n}\n\nint MXSymbolCreateGroup(uint32_t num_symbols, SymbolHandle* symbols, SymbolHandle* out) {\n  return NNSymbolCreateGroup(num_symbols, symbols, out);\n}\n\nint MXSymbolGetOutput(SymbolHandle symbol, uint32_t index, SymbolHandle* out) {\n  return NNSymbolGetOutput(symbol, index, out);\n}\n\nint MXSymbolGetInputs(SymbolHandle symbol, SymbolHandle* out) {\n  nnvm::Symbol* s = new nnvm::Symbol();\n  API_BEGIN();\n  std::vector<nnvm::ObjectPtr> inputs =\n      static_cast<nnvm::Symbol*>(symbol)->ListInputs(nnvm::Symbol::ListInputOption(0));\n  for (const nnvm::ObjectPtr& o : inputs) {\n    nnvm::NodeEntry e(o);\n    s->outputs.push_back(e);\n  }\n  *out = s;\n  API_END_HANDLE_ERROR(delete s);\n}\n\nint MXSymbolGetInternals(SymbolHandle symbol, SymbolHandle* out) {\n  nnvm::Symbol* s = new nnvm::Symbol();\n  API_BEGIN();\n  *s   = static_cast<nnvm::Symbol*>(symbol)->GetInternals();\n  *out = s;\n  API_END_HANDLE_ERROR(delete s);\n}\n\nint MXSymbolGetChildren(SymbolHandle symbol, SymbolHandle* out) {\n  nnvm::Symbol* s = new nnvm::Symbol();\n  API_BEGIN();\n  *s   = static_cast<nnvm::Symbol*>(symbol)->GetChildren();\n  *out = s;\n  API_END_HANDLE_ERROR(delete s);\n}\n\nint MXSymbolFree(SymbolHandle symbol) {\n  return NNSymbolFree(symbol);\n}\n\nint MXSymbolCopy(SymbolHandle symbol, SymbolHandle* out) {\n  return NNSymbolCopy(symbol, out);\n}\n\nint MXSymbolPrint(SymbolHandle symbol, const char** out_str) {\n  return NNSymbolPrint(symbol, out_str);\n}\n\nint MXSymbolGetName(SymbolHandle symbol, const char** out, int* success) {\n  return NNSymbolGetAttr(symbol, \"name\", out, success);\n}\n\nint MXSymbolGetAttr(SymbolHandle symbol, const char* key, const char** out, int* success) {\n  nnvm::Symbol* s              = static_cast<nnvm::Symbol*>(symbol);\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  API_BEGIN();\n  if (s->GetAttr(key, &(ret->ret_str))) {\n    *out     = (ret->ret_str).c_str();\n    *success = 1;\n  } else {\n    *out     = nullptr;\n    *success = 0;\n    if (std::find(kHiddenKeys.begin(), kHiddenKeys.end(), key) != kHiddenKeys.end()) {\n      std::string skey = \"__\" + std::string(key) + \"__\";\n      if (s->GetAttr(skey, &(ret->ret_str))) {\n        *out     = (ret->ret_str).c_str();\n        *success = 1;\n      }\n    }\n  }\n  API_END();\n}\n\nint MXSymbolSetAttr(SymbolHandle symbol, const char* key, const char* value) {\n  nnvm::Symbol* s = static_cast<nnvm::Symbol*>(symbol);\n  API_BEGIN();\n  std::vector<std::pair<std::string, std::string>> kwargs;\n  std::string skey(key), sval(value);\n  for (const auto& k : kHiddenKeys) {\n    size_t pos = skey.rfind(k);\n    if (pos == 0 && k.length() == skey.length()) {\n      skey = \"__\" + skey + \"__\";\n      break;\n    } else if (pos != std::string::npos && pos + k.length() == skey.length()) {\n      std::ostringstream os;\n      os << \"setting variable attributes with \" << key << \" is deprecated. \"\n         << \"please instead use\\nw = Variable(\" << k << \"=\" << value << \")\\n\"\n         << \"sym = YourSymbolName(\" << skey.substr(0, pos - 1) << \"=w)\";\n      throw dmlc::Error(os.str());\n    }\n  }\n  kwargs.emplace_back(std::make_pair(std::move(skey), std::move(sval)));\n  s->SetAttrs(kwargs);\n  API_END();\n}\n\nint MXSymbolListAttr(SymbolHandle symbol, uint32_t* out_size, const char*** out) {\n  nnvm::Symbol* s              = static_cast<nnvm::Symbol*>(symbol);\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  API_BEGIN();\n  std::vector<std::tuple<std::string, std::string, std::string>> attr = s->ListAttrsRecursive();\n\n  std::vector<std::string>& attr_list = ret->ret_vec_str;\n  attr_list.clear();\n  for (const auto& tp : attr) {\n    attr_list.emplace_back(std::get<0>(tp) + kNamespaceSeparator + std::get<1>(tp));\n    attr_list.emplace_back(std::get<2>(tp));\n    if (find(kReplacedHiddenKeys.begin(), kReplacedHiddenKeys.end(), std::get<1>(tp)) !=\n        kReplacedHiddenKeys.end()) {\n      attr_list.push_back(std::get<0>(tp) + kNamespaceSeparator +\n                          std::get<1>(tp).substr(2, std::get<1>(tp).length() - 4));\n      attr_list.push_back(std::get<2>(tp));\n    }\n  }\n  *out_size = attr_list.size() / 2;\n  ret->ret_vec_charp.clear();\n  for (const auto& attr : attr_list) {\n    ret->ret_vec_charp.push_back(attr.c_str());\n  }\n  *out = dmlc::BeginPtr(ret->ret_vec_charp);\n  API_END();\n}\n\nint MXSymbolListAttrShallow(SymbolHandle symbol, uint32_t* out_size, const char*** out) {\n  nnvm::Symbol* s              = static_cast<nnvm::Symbol*>(symbol);\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  API_BEGIN();\n  std::unordered_map<std::string, std::string> attr =\n      s->ListAttrs(static_cast<nnvm::Symbol::ListAttrOption>(1));  // NOLINT(*)\n\n  std::vector<std::string>& attr_list = ret->ret_vec_str;\n  attr_list.clear();\n  for (const auto& kv : attr) {\n    attr_list.push_back(kv.first);\n    attr_list.push_back(kv.second);\n    if (find(kReplacedHiddenKeys.begin(), kReplacedHiddenKeys.end(), kv.first) !=\n        kReplacedHiddenKeys.end()) {\n      attr_list.push_back(kv.first.substr(2, kv.first.length() - 4));\n      attr_list.push_back(kv.second);\n    }\n  }\n  *out_size = attr_list.size() / 2;\n  ret->ret_vec_charp.clear();\n  for (auto& attr : attr_list) {\n    ret->ret_vec_charp.push_back(attr.c_str());\n  }\n  *out = dmlc::BeginPtr(ret->ret_vec_charp);\n  API_END();\n}\n\nint MXSymbolListOutputs(SymbolHandle symbol, uint32_t* out_size, const char*** out_str_array) {\n  return NNSymbolListOutputNames(symbol, out_size, out_str_array);\n}\n\nint MXSymbolGetNumOutputs(SymbolHandle symbol, uint32_t* output_count) {\n  return NNSymbolGetNumOutputs(symbol, output_count);\n}\n\nint MXSymbolCompose(SymbolHandle sym,\n                    const char* name,\n                    uint32_t num_args,\n                    const char** keys,\n                    SymbolHandle* args) {\n  return NNSymbolCompose(sym, name, num_args, keys, args);\n}\n\n// adapter functions that re-implements the functions.\nint MXSymbolListArguments(SymbolHandle symbol, uint32_t* out_size, const char*** out_str_array) {\n  return NNSymbolListInputNames(symbol, 1, out_size, out_str_array);\n}\n\nint MXSymbolListAuxiliaryStates(SymbolHandle symbol,\n                                uint32_t* out_size,\n                                const char*** out_str_array) {\n  return NNSymbolListInputNames(symbol, 2, out_size, out_str_array);\n}\n\nint MXSymbolGetAtomicSymbolName(AtomicSymbolCreator creator, const char** out) {\n  API_BEGIN();\n  Op* e = static_cast<Op*>(creator);\n  *out  = e->name.c_str();\n  API_END();\n}\n\nnamespace mxnet {\n\nextern std::vector<nnvm::Symbol*> GetInputSymbols(const nnvm::Symbol& sym);\nextern bool CutGraphInputs(const std::vector<nnvm::NodeEntry*>& input_entries,\n                           bool skip_var,\n                           std::vector<nnvm::NodeEntry>* orig_entries);\n\n}  // namespace mxnet\n\nint MXSymbolGetInputSymbols(SymbolHandle sym, SymbolHandle** input_arr, int* input_size) {\n  API_BEGIN();\n  nnvm::Symbol* s                       = static_cast<nnvm::Symbol*>(sym);\n  std::vector<nnvm::Symbol*> input_syms = mxnet::GetInputSymbols(*s);\n  *input_size                           = input_syms.size();\n\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  ret->ret_handles.clear();\n  ret->ret_handles.reserve(*input_size);\n  for (int i = 0; i < *input_size; ++i)\n    ret->ret_handles.push_back(input_syms[i]);\n  *input_arr = reinterpret_cast<SymbolHandle*>(dmlc::BeginPtr(ret->ret_handles));\n  API_END_HANDLE_ERROR();\n}\n\nint MXSymbolCutSubgraph(SymbolHandle sym, SymbolHandle** input_symbols, int* input_size) {\n  // Given a graph, we want to fetch the nodes that have been marked as part of\n  // a subgraph.\n  API_BEGIN();\n  nnvm::Symbol* s             = static_cast<nnvm::Symbol*>(sym);\n  const std::string subg_attr = \"__subgraph_name__\";\n  auto out_node               = s->outputs[0].node;\n  auto it                     = out_node->attrs.dict.find(subg_attr);\n  if (it != out_node->attrs.dict.end()) {\n    const std::string& subg_name = it->second;\n    std::vector<nnvm::NodeEntry*> input_entries;\n    DFSVisit(s->outputs, [&subg_attr, &subg_name, &input_entries](nnvm::ObjectPtr n) {\n      // If the node itself isn't in the subgraph, we ignore it.\n      auto it = n->attrs.dict.find(subg_attr);\n      if (it == n->attrs.dict.end() || it->second != subg_name)\n        return;\n\n      // We search for nodes whose node entries aren't in the subgraph.\n      for (size_t j = 0; j < n->inputs.size(); j++) {\n        auto in_node = n->inputs[j].node;\n        auto it      = in_node->attrs.dict.find(subg_attr);\n        if (it == in_node->attrs.dict.end() || it->second != subg_name)\n          input_entries.push_back(&n->inputs[j]);\n      }\n    });\n\n    std::vector<nnvm::NodeEntry> orig_entries;\n    CutGraphInputs(input_entries, false, &orig_entries);\n    std::vector<nnvm::Symbol*> input_syms(orig_entries.size());\n    for (size_t i = 0; i < input_syms.size(); i++) {\n      input_syms[i] = new nnvm::Symbol();\n      input_syms[i]->outputs.push_back(orig_entries[i]);\n    }\n    *input_size = input_syms.size();\n\n    MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n    ret->ret_handles.clear();\n    ret->ret_handles.reserve(*input_size);\n    for (int i = 0; i < *input_size; ++i)\n      ret->ret_handles.push_back(input_syms[i]);\n    *input_symbols = reinterpret_cast<SymbolHandle*>(dmlc::BeginPtr(ret->ret_handles));\n  } else {\n    *input_size = 0;\n  }\n\n  API_END_HANDLE_ERROR();\n}\n\n/*!\n * \\brief Convert shape attr in graph nodes to comply with NumPy semantics for\n * legacy models (before 1.6.0) if global flag is_np_shape has been turned on,\n * i.e., use -1 to indicate unknown number of dimensions and unknown dimension sizes.\n */\nvoid ConvertShapeAttrToNumPyCompatible(nnvm::Graph* g) {\n  if (Imperative::Get()->is_np_shape() &&\n      (!g->HasAttr(\"is_np_shape\") || !g->GetAttr<int>(\"is_np_shape\"))) {\n    DFSVisit(g->outputs, [](nnvm::ObjectPtr n) {\n      if (n->is_variable()) {\n        auto it = n->attrs.dict.find(\"__shape__\");\n        if (it != n->attrs.dict.end()) {\n          mxnet::TShape shape;\n          std::istringstream is(it->second);\n          is >> shape;\n          common::ConvertToNumpyShape(&shape);\n          std::ostringstream os;\n          os << shape;\n          it->second = os.str();\n        }\n      }\n    });\n  }\n}\n\nint MXSymbolCreateFromFile(const char* fname, SymbolHandle* out) {\n  nnvm::Symbol* s = new nnvm::Symbol();\n  API_BEGIN();\n  std::unique_ptr<dmlc::Stream> fi(dmlc::Stream::Create(fname, \"r\"));\n  dmlc::istream is(fi.get());\n  nnvm::Graph g;\n  g.attrs[\"json\"] = std::make_shared<nnvm::any>(\n      std::string(std::istreambuf_iterator<char>(is), std::istreambuf_iterator<char>()));\n  g = nnvm::ApplyPass(g, \"LoadLegacyJSON\");\n  ConvertShapeAttrToNumPyCompatible(&g);\n  s->outputs = g.outputs;\n  *out       = s;\n  is.set_stream(nullptr);\n  API_END_HANDLE_ERROR(delete s);\n}\n\nint MXSymbolCreateFromJSON(const char* json, SymbolHandle* out) {\n  nnvm::Symbol* s = new nnvm::Symbol();\n  API_BEGIN();\n  nnvm::Graph g;\n  g.attrs[\"json\"] = std::make_shared<nnvm::any>(std::string(json));\n  g               = nnvm::ApplyPass(g, \"LoadLegacyJSON\");\n  ConvertShapeAttrToNumPyCompatible(&g);\n  s->outputs = g.outputs;\n  *out       = s;\n  API_END_HANDLE_ERROR(delete s);\n}\n\nint MXSymbolRemoveAmpCast(SymbolHandle sym_handle, SymbolHandle* ret_sym_handle) {\n  nnvm::Symbol* s = new nnvm::Symbol();\n  API_BEGIN();\n  nnvm::Symbol* source = static_cast<nnvm::Symbol*>(sym_handle);\n  *s                   = source->Copy();\n  s->outputs           = nnvm::ApplyPass(Symbol2Graph(*s), \"RemoveAmpCast\").outputs;\n  *ret_sym_handle      = s;\n  API_END_HANDLE_ERROR(delete s);\n}\n\nint MXSymbolSaveToFile(SymbolHandle symbol, const char* fname) {\n  nnvm::Symbol* s = static_cast<nnvm::Symbol*>(symbol);\n  API_BEGIN();\n  std::unique_ptr<dmlc::Stream> fo(dmlc::Stream::Create(fname, \"w\"));\n  dmlc::ostream os(fo.get());\n  os << nnvm::pass::SaveJSON(Symbol2Graph(*s));\n  // reset file pointer, force flush\n  os.set_stream(nullptr);\n  API_END();\n}\n\nint MXSymbolSaveToJSON(SymbolHandle symbol, const char** out_json) {\n  nnvm::Symbol* s              = static_cast<nnvm::Symbol*>(symbol);\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  API_BEGIN();\n  ret->ret_str = nnvm::pass::SaveJSON(Symbol2Graph(*s));\n  *out_json    = ret->ret_str.c_str();\n  API_END();\n}\n\nnamespace mxnet {\n\ntemplate <typename AttrType>\nvoid MatchArguments(const nnvm::IndexedGraph& idx,\n                    const std::unordered_map<std::string, AttrType>& known_arg_attrs,\n                    std::vector<AttrType>* arg_attrs,\n                    const char* source) {\n  auto& arg_nodes = idx.input_nodes();\n  CHECK_EQ(arg_attrs->size(), arg_nodes.size());\n  size_t nmatched = 0;\n  for (size_t i = 0; i < arg_nodes.size(); ++i) {\n    const std::string& name = idx[arg_nodes[i]].source->attrs.name;\n    auto it                 = known_arg_attrs.find(name);\n    if (it != known_arg_attrs.end()) {\n      arg_attrs->at(i) = it->second;\n      ++nmatched;\n    }\n  }\n  if (nmatched != known_arg_attrs.size()) {\n    std::unordered_set<std::string> keys;\n    std::ostringstream head, msg;\n    msg << \"\\nCandidate arguments:\\n\";\n    for (size_t i = 0; i < arg_nodes.size(); ++i) {\n      std::string arg_name = idx[arg_nodes[i]].source->attrs.name;\n      keys.insert(arg_name);\n      msg << \"\\t[\" << i << ']' << arg_name << '\\n';\n    }\n    for (const auto& kv : known_arg_attrs) {\n      const std::string& key = kv.first;\n      if (keys.count(key) == 0) {\n        LOG(FATAL) << source << \"Keyword argument name \" << key << \" not found.\" << msg.str();\n      }\n    }\n  }\n}\n\n}  // namespace mxnet\n\ntemplate <typename dtype, typename stype, typename itype>\ninline void SymbolInferShape(const char** keys,\n                             uint32_t num_args,\n                             const dtype* arg_shape_data,\n                             const itype* arg_ind_ptr,\n                             const int** in_shape_ndim,\n                             const dtype*** in_shape_data,\n                             const int** out_shape_ndim,\n                             const dtype*** out_shape_data,\n                             const int** aux_shape_ndim,\n                             const dtype*** aux_shape_data,\n                             nnvm::Symbol* s,\n                             MXAPIThreadLocalEntry<dtype>* ret,\n                             stype* in_shape_size,\n                             stype* out_shape_size,\n                             stype* aux_shape_size,\n                             int* complete) {\n  nnvm::Graph g = Symbol2Graph(*s);\n  mxnet::ShapeVector arg_shapes(g.indexed_graph().input_nodes().size(), mxnet::TShape());\n  if (keys == nullptr && num_args != 0) {\n    std::vector<uint32_t> read_only_args = mxnet::ReadOnlyArgIndices(g.indexed_graph());\n    CHECK_LE(num_args, read_only_args.size());\n    for (uint32_t i = 0; i < num_args; ++i) {\n      arg_shapes[read_only_args[i]] = mxnet::ShapeTypeCast(arg_shape_data + arg_ind_ptr[i],\n                                                           arg_shape_data + arg_ind_ptr[i + 1]);\n    }\n  } else {\n    std::unordered_map<std::string, mxnet::TShape> kwargs;\n    for (uint32_t i = 0; i < num_args; ++i) {\n      kwargs[keys[i]] = mxnet::ShapeTypeCast(arg_shape_data + arg_ind_ptr[i],\n                                             arg_shape_data + arg_ind_ptr[i + 1]);\n    }\n    mxnet::MatchArguments(g.indexed_graph(), kwargs, &arg_shapes, \"InferShape\");\n  }\n  try {\n    g = mxnet::exec::InferShape(std::move(g), std::move(arg_shapes), \"__shape__\");\n  } catch (const mxnet::op::InferShapeError& err) {\n    throw dmlc::Error(err.msg);\n  }\n  // if use legacy shape definition, need to convert numpy shape to legacy shape\n  mxnet::ShapeVector shapes = g.GetAttr<mxnet::ShapeVector>(\"shape\");\n  if (!Imperative::Get()->is_np_shape()) {\n    common::ConvertToLegacyShape(&shapes);\n  }\n  // copy back\n  CopyAttr(g.indexed_graph(), shapes, &(ret->arg_shapes), &(ret->out_shapes), &(ret->aux_shapes));\n  // copy data back\n  MXAPIThreadLocalEntry<dtype>::SetupShapeArrayReturnWithBufferEx(ret->arg_shapes,\n                                                                  &(ret->arg_shape_ndim_ex),\n                                                                  &(ret->arg_shape_data_ex),\n                                                                  &(ret->arg_shape_buffer_ex));\n  MXAPIThreadLocalEntry<dtype>::SetupShapeArrayReturnWithBufferEx(ret->out_shapes,\n                                                                  &(ret->out_shape_ndim_ex),\n                                                                  &(ret->out_shape_data_ex),\n                                                                  &(ret->out_shape_buffer_ex));\n  MXAPIThreadLocalEntry<dtype>::SetupShapeArrayReturnWithBufferEx(ret->aux_shapes,\n                                                                  &(ret->aux_shape_ndim_ex),\n                                                                  &(ret->aux_shape_data_ex),\n                                                                  &(ret->aux_shape_buffer_ex));\n  *in_shape_size  = static_cast<stype>(ret->arg_shapes.size());\n  *in_shape_ndim  = dmlc::BeginPtr(ret->arg_shape_ndim_ex);\n  *in_shape_data  = dmlc::BeginPtr(ret->arg_shape_data_ex);\n  *out_shape_size = static_cast<stype>(ret->out_shapes.size());\n  *out_shape_ndim = dmlc::BeginPtr(ret->out_shape_ndim_ex);\n  *out_shape_data = dmlc::BeginPtr(ret->out_shape_data_ex);\n  *aux_shape_size = static_cast<stype>(ret->aux_shapes.size());\n  *aux_shape_ndim = dmlc::BeginPtr(ret->aux_shape_ndim_ex);\n  *aux_shape_data = dmlc::BeginPtr(ret->aux_shape_data_ex);\n  // mark complete\n  *complete = (g.GetAttr<size_t>(\"shape_num_unknown_nodes\") == 0);\n}\n\n/*!\n * \\brief Symbol shape Inference\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=0 (by default)\n * \\param sym symbol handle\n * \\param num_args number of args\n * \\param keys keys\n * \\param arg_ind_ptr arg index pointer\n * \\param arg_shape_data arg shape data\n * \\param in_shape_size input shape size\n * \\param in_shape_ndim input shape number of dims\n * \\param in_shape_data input shape data\n * \\param out_shape_size ouput shape size\n * \\param out_shape_ndim output shape number of dims\n * \\param out_shape_data output shape data\n * \\param aux_shape_size shape size of auxiliary states\n * \\param aux_shape_ndim number of dims of auxiliary states shape\n * \\param aux_shape_data shape data of auxiliary states\n * \\param complete indicates completion of Shape Inference\n * \\return 0 when success, -1 when failure happens\n */\nint MXSymbolInferShape(SymbolHandle sym,\n                       uint32_t num_args,\n                       const char** keys,\n                       const uint32_t* arg_ind_ptr,\n                       const int* arg_shape_data,\n                       uint32_t* in_shape_size,\n                       const int** in_shape_ndim,\n                       const int*** in_shape_data,\n                       uint32_t* out_shape_size,\n                       const int** out_shape_ndim,\n                       const int*** out_shape_data,\n                       uint32_t* aux_shape_size,\n                       const int** aux_shape_ndim,\n                       const int*** aux_shape_data,\n                       int* complete) {\n  nnvm::Symbol* s              = static_cast<nnvm::Symbol*>(sym);\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  API_BEGIN();\n  SymbolInferShape<int, uint32_t, uint32_t>(keys,\n                                            num_args,\n                                            arg_shape_data,\n                                            arg_ind_ptr,\n                                            in_shape_ndim,\n                                            in_shape_data,\n                                            out_shape_ndim,\n                                            out_shape_data,\n                                            aux_shape_ndim,\n                                            aux_shape_data,\n                                            s,\n                                            ret,\n                                            in_shape_size,\n                                            out_shape_size,\n                                            aux_shape_size,\n                                            complete);\n  API_END();\n}\n\n/*!\n * \\brief Executor for Symbol Shape Inference\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=1 (not default) i.e. Large Tensor Support\n * \\param sym symbol handle\n * \\param num_args number of args\n * \\param keys keys\n * \\param arg_ind_ptr arg index pointer\n * \\param arg_shape_data arg shape data\n * \\param in_shape_size input shape size\n * \\param in_shape_ndim input shape number of dims\n * \\param in_shape_data input shape data\n * \\param out_shape_size ouput shape size\n * \\param out_shape_ndim output shape number of dims\n * \\param out_shape_data output shape data\n * \\param aux_shape_size shape size of auxiliary states\n * \\param aux_shape_ndim number of dims of auxiliary states shape\n * \\param aux_shape_data shape data of auxiliary states\n * \\param complete indicates completion of Shape Inference\n * \\return 0 when success, -1 when failure happens\n */\nint MXSymbolInferShape64(SymbolHandle sym,\n                         uint32_t num_args,\n                         const char** keys,\n                         const int64_t* arg_ind_ptr,\n                         const int64_t* arg_shape_data,\n                         size_t* in_shape_size,\n                         const int** in_shape_ndim,\n                         const int64_t*** in_shape_data,\n                         size_t* out_shape_size,\n                         const int** out_shape_ndim,\n                         const int64_t*** out_shape_data,\n                         size_t* aux_shape_size,\n                         const int** aux_shape_ndim,\n                         const int64_t*** aux_shape_data,\n                         int* complete) {\n  nnvm::Symbol* s                     = static_cast<nnvm::Symbol*>(sym);\n  MXAPIThreadLocalEntry<int64_t>* ret = MXAPIThreadLocalStore<int64_t>::Get();\n  API_BEGIN();\n  SymbolInferShape<int64_t, size_t, int64_t>(keys,\n                                             num_args,\n                                             arg_shape_data,\n                                             arg_ind_ptr,\n                                             in_shape_ndim,\n                                             in_shape_data,\n                                             out_shape_ndim,\n                                             out_shape_data,\n                                             aux_shape_ndim,\n                                             aux_shape_data,\n                                             s,\n                                             ret,\n                                             in_shape_size,\n                                             out_shape_size,\n                                             aux_shape_size,\n                                             complete);\n  API_END();\n}\n\n/*!\n * \\brief Executor for Symbol Partial Shape Inference\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=0 (by default)\n * \\param sym symbol handle\n * \\param num_args number of args\n * \\param keys keys\n * \\param arg_ind_ptr arg index pointer\n * \\param arg_shape_data arg shape data\n * \\param in_shape_size input shape size\n * \\param in_shape_ndim input shape number of dims\n * \\param in_shape_data input shape data\n * \\param out_shape_size ouput shape size\n * \\param out_shape_ndim output shape number of dims\n * \\param out_shape_data output shape data\n * \\param aux_shape_size shape size of auxiliary states\n * \\param aux_shape_ndim number of dims of auxiliary states shape\n * \\param aux_shape_data shape data of auxiliary states\n * \\param complete indicates completion of Shape Inference\n * \\return 0 when success, -1 when failure happens\n */\nint MXSymbolInferShapePartial(SymbolHandle sym,\n                              uint32_t num_args,\n                              const char** keys,\n                              const uint32_t* arg_ind_ptr,\n                              const int* arg_shape_data,\n                              uint32_t* in_shape_size,\n                              const int** in_shape_ndim,\n                              const int*** in_shape_data,\n                              uint32_t* out_shape_size,\n                              const int** out_shape_ndim,\n                              const int*** out_shape_data,\n                              uint32_t* aux_shape_size,\n                              const int** aux_shape_ndim,\n                              const int*** aux_shape_data,\n                              int* complete) {\n  int succ  = 0;\n  *complete = 1;\n  return MXSymbolInferShape(sym,\n                            num_args,\n                            keys,\n                            arg_ind_ptr,\n                            arg_shape_data,\n                            in_shape_size,\n                            in_shape_ndim,\n                            in_shape_data,\n                            out_shape_size,\n                            out_shape_ndim,\n                            out_shape_data,\n                            aux_shape_size,\n                            aux_shape_ndim,\n                            aux_shape_data,\n                            &succ);\n}\n\n/*!\n * \\brief Executor for Symbol Partial Shape Inference\n *  This api is available when MXNet is built with flag\n *  USE_INT64_TENSOR_SIZE=1 (not default) i.e. Large Tensor Support\n * \\param sym symbol handle\n * \\param num_args number of args\n * \\param keys keys\n * \\param arg_ind_ptr arg index pointer\n * \\param arg_shape_data arg shape data\n * \\param in_shape_size input shape size\n * \\param in_shape_ndim input shape number of dims\n * \\param in_shape_data input shape data\n * \\param out_shape_size ouput shape size\n * \\param out_shape_ndim output shape number of dims\n * \\param out_shape_data output shape data\n * \\param aux_shape_size shape size of auxiliary states\n * \\param aux_shape_ndim number of dims of auxiliary states shape\n * \\param aux_shape_data shape data of auxiliary states\n * \\param complete indicates completion of Shape Inference\n * \\return 0 when success, -1 when failure happens\n */\nint MXSymbolInferShapePartial64(SymbolHandle sym,\n                                uint32_t num_args,\n                                const char** keys,\n                                const int64_t* arg_ind_ptr,\n                                const int64_t* arg_shape_data,\n                                size_t* in_shape_size,\n                                const int** in_shape_ndim,\n                                const int64_t*** in_shape_data,\n                                size_t* out_shape_size,\n                                const int** out_shape_ndim,\n                                const int64_t*** out_shape_data,\n                                size_t* aux_shape_size,\n                                const int** aux_shape_ndim,\n                                const int64_t*** aux_shape_data,\n                                int* complete) {\n  int succ  = 0;\n  *complete = 1;\n  return MXSymbolInferShape64(sym,\n                              num_args,\n                              keys,\n                              arg_ind_ptr,\n                              arg_shape_data,\n                              in_shape_size,\n                              in_shape_ndim,\n                              in_shape_data,\n                              out_shape_size,\n                              out_shape_ndim,\n                              out_shape_data,\n                              aux_shape_size,\n                              aux_shape_ndim,\n                              aux_shape_data,\n                              &succ);\n}\n\nint MXSymbolInferType(SymbolHandle sym,\n                      uint32_t num_args,\n                      const char** keys,\n                      const int* arg_type_data,\n                      uint32_t* in_type_size,\n                      const int** in_type_data,\n                      uint32_t* out_type_size,\n                      const int** out_type_data,\n                      uint32_t* aux_type_size,\n                      const int** aux_type_data,\n                      int* complete) {\n  nnvm::Symbol* s              = static_cast<nnvm::Symbol*>(sym);\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  API_BEGIN();\n  nnvm::Graph g = Symbol2Graph(*s);\n  nnvm::DTypeVector arg_types(g.indexed_graph().input_nodes().size(), -1);\n  if (keys == nullptr && num_args != 0) {\n    std::vector<uint32_t> read_only_args = mxnet::ReadOnlyArgIndices(g.indexed_graph());\n    CHECK_LE(num_args, read_only_args.size());\n    for (uint32_t i = 0; i < num_args; ++i) {\n      arg_types[read_only_args[i]] = arg_type_data[i];\n    }\n  } else {\n    std::unordered_map<std::string, int> kwargs;\n    for (uint32_t i = 0; i < num_args; ++i) {\n      kwargs[keys[i]] = arg_type_data[i];\n    }\n    mxnet::MatchArguments(g.indexed_graph(), kwargs, &arg_types, \"InferType\");\n  }\n\n  g = mxnet::exec::InferType(std::move(g), std::move(arg_types), \"__dtype__\");\n  // copy back\n  CopyAttr(g.indexed_graph(),\n           g.GetAttr<nnvm::DTypeVector>(\"dtype\"),\n           &(ret->arg_types),\n           &(ret->out_types),\n           &(ret->aux_types));\n\n  *in_type_size  = static_cast<uint32_t>(ret->arg_types.size());\n  *in_type_data  = dmlc::BeginPtr(ret->arg_types);\n  *out_type_size = static_cast<uint32_t>(ret->out_types.size());\n  *out_type_data = dmlc::BeginPtr(ret->out_types);\n  *aux_type_size = static_cast<uint32_t>(ret->aux_types.size());\n  *aux_type_data = dmlc::BeginPtr(ret->aux_types);\n  *complete      = (g.GetAttr<size_t>(\"dtype_num_unknown_nodes\") == 0);\n  API_END();\n}\n\nint MXSymbolInferTypePartial(SymbolHandle sym,\n                             uint32_t num_args,\n                             const char** keys,\n                             const int* arg_type_data,\n                             uint32_t* in_type_size,\n                             const int** in_type_data,\n                             uint32_t* out_type_size,\n                             const int** out_type_data,\n                             uint32_t* aux_type_size,\n                             const int** aux_type_data,\n                             int* complete) {\n  int succ  = 0;\n  *complete = 1;\n  return MXSymbolInferType(sym,\n                           num_args,\n                           keys,\n                           arg_type_data,\n                           in_type_size,\n                           in_type_data,\n                           out_type_size,\n                           out_type_data,\n                           aux_type_size,\n                           aux_type_data,\n                           &succ);\n}\n\nint MXSymbolGrad(SymbolHandle sym, uint32_t num_wrt, const char** wrt, SymbolHandle* out) {\n  API_BEGIN();\n  LOG(FATAL) << \"not implemented\";\n  API_END();\n}\n\nint MXQuantizeSymbol(SymbolHandle sym_handle,\n                     SymbolHandle* ret_sym_handle,\n                     const int* dev_type,\n                     const uint32_t num_excluded_sym_names,\n                     const char** excluded_sym_names,\n                     const uint32_t num_excluded_op_names,\n                     const char** excluded_op_names,\n                     const uint32_t num_offline,\n                     const char** offline_params,\n                     const char* quantized_dtype,\n                     const bool calib_quantize,\n                     const char* quantize_mode,\n                     const char* quantize_granularity,\n                     mx_uint* out_num_calib_names,\n                     const char*** out_calib_names) {\n  nnvm::Symbol* s = new nnvm::Symbol();\n  API_BEGIN();\n  nnvm::Symbol* sym = static_cast<nnvm::Symbol*>(sym_handle);\n  nnvm::Graph g     = Symbol2Graph(*sym);\n  int target_dev    = *dev_type;\n  std::unordered_set<std::string> excluded_node_names;\n  for (size_t i = 0; i < num_excluded_sym_names; ++i) {\n    excluded_node_names.emplace(excluded_sym_names[i]);\n  }\n  std::unordered_set<std::string> excluded_op;\n  for (size_t i = 0; i < num_excluded_op_names; ++i) {\n    excluded_op.emplace(excluded_op_names[i]);\n  }\n  std::unordered_set<std::string> offline;\n  for (size_t i = 0; i < num_offline; ++i) {\n    offline.emplace(offline_params[i]);\n  }\n  std::string quantized_type(quantized_dtype);\n  std::string quantized_mode(quantize_mode);\n  std::string quantized_granularity(quantize_granularity);\n  g.attrs[\"excluded_nodes\"]       = std::make_shared<nnvm::any>(std::move(excluded_node_names));\n  g.attrs[\"excluded_ops\"]         = std::make_shared<nnvm::any>(std::move(excluded_op));\n  g.attrs[\"offline_params\"]       = std::make_shared<nnvm::any>(std::move(offline));\n  g.attrs[\"quantized_dtype\"]      = std::make_shared<nnvm::any>(std::move(quantized_type));\n  g.attrs[\"target_ctx\"]           = std::make_shared<nnvm::any>(target_dev);\n  g.attrs[\"quantize_mode\"]        = std::make_shared<nnvm::any>(std::move(quantized_mode));\n  g.attrs[\"quantize_granularity\"] = std::make_shared<nnvm::any>(std::move(quantized_granularity));\n  g                               = ApplyPass(std::move(g), \"QuantizeGraph\");\n  const auto& calib_nodes         = g.GetAttr<std::vector<std::string>>(\"calib_nodes\");\n  MXAPIThreadLocalEntry<>* ret    = MXAPIThreadLocalStore<>::Get();\n  ret->ret_vec_str                = calib_nodes;\n  *out_num_calib_names            = ret->ret_vec_str.size();\n  ret->ret_vec_charp.clear();\n  ret->ret_vec_charp.reserve(ret->ret_vec_str.size());\n  for (const auto& str : ret->ret_vec_str) {\n    ret->ret_vec_charp.push_back(str.c_str());\n  }\n  *out_calib_names = dmlc::BeginPtr(ret->ret_vec_charp);\n  s->outputs       = g.outputs;\n  *ret_sym_handle  = s;\n  API_END_HANDLE_ERROR(delete s);\n}\n\nint MXReducePrecisionSymbol(SymbolHandle sym_handle,\n                            SymbolHandle* ret_sym_handle,\n                            const int target_dtype,\n                            const int cast_params_offline,\n                            const char* const offline_param_cast_attr_p,\n                            const uint32_t num_inputs,\n                            const char** const input_names_p,\n                            const uint32_t num_all_args,\n                            const char** const all_arg_names_p,\n                            const int* all_arg_types_p,\n                            const uint32_t num_target_dtype_ops,\n                            const char** const target_dtype_ops_p,\n                            const uint32_t num_fp32_ops,\n                            const char** const fp32_ops_p,\n                            const uint32_t num_widest_dtype_ops,\n                            const char** const widest_dtype_ops_p) {\n  nnvm::Symbol* result_sym = new nnvm::Symbol();\n  API_BEGIN();\n  nnvm::Symbol* sym                   = static_cast<nnvm::Symbol*>(sym_handle);\n  nnvm::Graph g                       = Symbol2Graph(*sym);\n  std::string offline_param_cast_attr = offline_param_cast_attr_p;\n  CHECK_EQ(num_all_args, g.indexed_graph().input_nodes().size());\n\n  std::unordered_set<std::string> input_names(input_names_p, input_names_p + num_inputs);\n  std::unordered_set<std::string> target_dtype_ops(target_dtype_ops_p,\n                                                   target_dtype_ops_p + num_target_dtype_ops);\n  std::unordered_set<std::string> fp32_ops(fp32_ops_p, fp32_ops_p + num_fp32_ops);\n  std::unordered_set<std::string> widest_dtype_ops(widest_dtype_ops_p,\n                                                   widest_dtype_ops_p + num_widest_dtype_ops);\n\n  nnvm::DTypeVector arg_types(num_all_args);\n  std::unordered_map<std::string, int> node_name_to_type_map;\n  for (int i = 0; i < num_all_args; ++i) {\n    node_name_to_type_map[all_arg_names_p[i]] = all_arg_types_p[i];\n  }\n  mxnet::MatchArguments(g.indexed_graph(), node_name_to_type_map, &arg_types, \"InferType\");\n  g = mxnet::exec::InferType(std::move(g), std::move(arg_types), \"\");\n\n  // InferType sets the \"dtype\" attribute with all infered types\n  g.attrs[\"target_dtype\"]        = std::make_shared<nnvm::any>(target_dtype);\n  g.attrs[\"cast_params_offline\"] = std::make_shared<nnvm::any>(cast_params_offline);\n  g.attrs[\"offline_param_cast_attr\"] =\n      std::make_shared<nnvm::any>(std::move(offline_param_cast_attr));\n  g.attrs[\"input_names\"]      = std::make_shared<nnvm::any>(std::move(input_names));\n  g.attrs[\"target_dtype_ops\"] = std::make_shared<nnvm::any>(std::move(target_dtype_ops));\n  g.attrs[\"fp32_ops\"]         = std::make_shared<nnvm::any>(std::move(fp32_ops));\n  g.attrs[\"widest_dtype_ops\"] = std::make_shared<nnvm::any>(std::move(widest_dtype_ops));\n  g                           = ApplyPass(std::move(g), \"ReducePrecision\");\n\n  result_sym->outputs                      = g.outputs;\n  *ret_sym_handle                          = result_sym;\n  nnvm::Symbol* ret_sym                    = static_cast<nnvm::Symbol*>(*ret_sym_handle);\n  const std::vector<nnvm::ObjectPtr>& args = ret_sym->ListInputs(nnvm::Symbol::kAll);\n\n  API_END_HANDLE_ERROR(delete result_sym);\n}\n\nint MXSetCalibTableToQuantizedSymbol(SymbolHandle qsym_handle,\n                                     const uint32_t num_layers,\n                                     const char** layer_names,\n                                     const float* min_ranges,\n                                     const float* max_ranges,\n                                     SymbolHandle* ret_qsym_handle) {\n  nnvm::Symbol* s = new nnvm::Symbol();\n  API_BEGIN();\n  nnvm::Symbol* sym = static_cast<nnvm::Symbol*>(qsym_handle);\n  nnvm::Graph g     = Symbol2Graph(*sym);\n  std::unordered_map<std::string, std::pair<float, float>> calib_table;\n  for (size_t i = 0; i < num_layers; ++i) {\n    calib_table.emplace(layer_names[i], std::make_pair(min_ranges[i], max_ranges[i]));\n  }\n  g.attrs[\"calib_table\"] = std::make_shared<nnvm::any>(std::move(calib_table));\n  g                      = ApplyPass(std::move(g), \"SetCalibTableToQuantizedGraph\");\n  s->outputs             = g.outputs;\n  *ret_qsym_handle       = s;\n  API_END_HANDLE_ERROR(delete s);\n}\n\nint MXGenBackendSubgraph(SymbolHandle sym_handle,\n                         const char* backend_name,\n                         SymbolHandle* ret_sym_handle) {\n  nnvm::Symbol* s = new nnvm::Symbol();\n  API_BEGIN();\n  nnvm::Symbol* sym = static_cast<nnvm::Symbol*>(sym_handle);\n  *s                = sym->Copy();\n  auto backend      = mxnet::op::SubgraphBackendRegistry::Get()->GetSubgraphBackend(backend_name);\n  const auto& subgraph_prop_list = backend->GetSubgraphProperties();\n  for (auto property : subgraph_prop_list) {\n    if (property->HasAttr(\"disable\") && property->GetAttr<bool>(\"disable\") == true) {\n      auto full_name = property->HasAttr(\"property_name\") ?\n                           property->GetAttr<std::string>(\"property_name\") :\n                           std::string();\n      LOG(INFO) << \"subgraph property \" << full_name << \" from backend \" << backend_name\n                << \" is disabled.\";\n      continue;\n    }\n    nnvm::Graph g = Symbol2Graph(*s);\n    property->SetAttr(\"graph\", g);\n    g.attrs[\"subgraph_property\"] = std::make_shared<nnvm::any>(property);\n    g                            = ApplyPass(std::move(g), \"EliminateCommonNodesPass\");\n    g                            = ApplyPass(std::move(g), \"BuildSubgraph\");\n    property->RemoveAttr(\"graph\");\n    g.attrs.erase(\"subgraph_property\");\n    s->outputs = g.outputs;\n  }\n  *ret_sym_handle = s;\n  API_END_HANDLE_ERROR(delete s);\n}\n\nint MXGenAtomicSymbolFromSymbol(SymbolHandle sym_handle, SymbolHandle* ret_sym_handle) {\n  nnvm::Symbol* s = new nnvm::Symbol();\n  API_BEGIN();\n  nnvm::Symbol* source = static_cast<nnvm::Symbol*>(sym_handle);\n  CHECK_GE(source->outputs.size(), 1) << \"Input symbol does not have outputs.\";\n  const auto& node = source->outputs[0].node;\n  for (const auto& other_node : source->outputs) {\n    if (node.get() != other_node.node.get()) {\n      LOG(FATAL) << \"Generating atomic symbol from other symbol only works for nongrouped symbol.\";\n    }\n  }\n  const auto* op   = node->op();\n  const auto attrs = source->ListAttrs(nnvm::Symbol::ListAttrOption::kShallow);\n  *s               = nnvm::Symbol::CreateFunctor(op, attrs);\n  *ret_sym_handle  = s;\n  API_END_HANDLE_ERROR(delete s);\n}\n\nint MXShallowCopySymbol(SymbolHandle src, SymbolHandle* out) {\n  nnvm::Symbol* out_sym = new nnvm::Symbol;\n  API_BEGIN();\n  nnvm::Symbol* src_sym = static_cast<nnvm::Symbol*>(src);\n  *out_sym              = *src_sym;\n  *out                  = out_sym;\n  API_END_HANDLE_ERROR(delete out_sym);\n}\n\nint MXOptimizeForBackend(SymbolHandle sym_handle,\n                         const char* backend_name,\n                         const int dev_type,\n                         SymbolHandle* ret_sym_handle,\n                         const mx_uint args_len,\n                         NDArrayHandle* in_args_handle,\n                         const mx_uint aux_len,\n                         NDArrayHandle* in_aux_handle,\n                         const mx_uint num_options,\n                         const char** keys,\n                         const char** vals,\n                         const uint32_t num_input_shapes,\n                         const char** input_shape_names,\n                         const int64_t* input_shape_data,\n                         const uint32_t* input_shape_idx,\n                         const uint32_t num_input_dtypes,\n                         const char** input_dtype_names,\n                         const int* input_dtypes,\n                         const uint32_t num_input_stypes,\n                         const char** input_stype_names,\n                         const int* input_stypes,\n                         bool skip_infer,\n                         int* new_args_cnt,\n                         NDArrayHandle** new_args_handle,\n                         char*** new_arg_names_handle,\n                         int* new_aux_cnt,\n                         NDArrayHandle** new_aux_handle,\n                         char*** new_aux_names_handle) {\n  // create copy of input symbol\n  nnvm::Symbol* s = new nnvm::Symbol();\n  API_BEGIN();\n  nnvm::Symbol* sym = static_cast<nnvm::Symbol*>(sym_handle);\n  *s                = sym->Copy();\n\n  // create a data structure from pointer array\n  std::unordered_map<std::string, std::string> options_map;\n  for (mx_uint i = 0; i < num_options; ++i)\n    options_map.emplace(keys[i], vals[i]);\n\n  NDArray*** new_args_ptr = reinterpret_cast<NDArray***>(new_args_handle);\n  NDArray*** new_aux_ptr  = reinterpret_cast<NDArray***>(new_aux_handle);\n  NDArray** in_args_ptr   = reinterpret_cast<NDArray**>(in_args_handle);\n  NDArray** in_aux_ptr    = reinterpret_cast<NDArray**>(in_aux_handle);\n\n  auto init_graph = [&](auto s) {\n    nnvm::Graph g = Symbol2Graph(*s);\n\n    // EliminateCommonNodesPass must be performed before first call to the indexed graph,\n    // because otherwise changing graph via other passes will result in an error, due to the fact\n    // that once indexed_graph is created, it cannot be changed.\n    g                                    = ApplyPass(std::move(g), \"EliminateCommonNodesPass\");\n    const auto& indexed_graph            = g.indexed_graph();\n    const auto& mutable_nodes            = indexed_graph.mutable_input_nodes();\n    std::vector<std::string> input_names = s->ListInputNames(nnvm::Symbol::kAll);\n    size_t num_forward_inputs            = input_names.size();\n\n    if (args_len || aux_len) {\n      if (!skip_infer) {\n        Context default_ctx = Context::Create(static_cast<Context::DeviceType>(dev_type), 0);\n        mxnet::ShapeVector arg_shapes(args_len + aux_len);\n        nnvm::DTypeVector arg_dtypes(args_len + aux_len);\n        StorageTypeVector arg_stypes(args_len + aux_len);\n\n        // create the input shape, dtype and stype maps\n        std::unordered_map<std::string, mxnet::TShape> input_shape_map(num_input_shapes);\n        for (uint32_t i = 0; i < num_input_shapes; ++i) {\n          input_shape_map.emplace(input_shape_names[i],\n                                  mxnet::TShape(input_shape_data + input_shape_idx[i],\n                                                input_shape_data + input_shape_idx[i + 1]));\n        }\n        std::unordered_map<std::string, int> input_dtype_map(num_input_dtypes);\n        for (uint32_t i = 0; i < num_input_dtypes; ++i) {\n          input_dtype_map.emplace(input_dtype_names[i], input_dtypes[i]);\n        }\n        std::unordered_map<std::string, int> input_stype_map(num_input_stypes);\n        for (uint32_t i = 0; i < num_input_stypes; ++i) {\n          input_stype_map.emplace(input_stype_names[i], input_stypes[i]);\n        }\n\n        size_t args_top = 0, aux_top = 0;\n        // loop over inputs to symbol in order and add to args/aux if mutable\n        for (size_t i = 0; i < num_forward_inputs; ++i) {\n          const uint32_t nid = indexed_graph.input_nodes().at(i);\n          if (mutable_nodes.count(nid)) {\n            CHECK_LT(aux_top, aux_len)\n                << \"Cannot find aux '\" << input_names[i] << \"' in provided aux to optimize_for\";\n            if (in_aux_ptr[aux_top] != nullptr) {\n              const auto& in_arg = *(in_aux_ptr[aux_top]);\n              arg_shapes[i]      = in_arg.shape();\n              arg_dtypes[i]      = in_arg.dtype();\n              arg_stypes[i]      = in_arg.storage_type();\n            }\n            aux_top++;\n          } else {\n            auto name = input_names[i];\n            CHECK_LT(args_top, args_len)\n                << \"Cannot find arg '\" << name << \"' in provided args to optimize_for\";\n            if (in_args_ptr[args_top] != nullptr) {\n              const auto& in_arg = *(in_args_ptr[args_top]);\n              arg_shapes[i]      = in_arg.shape();\n              arg_dtypes[i]      = in_arg.dtype();\n              arg_stypes[i]      = in_arg.storage_type();\n            } else {\n              // input_names[i] is not in args but can be in the optional\n              // shape/type/stype attribute dicts.\n              auto it_shape = input_shape_map.find(name);\n              if (it_shape != input_shape_map.end()) {\n                arg_shapes[i] = it_shape->second;\n              }\n              auto it_type = input_dtype_map.find(name);\n              if (it_type != input_dtype_map.end()) {\n                arg_dtypes[i] = it_type->second;\n              }\n              it_type = input_stype_map.find(name);\n              if (it_type != input_stype_map.end()) {\n                arg_stypes[i] = it_type->second;\n              }\n            }\n            args_top++;\n          }\n        }\n\n        g.attrs[\"context\"] = std::make_shared<nnvm::any>(\n            exec::ContextVector(indexed_graph.num_nodes(), default_ctx));\n\n        // infer shapes\n        g = exec::InferShape(std::move(g), std::move(arg_shapes), \"__shape__\");\n        // infer dtypes\n        g = exec::InferType(std::move(g), std::move(arg_dtypes), \"__dtype__\");\n        // infer stypes\n        g = exec::InferStorageType(std::move(g), std::move(arg_stypes), \"__storage_type__\");\n      }\n      // set args/aux as attributes on graph so that subgraph property can use them\n      std::vector<std::string> arg_names = s->ListInputNames(nnvm::Symbol::kReadOnlyArgs);\n      g.attrs[\"in_args\"]                 = std::make_shared<nnvm::any>(in_args_ptr);\n      g.attrs[\"in_arg_names\"]            = std::make_shared<nnvm::any>(arg_names);\n\n      std::vector<std::string> aux_names = s->ListInputNames(nnvm::Symbol::kAuxiliaryStates);\n      g.attrs[\"in_aux\"]                  = std::make_shared<nnvm::any>(in_aux_ptr);\n      g.attrs[\"in_aux_names\"]            = std::make_shared<nnvm::any>(aux_names);\n    } else {\n      // args/aux were not specified, so set nullptr/empty-lists\n      NDArray** in_args_ptr = static_cast<NDArray**>(nullptr);\n      std::vector<std::string> arg_names;\n      g.attrs[\"in_args\"]      = std::make_shared<nnvm::any>(in_args_ptr);\n      g.attrs[\"in_arg_names\"] = std::make_shared<nnvm::any>(arg_names);\n\n      NDArray** in_aux_ptr = static_cast<NDArray**>(nullptr);\n      std::vector<std::string> aux_names;\n      g.attrs[\"in_aux\"]       = std::make_shared<nnvm::any>(in_aux_ptr);\n      g.attrs[\"in_aux_names\"] = std::make_shared<nnvm::any>(aux_names);\n    }\n\n    // set dedup option as attribute on graph to enable dedup during partitioning\n    if (options_map.count(\"dedup_subgraph\") > 0 &&\n        options_map.at(\"dedup_subgraph\").compare(\"True\") == 0)\n      g.attrs[\"dedup_subgraph\"] = std::make_shared<nnvm::any>(std::string(\"True\"));\n    return g;\n  };\n\n  if (mxnet::op::SubgraphBackendRegistry::Get()->backend_map_.count(backend_name) > 0) {\n    // use subgraph backend\n    const auto backend =\n        mxnet::op::SubgraphBackendRegistry ::Get()->GetSubgraphBackend(backend_name);\n    const auto& subgraph_prop_list = backend->GetSubgraphProperties();\n    for (auto property : subgraph_prop_list) {\n      if (property->HasAttr(\"disable\") && property->GetAttr<bool>(\"disable\") == true) {\n        auto full_name = property->HasAttr(\"property_name\") ?\n                             property->GetAttr<std::string>(\"property_name\") :\n                             std::string();\n        LOG(INFO) << \"subgraph property \" << full_name << \" from backend \" << backend_name\n                  << \" is disabled.\";\n        continue;\n      }\n      nnvm::Graph g = init_graph(s);\n      property->PrePartition(g, options_map);\n      g.attrs[\"subgraph_property\"] = std::make_shared<nnvm::any>(property);\n      g                            = ApplyPass(std::move(g), \"BuildSubgraph\");\n      g.attrs.erase(\"subgraph_property\");\n      property->PostPartition(g);\n      s->outputs = g.outputs;\n    }\n  } else if (dmlc::Registry<nnvm::PassFunctionReg>::Find(backend_name) != nullptr) {\n    // use graph pass\n    nnvm::Graph g          = init_graph(s);\n    g.attrs[\"options_map\"] = std::make_shared<nnvm::any>(options_map);\n    g.attrs[\"pass_name\"]   = std::make_shared<nnvm::any>(backend_name);\n    g                      = ApplyPass(std::move(g), backend_name);\n\n    std::vector<NDArray*> new_args         = g.GetAttr<std::vector<NDArray*>>(\"new_args\");\n    std::vector<NDArray*> new_aux          = g.GetAttr<std::vector<NDArray*>>(\"new_aux\");\n    std::vector<std::string> new_arg_names = g.GetAttr<std::vector<std::string>>(\"new_arg_names\");\n    std::vector<std::string> new_aux_names = g.GetAttr<std::vector<std::string>>(\"new_aux_names\");\n    g.attrs.erase(\"new_args\");\n    g.attrs.erase(\"new_aux\");\n    g.attrs.erase(\"new_arg_names\");\n    g.attrs.erase(\"new_aux_names\");\n    s->outputs = g.outputs;\n\n    NDArray** new_arg_arr = new NDArray*[new_arg_names.size()];\n    NDArray** new_aux_arr = new NDArray*[new_aux_names.size()];\n    char** new_arg_cstr   = new char*[new_arg_names.size()];\n    char** new_aux_cstr   = new char*[new_aux_names.size()];\n    for (unsigned i = 0; i < new_arg_names.size(); i++) {\n      new_arg_arr[i] = new_args[i];\n      std::string& s = new_arg_names[i];\n      char* tmp      = new char[s.length() + 1];\n      s.copy(tmp, s.length());\n      tmp[s.length()] = '\\0';\n      new_arg_cstr[i] = tmp;\n    }\n    for (unsigned i = 0; i < new_aux_names.size(); i++) {\n      new_aux_arr[i] = new_aux[i];\n      std::string& s = new_aux_names[i];\n      char* tmp      = new char[s.length() + 1];\n      s.copy(tmp, s.length());\n      tmp[s.length()] = '\\0';\n      new_aux_cstr[i] = tmp;\n    }\n    *new_args_cnt         = new_arg_names.size();\n    *new_aux_cnt          = new_aux_names.size();\n    *new_arg_names_handle = new_arg_cstr;\n    *new_aux_names_handle = new_aux_cstr;\n    *new_args_ptr         = new_arg_arr;\n    *new_aux_ptr          = new_aux_arr;\n  } else {\n    // cannot find graph pass or subgraph backend registered in this name\n    LOG(ERROR) << \"Error optimizing for backend '\" << backend_name << \"' cannot be found\";\n  }\n\n  *ret_sym_handle = s;\n  API_END_HANDLE_ERROR(delete s);\n}\n\nint MXCheckDynamicShapeOp(SymbolHandle sym_handle, bool* has_dynamic_shape) {\n  nnvm::Symbol* s = new nnvm::Symbol();\n  API_BEGIN();\n  *has_dynamic_shape = false;\n  // traverse the symbol and check if any dynamic shape is present\n  nnvm::Symbol* sym      = static_cast<nnvm::Symbol*>(sym_handle);\n  *s                     = sym->Copy();\n  nnvm::Graph g          = Symbol2Graph(*s);\n  const auto& infershape = nnvm::Op::GetAttr<mxnet::FInferShape>(\"FInferShape\");\n  DFSVisit(g.outputs, [infershape, has_dynamic_shape](const nnvm::ObjectPtr n) {\n    if (*has_dynamic_shape)\n      return;\n    if (!n->is_variable() && !infershape.count(n->op())) {\n      *has_dynamic_shape = true;\n      return;\n    }\n  });\n  API_END_HANDLE_ERROR(delete s);\n}\n"
  },
  {
    "path": "src/c_api/c_api_test.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file c_api_test.cc\n * \\brief C API of mxnet for the ease of testing backend in Python\n */\n#include <mxnet/c_api_test.h>\n#include <nnvm/pass.h>\n#include \"./c_api_common.h\"\n#include \"../operator/subgraph/subgraph_property.h\"\n#include \"../common/cuda/rtc.h\"\n\nint MXBuildSubgraphByOpNames(SymbolHandle sym_handle,\n                             const char* prop_name,\n                             const uint32_t num_ops,\n                             const char** op_names,\n                             SymbolHandle* ret_sym_handle) {\n  nnvm::Symbol* s = new nnvm::Symbol();\n  API_BEGIN();\n  std::unordered_set<std::string> op_name_set;\n  for (size_t i = 0; i < num_ops; ++i) {\n    op_name_set.emplace(op_names[i]);\n  }\n  nnvm::Symbol* sym = static_cast<nnvm::Symbol*>(sym_handle);\n  *s                = sym->Copy();\n  if (!op_name_set.empty()) {\n    auto& backend = mxnet::op::SubgraphBackendRegistry::Get()->GetSubgraphBackend(prop_name);\n    LOG(INFO) << \"Subgraph backend \" << backend->GetName() << \" is activated.\";\n    const auto& subgraph_prop_list = backend->GetSubgraphProperties();\n    for (auto property : subgraph_prop_list) {\n      nnvm::Graph g;\n      g.outputs = s->outputs;\n      property->SetAttr(\"graph\", g);\n      property->SetAttr(\"op_names\", op_name_set);\n      g.attrs[\"subgraph_property\"] = std::make_shared<nnvm::any>(property);\n      g                            = nnvm::ApplyPass(std::move(g), \"EliminateCommonNodesPass\");\n      g                            = nnvm::ApplyPass(std::move(g), \"BuildSubgraph\");\n      property->RemoveAttr(\"graph\");\n      g.attrs.erase(\"subgraph_property\");\n      s->outputs = g.outputs;\n    }\n  }\n  *ret_sym_handle = s;\n  API_END_HANDLE_ERROR(delete s);\n}\n\nint MXSetSubgraphPropertyOpNames(const char* prop_name,\n                                 const uint32_t num_ops,\n                                 const char** op_names) {\n  API_BEGIN();\n  std::unordered_set<std::string> op_name_set;\n  for (size_t i = 0; i < num_ops; ++i) {\n    op_name_set.emplace(op_names[i]);\n  }\n  (*mxnet::op::SubgraphPropertyOpNameSet::Get())[prop_name] = op_name_set;\n  API_END();\n}\n\nint MXSetSubgraphPropertyOpNamesV2(const char* prop_name,\n                                   const uint32_t num_ops,\n                                   const char** op_names) {\n  API_BEGIN();\n  std::unordered_set<std::string> op_name_set;\n  for (size_t i = 0; i < num_ops; ++i) {\n    op_name_set.emplace(op_names[i]);\n  }\n  auto& backend = mxnet::op::SubgraphBackendRegistry::Get()->GetSubgraphBackend(prop_name);\n  const auto& subgraph_prop_list = backend->GetSubgraphProperties();\n  for (auto& property : subgraph_prop_list) {\n    property->SetAttr(\"op_names\", op_name_set);\n  }\n  API_END();\n}\n\nint MXRemoveSubgraphPropertyOpNames(const char* prop_name) {\n  API_BEGIN();\n  mxnet::op::SubgraphPropertyOpNameSet::Get()->erase(prop_name);\n  API_END();\n}\n\nint MXRemoveSubgraphPropertyOpNamesV2(const char* prop_name) {\n  API_BEGIN();\n  auto& backend = mxnet::op::SubgraphBackendRegistry::Get()->GetSubgraphBackend(prop_name);\n  const auto& subgraph_prop_list = backend->GetSubgraphProperties();\n  for (auto& property : subgraph_prop_list) {\n    property->RemoveAttr(\"op_names\");\n  }\n  API_END();\n}\n\nint MXGetEnv(const char* name, const char** value) {\n  API_BEGIN();\n  *value = getenv(name);\n  API_END();\n}\n\nint MXSetEnv(const char* name, const char* value) {\n  API_BEGIN();\n#ifdef _WIN32\n  auto value_arg = (value == nullptr) ? \"\" : value;\n  _putenv_s(name, value_arg);\n#else\n  if (value == nullptr)\n    unsetenv(name);\n  else\n    setenv(name, value, 1);\n#endif\n  API_END();\n}\n\nint MXGetMaxSupportedArch(uint32_t* max_arch) {\n  API_BEGIN();\n#if MXNET_USE_CUDA\n  *max_arch = static_cast<uint32_t>(mxnet::common::cuda::rtc::GetMaxSupportedArch());\n#else\n  LOG(FATAL) << \"Compile with USE_CUDA=1 to have CUDA runtime compilation.\";\n#endif\n  API_END();\n}\n"
  },
  {
    "path": "src/common/alm.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file alm.cc\n * \\brief Automatic Layout Manager\n * \\author Dawid Tracz, Vladimir Cherepanov\n */\n\n#include \"alm.h\"\n\n#include <algorithm>\n#include <sstream>\n#include <unordered_set>\n#include <utility>\n\n#include \"../operator/nn/convolution-inl.h\"\n#include \"../operator/nn/deconvolution-inl.h\"\n#include \"../operator/tensor/matrix_op-inl.h\"\n\nnamespace mxnet {\nnamespace alm {\n\nnamespace {\n\nnnvm::ObjectPtr CreateTransposeNode(const std::string& name, const alm::Transpose& axes) {\n  nnvm::ObjectPtr newptr = nnvm::Node::Create();\n  newptr->attrs.op       = nnvm::Op::Get(\"transpose\");\n  newptr->attrs.name     = name;\n  // set tranpose axes\n  std::ostringstream ss;\n  ss << mxnet::TShape(axes.begin(), axes.end());\n  newptr->attrs.dict[\"axes\"] = ss.str();\n  newptr->op()->attr_parser(&(newptr->attrs));\n  return newptr;\n}\n\nmshadow::LayoutFlag TargetLayout(const nnvm::ObjectPtr& node) {\n  static const Op* conv_op   = Op::Get(\"Convolution\");\n  static const Op* deconv_op = Op::Get(\"Deconvolution\");\n\n  static const std::unordered_map<int, mshadow::LayoutFlag> ndim2layout{\n      {1, mshadow::kNWC},\n      {2, mshadow::kNHWC},\n      {3, mshadow::kNDHWC},\n  };\n\n  auto target_layout = [](const auto& param) {\n    auto it = ndim2layout.find(param.kernel.ndim());\n    CHECK(it != ndim2layout.end()) << \"Unexpected kernel dimensions: \" << param.kernel;\n    return it->second;\n  };\n\n  if (node->op() == conv_op)\n    return target_layout(nnvm::get<op::ConvolutionParam>(node->attrs.parsed));\n\n  if (node->op() == deconv_op)\n    return target_layout(nnvm::get<op::DeconvolutionParam>(node->attrs.parsed));\n\n  return mshadow::kUNKNOWN;\n}\n\n}  // namespace\n\nnnvm::Graph OptimizeLayout(nnvm::Graph&& g) {\n  static const auto& op_map     = Op::GetAttr<mxnet::alm::FChangeLayout>(\"FChangeLayout\");\n  static const Op* transpose_op = Op::Get(\"transpose\");\n  std::unordered_set<nnvm::ObjectPtr> outputs;\n  for (auto& o : g.outputs)\n    outputs.insert(o.node);\n  nnvm::NodeEntryMap<alm::Transpose> changed;\n  struct ToDelete {\n    nnvm::ObjectPtr node;  // output of the transpose\n    size_t input_idx;\n  };\n  std::vector<ToDelete> to_delete;\n  struct ToAdd {\n    nnvm::ObjectPtr node;\n    size_t input_idx;\n    alm::Transpose axes;\n  };\n  std::vector<ToAdd> to_add;\n  DFSVisit(g.outputs, [&outputs, &changed, &to_add, &to_delete](const nnvm::ObjectPtr& node) {\n    std::vector<alm::Transpose> input_axes(node->inputs.size());\n    for (size_t i = 0; i < node->inputs.size(); ++i) {\n      if (node->inputs[i].node->op() == transpose_op) {\n        const auto& param = nnvm::get<op::TransposeParam>(node->inputs[i].node->attrs.parsed);\n        if (IsIdentity(FromTShape(param.axes))) {\n          to_delete.push_back({node, i});\n          continue;\n        }\n      }\n      auto it = changed.find(node->inputs[i]);\n      if (it == changed.end())\n        continue;\n      input_axes[i] = it->second;\n    }\n    auto fchange = op_map.get(node->op(), nullptr);\n    if (fchange && outputs.count(node) == 0) {\n      std::vector<alm::Transpose> output_axes;\n      if (fchange(&node->attrs, TargetLayout(node), &input_axes, &output_axes))\n        node->op()->attr_parser(&node->attrs);\n      for (size_t i = 0; i < output_axes.size(); ++i) {\n        if (IsIdentity(output_axes[i]))\n          continue;\n        changed.insert(std::make_pair(nnvm::NodeEntry(node, i, 0), output_axes[i]));\n      }\n    }\n    for (size_t i = 0; i < input_axes.size(); ++i) {\n      if (IsIdentity(input_axes[i]))\n        continue;\n      to_add.push_back({node, i, input_axes[i]});\n    }\n  });\n  for (const auto& t : to_delete) {\n    auto& tnode = t.node->inputs[t.input_idx].node;\n    CHECK_EQ(tnode->inputs.size(), 1);\n    t.node->inputs[t.input_idx] = tnode->inputs[0];\n  }\n  size_t node_no = 0;\n  for (const auto& t : to_add) {\n    auto tnode = CreateTransposeNode(\"ALM_transpose_\" + std::to_string(node_no++), t.axes);\n    tnode->inputs.push_back(t.node->inputs[t.input_idx]);\n    t.node->inputs[t.input_idx] = nnvm::NodeEntry(tnode);\n  }\n  nnvm::Graph ret;\n  ret.outputs = g.outputs;\n  return ret;\n}\n\nTranspose Reverse(const Transpose& axes) {\n  Transpose rev(axes.size());\n  for (size_t i = 0; i < rev.size(); i++)\n    rev[axes[i]] = i;\n  return rev;\n}\n\nTranspose Compose(const Transpose& lhs, const Transpose& rhs) {\n  if (lhs.empty())\n    return rhs;\n  if (rhs.empty())\n    return lhs;\n  CHECK_EQ(lhs.size(), rhs.size());\n  Transpose ret(lhs.size());\n  for (auto i = 0; i < ret.size(); ++i)\n    ret[i] = lhs[rhs[i]];\n  return ret;\n}\n\nbool IsIdentity(const Transpose& t) {\n  for (size_t i = 0; i < t.size(); ++i) {\n    if (t[i] != i)\n      return false;\n  }\n  return true;\n}\n\nmshadow::LayoutFlag ApplyTranspose(mshadow::LayoutFlag layout, const Transpose& axes) {\n  auto ret = mshadow::layoutFlag(ApplyTranspose(mshadow::toString(layout), axes));\n  CHECK_NE(ret, mshadow::kUNKNOWN);\n  return ret;\n}\n\nstd::string ApplyTranspose(const std::string& layout, const Transpose& axes) {\n  std::string ret(layout.size(), ' ');\n  for (size_t i = 0; i < ret.size(); i++)\n    ret[i] = layout[axes[i]];\n  return ret;\n}\n\nTranspose FromTShape(const mxnet::TShape& s) {\n  Transpose ret(s.ndim());\n  std::copy(s.begin(), s.end(), ret.begin());\n  return ret;\n}\n\nTranspose FactorCommonTranspose(std::vector<Transpose>* axes) {\n  Transpose ret;\n  for (auto& t : *axes) {\n    if (IsIdentity(t))\n      continue;\n    if (IsIdentity(ret)) {\n      std::swap(t, ret);\n      continue;\n    }\n    auto rev = Reverse(ret);\n    t        = Compose(t, rev);\n  }\n  return ret;\n}\n\n}  // namespace alm\n}  // namespace mxnet\n"
  },
  {
    "path": "src/common/alm.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file alm.h\n * \\brief Automatic Layout Manager\n * \\author Dawid Tracz, Vladimir Cherepanov\n */\n\n#ifndef MXNET_COMMON_ALM_H_\n#define MXNET_COMMON_ALM_H_\n\n#include <mxnet/base.h>\n#include <nnvm/graph.h>\n#include <nnvm/node.h>\n#include <functional>\n#include <string>\n#include <unordered_map>\n#include <vector>\n\nnamespace mxnet {\nnamespace alm {\n\n/*!\n *  \\brief A singleton flag, set and read by MXSetOptimizeLayout and MXGetOptimizeLayout\n */\nstruct ALMParams {\n  bool optimize = false;\n\n  static ALMParams& get() {\n    static ALMParams alm;\n    return alm;\n  }\n};\n\n/*!\n * \\bried Top-level function to run layout optimization.\n */\nnnvm::Graph OptimizeLayout(nnvm::Graph&& g);\n\n/*!\n * \\brief Transpose, represented by permutation of axes.\n */\nusing Transpose = std::vector<size_t>;\n\nbool IsIdentity(const Transpose& t);\nTranspose Reverse(const Transpose& axes);\n\n/*!\n * \\bried Compose 2 transposes. Not commutative: a * b means b is applied first, then a.\n */\nTranspose Compose(const Transpose& lhs, const Transpose& rhs);\n\nmshadow::LayoutFlag ApplyTranspose(mshadow::LayoutFlag layout, const Transpose& axes);\nstd::string ApplyTranspose(const std::string& layout, const Transpose& axes);\n\nTranspose FromTShape(const mxnet::TShape& s);\n\n/*!\n * \\brief May change operator's layout. Used in LayoutOptimization.\n *\n * \\param target_layout The target layout to change to, or kUNKNOWN. In the latter case the target\n * layout is calculated based on in_axes, with a goal to cancel them out (at least some, ideally -\n * all).\n * \\param in_axes (in/out) On input - pending inputs' transposes. On output - inputs' transposes,\n * required by the new layout.\n * \\param out_axes (out) Outputs' transposes, required to convert to the original layouts.\n * \\return true if attrs changed and params need to be reparsed.\n */\nusing FChangeLayout = std::function<bool(nnvm::NodeAttrs*,\n                                         mshadow::LayoutFlag target_layout,\n                                         std::vector<Transpose>* in_axes,\n                                         std::vector<Transpose>* out_axes)>;\n\n/*!\n * \\brief Factors out and returns a common transpose, or default-constructed Transpose if all\n * axes (in/out parameter) are empty.\n */\nTranspose FactorCommonTranspose(std::vector<Transpose>* axes);\n\n}  // namespace alm\n}  // namespace mxnet\n\n#endif  // MXNET_COMMON_ALM_H_\n"
  },
  {
    "path": "src/common/cuda/cudnn_cxx.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file cudnn_cxx.cc\n */\n#include \"cudnn_cxx.h\"\n\n#include <mxnet/base.h>\n#if MXNET_USE_CUDNN == 1\n\n#include <mxnet/storage.h>\n#include <algorithm>\n#include <sstream>\n#include <utility>\n\nnamespace mxnet {\nnamespace cudnn_cxx {\n\nDescriptor Make(cudnnBackendDescriptorType_t type) {\n  cudnnBackendDescriptor_t desc{};\n  CUDNN_CALL(cudnnBackendCreateDescriptor(type, &desc));\n  return Descriptor(desc);\n}\n\nstd::vector<cudnnBackendDescriptor_t> MakeRawDescriptors(size_t n,\n                                                         cudnnBackendDescriptorType_t type) {\n  std::vector<cudnnBackendDescriptor_t> ret(n);\n  for (auto& d : ret)\n    CUDNN_CALL(cudnnBackendCreateDescriptor(type, &d));\n  return ret;\n}\n\nvoid SetAttr(const Descriptor& desc, cudnnBackendAttributeName_t name, const Descriptor& val) {\n  auto raw = val.get();\n  CUDNN_CALL(cudnnBackendSetAttribute(desc.get(), name, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &raw));\n}\n\nvoid SetAttr(const Descriptor& desc, cudnnBackendAttributeName_t name, const WeakDescriptor& val) {\n  auto raw = val.get();\n  CUDNN_CALL(cudnnBackendSetAttribute(desc.get(), name, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &raw));\n}\n\nvoid SetAttr(const Descriptor& desc,\n             cudnnBackendAttributeName_t name,\n             const std::vector<Descriptor>& val) {\n  std::vector<cudnnBackendDescriptor_t> raw(val.size());\n  std::transform(val.begin(), val.end(), raw.begin(), [](const Descriptor& d) { return d.get(); });\n  CUDNN_CALL(cudnnBackendSetAttribute(\n      desc.get(), name, CUDNN_TYPE_BACKEND_DESCRIPTOR, raw.size(), raw.data()));\n}\n\nDescriptor GetAttr(const Descriptor& desc,\n                   cudnnBackendAttributeName_t name,\n                   cudnnBackendDescriptorType_t type) {\n  cudnnBackendDescriptor_t ret{};\n  CUDNN_CALL(cudnnBackendCreateDescriptor(type, &ret));\n  int64_t count = 0;\n  CUDNN_CALL(\n      cudnnBackendGetAttribute(desc.get(), name, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &count, &ret));\n  CHECK_EQ(count, 1);\n  return Descriptor(ret);\n}\n\nstd::vector<Descriptor> GetAllAttrs(const Descriptor& desc,\n                                    cudnnBackendAttributeName_t name,\n                                    cudnnBackendDescriptorType_t type) {\n  int64_t count = 0;\n  CUDNN_CALL(cudnnBackendGetAttribute(\n      desc.get(), name, CUDNN_TYPE_BACKEND_DESCRIPTOR, 0, &count, nullptr));\n  auto raw = MakeRawDescriptors(count, type);\n  CUDNN_CALL(cudnnBackendGetAttribute(\n      desc.get(), name, CUDNN_TYPE_BACKEND_DESCRIPTOR, raw.size(), &count, raw.data()));\n\n  CHECK_LE(count, raw.size());\n  std::vector<Descriptor> ret(raw.begin(), raw.begin() + count);\n  for (size_t i = count; i < raw.size(); ++i)\n    CUDNN_CALL(cudnnBackendDestroyDescriptor(raw[i]));\n  return ret;\n}\n\nstd::vector<Descriptor> GetSomeAttrs(size_t max_n,\n                                     const Descriptor& desc,\n                                     cudnnBackendAttributeName_t name,\n                                     cudnnBackendDescriptorType_t type) {\n  auto raw      = MakeRawDescriptors(max_n, type);\n  int64_t count = 0;\n  CUDNN_CALL(cudnnBackendGetAttribute(\n      desc.get(), name, CUDNN_TYPE_BACKEND_DESCRIPTOR, raw.size(), &count, raw.data()));\n  std::vector<Descriptor> ret(count);\n  size_t i = 0;\n  for (; i < count; ++i)\n    ret[i] = Descriptor(raw[i]);\n  for (; i < max_n; ++i)\n    CUDNN_CALL(cudnnBackendDestroyDescriptor(raw[i]));\n  return ret;\n}\n\nstd::vector<Descriptor> GetPlans(cudnnBackendHeurMode_t h_mode,\n                                 cudnnHandle_t handle,\n                                 const Descriptor& op_graph,\n                                 size_t workspace_limit,\n                                 size_t* max_workspace,\n                                 const std::unordered_set<int64_t>& excl_engines,\n                                 const std::vector<cudnnBackendNumericalNote_t>& req_numeric,\n                                 const std::vector<cudnnBackendNumericalNote_t>& excl_numeric,\n#if CUDNN_VERSION >= 8200\n                                 const std::vector<cudnnBackendBehaviorNote_t>& req_behavior,\n                                 const std::vector<cudnnBackendBehaviorNote_t>& excl_behavior,\n#endif  // CUDNN_VERSION >= 8200\n                                 bool verbose_filter) {\n  auto heur = MakeFinalized(CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR,\n                            CUDNN_ATTR_ENGINEHEUR_OPERATION_GRAPH,\n                            op_graph,\n                            CUDNN_ATTR_ENGINEHEUR_MODE,\n                            h_mode);\n  auto cfgs = GetAllAttrs(heur, CUDNN_ATTR_ENGINEHEUR_RESULTS, CUDNN_BACKEND_ENGINECFG_DESCRIPTOR);\n  std::vector<Descriptor> plans;\n  if (max_workspace)\n    *max_workspace = 0;\n  for (const auto& cfg : cfgs) {\n    auto plan = Make(CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR,\n                     CUDNN_ATTR_EXECUTION_PLAN_HANDLE,\n                     handle,\n                     CUDNN_ATTR_EXECUTION_PLAN_ENGINE_CONFIG,\n                     cfg);\n    auto err  = cudnnBackendFinalize(plan.get());\n    if (err == CUDNN_STATUS_NOT_SUPPORTED || err == CUDNN_STATUS_ARCH_MISMATCH)\n      continue;\n    if (err != CUDNN_STATUS_SUCCESS) {\n      LOG(WARNING) << \"Unexpected cuDNN status: \" << err << \": \" << cudnnGetErrorString(err);\n      continue;\n    }\n    auto workspace = GetAttr<int64_t>(plan, CUDNN_ATTR_EXECUTION_PLAN_WORKSPACE_SIZE);\n    if (workspace_limit < workspace) {\n      if (verbose_filter)\n        LOG(INFO) << \"   Plan \" << PlanStr(plan) << \" exceeds workspace limit\";\n      continue;\n    }\n    auto engine = GetAttr(cfg, CUDNN_ATTR_ENGINECFG_ENGINE, CUDNN_BACKEND_ENGINE_DESCRIPTOR);\n    if (excl_engines.count(GetAttr<int64_t>(engine, CUDNN_ATTR_ENGINE_GLOBAL_INDEX))) {\n      if (verbose_filter)\n        LOG(INFO) << \"   Plan \" << PlanStr(plan) << \" excluded by engine\";\n      continue;\n    }\n    auto numerical = GetSomeAttrs<cudnnBackendNumericalNote_t>(\n        CUDNN_NUMERICAL_NOTE_TYPE_COUNT, engine, CUDNN_ATTR_ENGINE_NUMERICAL_NOTE);\n    if (!IsCompatible(numerical, req_numeric, excl_numeric)) {\n      if (verbose_filter)\n        LOG(INFO) << \"   Plan \" << PlanStr(plan) << \" has incompatible numerics\";\n      continue;\n    }\n#if CUDNN_VERSION >= 8200\n    auto behavior = GetSomeAttrs<cudnnBackendBehaviorNote_t>(\n        CUDNN_BEHAVIOR_NOTE_TYPE_COUNT, engine, CUDNN_ATTR_ENGINE_BEHAVIOR_NOTE);\n    if (!IsCompatible(behavior, req_behavior, excl_behavior)) {\n      if (verbose_filter)\n        LOG(INFO) << \"   Plan \" << PlanStr(plan) << \" has incompatible behavior\";\n      continue;\n    }\n#endif  // CUDNN_VERSION >= 8200\n    plans.push_back(std::move(plan));\n    if (max_workspace)\n      *max_workspace = std::max(*max_workspace, static_cast<size_t>(workspace));\n  }\n  return plans;\n}\n\n#if !defined(__CUDACC__)  // Can be removed when CUDA 10 support is dropped.\n\nSampler MakeAvgSampler(size_t n, float max_cutoff_msec, size_t warmups) {\n  size_t warmups_performed = 0;\n  size_t k                 = 0;\n  float s                  = 0.0f;\n  if (n < 1)\n    n = 1;\n\n  return [n, max_cutoff_msec, warmups, warmups_performed, k, s](float x) mutable {\n    if (warmups_performed < warmups && x < max_cutoff_msec) {\n      warmups_performed++;\n    } else {\n      // Add this sample to the average calculation\n      s += x;\n      k++;\n    }\n    bool keep_going = k < n && x < max_cutoff_msec;\n    return keep_going ? std::nullopt : std::optional(s / k);\n  };\n}\n\nstd::vector<FindResult> FindTopPlans(std::vector<Descriptor>&& plans,\n                                     size_t max_results,\n                                     cudnnHandle_t handle,\n                                     const Descriptor& var_pack,\n                                     Sampler sampler) {\n  // We're about to perform kernel timings, so we need to quiet the system by grabbing\n  // the Storage lock.  Concurrent cudaMalloc's can disrupt the accurate timing\n  // measurements of the algos, and can prevent the cuda driver's proper freeing\n  // of temporary workspace allocations.  Grabbing the lock might also\n  // impede other threads from launching work on the GPU.\n  std::lock_guard<std::mutex> lock(Storage::Get()->GetMutex(Context::kGPU));\n  std::array<cudaEvent_t, 2> ev;\n  for (auto& ee : ev)\n    CUDA_CALL(cudaEventCreate(&ee));\n  auto cmp = [](const FindResult& lhs, const FindResult& rhs) { return lhs.time < rhs.time; };\n  cudaStream_t stream{};\n  CUDNN_CALL(cudnnGetStream(handle, &stream));\n  std::vector<FindResult> h;\n  for (size_t i = 0; i < plans.size(); ++i) {\n    auto&& plan = plans[i];\n    // Make a copy of the unused sampler for each plan's timing.  Timed warm-up\n    // runs are handled by the sampler to enable early loop exit for slow kernels.\n    auto sampler_copy = sampler;\n    for (;;) {\n      CUDA_CALL(cudaEventRecord(ev[0], stream));\n      CUDNN_CALL(cudnnBackendExecute(handle, plan.get(), var_pack.get()));\n      CUDA_CALL(cudaEventRecord(ev[1], stream));\n      CUDA_CALL(cudaEventSynchronize(ev[1]));\n      float t = 0.0f;\n      CUDA_CALL(cudaEventElapsedTime(&t, ev[0], ev[1]));\n      if (auto r = sampler_copy(t); r) {\n        auto time_to_record = r.value();\n        if (h.size() == max_results) {\n          if (time_to_record < h[0].time) {\n            std::pop_heap(h.begin(), h.end(), cmp);\n            h.back() = {std::move(plan), i, time_to_record};\n            std::push_heap(h.begin(), h.end(), cmp);\n          }\n        } else {\n          h.push_back({std::move(plan), i, time_to_record});\n          std::push_heap(h.begin(), h.end(), cmp);\n        }\n        break;\n      }\n    }\n  }\n  for (auto& ee : ev)\n    CUDA_CALL(cudaEventDestroy(ee));\n  std::sort_heap(h.begin(), h.end(), cmp);\n  return h;\n}\n\n#endif  // !defined(__CUDACC__)\n\nstd::string NoteStr(cudnnBackendNumericalNote_t note) {\n  std::unordered_map<cudnnBackendNumericalNote_t, std::string> m{\n      {CUDNN_NUMERICAL_NOTE_TENSOR_CORE, \"tc\"},\n      {CUDNN_NUMERICAL_NOTE_DOWN_CONVERT_INPUTS, \"dci\"},\n      {CUDNN_NUMERICAL_NOTE_REDUCED_PRECISION_REDUCTION, \"rp\"},\n      {CUDNN_NUMERICAL_NOTE_FFT, \"fft\"},\n      {CUDNN_NUMERICAL_NOTE_NONDETERMINISTIC, \"nd\"},\n      {CUDNN_NUMERICAL_NOTE_WINOGRAD, \"w\"},\n  };\n  auto it = m.find(note);\n  return it != m.end() ? it->second : std::to_string(note);\n}\n\nstd::string KnobStr(cudnnBackendKnobType_t knob) {\n  std::unordered_map<cudnnBackendKnobType_t, std::string> m {\n    {CUDNN_KNOB_TYPE_SPLIT_K, \"split_k\"}, {CUDNN_KNOB_TYPE_SWIZZLE, \"swizzle\"},\n        {CUDNN_KNOB_TYPE_TILE_SIZE, \"tile_size\"}, {CUDNN_KNOB_TYPE_USE_TEX, \"use_tex\"},\n        {CUDNN_KNOB_TYPE_EDGE, \"edge\"}, {CUDNN_KNOB_TYPE_KBLOCK, \"kblock\"},\n        {CUDNN_KNOB_TYPE_LDGA, \"ldga\"}, {CUDNN_KNOB_TYPE_LDGB, \"ldgb\"},\n        {CUDNN_KNOB_TYPE_CHUNK_K, \"chunk_k\"}, {CUDNN_KNOB_TYPE_SPLIT_H, \"split_h\"},\n        {CUDNN_KNOB_TYPE_WINO_TILE, \"wino_tile\"}, {CUDNN_KNOB_TYPE_MULTIPLY, \"multiply\"},\n        {CUDNN_KNOB_TYPE_SPLIT_K_BUF, \"split_k_buf\"}, {CUDNN_KNOB_TYPE_TILEK, \"tilek\"},\n        {CUDNN_KNOB_TYPE_STAGES, \"stages\"}, {CUDNN_KNOB_TYPE_REDUCTION_MODE, \"reduction_mode\"},\n        {CUDNN_KNOB_TYPE_CTA_SPLIT_K_MODE, \"cta_split_k_mode\"},\n        {CUDNN_KNOB_TYPE_SPLIT_K_SLC, \"split_k_slc\"}, {CUDNN_KNOB_TYPE_IDX_MODE, \"idx_mode\"},\n        {CUDNN_KNOB_TYPE_SLICED, \"sliced\"}, {CUDNN_KNOB_TYPE_SPLIT_RS, \"split_rs\"},\n        {CUDNN_KNOB_TYPE_SINGLEBUFFER, \"singlebuffer\"}, {CUDNN_KNOB_TYPE_LDGC, \"ldgc\"},\n        {CUDNN_KNOB_TYPE_SPECFILT, \"specfilt\"},\n#if CUDNN_VERSION >= 8100\n        {CUDNN_KNOB_TYPE_KERNEL_CFG, \"kernel_cfg\"},\n#endif  // CUDNN_VERSION >= 8100\n  };\n  auto it = m.find(knob);\n  return it != m.end() ? it->second : std::to_string(knob);\n}\n\nstd::string PlanStr(const Descriptor& plan) {\n  auto wks = GetAttr<int64_t>(plan, CUDNN_ATTR_EXECUTION_PLAN_WORKSPACE_SIZE);\n  auto cfg =\n      GetAttr(plan, CUDNN_ATTR_EXECUTION_PLAN_ENGINE_CONFIG, CUDNN_BACKEND_ENGINECFG_DESCRIPTOR);\n  auto engine     = GetAttr(cfg, CUDNN_ATTR_ENGINECFG_ENGINE, CUDNN_BACKEND_ENGINE_DESCRIPTOR);\n  auto engine_idx = GetAttr<int64_t>(engine, CUDNN_ATTR_ENGINE_GLOBAL_INDEX);\n  std::ostringstream ss;\n  ss << \"eng:\" << engine_idx << \" wksp:\" << wks;\n  auto notes = GetSomeAttrs<cudnnBackendNumericalNote_t>(\n      CUDNN_NUMERICAL_NOTE_TYPE_COUNT, engine, CUDNN_ATTR_ENGINE_NUMERICAL_NOTE);\n  for (auto note : notes)\n    ss << \" \" << NoteStr(note);\n  auto choices = GetSomeAttrs(CUDNN_KNOB_TYPE_COUNTS,\n                              cfg,\n                              CUDNN_ATTR_ENGINECFG_KNOB_CHOICES,\n                              CUDNN_BACKEND_KNOB_CHOICE_DESCRIPTOR);\n  for (const auto& choice : choices) {\n    auto type = GetAttr<cudnnBackendKnobType_t>(choice, CUDNN_ATTR_KNOB_CHOICE_KNOB_TYPE);\n    auto val  = GetAttr<int64_t>(choice, CUDNN_ATTR_KNOB_CHOICE_KNOB_VALUE);\n    ss << \" \" << KnobStr(type) << \":\" << val;\n  }\n  return ss.str();\n}\n\n}  // namespace cudnn_cxx\n}  // namespace mxnet\n\n#endif  // MXNET_USE_CUDNN == 1\n"
  },
  {
    "path": "src/common/cuda/cudnn_cxx.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file cudnn_cxx.h\n * \\brief Convenience utilities to make coding against cuDNN v8 API less verbose\n */\n#ifndef MXNET_COMMON_CUDA_CUDNN_CXX_H_\n#define MXNET_COMMON_CUDA_CUDNN_CXX_H_\n\n#include <mxnet/base.h>\n#if MXNET_USE_CUDNN == 1\n\n#include <array>\n#include <cstddef>\n#include <cstdint>\n#include <functional>\n#include <memory>\n\n#if !defined(__CUDACC__)  // Can be removed when CUDA 10 support is dropped.\n#include <optional>       // NOLINT(build/include_order)\n#endif                    // !defined(__CUDACC__)\n\n#include <string>\n#include <unordered_set>\n#include <utility>\n#include <vector>\n\n#include \"utils.h\"\n\nSTATIC_ASSERT_CUDNN_VERSION_GE(8002);\n\nnamespace mxnet {\nnamespace cudnn_cxx {\n\nstruct DescriptorDestroyer {\n  using pointer = cudnnBackendDescriptor_t;\n\n  void operator()(cudnnBackendDescriptor_t desc) {\n    CUDNN_CALL_NONFATAL(cudnnBackendDestroyDescriptor(desc));\n  }\n};\n\nusing Descriptor = std::unique_ptr<cudnnBackendDescriptor_t, DescriptorDestroyer>;\n\nstruct WeakDescriptor {\n  cudnnBackendDescriptor_t desc = nullptr;\n\n  explicit WeakDescriptor(const Descriptor& other) : desc(other.get()) {}\n  cudnnBackendDescriptor_t get() const {\n    return desc;\n  }\n};\n\ntemplate <typename T>\nstruct AttrType;\n\ntemplate <>\nstruct AttrType<int64_t> {\n  static constexpr cudnnBackendAttributeType_t type = CUDNN_TYPE_INT64;\n};\n\ntemplate <>\nstruct AttrType<void*> {\n  static constexpr cudnnBackendAttributeType_t type = CUDNN_TYPE_VOID_PTR;\n};\n\ntemplate <>\nstruct AttrType<float> {\n  static constexpr cudnnBackendAttributeType_t type = CUDNN_TYPE_FLOAT;\n};\n\ntemplate <>\nstruct AttrType<double> {\n  static constexpr cudnnBackendAttributeType_t type = CUDNN_TYPE_DOUBLE;\n};\n\ntemplate <>\nstruct AttrType<cudnnHandle_t> {\n  static constexpr cudnnBackendAttributeType_t type = CUDNN_TYPE_HANDLE;\n};\n\ntemplate <>\nstruct AttrType<bool> {\n  static constexpr cudnnBackendAttributeType_t type = CUDNN_TYPE_BOOLEAN;\n};\n\ntemplate <>\nstruct AttrType<cudnnDataType_t> {\n  static constexpr cudnnBackendAttributeType_t type = CUDNN_TYPE_DATA_TYPE;\n};\n\ntemplate <>\nstruct AttrType<cudnnConvolutionMode_t> {\n  static constexpr cudnnBackendAttributeType_t type = CUDNN_TYPE_CONVOLUTION_MODE;\n};\n\ntemplate <>\nstruct AttrType<cudnnNanPropagation_t> {\n  static constexpr cudnnBackendAttributeType_t type = CUDNN_TYPE_NAN_PROPOGATION;\n};\n\ntemplate <>\nstruct AttrType<cudnnPointwiseMode_t> {\n  static constexpr cudnnBackendAttributeType_t type = CUDNN_TYPE_POINTWISE_MODE;\n};\n\ntemplate <>\nstruct AttrType<cudnnBackendHeurMode_t> {\n  static constexpr cudnnBackendAttributeType_t type = CUDNN_TYPE_HEUR_MODE;\n};\n\ntemplate <>\nstruct AttrType<cudnnBackendNumericalNote_t> {\n  static constexpr cudnnBackendAttributeType_t type = CUDNN_TYPE_NUMERICAL_NOTE;\n};\n\n#if CUDNN_VERSION >= 8100\ntemplate <>\nstruct AttrType<cudnnReduceTensorOp_t> {\n  static constexpr cudnnBackendAttributeType_t type = CUDNN_TYPE_REDUCTION_OPERATOR_TYPE;\n};\n#if CUDNN_VERSION >= 8200\ntemplate <>\nstruct AttrType<cudnnBackendBehaviorNote_t> {\n  static constexpr cudnnBackendAttributeType_t type = CUDNN_TYPE_BEHAVIOR_NOTE;\n};\n#endif  // CUDNN_VERSION >= 8200\n#endif  // CUDNN_VERSION >= 8100\n\ntemplate <>\nstruct AttrType<cudnnBackendKnobType_t> {\n  static constexpr cudnnBackendAttributeType_t type = CUDNN_TYPE_KNOB_TYPE;\n};\n\nvoid SetAttr(const Descriptor& desc, cudnnBackendAttributeName_t name, const Descriptor& val);\nvoid SetAttr(const Descriptor& desc, cudnnBackendAttributeName_t name, const WeakDescriptor& val);\nvoid SetAttr(const Descriptor& desc,\n             cudnnBackendAttributeName_t name,\n             const std::vector<Descriptor>& val);\n\ntemplate <typename T>\nvoid SetAttr(const Descriptor& desc, cudnnBackendAttributeName_t name, T val) {\n  CUDNN_CALL(cudnnBackendSetAttribute(desc.get(), name, AttrType<T>::type, 1, &val));\n}\n\ntemplate <typename T>\nvoid SetAttr(const Descriptor& desc, cudnnBackendAttributeName_t name, const std::vector<T>& val) {\n  CUDNN_CALL(cudnnBackendSetAttribute(desc.get(), name, AttrType<T>::type, val.size(), val.data()));\n}\n\ntemplate <typename T, size_t N>\nvoid SetAttr(const Descriptor& desc,\n             cudnnBackendAttributeName_t name,\n             const std::array<T, N>& val) {\n  CUDNN_CALL(cudnnBackendSetAttribute(desc.get(), name, AttrType<T>::type, val.size(), val.data()));\n}\n\ninline void SetAttrs(const Descriptor& desc) {}\n\ntemplate <typename T, typename... Attrs>\nvoid SetAttrs(const Descriptor& desc, cudnnBackendAttributeName_t name, T&& val, Attrs&&... rest) {\n  SetAttr(desc, name, std::forward<T>(val));\n  SetAttrs(desc, std::forward<Attrs>(rest)...);\n}\n\nstd::vector<cudnnBackendDescriptor_t> MakeRawDescriptors(size_t n,\n                                                         cudnnBackendDescriptorType_t type);\n\nDescriptor Make(cudnnBackendDescriptorType_t type);\n\ntemplate <typename... Attrs>\nDescriptor Make(cudnnBackendDescriptorType_t type, Attrs&&... attrs) {\n  auto desc = Make(type);\n  SetAttrs(desc, std::forward<Attrs>(attrs)...);\n  return desc;\n}\n\ntemplate <typename... Attrs>\nDescriptor MakeFinalized(cudnnBackendDescriptorType_t type, Attrs&&... attrs) {\n  auto desc = Make(type, std::forward<Attrs>(attrs)...);\n  CUDNN_CALL(cudnnBackendFinalize(desc.get()));\n  return desc;\n}\n\ntemplate <typename T>\nT GetAttr(const Descriptor& desc, cudnnBackendAttributeName_t name) {\n  T ret{};\n  int64_t ret_count = 0;\n  CUDNN_CALL(cudnnBackendGetAttribute(desc.get(), name, AttrType<T>::type, 1, &ret_count, &ret));\n  CHECK_EQ(ret_count, 1);\n  return ret;\n}\n\ntemplate <typename T>\nstd::vector<T> GetAllAttrs(const Descriptor& desc, cudnnBackendAttributeName_t name) {\n  int64_t count = 0;\n  CUDNN_CALL(cudnnBackendGetAttribute(desc.get(), name, AttrType<T>::type, 0, &count, nullptr));\n  std::vector<T> ret(count);\n  CUDNN_CALL(cudnnBackendGetAttribute(\n      desc.get(), name, AttrType<T>::type, ret.size(), &count, ret.data()));\n  return ret;\n}\n\ntemplate <typename T>\nstd::vector<T> GetSomeAttrs(size_t max_n,\n                            const Descriptor& desc,\n                            cudnnBackendAttributeName_t name) {\n  int64_t count = 0;\n  std::vector<T> ret(max_n);\n  CUDNN_CALL(cudnnBackendGetAttribute(\n      desc.get(), name, AttrType<T>::type, ret.size(), &count, ret.data()));\n  ret.resize(count);\n  return ret;\n}\n\nDescriptor GetAttr(const Descriptor& desc,\n                   cudnnBackendAttributeName_t name,\n                   cudnnBackendDescriptorType_t type);\n\nstd::vector<Descriptor> GetAllAttrs(const Descriptor& desc,\n                                    cudnnBackendAttributeName_t name,\n                                    cudnnBackendDescriptorType_t type);\n\nstd::vector<Descriptor> GetSomeAttrs(size_t max_n,\n                                     const Descriptor& desc,\n                                     cudnnBackendAttributeName_t name,\n                                     cudnnBackendDescriptorType_t type);\n\n// Order sets layout, as a permutation of dims, with N,C,<spacial dims> being identity.\ntemplate <typename T>\nstd::vector<T> PackedStrides(const std::vector<size_t>& order, const std::vector<T>& dims) {\n  CHECK_EQ(order.size(), dims.size());\n  std::vector<T> ret(dims.size(), 1);\n  for (size_t i = dims.size() - 1; i--;)\n    ret[order[i]] = dims[order[i + 1]] * ret[order[i + 1]];\n  return ret;\n}\n\n// Given an engine config's `notes`, return whether that config is compatible, i.e. does\n// the config have all of the required notes and none of the notes that are being excluded.\ntemplate <typename Note>\ninline bool IsCompatible(const std::vector<Note>& notes,\n                         const std::vector<Note>& require_notes,\n                         const std::vector<Note>& exclude_notes) {\n  for (auto rn : require_notes) {\n    auto it = std::find(notes.begin(), notes.end(), rn);\n    if (it == notes.end())\n      return false;\n  }\n  for (auto en : exclude_notes) {\n    auto it = std::find(notes.begin(), notes.end(), en);\n    if (it != notes.end())\n      return false;\n  }\n  return true;\n}\n\n// Execution plans are returned in the order of cuDNN heurstics, i.e. from best to worst.\n// - max_workspace is an out parameter - the maximum workspace requirement among returned plans,\n//   may be nullptr if not needed.\nstd::vector<Descriptor> GetPlans(cudnnBackendHeurMode_t h_mode,\n                                 cudnnHandle_t handle,\n                                 const Descriptor& op_graph,\n                                 size_t workspace_limit,\n                                 size_t* max_workspace,\n                                 const std::unordered_set<int64_t>& excl_engines,\n                                 const std::vector<cudnnBackendNumericalNote_t>& req_numeric,\n                                 const std::vector<cudnnBackendNumericalNote_t>& excl_numeric,\n#if CUDNN_VERSION >= 8200\n                                 const std::vector<cudnnBackendBehaviorNote_t>& req_behavior,\n                                 const std::vector<cudnnBackendBehaviorNote_t>& excl_behavior,\n#endif  // CUDNN_VERSION >= 8200\n                                 bool verbose_filter);\n\n#if !defined(__CUDACC__)  // Can be removed when CUDA 10 support is dropped.\n\n// Defines a sampling algorithm.\n// Returns an aggregate value, to be used as a metric for time comparison, or std::nullopt to\n// perform another time measurement.\nusing Sampler = std::function<std::optional<float>(float)>;\n\n// Return a sampler that after `n` trials returns the average.\n// Before tallying trials, `warmups` trials are first ignored.\n// If ever a trial that exceeds `max_cutoff_msec` is encountered (even during warmup),\n// that trial is tallied and the sampling ends with the then-current trial average.\nSampler MakeAvgSampler(size_t n, float max_cutoff_msec = 1000.0, size_t warmups = 1);\n\nstruct FindResult {\n  Descriptor plan;\n  size_t heur_i;\n  float time;\n};\n\n// Executes and times the plans. The results are returned in the order from best to worst.\nstd::vector<FindResult> FindTopPlans(std::vector<Descriptor>&& plans,\n                                     size_t max_results,\n                                     cudnnHandle_t handle,\n                                     const Descriptor& var_pack,\n                                     Sampler sampler);\n#endif  // !defined(__CUDACC__)\n\nstd::string PlanStr(const Descriptor& plan);\n\n}  // namespace cudnn_cxx\n}  // namespace mxnet\n\n#endif  // MXNET_USE_CUDNN == 1\n\n#endif  //  MXNET_COMMON_CUDA_CUDNN_CXX_H_\n"
  },
  {
    "path": "src/common/cuda/nvtx.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_COMMON_CUDA_NVTX_H_\n#define MXNET_COMMON_CUDA_NVTX_H_\n\n#if MXNET_USE_CUDA && MXNET_USE_NVTX\n#include <cuda.h>\n#include <cuda_runtime.h>\n#include <nvToolsExtCuda.h>\n#include <vector>\n#include <string>\n#include <cstring>\n\nnamespace mxnet {\nnamespace common {\nnamespace cuda {\n\nclass NVTXDuration {\n public:\n  explicit NVTXDuration(const char* name) noexcept : range_id_(0), name_(name) {}\n\n  inline void start() {\n    range_id_ = nvtxRangeStartA(name_);\n  }\n\n  inline void stop() {\n    nvtxRangeEnd(range_id_);\n  }\n\n private:\n  nvtxRangeId_t range_id_;\n  const char* name_;\n};\n\n// Utility class for NVTX\nclass nvtx {\n public:\n  // Palette of colors (make sure to add new colors to the vector in nameToColor()).\n  static const uint32_t kRed     = 0xFF0000;\n  static const uint32_t kGreen   = 0x00FF00;\n  static const uint32_t kBlue    = 0x0000FF;\n  static const uint32_t kYellow  = 0xB58900;\n  static const uint32_t kOrange  = 0xCB4B16;\n  static const uint32_t kRed1    = 0xDC322F;\n  static const uint32_t kMagenta = 0xD33682;\n  static const uint32_t kViolet  = 0x6C71C4;\n  static const uint32_t kBlue1   = 0x268BD2;\n  static const uint32_t kCyan    = 0x2AA198;\n  static const uint32_t kGreen1  = 0x859900;\n\n  static void gpuRangeStart(const uint32_t rgb, const std::string& range_name) {\n    nvtxEventAttributes_t att;\n    att.version       = NVTX_VERSION;\n    att.size          = NVTX_EVENT_ATTRIB_STRUCT_SIZE;\n    att.colorType     = NVTX_COLOR_ARGB;\n    att.color         = rgb | 0xff000000;\n    att.messageType   = NVTX_MESSAGE_TYPE_ASCII;\n    att.message.ascii = range_name.c_str();\n    nvtxRangePushEx(&att);\n  }\n\n  // Utility to map a range name prefix to a random color based on its hash\n  static uint32_t nameToColor(const std::string& range_name, int prefix_len) {\n    static std::vector<uint32_t> colors{\n        kRed, kGreen, kBlue, kYellow, kOrange, kRed1, kMagenta, kViolet, kBlue1, kCyan, kGreen1};\n    std::string s(range_name, 0, prefix_len);\n    std::hash<std::string> hash_fn;\n    return colors[hash_fn(s) % colors.size()];\n  }\n\n  // Utility to map a range name to a random color based on its hash\n  static uint32_t nameToColor(const std::string& range_name) {\n    return nameToColor(range_name, range_name.size());\n  }\n\n  static void gpuRangeStop() {\n    nvtxRangePop();\n  }\n};\n\n}  // namespace cuda\n}  // namespace common\n}  // namespace mxnet\n\n#endif  // MXNET_UDE_CUDA && MXNET_USE_NVTX\n#endif  // MXNET_COMMON_CUDA_NVTX_H_\n"
  },
  {
    "path": "src/common/cuda/rtc/backward_functions-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_COMMON_CUDA_RTC_BACKWARD_FUNCTIONS_INL_H_\n#define MXNET_COMMON_CUDA_RTC_BACKWARD_FUNCTIONS_INL_H_\n\n#if MXNET_USE_CUDA\n\nnamespace mxnet {\nnamespace common {\nnamespace cuda {\nnamespace rtc {\n\nconst char backward_function_definitions[] = R\"code(\n\nnamespace op {\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_relu(const DTypeGrad grad, const DType val) {\n  if (isnan(val)) return val;\n  return val > 0 ? grad : 0;\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_sigmoid(const DTypeGrad grad, const DType val) {\n  return grad * val * (1 - val);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_log_sigmoid(const DTypeGrad grad, const DType val) {\n  return grad * (1 - op::exp(val));\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_softrelu(const DTypeGrad grad, const DType val) {\n  const mixed_type<DTypeGrad, DType> v = val;\n  return grad * sigmoid(v);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_softsign(const DTypeGrad grad, const DType val) {\n  const mixed_type<DTypeGrad, DType> v = val;\n  const auto ap1 = 1 + op::abs(v);\n  return grad / (ap1 * ap1);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_abs(const DTypeGrad grad, const DType val) {\n  const mixed_type<DTypeGrad, DType> v = val;\n  return grad * op::sign(v);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_exp(const DTypeGrad grad, const DType val) {\n  const mixed_type<DTypeGrad, DType> v = val;\n  return grad * op::exp(v);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_expm1(const DTypeGrad grad, const DType val) {\n  return backward_exp(grad, val);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_log(const DTypeGrad grad, const DType val) {\n  return grad / val;\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_log10(const DTypeGrad grad, const DType val) {\n  return grad / (val * op::log(static_cast<DTypeGrad>(10)));\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_log2(const DTypeGrad grad, const DType val) {\n  return grad / (val * op::log(static_cast<DTypeGrad>(2)));\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_log1p(const DTypeGrad grad, const DType val) {\n  return grad / (1 + val);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_sin(const DTypeGrad grad, const DType val) {\n  const mixed_type<DTypeGrad, DType> v = val;\n  return grad * op::cos(v);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_cos(const DTypeGrad grad, const DType val) {\n  const mixed_type<DTypeGrad, DType> v = val;\n  return -grad * op::sin(v);\n}\n\n// Uses output from tan\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_tan(const DTypeGrad grad, const DType out) {\n  return grad * (out * out + 1);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_arcsin(const DTypeGrad grad, const DType val) {\n  const mixed_type<DTypeGrad, DType> v = val;\n  return grad / op::sqrt(1 - v*v);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_arccos(const DTypeGrad grad, const DType val) {\n  const mixed_type<DTypeGrad, DType> v = val;\n  return -grad / op::sqrt(1 - v*v);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_arctan(const DTypeGrad grad, const DType val) {\n  return grad / (1 + val*val);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_degrees(const DTypeGrad grad, const DType /* val */) {\n  return op::degrees(grad);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_radians(const DTypeGrad grad, const DType /* val */) {\n  return op::radians(grad);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_sinh(const DTypeGrad grad, const DType val) {\n  const mixed_type<DTypeGrad, DType> v = val;\n  return grad * op::cosh(v);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_cosh(const DTypeGrad grad, const DType val) {\n  const mixed_type<DTypeGrad, DType> v = val;\n  return grad * op::sinh(v);\n}\n\n// Uses tanh output\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_tanh(const DTypeGrad grad, const DType out) {\n  return grad * (1 - out * out);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_arcsinh(const DTypeGrad grad, const DType val) {\n  const mixed_type<DTypeGrad, DType> v = val;\n  return grad / op::sqrt(v * v + 1);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_arccosh(const DTypeGrad grad, const DType val) {\n  const mixed_type<DTypeGrad, DType> v = val;\n  return grad / op::sqrt(v * v - 1);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_arctanh(const DTypeGrad grad, const DType val) {\n  return grad / (1 - val * val);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_mish(const DTypeGrad grad, const DType val) {\n  const auto softrelu = op::softrelu(val);\n  const auto tanh_sr = op::tanh(softrelu);\n  return grad * (tanh_sr + val * sigmoid(val) * (1 - tanh_sr * tanh_sr));\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_sqrt(const DTypeGrad grad, const DType out) {\n  return 0.5 * grad / out;\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_rsqrt(const DTypeGrad grad, const DType val) {\n  const mixed_type<DTypeGrad, DType> v = val;\n  const auto inv = 1 / v;\n  return -0.5 * grad * op::sqrt(inv) * inv;\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_cbrt(const DTypeGrad grad, const DType out) {\n  return grad / (3.0f * out * out);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_rcbrt(const DTypeGrad grad, const DType val) {\n  const mixed_type<DTypeGrad, DType> v = val;\n  const auto inv = 1 / v;\n  return -1.f/3.f * grad * op::cbrt(inv) * inv;\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_square(const DTypeGrad grad, const DType val) {\n  return 2 * val * grad;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType div_rgrad(const DType val,\n                                  const DType2 val2) {\n  return -val / (val2 * val2);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_clip(const DTypeGrad grad, const DType val,\n              const float a_min, const float a_max) {\n  if (val > a_max || val < a_min) {\n    return 0;\n  } else {\n    return grad;\n  }\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_reciprocal(const DTypeGrad grad, const DType val) {\n  return -grad / (val * val);\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_erf(const DTypeGrad grad, const DType val) {\n  using type = mixed_type<DTypeGrad, DType>;\n  const type v = val;\n  constexpr type my_pi = pi;\n  return 2.0f / op::sqrt(my_pi) * op::exp(-(v*v)) * grad;\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_erfinv(const DTypeGrad grad, const DType val) {\n  using type = mixed_type<DTypeGrad, DType>;\n  constexpr type my_pi = pi;\n  const type g = grad;\n  const type v = val;\n  return 0.5f * op::sqrt(my_pi) * op::exp(v * v) * g;\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_gamma(const DTypeGrad grad, const DType val) {\n  using type = mixed_type<DTypeGrad, DType>;\n  const type v = val;\n  if (type_util::is_same<DTypeGrad, double>::value) {\n    return grad * op::gamma(v) * op::special_functions::cephes::psi<double>(v);\n  } else {\n    return grad * op::gamma(v) * op::special_functions::cephes::psi<float>(v);\n  }\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_gammaln(const DTypeGrad grad, const DType val) {\n  using type = mixed_type<DTypeGrad, DType>;\n  const type v = val;\n  if (type_util::is_same<DTypeGrad, double>::value) {\n    return grad * op::special_functions::cephes::psi<double>(v);\n  } else {\n    return grad * op::special_functions::cephes::psi<float>(v);\n  }\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_digamma(const DTypeGrad grad, const DType val) {\n  using type = mixed_type<DTypeGrad, DType>;\n  const type v = val;\n  if (type_util::is_same<DTypeGrad, double>::value) {\n    return grad * op::special_functions::trigamma<double>(v);\n  } else {\n    return grad * op::special_functions::trigamma<float>(v);\n  }\n}\n\ntemplate <typename DType, typename DTypeGrad>\n__device__ inline mixed_type<DTypeGrad, DType>\nbackward_gelu_erf(const DTypeGrad grad, const DType val) {\n  return 0.5f * (grad + grad * op::erf(val / op::sqrt(2.0f)) +\n                 val * backward_erf(grad, val / op::sqrt(2.0f)) / op::sqrt(2.0f));\n}\n\n}  // namespace op\n\n)code\";\n\nconst char grad_function_definitions[] = R\"code(\nnamespace op {\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nrdiv_grad(const DType val,\n          const DType2 val2) {\n  return -val2 / (val * val);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\ndiv_grad(const DType val,\n         const DType2 val2) {\n  const mixed_type<DType, DType2> temp = val2;\n  return op::reciprocal(temp);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType mod_grad(const DType val,\n                                 const DType2 val2) {\n  if (type_util::is_integral<DType>::value) {\n    return 0;\n  } else {\n    return 1;\n  }\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType mod_rgrad(const DType val,\n                                  const DType2 val2) {\n  if (type_util::is_integral<DType>::value) {\n    return 0;\n  } else {\n    return -op::floor(val / val2);\n  }\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType rmod_grad(const DType val,\n                                  const DType2 val2) {\n  if (type_util::is_integral<DType>::value) {\n    return 0;\n  } else {\n    return -op::floor(val2 / val);\n  }\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\npower_grad(const DType val,\n           const DType2 val2) {\n  return op::power(val, val2 - 1.f) * val2;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\npower_rgrad(const DType val,\n            const DType2 val2) {\n  const mixed_type<DType, DType2> temp = val;\n  return op::power(val, val2) * op::log(temp);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nrpower_grad(const DType val,\n            const DType2 val2) {\n  const mixed_type<DType, DType2> temp = val2;\n  return val * op::log(temp);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nhypot_grad_left(const DType val,\n                const DType2 val2) {\n  return val / op::hypot(val, val2);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nhypot_grad_right(const DType val,\n                 const DType2 val2) {\n  return val2 / op::hypot(val, val2);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\ncopysign_grad(const DType val,\n              const DType2 val2) {\n  return (val >= 0 && val2 >= 0) || (val < 0 && val2 < 0) ? 1 : -1;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nbitwise_left_shift_grad(const DType val,\n                        const DType2 val2) {\n  return op::power(static_cast<DType>(2), val2);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nbitwise_left_shift_rgrad(const DType val,\n                         const DType2 val2) {\n  using type = mixed_type<DType, DType2>;\n  return val * op::power(static_cast<DType>(2), val2) * op::log(static_cast<type>(2));\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nrbitwise_left_shift_grad(const DType val,\n                         const DType2 val2) {\n  using type = mixed_type<DType, DType2>;\n  return val2 * op::power(static_cast<DType>(2), val) * op::log(static_cast<type>(2));\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nbitwise_right_shift_grad(const DType val,\n                         const DType2 val2) {\n  return op::power(0.5f, val2);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nbitwise_right_shift_rgrad(const DType val,\n                          const DType2 val2) {\n  return val * op::power(0.5f, val2) * op::log(0.5f);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nrbitwise_right_shift_grad(const DType val,\n                          const DType2 val2) {\n  return val2 * op::power(0.5f, val) * op::log(0.5f);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\narctan2_grad(const DType val,\n             const DType2 val2) {\n  return val2 / (val * val + val2 * val2);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nrarctan2_grad(const DType val,\n              const DType2 val2) {\n  return val / (val * val + val2 * val2);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\narctan2_rgrad(const DType val,\n              const DType2 val2) {\n  return -rarctan2_grad(val, val2);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nldexp_grad(const DType val,\n           const DType2 val2) {\n  return op::power(static_cast<DType>(2), val2);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nrldexp_grad(const DType val,\n            const DType2 val2) {\n  using type = mixed_type<DType, DType2>;\n  return val2 * op::power(static_cast<type>(2), val) * op::log(static_cast<type>(2));\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nlogaddexp_grad(const DType val,\n           const DType2 val2) {\n  return op::exp(val) / (op::exp(val) + op::exp(val2));\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nlogaddexp_rgrad(const DType val,\n           const DType2 val2) {\n  return op::exp(val2) / (op::exp(val) + op::exp(val2));\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType smooth_l1_grad(const DType val, const DType2 scalar) {\n  auto bsq = scalar * scalar;\n  auto ibsq = 1.0f / bsq;\n  if (val > ibsq) {\n    return 1;\n  } else if (val < -ibsq) {\n    return -1;\n  } else {\n    return bsq * val;\n  }\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType2 xelu_grad(const DType val,\n                                   const DType2 val2) {\n  return (val > 0) ? 1 : val2;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType prelu_grad(const DType val,\n                                   const DType2 val2) {\n  return (val > 0) ? 0 : val;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType2, DType>\ngamma_implicit_grad(const DType a_in, const DType2 x_in) {\n    using OType = mixed_type<DType2, DType>;\n    const OType a = a_in;\n    const OType x = x_in;\n    if (x < 0.8f) {\n      OType numer = 1;\n      OType denom = a;\n      OType series1 = numer / denom;\n      OType series2 = numer / (denom * denom);\n#pragma unroll\n      for (int i = 1; i <= 5; i++) {\n        numer *= -x / static_cast<DType>(i);\n        denom += 1;\n        series1 += numer / denom;\n        series2 += numer / (denom * denom);\n      }\n      OType pow_x_alpha = op::power(x, a);\n      OType gamma_pdf = op::power(x, a - 1) * op::exp(-x);\n      OType gamma_cdf = pow_x_alpha * series1;\n      OType gamma_cdf_alpha =\n          (op::log(x) - OType(special_functions::cephes::psi<float>(a))) *\n              gamma_cdf -\n          pow_x_alpha * series2;\n      OType result = -gamma_cdf_alpha / gamma_pdf;\n      return op::isnan(result) ? 0.f : result;\n    }\n    if (a > 8.0f) {\n      if (0.9f * a <= x && x <= 1.1f * a) {\n        OType numer_1 = 1 + 24 * a * (1 + 12 * a);\n        OType numer_2 = 1440 * (a * a) + 6 * x * (53 - 120 * x) -\n                        65 * x * x / a + a * (107 + 3600 * x);\n        OType denom = 1244160 * (a * a) * (a * a);\n        return numer_1 * numer_2 / denom;\n      }\n      OType denom = op::sqrt(8 * a);\n      OType term2 = denom / (a - x);\n      OType term3 =\n          op::power(x - a - a * op::log(x / a), static_cast<OType>(-1.5));\n      OType term23 = (x < a) ? term2 - term3 : term2 + term3;\n      OType term1 = op::log(x / a) * term23 -\n                    op::sqrt(2 / a) * (a + x) / ((a - x) * (a - x));\n      OType stirling = 1.f + 1.f / (12.f * a) * (1.f + 1.f / (24.f * a));\n      OType numer = x * term1;\n      return -stirling * numer / denom;\n    }\n    OType u = op::log(x / a);\n    OType v = op::log(a);\n    OType coef_uv[3][8] = {\n        {0.16009398, -0.094634809, 0.025146376, -0.0030648343, 1, 0.32668115,\n         0.10406089, 0.0014179084},\n        {0.53487893, 0.1298071, 0.065735949, -0.0015649758, 0.16639465,\n         0.020070113, -0.0035938915, -0.00058392623},\n        {0.040121004, -0.0065914022, -0.0026286047, -0.0013441777, 0.017050642,\n         -0.0021309326, 0.00085092367, -1.5247877e-07},\n    };\n    OType coef_v[8];\n#pragma unroll\n    for (int i = 0; i < 8; i++) {\n      coef_v[i] = coef_uv[0][i] + u * (coef_uv[1][i] + u * coef_uv[2][i]);\n    }\n    OType p = coef_v[0] + v * (coef_v[1] + v * (coef_v[2] + v * coef_v[3]));\n    OType q = coef_v[4] + v * (coef_v[5] + v * (coef_v[6] + v * coef_v[7]));\n    return op::exp(p / q);\n}\n\n}  // namespace op\n)code\";\n\n}  // namespace rtc\n}  // namespace cuda\n}  // namespace common\n}  // namespace mxnet\n\n#endif  // MXNET_USE_CUDA\n\n#endif  // MXNET_COMMON_CUDA_RTC_BACKWARD_FUNCTIONS_INL_H_\n"
  },
  {
    "path": "src/common/cuda/rtc/forward_functions-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_COMMON_CUDA_RTC_FORWARD_FUNCTIONS_INL_H_\n#define MXNET_COMMON_CUDA_RTC_FORWARD_FUNCTIONS_INL_H_\n\n#if MXNET_USE_CUDA\n\nnamespace mxnet {\nnamespace common {\nnamespace cuda {\nnamespace rtc {\n\nconst char function_definitions_util[] = R\"code(\n\n#define INT_MAX (2147483647)\n\nnamespace op {\nusing type_util::mixed_type;\n\ntemplate <typename DType>\nstruct LoadType {\n  using Type = DType;\n};\n\ntemplate <>\nstruct LoadType<half> {\n  using Type = float;\n};\n\ntemplate <typename DType>\n__device__ inline typename LoadType<DType>::Type load(const DType input) {\n  return input;\n}\n\ntemplate <>\n__device__ inline float load(const half input) {\n  return __half2float(input);\n}\n\ntemplate <typename DType1, typename DType2>\n__device__ inline DType1 store(const DType2 input, DType1* ref) {\n  return input;\n}\n\ntemplate <typename DType>\n__device__ inline half store(const DType input, half* ref) {\n  return __float2half(input);\n}\n\ntemplate <int ndim>\nstruct Shape {\n   int x[ndim];\n   size_t size;\n   __device__ inline const int& operator [](const int i) const {\n       return x[i];\n   }\n   __device__ inline int& operator [](const int i) {\n       return x[i];\n   }\n   __device__ inline void set(const int def) {\n       #pragma unroll\n       for (int i = 0; i < ndim; i++) {\n           x[i] = def;\n       }\n   }\n};\n\ntemplate <>\nstruct Shape<0> {\n   size_t size;\n};\n\ntemplate <int nvec, typename DType, int ndim>\n__device__ inline vector::VectorizedStorage<DType, nvec> load_index(const DType * input, int i,\n                                                                    const Shape<ndim> &shape) {\n  using V = vector::VectorizedStorage<DType, nvec>;\n  if (i < shape.size) {\n    const auto* vector_input = reinterpret_cast<const typename V::LType *>(input + i);\n    return V(*vector_input);\n  } else {\n    return V({0});\n  }\n}\n\ntemplate <int nvec, typename DType, int ndim>\n__device__ inline vector::VectorizedStorage<DType, nvec> global_load_index(const DType * input,\n                    int i, const Shape<ndim> &shape) {\n  using V = vector::VectorizedStorage<DType, nvec>;\n  if (i < shape.size) {\n    const auto* vector_input = reinterpret_cast<const typename V::LType *>(input + i);\n    return V(__ldg(vector_input));\n  } else {\n    return V({0});\n  }\n}\n\ntemplate <int nvec, typename DType, int ndim>\n__device__ inline vector::VectorizedStorage<DType, nvec> load_slice(const DType * input,\n                                                                    const Shape<ndim>& shape,\n                                                                    Shape<ndim> begin,\n                                                                    Shape<ndim> end,\n                                                                    int offset) {\n  int idx[nvec];\n\n  Shape<ndim> ref_strides;\n  Shape<ndim> strides;\n  ref_strides[ndim-1] = 1;\n  strides[ndim-1] = 1;\n  #pragma unroll\n  for (int dim = ndim-1; dim >=0; dim--) {\n    if (begin[dim] < 0) begin[dim] = shape[dim] + begin[dim];\n    if (end[dim] < 0) end[dim] = shape[dim] + end[dim];\n    if (end[dim] == INT_MAX) end[dim] = shape[dim];\n    if (dim > 0) {\n      ref_strides[dim-1] = ref_strides[dim] * (end[dim] - begin[dim]);\n      strides[dim-1] = strides[dim] * shape[dim];\n    }\n  }\n  #pragma unroll\n  for (int j = 0; j < nvec; j++) {\n    idx[j] = 0;\n    int ref_idx = offset + j;\n    #pragma unroll\n    for (int dim = 0; dim < ndim; dim++) {\n       int stride = ref_strides[dim];\n       if (shape[dim] > 1) {\n         idx[j] += (ref_idx / stride + begin[dim]) * strides[dim];\n       }\n       ref_idx = ref_idx % stride;\n    }\n  }\n  vector::VectorizedStorage<DType, nvec> ret;\n  #pragma unroll\n  for (int j = 0; j < nvec; j++) {\n      ret.scratch_.separate[j] = idx[j] < shape.size ? *(input + idx[j]) : DType {};\n  }\n  return ret;\n}\n\ntemplate <int nvec, typename DType, int ndim>\n__device__ inline vector::VectorizedStorage<DType, nvec> fast_load_slice(const DType * input,\n                                                                         const Shape<ndim>& shape,\n                                                                         Shape<ndim> begin,\n                                                                         Shape<ndim> end,\n                                                                         int offset) {\n  int idx = 0;\n\n  Shape<ndim> ref_strides;\n  Shape<ndim> strides;\n  ref_strides[ndim-1] = 1;\n  strides[ndim-1] = 1;\n  #pragma unroll\n  for (int dim = ndim-1; dim >=0; dim--) {\n    if (begin[dim] < 0) begin[dim] = shape[dim] + begin[dim];\n    if (end[dim] < 0) end[dim] = shape[dim] + end[dim];\n    if (end[dim] == INT_MAX) end[dim] = shape[dim];\n    if (dim > 0) {\n      ref_strides[dim-1] = ref_strides[dim] * (end[dim] - begin[dim]);\n      strides[dim-1] = strides[dim] * shape[dim];\n    }\n  }\n  int ref_idx = offset;\n  #pragma unroll\n  for (int dim = 0; dim < ndim; dim++) {\n     int stride = ref_strides[dim];\n     if (shape[dim] > 1) {\n       idx += (ref_idx / stride + begin[dim]) * strides[dim];\n     }\n     ref_idx = ref_idx % stride;\n  }\n  return global_load_index<nvec>(input, idx, shape);\n}\n\ntemplate <int nvec, typename DType, int ndim>\n__device__ inline void store_index(const vector::VectorizedStorage<DType, nvec> value, int i,\n                        DType * output, const Shape<ndim>& shape) {\n  if (i < (shape.size + nvec - 1) / nvec) {\n    auto vector_output = reinterpret_cast<\n                          typename vector::VectorizedStorage<DType, nvec>::LType *>(output);\n    vector_output[i] = value.scratch_.aligned;\n  }\n}\n\ntemplate <int nvec, typename DType, int ndim>\n__device__ inline void store_add_index(const vector::VectorizedStorage<DType, nvec> value, int i,\n                            DType * output, const Shape<ndim>& shape) {\n  if (i < (shape.size + nvec - 1) / nvec) {\n    auto vector_output = reinterpret_cast<\n                          typename vector::VectorizedStorage<DType, nvec>::LType *>(output);\n    vector::VectorizedStorage<DType, nvec> ret(vector_output[i]);\n    ret += value;\n    vector_output[i] = ret.scratch_.aligned;\n  }\n}\n\n}  // namespace op\n)code\";\n\nconst char function_definitions_binary[] = R\"code(\nnamespace op {\n\ntemplate <typename DType>\n__device__ inline bool isnan(const DType val) {\n  return util::isnan(val);\n}\n\ntemplate <typename DType>\n__device__ inline bool_t isinf(const DType val) {\n  return util::isinf(val);\n}\n\ntemplate <typename DType>\n__device__ inline bool_t isposinf(const DType val) {\n  return util::isinf(val) && (val > 0);\n}\n\ntemplate <typename DType>\n__device__ inline bool_t isneginf(const DType val) {\n  return util::isinf(val) && (val < 0);\n}\n\ntemplate <typename DType>\n__device__ inline bool_t isfinite(const DType val) {\n  return !op::isnan(val) && !op::isinf(val);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nadd(const DType a, const DType2 b) {\n  return a + b;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nsub(const DType a, const DType2 b) {\n  return a - b;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nrsub(const DType a, const DType2 b) {\n  return b - a;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nfloor_divide(const DType a, const DType2 b) {\n  if (type_util::has_double_or_integral<DType, DType2>::value) {\n    return ::floor((double)a / (double)b);\n  } else {\n    return ::floorf((float)a / (float)b);\n  }\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nrfloor_divide(const DType a, const DType2 b) {\n  if (type_util::has_double_or_integral<DType, DType2>::value) {\n    return ::floor((double)b / (double)a);\n  } else {\n    return ::floorf((float)b / (float)a);\n  }\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nmul(const DType a, const DType2 b) {\n  return a * b;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\ndiv(const DType a, const DType2 b) {\n  return a / b;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nrdiv(const DType a, const DType2 b) {\n  return b / a;\n}\n\n#define DEFINE_BINARY_MATH_FUNC(name, double_version, float_version) \\\ntemplate <typename DType, typename DType2> \\\n__device__ inline mixed_type<DType, DType2> \\\nname (const DType a, const DType2 b) { \\\n  if (type_util::has_double_or_integral<DType, DType2>::value) { \\\n    return double_version ((double)a, (double)b); \\\n  } else { \\\n    return float_version ((float)a, (float)b); \\\n  } \\\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\npower (const DType a, const DType2 b) {\n  if (type_util::has_double<DType, DType2>::value) {\n    return ::pow ((double)a, (double)b); \\\n  } else {\n    return ::powf ((float)a, (float)b);\n  }\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nrpow(const DType a, const DType2 b) {\n  return power(b, a);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nmax(const DType a, const DType2 b) {\n  if (isnan(a)) return a;\n  return a > b ? a : b;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nfmax(const DType a, const DType2 b) {\n  if (isnan(b)) return a;\n  return a > b ? a : b;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nmin(const DType a, const DType2 b) {\n  if (isnan(a)) return a;\n  return a < b ? a : b;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nfmin(const DType a, const DType2 b) {\n  if (isnan(b)) return a;\n  return a < b ? a : b;\n}\n\nDEFINE_BINARY_MATH_FUNC(hypot, ::hypot, ::hypotf)\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nmod(const DType a, const DType2 b) {\n  if (b == 0) {\n    return 0;\n  }\n  const double ad = static_cast<double>(a);\n  const double bd = static_cast<double>(b);\n  if (bd < 0) {\n    if (ad < 0) {\n      return -::fmod(-ad, -bd);\n    } else {\n      return ::fmod(ad, -bd) +\n             (::fmod(ad, -bd) != 0 ? bd : 0);\n    }\n  } else {\n    if (ad < 0) {\n      return -::fmod(-ad, bd) +\n              (::fmod(-ad, bd) != 0 ? bd : 0);\n    } else {\n      return ::fmod(ad, bd);\n    }\n  }\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nfmod(const DType a, const DType2 b) {\n  if (b == 0) {\n    return 0;\n  }\n  return ::fmod(static_cast<double>(a), static_cast<double>(b));\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nrmod(const DType a, const DType2 b) {\n  return op::mod(b, a);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nrfmod(const DType a, const DType2 b) {\n  return op::fmod(b, a);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType equal(const DType a, const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a == real_b ? 1 : 0;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType not_equal(const DType a, const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a != real_b ? 1 : 0;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType greater(const DType a, const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a > real_b ? 1 : 0;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType greater_equal(const DType a, const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a >= real_b ? 1 : 0;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType less(const DType a, const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a < real_b ? 1 : 0;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType less_equal(const DType a, const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a <= real_b ? 1 : 0;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline bool_t np_equal(const DType a, const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a == real_b ? true : false;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline bool_t np_not_equal(const DType a, const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a != real_b ? true : false;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline bool_t np_greater(const DType a, const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a > real_b ? true : false;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline bool_t np_greater_equal(const DType a, const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a >= real_b ? true : false;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline bool_t np_less(const DType a, const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a < real_b ? true : false;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline bool_t np_less_equal(const DType a, const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a <= real_b ? true : false;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType logical_and(const DType a, const DType2 b) {\n  return a && b ? 1 : 0;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType logical_or(const DType a, const DType2 b) {\n  return a || b ? 1 : 0;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType logical_xor(const DType a, const DType2 b) {\n  return ((a || b) && !(a && b)) ? 1 : 0;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType copysign(const DType a, const DType2 b) {\n  return (a >= 0 && b >= 0) || (a < 0 && b < 0) ? a : -a;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType2 rcopysign(const DType a, const DType2 b) {\n  return copysign(b, a);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nlcm(const DType a, const DType2 b) {\n  if (type_util::is_integral<DType>::value &&\n      type_util::is_integral<DType2>::value) {\n    DType A = a;\n    DType2 B = b;\n    // minus cases.\n    if (a < 0) {\n      A = -a;\n    }\n    if (b < 0) {\n      B = -b;\n    }\n    // handle zero-valued cases.\n    DType c;\n    if (a == 0 || b == 0) {\n      c = 0;\n    } else {\n      DType tmp;\n      DType tmp_a = A;\n      DType tmp_b = B;\n      if (A < B) {\n        tmp = A;\n        A = B;\n        B = tmp;\n      }\n      while (A % B != 0) {\n        A = A % B;\n        tmp = A;\n        A = B;\n        B = tmp;\n      }\n      c = tmp_a / B * tmp_b;\n    }\n    return c;\n  } else {\n    return 0;\n  }\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\ngcd(const DType a, const DType2 b) {\n  if (type_util::is_integral<DType>::value &&\n      type_util::is_integral<DType2>::value) {\n    DType A = a;\n    DType2 B = b;\n    // minus cases.\n    if (a < 0) {\n      A = -a;\n    }\n    if (b < 0) {\n      B = -b;\n    }\n    // handle zero-valued cases.\n    DType c;\n    if (a == 0 && b != 0) {\n      c = B;\n    } else if (b == 0 && a != 0) {\n      c = A;\n    } else if (a == 0 && b == 0) {\n      c = 0;\n    } else {\n      DType tmp;\n      if (A < B) {\n        tmp = A;\n        A = B;\n        B = tmp;\n      }\n      while (A % B != 0) {\n        A = A % B;\n        tmp = A;\n        A = B;\n        B = tmp;\n      }\n      c = B;\n    }\n    return c;\n  } else {\n    return 0;\n  }\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2> bitwise_xor(const DType a,\n                                                                       const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a ^ real_b;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2> bitwise_or(const DType a,\n                                                                       const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a | real_b;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2> bitwise_and(const DType a,\n                                                                       const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a & real_b;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2> bitwise_left_shift(const DType a,\n                                                                              const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a << real_b;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2> rbitwise_left_shift(const DType a,\n                                                                               const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_b << real_a;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2> bitwise_right_shift(const DType a,\n                                                                               const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_a >> real_b;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2> rbitwise_right_shift(const DType a,\n                                                                                const DType2 b) {\n  const mixed_type<DType, DType2> real_a = a;\n  const mixed_type<DType, DType2> real_b = b;\n  return real_b >> real_a;\n}\n\nDEFINE_BINARY_MATH_FUNC(arctan2, ::atan2, ::atan2f)\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nrarctan2(const DType a, const DType2 b) {\n  return arctan2(b, a);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nldexp(const DType a, const DType2 b) {\n  if (type_util::has_double_or_integral<DType, DType2>::value) {\n    return a * ::pow(2.0, static_cast<double>(b));\n  } else {\n    return a * ::powf(2.0f, static_cast<float>(b));\n  }\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nrldexp(const DType a, const DType2 b) {\n  return ldexp(b, a);\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline mixed_type<DType, DType2>\nlogaddexp(const DType a, const DType2 b) {\n  if (type_util::has_double_or_integral<DType, DType2>::value) {\n    return ::log(::exp(static_cast<double>(a)) + ::exp(static_cast<double>(b)));\n  } else {\n    return ::log(::expf(static_cast<float>(a)) + ::expf(static_cast<float>(b)));\n  }\n}\n\n#undef DEFINE_BINARY_MATH_FUNC\n\ntemplate <typename DType, typename DType2>\n__device__ inline bool np_logical_and(const DType val, const DType2 val2) {\n  return (val && val2) ? true : false;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline bool np_logical_or(const DType val, const DType2 val2) {\n  return (val || val2) ? true : false;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline bool np_logical_xor(const DType val, const DType2 val2) {\n  return ((val || val2) && !(val && val2)) ? true : false;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType left(const DType left_val, const DType2 right_val) {\n  return left_val;\n}\n\ntemplate <typename DType, typename DType2>\n__device__ inline DType2 right(const DType left_val, const DType2 right_val) {\n  return right_val;\n}\n\n}  // namespace op\n)code\";\n\nconst char function_definitions_unary[] = R\"code(\nnamespace op {\n\ntemplate <typename DType>\n__device__ inline DType identity(const DType val) {\n  return val;\n}\n\ntemplate <typename DType>\n__device__ inline DType negation(const DType val) {\n  return -val;\n}\n\ntemplate <typename OutType, typename DType>\n__device__ inline typename LoadType<OutType>::Type cast(const DType val) {\n  return static_cast<typename LoadType<OutType>::Type>(val);\n}\n\n// activations\n\ntemplate <typename DType>\n__device__ inline DType relu(const DType val) {\n  return (isnan(val) || val > 0) ? val : 0;\n}\n\ntemplate <typename DType>\n__device__ inline DType sigmoid(const DType val) {\n  if (type_util::has_double_or_integral<DType>::value) {\n    return 1./(1 + ::exp(-val));\n  } else {\n    return 1.f/(1 + expf(-val));\n  }\n}\n\ntemplate <typename DType>\n__device__ inline DType log_sigmoid(const DType val) {\n  if (type_util::has_double_or_integral<DType>::value) {\n    return ::log(1./(1 + ::exp(-val)));\n  } else {\n    return ::logf(1.f/(1 + expf(-val)));\n  }\n}\n\ntemplate <typename DType>\n__device__ inline DType softrelu(const DType val) {\n  // Avoid overflow of exp for large inputs.\n  // The threshold 20 is chosen such that softrelu(a) = a\n  // for a > 20 using floating precision.\n  if (val > 20) return val;\n  if (type_util::has_double_or_integral<DType>::value) {\n    return ::log(1 + ::exp(val));\n  } else {\n    return logf(1 + expf(val));\n  }\n}\n\ntemplate <typename DType>\n__device__ inline DType softsign(const DType val) {\n  if (type_util::has_double_or_integral<DType>::value) {\n    return val / (1 + fabs(val));\n  } else {\n    return val / (1 + fabsf(val));\n  }\n}\n\n// exp and log\n\n#define DEFINE_UNARY_MATH_FUNC(name, double_version, float_version) \\\ntemplate <typename DType> \\\n__device__ inline DType name (const DType a) { \\\n  if (type_util::has_double_or_integral<DType>::value) { \\\n    return double_version ((double)a); \\\n  } else { \\\n    return float_version (a); \\\n  } \\\n}\n\nDEFINE_UNARY_MATH_FUNC(exp, ::exp, ::expf)\nDEFINE_UNARY_MATH_FUNC(expm1, ::expm1, ::expm1f)\nDEFINE_UNARY_MATH_FUNC(log, ::log, ::logf)\nDEFINE_UNARY_MATH_FUNC(log10, ::log10, ::log10f)\nDEFINE_UNARY_MATH_FUNC(log2, ::log2, ::log2f)\nDEFINE_UNARY_MATH_FUNC(log1p, ::log1p, ::log1pf)\n\n// trigonometric\n\nconstexpr double pi = 3.14159265358979323846;\n\ntemplate <typename DType>\n__device__ inline DType degrees(const DType val) {\n  if (type_util::has_double_or_integral<DType>::value) {\n    return (val / pi) * 180;\n  } else {\n    return (val / static_cast<float>(pi)) * 180.f;\n  }\n}\n\ntemplate <typename DType>\n__device__ inline DType radians(const DType val) {\n  if (type_util::has_double_or_integral<DType>::value) {\n    return (val / 180.0) * pi;\n  } else {\n    return (val / 180.0f) * static_cast<float>(pi);\n  }\n}\n\nDEFINE_UNARY_MATH_FUNC(sin, ::sin, ::sinf)\nDEFINE_UNARY_MATH_FUNC(cos, ::cos, ::cosf)\nDEFINE_UNARY_MATH_FUNC(tan, ::tan, ::tanf)\nDEFINE_UNARY_MATH_FUNC(arcsin, ::asin, ::asinf)\nDEFINE_UNARY_MATH_FUNC(arccos, ::acos, ::acosf)\nDEFINE_UNARY_MATH_FUNC(arctan, ::atan, ::atanf)\n\nDEFINE_UNARY_MATH_FUNC(sinh, ::sinh, ::sinhf)\nDEFINE_UNARY_MATH_FUNC(cosh, ::cosh, ::coshf)\nDEFINE_UNARY_MATH_FUNC(tanh, ::tanh, ::tanhf)\nDEFINE_UNARY_MATH_FUNC(arcsinh, ::asinh, ::asinhf)\nDEFINE_UNARY_MATH_FUNC(arccosh, ::acosh, ::acoshf)\nDEFINE_UNARY_MATH_FUNC(arctanh, ::atanh, ::atanhf)\n\ntemplate <typename DType>\n__device__ inline DType mish(const DType val) {\n  return val * op::tanh(op::softrelu(val));\n}\n\n// sqrt\n\nDEFINE_UNARY_MATH_FUNC(sqrt, ::sqrt, ::sqrtf)\nDEFINE_UNARY_MATH_FUNC(rsqrt, ::rsqrt, ::rsqrtf)\nDEFINE_UNARY_MATH_FUNC(cbrt, ::cbrt, ::cbrtf)\nDEFINE_UNARY_MATH_FUNC(rcbrt, ::rcbrt, ::rcbrtf)\n\ntemplate <typename DType>\n__device__ inline DType square(const DType val) {\n  return val * val;\n}\n\ntemplate <typename DType, typename... DTypes>\n__device__ inline typename LoadType<DType>::Type zero(const DType val, const DTypes... args) {\n  return 0;\n}\n\ntemplate <typename DType>\n__device__ inline typename LoadType<DType>::Type zero() {\n  return 0;\n}\n\ntemplate <typename DType, typename... DTypes>\n__device__ inline typename LoadType<DType>::Type one(const DType val, const DTypes... args) {\n  return 1;\n}\n\ntemplate <typename DType>\n__device__ inline typename LoadType<DType>::Type one() {\n  return 1;\n}\n\ntemplate <typename DType, typename... DTypes>\n__device__ inline typename LoadType<DType>::Type negone(const DType val, const DTypes... args) {\n  return -1;\n}\n\ntemplate <typename DType>\n__device__ inline typename LoadType<DType>::Type negone() {\n  return -1;\n}\n\ntemplate <typename DType>\n__device__ inline DType round(const DType val) {\n  if (type_util::has_double<DType>::value) {\n    return ::round((double)val);\n  } else if (type_util::is_integral<DType>::value) {\n    return val;\n  } else {\n    return ::roundf(val);\n  }\n}\n\ntemplate <typename DType>\n__device__ inline DType floor(const DType val) {\n  if (type_util::has_double<DType>::value) {\n    return ::floor((double)val);\n  } else if (type_util::is_integral<DType>::value) {\n    return val;\n  } else {\n    return ::floorf(val);\n  }\n}\n\ntemplate <typename DType>\n__device__ inline DType ceil(const DType val) {\n  if (type_util::has_double<DType>::value) {\n    return ::ceil((double)val);\n  } else if (type_util::is_integral<DType>::value) {\n    return val;\n  } else {\n    return ::ceilf(val);\n  }\n}\n\ntemplate <typename DType>\n__device__ inline DType rint(const DType val) {\n  if (type_util::has_double<DType>::value) {\n    return ::rint((double)val);\n  } else if (type_util::is_integral<DType>::value) {\n    return val;\n  } else {\n    return ::rintf(val);\n  }\n}\n\ntemplate <typename DType>\n__device__ inline DType fix(const DType val) {\n  const auto f = floor(val);\n  const auto c = ceil(val);\n  return (f > 0 ? f : -f) < (c > 0 ? c : -c) ? f : c;\n}\n\ntemplate <typename DType>\n__device__ inline DType trunc(const DType val) {\n  if (type_util::has_double<DType>::value) {\n    return ::trunc((double)val);\n  } else if (type_util::is_integral<DType>::value) {\n    return val;\n  } else {\n    return ::truncf(val);\n  }\n}\n\ntemplate <typename DType>\n__device__ inline DType clip(const DType val, const float a_min, const float a_max) {\n  if (val > a_max) {\n    return a_max;\n  } else if (val < a_min) {\n    return a_min;\n  } else {\n    return val;\n  }\n}\n\ntemplate <typename DType>\n__device__ inline DType sign(const DType val) {\n  if (val < 0) return -1;\n  return val > 0 ? 1 : 0;\n}\n\ntemplate <typename DType>\n__device__ inline DType reciprocal(const DType val) {\n  return 1.0f / val;\n}\n\nDEFINE_UNARY_MATH_FUNC(abs, ::fabs, ::fabsf)\nDEFINE_UNARY_MATH_FUNC(gamma, ::tgamma, ::tgammaf)\nDEFINE_UNARY_MATH_FUNC(gammaln, ::lgamma, ::lgammaf)\nDEFINE_UNARY_MATH_FUNC(erf, ::erf, ::erff)\nDEFINE_UNARY_MATH_FUNC(erfinv, ::erfinv, ::erfinvf)\n\ntemplate <typename DType>\n__device__ inline DType gelu_erf(const DType val) {\n  return 0.5f * val * (1.0f + op::erf(val / op::sqrt(2.0f)));\n}\n\ntemplate <typename DType1, typename DType2>\n__device__ inline DType1 smooth_l1(const DType1 val, const DType2 scalar) {\n  const auto bsq = scalar * scalar;\n  const auto ibsq = 1.0f / bsq;\n  if (val > ibsq) {\n    return val - 0.5f * ibsq;\n  } else if (val < -ibsq) {\n    return -val - 0.5f * ibsq;\n  } else {\n    return 0.5f * val * val * bsq;\n  }\n}\n\ntemplate <typename DType>\n__device__ inline DType digamma(const DType val) {\n  if (type_util::has_double_or_integral<DType>::value) {\n    return special_functions::cephes::psi<double>(val);\n  } else {\n    return special_functions::cephes::psi<float>(val);\n  }\n}\n\ntemplate <typename DType>\n__device__ inline DType logical_not(const DType val) {\n  return val != DType(0) ? DType(0) : DType(1);\n}\n\ntemplate <typename DType>\n__device__ inline bool_t np_logical_not(const DType val) {\n  return !static_cast<bool>(val);\n}\n\ntemplate <typename DType>\n__device__ inline bool_t NonZero(const DType val) {\n  return val != 0;\n}\n\n#undef DEFINE_UNARY_MATH_FUNC\n\ntemplate <typename DType>\n__device__ inline DType bitwise_not(const DType a) {\n  if (type_util::is_same<DType, bool_t>::value) {\n    return !a;\n  } else {\n    return ~static_cast<int64>(a);\n  }\n}\n\n}  // namespace op\n\n)code\";\n\n}  // namespace rtc\n}  // namespace cuda\n}  // namespace common\n}  // namespace mxnet\n\n#endif  // MXNET_USE_CUDA\n\n#endif  // MXNET_COMMON_CUDA_RTC_FORWARD_FUNCTIONS_INL_H_\n"
  },
  {
    "path": "src/common/cuda/rtc/half-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_COMMON_CUDA_RTC_HALF_INL_H_\n#define MXNET_COMMON_CUDA_RTC_HALF_INL_H_\n\n#if MXNET_USE_CUDA\n\nnamespace mxnet {\nnamespace common {\nnamespace cuda {\nnamespace rtc {\n\nconst char fp16_support_string[] = R\"code(\nstruct __align__(2) __half {\n  __host__ __device__ __half() : __x(0) { }\n  unsigned short __x;\n};\n/* Definitions of intrinsics */\n__device__ inline __half __float2half(const float f) {\n  __half val;\n asm(\"{  cvt.rn.f16.f32 %0, %1;}\\n\" : \"=h\"(val.__x) : \"f\"(f));\n  return val;\n}\n__device__ inline float __half2float(const __half h) {\n  float val;\n asm(\"{  cvt.f32.f16 %0, %1;}\\n\" : \"=f\"(val) : \"h\"(h.__x));\n  return val;\n}\n\ntypedef __half half;\n\ntemplate <typename DType>\nstruct AccType {\n  using type = DType;\n\n  __device__ static inline type from(const DType& val) {\n    return val;\n  }\n\n  __device__ static inline DType to(type val) {\n    return val;\n  }\n\n};\n\ntemplate<>\nstruct AccType<half> {\n  using type = float;\n\n  __device__ static inline type from(const half& val) {\n    return __half2float(val);\n  }\n\n  __device__ static inline half to(type val) {\n    return __float2half(val);\n  }\n};\n)code\";\n\n}  // namespace rtc\n}  // namespace cuda\n}  // namespace common\n}  // namespace mxnet\n\n#endif  // MXNET_USE_CUDA\n\n#endif  // MXNET_COMMON_CUDA_RTC_HALF_INL_H_\n"
  },
  {
    "path": "src/common/cuda/rtc/reducer-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_COMMON_CUDA_RTC_REDUCER_INL_H_\n#define MXNET_COMMON_CUDA_RTC_REDUCER_INL_H_\n\n#if MXNET_USE_CUDA\n\nnamespace mxnet {\nnamespace common {\nnamespace cuda {\nnamespace rtc {\n\nconst char reducer[] = R\"code(\nnamespace red {\n\nstruct sum {\n  /*! \\brief do reduction into dst */\n  template<typename DType, typename DType2>\n  __device__ inline static void Reduce(volatile DType& dst,  volatile DType2 src) {\n    dst = op::add(dst, src);\n  }\n\n  /*! \\brief do stable reduction into dst */\n  template<typename DType, typename DType2>\n  __device__ inline static void Reduce(volatile DType& dst,  volatile DType2 src,\n                                       volatile DType& residual) {\n    DType y = op::sub(src, residual);\n    DType t = dst + y;\n    if (util::isinf(t)) {\n      residual = 0;\n    } else {\n      residual = (t - dst) - y;\n    }\n    dst = t;\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst_val, volatile DType& src_val) {\n    Reduce(dst_val, src_val);\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst_val, volatile DType& dst_residual,\n                                      volatile DType& src_val, volatile DType& src_residual) {\n    DType t1 = dst_val + src_val;\n    if (util::isinf(t1)) {\n      dst_val = t1;\n      dst_residual = 0;\n    } else {\n      DType e = t1 - dst_val;\n      DType t2 = ((src_val - e) + (dst_val - (t1 - e))) + dst_residual + src_residual;\n      dst_val = t1 + t2;\n      dst_residual = t2 - (dst_val - t1);\n    }\n  }\n  /*! \\brief finalize reduction result */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& dst) {}\n  /*! \\brief finalize reduction result */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& dst, volatile DType& none) {}\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &initv) {\n    initv = 0;\n  }\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &initv, DType &residual) {\n    SetInitValue(initv);\n    residual = 0;\n  }\n};\n\nstruct product {\n  /*! \\brief do reduction into dst */\n  template<typename DType, typename DType2>\n  __device__ inline static void Reduce(volatile DType& dst, volatile DType2 src) {\n    dst = op::mul(dst, src);\n  }\n  /*! \\brief do reduction into dst */\n  template<typename DType, typename DType2>\n  __device__ inline static void Reduce(volatile DType& dst, volatile DType2 src,\n                                       volatile DType& none) {\n    Reduce(dst, src);\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst_val, volatile DType& src_val) {\n    Reduce(dst_val, src_val);\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst_val, volatile DType& dst_residual,\n                                      volatile DType& src_val, volatile DType& src_residual) {\n    Reduce(dst_val, src_val);\n  }\n  /*! \\brief finalize reduction result */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& dst) {}\n  /*! \\brief finalize reduction result */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& dst, volatile DType& none) {}\n  /*!\n  *\\brief set the initial value during reduction\n  */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &initv) {\n    initv = 1;\n  }\n  /*!\n  *\\brief set the initial value during reduction\n  */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &initv, DType &none) {\n    SetInitValue(initv);\n  }\n};\n\nstruct nansum {\n  /*! \\brief do reduction into dst */\n  template<typename DType, typename DType2>\n  __device__ inline static void Reduce(volatile DType& dst, volatile DType2 src) {\n    if (util::isnan(src)) return;\n    dst = op::add(dst, src);\n  }\n  /*! \\brief do reduction into dst */\n  template<typename DType>\n  __device__ inline static void Reduce(volatile DType& dst, volatile DType src,\n                                       volatile DType& residual) {\n    if (util::isnan(src)) return;\n    DType y = src - residual;\n    DType t = dst + y;\n    residual = (t - dst) - y;\n    dst = t;\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst_val, volatile DType& src_val) {\n    Reduce(dst_val, src_val);\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst_val, volatile DType& dst_residual,\n                                      volatile DType& src_val, volatile DType& src_residual) {\n    DType t1 = dst_val + src_val;\n    DType e = t1 - src_val;\n    DType t2 = ((src_val - e) + (dst_val - (t1 - e))) + dst_residual + src_residual;\n    dst_val = t1 + t2;\n    dst_residual = t2 - (dst_val - t1);\n  }\n  /*! \\brief finalize reduction result */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& dst) {}\n  /*! \\brief finalize reduction result */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& dst, volatile DType& none) {}\n  /*!\n  *\\brief set the initial value during reduction\n  */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType & initv) {\n      initv = 0;\n  }\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &initv, DType &residual) {\n    SetInitValue(initv);\n    residual = 0;\n  }\n};\n\nstruct nanprod {\n  /*! \\brief do reduction into dst */\n  template<typename DType, typename DType2>\n  __device__ inline static void Reduce(volatile DType& dst, volatile DType2 src) {\n    if (util::isnan(src)) return;\n    dst = op::mul(dst, src);\n  }\n  /*! \\brief do reduction into dst */\n  template<typename DType>\n  __device__ inline static void Reduce(volatile DType& dst, volatile DType src,\n                                       volatile DType& none) {\n    Reduce(dst, src);\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst_val, volatile DType& src_val) {\n    Reduce(dst_val, src_val);\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst_val, volatile DType& dst_residual,\n                                      volatile DType& src_val, volatile DType& src_residual) {\n    Reduce(dst_val, src_val);\n  }\n  /*! \\brief finalize reduction */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& dst) {}\n  /*! \\brief finalize reduction */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& dst, volatile DType& none) {}\n  /*!\n  *\\brief set the initial value during reduction\n  */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType & initv) {\n    initv = 1;\n  }\n  /*!\n  *\\brief set the initial value during reduction\n  */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &initv, DType &none) {\n    SetInitValue(initv);\n  }\n};\n\nstruct nrm2 {\n  /*! \\brief do reduction into dst */\n  template<typename AType, typename DType>\n  __device__ inline static void Reduce(volatile AType& sum_of_squares, volatile DType src) {\n    sum_of_squares = op::add(sum_of_square, src * src);\n  }\n  /*! \\brief do stable reduction into dst */\n  template<typename AType, typename DType>\n  __device__ inline static void Reduce(volatile AType& sum_of_squares,\n                                       volatile DType src, volatile DType& scale) {\n    if (src != 0) {\n      DType abs = op::abs(src);\n      if (scale < abs) {\n        sum_of_squares = 1 + sum_of_squares * (scale / abs) * (scale / abs);\n        scale = abs;\n      } else {\n        sum_of_squares = sum_of_squares + (abs / scale) * (abs / scale);\n      }\n    }\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst_val, volatile DType& src_val) {\n    dst_val = op::add(dst_val, src_val);\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst_ssq, volatile DType& dst_scale,\n                                      volatile DType& src_ssq, volatile DType& src_scale) {\n    if (dst_scale != 0 && dst_scale >= src_scale) {\n      dst_ssq = dst_ssq + src_ssq * (src_scale / dst_scale) * (src_scale / dst_scale);\n    } else if (src_scale != 0 && dst_scale < src_scale) {\n      dst_ssq = src_ssq + dst_ssq * (dst_scale / src_scale) * (dst_scale / src_scale);\n      dst_scale = src_scale;\n    }\n  }\n  /*! \\brief finalize reduction result */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& sum_of_squares) {\n    sum_of_squares = op::sqrt(sum_of_squares);\n  }\n  /*! \\brief finalize reduction result */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& sum_of_squares, volatile DType& scale) {\n    sum_of_squares = scale * op::sqrt(sum_of_squares);\n  }\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &sum_of_squares) {\n    sum_of_squares = 0;\n  }\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &sum_of_squares, DType &scale) {\n    SetInitValue(sum_of_squares);\n    scale = 0;\n  }\n};\n\nstruct nrmlp {\n  double lp;\n  /* \\brief power for Lp norm */\n  __device__ inline static double lp_power(volatile double src, volatile double p) {\n    if (p != 0.0) {\n      if (src == 0.0) {\n        return src;\n      } else {\n        return op::power(src, p);\n      }\n    } else {  // 0-norm, sparsity\n      return static_cast<double>(src != 0);\n    }\n  }\n\n  /*! \\brief do reduction into dst */\n  template<typename AType, typename DType>\n  __device__ inline void Reduce(volatile AType& sum_of_powers, volatile DType src) {\n    if (src != 0) {\n      sum_of_powers += AType(lp_power(static_cast<double>(src), lp));\n    }\n  }\n\n  /*! \\brief do stable reduction into dst */\n  template<typename AType, typename DType>\n  __device__ inline void Reduce(volatile AType& sum_of_powers, volatile DType src,\n                                volatile DType& scale) {\n    if (src != 0) {\n      DType src_abs = op::abs(src);\n      if (scale < src_abs) {\n        sum_of_powers = sum_of_powers * AType(lp_power(static_cast<double>(scale / src_abs), lp));\n        sum_of_powers = sum_of_powers + 1;\n        scale = src_abs;\n      } else {\n        sum_of_powers = sum_of_powers + AType(lp_power(static_cast<double>(src_abs / scale), lp));\n      }\n    }\n  }\n\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst_val, volatile DType& src_val) {\n    dst_val = dst_val + src_val;\n  }\n\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst_ssq, volatile DType& dst_scale,\n                                      volatile DType& src_ssq, volatile DType& src_scale) {\n    if (dst_scale != 0 && dst_scale >= src_scale) {\n      dst_ssq = dst_ssq + src_ssq * DType(lp_power(static_cast<double>(src_scale / dst_scale), 2));\n    } else if (src_scale != 0 && dst_scale < src_scale) {\n      dst_ssq = src_ssq + dst_ssq * DType(lp_power(static_cast<double>(dst_scale / src_scale), 2));\n      dst_scale = src_scale;\n    }\n  }\n\n  /*! \\brief finalize reduction result */\n  template<typename DType>\n  __device__ inline void Finalize(volatile DType& sum_of_powers) {\n    if (lp != 0.0) {\n      sum_of_powers = DType(lp_power(static_cast<double>(sum_of_powers), 1.0 / lp));\n    }\n  }\n\n  /*! \\brief finalize reduction result */\n  template<typename DType>\n  __device__ inline void Finalize(volatile DType& sum_of_powers, volatile DType& scale) {\n    if (lp != 0.0) {\n      sum_of_powers = scale * DType(lp_power(static_cast<double>(sum_of_powers), 1.0 / lp));\n    }\n  }\n\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &sum_of_powers) {\n    sum_of_powers = 0;\n  }\n\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &sum_of_powers, DType &scale) {\n    SetInitValue(sum_of_powers);\n    scale = 0;\n  }\n};\n\n}  // namespace red\n)code\";\n\nconst char logic_reducer[] = R\"code(\nnamespace red {\n\nstruct maximum {\n  /*! \\brief do reduction into dst */\n  template<typename DType, typename DType2>\n  __device__ inline static void Reduce(volatile DType& dst,  volatile DType2 src) { // NOLINT(*)\n    if (!util::isnan(dst)) {\n      if (!(dst >= src)) dst = src;\n    }\n  }\n  /*! \\brief do reduction into dst */\n  template<typename DType, typename DType2>\n  __device__ inline static void Reduce(volatile DType& dst,  volatile DType2 src,\n                                       volatile DType& none) {\n    Reduce(dst, src);\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst_val, volatile DType& src_val) {\n    Reduce(dst_val, src_val);\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst_val, volatile DType& dst_residual,\n                                      volatile DType& src_val, volatile DType& src_residual) {\n    Reduce(dst_val, src_val);\n  }\n  /*! \\brief finalize reduction result */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& dst) {}\n  /*! \\brief finalize reduction result */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& dst, volatile DType& none) {}\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &initv) {\n    initv = limits::NegInfValue<DType>();\n  }\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &initv, DType &none) {\n    SetInitValue(initv);\n  }\n};\n\nstruct minimum {\n  /*! \\brief do reduction into dst */\n  template<typename DType, typename DType2>\n  __device__ inline static void Reduce(volatile DType& dst,  volatile DType2 src) {\n    if (!util::isnan(dst)) {\n      if (!(dst <= src)) dst = src;\n    }\n  }\n  /*! \\brief do reduction into dst */\n  template<typename DType, typename DType2>\n  __device__ inline static void Reduce(volatile DType& dst,  volatile DType2 src,\n                                       volatile DType& none) {\n    Reduce(dst, src);\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst_val, volatile DType& src_val) {\n    Reduce(dst_val, src_val);\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst_val, volatile DType& dst_residual,\n                                      volatile DType& src_val, volatile DType& src_residual) {\n    Reduce(dst_val, src_val);\n  }\n  /*! \\brief finalize reduction result */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& dst) {}\n  /*! \\brief finalize reduction result */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& dst, volatile DType& none) {}\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &initv) {\n    initv = limits::PosInfValue<DType>();\n  }\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &initv, DType &none) {\n    SetInitValue(initv);\n  }\n};\n\nstruct argmax {\n  /*! \\brief do reduction into dst */\n  template<typename AType, typename DType>\n  __device__ inline static void Reduce(volatile AType& dst,  volatile DType src) {\n    if (dst.num < src.num || (dst.num == src.num && dst.idx > src.idx)) {\n      dst.num = src.num;\n      dst.idx = src.idx;\n    }\n  }\n  /*! \\brief do stable reduction into dst */\n  template<typename AType, typename DType>\n  __device__ inline static void Reduce(volatile AType& dst,  volatile DType src,\n                                       volatile DType&) {\n    if (dst.num < src.num || (dst.num == src.num && dst.idx > src.idx)) {\n      dst.num = src.num;\n      dst.idx = src.idx;\n    }\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst, volatile DType& src) {\n    if (dst.num < src.num || (dst.num == src.num && dst.idx > src.idx)) {\n      dst.num = src.num;\n      dst.idx = src.idx;\n    }\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst, volatile DType&,\n                                      volatile DType& src, volatile DType&) {\n    if (dst.num < src.num || (dst.num == src.num && dst.idx > src.idx)) {\n      dst.num = src.num;\n      dst.idx = src.idx;\n    }\n  }\n  /*! \\brief finalize reduction */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& dst) {}\n  /*! \\brief finalize reduction */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& dst, volatile DType&) {}\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &initv) {\n    initv.num = limits::NegInfValue<decltype(initv.num)>();\n  }\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &initv, DType &) {\n    initv.num = limits::NegInfValue<decltype(initv.num)>();\n  }\n};\n\nstruct argmin {\n  /*! \\brief do reduction into dst */\n  template<typename AType, typename DType>\n  __device__ inline static void Reduce(volatile AType& dst,  volatile DType src) {\n    if (dst.num > src.num || (dst.num == src.num && dst.idx > src.idx)) {\n      dst.num = src.num;\n      dst.idx = src.idx;\n    }\n  }\n  /*! \\brief do stable reduction into dst */\n  template<typename AType, typename DType>\n  __device__ inline static void Reduce(volatile AType& dst,  volatile DType src,\n                                       volatile DType& residual) {\n    if (dst.num > src.num || (dst.num == src.num && dst.idx > src.idx)) {\n      dst.num = src.num;\n      dst.idx = src.idx;\n    }\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst, volatile DType& src) {\n    if (dst.num > src.num || (dst.num == src.num && dst.idx > src.idx)) {\n      dst.num = src.num;\n      dst.idx = src.idx;\n    }\n  }\n  /*! \\brief combine the results of two reducers */\n  template<typename DType>\n  __device__ inline static void Merge(volatile DType& dst, volatile DType&,\n                                      volatile DType& src, volatile DType&) {\n    if (dst.num > src.num || (dst.num == src.num && dst.idx > src.idx)) {\n      dst.num = src.num;\n      dst.idx = src.idx;\n    }\n  }\n  /*! \\brief finalize reduction */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& dst) {}\n  /*! \\brief finalize reduction */\n  template<typename DType>\n  __device__ inline static void Finalize(volatile DType& dst, volatile DType& residual) {}\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &initv) {\n    initv.num = limits::PosInfValue<decltype(initv.num)>();\n  }\n  /*!\n   *\\brief set the initial value during reduction\n   */\n  template<typename DType>\n  __device__ inline static void SetInitValue(DType &initv, DType &residual) {\n    initv.num = limits::PosInfValue<decltype(initv.num)>();\n  }\n};\n}  // namespace red\n)code\";\n}  // namespace rtc\n}  // namespace cuda\n}  // namespace common\n}  // namespace mxnet\n\n#endif  // MXNET_USE_CUDA\n\n#endif  // MXNET_COMMON_CUDA_RTC_REDUCER_INL_H_\n"
  },
  {
    "path": "src/common/cuda/rtc/special_functions-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_COMMON_CUDA_RTC_SPECIAL_FUNCTIONS_INL_H_\n#define MXNET_COMMON_CUDA_RTC_SPECIAL_FUNCTIONS_INL_H_\n\n#include <cfloat>\n#include <string>\n\nnamespace mxnet {\nnamespace common {\nnamespace cuda {\nnamespace rtc {\n\n// This code is based on the Cephes Library availible at http://www.netlib.org/cephes\n// The original author, Stephen Moshier, has kindly given permission to use this code\n// in mxnet.  (See email below).\n//\n//     Date: Tue, 13 Sep 2016 09:28:20 -0400\n//     From: Stephen Moshier\n//     To: Flunkert, Valentin\n//     Subject: Re: cephes code in mxnet\n//\n//     Hello Valentin,\n//\n//     Thank you for writing.  You are welcome to use and modify the Cephes code\n//     and distribute it under the Apache license.\n//\n//     Good luck with your project,\n//     Steve Moshier\n//\n// Cephes Math Library Release 2.2:  June, 1992\n// Copyright 1984, 1987, 1992 by Stephen L. Moshier\n// Direct inquiries to 30 Frost Street, Cambridge, MA 02140\n//\nconst char special_functions_definitions[] = R\"code(\nnamespace op {\n\nnamespace special_functions {\n\ntemplate<typename DType>\n__device__ inline static DType trigamma(DType x);\n\ntemplate<>\n__device__ inline double trigamma<double>(double x) {\n  double PI(3.14159265358979323846);\n  double sign = +1;\n  double result = 0;\n  if (x < 0.5) {\n    sign = -1;\n    const double sin_pi_x = sin(PI * x);\n    result -= (PI * PI) / (sin_pi_x * sin_pi_x);\n    x = 1 - x;\n  }\n  for (int i = 0; i < 6; ++i) {\n    result += 1 / (x * x);\n    x += 1;\n  }\n  const double ixx = 1 / (x*x);\n  result += (1 + 1 / (2*x) + ixx * (1./6 - ixx * (1./30 - ixx * (1./42)))) / x;\n  return sign * result;\n}\n\ntemplate<>\n__device__ inline float trigamma<float>(float x) {\n  float PI(3.14159265358979323846);\n  float sign = +1;\n  float result = 0;\n  if (x < 0.5f) {\n    sign = -1;\n    const float sin_pi_x = sinf(PI * x);\n    result -= (PI * PI) / (sin_pi_x * sin_pi_x);\n    x = 1 - x;\n  }\n  for (int i = 0; i < 6; ++i) {\n    result += 1 / (x * x);\n    x += 1;\n  }\n  const float ixx = 1 / (x*x);\n  result += (1 + 1 / (2*x) + ixx * (1.f/6 - ixx * (1.f/30 - ixx * (1.f/42)))) / x;\n  return sign * result;\n}\n\nstruct cephes {\n  /*\n   * Helper to evaluate a polynomial given an array of coefficients.\n   */\n  template <typename DType>\n  __device__ inline static DType polevl(DType x, const DType coef[], int N) {\n    DType ans;\n    DType const *p;\n    int i;\n\n    p = coef;\n    ans = *p++;\n\n    i = N;\n    do {\n      ans = ans * x  +  *p++;\n    } while ( --i );\n\n    return( ans );\n  }\n\n\n  /*\n   * Helper function for psi that handles double/float specific differences\n   * in the algorithm.\n   */\n  template<typename DType>\n  __device__ inline static DType psi_helper(DType s);\n\n  /*\n   *\n   *\tPsi (digamma) function\n   *\n   *\n   * SYNOPSIS:\n   *\n   * float x, y, psif();\n   *\n   * y = psif( x );\n   *\n   *\n   * DESCRIPTION:\n   *\n   *              d      -\n   *   psi(x)  =  -- ln | (x)\n   *              dx\n   *\n   * is the logarithmic derivative of the gamma function.\n   * For integer x,\n   *                   n-1\n   *                    -\n   * psi(n) = -EUL  +   >  1/k.\n   *                    -\n   *                   k=1\n   *\n   * This formula is used for 0 < n <= 10.  If x is negative, it\n   * is transformed to a positive argument by the reflection\n   * formula  psi(1-x) = psi(x) + pi cot(pi x).\n   * For general positive x, the argument is made greater than 10\n   * using the recurrence  psi(x+1) = psi(x) + 1/x.\n   * Then the following asymptotic expansion is applied:\n   *\n   *                           inf.   B\n   *                            -      2k\n   * psi(x) = log(x) - 1/2x -   >   -------\n   *                            -        2k\n   *                           k=1   2k x\n   *\n   * where the B2k are Bernoulli numbers.\n   *\n   * ACCURACY:\n   *    Absolute error,  relative when |psi| > 1 :\n   * arithmetic   domain     # trials      peak         rms\n   *    IEEE      -33,0        30000      8.2e-7      1.2e-7\n   *    IEEE      0,33        100000      7.3e-7      7.7e-8\n   *\n   * ERROR MESSAGES:\n   *     message         condition      value returned\n   * psi singularity    x integer <=0      MAXNUMF\n   */\n  template<typename DType>\n  __device__ inline static DType psi(DType x) {\n    DType p, q, nz, s, w, y;\n    int i, n, negative;\n\n    DType EUL(0.57721566490153286061);\n    DType PI(3.14159265358979323846);\n\n    negative = 0;\n    nz = 0.0;\n\n    if ( x <= 0.0 ) {\n      negative = 1;\n      q = x;\n      p = ::floor(q);\n      if ( p == q ) {\n        return DBL_MAX;\n      }\n      /* Remove the zeros of tan(PI x)\n       * by subtracting the nearest integer from x\n       */\n      nz = q - p;\n      if ( nz != 0.5 ) {\n        if ( nz > 0.5 ) {\n          p += 1.0;\n          nz = q - p;\n        }\n        nz = PI/::tan(PI*nz);\n      } else {\n        nz = 0.0;\n      }\n      x = 1.0 - x;\n    }\n\n    /* check for positive integer up to 10 */\n    if ( (x <= 10.0) && (x == ::floor(x)) ) {\n      y = 0.0;\n      n = x;\n      for ( i = 1; i < n; i++ ) {\n        w = i;\n        y += 1.0/w;\n      }\n      y -= EUL;\n      goto done;\n    }\n\n    s = x;\n    w = 0.0;\n    while ( s < 10.0 ) {\n      w += 1.0/s;\n      s += 1.0;\n    }\n\n    y = psi_helper(s);\n\n    y = logf(s)  -  (0.5/s)  -  y  -  w;\n\ndone:\n\n    if ( negative ) {\n      y -= nz;\n    }\n\n    return(y);\n  }\n};\n\n\ntemplate<>\n__device__ inline double cephes::psi_helper<double>(double s) {\n  double z;\n  const double A[] = {\n    8.33333333333333333333E-2,\n    -2.10927960927960927961E-2,\n    7.57575757575757575758E-3,\n    -4.16666666666666666667E-3,\n    3.96825396825396825397E-3,\n    -8.33333333333333333333E-3,\n    8.33333333333333333333E-2\n  };\n\n  if ( s < 1.0e17 ) {\n    z = 1.0/(s * s);\n    return z * cephes::polevl<double>(z, A, 6);\n  } else {\n    return 0.0;\n  }\n}\n\ntemplate<>\n__device__ inline float cephes::psi_helper<float>(float s) {\n  float z;\n  const float A[] = {\n    -4.16666666666666666667E-3f,\n    3.96825396825396825397E-3f,\n    -8.33333333333333333333E-3f,\n    8.33333333333333333333E-2f\n  };\n\n  if ( s < 1.0e8 ) {\n    z = 1.0/(s * s);\n    return z * cephes::polevl<float>(z, A, 3);\n  } else {\n    return 0.0;\n  }\n}\n}  // namespace special_functions\n}  // namespace op\n)code\";\n\n}  // namespace rtc\n}  // namespace cuda\n}  // namespace common\n}  // namespace mxnet\n\n#endif  // MXNET_COMMON_CUDA_RTC_SPECIAL_FUNCTIONS_INL_H_\n"
  },
  {
    "path": "src/common/cuda/rtc/util-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_COMMON_CUDA_RTC_UTIL_INL_H_\n#define MXNET_COMMON_CUDA_RTC_UTIL_INL_H_\n\n#include <mxnet/base.h>\n\n#if MXNET_USE_CUDA\n\nnamespace mxnet {\nnamespace common {\nnamespace cuda {\nnamespace rtc {\n\nconst char type_support_string[] = R\"code(\nusing float32 = float;\nusing float64 = double;\nusing float16 = half;\nusing uint8 = unsigned char;\nusing int8 = char;\nusing int32 = int;\nusing int64 = long long;\nusing int16 = short;\nusing uint16 = unsigned short;\nusing uint32 = unsigned int;\nusing uint64 = unsigned long long;\n\nstatic_assert(sizeof(float32) == 4, \"Size of float32 is expected to be 4B\");\nstatic_assert(sizeof(float64) == 8, \"Size of float64 is expected to be 8B\");\nstatic_assert(sizeof(float16) == 2, \"Size of float16 is expected to be 2B\");\nstatic_assert(sizeof(uint8) == 1, \"Size of uint8 is expected to be 1B\");\nstatic_assert(sizeof(int8) == 1, \"Size of int8 is expected to be 1B\");\nstatic_assert(sizeof(int32) == 4, \"Size of int32 is expected to be 4B\");\nstatic_assert(sizeof(int64) == 8, \"Size of int64 is expected to be 8B\");\nstatic_assert(sizeof(int16) == 2, \"Size of int16 is expected to be 2B\");\nstatic_assert(sizeof(uint16) == 2, \"Size of uint16 is expected to be 2B\");\nstatic_assert(sizeof(uint32) == 4, \"Size of uint32 is expected to be 4B\");\nstatic_assert(sizeof(uint64) == 8, \"Size of uint64 is expected to be 8B\");\n\n)code\"\n#if MSHADOW_INT64_TENSOR_SIZE == 1\n                                   \"typedef int64 index_t;\\n\"\n#else\n                                   \"typedef int32 index_t;\\n\"\n#endif\n                                   R\"code(\n// bool and int8 need to be accumulated in index_t\n// but bool needs to be treated in the special way\n// for ops like bitwise_not\nstruct bool_t {\n  index_t value;\n\n  __device__ inline bool_t(const index_t& v) : value(v) {}\n  __device__ inline bool_t(const volatile index_t& v) : value(v) {}\n  __device__ inline bool_t() : value(0) {}\n\n  __device__ inline operator index_t() const volatile { return value; }\n  __device__ inline bool_t& operator= (const index_t& v) {\n    value = v;\n    return *this;\n  }\n  __device__ inline volatile bool_t& operator= (const index_t& v) volatile {\n    value = v;\n    return *this;\n  }\n  __device__ inline bool_t& operator= (const volatile index_t& v) {\n    value = v;\n    return *this;\n  }\n};\ntemplate<>\nstruct AccType<bool> {\n  using type = bool_t;\n\n  __device__ static inline type from(const bool& val) {\n    return val;\n  }\n\n  __device__ static inline bool to(type val) {\n    return val;\n  }\n};\n\ntemplate<>\nstruct AccType<int8> {\n  using type = index_t;\n\n  __device__ static inline type from(const int8& val) {\n    return val;\n  }\n\n  __device__ static inline int8 to(type val) {\n    return val;\n  }\n};\n\ntemplate<>\nstruct AccType<uint8> {\n  using type = index_t;\n\n  __device__ static inline type from(const uint8& val) {\n    return val;\n  }\n\n  __device__ static inline uint8 to(type val) {\n    return val;\n  }\n};\n\nnamespace type_util {\n\nstruct false_type {\n  static constexpr bool value = false;\n};\n\nstruct true_type {\n  static constexpr bool value = true;\n};\n\n// is_integral\ntemplate <typename T> struct is_integral : false_type {};\ntemplate <> struct is_integral<uint8> : true_type {};\ntemplate <> struct is_integral<uint16> : true_type {};\ntemplate <> struct is_integral<uint32> : true_type {};\ntemplate <> struct is_integral<uint64> : true_type {};\ntemplate <> struct is_integral<int8>  : true_type {};\ntemplate <> struct is_integral<int16>  : true_type {};\ntemplate <> struct is_integral<int32> : true_type {};\ntemplate <> struct is_integral<int64> : true_type {};\ntemplate <> struct is_integral<bool>  : true_type {};\ntemplate <> struct is_integral<bool_t>  : true_type {};\n\n// is_unsigned\ntemplate <typename T> struct is_unsigned : false_type {};\ntemplate <> struct is_unsigned<uint8> : true_type {};\ntemplate <> struct is_unsigned<uint16> : true_type {};\ntemplate <> struct is_unsigned<uint32> : true_type {};\ntemplate <> struct is_unsigned<uint64> : true_type {};\ntemplate <> struct is_unsigned<bool>  : true_type {};\ntemplate <> struct is_unsigned<bool_t>  : true_type {};\n\n// is_same\ntemplate <typename T, typename U>\nstruct is_same : false_type {};\ntemplate <typename T> struct is_same<T, T> : true_type {};\n\n// has_double\ntemplate <typename... T> struct has_double : false_type {};\n\ntemplate <typename A, typename... B>\nstruct has_double<A, B...> {\n    static constexpr bool value = is_same<A, double>::value ||\n                                  has_double<B...>::value;\n};\n\n// has_double_or_integral\ntemplate <typename... T> struct has_double_or_integral : false_type {};\n\ntemplate <typename A, typename... B>\nstruct has_double_or_integral<A, B...> {\n    static constexpr bool value = is_same<A, double>::value ||\n                                  is_integral<A>::value ||\n                                  has_double_or_integral<B...>::value;\n};\n\ntemplate <bool b>\nstruct enable_if {};\n\ntemplate <>\nstruct enable_if<true> {\n  using type = void;\n};\n\ntemplate <typename T, typename U, class Enable = void>\nstruct mixed_type_helper;\n\ntemplate <typename T>\nstruct mixed_type_helper<T, float64, typename enable_if<!is_same<float64, T>::value>::type> {\n  using type = float64;\n};\n\ntemplate <typename T>\nstruct mixed_type_helper<float64, T> {\n  using type = float64;\n};\n\ntemplate <typename T>\nstruct mixed_type_helper<T, float32, typename enable_if<!is_same<float64, T>::value &&\n                                                        !is_same<float32, T>::value>::type> {\n  using type = float32;\n};\n\ntemplate <typename T>\nstruct mixed_type_helper<float32, T, typename enable_if<!is_same<float64, T>::value>::type> {\n  using type = float32;\n};\n\ntemplate <typename T>\nstruct mixed_type_helper<T, float16, typename enable_if<is_same<float16, T>::value ||\n                                                        is_integral<T>::value>::type> {\n  using type = float16;\n};\n\ntemplate <typename T>\nstruct mixed_type_helper<float16, T, typename enable_if<is_integral<T>::value>::type> {\n  using type = float16;\n};\n\ntemplate <typename T, typename U>\nstruct mixed_type_helper<T, U, typename enable_if<is_integral<T>::value &&\n                                                  is_integral<U>::value &&\n                                                  is_unsigned<T>::value &&\n                                                  is_unsigned<U>::value &&\n                                                  !is_same<U, bool_t>::value &&\n                                                  sizeof(T) < sizeof(U)>::type> {\n  using type = U;\n};\n\ntemplate <typename T, typename U>\nstruct mixed_type_helper<T, U, typename enable_if<is_integral<T>::value &&\n                                                  is_integral<U>::value &&\n                                                  !is_unsigned<T>::value &&\n                                                  !is_unsigned<U>::value &&\n                                                  !is_same<U, bool_t>::value &&\n                                                  sizeof(T) < sizeof(U)>::type> {\n  using type = U;\n};\n\ntemplate <typename T, typename U>\nstruct mixed_type_helper<T, U, typename enable_if<is_integral<T>::value &&\n                                                  is_integral<U>::value &&\n                                                  is_unsigned<T>::value &&\n                                                  !is_unsigned<U>::value &&\n                                                  !is_same<U, bool_t>::value &&\n                                                  sizeof(T) < sizeof(U)>::type> {\n  using type = U;\n};\n\ntemplate <typename T, typename U>\nstruct mixed_type_helper<U, T, typename enable_if<is_integral<T>::value &&\n                                                  is_integral<U>::value &&\n                                                  is_unsigned<T>::value &&\n                                                  is_unsigned<U>::value &&\n                                                  !is_same<U, bool_t>::value &&\n                                                  sizeof(T) < sizeof(U)>::type> {\n  using type = U;\n};\n\ntemplate <typename T, typename U>\nstruct mixed_type_helper<U, T, typename enable_if<is_integral<T>::value &&\n                                                  is_integral<U>::value &&\n                                                  !is_unsigned<T>::value &&\n                                                  !is_unsigned<U>::value &&\n                                                  !is_same<U, bool_t>::value &&\n                                                  sizeof(T) < sizeof(U)>::type> {\n  using type = U;\n};\n\ntemplate <typename T, typename U>\nstruct mixed_type_helper<U, T, typename enable_if<is_integral<T>::value &&\n                                                  is_integral<U>::value &&\n                                                  is_unsigned<T>::value &&\n                                                  !is_unsigned<U>::value &&\n                                                  !is_same<U, bool_t>::value &&\n                                                  sizeof(T) < sizeof(U)>::type> {\n  using type = U;\n};\n\ntemplate <typename T, typename U>\nstruct mixed_type_helper<T, U, typename enable_if<is_integral<T>::value &&\n                                                  is_integral<U>::value &&\n                                                  !is_same<U, bool_t>::value &&\n                                                  is_same<T, U>::value>::type> {\n  using type = U;\n};\n\ntemplate<>\nstruct mixed_type_helper<int8, uint8> {\n  using type = int16;\n};\n\ntemplate<>\nstruct mixed_type_helper<uint8, int8> {\n  using type = int16;\n};\n\ntemplate<>\nstruct mixed_type_helper<int8, uint16> {\n  using type = int32;\n};\n\ntemplate<>\nstruct mixed_type_helper<uint16, int8> {\n  using type = int32;\n};\n\ntemplate<>\nstruct mixed_type_helper<int8, uint32> {\n  using type = int64;\n};\n\ntemplate<>\nstruct mixed_type_helper<uint32, int8> {\n  using type = int64;\n};\n\ntemplate<>\nstruct mixed_type_helper<int16, uint16> {\n  using type = int32;\n};\n\ntemplate<>\nstruct mixed_type_helper<uint16, int16> {\n  using type = int32;\n};\n\ntemplate<>\nstruct mixed_type_helper<int16, uint32> {\n  using type = int64;\n};\n\ntemplate<>\nstruct mixed_type_helper<uint32, int16> {\n  using type = int64;\n};\n\ntemplate<>\nstruct mixed_type_helper<int32, uint32> {\n  using type = int64;\n};\n\ntemplate<>\nstruct mixed_type_helper<uint32, int32> {\n  using type = int64;\n};\n\ntemplate<>\nstruct mixed_type_helper<uint64, index_t> {\n  using type = index_t;\n};\n\ntemplate<>\nstruct mixed_type_helper<index_t, uint64> {\n  using type = index_t;\n};\n\ntemplate <typename T>\nstruct mixed_type_helper<T, bool_t, typename enable_if<is_integral<T>::value &&\n                                                       sizeof(T) < sizeof(bool_t)>::type> {\n  using type = index_t;\n};\n\ntemplate <typename T>\nstruct mixed_type_helper<bool_t, T, typename enable_if<is_integral<T>::value &&\n                                                       sizeof(T) < sizeof(bool_t)>::type> {\n  using type = index_t;\n};\n\ntemplate <typename T>\nstruct mixed_type_helper<T, bool_t, typename enable_if<is_integral<T>::value &&\n                                                       sizeof(T) == sizeof(bool_t)>::type> {\n  using type = T;\n};\n\ntemplate <typename T>\nstruct mixed_type_helper<bool_t, T, typename enable_if<is_integral<T>::value &&\n                                                       !is_same<T, bool_t>::value &&\n                                                       sizeof(T) == sizeof(bool_t)>::type> {\n  using type = T;\n};\n\ntemplate <typename... Ts>\nstruct multi_mixed_type_helper;\n\ntemplate <>\nstruct multi_mixed_type_helper<> {\n    using type = void;\n};\n\ntemplate <typename T>\nstruct multi_mixed_type_helper<T> {\n    using type = T;\n};\n\ntemplate <typename T, typename U, typename... Ts>\nstruct multi_mixed_type_helper<T, U, Ts...> {\n    using type = typename mixed_type_helper<T,\n                                            typename multi_mixed_type_helper<U,\n                                                                             Ts...>::type>::type;\n};\n\ntemplate <typename... Ts>\nusing mixed_type = typename multi_mixed_type_helper<Ts...>::type;\n\n}  // namespace type_util\n)code\";\n\nconst char util_string[] = R\"code(\nenum class OpReqType {\n  kNullOp,\n  kWriteTo,\n  kWriteInplace,\n  kAddTo\n};\n\nconstexpr int kRTCMaxThreadsPerBlock = 512;\nconstexpr int warp_size = 32;\n\nnamespace util {\n\nconstexpr int MAX_DIM = 5;\n\ntemplate <int ndim>\n__device__ inline void unravel_dot(const index_t idx, const index_t (&shape)[MAX_DIM],\n  const index_t (&stridej)[MAX_DIM], const index_t (&stridek)[MAX_DIM], index_t* j, index_t* k) {\n  *j = 0;\n  *k = 0;\n  #pragma unroll\n  for (index_t i = ndim-1, idx_t = idx; i >=0; --i) {\n    const auto tmp = idx_t / shape[i];\n    const auto coord = idx_t - tmp*shape[i];\n    *j += coord*stridej[i];\n    *k += coord*stridek[i];\n    idx_t = tmp;\n  }\n}\n\ntemplate<int ndim>\n__device__ inline index_t unravel_dot(const index_t idx, const index_t (&shape)[MAX_DIM],\n  const index_t (&stride)[MAX_DIM]) {\n  index_t ret = 0;\n  #pragma unroll\n  for (index_t i = ndim-1, j = idx; i >=0; --i) {\n    auto tmp = j / shape[i];\n    ret += (j - tmp*shape[i])*stride[i];\n    j = tmp;\n  }\n  return ret;\n}\n\ntemplate<int ndim>\n__device__ inline index_t unravel_ravel(const index_t idx, const index_t (&shape1)[MAX_DIM],\n                                        const index_t (&shape2)[MAX_DIM]) {\n  index_t ret = 0;\n  index_t total_shape = 1;\n#pragma unroll\n  for (index_t i = ndim-1, j = idx; i >=0; --i) {\n    if (i != ndim - 1) {\n      total_shape *= shape2[i + 1];\n    }\n    auto tmp = j / shape1[i];\n    const index_t coord = j - tmp*shape1[i];\n    ret += total_shape * (shape2[i] > coord) * coord;\n    j = tmp;\n  }\n  return ret;\n}\n\ntemplate<int ndim, int ndim2>\n__device__ inline index_t ravel(const index_t (&coord)[ndim], const index_t (&shape)[ndim2]) {\n  index_t ret = 0;\n#pragma unroll\n  for (int i = 0; i < ndim; ++i) {\n    ret = ret * shape[i] + (shape[i] > coord[i]) * coord[i];\n  }\n  return ret;\n}\n\ntemplate<int ndim, int ndim2>\n__device__ inline void unravel(const index_t idx,\n                               const index_t (&shape)[ndim2],\n                               index_t (&coord)[ndim]) {\n#pragma unroll\n  for (index_t i = ndim-1, j = idx; i >=0; --i) {\n    auto tmp = j / shape[i];\n    coord[i] = j - tmp*shape[i];\n    j = tmp;\n  }\n}\n\ntemplate <typename DType>\n__device__ inline bool isinf(volatile const DType &val) {\n  return false;\n}\n\ntemplate <>\n__device__ inline bool isinf(volatile const float &val) {\n  return ::isinf(val);\n}\n\ntemplate <>\n__device__ inline bool isinf(volatile const double &val) {\n  return ::isinf(val);\n}\n\ntemplate <>\n__device__ inline bool isinf(volatile const long double &val) {\n  return ::isinf(val);\n}\n\ntemplate <>\n__device__ inline bool isinf(volatile const float16 &val) {\n  return ::isinf(__half2float(const_cast<const float16&>(val)));\n}\n\ntemplate <typename DType>\n__device__ inline bool isnan(volatile const DType &val) {\n  return false;\n}\n\ntemplate <>\n__device__ inline bool isnan(volatile const float &val) {\n  return ::isnan(val);\n}\n\ntemplate <>\n__device__ inline bool isnan(volatile const double &val) {\n  return ::isnan(val);\n}\n\ntemplate <>\n__device__ inline bool isnan(volatile const long double &val) {\n  return ::isnan(val);\n}\n\ntemplate <>\n__device__ inline bool isnan(volatile const float16 &val) {\n  return ::isnan(__half2float(const_cast<const float16&>(val)));\n}\n\ntemplate <int NVALUES = warp_size, typename OP, typename T>\n__device__ inline T warp_reduce(T value, OP redfun) {\n#pragma unroll\n  for (int i = warp_size / 2; i >= 1; i /= 2) {\n    if (NVALUES > i) value = redfun(value, __shfl_down_sync(0xffffffff, value, i));\n  }\n  return value;\n}\n\ntemplate <typename OP, typename T>\n__device__ inline T grouped_warp_reduce(T value, OP redfun, const int group_size) {\n  for (int i = 1; i < group_size; i *= 2) {\n    value = redfun(value, __shfl_down_sync(0xffffffff, value, i));\n  }\n  return value;\n}\n\ntemplate <typename OP, typename T>\n__device__ inline T grouped_warp_allreduce(T value, OP redfun, const int group_size) {\n  value = grouped_warp_reduce(value, redfun, group_size);\n  return __shfl_sync(0xffffffff, value, 0, group_size);\n}\n\ntemplate <typename OP, typename T>\n__device__ inline T strided_grouped_warp_reduce(T value, OP redfun, const int group_size) {\n  for (int i = warp_size / 2; i >= group_size; i /= 2) {\n    value = redfun(value, __shfl_down_sync(0xffffffff, value, i));\n  }\n  return value;\n}\n\ntemplate <typename OP, typename T>\n__device__ inline T strided_grouped_warp_allreduce(T value, OP redfun, const int group_size) {\n  value = strided_grouped_warp_reduce(value, redfun, group_size);\n  for (int i = group_size; i < warp_size; i *= 2) {\n    T tmp = __shfl_up_sync(0xffffffff, value, i);\n    if (threadIdx.x % warp_size >= i) {\n      value = tmp;\n    }\n  }\n  return value;\n}\n\n}  // namespace util\n)code\";\n\nconst char limits[] = R\"code(\nconstexpr double DBL_MAX = 1.7976931348623157081e+308;\nconstexpr float FLT_MAX = 3.4028234663852885981e+38;\n#define inf ((float)1e50)\n#define nan (inf - inf)\n\nnamespace limits {\n\ntemplate<typename DType>\n__device__ inline DType MinValue(void);\n\ntemplate<>\n__device__ inline float MinValue<float>(void) {\n  return -FLT_MAX;\n}\n/*! \\brief minimum value of double */\ntemplate<>\n__device__ inline double MinValue<double>(void) {\n  return -DBL_MAX;\n}\n/*! \\brief minimum value of uint8 */\ntemplate<>\n__device__ inline uint8 MinValue<uint8>(void) {\n  return 0;\n}\n/*! \\brief minimum value of uint16 */\ntemplate<>\n__device__ inline uint16 MinValue<uint16>(void) {\n  return 0;\n}\n/*! \\brief minimum value of uint32 */\ntemplate<>\n__device__ inline uint32 MinValue<uint32>(void) {\n  return 0;\n}\n/*! \\brief minimum value of uint64 */\ntemplate<>\n__device__ inline uint64 MinValue<uint64>(void) {\n  return 0;\n}\n/*! \\brief minimum value of int8_t */\ntemplate<>\n__device__ inline int8 MinValue<int8>(void) {\n  return -128;\n}\n/*! \\brief minimum value of int16 */\ntemplate<>\n__device__ inline int16 MinValue<int16>(void) {\n  return -32768;\n}\n/*! \\brief minimum value of int32 */\ntemplate<>\n__device__ inline int32 MinValue<int32>(void) {\n  return -2147483648;\n}\n/*! \\brief minimum value of int64_t */\ntemplate<>\n__device__ inline int64 MinValue<int64>(void) {\n  return -9223372036854775808LL;\n}\n/*! \\brief minimum value of bool */\ntemplate<>\n__device__ inline bool MinValue<bool>(void) {\n  return false;\n}\n/*! \\brief minimum value of bool_t */\ntemplate<>\n__device__ inline bool_t MinValue<bool_t>(void) {\n  return MinValue<index_t>();\n}\n\n/*!\n * \\brief negative infinity of certain types\n * \\tparam DType data type\n */\ntemplate<typename DType>\n__device__ inline DType NegInfValue(void) {\n  return MinValue<DType>();\n}\n/*! \\brief negative infinity value of float */\ntemplate<>\n__device__ inline float NegInfValue<float>(void) {\n  return -inf;\n}\n/*! \\brief negative infinity value of double */\ntemplate<>\n__device__ inline double NegInfValue<double>(void) {\n  return -inf;\n}\n\n/*!\n * \\brief maximum value of certain types\n * \\tparam DType data type\n */\ntemplate<typename DType>\n__device__ inline DType MaxValue(void);\n/*! \\brief maximum value of float */\ntemplate<>\n__device__ inline float MaxValue<float>(void) {\n  return FLT_MAX;\n}\n/*! \\brief maximum value of double */\ntemplate<>\n__device__ inline double MaxValue<double>(void) {\n  return DBL_MAX;\n}\n/*! \\brief maximum value of uint8 */\ntemplate<>\n__device__ inline uint8 MaxValue<uint8>(void) {\n  return 255;\n}\n/*! \\brief maximum value of uint16 */\ntemplate<>\n__device__ inline uint16 MaxValue<uint16>(void) {\n  return 65535;\n}\n/*! \\brief maximum value of uint32 */\ntemplate<>\n__device__ inline uint32 MaxValue<uint32>(void) {\n  return 4294967295;\n}\n/*! \\brief maximum value of uint64 */\ntemplate<>\n__device__ inline uint64 MaxValue<uint64>(void) {\n  return 18446744073709551615LL;\n}\n/*! \\brief maximum value of int8 */\ntemplate<>\n__device__ inline int8 MaxValue<int8>(void) {\n  return 127;\n}\n/*! \\brief maximum value of int16 */\ntemplate<>\n__device__ inline int16 MaxValue<int16>(void) {\n  return 32767;\n}\n/*! \\brief maximum value of int32 */\ntemplate<>\n__device__ inline int32 MaxValue<int32>(void) {\n  return 2147483647;\n}\n/*! \\brief maximum value of int64 */\ntemplate<>\n__device__ inline int64 MaxValue<int64>(void) {\n  return 9223372036854775807LL;\n}\n/*! \\brief maximum value of bool */\ntemplate<>\n__device__ inline bool MaxValue<bool>(void) {\n  return true;\n}\n/*! \\brief maximum value of bool_t */\ntemplate<>\n__device__ inline bool_t MaxValue<bool_t>(void) {\n  return MaxValue<index_t>();\n}\n/*!\n * \\brief positive infinity of certain types\n * \\tparam DType data type\n */\ntemplate<typename DType>\n__device__ inline DType PosInfValue(void) {\n  return MaxValue<DType>();\n}\n/*! \\brief positive infinity value of float */\ntemplate<>\n__device__ inline float PosInfValue<float>(void) {\n  return inf;\n}\n/*! \\brief positive infinity value of double */\ntemplate<>\n__device__ inline double PosInfValue<double>(void) {\n  return inf;\n}\n\n}  // namespace limits\n)code\";\n}  // namespace rtc\n}  // namespace cuda\n}  // namespace common\n}  // namespace mxnet\n\n#endif  // MXNET_USE_CUDA\n\n#endif  // MXNET_COMMON_CUDA_RTC_UTIL_INL_H_\n"
  },
  {
    "path": "src/common/cuda/rtc/vectorization-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_COMMON_CUDA_RTC_VECTORIZATION_INL_H_\n#define MXNET_COMMON_CUDA_RTC_VECTORIZATION_INL_H_\n\n#include <mxnet/base.h>\n\n#if MXNET_USE_CUDA\n\n#include <sstream>\n#include <string>\n#include <vector>\n#include <algorithm>\n\n#include \"../rtc.h\"\n#include \"../../utils.h\"\n\nnamespace mxnet {\nnamespace common {\nnamespace cuda {\nnamespace rtc {\n\nconst char vectorization_support_string[] = R\"code(\n\nnamespace vector {\n\nconstexpr int vectorized_kernel_thread_num = 512;\n\ntemplate <int size>\nstruct VectorType {\n    static_assert(size <= 32, \"VectorType needs to have size of at most 32B\");\n};\n\ntemplate <>\nstruct VectorType<1> {\n  using type = char;\n};\n\ntemplate <>\nstruct VectorType<2> {\n  using type = short;\n};\n\n\ntemplate <>\nstruct VectorType<4> {\n  using type = int;\n};\n\ntemplate <>\nstruct VectorType<8> {\n  using type = long long;\n};\n\ntemplate <>\nstruct VectorType<16> {\n  using type = ulonglong2;\n};\n\ntemplate <>\nstruct VectorType<32> {\n  using type = ulonglong4;\n};\n\ntemplate <typename DType>\n__device__ inline DType add_elem(const DType& x, const DType& y) {\n  return x + y;\n}\n\ntemplate <>\n__device__ inline half add_elem(const half& x, const half& y) {\n  return __float2half(__half2float(x) + __half2float(y));\n}\n\n/* \\brief Helper class that enables storing multiple values of type DType\n          as 1 value of type LType.\n*/\ntemplate <typename DType, int n>\nclass VectorizedStorage {\n public:\n  using LType = typename VectorType<sizeof(DType) * n>::type;\n  constexpr static int nvec = n;\n  union vectorized_storage {\n    LType aligned;\n    DType separate[nvec];  // NOLINT(*)\n\n    inline __device__ vectorized_storage() {}\n    inline __device__ ~vectorized_storage() {}\n  } scratch_;\n\n  inline __device__ VectorizedStorage() {}\n  inline __device__ VectorizedStorage (const VectorizedStorage<DType, n>& y2) {\n      scratch_.aligned = y2.scratch_.aligned;\n  }\n  inline __device__ VectorizedStorage (const LType &y2) {\n      scratch_.aligned = y2;\n  }\n  inline __device__ VectorizedStorage<DType, n>& operator+=(\n      const VectorizedStorage<DType, n>& rhs) {\n    #pragma unroll\n    for (int i = 0; i < nvec; ++i) {\n      scratch_.separate[i] = add_elem(scratch_.separate[i], rhs.scratch_.separate[i]);\n    }\n    return *this;\n  }\n  inline __device__ ~VectorizedStorage() {}\n};\n\n// Returns const LType is DType is const\ntemplate <typename DType, typename LType>\nstruct select_const {\n  using type = LType;\n};\n\ntemplate <typename DType, typename LType>\nstruct select_const<const DType, LType> {\n  using type = const LType;\n};\n\ntemplate <typename DType>\nstruct remove_const {\n  using type = DType;\n};\n\ntemplate <typename DType>\nstruct remove_const<const DType> {\n  using type = DType;\n};\n\n\n/* \\brief Helper class that enables accessing multiple values of type DType\n          as 1 value of type LType. Additional aligned template argument\n          allows performance optimizations if the pointer and the size of\n          the allocation is aligned to sizeof(LType) / sizeof(DType) elements.\n*/\ntemplate <typename DType, int nvec, bool aligned = false>\nclass VectorizedAccessor {\n public:\n  using StorageType = VectorizedStorage<typename remove_const<DType>::type,\n                                        nvec>;\n  using LType = typename select_const<DType, typename StorageType::LType>::type;\n  StorageType storage_;\n\n  LType* aligned_ptr_;\n  DType* unaligned_ptr_;\n  int alignment_;\n  index_t n_elems_;\n\n  inline __device__ VectorizedAccessor(DType* const ptr, const index_t size) {\n    unaligned_ptr_ = ptr;\n    if (aligned) {\n      alignment_ = 0;\n      aligned_ptr_ = reinterpret_cast<LType*>(ptr);\n      n_elems_ = (size + nvec - 1) / nvec;\n    } else {\n      size_t ptr_as_number = reinterpret_cast<size_t>(ptr);\n      alignment_ = (ptr_as_number % sizeof(LType)) / sizeof(DType);\n      aligned_ptr_ = reinterpret_cast<LType*>(ptr - alignment_);\n      n_elems_ = (size + alignment_ + nvec - 1) / nvec;\n    }\n  }\n\n  /* \\brief Alignment of the input pointer in elements. */\n  inline __device__ int alignment() const {\n    return alignment_;\n  }\n\n  /* \\brief Access to separate elements. */\n  inline __device__ DType* separate() {\n    return storage_.scratch_.separate;\n  }\n\n  /* \\brief Number of aligned elements that span the entire input tensor. */\n  inline __device__ index_t num_aligned_elements() const {\n    return n_elems_;\n  }\n\n  /* \\brief Load values from the input.\n     \\param id Aligned index of the element.\n     \\param N size of the tensor.\n  */\n  inline __device__ void load(const index_t id, const index_t N) {\n    if (aligned) {\n      storage_.scratch_.aligned = aligned_ptr_[id];\n    } else {\n      if (id > 0 && id < n_elems_ - 1) {\n        storage_.scratch_.aligned = aligned_ptr_[id];\n      } else {\n#pragma unroll\n        for (int j = 0; j < nvec; ++j) {\n          DType* ptr = reinterpret_cast<DType*>(&(aligned_ptr_[id])) + j;\n          if (reinterpret_cast<size_t>(ptr) >= reinterpret_cast<size_t>(unaligned_ptr_) &&\n              reinterpret_cast<size_t>(ptr) < reinterpret_cast<size_t>(unaligned_ptr_ + N)) {\n            storage_.scratch_.separate[j] = *ptr;\n          } else {\n            storage_.scratch_.separate[j] = DType();\n          }\n        }\n      }\n    }\n  }\n};\n\n/* \\brief Class used for vectorized read-only access. */\ntemplate <typename DType, int nvec, bool aligned = false>\nclass VectorizedLoader : public VectorizedAccessor<const DType, nvec, aligned> {\n public:\n  inline __device__ VectorizedLoader(const DType* ptr, const index_t N) :\n    VectorizedAccessor<const DType, nvec, aligned>(ptr, N) {\n  }\n};\n\n/* \\brief Class used for vectorized writable access. */\ntemplate <typename DType, int nvec, bool aligned = false>\nclass VectorizedStorer : public VectorizedAccessor<DType, nvec, aligned> {\n public:\n  inline __device__ VectorizedStorer(DType* ptr, const index_t N) :\n    VectorizedAccessor<DType, nvec, aligned>(ptr, N) {\n  }\n\n  /* \\brief Store values to the output.\n     \\param id Aligned index of the element.\n     \\param N size of the tensor.\n  */\n  inline __device__ void store(const index_t id, const index_t N) {\n    if (aligned) {\n      this->aligned_ptr_[id] = this->storage_.scratch_.aligned;\n    } else {\n      if (id > 0 && id < this->n_elems_ - 1) {\n        this->aligned_ptr_[id] = this->storage_.scratch_.aligned;\n      } else {\n#pragma unroll\n        for (int j = 0; j < nvec; ++j) {\n          DType* ptr = reinterpret_cast<DType*>(&(this->aligned_ptr_[id])) + j;\n          if (reinterpret_cast<size_t>(ptr) >= reinterpret_cast<size_t>(this->unaligned_ptr_) &&\n              reinterpret_cast<size_t>(ptr) < reinterpret_cast<size_t>(this->unaligned_ptr_ + N)) {\n            *ptr = this->storage_.scratch_.separate[j];\n          }\n        }\n      }\n    }\n  }\n};\n\n}  // namespace vector\n\n)code\";\n\nnamespace {\n\ninline index_t get_num_aligned_elements(const void* ptr,\n                                        const index_t lead_dim,\n                                        const int nvec,\n                                        const int size) {\n  size_t ptr_as_number = reinterpret_cast<size_t>(ptr);\n  int alignment        = (ptr_as_number % (nvec * size)) / size;\n  return (lead_dim + alignment + nvec - 1) / nvec;\n}\n\nenum class Alignment {\n  SAME_ALIGNED,    // All tensors aligned\n  SAME_UNALIGNED,  // All tensors have the same misalignment\n  DIFFERENT        // Tensors have different alignment\n};\n\ninline int CalcAlignment(const void* ptr, const int size) {\n  size_t ptr_as_number = reinterpret_cast<size_t>(ptr);\n  return ptr_as_number % size;\n}\n\n/* \\brief Check alignment of the inputs and outputs when using vectorized accesses.\n   \\param params Structure containing arrays with inputs' and outputs' pointers\n   \\param lead_dim Leading dimension of the tensors.\n   \\param other_dim The size of the other dimensions of the tensors.\n   \\param nvec Length of the vector.\n   \\param inputs Inputs to the operator.\n   \\param outputs Outputs of the operator.\n*/\ntemplate <typename Params>\nAlignment CheckAlignment(const Params& params,\n                         const index_t lead_dim,\n                         const index_t other_dim,\n                         const int nvec,\n                         const std::vector<TBlob>& inputs,\n                         const std::vector<TBlob>& outputs) {\n  using namespace common;\n  int align = -1;\n\n  size_t i = 0;\n  for (const void* ptr : params.inputs) {\n    if (ptr != nullptr) {\n      int new_align = CalcAlignment(ptr, mshadow_type_info(inputs[i].type_flag_).size * nvec);\n      if (align == -1) {\n        align = new_align;\n      } else {\n        if (align != new_align) {\n          return Alignment::DIFFERENT;\n        }\n      }\n    }\n    ++i;\n  }\n\n  i = 0;\n  for (const void* ptr : params.outputs) {\n    if (ptr != nullptr) {\n      int new_align = CalcAlignment(ptr, mshadow_type_info(outputs[i].type_flag_).size * nvec);\n      if (align == -1) {\n        align = new_align;\n      } else {\n        if (align != new_align) {\n          return Alignment::DIFFERENT;\n        }\n      }\n    }\n    ++i;\n  }\n\n  if ((other_dim != 1) && (lead_dim % nvec != 0)) {\n    return Alignment::DIFFERENT;\n  }\n\n  if ((align == 0) && (lead_dim % nvec == 0)) {\n    return Alignment::SAME_ALIGNED;\n  } else {\n    return Alignment::SAME_UNALIGNED;\n  }\n}\n\nconstexpr int vectorized_kernel_thread_num = 512;\n\n}  // namespace\n\n/*! \\brief Launcher helper for the kernels using vectorization.\n *  \\param parameters of the kernel (e.g. values of the template arguments)\n *  \\param kernel_name name of the kernel\n *  \\param code used for compilation of the kernel if not found in cache\n *  \\param nvec length of the vector used for loading/storing data\n *  \\param lead_dim size of leading dimension of the tensors\n *  \\param other_dim maximum of the total size of all the other dimensions of the tensors\n *  \\param s stream used to launch the kernel\n *  \\param inputs to the kernel\n *  \\param outputs of the kernel\n *  \\param dev_id id of the devide which the kernel will be launched on\n *  \\param lead_input_num number of input to use for checking alignment\n *                        (in case only a subset of inputs is used vectorized).\n *                        Default is 0.\n *  \\param blocks if provided and not 0, will launch the specified number of thread blocks.\n *                Default is 0.\n */\ntemplate <typename Params>\nvoid VectorizedKernelRTCLauncher(const std::string& parameters,\n                                 const std::string& kernel_name,\n                                 const std::string& code,\n                                 int nvec,\n                                 const index_t lead_dim,\n                                 const index_t other_dim,\n                                 mshadow::Stream<gpu>* s,\n                                 const Params params,\n                                 const std::vector<TBlob>& inputs,\n                                 const std::vector<TBlob>& outputs,\n                                 const int dev_id,\n                                 const int lead_input_num = 0,\n                                 const index_t blocks     = 0) {\n  const index_t N = lead_dim * other_dim;\n  nvec            = std::min(nvec, 4);  // Use at most 4-wide vectors\n  if (N != 0) {\n    auto align = CheckAlignment(params, lead_dim, other_dim, nvec, inputs, outputs);\n    std::string kernel_builder;\n    kernel_builder.reserve(2560);\n\n    // Fill input types\n    int counter = 0;\n    for (const auto& input : inputs) {\n      const auto& type_info = common::mshadow_type_info(input.type_flag_);\n      kernel_builder += \"using InputType\";\n      kernel_builder += std::to_string(counter);\n      kernel_builder += \" = \";\n      kernel_builder += type_info.name;\n      kernel_builder += \";\\n\";\n      ++counter;\n    }\n\n    // Fill output types\n    counter = 0;\n    for (const auto& output : outputs) {\n      const auto& type_info = common::mshadow_type_info(output.type_flag_);\n      kernel_builder += \"using OutputType\";\n      kernel_builder += std::to_string(counter);\n      kernel_builder += \" = \";\n      kernel_builder += type_info.name;\n      kernel_builder += \";\\n\";\n      ++counter;\n    }\n\n    switch (align) {\n      case Alignment::SAME_ALIGNED:\n        kernel_builder +=\n            \"const bool aligned = true;\\n\"\n            \"const int nvec = \";\n        kernel_builder += std::to_string(nvec);\n        kernel_builder += \";\\n\";\n        break;\n      case Alignment::SAME_UNALIGNED:\n        kernel_builder +=\n            \"const bool aligned = false;\\n\"\n            \"const int nvec = \";\n        kernel_builder += std::to_string(nvec);\n        kernel_builder += \";\\n\";\n        break;\n      case Alignment::DIFFERENT: {\n        // If the pointers are aligned differently we cannot vectorize\n        kernel_builder +=\n            \"const bool aligned = true;\\n\"\n            \"const int nvec = 1;\\n\";\n        nvec = 1;\n        break;\n      }\n    }\n\n    kernel_builder += parameters;\n\n    index_t num_aligned_elements =\n        get_num_aligned_elements(params.inputs[lead_input_num],\n                                 lead_dim,\n                                 nvec,\n                                 common::mshadow_type_info(inputs[lead_input_num].type_flag_).size);\n    constexpr int threads = vectorized_kernel_thread_num;\n    index_t num_blocks;\n    if (blocks != 0) {\n      num_blocks = blocks;\n    } else {\n      size_t num_elements      = other_dim * num_aligned_elements;\n      num_blocks               = (num_elements + threads - 1) / threads;\n      constexpr int max_blocks = 65535;\n      num_blocks               = std::min(static_cast<int>(num_blocks), max_blocks);\n    }\n    std::vector<const void*> args = {&params, &lead_dim, &other_dim, &N, &num_aligned_elements};\n    auto function = common::cuda::rtc::get_function(kernel_builder, kernel_name, code, dev_id);\n\n    common::cuda::rtc::launch(function,\n                              {static_cast<unsigned int>(num_blocks), 1, 1},\n                              {static_cast<unsigned int>(threads), 1, 1},\n                              0,\n                              s,\n                              &args);\n  }\n}\n\n}  // namespace rtc\n}  // namespace cuda\n}  // namespace common\n}  // namespace mxnet\n\n#endif  // MXNET_USE_CUDA\n\n#endif  // MXNET_COMMON_CUDA_RTC_VECTORIZATION_INL_H_\n"
  },
  {
    "path": "src/common/cuda/rtc.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#include \"mxnet/base.h\"\n\n#if MXNET_USE_CUDA\n\n#include <nvrtc.h>\n\n#include <mutex>\n#include <string>\n#include <fstream>\n#include <unordered_map>\n#include <vector>\n#include <tuple>\n#include <algorithm>\n\n#include \"rtc.h\"\n#include \"../../initialize.h\"\n#include \"rtc/half-inl.h\"\n#include \"rtc/util-inl.h\"\n#include \"rtc/forward_functions-inl.h\"\n#include \"rtc/backward_functions-inl.h\"\n#include \"rtc/vectorization-inl.h\"\n#include \"rtc/special_functions-inl.h\"\n#include \"rtc/reducer-inl.h\"\n#include \"utils.h\"\n\ntypedef CUresult (*cuDeviceGetPtr)(CUdevice* device, int ordinal);\ntypedef CUresult (*cuDevicePrimaryCtxRetainPtr)(CUcontext* pctx, CUdevice dev);\ntypedef CUresult (*cuModuleLoadDataExPtr)(CUmodule* module,\n                                          const void* image,\n                                          unsigned int numOptions,\n                                          CUjit_option* options,\n                                          void** optionValues);\ntypedef CUresult (*cuModuleGetFunctionPtr)(CUfunction* hfunc, CUmodule hmod, const char* name);\ntypedef CUresult (*cuLaunchKernelPtr)(CUfunction f,\n                                      unsigned int gridDimX,\n                                      unsigned int gridDimY,\n                                      unsigned int gridDimZ,\n                                      unsigned int blockDimX,\n                                      unsigned int blockDimY,\n                                      unsigned int blockDimZ,\n                                      unsigned int sharedMemBytes,\n                                      CUstream hStream,\n                                      void** kernelParams,\n                                      void** extra);\ntypedef CUresult (*cuGetErrorStringPtr)(CUresult error, const char** pStr);\n\nnamespace mxnet {\nnamespace common {\nnamespace cuda {\nnamespace rtc {\n\n#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)\nconst char cuda_lib_name[] = \"nvcuda.dll\";\n#else\nconst char cuda_lib_name[] = \"libcuda.so.1\";\n#endif\n\nstd::mutex lock;\n\nnamespace util {\n\nstd::string to_string(OpReqType req) {\n  switch (req) {\n    case kNullOp:\n      return \"OpReqType::kNullOp\";\n    case kWriteTo:\n    case kWriteInplace:\n      return \"OpReqType::kWriteTo\";\n    case kAddTo:\n      return \"OpReqType::kAddTo\";\n  }\n  LOG(FATAL) << \"Unrecognized req.\";\n  return \"\";\n}\n\n}  // namespace util\n\nint GetMaxSupportedArch() {\n#if CUDA_VERSION < 10000\n  constexpr int max_supported_sm_arch = 72;\n#elif CUDA_VERSION < 11000\n  constexpr int max_supported_sm_arch = 75;\n#elif CUDA_VERSION < 11010\n  constexpr int max_supported_sm_arch = 80;\n#elif CUDA_VERSION < 11020\n  constexpr int max_supported_sm_arch = 86;\n#else\n  // starting with cuda 11.2, nvrtc can report the max supported arch,\n  // removing the need to update this routine with each new cuda version.\n  static int max_supported_sm_arch = []() {\n    int num_archs = 0;\n    NVRTC_CALL(nvrtcGetNumSupportedArchs(&num_archs));\n    std::vector<int> archs(num_archs);\n    if (num_archs > 0) {\n      NVRTC_CALL(nvrtcGetSupportedArchs(archs.data()));\n    } else {\n      LOG(FATAL) << \"Could not determine supported cuda archs.\";\n    }\n    return archs[num_archs - 1];\n  }();\n#endif\n  return max_supported_sm_arch;\n}\n\nnamespace {\n\n// Obtain compilation log from the program.\nstd::string GetCompileLog(nvrtcProgram program) {\n  size_t log_size_including_null;\n  NVRTC_CALL(nvrtcGetProgramLogSize(program, &log_size_including_null));\n  std::string log(log_size_including_null - 1, '\\0');\n  // Room for terminating null character ensured since C++11\n  NVRTC_CALL(nvrtcGetProgramLog(program, &log[0]));\n  return log;\n}\n\n// Obtain compilation result (ptx assembly) from the program.\nstd::string GetCompiledCode(nvrtcProgram program, bool use_cubin) {\n#if CUDA_VERSION >= 11010\n  const auto getSize = use_cubin ? nvrtcGetCUBINSize : nvrtcGetPTXSize;\n  const auto getFunc = use_cubin ? nvrtcGetCUBIN : nvrtcGetPTX;\n#else\n  const auto getSize                  = nvrtcGetPTXSize;\n  const auto getFunc                  = nvrtcGetPTX;\n#endif\n  size_t ptx_size_including_null;\n  NVRTC_CALL(getSize(program, &ptx_size_including_null));\n  std::string ptx(ptx_size_including_null - 1, '\\0');\n  // Room for terminating null character ensured since C++11\n  NVRTC_CALL(getFunc(program, &ptx[0]));\n  return ptx;\n}\n\nstd::tuple<bool, std::string> GetArchString(const int sm_arch) {\n  const int sm_arch_as_used = std::min(sm_arch, GetMaxSupportedArch());\n  // Always use PTX for CUDA <= 11.0\n  const bool known_arch = (CUDA_VERSION > 11000) && (sm_arch == sm_arch_as_used);\n  if (known_arch) {\n    return {known_arch, \"sm_\" + std::to_string(sm_arch_as_used)};\n  } else {\n    return {known_arch, \"compute_\" + std::to_string(sm_arch_as_used)};\n  }\n}\n\n}  // namespace\n\nCUfunction get_function(const std::string& parameters,\n                        const std::string& kernel_name,\n                        const std::string& code,\n                        int dev_id) {\n  constexpr int CACHESIZE_WARN_THRESHOLD = 10000;\n  std::lock_guard<std::mutex> l(lock);\n  // Local class for value type of compile cache\n  struct KernelInfo {\n    std::string mangled_name;\n    std::string ptx;\n    std::vector<CUfunction> functions;\n  };\n  void* cuda_lib_handle = LibraryInitializer::Get()->lib_load(cuda_lib_name);\n\n  // Maps from the kernel name and parameters to the ptx and jit-compiled CUfunctions.\n  using KernelCache = std::unordered_map<std::string, KernelInfo>;\n  // Per-gpu-architecture compiled kernel cache with jit-compiled function for each device context\n  static std::unordered_map<int32_t, KernelCache> compiled_kernels;\n  int sm_arch = SMArch(dev_id);\n  // make null map as needed\n  KernelCache& compiled_kernels_this_arch = compiled_kernels[sm_arch];\n  // make KernelInfo as needed\n  KernelInfo& kinfo = compiled_kernels_this_arch[parameters + kernel_name];\n  if (kinfo.ptx.size() == 0) {\n    // It's the first time we've seen this kernel, so we need to generate the ptx and mangled_name.\n    static std::string common_header =\n        std::string(fp16_support_string) + \"\\n\" + type_support_string + \"\\n\" + util_string + \"\\n\" +\n        limits + \"\\n\" + special_functions_definitions + '\\n' + vectorization_support_string + \"\\n\" +\n        function_definitions_util + \"\\n\" + function_definitions_binary + \"\\n\" +\n        function_definitions_unary + \"\\n\" + backward_function_definitions + \"\\n\" +\n        grad_function_definitions + \"\\n\" + reducer + \"\\n\" + logic_reducer + \"\\n\";\n    std::string code_with_header = common_header + parameters + code;\n    // If verbose mode, output kernel source, though not including the common header\n    if (dmlc::GetEnv(\"MXNET_RTC_VERBOSE\", false)) {\n      LOG(INFO) << \"\\n\" << std::string(80, '-') << \"\\n\" << (parameters + code);\n    }\n    if (compiled_kernels_this_arch.size() == CACHESIZE_WARN_THRESHOLD + 1 &&\n        dmlc::GetEnv(\"MXNET_RTC_SIZE_WARNING\", true)) {\n      LOG(WARNING) << \"The number of different compiled kernels exceeds \"\n                   << CACHESIZE_WARN_THRESHOLD\n                   << \".  Set MXNET_RTC_SIZE_WARNING=0 to quiet this warning.\";\n    }\n    nvrtcProgram program;\n    NVRTC_CALL(nvrtcCreateProgram(&program,                              // prog\n                                  &code_with_header[0],                  // buffer\n                                  (kernel_name + \"_kernel.cu\").c_str(),  // name\n                                  0,                                     // num headers\n                                  nullptr,                               // headers\n                                  nullptr));                             // include names\n    const auto [use_cubin, gpu_arch] = GetArchString(sm_arch);           // NOLINT(*)\n    std::string gpu_arch_arg         = \"--gpu-architecture=\" + gpu_arch;\n    const char* opts[]               = {\n      gpu_arch_arg.c_str(),\n#if NDEBUG == 0\n      \"-G\",\n#endif\n      \"--std=c++14\"\n    };\n    const std::string& kernel_name_demangled = kernel_name;\n    NVRTC_CALL(nvrtcAddNameExpression(program, (kernel_name_demangled).c_str()));\n\n    nvrtcResult compileResult          = nvrtcCompileProgram(program,  // prog\n                                                    sizeof(opts) / sizeof(opts[0]),  // num options\n                                                    opts);  // options\n    static const std::string dump_file = \"mxnet_rtc_debug_code.log\";\n    if (compileResult != NVRTC_SUCCESS) {\n      std::ofstream f(dump_file);\n      f << code_with_header;\n      f.close();\n    }\n    CHECK_EQ(compileResult, NVRTC_SUCCESS)\n        << \"NVRTC Compilation failed.\\n\"\n        << \"The generated code was stored in \" << dump_file << \"\\n\"\n        << GetCompileLog(program);\n\n    kinfo.ptx = GetCompiledCode(program, use_cubin);\n    const char* mangled_name;\n    NVRTC_CALL(nvrtcGetLoweredName(program, kernel_name_demangled.c_str(), &mangled_name));\n    kinfo.mangled_name = mangled_name;\n    // Destroy the program.\n    NVRTC_CALL(nvrtcDestroyProgram(&program));\n  }\n  // Ensure function array is deep enough to index by dev_id\n  while (kinfo.functions.size() <= static_cast<size_t>(dev_id))\n    kinfo.functions.push_back(static_cast<CUfunction>(nullptr));\n  // Jit-compile ptx for the device as needed\n  if (kinfo.functions[dev_id] == static_cast<CUfunction>(nullptr)) {\n    // Make sure driver context is set to the proper device\n    CUdevice cu_device;\n    CUcontext context;\n    cuDeviceGetPtr device_get_ptr = get_func<cuDeviceGetPtr>(cuda_lib_handle, \"cuDeviceGet\");\n    CUDA_DRIVER_CALL((*device_get_ptr)(&cu_device, dev_id));\n    cuDevicePrimaryCtxRetainPtr device_primary_ctx_retain_ptr =\n        get_func<cuDevicePrimaryCtxRetainPtr>(cuda_lib_handle, \"cuDevicePrimaryCtxRetain\");\n    CUDA_DRIVER_CALL((*device_primary_ctx_retain_ptr)(&context, cu_device));\n\n    // Jit-compile ptx for the driver's current context\n    CUmodule module;\n\n#if NDEBUG == 0\n    intptr_t debug_info = 1;\n    intptr_t line_info  = 1;\n#else\n    intptr_t debug_info = 0;\n    intptr_t line_info  = 0;\n#endif\n\n    CUjit_option jit_opts[] = {CU_JIT_GENERATE_DEBUG_INFO, CU_JIT_GENERATE_LINE_INFO};\n    void* jit_opt_values[]  = {reinterpret_cast<void*>(debug_info),\n                              reinterpret_cast<void*>(line_info)};\n\n    cuModuleLoadDataExPtr module_load_data_ex_ptr =\n        get_func<cuModuleLoadDataExPtr>(cuda_lib_handle, \"cuModuleLoadDataEx\");\n    CUDA_DRIVER_CALL(\n        (*module_load_data_ex_ptr)(&module, kinfo.ptx.c_str(), 2, jit_opts, jit_opt_values));\n    cuModuleGetFunctionPtr module_get_function_ptr =\n        get_func<cuModuleGetFunctionPtr>(cuda_lib_handle, \"cuModuleGetFunction\");\n    CUDA_DRIVER_CALL(\n        (*module_get_function_ptr)(&kinfo.functions[dev_id], module, kinfo.mangled_name.c_str()));\n  }\n  return kinfo.functions[dev_id];\n}\n\nvoid launch(CUfunction function,\n            const dim3 grid_dim,\n            const dim3 block_dim,\n            unsigned int shared_mem_bytes,\n            mshadow::Stream<gpu>* stream,\n            std::vector<const void*>* args) {\n  CHECK(args->size() != 0) << \"Empty argument list passed to a kernel.\";\n  void* cuda_lib_handle = LibraryInitializer::Get()->lib_load(cuda_lib_name);\n  cuLaunchKernelPtr launch_kernel_ptr =\n      get_func<cuLaunchKernelPtr>(cuda_lib_handle, \"cuLaunchKernel\");\n  CUresult err = (*launch_kernel_ptr)(function,  // function to launch\n                                      grid_dim.x,\n                                      grid_dim.y,\n                                      grid_dim.z,  // grid dim\n                                      block_dim.x,\n                                      block_dim.y,\n                                      block_dim.z,                              // block dim\n                                      shared_mem_bytes,                         // shared memory\n                                      mshadow::Stream<gpu>::GetStream(stream),  // stream\n                                      const_cast<void**>(args->data()),         // arguments\n                                      nullptr);                                 // );\n  if (err != CUDA_SUCCESS) {\n    const char* error_string;\n    cuGetErrorStringPtr get_error_string_ptr =\n        get_func<cuGetErrorStringPtr>(cuda_lib_handle, \"cuGetErrorString\");\n    (*get_error_string_ptr)(err, &error_string);\n    LOG(FATAL) << \"cuLaunchKernel failed: \" << err << \" \" << error_string << \": \"\n               << reinterpret_cast<void*>(function) << \" \"\n               << \"(\" << grid_dim.x << \", \" << grid_dim.y << \", \" << grid_dim.z << \") \"\n               << \"(\" << block_dim.x << \", \" << block_dim.y << \", \" << block_dim.z << \") \"\n               << shared_mem_bytes << \" \" << args->size();\n  }\n}\n\n}  // namespace rtc\n}  // namespace cuda\n}  // namespace common\n}  // namespace mxnet\n\n#endif  // MXNET_USE_CUDA\n"
  },
  {
    "path": "src/common/cuda/rtc.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file cuda_rtc.h\n * \\brief Common CUDA utilities for\n *        runtime compilation.\n */\n\n#ifndef MXNET_COMMON_CUDA_RTC_H_\n#define MXNET_COMMON_CUDA_RTC_H_\n\n#include \"mxnet/base.h\"\n#include \"mxnet/op_attr_types.h\"\n\n#if MXNET_USE_CUDA\n\n#include <cuda.h>\n#include <cuda_runtime_api.h>\n\n#include <mutex>\n#include <string>\n#include <vector>\n\nnamespace mxnet {\nnamespace common {\nnamespace cuda {\nnamespace rtc {\n\nnamespace util {\n\n/*! \\brief Convert OpReqType to string.\n *  \\param req to convert\n */\nstd::string to_string(OpReqType req);\n\n}  // namespace util\n\nint GetMaxSupportedArch();\n\nextern std::mutex lock;\n\n/*! \\brief Compile and get the GPU kernel. Uses cache in order to\n *         eliminate the overhead of compilation.\n *  \\param parameters of the kernel (e.g. values of the template arguments, types used)\n *  \\param kernel_name name of the kernel\n *  \\param code used for compilation of the kernel if not found in cache\n *  \\param dev_id id of the device which the kernel will be launched on\n */\nCUfunction get_function(const std::string& parameters,\n                        const std::string& kernel_name,\n                        const std::string& code,\n                        int dev_id);\n\n/*! \\brief Launch a GPU kernel.\n *  \\param function to launch\n *  \\param grid_dim grid dimensions\n *  \\param block_dim block dimensions\n *  \\param shared_mem_bytes amount of dynamic shared memory needed by the kernel\n *  \\param stream used for launching the kernel\n *  \\param args arguments of the kernel\n */\nvoid launch(CUfunction function,\n            const dim3 grid_dim,\n            const dim3 block_dim,\n            unsigned int shared_mem_bytes,\n            mshadow::Stream<gpu>* stream,\n            std::vector<const void*>* args);\n\n}  // namespace rtc\n}  // namespace cuda\n}  // namespace common\n}  // namespace mxnet\n\n#endif  // MXNET_USE_CUDA\n\n#endif  // MXNET_COMMON_CUDA_RTC_H_\n"
  },
  {
    "path": "src/common/cuda/utils.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file cuda_utils.cc\n * \\brief Common CUDA utilities.\n */\n\n#include <mxnet/base.h>\n#include <mshadow/base.h>\n\n#include <algorithm>\n\n#include \"utils.h\"\n#include \"../utils.h\"\n\n#if MXNET_USE_CUDA\n\nnamespace mxnet {\nnamespace common {\nnamespace cuda {\n\nint get_load_type(size_t N) {\n  using namespace mshadow;\n  if (N % 8 == 0) {\n    return kFloat64;\n  } else if (N % 4 == 0) {\n    return kFloat32;\n  } else if (N % 2 == 0) {\n    return kFloat16;\n  } else {\n    return kUint8;\n  }\n}\n\nint get_rows_per_block(size_t row_size, int num_threads_per_block) {\n  const int warp_size = 32;\n  CHECK(IsPower2(num_threads_per_block))\n      << \"Number of threads in a block must be power of 2 to use get_rows_per_block function\";\n  // How many read instructions should 1 thread at least do\n  const int read_instructions           = 2;\n  const int desired_num_threads_per_row = (row_size + read_instructions - 1) / read_instructions;\n  int desired_num_warps_per_row         = (desired_num_threads_per_row + warp_size - 1) / warp_size;\n  int actual_num_warps_per_row =\n      std::min(desired_num_warps_per_row, num_threads_per_block / warp_size);\n  // actual number of warps needs to be power of 2\n  actual_num_warps_per_row = RoundToPower2(actual_num_warps_per_row);\n  return num_threads_per_block / (warp_size * actual_num_warps_per_row);\n}\n\n}  // namespace cuda\n}  // namespace common\n}  // namespace mxnet\n\n#endif  // MXNET_USE_CUDA\n"
  },
  {
    "path": "src/common/cuda/utils.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file utils.h\n * \\brief Common CUDA utilities.\n */\n#ifndef MXNET_COMMON_CUDA_UTILS_H_\n#define MXNET_COMMON_CUDA_UTILS_H_\n\n#include <dmlc/logging.h>\n#include <dmlc/parameter.h>\n#include <dmlc/optional.h>\n#include <mshadow/base.h>\n#include <mxnet/libinfo.h>\n\n/*! \\brief Macros/inlines to assist CLion to parse Cuda files (*.cu, *.cuh) */\n#ifdef __JETBRAINS_IDE__\n#define __CUDACC__ 1\n#define __host__\n#define __device__\n#define __global__\n#define __forceinline__\n#define __shared__\ninline void __syncthreads() {}\ninline void __threadfence_block() {}\ntemplate <class T>\ninline T __clz(const T val) {\n  return val;\n}\nstruct __cuda_fake_struct {\n  int x;\n  int y;\n  int z;\n};\nextern __cuda_fake_struct blockDim;\nextern __cuda_fake_struct threadIdx;\nextern __cuda_fake_struct blockIdx;\n#endif\n\n#define QUOTE(x)      #x\n#define QUOTEVALUE(x) QUOTE(x)\n\n#if MXNET_USE_CUDA\n\n#include <cuda_runtime.h>\n#include <cublas_v2.h>\n#include <curand.h>\n#if MXNET_USE_NVML\n#include <nvml.h>\n#endif  // MXNET_USE_NVML\n\n#include <vector>\n\n#define STATIC_ASSERT_CUDA_VERSION_GE(min_version) \\\n  static_assert(CUDA_VERSION >= min_version, \"Compiled-against CUDA version \" \\\n      QUOTEVALUE(CUDA_VERSION) \" is too old, please upgrade system to version \" \\\n      QUOTEVALUE(min_version) \" or later.\")\n\n/*!\n * \\brief When compiling a __device__ function, check that the architecture is >= Kepler (3.0)\n *        Note that __CUDA_ARCH__ is not defined outside of a __device__ function\n */\n#ifdef __CUDACC__\ninline __device__ bool __is_supported_cuda_architecture() {\n#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 300\n#error \"Fermi and earlier GPU architectures are not supported (architecture versions less than 3.0)\"\n  return false;\n#else\n  return true;\n#endif  // __CUDA_ARCH__ < 300\n}\n#endif  // __CUDACC__\n\n/*!\n * \\brief Check CUDA error.\n * \\param msg Message to print if an error occured.\n */\n#define CHECK_CUDA_ERROR(msg)                                                \\\n  {                                                                          \\\n    cudaError_t e = cudaGetLastError();                                      \\\n    CHECK_EQ(e, cudaSuccess) << (msg) << \" CUDA: \" << cudaGetErrorString(e); \\\n  }\n\n/*!\n * \\brief Protected CUDA call.\n * \\param func Expression to call.\n *\n * It checks for CUDA errors after invocation of the expression.\n */\n#define CUDA_CALL(func)                                                                            \\\n  {                                                                                                \\\n    cudaError_t e = (func);                                                                        \\\n    CHECK(e == cudaSuccess || e == cudaErrorCudartUnloading) << \"CUDA: \" << cudaGetErrorString(e); \\\n  }\n\n/*!\n * \\brief Protected cuBLAS call.\n * \\param func Expression to call.\n *\n * It checks for cuBLAS errors after invocation of the expression.\n */\n#define CUBLAS_CALL(func)                                              \\\n  {                                                                    \\\n    cublasStatus_t e = (func);                                         \\\n    CHECK_EQ(e, CUBLAS_STATUS_SUCCESS)                                 \\\n        << \"cuBLAS: \" << mxnet::common::cuda::CublasGetErrorString(e); \\\n  }\n\n/*!\n * \\brief Protected cuSolver call.\n * \\param func Expression to call.\n *\n * It checks for cuSolver errors after invocation of the expression.\n */\n#define CUSOLVER_CALL(func)                                                \\\n  {                                                                        \\\n    cusolverStatus_t e = (func);                                           \\\n    CHECK_EQ(e, CUSOLVER_STATUS_SUCCESS)                                   \\\n        << \"cuSolver: \" << mxnet::common::cuda::CusolverGetErrorString(e); \\\n  }\n\n/*!\n * \\brief Protected cuRAND call.\n * \\param func Expression to call.\n *\n * It checks for cuRAND errors after invocation of the expression.\n */\n#define CURAND_CALL(func)                                              \\\n  {                                                                    \\\n    curandStatus_t e = (func);                                         \\\n    CHECK_EQ(e, CURAND_STATUS_SUCCESS)                                 \\\n        << \"cuRAND: \" << mxnet::common::cuda::CurandGetErrorString(e); \\\n  }\n\n/*!\n * \\brief Protected NVRTC call.\n * \\param func Expression to call.\n *\n * It checks for NVRTC errors after invocation of the expression.\n */\n#define NVRTC_CALL(x)                                                                           \\\n  {                                                                                             \\\n    nvrtcResult result = x;                                                                     \\\n    CHECK_EQ(result, NVRTC_SUCCESS) << #x \" failed with error \" << nvrtcGetErrorString(result); \\\n  }\n\n/*!\n * \\brief Protected CUDA driver call.\n * \\param func Expression to call.\n *\n * It checks for CUDA driver errors after invocation of the expression.\n */\n#define CUDA_DRIVER_CALL(func)                                         \\\n  {                                                                    \\\n    CUresult e = (func);                                               \\\n    if (e != CUDA_SUCCESS) {                                           \\\n      char const* err_msg = nullptr;                                   \\\n      if (cuGetErrorString(e, &err_msg) == CUDA_ERROR_INVALID_VALUE) { \\\n        LOG(FATAL) << \"CUDA Driver: Unknown error \" << e;              \\\n      } else {                                                         \\\n        LOG(FATAL) << \"CUDA Driver: \" << e << \" \" << err_msg;          \\\n      }                                                                \\\n    }                                                                  \\\n  }\n\n#if MXNET_USE_NVML\n/*!\n * \\brief Protected NVML call.\n * \\param func Expression to call.\n *\n * It checks for NVML errors after invocation of the expression.\n */\n#define NVML_CALL(func)                                                                       \\\n  {                                                                                           \\\n    nvmlReturn_t result = (func);                                                             \\\n    CHECK_EQ(result, NVML_SUCCESS) << #func \" failed with error \" << nvmlErrorString(result); \\\n  }\n#endif  // MXNET_USE_NVML\n\n#if !defined(_MSC_VER)\n#define CUDA_UNROLL   _Pragma(\"unroll\")\n#define CUDA_NOUNROLL _Pragma(\"nounroll\")\n#else\n#define CUDA_UNROLL\n#define CUDA_NOUNROLL\n#endif\n\nnamespace mxnet {\nnamespace common {\n/*! \\brief common utils for cuda */\nnamespace cuda {\n/*!\n * \\brief Converts between C++ datatypes and enums/constants needed by cuBLAS.\n */\ntemplate <typename DType>\nstruct CublasType;\n\n// With CUDA v8, cuBLAS adopted use of cudaDataType_t instead of its own\n// datatype cublasDataType_t.  The older cudaDataType_t values could be\n// included below, but since this class was introduced to support the cuBLAS v8\n// call cublasGemmEx(), burdening the class with the legacy type values\n// was not needed.\n\ntemplate <>\nstruct CublasType<float> {\n  static const int kFlag = mshadow::kFloat32;\n#if CUDA_VERSION >= 8000\n  static const cudaDataType_t kCudaFlag = CUDA_R_32F;\n#endif\n  typedef float ScaleType;\n  static const float one;\n  static const float zero;\n};\ntemplate <>\nstruct CublasType<double> {\n  static const int kFlag = mshadow::kFloat64;\n#if CUDA_VERSION >= 8000\n  static const cudaDataType_t kCudaFlag = CUDA_R_64F;\n#endif\n  typedef double ScaleType;\n  static const double one;\n  static const double zero;\n};\ntemplate <>\nstruct CublasType<mshadow::half::half_t> {\n  static const int kFlag = mshadow::kFloat16;\n#if CUDA_VERSION >= 8000\n  static const cudaDataType_t kCudaFlag = CUDA_R_16F;\n#endif\n  typedef float ScaleType;\n  static const mshadow::half::half_t one;\n  static const mshadow::half::half_t zero;\n};\ntemplate <>\nstruct CublasType<uint8_t> {\n  static const int kFlag = mshadow::kUint8;\n#if CUDA_VERSION >= 8000\n  static const cudaDataType_t kCudaFlag = CUDA_R_8I;\n#endif\n  typedef uint8_t ScaleType;\n  static const uint8_t one  = 1;\n  static const uint8_t zero = 0;\n};\ntemplate <>\nstruct CublasType<int32_t> {\n  static const int kFlag = mshadow::kInt32;\n#if CUDA_VERSION >= 8000\n  static const cudaDataType_t kCudaFlag = CUDA_R_32I;\n#endif\n  typedef int32_t ScaleType;\n  static const int32_t one  = 1;\n  static const int32_t zero = 0;\n};\n\n/*!\n * \\brief Get string representation of cuBLAS errors.\n * \\param error The error.\n * \\return String representation.\n */\ninline const char* CublasGetErrorString(cublasStatus_t error) {\n  switch (error) {\n    case CUBLAS_STATUS_SUCCESS:\n      return \"CUBLAS_STATUS_SUCCESS\";\n    case CUBLAS_STATUS_NOT_INITIALIZED:\n      return \"CUBLAS_STATUS_NOT_INITIALIZED\";\n    case CUBLAS_STATUS_ALLOC_FAILED:\n      return \"CUBLAS_STATUS_ALLOC_FAILED\";\n    case CUBLAS_STATUS_INVALID_VALUE:\n      return \"CUBLAS_STATUS_INVALID_VALUE\";\n    case CUBLAS_STATUS_ARCH_MISMATCH:\n      return \"CUBLAS_STATUS_ARCH_MISMATCH\";\n    case CUBLAS_STATUS_MAPPING_ERROR:\n      return \"CUBLAS_STATUS_MAPPING_ERROR\";\n    case CUBLAS_STATUS_EXECUTION_FAILED:\n      return \"CUBLAS_STATUS_EXECUTION_FAILED\";\n    case CUBLAS_STATUS_INTERNAL_ERROR:\n      return \"CUBLAS_STATUS_INTERNAL_ERROR\";\n    case CUBLAS_STATUS_NOT_SUPPORTED:\n      return \"CUBLAS_STATUS_NOT_SUPPORTED\";\n    default:\n      break;\n  }\n  return \"Unknown cuBLAS status\";\n}\n\n#if CUDA_VERSION >= 8000\n/*!\n * \\brief Create the proper constant for indicating cuBLAS transposition, if desired.\n * \\param transpose Whether transposition should be performed.\n * \\return the yes/no transposition-indicating constant.\n */\ninline cublasOperation_t CublasTransposeOp(bool transpose) {\n  return transpose ? CUBLAS_OP_T : CUBLAS_OP_N;\n}\n#endif\n\n/*!\n * \\brief Get string representation of cuSOLVER errors.\n * \\param error The error.\n * \\return String representation.\n */\ninline const char* CusolverGetErrorString(cusolverStatus_t error) {\n  switch (error) {\n    case CUSOLVER_STATUS_SUCCESS:\n      return \"CUSOLVER_STATUS_SUCCESS\";\n    case CUSOLVER_STATUS_NOT_INITIALIZED:\n      return \"CUSOLVER_STATUS_NOT_INITIALIZED\";\n    case CUSOLVER_STATUS_ALLOC_FAILED:\n      return \"CUSOLVER_STATUS_ALLOC_FAILED\";\n    case CUSOLVER_STATUS_INVALID_VALUE:\n      return \"CUSOLVER_STATUS_INVALID_VALUE\";\n    case CUSOLVER_STATUS_ARCH_MISMATCH:\n      return \"CUSOLVER_STATUS_ARCH_MISMATCH\";\n    case CUSOLVER_STATUS_EXECUTION_FAILED:\n      return \"CUSOLVER_STATUS_EXECUTION_FAILED\";\n    case CUSOLVER_STATUS_INTERNAL_ERROR:\n      return \"CUSOLVER_STATUS_INTERNAL_ERROR\";\n    case CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED:\n      return \"CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED\";\n    default:\n      break;\n  }\n  return \"Unknown cuSOLVER status\";\n}\n\n/*!\n * \\brief Get string representation of cuRAND errors.\n * \\param status The status.\n * \\return String representation.\n */\ninline const char* CurandGetErrorString(curandStatus_t status) {\n  switch (status) {\n    case CURAND_STATUS_SUCCESS:\n      return \"CURAND_STATUS_SUCCESS\";\n    case CURAND_STATUS_VERSION_MISMATCH:\n      return \"CURAND_STATUS_VERSION_MISMATCH\";\n    case CURAND_STATUS_NOT_INITIALIZED:\n      return \"CURAND_STATUS_NOT_INITIALIZED\";\n    case CURAND_STATUS_ALLOCATION_FAILED:\n      return \"CURAND_STATUS_ALLOCATION_FAILED\";\n    case CURAND_STATUS_TYPE_ERROR:\n      return \"CURAND_STATUS_TYPE_ERROR\";\n    case CURAND_STATUS_OUT_OF_RANGE:\n      return \"CURAND_STATUS_OUT_OF_RANGE\";\n    case CURAND_STATUS_LENGTH_NOT_MULTIPLE:\n      return \"CURAND_STATUS_LENGTH_NOT_MULTIPLE\";\n    case CURAND_STATUS_DOUBLE_PRECISION_REQUIRED:\n      return \"CURAND_STATUS_DOUBLE_PRECISION_REQUIRED\";\n    case CURAND_STATUS_LAUNCH_FAILURE:\n      return \"CURAND_STATUS_LAUNCH_FAILURE\";\n    case CURAND_STATUS_PREEXISTING_FAILURE:\n      return \"CURAND_STATUS_PREEXISTING_FAILURE\";\n    case CURAND_STATUS_INITIALIZATION_FAILED:\n      return \"CURAND_STATUS_INITIALIZATION_FAILED\";\n    case CURAND_STATUS_ARCH_MISMATCH:\n      return \"CURAND_STATUS_ARCH_MISMATCH\";\n    case CURAND_STATUS_INTERNAL_ERROR:\n      return \"CURAND_STATUS_INTERNAL_ERROR\";\n  }\n  return \"Unknown cuRAND status\";\n}\n\ntemplate <typename DType>\ninline DType __device__ CudaMax(DType a, DType b) {\n  return a > b ? a : b;\n}\n\ntemplate <typename DType>\ninline DType __device__ CudaMin(DType a, DType b) {\n  return a < b ? a : b;\n}\n\nclass DeviceStore {\n public:\n  /*! \\brief default constructor- only optionally restores previous device */\n  explicit DeviceStore(int requested_device = -1, bool restore = true)\n      : restore_device_(-1), current_device_(requested_device), restore_(restore) {\n    if (restore_)\n      CUDA_CALL(cudaGetDevice(&restore_device_));\n    if (requested_device != restore_device_) {\n      SetDevice(requested_device);\n    }\n  }\n\n  ~DeviceStore() {\n    if (restore_ && current_device_ != restore_device_ && current_device_ != -1 &&\n        restore_device_ != -1)\n      CUDA_CALL(cudaSetDevice(restore_device_));\n  }\n\n  void SetDevice(int device) {\n    if (device != -1) {\n      CUDA_CALL(cudaSetDevice(device));\n      current_device_ = device;\n    }\n  }\n\n private:\n  int restore_device_;\n  int current_device_;\n  bool restore_;\n};\n\n/*!\n * \\brief Get the largest datatype suitable to read\n *         requested number of bytes.\n *\n *  \\input Number of bytes to be read\n *  \\return mshadow representation of type that could\n *          be used for reading\n */\nint get_load_type(size_t N);\n\n/*!\n * \\brief Determine how many rows in a 2D matrix should a block\n *        of threads handle based on the row size and the number\n *        of threads in a block.\n * \\param row_size Size of the row expressed in the number of reads required to fully\n *                 load it. For example, if the row has N elements, but  each thread\n *                 reads 2 elements with a single read, row_size should be N / 2.\n * \\param num_threads_per_block Number of threads in a block.\n * \\return the number of rows that should be handled by a single block.\n */\nint get_rows_per_block(size_t row_size, int num_threads_per_block);\n\n}  // namespace cuda\n}  // namespace common\n}  // namespace mxnet\n\n/*! \\brief Maximum number of GPUs */\nconstexpr size_t kMaxNumGpus = 64;\n\n// The implementations below assume that accesses of 32-bit ints are inherently atomic and\n// can be read/written by multiple threads without locks.  The values held should be < 2^31.\n\n/*!\n * \\brief Return an attribute GPU `device_id`.\n * \\param device_id The device index of the cuda-capable gpu of interest.\n * \\param cached_values An array of attributes for already-looked-up GPUs.\n * \\param attr The attribute, by number.\n * \\param attr_name A string representation of the attribute, for error messages.\n * \\return the gpu's attribute value.\n */\ninline int cudaAttributeLookup(int device_id,\n                               std::vector<int32_t>* cached_values,\n                               cudaDeviceAttr attr,\n                               const char* attr_name) {\n  if (device_id < 0 || device_id >= static_cast<int>(cached_values->size())) {\n    LOG(FATAL) << attr_name << \"(device_id) called with invalid id: \" << device_id;\n  } else if ((*cached_values)[device_id] < 0) {\n    int temp = -1;\n    CUDA_CALL(cudaDeviceGetAttribute(&temp, attr, device_id));\n    (*cached_values)[device_id] = static_cast<int32_t>(temp);\n  }\n  return (*cached_values)[device_id];\n}\n\n/*!\n * \\brief Determine major version number of the gpu's cuda compute architecture.\n * \\param device_id The device index of the cuda-capable gpu of interest.\n * \\return the major version number of the gpu's cuda compute architecture.\n */\ninline int ComputeCapabilityMajor(int device_id) {\n  static std::vector<int32_t> capability_major(kMaxNumGpus, -1);\n  return cudaAttributeLookup(\n      device_id, &capability_major, cudaDevAttrComputeCapabilityMajor, \"ComputeCapabilityMajor\");\n}\n\n/*!\n * \\brief Determine minor version number of the gpu's cuda compute architecture.\n * \\param device_id The device index of the cuda-capable gpu of interest.\n * \\return the minor version number of the gpu's cuda compute architecture.\n */\ninline int ComputeCapabilityMinor(int device_id) {\n  static std::vector<int32_t> capability_minor(kMaxNumGpus, -1);\n  return cudaAttributeLookup(\n      device_id, &capability_minor, cudaDevAttrComputeCapabilityMinor, \"ComputeCapabilityMinor\");\n}\n\n/*!\n * \\brief Return the integer SM architecture (e.g. Volta = 70).\n * \\param device_id The device index of the cuda-capable gpu of interest.\n * \\return the gpu's cuda compute architecture as an int.\n */\ninline int SMArch(int device_id) {\n  auto major = ComputeCapabilityMajor(device_id);\n  auto minor = ComputeCapabilityMinor(device_id);\n  return 10 * major + minor;\n}\n\n/*!\n * \\brief Return the number of streaming multiprocessors of GPU `device_id`.\n * \\param device_id The device index of the cuda-capable gpu of interest.\n * \\return the gpu's count of streaming multiprocessors.\n */\ninline int MultiprocessorCount(int device_id) {\n  static std::vector<int32_t> sm_counts(kMaxNumGpus, -1);\n  return cudaAttributeLookup(\n      device_id, &sm_counts, cudaDevAttrMultiProcessorCount, \"MultiprocessorCount\");\n}\n\n/*!\n * \\brief Return the shared memory size in bytes of each of the GPU's streaming multiprocessors.\n * \\param device_id The device index of the cuda-capable gpu of interest.\n * \\return the shared memory size per streaming multiprocessor.\n */\ninline int MaxSharedMemoryPerMultiprocessor(int device_id) {\n  static std::vector<int32_t> max_smem_per_mutiprocessor(kMaxNumGpus, -1);\n  return cudaAttributeLookup(device_id,\n                             &max_smem_per_mutiprocessor,\n                             cudaDevAttrMaxSharedMemoryPerMultiprocessor,\n                             \"MaxSharedMemoryPerMultiprocessor\");\n}\n\n/*!\n * \\brief Return whether the GPU `device_id` supports cooperative-group kernel launching.\n * \\param device_id The device index of the cuda-capable gpu of interest.\n * \\return the gpu's ability to run cooperative-group kernels.\n */\ninline bool SupportsCooperativeLaunch(int device_id) {\n  static std::vector<int32_t> coop_launch(kMaxNumGpus, -1);\n  return cudaAttributeLookup(\n      device_id, &coop_launch, cudaDevAttrCooperativeLaunch, \"SupportsCooperativeLaunch\");\n}\n\n/*!\n * \\brief Determine whether a cuda-capable gpu's architecture supports float16 math.\n *        Assume not if device_id is negative.\n * \\param device_id The device index of the cuda-capable gpu of interest.\n * \\return whether the gpu's architecture supports float16 math.\n */\ninline bool SupportsFloat16Compute(int device_id) {\n  if (device_id < 0) {\n    return false;\n  } else {\n    // Kepler and most Maxwell GPUs do not support fp16 compute\n    int computeCapabilityMajor = ComputeCapabilityMajor(device_id);\n    return (computeCapabilityMajor > 5) ||\n           (computeCapabilityMajor == 5 && ComputeCapabilityMinor(device_id) >= 3);\n  }\n}\n\n/*!\n * \\brief Determine whether a cuda-capable gpu's architecture supports Tensor Core math.\n *        Assume not if device_id is negative.\n * \\param device_id The device index of the cuda-capable gpu of interest.\n * \\return whether the gpu's architecture supports Tensor Core math.\n */\ninline bool SupportsTensorCore(int device_id) {\n  // Volta (sm_70) supports TensorCore algos\n  return device_id >= 0 && ComputeCapabilityMajor(device_id) >= 7;\n}\n\n// The policy if the user hasn't set the environment variable MXNET_CUDA_ALLOW_TENSOR_CORE\n#define MXNET_CUDA_ALLOW_TENSOR_CORE_DEFAULT true\n\n/*!\n * \\brief Returns global policy for TensorCore algo use.\n * \\return whether to allow TensorCore algo (if not specified by the Operator locally).\n */\ninline bool GetEnvAllowTensorCore() {\n  // Since these statics are in the '.h' file, they will exist and will be set\n  // separately in each compilation unit.  Not ideal, but cleaner than creating a\n  // cuda_utils.cc solely to have a single instance and initialization.\n  static bool allow_tensor_core = false;\n  static bool is_set            = false;\n  if (!is_set) {\n    // Use of optional<bool> here permits: \"0\", \"1\", \"true\" and \"false\" to all be legal.\n    bool default_value = MXNET_CUDA_ALLOW_TENSOR_CORE_DEFAULT;\n    allow_tensor_core =\n        dmlc::GetEnv(\"MXNET_CUDA_ALLOW_TENSOR_CORE\", dmlc::optional<bool>(default_value)).value();\n    is_set = true;\n  }\n  return allow_tensor_core;\n}\n\n// The policy if the user hasn't set the environment variable\n// CUDNN_TENSOR_OP_MATH_ALLOW_CONVERSION\n#define MXNET_CUDA_TENSOR_OP_MATH_ALLOW_CONVERSION_DEFAULT false\n\n/*!\n * \\brief Returns global policy for TensorCore implicit type casting\n */\ninline bool GetEnvAllowTensorCoreConversion() {\n  // Use of optional<bool> here permits: \"0\", \"1\", \"true\" and \"false\" to all be\n  // legal.\n  bool default_value = MXNET_CUDA_TENSOR_OP_MATH_ALLOW_CONVERSION_DEFAULT;\n  return dmlc::GetEnv(\"MXNET_CUDA_TENSOR_OP_MATH_ALLOW_CONVERSION\",\n                      dmlc::optional<bool>(default_value))\n      .value();\n}\n\n#if CUDA_VERSION >= 9000\n// Sets the cuBLAS math mode that determines the 'allow TensorCore' policy.  Returns previous.\ninline cublasMath_t SetCublasMathMode(cublasHandle_t blas_handle, cublasMath_t new_math_type) {\n  auto handle_math_mode = CUBLAS_DEFAULT_MATH;\n  CUBLAS_CALL(cublasGetMathMode(blas_handle, &handle_math_mode));\n  CUBLAS_CALL(cublasSetMathMode(blas_handle, new_math_type));\n  return handle_math_mode;\n}\n#endif\n\n#endif  // MXNET_USE_CUDA\n\n#if MXNET_USE_CUDNN\n\n#include <cudnn.h>\n\n// Creating CUDNN_VERSION_AS_STRING as follows avoids a static_assert error message that shows\n// the formula for CUDNN_VERSION, i.e. \"1000 * 7 + 100 * 6 + 0\" rather than number \"7600\".\nstatic_assert(CUDNN_PATCHLEVEL < 100 && CUDNN_MINOR < 10,\n              \"CUDNN_VERSION_AS_STRING macro assumptions violated.\");\n#if CUDNN_PATCHLEVEL >= 10\n#define CUDNN_VERSION_AS_STRING \\\n  QUOTEVALUE(CUDNN_MAJOR)       \\\n  QUOTEVALUE(CUDNN_MINOR)       \\\n  QUOTEVALUE(CUDNN_PATCHLEVEL)\n#else\n#define CUDNN_VERSION_AS_STRING \\\n  QUOTEVALUE(CUDNN_MAJOR)       \\\n  QUOTEVALUE(CUDNN_MINOR)       \\\n  \"0\" QUOTEVALUE(CUDNN_PATCHLEVEL)\n#endif\n\n#define STATIC_ASSERT_CUDNN_VERSION_GE(min_version)             \\\n  static_assert(                                                \\\n      CUDNN_VERSION >= min_version,                             \\\n      \"Compiled-against cuDNN version \" CUDNN_VERSION_AS_STRING \\\n      \" is too old, please upgrade system to version \" QUOTEVALUE(min_version) \" or later.\")\n\n#define CUDNN_CALL_S(f, s)                                       \\\n  {                                                              \\\n    cudnnStatus_t unclash_cxx_e = (f);                           \\\n    if (unclash_cxx_e != CUDNN_STATUS_SUCCESS)                   \\\n      LOG(s) << \"cuDNN: \" << cudnnGetErrorString(unclash_cxx_e); \\\n  }\n\n#define CUDNN_CALL(f)          CUDNN_CALL_S(f, FATAL)\n#define CUDNN_CALL_NONFATAL(f) CUDNN_CALL_S(f, WARNING)\n\n#define CUTENSOR_CALL(func)                                                            \\\n  {                                                                                    \\\n    cutensorStatus_t e = (func);                                                       \\\n    CHECK_EQ(e, CUTENSOR_STATUS_SUCCESS) << \"cuTensor: \" << cutensorGetErrorString(e); \\\n  }\n\n/*!\n * \\brief Return max number of perf structs cudnnFindConvolutionForwardAlgorithm()\n *        may want to populate.\n * \\param cudnn_handle cudnn handle needed to perform the inquiry.\n * \\return max number of perf structs cudnnFindConvolutionForwardAlgorithm() may\n *         want to populate.\n */\ninline int MaxForwardAlgos(cudnnHandle_t cudnn_handle) {\n  STATIC_ASSERT_CUDNN_VERSION_GE(7000);\n  int max_algos = 0;\n  CUDNN_CALL(cudnnGetConvolutionForwardAlgorithmMaxCount(cudnn_handle, &max_algos));\n  return max_algos;\n}\n\n/*!\n * \\brief Return max number of perf structs cudnnFindConvolutionBackwardFilterAlgorithm()\n *        may want to populate.\n * \\param cudnn_handle cudnn handle needed to perform the inquiry.\n * \\return max number of perf structs cudnnFindConvolutionBackwardFilterAlgorithm() may\n *         want to populate.\n */\ninline int MaxBackwardFilterAlgos(cudnnHandle_t cudnn_handle) {\n  STATIC_ASSERT_CUDNN_VERSION_GE(7000);\n  int max_algos = 0;\n  CUDNN_CALL(cudnnGetConvolutionBackwardFilterAlgorithmMaxCount(cudnn_handle, &max_algos));\n  return max_algos;\n}\n\n/*!\n * \\brief Return max number of perf structs cudnnFindConvolutionBackwardDataAlgorithm()\n *        may want to populate.\n * \\param cudnn_handle cudnn handle needed to perform the inquiry.\n * \\return max number of perf structs cudnnFindConvolutionBackwardDataAlgorithm() may\n *         want to populate.\n */\ninline int MaxBackwardDataAlgos(cudnnHandle_t cudnn_handle) {\n  STATIC_ASSERT_CUDNN_VERSION_GE(7000);\n  int max_algos = 0;\n  CUDNN_CALL(cudnnGetConvolutionBackwardDataAlgorithmMaxCount(cudnn_handle, &max_algos));\n  return max_algos;\n}\n\n#endif  // MXNET_USE_CUDNN\n\n// Overload atomicAdd to work for floats on all architectures\n#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 600\n// From CUDA Programming Guide\nstatic inline __device__ void atomicAdd(double* address, double val) {\n  unsigned long long* address_as_ull =                 // NOLINT(*)\n      reinterpret_cast<unsigned long long*>(address);  // NOLINT(*)\n  unsigned long long old = *address_as_ull;            // NOLINT(*)\n  unsigned long long assumed;                          // NOLINT(*)\n\n  do {\n    assumed = old;\n    old     = atomicCAS(\n        address_as_ull, assumed, __double_as_longlong(val + __longlong_as_double(assumed)));\n\n    // Note: uses integer comparison to avoid hang in case of NaN (since NaN != NaN)\n  } while (assumed != old);\n}\n#endif\n\n// Overload atomicAdd for half precision\n// Taken from:\n// https://github.com/torch/cutorch/blob/master/lib/THC/THCAtomics.cuh\n#ifdef __CUDACC__\nstatic inline __device__ void atomicAdd(mshadow::half::half_t* address, mshadow::half::half_t val) {\n  unsigned int* address_as_ui = reinterpret_cast<unsigned int*>(\n      reinterpret_cast<char*>(address) - (reinterpret_cast<size_t>(address) & 2));\n  unsigned int old = *address_as_ui;\n  unsigned int assumed;\n\n  do {\n    assumed = old;\n    mshadow::half::half_t hsum;\n    hsum.half_ = reinterpret_cast<size_t>(address) & 2 ? (old >> 16) : (old & 0xffff);\n    hsum += val;\n    old = reinterpret_cast<size_t>(address) & 2 ? (old & 0xffff) | (hsum.half_ << 16) :\n                                                  (old & 0xffff0000) | hsum.half_;\n    old = atomicCAS(address_as_ui, assumed, old);\n  } while (assumed != old);\n}\n\nstatic inline __device__ void atomicAdd(uint8_t* address, uint8_t val) {\n  unsigned int* address_as_ui = (unsigned int*)(address - ((size_t)address & 0x3));\n  unsigned int old            = *address_as_ui;\n  unsigned int shift          = (((size_t)address & 0x3) << 3);\n  unsigned int sum;\n  unsigned int assumed;\n\n  do {\n    assumed = old;\n    sum     = val + static_cast<uint8_t>((old >> shift) & 0xff);\n    old     = (old & ~(0x000000ff << shift)) | (sum << shift);\n    old     = atomicCAS(address_as_ui, assumed, old);\n  } while (assumed != old);\n}\n\nstatic inline __device__ void atomicAdd(int8_t* address, int8_t val) {\n  unsigned int* address_as_ui = (unsigned int*)(address - ((size_t)address & 0x3));\n  unsigned int old            = *address_as_ui;\n  unsigned int shift          = (((size_t)address & 0x3) << 3);\n  unsigned int sum;\n  unsigned int assumed;\n\n  do {\n    assumed = old;\n    sum     = val + static_cast<int8_t>((old >> shift) & 0xff);\n    old     = (old & ~(0x000000ff << shift)) | (sum << shift);\n    old     = atomicCAS(address_as_ui, assumed, old);\n  } while (assumed != old);\n}\n\n// Overload atomicAdd to work for signed int64 on all architectures\nstatic inline __device__ void atomicAdd(int64_t* address, int64_t val) {\n  atomicAdd(reinterpret_cast<unsigned long long*>(address),  // NOLINT\n            static_cast<unsigned long long>(val));           // NOLINT\n}\n\ntemplate <typename DType>\n__device__ inline DType ldg(const DType* address) {\n#if __CUDA_ARCH__ >= 350\n  return __ldg(address);\n#else\n  return *address;\n#endif\n}\n\nnamespace mxnet {\nnamespace common {\n/*! \\brief common utils for cuda */\nnamespace cuda {\n\nstatic constexpr const int warp_size = 32;\n\n/*! \\brief Reduction inside a warp.\n * Template parameters:\n * NVALUES - number of values to reduce (defaults to warp_size).\n * \\param value - values to be reduced.\n * \\param redfun - function used to perform reduction.\n */\ntemplate <int NVALUES = warp_size, typename OP, typename T>\n__device__ inline T warp_reduce(T value, OP redfun) {\n#pragma unroll\n  for (int i = warp_size / 2; i >= 1; i /= 2) {\n    if (NVALUES > i)\n      value = redfun(value, __shfl_down_sync(0xffffffff, value, i));\n  }\n  return value;\n}\n\ntemplate <typename OP, typename T>\n__device__ inline T grouped_warp_allreduce(T value, OP redfun, const int group_size) {\n  for (int i = 1; i < group_size; i *= 2) {\n    value = redfun(value, __shfl_down_sync(0xffffffff, value, i));\n  }\n  return __shfl_sync(0xffffffff, value, 0, group_size);\n}\n\ntemplate <int NValues = warp_size, typename OP>\n__device__ inline mshadow::half::half_t warp_reduce(mshadow::half::half_t value, OP redfun) {\n  float v = static_cast<float>(value);\n#pragma unroll\n  for (int i = warp_size / 2; i >= 1; i /= 2) {\n    if (NValues > i)\n      v = redfun(v, __shfl_down_sync(0xffffffff, v, i));\n  }\n  return mshadow::half::half_t(v);\n}\n\n/*! \\brief Reduction inside a block, requires all threads in a block to participate.\n *         It uses a 2 step approach:\n *          - all warps in a block perform intermediate reduction\n *          - first warp reduces the intermediate results.\n * Template parameters:\n * NTHREADS - number of threads in a block.\n * all_reduce - whether all threads need the result of the reduction. If set to\n *              true, then all threads return with the same value. If set to\n *              false, then only thread 0 has the valid result. Defaults to true.\n * \\param value - value from each thread to be reduced\n * \\param redfun - function used to perform reduction\n */\ntemplate <int NTHREADS, bool all_reduce = true, typename OP, typename T>\n__device__ inline T reduce(const T& value, OP redfun) {\n  static_assert(NTHREADS <= warp_size * warp_size, \"Number of threads too large for reduction\");\n  __shared__ T scratch[NTHREADS / warp_size];\n  const int thread_idx_in_warp = threadIdx.x % warp_size;\n  const int warp_id            = threadIdx.x / warp_size;\n  const T my_val               = warp_reduce<warp_size>(value, redfun);\n  if (thread_idx_in_warp == 0) {\n    scratch[warp_id] = my_val;\n  }\n  __syncthreads();\n  T ret = 0;\n  if (warp_id == 0) {\n    const T prev_val = threadIdx.x < (NTHREADS / warp_size) ? scratch[threadIdx.x] : 0;\n    const T my_val   = warp_reduce<NTHREADS / warp_size>(prev_val, redfun);\n    if (all_reduce) {\n      scratch[threadIdx.x] = my_val;\n    } else {\n      ret = my_val;\n    }\n  }\n  // Necessary to synchronize in order to use this function again\n  // as the shared memory scratch space is reused between calls\n  __syncthreads();\n  if (all_reduce) {\n    ret = scratch[0];\n    __syncthreads();\n  }\n  return ret;\n}\n\n}  // namespace cuda\n}  // namespace common\n}  // namespace mxnet\n\n#endif  // __CUDACC__\n\n#endif  // MXNET_COMMON_CUDA_UTILS_H_\n"
  },
  {
    "path": "src/common/exec_utils.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file exec_utils.cc\n * \\brief Implementation of executor util functions.\n */\n\n#include \"exec_utils.h\"\n#include <unordered_set>\n#include <unordered_map>\n#include <string>\n\nnamespace mxnet {\nnamespace common {\n\nvoid CopyGraph(nnvm::Graph* dst, const nnvm::Graph& src, bool copy_variables) {\n  using nnvm::Node;\n  using nnvm::NodeEntry;\n  using nnvm::ObjectPtr;\n  std::unordered_map<Node*, ObjectPtr> old_new;\n  // use DFSVisit to copy all the nodes\n  DFSVisit(src.outputs, [&old_new, copy_variables](const ObjectPtr& node) {\n    ObjectPtr np;\n    if (copy_variables || !node->is_variable()) {\n      np        = Node::Create();\n      np->attrs = node->attrs;\n    } else {\n      np = node;\n    }\n    old_new[node.get()] = std::move(np);\n  });\n  // connect nodes of new graph\n  for (const auto& kv : old_new) {\n    for (const NodeEntry& e : kv.first->inputs) {\n      Node* ptr = e.node.get();\n      kv.second->inputs.emplace_back(NodeEntry{old_new[ptr], e.index, e.version});\n    }\n    for (const ObjectPtr& p : kv.first->control_deps) {\n      kv.second->control_deps.emplace_back(old_new[p.get()]);\n    }\n  }\n  // set the head\n  for (const NodeEntry& e : src.outputs) {\n    (*dst).outputs.emplace_back(NodeEntry{old_new[e.node.get()], e.index, e.version});\n  }\n}\n\nbool CheckForInputNameDuplicates(const nnvm::IndexedGraph& idx) {\n  std::unordered_set<std::string> names;\n  for (const auto& nid : idx.input_nodes()) {\n    const std::string& name = idx[nid].source->attrs.name;\n    if (names.count(name)) {\n      LOG(WARNING) << \"Variable name \" << name << \" is used more than once!\";\n      return false;\n    }\n    names.insert(name);\n  }\n  return true;\n}\n\n}  // namespace common\n}  // namespace mxnet\n"
  },
  {
    "path": "src/common/exec_utils.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file exec_utils.h\n * \\brief Common utility functions for executors.\n */\n#ifndef MXNET_COMMON_EXEC_UTILS_H_\n#define MXNET_COMMON_EXEC_UTILS_H_\n\n#include <nnvm/graph.h>\n#include <nnvm/pass_functions.h>\n#include <map>\n#include <vector>\n#include <string>\n#include <utility>\n#include \"../common/utils.h\"\n#include \"../imperative/exec_pass.h\"\n\nnamespace mxnet {\nnamespace common {\n\n#if MXNET_USE_ONEDNN == 1\n// We have to make sure it's default storage and default layout.\n#define DEFAULT_DATA(x) x.IsDefaultData()\n#else\n#define DEFAULT_DATA(x) (x.storage_type() == kDefaultStorage)\n#endif\n\n/*\n * \\brief setup default-storage tblobs from source NDArrays. If any source NDArray has non-default\n *        storage, it creates a temp NDArray with default storage and uses the temp tblob. The\n *        function also records the indices of non-default source NDArrays and the indices of\n *        their corresponding temporary NDArrays in the temp array.\n * \\param src list of source NDArray\n * \\param blobs list of tblobs to return\n * \\param temp_src list of source NDArrays which requires temporary default storage representation\n * \\param temp_dst list of temporary destination NDArrays for default storage representation\n * \\param idx_map mapping from indices in source NDArrays to indices in temp_dst. When not set,\n          indices are not recorded\n * \\return true if any source NDArray need to cast storage\n */\ninline bool SetupDefaultBlobsIn(const std::vector<NDArray>& src,\n                                const std::vector<NDArray>* bufs,\n                                std::vector<TBlob>* blobs,\n                                std::vector<NDArray>* temp_src,\n                                std::vector<NDArray>* temp_dst,\n                                std::unordered_map<uint32_t, uint32_t>* idx_map) {\n  bool require_cast = false;\n  for (size_t i = 0; i < src.size(); i++) {\n    const auto& nd = src[i];\n    if (!DEFAULT_DATA(nd)) {\n      (*idx_map)[i] = temp_dst->size();\n      NDArray temp =\n          bufs != nullptr ? bufs->at(i) : NDArray(nd.shape(), nd.ctx(), true, nd.dtype());\n#if MXNET_USE_ONEDNN == 1\n      CHECK(temp.IsDefaultData());\n#endif\n      temp_src->emplace_back(nd);\n      temp_dst->emplace_back(temp);\n      blobs->emplace_back(temp.data());\n      require_cast = true;\n    } else {\n      blobs->push_back(nd.data());\n    }\n  }\n  return require_cast;\n}\n\ninline bool SetupDefaultBlobsOut(const std::vector<NDArray>& src,\n                                 const std::vector<NDArray>* bufs,\n                                 std::vector<OpReqType>* req,\n                                 std::vector<TBlob>* blobs,\n                                 std::vector<NDArray>* temp_src,\n                                 std::vector<NDArray>* temp_dst) {\n  bool require_cast = false;\n  for (size_t i = 0; i < src.size(); i++) {\n    const auto& nd = src[i];\n\n#if MXNET_USE_ONEDNN == 1\n    if (req->at(i) == kWriteInplace && nd.IsDNNLData())\n      // If it's write inplace and the output array doesn't use the default\n      // layout, we'll generate a temporary output array below, which means\n      // the input array and the output array are no longer the same array.\n      // we should change the request type.\n      req->at(i) = kWriteTo;\n      // We have to make sure it's default storage and default layout.\n#endif\n    if (!DEFAULT_DATA(nd)) {\n#if MXNET_USE_ONEDNN == 1\n      NDArray temp;\n      if (bufs != nullptr) {\n        temp = bufs->at(i);\n      } else if (kAddTo == req->at(i)) {\n        temp = nd.IsDNNLData() ? nd.Reorder2Default() : nd;\n      } else {\n        temp = NDArray(nd.shape(), nd.ctx(), true, nd.dtype());\n      }\n      CHECK(temp.IsDefaultData());\n#else\n      NDArray temp =\n          bufs != nullptr ? bufs->at(i) : NDArray(nd.shape(), nd.ctx(), true, nd.dtype());\n#endif\n      temp_src->emplace_back(nd);\n      temp_dst->emplace_back(temp);\n      blobs->emplace_back(temp.data());\n      require_cast = true;\n    } else {\n      blobs->push_back(nd.data());\n    }\n  }\n  return require_cast;\n}\n\n/*\n * \\brief setup default-storage tblobs for input and output NDArrays.\n *        If any NDArray has non-default storage,\n *        it creates a temp NDArray with default storage and uses the temp tblob. The\n *        function also records the indices of non-default source NDArrays and the indices of\n *        their corresponding temporary NDArrays in the temp array.\n */\ninline void SetupDefaultBlobsInOut(const std::vector<NDArray>& ndinputs,\n                                   const std::vector<NDArray>& ndoutputs,\n                                   const std::vector<NDArray>* in_bufs,\n                                   const std::vector<NDArray>* out_bufs,\n                                   std::vector<OpReqType>* req,\n                                   std::vector<TBlob>* input_blobs,\n                                   std::vector<TBlob>* output_blobs,\n                                   std::vector<NDArray>* pre_temp_src,\n                                   std::vector<NDArray>* pre_temp_dst,\n                                   std::vector<NDArray>* post_temp_src,\n                                   std::vector<NDArray>* post_temp_dst,\n                                   std::unordered_map<uint32_t, uint32_t>* in_temp_idx_map,\n                                   const std::vector<uint32_t>& mutate_idx) {\n  // populate input blobs\n  SetupDefaultBlobsIn(ndinputs, in_bufs, input_blobs, pre_temp_src, pre_temp_dst, in_temp_idx_map);\n  // populate output blobs\n  SetupDefaultBlobsOut(ndoutputs, out_bufs, req, output_blobs, post_temp_dst, post_temp_src);\n  // add mutable inputs to post temp list\n  for (const auto idx : mutate_idx) {\n    auto map_iter = in_temp_idx_map->find(idx);\n    if (map_iter != in_temp_idx_map->end()) {\n      post_temp_src->push_back(pre_temp_dst->at(map_iter->second));\n      post_temp_dst->push_back(ndinputs[idx]);\n    }\n  }\n}\n\n/*\n * \\brief cast the NDArrays in `src` and store the result in NDArrays in `dst`.\n *        This is only used for storage fallback in executor.\n * \\param src list of source NDArray to cast\n * \\param dst list of destionation NDArray which hold the result of cast_storage operation\n * \\param ctx operator context for cast_storage operation\n */\ninline void CastNonDefaultStorage(const std::vector<NDArray>& src,\n                                  const std::vector<NDArray>& dst,\n                                  const OpContext& ctx,\n                                  const bool is_gpu) {\n  CHECK_EQ(dst.size(), src.size());\n  for (size_t i = 0; i < src.size(); i++) {\n    if (is_gpu) {\n#if MXNET_USE_CUDA\n      CastStorageDispatch<gpu>(ctx, src[i], dst[i]);\n#else\n      LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR;\n#endif\n    } else {\n      CastStorageDispatch<cpu>(ctx, src[i], dst[i]);\n    }\n  }\n}\n\n/*! \\brief The default type inference function, which assigns all undefined\n *         types to the same type of one of the inputs or outputs.\n */\ninline bool SameType(const nnvm::NodeAttrs& attrs,\n                     std::vector<int>* iattr,\n                     std::vector<int>* oattr) {\n  int def_v = -1;\n  for (int v : *oattr) {\n    if (v != -1) {\n      def_v = v;\n      break;\n    }\n  }\n  if (def_v == -1) {\n    for (int v : *iattr) {\n      if (v != -1) {\n        def_v = v;\n        break;\n      }\n    }\n  }\n  if (def_v == -1)\n    return false;\n  for (int& v : *oattr) {\n    v = def_v;\n  }\n  for (int& v : *iattr) {\n    v = def_v;\n  }\n  return true;\n}\n\n/*! \\brief The default storage type inference function, which assigns all undefined\n *         storage types to kDefaultStorage. If all of input and output storage types\n *         are kDefaultStorage, DispatchMode::kFCompute is assigned to dispatch_mode. Otherwise,\n *         DispatchMode::kFComputeFallback is assigned to dispatch_mode.\n */\ninline bool DefaultStorageType(const nnvm::NodeAttrs& attrs,\n                               const int dev_mask,\n                               DispatchMode* dispatch_mode,\n                               std::vector<int>* iattr,\n                               std::vector<int>* oattr) {\n  bool fallback = false;\n  for (int& v : *oattr) {\n    if (v == -1)\n      v = kDefaultStorage;\n    if (v != kDefaultStorage)\n      fallback = true;\n  }\n  for (int& v : *iattr) {\n    if (v == -1)\n      v = kDefaultStorage;\n    if (v != kDefaultStorage)\n      fallback = true;\n  }\n  if (*dispatch_mode == DispatchMode::kUndefined) {\n    if (fallback) {\n      *dispatch_mode = DispatchMode::kFComputeFallback;\n    } else {\n      *dispatch_mode = DispatchMode::kFCompute;\n    }\n  }\n  return true;\n}\n\n// string representation of storage id\ninline std::string storage_str(int storage_id) {\n  std::string str;\n  if (storage_id == -1) {\n    str = \"var (-1)\";\n  } else if (storage_id == -2) {\n    str = \"external storage (-2)\";\n  } else {\n    str = \"group \" + std::to_string(storage_id);\n  }\n  return str;\n}\n\n/* log the static memory plan of the graph. Example:\n   node 0 var\n   node 1 _copy\n            input 0: [80,3,224,224] (47040 KB) -> var storage (-1)\n            output 1: [80,3,224,224] (47040 KB) -> group 0\n   node 2 var\n   node 3 var\n   node 4 var\n   node 5 var\n   node 6 BatchNorm\n            input 1: [80,3,224,224] (47040 KB) -> group 0\n            input 2: [3] (0 KB) -> var storage (-1)\n            input 3: [3] (0 KB) -> var storage (-1)\n            input 4: [3] (0 KB) -> var storage (-1)\n            input 5: [3] (0 KB) -> var storage (-1)\n            output 6: [80,3,224,224] (47040 KB) -> group 1\n            output 7: [3] (0 KB) -> group 3\n            output 8: [3] (0 KB) -> group 2\n   ...\n */\ninline void LogMemoryPlan(const nnvm::Graph& g) {\n  const auto& idx    = g.indexed_graph();\n  const auto& vshape = g.GetAttr<mxnet::ShapeVector>(\"shape\");\n  const auto& vtype  = g.GetAttr<nnvm::DTypeVector>(\"dtype\");\n  // find node range\n  uint32_t node_start = 0, node_end = idx.num_nodes();\n  if (g.attrs.count(\"node_range\")) {\n    const auto& range = g.GetAttr<std::pair<uint32_t, uint32_t> >(\"node_range\");\n    node_start        = range.first;\n    node_end          = range.second;\n  }\n  for (uint32_t nid = node_start; nid < node_end; ++nid) {\n    const auto& inode = idx[nid];\n    if (inode.source->is_variable()) {\n      LOG(INFO) << \"node \" << nid << \" var\";\n    } else {\n      LOG(INFO) << \"node \" << nid << \" \" << inode.source->attrs.op->name;\n      for (const auto& e : inode.inputs) {\n        auto eid          = idx.entry_id(e);\n        size_t kilo_bytes = vshape[eid].Size() * mshadow::mshadow_sizeof(vtype[eid]) / 1024;\n        LOG(INFO) << \"\\t\\tinput \" << eid << \": \" << vshape[eid] << \" (\" << kilo_bytes << \" KB)\";\n      }\n      for (uint32_t index = 0; index < inode.source->num_outputs(); ++index) {\n        uint32_t eid      = idx.entry_id(nid, index);\n        size_t kilo_bytes = vshape[eid].Size() * mshadow::mshadow_sizeof(vtype[eid]) / 1024;\n        LOG(INFO) << \"\\t\\toutput \" << eid << \": \" << vshape[eid] << \" (\" << kilo_bytes << \" KB)\";\n      }\n    }\n  }\n}\n\n/* log the static memory plan of the graph. Example:\n    node 0 var\n    node 1 _copy: fcompute\n                input 0: default\n                output 1: default\n    node 2 var\n    node 3 Convolution: fcompute\n                input 1: default\n                input 2: default\n                output 3: default\n    node 4 var\n    node 5 var\n    node 6 var\n    node 7 var\n    node 8 BatchNorm: fcompute\n                input 3: default\n                input 4: default\n                input 5: default\n                input 6: default\n                input 7: default\n                output 8: default\n                output 9: default\n                output 10: default\n    ...\n */\ninline void LogInferStorage(const nnvm::Graph& g) {\n  const auto& idx            = g.indexed_graph();\n  const auto& vstorage_type  = g.GetAttr<StorageTypeVector>(\"storage_type\");\n  const auto& dispatch_modes = g.GetAttr<DispatchModeVector>(\"dispatch_mode\");\n  uint32_t node_start = 0, node_end = idx.num_nodes();\n  if (g.attrs.count(\"node_range\")) {\n    const auto& range = g.GetAttr<std::pair<uint32_t, uint32_t> >(\"node_range\");\n    node_start        = range.first;\n    node_end          = range.second;\n  }\n  for (uint32_t nid = node_start; nid < node_end; ++nid) {\n    const auto& inode = idx[nid];\n    if (inode.source->is_variable()) {\n      LOG(INFO) << \"node \" << nid << \" var\";\n    } else {\n      LOG(INFO) << \"node \" << nid << \" \" << inode.source->attrs.op->name << \": \"\n                << dispatch_mode_string(dispatch_modes[nid]);\n      for (const auto& e : inode.inputs) {\n        auto eid = idx.entry_id(e);\n        LOG(INFO) << \"\\t\\tinput \" << eid << \": \" << stype_string(vstorage_type[eid]);\n      }\n      for (uint32_t index = 0; index < inode.source->num_outputs(); ++index) {\n        uint32_t eid = idx.entry_id(nid, index);\n        LOG(INFO) << \"\\t\\toutput \" << eid << \": \" << stype_string(vstorage_type[eid]);\n      }\n    }\n  }\n}\n\n/*!\n * \\brief If the requested ndarray's shape size is less than\n * the corresponding shared_data_array's shape size and the\n * storage type is shareable, reuse the memory allocation\n * in shared_buffer; otherwise, create a zero ndarray.\n * Shareable storages include both default storage and row_sparse storage\n * if enable_row_sparse_sharing is `True`, otherwise default storage only.\n */\ninline NDArray ReshapeOrCreate(const std::string& name,\n                               const mxnet::TShape& dest_arg_shape,\n                               const int dest_arg_dtype,\n                               const NDArrayStorageType dest_arg_stype,\n                               const Context& ctx,\n                               std::unordered_map<std::string, NDArray>* shared_buffer,\n                               bool enable_row_sparse_sharing) {\n  bool stype_shareable = dest_arg_stype == kDefaultStorage;\n  if (enable_row_sparse_sharing) {\n    stype_shareable = stype_shareable || dest_arg_stype == kRowSparseStorage;\n  }\n  auto it = shared_buffer->find(name);\n  if (it != shared_buffer->end()) {\n    // check if size is large enough for sharing\n    bool size_shareable = it->second.shape().Size() >= dest_arg_shape.Size();\n    if (size_shareable && stype_shareable) {  // memory can be reused\n      CHECK_EQ(it->second.dtype(), dest_arg_dtype)\n          << \"Requested arg array's dtype does not match that of the reusable ndarray\";\n      CHECK_EQ(it->second.storage_type(), dest_arg_stype)\n          << \"Requested arg array's stype does not match that of the reusable ndarray\";\n      return it->second.Reshape(dest_arg_shape);\n    } else if (stype_shareable) {\n      LOG(WARNING) << \"Bucketing: data \" << name << \" has a shape \" << dest_arg_shape\n                   << \", which is larger than already allocated shape \" << it->second.shape()\n                   << \". Need to re-allocate. Consider putting default bucket key to be \"\n                   << \"the bucket taking the largest input for better memory sharing.\";\n      // size is not large enough, creating a larger one for sharing\n      // the NDArrays in shared_buffer are guaranteed to be of shareable storages\n      it->second = InitZeros(dest_arg_stype, dest_arg_shape, ctx, dest_arg_dtype);\n      return it->second;\n    } else {\n      // not shareable storage\n      return InitZeros(dest_arg_stype, dest_arg_shape, ctx, dest_arg_dtype);\n    }\n  } else {\n    auto ret = InitZeros(dest_arg_stype, dest_arg_shape, ctx, dest_arg_dtype);\n    if (stype_shareable) {\n      shared_buffer->emplace(name, ret);\n    }\n    return ret;\n  }  // if (it != shared_buffer->end())\n}\n\n/*!\n * \\brief Assign context to the graph.\n * This is triggered by both simple_bind and bind flows.\n */\ninline nnvm::Graph AssignContext(nnvm::Graph g,\n                                 const Context& default_ctx,\n                                 const std::map<std::string, Context>& ctx_map,\n                                 const std::vector<Context>& in_arg_ctxes,\n                                 const std::vector<Context>& arg_grad_ctxes,\n                                 const std::vector<Context>& aux_state_ctxes,\n                                 const std::vector<OpReqType>& grad_req_types,\n                                 size_t num_forward_inputs,\n                                 size_t num_forward_outputs) {\n  const auto& idx           = g.indexed_graph();\n  const auto& mutable_nodes = idx.mutable_input_nodes();\n  // default use default context.\n  if (ctx_map.size() == 0) {\n    g.attrs[\"context\"] =\n        std::make_shared<nnvm::any>(exec::ContextVector(idx.num_nodes(), default_ctx));\n    for (const auto& x : in_arg_ctxes) {\n      CHECK(x == default_ctx) << \"Input array is in \" << x\n                              << \" while binding with ctx=\" << default_ctx\n                              << \". All arguments must be in global context (\" << default_ctx\n                              << \") unless group2ctx is specified for cross-device graph.\";\n    }\n    for (const auto& x : arg_grad_ctxes) {\n      CHECK(x == default_ctx) << \"Gradient array is in \" << x\n                              << \" while binding with ctx=\" << default_ctx\n                              << \". All gradients must be in global context (\" << default_ctx\n                              << \") unless group2ctx is specified for cross-device graph.\";\n    }\n    return g;\n  }\n\n  // otherwise, use context assignment.\n  std::map<Context, int> ctx2id;                   // map ctx to device id\n  std::vector<Context> ctx_list;                   // index is device id\n  nnvm::DeviceVector device(idx.num_nodes(), -1);  // index is node id\n  nnvm::DeviceAssignMap device_map;                // map arg name to device id\n\n  // loop through the user input ctx_map and\n  // populate maps and lists\n  for (auto& kv : ctx_map) {\n    if (ctx2id.count(kv.second) == 0) {  // if context has no device id, create one\n      ctx2id[kv.second] = static_cast<int>(ctx_list.size());  // assign device id to ctx\n      ctx_list.push_back(kv.second);                          // save ctx to the list\n    }\n    // assign device id to to the arg name with the corresponding ctx\n    device_map[kv.first] = ctx2id.at(kv.second);\n  }\n\n  // loop through all the rest of input nodes not specified\n  // in the ctx_map and populate maps and lists\n  size_t arg_top = 0, aux_top = 0;\n  for (size_t i = 0; i < num_forward_inputs; ++i) {\n    const uint32_t nid = idx.input_nodes().at(i);\n    Context ctx;\n    if (mutable_nodes.count(nid)) {  // aux node is mutable\n      CHECK_LT(aux_top, aux_state_ctxes.size());\n      ctx = aux_state_ctxes[aux_top];\n      ++aux_top;\n    } else {  // regular input node is immutable\n      CHECK_LT(arg_top, in_arg_ctxes.size());\n      ctx = in_arg_ctxes[arg_top];\n      ++arg_top;\n    }\n    if (ctx2id.count(ctx) == 0) {  // if the current ctx is not in the map of ctx and device id\n      ctx2id[ctx] = static_cast<int>(ctx_list.size());  // assign the current ctx with device id\n      ctx_list.push_back(ctx);                          // save the current ctx in the list\n    }\n    device[nid] = ctx2id.at(ctx);  // assign device id to the current node\n  }\n\n  // loop through backward input nodes and populate maps and lists\n  // the backward input nodes is the gradient of the loss wrt the output\n  size_t arg_grad_offset = 0;\n  // keep an offset into the arg_grad_ctxes vector,\n  // since g.outputs exclude arg_grad whose req == null\n  CHECK_GE(grad_req_types.size(), g.outputs.size() - num_forward_outputs)\n      << \"insufficient number of grad_reqs\";\n  for (size_t i = num_forward_outputs; i < g.outputs.size(); ++i, ++arg_grad_offset) {\n    while (grad_req_types[arg_grad_offset] == kNullOp)\n      ++arg_grad_offset;\n    const uint32_t nid = idx.outputs()[i].node_id;\n    Context ctx        = arg_grad_ctxes[arg_grad_offset];\n    if (ctx2id.count(ctx) == 0) {\n      ctx2id[ctx] = static_cast<int>(ctx_list.size());\n      ctx_list.push_back(ctx);\n    }\n    int devid = ctx2id.at(ctx);\n    if (device[nid] != -1) {\n      CHECK_EQ(device[nid], devid) << \"device of same output not equal to each other\";\n    } else {\n      device[nid] = devid;\n    }\n  }\n\n  g.attrs[\"device\"] = std::make_shared<dmlc::any>(std::move(device));\n  g                 = nnvm::pass::PlaceDevice(g, \"__ctx_group__\", device_map, \"_CrossDeviceCopy\");\n  const auto& assigned_devices = g.GetAttr<nnvm::DeviceVector>(\"device\");\n\n  exec::ContextVector vcontext;\n  for (auto context : assigned_devices) {\n    if (context == -1) {\n      vcontext.push_back(default_ctx);\n    } else {\n      vcontext.push_back(ctx_list[context]);\n    }\n  }\n\n  // after device planning, we should check again\n  // if the assigned device of gradient node\n  // corresponds to storage of grads\n  auto& new_idx   = g.indexed_graph();\n  arg_grad_offset = 0;\n  for (size_t i = num_forward_outputs; i < g.outputs.size(); ++i, ++arg_grad_offset) {\n    while (grad_req_types[arg_grad_offset] == kNullOp)\n      ++arg_grad_offset;\n    const uint32_t nid = new_idx.outputs()[i].node_id;\n    Context ctx        = arg_grad_ctxes[arg_grad_offset];\n    CHECK(ctx == vcontext[nid]) << \"Trying to save gradient to \" << ctx\n                                << \" while its source node \\\"\" << new_idx[nid].source->attrs.name\n                                << \"\\\" computes it on \" << vcontext[nid]\n                                << \". Check your ctx in NDArray allocation.\";\n  }\n\n  g.attrs[\"context\"] = std::make_shared<nnvm::any>(std::move(vcontext));\n  return g;\n}\n\n/*!\n * \\brief Copy the graph, optionally leaving original Variable nodes.\n *\n * \\param dst destination graph\n * \\param src source graph being copied\n * \\param copy_variable whether to copy or reuse Variable nodes from the\n *                      source graph\n */\nvoid CopyGraph(nnvm::Graph* dst, const nnvm::Graph& src, bool copy_variables);\n\n/*!\n * \\brief Check whether graph contains any duplicated names in its inputs.\n *\n * \\param idx Indexed graph being checked\n *\n * \\return true if there are no duplicates, false otherwise\n */\nbool CheckForInputNameDuplicates(const nnvm::IndexedGraph& idx);\n\n}  // namespace common\n}  // namespace mxnet\n#endif  // MXNET_COMMON_EXEC_UTILS_H_\n"
  },
  {
    "path": "src/common/lazy_alloc_array.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file lazy_alloc_array.h\n * \\brief An array that lazily allocate elements as\n *   First time the cell get visited.\n */\n#ifndef MXNET_COMMON_LAZY_ALLOC_ARRAY_H_\n#define MXNET_COMMON_LAZY_ALLOC_ARRAY_H_\n\n#include <dmlc/logging.h>\n#include <memory>\n#include <mutex>\n#include <array>\n#include <vector>\n#include <atomic>\n\nnamespace mxnet {\nnamespace common {\n\ntemplate <typename TElem>\nclass LazyAllocArray {\n public:\n  LazyAllocArray();\n  /*!\n   * \\brief Get element of corresponding index,\n   *  if it is not created create by creator\n   * \\param index the array index position\n   * \\param creator a lambda function to create new element when needed.\n   */\n  template <typename FCreate>\n  inline std::shared_ptr<TElem> Get(int index, FCreate creator);\n  /*!\n   * \\brief for each not null element of the array, call fvisit\n   * \\param fvisit a function of (size_t, TElem*)\n   */\n  template <typename FVisit>\n  inline void ForEach(FVisit fvisit);\n  /*! \\brief clear all the allocated elements in array */\n  inline void Clear();\n\n private:\n  template <typename SyncObject>\n  class unique_unlock {\n   public:\n    explicit unique_unlock(std::unique_lock<SyncObject>* lock) : lock_(lock) {\n      if (lock_) {\n        lock_->unlock();\n      }\n    }\n    ~unique_unlock() {\n      if (lock_) {\n        lock_->lock();\n      }\n    }\n\n   private:\n    std::unique_lock<SyncObject>* lock_;\n  };\n\n  /*! \\brief the initial size of the array */\n  static constexpr std::size_t kInitSize = 16;\n  /*! \\brief mutex used during creation */\n  std::mutex create_mutex_;\n  /*! \\brief internal data fir initial size */\n  std::array<std::shared_ptr<TElem>, kInitSize> head_;\n  /*! \\brief overflow array of more elements */\n  std::vector<std::shared_ptr<TElem> > more_;\n  /*! \\brief Signal shutdown of array */\n  std::atomic<bool> is_clearing_;\n};\n\ntemplate <typename TElem>\ninline LazyAllocArray<TElem>::LazyAllocArray() : is_clearing_(false) {}\n\n// implementations\ntemplate <typename TElem>\ntemplate <typename FCreate>\ninline std::shared_ptr<TElem> LazyAllocArray<TElem>::Get(int index, FCreate creator) {\n  CHECK_GE(index, 0);\n  size_t idx = static_cast<size_t>(index);\n  if (idx < kInitSize) {\n    std::shared_ptr<TElem> ptr = head_[idx];\n    if (ptr) {\n      return ptr;\n    } else {\n      std::lock_guard<std::mutex> lock(create_mutex_);\n      if (!is_clearing_.load()) {\n        std::shared_ptr<TElem> ptr = head_[idx];\n        if (ptr) {\n          return ptr;\n        }\n        ptr = head_[idx] = std::shared_ptr<TElem>(creator());\n        return ptr;\n      }\n    }\n  } else {\n    std::lock_guard<std::mutex> lock(create_mutex_);\n    if (!is_clearing_.load()) {\n      idx -= kInitSize;\n      if (more_.size() <= idx) {\n        more_.reserve(idx + 1);\n        while (more_.size() <= idx) {\n          more_.push_back(std::shared_ptr<TElem>(nullptr));\n        }\n      }\n      std::shared_ptr<TElem> ptr = more_[idx];\n      if (ptr) {\n        return ptr;\n      }\n      ptr = more_[idx] = std::shared_ptr<TElem>(creator());\n      return ptr;\n    }\n  }\n  return nullptr;\n}\n\ntemplate <typename TElem>\ninline void LazyAllocArray<TElem>::Clear() {\n  std::unique_lock<std::mutex> lock(create_mutex_);\n  is_clearing_.store(true);\n  // Currently, head_ and more_ never get smaller, so it's safe to\n  // iterate them outside of the lock.  The loops should catch\n  // any growth which might happen when create_mutex_ is unlocked\n  for (size_t i = 0; i < head_.size(); ++i) {\n    std::shared_ptr<TElem> p = head_[i];\n    head_[i]                 = std::shared_ptr<TElem>(nullptr);\n    unique_unlock<std::mutex> unlocker(&lock);\n    p = std::shared_ptr<TElem>(nullptr);\n  }\n  for (size_t i = 0; i < more_.size(); ++i) {\n    std::shared_ptr<TElem> p = more_[i];\n    more_[i]                 = std::shared_ptr<TElem>(nullptr);\n    unique_unlock<std::mutex> unlocker(&lock);\n    p = std::shared_ptr<TElem>(nullptr);\n  }\n  more_.clear();\n  is_clearing_.store(false);\n}\n\ntemplate <typename TElem>\ntemplate <typename FVisit>\ninline void LazyAllocArray<TElem>::ForEach(FVisit fvisit) {\n  std::lock_guard<std::mutex> lock(create_mutex_);\n  for (size_t i = 0; i < head_.size(); ++i) {\n    if (head_[i].get() != nullptr) {\n      fvisit(i, head_[i].get());\n    }\n  }\n  for (size_t i = 0; i < more_.size(); ++i) {\n    if (more_[i].get() != nullptr) {\n      fvisit(i + kInitSize, more_[i].get());\n    }\n  }\n}\n\n}  // namespace common\n}  // namespace mxnet\n#endif  // MXNET_COMMON_LAZY_ALLOC_ARRAY_H_\n"
  },
  {
    "path": "src/common/object_pool.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_COMMON_OBJECT_POOL_H_\n#define MXNET_COMMON_OBJECT_POOL_H_\n#include <dmlc/logging.h>\n#include <cstdlib>\n#include <mutex>\n#include <utility>\n#include <vector>\n\nnamespace mxnet {\nnamespace common {\n/*!\n * \\brief Object pool for fast allocation and deallocation.\n */\ntemplate <typename T>\nclass ObjectPool {\n public:\n  /*!\n   * \\brief Destructor.\n   */\n  ~ObjectPool();\n  /*!\n   * \\brief Create new object.\n   * \\return Pointer to the new object.\n   */\n  template <typename... Args>\n  T* New(Args&&... args);\n  /*!\n   * \\brief Delete an existing object.\n   * \\param ptr The pointer to delete.\n   *\n   * Make sure the pointer to delete is allocated from this pool.\n   */\n  void Delete(T* ptr);\n\n  /*!\n   * \\brief Get singleton instance of pool.\n   * \\return Object Pool.\n   */\n  static ObjectPool* Get();\n\n  /*!\n   * \\brief Get a shared ptr of the singleton instance of pool.\n   * \\return Shared pointer to the Object Pool.\n   */\n  static const std::shared_ptr<ObjectPool>& _GetSharedRef();\n\n private:\n  /*!\n   * \\brief Internal structure to hold pointers.\n   */\n  struct LinkedList {\n#if defined(_MSC_VER)\n    T t;\n    LinkedList* next{nullptr};\n#else\n    union {\n      T t;\n      LinkedList* next{nullptr};\n    };\n#endif\n  };\n  /*!\n   * \\brief Page size of allocation.\n   *\n   * Currently defined to be 4KB.\n   */\n  constexpr static std::size_t kPageSize = 1 << 12;\n  /*! \\brief internal mutex */\n  std::mutex m_;\n  /*!\n   * \\brief Head of free list.\n   */\n  LinkedList* head_{nullptr};\n  /*!\n   * \\brief Pages allocated.\n   */\n  std::vector<void*> allocated_;\n  /*!\n   * \\brief Private constructor.\n   */\n  ObjectPool();\n  /*!\n   * \\brief Allocate a page of raw objects.\n   *\n   * This function is not protected and must be called with caution.\n   */\n  void AllocateChunk();\n  DISALLOW_COPY_AND_ASSIGN(ObjectPool);\n};  // class ObjectPool\n\n/*!\n * \\brief Helper trait class for easy allocation and deallocation.\n */\ntemplate <typename T>\nstruct ObjectPoolAllocatable {\n  /*!\n   * \\brief Create new object.\n   * \\return Pointer to the new object.\n   */\n  template <typename... Args>\n  static T* New(Args&&... args);\n  /*!\n   * \\brief Delete an existing object.\n   * \\param ptr The pointer to delete.\n   *\n   * Make sure the pointer to delete is allocated from this pool.\n   */\n  static void Delete(T* ptr);\n};  // struct ObjectPoolAllocatable\n\ntemplate <typename T>\nObjectPool<T>::~ObjectPool() {\n  for (auto i : allocated_) {\n#ifdef _MSC_VER\n    _aligned_free(i);\n#else\n    free(i);\n#endif\n  }\n}\n\ntemplate <typename T>\ntemplate <typename... Args>\nT* ObjectPool<T>::New(Args&&... args) {\n  LinkedList* ret;\n  {\n    std::lock_guard<std::mutex> lock{m_};\n    if (head_->next == nullptr) {\n      AllocateChunk();\n    }\n    ret   = head_;\n    head_ = head_->next;\n  }\n  return new (static_cast<void*>(ret)) T(std::forward<Args>(args)...);\n}\n\ntemplate <typename T>\nvoid ObjectPool<T>::Delete(T* ptr) {\n  ptr->~T();\n  auto linked_list_ptr = reinterpret_cast<LinkedList*>(ptr);\n  {\n    std::lock_guard<std::mutex> lock{m_};\n    linked_list_ptr->next = head_;\n    head_                 = linked_list_ptr;\n  }\n}\n\ntemplate <typename T>\nObjectPool<T>* ObjectPool<T>::Get() {\n  return _GetSharedRef().get();\n}\n\ntemplate <typename T>\nconst std::shared_ptr<ObjectPool<T> >& ObjectPool<T>::_GetSharedRef() {\n  static std::shared_ptr<ObjectPool<T> > inst_ptr(new ObjectPool<T>());\n  return inst_ptr;\n}\n\ntemplate <typename T>\nObjectPool<T>::ObjectPool() {\n  AllocateChunk();\n}\n\ntemplate <typename T>\nvoid ObjectPool<T>::AllocateChunk() {\n  static_assert(sizeof(LinkedList) <= kPageSize, \"Object too big.\");\n  static_assert(sizeof(LinkedList) % alignof(LinkedList) == 0, \"ObjectPooll Invariant\");\n  static_assert(alignof(LinkedList) % alignof(T) == 0, \"ObjectPooll Invariant\");\n  static_assert(kPageSize % alignof(LinkedList) == 0, \"ObjectPooll Invariant\");\n  void* new_chunk_ptr;\n#ifdef _MSC_VER\n  new_chunk_ptr = _aligned_malloc(kPageSize, kPageSize);\n  CHECK(new_chunk_ptr != nullptr) << \"Allocation failed\";\n#else\n  int ret = posix_memalign(&new_chunk_ptr, kPageSize, kPageSize);\n  CHECK_EQ(ret, 0) << \"Allocation failed\";\n#endif\n  allocated_.emplace_back(new_chunk_ptr);\n  auto new_chunk = static_cast<LinkedList*>(new_chunk_ptr);\n  auto size      = kPageSize / sizeof(LinkedList);\n  for (std::size_t i = 0; i < size - 1; ++i) {\n    new_chunk[i].next = &new_chunk[i + 1];\n  }\n  new_chunk[size - 1].next = head_;\n  head_                    = new_chunk;\n}\n\ntemplate <typename T>\ntemplate <typename... Args>\nT* ObjectPoolAllocatable<T>::New(Args&&... args) {\n  return ObjectPool<T>::Get()->New(std::forward<Args>(args)...);\n}\n\ntemplate <typename T>\nvoid ObjectPoolAllocatable<T>::Delete(T* ptr) {\n  ObjectPool<T>::Get()->Delete(ptr);\n}\n\n}  // namespace common\n}  // namespace mxnet\n#endif  // MXNET_COMMON_OBJECT_POOL_H_\n"
  },
  {
    "path": "src/common/random_generator.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file random_generator.cu\n * \\brief gpu implements for parallel random number generator.\n */\n\n#include <mxnet/random_generator.h>\n#include <algorithm>\n#include \"../operator/mxnet_op.h\"\n\nnamespace mxnet {\nnamespace common {\nnamespace random {\n\ntemplate <>\nconst int RandGenerator<gpu, float>::kMinNumRandomPerThread = 64;\n\ntemplate <>\nconst int RandGenerator<gpu, float>::kNumRandomStates = 32768;\n\n__global__ void rand_generator_seed_kernel(curandStatePhilox4_32_10_t* states_,\n                                           const int size,\n                                           uint32_t seed) {\n  int id = blockIdx.x * blockDim.x + threadIdx.x;\n  if (id < size)\n    curand_init(seed, id, 0, states_ + id);\n}\n\ntemplate <>\nvoid RandGenerator<gpu, float>::Seed(mshadow::Stream<gpu>* s, uint32_t seed) {\n  using namespace mshadow::cuda;\n  int ngrid =\n      std::min(kMaxGridNum,\n               (RandGenerator<gpu, float>::kNumRandomStates + kBaseThreadNum - 1) / kBaseThreadNum);\n  rand_generator_seed_kernel<<<ngrid, kBaseThreadNum, 0, mshadow::Stream<gpu>::GetStream(s)>>>(\n      states_, RandGenerator<gpu, float>::kNumRandomStates, seed);\n  MSHADOW_CUDA_POST_KERNEL_CHECK(rand_generator_seed_kernel);\n  s->Wait();\n}\n\ntemplate <>\nvoid RandGenerator<gpu, float>::AllocState(RandGenerator<gpu>* inst) {\n  CUDA_CALL(cudaMalloc(&inst->states_, kNumRandomStates * sizeof(curandStatePhilox4_32_10_t)));\n}\n\ntemplate <>\nvoid RandGenerator<gpu, float>::FreeState(RandGenerator<gpu>* inst) {\n  CUDA_CALL(cudaFree(inst->states_));\n}\n\ntemplate <>\nvoid* RandGenerator<gpu, float>::GetStates() {\n  return static_cast<void*>(states_);\n}\n\n}  // namespace random\n}  // namespace common\n}  // namespace mxnet\n"
  },
  {
    "path": "src/common/rtc.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#include <mxnet/rtc.h>\n#include <typeinfo>\n\n#include \"cuda/utils.h\"\n#include \"../operator/operator_common.h\"\n\n#if MXNET_USE_CUDA\n\nnamespace mxnet {\nnamespace rtc {\n\nCudaModule::Chunk::Chunk(const char* source,\n                         const std::vector<std::string>& options,\n                         const std::vector<std::string>& exports) {\n  NVRTC_CALL(nvrtcCreateProgram(&prog_, source, \"source.cu\", 0, nullptr, nullptr));\n  for (const auto& i : exports)\n    exports_.insert(i);\n#if CUDA_VERSION >= 8000\n  for (const auto& func : exports) {\n    NVRTC_CALL(nvrtcAddNameExpression(prog_, func.c_str()));\n  }\n#else\n  CHECK_EQ(exports.size(), 0)\n      << \"Exporting is only supported with CUDA 8.0 and above. \"\n      << \"For lower version of CUDA, please prepend your kernel defintiions \"\n      << \"with extern \\\"C\\\" instead.\";\n#endif\n  std::vector<const char*> c_options;\n  for (const auto& i : options)\n    c_options.push_back(i.c_str());\n  nvrtcResult compile_res = nvrtcCompileProgram(prog_, c_options.size(), c_options.data());\n  if (compile_res != NVRTC_SUCCESS) {\n    size_t err_size;\n    NVRTC_CALL(nvrtcGetProgramLogSize(prog_, &err_size));\n    std::vector<char> err(err_size);\n    NVRTC_CALL(nvrtcGetProgramLog(prog_, err.data()));\n    LOG(FATAL) << err.data();\n  }\n\n  bool use_ptx = true;\n  for (const auto& opt : options) {\n    if (opt.find(\"sm_\") != std::string::npos) {\n      use_ptx = false;\n      break;\n    }\n  }\n\n  if (use_ptx) {\n    size_t ptx_size;\n    NVRTC_CALL(nvrtcGetPTXSize(prog_, &ptx_size));\n    ptx_.resize(ptx_size);\n    NVRTC_CALL(nvrtcGetPTX(prog_, ptx_.data()));\n  } else {\n#if CUDA_VERSION >= 11010\n    size_t cubin_size;\n    NVRTC_CALL(nvrtcGetCUBINSize(prog_, &cubin_size));\n    ptx_.resize(cubin_size);\n    NVRTC_CALL(nvrtcGetCUBIN(prog_, ptx_.data()));\n#else\n    LOG(FATAL) << \"Your CUDA version does not support compiling for sm_XX target. \"\n               << \"Use compute_XX target instead or upgrade to CUDA 11.1 or later.\";\n#endif\n  }\n}\n\nCudaModule::Chunk::~Chunk() {\n  for (const auto& kv : mod_) {\n    CUDA_DRIVER_CALL(cuModuleUnload(kv.second));\n  }\n  NVRTC_CALL(nvrtcDestroyProgram(&prog_));\n}\n\nCUfunction CudaModule::Chunk::GetFunction(const std::string& mangled_name, const Context& ctx) {\n  CHECK_EQ(ctx.dev_mask(), Context::kGPU) << \"CUDA Runtime compilation only supports Nvidia GPU.\";\n  auto iter = mod_.find(ctx.dev_id);\n  mxnet::common::cuda::DeviceStore device_store;\n  CUmodule module;\n  if (iter != mod_.end()) {\n    module = iter->second;\n  } else {\n    device_store.SetDevice(ctx.dev_id);\n    CUDA_DRIVER_CALL(cuModuleLoadDataEx(&module, ptx_.data(), 0, nullptr, nullptr));\n    mod_[ctx.dev_id] = module;\n  }\n  CUfunction function;\n  auto err = cuModuleGetFunction(&function, module, mangled_name.c_str());\n  if (err == CUDA_ERROR_NOT_FOUND) {\n    LOG(FATAL) << \"Cannot find cuda kernel with name '\" << mangled_name\n               << \"'. Please either prepend kernel definition \"\n               << \"with 'extern \\\"C\\\"' or add its name to exports \"\n               << \"when creating CudaModule.\";\n  }\n  CUDA_DRIVER_CALL(err);\n  return function;\n}\n\nstd::shared_ptr<CudaModule::Kernel> CudaModule::GetKernel(const std::string& name,\n                                                          const std::vector<ArgType>& signature) {\n  std::string mangled_name = name;\n#if CUDA_VERSION >= 8000\n  if (ptr_->exports_.count(name)) {\n    const char* c_mangled_name;\n    NVRTC_CALL(nvrtcGetLoweredName(ptr_->prog_, name.c_str(), &c_mangled_name));\n    mangled_name = c_mangled_name;\n  }\n#endif\n  return std::shared_ptr<Kernel>(new Kernel(ptr_, mangled_name, signature));\n}\n\nCudaModule::Kernel::Kernel(const std::shared_ptr<CudaModule::Chunk>& mod,\n                           const std::string& mangled_name,\n                           const std::vector<ArgType>& signature)\n    : mangled_name_(mangled_name), signature_(signature), mod_(mod) {}\n\nvoid CudaModule::Kernel::Launch(const Context& ctx,\n                                const std::vector<dmlc::any>& args,\n                                uint32_t grid_dim_x,\n                                uint32_t grid_dim_y,\n                                uint32_t grid_dim_z,\n                                uint32_t block_dim_x,\n                                uint32_t block_dim_y,\n                                uint32_t block_dim_z,\n                                uint32_t shared_mem) {\n  CHECK_EQ(ctx.dev_mask(), Context::kGPU) << \"CUDA Runtime compilation only supports Nvidia GPU.\";\n\n  auto mod       = mod_;\n  auto arg_types = signature();\n\n  CUfunction function;\n  auto iter = func_.find(ctx.dev_id);\n  if (iter != func_.end()) {\n    function = iter->second;\n  } else {\n    function          = mod_->GetFunction(mangled_name_, ctx);\n    func_[ctx.dev_id] = function;\n  }\n\n  std::vector<Engine::VarHandle> read_vars, write_vars;\n  for (size_t i = 0; i < arg_types.size(); ++i) {\n    if (!arg_types[i].is_ndarray)\n      continue;\n    const auto& array = dmlc::get<NDArray>(args[i]);\n    CHECK_EQ(array.dtype(), arg_types[i].dtype)\n        << \"The i-th argument is expected to be an NDArray of \"\n        << op::type_string(arg_types[i].dtype) << \" type, but got \"\n        << op::type_string(array.dtype()) << \" instead.\";\n    if (arg_types[i].is_const) {\n      read_vars.emplace_back(array.var());\n    } else {\n      write_vars.emplace_back(array.var());\n    }\n  }\n\n  Engine::Get()->PushSync(\n      [function,\n       mod,\n       args,\n       arg_types,\n       grid_dim_x,\n       grid_dim_y,\n       grid_dim_z,\n       block_dim_x,\n       block_dim_y,\n       block_dim_z,\n       shared_mem](RunContext rctx) {\n        std::vector<void*> p_args;\n        for (size_t i = 0; i < arg_types.size(); ++i) {\n          if (arg_types[i].is_ndarray) {\n            const auto& array = dmlc::get<NDArray>(args[i]);\n            p_args.push_back(reinterpret_cast<void*>(const_cast<void**>(&array.data().dptr_)));\n          } else {\n            MSHADOW_TYPE_SWITCH(arg_types[i].dtype, DType, {\n              const auto& number = dmlc::get<DType>(args[i]);\n              p_args.push_back(const_cast<DType*>(&number));\n            });\n          }\n        }\n\n        mshadow::Stream<gpu>* s = rctx.get_stream<gpu>();\n        CUDA_DRIVER_CALL(cuLaunchKernel(function,\n                                        grid_dim_x,\n                                        grid_dim_y,\n                                        grid_dim_z,\n                                        block_dim_x,\n                                        block_dim_y,\n                                        block_dim_z,\n                                        shared_mem,\n                                        s->stream_,\n                                        p_args.data(),\n                                        nullptr));\n        CUDA_CALL(cudaStreamSynchronize(s->stream_));\n      },\n      ctx,\n      read_vars,\n      write_vars,\n      FnProperty::kNormal,\n      0,\n      mangled_name_.c_str());\n}\n\n}  // namespace rtc\n}  // namespace mxnet\n\n#endif  // MXNET_USE_CUDA\n"
  },
  {
    "path": "src/common/static_array.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file static_array.h\n */\n#ifndef MXNET_COMMON_STATIC_ARRAY_H_\n#define MXNET_COMMON_STATIC_ARRAY_H_\n\n#include <mshadow/base.h>\n\nnamespace mxnet {\nnamespace common {\n\n/*! \\brief\n * Static array. This code is borrowed from struct Shape<ndim>,\n * except that users can specify the type of the elements of\n * the statically allocated array.\n * The object instance of the struct is copyable between CPU and GPU.\n * \\tparam T element type of the array, must be copyable between CPU and GPU\n * \\tparam num number of elements in the array\n */\ntemplate <typename T, int num>\nstruct StaticArray {\n  static const int kNum = num;\n\n  T array_[kNum];\n\n  /*! \\brief default constructor, do nothing */\n  MSHADOW_XINLINE StaticArray(void) {}\n\n  /*! \\brief constructor, fill in the array with the input value */\n  MSHADOW_XINLINE StaticArray(const T& val) {\n#pragma unroll\n    for (int i = 0; i < num; ++i) {\n      this->array_[i] = val;\n    }\n  }\n\n  /*! \\brief constuctor */\n  MSHADOW_XINLINE StaticArray(const StaticArray<T, num>& sa) {\n#pragma unroll\n    for (int i = 0; i < num; ++i) {\n      this->array_[i] = sa[i];\n    }\n  }\n\n  MSHADOW_XINLINE T& operator[](const index_t idx) {\n    return array_[idx];\n  }\n\n  MSHADOW_XINLINE const T& operator[](const index_t idx) const {\n    return array_[idx];\n  }\n};  // StaticArray\n\n}  // namespace common\n}  // namespace mxnet\n#endif  // MXNET_COMMON_STATIC_ARRAY_H_\n"
  },
  {
    "path": "src/common/tensor_inspector.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file tensor_inspector.h\n * \\brief utility to inspect tensor objects\n * \\author Zhaoqi Zhu\n */\n\n#ifndef MXNET_COMMON_TENSOR_INSPECTOR_H_\n#define MXNET_COMMON_TENSOR_INSPECTOR_H_\n\n#include <algorithm>\n#include <cmath>\n#include <string>\n#include <vector>\n#include <fstream>\n#include \"../../3rdparty/mshadow/mshadow/base.h\"\n\nnamespace mxnet {\n\n/*!\n * \\brief this singleton struct mediates individual TensorInspector objects\n * so that we can control the global behavior from each of them\n */\nstruct InspectorManager {\n  static InspectorManager* get() {\n    static std::mutex mtx;\n    static std::unique_ptr<InspectorManager> im = nullptr;\n    if (!im) {\n      std::unique_lock<std::mutex> lk(mtx);\n      if (!im)\n        im = std::make_unique<InspectorManager>();\n    }\n    return im.get();\n  }\n  /* !\\brief mutex used to lock interactive_print() and check_value() */\n  std::mutex mutex_;\n  /* !\\brief skip all interactive prints */\n  bool interactive_print_skip_all_ = false;\n  /* !\\brief skip all value checks */\n  bool check_value_skip_all_ = false;\n  /* !\\brief visit count for interactive print tags */\n  std::unordered_map<std::string, int> interactive_print_tag_counter_;\n  /* !\\brief visit count for check value tags */\n  std::unordered_map<std::string, int> check_value_tag_counter_;\n  /* !\\brief visit count for dump value tags */\n  std::unordered_map<std::string, int> dump_to_file_tag_counter_;\n};\n\n/*!\n * \\brief Enum for building value checkers for TensorInspector::check_value()\n */\nenum CheckerType {\n  NegativeChecker,  // check if is negative\n  PositiveChecker,  // check if is positive\n  ZeroChecker,      // check if is zero\n  NaNChecker,       // check if is NaN, will always return false if DType is not a float type\n  InfChecker,       // check if is infinity, will always return false if DType is not a float type\n  PositiveInfChecker,  // check if is positive infinity,\n                       // will always return false if DType is not a float type\n  NegativeInfChecker,  // check if is nagative infinity,\n                       // will always return false if DType is not a float type\n  FiniteChecker,       // check if is finite, will always return false if DType is not a float type\n  NormalChecker,       // check if is neither infinity nor NaN\n  AbnormalChecker,     // chekck if is infinity or nan\n};\n\n/**\n *  _______                      _____                           _\n * |__   __|                    |_   _|                         | |\n *    | | ___ _ __  ___  ___  _ __| |  _ __  ___ _ __   ___  ___| |_ ___  _ __\n *    | |/ _ \\ '_ \\/ __|/ _ \\| '__| | | '_ \\/ __| '_ \\ / _ \\/ __| __/ _ \\| '__|\n *    | |  __/ | | \\__ \\ (_) | | _| |_| | | \\__ \\ |_) |  __/ (__| || (_) | |\n *    |_|\\___|_| |_|___/\\___/|_||_____|_| |_|___/ .__/ \\___|\\___|\\__\\___/|_|\n *                                              | |\n *                                              |_|\n */\n\n/*!\n * \\brief This class provides a unified interface to inspect the value of all data types\n * including Tensor, TBlob, and NDArray. If the tensor resides on GPU, then it will be\n * copied from GPU memory back to CPU memory to be operated on. Internally, all data types\n * are stored as a TBlob object tb_.\n */\nclass TensorInspector {\n private:\n  /*!\n   * \\brief generate the tensor info, including data type and shape\n   * \\tparam DType the data type\n   * \\tparam StreamType the type of the stream object\n   * \\param os stream object to output to\n   */\n  template <typename DType, typename StreamType>\n  void tensor_info_to_string(StreamType* os) {\n    const int dimension = tb_.ndim();\n    *os << \"<\" << infer_type_string(typeid(DType)) << \" Tensor \";\n    *os << tb_.shape_[0];\n    for (int i = 1; i < dimension; ++i) {\n      *os << 'x' << tb_.shape_[i];\n    }\n    *os << \">\" << std::endl;\n  }\n\n  /*!\n   * \\brief output the tensor info, including data type and shape\n   * \\tparam DType the data type\n   * \\tparam StreamType the type of the stream object\n   * \\param os stream object to output to\n   * \\param shape the shape of the tensor\n   */\n  template <typename DType, typename StreamType>\n  void tensor_info_to_string(StreamType* os, const std::vector<index_t>& shape) {\n    const int dimension = shape.size();\n    *os << \"<\" << infer_type_string(typeid(DType)) << \" Tensor \";\n    *os << shape[0];\n    for (int i = 1; i < dimension; ++i) {\n      *os << 'x' << shape[i];\n    }\n    *os << \">\" << std::endl;\n  }\n\n  /*!\n   * \\brief output the tensor in a structured format\n   * \\tparam DType the data type\n   * \\tparam StreamType the type of the stream object\n   * \\param os stream object to output to\n   */\n  template <typename DType, typename StreamType>\n  void to_string_helper(StreamType* os) {\n#if MXNET_USE_CUDA\n    if (tb_.dev_mask() == gpu::kDevMask) {\n      TensorInspector(test::CAccessAsCPU(ctx_, tb_, false)(), ctx_).to_string_helper<DType>(os);\n      return;\n    }\n#endif  // MXNET_USE_CUDA\n    const int dimension = tb_.ndim();\n    std::vector<index_t> offsets;\n    index_t multiple = 1;\n    for (int i = dimension - 1; i >= 0; --i) {\n      multiple *= tb_.shape_[i];\n      offsets.push_back(multiple);\n    }\n    *os << std::string(dimension, '[');\n    *os << tb_.dptr<DType>()[0];\n    for (index_t i = 1; i < static_cast<index_t>(tb_.shape_.Size()); ++i) {\n      int n = 0;\n      for (auto off : offsets) {\n        n += (i % off == 0);\n      }\n      if (n) {\n        *os << std::string(n, ']') << \", \" << std::string(n, '[');\n      } else {\n        *os << \", \";\n      }\n      *os << tb_.dptr<DType>()[i];\n    }\n    *os << std::string(dimension, ']') << std::endl;\n    tensor_info_to_string<DType>(os);\n  }\n\n  /*!\n   * \\brief output the tensor in a structured format\n   * \\tparam DType the data type\n   * \\tparam StreamType the type of the stream object\n   * \\param os stream object to output to\n   * \\param dptr the data pointer\n   */\n  template <typename DType, typename StreamType>\n  void to_string_helper(StreamType* os, const DType* dptr) {\n#if MXNET_USE_CUDA\n    if (tb_.dev_mask() == gpu::kDevMask) {\n      TensorInspector(test::CAccessAsCPU(ctx_, tb_, false)(), ctx_)\n          .to_string_helper<DType>(os, dptr);\n      return;\n    }\n#endif  // MXNET_USE_CUDA\n    *os << *dptr << std::endl;\n    *os << \"<\" << typeid(*dptr).name() << \">\" << std::endl;\n  }\n\n  /*!\n   * \\brief output a part of the tensor in a structed format\n   * \\tparam DType the data type\n   * \\tparam StreamType the type of the stream object\n   * \\param os stream object to output to\n   * \\param sub_shape the sub-shape of the desired part of the tensor\n   * \\param offset the position of the first value of the desired part of the tensor\n   */\n  template <typename DType, typename StreamType>\n  void to_string_helper(StreamType* os, const std::vector<index_t>& sub_shape, index_t offset) {\n#if MXNET_USE_CUDA\n    if (tb_.dev_mask() == gpu::kDevMask) {\n      TensorInspector(test::CAccessAsCPU(ctx_, tb_, false)(), ctx_)\n          .to_string_helper<DType>(os, sub_shape, offset);\n      return;\n    }\n#endif  // MXNET_USE_CUDA\n    DType* dptr = tb_.dptr<DType>() + offset;\n    if (sub_shape.size() == 0) {\n      to_string_helper<DType>(os, dptr);\n      return;\n    }\n    const int dimension = sub_shape.size();\n    std::vector<index_t> offsets;\n    index_t multiple = 1;\n    for (int i = dimension - 1; i >= 0; --i) {\n      multiple *= sub_shape[i];\n      offsets.push_back(multiple);\n    }\n    std::stringstream ss;\n    *os << std::string(dimension, '[');\n    *os << dptr[0];\n    for (index_t i = 1; i < multiple; ++i) {\n      int n = 0;\n      for (auto off : offsets) {\n        n += (i % off == 0);\n      }\n      if (n) {\n        *os << std::string(n, ']') << \", \" << std::string(n, '[');\n      } else {\n        *os << \", \";\n      }\n      *os << dptr[i];\n    }\n    *os << std::string(dimension, ']') << std::endl;\n    tensor_info_to_string<DType>(os, sub_shape);\n  }\n\n  /*!\n   * \\brief helper function to calculate the sub_shape and offset for the desired part of the\n   * tensor, given its coordinates in the original tensor \\param pos the coordinates of the desired\n   * part of the tensor \\param sub_shape the sub-shape of the desired part of the tensor; calculated\n   * here \\param offset the position of the first value of the desired part of the tensor;\n   * calculated here\n   */\n  void print_locator(const std::vector<index_t>& pos,\n                     std::vector<index_t>* sub_shape,\n                     index_t* offset) {\n    const int dimension = tb_.ndim();\n    const int sub_dim   = dimension - pos.size();\n    sub_shape->resize(sub_dim);\n    index_t multiple = 1;\n    for (size_t i = pos.size(), j = 0; i < static_cast<size_t>(dimension); ++i, ++j) {\n      (*sub_shape)[j] = tb_.shape_[i];\n      multiple *= tb_.shape_[i];\n    }\n    index_t sum = 0;\n    index_t m   = 1;\n    for (index_t i = pos.size() - 1; i >= 0; --i) {\n      sum += pos[i] * m;\n      m *= tb_.shape_[i];\n    }\n    *offset = sum * multiple;\n  }\n\n  /*!\n   * \\brief parse the coordinate of the desired part of the tensor, given a string that represents\n   * that coordinate \\param pos the coordinates of the desired part of the tensor, calculated here\n   * \\param str the string that represents the coordinate\n   */\n  bool parse_position(std::vector<index_t>* pos, const std::string& str) {\n    const int dimension = tb_.ndim();\n    std::istringstream ss(str);\n    index_t n;\n    while (ss >> n) {\n      pos->push_back(n);\n      if (ss.peek() == ',') {\n        ss.ignore();\n      }\n    }\n    if (pos->size() > static_cast<size_t>(dimension)) {\n      return false;\n    }\n    for (size_t i = 0; i < pos->size(); ++i) {\n      if ((*pos)[i] > (tb_.shape_[i] - 1) || (*pos)[i] < 0) {\n        return false;\n      }\n    }\n    return !pos->empty();\n  }\n\n  /*!\n   * \\brief interactive print the tensor value\n   * \\tparam DType the data type\n   * \\param tag the name given to this call\n   */\n  template <typename DType>\n  void interactive_print_helper(std::string tag) {\n#if MXNET_USE_CUDA\n    if (tb_.dev_mask() == gpu::kDevMask) {\n      TensorInspector(test::CAccessAsCPU(ctx_, tb_, false)(), ctx_)\n          .interactive_print_helper<DType>(tag);\n      return;\n    }\n#endif  // MXNET_USE_CUDA\n    std::lock_guard<std::mutex> lock(InspectorManager::get()->mutex_);\n    InspectorManager::get()->interactive_print_tag_counter_[tag] += 1;\n    while (!InspectorManager::get()->interactive_print_skip_all_) {\n      std::cout << \"----------Interactive Print----------\" << std::endl;\n      if (tag != \"\") {\n        std::cout << \"Tag: \" << tag\n                  << \"  Visit: \" << InspectorManager::get()->interactive_print_tag_counter_[tag]\n                  << std::endl;\n      }\n      tensor_info_to_string<DType>(&std::cout);\n      std::cout << \"To print a part of the tensor, \"\n                << \"please specify a position, seperated by \\\",\\\"\" << std::endl;\n      std::cout << \"\\\"e\\\" for the entire tensor, \"\n                << \"\\\"d\\\" to dump value to file, \"\n                << \"\\\"b\\\" to break, \"\n                << \"\\\"s\\\" to skip all: \";\n      std::string str;\n      std::cin >> str;\n      if (str == \"b\") {\n        break;\n      } else if (str == \"e\") {\n        to_string_helper<DType>(&std::cout);\n        continue;\n      } else if (str == \"s\") {\n        InspectorManager::get()->interactive_print_skip_all_ = true;\n        break;\n      } else if (str == \"d\") {\n        while (true) {\n          std::cout << \"Please enter a tag: \";\n          std::cin >> str;\n          if (str.find(' ') != std::string::npos) {\n            std::cout << \"Invalid tag name. No space allowed.\";\n            continue;\n          }\n          dump_to_file_helper<DType>(str);\n          break;\n        }\n        continue;\n      }\n      std::vector<index_t> pos;\n      if (parse_position(&pos, str)) {\n        std::vector<index_t> sub_shape;\n        index_t offset;\n        print_locator(pos, &sub_shape, &offset);\n        to_string_helper<DType>(&std::cout, sub_shape, offset);\n      } else {\n        std::cout << \"invalid command/indices\" << std::endl;\n      }\n    }\n  }\n\n  /*!\n   * \\brief build the lambda function, aka the checker, given its type\n   * \\tparam DType the data type\n   * \\param ct the type of the checker\n   */\n  template <typename DType>\n  std::function<bool(DType)> get_checker(CheckerType ct) {\n    switch (ct) {\n      case NegativeChecker:\n        return [](DType x) { return x < 0; };\n      case PositiveChecker:\n        return [](DType x) { return x > 0; };\n      case ZeroChecker:\n        return [](DType x) { return x == 0; };\n      case NaNChecker:\n        if (std::is_same<DType, float>::value || std::is_same<DType, double>::value ||\n            std::is_same<DType, mshadow::half::half_t>::value) {\n          return [](DType x) { return x != x; };\n        } else {\n          LOG(WARNING) << \"NaNChecker only applies to float types. \"\n                       << \"Lambda will always return false.\";\n        }\n        break;\n      case InfChecker:\n        if (std::is_same<DType, float>::value || std::is_same<DType, double>::value ||\n            std::is_same<DType, mshadow::half::half_t>::value) {\n          return [](DType x) { return x == (DType)1.0 / 0.0f || x == -(DType)1.0 / 0.0f; };\n        } else {\n          LOG(WARNING) << \"InfChecker only applies to float types. \"\n                       << \"Lambda will always return false.\";\n        }\n        break;\n      case PositiveInfChecker:\n        if (std::is_same<DType, float>::value || std::is_same<DType, double>::value ||\n            std::is_same<DType, mshadow::half::half_t>::value) {\n          return [](DType x) { return x == (DType)1.0 / 0.0f; };\n        } else {\n          LOG(WARNING) << \"PositiveInfChecker only applies to float types. \"\n                       << \"Lambda will always return false.\";\n        }\n        break;\n      case NegativeInfChecker:\n        if (std::is_same<DType, float>::value || std::is_same<DType, double>::value ||\n            std::is_same<DType, mshadow::half::half_t>::value) {\n          return [](DType x) { return x == -(DType)1.0 / 0.0f; };\n        } else {\n          LOG(WARNING) << \"NegativeInfChecker only applies to float types. \"\n                       << \"Lambda will always return false.\";\n        }\n        break;\n      case FiniteChecker:\n        if (std::is_same<DType, float>::value || std::is_same<DType, double>::value ||\n            std::is_same<DType, mshadow::half::half_t>::value) {\n          return [](DType x) { return x != (DType)1.0 / 0.0f && x != -(DType)1.0 / 0.0f; };\n        } else {\n          LOG(WARNING) << \"FiniteChecker only applies to float types. \"\n                       << \"Lambda will always return false.\";\n        }\n        break;\n      case NormalChecker:\n        if (std::is_same<DType, float>::value || std::is_same<DType, double>::value ||\n            std::is_same<DType, mshadow::half::half_t>::value) {\n          return\n              [](DType x) { return x != (DType)1.0 / 0.0f && x != -(DType)1.0 / 0.0f && x == x; };\n        } else {\n          LOG(WARNING) << \"NormalChecker only applies to float types. \"\n                       << \"Lambda will always return false.\";\n        }\n        break;\n      case AbnormalChecker:\n        if (std::is_same<DType, float>::value || std::is_same<DType, double>::value ||\n            std::is_same<DType, mshadow::half::half_t>::value) {\n          return\n              [](DType x) { return x == (DType)1.0 / 0.0f || x == -(DType)1.0 / 0.0f || x != x; };\n        } else {\n          LOG(WARNING) << \"AbnormalChecker only applies to float types. \"\n                       << \"Lambda will always return false.\";\n        }\n        break;\n      default:\n        return [](DType x) { return false; };\n    }\n    return [](DType x) { return false; };\n  }\n\n  /*!\n   * \\brief calculate the coordinate of a value in the tensor, given its index\n   * \\param idx the index of the value in the tensor\n   */\n  std::vector<index_t> index_to_coordinates(index_t idx) {\n    const int dimension = tb_.ndim();\n    std::vector<index_t> ret;\n    for (int i = dimension - 1; i >= 0; --i) {\n      ret.push_back(idx % tb_.shape_[i]);\n      idx /= tb_.shape_[i];\n    }\n    std::reverse(ret.begin(), ret.end());\n    return ret;\n  }\n\n  /*!\n   * \\brief check/validate the values within the tensor, find the coordinates\n   * where the value checker evaluates to true\n   * \\tparam DType the data type\n   * \\param ret a vector of coordinates which itself is a vector of int; calculated here\n   * \\param checker the lambda function to check each value of within the tensor\n   * \\param interactive wherether to allow the user to interactively check the coordinates\n   * \\param tag the name given to this call\n   */\n  template <typename DType>\n  void check_value_helper(std::vector<std::vector<index_t>>* ret,\n                          const std::function<bool(DType)>& checker,\n                          bool interactive,\n                          std::string tag) {\n#if MXNET_USE_CUDA\n    if (tb_.dev_mask() == gpu::kDevMask) {\n      return TensorInspector(test::CAccessAsCPU(ctx_, tb_, false)(), ctx_)\n          .check_value_helper<DType>(ret, checker, interactive, tag);\n    }\n#endif  // MXNET_USE_CUDA\n    index_t count = 0;\n    std::stringstream ss;\n    ss << \"[\";\n    bool first_pass = true;\n    for (index_t i = 0; i < static_cast<index_t>(tb_.shape_.Size()); ++i) {\n      if (checker(tb_.dptr<DType>()[i])) {\n        ++count;\n        if (!first_pass) {\n          ss << \", \";\n        }\n        first_pass                  = false;\n        std::vector<index_t> coords = index_to_coordinates(i);\n        ss << \"(\" << coords[0];\n        for (size_t i = 1; i < coords.size(); ++i) {\n          ss << \", \" << coords[i];\n        }\n        ss << \")\";\n        ret->push_back(coords);\n      }\n    }\n    ss << \"]\" << std::endl;\n    if (interactive) {\n      std::lock_guard<std::mutex> lock(InspectorManager::get()->mutex_);\n      InspectorManager::get()->check_value_tag_counter_[tag] += 1;\n      while (!InspectorManager::get()->check_value_skip_all_) {\n        std::cout << \"----------Value Check----------\" << std::endl;\n        tensor_info_to_string<DType>(&std::cout);\n        if (tag != \"\") {\n          std::cout << \"Tag: \" << tag\n                    << \"  Visit: \" << InspectorManager::get()->check_value_tag_counter_[tag]\n                    << std::endl;\n        }\n        std::cout << count << \" value(s) found.\" << std::endl;\n        std::cout << \"To print a part of the tensor,\"\n                  << \" please specify a position, seperated by \\\",\\\"\" << std::endl;\n        std::cout << \"\\\"e\\\" for the entire tensor, \"\n                  << \"\\\"p\\\" to print the coordinates of the values found, \"\n                  << \"\\\"b\\\" to break, \"\n                  << \"\\\"s\\\" to skip all: \";\n        std::string str;\n        std::cin >> str;\n        if (str == \"b\") {\n          break;\n        } else if (str == \"e\") {\n          to_string_helper<DType>(&std::cout);\n          continue;\n        } else if (str == \"p\") {\n          std::cout << ss.str() << std::endl;\n          continue;\n        } else if (str == \"s\") {\n          InspectorManager::get()->check_value_skip_all_ = true;\n          break;\n        }\n        std::vector<index_t> pos;\n        if (parse_position(&pos, str)) {\n          std::vector<index_t> sub_shape;\n          index_t offset;\n          print_locator(pos, &sub_shape, &offset);\n          to_string_helper<DType>(&std::cout, sub_shape, offset);\n        } else {\n          std::cout << \"invalid command/indices\" << std::endl;\n        }\n      }\n    }\n  }\n\n  /*!\n   * \\brief infer the python type, given the c++ type\n   * \\tparam ti the type info\n   */\n  inline char infer_type(const std::type_info& ti) {\n    if (ti == typeid(float))\n      return 'f';\n    else if (ti == typeid(double))\n      return 'f';\n    else if (ti == typeid(mshadow::half::half_t))\n      return 'f';\n    else if (ti == typeid(uint8_t))\n      return 'u';\n    else if (ti == typeid(int32_t))\n      return 'i';\n    else if (ti == typeid(int64_t))\n      return 'i';\n    else\n      return '?';\n  }\n\n  /*!\n   * \\brief infer the python type, given the c++ type\n   * \\tparam ti the type info\n   */\n  inline std::string infer_type_string(const std::type_info& ti) {\n    if (ti == typeid(float))\n      return \"float\";\n    else if (ti == typeid(double))\n      return \"double\";\n    else if (ti == typeid(mshadow::half::half_t))\n      return \"mshasow::half::half_t\";\n    else if (ti == typeid(uint8_t))\n      return \"uint8_t\";\n    else if (ti == typeid(int32_t))\n      return \"int32_t\";\n    else if (ti == typeid(int64_t))\n      return \"int64_t\";\n    else\n      return \"unknown tyoe\";\n  }\n\n  /*!\n   * \\brief check if the host machine is big or small endian\n   */\n  inline char endian_test() {\n    int x = 1;\n    return (reinterpret_cast<char*>(&x)[0]) ? '<' : '>';\n  }\n\n  /*!\n   * \\brief generate the header following npy 1.0 format\n   * \\tparam DType the data type\n   */\n  template <typename DType>\n  std::string get_header() {\n    const int dimension = tb_.ndim();\n    std::string dict;\n    dict += \"{'descr':'\";\n    dict += endian_test();\n    dict += infer_type(typeid(DType));\n    dict += std::to_string(sizeof(DType));\n    dict += \"','fortran_order':False,'shape':(\";\n    dict += std::to_string(tb_.shape_[0]);\n    for (int i = 1; i < dimension; ++i) {\n      dict += ',';\n      dict += std::to_string(tb_.shape_[i]);\n    }\n    if (dimension == 1) {\n      dict += \",\";\n    }\n    dict += \")} \";\n    int padding_size = 64 - ((10 + dict.size()) % 64);\n    dict += std::string(padding_size, ' ');\n    dict.back() = '\\n';\n    std::string header;\n    header += static_cast<char>(0x93);\n    header += \"NUMPY\";\n    header += static_cast<char>(0x01);\n    header += static_cast<char>(0x00);\n    header += static_cast<char>((uint16_t)dict.size() & 0x00ff);\n    header += static_cast<char>(((uint16_t)dict.size() >> 8) & 0x00ff);\n    header += dict;\n    return header;\n  }\n\n  /*!\n   * \\brief write the header and the date to an npy file\n   * \\tparam DType the data type\n   * \\param header the header of the file\n   * \\param filename the file name\n   */\n  template <typename DType>\n  void write_npy(const std::string& header, const std::string& filename) {\n    std::ofstream file;\n    file.exceptions(std::ofstream::failbit | std::ofstream::badbit);\n    try {\n      file.open(filename, std::ios::out | std::ios::binary);\n      file.write(header.c_str(), header.size());\n      file.write(reinterpret_cast<char*>(tb_.dptr<DType>()), sizeof(DType) * tb_.shape_.Size());\n      file.close();\n      std::cout << \"Tensor dumped to file: \" << filename << std::endl;\n    } catch (std::ofstream::failure e) {\n      std::cerr << \"Exception opening/writing/closing file \" << filename << std::endl;\n    }\n  }\n\n  /*!\n   * \\brief dump the value of the tensor to a file with name \"[tag]_[visit count].npy\" in npy format\n   * the dump file follows npy 1.0 stantand\n   * \\tparam DType the data type\n   * \\param tag the name given to this call\n   */\n  template <typename DType>\n  void dump_to_file_helper(const std::string& tag) {\n#if MXNET_USE_CUDA\n    if (tb_.dev_mask() == gpu::kDevMask) {\n      TensorInspector(test::CAccessAsCPU(ctx_, tb_, false)(), ctx_).dump_to_file_helper<DType>(tag);\n      return;\n    }\n#endif  // MXNET_USE_CUDA\n    std::string header = get_header<DType>();\n    InspectorManager::get()->dump_to_file_tag_counter_[tag] += 1;\n    const int visit      = InspectorManager::get()->dump_to_file_tag_counter_[tag];\n    std::string filename = tag + \"_\" + std::to_string(visit) + \".npy\";\n    write_npy<DType>(header, filename);\n  }\n\n  /*!\n   * \\brief validate that the shape\n   */\n  inline void validate_shape() {\n    const int dimension = tb_.ndim();\n    CHECK(dimension > 0) << \"Tensor Inspector does not support empty tensors \"\n                         << \"or tensors of unknow shape.\";\n    for (int i = 0; i < dimension; ++i) {\n      CHECK(tb_.shape_[i] != 0) << \"Invalid tensor shape: shape_[\" << i << \"] is 0\";\n    }\n  }\n\n  /* !\\brief the tensor blob */\n  const TBlob tb_;\n  /* !\\brief the run context of the tensor */\n  const RunContext& ctx_;\n\n public:\n  /*!\n   * \\brief construct from Tensor object\n   * \\tparam Device the device the tensor resides in\n   * \\tparam dimension the dimension of the tensor\n   * \\tparam DType the data type\n   * \\param ts the source tensor object\n   * \\param ctx the run context of the tensor\n   */\n  template <typename Device, int dimension, typename DType>\n  TensorInspector(const mshadow::Tensor<Device, dimension, DType>& ts, const RunContext& ctx)\n      : tb_(ts), ctx_(ctx) {\n    validate_shape();\n  }\n\n  /*!\n   * \\brief construct from TBlob object\n   * \\param tb the source tblob object\n   * \\param ctx the run context of the tensor\n   */\n  TensorInspector(const TBlob& tb, const RunContext& ctx) : tb_(tb), ctx_(ctx) {\n    validate_shape();\n  }\n\n  /*!\n   * \\brief construct from NDArray object. Currently this only works with kDefaultStorage\n   * \\param arr the source ndarray object\n   * \\param ctx the run context of the tensor\n   */\n  TensorInspector(const NDArray& arr, const RunContext& ctx) : tb_(arr.data()), ctx_(ctx) {\n    validate_shape();\n  }\n\n  /*!\n   * \\brief print the tensor to std::cout\n   */\n  void print_string() {\n    std::cout << to_string() << std::endl;\n  }\n\n  /*!\n   * \\brief return a string which contains the values and other info of the tensor\n   */\n  std::string to_string() {\n    std::stringstream ss;\n    MSHADOW_TYPE_SWITCH(tb_.type_flag_, DType, { to_string_helper<DType>(&ss); });\n    return ss.str();\n  }\n\n  /*!\n   * \\brief interactively print the tensor value\n   * \\param tag the name given to this call\n   */\n  void interactive_print(std::string tag = \"\") {\n    MSHADOW_TYPE_SWITCH(tb_.type_flag_, DType, { interactive_print_helper<DType>(tag); });\n  }\n\n  /*!\n   * \\brief check/validate the values within the tensor, return the coordinates\n   * where the value checker evaluates to true\n   * \\tparam ValueChecker the type of the lambda\n   * \\param checker the lambda function to check each value of within the tensor\n   * \\param interactive wherether to allow the user to interactively check the coordinates\n   * \\param tag the name given to this call\n   */\n  template <typename ValueChecker>\n  std::vector<std::vector<index_t>> check_value(const ValueChecker& checker,\n                                                bool interactive = false,\n                                                std::string tag  = \"\") {\n    std::vector<std::vector<index_t>> ret;\n    MSHADOW_TYPE_SWITCH(tb_.type_flag_, DType, {\n      check_value_helper<DType>(&ret, checker, ret, interactive, tag);\n    });\n    return ret;\n  }\n\n  /*!\n   * \\brief check/validate the values within the tensor, return the coordinates\n   * where the lambda evaluates to true\n   * \\param ct the type of the checker\n   * \\param interactive wherether to allow the user to interactively check the coordinates\n   * \\param tag the name given to this call\n   */\n  std::vector<std::vector<index_t>> check_value(CheckerType ct,\n                                                bool interactive = false,\n                                                std::string tag  = \"\") {\n    std::vector<std::vector<index_t>> ret;\n    MSHADOW_TYPE_SWITCH(tb_.type_flag_, DType, {\n      check_value_helper<DType>(&ret, get_checker<DType>(ct), interactive, tag);\n    });\n    return ret;\n  }\n\n  /*!\n   * \\brief dump the value of the tensor to a file with name \"tag_[visit count].npy\" in npy format\n   * \\param tag the name given to this call\n   */\n  void dump_to_file(std::string tag) {\n    MSHADOW_TYPE_SWITCH(tb_.type_flag_, DType, { dump_to_file_helper<DType>(tag); });\n  }\n};\n\n}  // namespace mxnet\n\n#endif  // MXNET_COMMON_TENSOR_INSPECTOR_H_\n"
  },
  {
    "path": "src/common/utils.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file utils.cc\n * \\brief cpu implementation of util functions\n */\n\n#include \"./utils.h\"\n#include \"../operator/tensor/cast_storage-inl.h\"\n#include \"../operator/tensor/sparse_retain-inl.h\"\n\nnamespace mxnet {\nnamespace common {\n\ntemplate <>\nvoid CheckFormatWrapper<cpu>(const RunContext& rctx,\n                             const NDArray& input,\n                             const TBlob& err_cpu,\n                             const bool full_check) {\n  CheckFormatImpl<cpu>(rctx, input, err_cpu, full_check);\n}\n\ntemplate <>\nvoid SparseRetainOpForwardRspWrapper<cpu>(mshadow::Stream<cpu>* s,\n                                          const NDArray& input_nd,\n                                          const TBlob& idx_data,\n                                          const OpReqType req,\n                                          NDArray* output_nd) {\n  mxnet::op::SparseRetainOpForwardRspImpl<cpu>(s, input_nd, idx_data, req, output_nd);\n}\n\ntemplate <>\nvoid CastStorageDispatch<cpu>(const OpContext& ctx, const NDArray& input, const NDArray& output) {\n  mxnet::op::CastStorageComputeImpl<cpu>(ctx, input, output);\n}\n\nvoid ExecuteMonInputCallback(\n    const nnvm::IndexedGraph& idx,\n    const std::vector<NDArray*>& state_arrays,\n    size_t nid,\n    const std::function<void(const char*, const char*, void*)>& monitor_callback) {\n  static const auto& flist_inputs = nnvm::Op::GetAttr<nnvm::FListInputNames>(\"FListInputNames\");\n  std::vector<std::string> input_names;\n  const nnvm::IndexedGraph::Node& inode = idx[nid];\n  const nnvm::Node* node                = inode.source;\n  if (flist_inputs.count(node->op())) {\n    input_names = flist_inputs[node->op()](node->attrs);\n  } else {\n    for (size_t i = 0; i < node->num_inputs(); ++i) {\n      input_names.emplace_back(\"input\" + std::to_string(i));\n    }\n  }\n\n  for (size_t i = 0; i < node->num_inputs(); ++i) {\n    const nnvm::NodeEntry& input = node->inputs[i];\n    if (state_arrays[idx.entry_id(input)]->is_none()) {\n      continue;\n    }\n    NDArray* cpy     = new NDArray(*state_arrays[idx.entry_id(input)]);\n    std::string name = inode.source->attrs.name + \"_\" + input_names[i];\n    monitor_callback(name.c_str(), inode.source->op()->name.c_str(), reinterpret_cast<void*>(cpy));\n  }\n}\n\nvoid ExecuteMonOutputCallback(\n    const nnvm::IndexedGraph& idx,\n    const std::vector<NDArray*>& state_arrays,\n    size_t nid,\n    const std::function<void(const char*, const char*, void*)>& monitor_callback) {\n  static const auto& flist_outputs = nnvm::Op::GetAttr<nnvm::FListOutputNames>(\"FListOutputNames\");\n  std::vector<std::string> output_names;\n  const nnvm::IndexedGraph::Node& inode = idx[nid];\n  const nnvm::Node* node                = inode.source;\n  if (flist_outputs.count(node->op())) {\n    output_names = flist_outputs[node->op()](node->attrs);\n  } else {\n    for (size_t i = 0; i < node->num_outputs(); ++i) {\n      output_names.emplace_back(std::to_string(i));\n    }\n  }\n\n  for (size_t i = 0; i < node->num_outputs(); ++i) {\n    if (state_arrays[idx.entry_id(nid, i)]->is_none()) {\n      continue;\n    }\n    NDArray* cpy     = new NDArray(*state_arrays[idx.entry_id(nid, i)]);\n    std::string name = inode.source->attrs.name + \"_\" + output_names[i];\n    monitor_callback(name.c_str(), inode.source->op()->name.c_str(), reinterpret_cast<void*>(cpy));\n  }\n}\n\nMShadowTypeInfo mshadow_type_info(const int type_flag) {\n  using namespace mshadow;\n  switch (type_flag) {\n    case kFloat32:\n      return MShadowTypeInfo(\"float32\", sizeof(float));\n    case kFloat64:\n      return MShadowTypeInfo(\"float64\", sizeof(double));\n    case kFloat16:\n      return MShadowTypeInfo(\"float16\", 2, sizeof(float));\n    case kUint8:\n      return MShadowTypeInfo(\"uint8\", sizeof(uint8_t), sizeof(index_t));\n    case kUint16:\n      return MShadowTypeInfo(\"uint16\", sizeof(uint16_t));\n    case kUint32:\n      return MShadowTypeInfo(\"uint32\", sizeof(uint32_t));\n    case kUint64:\n      return MShadowTypeInfo(\"uint64\", sizeof(uint64_t));\n    case kInt16:\n      return MShadowTypeInfo(\"int16\", sizeof(int16_t));\n    case kInt32:\n      return MShadowTypeInfo(\"int32\", sizeof(int32_t));\n    case kInt8:\n      return MShadowTypeInfo(\"int8\", sizeof(int8_t), sizeof(index_t));\n    case kInt64:\n      return MShadowTypeInfo(\"int64\", sizeof(int64_t));\n    case kBool:\n      return MShadowTypeInfo(\"bool\", sizeof(bool), sizeof(index_t));\n    default:\n      LOG(FATAL) << \"Unknown type flag \" << type_flag;\n      return MShadowTypeInfo(\"INVALID\", 1);\n  }\n}\n\n}  // namespace common\n}  // namespace mxnet\n"
  },
  {
    "path": "src/common/utils.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file utils.cu\n * \\brief gpu implementation of util functions\n */\n\n#include \"./utils.h\"\n#include \"../operator/tensor/cast_storage-inl.h\"\n#include \"../operator/tensor/sparse_retain-inl.h\"\n\nnamespace mxnet {\nnamespace common {\n\ntemplate <>\nvoid CheckFormatWrapper<gpu>(const RunContext& rctx,\n                             const NDArray& input,\n                             const TBlob& err_cpu,\n                             const bool full_check) {\n  CheckFormatImpl<gpu>(rctx, input, err_cpu, full_check);\n}\n\ntemplate <>\nvoid SparseRetainOpForwardRspWrapper<gpu>(mshadow::Stream<gpu>* s,\n                                          const NDArray& input_nd,\n                                          const TBlob& idx_data,\n                                          const OpReqType req,\n                                          NDArray* output_nd) {\n  mxnet::op::SparseRetainOpForwardRspImpl<gpu>(s, input_nd, idx_data, req, output_nd);\n}\n\ntemplate <>\nvoid CastStorageDispatch<gpu>(const OpContext& ctx, const NDArray& input, const NDArray& output) {\n  mxnet::op::CastStorageComputeImpl<gpu>(ctx, input, output);\n}\n\n}  // namespace common\n}  // namespace mxnet\n"
  },
  {
    "path": "src/common/utils.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file utils.h\n * \\brief Basic utilility functions.\n */\n#ifndef MXNET_COMMON_UTILS_H_\n#define MXNET_COMMON_UTILS_H_\n\n#include <dmlc/logging.h>\n#include <dmlc/omp.h>\n#include <nnvm/graph.h>\n#include <nnvm/node.h>\n#include <mxnet/imperative.h>\n#include <mxnet/engine.h>\n#include <mxnet/ndarray.h>\n#include <mxnet/storage.h>\n#include <mxnet/op_attr_types.h>\n#include <mxnet/graph_attr_types.h>\n#include <nnvm/graph_attr_types.h>\n\n#include <memory>\n#include <vector>\n#include <type_traits>\n#include <utility>\n#include <random>\n#include <string>\n#include <thread>\n#include <algorithm>\n#include <functional>\n#include <limits>\n\n#include \"../operator/mxnet_op.h\"\n#if MXNET_USE_ONEDNN == 1\n#include \"../operator/nn/dnnl/dnnl_base-inl.h\"\n#endif\n\n#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)\n#include <windows.h>\n#else\n#include <unistd.h>\n#endif\n\nnamespace mxnet {\nnamespace common {\n\n#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)\ninline size_t current_process_id() {\n  return ::GetCurrentProcessId();\n}\n#else\ninline size_t current_process_id() {\n  return getpid();\n}\n#endif\n/*!\n * \\brief IndPtr should be non-negative, in non-decreasing order, start with 0\n *           and end with value equal with size of indices.\n */\nstruct csr_indptr_check {\n  template <typename DType, typename IType>\n  MSHADOW_XINLINE static void Map(int i,\n                                  DType* out,\n                                  const IType* indptr,\n                                  const nnvm::dim_t end,\n                                  const nnvm::dim_t idx_size) {\n    if (indptr[i + 1] < 0 || indptr[i + 1] < indptr[i] || (i == 0 && indptr[i] != 0) ||\n        (i == end - 1 && indptr[end] != idx_size))\n      *out = kCSRIndPtrErr;\n  }\n};\n\n/*!\n *  \\brief Indices should be non-negative, less than the number of columns\n *           and in ascending order per row.\n */\nstruct csr_idx_check {\n  template <typename DType, typename IType, typename RType>\n  MSHADOW_XINLINE static void Map(int i,\n                                  DType* out,\n                                  const IType* idx,\n                                  const RType* indptr,\n                                  const nnvm::dim_t ncols) {\n    for (RType j = indptr[i]; j < indptr[i + 1]; j++) {\n      if (idx[j] >= ncols || idx[j] < 0 || (j < indptr[i + 1] - 1 && idx[j] >= idx[j + 1])) {\n        *out = kCSRIdxErr;\n        break;\n      }\n    }\n  }\n};\n\n/*!\n *  \\brief Indices of RSPNDArray should be non-negative,\n *           less than the size of first dimension and in ascending order\n */\nstruct rsp_idx_check {\n  template <typename DType, typename IType>\n  MSHADOW_XINLINE static void Map(int i,\n                                  DType* out,\n                                  const IType* idx,\n                                  const nnvm::dim_t end,\n                                  const nnvm::dim_t nrows) {\n    if ((i < end && idx[i + 1] <= idx[i]) || idx[i] < 0 || idx[i] >= nrows)\n      *out = kRSPIdxErr;\n  }\n};\n\ntemplate <typename xpu>\nvoid CheckFormatWrapper(const RunContext& rctx,\n                        const NDArray& input,\n                        const TBlob& err_cpu,\n                        const bool full_check);\n\n/*!\n * \\brief Check the validity of CSRNDArray.\n * \\param rctx Execution context.\n * \\param input Input NDArray of CSRStorage.\n * \\param err_cpu Error number on cpu.\n * \\param full_check If true, rigorous check, O(N) operations,\n *          otherwise basic check, O(1) operations.\n */\ntemplate <typename xpu>\nvoid CheckFormatCSRImpl(const RunContext& rctx,\n                        const NDArray& input,\n                        const TBlob& err_cpu,\n                        const bool full_check) {\n  using namespace op::mxnet_op;\n  CHECK_EQ(input.storage_type(), kCSRStorage) << \"CheckFormatCSRImpl is for CSRNDArray\";\n  const mxnet::TShape shape         = input.shape();\n  const mxnet::TShape idx_shape     = input.aux_shape(csr::kIdx);\n  const mxnet::TShape indptr_shape  = input.aux_shape(csr::kIndPtr);\n  const mxnet::TShape storage_shape = input.storage_shape();\n  if ((shape.ndim() != 2) ||\n      (idx_shape.ndim() != 1 || indptr_shape.ndim() != 1 || storage_shape.ndim() != 1) ||\n      (indptr_shape[0] != shape[0] + 1) || (idx_shape[0] != storage_shape[0])) {\n    MSHADOW_TYPE_SWITCH(err_cpu.type_flag_, DType, {\n      DType* err = err_cpu.dptr<DType>();\n      *err       = kCSRShapeErr;\n    });\n    return;\n  }\n  if (full_check) {\n    MSHADOW_TYPE_SWITCH(err_cpu.type_flag_, DType, {\n      MSHADOW_IDX_TYPE_SWITCH(input.aux_type(csr::kIndPtr), RType, {\n        MSHADOW_IDX_TYPE_SWITCH(input.aux_type(csr::kIdx), IType, {\n          mshadow::Stream<xpu>* s = rctx.get_stream<xpu>();\n          NDArray ret_xpu = NDArray(mshadow::Shape1(1), rctx.get_ctx(), false, err_cpu.type_flag_);\n          TBlob val_xpu   = ret_xpu.data();\n          Kernel<set_to_int<kNormalErr>, xpu>::Launch(s, val_xpu.Size(), val_xpu.dptr<DType>());\n          Kernel<csr_indptr_check, xpu>::Launch(s,\n                                                indptr_shape[0] - 1,\n                                                val_xpu.dptr<DType>(),\n                                                input.aux_data(csr::kIndPtr).dptr<RType>(),\n                                                indptr_shape[0] - 1,\n                                                idx_shape[0]);\n          // no need to check indices if indices are empty\n          if (idx_shape[0] != 0) {\n            Kernel<csr_idx_check, xpu>::Launch(s,\n                                               indptr_shape[0] - 1,\n                                               val_xpu.dptr<DType>(),\n                                               input.aux_data(csr::kIdx).dptr<IType>(),\n                                               input.aux_data(csr::kIndPtr).dptr<RType>(),\n                                               shape[1]);\n          }\n          mshadow::Copy(err_cpu.get<cpu, 1, DType>(), val_xpu.get<xpu, 1, DType>(s), s);\n        });\n      });\n    });\n  }\n}\n\n/*!\n * \\brief Check the validity of RowSparseNDArray.\n * \\param rctx Execution context.\n * \\param input Input NDArray of RowSparseStorage.\n * \\param err_cpu Error number on cpu.\n * \\param full_check If true, rigorous check, O(N) operations,\n *          otherwise basic check, O(1) operations.\n */\ntemplate <typename xpu>\nvoid CheckFormatRSPImpl(const RunContext& rctx,\n                        const NDArray& input,\n                        const TBlob& err_cpu,\n                        const bool full_check) {\n  using namespace op::mxnet_op;\n  CHECK_EQ(input.storage_type(), kRowSparseStorage) << \"CheckFormatRSPImpl is for RSPNDArray\";\n  const mxnet::TShape idx_shape = input.aux_shape(rowsparse::kIdx);\n  if (idx_shape[0] != input.storage_shape()[0]) {\n    MSHADOW_TYPE_SWITCH(err_cpu.type_flag_, DType, {\n      DType* err = err_cpu.dptr<DType>();\n      *err       = kRSPShapeErr;\n    });\n    return;\n  }\n  if (idx_shape[0] == 0) {\n    return;\n  }\n  if (full_check) {\n    MSHADOW_TYPE_SWITCH(err_cpu.type_flag_, DType, {\n      MSHADOW_IDX_TYPE_SWITCH(input.aux_type(rowsparse::kIdx), IType, {\n        mshadow::Stream<xpu>* s = rctx.get_stream<xpu>();\n        NDArray ret_xpu = NDArray(mshadow::Shape1(1), rctx.get_ctx(), false, err_cpu.type_flag_);\n        TBlob val_xpu   = ret_xpu.data();\n        Kernel<set_to_int<kNormalErr>, xpu>::Launch(s, val_xpu.Size(), val_xpu.dptr<DType>());\n\n        Kernel<rsp_idx_check, xpu>::Launch(s,\n                                           idx_shape[0],\n                                           val_xpu.dptr<DType>(),\n                                           input.aux_data(rowsparse::kIdx).dptr<IType>(),\n                                           idx_shape[0] - 1,\n                                           input.shape()[0]);\n        mshadow::Copy(err_cpu.get<cpu, 1, DType>(), val_xpu.get<xpu, 1, DType>(s), s);\n      });\n    });\n  }\n}\n\ntemplate <typename xpu>\nvoid CheckFormatImpl(const RunContext& rctx,\n                     const NDArray& input,\n                     const TBlob& err_cpu,\n                     const bool full_check) {\n  int stype = input.storage_type();\n  if (stype == kCSRStorage) {\n    CheckFormatCSRImpl<xpu>(rctx, input, err_cpu, full_check);\n  } else if (stype == kRowSparseStorage) {\n    CheckFormatRSPImpl<xpu>(rctx, input, err_cpu, full_check);\n  } else if (stype == kDefaultStorage) {\n    // no-op for default storage\n  } else {\n    LOG(FATAL) << \"Unknown storage type \" << stype;\n  }\n}\n\n/*! \\brief Pick rows specified by user input index array from a row sparse ndarray\n *         and save them in the output sparse ndarray.\n */\ntemplate <typename xpu>\nvoid SparseRetainOpForwardRspWrapper(mshadow::Stream<xpu>* s,\n                                     const NDArray& input_nd,\n                                     const TBlob& idx_data,\n                                     const OpReqType req,\n                                     NDArray* output_nd);\n\n/* \\brief Casts tensor storage type to the new type.\n */\ntemplate <typename xpu>\nvoid CastStorageDispatch(const OpContext& ctx, const NDArray& input, const NDArray& output);\n\n/*! \\brief returns true if all storage types in `vstorage` are the same as target `stype`.\n *         false is returned for empty inputs.\n */\ninline bool ContainsOnlyStorage(const StorageTypeVector& vstorage, const NDArrayStorageType stype) {\n  if (!vstorage.empty()) {\n    for (const auto& i : vstorage) {\n      if (i != stype)\n        return false;\n    }\n    return true;\n  }\n  return false;\n}\n\n/*! \\brief returns true if all storage types in `vstorage` are the same as target `stype1`\n *         or `stype2'. Sets boolean if both found.\n *         false is returned for empty inputs.\n */\ninline bool ContainsOnlyStorage(const StorageTypeVector& vstorage,\n                                const NDArrayStorageType stype1,\n                                const NDArrayStorageType stype2,\n                                bool* has_both) {\n  if (has_both) {\n    *has_both = false;\n  }\n  if (!vstorage.empty()) {\n    uint8_t has = 0;\n    for (const auto i : vstorage) {\n      if (i == stype1) {\n        has |= 1;\n      } else if (i == stype2) {\n        has |= 2;\n      } else {\n        return false;\n      }\n    }\n    if (has_both) {\n      *has_both = has == 3;\n    }\n    return true;\n  }\n  return false;\n}\n\n/*! \\brief returns true if the storage types of arrays in `ndarrays`\n *         are the same as target `stype`. false is returned for empty inputs.\n */\ninline bool ContainsOnlyStorage(const std::vector<NDArray>& ndarrays,\n                                const NDArrayStorageType stype) {\n  if (!ndarrays.empty()) {\n    for (const auto& nd : ndarrays) {\n      if (nd.storage_type() != stype) {\n        return false;\n      }\n    }\n    return true;\n  }\n  return false;\n}\n\n/*! \\brief returns true if the storage types of arrays in `ndarrays`\n *         are the same as targets `stype1` or `stype2`. false is returned for empty inputs.\n */\ninline bool ContainsOnlyStorage(const std::vector<NDArray>& ndarrays,\n                                const NDArrayStorageType stype1,\n                                const NDArrayStorageType stype2,\n                                bool* has_both) {\n  if (has_both) {\n    *has_both = false;\n  }\n  if (!ndarrays.empty()) {\n    uint8_t has = 0;\n    for (const auto& nd : ndarrays) {\n      const NDArrayStorageType stype = nd.storage_type();\n      if (stype == stype1) {\n        has |= 1;\n      } else if (stype == stype2) {\n        has |= 2;\n      } else {\n        return false;\n      }\n    }\n    if (has_both) {\n      *has_both = has == 3;\n    }\n    return true;\n  }\n  return false;\n}\n\n/*! \\brief returns true if storage type of any array in `ndarrays`\n *         is the same as the target `stype`. false is returned for empty inputs.\n */\ninline bool ContainsStorageType(const std::vector<NDArray>& ndarrays,\n                                const NDArrayStorageType stype) {\n  if (!ndarrays.empty()) {\n    for (const auto& nd : ndarrays) {\n      if (nd.storage_type() == stype) {\n        return true;\n      }\n    }\n  }\n  return false;\n}\n\n/*! \\brief returns true if any storage type `ndstype` in `ndstypes`\n *         is the same as the target `stype`. false is returned for empty inputs.\n */\ninline bool ContainsStorageType(const std::vector<int>& ndstypes, const NDArrayStorageType stype) {\n  if (!ndstypes.empty()) {\n    for (const auto& ndstype : ndstypes) {\n      if (ndstype == stype) {\n        return true;\n      }\n    }\n  }\n  return false;\n}\n\n/*! \\brief get string representation of dispatch_mode */\ninline std::string dispatch_mode_string(const DispatchMode x) {\n  switch (x) {\n    case DispatchMode::kFCompute:\n      return \"fcompute\";\n    case DispatchMode::kFComputeEx:\n      return \"fcompute_ex\";\n    case DispatchMode::kFComputeFallback:\n      return \"fcompute_fallback\";\n    case DispatchMode::kVariable:\n      return \"variable\";\n    case DispatchMode::kUndefined:\n      return \"undefined\";\n  }\n  return \"unknown\";\n}\n\n/*! \\brief get string representation of storage_type */\ninline std::string stype_string(const int x) {\n  switch (x) {\n    case kDefaultStorage:\n      return \"default\";\n    case kCSRStorage:\n      return \"csr\";\n    case kRowSparseStorage:\n      return \"row_sparse\";\n  }\n  return \"unknown\";\n}\n\n/*! \\brief get string representation of device type */\ninline std::string dev_type_string(const int dev_type) {\n  switch (dev_type) {\n    case Context::kCPU:\n      return \"cpu\";\n    case Context::kGPU:\n      return \"gpu\";\n    case Context::kCPUPinned:\n      return \"cpu_pinned\";\n    case Context::kCPUShared:\n      return \"cpu_shared\";\n  }\n  return \"unknown\";\n}\n\ninline std::string attr_value_string(const nnvm::NodeAttrs& attrs,\n                                     const std::string& attr_name,\n                                     std::string default_val = \"\") {\n  if (attrs.dict.find(attr_name) == attrs.dict.end()) {\n    return default_val;\n  }\n  return attrs.dict.at(attr_name);\n}\n\n/*! \\brief Seeks an attribute in a node and its subgraphs and invokes a function on each. */\ntemplate <typename Fn>\ninline void attr_foreach(const nnvm::NodeAttrs& attrs, const std::string& attr_name, const Fn& fn) {\n  const auto& found_it = attrs.dict.find(attr_name);\n  if (found_it != attrs.dict.end()) {\n    fn(found_it->second);\n  }\n  for (const auto& subgraph : attrs.subgraphs) {\n    DFSVisit(subgraph->outputs,\n             [&](const nnvm::ObjectPtr& node) { attr_foreach(node->attrs, attr_name, fn); });\n  }\n}\n\ntemplate <typename ValueType>\ninline ValueType flag_attr_accumulate(const nnvm::NodeAttrs& attrs, const std::string& attr_name) {\n  static_assert(std::is_integral<ValueType>::value, \"ValueType must be an integral type.\");\n\n  ValueType result = 0;\n  attr_foreach(attrs, attr_name, [&](const std::string& attr_value) {\n    std::istringstream ss(attr_value);\n    ValueType temp;\n    ss >> temp;\n    result |= temp;\n\n    if (ss.fail() || !ss.eof()) {\n      LOG(WARNING) << \"Incorrect value of an attribute: \" << attr_name\n                   << \". Expected an integer, while got: \" << attr_value;\n    }\n  });\n  return result;\n}\n\n/*! \\brief get string representation of the operator stypes */\ninline std::string operator_stype_string(const nnvm::NodeAttrs& attrs,\n                                         const int dev_mask,\n                                         const std::vector<int>& in_attrs,\n                                         const std::vector<int>& out_attrs) {\n  std::ostringstream os;\n  os << \"operator = \" << attrs.op->name << \"\\ninput storage types = [\";\n  for (const int attr : in_attrs) {\n    os << stype_string(attr) << \", \";\n  }\n  os << \"]\\n\"\n     << \"output storage types = [\";\n  for (const int attr : out_attrs) {\n    os << stype_string(attr) << \", \";\n  }\n  os << \"]\\n\"\n     << \"params = {\";\n  for (auto kv : attrs.dict) {\n    os << \"\\\"\" << kv.first << \"\\\" : \" << kv.second << \", \";\n  }\n  os << \"}\\n\"\n     << \"context.dev_mask = \" << dev_type_string(dev_mask);\n  return os.str();\n}\n\n/*! \\brief get string representation of the operator */\ninline std::string operator_string(const nnvm::NodeAttrs& attrs,\n                                   const OpContext& ctx,\n                                   const std::vector<NDArray>& inputs,\n                                   const std::vector<OpReqType>& req,\n                                   const std::vector<NDArray>& outputs) {\n  std::string result = \"\";\n  std::vector<int> in_stypes;\n  std::vector<int> out_stypes;\n  in_stypes.reserve(inputs.size());\n  out_stypes.reserve(outputs.size());\n  auto xform = [](const NDArray arr) -> int { return arr.storage_type(); };\n  std::transform(inputs.begin(), inputs.end(), std::back_inserter(in_stypes), xform);\n  std::transform(outputs.begin(), outputs.end(), std::back_inserter(out_stypes), xform);\n  result += operator_stype_string(attrs, ctx.run_ctx.ctx.dev_mask(), in_stypes, out_stypes);\n  return result;\n}\n\n/*! \\brief log message once. Intended for storage fallback warning messages. */\ninline void LogOnce(const std::string& message) {\n  typedef dmlc::ThreadLocalStore<std::unordered_set<std::string>> LogStore;\n  auto log_store = LogStore::Get();\n  if (log_store->find(message) == log_store->end()) {\n    LOG(INFO) << message;\n    log_store->insert(message);\n  }\n}\n\n/*! \\brief log storage fallback event\n */\ninline void LogStorageFallback(const nnvm::NodeAttrs& attrs,\n                               const int dev_mask,\n                               const std::vector<int>* in_attrs,\n                               const std::vector<int>* out_attrs) {\n  static bool log = dmlc::GetEnv(\"MXNET_STORAGE_FALLBACK_LOG_VERBOSE\", true);\n  if (!log)\n    return;\n  const std::string op_str = operator_stype_string(attrs, dev_mask, *in_attrs, *out_attrs);\n  std::ostringstream os;\n  const char* warning =\n      \"\\n WARNING:\\n\"\n      \"Execution of the operator above will fallback to the generic implementation \"\n#if MXNET_USE_ONEDNN == 1\n      \"(not utilizing kernels from oneDNN library) \"\n#endif\n      \"with default dense storage type. You are seeing this warning message because \"\n#if MXNET_USE_ONEDNN == 1\n      \"MXNET_ONEDNN_ENABLED flag is set to 0, in which case you can re-enable the default \"\n      \"execution path by setting MXNET_ONEDNN_ENABLED back to 1, or \"\n#endif\n      \"the operator above is unable to process the given ndarrays with specified storage types, \"\n      \"context and/or parameter, in which case temporary dense ndarrays are generated in order to \"\n      \"execute the operator. The fallback does not affect the correctness of the programme. Using \"\n      \"default storage type performance degradation might be observed. \\nYou can set environment \"\n      \"variable MXNET_STORAGE_FALLBACK_LOG_VERBOSE to 0 to suppress this warning.\";\n  os << \"\\nStorage type fallback detected:\\n\" << op_str << warning;\n  LogOnce(os.str());\n#if MXNET_USE_ONEDNN == 1\n  if (GetDNNLCacheSize() != -1)\n    common::LogOnce(\n        \"MXNET_ONEDNN_CACHE_NUM is set.\"\n        \"Should only be set if \"\n        \"your model has variable input shapes, \"\n        \"as cache size may grow unbounded\");\n#endif\n}\n\n// heuristic to dermine number of threads per GPU\ninline int GetNumThreadsPerGPU() {\n  // This is resource efficient option.\n  return dmlc::GetEnv(\"MXNET_GPU_WORKER_NTHREADS\", 2);\n}\n\n// heuristic to get number of matching colors.\n// this decides how much parallelism we can get in each GPU.\ninline int GetExecNumMatchColor() {\n  // This is resource efficient option.\n  int num_match_color = dmlc::GetEnv(\"MXNET_EXEC_NUM_TEMP\", 1);\n  return std::min(num_match_color, GetNumThreadsPerGPU());\n}\n\ntemplate <typename T, typename V>\nV ParallelAccumulate(const T* a, const int n, V start) {\n  V sum = start;\n#pragma omp parallel for reduction(+ : sum)\n  for (int i = 0; i < n; ++i) {\n    sum += a[i];\n  }\n  return sum;\n}\n\n/*!\n * \\brief\n * Helper function for ParallelSort.\n * DO NOT call this function directly.\n * Use the interface ParallelSort instead.\n * Ref: https://github.com/dmlc/difacto/blob/master/src/common/parallel_sort.h\n */\ntemplate <typename RandomIt, typename Compare>\nvoid ParallelSortHelper(RandomIt first, size_t len, size_t grainsize, const Compare& comp) {\n  if (len < grainsize) {\n    std::sort(first, first + len, comp);\n  } else {\n    std::thread thr(ParallelSortHelper<RandomIt, Compare>, first, len / 2, grainsize, comp);\n    ParallelSortHelper(first + len / 2, len - len / 2, grainsize, comp);\n    thr.join();\n    std::inplace_merge(first, first + len / 2, first + len, comp);\n  }\n}\n\n/*!\n * \\brief\n * Sort the elements in the range [first, last) into the ascending order defined by\n * the comparator comp.\n * If the length of the range [first, last) is greater than a certain threshold,\n * the range will be recursively divided into two and assign two threads\n * to sort each half range.\n * Ref: https://github.com/dmlc/difacto/blob/master/src/common/parallel_sort.h\n */\ntemplate <typename RandomIt, typename Compare>\nvoid ParallelSort(RandomIt first, RandomIt last, size_t num_threads, Compare comp) {\n  const auto num   = std::distance(first, last);\n  size_t grainsize = std::max(num / num_threads + 5, static_cast<size_t>(1024 * 16));\n  ParallelSortHelper(first, num, grainsize, comp);\n}\n\n/*!\n * \\brief\n * Sort the elements in the range [first, last) into ascending order.\n * The elements are compared using the default < operator.\n * If the length of the range [first, last) is greater than a certain threshold,\n * the range will be recursively divided into two and assign two threads\n * to sort each half range.\n * Ref: https://github.com/dmlc/difacto/blob/master/src/common/parallel_sort.h\n */\ntemplate <typename RandomIt>\nvoid ParallelSort(RandomIt first, RandomIt last, size_t num_threads) {\n  ParallelSort(\n      first, last, num_threads, std::less<typename std::iterator_traits<RandomIt>::value_type>());\n}\n\n/*!\n * \\brief Random Engine\n */\ntypedef std::mt19937 RANDOM_ENGINE;\n\n/*!\n * \\brief Helper functions.\n */\nnamespace helper {\n\n/*!\n * \\brief Helper for non-array type `T`.\n */\ntemplate <class T>\nstruct UniqueIf {\n  /*!\n   * \\brief Type of `T`.\n   */\n  using SingleObject = std::unique_ptr<T>;\n};\n\n/*!\n * \\brief Helper for an array of unknown bound `T`.\n */\ntemplate <class T>\nstruct UniqueIf<T[]> {\n  /*!\n   * \\brief Type of `T`.\n   */\n  using UnknownBound = std::unique_ptr<T[]>;\n};\n\n/*!\n * \\brief Helper for an array of known bound `T`.\n */\ntemplate <class T, size_t kSize>\nstruct UniqueIf<T[kSize]> {\n  /*!\n   * \\brief Type of `T`.\n   */\n  using KnownBound = void;\n};\n\n}  // namespace helper\n\n/*!\n * \\brief Constructs an object of type `T` and wraps it in a\n *        `std``::``unique_ptr`.\n * \\param args List of arguments with which an instance of `T` will be\n *             constructed.\n * \\return `std``::``unique_ptr` of an instance of type `T`.\n *\n * Constructs a non-array type `T`. The arguments `args` are passed to the\n * constructor of `T`. The function does not participate in the overload\n * resolution if `T` is an array type.\n */\ntemplate <class T, class... Args>\ntypename helper::UniqueIf<T>::SingleObject MakeUnique(Args&&... args) {\n  return std::unique_ptr<T>(new T(std::forward<Args>(args)...));\n}\n\n/*!\n * \\brief Constructs an object of type `T` and wraps it in a\n *        `std``::``unique_ptr`.\n * \\param n The size of the array to construct.\n * \\return `std``::``unique_ptr` of an instance of type `T`.\n *\n * Constructs an array of unknown bound `T`. The function does not participate\n * in the overload resolution unless `T` is an array of unknown bound.\n */\ntemplate <class T>\ntypename helper::UniqueIf<T>::UnknownBound MakeUnique(size_t n) {\n  using U = typename std::remove_extent<T>::type;\n  return std::unique_ptr<T>(new U[n]{});\n}\n\n/*!\n * \\brief Constructs an object of type `T` and wraps it in a\n *        `std``::``unique_ptr`.\n * \\param args List of arguments with which an instance of `T` will be\n *             constructed.\n *\n * Constructs an arrays of known bound is disallowed.\n */\ntemplate <class T, class... Args>\ntypename helper::UniqueIf<T>::KnownBound MakeUnique(Args&&... args) = delete;\n\ntemplate <typename FCompType>\nFCompType GetFCompute(const nnvm::Op* op, const std::string& name, const Context& ctx) {\n  static auto& fcompute_cpu = nnvm::Op::GetAttr<FCompType>(name + \"<cpu>\");\n  static auto& fcompute_gpu = nnvm::Op::GetAttr<FCompType>(name + \"<gpu>\");\n\n  if (ctx.dev_mask() == cpu::kDevMask) {\n    return fcompute_cpu.get(op, nullptr);\n  } else if (ctx.dev_mask() == gpu::kDevMask) {\n    return fcompute_gpu.get(op, nullptr);\n  } else {\n    LOG(FATAL) << \"Unknown device mask \" << ctx.dev_mask();\n    return nullptr;\n  }\n}\n\n/*!\n * \\brief Return the max integer value representable in the type `T` without loss of precision.\n */\ntemplate <typename T>\nconstexpr size_t MaxIntegerValue() {\n  return std::is_integral<T>::value ? std::numeric_limits<T>::max() :\n                                      size_t(2) << (std::numeric_limits<T>::digits - 1);\n}\n\ntemplate <>\nconstexpr size_t MaxIntegerValue<mshadow::half::half_t>() {\n  return size_t(2) << 10;\n}\n\ntemplate <>\nconstexpr size_t MaxIntegerValue<mshadow::bfloat::bf16_t>() {\n  return size_t(2) << 14;\n}\n\nMSHADOW_XINLINE int ilog2ul(size_t a) {\n  int k = 1;\n  while (a >>= 1)\n    ++k;\n  return k;\n}\n\nMSHADOW_XINLINE int ilog2ui(unsigned int a) {\n  int k = 1;\n  while (a >>= 1)\n    ++k;\n  return k;\n}\n\n/*!\n * \\brief Return an NDArray of all zeros.\n */\ninline NDArray InitZeros(const NDArrayStorageType stype,\n                         const mxnet::TShape& shape,\n                         const Context& ctx,\n                         const int dtype) {\n  // NDArray with default storage\n  if (stype == kDefaultStorage) {\n    NDArray ret(shape, ctx, false, dtype);\n    ret = 0;\n    return ret;\n  }\n  // NDArray with non-default storage. Storage allocation is always delayed.\n  return NDArray(stype, shape, ctx, true, dtype);\n}\n\n/*!\n * \\brief Helper to add a NDArray of zeros to a std::vector.\n */\ninline void EmplaceBackZeros(const NDArrayStorageType stype,\n                             const mxnet::TShape& shape,\n                             const Context& ctx,\n                             const int dtype,\n                             std::vector<NDArray>* vec) {\n  // NDArray with default storage\n  if (stype == kDefaultStorage) {\n    vec->emplace_back(shape, ctx, false, dtype);\n    vec->back() = 0;\n  } else {\n    // NDArray with non-default storage. Storage allocation is always delayed.\n    vec->emplace_back(stype, shape, ctx, true, dtype);\n  }\n}\n\n/*!\n * \\brief parallelize copy by OpenMP.\n */\ntemplate <typename DType>\ninline void ParallelCopy(DType* dst, const DType* src, index_t size) {\n  static index_t copy_block_size = dmlc::GetEnv(\"MXNET_CPU_PARALLEL_SIZE\", 200000);\n  if (size >= copy_block_size) {\n#pragma omp parallel for num_threads(engine::OpenMP::Get()->GetRecommendedOMPThreadCount())\n    for (index_t i = 0; i < size; ++i) {\n      dst[i] = src[i];\n    }\n  } else {\n#pragma GCC diagnostic push\n#if __GNUC__ >= 8\n#pragma GCC diagnostic ignored \"-Wclass-memaccess\"\n#endif\n    std::memcpy(dst, src, sizeof(DType) * size);\n#pragma GCC diagnostic pop\n  }\n}\n\n/*!\n * \\breif parallelize add by OpenMP\n */\ntemplate <typename DType>\ninline void ParallelAdd(DType* dst, const DType* src, index_t size) {\n  static index_t add_block_size = dmlc::GetEnv(\"MXNET_CPU_PARALLEL_SIZE\", 200000);\n  if (size >= add_block_size) {\n#pragma omp parallel for num_threads(engine::OpenMP::Get()->GetRecommendedOMPThreadCount())\n    for (index_t i = 0; i < size; ++i) {\n      dst[i] += src[i];\n    }\n  } else {\n    for (index_t i = 0; i < size; ++i) {\n      dst[i] += src[i];\n    }\n  }\n}\n\n/*!\n * \\brief If numpy compatibility is turned off (default), the shapes passed in\n * by users follow the legacy shape definition:\n * 1. 0 ndim means the shape is completely unknown.\n * 2. 0 dim size means the dim size is unknown.\n * We need to convert those shapes to use the numpy shape definition:\n * 1. 0 ndim means it's a scalar tensor.\n * 2. -1 ndim means the shape is unknown.\n * 3. 0 dim size means no elements in that dimension.\n * 4. -1 dim size means the dimension's size is unknown.\n * so that operator's infer shape function can work in backend.\n * \\param shape to be converted.\n * Note: It is possible that the shape to be converted is already\n * numpy compatible. For example, when a subgraph operator's infer\n * shape function is called from the infer shape pass of the whole\n * graph, its input/output shapes have been converted to numpy\n * compatible shapes.\n */\ninline void ConvertToNumpyShape(mxnet::TShape* shape) {\n  if (shape->ndim() == 0) {    // legacy shape ndim = 0 means unknown\n    *shape = mxnet::TShape();  // unknown shape ndim = -1\n  } else {\n    for (int j = 0; j < shape->ndim(); ++j) {\n      if ((*shape)[j] == 0) {  // legacy shape dim_size = 0 means unknown\n        (*shape)[j] = -1;      // unknown dim size = -1\n      }\n    }\n  }\n}\n\ninline void ConvertToNumpyShape(mxnet::ShapeVector* shapes) {\n  for (size_t i = 0; i < shapes->size(); ++i) {\n    ConvertToNumpyShape(&(shapes->at(i)));\n  }\n}\n\n/*!\n * \\brief This is function is used to convert shapes returned by\n * the infer shape functions/pass to the legacy shape definition.\n */\ninline void ConvertToLegacyShape(mxnet::TShape* shape) {\n  if (!mxnet::ndim_is_known(*shape)) {\n    *shape = mxnet::TShape(0, -1);\n  } else {\n    for (int j = 0; j < shape->ndim(); ++j) {\n      if (!mxnet::dim_size_is_known(*shape, j)) {\n        (*shape)[j] = 0;\n      }\n    }\n  }\n}\n\ninline void ConvertToLegacyShape(mxnet::ShapeVector* shapes) {\n  for (size_t i = 0; i < shapes->size(); ++i) {\n    ConvertToLegacyShape(&(shapes->at(i)));\n  }\n}\nvoid ExecuteMonInputCallback(\n    const nnvm::IndexedGraph& idx,\n    const std::vector<NDArray*>& state_arrays,\n    size_t nid,\n    const std::function<void(const char*, const char*, void*)>& monitor_callback);\n\nvoid ExecuteMonOutputCallback(\n    const nnvm::IndexedGraph& idx,\n    const std::vector<NDArray*>& state_arrays,\n    size_t nid,\n    const std::function<void(const char*, const char*, void*)>& monitor_callback);\n\ninline mxnet::TShape CanonicalizeAxes(const mxnet::TShape& src) {\n  // convert negative axes to positive values\n  const int ndim     = src.ndim();\n  mxnet::TShape axes = src;\n  for (int i = 0; i < ndim; ++i) {\n    if (axes[i] < 0) {\n      axes[i] += ndim;\n    }\n    CHECK(axes[i] >= 0 && axes[i] < ndim)\n        << \"axes[\" << i << \"]=\" << axes[i] << \" exceeds the range [\" << 0 << \", \" << ndim << \")\";\n  }\n  return axes;\n}\n\ninline bool is_float(const int dtype) {\n  return dtype == mshadow::kFloat32 || dtype == mshadow::kFloat64 || dtype == mshadow::kFloat16 ||\n         dtype == mshadow::kBfloat16;\n}\n\ninline bool is_int(const int dtype) {\n  return dtype == mshadow::kUint8 || dtype == mshadow::kInt8 || dtype == mshadow::kUint16 ||\n         dtype == mshadow::kInt16 || dtype == mshadow::kUint32 || dtype == mshadow::kInt32 ||\n         dtype == mshadow::kUint64 || dtype == mshadow::kInt64;\n}\n\ninline bool is_signed_int(const int dtype) {\n  return dtype == mshadow::kInt8 || dtype == mshadow::kInt16 || dtype == mshadow::kInt32 ||\n         dtype == mshadow::kInt64;\n}\n\ninline bool is_unsigned_int(const int dtype) {\n  return dtype == mshadow::kUint8 || dtype == mshadow::kUint16 || dtype == mshadow::kUint32 ||\n         dtype == mshadow::kUint64;\n}\n\nstatic int bits_of(const int type_flag) {\n  switch (type_flag) {\n    case mshadow::kFloat32:\n      return sizeof(float) * CHAR_BIT;\n    case mshadow::kFloat64:\n      return sizeof(double) * CHAR_BIT;\n    case mshadow::kUint8:\n      return sizeof(uint8_t) * CHAR_BIT;\n    case mshadow::kInt32:\n      return sizeof(int32_t) * CHAR_BIT;\n    case mshadow::kInt8:\n      return sizeof(int8_t) * CHAR_BIT;\n    case mshadow::kInt64:\n      return sizeof(int64_t) * CHAR_BIT;\n    case mshadow::kBool:\n      return sizeof(bool) * CHAR_BIT;\n    case mshadow::kInt16:\n      return sizeof(int16_t) * CHAR_BIT;\n    case mshadow::kUint16:\n      return sizeof(uint16_t) * CHAR_BIT;\n    case mshadow::kUint32:\n      return sizeof(uint32_t) * CHAR_BIT;\n    case mshadow::kUint64:\n      return sizeof(uint64_t) * CHAR_BIT;\n    default: {\n      LOG(FATAL) << \"Unknown type_flag=\" << type_flag;\n      return -1;\n    }\n  }\n}\n\ninline int type_promotion(const int type1, const int type2) {\n  if (type1 == type2)\n    return type1;\n  if (is_float(type1) && is_float(type2)) {\n    if (type1 == mshadow::kFloat64 || type2 == mshadow::kFloat64) {\n      return mshadow::kFloat64;\n    }\n    if (type1 == mshadow::kFloat32 || type2 == mshadow::kFloat32) {\n      return mshadow::kFloat32;\n    }\n    return mshadow::kFloat16;\n  } else if (is_float(type1) || is_float(type2)) {\n    return is_float(type1) ? type1 : type2;\n  }\n  if (is_signed_int(type1) && is_signed_int(type2)) {\n    if (type1 == mshadow::kInt64 || type2 == mshadow::kInt64) {\n      return mshadow::kInt64;\n    }\n    if (type1 == mshadow::kInt32 || type2 == mshadow::kInt32) {\n      return mshadow::kInt32;\n    }\n    if (type1 == mshadow::kInt16 || type2 == mshadow::kInt16) {\n      return mshadow::kInt16;\n    }\n    return mshadow::kInt8;\n  } else if (is_unsigned_int(type1) && is_unsigned_int(type2)) {\n    if (type1 == mshadow::kUint64 || type2 == mshadow::kUint64) {\n      return mshadow::kUint64;\n    }\n    if (type1 == mshadow::kUint32 || type2 == mshadow::kUint32) {\n      return mshadow::kUint32;\n    }\n    if (type1 == mshadow::kUint16 || type2 == mshadow::kUint16) {\n      return mshadow::kUint16;\n    }\n    return mshadow::kUint8;\n  } else if (type1 == mshadow::kBool) {\n    return type2;\n  } else if (type2 == mshadow::kBool) {\n    return type1;\n  } else if (is_unsigned_int(type1) || is_unsigned_int(type2)) {\n    if (bits_of(type1) < bits_of(type2)) {\n      if (type1 == mshadow::kInt8 && type2 == mshadow::kUint16) {\n        return mshadow::kInt32;\n      } else if (type1 == mshadow::kInt8 && type2 == mshadow::kUint32) {\n        return mshadow::kInt64;\n      } else if (type1 == mshadow::kInt16 && type2 == mshadow::kUint32) {\n        return mshadow::kInt64;\n      } else if (type2 == mshadow::kUint64) {\n        LOG(FATAL) << \"Unsupported type promotions between \" << mshadow::dtype_string(type1)\n                   << \" and \" << mshadow::dtype_string(type2);\n      } else {\n        return type2;\n      }\n    } else if (bits_of(type2) < bits_of(type1)) {\n      if (type2 == mshadow::kInt8 && type1 == mshadow::kUint16) {\n        return mshadow::kInt32;\n      } else if (type2 == mshadow::kInt8 && type1 == mshadow::kUint32) {\n        return mshadow::kInt64;\n      } else if (type2 == mshadow::kInt16 && type1 == mshadow::kUint32) {\n        return mshadow::kInt64;\n      } else if (type1 == mshadow::kUint64) {\n        LOG(FATAL) << \"Unsupported type promotions between \" << mshadow::dtype_string(type1)\n                   << \" and \" << mshadow::dtype_string(type2);\n      } else {\n        return type1;\n      }\n    } else {\n      if (type1 == mshadow::kUint8 || type2 == mshadow::kUint8) {\n        return mshadow::kInt16;\n      }\n      if (type1 == mshadow::kUint16 || type2 == mshadow::kUint16) {\n        return mshadow::kInt32;\n      }\n      if (type1 == mshadow::kUint32 || type2 == mshadow::kUint32) {\n        return mshadow::kInt64;\n      }\n    }\n  }\n  LOG(FATAL) << \"Unsupported type promotions between \" << mshadow::dtype_string(type1) << \" and \"\n             << mshadow::dtype_string(type2);\n  return -1;\n}\n\ninline const std::string NodeAttrsGetProfilerScope(const nnvm::NodeAttrs& attrs) {\n  // obtain the profiler scope name, if assigned previously\n  std::string profiler_scope = MXNET_STORAGE_DEFAULT_PROFILER_SCOPE_CSTR;\n  const std::unordered_map<std::string, std::string>& node_attrs_dict = attrs.dict;\n  const std::unordered_map<std::string, std::string>::const_iterator profiler_scope_iter =\n      node_attrs_dict.find(\"__profiler_scope__\");\n  if (profiler_scope_iter != node_attrs_dict.end()) {\n    profiler_scope = profiler_scope_iter->second;\n  }\n  return profiler_scope;\n}\n\ninline int GetDefaultDtype() {\n  return Imperative::Get()->is_np_default_dtype() ? mshadow::kFloat64 : mshadow::kFloat32;\n}\n\ninline int GetDefaultDtype(int dtype) {\n  if (dtype != -1)\n    return dtype;\n  return Imperative::Get()->is_np_default_dtype() ? mshadow::kFloat64 : mshadow::kFloat32;\n}\n\nstruct MShadowTypeInfo {\n  std::string name;\n  int size;\n  int acc_size;\n\n  MShadowTypeInfo(const std::string name, const int size, const int acc_size)\n      : name(std::move(name)), size(size), acc_size(acc_size) {}\n\n  MShadowTypeInfo(const std::string name, const int size) : MShadowTypeInfo(name, size, size) {}\n};\n\nMShadowTypeInfo mshadow_type_info(const int type_flag);\n\ninline bool AlignedMemAlloc(void** ptr, size_t size, size_t alignment) {\n#if _MSC_VER\n  *ptr = _aligned_malloc(size, alignment);\n  if (*ptr == nullptr)\n    return false;\n#else\n  int res = posix_memalign(ptr, alignment, size);\n  if (res != 0)\n    return false;\n#endif\n  return true;\n}\n\ninline void AlignedMemFree(void* ptr) {\n#if _MSC_VER\n  _aligned_free(ptr);\n#else\n  free(ptr);\n#endif\n}\n\ninline index_t div_round(const index_t a, const index_t b) {\n  return (a + b - 1) / b;\n}\n\ninline bool IsPower2(size_t N) {\n  return ((N & (N - 1)) == 0) && N != 0;\n}\n\ninline size_t RoundToPower2(size_t N) {\n  size_t ret   = 1;\n  size_t copyN = N;\n  while (N >= 2) {\n    ret *= 2;\n    N /= 2;\n  }\n  if (ret < copyN) {\n    ret *= 2;\n  }\n  return ret;\n}\n\n}  // namespace common\n}  // namespace mxnet\n#endif  // MXNET_COMMON_UTILS_H_\n"
  },
  {
    "path": "src/engine/engine.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file engine.cc\n * \\brief Implementation of engine.\n */\n#include <mxnet/engine.h>\n#include <memory>\n#include <cstdlib>\n#include \"./engine_impl.h\"\n#include \"../common/cuda/utils.h\"\n\nnamespace mxnet {\nnamespace engine {\ninline Engine* CreateEngine() {\n  const char* type          = getenv(\"MXNET_ENGINE_TYPE\");\n  const bool default_engine = (type == nullptr);\n  if (type == nullptr)\n    type = \"ThreadedEnginePerDevice\";\n  std::string stype = type;\n\n  // The async tag is used later to determine if we use the GPU dependecy engine\n  std::string async_engine_tag = \"Async\";\n  auto tag_pos                 = stype.find(async_engine_tag);\n  if (tag_pos != std::string::npos && tag_pos + async_engine_tag.length() == stype.length()) {\n    stype = stype.substr(0, tag_pos);\n  }\n\n  Engine* ret = nullptr;\n#if MXNET_PREDICT_ONLY == 0\n  if (stype == \"NaiveEngine\") {\n    ret = CreateNaiveEngine();\n  } else if (stype == \"ThreadedEngine\") {\n    ret = CreateThreadedEnginePooled();\n  } else if (stype == \"ThreadedEnginePerDevice\") {\n    ret = CreateThreadedEnginePerDevice();\n  }\n#else\n  ret = CreateNaiveEngine();\n#endif\n\n  if (ret == nullptr) {\n    LOG(FATAL) << \"Cannot find Engine \" << type;\n  }\n  if (!default_engine) {\n    LOG(INFO) << \"MXNet start using engine: \" << type;\n  }\n  return ret;\n}\n\n#if MXNET_USE_CUDA\nCUDAEvent::CUDAEvent(Context const& ctx)\n    : event_(std::make_shared<cudaEvent_t>()), dev_id_(ctx.dev_id) {\n  cudaEvent_t ev;\n  common::cuda::DeviceStore device_store(dev_id_);\n  CUDA_CALL(cudaEventCreateWithFlags(&ev, cudaEventDisableTiming));\n  *event_ = ev;\n}\n\nCUDAEvent::~CUDAEvent() {\n  if (event_ && *event_ != nullptr) {\n    common::cuda::DeviceStore device_store(dev_id_);\n    CUDA_CALL(cudaEventSynchronize(*event_));\n    CUDA_CALL(cudaEventDestroy(*event_));\n  }\n}\n#endif\n}  // namespace engine\n\nconst std::shared_ptr<Engine>& Engine::_GetSharedRef() {\n  static std::shared_ptr<Engine> sptr(engine::CreateEngine());\n  return sptr;\n}\n\nEngine* Engine::Get() {\n  static Engine* inst = _GetSharedRef().get();\n  return inst;\n}\n}  // namespace mxnet\n"
  },
  {
    "path": "src/engine/engine_impl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file engine_impl.h\n * \\brief Internal implementation header of engine components.\n */\n#ifndef MXNET_ENGINE_ENGINE_IMPL_H_\n#define MXNET_ENGINE_ENGINE_IMPL_H_\n\n#include <mxnet/engine.h>\n\n/*! \\brief MACRO on whether or not enable debug option*/\n#define ENGINE_DEBUG 0\n\nnamespace mxnet {\nnamespace engine {\n\n/*! \\brief base class of engine operators, used for type checking */\nstruct Opr {\n#if ENGINE_DEBUG\n  virtual ~Opr() = default;\n#endif\n  /*!\n   * \\brief cast variable to derived type T\n   * \\tparam T the type we want to cast into.\n   * \\return A casted variable.\n   */\n  template <typename T>\n  inline T* Cast();\n};  // struct Opr\n\n// implementation of the inline functions\ntemplate <typename T>\ninline T* Var::Cast() {\n  static_assert(std::is_base_of<Var, T>::value, \"must inherit `mxnet::engine::Var`\");\n#if ENGINE_DEBUG\n  return dynamic_cast<T*>(this);\n#else\n  return static_cast<T*>(this);\n#endif\n}\n\ntemplate <typename T>\ninline T* Opr::Cast() {\n  static_assert(std::is_base_of<Opr, T>::value, \"must inherit `mxnet::engine::Opr`\");\n#if ENGINE_DEBUG\n  return dynamic_cast<T*>(this);\n#else\n  return static_cast<T*>(this);\n#endif\n}\n\n/*! \\brief Maximum number of GPUs */\nstatic constexpr std::size_t kMaxNumGPUs = 16;\n\n// predeclare factory function for each type of engine\n/*! \\return NaiveEngine instance */\nEngine* CreateNaiveEngine();\n#if MXNET_PREDICT_ONLY == 0\n/*! \\return ThreadedEnginePooled instance */\nEngine* CreateThreadedEnginePooled();\n/*! \\return ThreadedEnginePerDevie instance */\nEngine* CreateThreadedEnginePerDevice();\n#endif\n}  // namespace engine\n}  // namespace mxnet\n#endif  // MXNET_ENGINE_ENGINE_IMPL_H_\n"
  },
  {
    "path": "src/engine/naive_engine.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file naive_engine.cc\n * \\brief Implementation of NaiveEngine\n */\n#include <atomic>\n#include <future>\n#include <memory>\n#include <thread>\n#include <vector>\n#include \"./engine_impl.h\"\n#include \"../profiler/profiler.h\"\n#include \"./openmp.h\"\n#include \"../common/object_pool.h\"\n#include \"../profiler/custom_op_profiler.h\"\n\nnamespace mxnet {\nnamespace engine {\n\n/*!\n * \\brief var used in Naive Engine for tracking the version\n * of the objects it is associated with.\n */\nclass NaiveVar final : public Var, public common::ObjectPoolAllocatable<NaiveVar> {\n public:\n  inline static NaiveVar* CastFromBase(Var* ptr) {\n    return ptr->Cast<NaiveVar>();\n  }\n};  // class NaiveVar\n\n// implement naive engine\nclass NaiveEngine final : public Engine {\n public:\n  struct NaiveOpr : public Opr {\n    AsyncFn fn;\n    std::vector<VarHandle> const_vars;\n    std::vector<VarHandle> mutable_vars;\n    FnProperty prop;\n    std::string opr_name;\n    /*! \\brief indicate whether to profile this operator */\n    bool profiling{false};\n    /*! \\brief operator execution statistics */\n    std::unique_ptr<profiler::ProfileOperator> opr_profile;\n  };\n\n  NaiveEngine() {\n    objpool_opr_ref_ = common::ObjectPool<NaiveOpr>::_GetSharedRef();\n    objpool_var_ref_ = common::ObjectPool<NaiveVar>::_GetSharedRef();\n  }\n  // virtual destructor\n#if MXNET_USE_CUDA\n  ~NaiveEngine() override {\n    LOG(INFO) << \"Engine shutdown\";\n    for (size_t i = 0; i < streams_.size(); ++i) {\n      if (streams_[i] != nullptr) {\n        streams_[i] = nullptr;\n      }\n    }\n    for (size_t i = 0; i < aux_streams_.size(); ++i) {\n      if (aux_streams_[i] != nullptr) {\n        aux_streams_[i] = nullptr;\n      }\n    }\n  }\n#else\n  ~NaiveEngine() override = default;\n#endif\n\n  void Stop() override {}\n\n  void Start() override {}\n\n  // new variables\n  VarHandle NewVariable() override {\n    return NaiveVar::New();\n  }\n\n  OprHandle NewOperator(AsyncFn fn,\n                        std::vector<VarHandle> const& const_vars,\n                        std::vector<VarHandle> const& mutable_vars,\n                        FnProperty prop      = FnProperty::kNormal,\n                        const char* opr_name = nullptr,\n                        bool wait            = false) override {\n    NaiveOpr* opr     = new NaiveOpr();\n    opr->fn           = fn;\n    opr->const_vars   = const_vars;\n    opr->mutable_vars = mutable_vars;\n    opr->prop         = prop;\n    opr->opr_name     = opr_name ? std::string(opr_name) : std::string();\n    return opr;\n  }\n\n  void DeleteOperator(OprHandle op) override {\n    NaiveOpr* opr = op->Cast<NaiveOpr>();\n    delete opr;\n  }\n\n  void Push(OprHandle op, Context exec_ctx, int priority = 0, bool profiling = false) override {\n    profiler::Profiler* profiler = profiler::Profiler::Get();\n    NaiveOpr* opr                = op->Cast<NaiveOpr>();\n    opr->profiling = profiling && profiler->IsProfiling(profiler::Profiler::kSymbolic);\n    this->PushAsync(\n        [&](RunContext ctx, CallbackOnStart on_start, CallbackOnComplete on_complete) {\n          if (opr->profiling) {\n            std::unique_ptr<profiler::ProfileOperator::Attributes> attrs;\n            if (profiler->AggregateEnabled()) {\n              attrs = std::make_unique<profiler::ProfileOperator::Attributes>();\n            }\n            opr->opr_profile =\n                std::make_unique<profiler::ProfileOperator>(opr->opr_name.c_str(), attrs.release());\n            opr->opr_profile->startForDevice(exec_ctx.dev_type, exec_ctx.dev_id);\n          }\n          opr->fn(ctx, on_start, on_complete);\n          if (opr->profiling) {\n            opr->opr_profile->stop();\n          }\n        },\n        exec_ctx,\n        opr->const_vars,\n        opr->mutable_vars,\n        opr->prop,\n        priority,\n        opr->opr_name.c_str());\n  }\n\n  /*!\n   * \\brief NaiveEngine's PushAsync was intentionally synchronous.\n   * User should not make any assumption about execution order when using async interface of any\n   * engine.\n   */\n  void PushAsync(AsyncFn exec_fun,\n                 Context exec_ctx,\n                 std::vector<VarHandle> const& const_vars,\n                 std::vector<VarHandle> const& mutable_vars,\n                 FnProperty prop      = FnProperty::kNormal,\n                 int priority         = 0,\n                 const char* opr_name = nullptr,\n                 bool wait            = false) override {\n    std::promise<void> promise;\n    std::future<void> future     = promise.get_future();\n    CallbackOnStart on_start     = CreateOnStart(NaiveEngine::OnStart, &promise);\n    CallbackOnComplete callback  = CreateCallback(NaiveEngine::OnComplete, &promise);\n    profiler::Profiler* profiler = profiler::Profiler::Get();\n    auto opr_deleter             = [this](NaiveOpr* p) { this->DeleteOperator(p); };\n    std::unique_ptr<NaiveOpr, decltype(opr_deleter)> opr(nullptr, opr_deleter);\n    const bool profiling = opr_name && profiler->IsProfiling(profiler::Profiler::kImperative);\n    // GenerateDisplayName() will return a pointer to the correct name of the operator\n    const char* display_name =\n        profiling ? profiler::CustomOpProfiler::Get()->GenerateDisplayName(opr_name) : opr_name;\n    if (profiling) {\n      opr.reset(\n          NewOperator(exec_fun, const_vars, mutable_vars, prop, display_name)->Cast<NaiveOpr>());\n      opr->profiling = profiling;\n      std::unique_ptr<profiler::ProfileOperator::Attributes> attrs;\n      if (profiler->AggregateEnabled()) {\n        attrs = std::make_unique<profiler::ProfileOperator::Attributes>();\n      }\n      opr->opr_profile =\n          std::make_unique<profiler::ProfileOperator>(opr->opr_name.c_str(), attrs.release());\n      opr->opr_profile->startForDevice(exec_ctx.dev_type, exec_ctx.dev_id);\n    }\n    if (exec_ctx.dev_mask() == gpu::kDevMask) {\n#if MXNET_USE_CUDA\n      size_t dev_id = static_cast<size_t>(exec_ctx.dev_id);\n      cudaGetLastError();  // reset cuda error\n      MSHADOW_CATCH_ERROR(mshadow::SetDevice<gpu>(exec_ctx.dev_id));\n      if (streams_.size() <= dev_id) {\n        streams_.resize(dev_id + 1, nullptr);\n        aux_streams_.resize(dev_id + 1, nullptr);\n      }\n      if (streams_[dev_id] == nullptr) {\n        streams_[dev_id]     = mshadow::NewStream<gpu>(true, MXNET_USE_CUDNN != 0, dev_id);\n        aux_streams_[dev_id] = new GPUAuxStream(streams_[dev_id]);\n      }\n      exec_fun(RunContext{exec_ctx, streams_[dev_id], aux_streams_[dev_id]}, on_start, callback);\n#else\n      LOG(FATAL) << \"GPU is not enabled\";\n#endif\n    } else {\n      exec_fun(RunContext{exec_ctx, &cpu_stream_, nullptr}, on_start, callback);\n    }\n    future.wait();\n    // increment mutable var version\n    for (auto var : mutable_vars) {\n      ++var->version_;\n    }\n    if (profiling) {\n      opr->opr_profile->stop();\n    }\n  }\n\n  void DeleteVariable(SyncFn delete_fn, Context exec_ctx, VarHandle var) override {\n    NaiveVar* naive_var = NaiveVar::CastFromBase(var);\n    this->PushAsync(\n        [delete_fn, naive_var](\n            RunContext ctx, CallbackOnStart on_start, CallbackOnComplete on_complete) mutable {\n          on_start();\n          delete_fn(ctx);\n          NaiveVar::Delete(naive_var);\n          on_complete();\n        },\n        exec_ctx,\n        {},\n        {var},\n        FnProperty::kDeleteVar,\n        0,\n        \"DeleteVariable\");\n  }\n\n  void WaitForVar(VarHandle var) override {}\n\n  void WaitForAll() override {}\n\n  void Throw(VarHandle var) override {}\n\n  void NotifyShutdown() override {\n    shutdown_phase_.store(true);\n  }\n\n private:\n  // onstart\n  static void OnStart(Engine* engine, void* param, const dmlc::Error* error) {}\n  // callback to oncomplete\n  static void OnComplete(Engine* engine, void* param, const dmlc::Error* error) {\n    static_cast<std::promise<void>*>(param)->set_value();\n  }\n  /*! \\brief whether it is during shutdown phase*/\n  std::atomic<bool> shutdown_phase_{false};\n  // CPU stream\n  mshadow::Stream<cpu> cpu_stream_;\n  // GPU streams\n  std::vector<mshadow::Stream<gpu>*> streams_;\n#if MXNET_USE_CUDA\n  // GPU auxiliary streams\n  std::vector<GPUAuxStream*> aux_streams_;\n#endif\n  /*!\n   * \\brief Holding a shared_ptr to the object pool to prevent it from being destructed too early\n   * See also #309 (https://github.com/apache/mxnet/issues/309) and similar fix in\n   * threaded_engine.h. Without this, segfaults seen on CentOS7 in\n   * test_operator_gpu.py:test_convolution_multiple_streams\n   */\n  std::shared_ptr<common::ObjectPool<NaiveOpr> > objpool_opr_ref_;\n  std::shared_ptr<common::ObjectPool<NaiveVar> > objpool_var_ref_;\n};  // class NaiveEngine\n\nEngine* CreateNaiveEngine() {\n  return new NaiveEngine();\n}\n\n}  // namespace engine\n}  // namespace mxnet\n"
  },
  {
    "path": "src/engine/openmp.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n#include <dmlc/omp.h>\n#include <dmlc/base.h>\n#include <dmlc/parameter.h>\n#include <climits>\n#include \"./openmp.h\"\n\nnamespace mxnet {\nnamespace engine {\n\n#if defined(__i386__) || defined(_M_X86) || defined(_M_X64) || defined(__x86_64__)\n#define ARCH_IS_INTEL_X86\n#endif\n\nstatic inline bool is_env_set(const char* var) {\n  return dmlc::GetEnv(var, INT_MIN) != INT_MIN;\n}\n\nOpenMP* OpenMP::Get() {\n  static OpenMP openMP;\n  return &openMP;\n}\n\nOpenMP::OpenMP() : omp_num_threads_set_in_environment_(is_env_set(\"OMP_NUM_THREADS\")) {\n#ifdef _OPENMP\n  initialize_process();\n  const int max = dmlc::GetEnv(\"MXNET_OMP_MAX_THREADS\", INT_MIN);\n  if (max != INT_MIN) {\n    omp_thread_max_ = max;\n  } else {\n    if (!omp_num_threads_set_in_environment_) {\n      omp_thread_max_ = omp_get_num_procs();\n#ifdef ARCH_IS_INTEL_X86\n      omp_thread_max_ >>= 1;\n#endif\n      omp_set_num_threads(omp_thread_max_);\n    } else {\n      omp_thread_max_ = omp_get_max_threads();\n    }\n  }\n#else\n  enabled_        = false;\n  omp_thread_max_ = 1;\n#endif\n}\n\nvoid OpenMP::initialize_process() {\n#ifdef _OPENMP\n  omp_get_num_procs();  // will force OpenMP to be initialized\n#endif\n}\n\nvoid OpenMP::on_start_worker_thread(bool use_omp) {\n#ifdef _OPENMP\n  if (!omp_num_threads_set_in_environment_) {\n    omp_set_num_threads(use_omp ? GetRecommendedOMPThreadCount(true) : 1);\n  }\n#endif\n}\n\nvoid OpenMP::set_reserve_cores(int cores) {\n  CHECK_GE(cores, 0);\n  reserve_cores_ = cores;\n#ifdef _OPENMP\n  if (reserve_cores_ >= omp_thread_max_) {\n    omp_set_num_threads(1);\n  } else {\n    omp_set_num_threads(omp_thread_max_ - reserve_cores_);\n  }\n#endif\n}\n\nint OpenMP::GetRecommendedOMPThreadCount(bool exclude_reserved) const {\n#ifdef _OPENMP\n  if (enabled_) {\n    // OMP_NUM_THREADS was set in the environment at the time of static initialization\n    if (omp_num_threads_set_in_environment_) {\n      return omp_get_max_threads();\n    }\n    int thread_count = omp_get_max_threads();\n    if (exclude_reserved) {\n      if (reserve_cores_ >= thread_count) {\n        thread_count = 1;\n      } else {\n        thread_count -= reserve_cores_;\n      }\n    }\n    // Check that OMP doesn't suggest more than our 'omp_thread_max_' value\n    if (!omp_thread_max_ || thread_count < omp_thread_max_) {\n      return thread_count;\n    }\n    return omp_thread_max_;\n  } else {\n    return 1;\n  }\n#else\n  return 1;\n#endif\n}\n\nOpenMP* __init_omp__ = OpenMP::Get();\n\n}  // namespace engine\n}  // namespace mxnet\n"
  },
  {
    "path": "src/engine/openmp.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n#ifndef MXNET_ENGINE_OPENMP_H_\n#define MXNET_ENGINE_OPENMP_H_\n\nnamespace mxnet {\nnamespace engine {\n\n/*! \\brief OpenMP wrapper and management class\n *         This class manages a layer on top of the OMP implementation and does not\n *         interact bidirectionally with the OMP implementation for all behaviors\n *         (i.e. it's meant to be use explicitly for explicit arguments to omp pragmas\n *         without affecting the behavior when no arguments are given)\n */\nclass OpenMP {\n public:\n  OpenMP();\n\n  /*!\n   * \\brief Get the recommended number of OMP threads to use given the current context\n   * \\return Recommended number of OMP threads to use in a parallel operation\n   */\n  int GetRecommendedOMPThreadCount(bool exclude_reserved = true) const;\n\n  /*!\n   * \\brief Set whether clients of this class receive pro-OMP behavior guidance\n   * \\param enabled Set to 'true' if this class should provide OMP behavior\n   */\n  void set_enabled(bool enabled) {\n    enabled_ = enabled;\n  }\n  bool enabled() const {\n    return enabled_;\n  }\n\n  /*!\n   * \\brief Set maximum number of threads to be used in an OMP region\n   * \\param thread_max Maximum number of threads to be used in an OMP region\n   */\n  void set_thread_max(int thread_max) {\n    omp_thread_max_ = thread_max;\n  }\n  /*!\n   * \\brief Maximum number of threads to be used in an OMP region\n   * \\return Maximum number of threads\n   */\n  int thread_max() const {\n    return omp_thread_max_;\n  }\n\n  /*!\n   * \\brief Reserve cores to be excluded from OMP regions\n   * \\param cores Number of cores to be excluded from OMP region usage\n   */\n  void set_reserve_cores(int cores);\n  /*!\n   * \\brief Get number of cores to be excluded from OMP regions\n   * \\return Number of cores to be excluded from OMP regions\n   */\n  int reserve_cores() const {\n    return reserve_cores_;\n  }\n\n  /*!\n   * \\brief Call at the beginning of a worker thread's life.  This will set the omp_num_threads\n   *        for omp regions created by this thread\n   * \\param use_omp true if this thread plans to utilize parallel omp regions\n   */\n  void on_start_worker_thread(bool use_omp);\n\n  /*!\n   * \\brief Initialize a new process to use omp (after a fork,\n   *        in case you're starting threads in the atfork() that may interfere\n   *        with the initialization. Can serialize the init with this first.\n   */\n  void initialize_process();\n\n  /*!\n   * \\brief Get the OpenMP object's singleton pointer\n   * \\return Singleton OpenMP object pointer\n   */\n  static OpenMP* Get();\n\n private:\n  /*!\n   * \\brief Whether OpenMP layer is enabled (use more then one thread).  Independent of OMP library\n   *        behavior\n   */\n  volatile bool enabled_ = true;\n  /*!\n   * \\brief Maximum number of threads for any OMP region\n   */\n  volatile int omp_thread_max_ = 0;\n  /*!\n   * \\brief Number of cores to reserve for non-OMP regions\n   */\n  volatile int reserve_cores_ = 0;\n  /*!\n   * \\brief Whether OMP_NUM_THREADS was set in the environment.  If it is, we fall back to\n   *        the OMP's implementation's handling of that environment variable\n   */\n  const bool omp_num_threads_set_in_environment_;\n};\n\n}  // namespace engine\n}  // namespace mxnet\n\n#endif  // MXNET_ENGINE_OPENMP_H_\n"
  },
  {
    "path": "src/engine/stream_manager.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_ENGINE_STREAM_MANAGER_H_\n#define MXNET_ENGINE_STREAM_MANAGER_H_\n\n#include <dmlc/base.h>\n#include <mxnet/base.h>\n#include <cstddef>\n#include <array>\n#include <string>\n#include <memory>\n#include <mutex>\n#include \"./engine_impl.h\"\n#include \"../common/cuda/utils.h\"\n\nnamespace mxnet {\nnamespace engine {\n\n/*!\n * \\brief Stream manager.\n *\n * Uses a basic round-robin algorithm to dispatch GPU streams. Returns default\n * context on CPU.\n */\ntemplate <std::size_t kNumGpus, std::size_t kStreams>\nclass StreamManager {\n public:\n  StreamManager();\n  ~StreamManager() {\n    Finalize();\n  }\n  RunContext GetRunContext(Context const& ctx);\n  RunContext GetIORunContext(Context const& ctx);\n  void Finalize();\n\n private:\n  std::mutex mutex_;\n#if MXNET_USE_CUDA\n  std::array<std::array<mshadow::Stream<gpu>*, kStreams>, kNumGpus> gpu_streams_;\n  std::array<std::array<GPUAuxStream*, kStreams>, kNumGpus> gpu_aux_streams_;\n  std::array<mshadow::Stream<gpu>*, kNumGpus> gpu_io_streams_;\n  std::array<int, kNumGpus> gpu_cnt_;\n  std::array<std::unique_ptr<CUDAEventPool>, kNumGpus> event_pools_;\n#endif  // MXNET_USE_CUDA\n  DISALLOW_COPY_AND_ASSIGN(StreamManager);\n};  // class StreamManager\n\ntemplate <std::size_t kNumGpus, std::size_t kStreams>\nRunContext StreamManager<kNumGpus, kStreams>::GetRunContext(Context const& ctx) {\n  RunContext ret;\n  switch (ctx.dev_mask()) {\n    case cpu::kDevMask:\n      ret = RunContext{ctx, nullptr, nullptr};\n      break;\n    case gpu::kDevMask: {\n#if MXNET_USE_CUDA\n      std::size_t use_counter;\n      CUDAEventPool* event_pool;\n      {\n        std::lock_guard<std::mutex> lock{mutex_};\n        auto&& counter = gpu_cnt_.at(ctx.dev_id);\n        if (counter == -1) {\n          mxnet::common::cuda::DeviceStore device_store(ctx.dev_id);\n          for (auto&& primary_stream : gpu_streams_.at(ctx.dev_id)) {\n            primary_stream = mshadow::NewStream<gpu>(true, MXNET_USE_CUDNN != 0, ctx.dev_id);\n          }\n          int idx = 0;\n          for (auto&& aux_stream : gpu_aux_streams_.at(ctx.dev_id)) {\n            auto primary_stream = gpu_streams_.at(ctx.dev_id).at(idx++);\n            aux_stream          = new GPUAuxStream(primary_stream);\n          }\n          counter = 0;\n        }\n        if (event_pools_.at(ctx.dev_id) == nullptr) {\n          event_pools_[ctx.dev_id] = std::make_unique<CUDAEventPool>(ctx);\n        }\n        event_pool  = event_pools_.at(ctx.dev_id).get();\n        use_counter = counter;\n        counter     = (counter + 1) % kStreams;\n      }\n      ret = RunContext{ctx,\n                       gpu_streams_.at(ctx.dev_id).at(use_counter),\n                       gpu_aux_streams_.at(ctx.dev_id).at(use_counter),\n                       event_pool};\n      break;\n#else\n      LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR;\n#endif  // MXNET_USE_CUDA\n      default:\n        LOG(FATAL) << \"Not Reached\";\n    }\n  }\n  return ret;\n}\n\ntemplate <std::size_t kNumGpus, std::size_t kStreams>\nRunContext StreamManager<kNumGpus, kStreams>::GetIORunContext(Context const& ctx) {\n  RunContext ret;\n  switch (ctx.dev_mask()) {\n    case cpu::kDevMask:\n      ret = RunContext{ctx, nullptr, nullptr};\n      break;\n    case gpu::kDevMask: {\n#if MXNET_USE_CUDA\n      CUDAEventPool* event_pool;\n      {\n        std::lock_guard<std::mutex> lock{mutex_};\n        if (gpu_io_streams_.at(ctx.dev_id) == nullptr) {\n          mxnet::common::cuda::DeviceStore device_store(ctx.dev_id);\n          gpu_io_streams_.at(ctx.dev_id) = mshadow::NewStream<gpu>(false, false, ctx.dev_id);\n        }\n        if (event_pools_.at(ctx.dev_id) == nullptr) {\n          event_pools_[ctx.dev_id] = std::make_unique<CUDAEventPool>(ctx);\n        }\n        event_pool = event_pools_.at(ctx.dev_id).get();\n      }\n      ret = RunContext{ctx, gpu_io_streams_.at(ctx.dev_id), nullptr, event_pool};\n      break;\n#else\n      LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR;\n#endif  // MXNET_USE_CUDA\n      default:\n        LOG(FATAL) << \"Not Reached\";\n    }\n  }\n  return ret;\n}\n\ntemplate <std::size_t kNumGpus, std::size_t kStreams>\nStreamManager<kNumGpus, kStreams>::StreamManager() {\n#if MXNET_USE_CUDA\n  for (std::size_t i = 0; i < kNumGpus; ++i) {\n    gpu_cnt_.at(i) = -1;\n  }\n  for (auto&& i : gpu_io_streams_) {\n    i = nullptr;\n  }\n#endif  // MXNET_USE_CUDA\n}\n\ntemplate <std::size_t kNumGpus, std::size_t kStreams>\nvoid StreamManager<kNumGpus, kStreams>::Finalize() {\n#if MXNET_USE_CUDA\n  for (std::size_t i = 0; i < kNumGpus; ++i) {\n    if (gpu_cnt_.at(i) != -1) {\n      if (event_pools_.at(i) != nullptr) {\n        event_pools_[i].reset();\n      }\n      for (auto&& primary_stream : gpu_streams_.at(i)) {\n        // Catch exception for CUDA driver shutdown\n        MSHADOW_CATCH_ERROR(mshadow::DeleteStream<gpu>(primary_stream));\n      }\n      for (auto&& aux_stream : gpu_aux_streams_.at(i)) {\n        delete aux_stream;\n      }\n      gpu_cnt_.at(i) = -1;\n    }\n  }\n#endif  // MXNET_USE_CUDA\n}\n\n}  // namespace engine\n}  // namespace mxnet\n\n#endif  // MXNET_ENGINE_STREAM_MANAGER_H_\n"
  },
  {
    "path": "src/engine/thread_pool.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_ENGINE_THREAD_POOL_H_\n#define MXNET_ENGINE_THREAD_POOL_H_\n\n#include <dmlc/base.h>\n#include <dmlc/thread_group.h>\n#include <cstddef>\n#include <vector>\n#include <list>\n#include <thread>\n#include <utility>\n#include \"mxnet/base.h\"\n\nnamespace mxnet {\nnamespace engine {\n\n/*!\n * \\brief Thread pool.\n */\nclass ThreadPool {\n public:\n  /*! \\brief Signal event upon destruction, even for exceptions (RAII) */\n  struct SetReadyOnDestroy {\n    explicit inline SetReadyOnDestroy(const std::shared_ptr<dmlc::ManualEvent>& event)\n        : event_(event) {}\n    inline ~SetReadyOnDestroy() {\n      if (event_) {\n        event_->signal();\n      }\n    }\n    std::shared_ptr<dmlc::ManualEvent> event_;\n  };\n\n  /*!\n   * \\brief Constructor takes function to run.\n   * \\param size size of the thread pool.\n   * \\param func the function to run on the thread pool.\n   */\n  explicit ThreadPool(size_t size, std::function<void()> func) : worker_threads_(size) {\n    CHECK_GT(size, 0);\n    for (auto& i : worker_threads_) {\n      i = std::thread(func);\n    }\n  }\n  explicit ThreadPool(size_t size,\n                      std::function<void(std::shared_ptr<dmlc::ManualEvent> ready)> func,\n                      const bool wait)\n      : worker_threads_(size) {\n    CHECK_GT(size, 0);\n    for (auto& i : worker_threads_) {\n      std::shared_ptr<dmlc::ManualEvent> ptr = std::make_shared<dmlc::ManualEvent>();\n      ready_events_.emplace_back(ptr);\n      i = std::thread(func, ptr);\n    }\n    if (wait) {\n      WaitForReady();\n    }\n  }\n  ~ThreadPool() noexcept(false) {\n    for (auto&& i : worker_threads_) {\n      i.join();\n    }\n  }\n\n private:\n  /*!\n   * \\brief Wait for all started threads to signal that they're ready\n   */\n  void WaitForReady() {\n    for (const std::shared_ptr<dmlc::ManualEvent>& ptr : ready_events_) {\n      ptr->wait();\n    }\n  }\n\n  /*!\n   * \\brief Worker threads.\n   */\n  std::vector<std::thread> worker_threads_;\n  /*!\n   * \\brief Startup synchronization objects\n   */\n  std::list<std::shared_ptr<dmlc::ManualEvent>> ready_events_;\n  /*!\n   * \\brief Disallow default construction.\n   */\n  ThreadPool() = delete;\n  /*!\n   * \\brief Disallow copy construction and assignment.\n   */\n  DISALLOW_COPY_AND_ASSIGN(ThreadPool);\n};\n}  // namespace engine\n}  // namespace mxnet\n#endif  // MXNET_ENGINE_THREAD_POOL_H_\n"
  },
  {
    "path": "src/engine/threaded_engine.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file threaded_engine.cc\n * \\brief implements base threaded engine.\n * \\author Yutian Li\n */\n#include <dmlc/logging.h>\n#include <cassert>\n#include <algorithm>\n#include <condition_variable>\n#include <mutex>\n#include <utility>\n#include \"./threaded_engine.h\"\n#include \"../common/cuda/utils.h\"\n\nnamespace mxnet {\nnamespace engine {\n\n#if ENGINE_DEBUG\nstd::atomic<std::size_t> OprBlock::counter{0};\nstd::atomic<std::size_t> VersionedVarBlock::counter{0};\nstd::atomic<std::size_t> ThreadedVar::counter{0};\nstd::atomic<std::size_t> ThreadedOpr::counter{0};\n#endif  // ENGINE_DEBUG\n\nThreadedVar::ThreadedVar(VersionedVarBlock* head) : head_{head} {\n#if ENGINE_DEBUG\n  LOG(INFO) << __func__ << \" \" << ++counter;\n#endif  // ENGINE_DEBUG\n}\n\ninline void ThreadedVar::AppendReadDependency(OprBlock* opr_block) {\n  std::lock_guard<std::mutex> lock{mutex_};\n  if (pending_write_ == nullptr) {\n    // invariant: is_ready_to_read()\n    CHECK_GE(num_pending_reads_, 0);\n    // STATE CHANGE\n    ++num_pending_reads_;\n    // decrease wait counter\n    opr_block->decr_wait();\n  } else {\n    auto&& new_var_block = VersionedVarBlock::New();\n    assert(head_->next == nullptr);\n    assert(head_->trigger == nullptr);\n    assert(head_->write == false);\n    // append things to next.\n    head_->next    = new_var_block;\n    head_->trigger = opr_block;\n    head_          = new_var_block;\n  }\n}\n\ninline void ThreadedVar::AppendWriteDependency(OprBlock* opr_block) {\n  auto&& new_var_block = VersionedVarBlock::New();\n  std::lock_guard<std::mutex> lock{mutex_};\n  // invariant.\n  assert(head_->next == nullptr);\n  assert(head_->trigger == nullptr);\n  assert(head_->write == false);\n  // attach to head.\n  head_->next    = new_var_block;\n  head_->trigger = opr_block;\n  head_->write   = true;\n\n  // check if it is ready to write\n  if (pending_write_ == nullptr) {\n    // invariant: is_ready_to_read()\n    pending_write_ = head_;\n    CHECK_GE(num_pending_reads_, 0);\n    if (num_pending_reads_ == 0) {\n      // STATE CHANGE\n      opr_block->decr_wait();\n      num_pending_reads_ = kWriteTriggered;\n    }\n  } else {\n    CHECK_NE(num_pending_reads_, 0);\n  }\n  head_ = new_var_block;\n}\n\ntemplate <typename Dispatcher>\ninline void ThreadedVar::CompleteReadDependency(Dispatcher dispatcher) {\n  OprBlock* trigger = nullptr;\n  {\n    // this is lock scope\n    std::lock_guard<std::mutex> lock{mutex_};\n    CHECK_GT(num_pending_reads_, 0);\n\n    if (--num_pending_reads_ == 0) {\n      if (pending_write_ != nullptr) {\n        // STATE CHANGE\n        trigger            = pending_write_->trigger;\n        num_pending_reads_ = kWriteTriggered;\n      }\n    }\n  }\n  if (trigger != nullptr && trigger->decr_wait() == 0) {\n    dispatcher(trigger);\n  }\n}\n\ntemplate <typename Dispatcher>\ninline bool ThreadedVar::CompleteWriteDependency(Dispatcher dispatcher) {\n  // this is lock scope\n  VersionedVarBlock *old_pending_write, *end_of_read_chain;\n  OprBlock* trigger_write = nullptr;\n  {\n    std::lock_guard<std::mutex> lock{mutex_};\n    // invariants\n    assert(head_->next == nullptr);\n    assert(pending_write_ != nullptr);\n    CHECK_EQ(num_pending_reads_, kWriteTriggered);\n\n    // increment version number\n    ++version_;\n\n    // really delete\n    if (to_delete_) {\n      VersionedVarBlock* head = pending_write_->next;\n      VersionedVarBlock::Delete(pending_write_);\n      assert(head_ == head);\n      VersionedVarBlock::Delete(head);\n      return true;\n    }\n    // detach pending write\n    old_pending_write = pending_write_;\n    // search for chains to trigger\n    end_of_read_chain = old_pending_write->next;\n    // reset to 0 pending reads\n    num_pending_reads_ = 0;\n    while (end_of_read_chain != head_ && end_of_read_chain->write == false) {\n      ++num_pending_reads_;\n      end_of_read_chain = end_of_read_chain->next;\n    }\n    if (end_of_read_chain == head_) {\n      pending_write_ = nullptr;\n    } else {\n      // check if there is pending reads, if not trigger write\n      assert(end_of_read_chain->write == true);\n      pending_write_ = end_of_read_chain;\n      if (num_pending_reads_ == 0) {\n        // mark write as already activated in this var\n        num_pending_reads_ = kWriteTriggered;\n        trigger_write      = end_of_read_chain->trigger;\n      }\n    }\n  }\n  // This is outside of lock scope\n  // Be very carful, pending_write_ and num_pending_reads_\n  // can change now, do not rely on these two variables.\n  // The linked list \\in [old_pending_write, end_of_read_chain)\n  // is already detached from this Var.\n  // So it is safe to modify these\n  VersionedVarBlock* cur_head = old_pending_write->next;\n  VersionedVarBlock::Delete(old_pending_write);\n  // dispatch all the events\n  while (cur_head != end_of_read_chain) {\n    if (cur_head->trigger->decr_wait() == 0) {\n      dispatcher(cur_head->trigger);\n    }\n    auto prev = cur_head;\n    cur_head  = cur_head->next;\n    assert(cur_head != nullptr);\n    VersionedVarBlock::Delete(prev);\n  }\n  if (trigger_write != nullptr && trigger_write->decr_wait() == 0) {\n    dispatcher(trigger_write);\n  }\n  return false;\n}\n\ninline void ThreadedVar::SetToDelete() {\n  std::lock_guard<std::mutex> lock{mutex_};\n  to_delete_ = true;\n}\n\ninline bool ThreadedVar::ready_to_read() {\n  std::lock_guard<std::mutex> lock{mutex_};\n  return this->is_ready_to_read();\n}\n\ninline size_t ThreadedVar::version() {\n  std::lock_guard<std::mutex> lock{mutex_};\n  return this->version_;\n}\n\n// implementation of threaded engine\nThreadedVar* ThreadedEngine::NewVariable() {\n  return ThreadedVar::New(VersionedVarBlock::New());\n}\n\nThreadedOpr* ThreadedEngine::NewOperator(ThreadedEngine::AsyncFn fn,\n                                         std::vector<VarHandle> const& const_vars,\n                                         std::vector<VarHandle> const& mutable_vars,\n                                         FnProperty prop,\n                                         const char* opr_name,\n                                         bool wait) {\n  auto ret      = ThreadedOpr::New();\n  ret->opr_name = opr_name ? std::string(opr_name) : std::string();\n  ret->fn       = std::move(fn);\n  ret->prop     = prop;\n  ret->const_vars.resize(const_vars.size());\n  ret->mutable_vars.resize(mutable_vars.size());\n  ret->wait = wait;\n  std::transform(\n      const_vars.begin(), const_vars.end(), ret->const_vars.begin(), ThreadedVar::CastFromBase);\n  std::transform(mutable_vars.begin(),\n                 mutable_vars.end(),\n                 ret->mutable_vars.begin(),\n                 ThreadedVar::CastFromBase);\n  if (ENGINE_DEBUG != 0) {\n    CheckDuplicate(const_vars, mutable_vars);\n  }\n  return ret;\n}\n\nvoid ThreadedEngine::CheckDuplicate(std::vector<VarHandle> const& const_vars,\n                                    std::vector<VarHandle> const& mutable_vars) {\n  // Check for duplicates.\n  auto use                 = const_vars;\n  auto mutate              = mutable_vars;\n  const size_t use_size    = use.size();\n  const size_t mutate_size = mutate.size();\n  std::sort(use.begin(), use.end());\n  std::sort(mutate.begin(), mutate.end());\n  for (std::size_t i = 0; i < use_size; ++i) {\n    if (i != 0 && use.at(i) == use.at(i - 1)) {\n      LOG(FATAL) << \"duplicate items found in `const_vars`\";\n    }\n  }\n  for (std::size_t i = 0; i < mutate_size; ++i) {\n    if (i != 0 && mutate.at(i) == mutate.at(i - 1)) {\n      LOG(FATAL) << \"duplicate items found in `mutable_vars`\";\n    }\n  }\n  std::size_t j = 0;\n  for (std::size_t i = 0; i < use_size; ++i) {\n    while (j < mutate_size && mutate.at(j) < use.at(i)) {\n      ++j;\n    }\n    if (j == mutate_size) {\n      break;\n    }\n    if (mutate.at(j) == use.at(i)) {\n      LOG(FATAL) << \"duplicate items found between `const_vars` and `mutable_vars`\";\n    }\n  }\n}\n\nvoid ThreadedEngine::DeleteOperator(OprHandle op) {\n  ThreadedOpr* threaded_opr = ThreadedOpr::CastFromBase(op);\n  std::vector<VarHandle> deps;\n  deps.reserve(threaded_opr->const_vars.size() + threaded_opr->mutable_vars.size());\n  deps.insert(deps.end(), threaded_opr->const_vars.begin(), threaded_opr->const_vars.end());\n  deps.insert(deps.end(), threaded_opr->mutable_vars.begin(), threaded_opr->mutable_vars.end());\n  this->PushAsync(\n      [threaded_opr](RunContext, CallbackOnStart on_start, CallbackOnComplete on_complete) {\n        on_start();\n        ThreadedOpr::Delete(threaded_opr);\n        on_complete();\n      },\n      Context::CPU(),\n      {},\n      deps,\n      FnProperty::kDeleteVar,\n      0,\n      \"DeleteOperator\");\n}\n\nvoid ThreadedEngine::Push(OprHandle op, Context exec_ctx, int priority, bool profiling) {\n  BulkFlush();\n  ThreadedOpr* threaded_opr = ThreadedOpr::CastFromBase(op);\n  if (profiling) {\n    threaded_opr->opr_name =\n        profiler::CustomOpProfiler::Get()->GenerateDisplayName(threaded_opr->opr_name.c_str());\n  }\n  OprBlock* opr_block = OprBlock::New();\n  opr_block->opr      = threaded_opr;\n\n  opr_block->wait.store(\n      static_cast<int>(threaded_opr->const_vars.size() + threaded_opr->mutable_vars.size() + 1));\n  opr_block->ctx       = exec_ctx;\n  opr_block->priority  = priority;\n  opr_block->profiling = profiling;\n  ++pending_;\n  // Add read dependencies.\n  for (auto&& i : threaded_opr->const_vars) {\n    i->AppendReadDependency(opr_block);\n  }\n  // Add write dependencies.\n  for (auto&& i : threaded_opr->mutable_vars) {\n    i->AppendWriteDependency(opr_block);\n  }\n  if (opr_block->decr_wait() == 0) {\n    this->PushToExecute(opr_block, true);\n  }\n}\n\nvoid ThreadedEngine::PushAsync(AsyncFn fn,\n                               Context exec_ctx,\n                               std::vector<VarHandle> const& const_vars,\n                               std::vector<VarHandle> const& mutable_vars,\n                               FnProperty prop,\n                               int priority,\n                               const char* opr_name,\n                               bool wait) {\n#if MXNET_USE_CUDA\n  if (exec_ctx.dev_mask() == gpu::kDevMask) {\n    if (device_count_ < 0) {\n      int tmp = -1;\n      cudaGetDeviceCount(&tmp);\n      device_count_ = tmp;\n      CHECK_GT(device_count_, 0) << \"GPU usage requires at least 1 GPU\";\n    }\n    CHECK_LT(exec_ctx.dev_id, device_count_)\n        << \"Invalid GPU Id: \" << exec_ctx.dev_id\n        << \", Valid device id should be less than device_count: \" << device_count_;\n  }\n#endif\n  const bool profiling = profiler_->IsProfiling(profiler::Profiler::kImperative);\n  ThreadedOpr* opr     = NewOperator(std::move(fn), const_vars, mutable_vars, prop, opr_name, wait);\n  opr->temporary       = true;\n  Push(opr, exec_ctx, priority, profiling);\n}\n\nvoid ThreadedEngine::PushSync(SyncFn exec_fn,\n                              Context exec_ctx,\n                              std::vector<VarHandle> const& const_vars,\n                              std::vector<VarHandle> const& mutable_vars,\n                              FnProperty prop,\n                              int priority,\n                              const char* opr_name) {\n  if (!bulk_size() || prop != FnProperty::kNormal || priority) {\n    this->PushAsync(\n        [exec_fn](RunContext ctx, CallbackOnStart on_start, CallbackOnComplete on_complete) {\n          on_start();\n          exec_fn(ctx);\n          on_complete();\n        },\n        exec_ctx,\n        const_vars,\n        mutable_vars,\n        prop,\n        priority,\n        opr_name);\n    return;\n  }\n\n  const BulkStatus& bulk_status = *BulkStatusStore::Get();\n  if (bulk_status.count && exec_ctx != bulk_status.ctx)\n    BulkFlush();\n  BulkAppend(exec_fn, exec_ctx, const_vars, mutable_vars);\n}\n\nvoid ThreadedEngine::DeleteVariable(SyncFn delete_fn, Context exec_ctx, VarHandle var) {\n  ThreadedVar* threaded_var = ThreadedVar::CastFromBase(var);\n  this->PushAsync(\n      [delete_fn, threaded_var](\n          RunContext ctx, CallbackOnStart on_start, CallbackOnComplete on_complete) {\n        // Mark variable as orphan,\n        // so during `ThreadedEngine::OnComplete` it could be recycled.\n        on_start();\n        threaded_var->SetToDelete();\n        delete_fn(ctx);\n        on_complete();\n      },\n      exec_ctx,\n      {},\n      {var},\n      FnProperty::kDeleteVar,\n      0,\n      \"DeleteVariable\");\n}\n\nvoid ThreadedEngine::WaitForVar(VarHandle var) {\n  BulkFlush();\n  ThreadedVar* threaded_var = ThreadedVar::CastFromBase(var);\n  if (threaded_var->ready_to_read()) {\n    ThrowException(threaded_var);\n    return;\n  }\n  if (engine_info_) {\n    LOG(INFO) << \"Wait for \" << threaded_var;\n    debug_wait_var_ = threaded_var;\n  }\n  std::atomic<bool> done{false};\n  this->PushAsync(\n      [this, &done](RunContext, CallbackOnStart on_start, CallbackOnComplete on_complete) {\n        on_start();\n        if (engine_info_) {\n          LOG(INFO) << \"Sync is executed\";\n        }\n        {\n          std::unique_lock<std::mutex> lock{finished_m_};\n          done.store(true);\n        }\n        finished_cv_.notify_all();\n        if (engine_info_) {\n          LOG(INFO) << \"Sync is notified\";\n        }\n        on_complete();\n      },\n      Context::CPU(),\n      {var},\n      {},\n      FnProperty::kNormal,\n      0,\n      \"WaitForVar\",\n      true);\n  {\n    std::unique_lock<std::mutex> lock{finished_m_};\n    finished_cv_.wait(lock, [this, &done]() { return done.load() || kill_.load(); });\n  }\n\n  ThrowException(threaded_var);\n}\n\nvoid ThreadedEngine::WaitForAll() {\n  BulkFlush();\n  std::unique_lock<std::mutex> lock{finished_m_};\n  finished_cv_.wait(lock, [this]() { return pending_.load() == 0 || kill_.load(); });\n  std::exception_ptr exception_to_rethrow = nullptr;\n  if (!global_exception_refs_.empty()) {\n    // iterate through all exception refs\n    for (const auto& global_exception_ref : global_exception_refs_) {\n      // the first exception will be saved to be rethrown later\n      if (*global_exception_ref != nullptr && exception_to_rethrow == nullptr) {\n        exception_to_rethrow = *global_exception_ref;\n      }\n      // clear exceptions, WaitToRead following WaitForAll shouldn't throw\n      *global_exception_ref = nullptr;\n    }\n    // A waitall following a waitall shouldn't throw any exceptions\n    global_exception_refs_.clear();\n    if (exception_to_rethrow != nullptr) {\n      std::rethrow_exception(exception_to_rethrow);\n    }\n  }\n}\n\ninline void ThreadedEngine::OnComplete(ThreadedOpr* threaded_opr) {\n  bool is_temporary_opr = threaded_opr->temporary;\n  // Mark complete for read variables\n  for (auto&& i : threaded_opr->const_vars) {\n    i->CompleteReadDependency([this](OprBlock* opr) { this->PushToExecute(opr, false); });\n  }\n  // Mark complete for write variables.\n  for (auto&& i : threaded_opr->mutable_vars) {\n    if (threaded_opr->opr_exception && *threaded_opr->opr_exception) {\n      i->var_exception = threaded_opr->opr_exception;\n      // add current operator exceptions to global exceptions if not already\n      // added\n      AddToGlobalExceptions(threaded_opr->opr_exception);\n    }\n    const bool debug_info = (engine_info_ && debug_wait_var_ == i);\n    if (debug_info) {\n      LOG(INFO) << \"Complete write dep for \" << i;\n    }\n    const bool to_delete = i->CompleteWriteDependency([this, debug_info](OprBlock* opr) {\n      if (debug_info) {\n        LOG(INFO) << \"PushToExecute \" << opr;\n        debug_push_opr_ = opr;\n      }\n      this->PushToExecute(opr, false);\n      if (debug_info) {\n        LOG(INFO) << \"Fin PushToExecute \" << opr;\n      }\n    });\n    if (to_delete) {\n#if MXNET_USE_CUDA\n      auto& sync_obj = i->sync_object;\n      {\n        std::lock_guard<std::mutex> l(sync_obj.mutex);\n        sync_obj.reader_events.clear();\n        sync_obj.writer_event.clear();\n      }\n#endif\n      ThreadedVar::Delete(i);\n    }\n  }\n  // The function been pushed from `ThreadedEngine::DeleteOperator`\n  // could execute right after we mark all vars as complete, so if\n  // threaded_opr is not temporary, its value is not reliable\n  // anymore start from here.\n  int npending = 0;\n  {\n    std::unique_lock<std::mutex> lock{finished_m_};\n    npending = --pending_;\n  }\n  CHECK_GE(npending, 0);\n  if (npending == 0) {\n    // no need to grab lock when notify.\n    finished_cv_.notify_all();\n  }\n\n  // delete operator if it is temperory\n  if (is_temporary_opr) {\n    ThreadedOpr::Delete(threaded_opr);\n  }\n}\n\ninline void ThreadedEngine::ThrowException(ThreadedVar* threaded_var) {\n  if (threaded_var->var_exception && *threaded_var->var_exception) {\n    std::exception_ptr tmp       = *threaded_var->var_exception;\n    *threaded_var->var_exception = nullptr;\n    std::rethrow_exception(tmp);\n  }\n  return;\n}\n\nvoid ThreadedEngine::Throw(VarHandle var) {\n  ThreadedVar* threaded_var = ThreadedVar::CastFromBase(var);\n  ThrowException(threaded_var);\n}\n\nvoid ThreadedEngine::OnCompleteStatic(Engine* engine, void* opr_block_, const dmlc::Error* error) {\n  OprBlock* opr_block       = static_cast<OprBlock*>(opr_block_);\n  ThreadedOpr* threaded_opr = opr_block->opr;\n  if (error != nullptr) {\n    auto ex_p                   = std::make_exception_ptr(*error);\n    threaded_opr->opr_exception = std::make_shared<std::exception_ptr>(ex_p);\n  }\n  if (opr_block->profiling && threaded_opr->opr_name.size()) {\n    // record operator end timestamp\n    opr_block->opr_profile->stop();\n  }\n  static_cast<ThreadedEngine*>(engine)->OnComplete(threaded_opr);\n  OprBlock::Delete(opr_block);\n}\n\nvoid ThreadedEngine::OnStartStatic(Engine* engine, void* opr_block, const dmlc::Error* error) {\n  // no-op\n}\n\n#if MXNET_USE_CUDA\nstatic inline void AddEventHelper(std::unordered_map<cudaStream_t, EventInfo>* events_per_stream,\n                                  const EventInfo& cuda_event) {\n  auto event_stream = cuda_event.stream;\n  if (events_per_stream->count(event_stream) > 0) {\n    if ((*events_per_stream)[event_stream].pool_index < cuda_event.pool_index) {\n      (*events_per_stream)[event_stream] = cuda_event;\n    }\n  } else {\n    (*events_per_stream).emplace(event_stream, cuda_event);\n  }\n}\n\nstatic inline bool IsEngineAsync() {\n  std::string type = dmlc::GetEnv(\"MXNET_ENGINE_TYPE\", std::string(\"\"));\n  std::string async_engine_tag(\"Async\");\n  auto tag_pos = type.find(async_engine_tag);\n  return tag_pos != std::string::npos;\n}\n\nvoid ThreadedEngine::OnStartCPU(Engine* engine, void* opr_block, const dmlc::Error* error) {\n  static bool use_new_dep_engine = IsEngineAsync();\n  if (!use_new_dep_engine) {\n    return;\n  }\n  ThreadedOpr* threaded_opr = static_cast<OprBlock*>(opr_block)->opr;\n  std::unordered_map<cudaStream_t, EventInfo> event_per_stream;\n  for (auto* read_var : threaded_opr->const_vars) {\n    auto& sync_obj = read_var->sync_object;\n    std::lock_guard<std::mutex> l(sync_obj.mutex);\n    auto& reader_events = sync_obj.reader_events;\n    // check for expired events and delete them\n    reader_events.erase(std::remove_if(reader_events.begin(),\n                                       reader_events.end(),\n                                       [&](const EventInfo e_i) { return e_i.event.expired(); }),\n                        reader_events.end());\n    for (auto& cuda_event : reader_events) {\n      AddEventHelper(&event_per_stream, cuda_event);\n    }\n    if (!sync_obj.writer_event.empty()) {\n      if (sync_obj.writer_event[0].event.expired()) {\n        sync_obj.writer_event.clear();\n      } else {\n        AddEventHelper(&event_per_stream, sync_obj.writer_event[0]);\n      }\n    }\n  }\n\n  for (auto* write_var : threaded_opr->mutable_vars) {\n    auto& sync_obj = write_var->sync_object;\n    std::lock_guard<std::mutex> l(sync_obj.mutex);\n    auto& reader_events = sync_obj.reader_events;\n    // check for expired events and delete them\n    reader_events.erase(std::remove_if(reader_events.begin(),\n                                       reader_events.end(),\n                                       [&](const EventInfo e_i) { return e_i.event.expired(); }),\n                        reader_events.end());\n    for (auto& cuda_event : reader_events) {\n      AddEventHelper(&event_per_stream, cuda_event);\n    }\n    if (!sync_obj.writer_event.empty()) {\n      if (sync_obj.writer_event[0].event.expired()) {\n        sync_obj.writer_event.clear();\n      } else {\n        AddEventHelper(&event_per_stream, sync_obj.writer_event[0]);\n      }\n    }\n  }\n  for (auto event : event_per_stream) {\n    auto ev = event.second.event.lock();\n    MSHADOW_CUDA_CALL(cudaEventSynchronize(*ev));\n  }\n}\n\nvoid ThreadedEngine::OnStartGPU(Engine* engine, void* sync_info, const dmlc::Error* error) {\n  static bool use_new_dep_engine = IsEngineAsync();\n  if (!use_new_dep_engine) {\n    return;\n  }\n  auto* info = reinterpret_cast<GPUWorkerSyncInfo*>(sync_info);\n  CHECK(info->stream != nullptr);\n  auto* worker_stream       = reinterpret_cast<mshadow::Stream<gpu>*>(info->stream);\n  ThreadedOpr* threaded_opr = static_cast<OprBlock*>(info->opr_block)->opr;\n  std::unordered_map<cudaStream_t, EventInfo> event_per_stream;\n  for (auto* read_var : threaded_opr->const_vars) {\n    auto& sync_obj = read_var->sync_object;\n    std::lock_guard<std::mutex> l(sync_obj.mutex);\n    auto& reader_events = sync_obj.reader_events;\n    // check for expired events and delete them\n    reader_events.erase(std::remove_if(reader_events.begin(),\n                                       reader_events.end(),\n                                       [&](const EventInfo e_i) { return e_i.event.expired(); }),\n                        reader_events.end());\n    for (auto& writer : sync_obj.writer_event) {\n      if (writer.event.expired()) {\n        sync_obj.writer_event.clear();\n        break;\n      }\n      if (writer.stream != worker_stream->stream_) {\n        // if there is already a reader on the same stream as us,\n        // it already synced with that writer and we can rely on\n        // the ongoing sync\n        bool found = false;\n        for (const auto& reader : reader_events) {\n          if (reader.stream == worker_stream->stream_) {\n            found = true;\n            break;\n          }\n        }\n        if (!found) {\n          AddEventHelper(&event_per_stream, writer);\n        }\n      }\n    }\n  }\n  for (auto* write_var : threaded_opr->mutable_vars) {\n    auto& sync_obj = write_var->sync_object;\n    std::lock_guard<std::mutex> l(sync_obj.mutex);\n    // check for expired events and delete them\n    auto& reader_events = sync_obj.reader_events;\n    reader_events.erase(std::remove_if(reader_events.begin(),\n                                       reader_events.end(),\n                                       [&](const EventInfo e_i) { return e_i.event.expired(); }),\n                        reader_events.end());\n    // if there are some readers, we wait for them\n    for (auto& cuda_event : reader_events) {\n      if (worker_stream->stream_ != cuda_event.stream) {\n        AddEventHelper(&event_per_stream, cuda_event);\n      }\n    }\n    if (!sync_obj.writer_event.empty()) {\n      if (sync_obj.writer_event[0].event.expired()) {\n        sync_obj.writer_event.clear();\n      } else {\n        if (worker_stream->stream_ != sync_obj.writer_event[0].stream) {\n          AddEventHelper(&event_per_stream, sync_obj.writer_event[0]);\n        }\n      }\n    }\n  }\n  for (auto event : event_per_stream) {\n    auto ev = event.second.event.lock();\n    MSHADOW_CUDA_CALL(cudaStreamWaitEvent(worker_stream->stream_, *ev, 0));\n  }\n}\n\nvoid ThreadedEngine::OnCompleteGPU(Engine* engine, void* sync_info, const dmlc::Error* error) {\n  auto* info = reinterpret_cast<GPUWorkerSyncInfo*>(sync_info);\n  CHECK(info->stream != nullptr);\n\n  auto* worker_stream            = reinterpret_cast<mshadow::Stream<gpu>*>(info->stream);\n  static bool use_new_dep_engine = IsEngineAsync();\n\n  if (!use_new_dep_engine) {\n    worker_stream->Wait();\n    ThreadedEngine::OnCompleteStatic(engine, info->opr_block, error);\n    GPUWorkerSyncInfo::Delete(info);\n    return;\n  }\n\n  ThreadedOpr* threaded_opr    = static_cast<OprBlock*>(info->opr_block)->opr;\n  auto* event_pool             = static_cast<CUDAEventPool*>(info->event_pool);\n  auto [event, event_pool_idx] = event_pool->GetNextEvent();  // NOLINT(*)\n  auto ev                      = event.lock();\n  MSHADOW_CUDA_CALL(cudaEventRecord(*ev, worker_stream->stream_));\n  for (auto* read_var : threaded_opr->const_vars) {\n    auto& sync_obj = read_var->sync_object;\n    std::lock_guard<std::mutex> l(sync_obj.mutex);\n    // If some reader event is already recorded on the same stream,\n    // we want to replace ourselves by it\n    int i;\n    for (i = 0; i < sync_obj.reader_events.size(); ++i) {\n      auto stream = sync_obj.reader_events[i].stream;\n      if (stream == worker_stream->stream_) {\n        sync_obj.reader_events[i].event      = event;\n        sync_obj.reader_events[i].pool_index = event_pool_idx;\n        break;\n      }\n    }\n    if (i == sync_obj.reader_events.size()) {\n      sync_obj.reader_events.push_back({event, worker_stream->stream_, event_pool_idx});\n    }\n  }\n\n  for (auto* write_var : threaded_opr->mutable_vars) {\n    auto& sync_obj = write_var->sync_object;\n    std::lock_guard<std::mutex> l(sync_obj.mutex);\n    sync_obj.reader_events.clear();\n    sync_obj.writer_event.clear();\n    sync_obj.writer_event.push_back({event, worker_stream->stream_, event_pool_idx});\n  }\n\n  ThreadedEngine::OnCompleteStatic(engine, info->opr_block, error);\n  GPUWorkerSyncInfo::Delete(info);\n}\n#endif\n\n}  // namespace engine\n}  // namespace mxnet\n"
  },
  {
    "path": "src/engine/threaded_engine.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file threaded_engine.h\n * \\brief Implements base class of threaded engine\n *    that tracks the dependency and pushes actions to execute.\n * \\author Yutian Li\n */\n#ifndef MXNET_ENGINE_THREADED_ENGINE_H_\n#define MXNET_ENGINE_THREADED_ENGINE_H_\n\n#include <dmlc/base.h>\n#include <dmlc/logging.h>\n#include <dmlc/omp.h>\n#include <mxnet/storage.h>\n#include <vector>\n#include <functional>\n#include <condition_variable>\n#include <atomic>\n#include <utility>\n#include <mutex>\n#include <string>\n#include <thread>\n#include \"./engine_impl.h\"\n#include \"../profiler/profiler.h\"\n#include \"./openmp.h\"\n#include \"../common/object_pool.h\"\n#include \"../profiler/custom_op_profiler.h\"\n\nnamespace mxnet {\nnamespace engine {\n\n// Define helper macros for debug information.\n#if ENGINE_DEBUG\n#define DEFINE_ENGINE_DEBUG_INFO(Type)         \\\n  static std::atomic<std::size_t> counter;     \\\n  Type() {                                     \\\n    LOG(INFO) << __func__ << \" \" << ++counter; \\\n  }                                            \\\n  ~Type() {                                    \\\n    LOG(INFO) << __func__ << \" \" << --counter; \\\n  }\n#else\n#define DEFINE_ENGINE_DEBUG_INFO(Type)\n#endif\n\n// Forward declarations\nstruct ThreadedOpr;\n\n/*! shared_ptr to exception_ptr, used for exception handling */\ntypedef std::shared_ptr<std::exception_ptr> ExceptionRef;\n\n/*!\n * \\brief Operation block in the scheduler.\n *  Each OprBlock corresponds to an operation pushed to the engine.\n */\nstruct OprBlock : public common::ObjectPoolAllocatable<OprBlock> {\n  /*!\n   * \\brief wait number of pending tasks this OprBlock is waiting for.\n   */\n  std::atomic<int> wait{0};\n  /*! \\brief Pointer to information on performing real operation */\n  ThreadedOpr* opr{nullptr};\n  /*! \\brief The context this operator */\n  Context ctx;\n  /*! \\brief priority of the function */\n  int priority;\n  /*! \\brief indicate whether to profile this operator */\n  bool profiling{false};\n  /*! \\brief operator execution statistics */\n  std::unique_ptr<profiler::ProfileOperator> opr_profile;\n  // define possible debug information\n  DEFINE_ENGINE_DEBUG_INFO(OprBlock);\n  /*!\n   * \\brief call this function to decrease the wait counter.\n   * \\return the wait counter after the decreasement.\n   */\n  inline int decr_wait() {\n    // check invariant, avoid over trigger\n    const int ret = --wait;\n    CHECK_GE(ret, 0);\n    return ret;\n  }\n};  // struct OprBlock\n\n/*!\n * \\brief VersionedVarBlock that corresponding to a variable version.\n *  This is a basic unit of LinkedList in the ThreadedVar.\n */\nstruct VersionedVarBlock : public common::ObjectPoolAllocatable<VersionedVarBlock> {\n  /*! \\brief next block in the LinkedList */\n  VersionedVarBlock* next{nullptr};\n  /*! \\brief the operation this block triggers */\n  OprBlock* trigger{nullptr};\n  /*! \\brief whether this operation is a write(mutate) operation. */\n  bool write{false};\n  /*! \\brief define possible debug information */\n  DEFINE_ENGINE_DEBUG_INFO(VersionedVarBlock);\n};  // struct VersionedVarBlock\n\n/*!\n * \\brief Variable implementation.\n *  Each ThreadedVar is a linked list(queue) of operations to be performed.\n */\nclass ThreadedVar final : public Var, public common::ObjectPoolAllocatable<ThreadedVar> {\n public:\n  /*!\n   * \\brief constructor\n   * \\param head head block of the LinkedList,\n   *             need to be initialized with next==nullptr and trigger=nullptr.\n   */\n  explicit ThreadedVar(VersionedVarBlock* head);\n  /*!\n   * \\brief Schedule a read operation on this variable.\n   *  If the opr_block can be runed right away,\n   *  the wait counter of opr_block will be decreased.\n   *  Otherwise, the opr_block will be added to waiting queue.\n   * \\param opr_block The operation to be scheduled.\n   */\n  inline void AppendReadDependency(OprBlock* opr_block);\n  /*!\n   * \\brief Schedule a write operation on this variable.\n   *  If the opr_block can be runed right away,\n   *  the wait counter of opr_block will be decreased.\n   *  Otherwise, the opr_block will be added to waiting queue.\n   * \\param opr_block The operation to be scheduled.\n   */\n  inline void AppendWriteDependency(OprBlock* opr_block);\n  /*!\n   * \\brief A read operation is completed on this variable.\n   *  This function may trigger subsequent waiting operations on this variable.\n   *\n   * \\param dispatcher the function called to trigger the operation,\n   *            when all of its dependencies are satiesfied.\n   * \\tparam Dispatcher the function called to trigger an operation.\n   */\n  template <typename Dispatcher>\n  inline void CompleteReadDependency(Dispatcher dispatcher);\n  /*!\n   * \\brief A write operation is completed on this variable.\n   *  This function may trigger subsequent waiting operations on this variable.\n   *\n   * \\param dispatcher the function called to trigger the operation,\n   *            when all of its dependencies are satiesfied.\n   * \\tparam Dispatcher the function called to trigger an operation.\n   * \\return to_delete, whether this Variable can be deleted after this functin.\n   */\n  template <typename Dispatcher>\n  inline bool CompleteWriteDependency(Dispatcher dispatcher);\n  /*! \\brief Mark this variable to be deleted. */\n  inline void SetToDelete();\n  /*! \\return whether this variable is ready to read. */\n  inline bool ready_to_read();\n  inline size_t version() override;\n  /*!\n   * \\brief Cast a Var pointer to ThreadedVar pointer\n   * \\param ptr pointer from base.\n   * \\return a casted pointer.\n   */\n  inline static ThreadedVar* CastFromBase(Var* ptr) {\n    return ptr->Cast<ThreadedVar>();\n  }\n  // code for debug.\n#if ENGINE_DEBUG\n  static std::atomic<std::size_t> counter;\n  ~ThreadedVar() {\n    LOG(INFO) << __func__ << \" \" << --counter;\n  }\n#endif  // ENGINE_DEBUG\n  /*!\n   * \\brief exception_ptr associated with the ThreadedOpr\n   * cannot modify state of exception object since dereferencing\n   * exception_ptr is undefined behavior. Using shared_ptr to hold\n   * exception_ptr and overcome this limitation */\n  ExceptionRef var_exception;\n\n private:\n  // TODO(hotpxl) change this to spinlock for faster runtime\n  // TODO(hotpxl) consider rename head\n  /*! \\brief internal mutex of the ThreadedVar */\n  std::mutex mutex_;\n  /*!\n   * \\brief number of pending reads operation in the variable.\n   *  will be marked as -1 when there is a already triggered pending write.\n   */\n  int num_pending_reads_{0};\n  /*!\n   * \\brief Points to the last VersionedVarBlock in the queue.\n   *  head_ always points to a empty VersionedVarBlock.\n   *  So when we want to append an operation to the queue:\n   *    1) update head_->trigger to be new op\n   *    2) update head_->next to be a new VersionedVarBlock\n   *    3) move head to head->next.\n   */\n  VersionedVarBlock* head_{nullptr};\n  /*!\n   * \\brief The pointer to next write to perform.\n   *  This pointer will only be updated when the write completes.\n   *  This is actually the head(oldest operation) in the queue.\n   */\n  VersionedVarBlock* pending_write_{nullptr};\n  /*!\n   * \\brief If true, delete after operation completes.\n   */\n  bool to_delete_{false};\n  /*! \\brief special const on num_pending_reads_ to mark write being triggered */\n  static constexpr int kWriteTriggered = -1;\n  /*!\n   * \\brief derived invariant of ready to ready, without lock.\n   * \\return whether the current variable is ready to read.\n   */\n  inline bool is_ready_to_read() const {\n    return pending_write_ == nullptr;\n  }\n};  // struct ThreadedVar\n\n/*!\n * \\brief Operator used in ThreadedEngine.\n */\nstruct ThreadedOpr final : public Opr, public common::ObjectPoolAllocatable<ThreadedOpr> {\n  /*! \\brief The function to be invoked each time. */\n  Engine::AsyncFn fn;\n  /*! \\brief The variable this operation will read from. */\n  std::vector<ThreadedVar*> const_vars;\n  /*! \\brief The variable this operation will mutate. */\n  std::vector<ThreadedVar*> mutable_vars;\n  /*! \\brief The property of the operator */\n  FnProperty prop;\n  /*! \\brief The name of the operator */\n  std::string opr_name;\n  /*!\n   * \\brief Whether this is an temporary operator\n   *        that can be deleted right after the operation completed.\n   */\n  bool temporary{false};\n  /*!\n   * \\brief Whether this is a WaitForVar operation\n   */\n  bool wait{false};\n  /*!\n   * \\brief Cast a Opr pointer to ThreadedOpr pointer\n   * \\param ptr pointer from base.\n   * \\return a casted pointer.\n   */\n  inline static ThreadedOpr* CastFromBase(Opr* ptr) {\n    return ptr->Cast<ThreadedOpr>();\n  }\n  // define possible debug information\n  DEFINE_ENGINE_DEBUG_INFO(ThreadedOpr);\n  /*!\n   * \\brief exception_ptr associated with the ThreadedOpr\n   * cannot modify state of exception object since dereferencing\n   * exception_ptr is undefined behavior. Using shared_ptr to hold\n   * exception_ptr and overcome this limitation */\n  ExceptionRef opr_exception;\n};  // struct ThreadedOpr\n\n/*!\n * \\brief Base class of all ThreadedEngine.\n *  This class implements a thread safe version of engine.\n *  The engine tracks the dependencies, and will call PushToExecute\n *  to execute a specific task.\n *\n *  Subclass can implement PushToExecute to design specific\n *  execution policy for the tasks.\n */\nclass ThreadedEngine : public Engine {\n public:\n  // implementing all the functions from Engine.\n  ThreadedVar* NewVariable() override;\n  ThreadedOpr* NewOperator(AsyncFn fn,\n                           std::vector<VarHandle> const& const_vars,\n                           std::vector<VarHandle> const& mutable_vars,\n                           FnProperty prop      = FnProperty::kNormal,\n                           const char* opr_name = nullptr,\n                           bool wait            = false) override;\n  void DeleteOperator(OprHandle op) override;\n  void Push(OprHandle op, Context exec_ctx, int priority = 0, bool profiling = false) override;\n  void PushAsync(AsyncFn exec_fun,\n                 Context exec_ctx,\n                 std::vector<VarHandle> const& const_vars,\n                 std::vector<VarHandle> const& mutable_vars,\n                 FnProperty prop      = FnProperty::kNormal,\n                 int priority         = 0,\n                 const char* opr_name = nullptr,\n                 bool wait            = false) override;\n  void PushSync(SyncFn exec_fn,\n                Context exec_ctx,\n                std::vector<VarHandle> const& const_vars,\n                std::vector<VarHandle> const& mutable_vars,\n                FnProperty prop      = FnProperty::kNormal,\n                int priority         = 0,\n                const char* opr_name = nullptr) override;\n  void DeleteVariable(SyncFn delete_fn, Context exec_ctx, VarHandle var) override;\n  void WaitForVar(VarHandle var) override;\n  void WaitForAll() override;\n  void Throw(VarHandle var) override;\n  void NotifyShutdown() override {\n    shutdown_phase_.store(true);\n  }\n\n  ThreadedEngine() {\n    engine_info_ = dmlc::GetEnv(\"MXNET_ENGINE_INFO\", false);\n\n    objpool_opr_ref_    = common::ObjectPool<ThreadedOpr>::_GetSharedRef();\n    objpool_blk_ref_    = common::ObjectPool<OprBlock>::_GetSharedRef();\n    objpool_varblk_ref_ = common::ObjectPool<VersionedVarBlock>::_GetSharedRef();\n    objpool_var_ref_    = common::ObjectPool<ThreadedVar>::_GetSharedRef();\n\n    storage_ref_ = Storage::_GetSharedRef();\n\n    // Get a ref to the profiler so that it doesn't get killed before us\n    profiler::Profiler::Get(&profiler_);\n  }\n  ~ThreadedEngine() {\n    {\n      std::unique_lock<std::mutex> lock{finished_m_};\n      kill_.store(true);\n    }\n    finished_cv_.notify_all();\n  }\n\n protected:\n  /*!\n   * \\brief Push the opr block to execution queue to be executed.\n   *  This function is implemented by the corresponding subclass\n   *  for specific policy.\n   *\n   * \\param opr_block The operator block.\n   * \\param pusher_thread whether the caller is the thread that calls push\n   */\n  virtual void PushToExecute(OprBlock* opr_block, bool pusher_thread) = 0;\n  /*!\n   * \\brief Call this function to actually execute an opr_block\n   *  This function also deletes the opr_block after execution.\n   * \\param run_ctx runtime context used to execute the function.\n   * \\param opr_block the opr_block to be executed and deleted.\n   */\n  void ExecuteOprBlock(RunContext run_ctx,\n                       OprBlock* opr_block,\n                       CallbackOnStart on_start,\n                       CallbackOnComplete callback) {\n    ThreadedOpr* threaded_opr = opr_block->opr;\n    if (opr_block->profiling && threaded_opr->opr_name.size()) {\n      std::unique_ptr<profiler::ProfileOperator::Attributes> attrs;\n      if (profiler_->AggregateEnabled()) {\n        attrs.reset(new profiler::ProfileOperator::Attributes());\n      }\n      const Context& ctx = opr_block->ctx;\n      opr_block->opr_profile.reset(\n          new profiler::ProfileOperator(threaded_opr->opr_name.c_str(), attrs.release()));\n      opr_block->opr_profile->startForDevice(ctx.dev_type, ctx.dev_id);\n    }\n    const bool debug_info = (engine_info_ && debug_push_opr_ == opr_block);\n    if (debug_info) {\n      LOG(INFO) << \"ExecuteOprBlock \" << opr_block << \"shutdown_phase=\" << shutdown_phase_;\n    }\n    // still run cleanup in shutdown_phase\n    if (!shutdown_phase_ || threaded_opr->prop == FnProperty::kDeleteVar) {\n      try {\n        OnStart(threaded_opr);\n        if (debug_info) {\n          LOG(INFO) << \"ExecuteOprFn \";\n        }\n        try {\n          if ((!(threaded_opr->opr_exception && *threaded_opr->opr_exception) ||\n               threaded_opr->prop == FnProperty::kNoSkip) ||\n              threaded_opr->wait) {\n            threaded_opr->fn(run_ctx, on_start, callback);\n          } else {\n            on_start();\n            callback();\n          }\n        } catch (const std::exception& e) {\n          on_start();\n          threaded_opr->opr_exception =\n              std::make_shared<std::exception_ptr>(std::current_exception());\n          callback();\n        }\n        if (debug_info) {\n          LOG(INFO) << \"Fin ExecuteOprFn \";\n        }\n      } catch (std::exception& e) {\n        std::string what = e.what();\n        if (what.find(\"driver shutting down\") == std::string::npos && !shutdown_phase_) {\n          LOG(FATAL) << e.what() << \"\\n\"\n                     << \"A fatal error occurred in asynchronous engine operation. \"\n                        \"If you do not know what caused this error, \"\n                        \"you can try set environment variable MXNET_ENGINE_TYPE \"\n                        \"to NaiveEngine and run with debugger (i.e. gdb). \"\n                        \"This will force all operations to be synchronous and \"\n                        \"backtrace will give you the series of calls that lead \"\n                        \"to this error. Remember to set MXNET_ENGINE_TYPE back to \"\n                        \"empty after debugging.\";\n        }\n      }\n    } else {\n      on_start();\n      callback();\n    }\n  }\n\n  int bulk_size() const override {\n    const profiler::Profiler* prof = profiler::Profiler::Get();\n    return (prof && prof->AggregateRunning()) ? 0 : BulkStatusStore::Get()->bulk_size;\n  }\n\n  int set_bulk_size(int bulk_size) override {\n    BulkStatus& bulk_status = *BulkStatusStore::Get();\n    std::swap(bulk_status.bulk_size, bulk_size);\n    if (bulk_status.count >= bulk_status.bulk_size)\n      BulkFlush();\n    if (!bulk_status.functions) {\n      bulk_status.functions.reset(new std::vector<SyncFn>());\n    }\n    bulk_status.functions->reserve(bulk_size);\n    return bulk_size;\n  }\n\n protected:\n  static void OnStartStatic(Engine* engine, void* opr_block, const dmlc::Error* error);\n  static void OnCompleteStatic(Engine* engine, void* threaded_opr, const dmlc::Error* error);\n#if MXNET_USE_CUDA\n  static void OnStartCPU(Engine* engine, void* opr_block, const dmlc::Error* error);\n  static void OnStartGPU(Engine* engine, void* sync_info, const dmlc::Error* error);\n  static void OnCompleteGPU(Engine* engine, void* sync_info, const dmlc::Error* error);\n  struct GPUWorkerSyncInfo : public common::ObjectPoolAllocatable<GPUWorkerSyncInfo> {\n    void* opr_block{nullptr};\n    void* stream{nullptr};\n    void* event_pool{nullptr};\n  };\n\n  std::shared_ptr<common::ObjectPool<GPUWorkerSyncInfo>> objpool_gpu_sync_ref_;\n#endif\n\n private:\n  /*! \\brief structure for holding bulk execution status */\n  struct BulkStatus {\n    /*! \\brief maximum number of ops per bulk */\n    int bulk_size = 0;\n    /*! \\brief current number of ops in bulk */\n    int count = 0;\n    /*! \\brief context of current ops */\n    Context ctx;\n    /*! \\brief current op functions */\n    std::shared_ptr<std::vector<SyncFn>> functions;\n    /*! \\brief constant variables */\n    std::vector<VarHandle> const_vars;\n    /*! \\brief mutable variables */\n    std::vector<VarHandle> mutable_vars;\n  };\n  /*! thread local store for bulk */\n  typedef dmlc::ThreadLocalStore<BulkStatus> BulkStatusStore;\n\n  /*!\n   * \\brief check if thee is duplication in const_vars and mutable_vars.\n   * \\param const_vars the variables to read from.\n   * \\param mutable_vars the variables to mutate.\n   */\n  void CheckDuplicate(std::vector<VarHandle> const& const_vars,\n                      std::vector<VarHandle> const& mutable_vars);\n  /*!\n   * \\brief Callback on operation completion.\n   *\n   * On operation completion, this will trigger subsequent operations.\n   */\n  inline void OnComplete(ThreadedOpr* threaded_opr);\n  /*!\n   * \\brief rethrow caught exception in WaitForVar\n   * \\param threaded_var the var that we are waiting to read\n   */\n  inline void ThrowException(ThreadedVar* threaded_var);\n  /*!\n   * \\brief Mark exceptions before operation execution.\n   *\n   * Will mark the operator as a failure and associate exception_ptr\n   * if any of the read dependencies have exception associated.\n   */\n  inline void OnStart(ThreadedOpr* threaded_opr) {\n    for (auto&& i : threaded_opr->const_vars) {\n      if (i->var_exception && *i->var_exception) {\n        threaded_opr->opr_exception = i->var_exception;\n        AddToGlobalExceptions(threaded_opr->opr_exception);\n        break;\n      }\n    }\n    if (!(threaded_opr->opr_exception && *threaded_opr->opr_exception)) {\n      for (auto&& i : threaded_opr->mutable_vars) {\n        if (i->var_exception && *i->var_exception) {\n          threaded_opr->opr_exception = i->var_exception;\n          AddToGlobalExceptions(threaded_opr->opr_exception);\n          break;\n        }\n      }\n    }\n  }\n\n  /*!\n   * \\brief find exception in global_exception_refs and add it if missing\n   * \\param opr_exception the exception to be added to global_exception_refs\n   */\n  inline void AddToGlobalExceptions(const ExceptionRef& opr_exception) {\n    auto it =\n        std::find(global_exception_refs_.begin(), global_exception_refs_.end(), opr_exception);\n    if (it == global_exception_refs_.end()) {\n      global_exception_refs_.push_back(opr_exception);\n    }\n    return;\n  }\n  /*! \\brief append an operator to bulk */\n  inline void BulkAppend(SyncFn exec_fn,\n                         Context exec_ctx,\n                         std::vector<VarHandle> const& const_vars,\n                         std::vector<VarHandle> const& mutable_vars) {\n    BulkStatus& bulk_status = *BulkStatusStore::Get();\n    if (!bulk_status.functions) {\n      bulk_status.functions.reset(new std::vector<SyncFn>());\n    }\n    bulk_status.functions->push_back(exec_fn);\n    if (!bulk_status.count) {\n      bulk_status.ctx = exec_ctx;\n    }\n\n    ++bulk_status.count;\n    bulk_status.const_vars.insert(\n        bulk_status.const_vars.end(), const_vars.begin(), const_vars.end());\n    bulk_status.mutable_vars.insert(\n        bulk_status.mutable_vars.end(), mutable_vars.begin(), mutable_vars.end());\n\n    if (bulk_status.count >= bulk_status.bulk_size)\n      BulkFlush();\n  }\n  /*! \\brief flush current bulk to execution */\n  inline void BulkFlush() {\n    BulkStatus& bulk_status = *BulkStatusStore::Get();\n    if (!bulk_status.count)\n      return;\n    bulk_status.count = 0;\n    DeduplicateVarHandle(&bulk_status.const_vars, &bulk_status.mutable_vars);\n    auto functions = bulk_status.functions;\n    this->PushAsync(\n        [functions](RunContext ctx, CallbackOnStart on_start, CallbackOnComplete on_complete) {\n          on_start();\n          for (auto& fn : *functions) {\n            fn(ctx);\n          }\n          on_complete();\n        },\n        bulk_status.ctx,\n        bulk_status.const_vars,\n        bulk_status.mutable_vars,\n        FnProperty::kNormal,\n        0,\n        \"ImperativeBulk\");\n    bulk_status.functions.reset(new std::vector<SyncFn>());\n    bulk_status.functions->reserve(bulk_status.bulk_size);\n    bulk_status.const_vars.clear();\n    bulk_status.mutable_vars.clear();\n  }\n  /*!\n   * \\brief Number of pending operations.\n   */\n  std::atomic<int> pending_{0};\n  /*! \\brief whether we want to kill the waiters */\n  std::atomic<bool> kill_{false};\n  /*! \\brief whether it is during shutdown phase*/\n  std::atomic<bool> shutdown_phase_{false};\n  /*!\\brief show more information from engine actions */\n  bool engine_info_{false};\n  /*! \\brief debug information about wait for var. */\n  std::atomic<ThreadedVar*> debug_wait_var_{nullptr};\n  /*! \\brief debug information about wait for var. */\n  std::atomic<OprBlock*> debug_push_opr_{nullptr};\n  /*!\n   * \\brief Mutex and condition_variable,\n   *  used to Notify waits for single or all variables.\n   */\n  std::mutex finished_m_;\n  std::condition_variable finished_cv_;\n  /*! \\brief global exception refs, which are rethrown when WaitForAll is called */\n  std::vector<ExceptionRef> global_exception_refs_;\n\n  /*!\n   * \\brief Holding a shared_ptr to the object pool to prevent it from being destructed too early\n   * See also #309 (https://github.com/apache/mxnet/issues/309)\n   */\n  std::shared_ptr<common::ObjectPool<ThreadedOpr>> objpool_opr_ref_;\n  std::shared_ptr<common::ObjectPool<OprBlock>> objpool_blk_ref_;\n  std::shared_ptr<common::ObjectPool<VersionedVarBlock>> objpool_varblk_ref_;\n  std::shared_ptr<common::ObjectPool<ThreadedVar>> objpool_var_ref_;\n\n  /*!\n   * \\brief Async destruction of some objects is relied on storage,\n   *  prevent it from being destructed too early\n   */\n  std::shared_ptr<Storage> storage_ref_;\n\n#if MXNET_USE_CUDA\n  /*! \\brief Number of GPU devices available */\n  std::atomic<int> device_count_{-1};\n#endif\n\n  /*! \\brief Hold a ref count ot the profiler */\n  std::shared_ptr<profiler::Profiler> profiler_;\n\n  /*!\n   * \\brief Disallow copy construction and assignment.\n   * \\note This must be last\n   */\n  DISALLOW_COPY_AND_ASSIGN(ThreadedEngine);\n};  // class ThreadedEngine\n\n}  // namespace engine\n}  // namespace mxnet\n\n#endif  // MXNET_ENGINE_THREADED_ENGINE_H_\n"
  },
  {
    "path": "src/engine/threaded_engine_perdevice.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file threaded_engine_perdevice.cc\n * \\brief ThreadedEngine that uses fix amount of thread for each device.\n */\n#include <dmlc/base.h>\n#include <dmlc/omp.h>\n#include <dmlc/logging.h>\n#include <dmlc/parameter.h>\n#include <dmlc/concurrency.h>\n#include <dmlc/thread_group.h>\n\n#include <mutex>\n#include <memory>\n#include \"../initialize.h\"\n#include \"./threaded_engine.h\"\n#include \"./thread_pool.h\"\n#include \"../common/lazy_alloc_array.h\"\n#include \"../common/utils.h\"\n#include \"../common/cuda/nvtx.h\"\n\nnamespace mxnet {\nnamespace engine {\n/*!\n * \\brief ThreadedEngine uses per device threads.\n * The policy of this Engine:\n *  - Execute Async operation immediately if pushed from Pusher.\n *  - Use fixed amount of threads for each device.\n *  - Use special threads for copy operations.\n *  - Each stream is allocated and bound to each of the thread.\n */\nclass ThreadedEnginePerDevice : public ThreadedEngine {\n public:\n  static auto constexpr kFIFO          = dmlc::ConcurrentQueueType::kFIFO;\n  static auto constexpr kPriority      = dmlc::ConcurrentQueueType::kPriority;\n  static auto constexpr kCopyQueue     = kPriority;\n  static auto constexpr kPriorityQueue = kPriority;\n  static auto constexpr kWorkerQueue   = kFIFO;\n  static int constexpr kMaxStreams     = 256;\n\n  ThreadedEnginePerDevice() noexcept(false) {\n#if MXNET_USE_CUDA\n    // Make sure that the pool is not destroyed before the engine\n    objpool_gpu_sync_ref_ = common::ObjectPool<GPUWorkerSyncInfo>::_GetSharedRef();\n    streams_.reserve(kMaxStreams);\n#endif\n    this->Start();\n  }\n  ~ThreadedEnginePerDevice() noexcept(false) override {\n    this->StopNoWait();\n  }\n\n  void StopNoWait() {\n    SignalQueuesForKill();\n    gpu_normal_workers_.Clear();\n    gpu_priority_workers_.Clear();\n    gpu_copy_workers_.Clear();\n    cpu_normal_workers_.Clear();\n    cpu_priority_worker_.reset(nullptr);\n#if MXNET_USE_CUDA\n    streams_.clear();\n    cuda_event_pool_per_worker_.clear();\n#endif\n  }\n\n  void Stop() override {\n    if (is_worker_)\n      return;\n    WaitForAll();\n    StopNoWait();\n  }\n\n#if MXNET_USE_CUDA\n  void WaitForAll() override {\n    ThreadedEngine::WaitForAll();\n    for (auto s : streams_) {\n      s->Wait();\n    }\n  }\n#endif\n\n  void Start() override {\n    if (is_worker_)\n      return;\n    gpu_worker_nthreads_ = common::GetNumThreadsPerGPU();\n    // MXNET_CPU_WORKER_NTHREADS\n    cpu_worker_nthreads_ = LibraryInitializer::Get()->cpu_worker_nthreads_;\n    gpu_copy_nthreads_   = dmlc::GetEnv(\"MXNET_GPU_COPY_NTHREADS\", 2);\n    // create CPU task\n    int cpu_priority_nthreads  = dmlc::GetEnv(\"MXNET_CPU_PRIORITY_NTHREADS\", 4);\n    cpu_priority_worker_       = std::make_unique<ThreadWorkerBlock<kPriorityQueue>>();\n    cpu_priority_worker_->pool = std::make_unique<ThreadPool>(\n        cpu_priority_nthreads,\n        [this](std::shared_ptr<dmlc::ManualEvent> ready_event) {\n          this->CPUWorker(Context(), cpu_priority_worker_.get(), ready_event);\n        },\n        true);\n    // GPU tasks will be created lazily\n  }\n\n protected:\n  void PushToExecute(OprBlock* opr_block, bool pusher_thread) override {\n    const Context& ctx = opr_block->ctx;\n    if ((opr_block->opr->prop == FnProperty::kAsync ||\n         opr_block->opr->prop == FnProperty::kDeleteVar) &&\n        pusher_thread) {\n      if (ctx.dev_mask() == Context::kGPU) {\n#if MXNET_USE_CUDA\n        MSHADOW_CATCH_ERROR(mshadow::SetDevice<gpu>(ctx.dev_id));\n#endif\n      }\n      CallbackOnStart on_start = this->CreateOnStart(ThreadedEngine::OnStartStatic, opr_block);\n      CallbackOnComplete callback =\n          this->CreateCallback(ThreadedEngine::OnCompleteStatic, opr_block);\n      this->ExecuteOprBlock(RunContext{ctx, nullptr, nullptr}, opr_block, on_start, callback);\n    } else {\n      if (ctx.dev_mask() == Context::kCPU) {\n        // CPU execution.\n        if (opr_block->opr->prop == FnProperty::kCPUPrioritized) {\n          cpu_priority_worker_->task_queue.Push(opr_block, opr_block->priority);\n        } else {\n          int dev_id  = ctx.dev_id;\n          int nthread = cpu_worker_nthreads_;\n          auto ptr    = cpu_normal_workers_.Get(dev_id, [this, ctx, nthread]() {\n            auto blk  = new ThreadWorkerBlock<kWorkerQueue>();\n            blk->pool = std::make_unique<ThreadPool>(\n                nthread,\n                [this, ctx, blk](std::shared_ptr<dmlc::ManualEvent> ready_event) {\n                  this->CPUWorker(ctx, blk, ready_event);\n                },\n                true);\n            return blk;\n          });\n          if (ptr) {\n            if (opr_block->opr->prop == FnProperty::kDeleteVar) {\n              ptr->task_queue.PushFront(opr_block, opr_block->priority);\n            } else {\n              ptr->task_queue.Push(opr_block, opr_block->priority);\n            }\n          }\n        }\n      } else {\n        CHECK_EQ(ctx.dev_mask(), Context::kGPU);\n        // GPU execution.\n        const FnProperty prop = opr_block->opr->prop;\n        const bool is_copy = (prop == FnProperty::kCopyFromGPU || prop == FnProperty::kCopyToGPU);\n        if (is_copy) {\n          const size_t nthread = gpu_copy_nthreads_;\n          auto ptr             = gpu_copy_workers_.Get(ctx.dev_id, [this, ctx, is_copy, nthread]() {\n            // Signify to kernel that GPU is being used, so reserve cores as necessary\n            OpenMP::Get()->set_reserve_cores(GetReserveCoreCount(true));\n            auto blk  = new ThreadWorkerBlock<kCopyQueue>();\n            blk->pool = std::make_unique<ThreadPool>(\n                nthread,\n                [this, ctx, is_copy, blk](std::shared_ptr<dmlc::ManualEvent> ready_event) {\n                  this->GPUWorker(ctx, is_copy, blk, ready_event);\n                },\n                true);\n            return blk;\n          });\n          if (ptr) {\n            if (opr_block->opr->prop == FnProperty::kDeleteVar) {\n              ptr->task_queue.PushFront(opr_block, opr_block->priority);\n            } else {\n              ptr->task_queue.Push(opr_block, opr_block->priority);\n            }\n          }\n        } else {\n          const size_t nthread = gpu_worker_nthreads_;\n          // GPU priority task\n          if (opr_block->opr->prop == FnProperty::kGPUPrioritized) {\n            auto ptr = gpu_priority_workers_.Get(ctx.dev_id, [this, ctx, is_copy, nthread]() {\n              // Signify to kernel that GPU is being used, so reserve cores as necessary\n              OpenMP::Get()->set_reserve_cores(GetReserveCoreCount(true));\n              auto blk  = new ThreadWorkerBlock<kPriorityQueue>();\n              blk->pool = std::make_unique<ThreadPool>(\n                  nthread,\n                  [this, ctx, is_copy, blk](std::shared_ptr<dmlc::ManualEvent> ready_event) {\n                    this->GPUWorker(ctx, is_copy, blk, ready_event);\n                  },\n                  true);\n              return blk;\n            });\n            if (ptr) {\n              ptr->task_queue.Push(opr_block, opr_block->priority);\n            }\n          } else {\n            // GPU normal task\n            auto ptr = gpu_normal_workers_.Get(ctx.dev_id, [this, ctx, is_copy, nthread]() {\n              // Signify to kernel that GPU is being used, so reserve cores as necessary\n              OpenMP::Get()->set_reserve_cores(GetReserveCoreCount(true));\n              auto blk  = new ThreadWorkerBlock<kWorkerQueue>();\n              blk->pool = std::make_unique<ThreadPool>(\n                  nthread,\n                  [this, ctx, is_copy, blk](std::shared_ptr<dmlc::ManualEvent> ready_event) {\n                    this->GPUWorker(ctx, is_copy, blk, ready_event);\n                  },\n                  true);\n              return blk;\n            });\n            if (ptr) {\n              if (opr_block->opr->prop == FnProperty::kDeleteVar) {\n                ptr->task_queue.PushFront(opr_block, opr_block->priority);\n              } else {\n                ptr->task_queue.Push(opr_block, opr_block->priority);\n              }\n            }\n          }\n        }\n      }\n    }\n  }\n\n private:\n  // working unit for each of the task.\n  template <dmlc::ConcurrentQueueType type>\n  struct ThreadWorkerBlock {\n    // task queue on this task\n    dmlc::ConcurrentBlockingQueue<OprBlock*, type> task_queue;\n    // thread pool that works on this task\n    std::unique_ptr<ThreadPool> pool;\n    // constructor\n    ThreadWorkerBlock() = default;\n    // destructor\n    ~ThreadWorkerBlock() = default;\n  };\n\n  /*! \\brief whether this is a worker thread. */\n  static MX_THREAD_LOCAL bool is_worker_;\n  /*! \\brief number of concurrent thread cpu worker uses */\n  size_t cpu_worker_nthreads_;\n  /*! \\brief number of concurrent thread each gpu worker uses */\n  size_t gpu_worker_nthreads_;\n  /*! \\brief number of concurrent thread each gpu copy worker uses */\n  size_t gpu_copy_nthreads_;\n  // cpu worker\n  common::LazyAllocArray<ThreadWorkerBlock<kWorkerQueue>> cpu_normal_workers_;\n  // cpu priority worker\n  std::unique_ptr<ThreadWorkerBlock<kPriorityQueue>> cpu_priority_worker_;\n  // workers doing normal works on GPU\n  common::LazyAllocArray<ThreadWorkerBlock<kWorkerQueue>> gpu_normal_workers_;\n  // workers doing copy works from/to GPU\n  common::LazyAllocArray<ThreadWorkerBlock<kCopyQueue>> gpu_copy_workers_;\n  // gpu priority workers\n  common::LazyAllocArray<ThreadWorkerBlock<kPriorityQueue>> gpu_priority_workers_;\n#if MXNET_USE_CUDA\n  std::vector<mshadow::Stream<gpu>*> streams_;\n\n  std::unordered_map<int, std::unique_ptr<CUDAEventPool>> cuda_event_pool_per_worker_;\n#endif\n\n  /*!\n   * \\brief GPU worker that performs operations on a certain device.\n   * \\param dev_id The device id of the worker.\n   * \\param is_copy_worker whether the worker only do copy job\n   * \\param block The task block of the worker.\n   */\n  template <dmlc::ConcurrentQueueType type>\n  inline void GPUWorker(Context ctx,\n                        bool is_copy_worker,\n                        ThreadWorkerBlock<type>* block,\n                        const std::shared_ptr<dmlc::ManualEvent>& ready_event) {\n    this->is_worker_ = true;\n#if MXNET_USE_CUDA\n    CHECK(block != nullptr);\n    mshadow::Stream<gpu>* stream = nullptr;\n    GPUAuxStream* aux_stream     = nullptr;\n    CUDAEventPool* event_pool    = nullptr;\n    do {\n      ThreadPool::SetReadyOnDestroy setReady(ready_event);\n      // allocate stream\n      mshadow::SetDevice<gpu>(ctx.dev_id);\n      if (is_copy_worker) {\n        stream = mshadow::NewStream<gpu>(false, false, ctx.dev_id);\n      } else {\n        stream     = mshadow::NewStream<gpu>(true, MXNET_USE_CUDNN != 0, ctx.dev_id);\n        aux_stream = new GPUAuxStream(stream);\n      }\n      // With thread safety...\n      {\n        static std::mutex m;\n        std::lock_guard<std::mutex> lock(m);\n        // register stream\n        streams_.push_back(stream);\n        auto event_pool_it = cuda_event_pool_per_worker_.find(ctx.dev_id);\n        if (event_pool_it != cuda_event_pool_per_worker_.end()) {\n          event_pool = event_pool_it->second.get();\n        } else {\n          auto res =\n              cuda_event_pool_per_worker_.emplace(ctx.dev_id, std::make_unique<CUDAEventPool>(ctx));\n          event_pool = res.first->second.get();\n        }\n      }\n    } while (false);\n    // execute task\n    OprBlock* opr_block;\n    RunContext run_ctx{ctx, stream, aux_stream};\n    auto* task_queue = &(block->task_queue);\n\n    // Don't eat up omp threads for GPU jobs.  They're probably best used elsewhere,\n    // for example for image decoding or the optimizer pass\n    OpenMP::Get()->on_start_worker_thread(false);\n\n    while (task_queue->Pop(&opr_block)) {\n#if MXNET_USE_NVTX\n      auto nvtx_name       = opr_block->opr->opr_name != \"\" ? opr_block->opr->opr_name : \"Op\";\n      auto end_pos         = nvtx_name.find('{');\n      auto name_prefix_len = end_pos != std::string::npos ? end_pos : nvtx_name.size();\n      auto color           = common::cuda::nvtx::nameToColor(nvtx_name, name_prefix_len);\n      common::cuda::nvtx::gpuRangeStart(color, nvtx_name);\n#endif\n      auto* info                  = ThreadedEngine::GPUWorkerSyncInfo::New();\n      info->opr_block             = opr_block;\n      info->stream                = stream;\n      info->event_pool            = event_pool;\n      CallbackOnStart on_start    = this->CreateOnStart(ThreadedEngine::OnStartGPU, info);\n      CallbackOnComplete callback = this->CreateCallback(ThreadedEngine::OnCompleteGPU, info);\n      this->ExecuteOprBlock(run_ctx, opr_block, on_start, callback);\n#if MXNET_USE_NVTX\n      common::cuda::nvtx::gpuRangeStop();\n#endif\n    }\n#else\n    ready_event->signal();\n#endif\n  }\n  /*!\n   * \\brief CPU worker that performs operations on CPU.\n   * \\param block The task block of the worker.\n   */\n  template <dmlc::ConcurrentQueueType type>\n  inline void CPUWorker(Context ctx,\n                        ThreadWorkerBlock<type>* block,\n                        const std::shared_ptr<dmlc::ManualEvent>& ready_event) {\n    this->is_worker_ = true;\n    auto* task_queue = &(block->task_queue);\n    RunContext run_ctx{ctx, nullptr, nullptr};\n\n    // execute task\n    OprBlock* opr_block;\n    ready_event->signal();\n\n    // Set default number of threads for OMP parallel regions initiated by this thread\n    OpenMP::Get()->on_start_worker_thread(true);\n\n    while (task_queue->Pop(&opr_block)) {\n#if MXNET_USE_CUDA\n      CallbackOnStart on_start = this->CreateOnStart(ThreadedEngine::OnStartCPU, opr_block);\n#else\n      CallbackOnStart on_start = this->CreateOnStart(ThreadedEngine::OnStartStatic, opr_block);\n#endif\n      CallbackOnComplete callback =\n          this->CreateCallback(ThreadedEngine::OnCompleteStatic, opr_block);\n      this->ExecuteOprBlock(run_ctx, opr_block, on_start, callback);\n    }\n  }\n\n  /*!\n   * \\brief Get number of cores this engine should reserve for its own use\n   * \\param using_gpu Whether there is GPU usage\n   * \\return number of cores that this engine wishes to be reserved\n   * \\note Testing found no degradation of performance using these values\n   *       running cifar10 with resnet50 on various GPU systems,\n   *       including AWS p2.16xlarge, which has 16 GPU's\n   */\n  int GetReserveCoreCount(const bool using_gpu) const {\n    int reserve = 0;\n    if (using_gpu) {\n      // Save at least one for GPU tasks\n      ++reserve;\n      // If we have 8 or more real cores, reserve another core for GPU tasks\n      if (OpenMP::Get()->GetRecommendedOMPThreadCount(true) >= 8) {\n        ++reserve;\n      }\n    }\n    return reserve;\n  }\n\n  /*! \\brief Signal a single queue for shutdown */\n  template <typename Object>\n  static inline void SignalQueueForKill(common::LazyAllocArray<Object>* array) {\n    array->ForEach([](size_t i, Object* block) { block->task_queue.SignalForKill(); });\n  }\n\n  /*! Signal all queues for shutdown */\n  void SignalQueuesForKill() {\n    SignalQueueForKill(&gpu_priority_workers_);\n    SignalQueueForKill(&gpu_normal_workers_);\n    SignalQueueForKill(&gpu_copy_workers_);\n    SignalQueueForKill(&cpu_normal_workers_);\n    if (cpu_priority_worker_) {\n      cpu_priority_worker_->task_queue.SignalForKill();\n    }\n  }\n};\n\nEngine* CreateThreadedEnginePerDevice() {\n  return new ThreadedEnginePerDevice();\n}\n\nMX_THREAD_LOCAL bool ThreadedEnginePerDevice::is_worker_ = false;\n\n}  // namespace engine\n}  // namespace mxnet\n"
  },
  {
    "path": "src/engine/threaded_engine_pooled.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file threaded_engine_pooled.cc\n * \\brief Pooled threaded engine\n * \\author Yutian Li\n */\n#include <dmlc/base.h>\n#include <dmlc/logging.h>\n#include <dmlc/concurrency.h>\n#include <cassert>\n#include <memory>\n#include <utility>\n#include \"./threaded_engine.h\"\n#include \"./thread_pool.h\"\n#include \"./stream_manager.h\"\n#if MXNET_USE_CUDA\n#include \"../common/cuda/utils.h\"\n#endif\n\nnamespace mxnet {\nnamespace engine {\n/*!\n * \\brief ThreadedEngine using global thread pool across all devices.\n * The policy of this Engine:\n *  - Execute Async operation immediately if pushed from Pusher.\n *  - Use a common thread pool for normal operations on all devices.\n *  - Use special thread pool for copy operations.\n */\nclass ThreadedEnginePooled : public ThreadedEngine {\n public:\n  ThreadedEnginePooled() {\n#if MXNET_USE_CUDA\n    // Make sure that the pool is not destroyed before the engine\n    objpool_gpu_sync_ref_ = common::ObjectPool<ThreadedEngine::GPUWorkerSyncInfo>::_GetSharedRef();\n#endif\n    this->Start();\n  }\n\n  ~ThreadedEnginePooled() noexcept(false) override {\n    StopNoWait();\n  }\n\n  void StopNoWait() {\n    task_queue_->SignalForKill();\n    io_task_queue_->SignalForKill();\n    task_queue_     = nullptr;\n    io_task_queue_  = nullptr;\n    thread_pool_    = nullptr;\n    io_thread_pool_ = nullptr;\n    streams_->Finalize();\n    streams_ = nullptr;\n  }\n\n  void Stop() override {\n    WaitForAll();\n    StopNoWait();\n  }\n\n  void Start() override {\n    streams_ = std::make_unique<StreamManager<kMaxNumGpus, kNumStreamsPerGpu>>();\n    task_queue_.reset(new dmlc::ConcurrentBlockingQueue<OprBlock*>());\n    io_task_queue_.reset(new dmlc::ConcurrentBlockingQueue<OprBlock*>());\n    thread_pool_ = std::make_unique<ThreadPool>(\n        kNumWorkingThreads,\n        [this](std::shared_ptr<dmlc::ManualEvent> ready_event) {\n          ThreadWorker(task_queue_, ready_event);\n        },\n        true);\n    io_thread_pool_ = std::make_unique<ThreadPool>(\n        1,\n        [this](std::shared_ptr<dmlc::ManualEvent> ready_event) {\n          ThreadWorker(io_task_queue_, ready_event);\n        },\n        true);\n  }\n\n protected:\n  void PushToExecute(OprBlock* opr_block, bool pusher_thread) override {\n    if (opr_block->opr->prop == FnProperty::kAsync && pusher_thread) {\n      DoExecute(opr_block);\n    } else {\n      DoPushToQueue(opr_block);\n    }\n  }\n\n private:\n  /*! \\brief Concurrency for thread pool */\n  static constexpr std::size_t kNumWorkingThreads = 16;\n  /*! \\brief Maximum number of GPUs */\n  static constexpr std::size_t kMaxNumGpus = 16;\n  /*!\\brief number of streams allocated for each GPU */\n  static constexpr std::size_t kNumStreamsPerGpu = 16;\n  /*!\n   * \\brief Streams.\n   */\n  std::unique_ptr<StreamManager<kMaxNumGpus, kNumStreamsPerGpu>> streams_;\n  /*!\n   * \\brief Task queues.\n   */\n  std::shared_ptr<dmlc::ConcurrentBlockingQueue<OprBlock*>> task_queue_;\n  std::shared_ptr<dmlc::ConcurrentBlockingQueue<OprBlock*>> io_task_queue_;\n  /*!\n   * \\brief Thread pools.\n   */\n  std::unique_ptr<ThreadPool> thread_pool_;\n  std::unique_ptr<ThreadPool> io_thread_pool_;\n  /*!\n   * \\brief Worker.\n   * \\param task_queue Queue to work on.\n   *\n   * The method to pass to thread pool to parallelize.\n   */\n  void ThreadWorker(std::shared_ptr<dmlc::ConcurrentBlockingQueue<OprBlock*>> task_queue,\n                    const std::shared_ptr<dmlc::ManualEvent>& ready_event) {\n    OprBlock* opr_block;\n    ready_event->signal();\n    while (task_queue->Pop(&opr_block)) {\n      DoExecute(opr_block);\n    }\n  }\n  /*!\n   * \\brief Execute an operation.\n   * \\param opr_block The operator block.\n   */\n  void DoExecute(OprBlock* opr_block) {\n#if MXNET_USE_CUDA\n    mxnet::common::cuda::DeviceStore device_store(-1, false);\n#endif\n    assert(opr_block->wait.load() == 0);\n    if (opr_block->ctx.dev_mask() == gpu::kDevMask) {\n#if MXNET_USE_CUDA\n      device_store.SetDevice(opr_block->ctx.dev_id);\n#else   // MXNET_USE_CUDA\n      LOG(FATAL) << \"Please compile with CUDA enabled\";\n#endif  // MXNET_USE_CUDA\n    }\n    bool is_copy = (opr_block->opr->prop == FnProperty::kCopyFromGPU ||\n                    opr_block->opr->prop == FnProperty::kCopyToGPU);\n    auto&& rctx  = is_copy ? streams_->GetIORunContext(opr_block->ctx) :\n                            streams_->GetRunContext(opr_block->ctx);\n#if MXNET_USE_CUDA\n    CallbackOnStart on_start;\n    CallbackOnComplete callback;\n    if (opr_block->ctx.dev_mask() == Context::kCPU) {\n      on_start = this->CreateOnStart(ThreadedEngine::OnStartCPU, opr_block);\n      callback = this->CreateCallback(ThreadedEngine::OnCompleteStatic, opr_block);\n    } else {\n      CHECK_EQ(opr_block->ctx.dev_mask(), Context::kGPU);\n      auto stream      = rctx.get_stream<gpu>();\n      auto event_pool  = static_cast<CUDAEventPool*>(rctx.event_pool);\n      auto* info       = ThreadedEngine::GPUWorkerSyncInfo::New();\n      info->opr_block  = opr_block;\n      info->stream     = stream;\n      info->event_pool = event_pool;\n      on_start         = this->CreateOnStart(ThreadedEngine::OnStartGPU, info);\n      callback         = this->CreateCallback(ThreadedEngine::OnCompleteGPU, info);\n    }\n#else   // MXNET_USE_CUDA\n    CallbackOnStart on_start = this->CreateOnStart(ThreadedEngine::OnStartStatic, opr_block);\n    CallbackOnComplete callback = this->CreateCallback(ThreadedEngine::OnCompleteStatic, opr_block);\n#endif  // MXNET_USE_CUDA\n    this->ExecuteOprBlock(rctx, opr_block, on_start, callback);\n  }\n  /*!\n   * \\brief Push the operation to the queue.\n   * \\param opr_block The operator block.\n   */\n  void DoPushToQueue(OprBlock* opr_block) {\n    switch (opr_block->opr->prop) {\n      case FnProperty::kCopyFromGPU:\n      case FnProperty::kCopyToGPU: {\n        io_task_queue_->Push(opr_block);\n        break;\n      }\n      default: {\n        task_queue_->Push(opr_block);\n        break;\n      }\n    }\n  }\n};\n\nEngine* CreateThreadedEnginePooled() {\n  return new ThreadedEnginePooled();\n}\n}  // namespace engine\n}  // namespace mxnet\n"
  },
  {
    "path": "src/imperative/attach_op_execs_pass.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file attach_op_execs_pass.cc\n * \\brief Operator executor to execute each operator.\n */\n#include <mxnet/base.h>\n#include <mxnet/op_attr_types.h>\n#include <mxnet/graph_attr_types.h>\n#include <nnvm/graph_attr_types.h>\n\n#include <utility>\n#include \"../common/utils.h\"\n#include \"../common/exec_utils.h\"\n#include \"../imperative/imperative_utils.h\"\n\nnamespace mxnet {\n\nnamespace exec {\n\n#if MXNET_USE_ONEDNN == 1\n#define CREATE_DEFAULT_INPUTS_DNNL(in_array, in_array_fallback, attrs) \\\n  CREATE_DEFAULT_INPUTS(true, attrs, CreateDefaultInputs(in_array, in_array_fallback))\n#else\n#define CREATE_DEFAULT_INPUTS_DNNL(in_array, in_array_fallback, attrs)  // empty macro\n#endif\n\n// abstract OpExecutor which provides storage fallback procedure on\n// non-default inputs and outputs\n// FComputeExecutor and FStatefulComputeExecutor inherit from this class\nclass StorageFallbackOpExecutor : public OpExecutor {\n public:\n  explicit StorageFallbackOpExecutor(const NodeAttrs& attrs,\n                                     DispatchMode dispatch_mode,\n                                     std::vector<uint32_t> mutate_idx)\n      : OpExecutor(attrs, dispatch_mode), mutate_idx_(std::move(mutate_idx)) {}\n\n  void Setup() override {\n    init_ = false;\n  }\n\n protected:\n  // initialize the data blobs\n  void InitBlobs() {\n    if (!init_) {\n      pre_temp_buf_.clear();\n      post_temp_buf_.clear();\n      for (const auto& nd : in_array) {\n        pre_temp_buf_.emplace_back(nd.shape(), nd.ctx(), true, nd.dtype());\n      }\n      for (const auto& nd : out_array) {\n        post_temp_buf_.emplace_back(nd.shape(), nd.ctx(), true, nd.dtype());\n      }\n      init_ = true;\n    }\n  }\n\n  // storage fallback before fcompute is launched\n  void PreFCompute(bool is_gpu) {\n    using namespace common;\n    InitBlobs();\n    in_data_.clear();\n    out_data_.clear();\n    pre_temp_src_.clear();\n    pre_temp_dst_.clear();\n    post_temp_src_.clear();\n    post_temp_dst_.clear();\n    in_temp_idx_map_.clear();\n    tmp_req = req;\n    SetupDefaultBlobsInOut(in_array,\n                           out_array,\n                           &pre_temp_buf_,\n                           &post_temp_buf_,\n                           &req,\n                           &in_data_,\n                           &out_data_,\n                           &pre_temp_src_,\n                           &pre_temp_dst_,\n                           &post_temp_src_,\n                           &post_temp_dst_,\n                           &in_temp_idx_map_,\n                           mutate_idx_);\n    common::CastNonDefaultStorage(pre_temp_src_, pre_temp_dst_, op_ctx, is_gpu);\n  }\n\n  // storage fallback after fcompute is completed\n  void PostFCompute(bool is_gpu) {\n    common::CastNonDefaultStorage(post_temp_src_, post_temp_dst_, op_ctx, is_gpu);\n    req = tmp_req;\n  }\n\n  // output requirement on each output array.\n  // This temporarily saves the original output requirements.\n  std::vector<OpReqType> tmp_req;\n  // default storage tensor blobs for fcompute\n  std::vector<TBlob> in_data_, out_data_;\n  // These are NDArray buffers for cast storage.\n  std::vector<NDArray> pre_temp_buf_, post_temp_buf_;\n  // source NDArray for cast storage\n  std::vector<NDArray> pre_temp_src_, post_temp_src_;\n  // destination NDArray for cast storage\n  std::vector<NDArray> pre_temp_dst_, post_temp_dst_;\n  // mapping from index in input_blobs to index in pre_temp_dst\n  std::unordered_map<uint32_t, uint32_t> in_temp_idx_map_;\n  // indices of mutatable inputs\n  std::vector<uint32_t> mutate_idx_;\n  // whether blobs are initialized\n  bool init_;\n};\n\n// stateful compute executor\nclass StatefulComputeExecutor : public StorageFallbackOpExecutor {\n public:\n  void Run(RunContext rctx, bool is_gpu) override {\n    op_ctx.run_ctx = rctx;\n    INVALIDATE_OUTPUTS(out_array, req);\n    PreFCompute(is_gpu);\n    fcompute_(state_, op_ctx, in_data_, req, out_data_);\n    PostFCompute(is_gpu);\n  }\n\n  ExecType exec_type() const override {\n    return exec_type_;\n  }\n\n  engine::VarHandle var() const override {\n    return state_.get_var();\n  }\n\n  OpStatePtr state() const override {\n    return state_;\n  }\n\n  explicit StatefulComputeExecutor(const NodeAttrs& attrs,\n                                   DispatchMode dispatch_mode,\n                                   OpStatePtr state,\n                                   FStatefulCompute fcompute,\n                                   ExecType exec_type,\n                                   const std::vector<uint32_t>& mutate_idx)\n      : StorageFallbackOpExecutor(attrs, dispatch_mode, mutate_idx),\n        state_(std::move(state)),\n        fcompute_(std::move(fcompute)),\n        exec_type_(exec_type) {}\n\n private:\n  OpStatePtr state_;\n  FStatefulCompute fcompute_;\n  ExecType exec_type_;\n};\n\n// stateful compute_ex executor\nclass StatefulComputeExExecutor : public OpExecutor {\n public:\n  void Run(RunContext rctx, bool is_gpu) override {\n    op_ctx.run_ctx = rctx;\n    INVALIDATE_OUTPUTS(out_array, req);\n    std::vector<NDArray>* pInArray = &in_array;\n    CREATE_DEFAULT_INPUTS_DNNL(in_array, pInArray = &in_array_fallback, attrs);\n    fcompute_(state_, op_ctx, *pInArray, req, out_array);\n  }\n\n  void Setup() override {}\n\n  ExecType exec_type() const override {\n    return exec_type_;\n  }\n\n  engine::VarHandle var() const override {\n    return state_.get_var();\n  }\n\n  OpStatePtr state() const override {\n    return state_;\n  }\n\n  explicit StatefulComputeExExecutor(const NodeAttrs& attrs,\n                                     DispatchMode dispatch_mode,\n                                     OpStatePtr state,\n                                     FStatefulComputeEx fcompute,\n                                     ExecType exec_type)\n      : OpExecutor(attrs, dispatch_mode),\n        state_(std::move(state)),\n        fcompute_(std::move(fcompute)),\n        exec_type_(exec_type) {}\n\n private:\n  OpStatePtr state_;\n  FStatefulComputeEx fcompute_;\n  ExecType exec_type_;\n};\n\n// fcompute executor\nclass FComputeExecutor : public StorageFallbackOpExecutor {\n public:\n  void Run(RunContext rctx, bool is_gpu) override {\n    using namespace common;\n    op_ctx.run_ctx = rctx;\n    INVALIDATE_OUTPUTS(out_array, req);\n    PreFCompute(is_gpu);\n    fcompute_(attrs, op_ctx, in_data_, req, out_data_);\n    PostFCompute(is_gpu);\n  }\n\n  ExecType exec_type() const override {\n    return exec_type_;\n  }\n\n  explicit FComputeExecutor(const NodeAttrs& attrs,\n                            DispatchMode dispatch_mode,\n                            FCompute fcompute,\n                            ExecType exec_type,\n                            const std::vector<uint32_t>& mutate_idx)\n      : StorageFallbackOpExecutor(attrs, dispatch_mode, mutate_idx),\n        fcompute_(std::move(fcompute)),\n        exec_type_(exec_type) {}\n\n private:\n  FCompute fcompute_;\n  ExecType exec_type_;\n};\n\n// fcompute_ex executor\nclass FComputeExExecutor : public OpExecutor {\n public:\n  void Run(RunContext rctx, bool is_gpu) override {\n    op_ctx.run_ctx = rctx;\n    INVALIDATE_OUTPUTS(out_array, req);\n    std::vector<NDArray>* pInArray = &in_array;\n    CREATE_DEFAULT_INPUTS_DNNL(in_array, pInArray = &in_array_fallback, attrs);\n    fcompute_(attrs, op_ctx, *pInArray, req, out_array);\n  }\n\n  void Setup() override {}\n\n  ExecType exec_type() const override {\n    return exec_type_;\n  }\n\n  explicit FComputeExExecutor(const NodeAttrs& attrs,\n                              DispatchMode dispatch_mode,\n                              FComputeEx fcompute,\n                              ExecType exec_type)\n      : OpExecutor(attrs, dispatch_mode), fcompute_(std::move(fcompute)), exec_type_(exec_type) {}\n\n private:\n  FComputeEx fcompute_;\n  ExecType exec_type_;\n};\n\nvoid CreateOpExecs(const Graph& g, OpExecVector* p_ret, OpStateVector* p_state, size_t i) {\n  using mxnet::ShapeVector;\n  using nnvm::DTypeVector;\n  using nnvm::FMutateInputs;\n\n  static auto& fcreate_op_state  = nnvm::Op::GetAttr<FCreateOpState>(\"FCreateOpState\");\n  static auto& fmutate_inputs    = nnvm::Op::GetAttr<FMutateInputs>(\"FMutateInputs\");\n  static auto& fexec_type        = nnvm::Op::GetAttr<FExecType>(\"FExecType\");\n  static auto& is_layer_backward = nnvm::Op::GetAttr<bool>(\"TIsLayerOpBackward\");\n\n  const auto& vdtype         = g.GetAttr<DTypeVector>(\"dtype\");\n  const auto& vshape         = g.GetAttr<mxnet::ShapeVector>(\"shape\");\n  const auto& vctx           = g.GetAttr<ContextVector>(\"context\");\n  const auto& dispatch_modes = g.GetAttr<DispatchModeVector>(\"dispatch_mode\");\n  // get the graph\n  const auto& idx   = g.indexed_graph();\n  OpExecVector& ret = *p_ret;\n\n  // initialize the nodes\n  const auto& inode = idx[i];\n  if (inode.source->is_variable())\n    return;\n  const nnvm::Op* op = inode.source->op();\n  ExecType exec_type = ExecType::kSync;\n  std::vector<uint32_t> mutate_index;\n  if (fmutate_inputs.count(op)) {\n    mutate_index = fmutate_inputs[op](inode.source->attrs);\n  }\n  if (fexec_type.count(op)) {\n    exec_type = fexec_type[op](inode.source->attrs);\n  }\n  CHECK(dispatch_modes[i] != DispatchMode::kUndefined);\n  if (fcreate_op_state.count(op)) {\n    mxnet::ShapeVector ishape;\n    std::vector<int> itype;\n    for (const auto& e : inode.inputs) {\n      ishape.emplace_back(vshape[idx.entry_id(e)]);\n      itype.emplace_back(vdtype[idx.entry_id(e)]);\n    }\n\n    OpStatePtr state = fcreate_op_state[op](inode.source->attrs, vctx[i], ishape, itype);\n    if (p_state) {\n      CHECK_GT(p_state->size(), i);\n      p_state->at(i) = state;\n    }\n    FStatefulComputeEx fcompute_ex =\n        common::GetFCompute<FStatefulComputeEx>(op, \"FStatefulComputeEx\", vctx[i]);\n    // FStatefulComputeEx is dispatched only when dispatch_mode is DispatchMode::kFComputeEx\n    if (fcompute_ex != nullptr && dispatch_modes[i] == DispatchMode::kFComputeEx) {\n      ret[i] = std::make_shared<StatefulComputeExExecutor>(\n          inode.source->attrs, dispatch_modes[i], state, fcompute_ex, exec_type);\n    } else {\n      FStatefulCompute fcompute =\n          common::GetFCompute<FStatefulCompute>(op, \"FStatefulCompute\", vctx[i]);\n      CHECK(fcompute != nullptr)\n          << \"One of FStatefulCompute and FStatefulComputeEx must be registered \"\n          << \"for stateful operator \" << op->name;\n      ret[i] = std::make_shared<StatefulComputeExecutor>(\n          inode.source->attrs, dispatch_modes[i], state, fcompute, exec_type, mutate_index);\n    }\n  } else if (is_layer_backward.get(op, false)) {\n    CHECK_GE(inode.control_deps.size(), 1);\n    uint32_t fwd_id = inode.control_deps[0];\n    CHECK(vctx[fwd_id] == vctx[i]);\n    CHECK(ret[fwd_id] != nullptr);\n    FStatefulComputeEx fcompute_ex =\n        common::GetFCompute<FStatefulComputeEx>(op, \"FStatefulComputeEx\", vctx[i]);\n    // FStatefulComputeEx is dispatched only when dispatch_mode is DispatchMode::kFComputeEx\n    if (fcompute_ex != nullptr && dispatch_modes[i] == DispatchMode::kFComputeEx) {\n      ret[i] = std::make_shared<StatefulComputeExExecutor>(inode.source->attrs,\n                                                           dispatch_modes[i],\n                                                           ret[fwd_id].get()->state(),\n                                                           fcompute_ex,\n                                                           exec_type);\n    } else {\n      FStatefulCompute fcompute =\n          common::GetFCompute<FStatefulCompute>(op, \"FStatefulCompute\", vctx[i]);\n      CHECK(fcompute != nullptr)\n          << \"One of FStatefulCompute and FStatefulComputeEx must be registered \"\n          << \"for stateful operator \" << op->name;\n      ret[i] = std::make_shared<StatefulComputeExecutor>(inode.source->attrs,\n                                                         dispatch_modes[i],\n                                                         ret[fwd_id].get()->state(),\n                                                         fcompute,\n                                                         exec_type,\n                                                         mutate_index);\n    }\n  } else {\n    FCompute fcompute   = common::GetFCompute<FCompute>(op, \"FCompute\", vctx[i]);\n    FComputeEx fcomp_ex = common::GetFCompute<FComputeEx>(op, \"FComputeEx\", vctx[i]);\n    if (fcomp_ex != nullptr && dispatch_modes[i] == DispatchMode::kFComputeEx) {\n      ret[i] = std::make_shared<FComputeExExecutor>(\n          inode.source->attrs, dispatch_modes[i], fcomp_ex, exec_type);\n    } else if (fcompute != nullptr) {\n      ret[i] = std::make_shared<FComputeExecutor>(\n          inode.source->attrs, dispatch_modes[i], fcompute, exec_type, mutate_index);\n    } else {\n      LOG(INFO) << \"Neither FCompute nor FComputeEx registered \" << op->name;\n    }\n  }\n}\n\n// pass to attach operator executors\nGraph AttachOpExecs(Graph g) {\n  const auto& idx = g.indexed_graph();\n  OpExecVector ret(idx.num_nodes());\n  for (size_t i = 0; i < idx.num_nodes(); ++i) {\n    CreateOpExecs(g, &ret, nullptr, i);\n  }\n  g.attrs[\"op_execs\"] = std::make_shared<nnvm::any>(ret);\n  return g;\n}\n\n}  // namespace exec\n}  // namespace mxnet\n"
  },
  {
    "path": "src/imperative/attach_op_resource_pass.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file attach_op_resource_pass.cc\n * \\brief Pass to attach resource to OpExecVector of the graph.\n */\n#include <mxnet/resource.h>\n#include <mxnet/op_attr_types.h>\n#include \"./exec_pass.h\"\n\nnamespace mxnet {\nnamespace exec {\n\nvoid AttachOpResources(const Graph& g,\n                       const OpExecVector& op_execs,\n                       size_t start_nid,\n                       size_t end_nid) {\n  static auto& fresource    = nnvm::Op::GetAttr<FResourceRequest>(\"FResourceRequest\");\n  static auto& fresource_ex = nnvm::Op::GetAttr<FResourceRequestEx>(\"FResourceRequestEx\");\n  const auto& vctx          = g.GetAttr<ContextVector>(\"context\");\n  const auto& vdispatch     = g.GetAttr<DispatchModeVector>(\"dispatch_mode\");\n  const auto& dev_masks     = g.GetAttr<DevMaskVector>(\"dev_mask\");\n  const auto& idx           = g.indexed_graph();\n  // Use global resource pool for each executor for now.\n  std::map<Context, Resource> cached_temp;\n  // Resource allocation\n  for (uint32_t nid = start_nid; nid < end_nid; ++nid) {\n    const auto& inode = idx[nid];\n    if (inode.source->is_variable())\n      continue;\n    const Context& ctx = vctx[nid];\n    auto& requested    = op_execs[nid]->op_ctx.requested;\n    requested.clear();\n    const auto op         = inode.source->op();\n    const bool rsc_req    = (fresource.count(op) != 0);\n    const bool rsc_ex_req = (fresource_ex.count(op) != 0);\n    if (rsc_req || rsc_ex_req) {\n      auto reqs = rsc_ex_req ?\n                      fresource_ex[op](inode.source->attrs, dev_masks[nid], vdispatch[nid]) :\n                      fresource[op](inode.source->attrs);\n      // Get the resource of temporal space.\n      for (const ResourceRequest& req : reqs) {\n        switch (req.type) {\n          case ResourceRequest::kTempSpace: {\n            // the scope is needed when there's new declaration of variable.\n            if (cached_temp.count(ctx) != 0) {\n              requested.push_back(cached_temp.at(ctx));\n            } else {\n              Resource r = ResourceManager::Get()->Request(ctx, req);\n              requested.push_back(r);\n              cached_temp[ctx] = r;\n            }\n            break;\n          }\n          case ResourceRequest::kRandom: {\n            requested.push_back(ResourceManager::Get()->Request(ctx, req));\n            break;\n          }\n          case ResourceRequest::kParallelRandom: {\n            requested.push_back(ResourceManager::Get()->Request(ctx, req));\n            break;\n          }\n#if MXNET_USE_CUDNN == 1\n          case ResourceRequest::kCuDNNDropoutDesc: {\n            requested.push_back(ResourceManager::Get()->Request(ctx, req));\n            break;\n          }\n#endif  // MXNET_USE_CUDNN == 1\n          default:\n            LOG(FATAL) << \"resource type \" << req.type << \" is not yet supported\";\n        }\n      }\n      CHECK(vdispatch[nid] != DispatchMode::kUndefined);\n    }\n    // extra resource requests for storage fallback\n    if (vdispatch[nid] == DispatchMode::kFComputeFallback) {\n      requested.push_back(ResourceManager::Get()->Request(ctx, ResourceRequest::kTempSpace));\n    }\n  }\n}\n\nvoid AttachOpResources(const Graph& g) {\n  const auto& op_execs = g.GetAttr<OpExecVector>(\"op_execs\");\n  AttachOpResources(g, op_execs, 0, g.indexed_graph().num_nodes());\n}\n\n}  // namespace exec\n}  // namespace mxnet\n"
  },
  {
    "path": "src/imperative/cached_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n#include <memory>\n#include <unordered_set>\n#include <iostream>\n#include \"./imperative_utils.h\"\n#include \"./cached_op.h\"\n#include \"./exec_pass.h\"\n#include \"../profiler/profiler.h\"\n#include \"../operator/operator_common.h\"\n#include \"../operator/subgraph/common.h\"\n\nnamespace mxnet {\n\nDMLC_REGISTER_PARAMETER(CachedOpConfig);\n\nconstexpr uint32_t kEidNotExist = std::numeric_limits<uint32_t>::max();\n\nnnvm::Symbol CachedOp::GetOptimizedSymbol() const {\n  nnvm::Symbol ret;\n  ret.outputs = std::vector<nnvm::NodeEntry>(full_graph_.outputs.begin(),\n                                             full_graph_.outputs.begin() + num_outputs());\n  return ret.Copy();\n}\n\nCachedOp::CachedOp(const nnvm::Symbol& sym,\n                   const std::vector<std::pair<std::string, std::string> >& flags)\n    : sym_(sym), flags_(flags) {\n  config_.Init(flags);\n  this->dynamic_shape_checked_ = false;\n\n  if (config_.static_shape) {\n    CHECK(config_.static_alloc) << \"static_alloc must be True when static_shape is True\";\n  }\n\n  auto grad_graph = nnvm::Graph();\n  std::unordered_map<uint32_t, uint32_t> fwd_input_to_grad_output;\n  CreateFullGraph(sym.Copy(),\n                  &fwd_graph_,\n                  &grad_graph,\n                  &full_graph_,\n                  &ograd_entries_,\n                  &fwd_input_to_grad_output);\n\n  {\n    const auto& idx  = fwd_graph_.indexed_graph();\n    bwd_output_reqs_ = std::vector<OpReqType>(grad_graph.outputs.size(), kWriteTo);\n    inlining_        = !config_.static_alloc &&\n                (idx.num_nodes() - idx.input_nodes().size()) <= config_.inline_limit;\n  }\n\n  SetInputIndices(fwd_graph_, config_.param_indices, &config_.data_indices);\n\n  // Set the backward dependency vectors\n  {\n    const auto& idx            = full_graph_.indexed_graph();\n    size_t num_forward_inputs  = num_inputs();\n    size_t num_forward_outputs = num_outputs();\n    for (uint32_t i = 0; i < ograd_entries_.size(); ++i) {\n      if (!idx.exist(ograd_entries_[i].node.get()))\n        continue;\n      bwd_ograd_dep_.push_back(i);\n    }\n    save_inputs_.resize(num_forward_inputs, false);\n    for (uint32_t i = 0; i < num_forward_inputs; ++i) {\n      save_inputs_[i] = true;\n      bwd_in_dep_.push_back(i);\n    }\n    save_outputs_.resize(idx.outputs().size(), false);\n    for (uint32_t i = 0; i < num_forward_outputs; ++i) {\n      save_outputs_[i] = true;\n      bwd_out_dep_.push_back(i);\n    }\n  }\n\n  SetRefCounts(&fwd_graph_, full_graph_);\n}\n\nCachedOp::~CachedOp() = default;\n\nstd::vector<nnvm::NodeEntry> CachedOp::Gradient(const nnvm::ObjectPtr& node,\n                                                const std::vector<nnvm::NodeEntry>& ograds) const {\n  using namespace nnvm;\n  static const auto _backward_CachedOp = Op::Get(\"_backward_CachedOp\");\n  static const auto _NoGrad            = Op::Get(\"_NoGradient\");\n\n  auto p          = Node::Create();\n  p->attrs.op     = _backward_CachedOp;\n  p->attrs.name   = node->attrs.name + \"_backward\";\n  p->attrs.parsed = node->attrs.parsed;\n  p->control_deps.push_back(node);\n  p->inputs.reserve(bwd_ograd_dep_.size() + bwd_in_dep_.size() + bwd_out_dep_.size());\n  for (auto i : bwd_ograd_dep_)\n    p->inputs.push_back(ograds[i]);\n  for (auto i : bwd_in_dep_)\n    p->inputs.push_back(node->inputs[i]);\n  for (auto i : bwd_out_dep_)\n    p->inputs.emplace_back(node, i, 0);\n  std::vector<NodeEntry> ret;\n  ret.reserve(num_inputs());\n  const auto& auxs = mutable_input_nodes();\n  if (auxs.size()) {\n    auto nop        = Node::Create();\n    nop->attrs.op   = _NoGrad;\n    nop->attrs.name = \"NoGradient\";\n    uint32_t k      = 0;\n    for (const auto& i : fwd_graph_.indexed_graph().input_nodes()) {\n      if (auxs.count(i)) {\n        ret.emplace_back(nop);\n      } else {\n        ret.emplace_back(p, k++, 0);\n      }\n    }\n  } else {\n    for (uint32_t i = 0; i < num_inputs(); ++i)\n      ret.emplace_back(p, i, 0);\n  }\n  return ret;\n}\n\nbool CachedOp::CheckDynamicShapeExists(const Context& default_ctx,\n                                       const std::vector<NDArray*>& inputs,\n                                       bool erase_result) {\n  using namespace nnvm;\n  using namespace imperative;\n  if (this->dynamic_shape_checked_) {\n    return config_.is_dynamic;\n  } else {\n    this->dynamic_shape_checked_ = true;\n  }\n  CHECK_EQ(inputs.size(), num_inputs());\n\n  auto state_ptr = GetCachedOpState(default_ctx);\n  auto& state    = state_ptr.get_state<CachedOpState>();\n\n  nnvm::Graph& g = state.info.fwd_graph;\n  ShapeVector shape_inputs(inputs.size());\n  for (size_t i = 0; i < inputs.size(); ++i) {\n    shape_inputs[i] = inputs[state.info.input_map[i]]->shape();\n  }\n  // We leverage the shape inference pass to detect whether dynamic shape exists.\n  // If so, the pass will fail with `contain_dynamic_shape = true`,\n  // This method is only called once, so the overhead is negligible.\n  bool contain_dynamic_shape = false;\n  CheckAndInferShape(&g, std::move(shape_inputs), true, {0, 0}, {0, 0}, &contain_dynamic_shape);\n  if (!config_.static_shape && erase_result) {\n    g.attrs.erase(\"shape\");\n    g.attrs.erase(\"shape_inputs\");\n  }\n  return contain_dynamic_shape;\n}\n\nbool CachedOp::SetForwardGraph(const Context& default_ctx,\n                               GraphInfo* info,\n                               const bool recording,\n                               const std::vector<NDArray*>& inputs) {\n  using namespace nnvm;\n  using namespace imperative;\n  CHECK_EQ(inputs.size(), num_inputs());\n  nnvm::Graph& g = info->fwd_graph;\n\n  ShapeVector shape_inputs(inputs.size());\n  DTypeVector dtype_inputs(inputs.size());\n  StorageTypeVector storage_type_inputs(inputs.size());\n  for (size_t i = 0; i < inputs.size(); ++i) {\n    shape_inputs[i]        = inputs[info->input_map[i]]->shape();\n    dtype_inputs[i]        = inputs[info->input_map[i]]->dtype();\n    storage_type_inputs[i] = inputs[info->input_map[i]]->storage_type();\n  }\n\n  bool match                 = true;\n  bool contain_dynamic_shape = false;\n  match &=\n      CheckAndInferShape(&g, std::move(shape_inputs), true, {0, 0}, {0, 0}, &contain_dynamic_shape);\n  match &= CheckAndInferType(&g, std::move(dtype_inputs), true);\n  exec::DevMaskVector dev_mask(g.indexed_graph().num_nodes(), default_ctx.dev_mask());\n  match &= CheckAndInferStorageType(&g, std::move(dev_mask), std::move(storage_type_inputs), true);\n\n  // When dynmaic shape exists, it is not feasible to plan memory ahead of time\n  if (contain_dynamic_shape) {\n    g.attrs.erase(AddPrefix(FORWARD, MEM_PLAN));\n    g.attrs.erase(AddPrefix(FULL, MEM_PLAN));\n    return false;\n  }\n  const std::string& prefix = recording ? FULL : FORWARD;\n  if (!match) {\n    g.attrs.erase(AddPrefix(FORWARD, MEM_PLAN));\n    g.attrs.erase(AddPrefix(FULL, MEM_PLAN));\n  } else if (g.attrs.count(AddPrefix(prefix, MEM_PLAN))) {\n    return true;\n  }\n\n  const auto& idx = g.indexed_graph();\n\n  StorageVector storage(idx.num_node_entries(), exec::kBadStorageID);\n  const auto& stypes = g.GetAttr<StorageTypeVector>(\"storage_type\");\n  CHECK_EQ(stypes.size(), storage.size());\n  for (size_t i = 0; i < stypes.size(); i++) {\n    if (stypes[i] != kDefaultStorage)\n      storage[i] = exec::kDynamicStorageID;\n  }\n  for (const auto i : idx.input_nodes()) {\n    storage[idx.entry_id(i, 0)] = exec::kExternalStorageID;\n  }\n  for (size_t i = 0; i < idx.outputs().size(); ++i) {\n    storage[idx.entry_id(idx.outputs()[i])] = exec::kExternalStorageID;\n  }\n\n  auto mem_plan                        = MXPlanMemory(&g,\n                               std::move(storage),\n                               g.GetAttr<std::vector<uint32_t> >(AddPrefix(prefix, REF_COUNT)),\n                               AddPrefix(prefix, STORAGE_PLAN));\n  g.attrs[AddPrefix(prefix, MEM_PLAN)] = std::make_shared<dmlc::any>(std::move(mem_plan));\n\n  return false;\n}\n\n// Utility function to set backward input eids\nvoid SetBackwardInputEid(const std::vector<uint32_t>& bwd_in_dep,\n                         const std::vector<uint32_t>& bwd_out_dep,\n                         const std::vector<uint32_t>& bwd_ograd_dep,\n                         const std::vector<nnvm::NodeEntry>& ograd_entries,\n                         const nnvm::IndexedGraph& idx,\n                         std::vector<uint32_t>* bwd_input_eid) {\n  for (const auto& i : bwd_ograd_dep) {\n    auto ograd = ograd_entries[i];\n    if (idx.exist(ograd.node.get())) {\n      bwd_input_eid->push_back(idx.entry_id(ograd));\n    } else {\n      bwd_input_eid->push_back(kEidNotExist);\n    }\n  }\n  for (const auto& i : bwd_in_dep) {\n    auto eid = idx.entry_id(idx.input_nodes()[i], 0);\n    bwd_input_eid->push_back(eid);\n  }\n  for (const auto& i : bwd_out_dep) {\n    auto eid = idx.entry_id(idx.outputs()[i]);\n    bwd_input_eid->push_back(eid);\n  }\n}\n\nbool CachedOp::SetBackwardGraph(GraphInfo* info,\n                                const std::vector<OpReqType>& reqs,\n                                const std::vector<NDArray*>& inputs,\n                                bool detect_inplace_addto) {\n  using namespace nnvm;\n  using namespace imperative;\n  std::lock_guard<std::mutex> lock(mutex_);\n  Context default_ctx = inputs[0]->ctx();\n  nnvm::Graph& g      = info->full_graph;\n\n  if (info->bwd_output_reqs != reqs) {\n    info->bwd_output_reqs = reqs;\n    info->bwd_input_eid.clear();\n    g         = nnvm::Graph();\n    g.outputs = info->fwd_graph.outputs;\n    for (size_t i = 0; i < info->grad_graph.outputs.size(); ++i) {\n      if (info->bwd_output_reqs[i] == kNullOp)\n        continue;\n      g.outputs.emplace_back(info->grad_graph.outputs[i]);\n    }\n    g.attrs[\"context\"] = std::make_shared<dmlc::any>(\n        std::vector<Context>(g.indexed_graph().num_nodes(), default_ctx));\n  }\n\n  const auto& idx = g.indexed_graph();\n\n  if (info->bwd_input_eid.size() != inputs.size()) {\n    info->bwd_input_eid.clear();\n    SetBackwardInputEid(\n        bwd_in_dep_, bwd_out_dep_, bwd_ograd_dep_, info->ograd_entries, idx, &info->bwd_input_eid);\n    CHECK_EQ(inputs.size(), info->bwd_input_eid.size());\n  }\n\n  size_t num_forward_nodes   = info->fwd_graph.indexed_graph().num_nodes();\n  size_t num_forward_entries = info->fwd_graph.indexed_graph().num_node_entries();\n\n  if (!g.attrs.count(AddPrefix(BACKWARD, REF_COUNT))) {\n    std::vector<uint32_t> ref_count(idx.num_node_entries(), 0);\n    for (size_t i = num_forward_nodes; i < idx.num_nodes(); ++i) {\n      for (const auto& j : idx[i].inputs)\n        ++ref_count[idx.entry_id(j)];\n    }\n    for (size_t i = 0; i < inputs.size(); ++i) {\n      if (info->bwd_input_eid[i] != kEidNotExist) {\n        ++ref_count[info->bwd_input_eid[i]];\n      }\n    }\n    for (const auto& i : idx.outputs())\n      ++ref_count[idx.entry_id(i)];\n    g.attrs[AddPrefix(BACKWARD, REF_COUNT)] = std::make_shared<dmlc::any>(std::move(ref_count));\n  }\n\n  // Set AddTo Entry based on the req that users provide\n  if (detect_inplace_addto) {\n    std::vector<int> addto_entry(idx.num_node_entries(), 0);\n    for (size_t i = 0; i < info->grad_graph.outputs.size(); ++i) {\n      if (reqs[i] == kAddTo) {\n        auto entry = info->grad_graph.outputs[i];\n        if (!idx.exist(entry.node.get()))\n          continue;\n        auto eid         = idx.entry_id(entry);\n        addto_entry[eid] = 1;\n      }\n    }\n    g.attrs[\"addto_entry\"] = std::make_shared<nnvm::any>(std::move(addto_entry));\n  }\n\n  auto shapes = info->fwd_graph.GetAttr<mxnet::ShapeVector>(\"shape\");\n  shapes.resize(idx.num_node_entries(), mxnet::TShape());\n  auto dtypes = info->fwd_graph.GetAttr<DTypeVector>(\"dtype\");\n  dtypes.resize(idx.num_node_entries(), -1);\n  auto stypes = info->fwd_graph.GetAttr<StorageTypeVector>(\"storage_type\");\n  stypes.resize(idx.num_node_entries(), -1);\n\n  for (size_t i = 0; i < inputs.size(); ++i) {\n    if (info->bwd_input_eid[i] == kEidNotExist) {\n      continue;\n    }\n    size_t oi                      = BwdOriginalInput(info->input_map, i);\n    shapes[info->bwd_input_eid[i]] = inputs[oi]->shape();\n    dtypes[info->bwd_input_eid[i]] = inputs[oi]->dtype();\n    stypes[info->bwd_input_eid[i]] = inputs[oi]->storage_type();\n  }\n\n  std::pair<uint32_t, uint32_t> node_range, entry_range;\n  node_range  = {num_forward_nodes, idx.num_nodes()};\n  entry_range = {num_forward_entries, idx.num_node_entries()};\n\n  bool match = true;\n  match &= CheckAndInferShape(&g, std::move(shapes), false, node_range, entry_range);\n  match &= CheckAndInferType(&g, std::move(dtypes), false, node_range, entry_range);\n  exec::DevMaskVector dev_mask(idx.num_nodes(), default_ctx.dev_mask());\n  match &= CheckAndInferStorageType(\n      &g, std::move(dev_mask), std::move(stypes), false, node_range, entry_range);\n\n  if (!match) {\n    g.attrs.erase(AddPrefix(BACKWARD, MEM_PLAN));\n  } else if (g.attrs.count(AddPrefix(BACKWARD, MEM_PLAN))) {\n    return true;\n  }\n\n  StorageVector storage(idx.num_node_entries(), exec::kBadStorageID);\n  const auto& bwd_stypes = g.GetAttr<StorageTypeVector>(\"storage_type\");\n  for (size_t i = 0; i < bwd_stypes.size(); i++) {\n    if (bwd_stypes[i] != kDefaultStorage)\n      storage[i] = exec::kDynamicStorageID;\n  }\n  for (size_t i = 0; i < num_forward_entries; ++i)\n    storage[i] = exec::kExternalStorageID;\n  for (const auto i : idx.input_nodes())\n    storage[idx.entry_id(i, 0)] = exec::kExternalStorageID;\n  for (const auto i : idx.outputs())\n    storage[idx.entry_id(i)] = exec::kExternalStorageID;\n\n  auto mem_plan                          = MXPlanMemory(&g,\n                               std::move(storage),\n                               g.GetAttr<std::vector<uint32_t> >(AddPrefix(BACKWARD, REF_COUNT)),\n                               AddPrefix(BACKWARD, STORAGE_PLAN),\n                               {num_forward_nodes, idx.num_nodes()},\n                               {num_forward_entries, idx.num_node_entries()},\n                               detect_inplace_addto);\n  g.attrs[AddPrefix(BACKWARD, MEM_PLAN)] = std::make_shared<dmlc::any>(std::move(mem_plan));\n\n  return false;\n}\n\nOpStatePtr CachedOp::GetCachedOpState(const Context& ctx) {\n  std::lock_guard<std::mutex> lock(mutex_);\n  for (const auto& i : cached_op_states_[ctx]) {\n    // only create one state per device when not using static memory\n    if (!config_.static_alloc || i.unique()) {\n      return i;\n    }\n  }\n  auto state_ptr = OpStatePtr::Create<CachedOpState>(ctx, fwd_graph_, full_graph_, inlining_);\n\n  cached_op_states_[ctx].push_back(state_ptr);\n  return state_ptr;\n}\n\nvoid CachedOp::StaticAllocMemory(const OpStatePtr& state_ptr, bool recording, bool keep_fwd) {\n  using namespace nnvm;\n  using namespace imperative;\n\n  auto& state                   = state_ptr.get_state<CachedOpState>();\n  const auto& default_ctx       = state.context;\n  nnvm::Graph& g                = keep_fwd ? state.info.full_graph : state.info.fwd_graph;\n  const auto& idx               = g.indexed_graph();\n  const std::string& graph_type = keep_fwd ? BACKWARD : (recording ? FULL : FORWARD);\n  const auto& storage_plan_attr = AddPrefix(graph_type, STORAGE_PLAN);\n  const auto& storage_plan      = g.GetAttr<std::vector<int> >(storage_plan_attr);\n  const auto& mem_plan          = g.GetAttr<MemoryPlanVector>(AddPrefix(graph_type, MEM_PLAN));\n  std::vector<int> addto_entry;\n  if (g.attrs.count(\"addto_entry\")) {\n    addto_entry = g.GetAttr<std::vector<int> >(\"addto_entry\");\n  }\n  size_t start_eid = keep_fwd ? state.info.fwd_graph.indexed_graph().num_node_entries() : 0;\n  size_t end_eid   = idx.num_node_entries();\n\n  if (!keep_fwd)\n    state.fwd_alloc = false;\n  state.bwd_alloc = false;\n  for (size_t i = start_eid; i < state.buff.size(); ++i) {\n    state.buff[i]            = NDArray();\n    state.arrays[i]          = &state.buff[i];\n    state.array_reqs[i]      = kNullOp;\n    state.dynamic_entries[i] = false;\n  }\n\n  for (auto i : idx.input_nodes()) {\n    auto eid = idx.entry_id(i, 0);\n    if (eid >= start_eid)\n      state.dynamic_entries[eid] = true;\n  }\n  for (auto i : idx.outputs()) {\n    auto eid = idx.entry_id(i);\n    if (eid >= start_eid)\n      state.dynamic_entries[eid] = true;\n  }\n\n  for (size_t i = start_eid; i < end_eid; ++i) {\n    if (addto_entry.size() && addto_entry[i]) {\n      state.array_reqs[i] = kAddTo;\n    } else if (storage_plan[i] >= 0) {\n      state.array_reqs[i] = kWriteInplace;\n    } else if (storage_plan[i] == -2) {\n      // -2 indicate that the entry is never referenced.\n      state.array_reqs[i] = kNullOp;\n    } else {\n      state.array_reqs[i] = kWriteTo;\n    }\n  }\n\n  auto& reuse_pool = keep_fwd ? state.bwd_reuse_pool : state.fwd_reuse_pool;\n  reuse_pool       = imperative::AllocateMemory(g,\n                                          idx,\n                                          default_ctx,\n                                          start_eid,\n                                          end_eid,\n                                          mem_plan,\n                                          state.arrays,\n                                          &state.array_reqs,\n                                          std::move(reuse_pool));\n\n  state.recording = recording;\n  if (keep_fwd) {\n    state.bwd_alloc = true;\n  } else {\n    state.fwd_alloc = true;\n  }\n}\n\nvoid CachedOp::StaticInitExec(const OpStatePtr& state_ptr, bool recording, bool keep_fwd) {\n  using namespace nnvm;\n  using namespace imperative;\n\n  auto& state             = state_ptr.get_state<CachedOpState>();\n  const auto& default_ctx = state.context;\n  nnvm::Graph& g          = keep_fwd ? state.info.full_graph : state.info.fwd_graph;\n  const auto& idx         = g.indexed_graph();\n  std::vector<int> skip_plus_node;\n  if (g.attrs.count(\"skip_plus_node\")) {\n    skip_plus_node = g.GetAttr<std::vector<int> >(\"skip_plus_node\");\n  }\n  size_t start_nid = keep_fwd ? state.info.fwd_graph.indexed_graph().num_nodes() : 0;\n  size_t end_nid   = idx.num_nodes();\n\n  if (!keep_fwd)\n    state.fwd_exec_init = false;\n  state.bwd_exec_init = false;\n\n  for (size_t i = start_nid; i < state.execs.size(); ++i) {\n    state.execs[i].reset();\n    state.opr_segs[i] = EngineOprSeg();\n  }\n\n  if (!config_.static_shape) {\n    for (size_t i = start_nid; i < end_nid; ++i) {\n      state.opr_segs[i].next_nid = i + 1;\n      state.opr_segs[i].skip     = skip_plus_node.size() && skip_plus_node[i];\n    }\n  } else {\n    for (size_t i = start_nid; i < end_nid; ++i) {\n      exec::CreateOpExecs(g, &state.execs, &state.op_states, i);\n    }\n    exec::AttachOpResources(g, state.execs, start_nid, end_nid);\n\n    for (size_t i = start_nid; i < end_nid; ++i) {\n      bool skip = idx[i].source->is_variable();\n      for (size_t j = 0; !skip && j < idx[i].inputs.size(); ++j) {\n        skip = state.dynamic_entries[idx.entry_id(idx[i].inputs[j])];\n      }\n      for (size_t j = 0; !skip && j < idx[i].source->num_outputs(); ++j) {\n        skip = state.dynamic_entries[idx.entry_id(i, j)];\n      }\n      if (skip)\n        continue;\n      SetupOpExec(g, i, state.execs[i], state.arrays, state.array_reqs);\n    }\n\n    // Init bulk_size for Inference mode with bulking enabled (= entire forward graph).\n    size_t bulk_size = idx.num_nodes();\n    if (recording || keep_fwd) {\n      // Training mode\n      if (!Imperative::PreferBulkExecTrain())\n        bulk_size = 0;\n      else\n        bulk_size = keep_fwd ? config_.backward_bulk_size : config_.forward_bulk_size;\n    } else {\n      // Inference mode\n      if (!Imperative::PreferBulkExecInference())\n        bulk_size = 0;\n    }\n\n    CreateEngineOpSeg(idx,\n                      default_ctx,\n                      start_nid,\n                      end_nid,\n                      bulk_size,\n                      state.execs,\n                      skip_plus_node,\n                      &state.opr_segs);\n  }\n\n  if (keep_fwd) {\n    state.bwd_exec_init = true;\n  } else {\n    state.fwd_exec_init = true;\n  }\n}\n\nvoid CachedOp::StaticRunOps(const Context& default_ctx,\n                            const nnvm::Graph& g,\n                            const OpStatePtr& state_ptr,\n                            const std::vector<NDArray*>& state_arrays,\n                            size_t start_nid,\n                            size_t end_nid) {\n  static auto& createop          = nnvm::Op::GetAttr<FCreateOpState>(\"FCreateOpState\");\n  static auto& is_layer_backward = Op::GetAttr<bool>(\"TIsLayerOpBackward\");\n\n  bool profiling   = profiler::Profiler::Get()->GetState() == profiler::Profiler::kRunning;\n  bool is_training = Imperative::Get()->is_training();\n  auto& state      = state_ptr.get_state<CachedOpState>();\n  const auto& idx  = g.indexed_graph();\n  const auto& dispatch_modes = g.GetAttr<DispatchModeVector>(\"dispatch_mode\");\n  const auto& op_execs       = state.execs;\n\n  std::vector<NDArray*> ndinputs, ndoutputs;\n  mxnet::ShapeVector arg_shapes;\n  nnvm::DTypeVector arg_dtypes;\n  std::vector<OpReqType> req;\n\n  for (size_t i = start_nid; config_.static_shape && i < end_nid; ++i) {\n    if (op_execs[i])\n      op_execs[i]->op_ctx.is_train = is_training;\n  }\n\n  for (size_t i = start_nid; i < end_nid; i = state.opr_segs[i].next_nid) {\n    const auto& opr_seg = state.opr_segs[i];\n    if (opr_seg.skip)\n      continue;\n    if (opr_seg.opr != nullptr) {\n      Engine::Get()->Push(opr_seg.opr.get(), default_ctx, 0, profiling);\n    } else {\n      const nnvm::IndexedGraph::Node& node = idx[i];\n      if (node.source->is_variable())\n        continue;\n      auto num_outputs = node.source->num_outputs();\n      ndinputs.clear();\n      ndinputs.reserve(node.inputs.size());\n      for (const auto& j : node.inputs) {\n        ndinputs.emplace_back(state_arrays[idx.entry_id(j)]);\n        CHECK(!ndinputs.back()->is_none());\n      }\n      if (monitor_callback_ && monitor_all_) {\n        mxnet::common::ExecuteMonInputCallback(idx, state_arrays, i, monitor_callback_);\n      }\n      ndoutputs.clear();\n      ndoutputs.reserve(num_outputs);\n      req.clear();\n      req.reserve(num_outputs);\n      for (size_t j = 0; j < num_outputs; ++j) {\n        size_t eid = idx.entry_id(i, j);\n        ndoutputs.emplace_back(state_arrays[eid]);\n        req.push_back(state.array_reqs[eid]);\n        CHECK(req.back() == kNullOp || !ndoutputs.back()->is_none());\n      }\n      const DispatchMode dispatch_mode = dispatch_modes[i];\n\n      if (createop.count(node.source->op())) {\n        arg_shapes.clear();\n        arg_dtypes.clear();\n        arg_shapes.reserve(ndinputs.size());\n        arg_dtypes.reserve(ndinputs.size());\n        for (auto& ndinput : ndinputs) {\n          arg_shapes.emplace_back(ndinput->shape());\n          arg_dtypes.emplace_back(ndinput->dtype());\n        }\n        if (!config_.static_shape) {\n          state.op_states[i] =\n              createop[node.source->op()](node.source->attrs, default_ctx, arg_shapes, arg_dtypes);\n        }\n        Imperative::Get()->InvokeOp(default_ctx,\n                                    node.source->attrs,\n                                    ndinputs,\n                                    ndoutputs,\n                                    req,\n                                    dispatch_mode,\n                                    state.op_states[i]);\n      } else if (is_layer_backward.get(node.source->op(), false)) {\n        nnvm::Node* fwd_node = node.source->control_deps[0].get();\n        auto fwd_node_id     = idx.node_id(fwd_node);\n        Imperative::Get()->InvokeOp(default_ctx,\n                                    node.source->attrs,\n                                    ndinputs,\n                                    ndoutputs,\n                                    req,\n                                    dispatch_mode,\n                                    state.op_states[fwd_node_id]);\n      } else {\n        Imperative::Get()->InvokeOp(\n            default_ctx, node.source->attrs, ndinputs, ndoutputs, req, dispatch_mode);\n      }\n      if (monitor_callback_) {\n        mxnet::common::ExecuteMonOutputCallback(idx, state_arrays, i, monitor_callback_);\n      }\n    }\n  }\n}\n\n#define INIT_DETACHED(x, y) \\\n  if (!y->is_none())        \\\n  x->InitDetached(y)\n\nstatic void PrepareOutputs(const nnvm::Graph& g,\n                           const Context& default_ctx,\n                           const std::vector<NDArray*>& outputs,\n                           std::vector<NDArray*>* pArrays,\n                           bool detach) {\n  using namespace nnvm;\n  const auto& dtypes = g.GetAttr<DTypeVector>(\"dtype\");\n  const auto& shapes = g.GetAttr<mxnet::ShapeVector>(\"shape\");\n  const auto& stypes = g.GetAttr<StorageTypeVector>(\"storage_type\");\n\n  const auto& idx = g.indexed_graph();\n  auto& arrays    = *pArrays;\n  for (size_t i = 0; i < outputs.size(); ++i) {\n    const auto eid = idx.entry_id(idx.outputs()[i]);\n    // An input and an output may share the same array.\n    if (detach)\n      INIT_DETACHED(outputs[i], arrays[eid]);\n\n    arrays[eid] = outputs[i];\n    if (arrays[eid]->is_none())\n      arrays[eid]->ReInit(\n          static_cast<NDArrayStorageType>(stypes[eid]), shapes[eid], default_ctx, dtypes[eid]);\n    const nnvm::NodeAttrs& attrs = idx[idx.outputs()[i].node_id].source->attrs;\n    outputs[i]->AssignStorageInfo(common::NodeAttrsGetProfilerScope(attrs), attrs.name);\n  }\n}\n\nOpStatePtr CachedOp::StaticForward(const Context& default_ctx,\n                                   const std::vector<NDArray*>& inputs,\n                                   const std::vector<NDArray*>& outputs) {\n  using namespace nnvm;\n  using namespace imperative;\n\n  bool recording = Imperative::Get()->is_recording();\n  auto state_ptr = GetCachedOpState(default_ctx);\n  auto& state    = state_ptr.get_state<CachedOpState>();\n\n  // Need to lock the mutex on the state, this allows\n  // for multi context push of ops to dependency engine.\n  // Required to lock for the whole function since static\n  // alloc allocates memory, and executors once and reuses the alloced memory\n  // and executors for multiple forward invokes of the same op.\n  std::lock_guard<std::mutex> lock(state.mutex);\n\n  bool match = SetForwardGraph(default_ctx, &state.info, recording, inputs);\n  match      = match && state.recording == recording;\n\n  nnvm::Graph& g  = state.info.fwd_graph;\n  const auto& idx = g.indexed_graph();\n  if (!state.fwd_alloc || !match) {\n    StaticAllocMemory(state_ptr, recording, false);\n  }\n\n  // We are going to add input and output arrays to the array list.\n  // The input and output arrays should only be valid for this run,\n  // so we shouldn't modify the state's array list.\n  state.arrays_with_in_out = state.arrays;\n  auto& arrays             = state.arrays_with_in_out;\n  if (config_.static_shape) {\n    for (auto i : config_.param_indices) {\n      auto nid = idx.input_nodes()[i];\n      if (!arrays[idx.entry_id(nid, 0)]->IsSame(*inputs[state.info.input_map[i]])) {\n        match    = false;\n        auto ptr = &state.buff[idx.entry_id(nid, 0)];\n        CHECK_EQ(arrays[idx.entry_id(nid, 0)], ptr);\n        *arrays[idx.entry_id(nid, 0)]               = *inputs[state.info.input_map[i]];\n        state.dynamic_entries[idx.entry_id(nid, 0)] = false;\n      }\n    }\n    for (auto i : config_.data_indices) {\n      auto eid    = idx.entry_id(idx.input_nodes()[i], 0);\n      arrays[eid] = inputs[state.info.input_map[i]];\n    }\n  } else {\n    for (size_t i = 0; i < num_inputs(); ++i) {\n      auto nid                     = idx.input_nodes()[i];\n      arrays[idx.entry_id(nid, 0)] = inputs[state.info.input_map[i]];\n    }\n  }\n\n  if (!state.fwd_exec_init || !match) {\n    StaticInitExec(state_ptr, recording, false);\n  }\n\n  PrepareOutputs(g, default_ctx, outputs, &arrays, true);\n  StaticRunOps(default_ctx, g, state_ptr, arrays, 0, idx.num_nodes());\n\n  return recording ? state_ptr : OpStatePtr();\n}\n\nOpStatePtr CachedOp::DynamicForward(const Context& default_ctx,\n                                    const std::vector<NDArray*>& inputs,\n                                    const std::vector<NDArray*>& outputs,\n                                    bool use_naive_run) {\n  using namespace nnvm;\n  using namespace imperative;\n\n  // Initialize\n  bool recording = Imperative::Get()->is_recording();\n  auto op_state  = OpStatePtr::Create<DynamicRuntime>();\n  auto& runtime  = op_state.get_state<DynamicRuntime>();\n  {\n    auto state_ptr = GetCachedOpState(default_ctx);\n    auto& state    = state_ptr.get_state<CachedOpState>();\n    std::lock_guard<std::mutex> lock(state.mutex);\n    SetForwardGraph(default_ctx, &state.info, recording, inputs);\n    runtime.info.fwd_graph = state.info.fwd_graph;\n    runtime.info.input_map = state.info.input_map;\n  }\n  nnvm::Graph& g  = runtime.info.fwd_graph;\n  const auto& idx = g.indexed_graph();\n  auto& buff      = runtime.buff;\n  auto& states    = runtime.op_states;\n\n  // Allocate entries\n  buff.resize(idx.num_node_entries());\n  states.resize(idx.num_nodes());\n  std::vector<NDArray*> arrays;\n  arrays.reserve(buff.size());\n  for (auto& buffered_array : buff) {\n    arrays.push_back(&buffered_array);\n  }\n  std::vector<OpReqType> array_reqs(arrays.size(), kWriteTo);\n  const auto& dispatch_modes    = g.GetAttr<DispatchModeVector>(\"dispatch_mode\");\n  const std::string& graph_type = recording ? FULL : FORWARD;\n  std::vector<uint32_t> ref_count =\n      g.GetAttr<std::vector<uint32_t> >(AddPrefix(graph_type, REF_COUNT));\n  for (size_t i = 0; i < idx.num_node_entries(); ++i) {\n    if (ref_count[i] == 0)\n      array_reqs[i] = kNullOp;\n  }\n  CollectInputOutputNDRefs(g, inputs, runtime.info.input_map, outputs, &arrays);\n\n  if (!use_naive_run) {\n    const auto& mem_plan = g.GetAttr<MemoryPlanVector>(AddPrefix(graph_type, MEM_PLAN));\n    CreateGraphNDs(g, default_ctx, mem_plan, &array_reqs, &arrays);\n    // If CachedOp is running in the inline mode, it uses RunGraph to record\n    // computation; otherwise, CachedOp records computation itself.\n    // So if it's not the inline mode, we disable recording.\n    RunGraph(false,\n             idx,\n             arrays,\n             0,\n             idx.num_nodes(),\n             std::move(array_reqs),\n             std::move(ref_count),\n             &states,\n             dispatch_modes,\n             recording && inlining_,\n             nullptr,\n             monitor_callback_,\n             monitor_all_);\n  } else {\n    mxnet::ShapeVector shapes = g.GetAttr<mxnet::ShapeVector>(\"shape\");\n    NaiveRunGraph(false,\n                  default_ctx,\n                  idx,\n                  arrays,\n                  0,\n                  idx.num_nodes(),\n                  std::move(array_reqs),\n                  std::move(ref_count),\n                  &states,\n                  dispatch_modes,\n                  recording && inlining_,\n                  &shapes,\n                  monitor_callback_,\n                  monitor_all_);\n    {\n      auto state_ptr    = GetCachedOpState(default_ctx);\n      auto& state       = state_ptr.get_state<CachedOpState>();\n      auto copied_shape = shapes;\n      std::lock_guard<std::mutex> lock(state.mutex);\n      state.info.fwd_graph.attrs[\"shape\"] = std::make_shared<dmlc::any>(std::move(copied_shape));\n    }\n    g.attrs[\"shape\"] = std::make_shared<dmlc::any>(std::move(shapes));\n  }\n  return op_state;\n}\n\nOpStatePtr CachedOp::Forward(const std::shared_ptr<CachedOp>& op_ptr,\n                             const std::vector<NDArray*>& inputs,\n                             const std::vector<NDArray*>& outputs,\n                             const Context& default_ctx) {\n  static const auto cached_op = nnvm::Op::Get(\"_CachedOp\");\n\n  CHECK_EQ(inputs.size(), num_inputs());\n  // Assign the storage information for the input arguments. Similar to the\n  // implementation in `graph_executor.cc`, we use `mutable_input_nodes()` to\n  // distinguish between weight parameters and auxiliary states.\n  const auto& fwd_idx             = fwd_graph_.indexed_graph();\n  const auto& mutable_input_nodes = fwd_idx.mutable_input_nodes();\n  for (size_t i = 0; i < fwd_idx.input_nodes().size(); ++i) {\n    const uint32_t nid               = fwd_idx.input_nodes().at(i);\n    const nnvm::NodeAttrs& attrs     = fwd_idx[nid].source->attrs;\n    const std::string& arg_name      = attrs.name;\n    const std::string profiler_scope = common::NodeAttrsGetProfilerScope(attrs);\n    if (mutable_input_nodes.count(nid)) {\n      inputs[i]->AssignStorageInfo(profiler_scope + \"aux_state:\", arg_name);\n    } else {\n      inputs[i]->AssignStorageInfo(profiler_scope + \"in_arg:\", arg_name);\n    }\n  }\n\n  {\n    auto state_ptr = GetCachedOpState(default_ctx);\n    auto& state    = state_ptr.get_state<CachedOpState>();\n\n    const auto& idx = state.info.fwd_graph.indexed_graph();\n    for (size_t i = 0; i < inputs.size(); ++i) {\n      CHECK_EQ(inputs[i]->ctx(), default_ctx)\n          << \"CachedOp requires all inputs to live on the same context. But \"\n          << idx[idx.input_nodes()[0]].source->attrs.name << \" is on \" << default_ctx << \" while \"\n          << idx[idx.input_nodes()[i]].source->attrs.name << \" is on \" << inputs[i]->ctx();\n    }\n  }\n\n  int prev_bulk_size = Engine::Get()->set_bulk_size(config_.forward_bulk_size);\n\n  OpStatePtr op_state;\n  try {\n    if (config_.is_dynamic || CheckDynamicShapeExists(default_ctx, inputs, true)) {\n      config_.is_dynamic   = true;\n      config_.static_alloc = false;\n      op_state             = DynamicForward(default_ctx, inputs, outputs, true);\n    } else if (config_.static_alloc) {\n      op_state = StaticForward(default_ctx, inputs, outputs);\n    } else {\n      op_state = DynamicForward(default_ctx, inputs, outputs, false);\n    }\n  } catch (const dmlc::Error& e) {\n    Engine::Get()->set_bulk_size(prev_bulk_size);\n    throw e;\n  }\n\n  Engine::Get()->set_bulk_size(prev_bulk_size);\n\n  if (Imperative::Get()->is_recording() && !inlining_) {\n    nnvm::NodeAttrs attrs;\n    attrs.op     = cached_op;\n    attrs.name   = \"_cachedop\";\n    attrs.parsed = op_ptr;\n    Imperative::Get()->RecordOp(\n        std::move(attrs), inputs, outputs, op_state, &save_inputs(), &save_outputs());\n  }\n  return op_state;\n}\n\nvoid CachedOp::DynamicBackward(const bool retain_graph,\n                               const OpStatePtr& op_state,\n                               const std::vector<NDArray*>& inputs,\n                               const std::vector<OpReqType>& reqs,\n                               const std::vector<NDArray*>& outputs) {\n  using namespace nnvm;\n  using namespace imperative;\n\n  // Initialize\n  Context default_ctx = outputs[0]->ctx();\n  auto& runtime       = op_state.get_state<DynamicRuntime>();\n  {\n    auto state_ptr = GetCachedOpState(default_ctx);\n    auto& state    = state_ptr.get_state<CachedOpState>();\n    std::lock_guard<std::mutex> lock(state.mutex);\n    state.info.fwd_graph = runtime.info.fwd_graph;\n    state.info.input_map = runtime.info.input_map;\n    SetBackwardGraph(&state.info, reqs, inputs);\n    runtime.info.full_graph    = state.info.full_graph;\n    runtime.info.bwd_input_eid = state.info.bwd_input_eid;\n  }\n  nnvm::Graph& g  = runtime.info.full_graph;\n  const auto& idx = g.indexed_graph();\n  auto& buff      = runtime.buff;\n  auto& states    = runtime.op_states;\n\n  size_t num_forward_outputs = runtime.info.fwd_graph.outputs.size();\n  size_t num_forward_nodes   = runtime.info.fwd_graph.indexed_graph().num_nodes();\n  size_t num_forward_entries = runtime.info.fwd_graph.indexed_graph().num_node_entries();\n  buff.resize(idx.num_node_entries());\n  std::vector<NDArray*> arrays;\n  arrays.reserve(buff.size());\n  for (auto& buffered_array : buff) {\n    arrays.push_back(&buffered_array);\n  }\n  for (size_t i = 0; i < inputs.size(); ++i) {\n    if (runtime.info.bwd_input_eid[i] == kEidNotExist) {\n      continue;\n    }\n    arrays[runtime.info.bwd_input_eid[i]] = inputs[BwdOriginalInput(runtime.info.input_map, i)];\n  }\n  for (size_t i = 0, j = num_forward_outputs; i < reqs.size(); ++i) {\n    if (reqs[i] == kNullOp)\n      continue;\n    const auto eid = idx.entry_id(idx.outputs()[j++]);\n    // An input and an output may share the same array.\n    INIT_DETACHED(outputs[i], arrays[eid]);\n    arrays[eid] = outputs[i];\n  }\n\n  // Allocate NDArrays\n  auto ref_count = g.GetAttr<std::vector<uint32_t> >(AddPrefix(BACKWARD, REF_COUNT));\n  if (retain_graph) {\n    for (size_t i = 0; i < num_forward_entries; ++i)\n      ++ref_count[i];\n  }\n\n  std::vector<OpReqType> array_reqs(arrays.size(), kWriteTo);\n  // set output reqs\n  for (size_t i = 0, j = num_forward_outputs; i < reqs.size(); ++i) {\n    if (reqs[i] == kNullOp)\n      continue;\n    array_reqs[idx.entry_id(idx.outputs()[j++])] = reqs[i];\n  }\n  // set null reqs based on ref counts\n  for (size_t i = num_forward_entries; i < idx.num_node_entries(); ++i) {\n    if (ref_count[i] == 0)\n      array_reqs[i] = kNullOp;\n  }\n\n  const auto& mem_plan = g.GetAttr<MemoryPlanVector>(AddPrefix(BACKWARD, MEM_PLAN));\n  AllocateMemory(g,\n                 idx,\n                 default_ctx,\n                 num_forward_entries,\n                 idx.num_node_entries(),\n                 mem_plan,\n                 arrays,\n                 &array_reqs);\n\n  const auto& dispatch_modes = g.GetAttr<DispatchModeVector>(\"dispatch_mode\");\n\n  RunGraph(retain_graph,\n           idx,\n           arrays,\n           num_forward_nodes,\n           idx.num_nodes(),\n           std::move(array_reqs),\n           std::move(ref_count),\n           &states,\n           dispatch_modes,\n           Imperative::Get()->is_recording(),\n           nullptr,\n           monitor_callback_);\n\n  if (retain_graph) {\n    buff.resize(num_forward_entries);\n  } else {\n    buff.clear();\n    states.clear();\n  }\n}\n\nvoid CachedOp::StaticBackward(const bool retain_graph,\n                              const OpStatePtr& state_ptr,\n                              const std::vector<NDArray*>& inputs,\n                              const std::vector<OpReqType>& reqs,\n                              const std::vector<NDArray*>& outputs) {\n  using namespace nnvm;\n  using namespace imperative;\n\n  Context default_ctx = outputs[0]->ctx();\n\n  auto& state = state_ptr.get_state<CachedOpState>();\n  std::lock_guard<std::mutex> lock(state.mutex);\n\n  bool match = SetBackwardGraph(&state.info, reqs, inputs, true);\n\n  nnvm::Graph& g         = state.info.full_graph;\n  const auto& idx        = g.indexed_graph();\n  auto num_forward_nodes = state.info.fwd_graph.indexed_graph().num_nodes();\n\n  if (!state.bwd_alloc || !match) {\n    StaticAllocMemory(state_ptr, true, true);\n  }\n\n  // We are going to add input and output arrays to the array list.\n  // The input and output arrays should only be valid for this run,\n  // so we shouldn't modify the state's array list.\n  state.arrays_with_in_out = state.arrays;\n  auto& arrays             = state.arrays_with_in_out;\n  for (size_t i = 0; i < state.info.bwd_input_eid.size(); ++i) {\n    auto eid = state.info.bwd_input_eid[i];\n    if (eid == kEidNotExist || !state.dynamic_entries[eid])\n      continue;\n    arrays[eid] = inputs[BwdOriginalInput(state.info.input_map, i)];\n  }\n\n  if (config_.static_shape) {\n    for (auto i : config_.param_indices) {\n      const auto iter = state.info.fwd_input_to_grad_output.find(i);\n      if (iter == state.info.fwd_input_to_grad_output.end())\n        continue;\n      auto entry = state.info.grad_graph.outputs[iter->second];\n      if (!idx.exist(entry.node.get()))\n        continue;\n      auto eid = idx.entry_id(entry);\n      if ((!arrays[eid]->IsSame(*outputs[iter->second]) && state.array_reqs[eid] != kNullOp) ||\n          !(state.array_reqs[eid] == reqs[iter->second])) {\n        match                 = false;\n        state.array_reqs[eid] = reqs[iter->second];\n        // An input and an output may share the same array.\n        INIT_DETACHED(outputs[iter->second], arrays[eid]);\n        *arrays[eid]               = *outputs[iter->second];\n        state.dynamic_entries[eid] = false;\n      }\n    }\n    for (auto i : config_.data_indices) {\n      const auto iter = state.info.fwd_input_to_grad_output.find(i);\n      if (iter == state.info.fwd_input_to_grad_output.end())\n        continue;\n      auto entry = state.info.grad_graph.outputs[iter->second];\n      if (!idx.exist(entry.node.get()))\n        continue;\n      auto eid = idx.entry_id(entry);\n      // An input and an output may share the same array.\n      INIT_DETACHED(outputs[iter->second], arrays[eid]);\n      arrays[eid] = outputs[iter->second];\n    }\n  } else {\n    for (size_t i = 0; i < state.info.grad_graph.outputs.size(); ++i) {\n      auto entry = state.info.grad_graph.outputs[i];\n      if (!idx.exist(entry.node.get()))\n        continue;\n      auto eid = idx.entry_id(entry);\n      // An input and an output may share the same array.\n      INIT_DETACHED(outputs[i], arrays[eid]);\n      arrays[eid] = outputs[i];\n    }\n  }\n\n  if (!state.bwd_exec_init || !match) {\n    StaticInitExec(state_ptr, true, true);\n  }\n\n  StaticRunOps(default_ctx, g, state_ptr, arrays, num_forward_nodes, idx.num_nodes());\n}\n\nvoid CachedOp::Backward(const bool retain_graph,\n                        const OpStatePtr& state,\n                        const std::vector<NDArray*>& inputs,\n                        const std::vector<OpReqType>& reqs,\n                        const std::vector<NDArray*>& outputs) {\n  const auto& fwd_idx             = fwd_graph_.indexed_graph();\n  const auto& full_idx            = full_graph_.indexed_graph();\n  const auto& mutable_input_nodes = fwd_idx.mutable_input_nodes();\n  for (size_t i = 0, j = 0; i < fwd_idx.input_nodes().size(); ++i) {\n    const uint32_t nid          = fwd_idx.input_nodes().at(i);\n    const std::string& arg_name = fwd_idx[nid].source->attrs.name;\n    const std::string profiler_scope =\n        common::NodeAttrsGetProfilerScope(fwd_idx[nid].source->attrs);\n    if (mutable_input_nodes.count(nid)) {\n      continue;\n    }\n    outputs[j++]->AssignStorageInfo(profiler_scope + \"arg_grad:\", arg_name);\n  }\n  for (size_t i = fwd_idx.input_nodes().size(), j = 0; i < full_idx.input_nodes().size(); ++i) {\n    const nnvm::NodeAttrs& attrs     = full_idx[full_idx.input_nodes().at(i)].source->attrs;\n    const std::string& entry_name    = attrs.name;\n    const std::string profiler_scope = common::NodeAttrsGetProfilerScope(attrs);\n    inputs[j++]->AssignStorageInfo(profiler_scope, entry_name);\n  }\n\n  using namespace imperative;\n  CHECK(!Imperative::Get()->is_recording())\n      << \"CachedOp does not support higher order gradients. \"\n      << \"If you want to do backward with create_graph=True please \"\n      << \"do not use hybridize.\";\n\n  int prev_bulk_size = Engine::Get()->set_bulk_size(config_.backward_bulk_size);\n\n  try {\n    if (config_.static_alloc) {\n      StaticBackward(retain_graph, state, inputs, reqs, outputs);\n    } else {\n      DynamicBackward(retain_graph, state, inputs, reqs, outputs);\n    }\n  } catch (const dmlc::Error& e) {\n    Engine::Get()->set_bulk_size(prev_bulk_size);\n    throw e;\n  }\n\n  Engine::Get()->set_bulk_size(prev_bulk_size);\n}\n\n/*\n * This is the operator state of CachedOp when CachedOp is used in the symbol\n * executor. This is different from the OpState returned by CachedOp::Forward.\n * The main reason why we need this OpState is that CachedOp and the symbol executor\n * maintain OpState differently. The symbol executor generates OpState in advance\n * while CachedOp generates OpState after Forward is called. We need this data\n * structure to keep the OpState generated by CachedOp::Forward and pass it to\n * Backward.\n */\nstruct CachedOpActualState {\n  std::shared_ptr<CachedOp> op;\n  OpStatePtr forward_state;\n\n  explicit CachedOpActualState(std::shared_ptr<CachedOp> op) {\n    this->op = op;\n  }\n};\n\n/*\n * This is the forward computation when CachedOp is used as an operator in\n * a symbol executor.\n */\nvoid CachedOpForward(const OpStatePtr& state_ptr,\n                     const OpContext& ctx,\n                     const std::vector<NDArray>& inputs,\n                     const std::vector<OpReqType>& req,\n                     const std::vector<NDArray>& outputs) {\n  CachedOpActualState& s        = state_ptr.get_state<CachedOpActualState>();\n  std::vector<NDArray> in_bufs  = inputs;\n  std::vector<NDArray> out_bufs = outputs;\n  std::vector<NDArray*> in_ptrs(in_bufs.size());\n  std::vector<NDArray*> out_ptrs(out_bufs.size());\n  for (size_t i = 0; i < in_ptrs.size(); i++)\n    in_ptrs[i] = &in_bufs[i];\n  for (size_t i = 0; i < out_ptrs.size(); i++)\n    out_ptrs[i] = &out_bufs[i];\n\n  // Set is_recording correct for the imperative executor.\n  bool orig_is_record;\n  if (ctx.need_grad)\n    orig_is_record = Imperative::Get()->set_is_recording(true);\n  else\n    orig_is_record = Imperative::Get()->is_recording();\n  // Set is_training correct for the imperative executor.\n  bool orig_is_train;\n  if (ctx.is_train)\n    orig_is_train = Imperative::Get()->set_is_training(true);\n  else\n    orig_is_train = Imperative::Get()->is_training();\n  CHECK(inputs.size() > 0) << \"cached op forward requires at least 1 input\";\n  Context default_ctx = inputs[0].ctx();\n  s.forward_state     = s.op->Forward(nullptr, in_ptrs, out_ptrs, default_ctx);\n  Imperative::Get()->set_is_training(orig_is_train);\n  Imperative::Get()->set_is_recording(orig_is_record);\n  // The arrays in out_ptrs may be changed by CachedOp.\n  // If it is, we need to copy data back.\n  for (size_t i = 0; i < out_bufs.size(); i++)\n    if (!out_bufs[i].IsSame(outputs[i]))\n      CopyFromTo(out_bufs[i], outputs[i]);\n}\n\n/*\n * This is the backward computation when CachedOp is used as an operator in\n * a symbol executor.\n */\nvoid CachedOpBackward(const OpStatePtr& state_ptr,\n                      const OpContext& ctx,\n                      const std::vector<NDArray>& inputs,\n                      const std::vector<OpReqType>& req,\n                      const std::vector<NDArray>& outputs) {\n  using namespace nnvm;\n  using namespace imperative;\n  CachedOpActualState& s        = state_ptr.get_state<CachedOpActualState>();\n  std::vector<NDArray> in_bufs  = inputs;\n  std::vector<NDArray> out_bufs = outputs;\n  std::vector<NDArray*> in_ptrs;\n  std::vector<NDArray*> out_ptrs;\n  CHECK_EQ(s.op->num_backward_inputs(), inputs.size());\n  in_ptrs.reserve(s.op->num_backward_inputs());\n  out_ptrs.reserve(s.op->num_inputs());\n\n  const std::vector<bool>& save_inputs  = s.op->save_inputs();\n  const std::vector<bool>& save_outputs = s.op->save_outputs();\n  size_t bwd_in_dep                     = s.op->num_inputs();\n  size_t bwd_out_dep                    = s.op->num_outputs();\n  CHECK(s.op->num_backward_inputs() > bwd_in_dep + bwd_out_dep);\n  size_t bwd_ograd_dep = s.op->num_backward_inputs() - bwd_in_dep - bwd_out_dep;\n\n  // Find inputs, outputs and ograds\n  auto ograds_begin = in_bufs.begin();\n  auto ograds_end   = in_bufs.begin() + bwd_ograd_dep;\n  auto in_begin     = ograds_end;\n  auto in_end       = in_begin + bwd_in_dep;\n  auto out_begin    = in_end;\n  auto out_end      = in_bufs.end();\n\n  for (auto it = ograds_begin; it != ograds_end; it++)\n    in_ptrs.push_back(&(*it));\n\n  CHECK_EQ(save_inputs.size(), in_end - in_begin);\n  CHECK_EQ(s.op->num_outputs(), out_end - out_begin);\n  for (auto it = in_begin; it != in_end; it++) {\n    auto i = it - in_begin;\n    if (save_inputs[i])\n      in_ptrs.push_back(&(*it));\n  }\n  for (auto it = out_begin; it != out_end; it++) {\n    auto i = it - out_begin;\n    if (save_outputs[i])\n      in_ptrs.push_back(&(*it));\n  }\n  CHECK_EQ(in_ptrs.size(), s.op->num_backward_inputs());\n  for (auto& out_buf : out_bufs) {\n    out_ptrs.push_back(&out_buf);\n  }\n  CHECK_EQ(out_ptrs.size(), s.op->num_backward_outputs());\n  // Set is_training correct for the imperative executor.\n  bool orig_is_train;\n  if (ctx.is_train)\n    orig_is_train = Imperative::Get()->set_is_training(true);\n  else\n    orig_is_train = Imperative::Get()->is_training();\n  // TODO(zhengda) CachedOp supports recording computation when running\n  // the backward path. This is necessary if we want to support the second-order\n  // differentiation. However, MXNet operator doesn't have an interface to\n  // pass a flag to determine whether to record computation inside an operator.\n  // Let's use false here for now and design a solution when the second-order\n  // differentiation is supported.\n  s.op->Backward(false, s.forward_state, in_ptrs, req, out_ptrs);\n  Imperative::Get()->set_is_training(orig_is_train);\n\n  // Clean up what we recorded.\n  s.forward_state.reset();\n\n  // The arrays in out_ptrs may be changed by CachedOp.\n  // If it is, we need to copy data back.\n  // For example, when the inputs and outputs share the same NDArrays,\n  // the outputs will be replaced by inputs.\n  // https://github.com/apache/mxnet/blob/v1.2.0/src/imperative/cached_op.cc#L385\n  for (size_t i = 0; i < out_bufs.size(); i++)\n    if (!out_bufs[i].IsSame(outputs[i]))\n      CopyFromTo(out_bufs[i], outputs[i]);\n}\n\n/*\n * Register the callback to be called when the operator is executed\n */\nvoid CachedOp::RegisterOpHook(const CachedOp::CachedOpMonCallback& callback, bool monitor_all) {\n  CHECK(callback) << \"invalid callback\";\n  monitor_callback_ = callback;\n  monitor_all_      = monitor_all;\n}\n\nOpStatePtr CreateCachedOpState(const NodeAttrs& attrs,\n                               Context ctx,\n                               const mxnet::ShapeVector& in_shapes,\n                               const std::vector<int>& in_types) {\n  const CachedOpPtr& op = nnvm::get<CachedOpPtr>(attrs.parsed);\n  return OpStatePtr::Create<CachedOpActualState>(op);\n}\n\nbool CachedOp::BackwardStorageType(const nnvm::NodeAttrs& attrs,\n                                   const int dev_mask,\n                                   DispatchMode* dispatch_mode,\n                                   std::vector<int>* in_attrs,\n                                   std::vector<int>* out_attrs) {\n  using namespace imperative;\n  nnvm::Graph g(full_graph_);\n  const auto& idx                  = g.indexed_graph();\n  const auto& outputs              = idx.outputs();\n  const size_t num_forward_outputs = fwd_graph_.outputs.size();\n  CHECK_EQ(outputs.size(), num_forward_outputs + out_attrs->size());\n\n  // Construct bwd_input_eid\n  std::vector<uint32_t> bwd_input_eid;\n  SetBackwardInputEid(\n      bwd_in_dep_, bwd_out_dep_, bwd_ograd_dep_, ograd_entries_, idx, &bwd_input_eid);\n  CHECK_EQ(in_attrs->size(), bwd_input_eid.size());\n\n  // Prepare stypes and contexts based on inputs\n  StorageTypeVector stypes(idx.num_node_entries(), -1);\n  for (size_t i = 0; i < in_attrs->size(); ++i) {\n    stypes[bwd_input_eid[i]] = in_attrs->at(i);\n  }\n  // Some out_attr is known ahead of time (e.g. the grad stype is given by users).\n  // Prepare these to before invoking infer storage on the subgraph\n  for (size_t i = 0; i < out_attrs->size(); i++) {\n    const auto eid = idx.entry_id(outputs[i + num_forward_outputs]);\n    if (bwd_input_eid[i] == kEidNotExist) {\n      continue;\n    }\n    stypes[eid] = out_attrs->at(i);\n  }\n  exec::DevMaskVector dev_masks(idx.num_nodes(), dev_mask);\n\n  // Full graph storage type inference\n  CheckAndInferStorageType(&g, std::move(dev_masks), std::move(stypes), false);\n  // Retrieve result and set outputs\n  const auto& inferred_stypes = g.GetAttr<StorageTypeVector>(\"storage_type\");\n  for (size_t i = 0; i < out_attrs->size(); i++) {\n    const auto eid = idx.entry_id(outputs[i + num_forward_outputs]);\n    STORAGE_TYPE_ASSIGN_CHECK(*out_attrs, i, inferred_stypes[eid]);\n  }\n  DISPATCH_MODE_ASSIGN_CHECK(dispatch_mode, 0, DispatchMode::kFComputeEx);\n  return true;\n}\n\nvoid CachedOpParamParser(nnvm::NodeAttrs* attrs) {\n  CachedOpConfig param;\n  try {\n    param.Init(attrs->dict);\n  } catch (const dmlc::ParamError& e) {\n    std::ostringstream os;\n    os << e.what();\n    os << \", in operator \" << attrs->op->name << \"(\"\n       << \"name=\\\"\" << attrs->name << \"\\\"\";\n    for (const auto& k : attrs->dict) {\n      os << \", \" << k.first << \"=\\\"\" << k.second << \"\\\"\";\n    }\n    os << \")\";\n    throw dmlc::ParamError(os.str());\n  }\n  if (!param.subgraph.empty()) {\n    nnvm::Graph g = nnvm::pass::LoadJSON(param.subgraph);\n    CHECK(!g.outputs.empty());\n    nnvm::Symbol sym;\n    sym.outputs = g.outputs;\n    std::vector<std::pair<std::string, std::string> > flags;\n    for (const auto& attr : attrs->dict)\n      flags.emplace_back(attr.first, attr.second);\n    attrs->parsed = std::make_shared<CachedOp>(sym, flags);\n  }\n}\n\nsize_t CachedOp::BwdOriginalInput(const std::vector<size_t>& input_map, size_t new_i) {\n  CHECK_GE(input_map.size(), bwd_in_dep_.size());\n  if (new_i >= bwd_ograd_dep_.size() && new_i < bwd_ograd_dep_.size() + bwd_in_dep_.size())\n    return bwd_ograd_dep_.size() + input_map[new_i - bwd_ograd_dep_.size()];\n  return new_i;\n}\n\nNNVM_REGISTER_OP(_CachedOp)\n    .set_num_inputs([](const NodeAttrs& attrs) {\n      const CachedOpPtr& op = nnvm::get<CachedOpPtr>(attrs.parsed);\n      return op->num_inputs();\n    })\n    .set_num_outputs([](const NodeAttrs& attrs) {\n      const CachedOpPtr& op = nnvm::get<CachedOpPtr>(attrs.parsed);\n      return op->num_outputs();\n    })\n    .set_attr_parser(CachedOpParamParser)\n    .set_attr<nnvm::FGradient>(\"FGradient\",\n                               [](const nnvm::ObjectPtr& n,\n                                  const std::vector<nnvm::NodeEntry>& ograds) {\n                                 const CachedOpPtr& op = nnvm::get<CachedOpPtr>(n->attrs.parsed);\n                                 return op->Gradient(n, ograds);\n                               })\n    .set_attr<nnvm::FListInputNames>(\"FListInputNames\",\n                                     [](const nnvm::NodeAttrs& attrs) {\n                                       const CachedOpPtr& op = nnvm::get<CachedOpPtr>(attrs.parsed);\n                                       return op->ListForwardInputNames();\n                                     })\n    .set_attr<nnvm::FListOutputNames>(\"FListOutputNames\",\n                                      [](const nnvm::NodeAttrs& attrs) {\n                                        const CachedOpPtr& op =\n                                            nnvm::get<CachedOpPtr>(attrs.parsed);\n                                        return op->ListForwardOutputNames();\n                                      })\n    .set_attr<FCreateOpState>(\"FCreateOpState\", CreateCachedOpState)\n    .set_attr<mxnet::FInferShape>(\"FInferShape\",\n                                  [](const nnvm::NodeAttrs& attrs,\n                                     mxnet::ShapeVector* in_shapes,\n                                     mxnet::ShapeVector* out_shapes) {\n                                    const CachedOpPtr& op = nnvm::get<CachedOpPtr>(attrs.parsed);\n                                    return op::DefaultSubgraphOpShapeHelper(\n                                        op->GetForwardSym(), in_shapes, out_shapes);\n                                  })\n    .set_attr<nnvm::FInferType>(\n        \"FInferType\",\n        [](const nnvm::NodeAttrs& attrs, std::vector<int>* in_types, std::vector<int>* out_types) {\n          const CachedOpPtr& op = nnvm::get<CachedOpPtr>(attrs.parsed);\n          return op::DefaultSubgraphOpTypeHelper(op->GetForwardSym(), in_types, out_types);\n        })\n    .set_attr<FInferStorageType>(\n        \"FInferStorageType\",\n        [](const nnvm::NodeAttrs& attrs,\n           const int dev_mask,\n           DispatchMode* dispatch_mode,\n           std::vector<int>* in_stypes,\n           std::vector<int>* out_stypes) {\n          const CachedOpPtr& op = nnvm::get<CachedOpPtr>(attrs.parsed);\n          return op::DefaultSubgraphOpStorageTypeHelper(\n              op->GetForwardSym(), dev_mask, dispatch_mode, in_stypes, out_stypes);\n        })\n    .set_attr<FStatefulComputeEx>(\"FStatefulComputeEx<cpu>\", CachedOpForward)\n    .set_attr<FStatefulComputeEx>(\"FStatefulComputeEx<gpu>\", CachedOpForward)\n    .set_attr<nnvm::FMutateInputs>(\"FMutateInputs\",\n                                   [](const nnvm::NodeAttrs& attrs) {\n                                     const CachedOpPtr& op = nnvm::get<CachedOpPtr>(attrs.parsed);\n                                     return op::DefaultSubgraphOpMutableInputsHelper(\n                                         op->GetForwardSym());\n                                   })\n    .set_attr<FResourceRequest>(\"FResourceRequest\",\n                                [](const nnvm::NodeAttrs& attrs) {\n                                  const CachedOpPtr& op = nnvm::get<CachedOpPtr>(attrs.parsed);\n                                  return op::DefaultSubgraphOpResourceRequestHelper(\n                                      op->GetForwardSym());\n                                })\n    .set_attr<FExecType>(\"FExecType\", op::DefaultSubgraphOpExecType)\n    .add_argument(\"data\", \"NDArray-or-Symbol[]\", \"input data list\");\n\nNNVM_REGISTER_OP(_backward_CachedOp)\n    .set_num_inputs([](const NodeAttrs& attrs) {\n      const CachedOpPtr& op = nnvm::get<CachedOpPtr>(attrs.parsed);\n      return op->num_backward_inputs();\n    })\n    .set_num_outputs([](const NodeAttrs& attrs) {\n      const CachedOpPtr& op = nnvm::get<CachedOpPtr>(attrs.parsed);\n      return op->num_inputs() - op->mutable_input_nodes().size();\n    })\n    .set_attr<FInferStorageType>(\"FInferStorageType\",\n                                 [](const nnvm::NodeAttrs& attrs,\n                                    const int dev_mask,\n                                    DispatchMode* dispatch_mode,\n                                    std::vector<int>* in_attrs,\n                                    std::vector<int>* out_attrs) {\n                                   const CachedOpPtr& op = nnvm::get<CachedOpPtr>(attrs.parsed);\n                                   return op->BackwardStorageType(\n                                       attrs, dev_mask, dispatch_mode, in_attrs, out_attrs);\n                                 })\n    .set_attr<FStatefulComputeEx>(\"FStatefulComputeEx<cpu>\", CachedOpBackward)\n    .set_attr<FStatefulComputeEx>(\"FStatefulComputeEx<gpu>\", CachedOpBackward)\n    .set_attr<FExecType>(\"FExecType\", op::DefaultSubgraphOpExecType)\n    .set_attr<bool>(\"TIsLayerOpBackward\", true)\n    .set_attr<bool>(\"TIsBackward\", true);\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/imperative/cached_op.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_IMPERATIVE_CACHED_OP_H_\n#define MXNET_IMPERATIVE_CACHED_OP_H_\n\n#include <mxnet/imperative.h>\n#include <vector>\n#include <numeric>\n#include <atomic>\n#include <utility>\n#include <string>\n#include <unordered_map>\n#include <map>\n#include \"../common/alm.h\"\n#include \"../operator/operator_common.h\"\n#include \"../operator/subgraph/common.h\"\n#include \"./imperative_utils.h\"\n#include \"../nnvm/error.h\"\n\nnamespace mxnet {\nnamespace {\n\nstatic const char FULL[]         = \"full\";\nstatic const char FORWARD[]      = \"forward\";\nstatic const char BACKWARD[]     = \"backward\";\nstatic const char REF_COUNT[]    = \"ref_count\";\nstatic const char MEM_PLAN[]     = \"mem_plan\";\nstatic const char STORAGE_PLAN[] = \"storage_plan\";\n\nstd::string AddPrefix(const std::string& prefix, const std::string& s) {\n  return prefix + \"_\" + s;\n}\n\nnnvm::NodeEntry AggregateGradient(std::vector<nnvm::NodeEntry>&& v) {\n  using nnvm::Op;\n  static size_t inplace_sum_cap  = dmlc::GetEnv(\"MXNET_EXEC_INPLACE_GRAD_SUM_CAP\", 8);\n  static const Op* ewise_plus_op = Op::Get(\"_grad_add\");\n  static const Op* ewise_sum_op  = Op::Get(\"ElementWiseSum\");\n  static const Op* identity_op   = Op::Get(\"identity\");\n  static const Op* zeros_op      = Op::Get(\"_zeros\");\n  static const Op* zeros_like_op = Op::Get(\"zeros_like\");\n\n  if (v.empty()) {\n    nnvm::ObjectPtr ng = nnvm::Node::Create();\n    ng->attrs.op       = Op::Get(\"_zeros_without_dtype\");\n    ng->attrs.name     = \"zeros_without_dtype\";\n    ng->attrs.op->attr_parser(&(ng->attrs));\n    return nnvm::NodeEntry(std::move(ng), 0, 0);\n  }\n\n  // remove zero in the sum. at least keep 1.\n  auto begin = std::remove_if(v.begin(), v.end(), [](const nnvm::NodeEntry& nodeEntry) {\n    CHECK(nodeEntry.node);\n    return nodeEntry.node->op() == zeros_op || nodeEntry.node->op() == zeros_like_op;\n  });\n  if (begin == v.begin())\n    ++begin;\n  v.erase(begin, v.end());\n  CHECK(!v.empty());\n\n  if (v.size() == 1) {\n    return std::move(v[0]);\n  } else {\n    if (v.size() < inplace_sum_cap) {\n      nnvm::ObjectPtr sum_node         = nnvm::Node::Create();\n      sum_node->attrs.op               = ewise_sum_op;\n      sum_node->attrs.name             = \"sum_grad\";\n      sum_node->attrs.dict[\"num_args\"] = std::to_string(v.size());\n      sum_node->attrs.op->attr_parser(&(sum_node->attrs));\n      sum_node->inputs = std::move(v);\n      return nnvm::NodeEntry(std::move(sum_node), 0, 0);\n    } else {\n      // use a stream line of plus instead\n      nnvm::NodeEntry ret = v[0];\n      for (size_t i = 1; i < v.size(); ++i) {\n        // Add control flow dependency from to previous node\n        // This enforces the gradient sum order will be in the inverse\n        // order of forward traversal\n        // NOTE: adding control dependency can be dangerous and cause cycle in the dep.\n        // The curent usage is correct, because of the following invariant:\n        // assert: v[i-1] do not depend on v[i]\n        // To put in plain text: v is gradient vector that get pushed in the order\n        // that can generate them, which means if v[i] is not yet pushed,\n        // all previous gradient cannot depend on it.\n        // Note: For a symbol like the following:\n        // data = mx.sym.Variable('data')\n        // sym = data + data + data + data + data + data + data\n        // the node entries v passed in here are of the same node of\n        // op _identity_with_attr_like_rhs. We should skip adding a node\n        // to its own control_deps.\n        if (v[i - 1].node != v[i].node) {\n          v[i].node->control_deps.push_back(ret.node);\n        }\n\n        std::ostringstream os;\n        os << \"sum_grad_\" << i;\n        nnvm::ObjectPtr x = nnvm::Node::Create();\n        x->attrs.op       = ewise_plus_op;\n        x->attrs.name     = os.str();\n        x->inputs         = {ret, v[i]};\n        ret               = nnvm::NodeEntry(std::move(x), 0, 0);\n      }\n      // identity node is used to avoid exposure of dummy plus node\n      // when its output get assigned to another space.\n      nnvm::ObjectPtr id_node = nnvm::Node::Create();\n      id_node->attrs.op       = identity_op;\n      id_node->attrs.name     = \"sum_grad_final\";\n      id_node->inputs         = {ret};\n      return nnvm::NodeEntry{id_node, 0, 0};\n    }\n  }\n}\n\n/* \\brief collect pointers to input and output ndarrays\n * into a single data structure, this data structure can\n * be used for Memory allocation pass*/\n\nvoid CollectInputOutputNDRefs(const nnvm::Graph& g,\n                              const std::vector<NDArray*>& inputs,\n                              const std::vector<size_t>& input_map,\n                              const std::vector<NDArray*>& outputs,\n                              std::vector<NDArray*>* arrays) DMLC_ATTRIBUTE_UNUSED;\nvoid CollectInputOutputNDRefs(const nnvm::Graph& g,\n                              const std::vector<NDArray*>& inputs,\n                              const std::vector<size_t>& input_map,\n                              const std::vector<NDArray*>& outputs,\n                              std::vector<NDArray*>* arrays) {\n  const auto& idx   = g.indexed_graph();\n  size_t num_inputs = idx.input_nodes().size();\n  for (size_t i = 0; i < num_inputs; ++i) {\n    (*arrays)[idx.entry_id(idx.input_nodes()[i], 0)] = inputs[input_map[i]];\n  }\n  for (size_t i = 0; i < idx.outputs().size(); ++i) {\n    auto eid = idx.entry_id(idx.outputs()[i]);\n    if (!(*arrays)[eid]->is_none())\n      *outputs[i] = (*arrays)[eid]->Detach();\n    (*arrays)[eid] = outputs[i];\n  }\n}\n\n/* \\brief create ndarrays for the intermediate outputs and final outputs\n * from the allocated storage (happens in MXPlanMemory NNVM pass)*/\nvoid CreateGraphNDs(const nnvm::Graph& g,\n                    const mxnet::Context& default_ctx,\n                    const mxnet::imperative::MemoryPlanVector& mem_plan,\n                    std::vector<OpReqType>* array_reqs,\n                    std::vector<NDArray*>* arrays) DMLC_ATTRIBUTE_UNUSED;\nvoid CreateGraphNDs(const nnvm::Graph& g,\n                    const mxnet::Context& default_ctx,\n                    const mxnet::imperative::MemoryPlanVector& mem_plan,\n                    std::vector<OpReqType>* array_reqs,\n                    std::vector<NDArray*>* arrays) {\n  const auto& idx = g.indexed_graph();\n  mxnet::imperative::AllocateMemory(\n      g, idx, default_ctx, 0, idx.num_node_entries(), mem_plan, *arrays, array_reqs);\n  const auto& dtypes = g.GetAttr<nnvm::DTypeVector>(\"dtype\");\n  const auto& shapes = g.GetAttr<mxnet::ShapeVector>(\"shape\");\n  const auto& stypes = g.GetAttr<mxnet::StorageTypeVector>(\"storage_type\");\n  for (size_t i = 0; i < idx.outputs().size(); ++i) {\n    auto eid = idx.entry_id(idx.outputs()[i]);\n    if (!(*arrays)[eid]->is_none())\n      continue;\n    *((*arrays)[eid]) = NDArray(\n        static_cast<NDArrayStorageType>(stypes[eid]), shapes[eid], default_ctx, true, dtypes[eid]);\n    const nnvm::NodeAttrs& attrs = idx[idx.outputs()[i].node_id].source->attrs;\n    (*arrays)[eid]->AssignStorageInfo(common::NodeAttrsGetProfilerScope(attrs), attrs.name);\n  }\n}\n\n/* \\brief create a forward graph from they Symbol */\nvoid CreateForwardGraph(const nnvm::Symbol& sym, nnvm::Graph* fwd_graph) {\n  using namespace nnvm;\n  static const auto _copy_op = Op::Get(\"_copy\");\n  NodeEntryMap<size_t> dedup_out;\n  // Iterate through all node entries, emplace node entry outputs of symbol\n  // to graph outputs. Since node entry stores information about the node\n  // as well as the input node of the graph, a graph can be recreated from a\n  // symbol by just copying the outputs\n  for (const NodeEntry& nodeEntry : sym.outputs) {\n    if (dedup_out.find(nodeEntry) != dedup_out.end()) {\n      ObjectPtr copy_node = Node::Create();\n      copy_node->attrs.op = _copy_op;\n      copy_node->attrs.name =\n          nodeEntry.node->attrs.name + \"_copy\" + std::to_string(dedup_out[nodeEntry]++);\n      copy_node->inputs.emplace_back(nodeEntry);\n      if (_copy_op->attr_parser != nullptr) {\n        _copy_op->attr_parser(&(copy_node->attrs));\n      }\n      fwd_graph->outputs.emplace_back(std::move(copy_node));\n    } else {\n      dedup_out.emplace(nodeEntry, 0);\n      fwd_graph->outputs.push_back(nodeEntry);\n    }\n  }\n  if (alm::ALMParams::get().optimize)\n    *fwd_graph = alm::OptimizeLayout(std::move(*fwd_graph));\n}\n\n/* \\brief construct grad_graph from fwd_graph and ograd_entries*/\nvoid CreateBackwardGraph(nnvm::Graph* fwd_graph,\n                         nnvm::Graph* grad_graph,\n                         std::vector<nnvm::NodeEntry>* ograd_entries,\n                         std::unordered_map<uint32_t, uint32_t>* fwd_input_to_grad_output) {\n  using namespace nnvm;\n  static const std::vector<const Op*> zero_ops{Op::Get(\"zeros_like\"), Op::Get(\"_zeros\")};\n  ograd_entries->reserve(fwd_graph->outputs.size());\n  for (size_t i = 0; i < fwd_graph->outputs.size(); ++i) {\n    nnvm::ObjectPtr np                   = Node::Create();\n    const nnvm::NodeAttrs& attrs         = fwd_graph->outputs[i].node->attrs;\n    np->attrs.name                       = attrs.name + \"_head_grad\";\n    np->attrs.dict[\"__profiler_scope__\"] = common::NodeAttrsGetProfilerScope(attrs);\n    ograd_entries->emplace_back(np);\n  }\n\n  std::vector<NodeEntry> xs;\n  const IndexedGraph& indexed_graph = fwd_graph->indexed_graph();\n  // Create vector of inputs to be passed to the gradient pass\n  for (size_t i = 0; i < indexed_graph.input_nodes().size(); ++i) {\n    const uint32_t node_id = indexed_graph.input_nodes()[i];\n    // skip the mutable nodes, which store the auxiliary states,\n    // since we don't need to compute gradient w.r.t auxiliary states\n    if (indexed_graph.mutable_input_nodes().count(node_id))\n      continue;\n    // Hold a mapping of the node id to its igrad position\n    // Need this mapping in StaticBackward, to obtain the igrad node,\n    // corresponding to a fwd_graph node.\n    (*fwd_input_to_grad_output)[i] = xs.size();\n    xs.emplace_back(indexed_graph[node_id].weak_ref.lock());\n  }\n\n  // There are inputs in computation graph that require gradients\n  if (!xs.empty()) {\n    try {\n      *grad_graph = pass::MXGradient(*fwd_graph,\n                                     fwd_graph->outputs,\n                                     xs,\n                                     *ograd_entries,\n                                     mxnet::AggregateGradient,\n                                     nullptr,\n                                     zero_ops,\n                                     \"_copy\");\n    } catch (const nnvm::pass::InvalidGraphError& e) {\n      *grad_graph = nnvm::Graph();\n    }\n  } else {\n    *grad_graph = nnvm::Graph();\n  }\n}\n\n/* \\brief construct fwd_graph, grad_graph and full_graph from symbol */\nvoid CreateFullGraph(const nnvm::Symbol& sym,\n                     nnvm::Graph* fwd_graph,\n                     nnvm::Graph* grad_graph,\n                     nnvm::Graph* full_graph,\n                     std::vector<nnvm::NodeEntry>* ograd_entries,\n                     std::unordered_map<uint32_t, uint32_t>* fwd_input_to_grad_output) {\n  using namespace nnvm;\n  CreateForwardGraph(sym, fwd_graph);\n\n  bool do_elim_common_expr = dmlc::GetEnv(\"MXNET_ELIMINATE_COMMON_EXPR\", true);\n  if (do_elim_common_expr)\n    *fwd_graph = exec::EliminateCommonExpr(std::move(*fwd_graph));\n\n  // construct backward graph\n  CreateBackwardGraph(fwd_graph, grad_graph, ograd_entries, fwd_input_to_grad_output);\n\n  full_graph->outputs = fwd_graph->outputs;\n  // add backward graph outputs to full graph\n  for (const auto& i : grad_graph->outputs) {\n    full_graph->outputs.emplace_back(i);\n  }\n}\n\n/* \\brief Set Ref counts for node entries for forward graph */\nvoid SetForwardRefCounts(nnvm::Graph* fwd_graph) {\n  const auto& idx = fwd_graph->indexed_graph();\n\n  std::vector<uint32_t> ref_count(idx.num_node_entries(), 0);\n  for (const auto& i : idx.input_nodes())\n    ++ref_count[idx.entry_id(i, 0)];\n  for (const auto& i : idx.outputs())\n    ++ref_count[idx.entry_id(i)];\n  for (size_t i = 0; i < idx.num_nodes(); ++i) {\n    for (const auto& j : idx[i].inputs)\n      ++ref_count[idx.entry_id(j)];\n  }\n\n  fwd_graph->attrs[AddPrefix(FORWARD, REF_COUNT)] =\n      std::make_shared<dmlc::any>(std::move(ref_count));\n}\n\n/* \\brief Set Ref counts for node entries for forward graph and full graph */\nvoid SetRefCounts(nnvm::Graph* fwd_graph, const nnvm::Graph& full_graph) {\n  const auto& idx = fwd_graph->indexed_graph();\n  SetForwardRefCounts(fwd_graph);\n\n  size_t num_forward_nodes   = idx.num_nodes();\n  size_t num_forward_entries = idx.num_node_entries();\n\n  const auto& full_idx = full_graph.indexed_graph();\n\n  std::vector<uint32_t> temp_ref_count(full_idx.num_node_entries(), 0);\n  for (size_t i = num_forward_nodes; i < full_idx.num_nodes(); ++i) {\n    for (const auto& j : full_idx[i].inputs) {\n      ++temp_ref_count[full_idx.entry_id(j)];\n    }\n  }\n\n  auto full_ref_count = fwd_graph->GetAttr<std::vector<uint32_t>>(AddPrefix(FORWARD, REF_COUNT));\n  for (size_t i = 0; i < num_forward_entries; ++i)\n    full_ref_count.at(i) += temp_ref_count[i];\n  fwd_graph->attrs[AddPrefix(FULL, REF_COUNT)] =\n      std::make_shared<dmlc::any>(std::move(full_ref_count));\n}\n\nvoid OptimizeGraph(nnvm::Graph* full_graph,\n                   nnvm::Graph* fwd_graph,\n                   nnvm::Graph* grad_graph,\n                   std::vector<size_t>* input_map,\n                   const Context& context,\n                   size_t num_forward_outputs,\n                   const bool inlining) {\n  input_map->resize(full_graph->indexed_graph().input_nodes().size());\n  std::iota(input_map->begin(), input_map->end(), 0);\n#if MXNET_USE_CUDA && !defined(_WIN32)\n  if (context.dev_mask() == kGPU && !inlining && dmlc::GetEnv(\"MXNET_USE_FUSION\", true)) {\n    nnvm::Graph unoptimized_graph;\n    common::CopyGraph(&unoptimized_graph, *full_graph, false);\n\n    if (common::CheckForInputNameDuplicates(unoptimized_graph.indexed_graph())) {\n      *full_graph = exec::FusePointwise(*full_graph, num_forward_outputs);\n      // Fill in input_map - mapping from the new to the original input indices.\n      const auto& original_inputs = unoptimized_graph.indexed_graph().input_nodes();\n      const auto& new_inputs      = full_graph->indexed_graph().input_nodes();\n      if (original_inputs.size() != new_inputs.size()) {\n        LOG(WARNING) << \"Number of inputs after fusion does not match original number of inputs. \"\n                     << \"This is most probably a bug. Disabling fusion for this run.\";\n        *full_graph = unoptimized_graph;\n      } else {\n        std::unordered_map<std::string, size_t> original_input_map;\n        for (size_t i = 0; i < original_inputs.size(); ++i) {\n          auto r = original_input_map.insert(std::make_pair(\n              unoptimized_graph.indexed_graph()[original_inputs[i]].source->attrs.name, i));\n          CHECK(r.second);\n        }\n        for (size_t i = 0; i < new_inputs.size(); ++i) {\n          auto it = original_input_map.find(\n              full_graph->indexed_graph()[new_inputs[i]].source->attrs.name);\n          CHECK(it != original_input_map.end());\n          (*input_map)[i] = it->second;\n        }\n      }\n    } else {\n      LOG(WARNING)\n          << \"Graph contains duplicate names for some of its inputs - fusion is NOT enabled!\";\n    }\n  }\n#else\n  // Only warn user if MXNET_USE_FUSION env var is explicitly set\n  if (context.dev_mask() == kGPU && !inlining && dmlc::GetEnv(\"MXNET_USE_FUSION\", false)) {\n    exec::WarnFusionNotSupported();\n  }\n#endif  // MXNET_USE_CUDA && !defined(_WIN32)\n\n  *fwd_graph         = nnvm::Graph();\n  fwd_graph->outputs = std::vector<nnvm::NodeEntry>(\n      full_graph->outputs.begin(), full_graph->outputs.begin() + num_forward_outputs);\n  *grad_graph         = nnvm::Graph();\n  grad_graph->outputs = std::vector<nnvm::NodeEntry>(\n      full_graph->outputs.begin() + num_forward_outputs, full_graph->outputs.end());\n  SetRefCounts(fwd_graph, *full_graph);\n}\n\n/* \\brief Check if param indices and data indices are set, if not then set data indices */\nvoid SetInputIndices(const nnvm::Graph& fwd_graph,\n                     const mxnet::Tuple<uint32_t>& param_indices,\n                     mxnet::Tuple<uint32_t>* data_indices) DMLC_ATTRIBUTE_UNUSED;\nvoid SetInputIndices(const nnvm::Graph& fwd_graph,\n                     const mxnet::Tuple<uint32_t>& param_indices,\n                     mxnet::Tuple<uint32_t>* data_indices) {\n  const auto& indexed_graph = fwd_graph.indexed_graph();\n  if (data_indices->ndim() || param_indices.ndim()) {\n    CHECK_EQ(data_indices->ndim() + param_indices.ndim(),\n             static_cast<const int>(indexed_graph.input_nodes().size()));\n  } else {\n    std::vector<uint32_t> tmp;\n    tmp.reserve(indexed_graph.input_nodes().size());\n    for (size_t i = 0; i < indexed_graph.input_nodes().size(); ++i) {\n      tmp.emplace_back(i);\n    }\n    data_indices->assign(tmp.begin(), tmp.end());\n  }\n}\n\n}  // namespace\n\n/*! \\brief CachedOp Parameters */\nstruct CachedOpConfig : public dmlc::Parameter<CachedOpConfig> {\n  uint32_t inline_limit;\n  uint32_t forward_bulk_size;\n  uint32_t backward_bulk_size;\n  bool static_alloc;\n  bool static_shape;\n  bool is_dynamic;\n  mxnet::Tuple<uint32_t> data_indices;\n  mxnet::Tuple<uint32_t> param_indices;\n  std::string subgraph;\n  DMLC_DECLARE_PARAMETER(CachedOpConfig) {\n    DMLC_DECLARE_FIELD(static_alloc)\n        .set_default(false)\n        .describe(\n            \"Statically allocate memory to improve speed. \"\n            \"Memory usage may increase.\");\n    DMLC_DECLARE_FIELD(static_shape)\n        .set_default(false)\n        .describe(\n            \"Optimize for invariant input shapes between iterations. \"\n            \"Must also set static_alloc to True. \"\n            \"Change of input shapes is still allowed but slower.\");\n    DMLC_DECLARE_FIELD(inline_limit)\n        .set_default(2)\n        .describe(\"Maximum number of operators that can be inlined.\");\n    DMLC_DECLARE_FIELD(forward_bulk_size)\n        .set_default(Imperative::BulkExecMaxNodeTrainFwd())\n        .describe(\"Segment size of bulk execution during forward pass.\");\n    DMLC_DECLARE_FIELD(backward_bulk_size)\n        .set_default(Imperative::BulkExecMaxNodeTrainBwd())\n        .describe(\"Segment size of bulk execution during backward pass.\");\n    DMLC_DECLARE_FIELD(data_indices)\n        .set_default(mxnet::Tuple<uint32_t>())\n        .describe(\"Position of argument variables.\");\n    DMLC_DECLARE_FIELD(param_indices)\n        .set_default(mxnet::Tuple<uint32_t>())\n        .describe(\"Position of parameters.\");\n    DMLC_DECLARE_FIELD(subgraph)\n        .set_default(std::string(\"\"))\n        .describe(\"JSON string of a subgraph.\");\n    DMLC_DECLARE_FIELD(is_dynamic)\n        .set_default(false)\n        .describe(\"Whether the graph contains dynamic shape operators.\");\n  }\n};\n\nnamespace io {\nclass LazyTransformDataset;\n}\n\nclass CachedOp {\n  using CachedOpMonCallback = std::function<void(const char*, const char*, void*)>;\n\n public:\n  CachedOp(const nnvm::Symbol& sym, const std::vector<std::pair<std::string, std::string>>& flags);\n  virtual ~CachedOp();\n  nnvm::Symbol GetOptimizedSymbol() const;\n  uint32_t num_inputs() const {\n    return fwd_graph_.indexed_graph().input_nodes().size();\n  }\n  uint32_t num_outputs() const {\n    return fwd_graph_.outputs.size();\n  }\n  uint32_t num_backward_inputs() const {\n    return bwd_ograd_dep_.size() + bwd_in_dep_.size() + bwd_out_dep_.size();\n  }\n  uint32_t num_backward_outputs() const {\n    auto& idx = fwd_graph_.indexed_graph();\n    return idx.input_nodes().size() - idx.mutable_input_nodes().size();\n  }\n  std::vector<bool>& save_inputs() {\n    return save_inputs_;\n  }\n  std::vector<bool>& save_outputs() {\n    return save_outputs_;\n  }\n  const std::unordered_set<uint32_t>& mutable_input_nodes() const {\n    return fwd_graph_.indexed_graph().mutable_input_nodes();\n  }\n  virtual std::vector<nnvm::NodeEntry> Gradient(const nnvm::ObjectPtr& node,\n                                                const std::vector<nnvm::NodeEntry>& ograds) const;\n  virtual OpStatePtr Forward(const std::shared_ptr<CachedOp>& op_ptr,\n                             const std::vector<NDArray*>& inputs,\n                             const std::vector<NDArray*>& outputs,\n                             const Context& default_context);\n  virtual void Backward(const bool retain_graph,\n                        const OpStatePtr& state,\n                        const std::vector<NDArray*>& inputs,\n                        const std::vector<OpReqType>& reqs,\n                        const std::vector<NDArray*>& outputs);\n  // backward storage type inference\n  virtual bool BackwardStorageType(const nnvm::NodeAttrs& attrs,\n                                   const int dev_mask,\n                                   DispatchMode* dispatch_mode,\n                                   std::vector<int>* in_attrs,\n                                   std::vector<int>* out_attrs);\n  std::vector<std::string> ListForwardInputNames() const {\n    nnvm::Symbol sym = GetForwardSym();\n    return sym.ListInputNames(nnvm::Symbol::kAll);\n  }\n  std::vector<std::string> ListForwardOutputNames() const {\n    nnvm::Symbol sym = GetForwardSym();\n    return sym.ListOutputNames();\n  }\n  nnvm::Symbol GetForwardSym() const {\n    nnvm::Symbol sym;\n    sym.outputs = fwd_graph_.outputs;\n    return sym;\n  }\n  void RegisterOpHook(const CachedOp::CachedOpMonCallback& callback, bool monitor_all = false);\n\n protected:\n  struct GraphInfo {\n    nnvm::Graph fwd_graph;\n    nnvm::Graph grad_graph;\n    nnvm::Graph full_graph;\n    std::vector<size_t> input_map;  // the original index of an input\n    std::vector<nnvm::NodeEntry> ograd_entries;\n    std::unordered_map<uint32_t, uint32_t> fwd_input_to_grad_output;\n    std::vector<OpReqType> bwd_output_reqs;\n    std::vector<uint32_t> bwd_input_eid;\n  };\n\n  struct CachedOpState {\n    CachedOpState(const Context& context_,\n                  const nnvm::Graph& fwd_graph_,\n                  const nnvm::Graph& full_graph_,\n                  const bool inlining_) {\n      context = context_;\n      nnvm::Symbol sym;\n      sym.outputs = fwd_graph_.outputs;\n      CreateFullGraph(sym.Copy(),\n                      &info.fwd_graph,\n                      &info.grad_graph,\n                      &info.full_graph,\n                      &info.ograd_entries,\n                      &info.fwd_input_to_grad_output);\n\n      OptimizeGraph(&info.full_graph,\n                    &info.fwd_graph,\n                    &info.grad_graph,\n                    &info.input_map,\n                    context_,\n                    fwd_graph_.outputs.size(),\n                    inlining_);\n\n      size_t max_nodes                = info.full_graph.indexed_graph().num_nodes();\n      size_t max_entries              = info.full_graph.indexed_graph().num_node_entries();\n      info.fwd_graph.attrs[\"context\"] = std::make_shared<dmlc::any>(\n          std::vector<Context>(info.fwd_graph.indexed_graph().num_nodes(), context));\n      info.full_graph.attrs[\"context\"] =\n          std::make_shared<dmlc::any>(std::vector<Context>(max_nodes, context));\n\n      buff.resize(max_entries);\n      arrays.resize(max_entries);\n      array_reqs.resize(max_entries);\n      dynamic_entries.resize(max_entries, false);\n      op_states.resize(max_nodes);\n      execs.resize(max_nodes);\n      opr_segs.resize(max_nodes);\n    }\n\n    std::mutex mutex;\n    Context context;\n    GraphInfo info;\n\n    bool recording     = false;\n    bool fwd_alloc     = false;\n    bool bwd_alloc     = false;\n    bool fwd_exec_init = false;\n    bool bwd_exec_init = false;\n\n    std::vector<NDArray> buff;\n    std::vector<NDArray*> arrays;\n    std::vector<NDArray*> arrays_with_in_out;\n    std::vector<OpReqType> array_reqs;\n\n    std::vector<OpStatePtr> op_states;\n    std::vector<std::shared_ptr<exec::OpExecutor>> execs;\n    std::vector<imperative::EngineOprSeg> opr_segs;\n\n    std::vector<bool> dynamic_entries;\n    std::multimap<size_t, NDArray> fwd_reuse_pool;\n    std::multimap<size_t, NDArray> bwd_reuse_pool;\n  };\n\n  OpStatePtr GetCachedOpState(const Context& ctx);\n  bool SetForwardGraph(const Context& default_ctx,\n                       GraphInfo* info,\n                       const bool recording,\n                       const std::vector<NDArray*>& inputs);\n  bool SetBackwardGraph(GraphInfo* info,\n                        const std::vector<OpReqType>& reqs,\n                        const std::vector<NDArray*>& inputs,\n                        bool detect_inplace_addto = false);\n  bool CheckDynamicShapeExists(const Context& default_ctx,\n                               const std::vector<NDArray*>& inputs,\n                               bool erase_result);\n  void StaticAllocMemory(const OpStatePtr& state_ptr, bool recording, bool keep_fwd);\n  void StaticInitExec(const OpStatePtr& state_ptr, bool recording, bool keep_fwd);\n  void StaticRunOps(const Context& default_ctx,\n                    const nnvm::Graph& g,\n                    const OpStatePtr& state_ptr,\n                    const std::vector<NDArray*>& state_arrays,\n                    size_t start_nid,\n                    size_t end_nid);\n  OpStatePtr StaticForward(const Context& default_ctx,\n                           const std::vector<NDArray*>& inputs,\n                           const std::vector<NDArray*>& outputs);\n  struct DynamicRuntime;\n\n private:\n  OpStatePtr DynamicForward(const Context& default_ctx,\n                            const std::vector<NDArray*>& inputs,\n                            const std::vector<NDArray*>& outputs,\n                            bool use_naive_run = false);\n  void DynamicBackward(const bool retain_graph,\n                       const OpStatePtr& op_state,\n                       const std::vector<NDArray*>& inputs,\n                       const std::vector<OpReqType>& reqs,\n                       const std::vector<NDArray*>& outputs);\n  void StaticBackward(const bool retain_graph,\n                      const OpStatePtr& state_ptr,\n                      const std::vector<NDArray*>& inputs,\n                      const std::vector<OpReqType>& reqs,\n                      const std::vector<NDArray*>& outputs);\n  size_t BwdOriginalInput(const std::vector<size_t>& input_map, size_t new_i);\n\n  CachedOpConfig config_;\n  nnvm::Graph fwd_graph_;\n  nnvm::Graph full_graph_;\n  bool inlining_;\n  bool dynamic_shape_checked_;\n  std::vector<nnvm::NodeEntry> ograd_entries_;\n  std::vector<uint32_t> bwd_in_dep_, bwd_out_dep_, bwd_ograd_dep_;\n  std::vector<bool> save_inputs_, save_outputs_;\n  std::vector<OpReqType> bwd_output_reqs_;\n\n  std::function<void(const char*, const char*, NDArrayHandle)> monitor_callback_{nullptr};\n  bool monitor_all_{false};\n\n  std::mutex mutex_;\n  std::unordered_map<Context, std::vector<OpStatePtr>> cached_op_states_;\n\n  friend class ::mxnet::io::LazyTransformDataset;\n  nnvm::Symbol sym_;\n  std::vector<std::pair<std::string, std::string>> flags_;\n};\n\nstruct CachedOp::DynamicRuntime {\n  GraphInfo info;\n  std::vector<NDArray> buff;\n  std::vector<OpStatePtr> op_states;\n};\n\nusing CachedOpPtr = std::shared_ptr<CachedOp>;\n\n}  // namespace mxnet\n#endif  // MXNET_IMPERATIVE_CACHED_OP_H_\n"
  },
  {
    "path": "src/imperative/cached_op_threadsafe.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#include <unordered_set>\n#include <iostream>\n#include \"./imperative_utils.h\"\n#include \"./exec_pass.h\"\n#include \"./cached_op_threadsafe.h\"\n#include \"../profiler/profiler.h\"\n#include \"../operator/operator_common.h\"\n#include \"../operator/subgraph/common.h\"\n\nnamespace mxnet {\n\nDMLC_REGISTER_PARAMETER(CachedOpThreadSafeConfig);\n\nstruct CachedOpThreadSafe::GraphInfo {\n  nnvm::Graph fwd_graph;\n};\n\nstruct CachedOpThreadSafe::DynamicRuntime {\n  GraphInfo info;\n  std::vector<OpStatePtr> op_states;\n};\n\nOpStatePtr CachedOpThreadSafe::GetCachedOpState(const Context& ctx) {\n  for (const auto& i : cached_op_states_[ctx]) {\n    // only create one state per device when not using static memory\n    if (!config_.static_alloc || i.unique()) {\n      return i;\n    }\n  }\n  nnvm::Graph full_graph;\n  auto state_ptr = OpStatePtr::Create<CachedOpState>(ctx, fwd_graph_, full_graph, false);\n\n  cached_op_states_[ctx].push_back(state_ptr);\n  return state_ptr;\n}\n\nCachedOpThreadSafe::CachedOpThreadSafe(\n    const nnvm::Symbol& sym,\n    const std::vector<std::pair<std::string, std::string>>& flags)\n    : CachedOp(sym, flags) {\n  using namespace nnvm;\n  using namespace imperative;\n  static const std::vector<const Op*> zero_ops{Op::Get(\"zeros_like\"), Op::Get(\"_zeros\")};\n  config_.Init(flags);\n\n  if (config_.static_shape) {\n    CHECK(config_.static_alloc) << \"static_alloc must be True when static_shape is True\";\n  }\n\n  // construct forward graph\n  CreateForwardGraph(sym.Copy(), &fwd_graph_);\n  SetForwardRefCounts(&fwd_graph_);\n\n  SetInputIndices(fwd_graph_, config_.param_indices, &config_.data_indices);\n}\n\n/*\n * \\brief Thread safe version of DynamicForward, with thread local buffer\n * used to store intermediate nodes in the graph\n */\nOpStatePtr CachedOpThreadSafe::DynamicForward(const Context& default_ctx,\n                                              const std::vector<NDArray*>& inputs,\n                                              const std::vector<NDArray*>& outputs) {\n  using namespace nnvm;\n  using namespace imperative;\n\n  auto state_ptr = GetCachedOpState(default_ctx);\n  auto op_state  = OpStatePtr::Create<DynamicRuntime>();\n  auto& runtime  = op_state.get_state<DynamicRuntime>();\n  {\n    auto& state = state_ptr.get_state<CachedOpState>();\n    // Need to lock the mutex on the state, this allows\n    // for multi context push of ops to dependency engine.\n    // SetForwardGraph runs infer passes on graphs as well\n    // as the planmemory pass.\n    std::lock_guard<std::mutex> lock(state.mutex);\n    // the below call runs the NNVM graph passes: type inference,\n    // shape inference, storage type inference and if the graph\n    // doesn't have dynamic shapes it also plans and allocates memory\n    // for intermediate and final outputs in the graph\n    SetForwardGraph(default_ctx, &state.info, false, inputs);\n    runtime.info.fwd_graph = state.info.fwd_graph;\n  }\n  nnvm::Graph& g   = runtime.info.fwd_graph;\n  const auto& idx  = g.indexed_graph();\n  size_t max_nodes = runtime.info.fwd_graph.indexed_graph().num_nodes();\n  runtime.op_states.resize(max_nodes);\n  auto& states = runtime.op_states;\n\n  // Allocate entries\n  // This buff is thread local and used to store intermediate\n  // nodes in the graph\n  buff.resize(idx.num_node_entries());\n  states.resize(idx.num_nodes());\n  std::vector<NDArray*> arrays;\n  arrays.reserve(buff.size());\n  for (auto& buffered_array : buff) {\n    arrays.push_back(&buffered_array);\n  }\n  std::vector<OpReqType> array_reqs(arrays.size(), kWriteTo);\n  const auto& dispatch_modes      = g.GetAttr<DispatchModeVector>(\"dispatch_mode\");\n  std::vector<uint32_t> ref_count = g.GetAttr<std::vector<uint32_t>>(\"forward_ref_count\");\n  for (size_t i = 0; i < idx.num_node_entries(); ++i) {\n    if (ref_count[i] == 0)\n      array_reqs[i] = kNullOp;\n  }\n\n  const MemoryPlanVector& mem_plan = g.GetAttr<MemoryPlanVector>(\"forward_mem_plan\");\n  // Collect input output pointers to ndarray into the arrays data structure\n  std::vector<size_t> input_map(inputs.size());\n  std::iota(input_map.begin(), input_map.end(), 0);\n  CollectInputOutputNDRefs(g, inputs, input_map, outputs, &arrays);\n  // The SetForwardGraph call in DynamicForward runs the memory planning phase\n  // and allocates storage for intermediate and final outputs of the graph\n  // We need to still create NDArrays (pointer data structure), based on this\n  // allocated memory from memory planning phase. The CreateGraphNDs below does\n  // that.\n  CreateGraphNDs(g, default_ctx, mem_plan, &array_reqs, &arrays);\n  // Invokes operators in the graph in a topologically sorted manner\n  RunGraph(false,\n           idx,\n           arrays,\n           0,\n           idx.num_nodes(),\n           std::move(array_reqs),\n           std::move(ref_count),\n           &states,\n           dispatch_modes,\n           false);\n  return op_state;\n}\n\nOpStatePtr CachedOpThreadSafe::Forward(const std::shared_ptr<CachedOp>& op_ptr,\n                                       const std::vector<NDArray*>& inputs,\n                                       const std::vector<NDArray*>& outputs,\n                                       const Context& default_ctx) {\n  // Acquiring lock on the mutex in forward\n  // Without this there are issues with static_forward,\n  // specifically with static_shape=True and dynamic_forward.\n  // Adding the lock here for safety,\n  // The perf hit would be acceptable because this involves just pushing\n  // ops to engine and not actual execution\n  // We are putting this lock here because without this there is a hang\n  // in the accept4 call in CUDA lib.\n  // TODO(anirudh2290): Investigate this issue more as it also prevents parallel\n  // push of ops for different contexts\n  std::lock_guard<std::mutex> lock(mutex_);\n  CHECK_EQ(inputs.size(), num_inputs());\n  const auto& idx = fwd_graph_.indexed_graph();\n  for (size_t i = 0; i < inputs.size(); ++i) {\n    CHECK_EQ(inputs[i]->ctx(), default_ctx)\n        << \"CachedOp requires all inputs to live on the same context. But \"\n        << idx[idx.input_nodes()[0]].source->attrs.name << \" is on \" << default_ctx << \" while \"\n        << idx[idx.input_nodes()[i]].source->attrs.name << \" is on \" << inputs[i]->ctx();\n  }\n\n  int prev_bulk_size = Engine::Get()->set_bulk_size(config_.forward_bulk_size);\n  OpStatePtr op_state;\n  try {\n    if (CheckDynamicShapeExists(default_ctx, inputs, true)) {\n      LOG(FATAL) << \"Dynamic shapes aren't supported with thread-safe cached op\";\n    }\n    if (config_.static_alloc) {\n      op_state = StaticForward(default_ctx, inputs, outputs);\n    } else {\n      op_state = DynamicForward(default_ctx, inputs, outputs);\n    }\n  } catch (const dmlc::Error& e) {\n    Engine::Get()->set_bulk_size(prev_bulk_size);\n    throw e;\n  }\n  Engine::Get()->set_bulk_size(prev_bulk_size);\n  return op_state;\n}\n\nstruct CachedOpThreadSafeActualState {\n  std::shared_ptr<CachedOp> op;\n  OpStatePtr forward_state;\n\n  explicit CachedOpThreadSafeActualState(std::shared_ptr<CachedOp> op) {\n    this->op = op;\n  }\n};\nOpStatePtr CreateCachedOpThreadSafeState(const NodeAttrs& attrs,\n                                         Context ctx,\n                                         const mxnet::ShapeVector& in_shapes,\n                                         const std::vector<int>& in_types) {\n  const CachedOpThreadSafePtr& op = nnvm::get<CachedOpThreadSafePtr>(attrs.parsed);\n  return OpStatePtr::Create<CachedOpThreadSafeActualState>(op);\n}\n\nvoid CachedOpThreadSafeForward(const OpStatePtr& state_ptr,\n                               const OpContext& ctx,\n                               const std::vector<NDArray>& inputs,\n                               const std::vector<OpReqType>& req,\n                               const std::vector<NDArray>& outputs) {\n  CachedOpThreadSafeActualState& s = state_ptr.get_state<CachedOpThreadSafeActualState>();\n  std::vector<NDArray> in_bufs     = inputs;\n  std::vector<NDArray> out_bufs    = outputs;\n  std::vector<NDArray*> in_ptrs(in_bufs.size());\n  std::vector<NDArray*> out_ptrs(out_bufs.size());\n  for (size_t i = 0; i < in_ptrs.size(); i++)\n    in_ptrs[i] = &in_bufs[i];\n  for (size_t i = 0; i < out_ptrs.size(); i++)\n    out_ptrs[i] = &out_bufs[i];\n\n  // Set is_recording correct for the imperative executor.\n  CHECK(!ctx.need_grad) << \"Only inference use case supported with thread safe cached op\";\n  CHECK(!ctx.is_train) << \"Only inference use case supported with thread safe cached op\";\n  CHECK(inputs.size() > 0) << \"thread safe cached op requires at least one input\";\n  Context default_ctx = inputs[0].ctx();\n  s.forward_state     = s.op->Forward(nullptr, in_ptrs, out_ptrs, default_ctx);\n  // The arrays in out_ptrs may be changed by CachedOp.\n  // If it is, we need to copy data back.\n  for (size_t i = 0; i < out_bufs.size(); i++)\n    if (!out_bufs[i].IsSame(outputs[i]))\n      CopyFromTo(out_bufs[i], outputs[i]);\n}\n\nvoid CachedOpThreadSafeParamParser(nnvm::NodeAttrs* attrs) {\n  CachedOpThreadSafeConfig param;\n  try {\n    param.Init(attrs->dict);\n  } catch (const dmlc::ParamError& e) {\n    std::ostringstream os;\n    os << e.what();\n    os << \", in operator \" << attrs->op->name << \"(\"\n       << \"name=\\\"\" << attrs->name << \"\\\"\";\n    for (const auto& k : attrs->dict) {\n      os << \", \" << k.first << \"=\\\"\" << k.second << \"\\\"\";\n    }\n    os << \")\";\n    throw dmlc::ParamError(os.str());\n  }\n}\nCachedOpThreadSafe::~CachedOpThreadSafe() = default;\n\nNNVM_REGISTER_OP(_CachedOpThreadSafe)\n    .set_num_inputs([](const NodeAttrs& attrs) {\n      const CachedOpThreadSafePtr& op = nnvm::get<CachedOpThreadSafePtr>(attrs.parsed);\n      return op->num_inputs();\n    })\n    .set_num_outputs([](const NodeAttrs& attrs) {\n      const CachedOpThreadSafePtr& op = nnvm::get<CachedOpThreadSafePtr>(attrs.parsed);\n      return op->num_outputs();\n    })\n    .set_attr_parser(CachedOpThreadSafeParamParser)\n    .set_attr<nnvm::FListInputNames>(\"FListInputNames\",\n                                     [](const nnvm::NodeAttrs& attrs) {\n                                       const CachedOpThreadSafePtr& op =\n                                           nnvm::get<CachedOpThreadSafePtr>(attrs.parsed);\n                                       return op->ListForwardInputNames();\n                                     })\n    .set_attr<nnvm::FListOutputNames>(\"FListOutputNames\",\n                                      [](const nnvm::NodeAttrs& attrs) {\n                                        const CachedOpThreadSafePtr& op =\n                                            nnvm::get<CachedOpThreadSafePtr>(attrs.parsed);\n                                        return op->ListForwardOutputNames();\n                                      })\n    .set_attr<FCreateOpState>(\"FCreateOpState\", CreateCachedOpThreadSafeState)\n    .set_attr<mxnet::FInferShape>(\"FInferShape\",\n                                  [](const nnvm::NodeAttrs& attrs,\n                                     mxnet::ShapeVector* in_shapes,\n                                     mxnet::ShapeVector* out_shapes) {\n                                    const CachedOpThreadSafePtr& op =\n                                        nnvm::get<CachedOpThreadSafePtr>(attrs.parsed);\n                                    return op::DefaultSubgraphOpShapeHelper(\n                                        op->GetForwardSym(), in_shapes, out_shapes);\n                                  })\n    .set_attr<nnvm::FInferType>(\n        \"FInferType\",\n        [](const nnvm::NodeAttrs& attrs, std::vector<int>* in_types, std::vector<int>* out_types) {\n          const CachedOpThreadSafePtr& op = nnvm::get<CachedOpThreadSafePtr>(attrs.parsed);\n          return op::DefaultSubgraphOpTypeHelper(op->GetForwardSym(), in_types, out_types);\n        })\n    .set_attr<FInferStorageType>(\n        \"FInferStorageType\",\n        [](const nnvm::NodeAttrs& attrs,\n           const int dev_mask,\n           DispatchMode* dispatch_mode,\n           std::vector<int>* in_stypes,\n           std::vector<int>* out_stypes) {\n          const CachedOpThreadSafePtr& op = nnvm::get<CachedOpThreadSafePtr>(attrs.parsed);\n          return op::DefaultSubgraphOpStorageTypeHelper(\n              op->GetForwardSym(), dev_mask, dispatch_mode, in_stypes, out_stypes);\n        })\n    .set_attr<FStatefulComputeEx>(\"FStatefulComputeEx<cpu>\", CachedOpThreadSafeForward)\n    .set_attr<FStatefulComputeEx>(\"FStatefulComputeEx<gpu>\", CachedOpThreadSafeForward)\n    .set_attr<nnvm::FMutateInputs>(\"FMutateInputs\",\n                                   [](const nnvm::NodeAttrs& attrs) {\n                                     const CachedOpThreadSafePtr& op =\n                                         nnvm::get<CachedOpThreadSafePtr>(attrs.parsed);\n                                     return op::DefaultSubgraphOpMutableInputsHelper(\n                                         op->GetForwardSym());\n                                   })\n    .set_attr<FResourceRequest>(\"FResourceRequest\",\n                                [](const nnvm::NodeAttrs& attrs) {\n                                  const CachedOpThreadSafePtr& op =\n                                      nnvm::get<CachedOpThreadSafePtr>(attrs.parsed);\n                                  return op::DefaultSubgraphOpResourceRequestHelper(\n                                      op->GetForwardSym());\n                                })\n    .set_attr<FExecType>(\"FExecType\", op::DefaultSubgraphOpExecType)\n    .add_argument(\"data\", \"NDArray-or-Symbol[]\", \"input data list\");\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/imperative/cached_op_threadsafe.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n// Threadsafe and minimal functionality cached op version for Inference\n// lot of code reused from cached_op.h\n#ifndef MXNET_IMPERATIVE_CACHED_OP_THREADSAFE_H_\n#define MXNET_IMPERATIVE_CACHED_OP_THREADSAFE_H_\n\n#include <mxnet/imperative.h>\n#include <vector>\n#include <atomic>\n#include <utility>\n#include <string>\n#include <unordered_map>\n#include \"./cached_op.h\"\n\nnamespace mxnet {\n/*! \\brief CachedOp Parameters*/\nstruct CachedOpThreadSafeConfig : public dmlc::Parameter<CachedOpThreadSafeConfig> {\n  // keeping the config minimal\n  // inlining, bulking, dynamic shapes, static allocing and shaping not\n  // supported\n  // data_indices indicates which of the indices from the arguments are data\n  mxnet::Tuple<uint32_t> data_indices;\n  // param_indices indicates which of the indices from the arguments are params\n  mxnet::Tuple<uint32_t> param_indices;\n  // decides the bulk size for dynamic forward\n  uint32_t forward_bulk_size;\n  bool static_alloc;\n  bool static_shape;\n  DMLC_DECLARE_PARAMETER(CachedOpThreadSafeConfig) {\n    DMLC_DECLARE_FIELD(static_alloc)\n        .set_default(false)\n        .describe(\n            \"Statically allocate memory to improve speed. \"\n            \"Memory usage may increase.\");\n    DMLC_DECLARE_FIELD(static_shape)\n        .set_default(false)\n        .describe(\n            \"Optimize for invariant input shapes between iterations. \"\n            \"Must also set static_alloc to True. \"\n            \"Change of input shapes is still allowed but slower.\");\n    DMLC_DECLARE_FIELD(forward_bulk_size)\n        .set_default(Imperative::BulkExecMaxNodeTrainFwd())\n        .describe(\"Segment size of bulk execution during dynamic forward\");\n    DMLC_DECLARE_FIELD(data_indices)\n        .set_default(mxnet::Tuple<uint32_t>())\n        .describe(\"Position of argument variables.\");\n    DMLC_DECLARE_FIELD(param_indices)\n        .set_default(mxnet::Tuple<uint32_t>())\n        .describe(\"Position of parameters.\");\n  }\n};\n\n// Thread local buff to store internal states of the graph\n// Used in dynamic_forward\n#if DMLC_CXX11_THREAD_LOCAL\nstatic thread_local std::vector<NDArray> buff;\n#else\nstatic MX_THREAD_LOCAL std::vector<NDArray> buff;\n#endif\n\nclass CachedOpThreadSafe : public CachedOp {\n public:\n  CachedOpThreadSafe(const nnvm::Symbol& sym,\n                     const std::vector<std::pair<std::string, std::string>>& flags);\n  ~CachedOpThreadSafe();\n  uint32_t num_inputs() const {\n    return fwd_graph_.indexed_graph().input_nodes().size();\n  }\n  uint32_t num_outputs() const {\n    return fwd_graph_.outputs.size();\n  }\n  const std::unordered_set<uint32_t>& mutable_input_nodes() const {\n    return fwd_graph_.indexed_graph().mutable_input_nodes();\n  }\n  OpStatePtr Forward(const std::shared_ptr<CachedOp>& op_ptr,\n                     const std::vector<NDArray*>& inputs,\n                     const std::vector<NDArray*>& outputs,\n                     const Context& default_ctx);\n  std::vector<std::string> ListForwardInputNames() const {\n    nnvm::Symbol sym = GetForwardSym();\n    return sym.ListInputNames(nnvm::Symbol::kAll);\n  }\n  std::vector<std::string> ListForwardOutputNames() const {\n    nnvm::Symbol sym = GetForwardSym();\n    return sym.ListOutputNames();\n  }\n  nnvm::Symbol GetForwardSym() const {\n    nnvm::Symbol sym;\n    sym.outputs = fwd_graph_.outputs;\n    return sym;\n  }\n\n  struct GraphInfo;\n\n private:\n  struct DynamicRuntime;\n\n  OpStatePtr GetCachedOpState(const Context& ctx);\n\n  OpStatePtr DynamicForward(const Context& default_ctx,\n                            const std::vector<NDArray*>& inputs,\n                            const std::vector<NDArray*>& outputs);\n\n  CachedOpThreadSafeConfig config_;\n  nnvm::Graph fwd_graph_;\n  std::mutex mutex_;\n  std::unordered_map<Context, std::vector<OpStatePtr>> cached_op_states_;\n};\n\nusing CachedOpThreadSafePtr = std::shared_ptr<CachedOpThreadSafe>;\n\n}  // namespace mxnet\n#endif  // MXNET_IMPERATIVE_CACHED_OP_THREADSAFE_H_\n"
  },
  {
    "path": "src/imperative/cuda_graphs.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * Copyright (c) 2020 by Contributors\n * \\file cuda_graphs.h\n * \\brief Wrappers for use of CUDA Graphs API\n */\n#ifndef MXNET_IMPERATIVE_CUDA_GRAPHS_H_\n#define MXNET_IMPERATIVE_CUDA_GRAPHS_H_\n\n#include <mxnet/base.h>\n#include <vector>\n#include <string>\n#include <map>\n#include <set>\n#include <sstream>\n\n#include \"./exec_pass.h\"\n#include \"../common/cuda/utils.h\"\n\n#if MXNET_USE_CUDA\n#define CUDA_GRAPHS_AVAILABLE (CUDA_VERSION >= 10020)\n#else\n#define CUDA_GRAPHS_AVAILABLE (0)\n#endif\n\n#if CUDA_GRAPHS_AVAILABLE\n\nnamespace mxnet {\nnamespace cuda_graphs {\n\ninline std::string CudaDim3ToString(const dim3& dims) {\n  std::stringstream ss;\n  if (dims.z != 1)\n    ss << \"(\" << dims.x << \",\" << dims.y << \",\" << dims.z << \")\";\n  else if (dims.y != 1)\n    ss << \"(\" << dims.x << \",\" << dims.y << \")\";\n  else\n    ss << \"(\" << dims.x << \")\";\n  return ss.str();\n}\n\n// Return the list of CUDA Graph nodes from a graph\ninline std::vector<cudaGraphNode_t> GetCudaGraphNodes(cudaGraph_t cuda_graph) {\n  size_t numNodes;\n  CUDA_CALL(cudaGraphGetNodes(cuda_graph, static_cast<cudaGraphNode_t*>(nullptr), &numNodes));\n  if (numNodes == 0)\n    return std::vector<cudaGraphNode_t>();\n  std::vector<cudaGraphNode_t> graphNodes(numNodes);\n  CUDA_CALL(cudaGraphGetNodes(cuda_graph, graphNodes.data(), &numNodes));\n  return graphNodes;\n}\n\n// Create a description of a CUDA Graph node\ninline std::string CudaGraphNodeToString(const cudaGraphNode_t node) {\n  std::stringstream ss;\n\n  // The following introspection calls are made through the driver API in order to bypass\n  // problems that would arise if multiple statically-linked copies of the runtime exist.\n\n  CUgraphNode cu_node = node;\n  CUgraphNodeType t;\n  CUDA_DRIVER_CALL(cuGraphNodeGetType(cu_node, &t));\n  switch (t) {\n    case CU_GRAPH_NODE_TYPE_KERNEL: {\n      CUDA_KERNEL_NODE_PARAMS kparams;\n      auto err = cuGraphKernelNodeGetParams(cu_node, &kparams);\n      if (err == CUDA_SUCCESS) {\n        ss << \"GPUKernel@\" << kparams.func;\n        dim3 gridDim(kparams.gridDimX, kparams.gridDimY, kparams.gridDimZ);\n        dim3 blockDim(kparams.blockDimX, kparams.blockDimY, kparams.blockDimZ);\n        ss << \"<<<gridDim=\" << CudaDim3ToString(gridDim)\n           << \", blkDim=\" << CudaDim3ToString(blockDim) << \">>>\";\n        ss << \"(...\";\n        if (kparams.sharedMemBytes != 0)\n          ss << \", dynSharedMemBytes=\" << kparams.sharedMemBytes;\n        ss << \")\";\n      } else {\n        ss << \"GPU Kernel: cuGraphKernelNodeGetParams() fails with \" << err;\n      }\n    } break;\n    case CU_GRAPH_NODE_TYPE_MEMCPY: {\n      cudaMemcpy3DParms mparams = {};\n      CUDA_CALL(cudaGraphMemcpyNodeGetParams(node, &mparams));\n      // If memcpy is seen, return without setting up runnable executor\n      switch (mparams.kind) {\n        case cudaMemcpyHostToHost:\n          ss << \"Host->Host \";\n          break;\n        case cudaMemcpyHostToDevice:\n          ss << \"Host->Device \";\n          break;\n        case cudaMemcpyDeviceToHost:\n          ss << \"Device->Host \";\n          break;\n        case cudaMemcpyDeviceToDevice:\n          ss << \"Device->Device \";\n          break;\n        default:\n          break;\n      }\n      ss << \"Memcpy\";\n    } break;\n    case CU_GRAPH_NODE_TYPE_MEMSET: {\n      cudaMemsetParams mparams = {};\n      CUDA_CALL(cudaGraphMemsetNodeGetParams(node, &mparams));\n      if (mparams.height == 1 && mparams.elementSize == 1) {\n        ss << \"cudaMemset(devPtr=\" << mparams.dst << \", value=\" << mparams.value\n           << \", count=\" << mparams.width << \")\";\n      } else {\n        if (mparams.elementSize == 1)\n          ss << \"cudaMemset2D\";\n        else\n          ss << \"MemSet<elemBytes=\" << mparams.elementSize << \">\";\n        ss << \"(devPtr=\" << mparams.dst << \", pitch=\" << mparams.pitch\n           << \", value=\" << mparams.value << \", width=\" << mparams.width\n           << \", height=\" << mparams.height << \")\";\n      }\n    } break;\n    case CU_GRAPH_NODE_TYPE_HOST:\n      ss << \"Host (executable) node\";\n      break;\n    case CU_GRAPH_NODE_TYPE_GRAPH:\n      ss << \"Node which executes an embedded graph\";\n      break;\n    case CU_GRAPH_NODE_TYPE_EMPTY:\n      ss << \"Empty (no-op) node\";\n      break;\n    default:\n      ss << \"Unknown/Invalid node type \" << t;\n  }\n  return ss.str();\n}\n\n// CUDA Graphs are managed in RAII fashion by smart pointers below.\n// Function objects (preferred for readability) provide the deleter function.\nclass CudaGraphDeleter {\n public:\n  void operator()(cudaGraph_t graph) {\n    if (graph != nullptr)\n      CUDA_CALL(cudaGraphDestroy(graph));\n  }\n};\n\n// CUDA Graphs Executors are managed in RAII fashion by smart pointers below.\n// Function objects (preferred for readability) provide the deleter function.\nclass CudaGraphExecDeleter {\n public:\n  void operator()(cudaGraphExec_t graph_exec) {\n    if (graph_exec != nullptr)\n      CUDA_CALL(cudaGraphExecDestroy(graph_exec));\n  }\n};\n\n// A CUDA Graphs executor for a portion of an Operator Segment (i.e. a 'SubSegment'),\n// characterized by a starting index in the OpExecutor list and a number of ops.\nclass CudaGraphsSubSegExec {\n public:\n  CudaGraphsSubSegExec(const std::vector<std::shared_ptr<exec::OpExecutor>>& exec_list,\n                       const RunContext& rctx,\n                       bool is_gpu,\n                       bool verbose,\n                       int from_op_idx,\n                       int num_ops,\n                       bool ops_are_cuda_graph_compatible = true)\n      : from_op_idx_(from_op_idx),\n        num_ops_(num_ops),\n        graph_(nullptr),\n        graph_exec_(nullptr),\n        graph_exec_id_(0) {\n    if (ops_are_cuda_graph_compatible) {\n      MakeGraph(exec_list, rctx, is_gpu, verbose, from_op_idx, num_ops);\n      MakeGraphExec(exec_list, rctx);\n    }\n  }\n\n  void Update(const std::vector<std::shared_ptr<exec::OpExecutor>>& exec_list,\n              const RunContext& rctx,\n              bool is_gpu,\n              bool verbose) {\n    // Current executor should be Runnable with the same parameters\n    CHECK(IsRunnable());\n    MakeGraph(exec_list, rctx, is_gpu, verbose, from_op_idx_, num_ops_);\n\n    cudaGraphExecUpdateResult update_result = cudaGraphExecUpdateError;\n    cudaGraphNode_t error_node;\n    cudaError_t err =\n        cudaGraphExecUpdate(graph_exec_.get(), graph_.get(), &error_node, &update_result);\n    switch (err) {\n      case cudaErrorGraphExecUpdateFailure:\n        MakeGraphExec(exec_list, rctx);\n        break;\n      case cudaSuccess:\n        CHECK_EQ(update_result, cudaGraphExecUpdateSuccess);\n        break;\n      default:\n        // Respond normally to unusual cudaGraphExecUpdate() ret vals\n        CUDA_CALL(err);\n    }\n  }\n\n  void RunSubSeg(const std::vector<std::shared_ptr<exec::OpExecutor>>& exec_list,\n                 const RunContext& rctx,\n                 bool is_gpu) {\n    if (IsRunnable()) {\n      auto s                  = rctx.get_stream<gpu>();\n      const cudaStream_t cu_s = mshadow::Stream<gpu>::GetStream(s);\n      CUDA_CALL(cudaGraphLaunch(graph_exec_.get(), cu_s));\n    } else {\n      // No CUDA Graph could be made for this portion of the OpSegment.  Run conventionally.\n      for (int i = 0; i != num_ops_; ++i)\n        exec_list[from_op_idx_ + i]->Run(rctx, is_gpu);\n    }\n  }\n\n  bool IsRunnable() {\n    return graph_exec_ != nullptr;\n  }\n\n  int NumGraphNodes() {\n    size_t numNodes;\n    CUDA_CALL(cudaGraphGetNodes(graph_.get(), static_cast<cudaGraphNode_t*>(nullptr), &numNodes));\n    return numNodes;\n  }\n\n private:\n  void MakeGraph(const std::vector<std::shared_ptr<exec::OpExecutor>>& exec_list,\n                 const RunContext& rctx,\n                 bool is_gpu,\n                 bool verbose,\n                 int from_op_idx,\n                 int num_ops) {\n    auto s                  = rctx.get_stream<gpu>();\n    const cudaStream_t cu_s = mshadow::Stream<gpu>::GetStream(s);\n    // Create CUDA Graph\n    // Use of cudaStreamCaptureModeThreadLocal allows other threads like GPU Copy workers\n    // to sync their streams without disturbing this capture.\n    CUDA_CALL(cudaStreamBeginCapture(cu_s, cudaStreamCaptureModeThreadLocal));\n    // Run those oprs in the sub segment while capturing- no actual GPU work is launched.\n    for (int i = 0; i != num_ops; ++i)\n      exec_list[from_op_idx + i]->Run(rctx, is_gpu);\n    cudaGraph_t cuda_graph = nullptr;\n    CUDA_CALL(cudaStreamEndCapture(cu_s, &cuda_graph));\n    graph_.reset(cuda_graph, CudaGraphDeleter());\n\n    if (verbose) {\n      std::vector<cudaGraphNode_t> graph_nodes = GetCudaGraphNodes(cuda_graph);\n      size_t num_nodes                         = graph_nodes.size();\n      LOG(INFO) << \"  Graph has \" << num_nodes << \" nodes:\";\n      for (size_t i = 0; i != num_nodes; ++i) {\n        LOG(INFO) << \"    node \" << i << \" = \" << CudaGraphNodeToString(graph_nodes[i]);\n      }\n    }\n  }\n\n  void MakeGraphExec(const std::vector<std::shared_ptr<exec::OpExecutor>>& exec_list,\n                     const RunContext& rctx) {\n    // Note that this routine is not invoked when a graph executor is merely updated.\n    cudaGraphExec_t cuda_graph_exec;\n    cudaGraphNode_t error_node;\n    char log_buffer[1000];\n\n    CUDA_CALL(cudaGraphInstantiate(&cuda_graph_exec, graph_.get(), &error_node, log_buffer, 1000));\n    graph_exec_.reset(cuda_graph_exec, CudaGraphExecDeleter());\n\n    // At this point we have a CUDA Graph executor\n    static int num_graph_creations = 0;\n    graph_exec_id_                 = num_graph_creations++;\n\n    static size_t max_log_entries = dmlc::GetEnv(\"MXNET_CUDA_GRAPHS_MAX_LOG_ENTRIES\", 0);\n    if (graph_exec_id_ < max_log_entries) {\n      LOG(INFO) << \"Created CUDA graph \" << graph_exec_id_;\n      if (num_graph_creations == max_log_entries)\n        LOG(INFO) << \"Further CUDA graph creation log messages are suppressed.\";\n    }\n    // Create a .dot file for graph visualization if requested\n    static std::string dotfile_base = dmlc::GetEnv(\"MXNET_CUDA_GRAPHS_DBG_FILE\", std::string());\n    if (dotfile_base.size() > 0) {\n#if CUDA_VERSION >= 11030\n      static int dotfile_flags = dmlc::GetEnv(\"MXNET_CUDA_GRAPHS_DBG_FILE_FLAGS\",\n                                              static_cast<int>(cudaGraphDebugDotFlagsVerbose));\n      std::ostringstream filename;\n      const bool is_train = exec_list.size() > 0 && exec_list[0]->op_ctx.is_train;\n      int dev_id          = rctx.ctx.dev_id;\n      filename << dotfile_base << \"-\"\n               << \"dev\" << dev_id << \"-\" << (is_train ? \"trn\" : \"inf\") << \"-\" << graph_exec_id_\n               << \".dot\";\n      CUDA_CALL(cudaGraphDebugDotPrint(graph_.get(), filename.str().c_str(), dotfile_flags));\n#else\n      [[maybe_unused]] static bool dot_file_unsupported = []() {  // NOLINT\n        LOG(INFO) << \"MXNET_CUDA_GRAPHS_DBG_FILE setting ignored- requires CUDA version >= 11.3\";\n        return true;\n      }();\n#endif  // CUDA_VERSION >= 11030\n    }\n  }\n\n  int from_op_idx_;\n  int num_ops_;\n  using cudaGraphStruct_t     = typename std::remove_pointer<cudaGraph_t>::type;\n  using cudaGraphExecStruct_t = typename std::remove_pointer<cudaGraphExec_t>::type;\n  std::shared_ptr<cudaGraphStruct_t> graph_;\n  std::shared_ptr<cudaGraphExecStruct_t> graph_exec_;\n  size_t graph_exec_id_;\n};\n\n// The CudaGraph executor and associated Tempspace ptrs for which it is valid.\nstruct CudaGraphInfo {\n  std::vector<CudaGraphsSubSegExec> cuda_graph_subseg_execs;\n  bool has_been_run_conventionally = false;\n  std::vector<void*> tempspace_dptrs;\n};\n// A CUDA graph is maintained for every combination of cudaStream_t (i.e. GPU Worker) and\n// the state of the is_train flag of the OpContext.  If the tempspace_dptrs change, we\n// don't expect to ever see the old tempspace_dptrs config again, so we discard the CUDA graph.\nstruct CudaGraphCacheKey {\n  cudaStream_t cu_s;\n  bool is_train;\n  // overload '<' so CudaGraphCacheKey can be used as a std::map key\n  bool operator<(const CudaGraphCacheKey& other) const {\n    return cu_s < other.cu_s || (cu_s == other.cu_s && is_train < other.is_train);\n  }\n};\nusing CudaGraphCache = std::map<CudaGraphCacheKey, CudaGraphInfo>;\n\nclass CudaGraphsExec {\n public:\n  CudaGraphsExec(const std::vector<std::shared_ptr<exec::OpExecutor>>& exec_list,\n                 bool is_gpu,\n                 const char* opr_names)\n      : verbose_(false), is_enabled_(false) {\n    opr_names_ = opr_names ? std::string(opr_names) : std::string();\n    if (is_gpu) {\n      is_enabled_ = dmlc::GetEnv(\"MXNET_ENABLE_CUDA_GRAPHS\", false);\n      verbose_    = dmlc::GetEnv(\"MXNET_CUDA_GRAPHS_VERBOSE\", false);\n      SetTempSpaces(exec_list);\n    }\n  }\n\n  void RunAll(const std::vector<std::shared_ptr<exec::OpExecutor>>& exec_list,\n              const RunContext& rctx,\n              bool is_gpu) {\n    // If this a CPU op or CUDA Graphs use isn't possible, run normally and return\n    if (!is_gpu || !is_enabled_) {\n      // Run all opr in the sub-graph\n      exec::OpExecutor::RunAll(exec_list, rctx, is_gpu);\n      return;\n    }\n\n    // Also if we're in a warm-up period where tempspace pointers are likely\n    // to change, run normally and return\n    auto s                  = rctx.get_stream<gpu>();\n    const cudaStream_t cu_s = mshadow::Stream<gpu>::GetStream(s);\n    // All the ops in the bulked segment will have the same setting of is_train as the first op\n    const bool is_train         = exec_list.size() > 0 && exec_list[0]->op_ctx.is_train;\n    const CudaGraphCacheKey key = {cu_s, is_train};\n    // Look-up the CUDA Graph info for this combo of stream and is_train setting\n    // This may create a default-initialized new entry.\n    auto& cuda_graph_info = cache_[key];\n    if (!cuda_graph_info.has_been_run_conventionally) {\n      // Run all opr in the sub-graph\n      exec::OpExecutor::RunAll(exec_list, rctx, is_gpu);\n      cuda_graph_info.has_been_run_conventionally = true;\n      return;\n    }\n\n    // At this point we will launch one or more CUDA Graphs through CUDA Graphs 'executors'\n    //     (there might be more than one executor if some ops in the segment are not capturable)\n    auto before_exec_tempspace_ptrs = GetGPUTempspacePtrs(s);\n\n    // Executors exist, but the tempspace pts have changed, so update them in-place via 'recapture'.\n    if (cuda_graph_info.cuda_graph_subseg_execs.size() > 0 &&\n        cuda_graph_info.tempspace_dptrs != before_exec_tempspace_ptrs) {\n      // Update all runnable executors.  Non-runnable executors launch their ops conventionally.\n      for (auto& subseg_exec : cuda_graph_info.cuda_graph_subseg_execs) {\n        if (subseg_exec.IsRunnable())\n          subseg_exec.Update(exec_list, rctx, is_gpu, verbose_);\n      }\n    } else if (cuda_graph_info.cuda_graph_subseg_execs.size() == 0) {\n      // No executors exist yet, so create them.\n      if (verbose_)\n        LOG(INFO) << \"Capturing CUDA graph of op segment \" << opr_names_;\n      // Make one or more CUDA Graphs, avoiding ops that are not compatible.\n      for (size_t first_op_idx = 0; first_op_idx != exec_list.size();) {\n        int num_good_ops = 0;\n        for (size_t last_op_idx = first_op_idx; last_op_idx != exec_list.size(); ++last_op_idx) {\n          if (OpOK(exec_list[last_op_idx]))\n            num_good_ops++;\n          else\n            break;\n        }\n        if (num_good_ops > 0) {\n          CreateSubExecOverRegion(exec_list,\n                                  rctx,\n                                  is_gpu,\n                                  first_op_idx,\n                                  first_op_idx + num_good_ops,\n                                  &cuda_graph_info.cuda_graph_subseg_execs);\n          first_op_idx += num_good_ops;\n        }\n        if (first_op_idx != exec_list.size()) {\n          // We had to have hit an op that was not OK.\n          if (verbose_) {\n            LOG(INFO) << \"Bypassing notOK op segment[\" << first_op_idx << \",\" << first_op_idx << \"]\"\n                      << \" of op segment \" << opr_names_;\n          }\n          CudaGraphsSubSegExec notOK_opseg(exec_list, rctx, is_gpu, false, first_op_idx, 1, false);\n          cuda_graph_info.cuda_graph_subseg_execs.push_back(notOK_opseg);\n          first_op_idx++;\n        }\n      }\n      // During graph capture, the ops may be asking for the tempworkspace.  This should\n      // not alter the base pointers, since this op seg has been executed before on this\n      // stream (i.e. on this gpu worker).  Safest to double-check this though.\n      auto after_capture_tempspace_ptrs = GetGPUTempspacePtrs(s);\n      if (before_exec_tempspace_ptrs != after_capture_tempspace_ptrs)\n        LOG(FATAL) << \"Internal error: saw change in TempSpace ptrs during CUDA graph use.\";\n      cuda_graph_info.tempspace_dptrs = before_exec_tempspace_ptrs;\n    }\n    // Now execute the CUDA Graph that we either just created or looked-up in the cache.\n    if (verbose_) {\n      int runnable_execs = 0;\n      int bypassed_ops   = 0;\n      for (auto& subseg_exec : cuda_graph_info.cuda_graph_subseg_execs) {\n        if (subseg_exec.IsRunnable()) {\n          LOG(INFO) << \"Launching captured graph with \" << subseg_exec.NumGraphNodes() << \" nodes.\";\n          runnable_execs++;\n        } else {\n          bypassed_ops++;\n        }\n      }\n      if (bypassed_ops > 0)\n        LOG(INFO) << \"    (bypassing \" << bypassed_ops << \" un-capturable ops)\";\n    }\n    for (auto& subseg_exec : cuda_graph_info.cuda_graph_subseg_execs)\n      subseg_exec.RunSubSeg(exec_list, rctx, is_gpu);\n  }\n\n private:\n  // Make a CUDA Graph of the region of ops [from_op_idx, upto_op_idx).  If such a graph\n  // is not runnable, e.g. if it includes memcpys from unpinned cpu memory, then make a\n  // number of smaller graphs that avoid those ops with the memcpys.\n  void CreateSubExecOverRegion(const std::vector<std::shared_ptr<exec::OpExecutor>>& exec_list,\n                               const RunContext& rctx,\n                               bool is_gpu,\n                               size_t from_op_idx,\n                               size_t upto_op_idx,\n                               std::vector<CudaGraphsSubSegExec>* cuda_graph_subseg_execs) {\n    // Optimistically try to create a CUDA Graph of the entire op segment region\n\n    int num_ops = upto_op_idx - from_op_idx;\n    CudaGraphsSubSegExec full_opseg(exec_list, rctx, is_gpu, verbose_, from_op_idx, num_ops);\n    if (full_opseg.IsRunnable()) {\n      cuda_graph_subseg_execs->push_back(full_opseg);\n    } else {\n      if (verbose_)\n        LOG(INFO) << \"  Graph was not runnable- creating op sub-segments...\";\n      // Enter fall-back approach to making many sub-execs\n      for (size_t first_op_idx = from_op_idx; first_op_idx != upto_op_idx;) {\n        int num_good_ops = 0;\n        for (size_t last_op_idx = first_op_idx; last_op_idx != upto_op_idx; ++last_op_idx) {\n          CudaGraphsSubSegExec single_opseg(exec_list, rctx, is_gpu, false, last_op_idx, 1);\n          if (single_opseg.IsRunnable())\n            num_good_ops++;\n          // Is it time to create a subseg exec from accumulated good ops?\n          if (num_good_ops > 0 && (last_op_idx == upto_op_idx - 1 || !single_opseg.IsRunnable())) {\n            if (verbose_)\n              LOG(INFO) << \"Capturing CUDA graph of op sub segment[\" << first_op_idx << \":\"\n                        << (first_op_idx + num_good_ops - 1) << \"]\"\n                        << \" of op segment \" << opr_names_;\n            CudaGraphsSubSegExec good_opseg(\n                exec_list, rctx, is_gpu, verbose_, first_op_idx, num_good_ops);\n            CHECK(good_opseg.IsRunnable()) << \"Unexpected issue with CUDA Graphs creation\";\n            cuda_graph_subseg_execs->push_back(good_opseg);\n            first_op_idx += num_good_ops;\n          }\n          // If the last single op was not runnable, use the exec to handle that op conventionally\n          if (!single_opseg.IsRunnable()) {\n            if (verbose_) {\n              LOG(INFO) << \"Bypassing op sub segment[\" << last_op_idx << \",\" << last_op_idx << \"]\"\n                        << \" of op segment \" << opr_names_;\n              // Generate throw-away exec in order to produce a diagnostic listing of graph nodes\n              CudaGraphsSubSegExec dummy(exec_list, rctx, is_gpu, verbose_, last_op_idx, 1);\n            }\n            cuda_graph_subseg_execs->push_back(single_opseg);\n            first_op_idx++;\n            break;\n          }\n        }\n      }\n    }\n  }\n\n  // Is the Op OK to make part of a CUDA Graph?\n  bool OpOK(const std::shared_ptr<exec::OpExecutor>& exec) {\n    static auto& fgraphcompatible = Op::GetAttr<FIsCUDAGraphsCompatible>(\"FIsCUDAGraphsCompatible\");\n    static auto& fcompute_ex      = Op::GetAttr<FComputeEx>(\"FComputeEx<gpu>\");\n    static auto& fstatefulcompute = Op::GetAttr<FStatefulCompute>(\"FStatefulCompute<gpu>\");\n    static auto& fstatefulcompute_ex = Op::GetAttr<FStatefulComputeEx>(\"FStatefulComputeEx<gpu>\");\n    const auto& attrs                = exec->attrs;\n    if (attrs.op != nullptr) {\n      const auto f = fgraphcompatible.get(attrs.op, nullptr);\n      if (f != nullptr) {\n        return f(attrs, exec->op_ctx.is_train);\n      }\n      if (fstatefulcompute.get(attrs.op, nullptr) != nullptr ||\n          fstatefulcompute_ex.get(attrs.op, nullptr) != nullptr) {\n        if (verbose_) {\n          LOG(INFO) << \"Omitting stateful operator \" << attrs.op->name << \" from CUDA graph.\";\n        }\n        return false;\n      }\n      if ((fcompute_ex.get(attrs.op, nullptr) != nullptr &&\n           exec->dispatch_mode == DispatchMode::kFComputeEx) ||\n          exec->dispatch_mode == DispatchMode::kFComputeFallback) {\n        if (verbose_) {\n          LOG(INFO) << \"Omitting operator \" << attrs.op->name\n                    << \" from CUDA graph due to dispatch mode \"\n                    << static_cast<int>(exec->dispatch_mode);\n        }\n        return false;\n      }\n    }\n    for (auto& resource : exec->op_ctx.requested) {\n      if (!(resource.req.type == ResourceRequest::kTempSpace)) {\n        if (verbose_) {\n          LOG(INFO) << \"Omitting operator \" << attrs.op->name\n                    << \" from CUDA graph due to using the resource type \"\n                    << static_cast<int>(resource.req.type);\n        }\n        return false;\n      }\n    }\n    return true;\n  }\n\n  // Determine Tempspaces used by ops.  Other resource uses disable CUDA Graphs.\n  void SetTempSpaces(const std::vector<std::shared_ptr<exec::OpExecutor>>& exec_list) {\n    // Gather info about the ops use of TempSpace.\n    if (is_enabled_) {\n      std::set<Resource*> tempspaces_set;\n      for (auto& exec : exec_list) {\n        for (auto& resource : exec->op_ctx.requested) {\n          if (resource.req.type == ResourceRequest::kTempSpace) {\n            tempspaces_set.insert(&resource);\n          }\n        }\n      }\n      tempspaces_.assign(tempspaces_set.begin(), tempspaces_set.end());\n    }\n  }\n\n  // Return the addresses of the gpu TempSpace areas\n  std::vector<void*> GetGPUTempspacePtrs(mshadow::Stream<gpu>* s) {\n    std::vector<void*> ret;\n    for (const auto& resource : tempspaces_) {\n      // Ask for minimal allocation to get base pointer without increasing the size\n      auto* base_ptr = resource->get_space_typed<gpu, 1, char>(mshadow::Shape1(1), s).dptr_;\n      ret.push_back(static_cast<void*>(base_ptr));\n    }\n    return ret;\n  }\n\n  CudaGraphCache cache_;\n  std::vector<Resource*> tempspaces_;\n  std::string opr_names_;\n  bool verbose_;\n  bool is_enabled_;\n};\n\n}  // namespace cuda_graphs\n}  // namespace mxnet\n\n#endif  // CUDA_GRAPHS_AVAILABLE\n\n#endif  // MXNET_IMPERATIVE_CUDA_GRAPHS_H_\n"
  },
  {
    "path": "src/imperative/eliminate_common_expr_pass.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file eliminate_common_expr.cc\n * \\brief Eliminate common expressions in the graph\n * \\author Przemyslaw Tredak\n */\n\n#include <mxnet/base.h>\n#include <mxnet/op_attr_types.h>\n\n#include <vector>\n#include <map>\n#include <utility>\n#include <sstream>\n\nnamespace mxnet {\nnamespace exec {\n\nnamespace {\n\nusing nnvm::Graph;\nusing nnvm::IndexedGraph;\nusing nnvm::Node;\nusing nnvm::ObjectPtr;\n\n// NodeInput holds the sufficient subset of NodeEntry fields for Node-input equality tests\nusing NodeInput = std::pair<const Node*, uint32_t>;\n\n/*!\n * \\brief Convert a Node's input vector of `NodeEntry` to a vector of the simpler `NodeInput`\n */\nstd::vector<NodeInput> ConvertInputs(const std::vector<nnvm::NodeEntry>& inputs) {\n  std::vector<NodeInput> ret;\n  ret.reserve(inputs.size());\n  for (const auto& entry : inputs) {\n    ret.emplace_back(entry.node.get(), entry.index);\n  }\n  return ret;\n}\n\n/*!\n * \\brief Determine if two Nodes have equal function such that one Node can be eliminated.\n */\nbool NodeEqual(const Node* n, const Node* m) {\n  if (n->is_variable() || m->is_variable())\n    return false;\n  if (n->op() != m->op())\n    return false;\n  // Nodes with different attributes are considered not identical,\n  // though this may reject Node pairs that are in fact functionally the same.\n  if (n->attrs.dict != m->attrs.dict)\n    return false;\n\n  // Ops that mutate inputs cannot be optimized out\n  static auto& fmutate_inputs = Op::GetAttr<nnvm::FMutateInputs>(\"FMutateInputs\");\n  if (fmutate_inputs.get(n->op(), nullptr) != nullptr)\n    return false;\n\n  // Stateful ops cannot be be equal to each other\n  static auto& fstateful = Op::GetAttr<FCreateOpState>(\"FCreateOpState\");\n  if (fstateful.get(n->op(), nullptr) != nullptr)\n    return false;\n\n  // Check to see if the user has explicitly set THasDeterministicOutput to override the\n  // subsequent determination of Node equality based on resource use.\n  static auto& deterministic_output =\n      Op::GetAttr<THasDeterministicOutput>(\"THasDeterministicOutput\");\n  if (deterministic_output.contains(n->op()))\n    return deterministic_output[n->op()];\n\n  // Ops that require resource could ask for\n  // random resource, so need to be explicitly marked\n  // to be eligible\n  static auto& resource_request    = Op::GetAttr<FResourceRequest>(\"FResourceRequest\");\n  static auto& resource_request_ex = Op::GetAttr<FResourceRequestEx>(\"FResourceRequestEx\");\n  const auto fresource_request     = resource_request.get(n->op(), nullptr);\n  if (fresource_request != nullptr) {\n    const auto& requests = fresource_request(n->attrs);\n    for (const auto& req : requests) {\n      if (req.type != ResourceRequest::kTempSpace) {\n        return false;\n      }\n    }\n  }\n  if (resource_request_ex.get(n->op(), nullptr) != nullptr)\n    return false;\n\n  return true;\n}\n\n// Graph traversal to create a list of pairs of identical-function nodes that can be combined.\nstd::vector<std::pair<ObjectPtr, ObjectPtr> > GetCommonNodes(const Graph& g) {\n  std::vector<std::pair<ObjectPtr, ObjectPtr> > ret;\n  // A map between a vector of inputs and those nodes that have those inputs\n  std::map<std::vector<NodeInput>, std::vector<const ObjectPtr*> > grouped_nodes;\n  // Traverse the graph and group the nodes by their vector of inputs\n  nnvm::DFSVisit(g.outputs, [&grouped_nodes](const ObjectPtr& n) {\n    if (n->inputs.size() != 0) {\n      grouped_nodes[ConvertInputs(n->inputs)].push_back(&n);\n    }\n  });\n  // Now check for identical node ops within the node groups (having identical inputs)\n  for (const auto& pair : grouped_nodes) {\n    auto& node_group = pair.second;  // Group of nodes that share the same vector of inputs\n    if (node_group.size() > 1) {\n      std::unordered_set<size_t> visited;\n      for (size_t i = 0; i < node_group.size(); ++i) {\n        if (visited.count(i))\n          continue;\n        for (size_t j = i + 1; j < node_group.size(); ++j) {\n          // If the two Nodes have equal function, then one Node (called the 'replaced') can\n          // be eliminated in favor of the other Node (the 'src').\n          if (NodeEqual(node_group[i]->get(), node_group[j]->get())) {\n            visited.insert(j);\n            ObjectPtr src      = *node_group[i];\n            ObjectPtr replaced = *node_group[j];\n            ret.emplace_back(src, replaced);\n          }\n        }\n      }\n    }\n  }\n  return ret;\n}\n\n/*!\n * \\brief Do a single pass of Node elimination given pairs of identical Nodes.\n */\nvoid EliminateCommonNodes(Graph* g,\n                          const std::vector<std::pair<ObjectPtr, ObjectPtr> >& common_nodes) {\n  for (const auto& p : common_nodes) {\n    std::vector<ObjectPtr> nodes_to_change;\n    const ObjectPtr& src      = p.first;\n    const ObjectPtr& replaced = p.second;\n    // Create a `nodes_to_change` list containing the Nodes that refer to the `replaced` Node\n    // that is targeted for elimination.\n    DFSVisit(g->outputs, [replaced, &nodes_to_change](const ObjectPtr& n) {\n      for (const auto& dep : n->control_deps) {\n        if (dep == replaced) {\n          nodes_to_change.push_back(n);\n          return;\n        }\n      }\n      for (const auto& inp : n->inputs) {\n        if (inp.node == replaced) {\n          nodes_to_change.push_back(n);\n          return;\n        }\n      }\n    });\n\n    // Change references to the `replaced` Node within the `nodes_to_change` list to be\n    // references to the equivalent `src` Node.\n    for (auto& n : nodes_to_change) {\n      for (auto& dep : n->control_deps) {\n        if (dep == replaced) {\n          dep = src;\n        }\n      }\n      for (auto& inp : n->inputs) {\n        if (inp.node == replaced) {\n          inp.node = src;\n        }\n      }\n    }\n\n    // Add `replaced` Node control dependencies to those of the `src` Node.\n    for (const auto& n : replaced->control_deps) {\n      src->control_deps.push_back(n);\n    }\n\n    // Change graph outputs driven by the `replaced` Node to now point to the `src` Node.\n    for (auto& out : g->outputs) {\n      if (out.node == replaced) {\n        out.node = src;\n      }\n    }\n  }\n  // Check for duplicates in outputs and\n  // insert Copy nodes as appropriate\n  const Op* copy_op = Op::Get(\"_copy\");\n  nnvm::NodeEntryMap<size_t> unique_outputs;\n  for (auto& output : g->outputs) {\n    auto kv = unique_outputs.find(output);\n    if (kv == unique_outputs.end()) {\n      unique_outputs.emplace(output, 0);\n    } else {\n      ObjectPtr copy_node = Node::Create();\n      std::ostringstream os;\n      os << kv->first.node->attrs.name << \"_\" << kv->second << \"_copy\";\n      kv->second++;\n      copy_node->attrs.op   = copy_op;\n      copy_node->attrs.name = os.str();\n      copy_node->inputs.emplace_back(kv->first);\n      output = nnvm::NodeEntry{copy_node, 0, 0};\n    }\n  }\n}\n\n}  // namespace\n\n/*!\n * \\brief Simplify a graph by iteratively eliminating Nodes with identical inputs and function.\n */\nnnvm::Graph EliminateCommonExpr(nnvm::Graph&& g) {\n  using nnvm::ObjectPtr;\n  bool keep_running = true;\n  while (keep_running) {\n    const auto& common_nodes = GetCommonNodes(g);\n    if (common_nodes.empty()) {\n      keep_running = false;\n    } else {\n      EliminateCommonNodes(&g, common_nodes);\n    }\n  }\n  return std::move(g);\n}\n\n}  // namespace exec\n}  // namespace mxnet\n"
  },
  {
    "path": "src/imperative/exec_pass.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file exec_pass.h\n * \\brief All the execution related pass and data structures.\n */\n#ifndef MXNET_IMPERATIVE_EXEC_PASS_H_\n#define MXNET_IMPERATIVE_EXEC_PASS_H_\n\n#include <mxnet/base.h>\n#include <mxnet/ndarray.h>\n#include <mxnet/operator.h>\n#include <mxnet/graph_attr_types.h>\n#include <nnvm/graph.h>\n#include <nnvm/graph_attr_types.h>\n#include <utility>\n#include <vector>\n#include <memory>\n#include <string>\n#include <utility>\n#include <tuple>\n\nnamespace mxnet {\nnamespace exec {\n\ntemplate <typename Attr>\nusing FAccessSubgraphAttr =\n    std::function<std::tuple<const nnvm::ObjectPtr, std::vector<Attr>, std::vector<Attr>>(\n        const NodeAttrs& attrs)>;\n\nusing FAccessSubgraphShape       = FAccessSubgraphAttr<mxnet::TShape>;\nusing FAccessSubgraphType        = FAccessSubgraphAttr<int>;\nusing FAccessSubgraphStorageType = FAccessSubgraphAttr<int>;\n\ntemplate <typename Attr>\nusing FProvideSubgraphAttr        = std::function<void(const NodeAttrs& attrs,\n                                                const std::vector<nnvm::ObjectPtr>& nodes,\n                                                const std::vector<std::vector<Attr>>& in_attrs,\n                                                const std::vector<std::vector<Attr>>& out_attrs)>;\nusing FProvideSubgraphShape       = FProvideSubgraphAttr<mxnet::TShape>;\nusing FProvideSubgraphType        = FProvideSubgraphAttr<int>;\nusing FProvideSubgraphStorageType = FProvideSubgraphAttr<int>;\n\nusing TIsFusion       = bool;\nusing TIsFusionHelper = bool;\n\n/*! \\brief reuse graph definition */\nusing nnvm::Graph;\n\nconst int kBadStorageID      = -1;\nconst int kExternalStorageID = -2;\nconst int kDynamicStorageID  = -3;\n\nconst int kNonDefaultStorage = -2;\n\n/*!\n * \\brief executor to execute an operator\n * This is a graph executor dependent interface\n * that unifies all the operator\n */\nclass OpExecutor {\n public:\n  /*! \\brief input data arrays, which may be either input or aux */\n  std::vector<NDArray> in_array;\n  /*! \\brief output data arrays */\n  std::vector<NDArray> out_array;\n  /*! \\brief output requirement on each array */\n  std::vector<OpReqType> req;\n  /*! \\brief runtime op context, contains allocated resources */\n  OpContext op_ctx;\n  /*! \\brief attributes of the node */\n  NodeAttrs attrs;\n  /*! \\brief dispatch mode of the executor */\n  DispatchMode dispatch_mode;\n\n  explicit OpExecutor(NodeAttrs attrs, DispatchMode dispatch_mode)\n      : attrs(std::move(attrs)), dispatch_mode(dispatch_mode) {}\n  /*! \\brief virtual destructor */\n  virtual ~OpExecutor() {}\n  /*!\n   * \\brief Setup the executor for given NDArray member\n   *  This can be called multiple times if NDArray changed during reshape.\n   *  It is safe to call it via an asynchronous engine lambda.\n   */\n  virtual void Setup() = 0;\n  /*!\n   * \\brief run the operator given runtime context on device.\n   *  This function call does not synchronize the stream.\n   * \\param rctx The runtime context passed in by environment.\n   */\n  virtual void Run(RunContext rctx, bool is_gpu) = 0;\n  /*!\n   * \\brief run the operators of a vector of execs, given runtime context on device.\n   *  This function call does not synchronize the stream.\n   * \\param rctx The runtime context passed in by environment.\n   */\n  static void RunAll(const std::vector<std::shared_ptr<OpExecutor>>& execs,\n                     RunContext rctx,\n                     bool is_gpu) {\n    for (auto& exec : execs)\n      exec->Run(rctx, is_gpu);\n  }\n  /*! \\return the execution type */\n  virtual ExecType exec_type() const = 0;\n  /*! \\return return engine variable for operator states */\n  virtual engine::VarHandle var() const {\n    return nullptr;\n  }\n  /*! \\return return operator state */\n  virtual OpStatePtr state() const {\n    return OpStatePtr();\n  }\n\n  // TODO(alexzai): (MXNET-856) Remove instance member after subgraph feature added\n protected:\n  std::vector<NDArray> in_array_fallback;\n};\n\n/*!\n * \\brief per node vector of operator executors.\n * \\note stored under attribute \"op_exec\"\n */\nusing OpExecVector = std::vector<std::shared_ptr<OpExecutor>>;\n\n/*!\n * \\brief per node vector of operator states.\n * \\note stored under attribute \"op_states\"\n */\nusing OpStateVector = std::vector<OpStatePtr>;\n\n/*!\n * \\brief per node context vector\n * \\node stored under \"context\"\n */\nusing ContextVector = std::vector<Context>;\n\n/*!\n * \\brief per node device mask vector\n * \\node stored under \"dev_mask\"\n */\nusing DevMaskVector = std::vector<int>;\n\n/*!\n * \\brief create OpExecutor for a node in graph\n *\n * \\param g input graph\n * \\param p_ret OpExecVector for input and output\n * \\param p_state OpStateVector if it has.\n * \\param i the id of the node\n */\nvoid CreateOpExecs(const Graph& g, OpExecVector* p_ret, OpStateVector* p_state, size_t i);\n/*!\n * \\brief Attach OpExecutor to the graph attributes.\n *\n * \\param g input graph\n * \\return graph with new attribute \"op_exec\" of type OpExecVector\n *  The fields on the OpExecVector are not yet been setup.\n */\nGraph AttachOpExecs(Graph g);\n\n/*!\n * \\brief Attach Resource to the OpExecVector of the graph.\n *\n * \\param g input graph need to contain op_exec attribute.\n */\nvoid AttachOpResources(const Graph& g);\n/*!\n * \\brief Attach Resource to the OpExecVector\n *\n * \\param g input graph\n * \\param op_execs OpExecutor vector\n * \\param start_nid starting node id\n * \\param end_nid end node id\n */\nvoid AttachOpResources(const Graph& g,\n                       const OpExecVector& op_execs,\n                       size_t start_nid,\n                       size_t end_nid);\n/*!\n * \\brief Discover chance of inplace addto operators.\n *  i.e. z = plus(z, source_op), and encourage it to become z += source_op.\n *\n * This optimization is coupled with executor. This is helpful to reduce memory\n * and computation for gradient aggregation of RNN.\n *\n * Require storage placement to be already finished.\n *\n * \\param g input graph need to contain op_exec attribute.\n *\n * \\return graph two new attributes, changes attribute \"storage_id\".\n *  - \"addto_entry\", std::vector<bool> size=g.num_node_entries()\n *    - addto_entry[eid] == 1, the corresponding op need to be performed using req=kAddTo\n *  - \"skip_plus_node\", std::vector<int> if set to 1, current op's execution is skiped.\n */\nGraph DetectInplaceAddTo(Graph g);\n\n/*!\n * \\brief Eliminate common expressions in the graph.\n *\n * \\param g input forward graph\n *\n * \\return graph with common expressions eliminated\n */\nGraph EliminateCommonExpr(Graph&& g);\n\n/*!\n * \\brief Fuse pointwise operations in the graph.\n *\n * \\param g input graph (needs to be entire graph, not just forward part)\n * \\param num_forward_outputs number of outputs in the graph produced by the forward pass\n *\n * \\return copy of the graph with fused pointwise operations\n */\nGraph FusePointwise(const Graph& g, const size_t num_forward_outputs);\n\n/*!\n * \\brief Issue a one-time warning that fusion is not possible for this platform or build.\n */\nvoid WarnFusionNotSupported();\n\n/*!\n * \\brief Infer shapes in the graph given the information.\n * \\param graph The input graph.\n * \\param shape_inputs The shapes of input symbols to the graph.\n * \\param shape_attr_key The key to the node attribute that can indicate shape. This is\n *                       the place where manual hint for shapes could be injected.\n * \\return A graph with new attribute \"shape\" containing inferred shape of each NodeEntry.\n *         The index of ShapeVector is given by graph.indexed_graph().entry_id.\n */\nGraph InferShape(Graph&& graph,\n                 mxnet::ShapeVector&& shape_inputs = mxnet::ShapeVector(),\n                 const std::string& shape_attr_key = \"\");\n\n/*!\n * \\brief Infer types in the graph given the information.\n * \\param graph The input graph.\n * \\param dtype_inputs The types of input symbols to the graph.\n * \\param dtype_attr_key The key to the node attribute that can indicate types. This is\n *                       the place where manual hint for types could be injected.\n * \\return A graph with new attribute \"dtype\" containing inferred type of each NodeEntry.\n *         The index of ShapeVector is given by graph.indexed_graph().entry_id.\n */\nGraph InferType(Graph&& graph,\n                nnvm::DTypeVector&& dtype_inputs  = nnvm::DTypeVector(),\n                const std::string& dtype_attr_key = \"\");\n\n/*!\n * \\brief Infer storage types in the graph given the information.\n * \\param graph The input graph.\n * \\param storage_type_inputs The storage types of input symbols to the graph.\n * \\param storage_type_attr_key The key to the node attribute that can indicate storage types.\n                                This is the place where manual hint for types could be injected.\n * \\return A graph with new attribute \"storage_type\" containing inferred type of each NodeEntry.\n *         The index of StorageTypeVector is given by graph.indexed_graph().entry_id.\n */\nGraph InferStorageType(Graph&& graph,\n                       StorageTypeVector&& storage_type_inputs  = StorageTypeVector(),\n                       const std::string& storage_type_attr_key = \"\");\n\n}  // namespace exec\n}  // namespace mxnet\n\nnamespace nnvm {\nnamespace pass {\n/*!\n * \\brief Get the gradient graph whose outputs are gradients of xs wrt to ys.\n * \\param graph The input graph.\n * \\param ys The entries to take gradient from.\n * \\param xs The entries to take gradient with respect to.\n * \\param ys_out_grad The output gradients of ys.\n * \\param aggregate_fun The aggregation function used for summing gradients.\n * \\param mirror_fun The backward mirroring function that does mirroring to save memory.\n * \\param zero_ops The list of operators that output a single zero array, used\n *                 for generating zero gradient nodes. The first operator must\n *                 be zero_like.\n * \\param copy_op_str The name of the copy operator that handle gradient duplicates.\n * \\param in_arg_shapes The shapes of input arguments, used for shape inference.\n * \\param in_arg_dtpyes The data types of input arguments, used for data type inference.\n * \\return A new graph, whose outputs correspond to inputs of xs.\n */\ninline Graph MXGradient(\n    Graph graph,\n    std::vector<NodeEntry> ys,\n    std::vector<NodeEntry> xs,\n    std::vector<NodeEntry> ys_out_grad,\n    std::function<NodeEntry(std::vector<NodeEntry>&& inputs)> aggregate_fun = nullptr,\n    std::function<int(const Node& node)> mirror_fun                         = nullptr,\n    std::vector<const Op*> zero_ops  = std::vector<const Op*>(),\n    std::string copy_op_str          = std::string(),\n    mxnet::ShapeVector in_arg_shapes = mxnet::ShapeVector(),\n    DTypeVector in_arg_dtypes        = DTypeVector(),\n    std::vector<NodeEntry> us        = std::vector<NodeEntry>()) {\n  graph.attrs[\"grad_ys\"]          = std::make_shared<any>(std::move(ys));\n  graph.attrs[\"grad_xs\"]          = std::make_shared<any>(std::move(xs));\n  graph.attrs[\"grad_ys_out_grad\"] = std::make_shared<any>(std::move(ys_out_grad));\n  graph.attrs[\"in_arg_shapes\"]    = std::make_shared<any>(std::move(in_arg_shapes));\n  graph.attrs[\"in_arg_dtypes\"]    = std::make_shared<any>(std::move(in_arg_dtypes));\n  graph.attrs[\"grad_us\"]          = std::make_shared<any>(std::move(us));\n\n  if (aggregate_fun != nullptr) {\n    graph.attrs[\"grad_aggregate_fun\"] = std::make_shared<any>(aggregate_fun);\n  }\n  if (mirror_fun != nullptr) {\n    graph.attrs[\"mirror_fun\"] = std::make_shared<any>(mirror_fun);\n  }\n  if (zero_ops.size()) {\n    graph.attrs[\"zero_ops\"] = std::make_shared<any>(std::move(zero_ops));\n  }\n  if (copy_op_str != std::string()) {\n    graph.attrs[\"copy_op_str\"] = std::make_shared<any>(std::move(copy_op_str));\n  }\n  return ApplyPass(std::move(graph), \"MXGradient\");\n}\n}  // namespace pass\n}  // namespace nnvm\n\n#endif  // MXNET_IMPERATIVE_EXEC_PASS_H_\n"
  },
  {
    "path": "src/imperative/imperative.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n#include <algorithm>\n#include <iostream>\n#include <unordered_map>\n#include <unordered_set>\n\n#include \"./imperative_utils.h\"\n#include \"./cached_op.h\"\n\nnamespace nnvm {\nObjectPtr CreateVariableNode(const std::string& name);\n}\n\nnamespace mxnet {\n#if DMLC_CXX11_THREAD_LOCAL\nthread_local bool Imperative::is_train_                 = false;\nthread_local bool Imperative::is_recording_             = false;\nthread_local bool Imperative::is_deferred_compute_      = false;\nthread_local OptConstraint Imperative::opt_constraints_ = OptConstraint::None;\nthread_local bool Imperative::is_np_shape_thread_local_ = false;\n#else\nMX_THREAD_LOCAL bool Imperative::is_train_                 = false;\nMX_THREAD_LOCAL bool Imperative::is_recording_             = false;\nMX_THREAD_LOCAL bool Imperative::is_deferred_compute_      = false;\nMX_THREAD_LOCAL OptConstraint Imperative::opt_constraints_ = OptConstraint::None;\nMX_THREAD_LOCAL bool Imperative::is_np_shape_thread_local_ = false;\n#endif\n\nImperative* Imperative::Get() {\n  static Imperative inst;\n  return &inst;\n}\n\nOpStatePtr Imperative::InvokeOp(const Context& ctx,\n                                const nnvm::NodeAttrs& attrs,\n                                const std::vector<NDArray*>& inputs,\n                                const std::vector<NDArray*>& outputs,\n                                const std::vector<OpReqType>& req,\n                                const DispatchMode dispatch_mode,\n                                OpStatePtr state) {\n  using namespace imperative;\n  static auto& createop          = nnvm::Op::GetAttr<FCreateOpState>(\"FCreateOpState\");\n  static auto& is_layer_backward = Op::GetAttr<bool>(\"TIsLayerOpBackward\");\n  MXAPIThreadLocalEntry<>* ret   = MXAPIThreadLocalStore<>::Get();\n\n  const nnvm::Op* op = attrs.op;\n\n  std::vector<engine::VarHandle> read_vars, write_vars;\n  std::vector<Resource> requested;\n  std::vector<uint32_t> mutate_idx;\n  SetDependency(\n      attrs, ctx, inputs, outputs, &read_vars, &write_vars, &requested, &mutate_idx, dispatch_mode);\n\n  FCompute fn      = common::GetFCompute<FCompute>(op, \"FCompute\", ctx);\n  FComputeEx fn_ex = common::GetFCompute<FComputeEx>(op, \"FComputeEx\", ctx);\n\n  // FComputeEx is dispatched only when dispatch_mode is DispatchMode::kFComputeEx\n  CHECK(dispatch_mode != DispatchMode::kUndefined);\n  bool dispatch_fcompex = dispatch_mode == DispatchMode::kFComputeEx;\n  if (fn_ex && dispatch_fcompex) {\n    PushFComputeEx(fn_ex, op, attrs, ctx, read_vars, write_vars, requested, inputs, outputs, req);\n  } else if (fn) {\n    PushFCompute(\n        fn, op, attrs, ctx, read_vars, write_vars, requested, inputs, outputs, mutate_idx, req);\n  } else if (createop.count(op) || is_layer_backward.get(op, false)) {\n    if (!state) {\n      state = createop[op](attrs, ctx, ret->arg_shapes, ret->arg_types);\n    }\n    write_vars.push_back(state.get_var());\n    PushOperator(state,\n                 op,\n                 attrs,\n                 ctx,\n                 read_vars,\n                 write_vars,\n                 requested,\n                 inputs,\n                 outputs,\n                 mutate_idx,\n                 req,\n                 dispatch_mode);\n  } else {\n    LOG(FATAL) << \"Operator \" << op->name << \" is not implemented for \"\n               << (ctx.dev_mask() == gpu::kDevMask ? \"GPU.\" : \"CPU.\");\n  }\n\n  return state;\n}\n\nOpStatePtr Imperative::Invoke(const Context& default_ctx,\n                              const nnvm::NodeAttrs& attrs,\n                              const std::vector<NDArray*>& inputs,\n                              const std::vector<NDArray*>& outputs) {\n  using namespace imperative;\n  static auto& ndfunc = nnvm::Op::GetAttr<FNDArrayFunction>(\"FNDArrayFunction\");\n\n  if (ndfunc.count(attrs.op)) {\n    std::vector<NDArray> p_inputs, p_outputs;\n    DerefInputOutput(inputs, outputs, &p_inputs, &p_outputs);\n    ndfunc[attrs.op](attrs, p_inputs, &p_outputs);\n    for (size_t i = 0; i < outputs.size(); ++i)\n      *outputs[i] = std::move(p_outputs[i]);\n    return OpStatePtr();\n  }\n\n  // TODO(piiswrong): infer ctx\n  DispatchMode dispatch_mode = DispatchMode::kUndefined;\n  Context ctx                = GetContext(attrs, inputs, outputs, default_ctx);\n  SetShapeType(ctx, attrs, inputs, outputs, &dispatch_mode);\n  std::vector<OpReqType> req;\n  SetWriteInplaceReq(inputs, outputs, &req);\n  OpStatePtr ret = InvokeOp(ctx, attrs, inputs, outputs, req, dispatch_mode);\n  // the followinng loop is used for finding out the correct shape when some shapes are dynamic\n  for (auto output : outputs) {\n    if (!shape_is_known(output->shape())) {\n      // the WaitToRead overhead here does not seem to be avoidable\n      output->WaitToRead();\n      output->SetShapeFromChunk();\n    }\n  }\n  return ret;\n}\n\n// Create nnvm::NodeEntry for variables' and gradients' autograd_entry_\n// attribute and associate AGInfo with it's info attribute\nvoid Imperative::MarkVariables(const std::vector<NDArray*>& variables,\n                               const std::vector<uint32_t>& grad_reqs,\n                               const std::vector<NDArray*>& gradients) {\n  for (uint32_t i = 0; i < variables.size(); ++i) {\n    // Unmarked leaf nodes have null autograd_entry_, while marked nonleaf nodes don't.\n    if (!variables[i]->autograd_entry_.node || variables[i]->autograd_entry_.node->is_variable()) {\n      std::string str_c(std::to_string(variable_count_++));\n      variables[i]->autograd_entry_ =\n          nnvm::NodeEntry{nnvm::Symbol::CreateVariable(\"var\" + str_c).outputs[0].node, 0, 0};\n      AGInfo& info = AGInfo::Create(variables[i]->autograd_entry_.node);\n      info.outputs.emplace_back(variables[i]->Detach());\n      info.out_grads.emplace_back(gradients[i]->Detach());\n      info.grad_req = static_cast<OpReqType>(grad_reqs[i]);\n      info.ctx      = variables[i]->ctx();\n\n      gradients[i]->autograd_entry_ =\n          nnvm::NodeEntry{nnvm::Symbol::CreateVariable(\"grad\" + str_c).outputs[0].node, 0, 0};\n      AGInfo& grad_info = AGInfo::Create(gradients[i]->autograd_entry_.node);\n      grad_info.outputs.emplace_back(gradients[i]->Detach());\n      grad_info.ctx = gradients[i]->ctx();\n    } else {\n      AGInfo& info = AGInfo::Get(variables[i]->autograd_entry_.node);\n      CHECK_EQ(info.out_grads.size(), 0)\n          << \"The node has already been marked. Cannot mark it again.\";\n      info.out_grads.emplace_back(gradients[i]->Detach());\n      info.grad_req = static_cast<OpReqType>(grad_reqs[i]);\n      info.ctx      = variables[i]->ctx();\n    }\n  }\n}\n\n// Unmark the variables to free the memory.\nvoid Imperative::DropGrads(const std::vector<NDArray*>& variables) {\n  for (auto variable : variables) {\n    if (variable->autograd_entry_.node) {\n      AGInfo& info = AGInfo::Get(variable->autograd_entry_.node);\n      CHECK_NE(info.out_grads.size(), 0)\n          << \"The node has empty out_grads already. Cannot DropGrads again.\";\n      for (auto grad : info.out_grads) {\n        grad.ReInit();\n      }\n      info.out_grads.clear();\n      info.grad_req = kNullOp;\n    }\n  }\n}\n\nvoid Imperative::GetBackwardDependency(const nnvm::ObjectPtr& node,\n                                       uint32_t num_inputs,\n                                       uint32_t num_outputs,\n                                       std::vector<bool>* p_save_inputs,\n                                       std::vector<bool>* p_save_outputs) {\n  static auto& fgradient          = nnvm::Op::GetAttr<nnvm::FGradient>(\"FGradient\");\n  std::vector<bool>& save_inputs  = *p_save_inputs;\n  std::vector<bool>& save_outputs = *p_save_outputs;\n  save_inputs.resize(num_inputs);\n  save_outputs.resize(num_outputs);\n  std::fill(save_inputs.begin(), save_inputs.end(), false);\n  std::fill(save_outputs.begin(), save_outputs.end(), false);\n\n  node->inputs.clear();\n  node->inputs.reserve(num_inputs);\n  for (uint32_t i = 0; i < num_inputs; ++i) {\n    node->inputs.emplace_back(nnvm::NodeEntry{nullptr, i, 0});\n  }\n\n  if (fgradient.count(node->op())) {\n    std::vector<nnvm::NodeEntry> ograd_entries;\n    ograd_entries.reserve(num_outputs);\n    for (uint32_t i = 0; i < num_outputs; ++i) {\n      ograd_entries.emplace_back(nullptr, i, 1);\n    }\n    auto igrad_entries = fgradient[node->op()](node, ograd_entries);\n    for (const auto& i : igrad_entries) {\n      if (i.node == nullptr && i.version == 0) {\n        save_inputs[i.index] = true;\n      } else if (i.node == node) {\n        save_outputs[i.index] = true;\n      }\n    }\n    DFSVisit(igrad_entries, [&](const nnvm::ObjectPtr& gnode) {\n      if (!gnode || gnode == node)\n        return;\n      for (const auto& i : gnode->inputs) {\n        if (i.node == nullptr && i.version == 0) {\n          save_inputs[i.index] = true;\n        } else if (i.node == node) {\n          save_outputs[i.index] = true;\n        }\n      }\n    });\n  }\n}\n\nvoid Imperative::RecordOp(nnvm::NodeAttrs&& attrs,\n                          const std::vector<NDArray*>& inputs,\n                          const std::vector<NDArray*>& outputs,\n                          const OpStatePtr& state,\n                          std::vector<bool>* p_save_inputs,\n                          std::vector<bool>* p_save_outputs) {\n  MXAPIThreadLocalEntry<>* local_buff = MXAPIThreadLocalStore<>::Get();\n\n  CHECK(!is_deferred_compute())\n      << \"Autograd recording is not supported during deferred compute mode.\";\n\n  for (auto output : outputs) {\n    CHECK(AGInfo::IsNone(*output))\n        << \"Assigning to NDArrays that are already in a computational graph \"\n        << \"will cause undefined behavior when evaluating gradients. \"\n        << \"Please call backward first to clear the graph or do this out side of \"\n        << \"a record section. Also note that you cannot use inplace operations \"\n        << \"like +=, *=, relu(x, out=x), y[idx]=x, etc inside a record section. \"\n        << \"Issue occurred while recording op: \" << attrs.name;\n  }\n\n  bool need_grad = false;\n  for (const auto& i : inputs) {\n    if (AGInfo::IsNone(*i))\n      continue;\n    need_grad = true;\n    break;\n  }\n  if (!need_grad)\n    return;\n\n  nnvm::ObjectPtr node = nnvm::Node::Create();\n  node->attrs          = std::move(attrs);\n  // if node name is empty or node name is equal to op name - name it with unique name\n  if (node->attrs.name == \"\" || node->attrs.op->name == node->attrs.name) {\n    node->attrs.name = \"node_\" + std::to_string(node_count_++);\n  } else {\n    node_count_++;\n  }\n  AGInfo& info = AGInfo::Create(node);\n  info.state   = state;\n  info.ctx     = outputs[0]->ctx();\n\n  if (p_save_inputs == nullptr) {\n    p_save_inputs  = &(local_buff->save_inputs);\n    p_save_outputs = &(local_buff->save_outputs);\n    GetBackwardDependency(node, inputs.size(), outputs.size(), p_save_inputs, p_save_outputs);\n  } else {\n    node->inputs.resize(inputs.size());\n  }\n\n  std::vector<bool>& save_inputs  = *p_save_inputs;\n  std::vector<bool>& save_outputs = *p_save_outputs;\n\n  for (size_t i = 0; i < inputs.size(); ++i) {\n    if (AGInfo::IsNone(*(inputs[i]))) {\n      nnvm::NodeEntry entry{\n          nnvm::Symbol::CreateVariable(\"null\" + std::to_string(variable_count_++)).outputs[0].node,\n          0,\n          0};\n      AGInfo& input_info = AGInfo::Create(entry.node);\n      input_info.ctx     = inputs[i]->ctx();\n      if (save_inputs[i]) {\n        input_info.outputs.emplace_back(*inputs[i]);\n      } else {\n        // Put a dummy array here since it will not be used.\n        input_info.outputs.emplace_back();\n        input_info.outputs.back().shape_        = inputs[i]->shape();\n        input_info.outputs.back().dtype_        = inputs[i]->dtype();\n        input_info.outputs.back().storage_type_ = inputs[i]->storage_type();\n      }\n      inputs[i]->autograd_entry_ = std::move(entry);  // assign last to prevent cyclic reference\n    } else if (save_inputs[i]) {\n      nnvm::NodeEntry& entry                       = inputs[i]->autograd_entry_;\n      AGInfo::Get(entry.node).outputs[entry.index] = inputs[i]->Detach();\n    }\n    node->inputs[i] = inputs[i]->autograd_entry_;\n  }\n\n  for (auto output : outputs) {\n    CHECK(AGInfo::IsNone(*output))\n        << \"NotImplementedError: Inplace operations (+=, -=, x[:]=, etc) \"\n        << \"are not supported when recording with autograd.\";\n  }\n\n  for (uint32_t i = 0; i < outputs.size(); ++i) {\n    if (save_outputs[i]) {\n      info.outputs.emplace_back(outputs[i]->Detach());\n    } else {\n      // Put a dummy array here since it will not be used.\n      info.outputs.emplace_back();\n      info.outputs.back().shape_        = outputs[i]->shape();\n      info.outputs.back().dtype_        = outputs[i]->dtype();\n      info.outputs.back().storage_type_ = outputs[i]->storage_type();\n    }\n    outputs[i]->autograd_entry_ = nnvm::NodeEntry{node, i, 0};\n  }\n}\n\nvoid Imperative::RecordDeferredCompute(nnvm::NodeAttrs&& attrs,\n                                       const std::vector<NDArray*>& inputs,\n                                       const std::vector<NDArray*>& outputs) {\n  CHECK(!is_recording())\n      << \"MXNetError: Autograd recording is not supported during deferred compute mode.\";\n\n  for (const NDArray* input : inputs) {\n    CHECK(!DCInfo::IsNone(*input))\n        << \"ValueError: All inputs to deferred compute recording must be associated \"\n        << \"with a symbolic variable or be the output of a deferred compute operator.\";\n  }\n  for (const NDArray* output : outputs) {\n    CHECK(DCInfo::IsNone(*output))\n        << \"NotImplementedError: Inplace operations (+=, -=, x[:]=, etc) \"\n        << \"are not supported when recording in deferred compute mode.\";\n  }\n  DispatchMode dispatch_mode = DispatchMode::kUndefined;\n  Context ctx                = imperative::GetContext(attrs, inputs, outputs, Context::CPU());\n  imperative::SetShapeType(ctx, attrs, inputs, outputs, &dispatch_mode);\n\n  nnvm::ObjectPtr node = nnvm::Node::Create();\n  node->inputs.reserve(inputs.size());\n  // Get NodeEntries for inputs\n  for (const NDArray* array : inputs) {\n    CHECK(array->deferredcompute_entry_.node);  // Must not be nullptr\n    node->inputs.emplace_back(array->deferredcompute_entry_);\n  }\n  node->attrs = std::move(attrs);\n  // Need to support NameManager in imperative API to better name node->attrs.name\n  // if node name is empty or node name is equal to op name - name it with unique name\n  if (node->attrs.name == \"\" || node->attrs.op->name == node->attrs.name) {\n    node->attrs.name = \"node_\" + std::to_string(node_count_++);\n  } else {\n    node_count_++;\n  }\n\n  if (get_opt_constraints() != OptConstraint::None) {\n    node->attrs.dict[OPT_CONSTRAINT_ATTR] =\n        std::to_string(static_cast<OptConstraint_int_t>(get_opt_constraints()));\n  }\n\n  for (uint32_t i = 0; i < outputs.size(); ++i) {\n    outputs[i]->deferredcompute_entry_ = nnvm::NodeEntry{node, i, 0};\n  }\n\n  DCInfo::Create(node, inputs, outputs);\n}\n\nnnvm::Symbol Imperative::GetDeferredComputeSymbol(const std::vector<NDArray*>& outputs) {\n  nnvm::Symbol s;\n  s.outputs.reserve(outputs.size());\n  for (NDArray* ndoutput : outputs) {\n    CHECK(!Imperative::DCInfo::IsNone(*ndoutput))\n        << \"ValueError: output_arrays for GetDeferredComputeSymbol \"\n        << \"must have a deferred compute history associated with them.\";\n    s.outputs.emplace_back(ndoutput->deferredcompute_entry_);\n  }\n  return s.Copy();\n}\n\nvoid Imperative::SetDeferredComputeVariable(NDArrayHandle* arrays,\n                                            SymbolHandle* variables,\n                                            const int num) {\n  // Sanity check all inputs\n  for (int i = 0; i < num; i++) {\n    nnvm::Symbol* s = reinterpret_cast<nnvm::Symbol*>(variables[i]);\n    NDArray* nd     = reinterpret_cast<NDArray*>(arrays[i]);\n    CHECK_EQ(s->outputs.size(), 1)\n        << \"MXNDArraySetDeferredComputeVariable expects variables as input. \"\n        << \"Instead got a Symbol with \" << s->outputs.size() << \" outputs as input \" << i;\n    CHECK(s->outputs[0].node->is_variable())\n        << \"MXNDArraySetDeferredComputeVariable expects variables as input. \"\n        << \"Instead got a Symbol associated with an operator as input \" << i;\n    CHECK(DCInfo::IsNone(*nd) || nd->deferredcompute_entry_.node == s->outputs[0].node)\n        << \"ValueError: array \" << i << \" is already associated with a different variable. \"\n        << \"You can call array.detach() to obtain a copy without the variable\";\n  }\n\n  // Store variables in DCInfo of arrays\n  for (int i = 0; i < num; i++) {\n    nnvm::Symbol* s            = reinterpret_cast<nnvm::Symbol*>(variables[i]);\n    NDArray* nd                = reinterpret_cast<NDArray*>(arrays[i]);\n    nd->deferredcompute_entry_ = nnvm::NodeEntry{s->outputs[0].node, 0, 0};\n\n    std::vector<NDArray*> inputs;\n    std::vector<NDArray*> outputs;  // No need to specify outputs, as we will set is_computed_\n    Imperative::DCInfo& info = Imperative::DCInfo::Create(s->outputs[0].node, inputs, outputs);\n    info.is_computed_        = true;\n  }\n}\n\nvoid Imperative::DeferredComputeClear(NDArrayHandle* arrays, const int num) {\n  std::vector<nnvm::NodeEntry> outputs;\n  outputs.reserve(num);\n  for (int i = 0; i < num; i++) {\n    NDArray* nd = reinterpret_cast<NDArray*>(arrays[i]);\n    outputs.emplace_back(nd->deferredcompute_entry_);\n  }\n  nnvm::DFSVisit(outputs, [&](const nnvm::ObjectPtr& n) {\n    if (n != nullptr && !n->info.empty()) {\n      Imperative::DCInfo info = Imperative::DCInfo::Get(n);\n      info.inputs_.clear();\n      info.input_handles_.clear();\n      info.outputs_.clear();\n      info.Clear(n);\n    }\n  });\n}\n\nstd::vector<NDArray*> Imperative::Backward(const std::vector<NDArray*>& outputs,\n                                           const std::vector<NDArray*>& ograds,\n                                           const std::vector<NDArray*>& variables,\n                                           bool is_train,\n                                           bool retain_graph,\n                                           bool create_graph) {\n  using namespace nnvm;\n  using namespace imperative;\n  static const std::vector<const Op*> zero_ops{Op::Get(\"zeros_like\"), Op::Get(\"_zeros\")};\n  static const Op* copy_op = Op::Get(\"_copy\");\n\n  // Construct forward graph\n  Graph graph;\n  graph.outputs.reserve(outputs.size());\n  for (const auto& i : outputs) {\n    CHECK(!AGInfo::IsNone(*i))\n        << \"Cannot differentiate node because it is not in a computational graph. \"\n        << \"You need to set is_recording to true or use autograd.record() to save \"\n        << \"computational graphs for backward. If you want to differentiate the same \"\n        << \"graph twice, you need to pass retain_graph=True to backward.\";\n    graph.outputs.emplace_back(i->autograd_entry_);\n  }\n  size_t num_forward_outputs = graph.outputs.size();\n\n  // Prepare head gradients\n  std::vector<NodeEntry> ograd_entries;\n  ograd_entries.reserve(ograds.size());\n  for (size_t i = 0; i < outputs.size(); ++i) {\n    nnvm::ObjectPtr np = Node::Create();\n    np->attrs.name     = \"_head_grad_\" + std::to_string(i);\n    ograd_entries.emplace_back(NodeEntry{np, 0, 0});\n    AGInfo& info = AGInfo::Create(ograd_entries.back().node);\n    info.ctx     = outputs[i]->ctx();\n    if (ograds[i] != nullptr) {\n      info.outputs.emplace_back(*ograds[i]);\n    } else {\n      info.outputs.emplace_back(outputs[i]->shape(), outputs[i]->ctx(), true, outputs[i]->dtype());\n      if (info.outputs.back().shape().Size() != 0) {\n        info.outputs.back() = static_cast<real_t>(1.0);\n      }\n    }\n  }\n\n  // Get gradient graph\n  Symbol sym;\n  sym.outputs = graph.outputs;\n  std::vector<NodeEntry> xs;\n  std::vector<NDArray*> x_grads;\n  std::vector<OpReqType> x_reqs;\n  if (variables.size()) {\n    xs.reserve(variables.size());\n    x_grads.reserve(variables.size());\n    x_reqs.reserve(variables.size());\n    for (size_t i = 0; i < variables.size(); ++i) {\n      CHECK(!AGInfo::IsNone(*variables[i]) &&\n            AGInfo::IsVariable(variables[i]->autograd_entry_.node))\n          << \"Cannot differentiate with respect to the \" << i + 1 << \"-th variable\"\n          << \" because it does not require gradient.\";\n      xs.emplace_back(variables[i]->autograd_entry_);\n      x_grads.push_back(new NDArray());\n      x_reqs.push_back(kWriteTo);\n    }\n  } else {\n    std::vector<ObjectPtr> args = sym.ListInputs(Symbol::kReadOnlyArgs);\n    xs.reserve(args.size());\n    x_grads.reserve(args.size());\n    x_reqs.reserve(args.size());\n    for (const auto& i : args) {\n      AGInfo& info = AGInfo::Get(i);\n      if (info.grad_req == kNullOp)\n        continue;\n      xs.emplace_back(NodeEntry{i, 0, 0});\n      x_grads.push_back(&info.out_grads[0]);\n      x_reqs.push_back(info.grad_req);\n      info.fresh_out_grad = true;\n    }\n    CHECK_GT(xs.size(), 0) << \"There are no inputs in computation graph that require gradients.\";\n  }\n  std::vector<ObjectPtr> nleaf_vars = ListNonleafVariables(sym);\n  std::vector<NodeEntry> us;\n  us.reserve(nleaf_vars.size());\n  for (const auto& i : nleaf_vars) {\n    us.emplace_back(NodeEntry{i, 0, 0});\n  }\n\n  Graph g_graph = pass::MXGradient(graph,\n                                   graph.outputs,\n                                   xs,\n                                   ograd_entries,\n                                   mxnet::AggregateGradient,\n                                   nullptr,\n                                   zero_ops,\n                                   \"_copy\",\n                                   ShapeVector(),\n                                   DTypeVector(),\n                                   us);\n  CHECK_EQ(g_graph.outputs.size(), xs.size());\n  for (const auto& e : g_graph.outputs) {\n    if (e.node->op() == nullptr) {\n      auto node      = Node::Create();\n      node->attrs.op = copy_op;\n      node->inputs.push_back(e);\n      graph.outputs.emplace_back(std::move(node));\n    } else {\n      graph.outputs.push_back(e);\n    }\n  }\n  const auto& idx = graph.indexed_graph();\n  // get number of nodes used in forward pass\n  size_t num_forward_nodes   = 0;\n  size_t num_forward_entries = 0;\n  for (size_t i = 0; i < num_forward_outputs; ++i) {\n    num_forward_nodes =\n        std::max(num_forward_nodes, static_cast<size_t>(idx.outputs()[i].node_id + 1));\n    num_forward_entries =\n        std::max(num_forward_entries, static_cast<size_t>(idx.entry_id(idx.outputs()[i])) + 1);\n  }\n\n  // Allocate buffer\n  std::vector<NDArray> buff(idx.num_node_entries());\n  std::vector<uint32_t> ref_count(buff.size(), 0);\n  std::vector<OpStatePtr> states;\n  std::vector<NDArray*> arrays;\n  arrays.reserve(buff.size());\n  for (auto& buffered_array : buff) {\n    arrays.push_back(&buffered_array);\n  }\n  if (create_graph) {\n    states.resize(num_forward_nodes);\n    nnvm::DFSVisit(sym.outputs, [&](const nnvm::ObjectPtr& n) {\n      AGInfo& info                 = AGInfo::Get(n);\n      states[idx.node_id(n.get())] = info.state;\n      for (uint32_t i = 0; i < info.outputs.size(); ++i) {\n        CHECK(idx.exist(n.get()));\n        size_t nid                = idx.node_id(n.get());\n        size_t eid                = idx.entry_id(nid, i);\n        buff[eid]                 = info.outputs[i];\n        buff[eid].autograd_entry_ = NodeEntry{n, i, 0};\n        ref_count[eid]            = 1;\n      }\n    });\n    for (auto& ograd_entry : ograd_entries) {\n      AGInfo& info = AGInfo::Get(ograd_entry.node);\n      if (!idx.exist(ograd_entry.node.get()))\n        continue;\n      size_t eid                = idx.entry_id(ograd_entry);\n      buff[eid]                 = info.outputs[0];\n      buff[eid].autograd_entry_ = ograd_entry;\n    }\n  } else {\n    states.reserve(num_forward_nodes);\n    for (size_t i = 0; i < num_forward_nodes; ++i) {\n      const AGInfo& info = dmlc::get<AGInfo>(idx[i].source->info);\n      states.emplace_back(info.state);\n      for (size_t j = 0; j < info.outputs.size(); ++j) {\n        size_t eid  = idx.entry_id(i, j);\n        arrays[eid] = const_cast<NDArray*>(&(info.outputs[j]));\n\n        if (retain_graph || info.grad_req != kNullOp)\n          ref_count[eid] = 1;\n      }\n    }\n    for (auto& ograd_entry : ograd_entries) {\n      if (!idx.exist(ograd_entry.node.get()))\n        continue;\n      AGInfo& info                      = AGInfo::Get(ograd_entry.node);\n      arrays[idx.entry_id(ograd_entry)] = &info.outputs[0];\n    }\n  }\n  for (size_t i = num_forward_outputs; i < graph.outputs.size(); ++i) {\n    size_t eid     = idx.entry_id(graph.outputs[i]);\n    arrays[eid]    = x_grads[i - num_forward_outputs];\n    ref_count[eid] = 1;\n  }\n  const std::vector<NodeEntry>& us_grads = g_graph.GetAttr<std::vector<NodeEntry>>(\"nleaf_grads\");\n  CHECK_EQ(us_grads.size(), us.size())\n      << \"Size of queried nleaf_vars and size of their gradients don't match.\";\n  for (size_t i = 0; i < us_grads.size(); i++) {\n    size_t eid   = idx.entry_id(us_grads[i]);\n    AGInfo& info = AGInfo::Get(us[i].node);\n    if (arrays[eid]->dtype_ == -1) {\n      arrays[eid] = &info.out_grads[0];\n    } else {\n      info.out_grads[0] = *arrays[eid];\n    }\n    ref_count[eid] = 1;\n  }\n\n  // Assign context\n  auto vctx = PlaceDevice(idx);\n\n  // Infer shape type\n  {\n    std::pair<uint32_t, uint32_t> node_range, entry_range;\n    node_range  = {num_forward_nodes, idx.num_nodes()};\n    entry_range = {num_forward_entries, idx.num_node_entries()};\n\n    ShapeVector shapes;\n    shapes.reserve(idx.num_node_entries());\n    bool contain_unknown = false;\n    for (const auto& i : arrays)\n      shapes.emplace_back(i->shape());\n    CheckAndInferShape(&graph, std::move(shapes), false, node_range, entry_range, &contain_unknown);\n\n    DTypeVector dtypes;\n    dtypes.reserve(idx.num_node_entries());\n    for (const auto& i : arrays)\n      dtypes.emplace_back(i->dtype());\n    CheckAndInferType(&graph, std::move(dtypes), false, node_range, entry_range);\n\n    StorageTypeVector stypes;\n    stypes.reserve(idx.num_node_entries());\n    for (const auto& i : arrays)\n      stypes.emplace_back(i->storage_type());\n    exec::DevMaskVector dev_mask;\n    dev_mask.reserve(idx.num_nodes());\n    for (const auto& i : vctx)\n      dev_mask.emplace_back(i.dev_mask());\n    CheckAndInferStorageType(\n        &graph, std::move(dev_mask), std::move(stypes), false, node_range, entry_range);\n  }\n\n  // Calculate ref count\n  for (size_t i = num_forward_nodes; i < idx.num_nodes(); ++i) {\n    for (const auto& j : idx[i].inputs) {\n      ++ref_count[idx.entry_id(j)];\n    }\n  }\n\n  // Assign reqs\n  std::vector<OpReqType> array_reqs(arrays.size(), kWriteTo);\n  for (size_t i = num_forward_entries; i < idx.num_node_entries(); ++i) {\n    if (ref_count[i] == 0)\n      array_reqs[i] = kNullOp;\n  }\n  for (size_t i = num_forward_outputs; i < idx.outputs().size(); ++i) {\n    size_t eid      = idx.entry_id(idx.outputs()[i]);\n    array_reqs[eid] = x_reqs[i - num_forward_outputs];\n  }\n  for (size_t i = 0; i < us_grads.size(); i++) {\n    size_t eid      = idx.entry_id(us_grads[i]);\n    AGInfo& info    = AGInfo::Get(us[i].node);\n    array_reqs[eid] = info.grad_req;\n  }\n\n  const auto& shapes         = graph.GetAttr<mxnet::ShapeVector>(\"shape\");\n  const auto& dtypes         = graph.GetAttr<DTypeVector>(\"dtype\");\n  const auto& stypes         = graph.GetAttr<StorageTypeVector>(\"storage_type\");\n  const auto& dispatch_modes = graph.GetAttr<DispatchModeVector>(\"dispatch_mode\");\n\n  for (size_t i = num_forward_nodes; i < idx.num_nodes(); ++i) {\n    auto num_outputs = idx[i].source->num_outputs();\n    for (size_t j = 0; j < num_outputs; ++j) {\n      auto eid = idx.entry_id(i, j);\n      if (arrays[eid]->is_none())\n        arrays[eid]->ReInit(\n            static_cast<NDArrayStorageType>(stypes[eid]), shapes[eid], vctx[i], dtypes[eid]);\n    }\n  }\n\n  for (size_t nid = num_forward_nodes; nid < idx.num_nodes(); ++nid) {\n    const nnvm::NodeAttrs& attrs = idx[nid].source->attrs;\n    for (size_t oid = 0; oid < idx[nid].source->num_outputs(); ++oid) {\n      size_t eid = idx.entry_id(nid, oid);\n      arrays[eid]->AssignStorageInfo(common::NodeAttrsGetProfilerScope(attrs), attrs.name);\n    }\n  }  // for (nid ∈ [num_forward_nodes, idx.num_nodes()))\n\n  if (dmlc::GetEnv(\"MXNET_MEM_PLAN_VERBOSE_LOGGING\", false)) {\n    common::LogMemoryPlan(graph);\n  }\n\n  // Execution\n\n  bool prev_recording = set_is_recording(create_graph);\n  bool prev_training  = set_is_training(is_train);\n  int prev_bulk_size  = Engine::Get()->set_bulk_size(backward_bulk_size_);\n\n  try {\n    RunGraph(retain_graph,\n             idx,\n             arrays,\n             num_forward_nodes,\n             idx.num_nodes(),\n             std::move(array_reqs),\n             std::move(ref_count),\n             &states,\n             dispatch_modes,\n             is_recording());\n  } catch (const dmlc::Error& e) {\n    Engine::Get()->set_bulk_size(prev_bulk_size);\n    set_is_recording(prev_recording);\n    set_is_training(prev_training);\n    throw e;\n  }\n\n  Engine::Get()->set_bulk_size(prev_bulk_size);\n  set_is_recording(prev_recording);\n  set_is_training(prev_training);\n\n  // Clear history\n  if (!retain_graph) {\n    nnvm::DFSVisit(sym.outputs, [&](const nnvm::ObjectPtr& n) {\n      AGInfo::Clear(n);\n      n->inputs.clear();\n    });\n  }\n\n  if (variables.size()) {\n    return x_grads;\n  }\n  return {};\n}\n\nImperative::DCInfo::DCInfo(const std::vector<NDArray*>& inputs,\n                           const std::vector<NDArray*>& outputs) {\n  this->inputs_.reserve(inputs.size());\n  this->input_handles_.reserve(inputs.size());\n  for (const NDArray* arr : inputs) {\n    CHECK(!arr->is_none());\n    this->inputs_.push_back(*arr);\n    this->input_handles_.push_back(arr);\n  }\n\n  this->outputs_.reserve(outputs.size());\n  for (const NDArray* arr : outputs) {\n    CHECK(!arr->is_none());\n    this->outputs_.push_back(*arr);\n  }\n}\n\nImperative::DCInfo& Imperative::DCInfo::Create(const nnvm::ObjectPtr& node,\n                                               const std::vector<NDArray*>& inputs,\n                                               const std::vector<NDArray*>& outputs) {\n  node->info.construct<DCInfo>(inputs, outputs);\n  return Imperative::DCInfo::Get(node);\n}\n\nvoid Imperative::DCInfo::Compute(const NDArray& arr) {\n  if (Imperative::DCInfo::IsComputed(arr)) {\n    if (!shape_is_known(arr.shape())) {\n      // We can't call arr.WaitToRead(); here, as WaitToRead calls Compute\n      // leading to an infinite loop.\n      Engine::Get()->WaitForVar(arr.ptr_->var);\n      if (shape_is_known(arr.ptr_->storage_shape)) {\n        arr.SetShapeFromChunk();\n      } else {\n        CHECK(shape_is_known(arr.shape()));\n      }\n    }\n    return;\n  }\n\n  DCInfo& info      = Imperative::DCInfo::Get(arr.deferredcompute_entry_.node);\n  info.is_computed_ = true;  // We will Invoke at the end of this function.\n\n  // Recursively compute input arrays\n  for (const NDArray& input : info.inputs_) {\n    Compute(input);\n  }\n\n  // Prepare pointers\n  std::vector<NDArray*> ndinputs, ndoutputs;\n  ndinputs.reserve(info.inputs_.size());\n  ndoutputs.reserve(info.outputs_.size());\n  for (NDArray& input : info.inputs_)\n    ndinputs.push_back(&input);\n  for (NDArray& output : info.outputs_)\n    ndoutputs.push_back(&output);\n\n  // Compute this array\n  Imperative::Get()->Invoke(\n      Context::CPU(), arr.deferredcompute_entry_.node->attrs, ndinputs, ndoutputs);\n  if (!shape_is_known(arr.shape())) {\n    arr.WaitToRead();\n    arr.SetShapeFromChunk();\n  }\n\n  // Deallocate copies\n  info.inputs_.clear();\n  info.outputs_.clear();\n}\n\nstd::vector<nnvm::ObjectPtr> Imperative::ListNonleafVariables(const nnvm::Symbol& sym) const {\n  using namespace nnvm;\n  std::vector<ObjectPtr> ret;\n  DFSVisit(sym.outputs, [&ret](const ObjectPtr& node) {\n    AGInfo& info = AGInfo::Get(node);\n    if (info.out_grads.size() > 0 && !node->is_variable()) {\n      ret.push_back(node);\n    }\n  });\n  return ret;\n}\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/imperative/imperative_utils.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#include \"./imperative_utils.h\"\n#include \"./cached_op.h\"\n#include \"../operator/operator_common.h\"\n\nnamespace {\n\nstd::vector<NDArray*> NodeInputs(const nnvm::IndexedGraph& idx,\n                                 const int node_idx,\n                                 const std::vector<NDArray*>& arrays) {\n  const nnvm::IndexedGraph::Node& node = idx[node_idx];\n  const size_t num_inputs              = node.inputs.size();\n  std::vector<NDArray*> ndinputs;\n  ndinputs.reserve(num_inputs);\n  for (const auto& j : node.inputs) {\n    const size_t eid = idx.entry_id(j);\n    ndinputs.emplace_back(arrays[eid]);\n  }\n  return ndinputs;\n}\n\nstd::vector<NDArray*> NodeOutputs(const nnvm::IndexedGraph& idx,\n                                  const int node_idx,\n                                  const std::vector<NDArray*>& arrays) {\n  const nnvm::IndexedGraph::Node& node = idx[node_idx];\n  const size_t num_outputs             = node.source->num_outputs();\n  std::vector<NDArray*> ndoutputs;\n  ndoutputs.reserve(num_outputs);\n  for (size_t j = 0; j < num_outputs; ++j) {\n    const size_t eid = idx.entry_id(node_idx, j);\n    ndoutputs.emplace_back(arrays[eid]);\n  }\n  return ndoutputs;\n}\n\nstd::vector<OpReqType> NodeReq(const nnvm::IndexedGraph& idx,\n                               const int node_idx,\n                               const std::vector<OpReqType>& array_reqs) {\n  const nnvm::IndexedGraph::Node& node = idx[node_idx];\n  const size_t num_outputs             = node.source->num_outputs();\n  std::vector<OpReqType> req;\n  req.reserve(num_outputs);\n  for (size_t j = 0; j < num_outputs; ++j) {\n    const size_t eid = idx.entry_id(node_idx, j);\n    req.push_back(array_reqs[eid]);\n  }\n  return req;\n}\n\nvoid InvokeOperator(const nnvm::IndexedGraph& idx,\n                    const int node_idx,\n                    const bool retain_graph,\n                    const std::vector<NDArray*>& arrays,\n                    Context ctx,\n                    std::vector<OpStatePtr>* p_states,\n                    const std::vector<NDArray*>& ndinputs,\n                    const std::vector<NDArray*>& ndoutputs,\n                    std::vector<OpReqType>* p_req,\n                    std::vector<uint32_t>* p_ref_count,\n                    std::function<void(const OpStatePtr& state)> invoke) {\n  static const auto bwd_cached_op  = Op::Get(\"_backward_CachedOp\");\n  static auto& createop            = nnvm::Op::GetAttr<FCreateOpState>(\"FCreateOpState\");\n  static auto& is_layer_backward   = Op::GetAttr<bool>(\"TIsLayerOpBackward\");\n  std::vector<OpStatePtr>& states  = *p_states;\n  std::vector<OpReqType>& req      = *p_req;\n  std::vector<uint32_t>& ref_count = *p_ref_count;\n\n  const nnvm::IndexedGraph::Node& node = idx[node_idx];\n  if (node.source->op() == bwd_cached_op && node.source->attrs.name == \"_cachedop_backward\") {\n    const auto& cached_op = dmlc::get<CachedOpPtr>(node.source->attrs.parsed);\n    nnvm::Node* fwd_node  = node.source->control_deps[0].get();\n    auto fwd_node_id      = idx.node_id(fwd_node);\n    cached_op->Backward(retain_graph, states[fwd_node_id], ndinputs, req, ndoutputs);\n  } else if (createop.count(node.source->op())) {\n    mxnet::ShapeVector arg_shapes;\n    nnvm::DTypeVector arg_dtypes;\n    arg_shapes.reserve(ndinputs.size());\n    arg_dtypes.reserve(ndinputs.size());\n    for (auto& ndinput : ndinputs) {\n      arg_shapes.emplace_back(ndinput->shape());\n      arg_dtypes.emplace_back(ndinput->dtype());\n    }\n    states[node_idx] = createop[node.source->op()](node.source->attrs, ctx, arg_shapes, arg_dtypes);\n    invoke(states[node_idx]);\n  } else if (is_layer_backward.get(node.source->op(), false)) {\n    nnvm::Node* fwd_node = node.source->control_deps[0].get();\n    auto fwd_node_id     = idx.node_id(fwd_node);\n    invoke(states[fwd_node_id]);\n  } else {\n    invoke(OpStatePtr());\n  }\n  for (const auto& j : node.inputs) {\n    const size_t eid = idx.entry_id(j);\n    if (--ref_count[eid] == 0) {\n      arrays[eid]->ReInit();\n    }\n  }\n  for (size_t j = 0; j < ndoutputs.size(); ++j) {\n    const size_t eid = idx.entry_id(node_idx, j);\n    if (ref_count[eid] == 0) {\n      arrays[eid]->ReInit();\n    }\n  }\n}\n\n}  // namespace\n\nnamespace mxnet {\nnamespace imperative {\n\nvoid RunGraph(const bool retain_graph,\n              const nnvm::IndexedGraph& idx,\n              const std::vector<NDArray*>& arrays,\n              size_t node_start,\n              size_t node_end,\n              std::vector<OpReqType>&& array_reqs,\n              std::vector<uint32_t>&& ref_count,\n              std::vector<OpStatePtr>* p_states,\n              const DispatchModeVector& dispatch_modes,\n              bool recording,\n              mxnet::ShapeVector* shapes,\n              const imperative::CachedOpMonCallback& callback,\n              const bool monitor_all) {\n  CHECK(shapes == nullptr);\n  for (size_t i = node_start; i < node_end; ++i) {\n    const nnvm::IndexedGraph::Node& node = idx[i];\n    if (node.source->op() == nullptr) {\n      continue;\n    }\n    std::vector<NDArray*> ndinputs  = NodeInputs(idx, i, arrays);\n    std::vector<NDArray*> ndoutputs = NodeOutputs(idx, i, arrays);\n    std::vector<OpReqType> req      = NodeReq(idx, i, array_reqs);\n    Context ctx                     = ndoutputs[0]->ctx();\n    if (callback && monitor_all) {\n      mxnet::common::ExecuteMonInputCallback(idx, arrays, i, callback);\n    }\n    auto invoke = [&](const OpStatePtr& state) {\n      const nnvm::IndexedGraph::Node& node = idx[i];\n      DispatchMode dispatch_mode           = dispatch_modes[i];\n      Imperative::Get()->InvokeOp(\n          ctx, node.source->attrs, ndinputs, ndoutputs, req, dispatch_mode, state);\n      if (recording) {\n        Imperative::Get()->RecordOp(NodeAttrs(node.source->attrs), ndinputs, ndoutputs, state);\n      }\n    };\n    InvokeOperator(\n        idx, i, retain_graph, arrays, ctx, p_states, ndinputs, ndoutputs, &req, &ref_count, invoke);\n    if (callback) {\n      mxnet::common::ExecuteMonOutputCallback(idx, arrays, i, callback);\n    }\n  }\n}\n\nvoid NaiveRunGraph(const bool retain_graph,\n                   const Context& default_ctx,\n                   const nnvm::IndexedGraph& idx,\n                   const std::vector<NDArray*>& arrays,\n                   size_t node_start,\n                   size_t node_end,\n                   std::vector<OpReqType>&& array_reqs,\n                   std::vector<uint32_t>&& ref_count,\n                   std::vector<OpStatePtr>* p_states,\n                   const DispatchModeVector& dispatch_modes,\n                   bool recording,\n                   mxnet::ShapeVector* shapes,\n                   const imperative::CachedOpMonCallback& callback,\n                   const bool monitor_all,\n                   const bool skip_engine) {\n  for (size_t i = node_start; i < node_end; ++i) {\n    const nnvm::IndexedGraph::Node& node = idx[i];\n    if (node.source->op() == nullptr) {\n      continue;\n    }\n    std::vector<NDArray*> ndinputs  = NodeInputs(idx, i, arrays);\n    std::vector<NDArray*> ndoutputs = NodeOutputs(idx, i, arrays);\n    std::vector<OpReqType> req;\n    Context ctx = GetContext(node.source->attrs, ndinputs, ndoutputs, default_ctx);\n    if (callback && monitor_all) {\n      mxnet::common::ExecuteMonInputCallback(idx, arrays, i, callback);\n    }\n    auto invoke = [&](const OpStatePtr& state) {\n      const nnvm::IndexedGraph::Node& node = idx[i];\n      DispatchMode dispatch_mode           = DispatchMode::kUndefined;\n      SetShapeType(ctx, node.source->attrs, ndinputs, ndoutputs, &dispatch_mode);\n      SetWriteInplaceReq(ndinputs, ndoutputs, &req);\n      if (skip_engine) {\n        auto new_attr = node.source->attrs;\n        CHECK(new_attr.dict.find(SKIP_ENGINE) == new_attr.dict.end());\n        new_attr.dict[SKIP_ENGINE] = SKIP_ENGINE_SET;\n        Imperative::Get()->InvokeOp(ctx, new_attr, ndinputs, ndoutputs, req, dispatch_mode, state);\n      } else {\n        Imperative::Get()->InvokeOp(\n            ctx, node.source->attrs, ndinputs, ndoutputs, req, dispatch_mode, state);\n      }\n      for (size_t j = 0; j < ndoutputs.size(); ++j) {\n        if (mxnet::op::shape_is_none(ndoutputs[j]->shape())) {\n          ndoutputs[j]->WaitToRead();\n          ndoutputs[j]->SetShapeFromChunk();\n        }\n        size_t eid     = idx.entry_id(i, j);\n        auto shape     = ndoutputs[j]->shape();\n        (*shapes)[eid] = shape;\n      }\n      if (recording) {\n        Imperative::Get()->RecordOp(NodeAttrs(node.source->attrs), ndinputs, ndoutputs, state);\n      }\n    };\n    InvokeOperator(\n        idx, i, retain_graph, arrays, ctx, p_states, ndinputs, ndoutputs, &req, &ref_count, invoke);\n    if (callback) {\n      mxnet::common::ExecuteMonOutputCallback(idx, arrays, i, callback);\n    }\n  }\n}\n\n}  // namespace imperative\n}  // namespace mxnet\n"
  },
  {
    "path": "src/imperative/imperative_utils.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n// The first two includes below need to be in unalphabetical for the miscellaneous CI to pass.\n#include <mxnet/operator.h>\n#include <mxnet/imperative.h>\n#include <nnvm/pass_functions.h>\n\n#include <algorithm>\n#include <map>\n#include <string>\n#include <utility>\n#include <vector>\n\n#include \"./exec_pass.h\"\n#include \"./cuda_graphs.h\"\n#include \"../c_api/c_api_common.h\"\n#include \"../common/exec_utils.h\"\n#include \"../common/utils.h\"\n#include \"../operator/nn/dnnl/dnnl_base-inl.h\"\n#include \"../operator/operator_common.h\"\n#include \"./exec_pass.h\"\n\n#ifndef MXNET_IMPERATIVE_IMPERATIVE_UTILS_H_\n#define MXNET_IMPERATIVE_IMPERATIVE_UTILS_H_\n\nnamespace mxnet {\n\n#if MXNET_USE_ONEDNN == 1\ntemplate <typename T>\nT* pntr(T& obj) {  // NOLINT\n  return &obj;\n}\ntemplate <typename T>\nT* pntr(T* obj) {\n  return obj;\n}\n\ntemplate <typename T>\nvoid InvalidateOutputs(const std::vector<T>* pArrs, const std::vector<OpReqType>& reqs) {\n  auto arrs = *pArrs;\n  for (size_t i = 0; i < arrs.size(); i++) {\n    if (reqs[i] == kWriteTo || reqs[i] == kNullOp)\n      pntr(arrs[i])->InvalidateDNNLData();\n  }\n}\n\n// TODO(alexzai): (MXNET-856) Remove helper function after subgraph feature added\nstatic inline void CreateDefaultInputs(const std::vector<NDArray>& arrs,\n                                       std::vector<NDArray>* out_arrs) {\n  out_arrs->clear();\n  for (size_t i = 0; i < arrs.size(); ++i) {\n    if (arrs[i].IsDNNLData())\n      out_arrs->push_back(arrs[i].Reorder2Default());\n    else\n      out_arrs->push_back(arrs[i]);\n  }\n}\n\n// TODO(alexzai): (MXNET-856) Remove helper function after subgraph feature added\nstatic inline void CreateDefaultInputs(std::vector<NDArray>* pArrs) {\n  auto&& arrs = *pArrs;\n  for (size_t i = 0; i < arrs.size(); ++i)\n    arrs[i].SelfReorder2Default();\n}\n\n#define INVALIDATE_OUTPUTS(outputs, req) InvalidateOutputs(&outputs, req)\n// kCrossDeviceCopy is used for `_copy_to` operator, which doesn't compute immediately in\n// its FCcomputeEx, but AsyncPush the copy operation to engine.\n// So for the case that A is holding dnnl memory, and then copy A to B, and then copy B\n// back to A, we shouldn't invalidate outputs for copying B back to A, because at this time,\n// copying A to B may not happen, and will corrupt A's memory.\n#define INVALIDATE_OUTPUTS_COND(cond, outputs, req) \\\n  if (cond) {                                       \\\n    INVALIDATE_OUTPUTS(outputs, req);               \\\n  }\n\n// add for dnnl OP + no dnnl OP\n#define CREATE_DEFAULT_INPUTS(cond, attrs, func_call)  \\\n  if (cond) {                                          \\\n    const auto is_dnnl = Op::GetAttr<bool>(\"TIsDNNL\"); \\\n    if (!is_dnnl.get(attrs.op, false))                 \\\n      func_call;                                       \\\n  }\n\n#else\n#define INVALIDATE_OUTPUTS(outputs, ...)       // empty macros\n#define INVALIDATE_OUTPUTS_COND(outputs, ...)  // empty macro\n#define CREATE_DEFAULT_INPUTS(input, ...)      // empty macro\n#endif\n\nnamespace imperative {\n\nnamespace {\nstatic const char SKIP_ENGINE[]     = \"__skip_engine__\";\nstatic const char SKIP_ENGINE_SET[] = \"__true__\";\n\ninline bool CheckIfSkipEngine(const nnvm::NodeAttrs& attrs) {\n  const auto& skip_engine_attr = attrs.dict.find(SKIP_ENGINE);\n  if (skip_engine_attr == attrs.dict.end())\n    return false;\n  return (*skip_engine_attr).second == SKIP_ENGINE_SET;\n}\n}  // namespace\n\nstruct MemoryPlanInfo {\n  int storage_id;\n  uint32_t root;\n  size_t size;\n  bool inplace;\n};\n\nstruct EngineOprDeleter {\n  void operator()(engine::Opr* handle) {\n    Engine::Get()->DeleteOperator(handle);\n  }\n};\n\nstruct EngineOprSeg {\n  bool skip;\n  size_t next_nid;\n  std::unique_ptr<engine::Opr, EngineOprDeleter> opr;\n};\n\nusing MemoryPlanVector    = std::vector<MemoryPlanInfo>;\nusing CachedOpMonCallback = std::function<void(const char*, const char*, void*)>;\n\ninline Context GetContext(const nnvm::NodeAttrs& attrs,\n                          const std::vector<NDArray*>& inputs,\n                          const std::vector<NDArray*>& outputs,\n                          const Context& default_ctx) {\n  Context ctx;\n  if (inputs.size()) {\n    ctx = inputs[0]->ctx();\n    for (size_t i = 1; i < inputs.size(); ++i) {\n      CHECK_EQ(inputs[i]->ctx().dev_mask(), ctx.dev_mask())\n          << \"Operator \" << attrs.op->name << \" require all inputs live on the same context. \"\n          << \"But the first argument is on \" << ctx << \" while the \" << i + 1\n          << \"-th argument is on \" << inputs[i]->ctx();\n    }\n  } else if (outputs.size() && !outputs[0]->is_none()) {\n    ctx = outputs[0]->ctx();\n  } else if (attrs.dict.find(\"ctx\") != attrs.dict.end()) {\n    ctx = Context::FromString(attrs.dict.at(\"ctx\"));\n  } else {\n    ctx = default_ctx;\n  }\n  // Non-default context (pinned, shared) does not propagate\n  if (ctx.dev_mask() != ctx.dev_type && inputs.size() != 0U) {\n    ctx = Context::Create(ctx.dev_mask(), ctx.dev_id);\n  }\n#if !MXNET_USE_CUDA\n  if (ctx.dev_mask() == gpu::kDevMask) {\n    LOG(INFO) << \"GPU support is disabled. Compile MXNet with \"\n              << \"USE_CUDA=1 to enable GPU support.\";\n  }\n#endif  // MXNET_USE_CUDA\n  return ctx;\n}\n\n/*! \\brief Set the shape, dtype, storage type and dispatch mode via the\n * attribute inference functions\n *\n * Inferred information is stored in MXAPIThreadLocalEntry. Existing information\n * is overwritten.\n */\ninline void SetShapeType(const Context& ctx,\n                         const nnvm::NodeAttrs& attrs,\n                         const std::vector<NDArray*>& inputs,\n                         const std::vector<NDArray*>& outputs,\n                         DispatchMode* dispatch_mode) {\n  static auto& infershape      = nnvm::Op::GetAttr<mxnet::FInferShape>(\"FInferShape\");\n  static auto& infertype       = nnvm::Op::GetAttr<nnvm::FInferType>(\"FInferType\");\n  static auto& inferstorage    = nnvm::Op::GetAttr<FInferStorageType>(\"FInferStorageType\");\n  MXAPIThreadLocalEntry<>* ret = MXAPIThreadLocalStore<>::Get();\n  // infer shape\n  mxnet::ShapeVector& in_shapes = ret->arg_shapes;\n  in_shapes.clear();\n  in_shapes.reserve(inputs.size());\n  for (auto& i : inputs) {\n    in_shapes.push_back(i->shape());\n  }\n  mxnet::ShapeVector& out_shapes = ret->out_shapes;\n  out_shapes.clear();\n  out_shapes.reserve(outputs.size());\n  for (auto& i : outputs) {\n    out_shapes.push_back(i->shape());\n  }\n  bool is_dynamic_shape_existing = !infershape.count(attrs.op);\n  if (!is_dynamic_shape_existing) {\n    // If any of the inputs is a deferred computed array with unknown shape, we\n    // can't infer shapes.\n    for (const NDArray* i : inputs) {\n      if (!shape_is_known(i->shape()) && !Imperative::DCInfo::IsNone(*i)) {\n        is_dynamic_shape_existing = true;\n        break;\n      }\n    }\n  }\n\n  if (!is_dynamic_shape_existing) {\n    if (!Imperative::Get()->is_np_shape()) {\n      common::ConvertToNumpyShape(&in_shapes);\n      common::ConvertToNumpyShape(&out_shapes);\n    }\n    const bool success = infershape[attrs.op](attrs, &in_shapes, &out_shapes);\n    if (!success) {\n      std::stringstream os;\n      os << \"Operator \" << attrs.op->name << \" inferring shapes failed.\\n\";\n      os << \"input shapes:\\n\";\n      for (const auto& s : in_shapes) {\n        os << s << '\\n';\n      }\n      os << \"output shapes:\\n\";\n      for (const auto& s : out_shapes) {\n        os << s << '\\n';\n      }\n      os << \"operator attributes:\\n\";\n      for (const auto& kv : attrs.dict) {\n        os << kv.first << \" : \" << kv.second << '\\n';\n      }\n      LOG(FATAL) << os.str();\n    }\n    CHECK_EQ(out_shapes.size(), outputs.size());\n  }\n  // infer type\n  std::vector<int>& in_types = ret->arg_types;\n  in_types.clear();\n  in_types.reserve(inputs.size());\n  for (auto& i : inputs) {\n    in_types.push_back(i->dtype());\n  }\n  std::vector<int>& out_types = ret->out_types;\n  out_types.clear();\n  out_types.reserve(outputs.size());\n  for (auto& i : outputs) {\n    out_types.push_back(i->dtype());\n  }\n  bool infer_type_success = false;\n  if (infertype.count(attrs.op)) {\n    infer_type_success = infertype[attrs.op](attrs, &in_types, &out_types);\n  } else {\n    infer_type_success = common::SameType(attrs, &in_types, &out_types);\n  }\n  CHECK(infer_type_success) << \"Operator \" << attrs.op->name << \" is missing FInferType attribute\";\n  CHECK_EQ(out_types.size(), outputs.size());\n\n  // infer storage type\n  auto& in_storage_types = ret->arg_storage_types;\n  in_storage_types.clear();\n  in_storage_types.reserve(inputs.size());\n  for (auto& i : inputs) {\n    in_storage_types.push_back(i->storage_type());\n  }\n  auto& out_storage_types = ret->out_storage_types;\n  out_storage_types.clear();\n  out_storage_types.reserve(outputs.size());\n  for (auto& i : outputs) {\n    out_storage_types.push_back(i->storage_type());\n  }\n  bool infer_stype_success = false;\n  if (inferstorage.count(attrs.op)) {\n    infer_stype_success = inferstorage[attrs.op](\n        attrs, ctx.dev_mask(), dispatch_mode, &in_storage_types, &out_storage_types);\n  } else {\n    // if infer storage attr is not present, apply the default infer storage function\n    infer_stype_success = common::DefaultStorageType(\n        attrs, ctx.dev_mask(), dispatch_mode, &in_storage_types, &out_storage_types);\n  }\n  CHECK(infer_stype_success) << \"Operator not implemented: \"\n                             << common::operator_stype_string(\n                                    attrs, ctx.dev_mask(), in_storage_types, out_storage_types);\n  if (*dispatch_mode == DispatchMode::kFComputeFallback) {\n    common::LogStorageFallback(attrs, ctx.dev_mask(), &in_storage_types, &out_storage_types);\n  }\n\n  CHECK_EQ(out_storage_types.size(), outputs.size());\n  CHECK(*dispatch_mode != DispatchMode::kUndefined);\n  for (size_t i = 0; i < outputs.size(); ++i) {\n    if (outputs[i]->is_none() || (mxnet::op::shape_is_none(outputs[i]->shape()) &&\n                                  Imperative::DCInfo::IsNone(*outputs[i]))) {\n      if (!is_dynamic_shape_existing) {\n        const auto storage_type = static_cast<NDArrayStorageType>(out_storage_types[i]);\n        outputs[i]->ReInit(storage_type, out_shapes[i], ctx, out_types[i]);\n      } else {\n        *outputs[i] = NDArray(ctx, out_types[i]);\n      }\n      outputs[i]->AssignStorageInfo(common::NodeAttrsGetProfilerScope(attrs), attrs.name);\n    } else if (mxnet::op::shape_is_none(outputs[i]->shape())) {\n      // For deferred computed arrays with unknown shape (following dynamic\n      // shape operator), don't use copy assignment as it would destroy the\n      // deferredcompute metadata.\n      if (!is_dynamic_shape_existing) {\n        outputs[i]->Init(out_shapes[i]);\n      }\n      CHECK_EQ(outputs[i]->dtype(), out_types[i])\n          << i << \"-th output has invalid dtype. \"\n          << \"Expecting \" << out_types[i] << \" got \" << outputs[i]->dtype() << \" in operator \"\n          << attrs.op->name;\n    } else {\n      CHECK_EQ(outputs[i]->shape(), out_shapes[i])\n          << i << \"-th output has invalid shape. \"\n          << \"Expecting \" << out_shapes[i] << \" got \" << outputs[i]->shape() << \" in operator \"\n          << attrs.op->name;\n      CHECK_EQ(outputs[i]->dtype(), out_types[i])\n          << i << \"-th output has invalid dtype. \"\n          << \"Expecting \" << out_types[i] << \" got \" << outputs[i]->dtype() << \" in operator \"\n          << attrs.op->name;\n    }\n  }\n}\n\n/*! \\brief Set read and write vars, resource requests and mutate_idx\n *\n * For inputs and outputs arguments only NDArray::var() is accessed.\n */\ninline void SetDependency(const nnvm::NodeAttrs& attrs,\n                          const Context& ctx,\n                          const std::vector<NDArray*>& inputs,\n                          const std::vector<NDArray*>& outputs,\n                          std::vector<engine::VarHandle>* p_read_vars,\n                          std::vector<engine::VarHandle>* p_write_vars,\n                          std::vector<Resource>* p_requested,\n                          std::vector<uint32_t>* p_mutate_idx,\n                          const DispatchMode dispatch_mode) {\n  static auto& fmutate          = nnvm::Op::GetAttr<nnvm::FMutateInputs>(\"FMutateInputs\");\n  static auto& ftmp_resource    = nnvm::Op::GetAttr<FResourceRequest>(\"FResourceRequest\");\n  static auto& ftmp_resource_ex = nnvm::Op::GetAttr<FResourceRequestEx>(\"FResourceRequestEx\");\n\n  std::vector<engine::VarHandle>& read_vars  = *p_read_vars;\n  std::vector<engine::VarHandle>& write_vars = *p_write_vars;\n  std::vector<Resource>& requested           = *p_requested;\n  std::vector<uint32_t>& mutate_idx          = *p_mutate_idx;\n\n  if (fmutate.count(attrs.op)) {\n    mutate_idx = fmutate[attrs.op](attrs);\n  }\n  const bool rsc_req    = (ftmp_resource.count(attrs.op) != 0);\n  const bool rsc_ex_req = (ftmp_resource_ex.count(attrs.op) != 0);\n  if (rsc_req || rsc_ex_req) {\n    int ntmp           = 0;\n    auto resource_reqs = rsc_ex_req ? ftmp_resource_ex[attrs.op](\n                                          attrs, static_cast<int>(ctx.dev_mask()), dispatch_mode) :\n                                      ftmp_resource[attrs.op](attrs);\n    for (const auto& req : resource_reqs) {\n      switch (req.type) {\n        case ResourceRequest::kTempSpace:\n          ++ntmp;\n        case ResourceRequest::kRandom:\n          requested.push_back(ResourceManager::Get()->Request(ctx, req));\n          write_vars.push_back(requested.back().var);\n          break;\n        case ResourceRequest::kParallelRandom:\n          requested.push_back(ResourceManager::Get()->Request(ctx, req));\n          write_vars.push_back(requested.back().var);\n          break;\n#if MXNET_USE_CUDNN == 1\n        case ResourceRequest::kCuDNNDropoutDesc:\n          requested.push_back(ResourceManager::Get()->Request(ctx, req));\n          write_vars.push_back(requested.back().var);\n          break;\n#endif  // MXNET_USE_CUDNN == 1\n        default:\n          LOG(FATAL) << \"resource type not yet supported\";\n      }\n    }\n    CHECK_LE(ntmp, 1) << \"Only support 1 temp space request\";\n  }\n\n  // append extra resource requests for storage fallback\n  if (dispatch_mode == DispatchMode::kFComputeFallback) {\n    requested.push_back(ResourceManager::Get()->Request(ctx, ResourceRequest::kTempSpace));\n    write_vars.push_back(requested.back().var);\n  }\n\n  read_vars.reserve(inputs.size());\n  for (auto& i : inputs) {\n    read_vars.push_back(i->var());\n  }\n  write_vars.reserve(outputs.size() + mutate_idx.size());\n  for (auto& i : outputs) {\n    write_vars.push_back(i->var());\n  }\n  for (auto& i : mutate_idx) {\n    write_vars.push_back(inputs[i]->var());\n  }\n  Engine::Get()->DeduplicateVarHandle(&read_vars, &write_vars);\n}\n\n/*! \\brief Reset vector of OpReqType *req based on input and output NDArrays.\n *\n * Set to kWriteInplace if corresponding output shares variable with any input\n * NDArray. Set to kWriteTo otherwise.\n */\ninline void SetWriteInplaceReq(const std::vector<NDArray*>& inputs,\n                               const std::vector<NDArray*>& outputs,\n                               std::vector<OpReqType>* req) {\n  std::unordered_set<engine::VarHandle> in_vars;\n  in_vars.reserve(inputs.size());\n  for (auto& i : inputs) {\n    in_vars.insert(i->var());\n  }\n  req->clear();\n  req->resize(outputs.size(), kWriteTo);\n  for (size_t i = 0; i < outputs.size(); i++) {\n    // output NDArray shares the memory with the input NDArray\n    if (in_vars.find(outputs[i]->var()) != in_vars.end()) {\n      req->at(i) = kWriteInplace;\n    }\n  }\n}\n\n/*!\n * \\brief Parse parameter attributes into a nnvm::NodeAttrs structure\n * \\param op Pointer to the nnvm Operator object\n * \\param num_inputs Number of operator inputs\n * \\param num_params Number of parameters\n * \\param param_keys Array of string pointers representing the parameter keys\n * \\param param_vals Array of string pointers representing the associated values\n * \\return nnvm::NodeAttrs structure representing the parsed attributes\n */\ninline nnvm::NodeAttrs ParseAttrs(const nnvm::Op* op,\n                                  const int num_inputs,\n                                  const int num_params,\n                                  const char** param_keys,\n                                  const char** param_vals) {\n  static auto& num_args = nnvm::Op::GetAttr<std::string>(\"key_var_num_args\");\n\n  nnvm::NodeAttrs attrs;\n  attrs.op = op;\n  attrs.dict.reserve(num_params + 1);\n  for (int i = 0; i < num_params; ++i) {\n    attrs.dict.emplace(param_keys[i], param_vals[i]);\n  }\n  if (num_args.count(op)) {\n    attrs.dict.emplace(num_args[op], std::to_string(num_inputs));\n  }\n  if (op->attr_parser != nullptr) {\n    op->attr_parser(&attrs);\n  }\n\n  return attrs;\n}\n\n/*!\n * \\brief Determine number of outputs for the given operator\n * \\param op Pointer to the nnvm Operator object\n * \\param attrs  nnvm::NodeAttrs structure representing the operator's attributes\n * \\param num_inputs Number of inputs tot he operator\n * \\param infered_num_outputs The inferred number of outputs\n * \\param num_visible_outputs The actual number of visible outputs\n */\ninline void SetNumOutputs(const nnvm::Op* op,\n                          const nnvm::NodeAttrs& attrs,\n                          const int& num_inputs,\n                          int* infered_num_outputs,\n                          int* num_visible_outputs) {\n  static auto& visible_out = nnvm::Op::GetAttr<nnvm::FNumVisibleOutputs>(\"FNumVisibleOutputs\");\n  int infered_num_inputs;\n  if (op->get_num_inputs != nullptr) {\n    infered_num_inputs = op->get_num_inputs(attrs);\n  } else {\n    infered_num_inputs = op->num_inputs;\n  }\n  CHECK_EQ(num_inputs, infered_num_inputs)\n      << \"Operator \" << op->name << \" expects \" << infered_num_inputs << \" inputs, but got \"\n      << num_inputs << \" instead.\";\n  if (op->get_num_outputs != nullptr) {\n    *infered_num_outputs = op->get_num_outputs(attrs);\n  } else {\n    *infered_num_outputs = op->num_outputs;\n  }\n  *num_visible_outputs = *infered_num_outputs;\n  if (visible_out.count(op)) {\n    *num_visible_outputs = visible_out[op](attrs);\n    CHECK_LE(*num_visible_outputs, *infered_num_outputs);\n  }\n}\n\n/*!\n * \\brief Copy-construct NDArrays referenced by inputs and outputs to p_inputs and p_outputs\n */\ninline void DerefInputOutput(const std::vector<NDArray*>& inputs,\n                             const std::vector<NDArray*>& outputs,\n                             std::vector<NDArray>* p_inputs,\n                             std::vector<NDArray>* p_outputs) {\n  p_inputs->reserve(inputs.size());\n  p_outputs->reserve(outputs.size());\n  for (const auto i : inputs)\n    p_inputs->emplace_back(*i);\n  for (const auto i : outputs)\n    p_outputs->emplace_back(*i);\n}\n\ninline void DerefInputOutput(const std::vector<NDArray*>& inputs,\n                             const std::vector<NDArray*>& outputs,\n                             std::vector<NDArray*>* p_inputs,\n                             std::vector<NDArray*>* p_outputs) {\n  p_inputs->reserve(inputs.size());\n  p_outputs->reserve(outputs.size());\n  for (const auto i : inputs)\n    p_inputs->emplace_back(new NDArray(*i));\n  for (const auto i : outputs)\n    p_outputs->emplace_back(new NDArray(*i));\n}\n\ninline void DerefInputOutputRelease(const std::vector<NDArray*>& inputs,\n                                    const std::vector<NDArray*>& outputs) {\n  for (auto i : inputs)\n    delete i;\n  for (auto i : outputs)\n    delete i;\n}\n\n/*\n * \\brief setup default-storage tblobs from source NDArrays. If any source NDArray has non-default\n *        storage, it creates a temp NDArray with default storage and uses the temp tblob. The\n *        function also records the indices of non-default source NDArrays and the indices of\n *        their corresponding temporary NDArrays in the temp array.\n * \\param src list of source NDArray\n * \\param blobs list of tblobs to return\n * \\param temp_src list of source NDArrays which requires temporary default storage representation\n * \\param temp_dst list of temporary destination NDArrays for default storage representation\n * \\param idx_map mapping from indices in source NDArrays to indices in temp_dst. When not set,\n          indices are not recorded\n * \\return true if any source NDArray need to cast storage\n */\ninline bool SetupDefaultBlobsIn(const std::vector<NDArray*>& src,\n                                const std::vector<NDArray>* bufs,\n                                std::vector<TBlob>* blobs,\n                                std::vector<NDArray>* temp_src,\n                                std::vector<NDArray>* temp_dst,\n                                std::unordered_map<uint32_t, uint32_t>* idx_map) {\n  bool require_cast = false;\n  for (size_t i = 0; i < src.size(); i++) {\n    const auto& nd = *src[i];\n    if (!DEFAULT_DATA(nd)) {\n      (*idx_map)[i] = temp_dst->size();\n      NDArray temp =\n          bufs != nullptr ? bufs->at(i) : NDArray(nd.shape(), nd.ctx(), true, nd.dtype());\n#if MXNET_USE_ONEDNN == 1\n      CHECK(temp.IsDefaultData());\n#endif\n      temp_src->emplace_back(nd);\n      temp_dst->emplace_back(temp);\n      blobs->emplace_back(temp.data());\n      require_cast = true;\n    } else {\n      blobs->push_back(nd.data());\n    }\n  }\n  return require_cast;\n}\n\ninline bool SetupDefaultBlobsOut(const std::vector<NDArray*>& src,\n                                 const std::vector<NDArray>* bufs,\n                                 std::vector<OpReqType>* req,\n                                 std::vector<TBlob>* blobs,\n                                 std::vector<NDArray>* temp_src,\n                                 std::vector<NDArray>* temp_dst) {\n  bool require_cast = false;\n  for (size_t i = 0; i < src.size(); i++) {\n    const auto& nd = *src[i];\n\n#if MXNET_USE_ONEDNN == 1\n    if (req->at(i) == kWriteInplace && nd.IsDNNLData())\n      // If it's write inplace and the output array doesn't use the default\n      // layout, we'll generate a temporary output array below, which means\n      // the input array and the output array are no longer the same array.\n      // we should change the request type.\n      req->at(i) = kWriteTo;\n#endif\n    if (!DEFAULT_DATA(nd)) {\n#if MXNET_USE_ONEDNN == 1\n      NDArray temp;\n      if (bufs != nullptr) {\n        temp = bufs->at(i);\n      } else if (kAddTo == req->at(i)) {\n        temp = nd.IsDNNLData() ? nd.Reorder2Default() : nd;\n      } else {\n        temp = NDArray(nd.shape(), nd.ctx(), true, nd.dtype());\n      }\n      CHECK(temp.IsDefaultData());\n#else\n      NDArray temp =\n          bufs != nullptr ? bufs->at(i) : NDArray(nd.shape(), nd.ctx(), true, nd.dtype());\n#endif\n      temp_src->emplace_back(nd);\n      temp_dst->emplace_back(temp);\n      blobs->emplace_back(temp.data());\n      require_cast = true;\n    } else {\n      blobs->push_back(nd.data());\n    }\n  }\n  return require_cast;\n}\n\n/*\n * \\brief setup default-storage tblobs for input and output NDArrays.\n *        If any NDArray has non-default storage,\n *        it creates a temp NDArray with default storage and uses the temp tblob. The\n *        function also records the indices of non-default source NDArrays and the indices of\n *        their corresponding temporary NDArrays in the temp array.\n */\ninline void SetupDefaultBlobsInOut(const std::vector<NDArray*>& ndinputs,\n                                   const std::vector<NDArray*>& ndoutputs,\n                                   const std::vector<NDArray>* in_bufs,\n                                   const std::vector<NDArray>* out_bufs,\n                                   std::vector<OpReqType>* req,\n                                   std::vector<TBlob>* input_blobs,\n                                   std::vector<TBlob>* output_blobs,\n                                   std::vector<NDArray>* pre_temp_src,\n                                   std::vector<NDArray>* pre_temp_dst,\n                                   std::vector<NDArray>* post_temp_src,\n                                   std::vector<NDArray>* post_temp_dst,\n                                   std::unordered_map<uint32_t, uint32_t>* in_temp_idx_map,\n                                   const std::vector<uint32_t>& mutate_idx) {\n  // populate input blobs\n  SetupDefaultBlobsIn(ndinputs, in_bufs, input_blobs, pre_temp_src, pre_temp_dst, in_temp_idx_map);\n  // populate output blobs\n  SetupDefaultBlobsOut(ndoutputs, out_bufs, req, output_blobs, post_temp_dst, post_temp_src);\n  // add mutable inputs to post temp list\n  for (const auto idx : mutate_idx) {\n    auto map_iter = in_temp_idx_map->find(idx);\n    if (map_iter != in_temp_idx_map->end()) {\n      post_temp_src->push_back(pre_temp_dst->at(map_iter->second));\n      post_temp_dst->push_back(*ndinputs[idx]);\n    }\n  }\n}\n\n#define REDEFINE_INPUTS_OUTPUTS(in, out, newIn, newOut) \\\n  std::vector<NDArray> newIn, newOut;                   \\\n  DerefInputOutput(in, out, &newIn, &newOut);           \\\n  DerefInputOutputRelease(in, out)\n\ninline void PushFCompute(const FCompute& fn,\n                         const nnvm::Op* op,\n                         const nnvm::NodeAttrs& attrs,\n                         const Context& ctx,\n                         const std::vector<engine::VarHandle>& read_vars,\n                         const std::vector<engine::VarHandle>& write_vars,\n                         const std::vector<Resource>& requested,\n                         const std::vector<NDArray*>& p_inputs,\n                         const std::vector<NDArray*>& p_outputs,\n                         const std::vector<uint32_t>& mutate_idx,\n                         const std::vector<OpReqType>& req) {\n  using namespace common;\n  static auto& fexec_type = nnvm::Op::GetAttr<FExecType>(\"FExecType\");\n\n  bool is_train      = Imperative::Get()->is_training();\n  bool need_grad     = Imperative::Get()->is_recording();\n  ExecType exec_type = fexec_type.count(op) ? fexec_type[op](attrs) : ExecType::kSync;\n  CHECK(exec_type == ExecType::kSync);\n  std::vector<NDArray*> inputs, outputs;\n  DerefInputOutput(p_inputs, p_outputs, &inputs, &outputs);\n  const auto& run = [=](RunContext rctx) {\n    std::vector<TBlob> input_blobs, output_blobs;\n    // pre-fcompute and post-fcompute storage fallback src NDArrays and dst NDArrays\n    std::vector<NDArray> pre_temp_src, pre_temp_dst, post_temp_dst, post_temp_src;\n    // mapping from index in input_blobs to index in pre_temp_dst\n    std::unordered_map<uint32_t, uint32_t> in_temp_idx_map;\n    INVALIDATE_OUTPUTS_COND(exec_type != ExecType::kCrossDeviceCopy, outputs, req);\n    std::vector<OpReqType> tmp_req = req;\n    // setup blobs\n    SetupDefaultBlobsInOut(inputs,\n                           outputs,\n                           nullptr,\n                           nullptr,\n                           &tmp_req,\n                           &input_blobs,\n                           &output_blobs,\n                           &pre_temp_src,\n                           &pre_temp_dst,\n                           &post_temp_src,\n                           &post_temp_dst,\n                           &in_temp_idx_map,\n                           mutate_idx);\n    // setup context\n    OpContext opctx{need_grad, is_train, rctx, engine::CallbackOnComplete(), requested};\n    bool is_gpu = ctx.dev_mask() == gpu::kDevMask;\n    // pre-fcompute fallback, cast to default storage type\n    CastNonDefaultStorage(pre_temp_src, pre_temp_dst, opctx, is_gpu);\n    fn(attrs, opctx, input_blobs, tmp_req, output_blobs);\n    // post-fcompute fallback, cast to original storage type\n    CastNonDefaultStorage(post_temp_src, post_temp_dst, opctx, is_gpu);\n    DerefInputOutputRelease(inputs, outputs);\n  };\n  if (CheckIfSkipEngine(attrs)) {\n    // execute without engine\n    run(RunContext{ctx, nullptr, nullptr});\n  } else {\n    Engine::Get()->PushSync(\n        run, ctx, read_vars, write_vars, FnProperty::kNormal, 0, op->name.c_str());\n  }\n}\n\ninline void PushFComputeEx(const FComputeEx& fn,\n                           const nnvm::Op* op,\n                           const nnvm::NodeAttrs& attrs,\n                           const Context& ctx,\n                           const std::vector<engine::VarHandle>& read_vars,\n                           const std::vector<engine::VarHandle>& write_vars,\n                           const std::vector<Resource>& requested,\n                           const std::vector<NDArray*>& p_inputs,\n                           const std::vector<NDArray*>& p_outputs,\n                           const std::vector<OpReqType>& req) {\n  static auto& fexec_type = nnvm::Op::GetAttr<FExecType>(\"FExecType\");\n\n  const bool is_train          = Imperative::Get()->is_training();\n  const bool need_grad         = Imperative::Get()->is_recording();\n  const auto exec_type         = fexec_type.count(op) ? fexec_type[op](attrs) : ExecType::kSync;\n  const auto cross_device_copy = exec_type == ExecType::kCrossDeviceCopy;\n  std::vector<NDArray*> inputs, outputs;\n  DerefInputOutput(p_inputs, p_outputs, &inputs, &outputs);\n  const auto& run = [=](RunContext rctx) {\n    OpContext opctx{need_grad, is_train, rctx, engine::CallbackOnComplete(), requested};\n    REDEFINE_INPUTS_OUTPUTS(inputs, outputs, inputsA, outputsA);\n    INVALIDATE_OUTPUTS_COND(!cross_device_copy, outputsA, req);\n    CREATE_DEFAULT_INPUTS(!cross_device_copy, attrs, CreateDefaultInputs(&inputsA));\n    fn(attrs, opctx, inputsA, req, outputsA);\n  };\n  if (cross_device_copy || CheckIfSkipEngine(attrs)) {\n    run(RunContext{ctx, nullptr, nullptr});\n  } else {\n    CHECK(exec_type == ExecType::kSync);\n    Engine::Get()->PushSync(\n        run, ctx, read_vars, write_vars, FnProperty::kNormal, 0, op->name.c_str());\n  }\n}\n\ninline void PushOperator(const OpStatePtr& state,\n                         const nnvm::Op* op,\n                         const nnvm::NodeAttrs& attrs,\n                         const Context& ctx,\n                         const std::vector<engine::VarHandle>& read_vars,\n                         const std::vector<engine::VarHandle>& write_vars,\n                         const std::vector<Resource>& requested,\n                         const std::vector<NDArray*>& p_inputs,\n                         const std::vector<NDArray*>& p_outputs,\n                         const std::vector<uint32_t>& mutate_idx,\n                         const std::vector<OpReqType>& req,\n                         const DispatchMode dispatch_mode) {\n  using namespace common;\n  static auto& fexec_type = nnvm::Op::GetAttr<FExecType>(\"FExecType\");\n\n  bool is_train      = Imperative::Get()->is_training();\n  bool need_grad     = Imperative::Get()->is_recording();\n  ExecType exec_type = fexec_type.count(op) ? fexec_type[op](attrs) : ExecType::kSync;\n  std::vector<NDArray*> inputs, outputs;\n  DerefInputOutput(p_inputs, p_outputs, &inputs, &outputs);\n\n  auto fcompute_ex = common::GetFCompute<FStatefulComputeEx>(op, \"FStatefulComputeEx\", ctx);\n  if (fcompute_ex != nullptr && dispatch_mode == DispatchMode::kFComputeEx) {\n    const auto& run = [=](RunContext rctx,\n                          engine::CallbackOnStart on_start,\n                          engine::CallbackOnComplete on_complete) {\n      OpContext opctx{need_grad, is_train, rctx, on_complete, requested};\n      REDEFINE_INPUTS_OUTPUTS(inputs, outputs, inputsA, outputsA);\n      INVALIDATE_OUTPUTS_COND(\n          exec_type != ExecType::kCrossDeviceCopy && op->name != \"_CachedOp\", outputsA, req);\n      CREATE_DEFAULT_INPUTS(exec_type != ExecType::kCrossDeviceCopy && op->name != \"_CachedOp\",\n                            attrs,\n                            CreateDefaultInputs(&inputsA));\n      on_start();\n      fcompute_ex(state, opctx, inputsA, req, outputsA);\n    };\n\n    // For operators with subgraphs, we need to invoke them in the main thread\n    // instead of the threaded engine.\n    if (exec_type == ExecType::kSubgraphExec || CheckIfSkipEngine(attrs)) {\n      RunContext rctx{ctx, nullptr, nullptr};\n      run(rctx, engine::CallbackOnStart(), engine::CallbackOnComplete());\n    } else if (exec_type == ExecType::kSync) {\n      Engine::Get()->PushSync(\n          [=](RunContext rctx) {\n            run(rctx, engine::CallbackOnStart(), engine::CallbackOnComplete());\n          },\n          ctx,\n          read_vars,\n          write_vars,\n          FnProperty::kNormal,\n          0,\n          op->name.c_str());\n    } else {\n      CHECK(exec_type == ExecType::kAsync);\n      Engine::Get()->PushAsync(\n          run, ctx, read_vars, write_vars, FnProperty::kAsync, 0, op->name.c_str());\n    }\n  } else {\n    auto fcompute = common::GetFCompute<FStatefulCompute>(op, \"FStatefulCompute\", ctx);\n    CHECK(fcompute != nullptr)\n        << \"One of FStatefulCompute and FStatefulComputeEx must be registered \"\n        << \"for stateful operator \" << op->name;\n\n    const auto& run = [=](RunContext rctx,\n                          engine::CallbackOnStart on_start,\n                          engine::CallbackOnComplete on_complete) {\n      OpContext opctx{need_grad, is_train, rctx, on_complete, requested};\n\n      std::vector<TBlob> input_blobs, output_blobs;\n      // pre-fcompute and post-fcompute storage fallback src NDArrays and dst NDArrays\n      std::vector<NDArray> pre_temp_src, pre_temp_dst, post_temp_dst, post_temp_src;\n      // mapping from index in input_blobs to index in pre_temp_dst\n      std::unordered_map<uint32_t, uint32_t> in_temp_idx_map;\n      INVALIDATE_OUTPUTS_COND(exec_type != ExecType::kCrossDeviceCopy, outputs, req);\n\n      std::vector<OpReqType> tmp_req = req;\n      // populate input blobs and output blobs\n      SetupDefaultBlobsInOut(inputs,\n                             outputs,\n                             nullptr,\n                             nullptr,\n                             &tmp_req,\n                             &input_blobs,\n                             &output_blobs,\n                             &pre_temp_src,\n                             &pre_temp_dst,\n                             &post_temp_src,\n                             &post_temp_dst,\n                             &in_temp_idx_map,\n                             mutate_idx);\n      // setup contexts\n      const bool is_gpu = rctx.get_ctx().dev_mask() == gpu::kDevMask;\n      // pre-fcompute fallback\n      CastNonDefaultStorage(pre_temp_src, pre_temp_dst, opctx, is_gpu);\n      fcompute(state, opctx, input_blobs, tmp_req, output_blobs);\n      // post-fcompute fallback, cast to original storage type, if necessary\n      CastNonDefaultStorage(post_temp_src, post_temp_dst, opctx, is_gpu);\n      DerefInputOutputRelease(inputs, outputs);\n    };\n\n    if (exec_type == ExecType::kSubgraphExec || CheckIfSkipEngine(attrs)) {\n      RunContext rctx{ctx, nullptr};\n      run(rctx, engine::CallbackOnStart(), engine::CallbackOnComplete());\n    } else if (exec_type == ExecType::kSync) {\n      Engine::Get()->PushSync(\n          [=](RunContext rctx) {\n            run(rctx, engine::CallbackOnStart(), engine::CallbackOnComplete());\n          },\n          ctx,\n          read_vars,\n          write_vars,\n          FnProperty::kNormal,\n          0,\n          op->name.c_str());\n    } else {\n      CHECK(exec_type == ExecType::kAsync);\n      Engine::Get()->PushAsync(\n          run, ctx, read_vars, write_vars, FnProperty::kAsync, 0, op->name.c_str());\n    }\n  }\n}\n\ninline bool CheckAndInferShape(nnvm::Graph* p_g,\n                               mxnet::ShapeVector&& shapes,\n                               bool use_inputs,\n                               std::pair<uint32_t, uint32_t> node_range  = {0, 0},\n                               std::pair<uint32_t, uint32_t> entry_range = {0, 0},\n                               bool* contain_unknown                     = nullptr) {\n  using namespace nnvm;\n  if (contain_unknown != nullptr) {\n    *contain_unknown = false;\n  }\n  nnvm::Graph& g = *p_g;\n  if (use_inputs) {\n    if (g.attrs.count(\"shape_inputs\") && g.GetAttr<mxnet::ShapeVector>(\"shape_inputs\") == shapes)\n      return true;\n  } else if (g.attrs.count(\"shape\")) {\n    const auto& prev_shapes = g.GetAttr<mxnet::ShapeVector>(\"shape\");\n    if (prev_shapes.size() == shapes.size()) {\n      bool match = true;\n      for (size_t i = 0; i < shapes.size(); ++i) {\n        if (i == entry_range.first) {\n          i = entry_range.second;\n          if (i >= shapes.size())\n            break;\n        }\n        if (shapes[i] == prev_shapes[i])\n          continue;\n        match = false;\n        break;\n      }\n      if (match)\n        return true;\n    }\n  }\n  g.attrs.erase(\"shape\");\n  g.attrs.erase(\"shape_inputs\");\n  if (node_range.second > node_range.first) {\n    g.attrs[\"node_range\"] = std::make_shared<dmlc::any>(node_range);\n  }\n  if (use_inputs) {\n    g = exec::InferShape(std::move(g), std::move(shapes));\n  } else {\n    g.attrs[\"shape\"] = std::make_shared<dmlc::any>(std::move(shapes));\n    g                = exec::InferShape(std::move(g));\n  }\n  if (contain_unknown == nullptr) {\n    CHECK_EQ(g.GetAttr<size_t>(\"shape_num_unknown_nodes\"), 0U);\n  } else {\n    *contain_unknown = g.GetAttr<size_t>(\"shape_num_unknown_nodes\") != 0U;\n  }\n  return false;\n}\n\ninline bool CheckAndInferType(nnvm::Graph* p_g,\n                              nnvm::DTypeVector&& dtypes,\n                              bool use_inputs,\n                              std::pair<uint32_t, uint32_t> node_range  = {0, 0},\n                              std::pair<uint32_t, uint32_t> entry_range = {0, 0}) {\n  using namespace nnvm;\n  nnvm::Graph& g = *p_g;\n  if (use_inputs) {\n    if (g.attrs.count(\"dtype_inputs\") && g.GetAttr<DTypeVector>(\"dtype_inputs\") == dtypes)\n      return true;\n  } else if (g.attrs.count(\"dtype\")) {\n    const auto& prev_dtypes = g.GetAttr<DTypeVector>(\"dtype\");\n    CHECK_EQ(prev_dtypes.size(), dtypes.size());\n    bool match = true;\n    for (size_t i = 0; i < dtypes.size(); ++i) {\n      if (i == entry_range.first) {\n        i = entry_range.second;\n        if (i >= dtypes.size())\n          break;\n      }\n      if (dtypes[i] == prev_dtypes[i])\n        continue;\n      match = false;\n      break;\n    }\n    if (match)\n      return true;\n  }\n  g.attrs.erase(\"dtype\");\n  g.attrs.erase(\"dtype_inputs\");\n  if (node_range.second > node_range.first) {\n    g.attrs[\"node_range\"] = std::make_shared<dmlc::any>(node_range);\n  }\n  if (node_range.second > node_range.first) {\n    g.attrs[\"node_range\"] = std::make_shared<dmlc::any>(node_range);\n  }\n  if (use_inputs) {\n    g = exec::InferType(std::move(g), std::move(dtypes));\n  } else {\n    g.attrs[\"dtype\"] = std::make_shared<dmlc::any>(std::move(dtypes));\n    g                = exec::InferType(std::move(g));\n  }\n  CHECK_EQ(g.GetAttr<size_t>(\"dtype_num_unknown_nodes\"), 0U);\n\n  return false;\n}\n\ninline bool CheckAndInferStorageType(nnvm::Graph* p_g,\n                                     exec::DevMaskVector&& dev_mask,\n                                     StorageTypeVector&& storage_types,\n                                     bool use_inputs,\n                                     std::pair<uint32_t, uint32_t> node_range  = {0, 0},\n                                     std::pair<uint32_t, uint32_t> entry_range = {0, 0}) {\n  using namespace nnvm;\n  nnvm::Graph& g = *p_g;\n  bool dev_match =\n      g.attrs.count(\"dev_mask\") && g.GetAttr<exec::DevMaskVector>(\"dev_mask\") == dev_mask;\n  if (!dev_match) {\n    g.attrs[\"dev_mask\"] = std::make_shared<dmlc::any>(std::move(dev_mask));\n  }\n\n  if (dev_match && use_inputs) {\n    if (g.attrs.count(\"storage_type_inputs\") &&\n        g.GetAttr<StorageTypeVector>(\"storage_type_inputs\") == storage_types)\n      return true;\n  } else if (dev_match && g.attrs.count(\"storage_type\")) {\n    const auto& prev_storage_types = g.GetAttr<StorageTypeVector>(\"storage_type\");\n    CHECK_EQ(prev_storage_types.size(), storage_types.size());\n    bool match = true;\n    for (size_t i = 0; i < storage_types.size(); ++i) {\n      if (i == entry_range.first) {\n        i = entry_range.second;\n        if (i >= storage_types.size())\n          break;\n      }\n      if (storage_types[i] == prev_storage_types[i])\n        continue;\n      match = false;\n      break;\n    }\n    if (match)\n      return true;\n  }\n  g.attrs.erase(\"dispatch_mode\");\n  g.attrs.erase(\"storage_type\");\n  g.attrs.erase(\"storage_type_inputs\");\n  if (node_range.second > node_range.first) {\n    g.attrs[\"node_range\"] = std::make_shared<dmlc::any>(node_range);\n  }\n  if (use_inputs) {\n    g = exec::InferStorageType(std::move(g), std::move(storage_types));\n  } else {\n    g.attrs[\"storage_type\"] = std::make_shared<dmlc::any>(std::move(storage_types));\n    g                       = exec::InferStorageType(std::move(g));\n  }\n  CHECK_EQ(g.GetAttr<size_t>(\"storage_type_num_unknown_nodes\"), 0U);\n  return false;\n}\n\ninline std::vector<Context> PlaceDevice(const nnvm::IndexedGraph& idx) {\n  static const auto& _copyto = Op::Get(\"_copyto\");\n\n  std::vector<Context> vctx(idx.num_nodes(),\n                            Context::Create(static_cast<Context::DeviceType>(-1), 0));\n  // forward pass\n  for (size_t i = 0; i < idx.num_nodes(); ++i) {\n    if (!idx[i].source->info.empty()) {\n      vctx[i] = dmlc::get<Imperative::AGInfo>(idx[i].source->info).ctx;\n    } else if (idx[i].source->op() == _copyto) {\n      CHECK_GT(idx[i].source->control_deps.size(), 0);\n      auto fwd_nid = idx.node_id(idx[i].source->control_deps[0].get());\n      CHECK_EQ(idx[fwd_nid].source->op(), _copyto);\n      vctx[i] = vctx[idx[fwd_nid].inputs[0].node_id];\n    } else if (idx[i].control_deps.size() &&\n               vctx[idx[i].control_deps[0]].dev_type != static_cast<Context::DeviceType>(-1)) {\n      vctx[i] = vctx[idx[i].control_deps[0]];\n    } else {\n      for (const auto& in : idx[i].inputs) {\n        if (vctx[in.node_id].dev_type == static_cast<Context::DeviceType>(-1))\n          continue;\n        vctx[i] = vctx[in.node_id];\n        break;\n      }\n    }\n  }\n  // backward pass\n  for (int i = idx.num_nodes() - 1; i >= 0; --i) {\n    if (vctx[i].dev_type == static_cast<Context::DeviceType>(-1))\n      continue;\n    if (idx[i].source->op() == _copyto) {\n      auto in_nid = idx[i].inputs[0].node_id;\n      if (vctx[in_nid].dev_type != static_cast<Context::DeviceType>(-1))\n        continue;\n      CHECK_GT(idx[i].source->control_deps.size(), 0);\n      auto fwd_nid = idx.node_id(idx[i].source->control_deps[0].get());\n      CHECK_EQ(idx[fwd_nid].source->op(), _copyto);\n      vctx[in_nid] = vctx[fwd_nid];\n      continue;\n    }\n    for (const auto& j : idx[i].inputs) {\n      if (vctx[j.node_id].dev_type != static_cast<Context::DeviceType>(-1))\n        continue;\n      vctx[j.node_id] = vctx[i];\n    }\n  }\n  // check all context initialized\n  for (size_t i = 0; i < idx.num_nodes(); ++i) {\n    CHECK_NE(vctx[i].dev_type, -1)\n        << \"Cannot decide context for node \" << idx[i].source->attrs.name;\n    // Non-default context do not propagate.\n    vctx[i].dev_type = vctx[i].dev_mask();\n  }\n\n  return vctx;\n}\n\ninline MemoryPlanVector MXPlanMemory(nnvm::Graph* p_g,\n                                     nnvm::StorageVector&& storage,\n                                     const std::vector<uint32_t>& ref_count,\n                                     const std::string& storage_plan,\n                                     const std::pair<uint32_t, uint32_t>& node_range  = {0, 0},\n                                     const std::pair<uint32_t, uint32_t>& entry_range = {0, 0},\n                                     bool detect_inplace_addto                        = false) {\n  using namespace nnvm;\n  nnvm::Graph& g  = *p_g;\n  const auto& idx = g.indexed_graph();\n  if (node_range.second > node_range.first) {\n    g.attrs[\"node_range\"] = std::make_shared<dmlc::any>(node_range);\n  }\n  g.attrs[\"ref_count\"] = std::make_shared<dmlc::any>(ref_count);\n  g.attrs[\"storage\"]   = std::make_shared<dmlc::any>(std::move(storage));\n  g                    = nnvm::ApplyPass(g, \"MXPlanMemory\");\n  if (detect_inplace_addto)\n    g = exec::DetectInplaceAddTo(g);\n\n  const auto& dtypes          = g.GetAttr<DTypeVector>(\"dtype\");\n  const auto& shapes          = g.GetAttr<mxnet::ShapeVector>(\"shape\");\n  const auto& storage_inplace = g.GetAttr<std::vector<int> >(\"storage_inplace_index\");\n  g.attrs[storage_plan]       = std::make_shared<any>(storage_inplace);\n  const auto& storage_ids     = g.GetAttr<StorageVector>(\"storage_id\");\n  uint32_t entry_start        = entry_range.first;\n  uint32_t entry_end =\n      entry_range.second > entry_start ? entry_range.second : idx.num_node_entries();\n  MemoryPlanVector mem_plan(idx.num_node_entries());\n  std::unordered_map<int, uint32_t> sid_to_root;\n\n  for (uint32_t i = entry_start; i < entry_end; ++i) {\n    if (storage_ids[i] < 0) {\n      mem_plan[i] = {storage_ids[i], i, 0, false};\n    } else if (!sid_to_root.count(storage_ids[i])) {\n      CHECK_LT(storage_inplace[i], 0);\n      sid_to_root[storage_ids[i]] = i;\n      mem_plan[i]                 = {\n          storage_ids[i], i, mshadow::mshadow_sizeof(dtypes[i]) * shapes[i].Size(), false};\n    } else {\n      uint32_t root = sid_to_root[storage_ids[i]];\n      mem_plan[i]   = {storage_ids[i], root, 0, storage_inplace[i] >= 0};\n      mem_plan[root].size =\n          std::max(mem_plan[root].size, mshadow::mshadow_sizeof(dtypes[i]) * shapes[i].Size());\n    }\n  }\n\n  return mem_plan;\n}\n\ninline std::multimap<size_t, NDArray> AllocateMemory(\n    const nnvm::Graph& g,\n    const nnvm::IndexedGraph& idx,\n    const Context& default_ctx,\n    const uint32_t entry_start,\n    const uint32_t entry_end,\n    const MemoryPlanVector& mem_plan,\n    const std::vector<NDArray*>& arrays,\n    std::vector<OpReqType>* array_reqs,\n    std::multimap<size_t, NDArray>&& pool = std::multimap<size_t, NDArray>()) {\n  using namespace nnvm;\n  const auto& dtypes = g.GetAttr<DTypeVector>(\"dtype\");\n  const auto& shapes = g.GetAttr<mxnet::ShapeVector>(\"shape\");\n  const auto& stypes = g.GetAttr<StorageTypeVector>(\"storage_type\");\n  std::vector<std::string> data_entry_profiler_scopes(entry_end - entry_start);\n  std::vector<std::string> data_entry_names(entry_end - entry_start);\n\n  std::multimap<size_t, NDArray> new_pool;\n\n  for (uint32_t nid = 0; nid < idx.num_nodes(); ++nid) {\n    const std::string profiler_scope = common::NodeAttrsGetProfilerScope(idx[nid].source->attrs);\n    for (uint32_t i = 0; i < idx[nid].source->num_outputs(); ++i) {\n      uint32_t eid = idx.entry_id(nid, i);\n      if (eid < entry_start || eid >= entry_end) {\n        continue;\n      }\n      data_entry_profiler_scopes[eid - entry_start] = profiler_scope;\n      data_entry_names[eid - entry_start]           = idx[nid].source->attrs.name;\n    }\n  }\n\n  const NDArray* pntr;\n  for (uint32_t i = entry_start; i < entry_end; ++i) {\n    const auto& plan = mem_plan[i];\n    if (plan.storage_id == exec::kExternalStorageID)\n      continue;\n    CHECK(arrays[i]->is_none());\n    if (plan.storage_id == exec::kDynamicStorageID) {\n      *arrays[i] = NDArray(\n          static_cast<NDArrayStorageType>(stypes[i]), shapes[i], default_ctx, true, dtypes[i]);\n      arrays[i]->AssignStorageInfo(data_entry_profiler_scopes[i - entry_start],\n                                   data_entry_names[i - entry_start]);\n      continue;\n    }\n    CHECK_EQ(stypes[i], kDefaultStorage);\n    if (plan.root == i) {\n      auto iter = pool.lower_bound(plan.size);\n      if (iter != pool.end()) {\n        pntr = &new_pool.insert(*iter)->second;\n        pool.erase(iter);\n      } else {\n        NDArray buff(mxnet::TShape({static_cast<nnvm::dim_t>(plan.size)}),\n                     default_ctx,\n                     true,\n                     mshadow::kUint8);\n        buff.AssignStorageInfo(data_entry_profiler_scopes[i - entry_start],\n                               data_entry_names[i - entry_start]);\n        pntr = &new_pool.insert({plan.size, buff})->second;\n      }\n    } else {\n      CHECK_GE(mem_plan[plan.root].storage_id, 0);\n      pntr = arrays[plan.root];\n      if (plan.inplace && array_reqs->at(i) == kWriteTo)\n        array_reqs->at(i) = kWriteInplace;\n    }\n    arrays[i]->InitAsArray(*pntr, shapes[i], dtypes[i]);\n  }\n\n  return new_pool;\n}\n\ninline void SetupOpExec(const nnvm::Graph& g,\n                        size_t nid,\n                        const std::shared_ptr<exec::OpExecutor>& exec,\n                        const std::vector<NDArray*> arrays,\n                        const std::vector<OpReqType> array_reqs) {\n  const auto& idx   = g.indexed_graph();\n  const auto& inode = idx[nid];\n  CHECK_EQ(exec->in_array.size(), 0U);\n  CHECK_EQ(exec->out_array.size(), 0U);\n  for (const auto& e : inode.inputs) {\n    CHECK(!arrays[idx.entry_id(e)]->is_none()) << inode.source->attrs.name;\n    exec->in_array.push_back(*arrays[idx.entry_id(e)]);\n  }\n  for (uint32_t index = 0; index < inode.source->num_outputs(); ++index) {\n    uint32_t eid = idx.entry_id(nid, index);\n    CHECK(!arrays[eid]->is_none()) << inode.source->attrs.name;\n    exec->out_array.push_back(*arrays[eid]);\n    exec->req.push_back(array_reqs[eid]);\n  }\n\n  exec->Setup();\n}\n\ninline Engine::OprHandle CreateEngineOp(\n    const Context& default_ctx,\n    const std::vector<std::shared_ptr<exec::OpExecutor> >& execs,\n    const char* opr_names) {\n  CHECK_GT(execs.size(), 0);\n  std::vector<Engine::VarHandle> use_vars, mutate_vars;\n\n  for (const auto& exec : execs) {\n    CHECK_GT(exec->out_array.size(), 0);\n    CHECK(execs.size() == 1 || exec->exec_type() == ExecType::kSync);\n\n    // the variables\n    for (const auto& nd : exec->in_array) {\n      use_vars.push_back(nd.var());\n    }\n    for (auto& r : exec->op_ctx.requested) {\n      mutate_vars.push_back(r.var);\n    }\n    for (auto& nd : exec->out_array) {\n      mutate_vars.push_back(nd.var());\n    }\n    if (exec->var() != nullptr) {\n      mutate_vars.push_back(exec->var());\n    }\n  }\n\n  // dedup vars\n  Engine::Get()->DeduplicateVarHandle(&use_vars, &mutate_vars);\n  bool is_gpu   = default_ctx.dev_mask() == gpu::kDevMask;\n  bool is_async = execs.size() > 1 ? false : execs[0]->exec_type() == ExecType::kAsync;\n\n#if CUDA_GRAPHS_AVAILABLE\n  // Provide initialized `cuda_graphs_exec`, which when captured\n  // by exec_fun, acts like a static variable inside the mutable closure.\n  cuda_graphs::CudaGraphsExec cuda_graphs_exec(execs, is_gpu, opr_names);\n  auto exec_fun = [cuda_graphs_exec, execs, is_async, is_gpu](\n                      RunContext ctx,\n                      Engine::CallbackOnStart on_start,\n                      Engine::CallbackOnComplete on_complete) mutable {\n    on_start();\n    if (is_async) {\n      execs[0]->op_ctx.async_on_complete = on_complete;\n    }\n    // Run all opr in the sub-graph with CUDA graphs executor if possible\n    cuda_graphs_exec.RunAll(execs, ctx, is_gpu);\n#else\n  auto exec_fun = [execs, is_async, is_gpu](RunContext ctx,\n                                            Engine::CallbackOnStart on_start,\n                                            Engine::CallbackOnComplete on_complete) {\n    on_start();\n    if (is_async) {\n      execs[0]->op_ctx.async_on_complete = on_complete;\n    }\n    exec::OpExecutor::RunAll(execs, ctx, is_gpu);\n#endif\n    // call on complete only if it is async op\n    if (!is_async) {\n      if (is_gpu) {\n#if !MXNET_USE_CUDA\n        LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR;\n#endif\n      }\n      on_complete();\n    }\n  };\n\n  return Engine::Get()->NewOperator(\n      exec_fun, use_vars, mutate_vars, FnProperty::kNormal, opr_names);\n}\n\ninline void CreateEngineOpSeg(const nnvm::IndexedGraph& idx,\n                              const Context default_ctx,\n                              const size_t start_nid,\n                              const size_t end_nid,\n                              const size_t bulk_size,\n                              const std::vector<std::shared_ptr<exec::OpExecutor> >& execs,\n                              const std::vector<int> skip_plus_node,\n                              std::vector<EngineOprSeg>* opr_segs) {\n  size_t seg_start = start_nid;\n  std::vector<std::shared_ptr<exec::OpExecutor> > seg_execs;\n  std::string opr_names = \"[\";\n  for (size_t nid = start_nid; nid < end_nid; ++nid) {\n    const auto& node = idx[nid];\n    if (node.source->is_variable())\n      continue;\n    if (skip_plus_node.size() && skip_plus_node[nid])\n      continue;\n    auto& exec          = execs[nid];\n    const auto& op_name = node.source->op()->name;\n    bool is_async       = exec->exec_type() != ExecType::kSync;\n    bool valid          = exec->out_array.size() > 0;\n\n    // Stop at async nodes and invalid node (due to input/output is not allocated)\n    bool stop = is_async || !valid || seg_execs.size() >= bulk_size;\n\n    // Create opr segment for previous nodes.\n    if (stop && nid > seg_start) {\n      auto& seg = (*opr_segs)[seg_start];\n      if (seg_execs.size()) {\n        seg = EngineOprSeg{false, nid};\n        opr_names.pop_back();\n        opr_names += \"]\";\n        seg.opr.reset(CreateEngineOp(default_ctx, seg_execs, opr_names.c_str()));\n      } else {\n        seg = EngineOprSeg{true, nid, nullptr};\n      }\n      seg_start = nid;\n      seg_execs.clear();\n      opr_names.clear();\n    }\n\n    seg_execs.push_back(exec);\n\n    const auto& inode = idx[nid];\n    opr_names += op_name;\n    opr_names += \"{name=\" + inode.source->attrs.name + \";\";\n    const std::unordered_map<std::string, std::string>& dict = inode.source->attrs.dict;\n    auto num_dict_entries                                    = dict.size();\n    for (auto& k : dict) {\n      opr_names += k.first + \"=\" + k.second;\n      if (--num_dict_entries != 0)\n        opr_names += \";\";\n    }\n    opr_names += \"},\";\n\n    auto& seg = (*opr_segs)[nid];\n    if (!valid) {\n      seg = EngineOprSeg{false, nid + 1, nullptr};\n      seg_execs.clear();\n      opr_names.clear();\n      seg_start = nid + 1;\n    } else if (is_async) {\n      seg = EngineOprSeg{false, nid + 1};\n      opr_names.pop_back();\n      opr_names += \"]\";\n      seg.opr.reset(CreateEngineOp(default_ctx, seg_execs, opr_names.c_str()));\n      seg_execs.clear();\n      opr_names.clear();\n      seg_start = nid + 1;\n    }\n  }\n  // The last segment\n  if (end_nid > seg_start) {\n    auto& seg = (*opr_segs)[seg_start];\n    if (seg_execs.size()) {\n      seg = EngineOprSeg{false, end_nid};\n      opr_names.pop_back();\n      opr_names += \"]\";\n      seg.opr.reset(CreateEngineOp(default_ctx, seg_execs, opr_names.c_str()));\n    } else {\n      seg = EngineOprSeg{true, end_nid, nullptr};\n    }\n  }\n}\n\nvoid RunGraph(const bool retain_graph,\n              const nnvm::IndexedGraph& idx,\n              const std::vector<NDArray*>& arrays,\n              size_t node_start,\n              size_t node_end,\n              std::vector<OpReqType>&& array_reqs,\n              std::vector<uint32_t>&& ref_count,\n              std::vector<OpStatePtr>* p_states,\n              const DispatchModeVector& dispatch_modes,\n              bool recording,\n              mxnet::ShapeVector* shapes          = nullptr,\n              const CachedOpMonCallback& callback = nullptr,\n              const bool monitor_all_             = false);\n\nvoid NaiveRunGraph(const bool retain_graph,\n                   const Context& default_ctx,\n                   const nnvm::IndexedGraph& idx,\n                   const std::vector<NDArray*>& arrays,\n                   size_t node_start,\n                   size_t node_end,\n                   std::vector<OpReqType>&& array_reqs,\n                   std::vector<uint32_t>&& ref_count,\n                   std::vector<OpStatePtr>* p_states,\n                   const DispatchModeVector& dispatch_modes,\n                   bool recording,\n                   mxnet::ShapeVector* shapes,\n                   const CachedOpMonCallback& callback = nullptr,\n                   const bool monitor_all_             = false,\n                   const bool skip_engine              = false);\n\n}  // namespace imperative\n}  // namespace mxnet\n\n#endif  // MXNET_IMPERATIVE_IMPERATIVE_UTILS_H_\n"
  },
  {
    "path": "src/imperative/infer_graph_attr_pass.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file infer_graph_attr_pass.cc\n * \\brief infer graph shape, dtype, and storage type\n */\n\n#include <mxnet/op_attr_types.h>\n#include <mxnet/graph_attr_types.h>\n#include <mxnet/imperative.h>\n#include \"./exec_pass.h\"\n#include \"../operator/operator_common.h\"\n#include \"../common/exec_utils.h\"\n\nnamespace mxnet {\nnamespace exec {\n\ntemplate <typename AttrType, typename FInfer>\nbool ApplyOpInferAttr(const nnvm::Graph& g,\n                      const FInfer& finfer,\n                      const NodeAttrs& attrs,\n                      const uint32_t nid,\n                      std::vector<AttrType>* in_attrs,\n                      std::vector<AttrType>* out_attrs,\n                      DispatchMode* dispatch_mode) {\n  return finfer(attrs, in_attrs, out_attrs);\n}\n\ntemplate <>\nbool ApplyOpInferAttr<int, FInferStorageType>(const nnvm::Graph& g,\n                                              const FInferStorageType& finfer,\n                                              const NodeAttrs& attrs,\n                                              const uint32_t nid,\n                                              std::vector<int>* in_attrs,\n                                              std::vector<int>* out_attrs,\n                                              DispatchMode* dispatch_mode) {\n  const DevMaskVector& dev_masks = g.GetAttr<DevMaskVector>(\"dev_mask\");\n  const bool success = finfer(attrs, dev_masks[nid], dispatch_mode, in_attrs, out_attrs);\n  if (!success) {\n    LOG(FATAL) << \"Operator not implemented: \"\n               << common::operator_stype_string(attrs, dev_masks[nid], *in_attrs, *out_attrs);\n  }\n  if (*dispatch_mode == DispatchMode::kFComputeFallback) {\n    common::LogStorageFallback(attrs, dev_masks[nid], in_attrs, out_attrs);\n  }\n  return true;\n}\n\ntemplate <typename AttrType, typename IsNone>\ninline void GetAttrFromForwardNode(const uint32_t nid,\n                                   const nnvm::IndexedGraph& idx,\n                                   std::vector<AttrType>* rshape_ptr,\n                                   std::vector<bool>* inference_finished,\n                                   IsNone fis_none) {\n  std::vector<AttrType>& rshape         = *rshape_ptr;\n  const nnvm::IndexedGraph::Node& inode = idx[nid];\n  // gradient function, used to get node correspondence.\n  static auto& fgrad                    = Op::GetAttr<nnvm::FGradient>(\"FGradient\");\n  nnvm::ObjectPtr fwd_ptr               = inode.source->control_deps[0];\n  const nnvm::IndexedGraph::Node& fnode = idx[inode.control_deps[0]];\n  // use gradient function to find out the correspondence.\n  std::vector<nnvm::NodeEntry> ograd(fwd_ptr->num_outputs());\n  for (size_t i = 0; i < ograd.size(); ++i) {\n    ograd[i].index = static_cast<uint32_t>(i);\n  }\n  // input gradient list\n  const std::vector<nnvm::NodeEntry>& igrad = fgrad[fwd_ptr->op()](fwd_ptr, ograd);\n  const nnvm::Node* igrad_node              = nullptr;\n  bool all_attrs_known                      = true;\n  // Input gradient assignement\n  for (size_t i = 0; i < igrad.size(); ++i) {\n    if (igrad[i].node->op() == inode.source->op()) {\n      uint32_t eid = idx.entry_id(nid, igrad[i].index);\n      if (fis_none(rshape[idx.entry_id(fnode.inputs[i])])) {\n        // Need to skip empty forward shape, because it may not be\n        // available now and it is possible to infer the forward\n        // shape in one of the next a few passes\n        all_attrs_known = false;\n      } else {\n        if (fis_none(rshape[eid])) {\n          rshape[eid] = rshape[idx.entry_id(fnode.inputs[i])];\n        } else {\n          CHECK_EQ(rshape[eid], rshape[idx.entry_id(fnode.inputs[i])])\n              << \"Backward shape/type inconsistent with the forward shape/type\";\n        }\n      }\n      if (igrad_node == nullptr) {\n        igrad_node = igrad[i].node.get();\n      } else {\n        CHECK(igrad_node == igrad[i].node.get());\n      }\n    }\n  }\n  // out grad entries\n  CHECK(igrad_node != nullptr) << \"Cannot find matching backward op for \"\n                               << inode.source->attrs.name;\n  for (size_t i = 0; i < igrad_node->inputs.size(); ++i) {\n    const nnvm::NodeEntry& e = igrad_node->inputs[i];\n    if (e.node == nullptr) {\n      uint32_t eid = idx.entry_id(inode.inputs[i]);\n      if (fis_none(rshape[eid])) {\n        rshape[eid] = rshape[idx.entry_id(inode.control_deps[0], e.index)];\n      }\n      if (fis_none(rshape[eid])) {\n        // If the attr is still unknown\n        all_attrs_known = false;\n      }\n    }\n  }\n  (*inference_finished)[nid] = all_attrs_known;\n}\n\ntemplate <typename FAccessSubgraphType, typename AttrType, typename IsNone>\nvoid GetAttrFromFusedNode(uint32_t nid,\n                          const nnvm::IndexedGraph& idx,\n                          std::vector<AttrType>* rshape_ptr,\n                          std::vector<bool>* inference_finished,\n                          IsNone fis_none,\n                          const std::string& infer_fusion_name) {\n  std::vector<AttrType>& rshape = *rshape_ptr;\n  const auto& inode             = idx[nid];\n  // gradient function, used to get node correspondence.\n  static auto& fgrad              = Op::GetAttr<nnvm::FGradient>(\"FGradient\");\n  nnvm::ObjectPtr fused_fwd_ptr   = inode.source->control_deps[0];\n  static auto& finfer_fused_shape = Op::GetAttr<FAccessSubgraphType>(infer_fusion_name);\n  auto finfer                     = finfer_fused_shape.get(fused_fwd_ptr->op(), nullptr);\n  CHECK(finfer != nullptr) << \"Operator \" << fused_fwd_ptr->attrs.name\n                           << \" is marked as Fusion but does not allow accessing attributes\";\n  const auto& inferred_attrs = finfer(fused_fwd_ptr->attrs);\n  const auto& fwd_ptr        = std::get<0>(inferred_attrs);\n  const auto& input_attrs    = std::get<1>(inferred_attrs);\n  const auto& output_attrs   = std::get<2>(inferred_attrs);\n\n  // use gradient function to find out the correspondence.\n  std::vector<nnvm::NodeEntry> ograd(fwd_ptr->num_outputs());\n  for (size_t i = 0; i < ograd.size(); ++i) {\n    ograd[i].index = static_cast<uint32_t>(i);\n  }\n  // input gradient list\n  const std::vector<nnvm::NodeEntry>& igrad = fgrad[fwd_ptr->op()](fwd_ptr, ograd);\n  const nnvm::Node* igrad_node              = nullptr;\n  bool all_attrs_known                      = true;\n  // Set the attributes of output gradients\n  // using attributes of forward node inputs\n  for (size_t i = 0; i < igrad.size(); ++i) {\n    if (igrad[i].node->op() == inode.source->op()) {\n      uint32_t eid = idx.entry_id(nid, igrad[i].index);\n      if (fis_none(input_attrs[i])) {\n        // Need to skip empty forward shape, because it may not be\n        // available now and it is possible to infer the forward\n        // shape in one of the next a few passes\n        all_attrs_known = false;\n      } else {\n        if (fis_none(rshape[eid])) {\n          rshape[eid] = input_attrs[i];\n        } else {\n          CHECK_EQ(rshape[eid], input_attrs[i])\n              << \"Backward shape inconsistent with the forward shape\";\n        }\n      }\n      if (igrad_node == nullptr) {\n        igrad_node = igrad[i].node.get();\n      } else {\n        CHECK(igrad_node == igrad[i].node.get());\n      }\n    }\n  }\n\n  // Set the attributes of input gradients\n  // using attributes of forward node outputs\n  CHECK(igrad_node != nullptr) << \"Cannot find matching backward op for \"\n                               << inode.source->attrs.name;\n  for (size_t i = 0; i < igrad_node->inputs.size(); ++i) {\n    const nnvm::NodeEntry& e = igrad_node->inputs[i];\n    if (e.node == nullptr) {\n      uint32_t eid = idx.entry_id(inode.inputs[i]);\n      if (fis_none(rshape[eid])) {\n        rshape[eid] = output_attrs[e.index];\n      }\n      if (fis_none(rshape[eid])) {\n        // If the attr is still unknown\n        all_attrs_known = false;\n      }\n    }\n  }\n  (*inference_finished)[nid] = all_attrs_known;\n}\n\ntemplate <typename FProvideSubgraphType, typename AttrType>\nvoid ProvideAttrToFusion(const uint32_t nid,\n                         const nnvm::IndexedGraph& idx,\n                         const std::vector<AttrType>& rshape,\n                         const std::string& provide_fusion_name) {\n  const auto& inode = idx[nid];\n  std::vector<std::vector<AttrType>> in_attrs;\n  std::vector<std::vector<AttrType>> out_attrs;\n  for (const auto& dep_node : inode.source->control_deps) {\n    in_attrs.push_back({});\n    out_attrs.push_back({});\n    auto& current_in_attrs  = in_attrs.back();\n    auto& current_out_attrs = out_attrs.back();\n    uint32_t dep_node_id    = idx.node_id(dep_node.get());\n    for (const auto& e : idx[dep_node_id].inputs) {\n      current_in_attrs.push_back(rshape[idx.entry_id(e)]);\n    }\n    for (size_t i = 0; i < dep_node->num_outputs(); ++i) {\n      current_out_attrs.push_back(rshape[idx.entry_id(dep_node_id, i)]);\n    }\n  }\n  auto provide =\n      Op::GetAttr<FProvideSubgraphType>(provide_fusion_name).get(inode.source->op(), nullptr);\n  CHECK(provide != nullptr)\n      << \"Encountered Fusion operator that does not implement providing subgraph attr \"\n      << provide_fusion_name << \".\";\n  provide(inode.source->attrs, inode.source->control_deps, in_attrs, out_attrs);\n}\n\n/*!\\brief\n * This is a duplicate of the InferAttr function in nnvm with minor modification\n * to support inferring storage type whose function signature is different from\n * shape/type inference functions'. The nnvm InferAttr will be deprecated\n * in the future. Please use interfaces InferShape, InferType, and InferStorageType\n * to call this function.\n *\n * \\param ret graph used for attribute inference\n * \\param emmpty_val empty value of the attribute\n * \\param infer_name name of the function used for attribute inference\n * \\param infer_fusion_name name of the function used for accessing attributes in fused nodes\n * \\param input_name name of the attribute in the graph used to store the\n *                   input data for attribute inference\n * \\param attr_key_name name of the attribute used for inference for variable nodes\n * \\param attr_name name of the inferred attribute\n * \\param unknown_name name of the attribute storing number of entries\n *                     impossible to infer\n * \\param fis_none function returning true for not fully inferred values\n * \\param fdefault default function used for inference if the node does not\n *                 provide its own implementation.\n * \\param bwd_identity_assign whether the attributes of forward NDArray and backward\n *                            NDArray have to be the same. False only for storage\n *                            type inference\n * \\param dispatch_mode_name name of the dispatch mode attribute on the node. Used for\n *                           storage type inference\n * \\param default_mode_val default value of the dispatch mode attribute on the node. Used\n *                         for storage type inference\n */\ntemplate <typename AttrType,\n          typename FInferType,\n          typename FAccessSubgraphType,\n          typename FProvideSubgraphType,\n          typename IsNone,\n          typename FDefault>\nnnvm::Graph InferAttr(nnvm::Graph&& ret,\n                      const AttrType empty_val,\n                      const char* infer_name,\n                      const char* infer_fusion_name,\n                      const char* provide_fusion_name,\n                      const char* input_name,\n                      const char* attr_key_name,\n                      const char* attr_name,\n                      const char* unknown_name,\n                      IsNone fis_none,\n                      FDefault fdefault,\n                      bool bwd_identity_assign,\n                      const char* dispatch_mode_name,\n                      const DispatchMode default_mode_val = DispatchMode::kUndefined) {\n  using nnvm::IndexedGraph;\n  using nnvm::Op;\n  using AttrVector     = std::vector<AttrType>;\n  using NodeAttrVector = std::vector<DispatchMode>;\n  using dmlc::any;\n\n  const IndexedGraph& idx   = ret.indexed_graph();\n  static auto& finfer_shape = Op::GetAttr<FInferType>(infer_name);\n  static auto& is_backward  = Op::GetAttr<nnvm::TIsBackward>(\"TIsBackward\");\n  // reshape shape vector\n  AttrVector rshape;\n  // vector holding information which operators\n  // finished attribute inference\n  std::vector<bool> inference_finished(idx.num_nodes(), false);\n  // dispatch mode vector\n  DispatchModeVector dispatch_modes;\n  if (ret.attrs.count(attr_name) != 0) {\n    rshape = ret.MoveCopyAttr<AttrVector>(attr_name);\n  } else {\n    rshape.resize(idx.num_node_entries(), empty_val);\n  }\n\n  if (ret.attrs.count(input_name) != 0) {\n    const AttrVector& shape_args = ret.GetAttr<AttrVector>(input_name);\n    CHECK_LE(shape_args.size(), idx.input_nodes().size())\n        << \"More provided \" << attr_name << \"s than number of arguments.\";\n    for (size_t i = 0; i < shape_args.size(); ++i) {\n      rshape[idx.entry_id(idx.input_nodes()[i], 0)] = shape_args[i];\n    }\n  }\n\n  // get the shape hints\n  std::string shape_hints_key = std::string(attr_name) + \"_hints\";\n  if (ret.attrs.count(shape_hints_key)) {\n    nnvm::NodeEntryMap<AttrType> shape_hints =\n        ret.GetAttr<nnvm::NodeEntryMap<AttrType>>(shape_hints_key);\n    for (const auto& kv : shape_hints) {\n      nnvm::NodeEntry e = kv.first;\n      if (idx.exist(e.node.get())) {\n        rshape[idx.entry_id(kv.first)] = kv.second;\n      }\n    }\n  }\n\n  std::string shape_attr_key;\n  if (ret.attrs.count(attr_key_name) != 0) {\n    shape_attr_key = ret.GetAttr<std::string>(attr_key_name);\n    // erase the provided arguments\n    ret.attrs.erase(attr_key_name);\n  }\n\n  // limit inference to part of the graph\n  uint32_t node_start = 0, node_end = idx.num_nodes();\n  if (ret.attrs.count(\"node_range\")) {\n    const auto& range = ret.GetAttr<std::pair<uint32_t, uint32_t>>(\"node_range\");\n    node_start        = range.first;\n    node_end          = range.second;\n    CHECK_GE(node_start, 0);\n    CHECK_LE(node_end, idx.num_nodes());\n    ret.attrs.erase(\"node_range\");\n  }\n  uint32_t entry_start = 0, entry_end = idx.num_node_entries();\n  if (ret.attrs.count(\"entry_range\")) {\n    const auto& range = ret.GetAttr<std::pair<uint32_t, uint32_t>>(\"entry_range\");\n    entry_start       = range.first;\n    entry_end         = range.second;\n    CHECK_GE(entry_start, 0);\n    CHECK_LE(entry_end, idx.num_node_entries());\n    ret.attrs.erase(\"entry_range\");\n  }\n  // populate the node attribute vector\n  if (dispatch_mode_name != nullptr) {\n    if (ret.attrs.count(dispatch_mode_name) != 0) {\n      dispatch_modes = ret.MoveCopyAttr<NodeAttrVector>(dispatch_mode_name);\n    } else {\n      LOG(FATAL) << \"Node attribute \" << dispatch_mode_name << \" does not exist in the graph\";\n    }\n  }\n\n  // Temp space for shape inference.\n  std::vector<AttrType> ishape, oshape;\n\n  // inference step function for nid\n  auto infer_step = [&](uint32_t nid, bool last_iter) {\n    if (inference_finished[nid])\n      return;\n    const auto& inode          = idx[nid];\n    const uint32_t num_inputs  = inode.inputs.size();\n    const uint32_t num_outputs = inode.source->num_outputs();\n    if (inode.source->is_variable()) {\n      // Variable node. No operator. Only one output entry.\n      CHECK(inode.source->op() == nullptr);\n      CHECK_EQ(num_outputs, 1U);\n      const uint32_t out_ent_id = idx.entry_id(nid, 0);\n      if (shape_attr_key.length() != 0 && fis_none(rshape[out_ent_id])) {\n        auto it = inode.source->attrs.dict.find(shape_attr_key);\n        if (it != inode.source->attrs.dict.end()) {\n          std::istringstream is(it->second);\n          CHECK(is >> rshape[out_ent_id]) << \"Invalid attribute\";\n        }\n      }\n      if (!fis_none(rshape[out_ent_id])) {\n        inference_finished[nid] = true;\n      }\n      // assign a default value to node attribute\n      if (dispatch_mode_name != nullptr) {\n        op::dispatch_mode_assign(&dispatch_modes[nid], default_mode_val);\n      }\n    } else if (is_backward.get(inode.source->op(), false) && inode.source->control_deps.size() &&\n               bwd_identity_assign) {\n      CHECK(dispatch_mode_name == nullptr)\n          << \"Backward inference for node attributes is not available\";\n      CHECK_GE(inode.source->control_deps.size(), 1U)\n          << \"BackwardOp need to have control_deps to its forward op\";\n      nnvm::ObjectPtr fwd_ptr = inode.source->control_deps[0];\n      CHECK(fwd_ptr->op() != nullptr) << \"Forward op cannot be a variable\";\n\n      static auto& is_fusion_helper = Op::GetAttr<exec::TIsFusionHelper>(\"TIsFusionHelper\");\n      if (!is_fusion_helper.get(fwd_ptr->op(), false)) {\n        GetAttrFromForwardNode(nid, idx, &rshape, &inference_finished, fis_none);\n      } else {\n        GetAttrFromFusedNode<FAccessSubgraphType>(\n            nid, idx, &rshape, &inference_finished, fis_none, infer_fusion_name);\n      }\n    } else {\n      DispatchMode* dispatch_mode = nullptr;\n      // Forward operator inference.\n      ishape.resize(num_inputs, empty_val);\n      for (uint32_t i = 0; i < ishape.size(); ++i) {\n        ishape[i] = rshape[idx.entry_id(inode.inputs[i])];\n      }\n      oshape.resize(num_outputs, empty_val);\n      for (uint32_t i = 0; i < oshape.size(); ++i) {\n        oshape[i] = rshape[idx.entry_id(nid, i)];\n      }\n      if (dispatch_mode_name != nullptr) {\n        dispatch_mode = &dispatch_modes[nid];\n      }\n      auto finfer = finfer_shape.get(inode.source->op(), fdefault);\n      if (finfer != nullptr) {\n        // Call inference function of the operator.\n        try {\n          static auto& is_fusion = Op::GetAttr<exec::TIsFusion>(\"TIsFusion\");\n          if (is_fusion.get(inode.source->op(), false)) {\n            ProvideAttrToFusion<FProvideSubgraphType>(nid, idx, rshape, provide_fusion_name);\n          }\n          ApplyOpInferAttr(ret, finfer, inode.source->attrs, nid, &ishape, &oshape, dispatch_mode);\n          bool finished = true;\n          for (const auto& attr : ishape) {\n            if (fis_none(attr))\n              finished = false;\n          }\n          for (const auto& attr : oshape) {\n            if (fis_none(attr))\n              finished = false;\n          }\n          inference_finished[nid] = finished;\n        } catch (const std::exception& e) {\n          throw dmlc::Error(\"Error in operator \" + inode.source->attrs.name + \": \" + e.what());\n        }\n      } else {\n        // Operator does not provide sttribute inference function,\n        // so we need to test if everything was inferred by other operators\n        bool all_attrs_known = true;\n        for (const auto& attr : ishape) {\n          if (fis_none(attr)) {\n            all_attrs_known = false;\n          }\n        }\n        for (const auto& attr : oshape) {\n          if (fis_none(attr)) {\n            all_attrs_known = false;\n          }\n        }\n        inference_finished[nid] = all_attrs_known;\n        if (!all_attrs_known) {\n          CHECK(!last_iter) << \"Attribute \" << infer_name << \" is not registered by op \"\n                            << inode.source->op()->name\n                            << \". We are not able to complete the inference because of this\";\n        }\n      }\n      // Save to the result map.\n      for (uint32_t i = 0; i < num_inputs; ++i) {\n        rshape[idx.entry_id(inode.inputs[i])] = ishape[i];\n      }\n      for (uint32_t i = 0; i < num_outputs; ++i) {\n        rshape[idx.entry_id(nid, i)] = oshape[i];\n      }\n    }\n  };\n\n  size_t last_num_unknown;\n  size_t num_unknown_dispatch_mode = dispatch_mode_name ? node_end - node_start : 0;\n  size_t num_unknown_entry_attr    = entry_end - entry_start;\n  size_t num_unknown               = num_unknown_entry_attr + num_unknown_dispatch_mode;\n  bool last_iter                   = false;\n  bool do_next_iteration           = true;\n  int i                            = 0;\n  do {\n    if (i % 2 == 0) {\n      for (uint32_t nid = node_start; nid < node_end; ++nid) {\n        infer_step(nid, last_iter);\n      }\n    } else {\n      // backward inference\n      for (uint32_t i = node_end; i != node_start; --i) {\n        infer_step(i - 1, last_iter);\n      }\n    }\n    last_num_unknown = num_unknown;\n    num_unknown      = 0;\n    for (size_t j = entry_start; j < entry_end; ++j) {\n      if (fis_none(rshape[j])) {\n        ++num_unknown;\n      }\n    }\n    if (dispatch_mode_name) {\n      for (size_t i = node_start; i < node_end; i++) {\n        if (dispatch_modes[i] == DispatchMode::kUndefined)\n          ++num_unknown;\n      }\n    }\n    do_next_iteration = num_unknown > 0 && last_num_unknown > num_unknown;\n    if (!do_next_iteration && !last_iter) {\n      // Check if every op agrees that it should be\n      // the end of attribute inference. If not,\n      // perform one final step\n      for (const bool done : inference_finished) {\n        do_next_iteration = do_next_iteration || !done;\n      }\n      last_iter = true;\n    }\n    ++i;\n  } while (do_next_iteration);\n  // set the shapes\n  ret.attrs[attr_name] = std::make_shared<any>(std::move(rshape));\n  // set the shapes\n  if (dispatch_mode_name) {\n    ret.attrs[dispatch_mode_name] = std::make_shared<any>(std::move(dispatch_modes));\n  }\n  // number of nodes who knows the shape.\n  ret.attrs[unknown_name] = std::make_shared<any>(num_unknown);\n  return std::move(ret);\n}\n\n/*!\\brief\n * This is a version of the InferAttr function specifically for shape inference.\n *\n * \\param ret graph used for attribute inference\n * \\param emmpty_val empty value of the attribute\n * \\param infer_name name of the function used for attribute inference\n * \\param input_name name of the attribute in the graph used to store the\n *                   input data for attribute inference\n * \\param attr_key_name name of the attribute used for inference for variable nodes\n * \\param attr_name name of the inferred attribute\n * \\param unknown_name name of the attribute storing number of entries\n *                     impossible to infer\n * \\param fis_none function returning true for not fully inferred values\n * \\param fnum_unknown function returning how many elements are unknown in\n *                     partially inferred value of the attribute\n * \\param fdefault default function used for inference if the node does not\n *                 provide its own implementation.\n * \\param bwd_identity_assign whether the attributes of forward NDArray and backward\n *                            NDArray have to be the same. False only for storage\n *                            type inference\n * \\param dispatch_mode_name name of the dispatch mode attribute on the node. Used for\n *                           storage type inference\n * \\param default_mode_val default value of the dispatch mode attribute on the node. Used\n *                         for storage type inference\n */\ntemplate <typename IsNone, typename FDefault, typename FNumUnknown>\nnnvm::Graph InferShapeAttr(nnvm::Graph&& ret,\n                           const mxnet::TShape empty_val,\n                           const char* infer_name,\n                           const char* input_name,\n                           const char* attr_key_name,\n                           const char* attr_name,\n                           const char* unknown_name,\n                           IsNone fis_none,\n                           FNumUnknown fnum_unknown,\n                           FDefault fdefault,\n                           bool bwd_identity_assign,\n                           const char* dispatch_mode_name,\n                           const DispatchMode default_mode_val = DispatchMode::kUndefined) {\n  using nnvm::IndexedGraph;\n  using nnvm::Op;\n  using AttrType       = mxnet::TShape;\n  using FInferType     = mxnet::FInferShape;\n  using AttrVector     = std::vector<AttrType>;\n  using NodeAttrVector = std::vector<DispatchMode>;\n  using dmlc::any;\n  const IndexedGraph& idx   = ret.indexed_graph();\n  static auto& finfer_shape = Op::GetAttr<FInferType>(infer_name);\n  static auto& is_backward  = Op::GetAttr<nnvm::TIsBackward>(\"TIsBackward\");\n  // reshape shape vector\n  AttrVector rshape;\n  // vector holding information which operators\n  // finished attribute inference\n  std::vector<bool> inference_finished(idx.num_nodes(), false);\n  // dispatch mode vector\n  DispatchModeVector dispatch_modes;\n  if (ret.attrs.count(attr_name) != 0) {\n    rshape = ret.MoveCopyAttr<AttrVector>(attr_name);\n  } else {\n    rshape.resize(idx.num_node_entries(), empty_val);\n  }\n\n  if (ret.attrs.count(input_name) != 0) {\n    const AttrVector& shape_args = ret.GetAttr<AttrVector>(input_name);\n    CHECK_LE(shape_args.size(), idx.input_nodes().size())\n        << \"More provided \" << attr_name << \"s than number of arguments.\";\n    for (size_t i = 0; i < shape_args.size(); ++i) {\n      rshape[idx.entry_id(idx.input_nodes()[i], 0)] = shape_args[i];\n    }\n  }\n\n  // get the shape hints\n  std::string shape_hints_key = std::string(attr_name) + \"_hints\";\n  if (ret.attrs.count(shape_hints_key)) {\n    nnvm::NodeEntryMap<AttrType> shape_hints =\n        ret.GetAttr<nnvm::NodeEntryMap<AttrType>>(shape_hints_key);\n    for (const auto& kv : shape_hints) {\n      nnvm::NodeEntry e = kv.first;\n      if (idx.exist(e.node.get())) {\n        rshape[idx.entry_id(kv.first)] = kv.second;\n      }\n    }\n  }\n\n  std::string shape_attr_key;\n  if (ret.attrs.count(attr_key_name) != 0) {\n    shape_attr_key = ret.GetAttr<std::string>(attr_key_name);\n    // erase the provided arguments\n    ret.attrs.erase(attr_key_name);\n  }\n\n  // limit inference to part of the graph\n  uint32_t node_start = 0, node_end = idx.num_nodes();\n  if (ret.attrs.count(\"node_range\")) {\n    const auto& range = ret.GetAttr<std::pair<uint32_t, uint32_t>>(\"node_range\");\n    node_start        = range.first;\n    node_end          = range.second;\n    CHECK_GE(node_start, 0);\n    CHECK_LE(node_end, idx.num_nodes());\n    ret.attrs.erase(\"node_range\");\n  }\n  uint32_t entry_start = 0, entry_end = idx.num_node_entries();\n  if (ret.attrs.count(\"entry_range\")) {\n    const auto& range = ret.GetAttr<std::pair<uint32_t, uint32_t>>(\"entry_range\");\n    entry_start       = range.first;\n    entry_end         = range.second;\n    CHECK_GE(entry_start, 0);\n    CHECK_LE(entry_end, idx.num_node_entries());\n    ret.attrs.erase(\"entry_range\");\n  }\n  // populate the node attribute vector\n  if (dispatch_mode_name != nullptr) {\n    if (ret.attrs.count(dispatch_mode_name) != 0) {\n      dispatch_modes = ret.MoveCopyAttr<NodeAttrVector>(dispatch_mode_name);\n    } else {\n      LOG(FATAL) << \"Node attribute \" << dispatch_mode_name << \" does not exist in the graph\";\n    }\n  }\n\n  // Temp space for shape inference.\n  std::vector<AttrType> ishape, oshape;\n  // whether a shape is dynamic\n  std::vector<int> is_dynamic(rshape.size(), 0);\n\n  // convert to numpy compatible shape to use operator's infer shape function\n  if (!Imperative::Get()->is_np_shape()) {\n    common::ConvertToNumpyShape(&rshape);\n  }\n\n  // inference step function for nid\n  auto infer_step = [&](uint32_t nid, bool last_iter) {\n    if (inference_finished[nid])\n      return;\n    const auto& inode          = idx[nid];\n    const std::string name     = inode.source->attrs.name;\n    const uint32_t num_inputs  = inode.inputs.size();\n    const uint32_t num_outputs = inode.source->num_outputs();\n\n    if (inode.source->is_variable()) {\n      // Variable node. No operator. Only one output entry.\n      CHECK(inode.source->op() == nullptr);\n      CHECK_EQ(num_outputs, 1U);\n      const uint32_t out_ent_id = idx.entry_id(nid, 0);\n      if (shape_attr_key.length() != 0 && fis_none(rshape[out_ent_id])) {\n        auto it = inode.source->attrs.dict.find(shape_attr_key);\n        if (it != inode.source->attrs.dict.end()) {\n          std::istringstream is(it->second);\n          CHECK(is >> rshape[out_ent_id]) << \"Invalid attribute\";\n          if (!Imperative::Get()->is_np_shape()) {\n            common::ConvertToNumpyShape(&rshape[out_ent_id]);\n          }\n        }\n      }\n      if (!fis_none(rshape[out_ent_id])) {\n        inference_finished[nid] = true;\n      }\n      // assign a default value to node attribute\n      if (dispatch_mode_name != nullptr) {\n        op::dispatch_mode_assign(&dispatch_modes[nid], default_mode_val);\n      }\n    } else if (is_backward.get(inode.source->op(), false) && inode.source->control_deps.size() &&\n               bwd_identity_assign) {\n      CHECK(dispatch_mode_name == nullptr)\n          << \"Backward inference for node attributes is not available\";\n      CHECK_GE(inode.source->control_deps.size(), 1U)\n          << \"BackwardOp need to have control_deps to its forward op\";\n      nnvm::ObjectPtr fwd_ptr = inode.source->control_deps[0];\n      CHECK(fwd_ptr->op() != nullptr) << \"Forward op cannot be a variable\";\n\n      static auto& is_fusion_helper = Op::GetAttr<exec::TIsFusionHelper>(\"TIsFusionHelper\");\n      if (!is_fusion_helper.get(fwd_ptr->op(), false)) {\n        GetAttrFromForwardNode(nid, idx, &rshape, &inference_finished, fis_none);\n      } else {\n        GetAttrFromFusedNode<exec::FAccessSubgraphShape>(\n            nid, idx, &rshape, &inference_finished, fis_none, \"FAccessSubgraphShape\");\n      }\n    } else {\n      DispatchMode* dispatch_mode = nullptr;\n      // Forward operator inference.\n      ishape.resize(num_inputs, empty_val);\n      bool is_input_dynamic_shape = false;\n      for (uint32_t i = 0; i < ishape.size(); ++i) {\n        ishape[i] = rshape[idx.entry_id(inode.inputs[i])];\n        if (!mxnet::ndim_is_known(ishape[i]) && is_dynamic[idx.entry_id(inode.inputs[i])]) {\n          is_input_dynamic_shape = true;\n        }\n      }\n      oshape.resize(num_outputs, empty_val);\n      for (uint32_t i = 0; i < oshape.size(); ++i) {\n        oshape[i] = rshape[idx.entry_id(nid, i)];\n      }\n      if (dispatch_mode_name != nullptr) {\n        dispatch_mode = &dispatch_modes[nid];\n      }\n      auto finfer = finfer_shape.get(inode.source->op(), fdefault);\n      if (finfer == nullptr || is_input_dynamic_shape) {\n        for (uint32_t i = 0; i < oshape.size(); ++i) {\n          if (!mxnet::ndim_is_known(oshape[i].ndim())) {\n            is_dynamic[idx.entry_id(nid, i)] = 1;\n          }\n        }\n        inference_finished[nid] = true;\n      } else {\n        // Call inference function of the operator.\n        try {\n          static auto& is_fusion = Op::GetAttr<exec::TIsFusion>(\"TIsFusion\");\n          if (is_fusion.get(inode.source->op(), false)) {\n            ProvideAttrToFusion<exec::FProvideSubgraphShape>(\n                nid, idx, rshape, \"FProvideSubgraphShape\");\n          }\n          ApplyOpInferAttr(ret, finfer, inode.source->attrs, nid, &ishape, &oshape, dispatch_mode);\n          bool finished = true;\n          for (const auto& attr : ishape) {\n            if (fis_none(attr))\n              finished = false;\n          }\n          for (const auto& attr : oshape) {\n            if (fis_none(attr))\n              finished = false;\n          }\n          inference_finished[nid] = finished;\n        } catch (const std::exception& e) {\n          throw dmlc::Error(\"Error in operator \" + inode.source->attrs.name + \": \" + e.what());\n        }\n      }\n      // Save to the result map.\n      for (uint32_t i = 0; i < num_inputs; ++i) {\n        rshape[idx.entry_id(inode.inputs[i])] = ishape[i];\n      }\n      for (uint32_t i = 0; i < num_outputs; ++i) {\n        rshape[idx.entry_id(nid, i)] = oshape[i];\n      }\n    }\n  };\n\n  size_t last_num_unknown;\n  size_t num_unknown     = static_cast<size_t>(-1);  // Infinity\n  bool last_iter         = false;\n  bool do_next_iteration = true;\n\n  int i = 0;\n  do {\n    if (i % 2 == 0) {\n      // forward inference\n      for (uint32_t nid = node_start; nid < node_end; ++nid) {\n        infer_step(nid, last_iter);\n      }\n    } else {\n      // backward inference\n      for (uint32_t i = node_end; i != node_start; --i) {\n        infer_step(i - 1, last_iter);\n      }\n    }\n    last_num_unknown = num_unknown;\n    num_unknown      = 0;\n    for (size_t j = entry_start; j < entry_end; ++j) {\n      if (fis_none(rshape[j])) {\n        num_unknown += fnum_unknown(rshape[j]);\n      }\n    }\n    if (dispatch_mode_name) {\n      for (size_t i = node_start; i < node_end; i++) {\n        if (dispatch_modes[i] == DispatchMode::kUndefined) {\n          ++num_unknown;\n        }\n      }\n    }\n    do_next_iteration = num_unknown > 0 && last_num_unknown > num_unknown;\n    if (!do_next_iteration && !last_iter) {\n      // Check if every op agrees that it should be\n      // the end of attribute inference. If not,\n      // perform one final step\n      for (const bool done : inference_finished) {\n        do_next_iteration = do_next_iteration || !done;\n      }\n      last_iter = true;\n    }\n    ++i;\n  } while (do_next_iteration);\n  // set the shapes\n  ret.attrs[attr_name] = std::make_shared<any>(std::move(rshape));\n  // set the shapes\n  if (dispatch_mode_name) {\n    ret.attrs[dispatch_mode_name] = std::make_shared<any>(std::move(dispatch_modes));\n  }\n  // number of nodes who knows the shape.\n  ret.attrs[unknown_name] = std::make_shared<any>(num_unknown);\n  return std::move(ret);\n}\n\nnnvm::Graph InferShape(nnvm::Graph&& graph,\n                       mxnet::ShapeVector&& shape_inputs,\n                       const std::string& shape_attr_key) {\n  using dmlc::any;\n  if (shape_inputs.size() != 0) {\n    graph.attrs[\"shape_inputs\"] = std::make_shared<any>(std::move(shape_inputs));\n  }\n  if (shape_attr_key.length() != 0) {\n    graph.attrs[\"shape_attr_key\"] = std::make_shared<any>(shape_attr_key);\n  }\n  return InferShapeAttr(\n      std::move(graph),\n      mxnet::TShape(),\n      \"FInferShape\",\n      \"shape_inputs\",\n      \"shape_attr_key\",\n      \"shape\",\n      \"shape_num_unknown_nodes\",\n      [](const mxnet::TShape& s) { return !mxnet::shape_is_known(s); },\n      [](const mxnet::TShape& s) {\n        if (!mxnet::ndim_is_known(s)) {\n          return static_cast<size_t>(1);\n        }\n        size_t ret = 0;\n        for (const auto& val : s) {\n          if (!mxnet::dim_size_is_known(val)) {\n            ++ret;\n          }\n        }\n        return ret;\n      },\n      nullptr,\n      true,\n      nullptr);\n}\n\nnnvm::Graph InferType(nnvm::Graph&& graph,\n                      nnvm::DTypeVector&& dtype_inputs,\n                      const std::string& dtype_attr_key) {\n  using dmlc::any;\n  if (dtype_inputs.size() != 0) {\n    graph.attrs[\"dtype_inputs\"] = std::make_shared<any>(std::move(dtype_inputs));\n  }\n  if (dtype_attr_key.length() != 0) {\n    graph.attrs[\"dtype_attr_key\"] = std::make_shared<any>(dtype_attr_key);\n  }\n  return InferAttr<int, nnvm::FInferType, exec::FAccessSubgraphType, exec::FProvideSubgraphType>(\n      std::move(graph),\n      -1,\n      \"FInferType\",\n      \"FAccessSubgraphType\",\n      \"FProvideSubgraphType\",\n      \"dtype_inputs\",\n      \"dtype_attr_key\",\n      \"dtype\",\n      \"dtype_num_unknown_nodes\",\n      [](const int t) { return t == -1; },\n      common::SameType,\n      true,\n      nullptr);\n}\n\nnnvm::Graph InferStorageType(nnvm::Graph&& graph,\n                             StorageTypeVector&& storage_type_inputs,\n                             const std::string& storage_type_attr_key) {\n  using dmlc::any;\n  if (storage_type_inputs.size() != 0) {\n    graph.attrs[\"storage_type_inputs\"] = std::make_shared<any>(std::move(storage_type_inputs));\n  }\n  if (storage_type_attr_key.length() != 0) {\n    graph.attrs[\"storage_type_attr_key\"] = std::make_shared<any>(storage_type_attr_key);\n  }\n  // initialize unknown values for dispatch modes\n  if (graph.attrs.count(\"dispatch_mode\") == 0) {\n    DispatchModeVector dispatch_modes(graph.indexed_graph().num_nodes(), DispatchMode::kUndefined);\n    graph.attrs[\"dispatch_mode\"] = std::make_shared<any>(std::move(dispatch_modes));\n  }\n  // initialize the dev_mask vector from the context vector\n  if (graph.attrs.count(\"dev_mask\") == 0) {\n    CHECK_GT(graph.attrs.count(\"context\"), 0);\n    DevMaskVector dev_masks(graph.indexed_graph().num_nodes());\n    const ContextVector& vctx = graph.GetAttr<ContextVector>(\"context\");\n    for (size_t i = 0; i < vctx.size(); i++)\n      dev_masks[i] = vctx[i].dev_mask();\n    graph.attrs[\"dev_mask\"] = std::make_shared<any>(std::move(dev_masks));\n  }\n\n  // for storage type, the backward attr is not necessarily the same as it's correspondence\n  nnvm::Graph ret = InferAttr<int,\n                              FInferStorageType,\n                              exec::FAccessSubgraphStorageType,\n                              exec::FProvideSubgraphStorageType>(\n      std::move(graph),\n      -1,\n      \"FInferStorageType\",\n      \"FAccessSubgraphStorageType\",\n      \"FProvideSubgraphStorageType\",\n      \"storage_type_inputs\",\n      \"storage_type_attr_key\",\n      \"storage_type\",\n      \"storage_type_num_unknown_nodes\",\n      [](const int t) { return t == -1; },\n      common::DefaultStorageType,\n      false,\n      \"dispatch_mode\",\n      DispatchMode::kVariable);\n\n  // log the storage types and dispatch modes of the graph\n  static bool log_verbose = dmlc::GetEnv(\"MXNET_INFER_STORAGE_TYPE_VERBOSE_LOGGING\", false);\n  if (log_verbose) {\n    common::LogInferStorage(ret);\n  }\n  return ret;\n}\n\n}  // namespace exec\n}  // namespace mxnet\n"
  },
  {
    "path": "src/imperative/inplace_addto_detect_pass.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file inplace_addto_detect_pass.cc\n * \\brief Detect whether inplace addto operation is possible for certain op.\n */\n#include <mxnet/base.h>\n#include <mxnet/operator.h>\n#include <mxnet/op_attr_types.h>\n#include <nnvm/graph_attr_types.h>\n\n#include \"./exec_pass.h\"\n\nnamespace mxnet {\nnamespace exec {\n\nGraph DetectInplaceAddTo(Graph g) {\n  nnvm::StorageVector storage_id = g.MoveCopyAttr<nnvm::StorageVector>(\"storage_id\");\n  std::vector<int> storage_inplace_index =\n      g.MoveCopyAttr<std::vector<int> >(\"storage_inplace_index\");\n  static const Op* ewise_plus_op = Op::Get(\"_grad_add\");\n  auto& idx                      = g.indexed_graph();\n  // reference cont.\n  std::vector<int> ref_count(idx.num_node_entries(), 0);\n  std::vector<int> addto_entry;\n  if (g.attrs.count(\"addto_entry\")) {\n    addto_entry = g.GetAttr<std::vector<int> >(\"addto_entry\");\n  } else {\n    addto_entry = std::vector<int>(idx.num_node_entries(), 0);\n  }\n  std::vector<int> skip_plus_node(idx.num_nodes(), 0);\n\n  for (auto& e : idx.outputs()) {\n    ++ref_count[idx.entry_id(e)];\n  }\n  for (uint32_t nid = 0; nid < idx.num_nodes(); ++nid) {\n    for (auto& e : idx[nid].inputs) {\n      ++ref_count[idx.entry_id(e)];\n    }\n  }\n\n  for (uint32_t nid = 0; nid < idx.num_nodes(); ++nid) {\n    const auto& inode = idx[nid];\n    if (inode.source->op() != ewise_plus_op)\n      continue;\n    int sid = storage_id[idx.entry_id(inode.inputs[0])];\n    if (sid != storage_id[idx.entry_id(nid, 0)])\n      continue;\n    if (idx[inode.inputs[0].node_id].source->is_variable())\n      continue;\n    if (idx[inode.inputs[1].node_id].source->is_variable())\n      continue;\n    uint32_t eid_rhs = idx.entry_id(inode.inputs[1]);\n    if (ref_count[eid_rhs] != 1)\n      continue;\n    if (inode.inputs[0].node_id >= inode.inputs[1].node_id)\n      continue;\n    // TODO(haibin) support inplace addto for Dynamic Storage\n    if (storage_id[eid_rhs] == kDynamicStorageID)\n      continue;\n    CHECK_NE(storage_id[eid_rhs], sid);\n    storage_id[eid_rhs]            = sid;\n    addto_entry[eid_rhs]           = 1;\n    storage_inplace_index[eid_rhs] = -1;\n    skip_plus_node[nid]            = 1;\n  }\n\n  g.attrs[\"storage_id\"]            = std::make_shared<nnvm::any>(std::move(storage_id));\n  g.attrs[\"storage_inplace_index\"] = std::make_shared<nnvm::any>(std::move(storage_inplace_index));\n  g.attrs[\"addto_entry\"]           = std::make_shared<nnvm::any>(std::move(addto_entry));\n  g.attrs[\"skip_plus_node\"]        = std::make_shared<nnvm::any>(std::move(skip_plus_node));\n  return g;\n}\n\n}  // namespace exec\n}  // namespace mxnet\n"
  },
  {
    "path": "src/imperative/naive_cached_op.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n#include <unordered_set>\n#include <iostream>\n#include \"./imperative_utils.h\"\n#include \"./naive_cached_op.h\"\n#include \"./exec_pass.h\"\n#include \"../profiler/profiler.h\"\n#include \"../operator/operator_common.h\"\n#include \"../operator/subgraph/common.h\"\n\nnamespace mxnet {\nOpStatePtr NaiveCachedOp::Forward(const std::shared_ptr<CachedOp>& op_ptr,\n                                  const std::vector<NDArray*>& inputs,\n                                  const std::vector<NDArray*>& outputs,\n                                  const Context& default_ctx) {\n  CHECK_EQ(inputs.size(), num_inputs());\n\n  {\n    auto state_ptr = GetCachedOpState(default_ctx);\n    auto& state    = state_ptr.get_state<CachedOpState>();\n\n    const auto& idx = state.info.fwd_graph.indexed_graph();\n    for (size_t i = 0; i < inputs.size(); ++i) {\n      CHECK_EQ(inputs[i]->ctx(), default_ctx)\n          << \"CachedOp requires all inputs to live on the same context. But \"\n          << idx[idx.input_nodes()[0]].source->attrs.name << \" is on \" << default_ctx << \" while \"\n          << idx[idx.input_nodes()[i]].source->attrs.name << \" is on \" << inputs[i]->ctx();\n    }\n  }\n\n  OpStatePtr op_state;\n  try {\n    // Initialize\n    bool recording = false;\n    op_state       = OpStatePtr::Create<DynamicRuntime>();\n    auto& runtime  = op_state.get_state<DynamicRuntime>();\n    {\n      auto state_ptr = GetCachedOpState(default_ctx);\n      auto& state    = state_ptr.get_state<CachedOpState>();\n      std::lock_guard<std::mutex> lock(state.mutex);\n      SetForwardGraph(default_ctx, &state.info, recording, inputs);\n      runtime.info.fwd_graph = state.info.fwd_graph;\n      runtime.info.input_map = state.info.input_map;\n    }\n    nnvm::Graph& g  = runtime.info.fwd_graph;\n    const auto& idx = g.indexed_graph();\n    auto& buff      = runtime.buff;\n    auto& states    = runtime.op_states;\n\n    // Allocate entries\n    buff.resize(idx.num_node_entries());\n    states.resize(idx.num_nodes());\n    std::vector<NDArray*> arrays;\n    arrays.reserve(buff.size());\n    for (auto& buffered_array : buff) {\n      arrays.push_back(&buffered_array);\n    }\n    std::vector<OpReqType> array_reqs(arrays.size(), kWriteTo);\n    const auto& dispatch_modes    = g.GetAttr<DispatchModeVector>(\"dispatch_mode\");\n    const std::string& graph_type = recording ? FULL : FORWARD;\n    std::vector<uint32_t> ref_count =\n        g.GetAttr<std::vector<uint32_t> >(AddPrefix(graph_type, REF_COUNT));\n    for (size_t i = 0; i < idx.num_node_entries(); ++i) {\n      if (ref_count[i] == 0)\n        array_reqs[i] = kNullOp;\n    }\n    CollectInputOutputNDRefs(g, inputs, runtime.info.input_map, outputs, &arrays);\n\n    mxnet::ShapeVector shapes = g.GetAttr<mxnet::ShapeVector>(\"shape\");\n    imperative::NaiveRunGraph(false,\n                              default_ctx,\n                              idx,\n                              arrays,\n                              0,\n                              idx.num_nodes(),\n                              std::move(array_reqs),\n                              std::move(ref_count),\n                              &states,\n                              dispatch_modes,\n                              false,\n                              &shapes,\n                              nullptr,\n                              false,\n                              true);\n    {\n      auto state_ptr    = GetCachedOpState(default_ctx);\n      auto& state       = state_ptr.get_state<CachedOpState>();\n      auto copied_shape = shapes;\n      std::lock_guard<std::mutex> lock(state.mutex);\n      state.info.fwd_graph.attrs[\"shape\"] = std::make_shared<dmlc::any>(std::move(copied_shape));\n    }\n    g.attrs[\"shape\"] = std::make_shared<dmlc::any>(std::move(shapes));\n  } catch (const dmlc::Error& e) {\n    throw e;\n  }\n  return op_state;\n}\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/imperative/naive_cached_op.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n// Threadsafe and minimal functionality cached op version for Inference\n// lot of code reused from cached_op.h\n#ifndef MXNET_IMPERATIVE_NAIVE_CACHED_OP_H_\n#define MXNET_IMPERATIVE_NAIVE_CACHED_OP_H_\n\n#include <mxnet/imperative.h>\n#include <vector>\n#include <atomic>\n#include <utility>\n#include <string>\n#include <unordered_map>\n#include \"./cached_op.h\"\n\nnamespace mxnet {\n/*! \\brief NaiveCachedOp which does not involve engine which is useful when executed in parallel.\n    It does not support advanced features of CachedOp, including backward/recording, etc...\n */\nclass NaiveCachedOp : public CachedOp {\n public:\n  NaiveCachedOp(const nnvm::Symbol& sym,\n                const std::vector<std::pair<std::string, std::string>>& flags)\n      : CachedOp(sym, flags) {}\n  virtual ~NaiveCachedOp() {}\n  OpStatePtr Forward(const std::shared_ptr<CachedOp>& op_ptr,\n                     const std::vector<NDArray*>& inputs,\n                     const std::vector<NDArray*>& outputs,\n                     const Context& default_ctx) override;\n  void Backward(const bool retain_graph,\n                const OpStatePtr& state,\n                const std::vector<NDArray*>& inputs,\n                const std::vector<OpReqType>& reqs,\n                const std::vector<NDArray*>& outputs) override {\n    LOG(FATAL) << \"Backward is not supported in NaiveCachedOp.\";\n  }\n  // backward storage type inference\n  bool BackwardStorageType(const nnvm::NodeAttrs& attrs,\n                           const int dev_mask,\n                           DispatchMode* dispatch_mode,\n                           std::vector<int>* in_attrs,\n                           std::vector<int>* out_attrs) override {\n    LOG(FATAL) << \"Backward is not supported in NaiveCachedOp.\";\n    return false;\n  }\n};  // NaiveCachedOp\n\nusing NaiveCachedOpPtr = std::shared_ptr<NaiveCachedOp>;\n\n}  // namespace mxnet\n#endif  // MXNET_IMPERATIVE_NAIVE_CACHED_OP_H_\n"
  },
  {
    "path": "src/imperative/pointwise_fusion_pass.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file pointwise_fusion_pass.cc\n * \\brief Pass applying pointwise fusion.\n * \\author Clement Fuji Tsang\n */\n\n#include <mxnet/base.h>\n#include <mxnet/operator.h>\n#include <mxnet/op_attr_types.h>\n#include <nnvm/graph_attr_types.h>\n#include <nnvm/pass_functions.h>\n#include <algorithm>\n#include <queue>\n#include <chrono>\n#include \"./simple_partition_pass.h\"\n#include \"../operator/fusion/fused_op-inl.h\"\n#include \"../operator/fusion/fused_op.h\"\n#include \"../operator/operator_common.h\"\n\nnamespace mxnet {\nnamespace exec {\n\nvoid WarnFusionNotSupported() {\n  static bool issued_warning = false;\n  if (!issued_warning) {\n    issued_warning = true;\n#if defined(_WIN32)\n    LOG(WARNING) << \"Omitting dynamic fused op creation- not enabled on Windows.  \"\n                 << \"Unset env var MXNET_USE_FUSION=1 to quiet this message.\";\n#else\n    LOG(WARNING) << \"Omitting dynamic fused op creation- needs MXNet lib built with \"\n                 << \"USE_CUDA=1.  Unset env var MXNET_USE_FUSION=1 \"\n                 << \"to quiet this message.\";\n#endif  // defined(_WIN32)\n  }\n}\n\n#if MXNET_USE_CUDA\n\nnamespace {\n\nbool IsFusionCompatible(const nnvm::Node* n) {\n  using namespace mxnet::fusion;\n  if (n->op() == nullptr)\n    return false;\n  const std::string& op_name = n->op()->name;\n  if (ops_desc.count(op_name))\n    return true;\n  if (slice_ops.count(op_name))\n    return false;\n  if (std::find(variable_io_ops.begin(), variable_io_ops.end(), op_name) != variable_io_ops.end())\n    return true;\n  if (op_name == \"LeakyReLU\") {\n    std::string act_type = n->attrs.dict.at(\"act_type\");\n    if (LeakyReLU_ops.count(act_type))\n      return true;\n    else\n      return false;\n  }\n  if (op_name == \"_backward_LeakyReLU\") {\n    std::string act_type = n->attrs.dict.at(\"act_type\");\n    if (LeakyReLU_bwd_ops.count(act_type))\n      return true;\n    else\n      return false;\n  }\n  return false;\n}\n\nbool IsInputsOnlyCompatible(const nnvm::Node* n) {\n  using namespace mxnet::fusion;\n  if (n->op() == nullptr)\n    return false;\n  const std::string& op_name = n->op()->name;\n  if (slice_ops.count(op_name)) {\n    if (op_name == \"slice\") {\n      // slice with non-default step attribute is not supported\n      // currently\n      if (n->attrs.dict.count(\"step\") &&\n          !(n->attrs.dict.at(\"step\") == \"()\" || n->attrs.dict.at(\"step\") == \"[]\")) {\n        return false;\n      }\n    }\n    return true;\n  }\n  return false;\n}\n\nvoid CreateSubgraphNode(const nnvm::Graph& subgraph,\n                        size_t inputs_size,\n                        nnvm::Node* subgraph_node) {\n  static const Op* fused_op_ptr = Op::Get(\"_FusedOp\");\n  subgraph_node->attrs.subgraphs.emplace_back(std::make_shared<nnvm::Symbol>());\n  subgraph_node->attrs.subgraphs.back()->outputs = subgraph.outputs;\n  subgraph_node->attrs.dict[\"num_inputs\"]        = std::to_string(inputs_size);\n  subgraph_node->attrs.dict[\"num_outputs\"]       = std::to_string(subgraph.outputs.size());\n  subgraph_node->attrs.op                        = fused_op_ptr;\n  subgraph_node->op()->attr_parser(&(subgraph_node->attrs));\n}\n\nstruct EntryInfo {\n  int source_node;\n  int index;\n};\n\ninline int SetInsert(const EntryInfo& new_elem, std::vector<EntryInfo>* elements) {\n  for (size_t i = 0; i < elements->size(); ++i) {\n    if ((new_elem.source_node == elements->at(i).source_node) &&\n        (new_elem.index == elements->at(i).index)) {\n      return i;\n    }\n  }\n  elements->emplace_back(new_elem);\n  return elements->size() - 1;\n}\n\n}  // namespace\n\n/* \\brief Create (if necessary) copy of the graph, replacing subgraphs with\n *        FusedOps. If there are no subgraphs to be replaced, the\n *        original graph is returned.\n * \\param g original graph.\n * \\param subgraph_assignment assignment of nodes in g's IndexedGraphs to\n *                            subgraphs. Values from -1 to num_subgraphs - 1\n *                            are allowed, -1 means that the node is not in a\n *                            subgraph.\n * \\param num_subgraphs number of subgraphs.\n * \\param create_subgraph_node function used to prepare the subgraph node.\n */\ntemplate <typename FCreateNode>\nGraph CopyAndReplaceSubgraphs(const Graph& g,\n                              const std::vector<int>& subgraph_assignment,\n                              const int num_subgraphs,\n                              FCreateNode create_subgraph_node) {\n  if (num_subgraphs == 0) {\n    return g;\n  }\n\n  Graph ret;\n\n  const auto& idx = g.indexed_graph();\n\n  CHECK_EQ(idx.num_nodes(), subgraph_assignment.size())\n      << \"Every node in the graph needs to be included in subgraph assignment.\";\n\n  std::vector<nnvm::ObjectPtr> new_nodes;\n  new_nodes.reserve(idx.num_nodes());\n  struct SubgraphInfo {\n    nnvm::Graph graph;\n    nnvm::ObjectPtr subgraph_node;\n    std::vector<EntryInfo> outputs;\n    std::vector<EntryInfo> inputs;\n    std::vector<nnvm::ObjectPtr> input_nodes;\n  };\n\n  std::vector<SubgraphInfo> subgraphs(num_subgraphs);\n\n  for (auto& info : subgraphs) {\n    info.subgraph_node = nnvm::Node::Create();\n  }\n\n  for (size_t i = 0; i < idx.num_nodes(); ++i) {\n    // First copy the node, it will be used\n    // either in the new graph or inside a\n    // subgraph. Variables are not copied.\n    if (idx[i].source->op() != nullptr) {\n      new_nodes.emplace_back(nnvm::Node::Create());\n      auto& node_copy  = new_nodes.back();\n      node_copy->attrs = idx[i].source->attrs;\n      node_copy->info  = idx[i].source->info;\n    } else {\n      new_nodes.emplace_back(idx[i].weak_ref.lock());\n      continue;\n    }\n    auto& node_copy       = new_nodes.back();\n    const int subgraph_id = subgraph_assignment[i];\n    if (subgraph_id != -1) {\n      auto& info = subgraphs[subgraph_id];\n      for (const auto& input : idx[i].inputs) {\n        const int their_subgraph = subgraph_assignment[input.node_id];\n        if (their_subgraph == subgraph_id) {\n          node_copy->inputs.emplace_back(new_nodes[input.node_id], input.index, input.version);\n        } else {\n          int input_num;\n          int output_num;\n          if (their_subgraph == -1) {\n            input_num = SetInsert({static_cast<int>(input.node_id), static_cast<int>(input.index)},\n                                  &(info.inputs));\n          } else {\n            auto& their_subgraph_info = subgraphs[their_subgraph];\n            output_num = SetInsert({static_cast<int>(input.node_id), static_cast<int>(input.index)},\n                                   &(their_subgraph_info.outputs));\n            input_num  = SetInsert({static_cast<int>(idx.num_nodes() + their_subgraph), output_num},\n                                  &(info.inputs));\n          }\n          if (static_cast<size_t>(input_num) == info.input_nodes.size()) {\n            info.input_nodes.emplace_back(nnvm::Node::Create());\n            info.input_nodes.back()->attrs.name = \"input_\" + std::to_string(input_num);\n            if (their_subgraph == -1) {\n              info.subgraph_node->inputs.emplace_back(\n                  new_nodes[input.node_id], input.index, input.version);\n            } else {\n              info.subgraph_node->inputs.emplace_back(\n                  subgraphs[their_subgraph].subgraph_node, output_num, input.version);\n            }\n          }\n          node_copy->inputs.emplace_back(info.input_nodes[input_num], 0, 0);\n        }\n      }\n    } else {\n      for (const auto& input : idx[i].inputs) {\n        const int subgraph_id = subgraph_assignment[input.node_id];\n        if (subgraph_id == -1) {\n          node_copy->inputs.emplace_back(new_nodes[input.node_id], input.index, input.version);\n        } else {\n          auto& info           = subgraphs[subgraph_id];\n          const int output_num = SetInsert(\n              {static_cast<int>(input.node_id), static_cast<int>(input.index)}, &(info.outputs));\n          node_copy->inputs.emplace_back(info.subgraph_node, output_num, input.version);\n        }\n      }\n    }\n\n    // Control deps\n    for (const auto& dep : idx[i].control_deps) {\n      if (subgraph_id == subgraph_assignment[dep]) {\n        node_copy->control_deps.emplace_back(new_nodes[dep]);\n      }\n    }\n  }\n\n  ret.outputs.reserve(idx.outputs().size());\n  for (const auto& output : idx.outputs()) {\n    const int subgraph_id = subgraph_assignment[output.node_id];\n    if (subgraph_id == -1) {\n      ret.outputs.emplace_back(new_nodes[output.node_id], output.index, output.version);\n    } else {\n      const int output_num =\n          SetInsert({static_cast<int>(output.node_id), static_cast<int>(output.index)},\n                    &(subgraphs[subgraph_id].outputs));\n      ret.outputs.emplace_back(subgraphs[subgraph_id].subgraph_node, output_num, output.version);\n    }\n  }\n\n  for (auto& info : subgraphs) {\n    info.graph.outputs.reserve(info.outputs.size());\n    for (const auto& entry_info : info.outputs) {\n      info.graph.outputs.emplace_back(new_nodes[entry_info.source_node], entry_info.index, 0);\n    }\n    create_subgraph_node(info.graph, info.inputs.size(), info.subgraph_node.get());\n  }\n\n  for (size_t i = 0; i < idx.num_nodes(); ++i) {\n    // Add _FusedOpHelper nodes\n    const int subgraph_id = subgraph_assignment[i];\n    for (size_t dep_num = 0; dep_num < idx[i].control_deps.size(); ++dep_num) {\n      const auto& dep             = idx[i].control_deps[dep_num];\n      const int their_subgraph_id = subgraph_assignment[dep];\n      if (subgraph_id != -1 && their_subgraph_id == -1) {\n        // Not in any subgraph, use FusedOpOutHelper\n        auto& info     = subgraphs[subgraph_id];\n        size_t node_id = info.subgraph_node->control_deps.size();\n        info.subgraph_node->control_deps.emplace_back(new_nodes[dep]);\n        auto helper_node          = op::MakeNode(\"_FusedOpOutHelper\",\n                                        \"FusedOp_\" + new_nodes[i]->attrs.name + \"_outhelper\",\n                                        nullptr,\n                                        nullptr,\n                                        nullptr);\n        helper_node->attrs.parsed = FusedOpHelperParamPtr(new FusedOpHelperParam(\n            nnvm::get<FusedOpPtr>(info.subgraph_node->attrs.parsed), node_id));\n        new_nodes[i]->control_deps.insert(new_nodes[i]->control_deps.begin() + dep_num,\n                                          std::move(helper_node));\n      } else if (their_subgraph_id != subgraph_id && their_subgraph_id != -1) {\n        auto& info               = subgraphs[their_subgraph_id];\n        const auto& subgraph_idx = info.graph.indexed_graph();\n        uint32_t node_id         = subgraph_idx.node_id(new_nodes[dep].get());\n        auto helper_node         = op::MakeNode(\n            \"_FusedOpHelper\",\n            info.subgraph_node->attrs.name + \"_\" + idx[i].source->attrs.name + \"_helper\",\n            nullptr,\n            nullptr,\n            nullptr);\n        helper_node->attrs.parsed = FusedOpHelperParamPtr(new FusedOpHelperParam(\n            nnvm::get<FusedOpPtr>(info.subgraph_node->attrs.parsed), node_id));\n        new_nodes[i]->control_deps.insert(new_nodes[i]->control_deps.begin() + dep_num,\n                                          std::move(helper_node));\n      }\n    }\n  }\n  for (auto& info : subgraphs) {\n    const auto& idx         = info.graph.indexed_graph();\n    const auto& input_nodes = idx.input_nodes();\n    std::vector<nnvm::NodeEntry> subgraph_inputs;\n    subgraph_inputs.reserve(info.subgraph_node->inputs.size());\n    for (const int input : input_nodes) {\n      for (size_t i = 0; i < info.input_nodes.size(); ++i) {\n        const auto& input_ptr = info.input_nodes[i].get();\n        if (input_ptr == idx[input].source) {\n          subgraph_inputs.emplace_back(info.subgraph_node->inputs[i]);\n        }\n      }\n    }\n    info.subgraph_node->inputs.swap(subgraph_inputs);\n    std::string name;\n    for (size_t i = 0; i < idx.num_nodes(); ++i) {\n      if (idx[i].source->op() != nullptr) {\n        name += idx[i].source->op()->name + \"_\";\n      }\n    }\n    info.subgraph_node->attrs.name = name;\n  }\n  return ret;\n}\n\nGraph FusePointwise(const Graph& g, const size_t num_forward_outputs) {\n  auto start                            = std::chrono::steady_clock::now();\n  auto [subset_assignment, num_subsets] = GetCompatibleSubsets(g,                    // NOLINT(*)\n                                                               num_forward_outputs,  // NOLINT(*)\n                                                               IsFusionCompatible,\n                                                               IsInputsOnlyCompatible);\n  Graph ret = CopyAndReplaceSubgraphs(g, subset_assignment, num_subsets, CreateSubgraphNode);\n  auto end  = std::chrono::steady_clock::now();\n  if (dmlc::GetEnv(\"MXNET_RTC_VERBOSE\", false)) {\n    auto diff = end - start;\n    LOG(INFO) << \"Pointwise fusion graph pass took: \"\n              << std::chrono::duration<double, std::milli>(diff).count() << \"ms.\";\n  }\n  return ret;\n}\n#endif  // MXNET_USE_CUDA\n\n}  // namespace exec\n}  // namespace mxnet\n"
  },
  {
    "path": "src/imperative/simple_partition_pass.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file simple_partition_pass.cc\n * \\brief Utilities used in simple partition pass\n * \\author Przemyslaw Tredak\n */\n\n#include \"./simple_partition_pass.h\"\n#include <memory>\n#include <utility>\n\nnamespace mxnet {\nnamespace exec {\n\nnamespace detail {\n\nconst IntervalVec* LargerSet(const IntervalVec* const first,\n                             const IntervalVec* const second) noexcept {\n  const IntervalVec* ret = nullptr;\n  auto first_iter        = first->begin();\n  auto second_iter       = second->begin();\n  while (first_iter != first->end() && second_iter != second->end()) {\n    if (*first_iter == *second_iter) {\n      ++first_iter;\n      ++second_iter;\n    } else {\n      // Entry in first set not seen in the second set\n      if (first_iter->second < second_iter->first) {\n        if (ret == first || ret == nullptr) {\n          ret = first;\n          ++first_iter;\n        } else {\n          return nullptr;\n        }\n        continue;\n      }\n      // Entry in second set not seen in the first set\n      if (second_iter->second < first_iter->first) {\n        if (ret == second || ret == nullptr) {\n          ret = second;\n          ++second_iter;\n        } else {\n          return nullptr;\n        }\n        continue;\n      }\n      // Entry in first set fully encloses the entry in the second set\n      if (first_iter->first <= second_iter->first && first_iter->second >= second_iter->second) {\n        if (ret == first || ret == nullptr) {\n          ret = first;\n          ++second_iter;\n        } else {\n          return nullptr;\n        }\n        continue;\n      }\n      // Entry in second set fully encloses the entry in the first set\n      if (second_iter->first <= first_iter->first && second_iter->second >= first_iter->second) {\n        if (ret == second || ret == nullptr) {\n          ret = second;\n          ++first_iter;\n        } else {\n          return nullptr;\n        }\n        continue;\n      }\n      // Entries intersect but one is not fully enclosed in the other\n      return nullptr;\n    }\n  }\n  if (ret == nullptr) {\n    // The common part is the same\n    return second_iter == second->end() ? first : second;\n  } else {\n    if ((ret == first && second_iter == second->end()) ||\n        (ret == second && first_iter == first->end())) {\n      return ret;\n    }\n  }\n  return nullptr;\n}\n\nvoid MergeSets(const IntervalVec** const my_set,\n               const IntervalVec* const other_set,\n               std::vector<std::unique_ptr<const IntervalVec>>* const storage) noexcept {\n  if ((*my_set == nullptr) || (*my_set)->size() == 0) {\n    *my_set = other_set;\n    return;\n  }\n  if (other_set == nullptr || other_set->size() == 0) {\n    return;\n  }\n  auto* larger_set = LargerSet(*my_set, other_set);\n  if (larger_set != nullptr) {\n    *my_set = larger_set;\n    return;\n  }\n  auto my_iter    = (*my_set)->cbegin();\n  auto other_iter = other_set->cbegin();\n  auto new_set    = IntervalVec();\n  int last_end    = -10;  // less than -1\n  while (my_iter != (*my_set)->cend() && other_iter != other_set->cend()) {\n    const auto& mine  = *my_iter;\n    const auto& other = *other_iter;\n    if (other.second < mine.first - 1) {\n      // other interval is before ours\n      if (last_end >= other.first - 1) {\n        new_set.back().second = other.second;\n      } else {\n        new_set.emplace_back(other);\n      }\n      last_end = other.second;\n      ++other_iter;\n    } else if (other.first > mine.second + 1) {\n      // other interval is after ours\n      if (last_end >= mine.first - 1) {\n        new_set.back().second = mine.second;\n      } else {\n        new_set.emplace_back(mine);\n      }\n      last_end = mine.second;\n      ++my_iter;\n    } else {\n      // Intervals can be merged together\n      Interval n(std::min(mine.first, other.first), std::max(mine.second, other.second));\n      if (last_end >= n.first - 1) {\n        new_set.back().second = n.second;\n      } else {\n        new_set.emplace_back(n);\n      }\n      last_end = n.second;\n      if (other.second >= mine.second) {\n        ++my_iter;\n      }\n      if (mine.second >= other.second) {\n        ++other_iter;\n      }\n    }\n  }\n  auto remaining_iter = my_iter == (*my_set)->cend() ? other_iter : my_iter;\n  auto remaining_end  = my_iter == (*my_set)->cend() ? other_set->cend() : (*my_set)->cend();\n  // Add the rest of entries\n  for (; remaining_iter != remaining_end; ++remaining_iter) {\n    auto& mine        = new_set.back();\n    const auto& other = *remaining_iter;\n    if (other.second < mine.first - 1) {\n      // other interval is before ours, should never happen\n      continue;\n    } else if (other.first > mine.second + 1) {\n      // other interval is after ours\n      new_set.emplace_back(other);\n    } else {\n      // Intervals can be merged together\n      mine.first  = std::min(mine.first, other.first);\n      mine.second = std::max(mine.second, other.second);\n    }\n  }\n  storage->emplace_back(std::make_unique<IntervalVec>(std::move(new_set)));\n  *my_set = storage->back().get();\n}\n\nbool Intersect(const IntervalVec& checked_sets, const IntervalVec& excluded_sets) noexcept {\n  size_t current_interval = 0, current_other_interval = 0;\n  while (current_interval < checked_sets.size() && current_other_interval < excluded_sets.size()) {\n    const auto& mine  = checked_sets[current_interval];\n    const auto& other = excluded_sets[current_other_interval];\n    if (other.second < mine.first) {\n      // other interval is before ours\n      ++current_other_interval;\n    } else if (other.first > mine.second) {\n      // other interval is after ours\n      ++current_interval;\n    } else {\n      // Intervals intersect\n      return true;\n    }\n  }\n  return false;\n}\n\nvoid AddSet(const IntervalVec** const sets,\n            const int set_to_add,\n            std::vector<std::unique_ptr<const IntervalVec>>* const storage) noexcept {\n  if (*sets != nullptr && (*sets)->size() != 0) {\n    for (auto& interval : (**sets)) {\n      if (set_to_add >= interval.first && set_to_add <= interval.second) {\n        return;\n      }\n    }\n  }\n  storage->emplace_back(std::make_unique<IntervalVec>(1, std::make_pair(set_to_add, set_to_add)));\n  MergeSets(sets, storage->back().get(), storage);\n}\n\nint GetSetMapping(const int set, std::vector<int>* const set_mapping) noexcept {\n  if (set == -1)\n    return -1;\n  int temp = set;\n  while ((*set_mapping)[temp] != temp) {\n    temp = (*set_mapping)[temp];\n  }\n  (*set_mapping)[set] = temp;\n  return temp;\n}\n\nvoid CheckAndUpdateCombinedExcludedSets(\n    const IntervalVec** const combined_excluded_sets_ptr,\n    const IntervalVec* const new_excluded_sets,\n    std::vector<const IntervalVec*>* const excluded_sets_ptr,\n    const int set_id,\n    const int first_node_in_set,\n    const size_t new_node_id,\n    const std::vector<int>& set_assignment,\n    std::vector<int>* const set_mapping_ptr,\n    const IntervalVec& inverse_set_mapping,\n    std::vector<std::unique_ptr<const IntervalVec>>* const storage) noexcept {\n  const auto* previous_excluded_sets = *combined_excluded_sets_ptr;\n  MergeSets(combined_excluded_sets_ptr, new_excluded_sets, storage);\n  if (new_excluded_sets != nullptr) {\n    if (previous_excluded_sets == nullptr ||\n        *previous_excluded_sets != **(combined_excluded_sets_ptr)) {\n      // Their set's excluded sets list got larger, need to update the descendants\n      // of their set\n      auto& excluded_sets = *excluded_sets_ptr;\n      for (size_t j = first_node_in_set; j < new_node_id; ++j) {\n        if (GetSetMapping(set_assignment[j], set_mapping_ptr) == set_id ||\n            (excluded_sets[j] != nullptr && Intersect(inverse_set_mapping, *excluded_sets[j]))) {\n          MergeSets(&excluded_sets[j], *combined_excluded_sets_ptr, storage);\n        }\n      }\n    }\n  }\n}\n\n}  // namespace detail\n\n}  // namespace exec\n}  // namespace mxnet\n"
  },
  {
    "path": "src/imperative/simple_partition_pass.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file simple_partition_pass.h\n * \\brief Simple pass for partitioning a graph.\n * \\author Clement Fuji Tsang, Przemyslaw Tredak\n */\n#ifndef MXNET_IMPERATIVE_SIMPLE_PARTITION_PASS_H_\n#define MXNET_IMPERATIVE_SIMPLE_PARTITION_PASS_H_\n\n#include <mxnet/base.h>\n#include <mxnet/op_attr_types.h>\n#include <mxnet/operator.h>\n#include <nnvm/graph_attr_types.h>\n#include <utility>\n#include <deque>\n#include <algorithm>\n#include <vector>\n#include <tuple>\n\n#include \"exec_pass.h\"\n\nnamespace mxnet {\nnamespace exec {\n\nnamespace detail {\n\nusing Interval    = std::pair<int, int>;\nusing IntervalVec = std::vector<Interval>;\n\n/* \\brief Return the set that fully contains the other set, or nullptr\n *        if neither set is a subset of another.\n */\nconst IntervalVec* LargerSet(const IntervalVec* const first,\n                             const IntervalVec* const second) noexcept;\n\n/* \\brief Compute the sum of the 2 sets and store it in my_set.\n */\nvoid MergeSets(const IntervalVec** const my_set,\n               const IntervalVec* const other_set,\n               std::vector<std::unique_ptr<const IntervalVec>>* const storage) noexcept;\n\n/* \\brief Returns true if there is non-empty intersection\n *        between the 2 sets.\n */\nbool Intersect(const IntervalVec& checked_sets, const IntervalVec& excluded_sets) noexcept;\n\n/* \\brief Add a single entry to the sets.\n */\nvoid AddSet(const IntervalVec** const sets,\n            const int set_to_add,\n            std::vector<std::unique_ptr<const IntervalVec>>* const storage) noexcept;\n\n/* \\brief Get the true mapping of the set (which could change\n *        due to merging of multiple sets.\n */\nint GetSetMapping(const int set, std::vector<int>* const set_mapping) noexcept;\n\n/* \\brief Check if 2 ids are on the same side of the cutoff\n *        (so either both on the FWD side or the BWD side).\n */\ninline bool IsSamePass(const int my_id, const int their_id, const int cutoff) noexcept {\n  return (my_id > cutoff && their_id > cutoff) || (my_id <= cutoff && their_id <= cutoff);\n}\n\n/* \\brief Check if adding a new node to the set changes the excluded set of the future\n *        fused node. If so, update all descendants of the fused node.\n *\n * \\param combined_excluded_sets_ptr pointer to the set's list of excluded sets\n *                                   before adding the new node\n * \\param new_excluded_sets list of excluded sets of the new node\n * \\param excluded_sets_ptr pointer to the lists of excluded sets of all the nodes\n * \\param set_id number of the set, to which the new node is added\n * \\param first_node_in_set id of the first node in the set, according to topological ordering\n * \\param new_node_id id of the node added to the set\n * \\param set_assignment assignment of sets\n * \\param set_mapping_ptr pointer to the mappings of sets\n * \\param inverse_set_mapping inverse mapping of the set\n * \\param storage memory storage\n */\nvoid CheckAndUpdateCombinedExcludedSets(\n    const IntervalVec** const combined_excluded_sets_ptr,\n    const IntervalVec* const new_excluded_sets,\n    std::vector<const IntervalVec*>* const excluded_sets_ptr,\n    const int set_id,\n    const int first_node_in_set,\n    const size_t new_node_id,\n    const std::vector<int>& set_assignment,\n    std::vector<int>* const set_mapping_ptr,\n    const IntervalVec& inverse_set_mapping,\n    std::vector<std::unique_ptr<const IntervalVec>>* const storage) noexcept;\n\n}  // namespace detail\n\n/* \\brief Get all subsets of nodes, where:\n *  - graph constructed from nodes in each subset is a connected graph\n *  - every node fulfills a predicate is_compatible\n *  - if nodes u and v are part of a subset, then for each path between\n *    u and v in the original directed graph, all nodes on those paths\n *    are also part of the subset\n * \\param g NNVM graph\n * \\param num_forward_outputs Number of outputs from the graph that come\n *                            from the forward pass\n * \\param is_compatible A function taking nnvm::Node* and returning bool\n *                      which identifies which nodes could be included in\n *                      subsets.\n * \\param is_input_only_compatible A function taking nnvm::Node* and\n *                                 returning bool which identifies which\n *                                 nodes could be included in subsets only\n *                                 as the first operations (their inputs\n *                                 need to be excluded).\n * \\return tuple (subset assignment, number of found subsets)\n */\ntemplate <typename FCompatible, typename FInputOnlyCompatible>\nstd::tuple<std::vector<int>, int> GetCompatibleSubsets(\n    const Graph& g,\n    const size_t num_forward_outputs,\n    FCompatible is_compatible,\n    FInputOnlyCompatible is_input_only_compatible) {\n  using namespace detail;\n  const auto& idx = g.indexed_graph();\n  std::vector<int> set_assignment(idx.num_nodes(), -1);\n  std::vector<const std::vector<Interval>*> excluded_sets(idx.num_nodes());\n  std::vector<int> set_mapping;\n  std::vector<const std::vector<Interval>*> combined_excluded_sets;\n  std::vector<int> first_node_in_set;\n  std::vector<const std::vector<Interval>*> inverse_set_mapping;\n  std::vector<std::unique_ptr<const std::vector<Interval>>> storage;\n\n  int last_forward_node = -1;\n  for (size_t i = 0; i < num_forward_outputs; ++i) {\n    const int output_id = idx.outputs()[i].node_id;\n    if (last_forward_node < output_id) {\n      last_forward_node = output_id;\n    }\n  }\n\n  int num_sets = 0;\n  for (size_t i = 0; i < idx.num_nodes(); ++i) {\n    const auto& node       = idx[i];\n    auto& my_excluded_sets = excluded_sets[i];\n    for (const auto& input : node.inputs) {\n      MergeSets(&my_excluded_sets, excluded_sets[input.node_id], &storage);\n    }\n    if (is_compatible(node.source)) {\n      int my_set = -1;\n      for (const auto& input : node.inputs) {\n        int their_set = GetSetMapping(set_assignment[input.node_id], &set_mapping);\n        if (their_set != -1 && their_set != my_set &&\n            IsSamePass(i, input.node_id, last_forward_node) &&\n            (my_excluded_sets == nullptr ||\n             !Intersect(*inverse_set_mapping[their_set], *my_excluded_sets))) {\n          if (my_set == -1) {\n            my_set = their_set;\n            CheckAndUpdateCombinedExcludedSets(&(combined_excluded_sets[their_set]),\n                                               my_excluded_sets,\n                                               &excluded_sets,\n                                               their_set,\n                                               first_node_in_set[their_set],\n                                               i,\n                                               set_assignment,\n                                               &set_mapping,\n                                               *(inverse_set_mapping[their_set]),\n                                               &storage);\n          } else {\n            MergeSets(&inverse_set_mapping[my_set], inverse_set_mapping[their_set], &storage);\n            set_mapping[their_set] = my_set;\n            first_node_in_set[my_set] =\n                std::min(first_node_in_set[my_set], first_node_in_set[their_set]);\n            CheckAndUpdateCombinedExcludedSets(&(combined_excluded_sets[their_set]),\n                                               combined_excluded_sets[my_set],\n                                               &excluded_sets,\n                                               my_set,\n                                               first_node_in_set[my_set],\n                                               i,\n                                               set_assignment,\n                                               &set_mapping,\n                                               *(inverse_set_mapping[my_set]),\n                                               &storage);\n          }\n        }\n      }\n      if (my_set == -1) {\n        set_mapping.emplace_back(num_sets);\n        combined_excluded_sets.emplace_back(my_excluded_sets);\n        first_node_in_set.emplace_back(i);\n        storage.emplace_back(\n            std::make_unique<std::vector<Interval>>(1, std::make_pair(num_sets, num_sets)));\n        inverse_set_mapping.emplace_back(storage.back().get());\n        my_set = num_sets++;\n      }\n      set_assignment[i] = my_set;\n    } else {\n      for (const auto& input : node.inputs) {\n        int their_set = GetSetMapping(set_assignment[input.node_id], &set_mapping);\n        if (their_set != -1) {\n          AddSet(&my_excluded_sets, their_set, &storage);\n        }\n      }\n      if ((is_input_only_compatible != nullptr) && is_input_only_compatible(node.source)) {\n        set_mapping.emplace_back(num_sets);\n        combined_excluded_sets.emplace_back(my_excluded_sets);\n        first_node_in_set.emplace_back(i);\n        storage.emplace_back(\n            std::make_unique<std::vector<Interval>>(1, std::make_pair(num_sets, num_sets)));\n        inverse_set_mapping.emplace_back(storage.back().get());\n        set_assignment[i] = num_sets++;\n      }\n    }\n  }\n\n  for (int& set : set_assignment) {\n    set = GetSetMapping(set, &set_mapping);\n  }\n\n  std::vector<int> set_reorder(num_sets, 0);\n  // First count the number of elements in each set.\n  for (int& set : set_assignment) {\n    if (set != -1) {\n      ++set_reorder[set];\n    }\n  }\n  // Then reorder them, removing sets that have\n  // only a single element.\n  int final_num_sets = 0;\n  for (int& set : set_reorder) {\n    if (set > 1) {\n      set = final_num_sets++;\n    } else {\n      set = -1;\n    }\n  }\n\n  for (int& set : set_assignment) {\n    if (set != -1) {\n      set = set_reorder[set];\n    }\n  }\n\n  return {set_assignment, final_num_sets};\n}\n\n}  // namespace exec\n}  // namespace mxnet\n#endif  // MXNET_IMPERATIVE_SIMPLE_PARTITION_PASS_H_\n"
  },
  {
    "path": "src/initialize.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file initialize.cc\n * \\brief initialize mxnet library\n */\n#include \"initialize.h\"\n\n#include <algorithm>\n#include <csignal>\n\n#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)\n#include <windows.h>\n/*!\n * \\brief Retrieve the system error message for the last-error code\n * \\param err string that gets the error message\n */\nvoid win_err(char** err) {\n  uint32_t dw = GetLastError();\n  FormatMessage(\n      FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,\n      nullptr,\n      dw,\n      MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),\n      reinterpret_cast<char*>(err),\n      0,\n      nullptr);\n}\n#else\n#include <cxxabi.h>\n#include <dlfcn.h>\n#if MXNET_USE_SIGNAL_HANDLER && DMLC_LOG_STACK_TRACE\n#include <execinfo.h>\n#endif\n#include <cerrno>\n#endif\n\n#include <dmlc/logging.h>\n#include <mxnet/c_api.h>\n#include <mxnet/engine.h>\n\n#include \"./engine/openmp.h\"\n#include \"./operator/custom/custom-inl.h\"\n#if MXNET_USE_OPENCV\n#include <opencv2/opencv.hpp>\n#endif  // MXNET_USE_OPENCV\n#include \"common/utils.h\"\n#include \"engine/openmp.h\"\n\n#if defined(MKL_USE_SINGLE_DYNAMIC_LIBRARY)\n#include <mkl.h>\n#endif\n\nnamespace mxnet {\n\n// pthread_atfork handlers, delegated to LibraryInitializer members.\n\nvoid pthread_atfork_prepare() {\n  LibraryInitializer* library_initializer = LibraryInitializer::Get();\n  library_initializer->atfork_prepare();\n}\n\nvoid pthread_atfork_parent() {\n  LibraryInitializer* library_initializer = LibraryInitializer::Get();\n  library_initializer->atfork_parent();\n}\n\nvoid pthread_atfork_child() {\n  LibraryInitializer* library_initializer = LibraryInitializer::Get();\n  library_initializer->atfork_child();\n}\n\n// LibraryInitializer member functions\n\nLibraryInitializer::LibraryInitializer()\n    : original_pid_(common::current_process_id()),\n      mp_worker_nthreads_(dmlc::GetEnv(\"MXNET_MP_WORKER_NTHREADS\", 1)),\n      cpu_worker_nthreads_(dmlc::GetEnv(\"MXNET_CPU_WORKER_NTHREADS\", 1)),\n      mp_cv_num_threads_(dmlc::GetEnv(\"MXNET_MP_OPENCV_NUM_THREADS\", 0)) {\n  dmlc::InitLogging(\"mxnet\");\n  init_mkl_dynamic_library();\n  engine::OpenMP::Get();  // force OpenMP initialization\n  install_pthread_atfork_handlers();\n}\n\nLibraryInitializer::~LibraryInitializer() = default;\n\nbool LibraryInitializer::lib_is_loaded(const std::string& path) const {\n  return loaded_libs_.count(path) > 0;\n}\n\n/*!\n * \\brief Loads the dynamic shared library file\n * \\param path library file location\n * \\return handle a pointer for the loaded library, throws dmlc::error if library can't be loaded\n */\nvoid* LibraryInitializer::lib_load(const char* path) {\n  void* handle = nullptr;\n  // check if library was already loaded\n  if (!lib_is_loaded(path)) {\n    // if not, load it\n#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)\n    handle = LoadLibrary(path);\n    if (!handle) {\n      char* err_msg = nullptr;\n      win_err(&err_msg);\n      LOG(FATAL) << \"Error loading library: '\" << path << \"'\\n\" << err_msg;\n      LocalFree(err_msg);\n      return nullptr;\n    }\n#else\n    /* library loading flags:\n     *  RTLD_LAZY - Perform lazy binding. Only resolve symbols as the code that\n     *              references them is executed.\n     *  RTLD_LOCAL - Symbols defined in this library are not made available to\n     *              resolve references in subsequently loaded libraries.\n     */\n    handle = dlopen(path, RTLD_LAZY | RTLD_LOCAL);\n    if (!handle) {\n      LOG(FATAL) << \"Error loading library: '\" << path << \"'\\n\" << dlerror();\n      return nullptr;\n    }\n#endif  // _WIN32 or _WIN64 or __WINDOWS__\n    // then store the pointer to the library\n    loaded_libs_[path] = handle;\n  } else {\n    handle = loaded_libs_.at(path);\n  }\n  return handle;\n}\n\n/*!\n * \\brief Closes the loaded dynamic shared library file\n * \\param handle library file handle\n */\nvoid LibraryInitializer::lib_close(void* handle, const std::string& libpath) {\n#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)\n  FreeLibrary((HMODULE)handle);\n#else\n  if (dlclose(handle)) {\n    LOG(WARNING) << \"LibraryInitializer::lib_close: couldn't close library at address: \" << handle\n                 << \" loaded from: '\" << libpath << \"': \" << dlerror();\n  }\n#endif  // _WIN32 or _WIN64 or __WINDOWS__\n}\n\n/*!\n * \\brief Obtains address of given function in the loaded library\n * \\param handle pointer for the loaded library\n * \\param func function pointer that gets output address\n * \\param name function name to be fetched\n */\nvoid LibraryInitializer::get_sym(void* handle, void** func, const char* name) {\n#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)\n  *func = GetProcAddress((HMODULE)handle, name);\n  if (!(*func)) {\n    char* err_msg = nullptr;\n    win_err(&err_msg);\n    LOG(FATAL) << \"Error getting function '\" << name << \"' from library\\n\" << err_msg;\n    LocalFree(err_msg);\n  }\n#else\n  *func = dlsym(handle, name);\n  if (!(*func)) {\n    LOG(FATAL) << \"Error getting function '\" << name << \"' from library\\n\" << dlerror();\n  }\n#endif  // _WIN32 or _WIN64 or __WINDOWS__\n}\n\nbool LibraryInitializer::was_forked() const {\n  return common::current_process_id() != original_pid_;\n}\n\nvoid LibraryInitializer::atfork_prepare() {\n  using op::custom::CustomOperator;\n  CustomOperator::Get()->Stop();\n  Engine::Get()->Stop();\n}\n\nvoid LibraryInitializer::atfork_parent() {\n  using op::custom::CustomOperator;\n  Engine::Get()->Start();\n  CustomOperator::Get()->Start();\n}\n\nvoid LibraryInitializer::atfork_child() {\n  using op::custom::CustomOperator;\n  // Conservative thread management for multiprocess workers\n  this->cpu_worker_nthreads_ = this->mp_worker_nthreads_;\n#if MXNET_USE_OPENCV && !__APPLE__\n  cv::setNumThreads(mp_cv_num_threads_);\n#endif  // MXNET_USE_OPENCV\n  engine::OpenMP::Get()->initialize_process();\n  engine::OpenMP::Get()->set_thread_max(1);\n  engine::OpenMP::Get()->set_enabled(false);\n  Engine::Get()->Start();\n  CustomOperator::Get()->Start();\n}\n\nvoid LibraryInitializer::install_pthread_atfork_handlers() {\n#ifndef _WIN32\n  engine::OpenMP::Get()->initialize_process();  // force omp to set its atfork handler first\n  pthread_atfork(pthread_atfork_prepare, pthread_atfork_parent, pthread_atfork_child);\n#endif\n}\n\nvoid LibraryInitializer::init_mkl_dynamic_library() {\n#if !(defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__))\n#if MKL_USE_SINGLE_DYNAMIC_LIBRARY\n#if USE_INT64_TENSOR_SIZE\n  int interface = MKL_INTERFACE_ILP64;\n#else\n  int interface = MKL_INTERFACE_LP64;\n#endif\n#if defined(__INTEL_LLVM_COMPILER) || defined(__APPLE__)\n  mkl_set_threading_layer(MKL_THREADING_INTEL);\n#else\n  mkl_set_threading_layer(MKL_THREADING_GNU);\n  interface += MKL_INTERFACE_GNU;\n#endif\n  mkl_set_interface_layer(interface);\n#endif\n#endif\n}\n\n#if MXNET_USE_SIGNAL_HANDLER && DMLC_LOG_STACK_TRACE\n\nstatic inline void printStackTrace(FILE* out = stderr, const unsigned int max_frames = 63) {\n#if !defined(_WIN32) && !defined(_WIN64) && !defined(__WINDOWS__)\n  // storage array for stack trace address data\n  void* addrlist[max_frames + 1];\n\n  // retrieve current stack addresses\n  size_t addrlen = backtrace(addrlist, sizeof(addrlist) / sizeof(void*));\n\n  if (addrlen < 5) {\n    return;\n  } else {\n    addrlen = std::min(addrlen, dmlc::LogStackTraceLevel());\n  }\n  fprintf(out, \"Stack trace:\\n\");\n\n  // resolve addresses into strings containing \"filename(function+address)\",\n  // Actually it will be ## program address function + offset\n  // this array must be free()-ed\n  char** symbollist = backtrace_symbols(addrlist, addrlen);\n\n  size_t funcnamesize = 1024;\n  char funcname[1024];\n\n  // iterate over the returned symbol lines. skip the first, it is the\n  // address of this function.\n  for (unsigned int i = 4; i < addrlen; i++) {\n    char* begin_name   = nullptr;\n    char* begin_offset = nullptr;\n    char* end_offset   = nullptr;\n\n    // find parentheses and +address offset surrounding the mangled name\n#ifdef DARWIN\n    // OSX style stack trace\n    for (char* p = symbollist[i]; *p; ++p) {\n      if (*p == '_' && *(p - 1) == ' ') {\n        begin_name = p - 1;\n      } else if (*p == '+') {\n        begin_offset = p - 1;\n      }\n    }\n\n    if (begin_name && begin_offset && begin_name < begin_offset) {\n      *begin_name++   = '\\0';\n      *begin_offset++ = '\\0';\n\n      // mangled name is now in [begin_name, begin_offset) and caller\n      // offset in [begin_offset, end_offset). now apply\n      // __cxa_demangle():\n      int status;\n      char* ret = abi::__cxa_demangle(begin_name, &funcname[0], &funcnamesize, &status);\n      if (status == 0) {\n        funcname = ret;  // use possibly realloc()-ed string\n        fprintf(out, \"  %-30s %-40s %s\\n\", symbollist[i], funcname, begin_offset);\n      } else {\n        // demangling failed. Output function name as a C function with\n        // no arguments.\n        fprintf(out, \"  %-30s %-38s() %s\\n\", symbollist[i], begin_name, begin_offset);\n      }\n    } else {\n      // couldn't parse the line? print the whole line.\n      fprintf(out, \"  %-40s\\n\", symbollist[i]);\n    }\n#else\n    for (char* p = symbollist[i]; *p; ++p) {\n      if (*p == '(') {\n        begin_name = p;\n      } else if (*p == '+') {\n        begin_offset = p;\n      } else if (*p == ')' && (begin_offset || begin_name)) {\n        end_offset = p;\n      }\n    }\n\n    if (begin_name && end_offset && begin_name < end_offset) {\n      *begin_name++ = '\\0';\n      *end_offset++ = '\\0';\n      if (begin_offset) {\n        *begin_offset++ = '\\0';\n      }\n\n      // mangled name is now in [begin_name, begin_offset) and caller\n      // offset in [begin_offset, end_offset). now apply\n      // __cxa_demangle():\n\n      int status  = 0;\n      char* ret   = abi::__cxa_demangle(begin_name, funcname, &funcnamesize, &status);\n      char* fname = begin_name;\n      if (status == 0) {\n        fname = ret;\n      }\n\n      if (begin_offset) {\n        fprintf(\n            out, \"  %-30s ( %-40s  + %-6s) %s\\n\", symbollist[i], fname, begin_offset, end_offset);\n      } else {\n        fprintf(out, \"  %-30s ( %-40s    %-6s) %s\\n\", symbollist[i], fname, \"\", end_offset);\n      }\n    } else {\n      // couldn't parse the line? print the whole line.\n      fprintf(out, \"  %-40s\\n\", symbollist[i]);\n    }\n#endif  // !DARWIN - but is posix\n  }\n  free(symbollist);\n#endif\n}\n\n#define SIGNAL_HANDLER(SIGNAL, HANDLER_NAME, IS_FATAL)                          \\\n  std::shared_ptr<void(int)> HANDLER_NAME(                                      \\\n      signal(SIGNAL,                                                            \\\n             [](int signum) {                                                   \\\n               if (IS_FATAL) {                                                  \\\n                 printf(\"\\nFatal Error: %s\\n\", strsignal(SIGNAL));              \\\n                 printStackTrace();                                             \\\n                 signal(signum, SIG_DFL);                                       \\\n                 raise(signum);                                                 \\\n               } else {                                                         \\\n                 switch (signum) {                                              \\\n                   case SIGSEGV:                                                \\\n                     LOG(FATAL) << \"InternalError: \" << strsignal(SIGNAL);      \\\n                     break;                                                     \\\n                   case SIGFPE:                                                 \\\n                     LOG(FATAL) << \"FloatingPointError: \" << strsignal(SIGNAL); \\\n                     break;                                                     \\\n                   case SIGBUS:                                                 \\\n                     LOG(FATAL) << \"IOError: \" << strsignal(SIGNAL);            \\\n                     break;                                                     \\\n                   default:                                                     \\\n                     LOG(FATAL) << \"RuntimeError: \" << strsignal(SIGNAL);       \\\n                     break;                                                     \\\n                 }                                                              \\\n               }                                                                \\\n             }),                                                                \\\n      [](auto f) { signal(SIGNAL, f); });\n\nSIGNAL_HANDLER(SIGSEGV, SIGSEGVHandler, true);\nSIGNAL_HANDLER(SIGFPE, SIGFPEHandler, false);\nSIGNAL_HANDLER(SIGBUS, SIGBUSHandler, false);\n\n#endif\n\nvoid LibraryInitializer::close_open_libs() {\n  for (const auto& l : loaded_libs_) {\n    lib_close(l.second, l.first);\n  }\n  loaded_libs_.clear();\n}\n\n/**\n * Perform static initialization\n */\n#ifdef __GNUC__\n// In GCC we use constructor to perform initialization before any static initializer is able to run\n__attribute__((constructor)) static void LibraryInitializerEntry() {\n#pragma GCC diagnostic ignored \"-Wunused-variable\"\n  volatile LibraryInitializer* library_init = LibraryInitializer::Get();\n}\n#else\nstatic LibraryInitializer* __library_init = LibraryInitializer::Get();\n#endif\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/initialize.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file initialize.h\n * \\brief Library initialization\n */\n\n#include <cstdlib>\n#include <map>\n#include <string>\n\n#include \"dmlc/io.h\"\n\n#ifndef MXNET_INITIALIZE_H_\n#define MXNET_INITIALIZE_H_\n\nnamespace mxnet {\n\nvoid pthread_atfork_prepare();\nvoid pthread_atfork_parent();\nvoid pthread_atfork_child();\n\n/**\n * Perform library initialization and control multiprocessing behaviour.\n */\nclass LibraryInitializer {\n public:\n  typedef std::map<std::string, void*> loaded_libs_t;\n  static LibraryInitializer* Get() {\n    static LibraryInitializer inst;\n    return &inst;\n  }\n\n  /**\n   * Library initialization. Called on library loading via constructor attributes or\n   * C++ static initialization.\n   */\n  LibraryInitializer();\n\n  ~LibraryInitializer();\n\n  /**\n   * @return true if the current pid doesn't match the one that initialized the library\n   */\n  bool was_forked() const;\n\n  // Library loading\n  bool lib_is_loaded(const std::string& path) const;\n  void* lib_load(const char* path);\n  void lib_close(void* handle, const std::string& libpath);\n  static void get_sym(void* handle, void** func, const char* name);\n\n  /**\n   * Original pid of the process which first loaded and initialized the library\n   */\n  size_t original_pid_;\n  size_t mp_worker_nthreads_;\n  size_t cpu_worker_nthreads_;\n  size_t omp_num_threads_;\n  size_t mp_cv_num_threads_;\n\n  // Actual code for the atfork handlers as member functions.\n  void atfork_prepare();\n  void atfork_parent();\n  void atfork_child();\n\n private:\n  /**\n   * Pthread atfork handlers are used to reset the concurrency state of modules like CustomOperator\n   * and Engine when forking. When forking only the thread that forks is kept alive and memory is\n   * copied to the new process so state is inconsistent. This call install the handlers.\n   * Has no effect on Windows.\n   *\n   * https://pubs.opengroup.org/onlinepubs/009695399/functions/pthread_atfork.html\n   */\n  void install_pthread_atfork_handlers();\n\n  /**\n   * Sets the interface and threading layer for Intel® oneAPI MKL at run time.\n   * Use with the Single Dynamic Library.\n   */\n  void init_mkl_dynamic_library();\n  /**\n   * Install signal handlers (UNIX). Has no effect on Windows.\n   */\n  void install_signal_handlers();\n\n  void close_open_libs();\n\n  loaded_libs_t loaded_libs_;\n};\n\n/*!\n * \\brief fetches from the library a function pointer of any given datatype and name\n * \\param T a template parameter for data type of function pointer\n * \\param lib library handle\n * \\param func_name function name to search for in the library\n * \\return func a function pointer\n */\ntemplate <typename T>\nT get_func(void* lib, const char* func_name) {\n  T func;\n  LibraryInitializer::Get()->get_sym(lib, reinterpret_cast<void**>(&func), func_name);\n  if (!func)\n    LOG(FATAL) << \"Unable to get function '\" << func_name << \"' from library\";\n  return func;\n}\n\n}  // namespace mxnet\n#endif  // MXNET_INITIALIZE_H_\n"
  },
  {
    "path": "src/io/batchify.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file batchify.cc\n * \\brief Mini-batch data combination functions.\n */\n#include <dmlc/parameter.h>\n#include <dmlc/omp.h>\n#include <mxnet/io.h>\n#include <mshadow/tensor.h>\n#include <mshadow/extension.h>\n#include <mshadow/extension/slice.h>\n\n#include <stack>\n#include <cmath>\n\n#include \"./inst_vector.h\"\n#include \"../ndarray/ndarray_function.h\"\n\nnamespace mxnet {\nnamespace io {\n\n#define tostr(s) #s\n\n#ifdef _MSC_VER\n#if _MSC_VER < 1925\n#define omp_parallel(t) __pragma(omp parallel for num_threads(t))\n#else\n#define omp_parallel(t) _Pragma(tostr(omp parallel for num_threads( ## t ## )))\n#endif\n#else\n#define omp_parallel(t) _Pragma(tostr(omp parallel for num_threads(t)))\n#endif\n\nstruct GroupBatchifyParam : public dmlc::Parameter<GroupBatchifyParam> {\n  mxnet::Tuple<std::intptr_t> functions;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(GroupBatchifyParam) {\n    DMLC_DECLARE_FIELD(functions).describe(\n        \"Internal sequentially applied batchify functions. \"\n        \"The number of functions must match output of dataset items.\");\n  }\n};  // struct GroupBatchifyParam\nDMLC_REGISTER_PARAMETER(GroupBatchifyParam);\n\nclass GroupBatchify : public BatchifyFunction {\n public:\n  explicit GroupBatchify(const std::vector<std::pair<std::string, std::string>>& kwargs) {\n    param_.InitAllowUnknown(kwargs);\n    fs_.reserve(param_.functions.ndim());\n    for (int i = 0; i < param_.functions.ndim(); ++i) {\n      fs_.emplace_back(\n          *static_cast<BatchifyFunctionPtr*>(reinterpret_cast<void*>(param_.functions[i])));\n    }\n  }\n\n  bool Batchify(const std::vector<std::vector<NDArray>>& inputs,\n                std::vector<NDArray>* outputs) override {\n    auto bs = inputs.size();\n    CHECK_GT(bs, 0) << \"BatchifyFunction should handle at lease 1 sample\";\n    auto out_size = inputs[0].size();\n    CHECK_EQ(out_size, fs_.size()) << \"In GroupBatchifyFunction, Elem size \" << out_size\n                                   << \" and batchify function size \" << fs_.size() << \" must match\";\n    outputs->resize(out_size);\n    for (size_t i = 0; i < out_size; ++i) {\n      std::vector<std::vector<NDArray>> inp;\n      inp.reserve(inputs.size());\n      for (const auto& input : inputs) {\n        std::vector<NDArray> curr({input[i]});\n        inp.emplace_back(curr);\n      }\n      std::vector<NDArray> tmp;\n      if (!fs_[i]->Batchify(inp, &tmp))\n        return false;\n      (*outputs)[i] = tmp[0];\n    }\n    return true;\n  }\n\n private:\n  /*! \\brief params */\n  GroupBatchifyParam param_;\n  /*! \\brief internal batchify function pointers */\n  std::vector<BatchifyFunctionPtr> fs_;\n};  // class GroupBatchify\n\nMXNET_REGISTER_IO_BATCHIFY_FUNCTION(GroupBatchify)\n    .describe(R\"code(Returns the GroupBatchify function.\n    )code\" ADD_FILELINE)\n    .add_arguments(GroupBatchifyParam::__FIELDS__())\n    .set_body([](const std::vector<std::pair<std::string, std::string>>& kwargs) {\n      return new GroupBatchify(kwargs);\n    });\n\nstruct StackBatchifyParam : public dmlc::Parameter<StackBatchifyParam> {\n  /*! \\brief Length of the sequence. */\n  int use_shared_mem;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(StackBatchifyParam) {\n    DMLC_DECLARE_FIELD(use_shared_mem).set_default(0).describe(\"If 1, use shared memory.\");\n  }\n};  // struct StackBatchifyParam\n\nDMLC_REGISTER_PARAMETER(StackBatchifyParam);\n\nclass StackBatchify : public BatchifyFunction {\n public:\n  explicit StackBatchify(const std::vector<std::pair<std::string, std::string>>& kwargs) {\n    param_.InitAllowUnknown(kwargs);\n  }\n\n  bool Batchify(const std::vector<std::vector<NDArray>>& inputs,\n                std::vector<NDArray>* outputs) override {\n    auto out_size = SanityCheck(inputs);\n    auto bs       = inputs.size();\n    outputs->resize(out_size);\n    for (size_t i = 0; i < out_size; ++i) {\n      // Process i-th output\n      mxnet::TShape ashape = inputs[0][i].shape();\n      CHECK_GE(ashape.ndim(), 0) << \"Data dim must be larger than 0\";\n      // check if all shapes are same\n      for (size_t j = 1; j < bs; ++j) {\n        CHECK_EQ(ashape, inputs[j][i].shape())\n            << \"StackBatchify requires all data along batch dim to be the same, \"\n            << \"mismatch \" << ashape << \" vs. \" << inputs[j][i].shape();\n      }\n\n      // calculate output ndarray size\n      TShape sshape(ashape.ndim() + 1, 0);\n      sshape[0] = bs;\n      for (int k = 0; k < ashape.ndim(); ++k) {\n        sshape[k + 1] = ashape[k];\n      }\n\n      int dtype = inputs[0][i].dtype();\n      if (!(*outputs)[i].is_none() && (*outputs)[i].ctx() == mxnet::Context::CPU(0) &&\n          (*outputs)[i].dtype() == dtype && (*outputs)[i].storage_type() == kDefaultStorage) {\n        if ((*outputs)[i].shape() != sshape) {\n          // realloc\n          (*outputs)[i].ReshapeAndAlloc(sshape);\n        }\n      } else {\n        (*outputs)[i] = NDArray(sshape, mxnet::Context::CPU(0), false, inputs[0][i].dtype());\n      }\n      int sbs = static_cast<int>(bs);\n      MSHADOW_TYPE_SWITCH_WITH_BOOL(dtype, DType, {\n        omp_parallel(bs) for (int j = 0; j < sbs; ++j) {\n          omp_exc_.Run([&] {\n            // inputs[j][i].WaitToRead();\n            DType* ptr = (*outputs)[i].data().dptr<DType>();\n            auto asize = ashape.Size();\n            RunContext rctx{(*outputs)[i].ctx(), nullptr, nullptr};\n            auto dst = TBlob(ptr + asize * j, inputs[j][i].data().shape_, cpu::kDevMask, dtype, 0);\n            mxnet::ndarray::Copy<cpu, cpu>(\n                inputs[j][i].data(), &dst, Context::CPU(), Context::CPU(), rctx);\n          });\n        }\n        omp_exc_.Rethrow();\n      })\n    }\n    return true;\n  }\n\n private:\n  /*! \\brief parameters */\n  StackBatchifyParam param_;\n  /*! \\brief OMPException obj to store and rethrow exceptions from omp blocks*/\n  dmlc::OMPException omp_exc_;\n\n  std::size_t SanityCheck(const std::vector<std::vector<NDArray>>& inputs) {\n    auto bs = inputs.size();\n    CHECK_GT(bs, 0) << \"BatchifyFunction should handle at lease 1 sample\";\n    auto out_size = inputs[0].size();\n    // sanity check: each input has same size\n    for (size_t i = 1; i < bs; ++i) {\n      CHECK_EQ(inputs[i].size(), out_size) << i << \"-th input size does not match \" << out_size;\n    }\n    return out_size;\n  }\n};  // class StackBatchify\n\nMXNET_REGISTER_IO_BATCHIFY_FUNCTION(StackBatchify)\n    .describe(R\"code(Returns the StackBatchify function.\n    )code\" ADD_FILELINE)\n    .add_arguments(StackBatchifyParam::__FIELDS__())\n    .set_body([](const std::vector<std::pair<std::string, std::string>>& kwargs) {\n      return new StackBatchify(kwargs);\n    });\n\nstruct PadBatchifyParam : public dmlc::Parameter<PadBatchifyParam> {\n  int use_shared_mem;\n  double pad_val;\n  int dtype;\n  int round_to;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(PadBatchifyParam) {\n    DMLC_DECLARE_FIELD(use_shared_mem).set_default(0).describe(\"If 1, use shared memory.\");\n    DMLC_DECLARE_FIELD(pad_val).set_default(0).describe(\"The filled values, default to 0.\");\n    DMLC_DECLARE_FIELD(dtype).set_default(-1).describe(\n        \"If not -1, force to use dtype as output type, otherwise use input type.\");\n    DMLC_DECLARE_FIELD(round_to).set_default(-1).describe(\n        \"If > 0, the padded dimension will be rounded to be multiple of this value.\");\n  }\n};  // struct PadBatchifyParam\n\nDMLC_REGISTER_PARAMETER(PadBatchifyParam);\n\nclass PadBatchify : public BatchifyFunction {\n public:\n  explicit PadBatchify(const std::vector<std::pair<std::string, std::string>>& kwargs) {\n    param_.InitAllowUnknown(kwargs);\n  }\n\n  bool Batchify(const std::vector<std::vector<NDArray>>& inputs,\n                std::vector<NDArray>* outputs) override {\n    auto bs = inputs.size();\n    CHECK_GT(bs, 0) << \"BatchifyFunction should handle at lease 1 sample\";\n    auto out_size = inputs[0].size();\n    outputs->resize(out_size);\n    for (size_t i = 0; i < out_size; ++i) {\n      // Process i-th output\n      mxnet::TShape ashape = inputs[0][i].shape();\n      CHECK_GE(ashape.ndim(), 0) << \"Data dim must be larger than 0\";\n      // find the maximum size in each dim\n      for (size_t j = 1; j < bs; ++j) {\n        mxnet::TShape other_shape = inputs[j][i].shape();\n        CHECK_EQ(ashape.ndim(), other_shape.ndim())\n            << \"PadBatchify expects all inputs to have same dimensionality: given \" << ashape.ndim()\n            << \" vs. \" << other_shape.ndim();\n        for (dim_t k = 0; k < ashape.ndim(); ++k) {\n          ashape[k] = std::max(ashape[k], other_shape[k]);\n        }\n      }\n      for (dim_t k = 0; k < ashape.ndim(); ++k) {\n        // pad to multiple of round_to\n        if (param_.round_to > 0) {\n          ashape[k] = param_.round_to *\n                      static_cast<int>(std::ceil(static_cast<double>(ashape[k] / param_.round_to)));\n        }\n      }\n\n      // calculate output ndarray size\n      TShape sshape(ashape.ndim() + 1, 0);\n      sshape[0] = bs;\n      for (int k = 0; k < ashape.ndim(); ++k) {\n        sshape[k + 1] = ashape[k];\n      }\n\n      int dtype = param_.dtype > -1 ? param_.dtype : inputs[0][i].dtype();\n      if (!(*outputs)[i].is_none() && (*outputs)[i].ctx() == mxnet::Context::CPU(0) &&\n          (*outputs)[i].dtype() == dtype && (*outputs)[i].storage_type() == kDefaultStorage) {\n        if ((*outputs)[i].shape() != sshape) {\n          // realloc\n          (*outputs)[i].ReshapeAndAlloc(sshape);\n        }\n      } else {\n        (*outputs)[i] = NDArray(sshape, mxnet::Context::CPU(0), false, inputs[0][i].dtype());\n      }\n      MSHADOW_TYPE_SWITCH_WITH_BOOL(dtype, DType, {\n        // fill pad value first\n        std::fill((*outputs)[i].data().dptr<DType>(),\n                  (*outputs)[i].data().dptr<DType>() + sshape.Size(),\n                  static_cast<DType>(param_.pad_val));\n        DType* ptr = (*outputs)[i].data().dptr<DType>();\n        auto asize = ashape.Size();\n        int sbs    = static_cast<int>(bs);\n        omp_parallel(bs) for (int j = 0; j < sbs; ++j) {\n          using namespace mshadow::expr;\n          auto compact_shapes = CompactShapes(ashape, inputs[j][i].shape());\n          // inputs[j][i].WaitToRead();\n          auto& fshape = compact_shapes.first;\n          auto& cshape = compact_shapes.second;\n          switch (fshape.size()) {\n            case 1U: {\n              mshadow::Tensor<cpu, 1, DType> dst =\n                  TBlob(ptr + asize * j, ashape, cpu::kDevMask, dtype, 0)\n                      .get_with_shape<cpu, 1, DType>(mshadow::Shape1(fshape[0]));\n              mshadow::Tensor<cpu, 1, DType> src =\n                  inputs[j][i].data().get_with_shape<cpu, 1, DType>(mshadow::Shape1(cshape[0]));\n              slice<0>(dst, 0, cshape[0]) = src;\n              break;\n            }\n            case 2U: {\n              mshadow::Tensor<cpu, 2, DType> dst =\n                  TBlob(ptr + asize * j, ashape, cpu::kDevMask, dtype, 0)\n                      .get_with_shape<cpu, 2, DType>(mshadow::Shape2(fshape[0], fshape[1]));\n              mshadow::Tensor<cpu, 2, DType> src =\n                  inputs[j][i].data().get_with_shape<cpu, 2, DType>(\n                      mshadow::Shape2(cshape[0], cshape[1]));\n              slice<1>(slice<0>(dst, 0, cshape[0]), 0, cshape[1]) = src;\n              break;\n            }\n            case 3U: {\n              mshadow::Tensor<cpu, 3, DType> dst =\n                  TBlob(ptr + asize * j, ashape, cpu::kDevMask, dtype, 0)\n                      .get_with_shape<cpu, 3, DType>(\n                          mshadow::Shape3(fshape[0], fshape[1], fshape[2]));\n              mshadow::Tensor<cpu, 3, DType> src =\n                  inputs[j][i].data().get_with_shape<cpu, 3, DType>(\n                      mshadow::Shape3(cshape[0], cshape[1], cshape[2]));\n              slice<2>(slice<1>(slice<0>(dst, 0, cshape[0]), 0, cshape[1]), 0, cshape[2]) = src;\n              break;\n            }\n            case 4U: {\n              mshadow::Tensor<cpu, 4, DType> dst =\n                  TBlob(ptr + asize * j, ashape, cpu::kDevMask, dtype, 0)\n                      .get_with_shape<cpu, 4, DType>(\n                          mshadow::Shape4(fshape[0], fshape[1], fshape[2], fshape[3]));\n              mshadow::Tensor<cpu, 4, DType> src =\n                  inputs[j][i].data().get_with_shape<cpu, 4, DType>(\n                      mshadow::Shape4(cshape[0], cshape[1], cshape[2], cshape[3]));\n              slice<3>(slice<2>(slice<1>(slice<0>(dst, 0, cshape[0]), 0, cshape[1]), 0, cshape[2]),\n                       0,\n                       cshape[3]) = src;\n              break;\n            }\n            case 5U: {\n              mshadow::Tensor<cpu, 5, DType> dst =\n                  TBlob(ptr + asize * j, ashape, cpu::kDevMask, dtype, 0)\n                      .get_with_shape<cpu, 5, DType>(\n                          mshadow::Shape5(fshape[0], fshape[1], fshape[2], fshape[3], fshape[4]));\n              mshadow::Tensor<cpu, 5, DType> src =\n                  inputs[j][i].data().get_with_shape<cpu, 5, DType>(\n                      mshadow::Shape5(cshape[0], cshape[1], cshape[2], cshape[3], cshape[4]));\n              slice<4>(\n                  slice<3>(\n                      slice<2>(slice<1>(slice<0>(dst, 0, cshape[0]), 0, cshape[1]), 0, cshape[2]),\n                      0,\n                      cshape[3]),\n                  0,\n                  cshape[4]) = src;\n              break;\n            }\n            default: {\n              LOG(FATAL) << \"# dim to pad: \" << cshape.size() << \" exceeds limit of 5.\";\n            }\n          }\n        }\n      })\n    }\n    return true;\n  }\n\n private:\n  /*! \\brief parameters */\n  PadBatchifyParam param_;\n  /*! \\brief OMPException obj to store and rethrow exceptions from omp blocks*/\n  dmlc::OMPException omp_exc_;\n\n  std::pair<std::vector<dim_t>, std::vector<dim_t>> CompactShapes(const TShape& ashape,\n                                                                  const TShape& ishape) {\n    // squeeze dimensions that do not need pad\n    std::stack<dim_t> dim_stack;\n    std::vector<dim_t> full_shape;\n    std::vector<dim_t> data_shape;\n    for (dim_t k = 0; k < ishape.ndim(); ++k) {\n      if (ishape[k] == ashape[k]) {\n        dim_stack.push(ishape[k]);\n      } else {\n        dim_t ss = 1;\n        while (!dim_stack.empty()) {\n          ss *= dim_stack.top();\n          dim_stack.pop();\n        }\n        if (ss > 1) {\n          full_shape.emplace_back(ss);\n          data_shape.emplace_back(ss);\n        }\n        full_shape.emplace_back(ashape[k]);\n        data_shape.emplace_back(ishape[k]);\n      }\n    }\n    // clear the stack\n    index_t ss = 1;\n    while (!dim_stack.empty()) {\n      ss *= dim_stack.top();\n      dim_stack.pop();\n    }\n    if (ss > 1 || full_shape.empty()) {\n      full_shape.emplace_back(ss);\n      data_shape.emplace_back(ss);\n    }\n    CHECK_EQ(full_shape.size(), data_shape.size());\n    CHECK_GE(data_shape.size(), 1U);\n    return std::make_pair(full_shape, data_shape);\n  }\n};  // class PadBatchify\n\nMXNET_REGISTER_IO_BATCHIFY_FUNCTION(PadBatchify)\n    .describe(R\"code(Returns the StackBatchify function.\n    )code\" ADD_FILELINE)\n    .add_arguments(PadBatchifyParam::__FIELDS__())\n    .set_body([](const std::vector<std::pair<std::string, std::string>>& kwargs) {\n      return new PadBatchify(kwargs);\n    });\n}  // namespace io\n}  // namespace mxnet\n"
  },
  {
    "path": "src/io/dataloader.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file dataloader.cc\n * \\brief Pure c++ backed dataloader implementation\n */\n#include <dmlc/parameter.h>\n#include <dmlc/omp.h>\n#include <mxnet/io.h>\n\n#include \"./inst_vector.h\"\n#include \"./iter_prefetcher.h\"\n#include \"../profiler/custom_op_profiler.h\"\n\nnamespace mxnet {\nnamespace io {\nstruct ThreadedDataLoaderParam : public dmlc::Parameter<ThreadedDataLoaderParam> {\n  /*! \\brief Multithread worker number. */\n  int num_workers;\n  /*! \\brief dataset pointer.*/\n  std::intptr_t dataset;\n  /*! \\brief sampler pointer.*/\n  std::intptr_t sampler;\n  /*! \\brief batchify function pointer.*/\n  std::intptr_t batchify_fn;\n  /*! \\brief pin memory to device id.*/\n  int pin_device_id;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(ThreadedDataLoaderParam) {\n    DMLC_DECLARE_FIELD(num_workers).set_default(0).describe(\"Number of thread workers.\");\n    DMLC_DECLARE_FIELD(dataset).describe(\"Pointer to shared Dataset.\");\n    DMLC_DECLARE_FIELD(sampler).describe(\"Pointer to Sampler.\");\n    DMLC_DECLARE_FIELD(batchify_fn).describe(\"Pointer to Batchify function.\");\n    DMLC_DECLARE_FIELD(pin_device_id)\n        .set_default(-1)\n        .describe(\"If not negative, will move data to pinned memory.\");\n  }\n};  // struct ThreadedDataLoaderParam\n\nDMLC_REGISTER_PARAMETER(ThreadedDataLoaderParam);\n\ntemplate <typename DType = real_t>\nclass ThreadedDataLoader : public IIterator<TBlobBatch> {\n public:\n  ThreadedDataLoader() = default;\n  // destructor\n  ~ThreadedDataLoader() override = default;\n  // constructor\n  void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override {\n    param_.InitAllowUnknown(kwargs);\n    int maxthread, threadget;\n#pragma omp parallel\n    {\n      // be conservative, set number of real cores\n      maxthread = std::max(omp_get_num_procs(), 1);\n    }\n    param_.num_workers = std::min(maxthread, param_.num_workers);\n#pragma omp parallel num_threads(param_.num_workers)\n    { threadget = omp_get_num_threads(); }\n    param_.num_workers = std::max(1, threadget);\n    dataset_     = *static_cast<std::shared_ptr<Dataset>*>(reinterpret_cast<void*>(param_.dataset));\n    dataset_len_ = dataset_->GetLen();\n    sampler_     = static_cast<IIterator<DataBatch>*>(reinterpret_cast<void*>(param_.sampler));\n    batchify_fn_ = *static_cast<BatchifyFunctionPtr*>(reinterpret_cast<void*>(param_.batchify_fn));\n    this->BeforeFirst();\n  }\n  // before first\n  void BeforeFirst() override {\n    sampler_->BeforeFirst();\n  }\n\n  int64_t GetLenHint() const override {\n    return sampler_->GetLenHint();\n  }\n\n  bool Next() override {\n    bool has_next = sampler_->Next();\n    if (!has_next)\n      return false;\n    auto samples           = sampler_->Value();\n    auto batch_size        = samples.data[0].shape().Size();\n    int real_batch_size    = batch_size - samples.num_batch_padd;\n    const int64_t* idx_ptr = static_cast<int64_t*>(samples.data[0].data().dptr_);\n    std::vector<int64_t> idx_ptrs;\n    idx_ptrs.assign(idx_ptr, idx_ptr + real_batch_size);\n\n    // __getitem__\n    std::vector<std::vector<NDArray> > inputs(batch_size);\n    std::vector<int> is_scalars;\n    bool profiling = profiler::Profiler::Get()->IsProfiling(profiler::Profiler::kImperative);\n    if (profiling) {\n      profiler::CustomOpProfiler::Get()->OnCustomBegin(\"MXThreadedDataLoaderGetItems\");\n    }\n#pragma omp parallel for num_threads(param_.num_workers)\n    for (int i = 0; i < real_batch_size; ++i) {\n      omp_exc_.Run([&] {\n        auto idx = idx_ptrs[i];\n        CHECK(dataset_->GetItem(idx, &inputs[i])) << \"Error getting data # \" << idx;\n      });\n    }\n    if (profiling) {\n      profiler::CustomOpProfiler::Get()->OnCustomEnd();\n    }\n    omp_exc_.Rethrow();\n\n    // pad to normal batch size\n    for (size_t i = real_batch_size; i < batch_size; ++i) {\n      inputs[i] = inputs[0];\n    }\n\n    // batchify\n    if (profiling) {\n      profiler::CustomOpProfiler::Get()->OnCustomBegin(\"MXThreadedDataLoaderBatchify\");\n    }\n    CHECK(batchify_fn_->Batchify(inputs, &batched_buffer_))\n        << \"Error call batchify inside dataloader\";\n    if (profiling) {\n      profiler::CustomOpProfiler::Get()->OnCustomEnd();\n    }\n    out_.batch_size = batched_buffer_.size();\n    out_.data.resize(batched_buffer_.size());\n    for (size_t i = 0; i < batched_buffer_.size(); ++i) {\n      out_.data[i] = batched_buffer_[i].data();\n    }\n    out_.num_batch_padd = samples.num_batch_padd;\n    return true;\n  }\n\n  const TBlobBatch& Value() const override {\n    return out_;\n  }\n\n private:\n  /*! \\brief Params */\n  ThreadedDataLoaderParam param_;\n  /*! \\brief output */\n  TBlobBatch out_;\n  /*! \\brief batched buffer */\n  std::vector<NDArray> batched_buffer_;\n  /*! \\brief pointer to dataset */\n  std::shared_ptr<Dataset> dataset_;\n  /*! \\brief dataset length */\n  int64_t dataset_len_;\n  /*! \\brief pointer to sampler iterator */\n  IIterator<DataBatch>* sampler_;\n  /*! \\brief pointer to batchify function */\n  BatchifyFunctionPtr batchify_fn_;\n  /*! \\brief OMPException obj to store and rethrow exceptions from omp blocks*/\n  dmlc::OMPException omp_exc_;\n};  // class ThreadedDataLoader\n\nMXNET_REGISTER_IO_ITER(ThreadedDataLoader)\n    .describe(R\"code(Returns a threaded data loader iterator.\n)code\" ADD_FILELINE)\n    .add_arguments(ThreadedDataLoaderParam::__FIELDS__())\n    .add_arguments(PrefetcherParam::__FIELDS__())\n    .set_body([]() { return new PrefetcherIter(new ThreadedDataLoader<mxnet::real_t>()); });\n}  // namespace io\n}  // namespace mxnet\n"
  },
  {
    "path": "src/io/dataset.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file dataset.cc\n * \\brief High performance datasets implementation\n */\n#include <dmlc/parameter.h>\n#include <dmlc/recordio.h>\n#include <dmlc/io.h>\n#include <mxnet/io.h>\n#include <mxnet/ndarray.h>\n#include <mxnet/tensor_blob.h>\n\n#include <memory>\n#include <string>\n#include <vector>\n#include <algorithm>\n#include <thread>\n\n#include \"../imperative/cached_op.h\"\n#include \"../imperative/naive_cached_op.h\"\n#include \"../ndarray/ndarray_function.h\"\n\n#if MXNET_USE_OPENCV\n#include <opencv2/opencv.hpp>\n#include \"./opencv_compatibility.h\"\n#endif  // MXNET_USE_OPENCV\n\nnamespace mxnet {\nnamespace io {\n\nstruct RecordFileDatasetParam : public dmlc::Parameter<RecordFileDatasetParam> {\n  std::string rec_file;\n  std::string idx_file;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(RecordFileDatasetParam) {\n    DMLC_DECLARE_FIELD(rec_file).describe(\"The absolute path of record file.\");\n    DMLC_DECLARE_FIELD(idx_file).describe(\"The path of the idx file.\");\n  }\n};  // struct RecordFileDatasetParam\n\nDMLC_REGISTER_PARAMETER(RecordFileDatasetParam);\n\nclass RecordFileDataset final : public Dataset {\n public:\n  explicit RecordFileDataset(const std::vector<std::pair<std::string, std::string>>& kwargs) {\n    std::vector<std::pair<std::string, std::string>> kwargs_left;\n    param_.InitAllowUnknown(kwargs);\n    // read and process idx file\n    dmlc::Stream* idx_stream = dmlc::Stream::Create(param_.idx_file.c_str(), \"r\");\n    dmlc::istream is(idx_stream);\n    size_t key, idx;\n    while (is >> key >> idx) {\n      idx_[key] = idx;\n    }\n    delete idx_stream;\n  }\n\n  uint64_t GetLen() const override {\n    return idx_.size();\n  }\n\n  bool GetItem(uint64_t idx, std::vector<NDArray>* ret) override {\n    ret->resize(1);\n    auto& out = (*ret)[0];\n    static thread_local std::unique_ptr<dmlc::Stream> stream;\n    static thread_local std::unique_ptr<dmlc::RecordIOReader> reader;\n    if (!reader) {\n      auto s = dmlc::Stream::Create(param_.rec_file.c_str(), \"r\");\n      stream.reset(s);\n      reader = std::make_unique<dmlc::RecordIOReader>(s);\n    }\n    size_t pos = idx_[static_cast<size_t>(idx)];\n    reader->Seek(pos);\n    static thread_local std::string read_buff;\n    if (reader->NextRecord(&read_buff)) {\n      const char* buf   = read_buff.c_str();\n      const size_t size = read_buff.size();\n      out = NDArray(TShape({static_cast<dim_t>(size)}), Context::CPU(), false, mshadow::kInt8);\n      TBlob dst = out.data();\n      RunContext rctx{Context::CPU(), nullptr, nullptr};\n      mxnet::ndarray::Copy<cpu, cpu>(TBlob(const_cast<void*>(reinterpret_cast<const void*>(buf)),\n                                           out.shape(),\n                                           cpu::kDevMask,\n                                           out.dtype(),\n                                           0),\n                                     &dst,\n                                     Context::CPU(),\n                                     Context::CPU(),\n                                     rctx);\n    }\n    return true;\n  }\n\n private:\n  /*! \\brief parameters */\n  RecordFileDatasetParam param_;\n  /*! \\brief indices */\n  std::unordered_map<size_t, size_t> idx_;\n};\n\nMXNET_REGISTER_IO_DATASET(RecordFileDataset)\n    .describe(\"MXNet Record File Dataset\")\n    .add_arguments(RecordFileDatasetParam::__FIELDS__())\n    .set_body([](const std::vector<std::pair<std::string, std::string>>& kwargs) {\n      return new RecordFileDataset(kwargs);\n    });\n\nstruct ImageRecordFileDatasetParam : public dmlc::Parameter<ImageRecordFileDatasetParam> {\n  std::string rec_file;\n  std::string idx_file;\n  int flag;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(ImageRecordFileDatasetParam) {\n    DMLC_DECLARE_FIELD(rec_file).describe(\"The absolute path of record file.\");\n    DMLC_DECLARE_FIELD(idx_file).describe(\"The path of the idx file.\");\n    DMLC_DECLARE_FIELD(flag).set_default(1).describe(\n        \"If 1, always convert to colored, if 0 always convert to grayscale.\");\n  }\n};  // struct ImageRecordFileDatasetParam\n\nDMLC_REGISTER_PARAMETER(ImageRecordFileDatasetParam);\n\n#if MXNET_USE_OPENCV\ntemplate <int n_channels>\nvoid SwapImageChannels(const cv::Mat& img, NDArray* arr) {\n  int swap_indices[n_channels];  // NOLINT(*)\n  if (n_channels == 1) {\n    swap_indices[0] = 0;\n  } else if (n_channels == 3) {\n    swap_indices[0] = 2;\n    swap_indices[1] = 1;\n    swap_indices[2] = 0;\n  } else if (n_channels == 4) {\n    swap_indices[0] = 2;\n    swap_indices[1] = 1;\n    swap_indices[2] = 0;\n    swap_indices[3] = 3;\n  }\n\n  TShape arr_shape = TShape({img.rows, img.cols, n_channels});\n  if (arr->is_none() || arr->shape() != arr_shape || arr->ctx() != mxnet::Context::CPU(0) ||\n      arr->dtype() != mshadow::kUint8 || arr->storage_type() != kDefaultStorage) {\n    *arr = NDArray(arr_shape, mxnet::Context::CPU(0), false, mshadow::kUint8);\n  }\n  auto ptr = static_cast<uint8_t*>(arr->data().dptr_);\n\n  // swap channels while copying elements into buffer\n  for (int i = 0; i < img.rows; ++i) {\n    const uint8_t* im_data = img.ptr<uint8_t>(i);\n    uint8_t* buffer_data   = ptr + i * img.cols * n_channels;\n    for (int j = 0; j < img.cols; ++j) {\n      for (int k = 0; k < n_channels; ++k) {\n        buffer_data[k] = im_data[swap_indices[k]];\n      }\n      im_data += n_channels;\n      buffer_data += n_channels;\n    }\n  }\n}\n#endif\n\n/*! \\brief Struct for unpack recordio header */\n#pragma pack(1)\nstruct IRHeader {\n  uint32_t flag;\n  float label;\n  uint64_t id;\n  uint64_t id2;\n};  // struct IRHeader\n\nclass ImageRecordFileDataset : public Dataset {\n public:\n  explicit ImageRecordFileDataset(const std::vector<std::pair<std::string, std::string>>& kwargs) {\n    std::vector<std::pair<std::string, std::string>> kwargs_left;\n    param_.InitAllowUnknown(kwargs);\n    base_ = std::make_shared<RecordFileDataset>(kwargs);\n  }\n\n  uint64_t GetLen() const override {\n    return base_->GetLen();\n  }\n\n  bool GetItem(uint64_t idx, std::vector<NDArray>* ret) override {\n    CHECK_LT(idx, GetLen());\n    std::vector<NDArray> raw;\n    if (!base_->GetItem(idx, &raw))\n      return false;\n    CHECK_EQ(raw.size(), 1U) << \"RecordFileDataset should return size 1 NDArray vector\";\n    uint8_t* s  = reinterpret_cast<uint8_t*>(raw[0].data().dptr_);\n    size_t size = raw[0].shape().Size();\n    CHECK_GT(size, sizeof(IRHeader)) << \"Invalid size of bytes from Record File\";\n    IRHeader header;\n    std::memcpy(&header, s, sizeof(header));\n    size -= sizeof(header);\n    s += sizeof(header);\n    NDArray label = NDArray(Context::CPU(), mshadow::default_type_flag);\n    RunContext rctx{Context::CPU(), nullptr, nullptr};\n    if (header.flag > 0) {\n      auto label_shape = header.flag <= 1 ? TShape(0, 1) : TShape({header.flag});\n      label.ReshapeAndAlloc(label_shape);\n      TBlob dst = label.data();\n      mxnet::ndarray::Copy<cpu, cpu>(\n          TBlob(reinterpret_cast<void*>(s), label.shape(), cpu::kDevMask, label.dtype(), 0),\n          &dst,\n          Context::CPU(),\n          Context::CPU(),\n          rctx);\n      s += sizeof(float) * header.flag;\n      size -= sizeof(float) * header.flag;\n    } else {\n      // label is a scalar with ndim() == 0\n      label.ReshapeAndAlloc(TShape(0, 1));\n      TBlob dst            = label.data();\n      *(dst.dptr<float>()) = header.label;\n    }\n    ret->resize(2);\n    (*ret)[1] = label;\n#if MXNET_USE_OPENCV\n    cv::Mat buf(1, size, CV_8U, s);\n    cv::Mat res = cv::imdecode(buf, param_.flag);\n    CHECK(!res.empty()) << \"Decoding failed. Invalid image file.\";\n    const int n_channels = res.channels();\n    if (n_channels == 1) {\n      SwapImageChannels<1>(res, &(ret->at(0)));\n    } else if (n_channels == 3) {\n      SwapImageChannels<3>(res, &(ret->at(0)));\n    } else if (n_channels == 4) {\n      SwapImageChannels<4>(res, &(ret->at(0)));\n    }\n    return true;\n#else\n    LOG(FATAL) << \"Opencv is needed for image decoding.\";\n#endif\n    return false;  // should not reach here\n  }\n\n private:\n  /*! \\brief parameters */\n  ImageRecordFileDatasetParam param_;\n  /*! \\brief base recordIO reader */\n  std::shared_ptr<RecordFileDataset> base_;\n};\n\nMXNET_REGISTER_IO_DATASET(ImageRecordFileDataset)\n    .describe(\"MXNet Image Record File Dataset\")\n    .add_arguments(ImageRecordFileDatasetParam::__FIELDS__())\n    .set_body([](const std::vector<std::pair<std::string, std::string>>& kwargs) {\n      return new ImageRecordFileDataset(kwargs);\n    });\n\nstruct ImageSequenceDatasetParam : public dmlc::Parameter<ImageSequenceDatasetParam> {\n  /*! \\brief the list of absolute image paths, separated by \\0 characters */\n  std::string img_list;\n  /*! \\brief the path separator character, by default it's ; */\n  char path_sep;\n  /*! \\brief If flag is 0, always convert to grayscale(1 channel).\n   * If flag is 1, always convert to colored (3 channels).\n   * If flag is -1, keep channels unchanged.\n   */\n  int flag;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(ImageSequenceDatasetParam) {\n    DMLC_DECLARE_FIELD(img_list).describe(\"The list of image absolute paths.\");\n    DMLC_DECLARE_FIELD(path_sep).set_default('|').describe(\n        \"The path separator for joined image paths.\");\n    DMLC_DECLARE_FIELD(flag).set_default(1).describe(\n        \"If 1, always convert to colored, if 0 always convert to grayscale.\");\n  }\n};  // struct ImageSequenceDatasetParam\n\nDMLC_REGISTER_PARAMETER(ImageSequenceDatasetParam);\n\nclass ImageSequenceDataset final : public Dataset {\n public:\n  explicit ImageSequenceDataset(const std::vector<std::pair<std::string, std::string>>& kwargs) {\n    std::vector<std::pair<std::string, std::string>> kwargs_left;\n    param_.InitAllowUnknown(kwargs);\n    img_list_ = dmlc::Split(param_.img_list, param_.path_sep);\n  }\n\n  uint64_t GetLen() const override {\n    return img_list_.size();\n  }\n\n  bool GetItem(uint64_t idx, std::vector<NDArray>* ret) override {\n#if MXNET_USE_OPENCV\n    CHECK_LT(idx, img_list_.size())\n        << \"GetItem index: \" << idx << \" out of bound: \" << img_list_.size();\n    cv::Mat res = cv::imread(img_list_[idx], param_.flag);\n    CHECK(!res.empty()) << \"Decoding failed. Invalid image file.\";\n    const int n_channels = res.channels();\n    ret->resize(1);\n    if (n_channels == 1) {\n      SwapImageChannels<1>(res, &(ret->at(0)));\n    } else if (n_channels == 3) {\n      SwapImageChannels<3>(res, &(ret->at(0)));\n    } else if (n_channels == 4) {\n      SwapImageChannels<4>(res, &(ret->at(0)));\n    }\n    return true;\n#else\n    LOG(FATAL) << \"Opencv is needed for image decoding.\";\n#endif\n    return false;\n  }\n\n private:\n  /*! \\brief parameters */\n  ImageSequenceDatasetParam param_;\n  /*! \\brief image list */\n  std::vector<std::string> img_list_;\n};\n\nMXNET_REGISTER_IO_DATASET(ImageSequenceDataset)\n    .describe(\"Image Sequence Dataset\")\n    .add_arguments(ImageSequenceDatasetParam::__FIELDS__())\n    .set_body([](const std::vector<std::pair<std::string, std::string>>& kwargs) {\n      return new ImageSequenceDataset(kwargs);\n    });\n\nstruct NDArrayDatasetParam : public dmlc::Parameter<NDArrayDatasetParam> {\n  /*! \\brief the source ndarray */\n  std::intptr_t arr;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(NDArrayDatasetParam) {\n    DMLC_DECLARE_FIELD(arr).describe(\"Pointer to NDArray.\");\n  }\n};  // struct NDArrayDatasetParam\n\nDMLC_REGISTER_PARAMETER(NDArrayDatasetParam);\n\nclass NDArrayDataset final : public Dataset {\n public:\n  explicit NDArrayDataset(const std::vector<std::pair<std::string, std::string>>& kwargs) {\n    param_.InitAllowUnknown(kwargs);\n    data_ = *(static_cast<NDArray*>(reinterpret_cast<void*>(param_.arr)));\n    if (data_.shape().ndim() < 1) {\n      LOG(FATAL) << \"NDArray with no dim is not iterable\";\n    }\n    size_ = data_.shape().begin()[0];\n  }\n\n  uint64_t GetLen() const override {\n    return size_;\n  }\n\n  bool GetItem(uint64_t idx, std::vector<NDArray>* rets) override {\n    CHECK_LT(idx, size_) << \"GetItem index: \" << idx << \" out of bound: \" << size_;\n    rets->resize(1);\n    auto& ret = (*rets)[0];\n    ret       = data_.Slice(idx, idx + 1);\n    if (ret.shape().ndim() > 1) {\n      // remove first dim to be consistent with numpy\n      TShape new_shape;\n      new_shape.assign(ret.shape().begin() + 1, ret.shape().end());\n      ret = ret.Reshape(new_shape);\n    } else {\n      if (data_.shape().ndim() == 1) {\n        // scalar\n        TShape new_shape(0, 1);\n        ret = ret.Reshape(new_shape);\n      }\n    }\n    return true;\n  }\n\n private:\n  /*! \\brief parameters */\n  NDArrayDatasetParam param_;\n  /*! \\brief stored ndarray */\n  NDArray data_;\n  /*! \\brief stored ndarray shape */\n  int64_t size_;\n};  // class NDArrayDataset\n\nMXNET_REGISTER_IO_DATASET(NDArrayDataset)\n    .describe(\"Single NDArray Dataset\")\n    .add_arguments(NDArrayDatasetParam::__FIELDS__())\n    .set_body([](const std::vector<std::pair<std::string, std::string>>& kwargs) {\n      return new NDArrayDataset(kwargs);\n    });\n\nstruct GroupDatasetParam : public dmlc::Parameter<GroupDatasetParam> {\n  /*! \\brief the source ndarray */\n  Tuple<std::intptr_t> datasets;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(GroupDatasetParam) {\n    DMLC_DECLARE_FIELD(datasets).describe(\"A small set of pointers to other c++ datasets.\");\n  }\n};  // struct GroupDatasetParam\n\nDMLC_REGISTER_PARAMETER(GroupDatasetParam);\n\nclass GroupDataset final : public Dataset {\n public:\n  explicit GroupDataset(const std::vector<std::pair<std::string, std::string>>& kwargs) {\n    std::vector<std::pair<std::string, std::string>> kwargs_left;\n    param_.InitAllowUnknown(kwargs);\n    auto childs = param_.datasets;\n    childs_.reserve(childs.ndim());\n    size_t child_cnt = 0;\n    for (auto child : childs) {\n      auto d = *static_cast<std::shared_ptr<Dataset>*>(reinterpret_cast<void*>(child));\n      if (child_cnt == 0) {\n        size_ = d->GetLen();\n      } else {\n        CHECK_EQ(size_, d->GetLen()) << \"All child dataset of GroupDataset must be identical \"\n                                     << \"Given mismatch: \" << size_ << \" vs \" << d->GetLen();\n      }\n      childs_.emplace_back(d);\n      child_cnt++;\n    }\n  }\n\n  uint64_t GetLen() const override {\n    return size_;\n  }\n\n  bool GetItem(uint64_t idx, std::vector<NDArray>* ret) override {\n    CHECK_LT(idx, size_) << \"GetItem index: \" << idx << \" out of bound: \" << size_;\n    ret->clear();\n    for (const auto& child : childs_) {\n      std::vector<NDArray> temp_ret;\n      if (!child->GetItem(idx, &temp_ret))\n        return false;\n      ret->insert(ret->end(), temp_ret.begin(), temp_ret.end());\n    }\n    return true;\n  }\n\n private:\n  /*! \\brief parameters */\n  GroupDatasetParam param_;\n  /*! \\brief stored child datasets */\n  std::vector<std::shared_ptr<Dataset>> childs_;\n  /*! \\brief overall dataset size, equals to all child datasets */\n  uint64_t size_;\n};  // class GroupDataset\n\nMXNET_REGISTER_IO_DATASET(GroupDataset)\n    .describe(\"Grouped Dataset that combine a bunch of datasets\")\n    .add_arguments(GroupDatasetParam::__FIELDS__())\n    .set_body([](const std::vector<std::pair<std::string, std::string>>& kwargs) {\n      return new GroupDataset(kwargs);\n    });\n\nstruct IndexedDatasetParam : public dmlc::Parameter<IndexedDatasetParam> {\n  /*! \\brief the base dataset */\n  std::intptr_t base;\n  /*! \\brief the indices */\n  Tuple<uint64_t> indices;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(IndexedDatasetParam) {\n    DMLC_DECLARE_FIELD(base).describe(\n        \"Pointer to the internal c++ dataset that is going to be indexed.\");\n    DMLC_DECLARE_FIELD(indices).describe(\n        \"The indices for the internal dataset. Output[i] will be base[indices[i]].\");\n  }\n};  // struct IndexedDatasetParam\n\nDMLC_REGISTER_PARAMETER(IndexedDatasetParam);\n\nclass IndexedDataset final : public Dataset {\n public:\n  explicit IndexedDataset(const std::vector<std::pair<std::string, std::string>>& kwargs) {\n    param_.InitAllowUnknown(kwargs);\n    base_data_ = *static_cast<std::shared_ptr<Dataset>*>(reinterpret_cast<void*>(param_.base));\n  }\n\n  uint64_t GetLen() const override {\n    return param_.indices.ndim();\n  }\n\n  bool GetItem(uint64_t idx, std::vector<NDArray>* ret) override {\n    CHECK_GT(param_.indices.ndim(), idx)\n        << \"IndexError: \" << idx << \" from total: \" << param_.indices.ndim();\n    auto new_idx = param_.indices[idx];\n    CHECK_GT(base_data_->GetLen(), new_idx)\n        << \"IndexError: \" << new_idx\n        << \" from original dataset with size: \" << base_data_->GetLen();\n    return base_data_->GetItem(new_idx, ret);\n  }\n\n private:\n  /*! \\brief parameters */\n  IndexedDatasetParam param_;\n  /*! \\brief stored child dataset */\n  std::shared_ptr<Dataset> base_data_;\n};  // class IndexedDataset\n\nMXNET_REGISTER_IO_DATASET(IndexedDataset)\n    .describe(\"Grouped Dataset that combine a bunch of datasets\")\n    .add_arguments(IndexedDatasetParam::__FIELDS__())\n    .set_body([](const std::vector<std::pair<std::string, std::string>>& kwargs) {\n      return new IndexedDataset(kwargs);\n    });\n\nstruct LazyTransformDatasetParam : public dmlc::Parameter<LazyTransformDatasetParam> {\n  /*! \\brief the source ndarray */\n  std::intptr_t cached_op;\n  /*! \\brief internal dataset */\n  std::intptr_t dataset;\n  /*! \\brief indices for items that needs transformation */\n  Tuple<int> transform_indices;\n  /*! \\brief is_scalar information for outputs */\n  Tuple<int> scalar_outputs;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(LazyTransformDatasetParam) {\n    DMLC_DECLARE_FIELD(cached_op).describe(\"Pointer to cached transform function.\");\n    DMLC_DECLARE_FIELD(dataset).describe(\"Pointer to internal dataset.\");\n    DMLC_DECLARE_FIELD(transform_indices)\n        .set_default(Tuple<int>({}))\n        .describe(\n            \"The indices for dataset items that need to be transformed/processed. \"\n            \"If `transform_indices` is empty(default), \"\n            \"then all items will be processed.\");\n    DMLC_DECLARE_FIELD(scalar_outputs)\n        .describe(\"Indicate whether outputs are scalars, the size must match the output size.\");\n  }\n};  // struct LazyTransformDatasetParam\n\nDMLC_REGISTER_PARAMETER(LazyTransformDatasetParam);\n\nclass LazyTransformDataset final : public Dataset {\n public:\n  LazyTransformDataset(const LazyTransformDataset& other) {\n    this->param_                = other.param_;\n    this->pass_through_indices_ = other.pass_through_indices_;\n    this->use_input_indices_    = other.use_input_indices_;\n    this->num_outputs_          = other.num_outputs_;\n    this->cached_op_ =\n        std::make_shared<NaiveCachedOp>(other.cached_op_->sym_, other.cached_op_->flags_);\n    this->base_data_ = other.base_data_;\n  }\n\n  explicit LazyTransformDataset(const std::vector<std::pair<std::string, std::string>>& kwargs) {\n    param_.InitAllowUnknown(kwargs);\n    auto op    = *static_cast<CachedOpPtr*>(reinterpret_cast<void*>(param_.cached_op));\n    cached_op_ = std::make_shared<NaiveCachedOp>(op->sym_, op->flags_);\n    base_data_ = *static_cast<std::shared_ptr<Dataset>*>(reinterpret_cast<void*>(param_.dataset));\n\n    // use first item to calculate size info\n    CHECK_GT(GetLen(), 0) << \"LazyTransformDataset expect the base dataset to have at least 1 item\";\n    std::vector<NDArray> inputs;\n    CHECK(base_data_->GetItem(0, &inputs));\n    // check output size\n    CHECK_EQ(param_.scalar_outputs.ndim(), cached_op_->num_outputs())\n        << \"Output scalar info size: \" << param_.scalar_outputs.ndim()\n        << \" vs. output size: \" << cached_op_->num_outputs() << \" mismatch!\";\n    // check input size\n    if (param_.transform_indices.ndim() == 0) {\n      std::vector<int> default_indices;\n      default_indices.reserve(cached_op_->num_inputs());\n      for (size_t i = 0; i < cached_op_->num_inputs(); ++i) {\n        default_indices.emplace_back(static_cast<int>(i));\n      }\n      use_input_indices_ = default_indices;\n    } else {\n      use_input_indices_ =\n          std::vector<int>(param_.transform_indices.begin(), param_.transform_indices.end());\n    }\n    CHECK_EQ(use_input_indices_.size(), cached_op_->num_inputs())\n        << \"Mismatched transform indices and transform inputs: \" << use_input_indices_.size()\n        << \" vs. \" << cached_op_->num_inputs();\n    auto num_inputs = use_input_indices_.size();\n    CHECK_GE(inputs.size(), num_inputs) << \"LazyTransformDataset input size \" << inputs.size()\n                                        << \" smaller than transform input size: \" << num_inputs;\n    pass_through_indices_.clear();\n    for (size_t i = 0; i < inputs.size(); ++i) {\n      // filling output ndarray from unaltered inputs, transformed outputs are already inserted\n      if (std::find(use_input_indices_.begin(), use_input_indices_.end(), i) ==\n          use_input_indices_.end()) {\n        pass_through_indices_.emplace_back(i);\n      }\n    }\n    num_outputs_ = inputs.size() + cached_op_->num_outputs() - cached_op_->num_inputs();\n  }\n\n  ~LazyTransformDataset() override = default;\n\n  uint64_t GetLen() const override {\n    return base_data_->GetLen();\n  }\n\n  bool GetItem(uint64_t idx, std::vector<NDArray>* outputs) override {\n    std::vector<NDArray> inputs;\n    if (!base_data_->GetItem(idx, &inputs))\n      return false;\n    outputs->reserve(num_outputs_);\n    outputs->resize(cached_op_->num_outputs());\n    for (auto i : pass_through_indices_) {\n      outputs->emplace_back(inputs[i]);\n    }\n    CHECK_EQ(outputs->size(), num_outputs_);\n    // workspace for cached op\n    std::vector<NDArray*> ndinputs;\n    std::vector<NDArray*> ndoutputs;\n    ndinputs.reserve(inputs.size());\n    for (int use_input_indice : use_input_indices_) {\n      ndinputs.emplace_back(&(inputs[use_input_indice]));\n    }\n    ndoutputs.reserve(cached_op_->num_outputs());\n    CHECK_LE(cached_op_->num_outputs(), outputs->size());\n    for (size_t i = 0; i < cached_op_->num_outputs(); ++i) {\n      ndoutputs.emplace_back(&(outputs->at(i)));\n    }\n\n    for (auto& input : inputs) {\n      input.WaitToRead();\n    }\n    CHECK(inputs.size() > 0) << \"dataset getitem requires at least one input\";\n    Context default_ctx = inputs[0].ctx();\n    cached_op_->Forward(cached_op_, ndinputs, ndoutputs, default_ctx);\n    return true;\n  }\n\n private:\n  /*! \\brief parameters */\n  LazyTransformDatasetParam param_;\n  /*! \\brief stored cached op */\n  NaiveCachedOpPtr cached_op_;\n  /*! \\brief internal dataset */\n  std::shared_ptr<Dataset> base_data_;\n  std::vector<int> use_input_indices_;\n  std::vector<int> pass_through_indices_;\n  size_t num_outputs_;\n};  // class LazyTransformDataset\n\nMXNET_REGISTER_IO_DATASET(LazyTransformDataset)\n    .describe(\"Dataset that apply lazy transformation to internal dataset\")\n    .add_arguments(LazyTransformDatasetParam::__FIELDS__())\n    .set_body([](const std::vector<std::pair<std::string, std::string>>& kwargs) {\n      return new LazyTransformDataset(kwargs);\n    });\n}  // namespace io\n}  // namespace mxnet\n"
  },
  {
    "path": "src/io/image_aug_default.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file image_aug_default.cc\n * \\brief Default augmenter.\n */\n#include <mxnet/base.h>\n#include <dmlc/optional.h>\n#include <utility>\n#include <string>\n#include <algorithm>\n#include <vector>\n#include \"./image_augmenter.h\"\n#include \"../common/utils.h\"\n\n#if MXNET_USE_OPENCV\n#include \"./opencv_compatibility.h\"\n// Registers\nnamespace dmlc {\nDMLC_REGISTRY_ENABLE(::mxnet::io::ImageAugmenterReg);\n}  // namespace dmlc\n#endif\n\nnamespace mxnet {\nnamespace io {\n\n/*! \\brief image augmentation parameters*/\nstruct DefaultImageAugmentParam : public dmlc::Parameter<DefaultImageAugmentParam> {\n  /*! \\brief resize shorter edge to size before applying other augmentations */\n  int resize;\n  /*! \\brief whether we do random cropping */\n  bool rand_crop;\n  /*! \\brief whether we do random resized cropping */\n  bool random_resized_crop;\n  /*! \\brief [-max_rotate_angle, max_rotate_angle] */\n  int max_rotate_angle;\n  /*! \\brief max aspect ratio */\n  float max_aspect_ratio;\n  /*! \\brief min aspect ratio */\n  dmlc::optional<float> min_aspect_ratio;\n  /*! \\brief random shear the image [-max_shear_ratio, max_shear_ratio] */\n  float max_shear_ratio;\n  /*! \\brief max crop size */\n  int max_crop_size;\n  /*! \\brief min crop size */\n  int min_crop_size;\n  /*! \\brief max scale ratio */\n  float max_random_scale;\n  /*! \\brief min scale ratio */\n  float min_random_scale;\n  /*! \\brief max area */\n  float max_random_area;\n  /*! \\brief min area */\n  float min_random_area;\n  /*! \\brief min image size */\n  float min_img_size;\n  /*! \\brief max image size */\n  float max_img_size;\n  /*! \\brief max random brightness */\n  float brightness;\n  /*! \\brief max random contrast */\n  float contrast;\n  /*! \\brief max random saturation */\n  float saturation;\n  /*! \\brief pca noise level */\n  float pca_noise;\n  /*! \\brief max random in H channel */\n  int random_h;\n  /*! \\brief max random in S channel */\n  int random_s;\n  /*! \\brief max random in L channel */\n  int random_l;\n  /*! \\brief rotate angle */\n  int rotate;\n  /*! \\brief filled color while padding */\n  int fill_value;\n  /*! \\brief interpolation method 0-NN 1-bilinear 2-cubic 3-area 4-lanczos4 9-auto 10-rand  */\n  int inter_method;\n  /*! \\brief padding size */\n  int pad;\n  /*! \\brief shape of the image data*/\n  mxnet::TShape data_shape;\n\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(DefaultImageAugmentParam) {\n    DMLC_DECLARE_FIELD(resize).set_default(-1).describe(\n        \"Down scale the shorter edge to a new size  \"\n        \"before applying other augmentations.\");\n    DMLC_DECLARE_FIELD(rand_crop).set_default(false).describe(\"If or not randomly crop the image\");\n    DMLC_DECLARE_FIELD(random_resized_crop)\n        .set_default(false)\n        .describe(\n            \"If or not perform random resized cropping \"\n            \"on the image, as a standard preprocessing \"\n            \"for resnet training on ImageNet data.\");\n    DMLC_DECLARE_FIELD(max_rotate_angle)\n        .set_default(0.0f)\n        .describe(\"Rotate by a random degree in ``[-v, v]``\");\n    DMLC_DECLARE_FIELD(max_aspect_ratio)\n        .set_default(0.0f)\n        .describe(\n            \"Change the aspect (namely width/height) to a random value. \"\n            \"If min_aspect_ratio is None then the aspect ratio ins sampled from \"\n            \"[1 - max_aspect_ratio, 1 + max_aspect_ratio], \"\n            \"else it is in ``[min_aspect_ratio, max_aspect_ratio]``\");\n    DMLC_DECLARE_FIELD(min_aspect_ratio)\n        .set_default(dmlc::optional<float>())\n        .describe(\n            \"Change the aspect (namely width/height) to a random value \"\n            \"in ``[min_aspect_ratio, max_aspect_ratio]``\");\n    DMLC_DECLARE_FIELD(max_shear_ratio)\n        .set_default(0.0f)\n        .describe(\n            \"Apply a shear transformation (namely ``(x,y)->(x+my,y)``) \"\n            \"with ``m`` randomly chose from \"\n            \"``[-max_shear_ratio, max_shear_ratio]``\");\n    DMLC_DECLARE_FIELD(max_crop_size)\n        .set_default(-1)\n        .describe(\n            \"Crop both width and height into a random size in \"\n            \"``[min_crop_size, max_crop_size].``\"\n            \"Ignored if ``random_resized_crop`` is True.\");\n    DMLC_DECLARE_FIELD(min_crop_size)\n        .set_default(-1)\n        .describe(\n            \"Crop both width and height into a random size in \"\n            \"``[min_crop_size, max_crop_size].``\"\n            \"Ignored if ``random_resized_crop`` is True.\");\n    DMLC_DECLARE_FIELD(max_random_scale)\n        .set_default(1.0f)\n        .describe(\n            \"Resize into ``[width*s, height*s]`` with ``s`` randomly\"\n            \" chosen from ``[min_random_scale, max_random_scale]``. \"\n            \"Ignored if ``random_resized_crop`` is True.\");\n    DMLC_DECLARE_FIELD(min_random_scale)\n        .set_default(1.0f)\n        .describe(\n            \"Resize into ``[width*s, height*s]`` with ``s`` randomly\"\n            \" chosen from ``[min_random_scale, max_random_scale]``\"\n            \"Ignored if ``random_resized_crop`` is True.\");\n    DMLC_DECLARE_FIELD(max_random_area)\n        .set_default(1.0f)\n        .describe(\n            \"Change the area (namely width * height) to a random value \"\n            \"in ``[min_random_area, max_random_area]``. \"\n            \"Ignored if ``random_resized_crop`` is False.\");\n    DMLC_DECLARE_FIELD(min_random_area)\n        .set_default(1.0f)\n        .describe(\n            \"Change the area (namely width * height) to a random value \"\n            \"in ``[min_random_area, max_random_area]``. \"\n            \"Ignored if ``random_resized_crop`` is False.\");\n    DMLC_DECLARE_FIELD(max_img_size)\n        .set_default(1e10f)\n        .describe(\n            \"Set the maximal width and height after all resize and\"\n            \" rotate argumentation  are applied\");\n    DMLC_DECLARE_FIELD(min_img_size)\n        .set_default(0.0f)\n        .describe(\n            \"Set the minimal width and height after all resize and\"\n            \" rotate argumentation  are applied\");\n    DMLC_DECLARE_FIELD(brightness)\n        .set_default(0.0f)\n        .describe(\n            \"Add a random value in ``[-brightness, brightness]`` to \"\n            \"the brightness of image.\");\n    DMLC_DECLARE_FIELD(contrast).set_default(0.0f).describe(\n        \"Add a random value in ``[-contrast, contrast]`` to \"\n        \"the contrast of image.\");\n    DMLC_DECLARE_FIELD(saturation)\n        .set_default(0.0f)\n        .describe(\n            \"Add a random value in ``[-saturation, saturation]`` to \"\n            \"the saturation of image.\");\n    DMLC_DECLARE_FIELD(pca_noise).set_default(0.0f).describe(\"Add PCA based noise to the image.\");\n    DMLC_DECLARE_FIELD(random_h).set_default(0).describe(\n        \"Add a random value in ``[-random_h, random_h]`` to \"\n        \"the H channel in HSL color space.\");\n    DMLC_DECLARE_FIELD(random_s).set_default(0).describe(\n        \"Add a random value in ``[-random_s, random_s]`` to \"\n        \"the S channel in HSL color space.\");\n    DMLC_DECLARE_FIELD(random_l).set_default(0).describe(\n        \"Add a random value in ``[-random_l, random_l]`` to \"\n        \"the L channel in HSL color space.\");\n    DMLC_DECLARE_FIELD(rotate).set_default(-1.0f).describe(\n        \"Rotate by an angle. If set, it overwrites the ``max_rotate_angle`` option.\");\n    DMLC_DECLARE_FIELD(fill_value)\n        .set_default(255)\n        .describe(\"Set the padding pixels value to ``fill_value``.\");\n    DMLC_DECLARE_FIELD(data_shape)\n        .set_expect_ndim(3)\n        .enforce_nonzero()\n        .describe(\"The shape of a output image.\");\n    DMLC_DECLARE_FIELD(inter_method)\n        .set_default(1)\n        .describe(\n            \"The interpolation method: 0-NN 1-bilinear 2-cubic 3-area \"\n            \"4-lanczos4 9-auto 10-rand.\");\n    DMLC_DECLARE_FIELD(pad).set_default(0).describe(\n        \"Change size from ``[width, height]`` into \"\n        \"``[pad + width + pad, pad + height + pad]`` by padding pixes\");\n  }\n};\n\nDMLC_REGISTER_PARAMETER(DefaultImageAugmentParam);\n\nstd::vector<dmlc::ParamFieldInfo> ListDefaultAugParams() {\n  return DefaultImageAugmentParam::__FIELDS__();\n}\n\n#if MXNET_USE_OPENCV\n\n#ifdef _MSC_VER\n#define M_PI CV_PI\n#endif\n/*! \\brief helper class to do image augmentation */\nclass DefaultImageAugmenter : public ImageAugmenter {\n public:\n  // contructor\n  DefaultImageAugmenter() = default;\n  void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override {\n    std::vector<std::pair<std::string, std::string> > kwargs_left;\n    kwargs_left = param_.InitAllowUnknown(kwargs);\n    for (auto& kwarg : kwargs_left) {\n      if (!strcmp(kwarg.first.c_str(), \"rotate_list\")) {\n        const char* val = kwarg.second.c_str();\n        const char* end = val + strlen(val);\n        char buf[128];\n        while (val < end) {\n          sscanf(val, \"%[^,]\", buf);\n          val += strlen(buf) + 1;\n          rotate_list_.push_back(atoi(buf));\n        }\n      }\n    }\n  }\n  /*!\n   * \\brief get interpolation method with given inter_method, 0-CV_INTER_NN 1-CV_INTER_LINEAR\n   * 2-CV_INTER_CUBIC \\ 3-CV_INTER_AREA 4-CV_INTER_LANCZOS4 9-AUTO(cubic for enlarge, area for\n   * shrink, bilinear for others) 10-RAND\n   */\n  int GetInterMethod(int inter_method,\n                     int old_width,\n                     int old_height,\n                     int new_width,\n                     int new_height,\n                     common::RANDOM_ENGINE* prnd) {\n    if (inter_method == 9) {\n      if (new_width > old_width && new_height > old_height) {\n        return 2;  // CV_INTER_CUBIC for enlarge\n      } else if (new_width < old_width && new_height < old_height) {\n        return 3;  // CV_INTER_AREA for shrink\n      } else {\n        return 1;  // CV_INTER_LINEAR for others\n      }\n    } else if (inter_method == 10) {\n      std::uniform_int_distribution<size_t> rand_uniform_int(0, 4);\n      return rand_uniform_int(*prnd);\n    } else {\n      return inter_method;\n    }\n  }\n  cv::Mat Process(const cv::Mat& src,\n                  std::vector<float>* label,\n                  common::RANDOM_ENGINE* prnd) override {\n    using mshadow::index_t;\n    bool is_cropped = false;\n\n    float max_aspect_ratio = 1.0f;\n    float min_aspect_ratio = 1.0f;\n    if (param_.min_aspect_ratio.has_value()) {\n      max_aspect_ratio = param_.max_aspect_ratio;\n      min_aspect_ratio = param_.min_aspect_ratio.value();\n    } else {\n      max_aspect_ratio = 1 + param_.max_aspect_ratio;\n      min_aspect_ratio = 1 - param_.max_aspect_ratio;\n    }\n\n    cv::Mat res;\n    if (param_.resize != -1) {\n      int new_height, new_width;\n      if (src.rows > src.cols) {\n        new_height = param_.resize * src.rows / src.cols;\n        new_width  = param_.resize;\n      } else {\n        new_height = param_.resize;\n        new_width  = param_.resize * src.cols / src.rows;\n      }\n      CHECK((param_.inter_method >= 0 && param_.inter_method <= 4) ||\n            (param_.inter_method >= 9 && param_.inter_method <= 10))\n          << \"invalid inter_method: valid value 0,1,2,3,4,9,10\";\n      int interpolation_method =\n          GetInterMethod(param_.inter_method, src.cols, src.rows, new_width, new_height, prnd);\n      cv::resize(src, res, cv::Size(new_width, new_height), 0, 0, interpolation_method);\n    } else {\n      res = src;\n    }\n\n    // normal augmentation by affine transformation.\n    if (param_.max_rotate_angle > 0 || param_.max_shear_ratio > 0.0f || param_.rotate > 0 ||\n        rotate_list_.size() > 0 || param_.max_random_scale != 1.0f ||\n        param_.min_random_scale != 1.0 ||\n        (!param_.random_resized_crop && (min_aspect_ratio != 1.0f || max_aspect_ratio != 1.0f)) ||\n        param_.max_img_size != 1e10f || param_.min_img_size != 0.0f) {\n      std::uniform_real_distribution<float> rand_uniform(0, 1);\n      // shear\n      float s = rand_uniform(*prnd) * param_.max_shear_ratio * 2 - param_.max_shear_ratio;\n      // rotate\n      int angle = std::uniform_int_distribution<int>(-param_.max_rotate_angle,\n                                                     param_.max_rotate_angle)(*prnd);\n      if (param_.rotate > 0)\n        angle = param_.rotate;\n      if (rotate_list_.size() > 0) {\n        angle = rotate_list_[std::uniform_int_distribution<int>(0, rotate_list_.size() - 1)(*prnd)];\n      }\n      float a = cos(angle / 180.0 * M_PI);\n      float b = sin(angle / 180.0 * M_PI);\n      // scale\n      float scale = 1.0f;\n      if (!param_.random_resized_crop) {\n        scale = rand_uniform(*prnd) * (param_.max_random_scale - param_.min_random_scale) +\n                param_.min_random_scale;\n      }\n      // aspect ratio\n      float ratio = 1.0f;\n      if (!param_.random_resized_crop) {\n        ratio = rand_uniform(*prnd) * (max_aspect_ratio - min_aspect_ratio) + min_aspect_ratio;\n      }\n      float hs = 2 * scale / (1 + ratio);\n      float ws = ratio * hs;\n      // new width and height\n      float new_width =\n          std::max(param_.min_img_size, std::min(param_.max_img_size, scale * res.cols));\n      float new_height =\n          std::max(param_.min_img_size, std::min(param_.max_img_size, scale * res.rows));\n      cv::Mat M(2, 3, CV_32F);\n      M.at<float>(0, 0)       = hs * a - s * b * ws;\n      M.at<float>(1, 0)       = -b * ws;\n      M.at<float>(0, 1)       = hs * b + s * a * ws;\n      M.at<float>(1, 1)       = a * ws;\n      float ori_center_width  = M.at<float>(0, 0) * res.cols + M.at<float>(0, 1) * res.rows;\n      float ori_center_height = M.at<float>(1, 0) * res.cols + M.at<float>(1, 1) * res.rows;\n      M.at<float>(0, 2)       = (new_width - ori_center_width) / 2;\n      M.at<float>(1, 2)       = (new_height - ori_center_height) / 2;\n      CHECK((param_.inter_method >= 0 && param_.inter_method <= 4) ||\n            (param_.inter_method >= 9 && param_.inter_method <= 10))\n          << \"invalid inter_method: valid value 0,1,2,3,4,9,10\";\n      int interpolation_method =\n          GetInterMethod(param_.inter_method, res.cols, res.rows, new_width, new_height, prnd);\n      cv::warpAffine(res,\n                     temp_,\n                     M,\n                     cv::Size(new_width, new_height),\n                     interpolation_method,\n                     cv::BORDER_CONSTANT,\n                     cv::Scalar(param_.fill_value, param_.fill_value, param_.fill_value));\n      res = temp_;\n    }\n\n    // pad logic\n    if (param_.pad > 0) {\n      cv::copyMakeBorder(res,\n                         res,\n                         param_.pad,\n                         param_.pad,\n                         param_.pad,\n                         param_.pad,\n                         cv::BORDER_CONSTANT,\n                         cv::Scalar(param_.fill_value, param_.fill_value, param_.fill_value));\n    }\n\n    if (param_.random_resized_crop) {\n      // random resize crop\n      CHECK(param_.min_random_scale == 1.0f && param_.max_random_scale == 1.0f &&\n            param_.min_crop_size == -1 && param_.max_crop_size == -1 && !param_.rand_crop)\n          << \"\\nSetting random_resized_crop to true conflicts with \"\n             \"min_random_scale, max_random_scale, \"\n             \"min_crop_size, max_crop_size, \"\n             \"and rand_crop.\";\n\n      if (param_.max_random_area != 1.0f || param_.min_random_area != 1.0f ||\n          max_aspect_ratio != 1.0f || min_aspect_ratio != 1.0f) {\n        CHECK(min_aspect_ratio > 0.0f);\n        CHECK(param_.min_random_area <= param_.max_random_area);\n        CHECK(min_aspect_ratio <= max_aspect_ratio);\n        std::uniform_real_distribution<float> rand_uniform_area(param_.min_random_area,\n                                                                param_.max_random_area);\n        std::uniform_real_distribution<float> rand_uniform_ratio(min_aspect_ratio,\n                                                                 max_aspect_ratio);\n        std::uniform_real_distribution<float> rand_uniform(0, 1);\n        float area = res.rows * res.cols;\n        for (int i = 0; i < 10; ++i) {\n          float rand_area   = rand_uniform_area(*prnd);\n          float ratio       = rand_uniform_ratio(*prnd);\n          float target_area = area * rand_area;\n          int y_area        = std::round(std::sqrt(target_area / ratio));\n          int x_area        = std::round(std::sqrt(target_area * ratio));\n          if (rand_uniform(*prnd) > 0.5) {\n            float temp_y_area = y_area;\n            y_area            = x_area;\n            x_area            = temp_y_area;\n          }\n          if (y_area <= res.rows && x_area <= res.cols) {\n            index_t rand_y_area =\n                std::uniform_int_distribution<index_t>(0, res.rows - y_area)(*prnd);\n            index_t rand_x_area =\n                std::uniform_int_distribution<index_t>(0, res.cols - x_area)(*prnd);\n            cv::Rect roi(rand_x_area, rand_y_area, x_area, y_area);\n            int interpolation_method = GetInterMethod(param_.inter_method,\n                                                      x_area,\n                                                      y_area,\n                                                      param_.data_shape[2],\n                                                      param_.data_shape[1],\n                                                      prnd);\n            cv::resize(res(roi),\n                       res,\n                       cv::Size(param_.data_shape[2], param_.data_shape[1]),\n                       0,\n                       0,\n                       interpolation_method);\n            is_cropped = true;\n            break;\n          }\n        }\n      }\n    } else if (!param_.random_resized_crop &&\n               (param_.max_crop_size != -1 || param_.min_crop_size != -1)) {\n      // random_crop\n      CHECK(res.cols >= param_.max_crop_size && res.rows >= param_.max_crop_size &&\n            param_.max_crop_size >= param_.min_crop_size)\n          << \"input image size smaller than max_crop_size\";\n      index_t rand_crop_size =\n          std::uniform_int_distribution<index_t>(param_.min_crop_size, param_.max_crop_size)(*prnd);\n      index_t y = res.rows - rand_crop_size;\n      index_t x = res.cols - rand_crop_size;\n      if (param_.rand_crop != 0) {\n        y = std::uniform_int_distribution<index_t>(0, y)(*prnd);\n        x = std::uniform_int_distribution<index_t>(0, x)(*prnd);\n      } else {\n        y /= 2;\n        x /= 2;\n      }\n      cv::Rect roi(x, y, rand_crop_size, rand_crop_size);\n      int interpolation_method = GetInterMethod(param_.inter_method,\n                                                rand_crop_size,\n                                                rand_crop_size,\n                                                param_.data_shape[2],\n                                                param_.data_shape[1],\n                                                prnd);\n      cv::resize(res(roi),\n                 res,\n                 cv::Size(param_.data_shape[2], param_.data_shape[1]),\n                 0,\n                 0,\n                 interpolation_method);\n      is_cropped = true;\n    }\n\n    if (!is_cropped) {\n      // center crop\n      int interpolation_method = GetInterMethod(param_.inter_method,\n                                                res.cols,\n                                                res.rows,\n                                                param_.data_shape[2],\n                                                param_.data_shape[1],\n                                                prnd);\n      if (res.rows < param_.data_shape[1]) {\n        index_t new_cols =\n            static_cast<index_t>(static_cast<float>(param_.data_shape[1]) /\n                                 static_cast<float>(res.rows) * static_cast<float>(res.cols));\n        cv::resize(res, res, cv::Size(new_cols, param_.data_shape[1]), 0, 0, interpolation_method);\n      }\n      if (res.cols < param_.data_shape[2]) {\n        index_t new_rows =\n            static_cast<index_t>(static_cast<float>(param_.data_shape[2]) /\n                                 static_cast<float>(res.cols) * static_cast<float>(res.rows));\n        cv::resize(res, res, cv::Size(param_.data_shape[2], new_rows), 0, 0, interpolation_method);\n      }\n      CHECK(static_cast<index_t>(res.rows) >= param_.data_shape[1] &&\n            static_cast<index_t>(res.cols) >= param_.data_shape[2])\n          << \"input image size smaller than input shape\";\n      index_t y = res.rows - param_.data_shape[1];\n      index_t x = res.cols - param_.data_shape[2];\n      if (param_.rand_crop != 0) {\n        y = std::uniform_int_distribution<index_t>(0, y)(*prnd);\n        x = std::uniform_int_distribution<index_t>(0, x)(*prnd);\n      } else {\n        y /= 2;\n        x /= 2;\n      }\n      cv::Rect roi(x, y, param_.data_shape[2], param_.data_shape[1]);\n      res = res(roi);\n    }\n\n    // color jitter\n    if (param_.brightness > 0.0f || param_.contrast > 0.0f || param_.saturation > 0.0f) {\n      std::uniform_real_distribution<float> rand_uniform(0, 1);\n      float alpha_b =\n          1.0 + std::uniform_real_distribution<float>(-param_.brightness, param_.brightness)(*prnd);\n      float alpha_c =\n          1.0 + std::uniform_real_distribution<float>(-param_.contrast, param_.contrast)(*prnd);\n      float alpha_s =\n          1.0 + std::uniform_real_distribution<float>(-param_.saturation, param_.saturation)(*prnd);\n      int rand_order[3] = {0, 1, 2};\n      std::shuffle(std::begin(rand_order), std::end(rand_order), *prnd);\n      for (int i : rand_order) {\n        if (i == 0) {\n          // brightness\n          res.convertTo(res, -1, alpha_b, 0);\n        }\n        if (i == 1) {\n          // contrast\n          cvtColor(res, temp_, CV_RGB2GRAY);\n          float gray_mean = cv::mean(temp_)[0];\n          res.convertTo(res, -1, alpha_c, (1 - alpha_c) * gray_mean);\n        }\n        if (i == 2) {\n          // saturation\n          cvtColor(res, temp_, CV_RGB2GRAY);\n          cvtColor(temp_, temp_, CV_GRAY2BGR);\n          cv::addWeighted(res, alpha_s, temp_, 1 - alpha_s, 0.0, res);\n        }\n      }\n    }\n\n    // color space augmentation\n    if (param_.random_h != 0 || param_.random_s != 0 || param_.random_l != 0) {\n      std::uniform_real_distribution<float> rand_uniform(0, 1);\n      cvtColor(res, res, CV_BGR2HLS);\n      // use an approximation of gaussian distribution to reduce extreme value\n      float rh = rand_uniform(*prnd);\n      rh += 4 * rand_uniform(*prnd);\n      rh       = rh / 5;\n      float rs = rand_uniform(*prnd);\n      rs += 4 * rand_uniform(*prnd);\n      rs       = rs / 5;\n      float rl = rand_uniform(*prnd);\n      rl += 4 * rand_uniform(*prnd);\n      rl           = rl / 5;\n      int h        = rh * param_.random_h * 2 - param_.random_h;\n      int s        = rs * param_.random_s * 2 - param_.random_s;\n      int l        = rl * param_.random_l * 2 - param_.random_l;\n      int temp[3]  = {h, l, s};\n      int limit[3] = {180, 255, 255};\n      for (int i = 0; i < res.rows; ++i) {\n        for (int j = 0; j < res.cols; ++j) {\n          for (int k = 0; k < 3; ++k) {\n            int v = res.at<cv::Vec3b>(i, j)[k];\n            v += temp[k];\n            v                          = std::max(0, std::min(limit[k], v));\n            res.at<cv::Vec3b>(i, j)[k] = v;\n          }\n        }\n      }\n      cvtColor(res, res, CV_HLS2BGR);\n    }\n\n    // pca noise\n    if (param_.pca_noise > 0.0f) {\n      std::normal_distribution<float> rand_normal(0, param_.pca_noise);\n      float pca_alpha_r = rand_normal(*prnd);\n      float pca_alpha_g = rand_normal(*prnd);\n      float pca_alpha_b = rand_normal(*prnd);\n      float pca_r =\n          eigvec[0][0] * pca_alpha_r + eigvec[0][1] * pca_alpha_g + eigvec[0][2] * pca_alpha_b;\n      float pca_g =\n          eigvec[1][0] * pca_alpha_r + eigvec[1][1] * pca_alpha_g + eigvec[1][2] * pca_alpha_b;\n      float pca_b =\n          eigvec[2][0] * pca_alpha_r + eigvec[2][1] * pca_alpha_g + eigvec[2][2] * pca_alpha_b;\n      float pca[3] = {pca_b, pca_g, pca_r};\n      for (int i = 0; i < res.rows; ++i) {\n        for (int j = 0; j < res.cols; ++j) {\n          for (int k = 0; k < 3; ++k) {\n            int vp = res.at<cv::Vec3b>(i, j)[k];\n            vp += pca[k];\n            vp                         = std::max(0, std::min(255, vp));\n            res.at<cv::Vec3b>(i, j)[k] = vp;\n          }\n        }\n      }\n    }\n    return res;\n  }\n\n private:\n  // temporal space\n  cv::Mat temp_;\n  // eigval and eigvec for adding pca noise\n  // store eigval * eigvec as eigvec\n  float eigvec[3][3] = {{55.46f * -0.5675f, 4.794f * 0.7192f, 1.148f * 0.4009f},\n                        {55.46f * -0.5808f, 4.794f * -0.0045f, 1.148f * -0.8140f},\n                        {55.46f * -0.5836f, 4.794f * -0.6948f, 1.148f * 0.4203f}};\n  // parameters\n  DefaultImageAugmentParam param_;\n  /*! \\brief list of possible rotate angle */\n  std::vector<int> rotate_list_;\n};\n\nImageAugmenter* ImageAugmenter::Create(const std::string& name) {\n  return dmlc::Registry<ImageAugmenterReg>::Find(name)->body();\n}\n\nMXNET_REGISTER_IMAGE_AUGMENTER(aug_default).describe(\"default augmenter\").set_body([]() {\n  return new DefaultImageAugmenter();\n});\n#endif  // MXNET_USE_OPENCV\n}  // namespace io\n}  // namespace mxnet\n"
  },
  {
    "path": "src/io/image_augmenter.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file image_augmenter.h\n * \\brief Interface of opencv based image augmenter\n */\n#ifndef MXNET_IO_IMAGE_AUGMENTER_H_\n#define MXNET_IO_IMAGE_AUGMENTER_H_\n\n#include <dmlc/registry.h>\n\n#if MXNET_USE_OPENCV\n#include <opencv2/opencv.hpp>\n#include <vector>   // NOLINT(*)\n#include <utility>  // NOLINT(*)\n#include <string>   // NOLINT(*)\n\n#include \"../common/utils.h\"\n\nnamespace mxnet {\nnamespace io {\n/*!\n * \\brief OpenCV based Image augmenter,\n *  The augmenter can contain internal temp state.\n */\nclass ImageAugmenter {\n public:\n  /*!\n   *  \\brief Initialize the Operator by setting the parameters\n   *  This function need to be called before all other functions.\n   *  \\param kwargs the keyword arguments parameters\n   */\n  virtual void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) = 0;\n  /*!\n   * \\brief augment src image.\n   *   this function is not thread safe, and will only be called by one thread\n   *   however, it will tries to re-use memory space as much as possible\n   * \\param src the source image\n   * \\param prnd pointer to random number generator.\n   * \\return The processed image.\n   */\n  virtual cv::Mat Process(const cv::Mat& src,\n                          std::vector<float>* label,\n                          common::RANDOM_ENGINE* prnd) = 0;\n  // virtual destructor\n  virtual ~ImageAugmenter() {}\n  /*!\n   * \\brief factory function\n   * \\param name Name of the augmenter\n   * \\return The created augmenter.\n   */\n  static ImageAugmenter* Create(const std::string& name);\n};\n\n/*! \\brief typedef the factory function of data iterator */\ntypedef std::function<ImageAugmenter*()> ImageAugmenterFactory;\n/*!\n * \\brief Registry entry for DataIterator factory functions.\n */\nstruct ImageAugmenterReg\n    : public dmlc::FunctionRegEntryBase<ImageAugmenterReg, ImageAugmenterFactory> {};\n//--------------------------------------------------------------\n// The following part are API Registration of Iterators\n//--------------------------------------------------------------\n/*!\n * \\brief Macro to register image augmenter\n *\n * \\code\n * // example of registering a mnist iterator\n * REGISTER_IMAGE_AUGMENTER(aug_default)\n * .describe(\"default augmenter\")\n * .set_body([]() {\n *     return new DefaultAugmenter();\n *   });\n * \\endcode\n */\n#define MXNET_REGISTER_IMAGE_AUGMENTER(name) \\\n  DMLC_REGISTRY_REGISTER(::mxnet::io::ImageAugmenterReg, ImageAugmenterReg, name)\n}  // namespace io\n}  // namespace mxnet\n#endif  // MXNET_USE_OPENCV\n\nnamespace mxnet {\nnamespace io {\n/*! \\return the parameter of default augmenter */\nstd::vector<dmlc::ParamFieldInfo> ListDefaultAugParams();\nstd::vector<dmlc::ParamFieldInfo> ListDefaultDetAugParams();\n}  // namespace io\n}  // namespace mxnet\n#endif  // MXNET_IO_IMAGE_AUGMENTER_H_\n"
  },
  {
    "path": "src/io/image_det_aug_default.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file image_det_aug_default.cc\n * \\brief Default augmenter.\n */\n#include <mxnet/base.h>\n#include <utility>\n#include <string>\n#include <algorithm>\n#include <vector>\n#include <cmath>\n#include \"./image_augmenter.h\"\n#include \"../common/utils.h\"\n\nnamespace mxnet {\nnamespace io {\n\nusing mxnet::Tuple;\n\nnamespace image_det_aug_default_enum {\nenum ImageDetAugDefaultCropEmitMode { kCenter, kOverlap };\nenum ImageDetAugDefaultResizeMode { kForce, kShrink, kFit };\n}  // namespace image_det_aug_default_enum\n\n/*! \\brief image detection augmentation parameters*/\nstruct DefaultImageDetAugmentParam : public dmlc::Parameter<DefaultImageDetAugmentParam> {\n  /*! \\brief resize shorter edge to size before applying other augmentations */\n  int resize;\n  /*! \\brief probability we do random cropping, use prob <= 0 to disable */\n  float rand_crop_prob;\n  /*! \\brief min crop scales */\n  Tuple<float> min_crop_scales;\n  /*! \\brief max crop scales */\n  Tuple<float> max_crop_scales;\n  /*! \\brief min crop aspect ratios */\n  Tuple<float> min_crop_aspect_ratios;\n  /*! \\brief max crop aspect ratios */\n  Tuple<float> max_crop_aspect_ratios;\n  /*! \\brief min IOUs between ground-truths and crop boxes */\n  Tuple<float> min_crop_overlaps;\n  /*! \\brief max IOUs between ground-truths and crop boxes */\n  Tuple<float> max_crop_overlaps;\n  /*! \\brief min itersection/gt_area between ground-truths and crop boxes */\n  Tuple<float> min_crop_sample_coverages;\n  /*! \\brief max itersection/gt_area between ground-truths and crop boxes */\n  Tuple<float> max_crop_sample_coverages;\n  /*! \\brief min itersection/crop_area between ground-truths and crop boxes */\n  Tuple<float> min_crop_object_coverages;\n  /*! \\brief max itersection/crop_area between ground-truths and crop boxes */\n  Tuple<float> max_crop_object_coverages;\n  /*! \\brief number of crop samplers, skip random crop if <= 0 */\n  int num_crop_sampler;\n  /*! \\beief 0-emit ground-truth if center out of crop area\n   * 1-emit if overlap < emit_overlap_thresh\n   */\n  int crop_emit_mode;\n  /*! \\brief ground-truth emition threshold specific for crop_emit_mode == 1 */\n  float emit_overlap_thresh;\n  /*! \\brief maximum trials for cropping, skip cropping if fails exceed this number */\n  Tuple<int> max_crop_trials;\n  /*! \\brief random padding prob */\n  float rand_pad_prob;\n  /*!< \\brief maximum padding scale */\n  float max_pad_scale;\n  /*! \\brief max random in H channel */\n  int max_random_hue;\n  /*! \\brief random H prob */\n  float random_hue_prob;\n  /*! \\brief max random in S channel */\n  int max_random_saturation;\n  /*! \\brief random saturation prob */\n  float random_saturation_prob;\n  /*! \\brief max random in L channel */\n  int max_random_illumination;\n  /*! \\brief random illumination change prob */\n  float random_illumination_prob;\n  /*! \\brief max random contrast */\n  float max_random_contrast;\n  /*! \\brief random contrast prob */\n  float random_contrast_prob;\n  /*! \\brief random mirror prob */\n  float rand_mirror_prob;\n  /*! \\brief filled color while padding */\n  int fill_value;\n  /*! \\brief interpolation method 0-NN 1-bilinear 2-cubic 3-area 4-lanczos4 9-auto 10-rand  */\n  int inter_method;\n  /*! \\brief shape of the image data */\n  mxnet::TShape data_shape;\n  /*! \\brief resize mode, 0-force\n   * 1-Shrink to data_shape, preserve ratio,\n   * 2-fit to data_shape, preserve ratio\n   */\n  int resize_mode;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(DefaultImageDetAugmentParam) {\n    DMLC_DECLARE_FIELD(resize).set_default(-1).describe(\n        \"Augmentation Param: scale shorter edge to size \"\n        \"before applying other augmentations, -1 to disable.\");\n    DMLC_DECLARE_FIELD(rand_crop_prob)\n        .set_default(0.0f)\n        .describe(\"Augmentation Param: Probability of random cropping, <= 0 to disable\");\n    DMLC_DECLARE_FIELD(min_crop_scales)\n        .set_default(Tuple<float>({0.0f}))\n        .describe(\"Augmentation Param: Min crop scales.\");\n    DMLC_DECLARE_FIELD(max_crop_scales)\n        .set_default(Tuple<float>({1.0f}))\n        .describe(\"Augmentation Param: Max crop scales.\");\n    DMLC_DECLARE_FIELD(min_crop_aspect_ratios)\n        .set_default(Tuple<float>({1.0f}))\n        .describe(\"Augmentation Param: Min crop aspect ratios.\");\n    DMLC_DECLARE_FIELD(max_crop_aspect_ratios)\n        .set_default(Tuple<float>({1.0f}))\n        .describe(\"Augmentation Param: Max crop aspect ratios.\");\n    DMLC_DECLARE_FIELD(min_crop_overlaps)\n        .set_default(Tuple<float>({0.0f}))\n        .describe(\"Augmentation Param: Minimum crop IOU between crop_box and ground-truths.\");\n    DMLC_DECLARE_FIELD(max_crop_overlaps)\n        .set_default(Tuple<float>({1.0f}))\n        .describe(\"Augmentation Param: Maximum crop IOU between crop_box and ground-truth.\");\n    DMLC_DECLARE_FIELD(min_crop_sample_coverages)\n        .set_default(Tuple<float>({0.0f}))\n        .describe(\n            \"Augmentation Param: Minimum ratio of intersect/crop_area \"\n            \"between crop box and ground-truths.\");\n    DMLC_DECLARE_FIELD(max_crop_sample_coverages)\n        .set_default(Tuple<float>({1.0f}))\n        .describe(\n            \"Augmentation Param: Maximum ratio of intersect/crop_area \"\n            \"between crop box and ground-truths.\");\n    DMLC_DECLARE_FIELD(min_crop_object_coverages)\n        .set_default(Tuple<float>({0.0f}))\n        .describe(\n            \"Augmentation Param: Minimum ratio of intersect/gt_area \"\n            \"between crop box and ground-truths.\");\n    DMLC_DECLARE_FIELD(max_crop_object_coverages)\n        .set_default(Tuple<float>({1.0f}))\n        .describe(\n            \"Augmentation Param: Maximum ratio of intersect/gt_area \"\n            \"between crop box and ground-truths.\");\n    DMLC_DECLARE_FIELD(num_crop_sampler)\n        .set_default(1)\n        .describe(\"Augmentation Param: Number of crop samplers.\");\n    DMLC_DECLARE_FIELD(crop_emit_mode)\n        .add_enum(\"center\", image_det_aug_default_enum::kCenter)\n        .add_enum(\"overlap\", image_det_aug_default_enum::kOverlap)\n        .set_default(image_det_aug_default_enum::kCenter)\n        .describe(\n            \"Augmentation Param: Emition mode for invalid ground-truths after crop. \"\n            \"center: emit if centroid of object is out of crop region; \"\n            \"overlap: emit if overlap is less than emit_overlap_thresh. \");\n    DMLC_DECLARE_FIELD(emit_overlap_thresh)\n        .set_default(0.3f)\n        .describe(\"Augmentation Param: Emit overlap thresh for emit mode overlap only.\");\n    DMLC_DECLARE_FIELD(max_crop_trials)\n        .set_default(Tuple<int>({25}))\n        .describe(\n            \"Augmentation Param: Skip cropping if fail crop trail count \"\n            \"exceeds this number.\");\n    DMLC_DECLARE_FIELD(rand_pad_prob)\n        .set_default(0.0f)\n        .describe(\"Augmentation Param: Probability for random padding.\");\n    DMLC_DECLARE_FIELD(max_pad_scale)\n        .set_default(1.0f)\n        .describe(\"Augmentation Param: Maximum padding scale.\");\n    DMLC_DECLARE_FIELD(max_random_hue)\n        .set_default(0)\n        .describe(\"Augmentation Param: Maximum random value of H channel in HSL color space.\");\n    DMLC_DECLARE_FIELD(random_hue_prob)\n        .set_default(0.0f)\n        .describe(\"Augmentation Param: Probability to apply random hue.\");\n    DMLC_DECLARE_FIELD(max_random_saturation)\n        .set_default(0)\n        .describe(\"Augmentation Param: Maximum random value of S channel in HSL color space.\");\n    DMLC_DECLARE_FIELD(random_saturation_prob)\n        .set_default(0.0f)\n        .describe(\"Augmentation Param: Probability to apply random saturation.\");\n    DMLC_DECLARE_FIELD(max_random_illumination)\n        .set_default(0)\n        .describe(\"Augmentation Param: Maximum random value of L channel in HSL color space.\");\n    DMLC_DECLARE_FIELD(random_illumination_prob)\n        .set_default(0.0f)\n        .describe(\"Augmentation Param: Probability to apply random illumination.\");\n    DMLC_DECLARE_FIELD(max_random_contrast)\n        .set_default(0)\n        .describe(\"Augmentation Param: Maximum random value of delta contrast.\");\n    DMLC_DECLARE_FIELD(random_contrast_prob)\n        .set_default(0.0f)\n        .describe(\"Augmentation Param: Probability to apply random contrast.\");\n    DMLC_DECLARE_FIELD(rand_mirror_prob)\n        .set_default(0.0f)\n        .describe(\"Augmentation Param: Probability to apply horizontal flip aka. mirror.\");\n    DMLC_DECLARE_FIELD(fill_value)\n        .set_default(127)\n        .describe(\"Augmentation Param: Filled color value while padding.\");\n    DMLC_DECLARE_FIELD(inter_method)\n        .set_default(1)\n        .describe(\"Augmentation Param: 0-NN 1-bilinear 2-cubic 3-area 4-lanczos4 9-auto 10-rand.\");\n    DMLC_DECLARE_FIELD(data_shape)\n        .set_expect_ndim(3)\n        .enforce_nonzero()\n        .describe(\"Dataset Param: Shape of each instance generated by the DataIter.\");\n    DMLC_DECLARE_FIELD(resize_mode)\n        .add_enum(\"force\", image_det_aug_default_enum::kForce)\n        .add_enum(\"shrink\", image_det_aug_default_enum::kShrink)\n        .add_enum(\"fit\", image_det_aug_default_enum::kFit)\n        .set_default(image_det_aug_default_enum::kForce)\n        .describe(\n            \"Augmentation Param: How image data fit in data_shape. \"\n            \"force: force reshape to data_shape regardless of aspect ratio; \"\n            \"shrink: ensure each side fit in data_shape, preserve aspect ratio; \"\n            \"fit: fit image to data_shape, preserve ratio, will upscale if applicable.\");\n  }\n};\n\nDMLC_REGISTER_PARAMETER(DefaultImageDetAugmentParam);\n\nstd::vector<dmlc::ParamFieldInfo> ListDefaultDetAugParams() {\n  return DefaultImageDetAugmentParam::__FIELDS__();\n}\n\n#if MXNET_USE_OPENCV\n#include \"./opencv_compatibility.h\"\nusing Rect = cv::Rect_<float>;\n\n#ifdef _MSC_VER\n#define M_PI CV_PI\n#endif\n\n/*! \\brief helper class for better detection label handling */\nclass ImageDetLabel {\n public:\n  /*! \\brief Helper struct to store the coordinates and id for each object */\n  struct ImageDetObject {\n    float id;\n    float left;\n    float top;\n    float right;\n    float bottom;\n    std::vector<float> extra;  // store extra info other than id and coordinates\n\n    /*! \\brief Return converted Rect object */\n    Rect ToRect() const {\n      return Rect(left, top, right - left, bottom - top);\n    }\n\n    /*! \\brief Return projected coordinates according to new region */\n    ImageDetObject Project(Rect box) const {\n      ImageDetObject ret = *this;\n      ret.left           = std::max(0.f, (ret.left - box.x) / box.width);\n      ret.top            = std::max(0.f, (ret.top - box.y) / box.height);\n      ret.right          = std::min(1.f, (ret.right - box.x) / box.width);\n      ret.bottom         = std::min(1.f, (ret.bottom - box.y) / box.height);\n      return ret;\n    }\n\n    /*! \\brief Return Horizontally fliped coordinates */\n    ImageDetObject HorizontalFlip() const {\n      ImageDetObject ret = *this;\n      ret.left           = 1.f - this->right;\n      ret.right          = 1.f - this->left;\n      return ret;\n    }\n  };  // struct ImageDetObject\n\n  /*! \\brief constructor from raw array of detection labels */\n  explicit ImageDetLabel(const std::vector<float>& raw_label) {\n    FromArray(raw_label);\n  }\n\n  /*! \\brief construct from raw array with following format\n   * header_width, object_width, (extra_headers...),\n   * [id, xmin, ymin, xmax, ymax, (extra_object_info)] x N\n   */\n  void FromArray(const std::vector<float>& raw_label) {\n    int label_width = static_cast<int>(raw_label.size());\n    CHECK_GE(label_width, 7);  // at least 2(header) + 5(1 object)\n    int header_width = static_cast<int>(raw_label[0]);\n    CHECK_GE(header_width, 2);\n    object_width_ = static_cast<int>(raw_label[1]);\n    CHECK_GE(object_width_, 5);  // id, x1, y1, x2, y2...\n    header_.assign(raw_label.begin(), raw_label.begin() + header_width);\n    int num = (label_width - header_width) / object_width_;\n    CHECK_EQ((label_width - header_width) % object_width_, 0);\n    objects_.reserve(num);\n    for (int i = header_width; i < label_width; i += object_width_) {\n      ImageDetObject obj;\n      auto it    = raw_label.cbegin() + i;\n      obj.id     = *(it++);\n      obj.left   = *(it++);\n      obj.top    = *(it++);\n      obj.right  = *(it++);\n      obj.bottom = *(it++);\n      obj.extra.assign(it, it - 5 + object_width_);\n      if (obj.right > obj.left && obj.bottom > obj.top) {\n        objects_.push_back(obj);\n      }\n    }\n  }\n\n  /*! \\brief Convert back to raw array */\n  std::vector<float> ToArray() const {\n    std::vector<float> out(header_);\n    out.reserve(out.size() + objects_.size() * object_width_);\n    for (auto& obj : objects_) {\n      out.push_back(obj.id);\n      out.push_back(obj.left);\n      out.push_back(obj.top);\n      out.push_back(obj.right);\n      out.push_back(obj.bottom);\n      out.insert(out.end(), obj.extra.begin(), obj.extra.end());\n    }\n    return out;\n  }\n\n  /*! \\brief Intersection over Union between two rects */\n  static float RectIOU(Rect a, Rect b) {\n    float intersect = (a & b).area();\n    if (intersect <= 0.f)\n      return 0.f;\n    return intersect / (a.area() + b.area() - intersect);\n  }\n\n  /*! \\brief try crop image with given crop_box\n   * return false if fail to meet any of the constraints\n   * convert all objects if success\n   */\n  bool TryCrop(const Rect crop_box,\n               const float min_crop_overlap,\n               const float max_crop_overlap,\n               const float min_crop_sample_coverage,\n               const float max_crop_sample_coverage,\n               const float min_crop_object_coverage,\n               const float max_crop_object_coverage,\n               const int crop_emit_mode,\n               const float emit_overlap_thresh) {\n    if (objects_.size() < 1) {\n      return true;  // no object, raise error or just skip?\n    }\n    // check if crop_box valid\n    bool valid = false;\n    if (min_crop_overlap > 0.f || max_crop_overlap < 1.f || min_crop_sample_coverage > 0.f ||\n        max_crop_sample_coverage < 1.f || min_crop_object_coverage > 0.f ||\n        max_crop_object_coverage < 1.f) {\n      for (auto& obj : objects_) {\n        Rect gt_box = obj.ToRect();\n        if (min_crop_overlap > 0.f || max_crop_overlap < 1.f) {\n          float ovp = RectIOU(crop_box, gt_box);\n          if (ovp < min_crop_overlap || ovp > max_crop_overlap) {\n            continue;\n          }\n        }\n        if (min_crop_sample_coverage > 0.f || max_crop_sample_coverage < 1.f) {\n          float c = (crop_box & gt_box).area() / crop_box.area();\n          if (c < min_crop_sample_coverage || c > max_crop_sample_coverage) {\n            continue;\n          }\n        }\n        if (min_crop_object_coverage > 0.f || max_crop_object_coverage < 1.f) {\n          float c = (crop_box & gt_box).area() / gt_box.area();\n          if (c < min_crop_object_coverage || c > max_crop_object_coverage) {\n            continue;\n          }\n        }\n        valid = true;\n        break;\n      }\n    } else {\n      valid = true;\n    }\n\n    if (!valid)\n      return false;\n    // transform ground-truth labels\n    std::vector<ImageDetObject> new_objects;\n    for (auto& object : objects_) {\n      if (image_det_aug_default_enum::kCenter == crop_emit_mode) {\n        float center_x = (object.left + object.right) * 0.5f;\n        float center_y = (object.top + object.bottom) * 0.5f;\n        if (!crop_box.contains(cv::Point2f(center_x, center_y))) {\n          continue;\n        }\n        new_objects.push_back(object.Project(crop_box));\n      } else if (image_det_aug_default_enum::kOverlap == crop_emit_mode) {\n        Rect gt_box   = object.ToRect();\n        float overlap = (crop_box & gt_box).area() / gt_box.area();\n        if (overlap > emit_overlap_thresh) {\n          new_objects.push_back(object.Project(crop_box));\n        }\n      }\n    }\n    if (new_objects.size() < 1)\n      return false;\n    objects_ = new_objects;  // replace the old objects\n    return true;\n  }\n\n  /*! \\brief try pad image with given pad_box\n   * convert all objects afterwards\n   */\n  bool TryPad(const Rect pad_box) {\n    // update all objects inplace\n    for (auto& object : objects_) {\n      object = object.Project(pad_box);\n    }\n    return true;\n  }\n\n  /*! \\brief flip image and object coordinates horizontally */\n  bool TryMirror() {\n    // flip all objects horizontally\n    for (auto& object : objects_) {\n      object = object.HorizontalFlip();\n    }\n    return true;\n  }\n\n private:\n  /*! \\brief width for each object information, 5 at least */\n  int object_width_;\n  /*! \\brief vector to store original header info */\n  std::vector<float> header_;\n  /*! \\brief storing objects in more convenient formats */\n  std::vector<ImageDetObject> objects_;\n};  // class ImageDetLabel\n\n/*! \\brief helper class to do image augmentation */\nclass DefaultImageDetAugmenter : public ImageAugmenter {\n public:\n  // contructor\n  DefaultImageDetAugmenter() = default;\n\n  void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override {\n    std::vector<std::pair<std::string, std::string> > kwargs_left;\n    kwargs_left = param_.InitAllowUnknown(kwargs);\n\n    CHECK((param_.inter_method >= 0 && param_.inter_method <= 4) ||\n          (param_.inter_method >= 9 && param_.inter_method <= 10))\n        << \"invalid inter_method: valid value 0,1,2,3,9,10\";\n\n    // validate crop parameters\n    ValidateCropParameters(&param_.min_crop_scales, param_.num_crop_sampler);\n    ValidateCropParameters(&param_.max_crop_scales, param_.num_crop_sampler);\n    ValidateCropParameters(&param_.min_crop_aspect_ratios, param_.num_crop_sampler);\n    ValidateCropParameters(&param_.max_crop_aspect_ratios, param_.num_crop_sampler);\n    ValidateCropParameters(&param_.min_crop_overlaps, param_.num_crop_sampler);\n    ValidateCropParameters(&param_.max_crop_overlaps, param_.num_crop_sampler);\n    ValidateCropParameters(&param_.min_crop_sample_coverages, param_.num_crop_sampler);\n    ValidateCropParameters(&param_.max_crop_sample_coverages, param_.num_crop_sampler);\n    ValidateCropParameters(&param_.min_crop_object_coverages, param_.num_crop_sampler);\n    ValidateCropParameters(&param_.max_crop_object_coverages, param_.num_crop_sampler);\n    ValidateCropParameters(&param_.max_crop_trials, param_.num_crop_sampler);\n    for (int i = 0; i < param_.num_crop_sampler; ++i) {\n      CHECK_GE(param_.min_crop_scales[i], 0.0f);\n      CHECK_LE(param_.max_crop_scales[i], 1.0f);\n      CHECK_GT(param_.max_crop_scales[i], param_.min_crop_scales[i]);\n      CHECK_GE(param_.min_crop_aspect_ratios[i], 0.0f);\n      CHECK_GE(param_.max_crop_aspect_ratios[i], param_.min_crop_aspect_ratios[i]);\n      CHECK_GE(param_.max_crop_overlaps[i], param_.min_crop_overlaps[i]);\n      CHECK_GE(param_.max_crop_sample_coverages[i], param_.min_crop_sample_coverages[i]);\n      CHECK_GE(param_.max_crop_object_coverages[i], param_.min_crop_object_coverages[i]);\n    }\n    CHECK_GE(param_.emit_overlap_thresh, 0.0f);\n  }\n  /*!\n   * \\brief get interpolation method with given inter_method, 0-CV_INTER_NN 1-CV_INTER_LINEAR\n   * 2-CV_INTER_CUBIC \\ 3-CV_INTER_AREA 4-CV_INTER_LANCZOS4 9-AUTO(cubic for enlarge, area for\n   * shrink, bilinear for others) 10-RAND\n   */\n  int GetInterMethod(int inter_method,\n                     int old_width,\n                     int old_height,\n                     int new_width,\n                     int new_height,\n                     common::RANDOM_ENGINE* prnd) {\n    if (inter_method == 9) {\n      if (new_width > old_width && new_height > old_height) {\n        return 2;  // CV_INTER_CUBIC for enlarge\n      } else if (new_width < old_width && new_height < old_height) {\n        return 3;  // CV_INTER_AREA for shrink\n      } else {\n        return 1;  // CV_INTER_LINEAR for others\n      }\n    } else if (inter_method == 10) {\n      std::uniform_int_distribution<size_t> rand_uniform_int(0, 4);\n      return rand_uniform_int(*prnd);\n    } else {\n      return inter_method;\n    }\n  }\n\n  /*! \\brief Check number of crop samplers and given parameters */\n  template <typename DType>\n  void ValidateCropParameters(mxnet::Tuple<DType>* param, const int num_sampler) {\n    if (num_sampler == 1) {\n      CHECK_EQ(param->ndim(), 1);\n    } else if (num_sampler > 1) {\n      if (param->ndim() == 1) {\n        std::vector<DType> vec(num_sampler, (*param)[0]);\n        param->assign(vec.begin(), vec.end());\n      } else {\n        CHECK_EQ(param->ndim(), num_sampler) << \"# of parameters/crop_samplers mismatch \";\n      }\n    }\n  }\n\n  /*! \\brief Generate crop box region given cropping parameters */\n  Rect GenerateCropBox(const float min_crop_scale,\n                       const float max_crop_scale,\n                       const float min_crop_aspect_ratio,\n                       const float max_crop_aspect_ratio,\n                       common::RANDOM_ENGINE* prnd,\n                       const float img_aspect_ratio) {\n    float new_scale =\n        std::uniform_real_distribution<float>(min_crop_scale, max_crop_scale)(*prnd) + 1e-12f;\n    float min_ratio =\n        std::max<float>(min_crop_aspect_ratio / img_aspect_ratio, new_scale * new_scale);\n    float max_ratio =\n        std::min<float>(max_crop_aspect_ratio / img_aspect_ratio, 1. / (new_scale * new_scale));\n    float new_ratio = std::sqrt(std::uniform_real_distribution<float>(min_ratio, max_ratio)(*prnd));\n    float new_width = std::min(1.f, new_scale * new_ratio);\n    float new_height = std::min(1.f, new_scale / new_ratio);\n    float x0         = std::uniform_real_distribution<float>(0.f, 1 - new_width)(*prnd);\n    float y0         = std::uniform_real_distribution<float>(0.f, 1 - new_height)(*prnd);\n    return Rect(x0, y0, new_width, new_height);\n  }\n\n  /*! \\brief Generate padding box region given padding parameters */\n  Rect GeneratePadBox(const float max_pad_scale,\n                      common::RANDOM_ENGINE* prnd,\n                      const float threshold = 1.05f) {\n    float new_scale = std::uniform_real_distribution<float>(1.f, max_pad_scale)(*prnd);\n    if (new_scale < threshold)\n      return Rect(0, 0, 0, 0);\n    auto rand_uniform = std::uniform_real_distribution<float>(0.f, new_scale - 1);\n    float x0          = rand_uniform(*prnd);\n    float y0          = rand_uniform(*prnd);\n    return Rect(-x0, -y0, new_scale, new_scale);\n  }\n\n  cv::Mat Process(const cv::Mat& src,\n                  std::vector<float>* label,\n                  common::RANDOM_ENGINE* prnd) override {\n    using mshadow::index_t;\n    cv::Mat res;\n    if (param_.resize != -1) {\n      int new_height, new_width;\n      if (src.rows > src.cols) {\n        new_height = param_.resize * src.rows / src.cols;\n        new_width  = param_.resize;\n      } else {\n        new_height = param_.resize;\n        new_width  = param_.resize * src.cols / src.rows;\n      }\n      int interpolation_method =\n          GetInterMethod(param_.inter_method, src.cols, src.rows, new_width, new_height, prnd);\n      cv::resize(src, res, cv::Size(new_width, new_height), 0, 0, interpolation_method);\n    } else {\n      res = src;\n    }\n\n    // build a helper class for processing labels\n    ImageDetLabel det_label(*label);\n    // random engine\n    std::uniform_real_distribution<float> rand_uniform(0, 1);\n\n    // color space augmentation\n    if (param_.random_hue_prob > 0.f || param_.random_saturation_prob > 0.f ||\n        param_.random_illumination_prob > 0.f || param_.random_contrast_prob > 0.f) {\n      std::uniform_real_distribution<float> uniform_range(-1.f, 1.f);\n      int h   = uniform_range(*prnd) * param_.max_random_hue;\n      int s   = uniform_range(*prnd) * param_.max_random_saturation;\n      int l   = uniform_range(*prnd) * param_.max_random_illumination;\n      float c = uniform_range(*prnd) * param_.max_random_contrast;\n      h       = rand_uniform(*prnd) < param_.random_hue_prob ? h : 0;\n      s       = rand_uniform(*prnd) < param_.random_saturation_prob ? s : 0;\n      l       = rand_uniform(*prnd) < param_.random_illumination_prob ? l : 0;\n      c       = rand_uniform(*prnd) < param_.random_contrast_prob ? c : 0;\n      if (h != 0 || s != 0 || l != 0) {\n        int temp[3]  = {h, l, s};\n        int limit[3] = {180, 255, 255};\n        cv::cvtColor(res, res, CV_BGR2HLS);\n        for (int i = 0; i < res.rows; ++i) {\n          for (int j = 0; j < res.cols; ++j) {\n            for (int k = 0; k < 3; ++k) {\n              int v = res.at<cv::Vec3b>(i, j)[k];\n              v += temp[k];\n              v                          = std::max(0, std::min(limit[k], v));\n              res.at<cv::Vec3b>(i, j)[k] = v;\n            }\n          }\n        }\n        cv::cvtColor(res, res, CV_HLS2BGR);\n      }\n      if (std::fabs(c) > 1e-3) {\n        cv::Mat tmp = res;\n        tmp.convertTo(res, -1, c + 1.f, 0);\n      }\n    }\n\n    // random mirror logic\n    if (param_.rand_mirror_prob > 0 && rand_uniform(*prnd) < param_.rand_mirror_prob) {\n      if (det_label.TryMirror()) {\n        // flip image\n        cv::flip(res, temp_, 1);\n        res = temp_;\n      }\n    }\n\n    // random padding logic\n    if (param_.rand_pad_prob > 0 && param_.max_pad_scale > 1.f) {\n      if (rand_uniform(*prnd) < param_.rand_pad_prob) {\n        Rect pad_box = GeneratePadBox(param_.max_pad_scale, prnd);\n        if (pad_box.area() > 0) {\n          if (det_label.TryPad(pad_box)) {\n            // pad image\n            temp_     = res;\n            int left  = static_cast<int>(-pad_box.x * res.cols);\n            int top   = static_cast<int>(-pad_box.y * res.rows);\n            int right = static_cast<int>((pad_box.width + pad_box.x - 1) * res.cols);\n            int bot   = static_cast<int>((pad_box.height + pad_box.y - 1) * res.rows);\n            cv::copyMakeBorder(temp_,\n                               res,\n                               top,\n                               bot,\n                               left,\n                               right,\n                               cv::BORDER_ISOLATED,\n                               cv::Scalar(param_.fill_value, param_.fill_value, param_.fill_value));\n          }\n        }\n      }\n    }\n\n    // random crop logic\n    if (param_.rand_crop_prob > 0 && param_.num_crop_sampler > 0) {\n      if (rand_uniform(*prnd) < param_.rand_crop_prob) {\n        // random crop sampling logic: randomly pick a sampler, return if success\n        // continue to next sampler if failed(exceed max_trial)\n        // return original sample if every sampler has failed\n        std::vector<int> indices(param_.num_crop_sampler);\n        for (int i = 0; i < param_.num_crop_sampler; ++i) {\n          indices[i] = i;\n        }\n        std::shuffle(indices.begin(), indices.end(), *prnd);\n        int num_processed = 0;\n        for (auto idx : indices) {\n          if (num_processed > 0)\n            break;\n          for (int t = 0; t < param_.max_crop_trials[idx]; ++t) {\n            Rect crop_box = GenerateCropBox(param_.min_crop_scales[idx],\n                                            param_.max_crop_scales[idx],\n                                            param_.min_crop_aspect_ratios[idx],\n                                            param_.max_crop_aspect_ratios[idx],\n                                            prnd,\n                                            static_cast<float>(res.cols) / res.rows);\n            if (det_label.TryCrop(crop_box,\n                                  param_.min_crop_overlaps[idx],\n                                  param_.max_crop_overlaps[idx],\n                                  param_.min_crop_sample_coverages[idx],\n                                  param_.max_crop_sample_coverages[idx],\n                                  param_.min_crop_object_coverages[idx],\n                                  param_.max_crop_object_coverages[idx],\n                                  param_.crop_emit_mode,\n                                  param_.emit_overlap_thresh)) {\n              ++num_processed;\n              // crop image\n              int left   = static_cast<int>(crop_box.x * res.cols);\n              int top    = static_cast<int>(crop_box.y * res.rows);\n              int width  = static_cast<int>(crop_box.width * res.cols);\n              int height = static_cast<int>(crop_box.height * res.rows);\n              res        = res(cv::Rect(left, top, width, height));\n              break;\n            }\n          }\n        }\n      }\n    }\n\n    if (image_det_aug_default_enum::kForce == param_.resize_mode) {\n      // force resize to specified data_shape, regardless of aspect ratio\n      int new_height = param_.data_shape[1];\n      int new_width  = param_.data_shape[2];\n      int interpolation_method =\n          GetInterMethod(param_.inter_method, res.cols, res.rows, new_width, new_height, prnd);\n      cv::resize(res, res, cv::Size(new_width, new_height), 0, 0, interpolation_method);\n    } else if (image_det_aug_default_enum::kShrink == param_.resize_mode) {\n      // try to keep original size, shrink if too large\n      float h = param_.data_shape[1];\n      float w = param_.data_shape[2];\n      if (res.rows > h || res.cols > w) {\n        float ratio    = std::min(h / res.rows, w / res.cols);\n        int new_height = ratio * res.rows;\n        int new_width  = ratio * res.cols;\n        int interpolation_method =\n            GetInterMethod(param_.inter_method, res.cols, res.rows, new_width, new_height, prnd);\n        cv::resize(res, res, cv::Size(new_width, new_height), 0, 0, interpolation_method);\n      }\n    } else if (image_det_aug_default_enum::kFit == param_.resize_mode) {\n      float h        = param_.data_shape[1];\n      float w        = param_.data_shape[2];\n      float ratio    = std::min(h / res.rows, w / res.cols);\n      int new_height = ratio * res.rows;\n      int new_width  = ratio * res.cols;\n      int interpolation_method =\n          GetInterMethod(param_.inter_method, res.cols, res.rows, new_width, new_height, prnd);\n      cv::resize(res, res, cv::Size(new_width, new_height), 0, 0, interpolation_method);\n    }\n\n    *label = det_label.ToArray();  // put back processed labels\n    return res;\n  }\n\n private:\n  // temporal space\n  cv::Mat temp_;\n  // parameters\n  DefaultImageDetAugmentParam param_;\n};\n\nMXNET_REGISTER_IMAGE_AUGMENTER(det_aug_default)\n    .describe(\"default detection augmenter\")\n    .set_body([]() { return new DefaultImageDetAugmenter(); });\n#endif  // MXNET_USE_OPENCV\n}  // namespace io\n}  // namespace mxnet\n"
  },
  {
    "path": "src/io/image_io.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file optimizer_op-inl.h\n * \\brief Optimizer operators\n * \\author Junyuan Xie\n */\n#include <dmlc/parameter.h>\n#include <dmlc/logging.h>\n#include <mxnet/ndarray.h>\n#include <mxnet/operator.h>\n#include <mxnet/operator_util.h>\n#include <mxnet/op_attr_types.h>\n#include <mshadow/base.h>\n#include <nnvm/op.h>\n#include <nnvm/op_attr_types.h>\n\n#include <fstream>\n#include <cstring>\n\n#include \"../operator/elemwise_op_common.h\"\n#include \"../operator/image/resize-inl.h\"\n\n#if MXNET_USE_OPENCV\n#include <opencv2/opencv.hpp>\n#include \"./opencv_compatibility.h\"\n#endif  // MXNET_USE_OPENCV\n\nnamespace mxnet {\nnamespace io {\n\n// http://www.64lines.com/jpeg-width-height\n// Gets the JPEG size from the array of data passed to the function,\n// file reference: http://www.obrador.com/essentialjpeg/headerinfo.htm\nbool get_jpeg_size(const uint8_t* data, uint32_t data_size, int64_t* width, int64_t* height) {\n  // Check for valid JPEG image\n  uint32_t i = 0;  // Keeps track of the position within the file\n  if (data[i] == 0xFF && data[i + 1] == 0xD8 && data[i + 2] == 0xFF && data[i + 3] == 0xE0) {\n    i += 4;\n    // Check for valid JPEG header (null terminated JFIF)\n    if (data[i + 2] == 'J' && data[i + 3] == 'F' && data[i + 4] == 'I' && data[i + 5] == 'F' &&\n        data[i + 6] == 0x00) {\n      // Retrieve the block length of the first block since\n      // the first block will not contain the size of file\n      uint16_t block_length = data[i] * 256 + data[i + 1];\n      while (i < data_size) {\n        i += block_length;  // Increase the file index to get to the next block\n        if (i >= data_size)\n          return false;  // Check to protect against segmentation faults\n        if (data[i] != 0xFF)\n          return false;  // Check that we are truly at the start of another block\n        uint8_t m = data[i + 1];\n        if (m == 0xC0 || (m >= 0xC1 && m <= 0xCF && m != 0xC4 && m != 0xC8 && m != 0xCC)) {\n          // 0xFFC0 is the \"Start of frame\" marker which contains the file size\n          // The structure of the 0xFFC0 block is quite simple\n          // [0xFFC0][ushort length][uchar precision][ushort x][ushort y]\n          *height = data[i + 5] * 256 + data[i + 6];\n          *width  = data[i + 7] * 256 + data[i + 8];\n          return true;\n        } else {\n          i += 2;                                      // Skip the block marker\n          block_length = data[i] * 256 + data[i + 1];  // Go to the next block\n        }\n      }\n      return false;  // If this point is reached then no size was found\n    } else {\n      return false;  // Not a valid JFIF string\n    }\n  } else {\n    return false;  // Not a valid SOI header\n  }\n}\n\nbool get_png_size(const uint8_t* data, uint32_t data_size, int64_t* width, int64_t* height) {\n  if (data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E && data[3] == 0x47) {\n    uint8_t const* p = data + 16;\n    *width           = ((p[0] * 256 + p[1]) * 256 + p[2]) * 256 + p[3];\n    p += 4;\n    *height = ((p[0] * 256 + p[1]) * 256 + p[2]) * 256 + p[3];\n    return true;\n  } else {\n    return false;\n  }\n}\n\nstruct ImdecodeParam : public dmlc::Parameter<ImdecodeParam> {\n  int flag;\n  bool to_rgb;\n  DMLC_DECLARE_PARAMETER(ImdecodeParam) {\n    DMLC_DECLARE_FIELD(flag).set_lower_bound(0).set_default(1).describe(\n        \"Convert decoded image to grayscale (0) or color (1).\");\n    DMLC_DECLARE_FIELD(to_rgb).set_default(true).describe(\n        \"Whether to convert decoded image to mxnet's default RGB format \"\n        \"(instead of opencv's default BGR).\");\n  }\n};\n\nDMLC_REGISTER_PARAMETER(ImdecodeParam);\n\nstruct ImreadParam : public dmlc::Parameter<ImreadParam> {\n  std::string filename;\n  int flag;\n  bool to_rgb;\n  DMLC_DECLARE_PARAMETER(ImreadParam) {\n    DMLC_DECLARE_FIELD(filename).describe(\"Name of the image file to be loaded.\");\n    DMLC_DECLARE_FIELD(flag).set_lower_bound(0).set_default(1).describe(\n        \"Convert decoded image to grayscale (0) or color (1).\");\n    DMLC_DECLARE_FIELD(to_rgb).set_default(true).describe(\n        \"Whether to convert decoded image to mxnet's default RGB format \"\n        \"(instead of opencv's default BGR).\");\n  }\n};\n\nDMLC_REGISTER_PARAMETER(ImreadParam);\n\n#if MXNET_USE_OPENCV\nvoid ImdecodeImpl(int flag, bool to_rgb, void* data, size_t size, NDArray* out) {\n  cv::Mat buf(1, size, CV_8U, data);\n  cv::Mat dst;\n  if (out->is_none()) {\n    cv::Mat res = cv::imdecode(buf, flag);\n    CHECK(!res.empty()) << \"Decoding failed. Invalid image file.\";\n\n    *out = NDArray(mshadow::Shape3(res.rows, res.cols, flag == 0 ? 1 : 3),\n                   Context::CPU(),\n                   false,\n                   mshadow::kUint8);\n    dst = cv::Mat(out->shape()[0], out->shape()[1], flag == 0 ? CV_8U : CV_8UC3, out->data().dptr_);\n    res.copyTo(dst);\n    CHECK(!dst.empty()) << \"Failed copying buffer to output.\";\n  } else {\n    dst = cv::Mat(out->shape()[0], out->shape()[1], flag == 0 ? CV_8U : CV_8UC3, out->data().dptr_);\n#if (CV_MAJOR_VERSION > 3 || (CV_MAJOR_VERSION == 3 && CV_MINOR_VERSION >= 3))\n    cv::imdecode(buf, flag | cv::IMREAD_IGNORE_ORIENTATION, &dst);\n    CHECK(!dst.empty()) << \"Decoding failed. Invalid image file.\";\n#elif (CV_MAJOR_VERSION > 2 || (CV_MAJOR_VERSION == 2 && CV_MINOR_VERSION >= 4))  // NOLINT\n    cv::imdecode(buf, flag, &dst);\n    CHECK(!dst.empty()) << \"Decoding failed. Invalid image file.\";\n#else\n    cv::Mat tmp = cv::imdecode(buf, flag);\n    CHECK(!tmp.empty()) << \"Decoding failed. Invalid image file.\";\n    tmp.copyTo(dst);\n    CHECK(!dst.empty()) << \"Failed copying buffer to output.\";\n#endif\n  }\n  CHECK_EQ(static_cast<void*>(dst.ptr()), out->data().dptr_);\n  if (to_rgb && flag != 0) {\n    cv::cvtColor(dst, dst, CV_BGR2RGB);\n  }\n}\n#endif  // MXNET_USE_OPENCV\n\nvoid Imdecode(const nnvm::NodeAttrs& attrs,\n              const std::vector<NDArray>& inputs,\n              std::vector<NDArray>* outputs) {\n#if MXNET_USE_OPENCV\n  const auto& param = nnvm::get<ImdecodeParam>(attrs.parsed);\n\n  CHECK_EQ(inputs[0].ctx().dev_mask(), Context::kCPU) << \"Only supports cpu input\";\n  CHECK_EQ(inputs[0].dtype(), mshadow::kUint8) << \"Input needs to be uint8 buffer\";\n  inputs[0].WaitToRead();\n\n  uint8_t* str_img = inputs[0].data().dptr<uint8_t>();\n  size_t len       = inputs[0].shape().Size();\n  CHECK(len > 0) << \"Input cannot be an empty buffer\";\n\n  mxnet::TShape oshape(3, 1);\n  oshape[2] = param.flag == 0 ? 1 : 3;\n  if (get_jpeg_size(str_img, len, &oshape[1], &oshape[0])) {\n  } else if (get_png_size(str_img, len, &oshape[1], &oshape[0])) {\n  } else {\n    (*outputs)[0] = NDArray();\n    ImdecodeImpl(param.flag, param.to_rgb, str_img, len, &((*outputs)[0]));\n    return;\n  }\n\n  const NDArray& ndin = inputs[0];\n  NDArray& ndout      = (*outputs)[0];\n  ndout               = NDArray(oshape, Context::CPU(), true, mshadow::kUint8);\n  Engine::Get()->PushSync(\n      [ndin, ndout, str_img, len, param](RunContext ctx) {\n        ImdecodeImpl(param.flag, param.to_rgb, str_img, len, const_cast<NDArray*>(&ndout));\n      },\n      ndout.ctx(),\n      {ndin.var()},\n      {ndout.var()},\n      FnProperty::kNormal,\n      0,\n      \"Imdecode\");\n#else\n  LOG(FATAL) << \"Build with USE_OPENCV=1 for image io.\";\n#endif  // MXNET_USE_OPENCV\n}\n\nvoid Imread(const nnvm::NodeAttrs& attrs,\n            const std::vector<NDArray>& inputs,\n            std::vector<NDArray>* outputs) {\n#if MXNET_USE_OPENCV\n  const auto& param = nnvm::get<ImreadParam>(attrs.parsed);\n\n  std::ifstream file(param.filename, std::ios::binary | std::ios::ate);\n  // if file is not open we get bad alloc after tellg\n  CHECK(file.is_open()) << \"Imread: '\" << param.filename\n                        << \"' couldn't open file: \" << strerror(errno);\n  size_t fsize = file.tellg();\n  file.seekg(0, std::ios::beg);\n  std::shared_ptr<uint8_t> buff(new uint8_t[fsize], std::default_delete<uint8_t[]>());\n  file.read(reinterpret_cast<char*>(buff.get()), fsize);\n  CHECK(file.good()) << \"Failed reading image file: '\" << param.filename << \"' \" << strerror(errno);\n\n  mxnet::TShape oshape(3, 1);\n  oshape[2] = param.flag == 0 ? 1 : 3;\n  if (get_jpeg_size(buff.get(), fsize, &oshape[1], &oshape[0])) {\n  } else if (get_png_size(buff.get(), fsize, &oshape[1], &oshape[0])) {\n  } else {\n    (*outputs)[0] = NDArray();\n    ImdecodeImpl(param.flag, param.to_rgb, buff.get(), fsize, &((*outputs)[0]));\n    return;\n  }\n\n  NDArray& ndout = (*outputs)[0];\n  ndout          = NDArray(oshape, Context::CPU(), true, mshadow::kUint8);\n  Engine::Get()->PushSync(\n      [ndout, buff, fsize, param](RunContext ctx) {\n        ImdecodeImpl(param.flag, param.to_rgb, buff.get(), fsize, const_cast<NDArray*>(&ndout));\n      },\n      ndout.ctx(),\n      {},\n      {ndout.var()},\n      FnProperty::kNormal,\n      0,\n      \"Imread\");\n#else\n  LOG(FATAL) << \"Build with USE_OPENCV=1 for image io.\";\n#endif  // MXNET_USE_OPENCV\n}\n\nstruct ResizeParam : public dmlc::Parameter<ResizeParam> {\n  int w;\n  int h;\n  int interp;\n  DMLC_DECLARE_PARAMETER(ResizeParam) {\n    DMLC_DECLARE_FIELD(w).set_lower_bound(1).describe(\"Width of resized image.\");\n    DMLC_DECLARE_FIELD(h).set_lower_bound(1).describe(\"Height of resized image.\");\n    DMLC_DECLARE_FIELD(interp).set_default(1).describe(\n        \"Interpolation method (default=cv2.INTER_LINEAR).\");\n  }\n};\nDMLC_REGISTER_PARAMETER(ResizeParam);\n\ninline bool ResizeShape(const nnvm::NodeAttrs& attrs,\n                        mxnet::ShapeVector* ishape,\n                        mxnet::ShapeVector* oshape) {\n  const auto& param = nnvm::get<ResizeParam>(attrs.parsed);\n  if (ishape->size() != 1 || (*ishape)[0].ndim() != 3)\n    return false;\n\n  oshape->clear();\n  oshape->push_back(mshadow::Shape3(param.h, param.w, (*ishape)[0][2]));\n  return true;\n}\n\ninline void Imresize(const nnvm::NodeAttrs& attrs,\n                     const OpContext& ctx,\n                     const std::vector<TBlob>& inputs,\n                     const std::vector<OpReqType>& req,\n                     const std::vector<TBlob>& outputs) {\n  const auto& param = nnvm::get<ResizeParam>(attrs.parsed);\n  op::image::ResizeImpl(inputs, outputs, param.h, param.w, param.interp);\n}\n\nstruct MakeBorderParam : public dmlc::Parameter<MakeBorderParam> {\n  int top, bot, left, right;\n  int type;\n  double value;\n  mxnet::Tuple<double> values;\n  DMLC_DECLARE_PARAMETER(MakeBorderParam) {\n    DMLC_DECLARE_FIELD(top).describe(\"Top margin.\");\n    DMLC_DECLARE_FIELD(bot).describe(\"Bottom margin.\");\n    DMLC_DECLARE_FIELD(left).describe(\"Left margin.\");\n    DMLC_DECLARE_FIELD(right).describe(\"Right margin.\");\n    DMLC_DECLARE_FIELD(type).set_default(0).describe(\"Filling type (default=cv2.BORDER_CONSTANT).\");\n    DMLC_DECLARE_FIELD(value).set_default(0.0).describe(\n        \"(Deprecated! Use ``values`` instead.) Fill with single value.\");\n    DMLC_DECLARE_FIELD(values).set_default({}).describe(\n        \"Fill with value(RGB[A] or gray), up to 4 channels.\");\n  }\n};\nDMLC_REGISTER_PARAMETER(MakeBorderParam);\n\ninline bool MakeBorderShape(const nnvm::NodeAttrs& attrs,\n                            mxnet::ShapeVector* ishape,\n                            mxnet::ShapeVector* oshape) {\n  const auto& param = nnvm::get<MakeBorderParam>(attrs.parsed);\n  if (ishape->size() != 1 || (*ishape)[0].ndim() != 3)\n    return false;\n\n  oshape->clear();\n  oshape->push_back(mshadow::Shape3((*ishape)[0][0] + param.top + param.bot,\n                                    (*ishape)[0][1] + param.left + param.right,\n                                    (*ishape)[0][2]));\n  return true;\n}\n\ninline void copyMakeBorder(const nnvm::NodeAttrs& attrs,\n                           const OpContext& ctx,\n                           const std::vector<TBlob>& inputs,\n                           const std::vector<OpReqType>& req,\n                           const std::vector<TBlob>& outputs) {\n#if MXNET_USE_OPENCV\n  CHECK_NE(inputs[0].type_flag_, mshadow::kFloat16) << \"imresize doesn't support fp16\";\n  const int DTYPE[] = {CV_32F, CV_64F, -1, CV_8U, CV_32S};\n  int cv_type       = CV_MAKETYPE(DTYPE[inputs[0].type_flag_], inputs[0].shape_[2]);\n  const auto& param = nnvm::get<MakeBorderParam>(attrs.parsed);\n  cv::Mat buf(inputs[0].shape_[0], inputs[0].shape_[1], cv_type, inputs[0].dptr_);\n  cv::Mat dst(outputs[0].shape_[0], outputs[0].shape_[1], cv_type, outputs[0].dptr_);\n  cv::Scalar color(param.value, param.value, param.value);\n  if (param.values.ndim() > 0) {\n    color = cv::Scalar(cv::Vec<double, 4>(param.values.begin()));\n  }\n  cv::copyMakeBorder(buf, dst, param.top, param.bot, param.left, param.right, param.type, color);\n  CHECK(!dst.empty());\n  CHECK_EQ(static_cast<void*>(dst.ptr()), outputs[0].dptr_);\n#else\n  LOG(FATAL) << \"Build with USE_OPENCV=1 for image io.\";\n#endif  // MXNET_USE_OPENCV\n}\n\nNNVM_REGISTER_OP(_cvimdecode)\n    .add_alias(\"_npi_cvimdecode\")\n    .describe(\n        \"Decode image with OpenCV. \\n\"\n        \"Note: return image in RGB by default, \"\n        \"instead of OpenCV's default BGR.\")\n    .set_num_inputs(1)\n    .set_num_outputs(1)\n    .set_attr_parser(op::ParamParser<ImdecodeParam>)\n    .set_attr<FNDArrayFunction>(\"FNDArrayFunction\", Imdecode)\n    .add_argument(\"buf\", \"NDArray\", \"Buffer containing binary encoded image\")\n    .add_arguments(ImdecodeParam::__FIELDS__());\n\nNNVM_REGISTER_OP(_cvimread)\n    .add_alias(\"_npi_cvimread\")\n    .describe(\n        \"Read and decode image with OpenCV. \\n\"\n        \"Note: return image in RGB by default, \"\n        \"instead of OpenCV's default BGR.\")\n    .set_num_inputs(0)\n    .set_num_outputs(1)\n    .set_attr_parser(op::ParamParser<ImreadParam>)\n    .set_attr<FNDArrayFunction>(\"FNDArrayFunction\", Imread)\n    .add_arguments(ImreadParam::__FIELDS__());\n\nNNVM_REGISTER_OP(_cvimresize)\n    .add_alias(\"_npi_cvimresize\")\n    .describe(\"Resize image with OpenCV. \\n\")\n    .set_num_inputs(1)\n    .set_num_outputs(1)\n    .set_attr_parser(op::ParamParser<ResizeParam>)\n    .set_attr<mxnet::FInferShape>(\"FInferShape\", ResizeShape)\n    .set_attr<nnvm::FInferType>(\"FInferType\", op::ElemwiseType<1, 1>)\n    .set_attr<FCompute>(\"FCompute<cpu>\", Imresize)\n    .add_argument(\"src\", \"NDArray\", \"source image\")\n    .add_arguments(ResizeParam::__FIELDS__());\n\nNNVM_REGISTER_OP(_cvcopyMakeBorder)\n    .describe(\"Pad image border with OpenCV. \\n\")\n    .set_num_inputs(1)\n    .set_num_outputs(1)\n    .set_attr_parser(op::ParamParser<MakeBorderParam>)\n    .set_attr<mxnet::FInferShape>(\"FInferShape\", MakeBorderShape)\n    .set_attr<nnvm::FInferType>(\"FInferType\", op::ElemwiseType<1, 1>)\n    .set_attr<FCompute>(\"FCompute<cpu>\", copyMakeBorder)\n    .add_argument(\"src\", \"NDArray\", \"source image\")\n    .add_arguments(MakeBorderParam::__FIELDS__());\n\n}  // namespace io\n}  // namespace mxnet\n"
  },
  {
    "path": "src/io/image_iter_common.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file image_iter_common.h\n * \\brief common types used by image data iterators\n */\n\n#ifndef MXNET_IO_IMAGE_ITER_COMMON_H_\n#define MXNET_IO_IMAGE_ITER_COMMON_H_\n\n#include <mxnet/io.h>\n#include <vector>\n#include <unordered_map>\n#include <string>\n\nnamespace mxnet {\nnamespace io {\n/*! \\brief data structure to hold labels for images */\nclass ImageLabelMap {\n public:\n  /*!\n   * \\brief initialize the label list into memory\n   * \\param path_imglist path to the image list\n   * \\param label_width predefined label_width\n   */\n  explicit ImageLabelMap(const char* path_imglist, index_t label_width, bool silent) {\n    this->label_width = label_width;\n    image_index_.clear();\n    label_.clear();\n    idx2label_.clear();\n    dmlc::InputSplit* fi = dmlc::InputSplit::Create(path_imglist, 0, 1, \"text\");\n    dmlc::InputSplit::Blob rec;\n    while (fi->NextRecord(&rec)) {\n      // quick manual parsing\n      char* p   = reinterpret_cast<char*>(rec.dptr);\n      char* end = p + rec.size;\n      // skip space\n      while (isspace(*p) && p != end)\n        ++p;\n      image_index_.push_back(static_cast<size_t>(atol(p)));\n      for (index_t i = 0; i < label_width; ++i) {\n        // skip till space\n        while (!isspace(*p) && p != end)\n          ++p;\n        // skip space\n        while (isspace(*p) && p != end)\n          ++p;\n        CHECK(p != end) << \"Bad ImageList format\";\n        label_.push_back(static_cast<real_t>(atof(p)));\n      }\n    }\n    delete fi;\n    // be careful not to resize label_ afterwards\n    idx2label_.reserve(image_index_.size());\n    for (size_t i = 0; i < image_index_.size(); ++i) {\n      idx2label_[image_index_[i]] = dmlc::BeginPtr(label_) + i * label_width;\n    }\n    if (!silent) {\n      LOG(INFO) << \"Loaded ImageList from \" << path_imglist << ' ' << image_index_.size()\n                << \" Image records\";\n    }\n  }\n  /*! \\brief find a label for corresponding index */\n  inline mshadow::Tensor<cpu, 1> Find(size_t imid) const {\n    std::unordered_map<size_t, real_t*>::const_iterator it = idx2label_.find(imid);\n    CHECK(it != idx2label_.end()) << \"fail to find imagelabel for id \" << imid;\n    return mshadow::Tensor<cpu, 1>(it->second, mshadow::Shape1(label_width));\n  }\n  /*! \\brief find a label for corresponding index, return vector as copy */\n  inline std::vector<float> FindCopy(size_t imid) const {\n    std::unordered_map<size_t, real_t*>::const_iterator it = idx2label_.find(imid);\n    CHECK(it != idx2label_.end()) << \"fail to find imagelabel for id \" << imid;\n    const real_t* ptr = it->second;\n    return std::vector<float>(ptr, ptr + label_width);\n  }\n\n private:\n  // label with_\n  mshadow::index_t label_width;\n  // image index of each record\n  std::vector<size_t> image_index_;\n  // real label content\n  std::vector<real_t> label_;\n  // map index to label\n  std::unordered_map<size_t, real_t*> idx2label_;\n};\n\n// Define image record parser parameters\nstruct ImageRecParserParam : public dmlc::Parameter<ImageRecParserParam> {\n  /*! \\brief path to image list */\n  std::string path_imglist;\n  /*! \\brief path to image recordio */\n  std::string path_imgrec;\n  /*! \\brief path to index file */\n  std::string path_imgidx;\n  /*! \\brief a sequence of names of image augmenters, seperated by , */\n  std::string aug_seq;\n  /*! \\brief label-width */\n  int label_width;\n  /*! \\brief input shape */\n  mxnet::TShape data_shape;\n  /*! \\brief number of threads */\n  int preprocess_threads;\n  /*! \\brief whether to remain silent */\n  bool verbose;\n  /*! \\brief partition the data into multiple parts */\n  int num_parts;\n  /*! \\brief the index of the part will read */\n  int part_index;\n  /*! \\brief device id used to create context for internal NDArray */\n  int device_id;\n  /*! \\brief the size of a shuffle chunk */\n  size_t shuffle_chunk_size;\n  /*! \\brief the seed for chunk shuffling */\n  int shuffle_chunk_seed;\n  /*! \\brief random seed for augmentations */\n  dmlc::optional<int> seed_aug;\n\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(ImageRecParserParam) {\n    DMLC_DECLARE_FIELD(path_imglist)\n        .set_default(\"\")\n        .describe(\n            \"Path to the image list (.lst) file. Generally created with tools/im2rec.py. \"\n            \"Format (Tab separated): \"\n            \"<index of record>\\t<one or more labels>\\t<relative path from root folder>.\");\n    DMLC_DECLARE_FIELD(path_imgrec)\n        .set_default(\"\")\n        .describe(\n            \"Path to the image RecordIO (.rec) file or a directory path. \"\n            \"Created with tools/im2rec.py.\");\n    DMLC_DECLARE_FIELD(path_imgidx)\n        .set_default(\"\")\n        .describe(\n            \"Path to the image RecordIO index (.idx) file. \"\n            \"Created with tools/im2rec.py.\");\n    DMLC_DECLARE_FIELD(aug_seq)\n        .set_default(\"aug_default\")\n        .describe(\n            \"The augmenter names to represent\"\n            \" sequence of augmenters to be applied, seperated by comma.\"\n            \" Additional keyword parameters will be seen by these augmenters.\");\n    DMLC_DECLARE_FIELD(label_width)\n        .set_lower_bound(1)\n        .set_default(1)\n        .describe(\"The number of labels per image.\");\n    DMLC_DECLARE_FIELD(data_shape)\n        .set_expect_ndim(3)\n        .enforce_nonzero()\n        .describe(\"The shape of one output image in (channels, height, width) format.\");\n    DMLC_DECLARE_FIELD(preprocess_threads)\n        .set_lower_bound(1)\n        .set_default(4)\n        .describe(\"The number of threads to do preprocessing.\");\n    DMLC_DECLARE_FIELD(verbose).set_default(true).describe(\"If or not output verbose information.\");\n    DMLC_DECLARE_FIELD(num_parts).set_default(1).describe(\n        \"Virtually partition the data into these many parts.\");\n    DMLC_DECLARE_FIELD(part_index)\n        .set_default(0)\n        .describe(\"The *i*-th virtual partition to be read.\");\n    DMLC_DECLARE_FIELD(device_id).set_default(0).describe(\n        \"The device id used to create context for internal NDArray. \"\n        \"Setting device_id to -1 will create Context::CPU(0). Setting \"\n        \"device_id to valid positive device id will create \"\n        \"Context::CPUPinned(device_id). Default is 0.\");\n    DMLC_DECLARE_FIELD(shuffle_chunk_size)\n        .set_default(0)\n        .describe(\"The data shuffle buffer size in MB. Only valid if shuffle is true.\");\n    DMLC_DECLARE_FIELD(shuffle_chunk_seed).set_default(0).describe(\"The random seed for shuffling\");\n    DMLC_DECLARE_FIELD(seed_aug)\n        .set_default(dmlc::optional<int>())\n        .describe(\"Random seed for augmentations.\");\n  }\n};\n\n// Batch parameters\nstruct BatchParam : public dmlc::Parameter<BatchParam> {\n  /*! \\brief label width */\n  uint32_t batch_size;\n  /*! \\brief use round roubin to handle overflow batch */\n  bool round_batch;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(BatchParam) {\n    DMLC_DECLARE_FIELD(batch_size).describe(\"Batch size.\");\n    DMLC_DECLARE_FIELD(round_batch)\n        .set_default(true)\n        .describe(\"Whether to use round robin to handle overflow batch or not.\");\n  }\n};\n\n// Batch Sampler parameters\nstruct BatchSamplerParam : public dmlc::Parameter<BatchSamplerParam> {\n  /*! \\brief Last batch behavior type */\n  enum LastBatchType {\n    /*! \\brief Keep not fully filled last batch */\n    kKeep = 0,\n    /*! \\brief Roll over the remaining batch to next epoch */\n    kRollOver,\n    /*! \\brief Discard not fully filled last batch */\n    kDiscard\n  };  // enum LastBatchType\n  /*! \\brief batch size */\n  uint32_t batch_size;\n  /*! \\brief last batch behavior */\n  int last_batch;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(BatchSamplerParam) {\n    DMLC_DECLARE_FIELD(batch_size).describe(\"Batch size.\");\n    DMLC_DECLARE_FIELD(last_batch)\n        .set_default(kKeep)\n        .add_enum(\"keep\", kKeep)\n        .add_enum(\"rollover\", kRollOver)\n        .add_enum(\"discard\", kDiscard)\n        .describe(\n            \"Specifies how the last batch is handled if batch_size does not evenly \"\n            \"divide sequence length. \"\n            \"If 'keep', the last batch will be returned directly, but will contain \"\n            \"less element than `batch_size` requires. \"\n            \"If 'discard', the last batch will be discarded. \"\n            \"If 'rollover', the remaining elements will be rolled over to the next \"\n            \"iteration. Note: legacy batch param with round_batch will always round data \"\n            \"in order to always provide full batchs. Rollover behavior will instead result \"\n            \"in different iteration sizes for each epoch.\");\n  }\n};\n\n// Define image record parameters\nstruct ImageRecordParam : public dmlc::Parameter<ImageRecordParam> {\n  /*! \\brief whether to do shuffle */\n  bool shuffle;\n  /*! \\brief random seed */\n  int seed;\n  /*! \\brief whether to remain silent */\n  bool verbose;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(ImageRecordParam) {\n    DMLC_DECLARE_FIELD(shuffle).set_default(false).describe(\n        \"Whether to shuffle data randomly or not.\");\n    DMLC_DECLARE_FIELD(seed).set_default(0).describe(\"The random seed.\");\n    DMLC_DECLARE_FIELD(verbose).set_default(true).describe(\n        \"Whether to output verbose information or not.\");\n  }\n};\n\n// normalize parameters\nstruct ImageNormalizeParam : public dmlc::Parameter<ImageNormalizeParam> {\n  /*! \\brief random seed */\n  int seed;\n  /*! \\brief whether to mirror the image */\n  bool mirror;\n  /*! \\brief whether to perform rand mirror the image */\n  bool rand_mirror;\n  /*! \\brief mean file string */\n  std::string mean_img;\n  /*! \\brief mean value for r channel */\n  float mean_r;\n  /*! \\brief mean value for g channel */\n  float mean_g;\n  /*! \\brief mean value for b channel */\n  float mean_b;\n  /*! \\brief mean value for alpha channel */\n  float mean_a;\n  /*! \\brief standard deviation for r channel */\n  float std_r;\n  /*! \\brief standard deviation for g channel */\n  float std_g;\n  /*! \\brief standard deviation for b channel */\n  float std_b;\n  /*! \\brief standard deviation for alpha channel */\n  float std_a;\n  /*! \\brief scale on color space */\n  float scale;\n  /*! \\brief maximum ratio of contrast variation */\n  float max_random_contrast;\n  /*! \\brief maximum value of illumination variation */\n  float max_random_illumination;\n  /*! \\brief silent */\n  bool verbose;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(ImageNormalizeParam) {\n    DMLC_DECLARE_FIELD(seed).set_default(0).describe(\"The random seed.\");\n    DMLC_DECLARE_FIELD(mirror).set_default(false).describe(\n        \"Whether to mirror the image or not. If true, images are \"\n        \"flipped along the horizontal axis.\");\n    DMLC_DECLARE_FIELD(rand_mirror)\n        .set_default(false)\n        .describe(\n            \"Whether to randomly mirror images or not. If true, 50% of \"\n            \"the images will be randomly mirrored (flipped along the \"\n            \"horizontal axis)\");\n    DMLC_DECLARE_FIELD(mean_img).set_default(\"\").describe(\"Filename of the mean image.\");\n    DMLC_DECLARE_FIELD(mean_r).set_default(0.0f).describe(\n        \"The mean value to be subtracted on the R channel\");\n    DMLC_DECLARE_FIELD(mean_g).set_default(0.0f).describe(\n        \"The mean value to be subtracted on the G channel\");\n    DMLC_DECLARE_FIELD(mean_b).set_default(0.0f).describe(\n        \"The mean value to be subtracted on the B channel\");\n    DMLC_DECLARE_FIELD(mean_a).set_default(0.0f).describe(\n        \"The mean value to be subtracted on the alpha channel\");\n    DMLC_DECLARE_FIELD(std_r).set_default(1.0f).describe(\n        \"Augmentation Param: Standard deviation on R channel.\");\n    DMLC_DECLARE_FIELD(std_g).set_default(1.0f).describe(\n        \"Augmentation Param: Standard deviation on G channel.\");\n    DMLC_DECLARE_FIELD(std_b).set_default(1.0f).describe(\n        \"Augmentation Param: Standard deviation on B channel.\");\n    DMLC_DECLARE_FIELD(std_a).set_default(1.0f).describe(\n        \"Augmentation Param: Standard deviation on Alpha channel.\");\n    DMLC_DECLARE_FIELD(scale).set_default(1.0f).describe(\"Multiply the image with a scale value.\");\n    DMLC_DECLARE_FIELD(max_random_contrast)\n        .set_default(0.0f)\n        .describe(\n            \"Change the contrast with a value randomly chosen from \"\n            \"``[-max_random_contrast, max_random_contrast]``\");\n    DMLC_DECLARE_FIELD(max_random_illumination)\n        .set_default(0.0f)\n        .describe(\n            \"Change the illumination with a value randomly chosen from \"\n            \"``[-max_random_illumination, max_random_illumination]``\");\n    DMLC_DECLARE_FIELD(verbose).set_default(true).describe(\"If or not output verbose information.\");\n  }\n};\n\n// normalize det parameters\nstruct ImageDetNormalizeParam : public dmlc::Parameter<ImageDetNormalizeParam> {\n  /*! \\brief random seed */\n  int seed;\n  /*! \\brief mean file string */\n  std::string mean_img;\n  /*! \\brief mean value for r channel */\n  float mean_r;\n  /*! \\brief mean value for g channel */\n  float mean_g;\n  /*! \\brief mean value for b channel */\n  float mean_b;\n  /*! \\brief mean value for alpha channel */\n  float mean_a;\n  /*! \\brief standard deviation for r channel */\n  float std_r;\n  /*! \\brief standard deviation for g channel */\n  float std_g;\n  /*! \\brief standard deviation for b channel */\n  float std_b;\n  /*! \\brief standard deviation for alpha channel */\n  float std_a;\n  /*! \\brief scale on color space */\n  float scale;\n  /*! \\brief silent */\n  bool verbose;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(ImageDetNormalizeParam) {\n    DMLC_DECLARE_FIELD(seed).set_default(0).describe(\"Augmentation Param: Random Seed.\");\n    DMLC_DECLARE_FIELD(mean_img).set_default(\"\").describe(\n        \"Augmentation Param: Mean Image to be subtracted.\");\n    DMLC_DECLARE_FIELD(mean_r).set_default(0.0f).describe(\n        \"Augmentation Param: Mean value on R channel.\");\n    DMLC_DECLARE_FIELD(mean_g).set_default(0.0f).describe(\n        \"Augmentation Param: Mean value on G channel.\");\n    DMLC_DECLARE_FIELD(mean_b).set_default(0.0f).describe(\n        \"Augmentation Param: Mean value on B channel.\");\n    DMLC_DECLARE_FIELD(mean_a).set_default(0.0f).describe(\n        \"Augmentation Param: Mean value on Alpha channel.\");\n    DMLC_DECLARE_FIELD(std_r).set_default(0.0f).describe(\n        \"Augmentation Param: Standard deviation on R channel.\");\n    DMLC_DECLARE_FIELD(std_g).set_default(0.0f).describe(\n        \"Augmentation Param: Standard deviation on G channel.\");\n    DMLC_DECLARE_FIELD(std_b).set_default(0.0f).describe(\n        \"Augmentation Param: Standard deviation on B channel.\");\n    DMLC_DECLARE_FIELD(std_a).set_default(0.0f).describe(\n        \"Augmentation Param: Standard deviation on Alpha channel.\");\n    DMLC_DECLARE_FIELD(scale).set_default(1.0f).describe(\n        \"Augmentation Param: Scale in color space.\");\n    DMLC_DECLARE_FIELD(verbose).set_default(true).describe(\n        \"Augmentation Param: Whether to print augmentor info.\");\n  }\n};\n\n// Define prefetcher parameters\nstruct PrefetcherParam : public dmlc::Parameter<PrefetcherParam> {\n  enum CtxType { kGPU = 0, kCPU, kCPUPinned, kCPUShared };\n  /*! \\brief number of prefetched batches */\n  size_t prefetch_buffer;\n\n  /*! \\brief Context data loader optimized for */\n  int ctx;\n  int device_id;\n  /*! \\brief data type */\n  dmlc::optional<int> dtype;\n\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(PrefetcherParam) {\n    DMLC_DECLARE_FIELD(prefetch_buffer)\n        .set_default(4)\n        .describe(\"Maximum number of batches to prefetch.\");\n    DMLC_DECLARE_FIELD(ctx)\n        .set_default(kGPU)\n        .add_enum(\"cpu\", kCPU)\n        .add_enum(\"gpu\", kGPU)\n        .add_enum(\"cpu_pinned\", kCPUPinned)\n        .describe(\n            \"Context data loader optimized for. \"\n            \"Note that it only indicates the optimization strategy for devices, \"\n            \"by no means the prefetcher will load data to GPUs. \"\n            \"If ctx is 'cpu_pinned' and device_id is not -1, \"\n            \"it will use cpu_pinned(device_id) as ctx\");\n    DMLC_DECLARE_FIELD(device_id).set_default(-1).describe(\n        \"The default device id for context. -1 indicate it's on default device\");\n    DMLC_DECLARE_FIELD(dtype)\n        .add_enum(\"float32\", mshadow::kFloat32)\n        .add_enum(\"float64\", mshadow::kFloat64)\n        .add_enum(\"float16\", mshadow::kFloat16)\n        .add_enum(\"bfloat16\", mshadow::kBfloat16)\n        .add_enum(\"int64\", mshadow::kInt64)\n        .add_enum(\"int32\", mshadow::kInt32)\n        .add_enum(\"uint8\", mshadow::kUint8)\n        .add_enum(\"int8\", mshadow::kInt8)\n        .set_default(dmlc::optional<int>())\n        .describe(\"Output data type. ``None`` means no change.\");\n  }\n};\n\n}  // namespace io\n}  // namespace mxnet\n\n#endif  // MXNET_IO_IMAGE_ITER_COMMON_H_\n"
  },
  {
    "path": "src/io/image_recordio.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file image_recordio.h\n * \\brief image recordio struct\n */\n#ifndef MXNET_IO_IMAGE_RECORDIO_H_\n#define MXNET_IO_IMAGE_RECORDIO_H_\n\n#include <dmlc/base.h>\n#include <dmlc/io.h>\n#include <string>\n\nnamespace mxnet {\nnamespace io {\n/*! \\brief image recordio struct */\nstruct ImageRecordIO {\n  /*! \\brief header in image recordio */\n  struct Header {\n    /*!\n     * \\brief flag of the header,\n     *  used for future extension purposes\n     */\n    uint32_t flag;\n    /*!\n     * \\brief label field that returns label of images\n     *  when image list was not presented,\n     *\n     * NOTE: user do not need to repack recordio just to\n     * change label field, just supply a list file that\n     * maps image id to new labels\n     */\n    float label;\n    /*!\n     * \\brief unique image index\n     *  image_id[1] is always set to 0,\n     *  reserved for future purposes for 128bit id\n     *  image_id[0] is used to store image id\n     */\n    uint64_t image_id[2];\n  };\n  /*! \\brief header of image recordio */\n  Header header;\n  /*! \\brief point to label */\n  float* label;\n  /*! \\brief number of float labels */\n  int num_label;\n  /*! \\brief pointer to data content */\n  uint8_t* content;\n  /*! \\brief size of the content */\n  size_t content_size;\n  /*! \\brief constructor */\n  ImageRecordIO(void) : label(nullptr), num_label(0), content(nullptr), content_size(0) {\n    memset(&header, 0, sizeof(header));\n  }\n  /*! \\brief get image id from record */\n  inline uint64_t image_index(void) const {\n    return header.image_id[0];\n  }\n  /*!\n   * \\brief load header from a record content\n   * \\param buf the head of record\n   * \\param size the size of the entire record\n   */\n  inline void Load(void* buf, size_t size) {\n    CHECK(size >= sizeof(header));\n    std::memcpy(&header, buf, sizeof(header));\n    content      = reinterpret_cast<uint8_t*>(buf) + sizeof(header);\n    content_size = size - sizeof(header);\n    if (header.flag > 0) {\n      CHECK(content_size >= sizeof(float) * header.flag);\n      label     = reinterpret_cast<float*>(content);\n      num_label = header.flag;\n      content   = reinterpret_cast<uint8_t*>(label + header.flag);\n      content_size -= sizeof(float) * header.flag;\n    } else {\n      label     = nullptr;\n      num_label = 0;\n    }\n  }\n  /*!\n   * \\brief save the record header\n   */\n  inline void SaveHeader(std::string* blob) const {\n    blob->resize(sizeof(header));\n    std::memcpy(dmlc::BeginPtr(*blob), &header, sizeof(header));\n  }\n};\n}  // namespace io\n}  // namespace mxnet\n#endif  // MXNET_IO_IMAGE_RECORDIO_H_\n"
  },
  {
    "path": "src/io/inst_vector.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file inst_vector.h\n * \\brief holder of a sequence of DataInst in CPU\n *        that are not necessarily of same shape\n */\n\n#ifndef MXNET_IO_INST_VECTOR_H_\n#define MXNET_IO_INST_VECTOR_H_\n\n#include <mxnet/io.h>\n#include <mxnet/base.h>\n#include <mxnet/tensor_blob.h>\n#include <dmlc/base.h>\n#include <mshadow/tensor.h>\n#include <vector>\n#include <string>\n\nnamespace mxnet {\nnamespace io {\n/*!\n * \\brief a vector of tensor with various shape\n *\n * data are stored in memory continuously\n */\ntemplate <int dim, typename DType>\nclass TensorVector {\n public:\n  TensorVector(void) {\n    this->Clear();\n  }\n  /*! \\brief get the buffer to the i-th tensor */\n  inline mshadow::Tensor<cpu, dim, DType> operator[](size_t i) const {\n    CHECK_LT(i + 1, offset_.size());\n    CHECK_EQ(shape_[i].Size(), offset_[i + 1] - offset_[i]);\n    return mshadow::Tensor<cpu, dim, DType>(\n        (DType*)dmlc::BeginPtr(content_) + offset_[i],  // NOLINT(*)\n        shape_[i]);                                     // NOLINT(*)\n  }\n  inline mshadow::Tensor<cpu, dim, DType> Back() const {\n    return (*this)[Size() - 1];\n  }\n  inline size_t Size(void) const {\n    return shape_.size();\n  }\n  /*! \\brief allocate space given the shape (data are copied) */\n  inline void Push(mshadow::Shape<dim> shape) {\n    shape_.push_back(shape);\n    offset_.push_back(offset_.back() + shape.Size());\n    content_.resize(offset_.back());\n  }\n  inline void Clear(void) {\n    offset_.clear();\n    offset_.push_back(0);\n    content_.clear();\n    shape_.clear();\n  }\n\n private:\n  // offset of the data content\n  std::vector<size_t> offset_;\n  // data content\n  std::vector<DType> content_;\n  // shape of data\n  std::vector<mshadow::Shape<dim> > shape_;\n};\n\n/*!\n * \\brief a list of (label, example) pairs, examples can have various shape\n */\ntemplate <typename DType = real_t>\nclass InstVector {\n public:\n  /*! \\brief return the number of (label, example) pairs */\n  inline size_t Size(void) const {\n    return index_.size();\n  }\n  // get index\n  inline unsigned Index(unsigned i) const {\n    return index_[i];\n  }\n  // instance\n  /* \\brief get the i-th (label, example) pair */\n  inline DataInst operator[](size_t i) const {\n    DataInst inst;\n    inst.index = index_[i];\n    // ImageRecordIter depends on data vector\n    // here having size 2. If you want to\n    // change this assumption here, change it\n    // in there as well (InitBatch section)!\n    inst.data.push_back(TBlob(data_[i]));\n    inst.data.push_back(TBlob(label_[i]));\n    return inst;\n  }\n  /* \\brief get the last (label, example) pair */\n  inline DataInst Back() const {\n    return (*this)[Size() - 1];\n  }\n  inline void Clear(void) {\n    index_.clear();\n    data_.Clear();\n    label_.Clear();\n  }\n  /*\n   * \\brief push a (label, example) pair\n   * only reserved the space, while the data is not copied\n   */\n  inline void Push(unsigned index, mshadow::Shape<3> dshape, mshadow::Shape<1> lshape) {\n    index_.push_back(index);\n    data_.Push(dshape);\n    label_.Push(lshape);\n  }\n  /*! \\return the data content */\n  inline const TensorVector<3, DType>& data() const {\n    return data_;\n  }\n  /*! \\return the label content */\n  inline const TensorVector<1, real_t>& label() const {\n    return label_;\n  }\n\n private:\n  /*! \\brief index of the data */\n  std::vector<unsigned> index_;\n  // label\n  TensorVector<3, DType> data_;\n  // data\n  TensorVector<1, real_t> label_;\n};\n\n/*!\n * \\brief tblob batch\n *\n * data are stored in tblob before going into NDArray\n */\nstruct TBlobBatch {\n public:\n  /*! \\brief unique id for instance, can be NULL, sometimes is useful */\n  unsigned* inst_index;\n  /*! \\brief number of instance */\n  mshadow::index_t batch_size;\n  /*! \\brief number of padding elements in this batch,\n       this is used to indicate the last elements in the batch are only padded up to match the\n     batch, and should be discarded */\n  mshadow::index_t num_batch_padd;\n  /*! \\brief content of dense data */\n  std::vector<TBlob> data;\n  /*! \\brief extra data to be fed to the network */\n  std::string extra_data;\n  /*! \\brief constructor */\n  TBlobBatch(void) {\n    inst_index     = nullptr;\n    batch_size     = 0;\n    num_batch_padd = 0;\n  }\n  /*! \\brief destructor */\n  ~TBlobBatch() {\n    delete[] inst_index;\n  }\n};  // struct TBlobBatch\n\nclass TBlobContainer : public TBlob {\n public:\n  TBlobContainer(void) : TBlob(), tensor_container_(nullptr) {}\n  ~TBlobContainer() {\n    if (tensor_container_) {\n      release();\n    }\n  }\n  void resize(const mxnet::TShape& shape, int type_flag) {\n    if (tensor_container_) {\n      CHECK_EQ(this->type_flag_, type_flag);\n      this->shape_ = shape;\n      resize();\n    } else {\n      this->type_flag_ = type_flag;\n      this->shape_     = shape;\n      create();\n    }\n  }\n\n private:\n  void create() {\n    CHECK(tensor_container_ == nullptr);\n    CHECK_EQ(this->dev_mask(), mshadow::cpu::kDevMask);\n    MSHADOW_TYPE_SWITCH(this->type_flag_, DType, {\n      auto tensor_container = new mshadow::TensorContainer<mshadow::cpu, 1, DType>(false);\n      tensor_container->Resize(mshadow::Shape1(shape_.Size()));\n      dptr_             = tensor_container->dptr_;\n      tensor_container_ = tensor_container;\n    });\n  }\n  void resize() {\n    MSHADOW_TYPE_SWITCH(this->type_flag_, DType, {\n      auto tensor_container = (mshadow::TensorContainer<mshadow::cpu, 1, DType>*)tensor_container_;\n      tensor_container->Resize(mshadow::Shape1(shape_.Size()));\n    });\n  }\n  void release() {\n    MSHADOW_TYPE_SWITCH(this->type_flag_, DType, {\n      auto tensor_container = (mshadow::TensorContainer<mshadow::cpu, 1, DType>*)tensor_container_;\n      delete tensor_container;\n    });\n  }\n\n  void* tensor_container_;\n};\n\n}  // namespace io\n}  // namespace mxnet\n#endif  // MXNET_IO_INST_VECTOR_H_\n"
  },
  {
    "path": "src/io/io.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#include <mxnet/io.h>\n#include <dmlc/registry.h>\n#include \"./image_augmenter.h\"\n#include \"./image_iter_common.h\"\n\n// Registers\nnamespace dmlc {\nDMLC_REGISTRY_ENABLE(::mxnet::DataIteratorReg);\nDMLC_REGISTRY_ENABLE(::mxnet::DatasetReg);\nDMLC_REGISTRY_ENABLE(::mxnet::BatchifyFunctionReg);\n}  // namespace dmlc\n\nnamespace mxnet {\nnamespace io {\n// Register parameters in header files\nDMLC_REGISTER_PARAMETER(BatchParam);\nDMLC_REGISTER_PARAMETER(BatchSamplerParam);\nDMLC_REGISTER_PARAMETER(PrefetcherParam);\nDMLC_REGISTER_PARAMETER(ImageNormalizeParam);\nDMLC_REGISTER_PARAMETER(ImageRecParserParam);\nDMLC_REGISTER_PARAMETER(ImageRecordParam);\nDMLC_REGISTER_PARAMETER(ImageDetNormalizeParam);\n}  // namespace io\n}  // namespace mxnet\n"
  },
  {
    "path": "src/io/iter_batchloader.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file iter_batchloader.h\n * \\brief define a batch adapter to create tblob batch\n */\n#ifndef MXNET_IO_ITER_BATCHLOADER_H_\n#define MXNET_IO_ITER_BATCHLOADER_H_\n\n#include <mxnet/io.h>\n#include <mxnet/base.h>\n#include <dmlc/logging.h>\n#include <mshadow/tensor.h>\n#include <utility>\n#include <vector>\n#include <string>\n#include \"./inst_vector.h\"\n#include \"./image_iter_common.h\"\n\nnamespace mxnet {\nnamespace io {\n\n/*! \\brief create a batch iterator from single instance iterator */\nclass BatchLoader : public IIterator<TBlobBatch> {\n public:\n  explicit BatchLoader(IIterator<DataInst>* base) : head_(1), num_overflow_(0), base_(base) {}\n\n  virtual ~BatchLoader(void) {\n    delete base_;\n  }\n\n  inline void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) {\n    std::vector<std::pair<std::string, std::string> > kwargs_left;\n    // init batch param, it could have similar param with\n    kwargs_left = param_.InitAllowUnknown(kwargs);\n    // Init space for out\n    out_.inst_index = new unsigned[param_.batch_size];\n    out_.batch_size = param_.batch_size;\n    out_.data.clear();\n    // init base iterator\n    base_->Init(kwargs);\n  }\n\n  virtual void BeforeFirst(void) {\n    if (param_.round_batch == 0 || num_overflow_ == 0) {\n      // otherise, we already called before first\n      base_->BeforeFirst();\n    } else {\n      num_overflow_ = 0;\n    }\n    head_ = 1;\n  }\n\n  virtual bool Next(void) {\n    out_.num_batch_padd = 0;\n    out_.batch_size     = param_.batch_size;\n    this->head_         = 0;\n\n    // if overflow from previous round, directly return false, until before first is called\n    if (num_overflow_ != 0)\n      return false;\n    size_t top = 0;\n\n    while (base_->Next()) {\n      const DataInst& d    = base_->Value();\n      out_.inst_index[top] = d.index;\n      if (data_.size() == 0) {\n        this->InitData(d);\n      }\n      for (size_t i = 0; i < d.data.size(); ++i) {\n        CHECK_EQ(unit_size_[i], d.data[i].Size());\n        MSHADOW_TYPE_SWITCH(data_[i].type_flag_, DType, {\n          mshadow::Copy(\n              data_[i].get<cpu, 1, DType>().Slice(top * unit_size_[i], (top + 1) * unit_size_[i]),\n              d.data[i].get_with_shape<cpu, 1, DType>(mshadow::Shape1(unit_size_[i])));\n        });\n      }\n      if (++top >= param_.batch_size) {\n        return true;\n      }\n    }\n    if (top != 0) {\n      if (param_.round_batch != 0) {\n        num_overflow_ = 0;\n        base_->BeforeFirst();\n        for (; top < param_.batch_size; ++top, ++num_overflow_) {\n          CHECK(base_->Next()) << \"number of input must be bigger than batch size\";\n          const DataInst& d    = base_->Value();\n          out_.inst_index[top] = d.index;\n          // copy data\n          for (size_t i = 0; i < d.data.size(); ++i) {\n            CHECK_EQ(unit_size_[i], d.data[i].Size());\n            MSHADOW_TYPE_SWITCH(data_[i].type_flag_, DType, {\n              mshadow::Copy(\n                  data_[i].get<cpu, 1, DType>().Slice(top * unit_size_[i],\n                                                      (top + 1) * unit_size_[i]),\n                  d.data[i].get_with_shape<cpu, 1, DType>(mshadow::Shape1(unit_size_[i])));\n            });\n          }\n        }\n        out_.num_batch_padd = num_overflow_;\n      } else {\n        out_.num_batch_padd = param_.batch_size - top;\n      }\n      return true;\n    }\n    return false;\n  }\n  virtual const TBlobBatch& Value(void) const {\n    return out_;\n  }\n\n protected:\n  /*! \\brief batch parameters */\n  BatchParam param_;\n  /*! \\brief output data */\n  TBlobBatch out_;\n  /*! \\brief on first */\n  int head_;\n  /*! \\brief number of overflow instances that readed in round_batch mode */\n  int num_overflow_;\n  /*! \\brief tensor to hold data */\n  std::vector<TBlobContainer> data_;\n\n private:\n  /*! \\brief base iterator */\n  IIterator<DataInst>* base_;\n  /*! \\brief data shape */\n  mxnet::ShapeVector shape_;\n  /*! \\brief unit size */\n  std::vector<size_t> unit_size_;\n  // initialize the data holder by using from the first batch.\n  inline void InitData(const DataInst& first_batch) {\n    shape_.resize(first_batch.data.size());\n    data_.resize(first_batch.data.size());\n    unit_size_.resize(first_batch.data.size());\n    for (size_t i = 0; i < first_batch.data.size(); ++i) {\n      mxnet::TShape src_shape = first_batch.data[i].shape_;\n      int src_type_flag       = first_batch.data[i].type_flag_;\n      // init object attributes\n      std::vector<index_t> shape_vec;\n      shape_vec.push_back(param_.batch_size);\n      for (index_t dim = 0; dim < src_shape.ndim(); ++dim) {\n        shape_vec.push_back(src_shape[dim]);\n      }\n      mxnet::TShape dst_shape(shape_vec.begin(), shape_vec.end());\n      shape_[i] = dst_shape;\n      data_[i].resize(mshadow::Shape1(dst_shape.Size()), src_type_flag);\n      unit_size_[i] = src_shape.Size();\n      out_.data.push_back(TBlob(data_[i].dptr_, dst_shape, cpu::kDevMask, src_type_flag, 0));\n    }\n  }\n};  // class BatchLoader\n\n/*! \\brief create a batch sampler from single instance iterator\n *  Unlike BatchLoader, BatchSampler will handle flexible length during iteration.\n */\nclass BatchSampler : public IIterator<DataBatch> {\n public:\n  explicit BatchSampler(IIterator<DataInst>* base) : num_overflow_(0), base_(base) {}\n\n  virtual ~BatchSampler(void) {\n    delete base_;\n  }\n\n  inline void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) {\n    std::vector<std::pair<std::string, std::string> > kwargs_left;\n    // init batch param, it could have similar param with\n    kwargs_left = param_.InitAllowUnknown(kwargs);\n    // Init space for out\n    out_.data.clear();\n    // init base iterator\n    base_->Init(kwargs);\n  }\n\n  virtual void BeforeFirst(void) {\n    if (param_.last_batch != param_.kRollOver || num_overflow_ == 0) {\n      // otherise, we already called before first\n      base_->BeforeFirst();\n    }\n  }\n\n  virtual int64_t GetLenHint(void) const {\n    auto base_hint = base_->GetLenHint();\n    if (base_hint < 0) {\n      return base_hint;\n    } else if (param_.kKeep == param_.last_batch) {\n      return (base_hint + param_.batch_size - 1) / param_.batch_size;\n    } else if (param_.kDiscard == param_.last_batch) {\n      return base_hint / param_.batch_size;\n    } else if (param_.kRollOver == param_.last_batch) {\n      return (base_hint + num_overflow_) / param_.batch_size;\n    } else {\n      LOG(FATAL) << \"last_batch must be one of 'keep', 'discard', or 'rollover'\"\n                 << \" but got: \" << param_.last_batch;\n    }\n    return -1;\n  }\n\n  virtual bool Next(void) {\n    out_.num_batch_padd = 0;\n\n    size_t top = num_overflow_;  // start with last overflow index\n\n    while (base_->Next()) {\n      const DataInst& d = base_->Value();\n      // out_.inst_index[top] = d.index;\n      if (data_.size() == 0) {\n        this->InitData(d);\n      }\n      for (size_t i = 0; i < d.data.size(); ++i) {\n        CHECK_EQ(unit_size_[i], d.data[i].Size());\n        MSHADOW_TYPE_SWITCH(data_[i].type_flag_, DType, {\n          mshadow::Copy(\n              data_[i].get<cpu, 1, DType>().Slice(top * unit_size_[i], (top + 1) * unit_size_[i]),\n              d.data[i].get_with_shape<cpu, 1, DType>(mshadow::Shape1(unit_size_[i])));\n        });\n      }\n      if (++top >= param_.batch_size) {\n        num_overflow_ = 0;\n        return true;\n      }\n    }\n    if (top != 0) {\n      if (param_.last_batch == param_.kDiscard) {\n        // discard the batch\n        num_overflow_ = 0;\n        return false;\n      } else if (param_.last_batch == param_.kKeep) {\n        out_.num_batch_padd = param_.batch_size - top;\n        num_overflow_       = 0;\n        return true;\n      } else if (param_.last_batch == param_.kRollOver) {\n        if (num_overflow_ > 0) {\n          base_->BeforeFirst();\n          num_overflow_ = top;\n          return this->Next();\n        } else {\n          num_overflow_ = top;\n          return false;\n        }\n      } else {\n        LOG(FATAL) << \"Unknown last_batch type: \" << param_.last_batch;\n      }\n    }\n    return false;\n  }\n  virtual const DataBatch& Value(void) const {\n    return out_;\n  }\n\n protected:\n  /*! \\brief batch parameters */\n  BatchSamplerParam param_;\n  /*! \\brief output data */\n  DataBatch out_;\n  /*! \\brief number of overflow instances that readed in round_batch mode */\n  int num_overflow_;\n  /*! \\brief tensor to hold data */\n  std::vector<TBlobContainer> data_;\n\n private:\n  /*! \\brief base iterator */\n  IIterator<DataInst>* base_;\n  /*! \\brief data shape */\n  mxnet::ShapeVector shape_;\n  /*! \\brief unit size */\n  std::vector<size_t> unit_size_;\n  // initialize the data holder by using from the first batch.\n  inline void InitData(const DataInst& first_batch) {\n    shape_.resize(first_batch.data.size());\n    data_.resize(first_batch.data.size());\n    unit_size_.resize(first_batch.data.size());\n    for (size_t i = 0; i < first_batch.data.size(); ++i) {\n      mxnet::TShape src_shape = first_batch.data[i].shape_;\n      int src_type_flag       = first_batch.data[i].type_flag_;\n      // init object attributes\n      std::vector<index_t> shape_vec;\n      shape_vec.push_back(param_.batch_size);\n      for (index_t dim = 0; dim < src_shape.ndim(); ++dim) {\n        shape_vec.push_back(src_shape[dim]);\n      }\n      mxnet::TShape dst_shape(shape_vec.begin(), shape_vec.end());\n      shape_[i] = dst_shape;\n      data_[i].resize(mshadow::Shape1(dst_shape.Size()), src_type_flag);\n      unit_size_[i] = src_shape.Size();\n      out_.data.push_back(\n          NDArray(TBlob(data_[i].dptr_, dst_shape, cpu::kDevMask, src_type_flag, 0), 0));\n    }\n  }\n};  // class BatchSampler\n}  // namespace io\n}  // namespace mxnet\n#endif  // MXNET_IO_ITER_BATCHLOADER_H_\n"
  },
  {
    "path": "src/io/iter_csv.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file iter_csv.cc\n * \\brief define a CSV Reader to read in arrays\n */\n#include <mxnet/io.h>\n#include <dmlc/base.h>\n#include <dmlc/logging.h>\n#include <dmlc/parameter.h>\n#include <dmlc/data.h>\n#include \"./iter_prefetcher.h\"\n#include \"./iter_batchloader.h\"\n\nnamespace mxnet {\nnamespace io {\n// CSV parameters\nstruct CSVIterParam : public dmlc::Parameter<CSVIterParam> {\n  /*! \\brief path to data csv file */\n  std::string data_csv;\n  /*! \\brief data shape */\n  mxnet::TShape data_shape;\n  /*! \\brief path to label csv file */\n  std::string label_csv;\n  /*! \\brief label shape */\n  mxnet::TShape label_shape;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(CSVIterParam) {\n    DMLC_DECLARE_FIELD(data_csv).describe(\"The input CSV file or a directory path.\");\n    DMLC_DECLARE_FIELD(data_shape).describe(\"The shape of one example.\");\n    DMLC_DECLARE_FIELD(label_csv).set_default(\"NULL\").describe(\n        \"The input CSV file or a directory path. \"\n        \"If NULL, all labels will be returned as 0.\");\n    index_t shape1[] = {1};\n    DMLC_DECLARE_FIELD(label_shape)\n        .set_default(mxnet::TShape(shape1, shape1 + 1))\n        .describe(\"The shape of one label.\");\n  }\n};\n\nclass CSVIterBase : public IIterator<DataInst> {\n public:\n  CSVIterBase() {\n    out_.data.resize(2);\n  }\n  ~CSVIterBase() override = default;\n\n  // initialize iterator loads data in\n  void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override = 0;\n  /*! \\brief reset the iterator */\n  void BeforeFirst() override = 0;\n  /*! \\brief move to next item */\n  bool Next() override = 0;\n  /*! \\brief get current data */\n  const DataInst& Value() const override {\n    return out_;\n  }\n\n protected:\n  CSVIterParam param_;\n\n  DataInst out_;\n\n  // internal instance counter\n  unsigned inst_counter_{0};\n  // at end\n  bool end_{false};\n\n  // label parser\n  size_t label_ptr_{0}, label_size_{0};\n  size_t data_ptr_{0}, data_size_{0};\n};\n\ntemplate <typename DType>\nclass CSVIterTyped : public CSVIterBase {\n public:\n  ~CSVIterTyped() override = default;\n  // intialize iterator loads data in\n  void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override {\n    param_.InitAllowUnknown(kwargs);\n    data_parser_.reset(dmlc::Parser<uint32_t, DType>::Create(param_.data_csv.c_str(), 0, 1, \"csv\"));\n    if (param_.label_csv != \"NULL\") {\n      label_parser_.reset(\n          dmlc::Parser<uint32_t, DType>::Create(param_.label_csv.c_str(), 0, 1, \"csv\"));\n    } else {\n      dummy_label.set_pad(false);\n      dummy_label.Resize(mshadow::Shape1(1));\n      dummy_label = 0;\n    }\n  }\n\n  void BeforeFirst() override {\n    data_parser_->BeforeFirst();\n    if (label_parser_.get() != nullptr) {\n      label_parser_->BeforeFirst();\n    }\n    data_ptr_ = label_ptr_ = 0;\n    data_size_ = label_size_ = 0;\n    inst_counter_            = 0;\n    end_                     = false;\n  }\n\n  bool Next() override {\n    if (end_)\n      return false;\n    while (data_ptr_ >= data_size_) {\n      if (!data_parser_->Next()) {\n        end_ = true;\n        return false;\n      }\n      data_ptr_  = 0;\n      data_size_ = data_parser_->Value().size;\n    }\n    out_.index = inst_counter_++;\n    CHECK_LT(data_ptr_, data_size_);\n    out_.data[0] = AsTBlob(data_parser_->Value()[data_ptr_++], param_.data_shape);\n\n    if (label_parser_.get() != nullptr) {\n      while (label_ptr_ >= label_size_) {\n        CHECK(label_parser_->Next())\n            << \"Data CSV's row is smaller than the number of rows in label_csv\";\n        label_ptr_  = 0;\n        label_size_ = label_parser_->Value().size;\n      }\n      CHECK_LT(label_ptr_, label_size_);\n      out_.data[1] = AsTBlob(label_parser_->Value()[label_ptr_++], param_.label_shape);\n    } else {\n      out_.data[1] = dummy_label;\n    }\n    return true;\n  }\n\n private:\n  inline TBlob AsTBlob(const dmlc::Row<uint32_t, DType>& row, const mxnet::TShape& shape) {\n    CHECK_EQ(row.length, shape.Size())\n        << \"The data size in CSV do not match size of shape: \"\n        << \"specified shape=\" << shape << \", the csv row-length=\" << row.length;\n    const DType* ptr = row.value;\n    return TBlob((DType*)ptr, shape, cpu::kDevMask, 0);  // NOLINT(*)\n  }\n  // dummy label\n  mshadow::TensorContainer<cpu, 1, DType> dummy_label;\n  std::unique_ptr<dmlc::Parser<uint32_t, DType> > label_parser_;\n  std::unique_ptr<dmlc::Parser<uint32_t, DType> > data_parser_;\n};\n\nclass CSVIter : public IIterator<DataInst> {\n public:\n  CSVIter()           = default;\n  ~CSVIter() override = default;\n\n  // intialize iterator loads data in\n  void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override {\n    param_.InitAllowUnknown(kwargs);\n    bool dtype_has_value = false;\n    int target_dtype     = -1;\n    for (const auto& arg : kwargs) {\n      if (arg.first == \"dtype\") {\n        dtype_has_value = true;\n        if (arg.second == \"int32\") {\n          target_dtype = mshadow::kInt32;\n        } else if (arg.second == \"int64\") {\n          target_dtype = mshadow::kInt64;\n        } else if (arg.second == \"float32\") {\n          target_dtype = mshadow::kFloat32;\n        } else {\n          CHECK(false) << arg.second << \" is not supported for CSVIter\";\n        }\n      }\n    }\n    if (dtype_has_value && target_dtype == mshadow::kInt32) {\n      iterator_.reset(reinterpret_cast<CSVIterBase*>(new CSVIterTyped<int32_t>()));\n    } else if (dtype_has_value && target_dtype == mshadow::kInt64) {\n      iterator_.reset(reinterpret_cast<CSVIterBase*>(new CSVIterTyped<int64_t>()));\n    } else if (!dtype_has_value || target_dtype == mshadow::kFloat32) {\n      iterator_.reset(reinterpret_cast<CSVIterBase*>(new CSVIterTyped<float>()));\n    }\n    iterator_->Init(kwargs);\n  }\n\n  void BeforeFirst() override {\n    iterator_->BeforeFirst();\n  }\n\n  bool Next() override {\n    return iterator_->Next();\n  }\n\n  const DataInst& Value() const override {\n    return iterator_->Value();\n  }\n\n private:\n  CSVIterParam param_;\n  std::unique_ptr<CSVIterBase> iterator_;\n};\n\nDMLC_REGISTER_PARAMETER(CSVIterParam);\n\nMXNET_REGISTER_IO_ITER(CSVIter)\n    .describe(R\"code(Returns the CSV file iterator.\n\nIn this function, the `data_shape` parameter is used to set the shape of each line of the input data.\nIf a row in an input file is `1,2,3,4,5,6`` and `data_shape` is (3,2), that row\nwill be reshaped, yielding the array [[1,2],[3,4],[5,6]] of shape (3,2).\n\nBy default, the `CSVIter` has `round_batch` parameter set to ``True``. So, if `batch_size`\nis 3 and there are 4 total rows in CSV file, 2 more examples\nare consumed at the first round. If `reset` function is called after first round,\nthe call is ignored and remaining examples are returned in the second round.\n\nIf one wants all the instances in the second round after calling `reset`, make sure\nto set `round_batch` to False.\n\nIf ``data_csv = 'data/'`` is set, then all the files in this directory will be read.\n\n``reset()`` is expected to be called only after a complete pass of data.\n\nBy default, the CSVIter parses all entries in the data file as float32 data type,\nif `dtype` argument is set to be 'int32' or 'int64' then CSVIter will parse all entries in the file\nas int32 or int64 data type accordingly.\n\nExamples::\n\n  // Contents of CSV file ``data/data.csv``.\n  1,2,3\n  2,3,4\n  3,4,5\n  4,5,6\n\n  // Creates a `CSVIter` with `batch_size`=2 and default `round_batch`=True.\n  CSVIter = mx.io.CSVIter(data_csv = 'data/data.csv', data_shape = (3,),\n  batch_size = 2)\n\n  // Two batches read from the above iterator are as follows:\n  [[ 1.  2.  3.]\n  [ 2.  3.  4.]]\n  [[ 3.  4.  5.]\n  [ 4.  5.  6.]]\n\n  // Creates a `CSVIter` with default `round_batch` set to True.\n  CSVIter = mx.io.CSVIter(data_csv = 'data/data.csv', data_shape = (3,),\n  batch_size = 3)\n\n  // Two batches read from the above iterator in the first pass are as follows:\n  [[1.  2.  3.]\n  [2.  3.  4.]\n  [3.  4.  5.]]\n\n  [[4.  5.  6.]\n  [1.  2.  3.]\n  [2.  3.  4.]]\n\n  // Now, `reset` method is called.\n  CSVIter.reset()\n\n  // Batch read from the above iterator in the second pass is as follows:\n  [[ 3.  4.  5.]\n  [ 4.  5.  6.]\n  [ 1.  2.  3.]]\n\n  // Creates a `CSVIter` with `round_batch`=False.\n  CSVIter = mx.io.CSVIter(data_csv = 'data/data.csv', data_shape = (3,),\n  batch_size = 3, round_batch=False)\n\n  // Contents of two batches read from the above iterator in both passes, after calling\n  // `reset` method before second pass, is as follows:\n  [[1.  2.  3.]\n  [2.  3.  4.]\n  [3.  4.  5.]]\n\n  [[4.  5.  6.]\n  [2.  3.  4.]\n  [3.  4.  5.]]\n\n  // Creates a 'CSVIter' with `dtype`='int32'\n  CSVIter = mx.io.CSVIter(data_csv = 'data/data.csv', data_shape = (3,),\n  batch_size = 3, round_batch=False, dtype='int32')\n\n  // Contents of two batches read from the above iterator in both passes, after calling\n  // `reset` method before second pass, is as follows:\n  [[1  2  3]\n  [2  3  4]\n  [3  4  5]]\n\n  [[4  5  6]\n  [2  3  4]\n  [3  4  5]]\n\n)code\" ADD_FILELINE)\n    .add_arguments(CSVIterParam::__FIELDS__())\n    .add_arguments(BatchParam::__FIELDS__())\n    .add_arguments(PrefetcherParam::__FIELDS__())\n    .set_body([]() { return new PrefetcherIter(new BatchLoader(new CSVIter())); });\n\n}  // namespace io\n}  // namespace mxnet\n"
  },
  {
    "path": "src/io/iter_image_det_recordio.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file iter_image_recordio-inl.hpp\n * \\brief recordio data iterator\n */\n#include <mxnet/io.h>\n#include <dmlc/base.h>\n#include <dmlc/io.h>\n#include <dmlc/omp.h>\n#include <dmlc/common.h>\n#include <dmlc/input_split_shuffle.h>\n#include <dmlc/logging.h>\n#include <dmlc/parameter.h>\n#include <dmlc/recordio.h>\n#include <dmlc/threadediter.h>\n#include <memory>\n#include <unordered_map>\n#include <vector>\n#include <cstdlib>\n#include \"./inst_vector.h\"\n#include \"./image_recordio.h\"\n#include \"./image_augmenter.h\"\n#include \"./image_iter_common.h\"\n#include \"./iter_prefetcher.h\"\n#include \"./iter_normalize.h\"\n#include \"./iter_batchloader.h\"\n\nnamespace mxnet {\nnamespace io {\n/*! \\brief data structure to hold labels for image detection tasks\n *  support arbitrary label_width\n */\nclass ImageDetLabelMap {\n public:\n  /*!\n   * \\brief initialize the label list into memory\n   * \\param path_imglist path to the image list\n   * \\param label_width predefined label_width, -1 for arbitrary width\n   */\n  explicit ImageDetLabelMap(const char* path_imglist, int label_width, bool silent) {\n    image_index_.clear();\n    label_.clear();\n    idx2label_.clear();\n    dmlc::InputSplit* fi = dmlc::InputSplit::Create(path_imglist, 0, 1, \"text\");\n    dmlc::InputSplit::Blob rec;\n    while (fi->NextRecord(&rec)) {\n      // quick manual parsing\n      char* p   = reinterpret_cast<char*>(rec.dptr);\n      char* end = p + rec.size;\n      // skip space\n      while (isspace(*p) && p != end)\n        ++p;\n      image_index_.push_back(static_cast<size_t>(atol(p)));\n      size_t start_pos = label_.size();\n      if (label_width > 0) {\n        // provided label_width > 0, require width check\n        for (int i = 0; i < label_width; ++i) {\n          // skip till space\n          while (!isspace(*p) && p != end)\n            ++p;\n          // skip space\n          while (isspace(*p) && p != end)\n            ++p;\n          CHECK(p != end) << \"Bad ImageList format\";\n          label_.push_back(static_cast<real_t>(atof(p)));\n        }\n        CHECK_EQ(label_.size() - start_pos, label_width);\n      } else {\n        // arbitrary label width for each sample\n        while (!isspace(*p) && p != end)\n          ++p;\n        while (isspace(*p) && p != end)\n          ++p;\n        char* curr = p;\n        CHECK(curr != end) << \"Bad ImageList format\";\n        while (!isspace(*p) && p != end)\n          ++p;\n        while (isspace(*p) && p != end)\n          ++p;\n        char* next = p;\n        while (next != end) {\n          label_.push_back(static_cast<real_t>(atof(curr)));\n          curr = next;\n          while (!isspace(*next) && next != end)\n            ++next;\n          while (isspace(*next) && next != end)\n            ++next;\n        }\n        // skip the last one which should be the image_path\n        CHECK_GT(label_.size(), start_pos) << \"Bad ImageList format: empty label\";\n      }\n      // record label start_pos and width in map\n      idx2label_[image_index_.back()] =\n          std::pair<size_t, size_t>(start_pos, label_.size() - start_pos);\n    }\n    delete fi;\n    if (!silent) {\n      LOG(INFO) << \"Loaded ImageList from \" << path_imglist << ' ' << image_index_.size()\n                << \" Image records\";\n    }\n  }\n\n  /*! \\brief find a label for corresponding index, return vector as copy */\n  inline std::vector<float> FindCopy(size_t imid) const {\n    std::unordered_map<size_t, std::pair<size_t, size_t>>::const_iterator it =\n        idx2label_.find(imid);\n    CHECK(it != idx2label_.end()) << \"fail to find imagelabel for id \" << imid;\n    const real_t* ptr = dmlc::BeginPtr(label_) + it->second.first;\n    return std::vector<float>(ptr, ptr + it->second.second);\n  }\n\n  /*! \\brief Iterate through all labels, find the Maximum width of labels */\n  inline size_t MaxLabelWidth() const {\n    size_t max_width = 0;\n    for (auto i : idx2label_) {\n      size_t width = i.second.second;\n      if (width > max_width)\n        max_width = width;\n    }\n    return max_width;\n  }\n\n private:\n  /*! \\brief vector storing image indices */\n  std::vector<size_t> image_index_;\n  /*! \\brief vectors storing raw labels in 1D */\n  std::vector<real_t> label_;\n  /*! \\brief map storing image index to pair<label_start_pos, label_end_pos> */\n  std::unordered_map<size_t, std::pair<size_t, size_t>> idx2label_;\n};  // class ImageDetLabelMap\n\n// Define image record parser parameters\nstruct ImageDetRecParserParam : public dmlc::Parameter<ImageDetRecParserParam> {\n  /*! \\brief path to image list */\n  std::string path_imglist;\n  /*! \\brief path to image recordio */\n  std::string path_imgrec;\n  /*! \\brief a sequence of names of image augmenters, seperated by , */\n  std::string aug_seq;\n  /*! \\brief label-width, use -1 for variable width */\n  int label_width;\n  /*! \\brief input shape */\n  mxnet::TShape data_shape;\n  /*! \\brief number of threads */\n  int preprocess_threads;\n  /*! \\brief whether to remain silent */\n  bool verbose;\n  /*! \\brief partition the data into multiple parts */\n  int num_parts;\n  /*! \\brief the index of the part will read*/\n  int part_index;\n  /*! \\brief the size of a shuffle chunk*/\n  size_t shuffle_chunk_size;\n  /*! \\brief the seed for chunk shuffling*/\n  int shuffle_chunk_seed;\n  /*! \\brief pad label to specified length, -1 for auto estimate in whole dataset */\n  int label_pad_width;\n  /*! \\brief labe padding value */\n  float label_pad_value;\n\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(ImageDetRecParserParam) {\n    DMLC_DECLARE_FIELD(path_imglist).set_default(\"\").describe(\"Dataset Param: Path to image list.\");\n    DMLC_DECLARE_FIELD(path_imgrec)\n        .set_default(\"./data/imgrec.rec\")\n        .describe(\"Dataset Param: Path to image record file.\");\n    DMLC_DECLARE_FIELD(aug_seq)\n        .set_default(\"det_aug_default\")\n        .describe(\n            \"Augmentation Param: the augmenter names to represent\"\n            \" sequence of augmenters to be applied, seperated by comma.\"\n            \" Additional keyword parameters will be seen by these augmenters.\"\n            \" Make sure you don't use normal augmenters for detection tasks.\");\n    DMLC_DECLARE_FIELD(label_width)\n        .set_default(-1)\n        .describe(\"Dataset Param: How many labels for an image, -1 for variable label size.\");\n    DMLC_DECLARE_FIELD(data_shape)\n        .set_expect_ndim(3)\n        .enforce_nonzero()\n        .describe(\"Dataset Param: Shape of each instance generated by the DataIter.\");\n    DMLC_DECLARE_FIELD(preprocess_threads)\n        .set_lower_bound(1)\n        .set_default(4)\n        .describe(\"Backend Param: Number of thread to do preprocessing.\");\n    DMLC_DECLARE_FIELD(verbose).set_default(true).describe(\n        \"Auxiliary Param: Whether to output parser information.\");\n    DMLC_DECLARE_FIELD(num_parts).set_default(1).describe(\"partition the data into multiple parts\");\n    DMLC_DECLARE_FIELD(part_index).set_default(0).describe(\"the index of the part will read\");\n    DMLC_DECLARE_FIELD(shuffle_chunk_size)\n        .set_default(0)\n        .describe(\n            \"the size(MB) of the shuffle chunk, used with shuffle=True,\"\n            \" it can enable global shuffling\");\n    DMLC_DECLARE_FIELD(shuffle_chunk_seed).set_default(0).describe(\"the seed for chunk shuffling\");\n    DMLC_DECLARE_FIELD(label_pad_width)\n        .set_default(0)\n        .describe(\"pad output label width if set larger than 0, -1 for auto estimate\");\n    DMLC_DECLARE_FIELD(label_pad_value)\n        .set_default(-1.f)\n        .describe(\"label padding value if enabled\");\n  }\n};\n\n// parser to parse image recordio\ntemplate <typename DType>\nclass ImageDetRecordIOParser {\n public:\n  // initialize the parser\n  inline void Init(const std::vector<std::pair<std::string, std::string>>& kwargs);\n\n  // set record to the head\n  inline void BeforeFirst() {\n    return source_->BeforeFirst();\n  }\n  // parse next set of records, return an array of\n  // instance vector to the user\n  virtual inline bool ParseNext(std::vector<InstVector<DType>>* out);\n\n protected:\n  // magic number to see prng\n  static const int kRandMagic = 233;\n  /*! \\brief parameters */\n  ImageDetRecParserParam param_;\n#if MXNET_USE_OPENCV\n  /*! \\brief augmenters */\n  std::vector<std::vector<std::unique_ptr<ImageAugmenter>>> augmenters_;\n#endif\n  /*! \\brief random samplers */\n  std::vector<std::unique_ptr<common::RANDOM_ENGINE>> prnds_;\n  /*! \\brief data source */\n  std::unique_ptr<dmlc::InputSplit> source_;\n  /*! \\brief label information, if any */\n  std::unique_ptr<ImageDetLabelMap> label_map_;\n  /*! \\brief temp space */\n  mshadow::TensorContainer<cpu, 3> img_;\n  /*! \\brief OMPException obj to store and rethrow exceptions from omp blocks*/\n  dmlc::OMPException omp_exc_;\n};\n\ntemplate <typename DType>\ninline void ImageDetRecordIOParser<DType>::Init(\n    const std::vector<std::pair<std::string, std::string>>& kwargs) {\n#if MXNET_USE_OPENCV\n  // initialize parameter\n  // init image rec param\n  param_.InitAllowUnknown(kwargs);\n  int maxthread, threadget;\n#pragma omp parallel\n  {\n    // be conservative, set number of real cores - 1\n    maxthread = std::max(omp_get_num_procs() - 1, 1);\n  }\n  param_.preprocess_threads = std::min(maxthread, param_.preprocess_threads);\n#pragma omp parallel num_threads(param_.preprocess_threads)\n  { threadget = omp_get_num_threads(); }\n  param_.preprocess_threads = threadget;\n\n  std::vector<std::string> aug_names = dmlc::Split(param_.aug_seq, ',');\n  augmenters_.clear();\n  augmenters_.resize(threadget);\n  // setup decoders\n  for (int i = 0; i < threadget; ++i) {\n    for (const auto& aug_name : aug_names) {\n      augmenters_[i].emplace_back(ImageAugmenter::Create(aug_name));\n      augmenters_[i].back()->Init(kwargs);\n    }\n    prnds_.emplace_back(new common::RANDOM_ENGINE((i + 1) * kRandMagic));\n  }\n  if (param_.path_imglist.length() != 0) {\n    label_map_ = std::make_unique<ImageDetLabelMap>(\n        param_.path_imglist.c_str(), param_.label_width, !param_.verbose);\n  }\n  CHECK(param_.path_imgrec.length() != 0) << \"ImageDetRecordIOIterator: must specify image_rec\";\n\n  if (param_.verbose) {\n    LOG(INFO) << \"ImageDetRecordIOParser: \" << param_.path_imgrec << \", use \" << threadget\n              << \" threads for decoding..\";\n  }\n  source_.reset(dmlc::InputSplit::Create(\n      param_.path_imgrec.c_str(), param_.part_index, param_.num_parts, \"recordio\"));\n\n  // estimate padding width for labels\n  int max_label_width = 0;\n  if (label_map_ != nullptr) {\n    max_label_width = label_map_->MaxLabelWidth();\n  } else {\n    // iterate through recordio\n    dmlc::InputSplit::Blob chunk;\n    while (source_->NextChunk(&chunk)) {\n#pragma omp parallel num_threads(param_.preprocess_threads)\n      {\n        omp_exc_.Run([&] {\n          CHECK(omp_get_num_threads() == param_.preprocess_threads);\n          int max_width = 0;\n          int tid       = omp_get_thread_num();\n          dmlc::RecordIOChunkReader reader(chunk, tid, param_.preprocess_threads);\n          ImageRecordIO rec;\n          dmlc::InputSplit::Blob blob;\n          while (reader.NextRecord(&blob)) {\n            rec.Load(blob.dptr, blob.size);\n            if (rec.label != nullptr) {\n              if (param_.label_width > 0) {\n                CHECK_EQ(param_.label_width, rec.num_label) << \"rec file provide \" << rec.num_label\n                                                            << \"-dimensional label \"\n                                                               \"but label_width is set to \"\n                                                            << param_.label_width;\n              }\n              // update max value\n              max_width = std::max(max_width, rec.num_label);\n            } else {\n              LOG(FATAL) << \"Not enough label packed in img_list or rec file.\";\n            }\n          }\n#pragma omp critical\n          { max_label_width = std::max(max_label_width, max_width); }\n        });\n      }\n      omp_exc_.Rethrow();\n    }\n  }\n  if (max_label_width > param_.label_pad_width) {\n    if (param_.label_pad_width > 0) {\n      LOG(FATAL) << \"ImageDetRecordIOParser: label_pad_width: \" << param_.label_pad_width\n                 << \" smaller than estimated width: \" << max_label_width;\n    }\n    param_.label_pad_width = max_label_width;\n  }\n  if (param_.verbose) {\n    LOG(INFO) << \"ImageDetRecordIOParser: \" << param_.path_imgrec\n              << \", label padding width: \" << param_.label_pad_width;\n  }\n\n  source_.reset(dmlc::InputSplit::Create(\n      param_.path_imgrec.c_str(), param_.part_index, param_.num_parts, \"recordio\"));\n\n  if (param_.shuffle_chunk_size > 0) {\n    if (param_.shuffle_chunk_size > 4096) {\n      LOG(INFO) << \"Chunk size: \" << param_.shuffle_chunk_size\n                << \" MB which is larger than 4096 MB, please set \"\n                   \"smaller chunk size\";\n    }\n    if (param_.shuffle_chunk_size < 4) {\n      LOG(INFO) << \"Chunk size: \" << param_.shuffle_chunk_size\n                << \" MB which is less than 4 MB, please set \"\n                   \"larger chunk size\";\n    }\n    // 1.1 ratio is for a bit more shuffle parts to avoid boundary issue\n    unsigned num_shuffle_parts = std::ceil(\n        source_->GetTotalSize() * 1.1 / (param_.num_parts * (param_.shuffle_chunk_size << 20UL)));\n\n    if (num_shuffle_parts > 1) {\n      source_.reset(dmlc::InputSplitShuffle::Create(param_.path_imgrec.c_str(),\n                                                    param_.part_index,\n                                                    param_.num_parts,\n                                                    \"recordio\",\n                                                    num_shuffle_parts,\n                                                    param_.shuffle_chunk_seed));\n    }\n    source_->HintChunkSize(param_.shuffle_chunk_size << 17UL);\n  } else {\n    // use 64 MB chunk when possible\n    source_->HintChunkSize(8 << 20UL);\n  }\n#else\n  LOG(FATAL) << \"ImageDetRec need opencv to process\";\n#endif\n}\n\ntemplate <typename DType>\ninline bool ImageDetRecordIOParser<DType>::ParseNext(std::vector<InstVector<DType>>* out_vec) {\n  CHECK(source_ != nullptr);\n  dmlc::InputSplit::Blob chunk;\n  if (!source_->NextChunk(&chunk))\n    return false;\n#if MXNET_USE_OPENCV\n  // save opencv out\n  out_vec->resize(param_.preprocess_threads);\n#pragma omp parallel num_threads(param_.preprocess_threads)\n  {\n    omp_exc_.Run([&] {\n      CHECK(omp_get_num_threads() == param_.preprocess_threads);\n      int tid = omp_get_thread_num();\n      dmlc::RecordIOChunkReader reader(chunk, tid, param_.preprocess_threads);\n      ImageRecordIO rec;\n      dmlc::InputSplit::Blob blob;\n      // image data\n      InstVector<DType>& out = (*out_vec)[tid];\n      out.Clear();\n      while (reader.NextRecord(&blob)) {\n        // Opencv decode and augments\n        cv::Mat res;\n        rec.Load(blob.dptr, blob.size);\n        cv::Mat buf(1, rec.content_size, CV_8U, rec.content);\n        switch (param_.data_shape[0]) {\n          case 1:\n            res = cv::imdecode(buf, 0);\n            break;\n          case 3:\n            res = cv::imdecode(buf, 1);\n            break;\n          case 4:\n            // -1 to keep the number of channel of the encoded image, and not\n            // force gray or color.\n            res = cv::imdecode(buf, -1);\n            CHECK_EQ(res.channels(), 4) << \"Invalid image with index \" << rec.image_index()\n                                        << \". Expected 4 channels, got \" << res.channels();\n            break;\n          default:\n            LOG(FATAL) << \"Invalid output shape \" << param_.data_shape;\n        }\n        const int n_channels = res.channels();\n        // load label before augmentations\n        std::vector<float> label_buf;\n        if (this->label_map_ != nullptr) {\n          label_buf = label_map_->FindCopy(rec.image_index());\n        } else if (rec.label != nullptr) {\n          if (param_.label_width > 0) {\n            CHECK_EQ(param_.label_width, rec.num_label) << \"rec file provide \" << rec.num_label\n                                                        << \"-dimensional label \"\n                                                           \"but label_width is set to \"\n                                                        << param_.label_width;\n          }\n          label_buf.assign(rec.label, rec.label + rec.num_label);\n        } else {\n          LOG(FATAL) << \"Not enough label packed in img_list or rec file.\";\n        }\n        for (auto& aug : this->augmenters_[tid]) {\n          res = aug->Process(res, &label_buf, this->prnds_[tid].get());\n        }\n        out.Push(static_cast<unsigned>(rec.image_index()),\n                 mshadow::Shape3(n_channels, param_.data_shape[1], param_.data_shape[2]),\n                 mshadow::Shape1(param_.label_pad_width + 4));\n\n        mshadow::Tensor<cpu, 3, DType> data = out.data().Back();\n\n        // For RGB or RGBA data, swap the B and R channel:\n        // OpenCV store as BGR (or BGRA) and we want RGB (or RGBA)\n        std::vector<int> swap_indices;\n        if (n_channels == 1)\n          swap_indices = {0};\n        if (n_channels == 3)\n          swap_indices = {2, 1, 0};\n        if (n_channels == 4)\n          swap_indices = {2, 1, 0, 3};\n\n        for (int i = 0; i < res.rows; ++i) {\n          uchar* im_data = res.ptr<uchar>(i);\n          for (int j = 0; j < res.cols; ++j) {\n            for (int k = 0; k < n_channels; ++k) {\n              data[k][i][j] = im_data[swap_indices[k]];\n            }\n            im_data += n_channels;\n          }\n        }\n        mshadow::Tensor<cpu, 1> label = out.label().Back();\n        label                         = param_.label_pad_value;\n        // store info for real data_shape and label_width\n        label[0] = res.channels();\n        label[1] = res.rows;\n        label[2] = res.cols;\n        label[3] = label_buf.size();\n        mshadow::Copy(\n            label.Slice(4, 4 + label_buf.size()),\n            mshadow::Tensor<cpu, 1>(dmlc::BeginPtr(label_buf), mshadow::Shape1(label_buf.size())));\n        res.release();\n      }\n    });\n  }\n#else\n  LOG(FATAL) << \"Opencv is needed for image decoding and augmenting.\";\n#endif\n  omp_exc_.Rethrow();\n  return true;\n}\n\n// Define image record parameters\nstruct ImageDetRecordParam : public dmlc::Parameter<ImageDetRecordParam> {\n  /*! \\brief whether to do shuffle */\n  bool shuffle;\n  /*! \\brief random seed */\n  int seed;\n  /*! \\brief whether to remain silent */\n  bool verbose;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(ImageDetRecordParam) {\n    DMLC_DECLARE_FIELD(shuffle).set_default(false).describe(\n        \"Augmentation Param: Whether to shuffle data.\");\n    DMLC_DECLARE_FIELD(seed).set_default(0).describe(\"Augmentation Param: Random Seed.\");\n    DMLC_DECLARE_FIELD(verbose).set_default(true).describe(\n        \"Auxiliary Param: Whether to output information.\");\n  }\n};\n\n// iterator on image recordio\ntemplate <typename DType = real_t>\nclass ImageDetRecordIter : public IIterator<DataInst> {\n public:\n  ImageDetRecordIter() : data_(nullptr) {}\n  // destructor\n  ~ImageDetRecordIter() override {\n    iter_.Destroy();\n    delete data_;\n  }\n  // constructor\n  void Init(const std::vector<std::pair<std::string, std::string>>& kwargs) override {\n    param_.InitAllowUnknown(kwargs);\n    // use the kwarg to init parser\n    parser_.Init(kwargs);\n    // prefetch at most 4 minbatches\n    iter_.set_max_capacity(4);\n    // init thread iter\n    iter_.Init(\n        [this](std::vector<InstVector<DType>>** dptr) {\n          if (*dptr == nullptr) {\n            *dptr = new std::vector<InstVector<DType>>();\n          }\n          return parser_.ParseNext(*dptr);\n        },\n        [this]() { parser_.BeforeFirst(); });\n    inst_ptr_ = 0;\n    rnd_.seed(kRandMagic + param_.seed);\n  }\n  // before first\n  void BeforeFirst() override {\n    iter_.BeforeFirst();\n    inst_order_.clear();\n    inst_ptr_ = 0;\n  }\n\n  bool Next() override {\n    while (true) {\n      if (inst_ptr_ < inst_order_.size()) {\n        std::pair<unsigned, unsigned> p = inst_order_[inst_ptr_];\n        out_                            = (*data_)[p.first][p.second];\n        ++inst_ptr_;\n        return true;\n      } else {\n        if (data_ != nullptr)\n          iter_.Recycle(&data_);\n        if (!iter_.Next(&data_))\n          return false;\n        inst_order_.clear();\n        for (unsigned i = 0; i < data_->size(); ++i) {\n          const InstVector<DType>& tmp = (*data_)[i];\n          for (unsigned j = 0; j < tmp.Size(); ++j) {\n            inst_order_.emplace_back(i, j);\n          }\n        }\n        // shuffle instance order if needed\n        if (param_.shuffle != 0) {\n          std::shuffle(inst_order_.begin(), inst_order_.end(), rnd_);\n        }\n        inst_ptr_ = 0;\n      }\n    }\n    return false;\n  }\n\n  const DataInst& Value() const override {\n    return out_;\n  }\n\n private:\n  // random magic\n  static const int kRandMagic = 233;\n  // output instance\n  DataInst out_;\n  // data ptr\n  size_t inst_ptr_;\n  // internal instance order\n  std::vector<std::pair<unsigned, unsigned>> inst_order_;\n  // data\n  std::vector<InstVector<DType>>* data_;\n  // internal parser\n  ImageDetRecordIOParser<DType> parser_;\n  // backend thread\n  dmlc::ThreadedIter<std::vector<InstVector<DType>>> iter_;\n  // parameters\n  ImageDetRecordParam param_;\n  // random number generator\n  common::RANDOM_ENGINE rnd_;\n};\n\nDMLC_REGISTER_PARAMETER(ImageDetRecParserParam);\nDMLC_REGISTER_PARAMETER(ImageDetRecordParam);\n\nMXNET_REGISTER_IO_ITER(ImageDetRecordIter)\n    .describe(\"Create iterator for image detection dataset packed in recordio.\")\n    .add_arguments(ImageDetRecParserParam::__FIELDS__())\n    .add_arguments(ImageDetRecordParam::__FIELDS__())\n    .add_arguments(BatchParam::__FIELDS__())\n    .add_arguments(PrefetcherParam::__FIELDS__())\n    .add_arguments(ListDefaultDetAugParams())\n    .add_arguments(ImageDetNormalizeParam::__FIELDS__())\n    .set_body([]() {\n      return new PrefetcherIter(\n          new BatchLoader(new ImageDetNormalizeIter(new ImageDetRecordIter<real_t>())));\n    });\n}  // namespace io\n}  // namespace mxnet\n"
  },
  {
    "path": "src/io/iter_image_recordio.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file iter_image_recordio-inl.hpp\n * \\brief recordio data iterator\n */\n#include <mxnet/io.h>\n#include <dmlc/base.h>\n#include <dmlc/io.h>\n#include <dmlc/omp.h>\n#include <dmlc/common.h>\n#include <dmlc/input_split_shuffle.h>\n#include <dmlc/logging.h>\n#include <dmlc/parameter.h>\n#include <dmlc/recordio.h>\n#include <dmlc/threadediter.h>\n#include <memory>\n#include <unordered_map>\n#include <vector>\n#include <cstdlib>\n#include \"./image_iter_common.h\"\n#include \"./inst_vector.h\"\n#include \"./image_recordio.h\"\n#include \"./image_augmenter.h\"\n#include \"./iter_prefetcher.h\"\n#include \"./iter_normalize.h\"\n#include \"./iter_batchloader.h\"\n\nnamespace mxnet {\nnamespace io {\n// parser to parse image recordio\ntemplate <typename DType>\nclass ImageRecordIOParser {\n public:\n  // initialize the parser\n  inline void Init(const std::vector<std::pair<std::string, std::string>>& kwargs);\n\n  // set record to the head\n  inline void BeforeFirst() {\n    return source_->BeforeFirst();\n  }\n  // parse next set of records, return an array of\n  // instance vector to the user\n  inline bool ParseNext(std::vector<InstVector<DType>>* out);\n\n private:\n  // magic number to see prng\n  static const int kRandMagic = 111;\n  /*! \\brief parameters */\n  ImageRecParserParam param_;\n#if MXNET_USE_OPENCV\n  /*! \\brief augmenters */\n  std::vector<std::vector<std::unique_ptr<ImageAugmenter>>> augmenters_;\n#endif\n  /*! \\brief random samplers */\n  std::vector<std::unique_ptr<common::RANDOM_ENGINE>> prnds_;\n  /*! \\brief data source */\n  std::unique_ptr<dmlc::InputSplit> source_;\n  /*! \\brief label information, if any */\n  std::unique_ptr<ImageLabelMap> label_map_;\n  /*! \\brief temp space */\n  mshadow::TensorContainer<cpu, 3> img_;\n};\n\ntemplate <typename DType>\ninline void ImageRecordIOParser<DType>::Init(\n    const std::vector<std::pair<std::string, std::string>>& kwargs) {\n#if MXNET_USE_OPENCV\n  // initialize parameter\n  // init image rec param\n  param_.InitAllowUnknown(kwargs);\n  int maxthread, threadget;\n#pragma omp parallel\n  {\n    // be conservative, set number of real cores\n    maxthread = std::max(omp_get_num_procs() / 2 - 1, 1);\n  }\n  param_.preprocess_threads = std::min(maxthread, param_.preprocess_threads);\n#pragma omp parallel num_threads(param_.preprocess_threads)\n  { threadget = omp_get_num_threads(); }\n  param_.preprocess_threads = threadget;\n\n  std::vector<std::string> aug_names = dmlc::Split(param_.aug_seq, ',');\n  augmenters_.clear();\n  augmenters_.resize(threadget);\n  // setup decoders\n  for (int i = 0; i < threadget; ++i) {\n    for (const auto& aug_name : aug_names) {\n      augmenters_[i].emplace_back(ImageAugmenter::Create(aug_name));\n      augmenters_[i].back()->Init(kwargs);\n    }\n    prnds_.emplace_back(new common::RANDOM_ENGINE((i + 1) * kRandMagic));\n  }\n  if (param_.path_imglist.length() != 0) {\n    label_map_ = std::make_unique<ImageLabelMap>(\n        param_.path_imglist.c_str(), param_.label_width, !param_.verbose);\n  }\n  CHECK(param_.path_imgrec.length() != 0) << \"ImageRecordIOIterator: must specify image_rec\";\n\n  if (param_.verbose) {\n    LOG(INFO) << \"ImageRecordIOParser: \" << param_.path_imgrec << \", use \" << threadget\n              << \" threads for decoding..\";\n  }\n  source_.reset(dmlc::InputSplit::Create(\n      param_.path_imgrec.c_str(), param_.part_index, param_.num_parts, \"recordio\"));\n  if (param_.shuffle_chunk_size > 0) {\n    if (param_.shuffle_chunk_size > 4096) {\n      LOG(INFO) << \"Chunk size: \" << param_.shuffle_chunk_size\n                << \" MB which is larger than 4096 MB, please set \"\n                   \"smaller chunk size\";\n    }\n    if (param_.shuffle_chunk_size < 4) {\n      LOG(INFO) << \"Chunk size: \" << param_.shuffle_chunk_size\n                << \" MB which is less than 4 MB, please set \"\n                   \"larger chunk size\";\n    }\n    // 1.1 ratio is for a bit more shuffle parts to avoid boundary issue\n    unsigned num_shuffle_parts = std::ceil(\n        source_->GetTotalSize() * 1.1 / (param_.num_parts * (param_.shuffle_chunk_size << 20UL)));\n\n    if (num_shuffle_parts > 1) {\n      source_.reset(dmlc::InputSplitShuffle::Create(param_.path_imgrec.c_str(),\n                                                    param_.part_index,\n                                                    param_.num_parts,\n                                                    \"recordio\",\n                                                    num_shuffle_parts,\n                                                    param_.shuffle_chunk_seed));\n    }\n    source_->HintChunkSize(param_.shuffle_chunk_size << 17UL);\n  } else {\n    // use 64 MB chunk when possible\n    source_->HintChunkSize(8 << 20UL);\n  }\n#else\n  LOG(FATAL) << \"ImageRec need opencv to process\";\n#endif\n}\n\ntemplate <typename DType>\ninline bool ImageRecordIOParser<DType>::ParseNext(std::vector<InstVector<DType>>* out_vec) {\n  CHECK(source_ != nullptr);\n  dmlc::InputSplit::Blob chunk;\n  if (!source_->NextChunk(&chunk))\n    return false;\n#if MXNET_USE_OPENCV\n  // save opencv out\n  out_vec->resize(param_.preprocess_threads);\n#pragma omp parallel num_threads(param_.preprocess_threads)\n  {\n    CHECK(omp_get_num_threads() == param_.preprocess_threads);\n    int tid = omp_get_thread_num();\n    dmlc::RecordIOChunkReader reader(chunk, tid, param_.preprocess_threads);\n    ImageRecordIO rec;\n    dmlc::InputSplit::Blob blob;\n    // image data\n    InstVector<DType>& out = (*out_vec)[tid];\n    out.Clear();\n    while (reader.NextRecord(&blob)) {\n      // Opencv decode and augments\n      cv::Mat res;\n      rec.Load(blob.dptr, blob.size);\n      cv::Mat buf(1, rec.content_size, CV_8U, rec.content);\n      switch (param_.data_shape[0]) {\n        case 1:\n          res = cv::imdecode(buf, 0);\n          break;\n        case 3:\n          res = cv::imdecode(buf, 1);\n          break;\n        case 4:\n          // -1 to keep the number of channel of the encoded image, and not force gray or color.\n          res = cv::imdecode(buf, -1);\n          CHECK_EQ(res.channels(), 4) << \"Invalid image with index \" << rec.image_index()\n                                      << \". Expected 4 channels, got \" << res.channels();\n          break;\n        default:\n          LOG(FATAL) << \"Invalid output shape \" << param_.data_shape;\n      }\n      const int n_channels = res.channels();\n      for (auto& aug : augmenters_[tid]) {\n        res = aug->Process(res, nullptr, prnds_[tid].get());\n      }\n      out.Push(static_cast<unsigned>(rec.image_index()),\n               mshadow::Shape3(n_channels, res.rows, res.cols),\n               mshadow::Shape1(param_.label_width));\n\n      mshadow::Tensor<cpu, 3, DType> data = out.data().Back();\n\n      // For RGB or RGBA data, swap the B and R channel:\n      // OpenCV store as BGR (or BGRA) and we want RGB (or RGBA)\n      std::vector<int> swap_indices;\n      if (n_channels == 1)\n        swap_indices = {0};\n      if (n_channels == 3)\n        swap_indices = {2, 1, 0};\n      if (n_channels == 4)\n        swap_indices = {2, 1, 0, 3};\n\n      for (int i = 0; i < res.rows; ++i) {\n        uchar* im_data = res.ptr<uchar>(i);\n        for (int j = 0; j < res.cols; ++j) {\n          for (int k = 0; k < n_channels; ++k) {\n            data[k][i][j] = im_data[swap_indices[k]];\n          }\n          im_data += n_channels;\n        }\n      }\n\n      mshadow::Tensor<cpu, 1> label = out.label().Back();\n      if (label_map_ != nullptr) {\n        mshadow::Copy(label, label_map_->Find(rec.image_index()));\n      } else if (rec.label != nullptr) {\n        CHECK_EQ(param_.label_width, rec.num_label) << \"rec file provide \" << rec.num_label\n                                                    << \"-dimensional label \"\n                                                       \"but label_width is set to \"\n                                                    << param_.label_width;\n        mshadow::Copy(label, mshadow::Tensor<cpu, 1>(rec.label, mshadow::Shape1(rec.num_label)));\n      } else {\n        CHECK_EQ(param_.label_width, 1) << \"label_width must be 1 unless an imglist is provided \"\n                                           \"or the rec file is packed with multi dimensional label\";\n        label[0] = rec.header.label;\n      }\n      res.release();\n    }\n  }\n#else\n  LOG(FATAL) << \"Opencv is needed for image decoding and augmenting.\";\n#endif  // MXNET_USE_OPENCV\n  return true;\n}\n\n// iterator on image recordio\ntemplate <typename DType = real_t>\nclass ImageRecordIter : public IIterator<DataInst> {\n public:\n  ImageRecordIter() : data_(nullptr) {}\n  // destructor\n  ~ImageRecordIter() override {\n    iter_.Destroy();\n    delete data_;\n  }\n  // constructor\n  void Init(const std::vector<std::pair<std::string, std::string>>& kwargs) override {\n    param_.InitAllowUnknown(kwargs);\n    // use the kwarg to init parser\n    parser_.Init(kwargs);\n    // prefetch at most 4 minbatches\n    iter_.set_max_capacity(4);\n    // init thread iter\n    iter_.Init(\n        [this](std::vector<InstVector<DType>>** dptr) {\n          if (*dptr == nullptr) {\n            *dptr = new std::vector<InstVector<DType>>();\n          }\n          return parser_.ParseNext(*dptr);\n        },\n        [this]() { parser_.BeforeFirst(); });\n    inst_ptr_ = 0;\n    rnd_.seed(kRandMagic + param_.seed);\n  }\n  // before first\n  void BeforeFirst() override {\n    iter_.BeforeFirst();\n    inst_order_.clear();\n    inst_ptr_ = 0;\n  }\n\n  bool Next() override {\n    while (true) {\n      if (inst_ptr_ < inst_order_.size()) {\n        std::pair<unsigned, unsigned> p = inst_order_[inst_ptr_];\n        out_                            = (*data_)[p.first][p.second];\n        ++inst_ptr_;\n        return true;\n      } else {\n        if (data_ != nullptr)\n          iter_.Recycle(&data_);\n        if (!iter_.Next(&data_))\n          return false;\n        inst_order_.clear();\n        for (unsigned i = 0; i < data_->size(); ++i) {\n          const InstVector<DType>& tmp = (*data_)[i];\n          for (unsigned j = 0; j < tmp.Size(); ++j) {\n            inst_order_.emplace_back(i, j);\n          }\n        }\n        // shuffle instance order if needed\n        if (param_.shuffle != 0) {\n          std::shuffle(inst_order_.begin(), inst_order_.end(), rnd_);\n        }\n        inst_ptr_ = 0;\n      }\n    }\n    return false;\n  }\n\n  const DataInst& Value() const override {\n    return out_;\n  }\n\n private:\n  // random magic\n  static const int kRandMagic = 111;\n  // output instance\n  DataInst out_;\n  // data ptr\n  size_t inst_ptr_;\n  // internal instance order\n  std::vector<std::pair<unsigned, unsigned>> inst_order_;\n  // data\n  std::vector<InstVector<DType>>* data_;\n  // internal parser\n  ImageRecordIOParser<DType> parser_;\n  // backend thread\n  dmlc::ThreadedIter<std::vector<InstVector<DType>>> iter_;\n  // parameters\n  ImageRecordParam param_;\n  // random number generator\n  common::RANDOM_ENGINE rnd_;\n};\n\n// OLD VERSION - DEPRECATED\nMXNET_REGISTER_IO_ITER(ImageRecordIter_v1)\n    .describe(R\"code(Iterating on image RecordIO files\n\n.. note::\n\n  ``ImageRecordIter_v1`` is deprecated. Use ``ImageRecordIter`` instead.\n\n\nRead images batches from RecordIO files with a rich of data augmentation\noptions.\n\nOne can use ``tools/im2rec.py`` to pack individual image files into RecordIO\nfiles.\n\n)code\" ADD_FILELINE)\n    .add_arguments(ImageRecParserParam::__FIELDS__())\n    .add_arguments(ImageRecordParam::__FIELDS__())\n    .add_arguments(BatchParam::__FIELDS__())\n    .add_arguments(PrefetcherParam::__FIELDS__())\n    .add_arguments(ListDefaultAugParams())\n    .add_arguments(ImageNormalizeParam::__FIELDS__())\n    .set_body([]() {\n      return new PrefetcherIter(\n          new BatchLoader(new ImageNormalizeIter(new ImageRecordIter<real_t>())));\n    });\n\n// OLD VERSION - DEPRECATED\nMXNET_REGISTER_IO_ITER(ImageRecordUInt8Iter_v1)\n    .describe(R\"code(Iterating on image RecordIO files\n\n.. note::\n\n  ``ImageRecordUInt8Iter_v1`` is deprecated. Use ``ImageRecordUInt8Iter`` instead.\n\nThis iterator is identical to ``ImageRecordIter`` except for using ``uint8`` as\nthe data type instead of ``float``.\n\n)code\" ADD_FILELINE)\n    .add_arguments(ImageRecParserParam::__FIELDS__())\n    .add_arguments(ImageRecordParam::__FIELDS__())\n    .add_arguments(BatchParam::__FIELDS__())\n    .add_arguments(PrefetcherParam::__FIELDS__())\n    .add_arguments(ListDefaultAugParams())\n    .set_body([]() { return new PrefetcherIter(new BatchLoader(new ImageRecordIter<uint8_t>())); });\n}  // namespace io\n}  // namespace mxnet\n"
  },
  {
    "path": "src/io/iter_image_recordio_2.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file iter_image_recordio_2.cc\n * \\brief new version of recordio data iterator\n */\n\n#include <mxnet/io.h>\n#include <dmlc/parameter.h>\n#include <dmlc/threadediter.h>\n#include <dmlc/input_split_shuffle.h>\n#include <dmlc/recordio.h>\n#include <dmlc/base.h>\n#include <dmlc/io.h>\n#include <dmlc/omp.h>\n#include <dmlc/common.h>\n#include <dmlc/timer.h>\n#include <memory>\n#include <type_traits>\n#if MXNET_USE_LIBJPEG_TURBO\n#include <turbojpeg.h>\n#endif\n#include \"./image_recordio.h\"\n#include \"./image_augmenter.h\"\n#include \"./image_iter_common.h\"\n#include \"./inst_vector.h\"\n#include \"../common/utils.h\"\n#include \"../profiler/profiler.h\"\n\nnamespace mxnet {\n\nnamespace io {\n// parser to parse image recordio\ntemplate <typename DType>\nclass ImageRecordIOParser2 {\n public:\n  // initialize the parser\n  inline void Init(const std::vector<std::pair<std::string, std::string>>& kwargs);\n\n  // set record to the head\n  inline void BeforeFirst() {\n    if (batch_param_.round_batch == 0 || !overflow) {\n      n_parsed_ = 0;\n      return source_->BeforeFirst();\n    } else {\n      overflow = false;\n    }\n  }\n  // parse next set of records, return an array of\n  // instance vector to the user\n  inline bool ParseNext(DataBatch* out);\n\n private:\n#if MXNET_USE_OPENCV\n  template <int n_channels>\n  void ProcessImage(const cv::Mat& res,\n                    mshadow::Tensor<cpu, 3, DType>* data_ptr,\n                    const bool is_mirrored,\n                    const float contrast_scaled,\n                    const float illumination_scaled);\n#if MXNET_USE_LIBJPEG_TURBO\n  cv::Mat TJimdecode(cv::Mat buf, int color);\n#endif\n#endif\n  inline size_t ParseChunk(DType* data_dptr,\n                           real_t* label_dptr,\n                           const size_t current_size,\n                           dmlc::InputSplit::Blob* chunk);\n  inline void CreateMeanImg();\n\n  // magic number to seed prng\n  static const int kRandMagic          = 111;\n  static const int kRandMagicNormalize = 0;\n  /*! \\brief parameters */\n  ImageRecParserParam param_;\n  ImageRecordParam record_param_;\n  BatchParam batch_param_;\n  ImageNormalizeParam normalize_param_;\n\n#if MXNET_USE_OPENCV\n  /*! \\brief augmenters */\n  std::vector<std::vector<std::unique_ptr<ImageAugmenter>>> augmenters_;\n#endif\n  /*! \\brief random samplers */\n  std::vector<std::unique_ptr<common::RANDOM_ENGINE>> prnds_;\n  common::RANDOM_ENGINE rnd_;\n  /*! \\brief data source */\n  std::unique_ptr<dmlc::InputSplit> source_;\n  /*! \\brief label information, if any */\n  std::unique_ptr<ImageLabelMap> label_map_;\n  /*! \\brief temporary results */\n  std::vector<InstVector<DType>> temp_;\n  /*! \\brief temp space */\n  mshadow::TensorContainer<cpu, 3> img_;\n  /*! \\brief internal instance order */\n  std::vector<std::pair<size_t, size_t>> inst_order_;\n  size_t inst_index_;\n  /*! \\brief internal counter tracking number of already parsed entries */\n  size_t n_parsed_;\n  /*! \\brief overflow marker */\n  bool overflow;\n  /*! \\brief unit size */\n  std::vector<size_t> unit_size_;\n  /*! \\brief mean image, if needed */\n  mshadow::TensorContainer<cpu, 3> meanimg_;\n  // whether to use legacy shuffle\n  // (without IndexedRecordIO support)\n  bool legacy_shuffle_;\n  // whether mean image is ready.\n  bool meanfile_ready_;\n  /*! \\brief OMPException obj to store and rethrow exceptions from omp blocks*/\n  dmlc::OMPException omp_exc_;\n};\n\ntemplate <typename DType>\ninline void ImageRecordIOParser2<DType>::Init(\n    const std::vector<std::pair<std::string, std::string>>& kwargs) {\n#if MXNET_USE_OPENCV\n  // initialize parameter\n  // init image rec param\n  param_.InitAllowUnknown(kwargs);\n  record_param_.InitAllowUnknown(kwargs);\n  batch_param_.InitAllowUnknown(kwargs);\n  normalize_param_.InitAllowUnknown(kwargs);\n  PrefetcherParam prefetch_param;\n  prefetch_param.InitAllowUnknown(kwargs);\n  n_parsed_ = 0;\n  overflow  = false;\n  rnd_.seed(kRandMagic + record_param_.seed);\n  int maxthread, threadget;\n  if (prefetch_param.ctx == PrefetcherParam::CtxType::kCPU) {\n    threadget = engine::OpenMP::Get()->GetRecommendedOMPThreadCount();\n  } else {\n#pragma omp parallel\n    {\n      // be conservative, set number of real cores\n      maxthread = std::max(omp_get_num_procs() / 2, 1);\n    }\n    param_.preprocess_threads = std::min(maxthread, param_.preprocess_threads);\n#pragma omp parallel num_threads(param_.preprocess_threads)\n    { threadget = omp_get_num_threads(); }\n  }\n  param_.preprocess_threads = threadget;\n\n  std::vector<std::string> aug_names = dmlc::Split(param_.aug_seq, ',');\n  augmenters_.clear();\n  augmenters_.resize(threadget);\n  // setup decoders\n  for (int i = 0; i < threadget; ++i) {\n    for (const auto& aug_name : aug_names) {\n      augmenters_[i].emplace_back(ImageAugmenter::Create(aug_name));\n      augmenters_[i].back()->Init(kwargs);\n    }\n    prnds_.emplace_back(new common::RANDOM_ENGINE((i + 1) * kRandMagic));\n  }\n  if (param_.path_imglist.length() != 0) {\n    label_map_ = std::make_unique<ImageLabelMap>(\n        param_.path_imglist.c_str(), param_.label_width, !param_.verbose);\n  }\n  CHECK(param_.path_imgrec.length() != 0) << \"ImageRecordIter2: must specify image_rec\";\n\n  if (param_.verbose) {\n    LOG(INFO) << \"ImageRecordIOParser2: \" << param_.path_imgrec << \", use \" << threadget\n              << \" threads for decoding..\";\n  }\n  legacy_shuffle_ = false;\n  if (param_.path_imgidx.length() != 0) {\n    source_.reset(dmlc::InputSplit::Create(param_.path_imgrec.c_str(),\n                                           param_.path_imgidx.c_str(),\n                                           param_.part_index,\n                                           param_.num_parts,\n                                           \"indexed_recordio\",\n                                           record_param_.shuffle,\n                                           record_param_.seed,\n                                           batch_param_.batch_size));\n  } else {\n    source_.reset(dmlc::InputSplit::Create(\n        param_.path_imgrec.c_str(), param_.part_index, param_.num_parts, \"recordio\"));\n    if (record_param_.shuffle)\n      legacy_shuffle_ = true;\n    if (param_.shuffle_chunk_size > 0) {\n      if (param_.shuffle_chunk_size > 4096) {\n        LOG(INFO) << \"Chunk size: \" << param_.shuffle_chunk_size\n                  << \" MB which is larger than 4096 MB, please set \"\n                     \"smaller chunk size\";\n      }\n      if (param_.shuffle_chunk_size < 4) {\n        LOG(INFO) << \"Chunk size: \" << param_.shuffle_chunk_size\n                  << \" MB which is less than 4 MB, please set \"\n                     \"larger chunk size\";\n      }\n      // 1.1 ratio is for a bit more shuffle parts to avoid boundary issue\n      size_t num_shuffle_parts = std::ceil(\n          source_->GetTotalSize() * 1.1 / (param_.num_parts * (param_.shuffle_chunk_size << 20UL)));\n\n      if (num_shuffle_parts > 1) {\n        source_.reset(dmlc::InputSplitShuffle::Create(param_.path_imgrec.c_str(),\n                                                      param_.part_index,\n                                                      param_.num_parts,\n                                                      \"recordio\",\n                                                      num_shuffle_parts,\n                                                      param_.shuffle_chunk_seed));\n      }\n      source_->HintChunkSize(param_.shuffle_chunk_size << 17UL);\n    } else {\n      // use 64 MB chunk when possible\n      source_->HintChunkSize(64 << 20UL);\n    }\n  }\n  // Normalize init\n  if (!std::is_same<DType, uint8_t>::value) {\n    meanimg_.set_pad(false);\n    meanfile_ready_ = false;\n    if (normalize_param_.mean_img.length() != 0) {\n      std::unique_ptr<dmlc::Stream> fi(\n          dmlc::Stream::Create(normalize_param_.mean_img.c_str(), \"r\", true));\n      if (fi.get() == nullptr) {\n        this->CreateMeanImg();\n      } else {\n        fi.reset(nullptr);\n        if (param_.verbose) {\n          LOG(INFO) << \"Load mean image from \" << normalize_param_.mean_img;\n        }\n        // use python compatible ndarray store format\n        std::vector<NDArray> data;\n        std::vector<std::string> keys;\n        {\n          std::unique_ptr<dmlc::Stream> fi(\n              dmlc::Stream::Create(normalize_param_.mean_img.c_str(), \"r\"));\n          NDArray::Load(fi.get(), &data, &keys);\n        }\n        CHECK_EQ(data.size(), 1) << \"Invalid mean image file format\";\n        data[0].WaitToRead();\n        mshadow::Tensor<cpu, 3> src = data[0].data().get<cpu, 3, real_t>();\n        meanimg_.Resize(src.shape_);\n        mshadow::Copy(meanimg_, src);\n        meanfile_ready_ = true;\n        if (param_.verbose) {\n          LOG(INFO) << \"Load mean image from \" << normalize_param_.mean_img << \" completed\";\n        }\n      }\n    }\n  }\n#else\n  LOG(FATAL) << \"ImageRec need opencv to process\";\n#endif\n}\n\ntemplate <typename DType>\ninline bool ImageRecordIOParser2<DType>::ParseNext(DataBatch* out) {\n  if (overflow) {\n    return false;\n  }\n  CHECK(source_ != nullptr);\n  dmlc::InputSplit::Blob chunk;\n  size_t current_size = 0;\n  out->index.resize(batch_param_.batch_size);\n\n  // InitBatch\n  if (out->data.size() == 0) {\n    // This assumes that DataInst given by\n    // InstVector contains only 2 elements in\n    // data vector (operator[] implementation)\n    out->data.resize(2);\n    unit_size_.resize(2);\n\n    std::vector<index_t> shape_vec;\n    shape_vec.push_back(batch_param_.batch_size);\n    for (index_t dim = 0; dim < param_.data_shape.ndim(); ++dim) {\n      shape_vec.push_back(param_.data_shape[dim]);\n    }\n    mxnet::TShape data_shape(shape_vec.begin(), shape_vec.end());\n\n    shape_vec.clear();\n    shape_vec.push_back(batch_param_.batch_size);\n    shape_vec.push_back(param_.label_width);\n    mxnet::TShape label_shape(shape_vec.begin(), shape_vec.end());\n\n    auto ctx    = Context::CPU(0);\n    auto dev_id = param_.device_id;\n    if (dev_id != -1) {\n      ctx = Context::CPUPinned(dev_id);\n    }\n\n    const std::string profiler_scope =\n        profiler::ProfilerScope::Get()->GetCurrentProfilerScope() + \"image_io:\";\n\n    out->data.at(0) = NDArray(data_shape, ctx, false, mshadow::DataType<DType>::kFlag);\n    out->data.at(0).AssignStorageInfo(profiler_scope, \"data\");\n    out->data.at(1) = NDArray(label_shape, ctx, false, mshadow::DataType<real_t>::kFlag);\n    out->data.at(1).AssignStorageInfo(profiler_scope, \"label\");\n    unit_size_[0] = param_.data_shape.Size();\n    unit_size_[1] = param_.label_width;\n  }\n\n  while (current_size < batch_param_.batch_size) {\n    // int n_to_copy;\n    size_t n_to_out = 0;\n    if (n_parsed_ == 0) {\n      if (source_->NextBatch(&chunk, batch_param_.batch_size)) {\n        inst_order_.clear();\n        inst_index_        = 0;\n        DType* data_dptr   = static_cast<DType*>(out->data[0].data().dptr_);\n        real_t* label_dptr = static_cast<real_t*>(out->data[1].data().dptr_);\n        if (!legacy_shuffle_) {\n          n_to_out = ParseChunk(data_dptr, label_dptr, current_size, &chunk);\n        } else {\n          n_to_out = ParseChunk(nullptr, nullptr, batch_param_.batch_size, &chunk);\n        }\n        // Count number of parsed images that do not fit into current out\n        n_parsed_ = inst_order_.size();\n        // shuffle instance order if needed\n        if (legacy_shuffle_) {\n          std::shuffle(inst_order_.begin(), inst_order_.end(), rnd_);\n        }\n      } else {\n        if (current_size == 0) {\n          return false;\n        }\n        CHECK(!overflow) << \"number of input images must be bigger than the batch size\";\n        if (batch_param_.round_batch != 0) {\n          overflow = true;\n          source_->BeforeFirst();\n        } else {\n          current_size = batch_param_.batch_size;\n        }\n        out->num_batch_padd = batch_param_.batch_size - current_size;\n        n_to_out            = 0;\n      }\n    } else {\n      size_t n_to_copy =\n          std::min(n_parsed_, static_cast<size_t>(batch_param_.batch_size) - current_size);\n      n_parsed_ -= n_to_copy;\n// Copy\n#pragma omp parallel for num_threads(param_.preprocess_threads)\n      for (int i = 0; i < static_cast<int>(n_to_copy); ++i) {\n        omp_exc_.Run([&] {\n          std::pair<size_t, size_t> place = inst_order_[inst_index_ + i];\n          const DataInst& batch           = temp_[place.first][place.second];\n          for (size_t j = 0; j < batch.data.size(); ++j) {\n            CHECK_EQ(unit_size_[j], batch.data[j].Size());\n            MSHADOW_TYPE_SWITCH(out->data[j].data().type_flag_, dtype, {\n              mshadow::Copy(\n                  out->data[j].data().FlatTo1D<cpu, dtype>().Slice(\n                      (current_size + i) * unit_size_[j], (current_size + i + 1) * unit_size_[j]),\n                  batch.data[j].get_with_shape<cpu, 1, dtype>(mshadow::Shape1(unit_size_[j])));\n            });\n          }\n        });\n      }\n      omp_exc_.Rethrow();\n      n_to_out = n_to_copy;\n      inst_index_ += n_to_copy;\n    }\n\n    current_size += n_to_out;\n  }\n  return true;\n}\n\n#if MXNET_USE_OPENCV\ntemplate <typename DType>\ntemplate <int n_channels>\nvoid ImageRecordIOParser2<DType>::ProcessImage(const cv::Mat& res,\n                                               mshadow::Tensor<cpu, 3, DType>* data_ptr,\n                                               const bool is_mirrored,\n                                               const float contrast_scaled,\n                                               const float illumination_scaled) {\n  float RGBA_MULT[4]                   = {0};\n  float RGBA_BIAS[4]                   = {0};\n  float RGBA_MEAN[4]                   = {0};\n  int16_t RGBA_MEAN_INT[4]             = {0};\n  mshadow::Tensor<cpu, 3, DType>& data = (*data_ptr);\n  if (!std::is_same<DType, uint8_t>::value) {\n    RGBA_MULT[0] = contrast_scaled / normalize_param_.std_r;\n    RGBA_MULT[1] = contrast_scaled / normalize_param_.std_g;\n    RGBA_MULT[2] = contrast_scaled / normalize_param_.std_b;\n    RGBA_MULT[3] = contrast_scaled / normalize_param_.std_a;\n    RGBA_BIAS[0] = illumination_scaled / normalize_param_.std_r;\n    RGBA_BIAS[1] = illumination_scaled / normalize_param_.std_g;\n    RGBA_BIAS[2] = illumination_scaled / normalize_param_.std_b;\n    RGBA_BIAS[3] = illumination_scaled / normalize_param_.std_a;\n    if (!meanfile_ready_) {\n      RGBA_MEAN[0]     = normalize_param_.mean_r;\n      RGBA_MEAN[1]     = normalize_param_.mean_g;\n      RGBA_MEAN[2]     = normalize_param_.mean_b;\n      RGBA_MEAN[3]     = normalize_param_.mean_a;\n      RGBA_MEAN_INT[0] = std::round(normalize_param_.mean_r);\n      RGBA_MEAN_INT[1] = std::round(normalize_param_.mean_g);\n      RGBA_MEAN_INT[2] = std::round(normalize_param_.mean_b);\n      RGBA_MEAN_INT[3] = std::round(normalize_param_.mean_a);\n    }\n  }\n\n  int swap_indices[n_channels];  // NOLINT(*)\n  if (n_channels == 1) {\n    swap_indices[0] = 0;\n  } else if (n_channels == 3) {\n    swap_indices[0] = 2;\n    swap_indices[1] = 1;\n    swap_indices[2] = 0;\n  } else if (n_channels == 4) {\n    swap_indices[0] = 2;\n    swap_indices[1] = 1;\n    swap_indices[2] = 0;\n    swap_indices[3] = 3;\n  }\n\n  DType RGBA[n_channels] = {};\n  for (int i = 0; i < res.rows; ++i) {\n    const uchar* im_data = res.ptr<uchar>(i);\n    for (int j = 0; j < res.cols; ++j) {\n      if (std::is_same<DType, int8_t>::value) {\n        if (meanfile_ready_) {\n          for (int k = 0; k < n_channels; ++k) {\n            RGBA[k] = cv::saturate_cast<int8_t>(\n                im_data[swap_indices[k]] - static_cast<int16_t>(std::round(meanimg_[k][i][j])));\n          }\n        } else {\n          for (int k = 0; k < n_channels; ++k) {\n            RGBA[k] = cv::saturate_cast<int8_t>(im_data[swap_indices[k]] - RGBA_MEAN_INT[k]);\n          }\n        }\n      } else {\n        for (int k = 0; k < n_channels; ++k) {\n          RGBA[k] = im_data[swap_indices[k]];\n        }\n        if (!std::is_same<DType, uint8_t>::value) {\n          // normalize/mirror here to avoid memory copies\n          // logic from iter_normalize.h, function SetOutImg\n          for (int k = 0; k < n_channels; ++k) {\n            if (meanfile_ready_) {\n              RGBA[k] = (RGBA[k] - meanimg_[k][i][j]) * RGBA_MULT[k] + RGBA_BIAS[k];\n            } else {\n              RGBA[k] = (RGBA[k] - RGBA_MEAN[k]) * RGBA_MULT[k] + RGBA_BIAS[k];\n            }\n          }\n        }\n      }\n      for (int k = 0; k < n_channels; ++k) {\n        // mirror here to avoid memory copies\n        // logic from iter_normalize.h, function SetOutImg\n        if (is_mirrored) {\n          data[k][i][res.cols - j - 1] = RGBA[k];\n        } else {\n          data[k][i][j] = RGBA[k];\n        }\n      }\n      im_data += n_channels;\n    }\n  }\n}\n\n#if MXNET_USE_LIBJPEG_TURBO\n\nbool is_jpeg(unsigned char* file) {\n  if ((file[0] == 255) && (file[1] == 216)) {\n    return true;\n  } else {\n    return false;\n  }\n}\n\ntemplate <typename DType>\ncv::Mat ImageRecordIOParser2<DType>::TJimdecode(cv::Mat image, int color) {\n  unsigned char* jpeg = image.ptr();\n  size_t jpeg_size    = image.rows * image.cols;\n\n  if (!is_jpeg(jpeg)) {\n    // If it is not JPEG then fall back to OpenCV\n    return cv::imdecode(image, color);\n  }\n\n  tjhandle handle = tjInitDecompress();\n  int h, w, subsamp;\n  int err = tjDecompressHeader2(handle, jpeg, jpeg_size, &w, &h, &subsamp);\n  if (err != 0) {\n    // If it is a malformed JPEG then fall back to OpenCV\n    return cv::imdecode(image, color);\n  }\n  cv::Mat ret = cv::Mat(h, w, color ? CV_8UC3 : CV_8UC1);\n  err = tjDecompress2(handle, jpeg, jpeg_size, ret.ptr(), w, 0, h, color ? TJPF_BGR : TJPF_GRAY, 0);\n  if (err != 0) {\n    // If it is a malformed JPEG then fall back to OpenCV\n    return cv::imdecode(image, color);\n  }\n  tjDestroy(handle);\n  return ret;\n}\n#endif\n#endif\n\n// Returns the number of images that are put into output\ntemplate <typename DType>\ninline size_t ImageRecordIOParser2<DType>::ParseChunk(DType* data_dptr,\n                                                      real_t* label_dptr,\n                                                      const size_t current_size,\n                                                      dmlc::InputSplit::Blob* chunk) {\n  temp_.resize(param_.preprocess_threads);\n#if MXNET_USE_OPENCV\n  // save opencv out\n  dmlc::RecordIOChunkReader reader(*chunk, 0, 1);\n  size_t gl_idx = current_size;\n#pragma omp parallel num_threads(param_.preprocess_threads)\n  {\n    omp_exc_.Run([&] {\n      CHECK(omp_get_num_threads() == param_.preprocess_threads);\n      int tid = omp_get_thread_num();\n      // dmlc::RecordIOChunkReader reader(*chunk, tid, param_.preprocess_threads);\n      ImageRecordIO rec;\n      dmlc::InputSplit::Blob blob;\n      // image data\n      InstVector<DType>& out_tmp = temp_[tid];\n      out_tmp.Clear();\n      while (true) {\n        bool reader_has_data;\n        size_t idx;\n#pragma omp critical\n        {\n          reader_has_data = reader.NextRecord(&blob);\n          if (reader_has_data) {\n            idx = gl_idx++;\n            if (idx >= batch_param_.batch_size) {\n              inst_order_.push_back(std::make_pair(tid, out_tmp.Size()));\n            }\n          }\n        }\n        if (!reader_has_data)\n          break;\n        // Opencv decode and augments\n        cv::Mat res;\n        rec.Load(blob.dptr, blob.size);\n        cv::Mat buf(1, rec.content_size, CV_8U, rec.content);\n\n        // If augmentation seed is supplied\n        // Re-seed RNG to guarantee reproducible results\n        if (param_.seed_aug.has_value()) {\n          prnds_[tid]->seed(idx + param_.seed_aug.value() + kRandMagic);\n        }\n\n        switch (param_.data_shape[0]) {\n          case 1:\n#if MXNET_USE_LIBJPEG_TURBO\n            res = TJimdecode(buf, 0);\n#else\n            res = cv::imdecode(buf, 0);\n#endif\n            break;\n          case 3:\n#if MXNET_USE_LIBJPEG_TURBO\n            res = TJimdecode(buf, 1);\n#else\n            res = cv::imdecode(buf, 1);\n#endif\n            break;\n          case 4:\n            // -1 to keep the number of channel of the encoded image, and not force gray or color.\n            res = cv::imdecode(buf, -1);\n            CHECK_EQ(res.channels(), 4) << \"Invalid image with index \" << rec.image_index()\n                                        << \". Expected 4 channels, got \" << res.channels();\n            break;\n          default:\n            LOG(FATAL) << \"Invalid output shape \" << param_.data_shape;\n        }\n        const int n_channels = res.channels();\n        // load label before augmentations\n        std::vector<float> label_buf;\n        if (label_map_ != nullptr) {\n          label_buf = label_map_->FindCopy(rec.image_index());\n        } else if (rec.label != nullptr) {\n          CHECK_EQ(param_.label_width, rec.num_label) << \"rec file provide \" << rec.num_label\n                                                      << \"-dimensional label \"\n                                                         \"but label_width is set to \"\n                                                      << param_.label_width;\n          label_buf.assign(rec.label, rec.label + rec.num_label);\n        } else {\n          CHECK_EQ(param_.label_width, 1)\n              << \"label_width must be 1 unless an imglist is provided \"\n                 \"or the rec file is packed with multi dimensional label\";\n          label_buf.assign(&rec.header.label, &rec.header.label + 1);\n        }\n        for (auto& aug : augmenters_[tid]) {\n          res = aug->Process(res, &label_buf, prnds_[tid].get());\n        }\n        mshadow::Tensor<cpu, 3, DType> data;\n        if (idx < batch_param_.batch_size) {\n          data = mshadow::Tensor<cpu, 3, DType>(data_dptr + idx * unit_size_[0],\n                                                mshadow::Shape3(n_channels, res.rows, res.cols));\n        } else {\n          out_tmp.Push(static_cast<size_t>(rec.image_index()),\n                       mshadow::Shape3(n_channels, res.rows, res.cols),\n                       mshadow::Shape1(param_.label_width));\n          data = out_tmp.data().Back();\n        }\n\n        std::uniform_real_distribution<float> rand_uniform(0, 1);\n        std::bernoulli_distribution coin_flip(0.5);\n        bool is_mirrored =\n            (normalize_param_.rand_mirror && coin_flip(*(prnds_[tid]))) || normalize_param_.mirror;\n        float contrast_scaled     = 1;\n        float illumination_scaled = 0;\n        if (!std::is_same<DType, uint8_t>::value) {\n          contrast_scaled =\n              (rand_uniform(*(prnds_[tid])) * normalize_param_.max_random_contrast * 2 -\n               normalize_param_.max_random_contrast + 1) *\n              normalize_param_.scale;\n          illumination_scaled =\n              (rand_uniform(*(prnds_[tid])) * normalize_param_.max_random_illumination * 2 -\n               normalize_param_.max_random_illumination) *\n              normalize_param_.scale;\n        }\n        // For RGB or RGBA data, swap the B and R channel:\n        // OpenCV store as BGR (or BGRA) and we want RGB (or RGBA)\n        if (n_channels == 1) {\n          ProcessImage<1>(res, &data, is_mirrored, contrast_scaled, illumination_scaled);\n        } else if (n_channels == 3) {\n          ProcessImage<3>(res, &data, is_mirrored, contrast_scaled, illumination_scaled);\n        } else if (n_channels == 4) {\n          ProcessImage<4>(res, &data, is_mirrored, contrast_scaled, illumination_scaled);\n        }\n\n        mshadow::Tensor<cpu, 1, real_t> label;\n        if (idx < batch_param_.batch_size) {\n          label = mshadow::Tensor<cpu, 1, real_t>(label_dptr + idx * unit_size_[1],\n                                                  mshadow::Shape1(param_.label_width));\n        } else {\n          label = out_tmp.label().Back();\n        }\n\n        mshadow::Copy(\n            label,\n            mshadow::Tensor<cpu, 1>(dmlc::BeginPtr(label_buf), mshadow::Shape1(label_buf.size())));\n        res.release();\n      }\n    });\n  }\n  omp_exc_.Rethrow();\n  return (std::min(static_cast<size_t>(batch_param_.batch_size), gl_idx) - current_size);\n#else\n  LOG(FATAL) << \"Opencv is needed for image decoding and augmenting.\";\n  return 0;\n#endif\n}\n\n// create mean image.\ntemplate <typename DType>\ninline void ImageRecordIOParser2<DType>::CreateMeanImg() {\n  if (param_.verbose) {\n    LOG(INFO) << \"Cannot find \" << normalize_param_.mean_img\n              << \": create mean image, this will take some time...\";\n  }\n  double start = dmlc::GetTime();\n  dmlc::InputSplit::Blob chunk;\n  size_t imcnt = 0;  // NOLINT(*)\n  while (source_->NextChunk(&chunk)) {\n    inst_order_.clear();\n    // Parse chunk w/o putting anything in out\n    ParseChunk(nullptr, nullptr, batch_param_.batch_size, &chunk);\n    for (auto place : inst_order_) {\n      mshadow::Tensor<cpu, 3> outimg =\n          temp_[place.first][place.second].data[0].template get<cpu, 3, real_t>();\n      if (imcnt == 0) {\n        meanimg_.Resize(outimg.shape_);\n        mshadow::Copy(meanimg_, outimg);\n      } else {\n        meanimg_ += outimg;\n      }\n      imcnt += 1;\n      double elapsed = dmlc::GetTime() - start;\n      if (imcnt % 10000L == 0 && param_.verbose) {\n        LOG(INFO) << imcnt << \" images processed, \" << elapsed << \" sec elapsed\";\n      }\n    }\n  }\n  meanimg_ *= (1.0f / imcnt);\n  // save as mxnet python compatible format.\n  TBlob tmp = meanimg_;\n  {\n    std::unique_ptr<dmlc::Stream> fo(dmlc::Stream::Create(normalize_param_.mean_img.c_str(), \"w\"));\n    NDArray::Save(fo.get(), {NDArray(tmp, 0)}, {\"mean_img\"});\n  }\n  if (param_.verbose) {\n    LOG(INFO) << \"Save mean image to \" << normalize_param_.mean_img << \"..\";\n  }\n  meanfile_ready_ = true;\n  this->BeforeFirst();\n}\n\ntemplate <typename DType = real_t>\nclass ImageRecordIter2 : public IIterator<DataBatch> {\n public:\n  ImageRecordIter2() = default;\n\n  ~ImageRecordIter2() override {\n    iter_.Destroy();\n  }\n\n  void Init(const std::vector<std::pair<std::string, std::string>>& kwargs) override {\n    prefetch_param_.InitAllowUnknown(kwargs);\n    parser_.Init(kwargs);\n    // maximum prefetch threaded iter internal size\n    const int kMaxPrefetchBuffer = 16;\n    // init thread iter\n    iter_.set_max_capacity(kMaxPrefetchBuffer);\n    // init thread iter\n    iter_.Init(\n        [this](DataBatch** dptr) {\n          if (*dptr == nullptr) {\n            *dptr = new DataBatch();\n          }\n          return parser_.ParseNext(*dptr);\n        },\n        [this]() { parser_.BeforeFirst(); });\n  }\n\n  void BeforeFirst() override {\n    iter_.BeforeFirst();\n  }\n\n  // From iter_prefetcher.h\n  bool Next() override {\n    if (out_ != nullptr) {\n      recycle_queue_.push(out_);\n      out_ = nullptr;\n    }\n    // do recycle\n    if (recycle_queue_.size() == prefetch_param_.prefetch_buffer) {\n      DataBatch* old_batch = recycle_queue_.front();\n      // can be more efficient on engine\n      for (NDArray& arr : old_batch->data) {\n        arr.WaitToWrite();\n      }\n      recycle_queue_.pop();\n      iter_.Recycle(&old_batch);\n    }\n    return iter_.Next(&out_);\n  }\n\n  const DataBatch& Value() const override {\n    return *out_;\n  }\n\n private:\n  /*! \\brief Backend thread */\n  dmlc::ThreadedIter<DataBatch> iter_;\n  /*! \\brief Parameters */\n  PrefetcherParam prefetch_param_;\n  /*! \\brief output data */\n  DataBatch* out_{nullptr};\n  /*! \\brief queue to be recycled */\n  std::queue<DataBatch*> recycle_queue_;\n  /* \\brief parser */\n  ImageRecordIOParser2<DType> parser_;\n};\n\ntemplate <typename DType = real_t>\nclass ImageRecordIter2CPU : public IIterator<DataBatch> {\n public:\n  ImageRecordIter2CPU() {\n    out_ = new DataBatch();\n    var_ = Engine::Get()->NewVariable();\n  }\n\n  ~ImageRecordIter2CPU() override {\n    Engine::Get()->DeleteVariable([](mxnet::RunContext ctx) {}, Context::CPU(), var_);\n    delete out_;\n  }\n\n  void Init(const std::vector<std::pair<std::string, std::string>>& kwargs) override {\n    parser_.Init(kwargs);\n  }\n\n  void BeforeFirst() override {\n    parser_.BeforeFirst();\n  }\n\n  // From iter_prefetcher.h\n  bool Next() override {\n    bool result       = false;\n    const auto engine = Engine::Get();\n    engine->PushSync([this, &result](RunContext ctx) { result = this->parser_.ParseNext(out_); },\n                     Context::CPU(),\n                     {},\n                     {var_},\n                     FnProperty::kNormal,\n                     0,\n                     \"DataLoader\");\n    engine->WaitForVar(var_);\n    return result;\n  }\n\n  const DataBatch& Value() const override {\n    return *out_;\n  }\n\n private:\n  /*! \\brief Backend thread */\n  dmlc::ThreadedIter<DataBatch> iter_;\n  /*! \\brief output data */\n  DataBatch* out_;\n  Engine::VarHandle var_;\n  /*! \\brief queue to be recycled */\n  std::queue<DataBatch*> recycle_queue_;\n  /* \\brief parser */\n  ImageRecordIOParser2<DType> parser_;\n};\n\nclass ImageRecordIter2Wrapper : public IIterator<DataBatch> {\n public:\n  ~ImageRecordIter2Wrapper() override {\n    if (record_iter_)\n      delete record_iter_;\n  }\n  void Init(const std::vector<std::pair<std::string, std::string>>& kwargs) override {\n    PrefetcherParam prefetch_param;\n    prefetch_param.InitAllowUnknown(kwargs);\n    int dtype = mshadow::kFloat32;\n    if (prefetch_param.dtype.has_value()) {\n      dtype = prefetch_param.dtype.value();\n    }\n    if (prefetch_param.ctx == PrefetcherParam::CtxType::kCPU) {\n      LOG(INFO) << \"Create ImageRecordIter2 optimized for CPU backend.\"\n                << \"Use omp threads instead of preprocess_threads.\";\n      switch (dtype) {\n        case mshadow::kFloat32:\n          record_iter_ = new ImageRecordIter2CPU<float>();\n          break;\n        case mshadow::kUint8:\n          record_iter_ = new ImageRecordIter2CPU<uint8_t>();\n          break;\n        case mshadow::kInt8:\n          record_iter_ = new ImageRecordIter2CPU<int8_t>();\n          break;\n        default:\n          LOG(FATAL) << \"unknown dtype for ImageRecordIter2.\";\n      }\n    } else {\n      // For gpu\n      switch (dtype) {\n        case mshadow::kFloat32:\n          record_iter_ = new ImageRecordIter2<float>();\n          break;\n        case mshadow::kUint8:\n          record_iter_ = new ImageRecordIter2<uint8_t>();\n          break;\n        case mshadow::kInt8:\n          record_iter_ = new ImageRecordIter2<int8_t>();\n          break;\n        default:\n          LOG(FATAL) << \"unknown dtype for ImageRecordIter2.\";\n      }\n    }\n    record_iter_->Init(kwargs);\n  }\n\n  void BeforeFirst() override {\n    record_iter_->BeforeFirst();\n  }\n\n  // From iter_prefetcher.h\n  bool Next() override {\n    return record_iter_->Next();\n  }\n\n  const DataBatch& Value() const override {\n    return record_iter_->Value();\n  }\n\n private:\n  IIterator<DataBatch>* record_iter_ = nullptr;\n};\n\nMXNET_REGISTER_IO_ITER(ImageRecordIter)\n    .describe(R\"code(Iterates on image RecordIO files\n\nReads batches of images from .rec RecordIO files. One can use ``im2rec.py`` tool\n(in tools/) to pack raw image files into RecordIO files. This iterator is less\nflexible to customization but is fast and has lot of language bindings. To\niterate over raw images directly use ``ImageIter`` instead (in Python).\n\nExample::\n\n  data_iter = mx.io.ImageRecordIter(\n    path_imgrec=\"./sample.rec\", # The target record file.\n    data_shape=(3, 227, 227), # Output data shape; 227x227 region will be cropped from the original image.\n    batch_size=4, # Number of items per batch.\n    resize=256 # Resize the shorter edge to 256 before cropping.\n    # You can specify more augmentation options. Use help(mx.io.ImageRecordIter) to see all the options.\n    )\n  # You can now use the data_iter to access batches of images.\n  batch = data_iter.next() # first batch.\n  images = batch.data[0] # This will contain 4 (=batch_size) images each of 3x227x227.\n  # process the images\n  ...\n  data_iter.reset() # To restart the iterator from the beginning.\n\n)code\" ADD_FILELINE)\n    .add_arguments(ImageRecParserParam::__FIELDS__())\n    .add_arguments(ImageRecordParam::__FIELDS__())\n    .add_arguments(BatchParam::__FIELDS__())\n    .add_arguments(PrefetcherParam::__FIELDS__())\n    .add_arguments(ListDefaultAugParams())\n    .add_arguments(ImageNormalizeParam::__FIELDS__())\n    .set_body([]() { return new ImageRecordIter2Wrapper(); });\n\nMXNET_REGISTER_IO_ITER(ImageRecordUInt8Iter)\n    .describe(R\"code(Iterating on image RecordIO files\n\n.. note:: ImageRecordUInt8Iter is deprecated. Use ImageRecordIter(dtype='uint8') instead.\n\nThis iterator is identical to ``ImageRecordIter`` except for using ``uint8`` as\nthe data type instead of ``float``.\n\n)code\" ADD_FILELINE)\n    .add_arguments(ImageRecParserParam::__FIELDS__())\n    .add_arguments(ImageRecordParam::__FIELDS__())\n    .add_arguments(BatchParam::__FIELDS__())\n    .add_arguments(PrefetcherParam::__FIELDS__())\n    .add_arguments(ListDefaultAugParams())\n    .set_body([]() { return new ImageRecordIter2<uint8_t>(); });\n\nMXNET_REGISTER_IO_ITER(ImageRecordInt8Iter)\n    .describe(R\"code(Iterating on image RecordIO files\n\n.. note:: ``ImageRecordInt8Iter`` is deprecated. Use ImageRecordIter(dtype='int8') instead.\n\nThis iterator is identical to ``ImageRecordIter`` except for using ``int8`` as\nthe data type instead of ``float``.\n\n)code\" ADD_FILELINE)\n    .add_arguments(ImageRecParserParam::__FIELDS__())\n    .add_arguments(ImageRecordParam::__FIELDS__())\n    .add_arguments(BatchParam::__FIELDS__())\n    .add_arguments(PrefetcherParam::__FIELDS__())\n    .add_arguments(ListDefaultAugParams())\n    .set_body([]() { return new ImageRecordIter2<int8_t>(); });\n\n}  // namespace io\n}  // namespace mxnet\n"
  },
  {
    "path": "src/io/iter_libsvm.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file iter_libsvm.cc\n * \\brief define a LibSVM Reader to read in arrays\n */\n#include <mxnet/io.h>\n#include <dmlc/base.h>\n#include <dmlc/logging.h>\n#include <dmlc/parameter.h>\n#include <dmlc/data.h>\n#include \"./iter_sparse_prefetcher.h\"\n#include \"./iter_sparse_batchloader.h\"\n\nnamespace mxnet {\nnamespace io {\n// LibSVM parameters\nstruct LibSVMIterParam : public dmlc::Parameter<LibSVMIterParam> {\n  /*! \\brief path to data libsvm file */\n  std::string data_libsvm;\n  /*! \\brief data shape */\n  mxnet::TShape data_shape;\n  /*! \\brief path to label libsvm file */\n  std::string label_libsvm;\n  /*! \\brief label shape */\n  mxnet::TShape label_shape;\n  /*! \\brief partition the data into multiple parts */\n  int num_parts;\n  /*! \\brief the index of the part will read*/\n  int part_index;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(LibSVMIterParam) {\n    DMLC_DECLARE_FIELD(data_libsvm)\n        .describe(\"The input zero-base indexed LibSVM data file or a directory path.\");\n    DMLC_DECLARE_FIELD(data_shape).describe(\"The shape of one example.\");\n    DMLC_DECLARE_FIELD(label_libsvm)\n        .set_default(\"NULL\")\n        .describe(\n            \"The input LibSVM label file or a directory path. \"\n            \"If NULL, all labels will be read from ``data_libsvm``.\");\n    index_t shape1[] = {1};\n    DMLC_DECLARE_FIELD(label_shape)\n        .set_default(mxnet::TShape(shape1, shape1 + 1))\n        .describe(\"The shape of one label.\");\n    DMLC_DECLARE_FIELD(num_parts).set_default(1).describe(\"partition the data into multiple parts\");\n    DMLC_DECLARE_FIELD(part_index).set_default(0).describe(\"the index of the part will read\");\n  }\n};\n\nclass LibSVMIter : public SparseIIterator<DataInst> {\n public:\n  LibSVMIter()           = default;\n  ~LibSVMIter() override = default;\n\n  // intialize iterator loads data in\n  void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override {\n    param_.InitAllowUnknown(kwargs);\n    CHECK_EQ(param_.data_shape.ndim(), 1) << \"dimension of data_shape is expected to be 1\";\n    CHECK_GT(param_.num_parts, 0) << \"number of parts should be positive\";\n    CHECK_GE(param_.part_index, 0) << \"part index should be non-negative\";\n    data_parser_.reset(dmlc::Parser<uint64_t>::Create(\n        param_.data_libsvm.c_str(), param_.part_index, param_.num_parts, \"libsvm\"));\n    if (param_.label_libsvm != \"NULL\") {\n      label_parser_.reset(dmlc::Parser<uint64_t>::Create(\n          param_.label_libsvm.c_str(), param_.part_index, param_.num_parts, \"libsvm\"));\n      CHECK_GT(param_.label_shape.Size(), 1)\n          << \"label_shape is not expected to be (1,) when param_.label_libsvm is set.\";\n    } else {\n      CHECK_EQ(param_.label_shape.Size(), 1)\n          << \"label_shape is expected to be (1,) when param_.label_libsvm is NULL\";\n    }\n    // both data and label are of CSRStorage in libsvm format\n    if (param_.label_shape.Size() > 1) {\n      out_.data.resize(6);\n    } else {\n      // only data is of CSRStorage in libsvm format.\n      out_.data.resize(4);\n    }\n  }\n\n  void BeforeFirst() override {\n    data_parser_->BeforeFirst();\n    if (label_parser_.get() != nullptr) {\n      label_parser_->BeforeFirst();\n    }\n    data_ptr_ = label_ptr_ = 0;\n    data_size_ = label_size_ = 0;\n    inst_counter_            = 0;\n    end_                     = false;\n  }\n\n  bool Next() override {\n    if (end_)\n      return false;\n    while (data_ptr_ >= data_size_) {\n      if (!data_parser_->Next()) {\n        end_ = true;\n        return false;\n      }\n      data_ptr_  = 0;\n      data_size_ = data_parser_->Value().size;\n    }\n    out_.index = inst_counter_++;\n    CHECK_LT(data_ptr_, data_size_);\n    const auto data_row = data_parser_->Value()[data_ptr_++];\n    // data, indices and indptr\n    out_.data[0] = AsDataBlob(data_row);\n    out_.data[1] = AsIdxBlob(data_row);\n    out_.data[2] = AsIndPtrPlaceholder(data_row);\n\n    if (label_parser_.get() != nullptr) {\n      while (label_ptr_ >= label_size_) {\n        CHECK(label_parser_->Next())\n            << \"Data LibSVM's row is smaller than the number of rows in label_libsvm\";\n        label_ptr_  = 0;\n        label_size_ = label_parser_->Value().size;\n      }\n      CHECK_LT(label_ptr_, label_size_);\n      const auto label_row = label_parser_->Value()[label_ptr_++];\n      // data, indices and indptr\n      out_.data[3] = AsDataBlob(label_row);\n      out_.data[4] = AsIdxBlob(label_row);\n      out_.data[5] = AsIndPtrPlaceholder(label_row);\n    } else {\n      out_.data[3] = AsScalarLabelBlob(data_row);\n    }\n    return true;\n  }\n\n  const DataInst& Value() const override {\n    return out_;\n  }\n\n  const NDArrayStorageType GetStorageType(bool is_data) const override {\n    if (is_data)\n      return kCSRStorage;\n    return param_.label_shape.Size() > 1 ? kCSRStorage : kDefaultStorage;\n  }\n\n  const mxnet::TShape GetShape(bool is_data) const override {\n    if (is_data)\n      return param_.data_shape;\n    return param_.label_shape;\n  }\n\n private:\n  inline TBlob AsDataBlob(const dmlc::Row<uint64_t>& row) {\n    const real_t* ptr = row.value;\n    mxnet::TShape shape(mshadow::Shape1(row.length));\n    return TBlob((real_t*)ptr, shape, cpu::kDevMask);  // NOLINT(*)\n  }\n\n  inline TBlob AsIdxBlob(const dmlc::Row<uint64_t>& row) {\n    const uint64_t* ptr = row.index;\n    mxnet::TShape shape(mshadow::Shape1(row.length));\n    return TBlob((int64_t*)ptr, shape, cpu::kDevMask, mshadow::kInt64);  // NOLINT(*)\n  }\n\n  inline TBlob AsIndPtrPlaceholder(const dmlc::Row<uint64_t>& row) {\n    return TBlob(nullptr, mshadow::Shape1(0), cpu::kDevMask, mshadow::kInt64);\n  }\n\n  inline TBlob AsScalarLabelBlob(const dmlc::Row<uint64_t>& row) {\n    const real_t* ptr = row.label;\n    return TBlob((real_t*)ptr, mshadow::Shape1(1), cpu::kDevMask);  // NOLINT(*)\n  }\n\n  LibSVMIterParam param_;\n  // output instance\n  DataInst out_;\n  // internal instance counter\n  unsigned inst_counter_{0};\n  // at end\n  bool end_{false};\n  // label parser\n  size_t label_ptr_{0}, label_size_{0};\n  size_t data_ptr_{0}, data_size_{0};\n  std::unique_ptr<dmlc::Parser<uint64_t> > label_parser_;\n  std::unique_ptr<dmlc::Parser<uint64_t> > data_parser_;\n};\n\nDMLC_REGISTER_PARAMETER(LibSVMIterParam);\n\nMXNET_REGISTER_IO_ITER(LibSVMIter)\n    .describe(R\"code(Returns the LibSVM iterator which returns data with `csr`\nstorage type. This iterator is experimental and should be used with care.\n\nThe input data is stored in a format similar to LibSVM file format, except that the **indices\nare expected to be zero-based instead of one-based, and the column indices for each row are\nexpected to be sorted in ascending order**. Details of the LibSVM format are available\n`here. <https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/>`_\n\n\nThe `data_shape` parameter is used to set the shape of each line of the data.\nThe dimension of both `data_shape` and `label_shape` are expected to be 1.\n\nThe `data_libsvm` parameter is used to set the path input LibSVM file.\nWhen it is set to a directory, all the files in the directory will be read.\n\nWhen `label_libsvm` is set to ``NULL``, both data and label are read from the file specified\nby `data_libsvm`. In this case, the data is stored in `csr` storage type, while the label is a 1D\ndense array.\n\nThe `LibSVMIter` only support `round_batch` parameter set to ``True``. Therefore, if `batch_size`\nis 3 and there are 4 total rows in libsvm file, 2 more examples are consumed at the first round.\n\nWhen `num_parts` and `part_index` are provided, the data is split into `num_parts` partitions,\nand the iterator only reads the `part_index`-th partition. However, the partitions are not\nguaranteed to be even.\n\n``reset()`` is expected to be called only after a complete pass of data.\n\nExample::\n\n  # Contents of libsvm file ``data.t``.\n  1.0 0:0.5 2:1.2\n  -2.0\n  -3.0 0:0.6 1:2.4 2:1.2\n  4 2:-1.2\n\n  # Creates a `LibSVMIter` with `batch_size`=3.\n  >>> data_iter = mx.io.LibSVMIter(data_libsvm = 'data.t', data_shape = (3,), batch_size = 3)\n  # The data of the first batch is stored in csr storage type\n  >>> batch = data_iter.next()\n  >>> csr = batch.data[0]\n  <CSRNDArray 3x3 @cpu(0)>\n  >>> csr.asnumpy()\n  [[ 0.5        0.          1.2 ]\n  [ 0.          0.          0.  ]\n  [ 0.6         2.4         1.2]]\n  # The label of first batch\n  >>> label = batch.label[0]\n  >>> label\n  [ 1. -2. -3.]\n  <NDArray 3 @cpu(0)>\n\n  >>> second_batch = data_iter.next()\n  # The data of the second batch\n  >>> second_batch.data[0].asnumpy()\n  [[ 0.          0.         -1.2 ]\n   [ 0.5         0.          1.2 ]\n   [ 0.          0.          0. ]]\n  # The label of the second batch\n  >>> second_batch.label[0].asnumpy()\n  [ 4.  1. -2.]\n\n  >>> data_iter.reset()\n  # To restart the iterator for the second pass of the data\n\nWhen `label_libsvm` is set to the path to another LibSVM file,\ndata is read from `data_libsvm` and label from `label_libsvm`.\nIn this case, both data and label are stored in the csr format.\nIf the label column in the `data_libsvm` file is ignored.\n\nExample::\n\n  # Contents of libsvm file ``label.t``\n  1.0\n  -2.0 0:0.125\n  -3.0 2:1.2\n  4 1:1.0 2:-1.2\n\n  # Creates a `LibSVMIter` with specified label file\n  >>> data_iter = mx.io.LibSVMIter(data_libsvm = 'data.t', data_shape = (3,),\n                   label_libsvm = 'label.t', label_shape = (3,), batch_size = 3)\n\n  # Both data and label are in csr storage type\n  >>> batch = data_iter.next()\n  >>> csr_data = batch.data[0]\n  <CSRNDArray 3x3 @cpu(0)>\n  >>> csr_data.asnumpy()\n  [[ 0.5         0.          1.2  ]\n   [ 0.          0.          0.   ]\n   [ 0.6         2.4         1.2 ]]\n  >>> csr_label = batch.label[0]\n  <CSRNDArray 3x3 @cpu(0)>\n  >>> csr_label.asnumpy()\n  [[ 0.          0.          0.   ]\n   [ 0.125       0.          0.   ]\n   [ 0.          0.          1.2 ]]\n\n)code\" ADD_FILELINE)\n    .add_arguments(LibSVMIterParam::__FIELDS__())\n    .add_arguments(BatchParam::__FIELDS__())\n    .add_arguments(PrefetcherParam::__FIELDS__())\n    .set_body([]() { return new SparsePrefetcherIter(new SparseBatchLoader(new LibSVMIter())); });\n\n}  // namespace io\n}  // namespace mxnet\n"
  },
  {
    "path": "src/io/iter_mnist.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file iter_mnist.cc\n * \\brief register mnist iterator\n */\n#include <mxnet/io.h>\n#include <mxnet/base.h>\n#include <dmlc/io.h>\n#include <dmlc/logging.h>\n#include <dmlc/parameter.h>\n#include <string>\n#include <vector>\n#include <utility>\n#include <map>\n#include \"./iter_prefetcher.h\"\n#include \"../common/utils.h\"\n\nnamespace mxnet {\nnamespace io {\n// Define mnist io parameters\nstruct MNISTParam : public dmlc::Parameter<MNISTParam> {\n  /*! \\brief path */\n  std::string image, label;\n  /*! \\brief whether to do shuffle */\n  bool shuffle;\n  /*! \\brief whether to print info */\n  bool silent;\n  /*! \\brief batch size */\n  int batch_size;\n  /*! \\brief data mode */\n  bool flat;\n  /*! \\brief random seed */\n  int seed;\n  /*! \\brief partition the data into multiple parts */\n  int num_parts;\n  /*! \\brief the index of the part will read*/\n  int part_index;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(MNISTParam) {\n    DMLC_DECLARE_FIELD(image)\n        .set_default(\"./train-images-idx3-ubyte\")\n        .describe(\"Dataset Param: Mnist image path.\");\n    DMLC_DECLARE_FIELD(label)\n        .set_default(\"./train-labels-idx1-ubyte\")\n        .describe(\"Dataset Param: Mnist label path.\");\n    DMLC_DECLARE_FIELD(batch_size)\n        .set_lower_bound(1)\n        .set_default(128)\n        .describe(\"Batch Param: Batch Size.\");\n    DMLC_DECLARE_FIELD(shuffle).set_default(true).describe(\n        \"Augmentation Param: Whether to shuffle data.\");\n    DMLC_DECLARE_FIELD(flat).set_default(false).describe(\n        \"Augmentation Param: Whether to flat the data into 1D.\");\n    DMLC_DECLARE_FIELD(seed).set_default(0).describe(\"Augmentation Param: Random Seed.\");\n    DMLC_DECLARE_FIELD(silent).set_default(false).describe(\n        \"Auxiliary Param: Whether to print out data info.\");\n    DMLC_DECLARE_FIELD(num_parts).set_default(1).describe(\"partition the data into multiple parts\");\n    DMLC_DECLARE_FIELD(part_index).set_default(0).describe(\"the index of the part will read\");\n  }\n};\n\nclass MNISTIter : public IIterator<TBlobBatch> {\n public:\n  MNISTIter() {\n    img_.dptr_ = nullptr;\n    out_.data.resize(2);\n  }\n  ~MNISTIter() override {\n    delete[] img_.dptr_;\n  }\n  // intialize iterator loads data in\n  void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override {\n    std::map<std::string, std::string> kmap(kwargs.begin(), kwargs.end());\n    param_.InitAllowUnknown(kmap);\n    this->LoadImage();\n    this->LoadLabel();\n    if (param_.flat) {\n      batch_data_.shape_ = mshadow::Shape4(param_.batch_size, 1, 1, img_.size(1) * img_.size(2));\n    } else {\n      batch_data_.shape_ = mshadow::Shape4(param_.batch_size, 1, img_.size(1), img_.size(2));\n    }\n    out_.data.clear();\n    batch_label_.shape_  = mshadow::Shape2(param_.batch_size, 1);\n    batch_label_.stride_ = 1;\n    batch_data_.stride_  = batch_data_.size(3);\n    out_.batch_size      = param_.batch_size;\n    if (param_.shuffle)\n      this->Shuffle();\n    if (param_.silent == 0) {\n      mxnet::TShape s;\n      s = batch_data_.shape_;\n      if (param_.flat) {\n        LOG(INFO) << \"MNISTIter: load \" << (unsigned)img_.size(0)\n                  << \" images, shuffle=\" << param_.shuffle << \", shape=\" << s.FlatTo2D();\n      } else {\n        LOG(INFO) << \"MNISTIter: load \" << (unsigned)img_.size(0)\n                  << \" images, shuffle=\" << param_.shuffle << \", shape=\" << s;\n      }\n    }\n  }\n  void BeforeFirst() override {\n    this->loc_ = 0;\n  }\n  bool Next() override {\n    if (loc_ + param_.batch_size <= img_.size(0)) {\n      batch_data_.dptr_  = img_[loc_].dptr_;\n      batch_label_.dptr_ = &labels_[loc_];\n      out_.data.clear();\n      if (param_.flat) {\n        out_.data.emplace_back(batch_data_.FlatTo2D());\n      } else {\n        out_.data.emplace_back(batch_data_);\n      }\n      out_.data.emplace_back(batch_label_);\n      loc_ += param_.batch_size;\n      return true;\n    } else {\n      return false;\n    }\n  }\n  const TBlobBatch& Value() const override {\n    return out_;\n  }\n\n private:\n  inline void GetPart(int count, int* start, int* end) {\n    CHECK_GE(param_.part_index, 0);\n    CHECK_GT(param_.num_parts, 0);\n    CHECK_GT(param_.num_parts, param_.part_index);\n\n    *start = static_cast<int>(static_cast<double>(count) / param_.num_parts * param_.part_index);\n    *end =\n        static_cast<int>(static_cast<double>(count) / param_.num_parts * (param_.part_index + 1));\n  }\n\n  inline void LoadImage() {\n    dmlc::SeekStream* stdimg = dmlc::SeekStream::CreateForRead(param_.image.c_str());\n    ReadInt(stdimg);\n    int image_count = ReadInt(stdimg);\n    int image_rows  = ReadInt(stdimg);\n    int image_cols  = ReadInt(stdimg);\n\n    int start, end;\n    GetPart(image_count, &start, &end);\n    image_count = end - start;\n    if (start > 0) {\n      stdimg->Seek(stdimg->Tell() + start * image_rows * image_cols);\n    }\n\n    img_.shape_  = mshadow::Shape3(image_count, image_rows, image_cols);\n    img_.stride_ = img_.size(2);\n\n    // allocate continuous memory\n    img_.dptr_ = new float[img_.MSize()];\n    for (int i = 0; i < image_count; ++i) {\n      for (int j = 0; j < image_rows; ++j) {\n        for (int k = 0; k < image_cols; ++k) {\n          unsigned char ch;\n          CHECK(stdimg->Read(&ch, sizeof(ch) != 0));\n          img_[i][j][k] = ch;\n        }\n      }\n    }\n    // normalize to 0-1\n    img_ *= 1.0f / 256.0f;\n    delete stdimg;\n  }\n  inline void LoadLabel() {\n    dmlc::SeekStream* stdlabel = dmlc::SeekStream::CreateForRead(param_.label.c_str());\n    ReadInt(stdlabel);\n    int labels_count = ReadInt(stdlabel);\n\n    int start, end;\n    GetPart(labels_count, &start, &end);\n    labels_count = end - start;\n    if (start > 0) {\n      stdlabel->Seek(stdlabel->Tell() + start);\n    }\n\n    labels_.resize(labels_count);\n    for (int i = 0; i < labels_count; ++i) {\n      unsigned char ch;\n      CHECK(stdlabel->Read(&ch, sizeof(ch) != 0));\n      labels_[i] = ch;\n      inst_.push_back((unsigned)i + inst_offset_);\n    }\n    delete stdlabel;\n  }\n  inline void Shuffle() {\n    std::shuffle(inst_.begin(), inst_.end(), common::RANDOM_ENGINE(kRandMagic + param_.seed));\n    std::vector<float> tmplabel(labels_.size());\n    mshadow::TensorContainer<cpu, 3> tmpimg(img_.shape_);\n    for (size_t i = 0; i < inst_.size(); ++i) {\n      unsigned ridx = inst_[i] - inst_offset_;\n      mshadow::Copy(tmpimg[i], img_[ridx]);\n      tmplabel[i] = labels_[ridx];\n    }\n    // copy back\n    mshadow::Copy(img_, tmpimg);\n    labels_ = tmplabel;\n  }\n\n private:\n  inline static int ReadInt(dmlc::Stream* fi) {\n    unsigned char buf[4];\n    CHECK(fi->Read(buf, sizeof(buf)) == sizeof(buf)) << \"invalid mnist format\";\n#ifdef _MSC_VER\n    return (buf[0] << 24 | buf[1] << 16 | buf[2] << 8 | buf[3]);\n#else\n    return reinterpret_cast<int>(buf[0] << 24 | buf[1] << 16 | buf[2] << 8 | buf[3]);\n#endif\n  }\n\n private:\n  /*! \\brief MNIST iter params */\n  MNISTParam param_;\n  /*! \\brief output */\n  TBlobBatch out_;\n  /*! \\brief current location */\n  index_t loc_{0};\n  /*! \\brief image content */\n  mshadow::Tensor<cpu, 3> img_;\n  /*! \\brief label content */\n  std::vector<float> labels_;\n  /*! \\brief batch data tensor */\n  mshadow::Tensor<cpu, 4> batch_data_;\n  /*! \\brief batch label tensor  */\n  mshadow::Tensor<cpu, 2> batch_label_;\n  /*! \\brief instance index offset */\n  unsigned inst_offset_{0};\n  /*! \\brief instance index */\n  std::vector<unsigned> inst_;\n  // magic number to setup randomness\n  static const int kRandMagic = 0;\n};  // class MNISTIter\n\nDMLC_REGISTER_PARAMETER(MNISTParam);\n\nMXNET_REGISTER_IO_ITER(MNISTIter)\n    .describe(\"Iterating on the MNIST dataset.\" ADD_FILELINE)\n    .add_arguments(MNISTParam::__FIELDS__())\n    .add_arguments(PrefetcherParam::__FIELDS__())\n    .set_body([]() { return new PrefetcherIter(new MNISTIter()); });\n\n}  // namespace io\n}  // namespace mxnet\n"
  },
  {
    "path": "src/io/iter_normalize.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file iter_normalize.h\n * \\brief Iterator that subtracts mean and do a few augmentations.\n */\n#ifndef MXNET_IO_ITER_NORMALIZE_H_\n#define MXNET_IO_ITER_NORMALIZE_H_\n\n#include <mxnet/base.h>\n#include <mxnet/io.h>\n#include <mxnet/ndarray.h>\n#include <dmlc/logging.h>\n#include <dmlc/parameter.h>\n#include <dmlc/timer.h>\n#include <mshadow/tensor.h>\n#include <utility>\n#include <string>\n#include <vector>\n#include \"../common/utils.h\"\n#include \"./image_iter_common.h\"\n\nnamespace mxnet {\nnamespace io {\n\n/*!\n * \\brief Iterator that normalize a image.\n *  It also applies a few augmention before normalization.\n */\nclass ImageNormalizeIter : public IIterator<DataInst> {\n public:\n  explicit ImageNormalizeIter(IIterator<DataInst>* base) : base_(base), meanfile_ready_(false) {}\n\n  virtual void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) {\n    param_.InitAllowUnknown(kwargs);\n    base_->Init(kwargs);\n    rnd_.seed(kRandMagic + param_.seed);\n    outimg_.set_pad(false);\n    meanimg_.set_pad(false);\n    if (param_.mean_img.length() != 0) {\n      std::unique_ptr<dmlc::Stream> fi(dmlc::Stream::Create(param_.mean_img.c_str(), \"r\", true));\n      if (fi.get() == nullptr) {\n        this->CreateMeanImg();\n      } else {\n        fi.reset(nullptr);\n        if (param_.verbose) {\n          LOG(INFO) << \"Load mean image from \" << param_.mean_img;\n        }\n        // use python compatible ndarray store format\n        std::vector<NDArray> data;\n        std::vector<std::string> keys;\n        {\n          std::unique_ptr<dmlc::Stream> fi(dmlc::Stream::Create(param_.mean_img.c_str(), \"r\"));\n          NDArray::Load(fi.get(), &data, &keys);\n        }\n        CHECK_EQ(data.size(), 1U) << \"Invalid mean image file format\";\n        data[0].WaitToRead();\n        mshadow::Tensor<cpu, 3> src = data[0].data().get<cpu, 3, real_t>();\n        meanimg_.Resize(src.shape_);\n        mshadow::Copy(meanimg_, src);\n        meanfile_ready_ = true;\n      }\n    }\n  }\n\n  virtual void BeforeFirst(void) {\n    base_->BeforeFirst();\n  }\n\n  virtual const DataInst& Value(void) const {\n    return out_;\n  }\n\n  virtual bool Next(void) {\n    if (!this->Next_())\n      return false;\n    return true;\n  }\n\n private:\n  /*! \\brief base iterator */\n  std::unique_ptr<IIterator<DataInst> > base_;\n  /*! whether mean image is ready */\n  bool meanfile_ready_;\n  /*! \\brief output data */\n  DataInst out_;\n  // normalize parameter.\n  ImageNormalizeParam param_;\n  /*! \\brief mean image, if needed */\n  mshadow::TensorContainer<cpu, 3> meanimg_;\n  /*! \\brief temp space for output image */\n  mshadow::TensorContainer<cpu, 3> outimg_;\n  /*! \\brief random numeber engine */\n  common::RANDOM_ENGINE rnd_;\n  // random magic number of this iterator\n  static const int kRandMagic = 0;\n\n  /*! \\brief internal next function, inlined for fater processing. */\n  inline bool Next_(void) {\n    if (!base_->Next())\n      return false;\n    const DataInst& src = base_->Value();\n    this->SetOutImg(src);\n    out_.data.resize(2);\n    out_.data[0]    = outimg_;\n    out_.data[1]    = src.data[1];\n    out_.index      = src.index;\n    out_.extra_data = src.extra_data;\n    return true;\n  }\n  /*!\n   * \\brief Set the output image, after augmentation and normalization.\n   * \\param src The source image.\n   */\n  inline void SetOutImg(const DataInst& src) {\n    using namespace mshadow::expr;  // NOLINT(*)\n\n    std::uniform_real_distribution<float> rand_uniform(0, 1);\n    std::bernoulli_distribution coin_flip(0.5);\n    mshadow::Tensor<cpu, 3> data = src.data[0].get<cpu, 3, real_t>();\n\n    outimg_.Resize(data.shape_);\n    float contrast =\n        rand_uniform(rnd_) * param_.max_random_contrast * 2 - param_.max_random_contrast + 1;\n    float illumination =\n        rand_uniform(rnd_) * param_.max_random_illumination * 2 - param_.max_random_illumination;\n    bool flip = (param_.rand_mirror && coin_flip(rnd_)) || param_.mirror;\n\n    // one-liner channel-wise normalization\n    switch (data.shape_[0]) {\n      case 4:\n        if (meanfile_ready_ && flip) {\n          outimg_[3] = mirror((data[3] - meanimg_[3]) * contrast + illumination) * param_.scale /\n                       param_.std_a;\n        } else if (meanfile_ready_ && (!flip)) {\n          outimg_[3] =\n              ((data[3] - meanimg_[3]) * contrast + illumination) * param_.scale / param_.std_a;\n        } else if (!meanfile_ready_ && flip) {\n          outimg_[3] = mirror((data[3] - param_.mean_a) * contrast + illumination) * param_.scale /\n                       param_.std_a;\n        } else {\n          outimg_[3] =\n              ((data[3] - param_.mean_a) * contrast + illumination) * param_.scale / param_.std_a;\n        }\n      case 3:\n        if (meanfile_ready_ && flip) {\n          outimg_[2] = mirror((data[2] - meanimg_[2]) * contrast + illumination) * param_.scale /\n                       param_.std_b;\n        } else if (meanfile_ready_ && (!flip)) {\n          outimg_[2] =\n              ((data[2] - meanimg_[2]) * contrast + illumination) * param_.scale / param_.std_b;\n        } else if (!meanfile_ready_ && flip) {\n          outimg_[2] = mirror((data[2] - param_.mean_b) * contrast + illumination) * param_.scale /\n                       param_.std_b;\n        } else {\n          outimg_[2] =\n              ((data[2] - param_.mean_b) * contrast + illumination) * param_.scale / param_.std_b;\n        }\n      case 2:\n        if (meanfile_ready_ && flip) {\n          outimg_[1] = mirror((data[1] - meanimg_[1]) * contrast + illumination) * param_.scale /\n                       param_.std_g;\n        } else if (meanfile_ready_ && (!flip)) {\n          outimg_[1] =\n              ((data[1] - meanimg_[1]) * contrast + illumination) * param_.scale / param_.std_g;\n        } else if (!meanfile_ready_ && flip) {\n          outimg_[1] = mirror((data[1] - param_.mean_g) * contrast + illumination) * param_.scale /\n                       param_.std_g;\n        } else {\n          outimg_[1] =\n              ((data[1] - param_.mean_g) * contrast + illumination) * param_.scale / param_.std_g;\n        }\n      case 1:\n        if (meanfile_ready_ && flip) {\n          outimg_[0] = mirror((data[0] - meanimg_[0]) * contrast + illumination) * param_.scale /\n                       param_.std_r;\n        } else if (meanfile_ready_ && (!flip)) {\n          outimg_[0] =\n              ((data[0] - meanimg_[0]) * contrast + illumination) * param_.scale / param_.std_r;\n        } else if (!meanfile_ready_ && flip) {\n          outimg_[0] = mirror((data[0] - param_.mean_r) * contrast + illumination) * param_.scale /\n                       param_.std_r;\n        } else {\n          outimg_[0] =\n              ((data[0] - param_.mean_r) * contrast + illumination) * param_.scale / param_.std_r;\n        }\n        break;\n      default:\n        LOG(FATAL) << \"Expected image channels range 1-4, got \" << data.shape_[0];\n    }\n  }\n\n  // creat mean image.\n  inline void CreateMeanImg(void) {\n    if (param_.verbose) {\n      LOG(INFO) << \"Cannot find \" << param_.mean_img\n                << \": create mean image, this will take some time...\";\n    }\n    double start = dmlc::GetTime();\n    size_t imcnt = 1;  // NOLINT(*)\n    CHECK(this->Next_()) << \"input iterator failed.\";\n    meanimg_.Resize(outimg_.shape_);\n    mshadow::Copy(meanimg_, outimg_);\n    while (this->Next_()) {\n      meanimg_ += outimg_;\n      imcnt += 1;\n      double elapsed = dmlc::GetTime() - start;\n      if (imcnt % 10000L == 0 && param_.verbose) {\n        LOG(INFO) << imcnt << \" images processed, \" << elapsed << \" sec elapsed\";\n      }\n    }\n    meanimg_ *= (1.0f / imcnt);\n    // save as mxnet python compatible format.\n    TBlob tmp = meanimg_;\n    {\n      std::unique_ptr<dmlc::Stream> fo(dmlc::Stream::Create(param_.mean_img.c_str(), \"w\"));\n      NDArray::Save(fo.get(), {NDArray(tmp, 0)}, {\"mean_img\"});\n    }\n    if (param_.verbose) {\n      LOG(INFO) << \"Save mean image to \" << param_.mean_img << \"..\";\n    }\n    meanfile_ready_ = true;\n    this->BeforeFirst();\n  }\n};\n\n/*!\n * \\brief Iterator that normalize a image.\n *  It also applies a few augmention before normalization.\n */\nclass ImageDetNormalizeIter : public IIterator<DataInst> {\n public:\n  explicit ImageDetNormalizeIter(IIterator<DataInst>* base) : base_(base), meanfile_ready_(false) {}\n\n  virtual void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) {\n    param_.InitAllowUnknown(kwargs);\n    base_->Init(kwargs);\n    rnd_.seed(kRandMagic + param_.seed);\n    outimg_.set_pad(false);\n    meanimg_.set_pad(false);\n    if (param_.mean_img.length() != 0) {\n      std::unique_ptr<dmlc::Stream> fi(dmlc::Stream::Create(param_.mean_img.c_str(), \"r\", true));\n      if (fi.get() == nullptr) {\n        this->CreateMeanImg();\n      } else {\n        fi.reset(nullptr);\n        if (param_.verbose) {\n          LOG(INFO) << \"Load mean image from \" << param_.mean_img;\n        }\n        // use python compatible ndarray store format\n        std::vector<NDArray> data;\n        std::vector<std::string> keys;\n        {\n          std::unique_ptr<dmlc::Stream> fi(dmlc::Stream::Create(param_.mean_img.c_str(), \"r\"));\n          NDArray::Load(fi.get(), &data, &keys);\n        }\n        CHECK_EQ(data.size(), 1) << \"Invalid mean image file format\";\n        data[0].WaitToRead();\n        mshadow::Tensor<cpu, 3> src = data[0].data().get<cpu, 3, real_t>();\n        meanimg_.Resize(src.shape_);\n        mshadow::Copy(meanimg_, src);\n        meanfile_ready_ = true;\n      }\n    }\n  }\n\n  virtual void BeforeFirst(void) {\n    base_->BeforeFirst();\n  }\n\n  virtual const DataInst& Value(void) const {\n    return out_;\n  }\n\n  virtual bool Next(void) {\n    if (!this->Next_())\n      return false;\n    return true;\n  }\n\n private:\n  /*! \\brief base iterator */\n  std::unique_ptr<IIterator<DataInst> > base_;\n  // whether mean image is ready.\n  bool meanfile_ready_;\n  /*! \\brief output data */\n  DataInst out_;\n  // normalize parameter.\n  ImageDetNormalizeParam param_;\n  /*! \\brief mean image, if needed */\n  mshadow::TensorContainer<cpu, 3> meanimg_;\n  /*! \\brief temp space for output image */\n  mshadow::TensorContainer<cpu, 3> outimg_;\n  /*! \\brief random numeber engine */\n  common::RANDOM_ENGINE rnd_;\n  // random magic number of this iterator\n  static const int kRandMagic = 0;\n\n  /*! \\brief internal next function, inlined for fater processing. */\n  inline bool Next_(void) {\n    if (!base_->Next())\n      return false;\n    const DataInst& src = base_->Value();\n    this->SetOutImg(src);\n    out_.data.resize(2);\n    out_.data[0]    = outimg_;\n    out_.data[1]    = src.data[1];\n    out_.index      = src.index;\n    out_.extra_data = src.extra_data;\n    return true;\n  }\n  /*!\n   * \\brief Set the output image, after augmentation and normalization.\n   * \\param src The source image.\n   */\n  inline void SetOutImg(const DataInst& src) {\n    using namespace mshadow::expr;  // NOLINT(*)\n    mshadow::Tensor<cpu, 3> data = src.data[0].get<cpu, 3, real_t>();\n\n    outimg_.Resize(data.shape_);\n\n    if (param_.mean_r > 0.0f || param_.mean_g > 0.0f || param_.mean_b > 0.0f ||\n        param_.mean_a > 0.0f) {\n      // subtract mean per channel\n      data[0] -= param_.mean_r;\n      if (data.shape_[0] >= 3) {\n        data[1] -= param_.mean_g;\n        data[2] -= param_.mean_b;\n      }\n      if (data.shape_[0] == 4) {\n        data[3] -= param_.mean_a;\n      }\n    } else if (!meanfile_ready_ || param_.mean_img.length() == 0) {\n      // do not subtract anything\n    } else {\n      CHECK(meanfile_ready_);\n      data -= meanimg_;\n    }\n\n    // std\n    if (param_.std_r > 0.0f) {\n      data[0] /= param_.std_r;\n    }\n    if (data.shape_[0] >= 3 && param_.std_g > 0.0f) {\n      data[1] /= param_.std_g;\n    }\n    if (data.shape_[0] >= 3 && param_.std_b > 0.0f) {\n      data[2] /= param_.std_b;\n    }\n    if (data.shape_[0] == 4 && param_.std_a > 0.0f) {\n      data[3] /= param_.std_a;\n    }\n    outimg_ = data * param_.scale;\n  }\n\n  // creat mean image.\n  inline void CreateMeanImg(void) {\n    if (param_.verbose) {\n      LOG(INFO) << \"Cannot find \" << param_.mean_img\n                << \": create mean image, this will take some time...\";\n    }\n    double start = dmlc::GetTime();\n    size_t imcnt = 1;  // NOLINT(*)\n    CHECK(this->Next_()) << \"input iterator failed.\";\n    meanimg_.Resize(outimg_.shape_);\n    mshadow::Copy(meanimg_, outimg_);\n    while (this->Next_()) {\n      meanimg_ += outimg_;\n      imcnt += 1;\n      double elapsed = dmlc::GetTime() - start;\n      if (imcnt % 10000L == 0 && param_.verbose) {\n        LOG(INFO) << imcnt << \" images processed, \" << elapsed << \" sec elapsed\";\n      }\n    }\n    meanimg_ *= (1.0f / imcnt);\n    // save as mxnet python compatible format.\n    TBlob tmp = meanimg_;\n    {\n      std::unique_ptr<dmlc::Stream> fo(dmlc::Stream::Create(param_.mean_img.c_str(), \"w\"));\n      NDArray::Save(fo.get(), {NDArray(tmp, 0)}, {\"mean_img\"});\n    }\n    if (param_.verbose) {\n      LOG(INFO) << \"Save mean image to \" << param_.mean_img << \"..\";\n    }\n    meanfile_ready_ = true;\n    this->BeforeFirst();\n  }\n};\n}  // namespace io\n}  // namespace mxnet\n#endif  // MXNET_IO_ITER_NORMALIZE_H_\n"
  },
  {
    "path": "src/io/iter_prefetcher.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file iter_prefetcher.h\n * \\brief define a prefetcher using threaditer to keep k batch fetched\n */\n#ifndef MXNET_IO_ITER_PREFETCHER_H_\n#define MXNET_IO_ITER_PREFETCHER_H_\n\n#include <mxnet/io.h>\n#include <mxnet/base.h>\n#include <mxnet/ndarray.h>\n#include <dmlc/logging.h>\n#include <dmlc/threadediter.h>\n#include <dmlc/optional.h>\n#include <mshadow/tensor.h>\n#include <climits>\n#include <utility>\n#include <string>\n#include <vector>\n#include <queue>\n#include <algorithm>\n#include \"./inst_vector.h\"\n#include \"./image_iter_common.h\"\n\nnamespace mxnet {\nnamespace io {\n// iterator on image recordio\nclass PrefetcherIter : public IIterator<DataBatch> {\n public:\n  explicit PrefetcherIter(IIterator<TBlobBatch>* base)\n      : loader_(base), out_(nullptr), length_hint_(-1) {}\n\n  ~PrefetcherIter() {\n    while (recycle_queue_.size() != 0) {\n      DataBatch* batch = recycle_queue_.front();\n      recycle_queue_.pop();\n      delete batch;\n    }\n    delete out_;\n    iter.Destroy();\n  }\n\n  void InitParams(const std::vector<std::pair<std::string, std::string> >& kwargs) {\n    std::vector<std::pair<std::string, std::string> > kwargs_left;\n    // init image rec param\n    kwargs_left = param_.InitAllowUnknown(kwargs);\n    CHECK_GT(param_.prefetch_buffer, 0) << \"Prefetch_buffer must be positive number\";\n    // maximum prefetch threaded iter internal size\n    const int kMaxPrefetchBuffer = 16;\n    // init thread iter\n    iter.set_max_capacity(kMaxPrefetchBuffer);\n  }\n\n  virtual void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) {\n    InitParams(kwargs);\n    // use the kwarg to init batch loader\n    loader_->Init(kwargs);\n    length_hint_ = loader_->GetLenHint();\n    iter.Init(\n        [this](DataBatch** dptr) {\n          if (!loader_->Next())\n            return false;\n          const TBlobBatch& batch = loader_->Value();\n          if (*dptr == nullptr) {\n            // allocate databatch\n            *dptr                   = new DataBatch();\n            (*dptr)->num_batch_padd = batch.num_batch_padd;\n            (*dptr)->data.resize(batch.data.size());\n            (*dptr)->index.resize(batch.batch_size);\n            for (size_t i = 0; i < batch.data.size(); ++i) {\n              auto dtype = param_.dtype ? param_.dtype.value() : batch.data[i].type_flag_;\n              auto ctx = ((param_.ctx == PrefetcherParam::kCPUPinned) && (param_.device_id >= 0)) ?\n                             Context::CPUPinned(param_.device_id) :\n                             Context::CPU();\n              (*dptr)->data.at(i) = NDArray(batch.data[i].shape_, ctx, false, dtype);\n            }\n          }\n          CHECK(batch.data.size() == (*dptr)->data.size());\n          // copy data over\n          for (size_t i = 0; i < batch.data.size(); ++i) {\n            if ((*dptr)->data.at(i).shape() != batch.data[i].shape_) {\n              // TODO(zhreshold): memory pool for dynamic shaped data\n              (*dptr)->data.at(i).ReshapeAndAlloc(batch.data[i].shape_);\n            }\n            CHECK_EQ((*dptr)->data.at(i).shape(), batch.data[i].shape_);\n            MSHADOW_TYPE_SWITCH(batch.data[i].type_flag_, DType, {\n              mshadow::Copy(((*dptr)->data)[i].data().FlatTo2D<cpu, DType>(),\n                            batch.data[i].FlatTo2D<cpu, DType>());\n            });\n            (*dptr)->num_batch_padd = batch.num_batch_padd;\n          }\n          if (batch.inst_index) {\n            std::copy(\n                batch.inst_index, batch.inst_index + batch.batch_size, (*dptr)->index.begin());\n          }\n          return true;\n        },\n        [this]() {\n          loader_->BeforeFirst();\n          length_hint_ = loader_->GetLenHint();\n        });\n  }\n\n  virtual void BeforeFirst(void) {\n    iter.BeforeFirst();\n  }\n\n  virtual int64_t GetLenHint(void) const {\n    return length_hint_;\n  }\n\n  virtual bool Next(void) {\n    if (out_ != nullptr) {\n      recycle_queue_.push(out_);\n      out_ = nullptr;\n    }\n    // do recycle\n    if (recycle_queue_.size() == param_.prefetch_buffer) {\n      DataBatch* old_batch = recycle_queue_.front();\n      // can be more efficient on engine\n      for (NDArray& arr : old_batch->data) {\n        arr.WaitToWrite();\n      }\n      recycle_queue_.pop();\n      iter.Recycle(&old_batch);\n    }\n    return iter.Next(&out_);\n  }\n  virtual const DataBatch& Value(void) const {\n    return *out_;\n  }\n\n protected:\n  /*! \\brief prefetcher parameters */\n  PrefetcherParam param_;\n  /*! \\brief backend thread */\n  dmlc::ThreadedIter<DataBatch> iter;\n  /*! \\brief internal batch loader */\n  std::unique_ptr<IIterator<TBlobBatch> > loader_;\n\n private:\n  /*! \\brief output data */\n  DataBatch* out_;\n  /*! \\brief queue to be recycled */\n  std::queue<DataBatch*> recycle_queue_;\n  /*! \\brief size hint cache */\n  int64_t length_hint_;\n};\n}  // namespace io\n}  // namespace mxnet\n#endif  // MXNET_IO_ITER_PREFETCHER_H_\n"
  },
  {
    "path": "src/io/iter_sampler.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file iter_sampler.cc\n * \\brief The sampler iterator for access dataset elements.\n */\n#include <dmlc/parameter.h>\n#include <mshadow/random.h>\n#include <mxnet/io.h>\n#include <mxnet/base.h>\n#include <mxnet/resource.h>\n#include <memory>\n#include <numeric>\n#include \"../common/utils.h\"\n#include \"./iter_batchloader.h\"\n#include \"./iter_prefetcher.h\"\n\nnamespace mxnet {\nnamespace io {\nstruct SequentialSamplerParam : public dmlc::Parameter<SequentialSamplerParam> {\n  /*! \\brief Length of the sequence. */\n  size_t length;\n  /*! \\brief start index.*/\n  int start;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(SequentialSamplerParam) {\n    DMLC_DECLARE_FIELD(length).describe(\"Length of the sequence.\");\n    DMLC_DECLARE_FIELD(start).set_default(0).describe(\"Start of the index.\");\n  }\n};  // struct SequentialSamplerParam\n\nDMLC_REGISTER_PARAMETER(SequentialSamplerParam);\n\nclass SequentialSampler : public IIterator<DataInst> {\n public:\n  void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override {\n    param_.InitAllowUnknown(kwargs);\n    indices_.resize(param_.length);\n    std::iota(std::begin(indices_), std::end(indices_), 0);  // fill like arange\n    out_.data.resize(1);\n  }\n\n  void BeforeFirst() override {\n    pos_ = 0;\n  }\n\n  int64_t GetLenHint() const override {\n    return static_cast<int64_t>(indices_.size());\n  }\n\n  bool Next() override {\n    if (pos_ < indices_.size()) {\n      int64_t* ptr = indices_.data() + pos_;\n      out_.data[0] = TBlob(ptr,\n                           TShape({\n                               1,\n                           }),\n                           cpu::kDevMask,\n                           0);\n      ++pos_;\n      return true;\n    }\n    return false;\n  }\n\n  const DataInst& Value() const override {\n    return out_;\n  }\n\n private:\n  /*! \\brief Stored integer indices */\n  std::vector<int64_t> indices_;\n  /*! \\brief current position for iteration */\n  std::size_t pos_;\n  /*! \\brief data for next value */\n  DataInst out_;\n  /*! \\brief arguments */\n  SequentialSamplerParam param_;\n};  // class SequentialSampler\n\nMXNET_REGISTER_IO_ITER(SequentialSampler)\n    .describe(R\"code(Returns the sequential sampler iterator.\n)code\" ADD_FILELINE)\n    .add_arguments(SequentialSamplerParam::__FIELDS__())\n    .add_arguments(BatchSamplerParam::__FIELDS__())\n    .set_body([]() { return new BatchSampler(new SequentialSampler()); });\n\nstruct RandomSamplerParam : public dmlc::Parameter<RandomSamplerParam> {\n  /*! \\brief Length of the sequence. */\n  size_t length;\n  // declare parameters\n  DMLC_DECLARE_PARAMETER(RandomSamplerParam) {\n    DMLC_DECLARE_FIELD(length).describe(\"Length of the sequence.\");\n  }\n};  // struct RandomSamplerParam\n\nDMLC_REGISTER_PARAMETER(RandomSamplerParam);\n\nclass RandomSampler : public IIterator<DataInst> {\n public:\n  void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override {\n    param_.InitAllowUnknown(kwargs);\n    indices_.resize(param_.length);\n    std::iota(std::begin(indices_), std::end(indices_), 0);  // fill like arange\n    mshadow::Random<cpu>* ctx_rng = ResourceManager::Get()\n                                        ->Request(Context::CPU(), ResourceRequest::kRandom)\n                                        .get_random<cpu, real_t>(nullptr);\n    rng_ = std::make_unique<common::RANDOM_ENGINE>(ctx_rng->GetSeed());\n    out_.data.resize(1);\n    BeforeFirst();\n  }\n\n  void BeforeFirst() override {\n    std::shuffle(std::begin(indices_), std::end(indices_), *rng_);\n    pos_ = 0;\n  }\n\n  int64_t GetLenHint() const override {\n    return static_cast<int64_t>(indices_.size());\n  }\n\n  bool Next() override {\n    if (pos_ < indices_.size()) {\n      int64_t* ptr = indices_.data() + pos_;\n      out_.data[0] = TBlob(ptr,\n                           TShape({\n                               1,\n                           }),\n                           cpu::kDevMask,\n                           0);\n      ++pos_;\n      return true;\n    }\n    return false;\n  }\n\n  const DataInst& Value() const override {\n    return out_;\n  }\n\n private:\n  /*! \\brief Stored integer indices */\n  std::vector<int64_t> indices_;\n  /*! \\brief current position for iteration */\n  std::size_t pos_;\n  /*! \\brief data for next value */\n  DataInst out_;\n  /*! \\brief random generator engine */\n  std::unique_ptr<std::mt19937> rng_;\n  /*! \\brief arguments */\n  RandomSamplerParam param_;\n};  // class RandomSampler\n\nMXNET_REGISTER_IO_ITER(RandomSampler)\n    .describe(R\"code(Returns the random sampler iterator.\n)code\" ADD_FILELINE)\n    .add_arguments(RandomSamplerParam::__FIELDS__())\n    .add_arguments(BatchSamplerParam::__FIELDS__())\n    .set_body([]() { return new BatchSampler(new RandomSampler()); });\n\n}  // namespace io\n}  // namespace mxnet\n"
  },
  {
    "path": "src/io/iter_sparse.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file iter_sparse.h\n * \\brief mxnet sparse data iterator\n */\n#ifndef MXNET_IO_ITER_SPARSE_H_\n#define MXNET_IO_ITER_SPARSE_H_\n\n#include <mxnet/io.h>\n#include <mxnet/ndarray.h>\n\nnamespace mxnet {\n/*!\n * \\brief iterator type\n * \\param DType data type\n */\ntemplate <typename DType>\nclass SparseIIterator : public IIterator<DType> {\n public:\n  /*! \\brief storage type of the data or label */\n  virtual const NDArrayStorageType GetStorageType(bool is_data) const = 0;\n  /*! \\brief shape of the data or label */\n  virtual const mxnet::TShape GetShape(bool is_data) const = 0;\n};  // class SparseIIterator\n\n}  // namespace mxnet\n#endif  // MXNET_IO_ITER_SPARSE_H_\n"
  },
  {
    "path": "src/io/iter_sparse_batchloader.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file iter_sparse_batchloader.h\n * \\brief define a batch adapter to create sparse tblob batch\n */\n#ifndef MXNET_IO_ITER_SPARSE_BATCHLOADER_H_\n#define MXNET_IO_ITER_SPARSE_BATCHLOADER_H_\n\n#include <mxnet/io.h>\n#include <mxnet/base.h>\n#include <dmlc/logging.h>\n#include <mshadow/tensor.h>\n#include <utility>\n#include <vector>\n#include <string>\n#include \"./inst_vector.h\"\n#include \"./image_iter_common.h\"\n#include \"./iter_batchloader.h\"\n#include \"./iter_sparse.h\"\n\nnamespace mxnet {\nnamespace io {\n\n/*! \\brief create a batch iterator from single instance iterator */\nclass SparseBatchLoader : public BatchLoader, public SparseIIterator<TBlobBatch> {\n public:\n  explicit SparseBatchLoader(SparseIIterator<DataInst>* base)\n      : BatchLoader(base), sparse_base_(base) {}\n\n  virtual ~SparseBatchLoader(void) {}\n\n  inline void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) {\n    BatchLoader::Init(kwargs);\n    data_stype_  = sparse_base_->GetStorageType(true);\n    label_stype_ = sparse_base_->GetStorageType(false);\n    if (param_.round_batch == 0) {\n      LOG(FATAL) << \"sparse batch loader doesn't support round_batch == false yet\";\n    }\n  }\n\n  virtual void BeforeFirst(void) {\n    BatchLoader::BeforeFirst();\n  }\n\n  virtual bool Next(void) {\n    out_.num_batch_padd = 0;\n    out_.batch_size     = param_.batch_size;\n    this->head_         = 0;\n    // if overflown from previous round, directly return false, until before first is called\n    if (num_overflow_ != 0)\n      return false;\n    size_t top = 0;\n    offsets_.clear();\n    while (sparse_base_->Next()) {\n      const DataInst& inst = sparse_base_->Value();\n      // initialize the data buffer, only called once\n      if (data_.size() == 0)\n        this->InitData(inst);\n      // initialize the number of elements in each buffer, called once per batch\n      if (offsets_.size() == 0)\n        offsets_.resize(inst.data.size(), 0);\n      CopyData(inst, top);\n      if (++top >= param_.batch_size) {\n        SetOutputShape();\n        return true;\n      }\n    }\n    if (top != 0) {\n      CHECK_NE(param_.round_batch, 0)\n          << \"round_batch = False is not supported for sparse data iterator\";\n      num_overflow_ = 0;\n      sparse_base_->BeforeFirst();\n      for (; top < param_.batch_size; ++top, ++num_overflow_) {\n        CHECK(sparse_base_->Next()) << \"number of input must be bigger than batch size\";\n        const DataInst& inst = sparse_base_->Value();\n        // copy data\n        CopyData(inst, top);\n      }\n      SetOutputShape();\n      out_.num_batch_padd = num_overflow_;\n      return true;\n    }\n    // no more data instance\n    return false;\n  }\n\n  virtual const TBlobBatch& Value(void) const {\n    return BatchLoader::Value();\n  }\n\n  virtual const NDArrayStorageType GetStorageType(bool is_data) const {\n    return sparse_base_->GetStorageType(is_data);\n  }\n\n  virtual const mxnet::TShape GetShape(bool is_data) const {\n    mxnet::TShape inst_shape = sparse_base_->GetShape(is_data);\n    std::vector<index_t> shape_vec;\n    shape_vec.push_back(param_.batch_size);\n    for (index_t dim = 0; dim < inst_shape.ndim(); ++dim) {\n      shape_vec.push_back(inst_shape[dim]);\n    }\n    return mxnet::TShape(shape_vec.begin(), shape_vec.end());\n  }\n\n private:\n  /*! \\brief base sparse iterator */\n  SparseIIterator<DataInst>* sparse_base_;\n  /*! \\brief data storage type */\n  NDArrayStorageType data_stype_;\n  /*! \\brief data label type */\n  NDArrayStorageType label_stype_;\n  /*! \\brief tensor offsets for slicing */\n  std::vector<size_t> offsets_;\n  /*! \\brief tensor dtypes */\n  std::vector<int> dtypes_;\n  /*! \\brief whether the offset correspond to an indptr array */\n  std::vector<bool> indptr_;\n\n  // check whether ith position is the indptr tensor for a CSR tensor\n  inline bool IsIndPtr(size_t i) {\n    auto data_num_aux        = num_aux_data(data_stype_);\n    auto label_num_aux       = num_aux_data(label_stype_);\n    auto label_indptr_offset = data_num_aux + 1 + label_num_aux;\n    // data indptr\n    if (i == data_num_aux && data_stype_ == kCSRStorage) {\n      return true;\n    }\n    // label indptr\n    if (i == label_indptr_offset && label_stype_ == kCSRStorage && data_stype_ == kCSRStorage) {\n      return true;\n    }\n    return false;\n  }\n\n  // initialize the data holder by using from the batch\n  inline void InitData(const DataInst& first_inst) {\n    CHECK(data_stype_ == kCSRStorage || label_stype_ == kCSRStorage);\n    out_.data.clear();\n    data_.clear();\n    offsets_.clear();\n    indptr_.clear();\n\n    // num_arrays is the number of arrays in inputs\n    // if both data and label are in the csr format,\n    // num_arrays will be 3 + 3 = 6.\n    size_t num_arrays = first_inst.data.size();\n    data_.resize(num_arrays);\n    offsets_.resize(num_arrays, 0);\n    indptr_.resize(num_arrays, false);\n    // tensor buffer sizes\n    std::vector<size_t> buff_sizes(num_arrays, 0);\n    dtypes_.resize(num_arrays);\n    out_.data.resize(num_arrays);\n    // estimate the memory required for a batch\n    for (size_t i = 0; i < num_arrays; ++i) {\n      // shape for indptr\n      if (IsIndPtr(i)) {\n        buff_sizes[i] = param_.batch_size + 1;\n        indptr_[i]    = true;\n      } else {\n        // estimated the size for the whole batch based on the first instance\n        buff_sizes[i] = first_inst.data[i].Size() * param_.batch_size;\n        indptr_[i]    = false;\n      }\n      dtypes_[i] = first_inst.data[i].type_flag_;\n    }\n\n    CHECK_EQ(buff_sizes[0], buff_sizes[1]);\n    // allocate buffer\n    for (size_t i = 0; i < num_arrays; ++i) {\n      // init object attributes\n      mxnet::TShape dst_shape(mshadow::Shape1(buff_sizes[i]));\n      data_[i].resize(mshadow::Shape1(buff_sizes[i]), dtypes_[i]);\n      CHECK(data_[i].dptr_ != nullptr);\n    }\n  }\n\n  /* \\brief set the shape of the outputs based on actual shapes */\n  inline void SetOutputShape() {\n    for (size_t i = 0; i < out_.data.size(); i++) {\n      out_.data[i] = TBlob(data_[i].dptr_, mshadow::Shape1(offsets_[i]), Context::kCPU, dtypes_[i]);\n    }\n  }\n\n  /* \\brief increase the size of i-th data buffer by a factor of 2, while retaining the content */\n  inline void ResizeBuffer(size_t src_size, size_t i) {\n    MSHADOW_TYPE_SWITCH(data_[i].type_flag_, DType, {\n      TBlobContainer temp;\n      temp.resize(mshadow::Shape1(src_size), dtypes_[i]);\n      mshadow::Copy(temp.get<cpu, 1, DType>(), data_[i].get<cpu, 1, DType>().Slice(0, src_size));\n      // increase the size of space exponentially\n      size_t capacity = data_[i].Size();\n      capacity        = capacity * 2 + 1;\n      data_[i]        = TBlobContainer();\n      data_[i].resize(mshadow::Shape1(capacity), dtypes_[i]);\n      // copy back\n      mshadow::Copy(data_[i].get<cpu, 1, DType>().Slice(0, src_size), temp.get<cpu, 1, DType>());\n    });\n  }\n\n  /* \\brief copy the data instance to data buffer */\n  void CopyData(const DataInst& inst, const size_t top) {\n    int64_t unit_size    = 0;\n    out_.inst_index[top] = inst.index;\n    for (size_t i = 0; i < inst.data.size(); ++i) {\n      if (!indptr_[i]) {\n        // indices and values tensor\n        unit_size = inst.data[i].shape_.Size();\n        MSHADOW_TYPE_SWITCH(data_[i].type_flag_, DType, {\n          const size_t begin = offsets_[i];\n          const size_t end   = offsets_[i] + unit_size;\n          size_t capacity    = data_[i].Size();\n          // resize the data buffer if estimated space is not sufficient\n          while (capacity < end) {\n            ResizeBuffer(begin, i);\n            capacity = data_[i].Size();\n          }\n          mshadow::Copy(data_[i].get<cpu, 1, DType>().Slice(begin, end),\n                        inst.data[i].get_with_shape<cpu, 1, DType>(mshadow::Shape1(unit_size)));\n        });\n        offsets_[i] += unit_size;\n      } else {\n        // indptr placeholder\n        auto indptr = data_[i].get<cpu, 1, int64_t>();\n        // initialize the first indptr, which is always 0\n        if (top == 0)\n          indptr[0] = 0;\n        indptr[top + 1] = indptr[top] + unit_size;\n        offsets_[i]     = top + 2;\n      }\n    }\n  }\n};  // class BatchLoader\n}  // namespace io\n}  // namespace mxnet\n#endif  // MXNET_IO_ITER_SPARSE_BATCHLOADER_H_\n"
  },
  {
    "path": "src/io/iter_sparse_prefetcher.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file iter_sparse_prefetcher.h\n * \\brief define a prefetcher using threaditer to keep k batch fetched\n */\n#ifndef MXNET_IO_ITER_SPARSE_PREFETCHER_H_\n#define MXNET_IO_ITER_SPARSE_PREFETCHER_H_\n\n#include <mxnet/io.h>\n#include <mxnet/base.h>\n#include <mxnet/ndarray.h>\n#include <dmlc/logging.h>\n#include <dmlc/threadediter.h>\n#include <dmlc/optional.h>\n#include <mshadow/tensor.h>\n#include <climits>\n#include <utility>\n#include <string>\n#include <vector>\n#include <queue>\n#include <algorithm>\n#include \"./inst_vector.h\"\n#include \"./image_iter_common.h\"\n#include \"./iter_prefetcher.h\"\n#include \"./iter_sparse.h\"\n\nnamespace mxnet {\nnamespace io {\n// iterator on sparse data\nclass SparsePrefetcherIter : public PrefetcherIter {\n public:\n  explicit SparsePrefetcherIter(SparseIIterator<TBlobBatch>* base)\n      : PrefetcherIter(base), sparse_loader_(base) {}\n\n  ~SparsePrefetcherIter() {}\n\n  virtual void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) {\n    PrefetcherIter::InitParams(kwargs);\n    // use the kwarg to init batch loader\n    sparse_loader_->Init(kwargs);\n    iter.Init(\n        [this](DataBatch** dptr) {\n          if (!sparse_loader_->Next())\n            return false;\n          const TBlobBatch& batch = sparse_loader_->Value();\n          if (*dptr == nullptr) {\n            // allocate databatch\n            *dptr                   = new DataBatch();\n            (*dptr)->num_batch_padd = batch.num_batch_padd;\n            // (*dptr)->data.at(0) => data\n            // (*dptr)->data.at(1) => label\n            (*dptr)->data.resize(2);\n            (*dptr)->index.resize(batch.batch_size);\n            size_t data_iter = 0;\n            for (size_t i = 0; i < (*dptr)->data.size(); ++i) {\n              bool is_data = i == 0;\n              auto stype   = this->GetStorageType(is_data);\n              auto dtype   = param_.dtype ? param_.dtype.value() : batch.data[data_iter].type_flag_;\n              if (stype == kDefaultStorage) {\n                (*dptr)->data.at(i) =\n                    NDArray(batch.data[data_iter].shape_, Context::CPU(), false, dtype);\n              } else {\n                (*dptr)->data.at(i) =\n                    NDArray(stype, this->GetShape(is_data), Context::CPU(), false, dtype);\n              }\n              data_iter += num_aux_data(stype) + 1;\n            }\n          }\n          // copy data over\n          size_t data_iter = 0;\n          for (size_t i = 0; i < (*dptr)->data.size(); ++i) {\n            auto& nd     = ((*dptr)->data)[i];\n            auto stype   = nd.storage_type();\n            auto& data_i = ((*dptr)->data)[i];\n            if (stype == kDefaultStorage) {\n              CopyFromTo(data_i.data(), batch.data[data_iter]);\n            } else if (stype == kCSRStorage) {\n              auto& values  = batch.data[data_iter];\n              auto& indices = batch.data[data_iter + 1];\n              auto& indptr  = batch.data[data_iter + 2];\n              // allocate memory\n              CHECK_EQ(indices.shape_.Size(), values.shape_.Size());\n              nd.CheckAndAllocAuxData(csr::kIdx, indices.shape_);\n              nd.CheckAndAllocData(values.shape_);\n              nd.CheckAndAllocAuxData(csr::kIndPtr, indptr.shape_);\n              // copy values, indices and indptr\n              CopyFromTo(data_i.data(), values);\n              CopyFromTo(data_i.aux_data(csr::kIdx), indices);\n              CopyFromTo(data_i.aux_data(csr::kIndPtr), indptr);\n            } else {\n              LOG(FATAL) << \"Storage type not implemented: \" << stype;\n            }\n            data_iter += num_aux_data(stype) + 1;\n            (*dptr)->num_batch_padd = batch.num_batch_padd;\n          }\n          if (batch.inst_index) {\n            std::copy(\n                batch.inst_index, batch.inst_index + batch.batch_size, (*dptr)->index.begin());\n          }\n          return true;\n        },\n        [this]() { sparse_loader_->BeforeFirst(); });\n  }\n\n  virtual void BeforeFirst(void) {\n    PrefetcherIter::BeforeFirst();\n  }\n\n  virtual bool Next(void) {\n    return PrefetcherIter::Next();\n  }\n  virtual const DataBatch& Value(void) const {\n    return PrefetcherIter::Value();\n  }\n\n  virtual const NDArrayStorageType GetStorageType(bool is_data) const {\n    return sparse_loader_->GetStorageType(is_data);\n  }\n\n  virtual const mxnet::TShape GetShape(bool is_data) const {\n    return sparse_loader_->GetShape(is_data);\n  }\n\n private:\n  /*! \\brief internal sparse batch loader */\n  SparseIIterator<TBlobBatch>* sparse_loader_;\n\n  inline void CopyFromTo(TBlob dst, const TBlob src) {\n    MSHADOW_TYPE_SWITCH(src.type_flag_, DType, {\n      mshadow::Copy(dst.FlatTo1D<cpu, DType>(), src.FlatTo1D<cpu, DType>());\n    });\n  }\n};\n}  // namespace io\n}  // namespace mxnet\n#endif  // MXNET_IO_ITER_SPARSE_PREFETCHER_H_\n"
  },
  {
    "path": "src/io/opencv_compatibility.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file opencv_compatibility.h\n * \\brief To be compatible with multiple versions of opencv\n */\n#ifndef MXNET_IO_OPENCV_COMPATIBILITY_H_\n#define MXNET_IO_OPENCV_COMPATIBILITY_H_\n\n#if MXNET_USE_OPENCV\n#include <opencv2/core/version.hpp>\n\n#if CV_VERSION_MAJOR >= 4\n#include <opencv2/opencv.hpp>\n#define CV_RGB2GRAY cv::COLOR_RGB2GRAY\n#define CV_BGR2GRAY cv::COLOR_BGR2GRAY\n\n#define CV_GRAY2RGB cv::COLOR_GRAY2RGB\n#define CV_GRAY2BGR cv::COLOR_GRAY2BGR\n\n#define CV_RGB2HLS cv::COLOR_RGB2HLS\n#define CV_BGR2HLS cv::COLOR_BGR2HLS\n\n#define CV_HLS2RGB cv::COLOR_HLS2RGB\n#define CV_HLS2BGR cv::COLOR_HLS2BGR\n\n#define CV_RGB2BGR cv::COLOR_RGB2BGR\n#define CV_BGR2RGB cv::COLOR_BGR2RGB\n\n#define CV_INTER_LINEAR  cv::INTER_LINEAR\n#define CV_INTER_NEAREST cv::INTER_NEAREST\n\n#define CV_LOAD_IMAGE_COLOR        cv::IMREAD_COLOR\n#define CV_IMWRITE_PNG_COMPRESSION cv::IMWRITE_PNG_COMPRESSION\n#define CV_IMWRITE_JPEG_QUALITY    cv::IMWRITE_JPEG_QUALITY\n\n#endif  // CV_VERSION_MAJOR >= 4\n\n#endif  // MXNET_USE_OPENCV\n\n#endif  // MXNET_IO_OPENCV_COMPATIBILITY_H_\n"
  },
  {
    "path": "src/ir/expr.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file expr.cc\n * \\brief The expression AST nodes for the common IR infra.\n */\n// Acknowledgement: This file originates from incubator-tvm\n\n#include <mxnet/ir/expr.h>\n\nnamespace mxnet {\n\nIntImm::IntImm(MXNetDataType dtype, int64_t value) {\n  CHECK(dtype.is_scalar()) << \"ValueError: IntImm can only take scalar.\";\n  CHECK(dtype.is_int() || dtype.is_uint()) << \"ValueError: IntImm can only take scalar.\";\n  if (dtype.is_uint()) {\n    CHECK_GE(value, 0U);\n  }\n  runtime::ObjectPtr<IntImmNode> node = make_object<IntImmNode>();\n  node->dtype                         = dtype;\n  node->value                         = value;\n  data_                               = std::move(node);\n}\n\nFloatImm::FloatImm(MXNetDataType dtype, double value) {\n  CHECK_EQ(dtype.lanes(), 1) << \"ValueError: FloatImm can only take scalar.\";\n  runtime::ObjectPtr<FloatImmNode> node = make_object<FloatImmNode>();\n  node->dtype                           = dtype;\n  node->value                           = value;\n  data_                                 = std::move(node);\n}\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/kvstore/comm.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_KVSTORE_COMM_H_\n#define MXNET_KVSTORE_COMM_H_\n#include <dmlc/omp.h>\n#include <string>\n#include <algorithm>\n#include <utility>\n#include <limits>\n#include <vector>\n#include <tuple>\n#include <thread>\n#include \"mxnet/ndarray.h\"\n#include \"gradient_compression.h\"\n#include \"../ndarray/ndarray_function.h\"\n#include \"../operator/tensor/sparse_retain-inl.h\"\n#include \"../profiler/profiler.h\"\n#include \"./kvstore_utils.h\"\nnamespace mxnet {\nnamespace kvstore {\n/**\n * \\brief multiple device commmunication\n */\nclass Comm {\n public:\n  Comm() {\n    pinned_ctx_ = Context::CPUPinned(0);\n  }\n  virtual ~Comm() {}\n  /**\n   * \\brief init key with the data shape and storage shape\n   */\n  virtual void Init(int key,\n                    const NDArrayStorageType stype,\n                    const mxnet::TShape& shape,\n                    int dtype = mshadow::kFloat32) = 0;\n  /**\n   * \\brief returns src[0] + .. + src[src.size()-1]\n   */\n  virtual const NDArray& Reduce(int key, const std::vector<NDArray>& src, int priority) = 0;\n  /**\n   * \\brief copy from src to dst[i] for every i\n   */\n  virtual void Broadcast(int key,\n                         const NDArray& src,\n                         const std::vector<NDArray*> dst,\n                         int priority) = 0;\n\n  /**\n   * \\brief broadcast src to dst[i] with target row_ids for every i\n   * \\param key the identifier key for the stored ndarray\n   * \\param src the source row_sparse ndarray to broadcast\n   * \\param dst a list of destination row_sparse NDArray and its target row_ids to broadcast,\n            where the row_ids are expected to be unique and sorted in row_id.data()\n   * \\param priority the priority of the operation\n   */\n  virtual void BroadcastRowSparse(int key,\n                                  const NDArray& src,\n                                  const std::vector<std::pair<NDArray*, NDArray>>& dst,\n                                  const int priority) = 0;\n\n  /**\n   * \\brief return a pinned contex\n   */\n  Context pinned_ctx() const {\n    return pinned_ctx_;\n  }\n\n  /**\n   * \\brief Sets gradient compression parameters to be able to\n   * perform reduce with compressed gradients\n   */\n  void SetGradientCompression(std::shared_ptr<GradientCompression> gc) {\n    gc_ = gc;\n  }\n\n protected:\n  Context pinned_ctx_;\n\n  std::shared_ptr<GradientCompression> gc_;\n};\n\n/**\n * \\brief an implemention of Comm that first copy data to CPU memeory, and then\n * reduce there\n */\nclass CommCPU : public Comm {\n public:\n  CommCPU() {\n    nthread_reduction_ = dmlc::GetEnv(\"MXNET_KVSTORE_REDUCTION_NTHREADS\", 4);\n    bigarray_bound_    = dmlc::GetEnv(\"MXNET_KVSTORE_BIGARRAY_BOUND\", 1000 * 1000);\n    // TODO(junwu) delete the following data member, now for benchmark only\n    is_serial_push_ = dmlc::GetEnv(\"MXNET_KVSTORE_SERIAL_PUSH\", 0);\n  }\n  virtual ~CommCPU() {}\n\n  void Init(int key,\n            const NDArrayStorageType stype,\n            const mxnet::TShape& shape,\n            int type = mshadow::kFloat32) override {\n    // Delayed allocation - the dense merged buffer might not be used at all if push()\n    // only sees sparse arrays\n    bool delay_alloc       = true;\n    merge_buf_[key].merged = NDArray(shape, pinned_ctx_, delay_alloc, type);\n  }\n\n  const NDArray& Reduce(int key, const std::vector<NDArray>& src, int priority) override {\n    auto& buf        = merge_buf_[key];\n    const auto stype = src[0].storage_type();\n    // avoid extra copy for single device, but it may bring problems for\n    // abnormal usage of kvstore\n    if (src.size() == 1) {\n      if (stype == kDefaultStorage) {\n        return src[0];\n      } else {\n        // With 'local' kvstore, we could store the weight on CPU while compute\n        // the gradient on GPU when the weight is extremely large.\n        // To avoiding copying the weight to the same context of the gradient,\n        // we always copy the gradient to merged buf.\n        NDArray& merged = buf.merged_buf(stype);\n        CopyFromTo(src[0], &merged, priority);\n        return merged;\n      }\n    }\n\n    NDArray& buf_merged = buf.merged_buf(stype);\n    // normal dense reduce\n    if (stype == kDefaultStorage) {\n      std::vector<Engine::VarHandle> const_vars(src.size() - 1);\n      std::vector<NDArray> reduce(src.size());\n      CopyFromTo(src[0], &buf_merged, priority);\n      reduce[0] = buf_merged;\n\n      if (buf.copy_buf.empty()) {\n        buf.copy_buf.resize(src.size() - 1);\n        for (size_t j = 0; j < src.size() - 1; ++j) {\n          // allocate copy buffer\n          buf.copy_buf[j] = NDArray(src[0].shape(), pinned_ctx_, false, src[0].dtype());\n        }\n      }\n      CHECK(stype == buf.copy_buf[0].storage_type())\n          << \"Storage type mismatch detected. \" << stype << \"(src) vs. \"\n          << buf.copy_buf[0].storage_type() << \"(buf.copy_buf)\";\n      for (size_t i = 1; i < src.size(); ++i) {\n        CopyFromTo(src[i], &(buf.copy_buf[i - 1]), priority);\n        reduce[i]         = buf.copy_buf[i - 1];\n        const_vars[i - 1] = reduce[i].var();\n      }\n\n      Engine::Get()->PushAsync(\n          [reduce, this](RunContext rctx,\n                         Engine::CallbackOnStart on_start,\n                         Engine::CallbackOnComplete on_complete) {\n            on_start();\n            ReduceSumCPU(reduce);\n            on_complete();\n          },\n          Context::CPU(),\n          const_vars,\n          {reduce[0].var()},\n          FnProperty::kCPUPrioritized,\n          priority,\n          \"KVStoreReduce\");\n    } else {\n      // sparse reduce\n      std::vector<Engine::VarHandle> const_vars(src.size());\n      std::vector<NDArray> reduce(src.size());\n\n      if (buf.copy_buf.empty()) {\n        buf.copy_buf.resize(src.size());\n        for (size_t j = 0; j < src.size(); ++j) {\n          buf.copy_buf[j] =\n              NDArray(src[0].storage_type(), src[0].shape(), pinned_ctx_, true, src[0].dtype());\n        }\n      }\n      CHECK(stype == buf.copy_buf[0].storage_type())\n          << \"Storage type mismatch detected. \" << stype << \"(src) vs. \"\n          << buf.copy_buf[0].storage_type() << \"(buf.copy_buf)\";\n      for (size_t i = 0; i < src.size(); ++i) {\n        CopyFromTo(src[i], &(buf.copy_buf[i]), priority);\n        reduce[i]     = buf.copy_buf[i];\n        const_vars[i] = reduce[i].var();\n      }\n      Resource rsc = ResourceManager::Get()->Request(buf_merged.ctx(),\n                                                     ResourceRequest(ResourceRequest::kTempSpace));\n      Engine::Get()->PushAsync(\n          [reduce, buf_merged, rsc, this](RunContext rctx,\n                                          Engine::CallbackOnStart on_start,\n                                          Engine::CallbackOnComplete on_complete) {\n            on_start();\n            NDArray out = buf_merged;\n            is_serial_push_ ?\n                ReduceSumCPUExSerial(reduce, &out) :\n                mxnet::ndarray::ElementwiseSum(rctx.get_stream<cpu>(), rsc, reduce, &out);\n            on_complete();\n          },\n          Context::CPU(),\n          const_vars,\n          {buf_merged.var(), rsc.var},\n          FnProperty::kCPUPrioritized,\n          priority,\n          \"KVStoreReduce\");\n    }\n\n    return buf_merged;\n  }\n\n  void Broadcast(int key,\n                 const NDArray& src,\n                 const std::vector<NDArray*> dst,\n                 int priority) override {\n    int mask = src.ctx().dev_mask();\n    if (mask == Context::kCPU) {\n      for (auto d : dst)\n        CopyFromTo(src, d, priority);\n    } else {\n      // First copy data to pinned_ctx, then broadcast.\n      // Note that kv.init initializes the data on pinned_ctx.\n      // This branch indicates push() with ndarrays on gpus were called,\n      // and the source is copied to gpu ctx.\n      // Also indicates that buffers are already initialized during push().\n      auto& buf = merge_buf_[key].merged_buf(src.storage_type());\n      CopyFromTo(src, &buf, priority);\n      for (auto d : dst)\n        CopyFromTo(buf, d, priority);\n    }\n  }\n\n  void BroadcastRowSparse(int key,\n                          const NDArray& src,\n                          const std::vector<std::pair<NDArray*, NDArray>>& dst,\n                          const int priority) override {\n    using namespace mshadow;\n    CHECK_EQ(src.storage_type(), kRowSparseStorage)\n        << \"BroadcastRowSparse expects row-sparse src NDArray\";\n    CHECK_EQ(src.ctx().dev_mask(), Context::kCPU)\n        << \"BroadcastRowSparse with src on gpu context not supported\";\n    for (const auto& dst_kv : dst) {\n      NDArray* out   = dst_kv.first;\n      NDArray row_id = dst_kv.second;\n      CHECK_EQ(out->storage_type(), kRowSparseStorage)\n          << \"BroadcastRowSparse expects row_sparse dst NDArray\";\n      CHECK_EQ(row_id.ctx().dev_mask(), Context::kCPU)\n          << \"BroadcastRowSparse with row_indices on gpu context not supported\";\n      // retain according to unique indices\n      const bool is_same_ctx = out->ctx() == src.ctx();\n      const bool is_diff_var = out->var() != src.var();\n      NDArray retained_cpu =\n          (is_same_ctx && is_diff_var) ?\n              *out :\n              NDArray(\n                  kRowSparseStorage, src.shape(), src.ctx(), true, src.dtype(), src.aux_types());\n      if (!is_diff_var) {\n        common::LogOnce(\"The output of row_sparse_pull() on key \" + std::to_string(key) +\n                        \"refers to the same NDArray as the one stored in KVStore.\"\n                        \"Performing row_sparse_pull() with such output is going to change the \"\n                        \"data stored in KVStore. Incorrect result may be generated \"\n                        \"next time row_sparse_pull() is called. To avoid such an issue,\"\n                        \"consider create a new NDArray buffer to store the output.\");\n      }\n      Engine::Get()->PushAsync(\n          [=](RunContext rctx,\n              Engine::CallbackOnStart on_start,\n              Engine::CallbackOnComplete on_complete) {\n            on_start();\n            const TBlob& indices = row_id.data();\n            NDArray temp         = retained_cpu;  // get rid the of const qualifier\n            op::SparseRetainOpForwardRspImpl<cpu>(\n                rctx.get_stream<cpu>(), src, indices, kWriteTo, &temp);\n            on_complete();\n          },\n          Context::CPU(),\n          {src.var(), row_id.var()},\n          {retained_cpu.var()},\n          FnProperty::kNormal,\n          priority,\n          \"KVStoreSparseRetain\");\n      // if retained_cpu == out, CopyFromTo will ignore the copy operation\n      CopyFromTo(retained_cpu, out, priority);\n    }\n  }\n\n private:\n  // reduce sum into val[0]\n  inline void ReduceSumCPU(const std::vector<NDArray>& in_data) {\n    MSHADOW_TYPE_SWITCH(in_data[0].dtype(), DType, {\n      std::vector<DType*> dptr(in_data.size());\n      for (size_t i = 0; i < in_data.size(); ++i) {\n        TBlob data = in_data[i].data();\n        CHECK(data.CheckContiguous());\n        dptr[i] = data.FlatTo2D<cpu, DType>().dptr_;\n      }\n      size_t total = in_data[0].shape().Size();\n      ReduceSumCPUImpl(dptr, total);\n    });\n  }\n\n  // serial implementation of reduce sum for row sparse NDArray.\n  inline void ReduceSumCPUExSerial(const std::vector<NDArray>& in, NDArray* out) {\n    using namespace rowsparse;\n    using namespace mshadow;\n    auto stype = out->storage_type();\n    CHECK_EQ(stype, kRowSparseStorage) << \"Unexpected storage type \" << stype;\n    size_t total_num_rows = 0;\n    size_t num_in         = in.size();\n    // skip the ones with empty indices and values\n    std::vector<bool> skip(num_in, false);\n    // the values tensor of the inputs\n    MSHADOW_TYPE_SWITCH(out->dtype(), DType, {\n      MSHADOW_IDX_TYPE_SWITCH(out->aux_type(kIdx), IType, {\n        std::vector<Tensor<cpu, 2, DType>> in_vals(num_in);\n        std::vector<Tensor<cpu, 1, IType>> in_indices(num_in);\n        // offset to the values tensor of all inputs\n        std::vector<size_t> offsets(num_in, 0);\n        std::vector<size_t> num_rows(num_in, 0);\n        for (size_t i = 0; i < num_in; i++) {\n          if (!in[i].storage_initialized()) {\n            skip[i] = true;\n            continue;\n          }\n          auto size   = in[i].aux_shape(kIdx).Size();\n          num_rows[i] = size;\n          total_num_rows += size;\n          in_vals[i]    = in[i].data().FlatTo2D<cpu, DType>();\n          in_indices[i] = in[i].aux_data(kIdx).FlatTo1D<cpu, IType>();\n        }\n        std::vector<IType> indices;\n        indices.reserve(total_num_rows);\n        // gather indices from all inputs\n        for (size_t i = 0; i < num_in; i++) {\n          for (size_t j = 0; j < num_rows[i]; j++) {\n            indices.emplace_back(in_indices[i][j]);\n          }\n        }\n        CHECK_EQ(indices.size(), total_num_rows);\n        // dedup indices\n        std::sort(indices.begin(), indices.end());\n        indices.resize(std::unique(indices.begin(), indices.end()) - indices.begin());\n        // the one left are unique non-zero rows\n        size_t nnr = indices.size();\n        // allocate memory for output\n        out->CheckAndAlloc({Shape1(nnr)});\n        auto idx_data = out->aux_data(kIdx).FlatTo1D<cpu, IType>();\n        auto val_data = out->data().FlatTo2D<cpu, DType>();\n\n        for (size_t i = 0; i < nnr; i++) {\n          // copy indices back\n          idx_data[i] = indices[i];\n          bool zeros  = true;\n          for (size_t j = 0; j < num_in; j++) {\n            if (skip[j])\n              continue;\n            size_t offset = offsets[j];\n            if (offset < num_rows[j]) {\n              if (indices[i] == in_indices[j][offset]) {\n                if (zeros) {\n                  Copy(val_data[i], in_vals[j][offset], nullptr);\n                  zeros = false;\n                } else {\n                  val_data[i] += in_vals[j][offset];\n                }\n                offsets[j] += 1;\n              }\n            }\n          }\n        }\n      });\n    });\n  }\n\n  template <typename DType>\n  inline static void ReduceSumCPU(const std::vector<DType*>& dptr, size_t offset, index_t size) {\n    using namespace mshadow;  // NOLINT(*)\n    Tensor<cpu, 1, DType> in_0(dptr[0] + offset, Shape1(size));\n    for (size_t i = 1; i < dptr.size(); i += 4) {\n      switch (dptr.size() - i) {\n        case 1: {\n          Tensor<cpu, 1, DType> in_1(dptr[i] + offset, Shape1(size));\n          in_0 += in_1;\n          break;\n        }\n        case 2: {\n          Tensor<cpu, 1, DType> in_1(dptr[i] + offset, Shape1(size));\n          Tensor<cpu, 1, DType> in_2(dptr[i + 1] + offset, Shape1(size));\n          in_0 += in_1 + in_2;\n          break;\n        }\n        case 3: {\n          Tensor<cpu, 1, DType> in_1(dptr[i] + offset, Shape1(size));\n          Tensor<cpu, 1, DType> in_2(dptr[i + 1] + offset, Shape1(size));\n          Tensor<cpu, 1, DType> in_3(dptr[i + 2] + offset, Shape1(size));\n          in_0 += in_1 + in_2 + in_3;\n          break;\n        }\n        default: {\n          Tensor<cpu, 1, DType> in_1(dptr[i] + offset, Shape1(size));\n          Tensor<cpu, 1, DType> in_2(dptr[i + 1] + offset, Shape1(size));\n          Tensor<cpu, 1, DType> in_3(dptr[i + 2] + offset, Shape1(size));\n          Tensor<cpu, 1, DType> in_4(dptr[i + 3] + offset, Shape1(size));\n          in_0 += in_1 + in_2 + in_3 + in_4;\n          break;\n        }\n      }\n    }\n  }\n\n  template <typename DType>\n  inline void ReduceSumCPUImpl(std::vector<DType*> dptr, size_t total) {\n    const size_t step = std::min(bigarray_bound_, static_cast<size_t>(4 << 10));\n    long ntask        = (total + step - 1) / step;  // NOLINT(*)\n    if (total < bigarray_bound_ || nthread_reduction_ <= 1) {\n      ReduceSumCPU(dptr, 0, total);\n    } else {\n#pragma omp parallel for schedule(static) num_threads(nthread_reduction_)\n      for (long j = 0; j < ntask; ++j) {  // NOLINT(*)\n        size_t k     = static_cast<size_t>(j);\n        size_t begin = std::min(k * step, total);\n        size_t end   = std::min((k + 1) * step, total);\n        if (j == ntask - 1)\n          CHECK_EQ(end, total);\n        ReduceSumCPU(dptr, begin, static_cast<index_t>(end - begin));\n      }\n    }\n  }\n\n  /// \\brief temporal space for pushing and pulling\n  struct BufferEntry {\n    /// \\brief the merged value\n    NDArray merged;\n    /// \\brief the cpu buffer for gpu data\n    std::vector<NDArray> copy_buf;\n    /// \\brief the merged buffer for the given storage type\n    inline NDArray& merged_buf(NDArrayStorageType stype) {\n      if (stype == kDefaultStorage) {\n        return merged;\n      }\n      CHECK(stype == kRowSparseStorage) << \"unexpected storage type \" << stype;\n      // check if sparse_merged is initialized\n      if (sparse_merged.is_none()) {\n        CHECK(!merged.is_none());\n        sparse_merged =\n            NDArray(kRowSparseStorage, merged.shape(), merged.ctx(), true, merged.dtype());\n      }\n      return sparse_merged;\n    }\n\n   private:\n    /// \\brief the sparse merged value\n    NDArray sparse_merged;\n  };\n  std::unordered_map<int, BufferEntry> merge_buf_;\n  size_t bigarray_bound_;\n  int nthread_reduction_;\n  bool is_serial_push_;\n};\n\n/**\n * \\brief an implementation of Comm that performs reduction on device\n * directly.\n *\n * It is faster if the total device-to-device bandwidths is larger than\n * device-to-cpu, which is often true for 4 or 8 GPUs. But it uses more device\n * memory.\n */\nclass CommDevice : public Comm {\n public:\n  CommDevice() {\n    inited_ = false;\n  }\n\n  virtual ~CommDevice() {}\n\n  void Init(int key,\n            const NDArrayStorageType stype,\n            const mxnet::TShape& shape,\n            int dtype = mshadow::kFloat32) override {\n    sorted_key_attrs_.emplace_back(key, shape, dtype);\n    inited_ = false;\n  }\n\n  void InitBuffersAndComm(const std::vector<NDArray>& src) {\n    if (!inited_) {\n      std::vector<Context> devs;\n      for (const auto& a : src) {\n        devs.push_back(a.ctx());\n      }\n      InitMergeBuffer(devs);\n      if (dmlc::GetEnv(\"MXNET_ENABLE_GPU_P2P\", 1)) {\n        EnableP2P(devs);\n      }\n    }\n  }\n\n  const NDArray& ReduceRowSparse(int key, const std::vector<NDArray>& src, int priority) {\n    auto& buf = merge_buf_[key];\n    std::vector<NDArray> reduce(src.size());\n\n    const NDArrayStorageType stype = src[0].storage_type();\n    NDArray& buf_merged            = buf.merged_buf(stype);\n    if (buf.copy_buf.empty()) {\n      // initialize buffer for copying during reduce\n      buf.copy_buf.resize(src.size());\n      for (size_t j = 0; j < src.size(); ++j) {\n        buf.copy_buf[j] = NDArray(stype, src[0].shape(), buf_merged.ctx(), true, src[0].dtype());\n      }\n    }\n    CHECK(src[0].storage_type() == buf.copy_buf[0].storage_type())\n        << \"Storage type mismatch detected. \" << src[0].storage_type() << \"(src) vs. \"\n        << buf.copy_buf[0].storage_type() << \"(buf.copy_buf)\";\n    for (size_t i = 0; i < src.size(); ++i) {\n      CopyFromTo(src[i], &(buf.copy_buf[i]), priority);\n      reduce[i] = buf.copy_buf[i];\n    }\n    ElementwiseSum(reduce, &buf_merged, priority);\n    return buf_merged;\n  }\n\n  const NDArray& Reduce(int key, const std::vector<NDArray>& src, int priority) override {\n    // when this reduce is called from kvstore_dist, gc is not set\n    // we don't do compression twice in dist_sync_device\n    if ((gc_ != nullptr) && (gc_->get_type() != CompressionType::kNone)) {\n      return ReduceCompressed(key, src, priority);\n    }\n\n    // avoid extra copy for single device, but it may bring problems for\n    // abnormal usage of kvstore\n    if (src.size() == 1) {\n      return src[0];\n    }\n\n    InitBuffersAndComm(src);\n    auto& buf = merge_buf_[key];\n\n    const NDArrayStorageType stype = src[0].storage_type();\n    NDArray& buf_merged            = buf.merged_buf(stype);\n    // normal dense reduce\n    if (stype == kDefaultStorage) {\n      CopyFromTo(src[0], &buf_merged, priority);\n\n      std::vector<NDArray> reduce(src.size());\n      reduce[0] = buf_merged;\n\n      if (buf.copy_buf.empty()) {\n        // TODO(mli) this results in large device memory usage for huge ndarray,\n        // such as the largest fullc in VGG. consider to do segment reduce with\n        // NDArray.Slice or gpu direct memory access. for the latter, we need to\n        // remove some ctx check, and also it reduces 20% perf\n        buf.copy_buf.resize(src.size() - 1);\n        const std::string profiler_scope =\n            profiler::ProfilerScope::Get()->GetCurrentProfilerScope() + \"comm_dev:\";\n        for (size_t i = 0; i < src.size() - 1; ++i) {\n          buf.copy_buf[i] =\n              NDArray(buf_merged.shape(), buf_merged.ctx(), false, buf_merged.dtype());\n          buf.copy_buf[i].AssignStorageInfo(profiler_scope, \"copy_buf\");\n        }\n      }\n      for (size_t i = 0; i < src.size() - 1; ++i) {\n        CopyFromTo(src[i + 1], &(buf.copy_buf[i]), priority);\n        reduce[i + 1] = buf.copy_buf[i];\n      }\n      ElementwiseSum(reduce, &buf_merged, priority);\n    } else {\n      // sparse reduce\n      buf_merged = ReduceRowSparse(key, src, priority);\n    }\n    return buf_merged;\n  }\n\n  const NDArray& ReduceCompressed(int key, const std::vector<NDArray>& src, int priority) {\n    InitBuffersAndComm(src);\n    auto& buf = merge_buf_[key];\n    std::vector<NDArray> reduce(src.size());\n    if (buf.copy_buf.empty()) {\n      // one buf for each context\n      buf.copy_buf.resize(src.size());\n      buf.compressed_recv_buf.resize(src.size());\n      buf.compressed_send_buf.resize(src.size());\n      buf.residual.resize(src.size());\n      const std::string profiler_scope =\n          profiler::ProfilerScope::Get()->GetCurrentProfilerScope() + \"comm_dev:\";\n      for (size_t i = 0; i < src.size(); ++i) {\n        buf.copy_buf[i] = NDArray(buf.merged.shape(), buf.merged.ctx(), false, buf.merged.dtype());\n        buf.copy_buf[i].AssignStorageInfo(profiler_scope, \"copy_buf\");\n        buf.residual[i] = NDArray(buf.merged.shape(), src[i].ctx(), false, buf.merged.dtype());\n        buf.residual[i].AssignStorageInfo(profiler_scope, \"residual\");\n        buf.residual[i]    = 0;\n        int64_t small_size = gc_->GetCompressedSize(buf.merged.shape().Size());\n        buf.compressed_recv_buf[i] =\n            NDArray(mxnet::TShape{small_size}, buf.merged.ctx(), false, buf.merged.dtype());\n        buf.compressed_recv_buf[i].AssignStorageInfo(profiler_scope, \"compressed_recv_buf\");\n        buf.compressed_send_buf[i] =\n            NDArray(mxnet::TShape{small_size}, src[i].ctx(), false, buf.merged.dtype());\n        buf.compressed_send_buf[i].AssignStorageInfo(profiler_scope, \"compressed_send_buf\");\n      }\n    }\n\n    for (size_t i = 0; i < src.size(); ++i) {\n      // compress before copy\n      // this is done even if the data is on same context as copy_buf because\n      // we don't want the training to be biased towards data on this GPU\n      gc_->Quantize(src[i], &(buf.compressed_send_buf[i]), &(buf.residual[i]), priority);\n\n      if (buf.compressed_send_buf[i].ctx() != buf.compressed_recv_buf[i].ctx()) {\n        CopyFromTo(buf.compressed_send_buf[i], &(buf.compressed_recv_buf[i]), priority);\n      } else {\n        // avoid memory copy when they are on same context\n        buf.compressed_recv_buf[i] = buf.compressed_send_buf[i];\n      }\n\n      gc_->Dequantize(buf.compressed_recv_buf[i], &(buf.copy_buf[i]), priority);\n      reduce[i] = buf.copy_buf[i];\n    }\n    ElementwiseSum(reduce, &buf.merged);\n    return buf.merged;\n  }\n\n  void Broadcast(int key,\n                 const NDArray& src,\n                 const std::vector<NDArray*> dst,\n                 int priority) override {\n    if (!inited_) {\n      // copy to a random device first\n      int dev_id = key % dst.size();\n      CopyFromTo(src, dst[dev_id], priority);\n      for (size_t i = 0; i < dst.size(); ++i) {\n        if (i != static_cast<size_t>(dev_id)) {\n          CopyFromTo(*dst[dev_id], dst[i], priority);\n        }\n      }\n    } else {\n      auto& buf_merged = merge_buf_[key].merged_buf(src.storage_type());\n      CopyFromTo(src, &buf_merged, priority);\n      for (auto d : dst) {\n        CopyFromTo(buf_merged, d, priority);\n      }\n    }\n  }\n\n  void BroadcastRowSparse(int key,\n                          const NDArray& src,\n                          const std::vector<std::pair<NDArray*, NDArray>>& dst,\n                          const int priority) override {\n    CHECK_EQ(src.storage_type(), kRowSparseStorage)\n        << \"BroadcastRowSparse expects row-sparse src NDArray\";\n\n    for (const auto& dst_kv : dst) {\n      NDArray* out   = dst_kv.first;\n      NDArray row_id = dst_kv.second;\n      CHECK_EQ(out->storage_type(), kRowSparseStorage)\n          << \"BroadcastRowSparse expects row_sparse dst NDArray\";\n      CHECK_EQ(row_id.ctx(), src.ctx()) << \"row_id and src are expected to be on the same context\";\n\n      // retain according to indices\n      const bool is_same_ctx = out->ctx() == src.ctx();\n      const bool is_diff_var = out->var() != src.var();\n      NDArray retained_gpu =\n          (is_same_ctx && is_diff_var) ?\n              *out :\n              NDArray(\n                  kRowSparseStorage, out->shape(), src.ctx(), true, out->dtype(), out->aux_types());\n      if (!is_diff_var) {\n        common::LogOnce(\"The output of row_sparse_pull() on key \" + std::to_string(key) +\n                        \"refers to the same NDArray as the one stored in KVStore.\"\n                        \"Performing row_sparse_pull() with such output is going to change the \"\n                        \"data stored in KVStore. Incorrect result may be generated \"\n                        \"next time row_sparse_pull() is called. To avoid such an issue,\"\n                        \"consider create a new NDArray buffer to store the output.\");\n      }\n      bool is_gpu = retained_gpu.ctx().dev_mask() == gpu::kDevMask;\n      Engine::Get()->PushAsync(\n          [=](RunContext rctx,\n              Engine::CallbackOnStart on_start,\n              Engine::CallbackOnComplete on_complete) {\n            on_start();\n            const TBlob& indices = row_id.data();\n            using namespace mxnet::common;\n            NDArray temp = retained_gpu;\n            switch (temp.ctx().dev_mask()) {\n              case cpu::kDevMask: {\n                SparseRetainOpForwardRspWrapper<cpu>(\n                    rctx.get_stream<cpu>(), src, indices, kWriteTo, &temp);\n                break;\n              }\n#if MXNET_USE_CUDA\n              case gpu::kDevMask: {\n                SparseRetainOpForwardRspWrapper<gpu>(\n                    rctx.get_stream<gpu>(), src, indices, kWriteTo, &temp);\n                break;\n              }\n#endif\n              default:\n                LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR;\n            }\n            on_complete();\n          },\n          retained_gpu.ctx(),\n          {src.var(), row_id.var()},\n          {retained_gpu.var()},\n          is_gpu ? FnProperty::kGPUPrioritized : FnProperty::kCPUPrioritized,\n          priority,\n          \"KVStoreSparseRetain\");\n      CopyFromTo(retained_gpu, out, priority);\n    }\n  }\n\n  using KeyAttrs = std::tuple<int, mxnet::TShape, int>;\n  // try to allocate buff on device evenly\n  void InitMergeBuffer(const std::vector<Context>& devs) {\n    std::sort(sorted_key_attrs_.begin(),\n              sorted_key_attrs_.end(),\n              [](const KeyAttrs& a, const KeyAttrs& b) {\n                return std::get<1>(a).Size() > std::get<1>(b).Size();\n              });\n\n    std::unordered_map<int, std::pair<Context, size_t>> ctx_info;\n    for (auto d : devs) {\n      ctx_info[d.dev_id] = std::make_pair(d, 0);\n    }\n\n    const std::string profiler_scope =\n        profiler::ProfilerScope::Get()->GetCurrentProfilerScope() + \"kvstore:comm_dev:\";\n\n    for (auto& sorted_key_attr : sorted_key_attrs_) {\n      const int key              = std::get<0>(sorted_key_attr);\n      const mxnet::TShape& shape = std::get<1>(sorted_key_attr);\n      const int type             = std::get<2>(sorted_key_attr);\n      auto& buf                  = merge_buf_[key];\n      Context ctx;\n      size_t min_size = std::numeric_limits<size_t>::max();\n      for (auto& ctx_info_kv : ctx_info) {\n        size_t size = ctx_info_kv.second.second;\n        if (size <= min_size) {\n          ctx      = ctx_info_kv.second.first;\n          min_size = size;\n        }\n      }\n      // Delayed allocation - as the dense merged buffer might not be used at all if push()\n      // only sees sparse arrays\n      if (buf.merged.is_none()) {\n        bool delay_alloc = true;\n        buf.merged       = NDArray(shape, ctx, delay_alloc, type);\n        buf.merged.AssignStorageInfo(profiler_scope, \"merge_buf_\" + std::to_string(key));\n      }\n      ctx_info[ctx.dev_id].second += shape.Size();\n    }\n    inited_ = true;\n  }\n\n private:\n  void EnableP2P(const std::vector<Context>& devs) {\n#if MXNET_USE_CUDA\n    std::vector<int> gpus;\n    for (const auto& d : devs) {\n      if (d.dev_mask() == gpu::kDevMask) {\n        gpus.push_back(d.dev_id);\n      }\n    }\n    int n       = static_cast<int>(gpus.size());\n    int enabled = 0;\n    std::vector<int> p2p(n * n);\n\n    for (int i = 0; i < n; ++i) {\n      // Restores active device to what it was before EnableP2P\n      mxnet::common::cuda::DeviceStore device_store(gpus[i]);\n      for (int j = 0; j < n; j++) {\n        int access;\n        cudaDeviceCanAccessPeer(&access, gpus[i], gpus[j]);\n        if (access) {\n          cudaError_t e = cudaDeviceEnablePeerAccess(gpus[j], 0);\n          if (e == cudaSuccess || e == cudaErrorPeerAccessAlreadyEnabled) {\n            ++enabled;\n            p2p[i * n + j] = 1;\n          }\n        }\n      }\n    }\n    if (enabled != n * (n - 1)) {\n      // print warning info if not fully enabled\n      LOG(WARNING) << \"only \" << enabled << \" out of \" << n * (n - 1)\n                   << \" GPU pairs are enabled direct access. \"\n                   << \"It may affect the performance. \"\n                   << \"You can set MXNET_ENABLE_GPU_P2P=0 to turn it off\";\n      std::string access(n, '.');\n      for (int i = 0; i < n; ++i) {\n        for (int j = 0; j < n; ++j) {\n          access[j] = p2p[i * n + j] ? 'v' : '.';\n        }\n        LOG(WARNING) << access;\n      }\n    }\n#endif\n  }\n\n  /// \\brief temporal space for pushing and pulling\n  struct BufferEntry {\n    /// \\brief the dense merged value for reduce and broadcast operations\n    NDArray merged;\n    /// \\brief the gpu buffer for copy during reduce operation\n    std::vector<NDArray> copy_buf;\n    /// \\brief the residual buffer for gradient compression\n    std::vector<NDArray> residual;\n    /// \\brief the small buffer for compressed data in sender\n    std::vector<NDArray> compressed_send_buf;\n    /// \\brief the small buffer for compressed data in receiver\n    std::vector<NDArray> compressed_recv_buf;\n\n    /// \\brief the merged buffer for the given storage type (could be either dense or row_sparse)\n    inline NDArray& merged_buf(NDArrayStorageType stype) {\n      if (stype == kDefaultStorage) {\n        CHECK(!merged.is_none()) << \"unintialized merge buffer detected\";\n        return merged;\n      }\n      CHECK(stype == kRowSparseStorage) << \"unexpected storage type \" << stype;\n      // check if sparse_merged is initialized\n      if (sparse_merged.is_none()) {\n        CHECK(!merged.is_none());\n        sparse_merged =\n            NDArray(kRowSparseStorage, merged.shape(), merged.ctx(), true, merged.dtype());\n      }\n      return sparse_merged;\n    }\n\n   private:\n    /// \\brief the sparse merged value for reduce and rowsparse broadcast operations\n    NDArray sparse_merged;\n  };\n  std::unordered_map<int, BufferEntry> merge_buf_;\n\n public:\n  bool inited_;\n  std::vector<KeyAttrs> sorted_key_attrs_;\n};\n\n}  // namespace kvstore\n}  // namespace mxnet\n#endif  // MXNET_KVSTORE_COMM_H_\n"
  },
  {
    "path": "src/kvstore/comm_tree.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_KVSTORE_COMM_TREE_H_\n#define MXNET_KVSTORE_COMM_TREE_H_\n#include <dmlc/omp.h>\n#include <string>\n#include <algorithm>\n#include <utility>\n#include <limits>\n#include <vector>\n#include <tuple>\n#include <thread>\n#include <map>\n#include \"mxnet/ndarray.h\"\n#include \"gradient_compression.h\"\n#include \"../ndarray/ndarray_function.h\"\n#include \"../operator/tensor/sparse_retain-inl.h\"\n#include \"./kvstore_utils.h\"\n#include \"./gpu_topology.h\"\nnamespace mxnet {\nnamespace kvstore {\n/**\n * \\brief an implementation of Comm that performs reduction on device\n * directly using tree.\n *\n * It is faster if the total device-to-device bandwidths is larger than\n * device-to-cpu, which is often true for 4 or 8 GPUs. But it uses more device\n * memory.\n */\nclass CommDeviceTree : public CommDevice {\n public:\n  CommDeviceTree() {\n    inited_             = false;\n    gpuarray_bound_     = dmlc::GetEnv(\"MXNET_KVSTORE_TREE_ARRAY_BOUND\", 10000000);\n    backtrack_          = dmlc::GetEnv(\"MXNET_KVSTORE_TREE_BACKTRACK\", 0);\n    link_usage_penalty_ = dmlc::GetEnv(\"MXNET_KVSTORE_TREE_LINK_USAGE_PENALTY\", 0.7);\n  }\n\n  virtual ~CommDeviceTree() {}\n\n  void Init(int key,\n            const NDArrayStorageType stype,\n            const mxnet::TShape& shape,\n            int dtype = mshadow::kFloat32) override {\n    tree_sorted_key_attrs_.emplace_back(key, shape, dtype);\n    sorted_key_attrs_.emplace_back(key, shape, dtype);\n  }\n\n  void InitBuffersAndComm(const std::vector<NDArray>& src) {\n    if (!inited_) {\n      for (const auto& a : src) {\n        devs_.push_back(a.ctx());\n      }\n      QueryTopology();\n      // Note: delayed allocation set to true, because we do not want to allocate\n      // both in TreeBufferEntry and BufferEntry, so we use a size_t to keep\n      // track of each key's shape within BufferEntry\n      // -this information is required for inherited Reduce- and\n      //  BroadcastRowSparse\n      InitMergeBuffer(devs_);\n      InitMergeBufferTree();\n    }\n  }\n\n  /**\n   * \\brief Reduce src to tree_merge_buf_\n   * \\param key is the id of the gradient we are doing Reduce on\n   * \\param src is the array of values located on different GPUs\n   * \\param root is the id of the GPU we want to send result of reduce to\n   * \\param merged_row is the id of the slice we are taking\n   * \\param priority the priority of the operation\n   */\n  const NDArray& ReduceInner(int key,\n                             const std::vector<NDArray>& src,\n                             int root,\n                             int merged_row,\n                             int priority) {\n    std::vector<std::vector<NDArray>> reduce(devs_.size());\n\n    TreeBufferEntry& random_buf    = tree_merge_buf_[0][key];\n    const NDArrayStorageType stype = random_buf.merged[0].storage_type();\n    std::vector<size_t>& topology  = topology_[root];\n    NDArray buf_slice;\n\n    if (stype == kDefaultStorage) {\n      // Copy everything into buf.merged for each gpu\n      for (const auto& src_gpu_value : src) {\n        int start = scan_[root][depth_];\n        int end   = scan_[root][depth_ + 1];\n\n        for (int j = start; j < end; ++j) {\n          int topo_id          = topology[j];\n          TreeBufferEntry& buf = tree_merge_buf_[topo_id][key];\n\n          if (devs_[topo_id] == src_gpu_value.ctx()) {\n            CopyFromTo(src_gpu_value, &(buf.merged[merged_row]), priority);\n          }\n        }\n      }\n\n      for (int level = depth_; level > 0; --level) {\n        int start = scan_[root][level];\n        int end   = scan_[root][level + 1];\n\n        unsigned is_dest = 0;\n        int dest_id      = 0;\n        for (int j = start; j < end; ++j) {\n          int topo_id = topology[j];\n          dest_id     = (is_dest == 0) ? topo_id : dest_id;\n\n          TreeBufferEntry& buf_dest = tree_merge_buf_[dest_id][key];\n          TreeBufferEntry& buf_from = tree_merge_buf_[topo_id][key];\n\n          if (!is_dest) {\n            if (reduce[dest_id].size() == 0) {\n              reduce[dest_id].push_back(buf_dest.merged[merged_row]);\n            }\n          } else {\n            if (dest_id != topo_id) {\n              CopyFromTo(buf_from.merged[merged_row],\n                         &(buf_dest.copy_buf[merged_row][is_dest - 1]),\n                         priority);\n              reduce[dest_id].push_back(buf_dest.copy_buf[merged_row][is_dest - 1]);\n            }\n          }\n\n          is_dest = (is_dest == static_cast<unsigned>(kBranch) - 1) ? 0 : is_dest + 1;\n        }\n\n        start      = scan_[root][level - 1];\n        end        = scan_[root][level];\n        int source = end;\n        for (int i = start; i < end; ++i) {\n          int gpu_id = topology[i];\n\n          // source keeps track of 2 leaf nodes, while start keeps track of parent\n          int dest_id = topology[source];\n          int from_id = topology[source + 1];\n          source += 2;\n\n          // conditional to detect whether operation must be done\n          if (reduce[gpu_id].size() > 1 && dest_id != from_id) {\n            TreeBufferEntry& buf = tree_merge_buf_[gpu_id][key];\n            ElementwiseSum(reduce[gpu_id], &(buf.merged[merged_row]), priority);\n          }\n        }\n\n        // reset\n        for (unsigned i = 0; i < devs_.size(); ++i) {\n          reduce[i].clear();\n        }\n      }\n    } else {\n      LOG(FATAL) << \"Only dense input supported for now\";\n    }\n\n    int topo_id          = topology[0];\n    TreeBufferEntry& buf = tree_merge_buf_[topo_id][key];\n    return buf.merged[merged_row];\n  }\n\n  const NDArray& Reduce(int key, const std::vector<NDArray>& src, int priority) override {\n    // when this reduce is called from kvstore_dist, gc is not set\n    // we don't do compression twice in dist_sync_device\n    if ((gc_ != nullptr) && (gc_->get_type() != CompressionType::kNone)) {\n      return ReduceCompressed(key, src, priority);\n    }\n\n    // avoid extra copy for single device, but it may bring problems for\n    // abnormal usage of kvstore\n    if (src.size() == 1) {\n      return src[0];\n    }\n\n    InitBuffersAndComm(src);\n    std::vector<std::vector<NDArray>> slice(devs_.size());\n    std::vector<std::vector<NDArray*>> broadcast_slice(devs_.size());\n    std::vector<int> slice_scan(devs_.size() + 1);\n\n    int total_size      = src[0].shape().Size();\n    unsigned first_size = src[0].shape()[0];\n\n    const NDArrayStorageType stype = src[0].storage_type();\n    // normal dense reduce\n    if (stype == kDefaultStorage) {\n      if (total_size > gpuarray_bound_ && first_size >= 2 * devs_.size()) {\n        // Find slice bounds\n        slice_scan[0]  = 0;\n        int slice_size = first_size / devs_.size();\n        for (unsigned i = 1; i < devs_.size(); ++i) {\n          slice_scan[i] = slice_scan[i - 1] + slice_size;\n        }\n        slice_scan[devs_.size()] = src[0].shape()[0];\n\n        // row: which slice\n        // col: which gpu\n        for (unsigned row = 0; row < devs_.size(); ++row) {\n          for (unsigned col = 0; col < devs_.size(); ++col) {\n            TreeBufferEntry& buf = tree_merge_buf_[col][key];\n            NDArray curr_slice   = src[col].Slice(slice_scan[row], slice_scan[row + 1]);\n            slice[row].push_back(curr_slice);\n            broadcast_slice[row].push_back(&(buf.merged[row]));\n          }\n        }\n\n        // Do reduce-scatter (multiroot reduce)\n        // input:  slice (src)\n        // output: buf.merge_buf\n        for (unsigned i = 0; i < devs_.size(); ++i) {\n          ReduceInner(key, slice[i], i, i, priority);\n        }\n\n        for (unsigned i = 0; i < devs_.size(); ++i) {\n          BroadcastInner(key, *(broadcast_slice[i][i]), broadcast_slice[i], i, i, priority);\n        }\n      } else {\n        int root = 0;\n        ReduceInner(key, src, root, 0, priority);\n\n        TreeBufferEntry& buf = tree_merge_buf_[root][key];\n        return buf.merged[0];\n      }\n\n      // Copy from list of small NDArrays to one big NDArray, which is returned\n      int gpu_id = 0;\n      return src[gpu_id];\n    } else {\n      // sparse reduce\n      return ReduceRowSparse(key, src, priority);\n    }\n  }\n\n  void BroadcastInner(int key,\n                      const NDArray& src,\n                      const std::vector<NDArray*>& dst,\n                      int root,\n                      int merged_row,\n                      int priority) {\n    // copy to root of tree\n    std::vector<size_t>& topology = topology_[root];\n    std::vector<NDArray> temp(devs_.size());\n    int gpu_id = topology[0];\n    if (merged_row == -1)\n      CopyFromTo(src, dst[gpu_id], priority);\n    temp[gpu_id] = *dst[gpu_id];\n\n    for (int level = 1; level <= depth_; ++level) {\n      int start = scan_[root][level];\n      int end   = scan_[root][level + 1];\n\n      unsigned is_src = 0;\n      int src_id      = 0;\n      for (int j = start; j < end; ++j) {\n        int topo_id = topology[j];\n        src_id      = (is_src == 0) ? topo_id : src_id;\n\n        if (is_src && src_id != topo_id) {\n          CopyFromTo(temp[src_id], dst[topo_id], priority);\n          temp[topo_id] = *dst[topo_id];\n        }\n\n        is_src = (is_src == static_cast<unsigned>(kBranch) - 1) ? 0 : is_src + 1;\n      }\n    }\n  }\n\n  void Broadcast(int key,\n                 const NDArray& src,\n                 const std::vector<NDArray*> dst,\n                 int priority) override {\n    if (!inited_) {\n      // copy to a random device first\n      int dev_id = key % dst.size();\n      CopyFromTo(src, dst[dev_id], priority);\n      for (size_t i = 0; i < dst.size(); ++i) {\n        if (i != static_cast<size_t>(dev_id)) {\n          CopyFromTo(*dst[dev_id], dst[i], priority);\n        }\n      }\n    } else {\n      int total_size                 = src.shape().Size();\n      unsigned first_size            = src.shape()[0];\n      const NDArrayStorageType stype = src.storage_type();\n      // normal dense reduce\n      if (stype == kDefaultStorage) {\n        if (total_size > gpuarray_bound_ && first_size >= 2 * devs_.size()) {\n          std::vector<int> slice_scan(devs_.size() + 1);\n          slice_scan[0]  = 0;\n          int slice_size = (dst[0]->shape()[0]) / devs_.size();\n          for (unsigned i = 1; i < devs_.size(); ++i) {\n            slice_scan[i] = slice_scan[i - 1] + slice_size;\n          }\n          slice_scan[devs_.size()] = dst[0]->shape()[0];\n\n          for (unsigned gpu_id = 0; gpu_id < dst.size(); ++gpu_id) {\n            TreeBufferEntry& buf = tree_merge_buf_[gpu_id][key];\n            for (unsigned i = 0; i < devs_.size(); ++i) {\n              if (devs_[gpu_id] == dst[gpu_id]->ctx()) {\n                NDArray curr_slice = dst[gpu_id]->Slice(slice_scan[i], slice_scan[i + 1]);\n                CopyFromTo(buf.merged[i], &curr_slice, priority);\n              }\n            }\n          }\n        } else {\n          int root = 0;\n          BroadcastInner(key, src, dst, root, -1, priority);\n        }\n      } else {\n        LOG(FATAL) << \"Only dense input supported for now\";\n      }\n    }\n  }\n\n private:\n  void EnableP2P(std::vector<int>* p2p) {\n#if MXNET_USE_CUDA\n    std::vector<int> gpus;\n    for (const auto& d : devs_) {\n      if (d.dev_mask() == gpu::kDevMask) {\n        gpus.push_back(d.dev_id);\n      }\n    }\n    int n       = static_cast<int>(gpus.size());\n    int enabled = 0;\n    p2p->clear();\n    p2p->resize(n * n, 0);\n    for (int i = 0; i < n; ++i) {\n      mxnet::common::cuda::DeviceStore device_store(gpus[i]);\n      for (int j = 0; j < n; j++) {\n        int access;\n        cudaDeviceCanAccessPeer(&access, gpus[i], gpus[j]);\n        if (access) {\n          cudaError_t e = cudaDeviceEnablePeerAccess(gpus[j], 0);\n          if (e == cudaSuccess || e == cudaErrorPeerAccessAlreadyEnabled) {\n            ++enabled;\n            (*p2p)[i * n + j] = 1;\n          }\n        }\n      }\n    }\n    if (enabled != n * (n - 1)) {\n      // print warning info if not fully enabled\n      LOG(WARNING) << \"only \" << enabled << \" out of \" << n * (n - 1)\n                   << \" GPU pairs are enabled direct access. \"\n                   << \"It may affect the performance. \"\n                   << \"You can set MXNET_ENABLE_GPU_P2P=0 to turn it off\";\n      std::string access(n, '.');\n      for (int i = 0; i < n; ++i) {\n        for (int j = 0; j < n; ++j) {\n          access[j] = (*p2p)[i * n + j] ? 'v' : '.';\n        }\n        LOG(WARNING) << access;\n      }\n    }\n#endif\n  }\n\n  void QueryTopology() {\n#if MXNET_USE_CUDA\n    std::vector<float> link_matrix(devs_.size() * devs_.size());\n    std::vector<int> p2p_matrix(devs_.size() * devs_.size());\n    EnableP2P(&p2p_matrix);\n    GetP2PWeight(devs_, p2p_matrix, &link_matrix);\n    if (backtrack_)\n      LOG(INFO) << \"Using Backtracking to generate trees\";\n    else\n      LOG(INFO) << \"Using Kernighan-Lin to generate trees\";\n    ComputeTrees(link_matrix, devs_.size(), link_usage_penalty_, backtrack_, &topology_, &scan_);\n\n    depth_ = ComputeDepth(devs_.size());\n#endif\n  }\n\n  using KeyAttrs = std::tuple<int, mxnet::TShape, int>;\n  // try to allocate buff on device evenly\n  void InitMergeBufferTree() {\n    LOG(INFO) << \"Using Tree\";\n\n    // same as all-reduce, except:\n    // 1) Allocate copy_buf here instead of in Reduce()\n    // 2) Force copy_buf to be of kRecvBufferSize\n    // 3) Do not use greedy assignment; all keys are assigned to each GPU\n    for (unsigned i = 0; i < devs_.size(); ++i)\n      tree_merge_buf_.emplace_back();\n\n    bool delay_alloc = true;\n    std::map<int, int> key_dist;\n\n    const std::string profiler_scope =\n        profiler::ProfilerScope::Get()->GetCurrentProfilerScope() + \"comm_dev_tree:\";\n\n    for (auto& tree_sorted_key_attr : tree_sorted_key_attrs_) {\n      const int key              = std::get<0>(tree_sorted_key_attr);\n      const mxnet::TShape& shape = std::get<1>(tree_sorted_key_attr);\n      const int type             = std::get<2>(tree_sorted_key_attr);\n\n      if (key_dist.find(shape.Size()) == key_dist.end())\n        key_dist[shape.Size()] = 1;\n      else\n        key_dist[shape.Size()]++;\n\n      int start = scan_[0][depth_];\n      int end   = scan_[0][depth_ + 1];\n\n      // In order to generalize to any number of GPUs in arbitrary order, we use\n      // strategy of having found the mapping from 0, 1, ..., n_gpus to dev_id.\n      // For example, if the user wants to use --gpus 4,2,3,1,7,5,0, they can do      // so:\n      //\n      //   idx:    0 1 2 3 4 5 6\n      //   dev_id: 4 2 3 1 7 5 0\n      //\n      // From this, we:\n      // 1) generate a link topology matrix with dimensions n_gpus x n_gpus\n      //    (link_matrix)\n      //\n      // 2) the reduction trees are saved as indices from 0, 1, ..., n_gpus\n      //    in a vector of vectors (topology_):\n      //\n      //    index  | topology_[index]\n      //    -------------------------\n      //    0      | [Tree 0]\n      //    1      | [Tree 1]\n      //           .\n      //           .\n      //           .\n      //    n_gpus | [Tree n_gpus]\n      //\n      // 3) We use the mapping (devs_) to retrieve dev_id and device context\n      for (int j = start; j < end; ++j) {\n        int topo_id = topology_[0][j];\n        auto& buf   = tree_merge_buf_[topo_id][key];\n        Context ctx = devs_[topo_id];\n\n        // buf.merged enforces that we only visit each GPU once\n        if (buf.merged.empty()) {\n          mxnet::TShape shape_copy = shape;\n          int total_size           = shape.Size();\n          unsigned first_size      = shape[0];\n          if (total_size > gpuarray_bound_ && first_size >= 2 * devs_.size()) {\n            // Find slice bounds\n            int slice_size = first_size / devs_.size();\n            int last_slice = first_size - (devs_.size() - 1) * slice_size;\n            shape_copy[0]  = slice_size;\n            buf.merged.resize(devs_.size());\n            for (unsigned row = 0; row < devs_.size(); ++row) {\n              if (row == devs_.size() - 1)\n                shape_copy[0] = last_slice;\n              buf.merged[row] = NDArray(shape_copy, ctx, delay_alloc, type);\n              buf.merged[row].AssignStorageInfo(profiler_scope, \"merged_\" + std::to_string(key));\n              buf.copy_buf.emplace_back();\n              if (buf.copy_buf[row].empty()) {\n                buf.copy_buf[row].resize(kBranch - 1);\n                for (size_t col = 0; col < buf.copy_buf[0].size(); ++col) {\n                  buf.copy_buf[row][col] = NDArray(buf.merged[row].shape(),\n                                                   buf.merged[row].ctx(),\n                                                   delay_alloc,\n                                                   buf.merged[row].dtype());\n                  buf.copy_buf[row][col].AssignStorageInfo(profiler_scope, \"copy_buf\");\n                }\n              }\n            }\n          } else {\n            buf.merged.emplace_back(shape, ctx, false, type);\n            buf.merged.back().AssignStorageInfo(profiler_scope, \"merged_\" + std::to_string(key));\n            if (buf.copy_buf.empty()) {\n              buf.copy_buf.emplace_back();\n              buf.copy_buf[0].resize(kBranch - 1);\n              for (size_t col = 0; col < buf.copy_buf[0].size(); ++col) {\n                buf.copy_buf[0][col] = NDArray(\n                    buf.merged[0].shape(), buf.merged[0].ctx(), delay_alloc, buf.merged[0].dtype());\n                buf.copy_buf[0][col].AssignStorageInfo(profiler_scope, \"copy_buf\");\n              }\n            }\n          }\n        }\n      }\n    }\n\n    for (auto& kv : key_dist) {\n      LOG(INFO) << \"Size \" << kv.first << \" occurs \" << kv.second << \" times\";\n    }\n    inited_ = true;\n  }\n\n  std::vector<KeyAttrs> tree_sorted_key_attrs_;\n  /// \\brief temporal space for pushing and pulling\n  struct TreeBufferEntry {\n    /// \\brief the dense merged value for reduce and broadcast operations\n    std::vector<NDArray> merged;\n    /// \\brief the gpu buffer for copy during reduce operation\n    std::vector<std::vector<NDArray>> copy_buf;\n    /// \\brief the residual buffer for gradient compression\n    std::vector<NDArray> residual;\n    /// \\brief the small buffer for compressed data in sender\n    std::vector<NDArray> compressed_send_buf;\n    /// \\brief the small buffer for compressed data in receiver\n    std::vector<NDArray> compressed_recv_buf;\n\n   private:\n    /// \\brief the sparse merged value for reduce and rowsparse broadcast operations\n    NDArray sparse_merged;\n  };\n  /// \\brief intent of tree_merge_buf_ in old comm.h: store key->gpu mapping\n  ///        new intent: for every gpu: store key->memory mapping\n  std::vector<std::unordered_map<int, TreeBufferEntry>> tree_merge_buf_;\n\n  /// \\brief NVLink-connected topology in full binary tree format\n  std::vector<std::vector<size_t>> topology_;\n  std::vector<std::vector<size_t>> scan_;\n  std::vector<Context> devs_;\n\n  int depth_;\n  int gpuarray_bound_;\n  bool backtrack_;\n  float link_usage_penalty_;\n\n  /// \\brief constant for maximum size of recv buffer per GPU\n  ///        2: only receive from 1 other GPU\n  const int kBranch = 2;\n};\n\n}  // namespace kvstore\n}  // namespace mxnet\n#endif  // MXNET_KVSTORE_COMM_TREE_H_\n"
  },
  {
    "path": "src/kvstore/gpu_topology.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_KVSTORE_GPU_TOPOLOGY_H_\n#define MXNET_KVSTORE_GPU_TOPOLOGY_H_\n#if MXNET_USE_CUDA\n#include <cuda_runtime_api.h>\n#include <cuda.h>\n#endif\n#include <iostream>\n#include <vector>\n#include <algorithm>\n#include <utility>\n#include <limits>\n#include <random>\n#include <stack>\n#include <queue>\n#include <string>\n#include <unordered_set>\n#include <unordered_map>\n\n#define MXNET_KVSTORE_MAXDEPTH 16\n\nnamespace mxnet {\nnamespace kvstore {\n\nstatic bool kLogTree = dmlc::GetEnv(\"MXNET_KVSTORE_LOGTREE\", false);\n\ntemplate <typename T>\ninline void PrintVector(const std::string& str, const std::vector<T>& vec) {\n  LOG(INFO) << str << \":\";\n  std::string output;\n  for (unsigned i = 0; i < vec.size(); ++i)\n    output += std::to_string(vec[i]) + \" \";\n  LOG(INFO) << output;\n}\n\ntemplate <typename T>\ninline void PrintMatrix(const std::string& str,\n                        const std::vector<T>& matrix,\n                        int num_rows,\n                        int num_cols) {\n  LOG(INFO) << str << \":\";\n  int count = 0;\n  for (int row = 0; row < num_rows; ++row) {\n    std::string output;\n    for (int col = 0; col < num_cols; ++col) {\n      output += std::to_string(static_cast<int>(matrix[count++])) + \" \";\n    }\n    LOG(INFO) << output;\n  }\n}\n\ninline void PrintTopo(const std::string& str,\n                      const std::vector<size_t>& topo_row,\n                      std::vector<size_t> scan_row) {\n  LOG(INFO) << str << \":\";\n  int depth = scan_row.size() - 1;\n  for (int row = 0; row < depth; ++row) {\n    int start = scan_row[row];\n    int end   = scan_row[row + 1];\n    std::string output;\n    for (; start < end; start++) {\n      for (int i = 0; i < (2 << (depth - row - 2)) + 1; ++i) {\n        output += \" \";\n      }\n      output += std::to_string(topo_row[start]);\n    }\n    LOG(INFO) << output;\n  }\n}\n\n/**\n * \\brief Uses BFS to find whether undirected graph is connected or not given its\n * adjacency matrix\n * Note: only consider matrix values > 1, because we care about whether it is\n * connected using only NVLink connections\n */\ntemplate <typename T>\ninline bool IsConnected(const std::vector<T>& matrix, int num_gpus) {\n  int source = 0;\n  std::vector<bool> visited(num_gpus, false);\n  std::queue<int> work_list;\n\n  work_list.push(source);\n  visited[source] = true;\n  while (!work_list.empty()) {\n    int curr = work_list.front();\n    work_list.pop();\n\n    for (int i = 0; i < num_gpus; ++i) {\n      int neighbour = matrix[curr * num_gpus + i];\n      if (i != curr && neighbour > 1 && visited[i] == false) {\n        visited[i] = true;\n        work_list.push(i);\n      }\n    }\n  }\n\n  for (int i = 0; i < num_gpus; ++i) {\n    if (visited[i] == false)\n      return false;\n  }\n  return true;\n}\n\n/**\n * \\brief Generate adjacency matrix with row/col numbering from 0, 1, ..., n_gpu\n * \\param devs is a vector of GPU contexts\n * \\param p2p_matrix is adjacency matrix of P2P connections where\n *          0: no P2P connection\n *          1: P2P connection\n * \\param matrix is adjacency matrix of link topology graph\n *        where edge weight represents relative performance of NVIDIA GPUs\n *          0: Self-connection\n *          1: PCI-E\n *          2: 1 NVLink connection\n *          3: 2 NVLink connections\n */\ntemplate <typename T>\ninline void GetP2PWeight(const std::vector<Context>& devs,\n                         const std::vector<int>& p2p_matrix,\n                         std::vector<T>* matrix) {\n  int num_gpus = devs.size();\n  int count    = 0;\n  std::vector<int> zero_dev_id(num_gpus, -1);\n  for (auto d : devs) {\n    zero_dev_id[count] = d.dev_id;\n    count++;\n  }\n\n#if MXNET_USE_CUDA\n  cudaDeviceP2PAttr attr;\n  attr = cudaDevP2PAttrPerformanceRank;\n  std::vector<int> max(num_gpus, 0);\n\n  for (int row = 0; row < num_gpus; ++row) {\n    for (int col = 0; col < num_gpus; ++col) {\n      if (row == col) {\n        (*matrix)[row * num_gpus + col] = 0;\n      } else {\n        int value;\n        int row_gpu = zero_dev_id[row];\n        int col_gpu = zero_dev_id[col];\n        cudaDeviceGetP2PAttribute(&value, attr, row_gpu, col_gpu);\n        if (value > max[row])\n          max[row] = value;\n        (*matrix)[row * num_gpus + col] = static_cast<T>(value) + 1;\n      }\n    }\n  }\n\n  // Check that all P2P connections are detected by GetP2PAttribute\n  // If yes, then continue as before\n  // If not, then treat fallback to using p2p_matrix (from EnableP2P)\n  //\n  // We have observed that with CUDA 9.0 p3.16xlarge:\n  //\n  //   0 2 2 3 3 1 1 1    . v v v v . . .\n  //   2 0 3 2 1 3 1 1    v . v v . v . .\n  //   2 3 0 3 1 1 2 1    v v . v . . v .\n  //   3 2 3 0 1 1 1 2    v v v . . . . v\n  //   3 1 1 1 0 2 2 3    v . . . . v v v\n  //   1 3 1 1 2 0 3 2    . v . . v . v v\n  //   1 1 2 1 2 3 0 3    . . v . v v . v\n  //   1 1 1 2 3 2 3 0    . . . v v v v .\n  //\n  //        matrix           p2p_matrix\n  //\n  // Here, they are correctly detected, because the 2s and 3s correspond to\n  // links that have P2P connections between them. However for CUDA 9.2 p3.16xlarge:\n  //\n  //   0 2 2 1 1 1 1 1    . v v v v . . .\n  //   2 0 1 2 1 1 1 1    v . v v . v . .\n  //   2 1 0 1 1 1 2 1    v v . v . . v .\n  //   1 2 1 0 1 1 1 2    v v v . . . . v\n  //   1 1 1 1 0 2 2 1    v . . . . v v v\n  //   1 1 1 1 2 0 1 2    . v . . v . v v\n  //   1 1 2 1 2 1 0 1    . . v . v v . v\n  //   1 1 1 2 1 2 1 0    . . . v v v v .\n  //\n  //        matrix          p2p_matrix\n  //\n  // The fastest connections (3 - double NVLink) are not recognized as being any\n  if (kLogTree) {\n    PrintMatrix(\"matrix\", *matrix, num_gpus, num_gpus);\n    PrintMatrix(\"p2p_matrix\", p2p_matrix, num_gpus, num_gpus);\n  }\n\n  // different from (1 - non-P2P PCI-E). This is why we fallback to p2p_matrix.\n  bool matrix_correct = true;\n  for (unsigned i = 0; i < p2p_matrix.size(); ++i) {\n    if (p2p_matrix[i] > 0 && (*matrix)[i] == 1) {\n      matrix_correct = false;\n      break;\n    }\n  }\n\n  if (!matrix_correct) {\n    LOG(WARNING) << \"cudaDeviceGetP2PAttribute incorrect. \"\n                 << \"Falling back to cudaDeviceEnablePeerAccess for topology detection\";\n    for (unsigned i = 0; i < p2p_matrix.size(); ++i) {\n      if (p2p_matrix[i] > 0)\n        (*matrix)[i] = 2;\n      else\n        (*matrix)[i] = 1;\n    }\n  }\n\n  // If all GPUs are connected by NVLink, then we can use NVLink only\n  // to communicate instead of going over PCI-E, so we set PCI-E links to 0\n  //\n  // Otherwise, we will make distinction between PCI-E GPUDirect links and\n  // PCI-E through CPU links, which are slower and show queueing effect (i.e.\n  // The most packets there are, the slower).\n  //\n  // For the latter links, we will set links that were 0 to 1/num_gpus to\n  // account for this queuing effect.\n  bool connected = IsConnected(*matrix, num_gpus);\n\n  if (connected) {\n    for (auto& matrix_value : *matrix) {\n      matrix_value = (matrix_value == 1) ? 0 : matrix_value;\n    }\n  } else {\n    for (auto& matrix_value : *matrix) {\n      matrix_value = (matrix_value == 1) ? 1. / num_gpus : matrix_value;\n    }\n  }\n  if (kLogTree)\n    PrintMatrix(\"Weight\", *matrix, num_gpus, num_gpus);\n\n#else\n  LOG(WARNING) << \"GPU required for link topology\";\n#endif\n}\n\n/**\n * \\brief Dense matrix-vector multiplication\n * Assume: matrix is square\n *   y = A*x (no accumulate)\n */\ntemplate <typename T>\ninline void gemv(const std::vector<T>& A, const std::vector<int>& x, std::vector<T>* y) {\n  int nrows = x.size();\n  int count = 0;\n  for (int row = 0; row < nrows; ++row) {\n    (*y)[row] = 0;\n    for (int col = 0; col < nrows; ++col) {\n      (*y)[row] += A[count] * static_cast<T>(x[col]);\n      count++;\n    }\n  }\n}\n\n/**\n * \\brief Element-wise multiplication between 2 dense vectors\n *   w = w * alpha*u\n */\ntemplate <typename T>\ninline void ewisemult(const std::vector<int>& u, T alpha, std::vector<T>* w) {\n  int nelem = u.size();\n  for (int i = 0; i < nelem; ++i) {\n    (*w)[i] *= alpha * static_cast<T>(u[i]);\n  }\n}\n\n/**\n * \\brief Computes best 2 nodes a,b to swap given objective function:\n *   g = max_{a \\in A, b \\in B} D(a) + D(b) - 2*W(a,b)\n *\n * Optimization: Only need to look at upper triangular since weight matrix is\n * symmetric\n */\ntemplate <typename T>\ninline void FindBestMove(const std::vector<T>& W,\n                         const std::vector<int>& P_temp,\n                         const std::vector<T>& D,\n                         const std::unordered_set<int>& used,\n                         int* a,\n                         int* b,\n                         T* g) {\n  int nrows = P_temp.size();\n  *g        = 0;\n  *a        = -1;\n  *b        = -1;\n  for (int row = 0; row < nrows; ++row) {\n    if (P_temp[row] == 0 || used.find(row) != used.end())\n      continue;\n    for (int col = row + 1; col < nrows; ++col) {\n      if (P_temp[col] == 0 || P_temp[row] == P_temp[col])\n        continue;\n\n      T cost = D[row] + D[col] - 2 * W[row * nrows + col];\n      if (cost > *g) {\n        *g = cost;\n        *a = row;\n        *b = col;\n      }\n    }\n  }\n}\n\n/**\n * \\brief Performs partition on each existing partition in graph W if partition has\n * more than 4 elements in it\n * \\param stop returns true if no partitions with >=4 elements found\n *             returns false otherwise\n * \\param cluster_pairs stores the mapping that tells us which 2 clusters are\n *        the output of partitioning one large cluster\n */\ntemplate <typename T>\ninline bool KernighanLin(const std::vector<T>& W,\n                         std::vector<int>* P,\n                         int* num_partitions,\n                         std::vector<std::pair<int, int>>* cluster_pairs,\n                         std::mt19937* gen) {\n  std::vector<int> histogram(*num_partitions, 0);\n  std::vector<int> P_temp(P->size(), 0);\n  std::vector<int> P_temp2(P->size(), 0);\n  std::vector<T> D(P->size(), 0);\n  std::vector<T> D_temp(P->size(), 0);\n\n  // 0) For every partition, determine if it can be partitioned further.\n  //    To do this, we must do a histogram of each partition:\n  for (int partition : *P) {\n    histogram[partition]++;\n  }\n\n  bool stop = true;\n  for (unsigned color = 0; color < histogram.size(); ++color) {\n    int partition_size = histogram[color];\n    // Save cluster in preparation for push to topo in GenerateBinaryTree()\n    if (partition_size <= 2) {\n      cluster_pairs->push_back(std::pair<int, int>(static_cast<int>(color), -partition_size));\n\n      // Do Kernighan-Lin if clustering is necessary\n    } else {\n      stop = false;\n\n      // 1) If it has more than 4 elements, we can partition further.\n      //    Assign random balanced partition of it\n      //   -balanced is more important than random, so allocate first half to A\n      //    and rest to B\n      int first_partition  = 0;\n      int target_partition = partition_size / 2;\n      std::vector<int> cluster_list;\n\n      for (unsigned i = 0; i < P->size(); ++i) {\n        // Required to shift from [0,1] to {-1,1}\n        //  1 means vertex i is in Cluster A\n        // -1 means vertex i is in Cluster B\n        if ((*P)[i] == static_cast<int>(color)) {\n          cluster_list.push_back(i);\n        } else {\n          P_temp[i] = 0;\n        }\n      }\n\n      // 1b) Shuffle using random generator\n      std::shuffle(cluster_list.begin(), cluster_list.end(), *gen);\n      for (int cluster : cluster_list) {\n        if (first_partition < target_partition) {\n          int dest     = cluster;\n          P_temp[dest] = 1;\n          first_partition++;\n        } else {\n          int dest     = cluster;\n          P_temp[dest] = -1;\n        }\n      }\n\n      // 2) Do iterations of Kernighan-Lin until convergence\n      T g_max        = 0;\n      int g_k        = -1;\n      unsigned count = 0;\n      do {\n        count++;\n        P_temp2 = P_temp;\n\n        // a) Compute difference between external and internal costs of all\n        //    elements in vector D\n        gemv(W, P_temp, &D);\n        ewisemult(P_temp, -1.f, &D);\n\n        // av and bv are used to hold candidates for moving\n        // gv stores the score associated with move\n        std::vector<int> av;\n        std::vector<int> bv;\n        std::vector<T> gv;\n\n        std::unordered_set<int> used;\n\n        for (int iter = 0; iter < partition_size / 2; ++iter) {\n          // b) Find best move by looking through upper triangular of W matrix\n          int a, b;\n          T g;\n          FindBestMove(W, P_temp, D, used, &a, &b, &g);\n          if (g > 0) {\n          } else {\n            g_max = 0;\n            break;\n          }\n\n          // c) Store best move to av, bv, gv\n          av.push_back(a);\n          bv.push_back(b);\n          gv.push_back(g);\n\n          // d) Eliminate best move from consideration in vector P_temp\n          P_temp[a] *= -1;\n          P_temp[b] *= -1;\n          used.insert(a);\n          used.insert(b);\n\n          // e) Update D using P_temp\n          gemv(W, P_temp, &D);\n          ewisemult(P_temp, -1.f, &D);\n          D[a] = 0;\n          D[b] = 0;\n        }\n\n        // 3) Find when to stop by doing linear scan through gv\n        //    Recompute score g_max\n        for (unsigned k = 0; k < gv.size(); ++k) {\n          if (k > 0)\n            gv[k] += gv[k - 1];\n          if (gv[k] > g_max) {\n            g_max = gv[k];\n            g_k   = k + 1;\n          }\n        }\n\n        // 4) If move is \"good\", commit moves by updating P_temp and P_temp2\n        //    Otherwise, rollback changes to P_temp2\n        if (g_max > 0) {\n          for (int i = 0; i < g_k; i++) {\n            int a      = av[i];\n            int b      = bv[i];\n            int temp   = P_temp2[a];\n            P_temp2[a] = P_temp2[b];\n            P_temp2[b] = temp;\n\n            P_temp = P_temp2;\n          }\n        } else {\n          P_temp = P_temp2;\n        }\n      } while (g_max > 0 && count <= P->size());\n\n      // 5) Update P using P_temp\n      int moves = 0;\n      for (unsigned i = 0; i < P->size(); ++i) {\n        if (P_temp[i] == -1) {\n          (*P)[i] = *num_partitions;\n          moves++;\n        }\n      }\n      cluster_pairs->push_back(\n          std::pair<int, int>(static_cast<int>(color), static_cast<int>(*num_partitions)));\n\n      (*num_partitions)++;\n    }\n  }\n\n  return stop;\n}\n\n/**\n * \\brief Returns root of a given color if found in roots\n *        Returns -1 if it is not found\n */\ninline int GetRoot(const std::vector<int>& P, int color, const std::unordered_set<int>& roots) {\n  for (auto root : roots) {\n    if (P[root] == color)\n      return root;\n  }\n  return -1;\n}\n\n/**\n * \\brief Returns root of a given color if found in roots\n *        Returns -1 if it is not found\n */\ninline int GetChild(const std::vector<int>& P, int color, int parent) {\n  for (unsigned i = 0; i < P.size(); ++i) {\n    if (P[i] == color && static_cast<int>(i) != parent)\n      return i;\n  }\n  return -1;\n}\n\n// Computes highest weighted edge a-b\n//\n// Contraints:\n//  -vertex a must be parent\n//  -vertex b must be in dest_cluster\n//\n// @output: b is vector of candidates if a tie happens\n//          g is weight of edge\n// Optimization: Only need to look at row a in matrix\ntemplate <typename T>\ninline void FindBestEdge(const std::vector<T>& W,\n                         const std::vector<int>& P,\n                         int parent,\n                         int dest_cluster,\n                         std::vector<int>* b,\n                         T* g) {\n  int nrows = P.size();\n  int row   = parent;\n  *g        = 0;\n  b->push_back(-1);\n  for (int col = 0; col < nrows; ++col) {\n    if (col == row || P[col] != dest_cluster)\n      continue;\n\n    T cost = W[row * nrows + col];\n    if (cost > *g) {\n      b->clear();\n    }\n    if (cost >= *g) {\n      b->push_back(col);\n      *g = cost;\n    }\n  }\n}\n\n// Given a vector of color pairs, appends to binary tree matrix topo\n// @input:  W gives the link topology\n//          P gives the result of KL partitioning\n//          cluster_pairs gives pairing between clusters, an edge is found\n//                        between each pairing\n//          roots gives source vertices\n//          gen gives random number generation to break ties\n// @output: cluster_pairs\n//          topo_row says where new edges are appended to\n//          scan_row says where we should start looking for topo_row\ntemplate <typename T>\ninline int KLGenerateBinaryTree(const std::vector<T>& W,\n                                const std::vector<int>& P,\n                                std::vector<std::pair<int, int>>* cluster_pairs,\n                                std::unordered_set<int>* roots,\n                                std::vector<size_t>* topo_row,\n                                std::vector<size_t>* scan_row,\n                                std::mt19937* gen) {\n  std::unordered_set<int> new_roots;\n  std::unordered_map<int, int> new_topo;\n  int reset = 0;\n\n  for (unsigned i = 0; i < cluster_pairs->size(); ++i) {\n    if (i == 0)\n      scan_row->push_back(topo_row->size());\n    int parent, child = -1;\n    if ((*cluster_pairs)[i].second == -2) {\n      // Root must be color of pair.first\n      int color = (*cluster_pairs)[i].first;\n      parent    = GetRoot(P, color, *roots);\n      if (parent == -1)\n        return 1;\n      child = GetChild(P, color, parent);\n    } else if ((*cluster_pairs)[i].second == -1) {\n      int color = (*cluster_pairs)[i].first;\n      parent    = GetRoot(P, color, *roots);\n      if (parent == -1)\n        return 1;\n      child = parent;\n    } else {\n      // Root must exist in either first or second element of pair\n      int color = (*cluster_pairs)[i].first;\n      parent    = GetRoot(P, color, *roots);\n      color     = (parent == -1) ? (*cluster_pairs)[i].second : color;\n      parent    = (parent == -1) ? GetRoot(P, color, *roots) : parent;\n\n      int from_cluster = color;\n      int dest_cluster = (from_cluster == (*cluster_pairs)[i].first) ? (*cluster_pairs)[i].second :\n                                                                       (*cluster_pairs)[i].first;\n\n      std::vector<int> candidates;\n      T weight;\n      FindBestEdge(W, P, parent, dest_cluster, &candidates, &weight);\n\n      // If no candidates\n      if (candidates[0] != -1) {\n        std::shuffle(candidates.begin(), candidates.end(), *gen);\n        child = candidates[0];\n      }\n\n      if (child == -1) {\n        new_roots.insert(parent);\n        return 1;\n      } else {\n        new_roots.insert(parent);\n        new_roots.insert(child);\n      }\n    }\n\n    new_topo[parent] = child;\n  }\n\n  int depth = scan_row->size();\n  int start = (*scan_row)[depth - 2];\n  int end   = (*scan_row)[depth - 1];\n\n  for (int i = start; i < end; ++i) {\n    int parent = (*topo_row)[i];\n    int child;\n\n    // If not first, check previous level whether or not we are encountering\n    // this root for the first time in this level of the tree\n    if (i != start && parent == static_cast<int>((*topo_row)[i - 1]))\n      child = parent;\n    else\n      child = new_topo[parent];\n    topo_row->push_back(parent);\n    topo_row->push_back(child);\n  }\n\n  cluster_pairs->clear();\n  roots->clear();\n  *roots = std::move(new_roots);\n\n  return reset;\n}\n\n// @input: n is the number of nodes in a balanced binary tree\n// @output: returns how many levels of binary tree there are\ninline int ComputeDepth(int n) {\n  for (int depth = 0; depth < MXNET_KVSTORE_MAXDEPTH; ++depth) {\n    int num = 2 << depth;\n    if (n <= num)\n      return depth + 1;\n  }\n  return 0;\n}\n\n// Checks whether a given state forms a spanning tree that satisfies:\n//   -balanced\n//   -binary\n//   -each edge in tree corresponds to link in network topology\n//   -each edge in tree does not form self-loop\ntemplate <typename T>\ninline bool IsValid(const std::vector<T>& W,\n                    const std::vector<int>& state,\n                    int num_elements,\n                    int row,\n                    int depth) {\n  // At each level of tree, check whether edge:\n  //   -corresponds to link in network topology\n  //   -corresponds to self-loop\n  for (int i = 0; i < depth; ++i) {\n    int stride = 1 << i;\n    for (int j = 0; j + stride < row; j += 2 * stride) {\n      int from = state[j];\n      int dest = state[j + stride];\n      if (W[from * num_elements + dest] == static_cast<T>(0) && from != dest) {\n        return false;\n      }\n    }\n  }\n\n  // If we encounter GPU for first time, increment found_vec.\n  // Otherwise, do nothing\n  std::unordered_set<int> found;\n  std::vector<int> found_vec(num_elements, 0);\n  for (auto val : state) {\n    if (val == -1)\n      continue;\n    if (val < num_elements) {\n      if (found.find(val) == found.end()) {\n        found.insert(val);\n        found_vec[val] = 1;\n      }\n    } else {\n      return false;\n    }\n  }\n\n  // modifier is maximum number of repeats a single GPU can take\n  //   e.g. 5 GPUs in 3-level binary tree => one GPU can repeat 3x\n  //        GPU0 GPU0 GPU0 GPU0 GPU1 GPU2 GPU3 GPU4\n  int modifier  = (1 << depth) - num_elements;\n  int num_found = found.size();\n\n  // So we know we have an invalid state if we find:\n  //   -only 4 unique GPUs\n  //   -9 unique GPUs\n  if (row < num_elements) {\n    if (num_found > row || num_found < row - modifier) {\n      return false;\n    }\n\n    // If we are at last recursive level, we can apply a more stringent check:\n    //   -if some GPU is not found, then we are in invalid state\n  } else if (row == static_cast<int>(state.size())) {\n    for (int i = 0; i < num_elements; ++i) {\n      if (found_vec[i] == 0) {\n        return false;\n      }\n    }\n  }\n\n  return true;\n}\n\n// This function takes a spanning tree encoded as state (result), which may have\n// repeated GPUs representing NO-SENDs and converts it into a unique format.\n// This has the effect of recognizing redundant sends, grouping them together,\n// so that the Reduce call knows not to perform a CopyFromTo.\n//\n// Initial result: [3 0 0 4 1 2 5 6]\n// Final result:   [3 3 0 4 1 2 5 6]\n//\n// Initial:\n//         3\n//     3     1\n//   3   0   1   5\n// 3 0 0 4 1 2 5 6    // GPU3 will make redundant send to GPU0\n//\n// Final:\n//         3\n//     3     1\n//   3   0   1   5\n// 3 3 0 4 1 2 5 6    // GPU3 knows not to make redundant send to itself\ninline void Postprocess(std::vector<int>* result, int num_elements, int depth) {\n  for (int level = depth - 1; level >= 0; --level) {\n    int stride = 1 << level;\n    std::vector<int> histogram_above(num_elements, 0);\n    for (unsigned i = 0; i < result->size(); i += 2 * stride) {\n      int val = (*result)[i];\n      histogram_above[val]++;\n    }\n    std::vector<int> histogram(num_elements, 0);\n    for (unsigned i = 0; i < result->size(); i += stride) {\n      int val = (*result)[i];\n      histogram[val]++;\n    }\n\n    for (int i = result->size() - stride; i - stride >= 0; i -= 2 * stride) {\n      int from = (*result)[i];\n      int dest = (*result)[i - stride];\n      if ((histogram[from] > 1 || histogram_above[from] >= 1) && from != dest) {\n        (*result)[i] = dest;\n        histogram[from]--;\n      }\n    }\n  }\n}\n\n// Given a spanning tree encoded as a state (result) and weight of each edge\n// in the link topology graph, compute its weight.\n// @input: penalty controls whether or not penalties are applied to tree\n//         -usually turned on when backtracking to get better solutions\n//         -usually turned off when outside the penalty to get weight of tree\ntemplate <typename T>\ninline T ComputeTreeWeight(const std::vector<T>& W,\n                           const std::vector<int>& result,\n                           int num_elements,\n                           int depth,\n                           bool penalty) {\n  T weight = 0.f;\n  std::unordered_set<int> links_used;\n\n  for (int i = 0; i < depth; ++i) {\n    int stride = 1 << i;\n    std::vector<bool> nodes_used(num_elements, false);\n    for (unsigned j = 0; j + stride < result.size(); j += 2 * stride) {\n      int from = result[j];\n      int dest = result[j + stride];\n      if (from != dest) {\n        weight += W[from * num_elements + dest];\n\n        // Penalize: (1) use of redundant edges in a single tree\n        //           (2) repeated use of a GPU in a single tree at the same\n        //               level above the leaf level\n        if (links_used.find(from * num_elements + dest) != links_used.end() && penalty) {\n          weight -= 100;\n        }\n        links_used.insert(from * num_elements + dest);\n        links_used.insert(dest * num_elements + from);\n      }\n\n      nodes_used[from] = true;\n      if (i > 0 && nodes_used[dest] && penalty) {\n        weight -= 10;\n      }\n      nodes_used[dest] = true;\n    }\n  }\n\n  return weight;\n}\n\n/**\n * \\brief Given a spanning tree encoded as result, which was convenient for performing\n * backtracking, convert it topology_ and scan_ in the classic \"binary tree\n * stored in an array\" format. For binary trees scan_ is redundant, but this\n * additional data structure leaves future generalization to k-radix trees.\n *\n * Initial result: [3 3 0 4 1 2 5 6]\n * topology_:      [3 3 1 3 0 1 5 3 3 0 4 1 2 5 6]\n * scan_:          [0 1 3 7 15]\n *\n * topology_ is stored in the classic \"binary tree stored in an array\" format\n * e.g.    3\n *     3     1\n *   3   0   1   5\n * 3 3 0 4 1 2 5 6\n *\n * Returns false if invalid tree in result\n * Otherwise returns true\n */\ninline bool FormTopology(const std::vector<int>& result,\n                         std::vector<size_t>* topo_row,\n                         std::vector<size_t>* scan_row,\n                         int depth) {\n  for (int result_value : result)\n    if (result_value == -1)\n      return false;\n\n  scan_row->push_back(topo_row->size());\n  for (int i = depth; i > 0; --i) {\n    int stride = 1 << i;\n    for (unsigned j = 0; j < result.size(); j += stride) {\n      int from = result[j];\n      topo_row->push_back(from);\n    }\n    scan_row->push_back(topo_row->size());\n  }\n\n  // Insert at the end, result vector\n  topo_row->insert(topo_row->end(), result.begin(), result.end());\n  scan_row->push_back(topo_row->size());\n  return true;\n}\n\n/**\n * \\brief Recursive function that finds a spanning tree, which fulfills the following\n * conditions:\n *   -balanced\n *   -binary\n *   -maximum weight\n */\ntemplate <typename T>\ninline bool RecursiveBacktrack(const std::vector<T>& W,\n                               std::vector<int>* state,\n                               std::vector<int>* best_result,\n                               T* best_result_weight,\n                               int row,\n                               int num_elements,\n                               int depth,\n                               bool optimal) {\n  if (row == static_cast<int>(state->size())) {\n    std::vector<int> result = *state;\n    Postprocess(&result, num_elements, depth);\n    T weight = ComputeTreeWeight(W, result, num_elements, depth, true);\n\n    // Save this spanning tree if it is highest weight tree found sofar\n    if (weight > *best_result_weight) {\n      std::swap(*best_result_weight, weight);\n      *best_result = result;\n    }\n    return !optimal;\n  }\n\n  // If not last recursive level, try to find valid tree for next level\n  bool stop = false;\n  for (int j = 0; j < num_elements; ++j) {\n    (*state)[row] = j;\n    if (IsValid(W, state, num_elements, row + 1, depth))\n      stop = RecursiveBacktrack(\n          W, state, best_result, best_result_weight, row + 1, num_elements, depth, optimal);\n    (*state)[row] = -1;\n    if (stop)\n      return stop;\n  }\n  return stop;\n}\n\ntemplate <typename T>\ninline void IterativeBacktrack(const std::vector<T>& W,\n                               std::vector<int>* state,\n                               std::vector<int>* best_result,\n                               T* best_result_weight,\n                               int row,\n                               int num_elements,\n                               int depth,\n                               bool optimal) {\n  std::stack<int> state_stack;\n  row     = 1;\n  int pos = 0;\n  state_stack.push(pos);\n\n  while (true) {\n    // If there is no valid position, 2 cases:\n    // a) if stack is empty, break and stop search\n    // b) if stack is not empty, pop stack and set current position to next\n    //    position backtrack to previous row\n    while (!state_stack.empty() && pos >= num_elements) {\n      pos = state_stack.top();\n      pos++;\n      state_stack.pop();\n      (*state)[state_stack.size() + 1] = -1;\n      row--;\n    }\n    if (state_stack.empty())\n      break;\n\n    (*state)[row] = pos;\n    // If there is a valid position push the position to stack, set current\n    // position to 0 and move to next row\n    if (IsValid(W, *state, num_elements, row + 1, depth)) {\n      state_stack.push(pos);\n      pos = 0;\n      row++;\n    } else {\n      pos++;\n      (*state)[row] = -1;\n    }\n\n    // If stack has size N, a solution is found\n    // Pop stack, set current position to next position\n    // Backtrack to find next solution\n    if (row == static_cast<int>(state->size())) {\n      std::vector<int> result = *state;\n      Postprocess(&result, num_elements, depth);\n      T weight = ComputeTreeWeight(W, result, num_elements, depth, true);\n\n      // Save this spanning tree if it is highest weight tree found so far\n      if (weight > *best_result_weight) {\n        std::swap(*best_result_weight, weight);\n        *best_result = result;\n      }\n      if (!optimal)\n        break;\n\n      pos = state_stack.top();\n      pos++;\n      state_stack.pop();\n      (*state)[state_stack.size()] = -1;\n      row--;\n    }\n  }\n}\n\n/**\n * \\brief Apply penalty factor alpha to each link in link topology graph that is used\n * by the spanning tree\n */\ntemplate <typename T>\ninline void UpdateWeight(std::vector<T>* W,\n                         const std::vector<size_t>& topo_row,\n                         int num_elements,\n                         float alpha) {\n  for (unsigned i = 1; i < topo_row.size() - 1; i += 2) {\n    unsigned parent = topo_row[i];\n    unsigned child  = topo_row[i + 1];\n    if (!(parent >= num_elements * num_elements || child >= num_elements * num_elements) &&\n        (parent != child)) {\n      (*W)[parent * num_elements + child] *= alpha;\n      (*W)[child * num_elements + parent] *= alpha;\n    }\n  }\n}\n\n/**\n * \\brief Do brute-force backtracking approach if Kernighan-Lin fails to find a binary\n * tree of height Log P.\n *\n * Constraints:\n * 1) minimize depth (balance)\n * 2) maximize edge weight\n * 3) tree is binary\n */\ntemplate <typename T>\ninline bool BacktrackGenerateBinaryTree(std::vector<T>* W,\n                                        int num_elements,\n                                        int root,\n                                        std::vector<size_t>* topo_row,\n                                        std::vector<size_t>* scan_row) {\n  // Clear before starting\n  topo_row->clear();\n  scan_row->clear();\n\n  // Compute depth\n  // num_elements: depth\n  // 5: 3 8\n  // 6: 3 8\n  // 7: 3 8\n  // 8: 3 8\n  // 9: 4 16\n  int depth        = ComputeDepth(num_elements);\n  int depth_leaves = 1 << depth;\n\n  // State vector\n  // -1 means unplaced\n  std::vector<int> state(depth_leaves, -1);\n  std::vector<int> result(depth_leaves, -1);\n  T result_weight = std::numeric_limits<T>::lowest();\n\n  // Place root and try all combinations\n  state[0] = root;\n\n  // Seek optimal solution until depth <= 3 i.e. 8 GPUs\n  // For larger numbers of GPUs, settle for first tree found (non-optimal), but\n  // this saves a lot of runtime, because Backtrack is exponential time\n  if (depth <= 3) {\n    IterativeBacktrack(*W, &state, &result, &result_weight, 1, num_elements, depth, true);\n  } else {\n    IterativeBacktrack(*W, &state, &result, &result_weight, 1, num_elements, depth, false);\n  }\n  return FormTopology(result, topo_row, scan_row, depth);\n}\n\n/**\n * \\brief ComputeTreesFromRoot does the same thing as ComputeTrees, with the only\n * exception being it will do it from a fixed GPU as root\n */\ntemplate <typename T>\ninline void ComputeTreesFromRoot(std::vector<T>* W,\n                                 int num_elements,\n                                 int root,\n                                 float alpha,\n                                 bool backtrack,\n                                 std::vector<size_t>* topo,\n                                 std::vector<size_t>* scan) {\n  int num_partitions = 1;\n\n  // Initialize partition array to indicate which partition each element belongs\n  // to beginning with 0\n  std::vector<int> P(num_elements, 0);\n\n  // Initialize vector of pairs that will tell us edges between what 2 clusters\n  // we should be looking to build the tree from\n  std::vector<std::pair<int, int>> cluster_pairs;\n\n  // Initialize vector of roots that will tell us edges between\n  std::unordered_set<int> roots;\n  roots.insert(root);\n\n  // Will be used to obtain a seed for the random number engine\n  // RNG: Standard mersenne_twister_engine seeded with rd()\n  //     -use 0 for testing (TODO: remove this)\n  // std::random_device rd;\n  // std::mt19937 gen(rd());\n  std::mt19937 gen(1);\n\n  // Temporary variables for rewinding\n  std::vector<int> P_temp;\n  int num_partitions_temp;\n  std::unordered_set<int> roots_temp;\n  std::vector<size_t> topo_temp;\n  std::vector<size_t> scan_temp;\n\n  // Determine number of partition levels\n  // If first partition, determine root of maximal spanning tree\n  bool stop = false;\n  int reset = 1;\n  int level = 0;\n\n  while (!backtrack && (!stop || reset)) {\n    if (reset == 1) {\n      cluster_pairs.clear();\n      P_temp              = P;\n      num_partitions_temp = num_partitions;\n      roots_temp          = roots;\n      topo_temp           = *topo;\n      scan_temp           = *scan;\n    }\n\n    // Run Kernighan-Lin to generate partition\n    stop = KernighanLin(*W, &P_temp, &num_partitions_temp, &cluster_pairs, &gen);\n\n    // Use partitions found and a given root to find best inter-cluster edge for\n    // each pair of clusters, and returns them as roots of next cluster\n    // If reset is true, then rewind back to previous clustering\n    reset =\n        KLGenerateBinaryTree(*W, P_temp, &cluster_pairs, &roots_temp, &topo_temp, &scan_temp, &gen);\n\n    if (reset)\n      level++;\n    if (level > 10)\n      break;\n  }\n\n  bool success = true;\n  if (reset == 1) {\n    LOG(INFO) << \"No valid binary tree found from root \" << root << \", try backtracking\";\n    success = BacktrackGenerateBinaryTree(W, num_elements, root, topo, scan);\n  } else {\n    *topo = topo_temp;\n    *scan = scan_temp;\n    scan->push_back(topo->size());\n  }\n  if (success)\n    UpdateWeight(W, *topo, num_elements, alpha);\n  else\n    LOG(FATAL) << \"No valid binary tree found from root \" << root << \" using backtracking\";\n}\n\n/**\n * \\brief ComputeTrees computes balanced binary spanning trees of maximum edge weight\n * given a link topology graph stored in adjacency matrix format\n * \\param W is the link topology matrix\n * \\param num_elements is the number of GPUs\n * \\param alpha is the link usage penalty\n * \\param backtrack is whether or not we use backtracking to generate trees\n * \\param topo stores the trees generated\n * \\param scan stores the start of each level of each tree\n */\ntemplate <typename T>\ninline void ComputeTrees(const std::vector<T>& W,\n                         int num_elements,\n                         float alpha,\n                         bool backtrack,\n                         std::vector<std::vector<size_t>>* topo,\n                         std::vector<std::vector<size_t>>* scan) {\n  std::vector<T> W_copy = W;\n\n  topo->clear();\n  scan->clear();\n  for (int i = 0; i < num_elements; ++i) {\n    topo->push_back(std::vector<size_t>());\n    scan->push_back(std::vector<size_t>());\n    (*topo)[i].push_back(i);\n    (*scan)[i].push_back(0);\n    ComputeTreesFromRoot(&W_copy, num_elements, i, alpha, backtrack, &((*topo)[i]), &((*scan)[i]));\n  }\n\n  // Note: must sum up adj matrix to show link usage before we readjust topo\n  // from 0, 1, ..., n_gpus format to dev_id format, which will cause segfault\n  std::vector<int> adj(W.size(), 0);\n  for (int row = 0; row < num_elements; ++row) {\n    for (unsigned col = 1; col < (*topo)[0].size(); col += 2) {\n      int from = std::min((*topo)[row][col], (*topo)[row][col + 1]);\n      int dest = std::max((*topo)[row][col], (*topo)[row][col + 1]);\n      if (from != dest) {\n        adj.at(from * num_elements + dest) += 1;\n        adj.at(dest * num_elements + from) += 1;\n      }\n    }\n  }\n\n  std::vector<std::vector<size_t>> topo_temp(num_elements, std::vector<size_t>());\n\n  if (kLogTree) {\n    for (int i = 0; i < num_elements; ++i)\n      PrintTopo(\"Tree \" + std::to_string(i), (*topo)[i], (*scan)[i]);\n\n    PrintMatrix(\"W\", W, num_elements, num_elements);\n    PrintMatrix(\"Links\", adj, num_elements, num_elements);\n  }\n}\n}  // namespace kvstore\n}  // namespace mxnet\n#endif  // MXNET_KVSTORE_GPU_TOPOLOGY_H_\n"
  },
  {
    "path": "src/kvstore/gradient_compression-inl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file gradient_compression-inl.h\n * \\author Rahul Huilgol\n * \\brief Declares and defines functions used to quantize and dequantize data\n */\n#ifndef MXNET_KVSTORE_GRADIENT_COMPRESSION_INL_H_\n#define MXNET_KVSTORE_GRADIENT_COMPRESSION_INL_H_\n\n#include <vector>\n#include \"../operator/mxnet_op.h\"\n\nnamespace mxnet {\nnamespace kvstore {\n\n// these gpu functions are defined in gradient_compression.cu\nvoid Quantize1BitImpl(mshadow::Stream<mshadow::gpu>* s,\n                      const std::vector<mxnet::TBlob>& inputs,\n                      const float threshold);\nvoid Dequantize1BitImpl(mshadow::Stream<mshadow::gpu>* s,\n                        const std::vector<mxnet::TBlob>& inputs,\n                        const float threshold);\nvoid Quantize2BitImpl(mshadow::Stream<mshadow::gpu>* s,\n                      const std::vector<mxnet::TBlob>& inputs,\n                      const float threshold);\nvoid Dequantize2BitImpl(mshadow::Stream<mshadow::gpu>* s,\n                        const std::vector<mxnet::TBlob>& inputs,\n                        const float threshold);\n\nstruct quantize_1bit {\n  MSHADOW_XINLINE static void Map(int out_byte_id,\n                                  int original_size,\n                                  float* out,\n                                  float* grad,\n                                  float* residual,\n                                  const float threshold) {\n    // this byte contains the compressed representation of\n    // upto 8 values starting from (char*)out + out_byte_id\n    char* compr_byte = reinterpret_cast<char*>(out) + out_byte_id;\n\n    // init to 0\n    *compr_byte = 0;\n    // start and end are indices in original grad array\n    const int start = out_byte_id << 3;\n    const int end   = (start + 8 <= original_size) ? start + 8 : original_size;\n\n    // masks used to quantize data\n    const uint8_t bits[] = {0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01};\n    for (int i = start; i < end; ++i) {\n      // adds gradient to existing residual to get updated grad\n      residual[i] += grad[i];\n      if (residual[i] > threshold) {\n        // set data to 1\n        *compr_byte |= bits[(i & 7)];\n        // reduce residual by 1\n        residual[i] -= 1;\n      } else {\n        // do nothing on compr_byte because it is initialized to 0\n        // add residual by 1\n        // because current position will be dequantized to -1\n        residual[i] += 1;\n      }\n    }\n  }\n};\n\ntemplate <typename xpu>\nvoid Quantize1BitKernelLaunch(mshadow::Stream<xpu>* s,\n                              const std::vector<mxnet::TBlob>& inputs,\n                              const float threshold) {\n  mxnet::op::mxnet_op::Kernel<quantize_1bit, xpu>::Launch(\n      s,\n      inputs[2].Size() * 4,     // compressed array byte size\n      inputs[0].Size(),         // original size\n      inputs[2].dptr<float>(),  // compressed array\n      inputs[0].dptr<float>(),  // original array\n      inputs[1].dptr<float>(),  // residual array\n      threshold);               // threshold\n}\n\nstruct dequantize_1bit {\n  MSHADOW_XINLINE static void Map(int i, float* out, float* in, const float threshold) {\n    // get position of dequantized value to fill\n    float* outval = out + i;\n    // gets byte which holds quantized value for this position\n    char* ch_ptr = reinterpret_cast<char*>(in + (i >> 5));\n    ch_ptr += ((i & 31) >> 3);\n    // masks used to quantize data\n    const uint8_t bits[] = {0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01};\n    // col denotes which bit of a byte is set for this value\n    // col=0 implies the first bit, col=1 implies the second bit,...\n    const int col        = i & 7;\n    const uint8_t mask   = bits[col];\n    const uint8_t masked = *ch_ptr & mask;\n    if (masked == mask) {\n      *outval = +1;\n    } else {\n      // if current position of byte is 0\n      // dequantized it to -1\n      *outval = -1;\n    }\n  }\n};\n\ntemplate <typename xpu>\nvoid Dequantize1BitKernelLaunch(mshadow::Stream<xpu>* s,\n                                const std::vector<mxnet::TBlob>& inputs,\n                                const float threshold) {\n  mxnet::op::mxnet_op::Kernel<dequantize_1bit, xpu>::Launch(\n      s,\n      inputs[1].Size(),         // original size\n      inputs[1].dptr<float>(),  // out array\n      inputs[0].dptr<float>(),  // compressed array\n      threshold);               // threshold\n}\n\nstruct quantize_2bit {\n  MSHADOW_XINLINE static void Map(int out_byte_id,\n                                  int original_size,\n                                  float* out,\n                                  float* grad,\n                                  float* residual,\n                                  const float neg_threshold,\n                                  const float pos_threshold) {\n    // this block contains the compressed representation of\n    // upto 4 values starting from (char*)out + out_byte_id\n    char* compr_byte = reinterpret_cast<char*>(out) + out_byte_id;\n    // init to 0\n    *compr_byte = 0;\n    // start and end are indices in original grad array\n    const int start = out_byte_id << 2;\n    const int end   = (start + 4 <= original_size) ? start + 4 : original_size;\n\n    // masks to set bits when value meets pos_threshold\n    // 0xc0 is mask when value is to be represented by the first two bits in a char*\n    // 0xc0 means first two bits are set to 11\n    const uint8_t posbits[] = {0xc0, 0x30, 0x0c, 0x03};\n    // masks to set bits when value meets neg_threshold\n    const uint8_t negbits[] = {0x80, 0x20, 0x08, 0x02};\n    for (int i = start; i < end; i++) {\n      // adds gradient to existing residual to get updated grad\n      residual[i] += grad[i];\n      if (residual[i] >= pos_threshold) {\n        // set data to 11\n        *compr_byte |= posbits[(i & 3)];\n        // reduce residual by pos_threshold\n        residual[i] -= pos_threshold;\n      } else if (residual[i] <= neg_threshold) {\n        // set data to 10\n        *compr_byte |= negbits[(i & 3)];\n        residual[i] -= neg_threshold;\n      }\n    }\n  }\n};\n\ntemplate <typename xpu>\nvoid Quantize2BitKernelLaunch(mshadow::Stream<xpu>* s,\n                              const std::vector<mxnet::TBlob>& inputs,\n                              const float threshold) {\n  mxnet::op::mxnet_op::Kernel<quantize_2bit, xpu>::Launch(\n      s,\n      inputs[2].Size() * 4,     // compressed array byte size\n      inputs[0].Size(),         // original size\n      inputs[2].dptr<float>(),  // compressed array\n      inputs[0].dptr<float>(),  // original array\n      inputs[1].dptr<float>(),  // residual array\n      -1 * threshold,           // negative threshold\n      threshold);               // positive threshold\n}\n\nstruct dequantize_2bit {\n  MSHADOW_XINLINE static void Map(int i,\n                                  float* out,\n                                  float* in,\n                                  const float neg_threshold,\n                                  const float pos_threshold) {\n    // get position of dequantized value to fill\n    float* outval = out + i;\n    // gets byte which holds quantized value for this position\n    char* ch_ptr = reinterpret_cast<char*>(in + (i >> 4));\n    ch_ptr += ((i & 15) >> 2);\n    // masks used to quantize data\n    const uint8_t posbits[] = {0xc0, 0x30, 0x0c, 0x03};\n    const uint8_t negbits[] = {0x80, 0x20, 0x08, 0x02};\n    // col denotes which two bits of a byte are set for this value\n    // col=0 implies first two bits, col=3 implies last two bits,...\n    const int col         = i & 3;\n    const uint8_t mask    = posbits[col];\n    const uint8_t negmask = negbits[col];\n    const uint8_t masked  = *ch_ptr & mask;\n    if (masked == mask) {\n      *outval = pos_threshold;\n    } else if (masked == negmask) {\n      // use posbits for mask as posbits are both 1s\n      // then compare masked with negbits to see if only negbits were set\n      *outval = neg_threshold;\n    } else {\n      *outval = 0;\n    }\n  }\n};\n\ntemplate <typename xpu>\nvoid Dequantize2BitKernelLaunch(mshadow::Stream<xpu>* s,\n                                const std::vector<mxnet::TBlob>& inputs,\n                                const float threshold) {\n  mxnet::op::mxnet_op::Kernel<dequantize_2bit, xpu>::Launch(\n      s,\n      inputs[1].Size(),         // original size\n      inputs[1].dptr<float>(),  // out array\n      inputs[0].dptr<float>(),  // compressed array\n      -1 * threshold,           // negative threshold\n      threshold);               // positive threshold\n}\n\ninline void Quantize1BitImpl(mshadow::Stream<mshadow::cpu>* s,\n                             const std::vector<mxnet::TBlob>& inputs,\n                             const float threshold) {\n  Quantize1BitKernelLaunch(s, inputs, threshold);\n}\n\ninline void Dequantize1BitImpl(mshadow::Stream<mshadow::cpu>* s,\n                               const std::vector<mxnet::TBlob>& inputs,\n                               const float threshold) {\n  Dequantize1BitKernelLaunch(s, inputs, threshold);\n}\n\ninline void Quantize2BitImpl(mshadow::Stream<mshadow::cpu>* s,\n                             const std::vector<mxnet::TBlob>& inputs,\n                             const float threshold) {\n  Quantize2BitKernelLaunch(s, inputs, threshold);\n}\n\ninline void Dequantize2BitImpl(mshadow::Stream<mshadow::cpu>* s,\n                               const std::vector<mxnet::TBlob>& inputs,\n                               const float threshold) {\n  Dequantize2BitKernelLaunch(s, inputs, threshold);\n}\n}  // namespace kvstore\n}  // namespace mxnet\n\n#endif  // MXNET_KVSTORE_GRADIENT_COMPRESSION_INL_H_\n"
  },
  {
    "path": "src/kvstore/gradient_compression.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file gradient_compression.cc\n * \\brief Gradient compression for kvstore\n * \\author Rahul Huilgol\n */\n\n#include <vector>\n#include \"kvstore_local.h\"\n#include \"gradient_compression.h\"\n#include \"gradient_compression-inl.h\"\n\nnamespace mxnet {\nnamespace kvstore {\n\nDMLC_REGISTER_PARAMETER(GradientCompressionParam);\n\nGradientCompression::GradientCompression() {\n  type_ = CompressionType::kNone;\n}\n\nvoid GradientCompression::SetParams(\n    const std::vector<std::pair<std::string, std::string> >& kwargs) {\n  GradientCompressionParam params;\n  params.InitAllowUnknown(kwargs);\n  if (params.type == \"1bit\") {\n    SetOneBitCompression(params.threshold);\n  } else if (params.type == \"2bit\") {\n    CHECK_GT(params.threshold, 0) << \"threshold must be greater than 0 for two bit compression\";\n    SetTwoBitCompression(params.threshold);\n  } else {\n    LOG(FATAL) << \"Unknown type for gradient compression \" << params.type;\n  }\n}\n\nCompressionType GradientCompression::get_type() {\n  return type_;\n}\n\nstd::string GradientCompression::get_type_str() {\n  return std::to_string(static_cast<int>(type_));\n}\n\nvoid GradientCompression::SetOneBitCompression(const float threshold) {\n  type_      = CompressionType::kOneBit;\n  threshold_ = threshold;\n}\n\nvoid GradientCompression::SetTwoBitCompression(const float threshold) {\n  type_      = CompressionType::kTwoBit;\n  threshold_ = threshold;\n}\n\nstd::string GradientCompression::EncodeParams() {\n  using namespace std;  // to reduce length of next line\n  string rval = get_type_str();\n  if (type_ != CompressionType::kNone) {\n    rval += \",\" + to_string(threshold_);\n  }\n  return rval;\n}\n\nvoid GradientCompression::DecodeParams(const std::string& s) {\n  std::vector<std::string> elems;\n  mxnet::kvstore::split(s, ',', std::back_inserter(elems));\n  type_ = static_cast<CompressionType>(stoi(elems[0]));\n  if (elems.size() > 1) {\n    if (!elems[1].empty()) {\n      threshold_ = stof(elems[1]);\n    }\n  }\n}\n\nint GradientCompression::GetCompressionFactor() {\n  if (type_ == CompressionType::kOneBit) {\n    return 32;\n  } else if (type_ == CompressionType::kTwoBit) {\n    return 16;\n  } else {\n    LOG(FATAL) << \"Unsupported compression type: \" << get_type_str();\n    return 0;\n  }\n}\n\nint64_t GradientCompression::GetCompressedSize(const int64_t original_size) {\n  const int bits = GetCompressionFactor();\n  return ((original_size % bits == 0) ? original_size / bits : original_size / bits + 1);\n}\n\nvoid GradientCompression::Quantize(const mxnet::NDArray& from,\n                                   mxnet::NDArray* to,\n                                   mxnet::NDArray* residual,\n                                   const int priority) {\n  CHECK(shape_is_known(from.shape())) << \"source operand has undefined shape\";\n  CHECK(shape_is_known(to->shape())) << \"destination operand has undefined shape\";\n  CHECK(shape_is_known(residual->shape())) << \"residual operand has undefined shape\";\n  const int a           = from.ctx().dev_mask();\n  const int b           = to->ctx().dev_mask();\n  const float threshold = threshold_;\n  if (a == mshadow::cpu::kDevMask && b == mshadow::cpu::kDevMask) {\n    if (type_ == CompressionType::kOneBit) {\n      mxnet::Engine::Get()->PushSync(\n          [from, to, residual, threshold](mxnet::RunContext ctx) {\n            std::vector<mxnet::TBlob> inputs = {from.data(), residual->data(), to->data()};\n            Quantize1BitImpl(ctx.get_stream<mshadow::cpu>(), inputs, threshold);\n          },\n          from.ctx(),\n          {from.var()},\n          {to->var(), residual->var()},\n          mxnet::FnProperty::kNormal,\n          priority,\n          \"QuantizeCPU\");\n    } else if (type_ == CompressionType::kTwoBit) {\n      mxnet::Engine::Get()->PushSync(\n          [from, to, residual, threshold](mxnet::RunContext ctx) {\n            std::vector<mxnet::TBlob> inputs = {from.data(), residual->data(), to->data()};\n            Quantize2BitImpl(ctx.get_stream<mshadow::cpu>(), inputs, threshold);\n          },\n          from.ctx(),\n          {from.var()},\n          {to->var(), residual->var()},\n          mxnet::FnProperty::kNormal,\n          priority,\n          \"QuantizeCPU\");\n    } else {\n      LOG(FATAL) << \"Unsupported quantization of type \" << get_type_str();\n    }\n  } else {\n    if (a == mshadow::gpu::kDevMask && b == mshadow::gpu::kDevMask) {\n#if MXNET_USE_CUDA\n      if (type_ == CompressionType::kOneBit) {\n        mxnet::Engine::Get()->PushSync(\n            [from, to, residual, threshold](mxnet::RunContext ctx) {\n              std::vector<mxnet::TBlob> inputs = {from.data(), residual->data(), to->data()};\n              Quantize1BitImpl(ctx.get_stream<mshadow::gpu>(), inputs, threshold);\n            },\n            from.ctx(),\n            {from.var()},\n            {to->var(), residual->var()},\n            mxnet::FnProperty::kNormal,\n            priority,\n            \"QuantizeGPU\");\n      } else if (type_ == CompressionType::kTwoBit) {\n        mxnet::Engine::Get()->PushSync(\n            [from, to, residual, threshold](mxnet::RunContext ctx) {\n              std::vector<mxnet::TBlob> inputs = {from.data(), residual->data(), to->data()};\n              Quantize2BitImpl(ctx.get_stream<mshadow::gpu>(), inputs, threshold);\n            },\n            from.ctx(),\n            {from.var()},\n            {to->var(), residual->var()},\n            mxnet::FnProperty::kNormal,\n            priority,\n            \"QuantizeGPU\");\n      } else {\n        LOG(FATAL) << \"Unsupported quantization of type \" << get_type_str();\n      }\n#else\n      LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR;\n#endif\n    } else {\n      LOG(FATAL) << \"Unknown device mask, from device mask \" << a << \" to device mask \" << b;\n    }\n  }\n}\n\nvoid GradientCompression::Dequantize(const mxnet::NDArray& from,\n                                     mxnet::NDArray* to,\n                                     const int priority) {\n  CHECK(shape_is_known(from.shape())) << \"source operand has undefined shape\";\n  CHECK(shape_is_known(to->shape())) << \"destination operand has undefined shape\";\n  const int a           = from.ctx().dev_mask();\n  const int b           = to->ctx().dev_mask();\n  const float threshold = threshold_;\n  if (a == mshadow::cpu::kDevMask && b == mshadow::cpu::kDevMask) {\n    if (type_ == CompressionType::kOneBit) {\n      mxnet::Engine::Get()->PushSync(\n          [from, to, threshold](mxnet::RunContext ctx) {\n            std::vector<mxnet::TBlob> inputs = {from.data(), to->data()};\n            Dequantize1BitImpl(ctx.get_stream<mshadow::cpu>(), inputs, threshold);\n          },\n          from.ctx(),\n          {from.var()},\n          {to->var()},\n          mxnet::FnProperty::kNormal,\n          priority,\n          \"DequantizeCPU\");\n    } else if (type_ == CompressionType::kTwoBit) {\n      mxnet::Engine::Get()->PushSync(\n          [from, to, threshold](mxnet::RunContext ctx) {\n            std::vector<mxnet::TBlob> inputs = {from.data(), to->data()};\n            Dequantize2BitImpl(ctx.get_stream<mshadow::cpu>(), inputs, threshold);\n          },\n          from.ctx(),\n          {from.var()},\n          {to->var()},\n          mxnet::FnProperty::kNormal,\n          priority,\n          \"DequantizeCPU\");\n    } else {\n      LOG(FATAL) << \"Unsupported dequantization of type \" << get_type_str();\n    }\n  } else {\n    if (a == mshadow::gpu::kDevMask && b == mshadow::gpu::kDevMask) {\n#if MXNET_USE_CUDA\n      if (type_ == CompressionType::kOneBit) {\n        mxnet::Engine::Get()->PushSync(\n            [from, to, threshold](mxnet::RunContext ctx) {\n              std::vector<mxnet::TBlob> inputs = {from.data(), to->data()};\n              Dequantize1BitImpl(ctx.get_stream<mshadow::gpu>(), inputs, threshold);\n              // Wait GPU kernel to complete\n              ctx.get_stream<mshadow::gpu>()->Wait();\n            },\n            from.ctx(),\n            {from.var()},\n            {to->var()},\n            mxnet::FnProperty::kNormal,\n            priority,\n            \"DequantizeGPU\");\n      } else if (type_ == CompressionType::kTwoBit) {\n        mxnet::Engine::Get()->PushSync(\n            [from, to, threshold](mxnet::RunContext ctx) {\n              std::vector<mxnet::TBlob> inputs = {from.data(), to->data()};\n              Dequantize2BitImpl(ctx.get_stream<mshadow::gpu>(), inputs, threshold);\n              // Wait GPU kernel to completes\n              ctx.get_stream<mshadow::gpu>()->Wait();\n            },\n            from.ctx(),\n            {from.var()},\n            {to->var()},\n            mxnet::FnProperty::kNormal,\n            priority,\n            \"DequantizeGPU\");\n      } else {\n        LOG(FATAL) << \"Unsupported dequantization of type \" << get_type_str();\n      }\n#else\n      LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR;\n#endif\n    } else {\n      LOG(FATAL) << \"Unknown device mask, from device mask \" << a << \" to device mask \" << b;\n    }\n  }\n}\n}  // namespace kvstore\n}  // namespace mxnet\n"
  },
  {
    "path": "src/kvstore/gradient_compression.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file gradient_compression.cu\n * \\author Rahul Huilgol\n * \\brief Implementation for gpu version of code\n */\n\n#include \"gradient_compression-inl.h\"\n\nnamespace mxnet {\nnamespace kvstore {\nvoid Quantize1BitImpl(mshadow::Stream<gpu>* s,\n                      const std::vector<TBlob>& inputs,\n                      const float threshold) {\n  Quantize1BitKernelLaunch(s, inputs, threshold);\n}\n\nvoid Dequantize1BitImpl(mshadow::Stream<gpu>* s,\n                        const std::vector<TBlob>& inputs,\n                        const float threshold) {\n  Dequantize1BitKernelLaunch(s, inputs, threshold);\n}\n\nvoid Quantize2BitImpl(mshadow::Stream<gpu>* s,\n                      const std::vector<TBlob>& inputs,\n                      const float threshold) {\n  Quantize2BitKernelLaunch(s, inputs, threshold);\n}\n\nvoid Dequantize2BitImpl(mshadow::Stream<gpu>* s,\n                        const std::vector<TBlob>& inputs,\n                        const float threshold) {\n  Dequantize2BitKernelLaunch(s, inputs, threshold);\n}\n}  // namespace kvstore\n}  // namespace mxnet\n"
  },
  {
    "path": "src/kvstore/gradient_compression.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file gradient_compression.h\n * \\brief Gradient compression for kvstore\n * \\author Rahul Huilgol\n */\n\n#ifndef MXNET_KVSTORE_GRADIENT_COMPRESSION_H_\n#define MXNET_KVSTORE_GRADIENT_COMPRESSION_H_\n#include <dmlc/parameter.h>\n#include <string>\n#include <utility>\n#include <vector>\n#include \"mxnet/ndarray.h\"\n\nnamespace mxnet {\nnamespace kvstore {\n\nenum class CompressionType { kNone, kOneBit, kTwoBit };\n\nstruct GradientCompressionParam : public dmlc::Parameter<GradientCompressionParam> {\n  std::string type;\n  float threshold;\n  DMLC_DECLARE_PARAMETER(GradientCompressionParam) {\n    DMLC_DECLARE_FIELD(type).describe(\n        \"Type of gradient compression to use, like `2bit` for example\");\n    DMLC_DECLARE_FIELD(threshold).set_default(0.5).describe(\n        \"Threshold to use for 2bit gradient compression\");\n  }\n};\n\nclass GradientCompression {\n public:\n  GradientCompression();\n\n  virtual ~GradientCompression() {}\n\n  /*!\n   * \\brief sets parameters for gradient compression\n   * \\param kwargs a vector of pair of strings. A pair represents key and value\n   * of the parameter. Will be parsed by GradientCompressionParam\n   */\n  void SetParams(const std::vector<std::pair<std::string, std::string> >& kwargs);\n\n  /*!\n   * \\brief returns type of compression if any\n   */\n  CompressionType get_type();\n\n  /*!\n   * \\brief returns as string the enum value of compression type\n   */\n  std::string get_type_str();\n\n  /*!\n   * \\biref sets one bit gradient compression\n   * \\param threshold float value used for thresholding gradients\n   */\n  void SetOneBitCompression(const float threshold);\n\n  /*!\n   * \\brief sets two bit gradient compression\n   * \\param threshold float value used for thresholding gradients\n   */\n  void SetTwoBitCompression(const float threshold);\n\n  /*!\n   * \\brief encodes parameters of gc into a string\n   */\n  std::string EncodeParams();\n\n  /*!\n   * \\brief decodes parameters of gc from a string and assigns them to member variables\n   */\n  void DecodeParams(const std::string& s);\n\n  /*!\n   * \\brief returns compression factor, which is the factor by which size of gradient\n   * reduces when using a particular type of compression\n   */\n  int GetCompressionFactor();\n\n  /*!\n   * \\brief returns the size of compressed gradients given an original sized gradient array\n   */\n  int64_t GetCompressedSize(const int64_t original_size);\n\n  /*!\n   * \\brief Issues quantize operation to be scheduled by the engine\n   * Compresses `from` into `to` and accumulates the quantization error\n   * into 'residual', using the quantization of type `type_`\n   * \\param from the ndarray containing original data to be quantized\n   * \\param to the target ndarray which contains quantized data\n   * \\param residual the ndarray which accumulates quantization error\n   * \\param priority Priority of the action.\n   */\n  void Quantize(const mxnet::NDArray& from,\n                mxnet::NDArray* to,\n                mxnet::NDArray* residual,\n                const int priority);\n\n  /*!\n   * \\brief Issues dequantize operation to be scheduled by the engine\n   * Decompresses `from` into `to` using current parameters of `type` and `threshold`\n   * \\param from the ndarray containing quantized data\n   * \\param to the target ndarray which contains final dequantized data\n   * \\param priority Priority of the action.\n   */\n  void Dequantize(const mxnet::NDArray& from, mxnet::NDArray* to, const int priority);\n\n private:\n  /*!\n   * \\brief denotes the type of gradient compression which has been set\n   */\n  CompressionType type_;\n\n  /*!\n   * \\brief denotes threshold used for quantization and dequantization\n   * Must be a positive value. All positive gradients will be thresholded to `threshold_` and\n   * all negative gradients will be thresholded to -1*`threshold_`\n   */\n  float threshold_ = 0;\n};\n}  // namespace kvstore\n}  // namespace mxnet\n#endif  // MXNET_KVSTORE_GRADIENT_COMPRESSION_H_\n"
  },
  {
    "path": "src/kvstore/kvstore.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file kvstore.cc\n * \\brief implement kv_store\n */\n#include <mxnet/kvstore.h>\n#include <dmlc/logging.h>\n#include \"./kvstore_local.h\"\n\n#if MXNET_USE_DIST_KVSTORE\n#include \"./kvstore_dist.h\"\n#include \"./p3store_dist.h\"\nstd::atomic<int> mxnet::kvstore::KVStoreDist::customer_id_{0};\n#endif  // MXNET_USE_DIST_KVSTORE\n#if MXNET_USE_NCCL\n#include \"./kvstore_nccl.h\"\n#endif  // MXNET_USE_NCCL\n\n#include <cstdlib>\n\nnamespace mxnet {\n\nKVStore* KVStore::Create(const char* type_name) {\n  std::string tname = type_name;\n  std::transform(tname.begin(), tname.end(), tname.begin(), ::tolower);\n  KVStore* kv          = nullptr;\n  bool use_device_comm = false;\n  auto has             = [tname](const std::string& pattern) {\n    return tname.find(pattern) != std::string::npos;\n  };\n  if (has(\"device\")) {\n    use_device_comm = true;\n  }\n\n  if (has(\"dist\")) {\n#if MXNET_USE_DIST_KVSTORE\n    auto ps_type = dmlc::GetEnv(\"DMLC_PS_VAN_TYPE\", std::string(\"none\"));\n    if (ps_type == \"p3\") {\n      CHECK(!has(\"async\")) << \"Asynchronous update is not supported in P3StoreDist\";\n      kv = new kvstore::P3StoreDist(use_device_comm);\n    } else {\n      kv = new kvstore::KVStoreDist(use_device_comm);\n    }\n    if (!has(\"_async\") && kv->IsWorkerNode() && kv->get_rank() == 0) {\n      // configure the server to be the sync mode\n      kv->SendCommandToServers(static_cast<int>(kvstore::CommandType::kSyncMode), \"\");\n    }\n#else\n    LOG(FATAL) << \"compile with USE_DIST_KVSTORE=1 to use \" << tname;\n    return nullptr;\n#endif  // MXNET_USE_DIST_KVSTORE\n  } else {\n    if (has(\"nccl\")) {\n#if MXNET_USE_NCCL\n      kv = new kvstore::KVStoreNCCL();\n#else\n      LOG(FATAL) << \"compile with USE_NCCL=1 to use \" << tname;\n      return nullptr;\n#endif\n    } else {\n      kv = new kvstore::KVStoreLocal(use_device_comm);\n    }\n  }\n  kv->type_ = tname;\n  return kv;\n}\n\n}  // namespace mxnet\n"
  },
  {
    "path": "src/kvstore/kvstore_dist.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/**\n * @file   kvstore_dist.h\n * @brief  distributed implementation based on ps-lite\n */\n#ifndef MXNET_KVSTORE_KVSTORE_DIST_H_\n#define MXNET_KVSTORE_KVSTORE_DIST_H_\n#include <string>\n#include <vector>\n#include <algorithm>\n#include <utility>\n#include \"./kvstore_local.h\"\n#include \"mxnet/engine.h\"\n#include \"ps/ps.h\"\n#include \"./kvstore_dist_server.h\"\nnamespace mxnet {\nnamespace kvstore {\n\n/**\n * \\brief distributed kvstore\n *\n * it's the server node's job to control the data consistency among all\n * workers. see details on \\ref ServerHandle::Start\n */\nclass KVStoreDist : public KVStoreLocal {\n public:\n  explicit KVStoreDist(bool use_device_comm)\n      : KVStoreLocal(use_device_comm), ps_worker_(nullptr), server_(nullptr) {\n    if (IsWorkerNode()) {\n      int new_customer_id = GetNewCustomerId();\n      ps_worker_          = new ps::KVWorker<char>(0, new_customer_id);\n      ps::StartAsync(new_customer_id, \"mxnet\\0\");\n      if (!ps::Postoffice::Get()->is_recovery()) {\n        ps::Postoffice::Get()->Barrier(new_customer_id,\n                                       ps::kWorkerGroup + ps::kServerGroup + ps::kScheduler);\n      }\n    }\n    bigarray_bound_ = dmlc::GetEnv(\"MXNET_KVSTORE_BIGARRAY_BOUND\", 1000 * 1000);\n    log_verbose_    = dmlc::GetEnv(\"MXNET_KVSTORE_DIST_ROW_SPARSE_VERBOSE\", false);\n  }\n\n  virtual ~KVStoreDist() {\n    Engine::Get()->WaitForAll();\n    customer_id_ = 0;\n    if (IsWorkerNode()) {\n      if (barrier_before_exit_) {\n        Barrier();\n        if (get_rank() == 0 && ps_worker_->get_customer()->customer_id() == 0) {\n          // stop the executor at servers\n          SendCommandToServers(static_cast<int>(CommandType::kStopServer), \"\");\n        }\n      }\n      ps::Finalize(ps_worker_->get_customer()->customer_id(), barrier_before_exit_);\n      delete ps_worker_;\n    }\n  }\n\n  void set_updater(const Updater& updater) override {\n    CHECK(updater) << \"invalid updater\";\n    if (IsServerNode()) {\n      CHECK_NOTNULL(server_)->set_updater(updater);\n    } else {\n      updater_ = updater;\n    }\n  }\n\n  void SetGradientCompression(\n      const std::vector<std::pair<std::string, std::string>>& kwargs) override {\n    KVStoreLocal::SetGradientCompression(kwargs);\n    if (get_rank() == 0) {\n      SendCommandToServers(static_cast<int>(CommandType::kSetGradientCompression),\n                           gradient_compression_->EncodeParams());\n    }\n  }\n\n  void SetServerProfilerCommand(const KVStoreServerProfilerCommand type,\n                                const std::string& params) override {\n    if (get_rank() == 0) {\n      SendCommandToServers(static_cast<int>(CommandType::kSetProfilerParams),\n                           params + std::to_string(static_cast<int>(type)));\n    }\n  }\n\n  void Barrier() override {\n    ps::Postoffice::Get()->Barrier(ps_worker_->get_customer()->customer_id(), ps::kWorkerGroup);\n  }\n\n  void SendCommandToServers(int cmd_id, const std::string& cmd_body) override {\n    CHECK_NOTNULL(ps_worker_);\n    ps_worker_->Wait(ps_worker_->Request(cmd_id, cmd_body, ps::kServerGroup));\n  }\n\n  int get_group_size() const override {\n    return ps::NumWorkers();\n  }\n\n  int get_rank() const override {\n    return ps::MyRank();\n  }\n\n  int get_num_dead_node(int node_id, int timeout) const override {\n    int number              = 0;\n    auto dead_nodes         = ps::Postoffice::Get()->GetDeadNodes(timeout);\n    const auto& watch_nodes = ps::Postoffice::Get()->GetNodeIDs(node_id);\n    std::unordered_set<int> watch_set(watch_nodes.begin(), watch_nodes.end());\n    for (int r : dead_nodes) {\n      if (watch_set.find(r) != watch_set.end())\n        number++;\n    }\n    return number;\n  }\n\n  void RunServer(const Controller& controller) override {\n    CHECK(!IsWorkerNode());\n    if (IsServerNode()) {\n      server_ = new KVStoreDistServer();\n      server_->set_controller(controller);\n    }\n\n    ps::StartAsync(0, \"mxnet_server\\0\");\n    if (!ps::Postoffice::Get()->is_recovery()) {\n      ps::Postoffice::Get()->Barrier(0, ps::kWorkerGroup + ps::kServerGroup + ps::kScheduler);\n    }\n    if (server_)\n      server_->Run();\n    ps::Finalize(0, true);\n    delete server_;\n    server_ = nullptr;\n  }\n\n protected:\n  /**\n   * \\brief serialize access to ps_kv_ or push_ps_kv_/pull_ps_kv_ while encoding keys\n   */\n  std::mutex mu_;\n\n  /**\n   * \\brief for worker to push and pull data\n   */\n  ps::KVWorker<char>* ps_worker_;\n\n  /**\n   * \\brief struct for ps keys and lens\n   */\n  struct PSKV {\n    ps::SArray<ps::Key> keys;  // n keys\n    ps::SArray<int> lens;      // the length of the i-th value\n    int size;\n  };\n\n  struct ComprPSKV {\n    PSKV push;\n    PSKV pull;\n  };\n\n  /**\n   * \\brief cache all key partitions\n   *\n   * `ps_kv_` is used for pushes and pulls without gradient compression\n   * `compr_ps_kv_` is used for gradient compression. It contains different\n   * pskv for push and pull because sizes would be different in both cases.\n   * Note: `ps_kv_[k]` for some key k may not be the same as `compr_ps_kv_[k].pull`\n   * This is because sharding may cause slightly different divisions when size is\n   * not perfectly divisible.\n   */\n  std::unordered_map<int, PSKV> ps_kv_;\n  std::unordered_map<int, ComprPSKV> compr_ps_kv_;\n\n private:\n  static std::atomic<int> customer_id_;\n\n  static int GetNewCustomerId() {\n    return customer_id_++;\n  }\n\n  void InitImpl(const std::vector<int>& keys, const std::vector<NDArray>& values) override {\n    CheckUnique(keys);\n    for (size_t i = 0; i < keys.size(); ++i) {\n      InitKV(keys[i], values[i]);\n    }\n    if (get_rank() == 0 && this->ps_worker_->get_customer()->customer_id() == 0) {\n      Push_(keys, values, 0, false);\n      // wait until the push is finished\n      for (const int key : keys) {\n        comm_buf_[key].WaitToWrite();\n        compr_buf_[key].WaitToWrite();\n      }\n    } else {\n      // do nothing\n    }\n    if (!ps::Postoffice::Get()->is_recovery()) {\n      Barrier();\n    }\n  }\n\n  virtual inline void InitKV(const int key, const NDArray& value) {\n    comm_->Init(key, value.storage_type(), value.shape(), value.dtype());\n  }\n\n  void PushPullImpl(const std::vector<int>& vkeys,\n                    const std::vector<int>& okeys,\n                    const std::vector<NDArray>& values,\n                    const std::vector<NDArray*>& outputs,\n                    int priority) override {\n    std::vector<int> uniq_vkeys;\n    std::vector<int> uniq_okeys;\n    std::vector<std::vector<NDArray>> grouped_vals;\n    std::vector<std::vector<NDArray*>> grouped_outs;\n\n    GroupKVPairsPush(vkeys, values, &uniq_vkeys, &grouped_vals, false);\n    GroupKVPairsPull(okeys, outputs, &uniq_okeys, &grouped_outs, true);\n    CHECK_EQ(uniq_vkeys.size(), uniq_okeys.size()) << \"List of push and pull keys are different\";\n\n    for (size_t i = 0; i < uniq_vkeys.size(); ++i) {\n      CHECK_EQ(uniq_vkeys[i], uniq_okeys[i]) << \"Mismatch in push and pull key\";\n      int key          = uniq_vkeys[i];\n      const auto& vals = grouped_vals[i];\n      const auto& outs = grouped_outs[i];\n\n      NDArray merged = comm_->Reduce(key, vals, priority);\n\n      const auto push_stype = merged.storage_type();\n      const auto pull_stype = outs[0]->storage_type();\n      CHECK_EQ(push_stype, kDefaultStorage) << \"Expected push_stype of value to be kDefaultStorage\";\n      CHECK_EQ(pull_stype, kDefaultStorage) << \"Expected pull_stype of value to be kDefaultStorage\";\n\n      const int push_dtype = merged.dtype();\n      const int pull_dtype = outs[0]->dtype();\n      CHECK_EQ(push_dtype, pull_dtype) << \"Output buffer dtype is different\";\n\n      auto& comm_buf = comm_buf_[key];\n      if (merged.ctx().dev_mask() == cpu::kDevMask) {\n        comm_buf = merged;  // avoid memory copy\n      } else {\n        if (comm_buf.is_none()) {\n          comm_buf = NDArray(outs[0]->shape(), pinned_ctx_, true, pull_dtype);\n        }\n        CopyFromTo(merged, &comm_buf);\n      }\n\n      CHECK(gradient_compression_->get_type() == CompressionType::kNone)\n          << \"Compression not supported with PushPull\";\n      PushPullDefault(key, comm_buf, priority);\n      comm_->Broadcast(key, comm_buf, outs, priority);\n    }\n  }\n\n  void PushImpl(const std::vector<int>& keys,\n                const std::vector<NDArray>& values,\n                int priority) override {\n    Push_(keys, values, priority, true);\n  }\n\n  void PullImpl(const std::vector<int>& keys,\n                const std::vector<NDArray*>& values,\n                int priority,\n                bool ignore_sparse) override {\n    CHECK(ignore_sparse) << \"dist kvstore pull doesn't support ignore_sparse=False\";\n    std::vector<int> uniq_keys;\n    std::vector<std::vector<NDArray*>> grouped_vals;\n    GroupKVPairsPull(keys, values, &uniq_keys, &grouped_vals, true);\n\n    for (size_t i = 0; i < uniq_keys.size(); ++i) {\n      int key = uniq_keys[i];\n      // use the same array for merging to guarantee that pull always happens\n      // after the previous push on this key\n      auto& recv_buf          = comm_buf_[key];\n      const auto storage_type = grouped_vals[i][0]->storage_type();\n      CHECK_EQ(storage_type, kDefaultStorage) << \"Expected stype of value to be kDefaultStorage\";\n      if (recv_buf.is_none()) {\n        // it may happen for the first time a no-rank-0 worker pull the weight.\n        recv_buf =\n            NDArray(grouped_vals[i][0]->shape(), pinned_ctx_, true, grouped_vals[i][0]->dtype());\n      }\n      PullDefault(key, recv_buf, priority);\n\n      comm_->Broadcast(key, recv_buf, grouped_vals[i], priority);\n    }\n  }\n\n  void PullRowSparseImpl(const std::vector<int>& keys,\n                         const std::vector<std::pair<NDArray*, NDArray>>& val_rowids,\n                         int priority = 0) override {\n    std::vector<int> uniq_keys;\n    std::vector<std::vector<std::pair<NDArray*, NDArray>>> grouped_val_rowids;\n    GroupKVPairsPullRsp(keys, val_rowids, &uniq_keys, &grouped_val_rowids, false);\n\n    for (size_t i = 0; i < uniq_keys.size(); ++i) {\n      int key = uniq_keys[i];\n      // use the same array for merging to guarantee that pull always happens\n      // after the previous push on this key\n      auto& recv_buf          = comm_buf_[key];\n      auto& grouped_val_rowid = grouped_val_rowids[i];\n      const auto storage_type = grouped_val_rowid[0].first->storage_type();\n      CHECK_EQ(storage_type, kRowSparseStorage)\n          << \"expected kRowSparseStorage, but got \" << storage_type;\n      if (recv_buf.is_none()) {\n        // it may happen for the first time a no-rank-0 worker pull the weight.\n        recv_buf = NDArray(storage_type,\n                           grouped_val_rowid[0].first->shape(),\n                           pinned_ctx_,\n                           true,\n                           grouped_val_rowid[0].first->dtype());\n      }\n      auto& target_val_rowids = grouped_val_rowids[i];\n      const size_t num_vals   = target_val_rowids.size();\n      for (size_t i = 0; i < num_vals; i++) {\n        auto& row_id                = target_val_rowids[i].second;\n        target_val_rowids[i].second = Unique(row_id, pinned_ctx_, 0);\n      }\n      CHECK_EQ(num_vals, 1) << \"RowSparsePull with multiple values is not supported yet\";\n      NDArray& indices = target_val_rowids[0].second;\n      PullRowSparse_(key, recv_buf, indices, priority);\n      // The recv_buf contains values pulled from remote server with unique indices.\n      // Directly broadcast w/o rowids if num_vals == 1\n      auto get_val = [](const std::pair<NDArray*, NDArray>& p) { return p.first; };\n      std::vector<NDArray*> grouped_val(grouped_val_rowid.size());\n      std::transform(\n          grouped_val_rowid.begin(), grouped_val_rowid.end(), grouped_val.begin(), get_val);\n      comm_->Broadcast(key, recv_buf, grouped_val, priority);\n    }\n  }\n\n  void Push_(const std::vector<int>& keys,\n             const std::vector<NDArray>& values,\n             int priority,\n             bool do_merge) {\n    // first aggregate the values over keys\n    std::vector<int> uniq_keys;\n    std::vector<std::vector<NDArray>> grouped_vals;\n    GroupKVPairsPush(keys, values, &uniq_keys, &grouped_vals, false);\n\n    for (size_t i = 0; i < uniq_keys.size(); ++i) {\n      // merge over devices\n      int key          = uniq_keys[i];\n      const auto& vals = grouped_vals[i];\n      NDArray merged   = do_merge ? comm_->Reduce(key, vals, priority) : vals[0];\n\n      const auto storage_type = merged.storage_type();\n      auto& comm_buf          = comm_buf_[key];\n      if (merged.ctx().dev_mask() == cpu::kDevMask) {\n        // Start of a push doesn't guarantee that the previous pushes are completed.\n        // This shouldn't affect training of networks though because training involves\n        // a sequence of push, pull, then push. This imposes ordering that the\n        // second push happens after the first pull, and the pull happens after first push.\n        comm_buf = merged;  // avoid memory copy\n      } else {\n        if (comm_buf.is_none()) {\n          if (storage_type == kDefaultStorage) {\n            comm_buf = NDArray(merged.shape(), pinned_ctx_, true, merged.dtype());\n          } else {\n            comm_buf = NDArray(storage_type, merged.shape(), pinned_ctx_, true, merged.dtype());\n          }\n        }\n        CopyFromTo(merged, &comm_buf);\n      }\n      const int dtype     = merged.dtype();\n      const int num_bytes = mshadow::mshadow_sizeof(dtype);\n      // push to servers\n      if (storage_type == kDefaultStorage) {\n        if (gradient_compression_->get_type() == CompressionType::kNone) {\n          PSKV& pskv = EncodeDefaultKey(key, comm_buf.shape().Size(), num_bytes);\n          PushDefault(key, comm_buf, pskv, priority);\n        } else {\n          CHECK_EQ(dtype, mshadow::kFloat32) << \"Gradient compression is only supported for \"\n                                             << \"float32 type of parameters\";\n          // Note: gradient compression uses `do_merge` as proxy to\n          // detect whether the push is initialization of a key or not.\n          // is_active is false when push is initialization of key\n          bool is_active = do_merge;\n          PSKV& pskv     = EncodeCompressedKey(key, comm_buf.shape().Size(), is_active, num_bytes);\n          // Returns push_pskv if active, else pull_pskv\n          // we want inactive gc to send uncompressed gradients,\n          // but sharded in the same way as later pushes would when gc becomes active\n          if (is_active) {\n            PushCompressed(key, comm_buf, pskv, priority);\n          } else {\n            PushDefault(key, comm_buf, pskv, priority);\n          }\n        }\n      } else if (storage_type == kRowSparseStorage) {\n        CHECK(gradient_compression_->get_type() == CompressionType::kNone)\n            << \"Gradient compression for row sparse storage type is not supported\";\n        PushRowSparse(key, comm_buf, priority);\n      } else {\n        LOG(FATAL) << \"unknown storage type\";\n      }\n    }\n  }\n\n  virtual void PushCompressed(int key, const NDArray& comm_buf, const PSKV& pskv, int priority) {\n    auto& small_buf            = compr_buf_[key];\n    auto& res_buf              = residual_[key];\n    const size_t original_size = comm_buf.shape().Size();\n    const int dtype            = comm_buf.dtype();\n\n    // Init the small buffer and residual_ buffer for quantize\n    if (small_buf.is_none()) {\n      small_buf = NDArray(mxnet::TShape{pskv.size}, comm_buf.ctx(), false, dtype);\n      res_buf =\n          NDArray(mxnet::TShape{static_cast<int64_t>(original_size)}, comm_buf.ctx(), false, dtype);\n      res_buf = 0;\n    }\n    gradient_compression_->Quantize(comm_buf, &small_buf, &res_buf, priority);\n    auto push_to_servers = [this, key, dtype, pskv, small_buf](RunContext rctx,\n                                                               Engine::CallbackOnStart on_start,\n                                                               Engine::CallbackOnComplete cb) {\n      on_start();\n      size_t size = small_buf.shape().Size() * mshadow::mshadow_sizeof(dtype);\n      char* data  = static_cast<char*>(small_buf.data().dptr_);\n      // do push. false means no delete\n      ps::SArray<char> vals(data, size, false);\n      int cmd = GetCommandType(RequestType::kCompressedPushPull, dtype);\n      CHECK_NOTNULL(ps_worker_)->ZPush(pskv.keys, vals, pskv.lens, cmd, [cb]() { cb(); });\n    };\n    // acquire locks on both comm_buf and small_buf so that\n    // pull (which uses comm_buf) for the same key waits till push finishes\n    Engine::Get()->PushAsync(push_to_servers,\n                             pinned_ctx_,\n                             {small_buf.var(), comm_buf.var()},\n                             {},\n                             FnProperty::kNormal,\n                             priority,\n                             \"KVStoreDistCompressedPush\");\n  }\n\n  virtual void PushDefault(int key, const NDArray& send_buf, const PSKV& pskv, int priority) {\n    auto push_to_servers = [this, key, pskv, send_buf](RunContext rctx,\n                                                       Engine::CallbackOnStart on_start,\n                                                       Engine::CallbackOnComplete cb) {\n      on_start();\n      const int dtype = send_buf.dtype();\n      // convert to ps keys\n      const size_t size = send_buf.shape().Size() * mshadow::mshadow_sizeof(dtype);\n      char* data        = static_cast<char*>(send_buf.data().dptr_);\n      // do push. false means no delete\n      ps::SArray<char> vals(data, size, false);\n      int cmd = GetCommandType(RequestType::kDefaultPushPull, dtype);\n      CHECK_NOTNULL(ps_worker_)->ZPush(pskv.keys, vals, pskv.lens, cmd, [cb]() { cb(); });\n    };\n    Engine::Get()->PushAsync(push_to_servers,\n                             pinned_ctx_,\n                             {send_buf.var()},\n                             {},\n                             FnProperty::kNormal,\n                             priority,\n                             \"KVStoreDistDefaultPush\");\n  }\n\n  // push row sparse gradient\n  virtual void PushRowSparse(int key, const NDArray& send_buf, int priority) {\n    using namespace rowsparse;\n    auto push_to_servers = [this, key, send_buf](RunContext rctx,\n                                                 Engine::CallbackOnStart on_start,\n                                                 Engine::CallbackOnComplete cb) {\n      on_start();\n      char* data             = static_cast<char*>(send_buf.data().dptr_);\n      const int64_t num_rows = send_buf.aux_shape(kIdx)[0];\n      const auto offsets     = send_buf.aux_data(kIdx).dptr<int64_t>();\n      const auto unit_len    = send_buf.shape().ProdShape(1, send_buf.shape().ndim());\n      const int num_bytes    = mshadow::mshadow_sizeof(send_buf.dtype());\n      const int64_t size     = num_rows * unit_len;\n      // convert to ps keys in row sparse format\n      PSKV& pskv = EncodeRowSparseKey(\n          key, size, num_rows, offsets, unit_len, send_buf.shape()[0], num_bytes);\n      if (this->log_verbose_) {\n        LOG(INFO) << \"worker \" << get_rank() << \" push lens: \" << pskv.lens\n                  << \" keys: \" << pskv.keys << \" size: \" << size;\n      }\n      ps::SArray<char> vals(data, size * num_bytes, false);\n      const int cmd = GetCommandType(RequestType::kRowSparsePushPull, send_buf.dtype());\n      CHECK_NOTNULL(ps_worker_)->ZPush(pskv.keys, vals, pskv.lens, cmd, [cb]() { cb(); });\n    };\n    Engine::Get()->PushAsync(push_to_servers,\n                             pinned_ctx_,\n                             {send_buf.var()},\n                             {},\n                             FnProperty::kNormal,\n                             priority,\n                             \"KVStoreDistRowSparsePush\");\n  }\n\n  virtual void PullDefault(int key, const NDArray& recv_buf, int priority) {\n    auto pull_from_servers = [this, key, recv_buf](RunContext rctx,\n                                                   Engine::CallbackOnStart on_start,\n                                                   Engine::CallbackOnComplete cb) {\n      on_start();\n      // convert to ps keys\n      size_t size         = recv_buf.shape().Size();\n      const int dtype     = recv_buf.dtype();\n      const int num_bytes = mshadow::mshadow_sizeof(dtype);\n      PSKV& pskv          = (gradient_compression_->get_type() == CompressionType::kNone) ?\n                       EncodeDefaultKey(key, size, num_bytes) :\n                       EncodeCompressedKey(key, size, false, num_bytes);\n      char* data = static_cast<char*>(recv_buf.data().dptr_);\n      // false means not to delete data when SArray is deleted\n      auto vals = new ps::SArray<char>(data, size * num_bytes, false);\n      // issue pull\n      RequestType mode = (gradient_compression_->get_type() != CompressionType::kNone) ?\n                             RequestType::kCompressedPushPull :\n                             RequestType::kDefaultPushPull;\n      const int cmd = GetCommandType(mode, dtype);\n      CHECK_NOTNULL(ps_worker_)->ZPull(pskv.keys, vals, &pskv.lens, cmd, [vals, cb]() {\n        delete vals;\n        cb();\n      });\n    };\n\n    CHECK_NOTNULL(Engine::Get())\n        ->PushAsync(pull_from_servers,\n                    pinned_ctx_,\n                    {},\n                    {recv_buf.var()},\n                    FnProperty::kNormal,\n                    priority,\n                    \"KVStoreDistDefaultStoragePull\");\n  }\n\n  // pull row sparse weight into `recv_buf` based on indices given by `indices`\n  virtual void PullRowSparse_(const int key,\n                              const NDArray& recv_buf,\n                              const NDArray& indices,\n                              int priority) {\n    using namespace rowsparse;\n    auto pull_from_servers = [this, key, recv_buf, indices](RunContext rctx,\n                                                            Engine::CallbackOnStart on_start,\n                                                            Engine::CallbackOnComplete cb) {\n      on_start();\n      // allocate memory for the buffer\n      CHECK_EQ(indices.dtype(), mshadow::kInt64);\n      const TBlob idx_data  = indices.data();\n      const size_t num_rows = idx_data.shape_.Size();\n      recv_buf.CheckAndAlloc({mshadow::Shape1(num_rows)});\n      const int dtype     = recv_buf.dtype();\n      char* data          = static_cast<char*>(recv_buf.data().dptr_);\n      const auto offsets  = idx_data.dptr<int64_t>();\n      const auto unit_len = recv_buf.shape().ProdShape(1, recv_buf.shape().ndim());\n      const int64_t size  = num_rows * unit_len;\n      const int num_bytes = mshadow::mshadow_sizeof(dtype);\n      // convert to ps keys in row sparse format\n      PSKV& pskv = EncodeRowSparseKey(\n          key, size, num_rows, offsets, unit_len, recv_buf.shape()[0], num_bytes);\n      if (this->log_verbose_) {\n        LOG(INFO) << \"worker \" << get_rank() << \" pull lens: \" << pskv.lens\n                  << \" keys: \" << pskv.keys << \" size: \" << size;\n      }\n      auto vals     = new ps::SArray<char>(data, size * num_bytes, false);\n      const int cmd = GetCommandType(RequestType::kRowSparsePushPull, recv_buf.dtype());\n      // copy indices to recv_buf. this needs to be done before ZPull\n      // because after pull is done, the callback function returns and locks are released.\n      // at this point, later functions may access the indices variable while copy happens\n      mshadow::Copy(recv_buf.aux_data(kIdx).FlatTo1D<cpu, int64_t>(),\n                    idx_data.FlatTo1D<cpu, int64_t>());\n      CHECK_NOTNULL(ps_worker_)->ZPull(pskv.keys, vals, &pskv.lens, cmd, [vals, cb]() {\n        delete vals;\n        cb();\n      });\n    };\n    CHECK_NOTNULL(Engine::Get())\n        ->PushAsync(pull_from_servers,\n                    pinned_ctx_,\n                    {indices.var()},\n                    {recv_buf.var()},\n                    FnProperty::kNormal,\n                    priority,\n                    \"KVStoreDistRowSparsePull\");\n  }\n\n  virtual void PushPullDefault(int key, const NDArray& comm_buf, int priority) {\n    auto pushpull = [this, key, comm_buf](RunContext rctx,\n                                          Engine::CallbackOnStart on_start,\n                                          Engine::CallbackOnComplete cb) {\n      on_start();\n      size_t size         = comm_buf.shape().Size();\n      const int dtype     = comm_buf.dtype();\n      const int num_bytes = mshadow::mshadow_sizeof(dtype);\n      const int cmd       = GetCommandType(RequestType::kDefaultPushPull, dtype);\n\n      PSKV& pskv = EncodeDefaultKey(key, size, num_bytes);\n      char* data = static_cast<char*>(comm_buf.data().dptr_);\n      auto vals  = new ps::SArray<char>(data, size * num_bytes, false);\n\n      CHECK_NOTNULL(ps_worker_)->ZPushPull(pskv.keys, *vals, vals, &pskv.lens, cmd, [vals, cb]() {\n        delete vals;\n        cb();\n      });\n    };\n\n    CHECK_NOTNULL(Engine::Get())\n        ->PushAsync(pushpull,\n                    pinned_ctx_,\n                    {},\n                    {comm_buf.var()},\n                    FnProperty::kNormal,\n                    priority,\n                    \"KVStoreDistDefaultStoragePushPull\");\n  }\n\n  /**\n   * \\brief check if the keys are all unique\n   */\n  void CheckUnique(const std::vector<int>& keys) {\n    auto keys_copy = keys;\n    auto last      = std::unique(keys_copy.begin(), keys_copy.end());\n    CHECK_EQ(static_cast<size_t>(std::distance(keys_copy.begin(), last)),\n             static_cast<size_t>(keys.size()));\n  }\n\n  /**\n   * \\brief convert to pskv for parameter server\n   * \\param key\n   * \\param num_arr_elems number of elements in the value for key\n   * \\param num_bytes size of each element in number of bytes\n   * \\return PSKV used for both push and pull\n   */\n  virtual inline PSKV& EncodeDefaultKey(const int key,\n                                        const size_t num_arr_elems,\n                                        const int num_bytes) {\n    mu_.lock();\n    PSKV& pskv = ps_kv_[key];\n    mu_.unlock();\n    size_t pskv_size = num_arr_elems * num_bytes;\n    if (!pskv.keys.empty()) {\n      CHECK_EQ(static_cast<size_t>(pskv.size), pskv_size)\n          << \"The value size cannot be changed \" << pskv_size << \". Key is \" << key;\n    } else {\n      auto krs              = ps::Postoffice::Get()->GetServerKeyRanges();\n      const int num_servers = krs.size();\n      CHECK_GT(num_servers, 0);\n\n      // a simple heuristic for load balance\n      if (num_arr_elems < bigarray_bound_) {\n        // send it to a single random picked server\n        int server     = (key * 9973) % num_servers;\n        ps::Key ps_key = krs[server].begin() + key;\n        CHECK_LT(ps_key, krs[server].end());\n        pskv.keys.push_back(ps_key);\n        const int total_bytes = num_arr_elems * num_bytes;\n        pskv.lens.push_back(total_bytes);\n        pskv.size = total_bytes;\n      } else {\n        // parition it to all servers\n        pskv.size = 0;\n        for (int i = 0; i < num_servers; ++i) {\n          size_t part_size =\n              static_cast<size_t>(\n                  round(static_cast<double>(num_arr_elems) / num_servers * (i + 1))) -\n              static_cast<size_t>(round(static_cast<double>(num_arr_elems) / num_servers * i));\n          ps::Key ps_key = krs[i].begin() + key;\n          CHECK_LT(ps_key, krs[i].end());\n          pskv.keys.push_back(ps_key);\n          const int total_bytes = part_size * num_bytes;\n          pskv.lens.push_back(total_bytes);\n          pskv.size += total_bytes;\n        }\n      }\n      CHECK_EQ(static_cast<size_t>(pskv.size), pskv_size);\n    }\n    return pskv;\n  }\n\n  /**\n   * \\brief Convert to PSKV for pushes and pulls when gradient compression is used.\n   * Divides original array into equal parts for each server.\n   * Populates both push and pull pskv on first call.\n   * \\param key\n   * \\param num_arr_elems number of elements in the value for key\n   * \\param is_push whether this is push or pull\n   * \\param num_bytes size of each element in number of bytes\n   * \\return PSKV used for both push and pull\n   */\n  virtual inline PSKV& EncodeCompressedKey(const int key,\n                                           const size_t original_num_elem,\n                                           const bool is_push,\n                                           const int num_bytes) {\n    auto krs              = ps::Postoffice::Get()->GetServerKeyRanges();\n    const int num_servers = krs.size();\n    CHECK_GT(num_servers, 0);\n\n    // represents size of data to be sent\n    size_t compr_num_elem = gradient_compression_->GetCompressedSize(original_num_elem);\n    mu_.lock();\n    PSKV& pskv = (is_push) ? compr_ps_kv_[key].push : compr_ps_kv_[key].pull;\n    mu_.unlock();\n\n    if (!pskv.keys.empty()) {\n      const size_t num_elem = (is_push) ? compr_num_elem : original_num_elem;\n      CHECK_EQ(static_cast<size_t>(pskv.size), num_elem * num_bytes)\n          << \"The value size can't be changed. For key \" << key;\n    } else {\n      // populate both pull and push pskvs\n      // push pskv has sizes corresponding to compressed data\n      // pull pskv has decompressed sizes for parts in push_pskv\n      mu_.lock();\n      PSKV& pull_pskv = compr_ps_kv_[key].pull;\n      PSKV& push_pskv = compr_ps_kv_[key].push;\n      mu_.unlock();\n\n      if (original_num_elem < bigarray_bound_) {\n        // a simple heuristic for load balancing\n        // send it to a single random picked server\n        const int server = (key * 9973) % num_servers;\n        ps::Key ps_key   = krs[server].begin() + key;\n        CHECK_LT(ps_key, krs[server].end());\n        // meta info\n        push_pskv.keys.push_back(krs[server].begin() + original_num_elem);\n        push_pskv.lens.push_back(0);\n        // data\n        push_pskv.keys.push_back(ps_key);\n        pull_pskv.keys.push_back(ps_key);\n        const int compr_size    = compr_num_elem * num_bytes;\n        const int original_size = original_num_elem * num_bytes;\n        push_pskv.lens.push_back(compr_size);\n        pull_pskv.lens.push_back(original_size);\n        push_pskv.size = compr_size;\n        pull_pskv.size = original_size;\n      } else {\n        // partition it to all servers\n        push_pskv.size = 0;\n        pull_pskv.size = 0;\n\n        for (int i = 0; i < num_servers; ++i) {\n          size_t part_compr, part_orig;\n          if (i == num_servers - 1) {\n            part_compr = compr_num_elem - push_pskv.size;\n            part_orig  = original_num_elem - pull_pskv.size;\n          } else {\n            part_compr =\n                static_cast<size_t>(\n                    round(static_cast<double>(compr_num_elem) / num_servers * (i + 1))) -\n                static_cast<size_t>(round(static_cast<double>(compr_num_elem) / num_servers * (i)));\n            part_orig = part_compr * gradient_compression_->GetCompressionFactor();\n          }\n\n          // meta info\n          ps::Key ps_key_dummy = krs[i].begin() + part_orig;\n          CHECK_LT(ps_key_dummy, krs[i].end());\n          push_pskv.keys.push_back(ps_key_dummy);\n          push_pskv.lens.push_back(0);\n\n          // data\n          ps::Key ps_key = krs[i].begin() + key;\n          CHECK_LT(ps_key, krs[i].end());\n          push_pskv.keys.push_back(ps_key);\n          pull_pskv.keys.push_back(ps_key);\n          push_pskv.lens.push_back(part_compr * num_bytes);\n          pull_pskv.lens.push_back(part_orig * num_bytes);\n          // num elements need to be inserted below so that for last server,\n          // there is no round off error\n          push_pskv.size += part_compr;\n          pull_pskv.size += part_orig;\n        }\n        CHECK_EQ(static_cast<size_t>(push_pskv.size), compr_num_elem);\n        CHECK_EQ(static_cast<size_t>(pull_pskv.size), original_num_elem);\n        push_pskv.size *= num_bytes;\n        pull_pskv.size *= num_bytes;\n        CHECK_EQ(push_pskv.lens.size(), num_servers * 2);\n      }\n    }\n    return pskv;\n  }\n\n  // Note: this encoding method for row sparse keys doesn't allow cross-layer batching\n  virtual inline PSKV& EncodeRowSparseKey(const int key,\n                                          const int64_t num_elem,\n                                          const int64_t num_rows,\n                                          const int64_t* offsets,\n                                          const size_t unit_len,\n                                          const int64_t total_num_rows,\n                                          const int num_bytes) {\n    using namespace common;\n    mu_.lock();\n    PSKV& pskv = ps_kv_[key];\n    mu_.unlock();\n    pskv.keys.clear();\n    pskv.lens.clear();\n    // TODO(haibin) cache this information\n    auto krs              = ps::Postoffice::Get()->GetServerKeyRanges();\n    const int num_servers = krs.size();\n    CHECK_GT(num_servers, 0);\n\n    if (total_num_rows * unit_len >= bigarray_bound_) {\n      pskv.size         = 0;\n      int64_t start_row = 0;\n      // parition it to all servers\n      for (int i = 0; i < num_servers; ++i) {\n        ps::Key master_key = krs[i].begin() + key;\n        pskv.keys.push_back(master_key);\n        pskv.lens.push_back(0);\n        if (offsets && num_elem > 0) {\n          // calculate partition ranges\n          int64_t part_num_rows =\n              llround(static_cast<double>(total_num_rows) / num_servers * (i + 1)) -\n              llround(static_cast<double>(total_num_rows) / num_servers * i);\n          auto end_row = start_row + part_num_rows;\n          // search for offsets in [start_row, end_row)\n          auto lb = std::lower_bound(offsets, offsets + num_rows, start_row);\n          auto ub = std::upper_bound(offsets, offsets + num_rows, end_row - 1);\n          for (auto offset = lb; offset < ub; offset++) {\n            ps::Key ps_key = krs[i].begin() + key + (*offset - start_row);\n            CHECK_LT(ps_key, krs[i].end());\n            pskv.keys.push_back(ps_key);\n            const int part_size = unit_len * num_bytes;\n            pskv.lens.push_back(part_size);\n            pskv.size += (part_size);\n          }\n          start_row = end_row;\n        }\n      }\n      CHECK_EQ(static_cast<size_t>(pskv.size), num_elem * num_bytes);\n    } else {\n      // send it to a single random picked server\n      const int server   = (key * 9973) % num_servers;\n      ps::Key master_key = krs[server].begin() + key;\n      pskv.keys.push_back(master_key);\n      pskv.lens.push_back(0);\n      for (int64_t i = 0; i < num_rows; i++) {\n        ps::Key ps_key = krs[server].begin() + key + offsets[i];\n        CHECK_LT(ps_key, krs[server].end());\n        pskv.keys.push_back(ps_key);\n        pskv.lens.push_back(unit_len * num_bytes);\n      }\n      pskv.size = num_elem * num_bytes;\n    }\n    return pskv;\n  }\n\n  /**\n   * \\brief the server handle\n   */\n  KVStoreDistServer* server_;\n  /**\n   * \\brief threshold for partition\n   */\n  size_t bigarray_bound_;\n  /**\n   * \\brief buffer for non-compressed data.\n   * When gradient compression is active, this is used\n   * for the data in pull and for original data in push\n   */\n  std::unordered_map<int, NDArray> comm_buf_;\n  /**\n   * \\brief buffer for compressed data\n   * Used when gradient compression is active and action\n   * is push\n   */\n  std::unordered_map<int, NDArray> compr_buf_;\n  /**\n   * \\brief residual buffer to accumulate quantization error\n   * during gradient compression\n   */\n  std::unordered_map<int, NDArray> residual_;\n  bool log_verbose_;\n};\n\n}  // namespace kvstore\n}  // namespace mxnet\n\n#endif  // MXNET_KVSTORE_KVSTORE_DIST_H_\n"
  },
  {
    "path": "src/kvstore/kvstore_dist_server.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file mxnet_node.h\n * \\brief implement mxnet nodes\n */\n#ifndef MXNET_KVSTORE_KVSTORE_DIST_SERVER_H_\n#define MXNET_KVSTORE_KVSTORE_DIST_SERVER_H_\n#include <mxnet/c_api.h>\n#include <mxnet/kvstore.h>\n#include <ps/ps.h>\n#include <queue>\n#include <string>\n#include <mutex>\n#include <condition_variable>\n#include <memory>\n#include <functional>\n#include <future>\n#include <vector>\n#include \"../profiler/profiler.h\"\n#include \"../operator/tensor/elemwise_binary_op-inl.h\"\n#include \"../operator/tensor/init_op.h\"\n\nnamespace mxnet {\nnamespace kvstore {\n\n// maintain same order in frontend.\nenum class CommandType {\n  kController,\n  kSetMultiPrecision,\n  kStopServer,\n  kSyncMode,\n  kSetGradientCompression,\n  kSetProfilerParams\n};\n\nenum class RequestType { kDefaultPushPull, kRowSparsePushPull, kCompressedPushPull };\n\nstruct DataHandleType {\n  RequestType requestType;\n  int dtype;\n};\n\n/*!\n * Uses Cantor pairing function to generate a unique number given two numbers.\n * This number can also be inverted to find the unique pair whose Cantor value is this number.\n * Ref: https://en.wikipedia.org/wiki/Pairing_function#Cantor_pairing_function\n * \\param requestType RequestType\n * \\param dtype integer\n * \\return Cantor value of arguments\n */\nstatic int GetCommandType(RequestType requestType, int d) {\n  int m = static_cast<int>(requestType);\n  return (((m + d) * (m + d + 1)) / 2) + d;\n}\n\n/*!\n * Unpairs Cantor value and finds the two integers used to pair.\n * Then returns DataHandleType object with those numbers.\n * \\param cmd DataHandleCommand generated by GetCommandType function\n * \\return DataHandleType\n */\nstatic DataHandleType DepairDataHandleType(int cmd) {\n  int w = std::floor((std::sqrt(8 * cmd + 1) - 1) / 2);\n  int t = ((w * w) + w) / 2;\n  int y = cmd - t;\n  int x = w - y;\n  CHECK_GE(x, 0);\n  CHECK_GE(y, 0);\n  DataHandleType type;\n  type.requestType = static_cast<RequestType>(x);\n  type.dtype       = y;\n  return type;\n}\n\n/**\n * \\brief executor runs a function using the thread called \\ref Start\n */\nclass Executor {\n public:\n  /**\n   * \\brief start the executor\n   */\n  void Start() {\n    std::unique_lock<std::mutex> lk(mu_);\n    while (true) {\n      cond_.wait(lk, [this] { return !queue_.empty(); });\n      Block blk = std::move(queue_.front());\n      queue_.pop();\n      lk.unlock();\n\n      if (blk.f) {\n        blk.f();\n        blk.p->set_value();\n      } else {\n        blk.p->set_value();\n        break;\n      }\n      lk.lock();\n    }\n  }\n\n  /**\n   * \\brief function\n   */\n  typedef std::function<void()> Func;\n\n  /**\n   * \\brief let the thread called \\ref Start to exec a function. threadsafe\n   */\n  void Exec(const Func& func) {\n    Block blk(func);\n    auto fut = blk.p->get_future();\n    {\n      std::lock_guard<std::mutex> lk(mu_);\n      queue_.push(std::move(blk));\n      cond_.notify_one();\n    }\n    fut.wait();\n  }\n\n  /**\n   * \\brief stop the thread, threadsafe\n   */\n  void Stop() {\n    Exec(Func());\n  }\n\n private:\n  struct Block {\n    explicit Block(const Func& func) : f(func), p(std::make_shared<std::promise<void>>()) {}\n    Func f;\n    std::shared_ptr<std::promise<void>> p;\n  };\n  std::queue<Block> queue_;\n  std::mutex mu_;\n  std::condition_variable cond_;\n};\n\nclass KVStoreDistServer {\n public:\n  KVStoreDistServer() {\n    using namespace std::placeholders;\n    ps_server_ = new ps::KVServer<char>(0);\n    static_cast<ps::SimpleApp*>(ps_server_)\n        ->set_request_handle(std::bind(&KVStoreDistServer::CommandHandle, this, _1, _2));\n    ps_server_->set_request_handle(std::bind(&KVStoreDistServer::DataHandleEx, this, _1, _2, _3));\n    sync_mode_            = false;\n    gradient_compression_ = std::make_shared<GradientCompression>();\n    log_verbose_          = dmlc::GetEnv(\"MXNET_KVSTORE_DIST_ROW_SPARSE_VERBOSE\", false);\n  }\n\n  ~KVStoreDistServer() {\n    profiler::Profiler::Get()->SetState(profiler::Profiler::ProfilerState(0));\n    delete ps_server_;\n  }\n\n  void set_controller(const KVStore::Controller& controller) {\n    CHECK(controller);\n    controller_ = controller;\n  }\n\n  void set_updater(const KVStore::Updater& updater) {\n    CHECK(updater);\n    updater_ = updater;\n  }\n\n  /**\n   * \\brief blocked until received the command \\a kSyncMode\n   */\n  void Run() {\n    exec_.Start();\n  }\n\n private:\n  struct UpdateBuf {\n    std::vector<ps::KVMeta> request;\n    NDArray merged;\n    // temp_array is used to cast received values as float32 for computation if required\n    NDArray temp_array;\n  };\n\n  void CommandHandle(const ps::SimpleData& recved, ps::SimpleApp* app) {\n    CommandType recved_type = static_cast<CommandType>(recved.head);\n    switch (recved_type) {\n      case CommandType::kStopServer:\n        exec_.Stop();\n        break;\n      case CommandType::kSyncMode:\n        sync_mode_ = true;\n        break;\n      case CommandType::kSetGradientCompression:\n        gradient_compression_->DecodeParams(recved.body);\n        break;\n      case CommandType::kSetProfilerParams:\n        // last char is the type of profiler command\n        ProcessServerProfilerCommands(\n            static_cast<KVStoreServerProfilerCommand>(recved.body.back() - '0'), recved.body);\n        break;\n      case CommandType::kSetMultiPrecision:\n        // uses value 1 for message id from frontend\n        if (!multi_precision_) {\n          multi_precision_ = true;\n          CreateMultiPrecisionCopies();\n        }\n        break;\n      case CommandType::kController:\n        // this uses value 0 for message id from frontend\n        // let the main thread to execute ctrl, which is necessary for python\n        exec_.Exec([this, recved]() {\n          CHECK(controller_);\n          controller_(recved.head, recved.body);\n        });\n        break;\n    }\n    app->Response(recved);\n  }\n\n  /*\n   * For keys already initialized, if necessary create stored_realt.\n   * This will only be used if by some wrong usage of kvstore,\n   * some keys are initialized before optimizer is set.\n   */\n  void CreateMultiPrecisionCopies() {\n    for (auto const& stored_entry : store_) {\n      const int key         = stored_entry.first;\n      const NDArray& stored = stored_entry.second;\n      if (stored.dtype() != mshadow::kFloat32) {\n        auto& stored_realt = store_realt_[key];\n        if (stored.storage_type() == kRowSparseStorage) {\n          stored_realt =\n              NDArray(kRowSparseStorage, stored.shape(), stored.ctx(), true, mshadow::kFloat32);\n        } else {\n          stored_realt = NDArray(stored.shape(), stored.ctx(), false, mshadow::kFloat32);\n        }\n\n        auto& update = update_buf_[key];\n        if (!update.merged.is_none()) {\n          if (update.merged.storage_type() == kRowSparseStorage) {\n            update.merged = NDArray(kRowSparseStorage,\n                                    update.merged.shape(),\n                                    update.merged.ctx(),\n                                    true,\n                                    mshadow::kFloat32);\n          } else {\n            update.merged =\n                NDArray(update.merged.shape(), update.merged.ctx(), false, mshadow::kFloat32);\n          }\n        }\n        CHECK(update.request.size() == 0)\n            << ps::MyRank() << \"Multiprecision mode can not be set while pushes are underway.\"\n            << \"Please set optimizer before pushing keys.\" << key << \" \" << update.request.size();\n\n        CopyFromTo(stored, stored_realt);\n      }\n    }\n    for (auto const& stored_realt_entry : store_realt_) {\n      stored_realt_entry.second.WaitToRead();\n    }\n  }\n\n  void ProcessServerProfilerCommands(KVStoreServerProfilerCommand type, const std::string& body) {\n    switch (type) {\n      case KVStoreServerProfilerCommand::kSetConfig:\n        SetProfilerConfig(body.substr(0, body.size() - 1));\n        break;\n      case KVStoreServerProfilerCommand::kState:\n        MXSetProfilerState(static_cast<int>(body.front() - '0'));\n        break;\n      case KVStoreServerProfilerCommand::kPause:\n        MXProfilePause(static_cast<int>(body.front() - '0'));\n        break;\n      case KVStoreServerProfilerCommand::kDump:\n        MXDumpProfile(static_cast<int>(body.front() - '0'));\n        break;\n    }\n  }\n\n  void SetProfilerConfig(std::string params_str) {\n    std::vector<std::string> elems;\n    mxnet::kvstore::split(params_str, ',', std::back_inserter(elems));\n    std::vector<const char*> ckeys;\n    std::vector<const char*> cvals;\n    ckeys.reserve(elems.size());\n    cvals.reserve(elems.size());\n\n    for (size_t i = 0; i < elems.size(); i++) {\n      std::vector<std::string> parts;\n      mxnet::kvstore::split(elems[i], ':', std::back_inserter(parts));\n      CHECK_EQ(parts.size(), 2) << \"Improper profiler config passed from worker\";\n      CHECK(!parts[0].empty()) << \"ProfilerConfig parameter is empty\";\n      CHECK(!parts[1].empty()) << \"ProfilerConfig value is empty for parameter \" << parts[0];\n      if (parts[0] == \"filename\") {\n        parts[1] = \"rank\" + std::to_string(ps::MyRank()) + \"_\" + parts[1];\n      }\n      char* ckey = new char[parts[0].length() + 1];\n      std::snprintf(ckey, parts[0].length() + 1, \"%s\", parts[0].c_str());\n      ckeys.push_back(ckey);\n\n      char* cval = new char[parts[1].length() + 1];\n      std::snprintf(cval, parts[1].length() + 1, \"%s\", parts[1].c_str());\n      cvals.push_back(cval);\n    }\n    MXSetProfilerConfig(elems.size(), &ckeys[0], &cvals[0]);\n    for (size_t i = 0; i < ckeys.size(); i++) {\n      delete[] ckeys[i];\n      delete[] cvals[i];\n    }\n  }\n\n  void DataHandleEx(const ps::KVMeta& req_meta,\n                    const ps::KVPairs<char>& req_data,\n                    ps::KVServer<char>* server) {\n    DataHandleType type = DepairDataHandleType(req_meta.cmd);\n    switch (type.requestType) {\n      case RequestType::kRowSparsePushPull:\n        DataHandleRowSparse(type, req_meta, req_data, server);\n        break;\n      case RequestType::kCompressedPushPull:\n        DataHandleCompressed(type, req_meta, req_data, server);\n        break;\n      case RequestType::kDefaultPushPull:\n        DataHandleDefault(type, req_meta, req_data, server);\n        break;\n    }\n  }\n\n  inline bool has_multi_precision_copy(const DataHandleType type) {\n    return multi_precision_ && type.dtype != mshadow::kFloat32;\n  }\n\n  inline void ApplyUpdates(const DataHandleType type,\n                           const int key,\n                           const ps::KVPairs<char>& req_data,\n                           UpdateBuf* update_buf,\n                           ps::KVServer<char>* server) {\n    if (!sync_mode_ || update_buf->request.size() == (size_t)ps::NumWorkers()) {\n      // let the main thread to execute updater_, which is necessary for python\n      auto& stored = has_multi_precision_copy(type) ? store_realt_[key] : store_[key];\n      auto& update = sync_mode_ ? update_buf->merged : update_buf->temp_array;\n      if (updater_) {\n        exec_.Exec([this, key, &update, &stored]() {\n          CHECK(updater_);\n          updater_(key, update, &stored);\n        });\n      } else {\n        CHECK(sync_mode_) << \"Updater needs to be set for async mode\";\n        // if no updater, just copy\n        CopyFromTo(update_buf->merged, &stored);\n      }\n\n      if (log_verbose_) {\n        LOG(INFO) << \"sent response to \" << update_buf->request.size() << \" workers\";\n      }\n      /**\n       * Request can be for either push, pull or pushpull\n       * If pull flag is set, respond immediately with the updated values\n       * Otherwise, only send the notification\n       */\n      bool has_pull = false;\n      for (const auto& req : update_buf->request) {\n        has_pull = has_pull || req.pull;\n      }\n      if (has_pull) {\n        // if there is a pull request, perform WaitToRead() once before DefaultStorageResponse\n        if (has_multi_precision_copy(type))\n          CopyFromTo(stored, store_[key]);\n        stored.WaitToRead();\n        for (const auto& req : update_buf->request) {\n          if (req.pull) {\n            DefaultStorageResponse(type, key, req, req_data, server);\n          }\n        }\n        update_buf->request.clear();\n      } else {\n        // otherwise, send response directly\n        for (const auto& req : update_buf->request) {\n          server->Response(req);\n        }\n        update_buf->request.clear();\n        if (has_multi_precision_copy(type))\n          CopyFromTo(stored, store_[key]);\n        stored.WaitToRead();\n      }\n    } else {\n      update_buf->merged.WaitToRead();\n    }\n  }\n\n  void DecodeRowIds(const ps::SArray<ps::Key>& keys,\n                    int64_t* indices,\n                    const int64_t master_key,\n                    const int64_t num_rows) {\n    indices[0] = 0;\n    for (int64_t i = 1; i <= num_rows; i++) {\n      int key        = DecodeKey(keys[i]);\n      auto row_id    = key - master_key;\n      indices[i - 1] = row_id;\n    }\n  }\n\n  void AccumulateRowSparseGrads(const DataHandleType type,\n                                const NDArray& recved,\n                                UpdateBuf* updateBuf) {\n    NDArray out(kRowSparseStorage,\n                updateBuf->merged.shape(),\n                Context(),\n                true,\n                has_multi_precision_copy(type) ? mshadow::kFloat32 : type.dtype);\n    if (has_multi_precision_copy(type))\n      CopyFromTo(recved, updateBuf->temp_array);\n    const NDArray& to_merge = has_multi_precision_copy(type) ? updateBuf->temp_array : recved;\n    // accumulate row_sparse gradients\n    using namespace mshadow;\n    Engine::Get()->PushAsync(\n        [to_merge, updateBuf, out](RunContext ctx,\n                                   Engine::CallbackOnStart on_start,\n                                   Engine::CallbackOnComplete on_complete) {\n          on_start();\n          op::ElemwiseBinaryOp::ComputeEx<cpu, op::mshadow_op::plus>(\n              {}, {}, {to_merge, updateBuf->merged}, {kWriteTo}, {out});\n          on_complete();\n        },\n        to_merge.ctx(),\n        {to_merge.var(), updateBuf->merged.var()},\n        {out.var()},\n        FnProperty::kNormal,\n        0,\n        PROFILER_MESSAGE_FUNCNAME);\n    CopyFromTo(out, &(updateBuf->merged), 0);\n    updateBuf->merged.WaitToRead();\n  }\n\n  void RowSparsePullResponse(const DataHandleType type,\n                             const int master_key,\n                             const size_t num_rows,\n                             const ps::KVMeta& req_meta,\n                             const ps::KVPairs<char>& req_data,\n                             ps::KVServer<char>* server) {\n    if (log_verbose_)\n      LOG(INFO) << \"pull: \" << master_key;\n    ps::KVPairs<char> response;\n    if (num_rows == 0) {\n      std::vector<int> lens(req_data.keys.size(), 0);\n      response.keys = req_data.keys;\n      response.lens.CopyFrom(lens.begin(), lens.end());\n      server->Response(req_meta, response);\n      return;\n    }\n    const NDArray& stored = store_[master_key];\n    if (has_multi_precision_copy(type))\n      stored.WaitToRead();\n    CHECK(!stored.is_none()) << \"init \" << master_key << \" first\";\n    auto shape          = stored.shape();\n    auto unit_len       = shape.ProdShape(1, shape.ndim());\n    const int num_bytes = mshadow::mshadow_sizeof(type.dtype);\n    const int unit_size = unit_len * num_bytes;\n    const char* data    = static_cast<char*>(stored.data().dptr_);\n    auto len            = num_rows * unit_size;\n    // concat values\n    response.vals.resize(len);\n#pragma omp parallel for\n    for (size_t i = 1; i <= num_rows; i++) {\n      int key        = DecodeKey(req_data.keys[i]);\n      int64_t row_id = key - master_key;\n      const auto src = data + row_id * unit_size;\n      auto begin     = (i - 1) * unit_size;\n      auto end       = i * unit_size;\n      response.vals.segment(begin, end).CopyFrom(src, unit_size);\n    }\n    // setup response\n    response.keys = req_data.keys;\n    std::vector<int> lens(req_data.keys.size(), unit_len);\n    lens[0] = 0;\n    response.lens.CopyFrom(lens.begin(), lens.end());\n    server->Response(req_meta, response);\n  }\n\n  void InitRowSparseStored(const DataHandleType type,\n                           const int master_key,\n                           const size_t num_rows,\n                           const ps::KVMeta& req_meta,\n                           const ps::KVPairs<char>& req_data,\n                           ps::KVServer<char>* server) {\n    auto& stored  = has_multi_precision_copy(type) ? store_realt_[master_key] : store_[master_key];\n    int dtype     = type.dtype;\n    int num_bytes = mshadow::mshadow_sizeof(dtype);\n    auto unit_len = req_data.lens[1] / num_bytes;\n    CHECK_GT(unit_len, 0);\n    size_t ds[] = {num_rows, (size_t)unit_len};\n    mxnet::TShape dshape(ds, ds + 2);\n    CHECK_EQ(req_data.vals.size(), num_rows * unit_len * num_bytes);\n    TBlob recv_blob;\n    MSHADOW_REAL_TYPE_SWITCH(dtype, DType, {\n      recv_blob = TBlob(reinterpret_cast<DType*>(req_data.vals.data()), dshape, cpu::kDevMask);\n    })\n    NDArray recved = NDArray(recv_blob, 0);\n    stored         = NDArray(kRowSparseStorage,\n                     dshape,\n                     Context(),\n                     true,\n                     has_multi_precision_copy(type) ? mshadow::kFloat32 : type.dtype);\n    if (has_multi_precision_copy(type)) {\n      store_[master_key] = NDArray(kRowSparseStorage, dshape, Context(), true, type.dtype);\n    }\n    Engine::Get()->PushAsync(\n        [this, recved, stored, type](RunContext ctx,\n                                     Engine::CallbackOnStart on_start,\n                                     Engine::CallbackOnComplete on_complete) {\n          on_start();\n          NDArray rsp = stored;\n          stored.CheckAndAlloc({mshadow::Shape1(recved.shape()[0])});\n          mshadow::Stream<cpu>* s = ctx.get_stream<cpu>();\n          using namespace mxnet::op;\n          nnvm::dim_t nnr = rsp.shape()[0];\n          MSHADOW_IDX_TYPE_SWITCH(rsp.aux_type(rowsparse::kIdx), IType, {\n            IType* idx = rsp.aux_data(rowsparse::kIdx).dptr<IType>();\n            mxnet_op::Kernel<PopulateFullIdxRspKernel, cpu>::Launch(s, nnr, idx);\n          });\n          TBlob rsp_data = rsp.data();\n          // copies or casts as appropriate\n          ndarray::Copy<cpu, cpu>(recved.data(), &rsp_data, Context(), Context(), RunContext());\n          on_complete();\n        },\n        recved.ctx(),\n        {recved.var()},\n        {stored.var()},\n        FnProperty::kNormal,\n        0,\n        PROFILER_MESSAGE_FUNCNAME);\n    if (has_multi_precision_copy(type)) {\n      CopyFromTo(stored, store_[master_key]);\n      store_[master_key].WaitToRead();\n    }\n    stored.WaitToRead();\n    server->Response(req_meta);\n  }\n\n  void DataHandleRowSparse(const DataHandleType type,\n                           const ps::KVMeta& req_meta,\n                           const ps::KVPairs<char>& req_data,\n                           ps::KVServer<char>* server) {\n    int master_key = DecodeKey(req_data.keys[0]);\n    auto num_rows  = req_data.keys.size() - 1;\n    auto& stored   = store_[master_key];\n    if (req_meta.push) {\n      CHECK_GT(req_data.lens.size(), 0) << \"req_data.lens cannot be empty\";\n      CHECK_EQ(req_data.lens[0], 0);\n      if (stored.is_none()) {\n        if (log_verbose_)\n          LOG(INFO) << \"initial push: \" << master_key;\n        // initialization\n        CHECK_GT(num_rows, 0) << \"init with empty data is not supported\";\n        InitRowSparseStored(type, master_key, num_rows, req_meta, req_data, server);\n        return;\n      } else {\n        if (log_verbose_)\n          LOG(INFO) << \"push: \" << master_key << \" \" << req_data.keys;\n        auto& updates = update_buf_[master_key];\n        if (sync_mode_ && updates.merged.is_none()) {\n          updates.merged = NDArray(kRowSparseStorage,\n                                   stored.shape(),\n                                   Context(),\n                                   true,\n                                   has_multi_precision_copy(type) ? mshadow::kFloat32 : type.dtype);\n        }\n        if (has_multi_precision_copy(type) && updates.temp_array.is_none()) {\n          updates.temp_array =\n              NDArray(kRowSparseStorage, stored.shape(), Context(), false, mshadow::kFloat32);\n        }\n\n        if (num_rows == 0) {\n          if (sync_mode_) {\n            if (updates.request.empty()) {\n              // reset to zeros\n              int merged_dtype = has_multi_precision_copy(type) ? mshadow::kFloat32 : type.dtype;\n              updates.merged =\n                  NDArray(kRowSparseStorage, stored.shape(), Context(), true, merged_dtype);\n            }  // else nothing to aggregate\n            updates.request.push_back(req_meta);\n            ApplyUpdates(type, master_key, req_data, &updates, server);\n          } else {\n            server->Response(req_meta);\n          }\n        } else {\n          auto unit_len = req_data.lens[1] / mshadow::mshadow_sizeof(type.dtype);\n          CHECK_GT(unit_len, 0);\n          // indices\n          std::vector<int64_t> indices(num_rows);\n          DecodeRowIds(req_data.keys, indices.data(), master_key, num_rows);\n\n          // data\n          TBlob idx_blob(indices.data(), mshadow::Shape1(num_rows), cpu::kDevMask);\n          size_t ds[] = {(size_t)num_rows, (size_t)unit_len};\n          mxnet::TShape dshape(ds, ds + 2);\n          TBlob recv_blob;\n          MSHADOW_REAL_TYPE_SWITCH(type.dtype, DType, {\n            recv_blob =\n                TBlob(reinterpret_cast<DType*>(req_data.vals.data()), dshape, cpu::kDevMask);\n          })\n          // row_sparse NDArray\n          NDArray recved(kRowSparseStorage, stored.shape(), recv_blob, {idx_blob}, 0);\n\n          if (updates.request.empty()) {\n            if (sync_mode_) {\n              CopyFromTo(recved, updates.merged);\n            } else {\n              if (has_multi_precision_copy(type)) {\n                CopyFromTo(recved, updates.temp_array);\n              } else {\n                updates.temp_array = recved;\n              }\n            }\n          } else {\n            CHECK(sync_mode_);\n            AccumulateRowSparseGrads(type, recved, &updates);\n          }\n          updates.request.push_back(req_meta);\n          ApplyUpdates(type, master_key, req_data, &updates, server);\n        }\n      }\n    } else {\n      // pull\n      RowSparsePullResponse(type, master_key, num_rows, req_meta, req_data, server);\n    }\n  }\n\n  void DefaultStorageResponse(const DataHandleType type,\n                              const int key,\n                              const ps::KVMeta& req_meta,\n                              const ps::KVPairs<char>& req_data,\n                              ps::KVServer<char>* server) {\n    ps::KVPairs<char> response;\n    const NDArray& stored = store_[key];\n    CHECK(!stored.is_none()) << \"init \" << key << \" first\";\n\n    // as server returns when store_realt is ready in this case\n    if (has_multi_precision_copy(type))\n      stored.WaitToRead();\n\n    auto len      = stored.shape().Size() * mshadow::mshadow_sizeof(stored.dtype());\n    response.keys = req_data.keys;\n    response.lens = {len};\n    // TODO(mli) try to remove this CopyFrom\n    response.vals.CopyFrom(static_cast<const char*>(stored.data().dptr_), len);\n    server->Response(req_meta, response);\n  }\n\n  void DataHandleCompressed(const DataHandleType type,\n                            const ps::KVMeta& req_meta,\n                            const ps::KVPairs<char>& req_data,\n                            ps::KVServer<char>* server) {\n    CHECK_EQ(type.dtype, mshadow::kFloat32)\n        << \"Gradient compression is currently supported for fp32 only\";\n    if (req_meta.push) {\n      // there used several WaitToRead, this is because \\a recved's memory\n      // could be deallocated when this function returns. so we need to make sure\n      // the operators with \\a NDArray are actually finished\n\n      // first for dummy key which represents original size of array, whose len is 0\n      CHECK_EQ(req_data.keys.size(), (size_t)2);\n      CHECK_EQ(req_data.lens.size(), (size_t)2);\n      CHECK_EQ(req_data.vals.size(), (size_t)req_data.lens[1]);\n\n      int original_size = DecodeKey(req_data.keys[0]);\n      int key           = DecodeKey(req_data.keys[1]);\n      auto& stored      = store_[key];\n\n      size_t ds[] = {(size_t)req_data.lens[1] / mshadow::mshadow_sizeof(type.dtype)};\n      mxnet::TShape dshape(ds, ds + 1);\n      TBlob recv_blob(reinterpret_cast<real_t*>(req_data.vals.data()), dshape, cpu::kDevMask);\n      NDArray recved = NDArray(recv_blob, 0);\n\n      NDArray decomp_buf = decomp_buf_[key];\n      dshape             = mxnet::TShape{(int64_t)original_size};\n\n      if (decomp_buf.is_none()) {\n        decomp_buf = NDArray(dshape, Context());\n      }\n\n      if (stored.is_none()) {\n        stored = NDArray(dshape, Context());\n        gradient_compression_->Dequantize(recved, &stored, 0);\n        server->Response(req_meta);\n        stored.WaitToRead();\n      } else if (sync_mode_) {\n        // synced push\n        auto& merged = update_buf_[key];\n        if (merged.merged.is_none()) {\n          merged.merged = NDArray(dshape, Context());\n        }\n        if (merged.request.size() == 0) {\n          gradient_compression_->Dequantize(recved, &merged.merged, 0);\n        } else {\n          gradient_compression_->Dequantize(recved, &decomp_buf, 0);\n          merged.merged += decomp_buf;\n        }\n        merged.request.push_back(req_meta);\n        ApplyUpdates(type, key, req_data, &merged, server);\n      } else {\n        // async push\n        gradient_compression_->Dequantize(recved, &decomp_buf, 0);\n        exec_.Exec([this, key, &decomp_buf, &stored]() {\n          CHECK(updater_);\n          updater_(key, decomp_buf, &stored);\n        });\n        server->Response(req_meta);\n        stored.WaitToRead();\n      }\n    } else {  // pull\n      CHECK_EQ(req_data.keys.size(), (size_t)1);\n      CHECK_EQ(req_data.lens.size(), (size_t)0);\n      int key = DecodeKey(req_data.keys[0]);\n      DefaultStorageResponse(type, key, req_meta, req_data, server);\n    }\n  }\n\n  void DataHandleDefault(const DataHandleType type,\n                         const ps::KVMeta& req_meta,\n                         const ps::KVPairs<char>& req_data,\n                         ps::KVServer<char>* server) {\n    // do some check\n    CHECK_EQ(req_data.keys.size(), (size_t)1);\n    if (req_meta.push) {\n      CHECK_EQ(req_data.lens.size(), (size_t)1);\n      CHECK_EQ(req_data.vals.size(), (size_t)req_data.lens[0]);\n    }\n    int key      = DecodeKey(req_data.keys[0]);\n    auto& stored = has_multi_precision_copy(type) ? store_realt_[key] : store_[key];\n    // there used several WaitToRead, this is because \\a recved's memory\n    // could be deallocated when this function returns. so we need to make sure\n    // the operators with \\a NDArray are actually finished\n    if (req_meta.push) {\n      size_t ds[] = {(size_t)req_data.lens[0] / mshadow::mshadow_sizeof(type.dtype)};\n      mxnet::TShape dshape(ds, ds + 1);\n      TBlob recv_blob;\n      MSHADOW_REAL_TYPE_SWITCH(type.dtype, DType, {\n        recv_blob = TBlob(reinterpret_cast<DType*>(req_data.vals.data()), dshape, cpu::kDevMask);\n      })\n      NDArray recved = NDArray(recv_blob, 0);\n      if (stored.is_none()) {\n        // initialization\n        stored = NDArray(dshape,\n                         Context(),\n                         false,\n                         has_multi_precision_copy(type) ? mshadow::kFloat32 : type.dtype);\n        CopyFromTo(recved, &stored, 0);\n        server->Response(req_meta);\n        if (has_multi_precision_copy(type)) {\n          auto& stored_dtype = store_[key];\n          stored_dtype       = NDArray(dshape, Context(), false, type.dtype);\n          CopyFromTo(stored, stored_dtype);\n          stored_dtype.WaitToRead();\n        }\n        stored.WaitToRead();\n      } else {\n        auto& updates = update_buf_[key];\n        if (sync_mode_ && updates.merged.is_none()) {\n          updates.merged = NDArray(dshape,\n                                   Context(),\n                                   false,\n                                   has_multi_precision_copy(type) ? mshadow::kFloat32 : type.dtype);\n        }\n        if (has_multi_precision_copy(type) && updates.temp_array.is_none()) {\n          updates.temp_array = NDArray(dshape, Context(), false, mshadow::kFloat32);\n        }\n        if (updates.request.empty()) {\n          if (sync_mode_) {\n            CopyFromTo(recved, updates.merged);\n          } else {\n            if (has_multi_precision_copy(type)) {\n              CopyFromTo(recved, updates.temp_array);\n            } else {\n              updates.temp_array = recved;\n            }\n          }\n        } else {\n          CHECK(sync_mode_);\n          if (has_multi_precision_copy(type)) {\n            CopyFromTo(recved, updates.temp_array);\n            updates.merged += updates.temp_array;\n          } else {\n            updates.merged += recved;\n          }\n        }\n        updates.request.push_back(req_meta);\n        ApplyUpdates(type, key, req_data, &updates, server);\n      }\n    } else {\n      DefaultStorageResponse(type, key, req_meta, req_data, server);\n    }\n  }\n\n  int DecodeKey(ps::Key key) {\n    auto kr = ps::Postoffice::Get()->GetServerKeyRanges()[ps::MyRank()];\n    return key - kr.begin();\n  }\n\n  /**\n   * \\brief user defined mode for push\n   */\n  bool sync_mode_;\n  KVStore::Controller controller_;\n  KVStore::Updater updater_;\n\n  /**\n   * \\brief store_ contains the value at kvstore for each key\n   */\n  std::unordered_map<int, NDArray> store_;\n  std::unordered_map<int, NDArray> store_realt_;\n\n  /**\n   * \\brief merge_buf_ is a buffer used if sync_mode is true. It represents\n   * values from different workers being merged. The store will be updated\n   * to this value when values from all workers are pushed into this buffer.\n   */\n  std::unordered_map<int, UpdateBuf> update_buf_;\n\n  /**\n   * \\brief decomp_buf_ is a buffer into which compressed values are\n   * decompressed before merging to the store. used when compress_!='none'\n   */\n  std::unordered_map<int, NDArray> decomp_buf_;\n\n  Executor exec_;\n  ps::KVServer<char>* ps_server_;\n\n  // whether to LOG verbose information\n  bool log_verbose_;\n\n  /*\n   * \\brief whether to use multi precision mode.\n   * in multi precision mode, all weights are stored as float32.\n   * any gradient received will be cast to float32 before accumulation and updating of weights.\n   */\n  bool multi_precision_;\n\n  /**\n   * \\brief gradient compression object.\n   * starts with none, used after SetGradientCompression sets the type\n   * currently there is no support for unsetting gradient compression\n   */\n  std::shared_ptr<kvstore::GradientCompression> gradient_compression_;\n};\n\n}  // namespace kvstore\n}  // namespace mxnet\n\n#endif  // MXNET_KVSTORE_KVSTORE_DIST_SERVER_H_\n"
  },
  {
    "path": "src/kvstore/kvstore_local.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/**\n * @file   kvstore_local.h\n * @brief  local implementation\n */\n#ifndef MXNET_KVSTORE_KVSTORE_LOCAL_H_\n#define MXNET_KVSTORE_KVSTORE_LOCAL_H_\n\n#include <mxnet/kvstore.h>\n#include <unordered_map>\n#include <bitset>\n#include <vector>\n#include <string>\n#include <utility>\n#include <functional>\n#include <algorithm>\n#include \"./comm.h\"\n#include \"./comm_tree.h\"\n#include \"./kvstore_utils.h\"\n#include \"../ndarray/ndarray_function.h\"\n#include \"../profiler/profiler.h\"\n\nnamespace mxnet {\nnamespace kvstore {\n/*!\n * \\brief Splits a string into smaller strings using char as delimiter\n * Example: \"a,b,c,,d\" is split into [\"a\",\"b\",\"c\",\"\",\"d\"]\n * \\param s string to split\n * \\param delim char to split string around\n * \\param result container for tokens extracted after splitting\n */\ntemplate <typename Out>\nvoid split(const std::string& s, const char delim, Out result) {\n  std::stringstream ss;\n  ss.str(s);\n  std::string item;\n  while (std::getline(ss, item, delim)) {\n    *(result++) = item;\n  }\n}\n\nenum KeyType { kUndefinedKey = -1, kStringKey, kIntKey };\n\n/**\n * \\brief store data in local machine\n */\nclass KVStoreLocal : public KVStore {\n public:\n  /*\n   * \\param use_device_comm\n   */\n  explicit KVStoreLocal(bool use_device_comm) : KVStore() {\n    if (use_device_comm) {\n      bool tree = dmlc::GetEnv(\"MXNET_KVSTORE_USETREE\", 0) & MXNET_USE_CUDA;\n      if (tree) {\n        comm_ = new CommDeviceTree();\n      } else {\n        comm_ = new CommDevice();\n      }\n    } else {\n      comm_ = new CommCPU();\n    }\n    pinned_ctx_           = comm_->pinned_ctx();\n    gradient_compression_ = std::make_shared<GradientCompression>();\n  }\n\n  virtual ~KVStoreLocal() {\n    delete comm_;\n    comm_ = nullptr;\n  }\n\n  void Init(const std::vector<int>& keys, const std::vector<NDArray>& values) override {\n    SetKeyType(kIntKey);\n    InitImpl(keys, values);\n  }\n\n  void Init(const std::vector<std::string>& str_keys, const std::vector<NDArray>& values) override {\n    SetKeyType(kStringKey);\n    std::vector<int> keys(str_keys.size());\n    for (size_t i = 0; i < str_keys.size(); ++i) {\n      auto& str_key = str_keys[i];\n      CHECK(str_key_dict_.find(str_key) == str_key_dict_.end())\n          << \"duplicate init of key \" << str_key;\n      auto key               = next_str_key_++;\n      str_key_dict_[str_key] = key;\n      // record reverse mapping from int to string\n      reverse_str_key_dict_[key] = str_key;\n      keys[i]                    = key;\n    }\n    InitImpl(keys, values);\n  }\n\n  void Push(const std::vector<int>& keys,\n            const std::vector<NDArray>& values,\n            int priority) override {\n    SetKeyType(kIntKey);\n    PushImpl(keys, values, priority);\n  }\n\n  void Pull(const std::vector<int>& keys,\n            const std::vector<NDArray*>& values,\n            int priority,\n            bool ignore_sparse) override {\n    SetKeyType(kIntKey);\n    PullImpl(keys, values, priority, ignore_sparse);\n  }\n\n  void Broadcast(const std::vector<int>& vkeys,\n                 const std::vector<int>& okeys,\n                 const std::vector<NDArray>& values,\n                 const std::vector<NDArray*>& outs,\n                 int priority) override {\n    SetKeyType(kIntKey);\n    BroadcastImpl(vkeys, okeys, values, outs, priority);\n  }\n\n  void PushPull(const std::vector<int>& vkeys,\n                const std::vector<int>& okeys,\n                const std::vector<NDArray>& values,\n                const std::vector<NDArray*>& outs,\n                int priority) override {\n    SetKeyType(kIntKey);\n    PushPullImpl(vkeys, okeys, values, outs, priority);\n  }\n\n  void PullRowSparse(const std::vector<int>& keys,\n                     const std::vector<std::pair<NDArray*, NDArray>>& val_rowids,\n                     int priority = 0) override {\n    SetKeyType(kIntKey);\n    PullRowSparseImpl(keys, val_rowids, priority);\n  }\n\n  void Push(const std::vector<std::string>& str_keys,\n            const std::vector<NDArray>& values,\n            int priority) override {\n    SetKeyType(kStringKey);\n    std::vector<int> keys(str_keys.size());\n    LookupKeys(str_keys, &keys);\n    PushImpl(keys, values, priority);\n  }\n\n  void Pull(const std::vector<std::string>& str_keys,\n            const std::vector<NDArray*>& values,\n            int priority,\n            bool ignore_sparse) override {\n    SetKeyType(kStringKey);\n    std::vector<int> keys(str_keys.size());\n    LookupKeys(str_keys, &keys);\n    PullImpl(keys, values, priority, ignore_sparse);\n  }\n\n  void Broadcast(const std::vector<std::string>& str_vkeys,\n                 const std::vector<std::string>& str_okeys,\n                 const std::vector<NDArray>& values,\n                 const std::vector<NDArray*>& outs,\n                 int priority) override {\n    SetKeyType(kStringKey);\n    std::vector<int> vkeys(str_vkeys.size());\n    std::vector<int> okeys(str_okeys.size());\n    for (size_t i = 0; i < str_vkeys.size(); ++i) {\n      auto& str_key = str_vkeys[i];\n      CHECK(str_key_dict_.find(str_key) == str_key_dict_.end())\n          << \"duplicate init of key \" << str_key;\n      auto key               = next_str_key_++;\n      str_key_dict_[str_key] = key;\n      // record reverse mapping from int to string\n      reverse_str_key_dict_[key] = str_key;\n      vkeys[i]                   = key;\n    }\n    LookupKeys(str_okeys, &okeys);\n    BroadcastImpl(vkeys, okeys, values, outs, priority);\n  }\n\n  void PushPull(const std::vector<std::string>& str_vkeys,\n                const std::vector<std::string>& str_okeys,\n                const std::vector<NDArray>& values,\n                const std::vector<NDArray*>& outs,\n                int priority) override {\n    SetKeyType(kStringKey);\n    std::vector<int> vkeys(str_vkeys.size());\n    std::vector<int> okeys(str_okeys.size());\n    LookupKeys(str_vkeys, &vkeys);\n    LookupKeys(str_okeys, &okeys);\n    PushPullImpl(vkeys, okeys, values, outs, priority);\n  }\n\n  void PullRowSparse(const std::vector<std::string>& str_keys,\n                     const std::vector<std::pair<NDArray*, NDArray>>& val_rowids,\n                     int priority = 0) override {\n    SetKeyType(kStringKey);\n    std::vector<int> keys(str_keys.size());\n    LookupKeys(str_keys, &keys);\n    PullRowSparseImpl(keys, val_rowids, priority);\n  }\n\n  void SetGradientCompression(\n      const std::vector<std::pair<std::string, std::string>>& kwargs) override {\n    gradient_compression_->SetParams(kwargs);\n  }\n\n private:\n  virtual void InitImpl(const std::vector<int>& keys, const std::vector<NDArray>& values) {\n    for (size_t i = 0; i < keys.size(); ++i) {\n      CHECK(local_.find(keys[i]) == local_.end())\n          << \"duplicate init of key \" << keys[i]\n          << \". Please double check if you called kv.init or kv.broadcast with this key \"\n          << \"multiple times\";\n      local_[keys[i]] = values[i].Copy(pinned_ctx_);\n      comm_->Init(keys[i], values[i].storage_type(), values[i].shape(), values[i].dtype());\n    }\n    comm_->SetGradientCompression(gradient_compression_);\n  }\n\n  virtual void PushImpl(const std::vector<int>& keys,\n                        const std::vector<NDArray>& values,\n                        int priority) {\n    std::vector<int> uniq_keys;\n    std::vector<std::vector<NDArray>> grouped_vals;\n    GroupKVPairsPush(keys, values, &uniq_keys, &grouped_vals, false);\n    for (size_t i = 0; i < uniq_keys.size(); ++i) {\n      int key               = uniq_keys[i];\n      const NDArray& merged = comm_->Reduce(key, grouped_vals[i], priority);\n      NDArray& local        = local_[key];\n      if (key_type_ == kStringKey) {\n        local.AssignStorageInfo(\n            profiler::ProfilerScope::Get()->GetCurrentProfilerScope() + \"kvstore:push:\",\n            reverse_str_key_dict_[key]);\n      } else {\n        local.AssignStorageInfo(\n            profiler::ProfilerScope::Get()->GetCurrentProfilerScope() + \"kvstore:push:\",\n            \"local_\" + std::to_string(key));\n      }\n      if (updater_ != nullptr) {\n        CHECK(!local.is_none()) << \"key \" << key << \" has not been inited\";\n        // if merged is on gpu, we may need copy weight from cpu to gpu\n        if (merged.ctx().dev_mask() != cpu::kDevMask && local.ctx().dev_mask() == cpu::kDevMask) {\n          local = local.Copy(merged.ctx());\n        }\n        // call the updater with string keys\n        // if string keys are used and str_updater_ is available\n        // otherwise fallback to updater_ which uses int key interface\n        if (key_type_ == kStringKey && str_updater_ != nullptr) {\n          // TODO(haibin) CHECK(str_updater_ != nullptr) if use_str_key\n          // after all language bindings picks up string interface changes\n          const std::string& str_key = reverse_str_key_dict_[key];\n          // TODO(haibin) avoid reverse key lookup if use_str_key\n          str_updater_(str_key, merged, &local);\n        } else {\n          updater_(key, merged, &local);\n        }\n      } else {\n        if (merged.storage_type() != local.storage_type()) {\n          local = merged.Copy(local.ctx());\n        } else {\n          local = merged;\n        }\n      }\n    }\n  }\n\n  virtual void PullImpl(const std::vector<int>& keys,\n                        const std::vector<NDArray*>& values,\n                        int priority,\n                        bool ignore_sparse) {\n    std::vector<int> uniq_keys;\n    std::vector<std::vector<NDArray*>> grouped_vals;\n    GroupKVPairsPull(keys, values, &uniq_keys, &grouped_vals, ignore_sparse);\n\n    for (size_t i = 0; i < uniq_keys.size(); ++i) {\n      int key              = uniq_keys[i];\n      const NDArray& local = local_[key];\n      CHECK(!local.is_none()) << \"key \" << key << \" has not been inited\";\n      comm_->Broadcast(key, local, grouped_vals[i], priority);\n      for (std::vector<NDArray*>::iterator iter = grouped_vals[i].begin();\n           iter != grouped_vals[i].end();\n           ++iter) {\n        if (key_type_ == kStringKey) {\n          (*iter)->AssignStorageInfo(\n              profiler::ProfilerScope::Get()->GetCurrentProfilerScope() + \"kvstore:pull:\",\n              reverse_str_key_dict_[key]);\n        } else {\n          (*iter)->AssignStorageInfo(\n              profiler::ProfilerScope::Get()->GetCurrentProfilerScope() + \"kvstore:pull:\",\n              \"grouped_vals_\" + std::to_string(key));\n        }\n      }\n    }\n  }\n\n  virtual void PullRowSparseImpl(const std::vector<int>& keys,\n                                 const std::vector<std::pair<NDArray*, NDArray>>& val_rowids,\n                                 int priority = 0) {\n    std::vector<int> uniq_keys;\n    std::vector<std::vector<std::pair<NDArray*, NDArray>>> grouped_val_rowids;\n    GroupKVPairsPullRsp(keys, val_rowids, &uniq_keys, &grouped_val_rowids, false);\n    for (size_t i = 0; i < uniq_keys.size(); ++i) {\n      int key              = uniq_keys[i];\n      const NDArray& local = local_[key];\n      CHECK(!local.is_none()) << \"key \" << key << \" has not been inited\";\n      CHECK_EQ(local.storage_type(), kRowSparseStorage)\n          << \"PullRowSparse expects row_sparse src NDArray\";\n      auto& target_val_rowids = grouped_val_rowids[i];\n      const size_t num_vals   = target_val_rowids.size();\n      for (size_t j = 0; j < num_vals; j++) {\n        auto& row_id                = target_val_rowids[j].second;\n        target_val_rowids[j].second = Unique(row_id, local.ctx(), 0);\n      }\n      comm_->BroadcastRowSparse(key, local, grouped_val_rowids[i], priority);\n    }\n  }\n\n protected:\n  KVStoreLocal() : KVStore() {}\n  /**\n   * \\brief set the key type of the kvstore if haven't already.\n   * If the key type is already defined, check if it matches the provided key type\n   */\n  void SetKeyType(const KeyType key_type) {\n    if (key_type_ == kUndefinedKey)\n      key_type_ = key_type;\n    CHECK_EQ(key_type_, key_type) << \"Mixed key types are not allowed\";\n  }\n\n  virtual void BroadcastImpl(const std::vector<int>& vkeys,\n                             const std::vector<int>& okeys,\n                             const std::vector<NDArray>& values,\n                             const std::vector<NDArray*>& outs,\n                             int priority) {\n    InitImpl(vkeys, values);\n    PullImpl(okeys, outs, priority, true);\n  }\n\n  virtual void PushPullImpl(const std::vector<int>& vkeys,\n                            const std::vector<int>& okeys,\n                            const std::vector<NDArray>& values,\n                            const std::vector<NDArray*>& outs,\n                            int priority) {\n    PushImpl(vkeys, values, priority);\n    PullImpl(okeys, outs, priority, true);\n  }\n\n  /**\n   * \\brief group values on keys for push\n   */\n  virtual void GroupKVPairsPush(const std::vector<int>& keys,\n                                const std::vector<NDArray>& values,\n                                std::vector<int>* uniq_keys,\n                                std::vector<std::vector<NDArray>>* grouped_vals,\n                                bool ignore_sparse) {\n    // check if the storage type of a value is valid\n    auto validator = [](const int key, const NDArray& nd, bool ignore_sparse) -> bool {\n      CHECK(!ignore_sparse) << \"Cannot ignore sparse arrays for push\";\n      auto stype = nd.storage_type();\n      // valid NDArray\n      if (stype == kDefaultStorage || stype == kRowSparseStorage)\n        return true;\n      // invalid NDArray, abort\n      LOG(FATAL) << \"Unexpected storage type detected during kvstore push: \" << stype;\n      return false;\n    };\n    GroupKVPairs(keys, values, uniq_keys, grouped_vals, validator, ignore_sparse);\n  }\n  /**\n   * \\brief group values on keys for pull\n   */\n  virtual void GroupKVPairsPull(const std::vector<int>& keys,\n                                const std::vector<NDArray*>& values,\n                                std::vector<int>* uniq_keys,\n                                std::vector<std::vector<NDArray*>>* grouped_vals,\n                                bool ignore_sparse) {\n    // check if the storage type of a value is valid\n    auto validator = [this](const int key, const NDArray* nd, bool ignore_sparse) -> bool {\n      // valid\n      if (nd->storage_type() == kDefaultStorage || !ignore_sparse)\n        return true;\n      // invalid, print warning messages once\n      if (this->warnings_printed_.find(key) == this->warnings_printed_.end()) {\n        LOG(INFO) << \"Warning: non-default weights detected during kvstore pull. \"\n                     \"This call has been ignored. Please make sure to use \"\n                     \"kv.row_sparse_pull() or module.prepare() with row_ids.\";\n        this->warnings_printed_.insert(key);\n      }\n      return false;\n    };\n    GroupKVPairs(keys, values, uniq_keys, grouped_vals, validator, ignore_sparse);\n  }\n\n  typedef std::pair<NDArray*, NDArray> RSPVal;\n  /**\n   * \\brief group values on keys for row_sparse_pull\n   */\n  virtual void GroupKVPairsPullRsp(const std::vector<int>& keys,\n                                   const std::vector<RSPVal>& values,\n                                   std::vector<int>* uniq_keys,\n                                   std::vector<std::vector<RSPVal>>* grouped_vals,\n                                   bool ignore_sparse) {\n    // check if the storage type of a value is valid\n    auto validator = [](const int key, const RSPVal& val_rowid, bool ignore_sparse) -> bool {\n      CHECK(!ignore_sparse) << \"Cannot ignore sparse arrays in row_sparse_pull\";\n      auto val_stype   = val_rowid.first->storage_type();\n      auto rowid_stype = val_rowid.second.storage_type();\n      // check storage types\n      CHECK_EQ(val_stype, kRowSparseStorage)\n          << \"Expected row_sparse storage type for \"\n          << \"row_sparse_pull values, but detected storage type \" << val_stype;\n      CHECK_EQ(rowid_stype, kDefaultStorage)\n          << \"Expected default storage type for \"\n          << \"row_sparse_pull rowids, but detected storage type \" << rowid_stype;\n      return true;\n    };\n    GroupKVPairs(keys, values, uniq_keys, grouped_vals, validator, ignore_sparse);\n  }\n\n  /**\n   * \\brief group values on keys with validation.\n   * A value `v` is not included in the result if is_valid(v) returns false.\n   */\n  template <typename V, typename FValidate>\n  void GroupKVPairs(const std::vector<int>& keys,\n                    const std::vector<V>& values,\n                    std::vector<int>* uniq_keys,\n                    std::vector<std::vector<V>>* grouped_vals,\n                    const FValidate& is_valid,\n                    bool ignore_sparse) {\n    CHECK_EQ(keys.size(), values.size());\n    // TODO(mli) check if already sorted as an optimization\n    using Idx = std::pair<int, int>;\n    std::vector<Idx> idx(keys.size());\n    for (size_t i = 0; i < keys.size(); ++i) {\n      idx[i].first  = keys[i];\n      idx[i].second = i;\n    }\n    std::sort(idx.begin(), idx.end(), [](const Idx& a, const Idx& b) { return a.first < b.first; });\n\n    int pre_key = idx[0].first - 1;\n    for (auto i : idx) {\n      if (is_valid(i.first, values[i.second], ignore_sparse)) {\n        if (i.first != pre_key) {\n          uniq_keys->push_back(i.first);\n          grouped_vals->push_back({values[i.second]});\n          pre_key = i.first;\n        } else {\n          grouped_vals->back().push_back(values[i.second]);\n        }\n      }\n    }\n  }\n\n  void LookupKeys(const std::vector<std::string>& str_keys, std::vector<int>* keys) {\n    for (size_t i = 0; i < str_keys.size(); ++i) {\n      auto& str_key = str_keys[i];\n      CHECK(str_key_dict_.find(str_key) != str_key_dict_.end())\n          << \"key \" << str_key << \" doesn't exist. Did you init?\";\n      keys->at(i) = str_key_dict_[str_key];\n    }\n  }\n\n  /*\n   * \\brief Compute the unique values in data and store them in ascending order\n   * in an int64_t row_sparse ndarray on ctx. The opeartion is async. The result\n   * row_sparse ndarray stores the unique values in out.data(). The aux_data()\n   * contains values that are not necessarily meaningful and should be ignored.\n   * \\param data the input data\n   * \\param ctx the target context\n   * \\param priority the priority of the operation\n   */\n  NDArray Unique(const NDArray& data, Context ctx, int priority) {\n    // create kRowSparseStorage output ndarray\n    const size_t num_elements = data.shape().Size();\n    NDArray out(kRowSparseStorage, mshadow::Shape2(num_elements, 1), ctx, true, mshadow::kInt64);\n    bool diff_ctx       = data.ctx() != ctx;\n    NDArray data_in_ctx = diff_ctx ? NDArray(data.shape(), ctx, true, data.dtype()) : data;\n    // if data == data_in_ctx, CopyFromTo is smart enough to skip the copy\n    CopyFromTo(data, &data_in_ctx, priority);\n    // GPU requires temp resources\n    bool is_gpu = out.ctx().dev_mask() == gpu::kDevMask;\n    Engine::Get()->PushAsync(\n        [=](RunContext rctx,\n            Engine::CallbackOnStart on_start,\n            Engine::CallbackOnComplete on_complete) {\n          on_start();\n          // copy data.data() to out.data()\n          out.CheckAndAlloc({mshadow::Shape1(num_elements)});\n          TBlob out_data = out.data();\n          NDArray workspace;\n          switch (out.ctx().dev_mask()) {\n            case cpu::kDevMask: {\n              mshadow::Stream<cpu>* s = rctx.get_stream<cpu>();\n              ndarray::Copy<cpu, cpu>(data_in_ctx.data(), &out_data, ctx, ctx, rctx);\n              UniqueImpl(&workspace, s, out);\n              break;\n            }\n#if MXNET_USE_CUDA\n            case gpu::kDevMask: {\n              mshadow::Stream<gpu>* s = rctx.get_stream<gpu>();\n              ndarray::Copy<gpu, gpu>(data_in_ctx.data(), &out_data, ctx, ctx, rctx);\n              UniqueImpl(&workspace, s, out);\n              break;\n            }\n#endif\n            default:\n              LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR;\n          }\n          on_complete();\n        },\n        out.ctx(),\n        {data_in_ctx.var()},\n        {out.var()},\n        is_gpu ? FnProperty::kGPUPrioritized : FnProperty::kCPUPrioritized,\n        priority,\n        \"KVStoreUnique\");\n    return out;\n  }\n\n  /// reducer and broadcaster\n  Comm* comm_;\n  /// pinned context\n  Context pinned_ctx_;\n  /// \\brief buffer for storing local values\n  std::unordered_map<int, NDArray> local_;\n  /// key mapping for string -> integer\n  std::unordered_map<std::string, int> str_key_dict_;\n  /// reverse key mapping for integer -> string\n  std::unordered_map<int, std::string> reverse_str_key_dict_;\n  /// the next available integer for string->int key mapping\n  int next_str_key_ = 0;\n  /// whether printed warning due to mismatch stype in each key\n  std::unordered_set<int> warnings_printed_;\n  /// whether int or string is used for keys\n  KeyType key_type_ = kUndefinedKey;\n};\n}  // namespace kvstore\n}  // namespace mxnet\n#endif  // MXNET_KVSTORE_KVSTORE_LOCAL_H_\n"
  },
  {
    "path": "src/kvstore/kvstore_nccl.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/**\n * @file   kvstore_nccl.h\n * @brief  NCCL implementation of KVStore\n */\n#ifndef MXNET_KVSTORE_KVSTORE_NCCL_H_\n#define MXNET_KVSTORE_KVSTORE_NCCL_H_\n\n#if MXNET_USE_NCCL\n\n#include <mxnet/kvstore.h>\n#include <nccl.h>\n#include <unordered_map>\n#include <bitset>\n#include <vector>\n#include <string>\n#include <utility>\n#include <functional>\n#include <algorithm>\n#include <tuple>\n#include \"./comm.h\"\n#include \"./kvstore_local.h\"\n#include \"../common/cuda/utils.h\"\n\n// NCCL v2 introduces NCCL_MAJOR macro for versioning,\n// so if there is no such macro defined in nccl.h\n// then it is NCCL v1\n#ifndef NCCL_MAJOR\n#define NCCL_MAJOR 1\n#endif\n\n#if NCCL_MAJOR == 1\n#define ncclGroupStart()\n#define ncclGroupEnd()\n#define ncclNumTypes nccl_NUM_TYPES\n#endif  // NCCL_MAJOR == 1\n\nnamespace mxnet {\nnamespace kvstore {\n\n/**\n * \\brief store data in local machine using NCCL\n */\nclass KVStoreNCCL : public KVStoreLocal {\n public:\n  KVStoreNCCL() : KVStoreLocal() {\n    // Due to aggregation, we do not use the Comm interface\n    comm_       = nullptr;\n    pinned_ctx_ = Context::CPUPinned(0);\n    inited_     = false;\n  }\n\n  virtual ~KVStoreNCCL() {\n    for (auto e : nccl_data_) {\n      cudaStreamDestroy(e.second.stream);\n      ncclCommDestroy(e.second.comm);\n    }\n  }\n\n private:\n  void InitImpl(const std::vector<int>& keys, const std::vector<NDArray>& values) override {\n    for (size_t i = 0; i < keys.size(); ++i) {\n      CHECK(local_.find(keys[i]) == local_.end()) << \"duplicate init of key \" << keys[i];\n      local_[keys[i]] = values[i].Copy(pinned_ctx_);\n      InitKey(keys[i], values[i].storage_type(), values[i].shape(), values[i].dtype());\n    }\n  }\n\n  void PushImpl(const std::vector<int>& keys,\n                const std::vector<NDArray>& values,\n                int priority) override {\n    std::vector<int> uniq_keys;\n    std::vector<std::vector<NDArray>> grouped_vals;\n    // nccl kvstore doesn't support sparse ndarray\n    GroupKVPairsHelper(keys, values, &uniq_keys, &grouped_vals, true);\n\n    std::vector<const NDArray*> merged_ptrs;\n    std::vector<NDArray*> local_ptrs;\n    bool nccl_called = false;\n\n    Reduce(uniq_keys, grouped_vals, priority, &merged_ptrs);\n\n    for (size_t i = 0; i < uniq_keys.size(); ++i) {\n      int key = uniq_keys[i];\n      if (grouped_vals[i].size() > 1) {\n        // We issued NCCL kernels, need to synchronize\n        nccl_called = true;\n      }\n      auto& merged   = *(merged_ptrs[i]);\n      NDArray& local = local_[key];\n      if (updater_ != nullptr) {\n        CHECK(!local.is_none()) << \"key \" << key << \" has not been inited\";\n        // if merged is on gpu, we may need copy weight from cpu to gpu\n        if (merged.ctx().dev_mask() != cpu::kDevMask && local.ctx().dev_mask() == cpu::kDevMask) {\n          local = local.Copy(merged.ctx());\n        }\n      }\n      local_ptrs.push_back(&local);\n    }\n\n    // Sync after all reductions in a group\n    if (nccl_called) {\n      CommSync(merged_ptrs, priority);\n    }\n\n    for (size_t i = 0; i < uniq_keys.size(); ++i) {\n      int key        = uniq_keys[i];\n      auto& merged   = *(merged_ptrs[i]);\n      NDArray& local = *(local_ptrs[i]);\n      if (updater_ != nullptr) {\n        // call the updater with string keys\n        // if string keys are used and str_updater_ is available\n        // otherwise fallback to updater_ which uses int key interface\n        if (key_type_ == kStringKey && str_updater_ != nullptr) {\n          // after all language bindings picks up string interface changes\n          const std::string& str_key = reverse_str_key_dict_[key];\n          str_updater_(str_key, merged, &local);\n        } else {\n          updater_(key, merged, &local);\n        }\n      } else {\n        local = merged;\n      }\n    }\n  }\n\n  void PullImpl(const std::vector<int>& keys,\n                const std::vector<NDArray*>& values,\n                int priority,\n                bool ignore_sparse) override {\n    CHECK(ignore_sparse) << \"nccl kvstore pull doesn't support ignore_sparse=False\";\n    std::vector<int> uniq_keys;\n    std::vector<std::vector<NDArray*>> grouped_vals;\n    GroupKVPairsHelper(keys, values, &uniq_keys, &grouped_vals, true);\n    std::vector<NDArray> locals;\n    bool nccl_called = false;\n\n    for (size_t i = 0; i < uniq_keys.size(); ++i) {\n      int key              = uniq_keys[i];\n      const NDArray& local = local_[key];\n      locals.push_back(local_[key]);\n      CHECK(!local.is_none()) << \"key \" << key << \" has not been inited\";\n      if (grouped_vals[i].size() > 1) {\n        // We issued NCCL kernels, need to synchronize\n        nccl_called = true;\n      }\n    }\n\n    Broadcast(uniq_keys, locals, grouped_vals, priority);\n    // Sync after all broadcasts in a group\n    if (nccl_called) {\n      const std::vector<const NDArray*> values_copy(values.begin(), values.end());\n      CommSync(values_copy, priority);\n    }\n  }\n\n  void PullRowSparseImpl(const std::vector<int>& keys,\n                         const std::vector<std::pair<NDArray*, NDArray>>& val_rowids,\n                         int priority = 0) override {\n    LOG(FATAL) << \"NCCL kvstore does not support sparse storage type\";\n  }\n\n  void SetGradientCompression(\n      const std::vector<std::pair<std::string, std::string>>& kwargs) override {\n    LOG(FATAL) << \"NCCL kvstore does not support gradient compression\";\n  }\n\n protected:\n  /**\n   * \\brief group values on keys\n   */\n  template <typename T>\n  void GroupKVPairsHelper(const std::vector<int>& keys,\n                          const std::vector<T>& values,\n                          std::vector<int>* uniq_keys,\n                          std::vector<std::vector<T>>* grouped_vals,\n                          bool ignore_sparse) {\n    // check if the storage type of a value is valid\n    auto validator = [this](const int key, const T nd, bool ignore_sparse) -> bool {\n      CHECK(ignore_sparse) << \"nccl kvstore pull doesn't support ignore_sparse=False\";\n      auto stype = ptr(nd)->storage_type();\n      // valid NDArray\n      if (stype == kDefaultStorage)\n        return true;\n      // invalid NDArray, abort\n      LOG(FATAL) << \"NCCL kvstore does not support sparse storage type\";\n      return false;\n    };\n    GroupKVPairs(keys, values, uniq_keys, grouped_vals, validator, ignore_sparse);\n  }\n\n private:\n  // Aggregated reductions\n  virtual void Reduce(const std::vector<int> keys,\n                      const std::vector<std::vector<NDArray>>& srcs,\n                      int priority,\n                      std::vector<const NDArray*>* merged_ptrs) {\n    std::vector<size_t> root_ids(keys.size());\n    std::vector<NDArray> reduces(keys.size());\n    merged_ptrs->resize(keys.size());\n    std::vector<Engine::VarHandle> const_vars;\n    std::vector<Engine::VarHandle> mutate_vars;\n\n    for (size_t k = 0; k < keys.size(); ++k) {\n      auto& key     = keys[k];\n      auto& src     = srcs[k];\n      auto& root_id = root_ids[k];\n\n      // avoid extra copy for single device, but it may bring problems for\n      // abnormal usage of kvstore\n      if (src.size() == 1) {\n        (*merged_ptrs)[k] = &src[0];\n        continue;\n      }\n\n      if (!inited_) {\n        std::vector<Context> devs;\n        for (const auto& a : src) {\n          devs.push_back(a.ctx());\n        }\n        InitNCCL(devs);\n        InitMergeBuffer(devs);\n      }\n\n      // Check whether we got the same set of devices\n      std::vector<int> dev_ids;\n      for (auto e : src) {\n        dev_ids.push_back(e.ctx().dev_id);\n      }\n      std::sort(dev_ids.begin(), dev_ids.end());\n      CHECK(device_ids_ == dev_ids) << \"NCCL KVStore supports only single set of devices\";\n\n      auto& buf = merge_buf_[key];\n      int root  = buf.merged.ctx().dev_id;\n      root_id   = FindRootId(src, root);\n\n      auto& reduce      = buf.merged;\n      (*merged_ptrs)[k] = &reduce;\n      // Need to pass NDArrays by value to the engine\n      reduces[k] = reduce;\n\n      for (size_t i = 0; i < src.size(); ++i) {\n        const_vars.push_back(src[i].var());\n      }\n      mutate_vars.push_back(reduce.var());\n    }\n\n    Engine::Get()->PushSync(\n        [srcs, reduces, root_ids, this](RunContext rctx) {\n          std::lock_guard<std::mutex> l(Storage::Get()->GetMutex(Context::kGPU));\n#if (NCCL_MAJOR > 2 || (NCCL_MAJOR == 2 && NCCL_MINOR > 1))\n          ncclGroupStart();\n#endif\n          for (size_t k = 0; k < srcs.size(); ++k) {\n            auto& src     = srcs[k];\n            auto& root_id = root_ids[k];\n            auto& reduce  = reduces[k];\n            if (src.size() <= 1) {\n              continue;\n            }\n            int root = nccl_data_[src[root_id].ctx().dev_id].rank;\n            ncclGroupStart();\n            for (size_t i = 0; i < src.size(); ++i) {\n              NCCLEntry cur = nccl_data_[src[i].ctx().dev_id];\n              if (i == root_id) {\n                MSHADOW_TYPE_SWITCH(src[i].dtype(),\n                                    DType,\n                                    ncclReduce(src[i].data().dptr<DType>(),\n                                               reduce.data().dptr<DType>(),\n                                               src[i].shape().Size(),\n                                               GetNCCLType(src[i].dtype()),\n                                               ncclSum,\n                                               root,\n                                               cur.comm,\n                                               cur.stream););\n              } else {\n                MSHADOW_TYPE_SWITCH(src[i].dtype(),\n                                    DType,\n                                    ncclReduce(src[i].data().dptr<DType>(),\n                                               nullptr,\n                                               src[i].shape().Size(),\n                                               GetNCCLType(src[i].dtype()),\n                                               ncclSum,\n                                               root,\n                                               cur.comm,\n                                               cur.stream););\n              }\n            }\n            ncclGroupEnd();\n          }\n#if (NCCL_MAJOR > 2 || (NCCL_MAJOR == 2 && NCCL_MINOR > 1))\n          ncclGroupEnd();\n#endif\n        },\n        Context::CPU(),\n        const_vars,\n        mutate_vars,\n        FnProperty::kCPUPrioritized,\n        priority,\n        \"KVStoreReduce\");\n  }\n\n  virtual void Broadcast(const std::vector<int> keys,\n                         const std::vector<NDArray>& srcs,\n                         const std::vector<std::vector<NDArray*>>& dsts,\n                         int priority) {\n    std::vector<size_t> root_ids(keys.size());\n    std::vector<Engine::VarHandle> const_vars;\n    std::vector<Engine::VarHandle> mutable_vars;\n\n    for (size_t k = 0; k < keys.size(); ++k) {\n      auto& key     = keys[k];\n      auto& src     = srcs[k];\n      auto& dst     = dsts[k];\n      auto& root_id = root_ids[k];\n\n      if (!inited_) {\n        // copy to a random device first\n        int dev_id = key % dst.size();\n        CopyFromTo(src, *dst[dev_id], priority);\n        for (size_t i = 0; i < dst.size(); ++i) {\n          if (i != static_cast<size_t>(dev_id)) {\n            CopyFromTo(*dst[dev_id], *dst[i], priority);\n          }\n        }\n      } else {\n        auto& buf = merge_buf_[key];\n        int root  = src.ctx().dev_id;\n        assert(root == buf.merged.ctx().dev_id);\n        root_id = FindRootId(dst, root);\n\n        // Check whether we got the same set of devices\n        std::vector<int> dev_ids;\n        for (size_t i = 0; i < dst.size(); ++i) {\n          auto& bcast = (i == root_id) ? src : *dst[i];\n          dev_ids.push_back(bcast.ctx().dev_id);\n        }\n        std::sort(dev_ids.begin(), dev_ids.end());\n        CHECK(device_ids_ == dev_ids) << \"NCCL KVStore supports only single set of devices\";\n\n        // On root perform simple copy to the output\n        CopyFromTo(src, *dst[root_id], priority);\n        for (size_t i = 0; i < dst.size(); ++i) {\n          if (i != root_id)\n            mutable_vars.push_back(dst[i]->var());\n        }\n        const_vars.push_back(src.var());\n      }\n    }\n\n    // If not yet inited, then all work is already scheduled\n    if (!inited_) {\n      return;\n    }\n\n    // We need to capture NDArrays by value\n    // in order to push to the engine\n    std::vector<std::vector<NDArray>> broadcasts(dsts.size());\n    for (size_t i = 0; i < dsts.size(); ++i) {\n      auto& broadcast = broadcasts[i];\n      broadcast.resize(dsts[i].size());\n      for (size_t j = 0; j < dsts[i].size(); ++j) {\n        broadcast[j] = *(dsts[i][j]);\n      }\n    }\n\n    Engine::Get()->PushSync(\n        [srcs, broadcasts, root_ids, this](RunContext rctx) {\n          std::lock_guard<std::mutex> l(Storage::Get()->GetMutex(Context::kGPU));\n#if (NCCL_MAJOR > 2 || (NCCL_MAJOR == 2 && NCCL_MINOR > 1))\n          ncclGroupStart();\n#endif\n          for (size_t k = 0; k < srcs.size(); ++k) {\n            auto& src     = srcs[k];\n            auto& dst     = broadcasts[k];\n            auto& root_id = root_ids[k];\n            if (dst.size() <= 1) {\n              continue;\n            }\n\n            int root = nccl_data_[src.ctx().dev_id].rank;\n            ncclGroupStart();\n            for (size_t i = 0; i < dst.size(); ++i) {\n              auto& bcast   = (i == root_id) ? src : dst[i];\n              NCCLEntry cur = nccl_data_[bcast.ctx().dev_id];\n              MSHADOW_TYPE_SWITCH(bcast.dtype(),\n                                  DType,\n                                  ncclBcast(bcast.data().dptr<DType>(),\n                                            bcast.shape().Size(),\n                                            GetNCCLType(bcast.dtype()),\n                                            root,\n                                            cur.comm,\n                                            cur.stream););\n            }\n            ncclGroupEnd();\n          }\n#if (NCCL_MAJOR > 2 || (NCCL_MAJOR == 2 && NCCL_MINOR > 1))\n          ncclGroupEnd();\n#endif\n        },\n        Context::CPU(),\n        const_vars,\n        mutable_vars,\n        FnProperty::kCPUPrioritized,\n        priority,\n        \"KVStoreBCast\");\n  }\n\n  // Function that waits for NCCL collective to complete\n  template <typename T>\n  void CommSync(const std::vector<T>& dst, int priority) {\n    std::vector<Engine::VarHandle> mutate_vars;\n    for (size_t i = 0; i < dst.size(); ++i) {\n      mutate_vars.push_back(ptr(dst[i])->var());\n    }\n    Engine::Get()->PushSync(\n        [this](RunContext rctx) {\n          mxnet::common::cuda::DeviceStore device_store;\n          for (auto cur : nccl_data_) {\n            device_store.SetDevice(cur.second.dev_id);\n            CUDA_CALL(cudaStreamSynchronize(cur.second.stream));\n          }\n        },\n        Context::CPU(),\n        {},\n        mutate_vars,\n        FnProperty::kCPUPrioritized,\n        priority,\n        \"KVStoreStreamSync\");\n  }\n\n  // Initialize single key\n  void InitKey(int key,\n               const NDArrayStorageType stype,\n               const mxnet::TShape& shape,\n               int dtype = mshadow::kFloat32) {\n    if (stype == kDefaultStorage) {\n      key_attrs_.push_back(std::make_tuple(key, shape, dtype));\n    } else {\n      LOG(FATAL) << \"NCCL KVStore does not support sparse storage type\";\n    }\n  }\n\n  ncclDataType_t GetNCCLType(int dtype) {\n    switch (dtype) {\n      case mshadow::kFloat32:\n        return ncclFloat;\n      case mshadow::kFloat16:\n        return ncclHalf;\n      case mshadow::kFloat64:\n        return ncclDouble;\n      case mshadow::kUint8:\n        return ncclChar;\n      case mshadow::kInt32:\n        return ncclInt;\n      case mshadow::kInt64:\n        return ncclInt64;\n      default:\n        LOG(FATAL) << \"Unknown type passed to NCCL KVStore\";\n    }\n    return ncclNumTypes;\n  }\n\n  void InitNCCL(const std::vector<Context>& devs) {\n    for (size_t i = 0; i < devs.size(); ++i) {\n      device_ids_.push_back(devs[i].dev_id);\n    }\n    std::sort(device_ids_.begin(), device_ids_.end());\n    std::lock_guard<std::mutex> l(Storage::Get()->GetMutex(Context::kGPU));\n    std::vector<ncclComm_t> comms(devs.size());\n    ncclCommInitAll(&(comms[0]), devs.size(), &(device_ids_[0]));\n    mxnet::common::cuda::DeviceStore device_store;\n    for (size_t i = 0; i < devs.size(); ++i) {\n      NCCLEntry e;\n      e.dev_id = device_ids_[i];\n      e.comm   = comms[i];\n      e.rank   = i;\n      device_store.SetDevice(e.dev_id);\n      cudaStreamCreate(&(e.stream));\n      nccl_data_[device_ids_[i]] = e;\n    }\n  }\n\n  using KeyAttrs = std::tuple<int, mxnet::TShape, int>;\n  void InitMergeBuffer(const std::vector<Context>& devs) {\n    for (size_t i = 0; i < key_attrs_.size(); ++i) {\n      int key         = std::get<0>(key_attrs_[i]);\n      mxnet::TShape s = std::get<1>(key_attrs_[i]);\n      int type        = std::get<2>(key_attrs_[i]);\n      auto& buf       = merge_buf_[key];\n      // always use devs[0] as root\n      buf.merged = NDArray(s, devs[0], false, type);\n    }\n    inited_ = true;\n  }\n\n  // Functions that enable templates to work on both references\n  // and pointers\n  template <typename T>\n  const T* ptr(const T& obj) {\n    return &obj;\n  }\n\n  template <typename T>\n  const T* ptr(T* obj) {\n    return obj;\n  }\n\n  // Find which element of the vector\n  // corresponds to root dev_id\n  template <typename T>\n  size_t FindRootId(const std::vector<T>& vec, int root) {\n    size_t root_id = -1;\n    for (size_t i = 0; i < vec.size(); ++i) {\n      if (ptr(vec[i])->ctx().dev_id == root) {\n        root_id = i;\n        break;\n      }\n    }\n    return root_id;\n  }\n\n  std::vector<KeyAttrs> key_attrs_;\n  /// \\brief temporal space for pushing and pulling\n  struct BufferEntry {\n    /// \\brief the merged value\n    NDArray merged;\n  };\n  struct NCCLEntry {\n    /// \\brief device ID\n    int dev_id;\n    /// \\brief NCCL commmunicator\n    ncclComm_t comm;\n    /// \\brief NCCL rank\n    int rank;\n    /// \\brief GPU stream to use with NCCL\n    cudaStream_t stream;\n  };\n  std::unordered_map<int, BufferEntry> merge_buf_;\n  std::unordered_map<int, NCCLEntry> nccl_data_;\n  bool inited_;\n  // \\brief devices used with this KVStore\n  std::vector<int> device_ids_;\n};\n}  // namespace kvstore\n}  // namespace mxnet\n#endif  // MXNET_USE_NCCL\n#endif  // MXNET_KVSTORE_KVSTORE_NCCL_H_\n"
  },
  {
    "path": "src/kvstore/kvstore_utils.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file kvstore_utils.cc\n * \\brief cpu implementation of util functions\n */\n\n#include \"./kvstore_utils.h\"\n#include \"../common/utils.h\"\n\nnamespace mxnet {\nnamespace kvstore {\n\ntemplate <>\nvoid UniqueImpl<cpu>(NDArray* workspace, mshadow::Stream<cpu>* s, const NDArray& out) {\n  const size_t num_elements = out.shape().Size();\n  CHECK_EQ(out.storage_type(), kRowSparseStorage) << \"row_sparse NDArray is expected\";\n  MSHADOW_IDX_TYPE_SWITCH(out.dtype(), IType, {\n    IType* dptr = out.data().dptr<IType>();\n    common::ParallelSort(\n        dptr, dptr + num_elements, engine::OpenMP::Get()->GetRecommendedOMPThreadCount());\n    const size_t num_selected_out = std::unique(dptr, dptr + num_elements) - dptr;\n    // set the shape of data/aux_data according to the number of unique values\n    out.set_aux_shape(rowsparse::kIdx, mshadow::Shape1(num_selected_out));\n  });\n}\n\n}  // namespace kvstore\n}  // namespace mxnet\n"
  },
  {
    "path": "src/kvstore/kvstore_utils.cu",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file kvstore_utils.cu\n * \\brief gpu implementation of util functions\n */\n#if defined(_MSC_VER) && __CUDACC_VER_MAJOR__ == 8 && __CUDACC_VER_BUILD__ != 44\n// Many CUDA 8 compilers other than V8.0.44 crash on Windows\n#pragma warning(\"Potential crash on CUDA compiler detected. Switching sorting from CUB to Thrust\")\n#define SORT_WITH_THRUST\n#include <thrust/device_ptr.h>\n#include <thrust/sort.h>\n#include <thrust/system/cuda/execution_policy.h>\n#else\n#undef SORT_WITH_THRUST\n#endif\n#include \"./kvstore_utils.h\"\n#include <cub/cub.cuh>\n#include <mxnet/resource.h>\n#include \"../common/utils.h\"\n\nnamespace mxnet {\nnamespace kvstore {\n\ntemplate <typename IType>\nsize_t UniqueImplGPU(NDArray* workspace,\n                     mshadow::Stream<gpu>* s,\n                     IType* dptr,\n                     const size_t size,\n                     Context ctx) {\n  // estimate unique temp space. The first byte is reserved to store the number\n  // of unique values selected\n  const size_t num_selected_bytes = sizeof(size_t);\n  size_t unique_temp_bytes        = 0;\n  size_t* null_ptr                = nullptr;\n  size_t* null_dptr               = nullptr;\n  cudaStream_t stream             = mshadow::Stream<gpu>::GetStream(s);\n  cub::DeviceSelect::Unique(\n      nullptr, unique_temp_bytes, null_dptr, null_dptr, null_ptr, size, stream);\n  // estimate sort temp space\n  const size_t sort_output_bytes = size * sizeof(IType);\n  size_t sort_temp_bytes         = 0;\n#ifndef SORT_WITH_THRUST\n  // The least-significant bit index (inclusive) needed for key comparison\n  const int begin_bit = 0;\n  // The most-significant bit index (exclusive) needed for key comparison\n  const int end_bit = sizeof(IType) * 8;\n  cub::DeviceRadixSort::SortKeys(\n      nullptr, sort_temp_bytes, null_dptr, null_dptr, size, begin_bit, end_bit, stream);\n#else\n  // sort_temp_bytes remains 0 because thrust request memory by itself\n#endif\n  // request temp storage\n  const size_t total_workspace =\n      num_selected_bytes + sort_output_bytes + std::max(sort_temp_bytes, unique_temp_bytes);\n  *workspace           = NDArray(mshadow::Shape1((total_workspace + 3) / 4), ctx, false);\n  char* workspace_dptr = reinterpret_cast<char*>(workspace->data().dptr_);\n  // temp space layout: num_selected_ptr, sort_output_bytes, unique/sort_temp_storage\n  size_t* num_selected_ptr = reinterpret_cast<size_t*>(workspace_dptr);\n  IType* sort_output_ptr   = reinterpret_cast<IType*>(workspace_dptr + num_selected_bytes);\n  void* temp_storage = static_cast<void*>(workspace_dptr + num_selected_bytes + sort_output_bytes);\n  // execute the sort kernel\n#ifndef SORT_WITH_THRUST\n  cub::DeviceRadixSort::SortKeys(\n      temp_storage, sort_temp_bytes, dptr, sort_output_ptr, size, begin_bit, end_bit, stream);\n#else\n  thrust::sort(thrust::cuda::par.on(stream), dptr, dptr + size, thrust::greater<IType>());\n  CUDA_CALL(\n      cudaMemcpyAsync(sort_output_ptr, dptr, sort_output_bytes, cudaMemcpyDeviceToDevice, stream));\n#endif\n  // execute unique kernel\n  cub::DeviceSelect::Unique(\n      temp_storage, unique_temp_bytes, sort_output_ptr, dptr, num_selected_ptr, size, stream);\n  // retrieve num selected unique values\n  size_t num_selected_out = 0;\n  CUDA_CALL(cudaMemcpyAsync(\n      &num_selected_out, num_selected_ptr, num_selected_bytes, cudaMemcpyDeviceToHost, stream));\n  CUDA_CALL(cudaStreamSynchronize(stream));\n  return num_selected_out;\n}\n\ntemplate <>\nvoid UniqueImpl<gpu>(NDArray* workspace, mshadow::Stream<gpu>* s, const NDArray& out) {\n  const size_t num_elements = out.shape().Size();\n  CHECK_EQ(out.storage_type(), kRowSparseStorage) << \"row_sparse NDArray is expected\";\n  MSHADOW_IDX_TYPE_SWITCH(out.dtype(), IType, {\n    IType* dptr             = out.data().dptr<IType>();\n    size_t num_selected_out = UniqueImplGPU(workspace, s, dptr, num_elements, out.ctx());\n    // set the shape of data/aux_data according to the number of unique values\n    out.set_aux_shape(rowsparse::kIdx, mshadow::Shape1(num_selected_out));\n  });\n}\n\n}  // namespace kvstore\n}  // namespace mxnet\n"
  },
  {
    "path": "src/kvstore/kvstore_utils.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file kvstore_utils.h\n * \\brief Basic utilility functions.\n */\n#ifndef MXNET_KVSTORE_KVSTORE_UTILS_H_\n#define MXNET_KVSTORE_KVSTORE_UTILS_H_\n\n#include <dmlc/logging.h>\n#include <mxnet/ndarray.h>\n#include <mxnet/resource.h>\n#include <utility>\n#include <vector>\n\nnamespace mxnet {\nnamespace kvstore {\n\n/*!\n * \\brief compute unique and sorted values in a row_sparse ndarray.\n * \\param workspace Temp workspace for computation. Its a pointer to a\n              NDArray placeholder to make sure the NDArray is not free'd\n              during execution.\n * \\param s   Stream\n * \\param out Input and output ndarray. The ndarray stores the\n *            unique elements in out.data().\n */\ntemplate <typename xpu>\nvoid UniqueImpl(NDArray* workspace, mshadow::Stream<xpu>* s, const NDArray& out);\n}  // namespace kvstore\n}  // namespace mxnet\n\n#endif  // MXNET_KVSTORE_KVSTORE_UTILS_H_\n"
  },
  {
    "path": "src/lib_api.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file lib_api.cc\n * \\brief APIs to interact with libraries\n * This API specifies function prototypes to\n * register custom ops, partitioner, and passes\n * for library authors\n * See example/extension/lib_custom_op/README.md\n * See example/extension/lib_subgraph/README.md\n * See example/extension/lib_pass/README.md\n */\n\n#include \"mxnet/lib_api.h\"\n\nmxnet::ext::MXerrorMsgs* mxnet::ext::MXerrorMsgs::get() {\n  static MXerrorMsgs inst;\n  return &inst;\n}\n\nstd::stringstream& mxnet::ext::MXerrorMsgs::add(const char* file, int line) {\n  messages.emplace_back();\n  messages.back() << file << \"[\" << line << \"]: \";\n  return messages.back();\n}\n\nint mxnet::ext::MXerrorMsgs::size() {\n  return messages.size();\n}\n\nconst std::string* mxnet::ext::MXerrorMsgs::get(int idx) {\n  return new std::string(messages.at(idx).str());\n}\n\nmxnet::ext::MXContext::MXContext() : dev_type(\"error\"), dev_id(-1) {}\n\nmxnet::ext::MXContext::MXContext(std::string dev_type_, int dev_id_)\n    : dev_type(std::move(dev_type_)), dev_id(dev_id_) {}\n\nmxnet::ext::MXContext::MXContext(const char* dev_type_, int dev_id_)\n    : dev_type(dev_type_), dev_id(dev_id_) {}\n\nmxnet::ext::MXContext mxnet::ext::MXContext::CPU() {\n  return MXContext(\"cpu\", 0);\n}\n\nmxnet::ext::MXContext mxnet::ext::MXContext::GPU() {\n  return MXContext(\"gpu\", 0);\n}\n\nmxnet::ext::MXContext mxnet::ext::MXContext::CPU(int dev_id) {\n  return MXContext(\"cpu\", dev_id);\n}\n\nmxnet::ext::MXContext mxnet::ext::MXContext::GPU(int dev_id) {\n  return MXContext(\"gpu\", dev_id);\n}\n\nvoid mxnet::ext::MXSparse::set(void* data_ptr,\n                               const int64_t* dims,\n                               int ndims,\n                               void* idx,\n                               int64_t num_idx,\n                               void* idx_ptr,\n                               int64_t num_idx_ptr) {\n  data = data_ptr;\n  // If CSR, num of non-zero elemets is num_idx,\n  // If row sparse, num of elements is num_idx * width.\n  data_len = num_idx;\n  if (!idx_ptr) {\n    for (int i = 1; i < ndims; ++i)\n      data_len *= dims[i];\n  }\n\n  indices     = reinterpret_cast<int64_t*>(idx);\n  indices_len = num_idx;\n\n  if (idx_ptr) {\n    indptr     = reinterpret_cast<int64_t*>(idx_ptr);\n    indptr_len = num_idx_ptr;\n  }\n}\n\nmxnet::ext::MXTensor::MXTensor()\n    : data_ptr(nullptr), dtype(kUNSET), verID(0), stype(kDefaultStorage) {}\nmxnet::ext::MXTensor::MXTensor(const MXTensor& oth)\n    : data_ptr(oth.data_ptr),\n      shape(oth.shape),\n      dtype(oth.dtype),\n      verID(oth.verID),\n      ctx(oth.ctx),\n      stype(oth.stype) {\n  setDLTensor();\n}\n\nmxnet::ext::MXTensor::MXTensor(void* data_ptr,\n                               std::vector<int64_t> shape,\n                               MXDType dtype,\n                               size_t vID,\n                               MXContext mx_ctx,\n                               MXStorageType stype)\n    : data_ptr(data_ptr),\n      shape(std::move(shape)),\n      dtype(dtype),\n      verID(vID),\n      ctx(std::move(mx_ctx)),\n      stype(stype) {\n  setDLTensor();\n}\n\nvoid mxnet::ext::MXTensor::setTensor(void* dptr,\n                                     MXDType type,\n                                     const int64_t* dims,\n                                     int ndims,\n                                     size_t vID,\n                                     MXContext mx_ctx,\n                                     MXStorageType storage_type) {\n  data_ptr = dptr;\n  dtype    = type;\n  verID    = vID;\n  ctx      = mx_ctx;\n  stype    = storage_type;\n  shape.clear();\n  for (int j = 0; j < ndims; j++) {\n    shape.push_back(dims[j]);\n  }\n  setDLTensor();\n}\n\nvoid mxnet::ext::MXTensor::setDLTensor() {\n  dltensor.data          = data_ptr;\n  dltensor.ndim          = shape.size();\n  dltensor.shape         = const_cast<int64_t*>(shape.data());\n  dltensor.strides       = nullptr;\n  dltensor.byte_offset   = 0;\n  dltensor.dtype.lanes   = 1;\n  dltensor.ctx.device_id = ctx.dev_id;\n  if (ctx.dev_type == \"cpu\")\n    dltensor.ctx.device_type = kDLCPU;\n  else if (ctx.dev_type == \"gpu\")\n    dltensor.ctx.device_type = kDLGPU;\n  else if (ctx.dev_type == \"opencl\")\n    dltensor.ctx.device_type = kDLOpenCL;\n  else if (ctx.dev_type == \"vulcan\")\n    dltensor.ctx.device_type = kDLVulkan;\n  else if (ctx.dev_type == \"metal\")\n    dltensor.ctx.device_type = kDLMetal;\n  else if (ctx.dev_type == \"vpi\")\n    dltensor.ctx.device_type = kDLVPI;\n  else if (ctx.dev_type == \"rocm\")\n    dltensor.ctx.device_type = kDLROCM;\n  else\n    dltensor.ctx.device_type = kDLExtDev;\n  switch (dtype) {\n    case kFloat32:\n      dltensor.dtype.code = kDLFloat;\n      dltensor.dtype.bits = 32;\n      break;\n    case kFloat64:\n      dltensor.dtype.code = kDLFloat;\n      dltensor.dtype.bits = 64;\n      break;\n    case kFloat16:\n      dltensor.dtype.code = kDLFloat;\n      dltensor.dtype.bits = 16;\n      break;\n    case kUint8:\n      dltensor.dtype.code = kDLUInt;\n      dltensor.dtype.bits = 8;\n      break;\n    case kInt32:\n      dltensor.dtype.code = kDLInt;\n      dltensor.dtype.bits = 32;\n      break;\n    case kInt8:\n      dltensor.dtype.code = kDLInt;\n      dltensor.dtype.bits = 8;\n      break;\n    case kInt64:\n      dltensor.dtype.code = kDLInt;\n      dltensor.dtype.bits = 64;\n      break;\n    default:\n      dltensor.dtype.code = 0;\n      dltensor.dtype.bits = 0;\n      throw std::runtime_error(\n          \"Error! Invalid dtype flag: \" + std::to_string(static_cast<int>(dtype)) +\n          \" when constructing MXTensor\");\n  }\n}\n\nint64_t mxnet::ext::MXTensor::size() const {\n  int64_t size = 1;\n  for (auto& s : shape)\n    size *= s;\n  return size;\n}\n\nbool mxnet::ext::MXTensor::isSame(const MXTensor& oth) const {\n  return data_ptr == oth.data_ptr && dtype == oth.dtype && verID == oth.verID &&\n         ctx.dev_type == oth.ctx.dev_type && ctx.dev_id == oth.ctx.dev_id && shape == oth.shape &&\n         stype == oth.stype;\n}\n\nmxnet::ext::PassResource::PassResource(std::unordered_map<std::string, MXTensor>* new_args,\n                                       std::unordered_map<std::string, MXTensor>* new_aux,\n                                       nd_malloc_t nd_malloc,\n                                       const void* nd_alloc)\n    : new_args_(new_args), new_aux_(new_aux), nd_malloc_(nd_malloc), nd_alloc_(nd_alloc) {}\n\nmxnet::ext::MXTensor* mxnet::ext::PassResource::alloc_arg(const std::string& name,\n                                                          const std::vector<int64_t>& shapes,\n                                                          const mxnet::ext::MXContext& ctx,\n                                                          mxnet::ext::MXDType dtype) const {\n  void* data;\n  nd_malloc_(nd_alloc_,\n             shapes.data(),\n             shapes.size(),\n             ctx.dev_type.c_str(),\n             ctx.dev_id,\n             dtype,\n             name.c_str(),\n             1,\n             &data);\n  MXTensor tensor(data, shapes, dtype, 0, ctx, kDefaultStorage);\n  (*new_args_)[name] = tensor;\n  return &(new_args_->at(name));\n}\n\nmxnet::ext::MXTensor* mxnet::ext::PassResource::alloc_aux(const std::string& name,\n                                                          const std::vector<int64_t>& shapes,\n                                                          const mxnet::ext::MXContext& ctx,\n                                                          mxnet::ext::MXDType dtype) const {\n  void* data;\n  nd_malloc_(nd_alloc_,\n             shapes.data(),\n             shapes.size(),\n             ctx.dev_type.c_str(),\n             ctx.dev_id,\n             dtype,\n             name.c_str(),\n             0,\n             &data);\n  MXTensor tensor(data, shapes, dtype, 0, ctx, kDefaultStorage);\n  (*new_aux_)[name] = tensor;\n  return &(new_aux_->at(name));\n}\n\nmxnet::ext::OpResource::OpResource(xpu_malloc_t cpu_malloc_fp,\n                                   void* cpu_alloc_fp,\n                                   xpu_malloc_t gpu_malloc_fp,\n                                   void* gpu_alloc_fp,\n                                   void* stream,\n                                   sparse_malloc_t sparse_malloc_fp,\n                                   void* sparse_alloc_fp,\n                                   void* rng_cpu_states,\n                                   void* rng_gpu_states)\n    : cpu_malloc(cpu_malloc_fp),\n      gpu_malloc(gpu_malloc_fp),\n      cpu_alloc(cpu_alloc_fp),\n      gpu_alloc(gpu_alloc_fp),\n      cuda_stream(stream),\n      sparse_malloc(sparse_malloc_fp),\n      sparse_alloc(sparse_alloc_fp),\n      rand_cpu_states(rng_cpu_states),\n      rand_gpu_states(rng_gpu_states) {}\n\nvoid* mxnet::ext::OpResource::alloc_cpu(int size) const {\n  return cpu_malloc(cpu_alloc, size);\n}\n\nvoid* mxnet::ext::OpResource::alloc_gpu(int size) const {\n  return gpu_malloc(gpu_alloc, size);\n}\n\nvoid mxnet::ext::OpResource::alloc_sparse(mxnet::ext::MXSparse* sparse,\n                                          int index,\n                                          int indices_len,\n                                          int indptr_len) const {\n  sparse_malloc(sparse_alloc,\n                index,\n                indices_len,\n                indptr_len,\n                &(sparse->data),\n                &(sparse->indices),\n                &(sparse->indptr));\n}\n\nmxnet::ext::mx_cpu_rand_t* mxnet::ext::OpResource::get_cpu_rand_states() const {\n  return static_cast<mx_cpu_rand_t*>(rand_cpu_states);\n}\n\nstd::string mxnet::ext::getShapeAt(const std::string& shape, unsigned index) {\n  int idx = 1;  // start at 1 to skip the first square bracket [\n  // find the beginning of the output shape for the particular output index\n  for (unsigned x = 0; x < index; x++)\n    idx = shape.find('[', idx + 1);\n  int stop = shape.find(']', idx);  // find stop index for this output shape\n  // add this shape to the list\n  return shape.substr(idx, stop - idx + 1);\n}\n\nstd::string mxnet::ext::getDtypeAt(const std::string& dtype, unsigned index) {\n  // find the beginning of the output dtype for the particular output index\n  int idx = 0;\n  for (unsigned x = 0; x < index; x++)\n    idx = dtype.find(',', idx + 1);\n  int stop = dtype.find(',', idx + 1);  // find stop index for this output dtype\n  if (stop == -1)\n    stop = dtype.find(']', idx + 1);\n  return dtype.substr(idx + 1, stop - idx - 1);\n}\n\nmxnet::ext::JsonVal::JsonVal() : type(ERR), num(-1), str(\"\") {}\nmxnet::ext::JsonVal::JsonVal(mxnet::ext::JsonType t) : type(t), num(-1), str(\"\") {}\nmxnet::ext::JsonVal::JsonVal(std::string s) : type(STR), num(-1), str(std::move(s)) {}\nmxnet::ext::JsonVal::JsonVal(int n) : type(NUM), num(n), str(std::to_string(n)) {}\nmxnet::ext::JsonVal::JsonVal(JsonType t, int n, std::string s)\n    : type(t), num(n), str(std::move(s)) {}\n\nbool mxnet::ext::JsonVal::operator<(const mxnet::ext::JsonVal& o) const {\n  // for string JSON objects compare the string\n  if (type == STR)\n    return type == o.type && str < o.str;\n  // for number JSON objects compare the number\n  if (type == NUM)\n    return type == o.type && num < o.num;\n  // for list JSON objects, compare the size of list, and then each object in the list\n  if (type == LIST) {\n    if (list.size() != o.list.size())\n      return false;\n    for (unsigned int i = 0; i < list.size(); i++)\n      if (list[i] < o.list[i])\n        return false;  // if we find an object that doesnt match return\n    return true;       // all objects in lists matched\n  }\n  // for map JSON objects, compare the size of map, and then each key/value in the maps\n  if (type == MAP) {\n    if (map.size() != o.map.size())\n      return false;\n    for (auto& item : map) {\n      // if one map is missing a key in another return\n      if (o.map.find(item.first) == o.map.end())\n        return false;\n      if (item.second < o.map.at(item.first))\n        return false;\n    }\n    return true;\n  }\n  return type < o.type;\n}\n\nstd::string mxnet::ext::JsonVal::dump() const {\n  std::string ret;\n  switch (type) {\n    case ERR:\n      ret = \"json(Error)\";\n      break;\n    case STR:\n      ret = \"\\\"\" + str + \"\\\"\";\n      break;\n    case NUM:\n      ret = str;\n      break;\n    case LIST:\n      ret = \"[\";\n      for (unsigned i = 0; i < list.size(); i++) {\n        auto& item = list[i];\n        ret += item.dump();\n        if (i < list.size() - 1)\n          ret += \",\";\n      }\n      ret += \"]\";\n      break;\n    case MAP:\n      ret          = \"{\";\n      unsigned cnt = 0;\n      for (auto& item : map) {\n        ret += item.first.dump() + \" : \" + item.second.dump();\n        if (cnt++ < map.size() - 1)\n          ret += \",\";\n      }\n      ret += \"}\";\n      break;\n  }\n  return ret;\n}\n\nmxnet::ext::JsonVal mxnet::ext::JsonVal::parse(const std::string& json) {\n  unsigned int idx = 0;\n  return JsonVal::parse(json, &idx);\n}\n\nmxnet::ext::JsonVal mxnet::ext::JsonVal::parse_string(const std::string& json, unsigned int* idx) {\n  JsonVal ret(STR);\n  while (*idx < json.size()) {\n    if (json[*idx] == '\"' &&\n        (ret.str.size() == 0 || (ret.str.size() > 0 && ret.str.back() != '\\\\'))) {\n      ++(*idx);\n      return ret;\n    } else {\n      ret.str += json[*idx];\n      ++(*idx);\n    }\n  }\n  MX_ERROR_MSG << \"Error! Unable to parse string: '\" << json.substr(*idx) << \"'\" << std::endl;\n  return JsonVal();\n}\n\nmxnet::ext::JsonVal mxnet::ext::JsonVal::parse_num(const std::string& json, unsigned int* idx) {\n  JsonVal ret(NUM);\n  while (*idx < json.size()) {\n    if (json[*idx] >= '0' && json[*idx] <= '9') {\n      ret.str += json[*idx];\n      ++(*idx);\n    } else {\n      break;\n    }\n  }\n  ret.num = std::stoi(ret.str);\n  return ret;\n}\n\nmxnet::ext::JsonVal mxnet::ext::JsonVal::parse_list(const std::string& json, unsigned int* idx) {\n  JsonVal ret(LIST);\n  while (*idx < json.size()) {\n    if (json[*idx] == ']') {\n      ++(*idx);\n      return ret;\n    } else {\n      JsonVal item = JsonVal::parse(json, idx);\n      if (item.type != ERR)\n        ret.list.push_back(item);\n    }\n  }\n  MX_ERROR_MSG << \"Error! Unable to parse list: '\" << json.substr(*idx) << \"'\" << std::endl;\n  return JsonVal();\n}\n\nmxnet::ext::JsonVal mxnet::ext::JsonVal::parse_map(const std::string& json, unsigned int* idx) {\n  JsonVal ret(MAP), key;\n  while (*idx < json.size()) {\n    if (json[*idx] == '}') {\n      ++(*idx);\n      return ret;\n    } else {\n      JsonVal item = JsonVal::parse(json, idx);\n      if (key.type == ERR) {\n        key = item;\n      } else {\n        ret.map[key] = item;\n        key.type     = ERR;\n      }\n    }\n  }\n  MX_ERROR_MSG << \"Error! Unable to parse map: '\" << json.substr(*idx) << \"'\" << std::endl;\n  return mxnet::ext::JsonVal();\n}\n\nmxnet::ext::JsonVal mxnet::ext::JsonVal::parse(const std::string& json, unsigned int* idx) {\n  JsonVal ret;\n  while (*idx < json.size()) {\n    if (json[*idx] == '\"') {\n      ++(*idx);\n      ret = JsonVal::parse_string(json, idx);\n    } else if (json[*idx] >= '0' && json[*idx] <= '9') {\n      ret = JsonVal::parse_num(json, idx);\n    } else if (json[*idx] == '[') {\n      ++(*idx);\n      ret = JsonVal::parse_list(json, idx);\n    } else if (json[*idx] == '{') {\n      ++(*idx);\n      ret = JsonVal::parse_map(json, idx);\n    } else if (json[*idx] == ']' || json[*idx] == '}') {\n      return ret;\n    }\n    if (ret.type != ERR)\n      return ret;\n    ++(*idx);\n  }\n  return ret;\n}\n\nstd::string mxnet::ext::JsonVal::toString() const {\n  std::string ret;\n  switch (type) {\n    case ERR:\n      ret = \"json(Error)\";\n      break;\n    case STR:\n      ret = \"json(STR:\" + str + \")\";\n      break;\n    case NUM:\n      ret = \"json(INT:\" + str + \")\";\n      break;\n    case LIST:\n      ret = \"json(LIST:[\";\n      for (auto& item : list)\n        ret += item.toString() + \",\";\n      ret += \"])\";\n      break;\n    case MAP:\n      ret = \"json(MAP:{\";\n      for (auto& item : map)\n        ret += item.first.toString() + \" : \" + item.second.toString() + \",\";\n      ret += \"})\";\n      break;\n  }\n  return ret;\n}\n\nmxnet::ext::Node::Node() {\n  tensor = nullptr;\n}\n\nvoid mxnet::ext::Node::_setPassResource(mxnet::ext::PassResource* res_) {\n  res = res_;\n}\n\nvoid mxnet::ext::Node::alloc_arg(const std::vector<int64_t>& shapes,\n                                 const mxnet::ext::MXContext& ctx,\n                                 mxnet::ext::MXDType dtype) {\n  if (!res)\n    throw std::runtime_error(\"Node not initialized. Cannot use alloc_arg outside of graph passes.\");\n  tensor = res->alloc_arg(name, shapes, ctx, dtype);\n}\n\nvoid mxnet::ext::Node::alloc_aux(const std::vector<int64_t>& shapes,\n                                 const mxnet::ext::MXContext& ctx,\n                                 mxnet::ext::MXDType dtype) {\n  if (!res)\n    throw std::runtime_error(\"Node not initialized. Cannot use alloc_aux outside of graph passes.\");\n  tensor = res->alloc_aux(name, shapes, ctx, dtype);\n}\n\nmxnet::ext::Graph::Graph() : res(nullptr) {}\n\nmxnet::ext::Graph::~Graph() {\n  for (auto& node : nodes)\n    delete node;\n}\n\nmxnet::ext::Graph* mxnet::ext::Graph::fromString(const std::string& json) {\n  JsonVal val = JsonVal::parse(json);\n  return fromJson(val);\n}\n\nmxnet::ext::Graph* mxnet::ext::Graph::fromJson(mxnet::ext::JsonVal val) {\n  // get nodes list\n  JsonVal nodes = val.map[JsonVal(\"nodes\")];\n  Graph* g      = new Graph();\n\n  std::map<int, Node*> nodeMap;\n  // loop over nodes\n  for (int i = 0; i < nodes.list.size(); i++) {\n    Node* n = new Node();\n    g->nodes.push_back(n);\n    JsonVal node = nodes.list[i];\n\n    // set the op info\n    n->op   = node.map[JsonVal(\"op\")].str;\n    n->name = node.map[JsonVal(\"name\")].str;\n\n    // if op is null it is an input to the graph\n    if (n->op.compare(\"null\") == 0)\n      g->inputs.push_back(n);\n\n    // set attrs\n    JsonVal attributes = node.map[JsonVal(\"attrs\")];\n    for (auto& kv : attributes.map) {\n      n->attrs[kv.first.str] = kv.second.str;\n    }\n\n    // set subgraphs, parsing each into a graph\n    if (node.map.count(JsonVal(\"subgraphs\")) > 0) {\n      JsonVal subgraphs = node.map[JsonVal(\"subgraphs\")];\n      for (auto& subgraph : subgraphs.list) {\n        n->subgraphs.push_back(fromJson(subgraph));\n      }\n    }\n\n    // set node inputs\n    JsonVal node_inputs = node.map[JsonVal(\"inputs\")];\n    n->inputs.resize(node_inputs.list.size());\n    for (int j = 0; j < node_inputs.list.size(); j++) {\n      JsonVal input    = node_inputs.list[j];\n      NodeEntry& entry = n->inputs[j];\n      // get pointer to other node\n      entry.node = nodeMap[input.list[0].num];\n      // get the other node's output index\n      entry.entry = input.list[1].num;\n      // set other nodes output as connected to this node\n      entry.node->outputs.push_back({n, j});\n    }\n    nodeMap[i] = n;\n  }\n\n  // set graph level outputs\n  JsonVal& heads = val.map[JsonVal(\"heads\")];\n  g->outputs.resize(heads.list.size());\n  for (int i = 0; i < heads.list.size(); i++) {\n    JsonVal head        = heads.list[i];\n    g->outputs[i].node  = nodeMap[head.list[0].num];\n    g->outputs[i].entry = head.list[1].num;\n  }\n\n  // add all attributes to the graph\n  for (auto& kv : val.map) {\n    if (kv.first.str.compare(\"nodes\") != 0 && kv.first.str.compare(\"heads\") != 0 &&\n        kv.first.str.compare(\"node_row_ptr\") != 0 && kv.first.str.compare(\"arg_nodes\") != 0) {\n      g->attrs[kv.first.str] = kv.second;\n    }\n  }\n  return g;\n}\n\n/* \\brief convert graph object back to JSON object */\nmxnet::ext::JsonVal mxnet::ext::Graph::toJson() const {\n  // top level object is a map\n  JsonVal val(MAP);\n\n  // add attributes\n  for (auto& kv : attrs) {\n    val.map[JsonVal(kv.first)] = kv.second;\n  }\n\n  // sort graph nodes in topological order, create mapping of node to index\n  std::map<Node*, int> nodeMap;\n  std::vector<Node*> sorted = topological_sort();\n  // nodes are in reverse topological order in the vector (back is first)\n  // so loop from end to front over the vector 'sorted'\n  for (int i = sorted.size() - 1; i >= 0; i--) {\n    nodeMap[sorted[i]] = sorted.size() - 1 - i;\n  }\n\n  // create node_row_ptr entry\n  val.map[JsonVal(\"node_row_ptr\")] = JsonVal(LIST);\n  JsonVal& node_row_ptr            = val.map[JsonVal(\"node_row_ptr\")];\n  for (int i = 0; i < nodes.size(); i++)\n    node_row_ptr.list.emplace_back(i);\n\n  // add all input nodes\n  val.map[JsonVal(\"arg_nodes\")] = JsonVal(LIST);\n  JsonVal& arg_nodes            = val.map[JsonVal(\"arg_nodes\")];\n  for (auto& input : inputs)\n    arg_nodes.list.emplace_back(nodeMap[input]);\n\n  // add all output nodes\n  val.map[JsonVal(\"heads\")] = JsonVal(LIST);\n  JsonVal& heads            = val.map[JsonVal(\"heads\")];\n  for (int i = 0; i < outputs.size(); i++) {\n    heads.list.emplace_back(LIST);\n    JsonVal& out = heads.list[i];\n    out.list.emplace_back(nodeMap[outputs[i].node]);\n    out.list.emplace_back(outputs[i].entry);\n    out.list.emplace_back(0);\n  }\n\n  // add all graph nodes\n  val.map[JsonVal(\"nodes\")] = JsonVal(LIST);\n  JsonVal& nodes_           = val.map[JsonVal(\"nodes\")];\n  for (int i = sorted.size() - 1; i >= 0; i--) {\n    // each node is a map\n    nodes_.list.emplace_back(MAP);\n    Node* n     = sorted[i];\n    JsonVal& n_ = nodes_.list[nodes_.list.size() - 1];\n\n    n_.map[JsonVal(\"op\")]     = JsonVal(n->op);\n    n_.map[JsonVal(\"name\")]   = JsonVal(n->name);\n    n_.map[JsonVal(\"inputs\")] = JsonVal(LIST);\n\n    // add inputs for this node\n    JsonVal& inputs_ = n_.map[JsonVal(\"inputs\")];\n    for (int j = 0; j < n->inputs.size(); j++) {\n      inputs_.list.emplace_back(LIST);\n      NodeEntry& entry = n->inputs[j];\n      JsonVal& in      = inputs_.list[j];\n      in.list.emplace_back(nodeMap[entry.node]);\n      in.list.emplace_back(entry.entry);\n      in.list.emplace_back(0);\n    }\n\n    // add subgraphs for this node, convert each back to JSON\n    if (n->subgraphs.size() > 0) {\n      n_.map[JsonVal(\"subgraphs\")] = JsonVal(LIST);\n      JsonVal& subgraphs_          = n_.map[JsonVal(\"subgraphs\")];\n      for (Graph* subgraph : n->subgraphs) {\n        subgraphs_.list.push_back(subgraph->toJson());\n      }\n    }\n\n    // add attributes for this node\n    n_.map[JsonVal(\"attrs\")] = JsonVal(MAP);\n    JsonVal& attrs_          = n_.map[JsonVal(\"attrs\")];\n    for (auto& kv : n->attrs) {\n      attrs_.map[JsonVal(kv.first)] = JsonVal(kv.second);\n    }\n  }\n  return val;\n}\n\n/* \\brief convert graph object to JSON string */\nstd::string mxnet::ext::Graph::toString() const {\n  return toJson().dump();\n}\n\n/* \\brief visits a node \"n\" */\nvoid mxnet::ext::Graph::_dfs_util(Node* n,\n                                  std::unordered_set<mxnet::ext::Node*>* to_visit,\n                                  std::function<void(mxnet::ext::Node*)> handler) const {\n  to_visit->erase(n);  // remove node now that we're visiting it\n  for (NodeEntry& e : n->outputs) {\n    Node* o = e.node;\n    if (to_visit->count(o) != 0) {\n      _dfs_util(o, to_visit, handler);  // visit neighbor\n    }\n  }\n  handler(n);  // post-order visit this node\n}\n\n/* \\brief post-order DFS graph traversal */\nvoid mxnet::ext::Graph::DFS(std::function<void(Node*)> handler) const {\n  std::unordered_set<Node*> to_visit;\n  // put all nodes in set to visit\n  for (auto& n : nodes)\n    to_visit.insert(n);\n  // visit all inputs first\n  for (auto& i : inputs)\n    if (to_visit.count(i) != 0)\n      _dfs_util(i, &to_visit, handler);\n  // visit any nodes left\n  while (to_visit.size() > 0)\n    _dfs_util(*(to_visit.begin()), &to_visit, handler);\n}\n\n/* \\brief sort graph nodes in topological order */\nstd::vector<mxnet::ext::Node*> mxnet::ext::Graph::topological_sort() const {\n  std::vector<mxnet::ext::Node*> sorted;\n  auto handler = [&](mxnet::ext::Node* n) {\n    sorted.push_back(n);  // when visiting each node, add it in order to the vector\n  };\n  DFS(handler);\n  return sorted;\n}\n\n/* \\brief print out graph details */\nvoid mxnet::ext::Graph::print(int indent) const {\n  std::string space = \"\";\n  for (int i = 0; i < indent; i++)\n    space += \" \";\n\n  std::cout << space << \"########### Graph #############\" << std::endl;\n  std::cout << space << \"attributes: \" << std::endl;\n  for (auto& kv : attrs)\n    std::cout << space << \"\\t\" << kv.first << \" : \" << kv.second.str << std::endl;\n  std::cout << space << \"inputs: \" << inputs.size() << std::endl;\n  std::cout << space << \"outputs: \" << outputs.size() << std::endl;\n  std::cout << space << \"nodes: \" << nodes.size() << std::endl;\n  std::vector<mxnet::ext::Node*> sorted = topological_sort();\n  // loop over each node and print out its inputs/outputs\n  for (int i = sorted.size() - 1; i >= 0; i--) {\n    std::cout << space << \"Node: \" << sorted[i]->name << std::endl;\n    for (auto& input : sorted[i]->inputs) {\n      std::cout << space << \"\\tInput: \" << input.node->name << \" \" << input.entry << std::endl;\n    }\n    for (auto& output : sorted[i]->outputs) {\n      std::cout << space << \"\\tOutput: \" << output.node->name << \" \" << output.entry << std::endl;\n    }\n    if (sorted[i]->subgraphs.size() > 0) {\n      for (auto& subgraph : sorted[i]->subgraphs) {\n        std::cout << space << \"\\tSubgraph:\" << std::endl;\n        subgraph->print(indent + 2);\n      }\n    }\n  }\n  std::cout << space << \"###############################\" << std::endl;\n}\n\n/* \\brief add a new node to this graph */\nmxnet::ext::Node* mxnet::ext::Graph::addNode(const std::string& name, const std::string& op) {\n  Node* n = new Node();\n  nodes.push_back(n);\n  n->name = name;\n  n->op   = op;\n  if (res)\n    n->_setPassResource(res);\n  return n;\n}\n\n/* \\brief get node at index in graph */\nmxnet::ext::Node* mxnet::ext::Graph::getNode(size_t idx) {\n  return nodes[idx];\n}\n\n/* \\brief get const node at index in const graph */\nconst mxnet::ext::Node* mxnet::ext::Graph::getNode(size_t idx) const {\n  return nodes.at(idx);\n}\n\n/* \\brief get attribute on graph */\nconst mxnet::ext::JsonVal& mxnet::ext::Graph::getAttr(const std::string& key) const {\n  return attrs.at(key);\n}\n\n/* \\brief get number of nodes in the graph */\nsize_t mxnet::ext::Graph::size() const {\n  return nodes.size();\n}\n\n// internally set passResource to enable tensor allocation for graph passes\nvoid mxnet::ext::Graph::_setPassResource(PassResource* res_) {\n  res = res_;\n  // set passResource for each node\n  for (Node* node : nodes) {\n    node->_setPassResource(res);\n  }\n}\n\n// internally set arg/aux params when available\nvoid mxnet::ext::Graph::_setParams(std::unordered_map<std::string, mxnet::ext::MXTensor>* args,\n                                   std::unordered_map<std::string, mxnet::ext::MXTensor>* aux) {\n  // set params for each input node\n  for (Node* node : inputs) {\n    std::string name = node->name;\n    if (node->attrs.count(\"isArg\") > 0 && node->attrs[\"isArg\"].compare(\"True\") == 0)\n      // mapping name back to original node name from subgraph input name\n      name = node->attrs[\"argName\"];\n    if (args->count(name) > 0)\n      node->tensor = &args->at(name);\n    else if (aux->count(name) > 0)\n      node->tensor = &aux->at(name);\n  }\n}\n\nmxnet::ext::CustomOp::CustomOp(const char* op_name)\n    : name(op_name),\n      parse_attrs(nullptr),\n      infer_type(nullptr),\n      infer_storage_type(nullptr),\n      infer_shape(nullptr),\n      mutate_inputs(nullptr),\n      isSGop(false) {}\n\nmxnet::ext::CustomOp& mxnet::ext::CustomOp::setForward(mxnet::ext::fcomp_t fcomp, const char* ctx) {\n  if (forward_ctx_map.count(ctx) > 0)\n    raiseDuplicateContextError();\n  forward_ctx_map[ctx] = fcomp;\n  return *this;\n}\n\nmxnet::ext::CustomOp& mxnet::ext::CustomOp::setBackward(mxnet::ext::fcomp_t fgrad,\n                                                        const char* ctx) {\n  if (backward_ctx_map.count(ctx) > 0)\n    raiseDuplicateContextError();\n  backward_ctx_map[ctx] = fgrad;\n  return *this;\n}\n\nmxnet::ext::CustomOp& mxnet::ext::CustomOp::setParseAttrs(mxnet::ext::parseAttrs_t func) {\n  parse_attrs = func;\n  return *this;\n}\n\nmxnet::ext::CustomOp& mxnet::ext::CustomOp::setInferType(mxnet::ext::inferType_t func) {\n  infer_type = func;\n  return *this;\n}\n\nmxnet::ext::CustomOp& mxnet::ext::CustomOp::setInferSType(mxnet::ext::inferSType_t func) {\n  infer_storage_type = func;\n  return *this;\n}\n\nmxnet::ext::CustomOp& mxnet::ext::CustomOp::setInferShape(mxnet::ext::inferShape_t func) {\n  infer_shape = func;\n  return *this;\n}\n\nmxnet::ext::CustomOp& mxnet::ext::CustomOp::setMutateInputs(mxnet::ext::mutateInputs_t func) {\n  mutate_inputs = func;\n  return *this;\n}\n\nmxnet::ext::CustomOp& mxnet::ext::CustomOp::setCreateOpState(mxnet::ext::createOpState_t func,\n                                                             const char* ctx) {\n  if (create_op_ctx_map.count(ctx) > 0)\n    raiseDuplicateContextError();\n  create_op_ctx_map[ctx] = func;\n  return *this;\n}\n\nmxnet::ext::CustomOp& mxnet::ext::CustomOp::setIsSubgraphOp() {\n  isSGop = true;\n  return *this;\n}\n\nvoid mxnet::ext::CustomOp::mapToVector() {\n  for (auto kv : forward_ctx_map) {\n    forward_ctx_cstr.push_back(kv.first);\n    forward_fp.push_back(kv.second);\n  }\n  for (auto kv : backward_ctx_map) {\n    backward_ctx_cstr.push_back(kv.first);\n    backward_fp.push_back(kv.second);\n  }\n  for (auto kv : create_op_ctx_map) {\n    create_op_ctx_cstr.push_back(kv.first);\n    create_op_fp.push_back(kv.second);\n  }\n}\n\nvoid mxnet::ext::CustomOp::raiseDuplicateContextError() {\n  std::string op_name_str(name);\n  throw std::runtime_error(\n      \"Error! Error! Cannot register multiple functions under same context for operator '\" +\n      op_name_str + \"'\");\n}\n\nmxnet::ext::CustomStatefulOp::CustomStatefulOp() : ignore_warn(false), created(false) {}\nmxnet::ext::CustomStatefulOp::~CustomStatefulOp() = default;\n\nmxnet::ext::CustomStatefulOpWrapper::~CustomStatefulOpWrapper() {\n  destroy_(instance);\n}\n\nmxnet::ext::CustomPass::CustomPass() : name(\"ERROR\") {}\nmxnet::ext::CustomPass::CustomPass(const char* pass_name) : name(pass_name) {}\nmxnet::ext::CustomPass& mxnet::ext::CustomPass::setBody(graphPass_t fn) {\n  pass = fn;\n  return *this;\n}\n\nmxnet::ext::CustomPartitioner::CustomPartitioner() : name(\"ERROR\") {}\nmxnet::ext::CustomPartitioner::CustomPartitioner(const char* backend_name) : name(backend_name) {}\n\nmxnet::ext::CustomPartitioner& mxnet::ext::CustomPartitioner::addStrategy(const char* prop_name,\n                                                                          const char* sg_name) {\n  strategies.push_back(prop_name);\n  op_names.push_back(sg_name);\n  return *this;\n}\n\nmxnet::ext::CustomPartitioner& mxnet::ext::CustomPartitioner::setSupportedOps(\n    const char* prop_name,\n    mxnet::ext::supportedOps_t fn) {\n  supported_map[std::string(prop_name)] = fn;\n  return *this;\n}\n\nmxnet::ext::CustomPartitioner& mxnet::ext::CustomPartitioner::setCreateSelector(\n    const char* prop_name,\n    mxnet::ext::createSelector_t fn) {\n  selector_map[std::string(prop_name)] = fn;\n  return *this;\n}\n\nmxnet::ext::CustomPartitioner& mxnet::ext::CustomPartitioner::setReviewSubgraph(\n    const char* prop_name,\n    mxnet::ext::reviewSubgraph_t fn) {\n  review_map[std::string(prop_name)] = fn;\n  return *this;\n}\n\nmxnet::ext::supportedOps_t mxnet::ext::CustomPartitioner::getSupportedOps(int stg_id) {\n  std::string prop(strategies[stg_id]);\n  if (supported_map.count(prop) > 0)\n    return supported_map[prop];\n  else\n    return nullptr;\n}\n\nmxnet::ext::createSelector_t mxnet::ext::CustomPartitioner::getCreateSelector(int stg_id) {\n  std::string prop(strategies[stg_id]);\n  if (selector_map.count(prop) > 0)\n    return selector_map[prop];\n  else\n    return nullptr;\n}\n\nmxnet::ext::reviewSubgraph_t mxnet::ext::CustomPartitioner::getReviewSubgraph(int stg_id) {\n  std::string prop(strategies[stg_id]);\n  if (review_map.count(prop) > 0)\n    return review_map[prop];\n  else\n    return nullptr;\n}\n\n/*! \\brief returns MXNet library version */\nMX_INT_RET _opVersion() {\n  return MX_LIBRARY_VERSION;\n}\n\n/*! \\brief returns number of ops registered in this library */\nMX_INT_RET _opRegSize() {\n  return mxnet::ext::Registry<mxnet::ext::CustomOp>::get()->size();\n}\n\n/*! \\brief returns operator registration at specified index */\nMX_VOID_RET _opRegGet(int idx,\n                      const char** name,\n                      int* isSGop,\n                      const char*** forward_ctx,\n                      mxnet::ext::fcomp_t** forward_fp,\n                      int* forward_count,\n                      const char*** backward_ctx,\n                      mxnet::ext::fcomp_t** backward_fp,\n                      int* backward_count,\n                      const char*** create_op_ctx,\n                      mxnet::ext::createOpState_t** create_op_fp,\n                      int* create_op_count,\n                      mxnet::ext::parseAttrs_t* parse,\n                      mxnet::ext::inferType_t* type,\n                      mxnet::ext::inferSType_t* stype,\n                      mxnet::ext::inferShape_t* shape,\n                      mxnet::ext::mutateInputs_t* mutate) {\n  mxnet::ext::CustomOp& op = mxnet::ext::Registry<mxnet::ext::CustomOp>::get()->get(idx);\n  *name                    = op.name;\n  *parse                   = op.parse_attrs;\n  *type                    = op.infer_type;\n  *stype                   = op.infer_storage_type;\n  *shape                   = op.infer_shape;\n  *mutate                  = op.mutate_inputs;\n  *isSGop                  = op.isSGop;\n  op.mapToVector();\n  *forward_ctx     = op.forward_ctx_cstr.data();\n  *forward_fp      = op.forward_fp.data();\n  *forward_count   = op.forward_fp.size();\n  *backward_ctx    = op.backward_ctx_cstr.data();\n  *backward_fp     = op.backward_fp.data();\n  *backward_count  = op.backward_fp.size();\n  *create_op_ctx   = op.create_op_ctx_cstr.data();\n  *create_op_fp    = op.create_op_fp.data();\n  *create_op_count = op.create_op_fp.size();\n}\n\n/*! \\brief calls free from the external library for library allocated arrays */\nMX_VOID_RET _opCallFree(void* ptr) {\n  free(ptr);\n}\n\n/*! \\brief returns status of calling parse attributes function for operator from library */\nMX_INT_RET _opCallParseAttrs(mxnet::ext::parseAttrs_t parseAttrs,\n                             const char* const* keys,\n                             const char* const* vals,\n                             int num,\n                             int* num_in,\n                             int* num_out) {\n  // create map of attributes from list\n  std::unordered_map<std::string, std::string> attrs;\n  for (int i = 0; i < num; i++) {\n    attrs[std::string(keys[i])] = std::string(vals[i]);\n  }\n  return parseAttrs(attrs, num_in, num_out);\n}\n\n/*! \\brief returns status of calling inferShape function for operator from library */\nMX_INT_RET _opCallInferShape(mxnet::ext::inferShape_t inferShape,\n                             const char* const* keys,\n                             const char* const* vals,\n                             int num,\n                             unsigned int** inshapes,\n                             int* indims,\n                             int num_in,\n                             unsigned int*** mod_inshapes,\n                             int** mod_indims,\n                             unsigned int*** outshapes,\n                             int** outdims,\n                             int num_out) {\n  // create map of attributes from list\n  std::unordered_map<std::string, std::string> attrs;\n  for (int i = 0; i < num; i++) {\n    attrs[std::string(keys[i])] = std::string(vals[i]);\n  }\n\n  // create a vector of shapes for inputs\n  std::vector<std::vector<unsigned int> > in_shapes(num_in);\n  for (int i = 0; i < num_in; i++) {\n    for (int j = 0; j < indims[i]; j++) {\n      in_shapes[i].push_back(inshapes[i][j]);\n    }\n  }\n\n  // create a vector of shapes for outputs\n  std::vector<std::vector<unsigned int> > out_shapes(num_out);\n\n  int retval = inferShape(attrs, &in_shapes, &out_shapes);\n  if (!retval)\n    return retval;\n\n  // allocate space for modified input dims, shape\n  *mod_indims   = static_cast<int*>(malloc(num_in * sizeof(int)));\n  *mod_inshapes = static_cast<unsigned**>(malloc(num_in * sizeof(unsigned*)));\n\n  // copy modified input shapes\n  for (int i = 0; i < num_in; i++) {\n    (*mod_indims)[i]   = in_shapes[i].size();\n    (*mod_inshapes)[i] = static_cast<unsigned*>(malloc((*mod_indims)[i] * sizeof(unsigned)));\n    for (int j = 0; j < (*mod_indims)[i]; j++) {\n      (*mod_inshapes)[i][j] = in_shapes[i][j];\n    }\n  }\n\n  // allocate space for output dims, shape\n  *outdims   = static_cast<int*>(malloc(num_out * sizeof(int)));\n  *outshapes = static_cast<unsigned**>(malloc(num_out * sizeof(unsigned*)));\n\n  // copy output shapes\n  for (int i = 0; i < num_out; i++) {\n    (*outdims)[i]   = out_shapes[i].size();\n    (*outshapes)[i] = static_cast<unsigned*>(malloc((*outdims)[i] * sizeof(unsigned)));\n    for (int j = 0; j < (*outdims)[i]; j++) {\n      (*outshapes)[i][j] = out_shapes[i][j];\n    }\n  }\n  return retval;\n}\n\n/*! \\brief returns status of calling inferType function for operator from library */\nMX_INT_RET _opCallInferType(mxnet::ext::inferType_t inferType,\n                            const char* const* keys,\n                            const char* const* vals,\n                            int num,\n                            int* intypes,\n                            int num_in,\n                            int* outtypes,\n                            int num_out) {\n  // create map of attributes from list\n  std::unordered_map<std::string, std::string> attrs;\n  for (int i = 0; i < num; i++) {\n    attrs[std::string(keys[i])] = std::string(vals[i]);\n  }\n\n  // create a vector of types for inputs\n  std::vector<int> in_types(num_in);\n  for (int i = 0; i < num_in; i++) {\n    in_types[i] = intypes[i];\n  }\n\n  // create a vector of types for outputs\n  std::vector<int> out_types(num_out, -1);\n\n  int retval = inferType(attrs, &in_types, &out_types);\n  if (!retval)\n    return retval;\n\n  // copy modified input types\n  for (int i = 0; i < num_in; i++) {\n    intypes[i] = in_types[i];\n  }\n  // copy output types\n  for (int i = 0; i < num_out; i++) {\n    outtypes[i] = out_types[i];\n  }\n\n  return retval;\n}\n\n/*! \\brief returns status of calling inferSType function for operator from library */\nMX_INT_RET _opCallInferSType(mxnet::ext::inferSType_t inferSType,\n                             const char* const* keys,\n                             const char* const* vals,\n                             int num,\n                             int* instypes,\n                             int num_in,\n                             int* outstypes,\n                             int num_out) {\n  // create map of attributes from list\n  std::unordered_map<std::string, std::string> attrs;\n  for (int i = 0; i < num; i++) {\n    attrs[std::string(keys[i])] = std::string(vals[i]);\n  }\n\n  // create a vector of types for inputs\n  std::vector<int> in_stypes(num_in);\n  for (int i = 0; i < num_in; i++) {\n    in_stypes[i] = instypes[i];\n  }\n\n  // create a vector of types for outputs\n  std::vector<int> out_stypes(num_out, -1);\n\n  int retval = inferSType(attrs, &in_stypes, &out_stypes);\n\n  if (!retval)\n    return retval;\n\n  // copy modified input storage types\n  for (int i = 0; i < num_in; i++) {\n    instypes[i] = in_stypes[i];\n  }\n  // copy output storage types\n  for (int i = 0; i < num_out; i++) {\n    outstypes[i] = out_stypes[i];\n  }\n\n  return retval;\n}\n\n/*! \\brief returns status of calling Forward/Backward function for operator from library */\nMX_INT_RET _opCallFCompute(mxnet::ext::fcomp_t fcomp,\n                           const char* const* keys,\n                           const char* const* vals,\n                           int num,\n                           const int64_t** inshapes,\n                           int* indims,\n                           void** indata,\n                           int* intypes,\n                           size_t* inIDs,\n                           const char** indev_type,\n                           int* indev_id,\n                           int num_in,\n                           const int64_t** outshapes,\n                           int* outdims,\n                           void** outdata,\n                           int* outtypes,\n                           size_t* outIDs,\n                           const char** outdev_type,\n                           int* outdev_id,\n                           int num_out,\n                           mxnet::ext::xpu_malloc_t cpu_malloc,\n                           void* cpu_alloc,\n                           mxnet::ext::xpu_malloc_t gpu_malloc,\n                           void* gpu_alloc,\n                           void* cuda_stream,\n                           mxnet::ext::sparse_malloc_t sparse_malloc,\n                           void* sparse_alloc,\n                           int* instypes,\n                           int* outstypes,\n                           void** in_indices,\n                           void** out_indices,\n                           void** in_indptr,\n                           void** out_indptr,\n                           int64_t* in_indices_shapes,\n                           int64_t* out_indices_shapes,\n                           int64_t* in_indptr_shapes,\n                           int64_t* out_indptr_shapes,\n                           void* rng_cpu_states,\n                           void* rng_gpu_states) {\n  // create map of attributes from list\n  std::unordered_map<std::string, std::string> attrs;\n  for (int i = 0; i < num; i++) {\n    attrs[std::string(keys[i])] = std::string(vals[i]);\n  }\n\n  // create a vector of tensors for inputs\n  std::vector<mxnet::ext::MXTensor> inputs(num_in);\n  // create a vector for sparse inputs\n  std::vector<mxnet::ext::MXSparse> in_sparse(num_in);\n\n  for (int i = 0; i < num_in; i++) {\n    // Dense representation.\n    if (instypes[i] == 0) {\n      inputs[i].setTensor(indata[i],\n                          (mxnet::ext::MXDType)intypes[i],\n                          inshapes[i],\n                          indims[i],\n                          inIDs[i],\n                          mxnet::ext::MXContext(indev_type[i], indev_id[i]),\n                          mxnet::ext::kDefaultStorage);\n    } else {\n      // Sparse representation.\n      mxnet::ext::MXStorageType type;\n      if (instypes[i] == 1) {\n        type = mxnet::ext::kRowSparseStorage;\n        in_sparse[i].set(indata[i], inshapes[i], indims[i], in_indices[i], in_indices_shapes[i]);\n      } else {\n        type = mxnet::ext::kCSRStorage;\n        in_sparse[i].set(indata[i],\n                         inshapes[i],\n                         indims[i],\n                         in_indices[i],\n                         in_indices_shapes[i],\n                         in_indptr[i],\n                         in_indptr_shapes[i]);\n      }\n      inputs[i].setTensor(reinterpret_cast<void*>(&in_sparse[i]),\n                          (mxnet::ext::MXDType)intypes[i],\n                          inshapes[i],\n                          indims[i],\n                          inIDs[i],\n                          mxnet::ext::MXContext(indev_type[i], indev_id[i]),\n                          type);\n    }\n  }\n\n  // create a vector of tensors for outputs\n  std::vector<mxnet::ext::MXTensor> outputs(num_out);\n  std::vector<mxnet::ext::MXSparse> out_sparse(num_out);\n\n  for (int i = 0; i < num_out; i++) {\n    // Dense representation.\n    if (outstypes[i] == 0) {\n      outputs[i].setTensor(outdata[i],\n                           (mxnet::ext::MXDType)outtypes[i],\n                           outshapes[i],\n                           outdims[i],\n                           outIDs[i],\n                           mxnet::ext::MXContext(outdev_type[i], outdev_id[i]),\n                           mxnet::ext::kDefaultStorage);\n    } else {\n      // Sparse representation.\n      mxnet::ext::MXStorageType type;\n      if (outstypes[i] == 1) {\n        type = mxnet::ext::kRowSparseStorage;\n        out_sparse[i].set(\n            outdata[i], outshapes[i], outdims[i], out_indices[i], out_indices_shapes[i]);\n      } else {\n        type = mxnet::ext::kCSRStorage;\n        out_sparse[i].set(outdata[i],\n                          outshapes[i],\n                          outdims[i],\n                          out_indices[i],\n                          out_indices_shapes[i],\n                          out_indptr[i],\n                          out_indptr_shapes[i]);\n      }\n      outputs[i].setTensor(reinterpret_cast<void*>(&out_sparse[i]),\n                           (mxnet::ext::MXDType)outtypes[i],\n                           outshapes[i],\n                           outdims[i],\n                           outIDs[i],\n                           mxnet::ext::MXContext(outdev_type[i], outdev_id[i]),\n                           type);\n    }\n  }\n\n  mxnet::ext::OpResource res(cpu_malloc,\n                             cpu_alloc,\n                             gpu_malloc,\n                             gpu_alloc,\n                             cuda_stream,\n                             sparse_malloc,\n                             sparse_alloc,\n                             rng_cpu_states,\n                             rng_gpu_states);\n  return fcomp(attrs, &inputs, &outputs, res);\n}\n\n/*! \\brief returns status of calling mutateInputs function for operator from library */\nMX_INT_RET _opCallMutateInputs(mxnet::ext::mutateInputs_t mutate,\n                               const char* const* keys,\n                               const char* const* vals,\n                               int num,\n                               int** mutate_indices,\n                               int* indices_size) {\n  // create map of attributes from list\n  std::unordered_map<std::string, std::string> attrs;\n  for (int i = 0; i < num; i++) {\n    attrs[std::string(keys[i])] = std::string(vals[i]);\n  }\n\n  // create a vector of mutate input indices\n  std::vector<int> mut_ind;\n\n  int retval = mutate(attrs, &mut_ind);\n  if (!retval)\n    return retval;\n\n  // output the input indices\n  *indices_size   = mut_ind.size();\n  *mutate_indices = static_cast<int*>(malloc(*indices_size * sizeof(int)));\n  for (int i = 0; i < *indices_size; i++) {\n    (*mutate_indices)[i] = mut_ind[i];\n  }\n\n  return retval;\n}\n\n/*! \\brief returns status of calling createStatefulOp function for operator from library */\nMX_INT_RET _opCallCreateOpState(mxnet::ext::createOpState_t create_op,\n                                const char* const* keys,\n                                const char* const* vals,\n                                int num,\n                                const char* dev_type,\n                                int dev_id,\n                                unsigned int** inshapes,\n                                int* indims,\n                                int num_in,\n                                const int* intypes,\n                                void** state_op) {\n  // create map of attributes from list\n  std::unordered_map<std::string, std::string> attrs;\n  for (int i = 0; i < num; i++) {\n    attrs[std::string(keys[i])] = std::string(vals[i]);\n  }\n\n  mxnet::ext::MXContext ctx(dev_type, dev_id);\n\n  // create a vector of shapes for inputs\n  std::vector<std::vector<unsigned int> > in_shapes(num_in);\n  for (int i = 0; i < num_in; i++) {\n    for (int j = 0; j < indims[i]; j++) {\n      in_shapes[i].push_back(inshapes[i][j]);\n    }\n  }\n\n  // create a vector of types for inputs\n  std::vector<int> in_types(num_in);\n  for (int i = 0; i < num_in; i++) {\n    in_types[i] = intypes[i];\n  }\n\n  // void pointer to hold custom state op instance created in custom library\n  // eventually state_op pointer is populated by instance from custom library\n  mxnet::ext::CustomStatefulOp** op_ptr =\n      reinterpret_cast<mxnet::ext::CustomStatefulOp**>(state_op);\n  return create_op(attrs, ctx, in_shapes, in_types, op_ptr);\n}\n\n/*! \\brief calls StatefulOp destructor for operator from library */\nMX_VOID_RET _opCallDestroyOpState(void* state_op) {\n  mxnet::ext::CustomStatefulOp* op_ptr = reinterpret_cast<mxnet::ext::CustomStatefulOp*>(state_op);\n  delete op_ptr;\n}\n\n/*! \\brief returns status of calling Stateful Forward/Backward for operator from library */\nMX_INT_RET _opCallFStatefulCompute(int is_forward,\n                                   void* state_op,\n                                   const int64_t** inshapes,\n                                   int* indims,\n                                   void** indata,\n                                   int* intypes,\n                                   size_t* inIDs,\n                                   const char** indev_type,\n                                   int* indev_id,\n                                   int num_in,\n                                   const int64_t** outshapes,\n                                   int* outdims,\n                                   void** outdata,\n                                   int* outtypes,\n                                   size_t* outIDs,\n                                   const char** outdev_type,\n                                   int* outdev_id,\n                                   int num_out,\n                                   mxnet::ext::xpu_malloc_t cpu_malloc,\n                                   void* cpu_alloc,\n                                   mxnet::ext::xpu_malloc_t gpu_malloc,\n                                   void* gpu_alloc,\n                                   void* stream,\n                                   mxnet::ext::sparse_malloc_t sparse_malloc,\n                                   void* sparse_alloc,\n                                   int* instypes,\n                                   int* outstypes,\n                                   void** in_indices,\n                                   void** out_indices,\n                                   void** in_indptr,\n                                   void** out_indptr,\n                                   int64_t* in_indices_shapes,\n                                   int64_t* out_indices_shapes,\n                                   int64_t* in_indptr_shapes,\n                                   int64_t* out_indptr_shapes,\n                                   void* rng_cpu_states,\n                                   void* rng_gpu_states) {\n  // create a vector of tensors for inputs\n  std::vector<mxnet::ext::MXTensor> inputs(num_in);\n  // create a vector for sparse inputs\n  std::vector<mxnet::ext::MXSparse> in_sparse(num_in);\n\n  for (int i = 0; i < num_in; i++) {\n    if (instypes[i] == 0) {\n      // Dense representation.\n      inputs[i].setTensor(indata[i],\n                          (mxnet::ext::MXDType)intypes[i],\n                          inshapes[i],\n                          indims[i],\n                          inIDs[i],\n                          mxnet::ext::MXContext(indev_type[i], indev_id[i]),\n                          mxnet::ext::kDefaultStorage);\n    } else {\n      // Sparse representation.\n      mxnet::ext::MXStorageType type;\n      if (instypes[i] == 1) {\n        type = mxnet::ext::kRowSparseStorage;\n        in_sparse[i].set(indata[i], inshapes[i], indims[i], in_indices[i], in_indices_shapes[i]);\n      } else {\n        type = mxnet::ext::kCSRStorage;\n        in_sparse[i].set(indata[i],\n                         inshapes[i],\n                         indims[i],\n                         in_indices[i],\n                         in_indices_shapes[i],\n                         in_indptr[i],\n                         in_indptr_shapes[i]);\n      }\n      inputs[i].setTensor(reinterpret_cast<void*>(&in_sparse[i]),\n                          (mxnet::ext::MXDType)intypes[i],\n                          inshapes[i],\n                          indims[i],\n                          inIDs[i],\n                          mxnet::ext::MXContext(indev_type[i], indev_id[i]),\n                          type);\n    }\n  }\n\n  // create a vector of tensors for outputs\n  std::vector<mxnet::ext::MXTensor> outputs(num_out);\n  // create a vector for sparse outputs\n  std::vector<mxnet::ext::MXSparse> out_sparse(num_out);\n\n  for (int i = 0; i < num_out; i++) {\n    if (outstypes[i] == 0) {\n      // Dense representation.\n      outputs[i].setTensor(outdata[i],\n                           (mxnet::ext::MXDType)outtypes[i],\n                           outshapes[i],\n                           outdims[i],\n                           outIDs[i],\n                           mxnet::ext::MXContext(outdev_type[i], outdev_id[i]),\n                           mxnet::ext::kDefaultStorage);\n    } else {\n      // Sparse representation.\n      mxnet::ext::MXStorageType type;\n      if (outstypes[i] == 1) {\n        type = mxnet::ext::kRowSparseStorage;\n        out_sparse[i].set(\n            outdata[i], outshapes[i], outdims[i], out_indices[i], out_indices_shapes[i]);\n      } else {\n        type = mxnet::ext::kCSRStorage;\n        out_sparse[i].set(outdata[i],\n                          outshapes[i],\n                          outdims[i],\n                          out_indices[i],\n                          out_indices_shapes[i],\n                          out_indptr[i],\n                          out_indptr_shapes[i]);\n      }\n      outputs[i].setTensor(reinterpret_cast<void*>(&out_sparse[i]),\n                           (mxnet::ext::MXDType)outtypes[i],\n                           outshapes[i],\n                           outdims[i],\n                           outIDs[i],\n                           mxnet::ext::MXContext(outdev_type[i], outdev_id[i]),\n                           type);\n    }\n  }\n\n  mxnet::ext::OpResource res(cpu_malloc,\n                             cpu_alloc,\n                             gpu_malloc,\n                             gpu_alloc,\n                             stream,\n                             sparse_malloc,\n                             sparse_alloc,\n                             rng_cpu_states,\n                             rng_gpu_states);\n\n  mxnet::ext::CustomStatefulOp* op_ptr = reinterpret_cast<mxnet::ext::CustomStatefulOp*>(state_op);\n  if (is_forward) {\n    return op_ptr->Forward(&inputs, &outputs, res);\n  }\n  return op_ptr->Backward(&inputs, &outputs, res);\n}\n\n/*! \\brief returns number of partitioners registered in this library */\nMX_INT_RET _partRegSize() {\n  return mxnet::ext::Registry<mxnet::ext::CustomPartitioner>::get()->size();\n}\n\n/* returns number of strategies registered for partitioner\n * at specified index */\nMX_INT_RET _partRegGetCount(int idx, const char** name) {\n  mxnet::ext::CustomPartitioner part =\n      mxnet::ext::Registry<mxnet::ext::CustomPartitioner>::get()->get(idx);\n  *name = part.name;\n  return part.strategies.size();\n}\n\n/*! \\brief returns partitioner registration at specified index */\nMX_VOID_RET _partRegGet(int part_idx,\n                        int stg_idx,\n                        const char** strategy,\n                        mxnet::ext::supportedOps_t* supportedOps,\n                        mxnet::ext::createSelector_t* createSelector,\n                        mxnet::ext::reviewSubgraph_t* reviewSubgraph,\n                        const char** op_name) {\n  mxnet::ext::CustomPartitioner part =\n      mxnet::ext::Registry<mxnet::ext::CustomPartitioner>::get()->get(part_idx);\n  *strategy       = part.strategies[stg_idx];\n  *op_name        = part.op_names[stg_idx];\n  *supportedOps   = part.getSupportedOps(stg_idx);\n  *createSelector = part.getCreateSelector(stg_idx);\n  *reviewSubgraph = part.getReviewSubgraph(stg_idx);\n}\n\n/*! \\brief returns status of calling supported ops function from library */\nMX_INT_RET _partCallSupportedOps(mxnet::ext::supportedOps_t supportedOps,\n                                 const char* json,\n                                 int num_ids,\n                                 int* ids,\n                                 const char* const* opt_keys,\n                                 const char* const* opt_vals,\n                                 int num_opts) {\n  mxnet::ext::Graph* graph = mxnet::ext::Graph::fromString(json);\n  // create map of options from list\n  std::unordered_map<std::string, std::string> opts;\n  for (int i = 0; i < num_opts; i++)\n    opts[std::string(opt_keys[i])] = std::string(opt_vals[i]);\n\n  // create array of subgraph IDs for operator support\n  std::vector<int> _ids(num_ids, -2);\n  // call user's supportedOps function\n  mxnet::ext::MXReturnValue retval = supportedOps(graph, &_ids, opts);\n  if (!retval)\n    return retval;\n\n  // copy bools in ids to ints\n  for (int i = 0; i < num_ids; i++)\n    ids[i] = _ids[i];\n\n  return retval;\n}\n\n/*! \\brief returns status of calling create selector function from library */\nMX_INT_RET _partCallCreateSelector(mxnet::ext::createSelector_t createSelector,\n                                   const char* json,\n                                   void** selector,\n                                   const char* const* opt_keys,\n                                   const char* const* opt_vals,\n                                   int num_opts) {\n  mxnet::ext::Graph* graph = mxnet::ext::Graph::fromString(json);\n  // create map of options from list\n  std::unordered_map<std::string, std::string> opts;\n  for (int i = 0; i < num_opts; i++)\n    opts[std::string(opt_keys[i])] = std::string(opt_vals[i]);\n\n  // void pointer to hold selector instance created in custom library\n  // eventually pointer is populated by instance from custom library\n  mxnet::ext::CustomOpSelector** sel_ptr =\n      reinterpret_cast<mxnet::ext::CustomOpSelector**>(selector);\n\n  // call user's createSelector function\n  return createSelector(graph, sel_ptr, opts);\n}\n\n/*! \\brief returns status of calling select function from library */\nMX_VOID_RET _partCallSelect(void* sel_inst, int nodeID, int* selected) {\n  mxnet::ext::CustomOpSelector* sel_ptr = reinterpret_cast<mxnet::ext::CustomOpSelector*>(sel_inst);\n  *selected                             = sel_ptr->Select(nodeID);\n}\n\n/*! \\brief returns status of calling select input function from library */\nMX_VOID_RET _partCallSelectInput(void* sel_inst, int nodeID, int input_nodeID, int* selected) {\n  mxnet::ext::CustomOpSelector* sel_ptr = reinterpret_cast<mxnet::ext::CustomOpSelector*>(sel_inst);\n  *selected                             = sel_ptr->SelectInput(nodeID, input_nodeID);\n}\n\n/*! \\brief returns status of calling select output function from library */\nMX_VOID_RET _partCallSelectOutput(void* sel_inst, int nodeID, int output_nodeID, int* selected) {\n  mxnet::ext::CustomOpSelector* sel_ptr = reinterpret_cast<mxnet::ext::CustomOpSelector*>(sel_inst);\n  *selected                             = sel_ptr->SelectOutput(nodeID, output_nodeID);\n}\n\n/*! \\brief returns status of calling filter function from library */\nMX_VOID_RET _partCallFilter(void* sel_inst,\n                            int* candidates,\n                            int num_candidates,\n                            int** keep,\n                            int* num_keep) {\n  mxnet::ext::CustomOpSelector* sel_ptr = reinterpret_cast<mxnet::ext::CustomOpSelector*>(sel_inst);\n  std::vector<int> candidates_(num_candidates);\n  for (int i = 0; i < num_candidates; i++) {\n    candidates_[i] = candidates[i];\n  }\n  std::vector<int> keep_;\n\n  sel_ptr->Filter(candidates_, &keep_);\n\n  *num_keep = keep_.size();\n  *keep     = static_cast<int*>(malloc(keep_.size() * sizeof(int)));\n  for (unsigned i = 0; i < keep_.size(); i++)\n    (*keep)[i] = keep_[i];\n}\n\n/*! \\brief returns status of calling reset selector function from library */\nMX_VOID_RET _partCallReset(void* sel_inst) {\n  mxnet::ext::CustomOpSelector* sel_ptr = reinterpret_cast<mxnet::ext::CustomOpSelector*>(sel_inst);\n  sel_ptr->Reset();\n}\n\n/*! \\brief returns status of calling review subgraph function from library */\nMX_INT_RET _partCallReviewSubgraph(mxnet::ext::reviewSubgraph_t reviewSubgraph,\n                                   const char* json,\n                                   int subgraph_id,\n                                   int* accept,\n                                   const char* const* opt_keys,\n                                   const char* const* opt_vals,\n                                   int num_opts,\n                                   char*** attr_keys,\n                                   char*** attr_vals,\n                                   int* num_attrs,\n                                   const char* const* arg_names,\n                                   int num_args,\n                                   void* const* arg_data,\n                                   const int64_t* const* arg_shapes,\n                                   const int* arg_dims,\n                                   const int* arg_types,\n                                   const size_t* arg_IDs,\n                                   const char* const* arg_dev_type,\n                                   const int* arg_dev_id,\n                                   const char* const* aux_names,\n                                   int num_aux,\n                                   void* const* aux_data,\n                                   const int64_t* const* aux_shapes,\n                                   const int* aux_dims,\n                                   const int* aux_types,\n                                   const size_t* aux_IDs,\n                                   const char* const* aux_dev_type,\n                                   const int* aux_dev_id) {\n  mxnet::ext::Graph* subgraph = mxnet::ext::Graph::fromString(json);\n  bool accept_bool            = false;\n  // create map of attributes from list\n  std::unordered_map<std::string, std::string> opts;\n  for (int i = 0; i < num_opts; i++)\n    opts[std::string(opt_keys[i])] = std::string(opt_vals[i]);\n\n  // create a map of named tensors for args\n  std::unordered_map<std::string, mxnet::ext::MXTensor> args;\n  for (int i = 0; i < num_args; i++) {\n    std::vector<int64_t> shapes;\n    shapes.reserve(arg_dims[i]);\n    for (int j = 0; j < arg_dims[i]; j++)\n      shapes.push_back(arg_shapes[i][j]);\n\n    mxnet::ext::MXTensor tensor(arg_data[i],\n                                shapes,\n                                (mxnet::ext::MXDType)arg_types[i],\n                                arg_IDs[i],\n                                mxnet::ext::MXContext(arg_dev_type[i], arg_dev_id[i]));\n    args[arg_names[i]] = tensor;\n  }\n  // create a map of named tensors for aux\n  std::unordered_map<std::string, mxnet::ext::MXTensor> aux;\n  for (int i = 0; i < num_aux; i++) {\n    std::vector<int64_t> shapes;\n    shapes.reserve(aux_dims[i]);\n    for (int j = 0; j < aux_dims[i]; j++)\n      shapes.push_back(aux_shapes[i][j]);\n\n    mxnet::ext::MXTensor tensor(aux_data[i],\n                                shapes,\n                                (mxnet::ext::MXDType)aux_types[i],\n                                aux_IDs[i],\n                                mxnet::ext::MXContext(aux_dev_type[i], aux_dev_id[i]));\n    aux[aux_names[i]] = tensor;\n  }\n\n  subgraph->_setParams(&args, &aux);\n\n  std::unordered_map<std::string, std::string> attrs;\n  mxnet::ext::MXReturnValue retval =\n      reviewSubgraph(subgraph, subgraph_id, &accept_bool, opts, &attrs);\n  if (!retval)\n    return retval;\n\n  *accept = accept_bool;\n\n  if (attrs.size() > 0) {\n    *num_attrs = attrs.size();\n    // allocate space for attributes\n    *attr_keys = static_cast<char**>(malloc(*num_attrs * sizeof(char*)));\n    *attr_vals = static_cast<char**>(malloc(*num_attrs * sizeof(char*)));\n\n    // copy attributes\n    int i = 0;\n    for (auto kv : attrs) {\n      (*attr_keys)[i] = static_cast<char*>(malloc((kv.first.size() + 1) * sizeof(char)));  // NOLINT\n      (*attr_vals)[i] =\n          static_cast<char*>(malloc((kv.second.size() + 1) * sizeof(char)));  // NOLINT\n      snprintf((*attr_keys)[i], kv.first.size() + 1, \"%s\", kv.first.c_str());\n      snprintf((*attr_vals)[i], kv.second.size() + 1, \"%s\", kv.second.c_str());\n      i++;\n    }\n  }\n\n  return retval;\n}\n\n/*! \\brief returns number of graph passes registered in this library */\nMX_INT_RET _passRegSize() {\n  return mxnet::ext::Registry<mxnet::ext::CustomPass>::get()->size();\n}\n\n/*! \\brief returns pass registration at specified index */\nMX_VOID_RET _passRegGet(int pass_idx, mxnet::ext::graphPass_t* graphPass, const char** pass_name) {\n  mxnet::ext::CustomPass pass = mxnet::ext::Registry<mxnet::ext::CustomPass>::get()->get(pass_idx);\n  *graphPass                  = pass.pass;\n  *pass_name                  = pass.name;\n}\n\n/*! \\brief returns status of calling graph pass function from library */\nMX_INT_RET _passCallGraphPass(mxnet::ext::graphPass_t graphPass,\n                              const char* json,\n                              char** out_graph,\n                              const char* const* opt_keys,\n                              const char* const* opt_vals,\n                              int num_opts,\n                              const char* pass_name,\n                              const char* const* arg_names,\n                              int num_args,\n                              void* const* arg_data,\n                              const int64_t* const* arg_shapes,\n                              const int* arg_dims,\n                              const int* arg_types,\n                              const size_t* arg_IDs,\n                              const char* const* arg_dev_type,\n                              const int* arg_dev_id,\n                              const char* const* aux_names,\n                              int num_aux,\n                              void* const* aux_data,\n                              const int64_t* const* aux_shapes,\n                              const int* aux_dims,\n                              const int* aux_types,\n                              const size_t* aux_IDs,\n                              const char* const* aux_dev_type,\n                              const int* aux_dev_id,\n                              mxnet::ext::nd_malloc_t nd_malloc,\n                              const void* nd_alloc) {\n  mxnet::ext::Graph* graph = mxnet::ext::Graph::fromString(json);\n  // create map of attributes from list\n  std::unordered_map<std::string, std::string> opts;\n  for (int i = 0; i < num_opts; i++)\n    opts[std::string(opt_keys[i])] = std::string(opt_vals[i]);\n\n  // create a map of named tensors for args\n  std::unordered_map<std::string, mxnet::ext::MXTensor> args;\n  for (int i = 0; i < num_args; i++) {\n    std::vector<int64_t> shapes;\n    shapes.reserve(arg_dims[i]);\n    for (int j = 0; j < arg_dims[i]; j++)\n      shapes.push_back(arg_shapes[i][j]);\n\n    mxnet::ext::MXTensor tensor(arg_data[i],\n                                shapes,\n                                (mxnet::ext::MXDType)arg_types[i],\n                                arg_IDs[i],\n                                mxnet::ext::MXContext(arg_dev_type[i], arg_dev_id[i]));\n    args[arg_names[i]] = tensor;\n  }\n  // create a map of named tensors for aux\n  std::unordered_map<std::string, mxnet::ext::MXTensor> aux;\n  for (int i = 0; i < num_aux; i++) {\n    std::vector<int64_t> shapes;\n    shapes.reserve(aux_dims[i]);\n    for (int j = 0; j < aux_dims[i]; j++)\n      shapes.push_back(aux_shapes[i][j]);\n\n    mxnet::ext::MXTensor tensor(aux_data[i],\n                                shapes,\n                                (mxnet::ext::MXDType)aux_types[i],\n                                aux_IDs[i],\n                                mxnet::ext::MXContext(aux_dev_type[i], aux_dev_id[i]));\n    aux[aux_names[i]] = tensor;\n  }\n\n  std::unordered_map<std::string, mxnet::ext::MXTensor> new_args, new_aux;\n  mxnet::ext::PassResource res(&new_args, &new_aux, nd_malloc, nd_alloc);\n  graph->_setParams(&args, &aux);\n  graph->_setPassResource(&res);\n  mxnet::ext::MXReturnValue retval = graphPass(graph, opts);\n  if (!retval)\n    return retval;\n\n  std::string tmp = graph->toString();\n  *out_graph      = static_cast<char*>(malloc((tmp.size() + 1) * sizeof(char)));  // NOLINT\n  snprintf((*out_graph), tmp.size() + 1, \"%s\", tmp.c_str());\n  return retval;\n}\n\n/*!\n * \\brief Checks if the MXNet version is supported by the library.\n * If supported, initializes the library.\n * \\param version MXNet version number passed to library and defined as:\n *                MXNET_VERSION = (MXNET_MAJOR*10000 + MXNET_MINOR*100 + MXNET_PATCH)\n * \\return Non-zero value on error i.e. library incompatible with passed MXNet version\n */\n#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)\n__declspec(dllexport) mxnet::ext::MXReturnValue __cdecl\n#else\nmxnet::ext::MXReturnValue\n#endif\n    initialize(int version);\n\nMX_INT_RET _msgSize() {\n  return mxnet::ext::MXerrorMsgs::get()->size();\n}\n\n/*! \\brief returns operator registration at specified index */\nMX_VOID_RET _msgGet(int idx, const char** msg) {\n  *msg = mxnet::ext::MXerrorMsgs::get()->get(idx)->c_str();\n}\n"
  },
  {
    "path": "src/ndarray/ndarray_function.cc",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file ndarray_function.cc\n * \\brief CPU Implementation of ndarray function.\n */\n\n// this will be invoked by gcc and compile CPU version\n#include \"./ndarray_function.h\"\n#include \"./ndarray_function-inl.h\"\n#include \"../common/utils.h\"\n#include \"../operator/mxnet_op.h\"\n#include \"../operator/tensor/elemwise_binary_op-inl.h\"\n#include \"../operator/tensor/elemwise_sum.h\"\n\nnamespace mxnet {\nnamespace ndarray {\ntemplate <>\nvoid Copy<cpu, cpu>(const TBlob& from,\n                    TBlob* to,\n                    Context from_ctx,\n                    Context to_ctx,\n                    RunContext ctx) {\n  MSHADOW_TYPE_SWITCH_EXT_WITH_BOOL(to->type_flag_, DType, {\n    if (to->type_flag_ == from.type_flag_) {\n      if (!features::is_enabled(features::INT64_TENSOR_SIZE)) {\n        CHECK_LT(from.Size(), (int64_t{1} << 31) - 1)\n            << \"Size of tensor you are trying to allocate is larger than \"\n               \"2^31 elements. Please build with flag USE_INT64_TENSOR_SIZE=1\";\n      }\n      const index_t size = static_cast<index_t>(from.Size());\n      CHECK_EQ(size, to->Size()) << \"copying size mismatch, from: \" << size * sizeof(DType)\n                                 << \" bytes, to: \" << to->Size() * sizeof(DType) << \" bytes.\";\n      common::ParallelCopy(to->dptr<DType>(), from.dptr<DType>(), size);\n    } else {\n      MSHADOW_TYPE_SWITCH_EXT_WITH_BOOL(from.type_flag_, SrcDType, {\n        to->FlatTo1D<cpu, DType>() = mshadow::expr::tcast<DType>(from.FlatTo1D<cpu, SrcDType>());\n      })\n    }\n  })\n}\n\ntemplate <typename DType, typename IType>\nvoid ElementwiseSumRspImpl(mshadow::Stream<cpu>* s,\n                           const std::vector<NDArray>& nds,\n                           const std::vector<IType>& uniq_row_idx,\n                           NDArray* out,\n                           const int nthreads = 4) {\n#pragma omp parallel num_threads(nthreads)\n  {\n    const size_t nnr             = uniq_row_idx.size();\n    const int num_threads        = omp_get_num_threads();\n    size_t row_block_len         = (nnr + num_threads - 1) / num_threads;\n    const size_t row_block_start = omp_get_thread_num() * row_block_len;\n    if (row_block_start < nnr) {\n      const size_t row_block_end = std::min(row_block_start + row_block_len, nnr);\n\n      const size_t row_length = out->data().shape_.ProdShape(1, out->data().shape_.ndim());\n      auto out_values         = out->data().get_with_shape<cpu, 2, DType>(\n          mshadow::Shape2(out->storage_shape()[0], row_length), s);\n      auto out_indices = out->aux_data(rowsparse::kIdx).FlatTo1D<cpu, IType>();\n      for (size_t i = row_block_start; i < row_block_end; ++i) {\n        out_indices[i] = uniq_row_idx[i];\n      }\n      for (const auto& nd : nds) {\n        if (nd.storage_initialized()) {\n          const auto nd_indices = nd.aux_data(rowsparse::kIdx).FlatTo1D<cpu, IType>();\n          const auto nd_values  = nd.data().get_with_shape<cpu, 2, DType>(\n              mshadow::Shape2(nd.storage_shape()[0], row_length), s);\n          const auto nd_num_rows        = nd.aux_shape(rowsparse::kIdx).Size();\n          const IType* nd_indices_start = &nd_indices[0];\n          const IType* nd_indices_end   = nd_indices_start + nd_num_rows;\n          const IType* row_idx_ptr =\n              std::lower_bound(nd_indices_start, nd_indices_end, out_indices[row_block_start]);\n          // skip this nd if all of its row indices are smaller than out_indices[row_block_start]\n          // or current row block is not covered by [*row_idx_ptr, nd_indices_end).\n          if (nd_indices_end == row_idx_ptr || *row_idx_ptr > out_indices[row_block_end - 1]) {\n            continue;\n          }\n          for (size_t irow = row_block_start;\n               irow < row_block_end && row_idx_ptr != nd_indices_end;) {\n            if (out_indices[irow] == *row_idx_ptr) {\n              auto out_value_cur_row = out_values[irow];\n              const auto offset      = row_idx_ptr - nd_indices_start;\n              auto nd_value_cur_row  = nd_values[offset];\n              for (index_t j = 0; j < nd_value_cur_row.shape_[0]; ++j) {\n                out_value_cur_row[j] += nd_value_cur_row[j];\n              }\n              ++irow;\n              ++row_idx_ptr;\n            } else if (out_indices[irow] < *row_idx_ptr) {\n              ++irow;\n            } else {\n              ++row_idx_ptr;\n            }\n          }\n        }\n      }\n    }\n  }\n}\n\n/*!\n * \\brief Given a vector of ndarrays, generate a index vector containing\n * all the unique row indices of the ndarrays.\n */\ntemplate <typename IType>\nvoid GetUniqueRspRowIdx(const std::vector<NDArray>& nds, std::vector<IType>* uniq_row_idx) {\n  using namespace rowsparse;\n  size_t total_num_rows = 0;\n  for (const auto& nd : nds) {\n    CHECK_EQ(nd.storage_type(), kRowSparseStorage);\n    if (nd.storage_initialized()) {\n      total_num_rows += nd.aux_shape(kIdx).Size();\n    }\n  }\n\n  uniq_row_idx->resize(total_num_rows);\n  int nthreads = omp_get_max_threads();\n  int offset   = 0;\n  for (const auto& nd : nds) {\n    if (nd.storage_initialized()) {\n      const IType* nd_row_idx = nd.aux_data(kIdx).dptr<IType>();\n      const int num_rows      = nd.aux_shape(kIdx).Size();\n#pragma omp parallel for num_threads(nthreads)\n      for (int i = 0; i < num_rows; ++i) {\n        (*uniq_row_idx)[offset + i] = nd_row_idx[i];\n      }\n      offset += num_rows;\n    }\n  }\n\n  common::ParallelSort(uniq_row_idx->begin(), uniq_row_idx->end(), nthreads);\n  auto it = std::unique(uniq_row_idx->begin(), uniq_row_idx->end());\n  uniq_row_idx->resize(it - uniq_row_idx->begin());\n}\n\nvoid ElementwiseSumRsp(mshadow::Stream<cpu>* s,\n                       const Resource& rsc,\n                       const std::vector<NDArray>& nds,\n                       NDArray* out) {\n  if (nds.empty())\n    return;\n  using namespace rowsparse;\n  CHECK_EQ(out->storage_type(), kRowSparseStorage)\n      << \"Expected row sparse storage type (\" << out->storage_type() << \" given)\";\n\n  MSHADOW_TYPE_SWITCH(out->dtype(), DType, {\n    MSHADOW_IDX_TYPE_SWITCH(out->aux_type(kIdx), IType, {\n      // TODO(Jun): Use resource rsc for temporary vector instead of\n      //            allocating it directly in GetUniqueRspRowIdx\n      std::vector<IType> uniq_row_idx;\n      GetUniqueRspRowIdx(nds, &uniq_row_idx);\n      out->CheckAndAlloc({mshadow::Shape1(uniq_row_idx.size())});\n      out->data().FlatTo2D<cpu, DType>() = static_cast<DType>(0);\n      ElementwiseSumRspImpl<DType, IType>(s, nds, uniq_row_idx, out, omp_get_max_threads());\n    });\n  });\n}\n\nvoid ElementwiseSumDnsCsrDnsImpl(mshadow::Stream<cpu>* s,\n                                 const Resource& rsc,\n                                 const std::vector<NDArray>& nds,\n                                 NDArray* out) {\n  using namespace mxnet::op;\n  using namespace mxnet::op::mxnet_op;\n  const TBlob& out_data = out->data();\n  MSHADOW_TYPE_SWITCH(out->dtype(), DType, {  // data type\n    Kernel<Sum, cpu>::Launch(s,\n                             out_data.Size(),\n                             out_data.dptr<DType>(),\n                             kWriteTo,\n                             nds[0].data().dptr<DType>(),\n                             nds[2].data().dptr<DType>());\n    const TBlob& csr_data      = nds[1].data();\n    const TBlob& csr_indices   = nds[1].aux_data(csr::kIdx);\n    const TBlob& csr_indptr    = nds[1].aux_data(csr::kIndPtr);\n    const nnvm::dim_t num_rows = nds[1].shape()[0];\n    const nnvm::dim_t num_cols = nds[1].shape()[1];\n    MSHADOW_IDX_TYPE_SWITCH(csr_indices.type_flag_, IType, {   // indices type\n      MSHADOW_IDX_TYPE_SWITCH(csr_indptr.type_flag_, CType, {  // indptr type\n        if (nds[1].storage_initialized()) {\n          Kernel<ElemwiseDnsCsrDnsKernel<kWriteTo, mshadow_op::plus>, cpu>::Launch(\n              s,\n              num_rows,\n              out_data.dptr<DType>(),\n              out_data.dptr<DType>(),\n              csr_data.dptr<DType>(),\n              csr_indices.dptr<IType>(),\n              csr_indptr.dptr<CType>(),\n              num_rows,\n              num_cols);\n        }\n      });\n    });\n  });\n}\n\nvoid ElementwiseSumContainsDnsImpl(mshadow::Stream<cpu>* s,\n                                   const Resource& rsc,\n                                   const std::vector<NDArray>& nds,\n                                   NDArray* out) {\n  using namespace mxnet::op;\n  using namespace mxnet::op::mxnet_op;\n  const TBlob& out_data = out->data();\n  MSHADOW_TYPE_SWITCH(out->dtype(), DType, {  // data type\n    // Do not set_zero when output mem inplace with input[0] mem\n    // Now for add_n OP, output mem can be in-placed with the first input\n    if (nds[0].data().dptr<DType>() != out_data.dptr<DType>()) {\n      Kernel<set_zero, cpu>::Launch(s, out_data.Size(), out_data.dptr<DType>());\n    }\n    for (size_t i = 0; i < nds.size(); ++i) {\n      const NDArray& nd    = nds[i];\n      const TBlob& nd_data = nd.data();\n\n      if (i == 0) {\n        if (nd.storage_type() == kDefaultStorage) {\n          Kernel<op_with_req<mshadow_op::identity, kWriteTo>, cpu>::Launch(\n              s, out_data.Size(), out_data.dptr<DType>(), nd_data.dptr<DType>());\n          continue;\n        } else {\n          Kernel<set_zero, cpu>::Launch(s, out_data.Size(), out_data.dptr<DType>());\n        }\n      }\n\n      switch (nd.storage_type()) {\n        case kDefaultStorage: {\n          Kernel<op_with_req<mshadow_op::plus, kWriteTo>, cpu>::Launch(s,\n                                                                       out_data.Size(),\n                                                                       out_data.dptr<DType>(),\n                                                                       out_data.dptr<DType>(),\n                                                                       nd_data.dptr<DType>());\n          break;\n        }\n        case kCSRStorage: {\n          const TBlob& nd_indices    = nd.aux_data(csr::kIdx);\n          const TBlob& nd_indptr     = nd.aux_data(csr::kIndPtr);\n          const nnvm::dim_t num_rows = nd.shape()[0];\n          const nnvm::dim_t num_cols = nd.shape()[1];\n          MSHADOW_IDX_TYPE_SWITCH(nd_indices.type_flag_, IType, {   // indices type\n            MSHADOW_IDX_TYPE_SWITCH(nd_indptr.type_flag_, CType, {  // indptr type\n              if (nd.storage_initialized()) {\n                Kernel<ElemwiseDnsCsrDnsKernel<kWriteTo, mshadow_op::plus>, cpu>::Launch(\n                    s,\n                    num_rows,\n                    out_data.dptr<DType>(),\n                    out_data.dptr<DType>(),\n                    nd_data.dptr<DType>(),\n                    nd_indices.dptr<IType>(),\n                    nd_indptr.dptr<CType>(),\n                    num_rows,\n                    num_cols);\n              }\n            });\n          });\n          break;\n        }\n        case kRowSparseStorage: {\n          const TBlob& nd_indices    = nd.aux_data(rowsparse::kIdx);\n          const nnvm::dim_t num_rows = nd.shape()[0];\n          const nnvm::dim_t num_cols = nd.shape()[1];\n          MSHADOW_IDX_TYPE_SWITCH(nd_indices.type_flag_, IType, {  // indices type\n            if (nd.storage_initialized()) {\n              const nnvm::dim_t nz_rows = nd_indices.Size();\n              Kernel<ElemwiseDnsRspDnsKernel<kWriteTo, mshadow_op::plus>, cpu>::Launch(\n                  s,\n                  nz_rows * num_cols,\n                  out_data.dptr<DType>(),\n                  out_data.dptr<DType>(),\n                  nd_data.dptr<DType>(),\n                  nd_indices.dptr<IType>(),\n                  num_rows,\n                  nz_rows,\n                  num_cols);\n            }\n          });\n          break;\n        }\n        default:\n          LOG(FATAL) << \"unknown storage type \" << nd.storage_type() << \"encountered...\";\n      }\n    }\n  });\n}\n\n/*!\n * \\brief Parallel cpu impl of elemwise sum for sparse tensors.\n * Currently only support row sparse sum.\n */\ntemplate <>\nvoid ElementwiseSum<cpu>(mshadow::Stream<cpu>* s,\n                         const Resource& rsc,\n                         const std::vector<NDArray>& nds,\n                         NDArray* out) {\n  if (nds.empty())\n    return;\n  if (common::ContainsOnlyStorage(nds, kRowSparseStorage)) {\n    ElementwiseSumRsp(s, rsc, nds, out);\n  } else if (nds.size() == 3U && nds[0].storage_type() == kDefaultStorage &&\n             nds[1].storage_type() == kCSRStorage && nds[2].storage_type() == kDefaultStorage &&\n             out->storage_type() == kDefaultStorage) {\n    ElementwiseSumDnsCsrDnsImpl(s, rsc, nds, out);\n  } else if (nds.size() > 4U && common::ContainsStorageType(nds, kDefaultStorage) &&\n             out->storage_type() == kDefaultStorage) {\n    ElementwiseSumContainsDnsImpl(s, rsc, nds, out);\n  } else {\n    LOG(FATAL) << \"ElementwiseSum<cpu> has not been implemented for storage_type = << \"\n               << nds[0].storage_type();\n  }\n}\n\ntemplate <>\nvoid Eval<cpu>(mshadow::Stream<cpu>* s, const real_t val, const NDArray& dst) {\n  NDArray temp                   = dst;\n  const NDArrayStorageType stype = temp.storage_type();\n  if (stype == kRowSparseStorage) {\n    SetValueRspImpl(s, val, &temp);\n  } else {\n    LOG(FATAL) << \"Not implemented for storage type\" << stype;\n  }\n}\n\n}  // namespace ndarray\n}  // namespace mxnet\n"
  },
  {
    "path": "src/nnvm/error.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n#ifndef MXNET_NNVM_ERROR_H_\n#define MXNET_NNVM_ERROR_H_\n\n#include <exception>\n#include <string>\n\nnamespace nnvm {\nnamespace pass {\n\nclass InvalidGraphError : public std::exception {\n public:\n  explicit InvalidGraphError(const std::string& msg = \"invalid graph error\") : msg_(msg) {}\n  ~InvalidGraphError() throw() {}\n  virtual const char* what() const throw() {\n    return msg_.c_str();\n  }\n\n private:\n  std::string msg_;\n};\n\n}  // namespace pass\n}  // namespace nnvm\n#endif  // MXNET_NNVM_ERROR_H_\n"
  },
  {
    "path": "src/nnvm/graph_algorithm.h",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*!\n * \\file graph_algorithm.h\n * \\brief This header contains graph algorithms on StaticGraph.\n *  It is used  compute informations such as whether two\n *  operations can run in parallel, and helps allocation.\n */\n#ifndef MXNET_NNVM_GRAPH_ALGORITHM_H_\n#define MXNET_NNVM_GRAPH_ALGORITHM_H_\n\n#include <nnvm/graph.h>\n#include <vector>\n\nnamespace nnvm {\nnamespace pass {\n\n/*!\n * \\brief Find best path in the DAG, with reward defined\n *  by sum of reward of each node along the path.\n * \\param graph the original static graph.\n * \\param topo_order topo order of the nodes in the graph.\n * \\param node_reward the reward of each node.\n * \\param path the output path of nodes.\n * \\return the total reward of best path.\n */\ninline uint32_t MXFindBestPath(const IndexedGraph& graph,\n                               const std::vector<uint32_t>& node_reward,\n                               std::vector<uint32_t>* path) {\n  const uint32_t num_nodes = static_cast<uint32_t>(graph.num_nodes());\n  CHECK_EQ(num_nodes, node_reward.size());\n\n  std::vector<uint32_t> best_reward(node_reward.size(), 0);\n  std::vector<uint32_t> next_node(node_reward.size(), num_nodes);\n  uint32_t best_solution = 0, best_start_node = 0;\n\n  // traverse in reverse topo order\n  for (uint32_t i = static_cast<uint32_t>(graph.num_nodes()); i != 0; --i) {\n    const uint32_t nid = i - 1;\n    best_reward[nid] += node_reward[nid];\n    if (best_reward[nid] > best_solution) {\n      best_solution   = best_reward[nid];\n      best_start_node = nid;\n    }\n    for (const auto& e : graph[nid].inputs) {\n      const uint32_t prev = e.node_id;\n      if (best_reward[nid] > best_reward[prev]) {\n        best_reward[prev] = best_reward[nid];\n        next_node[prev]   = nid;\n      }\n    }\n  }\n  path->clear();\n  uint32_t reward = 0;\n  for (uint32_t nid = best_start_node; nid < num_nodes; nid = next_node[nid]) {\n    path->push_back(nid);\n    reward += node_reward[nid];\n  }\n  CHECK_EQ(reward, best_solution);\n  return best_solution;\n}\n\n/*!\n * \\brief Color the nodes in the graph into index.\n *  The coloring algorithm tries to assign node group\n *  such that node in the same group cannot run in parallel.\n *\n * \\param graph the original indexed graph.\n * \\param node_importance The importance of the node\n * \\param max_ncolor maximum number of colors allowed.\n * \\param color the color index of each of the node.\n * \\return the total number of colors.\n */\ninline uint32_t MXColorNodeGroup(const IndexedGraph& graph,\n                                 std::vector<uint32_t> node_importance,\n                                 uint32_t max_ncolor,\n                                 std::vector<uint32_t>* color) {\n  CHECK_NE(max_ncolor, 0U);\n  CHECK_EQ(graph.num_nodes(), node_importance.size());\n\n  color->clear();\n  color->resize(graph.num_nodes(), max_ncolor);\n  uint32_t cindex;\n  // greedy algorithm, every time\n  // find a path with best reward and assign a new color\n  // All the nodes in the path cannot run in parallel.\n  for (cindex = 0; cindex < max_ncolor - 1; ++cindex) {\n    std::vector<uint32_t> path;\n    uint32_t reward = MXFindBestPath(graph, node_importance, &path);\n    if (reward == 0)\n      break;\n    for (uint32_t nid : path) {\n      if (node_importance[nid] != 0) {\n        CHECK_EQ(color->at(nid), max_ncolor);\n        color->at(nid) = cindex;\n        // make the importance 0 after color is decided.\n        node_importance[nid] = 0;\n      }\n    }\n  }\n  // assign i for rest of the node\n  for (uint32_t i = 0; i < graph.num_nodes(); ++i) {\n    if (color->at(i) == max_ncolor) {\n      color->at(i) = cindex;\n    }\n  }\n  return cindex + 1;\n}\n\n}  // namespace pass\n}  // namespace nnvm\n\n#endif  // MXNET_NNVM_GRAPH_ALGORITHM_H_\n"
  }
]